3584 files changed, 222999 insertions, 54739 deletions

diff --git a/drivers/accessibility/Kconfig b/drivers/accessibility/Kconfig
index 00f7512c9cf4..f10c17dc1dee 100644
--- a/drivers/accessibility/Kconfig
+++ b/drivers/accessibility/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 menuconfig ACCESSIBILITY
 	bool "Accessibility support"
-	---help---
+	help
 	  Accessibility handles all special kinds of hardware devices or
 	  software adapters which help people with disabilities (e.g.
 	  blindness) to use computers.
@@ -21,7 +21,7 @@ config A11Y_BRAILLE_CONSOLE
 	bool "Console on braille device"
 	depends on VT
 	depends on SERIAL_CORE_CONSOLE
-	---help---
+	help
 	  Enables console output on a braille device connected to a 8250
 	  serial port. For now only the VisioBraille device is supported.
 
diff --git a/drivers/acpi/acpi_extlog.c b/drivers/acpi/acpi_extlog.c
index 8596a106a933..f138e12b7b82 100644
--- a/drivers/acpi/acpi_extlog.c
+++ b/drivers/acpi/acpi_extlog.c
@@ -42,8 +42,6 @@ struct extlog_l1_head {
 	u8  rev1[12];
 };
 
-static int old_edac_report_status;
-
 static u8 extlog_dsm_uuid[] __initdata = "663E35AF-CC10-41A4-88EA-5470AF055295";
 
 /* L1 table related physical address */
@@ -146,7 +144,7 @@ static int extlog_print(struct notifier_block *nb, unsigned long val,
 	static u32 err_seq;
 
 	estatus = extlog_elog_entry_check(cpu, bank);
-	if (estatus == NULL)
+	if (estatus == NULL || (mce->kflags & MCE_HANDLED_CEC))
 		return NOTIFY_DONE;
 
 	memcpy(elog_buf, (void *)estatus, ELOG_ENTRY_LEN);
@@ -176,7 +174,8 @@ static int extlog_print(struct notifier_block *nb, unsigned long val,
 	}
 
 out:
-	return NOTIFY_STOP;
+	mce->kflags |= MCE_HANDLED_EXTLOG;
+	return NOTIFY_OK;
 }
 
 static bool __init extlog_get_l1addr(void)
@@ -228,11 +227,6 @@ static int __init extlog_init(void)
 	if (!(cap & MCG_ELOG_P) || !extlog_get_l1addr())
 		return -ENODEV;
 
-	if (edac_get_report_status() == EDAC_REPORTING_FORCE) {
-		pr_warn("Not loading eMCA, error reporting force-enabled through EDAC.\n");
-		return -EPERM;
-	}
-
 	rc = -EINVAL;
 	/* get L1 header to fetch necessary information */
 	l1_hdr_size = sizeof(struct extlog_l1_head);
@@ -280,12 +274,6 @@ static int __init extlog_init(void)
 	if (elog_buf == NULL)
 		goto err_release_elog;
 
-	/*
-	 * eMCA event report method has higher priority than EDAC method,
-	 * unless EDAC event report method is mandatory.
-	 */
-	old_edac_report_status = edac_get_report_status();
-	edac_set_report_status(EDAC_REPORTING_DISABLED);
 	mce_register_decode_chain(&extlog_mce_dec);
 	/* enable OS to be involved to take over management from BIOS */
 	((struct extlog_l1_head *)extlog_l1_addr)->flags |= FLAG_OS_OPTIN;
@@ -307,7 +295,6 @@ err:
 
 static void __exit extlog_exit(void)
 {
-	edac_set_report_status(old_edac_report_status);
 	mce_unregister_decode_chain(&extlog_mce_dec);
 	((struct extlog_l1_head *)extlog_l1_addr)->flags &= ~FLAG_OS_OPTIN;
 	if (extlog_l1_addr)
diff --git a/drivers/acpi/acpica/dbdisply.c b/drivers/acpi/acpica/dbdisply.c
index f2df416d0d2d..d41eb9e67500 100644
--- a/drivers/acpi/acpica/dbdisply.c
+++ b/drivers/acpi/acpica/dbdisply.c
@@ -51,6 +51,8 @@ static acpi_adr_space_type acpi_gbl_space_id_list[] = {
 	ACPI_ADR_SPACE_IPMI,
 	ACPI_ADR_SPACE_GPIO,
 	ACPI_ADR_SPACE_GSBUS,
+	ACPI_ADR_SPACE_PLATFORM_COMM,
+	ACPI_ADR_SPACE_PLATFORM_RT,
 	ACPI_ADR_SPACE_DATA_TABLE,
 	ACPI_ADR_SPACE_FIXED_HARDWARE
 };
diff --git a/drivers/acpi/acpica/utdecode.c b/drivers/acpi/acpica/utdecode.c
index 177ab88d95de..ed9aedf604a1 100644
--- a/drivers/acpi/acpica/utdecode.c
+++ b/drivers/acpi/acpica/utdecode.c
@@ -78,7 +78,8 @@ const char *acpi_gbl_region_types[ACPI_NUM_PREDEFINED_REGIONS] = {
 	"IPMI",			/* 0x07 */
 	"GeneralPurposeIo",	/* 0x08 */
 	"GenericSerialBus",	/* 0x09 */
-	"PCC"			/* 0x0A */
+	"PCC",			/* 0x0A */
+	"PlatformRtMechanism"	/* 0x0B */
 };
 
 const char *acpi_ut_get_region_name(u8 space_id)
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index b44b12a931e7..94d91c67aeae 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -186,7 +186,7 @@ int acpi_device_set_power(struct acpi_device *device, int state)
 		 * possibly drop references to the power resources in use.
 		 */
 		state = ACPI_STATE_D3_HOT;
-		/* If _PR3 is not available, use D3hot as the target state. */
+		/* If D3cold is not supported, use D3hot as the target state. */
 		if (!device->power.states[ACPI_STATE_D3_COLD].flags.valid)
 			target_state = state;
 	} else if (!device->power.states[state].flags.valid) {
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index fa4500f9cfd1..7c138a4edc03 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -2293,7 +2293,7 @@ static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 	nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
 	if (!nd_set)
 		return -ENOMEM;
-	guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid);
+	import_guid(&nd_set->type_guid, spa->range_guid);
 
 	info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
 	if (!info)
diff --git a/drivers/acpi/nfit/mce.c b/drivers/acpi/nfit/mce.c
index f0ae48515b48..ee8d9973f60b 100644
--- a/drivers/acpi/nfit/mce.c
+++ b/drivers/acpi/nfit/mce.c
@@ -76,6 +76,7 @@ static int nfit_handle_mce(struct notifier_block *nb, unsigned long val,
 			 */
 			acpi_nfit_ars_rescan(acpi_desc, 0);
 		}
+		mce->kflags |= MCE_HANDLED_NFIT;
 		break;
 	}
 
diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c
index 47b4969d9b93..5be5a977da1b 100644
--- a/drivers/acpi/numa/srat.c
+++ b/drivers/acpi/numa/srat.c
@@ -35,6 +35,7 @@ int pxm_to_node(int pxm)
 		return NUMA_NO_NODE;
 	return pxm_to_node_map[pxm];
 }
+EXPORT_SYMBOL(pxm_to_node);
 
 int node_to_pxm(int node)
 {
diff --git a/drivers/acpi/pci_mcfg.c b/drivers/acpi/pci_mcfg.c
index 6b347d9920cc..54b36b7ad47d 100644
--- a/drivers/acpi/pci_mcfg.c
+++ b/drivers/acpi/pci_mcfg.c
@@ -29,7 +29,7 @@ struct mcfg_fixup {
 	u32 oem_revision;
 	u16 segment;
 	struct resource bus_range;
-	struct pci_ecam_ops *ops;
+	const struct pci_ecam_ops *ops;
 	struct resource cfgres;
 };
 
@@ -165,7 +165,7 @@ static int pci_mcfg_quirk_matches(struct mcfg_fixup *f, u16 segment,
 
 static void pci_mcfg_apply_quirks(struct acpi_pci_root *root,
 				  struct resource *cfgres,
-				  struct pci_ecam_ops **ecam_ops)
+				  const struct pci_ecam_ops **ecam_ops)
 {
 #ifdef CONFIG_PCI_QUIRKS
 	u16 segment = root->segment;
@@ -191,9 +191,9 @@ static void pci_mcfg_apply_quirks(struct acpi_pci_root *root,
 static LIST_HEAD(pci_mcfg_list);
 
 int pci_mcfg_lookup(struct acpi_pci_root *root, struct resource *cfgres,
-		    struct pci_ecam_ops **ecam_ops)
+		    const struct pci_ecam_ops **ecam_ops)
 {
-	struct pci_ecam_ops *ops = &pci_generic_ecam_ops;
+	const struct pci_ecam_ops *ops = &pci_generic_ecam_ops;
 	struct resource *bus_res = &root->secondary;
 	u16 seg = root->segment;
 	struct mcfg_entry *e;
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index ac8ad6cb82aa..f90e841c59f5 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -483,13 +483,8 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm,
 	if (IS_ENABLED(CONFIG_HOTPLUG_PCI_SHPC))
 		control |= OSC_PCI_SHPC_NATIVE_HP_CONTROL;
 
-	if (pci_aer_available()) {
-		if (aer_acpi_firmware_first())
-			dev_info(&device->dev,
-				 "PCIe AER handled by firmware\n");
-		else
-			control |= OSC_PCI_EXPRESS_AER_CONTROL;
-	}
+	if (pci_aer_available())
+		control |= OSC_PCI_EXPRESS_AER_CONTROL;
 
 	/*
 	 * Per the Downstream Port Containment Related Enhancements ECN to
@@ -938,7 +933,7 @@ struct pci_bus *acpi_pci_root_create(struct acpi_pci_root *root,
 	 * assignments made by firmware for this host bridge.
 	 */
 	obj = acpi_evaluate_dsm(ACPI_HANDLE(bus->bridge), &pci_acpi_dsm_guid, 1,
-	                        IGNORE_PCI_BOOT_CONFIG_DSM, NULL);
+				DSM_PCI_PRESERVE_BOOT_CONFIG, NULL);
 	if (obj && obj->type == ACPI_TYPE_INTEGER && obj->integer.value == 0)
 		host_bridge->preserve_config = 1;
 	ACPI_FREE(obj);
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 5287ab98b8c1..8777faced51a 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -15,8 +15,7 @@
 #include <linux/nls.h>
 #include <linux/dma-mapping.h>
 #include <linux/platform_data/x86/apple.h>
-
-#include <asm/pgtable.h>
+#include <linux/pgtable.h>
 
 #include "internal.h"
 
@@ -919,12 +918,9 @@ static void acpi_bus_init_power_state(struct acpi_device *device, int state)
 
 		if (buffer.length && package
 		    && package->type == ACPI_TYPE_PACKAGE
-		    && package->package.count) {
-			int err = acpi_extract_power_resources(package, 0,
-							       &ps->resources);
-			if (!err)
-				device->power.flags.power_resources = 1;
-		}
+		    && package->package.count)
+			acpi_extract_power_resources(package, 0, &ps->resources);
+
 		ACPI_FREE(buffer.pointer);
 	}
 
@@ -971,14 +967,27 @@ static void acpi_bus_get_power_flags(struct acpi_device *device)
 		acpi_bus_init_power_state(device, i);
 
 	INIT_LIST_HEAD(&device->power.states[ACPI_STATE_D3_COLD].resources);
-	if (!list_empty(&device->power.states[ACPI_STATE_D3_HOT].resources))
-		device->power.states[ACPI_STATE_D3_COLD].flags.valid = 1;
 
-	/* Set defaults for D0 and D3hot states (always valid) */
+	/* Set the defaults for D0 and D3hot (always supported). */
 	device->power.states[ACPI_STATE_D0].flags.valid = 1;
 	device->power.states[ACPI_STATE_D0].power = 100;
 	device->power.states[ACPI_STATE_D3_HOT].flags.valid = 1;
 
+	/*
+	 * Use power resources only if the D0 list of them is populated, because
+	 * some platforms may provide _PR3 only to indicate D3cold support and
+	 * in those cases the power resources list returned by it may be bogus.
+	 */
+	if (!list_empty(&device->power.states[ACPI_STATE_D0].resources)) {
+		device->power.flags.power_resources = 1;
+		/*
+		 * D3cold is supported if the D3hot list of power resources is
+		 * not empty.
+		 */
+		if (!list_empty(&device->power.states[ACPI_STATE_D3_HOT].resources))
+			device->power.states[ACPI_STATE_D3_COLD].flags.valid = 1;
+	}
+
 	if (acpi_bus_init_power(device))
 		device->flags.power_manageable = 0;
 }
diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c
index 8558b629880b..ecc304149067 100644
--- a/drivers/amba/bus.c
+++ b/drivers/amba/bus.c
@@ -505,7 +505,7 @@ static DECLARE_DELAYED_WORK(deferred_retry_work, amba_deferred_retry_func);
 
 #define DEFERRED_DEVICE_TIMEOUT (msecs_to_jiffies(5 * 1000))
 
-static void amba_deferred_retry_func(struct work_struct *dummy)
+static int amba_deferred_retry(void)
 {
 	struct deferred_device *ddev, *tmp;
 
@@ -521,11 +521,19 @@ static void amba_deferred_retry_func(struct work_struct *dummy)
 		kfree(ddev);
 	}
 
+	mutex_unlock(&deferred_devices_lock);
+
+	return 0;
+}
+late_initcall(amba_deferred_retry);
+
+static void amba_deferred_retry_func(struct work_struct *dummy)
+{
+	amba_deferred_retry();
+
 	if (!list_empty(&deferred_devices))
 		schedule_delayed_work(&deferred_retry_work,
 				      DEFERRED_DEVICE_TIMEOUT);
-
-	mutex_unlock(&deferred_devices_lock);
 }
 
 /**
diff --git a/drivers/amba/tegra-ahb.c b/drivers/amba/tegra-ahb.c
index 57d3b2e2d007..0b2c20fddb7c 100644
--- a/drivers/amba/tegra-ahb.c
+++ b/drivers/amba/tegra-ahb.c
@@ -120,7 +120,7 @@ static const u32 tegra_ahb_gizmo[] = {
 struct tegra_ahb {
 	void __iomem	*regs;
 	struct device	*dev;
-	u32		ctx[0];
+	u32		ctx[];
 };
 
 static inline u32 gizmo_readl(struct tegra_ahb *ahb, u32 offset)
diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
index 6fdf2abe4598..53b22e26266c 100644
--- a/drivers/android/Kconfig
+++ b/drivers/android/Kconfig
@@ -3,7 +3,7 @@ menu "Android"
 
 config ANDROID
 	bool "Android Drivers"
-	---help---
+	help
 	  Enable support for various drivers needed on the Android platform
 
 if ANDROID
@@ -12,7 +12,7 @@ config ANDROID_BINDER_IPC
 	bool "Android Binder IPC Driver"
 	depends on MMU
 	default n
-	---help---
+	help
 	  Binder is used in Android for both communication between processes,
 	  and remote method invocation.
 
@@ -24,7 +24,7 @@ config ANDROID_BINDERFS
 	bool "Android Binderfs filesystem"
 	depends on ANDROID_BINDER_IPC
 	default n
-	---help---
+	help
 	  Binderfs is a pseudo-filesystem for the Android Binder IPC driver
 	  which can be mounted per-ipc namespace allowing to run multiple
 	  instances of Android.
@@ -36,7 +36,7 @@ config ANDROID_BINDER_DEVICES
 	string "Android Binder devices"
 	depends on ANDROID_BINDER_IPC
 	default "binder,hwbinder,vndbinder"
-	---help---
+	help
 	  Default value for the binder.devices parameter.
 
 	  The binder.devices parameter is a comma-separated list of strings
@@ -47,7 +47,7 @@ config ANDROID_BINDER_DEVICES
 config ANDROID_BINDER_IPC_SELFTEST
 	bool "Android Binder IPC Driver Selftest"
 	depends on ANDROID_BINDER_IPC
-	---help---
+	help
 	  This feature allows binder selftest to run.
 
 	  Binder selftest checks the allocation and free of binder buffers
diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
index 2d8b9b91dee0..42c672f1584e 100644
--- a/drivers/android/binder_alloc.c
+++ b/drivers/android/binder_alloc.c
@@ -212,7 +212,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
 		mm = alloc->vma_vm_mm;
 
 	if (mm) {
-		down_read(&mm->mmap_sem);
+		mmap_read_lock(mm);
 		vma = alloc->vma;
 	}
 
@@ -270,7 +270,7 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
 		trace_binder_alloc_page_end(alloc, index);
 	}
 	if (mm) {
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 		mmput(mm);
 	}
 	return 0;
@@ -303,7 +303,7 @@ err_page_ptr_cleared:
 	}
 err_no_vma:
 	if (mm) {
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 		mmput(mm);
 	}
 	return vma ? -ENOMEM : -ESRCH;
@@ -932,8 +932,8 @@ enum lru_status binder_alloc_free_page(struct list_head *item,
 	mm = alloc->vma_vm_mm;
 	if (!mmget_not_zero(mm))
 		goto err_mmget;
-	if (!down_read_trylock(&mm->mmap_sem))
-		goto err_down_read_mmap_sem_failed;
+	if (!mmap_read_trylock(mm))
+		goto err_mmap_read_lock_failed;
 	vma = binder_alloc_get_vma(alloc);
 
 	list_lru_isolate(lru, item);
@@ -946,7 +946,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item,
 
 		trace_binder_unmap_user_end(alloc, index);
 	}
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	mmput(mm);
 
 	trace_binder_unmap_kernel_start(alloc, index);
@@ -960,7 +960,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item,
 	mutex_unlock(&alloc->mutex);
 	return LRU_REMOVED_RETRY;
 
-err_down_read_mmap_sem_failed:
+err_mmap_read_lock_failed:
 	mmput_async(mm);
 err_mmget:
 err_page_already_freed:
diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c
index 9ecad74183a3..7cf566aafe1f 100644
--- a/drivers/android/binderfs.c
+++ b/drivers/android/binderfs.c
@@ -650,7 +650,7 @@ static int binderfs_fill_super(struct super_block *sb, struct fs_context *fc)
 	struct binderfs_info *info;
 	struct binderfs_mount_opts *ctx = fc->fs_private;
 	struct inode *inode = NULL;
-	struct binderfs_device device_info = { 0 };
+	struct binderfs_device device_info = {};
 	const char *name;
 	size_t len;
 
@@ -747,7 +747,7 @@ static const struct fs_context_operations binderfs_fs_context_ops = {
 
 static int binderfs_init_fs_context(struct fs_context *fc)
 {
-	struct binderfs_mount_opts *ctx = fc->fs_private;
+	struct binderfs_mount_opts *ctx;
 
 	ctx = kzalloc(sizeof(struct binderfs_mount_opts), GFP_KERNEL);
 	if (!ctx)
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index 05ecdce1b702..030cb32da980 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -17,7 +17,7 @@ menuconfig ATA
 	depends on BLOCK
 	select SCSI
 	select GLOB
-	---help---
+	help
 	  If you want to use an ATA hard disk, ATA tape drive, ATA CD-ROM or
 	  any other ATA device under Linux, say Y and make sure that you know
 	  the name of your ATA host adapter (the card inside your computer
diff --git a/drivers/atm/Kconfig b/drivers/atm/Kconfig
index cfb0d16b60ad..8007e051876c 100644
--- a/drivers/atm/Kconfig
+++ b/drivers/atm/Kconfig
@@ -7,7 +7,7 @@ menuconfig ATM_DRIVERS
 	bool "ATM drivers"
 	depends on NETDEVICES && ATM
 	default y
-	---help---
+	help
 	  Say Y here to get to see options for Asynchronous Transfer Mode
 	  device drivers. This option alone does not add any kernel code.
 
@@ -40,7 +40,7 @@ config ATM_LANAI
 config ATM_ENI
 	tristate "Efficient Networks ENI155P"
 	depends on PCI
-	---help---
+	help
 	  Driver for the Efficient Networks ENI155p series and SMC ATM
 	  Power155 155 Mbps ATM adapters. Both, the versions with 512KB and
 	  2MB on-board RAM (Efficient calls them "C" and "S", respectively),
@@ -64,7 +64,7 @@ config ATM_ENI_DEBUG
 config ATM_ENI_TUNE_BURST
 	bool "Fine-tune burst settings"
 	depends on ATM_ENI
-	---help---
+	help
 	  In order to obtain good throughput, the ENI NIC can transfer
 	  multiple words of data per PCI bus access cycle. Such a multi-word
 	  transfer is called a burst.
@@ -256,7 +256,7 @@ config ATM_AMBASSADOR
 config ATM_AMBASSADOR_DEBUG
 	bool "Enable debugging messages"
 	depends on ATM_AMBASSADOR
-	---help---
+	help
 	  Somewhat useful debugging messages are available. The choice of
 	  messages is controlled by a bitmap.  This may be specified as a
 	  module argument (kernel command line argument as well?), changed
@@ -280,7 +280,7 @@ config ATM_HORIZON
 config ATM_HORIZON_DEBUG
 	bool "Enable debugging messages"
 	depends on ATM_HORIZON
-	---help---
+	help
 	  Somewhat useful debugging messages are available. The choice of
 	  messages is controlled by a bitmap.  This may be specified as a
 	  module argument (kernel command line argument as well?), changed
@@ -296,7 +296,7 @@ config ATM_HORIZON_DEBUG
 config ATM_IA
 	tristate "Interphase ATM PCI x575/x525/x531"
 	depends on PCI
-	---help---
+	help
 	  This is a driver for the Interphase (i)ChipSAR adapter cards
 	  which include a variety of variants in term of the size of the
 	  control memory (128K-1KVC, 512K-4KVC), the size of the packet
@@ -312,7 +312,7 @@ config ATM_IA
 config ATM_IA_DEBUG
 	bool "Enable debugging messages"
 	depends on ATM_IA
-	---help---
+	help
 	  Somewhat useful debugging messages are available. The choice of
 	  messages is controlled by a bitmap. This may be specified as a
 	  module argument (kernel command line argument as well?), changed
@@ -330,7 +330,7 @@ config ATM_FORE200E
 	tristate "FORE Systems 200E-series"
 	depends on (PCI || SBUS)
 	select FW_LOADER
-	---help---
+	help
 	  This is a driver for the FORE Systems 200E-series ATM adapter
 	  cards. It simultaneously supports PCA-200E and SBA-200E models
 	  on PCI and SBUS hosts. Say Y (or M to compile as a module
@@ -352,7 +352,7 @@ config ATM_FORE200E_TX_RETRY
 	int "Maximum number of tx retries"
 	depends on ATM_FORE200E
 	default "16"
-	---help---
+	help
 	  Specifies the number of times the driver attempts to transmit
 	  a message before giving up, if the transmit queue of the ATM card
 	  is transiently saturated.
diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c
index 8fbd36eb8941..f4ad7ce25ae8 100644
--- a/drivers/atm/fore200e.c
+++ b/drivers/atm/fore200e.c
@@ -25,6 +25,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/delay.h>
 #include <linux/firmware.h>
+#include <linux/pgtable.h>
 #include <asm/io.h>
 #include <asm/string.h>
 #include <asm/page.h>
@@ -40,7 +41,6 @@
 #include <asm/idprom.h>
 #include <asm/openprom.h>
 #include <asm/oplib.h>
-#include <asm/pgtable.h>
 #endif
 
 #if defined(CONFIG_ATM_FORE200E_USE_TASKLET) /* defer interrupt work to a tasklet */
diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig
index 48efa7a047f3..81757eeded68 100644
--- a/drivers/auxdisplay/Kconfig
+++ b/drivers/auxdisplay/Kconfig
@@ -8,7 +8,7 @@
 
 menuconfig AUXDISPLAY
 	bool "Auxiliary Display support"
-	---help---
+	help
 	  Say Y here to get to see options for auxiliary display drivers.
 	  This option alone does not add any kernel code.
 
@@ -20,7 +20,7 @@ config HD44780
 	tristate "HD44780 Character LCD support"
 	depends on GPIOLIB || COMPILE_TEST
 	select CHARLCD
-	---help---
+	help
 	  Enable support for Character LCDs using a HD44780 controller.
 	  The LCD is accessible through the /dev/lcd char device (10, 156).
 	  This code can either be compiled as a module, or linked into the
@@ -31,7 +31,7 @@ config KS0108
 	tristate "KS0108 LCD Controller"
 	depends on PARPORT_PC
 	default n
-	---help---
+	help
 	  If you have a LCD controlled by one or more KS0108
 	  controllers, say Y. You will need also another more specific
 	  driver for your LCD.
@@ -49,7 +49,7 @@ config KS0108_PORT
 	hex "Parallel port where the LCD is connected"
 	depends on KS0108
 	default 0x378
-	---help---
+	help
 	  The address of the parallel port where the LCD is connected.
 
 	  The first  standard parallel port address is 0x378.
@@ -71,7 +71,7 @@ config KS0108_DELAY
 	int "Delay between each control writing (microseconds)"
 	depends on KS0108
 	default "2"
-	---help---
+	help
 	  Amount of time the ks0108 should wait between each control write
 	  to the parallel port.
 
@@ -92,7 +92,7 @@ config CFAG12864B
 	select FB_SYS_IMAGEBLIT
 	select FB_SYS_FOPS
 	default n
-	---help---
+	help
 	  If you have a Crystalfontz 128x64 2-color LCD, cfag12864b Series,
 	  say Y. You also need the ks0108 LCD Controller driver.
 
@@ -114,7 +114,7 @@ config CFAG12864B_RATE
 	int "Refresh rate (hertz)"
 	depends on CFAG12864B
 	default "20"
-	---help---
+	help
 	  Refresh rate of the LCD.
 
 	  As the LCD is not memory mapped, the driver has to make the work by
@@ -168,7 +168,7 @@ menuconfig PARPORT_PANEL
 	tristate "Parallel port LCD/Keypad Panel support"
 	depends on PARPORT
 	select CHARLCD
-	---help---
+	help
 	  Say Y here if you have an HD44780 or KS-0074 LCD connected to your
 	  parallel port. This driver also features 4 and 6-key keypads. The LCD
 	  is accessible through the /dev/lcd char device (10, 156), and the
@@ -182,7 +182,7 @@ config PANEL_PARPORT
 	int "Default parallel port number (0=LPT1)"
 	range 0 255
 	default "0"
-	---help---
+	help
 	  This is the index of the parallel port the panel is connected to. One
 	  driver instance only supports one parallel port, so if your keypad
 	  and LCD are connected to two separate ports, you have to start two
@@ -193,7 +193,7 @@ config PANEL_PROFILE
 	int "Default panel profile (0-5, 0=custom)"
 	range 0 5
 	default "5"
-	---help---
+	help
 	  To ease configuration, the driver supports different configuration
 	  profiles for past and recent wirings. These profiles can also be
 	  used to define an approximative configuration, completed by a few
@@ -215,7 +215,7 @@ config PANEL_KEYPAD
 	int "Keypad type (0=none, 1=old 6 keys, 2=new 6 keys, 3=Nexcom 4 keys)"
 	range 0 3
 	default 0
-	---help---
+	help
 	  This enables and configures a keypad connected to the parallel port.
 	  The keys will be read from character device 10,185. Valid values are :
 
@@ -232,7 +232,7 @@ config PANEL_LCD
 	int "LCD type (0=none, 1=custom, 2=old //, 3=ks0074, 4=hantronix, 5=Nexcom)"
 	range 0 5
 	default 0
-	---help---
+	help
 	   This enables and configures an LCD connected to the parallel port.
 	   The driver includes an interpreter for escape codes starting with
 	   '\e[L' which are specific to the LCD, and a few ANSI codes. The
@@ -255,7 +255,7 @@ config PANEL_LCD_HEIGHT
 	int "Number of lines on the LCD (1-2)"
 	range 1 2
 	default 2
-	---help---
+	help
 	  This is the number of visible character lines on the LCD in custom profile.
 	  It can either be 1 or 2.
 
@@ -264,7 +264,7 @@ config PANEL_LCD_WIDTH
 	int "Number of characters per line on the LCD (1-40)"
 	range 1 40
 	default 40
-	---help---
+	help
 	  This is the number of characters per line on the LCD in custom profile.
 	  Common values are 16,20,24,40.
 
@@ -273,7 +273,7 @@ config PANEL_LCD_BWIDTH
 	int "Internal LCD line width (1-40, 40 by default)"
 	range 1 40
 	default 40
-	---help---
+	help
 	  Most LCDs use a standard controller which supports hardware lines of 40
 	  characters, although sometimes only 16, 20 or 24 of them are really wired
 	  to the terminal. This results in some non-visible but addressable characters,
@@ -289,7 +289,7 @@ config PANEL_LCD_HWIDTH
 	int "Hardware LCD line width (1-64, 64 by default)"
 	range 1 64
 	default 64
-	---help---
+	help
 	  Most LCDs use a single address bit to differentiate line 0 and line 1. Since
 	  some of them need to be able to address 40 chars with the lower bits, they
 	  often use the immediately superior power of 2, which is 64, to address the
@@ -303,7 +303,7 @@ config PANEL_LCD_CHARSET
 	int "LCD character set (0=normal, 1=KS0074)"
 	range 0 1
 	default 0
-	---help---
+	help
 	  Some controllers such as the KS0074 use a somewhat strange character set
 	  where many symbols are at unusual places. The driver knows how to map
 	  'standard' ASCII characters to the character sets used by these controllers.
@@ -319,7 +319,7 @@ config PANEL_LCD_PROTO
 	int "LCD communication mode (0=parallel 8 bits, 1=serial)"
 	range 0 1
 	default 0
-	---help---
+	help
 	  This driver now supports any serial or parallel LCD wired to a parallel
 	  port. But before assigning signals, the driver needs to know if it will
 	  be driving a serial LCD or a parallel one. Serial LCDs only use 2 wires
@@ -332,7 +332,7 @@ config PANEL_LCD_PIN_E
 	int "Parallel port pin number & polarity connected to the LCD E signal (-17...17) "
 	range -17 17
 	default 14
-	---help---
+	help
 	  This describes the number of the parallel port pin to which the LCD 'E'
 	  signal has been connected. It can be :
 
@@ -347,7 +347,7 @@ config PANEL_LCD_PIN_RS
 	int "Parallel port pin number & polarity connected to the LCD RS signal (-17...17) "
 	range -17 17
 	default 17
-	---help---
+	help
 	  This describes the number of the parallel port pin to which the LCD 'RS'
 	  signal has been connected. It can be :
 
@@ -362,7 +362,7 @@ config PANEL_LCD_PIN_RW
 	int "Parallel port pin number & polarity connected to the LCD RW signal (-17...17) "
 	range -17 17
 	default 16
-	---help---
+	help
 	  This describes the number of the parallel port pin to which the LCD 'RW'
 	  signal has been connected. It can be :
 
@@ -377,7 +377,7 @@ config PANEL_LCD_PIN_SCL
 	int "Parallel port pin number & polarity connected to the LCD SCL signal (-17...17) "
 	range -17 17
 	default 1
-	---help---
+	help
 	  This describes the number of the parallel port pin to which the serial
 	  LCD 'SCL' signal has been connected. It can be :
 
@@ -392,7 +392,7 @@ config PANEL_LCD_PIN_SDA
 	int "Parallel port pin number & polarity connected to the LCD SDA signal (-17...17) "
 	range -17 17
 	default 2
-	---help---
+	help
 	  This describes the number of the parallel port pin to which the serial
 	  LCD 'SDA' signal has been connected. It can be :
 
@@ -407,7 +407,7 @@ config PANEL_LCD_PIN_BL
 	int "Parallel port pin number & polarity connected to the LCD backlight signal (-17...17) "
 	range -17 17
 	default 0
-	---help---
+	help
 	  This describes the number of the parallel port pin to which the LCD 'BL' signal
 	  has been connected. It can be :
 
@@ -423,7 +423,7 @@ config PANEL_CHANGE_MESSAGE
 	bool "Change LCD initialization message ?"
 	depends on CHARLCD
 	default "n"
-	---help---
+	help
 	  This allows you to replace the boot message indicating the kernel version
 	  and the driver version with a custom message. This is useful on appliances
 	  where a simple 'Starting system' message can be enough to stop a customer
@@ -436,7 +436,7 @@ config PANEL_BOOT_MESSAGE
 	depends on PANEL_CHANGE_MESSAGE="y"
 	string "New initialization message"
 	default ""
-	---help---
+	help
 	  This allows you to replace the boot message indicating the kernel version
 	  and the driver version with a custom message. This is useful on appliances
 	  where a simple 'Starting system' message can be enough to stop a customer
@@ -448,7 +448,7 @@ config PANEL_BOOT_MESSAGE
 choice
 	prompt "Backlight initial state"
 	default CHARLCD_BL_FLASH
-	---help---
+	help
 	  Select the initial backlight state on boot or module load.
 
 	  Previously, there was no option for this: the backlight flashed
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index 5f0bc74d2409..8d7001712062 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -149,8 +149,9 @@ config DEBUG_TEST_DRIVER_REMOVE
 	  test this functionality.
 
 config PM_QOS_KUNIT_TEST
-	bool "KUnit Test for PM QoS features"
+	bool "KUnit Test for PM QoS features" if !KUNIT_ALL_TESTS
 	depends on KUNIT=y
+	default KUNIT_ALL_TESTS
 
 config HMEM_REPORTING
 	bool
diff --git a/drivers/base/base.h b/drivers/base/base.h
index 40fb069a8a7e..95c22c0f9036 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -153,6 +153,7 @@ extern char *make_class_name(const char *name, struct kobject *kobj);
 extern int devres_release_all(struct device *dev);
 extern void device_block_probing(void);
 extern void device_unblock_probing(void);
+extern void driver_deferred_probe_force_trigger(void);
 
 /* /sys/devices directory */
 extern struct kset *devices_kset;
diff --git a/drivers/base/core.c b/drivers/base/core.c
index de808c5a187b..67d39a90b45c 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -49,6 +49,9 @@ static LIST_HEAD(wait_for_suppliers);
 static DEFINE_MUTEX(wfs_lock);
 static LIST_HEAD(deferred_sync);
 static unsigned int defer_sync_state_count = 1;
+static unsigned int defer_fw_devlink_count;
+static DEFINE_MUTEX(defer_fw_devlink_lock);
+static bool fw_devlink_is_permissive(void);
 
 #ifdef CONFIG_SRCU
 static DEFINE_MUTEX(device_links_lock);
@@ -529,7 +532,7 @@ static void device_link_add_missing_supplier_links(void)
 		int ret = fwnode_call_int_op(dev->fwnode, add_links, dev);
 		if (!ret)
 			list_del_init(&dev->links.needs_suppliers);
-		else if (ret != -ENODEV)
+		else if (ret != -ENODEV || fw_devlink_is_permissive())
 			dev->links.need_for_probe = false;
 	}
 	mutex_unlock(&wfs_lock);
@@ -643,9 +646,17 @@ static void device_links_missing_supplier(struct device *dev)
 {
 	struct device_link *link;
 
-	list_for_each_entry(link, &dev->links.suppliers, c_node)
-		if (link->status == DL_STATE_CONSUMER_PROBE)
+	list_for_each_entry(link, &dev->links.suppliers, c_node) {
+		if (link->status != DL_STATE_CONSUMER_PROBE)
+			continue;
+
+		if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) {
 			WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
+		} else {
+			WARN_ON(!(link->flags & DL_FLAG_SYNC_STATE_ONLY));
+			WRITE_ONCE(link->status, DL_STATE_DORMANT);
+		}
+	}
 }
 
 /**
@@ -684,11 +695,11 @@ int device_links_check_suppliers(struct device *dev)
 	device_links_write_lock();
 
 	list_for_each_entry(link, &dev->links.suppliers, c_node) {
-		if (!(link->flags & DL_FLAG_MANAGED) ||
-		    link->flags & DL_FLAG_SYNC_STATE_ONLY)
+		if (!(link->flags & DL_FLAG_MANAGED))
 			continue;
 
-		if (link->status != DL_STATE_AVAILABLE) {
+		if (link->status != DL_STATE_AVAILABLE &&
+		    !(link->flags & DL_FLAG_SYNC_STATE_ONLY)) {
 			device_links_missing_supplier(dev);
 			ret = -EPROBE_DEFER;
 			break;
@@ -949,11 +960,21 @@ static void __device_links_no_driver(struct device *dev)
 		if (!(link->flags & DL_FLAG_MANAGED))
 			continue;
 
-		if (link->flags & DL_FLAG_AUTOREMOVE_CONSUMER)
+		if (link->flags & DL_FLAG_AUTOREMOVE_CONSUMER) {
 			device_link_drop_managed(link);
-		else if (link->status == DL_STATE_CONSUMER_PROBE ||
-			 link->status == DL_STATE_ACTIVE)
+			continue;
+		}
+
+		if (link->status != DL_STATE_CONSUMER_PROBE &&
+		    link->status != DL_STATE_ACTIVE)
+			continue;
+
+		if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) {
 			WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
+		} else {
+			WARN_ON(!(link->flags & DL_FLAG_SYNC_STATE_ONLY));
+			WRITE_ONCE(link->status, DL_STATE_DORMANT);
+		}
 	}
 
 	dev->links.status = DL_DEV_NO_DRIVER;
@@ -1162,6 +1183,150 @@ static void device_links_purge(struct device *dev)
 	device_links_write_unlock();
 }
 
+static u32 fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY;
+static int __init fw_devlink_setup(char *arg)
+{
+	if (!arg)
+		return -EINVAL;
+
+	if (strcmp(arg, "off") == 0) {
+		fw_devlink_flags = 0;
+	} else if (strcmp(arg, "permissive") == 0) {
+		fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY;
+	} else if (strcmp(arg, "on") == 0) {
+		fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER;
+	} else if (strcmp(arg, "rpm") == 0) {
+		fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER |
+				   DL_FLAG_PM_RUNTIME;
+	}
+	return 0;
+}
+early_param("fw_devlink", fw_devlink_setup);
+
+u32 fw_devlink_get_flags(void)
+{
+	return fw_devlink_flags;
+}
+
+static bool fw_devlink_is_permissive(void)
+{
+	return fw_devlink_flags == DL_FLAG_SYNC_STATE_ONLY;
+}
+
+static void fw_devlink_link_device(struct device *dev)
+{
+	int fw_ret;
+
+	if (!fw_devlink_flags)
+		return;
+
+	mutex_lock(&defer_fw_devlink_lock);
+	if (!defer_fw_devlink_count)
+		device_link_add_missing_supplier_links();
+
+	/*
+	 * The device's fwnode not having add_links() doesn't affect if other
+	 * consumers can find this device as a supplier.  So, this check is
+	 * intentionally placed after device_link_add_missing_supplier_links().
+	 */
+	if (!fwnode_has_op(dev->fwnode, add_links))
+		goto out;
+
+	/*
+	 * If fw_devlink is being deferred, assume all devices have mandatory
+	 * suppliers they need to link to later. Then, when the fw_devlink is
+	 * resumed, all these devices will get a chance to try and link to any
+	 * suppliers they have.
+	 */
+	if (!defer_fw_devlink_count) {
+		fw_ret = fwnode_call_int_op(dev->fwnode, add_links, dev);
+		if (fw_ret == -ENODEV && fw_devlink_is_permissive())
+			fw_ret = -EAGAIN;
+	} else {
+		fw_ret = -ENODEV;
+	}
+
+	if (fw_ret == -ENODEV)
+		device_link_wait_for_mandatory_supplier(dev);
+	else if (fw_ret)
+		device_link_wait_for_optional_supplier(dev);
+
+out:
+	mutex_unlock(&defer_fw_devlink_lock);
+}
+
+/**
+ * fw_devlink_pause - Pause parsing of fwnode to create device links
+ *
+ * Calling this function defers any fwnode parsing to create device links until
+ * fw_devlink_resume() is called. Both these functions are ref counted and the
+ * caller needs to match the calls.
+ *
+ * While fw_devlink is paused:
+ * - Any device that is added won't have its fwnode parsed to create device
+ *   links.
+ * - The probe of the device will also be deferred during this period.
+ * - Any devices that were already added, but waiting for suppliers won't be
+ *   able to link to newly added devices.
+ *
+ * Once fw_devlink_resume():
+ * - All the fwnodes that was not parsed will be parsed.
+ * - All the devices that were deferred probing will be reattempted if they
+ *   aren't waiting for any more suppliers.
+ *
+ * This pair of functions, is mainly meant to optimize the parsing of fwnodes
+ * when a lot of devices that need to link to each other are added in a short
+ * interval of time. For example, adding all the top level devices in a system.
+ *
+ * For example, if N devices are added and:
+ * - All the consumers are added before their suppliers
+ * - All the suppliers of the N devices are part of the N devices
+ *
+ * Then:
+ *
+ * - With the use of fw_devlink_pause() and fw_devlink_resume(), each device
+ *   will only need one parsing of its fwnode because it is guaranteed to find
+ *   all the supplier devices already registered and ready to link to. It won't
+ *   have to do another pass later to find one or more suppliers it couldn't
+ *   find in the first parse of the fwnode. So, we'll only need O(N) fwnode
+ *   parses.
+ *
+ * - Without the use of fw_devlink_pause() and fw_devlink_resume(), we would
+ *   end up doing O(N^2) parses of fwnodes because every device that's added is
+ *   guaranteed to trigger a parse of the fwnode of every device added before
+ *   it. This O(N^2) parse is made worse by the fact that when a fwnode of a
+ *   device is parsed, all it descendant devices might need to have their
+ *   fwnodes parsed too (even if the devices themselves aren't added).
+ */
+void fw_devlink_pause(void)
+{
+	mutex_lock(&defer_fw_devlink_lock);
+	defer_fw_devlink_count++;
+	mutex_unlock(&defer_fw_devlink_lock);
+}
+
+/** fw_devlink_resume - Resume parsing of fwnode to create device links
+ *
+ * This function is used in conjunction with fw_devlink_pause() and is ref
+ * counted. See documentation for fw_devlink_pause() for more details.
+ */
+void fw_devlink_resume(void)
+{
+	mutex_lock(&defer_fw_devlink_lock);
+	if (!defer_fw_devlink_count) {
+		WARN(true, "Unmatched fw_devlink pause/resume!");
+		goto out;
+	}
+
+	defer_fw_devlink_count--;
+	if (defer_fw_devlink_count)
+		goto out;
+
+	device_link_add_missing_supplier_links();
+	driver_deferred_probe_force_trigger();
+out:
+	mutex_unlock(&defer_fw_devlink_lock);
+}
 /* Device links support end. */
 
 int (*platform_notify)(struct device *dev) = NULL;
@@ -2364,36 +2529,6 @@ static int device_private_init(struct device *dev)
 	return 0;
 }
 
-static u32 fw_devlink_flags;
-static int __init fw_devlink_setup(char *arg)
-{
-	if (!arg)
-		return -EINVAL;
-
-	if (strcmp(arg, "off") == 0) {
-		fw_devlink_flags = 0;
-	} else if (strcmp(arg, "permissive") == 0) {
-		fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY;
-	} else if (strcmp(arg, "on") == 0) {
-		fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER;
-	} else if (strcmp(arg, "rpm") == 0) {
-		fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER |
-				   DL_FLAG_PM_RUNTIME;
-	}
-	return 0;
-}
-early_param("fw_devlink", fw_devlink_setup);
-
-u32 fw_devlink_get_flags(void)
-{
-	return fw_devlink_flags;
-}
-
-static bool fw_devlink_is_permissive(void)
-{
-	return fw_devlink_flags == DL_FLAG_SYNC_STATE_ONLY;
-}
-
 /**
  * device_add - add device to device hierarchy.
  * @dev: device.
@@ -2426,9 +2561,8 @@ int device_add(struct device *dev)
 	struct device *parent;
 	struct kobject *kobj;
 	struct class_interface *class_intf;
-	int error = -EINVAL, fw_ret;
+	int error = -EINVAL;
 	struct kobject *glue_dir = NULL;
-	bool is_fwnode_dev = false;
 
 	dev = get_device(dev);
 	if (!dev)
@@ -2526,11 +2660,6 @@ int device_add(struct device *dev)
 
 	kobject_uevent(&dev->kobj, KOBJ_ADD);
 
-	if (dev->fwnode && !dev->fwnode->dev) {
-		dev->fwnode->dev = dev;
-		is_fwnode_dev = true;
-	}
-
 	/*
 	 * Check if any of the other devices (consumers) have been waiting for
 	 * this device (supplier) to be added so that they can create a device
@@ -2539,19 +2668,13 @@ int device_add(struct device *dev)
 	 * This needs to happen after device_pm_add() because device_link_add()
 	 * requires the supplier be registered before it's called.
 	 *
-	 * But this also needs to happe before bus_probe_device() to make sure
+	 * But this also needs to happen before bus_probe_device() to make sure
 	 * waiting consumers can link to it before the driver is bound to the
 	 * device and the driver sync_state callback is called for this device.
 	 */
-	device_link_add_missing_supplier_links();
-
-	if (fw_devlink_flags && is_fwnode_dev &&
-	    fwnode_has_op(dev->fwnode, add_links)) {
-		fw_ret = fwnode_call_int_op(dev->fwnode, add_links, dev);
-		if (fw_ret == -ENODEV && !fw_devlink_is_permissive())
-			device_link_wait_for_mandatory_supplier(dev);
-		else if (fw_ret)
-			device_link_wait_for_optional_supplier(dev);
+	if (dev->fwnode && !dev->fwnode->dev) {
+		dev->fwnode->dev = dev;
+		fw_devlink_link_device(dev);
 	}
 
 	bus_probe_device(dev);
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 9a1c00fbbaef..d2136ab9b14a 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -562,6 +562,12 @@ ssize_t __weak cpu_show_itlb_multihit(struct device *dev,
 	return sprintf(buf, "Not affected\n");
 }
 
+ssize_t __weak cpu_show_srbds(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "Not affected\n");
+}
+
 static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL);
 static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL);
 static DEVICE_ATTR(spectre_v2, 0444, cpu_show_spectre_v2, NULL);
@@ -570,6 +576,7 @@ static DEVICE_ATTR(l1tf, 0444, cpu_show_l1tf, NULL);
 static DEVICE_ATTR(mds, 0444, cpu_show_mds, NULL);
 static DEVICE_ATTR(tsx_async_abort, 0444, cpu_show_tsx_async_abort, NULL);
 static DEVICE_ATTR(itlb_multihit, 0444, cpu_show_itlb_multihit, NULL);
+static DEVICE_ATTR(srbds, 0444, cpu_show_srbds, NULL);
 
 static struct attribute *cpu_root_vulnerabilities_attrs[] = {
 	&dev_attr_meltdown.attr,
@@ -580,6 +587,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = {
 	&dev_attr_mds.attr,
 	&dev_attr_tsx_async_abort.attr,
 	&dev_attr_itlb_multihit.attr,
+	&dev_attr_srbds.attr,
 	NULL
 };
 
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 94037be7f5d7..9a1d940342ac 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -164,6 +164,11 @@ static void driver_deferred_probe_trigger(void)
 	if (!driver_deferred_probe_enable)
 		return;
 
+	driver_deferred_probe_force_trigger();
+}
+
+void driver_deferred_probe_force_trigger(void)
+{
 	/*
 	 * A successful probe means that all the devices in the pending list
 	 * should be triggered to be reprobed.  Move all the deferred devices
@@ -254,12 +259,12 @@ __setup("deferred_probe_timeout=", deferred_probe_timeout_setup);
 int driver_deferred_probe_check_state(struct device *dev)
 {
 	if (!IS_ENABLED(CONFIG_MODULES) && initcalls_done) {
-		dev_warn(dev, "ignoring dependency for device, assuming no driver");
+		dev_warn(dev, "ignoring dependency for device, assuming no driver\n");
 		return -ENODEV;
 	}
 
 	if (!driver_deferred_probe_timeout && initcalls_done) {
-		dev_warn(dev, "deferred probe timeout, ignoring dependency");
+		dev_warn(dev, "deferred probe timeout, ignoring dependency\n");
 		return -ETIMEDOUT;
 	}
 
@@ -275,7 +280,7 @@ static void deferred_probe_timeout_work_func(struct work_struct *work)
 	flush_work(&deferred_probe_work);
 
 	list_for_each_entry_safe(private, p, &deferred_probe_pending_list, deferred_probe)
-		dev_info(private->device, "deferred probe pending");
+		dev_info(private->device, "deferred probe pending\n");
 	wake_up(&probe_timeout_waitqueue);
 }
 static DECLARE_DELAYED_WORK(deferred_probe_timeout_work, deferred_probe_timeout_work_func);
@@ -336,7 +341,7 @@ bool device_is_bound(struct device *dev)
 static void driver_bound(struct device *dev)
 {
 	if (device_is_bound(dev)) {
-		printk(KERN_WARNING "%s: device %s already bound\n",
+		pr_warn("%s: device %s already bound\n",
 			__func__, kobject_name(&dev->kobj));
 		return;
 	}
@@ -505,8 +510,8 @@ re_probe:
 	}
 
 	if (driver_sysfs_add(dev)) {
-		printk(KERN_ERR "%s: driver_sysfs_add(%s) failed\n",
-			__func__, dev_name(dev));
+		pr_err("%s: driver_sysfs_add(%s) failed\n",
+		       __func__, dev_name(dev));
 		goto probe_failed;
 	}
 
@@ -597,9 +602,8 @@ pinctrl_bind_failed:
 		break;
 	default:
 		/* driver matched but the probe failed */
-		printk(KERN_WARNING
-		       "%s: probe of %s failed with error %d\n",
-		       drv->name, dev_name(dev), ret);
+		pr_warn("%s: probe of %s failed with error %d\n",
+			drv->name, dev_name(dev), ret);
 	}
 	/*
 	 * Ignore errors returned by ->probe so that the next driver can try
@@ -624,8 +628,8 @@ static int really_probe_debug(struct device *dev, struct device_driver *drv)
 	ret = really_probe(dev, drv);
 	rettime = ktime_get();
 	delta = ktime_sub(rettime, calltime);
-	printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n",
-	       dev_name(dev), ret, (s64) ktime_to_us(delta));
+	pr_debug("probe of %s returned %d after %lld usecs\n",
+		 dev_name(dev), ret, (s64) ktime_to_us(delta));
 	return ret;
 }
 
@@ -713,8 +717,7 @@ static inline bool cmdline_requested_async_probing(const char *drv_name)
 static int __init save_async_options(char *buf)
 {
 	if (strlen(buf) >= ASYNC_DRV_NAMES_MAX_LEN)
-		printk(KERN_WARNING
-			"Too long list of driver names for 'driver_async_probe'!\n");
+		pr_warn("Too long list of driver names for 'driver_async_probe'!\n");
 
 	strlcpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN);
 	return 0;
@@ -789,7 +792,7 @@ static int __device_attach_driver(struct device_driver *drv, void *_data)
 		dev_dbg(dev, "Device match requests probe deferral\n");
 		driver_deferred_probe_add(dev);
 	} else if (ret < 0) {
-		dev_dbg(dev, "Bus failed to match device: %d", ret);
+		dev_dbg(dev, "Bus failed to match device: %d\n", ret);
 		return ret;
 	} /* ret > 0 means positive match */
 
@@ -1022,7 +1025,7 @@ static int __driver_attach(struct device *dev, void *data)
 		dev_dbg(dev, "Device match requests probe deferral\n");
 		driver_deferred_probe_add(dev);
 	} else if (ret < 0) {
-		dev_dbg(dev, "Bus failed to match device: %d", ret);
+		dev_dbg(dev, "Bus failed to match device: %d\n", ret);
 		return ret;
 	} /* ret > 0 means positive match */
 
diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c
index 1e9c96e3ed63..5327bfc6ba71 100644
--- a/drivers/base/firmware_loader/fallback.c
+++ b/drivers/base/firmware_loader/fallback.c
@@ -9,6 +9,7 @@
 #include <linux/umh.h>
 #include <linux/sysctl.h>
 #include <linux/vmalloc.h>
+#include <linux/module.h>
 
 #include "fallback.h"
 #include "firmware.h"
@@ -17,6 +18,8 @@
  * firmware fallback mechanism
  */
 
+MODULE_IMPORT_NS(FIRMWARE_LOADER_PRIVATE);
+
 extern struct firmware_fallback_config fw_fallback_config;
 
 /* These getters are vetted to use int properly */
@@ -460,7 +463,7 @@ static const struct attribute_group *fw_dev_attr_groups[] = {
 
 static struct fw_sysfs *
 fw_create_instance(struct firmware *firmware, const char *fw_name,
-		   struct device *device, enum fw_opt opt_flags)
+		   struct device *device, u32 opt_flags)
 {
 	struct fw_sysfs *fw_sysfs;
 	struct device *f_dev;
@@ -493,7 +496,7 @@ exit:
  * In charge of constructing a sysfs fallback interface for firmware loading.
  **/
 static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs,
-				  enum fw_opt opt_flags, long timeout)
+				  u32 opt_flags, long timeout)
 {
 	int retval = 0;
 	struct device *f_dev = &fw_sysfs->dev;
@@ -547,7 +550,7 @@ err_put_dev:
 
 static int fw_load_from_user_helper(struct firmware *firmware,
 				    const char *name, struct device *device,
-				    enum fw_opt opt_flags)
+				    u32 opt_flags)
 {
 	struct fw_sysfs *fw_sysfs;
 	long timeout;
@@ -588,7 +591,7 @@ out_unlock:
 	return ret;
 }
 
-static bool fw_force_sysfs_fallback(enum fw_opt opt_flags)
+static bool fw_force_sysfs_fallback(u32 opt_flags)
 {
 	if (fw_fallback_config.force_sysfs_fallback)
 		return true;
@@ -597,7 +600,7 @@ static bool fw_force_sysfs_fallback(enum fw_opt opt_flags)
 	return true;
 }
 
-static bool fw_run_sysfs_fallback(enum fw_opt opt_flags)
+static bool fw_run_sysfs_fallback(u32 opt_flags)
 {
 	int ret;
 
@@ -640,7 +643,7 @@ static bool fw_run_sysfs_fallback(enum fw_opt opt_flags)
  **/
 int firmware_fallback_sysfs(struct firmware *fw, const char *name,
 			    struct device *device,
-			    enum fw_opt opt_flags,
+			    u32 opt_flags,
 			    int ret)
 {
 	if (!fw_run_sysfs_fallback(opt_flags))
diff --git a/drivers/base/firmware_loader/fallback.h b/drivers/base/firmware_loader/fallback.h
index 06f4577733a8..2afdb6adb23f 100644
--- a/drivers/base/firmware_loader/fallback.h
+++ b/drivers/base/firmware_loader/fallback.h
@@ -33,7 +33,7 @@ struct firmware_fallback_config {
 #ifdef CONFIG_FW_LOADER_USER_HELPER
 int firmware_fallback_sysfs(struct firmware *fw, const char *name,
 			    struct device *device,
-			    enum fw_opt opt_flags,
+			    u32 opt_flags,
 			    int ret);
 void kill_pending_fw_fallback_reqs(bool only_kill_custom);
 
@@ -45,7 +45,7 @@ void unregister_sysfs_loader(void);
 #else /* CONFIG_FW_LOADER_USER_HELPER */
 static inline int firmware_fallback_sysfs(struct firmware *fw, const char *name,
 					  struct device *device,
-					  enum fw_opt opt_flags,
+					  u32 opt_flags,
 					  int ret)
 {
 	/* Keep carrying over the same error */
@@ -67,10 +67,10 @@ static inline void unregister_sysfs_loader(void)
 #endif /* CONFIG_FW_LOADER_USER_HELPER */
 
 #ifdef CONFIG_EFI_EMBEDDED_FIRMWARE
-int firmware_fallback_platform(struct fw_priv *fw_priv, enum fw_opt opt_flags);
+int firmware_fallback_platform(struct fw_priv *fw_priv, u32 opt_flags);
 #else
 static inline int firmware_fallback_platform(struct fw_priv *fw_priv,
-					     enum fw_opt opt_flags)
+					     u32 opt_flags)
 {
 	return -ENOENT;
 }
diff --git a/drivers/base/firmware_loader/fallback_platform.c b/drivers/base/firmware_loader/fallback_platform.c
index c88c745590fe..cdd2c9a9f38a 100644
--- a/drivers/base/firmware_loader/fallback_platform.c
+++ b/drivers/base/firmware_loader/fallback_platform.c
@@ -8,7 +8,7 @@
 #include "fallback.h"
 #include "firmware.h"
 
-int firmware_fallback_platform(struct fw_priv *fw_priv, enum fw_opt opt_flags)
+int firmware_fallback_platform(struct fw_priv *fw_priv, u32 opt_flags)
 {
 	const u8 *data;
 	size_t size;
diff --git a/drivers/base/firmware_loader/fallback_table.c b/drivers/base/firmware_loader/fallback_table.c
index a182e318bd09..46a731dede6f 100644
--- a/drivers/base/firmware_loader/fallback_table.c
+++ b/drivers/base/firmware_loader/fallback_table.c
@@ -21,7 +21,7 @@ struct firmware_fallback_config fw_fallback_config = {
 	.loading_timeout = 60,
 	.old_timeout = 60,
 };
-EXPORT_SYMBOL_GPL(fw_fallback_config);
+EXPORT_SYMBOL_NS_GPL(fw_fallback_config, FIRMWARE_LOADER_PRIVATE);
 
 #ifdef CONFIG_SYSCTL
 struct ctl_table firmware_config_table[] = {
diff --git a/drivers/base/firmware_loader/firmware.h b/drivers/base/firmware_loader/firmware.h
index 25836a6afc9f..933e2192fbe8 100644
--- a/drivers/base/firmware_loader/firmware.h
+++ b/drivers/base/firmware_loader/firmware.h
@@ -136,8 +136,7 @@ static inline void fw_state_done(struct fw_priv *fw_priv)
 	__fw_state_set(fw_priv, FW_STATUS_DONE);
 }
 
-int assign_fw(struct firmware *fw, struct device *device,
-	      enum fw_opt opt_flags);
+int assign_fw(struct firmware *fw, struct device *device, u32 opt_flags);
 
 #ifdef CONFIG_FW_LOADER_PAGED_BUF
 void fw_free_paged_buf(struct fw_priv *fw_priv);
diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c
index 76f79913916d..ca871b13524e 100644
--- a/drivers/base/firmware_loader/main.c
+++ b/drivers/base/firmware_loader/main.c
@@ -210,7 +210,7 @@ static struct fw_priv *__lookup_fw_priv(const char *fw_name)
 static int alloc_lookup_fw_priv(const char *fw_name,
 				struct firmware_cache *fwc,
 				struct fw_priv **fw_priv, void *dbuf,
-				size_t size, enum fw_opt opt_flags)
+				size_t size, u32 opt_flags)
 {
 	struct fw_priv *tmp;
 
@@ -548,9 +548,6 @@ static void firmware_free_data(const struct firmware *fw)
 static void fw_set_page_data(struct fw_priv *fw_priv, struct firmware *fw)
 {
 	fw->priv = fw_priv;
-#ifdef CONFIG_FW_LOADER_USER_HELPER
-	fw->pages = fw_priv->pages;
-#endif
 	fw->size = fw_priv->size;
 	fw->data = fw_priv->data;
 
@@ -635,8 +632,7 @@ static int fw_add_devm_name(struct device *dev, const char *name)
 }
 #endif
 
-int assign_fw(struct firmware *fw, struct device *device,
-	      enum fw_opt opt_flags)
+int assign_fw(struct firmware *fw, struct device *device, u32 opt_flags)
 {
 	struct fw_priv *fw_priv = fw->priv;
 	int ret;
@@ -687,7 +683,7 @@ int assign_fw(struct firmware *fw, struct device *device,
 static int
 _request_firmware_prepare(struct firmware **firmware_p, const char *name,
 			  struct device *device, void *dbuf, size_t size,
-			  enum fw_opt opt_flags)
+			  u32 opt_flags)
 {
 	struct firmware *firmware;
 	struct fw_priv *fw_priv;
@@ -753,7 +749,7 @@ static void fw_abort_batch_reqs(struct firmware *fw)
 static int
 _request_firmware(const struct firmware **firmware_p, const char *name,
 		  struct device *device, void *buf, size_t size,
-		  enum fw_opt opt_flags)
+		  u32 opt_flags)
 {
 	struct firmware *fw = NULL;
 	int ret;
@@ -990,7 +986,7 @@ struct firmware_work {
 	struct device *device;
 	void *context;
 	void (*cont)(const struct firmware *fw, void *context);
-	enum fw_opt opt_flags;
+	u32 opt_flags;
 };
 
 static void request_firmware_work_func(struct work_struct *work)
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 615c6b06b427..c0d0a5490ac6 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -153,23 +153,24 @@ EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource_byname);
  *		if (irq < 0)
  *			return irq;
  *
- * Return: IRQ number on success, negative error number on failure.
+ * Return: non-zero IRQ number on success, negative error number on failure.
  */
 int platform_get_irq_optional(struct platform_device *dev, unsigned int num)
 {
+	int ret;
 #ifdef CONFIG_SPARC
 	/* sparc does not have irqs represented as IORESOURCE_IRQ resources */
 	if (!dev || num >= dev->archdata.num_irqs)
 		return -ENXIO;
-	return dev->archdata.irqs[num];
+	ret = dev->archdata.irqs[num];
+	goto out;
 #else
 	struct resource *r;
-	int ret;
 
 	if (IS_ENABLED(CONFIG_OF_IRQ) && dev->dev.of_node) {
 		ret = of_irq_get(dev->dev.of_node, num);
 		if (ret > 0 || ret == -EPROBE_DEFER)
-			return ret;
+			goto out;
 	}
 
 	r = platform_get_resource(dev, IORESOURCE_IRQ, num);
@@ -177,7 +178,7 @@ int platform_get_irq_optional(struct platform_device *dev, unsigned int num)
 		if (r && r->flags & IORESOURCE_DISABLED) {
 			ret = acpi_irq_get(ACPI_HANDLE(&dev->dev), num, r);
 			if (ret)
-				return ret;
+				goto out;
 		}
 	}
 
@@ -191,13 +192,17 @@ int platform_get_irq_optional(struct platform_device *dev, unsigned int num)
 		struct irq_data *irqd;
 
 		irqd = irq_get_irq_data(r->start);
-		if (!irqd)
-			return -ENXIO;
+		if (!irqd) {
+			ret = -ENXIO;
+			goto out;
+		}
 		irqd_set_trigger_type(irqd, r->flags & IORESOURCE_BITS);
 	}
 
-	if (r)
-		return r->start;
+	if (r) {
+		ret = r->start;
+		goto out;
+	}
 
 	/*
 	 * For the index 0 interrupt, allow falling back to GpioInt
@@ -210,11 +215,14 @@ int platform_get_irq_optional(struct platform_device *dev, unsigned int num)
 		ret = acpi_dev_gpio_irq_get(ACPI_COMPANION(&dev->dev), num);
 		/* Our callers expect -ENXIO for missing IRQs. */
 		if (ret >= 0 || ret == -EPROBE_DEFER)
-			return ret;
+			goto out;
 	}
 
-	return -ENXIO;
+	ret = -ENXIO;
 #endif
+out:
+	WARN(ret == 0, "0 is an invalid IRQ number\n");
+	return ret;
 }
 EXPORT_SYMBOL_GPL(platform_get_irq_optional);
 
@@ -233,7 +241,7 @@ EXPORT_SYMBOL_GPL(platform_get_irq_optional);
  *		if (irq < 0)
  *			return irq;
  *
- * Return: IRQ number on success, negative error number on failure.
+ * Return: non-zero IRQ number on success, negative error number on failure.
  */
 int platform_get_irq(struct platform_device *dev, unsigned int num)
 {
@@ -305,8 +313,10 @@ static int __platform_get_irq_byname(struct platform_device *dev,
 	}
 
 	r = platform_get_resource_byname(dev, IORESOURCE_IRQ, name);
-	if (r)
+	if (r) {
+		WARN(r->start == 0, "0 is an invalid IRQ number\n");
 		return r->start;
+	}
 
 	return -ENXIO;
 }
@@ -318,7 +328,7 @@ static int __platform_get_irq_byname(struct platform_device *dev,
  *
  * Get an IRQ like platform_get_irq(), but then by name rather then by index.
  *
- * Return: IRQ number on success, negative error number on failure.
+ * Return: non-zero IRQ number on success, negative error number on failure.
  */
 int platform_get_irq_byname(struct platform_device *dev, const char *name)
 {
@@ -340,7 +350,7 @@ EXPORT_SYMBOL_GPL(platform_get_irq_byname);
  * Get an optional IRQ by name like platform_get_irq_byname(). Except that it
  * does not print an error message if an IRQ can not be obtained.
  *
- * Return: IRQ number on success, negative error number on failure.
+ * Return: non-zero IRQ number on success, negative error number on failure.
  */
 int platform_get_irq_byname_optional(struct platform_device *dev,
 				     const char *name)
@@ -672,7 +682,7 @@ EXPORT_SYMBOL_GPL(platform_device_unregister);
 struct platform_device *platform_device_register_full(
 		const struct platform_device_info *pdevinfo)
 {
-	int ret = -ENOMEM;
+	int ret;
 	struct platform_device *pdev;
 
 	pdev = platform_device_alloc(pdevinfo->name, pdevinfo->id);
@@ -853,6 +863,8 @@ int __init_or_module __platform_driver_probe(struct platform_driver *drv,
 	/* temporary section violation during probe() */
 	drv->probe = probe;
 	retval = code = __platform_driver_register(drv, module);
+	if (retval)
+		return retval;
 
 	/*
 	 * Fixup that section violation, being paranoid about code scanning
@@ -977,7 +989,7 @@ EXPORT_SYMBOL_GPL(__platform_register_drivers);
  * @drivers: an array of drivers to unregister
  * @count: the number of drivers to unregister
  *
- * Unegisters platform drivers specified by an array. This is typically used
+ * Unregisters platform drivers specified by an array. This is typically used
  * to complement an earlier call to platform_register_drivers(). Drivers are
  * unregistered in the reverse order in which they were registered.
  */
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index bb98b813554f..9dd85bea4026 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -519,7 +519,7 @@ static void dpm_watchdog_handler(struct timer_list *t)
 	struct dpm_watchdog *wd = from_timer(wd, t, timer);
 
 	dev_emerg(wd->dev, "**** DPM device timeout ****\n");
-	show_stack(wd->tsk, NULL);
+	show_stack(wd->tsk, NULL, KERN_EMERG);
 	panic("%s %s: unrecoverable failure\n",
 		dev_driver_string(wd->dev), dev_name(wd->dev));
 }
diff --git a/drivers/base/property.c b/drivers/base/property.c
index 5f35c0ccf5e0..1e6d75e65938 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -708,14 +708,23 @@ struct fwnode_handle *device_get_next_child_node(struct device *dev,
 						 struct fwnode_handle *child)
 {
 	struct acpi_device *adev = ACPI_COMPANION(dev);
-	struct fwnode_handle *fwnode = NULL;
+	struct fwnode_handle *fwnode = NULL, *next;
 
 	if (dev->of_node)
 		fwnode = &dev->of_node->fwnode;
 	else if (adev)
 		fwnode = acpi_fwnode_handle(adev);
 
-	return fwnode_get_next_child_node(fwnode, child);
+	/* Try to find a child in primary fwnode */
+	next = fwnode_get_next_child_node(fwnode, child);
+	if (next)
+		return next;
+
+	/* When no more children in primary, continue with secondary */
+	if (!IS_ERR_OR_NULL(fwnode->secondary))
+		next = fwnode_get_next_child_node(fwnode->secondary, child);
+
+	return next;
 }
 EXPORT_SYMBOL_GPL(device_get_next_child_node);
 
diff --git a/drivers/base/soc.c b/drivers/base/soc.c
index 4af11a423475..a5bae551167d 100644
--- a/drivers/base/soc.c
+++ b/drivers/base/soc.c
@@ -46,7 +46,7 @@ static umode_t soc_attribute_mode(struct kobject *kobj,
 				struct attribute *attr,
 				int index)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct soc_device *soc_dev = container_of(dev, struct soc_device, dev);
 
 	if ((attr == &dev_attr_machine.attr)
diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c
index 2079937ddb51..e5eb27375416 100644
--- a/drivers/base/swnode.c
+++ b/drivers/base/swnode.c
@@ -712,17 +712,18 @@ EXPORT_SYMBOL_GPL(software_node_register_nodes);
  * @nodes: Zero terminated array of software nodes to be unregistered
  *
  * Unregister multiple software nodes at once.
+ *
+ * NOTE: Be careful using this call if the nodes had parent pointers set up in
+ * them before registering.  If so, it is wiser to remove the nodes
+ * individually, in the correct order (child before parent) instead of relying
+ * on the sequential order of the list of nodes in the array.
  */
 void software_node_unregister_nodes(const struct software_node *nodes)
 {
-	struct swnode *swnode;
 	int i;
 
-	for (i = 0; nodes[i].name; i++) {
-		swnode = software_node_to_swnode(&nodes[i]);
-		if (swnode)
-			fwnode_remove_software_node(&swnode->fwnode);
-	}
+	for (i = 0; nodes[i].name; i++)
+		software_node_unregister(&nodes[i]);
 }
 EXPORT_SYMBOL_GPL(software_node_unregister_nodes);
 
@@ -789,6 +790,20 @@ int software_node_register(const struct software_node *node)
 }
 EXPORT_SYMBOL_GPL(software_node_register);
 
+/**
+ * software_node_unregister - Unregister static software node
+ * @node: The software node to be unregistered
+ */
+void software_node_unregister(const struct software_node *node)
+{
+	struct swnode *swnode;
+
+	swnode = software_node_to_swnode(node);
+	if (swnode)
+		fwnode_remove_software_node(&swnode->fwnode);
+}
+EXPORT_SYMBOL_GPL(software_node_unregister);
+
 struct fwnode_handle *
 fwnode_create_software_node(const struct property_entry *properties,
 			    const struct fwnode_handle *parent)
diff --git a/drivers/base/test/Kconfig b/drivers/base/test/Kconfig
index 305c7751184a..ba225eb1b761 100644
--- a/drivers/base/test/Kconfig
+++ b/drivers/base/test/Kconfig
@@ -9,5 +9,6 @@ config TEST_ASYNC_DRIVER_PROBE
 
 	  If unsure say N.
 config KUNIT_DRIVER_PE_TEST
-	bool "KUnit Tests for property entry API"
+	bool "KUnit Tests for property entry API" if !KUNIT_ALL_TESTS
 	depends on KUNIT=y
+	default KUNIT_ALL_TESTS
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index 025b1b77b11a..ecceaaa1a66f 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -7,7 +7,7 @@ menuconfig BLK_DEV
 	bool "Block devices"
 	depends on BLOCK
 	default y
-	---help---
+	help
 	  Say Y here to get to see options for various different block device
 	  drivers. This option alone does not add any kernel code.
 
@@ -27,7 +27,7 @@ config BLK_DEV_NULL_BLK_FAULT_INJECTION
 config BLK_DEV_FD
 	tristate "Normal floppy disk support"
 	depends on ARCH_MAY_HAVE_PC_FDC
-	---help---
+	help
 	  If you want to use the floppy disk drive(s) of your PC under Linux,
 	  say Y. Information about this driver, especially important for IBM
 	  Thinkpad users, is contained in
@@ -91,7 +91,7 @@ config GDROM
 config PARIDE
 	tristate "Parallel port IDE device support"
 	depends on PARPORT_PC
-	---help---
+	help
 	  There are many external CD-ROM and disk devices that connect through
 	  your computer's parallel port. Most of them are actually IDE devices
 	  using a parallel port IDE adapter. This option enables the PARIDE
@@ -124,7 +124,7 @@ source "drivers/block/zram/Kconfig"
 config BLK_DEV_UMEM
 	tristate "Micro Memory MM5415 Battery Backed RAM support"
 	depends on PCI
-	---help---
+	help
 	  Saying Y here will include support for the MM5415 family of
 	  battery backed (Non-volatile) RAM cards.
 	  <http://www.umem.com/>
@@ -141,7 +141,7 @@ config BLK_DEV_UMEM
 config BLK_DEV_UBD
 	bool "Virtual block device"
 	depends on UML
-	---help---
+	help
           The User-Mode Linux port includes a driver called UBD which will let
           you access arbitrary files on the host computer as block devices.
           Unless you know that you do not need such virtual block devices say
@@ -150,7 +150,7 @@ config BLK_DEV_UBD
 config BLK_DEV_UBD_SYNC
 	bool "Always do synchronous disk IO for UBD"
 	depends on BLK_DEV_UBD
-	---help---
+	help
 	  Writes to the virtual block device are not immediately written to the
 	  host's disk; this may cause problems if, for example, the User-Mode
 	  Linux 'Virtual Machine' uses a journalling filesystem and the host
@@ -173,7 +173,7 @@ config BLK_DEV_COW_COMMON
 
 config BLK_DEV_LOOP
 	tristate "Loopback device support"
-	---help---
+	help
 	  Saying Y here will allow you to use a regular file as a block
 	  device; you can then create a file system on that block device and
 	  mount it just as you would mount other block devices such as hard
@@ -234,7 +234,7 @@ config BLK_DEV_CRYPTOLOOP
 	select CRYPTO
 	select CRYPTO_CBC
 	depends on BLK_DEV_LOOP
-	---help---
+	help
 	  Say Y here if you want to be able to use the ciphers that are 
 	  provided by the CryptoAPI as loop transformation. This might be
 	  used as hard disk encryption.
@@ -249,7 +249,7 @@ source "drivers/block/drbd/Kconfig"
 config BLK_DEV_NBD
 	tristate "Network block device support"
 	depends on NET
-	---help---
+	help
 	  Saying Y here will allow your computer to be a client for network
 	  block devices, i.e. it will be able to use block devices exported by
 	  servers (mount file systems on them etc.). Communication between
@@ -277,7 +277,7 @@ config BLK_DEV_SKD
 	tristate "STEC S1120 Block Driver"
 	depends on PCI
 	depends on 64BIT
-	---help---
+	help
 	Saying Y or M here will enable support for the
 	STEC, Inc. S1120 PCIe SSD.
 
@@ -286,7 +286,7 @@ config BLK_DEV_SKD
 config BLK_DEV_SX8
 	tristate "Promise SATA SX8 support"
 	depends on PCI
-	---help---
+	help
 	  Saying Y or M here will enable support for the 
 	  Promise SATA SX8 controllers.
 
@@ -294,7 +294,7 @@ config BLK_DEV_SX8
 
 config BLK_DEV_RAM
 	tristate "RAM block device support"
-	---help---
+	help
 	  Saying Y here will allow you to use a portion of your RAM memory as
 	  a block device, so that you can make file systems on it, read and
 	  write to it and do all the other things that you can do with normal
@@ -428,7 +428,7 @@ config XEN_BLKDEV_BACKEND
 config VIRTIO_BLK
 	tristate "Virtio block driver"
 	depends on VIRTIO
-	---help---
+	help
 	  This is the virtual block driver for virtio.  It can be used with
           QEMU based VMMs (like KVM or Xen).  Say Y or M.
 
@@ -458,4 +458,6 @@ config BLK_DEV_RSXX
 	  To compile this driver as a module, choose M here: the
 	  module will be called rsxx.
 
+source "drivers/block/rnbd/Kconfig"
+
 endif # BLK_DEV
diff --git a/drivers/block/Makefile b/drivers/block/Makefile
index 795facd8cf19..e1f63117ee94 100644
--- a/drivers/block/Makefile
+++ b/drivers/block/Makefile
@@ -39,6 +39,7 @@ obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX)	+= mtip32xx/
 
 obj-$(CONFIG_BLK_DEV_RSXX) += rsxx/
 obj-$(CONFIG_ZRAM) += zram/
+obj-$(CONFIG_BLK_DEV_RNBD)	+= rnbd/
 
 obj-$(CONFIG_BLK_DEV_NULL_BLK)	+= null_blk.o
 null_blk-objs	:= null_blk_main.o
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 14345a87c7cc..33d0831c99b6 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -620,7 +620,7 @@ struct fifo_buffer {
 	unsigned int head_index;
 	unsigned int size;
 	int total; /* sum of all values */
-	int values[0];
+	int values[];
 };
 extern struct fifo_buffer *fifo_alloc(unsigned int fifo_size);
 
diff --git a/drivers/block/drbd/drbd_protocol.h b/drivers/block/drbd/drbd_protocol.h
index e6fc5ad72501..dea59c92ecc1 100644
--- a/drivers/block/drbd/drbd_protocol.h
+++ b/drivers/block/drbd/drbd_protocol.h
@@ -271,7 +271,7 @@ struct p_rs_param {
 	u32 resync_rate;
 
 	      /* Since protocol version 88 and higher. */
-	char verify_alg[0];
+	char verify_alg[];
 } __packed;
 
 struct p_rs_param_89 {
@@ -305,7 +305,7 @@ struct p_protocol {
 	u32 two_primaries;
 
 	/* Since protocol version 87 and higher. */
-	char integrity_alg[0];
+	char integrity_alg[];
 
 } __packed;
 
@@ -360,7 +360,7 @@ struct p_sizes {
 	u16	    dds_flags; /* use enum dds_flags here. */
 
 	/* optional queue_limits if (agreed_features & DRBD_FF_WSAME) */
-	struct o_qlim qlim[0];
+	struct o_qlim qlim[];
 } __packed;
 
 struct p_state {
@@ -409,7 +409,7 @@ struct p_compressed_bm {
 	 */
 	u8 encoding;
 
-	u8 code[0];
+	u8 code[];
 } __packed;
 
 struct p_delay_probe93 {
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 2e96d8b8758b..c33bbbfd1bd9 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -1390,7 +1390,7 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
 		goto out_unfreeze;
 
 	/* Mask out flags that can't be set using LOOP_SET_STATUS. */
-	lo->lo_flags &= ~LOOP_SET_STATUS_SETTABLE_FLAGS;
+	lo->lo_flags &= LOOP_SET_STATUS_SETTABLE_FLAGS;
 	/* For those flags, use the previous values instead */
 	lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_SETTABLE_FLAGS;
 	/* For flags that can't be cleared, use previous values too */
diff --git a/drivers/block/paride/Kconfig b/drivers/block/paride/Kconfig
index f8bd6ef3605a..7c6ae1036927 100644
--- a/drivers/block/paride/Kconfig
+++ b/drivers/block/paride/Kconfig
@@ -28,7 +28,7 @@ config PARIDE_PCD
 	depends on PARIDE
 	select CDROM
 	select BLK_SCSI_REQUEST # only for the generic cdrom code
-	---help---
+	help
 	  This option enables the high-level driver for ATAPI CD-ROM devices
 	  connected through a parallel port. If you chose to build PARIDE
 	  support into your kernel, you may answer Y here to build in the
@@ -71,7 +71,7 @@ config PARIDE_PT
 config PARIDE_PG
 	tristate "Parallel port generic ATAPI devices"
 	depends on PARIDE
-	---help---
+	help
 	  This option enables a special high-level driver for generic ATAPI
 	  devices connected through a parallel port. The driver allows user
 	  programs, such as cdrtools, to send ATAPI commands directly to a
@@ -111,7 +111,7 @@ config PARIDE_ATEN
 config PARIDE_BPCK
 	tristate "MicroSolutions backpack (Series 5) protocol"
 	depends on PARIDE
-	---help---
+	help
 	  This option enables support for the Micro Solutions BACKPACK
 	  parallel port Series 5 IDE protocol.  (Most BACKPACK drives made
 	  before 1999 were Series 5) Series 5 drives will NOT always have the
@@ -129,7 +129,7 @@ config PARIDE_BPCK
 config PARIDE_BPCK6
 	tristate "MicroSolutions backpack (Series 6) protocol"
 	depends on PARIDE && !64BIT
-	---help---
+	help
 	  This option enables support for the Micro Solutions BACKPACK
 	  parallel port Series 6 IDE protocol.  (Most BACKPACK drives made
 	  after 1999 were Series 6) Series 6 drives will have the Series noted
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 0b944ac96d6b..27a33adc41e4 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -1613,7 +1613,7 @@ static noinline_for_stack int pkt_get_last_written(struct pktcdvd_device *pd,
 	disc_information di;
 	track_information ti;
 	__u32 last_track;
-	int ret = -1;
+	int ret;
 
 	ret = pkt_get_disc_info(pd, &di);
 	if (ret)
diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c
index c5c6487a19d5..7b55811c2a81 100644
--- a/drivers/block/ps3disk.c
+++ b/drivers/block/ps3disk.c
@@ -454,7 +454,6 @@ static int ps3disk_probe(struct ps3_system_bus_device *_dev)
 	queue->queuedata = dev;
 
 	blk_queue_max_hw_sectors(queue, dev->bounce_size >> 9);
-	blk_queue_segment_boundary(queue, -1UL);
 	blk_queue_dma_alignment(queue, dev->blk_size-1);
 	blk_queue_logical_block_size(queue, dev->blk_size);
 
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 67d65ac785e9..4f61e9209461 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -836,6 +836,7 @@ enum {
 	Opt_lock_timeout,
 	/* int args above */
 	Opt_pool_ns,
+	Opt_compression_hint,
 	/* string args above */
 	Opt_read_only,
 	Opt_read_write,
@@ -844,8 +845,23 @@ enum {
 	Opt_notrim,
 };
 
+enum {
+	Opt_compression_hint_none,
+	Opt_compression_hint_compressible,
+	Opt_compression_hint_incompressible,
+};
+
+static const struct constant_table rbd_param_compression_hint[] = {
+	{"none",		Opt_compression_hint_none},
+	{"compressible",	Opt_compression_hint_compressible},
+	{"incompressible",	Opt_compression_hint_incompressible},
+	{}
+};
+
 static const struct fs_parameter_spec rbd_parameters[] = {
 	fsparam_u32	("alloc_size",			Opt_alloc_size),
+	fsparam_enum	("compression_hint",		Opt_compression_hint,
+			 rbd_param_compression_hint),
 	fsparam_flag	("exclusive",			Opt_exclusive),
 	fsparam_flag	("lock_on_read",		Opt_lock_on_read),
 	fsparam_u32	("lock_timeout",		Opt_lock_timeout),
@@ -867,6 +883,8 @@ struct rbd_options {
 	bool	lock_on_read;
 	bool	exclusive;
 	bool	trim;
+
+	u32 alloc_hint_flags;  /* CEPH_OSD_OP_ALLOC_HINT_FLAG_* */
 };
 
 #define RBD_QUEUE_DEPTH_DEFAULT	BLKDEV_MAX_RQ
@@ -1433,8 +1451,10 @@ static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
 static void rbd_osd_format_read(struct ceph_osd_request *osd_req)
 {
 	struct rbd_obj_request *obj_request = osd_req->r_priv;
+	struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
+	struct ceph_options *opt = rbd_dev->rbd_client->client->options;
 
-	osd_req->r_flags = CEPH_OSD_FLAG_READ;
+	osd_req->r_flags = CEPH_OSD_FLAG_READ | opt->read_from_replica;
 	osd_req->r_snapid = obj_request->img_request->snap_id;
 }
 
@@ -2253,7 +2273,8 @@ static void __rbd_osd_setup_write_ops(struct ceph_osd_request *osd_req,
 	    !(obj_req->flags & RBD_OBJ_FLAG_MAY_EXIST)) {
 		osd_req_op_alloc_hint_init(osd_req, which++,
 					   rbd_dev->layout.object_size,
-					   rbd_dev->layout.object_size);
+					   rbd_dev->layout.object_size,
+					   rbd_dev->opts->alloc_hint_flags);
 	}
 
 	if (rbd_obj_is_entire(obj_req))
@@ -6331,6 +6352,29 @@ static int rbd_parse_param(struct fs_parameter *param,
 		pctx->spec->pool_ns = param->string;
 		param->string = NULL;
 		break;
+	case Opt_compression_hint:
+		switch (result.uint_32) {
+		case Opt_compression_hint_none:
+			opt->alloc_hint_flags &=
+			    ~(CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE |
+			      CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE);
+			break;
+		case Opt_compression_hint_compressible:
+			opt->alloc_hint_flags |=
+			    CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE;
+			opt->alloc_hint_flags &=
+			    ~CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE;
+			break;
+		case Opt_compression_hint_incompressible:
+			opt->alloc_hint_flags |=
+			    CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE;
+			opt->alloc_hint_flags &=
+			    ~CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE;
+			break;
+		default:
+			BUG();
+		}
+		break;
 	case Opt_read_only:
 		opt->read_only = true;
 		break;
diff --git a/drivers/block/rbd_types.h b/drivers/block/rbd_types.h
index ac98ab6ccd3b..a600e0eb6b6f 100644
--- a/drivers/block/rbd_types.h
+++ b/drivers/block/rbd_types.h
@@ -93,7 +93,7 @@ struct rbd_image_header_ondisk {
 	__le32 snap_count;
 	__le32 reserved;
 	__le64 snap_names_len;
-	struct rbd_image_snap_ondisk snaps[0];
+	struct rbd_image_snap_ondisk snaps[];
 } __attribute__((packed));
 
 
diff --git a/drivers/block/rnbd/Kconfig b/drivers/block/rnbd/Kconfig
new file mode 100644
index 000000000000..4b6d3d816d1f
--- /dev/null
+++ b/drivers/block/rnbd/Kconfig
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+config BLK_DEV_RNBD
+	bool
+
+config BLK_DEV_RNBD_CLIENT
+	tristate "RDMA Network Block Device driver client"
+	depends on INFINIBAND_RTRS_CLIENT
+	select BLK_DEV_RNBD
+	help
+	  RNBD client is a network block device driver using rdma transport.
+
+	  RNBD client allows for mapping of a remote block devices over
+	  RTRS protocol from a target system where RNBD server is running.
+
+	  If unsure, say N.
+
+config BLK_DEV_RNBD_SERVER
+	tristate "RDMA Network Block Device driver server"
+	depends on INFINIBAND_RTRS_SERVER
+	select BLK_DEV_RNBD
+	help
+	  RNBD server is the server side of RNBD using rdma transport.
+
+	  RNBD server allows for exporting local block devices to a remote client
+	  over RTRS protocol.
+
+	  If unsure, say N.
diff --git a/drivers/block/rnbd/Makefile b/drivers/block/rnbd/Makefile
new file mode 100644
index 000000000000..5bb1a7ad1ada
--- /dev/null
+++ b/drivers/block/rnbd/Makefile
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+ccflags-y := -I$(srctree)/drivers/infiniband/ulp/rtrs
+
+rnbd-client-y := rnbd-clt.o \
+		  rnbd-clt-sysfs.o \
+		  rnbd-common.o
+
+rnbd-server-y := rnbd-common.o \
+		  rnbd-srv.o \
+		  rnbd-srv-dev.o \
+		  rnbd-srv-sysfs.o
+
+obj-$(CONFIG_BLK_DEV_RNBD_CLIENT) += rnbd-client.o
+obj-$(CONFIG_BLK_DEV_RNBD_SERVER) += rnbd-server.o
diff --git a/drivers/block/rnbd/README b/drivers/block/rnbd/README
new file mode 100644
index 000000000000..1773c0aa0bd4
--- /dev/null
+++ b/drivers/block/rnbd/README
@@ -0,0 +1,92 @@
+********************************
+RDMA Network Block Device (RNBD)
+********************************
+
+Introduction
+------------
+
+RNBD (RDMA Network Block Device) is a pair of kernel modules
+(client and server) that allow for remote access of a block device on
+the server over RTRS protocol using the RDMA (InfiniBand, RoCE, iWARP)
+transport. After being mapped, the remote block devices can be accessed
+on the client side as local block devices.
+
+I/O is transferred between client and server by the RTRS transport
+modules. The administration of RNBD and RTRS modules is done via
+sysfs entries.
+
+Requirements
+------------
+
+  RTRS kernel modules
+
+Quick Start
+-----------
+
+Server side:
+  # modprobe rnbd_server
+
+Client side:
+  # modprobe rnbd_client
+  # echo "sessname=blya path=ip:10.50.100.66 device_path=/dev/ram0" > \
+            /sys/devices/virtual/rnbd-client/ctl/map_device
+
+  Where "sessname=" is a session name, a string to identify the session
+  on client and on server sides; "path=" is a destination IP address or
+  a pair of a source and a destination IPs, separated by comma.  Multiple
+  "path=" options can be specified in order to use multipath  (see RTRS
+  description for details); "device_path=" is the block device to be
+  mapped from the server side. After the session to the server machine is
+  established, the mapped device will appear on the client side under
+  /dev/rnbd<N>.
+
+
+RNBD-Server Module Parameters
+=============================
+
+dev_search_path
+---------------
+
+When a device is mapped from the client, the server generates the path
+to the block device on the server side by concatenating dev_search_path
+and the "device_path" that was specified in the map_device operation.
+
+The default dev_search_path is: "/".
+
+dev_search_path option can also contain %SESSNAME% in order to provide
+different device namespaces for different sessions.  See "device_path"
+option for details.
+
+============================
+Protocol (rnbd/rnbd-proto.h)
+============================
+
+1. Before mapping first device from a given server, client sends an
+RNBD_MSG_SESS_INFO to the server. Server responds with
+RNBD_MSG_SESS_INFO_RSP. Currently the messages only contain the protocol
+version for backward compatibility.
+
+2. Client requests to open a device by sending RNBD_MSG_OPEN message. This
+contains the path to the device and access mode (read-only or writable).
+Server responds to the message with RNBD_MSG_OPEN_RSP. This contains
+a 32 bit device id to be used for  IOs and device "geometry" related
+information: side, max_hw_sectors, etc.
+
+3. Client attaches RNBD_MSG_IO to each IO message send to a device. This
+message contains device id, provided by server in his rnbd_msg_open_rsp,
+sector to be accessed, read-write flags and bi_size.
+
+4. Client closes a device by sending RNBD_MSG_CLOSE which contains only the
+device id provided by the server.
+
+=========================================
+Contributors List(in alphabetical order)
+=========================================
+Danil Kipnis <danil.kipnis@profitbricks.com>
+Fabian Holler <mail@fholler.de>
+Guoqing Jiang <guoqing.jiang@cloud.ionos.com>
+Jack Wang <jinpu.wang@profitbricks.com>
+Kleber Souza <kleber.souza@profitbricks.com>
+Lutz Pogrell <lutz.pogrell@cloud.ionos.com>
+Milind Dumbare <Milind.dumbare@gmail.com>
+Roman Penyaev <roman.penyaev@profitbricks.com>
diff --git a/drivers/block/rnbd/rnbd-clt-sysfs.c b/drivers/block/rnbd/rnbd-clt-sysfs.c
new file mode 100644
index 000000000000..4f4474eecadb
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-clt-sysfs.c
@@ -0,0 +1,639 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include <linux/types.h>
+#include <linux/ctype.h>
+#include <linux/parser.h>
+#include <linux/module.h>
+#include <linux/in6.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+#include <linux/device.h>
+#include <rdma/ib.h>
+#include <rdma/rdma_cm.h>
+
+#include "rnbd-clt.h"
+
+static struct device *rnbd_dev;
+static struct class *rnbd_dev_class;
+static struct kobject *rnbd_devs_kobj;
+
+enum {
+	RNBD_OPT_ERR		= 0,
+	RNBD_OPT_DEST_PORT	= 1 << 0,
+	RNBD_OPT_PATH		= 1 << 1,
+	RNBD_OPT_DEV_PATH	= 1 << 2,
+	RNBD_OPT_ACCESS_MODE	= 1 << 3,
+	RNBD_OPT_SESSNAME	= 1 << 6,
+};
+
+static const unsigned int rnbd_opt_mandatory[] = {
+	RNBD_OPT_PATH,
+	RNBD_OPT_DEV_PATH,
+	RNBD_OPT_SESSNAME,
+};
+
+static const match_table_t rnbd_opt_tokens = {
+	{RNBD_OPT_PATH,		"path=%s"	},
+	{RNBD_OPT_DEV_PATH,	"device_path=%s"},
+	{RNBD_OPT_DEST_PORT,	"dest_port=%d"  },
+	{RNBD_OPT_ACCESS_MODE,	"access_mode=%s"},
+	{RNBD_OPT_SESSNAME,	"sessname=%s"	},
+	{RNBD_OPT_ERR,		NULL		},
+};
+
+struct rnbd_map_options {
+	char *sessname;
+	struct rtrs_addr *paths;
+	size_t *path_cnt;
+	char *pathname;
+	u16 *dest_port;
+	enum rnbd_access_mode *access_mode;
+};
+
+static int rnbd_clt_parse_map_options(const char *buf, size_t max_path_cnt,
+				       struct rnbd_map_options *opt)
+{
+	char *options, *sep_opt;
+	char *p;
+	substring_t args[MAX_OPT_ARGS];
+	int opt_mask = 0;
+	int token;
+	int ret = -EINVAL;
+	int i, dest_port;
+	int p_cnt = 0;
+
+	options = kstrdup(buf, GFP_KERNEL);
+	if (!options)
+		return -ENOMEM;
+
+	sep_opt = strstrip(options);
+	while ((p = strsep(&sep_opt, " ")) != NULL) {
+		if (!*p)
+			continue;
+
+		token = match_token(p, rnbd_opt_tokens, args);
+		opt_mask |= token;
+
+		switch (token) {
+		case RNBD_OPT_SESSNAME:
+			p = match_strdup(args);
+			if (!p) {
+				ret = -ENOMEM;
+				goto out;
+			}
+			if (strlen(p) > NAME_MAX) {
+				pr_err("map_device: sessname too long\n");
+				ret = -EINVAL;
+				kfree(p);
+				goto out;
+			}
+			strlcpy(opt->sessname, p, NAME_MAX);
+			kfree(p);
+			break;
+
+		case RNBD_OPT_PATH:
+			if (p_cnt >= max_path_cnt) {
+				pr_err("map_device: too many (> %zu) paths provided\n",
+				       max_path_cnt);
+				ret = -ENOMEM;
+				goto out;
+			}
+			p = match_strdup(args);
+			if (!p) {
+				ret = -ENOMEM;
+				goto out;
+			}
+
+			ret = rtrs_addr_to_sockaddr(p, strlen(p),
+						    *opt->dest_port,
+						    &opt->paths[p_cnt]);
+			if (ret) {
+				pr_err("Can't parse path %s: %d\n", p, ret);
+				kfree(p);
+				goto out;
+			}
+
+			p_cnt++;
+
+			kfree(p);
+			break;
+
+		case RNBD_OPT_DEV_PATH:
+			p = match_strdup(args);
+			if (!p) {
+				ret = -ENOMEM;
+				goto out;
+			}
+			if (strlen(p) > NAME_MAX) {
+				pr_err("map_device: Device path too long\n");
+				ret = -EINVAL;
+				kfree(p);
+				goto out;
+			}
+			strlcpy(opt->pathname, p, NAME_MAX);
+			kfree(p);
+			break;
+
+		case RNBD_OPT_DEST_PORT:
+			if (match_int(args, &dest_port) || dest_port < 0 ||
+			    dest_port > 65535) {
+				pr_err("bad destination port number parameter '%d'\n",
+				       dest_port);
+				ret = -EINVAL;
+				goto out;
+			}
+			*opt->dest_port = dest_port;
+			break;
+
+		case RNBD_OPT_ACCESS_MODE:
+			p = match_strdup(args);
+			if (!p) {
+				ret = -ENOMEM;
+				goto out;
+			}
+
+			if (!strcmp(p, "ro")) {
+				*opt->access_mode = RNBD_ACCESS_RO;
+			} else if (!strcmp(p, "rw")) {
+				*opt->access_mode = RNBD_ACCESS_RW;
+			} else if (!strcmp(p, "migration")) {
+				*opt->access_mode = RNBD_ACCESS_MIGRATION;
+			} else {
+				pr_err("map_device: Invalid access_mode: '%s'\n",
+				       p);
+				ret = -EINVAL;
+				kfree(p);
+				goto out;
+			}
+
+			kfree(p);
+			break;
+
+		default:
+			pr_err("map_device: Unknown parameter or missing value '%s'\n",
+			       p);
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(rnbd_opt_mandatory); i++) {
+		if ((opt_mask & rnbd_opt_mandatory[i])) {
+			ret = 0;
+		} else {
+			pr_err("map_device: Parameters missing\n");
+			ret = -EINVAL;
+			break;
+		}
+	}
+
+out:
+	*opt->path_cnt = p_cnt;
+	kfree(options);
+	return ret;
+}
+
+static ssize_t state_show(struct kobject *kobj,
+			  struct kobj_attribute *attr, char *page)
+{
+	struct rnbd_clt_dev *dev;
+
+	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
+
+	switch (dev->dev_state) {
+	case DEV_STATE_INIT:
+		return snprintf(page, PAGE_SIZE, "init\n");
+	case DEV_STATE_MAPPED:
+		/* TODO fix cli tool before changing to proper state */
+		return snprintf(page, PAGE_SIZE, "open\n");
+	case DEV_STATE_MAPPED_DISCONNECTED:
+		/* TODO fix cli tool before changing to proper state */
+		return snprintf(page, PAGE_SIZE, "closed\n");
+	case DEV_STATE_UNMAPPED:
+		return snprintf(page, PAGE_SIZE, "unmapped\n");
+	default:
+		return snprintf(page, PAGE_SIZE, "unknown\n");
+	}
+}
+
+static struct kobj_attribute rnbd_clt_state_attr = __ATTR_RO(state);
+
+static ssize_t mapping_path_show(struct kobject *kobj,
+				 struct kobj_attribute *attr, char *page)
+{
+	struct rnbd_clt_dev *dev;
+
+	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%s\n", dev->pathname);
+}
+
+static struct kobj_attribute rnbd_clt_mapping_path_attr =
+	__ATTR_RO(mapping_path);
+
+static ssize_t access_mode_show(struct kobject *kobj,
+				struct kobj_attribute *attr, char *page)
+{
+	struct rnbd_clt_dev *dev;
+
+	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
+
+	return snprintf(page, PAGE_SIZE, "%s\n",
+			rnbd_access_mode_str(dev->access_mode));
+}
+
+static struct kobj_attribute rnbd_clt_access_mode =
+	__ATTR_RO(access_mode);
+
+static ssize_t rnbd_clt_unmap_dev_show(struct kobject *kobj,
+					struct kobj_attribute *attr, char *page)
+{
+	return scnprintf(page, PAGE_SIZE, "Usage: echo <normal|force> > %s\n",
+			 attr->attr.name);
+}
+
+static ssize_t rnbd_clt_unmap_dev_store(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct rnbd_clt_dev *dev;
+	char *opt, *options;
+	bool force;
+	int err;
+
+	opt = kstrdup(buf, GFP_KERNEL);
+	if (!opt)
+		return -ENOMEM;
+
+	options = strstrip(opt);
+	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
+	if (sysfs_streq(options, "normal")) {
+		force = false;
+	} else if (sysfs_streq(options, "force")) {
+		force = true;
+	} else {
+		rnbd_clt_err(dev,
+			      "unmap_device: Invalid value: %s\n",
+			      options);
+		err = -EINVAL;
+		goto out;
+	}
+
+	rnbd_clt_info(dev, "Unmapping device, option: %s.\n",
+		       force ? "force" : "normal");
+
+	/*
+	 * We take explicit module reference only for one reason: do not
+	 * race with lockless rnbd_destroy_sessions().
+	 */
+	if (!try_module_get(THIS_MODULE)) {
+		err = -ENODEV;
+		goto out;
+	}
+	err = rnbd_clt_unmap_device(dev, force, &attr->attr);
+	if (err) {
+		if (err != -EALREADY)
+			rnbd_clt_err(dev, "unmap_device: %d\n",  err);
+		goto module_put;
+	}
+
+	/*
+	 * Here device can be vanished!
+	 */
+
+	err = count;
+
+module_put:
+	module_put(THIS_MODULE);
+out:
+	kfree(opt);
+
+	return err;
+}
+
+static struct kobj_attribute rnbd_clt_unmap_device_attr =
+	__ATTR(unmap_device, 0644, rnbd_clt_unmap_dev_show,
+	       rnbd_clt_unmap_dev_store);
+
+static ssize_t rnbd_clt_resize_dev_show(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 char *page)
+{
+	return scnprintf(page, PAGE_SIZE,
+			 "Usage: echo <new size in sectors> > %s\n",
+			 attr->attr.name);
+}
+
+static ssize_t rnbd_clt_resize_dev_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf, size_t count)
+{
+	int ret;
+	unsigned long sectors;
+	struct rnbd_clt_dev *dev;
+
+	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
+
+	ret = kstrtoul(buf, 0, &sectors);
+	if (ret)
+		return ret;
+
+	ret = rnbd_clt_resize_disk(dev, (size_t)sectors);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static struct kobj_attribute rnbd_clt_resize_dev_attr =
+	__ATTR(resize, 0644, rnbd_clt_resize_dev_show,
+	       rnbd_clt_resize_dev_store);
+
+static ssize_t rnbd_clt_remap_dev_show(struct kobject *kobj,
+					struct kobj_attribute *attr, char *page)
+{
+	return scnprintf(page, PAGE_SIZE, "Usage: echo <1> > %s\n",
+			 attr->attr.name);
+}
+
+static ssize_t rnbd_clt_remap_dev_store(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct rnbd_clt_dev *dev;
+	char *opt, *options;
+	int err;
+
+	opt = kstrdup(buf, GFP_KERNEL);
+	if (!opt)
+		return -ENOMEM;
+
+	options = strstrip(opt);
+	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
+	if (!sysfs_streq(options, "1")) {
+		rnbd_clt_err(dev,
+			      "remap_device: Invalid value: %s\n",
+			      options);
+		err = -EINVAL;
+		goto out;
+	}
+	err = rnbd_clt_remap_device(dev);
+	if (likely(!err))
+		err = count;
+
+out:
+	kfree(opt);
+
+	return err;
+}
+
+static struct kobj_attribute rnbd_clt_remap_device_attr =
+	__ATTR(remap_device, 0644, rnbd_clt_remap_dev_show,
+	       rnbd_clt_remap_dev_store);
+
+static ssize_t session_show(struct kobject *kobj, struct kobj_attribute *attr,
+			    char *page)
+{
+	struct rnbd_clt_dev *dev;
+
+	dev = container_of(kobj, struct rnbd_clt_dev, kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%s\n", dev->sess->sessname);
+}
+
+static struct kobj_attribute rnbd_clt_session_attr =
+	__ATTR_RO(session);
+
+static struct attribute *rnbd_dev_attrs[] = {
+	&rnbd_clt_unmap_device_attr.attr,
+	&rnbd_clt_resize_dev_attr.attr,
+	&rnbd_clt_remap_device_attr.attr,
+	&rnbd_clt_mapping_path_attr.attr,
+	&rnbd_clt_state_attr.attr,
+	&rnbd_clt_session_attr.attr,
+	&rnbd_clt_access_mode.attr,
+	NULL,
+};
+
+void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev)
+{
+	/*
+	 * The module unload rnbd_client_exit path is racing with unmapping of
+	 * the last single device from the sysfs manually
+	 * i.e. rnbd_clt_unmap_dev_store() leading to a sysfs warning because
+	 * of sysfs link already was removed already.
+	 */
+	if (strlen(dev->blk_symlink_name) && try_module_get(THIS_MODULE)) {
+		sysfs_remove_link(rnbd_devs_kobj, dev->blk_symlink_name);
+		module_put(THIS_MODULE);
+	}
+}
+
+static struct kobj_type rnbd_dev_ktype = {
+	.sysfs_ops      = &kobj_sysfs_ops,
+	.default_attrs  = rnbd_dev_attrs,
+};
+
+static int rnbd_clt_add_dev_kobj(struct rnbd_clt_dev *dev)
+{
+	int ret;
+	struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
+
+	ret = kobject_init_and_add(&dev->kobj, &rnbd_dev_ktype, gd_kobj, "%s",
+				   "rnbd");
+	if (ret)
+		rnbd_clt_err(dev, "Failed to create device sysfs dir, err: %d\n",
+			      ret);
+
+	return ret;
+}
+
+static ssize_t rnbd_clt_map_device_show(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 char *page)
+{
+	return scnprintf(page, PAGE_SIZE,
+			 "Usage: echo \"[dest_port=server port number] sessname=<name of the rtrs session> path=<[srcaddr@]dstaddr> [path=<[srcaddr@]dstaddr>] device_path=<full path on remote side> [access_mode=<ro|rw|migration>]\" > %s\n\naddr ::= [ ip:<ipv4> | ip:<ipv6> | gid:<gid> ]\n",
+			 attr->attr.name);
+}
+
+static int rnbd_clt_get_path_name(struct rnbd_clt_dev *dev, char *buf,
+				   size_t len)
+{
+	int ret;
+	char pathname[NAME_MAX], *s;
+
+	strlcpy(pathname, dev->pathname, sizeof(pathname));
+	while ((s = strchr(pathname, '/')))
+		s[0] = '!';
+
+	ret = snprintf(buf, len, "%s", pathname);
+	if (ret >= len)
+		return -ENAMETOOLONG;
+
+	return 0;
+}
+
+static int rnbd_clt_add_dev_symlink(struct rnbd_clt_dev *dev)
+{
+	struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
+	int ret;
+
+	ret = rnbd_clt_get_path_name(dev, dev->blk_symlink_name,
+				      sizeof(dev->blk_symlink_name));
+	if (ret) {
+		rnbd_clt_err(dev, "Failed to get /sys/block symlink path, err: %d\n",
+			      ret);
+		goto out_err;
+	}
+
+	ret = sysfs_create_link(rnbd_devs_kobj, gd_kobj,
+				dev->blk_symlink_name);
+	if (ret) {
+		rnbd_clt_err(dev, "Creating /sys/block symlink failed, err: %d\n",
+			      ret);
+		goto out_err;
+	}
+
+	return 0;
+
+out_err:
+	dev->blk_symlink_name[0] = '\0';
+	return ret;
+}
+
+static ssize_t rnbd_clt_map_device_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct rnbd_clt_dev *dev;
+	struct rnbd_map_options opt;
+	int ret;
+	char pathname[NAME_MAX];
+	char sessname[NAME_MAX];
+	enum rnbd_access_mode access_mode = RNBD_ACCESS_RW;
+	u16 port_nr = RTRS_PORT;
+
+	struct sockaddr_storage *addrs;
+	struct rtrs_addr paths[6];
+	size_t path_cnt;
+
+	opt.sessname = sessname;
+	opt.paths = paths;
+	opt.path_cnt = &path_cnt;
+	opt.pathname = pathname;
+	opt.dest_port = &port_nr;
+	opt.access_mode = &access_mode;
+	addrs = kcalloc(ARRAY_SIZE(paths) * 2, sizeof(*addrs), GFP_KERNEL);
+	if (!addrs)
+		return -ENOMEM;
+
+	for (path_cnt = 0; path_cnt < ARRAY_SIZE(paths); path_cnt++) {
+		paths[path_cnt].src = &addrs[path_cnt * 2];
+		paths[path_cnt].dst = &addrs[path_cnt * 2 + 1];
+	}
+
+	ret = rnbd_clt_parse_map_options(buf, ARRAY_SIZE(paths), &opt);
+	if (ret)
+		goto out;
+
+	pr_info("Mapping device %s on session %s, (access_mode: %s)\n",
+		pathname, sessname,
+		rnbd_access_mode_str(access_mode));
+
+	dev = rnbd_clt_map_device(sessname, paths, path_cnt, port_nr, pathname,
+				  access_mode);
+	if (IS_ERR(dev)) {
+		ret = PTR_ERR(dev);
+		goto out;
+	}
+
+	ret = rnbd_clt_add_dev_kobj(dev);
+	if (ret)
+		goto unmap_dev;
+
+	ret = rnbd_clt_add_dev_symlink(dev);
+	if (ret)
+		goto unmap_dev;
+
+	kfree(addrs);
+	return count;
+
+unmap_dev:
+	rnbd_clt_unmap_device(dev, true, NULL);
+out:
+	kfree(addrs);
+	return ret;
+}
+
+static struct kobj_attribute rnbd_clt_map_device_attr =
+	__ATTR(map_device, 0644,
+	       rnbd_clt_map_device_show, rnbd_clt_map_device_store);
+
+static struct attribute *default_attrs[] = {
+	&rnbd_clt_map_device_attr.attr,
+	NULL,
+};
+
+static struct attribute_group default_attr_group = {
+	.attrs = default_attrs,
+};
+
+static const struct attribute_group *default_attr_groups[] = {
+	&default_attr_group,
+	NULL,
+};
+
+int rnbd_clt_create_sysfs_files(void)
+{
+	int err;
+
+	rnbd_dev_class = class_create(THIS_MODULE, "rnbd-client");
+	if (IS_ERR(rnbd_dev_class))
+		return PTR_ERR(rnbd_dev_class);
+
+	rnbd_dev = device_create_with_groups(rnbd_dev_class, NULL,
+					      MKDEV(0, 0), NULL,
+					      default_attr_groups, "ctl");
+	if (IS_ERR(rnbd_dev)) {
+		err = PTR_ERR(rnbd_dev);
+		goto cls_destroy;
+	}
+	rnbd_devs_kobj = kobject_create_and_add("devices", &rnbd_dev->kobj);
+	if (!rnbd_devs_kobj) {
+		err = -ENOMEM;
+		goto dev_destroy;
+	}
+
+	return 0;
+
+dev_destroy:
+	device_destroy(rnbd_dev_class, MKDEV(0, 0));
+cls_destroy:
+	class_destroy(rnbd_dev_class);
+
+	return err;
+}
+
+void rnbd_clt_destroy_default_group(void)
+{
+	sysfs_remove_group(&rnbd_dev->kobj, &default_attr_group);
+}
+
+void rnbd_clt_destroy_sysfs_files(void)
+{
+	kobject_del(rnbd_devs_kobj);
+	kobject_put(rnbd_devs_kobj);
+	device_destroy(rnbd_dev_class, MKDEV(0, 0));
+	class_destroy(rnbd_dev_class);
+}
diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c
new file mode 100644
index 000000000000..cc6a4e2587ae
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-clt.c
@@ -0,0 +1,1729 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include <linux/module.h>
+#include <linux/blkdev.h>
+#include <linux/hdreg.h>
+#include <linux/scatterlist.h>
+#include <linux/idr.h>
+
+#include "rnbd-clt.h"
+
+MODULE_DESCRIPTION("RDMA Network Block Device Client");
+MODULE_LICENSE("GPL");
+
+static int rnbd_client_major;
+static DEFINE_IDA(index_ida);
+static DEFINE_MUTEX(ida_lock);
+static DEFINE_MUTEX(sess_lock);
+static LIST_HEAD(sess_list);
+
+/*
+ * Maximum number of partitions an instance can have.
+ * 6 bits = 64 minors = 63 partitions (one minor is used for the device itself)
+ */
+#define RNBD_PART_BITS		6
+
+static inline bool rnbd_clt_get_sess(struct rnbd_clt_session *sess)
+{
+	return refcount_inc_not_zero(&sess->refcount);
+}
+
+static void free_sess(struct rnbd_clt_session *sess);
+
+static void rnbd_clt_put_sess(struct rnbd_clt_session *sess)
+{
+	might_sleep();
+
+	if (refcount_dec_and_test(&sess->refcount))
+		free_sess(sess);
+}
+
+static void rnbd_clt_put_dev(struct rnbd_clt_dev *dev)
+{
+	might_sleep();
+
+	if (!refcount_dec_and_test(&dev->refcount))
+		return;
+
+	mutex_lock(&ida_lock);
+	ida_simple_remove(&index_ida, dev->clt_device_id);
+	mutex_unlock(&ida_lock);
+	kfree(dev->hw_queues);
+	rnbd_clt_put_sess(dev->sess);
+	mutex_destroy(&dev->lock);
+	kfree(dev);
+}
+
+static inline bool rnbd_clt_get_dev(struct rnbd_clt_dev *dev)
+{
+	return refcount_inc_not_zero(&dev->refcount);
+}
+
+static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev,
+				 const struct rnbd_msg_open_rsp *rsp)
+{
+	struct rnbd_clt_session *sess = dev->sess;
+
+	if (!rsp->logical_block_size)
+		return -EINVAL;
+
+	dev->device_id		    = le32_to_cpu(rsp->device_id);
+	dev->nsectors		    = le64_to_cpu(rsp->nsectors);
+	dev->logical_block_size	    = le16_to_cpu(rsp->logical_block_size);
+	dev->physical_block_size    = le16_to_cpu(rsp->physical_block_size);
+	dev->max_write_same_sectors = le32_to_cpu(rsp->max_write_same_sectors);
+	dev->max_discard_sectors    = le32_to_cpu(rsp->max_discard_sectors);
+	dev->discard_granularity    = le32_to_cpu(rsp->discard_granularity);
+	dev->discard_alignment	    = le32_to_cpu(rsp->discard_alignment);
+	dev->secure_discard	    = le16_to_cpu(rsp->secure_discard);
+	dev->rotational		    = rsp->rotational;
+
+	dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE;
+	dev->max_segments = BMAX_SEGMENTS;
+
+	dev->max_hw_sectors = min_t(u32, dev->max_hw_sectors,
+				    le32_to_cpu(rsp->max_hw_sectors));
+	dev->max_segments = min_t(u16, dev->max_segments,
+				  le16_to_cpu(rsp->max_segments));
+
+	return 0;
+}
+
+static int rnbd_clt_change_capacity(struct rnbd_clt_dev *dev,
+				    size_t new_nsectors)
+{
+	int err = 0;
+
+	rnbd_clt_info(dev, "Device size changed from %zu to %zu sectors\n",
+		       dev->nsectors, new_nsectors);
+	dev->nsectors = new_nsectors;
+	set_capacity(dev->gd, dev->nsectors);
+	err = revalidate_disk(dev->gd);
+	if (err)
+		rnbd_clt_err(dev,
+			      "Failed to change device size from %zu to %zu, err: %d\n",
+			      dev->nsectors, new_nsectors, err);
+	return err;
+}
+
+static int process_msg_open_rsp(struct rnbd_clt_dev *dev,
+				struct rnbd_msg_open_rsp *rsp)
+{
+	int err = 0;
+
+	mutex_lock(&dev->lock);
+	if (dev->dev_state == DEV_STATE_UNMAPPED) {
+		rnbd_clt_info(dev,
+			       "Ignoring Open-Response message from server for  unmapped device\n");
+		err = -ENOENT;
+		goto out;
+	}
+	if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED) {
+		u64 nsectors = le64_to_cpu(rsp->nsectors);
+
+		/*
+		 * If the device was remapped and the size changed in the
+		 * meantime we need to revalidate it
+		 */
+		if (dev->nsectors != nsectors)
+			rnbd_clt_change_capacity(dev, nsectors);
+		rnbd_clt_info(dev, "Device online, device remapped successfully\n");
+	}
+	err = rnbd_clt_set_dev_attr(dev, rsp);
+	if (err)
+		goto out;
+	dev->dev_state = DEV_STATE_MAPPED;
+
+out:
+	mutex_unlock(&dev->lock);
+
+	return err;
+}
+
+int rnbd_clt_resize_disk(struct rnbd_clt_dev *dev, size_t newsize)
+{
+	int ret = 0;
+
+	mutex_lock(&dev->lock);
+	if (dev->dev_state != DEV_STATE_MAPPED) {
+		pr_err("Failed to set new size of the device, device is not opened\n");
+		ret = -ENOENT;
+		goto out;
+	}
+	ret = rnbd_clt_change_capacity(dev, newsize);
+
+out:
+	mutex_unlock(&dev->lock);
+
+	return ret;
+}
+
+static inline void rnbd_clt_dev_requeue(struct rnbd_queue *q)
+{
+	if (WARN_ON(!q->hctx))
+		return;
+
+	/* We can come here from interrupt, thus async=true */
+	blk_mq_run_hw_queue(q->hctx, true);
+}
+
+enum {
+	RNBD_DELAY_IFBUSY = -1,
+};
+
+/**
+ * rnbd_get_cpu_qlist() - finds a list with HW queues to be rerun
+ * @sess:	Session to find a queue for
+ * @cpu:	Cpu to start the search from
+ *
+ * Description:
+ *     Each CPU has a list of HW queues, which needs to be rerun.  If a list
+ *     is not empty - it is marked with a bit.  This function finds first
+ *     set bit in a bitmap and returns corresponding CPU list.
+ */
+static struct rnbd_cpu_qlist *
+rnbd_get_cpu_qlist(struct rnbd_clt_session *sess, int cpu)
+{
+	int bit;
+
+	/* Search from cpu to nr_cpu_ids */
+	bit = find_next_bit(sess->cpu_queues_bm, nr_cpu_ids, cpu);
+	if (bit < nr_cpu_ids) {
+		return per_cpu_ptr(sess->cpu_queues, bit);
+	} else if (cpu != 0) {
+		/* Search from 0 to cpu */
+		bit = find_next_bit(sess->cpu_queues_bm, cpu, 0);
+		if (bit < cpu)
+			return per_cpu_ptr(sess->cpu_queues, bit);
+	}
+
+	return NULL;
+}
+
+static inline int nxt_cpu(int cpu)
+{
+	return (cpu + 1) % nr_cpu_ids;
+}
+
+/**
+ * rnbd_rerun_if_needed() - rerun next queue marked as stopped
+ * @sess:	Session to rerun a queue on
+ *
+ * Description:
+ *     Each CPU has it's own list of HW queues, which should be rerun.
+ *     Function finds such list with HW queues, takes a list lock, picks up
+ *     the first HW queue out of the list and requeues it.
+ *
+ * Return:
+ *     True if the queue was requeued, false otherwise.
+ *
+ * Context:
+ *     Does not matter.
+ */
+static bool rnbd_rerun_if_needed(struct rnbd_clt_session *sess)
+{
+	struct rnbd_queue *q = NULL;
+	struct rnbd_cpu_qlist *cpu_q;
+	unsigned long flags;
+	int *cpup;
+
+	/*
+	 * To keep fairness and not to let other queues starve we always
+	 * try to wake up someone else in round-robin manner.  That of course
+	 * increases latency but queues always have a chance to be executed.
+	 */
+	cpup = get_cpu_ptr(sess->cpu_rr);
+	for (cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(*cpup)); cpu_q;
+	     cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(cpu_q->cpu))) {
+		if (!spin_trylock_irqsave(&cpu_q->requeue_lock, flags))
+			continue;
+		if (unlikely(!test_bit(cpu_q->cpu, sess->cpu_queues_bm)))
+			goto unlock;
+		q = list_first_entry_or_null(&cpu_q->requeue_list,
+					     typeof(*q), requeue_list);
+		if (WARN_ON(!q))
+			goto clear_bit;
+		list_del_init(&q->requeue_list);
+		clear_bit_unlock(0, &q->in_list);
+
+		if (list_empty(&cpu_q->requeue_list)) {
+			/* Clear bit if nothing is left */
+clear_bit:
+			clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
+		}
+unlock:
+		spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);
+
+		if (q)
+			break;
+	}
+
+	/**
+	 * Saves the CPU that is going to be requeued on the per-cpu var. Just
+	 * incrementing it doesn't work because rnbd_get_cpu_qlist() will
+	 * always return the first CPU with something on the queue list when the
+	 * value stored on the var is greater than the last CPU with something
+	 * on the list.
+	 */
+	if (cpu_q)
+		*cpup = cpu_q->cpu;
+	put_cpu_var(sess->cpu_rr);
+
+	if (q)
+		rnbd_clt_dev_requeue(q);
+
+	return q;
+}
+
+/**
+ * rnbd_rerun_all_if_idle() - rerun all queues left in the list if
+ *				 session is idling (there are no requests
+ *				 in-flight).
+ * @sess:	Session to rerun the queues on
+ *
+ * Description:
+ *     This function tries to rerun all stopped queues if there are no
+ *     requests in-flight anymore.  This function tries to solve an obvious
+ *     problem, when number of tags < than number of queues (hctx), which
+ *     are stopped and put to sleep.  If last permit, which has been just put,
+ *     does not wake up all left queues (hctxs), IO requests hang forever.
+ *
+ *     That can happen when all number of permits, say N, have been exhausted
+ *     from one CPU, and we have many block devices per session, say M.
+ *     Each block device has it's own queue (hctx) for each CPU, so eventually
+ *     we can put that number of queues (hctxs) to sleep: M x nr_cpu_ids.
+ *     If number of permits N < M x nr_cpu_ids finally we will get an IO hang.
+ *
+ *     To avoid this hang last caller of rnbd_put_permit() (last caller is the
+ *     one who observes sess->busy == 0) must wake up all remaining queues.
+ *
+ * Context:
+ *     Does not matter.
+ */
+static void rnbd_rerun_all_if_idle(struct rnbd_clt_session *sess)
+{
+	bool requeued;
+
+	do {
+		requeued = rnbd_rerun_if_needed(sess);
+	} while (atomic_read(&sess->busy) == 0 && requeued);
+}
+
+static struct rtrs_permit *rnbd_get_permit(struct rnbd_clt_session *sess,
+					     enum rtrs_clt_con_type con_type,
+					     int wait)
+{
+	struct rtrs_permit *permit;
+
+	permit = rtrs_clt_get_permit(sess->rtrs, con_type,
+				      wait ? RTRS_PERMIT_WAIT :
+				      RTRS_PERMIT_NOWAIT);
+	if (likely(permit))
+		/* We have a subtle rare case here, when all permits can be
+		 * consumed before busy counter increased.  This is safe,
+		 * because loser will get NULL as a permit, observe 0 busy
+		 * counter and immediately restart the queue himself.
+		 */
+		atomic_inc(&sess->busy);
+
+	return permit;
+}
+
+static void rnbd_put_permit(struct rnbd_clt_session *sess,
+			     struct rtrs_permit *permit)
+{
+	rtrs_clt_put_permit(sess->rtrs, permit);
+	atomic_dec(&sess->busy);
+	/* Paired with rnbd_clt_dev_add_to_requeue().  Decrement first
+	 * and then check queue bits.
+	 */
+	smp_mb__after_atomic();
+	rnbd_rerun_all_if_idle(sess);
+}
+
+static struct rnbd_iu *rnbd_get_iu(struct rnbd_clt_session *sess,
+				     enum rtrs_clt_con_type con_type,
+				     int wait)
+{
+	struct rnbd_iu *iu;
+	struct rtrs_permit *permit;
+
+	permit = rnbd_get_permit(sess, con_type,
+				  wait ? RTRS_PERMIT_WAIT :
+				  RTRS_PERMIT_NOWAIT);
+	if (unlikely(!permit))
+		return NULL;
+	iu = rtrs_permit_to_pdu(permit);
+	iu->permit = permit;
+	/*
+	 * 1st reference is dropped after finishing sending a "user" message,
+	 * 2nd reference is dropped after confirmation with the response is
+	 * returned.
+	 * 1st and 2nd can happen in any order, so the rnbd_iu should be
+	 * released (rtrs_permit returned to ibbtrs) only leased after both
+	 * are finished.
+	 */
+	atomic_set(&iu->refcount, 2);
+	init_waitqueue_head(&iu->comp.wait);
+	iu->comp.errno = INT_MAX;
+
+	return iu;
+}
+
+static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu)
+{
+	if (atomic_dec_and_test(&iu->refcount))
+		rnbd_put_permit(sess, iu->permit);
+}
+
+static void rnbd_softirq_done_fn(struct request *rq)
+{
+	struct rnbd_clt_dev *dev	= rq->rq_disk->private_data;
+	struct rnbd_clt_session *sess	= dev->sess;
+	struct rnbd_iu *iu;
+
+	iu = blk_mq_rq_to_pdu(rq);
+	rnbd_put_permit(sess, iu->permit);
+	blk_mq_end_request(rq, errno_to_blk_status(iu->errno));
+}
+
+static void msg_io_conf(void *priv, int errno)
+{
+	struct rnbd_iu *iu = priv;
+	struct rnbd_clt_dev *dev = iu->dev;
+	struct request *rq = iu->rq;
+	int rw = rq_data_dir(rq);
+
+	iu->errno = errno;
+
+	blk_mq_complete_request(rq);
+
+	if (errno)
+		rnbd_clt_info_rl(dev, "%s I/O failed with err: %d\n",
+				 rw == READ ? "read" : "write", errno);
+}
+
+static void wake_up_iu_comp(struct rnbd_iu *iu, int errno)
+{
+	iu->comp.errno = errno;
+	wake_up(&iu->comp.wait);
+}
+
+static void msg_conf(void *priv, int errno)
+{
+	struct rnbd_iu *iu = priv;
+
+	iu->errno = errno;
+	schedule_work(&iu->work);
+}
+
+enum wait_type {
+	NO_WAIT = 0,
+	WAIT    = 1
+};
+
+static int send_usr_msg(struct rtrs_clt *rtrs, int dir,
+			struct rnbd_iu *iu, struct kvec *vec, size_t nr,
+			size_t len, struct scatterlist *sg, unsigned int sg_len,
+			void (*conf)(struct work_struct *work),
+			int *errno, enum wait_type wait)
+{
+	int err;
+	struct rtrs_clt_req_ops req_ops;
+
+	INIT_WORK(&iu->work, conf);
+	req_ops = (struct rtrs_clt_req_ops) {
+		.priv = iu,
+		.conf_fn = msg_conf,
+	};
+	err = rtrs_clt_request(dir, &req_ops, rtrs, iu->permit,
+				vec, nr, len, sg, sg_len);
+	if (!err && wait) {
+		wait_event(iu->comp.wait, iu->comp.errno != INT_MAX);
+		*errno = iu->comp.errno;
+	} else {
+		*errno = 0;
+	}
+
+	return err;
+}
+
+static void msg_close_conf(struct work_struct *work)
+{
+	struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
+	struct rnbd_clt_dev *dev = iu->dev;
+
+	wake_up_iu_comp(iu, iu->errno);
+	rnbd_put_iu(dev->sess, iu);
+	rnbd_clt_put_dev(dev);
+}
+
+static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id, bool wait)
+{
+	struct rnbd_clt_session *sess = dev->sess;
+	struct rnbd_msg_close msg;
+	struct rnbd_iu *iu;
+	struct kvec vec = {
+		.iov_base = &msg,
+		.iov_len  = sizeof(msg)
+	};
+	int err, errno;
+
+	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
+	if (!iu)
+		return -ENOMEM;
+
+	iu->buf = NULL;
+	iu->dev = dev;
+
+	sg_mark_end(&iu->sglist[0]);
+
+	msg.hdr.type	= cpu_to_le16(RNBD_MSG_CLOSE);
+	msg.device_id	= cpu_to_le32(device_id);
+
+	WARN_ON(!rnbd_clt_get_dev(dev));
+	err = send_usr_msg(sess->rtrs, WRITE, iu, &vec, 1, 0, NULL, 0,
+			   msg_close_conf, &errno, wait);
+	if (err) {
+		rnbd_clt_put_dev(dev);
+		rnbd_put_iu(sess, iu);
+	} else {
+		err = errno;
+	}
+
+	rnbd_put_iu(sess, iu);
+	return err;
+}
+
+static void msg_open_conf(struct work_struct *work)
+{
+	struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
+	struct rnbd_msg_open_rsp *rsp = iu->buf;
+	struct rnbd_clt_dev *dev = iu->dev;
+	int errno = iu->errno;
+
+	if (errno) {
+		rnbd_clt_err(dev,
+			      "Opening failed, server responded: %d\n",
+			      errno);
+	} else {
+		errno = process_msg_open_rsp(dev, rsp);
+		if (errno) {
+			u32 device_id = le32_to_cpu(rsp->device_id);
+			/*
+			 * If server thinks its fine, but we fail to process
+			 * then be nice and send a close to server.
+			 */
+			(void)send_msg_close(dev, device_id, NO_WAIT);
+		}
+	}
+	kfree(rsp);
+	wake_up_iu_comp(iu, errno);
+	rnbd_put_iu(dev->sess, iu);
+	rnbd_clt_put_dev(dev);
+}
+
+static void msg_sess_info_conf(struct work_struct *work)
+{
+	struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
+	struct rnbd_msg_sess_info_rsp *rsp = iu->buf;
+	struct rnbd_clt_session *sess = iu->sess;
+
+	if (!iu->errno)
+		sess->ver = min_t(u8, rsp->ver, RNBD_PROTO_VER_MAJOR);
+
+	kfree(rsp);
+	wake_up_iu_comp(iu, iu->errno);
+	rnbd_put_iu(sess, iu);
+	rnbd_clt_put_sess(sess);
+}
+
+static int send_msg_open(struct rnbd_clt_dev *dev, bool wait)
+{
+	struct rnbd_clt_session *sess = dev->sess;
+	struct rnbd_msg_open_rsp *rsp;
+	struct rnbd_msg_open msg;
+	struct rnbd_iu *iu;
+	struct kvec vec = {
+		.iov_base = &msg,
+		.iov_len  = sizeof(msg)
+	};
+	int err, errno;
+
+	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
+	if (!rsp)
+		return -ENOMEM;
+
+	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
+	if (!iu) {
+		kfree(rsp);
+		return -ENOMEM;
+	}
+
+	iu->buf = rsp;
+	iu->dev = dev;
+
+	sg_init_one(iu->sglist, rsp, sizeof(*rsp));
+
+	msg.hdr.type	= cpu_to_le16(RNBD_MSG_OPEN);
+	msg.access_mode	= dev->access_mode;
+	strlcpy(msg.dev_name, dev->pathname, sizeof(msg.dev_name));
+
+	WARN_ON(!rnbd_clt_get_dev(dev));
+	err = send_usr_msg(sess->rtrs, READ, iu,
+			   &vec, 1, sizeof(*rsp), iu->sglist, 1,
+			   msg_open_conf, &errno, wait);
+	if (err) {
+		rnbd_clt_put_dev(dev);
+		rnbd_put_iu(sess, iu);
+		kfree(rsp);
+	} else {
+		err = errno;
+	}
+
+	rnbd_put_iu(sess, iu);
+	return err;
+}
+
+static int send_msg_sess_info(struct rnbd_clt_session *sess, bool wait)
+{
+	struct rnbd_msg_sess_info_rsp *rsp;
+	struct rnbd_msg_sess_info msg;
+	struct rnbd_iu *iu;
+	struct kvec vec = {
+		.iov_base = &msg,
+		.iov_len  = sizeof(msg)
+	};
+	int err, errno;
+
+	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
+	if (!rsp)
+		return -ENOMEM;
+
+	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
+	if (!iu) {
+		kfree(rsp);
+		return -ENOMEM;
+	}
+
+	iu->buf = rsp;
+	iu->sess = sess;
+
+	sg_init_one(iu->sglist, rsp, sizeof(*rsp));
+
+	msg.hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO);
+	msg.ver      = RNBD_PROTO_VER_MAJOR;
+
+	if (!rnbd_clt_get_sess(sess)) {
+		/*
+		 * That can happen only in one case, when RTRS has restablished
+		 * the connection and link_ev() is called, but session is almost
+		 * dead, last reference on session is put and caller is waiting
+		 * for RTRS to close everything.
+		 */
+		err = -ENODEV;
+		goto put_iu;
+	}
+	err = send_usr_msg(sess->rtrs, READ, iu,
+			   &vec, 1, sizeof(*rsp), iu->sglist, 1,
+			   msg_sess_info_conf, &errno, wait);
+	if (err) {
+		rnbd_clt_put_sess(sess);
+put_iu:
+		rnbd_put_iu(sess, iu);
+		kfree(rsp);
+	} else {
+		err = errno;
+	}
+
+	rnbd_put_iu(sess, iu);
+	return err;
+}
+
+static void set_dev_states_to_disconnected(struct rnbd_clt_session *sess)
+{
+	struct rnbd_clt_dev *dev;
+
+	mutex_lock(&sess->lock);
+	list_for_each_entry(dev, &sess->devs_list, list) {
+		rnbd_clt_err(dev, "Device disconnected.\n");
+
+		mutex_lock(&dev->lock);
+		if (dev->dev_state == DEV_STATE_MAPPED)
+			dev->dev_state = DEV_STATE_MAPPED_DISCONNECTED;
+		mutex_unlock(&dev->lock);
+	}
+	mutex_unlock(&sess->lock);
+}
+
+static void remap_devs(struct rnbd_clt_session *sess)
+{
+	struct rnbd_clt_dev *dev;
+	struct rtrs_attrs attrs;
+	int err;
+
+	/*
+	 * Careful here: we are called from RTRS link event directly,
+	 * thus we can't send any RTRS request and wait for response
+	 * or RTRS will not be able to complete request with failure
+	 * if something goes wrong (failing of outstanding requests
+	 * happens exactly from the context where we are blocking now).
+	 *
+	 * So to avoid deadlocks each usr message sent from here must
+	 * be asynchronous.
+	 */
+
+	err = send_msg_sess_info(sess, NO_WAIT);
+	if (err) {
+		pr_err("send_msg_sess_info(\"%s\"): %d\n", sess->sessname, err);
+		return;
+	}
+
+	rtrs_clt_query(sess->rtrs, &attrs);
+	mutex_lock(&sess->lock);
+	sess->max_io_size = attrs.max_io_size;
+
+	list_for_each_entry(dev, &sess->devs_list, list) {
+		bool skip;
+
+		mutex_lock(&dev->lock);
+		skip = (dev->dev_state == DEV_STATE_INIT);
+		mutex_unlock(&dev->lock);
+		if (skip)
+			/*
+			 * When device is establishing connection for the first
+			 * time - do not remap, it will be closed soon.
+			 */
+			continue;
+
+		rnbd_clt_info(dev, "session reconnected, remapping device\n");
+		err = send_msg_open(dev, NO_WAIT);
+		if (err) {
+			rnbd_clt_err(dev, "send_msg_open(): %d\n", err);
+			break;
+		}
+	}
+	mutex_unlock(&sess->lock);
+}
+
+static void rnbd_clt_link_ev(void *priv, enum rtrs_clt_link_ev ev)
+{
+	struct rnbd_clt_session *sess = priv;
+
+	switch (ev) {
+	case RTRS_CLT_LINK_EV_DISCONNECTED:
+		set_dev_states_to_disconnected(sess);
+		break;
+	case RTRS_CLT_LINK_EV_RECONNECTED:
+		remap_devs(sess);
+		break;
+	default:
+		pr_err("Unknown session event received (%d), session: %s\n",
+		       ev, sess->sessname);
+	}
+}
+
+static void rnbd_init_cpu_qlists(struct rnbd_cpu_qlist __percpu *cpu_queues)
+{
+	unsigned int cpu;
+	struct rnbd_cpu_qlist *cpu_q;
+
+	for_each_possible_cpu(cpu) {
+		cpu_q = per_cpu_ptr(cpu_queues, cpu);
+
+		cpu_q->cpu = cpu;
+		INIT_LIST_HEAD(&cpu_q->requeue_list);
+		spin_lock_init(&cpu_q->requeue_lock);
+	}
+}
+
+static void destroy_mq_tags(struct rnbd_clt_session *sess)
+{
+	if (sess->tag_set.tags)
+		blk_mq_free_tag_set(&sess->tag_set);
+}
+
+static inline void wake_up_rtrs_waiters(struct rnbd_clt_session *sess)
+{
+	sess->rtrs_ready = true;
+	wake_up_all(&sess->rtrs_waitq);
+}
+
+static void close_rtrs(struct rnbd_clt_session *sess)
+{
+	might_sleep();
+
+	if (!IS_ERR_OR_NULL(sess->rtrs)) {
+		rtrs_clt_close(sess->rtrs);
+		sess->rtrs = NULL;
+		wake_up_rtrs_waiters(sess);
+	}
+}
+
+static void free_sess(struct rnbd_clt_session *sess)
+{
+	WARN_ON(!list_empty(&sess->devs_list));
+
+	might_sleep();
+
+	close_rtrs(sess);
+	destroy_mq_tags(sess);
+	if (!list_empty(&sess->list)) {
+		mutex_lock(&sess_lock);
+		list_del(&sess->list);
+		mutex_unlock(&sess_lock);
+	}
+	free_percpu(sess->cpu_queues);
+	free_percpu(sess->cpu_rr);
+	mutex_destroy(&sess->lock);
+	kfree(sess);
+}
+
+static struct rnbd_clt_session *alloc_sess(const char *sessname)
+{
+	struct rnbd_clt_session *sess;
+	int err, cpu;
+
+	sess = kzalloc_node(sizeof(*sess), GFP_KERNEL, NUMA_NO_NODE);
+	if (!sess)
+		return ERR_PTR(-ENOMEM);
+	strlcpy(sess->sessname, sessname, sizeof(sess->sessname));
+	atomic_set(&sess->busy, 0);
+	mutex_init(&sess->lock);
+	INIT_LIST_HEAD(&sess->devs_list);
+	INIT_LIST_HEAD(&sess->list);
+	bitmap_zero(sess->cpu_queues_bm, NR_CPUS);
+	init_waitqueue_head(&sess->rtrs_waitq);
+	refcount_set(&sess->refcount, 1);
+
+	sess->cpu_queues = alloc_percpu(struct rnbd_cpu_qlist);
+	if (!sess->cpu_queues) {
+		err = -ENOMEM;
+		goto err;
+	}
+	rnbd_init_cpu_qlists(sess->cpu_queues);
+
+	/*
+	 * That is simple percpu variable which stores cpu indeces, which are
+	 * incremented on each access.  We need that for the sake of fairness
+	 * to wake up queues in a round-robin manner.
+	 */
+	sess->cpu_rr = alloc_percpu(int);
+	if (!sess->cpu_rr) {
+		err = -ENOMEM;
+		goto err;
+	}
+	for_each_possible_cpu(cpu)
+		* per_cpu_ptr(sess->cpu_rr, cpu) = cpu;
+
+	return sess;
+
+err:
+	free_sess(sess);
+
+	return ERR_PTR(err);
+}
+
+static int wait_for_rtrs_connection(struct rnbd_clt_session *sess)
+{
+	wait_event(sess->rtrs_waitq, sess->rtrs_ready);
+	if (IS_ERR_OR_NULL(sess->rtrs))
+		return -ECONNRESET;
+
+	return 0;
+}
+
+static void wait_for_rtrs_disconnection(struct rnbd_clt_session *sess)
+	__releases(&sess_lock)
+	__acquires(&sess_lock)
+{
+	DEFINE_WAIT(wait);
+
+	prepare_to_wait(&sess->rtrs_waitq, &wait, TASK_UNINTERRUPTIBLE);
+	if (IS_ERR_OR_NULL(sess->rtrs)) {
+		finish_wait(&sess->rtrs_waitq, &wait);
+		return;
+	}
+	mutex_unlock(&sess_lock);
+	/* loop in caller, see __find_and_get_sess().
+	 * You can't leave mutex locked and call schedule(), you will catch a
+	 * deadlock with a caller of free_sess(), which has just put the last
+	 * reference and is about to take the sess_lock in order to delete
+	 * the session from the list.
+	 */
+	schedule();
+	mutex_lock(&sess_lock);
+}
+
+static struct rnbd_clt_session *__find_and_get_sess(const char *sessname)
+	__releases(&sess_lock)
+	__acquires(&sess_lock)
+{
+	struct rnbd_clt_session *sess, *sn;
+	int err;
+
+again:
+	list_for_each_entry_safe(sess, sn, &sess_list, list) {
+		if (strcmp(sessname, sess->sessname))
+			continue;
+
+		if (sess->rtrs_ready && IS_ERR_OR_NULL(sess->rtrs))
+			/*
+			 * No RTRS connection, session is dying.
+			 */
+			continue;
+
+		if (rnbd_clt_get_sess(sess)) {
+			/*
+			 * Alive session is found, wait for RTRS connection.
+			 */
+			mutex_unlock(&sess_lock);
+			err = wait_for_rtrs_connection(sess);
+			if (err)
+				rnbd_clt_put_sess(sess);
+			mutex_lock(&sess_lock);
+
+			if (err)
+				/* Session is dying, repeat the loop */
+				goto again;
+
+			return sess;
+		}
+		/*
+		 * Ref is 0, session is dying, wait for RTRS disconnect
+		 * in order to avoid session names clashes.
+		 */
+		wait_for_rtrs_disconnection(sess);
+		/*
+		 * RTRS is disconnected and soon session will be freed,
+		 * so repeat a loop.
+		 */
+		goto again;
+	}
+
+	return NULL;
+}
+
+static struct
+rnbd_clt_session *find_or_create_sess(const char *sessname, bool *first)
+{
+	struct rnbd_clt_session *sess = NULL;
+
+	mutex_lock(&sess_lock);
+	sess = __find_and_get_sess(sessname);
+	if (!sess) {
+		sess = alloc_sess(sessname);
+		if (IS_ERR(sess)) {
+			mutex_unlock(&sess_lock);
+			return sess;
+		}
+		list_add(&sess->list, &sess_list);
+		*first = true;
+	} else
+		*first = false;
+	mutex_unlock(&sess_lock);
+
+	return sess;
+}
+
+static int rnbd_client_open(struct block_device *block_device, fmode_t mode)
+{
+	struct rnbd_clt_dev *dev = block_device->bd_disk->private_data;
+
+	if (dev->read_only && (mode & FMODE_WRITE))
+		return -EPERM;
+
+	if (dev->dev_state == DEV_STATE_UNMAPPED ||
+	    !rnbd_clt_get_dev(dev))
+		return -EIO;
+
+	return 0;
+}
+
+static void rnbd_client_release(struct gendisk *gen, fmode_t mode)
+{
+	struct rnbd_clt_dev *dev = gen->private_data;
+
+	rnbd_clt_put_dev(dev);
+}
+
+static int rnbd_client_getgeo(struct block_device *block_device,
+			      struct hd_geometry *geo)
+{
+	u64 size;
+	struct rnbd_clt_dev *dev;
+
+	dev = block_device->bd_disk->private_data;
+	size = dev->size * (dev->logical_block_size / SECTOR_SIZE);
+	geo->cylinders	= size >> 6;	/* size/64 */
+	geo->heads	= 4;
+	geo->sectors	= 16;
+	geo->start	= 0;
+
+	return 0;
+}
+
+static const struct block_device_operations rnbd_client_ops = {
+	.owner		= THIS_MODULE,
+	.open		= rnbd_client_open,
+	.release	= rnbd_client_release,
+	.getgeo		= rnbd_client_getgeo
+};
+
+/* The amount of data that belongs to an I/O and the amount of data that
+ * should be read or written to the disk (bi_size) can differ.
+ *
+ * E.g. When WRITE_SAME is used, only a small amount of data is
+ * transferred that is then written repeatedly over a lot of sectors.
+ *
+ * Get the size of data to be transferred via RTRS by summing up the size
+ * of the scather-gather list entries.
+ */
+static size_t rnbd_clt_get_sg_size(struct scatterlist *sglist, u32 len)
+{
+	struct scatterlist *sg;
+	size_t tsize = 0;
+	int i;
+
+	for_each_sg(sglist, sg, len, i)
+		tsize += sg->length;
+	return tsize;
+}
+
+static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev,
+				     struct request *rq,
+				     struct rnbd_iu *iu)
+{
+	struct rtrs_clt *rtrs = dev->sess->rtrs;
+	struct rtrs_permit *permit = iu->permit;
+	struct rnbd_msg_io msg;
+	struct rtrs_clt_req_ops req_ops;
+	unsigned int sg_cnt = 0;
+	struct kvec vec;
+	size_t size;
+	int err;
+
+	iu->rq		= rq;
+	iu->dev		= dev;
+	msg.sector	= cpu_to_le64(blk_rq_pos(rq));
+	msg.bi_size	= cpu_to_le32(blk_rq_bytes(rq));
+	msg.rw		= cpu_to_le32(rq_to_rnbd_flags(rq));
+	msg.prio	= cpu_to_le16(req_get_ioprio(rq));
+
+	/*
+	 * We only support discards with single segment for now.
+	 * See queue limits.
+	 */
+	if (req_op(rq) != REQ_OP_DISCARD)
+		sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sglist);
+
+	if (sg_cnt == 0)
+		/* Do not forget to mark the end */
+		sg_mark_end(&iu->sglist[0]);
+
+	msg.hdr.type	= cpu_to_le16(RNBD_MSG_IO);
+	msg.device_id	= cpu_to_le32(dev->device_id);
+
+	vec = (struct kvec) {
+		.iov_base = &msg,
+		.iov_len  = sizeof(msg)
+	};
+	size = rnbd_clt_get_sg_size(iu->sglist, sg_cnt);
+	req_ops = (struct rtrs_clt_req_ops) {
+		.priv = iu,
+		.conf_fn = msg_io_conf,
+	};
+	err = rtrs_clt_request(rq_data_dir(rq), &req_ops, rtrs, permit,
+			       &vec, 1, size, iu->sglist, sg_cnt);
+	if (unlikely(err)) {
+		rnbd_clt_err_rl(dev, "RTRS failed to transfer IO, err: %d\n",
+				 err);
+		return err;
+	}
+
+	return 0;
+}
+
+/**
+ * rnbd_clt_dev_add_to_requeue() - add device to requeue if session is busy
+ * @dev:	Device to be checked
+ * @q:		Queue to be added to the requeue list if required
+ *
+ * Description:
+ *     If session is busy, that means someone will requeue us when resources
+ *     are freed.  If session is not doing anything - device is not added to
+ *     the list and @false is returned.
+ */
+static bool rnbd_clt_dev_add_to_requeue(struct rnbd_clt_dev *dev,
+						struct rnbd_queue *q)
+{
+	struct rnbd_clt_session *sess = dev->sess;
+	struct rnbd_cpu_qlist *cpu_q;
+	unsigned long flags;
+	bool added = true;
+	bool need_set;
+
+	cpu_q = get_cpu_ptr(sess->cpu_queues);
+	spin_lock_irqsave(&cpu_q->requeue_lock, flags);
+
+	if (likely(!test_and_set_bit_lock(0, &q->in_list))) {
+		if (WARN_ON(!list_empty(&q->requeue_list)))
+			goto unlock;
+
+		need_set = !test_bit(cpu_q->cpu, sess->cpu_queues_bm);
+		if (need_set) {
+			set_bit(cpu_q->cpu, sess->cpu_queues_bm);
+			/* Paired with rnbd_put_permit(). Set a bit first
+			 * and then observe the busy counter.
+			 */
+			smp_mb__before_atomic();
+		}
+		if (likely(atomic_read(&sess->busy))) {
+			list_add_tail(&q->requeue_list, &cpu_q->requeue_list);
+		} else {
+			/* Very unlikely, but possible: busy counter was
+			 * observed as zero.  Drop all bits and return
+			 * false to restart the queue by ourselves.
+			 */
+			if (need_set)
+				clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
+			clear_bit_unlock(0, &q->in_list);
+			added = false;
+		}
+	}
+unlock:
+	spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);
+	put_cpu_ptr(sess->cpu_queues);
+
+	return added;
+}
+
+static void rnbd_clt_dev_kick_mq_queue(struct rnbd_clt_dev *dev,
+					struct blk_mq_hw_ctx *hctx,
+					int delay)
+{
+	struct rnbd_queue *q = hctx->driver_data;
+
+	if (delay != RNBD_DELAY_IFBUSY)
+		blk_mq_delay_run_hw_queue(hctx, delay);
+	else if (unlikely(!rnbd_clt_dev_add_to_requeue(dev, q)))
+		/*
+		 * If session is not busy we have to restart
+		 * the queue ourselves.
+		 */
+		blk_mq_delay_run_hw_queue(hctx, 10/*ms*/);
+}
+
+static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
+				   const struct blk_mq_queue_data *bd)
+{
+	struct request *rq = bd->rq;
+	struct rnbd_clt_dev *dev = rq->rq_disk->private_data;
+	struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
+	int err;
+
+	if (unlikely(dev->dev_state != DEV_STATE_MAPPED))
+		return BLK_STS_IOERR;
+
+	iu->permit = rnbd_get_permit(dev->sess, RTRS_IO_CON,
+				      RTRS_PERMIT_NOWAIT);
+	if (unlikely(!iu->permit)) {
+		rnbd_clt_dev_kick_mq_queue(dev, hctx, RNBD_DELAY_IFBUSY);
+		return BLK_STS_RESOURCE;
+	}
+
+	blk_mq_start_request(rq);
+	err = rnbd_client_xfer_request(dev, rq, iu);
+	if (likely(err == 0))
+		return BLK_STS_OK;
+	if (unlikely(err == -EAGAIN || err == -ENOMEM)) {
+		rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/);
+		rnbd_put_permit(dev->sess, iu->permit);
+		return BLK_STS_RESOURCE;
+	}
+
+	rnbd_put_permit(dev->sess, iu->permit);
+	return BLK_STS_IOERR;
+}
+
+static int rnbd_init_request(struct blk_mq_tag_set *set, struct request *rq,
+			      unsigned int hctx_idx, unsigned int numa_node)
+{
+	struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
+
+	sg_init_table(iu->sglist, BMAX_SEGMENTS);
+	return 0;
+}
+
+static struct blk_mq_ops rnbd_mq_ops = {
+	.queue_rq	= rnbd_queue_rq,
+	.init_request	= rnbd_init_request,
+	.complete	= rnbd_softirq_done_fn,
+};
+
+static int setup_mq_tags(struct rnbd_clt_session *sess)
+{
+	struct blk_mq_tag_set *tag_set = &sess->tag_set;
+
+	memset(tag_set, 0, sizeof(*tag_set));
+	tag_set->ops		= &rnbd_mq_ops;
+	tag_set->queue_depth	= sess->queue_depth;
+	tag_set->numa_node		= NUMA_NO_NODE;
+	tag_set->flags		= BLK_MQ_F_SHOULD_MERGE |
+				  BLK_MQ_F_TAG_SHARED;
+	tag_set->cmd_size		= sizeof(struct rnbd_iu);
+	tag_set->nr_hw_queues	= num_online_cpus();
+
+	return blk_mq_alloc_tag_set(tag_set);
+}
+
+static struct rnbd_clt_session *
+find_and_get_or_create_sess(const char *sessname,
+			    const struct rtrs_addr *paths,
+			    size_t path_cnt, u16 port_nr)
+{
+	struct rnbd_clt_session *sess;
+	struct rtrs_attrs attrs;
+	int err;
+	bool first;
+	struct rtrs_clt_ops rtrs_ops;
+
+	sess = find_or_create_sess(sessname, &first);
+	if (sess == ERR_PTR(-ENOMEM))
+		return ERR_PTR(-ENOMEM);
+	else if (!first)
+		return sess;
+
+	rtrs_ops = (struct rtrs_clt_ops) {
+		.priv = sess,
+		.link_ev = rnbd_clt_link_ev,
+	};
+	/*
+	 * Nothing was found, establish rtrs connection and proceed further.
+	 */
+	sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname,
+				   paths, path_cnt, port_nr,
+				   sizeof(struct rnbd_iu),
+				   RECONNECT_DELAY, BMAX_SEGMENTS,
+				   BLK_MAX_SEGMENT_SIZE,
+				   MAX_RECONNECTS);
+	if (IS_ERR(sess->rtrs)) {
+		err = PTR_ERR(sess->rtrs);
+		goto wake_up_and_put;
+	}
+	rtrs_clt_query(sess->rtrs, &attrs);
+	sess->max_io_size = attrs.max_io_size;
+	sess->queue_depth = attrs.queue_depth;
+
+	err = setup_mq_tags(sess);
+	if (err)
+		goto close_rtrs;
+
+	err = send_msg_sess_info(sess, WAIT);
+	if (err)
+		goto close_rtrs;
+
+	wake_up_rtrs_waiters(sess);
+
+	return sess;
+
+close_rtrs:
+	close_rtrs(sess);
+put_sess:
+	rnbd_clt_put_sess(sess);
+
+	return ERR_PTR(err);
+
+wake_up_and_put:
+	wake_up_rtrs_waiters(sess);
+	goto put_sess;
+}
+
+static inline void rnbd_init_hw_queue(struct rnbd_clt_dev *dev,
+				       struct rnbd_queue *q,
+				       struct blk_mq_hw_ctx *hctx)
+{
+	INIT_LIST_HEAD(&q->requeue_list);
+	q->dev  = dev;
+	q->hctx = hctx;
+}
+
+static void rnbd_init_mq_hw_queues(struct rnbd_clt_dev *dev)
+{
+	int i;
+	struct blk_mq_hw_ctx *hctx;
+	struct rnbd_queue *q;
+
+	queue_for_each_hw_ctx(dev->queue, hctx, i) {
+		q = &dev->hw_queues[i];
+		rnbd_init_hw_queue(dev, q, hctx);
+		hctx->driver_data = q;
+	}
+}
+
+static int setup_mq_dev(struct rnbd_clt_dev *dev)
+{
+	dev->queue = blk_mq_init_queue(&dev->sess->tag_set);
+	if (IS_ERR(dev->queue)) {
+		rnbd_clt_err(dev, "Initializing multiqueue queue failed, err: %ld\n",
+			      PTR_ERR(dev->queue));
+		return PTR_ERR(dev->queue);
+	}
+	rnbd_init_mq_hw_queues(dev);
+	return 0;
+}
+
+static void setup_request_queue(struct rnbd_clt_dev *dev)
+{
+	blk_queue_logical_block_size(dev->queue, dev->logical_block_size);
+	blk_queue_physical_block_size(dev->queue, dev->physical_block_size);
+	blk_queue_max_hw_sectors(dev->queue, dev->max_hw_sectors);
+	blk_queue_max_write_same_sectors(dev->queue,
+					 dev->max_write_same_sectors);
+
+	/*
+	 * we don't support discards to "discontiguous" segments
+	 * in on request
+	 */
+	blk_queue_max_discard_segments(dev->queue, 1);
+
+	blk_queue_max_discard_sectors(dev->queue, dev->max_discard_sectors);
+	dev->queue->limits.discard_granularity	= dev->discard_granularity;
+	dev->queue->limits.discard_alignment	= dev->discard_alignment;
+	if (dev->max_discard_sectors)
+		blk_queue_flag_set(QUEUE_FLAG_DISCARD, dev->queue);
+	if (dev->secure_discard)
+		blk_queue_flag_set(QUEUE_FLAG_SECERASE, dev->queue);
+
+	blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, dev->queue);
+	blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, dev->queue);
+	blk_queue_max_segments(dev->queue, dev->max_segments);
+	blk_queue_io_opt(dev->queue, dev->sess->max_io_size);
+	blk_queue_virt_boundary(dev->queue, SZ_4K - 1);
+	blk_queue_write_cache(dev->queue, true, true);
+	dev->queue->queuedata = dev;
+}
+
+static void rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx)
+{
+	dev->gd->major		= rnbd_client_major;
+	dev->gd->first_minor	= idx << RNBD_PART_BITS;
+	dev->gd->fops		= &rnbd_client_ops;
+	dev->gd->queue		= dev->queue;
+	dev->gd->private_data	= dev;
+	snprintf(dev->gd->disk_name, sizeof(dev->gd->disk_name), "rnbd%d",
+		 idx);
+	pr_debug("disk_name=%s, capacity=%zu\n",
+		 dev->gd->disk_name,
+		 dev->nsectors * (dev->logical_block_size / SECTOR_SIZE)
+		 );
+
+	set_capacity(dev->gd, dev->nsectors);
+
+	if (dev->access_mode == RNBD_ACCESS_RO) {
+		dev->read_only = true;
+		set_disk_ro(dev->gd, true);
+	} else {
+		dev->read_only = false;
+	}
+
+	if (!dev->rotational)
+		blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue);
+}
+
+static int rnbd_client_setup_device(struct rnbd_clt_session *sess,
+				     struct rnbd_clt_dev *dev, int idx)
+{
+	int err;
+
+	dev->size = dev->nsectors * dev->logical_block_size;
+
+	err = setup_mq_dev(dev);
+	if (err)
+		return err;
+
+	setup_request_queue(dev);
+
+	dev->gd = alloc_disk_node(1 << RNBD_PART_BITS,	NUMA_NO_NODE);
+	if (!dev->gd) {
+		blk_cleanup_queue(dev->queue);
+		return -ENOMEM;
+	}
+
+	rnbd_clt_setup_gen_disk(dev, idx);
+
+	return 0;
+}
+
+static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,
+				      enum rnbd_access_mode access_mode,
+				      const char *pathname)
+{
+	struct rnbd_clt_dev *dev;
+	int ret;
+
+	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, NUMA_NO_NODE);
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	dev->hw_queues = kcalloc(nr_cpu_ids, sizeof(*dev->hw_queues),
+				 GFP_KERNEL);
+	if (!dev->hw_queues) {
+		ret = -ENOMEM;
+		goto out_alloc;
+	}
+
+	mutex_lock(&ida_lock);
+	ret = ida_simple_get(&index_ida, 0, 1 << (MINORBITS - RNBD_PART_BITS),
+			     GFP_KERNEL);
+	mutex_unlock(&ida_lock);
+	if (ret < 0) {
+		pr_err("Failed to initialize device '%s' from session %s, allocating idr failed, err: %d\n",
+		       pathname, sess->sessname, ret);
+		goto out_queues;
+	}
+	dev->clt_device_id	= ret;
+	dev->sess		= sess;
+	dev->access_mode	= access_mode;
+	strlcpy(dev->pathname, pathname, sizeof(dev->pathname));
+	mutex_init(&dev->lock);
+	refcount_set(&dev->refcount, 1);
+	dev->dev_state = DEV_STATE_INIT;
+
+	/*
+	 * Here we called from sysfs entry, thus clt-sysfs is
+	 * responsible that session will not disappear.
+	 */
+	WARN_ON(!rnbd_clt_get_sess(sess));
+
+	return dev;
+
+out_queues:
+	kfree(dev->hw_queues);
+out_alloc:
+	kfree(dev);
+	return ERR_PTR(ret);
+}
+
+static bool __exists_dev(const char *pathname)
+{
+	struct rnbd_clt_session *sess;
+	struct rnbd_clt_dev *dev;
+	bool found = false;
+
+	list_for_each_entry(sess, &sess_list, list) {
+		mutex_lock(&sess->lock);
+		list_for_each_entry(dev, &sess->devs_list, list) {
+			if (!strncmp(dev->pathname, pathname,
+				     sizeof(dev->pathname))) {
+				found = true;
+				break;
+			}
+		}
+		mutex_unlock(&sess->lock);
+		if (found)
+			break;
+	}
+
+	return found;
+}
+
+static bool exists_devpath(const char *pathname)
+{
+	bool found;
+
+	mutex_lock(&sess_lock);
+	found = __exists_dev(pathname);
+	mutex_unlock(&sess_lock);
+
+	return found;
+}
+
+static bool insert_dev_if_not_exists_devpath(const char *pathname,
+					     struct rnbd_clt_session *sess,
+					     struct rnbd_clt_dev *dev)
+{
+	bool found;
+
+	mutex_lock(&sess_lock);
+	found = __exists_dev(pathname);
+	if (!found) {
+		mutex_lock(&sess->lock);
+		list_add_tail(&dev->list, &sess->devs_list);
+		mutex_unlock(&sess->lock);
+	}
+	mutex_unlock(&sess_lock);
+
+	return found;
+}
+
+static void delete_dev(struct rnbd_clt_dev *dev)
+{
+	struct rnbd_clt_session *sess = dev->sess;
+
+	mutex_lock(&sess->lock);
+	list_del(&dev->list);
+	mutex_unlock(&sess->lock);
+}
+
+struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname,
+					   struct rtrs_addr *paths,
+					   size_t path_cnt, u16 port_nr,
+					   const char *pathname,
+					   enum rnbd_access_mode access_mode)
+{
+	struct rnbd_clt_session *sess;
+	struct rnbd_clt_dev *dev;
+	int ret;
+
+	if (exists_devpath(pathname))
+		return ERR_PTR(-EEXIST);
+
+	sess = find_and_get_or_create_sess(sessname, paths, path_cnt, port_nr);
+	if (IS_ERR(sess))
+		return ERR_CAST(sess);
+
+	dev = init_dev(sess, access_mode, pathname);
+	if (IS_ERR(dev)) {
+		pr_err("map_device: failed to map device '%s' from session %s, can't initialize device, err: %ld\n",
+		       pathname, sess->sessname, PTR_ERR(dev));
+		ret = PTR_ERR(dev);
+		goto put_sess;
+	}
+	if (insert_dev_if_not_exists_devpath(pathname, sess, dev)) {
+		ret = -EEXIST;
+		goto put_dev;
+	}
+	ret = send_msg_open(dev, WAIT);
+	if (ret) {
+		rnbd_clt_err(dev,
+			      "map_device: failed, can't open remote device, err: %d\n",
+			      ret);
+		goto del_dev;
+	}
+	mutex_lock(&dev->lock);
+	pr_debug("Opened remote device: session=%s, path='%s'\n",
+		 sess->sessname, pathname);
+	ret = rnbd_client_setup_device(sess, dev, dev->clt_device_id);
+	if (ret) {
+		rnbd_clt_err(dev,
+			      "map_device: Failed to configure device, err: %d\n",
+			      ret);
+		mutex_unlock(&dev->lock);
+		goto del_dev;
+	}
+
+	rnbd_clt_info(dev,
+		       "map_device: Device mapped as %s (nsectors: %zu, logical_block_size: %d, physical_block_size: %d, max_write_same_sectors: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, rotational: %d)\n",
+		       dev->gd->disk_name, dev->nsectors,
+		       dev->logical_block_size, dev->physical_block_size,
+		       dev->max_write_same_sectors, dev->max_discard_sectors,
+		       dev->discard_granularity, dev->discard_alignment,
+		       dev->secure_discard, dev->max_segments,
+		       dev->max_hw_sectors, dev->rotational);
+
+	mutex_unlock(&dev->lock);
+
+	add_disk(dev->gd);
+	rnbd_clt_put_sess(sess);
+
+	return dev;
+
+del_dev:
+	delete_dev(dev);
+put_dev:
+	rnbd_clt_put_dev(dev);
+put_sess:
+	rnbd_clt_put_sess(sess);
+
+	return ERR_PTR(ret);
+}
+
+static void destroy_gen_disk(struct rnbd_clt_dev *dev)
+{
+	del_gendisk(dev->gd);
+	blk_cleanup_queue(dev->queue);
+	put_disk(dev->gd);
+}
+
+static void destroy_sysfs(struct rnbd_clt_dev *dev,
+			  const struct attribute *sysfs_self)
+{
+	rnbd_clt_remove_dev_symlink(dev);
+	if (dev->kobj.state_initialized) {
+		if (sysfs_self)
+			/* To avoid deadlock firstly remove itself */
+			sysfs_remove_file_self(&dev->kobj, sysfs_self);
+		kobject_del(&dev->kobj);
+		kobject_put(&dev->kobj);
+	}
+}
+
+int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force,
+			   const struct attribute *sysfs_self)
+{
+	struct rnbd_clt_session *sess = dev->sess;
+	int refcount, ret = 0;
+	bool was_mapped;
+
+	mutex_lock(&dev->lock);
+	if (dev->dev_state == DEV_STATE_UNMAPPED) {
+		rnbd_clt_info(dev, "Device is already being unmapped\n");
+		ret = -EALREADY;
+		goto err;
+	}
+	refcount = refcount_read(&dev->refcount);
+	if (!force && refcount > 1) {
+		rnbd_clt_err(dev,
+			      "Closing device failed, device is in use, (%d device users)\n",
+			      refcount - 1);
+		ret = -EBUSY;
+		goto err;
+	}
+	was_mapped = (dev->dev_state == DEV_STATE_MAPPED);
+	dev->dev_state = DEV_STATE_UNMAPPED;
+	mutex_unlock(&dev->lock);
+
+	delete_dev(dev);
+	destroy_sysfs(dev, sysfs_self);
+	destroy_gen_disk(dev);
+	if (was_mapped && sess->rtrs)
+		send_msg_close(dev, dev->device_id, WAIT);
+
+	rnbd_clt_info(dev, "Device is unmapped\n");
+
+	/* Likely last reference put */
+	rnbd_clt_put_dev(dev);
+
+	/*
+	 * Here device and session can be vanished!
+	 */
+
+	return 0;
+err:
+	mutex_unlock(&dev->lock);
+
+	return ret;
+}
+
+int rnbd_clt_remap_device(struct rnbd_clt_dev *dev)
+{
+	int err;
+
+	mutex_lock(&dev->lock);
+	if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED)
+		err = 0;
+	else if (dev->dev_state == DEV_STATE_UNMAPPED)
+		err = -ENODEV;
+	else if (dev->dev_state == DEV_STATE_MAPPED)
+		err = -EALREADY;
+	else
+		err = -EBUSY;
+	mutex_unlock(&dev->lock);
+	if (!err) {
+		rnbd_clt_info(dev, "Remapping device.\n");
+		err = send_msg_open(dev, WAIT);
+		if (err)
+			rnbd_clt_err(dev, "remap_device: %d\n", err);
+	}
+
+	return err;
+}
+
+static void unmap_device_work(struct work_struct *work)
+{
+	struct rnbd_clt_dev *dev;
+
+	dev = container_of(work, typeof(*dev), unmap_on_rmmod_work);
+	rnbd_clt_unmap_device(dev, true, NULL);
+}
+
+static void rnbd_destroy_sessions(void)
+{
+	struct rnbd_clt_session *sess, *sn;
+	struct rnbd_clt_dev *dev, *tn;
+
+	/* Firstly forbid access through sysfs interface */
+	rnbd_clt_destroy_default_group();
+	rnbd_clt_destroy_sysfs_files();
+
+	/*
+	 * Here at this point there is no any concurrent access to sessions
+	 * list and devices list:
+	 *   1. New session or device can'be be created - session sysfs files
+	 *      are removed.
+	 *   2. Device or session can't be removed - module reference is taken
+	 *      into account in unmap device sysfs callback.
+	 *   3. No IO requests inflight - each file open of block_dev increases
+	 *      module reference in get_disk().
+	 *
+	 * But still there can be user requests inflights, which are sent by
+	 * asynchronous send_msg_*() functions, thus before unmapping devices
+	 * RTRS session must be explicitly closed.
+	 */
+
+	list_for_each_entry_safe(sess, sn, &sess_list, list) {
+		WARN_ON(!rnbd_clt_get_sess(sess));
+		close_rtrs(sess);
+		list_for_each_entry_safe(dev, tn, &sess->devs_list, list) {
+			/*
+			 * Here unmap happens in parallel for only one reason:
+			 * blk_cleanup_queue() takes around half a second, so
+			 * on huge amount of devices the whole module unload
+			 * procedure takes minutes.
+			 */
+			INIT_WORK(&dev->unmap_on_rmmod_work, unmap_device_work);
+			queue_work(system_long_wq, &dev->unmap_on_rmmod_work);
+		}
+		rnbd_clt_put_sess(sess);
+	}
+	/* Wait for all scheduled unmap works */
+	flush_workqueue(system_long_wq);
+	WARN_ON(!list_empty(&sess_list));
+}
+
+static int __init rnbd_client_init(void)
+{
+	int err = 0;
+
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_hdr) != 4);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info) != 36);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info_rsp) != 36);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_open) != 264);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_close) != 8);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_open_rsp) != 56);
+	rnbd_client_major = register_blkdev(rnbd_client_major, "rnbd");
+	if (rnbd_client_major <= 0) {
+		pr_err("Failed to load module, block device registration failed\n");
+		return -EBUSY;
+	}
+
+	err = rnbd_clt_create_sysfs_files();
+	if (err) {
+		pr_err("Failed to load module, creating sysfs device files failed, err: %d\n",
+		       err);
+		unregister_blkdev(rnbd_client_major, "rnbd");
+	}
+
+	return err;
+}
+
+static void __exit rnbd_client_exit(void)
+{
+	rnbd_destroy_sessions();
+	unregister_blkdev(rnbd_client_major, "rnbd");
+	ida_destroy(&index_ida);
+}
+
+module_init(rnbd_client_init);
+module_exit(rnbd_client_exit);
diff --git a/drivers/block/rnbd/rnbd-clt.h b/drivers/block/rnbd/rnbd-clt.h
new file mode 100644
index 000000000000..ed33654aa486
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-clt.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#ifndef RNBD_CLT_H
+#define RNBD_CLT_H
+
+#include <linux/wait.h>
+#include <linux/in.h>
+#include <linux/inet.h>
+#include <linux/blk-mq.h>
+#include <linux/refcount.h>
+
+#include <rtrs.h>
+#include "rnbd-proto.h"
+#include "rnbd-log.h"
+
+/* Max. number of segments per IO request, Mellanox Connect X ~ Connect X5,
+ * choose minimial 30 for all, minus 1 for internal protocol, so 29.
+ */
+#define BMAX_SEGMENTS 29
+/*  time in seconds between reconnect tries, default to 30 s */
+#define RECONNECT_DELAY 30
+/*
+ * Number of times to reconnect on error before giving up, 0 for * disabled,
+ * -1 for forever
+ */
+#define MAX_RECONNECTS -1
+
+enum rnbd_clt_dev_state {
+	DEV_STATE_INIT,
+	DEV_STATE_MAPPED,
+	DEV_STATE_MAPPED_DISCONNECTED,
+	DEV_STATE_UNMAPPED,
+};
+
+struct rnbd_iu_comp {
+	wait_queue_head_t wait;
+	int errno;
+};
+
+struct rnbd_iu {
+	union {
+		struct request *rq; /* for block io */
+		void *buf; /* for user messages */
+	};
+	struct rtrs_permit	*permit;
+	union {
+		/* use to send msg associated with a dev */
+		struct rnbd_clt_dev *dev;
+		/* use to send msg associated with a sess */
+		struct rnbd_clt_session *sess;
+	};
+	struct scatterlist	sglist[BMAX_SEGMENTS];
+	struct work_struct	work;
+	int			errno;
+	struct rnbd_iu_comp	comp;
+	atomic_t		refcount;
+};
+
+struct rnbd_cpu_qlist {
+	struct list_head	requeue_list;
+	spinlock_t		requeue_lock;
+	unsigned int		cpu;
+};
+
+struct rnbd_clt_session {
+	struct list_head        list;
+	struct rtrs_clt        *rtrs;
+	wait_queue_head_t       rtrs_waitq;
+	bool                    rtrs_ready;
+	struct rnbd_cpu_qlist	__percpu
+				*cpu_queues;
+	DECLARE_BITMAP(cpu_queues_bm, NR_CPUS);
+	int	__percpu	*cpu_rr; /* per-cpu var for CPU round-robin */
+	atomic_t		busy;
+	int			queue_depth;
+	u32			max_io_size;
+	struct blk_mq_tag_set	tag_set;
+	struct mutex		lock; /* protects state and devs_list */
+	struct list_head        devs_list; /* list of struct rnbd_clt_dev */
+	refcount_t		refcount;
+	char			sessname[NAME_MAX];
+	u8			ver; /* protocol version */
+};
+
+/**
+ * Submission queues.
+ */
+struct rnbd_queue {
+	struct list_head	requeue_list;
+	unsigned long		in_list;
+	struct rnbd_clt_dev	*dev;
+	struct blk_mq_hw_ctx	*hctx;
+};
+
+struct rnbd_clt_dev {
+	struct rnbd_clt_session	*sess;
+	struct request_queue	*queue;
+	struct rnbd_queue	*hw_queues;
+	u32			device_id;
+	/* local Idr index - used to track minor number allocations. */
+	u32			clt_device_id;
+	struct mutex		lock;
+	enum rnbd_clt_dev_state	dev_state;
+	char			pathname[NAME_MAX];
+	enum rnbd_access_mode	access_mode;
+	bool			read_only;
+	bool			rotational;
+	u32			max_hw_sectors;
+	u32			max_write_same_sectors;
+	u32			max_discard_sectors;
+	u32			discard_granularity;
+	u32			discard_alignment;
+	u16			secure_discard;
+	u16			physical_block_size;
+	u16			logical_block_size;
+	u16			max_segments;
+	size_t			nsectors;
+	u64			size;		/* device size in bytes */
+	struct list_head        list;
+	struct gendisk		*gd;
+	struct kobject		kobj;
+	char			blk_symlink_name[NAME_MAX];
+	refcount_t		refcount;
+	struct work_struct	unmap_on_rmmod_work;
+};
+
+/* rnbd-clt.c */
+
+struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname,
+					   struct rtrs_addr *paths,
+					   size_t path_cnt, u16 port_nr,
+					   const char *pathname,
+					   enum rnbd_access_mode access_mode);
+int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force,
+			   const struct attribute *sysfs_self);
+
+int rnbd_clt_remap_device(struct rnbd_clt_dev *dev);
+int rnbd_clt_resize_disk(struct rnbd_clt_dev *dev, size_t newsize);
+
+/* rnbd-clt-sysfs.c */
+
+int rnbd_clt_create_sysfs_files(void);
+
+void rnbd_clt_destroy_sysfs_files(void);
+void rnbd_clt_destroy_default_group(void);
+
+void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev);
+
+#endif /* RNBD_CLT_H */
diff --git a/drivers/block/rnbd/rnbd-common.c b/drivers/block/rnbd/rnbd-common.c
new file mode 100644
index 000000000000..596c3f732403
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-common.c
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#include "rnbd-proto.h"
+
+const char *rnbd_access_mode_str(enum rnbd_access_mode mode)
+{
+	switch (mode) {
+	case RNBD_ACCESS_RO:
+		return "ro";
+	case RNBD_ACCESS_RW:
+		return "rw";
+	case RNBD_ACCESS_MIGRATION:
+		return "migration";
+	default:
+		return "unknown";
+	}
+}
diff --git a/drivers/block/rnbd/rnbd-log.h b/drivers/block/rnbd/rnbd-log.h
new file mode 100644
index 000000000000..136e7d6c3451
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-log.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#ifndef RNBD_LOG_H
+#define RNBD_LOG_H
+
+#include "rnbd-clt.h"
+#include "rnbd-srv.h"
+
+#define rnbd_clt_log(fn, dev, fmt, ...) (				\
+		fn("<%s@%s> " fmt, (dev)->pathname,			\
+		(dev)->sess->sessname,					\
+		   ##__VA_ARGS__))
+#define rnbd_srv_log(fn, dev, fmt, ...) (				\
+			fn("<%s@%s>: " fmt, (dev)->pathname,		\
+			   (dev)->sess->sessname, ##__VA_ARGS__))
+
+#define rnbd_clt_err(dev, fmt, ...)	\
+	rnbd_clt_log(pr_err, dev, fmt, ##__VA_ARGS__)
+#define rnbd_clt_err_rl(dev, fmt, ...)	\
+	rnbd_clt_log(pr_err_ratelimited, dev, fmt, ##__VA_ARGS__)
+#define rnbd_clt_info(dev, fmt, ...) \
+	rnbd_clt_log(pr_info, dev, fmt, ##__VA_ARGS__)
+#define rnbd_clt_info_rl(dev, fmt, ...) \
+	rnbd_clt_log(pr_info_ratelimited, dev, fmt, ##__VA_ARGS__)
+
+#define rnbd_srv_err(dev, fmt, ...)	\
+	rnbd_srv_log(pr_err, dev, fmt, ##__VA_ARGS__)
+#define rnbd_srv_err_rl(dev, fmt, ...)	\
+	rnbd_srv_log(pr_err_ratelimited, dev, fmt, ##__VA_ARGS__)
+#define rnbd_srv_info(dev, fmt, ...) \
+	rnbd_srv_log(pr_info, dev, fmt, ##__VA_ARGS__)
+#define rnbd_srv_info_rl(dev, fmt, ...) \
+	rnbd_srv_log(pr_info_ratelimited, dev, fmt, ##__VA_ARGS__)
+
+#endif /* RNBD_LOG_H */
diff --git a/drivers/block/rnbd/rnbd-proto.h b/drivers/block/rnbd/rnbd-proto.h
new file mode 100644
index 000000000000..ca166241452c
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-proto.h
@@ -0,0 +1,303 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#ifndef RNBD_PROTO_H
+#define RNBD_PROTO_H
+
+#include <linux/types.h>
+#include <linux/blkdev.h>
+#include <linux/limits.h>
+#include <linux/inet.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <rdma/ib.h>
+
+#define RNBD_PROTO_VER_MAJOR 2
+#define RNBD_PROTO_VER_MINOR 0
+
+/* The default port number the RTRS server is listening on. */
+#define RTRS_PORT 1234
+
+/**
+ * enum rnbd_msg_types - RNBD message types
+ * @RNBD_MSG_SESS_INFO:	initial session info from client to server
+ * @RNBD_MSG_SESS_INFO_RSP:	initial session info from server to client
+ * @RNBD_MSG_OPEN:		open (map) device request
+ * @RNBD_MSG_OPEN_RSP:		response to an @RNBD_MSG_OPEN
+ * @RNBD_MSG_IO:		block IO request operation
+ * @RNBD_MSG_CLOSE:		close (unmap) device request
+ */
+enum rnbd_msg_type {
+	RNBD_MSG_SESS_INFO,
+	RNBD_MSG_SESS_INFO_RSP,
+	RNBD_MSG_OPEN,
+	RNBD_MSG_OPEN_RSP,
+	RNBD_MSG_IO,
+	RNBD_MSG_CLOSE,
+};
+
+/**
+ * struct rnbd_msg_hdr - header of RNBD messages
+ * @type:	Message type, valid values see: enum rnbd_msg_types
+ */
+struct rnbd_msg_hdr {
+	__le16		type;
+	__le16		__padding;
+};
+
+/**
+ * We allow to map RO many times and RW only once. We allow to map yet another
+ * time RW, if MIGRATION is provided (second RW export can be required for
+ * example for VM migration)
+ */
+enum rnbd_access_mode {
+	RNBD_ACCESS_RO,
+	RNBD_ACCESS_RW,
+	RNBD_ACCESS_MIGRATION,
+};
+
+/**
+ * struct rnbd_msg_sess_info - initial session info from client to server
+ * @hdr:		message header
+ * @ver:		RNBD protocol version
+ */
+struct rnbd_msg_sess_info {
+	struct rnbd_msg_hdr hdr;
+	u8		ver;
+	u8		reserved[31];
+};
+
+/**
+ * struct rnbd_msg_sess_info_rsp - initial session info from server to client
+ * @hdr:		message header
+ * @ver:		RNBD protocol version
+ */
+struct rnbd_msg_sess_info_rsp {
+	struct rnbd_msg_hdr hdr;
+	u8		ver;
+	u8		reserved[31];
+};
+
+/**
+ * struct rnbd_msg_open - request to open a remote device.
+ * @hdr:		message header
+ * @access_mode:	the mode to open remote device, valid values see:
+ *			enum rnbd_access_mode
+ * @device_name:	device path on remote side
+ */
+struct rnbd_msg_open {
+	struct rnbd_msg_hdr hdr;
+	u8		access_mode;
+	u8		resv1;
+	s8		dev_name[NAME_MAX];
+	u8		reserved[3];
+};
+
+/**
+ * struct rnbd_msg_close - request to close a remote device.
+ * @hdr:	message header
+ * @device_id:	device_id on server side to identify the device
+ */
+struct rnbd_msg_close {
+	struct rnbd_msg_hdr hdr;
+	__le32		device_id;
+};
+
+/**
+ * struct rnbd_msg_open_rsp - response message to RNBD_MSG_OPEN
+ * @hdr:		message header
+ * @device_id:		device_id on server side to identify the device
+ * @nsectors:		number of sectors in the usual 512b unit
+ * @max_hw_sectors:	max hardware sectors in the usual 512b unit
+ * @max_write_same_sectors: max sectors for WRITE SAME in the 512b unit
+ * @max_discard_sectors: max. sectors that can be discarded at once in 512b
+ * unit.
+ * @discard_granularity: size of the internal discard allocation unit in bytes
+ * @discard_alignment: offset from internal allocation assignment in bytes
+ * @physical_block_size: physical block size device supports in bytes
+ * @logical_block_size: logical block size device supports in bytes
+ * @max_segments:	max segments hardware support in one transfer
+ * @secure_discard:	supports secure discard
+ * @rotation:		is a rotational disc?
+ */
+struct rnbd_msg_open_rsp {
+	struct rnbd_msg_hdr	hdr;
+	__le32			device_id;
+	__le64			nsectors;
+	__le32			max_hw_sectors;
+	__le32			max_write_same_sectors;
+	__le32			max_discard_sectors;
+	__le32			discard_granularity;
+	__le32			discard_alignment;
+	__le16			physical_block_size;
+	__le16			logical_block_size;
+	__le16			max_segments;
+	__le16			secure_discard;
+	u8			rotational;
+	u8			reserved[11];
+};
+
+/**
+ * struct rnbd_msg_io - message for I/O read/write
+ * @hdr:	message header
+ * @device_id:	device_id on server side to find the right device
+ * @sector:	bi_sector attribute from struct bio
+ * @rw:		valid values are defined in enum rnbd_io_flags
+ * @bi_size:    number of bytes for I/O read/write
+ * @prio:       priority
+ */
+struct rnbd_msg_io {
+	struct rnbd_msg_hdr hdr;
+	__le32		device_id;
+	__le64		sector;
+	__le32		rw;
+	__le32		bi_size;
+	__le16		prio;
+};
+
+#define RNBD_OP_BITS  8
+#define RNBD_OP_MASK  ((1 << RNBD_OP_BITS) - 1)
+
+/**
+ * enum rnbd_io_flags - RNBD request types from rq_flag_bits
+ * @RNBD_OP_READ:	     read sectors from the device
+ * @RNBD_OP_WRITE:	     write sectors to the device
+ * @RNBD_OP_FLUSH:	     flush the volatile write cache
+ * @RNBD_OP_DISCARD:        discard sectors
+ * @RNBD_OP_SECURE_ERASE:   securely erase sectors
+ * @RNBD_OP_WRITE_SAME:     write the same sectors many times
+
+ * @RNBD_F_SYNC:	     request is sync (sync write or read)
+ * @RNBD_F_FUA:             forced unit access
+ */
+enum rnbd_io_flags {
+
+	/* Operations */
+
+	RNBD_OP_READ		= 0,
+	RNBD_OP_WRITE		= 1,
+	RNBD_OP_FLUSH		= 2,
+	RNBD_OP_DISCARD	= 3,
+	RNBD_OP_SECURE_ERASE	= 4,
+	RNBD_OP_WRITE_SAME	= 5,
+
+	RNBD_OP_LAST,
+
+	/* Flags */
+
+	RNBD_F_SYNC  = 1<<(RNBD_OP_BITS + 0),
+	RNBD_F_FUA   = 1<<(RNBD_OP_BITS + 1),
+
+	RNBD_F_ALL   = (RNBD_F_SYNC | RNBD_F_FUA)
+
+};
+
+static inline u32 rnbd_op(u32 flags)
+{
+	return flags & RNBD_OP_MASK;
+}
+
+static inline u32 rnbd_flags(u32 flags)
+{
+	return flags & ~RNBD_OP_MASK;
+}
+
+static inline bool rnbd_flags_supported(u32 flags)
+{
+	u32 op;
+
+	op = rnbd_op(flags);
+	flags = rnbd_flags(flags);
+
+	if (op >= RNBD_OP_LAST)
+		return false;
+	if (flags & ~RNBD_F_ALL)
+		return false;
+
+	return true;
+}
+
+static inline u32 rnbd_to_bio_flags(u32 rnbd_opf)
+{
+	u32 bio_opf;
+
+	switch (rnbd_op(rnbd_opf)) {
+	case RNBD_OP_READ:
+		bio_opf = REQ_OP_READ;
+		break;
+	case RNBD_OP_WRITE:
+		bio_opf = REQ_OP_WRITE;
+		break;
+	case RNBD_OP_FLUSH:
+		bio_opf = REQ_OP_FLUSH | REQ_PREFLUSH;
+		break;
+	case RNBD_OP_DISCARD:
+		bio_opf = REQ_OP_DISCARD;
+		break;
+	case RNBD_OP_SECURE_ERASE:
+		bio_opf = REQ_OP_SECURE_ERASE;
+		break;
+	case RNBD_OP_WRITE_SAME:
+		bio_opf = REQ_OP_WRITE_SAME;
+		break;
+	default:
+		WARN(1, "Unknown RNBD type: %d (flags %d)\n",
+		     rnbd_op(rnbd_opf), rnbd_opf);
+		bio_opf = 0;
+	}
+
+	if (rnbd_opf & RNBD_F_SYNC)
+		bio_opf |= REQ_SYNC;
+
+	if (rnbd_opf & RNBD_F_FUA)
+		bio_opf |= REQ_FUA;
+
+	return bio_opf;
+}
+
+static inline u32 rq_to_rnbd_flags(struct request *rq)
+{
+	u32 rnbd_opf;
+
+	switch (req_op(rq)) {
+	case REQ_OP_READ:
+		rnbd_opf = RNBD_OP_READ;
+		break;
+	case REQ_OP_WRITE:
+		rnbd_opf = RNBD_OP_WRITE;
+		break;
+	case REQ_OP_DISCARD:
+		rnbd_opf = RNBD_OP_DISCARD;
+		break;
+	case REQ_OP_SECURE_ERASE:
+		rnbd_opf = RNBD_OP_SECURE_ERASE;
+		break;
+	case REQ_OP_WRITE_SAME:
+		rnbd_opf = RNBD_OP_WRITE_SAME;
+		break;
+	case REQ_OP_FLUSH:
+		rnbd_opf = RNBD_OP_FLUSH;
+		break;
+	default:
+		WARN(1, "Unknown request type %d (flags %llu)\n",
+		     req_op(rq), (unsigned long long)rq->cmd_flags);
+		rnbd_opf = 0;
+	}
+
+	if (op_is_sync(rq->cmd_flags))
+		rnbd_opf |= RNBD_F_SYNC;
+
+	if (op_is_flush(rq->cmd_flags))
+		rnbd_opf |= RNBD_F_FUA;
+
+	return rnbd_opf;
+}
+
+const char *rnbd_access_mode_str(enum rnbd_access_mode mode);
+
+#endif /* RNBD_PROTO_H */
diff --git a/drivers/block/rnbd/rnbd-srv-dev.c b/drivers/block/rnbd/rnbd-srv-dev.c
new file mode 100644
index 000000000000..5eddfd29ab64
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-srv-dev.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include "rnbd-srv-dev.h"
+#include "rnbd-log.h"
+
+struct rnbd_dev *rnbd_dev_open(const char *path, fmode_t flags,
+			       struct bio_set *bs)
+{
+	struct rnbd_dev *dev;
+	int ret;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	dev->blk_open_flags = flags;
+	dev->bdev = blkdev_get_by_path(path, flags, THIS_MODULE);
+	ret = PTR_ERR_OR_ZERO(dev->bdev);
+	if (ret)
+		goto err;
+
+	dev->blk_open_flags = flags;
+	bdevname(dev->bdev, dev->name);
+	dev->ibd_bio_set = bs;
+
+	return dev;
+
+err:
+	kfree(dev);
+	return ERR_PTR(ret);
+}
+
+void rnbd_dev_close(struct rnbd_dev *dev)
+{
+	blkdev_put(dev->bdev, dev->blk_open_flags);
+	kfree(dev);
+}
+
+static void rnbd_dev_bi_end_io(struct bio *bio)
+{
+	struct rnbd_dev_blk_io *io = bio->bi_private;
+
+	rnbd_endio(io->priv, blk_status_to_errno(bio->bi_status));
+	bio_put(bio);
+}
+
+/**
+ *	rnbd_bio_map_kern	-	map kernel address into bio
+ *	@data: pointer to buffer to map
+ *	@bs: bio_set to use.
+ *	@len: length in bytes
+ *	@gfp_mask: allocation flags for bio allocation
+ *
+ *	Map the kernel address into a bio suitable for io to a block
+ *	device. Returns an error pointer in case of error.
+ */
+static struct bio *rnbd_bio_map_kern(void *data, struct bio_set *bs,
+				     unsigned int len, gfp_t gfp_mask)
+{
+	unsigned long kaddr = (unsigned long)data;
+	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	unsigned long start = kaddr >> PAGE_SHIFT;
+	const int nr_pages = end - start;
+	int offset, i;
+	struct bio *bio;
+
+	bio = bio_alloc_bioset(gfp_mask, nr_pages, bs);
+	if (!bio)
+		return ERR_PTR(-ENOMEM);
+
+	offset = offset_in_page(kaddr);
+	for (i = 0; i < nr_pages; i++) {
+		unsigned int bytes = PAGE_SIZE - offset;
+
+		if (len <= 0)
+			break;
+
+		if (bytes > len)
+			bytes = len;
+
+		if (bio_add_page(bio, virt_to_page(data), bytes,
+				    offset) < bytes) {
+			/* we don't support partial mappings */
+			bio_put(bio);
+			return ERR_PTR(-EINVAL);
+		}
+
+		data += bytes;
+		len -= bytes;
+		offset = 0;
+	}
+
+	bio->bi_end_io = bio_put;
+	return bio;
+}
+
+int rnbd_dev_submit_io(struct rnbd_dev *dev, sector_t sector, void *data,
+		       size_t len, u32 bi_size, enum rnbd_io_flags flags,
+		       short prio, void *priv)
+{
+	struct rnbd_dev_blk_io *io;
+	struct bio *bio;
+
+	/* Generate bio with pages pointing to the rdma buffer */
+	bio = rnbd_bio_map_kern(data, dev->ibd_bio_set, len, GFP_KERNEL);
+	if (IS_ERR(bio))
+		return PTR_ERR(bio);
+
+	io = container_of(bio, struct rnbd_dev_blk_io, bio);
+
+	io->dev	= dev;
+	io->priv = priv;
+
+	bio->bi_end_io = rnbd_dev_bi_end_io;
+	bio->bi_private	= io;
+	bio->bi_opf = rnbd_to_bio_flags(flags);
+	bio->bi_iter.bi_sector = sector;
+	bio->bi_iter.bi_size = bi_size;
+	bio_set_prio(bio, prio);
+	bio_set_dev(bio, dev->bdev);
+
+	submit_bio(bio);
+
+	return 0;
+}
diff --git a/drivers/block/rnbd/rnbd-srv-dev.h b/drivers/block/rnbd/rnbd-srv-dev.h
new file mode 100644
index 000000000000..0f65b09a270e
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-srv-dev.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#ifndef RNBD_SRV_DEV_H
+#define RNBD_SRV_DEV_H
+
+#include <linux/fs.h>
+#include "rnbd-proto.h"
+
+struct rnbd_dev {
+	struct block_device	*bdev;
+	struct bio_set		*ibd_bio_set;
+	fmode_t			blk_open_flags;
+	char			name[BDEVNAME_SIZE];
+};
+
+struct rnbd_dev_blk_io {
+	struct rnbd_dev *dev;
+	void		 *priv;
+	/* have to be last member for front_pad usage of bioset_init */
+	struct bio	bio;
+};
+
+/**
+ * rnbd_dev_open() - Open a device
+ * @flags:	open flags
+ * @bs:		bio_set to use during block io,
+ */
+struct rnbd_dev *rnbd_dev_open(const char *path, fmode_t flags,
+			       struct bio_set *bs);
+
+/**
+ * rnbd_dev_close() - Close a device
+ */
+void rnbd_dev_close(struct rnbd_dev *dev);
+
+void rnbd_endio(void *priv, int error);
+
+static inline int rnbd_dev_get_max_segs(const struct rnbd_dev *dev)
+{
+	return queue_max_segments(bdev_get_queue(dev->bdev));
+}
+
+static inline int rnbd_dev_get_max_hw_sects(const struct rnbd_dev *dev)
+{
+	return queue_max_hw_sectors(bdev_get_queue(dev->bdev));
+}
+
+static inline int rnbd_dev_get_secure_discard(const struct rnbd_dev *dev)
+{
+	return blk_queue_secure_erase(bdev_get_queue(dev->bdev));
+}
+
+static inline int rnbd_dev_get_max_discard_sects(const struct rnbd_dev *dev)
+{
+	if (!blk_queue_discard(bdev_get_queue(dev->bdev)))
+		return 0;
+
+	return blk_queue_get_max_sectors(bdev_get_queue(dev->bdev),
+					 REQ_OP_DISCARD);
+}
+
+static inline int rnbd_dev_get_discard_granularity(const struct rnbd_dev *dev)
+{
+	return bdev_get_queue(dev->bdev)->limits.discard_granularity;
+}
+
+static inline int rnbd_dev_get_discard_alignment(const struct rnbd_dev *dev)
+{
+	return bdev_get_queue(dev->bdev)->limits.discard_alignment;
+}
+
+/**
+ * rnbd_dev_submit_io() - Submit an I/O to the disk
+ * @dev:	device to that the I/O is submitted
+ * @sector:	address to read/write data to
+ * @data:	I/O data to write or buffer to read I/O date into
+ * @len:	length of @data
+ * @bi_size:	Amount of data that will be read/written
+ * @prio:       IO priority
+ * @priv:	private data passed to @io_fn
+ */
+int rnbd_dev_submit_io(struct rnbd_dev *dev, sector_t sector, void *data,
+			size_t len, u32 bi_size, enum rnbd_io_flags flags,
+			short prio, void *priv);
+
+#endif /* RNBD_SRV_DEV_H */
diff --git a/drivers/block/rnbd/rnbd-srv-sysfs.c b/drivers/block/rnbd/rnbd-srv-sysfs.c
new file mode 100644
index 000000000000..106775c074d1
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-srv-sysfs.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include <uapi/linux/limits.h>
+#include <linux/kobject.h>
+#include <linux/sysfs.h>
+#include <linux/stat.h>
+#include <linux/genhd.h>
+#include <linux/list.h>
+#include <linux/moduleparam.h>
+#include <linux/device.h>
+
+#include "rnbd-srv.h"
+
+static struct device *rnbd_dev;
+static struct class *rnbd_dev_class;
+static struct kobject *rnbd_devs_kobj;
+
+static void rnbd_srv_dev_release(struct kobject *kobj)
+{
+	struct rnbd_srv_dev *dev;
+
+	dev = container_of(kobj, struct rnbd_srv_dev, dev_kobj);
+
+	kfree(dev);
+}
+
+static struct kobj_type dev_ktype = {
+	.sysfs_ops = &kobj_sysfs_ops,
+	.release = rnbd_srv_dev_release
+};
+
+int rnbd_srv_create_dev_sysfs(struct rnbd_srv_dev *dev,
+			       struct block_device *bdev,
+			       const char *dev_name)
+{
+	struct kobject *bdev_kobj;
+	int ret;
+
+	ret = kobject_init_and_add(&dev->dev_kobj, &dev_ktype,
+				   rnbd_devs_kobj, dev_name);
+	if (ret)
+		return ret;
+
+	dev->dev_sessions_kobj = kobject_create_and_add("sessions",
+							&dev->dev_kobj);
+	if (!dev->dev_sessions_kobj)
+		goto put_dev_kobj;
+
+	bdev_kobj = &disk_to_dev(bdev->bd_disk)->kobj;
+	ret = sysfs_create_link(&dev->dev_kobj, bdev_kobj, "block_dev");
+	if (ret)
+		goto put_sess_kobj;
+
+	return 0;
+
+put_sess_kobj:
+	kobject_put(dev->dev_sessions_kobj);
+put_dev_kobj:
+	kobject_put(&dev->dev_kobj);
+	return ret;
+}
+
+void rnbd_srv_destroy_dev_sysfs(struct rnbd_srv_dev *dev)
+{
+	sysfs_remove_link(&dev->dev_kobj, "block_dev");
+	kobject_del(dev->dev_sessions_kobj);
+	kobject_put(dev->dev_sessions_kobj);
+	kobject_del(&dev->dev_kobj);
+	kobject_put(&dev->dev_kobj);
+}
+
+static ssize_t read_only_show(struct kobject *kobj, struct kobj_attribute *attr,
+			      char *page)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+
+	sess_dev = container_of(kobj, struct rnbd_srv_sess_dev, kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%d\n",
+			 !(sess_dev->open_flags & FMODE_WRITE));
+}
+
+static struct kobj_attribute rnbd_srv_dev_session_ro_attr =
+	__ATTR_RO(read_only);
+
+static ssize_t access_mode_show(struct kobject *kobj,
+				struct kobj_attribute *attr,
+				char *page)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+
+	sess_dev = container_of(kobj, struct rnbd_srv_sess_dev, kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%s\n",
+			 rnbd_access_mode_str(sess_dev->access_mode));
+}
+
+static struct kobj_attribute rnbd_srv_dev_session_access_mode_attr =
+	__ATTR_RO(access_mode);
+
+static ssize_t mapping_path_show(struct kobject *kobj,
+				 struct kobj_attribute *attr, char *page)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+
+	sess_dev = container_of(kobj, struct rnbd_srv_sess_dev, kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%s\n", sess_dev->pathname);
+}
+
+static struct kobj_attribute rnbd_srv_dev_session_mapping_path_attr =
+	__ATTR_RO(mapping_path);
+
+static struct attribute *rnbd_srv_default_dev_sessions_attrs[] = {
+	&rnbd_srv_dev_session_access_mode_attr.attr,
+	&rnbd_srv_dev_session_ro_attr.attr,
+	&rnbd_srv_dev_session_mapping_path_attr.attr,
+	NULL,
+};
+
+static struct attribute_group rnbd_srv_default_dev_session_attr_group = {
+	.attrs = rnbd_srv_default_dev_sessions_attrs,
+};
+
+void rnbd_srv_destroy_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev)
+{
+	sysfs_remove_group(&sess_dev->kobj,
+			   &rnbd_srv_default_dev_session_attr_group);
+
+	kobject_del(&sess_dev->kobj);
+	kobject_put(&sess_dev->kobj);
+}
+
+static void rnbd_srv_sess_dev_release(struct kobject *kobj)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+
+	sess_dev = container_of(kobj, struct rnbd_srv_sess_dev, kobj);
+	rnbd_destroy_sess_dev(sess_dev);
+}
+
+static struct kobj_type rnbd_srv_sess_dev_ktype = {
+	.sysfs_ops	= &kobj_sysfs_ops,
+	.release	= rnbd_srv_sess_dev_release,
+};
+
+int rnbd_srv_create_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev)
+{
+	int ret;
+
+	ret = kobject_init_and_add(&sess_dev->kobj, &rnbd_srv_sess_dev_ktype,
+				   sess_dev->dev->dev_sessions_kobj, "%s",
+				   sess_dev->sess->sessname);
+	if (ret)
+		return ret;
+
+	ret = sysfs_create_group(&sess_dev->kobj,
+				 &rnbd_srv_default_dev_session_attr_group);
+	if (ret)
+		goto err;
+
+	return 0;
+
+err:
+	kobject_put(&sess_dev->kobj);
+
+	return ret;
+}
+
+int rnbd_srv_create_sysfs_files(void)
+{
+	int err;
+
+	rnbd_dev_class = class_create(THIS_MODULE, "rnbd-server");
+	if (IS_ERR(rnbd_dev_class))
+		return PTR_ERR(rnbd_dev_class);
+
+	rnbd_dev = device_create(rnbd_dev_class, NULL,
+				  MKDEV(0, 0), NULL, "ctl");
+	if (IS_ERR(rnbd_dev)) {
+		err = PTR_ERR(rnbd_dev);
+		goto cls_destroy;
+	}
+	rnbd_devs_kobj = kobject_create_and_add("devices", &rnbd_dev->kobj);
+	if (!rnbd_devs_kobj) {
+		err = -ENOMEM;
+		goto dev_destroy;
+	}
+
+	return 0;
+
+dev_destroy:
+	device_destroy(rnbd_dev_class, MKDEV(0, 0));
+cls_destroy:
+	class_destroy(rnbd_dev_class);
+
+	return err;
+}
+
+void rnbd_srv_destroy_sysfs_files(void)
+{
+	kobject_del(rnbd_devs_kobj);
+	kobject_put(rnbd_devs_kobj);
+	device_destroy(rnbd_dev_class, MKDEV(0, 0));
+	class_destroy(rnbd_dev_class);
+}
diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c
new file mode 100644
index 000000000000..86e61523907b
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-srv.c
@@ -0,0 +1,844 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include <linux/module.h>
+#include <linux/blkdev.h>
+
+#include "rnbd-srv.h"
+#include "rnbd-srv-dev.h"
+
+MODULE_DESCRIPTION("RDMA Network Block Device Server");
+MODULE_LICENSE("GPL");
+
+static u16 port_nr = RTRS_PORT;
+
+module_param_named(port_nr, port_nr, ushort, 0444);
+MODULE_PARM_DESC(port_nr,
+		 "The port number the server is listening on (default: "
+		 __stringify(RTRS_PORT)")");
+
+#define DEFAULT_DEV_SEARCH_PATH "/"
+
+static char dev_search_path[PATH_MAX] = DEFAULT_DEV_SEARCH_PATH;
+
+static int dev_search_path_set(const char *val, const struct kernel_param *kp)
+{
+	const char *p = strrchr(val, '\n') ? : val + strlen(val);
+
+	if (strlen(val) >= sizeof(dev_search_path))
+		return -EINVAL;
+
+	snprintf(dev_search_path, sizeof(dev_search_path), "%.*s",
+		 (int)(p - val), val);
+
+	pr_info("dev_search_path changed to '%s'\n", dev_search_path);
+
+	return 0;
+}
+
+static struct kparam_string dev_search_path_kparam_str = {
+	.maxlen	= sizeof(dev_search_path),
+	.string	= dev_search_path
+};
+
+static const struct kernel_param_ops dev_search_path_ops = {
+	.set	= dev_search_path_set,
+	.get	= param_get_string,
+};
+
+module_param_cb(dev_search_path, &dev_search_path_ops,
+		&dev_search_path_kparam_str, 0444);
+MODULE_PARM_DESC(dev_search_path,
+		 "Sets the dev_search_path. When a device is mapped this path is prepended to the device path from the map device operation.  If %SESSNAME% is specified in a path, then device will be searched in a session namespace. (default: "
+		 DEFAULT_DEV_SEARCH_PATH ")");
+
+static DEFINE_MUTEX(sess_lock);
+static DEFINE_SPINLOCK(dev_lock);
+
+static LIST_HEAD(sess_list);
+static LIST_HEAD(dev_list);
+
+struct rnbd_io_private {
+	struct rtrs_srv_op		*id;
+	struct rnbd_srv_sess_dev	*sess_dev;
+};
+
+static void rnbd_sess_dev_release(struct kref *kref)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+
+	sess_dev = container_of(kref, struct rnbd_srv_sess_dev, kref);
+	complete(sess_dev->destroy_comp);
+}
+
+static inline void rnbd_put_sess_dev(struct rnbd_srv_sess_dev *sess_dev)
+{
+	kref_put(&sess_dev->kref, rnbd_sess_dev_release);
+}
+
+void rnbd_endio(void *priv, int error)
+{
+	struct rnbd_io_private *rnbd_priv = priv;
+	struct rnbd_srv_sess_dev *sess_dev = rnbd_priv->sess_dev;
+
+	rnbd_put_sess_dev(sess_dev);
+
+	rtrs_srv_resp_rdma(rnbd_priv->id, error);
+
+	kfree(priv);
+}
+
+static struct rnbd_srv_sess_dev *
+rnbd_get_sess_dev(int dev_id, struct rnbd_srv_session *srv_sess)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+	int ret = 0;
+
+	rcu_read_lock();
+	sess_dev = xa_load(&srv_sess->index_idr, dev_id);
+	if (likely(sess_dev))
+		ret = kref_get_unless_zero(&sess_dev->kref);
+	rcu_read_unlock();
+
+	if (!sess_dev || !ret)
+		return ERR_PTR(-ENXIO);
+
+	return sess_dev;
+}
+
+static int process_rdma(struct rtrs_srv *sess,
+			struct rnbd_srv_session *srv_sess,
+			struct rtrs_srv_op *id, void *data, u32 datalen,
+			const void *usr, size_t usrlen)
+{
+	const struct rnbd_msg_io *msg = usr;
+	struct rnbd_io_private *priv;
+	struct rnbd_srv_sess_dev *sess_dev;
+	u32 dev_id;
+	int err;
+
+	priv = kmalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev_id = le32_to_cpu(msg->device_id);
+
+	sess_dev = rnbd_get_sess_dev(dev_id, srv_sess);
+	if (IS_ERR(sess_dev)) {
+		pr_err_ratelimited("Got I/O request on session %s for unknown device id %d\n",
+				   srv_sess->sessname, dev_id);
+		err = -ENOTCONN;
+		goto err;
+	}
+
+	priv->sess_dev = sess_dev;
+	priv->id = id;
+
+	err = rnbd_dev_submit_io(sess_dev->rnbd_dev, le64_to_cpu(msg->sector),
+				  data, datalen, le32_to_cpu(msg->bi_size),
+				  le32_to_cpu(msg->rw),
+				  srv_sess->ver < RNBD_PROTO_VER_MAJOR ||
+				  usrlen < sizeof(*msg) ?
+				  0 : le16_to_cpu(msg->prio), priv);
+	if (unlikely(err)) {
+		rnbd_srv_err(sess_dev, "Submitting I/O to device failed, err: %d\n",
+			      err);
+		goto sess_dev_put;
+	}
+
+	return 0;
+
+sess_dev_put:
+	rnbd_put_sess_dev(sess_dev);
+err:
+	kfree(priv);
+	return err;
+}
+
+static void destroy_device(struct rnbd_srv_dev *dev)
+{
+	WARN_ONCE(!list_empty(&dev->sess_dev_list),
+		  "Device %s is being destroyed but still in use!\n",
+		  dev->id);
+
+	spin_lock(&dev_lock);
+	list_del(&dev->list);
+	spin_unlock(&dev_lock);
+
+	mutex_destroy(&dev->lock);
+	if (dev->dev_kobj.state_in_sysfs)
+		/*
+		 * Destroy kobj only if it was really created.
+		 */
+		rnbd_srv_destroy_dev_sysfs(dev);
+	else
+		kfree(dev);
+}
+
+static void destroy_device_cb(struct kref *kref)
+{
+	struct rnbd_srv_dev *dev;
+
+	dev = container_of(kref, struct rnbd_srv_dev, kref);
+
+	destroy_device(dev);
+}
+
+static void rnbd_put_srv_dev(struct rnbd_srv_dev *dev)
+{
+	kref_put(&dev->kref, destroy_device_cb);
+}
+
+void rnbd_destroy_sess_dev(struct rnbd_srv_sess_dev *sess_dev)
+{
+	DECLARE_COMPLETION_ONSTACK(dc);
+
+	xa_erase(&sess_dev->sess->index_idr, sess_dev->device_id);
+	synchronize_rcu();
+	sess_dev->destroy_comp = &dc;
+	rnbd_put_sess_dev(sess_dev);
+	wait_for_completion(&dc); /* wait for inflights to drop to zero */
+
+	rnbd_dev_close(sess_dev->rnbd_dev);
+	list_del(&sess_dev->sess_list);
+	mutex_lock(&sess_dev->dev->lock);
+	list_del(&sess_dev->dev_list);
+	if (sess_dev->open_flags & FMODE_WRITE)
+		sess_dev->dev->open_write_cnt--;
+	mutex_unlock(&sess_dev->dev->lock);
+
+	rnbd_put_srv_dev(sess_dev->dev);
+
+	rnbd_srv_info(sess_dev, "Device closed\n");
+	kfree(sess_dev);
+}
+
+static void destroy_sess(struct rnbd_srv_session *srv_sess)
+{
+	struct rnbd_srv_sess_dev *sess_dev, *tmp;
+
+	if (list_empty(&srv_sess->sess_dev_list))
+		goto out;
+
+	mutex_lock(&srv_sess->lock);
+	list_for_each_entry_safe(sess_dev, tmp, &srv_sess->sess_dev_list,
+				 sess_list)
+		rnbd_srv_destroy_dev_session_sysfs(sess_dev);
+	mutex_unlock(&srv_sess->lock);
+
+out:
+	xa_destroy(&srv_sess->index_idr);
+	bioset_exit(&srv_sess->sess_bio_set);
+
+	pr_info("RTRS Session %s disconnected\n", srv_sess->sessname);
+
+	mutex_lock(&sess_lock);
+	list_del(&srv_sess->list);
+	mutex_unlock(&sess_lock);
+
+	mutex_destroy(&srv_sess->lock);
+	kfree(srv_sess);
+}
+
+static int create_sess(struct rtrs_srv *rtrs)
+{
+	struct rnbd_srv_session *srv_sess;
+	char sessname[NAME_MAX];
+	int err;
+
+	err = rtrs_srv_get_sess_name(rtrs, sessname, sizeof(sessname));
+	if (err) {
+		pr_err("rtrs_srv_get_sess_name(%s): %d\n", sessname, err);
+
+		return err;
+	}
+	srv_sess = kzalloc(sizeof(*srv_sess), GFP_KERNEL);
+	if (!srv_sess)
+		return -ENOMEM;
+
+	srv_sess->queue_depth = rtrs_srv_get_queue_depth(rtrs);
+	err = bioset_init(&srv_sess->sess_bio_set, srv_sess->queue_depth,
+			  offsetof(struct rnbd_dev_blk_io, bio),
+			  BIOSET_NEED_BVECS);
+	if (err) {
+		pr_err("Allocating srv_session for session %s failed\n",
+		       sessname);
+		kfree(srv_sess);
+		return err;
+	}
+
+	xa_init_flags(&srv_sess->index_idr, XA_FLAGS_ALLOC);
+	INIT_LIST_HEAD(&srv_sess->sess_dev_list);
+	mutex_init(&srv_sess->lock);
+	mutex_lock(&sess_lock);
+	list_add(&srv_sess->list, &sess_list);
+	mutex_unlock(&sess_lock);
+
+	srv_sess->rtrs = rtrs;
+	strlcpy(srv_sess->sessname, sessname, sizeof(srv_sess->sessname));
+
+	rtrs_srv_set_sess_priv(rtrs, srv_sess);
+
+	return 0;
+}
+
+static int rnbd_srv_link_ev(struct rtrs_srv *rtrs,
+			     enum rtrs_srv_link_ev ev, void *priv)
+{
+	struct rnbd_srv_session *srv_sess = priv;
+
+	switch (ev) {
+	case RTRS_SRV_LINK_EV_CONNECTED:
+		return create_sess(rtrs);
+
+	case RTRS_SRV_LINK_EV_DISCONNECTED:
+		if (WARN_ON_ONCE(!srv_sess))
+			return -EINVAL;
+
+		destroy_sess(srv_sess);
+		return 0;
+
+	default:
+		pr_warn("Received unknown RTRS session event %d from session %s\n",
+			ev, srv_sess->sessname);
+		return -EINVAL;
+	}
+}
+
+static int process_msg_close(struct rtrs_srv *rtrs,
+			     struct rnbd_srv_session *srv_sess,
+			     void *data, size_t datalen, const void *usr,
+			     size_t usrlen)
+{
+	const struct rnbd_msg_close *close_msg = usr;
+	struct rnbd_srv_sess_dev *sess_dev;
+
+	sess_dev = rnbd_get_sess_dev(le32_to_cpu(close_msg->device_id),
+				      srv_sess);
+	if (IS_ERR(sess_dev))
+		return 0;
+
+	rnbd_put_sess_dev(sess_dev);
+	mutex_lock(&srv_sess->lock);
+	rnbd_srv_destroy_dev_session_sysfs(sess_dev);
+	mutex_unlock(&srv_sess->lock);
+	return 0;
+}
+
+static int process_msg_open(struct rtrs_srv *rtrs,
+			    struct rnbd_srv_session *srv_sess,
+			    const void *msg, size_t len,
+			    void *data, size_t datalen);
+
+static int process_msg_sess_info(struct rtrs_srv *rtrs,
+				 struct rnbd_srv_session *srv_sess,
+				 const void *msg, size_t len,
+				 void *data, size_t datalen);
+
+static int rnbd_srv_rdma_ev(struct rtrs_srv *rtrs, void *priv,
+			     struct rtrs_srv_op *id, int dir,
+			     void *data, size_t datalen, const void *usr,
+			     size_t usrlen)
+{
+	struct rnbd_srv_session *srv_sess = priv;
+	const struct rnbd_msg_hdr *hdr = usr;
+	int ret = 0;
+	u16 type;
+
+	if (WARN_ON_ONCE(!srv_sess))
+		return -ENODEV;
+
+	type = le16_to_cpu(hdr->type);
+
+	switch (type) {
+	case RNBD_MSG_IO:
+		return process_rdma(rtrs, srv_sess, id, data, datalen, usr,
+				    usrlen);
+	case RNBD_MSG_CLOSE:
+		ret = process_msg_close(rtrs, srv_sess, data, datalen,
+					usr, usrlen);
+		break;
+	case RNBD_MSG_OPEN:
+		ret = process_msg_open(rtrs, srv_sess, usr, usrlen,
+				       data, datalen);
+		break;
+	case RNBD_MSG_SESS_INFO:
+		ret = process_msg_sess_info(rtrs, srv_sess, usr, usrlen,
+					    data, datalen);
+		break;
+	default:
+		pr_warn("Received unexpected message type %d with dir %d from session %s\n",
+			type, dir, srv_sess->sessname);
+		return -EINVAL;
+	}
+
+	rtrs_srv_resp_rdma(id, ret);
+	return 0;
+}
+
+static struct rnbd_srv_sess_dev
+*rnbd_sess_dev_alloc(struct rnbd_srv_session *srv_sess)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+	int error;
+
+	sess_dev = kzalloc(sizeof(*sess_dev), GFP_KERNEL);
+	if (!sess_dev)
+		return ERR_PTR(-ENOMEM);
+
+	error = xa_alloc(&srv_sess->index_idr, &sess_dev->device_id, sess_dev,
+			 xa_limit_32b, GFP_NOWAIT);
+	if (error < 0) {
+		pr_warn("Allocating idr failed, err: %d\n", error);
+		kfree(sess_dev);
+		return ERR_PTR(error);
+	}
+
+	return sess_dev;
+}
+
+static struct rnbd_srv_dev *rnbd_srv_init_srv_dev(const char *id)
+{
+	struct rnbd_srv_dev *dev;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	strlcpy(dev->id, id, sizeof(dev->id));
+	kref_init(&dev->kref);
+	INIT_LIST_HEAD(&dev->sess_dev_list);
+	mutex_init(&dev->lock);
+
+	return dev;
+}
+
+static struct rnbd_srv_dev *
+rnbd_srv_find_or_add_srv_dev(struct rnbd_srv_dev *new_dev)
+{
+	struct rnbd_srv_dev *dev;
+
+	spin_lock(&dev_lock);
+	list_for_each_entry(dev, &dev_list, list) {
+		if (!strncmp(dev->id, new_dev->id, sizeof(dev->id))) {
+			if (!kref_get_unless_zero(&dev->kref))
+				/*
+				 * We lost the race, device is almost dead.
+				 *  Continue traversing to find a valid one.
+				 */
+				continue;
+			spin_unlock(&dev_lock);
+			return dev;
+		}
+	}
+	list_add(&new_dev->list, &dev_list);
+	spin_unlock(&dev_lock);
+
+	return new_dev;
+}
+
+static int rnbd_srv_check_update_open_perm(struct rnbd_srv_dev *srv_dev,
+					    struct rnbd_srv_session *srv_sess,
+					    enum rnbd_access_mode access_mode)
+{
+	int ret = -EPERM;
+
+	mutex_lock(&srv_dev->lock);
+
+	switch (access_mode) {
+	case RNBD_ACCESS_RO:
+		ret = 0;
+		break;
+	case RNBD_ACCESS_RW:
+		if (srv_dev->open_write_cnt == 0)  {
+			srv_dev->open_write_cnt++;
+			ret = 0;
+		} else {
+			pr_err("Mapping device '%s' for session %s with RW permissions failed. Device already opened as 'RW' by %d client(s), access mode %s.\n",
+			       srv_dev->id, srv_sess->sessname,
+			       srv_dev->open_write_cnt,
+			       rnbd_access_mode_str(access_mode));
+		}
+		break;
+	case RNBD_ACCESS_MIGRATION:
+		if (srv_dev->open_write_cnt < 2) {
+			srv_dev->open_write_cnt++;
+			ret = 0;
+		} else {
+			pr_err("Mapping device '%s' for session %s with migration permissions failed. Device already opened as 'RW' by %d client(s), access mode %s.\n",
+			       srv_dev->id, srv_sess->sessname,
+			       srv_dev->open_write_cnt,
+			       rnbd_access_mode_str(access_mode));
+		}
+		break;
+	default:
+		pr_err("Received mapping request for device '%s' on session %s with invalid access mode: %d\n",
+		       srv_dev->id, srv_sess->sessname, access_mode);
+		ret = -EINVAL;
+	}
+
+	mutex_unlock(&srv_dev->lock);
+
+	return ret;
+}
+
+static struct rnbd_srv_dev *
+rnbd_srv_get_or_create_srv_dev(struct rnbd_dev *rnbd_dev,
+				struct rnbd_srv_session *srv_sess,
+				enum rnbd_access_mode access_mode)
+{
+	int ret;
+	struct rnbd_srv_dev *new_dev, *dev;
+
+	new_dev = rnbd_srv_init_srv_dev(rnbd_dev->name);
+	if (IS_ERR(new_dev))
+		return new_dev;
+
+	dev = rnbd_srv_find_or_add_srv_dev(new_dev);
+	if (dev != new_dev)
+		kfree(new_dev);
+
+	ret = rnbd_srv_check_update_open_perm(dev, srv_sess, access_mode);
+	if (ret) {
+		rnbd_put_srv_dev(dev);
+		return ERR_PTR(ret);
+	}
+
+	return dev;
+}
+
+static void rnbd_srv_fill_msg_open_rsp(struct rnbd_msg_open_rsp *rsp,
+					struct rnbd_srv_sess_dev *sess_dev)
+{
+	struct rnbd_dev *rnbd_dev = sess_dev->rnbd_dev;
+
+	rsp->hdr.type = cpu_to_le16(RNBD_MSG_OPEN_RSP);
+	rsp->device_id =
+		cpu_to_le32(sess_dev->device_id);
+	rsp->nsectors =
+		cpu_to_le64(get_capacity(rnbd_dev->bdev->bd_disk));
+	rsp->logical_block_size	=
+		cpu_to_le16(bdev_logical_block_size(rnbd_dev->bdev));
+	rsp->physical_block_size =
+		cpu_to_le16(bdev_physical_block_size(rnbd_dev->bdev));
+	rsp->max_segments =
+		cpu_to_le16(rnbd_dev_get_max_segs(rnbd_dev));
+	rsp->max_hw_sectors =
+		cpu_to_le32(rnbd_dev_get_max_hw_sects(rnbd_dev));
+	rsp->max_write_same_sectors =
+		cpu_to_le32(bdev_write_same(rnbd_dev->bdev));
+	rsp->max_discard_sectors =
+		cpu_to_le32(rnbd_dev_get_max_discard_sects(rnbd_dev));
+	rsp->discard_granularity =
+		cpu_to_le32(rnbd_dev_get_discard_granularity(rnbd_dev));
+	rsp->discard_alignment =
+		cpu_to_le32(rnbd_dev_get_discard_alignment(rnbd_dev));
+	rsp->secure_discard =
+		cpu_to_le16(rnbd_dev_get_secure_discard(rnbd_dev));
+	rsp->rotational =
+		!blk_queue_nonrot(bdev_get_queue(rnbd_dev->bdev));
+}
+
+static struct rnbd_srv_sess_dev *
+rnbd_srv_create_set_sess_dev(struct rnbd_srv_session *srv_sess,
+			      const struct rnbd_msg_open *open_msg,
+			      struct rnbd_dev *rnbd_dev, fmode_t open_flags,
+			      struct rnbd_srv_dev *srv_dev)
+{
+	struct rnbd_srv_sess_dev *sdev = rnbd_sess_dev_alloc(srv_sess);
+
+	if (IS_ERR(sdev))
+		return sdev;
+
+	kref_init(&sdev->kref);
+
+	strlcpy(sdev->pathname, open_msg->dev_name, sizeof(sdev->pathname));
+
+	sdev->rnbd_dev		= rnbd_dev;
+	sdev->sess		= srv_sess;
+	sdev->dev		= srv_dev;
+	sdev->open_flags	= open_flags;
+	sdev->access_mode	= open_msg->access_mode;
+
+	return sdev;
+}
+
+static char *rnbd_srv_get_full_path(struct rnbd_srv_session *srv_sess,
+				     const char *dev_name)
+{
+	char *full_path;
+	char *a, *b;
+
+	full_path = kmalloc(PATH_MAX, GFP_KERNEL);
+	if (!full_path)
+		return ERR_PTR(-ENOMEM);
+
+	/*
+	 * Replace %SESSNAME% with a real session name in order to
+	 * create device namespace.
+	 */
+	a = strnstr(dev_search_path, "%SESSNAME%", sizeof(dev_search_path));
+	if (a) {
+		int len = a - dev_search_path;
+
+		len = snprintf(full_path, PATH_MAX, "%.*s/%s/%s", len,
+			       dev_search_path, srv_sess->sessname, dev_name);
+		if (len >= PATH_MAX) {
+			pr_err("Too long path: %s, %s, %s\n",
+			       dev_search_path, srv_sess->sessname, dev_name);
+			kfree(full_path);
+			return ERR_PTR(-EINVAL);
+		}
+	} else {
+		snprintf(full_path, PATH_MAX, "%s/%s",
+			 dev_search_path, dev_name);
+	}
+
+	/* eliminitate duplicated slashes */
+	a = strchr(full_path, '/');
+	b = a;
+	while (*b != '\0') {
+		if (*b == '/' && *a == '/') {
+			b++;
+		} else {
+			a++;
+			*a = *b;
+			b++;
+		}
+	}
+	a++;
+	*a = '\0';
+
+	return full_path;
+}
+
+static int process_msg_sess_info(struct rtrs_srv *rtrs,
+				 struct rnbd_srv_session *srv_sess,
+				 const void *msg, size_t len,
+				 void *data, size_t datalen)
+{
+	const struct rnbd_msg_sess_info *sess_info_msg = msg;
+	struct rnbd_msg_sess_info_rsp *rsp = data;
+
+	srv_sess->ver = min_t(u8, sess_info_msg->ver, RNBD_PROTO_VER_MAJOR);
+	pr_debug("Session %s using protocol version %d (client version: %d, server version: %d)\n",
+		 srv_sess->sessname, srv_sess->ver,
+		 sess_info_msg->ver, RNBD_PROTO_VER_MAJOR);
+
+	rsp->hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO_RSP);
+	rsp->ver = srv_sess->ver;
+
+	return 0;
+}
+
+/**
+ * find_srv_sess_dev() - a dev is already opened by this name
+ * @srv_sess:	the session to search.
+ * @dev_name:	string containing the name of the device.
+ *
+ * Return struct rnbd_srv_sess_dev if srv_sess already opened the dev_name
+ * NULL if the session didn't open the device yet.
+ */
+static struct rnbd_srv_sess_dev *
+find_srv_sess_dev(struct rnbd_srv_session *srv_sess, const char *dev_name)
+{
+	struct rnbd_srv_sess_dev *sess_dev;
+
+	if (list_empty(&srv_sess->sess_dev_list))
+		return NULL;
+
+	list_for_each_entry(sess_dev, &srv_sess->sess_dev_list, sess_list)
+		if (!strcmp(sess_dev->pathname, dev_name))
+			return sess_dev;
+
+	return NULL;
+}
+
+static int process_msg_open(struct rtrs_srv *rtrs,
+			    struct rnbd_srv_session *srv_sess,
+			    const void *msg, size_t len,
+			    void *data, size_t datalen)
+{
+	int ret;
+	struct rnbd_srv_dev *srv_dev;
+	struct rnbd_srv_sess_dev *srv_sess_dev;
+	const struct rnbd_msg_open *open_msg = msg;
+	fmode_t open_flags;
+	char *full_path;
+	struct rnbd_dev *rnbd_dev;
+	struct rnbd_msg_open_rsp *rsp = data;
+
+	pr_debug("Open message received: session='%s' path='%s' access_mode=%d\n",
+		 srv_sess->sessname, open_msg->dev_name,
+		 open_msg->access_mode);
+	open_flags = FMODE_READ;
+	if (open_msg->access_mode != RNBD_ACCESS_RO)
+		open_flags |= FMODE_WRITE;
+
+	mutex_lock(&srv_sess->lock);
+
+	srv_sess_dev = find_srv_sess_dev(srv_sess, open_msg->dev_name);
+	if (srv_sess_dev)
+		goto fill_response;
+
+	if ((strlen(dev_search_path) + strlen(open_msg->dev_name))
+	    >= PATH_MAX) {
+		pr_err("Opening device for session %s failed, device path too long. '%s/%s' is longer than PATH_MAX (%d)\n",
+		       srv_sess->sessname, dev_search_path, open_msg->dev_name,
+		       PATH_MAX);
+		ret = -EINVAL;
+		goto reject;
+	}
+	if (strstr(open_msg->dev_name, "..")) {
+		pr_err("Opening device for session %s failed, device path %s contains relative path ..\n",
+		       srv_sess->sessname, open_msg->dev_name);
+		ret = -EINVAL;
+		goto reject;
+	}
+	full_path = rnbd_srv_get_full_path(srv_sess, open_msg->dev_name);
+	if (IS_ERR(full_path)) {
+		ret = PTR_ERR(full_path);
+		pr_err("Opening device '%s' for client %s failed, failed to get device full path, err: %d\n",
+		       open_msg->dev_name, srv_sess->sessname, ret);
+		goto reject;
+	}
+
+	rnbd_dev = rnbd_dev_open(full_path, open_flags,
+				 &srv_sess->sess_bio_set);
+	if (IS_ERR(rnbd_dev)) {
+		pr_err("Opening device '%s' on session %s failed, failed to open the block device, err: %ld\n",
+		       full_path, srv_sess->sessname, PTR_ERR(rnbd_dev));
+		ret = PTR_ERR(rnbd_dev);
+		goto free_path;
+	}
+
+	srv_dev = rnbd_srv_get_or_create_srv_dev(rnbd_dev, srv_sess,
+						  open_msg->access_mode);
+	if (IS_ERR(srv_dev)) {
+		pr_err("Opening device '%s' on session %s failed, creating srv_dev failed, err: %ld\n",
+		       full_path, srv_sess->sessname, PTR_ERR(srv_dev));
+		ret = PTR_ERR(srv_dev);
+		goto rnbd_dev_close;
+	}
+
+	srv_sess_dev = rnbd_srv_create_set_sess_dev(srv_sess, open_msg,
+						     rnbd_dev, open_flags,
+						     srv_dev);
+	if (IS_ERR(srv_sess_dev)) {
+		pr_err("Opening device '%s' on session %s failed, creating sess_dev failed, err: %ld\n",
+		       full_path, srv_sess->sessname, PTR_ERR(srv_sess_dev));
+		ret = PTR_ERR(srv_sess_dev);
+		goto srv_dev_put;
+	}
+
+	/* Create the srv_dev sysfs files if they haven't been created yet. The
+	 * reason to delay the creation is not to create the sysfs files before
+	 * we are sure the device can be opened.
+	 */
+	mutex_lock(&srv_dev->lock);
+	if (!srv_dev->dev_kobj.state_in_sysfs) {
+		ret = rnbd_srv_create_dev_sysfs(srv_dev, rnbd_dev->bdev,
+						 rnbd_dev->name);
+		if (ret) {
+			mutex_unlock(&srv_dev->lock);
+			rnbd_srv_err(srv_sess_dev,
+				      "Opening device failed, failed to create device sysfs files, err: %d\n",
+				      ret);
+			goto free_srv_sess_dev;
+		}
+	}
+
+	ret = rnbd_srv_create_dev_session_sysfs(srv_sess_dev);
+	if (ret) {
+		mutex_unlock(&srv_dev->lock);
+		rnbd_srv_err(srv_sess_dev,
+			      "Opening device failed, failed to create dev client sysfs files, err: %d\n",
+			      ret);
+		goto free_srv_sess_dev;
+	}
+
+	list_add(&srv_sess_dev->dev_list, &srv_dev->sess_dev_list);
+	mutex_unlock(&srv_dev->lock);
+
+	list_add(&srv_sess_dev->sess_list, &srv_sess->sess_dev_list);
+
+	rnbd_srv_info(srv_sess_dev, "Opened device '%s'\n", srv_dev->id);
+
+	kfree(full_path);
+
+fill_response:
+	rnbd_srv_fill_msg_open_rsp(rsp, srv_sess_dev);
+	mutex_unlock(&srv_sess->lock);
+	return 0;
+
+free_srv_sess_dev:
+	xa_erase(&srv_sess->index_idr, srv_sess_dev->device_id);
+	synchronize_rcu();
+	kfree(srv_sess_dev);
+srv_dev_put:
+	if (open_msg->access_mode != RNBD_ACCESS_RO) {
+		mutex_lock(&srv_dev->lock);
+		srv_dev->open_write_cnt--;
+		mutex_unlock(&srv_dev->lock);
+	}
+	rnbd_put_srv_dev(srv_dev);
+rnbd_dev_close:
+	rnbd_dev_close(rnbd_dev);
+free_path:
+	kfree(full_path);
+reject:
+	mutex_unlock(&srv_sess->lock);
+	return ret;
+}
+
+static struct rtrs_srv_ctx *rtrs_ctx;
+
+static struct rtrs_srv_ops rtrs_ops;
+static int __init rnbd_srv_init_module(void)
+{
+	int err;
+
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_hdr) != 4);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info) != 36);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info_rsp) != 36);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_open) != 264);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_close) != 8);
+	BUILD_BUG_ON(sizeof(struct rnbd_msg_open_rsp) != 56);
+	rtrs_ops = (struct rtrs_srv_ops) {
+		.rdma_ev = rnbd_srv_rdma_ev,
+		.link_ev = rnbd_srv_link_ev,
+	};
+	rtrs_ctx = rtrs_srv_open(&rtrs_ops, port_nr);
+	if (IS_ERR(rtrs_ctx)) {
+		err = PTR_ERR(rtrs_ctx);
+		pr_err("rtrs_srv_open(), err: %d\n", err);
+		return err;
+	}
+
+	err = rnbd_srv_create_sysfs_files();
+	if (err) {
+		pr_err("rnbd_srv_create_sysfs_files(), err: %d\n", err);
+		rtrs_srv_close(rtrs_ctx);
+		return err;
+	}
+
+	return 0;
+}
+
+static void __exit rnbd_srv_cleanup_module(void)
+{
+	rtrs_srv_close(rtrs_ctx);
+	WARN_ON(!list_empty(&sess_list));
+	rnbd_srv_destroy_sysfs_files();
+}
+
+module_init(rnbd_srv_init_module);
+module_exit(rnbd_srv_cleanup_module);
diff --git a/drivers/block/rnbd/rnbd-srv.h b/drivers/block/rnbd/rnbd-srv.h
new file mode 100644
index 000000000000..5a8544b5e74f
--- /dev/null
+++ b/drivers/block/rnbd/rnbd-srv.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#ifndef RNBD_SRV_H
+#define RNBD_SRV_H
+
+#include <linux/types.h>
+#include <linux/idr.h>
+#include <linux/kref.h>
+
+#include <rtrs.h>
+#include "rnbd-proto.h"
+#include "rnbd-log.h"
+
+struct rnbd_srv_session {
+	/* Entry inside global sess_list */
+	struct list_head        list;
+	struct rtrs_srv		*rtrs;
+	char			sessname[NAME_MAX];
+	int			queue_depth;
+	struct bio_set		sess_bio_set;
+
+	struct xarray		index_idr;
+	/* List of struct rnbd_srv_sess_dev */
+	struct list_head        sess_dev_list;
+	struct mutex		lock;
+	u8			ver;
+};
+
+struct rnbd_srv_dev {
+	/* Entry inside global dev_list */
+	struct list_head                list;
+	struct kobject                  dev_kobj;
+	struct kobject                  *dev_sessions_kobj;
+	struct kref                     kref;
+	char				id[NAME_MAX];
+	/* List of rnbd_srv_sess_dev structs */
+	struct list_head		sess_dev_list;
+	struct mutex			lock;
+	int				open_write_cnt;
+};
+
+/* Structure which binds N devices and N sessions */
+struct rnbd_srv_sess_dev {
+	/* Entry inside rnbd_srv_dev struct */
+	struct list_head		dev_list;
+	/* Entry inside rnbd_srv_session struct */
+	struct list_head		sess_list;
+	struct rnbd_dev			*rnbd_dev;
+	struct rnbd_srv_session		*sess;
+	struct rnbd_srv_dev		*dev;
+	struct kobject                  kobj;
+	u32                             device_id;
+	fmode_t                         open_flags;
+	struct kref			kref;
+	struct completion               *destroy_comp;
+	char				pathname[NAME_MAX];
+	enum rnbd_access_mode		access_mode;
+};
+
+/* rnbd-srv-sysfs.c */
+
+int rnbd_srv_create_dev_sysfs(struct rnbd_srv_dev *dev,
+			      struct block_device *bdev,
+			      const char *dir_name);
+void rnbd_srv_destroy_dev_sysfs(struct rnbd_srv_dev *dev);
+int rnbd_srv_create_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev);
+void rnbd_srv_destroy_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev);
+int rnbd_srv_create_sysfs_files(void);
+void rnbd_srv_destroy_sysfs_files(void);
+void rnbd_destroy_sess_dev(struct rnbd_srv_sess_dev *sess_dev);
+
+#endif /* RNBD_SRV_H */
diff --git a/drivers/block/umem.c b/drivers/block/umem.c
index d84e8a878df2..1e2aa5ae2796 100644
--- a/drivers/block/umem.c
+++ b/drivers/block/umem.c
@@ -784,7 +784,7 @@ static const struct block_device_operations mm_fops = {
 
 static int mm_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
-	int ret = -ENODEV;
+	int ret;
 	struct cardinfo *card = &cards[num_cards];
 	unsigned char	mem_present;
 	unsigned char	batt_status;
diff --git a/drivers/block/z2ram.c b/drivers/block/z2ram.c
index 600430685e28..0e734802ee7c 100644
--- a/drivers/block/z2ram.c
+++ b/drivers/block/z2ram.c
@@ -35,10 +35,10 @@
 #include <linux/bitops.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
+#include <linux/pgtable.h>
 
 #include <asm/setup.h>
 #include <asm/amigahw.h>
-#include <asm/pgtable.h>
 
 #include <linux/zorro.h>
 
diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c
index 5ee8e3fae551..33e3b76c4fa9 100644
--- a/drivers/block/zram/zcomp.c
+++ b/drivers/block/zram/zcomp.c
@@ -29,7 +29,6 @@ static const char * const backends[] = {
 #if IS_ENABLED(CONFIG_CRYPTO_ZSTD)
 	"zstd",
 #endif
-	NULL
 };
 
 static void zcomp_strm_free(struct zcomp_strm *zstrm)
@@ -64,7 +63,7 @@ bool zcomp_available_algorithm(const char *comp)
 {
 	int i;
 
-	i = __sysfs_match_string(backends, -1, comp);
+	i = sysfs_match_string(backends, comp);
 	if (i >= 0)
 		return true;
 
@@ -83,9 +82,9 @@ ssize_t zcomp_available_show(const char *comp, char *buf)
 {
 	bool known_algorithm = false;
 	ssize_t sz = 0;
-	int i = 0;
+	int i;
 
-	for (; backends[i]; i++) {
+	for (i = 0; i < ARRAY_SIZE(backends); i++) {
 		if (!strcmp(comp, backends[i])) {
 			known_algorithm = true;
 			sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 6d4e4497b59b..c8818e3b1079 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -20,6 +20,15 @@ config ARM_CCI400_PORT_CTRL
 	  Low level power management driver for CCI400 cache coherent
 	  interconnect for ARM platforms.
 
+config ARM_INTEGRATOR_LM
+	bool "ARM Integrator Logic Module bus"
+	depends on HAS_IOMEM
+	depends on ARCH_INTEGRATOR || COMPILE_TEST
+	default ARCH_INTEGRATOR
+	help
+	  Say y here to enable support for the ARM Logic Module bus
+	  found on the ARM Integrator AP (Application Platform)
+
 config BRCMSTB_GISB_ARB
 	bool "Broadcom STB GISB bus arbiter"
 	depends on ARM || ARM64 || MIPS
@@ -29,6 +38,36 @@ config BRCMSTB_GISB_ARB
 	  arbiter. This driver provides timeout and target abort error handling
 	  and internal bus master decoding.
 
+config BT1_APB
+	bool "Baikal-T1 APB-bus driver"
+	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
+	select REGMAP_MMIO
+	help
+	  Baikal-T1 AXI-APB bridge is used to access the SoC subsystem CSRs.
+	  IO requests are routed to this bus by means of the DW AMBA 3 AXI
+	  Interconnect. In case of any APB protocol collisions, slave device
+	  not responding on timeout an IRQ is raised with an erroneous address
+	  reported to the APB terminator (APB Errors Handler Block). This
+	  driver provides the interrupt handler to detect the erroneous
+	  address, prints an error message about the address fault, updates an
+	  errors counter. The counter and the APB-bus operations timeout can be
+	  accessed via corresponding sysfs nodes.
+
+config BT1_AXI
+	bool "Baikal-T1 AXI-bus driver"
+	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
+	select MFD_SYSCON
+	help
+	  AXI3-bus is the main communication bus connecting all high-speed
+	  peripheral IP-cores with RAM controller and with MIPS P5600 cores on
+	  Baikal-T1 SoC. Traffic arbitration is done by means of DW AMBA 3 AXI
+	  Interconnect (so called AXI Main Interconnect) routing IO requests
+	  from one SoC block to another. This driver provides a way to detect
+	  any bus protocol errors and device not responding situations by
+	  means of an embedded on top of the interconnect errors handler
+	  block (EHB). AXI Interconnect QoS arbitration tuning is currently
+	  unsupported.
+
 config MOXTET
 	tristate "CZ.NIC Turris Mox module configuration bus"
 	depends on SPI_MASTER && OF
@@ -183,7 +222,7 @@ config UNIPHIER_SYSTEM_BUS
 	  needed to use on-board devices connected to UniPhier SoCs.
 
 config VEXPRESS_CONFIG
-	bool "Versatile Express configuration bus"
+	tristate "Versatile Express configuration bus"
 	default y if ARCH_VEXPRESS
 	depends on ARM || ARM64
 	depends on OF
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index 05f32cd694a4..397e35392bff 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -5,7 +5,7 @@
 
 # Interconnect bus drivers for ARM platforms
 obj-$(CONFIG_ARM_CCI)		+= arm-cci.o
-
+obj-$(CONFIG_ARM_INTEGRATOR_LM)	+= arm-integrator-lm.o
 obj-$(CONFIG_HISILICON_LPC)	+= hisi_lpc.o
 obj-$(CONFIG_BRCMSTB_GISB_ARB)	+= brcmstb_gisb.o
 obj-$(CONFIG_MOXTET)		+= moxtet.o
@@ -13,6 +13,8 @@ obj-$(CONFIG_MOXTET)		+= moxtet.o
 # DPAA2 fsl-mc bus
 obj-$(CONFIG_FSL_MC_BUS)	+= fsl-mc/
 
+obj-$(CONFIG_BT1_APB)		+= bt1-apb.o
+obj-$(CONFIG_BT1_AXI)		+= bt1-axi.o
 obj-$(CONFIG_IMX_WEIM)		+= imx-weim.o
 obj-$(CONFIG_MIPS_CDMM)		+= mips_cdmm.o
 obj-$(CONFIG_MVEBU_MBUS) 	+= mvebu-mbus.o
diff --git a/drivers/bus/arm-integrator-lm.c b/drivers/bus/arm-integrator-lm.c
new file mode 100644
index 000000000000..845b6c43fef8
--- /dev/null
+++ b/drivers/bus/arm-integrator-lm.c
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ARM Integrator Logical Module bus driver
+ * Copyright (C) 2020 Linaro Ltd.
+ * Author: Linus Walleij <linus.walleij@linaro.org>
+ *
+ * See the device tree bindings for this block for more details on the
+ * hardware.
+ */
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/bitops.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+
+/* All information about the connected logic modules are in here */
+#define INTEGRATOR_SC_DEC_OFFSET	0x10
+
+/* Base address for the expansion modules */
+#define INTEGRATOR_AP_EXP_BASE		0xc0000000
+#define INTEGRATOR_AP_EXP_STRIDE	0x10000000
+
+static int integrator_lm_populate(int num, struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	struct device_node *child;
+	u32 base;
+	int ret;
+
+	base = INTEGRATOR_AP_EXP_BASE + (num * INTEGRATOR_AP_EXP_STRIDE);
+
+	/* Walk over the child nodes and see what chipselects we use */
+	for_each_available_child_of_node(np, child) {
+		struct resource res;
+
+		ret = of_address_to_resource(child, 0, &res);
+		if (ret) {
+			dev_info(dev, "no valid address on child\n");
+			continue;
+		}
+
+		/* First populate the syscon then any devices */
+		if (res.start == base) {
+			dev_info(dev, "populate module @0x%08x from DT\n",
+				 base);
+			ret = of_platform_default_populate(child, NULL, dev);
+			if (ret) {
+				dev_err(dev, "failed to populate module\n");
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static const struct of_device_id integrator_ap_syscon_match[] = {
+	{ .compatible = "arm,integrator-ap-syscon"},
+	{ },
+};
+
+static int integrator_ap_lm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *syscon;
+	static struct regmap *map;
+	u32 val;
+	int ret;
+	int i;
+
+	/* Look up the system controller */
+	syscon = of_find_matching_node(NULL, integrator_ap_syscon_match);
+	if (!syscon) {
+		dev_err(dev,
+			"could not find Integrator/AP system controller\n");
+		return -ENODEV;
+	}
+	map = syscon_node_to_regmap(syscon);
+	if (IS_ERR(map)) {
+		dev_err(dev,
+			"could not find Integrator/AP system controller\n");
+		return PTR_ERR(map);
+	}
+
+	ret = regmap_read(map, INTEGRATOR_SC_DEC_OFFSET, &val);
+	if (ret) {
+		dev_err(dev, "could not read from Integrator/AP syscon\n");
+		return ret;
+	}
+
+	/* Loop over the connected modules */
+	for (i = 0; i < 4; i++) {
+		if (!(val & BIT(4 + i)))
+			continue;
+
+		dev_info(dev, "detected module in slot %d\n", i);
+		ret = integrator_lm_populate(i, dev);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id integrator_ap_lm_match[] = {
+	{ .compatible = "arm,integrator-ap-lm"},
+	{ },
+};
+
+static struct platform_driver integrator_ap_lm_driver = {
+	.probe = integrator_ap_lm_probe,
+	.driver = {
+		.name = "integratorap-lm",
+		.of_match_table = integrator_ap_lm_match,
+	},
+};
+module_platform_driver(integrator_ap_lm_driver);
+MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
+MODULE_DESCRIPTION("Integrator AP Logical Module driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/bt1-apb.c b/drivers/bus/bt1-apb.c
new file mode 100644
index 000000000000..b25ff941e7c7
--- /dev/null
+++ b/drivers/bus/bt1-apb.c
@@ -0,0 +1,421 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *
+ * Baikal-T1 APB-bus driver
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/atomic.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/nmi.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/time64.h>
+#include <linux/clk.h>
+#include <linux/sysfs.h>
+
+#define APB_EHB_ISR			0x00
+#define APB_EHB_ISR_PENDING		BIT(0)
+#define APB_EHB_ISR_MASK		BIT(1)
+#define APB_EHB_ADDR			0x04
+#define APB_EHB_TIMEOUT			0x08
+
+#define APB_EHB_TIMEOUT_MIN		0x000003FFU
+#define APB_EHB_TIMEOUT_MAX		0xFFFFFFFFU
+
+/*
+ * struct bt1_apb - Baikal-T1 APB EHB private data
+ * @dev: Pointer to the device structure.
+ * @regs: APB EHB registers map.
+ * @res: No-device error injection memory region.
+ * @irq: Errors IRQ number.
+ * @rate: APB-bus reference clock rate.
+ * @pclk: APB-reference clock.
+ * @prst: APB domain reset line.
+ * @count: Number of errors detected.
+ */
+struct bt1_apb {
+	struct device *dev;
+
+	struct regmap *regs;
+	void __iomem *res;
+	int irq;
+
+	unsigned long rate;
+	struct clk *pclk;
+
+	struct reset_control *prst;
+
+	atomic_t count;
+};
+
+static const struct regmap_config bt1_apb_regmap_cfg = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = APB_EHB_TIMEOUT,
+	.fast_io = true
+};
+
+static inline unsigned long bt1_apb_n_to_timeout_us(struct bt1_apb *apb, u32 n)
+{
+	u64 timeout = (u64)n * USEC_PER_SEC;
+
+	do_div(timeout, apb->rate);
+
+	return timeout;
+
+}
+
+static inline unsigned long bt1_apb_timeout_to_n_us(struct bt1_apb *apb,
+						    unsigned long timeout)
+{
+	u64 n = (u64)timeout * apb->rate;
+
+	do_div(n, USEC_PER_SEC);
+
+	return n;
+
+}
+
+static irqreturn_t bt1_apb_isr(int irq, void *data)
+{
+	struct bt1_apb *apb = data;
+	u32 addr = 0;
+
+	regmap_read(apb->regs, APB_EHB_ADDR, &addr);
+
+	dev_crit_ratelimited(apb->dev,
+		"APB-bus fault %d: Slave access timeout at 0x%08x\n",
+		atomic_inc_return(&apb->count),
+		addr);
+
+	/*
+	 * Print backtrace on each CPU. This might be pointless if the fault
+	 * has happened on the same CPU as the IRQ handler is executed or
+	 * the other core proceeded further execution despite the error.
+	 * But if it's not, by looking at the trace we would get straight to
+	 * the cause of the problem.
+	 */
+	trigger_all_cpu_backtrace();
+
+	regmap_update_bits(apb->regs, APB_EHB_ISR, APB_EHB_ISR_PENDING, 0);
+
+	return IRQ_HANDLED;
+}
+
+static void bt1_apb_clear_data(void *data)
+{
+	struct bt1_apb *apb = data;
+	struct platform_device *pdev = to_platform_device(apb->dev);
+
+	platform_set_drvdata(pdev, NULL);
+}
+
+static struct bt1_apb *bt1_apb_create_data(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct bt1_apb *apb;
+	int ret;
+
+	apb = devm_kzalloc(dev, sizeof(*apb), GFP_KERNEL);
+	if (!apb)
+		return ERR_PTR(-ENOMEM);
+
+	ret = devm_add_action(dev, bt1_apb_clear_data, apb);
+	if (ret) {
+		dev_err(dev, "Can't add APB EHB data clear action\n");
+		return ERR_PTR(ret);
+	}
+
+	apb->dev = dev;
+	atomic_set(&apb->count, 0);
+	platform_set_drvdata(pdev, apb);
+
+	return apb;
+}
+
+static int bt1_apb_request_regs(struct bt1_apb *apb)
+{
+	struct platform_device *pdev = to_platform_device(apb->dev);
+	void __iomem *regs;
+
+	regs = devm_platform_ioremap_resource_byname(pdev, "ehb");
+	if (IS_ERR(regs)) {
+		dev_err(apb->dev, "Couldn't map APB EHB registers\n");
+		return PTR_ERR(regs);
+	}
+
+	apb->regs = devm_regmap_init_mmio(apb->dev, regs, &bt1_apb_regmap_cfg);
+	if (IS_ERR(apb->regs)) {
+		dev_err(apb->dev, "Couldn't create APB EHB regmap\n");
+		return PTR_ERR(apb->regs);
+	}
+
+	apb->res = devm_platform_ioremap_resource_byname(pdev, "nodev");
+	if (IS_ERR(apb->res))
+		dev_err(apb->dev, "Couldn't map reserved region\n");
+
+	return PTR_ERR_OR_ZERO(apb->res);
+}
+
+static int bt1_apb_request_rst(struct bt1_apb *apb)
+{
+	int ret;
+
+	apb->prst = devm_reset_control_get_optional_exclusive(apb->dev, "prst");
+	if (IS_ERR(apb->prst)) {
+		dev_warn(apb->dev, "Couldn't get reset control line\n");
+		return PTR_ERR(apb->prst);
+	}
+
+	ret = reset_control_deassert(apb->prst);
+	if (ret)
+		dev_err(apb->dev, "Failed to deassert the reset line\n");
+
+	return ret;
+}
+
+static void bt1_apb_disable_clk(void *data)
+{
+	struct bt1_apb *apb = data;
+
+	clk_disable_unprepare(apb->pclk);
+}
+
+static int bt1_apb_request_clk(struct bt1_apb *apb)
+{
+	int ret;
+
+	apb->pclk = devm_clk_get(apb->dev, "pclk");
+	if (IS_ERR(apb->pclk)) {
+		dev_err(apb->dev, "Couldn't get APB clock descriptor\n");
+		return PTR_ERR(apb->pclk);
+	}
+
+	ret = clk_prepare_enable(apb->pclk);
+	if (ret) {
+		dev_err(apb->dev, "Couldn't enable the APB clock\n");
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(apb->dev, bt1_apb_disable_clk, apb);
+	if (ret) {
+		dev_err(apb->dev, "Can't add APB EHB clocks disable action\n");
+		return ret;
+	}
+
+	apb->rate = clk_get_rate(apb->pclk);
+	if (!apb->rate) {
+		dev_err(apb->dev, "Invalid clock rate\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void bt1_apb_clear_irq(void *data)
+{
+	struct bt1_apb *apb = data;
+
+	regmap_update_bits(apb->regs, APB_EHB_ISR, APB_EHB_ISR_MASK, 0);
+}
+
+static int bt1_apb_request_irq(struct bt1_apb *apb)
+{
+	struct platform_device *pdev = to_platform_device(apb->dev);
+	int ret;
+
+	apb->irq = platform_get_irq(pdev, 0);
+	if (apb->irq < 0)
+		return apb->irq;
+
+	ret = devm_request_irq(apb->dev, apb->irq, bt1_apb_isr, IRQF_SHARED,
+			       "bt1-apb", apb);
+	if (ret) {
+		dev_err(apb->dev, "Couldn't request APB EHB IRQ\n");
+		return ret;
+	}
+
+	ret = devm_add_action(apb->dev, bt1_apb_clear_irq, apb);
+	if (ret) {
+		dev_err(apb->dev, "Can't add APB EHB IRQs clear action\n");
+		return ret;
+	}
+
+	/* Unmask IRQ and clear it' pending flag. */
+	regmap_update_bits(apb->regs, APB_EHB_ISR,
+			   APB_EHB_ISR_PENDING | APB_EHB_ISR_MASK,
+			   APB_EHB_ISR_MASK);
+
+	return 0;
+}
+
+static ssize_t count_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	struct bt1_apb *apb = dev_get_drvdata(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&apb->count));
+}
+static DEVICE_ATTR_RO(count);
+
+static ssize_t timeout_show(struct device *dev, struct device_attribute *attr,
+			    char *buf)
+{
+	struct bt1_apb *apb = dev_get_drvdata(dev);
+	unsigned long timeout;
+	int ret;
+	u32 n;
+
+	ret = regmap_read(apb->regs, APB_EHB_TIMEOUT, &n);
+	if (ret)
+		return ret;
+
+	timeout = bt1_apb_n_to_timeout_us(apb, n);
+
+	return scnprintf(buf, PAGE_SIZE, "%lu\n", timeout);
+}
+
+static ssize_t timeout_store(struct device *dev,
+			     struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	struct bt1_apb *apb = dev_get_drvdata(dev);
+	unsigned long timeout;
+	int ret;
+	u32 n;
+
+	if (kstrtoul(buf, 0, &timeout) < 0)
+		return -EINVAL;
+
+	n = bt1_apb_timeout_to_n_us(apb, timeout);
+	n = clamp(n, APB_EHB_TIMEOUT_MIN, APB_EHB_TIMEOUT_MAX);
+
+	ret = regmap_write(apb->regs, APB_EHB_TIMEOUT, n);
+
+	return ret ?: count;
+}
+static DEVICE_ATTR_RW(timeout);
+
+static ssize_t inject_error_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "Error injection: nodev irq\n");
+}
+
+static ssize_t inject_error_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *data, size_t count)
+{
+	struct bt1_apb *apb = dev_get_drvdata(dev);
+
+	/*
+	 * Either dummy read from the unmapped address in the APB IO area
+	 * or manually set the IRQ status.
+	 */
+	if (sysfs_streq(data, "nodev"))
+		readl(apb->res);
+	else if (sysfs_streq(data, "irq"))
+		regmap_update_bits(apb->regs, APB_EHB_ISR, APB_EHB_ISR_PENDING,
+				   APB_EHB_ISR_PENDING);
+	else
+		return -EINVAL;
+
+	return count;
+}
+static DEVICE_ATTR_RW(inject_error);
+
+static struct attribute *bt1_apb_sysfs_attrs[] = {
+	&dev_attr_count.attr,
+	&dev_attr_timeout.attr,
+	&dev_attr_inject_error.attr,
+	NULL
+};
+ATTRIBUTE_GROUPS(bt1_apb_sysfs);
+
+static void bt1_apb_remove_sysfs(void *data)
+{
+	struct bt1_apb *apb = data;
+
+	device_remove_groups(apb->dev, bt1_apb_sysfs_groups);
+}
+
+static int bt1_apb_init_sysfs(struct bt1_apb *apb)
+{
+	int ret;
+
+	ret = device_add_groups(apb->dev, bt1_apb_sysfs_groups);
+	if (ret) {
+		dev_err(apb->dev, "Failed to create EHB APB sysfs nodes\n");
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(apb->dev, bt1_apb_remove_sysfs, apb);
+	if (ret)
+		dev_err(apb->dev, "Can't add APB EHB sysfs remove action\n");
+
+	return ret;
+}
+
+static int bt1_apb_probe(struct platform_device *pdev)
+{
+	struct bt1_apb *apb;
+	int ret;
+
+	apb = bt1_apb_create_data(pdev);
+	if (IS_ERR(apb))
+		return PTR_ERR(apb);
+
+	ret = bt1_apb_request_regs(apb);
+	if (ret)
+		return ret;
+
+	ret = bt1_apb_request_rst(apb);
+	if (ret)
+		return ret;
+
+	ret = bt1_apb_request_clk(apb);
+	if (ret)
+		return ret;
+
+	ret = bt1_apb_request_irq(apb);
+	if (ret)
+		return ret;
+
+	ret = bt1_apb_init_sysfs(apb);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id bt1_apb_of_match[] = {
+	{ .compatible = "baikal,bt1-apb" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, bt1_apb_of_match);
+
+static struct platform_driver bt1_apb_driver = {
+	.probe = bt1_apb_probe,
+	.driver = {
+		.name = "bt1-apb",
+		.of_match_table = bt1_apb_of_match
+	}
+};
+module_platform_driver(bt1_apb_driver);
+
+MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
+MODULE_DESCRIPTION("Baikal-T1 APB-bus driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/bt1-axi.c b/drivers/bus/bt1-axi.c
new file mode 100644
index 000000000000..e7a6744acc7b
--- /dev/null
+++ b/drivers/bus/bt1-axi.c
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *
+ * Baikal-T1 AXI-bus driver
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/bitfield.h>
+#include <linux/device.h>
+#include <linux/atomic.h>
+#include <linux/regmap.h>
+#include <linux/platform_device.h>
+#include <linux/mfd/syscon.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/nmi.h>
+#include <linux/of.h>
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/sysfs.h>
+
+#define BT1_AXI_WERRL			0x110
+#define BT1_AXI_WERRH			0x114
+#define BT1_AXI_WERRH_TYPE		BIT(23)
+#define BT1_AXI_WERRH_ADDR_FLD		24
+#define BT1_AXI_WERRH_ADDR_MASK		GENMASK(31, BT1_AXI_WERRH_ADDR_FLD)
+
+/*
+ * struct bt1_axi - Baikal-T1 AXI-bus private data
+ * @dev: Pointer to the device structure.
+ * @qos_regs: AXI Interconnect QoS tuning registers.
+ * @sys_regs: Baikal-T1 System Controller registers map.
+ * @irq: Errors IRQ number.
+ * @aclk: AXI reference clock.
+ * @arst: AXI Interconnect reset line.
+ * @count: Number of errors detected.
+ */
+struct bt1_axi {
+	struct device *dev;
+
+	void __iomem *qos_regs;
+	struct regmap *sys_regs;
+	int irq;
+
+	struct clk *aclk;
+
+	struct reset_control *arst;
+
+	atomic_t count;
+};
+
+static irqreturn_t bt1_axi_isr(int irq, void *data)
+{
+	struct bt1_axi *axi = data;
+	u32 low = 0, high = 0;
+
+	regmap_read(axi->sys_regs, BT1_AXI_WERRL, &low);
+	regmap_read(axi->sys_regs, BT1_AXI_WERRH, &high);
+
+	dev_crit_ratelimited(axi->dev,
+		"AXI-bus fault %d: %s at 0x%x%08x\n",
+		atomic_inc_return(&axi->count),
+		high & BT1_AXI_WERRH_TYPE ? "no slave" : "slave protocol error",
+		high, low);
+
+	/*
+	 * Print backtrace on each CPU. This might be pointless if the fault
+	 * has happened on the same CPU as the IRQ handler is executed or
+	 * the other core proceeded further execution despite the error.
+	 * But if it's not, by looking at the trace we would get straight to
+	 * the cause of the problem.
+	 */
+	trigger_all_cpu_backtrace();
+
+	return IRQ_HANDLED;
+}
+
+static void bt1_axi_clear_data(void *data)
+{
+	struct bt1_axi *axi = data;
+	struct platform_device *pdev = to_platform_device(axi->dev);
+
+	platform_set_drvdata(pdev, NULL);
+}
+
+static struct bt1_axi *bt1_axi_create_data(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct bt1_axi *axi;
+	int ret;
+
+	axi = devm_kzalloc(dev, sizeof(*axi), GFP_KERNEL);
+	if (!axi)
+		return ERR_PTR(-ENOMEM);
+
+	ret = devm_add_action(dev, bt1_axi_clear_data, axi);
+	if (ret) {
+		dev_err(dev, "Can't add AXI EHB data clear action\n");
+		return ERR_PTR(ret);
+	}
+
+	axi->dev = dev;
+	atomic_set(&axi->count, 0);
+	platform_set_drvdata(pdev, axi);
+
+	return axi;
+}
+
+static int bt1_axi_request_regs(struct bt1_axi *axi)
+{
+	struct platform_device *pdev = to_platform_device(axi->dev);
+	struct device *dev = axi->dev;
+
+	axi->sys_regs = syscon_regmap_lookup_by_phandle(dev->of_node, "syscon");
+	if (IS_ERR(axi->sys_regs)) {
+		dev_err(dev, "Couldn't find syscon registers\n");
+		return PTR_ERR(axi->sys_regs);
+	}
+
+	axi->qos_regs = devm_platform_ioremap_resource_byname(pdev, "qos");
+	if (IS_ERR(axi->qos_regs))
+		dev_err(dev, "Couldn't map AXI-bus QoS registers\n");
+
+	return PTR_ERR_OR_ZERO(axi->qos_regs);
+}
+
+static int bt1_axi_request_rst(struct bt1_axi *axi)
+{
+	int ret;
+
+	axi->arst = devm_reset_control_get_optional_exclusive(axi->dev, "arst");
+	if (IS_ERR(axi->arst)) {
+		dev_warn(axi->dev, "Couldn't get reset control line\n");
+		return PTR_ERR(axi->arst);
+	}
+
+	ret = reset_control_deassert(axi->arst);
+	if (ret)
+		dev_err(axi->dev, "Failed to deassert the reset line\n");
+
+	return ret;
+}
+
+static void bt1_axi_disable_clk(void *data)
+{
+	struct bt1_axi *axi = data;
+
+	clk_disable_unprepare(axi->aclk);
+}
+
+static int bt1_axi_request_clk(struct bt1_axi *axi)
+{
+	int ret;
+
+	axi->aclk = devm_clk_get(axi->dev, "aclk");
+	if (IS_ERR(axi->aclk)) {
+		dev_err(axi->dev, "Couldn't get AXI Interconnect clock\n");
+		return PTR_ERR(axi->aclk);
+	}
+
+	ret = clk_prepare_enable(axi->aclk);
+	if (ret) {
+		dev_err(axi->dev, "Couldn't enable the AXI clock\n");
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(axi->dev, bt1_axi_disable_clk, axi);
+	if (ret)
+		dev_err(axi->dev, "Can't add AXI clock disable action\n");
+
+	return ret;
+}
+
+static int bt1_axi_request_irq(struct bt1_axi *axi)
+{
+	struct platform_device *pdev = to_platform_device(axi->dev);
+	int ret;
+
+	axi->irq = platform_get_irq(pdev, 0);
+	if (axi->irq < 0)
+		return axi->irq;
+
+	ret = devm_request_irq(axi->dev, axi->irq, bt1_axi_isr, IRQF_SHARED,
+			       "bt1-axi", axi);
+	if (ret)
+		dev_err(axi->dev, "Couldn't request AXI EHB IRQ\n");
+
+	return ret;
+}
+
+static ssize_t count_show(struct device *dev,
+			  struct device_attribute *attr, char *buf)
+{
+	struct bt1_axi *axi = dev_get_drvdata(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n", atomic_read(&axi->count));
+}
+static DEVICE_ATTR_RO(count);
+
+static ssize_t inject_error_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "Error injection: bus unaligned\n");
+}
+
+static ssize_t inject_error_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *data, size_t count)
+{
+	struct bt1_axi *axi = dev_get_drvdata(dev);
+
+	/*
+	 * Performing unaligned read from the memory will cause the CM2 bus
+	 * error while unaligned writing - the AXI bus write error handled
+	 * by this driver.
+	 */
+	if (sysfs_streq(data, "bus"))
+		readb(axi->qos_regs);
+	else if (sysfs_streq(data, "unaligned"))
+		writeb(0, axi->qos_regs);
+	else
+		return -EINVAL;
+
+	return count;
+}
+static DEVICE_ATTR_RW(inject_error);
+
+static struct attribute *bt1_axi_sysfs_attrs[] = {
+	&dev_attr_count.attr,
+	&dev_attr_inject_error.attr,
+	NULL
+};
+ATTRIBUTE_GROUPS(bt1_axi_sysfs);
+
+static void bt1_axi_remove_sysfs(void *data)
+{
+	struct bt1_axi *axi = data;
+
+	device_remove_groups(axi->dev, bt1_axi_sysfs_groups);
+}
+
+static int bt1_axi_init_sysfs(struct bt1_axi *axi)
+{
+	int ret;
+
+	ret = device_add_groups(axi->dev, bt1_axi_sysfs_groups);
+	if (ret) {
+		dev_err(axi->dev, "Failed to add sysfs files group\n");
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(axi->dev, bt1_axi_remove_sysfs, axi);
+	if (ret)
+		dev_err(axi->dev, "Can't add AXI EHB sysfs remove action\n");
+
+	return ret;
+}
+
+static int bt1_axi_probe(struct platform_device *pdev)
+{
+	struct bt1_axi *axi;
+	int ret;
+
+	axi = bt1_axi_create_data(pdev);
+	if (IS_ERR(axi))
+		return PTR_ERR(axi);
+
+	ret = bt1_axi_request_regs(axi);
+	if (ret)
+		return ret;
+
+	ret = bt1_axi_request_rst(axi);
+	if (ret)
+		return ret;
+
+	ret = bt1_axi_request_clk(axi);
+	if (ret)
+		return ret;
+
+	ret = bt1_axi_request_irq(axi);
+	if (ret)
+		return ret;
+
+	ret = bt1_axi_init_sysfs(axi);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id bt1_axi_of_match[] = {
+	{ .compatible = "baikal,bt1-axi" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, bt1_axi_of_match);
+
+static struct platform_driver bt1_axi_driver = {
+	.probe = bt1_axi_probe,
+	.driver = {
+		.name = "bt1-axi",
+		.of_match_table = bt1_axi_of_match
+	}
+};
+module_platform_driver(bt1_axi_driver);
+
+MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
+MODULE_DESCRIPTION("Baikal-T1 AXI-bus driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/mhi/core/boot.c b/drivers/bus/mhi/core/boot.c
index ebad5eb48e5a..0b38014d040e 100644
--- a/drivers/bus/mhi/core/boot.c
+++ b/drivers/bus/mhi/core/boot.c
@@ -43,10 +43,7 @@ void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl,
 		      lower_32_bits(mhi_buf->dma_addr));
 
 	mhi_write_reg(mhi_cntrl, base, BHIE_RXVECSIZE_OFFS, mhi_buf->len);
-	sequence_id = prandom_u32() & BHIE_RXVECSTATUS_SEQNUM_BMSK;
-
-	if (unlikely(!sequence_id))
-		sequence_id = 1;
+	sequence_id = MHI_RANDOM_U32_NONZERO(BHIE_RXVECSTATUS_SEQNUM_BMSK);
 
 	mhi_write_reg_field(mhi_cntrl, base, BHIE_RXVECDB_OFFS,
 			    BHIE_RXVECDB_SEQNUM_BMSK, BHIE_RXVECDB_SEQNUM_SHFT,
@@ -121,7 +118,8 @@ static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl)
 		ee = mhi_get_exec_env(mhi_cntrl);
 	}
 
-	dev_dbg(dev, "Waiting for image download completion, current EE: %s\n",
+	dev_dbg(dev,
+		"Waiting for RDDM image download via BHIe, current EE:%s\n",
 		TO_MHI_EXEC_STR(ee));
 
 	while (retry--) {
@@ -152,11 +150,14 @@ static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl)
 int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic)
 {
 	void __iomem *base = mhi_cntrl->bhie;
+	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 	u32 rx_status;
 
 	if (in_panic)
 		return __mhi_download_rddm_in_panic(mhi_cntrl);
 
+	dev_dbg(dev, "Waiting for RDDM image download via BHIe\n");
+
 	/* Wait for the image download to complete */
 	wait_event_timeout(mhi_cntrl->state_event,
 			   mhi_read_reg_field(mhi_cntrl, base,
@@ -174,8 +175,10 @@ static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl,
 			    const struct mhi_buf *mhi_buf)
 {
 	void __iomem *base = mhi_cntrl->bhie;
+	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 	rwlock_t *pm_lock = &mhi_cntrl->pm_lock;
 	u32 tx_status, sequence_id;
+	int ret;
 
 	read_lock_bh(pm_lock);
 	if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
@@ -183,6 +186,9 @@ static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl,
 		return -EIO;
 	}
 
+	sequence_id = MHI_RANDOM_U32_NONZERO(BHIE_TXVECSTATUS_SEQNUM_BMSK);
+	dev_dbg(dev, "Starting AMSS download via BHIe. Sequence ID:%u\n",
+		sequence_id);
 	mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_HIGH_OFFS,
 		      upper_32_bits(mhi_buf->dma_addr));
 
@@ -191,26 +197,25 @@ static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl,
 
 	mhi_write_reg(mhi_cntrl, base, BHIE_TXVECSIZE_OFFS, mhi_buf->len);
 
-	sequence_id = prandom_u32() & BHIE_TXVECSTATUS_SEQNUM_BMSK;
 	mhi_write_reg_field(mhi_cntrl, base, BHIE_TXVECDB_OFFS,
 			    BHIE_TXVECDB_SEQNUM_BMSK, BHIE_TXVECDB_SEQNUM_SHFT,
 			    sequence_id);
 	read_unlock_bh(pm_lock);
 
 	/* Wait for the image download to complete */
-	wait_event_timeout(mhi_cntrl->state_event,
-			   MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
-			   mhi_read_reg_field(mhi_cntrl, base,
-					      BHIE_TXVECSTATUS_OFFS,
-					      BHIE_TXVECSTATUS_STATUS_BMSK,
-					      BHIE_TXVECSTATUS_STATUS_SHFT,
-					      &tx_status) || tx_status,
-			   msecs_to_jiffies(mhi_cntrl->timeout_ms));
-
-	if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+	ret = wait_event_timeout(mhi_cntrl->state_event,
+				 MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
+				 mhi_read_reg_field(mhi_cntrl, base,
+						   BHIE_TXVECSTATUS_OFFS,
+						   BHIE_TXVECSTATUS_STATUS_BMSK,
+						   BHIE_TXVECSTATUS_STATUS_SHFT,
+						   &tx_status) || tx_status,
+				 msecs_to_jiffies(mhi_cntrl->timeout_ms));
+	if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
+	    tx_status != BHIE_TXVECSTATUS_STATUS_XFER_COMPL)
 		return -EIO;
 
-	return (tx_status == BHIE_TXVECSTATUS_STATUS_XFER_COMPL) ? 0 : -EIO;
+	return (!ret) ? -ETIMEDOUT : 0;
 }
 
 static int mhi_fw_load_sbl(struct mhi_controller *mhi_cntrl,
@@ -239,14 +244,15 @@ static int mhi_fw_load_sbl(struct mhi_controller *mhi_cntrl,
 		goto invalid_pm_state;
 	}
 
-	dev_dbg(dev, "Starting SBL download via BHI\n");
+	session_id = MHI_RANDOM_U32_NONZERO(BHI_TXDB_SEQNUM_BMSK);
+	dev_dbg(dev, "Starting SBL download via BHI. Session ID:%u\n",
+		session_id);
 	mhi_write_reg(mhi_cntrl, base, BHI_STATUS, 0);
 	mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_HIGH,
 		      upper_32_bits(dma_addr));
 	mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_LOW,
 		      lower_32_bits(dma_addr));
 	mhi_write_reg(mhi_cntrl, base, BHI_IMGSIZE, size);
-	session_id = prandom_u32() & BHI_TXDB_SEQNUM_BMSK;
 	mhi_write_reg(mhi_cntrl, base, BHI_IMGTXDB, session_id);
 	read_unlock_bh(pm_lock);
 
@@ -377,30 +383,18 @@ static void mhi_firmware_copy(struct mhi_controller *mhi_cntrl,
 	}
 }
 
-void mhi_fw_load_worker(struct work_struct *work)
+void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl)
 {
-	struct mhi_controller *mhi_cntrl;
 	const struct firmware *firmware = NULL;
 	struct image_info *image_info;
-	struct device *dev;
+	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 	const char *fw_name;
 	void *buf;
 	dma_addr_t dma_addr;
 	size_t size;
 	int ret;
 
-	mhi_cntrl = container_of(work, struct mhi_controller, fw_worker);
-	dev = &mhi_cntrl->mhi_dev->dev;
-
-	dev_dbg(dev, "Waiting for device to enter PBL from: %s\n",
-		TO_MHI_EXEC_STR(mhi_cntrl->ee));
-
-	ret = wait_event_timeout(mhi_cntrl->state_event,
-				 MHI_IN_PBL(mhi_cntrl->ee) ||
-				 MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
-				 msecs_to_jiffies(mhi_cntrl->timeout_ms));
-
-	if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+	if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
 		dev_err(dev, "Device MHI is not in valid state\n");
 		return;
 	}
@@ -446,7 +440,12 @@ void mhi_fw_load_worker(struct work_struct *work)
 		release_firmware(firmware);
 
 	/* Error or in EDL mode, we're done */
-	if (ret || mhi_cntrl->ee == MHI_EE_EDL)
+	if (ret) {
+		dev_err(dev, "MHI did not load SBL, ret:%d\n", ret);
+		return;
+	}
+
+	if (mhi_cntrl->ee == MHI_EE_EDL)
 		return;
 
 	write_lock_irq(&mhi_cntrl->pm_lock);
@@ -474,8 +473,10 @@ fw_load_ee_pthru:
 	if (!mhi_cntrl->fbc_download)
 		return;
 
-	if (ret)
+	if (ret) {
+		dev_err(dev, "MHI did not enter READY state\n");
 		goto error_read;
+	}
 
 	/* Wait for the SBL event */
 	ret = wait_event_timeout(mhi_cntrl->state_event,
@@ -493,6 +494,8 @@ fw_load_ee_pthru:
 	ret = mhi_fw_load_amss(mhi_cntrl,
 			       /* Vector table is the last entry */
 			       &image_info->mhi_buf[image_info->entries - 1]);
+	if (ret)
+		dev_err(dev, "MHI did not load AMSS, ret:%d\n", ret);
 
 	release_firmware(firmware);
 
diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c
index 1f8c82603179..e43a190a7a36 100644
--- a/drivers/bus/mhi/core/init.c
+++ b/drivers/bus/mhi/core/init.c
@@ -34,6 +34,8 @@ const char * const dev_state_tran_str[DEV_ST_TRANSITION_MAX] = {
 	[DEV_ST_TRANSITION_READY] = "READY",
 	[DEV_ST_TRANSITION_SBL] = "SBL",
 	[DEV_ST_TRANSITION_MISSION_MODE] = "MISSION_MODE",
+	[DEV_ST_TRANSITION_SYS_ERR] = "SYS_ERR",
+	[DEV_ST_TRANSITION_DISABLE] = "DISABLE",
 };
 
 const char * const mhi_state_str[MHI_STATE_MAX] = {
@@ -835,8 +837,6 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl,
 	spin_lock_init(&mhi_cntrl->transition_lock);
 	spin_lock_init(&mhi_cntrl->wlock);
 	INIT_WORK(&mhi_cntrl->st_worker, mhi_pm_st_worker);
-	INIT_WORK(&mhi_cntrl->syserr_worker, mhi_pm_sys_err_worker);
-	INIT_WORK(&mhi_cntrl->fw_worker, mhi_fw_load_worker);
 	init_waitqueue_head(&mhi_cntrl->state_event);
 
 	mhi_cmd = mhi_cntrl->mhi_cmd;
@@ -864,6 +864,10 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl,
 		mutex_init(&mhi_chan->mutex);
 		init_completion(&mhi_chan->completion);
 		rwlock_init(&mhi_chan->lock);
+
+		/* used in setting bei field of TRE */
+		mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
+		mhi_chan->intmod = mhi_event->intmod;
 	}
 
 	if (mhi_cntrl->bounce_buf) {
diff --git a/drivers/bus/mhi/core/internal.h b/drivers/bus/mhi/core/internal.h
index 095d95bc0e37..b1f640b75a94 100644
--- a/drivers/bus/mhi/core/internal.h
+++ b/drivers/bus/mhi/core/internal.h
@@ -386,6 +386,8 @@ enum dev_st_transition {
 	DEV_ST_TRANSITION_READY,
 	DEV_ST_TRANSITION_SBL,
 	DEV_ST_TRANSITION_MISSION_MODE,
+	DEV_ST_TRANSITION_SYS_ERR,
+	DEV_ST_TRANSITION_DISABLE,
 	DEV_ST_TRANSITION_MAX,
 };
 
@@ -452,6 +454,7 @@ enum mhi_pm_state {
 #define PRIMARY_CMD_RING		0
 #define MHI_DEV_WAKE_DB			127
 #define MHI_MAX_MTU			0xffff
+#define MHI_RANDOM_U32_NONZERO(bmsk)	(prandom_u32_max(bmsk) + 1)
 
 enum mhi_er_type {
 	MHI_ER_TYPE_INVALID = 0x0,
@@ -586,7 +589,7 @@ enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl);
 int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl,
 			       enum dev_st_transition state);
 void mhi_pm_st_worker(struct work_struct *work);
-void mhi_pm_sys_err_worker(struct work_struct *work);
+void mhi_pm_sys_err_handler(struct mhi_controller *mhi_cntrl);
 void mhi_fw_load_worker(struct work_struct *work);
 int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl);
 void mhi_ctrl_ev_task(unsigned long data);
@@ -627,6 +630,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl);
 void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl);
 void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl,
 		      struct image_info *img_info);
+void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl);
 int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
 			struct mhi_chan *mhi_chan);
 int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl,
@@ -670,8 +674,7 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *dev);
 irqreturn_t mhi_intvec_handler(int irq_number, void *dev);
 
 int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
-		void *buf, void *cb, size_t buf_len, enum mhi_flags flags);
-
+		struct mhi_buf_info *info, enum mhi_flags flags);
 int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl,
 			 struct mhi_buf_info *buf_info);
 int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl,
diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c
index 97e06cc586e4..1f622ce6be8b 100644
--- a/drivers/bus/mhi/core/main.c
+++ b/drivers/bus/mhi/core/main.c
@@ -258,7 +258,7 @@ int mhi_destroy_device(struct device *dev, void *data)
 	return 0;
 }
 
-static void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
+void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
 {
 	struct mhi_driver *mhi_drv;
 
@@ -270,6 +270,7 @@ static void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason)
 	if (mhi_drv->status_cb)
 		mhi_drv->status_cb(mhi_dev, cb_reason);
 }
+EXPORT_SYMBOL_GPL(mhi_notify);
 
 /* Bind MHI channels to MHI devices */
 void mhi_create_devices(struct mhi_controller *mhi_cntrl)
@@ -368,30 +369,37 @@ irqreturn_t mhi_irq_handler(int irq_number, void *dev)
 	return IRQ_HANDLED;
 }
 
-irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *dev)
+irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv)
 {
-	struct mhi_controller *mhi_cntrl = dev;
+	struct mhi_controller *mhi_cntrl = priv;
+	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 	enum mhi_state state = MHI_STATE_MAX;
 	enum mhi_pm_state pm_state = 0;
 	enum mhi_ee_type ee = 0;
 
 	write_lock_irq(&mhi_cntrl->pm_lock);
-	if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
-		state = mhi_get_mhi_state(mhi_cntrl);
-		ee = mhi_cntrl->ee;
-		mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
+	if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+		write_unlock_irq(&mhi_cntrl->pm_lock);
+		goto exit_intvec;
 	}
 
+	state = mhi_get_mhi_state(mhi_cntrl);
+	ee = mhi_cntrl->ee;
+	mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
+	dev_dbg(dev, "local ee:%s device ee:%s dev_state:%s\n",
+		TO_MHI_EXEC_STR(mhi_cntrl->ee), TO_MHI_EXEC_STR(ee),
+		TO_MHI_STATE_STR(state));
+
 	if (state == MHI_STATE_SYS_ERR) {
-		dev_dbg(&mhi_cntrl->mhi_dev->dev, "System error detected\n");
+		dev_dbg(dev, "System error detected\n");
 		pm_state = mhi_tryset_pm_state(mhi_cntrl,
 					       MHI_PM_SYS_ERR_DETECT);
 	}
 	write_unlock_irq(&mhi_cntrl->pm_lock);
 
-	/* If device in RDDM don't bother processing SYS error */
-	if (mhi_cntrl->ee == MHI_EE_RDDM) {
-		if (mhi_cntrl->ee != ee) {
+	 /* If device supports RDDM don't bother processing SYS error */
+	if (mhi_cntrl->rddm_image) {
+		if (mhi_cntrl->ee == MHI_EE_RDDM && mhi_cntrl->ee != ee) {
 			mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM);
 			wake_up_all(&mhi_cntrl->state_event);
 		}
@@ -405,7 +413,7 @@ irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *dev)
 		if (MHI_IN_PBL(ee))
 			mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR);
 		else
-			schedule_work(&mhi_cntrl->syserr_worker);
+			mhi_pm_sys_err_handler(mhi_cntrl);
 	}
 
 exit_intvec:
@@ -513,7 +521,10 @@ static int parse_xfer_event(struct mhi_controller *mhi_cntrl,
 				mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
 
 			result.buf_addr = buf_info->cb_buf;
-			result.bytes_xferd = xfer_len;
+
+			/* truncate to buf len if xfer_len is larger */
+			result.bytes_xferd =
+				min_t(u16, xfer_len, buf_info->len);
 			mhi_del_ring_element(mhi_cntrl, buf_ring);
 			mhi_del_ring_element(mhi_cntrl, tre_ring);
 			local_rp = tre_ring->rp;
@@ -597,7 +608,9 @@ static int parse_rsc_event(struct mhi_controller *mhi_cntrl,
 
 	result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
 		-EOVERFLOW : 0;
-	result.bytes_xferd = xfer_len;
+
+	/* truncate to buf len if xfer_len is larger */
+	result.bytes_xferd = min_t(u16, xfer_len, buf_info->len);
 	result.buf_addr = buf_info->cb_buf;
 	result.dir = mhi_chan->dir;
 
@@ -722,13 +735,18 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
 			{
 				enum mhi_pm_state new_state;
 
+				/* skip SYS_ERROR handling if RDDM supported */
+				if (mhi_cntrl->ee == MHI_EE_RDDM ||
+				    mhi_cntrl->rddm_image)
+					break;
+
 				dev_dbg(dev, "System error detected\n");
 				write_lock_irq(&mhi_cntrl->pm_lock);
 				new_state = mhi_tryset_pm_state(mhi_cntrl,
 							MHI_PM_SYS_ERR_DETECT);
 				write_unlock_irq(&mhi_cntrl->pm_lock);
 				if (new_state == MHI_PM_SYS_ERR_DETECT)
-					schedule_work(&mhi_cntrl->syserr_worker);
+					mhi_pm_sys_err_handler(mhi_cntrl);
 				break;
 			}
 			default:
@@ -774,9 +792,18 @@ int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
 		}
 		case MHI_PKT_TYPE_TX_EVENT:
 			chan = MHI_TRE_GET_EV_CHID(local_rp);
-			mhi_chan = &mhi_cntrl->mhi_chan[chan];
-			parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
-			event_quota--;
+
+			WARN_ON(chan >= mhi_cntrl->max_chan);
+
+			/*
+			 * Only process the event ring elements whose channel
+			 * ID is within the maximum supported range.
+			 */
+			if (chan < mhi_cntrl->max_chan) {
+				mhi_chan = &mhi_cntrl->mhi_chan[chan];
+				parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
+				event_quota--;
+			}
 			break;
 		default:
 			dev_err(dev, "Unhandled event type: %d\n", type);
@@ -819,14 +846,23 @@ int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
 		enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp);
 
 		chan = MHI_TRE_GET_EV_CHID(local_rp);
-		mhi_chan = &mhi_cntrl->mhi_chan[chan];
-
-		if (likely(type == MHI_PKT_TYPE_TX_EVENT)) {
-			parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
-			event_quota--;
-		} else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) {
-			parse_rsc_event(mhi_cntrl, local_rp, mhi_chan);
-			event_quota--;
+
+		WARN_ON(chan >= mhi_cntrl->max_chan);
+
+		/*
+		 * Only process the event ring elements whose channel
+		 * ID is within the maximum supported range.
+		 */
+		if (chan < mhi_cntrl->max_chan) {
+			mhi_chan = &mhi_cntrl->mhi_chan[chan];
+
+			if (likely(type == MHI_PKT_TYPE_TX_EVENT)) {
+				parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
+				event_quota--;
+			} else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) {
+				parse_rsc_event(mhi_cntrl, local_rp, mhi_chan);
+				event_quota--;
+			}
 		}
 
 		mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
@@ -896,7 +932,7 @@ void mhi_ctrl_ev_task(unsigned long data)
 		}
 		write_unlock_irq(&mhi_cntrl->pm_lock);
 		if (pm_state == MHI_PM_SYS_ERR_DETECT)
-			schedule_work(&mhi_cntrl->syserr_worker);
+			mhi_pm_sys_err_handler(mhi_cntrl);
 	}
 }
 
@@ -918,9 +954,7 @@ int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 	struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
 							     mhi_dev->dl_chan;
 	struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
-	struct mhi_ring *buf_ring = &mhi_chan->buf_ring;
-	struct mhi_buf_info *buf_info;
-	struct mhi_tre *mhi_tre;
+	struct mhi_buf_info buf_info = { };
 	int ret;
 
 	/* If MHI host pre-allocates buffers then client drivers cannot queue */
@@ -945,27 +979,15 @@ int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 	/* Toggle wake to exit out of M2 */
 	mhi_cntrl->wake_toggle(mhi_cntrl);
 
-	/* Generate the TRE */
-	buf_info = buf_ring->wp;
+	buf_info.v_addr = skb->data;
+	buf_info.cb_buf = skb;
+	buf_info.len = len;
 
-	buf_info->v_addr = skb->data;
-	buf_info->cb_buf = skb;
-	buf_info->wp = tre_ring->wp;
-	buf_info->dir = mhi_chan->dir;
-	buf_info->len = len;
-	ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
-	if (ret)
-		goto map_error;
-
-	mhi_tre = tre_ring->wp;
-
-	mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
-	mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_info->len);
-	mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(1, 1, 0, 0);
-
-	/* increment WP */
-	mhi_add_ring_element(mhi_cntrl, tre_ring);
-	mhi_add_ring_element(mhi_cntrl, buf_ring);
+	ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
+	if (unlikely(ret)) {
+		read_unlock_bh(&mhi_cntrl->pm_lock);
+		return ret;
+	}
 
 	if (mhi_chan->dir == DMA_TO_DEVICE)
 		atomic_inc(&mhi_cntrl->pending_pkts);
@@ -979,11 +1001,6 @@ int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 	read_unlock_bh(&mhi_cntrl->pm_lock);
 
 	return 0;
-
-map_error:
-	read_unlock_bh(&mhi_cntrl->pm_lock);
-
-	return ret;
 }
 EXPORT_SYMBOL_GPL(mhi_queue_skb);
 
@@ -995,9 +1012,8 @@ int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 							     mhi_dev->dl_chan;
 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 	struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
-	struct mhi_ring *buf_ring = &mhi_chan->buf_ring;
-	struct mhi_buf_info *buf_info;
-	struct mhi_tre *mhi_tre;
+	struct mhi_buf_info buf_info = { };
+	int ret;
 
 	/* If MHI host pre-allocates buffers then client drivers cannot queue */
 	if (mhi_chan->pre_alloc)
@@ -1024,25 +1040,16 @@ int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 	/* Toggle wake to exit out of M2 */
 	mhi_cntrl->wake_toggle(mhi_cntrl);
 
-	/* Generate the TRE */
-	buf_info = buf_ring->wp;
-	WARN_ON(buf_info->used);
-	buf_info->p_addr = mhi_buf->dma_addr;
-	buf_info->pre_mapped = true;
-	buf_info->cb_buf = mhi_buf;
-	buf_info->wp = tre_ring->wp;
-	buf_info->dir = mhi_chan->dir;
-	buf_info->len = len;
-
-	mhi_tre = tre_ring->wp;
-
-	mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
-	mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_info->len);
-	mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(1, 1, 0, 0);
+	buf_info.p_addr = mhi_buf->dma_addr;
+	buf_info.cb_buf = mhi_buf;
+	buf_info.pre_mapped = true;
+	buf_info.len = len;
 
-	/* increment WP */
-	mhi_add_ring_element(mhi_cntrl, tre_ring);
-	mhi_add_ring_element(mhi_cntrl, buf_ring);
+	ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
+	if (unlikely(ret)) {
+		read_unlock_bh(&mhi_cntrl->pm_lock);
+		return ret;
+	}
 
 	if (mhi_chan->dir == DMA_TO_DEVICE)
 		atomic_inc(&mhi_cntrl->pending_pkts);
@@ -1060,7 +1067,7 @@ int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 EXPORT_SYMBOL_GPL(mhi_queue_dma);
 
 int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
-		void *buf, void *cb, size_t buf_len, enum mhi_flags flags)
+			struct mhi_buf_info *info, enum mhi_flags flags)
 {
 	struct mhi_ring *buf_ring, *tre_ring;
 	struct mhi_tre *mhi_tre;
@@ -1072,15 +1079,22 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
 	tre_ring = &mhi_chan->tre_ring;
 
 	buf_info = buf_ring->wp;
-	buf_info->v_addr = buf;
-	buf_info->cb_buf = cb;
+	WARN_ON(buf_info->used);
+	buf_info->pre_mapped = info->pre_mapped;
+	if (info->pre_mapped)
+		buf_info->p_addr = info->p_addr;
+	else
+		buf_info->v_addr = info->v_addr;
+	buf_info->cb_buf = info->cb_buf;
 	buf_info->wp = tre_ring->wp;
 	buf_info->dir = mhi_chan->dir;
-	buf_info->len = buf_len;
+	buf_info->len = info->len;
 
-	ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
-	if (ret)
-		return ret;
+	if (!info->pre_mapped) {
+		ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
+		if (ret)
+			return ret;
+	}
 
 	eob = !!(flags & MHI_EOB);
 	eot = !!(flags & MHI_EOT);
@@ -1089,7 +1103,7 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
 
 	mhi_tre = tre_ring->wp;
 	mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
-	mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_len);
+	mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(info->len);
 	mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(bei, eot, eob, chain);
 
 	/* increment WP */
@@ -1106,6 +1120,7 @@ int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 	struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan :
 							     mhi_dev->dl_chan;
 	struct mhi_ring *tre_ring;
+	struct mhi_buf_info buf_info = { };
 	unsigned long flags;
 	int ret;
 
@@ -1121,7 +1136,11 @@ int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir,
 	if (mhi_is_ring_full(mhi_cntrl, tre_ring))
 		return -ENOMEM;
 
-	ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf, buf, len, mflags);
+	buf_info.v_addr = buf;
+	buf_info.cb_buf = buf;
+	buf_info.len = len;
+
+	ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags);
 	if (unlikely(ret))
 		return ret;
 
@@ -1322,7 +1341,7 @@ int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
 
 		while (nr_el--) {
 			void *buf;
-
+			struct mhi_buf_info info = { };
 			buf = kmalloc(len, GFP_KERNEL);
 			if (!buf) {
 				ret = -ENOMEM;
@@ -1330,8 +1349,10 @@ int mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
 			}
 
 			/* Prepare transfer descriptors */
-			ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf, buf,
-					  len, MHI_EOT);
+			info.v_addr = buf;
+			info.cb_buf = buf;
+			info.len = len;
+			ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &info, MHI_EOT);
 			if (ret) {
 				kfree(buf);
 				goto error_pre_alloc;
diff --git a/drivers/bus/mhi/core/pm.c b/drivers/bus/mhi/core/pm.c
index dc83d65f7784..796098078083 100644
--- a/drivers/bus/mhi/core/pm.c
+++ b/drivers/bus/mhi/core/pm.c
@@ -288,14 +288,18 @@ int mhi_pm_m0_transition(struct mhi_controller *mhi_cntrl)
 	for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
 		struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
 
-		write_lock_irq(&mhi_chan->lock);
-		if (mhi_chan->db_cfg.reset_req)
+		if (mhi_chan->db_cfg.reset_req) {
+			write_lock_irq(&mhi_chan->lock);
 			mhi_chan->db_cfg.db_mode = true;
+			write_unlock_irq(&mhi_chan->lock);
+		}
+
+		read_lock_irq(&mhi_chan->lock);
 
 		/* Only ring DB if ring is not empty */
 		if (tre_ring->base && tre_ring->wp  != tre_ring->rp)
 			mhi_ring_chan_db(mhi_cntrl, mhi_chan);
-		write_unlock_irq(&mhi_chan->lock);
+		read_unlock_irq(&mhi_chan->lock);
 	}
 
 	mhi_cntrl->wake_put(mhi_cntrl, false);
@@ -449,19 +453,8 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl,
 		to_mhi_pm_state_str(transition_state));
 
 	/* We must notify MHI control driver so it can clean up first */
-	if (transition_state == MHI_PM_SYS_ERR_PROCESS) {
-		/*
-		 * If controller supports RDDM, we do not process
-		 * SYS error state, instead we will jump directly
-		 * to RDDM state
-		 */
-		if (mhi_cntrl->rddm_image) {
-			dev_dbg(dev,
-				 "Controller supports RDDM, so skip SYS_ERR\n");
-			return;
-		}
+	if (transition_state == MHI_PM_SYS_ERR_PROCESS)
 		mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_SYS_ERROR);
-	}
 
 	mutex_lock(&mhi_cntrl->pm_mutex);
 	write_lock_irq(&mhi_cntrl->pm_lock);
@@ -527,8 +520,6 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl,
 	mutex_unlock(&mhi_cntrl->pm_mutex);
 	dev_dbg(dev, "Waiting for all pending threads to complete\n");
 	wake_up_all(&mhi_cntrl->state_event);
-	flush_work(&mhi_cntrl->st_worker);
-	flush_work(&mhi_cntrl->fw_worker);
 
 	dev_dbg(dev, "Reset all active channels and remove MHI devices\n");
 	device_for_each_child(mhi_cntrl->cntrl_dev, NULL, mhi_destroy_device);
@@ -608,13 +599,17 @@ int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl,
 }
 
 /* SYS_ERR worker */
-void mhi_pm_sys_err_worker(struct work_struct *work)
+void mhi_pm_sys_err_handler(struct mhi_controller *mhi_cntrl)
 {
-	struct mhi_controller *mhi_cntrl = container_of(work,
-							struct mhi_controller,
-							syserr_worker);
+	struct device *dev = &mhi_cntrl->mhi_dev->dev;
+
+	/* skip if controller supports RDDM */
+	if (mhi_cntrl->rddm_image) {
+		dev_dbg(dev, "Controller supports RDDM, skip SYS_ERROR\n");
+		return;
+	}
 
-	mhi_pm_disable_transition(mhi_cntrl, MHI_PM_SYS_ERR_PROCESS);
+	mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_SYS_ERR);
 }
 
 /* Device State Transition worker */
@@ -643,7 +638,7 @@ void mhi_pm_st_worker(struct work_struct *work)
 				mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
 			write_unlock_irq(&mhi_cntrl->pm_lock);
 			if (MHI_IN_PBL(mhi_cntrl->ee))
-				wake_up_all(&mhi_cntrl->state_event);
+				mhi_fw_load_handler(mhi_cntrl);
 			break;
 		case DEV_ST_TRANSITION_SBL:
 			write_lock_irq(&mhi_cntrl->pm_lock);
@@ -662,6 +657,14 @@ void mhi_pm_st_worker(struct work_struct *work)
 		case DEV_ST_TRANSITION_READY:
 			mhi_ready_state_transition(mhi_cntrl);
 			break;
+		case DEV_ST_TRANSITION_SYS_ERR:
+			mhi_pm_disable_transition
+				(mhi_cntrl, MHI_PM_SYS_ERR_PROCESS);
+			break;
+		case DEV_ST_TRANSITION_DISABLE:
+			mhi_pm_disable_transition
+				(mhi_cntrl, MHI_PM_SHUTDOWN_PROCESS);
+			break;
 		default:
 			break;
 		}
@@ -669,6 +672,149 @@ void mhi_pm_st_worker(struct work_struct *work)
 	}
 }
 
+int mhi_pm_suspend(struct mhi_controller *mhi_cntrl)
+{
+	struct mhi_chan *itr, *tmp;
+	struct device *dev = &mhi_cntrl->mhi_dev->dev;
+	enum mhi_pm_state new_state;
+	int ret;
+
+	if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
+		return -EINVAL;
+
+	if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+		return -EIO;
+
+	/* Return busy if there are any pending resources */
+	if (atomic_read(&mhi_cntrl->dev_wake))
+		return -EBUSY;
+
+	/* Take MHI out of M2 state */
+	read_lock_bh(&mhi_cntrl->pm_lock);
+	mhi_cntrl->wake_get(mhi_cntrl, false);
+	read_unlock_bh(&mhi_cntrl->pm_lock);
+
+	ret = wait_event_timeout(mhi_cntrl->state_event,
+				 mhi_cntrl->dev_state == MHI_STATE_M0 ||
+				 mhi_cntrl->dev_state == MHI_STATE_M1 ||
+				 MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+				 msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+	read_lock_bh(&mhi_cntrl->pm_lock);
+	mhi_cntrl->wake_put(mhi_cntrl, false);
+	read_unlock_bh(&mhi_cntrl->pm_lock);
+
+	if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+		dev_err(dev,
+			"Could not enter M0/M1 state");
+		return -EIO;
+	}
+
+	write_lock_irq(&mhi_cntrl->pm_lock);
+
+	if (atomic_read(&mhi_cntrl->dev_wake)) {
+		write_unlock_irq(&mhi_cntrl->pm_lock);
+		return -EBUSY;
+	}
+
+	dev_info(dev, "Allowing M3 transition\n");
+	new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_ENTER);
+	if (new_state != MHI_PM_M3_ENTER) {
+		write_unlock_irq(&mhi_cntrl->pm_lock);
+		dev_err(dev,
+			"Error setting to PM state: %s from: %s\n",
+			to_mhi_pm_state_str(MHI_PM_M3_ENTER),
+			to_mhi_pm_state_str(mhi_cntrl->pm_state));
+		return -EIO;
+	}
+
+	/* Set MHI to M3 and wait for completion */
+	mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M3);
+	write_unlock_irq(&mhi_cntrl->pm_lock);
+	dev_info(dev, "Wait for M3 completion\n");
+
+	ret = wait_event_timeout(mhi_cntrl->state_event,
+				 mhi_cntrl->dev_state == MHI_STATE_M3 ||
+				 MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+				 msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+	if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+		dev_err(dev,
+			"Did not enter M3 state, MHI state: %s, PM state: %s\n",
+			TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+			to_mhi_pm_state_str(mhi_cntrl->pm_state));
+		return -EIO;
+	}
+
+	/* Notify clients about entering LPM */
+	list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
+		mutex_lock(&itr->mutex);
+		if (itr->mhi_dev)
+			mhi_notify(itr->mhi_dev, MHI_CB_LPM_ENTER);
+		mutex_unlock(&itr->mutex);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mhi_pm_suspend);
+
+int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
+{
+	struct mhi_chan *itr, *tmp;
+	struct device *dev = &mhi_cntrl->mhi_dev->dev;
+	enum mhi_pm_state cur_state;
+	int ret;
+
+	dev_info(dev, "Entered with PM state: %s, MHI state: %s\n",
+		 to_mhi_pm_state_str(mhi_cntrl->pm_state),
+		 TO_MHI_STATE_STR(mhi_cntrl->dev_state));
+
+	if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
+		return 0;
+
+	if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+		return -EIO;
+
+	/* Notify clients about exiting LPM */
+	list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
+		mutex_lock(&itr->mutex);
+		if (itr->mhi_dev)
+			mhi_notify(itr->mhi_dev, MHI_CB_LPM_EXIT);
+		mutex_unlock(&itr->mutex);
+	}
+
+	write_lock_irq(&mhi_cntrl->pm_lock);
+	cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_EXIT);
+	if (cur_state != MHI_PM_M3_EXIT) {
+		write_unlock_irq(&mhi_cntrl->pm_lock);
+		dev_info(dev,
+			 "Error setting to PM state: %s from: %s\n",
+			 to_mhi_pm_state_str(MHI_PM_M3_EXIT),
+			 to_mhi_pm_state_str(mhi_cntrl->pm_state));
+		return -EIO;
+	}
+
+	/* Set MHI to M0 and wait for completion */
+	mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0);
+	write_unlock_irq(&mhi_cntrl->pm_lock);
+
+	ret = wait_event_timeout(mhi_cntrl->state_event,
+				 mhi_cntrl->dev_state == MHI_STATE_M0 ||
+				 MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+				 msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+	if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+		dev_err(dev,
+			"Did not enter M0 state, MHI state: %s, PM state: %s\n",
+			TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+			to_mhi_pm_state_str(mhi_cntrl->pm_state));
+		return -EIO;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mhi_pm_resume);
+
 int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl)
 {
 	int ret;
@@ -760,6 +906,7 @@ static void mhi_deassert_dev_wake(struct mhi_controller *mhi_cntrl,
 
 int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
 {
+	enum mhi_state state;
 	enum mhi_ee_type current_ee;
 	enum dev_st_transition next_state;
 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
@@ -829,13 +976,36 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
 		goto error_bhi_offset;
 	}
 
+	state = mhi_get_mhi_state(mhi_cntrl);
+	if (state == MHI_STATE_SYS_ERR) {
+		mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
+		ret = wait_event_timeout(mhi_cntrl->state_event,
+				MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state) ||
+					mhi_read_reg_field(mhi_cntrl,
+							   mhi_cntrl->regs,
+							   MHICTRL,
+							   MHICTRL_RESET_MASK,
+							   MHICTRL_RESET_SHIFT,
+							   &val) ||
+					!val,
+				msecs_to_jiffies(mhi_cntrl->timeout_ms));
+		if (ret) {
+			ret = -EIO;
+			dev_info(dev, "Failed to reset MHI due to syserr state\n");
+			goto error_bhi_offset;
+		}
+
+		/*
+		 * device cleares INTVEC as part of RESET processing,
+		 * re-program it
+		 */
+		mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
+	}
+
 	/* Transition to next state */
 	next_state = MHI_IN_PBL(current_ee) ?
 		DEV_ST_TRANSITION_PBL : DEV_ST_TRANSITION_READY;
 
-	if (next_state == DEV_ST_TRANSITION_PBL)
-		schedule_work(&mhi_cntrl->fw_worker);
-
 	mhi_queue_state_transition(mhi_cntrl, next_state);
 
 	mutex_unlock(&mhi_cntrl->pm_mutex);
@@ -876,7 +1046,12 @@ void mhi_power_down(struct mhi_controller *mhi_cntrl, bool graceful)
 				to_mhi_pm_state_str(MHI_PM_LD_ERR_FATAL_DETECT),
 				to_mhi_pm_state_str(mhi_cntrl->pm_state));
 	}
-	mhi_pm_disable_transition(mhi_cntrl, MHI_PM_SHUTDOWN_PROCESS);
+
+	mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_DISABLE);
+
+	/* Wait for shutdown to complete */
+	flush_work(&mhi_cntrl->st_worker);
+
 	mhi_deinit_free_irq(mhi_cntrl);
 
 	if (!mhi_cntrl->pre_init) {
diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c
index e5f5f48d69d2..3affd180baac 100644
--- a/drivers/bus/ti-sysc.c
+++ b/drivers/bus/ti-sysc.c
@@ -1275,13 +1275,6 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
 		   SYSC_QUIRK_LEGACY_IDLE),
 	SYSC_QUIRK("smartreflex", 0, -ENODEV, 0x38, -ENODEV, 0x00000000, 0xffffffff,
 		   SYSC_QUIRK_LEGACY_IDLE),
-	SYSC_QUIRK("timer", 0, 0, 0x10, 0x14, 0x00000015, 0xffffffff,
-		   0),
-	/* Some timers on omap4 and later */
-	SYSC_QUIRK("timer", 0, 0, 0x10, -ENODEV, 0x50002100, 0xffffffff,
-		   0),
-	SYSC_QUIRK("timer", 0, 0, 0x10, -ENODEV, 0x4fff1301, 0xffff00ff,
-		   0),
 	SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000046, 0xffffffff,
 		   SYSC_QUIRK_SWSUP_SIDLE | SYSC_QUIRK_LEGACY_IDLE),
 	SYSC_QUIRK("uart", 0, 0x50, 0x54, 0x58, 0x00000052, 0xffffffff,
@@ -1404,6 +1397,13 @@ static const struct sysc_revision_quirk sysc_revision_quirks[] = {
 	SYSC_QUIRK("slimbus", 0, 0, 0x10, -ENODEV, 0x40002903, 0xffffffff, 0),
 	SYSC_QUIRK("spinlock", 0, 0, 0x10, -ENODEV, 0x50020000, 0xffffffff, 0),
 	SYSC_QUIRK("rng", 0, 0x1fe0, 0x1fe4, -ENODEV, 0x00000020, 0xffffffff, 0),
+	SYSC_QUIRK("timer", 0, 0, 0x10, 0x14, 0x00000013, 0xffffffff, 0),
+	SYSC_QUIRK("timer", 0, 0, 0x10, 0x14, 0x00000015, 0xffffffff, 0),
+	/* Some timers on omap4 and later */
+	SYSC_QUIRK("timer", 0, 0, 0x10, -ENODEV, 0x50002100, 0xffffffff, 0),
+	SYSC_QUIRK("timer", 0, 0, 0x10, -ENODEV, 0x4fff1301, 0xffff00ff, 0),
+	SYSC_QUIRK("timer32k", 0, 0, 0x4, -ENODEV, 0x00000040, 0xffffffff, 0),
+	SYSC_QUIRK("timer32k", 0, 0, 0x4, -ENODEV, 0x00000011, 0xffffffff, 0),
 	SYSC_QUIRK("timer32k", 0, 0, 0x4, -ENODEV, 0x00000060, 0xffffffff, 0),
 	SYSC_QUIRK("tpcc", 0, 0, -ENODEV, -ENODEV, 0x40014c00, 0xffffffff, 0),
 	SYSC_QUIRK("usbhstll", 0, 0, 0x10, 0x14, 0x00000004, 0xffffffff, 0),
@@ -2744,6 +2744,17 @@ static int sysc_init_soc(struct sysc *ddata)
 	if (match && match->data)
 		sysc_soc->soc = (int)match->data;
 
+	/* Ignore devices that are not available on HS and EMU SoCs */
+	if (!sysc_soc->general_purpose) {
+		switch (sysc_soc->soc) {
+		case SOC_3430 ... SOC_3630:
+			sysc_add_disabled(0x48304000);	/* timer12 */
+			break;
+		default:
+			break;
+		};
+	}
+
 	match = soc_device_match(sysc_soc_feat_match);
 	if (!match)
 		return 0;
diff --git a/drivers/bus/vexpress-config.c b/drivers/bus/vexpress-config.c
index ff70575b2db6..a58ac0c8e282 100644
--- a/drivers/bus/vexpress-config.c
+++ b/drivers/bus/vexpress-config.c
@@ -6,10 +6,61 @@
 
 #include <linux/err.h>
 #include <linux/init.h>
+#include <linux/io.h>
+#include <linux/module.h>
 #include <linux/of.h>
+#include <linux/platform_device.h>
 #include <linux/of_device.h>
+#include <linux/sched/signal.h>
+#include <linux/slab.h>
 #include <linux/vexpress.h>
 
+#define SYS_MISC		0x0
+#define SYS_MISC_MASTERSITE	(1 << 14)
+
+#define SYS_PROCID0		0x24
+#define SYS_PROCID1		0x28
+#define SYS_HBI_MASK		0xfff
+#define SYS_PROCIDx_HBI_SHIFT	0
+
+#define SYS_CFGDATA		0x40
+
+#define SYS_CFGCTRL		0x44
+#define SYS_CFGCTRL_START	(1 << 31)
+#define SYS_CFGCTRL_WRITE	(1 << 30)
+#define SYS_CFGCTRL_DCC(n)	(((n) & 0xf) << 26)
+#define SYS_CFGCTRL_FUNC(n)	(((n) & 0x3f) << 20)
+#define SYS_CFGCTRL_SITE(n)	(((n) & 0x3) << 16)
+#define SYS_CFGCTRL_POSITION(n)	(((n) & 0xf) << 12)
+#define SYS_CFGCTRL_DEVICE(n)	(((n) & 0xfff) << 0)
+
+#define SYS_CFGSTAT		0x48
+#define SYS_CFGSTAT_ERR		(1 << 1)
+#define SYS_CFGSTAT_COMPLETE	(1 << 0)
+
+#define VEXPRESS_SITE_MB		0
+#define VEXPRESS_SITE_DB1		1
+#define VEXPRESS_SITE_DB2		2
+#define VEXPRESS_SITE_MASTER		0xf
+
+struct vexpress_syscfg {
+	struct device *dev;
+	void __iomem *base;
+	struct list_head funcs;
+};
+
+struct vexpress_syscfg_func {
+	struct list_head list;
+	struct vexpress_syscfg *syscfg;
+	struct regmap *regmap;
+	int num_templates;
+	u32 template[]; /* Keep it last! */
+};
+
+struct vexpress_config_bridge_ops {
+	struct regmap * (*regmap_init)(struct device *dev, void *context);
+	void (*regmap_exit)(struct regmap *regmap, void *context);
+};
 
 struct vexpress_config_bridge {
 	struct vexpress_config_bridge_ops *ops;
@@ -18,26 +69,20 @@ struct vexpress_config_bridge {
 
 
 static DEFINE_MUTEX(vexpress_config_mutex);
-static struct class *vexpress_config_class;
 static u32 vexpress_config_site_master = VEXPRESS_SITE_MASTER;
 
 
-void vexpress_config_set_master(u32 site)
+static void vexpress_config_set_master(u32 site)
 {
 	vexpress_config_site_master = site;
 }
 
-u32 vexpress_config_get_master(void)
-{
-	return vexpress_config_site_master;
-}
-
-void vexpress_config_lock(void *arg)
+static void vexpress_config_lock(void *arg)
 {
 	mutex_lock(&vexpress_config_mutex);
 }
 
-void vexpress_config_unlock(void *arg)
+static void vexpress_config_unlock(void *arg)
 {
 	mutex_unlock(&vexpress_config_mutex);
 }
@@ -59,7 +104,7 @@ static void vexpress_config_find_prop(struct device_node *node,
 	}
 }
 
-int vexpress_config_get_topo(struct device_node *node, u32 *site,
+static int vexpress_config_get_topo(struct device_node *node, u32 *site,
 		u32 *position, u32 *dcc)
 {
 	vexpress_config_find_prop(node, "arm,vexpress,site", site);
@@ -88,9 +133,6 @@ struct regmap *devm_regmap_init_vexpress_config(struct device *dev)
 	struct regmap *regmap;
 	struct regmap **res;
 
-	if (WARN_ON(dev->parent->class != vexpress_config_class))
-		return ERR_PTR(-ENODEV);
-
 	bridge = dev_get_drvdata(dev->parent);
 	if (WARN_ON(!bridge))
 		return ERR_PTR(-EINVAL);
@@ -113,91 +155,265 @@ struct regmap *devm_regmap_init_vexpress_config(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(devm_regmap_init_vexpress_config);
 
-struct device *vexpress_config_bridge_register(struct device *parent,
-		struct vexpress_config_bridge_ops *ops, void *context)
+static int vexpress_syscfg_exec(struct vexpress_syscfg_func *func,
+		int index, bool write, u32 *data)
 {
-	struct device *dev;
-	struct vexpress_config_bridge *bridge;
+	struct vexpress_syscfg *syscfg = func->syscfg;
+	u32 command, status;
+	int tries;
+	long timeout;
 
-	if (!vexpress_config_class) {
-		vexpress_config_class = class_create(THIS_MODULE,
-				"vexpress-config");
-		if (IS_ERR(vexpress_config_class))
-			return (void *)vexpress_config_class;
+	if (WARN_ON(index >= func->num_templates))
+		return -EINVAL;
+
+	command = readl(syscfg->base + SYS_CFGCTRL);
+	if (WARN_ON(command & SYS_CFGCTRL_START))
+		return -EBUSY;
+
+	command = func->template[index];
+	command |= SYS_CFGCTRL_START;
+	command |= write ? SYS_CFGCTRL_WRITE : 0;
+
+	/* Use a canary for reads */
+	if (!write)
+		*data = 0xdeadbeef;
+
+	dev_dbg(syscfg->dev, "func %p, command %x, data %x\n",
+			func, command, *data);
+	writel(*data, syscfg->base + SYS_CFGDATA);
+	writel(0, syscfg->base + SYS_CFGSTAT);
+	writel(command, syscfg->base + SYS_CFGCTRL);
+	mb();
+
+	/* The operation can take ages... Go to sleep, 100us initially */
+	tries = 100;
+	timeout = 100;
+	do {
+		if (!irqs_disabled()) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_timeout(usecs_to_jiffies(timeout));
+			if (signal_pending(current))
+				return -EINTR;
+		} else {
+			udelay(timeout);
+		}
+
+		status = readl(syscfg->base + SYS_CFGSTAT);
+		if (status & SYS_CFGSTAT_ERR)
+			return -EFAULT;
+
+		if (timeout > 20)
+			timeout -= 20;
+	} while (--tries && !(status & SYS_CFGSTAT_COMPLETE));
+	if (WARN_ON_ONCE(!tries))
+		return -ETIMEDOUT;
+
+	if (!write) {
+		*data = readl(syscfg->base + SYS_CFGDATA);
+		dev_dbg(syscfg->dev, "func %p, read data %x\n", func, *data);
 	}
 
-	dev = device_create(vexpress_config_class, parent, 0,
-			NULL, "%s.bridge", dev_name(parent));
+	return 0;
+}
+
+static int vexpress_syscfg_read(void *context, unsigned int index,
+		unsigned int *val)
+{
+	struct vexpress_syscfg_func *func = context;
+
+	return vexpress_syscfg_exec(func, index, false, val);
+}
+
+static int vexpress_syscfg_write(void *context, unsigned int index,
+		unsigned int val)
+{
+	struct vexpress_syscfg_func *func = context;
+
+	return vexpress_syscfg_exec(func, index, true, &val);
+}
+
+static struct regmap_config vexpress_syscfg_regmap_config = {
+	.lock = vexpress_config_lock,
+	.unlock = vexpress_config_unlock,
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_read = vexpress_syscfg_read,
+	.reg_write = vexpress_syscfg_write,
+	.reg_format_endian = REGMAP_ENDIAN_LITTLE,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+};
+
 
-	if (IS_ERR(dev))
-		return dev;
+static struct regmap *vexpress_syscfg_regmap_init(struct device *dev,
+		void *context)
+{
+	int err;
+	struct vexpress_syscfg *syscfg = context;
+	struct vexpress_syscfg_func *func;
+	struct property *prop;
+	const __be32 *val = NULL;
+	__be32 energy_quirk[4];
+	int num;
+	u32 site, position, dcc;
+	int i;
+
+	err = vexpress_config_get_topo(dev->of_node, &site,
+				&position, &dcc);
+	if (err)
+		return ERR_PTR(err);
+
+	prop = of_find_property(dev->of_node,
+			"arm,vexpress-sysreg,func", NULL);
+	if (!prop)
+		return ERR_PTR(-EINVAL);
 
-	bridge = devm_kmalloc(dev, sizeof(*bridge), GFP_KERNEL);
-	if (!bridge) {
-		put_device(dev);
-		device_unregister(dev);
+	num = prop->length / sizeof(u32) / 2;
+	val = prop->value;
+
+	/*
+	 * "arm,vexpress-energy" function used to be described
+	 * by its first device only, now it requires both
+	 */
+	if (num == 1 && of_device_is_compatible(dev->of_node,
+			"arm,vexpress-energy")) {
+		num = 2;
+		energy_quirk[0] = *val;
+		energy_quirk[2] = *val++;
+		energy_quirk[1] = *val;
+		energy_quirk[3] = cpu_to_be32(be32_to_cpup(val) + 1);
+		val = energy_quirk;
+	}
+
+	func = kzalloc(struct_size(func, template, num), GFP_KERNEL);
+	if (!func)
 		return ERR_PTR(-ENOMEM);
+
+	func->syscfg = syscfg;
+	func->num_templates = num;
+
+	for (i = 0; i < num; i++) {
+		u32 function, device;
+
+		function = be32_to_cpup(val++);
+		device = be32_to_cpup(val++);
+
+		dev_dbg(dev, "func %p: %u/%u/%u/%u/%u\n",
+				func, site, position, dcc,
+				function, device);
+
+		func->template[i] = SYS_CFGCTRL_DCC(dcc);
+		func->template[i] |= SYS_CFGCTRL_SITE(site);
+		func->template[i] |= SYS_CFGCTRL_POSITION(position);
+		func->template[i] |= SYS_CFGCTRL_FUNC(function);
+		func->template[i] |= SYS_CFGCTRL_DEVICE(device);
 	}
-	bridge->ops = ops;
-	bridge->context = context;
 
-	dev_set_drvdata(dev, bridge);
+	vexpress_syscfg_regmap_config.max_register = num - 1;
 
-	dev_dbg(parent, "Registered bridge '%s', parent node %p\n",
-			dev_name(dev), parent->of_node);
+	func->regmap = regmap_init(dev, NULL, func,
+			&vexpress_syscfg_regmap_config);
 
-	return dev;
-}
+	if (IS_ERR(func->regmap)) {
+		void *err = func->regmap;
 
+		kfree(func);
+		return err;
+	}
+
+	list_add(&func->list, &syscfg->funcs);
 
-static int vexpress_config_node_match(struct device *dev, const void *data)
+	return func->regmap;
+}
+
+static void vexpress_syscfg_regmap_exit(struct regmap *regmap, void *context)
 {
-	const struct device_node *node = data;
+	struct vexpress_syscfg *syscfg = context;
+	struct vexpress_syscfg_func *func, *tmp;
 
-	dev_dbg(dev, "Parent node %p, looking for %p\n",
-			dev->parent->of_node, node);
+	regmap_exit(regmap);
 
-	return dev->parent->of_node == node;
+	list_for_each_entry_safe(func, tmp, &syscfg->funcs, list) {
+		if (func->regmap == regmap) {
+			list_del(&syscfg->funcs);
+			kfree(func);
+			break;
+		}
+	}
 }
 
-static int vexpress_config_populate(struct device_node *node)
+static struct vexpress_config_bridge_ops vexpress_syscfg_bridge_ops = {
+	.regmap_init = vexpress_syscfg_regmap_init,
+	.regmap_exit = vexpress_syscfg_regmap_exit,
+};
+
+
+static int vexpress_syscfg_probe(struct platform_device *pdev)
 {
-	struct device_node *bridge;
-	struct device *parent;
-	int ret;
+	struct vexpress_syscfg *syscfg;
+	struct resource *res;
+	struct vexpress_config_bridge *bridge;
+	struct device_node *node;
+	int master;
+	u32 dt_hbi;
+
+	syscfg = devm_kzalloc(&pdev->dev, sizeof(*syscfg), GFP_KERNEL);
+	if (!syscfg)
+		return -ENOMEM;
+	syscfg->dev = &pdev->dev;
+	INIT_LIST_HEAD(&syscfg->funcs);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	syscfg->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(syscfg->base))
+		return PTR_ERR(syscfg->base);
 
-	bridge = of_parse_phandle(node, "arm,vexpress,config-bridge", 0);
+	bridge = devm_kmalloc(&pdev->dev, sizeof(*bridge), GFP_KERNEL);
 	if (!bridge)
-		return -EINVAL;
+		return -ENOMEM;
 
-	parent = class_find_device(vexpress_config_class, NULL, bridge,
-			vexpress_config_node_match);
-	of_node_put(bridge);
-	if (WARN_ON(!parent))
-		return -ENODEV;
+	bridge->ops = &vexpress_syscfg_bridge_ops;
+	bridge->context = syscfg;
 
-	ret = of_platform_populate(node, NULL, NULL, parent);
+	dev_set_drvdata(&pdev->dev, bridge);
 
-	put_device(parent);
+	master = readl(syscfg->base + SYS_MISC) & SYS_MISC_MASTERSITE ?
+			VEXPRESS_SITE_DB2 : VEXPRESS_SITE_DB1;
+	vexpress_config_set_master(master);
 
-	return ret;
-}
+	/* Confirm board type against DT property, if available */
+	if (of_property_read_u32(of_root, "arm,hbi", &dt_hbi) == 0) {
+		u32 id = readl(syscfg->base + (master == VEXPRESS_SITE_DB1 ?
+				 SYS_PROCID0 : SYS_PROCID1));
+		u32 hbi = (id >> SYS_PROCIDx_HBI_SHIFT) & SYS_HBI_MASK;
 
-static int __init vexpress_config_init(void)
-{
-	int err = 0;
-	struct device_node *node;
+		if (WARN_ON(dt_hbi != hbi))
+			dev_warn(&pdev->dev, "DT HBI (%x) is not matching hardware (%x)!\n",
+					dt_hbi, hbi);
+	}
 
-	/* Need the config devices early, before the "normal" devices... */
 	for_each_compatible_node(node, NULL, "arm,vexpress,config-bus") {
-		err = vexpress_config_populate(node);
-		if (err) {
-			of_node_put(node);
-			break;
-		}
+		struct device_node *bridge_np;
+
+		bridge_np = of_parse_phandle(node, "arm,vexpress,config-bridge", 0);
+		if (bridge_np != pdev->dev.parent->of_node)
+			continue;
+
+		of_platform_populate(node, NULL, NULL, &pdev->dev);
 	}
 
-	return err;
+	return 0;
 }
-postcore_initcall(vexpress_config_init);
 
+static const struct platform_device_id vexpress_syscfg_id_table[] = {
+	{ "vexpress-syscfg", },
+	{},
+};
+MODULE_DEVICE_TABLE(platform, vexpress_syscfg_id_table);
+
+static struct platform_driver vexpress_syscfg_driver = {
+	.driver.name = "vexpress-syscfg",
+	.id_table = vexpress_syscfg_id_table,
+	.probe = vexpress_syscfg_probe,
+};
+module_platform_driver(vexpress_syscfg_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c
index a0a7ae705de8..d82b3b7658bd 100644
--- a/drivers/cdrom/cdrom.c
+++ b/drivers/cdrom/cdrom.c
@@ -3535,7 +3535,7 @@ static int cdrom_print_info(const char *header, int val, char *info,
 }
 
 static int cdrom_sysctl_info(struct ctl_table *ctl, int write,
-                           void __user *buffer, size_t *lenp, loff_t *ppos)
+                           void *buffer, size_t *lenp, loff_t *ppos)
 {
 	int pos;
 	char *info = cdrom_sysctl_settings.info;
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index d4665fe9ccd2..98c3a5d8003e 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -11,7 +11,7 @@ config TTY_PRINTK
 	tristate "TTY driver to output user messages via printk"
 	depends on EXPERT && TTY
 	default n
-	---help---
+	help
 	  If you say Y here, the support for writing user messages (i.e.
 	  console messages) via printk is available.
 
@@ -33,7 +33,7 @@ config TTY_PRINTK_LEVEL
 config PRINTER
 	tristate "Parallel printer support"
 	depends on PARPORT
-	---help---
+	help
 	  If you intend to attach a printer to the parallel port of your Linux
 	  box (as opposed to using a serial printer; if the connector at the
 	  printer has 9 or 25 holes ["female"], then it's serial), say Y.
@@ -59,7 +59,7 @@ config PRINTER
 config LP_CONSOLE
 	bool "Support for console on line printer"
 	depends on PRINTER
-	---help---
+	help
 	  If you want kernel messages to be printed out as they occur, you
 	  can have a console on the printer. This option adds support for
 	  doing that; to actually get it to happen you need to pass the
@@ -76,7 +76,7 @@ config LP_CONSOLE
 config PPDEV
 	tristate "Support for user-space parallel port device drivers"
 	depends on PARPORT
-	---help---
+	help
 	  Saying Y to this adds support for /dev/parport device nodes.  This
 	  is needed for programs that want portable access to the parallel
 	  port, for instance deviceid (which displays Plug-and-Play device
@@ -146,7 +146,7 @@ config DS1620
 config NWBUTTON
 	tristate "NetWinder Button"
 	depends on ARCH_NETWINDER
-	---help---
+	help
 	  If you say Y here and create a character device node /dev/nwbutton
 	  with major and minor numbers 10 and 158 ("man mknod"), then every
 	  time the orange button is pressed a number of times, the number of
@@ -182,7 +182,7 @@ config NWBUTTON_REBOOT
 config NWFLASH
 	tristate "NetWinder flash support"
 	depends on ARCH_NETWINDER
-	---help---
+	help
 	  If you say Y here and create a character device /dev/flash with
 	  major 10 and minor 160 you can manipulate the flash ROM containing
 	  the NetWinder firmware. Be careful as accidentally overwriting the
@@ -209,7 +209,7 @@ config DTLK
 
 config XILINX_HWICAP
 	tristate "Xilinx HWICAP Support"
-	depends on XILINX_VIRTEX || MICROBLAZE
+	depends on MICROBLAZE
 	help
 	  This option enables support for Xilinx Internal Configuration
 	  Access Port (ICAP) driver.  The ICAP is used on Xilinx Virtex
@@ -220,7 +220,7 @@ config XILINX_HWICAP
 config R3964
 	tristate "Siemens R3964 line discipline"
 	depends on TTY && BROKEN
-	---help---
+	help
 	  This driver allows synchronous communication with devices using the
 	  Siemens R3964 packet protocol. Unless you are dealing with special
 	  hardware like PLCs, you are unlikely to need this.
@@ -233,7 +233,7 @@ config R3964
 config APPLICOM
 	tristate "Applicom intelligent fieldbus card support"
 	depends on PCI
-	---help---
+	help
 	  This driver provides the kernel-side support for the intelligent
 	  fieldbus cards made by Applicom International. More information
 	  about these cards can be found on the WWW at the address
@@ -248,7 +248,7 @@ config APPLICOM
 config SONYPI
 	tristate "Sony Vaio Programmable I/O Control Device support"
 	depends on X86_32 && PCI && INPUT
-	---help---
+	help
 	  This driver enables access to the Sony Programmable I/O Control
 	  Device which can be found in many (all ?) Sony Vaio laptops.
 
@@ -269,7 +269,7 @@ config MWAVE
 	tristate "ACP Modem (Mwave) support"
 	depends on X86 && TTY
 	select SERIAL_8250
-	---help---
+	help
 	  The ACP modem (Mwave) for Linux is a WinModem. It is composed of a
 	  kernel driver and a user level application. Together these components
 	  support direct attachment to public switched telephone networks (PSTNs)
@@ -347,7 +347,7 @@ config NVRAM
 	tristate "/dev/nvram support"
 	depends on X86 || HAVE_ARCH_NVRAM_OPS
 	default M68K || PPC
-	---help---
+	help
 	  If you say Y here and create a character special file /dev/nvram
 	  with major number 10 and minor number 144 using mknod ("man mknod"),
 	  you get read and write access to the non-volatile memory.
diff --git a/drivers/char/agp/Kconfig b/drivers/char/agp/Kconfig
index bc54235a7022..a086dd34f932 100644
--- a/drivers/char/agp/Kconfig
+++ b/drivers/char/agp/Kconfig
@@ -3,7 +3,7 @@ menuconfig AGP
 	tristate "/dev/agpgart (AGP Support)"
 	depends on ALPHA || IA64 || PARISC || PPC || X86
 	depends on PCI
-	---help---
+	help
 	  AGP (Accelerated Graphics Port) is a bus system mainly used to
 	  connect graphics cards to the rest of the system.
 
@@ -30,7 +30,7 @@ menuconfig AGP
 config AGP_ALI
 	tristate "ALI chipset support"
 	depends on AGP && X86_32
-	---help---
+	help
 	  This option gives you AGP support for the GLX component of
 	  X on the following ALi chipsets.  The supported chipsets
 	  include M1541, M1621, M1631, M1632, M1641,M1647,and M1651.
@@ -45,7 +45,7 @@ config AGP_ALI
 config AGP_ATI
 	tristate "ATI chipset support"
 	depends on AGP && X86_32
-	---help---
+	help
 	  This option gives you AGP support for the GLX component of
 	  X on the ATI RadeonIGP family of chipsets.
 
diff --git a/drivers/char/agp/frontend.c b/drivers/char/agp/frontend.c
index 47098648502d..00ff5fcb808a 100644
--- a/drivers/char/agp/frontend.c
+++ b/drivers/char/agp/frontend.c
@@ -39,7 +39,6 @@
 #include <linux/fs.h>
 #include <linux/sched.h>
 #include <linux/uaccess.h>
-#include <asm/pgtable.h>
 #include "agp.h"
 
 struct agp_front_data agp_fe;
diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index 9e84239f88d4..3ffbb1c80c5c 100644
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -42,7 +42,6 @@
 #ifdef CONFIG_X86
 #include <asm/set_memory.h>
 #endif
-#include <asm/pgtable.h>
 #include "agp.h"
 
 __u32 *agp_gatt_table;
diff --git a/drivers/char/bsr.c b/drivers/char/bsr.c
index e5e5333f302d..cce2af5df7b4 100644
--- a/drivers/char/bsr.c
+++ b/drivers/char/bsr.c
@@ -17,7 +17,6 @@
 #include <linux/list.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
-#include <asm/pgtable.h>
 #include <asm/io.h>
 
 /*
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index ac00d78ee9cc..0ad17efc96df 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -6,7 +6,7 @@
 menuconfig HW_RANDOM
 	tristate "Hardware Random Number Generator Core support"
 	default m
-	---help---
+	help
 	  Hardware Random Number Generator Core infrastructure.
 
 	  To compile this driver as a module, choose M here: the
@@ -24,7 +24,7 @@ if HW_RANDOM
 config HW_RANDOM_TIMERIOMEM
 	tristate "Timer IOMEM HW Random Number Generator support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  This driver provides kernel-side support for a generic Random
 	  Number Generator used by reading a 'dumb' iomem address that
 	  is to be read no faster than, for example, once a second;
@@ -39,7 +39,7 @@ config HW_RANDOM_INTEL
 	tristate "Intel HW Random Number Generator support"
 	depends on (X86 || IA64) && PCI
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Intel i8xx-based motherboards.
 
@@ -52,7 +52,7 @@ config HW_RANDOM_AMD
 	tristate "AMD HW Random Number Generator support"
 	depends on (X86 || PPC_MAPLE) && PCI
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on AMD 76x-based motherboards.
 
@@ -65,7 +65,7 @@ config HW_RANDOM_ATMEL
 	tristate "Atmel Random Number Generator support"
 	depends on ARCH_AT91 && HAVE_CLK && OF
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Atmel AT91 devices.
 
@@ -79,7 +79,7 @@ config HW_RANDOM_BCM2835
 	depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X || \
 		   ARCH_BCM_63XX || BCM63XX || BMIPS_GENERIC
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on the Broadcom BCM2835 and BCM63xx SoCs.
 
@@ -92,7 +92,7 @@ config HW_RANDOM_IPROC_RNG200
 	tristate "Broadcom iProc/STB RNG200 support"
 	depends on ARCH_BCM_IPROC || ARCH_BCM2835 || ARCH_BRCMSTB
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the RNG200
 	  hardware found on the Broadcom iProc and STB SoCs.
 
@@ -105,7 +105,7 @@ config HW_RANDOM_GEODE
 	tristate "AMD Geode HW Random Number Generator support"
 	depends on X86_32 && PCI
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on the AMD Geode LX.
 
@@ -118,7 +118,7 @@ config HW_RANDOM_N2RNG
 	tristate "Niagara2 Random Number Generator support"
 	depends on SPARC64
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Niagara2 cpus.
 
@@ -131,7 +131,7 @@ config HW_RANDOM_VIA
 	tristate "VIA HW Random Number Generator support"
 	depends on X86
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on VIA based motherboards.
 
@@ -144,7 +144,7 @@ config HW_RANDOM_IXP4XX
 	tristate "Intel IXP4xx NPU HW Pseudo-Random Number Generator support"
 	depends on ARCH_IXP4XX
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Pseudo-Random
 	  Number Generator hardware found on the Intel IXP45x/46x NPU.
 
@@ -157,7 +157,7 @@ config HW_RANDOM_OMAP
 	tristate "OMAP Random Number Generator support"
 	depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU
 	default HW_RANDOM
- 	---help---
+	help
  	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on OMAP16xx, OMAP2/3/4/5, AM33xx/AM43xx
 	  multimedia processors, and Marvell Armada 7k/8k SoCs.
@@ -171,7 +171,7 @@ config HW_RANDOM_OMAP3_ROM
 	tristate "OMAP3 ROM Random Number Generator support"
 	depends on ARCH_OMAP3
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on OMAP34xx processors.
 
@@ -184,7 +184,7 @@ config HW_RANDOM_OCTEON
 	tristate "Octeon Random Number Generator support"
 	depends on CAVIUM_OCTEON_SOC
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Octeon processors.
 
@@ -197,7 +197,7 @@ config HW_RANDOM_PASEMI
 	tristate "PA Semi HW Random Number Generator support"
 	depends on PPC_PASEMI
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on PA Semi PWRficient SoCs.
 
@@ -209,7 +209,7 @@ config HW_RANDOM_PASEMI
 config HW_RANDOM_VIRTIO
 	tristate "VirtIO Random Number Generator support"
 	depends on VIRTIO
-	---help---
+	help
 	  This driver provides kernel-side support for the virtual Random Number
 	  Generator hardware.
 
@@ -220,7 +220,7 @@ config HW_RANDOM_TX4939
 	tristate "TX4939 Random Number Generator support"
 	depends on SOC_TX4939
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on TX4939 SoC.
 
@@ -233,7 +233,7 @@ config HW_RANDOM_MXC_RNGA
 	tristate "Freescale i.MX RNGA Random Number Generator"
 	depends on SOC_IMX31
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Freescale i.MX processors.
 
@@ -247,7 +247,7 @@ config HW_RANDOM_IMX_RNGC
 	depends on HAS_IOMEM && HAVE_CLK
 	depends on SOC_IMX25 || COMPILE_TEST
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator Version C hardware found on some Freescale i.MX
 	  processors. Version B is also supported by this driver.
@@ -261,7 +261,7 @@ config HW_RANDOM_NOMADIK
 	tristate "ST-Ericsson Nomadik Random Number Generator support"
 	depends on ARCH_NOMADIK
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on ST-Ericsson SoCs (8815 and 8500).
 
@@ -274,7 +274,7 @@ config HW_RANDOM_PSERIES
 	tristate "pSeries HW Random Number Generator support"
 	depends on PPC64 && IBMVIO
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on POWER7+ machines and above
 
@@ -287,7 +287,7 @@ config HW_RANDOM_POWERNV
 	tristate "PowerNV Random Number Generator support"
 	depends on PPC_POWERNV
 	default HW_RANDOM
-	---help---
+	help
 	  This is the driver for Random Number Generator hardware found
 	  in POWER7+ and above machines for PowerNV platform.
 
@@ -300,7 +300,7 @@ config HW_RANDOM_HISI
 	tristate "Hisilicon Random Number Generator support"
 	depends on HW_RANDOM && ARCH_HISI
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Hisilicon Hip04 and Hip05 SoC.
 
@@ -325,7 +325,7 @@ config HW_RANDOM_HISI_V2
 config HW_RANDOM_ST
 	tristate "ST Microelectronics HW Random Number Generator support"
 	depends on HW_RANDOM && ARCH_STI
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on STi series of SoCs.
 
@@ -336,7 +336,7 @@ config HW_RANDOM_XGENE
 	tristate "APM X-Gene True Random Number Generator (TRNG) support"
 	depends on HW_RANDOM && ARCH_XGENE
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on APM X-Gene SoC.
 
@@ -363,7 +363,7 @@ config HW_RANDOM_PIC32
 	tristate "Microchip PIC32 Random Number Generator support"
 	depends on HW_RANDOM && MACH_PIC32
 	default y
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on a PIC32.
 
@@ -377,7 +377,7 @@ config HW_RANDOM_MESON
 	depends on HW_RANDOM
 	depends on ARCH_MESON || COMPILE_TEST
 	default y
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Amlogic Meson SoCs.
 
@@ -390,7 +390,7 @@ config HW_RANDOM_CAVIUM
 	tristate "Cavium ThunderX Random Number Generator support"
 	depends on HW_RANDOM && PCI && (ARM64 || (COMPILE_TEST && 64BIT))
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Cavium SoCs.
 
@@ -404,7 +404,7 @@ config HW_RANDOM_MTK
 	depends on HW_RANDOM
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	default y
-	---help---
+	help
 	  This driver provides kernel-side support for the Random Number
 	  Generator hardware found on Mediatek SoCs.
 
@@ -417,7 +417,7 @@ config HW_RANDOM_S390
 	tristate "S390 True Random Number Generator support"
 	depends on S390
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides kernel-side support for the True
 	  Random Number Generator available as CPACF extension
 	  on modern s390 hardware platforms.
@@ -431,7 +431,7 @@ config HW_RANDOM_EXYNOS
 	tristate "Samsung Exynos True Random Number Generator support"
 	depends on ARCH_EXYNOS || COMPILE_TEST
 	default HW_RANDOM
-	---help---
+	help
 	  This driver provides support for the True Random Number
 	  Generator available in Exynos SoCs.
 
diff --git a/drivers/char/ipmi/bt-bmc.c b/drivers/char/ipmi/bt-bmc.c
index d36aeacb290e..a395e2e70dc5 100644
--- a/drivers/char/ipmi/bt-bmc.c
+++ b/drivers/char/ipmi/bt-bmc.c
@@ -399,15 +399,15 @@ static int bt_bmc_config_irq(struct bt_bmc *bt_bmc,
 	struct device *dev = &pdev->dev;
 	int rc;
 
-	bt_bmc->irq = platform_get_irq(pdev, 0);
-	if (!bt_bmc->irq)
-		return -ENODEV;
+	bt_bmc->irq = platform_get_irq_optional(pdev, 0);
+	if (bt_bmc->irq < 0)
+		return bt_bmc->irq;
 
 	rc = devm_request_irq(dev, bt_bmc->irq, bt_bmc_irq, IRQF_SHARED,
 			      DEVICE_NAME, bt_bmc);
 	if (rc < 0) {
 		dev_warn(dev, "Unable to request IRQ %d\n", bt_bmc->irq);
-		bt_bmc->irq = 0;
+		bt_bmc->irq = rc;
 		return rc;
 	}
 
@@ -430,9 +430,6 @@ static int bt_bmc_probe(struct platform_device *pdev)
 	struct device *dev;
 	int rc;
 
-	if (!pdev || !pdev->dev.of_node)
-		return -ENODEV;
-
 	dev = &pdev->dev;
 	dev_info(dev, "Found bt bmc device\n");
 
@@ -466,9 +463,9 @@ static int bt_bmc_probe(struct platform_device *pdev)
 	init_waitqueue_head(&bt_bmc->queue);
 
 	bt_bmc->miscdev.minor	= MISC_DYNAMIC_MINOR,
-		bt_bmc->miscdev.name	= DEVICE_NAME,
-		bt_bmc->miscdev.fops	= &bt_bmc_fops,
-		bt_bmc->miscdev.parent = dev;
+	bt_bmc->miscdev.name	= DEVICE_NAME,
+	bt_bmc->miscdev.fops	= &bt_bmc_fops,
+	bt_bmc->miscdev.parent = dev;
 	rc = misc_register(&bt_bmc->miscdev);
 	if (rc) {
 		dev_err(dev, "Unable to register misc device\n");
@@ -477,7 +474,7 @@ static int bt_bmc_probe(struct platform_device *pdev)
 
 	bt_bmc_config_irq(bt_bmc, pdev);
 
-	if (bt_bmc->irq) {
+	if (bt_bmc->irq >= 0) {
 		dev_info(dev, "Using IRQ %d\n", bt_bmc->irq);
 	} else {
 		dev_info(dev, "No IRQ; using timer\n");
@@ -503,7 +500,7 @@ static int bt_bmc_remove(struct platform_device *pdev)
 	struct bt_bmc *bt_bmc = dev_get_drvdata(&pdev->dev);
 
 	misc_deregister(&bt_bmc->miscdev);
-	if (!bt_bmc->irq)
+	if (bt_bmc->irq < 0)
 		del_timer_sync(&bt_bmc->poll_timer);
 	return 0;
 }
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index c48d8f086382..e1b22fe0916c 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -33,6 +33,7 @@
 #include <linux/workqueue.h>
 #include <linux/uuid.h>
 #include <linux/nospec.h>
+#include <linux/vmalloc.h>
 
 #define IPMI_DRIVER_VERSION "39.2"
 
@@ -1153,7 +1154,7 @@ static void free_user_work(struct work_struct *work)
 					      remove_work);
 
 	cleanup_srcu_struct(&user->release_barrier);
-	kfree(user);
+	vfree(user);
 }
 
 int ipmi_create_user(unsigned int          if_num,
@@ -1185,7 +1186,7 @@ int ipmi_create_user(unsigned int          if_num,
 	if (rv)
 		return rv;
 
-	new_user = kmalloc(sizeof(*new_user), GFP_KERNEL);
+	new_user = vzalloc(sizeof(*new_user));
 	if (!new_user)
 		return -ENOMEM;
 
@@ -1232,7 +1233,7 @@ int ipmi_create_user(unsigned int          if_num,
 
 out_kfree:
 	srcu_read_unlock(&ipmi_interfaces_srcu, index);
-	kfree(new_user);
+	vfree(new_user);
 	return rv;
 }
 EXPORT_SYMBOL(ipmi_create_user);
@@ -3171,7 +3172,7 @@ static void guid_handler(struct ipmi_smi *intf, struct ipmi_recv_msg *msg)
 		goto out;
 	}
 
-	guid_copy(&bmc->fetch_guid, (guid_t *)(msg->msg.data + 1));
+	import_guid(&bmc->fetch_guid, msg->msg.data + 1);
 	/*
 	 * Make sure the guid data is available before setting
 	 * dyn_guid_set.
diff --git a/drivers/char/ipmi/ipmi_si_platform.c b/drivers/char/ipmi/ipmi_si_platform.c
index 638c693e17ad..129b5713f187 100644
--- a/drivers/char/ipmi/ipmi_si_platform.c
+++ b/drivers/char/ipmi/ipmi_si_platform.c
@@ -393,6 +393,8 @@ static int acpi_ipmi_probe(struct platform_device *pdev)
 	dev_info(io.dev, "%pR regsize %d spacing %d irq %d\n",
 		 res, io.regsize, io.regspacing, io.irq);
 
+	request_module("acpi_ipmi");
+
 	return ipmi_si_add_smi(&io);
 
 err_free:
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index 2704470e021d..198b65d45c5e 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -189,8 +189,6 @@ struct ssif_addr_info {
 	struct device *dev;
 	struct i2c_client *client;
 
-	struct i2c_client *added_client;
-
 	struct mutex clients_mutex;
 	struct list_head clients;
 
@@ -1472,6 +1470,7 @@ static bool check_acpi(struct ssif_info *ssif_info, struct device *dev)
 	if (acpi_handle) {
 		ssif_info->addr_source = SI_ACPI;
 		ssif_info->addr_info.acpi_info.acpi_handle = acpi_handle;
+		request_module("acpi_ipmi");
 		return true;
 	}
 #endif
@@ -1940,21 +1939,6 @@ out_remove_attr:
 	goto out;
 }
 
-static int ssif_adapter_handler(struct device *adev, void *opaque)
-{
-	struct ssif_addr_info *addr_info = opaque;
-
-	if (adev->type != &i2c_adapter_type)
-		return 0;
-
-	addr_info->added_client = i2c_new_client_device(to_i2c_adapter(adev),
-							&addr_info->binfo);
-
-	if (!addr_info->adapter_name)
-		return 1; /* Only try the first I2C adapter by default. */
-	return 0;
-}
-
 static int new_ssif_client(int addr, char *adapter_name,
 			   int debug, int slave_addr,
 			   enum ipmi_addr_src addr_src,
@@ -1998,9 +1982,7 @@ static int new_ssif_client(int addr, char *adapter_name,
 
 	list_add_tail(&addr_info->link, &ssif_infos);
 
-	if (initialized)
-		i2c_for_each_dev(addr_info, ssif_adapter_handler);
-	/* Otherwise address list will get it */
+	/* Address list will get it */
 
 out_unlock:
 	mutex_unlock(&ssif_infos_mutex);
@@ -2120,8 +2102,6 @@ static int ssif_platform_remove(struct platform_device *dev)
 		return 0;
 
 	mutex_lock(&ssif_infos_mutex);
-	i2c_unregister_device(addr_info->added_client);
-
 	list_del(&addr_info->link);
 	kfree(addr_info);
 	mutex_unlock(&ssif_infos_mutex);
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 43dd0891ca1e..934c92dcb9ab 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -31,11 +31,15 @@
 #include <linux/uio.h>
 #include <linux/uaccess.h>
 #include <linux/security.h>
+#include <linux/pseudo_fs.h>
+#include <uapi/linux/magic.h>
+#include <linux/mount.h>
 
 #ifdef CONFIG_IA64
 # include <linux/efi.h>
 #endif
 
+#define DEVMEM_MINOR	1
 #define DEVPORT_MINOR	4
 
 static inline unsigned long size_inside_page(unsigned long start,
@@ -167,7 +171,7 @@ static ssize_t read_mem(struct file *file, char __user *buf,
 			if (!ptr)
 				goto failed;
 
-			probe = probe_kernel_read(bounce, ptr, sz);
+			probe = copy_from_kernel_nofault(bounce, ptr, sz);
 			unxlate_dev_mem_ptr(p, ptr);
 			if (probe)
 				goto failed;
@@ -805,12 +809,64 @@ static loff_t memory_lseek(struct file *file, loff_t offset, int orig)
 	return ret;
 }
 
+static struct inode *devmem_inode;
+
+#ifdef CONFIG_IO_STRICT_DEVMEM
+void revoke_devmem(struct resource *res)
+{
+	struct inode *inode = READ_ONCE(devmem_inode);
+
+	/*
+	 * Check that the initialization has completed. Losing the race
+	 * is ok because it means drivers are claiming resources before
+	 * the fs_initcall level of init and prevent /dev/mem from
+	 * establishing mappings.
+	 */
+	if (!inode)
+		return;
+
+	/*
+	 * The expectation is that the driver has successfully marked
+	 * the resource busy by this point, so devmem_is_allowed()
+	 * should start returning false, however for performance this
+	 * does not iterate the entire resource range.
+	 */
+	if (devmem_is_allowed(PHYS_PFN(res->start)) &&
+	    devmem_is_allowed(PHYS_PFN(res->end))) {
+		/*
+		 * *cringe* iomem=relaxed says "go ahead, what's the
+		 * worst that can happen?"
+		 */
+		return;
+	}
+
+	unmap_mapping_range(inode->i_mapping, res->start, resource_size(res), 1);
+}
+#endif
+
 static int open_port(struct inode *inode, struct file *filp)
 {
+	int rc;
+
 	if (!capable(CAP_SYS_RAWIO))
 		return -EPERM;
 
-	return security_locked_down(LOCKDOWN_DEV_MEM);
+	rc = security_locked_down(LOCKDOWN_DEV_MEM);
+	if (rc)
+		return rc;
+
+	if (iminor(inode) != DEVMEM_MINOR)
+		return 0;
+
+	/*
+	 * Use a unified address space to have a single point to manage
+	 * revocations when drivers want to take over a /dev/mem mapped
+	 * range.
+	 */
+	inode->i_mapping = devmem_inode->i_mapping;
+	filp->f_mapping = inode->i_mapping;
+
+	return 0;
 }
 
 #define zero_lseek	null_lseek
@@ -885,7 +941,7 @@ static const struct memdev {
 	fmode_t fmode;
 } devlist[] = {
 #ifdef CONFIG_DEVMEM
-	 [1] = { "mem", 0, &mem_fops, FMODE_UNSIGNED_OFFSET },
+	 [DEVMEM_MINOR] = { "mem", 0, &mem_fops, FMODE_UNSIGNED_OFFSET },
 #endif
 #ifdef CONFIG_DEVKMEM
 	 [2] = { "kmem", 0, &kmem_fops, FMODE_UNSIGNED_OFFSET },
@@ -939,6 +995,45 @@ static char *mem_devnode(struct device *dev, umode_t *mode)
 
 static struct class *mem_class;
 
+static int devmem_fs_init_fs_context(struct fs_context *fc)
+{
+	return init_pseudo(fc, DEVMEM_MAGIC) ? 0 : -ENOMEM;
+}
+
+static struct file_system_type devmem_fs_type = {
+	.name		= "devmem",
+	.owner		= THIS_MODULE,
+	.init_fs_context = devmem_fs_init_fs_context,
+	.kill_sb	= kill_anon_super,
+};
+
+static int devmem_init_inode(void)
+{
+	static struct vfsmount *devmem_vfs_mount;
+	static int devmem_fs_cnt;
+	struct inode *inode;
+	int rc;
+
+	rc = simple_pin_fs(&devmem_fs_type, &devmem_vfs_mount, &devmem_fs_cnt);
+	if (rc < 0) {
+		pr_err("Cannot mount /dev/mem pseudo filesystem: %d\n", rc);
+		return rc;
+	}
+
+	inode = alloc_anon_inode(devmem_vfs_mount->mnt_sb);
+	if (IS_ERR(inode)) {
+		rc = PTR_ERR(inode);
+		pr_err("Cannot allocate inode for /dev/mem: %d\n", rc);
+		simple_release_fs(&devmem_vfs_mount, &devmem_fs_cnt);
+		return rc;
+	}
+
+	/* publish /dev/mem initialized */
+	WRITE_ONCE(devmem_inode, inode);
+
+	return 0;
+}
+
 static int __init chr_dev_init(void)
 {
 	int minor;
@@ -960,6 +1055,8 @@ static int __init chr_dev_init(void)
 		 */
 		if ((minor == DEVPORT_MINOR) && !arch_has_dev_port())
 			continue;
+		if ((minor == DEVMEM_MINOR) && devmem_init_inode() != 0)
+			continue;
 
 		device_create(mem_class, NULL, MKDEV(MEM_MAJOR, minor),
 			      NULL, devlist[minor].name);
diff --git a/drivers/char/mspec.c b/drivers/char/mspec.c
index 7d583222e8fa..0fae33319d2e 100644
--- a/drivers/char/mspec.c
+++ b/drivers/char/mspec.c
@@ -39,7 +39,6 @@
 #include <linux/numa.h>
 #include <linux/refcount.h>
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <linux/atomic.h>
 #include <asm/tlbflush.h>
 #include <asm/uncached.h>
@@ -65,7 +64,7 @@ enum mspec_page_type {
  * This structure is shared by all vma's that are split off from the
  * original vma when split_vma()'s are done.
  *
- * The refcnt is incremented atomically because mm->mmap_sem does not
+ * The refcnt is incremented atomically because mm->mmap_lock does not
  * protect in fork case where multiple tasks share the vma_data.
  */
 struct vma_data {
diff --git a/drivers/char/random.c b/drivers/char/random.c
index a7cf6aa65908..2a41b21623ae 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -2087,7 +2087,7 @@ static int proc_do_uuid(struct ctl_table *table, int write,
  * Return entropy available scaled to integral bits
  */
 static int proc_do_entropy(struct ctl_table *table, int write,
-			   void __user *buffer, size_t *lenp, loff_t *ppos)
+			   void *buffer, size_t *lenp, loff_t *ppos)
 {
 	struct ctl_table fake_table;
 	int entropy_count;
diff --git a/drivers/char/tlclk.c b/drivers/char/tlclk.c
index 6d81bb3bb503..896a3550fba9 100644
--- a/drivers/char/tlclk.c
+++ b/drivers/char/tlclk.c
@@ -777,17 +777,21 @@ static int __init tlclk_init(void)
 {
 	int ret;
 
+	telclk_interrupt = (inb(TLCLK_REG7) & 0x0f);
+
+	alarm_events = kzalloc( sizeof(struct tlclk_alarms), GFP_KERNEL);
+	if (!alarm_events) {
+		ret = -ENOMEM;
+		goto out1;
+	}
+
 	ret = register_chrdev(tlclk_major, "telco_clock", &tlclk_fops);
 	if (ret < 0) {
 		printk(KERN_ERR "tlclk: can't get major %d.\n", tlclk_major);
+		kfree(alarm_events);
 		return ret;
 	}
 	tlclk_major = ret;
-	alarm_events = kzalloc( sizeof(struct tlclk_alarms), GFP_KERNEL);
-	if (!alarm_events) {
-		ret = -ENOMEM;
-		goto out1;
-	}
 
 	/* Read telecom clock IRQ number (Set by BIOS) */
 	if (!request_region(TLCLK_BASE, 8, "telco_clock")) {
@@ -796,7 +800,6 @@ static int __init tlclk_init(void)
 		ret = -EBUSY;
 		goto out2;
 	}
-	telclk_interrupt = (inb(TLCLK_REG7) & 0x0f);
 
 	if (0x0F == telclk_interrupt ) { /* not MCPBL0010 ? */
 		printk(KERN_ERR "telclk_interrupt = 0x%x non-mcpbl0010 hw.\n",
@@ -837,8 +840,8 @@ out3:
 	release_region(TLCLK_BASE, 8);
 out2:
 	kfree(alarm_events);
-out1:
 	unregister_chrdev(tlclk_major, "telco_clock");
+out1:
 	return ret;
 }
 
diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig
index aacdeed93320..58b4c573d176 100644
--- a/drivers/char/tpm/Kconfig
+++ b/drivers/char/tpm/Kconfig
@@ -9,7 +9,7 @@ menuconfig TCG_TPM
 	imply SECURITYFS
 	select CRYPTO
 	select CRYPTO_HASH_INFO
-	---help---
+	help
 	  If you have a TPM security chip in your system, which
 	  implements the Trusted Computing Group's specification,
 	  say Yes and it will be accessible from within Linux.  For
@@ -31,7 +31,7 @@ config HW_RANDOM_TPM
 	bool "TPM HW Random Number Generator support"
 	depends on TCG_TPM && HW_RANDOM && !(TCG_TPM=y && HW_RANDOM=m)
 	default y
-	---help---
+	help
 	  This setting exposes the TPM's Random Number Generator as a hwrng
 	  device. This allows the kernel to collect randomness from the TPM at
 	  boot, and provides the TPM randomines in /dev/hwrng.
@@ -40,7 +40,7 @@ config HW_RANDOM_TPM
 
 config TCG_TIS_CORE
 	tristate
-	---help---
+	help
 	TCG TIS TPM core driver. It implements the TPM TCG TIS logic and hooks
 	into the TPM kernel APIs. Physical layers will register against it.
 
@@ -48,7 +48,7 @@ config TCG_TIS
 	tristate "TPM Interface Specification 1.2 Interface / TPM 2.0 FIFO Interface"
 	depends on X86 || OF
 	select TCG_TIS_CORE
-	---help---
+	help
 	  If you have a TPM security chip that is compliant with the
 	  TCG TIS 1.2 TPM specification (TPM1.2) or the TCG PTP FIFO
 	  specification (TPM2.0) say Yes and it will be accessible from
@@ -59,7 +59,7 @@ config TCG_TIS_SPI
 	tristate "TPM Interface Specification 1.3 Interface / TPM 2.0 FIFO Interface - (SPI)"
 	depends on SPI
 	select TCG_TIS_CORE
-	---help---
+	help
 	  If you have a TPM security chip which is connected to a regular,
 	  non-tcg SPI master (i.e. most embedded platforms) that is compliant with the
 	  TCG TIS 1.3 TPM specification (TPM1.2) or the TCG PTP FIFO
@@ -77,7 +77,7 @@ config TCG_TIS_SPI_CR50
 config TCG_TIS_I2C_ATMEL
 	tristate "TPM Interface Specification 1.2 Interface (I2C - Atmel)"
 	depends on I2C
-	---help---
+	help
 	  If you have an Atmel I2C TPM security chip say Yes and it will be
 	  accessible from within Linux.
 	  To compile this driver as a module, choose M here; the module will
@@ -86,7 +86,7 @@ config TCG_TIS_I2C_ATMEL
 config TCG_TIS_I2C_INFINEON
 	tristate "TPM Interface Specification 1.2 Interface (I2C - Infineon)"
 	depends on I2C
-	---help---
+	help
 	  If you have a TPM security chip that is compliant with the
 	  TCG TIS 1.2 TPM specification and Infineon's I2C Protocol Stack
 	  Specification 0.20 say Yes and it will be accessible from within
@@ -97,7 +97,7 @@ config TCG_TIS_I2C_INFINEON
 config TCG_TIS_I2C_NUVOTON
 	tristate "TPM Interface Specification 1.2 Interface (I2C - Nuvoton)"
 	depends on I2C
-	---help---
+	help
 	  If you have a TPM security chip with an I2C interface from
 	  Nuvoton Technology Corp. say Yes and it will be accessible
 	  from within Linux.
@@ -107,7 +107,7 @@ config TCG_TIS_I2C_NUVOTON
 config TCG_NSC
 	tristate "National Semiconductor TPM Interface"
 	depends on X86
-	---help---
+	help
 	  If you have a TPM security chip from National Semiconductor 
 	  say Yes and it will be accessible from within Linux.  To 
 	  compile this driver as a module, choose M here; the module 
@@ -116,7 +116,7 @@ config TCG_NSC
 config TCG_ATMEL
 	tristate "Atmel TPM Interface"
 	depends on PPC64 || HAS_IOPORT_MAP
-	---help---
+	help
 	  If you have a TPM security chip from Atmel say Yes and it 
 	  will be accessible from within Linux.  To compile this driver 
 	  as a module, choose M here; the module will be called tpm_atmel.
@@ -124,7 +124,7 @@ config TCG_ATMEL
 config TCG_INFINEON
 	tristate "Infineon Technologies TPM Interface"
 	depends on PNP
-	---help---
+	help
 	  If you have a TPM security chip from Infineon Technologies
 	  (either SLD 9630 TT 1.1 or SLB 9635 TT 1.2) say Yes and it
 	  will be accessible from within Linux.
@@ -136,7 +136,7 @@ config TCG_INFINEON
 config TCG_IBMVTPM
 	tristate "IBM VTPM Interface"
 	depends on PPC_PSERIES
-	---help---
+	help
 	  If you have IBM virtual TPM (VTPM) support say Yes and it
 	  will be accessible from within Linux.  To compile this driver
 	  as a module, choose M here; the module will be called tpm_ibmvtpm.
@@ -145,7 +145,7 @@ config TCG_XEN
 	tristate "XEN TPM Interface"
 	depends on TCG_TPM && XEN
 	select XEN_XENBUS_FRONTEND
-	---help---
+	help
 	  If you want to make TPM support available to a Xen user domain,
 	  say Yes and it will be accessible from within Linux. See
 	  the manpages for xl, xl.conf, and docs/misc/vtpm.txt in
@@ -156,7 +156,7 @@ config TCG_XEN
 config TCG_CRB
 	tristate "TPM 2.0 CRB Interface"
 	depends on ACPI
-	---help---
+	help
 	  If you have a TPM security chip that is compliant with the
 	  TCG CRB 2.0 TPM specification say Yes and it will be accessible
 	  from within Linux.  To compile this driver as a module, choose
@@ -165,7 +165,7 @@ config TCG_CRB
 config TCG_VTPM_PROXY
 	tristate "VTPM Proxy Interface"
 	depends on TCG_TPM
-	---help---
+	help
 	  This driver proxies for an emulated TPM (vTPM) running in userspace.
 	  A device /dev/vtpmx is provided that creates a device pair
 	  /dev/vtpmX and a server-side file descriptor on which the vTPM
diff --git a/drivers/char/tpm/st33zp24/Kconfig b/drivers/char/tpm/st33zp24/Kconfig
index e582145076dc..601c2ae5bfcb 100644
--- a/drivers/char/tpm/st33zp24/Kconfig
+++ b/drivers/char/tpm/st33zp24/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config TCG_TIS_ST33ZP24
 	tristate
-	---help---
+	help
 	  STMicroelectronics ST33ZP24 core driver. It implements the core
 	  TPM1.2 logic and hooks into the TPM kernel APIs. Physical layers will
 	  register against it.
@@ -13,7 +13,7 @@ config TCG_TIS_ST33ZP24_I2C
 	tristate "STMicroelectronics TPM Interface Specification 1.2 Interface (I2C)"
 	depends on I2C
 	select TCG_TIS_ST33ZP24
-	---help---
+	help
 	  This module adds support for the STMicroelectronics TPM security chip
 	  ST33ZP24 with i2c interface.
 	  To compile this driver as a module, choose M here; the module will be
@@ -23,7 +23,7 @@ config TCG_TIS_ST33ZP24_SPI
 	tristate "STMicroelectronics TPM Interface Specification 1.2 Interface (SPI)"
 	depends on SPI
 	select TCG_TIS_ST33ZP24
-	---help---
+	help
 	  This module adds support for the STMicroelectronics TPM security chip
 	  ST33ZP24 with spi interface.
 	  To compile this driver as a module, choose M here; the module will be
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index bcb257baed06..69934c0c3dd8 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -1,5 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0
 
+config HAVE_CLK
+	bool
+	help
+	  The <linux/clk.h> calls support software clock gating and
+	  thus are a key power management tool on many systems.
+
 config CLKDEV_LOOKUP
 	bool
 	select HAVE_CLK
@@ -7,26 +13,35 @@ config CLKDEV_LOOKUP
 config HAVE_CLK_PREPARE
 	bool
 
-config COMMON_CLK
+config HAVE_LEGACY_CLK # TODO: Remove once all legacy users are migrated
 	bool
+	select HAVE_CLK
+	help
+	  Select this option when the clock API in <linux/clk.h> is implemented
+	  by platform/architecture code. This method is deprecated. Modern
+	  code should select COMMON_CLK instead and not define a custom
+	  'struct clk'.
+
+menuconfig COMMON_CLK
+	bool "Common Clock Framework"
+	depends on !HAVE_LEGACY_CLK
 	select HAVE_CLK_PREPARE
 	select CLKDEV_LOOKUP
 	select SRCU
 	select RATIONAL
-	---help---
+	help
 	  The common clock framework is a single definition of struct
 	  clk, useful across many platforms, as well as an
 	  implementation of the clock API in include/linux/clk.h.
 	  Architectures utilizing the common struct clk should select
 	  this option.
 
-menu "Common Clock Framework"
-	depends on COMMON_CLK
+if COMMON_CLK
 
 config COMMON_CLK_WM831X
 	tristate "Clock driver for WM831x/2x PMICs"
 	depends on MFD_WM831X
-	---help---
+	help
 	  Supports the clocking subsystem of the WM831x/2x series of
 	  PMICs from Wolfson Microelectronics.
 
@@ -35,14 +50,14 @@ source "drivers/clk/versatile/Kconfig"
 config CLK_HSDK
 	bool "PLL Driver for HSDK platform"
 	depends on OF || COMPILE_TEST
-	---help---
+	help
 	  This driver supports the HSDK core, system, ddr, tunnel and hdmi PLLs
 	  control.
 
 config COMMON_CLK_MAX77686
 	tristate "Clock driver for Maxim 77620/77686/77802 MFD"
 	depends on MFD_MAX77686 || MFD_MAX77620 || COMPILE_TEST
-	---help---
+	help
 	  This driver supports Maxim 77620/77686/77802 crystal oscillator
 	  clock.
 
@@ -55,7 +70,7 @@ config COMMON_CLK_MAX9485
 config COMMON_CLK_RK808
 	tristate "Clock driver for RK805/RK808/RK809/RK817/RK818"
 	depends on MFD_RK808
-	---help---
+	help
 	  This driver supports RK805, RK809 and RK817, RK808 and RK818 crystal oscillator clock.
 	  These multi-function devices have two fixed-rate oscillators, clocked at 32KHz each.
 	  Clkout1 is always on, Clkout2 can off by control register.
@@ -65,7 +80,7 @@ config COMMON_CLK_HI655X
 	depends on (MFD_HI655X_PMIC || COMPILE_TEST)
 	depends on REGMAP
 	default MFD_HI655X_PMIC
-	---help---
+	help
 	  This driver supports the hi655x PMIC clock. This
 	  multi-function device has one fixed-rate oscillator, clocked
 	  at 32KHz.
@@ -73,7 +88,7 @@ config COMMON_CLK_HI655X
 config COMMON_CLK_SCMI
 	tristate "Clock driver controlled via SCMI interface"
 	depends on ARM_SCMI_PROTOCOL || COMPILE_TEST
-	  ---help---
+	help
 	  This driver provides support for clocks that are controlled
 	  by firmware that implements the SCMI interface.
 
@@ -83,7 +98,7 @@ config COMMON_CLK_SCMI
 config COMMON_CLK_SCPI
 	tristate "Clock driver controlled via SCPI interface"
 	depends on ARM_SCPI_PROTOCOL || COMPILE_TEST
-	  ---help---
+	help
 	  This driver provides support for clocks that are controlled
 	  by firmware that implements the SCPI interface.
 
@@ -106,7 +121,7 @@ config COMMON_CLK_SI5351
 	depends on I2C
 	select REGMAP_I2C
 	select RATIONAL
-	---help---
+	help
 	  This driver supports Silicon Labs 5351A/B/C programmable clock
 	  generators.
 
@@ -148,7 +163,7 @@ config COMMON_CLK_CDCE706
 	depends on I2C
 	select REGMAP_I2C
 	select RATIONAL
-	---help---
+	help
 	  This driver supports TI CDCE706 programmable 3-PLL clock synthesizer.
 
 config COMMON_CLK_CDCE925
@@ -191,7 +206,7 @@ config COMMON_CLK_GEMINI
 	depends on ARCH_GEMINI || COMPILE_TEST
 	select MFD_SYSCON
 	select RESET_CONTROLLER
-	---help---
+	help
 	  This driver supports the SoC clocks on the Cortina Systems Gemini
 	  platform, also known as SL3516 or CS3516.
 
@@ -201,7 +216,7 @@ config COMMON_CLK_ASPEED
 	default ARCH_ASPEED
 	select MFD_SYSCON
 	select RESET_CONTROLLER
-	---help---
+	help
 	  This driver supports the SoC clocks on the Aspeed BMC platforms.
 
 	  The G4 and G5 series, including the ast2400 and ast2500, are supported
@@ -210,7 +225,7 @@ config COMMON_CLK_ASPEED
 config COMMON_CLK_S2MPS11
 	tristate "Clock driver for S2MPS1X/S5M8767 MFD"
 	depends on MFD_SEC_CORE || COMPILE_TEST
-	---help---
+	help
 	  This driver supports S2MPS11/S2MPS14/S5M8767 crystal oscillator
 	  clock. These multi-function devices have two (S2MPS14) or three
 	  (S2MPS11, S5M8767) fixed-rate oscillators, clocked at 32KHz each.
@@ -218,7 +233,7 @@ config COMMON_CLK_S2MPS11
 config CLK_TWL6040
 	tristate "External McPDM functional clock from twl6040"
 	depends on TWL6040_CORE
-	---help---
+	help
 	  Enable the external functional clock support on OMAP4+ platforms for
 	  McPDM. McPDM module is using the external bit clock on the McPDM bus
 	  as functional clock.
@@ -233,7 +248,7 @@ config COMMON_CLK_AXI_CLKGEN
 config CLK_QORIQ
 	bool "Clock driver for Freescale QorIQ platforms"
 	depends on (PPC_E500MC || ARM || ARM64 || COMPILE_TEST) && OF
-	---help---
+	help
 	  This adds the clock driver support for Freescale QorIQ platforms
 	  using common clock framework.
 
@@ -251,8 +266,8 @@ config COMMON_CLK_XGENE
 	bool "Clock driver for APM XGene SoC"
 	default ARCH_XGENE
 	depends on ARM64 || COMPILE_TEST
-	---help---
-	  Sypport for the APM X-Gene SoC reference, PLL, and device clocks.
+	help
+	  Support for the APM X-Gene SoC reference, PLL, and device clocks.
 
 config COMMON_CLK_LOCHNAGAR
 	tristate "Cirrus Logic Lochnagar clock driver"
@@ -265,26 +280,26 @@ config COMMON_CLK_NXP
 	def_bool COMMON_CLK && (ARCH_LPC18XX || ARCH_LPC32XX)
 	select REGMAP_MMIO if ARCH_LPC32XX
 	select MFD_SYSCON if ARCH_LPC18XX
-	---help---
+	help
 	  Support for clock providers on NXP platforms.
 
 config COMMON_CLK_PALMAS
 	tristate "Clock driver for TI Palmas devices"
 	depends on MFD_PALMAS
-	---help---
+	help
 	  This driver supports TI Palmas devices 32KHz output KG and KG_AUDIO
 	  using common clock framework.
 
 config COMMON_CLK_PWM
 	tristate "Clock driver for PWMs used as clock outputs"
 	depends on PWM
-	---help---
+	help
 	  Adapter driver so that any PWM output can be (mis)used as clock signal
 	  at 50% duty cycle.
 
 config COMMON_CLK_PXA
 	def_bool COMMON_CLK && ARCH_PXA
-	---help---
+	help
 	  Support for the Marvell PXA SoC.
 
 config COMMON_CLK_PIC32
@@ -294,7 +309,7 @@ config COMMON_CLK_OXNAS
 	bool "Clock driver for the OXNAS SoC Family"
 	depends on ARCH_OXNAS || COMPILE_TEST
 	select MFD_SYSCON
-	---help---
+	help
 	  Support for the OXNAS SoC Family clocks.
 
 config COMMON_CLK_VC5
@@ -326,6 +341,12 @@ config COMMON_CLK_MMP2
 	help
 	  Support for Marvell MMP2 and MMP3 SoC clocks
 
+config COMMON_CLK_MMP2_AUDIO
+        tristate "Clock driver for MMP2 Audio subsystem"
+        depends on COMMON_CLK_MMP2 || COMPILE_TEST
+        help
+          This driver supports clocks for Audio subsystem on MMP2 SoC.
+
 config COMMON_CLK_BD718XX
 	tristate "Clock driver for 32K clk gates on ROHM PMICs"
 	depends on MFD_ROHM_BD718XX || MFD_ROHM_BD70528 || MFD_ROHM_BD71828
@@ -341,6 +362,7 @@ config COMMON_CLK_FIXED_MMIO
 
 source "drivers/clk/actions/Kconfig"
 source "drivers/clk/analogbits/Kconfig"
+source "drivers/clk/baikal-t1/Kconfig"
 source "drivers/clk/bcm/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
 source "drivers/clk/imgtec/Kconfig"
@@ -360,6 +382,7 @@ source "drivers/clk/sunxi-ng/Kconfig"
 source "drivers/clk/tegra/Kconfig"
 source "drivers/clk/ti/Kconfig"
 source "drivers/clk/uniphier/Kconfig"
+source "drivers/clk/x86/Kconfig"
 source "drivers/clk/zynqmp/Kconfig"
 
-endmenu
+endif
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index f4169cc2fd31..ca9af11d3391 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -75,6 +75,7 @@ obj-y					+= analogbits/
 obj-$(CONFIG_COMMON_CLK_AT91)		+= at91/
 obj-$(CONFIG_ARCH_ARTPEC)		+= axis/
 obj-$(CONFIG_ARC_PLAT_AXS10X)		+= axs10x/
+obj-$(CONFIG_CLK_BAIKAL_T1)		+= baikal-t1/
 obj-y					+= bcm/
 obj-$(CONFIG_ARCH_BERLIN)		+= berlin/
 obj-$(CONFIG_ARCH_DAVINCI)		+= davinci/
@@ -104,17 +105,18 @@ obj-$(CONFIG_COMMON_CLK_SAMSUNG)	+= samsung/
 obj-$(CONFIG_CLK_SIFIVE)		+= sifive/
 obj-$(CONFIG_ARCH_SIRF)			+= sirf/
 obj-$(CONFIG_ARCH_SOCFPGA)		+= socfpga/
+obj-$(CONFIG_ARCH_AGILEX)		+= socfpga/
+obj-$(CONFIG_ARCH_STRATIX10)		+= socfpga/
 obj-$(CONFIG_PLAT_SPEAR)		+= spear/
-obj-$(CONFIG_ARCH_SPRD)			+= sprd/
+obj-y					+= sprd/
 obj-$(CONFIG_ARCH_STI)			+= st/
-obj-$(CONFIG_ARCH_STRATIX10)		+= socfpga/
 obj-$(CONFIG_ARCH_SUNXI)		+= sunxi/
 obj-$(CONFIG_SUNXI_CCU)			+= sunxi-ng/
 obj-$(CONFIG_ARCH_TEGRA)		+= tegra/
 obj-y					+= ti/
 obj-$(CONFIG_CLK_UNIPHIER)		+= uniphier/
 obj-$(CONFIG_ARCH_U8500)		+= ux500/
-obj-$(CONFIG_COMMON_CLK_VERSATILE)	+= versatile/
+obj-y					+= versatile/
 ifeq ($(CONFIG_COMMON_CLK), y)
 obj-$(CONFIG_X86)			+= x86/
 endif
diff --git a/drivers/clk/at91/at91rm9200.c b/drivers/clk/at91/at91rm9200.c
index c44a431b6c97..38bdb4981315 100644
--- a/drivers/clk/at91/at91rm9200.c
+++ b/drivers/clk/at91/at91rm9200.c
@@ -98,9 +98,9 @@ static void __init at91rm9200_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	at91rm9200_pmc = pmc_data_allocate(PMC_MAIN + 1,
+	at91rm9200_pmc = pmc_data_allocate(PMC_PLLBCK + 1,
 					    nck(at91rm9200_systemck),
-					    nck(at91rm9200_periphck), 0);
+					    nck(at91rm9200_periphck), 0, 4);
 	if (!at91rm9200_pmc)
 		return;
 
@@ -123,12 +123,16 @@ static void __init at91rm9200_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91rm9200_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_pll(regmap, "pllbck", "mainck", 1,
 				   &at91rm9200_pll_layout,
 				   &rm9200_pll_characteristics);
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91rm9200_pmc->chws[PMC_PLLBCK] = hw;
+
 	parent_names[0] = slowxtal_name;
 	parent_names[1] = "mainck";
 	parent_names[2] = "pllack";
@@ -159,6 +163,8 @@ static void __init at91rm9200_pmc_setup(struct device_node *np)
 						    &at91rm9200_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		at91rm9200_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(at91rm9200_systemck); i++) {
@@ -187,7 +193,7 @@ static void __init at91rm9200_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(at91rm9200_pmc);
+	kfree(at91rm9200_pmc);
 }
 /*
  * While the TCB can be used as the clocksource, the system timer is most likely
diff --git a/drivers/clk/at91/at91sam9260.c b/drivers/clk/at91/at91sam9260.c
index a9d4234758d7..6d0723aa8b13 100644
--- a/drivers/clk/at91/at91sam9260.c
+++ b/drivers/clk/at91/at91sam9260.c
@@ -352,9 +352,10 @@ static void __init at91sam926x_pmc_setup(struct device_node *np,
 	if (IS_ERR(regmap))
 		return;
 
-	at91sam9260_pmc = pmc_data_allocate(PMC_MAIN + 1,
+	at91sam9260_pmc = pmc_data_allocate(PMC_PLLBCK + 1,
 					    ndck(data->sck, data->num_sck),
-					    ndck(data->pck, data->num_pck), 0);
+					    ndck(data->pck, data->num_pck),
+					    0, data->num_progck);
 	if (!at91sam9260_pmc)
 		return;
 
@@ -398,12 +399,16 @@ static void __init at91sam926x_pmc_setup(struct device_node *np,
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91sam9260_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_pll(regmap, "pllbck", "mainck", 1,
 				   data->pllb_layout,
 				   data->pllb_characteristics);
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91sam9260_pmc->chws[PMC_PLLBCK] = hw;
+
 	parent_names[0] = slck_name;
 	parent_names[1] = "mainck";
 	parent_names[2] = "pllack";
@@ -434,6 +439,8 @@ static void __init at91sam926x_pmc_setup(struct device_node *np,
 						    &at91rm9200_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		at91sam9260_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < data->num_sck; i++) {
@@ -462,7 +469,7 @@ static void __init at91sam926x_pmc_setup(struct device_node *np,
 	return;
 
 err_free:
-	pmc_data_free(at91sam9260_pmc);
+	kfree(at91sam9260_pmc);
 }
 
 static void __init at91sam9260_pmc_setup(struct device_node *np)
diff --git a/drivers/clk/at91/at91sam9g45.c b/drivers/clk/at91/at91sam9g45.c
index 38a7d2d2df0c..9873b583c260 100644
--- a/drivers/clk/at91/at91sam9g45.c
+++ b/drivers/clk/at91/at91sam9g45.c
@@ -115,9 +115,9 @@ static void __init at91sam9g45_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	at91sam9g45_pmc = pmc_data_allocate(PMC_MAIN + 1,
+	at91sam9g45_pmc = pmc_data_allocate(PMC_PLLACK + 1,
 					    nck(at91sam9g45_systemck),
-					    nck(at91sam9g45_periphck), 0);
+					    nck(at91sam9g45_periphck), 0, 2);
 	if (!at91sam9g45_pmc)
 		return;
 
@@ -143,6 +143,8 @@ static void __init at91sam9g45_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91sam9g45_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_utmi(regmap, NULL, "utmick", "mainck");
 	if (IS_ERR(hw))
 		goto err_free;
@@ -182,6 +184,8 @@ static void __init at91sam9g45_pmc_setup(struct device_node *np)
 						    &at91sam9g45_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		at91sam9g45_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(at91sam9g45_systemck); i++) {
@@ -210,7 +214,7 @@ static void __init at91sam9g45_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(at91sam9g45_pmc);
+	kfree(at91sam9g45_pmc);
 }
 /*
  * The TCB is used as the clocksource so its clock is needed early. This means
diff --git a/drivers/clk/at91/at91sam9n12.c b/drivers/clk/at91/at91sam9n12.c
index 8bb39d2ba84b..630dc5d87171 100644
--- a/drivers/clk/at91/at91sam9n12.c
+++ b/drivers/clk/at91/at91sam9n12.c
@@ -128,8 +128,8 @@ static void __init at91sam9n12_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	at91sam9n12_pmc = pmc_data_allocate(PMC_MAIN + 1,
-					   nck(at91sam9n12_systemck), 31, 0);
+	at91sam9n12_pmc = pmc_data_allocate(PMC_PLLBCK + 1,
+					   nck(at91sam9n12_systemck), 31, 0, 2);
 	if (!at91sam9n12_pmc)
 		return;
 
@@ -162,11 +162,15 @@ static void __init at91sam9n12_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91sam9n12_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_pll(regmap, "pllbck", "mainck", 1,
 				   &at91rm9200_pll_layout, &pllb_characteristics);
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91sam9n12_pmc->chws[PMC_PLLBCK] = hw;
+
 	parent_names[0] = slck_name;
 	parent_names[1] = "mainck";
 	parent_names[2] = "plladivck";
@@ -198,6 +202,8 @@ static void __init at91sam9n12_pmc_setup(struct device_node *np)
 						    &at91sam9x5_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		at91sam9n12_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(at91sam9n12_systemck); i++) {
@@ -228,7 +234,7 @@ static void __init at91sam9n12_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(at91sam9n12_pmc);
+	kfree(at91sam9n12_pmc);
 }
 /*
  * The TCB is used as the clocksource so its clock is needed early. This means
diff --git a/drivers/clk/at91/at91sam9rl.c b/drivers/clk/at91/at91sam9rl.c
index 77fe83a73bf4..0d1cc44b056f 100644
--- a/drivers/clk/at91/at91sam9rl.c
+++ b/drivers/clk/at91/at91sam9rl.c
@@ -87,9 +87,9 @@ static void __init at91sam9rl_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	at91sam9rl_pmc = pmc_data_allocate(PMC_MAIN + 1,
+	at91sam9rl_pmc = pmc_data_allocate(PMC_PLLACK + 1,
 					   nck(at91sam9rl_systemck),
-					   nck(at91sam9rl_periphck), 0);
+					   nck(at91sam9rl_periphck), 0, 2);
 	if (!at91sam9rl_pmc)
 		return;
 
@@ -105,6 +105,8 @@ static void __init at91sam9rl_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91sam9rl_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_utmi(regmap, NULL, "utmick", "mainck");
 	if (IS_ERR(hw))
 		goto err_free;
@@ -138,6 +140,8 @@ static void __init at91sam9rl_pmc_setup(struct device_node *np)
 						    &at91rm9200_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		at91sam9rl_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(at91sam9rl_systemck); i++) {
@@ -166,6 +170,6 @@ static void __init at91sam9rl_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(at91sam9rl_pmc);
+	kfree(at91sam9rl_pmc);
 }
 CLK_OF_DECLARE_DRIVER(at91sam9rl_pmc, "atmel,at91sam9rl-pmc", at91sam9rl_pmc_setup);
diff --git a/drivers/clk/at91/at91sam9x5.c b/drivers/clk/at91/at91sam9x5.c
index 086cf0b4955c..0ce3da080287 100644
--- a/drivers/clk/at91/at91sam9x5.c
+++ b/drivers/clk/at91/at91sam9x5.c
@@ -150,8 +150,8 @@ static void __init at91sam9x5_pmc_setup(struct device_node *np,
 	if (IS_ERR(regmap))
 		return;
 
-	at91sam9x5_pmc = pmc_data_allocate(PMC_MAIN + 1,
-					   nck(at91sam9x5_systemck), 31, 0);
+	at91sam9x5_pmc = pmc_data_allocate(PMC_PLLACK + 1,
+					   nck(at91sam9x5_systemck), 31, 0, 2);
 	if (!at91sam9x5_pmc)
 		return;
 
@@ -184,6 +184,8 @@ static void __init at91sam9x5_pmc_setup(struct device_node *np,
 	if (IS_ERR(hw))
 		goto err_free;
 
+	at91sam9x5_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_utmi(regmap, NULL, "utmick", "mainck");
 	if (IS_ERR(hw))
 		goto err_free;
@@ -227,6 +229,8 @@ static void __init at91sam9x5_pmc_setup(struct device_node *np,
 						    &at91sam9x5_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		at91sam9x5_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(at91sam9x5_systemck); i++) {
@@ -278,7 +282,7 @@ static void __init at91sam9x5_pmc_setup(struct device_node *np,
 	return;
 
 err_free:
-	pmc_data_free(at91sam9x5_pmc);
+	kfree(at91sam9x5_pmc);
 }
 
 static void __init at91sam9g15_pmc_setup(struct device_node *np)
diff --git a/drivers/clk/at91/pmc.c b/drivers/clk/at91/pmc.c
index b71515acdec1..20ee9dccee78 100644
--- a/drivers/clk/at91/pmc.c
+++ b/drivers/clk/at91/pmc.c
@@ -67,6 +67,10 @@ struct clk_hw *of_clk_hw_pmc_get(struct of_phandle_args *clkspec, void *data)
 		if (idx < pmc_data->ngck)
 			return pmc_data->ghws[idx];
 		break;
+	case PMC_TYPE_PROGRAMMABLE:
+		if (idx < pmc_data->npck)
+			return pmc_data->pchws[idx];
+		break;
 	default:
 		break;
 	}
@@ -76,48 +80,34 @@ struct clk_hw *of_clk_hw_pmc_get(struct of_phandle_args *clkspec, void *data)
 	return ERR_PTR(-EINVAL);
 }
 
-void pmc_data_free(struct pmc_data *pmc_data)
-{
-	kfree(pmc_data->chws);
-	kfree(pmc_data->shws);
-	kfree(pmc_data->phws);
-	kfree(pmc_data->ghws);
-}
-
 struct pmc_data *pmc_data_allocate(unsigned int ncore, unsigned int nsystem,
-				   unsigned int nperiph, unsigned int ngck)
+				   unsigned int nperiph, unsigned int ngck,
+				   unsigned int npck)
 {
-	struct pmc_data *pmc_data = kzalloc(sizeof(*pmc_data), GFP_KERNEL);
+	unsigned int num_clks = ncore + nsystem + nperiph + ngck + npck;
+	struct pmc_data *pmc_data;
 
+	pmc_data = kzalloc(struct_size(pmc_data, hwtable, num_clks),
+			   GFP_KERNEL);
 	if (!pmc_data)
 		return NULL;
 
 	pmc_data->ncore = ncore;
-	pmc_data->chws = kcalloc(ncore, sizeof(struct clk_hw *), GFP_KERNEL);
-	if (!pmc_data->chws)
-		goto err;
+	pmc_data->chws = pmc_data->hwtable;
 
 	pmc_data->nsystem = nsystem;
-	pmc_data->shws = kcalloc(nsystem, sizeof(struct clk_hw *), GFP_KERNEL);
-	if (!pmc_data->shws)
-		goto err;
+	pmc_data->shws = pmc_data->chws + ncore;
 
 	pmc_data->nperiph = nperiph;
-	pmc_data->phws = kcalloc(nperiph, sizeof(struct clk_hw *), GFP_KERNEL);
-	if (!pmc_data->phws)
-		goto err;
+	pmc_data->phws = pmc_data->shws + nsystem;
 
 	pmc_data->ngck = ngck;
-	pmc_data->ghws = kcalloc(ngck, sizeof(struct clk_hw *), GFP_KERNEL);
-	if (!pmc_data->ghws)
-		goto err;
+	pmc_data->ghws = pmc_data->phws + nperiph;
 
-	return pmc_data;
-
-err:
-	pmc_data_free(pmc_data);
+	pmc_data->npck = npck;
+	pmc_data->pchws = pmc_data->ghws + ngck;
 
-	return NULL;
+	return pmc_data;
 }
 
 #ifdef CONFIG_PM
@@ -274,8 +264,11 @@ static int __init pmc_register_ops(void)
 	struct device_node *np;
 
 	np = of_find_matching_node(NULL, sama5d2_pmc_dt_ids);
+	if (!np)
+		return -ENODEV;
 
 	pmcreg = device_node_to_regmap(np);
+	of_node_put(np);
 	if (IS_ERR(pmcreg))
 		return PTR_ERR(pmcreg);
 
diff --git a/drivers/clk/at91/pmc.h b/drivers/clk/at91/pmc.h
index 9b8db9cdcda5..df616f2937e7 100644
--- a/drivers/clk/at91/pmc.h
+++ b/drivers/clk/at91/pmc.h
@@ -24,6 +24,10 @@ struct pmc_data {
 	struct clk_hw **phws;
 	unsigned int ngck;
 	struct clk_hw **ghws;
+	unsigned int npck;
+	struct clk_hw **pchws;
+
+	struct clk_hw *hwtable[];
 };
 
 struct clk_range {
@@ -94,8 +98,8 @@ struct clk_pcr_layout {
 #define ndck(a, s) (a[s - 1].id + 1)
 #define nck(a) (a[ARRAY_SIZE(a) - 1].id + 1)
 struct pmc_data *pmc_data_allocate(unsigned int ncore, unsigned int nsystem,
-				   unsigned int nperiph, unsigned int ngck);
-void pmc_data_free(struct pmc_data *pmc_data);
+				   unsigned int nperiph, unsigned int ngck,
+				   unsigned int npck);
 
 int of_at91_get_clk_range(struct device_node *np, const char *propname,
 			  struct clk_range *range);
diff --git a/drivers/clk/at91/sam9x60.c b/drivers/clk/at91/sam9x60.c
index cc19e8fb83be..3e20aa68259f 100644
--- a/drivers/clk/at91/sam9x60.c
+++ b/drivers/clk/at91/sam9x60.c
@@ -182,10 +182,10 @@ static void __init sam9x60_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	sam9x60_pmc = pmc_data_allocate(PMC_MAIN + 1,
+	sam9x60_pmc = pmc_data_allocate(PMC_PLLACK + 1,
 					nck(sam9x60_systemck),
 					nck(sam9x60_periphck),
-					nck(sam9x60_gck));
+					nck(sam9x60_gck), 8);
 	if (!sam9x60_pmc)
 		return;
 
@@ -214,6 +214,8 @@ static void __init sam9x60_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	sam9x60_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = sam9x60_clk_register_pll(regmap, &pmc_pll_lock, "upllck",
 				      "main_osc", 1, &upll_characteristics);
 	if (IS_ERR(hw))
@@ -255,6 +257,8 @@ static void __init sam9x60_pmc_setup(struct device_node *np)
 						    &sam9x60_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		sam9x60_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(sam9x60_systemck); i++) {
@@ -299,7 +303,7 @@ static void __init sam9x60_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(sam9x60_pmc);
+	kfree(sam9x60_pmc);
 }
 /* Some clks are used for a clocksource */
 CLK_OF_DECLARE(sam9x60_pmc, "microchip,sam9x60-pmc", sam9x60_pmc_setup);
diff --git a/drivers/clk/at91/sama5d2.c b/drivers/clk/at91/sama5d2.c
index ff7e3f727082..d69421d71daf 100644
--- a/drivers/clk/at91/sama5d2.c
+++ b/drivers/clk/at91/sama5d2.c
@@ -89,6 +89,7 @@ static const struct {
 	{ .n = "i2s1_clk",    .id = 55, .r = { .min = 0, .max = 83000000 }, },
 	{ .n = "can0_clk",    .id = 56, .r = { .min = 0, .max = 83000000 }, },
 	{ .n = "can1_clk",    .id = 57, .r = { .min = 0, .max = 83000000 }, },
+	{ .n = "ptc_clk",     .id = 58, .r = { .min = 0, .max = 83000000 }, },
 	{ .n = "classd_clk",  .id = 59, .r = { .min = 0, .max = 83000000 }, },
 };
 
@@ -166,10 +167,10 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	sama5d2_pmc = pmc_data_allocate(PMC_I2S1_MUX + 1,
+	sama5d2_pmc = pmc_data_allocate(PMC_AUDIOPLLCK + 1,
 					nck(sama5d2_systemck),
 					nck(sama5d2_periph32ck),
-					nck(sama5d2_gck));
+					nck(sama5d2_gck), 3);
 	if (!sama5d2_pmc)
 		return;
 
@@ -202,6 +203,8 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	sama5d2_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_audio_pll_frac(regmap, "audiopll_fracck",
 					      "mainck");
 	if (IS_ERR(hw))
@@ -217,6 +220,8 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	sama5d2_pmc->chws[PMC_AUDIOPLLCK] = hw;
+
 	regmap_sfr = syscon_regmap_lookup_by_compatible("atmel,sama5d2-sfr");
 	if (IS_ERR(regmap_sfr))
 		regmap_sfr = NULL;
@@ -267,6 +272,8 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 						    &sama5d2_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		sama5d2_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(sama5d2_systemck); i++) {
@@ -350,6 +357,6 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(sama5d2_pmc);
+	kfree(sama5d2_pmc);
 }
 CLK_OF_DECLARE_DRIVER(sama5d2_pmc, "atmel,sama5d2-pmc", sama5d2_pmc_setup);
diff --git a/drivers/clk/at91/sama5d3.c b/drivers/clk/at91/sama5d3.c
index 88506f909c08..5e4e44dd4c37 100644
--- a/drivers/clk/at91/sama5d3.c
+++ b/drivers/clk/at91/sama5d3.c
@@ -125,9 +125,9 @@ static void __init sama5d3_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	sama5d3_pmc = pmc_data_allocate(PMC_MAIN + 1,
+	sama5d3_pmc = pmc_data_allocate(PMC_PLLACK + 1,
 					nck(sama5d3_systemck),
-					nck(sama5d3_periphck), 0);
+					nck(sama5d3_periphck), 0, 3);
 	if (!sama5d3_pmc)
 		return;
 
@@ -158,6 +158,8 @@ static void __init sama5d3_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	sama5d3_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_utmi(regmap, NULL, "utmick", "mainck");
 	if (IS_ERR(hw))
 		goto err_free;
@@ -201,6 +203,8 @@ static void __init sama5d3_pmc_setup(struct device_node *np)
 						    &at91sam9x5_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		sama5d3_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(sama5d3_systemck); i++) {
@@ -231,7 +235,7 @@ static void __init sama5d3_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(sama5d3_pmc);
+	kfree(sama5d3_pmc);
 }
 /*
  * The TCB is used as the clocksource so its clock is needed early. This means
diff --git a/drivers/clk/at91/sama5d4.c b/drivers/clk/at91/sama5d4.c
index a6dee4a3b6e4..662ff5fa6e98 100644
--- a/drivers/clk/at91/sama5d4.c
+++ b/drivers/clk/at91/sama5d4.c
@@ -140,9 +140,9 @@ static void __init sama5d4_pmc_setup(struct device_node *np)
 	if (IS_ERR(regmap))
 		return;
 
-	sama5d4_pmc = pmc_data_allocate(PMC_MCK2 + 1,
+	sama5d4_pmc = pmc_data_allocate(PMC_PLLACK + 1,
 					nck(sama5d4_systemck),
-					nck(sama5d4_periph32ck), 0);
+					nck(sama5d4_periph32ck), 0, 3);
 	if (!sama5d4_pmc)
 		return;
 
@@ -173,6 +173,8 @@ static void __init sama5d4_pmc_setup(struct device_node *np)
 	if (IS_ERR(hw))
 		goto err_free;
 
+	sama5d4_pmc->chws[PMC_PLLACK] = hw;
+
 	hw = at91_clk_register_utmi(regmap, NULL, "utmick", "mainck");
 	if (IS_ERR(hw))
 		goto err_free;
@@ -224,6 +226,8 @@ static void __init sama5d4_pmc_setup(struct device_node *np)
 						    &at91sam9x5_programmable_layout);
 		if (IS_ERR(hw))
 			goto err_free;
+
+		sama5d4_pmc->pchws[i] = hw;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(sama5d4_systemck); i++) {
@@ -267,6 +271,6 @@ static void __init sama5d4_pmc_setup(struct device_node *np)
 	return;
 
 err_free:
-	pmc_data_free(sama5d4_pmc);
+	kfree(sama5d4_pmc);
 }
 CLK_OF_DECLARE_DRIVER(sama5d4_pmc, "atmel,sama5d4-pmc", sama5d4_pmc_setup);
diff --git a/drivers/clk/baikal-t1/Kconfig b/drivers/clk/baikal-t1/Kconfig
new file mode 100644
index 000000000000..03102f1094bc
--- /dev/null
+++ b/drivers/clk/baikal-t1/Kconfig
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config CLK_BAIKAL_T1
+	bool "Baikal-T1 Clocks Control Unit interface"
+	depends on (MIPS_BAIKAL_T1 && OF) || COMPILE_TEST
+	default MIPS_BAIKAL_T1
+	help
+	  Clocks Control Unit is the core of Baikal-T1 SoC System Controller
+	  responsible for the chip subsystems clocking and resetting. It
+	  consists of multiple global clock domains, which can be reset by
+	  means of the CCU control registers. These domains and devices placed
+	  in them are fed with clocks generated by a hierarchy of PLLs,
+	  configurable and fixed clock dividers. Enable this option to be able
+	  to select Baikal-T1 CCU PLLs and Dividers drivers.
+
+if CLK_BAIKAL_T1
+
+config CLK_BT1_CCU_PLL
+	bool "Baikal-T1 CCU PLLs support"
+	select MFD_SYSCON
+	default MIPS_BAIKAL_T1
+	help
+	  Enable this to support the PLLs embedded into the Baikal-T1 SoC
+	  System Controller. These are five PLLs placed at the root of the
+	  clocks hierarchy, right after an external reference oscillator
+	  (normally of 25MHz). They are used to generate high frequency
+	  signals, which are either directly wired to the consumers (like
+	  CPUs, DDR, etc.) or passed over the clock dividers to be only
+	  then used as an individual reference clock of a target device.
+
+config CLK_BT1_CCU_DIV
+	bool "Baikal-T1 CCU Dividers support"
+	select RESET_CONTROLLER
+	select MFD_SYSCON
+	default MIPS_BAIKAL_T1
+	help
+	  Enable this to support the CCU dividers used to distribute clocks
+	  between AXI-bus and system devices coming from CCU PLLs of Baikal-T1
+	  SoC. CCU dividers can be either configurable or with fixed divider,
+	  either gateable or ungateable. Some of the CCU dividers can be as well
+	  used to reset the domains they're supplying clock to.
+
+endif
diff --git a/drivers/clk/baikal-t1/Makefile b/drivers/clk/baikal-t1/Makefile
new file mode 100644
index 000000000000..b3b9590b95ed
--- /dev/null
+++ b/drivers/clk/baikal-t1/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_CLK_BT1_CCU_PLL) += ccu-pll.o clk-ccu-pll.o
+obj-$(CONFIG_CLK_BT1_CCU_DIV) += ccu-div.o clk-ccu-div.o
diff --git a/drivers/clk/baikal-t1/ccu-div.c b/drivers/clk/baikal-t1/ccu-div.c
new file mode 100644
index 000000000000..4062092d67f9
--- /dev/null
+++ b/drivers/clk/baikal-t1/ccu-div.c
@@ -0,0 +1,602 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *   Dmitry Dunaev <dmitry.dunaev@baikalelectronics.ru>
+ *
+ * Baikal-T1 CCU Dividers interface driver
+ */
+
+#define pr_fmt(fmt) "bt1-ccu-div: " fmt
+
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/slab.h>
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/spinlock.h>
+#include <linux/regmap.h>
+#include <linux/delay.h>
+#include <linux/time64.h>
+#include <linux/debugfs.h>
+
+#include "ccu-div.h"
+
+#define CCU_DIV_CTL			0x00
+#define CCU_DIV_CTL_EN			BIT(0)
+#define CCU_DIV_CTL_RST			BIT(1)
+#define CCU_DIV_CTL_SET_CLKDIV		BIT(2)
+#define CCU_DIV_CTL_CLKDIV_FLD		4
+#define CCU_DIV_CTL_CLKDIV_MASK(_width) \
+	GENMASK((_width) + CCU_DIV_CTL_CLKDIV_FLD - 1, CCU_DIV_CTL_CLKDIV_FLD)
+#define CCU_DIV_CTL_LOCK_SHIFTED	BIT(27)
+#define CCU_DIV_CTL_LOCK_NORMAL		BIT(31)
+
+#define CCU_DIV_RST_DELAY_US		1
+#define CCU_DIV_LOCK_CHECK_RETRIES	50
+
+#define CCU_DIV_CLKDIV_MIN		0
+#define CCU_DIV_CLKDIV_MAX(_mask) \
+	((_mask) >> CCU_DIV_CTL_CLKDIV_FLD)
+
+/*
+ * Use the next two methods until there are generic field setter and
+ * getter available with non-constant mask support.
+ */
+static inline u32 ccu_div_get(u32 mask, u32 val)
+{
+	return (val & mask) >> CCU_DIV_CTL_CLKDIV_FLD;
+}
+
+static inline u32 ccu_div_prep(u32 mask, u32 val)
+{
+	return (val << CCU_DIV_CTL_CLKDIV_FLD) & mask;
+}
+
+static inline unsigned long ccu_div_lock_delay_ns(unsigned long ref_clk,
+						  unsigned long div)
+{
+	u64 ns = 4ULL * (div ?: 1) * NSEC_PER_SEC;
+
+	do_div(ns, ref_clk);
+
+	return ns;
+}
+
+static inline unsigned long ccu_div_calc_freq(unsigned long ref_clk,
+					      unsigned long div)
+{
+	return ref_clk / (div ?: 1);
+}
+
+static int ccu_div_var_update_clkdiv(struct ccu_div *div,
+				     unsigned long parent_rate,
+				     unsigned long divider)
+{
+	unsigned long nd;
+	u32 val = 0;
+	u32 lock;
+	int count;
+
+	nd = ccu_div_lock_delay_ns(parent_rate, divider);
+
+	if (div->features & CCU_DIV_LOCK_SHIFTED)
+		lock = CCU_DIV_CTL_LOCK_SHIFTED;
+	else
+		lock = CCU_DIV_CTL_LOCK_NORMAL;
+
+	regmap_update_bits(div->sys_regs, div->reg_ctl,
+			   CCU_DIV_CTL_SET_CLKDIV, CCU_DIV_CTL_SET_CLKDIV);
+
+	/*
+	 * Until there is nsec-version of readl_poll_timeout() is available
+	 * we have to implement the next polling loop.
+	 */
+	count = CCU_DIV_LOCK_CHECK_RETRIES;
+	do {
+		ndelay(nd);
+		regmap_read(div->sys_regs, div->reg_ctl, &val);
+		if (val & lock)
+			return 0;
+	} while (--count);
+
+	return -ETIMEDOUT;
+}
+
+static int ccu_div_var_enable(struct clk_hw *hw)
+{
+	struct clk_hw *parent_hw = clk_hw_get_parent(hw);
+	struct ccu_div *div = to_ccu_div(hw);
+	unsigned long flags;
+	u32 val = 0;
+	int ret;
+
+	if (!parent_hw) {
+		pr_err("Can't enable '%s' with no parent", clk_hw_get_name(hw));
+		return -EINVAL;
+	}
+
+	regmap_read(div->sys_regs, div->reg_ctl, &val);
+	if (val & CCU_DIV_CTL_EN)
+		return 0;
+
+	spin_lock_irqsave(&div->lock, flags);
+	ret = ccu_div_var_update_clkdiv(div, clk_hw_get_rate(parent_hw),
+					ccu_div_get(div->mask, val));
+	if (!ret)
+		regmap_update_bits(div->sys_regs, div->reg_ctl,
+				   CCU_DIV_CTL_EN, CCU_DIV_CTL_EN);
+	spin_unlock_irqrestore(&div->lock, flags);
+	if (ret)
+		pr_err("Divider '%s' lock timed out\n", clk_hw_get_name(hw));
+
+	return ret;
+}
+
+static int ccu_div_gate_enable(struct clk_hw *hw)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&div->lock, flags);
+	regmap_update_bits(div->sys_regs, div->reg_ctl,
+			   CCU_DIV_CTL_EN, CCU_DIV_CTL_EN);
+	spin_unlock_irqrestore(&div->lock, flags);
+
+	return 0;
+}
+
+static void ccu_div_gate_disable(struct clk_hw *hw)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&div->lock, flags);
+	regmap_update_bits(div->sys_regs, div->reg_ctl, CCU_DIV_CTL_EN, 0);
+	spin_unlock_irqrestore(&div->lock, flags);
+}
+
+static int ccu_div_gate_is_enabled(struct clk_hw *hw)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	u32 val = 0;
+
+	regmap_read(div->sys_regs, div->reg_ctl, &val);
+
+	return !!(val & CCU_DIV_CTL_EN);
+}
+
+static unsigned long ccu_div_var_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	unsigned long divider;
+	u32 val = 0;
+
+	regmap_read(div->sys_regs, div->reg_ctl, &val);
+	divider = ccu_div_get(div->mask, val);
+
+	return ccu_div_calc_freq(parent_rate, divider);
+}
+
+static inline unsigned long ccu_div_var_calc_divider(unsigned long rate,
+						     unsigned long parent_rate,
+						     unsigned int mask)
+{
+	unsigned long divider;
+
+	divider = parent_rate / rate;
+	return clamp_t(unsigned long, divider, CCU_DIV_CLKDIV_MIN,
+		       CCU_DIV_CLKDIV_MAX(mask));
+}
+
+static long ccu_div_var_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *parent_rate)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	unsigned long divider;
+
+	divider = ccu_div_var_calc_divider(rate, *parent_rate, div->mask);
+
+	return ccu_div_calc_freq(*parent_rate, divider);
+}
+
+/*
+ * This method is used for the clock divider blocks, which support the
+ * on-the-fly rate change. So due to lacking the EN bit functionality
+ * they can't be gated before the rate adjustment.
+ */
+static int ccu_div_var_set_rate_slow(struct clk_hw *hw, unsigned long rate,
+				     unsigned long parent_rate)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	unsigned long flags, divider;
+	u32 val;
+	int ret;
+
+	divider = ccu_div_var_calc_divider(rate, parent_rate, div->mask);
+	if (divider == 1 && div->features & CCU_DIV_SKIP_ONE) {
+		divider = 0;
+	} else if (div->features & CCU_DIV_SKIP_ONE_TO_THREE) {
+		if (divider == 1 || divider == 2)
+			divider = 0;
+		else if (divider == 3)
+			divider = 4;
+	}
+
+	val = ccu_div_prep(div->mask, divider);
+
+	spin_lock_irqsave(&div->lock, flags);
+	regmap_update_bits(div->sys_regs, div->reg_ctl, div->mask, val);
+	ret = ccu_div_var_update_clkdiv(div, parent_rate, divider);
+	spin_unlock_irqrestore(&div->lock, flags);
+	if (ret)
+		pr_err("Divider '%s' lock timed out\n", clk_hw_get_name(hw));
+
+	return ret;
+}
+
+/*
+ * This method is used for the clock divider blocks, which don't support
+ * the on-the-fly rate change.
+ */
+static int ccu_div_var_set_rate_fast(struct clk_hw *hw, unsigned long rate,
+				     unsigned long parent_rate)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	unsigned long flags, divider;
+	u32 val;
+
+	divider = ccu_div_var_calc_divider(rate, parent_rate, div->mask);
+	val = ccu_div_prep(div->mask, divider);
+
+	/*
+	 * Also disable the clock divider block if it was enabled by default
+	 * or by the bootloader.
+	 */
+	spin_lock_irqsave(&div->lock, flags);
+	regmap_update_bits(div->sys_regs, div->reg_ctl,
+			   div->mask | CCU_DIV_CTL_EN, val);
+	spin_unlock_irqrestore(&div->lock, flags);
+
+	return 0;
+}
+
+static unsigned long ccu_div_fixed_recalc_rate(struct clk_hw *hw,
+					       unsigned long parent_rate)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+
+	return ccu_div_calc_freq(parent_rate, div->divider);
+}
+
+static long ccu_div_fixed_round_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long *parent_rate)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+
+	return ccu_div_calc_freq(*parent_rate, div->divider);
+}
+
+static int ccu_div_fixed_set_rate(struct clk_hw *hw, unsigned long rate,
+				  unsigned long parent_rate)
+{
+	return 0;
+}
+
+int ccu_div_reset_domain(struct ccu_div *div)
+{
+	unsigned long flags;
+
+	if (!div || !(div->features & CCU_DIV_RESET_DOMAIN))
+		return -EINVAL;
+
+	spin_lock_irqsave(&div->lock, flags);
+	regmap_update_bits(div->sys_regs, div->reg_ctl,
+			   CCU_DIV_CTL_RST, CCU_DIV_CTL_RST);
+	spin_unlock_irqrestore(&div->lock, flags);
+
+	/* The next delay must be enough to cover all the resets. */
+	udelay(CCU_DIV_RST_DELAY_US);
+
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+
+struct ccu_div_dbgfs_bit {
+	struct ccu_div *div;
+	const char *name;
+	u32 mask;
+};
+
+#define CCU_DIV_DBGFS_BIT_ATTR(_name, _mask) {	\
+		.name = _name,			\
+		.mask = _mask			\
+	}
+
+static const struct ccu_div_dbgfs_bit ccu_div_bits[] = {
+	CCU_DIV_DBGFS_BIT_ATTR("div_en", CCU_DIV_CTL_EN),
+	CCU_DIV_DBGFS_BIT_ATTR("div_rst", CCU_DIV_CTL_RST),
+	CCU_DIV_DBGFS_BIT_ATTR("div_bypass", CCU_DIV_CTL_SET_CLKDIV),
+	CCU_DIV_DBGFS_BIT_ATTR("div_lock", CCU_DIV_CTL_LOCK_NORMAL)
+};
+
+#define CCU_DIV_DBGFS_BIT_NUM	ARRAY_SIZE(ccu_div_bits)
+
+/*
+ * It can be dangerous to change the Divider settings behind clock framework
+ * back, therefore we don't provide any kernel config based compile time option
+ * for this feature to enable.
+ */
+#undef CCU_DIV_ALLOW_WRITE_DEBUGFS
+#ifdef CCU_DIV_ALLOW_WRITE_DEBUGFS
+
+static int ccu_div_dbgfs_bit_set(void *priv, u64 val)
+{
+	const struct ccu_div_dbgfs_bit *bit = priv;
+	struct ccu_div *div = bit->div;
+	unsigned long flags;
+
+	spin_lock_irqsave(&div->lock, flags);
+	regmap_update_bits(div->sys_regs, div->reg_ctl,
+			   bit->mask, val ? bit->mask : 0);
+	spin_unlock_irqrestore(&div->lock, flags);
+
+	return 0;
+}
+
+static int ccu_div_dbgfs_var_clkdiv_set(void *priv, u64 val)
+{
+	struct ccu_div *div = priv;
+	unsigned long flags;
+	u32 data;
+
+	val = clamp_t(u64, val, CCU_DIV_CLKDIV_MIN,
+		      CCU_DIV_CLKDIV_MAX(div->mask));
+	data = ccu_div_prep(div->mask, val);
+
+	spin_lock_irqsave(&div->lock, flags);
+	regmap_update_bits(div->sys_regs, div->reg_ctl, div->mask, data);
+	spin_unlock_irqrestore(&div->lock, flags);
+
+	return 0;
+}
+
+#define ccu_div_dbgfs_mode		0644
+
+#else /* !CCU_DIV_ALLOW_WRITE_DEBUGFS */
+
+#define ccu_div_dbgfs_bit_set		NULL
+#define ccu_div_dbgfs_var_clkdiv_set	NULL
+#define ccu_div_dbgfs_mode		0444
+
+#endif /* !CCU_DIV_ALLOW_WRITE_DEBUGFS */
+
+static int ccu_div_dbgfs_bit_get(void *priv, u64 *val)
+{
+	const struct ccu_div_dbgfs_bit *bit = priv;
+	struct ccu_div *div = bit->div;
+	u32 data = 0;
+
+	regmap_read(div->sys_regs, div->reg_ctl, &data);
+	*val = !!(data & bit->mask);
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(ccu_div_dbgfs_bit_fops,
+	ccu_div_dbgfs_bit_get, ccu_div_dbgfs_bit_set, "%llu\n");
+
+static int ccu_div_dbgfs_var_clkdiv_get(void *priv, u64 *val)
+{
+	struct ccu_div *div = priv;
+	u32 data = 0;
+
+	regmap_read(div->sys_regs, div->reg_ctl, &data);
+	*val = ccu_div_get(div->mask, data);
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(ccu_div_dbgfs_var_clkdiv_fops,
+	ccu_div_dbgfs_var_clkdiv_get, ccu_div_dbgfs_var_clkdiv_set, "%llu\n");
+
+static int ccu_div_dbgfs_fixed_clkdiv_get(void *priv, u64 *val)
+{
+	struct ccu_div *div = priv;
+
+	*val = div->divider;
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(ccu_div_dbgfs_fixed_clkdiv_fops,
+	ccu_div_dbgfs_fixed_clkdiv_get, NULL, "%llu\n");
+
+static void ccu_div_var_debug_init(struct clk_hw *hw, struct dentry *dentry)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	struct ccu_div_dbgfs_bit *bits;
+	int didx, bidx, num = 2;
+	const char *name;
+
+	num += !!(div->flags & CLK_SET_RATE_GATE) +
+		!!(div->features & CCU_DIV_RESET_DOMAIN);
+
+	bits = kcalloc(num, sizeof(*bits), GFP_KERNEL);
+	if (!bits)
+		return;
+
+	for (didx = 0, bidx = 0; bidx < CCU_DIV_DBGFS_BIT_NUM; ++bidx) {
+		name = ccu_div_bits[bidx].name;
+		if (!(div->flags & CLK_SET_RATE_GATE) &&
+		    !strcmp("div_en", name)) {
+			continue;
+		}
+
+		if (!(div->features & CCU_DIV_RESET_DOMAIN) &&
+		    !strcmp("div_rst", name)) {
+			continue;
+		}
+
+		bits[didx] = ccu_div_bits[bidx];
+		bits[didx].div = div;
+
+		if (div->features & CCU_DIV_LOCK_SHIFTED &&
+		    !strcmp("div_lock", name)) {
+			bits[didx].mask = CCU_DIV_CTL_LOCK_SHIFTED;
+		}
+
+		debugfs_create_file_unsafe(bits[didx].name, ccu_div_dbgfs_mode,
+					   dentry, &bits[didx],
+					   &ccu_div_dbgfs_bit_fops);
+		++didx;
+	}
+
+	debugfs_create_file_unsafe("div_clkdiv", ccu_div_dbgfs_mode, dentry,
+				   div, &ccu_div_dbgfs_var_clkdiv_fops);
+}
+
+static void ccu_div_gate_debug_init(struct clk_hw *hw, struct dentry *dentry)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+	struct ccu_div_dbgfs_bit *bit;
+
+	bit = kmalloc(sizeof(*bit), GFP_KERNEL);
+	if (!bit)
+		return;
+
+	*bit = ccu_div_bits[0];
+	bit->div = div;
+	debugfs_create_file_unsafe(bit->name, ccu_div_dbgfs_mode, dentry, bit,
+				   &ccu_div_dbgfs_bit_fops);
+
+	debugfs_create_file_unsafe("div_clkdiv", 0400, dentry, div,
+				   &ccu_div_dbgfs_fixed_clkdiv_fops);
+}
+
+static void ccu_div_fixed_debug_init(struct clk_hw *hw, struct dentry *dentry)
+{
+	struct ccu_div *div = to_ccu_div(hw);
+
+	debugfs_create_file_unsafe("div_clkdiv", 0400, dentry, div,
+				   &ccu_div_dbgfs_fixed_clkdiv_fops);
+}
+
+#else /* !CONFIG_DEBUG_FS */
+
+#define ccu_div_var_debug_init NULL
+#define ccu_div_gate_debug_init NULL
+#define ccu_div_fixed_debug_init NULL
+
+#endif /* !CONFIG_DEBUG_FS */
+
+static const struct clk_ops ccu_div_var_gate_to_set_ops = {
+	.enable = ccu_div_var_enable,
+	.disable = ccu_div_gate_disable,
+	.is_enabled = ccu_div_gate_is_enabled,
+	.recalc_rate = ccu_div_var_recalc_rate,
+	.round_rate = ccu_div_var_round_rate,
+	.set_rate = ccu_div_var_set_rate_fast,
+	.debug_init = ccu_div_var_debug_init
+};
+
+static const struct clk_ops ccu_div_var_nogate_ops = {
+	.recalc_rate = ccu_div_var_recalc_rate,
+	.round_rate = ccu_div_var_round_rate,
+	.set_rate = ccu_div_var_set_rate_slow,
+	.debug_init = ccu_div_var_debug_init
+};
+
+static const struct clk_ops ccu_div_gate_ops = {
+	.enable = ccu_div_gate_enable,
+	.disable = ccu_div_gate_disable,
+	.is_enabled = ccu_div_gate_is_enabled,
+	.recalc_rate = ccu_div_fixed_recalc_rate,
+	.round_rate = ccu_div_fixed_round_rate,
+	.set_rate = ccu_div_fixed_set_rate,
+	.debug_init = ccu_div_gate_debug_init
+};
+
+static const struct clk_ops ccu_div_fixed_ops = {
+	.recalc_rate = ccu_div_fixed_recalc_rate,
+	.round_rate = ccu_div_fixed_round_rate,
+	.set_rate = ccu_div_fixed_set_rate,
+	.debug_init = ccu_div_fixed_debug_init
+};
+
+struct ccu_div *ccu_div_hw_register(const struct ccu_div_init_data *div_init)
+{
+	struct clk_parent_data parent_data = { };
+	struct clk_init_data hw_init = { };
+	struct ccu_div *div;
+	int ret;
+
+	if (!div_init)
+		return ERR_PTR(-EINVAL);
+
+	div = kzalloc(sizeof(*div), GFP_KERNEL);
+	if (!div)
+		return ERR_PTR(-ENOMEM);
+
+	/*
+	 * Note since Baikal-T1 System Controller registers are MMIO-backed
+	 * we won't check the regmap IO operations return status, because it
+	 * must be zero anyway.
+	 */
+	div->hw.init = &hw_init;
+	div->id = div_init->id;
+	div->reg_ctl = div_init->base + CCU_DIV_CTL;
+	div->sys_regs = div_init->sys_regs;
+	div->flags = div_init->flags;
+	div->features = div_init->features;
+	spin_lock_init(&div->lock);
+
+	hw_init.name = div_init->name;
+	hw_init.flags = div_init->flags;
+
+	if (div_init->type == CCU_DIV_VAR) {
+		if (hw_init.flags & CLK_SET_RATE_GATE)
+			hw_init.ops = &ccu_div_var_gate_to_set_ops;
+		else
+			hw_init.ops = &ccu_div_var_nogate_ops;
+		div->mask = CCU_DIV_CTL_CLKDIV_MASK(div_init->width);
+	} else if (div_init->type == CCU_DIV_GATE) {
+		hw_init.ops = &ccu_div_gate_ops;
+		div->divider = div_init->divider;
+	} else if (div_init->type == CCU_DIV_FIXED) {
+		hw_init.ops = &ccu_div_fixed_ops;
+		div->divider = div_init->divider;
+	} else {
+		ret = -EINVAL;
+		goto err_free_div;
+	}
+
+	if (!div_init->parent_name) {
+		ret = -EINVAL;
+		goto err_free_div;
+	}
+	parent_data.fw_name = div_init->parent_name;
+	hw_init.parent_data = &parent_data;
+	hw_init.num_parents = 1;
+
+	ret = of_clk_hw_register(div_init->np, &div->hw);
+	if (ret)
+		goto err_free_div;
+
+	return div;
+
+err_free_div:
+	kfree(div);
+
+	return ERR_PTR(ret);
+}
+
+void ccu_div_hw_unregister(struct ccu_div *div)
+{
+	clk_hw_unregister(&div->hw);
+
+	kfree(div);
+}
diff --git a/drivers/clk/baikal-t1/ccu-div.h b/drivers/clk/baikal-t1/ccu-div.h
new file mode 100644
index 000000000000..795665caefbd
--- /dev/null
+++ b/drivers/clk/baikal-t1/ccu-div.h
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Baikal-T1 CCU Dividers interface driver
+ */
+#ifndef __CLK_BT1_CCU_DIV_H__
+#define __CLK_BT1_CCU_DIV_H__
+
+#include <linux/clk-provider.h>
+#include <linux/spinlock.h>
+#include <linux/regmap.h>
+#include <linux/bits.h>
+#include <linux/of.h>
+
+/*
+ * CCU Divider private flags
+ * @CCU_DIV_SKIP_ONE: Due to some reason divider can't be set to 1.
+ *		      It can be 0 though, which is functionally the same.
+ * @CCU_DIV_SKIP_ONE_TO_THREE: For some reason divider can't be within [1,3].
+ *			       It can be either 0 or greater than 3.
+ * @CCU_DIV_LOCK_SHIFTED: Find lock-bit at non-standard position.
+ * @CCU_DIV_RESET_DOMAIN: Provide reset clock domain method.
+ */
+#define CCU_DIV_SKIP_ONE		BIT(1)
+#define CCU_DIV_SKIP_ONE_TO_THREE	BIT(2)
+#define CCU_DIV_LOCK_SHIFTED		BIT(3)
+#define CCU_DIV_RESET_DOMAIN		BIT(4)
+
+/*
+ * enum ccu_div_type - CCU Divider types
+ * @CCU_DIV_VAR: Clocks gate with variable divider.
+ * @CCU_DIV_GATE: Clocks gate with fixed divider.
+ * @CCU_DIV_FIXED: Ungateable clock with fixed divider.
+ */
+enum ccu_div_type {
+	CCU_DIV_VAR,
+	CCU_DIV_GATE,
+	CCU_DIV_FIXED
+};
+
+/*
+ * struct ccu_div_init_data - CCU Divider initialization data
+ * @id: Clocks private identifier.
+ * @name: Clocks name.
+ * @parent_name: Parent clocks name in a fw node.
+ * @base: Divider register base address with respect to the sys_regs base.
+ * @sys_regs: Baikal-T1 System Controller registers map.
+ * @np: Pointer to the node describing the CCU Dividers.
+ * @type: CCU divider type (variable, fixed with and without gate).
+ * @width: Divider width if it's variable.
+ * @divider: Divider fixed value.
+ * @flags: CCU Divider clock flags.
+ * @features: CCU Divider private features.
+ */
+struct ccu_div_init_data {
+	unsigned int id;
+	const char *name;
+	const char *parent_name;
+	unsigned int base;
+	struct regmap *sys_regs;
+	struct device_node *np;
+	enum ccu_div_type type;
+	union {
+		unsigned int width;
+		unsigned int divider;
+	};
+	unsigned long flags;
+	unsigned long features;
+};
+
+/*
+ * struct ccu_div - CCU Divider descriptor
+ * @hw: clk_hw of the divider.
+ * @id: Clock private identifier.
+ * @reg_ctl: Divider control register base address.
+ * @sys_regs: Baikal-T1 System Controller registers map.
+ * @lock: Divider state change spin-lock.
+ * @mask: Divider field mask.
+ * @divider: Divider fixed value.
+ * @flags: Divider clock flags.
+ * @features: CCU Divider private features.
+ */
+struct ccu_div {
+	struct clk_hw hw;
+	unsigned int id;
+	unsigned int reg_ctl;
+	struct regmap *sys_regs;
+	spinlock_t lock;
+	union {
+		u32 mask;
+		unsigned int divider;
+	};
+	unsigned long flags;
+	unsigned long features;
+};
+#define to_ccu_div(_hw) container_of(_hw, struct ccu_div, hw)
+
+static inline struct clk_hw *ccu_div_get_clk_hw(struct ccu_div *div)
+{
+	return div ? &div->hw : NULL;
+}
+
+struct ccu_div *ccu_div_hw_register(const struct ccu_div_init_data *init);
+
+void ccu_div_hw_unregister(struct ccu_div *div);
+
+int ccu_div_reset_domain(struct ccu_div *div);
+
+#endif /* __CLK_BT1_CCU_DIV_H__ */
diff --git a/drivers/clk/baikal-t1/ccu-pll.c b/drivers/clk/baikal-t1/ccu-pll.c
new file mode 100644
index 000000000000..13ef28001439
--- /dev/null
+++ b/drivers/clk/baikal-t1/ccu-pll.c
@@ -0,0 +1,558 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *   Dmitry Dunaev <dmitry.dunaev@baikalelectronics.ru>
+ *
+ * Baikal-T1 CCU PLL interface driver
+ */
+
+#define pr_fmt(fmt) "bt1-ccu-pll: " fmt
+
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/limits.h>
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/slab.h>
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/spinlock.h>
+#include <linux/regmap.h>
+#include <linux/iopoll.h>
+#include <linux/time64.h>
+#include <linux/rational.h>
+#include <linux/debugfs.h>
+
+#include "ccu-pll.h"
+
+#define CCU_PLL_CTL			0x000
+#define CCU_PLL_CTL_EN			BIT(0)
+#define CCU_PLL_CTL_RST			BIT(1)
+#define CCU_PLL_CTL_CLKR_FLD		2
+#define CCU_PLL_CTL_CLKR_MASK		GENMASK(7, CCU_PLL_CTL_CLKR_FLD)
+#define CCU_PLL_CTL_CLKF_FLD		8
+#define CCU_PLL_CTL_CLKF_MASK		GENMASK(20, CCU_PLL_CTL_CLKF_FLD)
+#define CCU_PLL_CTL_CLKOD_FLD		21
+#define CCU_PLL_CTL_CLKOD_MASK		GENMASK(24, CCU_PLL_CTL_CLKOD_FLD)
+#define CCU_PLL_CTL_BYPASS		BIT(30)
+#define CCU_PLL_CTL_LOCK		BIT(31)
+#define CCU_PLL_CTL1			0x004
+#define CCU_PLL_CTL1_BWADJ_FLD		3
+#define CCU_PLL_CTL1_BWADJ_MASK		GENMASK(14, CCU_PLL_CTL1_BWADJ_FLD)
+
+#define CCU_PLL_LOCK_CHECK_RETRIES	50
+
+#define CCU_PLL_NR_MAX \
+	((CCU_PLL_CTL_CLKR_MASK >> CCU_PLL_CTL_CLKR_FLD) + 1)
+#define CCU_PLL_NF_MAX \
+	((CCU_PLL_CTL_CLKF_MASK >> (CCU_PLL_CTL_CLKF_FLD + 1)) + 1)
+#define CCU_PLL_OD_MAX \
+	((CCU_PLL_CTL_CLKOD_MASK >> CCU_PLL_CTL_CLKOD_FLD) + 1)
+#define CCU_PLL_NB_MAX \
+	((CCU_PLL_CTL1_BWADJ_MASK >> CCU_PLL_CTL1_BWADJ_FLD) + 1)
+#define CCU_PLL_FDIV_MIN		427000UL
+#define CCU_PLL_FDIV_MAX		3500000000UL
+#define CCU_PLL_FOUT_MIN		200000000UL
+#define CCU_PLL_FOUT_MAX		2500000000UL
+#define CCU_PLL_FVCO_MIN		700000000UL
+#define CCU_PLL_FVCO_MAX		3500000000UL
+#define CCU_PLL_CLKOD_FACTOR		2
+
+static inline unsigned long ccu_pll_lock_delay_us(unsigned long ref_clk,
+						  unsigned long nr)
+{
+	u64 us = 500ULL * nr * USEC_PER_SEC;
+
+	do_div(us, ref_clk);
+
+	return us;
+}
+
+static inline unsigned long ccu_pll_calc_freq(unsigned long ref_clk,
+					      unsigned long nr,
+					      unsigned long nf,
+					      unsigned long od)
+{
+	u64 tmp = ref_clk;
+
+	do_div(tmp, nr);
+	tmp *= nf;
+	do_div(tmp, od);
+
+	return tmp;
+}
+
+static int ccu_pll_reset(struct ccu_pll *pll, unsigned long ref_clk,
+			 unsigned long nr)
+{
+	unsigned long ud, ut;
+	u32 val;
+
+	ud = ccu_pll_lock_delay_us(ref_clk, nr);
+	ut = ud * CCU_PLL_LOCK_CHECK_RETRIES;
+
+	regmap_update_bits(pll->sys_regs, pll->reg_ctl,
+			   CCU_PLL_CTL_RST, CCU_PLL_CTL_RST);
+
+	return regmap_read_poll_timeout_atomic(pll->sys_regs, pll->reg_ctl, val,
+					       val & CCU_PLL_CTL_LOCK, ud, ut);
+}
+
+static int ccu_pll_enable(struct clk_hw *hw)
+{
+	struct clk_hw *parent_hw = clk_hw_get_parent(hw);
+	struct ccu_pll *pll = to_ccu_pll(hw);
+	unsigned long flags;
+	u32 val = 0;
+	int ret;
+
+	if (!parent_hw) {
+		pr_err("Can't enable '%s' with no parent", clk_hw_get_name(hw));
+		return -EINVAL;
+	}
+
+	regmap_read(pll->sys_regs, pll->reg_ctl, &val);
+	if (val & CCU_PLL_CTL_EN)
+		return 0;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	regmap_write(pll->sys_regs, pll->reg_ctl, val | CCU_PLL_CTL_EN);
+	ret = ccu_pll_reset(pll, clk_hw_get_rate(parent_hw),
+			    FIELD_GET(CCU_PLL_CTL_CLKR_MASK, val) + 1);
+	spin_unlock_irqrestore(&pll->lock, flags);
+	if (ret)
+		pr_err("PLL '%s' reset timed out\n", clk_hw_get_name(hw));
+
+	return ret;
+}
+
+static void ccu_pll_disable(struct clk_hw *hw)
+{
+	struct ccu_pll *pll = to_ccu_pll(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	regmap_update_bits(pll->sys_regs, pll->reg_ctl, CCU_PLL_CTL_EN, 0);
+	spin_unlock_irqrestore(&pll->lock, flags);
+}
+
+static int ccu_pll_is_enabled(struct clk_hw *hw)
+{
+	struct ccu_pll *pll = to_ccu_pll(hw);
+	u32 val = 0;
+
+	regmap_read(pll->sys_regs, pll->reg_ctl, &val);
+
+	return !!(val & CCU_PLL_CTL_EN);
+}
+
+static unsigned long ccu_pll_recalc_rate(struct clk_hw *hw,
+					 unsigned long parent_rate)
+{
+	struct ccu_pll *pll = to_ccu_pll(hw);
+	unsigned long nr, nf, od;
+	u32 val = 0;
+
+	regmap_read(pll->sys_regs, pll->reg_ctl, &val);
+	nr = FIELD_GET(CCU_PLL_CTL_CLKR_MASK, val) + 1;
+	nf = FIELD_GET(CCU_PLL_CTL_CLKF_MASK, val) + 1;
+	od = FIELD_GET(CCU_PLL_CTL_CLKOD_MASK, val) + 1;
+
+	return ccu_pll_calc_freq(parent_rate, nr, nf, od);
+}
+
+static void ccu_pll_calc_factors(unsigned long rate, unsigned long parent_rate,
+				 unsigned long *nr, unsigned long *nf,
+				 unsigned long *od)
+{
+	unsigned long err, freq, min_err = ULONG_MAX;
+	unsigned long num, denom, n1, d1, nri;
+	unsigned long nr_max, nf_max, od_max;
+
+	/*
+	 * Make sure PLL is working with valid input signal (Fdiv). If
+	 * you want to speed the function up just reduce CCU_PLL_NR_MAX.
+	 * This will cause a worse approximation though.
+	 */
+	nri = (parent_rate / CCU_PLL_FDIV_MAX) + 1;
+	nr_max = min(parent_rate / CCU_PLL_FDIV_MIN, CCU_PLL_NR_MAX);
+
+	/*
+	 * Find a closest [nr;nf;od] vector taking into account the
+	 * limitations like: 1) 700MHz <= Fvco <= 3.5GHz, 2) PLL Od is
+	 * either 1 or even number within the acceptable range (alas 1s
+	 * is also excluded by the next loop).
+	 */
+	for (; nri <= nr_max; ++nri) {
+		/* Use Od factor to fulfill the limitation 2). */
+		num = CCU_PLL_CLKOD_FACTOR * rate;
+		denom = parent_rate / nri;
+
+		/*
+		 * Make sure Fvco is within the acceptable range to fulfill
+		 * the condition 1). Note due to the CCU_PLL_CLKOD_FACTOR value
+		 * the actual upper limit is also divided by that factor.
+		 * It's not big problem for us since practically there is no
+		 * need in clocks with that high frequency.
+		 */
+		nf_max = min(CCU_PLL_FVCO_MAX / denom, CCU_PLL_NF_MAX);
+		od_max = CCU_PLL_OD_MAX / CCU_PLL_CLKOD_FACTOR;
+
+		/*
+		 * Bypass the out-of-bound values, which can't be properly
+		 * handled by the rational fraction approximation algorithm.
+		 */
+		if (num / denom >= nf_max) {
+			n1 = nf_max;
+			d1 = 1;
+		} else if (denom / num >= od_max) {
+			n1 = 1;
+			d1 = od_max;
+		} else {
+			rational_best_approximation(num, denom, nf_max, od_max,
+						    &n1, &d1);
+		}
+
+		/* Select the best approximation of the target rate. */
+		freq = ccu_pll_calc_freq(parent_rate, nri, n1, d1);
+		err = abs((int64_t)freq - num);
+		if (err < min_err) {
+			min_err = err;
+			*nr = nri;
+			*nf = n1;
+			*od = CCU_PLL_CLKOD_FACTOR * d1;
+		}
+	}
+}
+
+static long ccu_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long *parent_rate)
+{
+	unsigned long nr = 1, nf = 1, od = 1;
+
+	ccu_pll_calc_factors(rate, *parent_rate, &nr, &nf, &od);
+
+	return ccu_pll_calc_freq(*parent_rate, nr, nf, od);
+}
+
+/*
+ * This method is used for PLLs, which support the on-the-fly dividers
+ * adjustment. So there is no need in gating such clocks.
+ */
+static int ccu_pll_set_rate_reset(struct clk_hw *hw, unsigned long rate,
+				  unsigned long parent_rate)
+{
+	struct ccu_pll *pll = to_ccu_pll(hw);
+	unsigned long nr, nf, od;
+	unsigned long flags;
+	u32 mask, val;
+	int ret;
+
+	ccu_pll_calc_factors(rate, parent_rate, &nr, &nf, &od);
+
+	mask = CCU_PLL_CTL_CLKR_MASK | CCU_PLL_CTL_CLKF_MASK |
+	       CCU_PLL_CTL_CLKOD_MASK;
+	val = FIELD_PREP(CCU_PLL_CTL_CLKR_MASK, nr - 1) |
+	      FIELD_PREP(CCU_PLL_CTL_CLKF_MASK, nf - 1) |
+	      FIELD_PREP(CCU_PLL_CTL_CLKOD_MASK, od - 1);
+
+	spin_lock_irqsave(&pll->lock, flags);
+	regmap_update_bits(pll->sys_regs, pll->reg_ctl, mask, val);
+	ret = ccu_pll_reset(pll, parent_rate, nr);
+	spin_unlock_irqrestore(&pll->lock, flags);
+	if (ret)
+		pr_err("PLL '%s' reset timed out\n", clk_hw_get_name(hw));
+
+	return ret;
+}
+
+/*
+ * This method is used for PLLs, which don't support the on-the-fly dividers
+ * adjustment. So the corresponding clocks are supposed to be gated first.
+ */
+static int ccu_pll_set_rate_norst(struct clk_hw *hw, unsigned long rate,
+				  unsigned long parent_rate)
+{
+	struct ccu_pll *pll = to_ccu_pll(hw);
+	unsigned long nr, nf, od;
+	unsigned long flags;
+	u32 mask, val;
+
+	ccu_pll_calc_factors(rate, parent_rate, &nr, &nf, &od);
+
+	/*
+	 * Disable PLL if it was enabled by default or left enabled by the
+	 * system bootloader.
+	 */
+	mask = CCU_PLL_CTL_CLKR_MASK | CCU_PLL_CTL_CLKF_MASK |
+	       CCU_PLL_CTL_CLKOD_MASK | CCU_PLL_CTL_EN;
+	val = FIELD_PREP(CCU_PLL_CTL_CLKR_MASK, nr - 1) |
+	      FIELD_PREP(CCU_PLL_CTL_CLKF_MASK, nf - 1) |
+	      FIELD_PREP(CCU_PLL_CTL_CLKOD_MASK, od - 1);
+
+	spin_lock_irqsave(&pll->lock, flags);
+	regmap_update_bits(pll->sys_regs, pll->reg_ctl, mask, val);
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+
+struct ccu_pll_dbgfs_bit {
+	struct ccu_pll *pll;
+	const char *name;
+	unsigned int reg;
+	u32 mask;
+};
+
+struct ccu_pll_dbgfs_fld {
+	struct ccu_pll *pll;
+	const char *name;
+	unsigned int reg;
+	unsigned int lsb;
+	u32 mask;
+	u32 min;
+	u32 max;
+};
+
+#define CCU_PLL_DBGFS_BIT_ATTR(_name, _reg, _mask)	\
+	{						\
+		.name = _name,				\
+		.reg = _reg,				\
+		.mask = _mask				\
+	}
+
+#define CCU_PLL_DBGFS_FLD_ATTR(_name, _reg, _lsb, _mask, _min, _max)	\
+	{								\
+		.name = _name,						\
+		.reg = _reg,						\
+		.lsb = _lsb,						\
+		.mask = _mask,						\
+		.min = _min,						\
+		.max = _max						\
+	}
+
+static const struct ccu_pll_dbgfs_bit ccu_pll_bits[] = {
+	CCU_PLL_DBGFS_BIT_ATTR("pll_en", CCU_PLL_CTL, CCU_PLL_CTL_EN),
+	CCU_PLL_DBGFS_BIT_ATTR("pll_rst", CCU_PLL_CTL, CCU_PLL_CTL_RST),
+	CCU_PLL_DBGFS_BIT_ATTR("pll_bypass", CCU_PLL_CTL, CCU_PLL_CTL_BYPASS),
+	CCU_PLL_DBGFS_BIT_ATTR("pll_lock", CCU_PLL_CTL, CCU_PLL_CTL_LOCK)
+};
+
+#define CCU_PLL_DBGFS_BIT_NUM	ARRAY_SIZE(ccu_pll_bits)
+
+static const struct ccu_pll_dbgfs_fld ccu_pll_flds[] = {
+	CCU_PLL_DBGFS_FLD_ATTR("pll_nr", CCU_PLL_CTL, CCU_PLL_CTL_CLKR_FLD,
+				CCU_PLL_CTL_CLKR_MASK, 1, CCU_PLL_NR_MAX),
+	CCU_PLL_DBGFS_FLD_ATTR("pll_nf", CCU_PLL_CTL, CCU_PLL_CTL_CLKF_FLD,
+				CCU_PLL_CTL_CLKF_MASK, 1, CCU_PLL_NF_MAX),
+	CCU_PLL_DBGFS_FLD_ATTR("pll_od", CCU_PLL_CTL, CCU_PLL_CTL_CLKOD_FLD,
+				CCU_PLL_CTL_CLKOD_MASK, 1, CCU_PLL_OD_MAX),
+	CCU_PLL_DBGFS_FLD_ATTR("pll_nb", CCU_PLL_CTL1, CCU_PLL_CTL1_BWADJ_FLD,
+				CCU_PLL_CTL1_BWADJ_MASK, 1, CCU_PLL_NB_MAX)
+};
+
+#define CCU_PLL_DBGFS_FLD_NUM	ARRAY_SIZE(ccu_pll_flds)
+
+/*
+ * It can be dangerous to change the PLL settings behind clock framework back,
+ * therefore we don't provide any kernel config based compile time option for
+ * this feature to enable.
+ */
+#undef CCU_PLL_ALLOW_WRITE_DEBUGFS
+#ifdef CCU_PLL_ALLOW_WRITE_DEBUGFS
+
+static int ccu_pll_dbgfs_bit_set(void *priv, u64 val)
+{
+	const struct ccu_pll_dbgfs_bit *bit = priv;
+	struct ccu_pll *pll = bit->pll;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	regmap_update_bits(pll->sys_regs, pll->reg_ctl + bit->reg,
+			   bit->mask, val ? bit->mask : 0);
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	return 0;
+}
+
+static int ccu_pll_dbgfs_fld_set(void *priv, u64 val)
+{
+	struct ccu_pll_dbgfs_fld *fld = priv;
+	struct ccu_pll *pll = fld->pll;
+	unsigned long flags;
+	u32 data;
+
+	val = clamp_t(u64, val, fld->min, fld->max);
+	data = ((val - 1) << fld->lsb) & fld->mask;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	regmap_update_bits(pll->sys_regs, pll->reg_ctl + fld->reg, fld->mask,
+			   data);
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	return 0;
+}
+
+#define ccu_pll_dbgfs_mode	0644
+
+#else /* !CCU_PLL_ALLOW_WRITE_DEBUGFS */
+
+#define ccu_pll_dbgfs_bit_set	NULL
+#define ccu_pll_dbgfs_fld_set	NULL
+#define ccu_pll_dbgfs_mode	0444
+
+#endif /* !CCU_PLL_ALLOW_WRITE_DEBUGFS */
+
+static int ccu_pll_dbgfs_bit_get(void *priv, u64 *val)
+{
+	struct ccu_pll_dbgfs_bit *bit = priv;
+	struct ccu_pll *pll = bit->pll;
+	u32 data = 0;
+
+	regmap_read(pll->sys_regs, pll->reg_ctl + bit->reg, &data);
+	*val = !!(data & bit->mask);
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(ccu_pll_dbgfs_bit_fops,
+	ccu_pll_dbgfs_bit_get, ccu_pll_dbgfs_bit_set, "%llu\n");
+
+static int ccu_pll_dbgfs_fld_get(void *priv, u64 *val)
+{
+	struct ccu_pll_dbgfs_fld *fld = priv;
+	struct ccu_pll *pll = fld->pll;
+	u32 data = 0;
+
+	regmap_read(pll->sys_regs, pll->reg_ctl + fld->reg, &data);
+	*val = ((data & fld->mask) >> fld->lsb) + 1;
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(ccu_pll_dbgfs_fld_fops,
+	ccu_pll_dbgfs_fld_get, ccu_pll_dbgfs_fld_set, "%llu\n");
+
+static void ccu_pll_debug_init(struct clk_hw *hw, struct dentry *dentry)
+{
+	struct ccu_pll *pll = to_ccu_pll(hw);
+	struct ccu_pll_dbgfs_bit *bits;
+	struct ccu_pll_dbgfs_fld *flds;
+	int idx;
+
+	bits = kcalloc(CCU_PLL_DBGFS_BIT_NUM, sizeof(*bits), GFP_KERNEL);
+	if (!bits)
+		return;
+
+	for (idx = 0; idx < CCU_PLL_DBGFS_BIT_NUM; ++idx) {
+		bits[idx] = ccu_pll_bits[idx];
+		bits[idx].pll = pll;
+
+		debugfs_create_file_unsafe(bits[idx].name, ccu_pll_dbgfs_mode,
+					   dentry, &bits[idx],
+					   &ccu_pll_dbgfs_bit_fops);
+	}
+
+	flds = kcalloc(CCU_PLL_DBGFS_FLD_NUM, sizeof(*flds), GFP_KERNEL);
+	if (!flds)
+		return;
+
+	for (idx = 0; idx < CCU_PLL_DBGFS_FLD_NUM; ++idx) {
+		flds[idx] = ccu_pll_flds[idx];
+		flds[idx].pll = pll;
+
+		debugfs_create_file_unsafe(flds[idx].name, ccu_pll_dbgfs_mode,
+					   dentry, &flds[idx],
+					   &ccu_pll_dbgfs_fld_fops);
+	}
+}
+
+#else /* !CONFIG_DEBUG_FS */
+
+#define ccu_pll_debug_init NULL
+
+#endif /* !CONFIG_DEBUG_FS */
+
+static const struct clk_ops ccu_pll_gate_to_set_ops = {
+	.enable = ccu_pll_enable,
+	.disable = ccu_pll_disable,
+	.is_enabled = ccu_pll_is_enabled,
+	.recalc_rate = ccu_pll_recalc_rate,
+	.round_rate = ccu_pll_round_rate,
+	.set_rate = ccu_pll_set_rate_norst,
+	.debug_init = ccu_pll_debug_init
+};
+
+static const struct clk_ops ccu_pll_straight_set_ops = {
+	.enable = ccu_pll_enable,
+	.disable = ccu_pll_disable,
+	.is_enabled = ccu_pll_is_enabled,
+	.recalc_rate = ccu_pll_recalc_rate,
+	.round_rate = ccu_pll_round_rate,
+	.set_rate = ccu_pll_set_rate_reset,
+	.debug_init = ccu_pll_debug_init
+};
+
+struct ccu_pll *ccu_pll_hw_register(const struct ccu_pll_init_data *pll_init)
+{
+	struct clk_parent_data parent_data = { };
+	struct clk_init_data hw_init = { };
+	struct ccu_pll *pll;
+	int ret;
+
+	if (!pll_init)
+		return ERR_PTR(-EINVAL);
+
+	pll = kzalloc(sizeof(*pll), GFP_KERNEL);
+	if (!pll)
+		return ERR_PTR(-ENOMEM);
+
+	/*
+	 * Note since Baikal-T1 System Controller registers are MMIO-backed
+	 * we won't check the regmap IO operations return status, because it
+	 * must be zero anyway.
+	 */
+	pll->hw.init = &hw_init;
+	pll->reg_ctl = pll_init->base + CCU_PLL_CTL;
+	pll->reg_ctl1 = pll_init->base + CCU_PLL_CTL1;
+	pll->sys_regs = pll_init->sys_regs;
+	pll->id = pll_init->id;
+	spin_lock_init(&pll->lock);
+
+	hw_init.name = pll_init->name;
+	hw_init.flags = pll_init->flags;
+
+	if (hw_init.flags & CLK_SET_RATE_GATE)
+		hw_init.ops = &ccu_pll_gate_to_set_ops;
+	else
+		hw_init.ops = &ccu_pll_straight_set_ops;
+
+	if (!pll_init->parent_name) {
+		ret = -EINVAL;
+		goto err_free_pll;
+	}
+	parent_data.fw_name = pll_init->parent_name;
+	hw_init.parent_data = &parent_data;
+	hw_init.num_parents = 1;
+
+	ret = of_clk_hw_register(pll_init->np, &pll->hw);
+	if (ret)
+		goto err_free_pll;
+
+	return pll;
+
+err_free_pll:
+	kfree(pll);
+
+	return ERR_PTR(ret);
+}
+
+void ccu_pll_hw_unregister(struct ccu_pll *pll)
+{
+	clk_hw_unregister(&pll->hw);
+
+	kfree(pll);
+}
diff --git a/drivers/clk/baikal-t1/ccu-pll.h b/drivers/clk/baikal-t1/ccu-pll.h
new file mode 100644
index 000000000000..76cd9132a219
--- /dev/null
+++ b/drivers/clk/baikal-t1/ccu-pll.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Baikal-T1 CCU PLL interface driver
+ */
+#ifndef __CLK_BT1_CCU_PLL_H__
+#define __CLK_BT1_CCU_PLL_H__
+
+#include <linux/clk-provider.h>
+#include <linux/spinlock.h>
+#include <linux/regmap.h>
+#include <linux/bits.h>
+#include <linux/of.h>
+
+/*
+ * struct ccu_pll_init_data - CCU PLL initialization data
+ * @id: Clock private identifier.
+ * @name: Clocks name.
+ * @parent_name: Clocks parent name in a fw node.
+ * @base: PLL registers base address with respect to the sys_regs base.
+ * @sys_regs: Baikal-T1 System Controller registers map.
+ * @np: Pointer to the node describing the CCU PLLs.
+ * @flags: PLL clock flags.
+ */
+struct ccu_pll_init_data {
+	unsigned int id;
+	const char *name;
+	const char *parent_name;
+	unsigned int base;
+	struct regmap *sys_regs;
+	struct device_node *np;
+	unsigned long flags;
+};
+
+/*
+ * struct ccu_pll - CCU PLL descriptor
+ * @hw: clk_hw of the PLL.
+ * @id: Clock private identifier.
+ * @reg_ctl: PLL control register base.
+ * @reg_ctl1: PLL control1 register base.
+ * @sys_regs: Baikal-T1 System Controller registers map.
+ * @lock: PLL state change spin-lock.
+ */
+struct ccu_pll {
+	struct clk_hw hw;
+	unsigned int id;
+	unsigned int reg_ctl;
+	unsigned int reg_ctl1;
+	struct regmap *sys_regs;
+	spinlock_t lock;
+};
+#define to_ccu_pll(_hw) container_of(_hw, struct ccu_pll, hw)
+
+static inline struct clk_hw *ccu_pll_get_clk_hw(struct ccu_pll *pll)
+{
+	return pll ? &pll->hw : NULL;
+}
+
+struct ccu_pll *ccu_pll_hw_register(const struct ccu_pll_init_data *init);
+
+void ccu_pll_hw_unregister(struct ccu_pll *pll);
+
+#endif /* __CLK_BT1_CCU_PLL_H__ */
diff --git a/drivers/clk/baikal-t1/clk-ccu-div.c b/drivers/clk/baikal-t1/clk-ccu-div.c
new file mode 100644
index 000000000000..f141fda12b09
--- /dev/null
+++ b/drivers/clk/baikal-t1/clk-ccu-div.c
@@ -0,0 +1,485 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *   Dmitry Dunaev <dmitry.dunaev@baikalelectronics.ru>
+ *
+ * Baikal-T1 CCU Dividers clock driver
+ */
+
+#define pr_fmt(fmt) "bt1-ccu-div: " fmt
+
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/clk-provider.h>
+#include <linux/reset-controller.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/ioport.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/bt1-ccu.h>
+#include <dt-bindings/reset/bt1-ccu.h>
+
+#include "ccu-div.h"
+
+#define CCU_AXI_MAIN_BASE		0x030
+#define CCU_AXI_DDR_BASE		0x034
+#define CCU_AXI_SATA_BASE		0x038
+#define CCU_AXI_GMAC0_BASE		0x03C
+#define CCU_AXI_GMAC1_BASE		0x040
+#define CCU_AXI_XGMAC_BASE		0x044
+#define CCU_AXI_PCIE_M_BASE		0x048
+#define CCU_AXI_PCIE_S_BASE		0x04C
+#define CCU_AXI_USB_BASE		0x050
+#define CCU_AXI_HWA_BASE		0x054
+#define CCU_AXI_SRAM_BASE		0x058
+
+#define CCU_SYS_SATA_REF_BASE		0x060
+#define CCU_SYS_APB_BASE		0x064
+#define CCU_SYS_GMAC0_BASE		0x068
+#define CCU_SYS_GMAC1_BASE		0x06C
+#define CCU_SYS_XGMAC_BASE		0x070
+#define CCU_SYS_USB_BASE		0x074
+#define CCU_SYS_PVT_BASE		0x078
+#define CCU_SYS_HWA_BASE		0x07C
+#define CCU_SYS_UART_BASE		0x084
+#define CCU_SYS_TIMER0_BASE		0x088
+#define CCU_SYS_TIMER1_BASE		0x08C
+#define CCU_SYS_TIMER2_BASE		0x090
+#define CCU_SYS_WDT_BASE		0x150
+
+#define CCU_DIV_VAR_INFO(_id, _name, _pname, _base, _width, _flags, _features) \
+	{								\
+		.id = _id,						\
+		.name = _name,						\
+		.parent_name = _pname,					\
+		.base = _base,						\
+		.type = CCU_DIV_VAR,					\
+		.width = _width,					\
+		.flags = _flags,					\
+		.features = _features					\
+	}
+
+#define CCU_DIV_GATE_INFO(_id, _name, _pname, _base, _divider)	\
+	{							\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_name = _pname,				\
+		.base = _base,					\
+		.type = CCU_DIV_GATE,				\
+		.divider = _divider				\
+	}
+
+#define CCU_DIV_FIXED_INFO(_id, _name, _pname, _divider)	\
+	{							\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_name = _pname,				\
+		.type = CCU_DIV_FIXED,				\
+		.divider = _divider				\
+	}
+
+#define CCU_DIV_RST_MAP(_rst_id, _clk_id)	\
+	{					\
+		.rst_id = _rst_id,		\
+		.clk_id = _clk_id		\
+	}
+
+struct ccu_div_info {
+	unsigned int id;
+	const char *name;
+	const char *parent_name;
+	unsigned int base;
+	enum ccu_div_type type;
+	union {
+		unsigned int width;
+		unsigned int divider;
+	};
+	unsigned long flags;
+	unsigned long features;
+};
+
+struct ccu_div_rst_map {
+	unsigned int rst_id;
+	unsigned int clk_id;
+};
+
+struct ccu_div_data {
+	struct device_node *np;
+	struct regmap *sys_regs;
+
+	unsigned int divs_num;
+	const struct ccu_div_info *divs_info;
+	struct ccu_div **divs;
+
+	unsigned int rst_num;
+	const struct ccu_div_rst_map *rst_map;
+	struct reset_controller_dev rcdev;
+};
+#define to_ccu_div_data(_rcdev) container_of(_rcdev, struct ccu_div_data, rcdev)
+
+/*
+ * AXI Main Interconnect (axi_main_clk) and DDR AXI-bus (axi_ddr_clk) clocks
+ * must be left enabled in any case, since former one is responsible for
+ * clocking a bus between CPU cores and the rest of the SoC components, while
+ * the later is clocking the AXI-bus between DDR controller and the Main
+ * Interconnect. So should any of these clocks get to be disabled, the system
+ * will literally stop working. That's why we marked them as critical.
+ */
+static const struct ccu_div_info axi_info[] = {
+	CCU_DIV_VAR_INFO(CCU_AXI_MAIN_CLK, "axi_main_clk", "pcie_clk",
+			 CCU_AXI_MAIN_BASE, 4,
+			 CLK_IS_CRITICAL, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_DDR_CLK, "axi_ddr_clk", "sata_clk",
+			 CCU_AXI_DDR_BASE, 4,
+			 CLK_IS_CRITICAL | CLK_SET_RATE_GATE,
+			 CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_SATA_CLK, "axi_sata_clk", "sata_clk",
+			 CCU_AXI_SATA_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_GMAC0_CLK, "axi_gmac0_clk", "eth_clk",
+			 CCU_AXI_GMAC0_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_GMAC1_CLK, "axi_gmac1_clk", "eth_clk",
+			 CCU_AXI_GMAC1_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_XGMAC_CLK, "axi_xgmac_clk", "eth_clk",
+			 CCU_AXI_XGMAC_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_PCIE_M_CLK, "axi_pcie_m_clk", "pcie_clk",
+			 CCU_AXI_PCIE_M_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_PCIE_S_CLK, "axi_pcie_s_clk", "pcie_clk",
+			 CCU_AXI_PCIE_S_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_USB_CLK, "axi_usb_clk", "sata_clk",
+			 CCU_AXI_USB_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_HWA_CLK, "axi_hwa_clk", "sata_clk",
+			 CCU_AXI_HWA_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_AXI_SRAM_CLK, "axi_sram_clk", "eth_clk",
+			 CCU_AXI_SRAM_BASE, 4,
+			 CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN)
+};
+
+static const struct ccu_div_rst_map axi_rst_map[] = {
+	CCU_DIV_RST_MAP(CCU_AXI_MAIN_RST, CCU_AXI_MAIN_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_DDR_RST, CCU_AXI_DDR_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_SATA_RST, CCU_AXI_SATA_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_GMAC0_RST, CCU_AXI_GMAC0_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_GMAC1_RST, CCU_AXI_GMAC1_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_XGMAC_RST, CCU_AXI_XGMAC_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_PCIE_M_RST, CCU_AXI_PCIE_M_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_PCIE_S_RST, CCU_AXI_PCIE_S_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_USB_RST, CCU_AXI_USB_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_HWA_RST, CCU_AXI_HWA_CLK),
+	CCU_DIV_RST_MAP(CCU_AXI_SRAM_RST, CCU_AXI_SRAM_CLK)
+};
+
+/*
+ * APB-bus clock is marked as critical since it's a main communication bus
+ * for the SoC devices registers IO-operations.
+ */
+static const struct ccu_div_info sys_info[] = {
+	CCU_DIV_VAR_INFO(CCU_SYS_SATA_REF_CLK, "sys_sata_ref_clk",
+			 "sata_clk", CCU_SYS_SATA_REF_BASE, 4,
+			 CLK_SET_RATE_GATE,
+			 CCU_DIV_SKIP_ONE | CCU_DIV_LOCK_SHIFTED |
+			 CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_VAR_INFO(CCU_SYS_APB_CLK, "sys_apb_clk",
+			 "pcie_clk", CCU_SYS_APB_BASE, 5,
+			 CLK_IS_CRITICAL, CCU_DIV_RESET_DOMAIN),
+	CCU_DIV_GATE_INFO(CCU_SYS_GMAC0_TX_CLK, "sys_gmac0_tx_clk",
+			  "eth_clk", CCU_SYS_GMAC0_BASE, 5),
+	CCU_DIV_FIXED_INFO(CCU_SYS_GMAC0_PTP_CLK, "sys_gmac0_ptp_clk",
+			   "eth_clk", 10),
+	CCU_DIV_GATE_INFO(CCU_SYS_GMAC1_TX_CLK, "sys_gmac1_tx_clk",
+			  "eth_clk", CCU_SYS_GMAC1_BASE, 5),
+	CCU_DIV_FIXED_INFO(CCU_SYS_GMAC1_PTP_CLK, "sys_gmac1_ptp_clk",
+			   "eth_clk", 10),
+	CCU_DIV_GATE_INFO(CCU_SYS_XGMAC_REF_CLK, "sys_xgmac_ref_clk",
+			  "eth_clk", CCU_SYS_XGMAC_BASE, 8),
+	CCU_DIV_FIXED_INFO(CCU_SYS_XGMAC_PTP_CLK, "sys_xgmac_ptp_clk",
+			   "eth_clk", 10),
+	CCU_DIV_GATE_INFO(CCU_SYS_USB_CLK, "sys_usb_clk",
+			  "eth_clk", CCU_SYS_USB_BASE, 10),
+	CCU_DIV_VAR_INFO(CCU_SYS_PVT_CLK, "sys_pvt_clk",
+			 "ref_clk", CCU_SYS_PVT_BASE, 5,
+			 CLK_SET_RATE_GATE, 0),
+	CCU_DIV_VAR_INFO(CCU_SYS_HWA_CLK, "sys_hwa_clk",
+			 "sata_clk", CCU_SYS_HWA_BASE, 4,
+			 CLK_SET_RATE_GATE, 0),
+	CCU_DIV_VAR_INFO(CCU_SYS_UART_CLK, "sys_uart_clk",
+			 "eth_clk", CCU_SYS_UART_BASE, 17,
+			 CLK_SET_RATE_GATE, 0),
+	CCU_DIV_FIXED_INFO(CCU_SYS_I2C1_CLK, "sys_i2c1_clk",
+			   "eth_clk", 10),
+	CCU_DIV_FIXED_INFO(CCU_SYS_I2C2_CLK, "sys_i2c2_clk",
+			   "eth_clk", 10),
+	CCU_DIV_FIXED_INFO(CCU_SYS_GPIO_CLK, "sys_gpio_clk",
+			   "ref_clk", 25),
+	CCU_DIV_VAR_INFO(CCU_SYS_TIMER0_CLK, "sys_timer0_clk",
+			 "ref_clk", CCU_SYS_TIMER0_BASE, 17,
+			 CLK_SET_RATE_GATE, 0),
+	CCU_DIV_VAR_INFO(CCU_SYS_TIMER1_CLK, "sys_timer1_clk",
+			 "ref_clk", CCU_SYS_TIMER1_BASE, 17,
+			 CLK_SET_RATE_GATE, 0),
+	CCU_DIV_VAR_INFO(CCU_SYS_TIMER2_CLK, "sys_timer2_clk",
+			 "ref_clk", CCU_SYS_TIMER2_BASE, 17,
+			 CLK_SET_RATE_GATE, 0),
+	CCU_DIV_VAR_INFO(CCU_SYS_WDT_CLK, "sys_wdt_clk",
+			 "eth_clk", CCU_SYS_WDT_BASE, 17,
+			 CLK_SET_RATE_GATE, CCU_DIV_SKIP_ONE_TO_THREE)
+};
+
+static const struct ccu_div_rst_map sys_rst_map[] = {
+	CCU_DIV_RST_MAP(CCU_SYS_SATA_REF_RST, CCU_SYS_SATA_REF_CLK),
+	CCU_DIV_RST_MAP(CCU_SYS_APB_RST, CCU_SYS_APB_CLK),
+};
+
+static struct ccu_div *ccu_div_find_desc(struct ccu_div_data *data,
+					 unsigned int clk_id)
+{
+	struct ccu_div *div;
+	int idx;
+
+	for (idx = 0; idx < data->divs_num; ++idx) {
+		div = data->divs[idx];
+		if (div && div->id == clk_id)
+			return div;
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static int ccu_div_reset(struct reset_controller_dev *rcdev,
+			 unsigned long rst_id)
+{
+	struct ccu_div_data *data = to_ccu_div_data(rcdev);
+	const struct ccu_div_rst_map *map;
+	struct ccu_div *div;
+	int idx, ret;
+
+	for (idx = 0, map = data->rst_map; idx < data->rst_num; ++idx, ++map) {
+		if (map->rst_id == rst_id)
+			break;
+	}
+	if (idx == data->rst_num) {
+		pr_err("Invalid reset ID %lu specified\n", rst_id);
+		return -EINVAL;
+	}
+
+	div = ccu_div_find_desc(data, map->clk_id);
+	if (IS_ERR(div)) {
+		pr_err("Invalid clock ID %d in mapping\n", map->clk_id);
+		return PTR_ERR(div);
+	}
+
+	ret = ccu_div_reset_domain(div);
+	if (ret) {
+		pr_err("Reset isn't supported by divider %s\n",
+			clk_hw_get_name(ccu_div_get_clk_hw(div)));
+	}
+
+	return ret;
+}
+
+static const struct reset_control_ops ccu_div_rst_ops = {
+	.reset = ccu_div_reset,
+};
+
+static struct ccu_div_data *ccu_div_create_data(struct device_node *np)
+{
+	struct ccu_div_data *data;
+	int ret;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
+
+	data->np = np;
+	if (of_device_is_compatible(np, "baikal,bt1-ccu-axi")) {
+		data->divs_num = ARRAY_SIZE(axi_info);
+		data->divs_info = axi_info;
+		data->rst_num = ARRAY_SIZE(axi_rst_map);
+		data->rst_map = axi_rst_map;
+	} else if (of_device_is_compatible(np, "baikal,bt1-ccu-sys")) {
+		data->divs_num = ARRAY_SIZE(sys_info);
+		data->divs_info = sys_info;
+		data->rst_num = ARRAY_SIZE(sys_rst_map);
+		data->rst_map = sys_rst_map;
+	} else {
+		pr_err("Incompatible DT node '%s' specified\n",
+			of_node_full_name(np));
+		ret = -EINVAL;
+		goto err_kfree_data;
+	}
+
+	data->divs = kcalloc(data->divs_num, sizeof(*data->divs), GFP_KERNEL);
+	if (!data->divs) {
+		ret = -ENOMEM;
+		goto err_kfree_data;
+	}
+
+	return data;
+
+err_kfree_data:
+	kfree(data);
+
+	return ERR_PTR(ret);
+}
+
+static void ccu_div_free_data(struct ccu_div_data *data)
+{
+	kfree(data->divs);
+
+	kfree(data);
+}
+
+static int ccu_div_find_sys_regs(struct ccu_div_data *data)
+{
+	data->sys_regs = syscon_node_to_regmap(data->np->parent);
+	if (IS_ERR(data->sys_regs)) {
+		pr_err("Failed to find syscon regs for '%s'\n",
+			of_node_full_name(data->np));
+		return PTR_ERR(data->sys_regs);
+	}
+
+	return 0;
+}
+
+static struct clk_hw *ccu_div_of_clk_hw_get(struct of_phandle_args *clkspec,
+					    void *priv)
+{
+	struct ccu_div_data *data = priv;
+	struct ccu_div *div;
+	unsigned int clk_id;
+
+	clk_id = clkspec->args[0];
+	div = ccu_div_find_desc(data, clk_id);
+	if (IS_ERR(div)) {
+		pr_info("Invalid clock ID %d specified\n", clk_id);
+		return ERR_CAST(div);
+	}
+
+	return ccu_div_get_clk_hw(div);
+}
+
+static int ccu_div_clk_register(struct ccu_div_data *data)
+{
+	int idx, ret;
+
+	for (idx = 0; idx < data->divs_num; ++idx) {
+		const struct ccu_div_info *info = &data->divs_info[idx];
+		struct ccu_div_init_data init = {0};
+
+		init.id = info->id;
+		init.name = info->name;
+		init.parent_name = info->parent_name;
+		init.np = data->np;
+		init.type = info->type;
+		init.flags = info->flags;
+		init.features = info->features;
+
+		if (init.type == CCU_DIV_VAR) {
+			init.base = info->base;
+			init.sys_regs = data->sys_regs;
+			init.width = info->width;
+		} else if (init.type == CCU_DIV_GATE) {
+			init.base = info->base;
+			init.sys_regs = data->sys_regs;
+			init.divider = info->divider;
+		} else {
+			init.divider = info->divider;
+		}
+
+		data->divs[idx] = ccu_div_hw_register(&init);
+		if (IS_ERR(data->divs[idx])) {
+			ret = PTR_ERR(data->divs[idx]);
+			pr_err("Couldn't register divider '%s' hw\n",
+				init.name);
+			goto err_hw_unregister;
+		}
+	}
+
+	ret = of_clk_add_hw_provider(data->np, ccu_div_of_clk_hw_get, data);
+	if (ret) {
+		pr_err("Couldn't register dividers '%s' clock provider\n",
+			of_node_full_name(data->np));
+		goto err_hw_unregister;
+	}
+
+	return 0;
+
+err_hw_unregister:
+	for (--idx; idx >= 0; --idx)
+		ccu_div_hw_unregister(data->divs[idx]);
+
+	return ret;
+}
+
+static void ccu_div_clk_unregister(struct ccu_div_data *data)
+{
+	int idx;
+
+	of_clk_del_provider(data->np);
+
+	for (idx = 0; idx < data->divs_num; ++idx)
+		ccu_div_hw_unregister(data->divs[idx]);
+}
+
+static int ccu_div_rst_register(struct ccu_div_data *data)
+{
+	int ret;
+
+	data->rcdev.ops = &ccu_div_rst_ops;
+	data->rcdev.of_node = data->np;
+	data->rcdev.nr_resets = data->rst_num;
+
+	ret = reset_controller_register(&data->rcdev);
+	if (ret)
+		pr_err("Couldn't register divider '%s' reset controller\n",
+			of_node_full_name(data->np));
+
+	return ret;
+}
+
+static void ccu_div_init(struct device_node *np)
+{
+	struct ccu_div_data *data;
+	int ret;
+
+	data = ccu_div_create_data(np);
+	if (IS_ERR(data))
+		return;
+
+	ret = ccu_div_find_sys_regs(data);
+	if (ret)
+		goto err_free_data;
+
+	ret = ccu_div_clk_register(data);
+	if (ret)
+		goto err_free_data;
+
+	ret = ccu_div_rst_register(data);
+	if (ret)
+		goto err_clk_unregister;
+
+	return;
+
+err_clk_unregister:
+	ccu_div_clk_unregister(data);
+
+err_free_data:
+	ccu_div_free_data(data);
+}
+
+CLK_OF_DECLARE(ccu_axi, "baikal,bt1-ccu-axi", ccu_div_init);
+CLK_OF_DECLARE(ccu_sys, "baikal,bt1-ccu-sys", ccu_div_init);
diff --git a/drivers/clk/baikal-t1/clk-ccu-pll.c b/drivers/clk/baikal-t1/clk-ccu-pll.c
new file mode 100644
index 000000000000..1eec8c0b8f50
--- /dev/null
+++ b/drivers/clk/baikal-t1/clk-ccu-pll.c
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *   Dmitry Dunaev <dmitry.dunaev@baikalelectronics.ru>
+ *
+ * Baikal-T1 CCU PLL clocks driver
+ */
+
+#define pr_fmt(fmt) "bt1-ccu-pll: " fmt
+
+#include <linux/kernel.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/ioport.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/bt1-ccu.h>
+
+#include "ccu-pll.h"
+
+#define CCU_CPU_PLL_BASE		0x000
+#define CCU_SATA_PLL_BASE		0x008
+#define CCU_DDR_PLL_BASE		0x010
+#define CCU_PCIE_PLL_BASE		0x018
+#define CCU_ETH_PLL_BASE		0x020
+
+#define CCU_PLL_INFO(_id, _name, _pname, _base, _flags)	\
+	{						\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _pname,			\
+		.base = _base,				\
+		.flags = _flags				\
+	}
+
+#define CCU_PLL_NUM			ARRAY_SIZE(pll_info)
+
+struct ccu_pll_info {
+	unsigned int id;
+	const char *name;
+	const char *parent_name;
+	unsigned int base;
+	unsigned long flags;
+};
+
+/*
+ * Mark as critical all PLLs except Ethernet one. CPU and DDR PLLs are sources
+ * of CPU cores and DDR controller reference clocks, due to which they
+ * obviously shouldn't be ever gated. SATA and PCIe PLLs are the parents of
+ * APB-bus and DDR controller AXI-bus clocks. If they are gated the system will
+ * be unusable.
+ */
+static const struct ccu_pll_info pll_info[] = {
+	CCU_PLL_INFO(CCU_CPU_PLL, "cpu_pll", "ref_clk", CCU_CPU_PLL_BASE,
+		     CLK_IS_CRITICAL),
+	CCU_PLL_INFO(CCU_SATA_PLL, "sata_pll", "ref_clk", CCU_SATA_PLL_BASE,
+		     CLK_IS_CRITICAL | CLK_SET_RATE_GATE),
+	CCU_PLL_INFO(CCU_DDR_PLL, "ddr_pll", "ref_clk", CCU_DDR_PLL_BASE,
+		     CLK_IS_CRITICAL | CLK_SET_RATE_GATE),
+	CCU_PLL_INFO(CCU_PCIE_PLL, "pcie_pll", "ref_clk", CCU_PCIE_PLL_BASE,
+		     CLK_IS_CRITICAL),
+	CCU_PLL_INFO(CCU_ETH_PLL, "eth_pll", "ref_clk", CCU_ETH_PLL_BASE,
+		     CLK_SET_RATE_GATE)
+};
+
+struct ccu_pll_data {
+	struct device_node *np;
+	struct regmap *sys_regs;
+	struct ccu_pll *plls[CCU_PLL_NUM];
+};
+
+static struct ccu_pll *ccu_pll_find_desc(struct ccu_pll_data *data,
+					 unsigned int clk_id)
+{
+	struct ccu_pll *pll;
+	int idx;
+
+	for (idx = 0; idx < CCU_PLL_NUM; ++idx) {
+		pll = data->plls[idx];
+		if (pll && pll->id == clk_id)
+			return pll;
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static struct ccu_pll_data *ccu_pll_create_data(struct device_node *np)
+{
+	struct ccu_pll_data *data;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
+
+	data->np = np;
+
+	return data;
+}
+
+static void ccu_pll_free_data(struct ccu_pll_data *data)
+{
+	kfree(data);
+}
+
+static int ccu_pll_find_sys_regs(struct ccu_pll_data *data)
+{
+	data->sys_regs = syscon_node_to_regmap(data->np->parent);
+	if (IS_ERR(data->sys_regs)) {
+		pr_err("Failed to find syscon regs for '%s'\n",
+			of_node_full_name(data->np));
+		return PTR_ERR(data->sys_regs);
+	}
+
+	return 0;
+}
+
+static struct clk_hw *ccu_pll_of_clk_hw_get(struct of_phandle_args *clkspec,
+					    void *priv)
+{
+	struct ccu_pll_data *data = priv;
+	struct ccu_pll *pll;
+	unsigned int clk_id;
+
+	clk_id = clkspec->args[0];
+	pll = ccu_pll_find_desc(data, clk_id);
+	if (IS_ERR(pll)) {
+		pr_info("Invalid PLL clock ID %d specified\n", clk_id);
+		return ERR_CAST(pll);
+	}
+
+	return ccu_pll_get_clk_hw(pll);
+}
+
+static int ccu_pll_clk_register(struct ccu_pll_data *data)
+{
+	int idx, ret;
+
+	for (idx = 0; idx < CCU_PLL_NUM; ++idx) {
+		const struct ccu_pll_info *info = &pll_info[idx];
+		struct ccu_pll_init_data init = {0};
+
+		init.id = info->id;
+		init.name = info->name;
+		init.parent_name = info->parent_name;
+		init.base = info->base;
+		init.sys_regs = data->sys_regs;
+		init.np = data->np;
+		init.flags = info->flags;
+
+		data->plls[idx] = ccu_pll_hw_register(&init);
+		if (IS_ERR(data->plls[idx])) {
+			ret = PTR_ERR(data->plls[idx]);
+			pr_err("Couldn't register PLL hw '%s'\n",
+				init.name);
+			goto err_hw_unregister;
+		}
+	}
+
+	ret = of_clk_add_hw_provider(data->np, ccu_pll_of_clk_hw_get, data);
+	if (ret) {
+		pr_err("Couldn't register PLL provider of '%s'\n",
+			of_node_full_name(data->np));
+		goto err_hw_unregister;
+	}
+
+	return 0;
+
+err_hw_unregister:
+	for (--idx; idx >= 0; --idx)
+		ccu_pll_hw_unregister(data->plls[idx]);
+
+	return ret;
+}
+
+static __init void ccu_pll_init(struct device_node *np)
+{
+	struct ccu_pll_data *data;
+	int ret;
+
+	data = ccu_pll_create_data(np);
+	if (IS_ERR(data))
+		return;
+
+	ret = ccu_pll_find_sys_regs(data);
+	if (ret)
+		goto err_free_data;
+
+	ret = ccu_pll_clk_register(data);
+	if (ret)
+		goto err_free_data;
+
+	return;
+
+err_free_data:
+	ccu_pll_free_data(data);
+}
+CLK_OF_DECLARE(ccu_pll, "baikal,bt1-ccu-pll", ccu_pll_init);
diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c
index ded13ccf768e..6bb7efa12037 100644
--- a/drivers/clk/bcm/clk-bcm2835.c
+++ b/drivers/clk/bcm/clk-bcm2835.c
@@ -396,8 +396,8 @@ out:
 }
 
 static void bcm2835_debugfs_regset(struct bcm2835_cprman *cprman, u32 base,
-				  struct debugfs_reg32 *regs, size_t nregs,
-				  struct dentry *dentry)
+				   const struct debugfs_reg32 *regs,
+				   size_t nregs, struct dentry *dentry)
 {
 	struct debugfs_regset32 *regset;
 
@@ -1240,7 +1240,7 @@ static u8 bcm2835_clock_get_parent(struct clk_hw *hw)
 	return (src & CM_SRC_MASK) >> CM_SRC_SHIFT;
 }
 
-static struct debugfs_reg32 bcm2835_debugfs_clock_reg32[] = {
+static const struct debugfs_reg32 bcm2835_debugfs_clock_reg32[] = {
 	{
 		.name = "ctl",
 		.offset = 0,
@@ -1296,8 +1296,9 @@ static const struct clk_ops bcm2835_vpu_clock_clk_ops = {
 };
 
 static struct clk_hw *bcm2835_register_pll(struct bcm2835_cprman *cprman,
-					   const struct bcm2835_pll_data *data)
+					   const void *data)
 {
+	const struct bcm2835_pll_data *pll_data = data;
 	struct bcm2835_pll *pll;
 	struct clk_init_data init;
 	int ret;
@@ -1307,7 +1308,7 @@ static struct clk_hw *bcm2835_register_pll(struct bcm2835_cprman *cprman,
 	/* All of the PLLs derive from the external oscillator. */
 	init.parent_names = &cprman->real_parent_names[0];
 	init.num_parents = 1;
-	init.name = data->name;
+	init.name = pll_data->name;
 	init.ops = &bcm2835_pll_clk_ops;
 	init.flags = CLK_IGNORE_UNUSED;
 
@@ -1316,7 +1317,7 @@ static struct clk_hw *bcm2835_register_pll(struct bcm2835_cprman *cprman,
 		return NULL;
 
 	pll->cprman = cprman;
-	pll->data = data;
+	pll->data = pll_data;
 	pll->hw.init = &init;
 
 	ret = devm_clk_hw_register(cprman->dev, &pll->hw);
@@ -1327,35 +1328,36 @@ static struct clk_hw *bcm2835_register_pll(struct bcm2835_cprman *cprman,
 
 static struct clk_hw *
 bcm2835_register_pll_divider(struct bcm2835_cprman *cprman,
-			     const struct bcm2835_pll_divider_data *data)
+			     const void *data)
 {
+	const struct bcm2835_pll_divider_data *divider_data = data;
 	struct bcm2835_pll_divider *divider;
 	struct clk_init_data init;
 	const char *divider_name;
 	int ret;
 
-	if (data->fixed_divider != 1) {
+	if (divider_data->fixed_divider != 1) {
 		divider_name = devm_kasprintf(cprman->dev, GFP_KERNEL,
-					      "%s_prediv", data->name);
+					      "%s_prediv", divider_data->name);
 		if (!divider_name)
 			return NULL;
 	} else {
-		divider_name = data->name;
+		divider_name = divider_data->name;
 	}
 
 	memset(&init, 0, sizeof(init));
 
-	init.parent_names = &data->source_pll;
+	init.parent_names = &divider_data->source_pll;
 	init.num_parents = 1;
 	init.name = divider_name;
 	init.ops = &bcm2835_pll_divider_clk_ops;
-	init.flags = data->flags | CLK_IGNORE_UNUSED;
+	init.flags = divider_data->flags | CLK_IGNORE_UNUSED;
 
 	divider = devm_kzalloc(cprman->dev, sizeof(*divider), GFP_KERNEL);
 	if (!divider)
 		return NULL;
 
-	divider->div.reg = cprman->regs + data->a2w_reg;
+	divider->div.reg = cprman->regs + divider_data->a2w_reg;
 	divider->div.shift = A2W_PLL_DIV_SHIFT;
 	divider->div.width = A2W_PLL_DIV_BITS;
 	divider->div.flags = CLK_DIVIDER_MAX_AT_ZERO;
@@ -1364,7 +1366,7 @@ bcm2835_register_pll_divider(struct bcm2835_cprman *cprman,
 	divider->div.table = NULL;
 
 	divider->cprman = cprman;
-	divider->data = data;
+	divider->data = divider_data;
 
 	ret = devm_clk_hw_register(cprman->dev, &divider->div.hw);
 	if (ret)
@@ -1374,20 +1376,22 @@ bcm2835_register_pll_divider(struct bcm2835_cprman *cprman,
 	 * PLLH's channels have a fixed divide by 10 afterwards, which
 	 * is what our consumers are actually using.
 	 */
-	if (data->fixed_divider != 1) {
-		return clk_hw_register_fixed_factor(cprman->dev, data->name,
+	if (divider_data->fixed_divider != 1) {
+		return clk_hw_register_fixed_factor(cprman->dev,
+						    divider_data->name,
 						    divider_name,
 						    CLK_SET_RATE_PARENT,
 						    1,
-						    data->fixed_divider);
+						    divider_data->fixed_divider);
 	}
 
 	return &divider->div.hw;
 }
 
 static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
-					  const struct bcm2835_clock_data *data)
+					     const void *data)
 {
+	const struct bcm2835_clock_data *clock_data = data;
 	struct bcm2835_clock *clock;
 	struct clk_init_data init;
 	const char *parents[1 << CM_SRC_BITS];
@@ -1398,8 +1402,8 @@ static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 	 * Replace our strings referencing parent clocks with the
 	 * actual clock-output-name of the parent.
 	 */
-	for (i = 0; i < data->num_mux_parents; i++) {
-		parents[i] = data->parents[i];
+	for (i = 0; i < clock_data->num_mux_parents; i++) {
+		parents[i] = clock_data->parents[i];
 
 		ret = match_string(cprman_parent_names,
 				   ARRAY_SIZE(cprman_parent_names),
@@ -1410,18 +1414,18 @@ static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 
 	memset(&init, 0, sizeof(init));
 	init.parent_names = parents;
-	init.num_parents = data->num_mux_parents;
-	init.name = data->name;
-	init.flags = data->flags | CLK_IGNORE_UNUSED;
+	init.num_parents = clock_data->num_mux_parents;
+	init.name = clock_data->name;
+	init.flags = clock_data->flags | CLK_IGNORE_UNUSED;
 
 	/*
 	 * Pass the CLK_SET_RATE_PARENT flag if we are allowed to propagate
 	 * rate changes on at least of the parents.
 	 */
-	if (data->set_rate_parent)
+	if (clock_data->set_rate_parent)
 		init.flags |= CLK_SET_RATE_PARENT;
 
-	if (data->is_vpu_clock) {
+	if (clock_data->is_vpu_clock) {
 		init.ops = &bcm2835_vpu_clock_clk_ops;
 	} else {
 		init.ops = &bcm2835_clock_clk_ops;
@@ -1430,7 +1434,7 @@ static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 		/* If the clock wasn't actually enabled at boot, it's not
 		 * critical.
 		 */
-		if (!(cprman_read(cprman, data->ctl_reg) & CM_ENABLE))
+		if (!(cprman_read(cprman, clock_data->ctl_reg) & CM_ENABLE))
 			init.flags &= ~CLK_IS_CRITICAL;
 	}
 
@@ -1439,7 +1443,7 @@ static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 		return NULL;
 
 	clock->cprman = cprman;
-	clock->data = data;
+	clock->data = clock_data;
 	clock->hw.init = &init;
 
 	ret = devm_clk_hw_register(cprman->dev, &clock->hw);
@@ -1448,25 +1452,27 @@ static struct clk_hw *bcm2835_register_clock(struct bcm2835_cprman *cprman,
 	return &clock->hw;
 }
 
-static struct clk *bcm2835_register_gate(struct bcm2835_cprman *cprman,
-					 const struct bcm2835_gate_data *data)
+static struct clk_hw *bcm2835_register_gate(struct bcm2835_cprman *cprman,
+					    const void *data)
 {
-	return clk_register_gate(cprman->dev, data->name, data->parent,
-				 CLK_IGNORE_UNUSED | CLK_SET_RATE_GATE,
-				 cprman->regs + data->ctl_reg,
-				 CM_GATE_BIT, 0, &cprman->regs_lock);
+	const struct bcm2835_gate_data *gate_data = data;
+
+	return clk_hw_register_gate(cprman->dev, gate_data->name,
+				    gate_data->parent,
+				    CLK_IGNORE_UNUSED | CLK_SET_RATE_GATE,
+				    cprman->regs + gate_data->ctl_reg,
+				    CM_GATE_BIT, 0, &cprman->regs_lock);
 }
 
-typedef struct clk_hw *(*bcm2835_clk_register)(struct bcm2835_cprman *cprman,
-					       const void *data);
 struct bcm2835_clk_desc {
-	bcm2835_clk_register clk_register;
+	struct clk_hw *(*clk_register)(struct bcm2835_cprman *cprman,
+				       const void *data);
 	unsigned int supported;
 	const void *data;
 };
 
 /* assignment helper macros for different clock types */
-#define _REGISTER(f, s, ...) { .clk_register = (bcm2835_clk_register)f, \
+#define _REGISTER(f, s, ...) { .clk_register = f, \
 			       .supported = s,				\
 			       .data = __VA_ARGS__ }
 #define REGISTER_PLL(s, ...)	_REGISTER(&bcm2835_register_pll,	\
diff --git a/drivers/clk/clk-ast2600.c b/drivers/clk/clk-ast2600.c
index 392d01705b97..99afc949925f 100644
--- a/drivers/clk/clk-ast2600.c
+++ b/drivers/clk/clk-ast2600.c
@@ -642,14 +642,22 @@ static const u32 ast2600_a0_axi_ahb_div_table[] = {
 	2, 2, 3, 5,
 };
 
-static const u32 ast2600_a1_axi_ahb_div_table[] = {
-	4, 6, 2, 4,
+static const u32 ast2600_a1_axi_ahb_div0_tbl[] = {
+	3, 2, 3, 4,
+};
+
+static const u32 ast2600_a1_axi_ahb_div1_tbl[] = {
+	3, 4, 6, 8,
+};
+
+static const u32 ast2600_a1_axi_ahb200_tbl[] = {
+	3, 4, 3, 4, 2, 2, 2, 2,
 };
 
 static void __init aspeed_g6_cc(struct regmap *map)
 {
 	struct clk_hw *hw;
-	u32 val, div, chip_id, axi_div, ahb_div;
+	u32 val, div, divbits, chip_id, axi_div, ahb_div;
 
 	clk_hw_register_fixed_rate(NULL, "clkin", NULL, 0, 25000000);
 
@@ -679,11 +687,22 @@ static void __init aspeed_g6_cc(struct regmap *map)
 	else
 		axi_div = 2;
 
+	divbits = (val >> 11) & 0x3;
 	regmap_read(map, ASPEED_G6_SILICON_REV, &chip_id);
-	if (chip_id & BIT(16))
-		ahb_div = ast2600_a1_axi_ahb_div_table[(val >> 11) & 0x3];
-	else
+	if (chip_id & BIT(16)) {
+		if (!divbits) {
+			ahb_div = ast2600_a1_axi_ahb200_tbl[(val >> 8) & 0x3];
+			if (val & BIT(16))
+				ahb_div *= 2;
+		} else {
+			if (val & BIT(16))
+				ahb_div = ast2600_a1_axi_ahb_div1_tbl[divbits];
+			else
+				ahb_div = ast2600_a1_axi_ahb_div0_tbl[divbits];
+		}
+	} else {
 		ahb_div = ast2600_a0_axi_ahb_div_table[(val >> 11) & 0x3];
+	}
 
 	hw = clk_hw_register_fixed_factor(NULL, "ahb", "hpll", 0, 1, axi_div * ahb_div);
 	aspeed_g6_clk_data->hws[ASPEED_CLK_AHB] = hw;
diff --git a/drivers/clk/clk-hsdk-pll.c b/drivers/clk/clk-hsdk-pll.c
index 97d1e8c35b71..b4f8852201cb 100644
--- a/drivers/clk/clk-hsdk-pll.c
+++ b/drivers/clk/clk-hsdk-pll.c
@@ -53,35 +53,38 @@ struct hsdk_pll_cfg {
 	u32 fbdiv;
 	u32 odiv;
 	u32 band;
+	u32 bypass;
 };
 
 static const struct hsdk_pll_cfg asdt_pll_cfg[] = {
-	{ 100000000,  0, 11, 3, 0 },
-	{ 133000000,  0, 15, 3, 0 },
-	{ 200000000,  1, 47, 3, 0 },
-	{ 233000000,  1, 27, 2, 0 },
-	{ 300000000,  1, 35, 2, 0 },
-	{ 333000000,  1, 39, 2, 0 },
-	{ 400000000,  1, 47, 2, 0 },
-	{ 500000000,  0, 14, 1, 0 },
-	{ 600000000,  0, 17, 1, 0 },
-	{ 700000000,  0, 20, 1, 0 },
-	{ 800000000,  0, 23, 1, 0 },
-	{ 900000000,  1, 26, 0, 0 },
-	{ 1000000000, 1, 29, 0, 0 },
-	{ 1100000000, 1, 32, 0, 0 },
-	{ 1200000000, 1, 35, 0, 0 },
-	{ 1300000000, 1, 38, 0, 0 },
-	{ 1400000000, 1, 41, 0, 0 },
-	{ 1500000000, 1, 44, 0, 0 },
-	{ 1600000000, 1, 47, 0, 0 },
+	{ 100000000,  0, 11, 3, 0, 0 },
+	{ 133000000,  0, 15, 3, 0, 0 },
+	{ 200000000,  1, 47, 3, 0, 0 },
+	{ 233000000,  1, 27, 2, 0, 0 },
+	{ 300000000,  1, 35, 2, 0, 0 },
+	{ 333000000,  1, 39, 2, 0, 0 },
+	{ 400000000,  1, 47, 2, 0, 0 },
+	{ 500000000,  0, 14, 1, 0, 0 },
+	{ 600000000,  0, 17, 1, 0, 0 },
+	{ 700000000,  0, 20, 1, 0, 0 },
+	{ 800000000,  0, 23, 1, 0, 0 },
+	{ 900000000,  1, 26, 0, 0, 0 },
+	{ 1000000000, 1, 29, 0, 0, 0 },
+	{ 1100000000, 1, 32, 0, 0, 0 },
+	{ 1200000000, 1, 35, 0, 0, 0 },
+	{ 1300000000, 1, 38, 0, 0, 0 },
+	{ 1400000000, 1, 41, 0, 0, 0 },
+	{ 1500000000, 1, 44, 0, 0, 0 },
+	{ 1600000000, 1, 47, 0, 0, 0 },
 	{}
 };
 
 static const struct hsdk_pll_cfg hdmi_pll_cfg[] = {
-	{ 297000000,  0, 21, 2, 0 },
-	{ 540000000,  0, 19, 1, 0 },
-	{ 594000000,  0, 21, 1, 0 },
+	{ 27000000,   0, 0,  0, 0, 1 },
+	{ 148500000,  0, 21, 3, 0, 0 },
+	{ 297000000,  0, 21, 2, 0, 0 },
+	{ 540000000,  0, 19, 1, 0, 0 },
+	{ 594000000,  0, 21, 1, 0, 0 },
 	{}
 };
 
@@ -134,11 +137,16 @@ static inline void hsdk_pll_set_cfg(struct hsdk_pll_clk *clk,
 {
 	u32 val = 0;
 
-	/* Powerdown and Bypass bits should be cleared */
-	val |= cfg->idiv << CGU_PLL_CTRL_IDIV_SHIFT;
-	val |= cfg->fbdiv << CGU_PLL_CTRL_FBDIV_SHIFT;
-	val |= cfg->odiv << CGU_PLL_CTRL_ODIV_SHIFT;
-	val |= cfg->band << CGU_PLL_CTRL_BAND_SHIFT;
+	if (cfg->bypass) {
+		val = hsdk_pll_read(clk, CGU_PLL_CTRL);
+		val |= CGU_PLL_CTRL_BYPASS;
+	} else {
+		/* Powerdown and Bypass bits should be cleared */
+		val |= cfg->idiv << CGU_PLL_CTRL_IDIV_SHIFT;
+		val |= cfg->fbdiv << CGU_PLL_CTRL_FBDIV_SHIFT;
+		val |= cfg->odiv << CGU_PLL_CTRL_ODIV_SHIFT;
+		val |= cfg->band << CGU_PLL_CTRL_BAND_SHIFT;
+	}
 
 	dev_dbg(clk->dev, "write configuration: %#x\n", val);
 
@@ -172,14 +180,14 @@ static unsigned long hsdk_pll_recalc_rate(struct clk_hw *hw,
 
 	dev_dbg(clk->dev, "current configuration: %#x\n", val);
 
-	/* Check if PLL is disabled */
-	if (val & CGU_PLL_CTRL_PD)
-		return 0;
-
 	/* Check if PLL is bypassed */
 	if (val & CGU_PLL_CTRL_BYPASS)
 		return parent_rate;
 
+	/* Check if PLL is disabled */
+	if (val & CGU_PLL_CTRL_PD)
+		return 0;
+
 	/* input divider = reg.idiv + 1 */
 	idiv = 1 + ((val & CGU_PLL_CTRL_IDIV_MASK) >> CGU_PLL_CTRL_IDIV_SHIFT);
 	/* fb divider = 2*(reg.fbdiv + 1) */
diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c
index 3c228b018116..3d7acab9d280 100644
--- a/drivers/clk/clk-si5341.c
+++ b/drivers/clk/clk-si5341.c
@@ -1,8 +1,14 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Driver for Silicon Labs Si5341/Si5340 Clock generator
+ * Driver for Silicon Labs Si5340, Si5341, Si5342, Si5344 and Si5345
  * Copyright (C) 2019 Topic Embedded Products
  * Author: Mike Looijmans <mike.looijmans@topic.nl>
+ *
+ * The Si5341 has 10 outputs and 5 synthesizers.
+ * The Si5340 is a smaller version of the Si5341 with only 4 outputs.
+ * The Si5345 is similar to the Si5341, with the addition of fractional input
+ * dividers and automatic input selection.
+ * The Si5342 and Si5344 are smaller versions of the Si5345.
  */
 
 #include <linux/clk.h>
@@ -18,11 +24,17 @@
 
 #define SI5341_NUM_INPUTS 4
 
-#define SI5341_MAX_NUM_OUTPUTS 10
 #define SI5340_MAX_NUM_OUTPUTS 4
+#define SI5341_MAX_NUM_OUTPUTS 10
+#define SI5342_MAX_NUM_OUTPUTS 2
+#define SI5344_MAX_NUM_OUTPUTS 4
+#define SI5345_MAX_NUM_OUTPUTS 10
 
-#define SI5341_NUM_SYNTH 5
 #define SI5340_NUM_SYNTH 4
+#define SI5341_NUM_SYNTH 5
+#define SI5342_NUM_SYNTH 2
+#define SI5344_NUM_SYNTH 4
+#define SI5345_NUM_SYNTH 5
 
 /* Range of the synthesizer fractional divider */
 #define SI5341_SYNTH_N_MIN	10
@@ -65,6 +77,7 @@ struct clk_si5341 {
 	u64 freq_vco; /* 13500–14256 MHz */
 	u8 num_outputs;
 	u8 num_synth;
+	u16 chip_id;
 };
 #define to_clk_si5341(_hw)	container_of(_hw, struct clk_si5341, hw)
 
@@ -142,6 +155,7 @@ static const char * const si5341_input_clock_names[] = {
 };
 
 /* Output configuration registers 0..9 are not quite logically organized */
+/* Also for si5345 */
 static const u16 si5341_reg_output_offset[] = {
 	0x0108,
 	0x010D,
@@ -155,6 +169,7 @@ static const u16 si5341_reg_output_offset[] = {
 	0x013A,
 };
 
+/* for si5340, si5342 and si5344 */
 static const u16 si5340_reg_output_offset[] = {
 	0x0112,
 	0x0117,
@@ -974,12 +989,32 @@ static int si5341_probe_chip_id(struct clk_si5341 *data)
 		data->reg_output_offset = si5341_reg_output_offset;
 		data->reg_rdiv_offset = si5341_reg_rdiv_offset;
 		break;
+	case 0x5342:
+		data->num_outputs = SI5342_MAX_NUM_OUTPUTS;
+		data->num_synth = SI5342_NUM_SYNTH;
+		data->reg_output_offset = si5340_reg_output_offset;
+		data->reg_rdiv_offset = si5340_reg_rdiv_offset;
+		break;
+	case 0x5344:
+		data->num_outputs = SI5344_MAX_NUM_OUTPUTS;
+		data->num_synth = SI5344_NUM_SYNTH;
+		data->reg_output_offset = si5340_reg_output_offset;
+		data->reg_rdiv_offset = si5340_reg_rdiv_offset;
+		break;
+	case 0x5345:
+		data->num_outputs = SI5345_MAX_NUM_OUTPUTS;
+		data->num_synth = SI5345_NUM_SYNTH;
+		data->reg_output_offset = si5341_reg_output_offset;
+		data->reg_rdiv_offset = si5341_reg_rdiv_offset;
+		break;
 	default:
 		dev_err(&data->i2c_client->dev, "Model '%x' not supported\n",
 			model);
 		return -EINVAL;
 	}
 
+	data->chip_id = model;
+
 	return 0;
 }
 
@@ -1054,6 +1089,11 @@ static const struct si5341_reg_default si5341_preamble[] = {
 	{ 0x0B4E, 0x1A },
 };
 
+static const struct si5341_reg_default si5345_preamble[] = {
+	{ 0x0B25, 0x00 },
+	{ 0x0540, 0x01 },
+};
+
 static int si5341_send_preamble(struct clk_si5341 *data)
 {
 	int res;
@@ -1068,8 +1108,14 @@ static int si5341_send_preamble(struct clk_si5341 *data)
 	res = regmap_write(data->regmap, 0xB24, revision < 2 ? 0xD8 : 0xC0);
 	if (res < 0)
 		return res;
-	res = si5341_write_multiple(data,
-		si5341_preamble, ARRAY_SIZE(si5341_preamble));
+
+	/* The si5342..si5345 require a different preamble */
+	if (data->chip_id > 0x5341)
+		res = si5341_write_multiple(data,
+			si5345_preamble, ARRAY_SIZE(si5345_preamble));
+	else
+		res = si5341_write_multiple(data,
+			si5341_preamble, ARRAY_SIZE(si5341_preamble));
 	if (res < 0)
 		return res;
 
@@ -1095,6 +1141,13 @@ static int si5341_finalize_defaults(struct clk_si5341 *data)
 	if (res < 0)
 		return res;
 
+	/* The si5342..si5345 have an additional post-amble */
+	if (data->chip_id > 0x5341) {
+		res = regmap_write(data->regmap, 0x540, 0x0);
+		if (res < 0)
+			return res;
+	}
+
 	/* Datasheet does not explain these nameless registers */
 	res = regmap_write(data->regmap, 0xB24, revision < 2 ? 0xDB : 0xC3);
 	if (res < 0)
@@ -1499,6 +1552,9 @@ static int si5341_probe(struct i2c_client *client,
 static const struct i2c_device_id si5341_id[] = {
 	{ "si5340", 0 },
 	{ "si5341", 1 },
+	{ "si5342", 2 },
+	{ "si5344", 4 },
+	{ "si5345", 5 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, si5341_id);
@@ -1506,6 +1562,9 @@ MODULE_DEVICE_TABLE(i2c, si5341_id);
 static const struct of_device_id clk_si5341_of_match[] = {
 	{ .compatible = "silabs,si5340" },
 	{ .compatible = "silabs,si5341" },
+	{ .compatible = "silabs,si5342" },
+	{ .compatible = "silabs,si5344" },
+	{ .compatible = "silabs,si5345" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, clk_si5341_of_match);
diff --git a/drivers/clk/clk-versaclock5.c b/drivers/clk/clk-versaclock5.c
index 24fef51fbcb5..fa96659f8023 100644
--- a/drivers/clk/clk-versaclock5.c
+++ b/drivers/clk/clk-versaclock5.c
@@ -124,6 +124,7 @@ enum vc5_model {
 	IDT_VC5_5P49V5933,
 	IDT_VC5_5P49V5935,
 	IDT_VC6_5P49V6901,
+	IDT_VC6_5P49V6965,
 };
 
 /* Structure to describe features of a particular VC5 model */
@@ -683,6 +684,7 @@ static int vc5_map_index_to_output(const enum vc5_model model,
 	case IDT_VC5_5P49V5925:
 	case IDT_VC5_5P49V5935:
 	case IDT_VC6_5P49V6901:
+	case IDT_VC6_5P49V6965:
 	default:
 		return n;
 	}
@@ -956,12 +958,20 @@ static const struct vc5_chip_info idt_5p49v6901_info = {
 	.flags = VC5_HAS_PFD_FREQ_DBL,
 };
 
+static const struct vc5_chip_info idt_5p49v6965_info = {
+	.model = IDT_VC6_5P49V6965,
+	.clk_fod_cnt = 4,
+	.clk_out_cnt = 5,
+	.flags = 0,
+};
+
 static const struct i2c_device_id vc5_id[] = {
 	{ "5p49v5923", .driver_data = IDT_VC5_5P49V5923 },
 	{ "5p49v5925", .driver_data = IDT_VC5_5P49V5925 },
 	{ "5p49v5933", .driver_data = IDT_VC5_5P49V5933 },
 	{ "5p49v5935", .driver_data = IDT_VC5_5P49V5935 },
 	{ "5p49v6901", .driver_data = IDT_VC6_5P49V6901 },
+	{ "5p49v6965", .driver_data = IDT_VC6_5P49V6965 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, vc5_id);
@@ -972,6 +982,7 @@ static const struct of_device_id clk_vc5_of_match[] = {
 	{ .compatible = "idt,5p49v5933", .data = &idt_5p49v5933_info },
 	{ .compatible = "idt,5p49v5935", .data = &idt_5p49v5935_info },
 	{ .compatible = "idt,5p49v6901", .data = &idt_5p49v6901_info },
+	{ .compatible = "idt,5p49v6965", .data = &idt_5p49v6965_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, clk_vc5_of_match);
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index 407f6919604c..3f588ed06ce3 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -3299,10 +3299,6 @@ static int __init clk_debug_init(void)
 late_initcall(clk_debug_init);
 #else
 static inline void clk_debug_register(struct clk_core *core) { }
-static inline void clk_debug_reparent(struct clk_core *core,
-				      struct clk_core *new_parent)
-{
-}
 static inline void clk_debug_unregister(struct clk_core *core)
 {
 }
diff --git a/drivers/clk/imgtec/Kconfig b/drivers/clk/imgtec/Kconfig
index 30f5265cca8d..c965fd0917dd 100644
--- a/drivers/clk/imgtec/Kconfig
+++ b/drivers/clk/imgtec/Kconfig
@@ -3,7 +3,7 @@ config COMMON_CLK_BOSTON
 	bool "Clock driver for MIPS Boston boards"
 	depends on MIPS || COMPILE_TEST
 	select MFD_SYSCON
-	---help---
+	help
 	  Enable this to support the system & CPU clocks on the MIPS Boston
 	  development board from Imagination Technologies. These are simple
 	  fixed rate clocks whose rate is determined by reading a platform
diff --git a/drivers/clk/imx/Kconfig b/drivers/clk/imx/Kconfig
index 01eadee88d66..db0253fa3d64 100644
--- a/drivers/clk/imx/Kconfig
+++ b/drivers/clk/imx/Kconfig
@@ -10,25 +10,25 @@ config MXC_CLK_SCU
 
 config CLK_IMX8MM
 	bool "IMX8MM CCM Clock Driver"
-	depends on ARCH_MXC && ARM64
+	depends on ARCH_MXC
 	help
 	    Build the driver for i.MX8MM CCM Clock Driver
 
 config CLK_IMX8MN
 	bool "IMX8MN CCM Clock Driver"
-	depends on ARCH_MXC && ARM64
+	depends on ARCH_MXC
 	help
 	    Build the driver for i.MX8MN CCM Clock Driver
 
 config CLK_IMX8MP
 	bool "IMX8MP CCM Clock Driver"
-	depends on ARCH_MXC && ARM64
+	depends on ARCH_MXC
 	help
 	    Build the driver for i.MX8MP CCM Clock Driver
 
 config CLK_IMX8MQ
 	bool "IMX8MQ CCM Clock Driver"
-	depends on ARCH_MXC && ARM64
+	depends on ARCH_MXC
 	help
 	    Build the driver for i.MX8MQ CCM Clock Driver
 
diff --git a/drivers/clk/imx/clk-composite-8m.c b/drivers/clk/imx/clk-composite-8m.c
index 99773519b5a5..d2b5af826f2c 100644
--- a/drivers/clk/imx/clk-composite-8m.c
+++ b/drivers/clk/imx/clk-composite-8m.c
@@ -124,6 +124,52 @@ static const struct clk_ops imx8m_clk_composite_divider_ops = {
 	.set_rate = imx8m_clk_composite_divider_set_rate,
 };
 
+static u8 imx8m_clk_composite_mux_get_parent(struct clk_hw *hw)
+{
+	return clk_mux_ops.get_parent(hw);
+}
+
+static int imx8m_clk_composite_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_mux *mux = to_clk_mux(hw);
+	u32 val = clk_mux_index_to_val(mux->table, mux->flags, index);
+	unsigned long flags = 0;
+	u32 reg;
+
+	if (mux->lock)
+		spin_lock_irqsave(mux->lock, flags);
+
+	reg = readl(mux->reg);
+	reg &= ~(mux->mask << mux->shift);
+	val = val << mux->shift;
+	reg |= val;
+	/*
+	 * write twice to make sure non-target interface
+	 * SEL_A/B point the same clk input.
+	 */
+	writel(reg, mux->reg);
+	writel(reg, mux->reg);
+
+	if (mux->lock)
+		spin_unlock_irqrestore(mux->lock, flags);
+
+	return 0;
+}
+
+static int
+imx8m_clk_composite_mux_determine_rate(struct clk_hw *hw,
+				       struct clk_rate_request *req)
+{
+	return clk_mux_ops.determine_rate(hw, req);
+}
+
+
+static const struct clk_ops imx8m_clk_composite_mux_ops = {
+	.get_parent = imx8m_clk_composite_mux_get_parent,
+	.set_parent = imx8m_clk_composite_mux_set_parent,
+	.determine_rate = imx8m_clk_composite_mux_determine_rate,
+};
+
 struct clk_hw *imx8m_clk_hw_composite_flags(const char *name,
 					const char * const *parent_names,
 					int num_parents, void __iomem *reg,
@@ -136,6 +182,7 @@ struct clk_hw *imx8m_clk_hw_composite_flags(const char *name,
 	struct clk_gate *gate = NULL;
 	struct clk_mux *mux = NULL;
 	const struct clk_ops *divider_ops;
+	const struct clk_ops *mux_ops;
 
 	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
 	if (!mux)
@@ -157,10 +204,17 @@ struct clk_hw *imx8m_clk_hw_composite_flags(const char *name,
 		div->shift = PCG_DIV_SHIFT;
 		div->width = PCG_CORE_DIV_WIDTH;
 		divider_ops = &clk_divider_ops;
+		mux_ops = &imx8m_clk_composite_mux_ops;
+	} else if (composite_flags & IMX_COMPOSITE_BUS) {
+		div->shift = PCG_PREDIV_SHIFT;
+		div->width = PCG_PREDIV_WIDTH;
+		divider_ops = &imx8m_clk_composite_divider_ops;
+		mux_ops = &imx8m_clk_composite_mux_ops;
 	} else {
 		div->shift = PCG_PREDIV_SHIFT;
 		div->width = PCG_PREDIV_WIDTH;
 		divider_ops = &imx8m_clk_composite_divider_ops;
+		mux_ops = &clk_mux_ops;
 	}
 
 	div->lock = &imx_ccm_lock;
@@ -176,7 +230,7 @@ struct clk_hw *imx8m_clk_hw_composite_flags(const char *name,
 	gate->lock = &imx_ccm_lock;
 
 	hw = clk_hw_register_composite(NULL, name, parent_names, num_parents,
-			mux_hw, &clk_mux_ops, div_hw,
+			mux_hw, mux_ops, div_hw,
 			divider_ops, gate_hw, &clk_gate_ops, flags);
 	if (IS_ERR(hw))
 		goto fail;
diff --git a/drivers/clk/imx/clk-gate2.c b/drivers/clk/imx/clk-gate2.c
index ce0060e8873e..b87ab3c3ba1e 100644
--- a/drivers/clk/imx/clk-gate2.c
+++ b/drivers/clk/imx/clk-gate2.c
@@ -41,21 +41,26 @@ static int clk_gate2_enable(struct clk_hw *hw)
 	struct clk_gate2 *gate = to_clk_gate2(hw);
 	u32 reg;
 	unsigned long flags;
+	int ret = 0;
 
 	spin_lock_irqsave(gate->lock, flags);
 
 	if (gate->share_count && (*gate->share_count)++ > 0)
 		goto out;
 
-	reg = readl(gate->reg);
-	reg &= ~(3 << gate->bit_idx);
-	reg |= gate->cgr_val << gate->bit_idx;
-	writel(reg, gate->reg);
+	if (gate->flags & IMX_CLK_GATE2_SINGLE_BIT) {
+		ret = clk_gate_ops.enable(hw);
+	} else {
+		reg = readl(gate->reg);
+		reg &= ~(3 << gate->bit_idx);
+		reg |= gate->cgr_val << gate->bit_idx;
+		writel(reg, gate->reg);
+	}
 
 out:
 	spin_unlock_irqrestore(gate->lock, flags);
 
-	return 0;
+	return ret;
 }
 
 static void clk_gate2_disable(struct clk_hw *hw)
@@ -73,9 +78,13 @@ static void clk_gate2_disable(struct clk_hw *hw)
 			goto out;
 	}
 
-	reg = readl(gate->reg);
-	reg &= ~(3 << gate->bit_idx);
-	writel(reg, gate->reg);
+	if (gate->flags & IMX_CLK_GATE2_SINGLE_BIT) {
+		clk_gate_ops.disable(hw);
+	} else {
+		reg = readl(gate->reg);
+		reg &= ~(3 << gate->bit_idx);
+		writel(reg, gate->reg);
+	}
 
 out:
 	spin_unlock_irqrestore(gate->lock, flags);
@@ -95,6 +104,9 @@ static int clk_gate2_is_enabled(struct clk_hw *hw)
 {
 	struct clk_gate2 *gate = to_clk_gate2(hw);
 
+	if (gate->flags & IMX_CLK_GATE2_SINGLE_BIT)
+		return clk_gate_ops.is_enabled(hw);
+
 	return clk_gate2_reg_is_enabled(gate->reg, gate->bit_idx);
 }
 
@@ -104,6 +116,9 @@ static void clk_gate2_disable_unused(struct clk_hw *hw)
 	unsigned long flags;
 	u32 reg;
 
+	if (gate->flags & IMX_CLK_GATE2_SINGLE_BIT)
+		return;
+
 	spin_lock_irqsave(gate->lock, flags);
 
 	if (!gate->share_count || *gate->share_count == 0) {
diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c
index dafc8806b03e..5dbb6a937732 100644
--- a/drivers/clk/imx/clk-imx6ul.c
+++ b/drivers/clk/imx/clk-imx6ul.c
@@ -503,7 +503,7 @@ static void __init imx6ul_clocks_init(struct device_node *ccm_node)
 		clk_prepare_enable(hws[IMX6UL_CLK_USBPHY2_GATE]->clk);
 	}
 
-	clk_set_parent(hws[IMX6UL_CLK_CAN_SEL]->clk, hws[IMX6UL_CLK_PLL3_60M]->clk);
+	clk_set_parent(hws[IMX6UL_CLK_CAN_SEL]->clk, hws[IMX6UL_CLK_PLL3_80M]->clk);
 	if (clk_on_imx6ul())
 		clk_set_parent(hws[IMX6UL_CLK_SIM_PRE_SEL]->clk, hws[IMX6UL_CLK_PLL3_USB_OTG]->clk);
 	else if (clk_on_imx6ull())
diff --git a/drivers/clk/imx/clk-imx7ulp.c b/drivers/clk/imx/clk-imx7ulp.c
index 3710aa0dee9b..634c0b6636b0 100644
--- a/drivers/clk/imx/clk-imx7ulp.c
+++ b/drivers/clk/imx/clk-imx7ulp.c
@@ -29,7 +29,7 @@ static const char * const ddr_sels[]		= { "apll_pfd_sel", "dummy", "dummy", "dum
 static const char * const nic_sels[]		= { "firc", "ddr_clk", };
 static const char * const periph_plat_sels[]	= { "dummy", "nic1_bus_clk", "nic1_clk", "ddr_clk", "apll_pfd2", "apll_pfd1", "apll_pfd0", "upll", };
 static const char * const periph_bus_sels[]	= { "dummy", "sosc_bus_clk", "dummy", "firc_bus_clk", "rosc", "nic1_bus_clk", "nic1_clk", "spll_bus_clk", };
-static const char * const arm_sels[]		= { "divcore", "dummy", "dummy", "hsrun_divcore", };
+static const char * const arm_sels[]		= { "core", "dummy", "dummy", "hsrun_core", };
 
 /* used by sosc/sirc/firc/ddr/spll/apll dividers */
 static const struct clk_div_table ulp_div_table[] = {
@@ -121,7 +121,9 @@ static void __init imx7ulp_clk_scg1_init(struct device_node *np)
 	hws[IMX7ULP_CLK_DDR_SEL]	= imx_clk_hw_mux_flags("ddr_sel", base + 0x30, 24, 2, ddr_sels, ARRAY_SIZE(ddr_sels), CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE);
 
 	hws[IMX7ULP_CLK_CORE_DIV]	= imx_clk_hw_divider_flags("divcore",	"scs_sel",  base + 0x14, 16, 4, CLK_SET_RATE_PARENT);
+	hws[IMX7ULP_CLK_CORE]		= imx_clk_hw_cpu("core", "divcore", hws[IMX7ULP_CLK_CORE_DIV]->clk, hws[IMX7ULP_CLK_SYS_SEL]->clk, hws[IMX7ULP_CLK_SPLL_SEL]->clk, hws[IMX7ULP_CLK_FIRC]->clk);
 	hws[IMX7ULP_CLK_HSRUN_CORE_DIV] = imx_clk_hw_divider_flags("hsrun_divcore", "hsrun_scs_sel", base + 0x1c, 16, 4, CLK_SET_RATE_PARENT);
+	hws[IMX7ULP_CLK_HSRUN_CORE] = imx_clk_hw_cpu("hsrun_core", "hsrun_divcore", hws[IMX7ULP_CLK_HSRUN_CORE_DIV]->clk, hws[IMX7ULP_CLK_HSRUN_SYS_SEL]->clk, hws[IMX7ULP_CLK_SPLL_SEL]->clk, hws[IMX7ULP_CLK_FIRC]->clk);
 
 	hws[IMX7ULP_CLK_DDR_DIV]	= imx_clk_hw_divider_gate("ddr_clk", "ddr_sel", CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, base + 0x30, 0, 3,
 							       0, ulp_div_table, &imx_ccm_lock);
@@ -270,7 +272,7 @@ static void __init imx7ulp_clk_smc1_init(struct device_node *np)
 	base = of_iomap(np, 0);
 	WARN_ON(!base);
 
-	hws[IMX7ULP_CLK_ARM] = imx_clk_hw_mux_flags("arm", base + 0x10, 8, 2, arm_sels, ARRAY_SIZE(arm_sels), CLK_IS_CRITICAL);
+	hws[IMX7ULP_CLK_ARM] = imx_clk_hw_mux_flags("arm", base + 0x10, 8, 2, arm_sels, ARRAY_SIZE(arm_sels), CLK_SET_RATE_PARENT);
 
 	imx_check_clk_hws(hws, clk_data->num);
 
diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c
index 925670438f23..b793264c21c6 100644
--- a/drivers/clk/imx/clk-imx8mm.c
+++ b/drivers/clk/imx/clk-imx8mm.c
@@ -416,9 +416,9 @@ static int imx8mm_clocks_probe(struct platform_device *pdev)
 		return PTR_ERR(base);
 
 	/* Core Slice */
-	hws[IMX8MM_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mm_a53_sels, ARRAY_SIZE(imx8mm_a53_sels));
-	hws[IMX8MM_CLK_A53_CG] = imx_clk_hw_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28);
-	hws[IMX8MM_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3);
+	hws[IMX8MM_CLK_A53_DIV] = imx8m_clk_hw_composite_core("arm_a53_div", imx8mm_a53_sels, base + 0x8000);
+	hws[IMX8MM_CLK_A53_CG] = hws[IMX8MM_CLK_A53_DIV];
+	hws[IMX8MM_CLK_A53_SRC] = hws[IMX8MM_CLK_A53_DIV];
 
 	hws[IMX8MM_CLK_M4_CORE] = imx8m_clk_hw_composite_core("arm_m4_core", imx8mm_m4_sels, base + 0x8080);
 	hws[IMX8MM_CLK_VPU_CORE] = imx8m_clk_hw_composite_core("vpu_core", imx8mm_vpu_sels, base + 0x8100);
@@ -444,21 +444,21 @@ static int imx8mm_clocks_probe(struct platform_device *pdev)
 
 	/* BUS */
 	hws[IMX8MM_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi",  imx8mm_main_axi_sels, base + 0x8800);
-	hws[IMX8MM_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mm_enet_axi_sels, base + 0x8880);
+	hws[IMX8MM_CLK_ENET_AXI] = imx8m_clk_hw_composite_bus("enet_axi", imx8mm_enet_axi_sels, base + 0x8880);
 	hws[IMX8MM_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite_critical("nand_usdhc_bus", imx8mm_nand_usdhc_sels, base + 0x8900);
-	hws[IMX8MM_CLK_VPU_BUS] = imx8m_clk_hw_composite("vpu_bus", imx8mm_vpu_bus_sels, base + 0x8980);
-	hws[IMX8MM_CLK_DISP_AXI] = imx8m_clk_hw_composite("disp_axi", imx8mm_disp_axi_sels, base + 0x8a00);
-	hws[IMX8MM_CLK_DISP_APB] = imx8m_clk_hw_composite("disp_apb", imx8mm_disp_apb_sels, base + 0x8a80);
-	hws[IMX8MM_CLK_DISP_RTRM] = imx8m_clk_hw_composite("disp_rtrm", imx8mm_disp_rtrm_sels, base + 0x8b00);
-	hws[IMX8MM_CLK_USB_BUS] = imx8m_clk_hw_composite("usb_bus", imx8mm_usb_bus_sels, base + 0x8b80);
-	hws[IMX8MM_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mm_gpu_axi_sels, base + 0x8c00);
-	hws[IMX8MM_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mm_gpu_ahb_sels, base + 0x8c80);
+	hws[IMX8MM_CLK_VPU_BUS] = imx8m_clk_hw_composite_bus("vpu_bus", imx8mm_vpu_bus_sels, base + 0x8980);
+	hws[IMX8MM_CLK_DISP_AXI] = imx8m_clk_hw_composite_bus("disp_axi", imx8mm_disp_axi_sels, base + 0x8a00);
+	hws[IMX8MM_CLK_DISP_APB] = imx8m_clk_hw_composite_bus("disp_apb", imx8mm_disp_apb_sels, base + 0x8a80);
+	hws[IMX8MM_CLK_DISP_RTRM] = imx8m_clk_hw_composite_bus("disp_rtrm", imx8mm_disp_rtrm_sels, base + 0x8b00);
+	hws[IMX8MM_CLK_USB_BUS] = imx8m_clk_hw_composite_bus("usb_bus", imx8mm_usb_bus_sels, base + 0x8b80);
+	hws[IMX8MM_CLK_GPU_AXI] = imx8m_clk_hw_composite_bus("gpu_axi", imx8mm_gpu_axi_sels, base + 0x8c00);
+	hws[IMX8MM_CLK_GPU_AHB] = imx8m_clk_hw_composite_bus("gpu_ahb", imx8mm_gpu_ahb_sels, base + 0x8c80);
 	hws[IMX8MM_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mm_noc_sels, base + 0x8d00);
 	hws[IMX8MM_CLK_NOC_APB] = imx8m_clk_hw_composite_critical("noc_apb", imx8mm_noc_apb_sels, base + 0x8d80);
 
 	/* AHB */
 	hws[IMX8MM_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb", imx8mm_ahb_sels, base + 0x9000);
-	hws[IMX8MM_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mm_audio_ahb_sels, base + 0x9100);
+	hws[IMX8MM_CLK_AUDIO_AHB] = imx8m_clk_hw_composite_bus("audio_ahb", imx8mm_audio_ahb_sels, base + 0x9100);
 
 	/* IPG */
 	hws[IMX8MM_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb", base + 0x9080, 0, 1);
@@ -614,9 +614,6 @@ static int imx8mm_clocks_probe(struct platform_device *pdev)
 					   hws[IMX8MM_ARM_PLL_OUT]->clk,
 					   hws[IMX8MM_CLK_A53_DIV]->clk);
 
-	clk_hw_set_parent(hws[IMX8MM_CLK_A53_SRC], hws[IMX8MM_SYS_PLL1_800M]);
-	clk_hw_set_parent(hws[IMX8MM_CLK_A53_CORE], hws[IMX8MM_ARM_PLL_OUT]);
-
 	imx_check_clk_hws(hws, IMX8MM_CLK_END);
 
 	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c
index 0bc7070235bd..213cc37b3173 100644
--- a/drivers/clk/imx/clk-imx8mn.c
+++ b/drivers/clk/imx/clk-imx8mn.c
@@ -413,9 +413,9 @@ static int imx8mn_clocks_probe(struct platform_device *pdev)
 	}
 
 	/* CORE */
-	hws[IMX8MN_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mn_a53_sels, ARRAY_SIZE(imx8mn_a53_sels));
-	hws[IMX8MN_CLK_A53_CG] = imx_clk_hw_gate3("arm_a53_cg", "arm_a53_src", base + 0x8000, 28);
-	hws[IMX8MN_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3);
+	hws[IMX8MN_CLK_A53_DIV] = imx8m_clk_hw_composite_core("arm_a53_div", imx8mn_a53_sels, base + 0x8000);
+	hws[IMX8MN_CLK_A53_SRC] = hws[IMX8MN_CLK_A53_DIV];
+	hws[IMX8MN_CLK_A53_CG] = hws[IMX8MN_CLK_A53_DIV];
 
 	hws[IMX8MN_CLK_GPU_CORE] = imx8m_clk_hw_composite_core("gpu_core", imx8mn_gpu_core_sels, base + 0x8180);
 	hws[IMX8MN_CLK_GPU_SHADER] = imx8m_clk_hw_composite_core("gpu_shader", imx8mn_gpu_shader_sels, base + 0x8200);
@@ -432,17 +432,17 @@ static int imx8mn_clocks_probe(struct platform_device *pdev)
 
 	/* BUS */
 	hws[IMX8MN_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi", imx8mn_main_axi_sels, base + 0x8800);
-	hws[IMX8MN_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mn_enet_axi_sels, base + 0x8880);
-	hws[IMX8MN_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite("nand_usdhc_bus", imx8mn_nand_usdhc_sels, base + 0x8900);
-	hws[IMX8MN_CLK_DISP_AXI] = imx8m_clk_hw_composite("disp_axi", imx8mn_disp_axi_sels, base + 0x8a00);
-	hws[IMX8MN_CLK_DISP_APB] = imx8m_clk_hw_composite("disp_apb", imx8mn_disp_apb_sels, base + 0x8a80);
-	hws[IMX8MN_CLK_USB_BUS] = imx8m_clk_hw_composite("usb_bus", imx8mn_usb_bus_sels, base + 0x8b80);
-	hws[IMX8MN_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mn_gpu_axi_sels, base + 0x8c00);
-	hws[IMX8MN_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mn_gpu_ahb_sels, base + 0x8c80);
+	hws[IMX8MN_CLK_ENET_AXI] = imx8m_clk_hw_composite_bus("enet_axi", imx8mn_enet_axi_sels, base + 0x8880);
+	hws[IMX8MN_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite_bus("nand_usdhc_bus", imx8mn_nand_usdhc_sels, base + 0x8900);
+	hws[IMX8MN_CLK_DISP_AXI] = imx8m_clk_hw_composite_bus("disp_axi", imx8mn_disp_axi_sels, base + 0x8a00);
+	hws[IMX8MN_CLK_DISP_APB] = imx8m_clk_hw_composite_bus("disp_apb", imx8mn_disp_apb_sels, base + 0x8a80);
+	hws[IMX8MN_CLK_USB_BUS] = imx8m_clk_hw_composite_bus("usb_bus", imx8mn_usb_bus_sels, base + 0x8b80);
+	hws[IMX8MN_CLK_GPU_AXI] = imx8m_clk_hw_composite_bus("gpu_axi", imx8mn_gpu_axi_sels, base + 0x8c00);
+	hws[IMX8MN_CLK_GPU_AHB] = imx8m_clk_hw_composite_bus("gpu_ahb", imx8mn_gpu_ahb_sels, base + 0x8c80);
 	hws[IMX8MN_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mn_noc_sels, base + 0x8d00);
 
 	hws[IMX8MN_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb", imx8mn_ahb_sels, base + 0x9000);
-	hws[IMX8MN_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mn_audio_ahb_sels, base + 0x9100);
+	hws[IMX8MN_CLK_AUDIO_AHB] = imx8m_clk_hw_composite_bus("audio_ahb", imx8mn_audio_ahb_sels, base + 0x9100);
 	hws[IMX8MN_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb", base + 0x9080, 0, 1);
 	hws[IMX8MN_CLK_IPG_AUDIO_ROOT] = imx_clk_hw_divider2("ipg_audio_root", "audio_ahb", base + 0x9180, 0, 1);
 	hws[IMX8MN_CLK_DRAM_CORE] = imx_clk_hw_mux2_flags("dram_core_clk", base + 0x9800, 24, 1, imx8mn_dram_core_sels, ARRAY_SIZE(imx8mn_dram_core_sels), CLK_IS_CRITICAL);
@@ -565,9 +565,6 @@ static int imx8mn_clocks_probe(struct platform_device *pdev)
 					   hws[IMX8MN_ARM_PLL_OUT]->clk,
 					   hws[IMX8MN_CLK_A53_DIV]->clk);
 
-	clk_hw_set_parent(hws[IMX8MN_CLK_A53_SRC], hws[IMX8MN_SYS_PLL1_800M]);
-	clk_hw_set_parent(hws[IMX8MN_CLK_A53_CORE], hws[IMX8MN_ARM_PLL_OUT]);
-
 	imx_check_clk_hws(hws, IMX8MN_CLK_END);
 
 	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
diff --git a/drivers/clk/imx/clk-imx8mp.c b/drivers/clk/imx/clk-imx8mp.c
index 41469e2cc3de..b4d9db9d5bf1 100644
--- a/drivers/clk/imx/clk-imx8mp.c
+++ b/drivers/clk/imx/clk-imx8mp.c
@@ -486,16 +486,16 @@ static int imx8mp_clocks_probe(struct platform_device *pdev)
 	hws[IMX8MP_SYS_PLL2] = imx_clk_hw_pll14xx("sys_pll2", "sys_pll2_ref_sel", anatop_base + 0x104, &imx_1416x_pll);
 	hws[IMX8MP_SYS_PLL3] = imx_clk_hw_pll14xx("sys_pll3", "sys_pll3_ref_sel", anatop_base + 0x114, &imx_1416x_pll);
 
-	hws[IMX8MP_AUDIO_PLL1_BYPASS] = imx_clk_hw_mux_flags("audio_pll1_bypass", anatop_base, 4, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_AUDIO_PLL2_BYPASS] = imx_clk_hw_mux_flags("audio_pll2_bypass", anatop_base + 0x14, 4, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_VIDEO_PLL1_BYPASS] = imx_clk_hw_mux_flags("video_pll1_bypass", anatop_base + 0x28, 4, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_DRAM_PLL_BYPASS] = imx_clk_hw_mux_flags("dram_pll_bypass", anatop_base + 0x50, 4, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_GPU_PLL_BYPASS] = imx_clk_hw_mux_flags("gpu_pll_bypass", anatop_base + 0x64, 4, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_VPU_PLL_BYPASS] = imx_clk_hw_mux_flags("vpu_pll_bypass", anatop_base + 0x74, 4, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_ARM_PLL_BYPASS] = imx_clk_hw_mux_flags("arm_pll_bypass", anatop_base + 0x84, 4, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_SYS_PLL1_BYPASS] = imx_clk_hw_mux_flags("sys_pll1_bypass", anatop_base + 0x94, 4, 1, sys_pll1_bypass_sels, ARRAY_SIZE(sys_pll1_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_SYS_PLL2_BYPASS] = imx_clk_hw_mux_flags("sys_pll2_bypass", anatop_base + 0x104, 4, 1, sys_pll2_bypass_sels, ARRAY_SIZE(sys_pll2_bypass_sels), CLK_SET_RATE_PARENT);
-	hws[IMX8MP_SYS_PLL3_BYPASS] = imx_clk_hw_mux_flags("sys_pll3_bypass", anatop_base + 0x114, 4, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_AUDIO_PLL1_BYPASS] = imx_clk_hw_mux_flags("audio_pll1_bypass", anatop_base, 16, 1, audio_pll1_bypass_sels, ARRAY_SIZE(audio_pll1_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_AUDIO_PLL2_BYPASS] = imx_clk_hw_mux_flags("audio_pll2_bypass", anatop_base + 0x14, 16, 1, audio_pll2_bypass_sels, ARRAY_SIZE(audio_pll2_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_VIDEO_PLL1_BYPASS] = imx_clk_hw_mux_flags("video_pll1_bypass", anatop_base + 0x28, 16, 1, video_pll1_bypass_sels, ARRAY_SIZE(video_pll1_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_DRAM_PLL_BYPASS] = imx_clk_hw_mux_flags("dram_pll_bypass", anatop_base + 0x50, 16, 1, dram_pll_bypass_sels, ARRAY_SIZE(dram_pll_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_GPU_PLL_BYPASS] = imx_clk_hw_mux_flags("gpu_pll_bypass", anatop_base + 0x64, 28, 1, gpu_pll_bypass_sels, ARRAY_SIZE(gpu_pll_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_VPU_PLL_BYPASS] = imx_clk_hw_mux_flags("vpu_pll_bypass", anatop_base + 0x74, 28, 1, vpu_pll_bypass_sels, ARRAY_SIZE(vpu_pll_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_ARM_PLL_BYPASS] = imx_clk_hw_mux_flags("arm_pll_bypass", anatop_base + 0x84, 28, 1, arm_pll_bypass_sels, ARRAY_SIZE(arm_pll_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_SYS_PLL1_BYPASS] = imx_clk_hw_mux_flags("sys_pll1_bypass", anatop_base + 0x94, 28, 1, sys_pll1_bypass_sels, ARRAY_SIZE(sys_pll1_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_SYS_PLL2_BYPASS] = imx_clk_hw_mux_flags("sys_pll2_bypass", anatop_base + 0x104, 28, 1, sys_pll2_bypass_sels, ARRAY_SIZE(sys_pll2_bypass_sels), CLK_SET_RATE_PARENT);
+	hws[IMX8MP_SYS_PLL3_BYPASS] = imx_clk_hw_mux_flags("sys_pll3_bypass", anatop_base + 0x114, 28, 1, sys_pll3_bypass_sels, ARRAY_SIZE(sys_pll3_bypass_sels), CLK_SET_RATE_PARENT);
 
 	hws[IMX8MP_AUDIO_PLL1_OUT] = imx_clk_hw_gate("audio_pll1_out", "audio_pll1_bypass", anatop_base, 13);
 	hws[IMX8MP_AUDIO_PLL2_OUT] = imx_clk_hw_gate("audio_pll2_out", "audio_pll2_bypass", anatop_base + 0x14, 13);
@@ -504,79 +504,82 @@ static int imx8mp_clocks_probe(struct platform_device *pdev)
 	hws[IMX8MP_GPU_PLL_OUT] = imx_clk_hw_gate("gpu_pll_out", "gpu_pll_bypass", anatop_base + 0x64, 11);
 	hws[IMX8MP_VPU_PLL_OUT] = imx_clk_hw_gate("vpu_pll_out", "vpu_pll_bypass", anatop_base + 0x74, 11);
 	hws[IMX8MP_ARM_PLL_OUT] = imx_clk_hw_gate("arm_pll_out", "arm_pll_bypass", anatop_base + 0x84, 11);
-	hws[IMX8MP_SYS_PLL1_OUT] = imx_clk_hw_gate("sys_pll1_out", "sys_pll1_bypass", anatop_base + 0x94, 11);
-	hws[IMX8MP_SYS_PLL2_OUT] = imx_clk_hw_gate("sys_pll2_out", "sys_pll2_bypass", anatop_base + 0x104, 11);
 	hws[IMX8MP_SYS_PLL3_OUT] = imx_clk_hw_gate("sys_pll3_out", "sys_pll3_bypass", anatop_base + 0x114, 11);
 
-	hws[IMX8MP_SYS_PLL1_40M] = imx_clk_hw_fixed_factor("sys_pll1_40m", "sys_pll1_out", 1, 20);
-	hws[IMX8MP_SYS_PLL1_80M] = imx_clk_hw_fixed_factor("sys_pll1_80m", "sys_pll1_out", 1, 10);
-	hws[IMX8MP_SYS_PLL1_100M] = imx_clk_hw_fixed_factor("sys_pll1_100m", "sys_pll1_out", 1, 8);
-	hws[IMX8MP_SYS_PLL1_133M] = imx_clk_hw_fixed_factor("sys_pll1_133m", "sys_pll1_out", 1, 6);
-	hws[IMX8MP_SYS_PLL1_160M] = imx_clk_hw_fixed_factor("sys_pll1_160m", "sys_pll1_out", 1, 5);
-	hws[IMX8MP_SYS_PLL1_200M] = imx_clk_hw_fixed_factor("sys_pll1_200m", "sys_pll1_out", 1, 4);
-	hws[IMX8MP_SYS_PLL1_266M] = imx_clk_hw_fixed_factor("sys_pll1_266m", "sys_pll1_out", 1, 3);
-	hws[IMX8MP_SYS_PLL1_400M] = imx_clk_hw_fixed_factor("sys_pll1_400m", "sys_pll1_out", 1, 2);
+	hws[IMX8MP_SYS_PLL1_40M_CG] = imx_clk_hw_gate("sys_pll1_40m_cg", "sys_pll1_bypass", anatop_base + 0x94, 27);
+	hws[IMX8MP_SYS_PLL1_80M_CG] = imx_clk_hw_gate("sys_pll1_80m_cg", "sys_pll1_bypass", anatop_base + 0x94, 25);
+	hws[IMX8MP_SYS_PLL1_100M_CG] = imx_clk_hw_gate("sys_pll1_100m_cg", "sys_pll1_bypass", anatop_base + 0x94, 23);
+	hws[IMX8MP_SYS_PLL1_133M_CG] = imx_clk_hw_gate("sys_pll1_133m_cg", "sys_pll1_bypass", anatop_base + 0x94, 21);
+	hws[IMX8MP_SYS_PLL1_160M_CG] = imx_clk_hw_gate("sys_pll1_160m_cg", "sys_pll1_bypass", anatop_base + 0x94, 19);
+	hws[IMX8MP_SYS_PLL1_200M_CG] = imx_clk_hw_gate("sys_pll1_200m_cg", "sys_pll1_bypass", anatop_base + 0x94, 17);
+	hws[IMX8MP_SYS_PLL1_266M_CG] = imx_clk_hw_gate("sys_pll1_266m_cg", "sys_pll1_bypass", anatop_base + 0x94, 15);
+	hws[IMX8MP_SYS_PLL1_400M_CG] = imx_clk_hw_gate("sys_pll1_400m_cg", "sys_pll1_bypass", anatop_base + 0x94, 13);
+	hws[IMX8MP_SYS_PLL1_OUT] = imx_clk_hw_gate("sys_pll1_out", "sys_pll1_bypass", anatop_base + 0x94, 11);
+
+	hws[IMX8MP_SYS_PLL1_40M] = imx_clk_hw_fixed_factor("sys_pll1_40m", "sys_pll1_40m_cg", 1, 20);
+	hws[IMX8MP_SYS_PLL1_80M] = imx_clk_hw_fixed_factor("sys_pll1_80m", "sys_pll1_80m_cg", 1, 10);
+	hws[IMX8MP_SYS_PLL1_100M] = imx_clk_hw_fixed_factor("sys_pll1_100m", "sys_pll1_100m_cg", 1, 8);
+	hws[IMX8MP_SYS_PLL1_133M] = imx_clk_hw_fixed_factor("sys_pll1_133m", "sys_pll1_133m_cg", 1, 6);
+	hws[IMX8MP_SYS_PLL1_160M] = imx_clk_hw_fixed_factor("sys_pll1_160m", "sys_pll1_160m_cg", 1, 5);
+	hws[IMX8MP_SYS_PLL1_200M] = imx_clk_hw_fixed_factor("sys_pll1_200m", "sys_pll1_200m_cg", 1, 4);
+	hws[IMX8MP_SYS_PLL1_266M] = imx_clk_hw_fixed_factor("sys_pll1_266m", "sys_pll1_266m_cg", 1, 3);
+	hws[IMX8MP_SYS_PLL1_400M] = imx_clk_hw_fixed_factor("sys_pll1_400m", "sys_pll1_400m_cg", 1, 2);
 	hws[IMX8MP_SYS_PLL1_800M] = imx_clk_hw_fixed_factor("sys_pll1_800m", "sys_pll1_out", 1, 1);
 
-	hws[IMX8MP_SYS_PLL2_50M] = imx_clk_hw_fixed_factor("sys_pll2_50m", "sys_pll2_out", 1, 20);
-	hws[IMX8MP_SYS_PLL2_100M] = imx_clk_hw_fixed_factor("sys_pll2_100m", "sys_pll2_out", 1, 10);
-	hws[IMX8MP_SYS_PLL2_125M] = imx_clk_hw_fixed_factor("sys_pll2_125m", "sys_pll2_out", 1, 8);
-	hws[IMX8MP_SYS_PLL2_166M] = imx_clk_hw_fixed_factor("sys_pll2_166m", "sys_pll2_out", 1, 6);
-	hws[IMX8MP_SYS_PLL2_200M] = imx_clk_hw_fixed_factor("sys_pll2_200m", "sys_pll2_out", 1, 5);
-	hws[IMX8MP_SYS_PLL2_250M] = imx_clk_hw_fixed_factor("sys_pll2_250m", "sys_pll2_out", 1, 4);
-	hws[IMX8MP_SYS_PLL2_333M] = imx_clk_hw_fixed_factor("sys_pll2_333m", "sys_pll2_out", 1, 3);
-	hws[IMX8MP_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_out", 1, 2);
+	hws[IMX8MP_SYS_PLL2_50M_CG] = imx_clk_hw_gate("sys_pll2_50m_cg", "sys_pll2_bypass", anatop_base + 0x104, 27);
+	hws[IMX8MP_SYS_PLL2_100M_CG] = imx_clk_hw_gate("sys_pll2_100m_cg", "sys_pll2_bypass", anatop_base + 0x104, 25);
+	hws[IMX8MP_SYS_PLL2_125M_CG] = imx_clk_hw_gate("sys_pll2_125m_cg", "sys_pll2_bypass", anatop_base + 0x104, 23);
+	hws[IMX8MP_SYS_PLL2_166M_CG] = imx_clk_hw_gate("sys_pll2_166m_cg", "sys_pll2_bypass", anatop_base + 0x104, 21);
+	hws[IMX8MP_SYS_PLL2_200M_CG] = imx_clk_hw_gate("sys_pll2_200m_cg", "sys_pll2_bypass", anatop_base + 0x104, 19);
+	hws[IMX8MP_SYS_PLL2_250M_CG] = imx_clk_hw_gate("sys_pll2_250m_cg", "sys_pll2_bypass", anatop_base + 0x104, 17);
+	hws[IMX8MP_SYS_PLL2_333M_CG] = imx_clk_hw_gate("sys_pll2_333m_cg", "sys_pll2_bypass", anatop_base + 0x104, 15);
+	hws[IMX8MP_SYS_PLL2_500M_CG] = imx_clk_hw_gate("sys_pll2_500m_cg", "sys_pll2_bypass", anatop_base + 0x104, 13);
+	hws[IMX8MP_SYS_PLL2_OUT] = imx_clk_hw_gate("sys_pll2_out", "sys_pll2_bypass", anatop_base + 0x104, 11);
+
+	hws[IMX8MP_SYS_PLL2_50M] = imx_clk_hw_fixed_factor("sys_pll2_50m", "sys_pll2_50m_cg", 1, 20);
+	hws[IMX8MP_SYS_PLL2_100M] = imx_clk_hw_fixed_factor("sys_pll2_100m", "sys_pll2_100m_cg", 1, 10);
+	hws[IMX8MP_SYS_PLL2_125M] = imx_clk_hw_fixed_factor("sys_pll2_125m", "sys_pll2_125m_cg", 1, 8);
+	hws[IMX8MP_SYS_PLL2_166M] = imx_clk_hw_fixed_factor("sys_pll2_166m", "sys_pll2_166m_cg", 1, 6);
+	hws[IMX8MP_SYS_PLL2_200M] = imx_clk_hw_fixed_factor("sys_pll2_200m", "sys_pll2_200m_cg", 1, 5);
+	hws[IMX8MP_SYS_PLL2_250M] = imx_clk_hw_fixed_factor("sys_pll2_250m", "sys_pll2_250m_cg", 1, 4);
+	hws[IMX8MP_SYS_PLL2_333M] = imx_clk_hw_fixed_factor("sys_pll2_333m", "sys_pll2_333m_cg", 1, 3);
+	hws[IMX8MP_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2);
 	hws[IMX8MP_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1);
 
-	hws[IMX8MP_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", ccm_base + 0x8000, 24, 3, imx8mp_a53_sels, ARRAY_SIZE(imx8mp_a53_sels));
-	hws[IMX8MP_CLK_M7_SRC] = imx_clk_hw_mux2("arm_m7_src", ccm_base + 0x8080, 24, 3, imx8mp_m7_sels, ARRAY_SIZE(imx8mp_m7_sels));
-	hws[IMX8MP_CLK_ML_SRC] = imx_clk_hw_mux2("ml_src", ccm_base + 0x8100, 24, 3, imx8mp_ml_sels, ARRAY_SIZE(imx8mp_ml_sels));
-	hws[IMX8MP_CLK_GPU3D_CORE_SRC] = imx_clk_hw_mux2("gpu3d_core_src", ccm_base + 0x8180, 24, 3,  imx8mp_gpu3d_core_sels, ARRAY_SIZE(imx8mp_gpu3d_core_sels));
-	hws[IMX8MP_CLK_GPU3D_SHADER_SRC] = imx_clk_hw_mux2("gpu3d_shader_src", ccm_base + 0x8200, 24, 3, imx8mp_gpu3d_shader_sels, ARRAY_SIZE(imx8mp_gpu3d_shader_sels));
-	hws[IMX8MP_CLK_GPU2D_SRC] = imx_clk_hw_mux2("gpu2d_src", ccm_base + 0x8280, 24, 3, imx8mp_gpu2d_sels, ARRAY_SIZE(imx8mp_gpu2d_sels));
-	hws[IMX8MP_CLK_AUDIO_AXI_SRC] = imx_clk_hw_mux2("audio_axi_src", ccm_base + 0x8300, 24, 3, imx8mp_audio_axi_sels, ARRAY_SIZE(imx8mp_audio_axi_sels));
-	hws[IMX8MP_CLK_HSIO_AXI_SRC] = imx_clk_hw_mux2("hsio_axi_src", ccm_base + 0x8380, 24, 3, imx8mp_hsio_axi_sels, ARRAY_SIZE(imx8mp_hsio_axi_sels));
-	hws[IMX8MP_CLK_MEDIA_ISP_SRC] = imx_clk_hw_mux2("media_isp_src", ccm_base + 0x8400, 24, 3, imx8mp_media_isp_sels, ARRAY_SIZE(imx8mp_media_isp_sels));
-	hws[IMX8MP_CLK_A53_CG] = imx_clk_hw_gate3("arm_a53_cg", "arm_a53_src", ccm_base + 0x8000, 28);
-	hws[IMX8MP_CLK_M4_CG] = imx_clk_hw_gate3("arm_m7_cg", "arm_m7_src", ccm_base + 0x8080, 28);
-	hws[IMX8MP_CLK_ML_CG] = imx_clk_hw_gate3("ml_cg", "ml_src", ccm_base + 0x8100, 28);
-	hws[IMX8MP_CLK_GPU3D_CORE_CG] = imx_clk_hw_gate3("gpu3d_core_cg", "gpu3d_core_src", ccm_base + 0x8180, 28);
-	hws[IMX8MP_CLK_GPU3D_SHADER_CG] = imx_clk_hw_gate3("gpu3d_shader_cg", "gpu3d_shader_src", ccm_base + 0x8200, 28);
-	hws[IMX8MP_CLK_GPU2D_CG] = imx_clk_hw_gate3("gpu2d_cg", "gpu2d_src", ccm_base + 0x8280, 28);
-	hws[IMX8MP_CLK_AUDIO_AXI_CG] = imx_clk_hw_gate3("audio_axi_cg", "audio_axi_src", ccm_base + 0x8300, 28);
-	hws[IMX8MP_CLK_HSIO_AXI_CG] = imx_clk_hw_gate3("hsio_axi_cg", "hsio_axi_src", ccm_base + 0x8380, 28);
-	hws[IMX8MP_CLK_MEDIA_ISP_CG] = imx_clk_hw_gate3("media_isp_cg", "media_isp_src", ccm_base + 0x8400, 28);
-	hws[IMX8MP_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", ccm_base + 0x8000, 0, 3);
-	hws[IMX8MP_CLK_M7_DIV] = imx_clk_hw_divider2("arm_m7_div", "arm_m7_cg", ccm_base + 0x8080, 0, 3);
-	hws[IMX8MP_CLK_ML_DIV] = imx_clk_hw_divider2("ml_div", "ml_cg", ccm_base + 0x8100, 0, 3);
-	hws[IMX8MP_CLK_GPU3D_CORE_DIV] = imx_clk_hw_divider2("gpu3d_core_div", "gpu3d_core_cg", ccm_base + 0x8180, 0, 3);
-	hws[IMX8MP_CLK_GPU3D_SHADER_DIV] = imx_clk_hw_divider2("gpu3d_shader_div", "gpu3d_shader_cg", ccm_base + 0x8200, 0, 3);
-	hws[IMX8MP_CLK_GPU2D_DIV] = imx_clk_hw_divider2("gpu2d_div", "gpu2d_cg", ccm_base + 0x8280, 0, 3);
-	hws[IMX8MP_CLK_AUDIO_AXI_DIV] = imx_clk_hw_divider2("audio_axi_div", "audio_axi_cg", ccm_base + 0x8300, 0, 3);
-	hws[IMX8MP_CLK_HSIO_AXI_DIV] = imx_clk_hw_divider2("hsio_axi_div", "hsio_axi_cg", ccm_base + 0x8380, 0, 3);
-	hws[IMX8MP_CLK_MEDIA_ISP_DIV] = imx_clk_hw_divider2("media_isp_div", "media_isp_cg", ccm_base + 0x8400, 0, 3);
+	hws[IMX8MP_CLK_A53_DIV] = imx8m_clk_hw_composite_core("arm_a53_div", imx8mp_a53_sels, ccm_base + 0x8000);
+	hws[IMX8MP_CLK_A53_SRC] = hws[IMX8MP_CLK_A53_DIV];
+	hws[IMX8MP_CLK_A53_CG] = hws[IMX8MP_CLK_A53_DIV];
+	hws[IMX8MP_CLK_M7_CORE] = imx8m_clk_hw_composite_core("m7_core", imx8mp_m7_sels, ccm_base + 0x8080);
+	hws[IMX8MP_CLK_ML_CORE] = imx8m_clk_hw_composite_core("ml_core", imx8mp_ml_sels, ccm_base + 0x8100);
+	hws[IMX8MP_CLK_GPU3D_CORE] = imx8m_clk_hw_composite_core("gpu3d_core", imx8mp_gpu3d_core_sels, ccm_base + 0x8180);
+	hws[IMX8MP_CLK_GPU3D_SHADER_CORE] = imx8m_clk_hw_composite("gpu3d_shader_core", imx8mp_gpu3d_shader_sels, ccm_base + 0x8200);
+	hws[IMX8MP_CLK_GPU2D_CORE] = imx8m_clk_hw_composite("gpu2d_core", imx8mp_gpu2d_sels, ccm_base + 0x8280);
+	hws[IMX8MP_CLK_AUDIO_AXI] = imx8m_clk_hw_composite("audio_axi", imx8mp_audio_axi_sels, ccm_base + 0x8300);
+	hws[IMX8MP_CLK_AUDIO_AXI_SRC] = hws[IMX8MP_CLK_AUDIO_AXI];
+	hws[IMX8MP_CLK_HSIO_AXI] = imx8m_clk_hw_composite("hsio_axi", imx8mp_hsio_axi_sels, ccm_base + 0x8380);
+	hws[IMX8MP_CLK_MEDIA_ISP] = imx8m_clk_hw_composite("media_isp", imx8mp_media_isp_sels, ccm_base + 0x8400);
 
 	/* CORE SEL */
 	hws[IMX8MP_CLK_A53_CORE] = imx_clk_hw_mux2("arm_a53_core", ccm_base + 0x9880, 24, 1, imx8mp_a53_core_sels, ARRAY_SIZE(imx8mp_a53_core_sels));
 
 	hws[IMX8MP_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi", imx8mp_main_axi_sels, ccm_base + 0x8800);
-	hws[IMX8MP_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mp_enet_axi_sels, ccm_base + 0x8880);
+	hws[IMX8MP_CLK_ENET_AXI] = imx8m_clk_hw_composite_bus("enet_axi", imx8mp_enet_axi_sels, ccm_base + 0x8880);
 	hws[IMX8MP_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite_critical("nand_usdhc_bus", imx8mp_nand_usdhc_sels, ccm_base + 0x8900);
-	hws[IMX8MP_CLK_VPU_BUS] = imx8m_clk_hw_composite("vpu_bus", imx8mp_vpu_bus_sels, ccm_base + 0x8980);
-	hws[IMX8MP_CLK_MEDIA_AXI] = imx8m_clk_hw_composite("media_axi", imx8mp_media_axi_sels, ccm_base + 0x8a00);
-	hws[IMX8MP_CLK_MEDIA_APB] = imx8m_clk_hw_composite("media_apb", imx8mp_media_apb_sels, ccm_base + 0x8a80);
-	hws[IMX8MP_CLK_HDMI_APB] = imx8m_clk_hw_composite("hdmi_apb", imx8mp_media_apb_sels, ccm_base + 0x8b00);
-	hws[IMX8MP_CLK_HDMI_AXI] = imx8m_clk_hw_composite("hdmi_axi", imx8mp_media_axi_sels, ccm_base + 0x8b80);
-	hws[IMX8MP_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mp_gpu_axi_sels, ccm_base + 0x8c00);
-	hws[IMX8MP_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mp_gpu_ahb_sels, ccm_base + 0x8c80);
+	hws[IMX8MP_CLK_VPU_BUS] = imx8m_clk_hw_composite_bus("vpu_bus", imx8mp_vpu_bus_sels, ccm_base + 0x8980);
+	hws[IMX8MP_CLK_MEDIA_AXI] = imx8m_clk_hw_composite_bus("media_axi", imx8mp_media_axi_sels, ccm_base + 0x8a00);
+	hws[IMX8MP_CLK_MEDIA_APB] = imx8m_clk_hw_composite_bus("media_apb", imx8mp_media_apb_sels, ccm_base + 0x8a80);
+	hws[IMX8MP_CLK_HDMI_APB] = imx8m_clk_hw_composite_bus("hdmi_apb", imx8mp_media_apb_sels, ccm_base + 0x8b00);
+	hws[IMX8MP_CLK_HDMI_AXI] = imx8m_clk_hw_composite_bus("hdmi_axi", imx8mp_media_axi_sels, ccm_base + 0x8b80);
+	hws[IMX8MP_CLK_GPU_AXI] = imx8m_clk_hw_composite_bus("gpu_axi", imx8mp_gpu_axi_sels, ccm_base + 0x8c00);
+	hws[IMX8MP_CLK_GPU_AHB] = imx8m_clk_hw_composite_bus("gpu_ahb", imx8mp_gpu_ahb_sels, ccm_base + 0x8c80);
 	hws[IMX8MP_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mp_noc_sels, ccm_base + 0x8d00);
 	hws[IMX8MP_CLK_NOC_IO] = imx8m_clk_hw_composite_critical("noc_io", imx8mp_noc_io_sels, ccm_base + 0x8d80);
-	hws[IMX8MP_CLK_ML_AXI] = imx8m_clk_hw_composite("ml_axi", imx8mp_ml_axi_sels, ccm_base + 0x8e00);
-	hws[IMX8MP_CLK_ML_AHB] = imx8m_clk_hw_composite("ml_ahb", imx8mp_ml_ahb_sels, ccm_base + 0x8e80);
+	hws[IMX8MP_CLK_ML_AXI] = imx8m_clk_hw_composite_bus("ml_axi", imx8mp_ml_axi_sels, ccm_base + 0x8e00);
+	hws[IMX8MP_CLK_ML_AHB] = imx8m_clk_hw_composite_bus("ml_ahb", imx8mp_ml_ahb_sels, ccm_base + 0x8e80);
 
 	hws[IMX8MP_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb_root", imx8mp_ahb_sels, ccm_base + 0x9000);
-	hws[IMX8MP_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mp_audio_ahb_sels, ccm_base + 0x9100);
-	hws[IMX8MP_CLK_MIPI_DSI_ESC_RX] = imx8m_clk_hw_composite("mipi_dsi_esc_rx", imx8mp_mipi_dsi_esc_rx_sels, ccm_base + 0x9200);
+	hws[IMX8MP_CLK_AUDIO_AHB] = imx8m_clk_hw_composite_bus("audio_ahb", imx8mp_audio_ahb_sels, ccm_base + 0x9100);
+	hws[IMX8MP_CLK_MIPI_DSI_ESC_RX] = imx8m_clk_hw_composite_bus("mipi_dsi_esc_rx", imx8mp_mipi_dsi_esc_rx_sels, ccm_base + 0x9200);
 
 	hws[IMX8MP_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb_root", ccm_base + 0x9080, 0, 1);
 	hws[IMX8MP_CLK_IPG_AUDIO_ROOT] = imx_clk_hw_divider2("ipg_audio_root", "audio_ahb", ccm_base + 0x9180, 0, 1);
@@ -695,8 +698,8 @@ static int imx8mp_clocks_probe(struct platform_device *pdev)
 	hws[IMX8MP_CLK_SDMA1_ROOT] = imx_clk_hw_gate4("sdma1_root_clk", "ipg_root", ccm_base + 0x43a0, 0);
 	hws[IMX8MP_CLK_ENET_QOS_ROOT] = imx_clk_hw_gate4("enet_qos_root_clk", "sim_enet_root_clk", ccm_base + 0x43b0, 0);
 	hws[IMX8MP_CLK_SIM_ENET_ROOT] = imx_clk_hw_gate4("sim_enet_root_clk", "enet_axi", ccm_base + 0x4400, 0);
-	hws[IMX8MP_CLK_GPU2D_ROOT] = imx_clk_hw_gate4("gpu2d_root_clk", "gpu2d_div", ccm_base + 0x4450, 0);
-	hws[IMX8MP_CLK_GPU3D_ROOT] = imx_clk_hw_gate4("gpu3d_root_clk", "gpu3d_core_div", ccm_base + 0x4460, 0);
+	hws[IMX8MP_CLK_GPU2D_ROOT] = imx_clk_hw_gate4("gpu2d_root_clk", "gpu2d_core", ccm_base + 0x4450, 0);
+	hws[IMX8MP_CLK_GPU3D_ROOT] = imx_clk_hw_gate4("gpu3d_root_clk", "gpu3d_core", ccm_base + 0x4460, 0);
 	hws[IMX8MP_CLK_SNVS_ROOT] = imx_clk_hw_gate4("snvs_root_clk", "ipg_root", ccm_base + 0x4470, 0);
 	hws[IMX8MP_CLK_UART1_ROOT] = imx_clk_hw_gate4("uart1_root_clk", "uart1", ccm_base + 0x4490, 0);
 	hws[IMX8MP_CLK_UART2_ROOT] = imx_clk_hw_gate4("uart2_root_clk", "uart2", ccm_base + 0x44a0, 0);
@@ -713,7 +716,7 @@ static int imx8mp_clocks_probe(struct platform_device *pdev)
 	hws[IMX8MP_CLK_GPU_ROOT] = imx_clk_hw_gate4("gpu_root_clk", "gpu_axi", ccm_base + 0x4570, 0);
 	hws[IMX8MP_CLK_VPU_VC8KE_ROOT] = imx_clk_hw_gate4("vpu_vc8ke_root_clk", "vpu_vc8000e", ccm_base + 0x4590, 0);
 	hws[IMX8MP_CLK_VPU_G2_ROOT] = imx_clk_hw_gate4("vpu_g2_root_clk", "vpu_g2", ccm_base + 0x45a0, 0);
-	hws[IMX8MP_CLK_NPU_ROOT] = imx_clk_hw_gate4("npu_root_clk", "ml_div", ccm_base + 0x45b0, 0);
+	hws[IMX8MP_CLK_NPU_ROOT] = imx_clk_hw_gate4("npu_root_clk", "ml_core", ccm_base + 0x45b0, 0);
 	hws[IMX8MP_CLK_HSIO_ROOT] = imx_clk_hw_gate4("hsio_root_clk", "ipg_root", ccm_base + 0x45c0, 0);
 	hws[IMX8MP_CLK_MEDIA_APB_ROOT] = imx_clk_hw_gate2_shared2("media_apb_root_clk", "media_apb", ccm_base + 0x45d0, 0, &share_count_media);
 	hws[IMX8MP_CLK_MEDIA_AXI_ROOT] = imx_clk_hw_gate2_shared2("media_axi_root_clk", "media_axi", ccm_base + 0x45d0, 0, &share_count_media);
@@ -721,7 +724,7 @@ static int imx8mp_clocks_probe(struct platform_device *pdev)
 	hws[IMX8MP_CLK_MEDIA_CAM2_PIX_ROOT] = imx_clk_hw_gate2_shared2("media_cam2_pix_root_clk", "media_cam2_pix", ccm_base + 0x45d0, 0, &share_count_media);
 	hws[IMX8MP_CLK_MEDIA_DISP1_PIX_ROOT] = imx_clk_hw_gate2_shared2("media_disp1_pix_root_clk", "media_disp1_pix", ccm_base + 0x45d0, 0, &share_count_media);
 	hws[IMX8MP_CLK_MEDIA_DISP2_PIX_ROOT] = imx_clk_hw_gate2_shared2("media_disp2_pix_root_clk", "media_disp2_pix", ccm_base + 0x45d0, 0, &share_count_media);
-	hws[IMX8MP_CLK_MEDIA_ISP_ROOT] = imx_clk_hw_gate2_shared2("media_isp_root_clk", "media_isp_div", ccm_base + 0x45d0, 0, &share_count_media);
+	hws[IMX8MP_CLK_MEDIA_ISP_ROOT] = imx_clk_hw_gate2_shared2("media_isp_root_clk", "media_isp", ccm_base + 0x45d0, 0, &share_count_media);
 
 	hws[IMX8MP_CLK_USDHC3_ROOT] = imx_clk_hw_gate4("usdhc3_root_clk", "usdhc3", ccm_base + 0x45e0, 0);
 	hws[IMX8MP_CLK_HDMI_ROOT] = imx_clk_hw_gate4("hdmi_root_clk", "hdmi_axi", ccm_base + 0x45f0, 0);
@@ -735,9 +738,6 @@ static int imx8mp_clocks_probe(struct platform_device *pdev)
 					     hws[IMX8MP_ARM_PLL_OUT]->clk,
 					     hws[IMX8MP_CLK_A53_DIV]->clk);
 
-	clk_hw_set_parent(hws[IMX8MP_CLK_A53_SRC], hws[IMX8MP_SYS_PLL1_800M]);
-	clk_hw_set_parent(hws[IMX8MP_CLK_A53_CORE], hws[IMX8MP_ARM_PLL_OUT]);
-
 	imx_check_clk_hws(hws, IMX8MP_CLK_END);
 
 	of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c
index fdc68db68de5..a64aace213c2 100644
--- a/drivers/clk/imx/clk-imx8mq.c
+++ b/drivers/clk/imx/clk-imx8mq.c
@@ -405,9 +405,9 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
 		return PTR_ERR(base);
 
 	/* CORE */
-	hws[IMX8MQ_CLK_A53_SRC] = imx_clk_hw_mux2("arm_a53_src", base + 0x8000, 24, 3, imx8mq_a53_sels, ARRAY_SIZE(imx8mq_a53_sels));
-	hws[IMX8MQ_CLK_A53_CG] = imx_clk_hw_gate3_flags("arm_a53_cg", "arm_a53_src", base + 0x8000, 28, CLK_IS_CRITICAL);
-	hws[IMX8MQ_CLK_A53_DIV] = imx_clk_hw_divider2("arm_a53_div", "arm_a53_cg", base + 0x8000, 0, 3);
+	hws[IMX8MQ_CLK_A53_DIV] = imx8m_clk_hw_composite_core("arm_a53_div", imx8mq_a53_sels, base + 0x8000);
+	hws[IMX8MQ_CLK_A53_CG] = hws[IMX8MQ_CLK_A53_DIV];
+	hws[IMX8MQ_CLK_A53_SRC] = hws[IMX8MQ_CLK_A53_DIV];
 
 	hws[IMX8MQ_CLK_M4_CORE] = imx8m_clk_hw_composite_core("arm_m4_core", imx8mq_arm_m4_sels, base + 0x8080);
 	hws[IMX8MQ_CLK_VPU_CORE] = imx8m_clk_hw_composite_core("vpu_core", imx8mq_vpu_sels, base + 0x8100);
@@ -432,22 +432,22 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
 
 	/* BUS */
 	hws[IMX8MQ_CLK_MAIN_AXI] = imx8m_clk_hw_composite_critical("main_axi", imx8mq_main_axi_sels, base + 0x8800);
-	hws[IMX8MQ_CLK_ENET_AXI] = imx8m_clk_hw_composite("enet_axi", imx8mq_enet_axi_sels, base + 0x8880);
-	hws[IMX8MQ_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite("nand_usdhc_bus", imx8mq_nand_usdhc_sels, base + 0x8900);
-	hws[IMX8MQ_CLK_VPU_BUS] = imx8m_clk_hw_composite("vpu_bus", imx8mq_vpu_bus_sels, base + 0x8980);
-	hws[IMX8MQ_CLK_DISP_AXI] = imx8m_clk_hw_composite("disp_axi", imx8mq_disp_axi_sels, base + 0x8a00);
-	hws[IMX8MQ_CLK_DISP_APB] = imx8m_clk_hw_composite("disp_apb", imx8mq_disp_apb_sels, base + 0x8a80);
-	hws[IMX8MQ_CLK_DISP_RTRM] = imx8m_clk_hw_composite("disp_rtrm", imx8mq_disp_rtrm_sels, base + 0x8b00);
-	hws[IMX8MQ_CLK_USB_BUS] = imx8m_clk_hw_composite("usb_bus", imx8mq_usb_bus_sels, base + 0x8b80);
-	hws[IMX8MQ_CLK_GPU_AXI] = imx8m_clk_hw_composite("gpu_axi", imx8mq_gpu_axi_sels, base + 0x8c00);
-	hws[IMX8MQ_CLK_GPU_AHB] = imx8m_clk_hw_composite("gpu_ahb", imx8mq_gpu_ahb_sels, base + 0x8c80);
+	hws[IMX8MQ_CLK_ENET_AXI] = imx8m_clk_hw_composite_bus("enet_axi", imx8mq_enet_axi_sels, base + 0x8880);
+	hws[IMX8MQ_CLK_NAND_USDHC_BUS] = imx8m_clk_hw_composite_bus("nand_usdhc_bus", imx8mq_nand_usdhc_sels, base + 0x8900);
+	hws[IMX8MQ_CLK_VPU_BUS] = imx8m_clk_hw_composite_bus("vpu_bus", imx8mq_vpu_bus_sels, base + 0x8980);
+	hws[IMX8MQ_CLK_DISP_AXI] = imx8m_clk_hw_composite_bus("disp_axi", imx8mq_disp_axi_sels, base + 0x8a00);
+	hws[IMX8MQ_CLK_DISP_APB] = imx8m_clk_hw_composite_bus("disp_apb", imx8mq_disp_apb_sels, base + 0x8a80);
+	hws[IMX8MQ_CLK_DISP_RTRM] = imx8m_clk_hw_composite_bus("disp_rtrm", imx8mq_disp_rtrm_sels, base + 0x8b00);
+	hws[IMX8MQ_CLK_USB_BUS] = imx8m_clk_hw_composite_bus("usb_bus", imx8mq_usb_bus_sels, base + 0x8b80);
+	hws[IMX8MQ_CLK_GPU_AXI] = imx8m_clk_hw_composite_bus("gpu_axi", imx8mq_gpu_axi_sels, base + 0x8c00);
+	hws[IMX8MQ_CLK_GPU_AHB] = imx8m_clk_hw_composite_bus("gpu_ahb", imx8mq_gpu_ahb_sels, base + 0x8c80);
 	hws[IMX8MQ_CLK_NOC] = imx8m_clk_hw_composite_critical("noc", imx8mq_noc_sels, base + 0x8d00);
 	hws[IMX8MQ_CLK_NOC_APB] = imx8m_clk_hw_composite_critical("noc_apb", imx8mq_noc_apb_sels, base + 0x8d80);
 
 	/* AHB */
 	/* AHB clock is used by the AHB bus therefore marked as critical */
 	hws[IMX8MQ_CLK_AHB] = imx8m_clk_hw_composite_critical("ahb", imx8mq_ahb_sels, base + 0x9000);
-	hws[IMX8MQ_CLK_AUDIO_AHB] = imx8m_clk_hw_composite("audio_ahb", imx8mq_audio_ahb_sels, base + 0x9100);
+	hws[IMX8MQ_CLK_AUDIO_AHB] = imx8m_clk_hw_composite_bus("audio_ahb", imx8mq_audio_ahb_sels, base + 0x9100);
 
 	/* IPG */
 	hws[IMX8MQ_CLK_IPG_ROOT] = imx_clk_hw_divider2("ipg_root", "ahb", base + 0x9080, 0, 1);
@@ -599,9 +599,6 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
 					   hws[IMX8MQ_ARM_PLL_OUT]->clk,
 					   hws[IMX8MQ_CLK_A53_DIV]->clk);
 
-	clk_hw_set_parent(hws[IMX8MQ_CLK_A53_SRC], hws[IMX8MQ_SYS1_PLL_800M]);
-	clk_hw_set_parent(hws[IMX8MQ_CLK_A53_CORE], hws[IMX8MQ_ARM_PLL_OUT]);
-
 	imx_check_clk_hws(hws, IMX8MQ_CLK_END);
 
 	err = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_hw_data);
diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c
index a83bbbee77d9..f9eb189b93c0 100644
--- a/drivers/clk/imx/clk-pll14xx.c
+++ b/drivers/clk/imx/clk-pll14xx.c
@@ -378,9 +378,9 @@ static const struct clk_ops clk_pll1443x_ops = {
 	.set_rate	= clk_pll1443x_set_rate,
 };
 
-struct clk_hw *imx_clk_hw_pll14xx(const char *name, const char *parent_name,
-				  void __iomem *base,
-				  const struct imx_pll14xx_clk *pll_clk)
+struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name,
+				const char *parent_name, void __iomem *base,
+				const struct imx_pll14xx_clk *pll_clk)
 {
 	struct clk_pll14xx *pll;
 	struct clk_hw *hw;
@@ -426,7 +426,7 @@ struct clk_hw *imx_clk_hw_pll14xx(const char *name, const char *parent_name,
 
 	hw = &pll->hw;
 
-	ret = clk_hw_register(NULL, hw);
+	ret = clk_hw_register(dev, hw);
 	if (ret) {
 		pr_err("%s: failed to register pll %s %d\n",
 			__func__, name, ret);
diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c
index df91a8244fb4..a7db93030e02 100644
--- a/drivers/clk/imx/clk-pllv3.c
+++ b/drivers/clk/imx/clk-pllv3.c
@@ -7,6 +7,7 @@
 #include <linux/clk-provider.h>
 #include <linux/delay.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/slab.h>
 #include <linux/jiffies.h>
 #include <linux/err.h>
@@ -25,6 +26,8 @@
 #define IMX7_ENET_PLL_POWER	(0x1 << 5)
 #define IMX7_DDR_PLL_POWER	(0x1 << 20)
 
+#define PLL_LOCK_TIMEOUT	10000
+
 /**
  * struct clk_pllv3 - IMX PLL clock version 3
  * @clk_hw:	 clock source
@@ -53,23 +56,14 @@ struct clk_pllv3 {
 
 static int clk_pllv3_wait_lock(struct clk_pllv3 *pll)
 {
-	unsigned long timeout = jiffies + msecs_to_jiffies(10);
 	u32 val = readl_relaxed(pll->base) & pll->power_bit;
 
 	/* No need to wait for lock when pll is not powered up */
 	if ((pll->powerup_set && !val) || (!pll->powerup_set && val))
 		return 0;
 
-	/* Wait for PLL to lock */
-	do {
-		if (readl_relaxed(pll->base) & BM_PLL_LOCK)
-			break;
-		if (time_after(jiffies, timeout))
-			break;
-		usleep_range(50, 500);
-	} while (1);
-
-	return readl_relaxed(pll->base) & BM_PLL_LOCK ? 0 : -ETIMEDOUT;
+	return readl_relaxed_poll_timeout(pll->base, val, val & BM_PLL_LOCK,
+					  500, PLL_LOCK_TIMEOUT);
 }
 
 static int clk_pllv3_prepare(struct clk_hw *hw)
diff --git a/drivers/clk/imx/clk-sscg-pll.c b/drivers/clk/imx/clk-sscg-pll.c
index d4a2be16d132..773d8a545cdf 100644
--- a/drivers/clk/imx/clk-sscg-pll.c
+++ b/drivers/clk/imx/clk-sscg-pll.c
@@ -72,7 +72,6 @@ struct clk_sscg_pll_setup {
 	int divr2, divf2;
 	int divq;
 	int bypass;
-
 	uint64_t vco1;
 	uint64_t vco2;
 	uint64_t fout;
@@ -86,11 +85,8 @@ struct clk_sscg_pll_setup {
 struct clk_sscg_pll {
 	struct clk_hw	hw;
 	const struct clk_ops  ops;
-
 	void __iomem *base;
-
 	struct clk_sscg_pll_setup setup;
-
 	u8 parent;
 	u8 bypass1;
 	u8 bypass2;
@@ -194,7 +190,6 @@ static int clk_sscg_pll2_find_setup(struct clk_sscg_pll_setup *setup,
 					struct clk_sscg_pll_setup *temp_setup,
 					uint64_t ref)
 {
-
 	int ret;
 
 	if (ref < PLL_STAGE1_MIN_FREQ || ref > PLL_STAGE1_MAX_FREQ)
@@ -253,7 +248,6 @@ static int clk_sscg_pll1_find_setup(struct clk_sscg_pll_setup *setup,
 					struct clk_sscg_pll_setup *temp_setup,
 					uint64_t ref)
 {
-
 	int ret;
 
 	if (ref < PLL_REF_MIN_FREQ || ref > PLL_REF_MAX_FREQ)
@@ -280,7 +274,6 @@ static int clk_sscg_pll_find_setup(struct clk_sscg_pll_setup *setup,
 	temp_setup.fout_request = rate;
 
 	switch (try_bypass) {
-
 	case PLL_BYPASS2:
 		if (prate == rate) {
 			setup->bypass = PLL_BYPASS2;
@@ -288,11 +281,9 @@ static int clk_sscg_pll_find_setup(struct clk_sscg_pll_setup *setup,
 			ret = 0;
 		}
 		break;
-
 	case PLL_BYPASS1:
 		ret = clk_sscg_pll2_find_setup(setup, &temp_setup, prate);
 		break;
-
 	case PLL_BYPASS_NONE:
 		ret = clk_sscg_pll1_find_setup(setup, &temp_setup, prate);
 		break;
@@ -301,7 +292,6 @@ static int clk_sscg_pll_find_setup(struct clk_sscg_pll_setup *setup,
 	return ret;
 }
 
-
 static int clk_sscg_pll_is_prepared(struct clk_hw *hw)
 {
 	struct clk_sscg_pll *pll = to_clk_sscg_pll(hw);
diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h
index f074dd8ec42e..16adbc34e05f 100644
--- a/drivers/clk/imx/clk.h
+++ b/drivers/clk/imx/clk.h
@@ -5,6 +5,8 @@
 #include <linux/spinlock.h>
 #include <linux/clk-provider.h>
 
+#define IMX_CLK_GATE2_SINGLE_BIT	1
+
 extern spinlock_t imx_ccm_lock;
 
 void imx_check_clocks(struct clk *clks[], unsigned int count);
@@ -131,9 +133,9 @@ struct clk *imx_clk_pll14xx(const char *name, const char *parent_name,
 #define imx_clk_pll14xx(name, parent_name, base, pll_clk) \
 	to_clk(imx_clk_hw_pll14xx(name, parent_name, base, pll_clk))
 
-struct clk_hw *imx_clk_hw_pll14xx(const char *name, const char *parent_name,
-				  void __iomem *base,
-				  const struct imx_pll14xx_clk *pll_clk);
+struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name,
+				const char *parent_name, void __iomem *base,
+				const struct imx_pll14xx_clk *pll_clk);
 
 struct clk_hw *imx_clk_hw_pllv1(enum imx_pllv1_type type, const char *name,
 		const char *parent, void __iomem *base);
@@ -240,6 +242,13 @@ static inline struct clk *to_clk(struct clk_hw *hw)
 	return hw->clk;
 }
 
+static inline struct clk_hw *imx_clk_hw_pll14xx(const char *name, const char *parent_name,
+				  void __iomem *base,
+				  const struct imx_pll14xx_clk *pll_clk)
+{
+	return imx_dev_clk_hw_pll14xx(NULL, name, parent_name, base, pll_clk);
+}
+
 static inline struct clk_hw *imx_clk_hw_fixed(const char *name, int rate)
 {
 	return clk_hw_register_fixed_rate(NULL, name, NULL, 0, rate);
@@ -310,6 +319,13 @@ static inline struct clk_hw *imx_clk_hw_gate(const char *name, const char *paren
 				    shift, 0, &imx_ccm_lock);
 }
 
+static inline struct clk_hw *imx_dev_clk_hw_gate(struct device *dev, const char *name,
+						const char *parent, void __iomem *reg, u8 shift)
+{
+	return clk_hw_register_gate(dev, name, parent, CLK_SET_RATE_PARENT, reg,
+				    shift, 0, &imx_ccm_lock);
+}
+
 static inline struct clk_hw *imx_clk_hw_gate_dis(const char *name, const char *parent,
 		void __iomem *reg, u8 shift)
 {
@@ -355,6 +371,17 @@ static inline struct clk_hw *imx_clk_hw_gate2_shared2(const char *name,
 				  &imx_ccm_lock, share_count);
 }
 
+static inline struct clk_hw *imx_dev_clk_hw_gate_shared(struct device *dev,
+				const char *name, const char *parent,
+				void __iomem *reg, u8 shift,
+				unsigned int *share_count)
+{
+	return clk_hw_register_gate2(NULL, name, parent, CLK_SET_RATE_PARENT |
+					CLK_OPS_PARENT_ENABLE, reg, shift, 0x3,
+					IMX_CLK_GATE2_SINGLE_BIT,
+					&imx_ccm_lock, share_count);
+}
+
 static inline struct clk *imx_clk_gate2_cgr(const char *name,
 		const char *parent, void __iomem *reg, u8 shift, u8 cgr_val)
 {
@@ -411,6 +438,15 @@ static inline struct clk_hw *imx_clk_hw_mux(const char *name, void __iomem *reg,
 			width, 0, &imx_ccm_lock);
 }
 
+static inline struct clk_hw *imx_dev_clk_hw_mux(struct device *dev,
+			const char *name, void __iomem *reg, u8 shift,
+			u8 width, const char * const *parents, int num_parents)
+{
+	return clk_hw_register_mux(dev, name, parents, num_parents,
+			CLK_SET_RATE_NO_REPARENT | CLK_SET_PARENT_GATE,
+			reg, shift, width, 0, &imx_ccm_lock);
+}
+
 static inline struct clk *imx_clk_mux2(const char *name, void __iomem *reg,
 			u8 shift, u8 width, const char * const *parents,
 			int num_parents)
@@ -473,11 +509,25 @@ static inline struct clk_hw *imx_clk_hw_mux_flags(const char *name,
 				   reg, shift, width, 0, &imx_ccm_lock);
 }
 
+static inline struct clk_hw *imx_dev_clk_hw_mux_flags(struct device *dev,
+						  const char *name,
+						  void __iomem *reg, u8 shift,
+						  u8 width,
+						  const char * const *parents,
+						  int num_parents,
+						  unsigned long flags)
+{
+	return clk_hw_register_mux(dev, name, parents, num_parents,
+				   flags | CLK_SET_RATE_NO_REPARENT,
+				   reg, shift, width, 0, &imx_ccm_lock);
+}
+
 struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name,
 		struct clk *div, struct clk *mux, struct clk *pll,
 		struct clk *step);
 
 #define IMX_COMPOSITE_CORE	BIT(0)
+#define IMX_COMPOSITE_BUS	BIT(1)
 
 struct clk_hw *imx8m_clk_hw_composite_flags(const char *name,
 					    const char * const *parent_names,
@@ -486,6 +536,12 @@ struct clk_hw *imx8m_clk_hw_composite_flags(const char *name,
 					    u32 composite_flags,
 					    unsigned long flags);
 
+#define imx8m_clk_hw_composite_bus(name, parent_names, reg)	\
+	imx8m_clk_hw_composite_flags(name, parent_names, \
+			ARRAY_SIZE(parent_names), reg, \
+			IMX_COMPOSITE_BUS, \
+			CLK_SET_RATE_NO_REPARENT | CLK_OPS_PARENT_ENABLE)
+
 #define imx8m_clk_hw_composite_core(name, parent_names, reg)	\
 	imx8m_clk_hw_composite_flags(name, parent_names, \
 			ARRAY_SIZE(parent_names), reg, \
diff --git a/drivers/clk/ingenic/Kconfig b/drivers/clk/ingenic/Kconfig
index b4555b465ea6..580b0cf69ed5 100644
--- a/drivers/clk/ingenic/Kconfig
+++ b/drivers/clk/ingenic/Kconfig
@@ -55,6 +55,16 @@ config INGENIC_CGU_X1000
 
 	  If building for a X1000 SoC, you want to say Y here.
 
+config INGENIC_CGU_X1830
+	bool "Ingenic X1830 CGU driver"
+	default MACH_X1830
+	select INGENIC_CGU_COMMON
+	help
+	  Support the clocks provided by the CGU hardware on Ingenic X1830
+	  and compatible SoCs.
+
+	  If building for a X1830 SoC, you want to say Y here.
+
 config INGENIC_TCU_CLK
 	bool "Ingenic JZ47xx TCU clocks driver"
 	default MACH_INGENIC
diff --git a/drivers/clk/ingenic/Makefile b/drivers/clk/ingenic/Makefile
index 8b1dad9b74a7..aaa4bffe03c6 100644
--- a/drivers/clk/ingenic/Makefile
+++ b/drivers/clk/ingenic/Makefile
@@ -5,4 +5,5 @@ obj-$(CONFIG_INGENIC_CGU_JZ4725B)	+= jz4725b-cgu.o
 obj-$(CONFIG_INGENIC_CGU_JZ4770)	+= jz4770-cgu.o
 obj-$(CONFIG_INGENIC_CGU_JZ4780)	+= jz4780-cgu.o
 obj-$(CONFIG_INGENIC_CGU_X1000)		+= x1000-cgu.o
+obj-$(CONFIG_INGENIC_CGU_X1830)		+= x1830-cgu.o
 obj-$(CONFIG_INGENIC_TCU_CLK)		+= tcu.o
diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c
index 6e963031cd87..d7981b670221 100644
--- a/drivers/clk/ingenic/cgu.c
+++ b/drivers/clk/ingenic/cgu.c
@@ -76,16 +76,13 @@ ingenic_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	const struct ingenic_cgu_pll_info *pll_info;
 	unsigned m, n, od_enc, od;
 	bool bypass;
-	unsigned long flags;
 	u32 ctl;
 
 	clk_info = &cgu->clock_info[ingenic_clk->idx];
 	BUG_ON(clk_info->type != CGU_CLK_PLL);
 	pll_info = &clk_info->pll;
 
-	spin_lock_irqsave(&cgu->lock, flags);
 	ctl = readl(cgu->base + pll_info->reg);
-	spin_unlock_irqrestore(&cgu->lock, flags);
 
 	m = (ctl >> pll_info->m_shift) & GENMASK(pll_info->m_bits - 1, 0);
 	m += pll_info->m_offset;
@@ -93,6 +90,9 @@ ingenic_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	n += pll_info->n_offset;
 	od_enc = ctl >> pll_info->od_shift;
 	od_enc &= GENMASK(pll_info->od_bits - 1, 0);
+
+	ctl = readl(cgu->base + pll_info->bypass_reg);
+
 	bypass = !pll_info->no_bypass_bit &&
 		 !!(ctl & BIT(pll_info->bypass_bit));
 
@@ -106,7 +106,8 @@ ingenic_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	BUG_ON(od == pll_info->od_max);
 	od++;
 
-	return div_u64((u64)parent_rate * m, n * od);
+	return div_u64((u64)parent_rate * m * pll_info->rate_multiplier,
+		n * od);
 }
 
 static unsigned long
@@ -139,7 +140,8 @@ ingenic_pll_calc(const struct ingenic_cgu_clk_info *clk_info,
 	if (pod)
 		*pod = od;
 
-	return div_u64((u64)parent_rate * m, n * od);
+	return div_u64((u64)parent_rate * m * pll_info->rate_multiplier,
+		n * od);
 }
 
 static inline const struct ingenic_cgu_clk_info *to_clk_info(
@@ -212,9 +214,14 @@ static int ingenic_pll_enable(struct clk_hw *hw)
 	u32 ctl;
 
 	spin_lock_irqsave(&cgu->lock, flags);
-	ctl = readl(cgu->base + pll_info->reg);
+	ctl = readl(cgu->base + pll_info->bypass_reg);
 
 	ctl &= ~BIT(pll_info->bypass_bit);
+
+	writel(ctl, cgu->base + pll_info->bypass_reg);
+
+	ctl = readl(cgu->base + pll_info->reg);
+
 	ctl |= BIT(pll_info->enable_bit);
 
 	writel(ctl, cgu->base + pll_info->reg);
@@ -259,12 +266,9 @@ static int ingenic_pll_is_enabled(struct clk_hw *hw)
 	struct ingenic_cgu *cgu = ingenic_clk->cgu;
 	const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk);
 	const struct ingenic_cgu_pll_info *pll_info = &clk_info->pll;
-	unsigned long flags;
 	u32 ctl;
 
-	spin_lock_irqsave(&cgu->lock, flags);
 	ctl = readl(cgu->base + pll_info->reg);
-	spin_unlock_irqrestore(&cgu->lock, flags);
 
 	return !!(ctl & BIT(pll_info->enable_bit));
 }
@@ -562,16 +566,12 @@ static int ingenic_clk_is_enabled(struct clk_hw *hw)
 	struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw);
 	struct ingenic_cgu *cgu = ingenic_clk->cgu;
 	const struct ingenic_cgu_clk_info *clk_info;
-	unsigned long flags;
 	int enabled = 1;
 
 	clk_info = &cgu->clock_info[ingenic_clk->idx];
 
-	if (clk_info->type & CGU_CLK_GATE) {
-		spin_lock_irqsave(&cgu->lock, flags);
+	if (clk_info->type & CGU_CLK_GATE)
 		enabled = !ingenic_cgu_gate_get(cgu, &clk_info->gate);
-		spin_unlock_irqrestore(&cgu->lock, flags);
-	}
 
 	return enabled;
 }
diff --git a/drivers/clk/ingenic/cgu.h b/drivers/clk/ingenic/cgu.h
index 0dc8004079ee..2c75ef4a36f5 100644
--- a/drivers/clk/ingenic/cgu.h
+++ b/drivers/clk/ingenic/cgu.h
@@ -17,6 +17,7 @@
 /**
  * struct ingenic_cgu_pll_info - information about a PLL
  * @reg: the offset of the PLL's control register within the CGU
+ * @rate_multiplier: the multiplier needed by pll rate calculation
  * @m_shift: the number of bits to shift the multiplier value by (ie. the
  *           index of the lowest bit of the multiplier value in the PLL's
  *           control register)
@@ -37,6 +38,7 @@
  * @od_encoding: a pointer to an array mapping post-VCO divider values to
  *               their encoded values in the PLL control register, or -1 for
  *               unsupported values
+ * @bypass_reg: the offset of the bypass control register within the CGU
  * @bypass_bit: the index of the bypass bit in the PLL control register
  * @enable_bit: the index of the enable bit in the PLL control register
  * @stable_bit: the index of the stable bit in the PLL control register
@@ -44,10 +46,12 @@
  */
 struct ingenic_cgu_pll_info {
 	unsigned reg;
+	unsigned rate_multiplier;
 	const s8 *od_encoding;
 	u8 m_shift, m_bits, m_offset;
 	u8 n_shift, n_bits, n_offset;
 	u8 od_shift, od_bits, od_max;
+	unsigned bypass_reg;
 	u8 bypass_bit;
 	u8 enable_bit;
 	u8 stable_bit;
diff --git a/drivers/clk/ingenic/jz4725b-cgu.c b/drivers/clk/ingenic/jz4725b-cgu.c
index a3b4635f6278..8c38e72d14a7 100644
--- a/drivers/clk/ingenic/jz4725b-cgu.c
+++ b/drivers/clk/ingenic/jz4725b-cgu.c
@@ -9,7 +9,9 @@
 #include <linux/clk-provider.h>
 #include <linux/delay.h>
 #include <linux/of.h>
+
 #include <dt-bindings/clock/jz4725b-cgu.h>
+
 #include "cgu.h"
 #include "pm.h"
 
@@ -54,6 +56,7 @@ static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = {
 		.parents = { JZ4725B_CLK_EXT, -1, -1, -1 },
 		.pll = {
 			.reg = CGU_REG_CPPCR,
+			.rate_multiplier = 1,
 			.m_shift = 23,
 			.m_bits = 9,
 			.m_offset = 2,
@@ -65,6 +68,7 @@ static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = {
 			.od_max = 4,
 			.od_encoding = pll_od_encoding,
 			.stable_bit = 10,
+			.bypass_reg = CGU_REG_CPPCR,
 			.bypass_bit = 9,
 			.enable_bit = 8,
 		},
diff --git a/drivers/clk/ingenic/jz4740-cgu.c b/drivers/clk/ingenic/jz4740-cgu.c
index 4f0e92c877d6..c0ac9196a581 100644
--- a/drivers/clk/ingenic/jz4740-cgu.c
+++ b/drivers/clk/ingenic/jz4740-cgu.c
@@ -10,7 +10,9 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/of.h>
+
 #include <dt-bindings/clock/jz4740-cgu.h>
+
 #include "cgu.h"
 #include "pm.h"
 
@@ -69,6 +71,7 @@ static const struct ingenic_cgu_clk_info jz4740_cgu_clocks[] = {
 		.parents = { JZ4740_CLK_EXT, -1, -1, -1 },
 		.pll = {
 			.reg = CGU_REG_CPPCR,
+			.rate_multiplier = 1,
 			.m_shift = 23,
 			.m_bits = 9,
 			.m_offset = 2,
@@ -80,6 +83,7 @@ static const struct ingenic_cgu_clk_info jz4740_cgu_clocks[] = {
 			.od_max = 4,
 			.od_encoding = pll_od_encoding,
 			.stable_bit = 10,
+			.bypass_reg = CGU_REG_CPPCR,
 			.bypass_bit = 9,
 			.enable_bit = 8,
 		},
diff --git a/drivers/clk/ingenic/jz4770-cgu.c b/drivers/clk/ingenic/jz4770-cgu.c
index c051ecba5cf8..9ea4490ecb7f 100644
--- a/drivers/clk/ingenic/jz4770-cgu.c
+++ b/drivers/clk/ingenic/jz4770-cgu.c
@@ -9,7 +9,9 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/of.h>
+
 #include <dt-bindings/clock/jz4770-cgu.h>
+
 #include "cgu.h"
 #include "pm.h"
 
@@ -102,6 +104,7 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
 		.parents = { JZ4770_CLK_EXT },
 		.pll = {
 			.reg = CGU_REG_CPPCR0,
+			.rate_multiplier = 1,
 			.m_shift = 24,
 			.m_bits = 7,
 			.m_offset = 1,
@@ -112,6 +115,7 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
 			.od_bits = 2,
 			.od_max = 8,
 			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_CPPCR0,
 			.bypass_bit = 9,
 			.enable_bit = 8,
 			.stable_bit = 10,
@@ -124,6 +128,7 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
 		.parents = { JZ4770_CLK_EXT },
 		.pll = {
 			.reg = CGU_REG_CPPCR1,
+			.rate_multiplier = 1,
 			.m_shift = 24,
 			.m_bits = 7,
 			.m_offset = 1,
@@ -134,9 +139,10 @@ static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
 			.od_bits = 2,
 			.od_max = 8,
 			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_CPPCR1,
+			.no_bypass_bit = true,
 			.enable_bit = 7,
 			.stable_bit = 6,
-			.no_bypass_bit = true,
 		},
 	},
 
diff --git a/drivers/clk/ingenic/jz4780-cgu.c b/drivers/clk/ingenic/jz4780-cgu.c
index c758f1643067..6c5b8029cc8a 100644
--- a/drivers/clk/ingenic/jz4780-cgu.c
+++ b/drivers/clk/ingenic/jz4780-cgu.c
@@ -13,6 +13,7 @@
 #include <linux/of.h>
 
 #include <dt-bindings/clock/jz4780-cgu.h>
+
 #include "cgu.h"
 #include "pm.h"
 
@@ -266,6 +267,7 @@ static const struct ingenic_cgu_clk_info jz4780_cgu_clocks[] = {
 
 #define DEF_PLL(name) { \
 	.reg = CGU_REG_ ## name, \
+	.rate_multiplier = 1, \
 	.m_shift = 19, \
 	.m_bits = 13, \
 	.m_offset = 1, \
@@ -277,6 +279,7 @@ static const struct ingenic_cgu_clk_info jz4780_cgu_clocks[] = {
 	.od_max = 16, \
 	.od_encoding = pll_od_encoding, \
 	.stable_bit = 6, \
+	.bypass_reg = CGU_REG_ ## name, \
 	.bypass_bit = 1, \
 	.enable_bit = 0, \
 }
diff --git a/drivers/clk/ingenic/tcu.c b/drivers/clk/ingenic/tcu.c
index 153a954b0d2f..9382dc3aa27e 100644
--- a/drivers/clk/ingenic/tcu.c
+++ b/drivers/clk/ingenic/tcu.c
@@ -323,7 +323,7 @@ static const struct ingenic_soc_info x1000_soc_info = {
 	.has_tcu_clk = false,
 };
 
-static const struct of_device_id ingenic_tcu_of_match[] __initconst = {
+static const struct of_device_id __maybe_unused ingenic_tcu_of_match[] __initconst = {
 	{ .compatible = "ingenic,jz4740-tcu", .data = &jz4740_soc_info, },
 	{ .compatible = "ingenic,jz4725b-tcu", .data = &jz4725b_soc_info, },
 	{ .compatible = "ingenic,jz4770-tcu", .data = &jz4770_soc_info, },
diff --git a/drivers/clk/ingenic/x1000-cgu.c b/drivers/clk/ingenic/x1000-cgu.c
index b22d87b3f555..453f3323cb99 100644
--- a/drivers/clk/ingenic/x1000-cgu.c
+++ b/drivers/clk/ingenic/x1000-cgu.c
@@ -1,13 +1,16 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * X1000 SoC CGU driver
- * Copyright (c) 2019 Zhou Yanjie <zhouyanjie@zoho.com>
+ * Copyright (c) 2019 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
  */
 
 #include <linux/clk-provider.h>
 #include <linux/delay.h>
+#include <linux/io.h>
 #include <linux/of.h>
+
 #include <dt-bindings/clock/x1000-cgu.h>
+
 #include "cgu.h"
 #include "pm.h"
 
@@ -18,6 +21,9 @@
 #define CGU_REG_CLKGR		0x20
 #define CGU_REG_OPCR		0x24
 #define CGU_REG_DDRCDR		0x2c
+#define CGU_REG_USBPCR		0x3c
+#define CGU_REG_USBPCR1		0x48
+#define CGU_REG_USBCDR		0x50
 #define CGU_REG_MACCDR		0x54
 #define CGU_REG_I2SCDR		0x60
 #define CGU_REG_LPCDR		0x64
@@ -38,8 +44,47 @@
 #define OPCR_SPENDN0		BIT(7)
 #define OPCR_SPENDN1		BIT(6)
 
+/* bits within the USBPCR register */
+#define USBPCR_SIDDQ		BIT(21)
+#define USBPCR_OTG_DISABLE	BIT(20)
+
 static struct ingenic_cgu *cgu;
 
+static int x1000_usb_phy_enable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr	= cgu->base + CGU_REG_USBPCR;
+
+	writel(readl(reg_opcr) | OPCR_SPENDN0, reg_opcr);
+	writel(readl(reg_usbpcr) & ~USBPCR_OTG_DISABLE & ~USBPCR_SIDDQ, reg_usbpcr);
+	return 0;
+}
+
+static void x1000_usb_phy_disable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr	= cgu->base + CGU_REG_USBPCR;
+
+	writel(readl(reg_opcr) & ~OPCR_SPENDN0, reg_opcr);
+	writel(readl(reg_usbpcr) | USBPCR_OTG_DISABLE | USBPCR_SIDDQ, reg_usbpcr);
+}
+
+static int x1000_usb_phy_is_enabled(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr	= cgu->base + CGU_REG_USBPCR;
+
+	return (readl(reg_opcr) & OPCR_SPENDN0) &&
+		!(readl(reg_usbpcr) & USBPCR_SIDDQ) &&
+		!(readl(reg_usbpcr) & USBPCR_OTG_DISABLE);
+}
+
+static const struct clk_ops x1000_otg_phy_ops = {
+	.enable		= x1000_usb_phy_enable,
+	.disable	= x1000_usb_phy_disable,
+	.is_enabled	= x1000_usb_phy_is_enabled,
+};
+
 static const s8 pll_od_encoding[8] = {
 	0x0, 0x1, -1, 0x2, -1, -1, -1, 0x3,
 };
@@ -58,6 +103,7 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 		.parents = { X1000_CLK_EXCLK, -1, -1, -1 },
 		.pll = {
 			.reg = CGU_REG_APLL,
+			.rate_multiplier = 1,
 			.m_shift = 24,
 			.m_bits = 7,
 			.m_offset = 1,
@@ -68,6 +114,7 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 			.od_bits = 2,
 			.od_max = 8,
 			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_APLL,
 			.bypass_bit = 9,
 			.enable_bit = 8,
 			.stable_bit = 10,
@@ -79,6 +126,7 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 		.parents = { X1000_CLK_EXCLK, -1, -1, -1 },
 		.pll = {
 			.reg = CGU_REG_MPLL,
+			.rate_multiplier = 1,
 			.m_shift = 24,
 			.m_bits = 7,
 			.m_offset = 1,
@@ -89,12 +137,22 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 			.od_bits = 2,
 			.od_max = 8,
 			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_MPLL,
 			.bypass_bit = 6,
 			.enable_bit = 7,
 			.stable_bit = 0,
 		},
 	},
 
+
+	/* Custom (SoC-specific) OTG PHY */
+
+	[X1000_CLK_OTGPHY] = {
+		"otg_phy", CGU_CLK_CUSTOM,
+		.parents = { -1, -1, X1000_CLK_EXCLK, -1 },
+		.custom = { &x1000_otg_phy_ops },
+	},
+
 	/* Muxes & dividers */
 
 	[X1000_CLK_SCLKA] = {
@@ -110,9 +168,10 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 	},
 
 	[X1000_CLK_CPU] = {
-		"cpu", CGU_CLK_DIV,
+		"cpu", CGU_CLK_DIV | CGU_CLK_GATE,
 		.parents = { X1000_CLK_CPUMUX, -1, -1, -1 },
 		.div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 },
+		.gate = { CGU_REG_CLKGR, 30 },
 	},
 
 	[X1000_CLK_L2CACHE] = {
@@ -141,9 +200,10 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 	},
 
 	[X1000_CLK_PCLK] = {
-		"pclk", CGU_CLK_DIV,
+		"pclk", CGU_CLK_DIV | CGU_CLK_GATE,
 		.parents = { X1000_CLK_AHB2PMUX, -1, -1, -1 },
 		.div = { CGU_REG_CPCCR, 16, 1, 4, 20, -1, -1 },
+		.gate = { CGU_REG_CLKGR, 28 },
 	},
 
 	[X1000_CLK_DDR] = {
@@ -156,12 +216,20 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 
 	[X1000_CLK_MAC] = {
 		"mac", CGU_CLK_MUX | CGU_CLK_DIV | CGU_CLK_GATE,
-		.parents = { X1000_CLK_SCLKA, X1000_CLK_MPLL},
+		.parents = { X1000_CLK_SCLKA, X1000_CLK_MPLL },
 		.mux = { CGU_REG_MACCDR, 31, 1 },
 		.div = { CGU_REG_MACCDR, 0, 1, 8, 29, 28, 27 },
 		.gate = { CGU_REG_CLKGR, 25 },
 	},
 
+	[X1000_CLK_LCD] = {
+		"lcd", CGU_CLK_MUX | CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { X1000_CLK_SCLKA, X1000_CLK_MPLL },
+		.mux = { CGU_REG_LPCDR, 31, 1 },
+		.div = { CGU_REG_LPCDR, 0, 1, 8, 28, 27, 26 },
+		.gate = { CGU_REG_CLKGR, 23 },
+	},
+
 	[X1000_CLK_MSCMUX] = {
 		"msc_mux", CGU_CLK_MUX,
 		.parents = { X1000_CLK_SCLKA, X1000_CLK_MPLL},
@@ -182,6 +250,15 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 		.gate = { CGU_REG_CLKGR, 5 },
 	},
 
+	[X1000_CLK_OTG] = {
+		"otg", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { X1000_CLK_EXCLK, -1,
+					 X1000_CLK_APLL, X1000_CLK_MPLL },
+		.mux = { CGU_REG_USBCDR, 30, 2 },
+		.div = { CGU_REG_USBCDR, 0, 1, 8, 29, 28, 27 },
+		.gate = { CGU_REG_CLKGR, 3 },
+	},
+
 	[X1000_CLK_SSIPLL] = {
 		"ssi_pll", CGU_CLK_MUX | CGU_CLK_DIV,
 		.parents = { X1000_CLK_SCLKA, X1000_CLK_MPLL, -1, -1 },
@@ -189,14 +266,32 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 		.div = { CGU_REG_SSICDR, 0, 1, 8, 29, 28, 27 },
 	},
 
+	[X1000_CLK_SSIPLL_DIV2] = {
+		"ssi_pll_div2", CGU_CLK_FIXDIV,
+		.parents = { X1000_CLK_SSIPLL },
+		.fixdiv = { 2 },
+	},
+
 	[X1000_CLK_SSIMUX] = {
 		"ssi_mux", CGU_CLK_MUX,
-		.parents = { X1000_CLK_EXCLK, X1000_CLK_SSIPLL, -1, -1 },
+		.parents = { X1000_CLK_EXCLK, X1000_CLK_SSIPLL_DIV2, -1, -1 },
 		.mux = { CGU_REG_SSICDR, 30, 1 },
 	},
 
 	/* Gate-only clocks */
 
+	[X1000_CLK_EMC] = {
+		"emc", CGU_CLK_GATE,
+		.parents = { X1000_CLK_AHB2, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR, 0 },
+	},
+
+	[X1000_CLK_EFUSE] = {
+		"efuse", CGU_CLK_GATE,
+		.parents = { X1000_CLK_AHB2, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR, 1 },
+	},
+
 	[X1000_CLK_SFC] = {
 		"sfc", CGU_CLK_GATE,
 		.parents = { X1000_CLK_SSIPLL, -1, -1, -1 },
@@ -239,12 +334,24 @@ static const struct ingenic_cgu_clk_info x1000_cgu_clocks[] = {
 		.gate = { CGU_REG_CLKGR, 16 },
 	},
 
+	[X1000_CLK_TCU] = {
+		"tcu", CGU_CLK_GATE,
+		.parents = { X1000_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR, 18 },
+	},
+
 	[X1000_CLK_SSI] = {
 		"ssi", CGU_CLK_GATE,
 		.parents = { X1000_CLK_SSIMUX, -1, -1, -1 },
 		.gate = { CGU_REG_CLKGR, 19 },
 	},
 
+	[X1000_CLK_OST] = {
+		"ost", CGU_CLK_GATE,
+		.parents = { X1000_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR, 20 },
+	},
+
 	[X1000_CLK_PDMA] = {
 		"pdma", CGU_CLK_GATE,
 		.parents = { X1000_CLK_EXCLK, -1, -1, -1 },
@@ -271,4 +378,8 @@ static void __init x1000_cgu_init(struct device_node *np)
 
 	ingenic_cgu_register_syscore_ops(cgu);
 }
-CLK_OF_DECLARE(x1000_cgu, "ingenic,x1000-cgu", x1000_cgu_init);
+/*
+ * CGU has some children devices, this is useful for probing children devices
+ * in the case where the device node is compatible with "simple-mfd".
+ */
+CLK_OF_DECLARE_DRIVER(x1000_cgu, "ingenic,x1000-cgu", x1000_cgu_init);
diff --git a/drivers/clk/ingenic/x1830-cgu.c b/drivers/clk/ingenic/x1830-cgu.c
new file mode 100644
index 000000000000..a1b2ff0ee487
--- /dev/null
+++ b/drivers/clk/ingenic/x1830-cgu.c
@@ -0,0 +1,448 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * X1830 SoC CGU driver
+ * Copyright (c) 2019 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/of.h>
+
+#include <dt-bindings/clock/x1830-cgu.h>
+
+#include "cgu.h"
+#include "pm.h"
+
+/* CGU register offsets */
+#define CGU_REG_CPCCR		0x00
+#define CGU_REG_CPPCR		0x0c
+#define CGU_REG_APLL		0x10
+#define CGU_REG_MPLL		0x14
+#define CGU_REG_CLKGR0		0x20
+#define CGU_REG_OPCR		0x24
+#define CGU_REG_CLKGR1		0x28
+#define CGU_REG_DDRCDR		0x2c
+#define CGU_REG_USBPCR		0x3c
+#define CGU_REG_USBRDT		0x40
+#define CGU_REG_USBVBFIL	0x44
+#define CGU_REG_USBPCR1		0x48
+#define CGU_REG_MACCDR		0x54
+#define CGU_REG_EPLL		0x58
+#define CGU_REG_I2SCDR		0x60
+#define CGU_REG_LPCDR		0x64
+#define CGU_REG_MSC0CDR		0x68
+#define CGU_REG_I2SCDR1		0x70
+#define CGU_REG_SSICDR		0x74
+#define CGU_REG_CIMCDR		0x7c
+#define CGU_REG_MSC1CDR		0xa4
+#define CGU_REG_CMP_INTR	0xb0
+#define CGU_REG_CMP_INTRE	0xb4
+#define CGU_REG_DRCG		0xd0
+#define CGU_REG_CPCSR		0xd4
+#define CGU_REG_VPLL		0xe0
+#define CGU_REG_MACPHYC		0xe8
+
+/* bits within the OPCR register */
+#define OPCR_GATE_USBPHYCLK	BIT(23)
+#define OPCR_SPENDN0		BIT(7)
+#define OPCR_SPENDN1		BIT(6)
+
+/* bits within the USBPCR register */
+#define USBPCR_SIDDQ		BIT(21)
+#define USBPCR_OTG_DISABLE	BIT(20)
+
+static struct ingenic_cgu *cgu;
+
+static int x1830_usb_phy_enable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr	= cgu->base + CGU_REG_USBPCR;
+
+	writel((readl(reg_opcr) | OPCR_SPENDN0) & ~OPCR_GATE_USBPHYCLK, reg_opcr);
+	writel(readl(reg_usbpcr) & ~USBPCR_OTG_DISABLE & ~USBPCR_SIDDQ, reg_usbpcr);
+	return 0;
+}
+
+static void x1830_usb_phy_disable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr	= cgu->base + CGU_REG_USBPCR;
+
+	writel((readl(reg_opcr) & ~OPCR_SPENDN0) | OPCR_GATE_USBPHYCLK, reg_opcr);
+	writel(readl(reg_usbpcr) | USBPCR_OTG_DISABLE | USBPCR_SIDDQ, reg_usbpcr);
+}
+
+static int x1830_usb_phy_is_enabled(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr	= cgu->base + CGU_REG_USBPCR;
+
+	return (readl(reg_opcr) & OPCR_SPENDN0) &&
+		!(readl(reg_usbpcr) & USBPCR_SIDDQ) &&
+		!(readl(reg_usbpcr) & USBPCR_OTG_DISABLE);
+}
+
+static const struct clk_ops x1830_otg_phy_ops = {
+	.enable		= x1830_usb_phy_enable,
+	.disable	= x1830_usb_phy_disable,
+	.is_enabled	= x1830_usb_phy_is_enabled,
+};
+
+static const s8 pll_od_encoding[64] = {
+	0x0, 0x1,  -1, 0x2,  -1,  -1,  -1, 0x3,
+	 -1,  -1,  -1,  -1,  -1,  -1,  -1, 0x4,
+	 -1,  -1,  -1,  -1,  -1,  -1,  -1,  -1,
+	 -1,  -1,  -1,  -1,  -1,  -1,  -1, 0x5,
+	 -1,  -1,  -1,  -1,  -1,  -1,  -1,  -1,
+	 -1,  -1,  -1,  -1,  -1,  -1,  -1,  -1,
+	 -1,  -1,  -1,  -1,  -1,  -1,  -1,  -1,
+	 -1,  -1,  -1,  -1,  -1,  -1,  -1, 0x6,
+};
+
+static const struct ingenic_cgu_clk_info x1830_cgu_clocks[] = {
+
+	/* External clocks */
+
+	[X1830_CLK_EXCLK] = { "ext", CGU_CLK_EXT },
+	[X1830_CLK_RTCLK] = { "rtc", CGU_CLK_EXT },
+
+	/* PLLs */
+
+	[X1830_CLK_APLL] = {
+		"apll", CGU_CLK_PLL,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.pll = {
+			.reg = CGU_REG_APLL,
+			.rate_multiplier = 2,
+			.m_shift = 20,
+			.m_bits = 9,
+			.m_offset = 1,
+			.n_shift = 14,
+			.n_bits = 6,
+			.n_offset = 1,
+			.od_shift = 11,
+			.od_bits = 3,
+			.od_max = 64,
+			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_CPPCR,
+			.bypass_bit = 30,
+			.enable_bit = 0,
+			.stable_bit = 3,
+		},
+	},
+
+	[X1830_CLK_MPLL] = {
+		"mpll", CGU_CLK_PLL,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.pll = {
+			.reg = CGU_REG_MPLL,
+			.rate_multiplier = 2,
+			.m_shift = 20,
+			.m_bits = 9,
+			.m_offset = 1,
+			.n_shift = 14,
+			.n_bits = 6,
+			.n_offset = 1,
+			.od_shift = 11,
+			.od_bits = 3,
+			.od_max = 64,
+			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_CPPCR,
+			.bypass_bit = 28,
+			.enable_bit = 0,
+			.stable_bit = 3,
+		},
+	},
+
+	[X1830_CLK_EPLL] = {
+		"epll", CGU_CLK_PLL,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.pll = {
+			.reg = CGU_REG_EPLL,
+			.rate_multiplier = 2,
+			.m_shift = 20,
+			.m_bits = 9,
+			.m_offset = 1,
+			.n_shift = 14,
+			.n_bits = 6,
+			.n_offset = 1,
+			.od_shift = 11,
+			.od_bits = 3,
+			.od_max = 64,
+			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_CPPCR,
+			.bypass_bit = 24,
+			.enable_bit = 0,
+			.stable_bit = 3,
+		},
+	},
+
+	[X1830_CLK_VPLL] = {
+		"vpll", CGU_CLK_PLL,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.pll = {
+			.reg = CGU_REG_VPLL,
+			.rate_multiplier = 2,
+			.m_shift = 20,
+			.m_bits = 9,
+			.m_offset = 1,
+			.n_shift = 14,
+			.n_bits = 6,
+			.n_offset = 1,
+			.od_shift = 11,
+			.od_bits = 3,
+			.od_max = 64,
+			.od_encoding = pll_od_encoding,
+			.bypass_reg = CGU_REG_CPPCR,
+			.bypass_bit = 26,
+			.enable_bit = 0,
+			.stable_bit = 3,
+		},
+	},
+
+	/* Custom (SoC-specific) OTG PHY */
+
+	[X1830_CLK_OTGPHY] = {
+		"otg_phy", CGU_CLK_CUSTOM,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.custom = { &x1830_otg_phy_ops },
+	},
+
+	/* Muxes & dividers */
+
+	[X1830_CLK_SCLKA] = {
+		"sclk_a", CGU_CLK_MUX,
+		.parents = { -1, X1830_CLK_EXCLK, X1830_CLK_APLL, -1 },
+		.mux = { CGU_REG_CPCCR, 30, 2 },
+	},
+
+	[X1830_CLK_CPUMUX] = {
+		"cpu_mux", CGU_CLK_MUX,
+		.parents = { -1, X1830_CLK_SCLKA, X1830_CLK_MPLL, -1 },
+		.mux = { CGU_REG_CPCCR, 28, 2 },
+	},
+
+	[X1830_CLK_CPU] = {
+		"cpu", CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { X1830_CLK_CPUMUX, -1, -1, -1 },
+		.div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 },
+		.gate = { CGU_REG_CLKGR1, 15 },
+	},
+
+	[X1830_CLK_L2CACHE] = {
+		"l2cache", CGU_CLK_DIV,
+		.parents = { X1830_CLK_CPUMUX, -1, -1, -1 },
+		.div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 },
+	},
+
+	[X1830_CLK_AHB0] = {
+		"ahb0", CGU_CLK_MUX | CGU_CLK_DIV,
+		.parents = { -1, X1830_CLK_SCLKA, X1830_CLK_MPLL, -1 },
+		.mux = { CGU_REG_CPCCR, 26, 2 },
+		.div = { CGU_REG_CPCCR, 8, 1, 4, 21, -1, -1 },
+	},
+
+	[X1830_CLK_AHB2PMUX] = {
+		"ahb2_apb_mux", CGU_CLK_MUX,
+		.parents = { -1, X1830_CLK_SCLKA, X1830_CLK_MPLL, -1 },
+		.mux = { CGU_REG_CPCCR, 24, 2 },
+	},
+
+	[X1830_CLK_AHB2] = {
+		"ahb2", CGU_CLK_DIV,
+		.parents = { X1830_CLK_AHB2PMUX, -1, -1, -1 },
+		.div = { CGU_REG_CPCCR, 12, 1, 4, 20, -1, -1 },
+	},
+
+	[X1830_CLK_PCLK] = {
+		"pclk", CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { X1830_CLK_AHB2PMUX, -1, -1, -1 },
+		.div = { CGU_REG_CPCCR, 16, 1, 4, 20, -1, -1 },
+		.gate = { CGU_REG_CLKGR1, 14 },
+	},
+
+	[X1830_CLK_DDR] = {
+		"ddr", CGU_CLK_MUX | CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { -1, X1830_CLK_SCLKA, X1830_CLK_MPLL, -1 },
+		.mux = { CGU_REG_DDRCDR, 30, 2 },
+		.div = { CGU_REG_DDRCDR, 0, 1, 4, 29, 28, 27 },
+		.gate = { CGU_REG_CLKGR0, 31 },
+	},
+
+	[X1830_CLK_MAC] = {
+		"mac", CGU_CLK_MUX | CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { X1830_CLK_SCLKA, X1830_CLK_MPLL,
+					 X1830_CLK_VPLL, X1830_CLK_EPLL },
+		.mux = { CGU_REG_MACCDR, 30, 2 },
+		.div = { CGU_REG_MACCDR, 0, 1, 8, 29, 28, 27 },
+		.gate = { CGU_REG_CLKGR1, 4 },
+	},
+
+	[X1830_CLK_LCD] = {
+		"lcd", CGU_CLK_MUX | CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { X1830_CLK_SCLKA, X1830_CLK_MPLL,
+					 X1830_CLK_VPLL, X1830_CLK_EPLL },
+		.mux = { CGU_REG_LPCDR, 30, 2 },
+		.div = { CGU_REG_LPCDR, 0, 1, 8, 28, 27, 26 },
+		.gate = { CGU_REG_CLKGR1, 9 },
+	},
+
+	[X1830_CLK_MSCMUX] = {
+		"msc_mux", CGU_CLK_MUX,
+		.parents = { X1830_CLK_SCLKA, X1830_CLK_MPLL,
+					 X1830_CLK_VPLL, X1830_CLK_EPLL },
+		.mux = { CGU_REG_MSC0CDR, 30, 2 },
+	},
+
+	[X1830_CLK_MSC0] = {
+		"msc0", CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { X1830_CLK_MSCMUX, -1, -1, -1 },
+		.div = { CGU_REG_MSC0CDR, 0, 2, 8, 29, 28, 27 },
+		.gate = { CGU_REG_CLKGR0, 4 },
+	},
+
+	[X1830_CLK_MSC1] = {
+		"msc1", CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { X1830_CLK_MSCMUX, -1, -1, -1 },
+		.div = { CGU_REG_MSC1CDR, 0, 2, 8, 29, 28, 27 },
+		.gate = { CGU_REG_CLKGR0, 5 },
+	},
+
+	[X1830_CLK_SSIPLL] = {
+		"ssi_pll", CGU_CLK_MUX | CGU_CLK_DIV,
+		.parents = { X1830_CLK_SCLKA, X1830_CLK_MPLL,
+					 X1830_CLK_VPLL, X1830_CLK_EPLL },
+		.mux = { CGU_REG_SSICDR, 30, 2 },
+		.div = { CGU_REG_SSICDR, 0, 1, 8, 28, 27, 26 },
+	},
+
+	[X1830_CLK_SSIPLL_DIV2] = {
+		"ssi_pll_div2", CGU_CLK_FIXDIV,
+		.parents = { X1830_CLK_SSIPLL },
+		.fixdiv = { 2 },
+	},
+
+	[X1830_CLK_SSIMUX] = {
+		"ssi_mux", CGU_CLK_MUX,
+		.parents = { X1830_CLK_EXCLK, X1830_CLK_SSIPLL_DIV2, -1, -1 },
+		.mux = { CGU_REG_SSICDR, 29, 1 },
+	},
+
+	/* Gate-only clocks */
+
+	[X1830_CLK_EMC] = {
+		"emc", CGU_CLK_GATE,
+		.parents = { X1830_CLK_AHB2, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 0 },
+	},
+
+	[X1830_CLK_EFUSE] = {
+		"efuse", CGU_CLK_GATE,
+		.parents = { X1830_CLK_AHB2, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 1 },
+	},
+
+	[X1830_CLK_OTG] = {
+		"otg", CGU_CLK_GATE,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 3 },
+	},
+
+	[X1830_CLK_SSI0] = {
+		"ssi0", CGU_CLK_GATE,
+		.parents = { X1830_CLK_SSIMUX, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 6 },
+	},
+
+	[X1830_CLK_SMB0] = {
+		"smb0", CGU_CLK_GATE,
+		.parents = { X1830_CLK_PCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 7 },
+	},
+
+	[X1830_CLK_SMB1] = {
+		"smb1", CGU_CLK_GATE,
+		.parents = { X1830_CLK_PCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 8 },
+	},
+
+	[X1830_CLK_SMB2] = {
+		"smb2", CGU_CLK_GATE,
+		.parents = { X1830_CLK_PCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 9 },
+	},
+
+	[X1830_CLK_UART0] = {
+		"uart0", CGU_CLK_GATE,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 14 },
+	},
+
+	[X1830_CLK_UART1] = {
+		"uart1", CGU_CLK_GATE,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 15 },
+	},
+
+	[X1830_CLK_SSI1] = {
+		"ssi1", CGU_CLK_GATE,
+		.parents = { X1830_CLK_SSIMUX, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 19 },
+	},
+
+	[X1830_CLK_SFC] = {
+		"sfc", CGU_CLK_GATE,
+		.parents = { X1830_CLK_SSIPLL, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 20 },
+	},
+
+	[X1830_CLK_PDMA] = {
+		"pdma", CGU_CLK_GATE,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 21 },
+	},
+
+	[X1830_CLK_TCU] = {
+		"tcu", CGU_CLK_GATE,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 30 },
+	},
+
+	[X1830_CLK_DTRNG] = {
+		"dtrng", CGU_CLK_GATE,
+		.parents = { X1830_CLK_PCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR1, 1 },
+	},
+
+	[X1830_CLK_OST] = {
+		"ost", CGU_CLK_GATE,
+		.parents = { X1830_CLK_EXCLK, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR1, 11 },
+	},
+};
+
+static void __init x1830_cgu_init(struct device_node *np)
+{
+	int retval;
+
+	cgu = ingenic_cgu_new(x1830_cgu_clocks,
+			      ARRAY_SIZE(x1830_cgu_clocks), np);
+	if (!cgu) {
+		pr_err("%s: failed to initialise CGU\n", __func__);
+		return;
+	}
+
+	retval = ingenic_cgu_register_clocks(cgu);
+	if (retval) {
+		pr_err("%s: failed to register CGU Clocks\n", __func__);
+		return;
+	}
+
+	ingenic_cgu_register_syscore_ops(cgu);
+}
+/*
+ * CGU has some children devices, this is useful for probing children devices
+ * in the case where the device node is compatible with "simple-mfd".
+ */
+CLK_OF_DECLARE_DRIVER(x1830_cgu, "ingenic,x1830-cgu", x1830_cgu_init);
diff --git a/drivers/clk/keystone/Kconfig b/drivers/clk/keystone/Kconfig
index ab613f28b502..e64d6726048f 100644
--- a/drivers/clk/keystone/Kconfig
+++ b/drivers/clk/keystone/Kconfig
@@ -2,7 +2,7 @@
 config COMMON_CLK_KEYSTONE
 	tristate "Clock drivers for Keystone based SOCs"
 	depends on (ARCH_KEYSTONE || COMPILE_TEST) && OF
-	---help---
+	help
 	  Supports clock drivers for Keystone based SOCs. These SOCs have local
 	  a power sleep control module that gate the clock to the IPs and PLLs.
 
@@ -11,7 +11,7 @@ config TI_SCI_CLK
 	depends on (ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST) && OF
 	depends on TI_SCI_PROTOCOL
 	default ARCH_KEYSTONE
-	---help---
+	help
 	  This adds the clock driver support over TI System Control Interface.
 	  If you wish to use clock resources from the PMMC firmware, say Y.
 	  Otherwise, say N.
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
index ea3c70d1307e..89ceb2fbc7c4 100644
--- a/drivers/clk/mediatek/Kconfig
+++ b/drivers/clk/mediatek/Kconfig
@@ -8,7 +8,7 @@ menu "Clock driver for MediaTek SoC"
 config COMMON_CLK_MEDIATEK
 	bool
 	select RESET_CONTROLLER
-	---help---
+	help
 	  MediaTek SoCs' clock support.
 
 config COMMON_CLK_MT2701
@@ -16,55 +16,55 @@ config COMMON_CLK_MT2701
 	depends on (ARCH_MEDIATEK && ARM) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM
-	---help---
+	help
 	  This driver supports MediaTek MT2701 basic clocks.
 
 config COMMON_CLK_MT2701_MMSYS
 	bool "Clock driver for MediaTek MT2701 mmsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports MediaTek MT2701 mmsys clocks.
 
 config COMMON_CLK_MT2701_IMGSYS
 	bool "Clock driver for MediaTek MT2701 imgsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports MediaTek MT2701 imgsys clocks.
 
 config COMMON_CLK_MT2701_VDECSYS
 	bool "Clock driver for MediaTek MT2701 vdecsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports MediaTek MT2701 vdecsys clocks.
 
 config COMMON_CLK_MT2701_HIFSYS
 	bool "Clock driver for MediaTek MT2701 hifsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports MediaTek MT2701 hifsys clocks.
 
 config COMMON_CLK_MT2701_ETHSYS
 	bool "Clock driver for MediaTek MT2701 ethsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports MediaTek MT2701 ethsys clocks.
 
 config COMMON_CLK_MT2701_BDPSYS
 	bool "Clock driver for MediaTek MT2701 bdpsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports MediaTek MT2701 bdpsys clocks.
 
 config COMMON_CLK_MT2701_AUDSYS
 	bool "Clock driver for Mediatek MT2701 audsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports Mediatek MT2701 audsys clocks.
 
 config COMMON_CLK_MT2701_G3DSYS
 	bool "Clock driver for MediaTek MT2701 g3dsys"
 	depends on COMMON_CLK_MT2701
-	---help---
+	help
 	  This driver supports MediaTek MT2701 g3dsys clocks.
 
 config COMMON_CLK_MT2712
@@ -72,51 +72,137 @@ config COMMON_CLK_MT2712
 	depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM64
-	---help---
+	help
 	  This driver supports MediaTek MT2712 basic clocks.
 
 config COMMON_CLK_MT2712_BDPSYS
 	bool "Clock driver for MediaTek MT2712 bdpsys"
 	depends on COMMON_CLK_MT2712
-	---help---
+	help
 	  This driver supports MediaTek MT2712 bdpsys clocks.
 
 config COMMON_CLK_MT2712_IMGSYS
 	bool "Clock driver for MediaTek MT2712 imgsys"
 	depends on COMMON_CLK_MT2712
-	---help---
+	help
 	  This driver supports MediaTek MT2712 imgsys clocks.
 
 config COMMON_CLK_MT2712_JPGDECSYS
 	bool "Clock driver for MediaTek MT2712 jpgdecsys"
 	depends on COMMON_CLK_MT2712
-	---help---
+	help
 	  This driver supports MediaTek MT2712 jpgdecsys clocks.
 
 config COMMON_CLK_MT2712_MFGCFG
 	bool "Clock driver for MediaTek MT2712 mfgcfg"
 	depends on COMMON_CLK_MT2712
-	---help---
+	help
 	  This driver supports MediaTek MT2712 mfgcfg clocks.
 
 config COMMON_CLK_MT2712_MMSYS
 	bool "Clock driver for MediaTek MT2712 mmsys"
 	depends on COMMON_CLK_MT2712
-	---help---
+	help
 	  This driver supports MediaTek MT2712 mmsys clocks.
 
 config COMMON_CLK_MT2712_VDECSYS
 	bool "Clock driver for MediaTek MT2712 vdecsys"
 	depends on COMMON_CLK_MT2712
-	---help---
+	help
 	  This driver supports MediaTek MT2712 vdecsys clocks.
 
 config COMMON_CLK_MT2712_VENCSYS
 	bool "Clock driver for MediaTek MT2712 vencsys"
 	depends on COMMON_CLK_MT2712
-	---help---
+	help
 	  This driver supports MediaTek MT2712 vencsys clocks.
 
+config COMMON_CLK_MT6765
+       bool "Clock driver for MediaTek MT6765"
+       depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
+       select COMMON_CLK_MEDIATEK
+       default ARCH_MEDIATEK && ARM64
+       help
+         This driver supports MediaTek MT6765 basic clocks.
+
+config COMMON_CLK_MT6765_AUDIOSYS
+       bool "Clock driver for MediaTek MT6765 audiosys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 audiosys clocks.
+
+config COMMON_CLK_MT6765_CAMSYS
+       bool "Clock driver for MediaTek MT6765 camsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 camsys clocks.
+
+config COMMON_CLK_MT6765_GCESYS
+       bool "Clock driver for MediaTek MT6765 gcesys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 gcesys clocks.
+
+config COMMON_CLK_MT6765_MMSYS
+       bool "Clock driver for MediaTek MT6765 mmsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 mmsys clocks.
+
+config COMMON_CLK_MT6765_IMGSYS
+       bool "Clock driver for MediaTek MT6765 imgsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 imgsys clocks.
+
+config COMMON_CLK_MT6765_VCODECSYS
+       bool "Clock driver for MediaTek MT6765 vcodecsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 vcodecsys clocks.
+
+config COMMON_CLK_MT6765_MFGSYS
+       bool "Clock driver for MediaTek MT6765 mfgsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 mfgsys clocks.
+
+config COMMON_CLK_MT6765_MIPI0ASYS
+       bool "Clock driver for MediaTek MT6765 mipi0asys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 mipi0asys clocks.
+
+config COMMON_CLK_MT6765_MIPI0BSYS
+       bool "Clock driver for MediaTek MT6765 mipi0bsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 mipi0bsys clocks.
+
+config COMMON_CLK_MT6765_MIPI1ASYS
+       bool "Clock driver for MediaTek MT6765 mipi1asys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 mipi1asys clocks.
+
+config COMMON_CLK_MT6765_MIPI1BSYS
+       bool "Clock driver for MediaTek MT6765 mipi1bsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 mipi1bsys clocks.
+
+config COMMON_CLK_MT6765_MIPI2ASYS
+       bool "Clock driver for MediaTek MT6765 mipi2asys"
+       depends on COMMON_CLK_MT6765
+      help
+         This driver supports MediaTek MT6765 mipi2asys clocks.
+
+config COMMON_CLK_MT6765_MIPI2BSYS
+       bool "Clock driver for MediaTek MT6765 mipi2bsys"
+       depends on COMMON_CLK_MT6765
+       help
+         This driver supports MediaTek MT6765 mipi2bsys clocks.
+
 config COMMON_CLK_MT6779
 	bool "Clock driver for MediaTek MT6779"
 	depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
@@ -178,31 +264,31 @@ config COMMON_CLK_MT6797
 	depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM64
-	---help---
+	help
 	  This driver supports MediaTek MT6797 basic clocks.
 
 config COMMON_CLK_MT6797_MMSYS
 	bool "Clock driver for MediaTek MT6797 mmsys"
 	depends on COMMON_CLK_MT6797
-	---help---
+	help
 	  This driver supports MediaTek MT6797 mmsys clocks.
 
 config COMMON_CLK_MT6797_IMGSYS
 	bool "Clock driver for MediaTek MT6797 imgsys"
 	depends on COMMON_CLK_MT6797
-	---help---
+	help
 	  This driver supports MediaTek MT6797 imgsys clocks.
 
 config COMMON_CLK_MT6797_VDECSYS
 	bool "Clock driver for MediaTek MT6797 vdecsys"
 	depends on COMMON_CLK_MT6797
-	---help---
+	help
 	  This driver supports MediaTek MT6797 vdecsys clocks.
 
 config COMMON_CLK_MT6797_VENCSYS
 	bool "Clock driver for MediaTek MT6797 vencsys"
 	depends on COMMON_CLK_MT6797
-	---help---
+	help
 	  This driver supports MediaTek MT6797 vencsys clocks.
 
 config COMMON_CLK_MT7622
@@ -210,28 +296,28 @@ config COMMON_CLK_MT7622
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK
-	---help---
+	help
 	  This driver supports MediaTek MT7622 basic clocks and clocks
 	  required for various periperals found on MediaTek.
 
 config COMMON_CLK_MT7622_ETHSYS
 	bool "Clock driver for MediaTek MT7622 ETHSYS"
 	depends on COMMON_CLK_MT7622
-	---help---
+	help
 	  This driver add support for clocks for Ethernet and SGMII
 	  required on MediaTek MT7622 SoC.
 
 config COMMON_CLK_MT7622_HIFSYS
 	bool "Clock driver for MediaTek MT7622 HIFSYS"
 	depends on COMMON_CLK_MT7622
-	---help---
+	help
 	  This driver supports MediaTek MT7622 HIFSYS clocks providing
 	  to PCI-E and USB.
 
 config COMMON_CLK_MT7622_AUDSYS
 	bool "Clock driver for MediaTek MT7622 AUDSYS"
 	depends on COMMON_CLK_MT7622
-	---help---
+	help
 	  This driver supports MediaTek MT7622 AUDSYS clocks providing
 	  to audio consumers such as I2S and TDM.
 
@@ -240,21 +326,21 @@ config COMMON_CLK_MT7629
 	depends on (ARCH_MEDIATEK && ARM) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM
-	---help---
+	help
 	  This driver supports MediaTek MT7629 basic clocks and clocks
 	  required for various periperals found on MediaTek.
 
 config COMMON_CLK_MT7629_ETHSYS
 	bool "Clock driver for MediaTek MT7629 ETHSYS"
 	depends on COMMON_CLK_MT7629
-	---help---
+	help
 	  This driver add support for clocks for Ethernet and SGMII
 	  required on MediaTek MT7629 SoC.
 
 config COMMON_CLK_MT7629_HIFSYS
 	bool "Clock driver for MediaTek MT7629 HIFSYS"
 	depends on COMMON_CLK_MT7629
-	---help---
+	help
 	  This driver supports MediaTek MT7629 HIFSYS clocks providing
 	  to PCI-E and USB.
 
@@ -263,7 +349,7 @@ config COMMON_CLK_MT8135
 	depends on (ARCH_MEDIATEK && ARM) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM
-	---help---
+	help
 	  This driver supports MediaTek MT8135 clocks.
 
 config COMMON_CLK_MT8173
@@ -271,9 +357,16 @@ config COMMON_CLK_MT8173
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK
-	---help---
+	help
 	  This driver supports MediaTek MT8173 clocks.
 
+config COMMON_CLK_MT8173_MMSYS
+	bool "Clock driver for MediaTek MT8173 mmsys"
+	depends on COMMON_CLK_MT8173
+	default COMMON_CLK_MT8173
+	help
+	  This driver supports MediaTek MT8173 mmsys clocks.
+
 config COMMON_CLK_MT8183
 	bool "Clock driver for MediaTek MT8183"
 	depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile
index 8cdb76a5cd71..959b556d32ea 100644
--- a/drivers/clk/mediatek/Makefile
+++ b/drivers/clk/mediatek/Makefile
@@ -1,6 +1,13 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_COMMON_CLK_MEDIATEK) += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o clk-cpumux.o reset.o clk-mux.o
 
+obj-$(CONFIG_COMMON_CLK_MT6765) += clk-mt6765.o
+obj-$(CONFIG_COMMON_CLK_MT6765_AUDIOSYS) += clk-mt6765-audio.o
+obj-$(CONFIG_COMMON_CLK_MT6765_CAMSYS) += clk-mt6765-cam.o
+obj-$(CONFIG_COMMON_CLK_MT6765_IMGSYS) += clk-mt6765-img.o
+obj-$(CONFIG_COMMON_CLK_MT6765_MIPI0ASYS) += clk-mt6765-mipi0a.o
+obj-$(CONFIG_COMMON_CLK_MT6765_MMSYS) += clk-mt6765-mm.o
+obj-$(CONFIG_COMMON_CLK_MT6765_VCODECSYS) += clk-mt6765-vcodec.o
 obj-$(CONFIG_COMMON_CLK_MT6779) += clk-mt6779.o
 obj-$(CONFIG_COMMON_CLK_MT6779_MMSYS) += clk-mt6779-mm.o
 obj-$(CONFIG_COMMON_CLK_MT6779_IMGSYS) += clk-mt6779-img.o
@@ -41,6 +48,7 @@ obj-$(CONFIG_COMMON_CLK_MT7629_ETHSYS) += clk-mt7629-eth.o
 obj-$(CONFIG_COMMON_CLK_MT7629_HIFSYS) += clk-mt7629-hif.o
 obj-$(CONFIG_COMMON_CLK_MT8135) += clk-mt8135.o
 obj-$(CONFIG_COMMON_CLK_MT8173) += clk-mt8173.o
+obj-$(CONFIG_COMMON_CLK_MT8173_MMSYS) += clk-mt8173-mm.o
 obj-$(CONFIG_COMMON_CLK_MT8183) += clk-mt8183.o
 obj-$(CONFIG_COMMON_CLK_MT8183_AUDIOSYS) += clk-mt8183-audio.o
 obj-$(CONFIG_COMMON_CLK_MT8183_CAMSYS) += clk-mt8183-cam.o
diff --git a/drivers/clk/mediatek/clk-mt2701-mm.c b/drivers/clk/mediatek/clk-mt2701-mm.c
index 054b597d4a73..cb18e1849492 100644
--- a/drivers/clk/mediatek/clk-mt2701-mm.c
+++ b/drivers/clk/mediatek/clk-mt2701-mm.c
@@ -79,16 +79,12 @@ static const struct mtk_gate mm_clks[] = {
 	GATE_DISP1(CLK_MM_TVE_FMM, "mm_tve_fmm", "mm_sel", 14),
 };
 
-static const struct of_device_id of_match_clk_mt2701_mm[] = {
-	{ .compatible = "mediatek,mt2701-mmsys", },
-	{}
-};
-
 static int clk_mt2701_mm_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->parent->of_node;
 	struct clk_onecell_data *clk_data;
 	int r;
-	struct device_node *node = pdev->dev.of_node;
 
 	clk_data = mtk_alloc_clk_data(CLK_MM_NR);
 
@@ -108,7 +104,6 @@ static struct platform_driver clk_mt2701_mm_drv = {
 	.probe = clk_mt2701_mm_probe,
 	.driver = {
 		.name = "clk-mt2701-mm",
-		.of_match_table = of_match_clk_mt2701_mm,
 	},
 };
 
diff --git a/drivers/clk/mediatek/clk-mt2712-mm.c b/drivers/clk/mediatek/clk-mt2712-mm.c
index 1c5948be35f3..5519c3d68c1f 100644
--- a/drivers/clk/mediatek/clk-mt2712-mm.c
+++ b/drivers/clk/mediatek/clk-mt2712-mm.c
@@ -128,9 +128,10 @@ static const struct mtk_gate mm_clks[] = {
 
 static int clk_mt2712_mm_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->parent->of_node;
 	struct clk_onecell_data *clk_data;
 	int r;
-	struct device_node *node = pdev->dev.of_node;
 
 	clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK);
 
@@ -146,16 +147,10 @@ static int clk_mt2712_mm_probe(struct platform_device *pdev)
 	return r;
 }
 
-static const struct of_device_id of_match_clk_mt2712_mm[] = {
-	{ .compatible = "mediatek,mt2712-mmsys", },
-	{}
-};
-
 static struct platform_driver clk_mt2712_mm_drv = {
 	.probe = clk_mt2712_mm_probe,
 	.driver = {
 		.name = "clk-mt2712-mm",
-		.of_match_table = of_match_clk_mt2712_mm,
 	},
 };
 
diff --git a/drivers/clk/mediatek/clk-mt6765-audio.c b/drivers/clk/mediatek/clk-mt6765-audio.c
new file mode 100644
index 000000000000..4c989165d795
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt6765-audio.c
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt6765-clk.h>
+
+static const struct mtk_gate_regs audio0_cg_regs = {
+	.set_ofs = 0x0,
+	.clr_ofs = 0x0,
+	.sta_ofs = 0x0,
+};
+
+static const struct mtk_gate_regs audio1_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x4,
+	.sta_ofs = 0x4,
+};
+
+#define GATE_AUDIO0(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &audio0_cg_regs,		\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr,	\
+	}
+
+#define GATE_AUDIO1(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &audio1_cg_regs,		\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr,	\
+	}
+
+static const struct mtk_gate audio_clks[] = {
+	/* AUDIO0 */
+	GATE_AUDIO0(CLK_AUDIO_AFE, "aud_afe", "audio_ck", 2),
+	GATE_AUDIO0(CLK_AUDIO_22M, "aud_22m", "aud_engen1_ck", 8),
+	GATE_AUDIO0(CLK_AUDIO_APLL_TUNER, "aud_apll_tuner",
+		    "aud_engen1_ck", 19),
+	GATE_AUDIO0(CLK_AUDIO_ADC, "aud_adc", "audio_ck", 24),
+	GATE_AUDIO0(CLK_AUDIO_DAC, "aud_dac", "audio_ck", 25),
+	GATE_AUDIO0(CLK_AUDIO_DAC_PREDIS, "aud_dac_predis",
+		    "audio_ck", 26),
+	GATE_AUDIO0(CLK_AUDIO_TML, "aud_tml", "audio_ck", 27),
+	/* AUDIO1 */
+	GATE_AUDIO1(CLK_AUDIO_I2S1_BCLK, "aud_i2s1_bclk",
+		    "audio_ck", 4),
+	GATE_AUDIO1(CLK_AUDIO_I2S2_BCLK, "aud_i2s2_bclk",
+		    "audio_ck", 5),
+	GATE_AUDIO1(CLK_AUDIO_I2S3_BCLK, "aud_i2s3_bclk",
+		    "audio_ck", 6),
+	GATE_AUDIO1(CLK_AUDIO_I2S4_BCLK, "aud_i2s4_bclk",
+		    "audio_ck", 7),
+};
+
+static int clk_mt6765_audio_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_AUDIO_NR_CLK);
+
+	mtk_clk_register_gates(node, audio_clks,
+			       ARRAY_SIZE(audio_clks), clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt6765_audio[] = {
+	{ .compatible = "mediatek,mt6765-audsys", },
+	{}
+};
+
+static struct platform_driver clk_mt6765_audio_drv = {
+	.probe = clk_mt6765_audio_probe,
+	.driver = {
+		.name = "clk-mt6765-audio",
+		.of_match_table = of_match_clk_mt6765_audio,
+	},
+};
+
+builtin_platform_driver(clk_mt6765_audio_drv);
diff --git a/drivers/clk/mediatek/clk-mt6765-cam.c b/drivers/clk/mediatek/clk-mt6765-cam.c
new file mode 100644
index 000000000000..c96394893bcf
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt6765-cam.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt6765-clk.h>
+
+static const struct mtk_gate_regs cam_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_CAM(_id, _name, _parent, _shift) {		\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &cam_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate cam_clks[] = {
+	GATE_CAM(CLK_CAM_LARB3, "cam_larb3", "mm_ck", 0),
+	GATE_CAM(CLK_CAM_DFP_VAD, "cam_dfp_vad", "mm_ck", 1),
+	GATE_CAM(CLK_CAM, "cam", "mm_ck", 6),
+	GATE_CAM(CLK_CAMTG, "camtg", "mm_ck", 7),
+	GATE_CAM(CLK_CAM_SENINF, "cam_seninf", "mm_ck", 8),
+	GATE_CAM(CLK_CAMSV0, "camsv0", "mm_ck", 9),
+	GATE_CAM(CLK_CAMSV1, "camsv1", "mm_ck", 10),
+	GATE_CAM(CLK_CAMSV2, "camsv2", "mm_ck", 11),
+	GATE_CAM(CLK_CAM_CCU, "cam_ccu", "mm_ck", 12),
+};
+
+static int clk_mt6765_cam_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_CAM_NR_CLK);
+
+	mtk_clk_register_gates(node, cam_clks, ARRAY_SIZE(cam_clks), clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt6765_cam[] = {
+	{ .compatible = "mediatek,mt6765-camsys", },
+	{}
+};
+
+static struct platform_driver clk_mt6765_cam_drv = {
+	.probe = clk_mt6765_cam_probe,
+	.driver = {
+		.name = "clk-mt6765-cam",
+		.of_match_table = of_match_clk_mt6765_cam,
+	},
+};
+
+builtin_platform_driver(clk_mt6765_cam_drv);
diff --git a/drivers/clk/mediatek/clk-mt6765-img.c b/drivers/clk/mediatek/clk-mt6765-img.c
new file mode 100644
index 000000000000..6fd8bf8030fc
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt6765-img.c
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt6765-clk.h>
+
+static const struct mtk_gate_regs img_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_IMG(_id, _name, _parent, _shift) {		\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &img_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate img_clks[] = {
+	GATE_IMG(CLK_IMG_LARB2, "img_larb2", "mm_ck", 0),
+	GATE_IMG(CLK_IMG_DIP, "img_dip", "mm_ck", 2),
+	GATE_IMG(CLK_IMG_FDVT, "img_fdvt", "mm_ck", 3),
+	GATE_IMG(CLK_IMG_DPE, "img_dpe", "mm_ck", 4),
+	GATE_IMG(CLK_IMG_RSC, "img_rsc", "mm_ck", 5),
+};
+
+static int clk_mt6765_img_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_IMG_NR_CLK);
+
+	mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt6765_img[] = {
+	{ .compatible = "mediatek,mt6765-imgsys", },
+	{}
+};
+
+static struct platform_driver clk_mt6765_img_drv = {
+	.probe = clk_mt6765_img_probe,
+	.driver = {
+		.name = "clk-mt6765-img",
+		.of_match_table = of_match_clk_mt6765_img,
+	},
+};
+
+builtin_platform_driver(clk_mt6765_img_drv);
diff --git a/drivers/clk/mediatek/clk-mt6765-mipi0a.c b/drivers/clk/mediatek/clk-mt6765-mipi0a.c
new file mode 100644
index 000000000000..81744d0f95a0
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt6765-mipi0a.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt6765-clk.h>
+
+static const struct mtk_gate_regs mipi0a_cg_regs = {
+	.set_ofs = 0x80,
+	.clr_ofs = 0x80,
+	.sta_ofs = 0x80,
+};
+
+#define GATE_MIPI0A(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &mipi0a_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr_inv,	\
+	}
+
+static const struct mtk_gate mipi0a_clks[] = {
+	GATE_MIPI0A(CLK_MIPI0A_CSR_CSI_EN_0A,
+		    "mipi0a_csr_0a", "f_fseninf_ck", 1),
+};
+
+static int clk_mt6765_mipi0a_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_MIPI0A_NR_CLK);
+
+	mtk_clk_register_gates(node, mipi0a_clks,
+			       ARRAY_SIZE(mipi0a_clks), clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt6765_mipi0a[] = {
+	{ .compatible = "mediatek,mt6765-mipi0a", },
+	{}
+};
+
+static struct platform_driver clk_mt6765_mipi0a_drv = {
+	.probe = clk_mt6765_mipi0a_probe,
+	.driver = {
+		.name = "clk-mt6765-mipi0a",
+		.of_match_table = of_match_clk_mt6765_mipi0a,
+	},
+};
+
+builtin_platform_driver(clk_mt6765_mipi0a_drv);
diff --git a/drivers/clk/mediatek/clk-mt6765-mm.c b/drivers/clk/mediatek/clk-mt6765-mm.c
new file mode 100644
index 000000000000..6d8214c51684
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt6765-mm.c
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt6765-clk.h>
+
+static const struct mtk_gate_regs mm_cg_regs = {
+	.set_ofs = 0x104,
+	.clr_ofs = 0x108,
+	.sta_ofs = 0x100,
+};
+
+#define GATE_MM(_id, _name, _parent, _shift) {		\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &mm_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate mm_clks[] = {
+	/* MM */
+	GATE_MM(CLK_MM_MDP_RDMA0, "mm_mdp_rdma0", "mm_ck", 0),
+	GATE_MM(CLK_MM_MDP_CCORR0, "mm_mdp_ccorr0", "mm_ck", 1),
+	GATE_MM(CLK_MM_MDP_RSZ0, "mm_mdp_rsz0", "mm_ck", 2),
+	GATE_MM(CLK_MM_MDP_RSZ1, "mm_mdp_rsz1", "mm_ck", 3),
+	GATE_MM(CLK_MM_MDP_TDSHP0, "mm_mdp_tdshp0", "mm_ck", 4),
+	GATE_MM(CLK_MM_MDP_WROT0, "mm_mdp_wrot0", "mm_ck", 5),
+	GATE_MM(CLK_MM_MDP_WDMA0, "mm_mdp_wdma0", "mm_ck", 6),
+	GATE_MM(CLK_MM_DISP_OVL0, "mm_disp_ovl0", "mm_ck", 7),
+	GATE_MM(CLK_MM_DISP_OVL0_2L, "mm_disp_ovl0_2l", "mm_ck", 8),
+	GATE_MM(CLK_MM_DISP_RSZ0, "mm_disp_rsz0", "mm_ck", 9),
+	GATE_MM(CLK_MM_DISP_RDMA0, "mm_disp_rdma0", "mm_ck", 10),
+	GATE_MM(CLK_MM_DISP_WDMA0, "mm_disp_wdma0", "mm_ck", 11),
+	GATE_MM(CLK_MM_DISP_COLOR0, "mm_disp_color0", "mm_ck", 12),
+	GATE_MM(CLK_MM_DISP_CCORR0, "mm_disp_ccorr0", "mm_ck", 13),
+	GATE_MM(CLK_MM_DISP_AAL0, "mm_disp_aal0", "mm_ck", 14),
+	GATE_MM(CLK_MM_DISP_GAMMA0, "mm_disp_gamma0", "mm_ck", 15),
+	GATE_MM(CLK_MM_DISP_DITHER0, "mm_disp_dither0", "mm_ck", 16),
+	GATE_MM(CLK_MM_DSI0, "mm_dsi0", "mm_ck", 17),
+	GATE_MM(CLK_MM_FAKE_ENG, "mm_fake_eng", "mm_ck", 18),
+	GATE_MM(CLK_MM_SMI_COMMON, "mm_smi_common", "mm_ck", 19),
+	GATE_MM(CLK_MM_SMI_LARB0, "mm_smi_larb0", "mm_ck", 20),
+	GATE_MM(CLK_MM_SMI_COMM0, "mm_smi_comm0", "mm_ck", 21),
+	GATE_MM(CLK_MM_SMI_COMM1, "mm_smi_comm1", "mm_ck", 22),
+	GATE_MM(CLK_MM_CAM_MDP, "mm_cam_mdp_ck", "mm_ck", 23),
+	GATE_MM(CLK_MM_SMI_IMG, "mm_smi_img_ck", "mm_ck", 24),
+	GATE_MM(CLK_MM_SMI_CAM, "mm_smi_cam_ck", "mm_ck", 25),
+	GATE_MM(CLK_MM_IMG_DL_RELAY, "mm_img_dl_relay", "mm_ck", 26),
+	GATE_MM(CLK_MM_IMG_DL_ASYNC_TOP, "mm_imgdl_async", "mm_ck", 27),
+	GATE_MM(CLK_MM_DIG_DSI, "mm_dig_dsi_ck", "mm_ck", 28),
+	GATE_MM(CLK_MM_F26M_HRTWT, "mm_hrtwt", "f_f26m_ck", 29),
+};
+
+static int clk_mt6765_mm_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK);
+
+	mtk_clk_register_gates(node, mm_clks, ARRAY_SIZE(mm_clks), clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt6765_mm[] = {
+	{ .compatible = "mediatek,mt6765-mmsys", },
+	{}
+};
+
+static struct platform_driver clk_mt6765_mm_drv = {
+	.probe = clk_mt6765_mm_probe,
+	.driver = {
+		.name = "clk-mt6765-mm",
+		.of_match_table = of_match_clk_mt6765_mm,
+	},
+};
+
+builtin_platform_driver(clk_mt6765_mm_drv);
diff --git a/drivers/clk/mediatek/clk-mt6765-vcodec.c b/drivers/clk/mediatek/clk-mt6765-vcodec.c
new file mode 100644
index 000000000000..baae665fab31
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt6765-vcodec.c
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt6765-clk.h>
+
+static const struct mtk_gate_regs venc_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_VENC(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &venc_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr_inv,	\
+	}
+
+static const struct mtk_gate venc_clks[] = {
+	GATE_VENC(CLK_VENC_SET0_LARB, "venc_set0_larb", "mm_ck", 0),
+	GATE_VENC(CLK_VENC_SET1_VENC, "venc_set1_venc", "mm_ck", 4),
+	GATE_VENC(CLK_VENC_SET2_JPGENC, "jpgenc", "mm_ck", 8),
+	GATE_VENC(CLK_VENC_SET3_VDEC, "venc_set3_vdec", "mm_ck", 12),
+};
+
+static int clk_mt6765_vcodec_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_VENC_NR_CLK);
+
+	mtk_clk_register_gates(node, venc_clks,
+			       ARRAY_SIZE(venc_clks), clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt6765_vcodec[] = {
+	{ .compatible = "mediatek,mt6765-vcodecsys", },
+	{}
+};
+
+static struct platform_driver clk_mt6765_vcodec_drv = {
+	.probe = clk_mt6765_vcodec_probe,
+	.driver = {
+		.name = "clk-mt6765-vcodec",
+		.of_match_table = of_match_clk_mt6765_vcodec,
+	},
+};
+
+builtin_platform_driver(clk_mt6765_vcodec_drv);
diff --git a/drivers/clk/mediatek/clk-mt6765.c b/drivers/clk/mediatek/clk-mt6765.c
new file mode 100644
index 000000000000..db8db1b3b79d
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt6765.c
@@ -0,0 +1,922 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+#include "clk-mux.h"
+
+#include <dt-bindings/clock/mt6765-clk.h>
+
+/*fmeter div select 4*/
+#define _DIV4_ 1
+
+static DEFINE_SPINLOCK(mt6765_clk_lock);
+
+/* Total 12 subsys */
+static void __iomem *cksys_base;
+static void __iomem *apmixed_base;
+
+/* CKSYS */
+#define CLK_SCP_CFG_0		(cksys_base + 0x200)
+#define CLK_SCP_CFG_1		(cksys_base + 0x204)
+
+/* CG */
+#define AP_PLL_CON3		(apmixed_base + 0x0C)
+#define PLLON_CON0		(apmixed_base + 0x44)
+#define PLLON_CON1		(apmixed_base + 0x48)
+
+/* clk cfg update */
+#define CLK_CFG_0		0x40
+#define CLK_CFG_0_SET		0x44
+#define CLK_CFG_0_CLR		0x48
+#define CLK_CFG_1		0x50
+#define CLK_CFG_1_SET		0x54
+#define CLK_CFG_1_CLR		0x58
+#define CLK_CFG_2		0x60
+#define CLK_CFG_2_SET		0x64
+#define CLK_CFG_2_CLR		0x68
+#define CLK_CFG_3		0x70
+#define CLK_CFG_3_SET		0x74
+#define CLK_CFG_3_CLR		0x78
+#define CLK_CFG_4		0x80
+#define CLK_CFG_4_SET		0x84
+#define CLK_CFG_4_CLR		0x88
+#define CLK_CFG_5		0x90
+#define CLK_CFG_5_SET		0x94
+#define CLK_CFG_5_CLR		0x98
+#define CLK_CFG_6		0xa0
+#define CLK_CFG_6_SET		0xa4
+#define CLK_CFG_6_CLR		0xa8
+#define CLK_CFG_7		0xb0
+#define CLK_CFG_7_SET		0xb4
+#define CLK_CFG_7_CLR		0xb8
+#define CLK_CFG_8		0xc0
+#define CLK_CFG_8_SET		0xc4
+#define CLK_CFG_8_CLR		0xc8
+#define CLK_CFG_9		0xd0
+#define CLK_CFG_9_SET		0xd4
+#define CLK_CFG_9_CLR		0xd8
+#define CLK_CFG_10		0xe0
+#define CLK_CFG_10_SET		0xe4
+#define CLK_CFG_10_CLR		0xe8
+#define CLK_CFG_UPDATE		0x004
+
+static const struct mtk_fixed_clk fixed_clks[] = {
+	FIXED_CLK(CLK_TOP_F_FRTC, "f_frtc_ck", "clk32k", 32768),
+	FIXED_CLK(CLK_TOP_CLK26M, "clk_26m_ck", "clk26m", 26000000),
+	FIXED_CLK(CLK_TOP_DMPLL, "dmpll_ck", NULL, 466000000),
+};
+
+static const struct mtk_fixed_factor top_divs[] = {
+	FACTOR(CLK_TOP_SYSPLL, "syspll_ck", "mainpll", 1, 1),
+	FACTOR(CLK_TOP_SYSPLL_D2, "syspll_d2", "mainpll", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL1_D2, "syspll1_d2", "syspll_d2", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL1_D4, "syspll1_d4", "syspll_d2", 1, 4),
+	FACTOR(CLK_TOP_SYSPLL1_D8, "syspll1_d8", "syspll_d2", 1, 8),
+	FACTOR(CLK_TOP_SYSPLL1_D16, "syspll1_d16", "syspll_d2", 1, 16),
+	FACTOR(CLK_TOP_SYSPLL_D3, "syspll_d3", "mainpll", 1, 3),
+	FACTOR(CLK_TOP_SYSPLL2_D2, "syspll2_d2", "syspll_d3", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL2_D4, "syspll2_d4", "syspll_d3", 1, 4),
+	FACTOR(CLK_TOP_SYSPLL2_D8, "syspll2_d8", "syspll_d3", 1, 8),
+	FACTOR(CLK_TOP_SYSPLL_D5, "syspll_d5", "mainpll", 1, 5),
+	FACTOR(CLK_TOP_SYSPLL3_D2, "syspll3_d2", "syspll_d5", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL3_D4, "syspll3_d4", "syspll_d5", 1, 4),
+	FACTOR(CLK_TOP_SYSPLL_D7, "syspll_d7", "mainpll", 1, 7),
+	FACTOR(CLK_TOP_SYSPLL4_D2, "syspll4_d2", "syspll_d7", 1, 2),
+	FACTOR(CLK_TOP_SYSPLL4_D4, "syspll4_d4", "syspll_d7", 1, 4),
+	FACTOR(CLK_TOP_UNIVPLL, "univpll", "univ2pll", 1, 2),
+	FACTOR(CLK_TOP_USB20_192M, "usb20_192m_ck", "univpll", 2, 13),
+	FACTOR(CLK_TOP_USB20_192M_D4, "usb20_192m_d4", "usb20_192m_ck", 1, 4),
+	FACTOR(CLK_TOP_USB20_192M_D8, "usb20_192m_d8", "usb20_192m_ck", 1, 8),
+	FACTOR(CLK_TOP_USB20_192M_D16,
+	       "usb20_192m_d16", "usb20_192m_ck", 1, 16),
+	FACTOR(CLK_TOP_USB20_192M_D32,
+	       "usb20_192m_d32", "usb20_192m_ck", 1, 32),
+	FACTOR(CLK_TOP_UNIVPLL_D2, "univpll_d2", "univpll", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL1_D2, "univpll1_d2", "univpll_d2", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL1_D4, "univpll1_d4", "univpll_d2", 1, 4),
+	FACTOR(CLK_TOP_UNIVPLL_D3, "univpll_d3", "univpll", 1, 3),
+	FACTOR(CLK_TOP_UNIVPLL2_D2, "univpll2_d2", "univpll_d3", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL2_D4, "univpll2_d4", "univpll_d3", 1, 4),
+	FACTOR(CLK_TOP_UNIVPLL2_D8, "univpll2_d8", "univpll_d3", 1, 8),
+	FACTOR(CLK_TOP_UNIVPLL2_D32, "univpll2_d32", "univpll_d3", 1, 32),
+	FACTOR(CLK_TOP_UNIVPLL_D5, "univpll_d5", "univpll", 1, 5),
+	FACTOR(CLK_TOP_UNIVPLL3_D2, "univpll3_d2", "univpll_d5", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL3_D4, "univpll3_d4", "univpll_d5", 1, 4),
+	FACTOR(CLK_TOP_MMPLL, "mmpll_ck", "mmpll", 1, 1),
+	FACTOR(CLK_TOP_MMPLL_D2, "mmpll_d2", "mmpll_ck", 1, 2),
+	FACTOR(CLK_TOP_MPLL, "mpll_ck", "mpll", 1, 1),
+	FACTOR(CLK_TOP_DA_MPLL_104M_DIV, "mpll_104m_div", "mpll_ck", 1, 2),
+	FACTOR(CLK_TOP_DA_MPLL_52M_DIV, "mpll_52m_div", "mpll_ck", 1, 4),
+	FACTOR(CLK_TOP_MFGPLL, "mfgpll_ck", "mfgpll", 1, 1),
+	FACTOR(CLK_TOP_MSDCPLL, "msdcpll_ck", "msdcpll", 1, 1),
+	FACTOR(CLK_TOP_MSDCPLL_D2, "msdcpll_d2", "msdcpll_ck", 1, 2),
+	FACTOR(CLK_TOP_APLL1, "apll1_ck", "apll1", 1, 1),
+	FACTOR(CLK_TOP_APLL1_D2, "apll1_d2", "apll1_ck", 1, 2),
+	FACTOR(CLK_TOP_APLL1_D4, "apll1_d4", "apll1_ck", 1, 4),
+	FACTOR(CLK_TOP_APLL1_D8, "apll1_d8", "apll1_ck", 1, 8),
+	FACTOR(CLK_TOP_ULPOSC1, "ulposc1_ck", "ulposc1", 1, 1),
+	FACTOR(CLK_TOP_ULPOSC1_D2, "ulposc1_d2", "ulposc1_ck", 1, 2),
+	FACTOR(CLK_TOP_ULPOSC1_D4, "ulposc1_d4", "ulposc1_ck", 1, 4),
+	FACTOR(CLK_TOP_ULPOSC1_D8, "ulposc1_d8", "ulposc1_ck", 1, 8),
+	FACTOR(CLK_TOP_ULPOSC1_D16, "ulposc1_d16", "ulposc1_ck", 1, 16),
+	FACTOR(CLK_TOP_ULPOSC1_D32, "ulposc1_d32", "ulposc1_ck", 1, 32),
+	FACTOR(CLK_TOP_F_F26M, "f_f26m_ck", "clk_26m_ck", 1, 1),
+	FACTOR(CLK_TOP_AXI, "axi_ck", "axi_sel", 1, 1),
+	FACTOR(CLK_TOP_MM, "mm_ck", "mm_sel", 1, 1),
+	FACTOR(CLK_TOP_SCP, "scp_ck", "scp_sel", 1, 1),
+	FACTOR(CLK_TOP_MFG, "mfg_ck", "mfg_sel", 1, 1),
+	FACTOR(CLK_TOP_F_FUART, "f_fuart_ck", "uart_sel", 1, 1),
+	FACTOR(CLK_TOP_SPI, "spi_ck", "spi_sel", 1, 1),
+	FACTOR(CLK_TOP_MSDC50_0, "msdc50_0_ck", "msdc50_0_sel", 1, 1),
+	FACTOR(CLK_TOP_MSDC30_1, "msdc30_1_ck", "msdc30_1_sel", 1, 1),
+	FACTOR(CLK_TOP_AUDIO, "audio_ck", "audio_sel", 1, 1),
+	FACTOR(CLK_TOP_AUD_1, "aud_1_ck", "aud_1_sel", 1, 1),
+	FACTOR(CLK_TOP_AUD_ENGEN1, "aud_engen1_ck", "aud_engen1_sel", 1, 1),
+	FACTOR(CLK_TOP_F_FDISP_PWM, "f_fdisp_pwm_ck", "disp_pwm_sel", 1, 1),
+	FACTOR(CLK_TOP_SSPM, "sspm_ck", "sspm_sel", 1, 1),
+	FACTOR(CLK_TOP_DXCC, "dxcc_ck", "dxcc_sel", 1, 1),
+	FACTOR(CLK_TOP_I2C, "i2c_ck", "i2c_sel", 1, 1),
+	FACTOR(CLK_TOP_F_FPWM, "f_fpwm_ck", "pwm_sel", 1, 1),
+	FACTOR(CLK_TOP_F_FSENINF, "f_fseninf_ck", "seninf_sel", 1, 1),
+	FACTOR(CLK_TOP_AES_FDE, "aes_fde_ck", "aes_fde_sel", 1, 1),
+	FACTOR(CLK_TOP_F_BIST2FPC, "f_bist2fpc_ck", "univpll2_d2", 1, 1),
+	FACTOR(CLK_TOP_ARMPLL_DIVIDER_PLL0, "arm_div_pll0", "syspll_d2", 1, 1),
+	FACTOR(CLK_TOP_ARMPLL_DIVIDER_PLL1, "arm_div_pll1", "syspll_ck", 1, 1),
+	FACTOR(CLK_TOP_ARMPLL_DIVIDER_PLL2, "arm_div_pll2", "univpll_d2", 1, 1),
+	FACTOR(CLK_TOP_DA_USB20_48M_DIV,
+	       "usb20_48m_div", "usb20_192m_d4", 1, 1),
+	FACTOR(CLK_TOP_DA_UNIV_48M_DIV, "univ_48m_div", "usb20_192m_d4", 1, 1),
+};
+
+static const char * const axi_parents[] = {
+	"clk26m",
+	"syspll_d7",
+	"syspll1_d4",
+	"syspll3_d2"
+};
+
+static const char * const mem_parents[] = {
+	"clk26m",
+	"dmpll_ck",
+	"apll1_ck"
+};
+
+static const char * const mm_parents[] = {
+	"clk26m",
+	"mmpll_ck",
+	"syspll1_d2",
+	"syspll_d5",
+	"syspll1_d4",
+	"univpll_d5",
+	"univpll1_d2",
+	"mmpll_d2"
+};
+
+static const char * const scp_parents[] = {
+	"clk26m",
+	"syspll4_d2",
+	"univpll2_d2",
+	"syspll1_d2",
+	"univpll1_d2",
+	"syspll_d3",
+	"univpll_d3"
+};
+
+static const char * const mfg_parents[] = {
+	"clk26m",
+	"mfgpll_ck",
+	"syspll_d3",
+	"univpll_d3"
+};
+
+static const char * const atb_parents[] = {
+	"clk26m",
+	"syspll1_d4",
+	"syspll1_d2"
+};
+
+static const char * const camtg_parents[] = {
+	"clk26m",
+	"usb20_192m_d8",
+	"univpll2_d8",
+	"usb20_192m_d4",
+	"univpll2_d32",
+	"usb20_192m_d16",
+	"usb20_192m_d32"
+};
+
+static const char * const uart_parents[] = {
+	"clk26m",
+	"univpll2_d8"
+};
+
+static const char * const spi_parents[] = {
+	"clk26m",
+	"syspll3_d2",
+	"syspll4_d2",
+	"syspll2_d4"
+};
+
+static const char * const msdc5hclk_parents[] = {
+	"clk26m",
+	"syspll1_d2",
+	"univpll1_d4",
+	"syspll2_d2"
+};
+
+static const char * const msdc50_0_parents[] = {
+	"clk26m",
+	"msdcpll_ck",
+	"syspll2_d2",
+	"syspll4_d2",
+	"univpll1_d2",
+	"syspll1_d2",
+	"univpll_d5",
+	"univpll1_d4"
+};
+
+static const char * const msdc30_1_parents[] = {
+	"clk26m",
+	"msdcpll_d2",
+	"univpll2_d2",
+	"syspll2_d2",
+	"syspll1_d4",
+	"univpll1_d4",
+	"usb20_192m_d4",
+	"syspll2_d4"
+};
+
+static const char * const audio_parents[] = {
+	"clk26m",
+	"syspll3_d4",
+	"syspll4_d4",
+	"syspll1_d16"
+};
+
+static const char * const aud_intbus_parents[] = {
+	"clk26m",
+	"syspll1_d4",
+	"syspll4_d2"
+};
+
+static const char * const aud_1_parents[] = {
+	"clk26m",
+	"apll1_ck"
+};
+
+static const char * const aud_engen1_parents[] = {
+	"clk26m",
+	"apll1_d2",
+	"apll1_d4",
+	"apll1_d8"
+};
+
+static const char * const disp_pwm_parents[] = {
+	"clk26m",
+	"univpll2_d4",
+	"ulposc1_d2",
+	"ulposc1_d8"
+};
+
+static const char * const sspm_parents[] = {
+	"clk26m",
+	"syspll1_d2",
+	"syspll_d3"
+};
+
+static const char * const dxcc_parents[] = {
+	"clk26m",
+	"syspll1_d2",
+	"syspll1_d4",
+	"syspll1_d8"
+};
+
+static const char * const usb_top_parents[] = {
+	"clk26m",
+	"univpll3_d4"
+};
+
+static const char * const spm_parents[] = {
+	"clk26m",
+	"syspll1_d8"
+};
+
+static const char * const i2c_parents[] = {
+	"clk26m",
+	"univpll3_d4",
+	"univpll3_d2",
+	"syspll1_d8",
+	"syspll2_d8"
+};
+
+static const char * const pwm_parents[] = {
+	"clk26m",
+	"univpll3_d4",
+	"syspll1_d8"
+};
+
+static const char * const seninf_parents[] = {
+	"clk26m",
+	"univpll1_d4",
+	"univpll1_d2",
+	"univpll2_d2"
+};
+
+static const char * const aes_fde_parents[] = {
+	"clk26m",
+	"msdcpll_ck",
+	"univpll_d3",
+	"univpll2_d2",
+	"univpll1_d2",
+	"syspll1_d2"
+};
+
+static const char * const ulposc_parents[] = {
+	"clk26m",
+	"ulposc1_d4",
+	"ulposc1_d8",
+	"ulposc1_d16",
+	"ulposc1_d32"
+};
+
+static const char * const camtm_parents[] = {
+	"clk26m",
+	"univpll1_d4",
+	"univpll1_d2",
+	"univpll2_d2"
+};
+
+#define INVALID_UPDATE_REG 0xFFFFFFFF
+#define INVALID_UPDATE_SHIFT -1
+#define INVALID_MUX_GATE -1
+
+static const struct mtk_mux top_muxes[] = {
+	/* CLK_CFG_0 */
+	MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_AXI_SEL, "axi_sel", axi_parents,
+			      CLK_CFG_0, CLK_CFG_0_SET, CLK_CFG_0_CLR,
+			      0, 2, 7, CLK_CFG_UPDATE, 0, CLK_IS_CRITICAL),
+	MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_MEM_SEL, "mem_sel", mem_parents,
+			      CLK_CFG_0, CLK_CFG_0_SET, CLK_CFG_0_CLR,
+			      8, 2, 15, CLK_CFG_UPDATE, 1, CLK_IS_CRITICAL),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MM_SEL, "mm_sel", mm_parents, CLK_CFG_0,
+			CLK_CFG_0_SET, CLK_CFG_0_CLR, 16, 3, 23,
+			CLK_CFG_UPDATE, 2),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_SCP_SEL, "scp_sel", scp_parents, CLK_CFG_0,
+			CLK_CFG_0_SET, CLK_CFG_0_CLR, 24, 3, 31,
+			CLK_CFG_UPDATE, 3),
+	/* CLK_CFG_1 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MFG_SEL, "mfg_sel", mfg_parents, CLK_CFG_1,
+			CLK_CFG_1_SET, CLK_CFG_1_CLR, 0, 2, 7,
+			CLK_CFG_UPDATE, 4),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_ATB_SEL, "atb_sel", atb_parents, CLK_CFG_1,
+			CLK_CFG_1_SET, CLK_CFG_1_CLR, 8, 2, 15,
+			CLK_CFG_UPDATE, 5),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG_SEL, "camtg_sel",
+			camtg_parents, CLK_CFG_1, CLK_CFG_1_SET,
+			CLK_CFG_1_CLR, 16, 3, 23, CLK_CFG_UPDATE, 6),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG1_SEL, "camtg1_sel", camtg_parents,
+			CLK_CFG_1, CLK_CFG_1_SET, CLK_CFG_1_CLR,
+			24, 3, 31, CLK_CFG_UPDATE, 7),
+	/* CLK_CFG_2 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG2_SEL, "camtg2_sel",
+			camtg_parents, CLK_CFG_2, CLK_CFG_2_SET,
+			CLK_CFG_2_CLR, 0, 3, 7, CLK_CFG_UPDATE, 8),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG3_SEL, "camtg3_sel", camtg_parents,
+			CLK_CFG_2, CLK_CFG_2_SET, CLK_CFG_2_CLR,
+			8, 3, 15, CLK_CFG_UPDATE, 9),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_UART_SEL, "uart_sel", uart_parents,
+			CLK_CFG_2, CLK_CFG_2_SET, CLK_CFG_2_CLR, 16, 1, 23,
+			CLK_CFG_UPDATE, 10),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_SPI_SEL, "spi_sel", spi_parents, CLK_CFG_2,
+			CLK_CFG_2_SET, CLK_CFG_2_CLR, 24, 2, 31,
+			CLK_CFG_UPDATE, 11),
+	/* CLK_CFG_3 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC50_0_HCLK_SEL, "msdc5hclk",
+			msdc5hclk_parents, CLK_CFG_3, CLK_CFG_3_SET,
+			CLK_CFG_3_CLR, 0, 2, 7, CLK_CFG_UPDATE, 12),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC50_0_SEL, "msdc50_0_sel",
+			msdc50_0_parents, CLK_CFG_3, CLK_CFG_3_SET,
+			CLK_CFG_3_CLR, 8, 3, 15, CLK_CFG_UPDATE, 13),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC30_1_SEL, "msdc30_1_sel",
+			msdc30_1_parents, CLK_CFG_3, CLK_CFG_3_SET,
+			CLK_CFG_3_CLR, 16, 3, 23, CLK_CFG_UPDATE, 14),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_AUDIO_SEL, "audio_sel", audio_parents,
+			CLK_CFG_3, CLK_CFG_3_SET, CLK_CFG_3_CLR,
+			24, 2, 31, CLK_CFG_UPDATE, 15),
+	/* CLK_CFG_4 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_INTBUS_SEL, "aud_intbus_sel",
+			aud_intbus_parents, CLK_CFG_4, CLK_CFG_4_SET,
+			CLK_CFG_4_CLR, 0, 2, 7, CLK_CFG_UPDATE, 16),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_1_SEL, "aud_1_sel", aud_1_parents,
+			CLK_CFG_4, CLK_CFG_4_SET, CLK_CFG_4_CLR,
+			8, 1, 15, CLK_CFG_UPDATE, 17),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_ENGEN1_SEL, "aud_engen1_sel",
+			aud_engen1_parents, CLK_CFG_4, CLK_CFG_4_SET,
+			CLK_CFG_4_CLR, 16, 2, 23, CLK_CFG_UPDATE, 18),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_DISP_PWM_SEL, "disp_pwm_sel",
+			disp_pwm_parents, CLK_CFG_4, CLK_CFG_4_SET,
+			CLK_CFG_4_CLR, 24, 2, 31, CLK_CFG_UPDATE, 19),
+	/* CLK_CFG_5 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_SSPM_SEL, "sspm_sel", sspm_parents,
+			CLK_CFG_5, CLK_CFG_5_SET, CLK_CFG_5_CLR, 0, 2, 7,
+			CLK_CFG_UPDATE, 20),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_DXCC_SEL, "dxcc_sel", dxcc_parents,
+			CLK_CFG_5, CLK_CFG_5_SET, CLK_CFG_5_CLR, 8, 2, 15,
+			CLK_CFG_UPDATE, 21),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_USB_TOP_SEL, "usb_top_sel",
+			usb_top_parents, CLK_CFG_5, CLK_CFG_5_SET,
+			CLK_CFG_5_CLR, 16, 1, 23, CLK_CFG_UPDATE, 22),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_SPM_SEL, "spm_sel", spm_parents, CLK_CFG_5,
+			CLK_CFG_5_SET, CLK_CFG_5_CLR, 24, 1, 31,
+			CLK_CFG_UPDATE, 23),
+	/* CLK_CFG_6 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_I2C_SEL, "i2c_sel", i2c_parents, CLK_CFG_6,
+			CLK_CFG_6_SET, CLK_CFG_6_CLR, 0, 3, 7, CLK_CFG_UPDATE,
+			24),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_PWM_SEL, "pwm_sel", pwm_parents, CLK_CFG_6,
+			CLK_CFG_6_SET, CLK_CFG_6_CLR, 8, 2, 15, CLK_CFG_UPDATE,
+			25),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_SENINF_SEL, "seninf_sel", seninf_parents,
+			CLK_CFG_6, CLK_CFG_6_SET, CLK_CFG_6_CLR, 16, 2, 23,
+			CLK_CFG_UPDATE, 26),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_AES_FDE_SEL, "aes_fde_sel",
+			aes_fde_parents, CLK_CFG_6, CLK_CFG_6_SET,
+			CLK_CFG_6_CLR, 24, 3, 31, CLK_CFG_UPDATE, 27),
+	/* CLK_CFG_7 */
+	MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_PWRAP_ULPOSC_SEL, "ulposc_sel",
+			      ulposc_parents, CLK_CFG_7, CLK_CFG_7_SET,
+			      CLK_CFG_7_CLR, 0, 3, 7, CLK_CFG_UPDATE, 28,
+			      CLK_IS_CRITICAL),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTM_SEL, "camtm_sel", camtm_parents,
+			CLK_CFG_7, CLK_CFG_7_SET, CLK_CFG_7_CLR, 8, 2, 15,
+			CLK_CFG_UPDATE, 29),
+};
+
+static const struct mtk_gate_regs top0_cg_regs = {
+	.set_ofs = 0x0,
+	.clr_ofs = 0x0,
+	.sta_ofs = 0x0,
+};
+
+static const struct mtk_gate_regs top1_cg_regs = {
+	.set_ofs = 0x104,
+	.clr_ofs = 0x104,
+	.sta_ofs = 0x104,
+};
+
+static const struct mtk_gate_regs top2_cg_regs = {
+	.set_ofs = 0x320,
+	.clr_ofs = 0x320,
+	.sta_ofs = 0x320,
+};
+
+#define GATE_TOP0(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top0_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr,	\
+	}
+
+#define GATE_TOP1(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top1_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr_inv,	\
+	}
+
+#define GATE_TOP2(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top2_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr,	\
+	}
+
+static const struct mtk_gate top_clks[] = {
+	/* TOP0 */
+	GATE_TOP0(CLK_TOP_MD_32K, "md_32k", "f_frtc_ck", 8),
+	GATE_TOP0(CLK_TOP_MD_26M, "md_26m", "f_f26m_ck", 9),
+	GATE_TOP0(CLK_TOP_MD2_32K, "md2_32k", "f_frtc_ck", 10),
+	GATE_TOP0(CLK_TOP_MD2_26M, "md2_26m", "f_f26m_ck", 11),
+	/* TOP1 */
+	GATE_TOP1(CLK_TOP_ARMPLL_DIVIDER_PLL0_EN,
+		  "arm_div_pll0_en", "arm_div_pll0", 3),
+	GATE_TOP1(CLK_TOP_ARMPLL_DIVIDER_PLL1_EN,
+		  "arm_div_pll1_en", "arm_div_pll1", 4),
+	GATE_TOP1(CLK_TOP_ARMPLL_DIVIDER_PLL2_EN,
+		  "arm_div_pll2_en", "arm_div_pll2", 5),
+	GATE_TOP1(CLK_TOP_FMEM_OCC_DRC_EN, "drc_en", "univpll2_d2", 6),
+	GATE_TOP1(CLK_TOP_USB20_48M_EN, "usb20_48m_en", "usb20_48m_div", 8),
+	GATE_TOP1(CLK_TOP_UNIVPLL_48M_EN, "univpll_48m_en", "univ_48m_div", 9),
+	GATE_TOP1(CLK_TOP_F_UFS_MP_SAP_CFG_EN, "ufs_sap", "f_f26m_ck", 12),
+	GATE_TOP1(CLK_TOP_F_BIST2FPC_EN, "bist2fpc", "f_bist2fpc_ck", 16),
+	/* TOP2 */
+	GATE_TOP2(CLK_TOP_APLL12_DIV0, "apll12_div0", "aud_1_ck", 2),
+	GATE_TOP2(CLK_TOP_APLL12_DIV1, "apll12_div1", "aud_1_ck", 3),
+	GATE_TOP2(CLK_TOP_APLL12_DIV2, "apll12_div2", "aud_1_ck", 4),
+	GATE_TOP2(CLK_TOP_APLL12_DIV3, "apll12_div3", "aud_1_ck", 5),
+};
+
+static const struct mtk_gate_regs ifr2_cg_regs = {
+	.set_ofs = 0x80,
+	.clr_ofs = 0x84,
+	.sta_ofs = 0x90,
+};
+
+static const struct mtk_gate_regs ifr3_cg_regs = {
+	.set_ofs = 0x88,
+	.clr_ofs = 0x8c,
+	.sta_ofs = 0x94,
+};
+
+static const struct mtk_gate_regs ifr4_cg_regs = {
+	.set_ofs = 0xa4,
+	.clr_ofs = 0xa8,
+	.sta_ofs = 0xac,
+};
+
+static const struct mtk_gate_regs ifr5_cg_regs = {
+	.set_ofs = 0xc0,
+	.clr_ofs = 0xc4,
+	.sta_ofs = 0xc8,
+};
+
+#define GATE_IFR2(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &ifr2_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_IFR3(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &ifr3_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_IFR4(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &ifr4_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_IFR5(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &ifr5_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+static const struct mtk_gate ifr_clks[] = {
+	/* INFRA_TOPAXI */
+	/* INFRA PERI */
+	/* INFRA mode 0 */
+	GATE_IFR2(CLK_IFR_ICUSB, "ifr_icusb", "axi_ck", 8),
+	GATE_IFR2(CLK_IFR_GCE, "ifr_gce", "axi_ck", 9),
+	GATE_IFR2(CLK_IFR_THERM, "ifr_therm", "axi_ck", 10),
+	GATE_IFR2(CLK_IFR_I2C_AP, "ifr_i2c_ap", "i2c_ck", 11),
+	GATE_IFR2(CLK_IFR_I2C_CCU, "ifr_i2c_ccu", "i2c_ck", 12),
+	GATE_IFR2(CLK_IFR_I2C_SSPM, "ifr_i2c_sspm", "i2c_ck", 13),
+	GATE_IFR2(CLK_IFR_I2C_RSV, "ifr_i2c_rsv", "i2c_ck", 14),
+	GATE_IFR2(CLK_IFR_PWM_HCLK, "ifr_pwm_hclk", "axi_ck", 15),
+	GATE_IFR2(CLK_IFR_PWM1, "ifr_pwm1", "f_fpwm_ck", 16),
+	GATE_IFR2(CLK_IFR_PWM2, "ifr_pwm2", "f_fpwm_ck", 17),
+	GATE_IFR2(CLK_IFR_PWM3, "ifr_pwm3", "f_fpwm_ck", 18),
+	GATE_IFR2(CLK_IFR_PWM4, "ifr_pwm4", "f_fpwm_ck", 19),
+	GATE_IFR2(CLK_IFR_PWM5, "ifr_pwm5", "f_fpwm_ck", 20),
+	GATE_IFR2(CLK_IFR_PWM, "ifr_pwm", "f_fpwm_ck", 21),
+	GATE_IFR2(CLK_IFR_UART0, "ifr_uart0", "f_fuart_ck", 22),
+	GATE_IFR2(CLK_IFR_UART1, "ifr_uart1", "f_fuart_ck", 23),
+	GATE_IFR2(CLK_IFR_GCE_26M, "ifr_gce_26m", "f_f26m_ck", 27),
+	GATE_IFR2(CLK_IFR_CQ_DMA_FPC, "ifr_dma", "axi_ck", 28),
+	GATE_IFR2(CLK_IFR_BTIF, "ifr_btif", "axi_ck", 31),
+	/* INFRA mode 1 */
+	GATE_IFR3(CLK_IFR_SPI0, "ifr_spi0", "spi_ck", 1),
+	GATE_IFR3(CLK_IFR_MSDC0, "ifr_msdc0", "msdc5hclk", 2),
+	GATE_IFR3(CLK_IFR_MSDC1, "ifr_msdc1", "axi_ck", 4),
+	GATE_IFR3(CLK_IFR_TRNG, "ifr_trng", "axi_ck", 9),
+	GATE_IFR3(CLK_IFR_AUXADC, "ifr_auxadc", "f_f26m_ck", 10),
+	GATE_IFR3(CLK_IFR_CCIF1_AP, "ifr_ccif1_ap", "axi_ck", 12),
+	GATE_IFR3(CLK_IFR_CCIF1_MD, "ifr_ccif1_md", "axi_ck", 13),
+	GATE_IFR3(CLK_IFR_AUXADC_MD, "ifr_auxadc_md", "f_f26m_ck", 14),
+	GATE_IFR3(CLK_IFR_AP_DMA, "ifr_ap_dma", "axi_ck", 18),
+	GATE_IFR3(CLK_IFR_DEVICE_APC, "ifr_dapc", "axi_ck", 20),
+	GATE_IFR3(CLK_IFR_CCIF_AP, "ifr_ccif_ap", "axi_ck", 23),
+	GATE_IFR3(CLK_IFR_AUDIO, "ifr_audio", "axi_ck", 25),
+	GATE_IFR3(CLK_IFR_CCIF_MD, "ifr_ccif_md", "axi_ck", 26),
+	/* INFRA mode 2 */
+	GATE_IFR4(CLK_IFR_RG_PWM_FBCLK6, "ifr_pwmfb", "f_f26m_ck", 0),
+	GATE_IFR4(CLK_IFR_DISP_PWM, "ifr_disp_pwm", "f_fdisp_pwm_ck", 2),
+	GATE_IFR4(CLK_IFR_CLDMA_BCLK, "ifr_cldmabclk", "axi_ck", 3),
+	GATE_IFR4(CLK_IFR_AUDIO_26M_BCLK, "ifr_audio26m", "f_f26m_ck", 4),
+	GATE_IFR4(CLK_IFR_SPI1, "ifr_spi1", "spi_ck", 6),
+	GATE_IFR4(CLK_IFR_I2C4, "ifr_i2c4", "i2c_ck", 7),
+	GATE_IFR4(CLK_IFR_SPI2, "ifr_spi2", "spi_ck", 9),
+	GATE_IFR4(CLK_IFR_SPI3, "ifr_spi3", "spi_ck", 10),
+	GATE_IFR4(CLK_IFR_I2C5, "ifr_i2c5", "i2c_ck", 18),
+	GATE_IFR4(CLK_IFR_I2C5_ARBITER, "ifr_i2c5a", "i2c_ck", 19),
+	GATE_IFR4(CLK_IFR_I2C5_IMM, "ifr_i2c5_imm", "i2c_ck", 20),
+	GATE_IFR4(CLK_IFR_I2C1_ARBITER, "ifr_i2c1a", "i2c_ck", 21),
+	GATE_IFR4(CLK_IFR_I2C1_IMM, "ifr_i2c1_imm", "i2c_ck", 22),
+	GATE_IFR4(CLK_IFR_I2C2_ARBITER, "ifr_i2c2a", "i2c_ck", 23),
+	GATE_IFR4(CLK_IFR_I2C2_IMM, "ifr_i2c2_imm", "i2c_ck", 24),
+	GATE_IFR4(CLK_IFR_SPI4, "ifr_spi4", "spi_ck", 25),
+	GATE_IFR4(CLK_IFR_SPI5, "ifr_spi5", "spi_ck", 26),
+	GATE_IFR4(CLK_IFR_CQ_DMA, "ifr_cq_dma", "axi_ck", 27),
+	GATE_IFR4(CLK_IFR_FAES_FDE, "ifr_faes_fde_ck", "aes_fde_ck", 29),
+	/* INFRA mode 3 */
+	GATE_IFR5(CLK_IFR_MSDC0_SELF, "ifr_msdc0sf", "msdc50_0_ck", 0),
+	GATE_IFR5(CLK_IFR_MSDC1_SELF, "ifr_msdc1sf", "msdc50_0_ck", 1),
+	GATE_IFR5(CLK_IFR_I2C6, "ifr_i2c6", "i2c_ck", 6),
+	GATE_IFR5(CLK_IFR_AP_MSDC0, "ifr_ap_msdc0", "msdc50_0_ck", 7),
+	GATE_IFR5(CLK_IFR_MD_MSDC0, "ifr_md_msdc0", "msdc50_0_ck", 8),
+	GATE_IFR5(CLK_IFR_MSDC0_SRC, "ifr_msdc0_clk", "msdc50_0_ck", 9),
+	GATE_IFR5(CLK_IFR_MSDC1_SRC, "ifr_msdc1_clk", "msdc30_1_ck", 10),
+	GATE_IFR5(CLK_IFR_MCU_PM_BCLK, "ifr_mcu_pm_bclk", "axi_ck", 17),
+	GATE_IFR5(CLK_IFR_CCIF2_AP, "ifr_ccif2_ap", "axi_ck", 18),
+	GATE_IFR5(CLK_IFR_CCIF2_MD, "ifr_ccif2_md", "axi_ck", 19),
+	GATE_IFR5(CLK_IFR_CCIF3_AP, "ifr_ccif3_ap", "axi_ck", 20),
+	GATE_IFR5(CLK_IFR_CCIF3_MD, "ifr_ccif3_md", "axi_ck", 21),
+};
+
+/* additional CCF control for mipi26M race condition(disp/camera) */
+static const struct mtk_gate_regs apmixed_cg_regs = {
+	.set_ofs = 0x14,
+	.clr_ofs = 0x14,
+	.sta_ofs = 0x14,
+};
+
+#define GATE_APMIXED(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &apmixed_cg_regs,		\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr_inv,		\
+	}
+
+static const struct mtk_gate apmixed_clks[] = {
+	/* AUDIO0 */
+	GATE_APMIXED(CLK_APMIXED_SSUSB26M, "apmixed_ssusb26m", "f_f26m_ck",
+		     4),
+	GATE_APMIXED(CLK_APMIXED_APPLL26M, "apmixed_appll26m", "f_f26m_ck",
+		     5),
+	GATE_APMIXED(CLK_APMIXED_MIPIC0_26M, "apmixed_mipic026m", "f_f26m_ck",
+		     6),
+	GATE_APMIXED(CLK_APMIXED_MDPLLGP26M, "apmixed_mdpll26m", "f_f26m_ck",
+		     7),
+	GATE_APMIXED(CLK_APMIXED_MMSYS_F26M, "apmixed_mmsys26m", "f_f26m_ck",
+		     8),
+	GATE_APMIXED(CLK_APMIXED_UFS26M, "apmixed_ufs26m", "f_f26m_ck",
+		     9),
+	GATE_APMIXED(CLK_APMIXED_MIPIC1_26M, "apmixed_mipic126m", "f_f26m_ck",
+		     11),
+	GATE_APMIXED(CLK_APMIXED_MEMPLL26M, "apmixed_mempll26m", "f_f26m_ck",
+		     13),
+	GATE_APMIXED(CLK_APMIXED_CLKSQ_LVPLL_26M, "apmixed_lvpll26m",
+		     "f_f26m_ck", 14),
+	GATE_APMIXED(CLK_APMIXED_MIPID0_26M, "apmixed_mipid026m", "f_f26m_ck",
+		     16),
+};
+
+#define MT6765_PLL_FMAX		(3800UL * MHZ)
+#define MT6765_PLL_FMIN		(1500UL * MHZ)
+
+#define CON0_MT6765_RST_BAR	BIT(23)
+
+#define PLL_INFO_NULL		(0xFF)
+
+#define PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits,	\
+		_pcwibits, _pd_reg, _pd_shift, _tuner_reg, _tuner_en_reg,\
+		_tuner_en_bit, _pcw_reg, _pcw_shift, _div_table) {\
+		.id = _id,						\
+		.name = _name,						\
+		.reg = _reg,						\
+		.pwr_reg = _pwr_reg,					\
+		.en_mask = _en_mask,					\
+		.flags = _flags,					\
+		.rst_bar_mask = CON0_MT6765_RST_BAR,			\
+		.fmax = MT6765_PLL_FMAX,				\
+		.fmin = MT6765_PLL_FMIN,				\
+		.pcwbits = _pcwbits,					\
+		.pcwibits = _pcwibits,					\
+		.pd_reg = _pd_reg,					\
+		.pd_shift = _pd_shift,					\
+		.tuner_reg = _tuner_reg,				\
+		.tuner_en_reg = _tuner_en_reg,				\
+		.tuner_en_bit = _tuner_en_bit,				\
+		.pcw_reg = _pcw_reg,					\
+		.pcw_shift = _pcw_shift,				\
+		.div_table = _div_table,				\
+	}
+
+#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits,	\
+			_pcwibits, _pd_reg, _pd_shift, _tuner_reg,	\
+			_tuner_en_reg, _tuner_en_bit, _pcw_reg,	\
+			_pcw_shift)	\
+		PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags,	\
+			_pcwbits, _pcwibits, _pd_reg, _pd_shift,	\
+			_tuner_reg, _tuner_en_reg, _tuner_en_bit,	\
+			_pcw_reg, _pcw_shift, NULL)	\
+
+static const struct mtk_pll_data plls[] = {
+	PLL(CLK_APMIXED_ARMPLL_L, "armpll_l", 0x021C, 0x0228, BIT(0),
+	    PLL_AO, 22, 8, 0x0220, 24, 0, 0, 0, 0x0220, 0),
+	PLL(CLK_APMIXED_ARMPLL, "armpll", 0x020C, 0x0218, BIT(0),
+	    PLL_AO, 22, 8, 0x0210, 24, 0, 0, 0, 0x0210, 0),
+	PLL(CLK_APMIXED_CCIPLL, "ccipll", 0x022C, 0x0238, BIT(0),
+	    PLL_AO, 22, 8, 0x0230, 24, 0, 0, 0, 0x0230, 0),
+	PLL(CLK_APMIXED_MAINPLL, "mainpll", 0x023C, 0x0248, BIT(0),
+	    (HAVE_RST_BAR | PLL_AO), 22, 8, 0x0240, 24, 0, 0, 0, 0x0240,
+	    0),
+	PLL(CLK_APMIXED_MFGPLL, "mfgpll", 0x024C, 0x0258, BIT(0),
+	    0, 22, 8, 0x0250, 24, 0, 0, 0, 0x0250, 0),
+	PLL(CLK_APMIXED_MMPLL, "mmpll", 0x025C, 0x0268, BIT(0),
+	    0, 22, 8, 0x0260, 24, 0, 0, 0, 0x0260, 0),
+	PLL(CLK_APMIXED_UNIV2PLL, "univ2pll", 0x026C, 0x0278, BIT(0),
+	    HAVE_RST_BAR, 22, 8, 0x0270, 24, 0, 0, 0, 0x0270, 0),
+	PLL(CLK_APMIXED_MSDCPLL, "msdcpll", 0x027C, 0x0288, BIT(0),
+	    0, 22, 8, 0x0280, 24, 0, 0, 0, 0x0280, 0),
+	PLL(CLK_APMIXED_APLL1, "apll1", 0x028C, 0x029C, BIT(0),
+	    0, 32, 8, 0x0290, 24, 0x0040, 0x000C, 0, 0x0294, 0),
+	PLL(CLK_APMIXED_MPLL, "mpll", 0x02A0, 0x02AC, BIT(0),
+	    PLL_AO, 22, 8, 0x02A4, 24, 0, 0, 0, 0x02A4, 0),
+};
+
+static int clk_mt6765_apmixed_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+	void __iomem *base;
+	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base)) {
+		pr_err("%s(): ioremap failed\n", __func__);
+		return PTR_ERR(base);
+	}
+
+	clk_data = mtk_alloc_clk_data(CLK_APMIXED_NR_CLK);
+
+	mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data);
+
+	mtk_clk_register_gates(node, apmixed_clks,
+			       ARRAY_SIZE(apmixed_clks), clk_data);
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	apmixed_base = base;
+	/* MPLL, CCIPLL, MAINPLL set HW mode, TDCLKSQ, CLKSQ1 */
+	writel(readl(AP_PLL_CON3) & 0xFFFFFFE1, AP_PLL_CON3);
+	writel(readl(PLLON_CON0) & 0x01041041, PLLON_CON0);
+	writel(readl(PLLON_CON1) & 0x01041041, PLLON_CON1);
+
+	return r;
+}
+
+static int clk_mt6765_top_probe(struct platform_device *pdev)
+{
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+	void __iomem *base;
+	struct clk_onecell_data *clk_data;
+	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base)) {
+		pr_err("%s(): ioremap failed\n", __func__);
+		return PTR_ERR(base);
+	}
+
+	clk_data = mtk_alloc_clk_data(CLK_TOP_NR_CLK);
+
+	mtk_clk_register_fixed_clks(fixed_clks, ARRAY_SIZE(fixed_clks),
+				    clk_data);
+	mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs),
+				 clk_data);
+	mtk_clk_register_muxes(top_muxes, ARRAY_SIZE(top_muxes), node,
+			       &mt6765_clk_lock, clk_data);
+	mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks),
+			       clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	cksys_base = base;
+	/* [4]:no need */
+	writel(readl(CLK_SCP_CFG_0) | 0x3EF, CLK_SCP_CFG_0);
+	/*[1,2,3,8]: no need*/
+	writel(readl(CLK_SCP_CFG_1) | 0x1, CLK_SCP_CFG_1);
+
+	return r;
+}
+
+static int clk_mt6765_ifr_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+	void __iomem *base;
+	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base)) {
+		pr_err("%s(): ioremap failed\n", __func__);
+		return PTR_ERR(base);
+	}
+
+	clk_data = mtk_alloc_clk_data(CLK_IFR_NR_CLK);
+
+	mtk_clk_register_gates(node, ifr_clks, ARRAY_SIZE(ifr_clks),
+			       clk_data);
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+		       __func__, r);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt6765[] = {
+	{
+		.compatible = "mediatek,mt6765-apmixedsys",
+		.data = clk_mt6765_apmixed_probe,
+	}, {
+		.compatible = "mediatek,mt6765-topckgen",
+		.data = clk_mt6765_top_probe,
+	}, {
+		.compatible = "mediatek,mt6765-infracfg",
+		.data = clk_mt6765_ifr_probe,
+	}, {
+		/* sentinel */
+	}
+};
+
+static int clk_mt6765_probe(struct platform_device *pdev)
+{
+	int (*clk_probe)(struct platform_device *d);
+	int r;
+
+	clk_probe = of_device_get_match_data(&pdev->dev);
+	if (!clk_probe)
+		return -EINVAL;
+
+	r = clk_probe(pdev);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt6765_drv = {
+	.probe = clk_mt6765_probe,
+	.driver = {
+		.name = "clk-mt6765",
+		.owner = THIS_MODULE,
+		.of_match_table = of_match_clk_mt6765,
+	},
+};
+
+static int __init clk_mt6765_init(void)
+{
+	return platform_driver_register(&clk_mt6765_drv);
+}
+
+arch_initcall(clk_mt6765_init);
diff --git a/drivers/clk/mediatek/clk-mt6779-mm.c b/drivers/clk/mediatek/clk-mt6779-mm.c
index fb5fbb8e3e41..059c1a41ac7a 100644
--- a/drivers/clk/mediatek/clk-mt6779-mm.c
+++ b/drivers/clk/mediatek/clk-mt6779-mm.c
@@ -84,15 +84,11 @@ static const struct mtk_gate mm_clks[] = {
 	GATE_MM1(CLK_MM_DISP_OVL_FBDC, "mm_disp_ovl_fbdc", "mm_sel", 16),
 };
 
-static const struct of_device_id of_match_clk_mt6779_mm[] = {
-	{ .compatible = "mediatek,mt6779-mmsys", },
-	{}
-};
-
 static int clk_mt6779_mm_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->parent->of_node;
 	struct clk_onecell_data *clk_data;
-	struct device_node *node = pdev->dev.of_node;
 
 	clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK);
 
@@ -106,7 +102,6 @@ static struct platform_driver clk_mt6779_mm_drv = {
 	.probe = clk_mt6779_mm_probe,
 	.driver = {
 		.name = "clk-mt6779-mm",
-		.of_match_table = of_match_clk_mt6779_mm,
 	},
 };
 
diff --git a/drivers/clk/mediatek/clk-mt6797-mm.c b/drivers/clk/mediatek/clk-mt6797-mm.c
index 8f05653b387d..01fdce287247 100644
--- a/drivers/clk/mediatek/clk-mt6797-mm.c
+++ b/drivers/clk/mediatek/clk-mt6797-mm.c
@@ -92,16 +92,12 @@ static const struct mtk_gate mm_clks[] = {
 		 "clk26m", 3),
 };
 
-static const struct of_device_id of_match_clk_mt6797_mm[] = {
-	{ .compatible = "mediatek,mt6797-mmsys", },
-	{}
-};
-
 static int clk_mt6797_mm_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->parent->of_node;
 	struct clk_onecell_data *clk_data;
 	int r;
-	struct device_node *node = pdev->dev.of_node;
 
 	clk_data = mtk_alloc_clk_data(CLK_MM_NR);
 
@@ -121,7 +117,6 @@ static struct platform_driver clk_mt6797_mm_drv = {
 	.probe = clk_mt6797_mm_probe,
 	.driver = {
 		.name = "clk-mt6797-mm",
-		.of_match_table = of_match_clk_mt6797_mm,
 	},
 };
 
diff --git a/drivers/clk/mediatek/clk-mt8173-mm.c b/drivers/clk/mediatek/clk-mt8173-mm.c
new file mode 100644
index 000000000000..36fa20be77b6
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8173-mm.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: James Liao <jamesjj.liao@mediatek.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#include "clk-gate.h"
+#include "clk-mtk.h"
+
+#include <dt-bindings/clock/mt8173-clk.h>
+
+static const struct mtk_gate_regs mm0_cg_regs = {
+	.set_ofs = 0x0104,
+	.clr_ofs = 0x0108,
+	.sta_ofs = 0x0100,
+};
+
+static const struct mtk_gate_regs mm1_cg_regs = {
+	.set_ofs = 0x0114,
+	.clr_ofs = 0x0118,
+	.sta_ofs = 0x0110,
+};
+
+#define GATE_MM0(_id, _name, _parent, _shift) {			\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_name = _parent,				\
+		.regs = &mm0_cg_regs,				\
+		.shift = _shift,				\
+		.ops = &mtk_clk_gate_ops_setclr,		\
+	}
+
+#define GATE_MM1(_id, _name, _parent, _shift) {			\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_name = _parent,				\
+		.regs = &mm1_cg_regs,				\
+		.shift = _shift,				\
+		.ops = &mtk_clk_gate_ops_setclr,		\
+	}
+
+static const struct mtk_gate mt8173_mm_clks[] = {
+	/* MM0 */
+	GATE_MM0(CLK_MM_SMI_COMMON, "mm_smi_common", "mm_sel", 0),
+	GATE_MM0(CLK_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 1),
+	GATE_MM0(CLK_MM_CAM_MDP, "mm_cam_mdp", "mm_sel", 2),
+	GATE_MM0(CLK_MM_MDP_RDMA0, "mm_mdp_rdma0", "mm_sel", 3),
+	GATE_MM0(CLK_MM_MDP_RDMA1, "mm_mdp_rdma1", "mm_sel", 4),
+	GATE_MM0(CLK_MM_MDP_RSZ0, "mm_mdp_rsz0", "mm_sel", 5),
+	GATE_MM0(CLK_MM_MDP_RSZ1, "mm_mdp_rsz1", "mm_sel", 6),
+	GATE_MM0(CLK_MM_MDP_RSZ2, "mm_mdp_rsz2", "mm_sel", 7),
+	GATE_MM0(CLK_MM_MDP_TDSHP0, "mm_mdp_tdshp0", "mm_sel", 8),
+	GATE_MM0(CLK_MM_MDP_TDSHP1, "mm_mdp_tdshp1", "mm_sel", 9),
+	GATE_MM0(CLK_MM_MDP_WDMA, "mm_mdp_wdma", "mm_sel", 11),
+	GATE_MM0(CLK_MM_MDP_WROT0, "mm_mdp_wrot0", "mm_sel", 12),
+	GATE_MM0(CLK_MM_MDP_WROT1, "mm_mdp_wrot1", "mm_sel", 13),
+	GATE_MM0(CLK_MM_FAKE_ENG, "mm_fake_eng", "mm_sel", 14),
+	GATE_MM0(CLK_MM_MUTEX_32K, "mm_mutex_32k", "rtc_sel", 15),
+	GATE_MM0(CLK_MM_DISP_OVL0, "mm_disp_ovl0", "mm_sel", 16),
+	GATE_MM0(CLK_MM_DISP_OVL1, "mm_disp_ovl1", "mm_sel", 17),
+	GATE_MM0(CLK_MM_DISP_RDMA0, "mm_disp_rdma0", "mm_sel", 18),
+	GATE_MM0(CLK_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 19),
+	GATE_MM0(CLK_MM_DISP_RDMA2, "mm_disp_rdma2", "mm_sel", 20),
+	GATE_MM0(CLK_MM_DISP_WDMA0, "mm_disp_wdma0", "mm_sel", 21),
+	GATE_MM0(CLK_MM_DISP_WDMA1, "mm_disp_wdma1", "mm_sel", 22),
+	GATE_MM0(CLK_MM_DISP_COLOR0, "mm_disp_color0", "mm_sel", 23),
+	GATE_MM0(CLK_MM_DISP_COLOR1, "mm_disp_color1", "mm_sel", 24),
+	GATE_MM0(CLK_MM_DISP_AAL, "mm_disp_aal", "mm_sel", 25),
+	GATE_MM0(CLK_MM_DISP_GAMMA, "mm_disp_gamma", "mm_sel", 26),
+	GATE_MM0(CLK_MM_DISP_UFOE, "mm_disp_ufoe", "mm_sel", 27),
+	GATE_MM0(CLK_MM_DISP_SPLIT0, "mm_disp_split0", "mm_sel", 28),
+	GATE_MM0(CLK_MM_DISP_SPLIT1, "mm_disp_split1", "mm_sel", 29),
+	GATE_MM0(CLK_MM_DISP_MERGE, "mm_disp_merge", "mm_sel", 30),
+	GATE_MM0(CLK_MM_DISP_OD, "mm_disp_od", "mm_sel", 31),
+	/* MM1 */
+	GATE_MM1(CLK_MM_DISP_PWM0MM, "mm_disp_pwm0mm", "mm_sel", 0),
+	GATE_MM1(CLK_MM_DISP_PWM026M, "mm_disp_pwm026m", "pwm_sel", 1),
+	GATE_MM1(CLK_MM_DISP_PWM1MM, "mm_disp_pwm1mm", "mm_sel", 2),
+	GATE_MM1(CLK_MM_DISP_PWM126M, "mm_disp_pwm126m", "pwm_sel", 3),
+	GATE_MM1(CLK_MM_DSI0_ENGINE, "mm_dsi0_engine", "mm_sel", 4),
+	GATE_MM1(CLK_MM_DSI0_DIGITAL, "mm_dsi0_digital", "dsi0_dig", 5),
+	GATE_MM1(CLK_MM_DSI1_ENGINE, "mm_dsi1_engine", "mm_sel", 6),
+	GATE_MM1(CLK_MM_DSI1_DIGITAL, "mm_dsi1_digital", "dsi1_dig", 7),
+	GATE_MM1(CLK_MM_DPI_PIXEL, "mm_dpi_pixel", "dpi0_sel", 8),
+	GATE_MM1(CLK_MM_DPI_ENGINE, "mm_dpi_engine", "mm_sel", 9),
+	GATE_MM1(CLK_MM_DPI1_PIXEL, "mm_dpi1_pixel", "lvds_pxl", 10),
+	GATE_MM1(CLK_MM_DPI1_ENGINE, "mm_dpi1_engine", "mm_sel", 11),
+	GATE_MM1(CLK_MM_HDMI_PIXEL, "mm_hdmi_pixel", "dpi0_sel", 12),
+	GATE_MM1(CLK_MM_HDMI_PLLCK, "mm_hdmi_pllck", "hdmi_sel", 13),
+	GATE_MM1(CLK_MM_HDMI_AUDIO, "mm_hdmi_audio", "apll1", 14),
+	GATE_MM1(CLK_MM_HDMI_SPDIF, "mm_hdmi_spdif", "apll2", 15),
+	GATE_MM1(CLK_MM_LVDS_PIXEL, "mm_lvds_pixel", "lvds_pxl", 16),
+	GATE_MM1(CLK_MM_LVDS_CTS, "mm_lvds_cts", "lvds_cts", 17),
+	GATE_MM1(CLK_MM_SMI_LARB4, "mm_smi_larb4", "mm_sel", 18),
+	GATE_MM1(CLK_MM_HDMI_HDCP, "mm_hdmi_hdcp", "hdcp_sel", 19),
+	GATE_MM1(CLK_MM_HDMI_HDCP24M, "mm_hdmi_hdcp24m", "hdcp_24m_sel", 20),
+};
+
+struct clk_mt8173_mm_driver_data {
+	const struct mtk_gate *gates_clk;
+	int gates_num;
+};
+
+static const struct clk_mt8173_mm_driver_data mt8173_mmsys_driver_data = {
+	.gates_clk = mt8173_mm_clks,
+	.gates_num = ARRAY_SIZE(mt8173_mm_clks),
+};
+
+static int clk_mt8173_mm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->parent->of_node;
+	const struct clk_mt8173_mm_driver_data *data;
+	struct clk_onecell_data *clk_data;
+	int ret;
+
+	clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK);
+	if (!clk_data)
+		return -ENOMEM;
+
+	data = &mt8173_mmsys_driver_data;
+
+	ret = mtk_clk_register_gates(node, data->gates_clk, data->gates_num,
+				     clk_data);
+	if (ret)
+		return ret;
+
+	ret = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct platform_driver clk_mt8173_mm_drv = {
+	.driver = {
+		.name = "clk-mt8173-mm",
+	},
+	.probe = clk_mt8173_mm_probe,
+};
+
+builtin_platform_driver(clk_mt8173_mm_drv);
diff --git a/drivers/clk/mediatek/clk-mt8173.c b/drivers/clk/mediatek/clk-mt8173.c
index 537a7f49b0f7..8f898ac476c0 100644
--- a/drivers/clk/mediatek/clk-mt8173.c
+++ b/drivers/clk/mediatek/clk-mt8173.c
@@ -753,93 +753,6 @@ static const struct mtk_gate img_clks[] __initconst = {
 	GATE_IMG(CLK_IMG_FD, "img_fd", "mm_sel", 11),
 };
 
-static const struct mtk_gate_regs mm0_cg_regs __initconst = {
-	.set_ofs = 0x0104,
-	.clr_ofs = 0x0108,
-	.sta_ofs = 0x0100,
-};
-
-static const struct mtk_gate_regs mm1_cg_regs __initconst = {
-	.set_ofs = 0x0114,
-	.clr_ofs = 0x0118,
-	.sta_ofs = 0x0110,
-};
-
-#define GATE_MM0(_id, _name, _parent, _shift) {			\
-		.id = _id,					\
-		.name = _name,					\
-		.parent_name = _parent,				\
-		.regs = &mm0_cg_regs,				\
-		.shift = _shift,				\
-		.ops = &mtk_clk_gate_ops_setclr,		\
-	}
-
-#define GATE_MM1(_id, _name, _parent, _shift) {			\
-		.id = _id,					\
-		.name = _name,					\
-		.parent_name = _parent,				\
-		.regs = &mm1_cg_regs,				\
-		.shift = _shift,				\
-		.ops = &mtk_clk_gate_ops_setclr,		\
-	}
-
-static const struct mtk_gate mm_clks[] __initconst = {
-	/* MM0 */
-	GATE_MM0(CLK_MM_SMI_COMMON, "mm_smi_common", "mm_sel", 0),
-	GATE_MM0(CLK_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 1),
-	GATE_MM0(CLK_MM_CAM_MDP, "mm_cam_mdp", "mm_sel", 2),
-	GATE_MM0(CLK_MM_MDP_RDMA0, "mm_mdp_rdma0", "mm_sel", 3),
-	GATE_MM0(CLK_MM_MDP_RDMA1, "mm_mdp_rdma1", "mm_sel", 4),
-	GATE_MM0(CLK_MM_MDP_RSZ0, "mm_mdp_rsz0", "mm_sel", 5),
-	GATE_MM0(CLK_MM_MDP_RSZ1, "mm_mdp_rsz1", "mm_sel", 6),
-	GATE_MM0(CLK_MM_MDP_RSZ2, "mm_mdp_rsz2", "mm_sel", 7),
-	GATE_MM0(CLK_MM_MDP_TDSHP0, "mm_mdp_tdshp0", "mm_sel", 8),
-	GATE_MM0(CLK_MM_MDP_TDSHP1, "mm_mdp_tdshp1", "mm_sel", 9),
-	GATE_MM0(CLK_MM_MDP_WDMA, "mm_mdp_wdma", "mm_sel", 11),
-	GATE_MM0(CLK_MM_MDP_WROT0, "mm_mdp_wrot0", "mm_sel", 12),
-	GATE_MM0(CLK_MM_MDP_WROT1, "mm_mdp_wrot1", "mm_sel", 13),
-	GATE_MM0(CLK_MM_FAKE_ENG, "mm_fake_eng", "mm_sel", 14),
-	GATE_MM0(CLK_MM_MUTEX_32K, "mm_mutex_32k", "rtc_sel", 15),
-	GATE_MM0(CLK_MM_DISP_OVL0, "mm_disp_ovl0", "mm_sel", 16),
-	GATE_MM0(CLK_MM_DISP_OVL1, "mm_disp_ovl1", "mm_sel", 17),
-	GATE_MM0(CLK_MM_DISP_RDMA0, "mm_disp_rdma0", "mm_sel", 18),
-	GATE_MM0(CLK_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 19),
-	GATE_MM0(CLK_MM_DISP_RDMA2, "mm_disp_rdma2", "mm_sel", 20),
-	GATE_MM0(CLK_MM_DISP_WDMA0, "mm_disp_wdma0", "mm_sel", 21),
-	GATE_MM0(CLK_MM_DISP_WDMA1, "mm_disp_wdma1", "mm_sel", 22),
-	GATE_MM0(CLK_MM_DISP_COLOR0, "mm_disp_color0", "mm_sel", 23),
-	GATE_MM0(CLK_MM_DISP_COLOR1, "mm_disp_color1", "mm_sel", 24),
-	GATE_MM0(CLK_MM_DISP_AAL, "mm_disp_aal", "mm_sel", 25),
-	GATE_MM0(CLK_MM_DISP_GAMMA, "mm_disp_gamma", "mm_sel", 26),
-	GATE_MM0(CLK_MM_DISP_UFOE, "mm_disp_ufoe", "mm_sel", 27),
-	GATE_MM0(CLK_MM_DISP_SPLIT0, "mm_disp_split0", "mm_sel", 28),
-	GATE_MM0(CLK_MM_DISP_SPLIT1, "mm_disp_split1", "mm_sel", 29),
-	GATE_MM0(CLK_MM_DISP_MERGE, "mm_disp_merge", "mm_sel", 30),
-	GATE_MM0(CLK_MM_DISP_OD, "mm_disp_od", "mm_sel", 31),
-	/* MM1 */
-	GATE_MM1(CLK_MM_DISP_PWM0MM, "mm_disp_pwm0mm", "mm_sel", 0),
-	GATE_MM1(CLK_MM_DISP_PWM026M, "mm_disp_pwm026m", "pwm_sel", 1),
-	GATE_MM1(CLK_MM_DISP_PWM1MM, "mm_disp_pwm1mm", "mm_sel", 2),
-	GATE_MM1(CLK_MM_DISP_PWM126M, "mm_disp_pwm126m", "pwm_sel", 3),
-	GATE_MM1(CLK_MM_DSI0_ENGINE, "mm_dsi0_engine", "mm_sel", 4),
-	GATE_MM1(CLK_MM_DSI0_DIGITAL, "mm_dsi0_digital", "dsi0_dig", 5),
-	GATE_MM1(CLK_MM_DSI1_ENGINE, "mm_dsi1_engine", "mm_sel", 6),
-	GATE_MM1(CLK_MM_DSI1_DIGITAL, "mm_dsi1_digital", "dsi1_dig", 7),
-	GATE_MM1(CLK_MM_DPI_PIXEL, "mm_dpi_pixel", "dpi0_sel", 8),
-	GATE_MM1(CLK_MM_DPI_ENGINE, "mm_dpi_engine", "mm_sel", 9),
-	GATE_MM1(CLK_MM_DPI1_PIXEL, "mm_dpi1_pixel", "lvds_pxl", 10),
-	GATE_MM1(CLK_MM_DPI1_ENGINE, "mm_dpi1_engine", "mm_sel", 11),
-	GATE_MM1(CLK_MM_HDMI_PIXEL, "mm_hdmi_pixel", "dpi0_sel", 12),
-	GATE_MM1(CLK_MM_HDMI_PLLCK, "mm_hdmi_pllck", "hdmi_sel", 13),
-	GATE_MM1(CLK_MM_HDMI_AUDIO, "mm_hdmi_audio", "apll1", 14),
-	GATE_MM1(CLK_MM_HDMI_SPDIF, "mm_hdmi_spdif", "apll2", 15),
-	GATE_MM1(CLK_MM_LVDS_PIXEL, "mm_lvds_pixel", "lvds_pxl", 16),
-	GATE_MM1(CLK_MM_LVDS_CTS, "mm_lvds_cts", "lvds_cts", 17),
-	GATE_MM1(CLK_MM_SMI_LARB4, "mm_smi_larb4", "mm_sel", 18),
-	GATE_MM1(CLK_MM_HDMI_HDCP, "mm_hdmi_hdcp", "hdcp_sel", 19),
-	GATE_MM1(CLK_MM_HDMI_HDCP24M, "mm_hdmi_hdcp24m", "hdcp_24m_sel", 20),
-};
-
 static const struct mtk_gate_regs vdec0_cg_regs __initconst = {
 	.set_ofs = 0x0000,
 	.clr_ofs = 0x0004,
@@ -1144,23 +1057,6 @@ static void __init mtk_imgsys_init(struct device_node *node)
 }
 CLK_OF_DECLARE(mtk_imgsys, "mediatek,mt8173-imgsys", mtk_imgsys_init);
 
-static void __init mtk_mmsys_init(struct device_node *node)
-{
-	struct clk_onecell_data *clk_data;
-	int r;
-
-	clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK);
-
-	mtk_clk_register_gates(node, mm_clks, ARRAY_SIZE(mm_clks),
-						clk_data);
-
-	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
-	if (r)
-		pr_err("%s(): could not register clock provider: %d\n",
-			__func__, r);
-}
-CLK_OF_DECLARE(mtk_mmsys, "mediatek,mt8173-mmsys", mtk_mmsys_init);
-
 static void __init mtk_vdecsys_init(struct device_node *node)
 {
 	struct clk_onecell_data *clk_data;
diff --git a/drivers/clk/mediatek/clk-mt8183-mm.c b/drivers/clk/mediatek/clk-mt8183-mm.c
index 720c696b506d..9d60e09619c1 100644
--- a/drivers/clk/mediatek/clk-mt8183-mm.c
+++ b/drivers/clk/mediatek/clk-mt8183-mm.c
@@ -84,8 +84,9 @@ static const struct mtk_gate mm_clks[] = {
 
 static int clk_mt8183_mm_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->parent->of_node;
 	struct clk_onecell_data *clk_data;
-	struct device_node *node = pdev->dev.of_node;
 
 	clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK);
 
@@ -95,16 +96,10 @@ static int clk_mt8183_mm_probe(struct platform_device *pdev)
 	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
 }
 
-static const struct of_device_id of_match_clk_mt8183_mm[] = {
-	{ .compatible = "mediatek,mt8183-mmsys", },
-	{}
-};
-
 static struct platform_driver clk_mt8183_mm_drv = {
 	.probe = clk_mt8183_mm_probe,
 	.driver = {
 		.name = "clk-mt8183-mm",
-		.of_match_table = of_match_clk_mt8183_mm,
 	},
 };
 
diff --git a/drivers/clk/mediatek/clk-mux.c b/drivers/clk/mediatek/clk-mux.c
index 76f9cd039195..14e127e9a740 100644
--- a/drivers/clk/mediatek/clk-mux.c
+++ b/drivers/clk/mediatek/clk-mux.c
@@ -160,7 +160,7 @@ struct clk *mtk_clk_register_mux(const struct mtk_mux *mux,
 				 spinlock_t *lock)
 {
 	struct mtk_clk_mux *clk_mux;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk *clk;
 
 	clk_mux = kzalloc(sizeof(*clk_mux), GFP_KERNEL);
diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c
index fad616cac01e..30c15766ebb1 100644
--- a/drivers/clk/meson/g12a.c
+++ b/drivers/clk/meson/g12a.c
@@ -3702,7 +3702,9 @@ static struct clk_regmap g12a_hdmi = {
 
 /*
  * The MALI IP is clocked by two identical clocks (mali_0 and mali_1)
- * muxed by a glitch-free switch.
+ * muxed by a glitch-free switch. The CCF can manage this glitch-free
+ * mux because it does top-to-bottom updates the each clock tree and
+ * switches to the "inactive" one when CLK_SET_RATE_GATE is set.
  */
 static const struct clk_parent_data g12a_mali_0_1_parent_data[] = {
 	{ .fw_name = "xtal", },
@@ -3726,7 +3728,13 @@ static struct clk_regmap g12a_mali_0_sel = {
 		.ops = &clk_regmap_mux_ops,
 		.parent_data = g12a_mali_0_1_parent_data,
 		.num_parents = 8,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		/*
+		 * Don't request the parent to change the rate because
+		 * all GPU frequencies can be derived from the fclk_*
+		 * clocks and one special GP0_PLL setting. This is
+		 * important because we need the MPLL clocks for audio.
+		 */
+		.flags = 0,
 	},
 };
 
@@ -3743,7 +3751,7 @@ static struct clk_regmap g12a_mali_0_div = {
 			&g12a_mali_0_sel.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -3759,7 +3767,7 @@ static struct clk_regmap g12a_mali_0 = {
 			&g12a_mali_0_div.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -3774,7 +3782,13 @@ static struct clk_regmap g12a_mali_1_sel = {
 		.ops = &clk_regmap_mux_ops,
 		.parent_data = g12a_mali_0_1_parent_data,
 		.num_parents = 8,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		/*
+		 * Don't request the parent to change the rate because
+		 * all GPU frequencies can be derived from the fclk_*
+		 * clocks and one special GP0_PLL setting. This is
+		 * important because we need the MPLL clocks for audio.
+		 */
+		.flags = 0,
 	},
 };
 
@@ -3791,7 +3805,7 @@ static struct clk_regmap g12a_mali_1_div = {
 			&g12a_mali_1_sel.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -3807,7 +3821,7 @@ static struct clk_regmap g12a_mali_1 = {
 			&g12a_mali_1_div.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -3827,7 +3841,7 @@ static struct clk_regmap g12a_mali = {
 		.ops = &clk_regmap_mux_ops,
 		.parent_hws = g12a_mali_parent_hws,
 		.num_parents = 2,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c
index 5fd6a574f8c3..0a68af6eec3d 100644
--- a/drivers/clk/meson/gxbb.c
+++ b/drivers/clk/meson/gxbb.c
@@ -957,7 +957,9 @@ static struct clk_regmap gxbb_sar_adc_clk = {
 
 /*
  * The MALI IP is clocked by two identical clocks (mali_0 and mali_1)
- * muxed by a glitch-free switch.
+ * muxed by a glitch-free switch. The CCF can manage this glitch-free
+ * mux because it does top-to-bottom updates the each clock tree and
+ * switches to the "inactive" one when CLK_SET_RATE_GATE is set.
  */
 
 static const struct clk_parent_data gxbb_mali_0_1_parent_data[] = {
@@ -980,14 +982,15 @@ static struct clk_regmap gxbb_mali_0_sel = {
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_0_sel",
 		.ops = &clk_regmap_mux_ops,
-		/*
-		 * bits 10:9 selects from 8 possible parents:
-		 * xtal, gp0_pll, mpll2, mpll1, fclk_div7,
-		 * fclk_div4, fclk_div3, fclk_div5
-		 */
 		.parent_data = gxbb_mali_0_1_parent_data,
 		.num_parents = 8,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		/*
+		 * Don't request the parent to change the rate because
+		 * all GPU frequencies can be derived from the fclk_*
+		 * clocks and one special GP0_PLL setting. This is
+		 * important because we need the MPLL clocks for audio.
+		 */
+		.flags = 0,
 	},
 };
 
@@ -1004,7 +1007,7 @@ static struct clk_regmap gxbb_mali_0_div = {
 			&gxbb_mali_0_sel.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -1020,7 +1023,7 @@ static struct clk_regmap gxbb_mali_0 = {
 			&gxbb_mali_0_div.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -1033,14 +1036,15 @@ static struct clk_regmap gxbb_mali_1_sel = {
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_1_sel",
 		.ops = &clk_regmap_mux_ops,
-		/*
-		 * bits 10:9 selects from 8 possible parents:
-		 * xtal, gp0_pll, mpll2, mpll1, fclk_div7,
-		 * fclk_div4, fclk_div3, fclk_div5
-		 */
 		.parent_data = gxbb_mali_0_1_parent_data,
 		.num_parents = 8,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		/*
+		 * Don't request the parent to change the rate because
+		 * all GPU frequencies can be derived from the fclk_*
+		 * clocks and one special GP0_PLL setting. This is
+		 * important because we need the MPLL clocks for audio.
+		 */
+		.flags = 0,
 	},
 };
 
@@ -1057,7 +1061,7 @@ static struct clk_regmap gxbb_mali_1_div = {
 			&gxbb_mali_1_sel.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -1073,7 +1077,7 @@ static struct clk_regmap gxbb_mali_1 = {
 			&gxbb_mali_1_div.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -1093,7 +1097,7 @@ static struct clk_regmap gxbb_mali = {
 		.ops = &clk_regmap_mux_ops,
 		.parent_hws = gxbb_mali_parent_hws,
 		.num_parents = 2,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c
index 34a70c4b4899..edc09d050ecf 100644
--- a/drivers/clk/meson/meson8b.c
+++ b/drivers/clk/meson/meson8b.c
@@ -1077,7 +1077,7 @@ static struct clk_regmap meson8b_vid_pll_in_sel = {
 		 * Meson8m2: vid2_pll
 		 */
 		.parent_hws = (const struct clk_hw *[]) {
-			&meson8b_hdmi_pll_dco.hw
+			&meson8b_hdmi_pll_lvds_out.hw
 		},
 		.num_parents = 1,
 		.flags = CLK_SET_RATE_PARENT,
@@ -1213,7 +1213,7 @@ static struct clk_regmap meson8b_vclk_in_en = {
 
 static struct clk_regmap meson8b_vclk_div1_gate = {
 	.data = &(struct clk_regmap_gate_data){
-		.offset = HHI_VID_CLK_DIV,
+		.offset = HHI_VID_CLK_CNTL,
 		.bit_idx = 0,
 	},
 	.hw.init = &(struct clk_init_data){
@@ -1243,7 +1243,7 @@ static struct clk_fixed_factor meson8b_vclk_div2_div = {
 
 static struct clk_regmap meson8b_vclk_div2_div_gate = {
 	.data = &(struct clk_regmap_gate_data){
-		.offset = HHI_VID_CLK_DIV,
+		.offset = HHI_VID_CLK_CNTL,
 		.bit_idx = 1,
 	},
 	.hw.init = &(struct clk_init_data){
@@ -1273,7 +1273,7 @@ static struct clk_fixed_factor meson8b_vclk_div4_div = {
 
 static struct clk_regmap meson8b_vclk_div4_div_gate = {
 	.data = &(struct clk_regmap_gate_data){
-		.offset = HHI_VID_CLK_DIV,
+		.offset = HHI_VID_CLK_CNTL,
 		.bit_idx = 2,
 	},
 	.hw.init = &(struct clk_init_data){
@@ -1303,7 +1303,7 @@ static struct clk_fixed_factor meson8b_vclk_div6_div = {
 
 static struct clk_regmap meson8b_vclk_div6_div_gate = {
 	.data = &(struct clk_regmap_gate_data){
-		.offset = HHI_VID_CLK_DIV,
+		.offset = HHI_VID_CLK_CNTL,
 		.bit_idx = 3,
 	},
 	.hw.init = &(struct clk_init_data){
@@ -1333,7 +1333,7 @@ static struct clk_fixed_factor meson8b_vclk_div12_div = {
 
 static struct clk_regmap meson8b_vclk_div12_div_gate = {
 	.data = &(struct clk_regmap_gate_data){
-		.offset = HHI_VID_CLK_DIV,
+		.offset = HHI_VID_CLK_CNTL,
 		.bit_idx = 4,
 	},
 	.hw.init = &(struct clk_init_data){
@@ -1725,7 +1725,7 @@ static struct clk_regmap meson8b_hdmi_sys_sel = {
 	},
 	.hw.init = &(struct clk_init_data){
 		.name = "hdmi_sys_sel",
-		.ops = &clk_regmap_mux_ro_ops,
+		.ops = &clk_regmap_mux_ops,
 		/* FIXME: all other parents are unknown */
 		.parent_data = &(const struct clk_parent_data) {
 			.fw_name = "xtal",
@@ -1745,7 +1745,7 @@ static struct clk_regmap meson8b_hdmi_sys_div = {
 	},
 	.hw.init = &(struct clk_init_data){
 		.name = "hdmi_sys_div",
-		.ops = &clk_regmap_divider_ro_ops,
+		.ops = &clk_regmap_divider_ops,
 		.parent_hws = (const struct clk_hw *[]) {
 			&meson8b_hdmi_sys_sel.hw
 		},
@@ -1761,7 +1761,7 @@ static struct clk_regmap meson8b_hdmi_sys = {
 	},
 	.hw.init = &(struct clk_init_data) {
 		.name = "hdmi_sys",
-		.ops = &clk_regmap_gate_ro_ops,
+		.ops = &clk_regmap_gate_ops,
 		.parent_hws = (const struct clk_hw *[]) {
 			&meson8b_hdmi_sys_div.hw
 		},
@@ -1918,6 +1918,13 @@ static struct clk_regmap meson8b_mali = {
 	},
 };
 
+static const struct reg_sequence meson8m2_gp_pll_init_regs[] = {
+	{ .reg = HHI_GP_PLL_CNTL2,	.def = 0x59c88000 },
+	{ .reg = HHI_GP_PLL_CNTL3,	.def = 0xca463823 },
+	{ .reg = HHI_GP_PLL_CNTL4,	.def = 0x0286a027 },
+	{ .reg = HHI_GP_PLL_CNTL5,	.def = 0x00003000 },
+};
+
 static const struct pll_params_table meson8m2_gp_pll_params_table[] = {
 	PLL_PARAMS(182, 3),
 	{ /* sentinel */ },
@@ -1951,6 +1958,8 @@ static struct clk_regmap meson8m2_gp_pll_dco = {
 			.width   = 1,
 		},
 		.table = meson8m2_gp_pll_params_table,
+		.init_regs = meson8m2_gp_pll_init_regs,
+		.init_count = ARRAY_SIZE(meson8m2_gp_pll_init_regs),
 	},
 	.hw.init = &(struct clk_init_data){
 		.name = "gp_pll_dco",
@@ -2063,7 +2072,7 @@ static struct clk_regmap meson8b_vpu_0 = {
 			&meson8b_vpu_0_div.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -2134,10 +2143,18 @@ static struct clk_regmap meson8b_vpu_1 = {
 			&meson8b_vpu_1_div.hw
 		},
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT,
+		.flags = CLK_SET_RATE_GATE | CLK_SET_RATE_PARENT,
 	},
 };
 
+/*
+ * The VPU clock has two two identical clock trees (vpu_0 and vpu_1)
+ * muxed by a glitch-free switch on Meson8b and Meson8m2. The CCF can
+ * actually manage this glitch-free mux because it does top-to-bottom
+ * updates the each clock tree and switches to the "inactive" one when
+ * CLK_SET_RATE_GATE is set.
+ * Meson8 only has vpu_0 and no glitch-free mux.
+ */
 static struct clk_regmap meson8b_vpu = {
 	.data = &(struct clk_regmap_mux_data){
 		.offset = HHI_VPU_CLK_CNTL,
@@ -2152,7 +2169,7 @@ static struct clk_regmap meson8b_vpu = {
 			&meson8b_vpu_1.hw,
 		},
 		.num_parents = 2,
-		.flags = CLK_SET_RATE_NO_REPARENT,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -3506,54 +3523,87 @@ static struct clk_regmap *const meson8b_clk_regmaps[] = {
 static const struct meson8b_clk_reset_line {
 	u32 reg;
 	u8 bit_idx;
+	bool active_low;
 } meson8b_clk_reset_bits[] = {
 	[CLKC_RESET_L2_CACHE_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 30
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 30,
+		.active_low = false,
 	},
 	[CLKC_RESET_AXI_64_TO_128_BRIDGE_A5_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 29
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 29,
+		.active_low = false,
 	},
 	[CLKC_RESET_SCU_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 28
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 28,
+		.active_low = false,
 	},
 	[CLKC_RESET_CPU3_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 27
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 27,
+		.active_low = false,
 	},
 	[CLKC_RESET_CPU2_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 26
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 26,
+		.active_low = false,
 	},
 	[CLKC_RESET_CPU1_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 25
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 25,
+		.active_low = false,
 	},
 	[CLKC_RESET_CPU0_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 24
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 24,
+		.active_low = false,
 	},
 	[CLKC_RESET_A5_GLOBAL_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 18
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 18,
+		.active_low = false,
 	},
 	[CLKC_RESET_A5_AXI_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 17
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 17,
+		.active_low = false,
 	},
 	[CLKC_RESET_A5_ABP_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 16
+		.reg = HHI_SYS_CPU_CLK_CNTL0,
+		.bit_idx = 16,
+		.active_low = false,
 	},
 	[CLKC_RESET_AXI_64_TO_128_BRIDGE_MMC_SOFT_RESET] = {
-		.reg = HHI_SYS_CPU_CLK_CNTL1, .bit_idx = 30
+		.reg = HHI_SYS_CPU_CLK_CNTL1,
+		.bit_idx = 30,
+		.active_low = false,
 	},
 	[CLKC_RESET_VID_CLK_CNTL_SOFT_RESET] = {
-		.reg = HHI_VID_CLK_CNTL, .bit_idx = 15
+		.reg = HHI_VID_CLK_CNTL,
+		.bit_idx = 15,
+		.active_low = false,
 	},
 	[CLKC_RESET_VID_DIVIDER_CNTL_SOFT_RESET_POST] = {
-		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 7
+		.reg = HHI_VID_DIVIDER_CNTL,
+		.bit_idx = 7,
+		.active_low = false,
 	},
 	[CLKC_RESET_VID_DIVIDER_CNTL_SOFT_RESET_PRE] = {
-		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 3
+		.reg = HHI_VID_DIVIDER_CNTL,
+		.bit_idx = 3,
+		.active_low = false,
 	},
 	[CLKC_RESET_VID_DIVIDER_CNTL_RESET_N_POST] = {
-		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 1
+		.reg = HHI_VID_DIVIDER_CNTL,
+		.bit_idx = 1,
+		.active_low = true,
 	},
 	[CLKC_RESET_VID_DIVIDER_CNTL_RESET_N_PRE] = {
-		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 0
+		.reg = HHI_VID_DIVIDER_CNTL,
+		.bit_idx = 0,
+		.active_low = true,
 	},
 };
 
@@ -3562,22 +3612,22 @@ static int meson8b_clk_reset_update(struct reset_controller_dev *rcdev,
 {
 	struct meson8b_clk_reset *meson8b_clk_reset =
 		container_of(rcdev, struct meson8b_clk_reset, reset);
-	unsigned long flags;
 	const struct meson8b_clk_reset_line *reset;
+	unsigned int value = 0;
+	unsigned long flags;
 
 	if (id >= ARRAY_SIZE(meson8b_clk_reset_bits))
 		return -EINVAL;
 
 	reset = &meson8b_clk_reset_bits[id];
 
+	if (assert != reset->active_low)
+		value = BIT(reset->bit_idx);
+
 	spin_lock_irqsave(&meson_clk_lock, flags);
 
-	if (assert)
-		regmap_update_bits(meson8b_clk_reset->regmap, reset->reg,
-				   BIT(reset->bit_idx), BIT(reset->bit_idx));
-	else
-		regmap_update_bits(meson8b_clk_reset->regmap, reset->reg,
-				   BIT(reset->bit_idx), 0);
+	regmap_update_bits(meson8b_clk_reset->regmap, reset->reg,
+			   BIT(reset->bit_idx), value);
 
 	spin_unlock_irqrestore(&meson_clk_lock, flags);
 
diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h
index c889fbeec30f..cd38ae2a9cb5 100644
--- a/drivers/clk/meson/meson8b.h
+++ b/drivers/clk/meson/meson8b.h
@@ -20,6 +20,10 @@
  * [0] http://dn.odroid.com/S805/Datasheet/S805_Datasheet%20V0.8%2020150126.pdf
  */
 #define HHI_GP_PLL_CNTL			0x40  /* 0x10 offset in data sheet */
+#define HHI_GP_PLL_CNTL2		0x44  /* 0x11 offset in data sheet */
+#define HHI_GP_PLL_CNTL3		0x48  /* 0x12 offset in data sheet */
+#define HHI_GP_PLL_CNTL4		0x4C  /* 0x13 offset in data sheet */
+#define HHI_GP_PLL_CNTL5		0x50  /* 0x14 offset in data sheet */
 #define HHI_VIID_CLK_DIV		0x128 /* 0x4a offset in data sheet */
 #define HHI_VIID_CLK_CNTL		0x12c /* 0x4b offset in data sheet */
 #define HHI_GCLK_MPEG0			0x140 /* 0x50 offset in data sheet */
@@ -146,7 +150,6 @@
 #define CLKID_CTS_VDAC0		171
 #define CLKID_HDMI_SYS_SEL	172
 #define CLKID_HDMI_SYS_DIV	173
-#define CLKID_HDMI_SYS		174
 #define CLKID_MALI_0_SEL	175
 #define CLKID_MALI_0_DIV	176
 #define CLKID_MALI_0		177
diff --git a/drivers/clk/mmp/Makefile b/drivers/clk/mmp/Makefile
index 14dc8a8a9d08..cbcc2f8430a2 100644
--- a/drivers/clk/mmp/Makefile
+++ b/drivers/clk/mmp/Makefile
@@ -8,7 +8,8 @@ obj-y += clk-apbc.o clk-apmu.o clk-frac.o clk-mix.o clk-gate.o clk.o
 obj-$(CONFIG_RESET_CONTROLLER) += reset.o
 
 obj-$(CONFIG_MACH_MMP_DT) += clk-of-pxa168.o clk-of-pxa910.o
-obj-$(CONFIG_COMMON_CLK_MMP2) += clk-of-mmp2.o clk-pll.o
+obj-$(CONFIG_COMMON_CLK_MMP2) += clk-of-mmp2.o clk-pll.o pwr-island.o
+obj-$(CONFIG_COMMON_CLK_MMP2_AUDIO) += clk-audio.o
 
 obj-$(CONFIG_CPU_PXA168) += clk-pxa168.o
 obj-$(CONFIG_CPU_PXA910) += clk-pxa910.o
diff --git a/drivers/clk/mmp/clk-audio.c b/drivers/clk/mmp/clk-audio.c
new file mode 100644
index 000000000000..eea69d498bd2
--- /dev/null
+++ b/drivers/clk/mmp/clk-audio.c
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * MMP Audio Clock Controller driver
+ *
+ * Copyright (C) 2020 Lubomir Rintel <lkundrak@v3.sk>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_clock.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <dt-bindings/clock/marvell,mmp2-audio.h>
+
+/* Audio Controller Registers */
+#define SSPA_AUD_CTRL				0x04
+#define SSPA_AUD_PLL_CTRL0			0x08
+#define SSPA_AUD_PLL_CTRL1			0x0c
+
+/* SSPA Audio Control Register */
+#define SSPA_AUD_CTRL_SYSCLK_SHIFT		0
+#define SSPA_AUD_CTRL_SYSCLK_DIV_SHIFT		1
+#define SSPA_AUD_CTRL_SSPA0_MUX_SHIFT		7
+#define SSPA_AUD_CTRL_SSPA0_SHIFT		8
+#define SSPA_AUD_CTRL_SSPA0_DIV_SHIFT		9
+#define SSPA_AUD_CTRL_SSPA1_SHIFT		16
+#define SSPA_AUD_CTRL_SSPA1_DIV_SHIFT		17
+#define SSPA_AUD_CTRL_SSPA1_MUX_SHIFT		23
+#define SSPA_AUD_CTRL_DIV_MASK			0x7e
+
+/* SSPA Audio PLL Control 0 Register */
+#define SSPA_AUD_PLL_CTRL0_DIV_OCLK_MODULO_MASK (0x7 << 28)
+#define SSPA_AUD_PLL_CTRL0_DIV_OCLK_MODULO(x)	((x) << 28)
+#define SSPA_AUD_PLL_CTRL0_FRACT_MASK		(0xfffff << 8)
+#define SSPA_AUD_PLL_CTRL0_FRACT(x)		((x) << 8)
+#define SSPA_AUD_PLL_CTRL0_ENA_DITHER		(1 << 7)
+#define SSPA_AUD_PLL_CTRL0_ICP_2UA		(0 << 5)
+#define SSPA_AUD_PLL_CTRL0_ICP_5UA		(1 << 5)
+#define SSPA_AUD_PLL_CTRL0_ICP_7UA		(2 << 5)
+#define SSPA_AUD_PLL_CTRL0_ICP_10UA		(3 << 5)
+#define SSPA_AUD_PLL_CTRL0_DIV_FBCCLK_MASK	(0x3 << 3)
+#define SSPA_AUD_PLL_CTRL0_DIV_FBCCLK(x)	((x) << 3)
+#define SSPA_AUD_PLL_CTRL0_DIV_MCLK_MASK	(0x1 << 2)
+#define SSPA_AUD_PLL_CTRL0_DIV_MCLK(x)		((x) << 2)
+#define SSPA_AUD_PLL_CTRL0_PD_OVPROT_DIS	(1 << 1)
+#define SSPA_AUD_PLL_CTRL0_PU			(1 << 0)
+
+/* SSPA Audio PLL Control 1 Register */
+#define SSPA_AUD_PLL_CTRL1_SEL_FAST_CLK		(1 << 24)
+#define SSPA_AUD_PLL_CTRL1_CLK_SEL_MASK		(1 << 11)
+#define SSPA_AUD_PLL_CTRL1_CLK_SEL_AUDIO_PLL	(1 << 11)
+#define SSPA_AUD_PLL_CTRL1_CLK_SEL_VCXO		(0 << 11)
+#define SSPA_AUD_PLL_CTRL1_DIV_OCLK_PATTERN_MASK (0x7ff << 0)
+#define SSPA_AUD_PLL_CTRL1_DIV_OCLK_PATTERN(x)	((x) << 0)
+
+struct mmp2_audio_clk {
+	void __iomem *mmio_base;
+
+	struct clk_hw audio_pll_hw;
+	struct clk_mux sspa_mux;
+	struct clk_mux sspa1_mux;
+	struct clk_divider sysclk_div;
+	struct clk_divider sspa0_div;
+	struct clk_divider sspa1_div;
+	struct clk_gate sysclk_gate;
+	struct clk_gate sspa0_gate;
+	struct clk_gate sspa1_gate;
+
+	u32 aud_ctrl;
+	u32 aud_pll_ctrl0;
+	u32 aud_pll_ctrl1;
+
+	spinlock_t lock;
+
+	/* Must be last */
+	struct clk_hw_onecell_data clk_data;
+};
+
+static const struct {
+	unsigned long parent_rate;
+	unsigned long freq_vco;
+	unsigned char mclk;
+	unsigned char fbcclk;
+	unsigned short fract;
+} predivs[] = {
+	{ 26000000, 135475200, 0, 0, 0x8a18 },
+	{ 26000000, 147456000, 0, 1, 0x0da1 },
+	{ 38400000, 135475200, 1, 2, 0x8208 },
+	{ 38400000, 147456000, 1, 3, 0xaaaa },
+};
+
+static const struct {
+	unsigned char divisor;
+	unsigned char modulo;
+	unsigned char pattern;
+} postdivs[] = {
+	{   1,	3,  0, },
+	{   2,	5,  0, },
+	{   4,	0,  0, },
+	{   6,	1,  1, },
+	{   8,	1,  0, },
+	{   9,	1,  2, },
+	{  12,	2,  1, },
+	{  16,	2,  0, },
+	{  18,	2,  2, },
+	{  24,	4,  1, },
+	{  36,	4,  2, },
+	{  48,	6,  1, },
+	{  72,	6,  2, },
+};
+
+static unsigned long audio_pll_recalc_rate(struct clk_hw *hw,
+					   unsigned long parent_rate)
+{
+	struct mmp2_audio_clk *priv = container_of(hw, struct mmp2_audio_clk, audio_pll_hw);
+	unsigned int prediv;
+	unsigned int postdiv;
+	u32 aud_pll_ctrl0;
+	u32 aud_pll_ctrl1;
+
+	aud_pll_ctrl0 = readl(priv->mmio_base + SSPA_AUD_PLL_CTRL0);
+	aud_pll_ctrl0 &= SSPA_AUD_PLL_CTRL0_DIV_OCLK_MODULO_MASK |
+			 SSPA_AUD_PLL_CTRL0_FRACT_MASK |
+			 SSPA_AUD_PLL_CTRL0_ENA_DITHER |
+			 SSPA_AUD_PLL_CTRL0_DIV_FBCCLK_MASK |
+			 SSPA_AUD_PLL_CTRL0_DIV_MCLK_MASK |
+			 SSPA_AUD_PLL_CTRL0_PU;
+
+	aud_pll_ctrl1 = readl(priv->mmio_base + SSPA_AUD_PLL_CTRL1);
+	aud_pll_ctrl1 &= SSPA_AUD_PLL_CTRL1_CLK_SEL_MASK |
+			 SSPA_AUD_PLL_CTRL1_DIV_OCLK_PATTERN_MASK;
+
+	for (prediv = 0; prediv < ARRAY_SIZE(predivs); prediv++) {
+		if (predivs[prediv].parent_rate != parent_rate)
+			continue;
+		for (postdiv = 0; postdiv < ARRAY_SIZE(postdivs); postdiv++) {
+			unsigned long freq;
+			u32 val;
+
+			val = SSPA_AUD_PLL_CTRL0_ENA_DITHER;
+			val |= SSPA_AUD_PLL_CTRL0_PU;
+			val |= SSPA_AUD_PLL_CTRL0_DIV_OCLK_MODULO(postdivs[postdiv].modulo);
+			val |= SSPA_AUD_PLL_CTRL0_FRACT(predivs[prediv].fract);
+			val |= SSPA_AUD_PLL_CTRL0_DIV_FBCCLK(predivs[prediv].fbcclk);
+			val |= SSPA_AUD_PLL_CTRL0_DIV_MCLK(predivs[prediv].mclk);
+			if (val != aud_pll_ctrl0)
+				continue;
+
+			val = SSPA_AUD_PLL_CTRL1_CLK_SEL_AUDIO_PLL;
+			val |= SSPA_AUD_PLL_CTRL1_DIV_OCLK_PATTERN(postdivs[postdiv].pattern);
+			if (val != aud_pll_ctrl1)
+				continue;
+
+			freq = predivs[prediv].freq_vco;
+			freq /= postdivs[postdiv].divisor;
+			return freq;
+		}
+	}
+
+	return 0;
+}
+
+static long audio_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				 unsigned long *parent_rate)
+{
+	unsigned int prediv;
+	unsigned int postdiv;
+	long rounded = 0;
+
+	for (prediv = 0; prediv < ARRAY_SIZE(predivs); prediv++) {
+		if (predivs[prediv].parent_rate != *parent_rate)
+			continue;
+		for (postdiv = 0; postdiv < ARRAY_SIZE(postdivs); postdiv++) {
+			long freq = predivs[prediv].freq_vco;
+
+			freq /= postdivs[postdiv].divisor;
+			if (freq == rate)
+				return rate;
+			if (freq < rate)
+				continue;
+			if (rounded && freq > rounded)
+				continue;
+			rounded = freq;
+		}
+	}
+
+	return rounded;
+}
+
+static int audio_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+			      unsigned long parent_rate)
+{
+	struct mmp2_audio_clk *priv = container_of(hw, struct mmp2_audio_clk, audio_pll_hw);
+	unsigned int prediv;
+	unsigned int postdiv;
+	unsigned long val;
+
+	for (prediv = 0; prediv < ARRAY_SIZE(predivs); prediv++) {
+		if (predivs[prediv].parent_rate != parent_rate)
+			continue;
+
+		for (postdiv = 0; postdiv < ARRAY_SIZE(postdivs); postdiv++) {
+			if (rate * postdivs[postdiv].divisor != predivs[prediv].freq_vco)
+				continue;
+
+			val = SSPA_AUD_PLL_CTRL0_ENA_DITHER;
+			val |= SSPA_AUD_PLL_CTRL0_PU;
+			val |= SSPA_AUD_PLL_CTRL0_DIV_OCLK_MODULO(postdivs[postdiv].modulo);
+			val |= SSPA_AUD_PLL_CTRL0_FRACT(predivs[prediv].fract);
+			val |= SSPA_AUD_PLL_CTRL0_DIV_FBCCLK(predivs[prediv].fbcclk);
+			val |= SSPA_AUD_PLL_CTRL0_DIV_MCLK(predivs[prediv].mclk);
+			writel(val, priv->mmio_base + SSPA_AUD_PLL_CTRL0);
+
+			val = SSPA_AUD_PLL_CTRL1_CLK_SEL_AUDIO_PLL;
+			val |= SSPA_AUD_PLL_CTRL1_DIV_OCLK_PATTERN(postdivs[postdiv].pattern);
+			writel(val, priv->mmio_base + SSPA_AUD_PLL_CTRL1);
+
+			return 0;
+		}
+	}
+
+	return -ERANGE;
+}
+
+static const struct clk_ops audio_pll_ops = {
+	.recalc_rate = audio_pll_recalc_rate,
+	.round_rate = audio_pll_round_rate,
+	.set_rate = audio_pll_set_rate,
+};
+
+static int register_clocks(struct mmp2_audio_clk *priv, struct device *dev)
+{
+	const struct clk_parent_data sspa_mux_parents[] = {
+		{ .hw = &priv->audio_pll_hw },
+		{ .fw_name = "i2s0" },
+	};
+	const struct clk_parent_data sspa1_mux_parents[] = {
+		{ .hw = &priv->audio_pll_hw },
+		{ .fw_name = "i2s1" },
+	};
+	int ret;
+
+	priv->audio_pll_hw.init = CLK_HW_INIT_FW_NAME("audio_pll",
+				"vctcxo", &audio_pll_ops,
+				CLK_SET_RATE_PARENT);
+	ret = devm_clk_hw_register(dev, &priv->audio_pll_hw);
+	if (ret)
+		return ret;
+
+	priv->sspa_mux.hw.init = CLK_HW_INIT_PARENTS_DATA("sspa_mux",
+				sspa_mux_parents, &clk_mux_ops,
+				CLK_SET_RATE_PARENT);
+	priv->sspa_mux.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sspa_mux.mask = 1;
+	priv->sspa_mux.shift = SSPA_AUD_CTRL_SSPA0_MUX_SHIFT;
+	ret = devm_clk_hw_register(dev, &priv->sspa_mux.hw);
+	if (ret)
+		return ret;
+
+	priv->sysclk_div.hw.init = CLK_HW_INIT_HW("sys_div",
+				&priv->sspa_mux.hw, &clk_divider_ops,
+				CLK_SET_RATE_PARENT);
+	priv->sysclk_div.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sysclk_div.shift = SSPA_AUD_CTRL_SYSCLK_DIV_SHIFT;
+	priv->sysclk_div.width = 6;
+	priv->sysclk_div.flags = CLK_DIVIDER_ONE_BASED;
+	priv->sysclk_div.flags |= CLK_DIVIDER_ROUND_CLOSEST;
+	priv->sysclk_div.flags |= CLK_DIVIDER_ALLOW_ZERO;
+	ret = devm_clk_hw_register(dev, &priv->sysclk_div.hw);
+	if (ret)
+		return ret;
+
+	priv->sysclk_gate.hw.init = CLK_HW_INIT_HW("sys_clk",
+				&priv->sysclk_div.hw, &clk_gate_ops,
+				CLK_SET_RATE_PARENT);
+	priv->sysclk_gate.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sysclk_gate.bit_idx = SSPA_AUD_CTRL_SYSCLK_SHIFT;
+	ret = devm_clk_hw_register(dev, &priv->sysclk_gate.hw);
+	if (ret)
+		return ret;
+
+	priv->sspa0_div.hw.init = CLK_HW_INIT_HW("sspa0_div",
+				&priv->sspa_mux.hw, &clk_divider_ops, 0);
+	priv->sspa0_div.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sspa0_div.shift = SSPA_AUD_CTRL_SSPA0_DIV_SHIFT;
+	priv->sspa0_div.width = 6;
+	priv->sspa0_div.flags = CLK_DIVIDER_ONE_BASED;
+	priv->sspa0_div.flags |= CLK_DIVIDER_ROUND_CLOSEST;
+	priv->sspa0_div.flags |= CLK_DIVIDER_ALLOW_ZERO;
+	ret = devm_clk_hw_register(dev, &priv->sspa0_div.hw);
+	if (ret)
+		return ret;
+
+	priv->sspa0_gate.hw.init = CLK_HW_INIT_HW("sspa0_clk",
+				&priv->sspa0_div.hw, &clk_gate_ops,
+				CLK_SET_RATE_PARENT);
+	priv->sspa0_gate.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sspa0_gate.bit_idx = SSPA_AUD_CTRL_SSPA0_SHIFT;
+	ret = devm_clk_hw_register(dev, &priv->sspa0_gate.hw);
+	if (ret)
+		return ret;
+
+	priv->sspa1_mux.hw.init = CLK_HW_INIT_PARENTS_DATA("sspa1_mux",
+				sspa1_mux_parents, &clk_mux_ops,
+				CLK_SET_RATE_PARENT);
+	priv->sspa1_mux.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sspa1_mux.mask = 1;
+	priv->sspa1_mux.shift = SSPA_AUD_CTRL_SSPA1_MUX_SHIFT;
+	ret = devm_clk_hw_register(dev, &priv->sspa1_mux.hw);
+	if (ret)
+		return ret;
+
+	priv->sspa1_div.hw.init = CLK_HW_INIT_HW("sspa1_div",
+				&priv->sspa1_mux.hw, &clk_divider_ops, 0);
+	priv->sspa1_div.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sspa1_div.shift = SSPA_AUD_CTRL_SSPA1_DIV_SHIFT;
+	priv->sspa1_div.width = 6;
+	priv->sspa1_div.flags = CLK_DIVIDER_ONE_BASED;
+	priv->sspa1_div.flags |= CLK_DIVIDER_ROUND_CLOSEST;
+	priv->sspa1_div.flags |= CLK_DIVIDER_ALLOW_ZERO;
+	ret = devm_clk_hw_register(dev, &priv->sspa1_div.hw);
+	if (ret)
+		return ret;
+
+	priv->sspa1_gate.hw.init = CLK_HW_INIT_HW("sspa1_clk",
+				&priv->sspa1_div.hw, &clk_gate_ops,
+				CLK_SET_RATE_PARENT);
+	priv->sspa1_gate.reg = priv->mmio_base + SSPA_AUD_CTRL;
+	priv->sspa1_gate.bit_idx = SSPA_AUD_CTRL_SSPA1_SHIFT;
+	ret = devm_clk_hw_register(dev, &priv->sspa1_gate.hw);
+	if (ret)
+		return ret;
+
+	priv->clk_data.hws[MMP2_CLK_AUDIO_SYSCLK] = &priv->sysclk_gate.hw;
+	priv->clk_data.hws[MMP2_CLK_AUDIO_SSPA0] = &priv->sspa0_gate.hw;
+	priv->clk_data.hws[MMP2_CLK_AUDIO_SSPA1] = &priv->sspa1_gate.hw;
+	priv->clk_data.num = MMP2_CLK_AUDIO_NR_CLKS;
+
+	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
+				      &priv->clk_data);
+}
+
+static int mmp2_audio_clk_probe(struct platform_device *pdev)
+{
+	struct mmp2_audio_clk *priv;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev,
+			    struct_size(priv, clk_data.hws,
+					MMP2_CLK_AUDIO_NR_CLKS),
+			    GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	spin_lock_init(&priv->lock);
+	platform_set_drvdata(pdev, priv);
+
+	priv->mmio_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(priv->mmio_base))
+		return PTR_ERR(priv->mmio_base);
+
+	pm_runtime_enable(&pdev->dev);
+	ret = pm_clk_create(&pdev->dev);
+	if (ret)
+		goto disable_pm_runtime;
+
+	ret = pm_clk_add(&pdev->dev, "audio");
+	if (ret)
+		goto destroy_pm_clk;
+
+	ret = register_clocks(priv, &pdev->dev);
+	if (ret)
+		goto destroy_pm_clk;
+
+	return 0;
+
+destroy_pm_clk:
+	pm_clk_destroy(&pdev->dev);
+disable_pm_runtime:
+	pm_runtime_disable(&pdev->dev);
+
+	return ret;
+}
+
+static int mmp2_audio_clk_remove(struct platform_device *pdev)
+{
+	pm_clk_destroy(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static int __maybe_unused mmp2_audio_clk_suspend(struct device *dev)
+{
+	struct mmp2_audio_clk *priv = dev_get_drvdata(dev);
+
+	priv->aud_ctrl = readl(priv->mmio_base + SSPA_AUD_CTRL);
+	priv->aud_pll_ctrl0 = readl(priv->mmio_base + SSPA_AUD_PLL_CTRL0);
+	priv->aud_pll_ctrl1 = readl(priv->mmio_base + SSPA_AUD_PLL_CTRL1);
+	pm_clk_suspend(dev);
+
+	return 0;
+}
+
+static int __maybe_unused mmp2_audio_clk_resume(struct device *dev)
+{
+	struct mmp2_audio_clk *priv = dev_get_drvdata(dev);
+
+	pm_clk_resume(dev);
+	writel(priv->aud_ctrl, priv->mmio_base + SSPA_AUD_CTRL);
+	writel(priv->aud_pll_ctrl0, priv->mmio_base + SSPA_AUD_PLL_CTRL0);
+	writel(priv->aud_pll_ctrl1, priv->mmio_base + SSPA_AUD_PLL_CTRL1);
+
+	return 0;
+}
+
+static const struct dev_pm_ops mmp2_audio_clk_pm_ops = {
+	SET_RUNTIME_PM_OPS(mmp2_audio_clk_suspend, mmp2_audio_clk_resume, NULL)
+};
+
+static const struct of_device_id mmp2_audio_clk_of_match[] = {
+	{ .compatible = "marvell,mmp2-audio-clock" },
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, mmp2_audio_clk_of_match);
+
+static struct platform_driver mmp2_audio_clk_driver = {
+	.driver = {
+		.name = "mmp2-audio-clock",
+		.of_match_table = of_match_ptr(mmp2_audio_clk_of_match),
+		.pm = &mmp2_audio_clk_pm_ops,
+	},
+	.probe = mmp2_audio_clk_probe,
+	.remove = mmp2_audio_clk_remove,
+};
+module_platform_driver(mmp2_audio_clk_driver);
+
+MODULE_AUTHOR("Lubomir Rintel <lkundrak@v3.sk>");
+MODULE_DESCRIPTION("Clock driver for MMP2 Audio subsystem");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/mmp/clk-frac.c b/drivers/clk/mmp/clk-frac.c
index fabc09aca6c4..48f592bd633d 100644
--- a/drivers/clk/mmp/clk-frac.c
+++ b/drivers/clk/mmp/clk-frac.c
@@ -28,13 +28,15 @@ static long clk_factor_round_rate(struct clk_hw *hw, unsigned long drate,
 		unsigned long *prate)
 {
 	struct mmp_clk_factor *factor = to_clk_factor(hw);
-	unsigned long rate = 0, prev_rate;
+	u64 rate = 0, prev_rate;
 	int i;
 
 	for (i = 0; i < factor->ftbl_cnt; i++) {
 		prev_rate = rate;
-		rate = (((*prate / 10000) * factor->ftbl[i].den) /
-			(factor->ftbl[i].num * factor->masks->factor)) * 10000;
+		rate = *prate;
+		rate *= factor->ftbl[i].den;
+		do_div(rate, factor->ftbl[i].num * factor->masks->factor);
+
 		if (rate > drate)
 			break;
 	}
@@ -54,6 +56,7 @@ static unsigned long clk_factor_recalc_rate(struct clk_hw *hw,
 	struct mmp_clk_factor *factor = to_clk_factor(hw);
 	struct mmp_clk_factor_masks *masks = factor->masks;
 	unsigned int val, num, den;
+	u64 rate;
 
 	val = readl_relaxed(factor->base);
 
@@ -66,8 +69,11 @@ static unsigned long clk_factor_recalc_rate(struct clk_hw *hw,
 	if (!den)
 		return 0;
 
-	return (((parent_rate / 10000)  * den) /
-			(num * factor->masks->factor)) * 10000;
+	rate = parent_rate;
+	rate *= den;
+	do_div(rate, num * factor->masks->factor);
+
+	return rate;
 }
 
 /* Configures new clock rate*/
@@ -78,12 +84,14 @@ static int clk_factor_set_rate(struct clk_hw *hw, unsigned long drate,
 	struct mmp_clk_factor_masks *masks = factor->masks;
 	int i;
 	unsigned long val;
-	unsigned long rate = 0;
 	unsigned long flags = 0;
+	u64 rate = 0;
 
 	for (i = 0; i < factor->ftbl_cnt; i++) {
-		rate = (((prate / 10000) * factor->ftbl[i].den) /
-			(factor->ftbl[i].num * factor->masks->factor)) * 10000;
+		rate = prate;
+		rate *= factor->ftbl[i].den;
+		do_div(rate, factor->ftbl[i].num * factor->masks->factor);
+
 		if (rate > drate)
 			break;
 	}
@@ -140,7 +148,10 @@ static int clk_factor_init(struct clk_hw *hw)
 		val &= ~(masks->den_mask << masks->den_shift);
 		val |= (factor->ftbl[0].den & masks->den_mask) <<
 			masks->den_shift;
+	}
 
+	if (!(val & masks->enable_mask) || i >= factor->ftbl_cnt) {
+		val |= masks->enable_mask;
 		writel(val, factor->base);
 	}
 
diff --git a/drivers/clk/mmp/clk-of-mmp2.c b/drivers/clk/mmp/clk-of-mmp2.c
index 52dc8b43acd9..67208aea94c5 100644
--- a/drivers/clk/mmp/clk-of-mmp2.c
+++ b/drivers/clk/mmp/clk-of-mmp2.c
@@ -17,8 +17,10 @@
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/of_address.h>
+#include <linux/clk.h>
 
 #include <dt-bindings/clock/marvell,mmp2.h>
+#include <dt-bindings/power/marvell,mmp2.h>
 
 #include "clk.h"
 #include "reset.h"
@@ -45,6 +47,10 @@
 #define APBC_SSP1	0x54
 #define APBC_SSP2	0x58
 #define APBC_SSP3	0x5c
+#define APBC_THERMAL0	0x90
+#define APBC_THERMAL1	0x98
+#define APBC_THERMAL2	0x9c
+#define APBC_THERMAL3	0xa0
 #define APMU_SDH0	0x54
 #define APMU_SDH1	0x58
 #define APMU_SDH2	0xe8
@@ -55,18 +61,19 @@
 #define APMU_DISP1	0x110
 #define APMU_CCIC0	0x50
 #define APMU_CCIC1	0xf4
-#define APBC_THERMAL0	0x90
-#define APBC_THERMAL1	0x98
-#define APBC_THERMAL2	0x9c
-#define APBC_THERMAL3	0xa0
 #define APMU_USBHSIC0	0xf8
 #define APMU_USBHSIC1	0xfc
 #define APMU_GPU	0xcc
+#define APMU_AUDIO	0x10c
+#define APMU_CAMERA	0x1fc
 
 #define MPMU_FCCR		0x8
 #define MPMU_POSR		0x10
 #define MPMU_UART_PLL		0x14
 #define MPMU_PLL2_CR		0x34
+#define MPMU_I2S0_PLL		0x40
+#define MPMU_I2S1_PLL		0x44
+#define MPMU_ACGR		0x1024
 /* MMP3 specific below */
 #define MPMU_PLL3_CR		0x50
 #define MPMU_PLL3_CTRL1		0x58
@@ -82,6 +89,8 @@ enum mmp2_clk_model {
 struct mmp2_clk_unit {
 	struct mmp_clk_unit unit;
 	enum mmp2_clk_model model;
+	struct genpd_onecell_data pd_data;
+	struct generic_pm_domain *pm_domains[MMP2_NR_POWER_DOMAINS];
 	void __iomem *mpmu_base;
 	void __iomem *apmu_base;
 	void __iomem *apbc_base;
@@ -91,6 +100,7 @@ static struct mmp_param_fixed_rate_clk fixed_rate_clks[] = {
 	{MMP2_CLK_CLK32, "clk32", NULL, 0, 32768},
 	{MMP2_CLK_VCTCXO, "vctcxo", NULL, 0, 26000000},
 	{MMP2_CLK_USB_PLL, "usb_pll", NULL, 0, 480000000},
+	{0, "i2s_pll", NULL, 0, 99666667},
 };
 
 static struct mmp_param_pll_clk pll_clks[] = {
@@ -139,7 +149,35 @@ static struct mmp_clk_factor_tbl uart_factor_tbl[] = {
 	{.num = 3521, .den = 689},	/*19.23MHZ */
 };
 
-static void mmp2_pll_init(struct mmp2_clk_unit *pxa_unit)
+static struct mmp_clk_factor_masks i2s_factor_masks = {
+	.factor = 2,
+	.num_mask = 0x7fff,
+	.den_mask = 0x1fff,
+	.num_shift = 0,
+	.den_shift = 15,
+	.enable_mask = 0xd0000000,
+};
+
+static struct mmp_clk_factor_tbl i2s_factor_tbl[] = {
+	{.num = 24868, .den =  511},	/*  2.0480 MHz */
+	{.num = 28003, .den =  793},	/*  2.8224 MHz */
+	{.num = 24941, .den = 1025},	/*  4.0960 MHz */
+	{.num = 28003, .den = 1586},	/*  5.6448 MHz */
+	{.num = 31158, .den = 2561},	/*  8.1920 MHz */
+	{.num = 16288, .den = 1845},	/* 11.2896 MHz */
+	{.num = 20772, .den = 2561},	/* 12.2880 MHz */
+	{.num =  8144, .den = 1845},	/* 22.5792 MHz */
+	{.num = 10386, .den = 2561},	/* 24.5760 MHz */
+};
+
+static DEFINE_SPINLOCK(acgr_lock);
+
+static struct mmp_param_gate_clk mpmu_gate_clks[] = {
+	{MMP2_CLK_I2S0, "i2s0_clk", "i2s0_pll", CLK_SET_RATE_PARENT, MPMU_ACGR, 0x200000, 0x200000, 0x0, 0, &acgr_lock},
+	{MMP2_CLK_I2S1, "i2s1_clk", "i2s1_pll", CLK_SET_RATE_PARENT, MPMU_ACGR, 0x100000, 0x100000, 0x0, 0, &acgr_lock},
+};
+
+static void mmp2_main_clk_init(struct mmp2_clk_unit *pxa_unit)
 {
 	struct clk *clk;
 	struct mmp_clk_unit *unit = &pxa_unit->unit;
@@ -166,6 +204,20 @@ static void mmp2_pll_init(struct mmp2_clk_unit *pxa_unit)
 				&uart_factor_masks, uart_factor_tbl,
 				ARRAY_SIZE(uart_factor_tbl), NULL);
 	mmp_clk_add(unit, MMP2_CLK_UART_PLL, clk);
+
+	mmp_clk_register_factor("i2s0_pll", "pll1_4",
+				CLK_SET_RATE_PARENT,
+				pxa_unit->mpmu_base + MPMU_I2S0_PLL,
+				&i2s_factor_masks, i2s_factor_tbl,
+				ARRAY_SIZE(i2s_factor_tbl), NULL);
+	mmp_clk_register_factor("i2s1_pll", "pll1_4",
+				CLK_SET_RATE_PARENT,
+				pxa_unit->mpmu_base + MPMU_I2S1_PLL,
+				&i2s_factor_masks, i2s_factor_tbl,
+				ARRAY_SIZE(i2s_factor_tbl), NULL);
+
+	mmp_register_gate_clks(unit, mpmu_gate_clks, pxa_unit->mpmu_base,
+				ARRAY_SIZE(mpmu_gate_clks));
 }
 
 static DEFINE_SPINLOCK(uart0_lock);
@@ -271,6 +323,8 @@ static u32 mmp2_gpu_bus_parent_table[] =         { 0x0000,   0x0020,   0x0030,
 static const char * const mmp3_gpu_bus_parent_names[] = {"pll1_4", "pll1_6", "pll1_2", "pll2_2"};
 static const char * const mmp3_gpu_gc_parent_names[] =  {"pll1",   "pll2",   "pll1_p", "pll2_p"};
 
+static DEFINE_SPINLOCK(audio_lock);
+
 static struct mmp_clk_mix_config ccic0_mix_config = {
 	.reg_info = DEFINE_MIX_REG_INFO(4, 17, 2, 6, 32),
 };
@@ -326,6 +380,7 @@ static struct mmp_param_gate_clk apmu_gate_clks[] = {
 	{MMP2_CLK_CCIC1_PHY, "ccic1_phy_clk", "ccic1_mix_clk", CLK_SET_RATE_PARENT, APMU_CCIC1, 0x24, 0x24, 0x0, 0, &ccic1_lock},
 	{MMP2_CLK_CCIC1_SPHY, "ccic1_sphy_clk", "ccic1_sphy_div", CLK_SET_RATE_PARENT, APMU_CCIC1, 0x300, 0x300, 0x0, 0, &ccic1_lock},
 	{MMP2_CLK_GPU_BUS, "gpu_bus_clk", "gpu_bus_mux", CLK_SET_RATE_PARENT, APMU_GPU, 0xa, 0xa, 0x0, MMP_CLK_GATE_NEED_DELAY, &gpu_lock},
+	{MMP2_CLK_AUDIO, "audio_clk", "audio_mix_clk", CLK_SET_RATE_PARENT, APMU_AUDIO, 0x12, 0x12, 0x0, 0, &audio_lock},
 };
 
 static struct mmp_param_gate_clk mmp2_apmu_gate_clks[] = {
@@ -423,6 +478,41 @@ static void mmp2_clk_reset_init(struct device_node *np,
 	mmp_clk_reset_register(np, cells, nr_resets);
 }
 
+static void mmp2_pm_domain_init(struct device_node *np,
+				struct mmp2_clk_unit *pxa_unit)
+{
+	if (pxa_unit->model == CLK_MODEL_MMP3) {
+		pxa_unit->pm_domains[MMP2_POWER_DOMAIN_GPU]
+			= mmp_pm_domain_register("gpu",
+				pxa_unit->apmu_base + APMU_GPU,
+				0x0600, 0x40003, 0x18000c, 0, &gpu_lock);
+	} else {
+		pxa_unit->pm_domains[MMP2_POWER_DOMAIN_GPU]
+			= mmp_pm_domain_register("gpu",
+				pxa_unit->apmu_base + APMU_GPU,
+				0x8600, 0x00003, 0x00000c,
+				MMP_PM_DOMAIN_NO_DISABLE, &gpu_lock);
+	}
+	pxa_unit->pd_data.num_domains++;
+
+	pxa_unit->pm_domains[MMP2_POWER_DOMAIN_AUDIO]
+		= mmp_pm_domain_register("audio",
+			pxa_unit->apmu_base + APMU_AUDIO,
+			0x600, 0x2, 0, 0, &audio_lock);
+	pxa_unit->pd_data.num_domains++;
+
+	if (pxa_unit->model == CLK_MODEL_MMP3) {
+		pxa_unit->pm_domains[MMP3_POWER_DOMAIN_CAMERA]
+			= mmp_pm_domain_register("camera",
+				pxa_unit->apmu_base + APMU_CAMERA,
+				0x600, 0, 0, 0, NULL);
+		pxa_unit->pd_data.num_domains++;
+	}
+
+	pxa_unit->pd_data.domains = pxa_unit->pm_domains;
+	of_genpd_add_provider_onecell(np, &pxa_unit->pd_data);
+}
+
 static void __init mmp2_clk_init(struct device_node *np)
 {
 	struct mmp2_clk_unit *pxa_unit;
@@ -454,9 +544,11 @@ static void __init mmp2_clk_init(struct device_node *np)
 		goto unmap_apmu_region;
 	}
 
+	mmp2_pm_domain_init(np, pxa_unit);
+
 	mmp_clk_init(np, &pxa_unit->unit, MMP2_NR_CLKS);
 
-	mmp2_pll_init(pxa_unit);
+	mmp2_main_clk_init(pxa_unit);
 
 	mmp2_apb_periph_clk_init(pxa_unit);
 
diff --git a/drivers/clk/mmp/clk.h b/drivers/clk/mmp/clk.h
index 20dc1e5dd756..55ac05379781 100644
--- a/drivers/clk/mmp/clk.h
+++ b/drivers/clk/mmp/clk.h
@@ -3,6 +3,7 @@
 #define __MACH_MMP_CLK_H
 
 #include <linux/clk-provider.h>
+#include <linux/pm_domain.h>
 #include <linux/clkdev.h>
 
 #define APBC_NO_BUS_CTRL	BIT(0)
@@ -16,6 +17,7 @@ struct mmp_clk_factor_masks {
 	unsigned int den_mask;
 	unsigned int num_shift;
 	unsigned int den_shift;
+	unsigned int enable_mask;
 };
 
 struct mmp_clk_factor_tbl {
@@ -251,4 +253,13 @@ void mmp_clk_init(struct device_node *np, struct mmp_clk_unit *unit,
 		int nr_clks);
 void mmp_clk_add(struct mmp_clk_unit *unit, unsigned int id,
 		struct clk *clk);
+
+/* Power islands */
+#define MMP_PM_DOMAIN_NO_DISABLE		BIT(0)
+
+struct generic_pm_domain *mmp_pm_domain_register(const char *name,
+		void __iomem *reg,
+		u32 power_on, u32 reset, u32 clock_enable,
+		unsigned int flags, spinlock_t *lock);
+
 #endif
diff --git a/drivers/clk/mmp/pwr-island.c b/drivers/clk/mmp/pwr-island.c
new file mode 100644
index 000000000000..ab57c0e995c1
--- /dev/null
+++ b/drivers/clk/mmp/pwr-island.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * MMP PMU power island support
+ *
+ * Copyright (C) 2020 Lubomir Rintel <lkundrak@v3.sk>
+ */
+
+#include <linux/pm_domain.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+
+#include "clk.h"
+
+#define to_mmp_pm_domain(genpd) container_of(genpd, struct mmp_pm_domain, genpd)
+
+struct mmp_pm_domain {
+	struct generic_pm_domain genpd;
+	void __iomem *reg;
+	spinlock_t *lock;
+	u32 power_on;
+	u32 reset;
+	u32 clock_enable;
+	unsigned int flags;
+};
+
+static int mmp_pm_domain_power_on(struct generic_pm_domain *genpd)
+{
+	struct mmp_pm_domain *pm_domain = to_mmp_pm_domain(genpd);
+	unsigned long flags = 0;
+	u32 val;
+
+	if (pm_domain->lock)
+		spin_lock_irqsave(pm_domain->lock, flags);
+
+	val = readl(pm_domain->reg);
+
+	/* Turn on the power island */
+	val |= pm_domain->power_on;
+	writel(val, pm_domain->reg);
+
+	/* Disable isolation */
+	val |= 0x100;
+	writel(val, pm_domain->reg);
+
+	/* Some blocks need to be reset after a power up */
+	if (pm_domain->reset || pm_domain->clock_enable) {
+		u32 after_power_on = val;
+
+		val &= ~pm_domain->reset;
+		writel(val, pm_domain->reg);
+
+		val |= pm_domain->clock_enable;
+		writel(val, pm_domain->reg);
+
+		val |= pm_domain->reset;
+		writel(val, pm_domain->reg);
+
+		writel(after_power_on, pm_domain->reg);
+	}
+
+	if (pm_domain->lock)
+		spin_unlock_irqrestore(pm_domain->lock, flags);
+
+	return 0;
+}
+
+static int mmp_pm_domain_power_off(struct generic_pm_domain *genpd)
+{
+	struct mmp_pm_domain *pm_domain = to_mmp_pm_domain(genpd);
+	unsigned long flags = 0;
+	u32 val;
+
+	if (pm_domain->flags & MMP_PM_DOMAIN_NO_DISABLE)
+		return 0;
+
+	if (pm_domain->lock)
+		spin_lock_irqsave(pm_domain->lock, flags);
+
+	/* Turn off and isolate the the power island. */
+	val = readl(pm_domain->reg);
+	val &= ~pm_domain->power_on;
+	val &= ~0x100;
+	writel(val, pm_domain->reg);
+
+	if (pm_domain->lock)
+		spin_unlock_irqrestore(pm_domain->lock, flags);
+
+	return 0;
+}
+
+struct generic_pm_domain *mmp_pm_domain_register(const char *name,
+		void __iomem *reg,
+		u32 power_on, u32 reset, u32 clock_enable,
+		unsigned int flags, spinlock_t *lock)
+{
+	struct mmp_pm_domain *pm_domain;
+
+	pm_domain = kzalloc(sizeof(*pm_domain), GFP_KERNEL);
+	if (!pm_domain)
+		return ERR_PTR(-ENOMEM);
+
+	pm_domain->reg = reg;
+	pm_domain->power_on = power_on;
+	pm_domain->reset = reset;
+	pm_domain->clock_enable = clock_enable;
+	pm_domain->flags = flags;
+	pm_domain->lock = lock;
+
+	pm_genpd_init(&pm_domain->genpd, NULL, true);
+	pm_domain->genpd.name = name;
+	pm_domain->genpd.power_on = mmp_pm_domain_power_on;
+	pm_domain->genpd.power_off = mmp_pm_domain_power_off;
+
+	return &pm_domain->genpd;
+}
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index abb121f8de52..cde6ca90a06b 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -142,6 +142,14 @@ config MSM_GCC_8916
 	  Say Y if you want to use devices such as UART, SPI i2c, USB,
 	  SD/eMMC, display, graphics, camera etc.
 
+config MSM_GCC_8939
+	tristate "MSM8939 Global Clock Controller"
+	select QCOM_GDSC
+	help
+	  Support for the global clock controller on msm8939 devices.
+	  Say Y if you want to use devices such as UART, SPI i2c, USB,
+	  SD/eMMC, display, graphics, camera etc.
+
 config MSM_GCC_8960
 	tristate "APQ8064/MSM8960 Global Clock Controller"
 	help
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 691efbf7e81f..7ec8561a1270 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_MDM_GCC_9615) += gcc-mdm9615.o
 obj-$(CONFIG_MDM_LCC_9615) += lcc-mdm9615.o
 obj-$(CONFIG_MSM_GCC_8660) += gcc-msm8660.o
 obj-$(CONFIG_MSM_GCC_8916) += gcc-msm8916.o
+obj-$(CONFIG_MSM_GCC_8939) += gcc-msm8939.o
 obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o
 obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o
 obj-$(CONFIG_MSM_GCC_8994) += gcc-msm8994.o
diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c
index 4e329a7baf2b..17e4a5a2a9fd 100644
--- a/drivers/clk/qcom/gcc-msm8916.c
+++ b/drivers/clk/qcom/gcc-msm8916.c
@@ -260,7 +260,7 @@ static struct clk_pll gpll0 = {
 	.l_reg = 0x21004,
 	.m_reg = 0x21008,
 	.n_reg = 0x2100c,
-	.config_reg = 0x21014,
+	.config_reg = 0x21010,
 	.mode_reg = 0x21000,
 	.status_reg = 0x2101c,
 	.status_bit = 17,
@@ -287,7 +287,7 @@ static struct clk_pll gpll1 = {
 	.l_reg = 0x20004,
 	.m_reg = 0x20008,
 	.n_reg = 0x2000c,
-	.config_reg = 0x20014,
+	.config_reg = 0x20010,
 	.mode_reg = 0x20000,
 	.status_reg = 0x2001c,
 	.status_bit = 17,
@@ -314,7 +314,7 @@ static struct clk_pll gpll2 = {
 	.l_reg = 0x4a004,
 	.m_reg = 0x4a008,
 	.n_reg = 0x4a00c,
-	.config_reg = 0x4a014,
+	.config_reg = 0x4a010,
 	.mode_reg = 0x4a000,
 	.status_reg = 0x4a01c,
 	.status_bit = 17,
@@ -341,7 +341,7 @@ static struct clk_pll bimc_pll = {
 	.l_reg = 0x23004,
 	.m_reg = 0x23008,
 	.n_reg = 0x2300c,
-	.config_reg = 0x23014,
+	.config_reg = 0x23010,
 	.mode_reg = 0x23000,
 	.status_reg = 0x2301c,
 	.status_bit = 17,
diff --git a/drivers/clk/qcom/gcc-msm8939.c b/drivers/clk/qcom/gcc-msm8939.c
new file mode 100644
index 000000000000..778354f82b1e
--- /dev/null
+++ b/drivers/clk/qcom/gcc-msm8939.c
@@ -0,0 +1,3988 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2020 Linaro Limited
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/clock/qcom,gcc-msm8939.h>
+#include <dt-bindings/reset/qcom,gcc-msm8939.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "reset.h"
+#include "gdsc.h"
+
+enum {
+	P_XO,
+	P_GPLL0,
+	P_GPLL0_AUX,
+	P_BIMC,
+	P_GPLL1,
+	P_GPLL1_AUX,
+	P_GPLL2,
+	P_GPLL2_AUX,
+	P_GPLL3,
+	P_GPLL3_AUX,
+	P_GPLL4,
+	P_GPLL5,
+	P_GPLL5_AUX,
+	P_GPLL5_EARLY,
+	P_GPLL6,
+	P_GPLL6_AUX,
+	P_SLEEP_CLK,
+	P_DSI0_PHYPLL_BYTE,
+	P_DSI0_PHYPLL_DSI,
+	P_EXT_PRI_I2S,
+	P_EXT_SEC_I2S,
+	P_EXT_MCLK,
+};
+
+static struct clk_pll gpll0 = {
+	.l_reg = 0x21004,
+	.m_reg = 0x21008,
+	.n_reg = 0x2100c,
+	.config_reg = 0x21010,
+	.mode_reg = 0x21000,
+	.status_reg = 0x2101c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll0_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(0),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll0_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll gpll1 = {
+	.l_reg = 0x20004,
+	.m_reg = 0x20008,
+	.n_reg = 0x2000c,
+	.config_reg = 0x20010,
+	.mode_reg = 0x20000,
+	.status_reg = 0x2001c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll1",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll1_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(1),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll1_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &gpll1.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll gpll2 = {
+	.l_reg = 0x4a004,
+	.m_reg = 0x4a008,
+	.n_reg = 0x4a00c,
+	.config_reg = 0x4a010,
+	.mode_reg = 0x4a000,
+	.status_reg = 0x4a01c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll2",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll2_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(2),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll2_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &gpll2.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll bimc_pll = {
+	.l_reg = 0x23004,
+	.m_reg = 0x23008,
+	.n_reg = 0x2300c,
+	.config_reg = 0x23010,
+	.mode_reg = 0x23000,
+	.status_reg = 0x2301c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "bimc_pll",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap bimc_pll_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(3),
+	.hw.init = &(struct clk_init_data){
+		.name = "bimc_pll_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &bimc_pll.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll gpll3 = {
+	.l_reg = 0x22004,
+	.m_reg = 0x22008,
+	.n_reg = 0x2200c,
+	.config_reg = 0x22010,
+	.mode_reg = 0x22000,
+	.status_reg = 0x2201c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll3",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll3_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(4),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll3_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &gpll3.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+/* GPLL3 at 1100 MHz, main output enabled. */
+static const struct pll_config gpll3_config = {
+	.l = 57,
+	.m = 7,
+	.n = 24,
+	.vco_val = 0x0,
+	.vco_mask = BIT(20),
+	.pre_div_val = 0x0,
+	.pre_div_mask = BIT(12),
+	.post_div_val = 0x0,
+	.post_div_mask = BIT(9) | BIT(8),
+	.mn_ena_mask = BIT(24),
+	.main_output_mask = BIT(0),
+	.aux_output_mask = BIT(1),
+};
+
+static struct clk_pll gpll4 = {
+	.l_reg = 0x24004,
+	.m_reg = 0x24008,
+	.n_reg = 0x2400c,
+	.config_reg = 0x24010,
+	.mode_reg = 0x24000,
+	.status_reg = 0x2401c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll4_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(5),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll4_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &gpll4.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+/* GPLL4 at 1200 MHz, main output enabled. */
+static struct pll_config gpll4_config = {
+	.l = 62,
+	.m = 1,
+	.n = 2,
+	.vco_val = 0x0,
+	.vco_mask = BIT(20),
+	.pre_div_val = 0x0,
+	.pre_div_mask = BIT(12),
+	.post_div_val = 0x0,
+	.post_div_mask = BIT(9) | BIT(8),
+	.mn_ena_mask = BIT(24),
+	.main_output_mask = BIT(0),
+};
+
+static struct clk_pll gpll5 = {
+	.l_reg = 0x25004,
+	.m_reg = 0x25008,
+	.n_reg = 0x2500c,
+	.config_reg = 0x25010,
+	.mode_reg = 0x25000,
+	.status_reg = 0x2501c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll5",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll5_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(6),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll5_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &gpll5.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll gpll6 = {
+	.l_reg = 0x37004,
+	.m_reg = 0x37008,
+	.n_reg = 0x3700c,
+	.config_reg = 0x37010,
+	.mode_reg = 0x37000,
+	.status_reg = 0x3701c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll6",
+		.parent_data = &(const struct clk_parent_data) {
+			.fw_name = "xo",
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll6_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(7),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll6_vote",
+		.parent_data = &(const struct clk_parent_data) {
+			.hw = &gpll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static const struct parent_map gcc_xo_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_bimc_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_BIMC, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_bimc_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &bimc_pll_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll6a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL6_AUX, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll6a_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll6_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll2a_gpll3_gpll6a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL2_AUX, 4 },
+	{ P_GPLL3, 2 },
+	{ P_GPLL6_AUX, 3 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll2a_gpll3_gpll6a_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll2_vote.hw },
+	{ .hw = &gpll3_vote.hw },
+	{ .hw = &gpll6_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll2_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL2, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll2_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll2_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll2_gpll4_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL2, 3 },
+	{ P_GPLL4, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll2_gpll4_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll2_vote.hw },
+	{ .hw = &gpll4_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_AUX, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0a_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll1a_sleep_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL1_AUX, 2 },
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll1a_sleep_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll1_vote.hw },
+	{ .fw_name = "sleep_clk", .name = "sleep_clk" },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll1a_gpll6_sleep_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL1_AUX, 2 },
+	{ P_GPLL6, 2 },
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll1a_gpll6_sleep_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll1_vote.hw },
+	{ .hw = &gpll6_vote.hw },
+	{ .fw_name = "sleep_clk", .name = "sleep_clk" },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll1a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL1_AUX, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll1a_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll1_vote.hw },
+};
+
+static const struct parent_map gcc_xo_dsibyte_map[] = {
+	{ P_XO, 0, },
+	{ P_DSI0_PHYPLL_BYTE, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_dsibyte_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .fw_name = "dsi0pllbyte", .name = "dsi0pllbyte" },
+};
+
+static const struct parent_map gcc_xo_gpll0a_dsibyte_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_AUX, 2 },
+	{ P_DSI0_PHYPLL_BYTE, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0a_dsibyte_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .fw_name = "dsi0pllbyte", .name = "dsi0pllbyte" },
+};
+
+static const struct parent_map gcc_xo_gpll1_dsiphy_gpll6_gpll3a_gpll0a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL1, 1 },
+	{ P_DSI0_PHYPLL_DSI, 2 },
+	{ P_GPLL6, 3 },
+	{ P_GPLL3_AUX, 4 },
+	{ P_GPLL0_AUX, 5 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll1_dsiphy_gpll6_gpll3a_gpll0a_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll1_vote.hw },
+	{ .fw_name = "dsi0pll", .name = "dsi0pll" },
+	{ .hw = &gpll6_vote.hw },
+	{ .hw = &gpll3_vote.hw },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0a_dsiphy_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_AUX, 2 },
+	{ P_DSI0_PHYPLL_DSI, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0a_dsiphy_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .fw_name = "dsi0pll", .name = "dsi0pll" },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll5a_gpll6_bimc_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL5_AUX, 3 },
+	{ P_GPLL6, 2 },
+	{ P_BIMC, 4 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll5a_gpll6_bimc_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll5_vote.hw },
+	{ .hw = &gpll6_vote.hw },
+	{ .hw = &bimc_pll_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll1_sleep_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL1, 2 },
+	{ P_SLEEP_CLK, 6 }
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll1_sleep_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll1_vote.hw },
+	{ .fw_name = "sleep_clk", .name = "sleep_clk" },
+};
+
+static const struct parent_map gcc_xo_gpll1_epi2s_emclk_sleep_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL1, 1 },
+	{ P_EXT_PRI_I2S, 2 },
+	{ P_EXT_MCLK, 3 },
+	{ P_SLEEP_CLK, 6 }
+};
+
+static const struct clk_parent_data gcc_xo_gpll1_epi2s_emclk_sleep_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll0_vote.hw },
+	{ .fw_name = "ext_pri_i2s", .name = "ext_pri_i2s" },
+	{ .fw_name = "ext_mclk", .name = "ext_mclk" },
+	{ .fw_name = "sleep_clk", .name = "sleep_clk" },
+};
+
+static const struct parent_map gcc_xo_gpll1_esi2s_emclk_sleep_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL1, 1 },
+	{ P_EXT_SEC_I2S, 2 },
+	{ P_EXT_MCLK, 3 },
+	{ P_SLEEP_CLK, 6 }
+};
+
+static const struct clk_parent_data gcc_xo_gpll1_esi2s_emclk_sleep_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll1_vote.hw },
+	{ .fw_name = "ext_sec_i2s", .name = "ext_sec_i2s" },
+	{ .fw_name = "ext_mclk", .name = "ext_mclk" },
+	{ .fw_name = "sleep_clk", .name = "sleep_clk" },
+};
+
+static const struct parent_map gcc_xo_sleep_map[] = {
+	{ P_XO, 0 },
+	{ P_SLEEP_CLK, 6 }
+};
+
+static const struct clk_parent_data gcc_xo_sleep_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .fw_name = "sleep_clk", .name = "sleep_clk" },
+};
+
+static const struct parent_map gcc_xo_gpll1_emclk_sleep_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL1, 1 },
+	{ P_EXT_MCLK, 2 },
+	{ P_SLEEP_CLK, 6 }
+};
+
+static const struct clk_parent_data gcc_xo_gpll1_emclk_sleep_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll1_vote.hw },
+	{ .fw_name = "ext_mclk", .name = "ext_mclk" },
+	{ .fw_name = "sleep_clk", .name = "sleep_clk" },
+};
+
+static const struct parent_map gcc_xo_gpll6_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL6, 1 },
+	{ P_GPLL0, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll6_gpll0_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll6_vote.hw },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll6_gpll0a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL6, 1 },
+	{ P_GPLL0_AUX, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll6_gpll0a_parent_data[] = {
+	{ .fw_name = "xo" },
+	{ .hw = &gpll6_vote.hw },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static struct clk_rcg2 pcnoc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x27000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcnoc_bfdcd_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 system_noc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x26004,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6a_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "system_noc_bfdcd_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6a_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 bimc_ddr_clk_src = {
+	.cmd_rcgr = 0x32004,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_bimc_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "bimc_ddr_clk_src",
+		.parent_data = gcc_xo_gpll0_bimc_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+		.flags = CLK_GET_RATE_NOCACHE,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_ahb_clk[] = {
+	F(40000000, P_GPLL0, 10, 1, 2),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 camss_ahb_clk_src = {
+	.cmd_rcgr = 0x5a000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_ahb_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_ahb_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_apss_ahb_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(133330000, P_GPLL0, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 apss_ahb_clk_src = {
+	.cmd_rcgr = 0x46000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_apss_ahb_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "apss_ahb_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_csi0_1_clk[] = {
+	F(100000000, P_GPLL0, 8, 0,	0),
+	F(200000000, P_GPLL0, 4, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 csi0_clk_src = {
+	.cmd_rcgr = 0x4e020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi0_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi1_clk_src = {
+	.cmd_rcgr = 0x4f020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi1_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_oxili_gfx3d_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(220000000, P_GPLL3, 5, 0, 0),
+	F(266670000, P_GPLL0, 3, 0, 0),
+	F(310000000, P_GPLL2_AUX, 3, 0, 0),
+	F(400000000, P_GPLL0, 2, 0, 0),
+	F(465000000, P_GPLL2_AUX, 2, 0, 0),
+	F(550000000, P_GPLL3, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gfx3d_clk_src = {
+	.cmd_rcgr = 0x59000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll2a_gpll3_gpll6a_map,
+	.freq_tbl = ftbl_gcc_oxili_gfx3d_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gfx3d_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll2a_gpll3_gpll6a_parent_data,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_vfe0_clk[] = {
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(177780000, P_GPLL0, 4.5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(266670000, P_GPLL0, 3, 0, 0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	F(400000000, P_GPLL0, 2, 0, 0),
+	F(465000000, P_GPLL2, 2, 0, 0),
+	F(480000000, P_GPLL4, 2.5, 0, 0),
+	F(600000000, P_GPLL4, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 vfe0_clk_src = {
+	.cmd_rcgr = 0x58000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll2_gpll4_map,
+	.freq_tbl = ftbl_gcc_camss_vfe0_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vfe0_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll2_gpll4_parent_data,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_blsp1_qup1_6_i2c_apps_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0200c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_blsp1_qup1_6_spi_apps_clk[] = {
+	F(960000, P_XO, 10, 1, 2),
+	F(4800000, P_XO, 4, 0, 0),
+	F(9600000, P_XO, 2, 0, 0),
+	F(16000000, P_GPLL0, 10, 1, 5),
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0, 16, 1, 2),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x02024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x03000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x03014,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x04000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x04024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x05000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x05024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x06000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup5_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = {
+	.cmd_rcgr = 0x06024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup5_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x07000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup6_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = {
+	.cmd_rcgr = 0x07024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup6_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_blsp1_uart1_6_apps_clk[] = {
+	F(3686400, P_GPLL0, 1, 72, 15625),
+	F(7372800, P_GPLL0, 1, 144, 15625),
+	F(14745600, P_GPLL0, 1, 288, 15625),
+	F(16000000, P_GPLL0, 10, 1, 5),
+	F(19200000, P_XO, 1, 0, 0),
+	F(24000000, P_GPLL0, 1, 3, 100),
+	F(25000000, P_GPLL0, 16, 1, 2),
+	F(32000000, P_GPLL0, 1, 1, 25),
+	F(40000000, P_GPLL0, 1, 1, 20),
+	F(46400000, P_GPLL0, 1, 29, 500),
+	F(48000000, P_GPLL0, 1, 3, 50),
+	F(51200000, P_GPLL0, 1, 8, 125),
+	F(56000000, P_GPLL0, 1, 7, 100),
+	F(58982400, P_GPLL0, 1, 1152, 15625),
+	F(60000000, P_GPLL0, 1, 3, 40),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x02044,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart1_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x03034,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart2_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_cci_clk[] = {
+	F(19200000,	P_XO, 1, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 cci_clk_src = {
+	.cmd_rcgr = 0x51000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_map,
+	.freq_tbl = ftbl_gcc_camss_cci_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "cci_clk_src",
+		.parent_data = gcc_xo_gpll0a_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_gp0_1_clk[] = {
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 camss_gp0_clk_src = {
+	.cmd_rcgr = 0x54000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_sleep_map,
+	.freq_tbl = ftbl_gcc_camss_gp0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_gp0_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 camss_gp1_clk_src = {
+	.cmd_rcgr = 0x55000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_sleep_map,
+	.freq_tbl = ftbl_gcc_camss_gp0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_gp1_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_jpeg0_clk[] = {
+	F(133330000, P_GPLL0, 6, 0,	0),
+	F(266670000, P_GPLL0, 3, 0,	0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 jpeg0_clk_src = {
+	.cmd_rcgr = 0x57000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_jpeg0_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "jpeg0_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_mclk0_1_clk[] = {
+	F(24000000, P_GPLL0, 1, 1, 45),
+	F(66670000, P_GPLL0, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 mclk0_clk_src = {
+	.cmd_rcgr = 0x52000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_gpll6_sleep_map,
+	.freq_tbl = ftbl_gcc_camss_mclk0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk0_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_gpll6_sleep_parent_data,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 mclk1_clk_src = {
+	.cmd_rcgr = 0x53000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_gpll6_sleep_map,
+	.freq_tbl = ftbl_gcc_camss_mclk0_1_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk1_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_gpll6_sleep_parent_data,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_csi0_1phytimer_clk[] = {
+	F(100000000, P_GPLL0, 8, 0,	0),
+	F(200000000, P_GPLL0, 4, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x4e000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1phytimer_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi0phytimer_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x4f000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1phytimer_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi1phytimer_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_cpp_clk[] = {
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	F(465000000, P_GPLL2, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cpp_clk_src = {
+	.cmd_rcgr = 0x58018,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll2_map,
+	.freq_tbl = ftbl_gcc_camss_cpp_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "cpp_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll2_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_crypto_clk[] = {
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	{ }
+};
+
+/* This is not in the documentation but is in the downstream driver */
+static struct clk_rcg2 crypto_clk_src = {
+	.cmd_rcgr = 0x16004,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_crypto_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "crypto_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gp1_3_clk[] = {
+	F(19200000, P_XO, 1, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 gp1_clk_src = {
+	.cmd_rcgr = 0x08004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_sleep_map,
+	.freq_tbl = ftbl_gcc_gp1_3_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp1_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp2_clk_src = {
+	.cmd_rcgr = 0x09004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_sleep_map,
+	.freq_tbl = ftbl_gcc_gp1_3_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp2_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp3_clk_src = {
+	.cmd_rcgr = 0x0a004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll1a_sleep_map,
+	.freq_tbl = ftbl_gcc_gp1_3_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp3_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 byte0_clk_src = {
+	.cmd_rcgr = 0x4d044,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_dsibyte_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "byte0_clk_src",
+		.parent_data = gcc_xo_gpll0a_dsibyte_parent_data,
+		.num_parents = 3,
+		.ops = &clk_byte2_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_rcg2 byte1_clk_src = {
+	.cmd_rcgr = 0x4d0b0,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_dsibyte_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "byte1_clk_src",
+		.parent_data = gcc_xo_gpll0a_dsibyte_parent_data,
+		.num_parents = 3,
+		.ops = &clk_byte2_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_mdss_esc_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 esc0_clk_src = {
+	.cmd_rcgr = 0x4d060,
+	.hid_width = 5,
+	.parent_map = gcc_xo_dsibyte_map,
+	.freq_tbl = ftbl_gcc_mdss_esc_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "esc0_clk_src",
+		.parent_data = gcc_xo_dsibyte_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 esc1_clk_src = {
+	.cmd_rcgr = 0x4d0a8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_dsibyte_map,
+	.freq_tbl = ftbl_gcc_mdss_esc_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "esc1_clk_src",
+		.parent_data = gcc_xo_dsibyte_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_mdss_mdp_clk[] = {
+	F(50000000, P_GPLL0_AUX, 16, 0, 0),
+	F(80000000, P_GPLL0_AUX, 10, 0, 0),
+	F(100000000, P_GPLL0_AUX, 8, 0, 0),
+	F(160000000, P_GPLL0_AUX, 5, 0, 0),
+	F(177780000, P_GPLL0_AUX, 4.5, 0, 0),
+	F(200000000, P_GPLL0_AUX, 4, 0, 0),
+	F(266670000, P_GPLL0_AUX, 3, 0, 0),
+	F(307200000, P_GPLL1, 2, 0, 0),
+	F(366670000, P_GPLL3_AUX, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 mdp_clk_src = {
+	.cmd_rcgr = 0x4d014,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll1_dsiphy_gpll6_gpll3a_gpll0a_map,
+	.freq_tbl = ftbl_gcc_mdss_mdp_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mdp_clk_src",
+		.parent_data = gcc_xo_gpll1_dsiphy_gpll6_gpll3a_gpll0a_parent_data,
+		.num_parents = 6,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 pclk0_clk_src = {
+	.cmd_rcgr = 0x4d000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_dsiphy_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pclk0_clk_src",
+		.parent_data = gcc_xo_gpll0a_dsiphy_parent_data,
+		.num_parents = 3,
+		.ops = &clk_pixel_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_rcg2 pclk1_clk_src = {
+	.cmd_rcgr = 0x4d0b8,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_dsiphy_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pclk1_clk_src",
+		.parent_data = gcc_xo_gpll0a_dsiphy_parent_data,
+		.num_parents = 3,
+		.ops = &clk_pixel_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_mdss_vsync_clk[] = {
+	F(19200000, P_XO, 1, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 vsync_clk_src = {
+	.cmd_rcgr = 0x4d02c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_map,
+	.freq_tbl = ftbl_gcc_mdss_vsync_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vsync_clk_src",
+		.parent_data = gcc_xo_gpll0a_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk[] = {
+	F(64000000, P_GPLL0, 12.5, 0, 0),
+	{ }
+};
+
+/* This is not in the documentation but is in the downstream driver */
+static struct clk_rcg2 pdm2_clk_src = {
+	.cmd_rcgr = 0x44010,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_pdm2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pdm2_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc_apps_clk[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 10, 1, 4),
+	F(25000000, P_GPLL0, 16, 1, 2),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(177770000, P_GPLL0, 4.5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x42004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_sdcc_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc1_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static struct clk_rcg2 sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x43004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_sdcc_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc2_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_apss_tcu_clk[] = {
+	F(154285000, P_GPLL6, 7, 0, 0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	F(400000000, P_GPLL0, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 apss_tcu_clk_src = {
+	.cmd_rcgr = 0x1207c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll5a_gpll6_bimc_map,
+	.freq_tbl = ftbl_gcc_apss_tcu_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "apss_tcu_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll5a_gpll6_bimc_parent_data,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_bimc_gpu_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(266500000, P_BIMC, 4, 0, 0),
+	F(400000000, P_GPLL0, 2, 0, 0),
+	F(533000000, P_BIMC, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 bimc_gpu_clk_src = {
+	.cmd_rcgr = 0x31028,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll5a_gpll6_bimc_map,
+	.freq_tbl = ftbl_gcc_bimc_gpu_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "bimc_gpu_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll5a_gpll6_bimc_parent_data,
+		.num_parents = 5,
+		.flags = CLK_GET_RATE_NOCACHE,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb_hs_system_clk[] = {
+	F(80000000, P_GPLL0, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb_hs_system_clk_src = {
+	.cmd_rcgr = 0x41010,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_usb_hs_system_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb_hs_system_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb_fs_system_clk[] = {
+	F(64000000, P_GPLL0, 12.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb_fs_system_clk_src = {
+	.cmd_rcgr = 0x3f010,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_usb_fs_system_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb_fs_system_clk_src",
+		.parent_data = gcc_xo_gpll6_gpll0_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb_fs_ic_clk[] = {
+	F(60000000, P_GPLL6, 1, 1, 18),
+	{ }
+};
+
+static struct clk_rcg2 usb_fs_ic_clk_src = {
+	.cmd_rcgr = 0x3f034,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_usb_fs_ic_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb_fs_ic_clk_src",
+		.parent_data = gcc_xo_gpll6_gpll0a_parent_data,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ultaudio_ahb_clk[] = {
+	F(3200000, P_XO, 6, 0, 0),
+	F(6400000, P_XO, 3, 0, 0),
+	F(9600000, P_XO, 2, 0, 0),
+	F(19200000, P_XO, 1, 0, 0),
+	F(40000000, P_GPLL0, 10, 1, 2),
+	F(66670000, P_GPLL0, 12, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ultaudio_ahbfabric_clk_src = {
+	.cmd_rcgr = 0x1c010,
+	.hid_width = 5,
+	.mnd_width = 8,
+	.parent_map = gcc_xo_gpll0_gpll1_sleep_map,
+	.freq_tbl = ftbl_gcc_ultaudio_ahb_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ultaudio_ahbfabric_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll1_sleep_parent_data,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_ultaudio_ahbfabric_ixfabric_clk = {
+	.halt_reg = 0x1c028,
+	.clkr = {
+		.enable_reg = 0x1c028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_ahbfabric_ixfabric_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &ultaudio_ahbfabric_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ultaudio_ahbfabric_ixfabric_lpm_clk = {
+	.halt_reg = 0x1c024,
+	.clkr = {
+		.enable_reg = 0x1c024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_ahbfabric_ixfabric_lpm_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &ultaudio_ahbfabric_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ultaudio_lpaif_i2s_clk[] = {
+	F(128000, P_XO, 10, 1, 15),
+	F(256000, P_XO, 5, 1, 15),
+	F(384000, P_XO, 5, 1, 10),
+	F(512000, P_XO, 5, 2, 15),
+	F(576000, P_XO, 5, 3, 20),
+	F(705600, P_GPLL1, 16, 1, 80),
+	F(768000, P_XO, 5, 1, 5),
+	F(800000, P_XO, 5, 5, 24),
+	F(1024000, P_XO, 5, 4, 15),
+	F(1152000, P_XO, 1, 3, 50),
+	F(1411200, P_GPLL1, 16, 1, 40),
+	F(1536000, P_XO, 1, 2, 25),
+	F(1600000, P_XO, 12, 0, 0),
+	F(1728000, P_XO, 5, 9, 20),
+	F(2048000, P_XO, 5, 8, 15),
+	F(2304000, P_XO, 5, 3, 5),
+	F(2400000, P_XO, 8, 0, 0),
+	F(2822400, P_GPLL1, 16, 1, 20),
+	F(3072000, P_XO, 5, 4, 5),
+	F(4096000, P_GPLL1, 9, 2, 49),
+	F(4800000, P_XO, 4, 0, 0),
+	F(5644800, P_GPLL1, 16, 1, 10),
+	F(6144000, P_GPLL1, 7, 1, 21),
+	F(8192000, P_GPLL1, 9, 4, 49),
+	F(9600000, P_XO, 2, 0, 0),
+	F(11289600, P_GPLL1, 16, 1, 5),
+	F(12288000, P_GPLL1, 7, 2, 21),
+	{ }
+};
+
+static struct clk_rcg2 ultaudio_lpaif_pri_i2s_clk_src = {
+	.cmd_rcgr = 0x1c054,
+	.hid_width = 5,
+	.mnd_width = 8,
+	.parent_map = gcc_xo_gpll1_epi2s_emclk_sleep_map,
+	.freq_tbl = ftbl_gcc_ultaudio_lpaif_i2s_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ultaudio_lpaif_pri_i2s_clk_src",
+		.parent_data = gcc_xo_gpll1_epi2s_emclk_sleep_parent_data,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_ultaudio_lpaif_pri_i2s_clk = {
+	.halt_reg = 0x1c068,
+	.clkr = {
+		.enable_reg = 0x1c068,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_lpaif_pri_i2s_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &ultaudio_lpaif_pri_i2s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 ultaudio_lpaif_sec_i2s_clk_src = {
+	.cmd_rcgr = 0x1c06c,
+	.hid_width = 5,
+	.mnd_width = 8,
+	.parent_map = gcc_xo_gpll1_esi2s_emclk_sleep_map,
+	.freq_tbl = ftbl_gcc_ultaudio_lpaif_i2s_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ultaudio_lpaif_sec_i2s_clk_src",
+		.parent_data = gcc_xo_gpll1_esi2s_emclk_sleep_parent_data,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_ultaudio_lpaif_sec_i2s_clk = {
+	.halt_reg = 0x1c080,
+	.clkr = {
+		.enable_reg = 0x1c080,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_lpaif_sec_i2s_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &ultaudio_lpaif_sec_i2s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_rcg2 ultaudio_lpaif_aux_i2s_clk_src = {
+	.cmd_rcgr = 0x1c084,
+	.hid_width = 5,
+	.mnd_width = 8,
+	.parent_map = gcc_xo_gpll1_emclk_sleep_map,
+	.freq_tbl = ftbl_gcc_ultaudio_lpaif_i2s_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ultaudio_lpaif_aux_i2s_clk_src",
+		.parent_data = gcc_xo_gpll1_esi2s_emclk_sleep_parent_data,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_ultaudio_lpaif_aux_i2s_clk = {
+	.halt_reg = 0x1c098,
+	.clkr = {
+		.enable_reg = 0x1c098,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_lpaif_aux_i2s_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &ultaudio_lpaif_aux_i2s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ultaudio_xo_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ultaudio_xo_clk_src = {
+	.cmd_rcgr = 0x1c034,
+	.hid_width = 5,
+	.parent_map = gcc_xo_sleep_map,
+	.freq_tbl = ftbl_gcc_ultaudio_xo_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ultaudio_xo_clk_src",
+		.parent_data = gcc_xo_sleep_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_ultaudio_avsync_xo_clk = {
+	.halt_reg = 0x1c04c,
+	.clkr = {
+		.enable_reg = 0x1c04c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_avsync_xo_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &ultaudio_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ultaudio_stc_xo_clk = {
+	.halt_reg = 0x1c050,
+	.clkr = {
+		.enable_reg = 0x1c050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_stc_xo_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &ultaudio_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_codec_clk[] = {
+	F(9600000, P_XO, 2, 0, 0),
+	F(12288000, P_XO, 1, 16, 25),
+	F(19200000, P_XO, 1, 0, 0),
+	F(11289600, P_EXT_MCLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 codec_digcodec_clk_src = {
+	.cmd_rcgr = 0x1c09c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll1_emclk_sleep_map,
+	.freq_tbl = ftbl_codec_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "codec_digcodec_clk_src",
+		.parent_data = gcc_xo_gpll1_emclk_sleep_parent_data,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_codec_digcodec_clk = {
+	.halt_reg = 0x1c0b0,
+	.clkr = {
+		.enable_reg = 0x1c0b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_codec_digcodec_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &codec_digcodec_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ultaudio_pcnoc_mport_clk = {
+	.halt_reg = 0x1c000,
+	.clkr = {
+		.enable_reg = 0x1c000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_pcnoc_mport_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ultaudio_pcnoc_sway_clk = {
+	.halt_reg = 0x1c004,
+	.clkr = {
+		.enable_reg = 0x1c004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ultaudio_pcnoc_sway_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_venus0_vcodec0_clk[] = {
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(228570000, P_GPLL0, 3.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 vcodec0_clk_src = {
+	.cmd_rcgr = 0x4C000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_venus0_vcodec0_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vcodec0_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_blsp1_ahb_clk = {
+	.halt_reg = 0x01008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_sleep_clk = {
+	.halt_reg = 0x01004,
+	.clkr = {
+		.enable_reg = 0x01004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
+	.halt_reg = 0x02008,
+	.clkr = {
+		.enable_reg = 0x02008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_i2c_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup1_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
+	.halt_reg = 0x02004,
+	.clkr = {
+		.enable_reg = 0x02004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_spi_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup1_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
+	.halt_reg = 0x03010,
+	.clkr = {
+		.enable_reg = 0x03010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_i2c_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup2_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
+	.halt_reg = 0x0300c,
+	.clkr = {
+		.enable_reg = 0x0300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_spi_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup2_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
+	.halt_reg = 0x04020,
+	.clkr = {
+		.enable_reg = 0x04020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_i2c_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup3_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
+	.halt_reg = 0x0401c,
+	.clkr = {
+		.enable_reg = 0x0401c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_spi_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup3_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
+	.halt_reg = 0x05020,
+	.clkr = {
+		.enable_reg = 0x05020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_i2c_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup4_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
+	.halt_reg = 0x0501c,
+	.clkr = {
+		.enable_reg = 0x0501c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_spi_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup4_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = {
+	.halt_reg = 0x06020,
+	.clkr = {
+		.enable_reg = 0x06020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_i2c_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup5_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = {
+	.halt_reg = 0x0601c,
+	.clkr = {
+		.enable_reg = 0x0601c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_spi_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup5_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = {
+	.halt_reg = 0x07020,
+	.clkr = {
+		.enable_reg = 0x07020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_i2c_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup6_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = {
+	.halt_reg = 0x0701c,
+	.clkr = {
+		.enable_reg = 0x0701c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_spi_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_qup6_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart1_apps_clk = {
+	.halt_reg = 0x0203c,
+	.clkr = {
+		.enable_reg = 0x0203c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart1_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_uart1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart2_apps_clk = {
+	.halt_reg = 0x0302c,
+	.clkr = {
+		.enable_reg = 0x0302c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart2_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &blsp1_uart2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x1300c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(7),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_boot_rom_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cci_ahb_clk = {
+	.halt_reg = 0x5101c,
+	.clkr = {
+		.enable_reg = 0x5101c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cci_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cci_clk = {
+	.halt_reg = 0x51018,
+	.clkr = {
+		.enable_reg = 0x51018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cci_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &cci_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0_ahb_clk = {
+	.halt_reg = 0x4e040,
+	.clkr = {
+		.enable_reg = 0x4e040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0_clk = {
+	.halt_reg = 0x4e03c,
+	.clkr = {
+		.enable_reg = 0x4e03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0phy_clk = {
+	.halt_reg = 0x4e048,
+	.clkr = {
+		.enable_reg = 0x4e048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0phy_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0pix_clk = {
+	.halt_reg = 0x4e058,
+	.clkr = {
+		.enable_reg = 0x4e058,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0pix_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0rdi_clk = {
+	.halt_reg = 0x4e050,
+	.clkr = {
+		.enable_reg = 0x4e050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0rdi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1_ahb_clk = {
+	.halt_reg = 0x4f040,
+	.clkr = {
+		.enable_reg = 0x4f040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1_clk = {
+	.halt_reg = 0x4f03c,
+	.clkr = {
+		.enable_reg = 0x4f03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1phy_clk = {
+	.halt_reg = 0x4f048,
+	.clkr = {
+		.enable_reg = 0x4f048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1phy_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1pix_clk = {
+	.halt_reg = 0x4f058,
+	.clkr = {
+		.enable_reg = 0x4f058,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1pix_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1rdi_clk = {
+	.halt_reg = 0x4f050,
+	.clkr = {
+		.enable_reg = 0x4f050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1rdi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi_vfe0_clk = {
+	.halt_reg = 0x58050,
+	.clkr = {
+		.enable_reg = 0x58050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi_vfe0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &vfe0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_gp0_clk = {
+	.halt_reg = 0x54018,
+	.clkr = {
+		.enable_reg = 0x54018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_gp0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_gp0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_gp1_clk = {
+	.halt_reg = 0x55018,
+	.clkr = {
+		.enable_reg = 0x55018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_gp1_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_ispif_ahb_clk = {
+	.halt_reg = 0x50004,
+	.clkr = {
+		.enable_reg = 0x50004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_ispif_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_jpeg0_clk = {
+	.halt_reg = 0x57020,
+	.clkr = {
+		.enable_reg = 0x57020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_jpeg0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &jpeg0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_jpeg_ahb_clk = {
+	.halt_reg = 0x57024,
+	.clkr = {
+		.enable_reg = 0x57024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_jpeg_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_jpeg_axi_clk = {
+	.halt_reg = 0x57028,
+	.clkr = {
+		.enable_reg = 0x57028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_jpeg_axi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk0_clk = {
+	.halt_reg = 0x52018,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_mclk0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &mclk0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk1_clk = {
+	.halt_reg = 0x53018,
+	.clkr = {
+		.enable_reg = 0x53018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_mclk1_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &mclk1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_micro_ahb_clk = {
+	.halt_reg = 0x5600c,
+	.clkr = {
+		.enable_reg = 0x5600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_micro_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0phytimer_clk = {
+	.halt_reg = 0x4e01c,
+	.clkr = {
+		.enable_reg = 0x4e01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0phytimer_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi0phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1phytimer_clk = {
+	.halt_reg = 0x4f01c,
+	.clkr = {
+		.enable_reg = 0x4f01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1phytimer_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &csi1phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_ahb_clk = {
+	.halt_reg = 0x5a014,
+	.clkr = {
+		.enable_reg = 0x5a014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_top_ahb_clk = {
+	.halt_reg = 0x56004,
+	.clkr = {
+		.enable_reg = 0x56004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_top_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cpp_ahb_clk = {
+	.halt_reg = 0x58040,
+	.clkr = {
+		.enable_reg = 0x58040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cpp_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cpp_clk = {
+	.halt_reg = 0x5803c,
+	.clkr = {
+		.enable_reg = 0x5803c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cpp_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &cpp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe0_clk = {
+	.halt_reg = 0x58038,
+	.clkr = {
+		.enable_reg = 0x58038,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_vfe0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &vfe0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe_ahb_clk = {
+	.halt_reg = 0x58044,
+	.clkr = {
+		.enable_reg = 0x58044,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_vfe_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &camss_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe_axi_clk = {
+	.halt_reg = 0x58048,
+	.clkr = {
+		.enable_reg = 0x58048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_vfe_axi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_crypto_ahb_clk = {
+	.halt_reg = 0x16024,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_crypto_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_crypto_axi_clk = {
+	.halt_reg = 0x16020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_crypto_axi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_crypto_clk = {
+	.halt_reg = 0x1601c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_crypto_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &crypto_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_gmem_clk = {
+	.halt_reg = 0x59024,
+	.clkr = {
+		.enable_reg = 0x59024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_gmem_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &gfx3d_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x08000,
+	.clkr = {
+		.enable_reg = 0x08000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp1_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x09000,
+	.clkr = {
+		.enable_reg = 0x09000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp2_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &gp2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x0a000,
+	.clkr = {
+		.enable_reg = 0x0a000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp3_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &gp3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_ahb_clk = {
+	.halt_reg = 0x4d07c,
+	.clkr = {
+		.enable_reg = 0x4d07c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_axi_clk = {
+	.halt_reg = 0x4d080,
+	.clkr = {
+		.enable_reg = 0x4d080,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_axi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_byte0_clk = {
+	.halt_reg = 0x4d094,
+	.clkr = {
+		.enable_reg = 0x4d094,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_byte0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &byte0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_byte1_clk = {
+	.halt_reg = 0x4d0a0,
+	.clkr = {
+		.enable_reg = 0x4d0a0,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_byte1_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &byte1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_esc0_clk = {
+	.halt_reg = 0x4d098,
+	.clkr = {
+		.enable_reg = 0x4d098,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_esc0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &esc0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_esc1_clk = {
+	.halt_reg = 0x4d09c,
+	.clkr = {
+		.enable_reg = 0x4d09c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_esc1_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &esc1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_mdp_clk = {
+	.halt_reg = 0x4D088,
+	.clkr = {
+		.enable_reg = 0x4D088,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_mdp_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &mdp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_pclk0_clk = {
+	.halt_reg = 0x4d084,
+	.clkr = {
+		.enable_reg = 0x4d084,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_pclk0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pclk0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_pclk1_clk = {
+	.halt_reg = 0x4d0a4,
+	.clkr = {
+		.enable_reg = 0x4d0a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_pclk1_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pclk1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_vsync_clk = {
+	.halt_reg = 0x4d090,
+	.clkr = {
+		.enable_reg = 0x4d090,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_vsync_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &vsync_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_cfg_ahb_clk = {
+	.halt_reg = 0x49000,
+	.clkr = {
+		.enable_reg = 0x49000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_cfg_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_q6_bimc_axi_clk = {
+	.halt_reg = 0x49004,
+	.clkr = {
+		.enable_reg = 0x49004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_q6_bimc_axi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &bimc_ddr_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_ahb_clk = {
+	.halt_reg = 0x59028,
+	.clkr = {
+		.enable_reg = 0x59028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_gfx3d_clk = {
+	.halt_reg = 0x59020,
+	.clkr = {
+		.enable_reg = 0x59020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_gfx3d_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &gfx3d_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x4400c,
+	.clkr = {
+		.enable_reg = 0x4400c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm2_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pdm2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x44004,
+	.clkr = {
+		.enable_reg = 0x44004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_prng_ahb_clk = {
+	.halt_reg = 0x13004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(8),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_prng_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0x4201c,
+	.clkr = {
+		.enable_reg = 0x4201c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x42018,
+	.clkr = {
+		.enable_reg = 0x42018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &sdcc1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x4301c,
+	.clkr = {
+		.enable_reg = 0x4301c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x43018,
+	.clkr = {
+		.enable_reg = 0x43018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &sdcc2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_apss_tcu_clk = {
+	.halt_reg = 0x12018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_apss_tcu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &bimc_ddr_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gfx_tcu_clk = {
+	.halt_reg = 0x12020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gfx_tcu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &bimc_ddr_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gfx_tbu_clk = {
+	.halt_reg = 0x12010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(3),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gfx_tbu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &bimc_ddr_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdp_tbu_clk = {
+	.halt_reg = 0x1201c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdp_tbu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus_tbu_clk = {
+	.halt_reg = 0x12014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(5),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus_tbu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vfe_tbu_clk = {
+	.halt_reg = 0x1203c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vfe_tbu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_jpeg_tbu_clk = {
+	.halt_reg = 0x12034,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_jpeg_tbu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_smmu_cfg_clk = {
+	.halt_reg = 0x12038,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(12),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_smmu_cfg_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gtcu_ahb_clk = {
+	.halt_reg = 0x12044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(13),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gtcu_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cpp_tbu_clk = {
+	.halt_reg = 0x12040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(14),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cpp_tbu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdp_rt_tbu_clk = {
+	.halt_reg = 0x1201c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdp_rt_tbu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_bimc_gfx_clk = {
+	.halt_reg = 0x31024,
+	.clkr = {
+		.enable_reg = 0x31024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_bimc_gfx_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &bimc_gpu_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_bimc_gpu_clk = {
+	.halt_reg = 0x31040,
+	.clkr = {
+		.enable_reg = 0x31040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_bimc_gpu_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &bimc_gpu_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb2a_phy_sleep_clk = {
+	.halt_reg = 0x4102c,
+	.clkr = {
+		.enable_reg = 0x4102c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb2a_phy_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_fs_ahb_clk = {
+	.halt_reg = 0x3f008,
+	.clkr = {
+		.enable_reg = 0x3f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_fs_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_fs_ic_clk = {
+	.halt_reg = 0x3f030,
+	.clkr = {
+		.enable_reg = 0x3f030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_fs_ic_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &usb_fs_ic_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_fs_system_clk = {
+	.halt_reg = 0x3f004,
+	.clkr = {
+		.enable_reg = 0x3f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_fs_system_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &usb_fs_system_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_hs_ahb_clk = {
+	.halt_reg = 0x41008,
+	.clkr = {
+		.enable_reg = 0x41008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_hs_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_hs_system_clk = {
+	.halt_reg = 0x41004,
+	.clkr = {
+		.enable_reg = 0x41004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_hs_system_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &usb_hs_system_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_ahb_clk = {
+	.halt_reg = 0x4c020,
+	.clkr = {
+		.enable_reg = 0x4c020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_ahb_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_axi_clk = {
+	.halt_reg = 0x4c024,
+	.clkr = {
+		.enable_reg = 0x4c024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_axi_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &system_noc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_vcodec0_clk = {
+	.halt_reg = 0x4c01c,
+	.clkr = {
+		.enable_reg = 0x4c01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_vcodec0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &vcodec0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_core0_vcodec0_clk = {
+	.halt_reg = 0x4c02c,
+	.clkr = {
+		.enable_reg = 0x4c02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_core0_vcodec0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &vcodec0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_core1_vcodec0_clk = {
+	.halt_reg = 0x4c034,
+	.clkr = {
+		.enable_reg = 0x4c034,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_core1_vcodec0_clk",
+			.parent_data = &(const struct clk_parent_data){
+				.hw = &vcodec0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_timer_clk = {
+	.halt_reg = 0x59040,
+	.clkr = {
+		.enable_reg = 0x59040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_timer_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc venus_gdsc = {
+	.gdscr = 0x4c018,
+	.pd = {
+		.name = "venus",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc mdss_gdsc = {
+	.gdscr = 0x4d078,
+	.pd = {
+		.name = "mdss",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc jpeg_gdsc = {
+	.gdscr = 0x5701c,
+	.pd = {
+		.name = "jpeg",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc vfe_gdsc = {
+	.gdscr = 0x58034,
+	.pd = {
+		.name = "vfe",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc oxili_gdsc = {
+	.gdscr = 0x5901c,
+	.pd = {
+		.name = "oxili",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc venus_core0_gdsc = {
+	.gdscr = 0x4c028,
+	.pd = {
+		.name = "venus_core0",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc venus_core1_gdsc = {
+	.gdscr = 0x4c030,
+	.pd = {
+		.name = "venus_core1",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct clk_regmap *gcc_msm8939_clocks[] = {
+	[GPLL0] = &gpll0.clkr,
+	[GPLL0_VOTE] = &gpll0_vote,
+	[BIMC_PLL] = &bimc_pll.clkr,
+	[BIMC_PLL_VOTE] = &bimc_pll_vote,
+	[GPLL1] = &gpll1.clkr,
+	[GPLL1_VOTE] = &gpll1_vote,
+	[GPLL2] = &gpll2.clkr,
+	[GPLL2_VOTE] = &gpll2_vote,
+	[PCNOC_BFDCD_CLK_SRC] = &pcnoc_bfdcd_clk_src.clkr,
+	[SYSTEM_NOC_BFDCD_CLK_SRC] = &system_noc_bfdcd_clk_src.clkr,
+	[CAMSS_AHB_CLK_SRC] = &camss_ahb_clk_src.clkr,
+	[APSS_AHB_CLK_SRC] = &apss_ahb_clk_src.clkr,
+	[CSI0_CLK_SRC] = &csi0_clk_src.clkr,
+	[CSI1_CLK_SRC] = &csi1_clk_src.clkr,
+	[GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr,
+	[VFE0_CLK_SRC] = &vfe0_clk_src.clkr,
+	[BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr,
+	[BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr,
+	[BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr,
+	[BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr,
+	[BLSP1_QUP5_I2C_APPS_CLK_SRC] = &blsp1_qup5_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP5_SPI_APPS_CLK_SRC] = &blsp1_qup5_spi_apps_clk_src.clkr,
+	[BLSP1_QUP6_I2C_APPS_CLK_SRC] = &blsp1_qup6_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP6_SPI_APPS_CLK_SRC] = &blsp1_qup6_spi_apps_clk_src.clkr,
+	[BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr,
+	[BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr,
+	[CCI_CLK_SRC] = &cci_clk_src.clkr,
+	[CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr,
+	[CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr,
+	[JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr,
+	[MCLK0_CLK_SRC] = &mclk0_clk_src.clkr,
+	[MCLK1_CLK_SRC] = &mclk1_clk_src.clkr,
+	[CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr,
+	[CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr,
+	[CPP_CLK_SRC] = &cpp_clk_src.clkr,
+	[CRYPTO_CLK_SRC] = &crypto_clk_src.clkr,
+	[GP1_CLK_SRC] = &gp1_clk_src.clkr,
+	[GP2_CLK_SRC] = &gp2_clk_src.clkr,
+	[GP3_CLK_SRC] = &gp3_clk_src.clkr,
+	[BYTE0_CLK_SRC] = &byte0_clk_src.clkr,
+	[ESC0_CLK_SRC] = &esc0_clk_src.clkr,
+	[MDP_CLK_SRC] = &mdp_clk_src.clkr,
+	[PCLK0_CLK_SRC] = &pclk0_clk_src.clkr,
+	[VSYNC_CLK_SRC] = &vsync_clk_src.clkr,
+	[PDM2_CLK_SRC] = &pdm2_clk_src.clkr,
+	[SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr,
+	[SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr,
+	[APSS_TCU_CLK_SRC] = &apss_tcu_clk_src.clkr,
+	[USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr,
+	[VCODEC0_CLK_SRC] = &vcodec0_clk_src.clkr,
+	[GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr,
+	[GCC_BLSP1_SLEEP_CLK] = &gcc_blsp1_sleep_clk.clkr,
+	[GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_I2C_APPS_CLK] = &gcc_blsp1_qup5_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP5_SPI_APPS_CLK] = &gcc_blsp1_qup5_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_I2C_APPS_CLK] = &gcc_blsp1_qup6_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP6_SPI_APPS_CLK] = &gcc_blsp1_qup6_spi_apps_clk.clkr,
+	[GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr,
+	[GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_CAMSS_CCI_AHB_CLK] = &gcc_camss_cci_ahb_clk.clkr,
+	[GCC_CAMSS_CCI_CLK] = &gcc_camss_cci_clk.clkr,
+	[GCC_CAMSS_CSI0_AHB_CLK] = &gcc_camss_csi0_ahb_clk.clkr,
+	[GCC_CAMSS_CSI0_CLK] = &gcc_camss_csi0_clk.clkr,
+	[GCC_CAMSS_CSI0PHY_CLK] = &gcc_camss_csi0phy_clk.clkr,
+	[GCC_CAMSS_CSI0PIX_CLK] = &gcc_camss_csi0pix_clk.clkr,
+	[GCC_CAMSS_CSI0RDI_CLK] = &gcc_camss_csi0rdi_clk.clkr,
+	[GCC_CAMSS_CSI1_AHB_CLK] = &gcc_camss_csi1_ahb_clk.clkr,
+	[GCC_CAMSS_CSI1_CLK] = &gcc_camss_csi1_clk.clkr,
+	[GCC_CAMSS_CSI1PHY_CLK] = &gcc_camss_csi1phy_clk.clkr,
+	[GCC_CAMSS_CSI1PIX_CLK] = &gcc_camss_csi1pix_clk.clkr,
+	[GCC_CAMSS_CSI1RDI_CLK] = &gcc_camss_csi1rdi_clk.clkr,
+	[GCC_CAMSS_CSI_VFE0_CLK] = &gcc_camss_csi_vfe0_clk.clkr,
+	[GCC_CAMSS_GP0_CLK] = &gcc_camss_gp0_clk.clkr,
+	[GCC_CAMSS_GP1_CLK] = &gcc_camss_gp1_clk.clkr,
+	[GCC_CAMSS_ISPIF_AHB_CLK] = &gcc_camss_ispif_ahb_clk.clkr,
+	[GCC_CAMSS_JPEG0_CLK] = &gcc_camss_jpeg0_clk.clkr,
+	[GCC_CAMSS_JPEG_AHB_CLK] = &gcc_camss_jpeg_ahb_clk.clkr,
+	[GCC_CAMSS_JPEG_AXI_CLK] = &gcc_camss_jpeg_axi_clk.clkr,
+	[GCC_CAMSS_MCLK0_CLK] = &gcc_camss_mclk0_clk.clkr,
+	[GCC_CAMSS_MCLK1_CLK] = &gcc_camss_mclk1_clk.clkr,
+	[GCC_CAMSS_MICRO_AHB_CLK] = &gcc_camss_micro_ahb_clk.clkr,
+	[GCC_CAMSS_CSI0PHYTIMER_CLK] = &gcc_camss_csi0phytimer_clk.clkr,
+	[GCC_CAMSS_CSI1PHYTIMER_CLK] = &gcc_camss_csi1phytimer_clk.clkr,
+	[GCC_CAMSS_AHB_CLK] = &gcc_camss_ahb_clk.clkr,
+	[GCC_CAMSS_TOP_AHB_CLK] = &gcc_camss_top_ahb_clk.clkr,
+	[GCC_CAMSS_CPP_AHB_CLK] = &gcc_camss_cpp_ahb_clk.clkr,
+	[GCC_CAMSS_CPP_CLK] = &gcc_camss_cpp_clk.clkr,
+	[GCC_CAMSS_VFE0_CLK] = &gcc_camss_vfe0_clk.clkr,
+	[GCC_CAMSS_VFE_AHB_CLK] = &gcc_camss_vfe_ahb_clk.clkr,
+	[GCC_CAMSS_VFE_AXI_CLK] = &gcc_camss_vfe_axi_clk.clkr,
+	[GCC_CRYPTO_AHB_CLK] = &gcc_crypto_ahb_clk.clkr,
+	[GCC_CRYPTO_AXI_CLK] = &gcc_crypto_axi_clk.clkr,
+	[GCC_CRYPTO_CLK] = &gcc_crypto_clk.clkr,
+	[GCC_OXILI_GMEM_CLK] = &gcc_oxili_gmem_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_MDSS_AHB_CLK] = &gcc_mdss_ahb_clk.clkr,
+	[GCC_MDSS_AXI_CLK] = &gcc_mdss_axi_clk.clkr,
+	[GCC_MDSS_BYTE0_CLK] = &gcc_mdss_byte0_clk.clkr,
+	[GCC_MDSS_ESC0_CLK] = &gcc_mdss_esc0_clk.clkr,
+	[GCC_MDSS_MDP_CLK] = &gcc_mdss_mdp_clk.clkr,
+	[GCC_MDSS_PCLK0_CLK] = &gcc_mdss_pclk0_clk.clkr,
+	[GCC_MDSS_VSYNC_CLK] = &gcc_mdss_vsync_clk.clkr,
+	[GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr,
+	[GCC_OXILI_AHB_CLK] = &gcc_oxili_ahb_clk.clkr,
+	[GCC_OXILI_GFX3D_CLK] = &gcc_oxili_gfx3d_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_GTCU_AHB_CLK] = &gcc_gtcu_ahb_clk.clkr,
+	[GCC_JPEG_TBU_CLK] = &gcc_jpeg_tbu_clk.clkr,
+	[GCC_MDP_TBU_CLK] = &gcc_mdp_tbu_clk.clkr,
+	[GCC_SMMU_CFG_CLK] = &gcc_smmu_cfg_clk.clkr,
+	[GCC_VENUS_TBU_CLK] = &gcc_venus_tbu_clk.clkr,
+	[GCC_VFE_TBU_CLK] = &gcc_vfe_tbu_clk.clkr,
+	[GCC_USB2A_PHY_SLEEP_CLK] = &gcc_usb2a_phy_sleep_clk.clkr,
+	[GCC_USB_HS_AHB_CLK] = &gcc_usb_hs_ahb_clk.clkr,
+	[GCC_USB_HS_SYSTEM_CLK] = &gcc_usb_hs_system_clk.clkr,
+	[GCC_VENUS0_AHB_CLK] = &gcc_venus0_ahb_clk.clkr,
+	[GCC_VENUS0_AXI_CLK] = &gcc_venus0_axi_clk.clkr,
+	[GCC_VENUS0_VCODEC0_CLK] = &gcc_venus0_vcodec0_clk.clkr,
+	[BIMC_DDR_CLK_SRC] = &bimc_ddr_clk_src.clkr,
+	[GCC_APSS_TCU_CLK] = &gcc_apss_tcu_clk.clkr,
+	[GCC_GFX_TCU_CLK] = &gcc_gfx_tcu_clk.clkr,
+	[BIMC_GPU_CLK_SRC] = &bimc_gpu_clk_src.clkr,
+	[GCC_BIMC_GFX_CLK] = &gcc_bimc_gfx_clk.clkr,
+	[GCC_BIMC_GPU_CLK] = &gcc_bimc_gpu_clk.clkr,
+	[ULTAUDIO_AHBFABRIC_CLK_SRC] = &ultaudio_ahbfabric_clk_src.clkr,
+	[ULTAUDIO_LPAIF_PRI_I2S_CLK_SRC] = &ultaudio_lpaif_pri_i2s_clk_src.clkr,
+	[ULTAUDIO_LPAIF_SEC_I2S_CLK_SRC] = &ultaudio_lpaif_sec_i2s_clk_src.clkr,
+	[ULTAUDIO_LPAIF_AUX_I2S_CLK_SRC] = &ultaudio_lpaif_aux_i2s_clk_src.clkr,
+	[ULTAUDIO_XO_CLK_SRC] = &ultaudio_xo_clk_src.clkr,
+	[CODEC_DIGCODEC_CLK_SRC] = &codec_digcodec_clk_src.clkr,
+	[GCC_ULTAUDIO_PCNOC_MPORT_CLK] = &gcc_ultaudio_pcnoc_mport_clk.clkr,
+	[GCC_ULTAUDIO_PCNOC_SWAY_CLK] = &gcc_ultaudio_pcnoc_sway_clk.clkr,
+	[GCC_ULTAUDIO_AVSYNC_XO_CLK] = &gcc_ultaudio_avsync_xo_clk.clkr,
+	[GCC_ULTAUDIO_STC_XO_CLK] = &gcc_ultaudio_stc_xo_clk.clkr,
+	[GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_CLK] = &gcc_ultaudio_ahbfabric_ixfabric_clk.clkr,
+	[GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_LPM_CLK] = &gcc_ultaudio_ahbfabric_ixfabric_lpm_clk.clkr,
+	[GCC_ULTAUDIO_LPAIF_PRI_I2S_CLK] = &gcc_ultaudio_lpaif_pri_i2s_clk.clkr,
+	[GCC_ULTAUDIO_LPAIF_SEC_I2S_CLK] = &gcc_ultaudio_lpaif_sec_i2s_clk.clkr,
+	[GCC_ULTAUDIO_LPAIF_AUX_I2S_CLK] = &gcc_ultaudio_lpaif_aux_i2s_clk.clkr,
+	[GCC_CODEC_DIGCODEC_CLK] = &gcc_codec_digcodec_clk.clkr,
+	[GCC_MSS_Q6_BIMC_AXI_CLK] = &gcc_mss_q6_bimc_axi_clk.clkr,
+	[GPLL3] = &gpll3.clkr,
+	[GPLL3_VOTE] = &gpll3_vote,
+	[GPLL4] = &gpll4.clkr,
+	[GPLL4_VOTE] = &gpll4_vote,
+	[GPLL5] = &gpll5.clkr,
+	[GPLL5_VOTE] = &gpll5_vote,
+	[GPLL6] = &gpll6.clkr,
+	[GPLL6_VOTE] = &gpll6_vote,
+	[BYTE1_CLK_SRC] = &byte1_clk_src.clkr,
+	[GCC_MDSS_BYTE1_CLK] = &gcc_mdss_byte1_clk.clkr,
+	[ESC1_CLK_SRC] = &esc1_clk_src.clkr,
+	[GCC_MDSS_ESC1_CLK] = &gcc_mdss_esc1_clk.clkr,
+	[PCLK1_CLK_SRC] = &pclk1_clk_src.clkr,
+	[GCC_MDSS_PCLK1_CLK] = &gcc_mdss_pclk1_clk.clkr,
+	[GCC_GFX_TBU_CLK] = &gcc_gfx_tbu_clk.clkr,
+	[GCC_CPP_TBU_CLK] = &gcc_cpp_tbu_clk.clkr,
+	[GCC_MDP_RT_TBU_CLK] = &gcc_mdp_rt_tbu_clk.clkr,
+	[USB_FS_SYSTEM_CLK_SRC] = &usb_fs_system_clk_src.clkr,
+	[USB_FS_IC_CLK_SRC] = &usb_fs_ic_clk_src.clkr,
+	[GCC_USB_FS_AHB_CLK] = &gcc_usb_fs_ahb_clk.clkr,
+	[GCC_USB_FS_IC_CLK] = &gcc_usb_fs_ic_clk.clkr,
+	[GCC_USB_FS_SYSTEM_CLK] = &gcc_usb_fs_system_clk.clkr,
+	[GCC_VENUS0_CORE0_VCODEC0_CLK] = &gcc_venus0_core0_vcodec0_clk.clkr,
+	[GCC_VENUS0_CORE1_VCODEC0_CLK] = &gcc_venus0_core1_vcodec0_clk.clkr,
+	[GCC_OXILI_TIMER_CLK] = &gcc_oxili_timer_clk.clkr,
+};
+
+static struct gdsc *gcc_msm8939_gdscs[] = {
+	[VENUS_GDSC] = &venus_gdsc,
+	[MDSS_GDSC] = &mdss_gdsc,
+	[JPEG_GDSC] = &jpeg_gdsc,
+	[VFE_GDSC] = &vfe_gdsc,
+	[OXILI_GDSC] = &oxili_gdsc,
+	[VENUS_CORE0_GDSC] = &venus_core0_gdsc,
+	[VENUS_CORE1_GDSC] = &venus_core1_gdsc,
+};
+
+static const struct qcom_reset_map gcc_msm8939_resets[] = {
+	[GCC_BLSP1_BCR] = { 0x01000 },
+	[GCC_BLSP1_QUP1_BCR] = { 0x02000 },
+	[GCC_BLSP1_UART1_BCR] = { 0x02038 },
+	[GCC_BLSP1_QUP2_BCR] = { 0x03008 },
+	[GCC_BLSP1_UART2_BCR] = { 0x03028 },
+	[GCC_BLSP1_QUP3_BCR] = { 0x04018 },
+	[GCC_BLSP1_UART3_BCR] = { 0x04038 },
+	[GCC_BLSP1_QUP4_BCR] = { 0x05018 },
+	[GCC_BLSP1_QUP5_BCR] = { 0x06018 },
+	[GCC_BLSP1_QUP6_BCR] = { 0x07018 },
+	[GCC_IMEM_BCR] = { 0x0e000 },
+	[GCC_SMMU_BCR] = { 0x12000 },
+	[GCC_APSS_TCU_BCR] = { 0x12050 },
+	[GCC_SMMU_XPU_BCR] = { 0x12054 },
+	[GCC_PCNOC_TBU_BCR] = { 0x12058 },
+	[GCC_PRNG_BCR] = { 0x13000 },
+	[GCC_BOOT_ROM_BCR] = { 0x13008 },
+	[GCC_CRYPTO_BCR] = { 0x16000 },
+	[GCC_SEC_CTRL_BCR] = { 0x1a000 },
+	[GCC_AUDIO_CORE_BCR] = { 0x1c008 },
+	[GCC_ULT_AUDIO_BCR] = { 0x1c0b4 },
+	[GCC_DEHR_BCR] = { 0x1f000 },
+	[GCC_SYSTEM_NOC_BCR] = { 0x26000 },
+	[GCC_PCNOC_BCR] = { 0x27018 },
+	[GCC_TCSR_BCR] = { 0x28000 },
+	[GCC_QDSS_BCR] = { 0x29000 },
+	[GCC_DCD_BCR] = { 0x2a000 },
+	[GCC_MSG_RAM_BCR] = { 0x2b000 },
+	[GCC_MPM_BCR] = { 0x2c000 },
+	[GCC_SPMI_BCR] = { 0x2e000 },
+	[GCC_SPDM_BCR] = { 0x2f000 },
+	[GCC_MM_SPDM_BCR] = { 0x2f024 },
+	[GCC_BIMC_BCR] = { 0x31000 },
+	[GCC_RBCPR_BCR] = { 0x33000 },
+	[GCC_TLMM_BCR] = { 0x34000 },
+	[GCC_CAMSS_CSI2_BCR] = { 0x3c038 },
+	[GCC_CAMSS_CSI2PHY_BCR] = { 0x3c044 },
+	[GCC_CAMSS_CSI2RDI_BCR] = { 0x3c04c },
+	[GCC_CAMSS_CSI2PIX_BCR] = { 0x3c054 },
+	[GCC_USB_FS_BCR] = { 0x3f000 },
+	[GCC_USB_HS_BCR] = { 0x41000 },
+	[GCC_USB2A_PHY_BCR] = { 0x41028 },
+	[GCC_SDCC1_BCR] = { 0x42000 },
+	[GCC_SDCC2_BCR] = { 0x43000 },
+	[GCC_PDM_BCR] = { 0x44000 },
+	[GCC_SNOC_BUS_TIMEOUT0_BCR] = { 0x47000 },
+	[GCC_PCNOC_BUS_TIMEOUT0_BCR] = { 0x48000 },
+	[GCC_PCNOC_BUS_TIMEOUT1_BCR] = { 0x48008 },
+	[GCC_PCNOC_BUS_TIMEOUT2_BCR] = { 0x48010 },
+	[GCC_PCNOC_BUS_TIMEOUT3_BCR] = { 0x48018 },
+	[GCC_PCNOC_BUS_TIMEOUT4_BCR] = { 0x48020 },
+	[GCC_PCNOC_BUS_TIMEOUT5_BCR] = { 0x48028 },
+	[GCC_PCNOC_BUS_TIMEOUT6_BCR] = { 0x48030 },
+	[GCC_PCNOC_BUS_TIMEOUT7_BCR] = { 0x48038 },
+	[GCC_PCNOC_BUS_TIMEOUT8_BCR] = { 0x48040 },
+	[GCC_PCNOC_BUS_TIMEOUT9_BCR] = { 0x48048 },
+	[GCC_MMSS_BCR] = { 0x4b000 },
+	[GCC_VENUS0_BCR] = { 0x4c014 },
+	[GCC_MDSS_BCR] = { 0x4d074 },
+	[GCC_CAMSS_PHY0_BCR] = { 0x4e018 },
+	[GCC_CAMSS_CSI0_BCR] = { 0x4e038 },
+	[GCC_CAMSS_CSI0PHY_BCR] = { 0x4e044 },
+	[GCC_CAMSS_CSI0RDI_BCR] = { 0x4e04c },
+	[GCC_CAMSS_CSI0PIX_BCR] = { 0x4e054 },
+	[GCC_CAMSS_PHY1_BCR] = { 0x4f018 },
+	[GCC_CAMSS_CSI1_BCR] = { 0x4f038 },
+	[GCC_CAMSS_CSI1PHY_BCR] = { 0x4f044 },
+	[GCC_CAMSS_CSI1RDI_BCR] = { 0x4f04c },
+	[GCC_CAMSS_CSI1PIX_BCR] = { 0x4f054 },
+	[GCC_CAMSS_ISPIF_BCR] = { 0x50000 },
+	[GCC_BLSP1_QUP4_SPI_APPS_CBCR] = { 0x0501c },
+	[GCC_CAMSS_CCI_BCR] = { 0x51014 },
+	[GCC_CAMSS_MCLK0_BCR] = { 0x52014 },
+	[GCC_CAMSS_MCLK1_BCR] = { 0x53014 },
+	[GCC_CAMSS_GP0_BCR] = { 0x54014 },
+	[GCC_CAMSS_GP1_BCR] = { 0x55014 },
+	[GCC_CAMSS_TOP_BCR] = { 0x56000 },
+	[GCC_CAMSS_MICRO_BCR] = { 0x56008 },
+	[GCC_CAMSS_JPEG_BCR] = { 0x57018 },
+	[GCC_CAMSS_VFE_BCR] = { 0x58030 },
+	[GCC_CAMSS_CSI_VFE0_BCR] = { 0x5804c },
+	[GCC_OXILI_BCR] = { 0x59018 },
+	[GCC_GMEM_BCR] = { 0x5902c },
+	[GCC_CAMSS_AHB_BCR] = { 0x5a018 },
+	[GCC_CAMSS_MCLK2_BCR] = { 0x5c014 },
+	[GCC_MDP_TBU_BCR] = { 0x62000 },
+	[GCC_GFX_TBU_BCR] = { 0x63000 },
+	[GCC_GFX_TCU_BCR] = { 0x64000 },
+	[GCC_MSS_TBU_AXI_BCR] = { 0x65000 },
+	[GCC_MSS_TBU_GSS_AXI_BCR] = { 0x66000 },
+	[GCC_MSS_TBU_Q6_AXI_BCR] = { 0x67000 },
+	[GCC_GTCU_AHB_BCR] = { 0x68000 },
+	[GCC_SMMU_CFG_BCR] = { 0x69000 },
+	[GCC_VFE_TBU_BCR] = { 0x6a000 },
+	[GCC_VENUS_TBU_BCR] = { 0x6b000 },
+	[GCC_JPEG_TBU_BCR] = { 0x6c000 },
+	[GCC_PRONTO_TBU_BCR] = { 0x6d000 },
+	[GCC_CPP_TBU_BCR] = { 0x6e000 },
+	[GCC_MDP_RT_TBU_BCR] = { 0x6f000 },
+	[GCC_SMMU_CATS_BCR] = { 0x7c000 },
+};
+
+static const struct regmap_config gcc_msm8939_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x80000,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc gcc_msm8939_desc = {
+	.config = &gcc_msm8939_regmap_config,
+	.clks = gcc_msm8939_clocks,
+	.num_clks = ARRAY_SIZE(gcc_msm8939_clocks),
+	.resets = gcc_msm8939_resets,
+	.num_resets = ARRAY_SIZE(gcc_msm8939_resets),
+	.gdscs = gcc_msm8939_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_msm8939_gdscs),
+};
+
+static const struct of_device_id gcc_msm8939_match_table[] = {
+	{ .compatible = "qcom,gcc-msm8939" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_msm8939_match_table);
+
+static int gcc_msm8939_probe(struct platform_device *pdev)
+{
+	struct regmap *regmap;
+
+	regmap = qcom_cc_map(pdev, &gcc_msm8939_desc);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	clk_pll_configure_sr_hpm_lp(&gpll3, regmap, &gpll3_config, true);
+	clk_pll_configure_sr_hpm_lp(&gpll4, regmap, &gpll4_config, true);
+
+	return qcom_cc_really_probe(pdev, &gcc_msm8939_desc, regmap);
+}
+
+static struct platform_driver gcc_msm8939_driver = {
+	.probe		= gcc_msm8939_probe,
+	.driver		= {
+		.name	= "gcc-msm8939",
+		.of_match_table = gcc_msm8939_match_table,
+	},
+};
+
+static int __init gcc_msm8939_init(void)
+{
+	return platform_driver_register(&gcc_msm8939_driver);
+}
+core_initcall(gcc_msm8939_init);
+
+static void __exit gcc_msm8939_exit(void)
+{
+	platform_driver_unregister(&gcc_msm8939_driver);
+}
+module_exit(gcc_msm8939_exit);
+
+MODULE_DESCRIPTION("Qualcomm GCC MSM8939 Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c
index df1d7056436c..9d7016bcd680 100644
--- a/drivers/clk/qcom/gcc-msm8998.c
+++ b/drivers/clk/qcom/gcc-msm8998.c
@@ -1110,6 +1110,27 @@ static struct clk_rcg2 ufs_axi_clk_src = {
 	},
 };
 
+static const struct freq_tbl ftbl_ufs_unipro_core_clk_src[] = {
+	F(37500000, P_GPLL0_OUT_MAIN, 16, 0, 0),
+	F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ufs_unipro_core_clk_src = {
+	.cmd_rcgr = 0x76028,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_ufs_unipro_core_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ufs_unipro_core_clk_src",
+		.parent_names = gcc_parent_names_0,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
 static const struct freq_tbl ftbl_usb30_master_clk_src[] = {
 	F(19200000, P_XO, 1, 0, 0),
 	F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0),
@@ -2549,6 +2570,11 @@ static struct clk_branch gcc_ufs_unipro_core_clk = {
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
 			.name = "gcc_ufs_unipro_core_clk",
+			.parent_names = (const char *[]){
+				"ufs_unipro_core_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
 			.ops = &clk_branch2_ops,
 		},
 	},
@@ -2904,6 +2930,7 @@ static struct clk_regmap *gcc_msm8998_clocks[] = {
 	[SDCC4_APPS_CLK_SRC] = &sdcc4_apps_clk_src.clkr,
 	[TSIF_REF_CLK_SRC] = &tsif_ref_clk_src.clkr,
 	[UFS_AXI_CLK_SRC] = &ufs_axi_clk_src.clkr,
+	[UFS_UNIPRO_CORE_CLK_SRC] = &ufs_unipro_core_clk_src.clkr,
 	[USB30_MASTER_CLK_SRC] = &usb30_master_clk_src.clkr,
 	[USB30_MOCK_UTMI_CLK_SRC] = &usb30_mock_utmi_clk_src.clkr,
 	[USB3_PHY_AUX_CLK_SRC] = &usb3_phy_aux_clk_src.clkr,
diff --git a/drivers/clk/qcom/gcc-sc7180.c b/drivers/clk/qcom/gcc-sc7180.c
index 6a51b5b5fc19..ca4383e3a02a 100644
--- a/drivers/clk/qcom/gcc-sc7180.c
+++ b/drivers/clk/qcom/gcc-sc7180.c
@@ -390,6 +390,7 @@ static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = {
 	F(29491200, P_GPLL0_OUT_EVEN, 1, 1536, 15625),
 	F(32000000, P_GPLL0_OUT_EVEN, 1, 8, 75),
 	F(48000000, P_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GPLL6_OUT_MAIN, 7.5, 0, 0),
 	F(64000000, P_GPLL0_OUT_EVEN, 1, 16, 75),
 	F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0),
 	F(80000000, P_GPLL0_OUT_EVEN, 1, 4, 15),
@@ -405,8 +406,8 @@ static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = {
 
 static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = {
 	.name = "gcc_qupv3_wrap0_s0_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -414,15 +415,15 @@ static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = {
 	.cmd_rcgr = 0x17034,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = {
 	.name = "gcc_qupv3_wrap0_s1_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -430,15 +431,15 @@ static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = {
 	.cmd_rcgr = 0x17164,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = {
 	.name = "gcc_qupv3_wrap0_s2_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -446,15 +447,15 @@ static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = {
 	.cmd_rcgr = 0x17294,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = {
 	.name = "gcc_qupv3_wrap0_s3_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -462,15 +463,15 @@ static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = {
 	.cmd_rcgr = 0x173c4,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = {
 	.name = "gcc_qupv3_wrap0_s4_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -478,15 +479,15 @@ static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = {
 	.cmd_rcgr = 0x174f4,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = {
 	.name = "gcc_qupv3_wrap0_s5_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -494,15 +495,15 @@ static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = {
 	.cmd_rcgr = 0x17624,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = {
 	.name = "gcc_qupv3_wrap1_s0_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -510,15 +511,15 @@ static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = {
 	.cmd_rcgr = 0x18018,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = {
 	.name = "gcc_qupv3_wrap1_s1_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -526,15 +527,15 @@ static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = {
 	.cmd_rcgr = 0x18148,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap1_s2_clk_src_init = {
 	.name = "gcc_qupv3_wrap1_s2_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -542,15 +543,15 @@ static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = {
 	.cmd_rcgr = 0x18278,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap1_s2_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = {
 	.name = "gcc_qupv3_wrap1_s3_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -558,15 +559,15 @@ static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = {
 	.cmd_rcgr = 0x183a8,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = {
 	.name = "gcc_qupv3_wrap1_s4_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -574,15 +575,15 @@ static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = {
 	.cmd_rcgr = 0x184d8,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init,
 };
 
 static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = {
 	.name = "gcc_qupv3_wrap1_s5_clk_src",
-	.parent_data = gcc_parent_data_0,
-	.num_parents = 4,
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
 	.ops = &clk_rcg2_ops,
 };
 
@@ -590,7 +591,7 @@ static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = {
 	.cmd_rcgr = 0x18608,
 	.mnd_width = 16,
 	.hid_width = 5,
-	.parent_map = gcc_parent_map_0,
+	.parent_map = gcc_parent_map_1,
 	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
 	.clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init,
 };
@@ -816,6 +817,26 @@ static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = {
 	},
 };
 
+static const struct freq_tbl ftbl_gcc_sec_ctrl_clk_src[] = {
+	F(4800000, P_BI_TCXO, 4, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sec_ctrl_clk_src = {
+	.cmd_rcgr = 0x3d030,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_sec_ctrl_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gcc_sec_ctrl_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
 static struct clk_branch gcc_aggre_ufs_phy_axi_clk = {
 	.halt_reg = 0x82024,
 	.halt_check = BRANCH_HALT_DELAY,
@@ -2406,6 +2427,7 @@ static struct clk_regmap *gcc_sc7180_clocks[] = {
 	[GCC_MSS_NAV_AXI_CLK] = &gcc_mss_nav_axi_clk.clkr,
 	[GCC_MSS_Q6_MEMNOC_AXI_CLK] = &gcc_mss_q6_memnoc_axi_clk.clkr,
 	[GCC_MSS_SNOC_AXI_CLK] = &gcc_mss_snoc_axi_clk.clkr,
+	[GCC_SEC_CTRL_CLK_SRC] = &gcc_sec_ctrl_clk_src.clkr,
 };
 
 static const struct qcom_reset_map gcc_sc7180_resets[] = {
diff --git a/drivers/clk/qcom/gcc-sm8150.c b/drivers/clk/qcom/gcc-sm8150.c
index 732bc7c937e6..72524cf11048 100644
--- a/drivers/clk/qcom/gcc-sm8150.c
+++ b/drivers/clk/qcom/gcc-sm8150.c
@@ -1616,6 +1616,36 @@ static struct clk_branch gcc_gpu_cfg_ahb_clk = {
 	},
 };
 
+static struct clk_branch gcc_gpu_gpll0_clk_src = {
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(15),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_gpll0_clk_src",
+			.parent_hws = (const struct clk_hw *[]){
+				&gpll0.clkr.hw },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_div_clk_src = {
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(16),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gpu_gpll0_div_clk_src",
+			.parent_hws = (const struct clk_hw *[]){
+				&gcc_gpu_gpll0_clk_src.clkr.hw },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_gpu_iref_clk = {
 	.halt_reg = 0x8c010,
 	.halt_check = BRANCH_HALT,
@@ -1698,6 +1728,36 @@ static struct clk_branch gcc_npu_cfg_ahb_clk = {
 	},
 };
 
+static struct clk_branch gcc_npu_gpll0_clk_src = {
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(18),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_npu_gpll0_clk_src",
+			.parent_hws = (const struct clk_hw *[]){
+				&gpll0.clkr.hw },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_npu_gpll0_div_clk_src = {
+	.clkr = {
+		.enable_reg = 0x52004,
+		.enable_mask = BIT(19),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_npu_gpll0_div_clk_src",
+			.parent_hws = (const struct clk_hw *[]){
+				&gcc_npu_gpll0_clk_src.clkr.hw },
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_npu_trig_clk = {
 	.halt_reg = 0x4d00c,
 	.halt_check = BRANCH_VOTED,
@@ -2812,6 +2872,45 @@ static struct clk_branch gcc_ufs_card_phy_aux_hw_ctl_clk = {
 	},
 };
 
+/* external clocks so add BRANCH_HALT_SKIP */
+static struct clk_branch gcc_ufs_card_rx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x7501c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_rx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+/* external clocks so add BRANCH_HALT_SKIP */
+static struct clk_branch gcc_ufs_card_rx_symbol_1_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x750ac,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_rx_symbol_1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+/* external clocks so add BRANCH_HALT_SKIP */
+static struct clk_branch gcc_ufs_card_tx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x75018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_card_tx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_ufs_card_unipro_core_clk = {
 	.halt_reg = 0x75058,
 	.halt_check = BRANCH_HALT,
@@ -2992,6 +3091,45 @@ static struct clk_branch gcc_ufs_phy_phy_aux_hw_ctl_clk = {
 	},
 };
 
+/* external clocks so add BRANCH_HALT_SKIP */
+static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x7701c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_rx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+/* external clocks so add BRANCH_HALT_SKIP */
+static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x770ac,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_rx_symbol_1_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+/* external clocks so add BRANCH_HALT_SKIP */
+static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = {
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x77018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ufs_phy_tx_symbol_0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_ufs_phy_unipro_core_clk = {
 	.halt_reg = 0x77058,
 	.halt_check = BRANCH_HALT,
@@ -3374,12 +3512,16 @@ static struct clk_regmap *gcc_sm8150_clocks[] = {
 	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
 	[GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr,
 	[GCC_GPU_CFG_AHB_CLK] = &gcc_gpu_cfg_ahb_clk.clkr,
+	[GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr,
+	[GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr,
 	[GCC_GPU_IREF_CLK] = &gcc_gpu_iref_clk.clkr,
 	[GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr,
 	[GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr,
 	[GCC_NPU_AT_CLK] = &gcc_npu_at_clk.clkr,
 	[GCC_NPU_AXI_CLK] = &gcc_npu_axi_clk.clkr,
 	[GCC_NPU_CFG_AHB_CLK] = &gcc_npu_cfg_ahb_clk.clkr,
+	[GCC_NPU_GPLL0_CLK_SRC] = &gcc_npu_gpll0_clk_src.clkr,
+	[GCC_NPU_GPLL0_DIV_CLK_SRC] = &gcc_npu_gpll0_div_clk_src.clkr,
 	[GCC_NPU_TRIG_CLK] = &gcc_npu_trig_clk.clkr,
 	[GCC_PCIE0_PHY_REFGEN_CLK] = &gcc_pcie0_phy_refgen_clk.clkr,
 	[GCC_PCIE1_PHY_REFGEN_CLK] = &gcc_pcie1_phy_refgen_clk.clkr,
@@ -3484,6 +3626,9 @@ static struct clk_regmap *gcc_sm8150_clocks[] = {
 	[GCC_UFS_CARD_PHY_AUX_CLK_SRC] = &gcc_ufs_card_phy_aux_clk_src.clkr,
 	[GCC_UFS_CARD_PHY_AUX_HW_CTL_CLK] =
 		&gcc_ufs_card_phy_aux_hw_ctl_clk.clkr,
+	[GCC_UFS_CARD_RX_SYMBOL_0_CLK] = &gcc_ufs_card_rx_symbol_0_clk.clkr,
+	[GCC_UFS_CARD_RX_SYMBOL_1_CLK] = &gcc_ufs_card_rx_symbol_1_clk.clkr,
+	[GCC_UFS_CARD_TX_SYMBOL_0_CLK] = &gcc_ufs_card_tx_symbol_0_clk.clkr,
 	[GCC_UFS_CARD_UNIPRO_CORE_CLK] = &gcc_ufs_card_unipro_core_clk.clkr,
 	[GCC_UFS_CARD_UNIPRO_CORE_CLK_SRC] =
 		&gcc_ufs_card_unipro_core_clk_src.clkr,
@@ -3501,6 +3646,9 @@ static struct clk_regmap *gcc_sm8150_clocks[] = {
 	[GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr,
 	[GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr,
 	[GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_phy_phy_aux_hw_ctl_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr,
 	[GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr,
 	[GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] =
 		&gcc_ufs_phy_unipro_core_clk_src.clkr,
diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c
index a250f59708d8..04944f11659b 100644
--- a/drivers/clk/qcom/gdsc.c
+++ b/drivers/clk/qcom/gdsc.c
@@ -11,6 +11,7 @@
 #include <linux/ktime.h>
 #include <linux/pm_domain.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <linux/reset-controller.h>
 #include <linux/slab.h>
 #include "gdsc.h"
@@ -112,6 +113,12 @@ static int gdsc_toggle_logic(struct gdsc *sc, enum gdsc_status status)
 	int ret;
 	u32 val = (status == GDSC_ON) ? 0 : SW_COLLAPSE_MASK;
 
+	if (status == GDSC_ON && sc->rsupply) {
+		ret = regulator_enable(sc->rsupply);
+		if (ret < 0)
+			return ret;
+	}
+
 	ret = regmap_update_bits(sc->regmap, sc->gdscr, SW_COLLAPSE_MASK, val);
 	if (ret)
 		return ret;
@@ -143,6 +150,13 @@ static int gdsc_toggle_logic(struct gdsc *sc, enum gdsc_status status)
 
 	ret = gdsc_poll_status(sc, status);
 	WARN(ret, "%s status stuck at 'o%s'", sc->pd.name, status ? "ff" : "n");
+
+	if (!ret && status == GDSC_OFF && sc->rsupply) {
+		ret = regulator_disable(sc->rsupply);
+		if (ret < 0)
+			return ret;
+	}
+
 	return ret;
 }
 
@@ -371,6 +385,15 @@ int gdsc_register(struct gdsc_desc *desc,
 	if (!data->domains)
 		return -ENOMEM;
 
+	for (i = 0; i < num; i++) {
+		if (!scs[i] || !scs[i]->supply)
+			continue;
+
+		scs[i]->rsupply = devm_regulator_get(dev, scs[i]->supply);
+		if (IS_ERR(scs[i]->rsupply))
+			return PTR_ERR(scs[i]->rsupply);
+	}
+
 	data->num_domains = num;
 	for (i = 0; i < num; i++) {
 		if (!scs[i])
diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h
index 64cdc8cf0d4d..c36fc26dcdff 100644
--- a/drivers/clk/qcom/gdsc.h
+++ b/drivers/clk/qcom/gdsc.h
@@ -10,6 +10,7 @@
 #include <linux/pm_domain.h>
 
 struct regmap;
+struct regulator;
 struct reset_controller_dev;
 
 /**
@@ -52,6 +53,9 @@ struct gdsc {
 	struct reset_controller_dev	*rcdev;
 	unsigned int			*resets;
 	unsigned int			reset_count;
+
+	const char 			*supply;
+	struct regulator		*rsupply;
 };
 
 struct gdsc_desc {
diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c
index 6c7592ddf8bb..3b3aac07fb2d 100644
--- a/drivers/clk/qcom/mmcc-msm8996.c
+++ b/drivers/clk/qcom/mmcc-msm8996.c
@@ -3064,7 +3064,9 @@ static struct gdsc gpu_gx_gdsc = {
 		.name = "gpu_gx",
 	},
 	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &gpu_gdsc.pd,
 	.flags = CLAMP_IO,
+	.supply = "vdd-gfx",
 };
 
 static struct clk_regmap *mmcc_msm8996_clocks[] = {
diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig
index ac2dd92ce2ef..9eb79bf90643 100644
--- a/drivers/clk/renesas/Kconfig
+++ b/drivers/clk/renesas/Kconfig
@@ -8,6 +8,7 @@ config CLK_RENESAS
 	select CLK_R7S9210 if ARCH_R7S9210
 	select CLK_R8A73A4 if ARCH_R8A73A4
 	select CLK_R8A7740 if ARCH_R8A7740
+	select CLK_R8A7742 if ARCH_R8A7742
 	select CLK_R8A7743 if ARCH_R8A7743 || ARCH_R8A7744
 	select CLK_R8A7745 if ARCH_R8A7745
 	select CLK_R8A77470 if ARCH_R8A77470
@@ -55,6 +56,10 @@ config CLK_R8A7740
 	select CLK_RENESAS_CPG_MSTP
 	select CLK_RENESAS_DIV6
 
+config CLK_R8A7742
+	bool "RZ/G1H clock support" if COMPILE_TEST
+	select CLK_RCAR_GEN2_CPG
+
 config CLK_R8A7743
 	bool "RZ/G1M clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2_CPG
@@ -90,12 +95,10 @@ config CLK_R8A7779
 config CLK_R8A7790
 	bool "R-Car H2 clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2_CPG
-	select CLK_RENESAS_DIV6
 
 config CLK_R8A7791
 	bool "R-Car M2-W/N clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2_CPG
-	select CLK_RENESAS_DIV6
 
 config CLK_R8A7792
 	bool "R-Car V2H clock support" if COMPILE_TEST
@@ -104,7 +107,6 @@ config CLK_R8A7792
 config CLK_R8A7794
 	bool "R-Car E2 clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2_CPG
-	select CLK_RENESAS_DIV6
 
 config CLK_R8A7795
 	bool "R-Car H3 clock support" if COMPILE_TEST
diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile
index 4a722bc5aac7..a4066f9b34ef 100644
--- a/drivers/clk/renesas/Makefile
+++ b/drivers/clk/renesas/Makefile
@@ -5,6 +5,7 @@ obj-$(CONFIG_CLK_RZA1)			+= clk-rz.o
 obj-$(CONFIG_CLK_R7S9210)		+= r7s9210-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A73A4)		+= clk-r8a73a4.o
 obj-$(CONFIG_CLK_R8A7740)		+= clk-r8a7740.o
+obj-$(CONFIG_CLK_R8A7742)		+= r8a7742-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7743)		+= r8a7743-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7745)		+= r8a7745-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A77470)		+= r8a77470-cpg-mssr.o
diff --git a/drivers/clk/renesas/r8a7742-cpg-mssr.c b/drivers/clk/renesas/r8a7742-cpg-mssr.c
new file mode 100644
index 000000000000..e919828668a4
--- /dev/null
+++ b/drivers/clk/renesas/r8a7742-cpg-mssr.c
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * r8a7742 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2020 Renesas Electronics Corp.
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+#include <dt-bindings/clock/r8a7742-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen2-cpg.h"
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A7742_CLK_OSC,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+	CLK_USB_EXTAL,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL3,
+	CLK_PLL1_DIV2,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+static const struct cpg_core_clk r8a7742_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",	CLK_EXTAL),
+	DEF_INPUT("usb_extal",	CLK_USB_EXTAL),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",	CLK_MAIN, CLK_TYPE_GEN2_MAIN, CLK_EXTAL),
+	DEF_BASE(".pll0",	CLK_PLL0, CLK_TYPE_GEN2_PLL0, CLK_MAIN),
+	DEF_BASE(".pll1",	CLK_PLL1, CLK_TYPE_GEN2_PLL1, CLK_MAIN),
+	DEF_BASE(".pll3",	CLK_PLL3, CLK_TYPE_GEN2_PLL3, CLK_MAIN),
+
+	DEF_FIXED(".pll1_div2",	CLK_PLL1_DIV2, CLK_PLL1, 2, 1),
+
+	/* Core Clock Outputs */
+	DEF_BASE("z",    R8A7742_CLK_Z,    CLK_TYPE_GEN2_Z,	CLK_PLL0),
+	DEF_BASE("lb",   R8A7742_CLK_LB,   CLK_TYPE_GEN2_LB,	CLK_PLL1),
+	DEF_BASE("sdh",  R8A7742_CLK_SDH,  CLK_TYPE_GEN2_SDH,	CLK_PLL1),
+	DEF_BASE("sd0",  R8A7742_CLK_SD0,  CLK_TYPE_GEN2_SD0,	CLK_PLL1),
+	DEF_BASE("sd1",  R8A7742_CLK_SD1,  CLK_TYPE_GEN2_SD1,	CLK_PLL1),
+	DEF_BASE("qspi", R8A7742_CLK_QSPI, CLK_TYPE_GEN2_QSPI,	CLK_PLL1_DIV2),
+	DEF_BASE("rcan", R8A7742_CLK_RCAN, CLK_TYPE_GEN2_RCAN,	CLK_USB_EXTAL),
+
+	DEF_FIXED("z2",    R8A7742_CLK_Z2,	CLK_PLL1,	    2, 1),
+	DEF_FIXED("zg",    R8A7742_CLK_ZG,	CLK_PLL1,	    3, 1),
+	DEF_FIXED("zx",    R8A7742_CLK_ZX,	CLK_PLL1,	    3, 1),
+	DEF_FIXED("zs",    R8A7742_CLK_ZS,	CLK_PLL1,	    6, 1),
+	DEF_FIXED("hp",    R8A7742_CLK_HP,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("b",     R8A7742_CLK_B,	CLK_PLL1,	   12, 1),
+	DEF_FIXED("p",     R8A7742_CLK_P,	CLK_PLL1,	   24, 1),
+	DEF_FIXED("cl",    R8A7742_CLK_CL,	CLK_PLL1,	   48, 1),
+	DEF_FIXED("m2",    R8A7742_CLK_M2,	CLK_PLL1,	    8, 1),
+	DEF_FIXED("zb3",   R8A7742_CLK_ZB3,	CLK_PLL3,	    4, 1),
+	DEF_FIXED("zb3d2", R8A7742_CLK_ZB3D2,	CLK_PLL3,	    8, 1),
+	DEF_FIXED("ddr",   R8A7742_CLK_DDR,	CLK_PLL3,	    8, 1),
+	DEF_FIXED("mp",    R8A7742_CLK_MP,	CLK_PLL1_DIV2,	   15, 1),
+	DEF_FIXED("cp",    R8A7742_CLK_CP,	CLK_EXTAL,	    2, 1),
+	DEF_FIXED("r",     R8A7742_CLK_R,	CLK_PLL1,	49152, 1),
+	DEF_FIXED("osc",   R8A7742_CLK_OSC,	CLK_PLL1,	12288, 1),
+
+	DEF_DIV6P1("sd2",  R8A7742_CLK_SD2,	CLK_PLL1_DIV2,	0x078),
+	DEF_DIV6P1("sd3",  R8A7742_CLK_SD3,	CLK_PLL1_DIV2,	0x26c),
+	DEF_DIV6P1("mmc0", R8A7742_CLK_MMC0,	CLK_PLL1_DIV2,	0x240),
+	DEF_DIV6P1("mmc1", R8A7742_CLK_MMC1,	CLK_PLL1_DIV2,	0x244),
+};
+
+static const struct mssr_mod_clk r8a7742_mod_clks[] __initconst = {
+	DEF_MOD("msiof0",		   0,	R8A7742_CLK_MP),
+	DEF_MOD("vcp1",			 100,	R8A7742_CLK_ZS),
+	DEF_MOD("vcp0",			 101,	R8A7742_CLK_ZS),
+	DEF_MOD("vpc1",			 102,	R8A7742_CLK_ZS),
+	DEF_MOD("vpc0",			 103,	R8A7742_CLK_ZS),
+	DEF_MOD("tmu1",			 111,	R8A7742_CLK_P),
+	DEF_MOD("3dg",			 112,	R8A7742_CLK_ZG),
+	DEF_MOD("2d-dmac",		 115,	R8A7742_CLK_ZS),
+	DEF_MOD("fdp1-2",		 117,	R8A7742_CLK_ZS),
+	DEF_MOD("fdp1-1",		 118,	R8A7742_CLK_ZS),
+	DEF_MOD("fdp1-0",		 119,	R8A7742_CLK_ZS),
+	DEF_MOD("tmu3",			 121,	R8A7742_CLK_P),
+	DEF_MOD("tmu2",			 122,	R8A7742_CLK_P),
+	DEF_MOD("cmt0",			 124,	R8A7742_CLK_R),
+	DEF_MOD("tmu0",			 125,	R8A7742_CLK_CP),
+	DEF_MOD("vsp1du1",		 127,	R8A7742_CLK_ZS),
+	DEF_MOD("vsp1du0",		 128,	R8A7742_CLK_ZS),
+	DEF_MOD("vsp1-sy",		 131,	R8A7742_CLK_ZS),
+	DEF_MOD("scifa2",		 202,	R8A7742_CLK_MP),
+	DEF_MOD("scifa1",		 203,	R8A7742_CLK_MP),
+	DEF_MOD("scifa0",		 204,	R8A7742_CLK_MP),
+	DEF_MOD("msiof2",		 205,	R8A7742_CLK_MP),
+	DEF_MOD("scifb0",		 206,	R8A7742_CLK_MP),
+	DEF_MOD("scifb1",		 207,	R8A7742_CLK_MP),
+	DEF_MOD("msiof1",		 208,	R8A7742_CLK_MP),
+	DEF_MOD("msiof3",		 215,	R8A7742_CLK_MP),
+	DEF_MOD("scifb2",		 216,	R8A7742_CLK_MP),
+	DEF_MOD("sys-dmac1",		 218,	R8A7742_CLK_ZS),
+	DEF_MOD("sys-dmac0",		 219,	R8A7742_CLK_ZS),
+	DEF_MOD("iic2",			 300,	R8A7742_CLK_HP),
+	DEF_MOD("tpu0",			 304,	R8A7742_CLK_CP),
+	DEF_MOD("mmcif1",		 305,	R8A7742_CLK_MMC1),
+	DEF_MOD("scif2",		 310,	R8A7742_CLK_P),
+	DEF_MOD("sdhi3",		 311,	R8A7742_CLK_SD3),
+	DEF_MOD("sdhi2",		 312,	R8A7742_CLK_SD2),
+	DEF_MOD("sdhi1",		 313,	R8A7742_CLK_SD1),
+	DEF_MOD("sdhi0",		 314,	R8A7742_CLK_SD0),
+	DEF_MOD("mmcif0",		 315,	R8A7742_CLK_MMC0),
+	DEF_MOD("iic0",			 318,	R8A7742_CLK_HP),
+	DEF_MOD("pciec",		 319,	R8A7742_CLK_MP),
+	DEF_MOD("iic1",			 323,	R8A7742_CLK_HP),
+	DEF_MOD("usb3.0",		 328,	R8A7742_CLK_MP),
+	DEF_MOD("cmt1",			 329,	R8A7742_CLK_R),
+	DEF_MOD("usbhs-dmac0",		 330,	R8A7742_CLK_HP),
+	DEF_MOD("usbhs-dmac1",		 331,	R8A7742_CLK_HP),
+	DEF_MOD("rwdt",			 402,	R8A7742_CLK_R),
+	DEF_MOD("irqc",			 407,	R8A7742_CLK_CP),
+	DEF_MOD("intc-sys",		 408,	R8A7742_CLK_ZS),
+	DEF_MOD("audio-dmac1",		 501,	R8A7742_CLK_HP),
+	DEF_MOD("audio-dmac0",		 502,	R8A7742_CLK_HP),
+	DEF_MOD("thermal",		 522,	CLK_EXTAL),
+	DEF_MOD("pwm",			 523,	R8A7742_CLK_P),
+	DEF_MOD("usb-ehci",		 703,	R8A7742_CLK_MP),
+	DEF_MOD("usbhs",		 704,	R8A7742_CLK_HP),
+	DEF_MOD("hscif1",		 716,	R8A7742_CLK_ZS),
+	DEF_MOD("hscif0",		 717,	R8A7742_CLK_ZS),
+	DEF_MOD("scif1",		 720,	R8A7742_CLK_P),
+	DEF_MOD("scif0",		 721,	R8A7742_CLK_P),
+	DEF_MOD("du2",			 722,	R8A7742_CLK_ZX),
+	DEF_MOD("du1",			 723,	R8A7742_CLK_ZX),
+	DEF_MOD("du0",			 724,	R8A7742_CLK_ZX),
+	DEF_MOD("lvds1",		 725,	R8A7742_CLK_ZX),
+	DEF_MOD("lvds0",		 726,	R8A7742_CLK_ZX),
+	DEF_MOD("r-gp2d",		 807,	R8A7742_CLK_ZX),
+	DEF_MOD("vin3",			 808,	R8A7742_CLK_ZG),
+	DEF_MOD("vin2",			 809,	R8A7742_CLK_ZG),
+	DEF_MOD("vin1",			 810,	R8A7742_CLK_ZG),
+	DEF_MOD("vin0",			 811,	R8A7742_CLK_ZG),
+	DEF_MOD("etheravb",		 812,	R8A7742_CLK_HP),
+	DEF_MOD("ether",		 813,	R8A7742_CLK_P),
+	DEF_MOD("sata1",		 814,	R8A7742_CLK_ZS),
+	DEF_MOD("sata0",		 815,	R8A7742_CLK_ZS),
+	DEF_MOD("imr-x2-1",		 820,	R8A7742_CLK_ZG),
+	DEF_MOD("imr-x2-0",		 821,	R8A7742_CLK_HP),
+	DEF_MOD("imr-lsx2-1",		 822,	R8A7742_CLK_P),
+	DEF_MOD("imr-lsx2-0",		 823,	R8A7742_CLK_ZS),
+	DEF_MOD("gpio5",		 907,	R8A7742_CLK_CP),
+	DEF_MOD("gpio4",		 908,	R8A7742_CLK_CP),
+	DEF_MOD("gpio3",		 909,	R8A7742_CLK_CP),
+	DEF_MOD("gpio2",		 910,	R8A7742_CLK_CP),
+	DEF_MOD("gpio1",		 911,	R8A7742_CLK_CP),
+	DEF_MOD("gpio0",		 912,	R8A7742_CLK_CP),
+	DEF_MOD("can1",			 915,	R8A7742_CLK_P),
+	DEF_MOD("can0",			 916,	R8A7742_CLK_P),
+	DEF_MOD("qspi_mod",		 917,	R8A7742_CLK_QSPI),
+	DEF_MOD("iicdvfs",		 926,	R8A7742_CLK_CP),
+	DEF_MOD("i2c3",			 928,	R8A7742_CLK_HP),
+	DEF_MOD("i2c2",			 929,	R8A7742_CLK_HP),
+	DEF_MOD("i2c1",			 930,	R8A7742_CLK_HP),
+	DEF_MOD("i2c0",			 931,	R8A7742_CLK_HP),
+	DEF_MOD("ssi-all",		1005,	R8A7742_CLK_P),
+	DEF_MOD("ssi9",			1006,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi8",			1007,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi7",			1008,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi6",			1009,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi5",			1010,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi4",			1011,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi3",			1012,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi2",			1013,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi1",			1014,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi0",			1015,	MOD_CLK_ID(1005)),
+	DEF_MOD("scu-all",		1017,	R8A7742_CLK_P),
+	DEF_MOD("scu-dvc1",		1018,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-dvc0",		1019,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu1-mix1",	1020,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu0-mix0",	1021,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src9",		1022,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src8",		1023,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src7",		1024,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src6",		1025,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src5",		1026,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src4",		1027,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src3",		1028,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src2",		1029,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src1",		1030,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src0",		1031,	MOD_CLK_ID(1017)),
+};
+
+static const unsigned int r8a7742_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(402),	/* RWDT */
+	MOD_CLK_ID(408),	/* INTC-SYS (GIC) */
+};
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ *    MD	EXTAL		PLL0	PLL1	PLL3
+ * 14 13 19	(MHz)		*1	*1
+ *---------------------------------------------------
+ * 0  0  0	15		x172/2	x208/2	x106
+ * 0  0  1	15		x172/2	x208/2	x88
+ * 0  1  0	20		x130/2	x156/2	x80
+ * 0  1  1	20		x130/2	x156/2	x66
+ * 1  0  0	26 / 2		x200/2	x240/2	x122
+ * 1  0  1	26 / 2		x200/2	x240/2	x102
+ * 1  1  0	30 / 2		x172/2	x208/2	x106
+ * 1  1  1	30 / 2		x172/2	x208/2	x88
+ *
+ * *1 :	Table 7.5a indicates VCO output (PLLx = VCO/2)
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	((((md) & BIT(14)) >> 12) | \
+					 (((md) & BIT(13)) >> 12) | \
+					 (((md) & BIT(19)) >> 19))
+
+static const struct rcar_gen2_cpg_pll_config cpg_pll_configs[8] __initconst = {
+	/* EXTAL div	PLL1 mult	PLL3 mult */
+	{ 1,		208,		106,	},
+	{ 1,		208,		88,	},
+	{ 1,		156,		80,	},
+	{ 1,		156,		66,	},
+	{ 2,		240,		122,	},
+	{ 2,		240,		102,	},
+	{ 2,		208,		106,	},
+	{ 2,		208,		88,	},
+};
+
+static int __init r8a7742_cpg_mssr_init(struct device *dev)
+{
+	const struct rcar_gen2_cpg_pll_config *cpg_pll_config;
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+
+	return rcar_gen2_cpg_init(cpg_pll_config, 2, cpg_mode);
+}
+
+const struct cpg_mssr_info r8a7742_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a7742_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a7742_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a7742_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a7742_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a7742_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a7742_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a7742_cpg_mssr_init,
+	.cpg_clk_register = rcar_gen2_cpg_clk_register,
+};
diff --git a/drivers/clk/renesas/r9a06g032-clocks.c b/drivers/clk/renesas/r9a06g032-clocks.c
index 1907ee195a08..d900f6bf53d0 100644
--- a/drivers/clk/renesas/r9a06g032-clocks.c
+++ b/drivers/clk/renesas/r9a06g032-clocks.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * R9A09G032 clock driver
+ * R9A06G032 clock driver
  *
  * Copyright (C) 2018 Renesas Electronics Europe Limited
  *
@@ -338,8 +338,8 @@ clk_rdesc_get(struct r9a06g032_priv *clocks,
 }
 
 /*
- * This implements the R9A09G032 clock gate 'driver'. We cannot use the system's
- * clock gate framework as the gates on the R9A09G032 have a special enabling
+ * This implements the R9A06G032 clock gate 'driver'. We cannot use the system's
+ * clock gate framework as the gates on the R9A06G032 have a special enabling
  * sequence, therefore we use this little proxy.
  */
 struct r9a06g032_clk_gate {
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index a2663fbbd7a5..dcb6e2706d37 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -673,6 +673,12 @@ static const struct of_device_id cpg_mssr_match[] = {
 		.data = &r7s9210_cpg_mssr_info,
 	},
 #endif
+#ifdef CONFIG_CLK_R8A7742
+	{
+		.compatible = "renesas,r8a7742-cpg-mssr",
+		.data = &r8a7742_cpg_mssr_info,
+	},
+#endif
 #ifdef CONFIG_CLK_R8A7743
 	{
 		.compatible = "renesas,r8a7743-cpg-mssr",
@@ -812,7 +818,8 @@ static int cpg_mssr_suspend_noirq(struct device *dev)
 	/* Save module registers with bits under our control */
 	for (reg = 0; reg < ARRAY_SIZE(priv->smstpcr_saved); reg++) {
 		if (priv->smstpcr_saved[reg].mask)
-			priv->smstpcr_saved[reg].val =
+			priv->smstpcr_saved[reg].val = priv->stbyctrl ?
+				readb(priv->base + STBCR(reg)) :
 				readl(priv->base + SMSTPCR(reg));
 	}
 
@@ -872,8 +879,9 @@ static int cpg_mssr_resume_noirq(struct device *dev)
 		}
 
 		if (!i)
-			dev_warn(dev, "Failed to enable SMSTP %p[0x%x]\n",
-				 priv->base + SMSTPCR(reg), oldval & mask);
+			dev_warn(dev, "Failed to enable %s%u[0x%x]\n",
+				 priv->stbyctrl ? "STB" : "SMSTP", reg,
+				 oldval & mask);
 	}
 
 	return 0;
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h
index 3b852ba0ecec..55a18ef0efaf 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.h
+++ b/drivers/clk/renesas/renesas-cpg-mssr.h
@@ -155,6 +155,7 @@ struct cpg_mssr_info {
 };
 
 extern const struct cpg_mssr_info r7s9210_cpg_mssr_info;
+extern const struct cpg_mssr_info r8a7742_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7743_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7745_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a77470_cpg_mssr_info;
diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c
index c9e5a1fb6653..fea33399a632 100644
--- a/drivers/clk/samsung/clk-exynos5420.c
+++ b/drivers/clk/samsung/clk-exynos5420.c
@@ -540,7 +540,7 @@ static const struct samsung_div_clock exynos5800_div_clks[] __initconst = {
 
 static const struct samsung_gate_clock exynos5800_gate_clks[] __initconst = {
 	GATE(CLK_ACLK550_CAM, "aclk550_cam", "mout_user_aclk550_cam",
-				GATE_BUS_TOP, 24, 0, 0),
+				GATE_BUS_TOP, 24, CLK_IS_CRITICAL, 0),
 	GATE(CLK_ACLK432_SCALER, "aclk432_scaler", "mout_user_aclk432_scaler",
 				GATE_BUS_TOP, 27, CLK_IS_CRITICAL, 0),
 };
@@ -943,25 +943,25 @@ static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = {
 	GATE(0, "aclk300_jpeg", "mout_user_aclk300_jpeg",
 			GATE_BUS_TOP, 4, CLK_IGNORE_UNUSED, 0),
 	GATE(0, "aclk333_432_isp0", "mout_user_aclk333_432_isp0",
-			GATE_BUS_TOP, 5, 0, 0),
+			GATE_BUS_TOP, 5, CLK_IS_CRITICAL, 0),
 	GATE(0, "aclk300_gscl", "mout_user_aclk300_gscl",
 			GATE_BUS_TOP, 6, CLK_IS_CRITICAL, 0),
 	GATE(0, "aclk333_432_gscl", "mout_user_aclk333_432_gscl",
 			GATE_BUS_TOP, 7, CLK_IGNORE_UNUSED, 0),
 	GATE(0, "aclk333_432_isp", "mout_user_aclk333_432_isp",
-			GATE_BUS_TOP, 8, 0, 0),
+			GATE_BUS_TOP, 8, CLK_IS_CRITICAL, 0),
 	GATE(CLK_PCLK66_GPIO, "pclk66_gpio", "mout_user_pclk66_gpio",
 			GATE_BUS_TOP, 9, CLK_IGNORE_UNUSED, 0),
 	GATE(0, "aclk66_psgen", "mout_user_aclk66_psgen",
 			GATE_BUS_TOP, 10, CLK_IGNORE_UNUSED, 0),
 	GATE(0, "aclk266_isp", "mout_user_aclk266_isp",
-			GATE_BUS_TOP, 13, 0, 0),
+			GATE_BUS_TOP, 13, CLK_IS_CRITICAL, 0),
 	GATE(0, "aclk166", "mout_user_aclk166",
 			GATE_BUS_TOP, 14, CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_ACLK333, "aclk333", "mout_user_aclk333",
 			GATE_BUS_TOP, 15, CLK_IS_CRITICAL, 0),
 	GATE(0, "aclk400_isp", "mout_user_aclk400_isp",
-			GATE_BUS_TOP, 16, 0, 0),
+			GATE_BUS_TOP, 16, CLK_IS_CRITICAL, 0),
 	GATE(0, "aclk400_mscl", "mout_user_aclk400_mscl",
 			GATE_BUS_TOP, 17, CLK_IS_CRITICAL, 0),
 	GATE(0, "aclk200_disp1", "mout_user_aclk200_disp1",
@@ -1161,9 +1161,11 @@ static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = {
 			GATE_IP_GSCL1, 3, 0, 0),
 	GATE(CLK_SMMU_FIMCL1, "smmu_fimcl1", "dout_gscl_blk_333",
 			GATE_IP_GSCL1, 4, 0, 0),
-	GATE(CLK_GSCL_WA, "gscl_wa", "sclk_gscl_wa", GATE_IP_GSCL1, 12, 0, 0),
-	GATE(CLK_GSCL_WB, "gscl_wb", "sclk_gscl_wb", GATE_IP_GSCL1, 13, 0, 0),
-	GATE(CLK_SMMU_FIMCL3, "smmu_fimcl3,", "dout_gscl_blk_333",
+	GATE(CLK_GSCL_WA, "gscl_wa", "sclk_gscl_wa", GATE_IP_GSCL1, 12,
+			CLK_IS_CRITICAL, 0),
+	GATE(CLK_GSCL_WB, "gscl_wb", "sclk_gscl_wb", GATE_IP_GSCL1, 13,
+			CLK_IS_CRITICAL, 0),
+	GATE(CLK_SMMU_FIMCL3, "smmu_fimcl3", "dout_gscl_blk_333",
 			GATE_IP_GSCL1, 16, 0, 0),
 	GATE(CLK_FIMC_LITE3, "fimc_lite3", "aclk333_432_gscl",
 			GATE_IP_GSCL1, 17, 0, 0),
diff --git a/drivers/clk/samsung/clk-exynos5433.c b/drivers/clk/samsung/clk-exynos5433.c
index 4b1aa9382ad2..6f29ecd0442e 100644
--- a/drivers/clk/samsung/clk-exynos5433.c
+++ b/drivers/clk/samsung/clk-exynos5433.c
@@ -1706,7 +1706,8 @@ static const struct samsung_gate_clock peric_gate_clks[] __initconst = {
 	GATE(CLK_SCLK_PCM1, "sclk_pcm1", "sclk_pcm1_peric",
 			ENABLE_SCLK_PERIC, 7, CLK_SET_RATE_PARENT, 0),
 	GATE(CLK_SCLK_I2S1, "sclk_i2s1", "sclk_i2s1_peric",
-			ENABLE_SCLK_PERIC, 6, CLK_SET_RATE_PARENT, 0),
+			ENABLE_SCLK_PERIC, 6,
+			CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, 0),
 	GATE(CLK_SCLK_SPI2, "sclk_spi2", "sclk_spi2_peric", ENABLE_SCLK_PERIC,
 			5, CLK_SET_RATE_PARENT, 0),
 	GATE(CLK_SCLK_SPI1, "sclk_spi1", "sclk_spi1_peric", ENABLE_SCLK_PERIC,
diff --git a/drivers/clk/samsung/clk-s3c2443.c b/drivers/clk/samsung/clk-s3c2443.c
index 5f30fe72cd51..c7aba1e1af70 100644
--- a/drivers/clk/samsung/clk-s3c2443.c
+++ b/drivers/clk/samsung/clk-s3c2443.c
@@ -387,7 +387,7 @@ void __init s3c2443_common_clk_init(struct device_node *np, unsigned long xti_f,
 				ARRAY_SIZE(s3c2450_gates));
 		samsung_clk_register_alias(ctx, s3c2450_aliases,
 				ARRAY_SIZE(s3c2450_aliases));
-		/* fall through - as s3c2450 extends the s3c2416 clocks */
+		fallthrough;	/* as s3c2450 extends the s3c2416 clocks */
 	case S3C2416:
 		samsung_clk_register_div(ctx, s3c2416_dividers,
 				ARRAY_SIZE(s3c2416_dividers));
diff --git a/drivers/clk/socfpga/Makefile b/drivers/clk/socfpga/Makefile
index ce5aa7802eb8..bf736f8d201a 100644
--- a/drivers/clk/socfpga/Makefile
+++ b/drivers/clk/socfpga/Makefile
@@ -3,3 +3,5 @@ obj-$(CONFIG_ARCH_SOCFPGA) += clk.o clk-gate.o clk-pll.o clk-periph.o
 obj-$(CONFIG_ARCH_SOCFPGA) += clk-pll-a10.o clk-periph-a10.o clk-gate-a10.o
 obj-$(CONFIG_ARCH_STRATIX10) += clk-s10.o
 obj-$(CONFIG_ARCH_STRATIX10) += clk-pll-s10.o clk-periph-s10.o clk-gate-s10.o
+obj-$(CONFIG_ARCH_AGILEX) += clk-agilex.o
+obj-$(CONFIG_ARCH_AGILEX) += clk-pll-s10.o clk-periph-s10.o clk-gate-s10.o
diff --git a/drivers/clk/socfpga/clk-agilex.c b/drivers/clk/socfpga/clk-agilex.c
new file mode 100644
index 000000000000..699527f7e764
--- /dev/null
+++ b/drivers/clk/socfpga/clk-agilex.c
@@ -0,0 +1,454 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019, Intel Corporation
+ */
+#include <linux/slab.h>
+#include <linux/clk-provider.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+
+#include <dt-bindings/clock/agilex-clock.h>
+
+#include "stratix10-clk.h"
+
+static const struct clk_parent_data pll_mux[] = {
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data cntr_mux[] = {
+	{ .fw_name = "main_pll",
+	  .name = "main_pll", },
+	{ .fw_name = "periph_pll",
+	  .name = "periph_pll", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data boot_mux[] = {
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+};
+
+static const struct clk_parent_data mpu_free_mux[] = {
+	{ .fw_name = "main_pll_c0",
+	  .name = "main_pll_c0", },
+	{ .fw_name = "peri_pll_c0",
+	  .name = "peri_pll_c0", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data noc_free_mux[] = {
+	{ .fw_name = "main_pll_c1",
+	  .name = "main_pll_c1", },
+	{ .fw_name = "peri_pll_c1",
+	  .name = "peri_pll_c1", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data emaca_free_mux[] = {
+	{ .fw_name = "main_pll_c2",
+	  .name = "main_pll_c2", },
+	{ .fw_name = "peri_pll_c2",
+	  .name = "peri_pll_c2", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data emacb_free_mux[] = {
+	{ .fw_name = "main_pll_c3",
+	  .name = "main_pll_c3", },
+	{ .fw_name = "peri_pll_c3",
+	  .name = "peri_pll_c3", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data emac_ptp_free_mux[] = {
+	{ .fw_name = "main_pll_c3",
+	  .name = "main_pll_c3", },
+	{ .fw_name = "peri_pll_c3",
+	  .name = "peri_pll_c3", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data gpio_db_free_mux[] = {
+	{ .fw_name = "main_pll_c3",
+	  .name = "main_pll_c3", },
+	{ .fw_name = "peri_pll_c3",
+	  .name = "peri_pll_c3", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data psi_ref_free_mux[] = {
+	{ .fw_name = "main_pll_c3",
+	  .name = "main_pll_c3", },
+	{ .fw_name = "peri_pll_c3",
+	  .name = "peri_pll_c3", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data sdmmc_free_mux[] = {
+	{ .fw_name = "main_pll_c3",
+	  .name = "main_pll_c3", },
+	{ .fw_name = "peri_pll_c3",
+	  .name = "peri_pll_c3", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data s2f_usr0_free_mux[] = {
+	{ .fw_name = "main_pll_c2",
+	  .name = "main_pll_c2", },
+	{ .fw_name = "peri_pll_c2",
+	  .name = "peri_pll_c2", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data s2f_usr1_free_mux[] = {
+	{ .fw_name = "main_pll_c2",
+	  .name = "main_pll_c2", },
+	{ .fw_name = "peri_pll_c2",
+	  .name = "peri_pll_c2", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data mpu_mux[] = {
+	{ .fw_name = "mpu_free_clk",
+	  .name = "mpu_free_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data s2f_usr0_mux[] = {
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data emac_mux[] = {
+	{ .fw_name = "emaca_free_clk",
+	  .name = "emaca_free_clk", },
+	{ .fw_name = "emacb_free_clk",
+	  .name = "emacb_free_clk", },
+};
+
+static const struct clk_parent_data noc_mux[] = {
+	{ .fw_name = "noc_free_clk",
+	  .name = "noc_free_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+/* clocks in AO (always on) controller */
+static const struct stratix10_pll_clock agilex_pll_clks[] = {
+	{ AGILEX_BOOT_CLK, "boot_clk", boot_mux, ARRAY_SIZE(boot_mux), 0,
+	  0x0},
+	{ AGILEX_MAIN_PLL_CLK, "main_pll", pll_mux, ARRAY_SIZE(pll_mux),
+	  0, 0x48},
+	{ AGILEX_PERIPH_PLL_CLK, "periph_pll", pll_mux, ARRAY_SIZE(pll_mux),
+	  0, 0x9c},
+};
+
+static const struct stratix10_perip_c_clock agilex_main_perip_c_clks[] = {
+	{ AGILEX_MAIN_PLL_C0_CLK, "main_pll_c0", "main_pll", NULL, 1, 0, 0x58},
+	{ AGILEX_MAIN_PLL_C1_CLK, "main_pll_c1", "main_pll", NULL, 1, 0, 0x5C},
+	{ AGILEX_MAIN_PLL_C2_CLK, "main_pll_c2", "main_pll", NULL, 1, 0, 0x64},
+	{ AGILEX_MAIN_PLL_C3_CLK, "main_pll_c3", "main_pll", NULL, 1, 0, 0x68},
+	{ AGILEX_PERIPH_PLL_C0_CLK, "peri_pll_c0", "periph_pll", NULL, 1, 0, 0xAC},
+	{ AGILEX_PERIPH_PLL_C1_CLK, "peri_pll_c1", "periph_pll", NULL, 1, 0, 0xB0},
+	{ AGILEX_PERIPH_PLL_C2_CLK, "peri_pll_c2", "periph_pll", NULL, 1, 0, 0xB8},
+	{ AGILEX_PERIPH_PLL_C3_CLK, "peri_pll_c3", "periph_pll", NULL, 1, 0, 0xBC},
+};
+
+static const struct stratix10_perip_cnt_clock agilex_main_perip_cnt_clks[] = {
+	{ AGILEX_MPU_FREE_CLK, "mpu_free_clk", NULL, mpu_free_mux, ARRAY_SIZE(mpu_free_mux),
+	   0, 0x3C, 0, 0, 0},
+	{ AGILEX_NOC_FREE_CLK, "noc_free_clk", NULL, noc_free_mux, ARRAY_SIZE(noc_free_mux),
+	  0, 0x40, 0, 0, 1},
+	{ AGILEX_L4_SYS_FREE_CLK, "l4_sys_free_clk", "noc_free_clk", NULL, 1, 0,
+	  0, 4, 0, 0},
+	{ AGILEX_NOC_CLK, "noc_clk", NULL, noc_mux, ARRAY_SIZE(noc_mux),
+	  0, 0, 0, 0x30, 1},
+	{ AGILEX_EMAC_A_FREE_CLK, "emaca_free_clk", NULL, emaca_free_mux, ARRAY_SIZE(emaca_free_mux),
+	  0, 0xD4, 0, 0x88, 0},
+	{ AGILEX_EMAC_B_FREE_CLK, "emacb_free_clk", NULL, emacb_free_mux, ARRAY_SIZE(emacb_free_mux),
+	  0, 0xD8, 0, 0x88, 1},
+	{ AGILEX_EMAC_PTP_FREE_CLK, "emac_ptp_free_clk", NULL, emac_ptp_free_mux,
+	  ARRAY_SIZE(emac_ptp_free_mux), 0, 0xDC, 0, 0x88, 2},
+	{ AGILEX_GPIO_DB_FREE_CLK, "gpio_db_free_clk", NULL, gpio_db_free_mux,
+	  ARRAY_SIZE(gpio_db_free_mux), 0, 0xE0, 0, 0x88, 3},
+	{ AGILEX_SDMMC_FREE_CLK, "sdmmc_free_clk", NULL, sdmmc_free_mux,
+	  ARRAY_SIZE(sdmmc_free_mux), 0, 0xE4, 0, 0x88, 4},
+	{ AGILEX_S2F_USER0_FREE_CLK, "s2f_user0_free_clk", NULL, s2f_usr0_free_mux,
+	  ARRAY_SIZE(s2f_usr0_free_mux), 0, 0xE8, 0, 0, 0},
+	{ AGILEX_S2F_USER1_FREE_CLK, "s2f_user1_free_clk", NULL, s2f_usr1_free_mux,
+	  ARRAY_SIZE(s2f_usr1_free_mux), 0, 0xEC, 0, 0x88, 5},
+	{ AGILEX_PSI_REF_FREE_CLK, "psi_ref_free_clk", NULL, psi_ref_free_mux,
+	  ARRAY_SIZE(psi_ref_free_mux), 0, 0xF0, 0, 0x88, 6},
+};
+
+static const struct stratix10_gate_clock agilex_gate_clks[] = {
+	{ AGILEX_MPU_CLK, "mpu_clk", NULL, mpu_mux, ARRAY_SIZE(mpu_mux), 0, 0x24,
+	  0, 0, 0, 0, 0x30, 0, 0},
+	{ AGILEX_MPU_PERIPH_CLK, "mpu_periph_clk", "mpu_clk", NULL, 1, 0, 0x24,
+	  0, 0, 0, 0, 0, 0, 4},
+	{ AGILEX_MPU_L2RAM_CLK, "mpu_l2ram_clk", "mpu_clk", NULL, 1, 0, 0x24,
+	  0, 0, 0, 0, 0, 0, 2},
+	{ AGILEX_L4_MAIN_CLK, "l4_main_clk", "noc_clk", NULL, 1, 0, 0x24,
+	  1, 0x44, 0, 2, 0, 0, 0},
+	{ AGILEX_L4_MP_CLK, "l4_mp_clk", "noc_clk", NULL, 1, 0, 0x24,
+	  2, 0x44, 8, 2, 0, 0, 0},
+	/*
+	 * The l4_sp_clk feeds a 100 MHz clock to various peripherals, one of them
+	 * being the SP timers, thus cannot get gated.
+	 */
+	{ AGILEX_L4_SP_CLK, "l4_sp_clk", "noc_clk", NULL, 1, CLK_IS_CRITICAL, 0x24,
+	  3, 0x44, 16, 2, 0, 0, 0},
+	{ AGILEX_CS_AT_CLK, "cs_at_clk", "noc_clk", NULL, 1, 0, 0x24,
+	  4, 0x44, 24, 2, 0, 0, 0},
+	{ AGILEX_CS_TRACE_CLK, "cs_trace_clk", "noc_clk", NULL, 1, 0, 0x24,
+	  4, 0x44, 26, 2, 0, 0, 0},
+	{ AGILEX_CS_PDBG_CLK, "cs_pdbg_clk", "cs_at_clk", NULL, 1, 0, 0x24,
+	  4, 0x44, 28, 1, 0, 0, 0},
+	{ AGILEX_CS_TIMER_CLK, "cs_timer_clk", "noc_clk", NULL, 1, 0, 0x24,
+	  5, 0, 0, 0, 0, 0, 0},
+	{ AGILEX_S2F_USER0_CLK, "s2f_user0_clk", NULL, s2f_usr0_mux, ARRAY_SIZE(s2f_usr0_mux), 0, 0x24,
+	  6, 0, 0, 0, 0, 0, 0},
+	{ AGILEX_EMAC0_CLK, "emac0_clk", NULL, emac_mux, ARRAY_SIZE(emac_mux), 0, 0x7C,
+	  0, 0, 0, 0, 0x94, 26, 0},
+	{ AGILEX_EMAC1_CLK, "emac1_clk", NULL, emac_mux, ARRAY_SIZE(emac_mux), 0, 0x7C,
+	  1, 0, 0, 0, 0x94, 27, 0},
+	{ AGILEX_EMAC2_CLK, "emac2_clk", NULL, emac_mux, ARRAY_SIZE(emac_mux), 0, 0x7C,
+	  2, 0, 0, 0, 0x94, 28, 0},
+	{ AGILEX_EMAC_PTP_CLK, "emac_ptp_clk", "emac_ptp_free_clk", NULL, 1, 0, 0x7C,
+	  3, 0, 0, 0, 0, 0, 0},
+	{ AGILEX_GPIO_DB_CLK, "gpio_db_clk", "gpio_db_free_clk", NULL, 1, 0, 0x7C,
+	  4, 0x98, 0, 16, 0, 0, 0},
+	{ AGILEX_SDMMC_CLK, "sdmmc_clk", "sdmmc_free_clk", NULL, 1, 0, 0x7C,
+	  5, 0, 0, 0, 0, 0, 4},
+	{ AGILEX_S2F_USER1_CLK, "s2f_user1_clk", "s2f_user1_free_clk", NULL, 1, 0, 0x7C,
+	  6, 0, 0, 0, 0, 0, 0},
+	{ AGILEX_PSI_REF_CLK, "psi_ref_clk", "psi_ref_free_clk", NULL, 1, 0, 0x7C,
+	  7, 0, 0, 0, 0, 0, 0},
+	{ AGILEX_USB_CLK, "usb_clk", "l4_mp_clk", NULL, 1, 0, 0x7C,
+	  8, 0, 0, 0, 0, 0, 0},
+	{ AGILEX_SPI_M_CLK, "spi_m_clk", "l4_mp_clk", NULL, 1, 0, 0x7C,
+	  9, 0, 0, 0, 0, 0, 0},
+	{ AGILEX_NAND_CLK, "nand_clk", "l4_main_clk", NULL, 1, 0, 0x7C,
+	  10, 0, 0, 0, 0, 0, 0},
+};
+
+static int agilex_clk_register_c_perip(const struct stratix10_perip_c_clock *clks,
+				       int nums, struct stratix10_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		clk = s10_register_periph(&clks[i], base);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n",
+			       __func__, clks[i].name);
+			continue;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+	}
+	return 0;
+}
+
+static int agilex_clk_register_cnt_perip(const struct stratix10_perip_cnt_clock *clks,
+					 int nums, struct stratix10_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		clk = s10_register_cnt_periph(&clks[i], base);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n",
+			       __func__, clks[i].name);
+			continue;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+	}
+
+	return 0;
+}
+
+static int agilex_clk_register_gate(const struct stratix10_gate_clock *clks,					    int nums, struct stratix10_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		clk = s10_register_gate(&clks[i], base);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n",
+			       __func__, clks[i].name);
+			continue;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+	}
+
+	return 0;
+}
+
+static int agilex_clk_register_pll(const struct stratix10_pll_clock *clks,
+				 int nums, struct stratix10_clock_data *data)
+{
+	struct clk *clk;
+	void __iomem *base = data->base;
+	int i;
+
+	for (i = 0; i < nums; i++) {
+		clk = agilex_register_pll(&clks[i], base);
+		if (IS_ERR(clk)) {
+			pr_err("%s: failed to register clock %s\n",
+			       __func__, clks[i].name);
+			continue;
+		}
+		data->clk_data.clks[clks[i].id] = clk;
+	}
+
+	return 0;
+}
+
+static struct stratix10_clock_data *__socfpga_agilex_clk_init(struct platform_device *pdev,
+						    int nr_clks)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
+	struct stratix10_clock_data *clk_data;
+	struct clk **clk_table;
+	struct resource *res;
+	void __iomem *base;
+	int ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return ERR_CAST(base);
+
+	clk_data = devm_kzalloc(dev, sizeof(*clk_data), GFP_KERNEL);
+	if (!clk_data)
+		return ERR_PTR(-ENOMEM);
+
+	clk_data->base = base;
+	clk_table = devm_kcalloc(dev, nr_clks, sizeof(*clk_table), GFP_KERNEL);
+	if (!clk_table)
+		return ERR_PTR(-ENOMEM);
+
+	clk_data->clk_data.clks = clk_table;
+	clk_data->clk_data.clk_num = nr_clks;
+	ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data->clk_data);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return clk_data;
+}
+
+static int agilex_clkmgr_probe(struct platform_device *pdev)
+{
+	struct stratix10_clock_data *clk_data;
+
+	clk_data = __socfpga_agilex_clk_init(pdev, AGILEX_NUM_CLKS);
+	if (IS_ERR(clk_data))
+		return PTR_ERR(clk_data);
+
+	agilex_clk_register_pll(agilex_pll_clks, ARRAY_SIZE(agilex_pll_clks), clk_data);
+
+	agilex_clk_register_c_perip(agilex_main_perip_c_clks,
+				 ARRAY_SIZE(agilex_main_perip_c_clks), clk_data);
+
+	agilex_clk_register_cnt_perip(agilex_main_perip_cnt_clks,
+				   ARRAY_SIZE(agilex_main_perip_cnt_clks),
+				   clk_data);
+
+	agilex_clk_register_gate(agilex_gate_clks, ARRAY_SIZE(agilex_gate_clks),
+			      clk_data);
+	return 0;
+}
+
+static const struct of_device_id agilex_clkmgr_match_table[] = {
+	{ .compatible = "intel,agilex-clkmgr",
+	  .data = agilex_clkmgr_probe },
+	{ }
+};
+
+static struct platform_driver agilex_clkmgr_driver = {
+	.probe		= agilex_clkmgr_probe,
+	.driver		= {
+		.name	= "agilex-clkmgr",
+		.suppress_bind_attrs = true,
+		.of_match_table = agilex_clkmgr_match_table,
+	},
+};
+
+static int __init agilex_clk_init(void)
+{
+	return platform_driver_register(&agilex_clkmgr_driver);
+}
+core_initcall(agilex_clk_init);
diff --git a/drivers/clk/socfpga/clk-gate-s10.c b/drivers/clk/socfpga/clk-gate-s10.c
index 8be4722f6064..083b2ec21fdd 100644
--- a/drivers/clk/socfpga/clk-gate-s10.c
+++ b/drivers/clk/socfpga/clk-gate-s10.c
@@ -70,7 +70,6 @@ struct clk *s10_register_gate(const struct stratix10_gate_clock *clks, void __io
 	struct clk *clk;
 	struct socfpga_gate_clk *socfpga_clk;
 	struct clk_init_data init;
-	const char * const *parent_names = clks->parent_names;
 	const char *parent_name = clks->parent_name;
 
 	socfpga_clk = kzalloc(sizeof(*socfpga_clk), GFP_KERNEL);
@@ -108,7 +107,9 @@ struct clk *s10_register_gate(const struct stratix10_gate_clock *clks, void __io
 	init.flags = clks->flags;
 
 	init.num_parents = clks->num_parents;
-	init.parent_names = parent_names ? parent_names : &parent_name;
+	init.parent_names = parent_name ? &parent_name : NULL;
+	if (init.parent_names == NULL)
+		init.parent_data = clks->parent_data;
 	socfpga_clk->hw.hw.init = &init;
 
 	clk = clk_register(NULL, &socfpga_clk->hw.hw);
diff --git a/drivers/clk/socfpga/clk-periph-s10.c b/drivers/clk/socfpga/clk-periph-s10.c
index dd6d4056e9de..397b77b89b16 100644
--- a/drivers/clk/socfpga/clk-periph-s10.c
+++ b/drivers/clk/socfpga/clk-periph-s10.c
@@ -81,7 +81,6 @@ struct clk *s10_register_periph(const struct stratix10_perip_c_clock *clks,
 	struct clk_init_data init;
 	const char *name = clks->name;
 	const char *parent_name = clks->parent_name;
-	const char * const *parent_names = clks->parent_names;
 
 	periph_clk = kzalloc(sizeof(*periph_clk), GFP_KERNEL);
 	if (WARN_ON(!periph_clk))
@@ -94,7 +93,9 @@ struct clk *s10_register_periph(const struct stratix10_perip_c_clock *clks,
 	init.flags = clks->flags;
 
 	init.num_parents = clks->num_parents;
-	init.parent_names = parent_names ? parent_names : &parent_name;
+	init.parent_names = parent_name ? &parent_name : NULL;
+	if (init.parent_names == NULL)
+		init.parent_data = clks->parent_data;
 
 	periph_clk->hw.hw.init = &init;
 
@@ -114,7 +115,6 @@ struct clk *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *clks
 	struct clk_init_data init;
 	const char *name = clks->name;
 	const char *parent_name = clks->parent_name;
-	const char * const *parent_names = clks->parent_names;
 
 	periph_clk = kzalloc(sizeof(*periph_clk), GFP_KERNEL);
 	if (WARN_ON(!periph_clk))
@@ -137,7 +137,9 @@ struct clk *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *clks
 	init.flags = clks->flags;
 
 	init.num_parents = clks->num_parents;
-	init.parent_names = parent_names ? parent_names : &parent_name;
+	init.parent_names = parent_name ? &parent_name : NULL;
+	if (init.parent_names == NULL)
+		init.parent_data = clks->parent_data;
 
 	periph_clk->hw.hw.init = &init;
 
diff --git a/drivers/clk/socfpga/clk-pll-a10.c b/drivers/clk/socfpga/clk-pll-a10.c
index 3816fc04b274..db54f7d806a0 100644
--- a/drivers/clk/socfpga/clk-pll-a10.c
+++ b/drivers/clk/socfpga/clk-pll-a10.c
@@ -58,7 +58,7 @@ static u8 clk_pll_get_parent(struct clk_hw *hwclk)
 		CLK_MGR_PLL_CLK_SRC_MASK;
 }
 
-static struct clk_ops clk_pll_ops = {
+static const struct clk_ops clk_pll_ops = {
 	.recalc_rate = clk_pll_recalc_rate,
 	.get_parent = clk_pll_get_parent,
 };
@@ -102,8 +102,6 @@ static struct clk * __init __socfpga_pll_init(struct device_node *node,
 	pll_clk->hw.hw.init = &init;
 
 	pll_clk->hw.bit_idx = SOCFPGA_PLL_EXT_ENA;
-	clk_pll_ops.enable = clk_gate_ops.enable;
-	clk_pll_ops.disable = clk_gate_ops.disable;
 
 	clk = clk_register(NULL, &pll_clk->hw.hw);
 	if (WARN_ON(IS_ERR(clk))) {
diff --git a/drivers/clk/socfpga/clk-pll-s10.c b/drivers/clk/socfpga/clk-pll-s10.c
index a301bb22f36c..4e268953b7da 100644
--- a/drivers/clk/socfpga/clk-pll-s10.c
+++ b/drivers/clk/socfpga/clk-pll-s10.c
@@ -18,8 +18,12 @@
 #define SOCFPGA_PLL_RESET_MASK		0x2
 #define SOCFPGA_PLL_REFDIV_MASK		0x00003F00
 #define SOCFPGA_PLL_REFDIV_SHIFT	8
+#define SOCFPGA_PLL_AREFDIV_MASK	0x00000F00
+#define SOCFPGA_PLL_DREFDIV_MASK	0x00003000
+#define SOCFPGA_PLL_DREFDIV_SHIFT	12
 #define SOCFPGA_PLL_MDIV_MASK		0xFF000000
 #define SOCFPGA_PLL_MDIV_SHIFT		24
+#define SOCFPGA_AGILEX_PLL_MDIV_MASK	0x000003FF
 #define SWCTRLBTCLKSEL_MASK		0x200
 #define SWCTRLBTCLKSEL_SHIFT		9
 
@@ -27,6 +31,27 @@
 
 #define to_socfpga_clk(p) container_of(p, struct socfpga_pll, hw.hw)
 
+static unsigned long agilex_clk_pll_recalc_rate(struct clk_hw *hwclk,
+						unsigned long parent_rate)
+{
+	struct socfpga_pll *socfpgaclk = to_socfpga_clk(hwclk);
+	unsigned long arefdiv, reg, mdiv;
+	unsigned long long vco_freq;
+
+	/* read VCO1 reg for numerator and denominator */
+	reg = readl(socfpgaclk->hw.reg);
+	arefdiv = (reg & SOCFPGA_PLL_AREFDIV_MASK) >> SOCFPGA_PLL_REFDIV_SHIFT;
+
+	vco_freq = (unsigned long long)parent_rate / arefdiv;
+
+	/* Read mdiv and fdiv from the fdbck register */
+	reg = readl(socfpgaclk->hw.reg + 0x24);
+	mdiv = reg & SOCFPGA_AGILEX_PLL_MDIV_MASK;
+
+	vco_freq = (unsigned long long)vco_freq * mdiv;
+	return (unsigned long)vco_freq;
+}
+
 static unsigned long clk_pll_recalc_rate(struct clk_hw *hwclk,
 					 unsigned long parent_rate)
 {
@@ -98,13 +123,19 @@ static int clk_pll_prepare(struct clk_hw *hwclk)
 	return 0;
 }
 
-static struct clk_ops clk_pll_ops = {
+static const struct clk_ops agilex_clk_pll_ops = {
+	.recalc_rate = agilex_clk_pll_recalc_rate,
+	.get_parent = clk_pll_get_parent,
+	.prepare = clk_pll_prepare,
+};
+
+static const struct clk_ops clk_pll_ops = {
 	.recalc_rate = clk_pll_recalc_rate,
 	.get_parent = clk_pll_get_parent,
 	.prepare = clk_pll_prepare,
 };
 
-static struct clk_ops clk_boot_ops = {
+static const struct clk_ops clk_boot_ops = {
 	.recalc_rate = clk_boot_clk_recalc_rate,
 	.get_parent = clk_boot_get_parent,
 	.prepare = clk_pll_prepare,
@@ -117,7 +148,6 @@ struct clk *s10_register_pll(const struct stratix10_pll_clock *clks,
 	struct socfpga_pll *pll_clk;
 	struct clk_init_data init;
 	const char *name = clks->name;
-	const char * const *parent_names = clks->parent_names;
 
 	pll_clk = kzalloc(sizeof(*pll_clk), GFP_KERNEL);
 	if (WARN_ON(!pll_clk))
@@ -134,12 +164,48 @@ struct clk *s10_register_pll(const struct stratix10_pll_clock *clks,
 	init.flags = clks->flags;
 
 	init.num_parents = clks->num_parents;
-	init.parent_names = parent_names;
+	init.parent_names = NULL;
+	init.parent_data = clks->parent_data;
+	pll_clk->hw.hw.init = &init;
+
+	pll_clk->hw.bit_idx = SOCFPGA_PLL_POWER;
+
+	clk = clk_register(NULL, &pll_clk->hw.hw);
+	if (WARN_ON(IS_ERR(clk))) {
+		kfree(pll_clk);
+		return NULL;
+	}
+	return clk;
+}
+
+struct clk *agilex_register_pll(const struct stratix10_pll_clock *clks,
+				void __iomem *reg)
+{
+	struct clk *clk;
+	struct socfpga_pll *pll_clk;
+	struct clk_init_data init;
+	const char *name = clks->name;
+
+	pll_clk = kzalloc(sizeof(*pll_clk), GFP_KERNEL);
+	if (WARN_ON(!pll_clk))
+		return NULL;
+
+	pll_clk->hw.reg = reg + clks->offset;
+
+	if (streq(name, SOCFPGA_BOOT_CLK))
+		init.ops = &clk_boot_ops;
+	else
+		init.ops = &agilex_clk_pll_ops;
+
+	init.name = name;
+	init.flags = clks->flags;
+
+	init.num_parents = clks->num_parents;
+	init.parent_names = NULL;
+	init.parent_data = clks->parent_data;
 	pll_clk->hw.hw.init = &init;
 
 	pll_clk->hw.bit_idx = SOCFPGA_PLL_POWER;
-	clk_pll_ops.enable = clk_gate_ops.enable;
-	clk_pll_ops.disable = clk_gate_ops.disable;
 
 	clk = clk_register(NULL, &pll_clk->hw.hw);
 	if (WARN_ON(IS_ERR(clk))) {
diff --git a/drivers/clk/socfpga/clk-pll.c b/drivers/clk/socfpga/clk-pll.c
index dc65cc0fd3bd..e5fb786843f3 100644
--- a/drivers/clk/socfpga/clk-pll.c
+++ b/drivers/clk/socfpga/clk-pll.c
@@ -65,7 +65,7 @@ static u8 clk_pll_get_parent(struct clk_hw *hwclk)
 			CLK_MGR_PLL_CLK_SRC_MASK;
 }
 
-static struct clk_ops clk_pll_ops = {
+static const struct clk_ops clk_pll_ops = {
 	.recalc_rate = clk_pll_recalc_rate,
 	.get_parent = clk_pll_get_parent,
 };
@@ -105,8 +105,6 @@ static __init struct clk *__socfpga_pll_init(struct device_node *node,
 	pll_clk->hw.hw.init = &init;
 
 	pll_clk->hw.bit_idx = SOCFPGA_PLL_EXT_ENA;
-	clk_pll_ops.enable = clk_gate_ops.enable;
-	clk_pll_ops.disable = clk_gate_ops.disable;
 
 	clk = clk_register(NULL, &pll_clk->hw.hw);
 	if (WARN_ON(IS_ERR(clk))) {
diff --git a/drivers/clk/socfpga/clk-s10.c b/drivers/clk/socfpga/clk-s10.c
index dea7c6c7d269..c1dfc9b34e4e 100644
--- a/drivers/clk/socfpga/clk-s10.c
+++ b/drivers/clk/socfpga/clk-s10.c
@@ -12,35 +12,137 @@
 
 #include "stratix10-clk.h"
 
-static const char * const pll_mux[] = { "osc1", "cb-intosc-hs-div2-clk",
-					"f2s-free-clk",};
-static const char * const cntr_mux[] = { "main_pll", "periph_pll",
-					 "osc1", "cb-intosc-hs-div2-clk",
-					 "f2s-free-clk"};
-static const char * const boot_mux[] = { "osc1", "cb-intosc-hs-div2-clk",};
-
-static const char * const noc_free_mux[] = {"main_noc_base_clk",
-					    "peri_noc_base_clk",
-					    "osc1", "cb-intosc-hs-div2-clk",
-					    "f2s-free-clk"};
-
-static const char * const emaca_free_mux[] = {"peri_emaca_clk", "boot_clk"};
-static const char * const emacb_free_mux[] = {"peri_emacb_clk", "boot_clk"};
-static const char * const emac_ptp_free_mux[] = {"peri_emac_ptp_clk", "boot_clk"};
-static const char * const gpio_db_free_mux[] = {"peri_gpio_db_clk", "boot_clk"};
-static const char * const sdmmc_free_mux[] = {"main_sdmmc_clk", "boot_clk"};
-static const char * const s2f_usr1_free_mux[] = {"peri_s2f_usr1_clk", "boot_clk"};
-static const char * const psi_ref_free_mux[] = {"peri_psi_ref_clk", "boot_clk"};
-static const char * const mpu_mux[] = { "mpu_free_clk", "boot_clk",};
-
-static const char * const s2f_usr0_mux[] = {"f2s-free-clk", "boot_clk"};
-static const char * const emac_mux[] = {"emaca_free_clk", "emacb_free_clk"};
-static const char * const noc_mux[] = {"noc_free_clk", "boot_clk"};
-
-static const char * const mpu_free_mux[] = {"main_mpu_base_clk",
-					    "peri_mpu_base_clk",
-					    "osc1", "cb-intosc-hs-div2-clk",
-					    "f2s-free-clk"};
+static const struct clk_parent_data pll_mux[] = {
+	{ .fw_name = "osc1",
+	  .name = "osc1" },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk" },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk" },
+};
+
+static const struct clk_parent_data cntr_mux[] = {
+	{ .fw_name =  "main_pll",
+	  .name = "main_pll", },
+	{ .fw_name = "periph_pll",
+	  .name = "periph_pll", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data boot_mux[] = {
+	{ .fw_name = "osc1",
+	  .name = "osc1" },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk" },
+};
+
+static const struct clk_parent_data noc_free_mux[] = {
+	{ .fw_name = "main_noc_base_clk",
+	  .name = "main_noc_base_clk", },
+	{ .fw_name = "peri_noc_base_clk",
+	  .name = "peri_noc_base_clk", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
+
+static const struct clk_parent_data emaca_free_mux[] = {
+	{ .fw_name = "peri_emaca_clk",
+	  .name = "peri_emaca_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data emacb_free_mux[] = {
+	{ .fw_name = "peri_emacb_clk",
+	  .name = "peri_emacb_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data emac_ptp_free_mux[] = {
+	{ .fw_name = "peri_emac_ptp_clk",
+	  .name = "peri_emac_ptp_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data gpio_db_free_mux[] = {
+	{ .fw_name = "peri_gpio_db_clk",
+	  .name = "peri_gpio_db_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data sdmmc_free_mux[] = {
+	{ .fw_name = "main_sdmmc_clk",
+	  .name = "main_sdmmc_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data s2f_usr1_free_mux[] = {
+	{ .fw_name = "peri_s2f_usr1_clk",
+	  .name = "peri_s2f_usr1_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data psi_ref_free_mux[] = {
+	{ .fw_name = "peri_psi_ref_clk",
+	  .name = "peri_psi_ref_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data mpu_mux[] = {
+	{ .fw_name = "mpu_free_clk",
+	  .name = "mpu_free_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data s2f_usr0_mux[] = {
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data emac_mux[] = {
+	{ .fw_name = "emaca_free_clk",
+	  .name = "emaca_free_clk", },
+	{ .fw_name = "emacb_free_clk",
+	  .name = "emacb_free_clk", },
+};
+
+static const struct clk_parent_data noc_mux[] = {
+	{ .fw_name = "noc_free_clk",
+	  .name = "noc_free_clk", },
+	{ .fw_name = "boot_clk",
+	  .name = "boot_clk", },
+};
+
+static const struct clk_parent_data mpu_free_mux[] = {
+	{ .fw_name = "main_mpu_base_clk",
+	  .name = "main_mpu_base_clk", },
+	{ .fw_name = "peri_mpu_base_clk",
+	  .name = "peri_mpu_base_clk", },
+	{ .fw_name = "osc1",
+	  .name = "osc1", },
+	{ .fw_name = "cb-intosc-hs-div2-clk",
+	  .name = "cb-intosc-hs-div2-clk", },
+	{ .fw_name = "f2s-free-clk",
+	  .name = "f2s-free-clk", },
+};
 
 /* clocks in AO (always on) controller */
 static const struct stratix10_pll_clock s10_pll_clks[] = {
diff --git a/drivers/clk/socfpga/stratix10-clk.h b/drivers/clk/socfpga/stratix10-clk.h
index fcabef42249c..f9d5d724c694 100644
--- a/drivers/clk/socfpga/stratix10-clk.h
+++ b/drivers/clk/socfpga/stratix10-clk.h
@@ -14,7 +14,7 @@ struct stratix10_clock_data {
 struct stratix10_pll_clock {
 	unsigned int		id;
 	const char		*name;
-	const char		*const *parent_names;
+	const struct clk_parent_data	*parent_data;
 	u8			num_parents;
 	unsigned long		flags;
 	unsigned long		offset;
@@ -24,7 +24,7 @@ struct stratix10_perip_c_clock {
 	unsigned int		id;
 	const char		*name;
 	const char		*parent_name;
-	const char		*const *parent_names;
+	const struct clk_parent_data	*parent_data;
 	u8			num_parents;
 	unsigned long		flags;
 	unsigned long		offset;
@@ -34,7 +34,7 @@ struct stratix10_perip_cnt_clock {
 	unsigned int		id;
 	const char		*name;
 	const char		*parent_name;
-	const char		*const *parent_names;
+	const struct clk_parent_data	*parent_data;
 	u8			num_parents;
 	unsigned long		flags;
 	unsigned long		offset;
@@ -47,7 +47,7 @@ struct stratix10_gate_clock {
 	unsigned int		id;
 	const char		*name;
 	const char		*parent_name;
-	const char		*const *parent_names;
+	const struct clk_parent_data	*parent_data;
 	u8			num_parents;
 	unsigned long		flags;
 	unsigned long		gate_reg;
@@ -62,6 +62,8 @@ struct stratix10_gate_clock {
 
 struct clk *s10_register_pll(const struct stratix10_pll_clock *,
 			     void __iomem *);
+struct clk *agilex_register_pll(const struct stratix10_pll_clock *,
+				void __iomem *);
 struct clk *s10_register_periph(const struct stratix10_perip_c_clock *,
 				void __iomem *);
 struct clk *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *,
diff --git a/drivers/clk/sprd/gate.c b/drivers/clk/sprd/gate.c
index 574cfc116bbc..56e1714b541e 100644
--- a/drivers/clk/sprd/gate.c
+++ b/drivers/clk/sprd/gate.c
@@ -94,8 +94,15 @@ static int sprd_gate_is_enabled(struct clk_hw *hw)
 {
 	struct sprd_gate *sg = hw_to_sprd_gate(hw);
 	struct sprd_clk_common *common = &sg->common;
+	struct clk_hw *parent;
 	unsigned int reg;
 
+	if (sg->flags & SPRD_GATE_NON_AON) {
+		parent = clk_hw_get_parent(hw);
+		if (!parent || !clk_hw_is_enabled(parent))
+			return 0;
+	}
+
 	regmap_read(common->regmap, common->reg, &reg);
 
 	if (sg->flags & CLK_GATE_SET_TO_DISABLE)
diff --git a/drivers/clk/sprd/gate.h b/drivers/clk/sprd/gate.h
index b55817869367..e738dafa4fe9 100644
--- a/drivers/clk/sprd/gate.h
+++ b/drivers/clk/sprd/gate.h
@@ -19,6 +19,15 @@ struct sprd_gate {
 	struct sprd_clk_common	common;
 };
 
+/*
+ * sprd_gate->flags is used for:
+ * CLK_GATE_SET_TO_DISABLE	BIT(0)
+ * CLK_GATE_HIWORD_MASK		BIT(1)
+ * CLK_GATE_BIG_ENDIAN		BIT(2)
+ * so we define new flags from	BIT(3)
+ */
+#define SPRD_GATE_NON_AON BIT(3) /* not alway powered on, check before read */
+
 #define SPRD_SC_GATE_CLK_HW_INIT_FN(_struct, _name, _parent, _reg,	\
 				    _sc_offset, _enable_mask, _flags,	\
 				    _gate_flags, _udelay, _ops, _fn)	\
diff --git a/drivers/clk/sprd/pll.c b/drivers/clk/sprd/pll.c
index 15791484388f..13a322b2535a 100644
--- a/drivers/clk/sprd/pll.c
+++ b/drivers/clk/sprd/pll.c
@@ -106,7 +106,7 @@ static unsigned long _sprd_pll_recalc_rate(const struct sprd_pll *pll,
 
 	cfg = kcalloc(regs_num, sizeof(*cfg), GFP_KERNEL);
 	if (!cfg)
-		return -ENOMEM;
+		return parent_rate;
 
 	for (i = 0; i < regs_num; i++)
 		cfg[i] = sprd_pll_read(pll, i);
diff --git a/drivers/clk/sprd/sc9863a-clk.c b/drivers/clk/sprd/sc9863a-clk.c
index 2e2dfb2d48ff..ad2e0f9f8563 100644
--- a/drivers/clk/sprd/sc9863a-clk.c
+++ b/drivers/clk/sprd/sc9863a-clk.c
@@ -23,22 +23,22 @@
 #include "pll.h"
 
 /* mpll*_gate clocks control cpu cores, they were enabled by default */
-SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll0_gate, "mpll0-gate", "ext-26m", 0x94,
-			     0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240);
-SPRD_PLL_SC_GATE_CLK_FW_NAME(dpll0_gate, "dpll0-gate", "ext-26m", 0x98,
-			     0x1000, BIT(0), 0, 0, 240);
-SPRD_PLL_SC_GATE_CLK_FW_NAME(lpll_gate, "lpll-gate", "ext-26m", 0x9c,
-			     0x1000, BIT(0), 0, 0, 240);
-SPRD_PLL_SC_GATE_CLK_FW_NAME(gpll_gate, "gpll-gate", "ext-26m", 0xa8,
-			     0x1000, BIT(0), 0, 0, 240);
-SPRD_PLL_SC_GATE_CLK_FW_NAME(dpll1_gate, "dpll1-gate", "ext-26m", 0x1dc,
-			     0x1000, BIT(0), 0, 0, 240);
-SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll1_gate, "mpll1-gate", "ext-26m", 0x1e0,
-			     0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240);
-SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll2_gate, "mpll2-gate", "ext-26m", 0x1e4,
-			     0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240);
-SPRD_PLL_SC_GATE_CLK_FW_NAME(isppll_gate, "isppll-gate", "ext-26m", 0x1e8,
-			     0x1000, BIT(0), 0, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll0_gate, "mpll0-gate", "ext-26m", 0x94,
+				    0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(dpll0_gate, "dpll0-gate", "ext-26m", 0x98,
+				    0x1000, BIT(0), 0, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(lpll_gate, "lpll-gate", "ext-26m", 0x9c,
+				    0x1000, BIT(0), 0, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(gpll_gate, "gpll-gate", "ext-26m", 0xa8,
+				    0x1000, BIT(0), 0, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(dpll1_gate, "dpll1-gate", "ext-26m", 0x1dc,
+				    0x1000, BIT(0), 0, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll1_gate, "mpll1-gate", "ext-26m", 0x1e0,
+				    0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll2_gate, "mpll2-gate", "ext-26m", 0x1e4,
+				    0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240);
+static SPRD_PLL_SC_GATE_CLK_FW_NAME(isppll_gate, "isppll-gate", "ext-26m",
+				    0x1e8, 0x1000, BIT(0), 0, 0, 240);
 
 static struct sprd_clk_common *sc9863a_pmu_gate_clks[] = {
 	/* address base is 0x402b0000 */
@@ -1615,6 +1615,36 @@ static const struct sprd_clk_desc sc9863a_mm_gate_desc = {
 	.hw_clks	= &sc9863a_mm_gate_hws,
 };
 
+/* camera sensor clocks */
+static SPRD_GATE_CLK_HW(mipi_csi_clk, "mipi-csi-clk", &mahb_ckg_eb.common.hw,
+			0x20, BIT(16), 0, SPRD_GATE_NON_AON);
+static SPRD_GATE_CLK_HW(mipi_csi_s_clk, "mipi-csi-s-clk", &mahb_ckg_eb.common.hw,
+			0x24, BIT(16), 0, SPRD_GATE_NON_AON);
+static SPRD_GATE_CLK_HW(mipi_csi_m_clk, "mipi-csi-m-clk", &mahb_ckg_eb.common.hw,
+			0x28, BIT(16), 0, SPRD_GATE_NON_AON);
+
+static struct sprd_clk_common *sc9863a_mm_clk_clks[] = {
+	/* address base is 0x60900000 */
+	&mipi_csi_clk.common,
+	&mipi_csi_s_clk.common,
+	&mipi_csi_m_clk.common,
+};
+
+static struct clk_hw_onecell_data sc9863a_mm_clk_hws = {
+	.hws	= {
+		[CLK_MIPI_CSI]		= &mipi_csi_clk.common.hw,
+		[CLK_MIPI_CSI_S]	= &mipi_csi_s_clk.common.hw,
+		[CLK_MIPI_CSI_M]	= &mipi_csi_m_clk.common.hw,
+	},
+	.num	= CLK_MM_CLK_NUM,
+};
+
+static const struct sprd_clk_desc sc9863a_mm_clk_desc = {
+	.clk_clks	= sc9863a_mm_clk_clks,
+	.num_clk_clks	= ARRAY_SIZE(sc9863a_mm_clk_clks),
+	.hw_clks	= &sc9863a_mm_clk_hws,
+};
+
 static SPRD_SC_GATE_CLK_FW_NAME(sim0_eb,	"sim0-eb",	"ext-26m", 0x0,
 				0x1000, BIT(0), 0, 0);
 static SPRD_SC_GATE_CLK_FW_NAME(iis0_eb,	"iis0-eb",	"ext-26m", 0x0,
@@ -1738,6 +1768,8 @@ static const struct of_device_id sprd_sc9863a_clk_ids[] = {
 	  .data = &sc9863a_aonapb_gate_desc },
 	{ .compatible = "sprd,sc9863a-mm-gate",	/* 0x60800000 */
 	  .data = &sc9863a_mm_gate_desc },
+	{ .compatible = "sprd,sc9863a-mm-clk",	/* 0x60900000 */
+	  .data = &sc9863a_mm_clk_desc },
 	{ .compatible = "sprd,sc9863a-apapb-gate",	/* 0x71300000 */
 	  .data = &sc9863a_apapb_gate_desc },
 	{ }
diff --git a/drivers/clk/st/clk-flexgen.c b/drivers/clk/st/clk-flexgen.c
index 4413b6e04a8e..55873d4b7603 100644
--- a/drivers/clk/st/clk-flexgen.c
+++ b/drivers/clk/st/clk-flexgen.c
@@ -375,6 +375,7 @@ static void __init st_of_flexgen_setup(struct device_node *np)
 			break;
 		}
 
+		flex_flags &= ~CLK_IS_CRITICAL;
 		of_clk_detect_critical(np, i, &flex_flags);
 
 		/*
diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c
index 27201fd26e44..e1aa1fbac48a 100644
--- a/drivers/clk/sunxi/clk-sunxi.c
+++ b/drivers/clk/sunxi/clk-sunxi.c
@@ -90,7 +90,7 @@ static void sun6i_a31_get_pll1_factors(struct factors_request *req)
 	 * Round down the frequency to the closest multiple of either
 	 * 6 or 16
 	 */
-	u32 round_freq_6 = round_down(freq_mhz, 6);
+	u32 round_freq_6 = rounddown(freq_mhz, 6);
 	u32 round_freq_16 = round_down(freq_mhz, 16);
 
 	if (round_freq_6 > round_freq_16)
diff --git a/drivers/clk/tegra/Kconfig b/drivers/clk/tegra/Kconfig
index 4d99a8770485..deaa4605824c 100644
--- a/drivers/clk/tegra/Kconfig
+++ b/drivers/clk/tegra/Kconfig
@@ -1,8 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
-config TEGRA_CLK_EMC
-	def_bool y
-	depends on TEGRA124_EMC
-
 config CLK_TEGRA_BPMP
 	def_bool y
 	depends on TEGRA_BPMP
diff --git a/drivers/clk/tegra/Makefile b/drivers/clk/tegra/Makefile
index 1f7c30f87ece..eec2313fd37e 100644
--- a/drivers/clk/tegra/Makefile
+++ b/drivers/clk/tegra/Makefile
@@ -13,8 +13,8 @@ obj-y					+= clk-super.o
 obj-y					+= clk-tegra-audio.o
 obj-y					+= clk-tegra-periph.o
 obj-y					+= clk-tegra-fixed.o
+obj-y					+= clk-tegra-super-cclk.o
 obj-y					+= clk-tegra-super-gen4.o
-obj-$(CONFIG_TEGRA_CLK_EMC)		+= clk-emc.o
 obj-$(CONFIG_ARCH_TEGRA_2x_SOC)         += clk-tegra20.o
 obj-$(CONFIG_ARCH_TEGRA_2x_SOC)		+= clk-tegra20-emc.o
 obj-$(CONFIG_ARCH_TEGRA_3x_SOC)         += clk-tegra30.o
@@ -22,8 +22,10 @@ obj-$(CONFIG_ARCH_TEGRA_3x_SOC)		+= clk-tegra20-emc.o
 obj-$(CONFIG_ARCH_TEGRA_114_SOC)	+= clk-tegra114.o
 obj-$(CONFIG_ARCH_TEGRA_124_SOC)	+= clk-tegra124.o
 obj-$(CONFIG_TEGRA_CLK_DFLL)		+= clk-tegra124-dfll-fcpu.o
+obj-$(CONFIG_TEGRA124_EMC)		+= clk-tegra124-emc.o
 obj-$(CONFIG_ARCH_TEGRA_132_SOC)	+= clk-tegra124.o
 obj-y					+= cvb.o
 obj-$(CONFIG_ARCH_TEGRA_210_SOC)	+= clk-tegra210.o
+obj-$(CONFIG_ARCH_TEGRA_210_SOC)	+= clk-tegra210-emc.o
 obj-$(CONFIG_CLK_TEGRA_BPMP)		+= clk-bpmp.o
 obj-y					+= clk-utils.o
diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c
index 531c2b3d814e..0b212cf2e794 100644
--- a/drivers/clk/tegra/clk-pll.c
+++ b/drivers/clk/tegra/clk-pll.c
@@ -744,13 +744,19 @@ static int _program_pll(struct clk_hw *hw, struct tegra_clk_pll_freq_table *cfg,
 
 	state = clk_pll_is_enabled(hw);
 
+	if (state && pll->params->pre_rate_change) {
+		ret = pll->params->pre_rate_change();
+		if (WARN_ON(ret))
+			return ret;
+	}
+
 	_get_pll_mnp(pll, &old_cfg);
 
 	if (state && pll->params->defaults_set && pll->params->dyn_ramp &&
 			(cfg->m == old_cfg.m) && (cfg->p == old_cfg.p)) {
 		ret = pll->params->dyn_ramp(pll, cfg);
 		if (!ret)
-			return 0;
+			goto done;
 	}
 
 	if (state) {
@@ -772,6 +778,10 @@ static int _program_pll(struct clk_hw *hw, struct tegra_clk_pll_freq_table *cfg,
 		pll_clk_start_ss(pll);
 	}
 
+done:
+	if (state && pll->params->post_rate_change)
+		pll->params->post_rate_change();
+
 	return ret;
 }
 
diff --git a/drivers/clk/tegra/clk-tegra-super-cclk.c b/drivers/clk/tegra/clk-tegra-super-cclk.c
new file mode 100644
index 000000000000..a03119c30456
--- /dev/null
+++ b/drivers/clk/tegra/clk-tegra-super-cclk.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Based on clk-super.c
+ * Copyright (c) 2012, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Based on older tegra20-cpufreq driver by Colin Cross <ccross@google.com>
+ * Copyright (C) 2010 Google, Inc.
+ *
+ * Author: Dmitry Osipenko <digetx@gmail.com>
+ * Copyright (C) 2019 GRATE-DRIVER project
+ */
+
+#include <linux/bits.h>
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "clk.h"
+
+#define PLLP_INDEX		4
+#define PLLX_INDEX		8
+
+#define SUPER_CDIV_ENB		BIT(31)
+
+static struct tegra_clk_super_mux *cclk_super;
+static bool cclk_on_pllx;
+
+static u8 cclk_super_get_parent(struct clk_hw *hw)
+{
+	return tegra_clk_super_ops.get_parent(hw);
+}
+
+static int cclk_super_set_parent(struct clk_hw *hw, u8 index)
+{
+	return tegra_clk_super_ops.set_parent(hw, index);
+}
+
+static int cclk_super_set_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long parent_rate)
+{
+	return tegra_clk_super_ops.set_rate(hw, rate, parent_rate);
+}
+
+static unsigned long cclk_super_recalc_rate(struct clk_hw *hw,
+					    unsigned long parent_rate)
+{
+	if (cclk_super_get_parent(hw) == PLLX_INDEX)
+		return parent_rate;
+
+	return tegra_clk_super_ops.recalc_rate(hw, parent_rate);
+}
+
+static int cclk_super_determine_rate(struct clk_hw *hw,
+				     struct clk_rate_request *req)
+{
+	struct clk_hw *pllp_hw = clk_hw_get_parent_by_index(hw, PLLP_INDEX);
+	struct clk_hw *pllx_hw = clk_hw_get_parent_by_index(hw, PLLX_INDEX);
+	struct tegra_clk_super_mux *super = to_clk_super_mux(hw);
+	unsigned long pllp_rate;
+	long rate = req->rate;
+
+	if (WARN_ON_ONCE(!pllp_hw || !pllx_hw))
+		return -EINVAL;
+
+	/*
+	 * Switch parent to PLLP for all CCLK rates that are suitable for PLLP.
+	 * PLLX will be disabled in this case, saving some power.
+	 */
+	pllp_rate = clk_hw_get_rate(pllp_hw);
+
+	if (rate <= pllp_rate) {
+		if (super->flags & TEGRA20_SUPER_CLK)
+			rate = pllp_rate;
+		else
+			rate = tegra_clk_super_ops.round_rate(hw, rate,
+							      &pllp_rate);
+
+		req->best_parent_rate = pllp_rate;
+		req->best_parent_hw = pllp_hw;
+		req->rate = rate;
+	} else {
+		rate = clk_hw_round_rate(pllx_hw, rate);
+		req->best_parent_rate = rate;
+		req->best_parent_hw = pllx_hw;
+		req->rate = rate;
+	}
+
+	if (WARN_ON_ONCE(rate <= 0))
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct clk_ops tegra_cclk_super_ops = {
+	.get_parent = cclk_super_get_parent,
+	.set_parent = cclk_super_set_parent,
+	.set_rate = cclk_super_set_rate,
+	.recalc_rate = cclk_super_recalc_rate,
+	.determine_rate = cclk_super_determine_rate,
+};
+
+static const struct clk_ops tegra_cclk_super_mux_ops = {
+	.get_parent = cclk_super_get_parent,
+	.set_parent = cclk_super_set_parent,
+	.determine_rate = cclk_super_determine_rate,
+};
+
+struct clk *tegra_clk_register_super_cclk(const char *name,
+		const char * const *parent_names, u8 num_parents,
+		unsigned long flags, void __iomem *reg, u8 clk_super_flags,
+		spinlock_t *lock)
+{
+	struct tegra_clk_super_mux *super;
+	struct clk *clk;
+	struct clk_init_data init;
+	u32 val;
+
+	if (WARN_ON(cclk_super))
+		return ERR_PTR(-EBUSY);
+
+	super = kzalloc(sizeof(*super), GFP_KERNEL);
+	if (!super)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.flags = flags;
+	init.parent_names = parent_names;
+	init.num_parents = num_parents;
+
+	super->reg = reg;
+	super->lock = lock;
+	super->width = 4;
+	super->flags = clk_super_flags;
+	super->hw.init = &init;
+
+	if (super->flags & TEGRA20_SUPER_CLK) {
+		init.ops = &tegra_cclk_super_mux_ops;
+	} else {
+		init.ops = &tegra_cclk_super_ops;
+
+		super->frac_div.reg = reg + 4;
+		super->frac_div.shift = 16;
+		super->frac_div.width = 8;
+		super->frac_div.frac_width = 1;
+		super->frac_div.lock = lock;
+		super->div_ops = &tegra_clk_frac_div_ops;
+	}
+
+	/*
+	 * Tegra30+ has the following CPUG clock topology:
+	 *
+	 *        +---+  +-------+  +-+            +-+                +-+
+	 * PLLP+->+   +->+DIVIDER+->+0|  +-------->+0|  ------------->+0|
+	 *        |   |  +-------+  | |  |  +---+  | |  |             | |
+	 * PLLC+->+MUX|             | +->+  | S |  | +->+             | +->+CPU
+	 *  ...   |   |             | |  |  | K |  | |  |  +-------+  | |
+	 * PLLX+->+-->+------------>+1|  +->+ I +->+1|  +->+ DIV2  +->+1|
+	 *        +---+             +++     | P |  +++     |SKIPPER|  +++
+	 *                           ^      | P |   ^      +-------+   ^
+	 *                           |      | E |   |                  |
+	 *                PLLX_SEL+--+      | R |   |       OVERHEAT+--+
+	 *                                  +---+   |
+	 *                                          |
+	 *                         SUPER_CDIV_ENB+--+
+	 *
+	 * Tegra20 is similar, but simpler. It doesn't have the divider and
+	 * thermal DIV2 skipper.
+	 *
+	 * At least for now we're not going to use clock-skipper, hence let's
+	 * ensure that it is disabled.
+	 */
+	val = readl_relaxed(reg + 4);
+	val &= ~SUPER_CDIV_ENB;
+	writel_relaxed(val, reg + 4);
+
+	clk = clk_register(NULL, &super->hw);
+	if (IS_ERR(clk))
+		kfree(super);
+	else
+		cclk_super = super;
+
+	return clk;
+}
+
+int tegra_cclk_pre_pllx_rate_change(void)
+{
+	if (IS_ERR_OR_NULL(cclk_super))
+		return -EINVAL;
+
+	if (cclk_super_get_parent(&cclk_super->hw) == PLLX_INDEX)
+		cclk_on_pllx = true;
+	else
+		cclk_on_pllx = false;
+
+	/*
+	 * CPU needs to be temporarily re-parented away from PLLX if PLLX
+	 * changes its rate. PLLP is a safe parent for CPU on all Tegra SoCs.
+	 */
+	if (cclk_on_pllx)
+		cclk_super_set_parent(&cclk_super->hw, PLLP_INDEX);
+
+	return 0;
+}
+
+void tegra_cclk_post_pllx_rate_change(void)
+{
+	if (cclk_on_pllx)
+		cclk_super_set_parent(&cclk_super->hw, PLLX_INDEX);
+}
diff --git a/drivers/clk/tegra/clk-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c
index 745f9faa98d8..745f9faa98d8 100644
--- a/drivers/clk/tegra/clk-emc.c
+++ b/drivers/clk/tegra/clk-tegra124-emc.c
diff --git a/drivers/clk/tegra/clk-tegra20.c b/drivers/clk/tegra/clk-tegra20.c
index 085feb04e913..3efc651b42e3 100644
--- a/drivers/clk/tegra/clk-tegra20.c
+++ b/drivers/clk/tegra/clk-tegra20.c
@@ -391,6 +391,8 @@ static struct tegra_clk_pll_params pll_x_params = {
 	.lock_delay = 300,
 	.freq_table = pll_x_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.pre_rate_change = tegra_cclk_pre_pllx_rate_change,
+	.post_rate_change = tegra_cclk_post_pllx_rate_change,
 };
 
 static struct tegra_clk_pll_params pll_e_params = {
@@ -702,9 +704,10 @@ static void tegra20_super_clk_init(void)
 	struct clk *clk;
 
 	/* CCLK */
-	clk = tegra_clk_register_super_mux("cclk", cclk_parents,
+	clk = tegra_clk_register_super_cclk("cclk", cclk_parents,
 			      ARRAY_SIZE(cclk_parents), CLK_SET_RATE_PARENT,
-			      clk_base + CCLK_BURST_POLICY, 0, 4, 0, 0, NULL);
+			      clk_base + CCLK_BURST_POLICY, TEGRA20_SUPER_CLK,
+			      NULL);
 	clks[TEGRA20_CLK_CCLK] = clk;
 
 	/* SCLK */
diff --git a/drivers/clk/tegra/clk-tegra210-emc.c b/drivers/clk/tegra/clk-tegra210-emc.c
new file mode 100644
index 000000000000..352a2c3fc374
--- /dev/null
+++ b/drivers/clk/tegra/clk-tegra210-emc.c
@@ -0,0 +1,369 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2015-2020, NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/clk/tegra.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+
+#define CLK_SOURCE_EMC 0x19c
+#define  CLK_SOURCE_EMC_2X_CLK_SRC GENMASK(31, 29)
+#define  CLK_SOURCE_EMC_MC_EMC_SAME_FREQ BIT(16)
+#define  CLK_SOURCE_EMC_2X_CLK_DIVISOR GENMASK(7, 0)
+
+#define CLK_SRC_PLLM 0
+#define CLK_SRC_PLLC 1
+#define CLK_SRC_PLLP 2
+#define CLK_SRC_CLK_M 3
+#define CLK_SRC_PLLM_UD 4
+#define CLK_SRC_PLLMB_UD 5
+#define CLK_SRC_PLLMB 6
+#define CLK_SRC_PLLP_UD 7
+
+struct tegra210_clk_emc {
+	struct clk_hw hw;
+	void __iomem *regs;
+
+	struct tegra210_clk_emc_provider *provider;
+
+	struct clk *parents[8];
+};
+
+static inline struct tegra210_clk_emc *
+to_tegra210_clk_emc(struct clk_hw *hw)
+{
+	return container_of(hw, struct tegra210_clk_emc, hw);
+}
+
+static const char *tegra210_clk_emc_parents[] = {
+	"pll_m", "pll_c", "pll_p", "clk_m", "pll_m_ud", "pll_mb_ud",
+	"pll_mb", "pll_p_ud",
+};
+
+static u8 tegra210_clk_emc_get_parent(struct clk_hw *hw)
+{
+	struct tegra210_clk_emc *emc = to_tegra210_clk_emc(hw);
+	u32 value;
+	u8 src;
+
+	value = readl_relaxed(emc->regs + CLK_SOURCE_EMC);
+	src = FIELD_GET(CLK_SOURCE_EMC_2X_CLK_SRC, value);
+
+	return src;
+}
+
+static unsigned long tegra210_clk_emc_recalc_rate(struct clk_hw *hw,
+						  unsigned long parent_rate)
+{
+	struct tegra210_clk_emc *emc = to_tegra210_clk_emc(hw);
+	u32 value, div;
+
+	/*
+	 * CCF assumes that neither the parent nor its rate will change during
+	 * ->set_rate(), so the parent rate passed in here was cached from the
+	 * parent before the ->set_rate() call.
+	 *
+	 * This can lead to wrong results being reported for the EMC clock if
+	 * the parent and/or parent rate have changed as part of the EMC rate
+	 * change sequence. Fix this by overriding the parent clock with what
+	 * we know to be the correct value after the rate change.
+	 */
+	parent_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
+
+	value = readl_relaxed(emc->regs + CLK_SOURCE_EMC);
+
+	div = FIELD_GET(CLK_SOURCE_EMC_2X_CLK_DIVISOR, value);
+	div += 2;
+
+	return DIV_ROUND_UP(parent_rate * 2, div);
+}
+
+static long tegra210_clk_emc_round_rate(struct clk_hw *hw, unsigned long rate,
+					unsigned long *prate)
+{
+	struct tegra210_clk_emc *emc = to_tegra210_clk_emc(hw);
+	struct tegra210_clk_emc_provider *provider = emc->provider;
+	unsigned int i;
+
+	if (!provider || !provider->configs || provider->num_configs == 0)
+		return clk_hw_get_rate(hw);
+
+	for (i = 0; i < provider->num_configs; i++) {
+		if (provider->configs[i].rate >= rate)
+			return provider->configs[i].rate;
+	}
+
+	return provider->configs[i - 1].rate;
+}
+
+static struct clk *tegra210_clk_emc_find_parent(struct tegra210_clk_emc *emc,
+						u8 index)
+{
+	struct clk_hw *parent = clk_hw_get_parent_by_index(&emc->hw, index);
+	const char *name = clk_hw_get_name(parent);
+
+	/* XXX implement cache? */
+
+	return __clk_lookup(name);
+}
+
+static int tegra210_clk_emc_set_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long parent_rate)
+{
+	struct tegra210_clk_emc *emc = to_tegra210_clk_emc(hw);
+	struct tegra210_clk_emc_provider *provider = emc->provider;
+	struct tegra210_clk_emc_config *config;
+	struct device *dev = provider->dev;
+	struct clk_hw *old, *new, *parent;
+	u8 old_idx, new_idx, index;
+	struct clk *clk;
+	unsigned int i;
+	int err;
+
+	if (!provider || !provider->configs || provider->num_configs == 0)
+		return -EINVAL;
+
+	for (i = 0; i < provider->num_configs; i++) {
+		if (provider->configs[i].rate >= rate) {
+			config = &provider->configs[i];
+			break;
+		}
+	}
+
+	if (i == provider->num_configs)
+		config = &provider->configs[i - 1];
+
+	old_idx = tegra210_clk_emc_get_parent(hw);
+	new_idx = FIELD_GET(CLK_SOURCE_EMC_2X_CLK_SRC, config->value);
+
+	old = clk_hw_get_parent_by_index(hw, old_idx);
+	new = clk_hw_get_parent_by_index(hw, new_idx);
+
+	/* if the rate has changed... */
+	if (config->parent_rate != clk_hw_get_rate(old)) {
+		/* ... but the clock source remains the same ... */
+		if (new_idx == old_idx) {
+			/* ... switch to the alternative clock source. */
+			switch (new_idx) {
+			case CLK_SRC_PLLM:
+				new_idx = CLK_SRC_PLLMB;
+				break;
+
+			case CLK_SRC_PLLM_UD:
+				new_idx = CLK_SRC_PLLMB_UD;
+				break;
+
+			case CLK_SRC_PLLMB_UD:
+				new_idx = CLK_SRC_PLLM_UD;
+				break;
+
+			case CLK_SRC_PLLMB:
+				new_idx = CLK_SRC_PLLM;
+				break;
+			}
+
+			/*
+			 * This should never happen because we can't deal with
+			 * it.
+			 */
+			if (WARN_ON(new_idx == old_idx))
+				return -EINVAL;
+
+			new = clk_hw_get_parent_by_index(hw, new_idx);
+		}
+
+		index = new_idx;
+		parent = new;
+	} else {
+		index = old_idx;
+		parent = old;
+	}
+
+	clk = tegra210_clk_emc_find_parent(emc, index);
+	if (IS_ERR(clk)) {
+		err = PTR_ERR(clk);
+		dev_err(dev, "failed to get parent clock for index %u: %d\n",
+			index, err);
+		return err;
+	}
+
+	/* set the new parent clock to the required rate */
+	if (clk_get_rate(clk) != config->parent_rate) {
+		err = clk_set_rate(clk, config->parent_rate);
+		if (err < 0) {
+			dev_err(dev, "failed to set rate %lu Hz for %pC: %d\n",
+				config->parent_rate, clk, err);
+			return err;
+		}
+	}
+
+	/* enable the new parent clock */
+	if (parent != old) {
+		err = clk_prepare_enable(clk);
+		if (err < 0) {
+			dev_err(dev, "failed to enable parent clock %pC: %d\n",
+				clk, err);
+			return err;
+		}
+	}
+
+	/* update the EMC source configuration to reflect the new parent */
+	config->value &= ~CLK_SOURCE_EMC_2X_CLK_SRC;
+	config->value |= FIELD_PREP(CLK_SOURCE_EMC_2X_CLK_SRC, index);
+
+	/*
+	 * Finally, switch the EMC programming with both old and new parent
+	 * clocks enabled.
+	 */
+	err = provider->set_rate(dev, config);
+	if (err < 0) {
+		dev_err(dev, "failed to set EMC rate to %lu Hz: %d\n", rate,
+			err);
+
+		/*
+		 * If we're unable to switch to the new EMC frequency, we no
+		 * longer need the new parent to be enabled.
+		 */
+		if (parent != old)
+			clk_disable_unprepare(clk);
+
+		return err;
+	}
+
+	/* reparent to new parent clock and disable the old parent clock */
+	if (parent != old) {
+		clk = tegra210_clk_emc_find_parent(emc, old_idx);
+		if (IS_ERR(clk)) {
+			err = PTR_ERR(clk);
+			dev_err(dev,
+				"failed to get parent clock for index %u: %d\n",
+				old_idx, err);
+			return err;
+		}
+
+		clk_hw_reparent(hw, parent);
+		clk_disable_unprepare(clk);
+	}
+
+	return err;
+}
+
+static const struct clk_ops tegra210_clk_emc_ops = {
+	.get_parent = tegra210_clk_emc_get_parent,
+	.recalc_rate = tegra210_clk_emc_recalc_rate,
+	.round_rate = tegra210_clk_emc_round_rate,
+	.set_rate = tegra210_clk_emc_set_rate,
+};
+
+struct clk *tegra210_clk_register_emc(struct device_node *np,
+				      void __iomem *regs)
+{
+	struct tegra210_clk_emc *emc;
+	struct clk_init_data init;
+	struct clk *clk;
+
+	emc = kzalloc(sizeof(*emc), GFP_KERNEL);
+	if (!emc)
+		return ERR_PTR(-ENOMEM);
+
+	emc->regs = regs;
+
+	init.name = "emc";
+	init.ops = &tegra210_clk_emc_ops;
+	init.flags = CLK_IS_CRITICAL | CLK_GET_RATE_NOCACHE;
+	init.parent_names = tegra210_clk_emc_parents;
+	init.num_parents = ARRAY_SIZE(tegra210_clk_emc_parents);
+	emc->hw.init = &init;
+
+	clk = clk_register(NULL, &emc->hw);
+	if (IS_ERR(clk)) {
+		kfree(emc);
+		return clk;
+	}
+
+	return clk;
+}
+
+int tegra210_clk_emc_attach(struct clk *clk,
+			    struct tegra210_clk_emc_provider *provider)
+{
+	struct clk_hw *hw = __clk_get_hw(clk);
+	struct tegra210_clk_emc *emc = to_tegra210_clk_emc(hw);
+	struct device *dev = provider->dev;
+	unsigned int i;
+	int err;
+
+	if (!try_module_get(provider->owner))
+		return -ENODEV;
+
+	for (i = 0; i < provider->num_configs; i++) {
+		struct tegra210_clk_emc_config *config = &provider->configs[i];
+		struct clk_hw *parent;
+		bool same_freq;
+		u8 div, src;
+
+		div = FIELD_GET(CLK_SOURCE_EMC_2X_CLK_DIVISOR, config->value);
+		src = FIELD_GET(CLK_SOURCE_EMC_2X_CLK_SRC, config->value);
+
+		/* do basic sanity checking on the EMC timings */
+		if (div & 0x1) {
+			dev_err(dev, "invalid odd divider %u for rate %lu Hz\n",
+				div, config->rate);
+			err = -EINVAL;
+			goto put;
+		}
+
+		same_freq = config->value & CLK_SOURCE_EMC_MC_EMC_SAME_FREQ;
+
+		if (same_freq != config->same_freq) {
+			dev_err(dev,
+				"ambiguous EMC to MC ratio for rate %lu Hz\n",
+				config->rate);
+			err = -EINVAL;
+			goto put;
+		}
+
+		parent = clk_hw_get_parent_by_index(hw, src);
+		config->parent = src;
+
+		if (src == CLK_SRC_PLLM || src == CLK_SRC_PLLM_UD) {
+			config->parent_rate = config->rate * (1 + div / 2);
+		} else {
+			unsigned long rate = config->rate * (1 + div / 2);
+
+			config->parent_rate = clk_hw_get_rate(parent);
+
+			if (config->parent_rate != rate) {
+				dev_err(dev,
+					"rate %lu Hz does not match input\n",
+					config->rate);
+				err = -EINVAL;
+				goto put;
+			}
+		}
+	}
+
+	emc->provider = provider;
+
+	return 0;
+
+put:
+	module_put(provider->owner);
+	return err;
+}
+EXPORT_SYMBOL_GPL(tegra210_clk_emc_attach);
+
+void tegra210_clk_emc_detach(struct clk *clk)
+{
+	struct tegra210_clk_emc *emc = to_tegra210_clk_emc(__clk_get_hw(clk));
+
+	module_put(emc->provider->owner);
+	emc->provider = NULL;
+}
+EXPORT_SYMBOL_GPL(tegra210_clk_emc_detach);
diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
index defe3b7ebfa4..68cbb98af567 100644
--- a/drivers/clk/tegra/clk-tegra210.c
+++ b/drivers/clk/tegra/clk-tegra210.c
@@ -37,6 +37,7 @@
 #define CLK_SOURCE_LA 0x1f8
 #define CLK_SOURCE_SDMMC2 0x154
 #define CLK_SOURCE_SDMMC4 0x164
+#define CLK_SOURCE_EMC_DLL 0x664
 
 #define PLLC_BASE 0x80
 #define PLLC_OUT 0x84
@@ -227,6 +228,10 @@
 #define RST_DFLL_DVCO 0x2f4
 #define DVFS_DFLL_RESET_SHIFT 0
 
+#define CLK_RST_CONTROLLER_CLK_OUT_ENB_X_SET	0x284
+#define CLK_RST_CONTROLLER_CLK_OUT_ENB_X_CLR	0x288
+#define CLK_OUT_ENB_X_CLK_ENB_EMC_DLL		BIT(14)
+
 #define CLK_RST_CONTROLLER_RST_DEV_Y_SET 0x2a8
 #define CLK_RST_CONTROLLER_RST_DEV_Y_CLR 0x2ac
 #define CPU_SOFTRST_CTRL 0x380
@@ -314,12 +319,6 @@ static unsigned long tegra210_input_freq[] = {
 	[8] = 12000000,
 };
 
-static const char *mux_pllmcp_clkm[] = {
-	"pll_m", "pll_c", "pll_p", "clk_m", "pll_m_ud", "pll_mb", "pll_mb",
-	"pll_p",
-};
-#define mux_pllmcp_clkm_idx NULL
-
 #define PLL_ENABLE			(1 << 30)
 
 #define PLLCX_MISC1_IDDQ		(1 << 27)
@@ -555,6 +554,27 @@ void tegra210_set_sata_pll_seq_sw(bool state)
 }
 EXPORT_SYMBOL_GPL(tegra210_set_sata_pll_seq_sw);
 
+void tegra210_clk_emc_dll_enable(bool flag)
+{
+	u32 offset = flag ? CLK_RST_CONTROLLER_CLK_OUT_ENB_X_SET :
+		     CLK_RST_CONTROLLER_CLK_OUT_ENB_X_CLR;
+
+	writel_relaxed(CLK_OUT_ENB_X_CLK_ENB_EMC_DLL, clk_base + offset);
+}
+EXPORT_SYMBOL_GPL(tegra210_clk_emc_dll_enable);
+
+void tegra210_clk_emc_dll_update_setting(u32 emc_dll_src_value)
+{
+	writel_relaxed(emc_dll_src_value, clk_base + CLK_SOURCE_EMC_DLL);
+}
+EXPORT_SYMBOL_GPL(tegra210_clk_emc_dll_update_setting);
+
+void tegra210_clk_emc_update_setting(u32 emc_src_value)
+{
+	writel_relaxed(emc_src_value, clk_base + CLK_SOURCE_EMC);
+}
+EXPORT_SYMBOL_GPL(tegra210_clk_emc_update_setting);
+
 static void tegra210_generic_mbist_war(struct tegra210_domain_mbist_war *mbist)
 {
 	u32 val;
@@ -2310,7 +2330,6 @@ static struct tegra_clk tegra210_clks[tegra_clk_max] __initdata = {
 	[tegra_clk_i2c2] = { .dt_id = TEGRA210_CLK_I2C2, .present = true },
 	[tegra_clk_uartc_8] = { .dt_id = TEGRA210_CLK_UARTC, .present = true },
 	[tegra_clk_mipi_cal] = { .dt_id = TEGRA210_CLK_MIPI_CAL, .present = true },
-	[tegra_clk_emc] = { .dt_id = TEGRA210_CLK_EMC, .present = true },
 	[tegra_clk_usb2] = { .dt_id = TEGRA210_CLK_USB2, .present = true },
 	[tegra_clk_bsev] = { .dt_id = TEGRA210_CLK_BSEV, .present = true },
 	[tegra_clk_uartd_8] = { .dt_id = TEGRA210_CLK_UARTD, .present = true },
@@ -2953,6 +2972,27 @@ static const char * const sor1_parents[] = {
 
 static u32 sor1_parents_idx[] = { 0, 2, 5, 6 };
 
+static const struct clk_div_table mc_div_table_tegra210[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 4 },
+	{ .val = 2, .div = 1 },
+	{ .val = 3, .div = 2 },
+	{ .val = 0, .div = 0 },
+};
+
+static void tegra210_clk_register_mc(const char *name,
+				     const char *parent_name)
+{
+	struct clk *clk;
+
+	clk = clk_register_divider_table(NULL, name, parent_name,
+					 CLK_IS_CRITICAL,
+					 clk_base + CLK_SOURCE_EMC,
+					 15, 2, CLK_DIVIDER_READ_ONLY,
+					 mc_div_table_tegra210, &emc_lock);
+	clks[TEGRA210_CLK_MC] = clk;
+}
+
 static const char * const sor1_out_parents[] = {
 	/*
 	 * Bit 0 of the mux selects sor1_pad_clkout, irrespective of bit 1, so
@@ -2995,7 +3035,8 @@ static const char * const la_parents[] = {
 static struct tegra_clk_periph tegra210_la =
 	TEGRA_CLK_PERIPH(29, 7, 9, 0, 8, 1, TEGRA_DIVIDER_ROUND_UP, 76, 0, NULL, NULL);
 
-static __init void tegra210_periph_clk_init(void __iomem *clk_base,
+static __init void tegra210_periph_clk_init(struct device_node *np,
+					    void __iomem *clk_base,
 					    void __iomem *pmc_base)
 {
 	struct clk *clk;
@@ -3035,22 +3076,19 @@ static __init void tegra210_periph_clk_init(void __iomem *clk_base,
 					     periph_clk_enb_refcnt);
 	clks[TEGRA210_CLK_DSIB] = clk;
 
+	/* csi_tpg */
+	clk = clk_register_gate(NULL, "csi_tpg", "pll_d",
+				CLK_SET_RATE_PARENT, clk_base + PLLD_BASE,
+				23, 0, &pll_d_lock);
+	clk_register_clkdev(clk, "csi_tpg", NULL);
+	clks[TEGRA210_CLK_CSI_TPG] = clk;
+
 	/* la */
 	clk = tegra_clk_register_periph("la", la_parents,
 			ARRAY_SIZE(la_parents), &tegra210_la, clk_base,
 			CLK_SOURCE_LA, 0);
 	clks[TEGRA210_CLK_LA] = clk;
 
-	/* emc mux */
-	clk = clk_register_mux(NULL, "emc_mux", mux_pllmcp_clkm,
-			       ARRAY_SIZE(mux_pllmcp_clkm), 0,
-			       clk_base + CLK_SOURCE_EMC,
-			       29, 3, 0, &emc_lock);
-
-	clk = tegra_clk_register_mc("mc", "emc_mux", clk_base + CLK_SOURCE_EMC,
-				    &emc_lock);
-	clks[TEGRA210_CLK_MC] = clk;
-
 	/* cml0 */
 	clk = clk_register_gate(NULL, "cml0", "pll_e", 0, clk_base + PLLE_AUX,
 				0, 0, &pll_e_lock);
@@ -3093,6 +3131,13 @@ static __init void tegra210_periph_clk_init(void __iomem *clk_base,
 	}
 
 	tegra_periph_clk_init(clk_base, pmc_base, tegra210_clks, &pll_p_params);
+
+	/* emc */
+	clk = tegra210_clk_register_emc(np, clk_base);
+	clks[TEGRA210_CLK_EMC] = clk;
+
+	/* mc */
+	tegra210_clk_register_mc("mc", "emc");
 }
 
 static void __init tegra210_pll_init(void __iomem *clk_base,
@@ -3153,6 +3198,17 @@ static void __init tegra210_pll_init(void __iomem *clk_base,
 	clk_register_clkdev(clk, "pll_m_ud", NULL);
 	clks[TEGRA210_CLK_PLL_M_UD] = clk;
 
+	/* PLLMB_UD */
+	clk = clk_register_fixed_factor(NULL, "pll_mb_ud", "pll_mb",
+					CLK_SET_RATE_PARENT, 1, 1);
+	clk_register_clkdev(clk, "pll_mb_ud", NULL);
+	clks[TEGRA210_CLK_PLL_MB_UD] = clk;
+
+	/* PLLP_UD */
+	clk = clk_register_fixed_factor(NULL, "pll_p_ud", "pll_p",
+					0, 1, 1);
+	clks[TEGRA210_CLK_PLL_P_UD] = clk;
+
 	/* PLLU_VCO */
 	if (!tegra210_init_pllu()) {
 		clk = clk_register_fixed_rate(NULL, "pll_u_vco", "pll_ref", 0,
@@ -3680,7 +3736,7 @@ static void __init tegra210_clock_init(struct device_node *np)
 
 	tegra_fixed_clk_init(tegra210_clks);
 	tegra210_pll_init(clk_base, pmc_base);
-	tegra210_periph_clk_init(clk_base, pmc_base);
+	tegra210_periph_clk_init(np, clk_base, pmc_base);
 	tegra_audio_clk_init(clk_base, pmc_base, tegra210_clks,
 			     tegra210_audio_plls,
 			     ARRAY_SIZE(tegra210_audio_plls), 24576000);
diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c
index 3255f82e61b5..37244a7e68c2 100644
--- a/drivers/clk/tegra/clk-tegra30.c
+++ b/drivers/clk/tegra/clk-tegra30.c
@@ -499,6 +499,8 @@ static struct tegra_clk_pll_params pll_x_params __ro_after_init = {
 	.freq_table = pll_x_freq_table,
 	.flags = TEGRA_PLL_HAS_CPCON | TEGRA_PLL_SET_DCCON |
 		 TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.pre_rate_change = tegra_cclk_pre_pllx_rate_change,
+	.post_rate_change = tegra_cclk_post_pllx_rate_change,
 };
 
 static struct tegra_clk_pll_params pll_e_params __ro_after_init = {
@@ -926,11 +928,11 @@ static void __init tegra30_super_clk_init(void)
 	clk_register_clkdev(clk, "pll_p_out4_cclkg", NULL);
 
 	/* CCLKG */
-	clk = tegra_clk_register_super_mux("cclk_g", cclk_g_parents,
+	clk = tegra_clk_register_super_cclk("cclk_g", cclk_g_parents,
 				  ARRAY_SIZE(cclk_g_parents),
 				  CLK_SET_RATE_PARENT,
 				  clk_base + CCLKG_BURST_POLICY,
-				  0, 4, 0, 0, NULL);
+				  0, NULL);
 	clks[TEGRA30_CLK_CCLK_G] = clk;
 
 	/*
diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h
index 2c9a68302e02..6b565f6b5f66 100644
--- a/drivers/clk/tegra/clk.h
+++ b/drivers/clk/tegra/clk.h
@@ -266,6 +266,10 @@ struct tegra_clk_pll;
  *				disabled.
  * @dyn_ramp:			Callback which can be used to define a custom
  *				dynamic ramp function for a given PLL.
+ * @pre_rate_change:		Callback which is invoked just before changing
+ *				PLL's rate.
+ * @post_rate_change:		Callback which is invoked right after changing
+ *				PLL's rate.
  *
  * Flags:
  * TEGRA_PLL_USE_LOCK - This flag indicated to use lock bits for
@@ -342,6 +346,8 @@ struct tegra_clk_pll_params {
 	void	(*set_defaults)(struct tegra_clk_pll *pll);
 	int	(*dyn_ramp)(struct tegra_clk_pll *pll,
 			struct tegra_clk_pll_freq_table *cfg);
+	int	(*pre_rate_change)(void);
+	void	(*post_rate_change)(void);
 };
 
 #define TEGRA_PLL_USE_LOCK BIT(0)
@@ -729,8 +735,10 @@ struct clk *tegra_clk_register_periph_data(void __iomem *clk_base,
  * TEGRA_DIVIDER_2 - LP cluster has additional divider. This flag indicates
  *     that this is LP cluster clock.
  * TEGRA210_CPU_CLK - This flag is used to identify CPU cluster for gen5
- * super mux parent using PLLP branches. To use PLLP branches to CPU, need
- * to configure additional bit PLLP_OUT_CPU in the clock registers.
+ *     super mux parent using PLLP branches. To use PLLP branches to CPU, need
+ *     to configure additional bit PLLP_OUT_CPU in the clock registers.
+ * TEGRA20_SUPER_CLK - Tegra20 doesn't have a dedicated divider for Super
+ *     clocks, it only has a clock-skipper.
  */
 struct tegra_clk_super_mux {
 	struct clk_hw	hw;
@@ -748,6 +756,7 @@ struct tegra_clk_super_mux {
 
 #define TEGRA_DIVIDER_2 BIT(0)
 #define TEGRA210_CPU_CLK BIT(1)
+#define TEGRA20_SUPER_CLK BIT(2)
 
 extern const struct clk_ops tegra_clk_super_ops;
 struct clk *tegra_clk_register_super_mux(const char *name,
@@ -758,6 +767,12 @@ struct clk *tegra_clk_register_super_clk(const char *name,
 		const char * const *parent_names, u8 num_parents,
 		unsigned long flags, void __iomem *reg, u8 clk_super_flags,
 		spinlock_t *lock);
+struct clk *tegra_clk_register_super_cclk(const char *name,
+		const char * const *parent_names, u8 num_parents,
+		unsigned long flags, void __iomem *reg, u8 clk_super_flags,
+		spinlock_t *lock);
+int tegra_cclk_pre_pllx_rate_change(void);
+void tegra_cclk_post_pllx_rate_change(void);
 
 /**
  * struct tegra_sdmmc_mux - switch divider with Low Jitter inputs for SDMMC
@@ -866,7 +881,7 @@ void tegra_super_clk_gen5_init(void __iomem *clk_base,
 			void __iomem *pmc_base, struct tegra_clk *tegra_clks,
 			struct tegra_clk_pll_params *pll_params);
 
-#ifdef CONFIG_TEGRA_CLK_EMC
+#ifdef CONFIG_TEGRA124_EMC
 struct clk *tegra_clk_register_emc(void __iomem *base, struct device_node *np,
 				   spinlock_t *lock);
 #else
@@ -907,4 +922,7 @@ void tegra_clk_periph_resume(void);
 bool tegra20_clk_emc_driver_available(struct clk_hw *emc_hw);
 struct clk *tegra20_clk_register_emc(void __iomem *ioaddr, bool low_jitter);
 
+struct clk *tegra210_clk_register_emc(struct device_node *np,
+				      void __iomem *regs);
+
 #endif /* TEGRA_CLK_H */
diff --git a/drivers/clk/ti/Kconfig b/drivers/clk/ti/Kconfig
index d913d8663f73..4972912307ef 100644
--- a/drivers/clk/ti/Kconfig
+++ b/drivers/clk/ti/Kconfig
@@ -3,5 +3,5 @@ config COMMON_CLK_TI_ADPLL
 	tristate "Clock driver for dm814x ADPLL"
 	depends on ARCH_OMAP2PLUS || COMPILE_TEST
 	default y if SOC_TI81XX
-	---help---
+	help
 	  ADPLL clock driver for the dm814x SoC using common clock framework.
diff --git a/drivers/clk/ti/clk-44xx.c b/drivers/clk/ti/clk-44xx.c
index 312a20f8ec0e..a38c92153979 100644
--- a/drivers/clk/ti/clk-44xx.c
+++ b/drivers/clk/ti/clk-44xx.c
@@ -606,13 +606,13 @@ static const struct omap_clkctrl_reg_data omap4_l4_per_clkctrl_regs[] __initcons
 
 static const struct
 omap_clkctrl_reg_data omap4_l4_secure_clkctrl_regs[] __initconst = {
-	{ OMAP4_AES1_CLKCTRL, NULL, CLKF_SW_SUP, "" },
-	{ OMAP4_AES2_CLKCTRL, NULL, CLKF_SW_SUP, "" },
-	{ OMAP4_DES3DES_CLKCTRL, NULL, CLKF_SW_SUP, "" },
-	{ OMAP4_PKA_CLKCTRL, NULL, CLKF_SW_SUP, "" },
-	{ OMAP4_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "" },
-	{ OMAP4_SHA2MD5_CLKCTRL, NULL, CLKF_SW_SUP, "" },
-	{ OMAP4_CRYPTODMA_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "" },
+	{ OMAP4_AES1_CLKCTRL, NULL, CLKF_SW_SUP, "l3_div_ck" },
+	{ OMAP4_AES2_CLKCTRL, NULL, CLKF_SW_SUP, "l3_div_ck" },
+	{ OMAP4_DES3DES_CLKCTRL, NULL, CLKF_SW_SUP, "l4_div_ck" },
+	{ OMAP4_PKA_CLKCTRL, NULL, CLKF_SW_SUP, "l4_div_ck" },
+	{ OMAP4_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l4_div_ck" },
+	{ OMAP4_SHA2MD5_CLKCTRL, NULL, CLKF_SW_SUP, "l3_div_ck" },
+	{ OMAP4_CRYPTODMA_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l3_div_ck" },
 	{ 0 },
 };
 
diff --git a/drivers/clk/ti/clk-54xx.c b/drivers/clk/ti/clk-54xx.c
index 92bf2dda95b9..8694bc9f5fc7 100644
--- a/drivers/clk/ti/clk-54xx.c
+++ b/drivers/clk/ti/clk-54xx.c
@@ -303,13 +303,13 @@ static const struct omap_clkctrl_reg_data omap5_l4per_clkctrl_regs[] __initconst
 
 static const struct
 omap_clkctrl_reg_data omap5_l4_secure_clkctrl_regs[] __initconst = {
-	{ OMAP5_AES1_CLKCTRL, NULL, CLKF_HW_SUP, "" },
-	{ OMAP5_AES2_CLKCTRL, NULL, CLKF_HW_SUP, "" },
-	{ OMAP5_DES3DES_CLKCTRL, NULL, CLKF_HW_SUP, "" },
-	{ OMAP5_FPKA_CLKCTRL, NULL, CLKF_SW_SUP, "" },
-	{ OMAP5_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "" },
-	{ OMAP5_SHA2MD5_CLKCTRL, NULL, CLKF_HW_SUP, "" },
-	{ OMAP5_DMA_CRYPTO_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "" },
+	{ OMAP5_AES1_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ OMAP5_AES2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ OMAP5_DES3DES_CLKCTRL, NULL, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_FPKA_CLKCTRL, NULL, CLKF_SW_SUP, "l4_root_clk_div" },
+	{ OMAP5_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l4_root_clk_div" },
+	{ OMAP5_SHA2MD5_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ OMAP5_DMA_CRYPTO_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l3_iclk_div" },
 	{ 0 },
 };
 
diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c
index 14b645093107..b4cf578a69e1 100644
--- a/drivers/clk/ti/clk-7xx.c
+++ b/drivers/clk/ti/clk-7xx.c
@@ -312,15 +312,6 @@ static const char * const dra7_gpu_hyd_mux_parents[] __initconst = {
 	NULL,
 };
 
-static const char * const dra7_gpu_sys_clk_parents[] __initconst = {
-	"sys_clkin",
-	NULL,
-};
-
-static const struct omap_clkctrl_div_data dra7_gpu_sys_clk_data __initconst = {
-	.max_div = 2,
-};
-
 static const struct omap_clkctrl_bit_data dra7_gpu_core_bit_data[] __initconst = {
 	{ 24, TI_CLK_MUX, dra7_gpu_core_mux_parents, NULL, },
 	{ 26, TI_CLK_MUX, dra7_gpu_hyd_mux_parents, NULL, },
@@ -328,7 +319,7 @@ static const struct omap_clkctrl_bit_data dra7_gpu_core_bit_data[] __initconst =
 };
 
 static const struct omap_clkctrl_reg_data dra7_gpu_clkctrl_regs[] __initconst = {
-	{ DRA7_GPU_CLKCTRL, dra7_gpu_core_bit_data, CLKF_SW_SUP, "gpu_cm:clk:0000:24", },
+	{ DRA7_GPU_CLKCTRL, dra7_gpu_core_bit_data, CLKF_SW_SUP, "gpu-clkctrl:0000:24", },
 	{ 0 },
 };
 
@@ -644,7 +635,7 @@ static const struct omap_clkctrl_reg_data dra7_l4sec_clkctrl_regs[] __initconst
 	{ DRA7_L4SEC_AES1_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
 	{ DRA7_L4SEC_AES2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
 	{ DRA7_L4SEC_DES_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
-	{ DRA7_L4SEC_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "" },
+	{ DRA7_L4SEC_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l4_root_clk_div" },
 	{ DRA7_L4SEC_SHAM_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
 	{ 0 },
 };
@@ -815,7 +806,7 @@ static const struct omap_clkctrl_reg_data dra7_wkupaon_clkctrl_regs[] __initcons
 	{ DRA7_WKUPAON_COUNTER_32K_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
 	{ DRA7_WKUPAON_UART10_CLKCTRL, dra7_uart10_bit_data, CLKF_SW_SUP, "wkupaon-clkctrl:0060:24" },
 	{ DRA7_WKUPAON_DCAN1_CLKCTRL, dra7_dcan1_bit_data, CLKF_SW_SUP, "wkupaon-clkctrl:0068:24" },
-	{ DRA7_WKUPAON_ADC_CLKCTRL, NULL, CLKF_SW_SUP, "mcan_clk" },
+	{ DRA7_WKUPAON_ADC_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_SOC_DRA76, "mcan_clk" },
 	{ 0 },
 };
 
diff --git a/drivers/clk/ti/clk-816x.c b/drivers/clk/ti/clk-816x.c
index 7d215cdf9dda..9daf3825f289 100644
--- a/drivers/clk/ti/clk-816x.c
+++ b/drivers/clk/ti/clk-816x.c
@@ -73,6 +73,7 @@ static const char *enable_init_clks[] = {
 	"ddr_pll_clk1",
 	"ddr_pll_clk2",
 	"ddr_pll_clk3",
+	"sysclk6_ck",
 };
 
 int __init dm816x_dt_clk_init(void)
diff --git a/drivers/clk/ti/composite.c b/drivers/clk/ti/composite.c
index 6a89936ba03a..eaa43575cfa5 100644
--- a/drivers/clk/ti/composite.c
+++ b/drivers/clk/ti/composite.c
@@ -196,6 +196,7 @@ cleanup:
 		if (!cclk->comp_clks[i])
 			continue;
 		list_del(&cclk->comp_clks[i]->link);
+		kfree(cclk->comp_clks[i]->parent_names);
 		kfree(cclk->comp_clks[i]);
 	}
 
diff --git a/drivers/clk/versatile/Kconfig b/drivers/clk/versatile/Kconfig
index c2618f1477a2..91f0ff54237d 100644
--- a/drivers/clk/versatile/Kconfig
+++ b/drivers/clk/versatile/Kconfig
@@ -1,33 +1,32 @@
 # SPDX-License-Identifier: GPL-2.0-only
-config ICST
-	bool
 
-config COMMON_CLK_VERSATILE
-	bool "Clock driver for ARM Reference designs"
+menu "Clock driver for ARM Reference designs"
 	depends on ARCH_INTEGRATOR || ARCH_REALVIEW || \
-		ARCH_VERSATILE || ARCH_VEXPRESS || ARM64 || \
-		COMPILE_TEST
+		ARCH_VERSATILE || ARCH_VEXPRESS || COMPILE_TEST
+
+config ICST
+	bool "Clock driver for ARM Reference designs ICST"
 	select REGMAP_MMIO
-	---help---
+	help
 	  Supports clocking on ARM Reference designs:
 	  - Integrator/AP and Integrator/CP
 	  - RealView PB1176, EB, PB11MP and PBX
-	  - Versatile Express
 
 config CLK_SP810
 	bool "Clock driver for ARM SP810 System Controller"
-	depends on COMMON_CLK_VERSATILE
-	default y if ARCH_VEXPRESS
-	---help---
+	default y if (ARCH_VEXPRESS && ARM)
+	help
 	  Supports clock muxing (REFCLK/TIMCLK to TIMERCLKEN0-3) capabilities
 	  of the ARM SP810 System Controller cell.
 
 config CLK_VEXPRESS_OSC
-	bool "Clock driver for Versatile Express OSC clock generators"
-	depends on COMMON_CLK_VERSATILE
+	tristate "Clock driver for Versatile Express OSC clock generators"
 	depends on VEXPRESS_CONFIG
+	select REGMAP_MMIO
 	default y if ARCH_VEXPRESS
-	---help---
+	help
 	  Simple regmap-based driver driving clock generators on Versatile
 	  Express platforms hidden behind its configuration infrastructure,
 	  commonly known as OSCs.
+
+endmenu
diff --git a/drivers/clk/versatile/clk-impd1.c b/drivers/clk/versatile/clk-impd1.c
index f9f4babe3ca6..ca798249544d 100644
--- a/drivers/clk/versatile/clk-impd1.c
+++ b/drivers/clk/versatile/clk-impd1.c
@@ -8,7 +8,6 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
-#include <linux/platform_data/clk-integrator.h>
 #include <linux/module.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
@@ -20,26 +19,6 @@
 #define IMPD1_OSC2	0x04
 #define IMPD1_LOCK	0x08
 
-struct impd1_clk {
-	char *pclkname;
-	struct clk *pclk;
-	char *vco1name;
-	struct clk *vco1clk;
-	char *vco2name;
-	struct clk *vco2clk;
-	struct clk *mmciclk;
-	char *uartname;
-	struct clk *uartclk;
-	char *spiname;
-	struct clk *spiclk;
-	char *scname;
-	struct clk *scclk;
-	struct clk_lookup *clks[15];
-};
-
-/* One entry for each connected IM-PD1 LM */
-static struct impd1_clk impd1_clks[4];
-
 /*
  * There are two VCO's on the IM-PD1
  */
@@ -80,106 +59,6 @@ static const struct clk_icst_desc impd1_icst2_desc = {
 	.lock_offset = IMPD1_LOCK,
 };
 
-/**
- * integrator_impd1_clk_init() - set up the integrator clock tree
- * @base: base address of the logic module (LM)
- * @id: the ID of this LM
- */
-void integrator_impd1_clk_init(void __iomem *base, unsigned int id)
-{
-	struct impd1_clk *imc;
-	struct clk *clk;
-	struct clk *pclk;
-	int i;
-
-	if (id > 3) {
-		pr_crit("no more than 4 LMs can be attached\n");
-		return;
-	}
-	imc = &impd1_clks[id];
-
-	/* Register the fixed rate PCLK */
-	imc->pclkname = kasprintf(GFP_KERNEL, "lm%x-pclk", id);
-	pclk = clk_register_fixed_rate(NULL, imc->pclkname, NULL, 0, 0);
-	imc->pclk = pclk;
-
-	imc->vco1name = kasprintf(GFP_KERNEL, "lm%x-vco1", id);
-	clk = icst_clk_register(NULL, &impd1_icst1_desc, imc->vco1name, NULL,
-				base);
-	imc->vco1clk = clk;
-	imc->clks[0] = clkdev_alloc(pclk, "apb_pclk", "lm%x:01000", id);
-	imc->clks[1] = clkdev_alloc(clk, NULL, "lm%x:01000", id);
-
-	/* VCO2 is also called "CLK2" */
-	imc->vco2name = kasprintf(GFP_KERNEL, "lm%x-vco2", id);
-	clk = icst_clk_register(NULL, &impd1_icst2_desc, imc->vco2name, NULL,
-				base);
-	imc->vco2clk = clk;
-
-	/* MMCI uses CLK2 right off */
-	imc->clks[2] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00700", id);
-	imc->clks[3] = clkdev_alloc(clk, NULL, "lm%x:00700", id);
-
-	/* UART reference clock divides CLK2 by a fixed factor 4 */
-	imc->uartname = kasprintf(GFP_KERNEL, "lm%x-uartclk", id);
-	clk = clk_register_fixed_factor(NULL, imc->uartname, imc->vco2name,
-				   CLK_IGNORE_UNUSED, 1, 4);
-	imc->uartclk = clk;
-	imc->clks[4] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00100", id);
-	imc->clks[5] = clkdev_alloc(clk, NULL, "lm%x:00100", id);
-	imc->clks[6] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00200", id);
-	imc->clks[7] = clkdev_alloc(clk, NULL, "lm%x:00200", id);
-
-	/* SPI PL022 clock divides CLK2 by a fixed factor 64 */
-	imc->spiname = kasprintf(GFP_KERNEL, "lm%x-spiclk", id);
-	clk = clk_register_fixed_factor(NULL, imc->spiname, imc->vco2name,
-				   CLK_IGNORE_UNUSED, 1, 64);
-	imc->clks[8] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00300", id);
-	imc->clks[9] = clkdev_alloc(clk, NULL, "lm%x:00300", id);
-
-	/* The GPIO blocks and AACI have only PCLK */
-	imc->clks[10] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00400", id);
-	imc->clks[11] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00500", id);
-	imc->clks[12] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00800", id);
-
-	/* Smart Card clock divides CLK2 by a fixed factor 4 */
-	imc->scname = kasprintf(GFP_KERNEL, "lm%x-scclk", id);
-	clk = clk_register_fixed_factor(NULL, imc->scname, imc->vco2name,
-				   CLK_IGNORE_UNUSED, 1, 4);
-	imc->scclk = clk;
-	imc->clks[13] = clkdev_alloc(pclk, "apb_pclk", "lm%x:00600", id);
-	imc->clks[14] = clkdev_alloc(clk, NULL, "lm%x:00600", id);
-
-	for (i = 0; i < ARRAY_SIZE(imc->clks); i++)
-		clkdev_add(imc->clks[i]);
-}
-EXPORT_SYMBOL_GPL(integrator_impd1_clk_init);
-
-void integrator_impd1_clk_exit(unsigned int id)
-{
-	int i;
-	struct impd1_clk *imc;
-
-	if (id > 3)
-		return;
-	imc = &impd1_clks[id];
-
-	for (i = 0; i < ARRAY_SIZE(imc->clks); i++)
-		clkdev_drop(imc->clks[i]);
-	clk_unregister(imc->spiclk);
-	clk_unregister(imc->uartclk);
-	clk_unregister(imc->vco2clk);
-	clk_unregister(imc->vco1clk);
-	clk_unregister(imc->pclk);
-	kfree(imc->scname);
-	kfree(imc->spiname);
-	kfree(imc->uartname);
-	kfree(imc->vco2name);
-	kfree(imc->vco1name);
-	kfree(imc->pclkname);
-}
-EXPORT_SYMBOL_GPL(integrator_impd1_clk_exit);
-
 static int integrator_impd1_clk_spawn(struct device *dev,
 				      struct device_node *parent,
 				      struct device_node *np)
diff --git a/drivers/clk/versatile/clk-versatile.c b/drivers/clk/versatile/clk-versatile.c
index fd54d5c0251c..8ed7a179f651 100644
--- a/drivers/clk/versatile/clk-versatile.c
+++ b/drivers/clk/versatile/clk-versatile.c
@@ -56,7 +56,7 @@ static const struct clk_icst_desc versatile_auxosc_desc __initconst = {
 static void __init cm_osc_setup(struct device_node *np,
 				const struct clk_icst_desc *desc)
 {
-	struct clk *clk = ERR_PTR(-EINVAL);
+	struct clk *clk;
 	const char *clk_name = np->name;
 	const char *parent_name;
 
diff --git a/drivers/clk/versatile/clk-vexpress-osc.c b/drivers/clk/versatile/clk-vexpress-osc.c
index 7ade146a3ea9..b2b32fa2d7c3 100644
--- a/drivers/clk/versatile/clk-vexpress-osc.c
+++ b/drivers/clk/versatile/clk-vexpress-osc.c
@@ -7,6 +7,7 @@
 #include <linux/clkdev.h>
 #include <linux/clk-provider.h>
 #include <linux/err.h>
+#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -65,8 +66,8 @@ static int vexpress_osc_probe(struct platform_device *pdev)
 {
 	struct clk_init_data init;
 	struct vexpress_osc *osc;
-	struct clk *clk;
 	u32 range[2];
+	int ret;
 
 	osc = devm_kzalloc(&pdev->dev, sizeof(*osc), GFP_KERNEL);
 	if (!osc)
@@ -92,11 +93,11 @@ static int vexpress_osc_probe(struct platform_device *pdev)
 
 	osc->hw.init = &init;
 
-	clk = clk_register(NULL, &osc->hw);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ret = devm_clk_hw_register(&pdev->dev, &osc->hw);
+	if (ret < 0)
+		return ret;
 
-	of_clk_add_provider(pdev->dev.of_node, of_clk_src_simple_get, clk);
+	devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_simple_get, &osc->hw);
 	clk_hw_set_rate_range(&osc->hw, osc->rate_min, osc->rate_max);
 
 	dev_dbg(&pdev->dev, "Registered clock '%s'\n", init.name);
@@ -108,6 +109,7 @@ static const struct of_device_id vexpress_osc_of_match[] = {
 	{ .compatible = "arm,vexpress-osc", },
 	{}
 };
+MODULE_DEVICE_TABLE(of, vexpress_osc_of_match);
 
 static struct platform_driver vexpress_osc_driver = {
 	.driver	= {
@@ -116,9 +118,5 @@ static struct platform_driver vexpress_osc_driver = {
 	},
 	.probe = vexpress_osc_probe,
 };
-
-static int __init vexpress_osc_init(void)
-{
-	return platform_driver_register(&vexpress_osc_driver);
-}
-core_initcall(vexpress_osc_init);
+module_platform_driver(vexpress_osc_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/x86/Kconfig b/drivers/clk/x86/Kconfig
new file mode 100644
index 000000000000..69642e15fcc1
--- /dev/null
+++ b/drivers/clk/x86/Kconfig
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config CLK_LGM_CGU
+	depends on OF && HAS_IOMEM && (X86 || COMPILE_TEST)
+	select OF_EARLY_FLATTREE
+	bool "Clock driver for Lightning Mountain(LGM) platform"
+	help
+	  Clock Generation Unit(CGU) driver for Intel Lightning Mountain(LGM)
+	  network processor SoC.
diff --git a/drivers/clk/x86/Makefile b/drivers/clk/x86/Makefile
index e3ec81e2a1c2..7c774ea7ddeb 100644
--- a/drivers/clk/x86/Makefile
+++ b/drivers/clk/x86/Makefile
@@ -3,3 +3,4 @@ obj-$(CONFIG_PMC_ATOM)		+= clk-pmc-atom.o
 obj-$(CONFIG_X86_AMD_PLATFORM_DEVICE)	+= clk-st.o
 clk-x86-lpss-objs		:= clk-lpt.o
 obj-$(CONFIG_X86_INTEL_LPSS)	+= clk-x86-lpss.o
+obj-$(CONFIG_CLK_LGM_CGU)	+= clk-cgu.o clk-cgu-pll.o clk-lgm.o
diff --git a/drivers/clk/x86/clk-cgu-pll.c b/drivers/clk/x86/clk-cgu-pll.c
new file mode 100644
index 000000000000..c03cc6b85b9f
--- /dev/null
+++ b/drivers/clk/x86/clk-cgu-pll.c
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Intel Corporation.
+ * Zhu YiXin <yixin.zhu@intel.com>
+ * Rahul Tanwar <rahul.tanwar@intel.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/iopoll.h>
+#include <linux/of.h>
+
+#include "clk-cgu.h"
+
+#define to_lgm_clk_pll(_hw)	container_of(_hw, struct lgm_clk_pll, hw)
+#define PLL_REF_DIV(x)		((x) + 0x08)
+
+/*
+ * Calculate formula:
+ * rate = (prate * mult + (prate * frac) / frac_div) / div
+ */
+static unsigned long
+lgm_pll_calc_rate(unsigned long prate, unsigned int mult,
+		  unsigned int div, unsigned int frac, unsigned int frac_div)
+{
+	u64 crate, frate, rate64;
+
+	rate64 = prate;
+	crate = rate64 * mult;
+	frate = rate64 * frac;
+	do_div(frate, frac_div);
+	crate += frate;
+	do_div(crate, div);
+
+	return crate;
+}
+
+static unsigned long lgm_pll_recalc_rate(struct clk_hw *hw, unsigned long prate)
+{
+	struct lgm_clk_pll *pll = to_lgm_clk_pll(hw);
+	unsigned int div, mult, frac;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	mult = lgm_get_clk_val(pll->membase, PLL_REF_DIV(pll->reg), 0, 12);
+	div = lgm_get_clk_val(pll->membase, PLL_REF_DIV(pll->reg), 18, 6);
+	frac = lgm_get_clk_val(pll->membase, pll->reg, 2, 24);
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	if (pll->type == TYPE_LJPLL)
+		div *= 4;
+
+	return lgm_pll_calc_rate(prate, mult, div, frac, BIT(24));
+}
+
+static int lgm_pll_is_enabled(struct clk_hw *hw)
+{
+	struct lgm_clk_pll *pll = to_lgm_clk_pll(hw);
+	unsigned long flags;
+	unsigned int ret;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	ret = lgm_get_clk_val(pll->membase, pll->reg, 0, 1);
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	return ret;
+}
+
+static int lgm_pll_enable(struct clk_hw *hw)
+{
+	struct lgm_clk_pll *pll = to_lgm_clk_pll(hw);
+	unsigned long flags;
+	u32 val;
+	int ret;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	lgm_set_clk_val(pll->membase, pll->reg, 0, 1, 1);
+	ret = readl_poll_timeout_atomic(pll->membase + pll->reg,
+					val, (val & 0x1), 1, 100);
+	spin_unlock_irqrestore(&pll->lock, flags);
+
+	return ret;
+}
+
+static void lgm_pll_disable(struct clk_hw *hw)
+{
+	struct lgm_clk_pll *pll = to_lgm_clk_pll(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&pll->lock, flags);
+	lgm_set_clk_val(pll->membase, pll->reg, 0, 1, 0);
+	spin_unlock_irqrestore(&pll->lock, flags);
+}
+
+static const struct clk_ops lgm_pll_ops = {
+	.recalc_rate = lgm_pll_recalc_rate,
+	.is_enabled = lgm_pll_is_enabled,
+	.enable = lgm_pll_enable,
+	.disable = lgm_pll_disable,
+};
+
+static struct clk_hw *
+lgm_clk_register_pll(struct lgm_clk_provider *ctx,
+		     const struct lgm_pll_clk_data *list)
+{
+	struct clk_init_data init = {};
+	struct lgm_clk_pll *pll;
+	struct device *dev = ctx->dev;
+	struct clk_hw *hw;
+	int ret;
+
+	init.ops = &lgm_pll_ops;
+	init.name = list->name;
+	init.flags = list->flags;
+	init.parent_data = list->parent_data;
+	init.num_parents = list->num_parents;
+
+	pll = devm_kzalloc(dev, sizeof(*pll), GFP_KERNEL);
+	if (!pll)
+		return ERR_PTR(-ENOMEM);
+
+	pll->membase = ctx->membase;
+	pll->lock = ctx->lock;
+	pll->reg = list->reg;
+	pll->flags = list->flags;
+	pll->type = list->type;
+	pll->hw.init = &init;
+
+	hw = &pll->hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return hw;
+}
+
+int lgm_clk_register_plls(struct lgm_clk_provider *ctx,
+			  const struct lgm_pll_clk_data *list,
+			  unsigned int nr_clk)
+{
+	struct clk_hw *hw;
+	int i;
+
+	for (i = 0; i < nr_clk; i++, list++) {
+		hw = lgm_clk_register_pll(ctx, list);
+		if (IS_ERR(hw)) {
+			dev_err(ctx->dev, "failed to register pll: %s\n",
+				list->name);
+			return PTR_ERR(hw);
+		}
+		ctx->clk_data.hws[list->id] = hw;
+	}
+
+	return 0;
+}
diff --git a/drivers/clk/x86/clk-cgu.c b/drivers/clk/x86/clk-cgu.c
new file mode 100644
index 000000000000..56af0e04ec1e
--- /dev/null
+++ b/drivers/clk/x86/clk-cgu.c
@@ -0,0 +1,636 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Intel Corporation.
+ * Zhu YiXin <yixin.zhu@intel.com>
+ * Rahul Tanwar <rahul.tanwar@intel.com>
+ */
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/of.h>
+
+#include "clk-cgu.h"
+
+#define GATE_HW_REG_STAT(reg)	((reg) + 0x0)
+#define GATE_HW_REG_EN(reg)	((reg) + 0x4)
+#define GATE_HW_REG_DIS(reg)	((reg) + 0x8)
+#define MAX_DDIV_REG	8
+#define MAX_DIVIDER_VAL 64
+
+#define to_lgm_clk_mux(_hw) container_of(_hw, struct lgm_clk_mux, hw)
+#define to_lgm_clk_divider(_hw) container_of(_hw, struct lgm_clk_divider, hw)
+#define to_lgm_clk_gate(_hw) container_of(_hw, struct lgm_clk_gate, hw)
+#define to_lgm_clk_ddiv(_hw) container_of(_hw, struct lgm_clk_ddiv, hw)
+
+static struct clk_hw *lgm_clk_register_fixed(struct lgm_clk_provider *ctx,
+					     const struct lgm_clk_branch *list)
+{
+	unsigned long flags;
+
+	if (list->div_flags & CLOCK_FLAG_VAL_INIT) {
+		spin_lock_irqsave(&ctx->lock, flags);
+		lgm_set_clk_val(ctx->membase, list->div_off, list->div_shift,
+				list->div_width, list->div_val);
+		spin_unlock_irqrestore(&ctx->lock, flags);
+	}
+
+	return clk_hw_register_fixed_rate(NULL, list->name,
+					  list->parent_data[0].name,
+					  list->flags, list->mux_flags);
+}
+
+static u8 lgm_clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct lgm_clk_mux *mux = to_lgm_clk_mux(hw);
+	unsigned long flags;
+	u32 val;
+
+	spin_lock_irqsave(&mux->lock, flags);
+	if (mux->flags & MUX_CLK_SW)
+		val = mux->reg;
+	else
+		val = lgm_get_clk_val(mux->membase, mux->reg, mux->shift,
+				      mux->width);
+	spin_unlock_irqrestore(&mux->lock, flags);
+	return clk_mux_val_to_index(hw, NULL, mux->flags, val);
+}
+
+static int lgm_clk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct lgm_clk_mux *mux = to_lgm_clk_mux(hw);
+	unsigned long flags;
+	u32 val;
+
+	val = clk_mux_index_to_val(NULL, mux->flags, index);
+	spin_lock_irqsave(&mux->lock, flags);
+	if (mux->flags & MUX_CLK_SW)
+		mux->reg = val;
+	else
+		lgm_set_clk_val(mux->membase, mux->reg, mux->shift,
+				mux->width, val);
+	spin_unlock_irqrestore(&mux->lock, flags);
+
+	return 0;
+}
+
+static int lgm_clk_mux_determine_rate(struct clk_hw *hw,
+				      struct clk_rate_request *req)
+{
+	struct lgm_clk_mux *mux = to_lgm_clk_mux(hw);
+
+	return clk_mux_determine_rate_flags(hw, req, mux->flags);
+}
+
+static const struct clk_ops lgm_clk_mux_ops = {
+	.get_parent = lgm_clk_mux_get_parent,
+	.set_parent = lgm_clk_mux_set_parent,
+	.determine_rate = lgm_clk_mux_determine_rate,
+};
+
+static struct clk_hw *
+lgm_clk_register_mux(struct lgm_clk_provider *ctx,
+		     const struct lgm_clk_branch *list)
+{
+	unsigned long flags, cflags = list->mux_flags;
+	struct device *dev = ctx->dev;
+	u8 shift = list->mux_shift;
+	u8 width = list->mux_width;
+	struct clk_init_data init = {};
+	struct lgm_clk_mux *mux;
+	u32 reg = list->mux_off;
+	struct clk_hw *hw;
+	int ret;
+
+	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = list->name;
+	init.ops = &lgm_clk_mux_ops;
+	init.flags = list->flags;
+	init.parent_data = list->parent_data;
+	init.num_parents = list->num_parents;
+
+	mux->membase = ctx->membase;
+	mux->lock = ctx->lock;
+	mux->reg = reg;
+	mux->shift = shift;
+	mux->width = width;
+	mux->flags = cflags;
+	mux->hw.init = &init;
+
+	hw = &mux->hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (cflags & CLOCK_FLAG_VAL_INIT) {
+		spin_lock_irqsave(&mux->lock, flags);
+		lgm_set_clk_val(mux->membase, reg, shift, width, list->mux_val);
+		spin_unlock_irqrestore(&mux->lock, flags);
+	}
+
+	return hw;
+}
+
+static unsigned long
+lgm_clk_divider_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct lgm_clk_divider *divider = to_lgm_clk_divider(hw);
+	unsigned long flags;
+	unsigned int val;
+
+	spin_lock_irqsave(&divider->lock, flags);
+	val = lgm_get_clk_val(divider->membase, divider->reg,
+			      divider->shift, divider->width);
+	spin_unlock_irqrestore(&divider->lock, flags);
+
+	return divider_recalc_rate(hw, parent_rate, val, divider->table,
+				   divider->flags, divider->width);
+}
+
+static long
+lgm_clk_divider_round_rate(struct clk_hw *hw, unsigned long rate,
+			   unsigned long *prate)
+{
+	struct lgm_clk_divider *divider = to_lgm_clk_divider(hw);
+
+	return divider_round_rate(hw, rate, prate, divider->table,
+				  divider->width, divider->flags);
+}
+
+static int
+lgm_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
+			 unsigned long prate)
+{
+	struct lgm_clk_divider *divider = to_lgm_clk_divider(hw);
+	unsigned long flags;
+	int value;
+
+	value = divider_get_val(rate, prate, divider->table,
+				divider->width, divider->flags);
+	if (value < 0)
+		return value;
+
+	spin_lock_irqsave(&divider->lock, flags);
+	lgm_set_clk_val(divider->membase, divider->reg,
+			divider->shift, divider->width, value);
+	spin_unlock_irqrestore(&divider->lock, flags);
+
+	return 0;
+}
+
+static int lgm_clk_divider_enable_disable(struct clk_hw *hw, int enable)
+{
+	struct lgm_clk_divider *div = to_lgm_clk_divider(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&div->lock, flags);
+	lgm_set_clk_val(div->membase, div->reg, div->shift_gate,
+			div->width_gate, enable);
+	spin_unlock_irqrestore(&div->lock, flags);
+	return 0;
+}
+
+static int lgm_clk_divider_enable(struct clk_hw *hw)
+{
+	return lgm_clk_divider_enable_disable(hw, 1);
+}
+
+static void lgm_clk_divider_disable(struct clk_hw *hw)
+{
+	lgm_clk_divider_enable_disable(hw, 0);
+}
+
+static const struct clk_ops lgm_clk_divider_ops = {
+	.recalc_rate = lgm_clk_divider_recalc_rate,
+	.round_rate = lgm_clk_divider_round_rate,
+	.set_rate = lgm_clk_divider_set_rate,
+	.enable = lgm_clk_divider_enable,
+	.disable = lgm_clk_divider_disable,
+};
+
+static struct clk_hw *
+lgm_clk_register_divider(struct lgm_clk_provider *ctx,
+			 const struct lgm_clk_branch *list)
+{
+	unsigned long flags, cflags = list->div_flags;
+	struct device *dev = ctx->dev;
+	struct lgm_clk_divider *div;
+	struct clk_init_data init = {};
+	u8 shift = list->div_shift;
+	u8 width = list->div_width;
+	u8 shift_gate = list->div_shift_gate;
+	u8 width_gate = list->div_width_gate;
+	u32 reg = list->div_off;
+	struct clk_hw *hw;
+	int ret;
+
+	div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL);
+	if (!div)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = list->name;
+	init.ops = &lgm_clk_divider_ops;
+	init.flags = list->flags;
+	init.parent_data = list->parent_data;
+	init.num_parents = 1;
+
+	div->membase = ctx->membase;
+	div->lock = ctx->lock;
+	div->reg = reg;
+	div->shift = shift;
+	div->width = width;
+	div->shift_gate	= shift_gate;
+	div->width_gate	= width_gate;
+	div->flags = cflags;
+	div->table = list->div_table;
+	div->hw.init = &init;
+
+	hw = &div->hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (cflags & CLOCK_FLAG_VAL_INIT) {
+		spin_lock_irqsave(&div->lock, flags);
+		lgm_set_clk_val(div->membase, reg, shift, width, list->div_val);
+		spin_unlock_irqrestore(&div->lock, flags);
+	}
+
+	return hw;
+}
+
+static struct clk_hw *
+lgm_clk_register_fixed_factor(struct lgm_clk_provider *ctx,
+			      const struct lgm_clk_branch *list)
+{
+	unsigned long flags;
+	struct clk_hw *hw;
+
+	hw = clk_hw_register_fixed_factor(ctx->dev, list->name,
+					  list->parent_data[0].name, list->flags,
+					  list->mult, list->div);
+	if (IS_ERR(hw))
+		return ERR_CAST(hw);
+
+	if (list->div_flags & CLOCK_FLAG_VAL_INIT) {
+		spin_lock_irqsave(&ctx->lock, flags);
+		lgm_set_clk_val(ctx->membase, list->div_off, list->div_shift,
+				list->div_width, list->div_val);
+		spin_unlock_irqrestore(&ctx->lock, flags);
+	}
+
+	return hw;
+}
+
+static int lgm_clk_gate_enable(struct clk_hw *hw)
+{
+	struct lgm_clk_gate *gate = to_lgm_clk_gate(hw);
+	unsigned long flags;
+	unsigned int reg;
+
+	spin_lock_irqsave(&gate->lock, flags);
+	reg = GATE_HW_REG_EN(gate->reg);
+	lgm_set_clk_val(gate->membase, reg, gate->shift, 1, 1);
+	spin_unlock_irqrestore(&gate->lock, flags);
+
+	return 0;
+}
+
+static void lgm_clk_gate_disable(struct clk_hw *hw)
+{
+	struct lgm_clk_gate *gate = to_lgm_clk_gate(hw);
+	unsigned long flags;
+	unsigned int reg;
+
+	spin_lock_irqsave(&gate->lock, flags);
+	reg = GATE_HW_REG_DIS(gate->reg);
+	lgm_set_clk_val(gate->membase, reg, gate->shift, 1, 1);
+	spin_unlock_irqrestore(&gate->lock, flags);
+}
+
+static int lgm_clk_gate_is_enabled(struct clk_hw *hw)
+{
+	struct lgm_clk_gate *gate = to_lgm_clk_gate(hw);
+	unsigned int reg, ret;
+	unsigned long flags;
+
+	spin_lock_irqsave(&gate->lock, flags);
+	reg = GATE_HW_REG_STAT(gate->reg);
+	ret = lgm_get_clk_val(gate->membase, reg, gate->shift, 1);
+	spin_unlock_irqrestore(&gate->lock, flags);
+
+	return ret;
+}
+
+static const struct clk_ops lgm_clk_gate_ops = {
+	.enable = lgm_clk_gate_enable,
+	.disable = lgm_clk_gate_disable,
+	.is_enabled = lgm_clk_gate_is_enabled,
+};
+
+static struct clk_hw *
+lgm_clk_register_gate(struct lgm_clk_provider *ctx,
+		      const struct lgm_clk_branch *list)
+{
+	unsigned long flags, cflags = list->gate_flags;
+	const char *pname = list->parent_data[0].name;
+	struct device *dev = ctx->dev;
+	u8 shift = list->gate_shift;
+	struct clk_init_data init = {};
+	struct lgm_clk_gate *gate;
+	u32 reg = list->gate_off;
+	struct clk_hw *hw;
+	int ret;
+
+	gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL);
+	if (!gate)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = list->name;
+	init.ops = &lgm_clk_gate_ops;
+	init.flags = list->flags;
+	init.parent_names = pname ? &pname : NULL;
+	init.num_parents = pname ? 1 : 0;
+
+	gate->membase = ctx->membase;
+	gate->lock = ctx->lock;
+	gate->reg = reg;
+	gate->shift = shift;
+	gate->flags = cflags;
+	gate->hw.init = &init;
+
+	hw = &gate->hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (cflags & CLOCK_FLAG_VAL_INIT) {
+		spin_lock_irqsave(&gate->lock, flags);
+		lgm_set_clk_val(gate->membase, reg, shift, 1, list->gate_val);
+		spin_unlock_irqrestore(&gate->lock, flags);
+	}
+
+	return hw;
+}
+
+int lgm_clk_register_branches(struct lgm_clk_provider *ctx,
+			      const struct lgm_clk_branch *list,
+			      unsigned int nr_clk)
+{
+	struct clk_hw *hw;
+	unsigned int idx;
+
+	for (idx = 0; idx < nr_clk; idx++, list++) {
+		switch (list->type) {
+		case CLK_TYPE_FIXED:
+			hw = lgm_clk_register_fixed(ctx, list);
+			break;
+		case CLK_TYPE_MUX:
+			hw = lgm_clk_register_mux(ctx, list);
+			break;
+		case CLK_TYPE_DIVIDER:
+			hw = lgm_clk_register_divider(ctx, list);
+			break;
+		case CLK_TYPE_FIXED_FACTOR:
+			hw = lgm_clk_register_fixed_factor(ctx, list);
+			break;
+		case CLK_TYPE_GATE:
+			hw = lgm_clk_register_gate(ctx, list);
+			break;
+		default:
+			dev_err(ctx->dev, "invalid clk type\n");
+			return -EINVAL;
+		}
+
+		if (IS_ERR(hw)) {
+			dev_err(ctx->dev,
+				"register clk: %s, type: %u failed!\n",
+				list->name, list->type);
+			return -EIO;
+		}
+		ctx->clk_data.hws[list->id] = hw;
+	}
+
+	return 0;
+}
+
+static unsigned long
+lgm_clk_ddiv_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct lgm_clk_ddiv *ddiv = to_lgm_clk_ddiv(hw);
+	unsigned int div0, div1, exdiv;
+	unsigned long flags;
+	u64 prate;
+
+	spin_lock_irqsave(&ddiv->lock, flags);
+	div0 = lgm_get_clk_val(ddiv->membase, ddiv->reg,
+			       ddiv->shift0, ddiv->width0) + 1;
+	div1 = lgm_get_clk_val(ddiv->membase, ddiv->reg,
+			       ddiv->shift1, ddiv->width1) + 1;
+	exdiv = lgm_get_clk_val(ddiv->membase, ddiv->reg,
+				ddiv->shift2, ddiv->width2);
+	spin_unlock_irqrestore(&ddiv->lock, flags);
+
+	prate = (u64)parent_rate;
+	do_div(prate, div0);
+	do_div(prate, div1);
+
+	if (exdiv) {
+		do_div(prate, ddiv->div);
+		prate *= ddiv->mult;
+	}
+
+	return prate;
+}
+
+static int lgm_clk_ddiv_enable(struct clk_hw *hw)
+{
+	struct lgm_clk_ddiv *ddiv = to_lgm_clk_ddiv(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&ddiv->lock, flags);
+	lgm_set_clk_val(ddiv->membase, ddiv->reg, ddiv->shift_gate,
+			ddiv->width_gate, 1);
+	spin_unlock_irqrestore(&ddiv->lock, flags);
+	return 0;
+}
+
+static void lgm_clk_ddiv_disable(struct clk_hw *hw)
+{
+	struct lgm_clk_ddiv *ddiv = to_lgm_clk_ddiv(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&ddiv->lock, flags);
+	lgm_set_clk_val(ddiv->membase, ddiv->reg, ddiv->shift_gate,
+			ddiv->width_gate, 0);
+	spin_unlock_irqrestore(&ddiv->lock, flags);
+}
+
+static int
+lgm_clk_get_ddiv_val(u32 div, u32 *ddiv1, u32 *ddiv2)
+{
+	u32 idx, temp;
+
+	*ddiv1 = 1;
+	*ddiv2 = 1;
+
+	if (div > MAX_DIVIDER_VAL)
+		div = MAX_DIVIDER_VAL;
+
+	if (div > 1) {
+		for (idx = 2; idx <= MAX_DDIV_REG; idx++) {
+			temp = DIV_ROUND_UP_ULL((u64)div, idx);
+			if (div % idx == 0 && temp <= MAX_DDIV_REG)
+				break;
+		}
+
+		if (idx > MAX_DDIV_REG)
+			return -EINVAL;
+
+		*ddiv1 = temp;
+		*ddiv2 = idx;
+	}
+
+	return 0;
+}
+
+static int
+lgm_clk_ddiv_set_rate(struct clk_hw *hw, unsigned long rate,
+		      unsigned long prate)
+{
+	struct lgm_clk_ddiv *ddiv = to_lgm_clk_ddiv(hw);
+	u32 div, ddiv1, ddiv2;
+	unsigned long flags;
+
+	div = DIV_ROUND_CLOSEST_ULL((u64)prate, rate);
+
+	spin_lock_irqsave(&ddiv->lock, flags);
+	if (lgm_get_clk_val(ddiv->membase, ddiv->reg, ddiv->shift2, 1)) {
+		div = DIV_ROUND_CLOSEST_ULL((u64)div, 5);
+		div = div * 2;
+	}
+
+	if (div <= 0) {
+		spin_unlock_irqrestore(&ddiv->lock, flags);
+		return -EINVAL;
+	}
+
+	if (lgm_clk_get_ddiv_val(div, &ddiv1, &ddiv2)) {
+		spin_unlock_irqrestore(&ddiv->lock, flags);
+		return -EINVAL;
+	}
+
+	lgm_set_clk_val(ddiv->membase, ddiv->reg, ddiv->shift0, ddiv->width0,
+			ddiv1 - 1);
+
+	lgm_set_clk_val(ddiv->membase, ddiv->reg,  ddiv->shift1, ddiv->width1,
+			ddiv2 - 1);
+	spin_unlock_irqrestore(&ddiv->lock, flags);
+
+	return 0;
+}
+
+static long
+lgm_clk_ddiv_round_rate(struct clk_hw *hw, unsigned long rate,
+			unsigned long *prate)
+{
+	struct lgm_clk_ddiv *ddiv = to_lgm_clk_ddiv(hw);
+	u32 div, ddiv1, ddiv2;
+	unsigned long flags;
+	u64 rate64;
+
+	div = DIV_ROUND_CLOSEST_ULL((u64)*prate, rate);
+
+	/* if predivide bit is enabled, modify div by factor of 2.5 */
+	spin_lock_irqsave(&ddiv->lock, flags);
+	if (lgm_get_clk_val(ddiv->membase, ddiv->reg, ddiv->shift2, 1)) {
+		div = div * 2;
+		div = DIV_ROUND_CLOSEST_ULL((u64)div, 5);
+	}
+
+	if (div <= 0) {
+		spin_unlock_irqrestore(&ddiv->lock, flags);
+		return *prate;
+	}
+
+	if (lgm_clk_get_ddiv_val(div, &ddiv1, &ddiv2) != 0) {
+		if (lgm_clk_get_ddiv_val(div + 1, &ddiv1, &ddiv2) != 0) {
+			spin_unlock_irqrestore(&ddiv->lock, flags);
+			return -EINVAL;
+		}
+	}
+
+	rate64 = *prate;
+	do_div(rate64, ddiv1);
+	do_div(rate64, ddiv2);
+
+	/* if predivide bit is enabled, modify rounded rate by factor of 2.5 */
+	if (lgm_get_clk_val(ddiv->membase, ddiv->reg, ddiv->shift2, 1)) {
+		rate64 = rate64 * 2;
+		rate64 = DIV_ROUND_CLOSEST_ULL(rate64, 5);
+	}
+	spin_unlock_irqrestore(&ddiv->lock, flags);
+
+	return rate64;
+}
+
+static const struct clk_ops lgm_clk_ddiv_ops = {
+	.recalc_rate = lgm_clk_ddiv_recalc_rate,
+	.enable	= lgm_clk_ddiv_enable,
+	.disable = lgm_clk_ddiv_disable,
+	.set_rate = lgm_clk_ddiv_set_rate,
+	.round_rate = lgm_clk_ddiv_round_rate,
+};
+
+int lgm_clk_register_ddiv(struct lgm_clk_provider *ctx,
+			  const struct lgm_clk_ddiv_data *list,
+			  unsigned int nr_clk)
+{
+	struct device *dev = ctx->dev;
+	struct clk_init_data init = {};
+	struct lgm_clk_ddiv *ddiv;
+	struct clk_hw *hw;
+	unsigned int idx;
+	int ret;
+
+	for (idx = 0; idx < nr_clk; idx++, list++) {
+		ddiv = NULL;
+		ddiv = devm_kzalloc(dev, sizeof(*ddiv), GFP_KERNEL);
+		if (!ddiv)
+			return -ENOMEM;
+
+		memset(&init, 0, sizeof(init));
+		init.name = list->name;
+		init.ops = &lgm_clk_ddiv_ops;
+		init.flags = list->flags;
+		init.parent_data = list->parent_data;
+		init.num_parents = 1;
+
+		ddiv->membase = ctx->membase;
+		ddiv->lock = ctx->lock;
+		ddiv->reg = list->reg;
+		ddiv->shift0 = list->shift0;
+		ddiv->width0 = list->width0;
+		ddiv->shift1 = list->shift1;
+		ddiv->width1 = list->width1;
+		ddiv->shift_gate = list->shift_gate;
+		ddiv->width_gate = list->width_gate;
+		ddiv->shift2 = list->ex_shift;
+		ddiv->width2 = list->ex_width;
+		ddiv->flags = list->div_flags;
+		ddiv->mult = 2;
+		ddiv->div = 5;
+		ddiv->hw.init = &init;
+
+		hw = &ddiv->hw;
+		ret = clk_hw_register(dev, hw);
+		if (ret) {
+			dev_err(dev, "register clk: %s failed!\n", list->name);
+			return ret;
+		}
+		ctx->clk_data.hws[list->id] = hw;
+	}
+
+	return 0;
+}
diff --git a/drivers/clk/x86/clk-cgu.h b/drivers/clk/x86/clk-cgu.h
new file mode 100644
index 000000000000..4e22bfb22312
--- /dev/null
+++ b/drivers/clk/x86/clk-cgu.h
@@ -0,0 +1,335 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright(c) 2020 Intel Corporation.
+ * Zhu YiXin <yixin.zhu@intel.com>
+ * Rahul Tanwar <rahul.tanwar@intel.com>
+ */
+
+#ifndef __CLK_CGU_H
+#define __CLK_CGU_H
+
+#include <linux/io.h>
+
+struct lgm_clk_mux {
+	struct clk_hw hw;
+	void __iomem *membase;
+	unsigned int reg;
+	u8 shift;
+	u8 width;
+	unsigned long flags;
+	spinlock_t lock;
+};
+
+struct lgm_clk_divider {
+	struct clk_hw hw;
+	void __iomem *membase;
+	unsigned int reg;
+	u8 shift;
+	u8 width;
+	u8 shift_gate;
+	u8 width_gate;
+	unsigned long flags;
+	const struct clk_div_table *table;
+	spinlock_t lock;
+};
+
+struct lgm_clk_ddiv {
+	struct clk_hw hw;
+	void __iomem *membase;
+	unsigned int reg;
+	u8 shift0;
+	u8 width0;
+	u8 shift1;
+	u8 width1;
+	u8 shift2;
+	u8 width2;
+	u8 shift_gate;
+	u8 width_gate;
+	unsigned int mult;
+	unsigned int div;
+	unsigned long flags;
+	spinlock_t lock;
+};
+
+struct lgm_clk_gate {
+	struct clk_hw hw;
+	void __iomem *membase;
+	unsigned int reg;
+	u8 shift;
+	unsigned long flags;
+	spinlock_t lock;
+};
+
+enum lgm_clk_type {
+	CLK_TYPE_FIXED,
+	CLK_TYPE_MUX,
+	CLK_TYPE_DIVIDER,
+	CLK_TYPE_FIXED_FACTOR,
+	CLK_TYPE_GATE,
+	CLK_TYPE_NONE,
+};
+
+/**
+ * struct lgm_clk_provider
+ * @membase: IO mem base address for CGU.
+ * @np: device node
+ * @dev: device
+ * @clk_data: array of hw clocks and clk number.
+ */
+struct lgm_clk_provider {
+	void __iomem *membase;
+	struct device_node *np;
+	struct device *dev;
+	struct clk_hw_onecell_data clk_data;
+	spinlock_t lock;
+};
+
+enum pll_type {
+	TYPE_ROPLL,
+	TYPE_LJPLL,
+	TYPE_NONE,
+};
+
+struct lgm_clk_pll {
+	struct clk_hw hw;
+	void __iomem *membase;
+	unsigned int reg;
+	unsigned long flags;
+	enum pll_type type;
+	spinlock_t lock;
+};
+
+/**
+ * struct lgm_pll_clk_data
+ * @id: platform specific id of the clock.
+ * @name: name of this pll clock.
+ * @parent_data: parent clock data.
+ * @num_parents: number of parents.
+ * @flags: optional flags for basic clock.
+ * @type: platform type of pll.
+ * @reg: offset of the register.
+ */
+struct lgm_pll_clk_data {
+	unsigned int id;
+	const char *name;
+	const struct clk_parent_data *parent_data;
+	u8 num_parents;
+	unsigned long flags;
+	enum pll_type type;
+	int reg;
+};
+
+#define LGM_PLL(_id, _name, _pdata, _flags,		\
+		_reg, _type)				\
+	{						\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_data = _pdata,			\
+		.num_parents = ARRAY_SIZE(_pdata),	\
+		.flags = _flags,			\
+		.reg = _reg,				\
+		.type = _type,				\
+	}
+
+struct lgm_clk_ddiv_data {
+	unsigned int id;
+	const char *name;
+	const struct clk_parent_data *parent_data;
+	u8 flags;
+	unsigned long div_flags;
+	unsigned int reg;
+	u8 shift0;
+	u8 width0;
+	u8 shift1;
+	u8 width1;
+	u8 shift_gate;
+	u8 width_gate;
+	u8 ex_shift;
+	u8 ex_width;
+};
+
+#define LGM_DDIV(_id, _name, _pname, _flags, _reg,		\
+		 _shft0, _wdth0, _shft1, _wdth1,		\
+		 _shft_gate, _wdth_gate, _xshft, _df)		\
+	{							\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_data = &(const struct clk_parent_data){	\
+			.fw_name = _pname,			\
+			.name = _pname,				\
+		},						\
+		.flags = _flags,				\
+		.reg = _reg,					\
+		.shift0 = _shft0,				\
+		.width0 = _wdth0,				\
+		.shift1 = _shft1,				\
+		.width1 = _wdth1,				\
+		.shift_gate = _shft_gate,			\
+		.width_gate = _wdth_gate,			\
+		.ex_shift = _xshft,				\
+		.ex_width = 1,					\
+		.div_flags = _df,				\
+	}
+
+struct lgm_clk_branch {
+	unsigned int id;
+	enum lgm_clk_type type;
+	const char *name;
+	const struct clk_parent_data *parent_data;
+	u8 num_parents;
+	unsigned long flags;
+	unsigned int mux_off;
+	u8 mux_shift;
+	u8 mux_width;
+	unsigned long mux_flags;
+	unsigned int mux_val;
+	unsigned int div_off;
+	u8 div_shift;
+	u8 div_width;
+	u8 div_shift_gate;
+	u8 div_width_gate;
+	unsigned long div_flags;
+	unsigned int div_val;
+	const struct clk_div_table *div_table;
+	unsigned int gate_off;
+	u8 gate_shift;
+	unsigned long gate_flags;
+	unsigned int gate_val;
+	unsigned int mult;
+	unsigned int div;
+};
+
+/* clock flags definition */
+#define CLOCK_FLAG_VAL_INIT	BIT(16)
+#define MUX_CLK_SW		BIT(17)
+
+#define LGM_MUX(_id, _name, _pdata, _f, _reg,		\
+		_shift, _width, _cf, _v)		\
+	{						\
+		.id = _id,				\
+		.type = CLK_TYPE_MUX,			\
+		.name = _name,				\
+		.parent_data = _pdata,			\
+		.num_parents = ARRAY_SIZE(_pdata),	\
+		.flags = _f,				\
+		.mux_off = _reg,			\
+		.mux_shift = _shift,			\
+		.mux_width = _width,			\
+		.mux_flags = _cf,			\
+		.mux_val = _v,				\
+	}
+
+#define LGM_DIV(_id, _name, _pname, _f, _reg, _shift, _width,	\
+		_shift_gate, _width_gate, _cf, _v, _dtable)	\
+	{							\
+		.id = _id,					\
+		.type = CLK_TYPE_DIVIDER,			\
+		.name = _name,					\
+		.parent_data = &(const struct clk_parent_data){	\
+			.fw_name = _pname,			\
+			.name = _pname,				\
+		},						\
+		.num_parents = 1,				\
+		.flags = _f,					\
+		.div_off = _reg,				\
+		.div_shift = _shift,				\
+		.div_width = _width,				\
+		.div_shift_gate = _shift_gate,			\
+		.div_width_gate = _width_gate,			\
+		.div_flags = _cf,				\
+		.div_val = _v,					\
+		.div_table = _dtable,				\
+	}
+
+#define LGM_GATE(_id, _name, _pname, _f, _reg,			\
+		 _shift, _cf, _v)				\
+	{							\
+		.id = _id,					\
+		.type = CLK_TYPE_GATE,				\
+		.name = _name,					\
+		.parent_data = &(const struct clk_parent_data){	\
+			.fw_name = _pname,			\
+			.name = _pname,				\
+		},						\
+		.num_parents = !_pname ? 0 : 1,			\
+		.flags = _f,					\
+		.gate_off = _reg,				\
+		.gate_shift = _shift,				\
+		.gate_flags = _cf,				\
+		.gate_val = _v,					\
+	}
+
+#define LGM_FIXED(_id, _name, _pname, _f, _reg,			\
+		  _shift, _width, _cf, _freq, _v)		\
+	{							\
+		.id = _id,					\
+		.type = CLK_TYPE_FIXED,				\
+		.name = _name,					\
+		.parent_data = &(const struct clk_parent_data){	\
+			.fw_name = _pname,			\
+			.name = _pname,				\
+		},						\
+		.num_parents = !_pname ? 0 : 1,			\
+		.flags = _f,					\
+		.div_off = _reg,				\
+		.div_shift = _shift,				\
+		.div_width = _width,				\
+		.div_flags = _cf,				\
+		.div_val = _v,					\
+		.mux_flags = _freq,				\
+	}
+
+#define LGM_FIXED_FACTOR(_id, _name, _pname, _f, _reg,		\
+			 _shift, _width, _cf, _v, _m, _d)	\
+	{							\
+		.id = _id,					\
+		.type = CLK_TYPE_FIXED_FACTOR,			\
+		.name = _name,					\
+		.parent_data = &(const struct clk_parent_data){	\
+			.fw_name = _pname,			\
+			.name = _pname,				\
+		},						\
+		.num_parents = 1,				\
+		.flags = _f,					\
+		.div_off = _reg,				\
+		.div_shift = _shift,				\
+		.div_width = _width,				\
+		.div_flags = _cf,				\
+		.div_val = _v,					\
+		.mult = _m,					\
+		.div = _d,					\
+	}
+
+static inline void lgm_set_clk_val(void __iomem *membase, u32 reg,
+				   u8 shift, u8 width, u32 set_val)
+{
+	u32 mask = (GENMASK(width - 1, 0) << shift);
+	u32 regval;
+
+	regval = readl(membase + reg);
+	regval = (regval & ~mask) | ((set_val << shift) & mask);
+	writel(regval, membase + reg);
+}
+
+static inline u32 lgm_get_clk_val(void __iomem *membase, u32 reg,
+				  u8 shift, u8 width)
+{
+	u32 mask = (GENMASK(width - 1, 0) << shift);
+	u32 val;
+
+	val = readl(membase + reg);
+	val = (val & mask) >> shift;
+
+	return val;
+}
+
+int lgm_clk_register_branches(struct lgm_clk_provider *ctx,
+			      const struct lgm_clk_branch *list,
+			      unsigned int nr_clk);
+int lgm_clk_register_plls(struct lgm_clk_provider *ctx,
+			  const struct lgm_pll_clk_data *list,
+			  unsigned int nr_clk);
+int lgm_clk_register_ddiv(struct lgm_clk_provider *ctx,
+			  const struct lgm_clk_ddiv_data *list,
+			  unsigned int nr_clk);
+#endif	/* __CLK_CGU_H */
diff --git a/drivers/clk/x86/clk-lgm.c b/drivers/clk/x86/clk-lgm.c
new file mode 100644
index 000000000000..020f4e83a5cc
--- /dev/null
+++ b/drivers/clk/x86/clk-lgm.c
@@ -0,0 +1,475 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Intel Corporation.
+ * Zhu YiXin <yixin.zhu@intel.com>
+ * Rahul Tanwar <rahul.tanwar@intel.com>
+ */
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/clock/intel,lgm-clk.h>
+#include "clk-cgu.h"
+
+#define PLL_DIV_WIDTH		4
+#define PLL_DDIV_WIDTH		3
+
+/* Gate0 clock shift */
+#define G_C55_SHIFT		7
+#define G_QSPI_SHIFT		9
+#define G_EIP197_SHIFT		11
+#define G_VAULT130_SHIFT	12
+#define G_TOE_SHIFT		13
+#define G_SDXC_SHIFT		14
+#define G_EMMC_SHIFT		15
+#define G_SPIDBG_SHIFT		17
+#define G_DMA3_SHIFT		28
+
+/* Gate1 clock shift */
+#define G_DMA0_SHIFT		0
+#define G_LEDC0_SHIFT		1
+#define G_LEDC1_SHIFT		2
+#define G_I2S0_SHIFT		3
+#define G_I2S1_SHIFT		4
+#define G_EBU_SHIFT		5
+#define G_PWM_SHIFT		6
+#define G_I2C0_SHIFT		7
+#define G_I2C1_SHIFT		8
+#define G_I2C2_SHIFT		9
+#define G_I2C3_SHIFT		10
+
+#define G_SSC0_SHIFT		12
+#define G_SSC1_SHIFT		13
+#define G_SSC2_SHIFT		14
+#define G_SSC3_SHIFT		15
+
+#define G_GPTC0_SHIFT		17
+#define G_GPTC1_SHIFT		18
+#define G_GPTC2_SHIFT		19
+#define G_GPTC3_SHIFT		20
+
+#define G_ASC0_SHIFT		22
+#define G_ASC1_SHIFT		23
+#define G_ASC2_SHIFT		24
+#define G_ASC3_SHIFT		25
+
+#define G_PCM0_SHIFT		27
+#define G_PCM1_SHIFT		28
+#define G_PCM2_SHIFT		29
+
+/* Gate2 clock shift */
+#define G_PCIE10_SHIFT		1
+#define G_PCIE11_SHIFT		2
+#define G_PCIE30_SHIFT		3
+#define G_PCIE31_SHIFT		4
+#define G_PCIE20_SHIFT		5
+#define G_PCIE21_SHIFT		6
+#define G_PCIE40_SHIFT		7
+#define G_PCIE41_SHIFT		8
+
+#define G_XPCS0_SHIFT		10
+#define G_XPCS1_SHIFT		11
+#define G_XPCS2_SHIFT		12
+#define G_XPCS3_SHIFT		13
+#define G_SATA0_SHIFT		14
+#define G_SATA1_SHIFT		15
+#define G_SATA2_SHIFT		16
+#define G_SATA3_SHIFT		17
+
+/* Gate3 clock shift */
+#define G_ARCEM4_SHIFT		0
+#define G_IDMAR1_SHIFT		2
+#define G_IDMAT0_SHIFT		3
+#define G_IDMAT1_SHIFT		4
+#define G_IDMAT2_SHIFT		5
+
+#define G_PPV4_SHIFT		8
+#define G_GSWIPO_SHIFT		9
+#define G_CQEM_SHIFT		10
+#define G_XPCS5_SHIFT		14
+#define G_USB1_SHIFT		25
+#define G_USB2_SHIFT		26
+
+
+/* Register definition */
+#define CGU_PLL0CZ_CFG0		0x000
+#define CGU_PLL0CM0_CFG0	0x020
+#define CGU_PLL0CM1_CFG0	0x040
+#define CGU_PLL0B_CFG0		0x060
+#define CGU_PLL1_CFG0		0x080
+#define CGU_PLL2_CFG0		0x0A0
+#define CGU_PLLPP_CFG0		0x0C0
+#define CGU_LJPLL3_CFG0		0x0E0
+#define CGU_LJPLL4_CFG0		0x100
+#define CGU_C55_PCMCR		0x18C
+#define CGU_PCMCR		0x190
+#define CGU_IF_CLK1		0x1A0
+#define CGU_IF_CLK2		0x1A4
+#define CGU_GATE0		0x300
+#define CGU_GATE1		0x310
+#define CGU_GATE2		0x320
+#define CGU_GATE3		0x310
+
+#define PLL_DIV(x)		((x) + 0x04)
+#define PLL_SSC(x)		((x) + 0x10)
+
+#define CLK_NR_CLKS		(LGM_GCLK_USB2 + 1)
+
+/*
+ * Below table defines the pair's of regval & effective dividers.
+ * It's more efficient to provide an explicit table due to non-linear
+ * relation between values.
+ */
+static const struct clk_div_table pll_div[] = {
+	{ .val = 0, .div = 1 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 3 },
+	{ .val = 3, .div = 4 },
+	{ .val = 4, .div = 5 },
+	{ .val = 5, .div = 6 },
+	{ .val = 6, .div = 8 },
+	{ .val = 7, .div = 10 },
+	{ .val = 8, .div = 12 },
+	{ .val = 9, .div = 16 },
+	{ .val = 10, .div = 20 },
+	{ .val = 11, .div = 24 },
+	{ .val = 12, .div = 32 },
+	{ .val = 13, .div = 40 },
+	{ .val = 14, .div = 48 },
+	{ .val = 15, .div = 64 },
+	{}
+};
+
+static const struct clk_div_table dcl_div[] = {
+	{ .val = 0, .div = 6  },
+	{ .val = 1, .div = 12 },
+	{ .val = 2, .div = 24 },
+	{ .val = 3, .div = 32 },
+	{ .val = 4, .div = 48 },
+	{ .val = 5, .div = 96 },
+	{}
+};
+
+static const struct clk_parent_data pll_p[] = {
+	{ .fw_name = "osc", .name = "osc" },
+};
+static const struct clk_parent_data pllcm_p[] = {
+	{ .fw_name = "cpu_cm", .name = "cpu_cm" },
+};
+static const struct clk_parent_data emmc_p[] = {
+	{ .fw_name = "emmc4", .name = "emmc4" },
+	{ .fw_name = "noc4", .name = "noc4" },
+};
+static const struct clk_parent_data sdxc_p[] = {
+	{ .fw_name = "sdxc3", .name = "sdxc3" },
+	{ .fw_name = "sdxc2", .name = "sdxc2" },
+};
+static const struct clk_parent_data pcm_p[] = {
+	{ .fw_name = "v_docsis", .name = "v_docsis" },
+	{ .fw_name = "dcl", .name = "dcl" },
+};
+static const struct clk_parent_data cbphy_p[] = {
+	{ .fw_name = "dd_serdes", .name = "dd_serdes" },
+	{ .fw_name = "dd_pcie", .name = "dd_pcie" },
+};
+
+static const struct lgm_pll_clk_data lgm_pll_clks[] = {
+	LGM_PLL(LGM_CLK_PLL0CZ, "pll0cz", pll_p, CLK_IGNORE_UNUSED,
+		CGU_PLL0CZ_CFG0, TYPE_ROPLL),
+	LGM_PLL(LGM_CLK_PLL0CM0, "pll0cm0", pllcm_p, CLK_IGNORE_UNUSED,
+		CGU_PLL0CM0_CFG0, TYPE_ROPLL),
+	LGM_PLL(LGM_CLK_PLL0CM1, "pll0cm1", pllcm_p, CLK_IGNORE_UNUSED,
+		CGU_PLL0CM1_CFG0, TYPE_ROPLL),
+	LGM_PLL(LGM_CLK_PLL0B, "pll0b", pll_p, CLK_IGNORE_UNUSED,
+		CGU_PLL0B_CFG0, TYPE_ROPLL),
+	LGM_PLL(LGM_CLK_PLL1, "pll1", pll_p, 0, CGU_PLL1_CFG0, TYPE_ROPLL),
+	LGM_PLL(LGM_CLK_PLL2, "pll2", pll_p, CLK_IGNORE_UNUSED,
+		CGU_PLL2_CFG0, TYPE_ROPLL),
+	LGM_PLL(LGM_CLK_PLLPP, "pllpp", pll_p, 0, CGU_PLLPP_CFG0, TYPE_ROPLL),
+	LGM_PLL(LGM_CLK_LJPLL3, "ljpll3", pll_p, 0, CGU_LJPLL3_CFG0, TYPE_LJPLL),
+	LGM_PLL(LGM_CLK_LJPLL4, "ljpll4", pll_p, 0, CGU_LJPLL4_CFG0, TYPE_LJPLL),
+};
+
+static const struct lgm_clk_branch lgm_branch_clks[] = {
+	LGM_DIV(LGM_CLK_PP_HW, "pp_hw", "pllpp", 0, PLL_DIV(CGU_PLLPP_CFG0),
+		0, PLL_DIV_WIDTH, 24, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_PP_UC, "pp_uc", "pllpp", 0, PLL_DIV(CGU_PLLPP_CFG0),
+		4, PLL_DIV_WIDTH, 25, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_PP_FXD, "pp_fxd", "pllpp", 0, PLL_DIV(CGU_PLLPP_CFG0),
+		8, PLL_DIV_WIDTH, 26, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_PP_TBM, "pp_tbm", "pllpp", 0, PLL_DIV(CGU_PLLPP_CFG0),
+		12, PLL_DIV_WIDTH, 27, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_DDR, "ddr", "pll2", CLK_IGNORE_UNUSED,
+		PLL_DIV(CGU_PLL2_CFG0), 0, PLL_DIV_WIDTH, 24, 1, 0, 0,
+		pll_div),
+	LGM_DIV(LGM_CLK_CM, "cpu_cm", "pll0cz", 0, PLL_DIV(CGU_PLL0CZ_CFG0),
+		0, PLL_DIV_WIDTH, 24, 1, 0, 0, pll_div),
+
+	LGM_DIV(LGM_CLK_IC, "cpu_ic", "pll0cz", CLK_IGNORE_UNUSED,
+		PLL_DIV(CGU_PLL0CZ_CFG0), 4, PLL_DIV_WIDTH, 25,
+		1, 0, 0, pll_div),
+
+	LGM_DIV(LGM_CLK_SDXC3, "sdxc3", "pll0cz", 0, PLL_DIV(CGU_PLL0CZ_CFG0),
+		8, PLL_DIV_WIDTH, 26, 1, 0, 0, pll_div),
+
+	LGM_DIV(LGM_CLK_CPU0, "cm0", "pll0cm0",
+		CLK_IGNORE_UNUSED, PLL_DIV(CGU_PLL0CM0_CFG0),
+		0, PLL_DIV_WIDTH, 24, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_CPU1, "cm1", "pll0cm1",
+		CLK_IGNORE_UNUSED, PLL_DIV(CGU_PLL0CM1_CFG0),
+		0, PLL_DIV_WIDTH, 24, 1, 0, 0, pll_div),
+
+	/*
+	 * Marking ngi_clk (next generation interconnect) and noc_clk
+	 * (network on chip peripheral clk) as critical clocks because
+	 * these are shared parent clock sources for many different
+	 * peripherals.
+	 */
+	LGM_DIV(LGM_CLK_NGI, "ngi", "pll0b",
+		(CLK_IGNORE_UNUSED|CLK_IS_CRITICAL), PLL_DIV(CGU_PLL0B_CFG0),
+		0, PLL_DIV_WIDTH, 24, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_NOC4, "noc4", "pll0b",
+		(CLK_IGNORE_UNUSED|CLK_IS_CRITICAL), PLL_DIV(CGU_PLL0B_CFG0),
+		4, PLL_DIV_WIDTH, 25, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_SW, "switch", "pll0b", 0, PLL_DIV(CGU_PLL0B_CFG0),
+		8, PLL_DIV_WIDTH, 26, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_QSPI, "qspi", "pll0b", 0, PLL_DIV(CGU_PLL0B_CFG0),
+		12, PLL_DIV_WIDTH, 27, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_CT, "v_ct", "pll1", 0, PLL_DIV(CGU_PLL1_CFG0),
+		0, PLL_DIV_WIDTH, 24, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_DSP, "v_dsp", "pll1", 0, PLL_DIV(CGU_PLL1_CFG0),
+		8, PLL_DIV_WIDTH, 26, 1, 0, 0, pll_div),
+	LGM_DIV(LGM_CLK_VIF, "v_ifclk", "pll1", 0, PLL_DIV(CGU_PLL1_CFG0),
+		12, PLL_DIV_WIDTH, 27, 1, 0, 0, pll_div),
+
+	LGM_FIXED_FACTOR(LGM_CLK_EMMC4, "emmc4", "sdxc3", 0,  0,
+			 0, 0, 0, 0, 1, 4),
+	LGM_FIXED_FACTOR(LGM_CLK_SDXC2, "sdxc2", "noc4", 0,  0,
+			 0, 0, 0, 0, 1, 4),
+	LGM_MUX(LGM_CLK_EMMC, "emmc", emmc_p, 0, CGU_IF_CLK1,
+		0, 1, CLK_MUX_ROUND_CLOSEST, 0),
+	LGM_MUX(LGM_CLK_SDXC, "sdxc", sdxc_p, 0, CGU_IF_CLK1,
+		1, 1, CLK_MUX_ROUND_CLOSEST, 0),
+	LGM_FIXED(LGM_CLK_OSC, "osc", NULL, 0, 0, 0, 0, 0, 40000000, 0),
+	LGM_FIXED(LGM_CLK_SLIC, "slic", NULL, 0, CGU_IF_CLK1,
+		  8, 2, CLOCK_FLAG_VAL_INIT, 8192000, 2),
+	LGM_FIXED(LGM_CLK_DOCSIS, "v_docsis", NULL, 0, 0, 0, 0, 0, 16000000, 0),
+	LGM_DIV(LGM_CLK_DCL, "dcl", "v_ifclk", 0, CGU_PCMCR,
+		25, 3, 0, 0, 0, 0, dcl_div),
+	LGM_MUX(LGM_CLK_PCM, "pcm", pcm_p, 0, CGU_C55_PCMCR,
+		0, 1, CLK_MUX_ROUND_CLOSEST, 0),
+	LGM_FIXED_FACTOR(LGM_CLK_DDR_PHY, "ddr_phy", "ddr",
+			 CLK_IGNORE_UNUSED, 0,
+			 0, 0, 0, 0, 2, 1),
+	LGM_FIXED_FACTOR(LGM_CLK_PONDEF, "pondef", "dd_pool",
+			 CLK_SET_RATE_PARENT, 0, 0, 0, 0, 0, 1, 2),
+	LGM_MUX(LGM_CLK_CBPHY0, "cbphy0", cbphy_p, 0, 0,
+		0, 0, MUX_CLK_SW | CLK_MUX_ROUND_CLOSEST, 0),
+	LGM_MUX(LGM_CLK_CBPHY1, "cbphy1", cbphy_p, 0, 0,
+		0, 0, MUX_CLK_SW | CLK_MUX_ROUND_CLOSEST, 0),
+	LGM_MUX(LGM_CLK_CBPHY2, "cbphy2", cbphy_p, 0, 0,
+		0, 0, MUX_CLK_SW | CLK_MUX_ROUND_CLOSEST, 0),
+	LGM_MUX(LGM_CLK_CBPHY3, "cbphy3", cbphy_p, 0, 0,
+		0, 0, MUX_CLK_SW | CLK_MUX_ROUND_CLOSEST, 0),
+
+	LGM_GATE(LGM_GCLK_C55, "g_c55", NULL, 0, CGU_GATE0,
+		 G_C55_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_QSPI, "g_qspi", "qspi", 0, CGU_GATE0,
+		 G_QSPI_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_EIP197, "g_eip197", NULL, 0, CGU_GATE0,
+		 G_EIP197_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_VAULT, "g_vault130", NULL, 0, CGU_GATE0,
+		 G_VAULT130_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_TOE, "g_toe", NULL, 0, CGU_GATE0,
+		 G_TOE_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SDXC, "g_sdxc", "sdxc", 0, CGU_GATE0,
+		 G_SDXC_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_EMMC, "g_emmc", "emmc", 0, CGU_GATE0,
+		 G_EMMC_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SPI_DBG, "g_spidbg", NULL, 0, CGU_GATE0,
+		 G_SPIDBG_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_DMA3, "g_dma3", NULL, 0, CGU_GATE0,
+		 G_DMA3_SHIFT, 0, 0),
+
+	LGM_GATE(LGM_GCLK_DMA0, "g_dma0", NULL, 0, CGU_GATE1,
+		 G_DMA0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_LEDC0, "g_ledc0", NULL, 0, CGU_GATE1,
+		 G_LEDC0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_LEDC1, "g_ledc1", NULL, 0, CGU_GATE1,
+		 G_LEDC1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_I2S0, "g_i2s0", NULL, 0, CGU_GATE1,
+		 G_I2S0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_I2S1, "g_i2s1", NULL, 0, CGU_GATE1,
+		 G_I2S1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_EBU, "g_ebu", NULL, 0, CGU_GATE1,
+		 G_EBU_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PWM, "g_pwm", NULL, 0, CGU_GATE1,
+		 G_PWM_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_I2C0, "g_i2c0", NULL, 0, CGU_GATE1,
+		 G_I2C0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_I2C1, "g_i2c1", NULL, 0, CGU_GATE1,
+		 G_I2C1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_I2C2, "g_i2c2", NULL, 0, CGU_GATE1,
+		 G_I2C2_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_I2C3, "g_i2c3", NULL, 0, CGU_GATE1,
+		 G_I2C3_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SSC0, "g_ssc0", "noc4", 0, CGU_GATE1,
+		 G_SSC0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SSC1, "g_ssc1", "noc4", 0, CGU_GATE1,
+		 G_SSC1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SSC2, "g_ssc2", "noc4", 0, CGU_GATE1,
+		 G_SSC2_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SSC3, "g_ssc3", "noc4", 0, CGU_GATE1,
+		 G_SSC3_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_GPTC0, "g_gptc0", "noc4", 0, CGU_GATE1,
+		 G_GPTC0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_GPTC1, "g_gptc1", "noc4", 0, CGU_GATE1,
+		 G_GPTC1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_GPTC2, "g_gptc2", "noc4", 0, CGU_GATE1,
+		 G_GPTC2_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_GPTC3, "g_gptc3", "osc", 0, CGU_GATE1,
+		 G_GPTC3_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_ASC0, "g_asc0", "noc4", 0, CGU_GATE1,
+		 G_ASC0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_ASC1, "g_asc1", "noc4", 0, CGU_GATE1,
+		 G_ASC1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_ASC2, "g_asc2", "noc4", 0, CGU_GATE1,
+		 G_ASC2_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_ASC3, "g_asc3", "osc", 0, CGU_GATE1,
+		 G_ASC3_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCM0, "g_pcm0", NULL, 0, CGU_GATE1,
+		 G_PCM0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCM1, "g_pcm1", NULL, 0, CGU_GATE1,
+		 G_PCM1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCM2, "g_pcm2", NULL, 0, CGU_GATE1,
+		 G_PCM2_SHIFT, 0, 0),
+
+	LGM_GATE(LGM_GCLK_PCIE10, "g_pcie10", NULL, 0, CGU_GATE2,
+		 G_PCIE10_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCIE11, "g_pcie11", NULL, 0, CGU_GATE2,
+		 G_PCIE11_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCIE30, "g_pcie30", NULL, 0, CGU_GATE2,
+		 G_PCIE30_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCIE31, "g_pcie31", NULL, 0, CGU_GATE2,
+		 G_PCIE31_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCIE20, "g_pcie20", NULL, 0, CGU_GATE2,
+		 G_PCIE20_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCIE21, "g_pcie21", NULL, 0, CGU_GATE2,
+		 G_PCIE21_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCIE40, "g_pcie40", NULL, 0, CGU_GATE2,
+		 G_PCIE40_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PCIE41, "g_pcie41", NULL, 0, CGU_GATE2,
+		 G_PCIE41_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_XPCS0, "g_xpcs0", NULL, 0, CGU_GATE2,
+		 G_XPCS0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_XPCS1, "g_xpcs1", NULL, 0, CGU_GATE2,
+		 G_XPCS1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_XPCS2, "g_xpcs2", NULL, 0, CGU_GATE2,
+		 G_XPCS2_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_XPCS3, "g_xpcs3", NULL, 0, CGU_GATE2,
+		 G_XPCS3_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SATA0, "g_sata0", NULL, 0, CGU_GATE2,
+		 G_SATA0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SATA1, "g_sata1", NULL, 0, CGU_GATE2,
+		 G_SATA1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SATA2, "g_sata2", NULL, 0, CGU_GATE2,
+		 G_SATA2_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_SATA3, "g_sata3", NULL, 0, CGU_GATE2,
+		 G_SATA3_SHIFT, 0, 0),
+
+	LGM_GATE(LGM_GCLK_ARCEM4, "g_arcem4", NULL, 0, CGU_GATE3,
+		 G_ARCEM4_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_IDMAR1, "g_idmar1", NULL, 0, CGU_GATE3,
+		 G_IDMAR1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_IDMAT0, "g_idmat0", NULL, 0, CGU_GATE3,
+		 G_IDMAT0_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_IDMAT1, "g_idmat1", NULL, 0, CGU_GATE3,
+		 G_IDMAT1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_IDMAT2, "g_idmat2", NULL, 0, CGU_GATE3,
+		 G_IDMAT2_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_PPV4, "g_ppv4", NULL, 0, CGU_GATE3,
+		 G_PPV4_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_GSWIPO, "g_gswipo", "switch", 0, CGU_GATE3,
+		 G_GSWIPO_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_CQEM, "g_cqem", "switch", 0, CGU_GATE3,
+		 G_CQEM_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_XPCS5, "g_xpcs5", NULL, 0, CGU_GATE3,
+		 G_XPCS5_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_USB1, "g_usb1", NULL, 0, CGU_GATE3,
+		 G_USB1_SHIFT, 0, 0),
+	LGM_GATE(LGM_GCLK_USB2, "g_usb2", NULL, 0, CGU_GATE3,
+		 G_USB2_SHIFT, 0, 0),
+};
+
+
+static const struct lgm_clk_ddiv_data lgm_ddiv_clks[] = {
+	LGM_DDIV(LGM_CLK_CML, "dd_cml", "ljpll3", 0,
+		 PLL_DIV(CGU_LJPLL3_CFG0), 0, PLL_DDIV_WIDTH,
+		 3, PLL_DDIV_WIDTH, 24, 1, 29, 0),
+	LGM_DDIV(LGM_CLK_SERDES, "dd_serdes", "ljpll3", 0,
+		 PLL_DIV(CGU_LJPLL3_CFG0), 6, PLL_DDIV_WIDTH,
+		 9, PLL_DDIV_WIDTH, 25, 1, 28, 0),
+	LGM_DDIV(LGM_CLK_POOL, "dd_pool", "ljpll3", 0,
+		 PLL_DIV(CGU_LJPLL3_CFG0), 12, PLL_DDIV_WIDTH,
+		 15, PLL_DDIV_WIDTH, 26, 1, 28, 0),
+	LGM_DDIV(LGM_CLK_PTP, "dd_ptp", "ljpll3", 0,
+		 PLL_DIV(CGU_LJPLL3_CFG0), 18, PLL_DDIV_WIDTH,
+		 21, PLL_DDIV_WIDTH, 27, 1, 28, 0),
+	LGM_DDIV(LGM_CLK_PCIE, "dd_pcie", "ljpll4", 0,
+		 PLL_DIV(CGU_LJPLL4_CFG0), 0, PLL_DDIV_WIDTH,
+		 3, PLL_DDIV_WIDTH, 24, 1, 29, 0),
+};
+
+static int lgm_cgu_probe(struct platform_device *pdev)
+{
+	struct lgm_clk_provider *ctx;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	ctx = devm_kzalloc(dev, struct_size(ctx, clk_data.hws, CLK_NR_CLKS),
+			   GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->clk_data.num = CLK_NR_CLKS;
+
+	ctx->membase = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ctx->membase))
+		return PTR_ERR(ctx->membase);
+
+	ctx->np = np;
+	ctx->dev = dev;
+	spin_lock_init(&ctx->lock);
+
+	ret = lgm_clk_register_plls(ctx, lgm_pll_clks,
+				    ARRAY_SIZE(lgm_pll_clks));
+	if (ret)
+		return ret;
+
+	ret = lgm_clk_register_branches(ctx, lgm_branch_clks,
+					ARRAY_SIZE(lgm_branch_clks));
+	if (ret)
+		return ret;
+
+	ret = lgm_clk_register_ddiv(ctx, lgm_ddiv_clks,
+				    ARRAY_SIZE(lgm_ddiv_clks));
+	if (ret)
+		return ret;
+
+	return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get,
+					   &ctx->clk_data);
+}
+
+static const struct of_device_id of_lgm_cgu_match[] = {
+	{ .compatible = "intel,cgu-lgm" },
+	{}
+};
+
+static struct platform_driver lgm_cgu_driver = {
+	.probe = lgm_cgu_probe,
+	.driver = {
+		   .name = "cgu-lgm",
+		   .of_match_table = of_lgm_cgu_match,
+	},
+};
+builtin_platform_driver(lgm_cgu_driver);
diff --git a/drivers/clk/zynqmp/clk-gate-zynqmp.c b/drivers/clk/zynqmp/clk-gate-zynqmp.c
index 83b236f20fff..10c9b889324f 100644
--- a/drivers/clk/zynqmp/clk-gate-zynqmp.c
+++ b/drivers/clk/zynqmp/clk-gate-zynqmp.c
@@ -37,9 +37,8 @@ static int zynqmp_clk_gate_enable(struct clk_hw *hw)
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 clk_id = gate->clk_id;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_enable(clk_id);
+	ret = zynqmp_pm_clock_enable(clk_id);
 
 	if (ret)
 		pr_warn_once("%s() clock enabled failed for %s, ret = %d\n",
@@ -58,9 +57,8 @@ static void zynqmp_clk_gate_disable(struct clk_hw *hw)
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 clk_id = gate->clk_id;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_disable(clk_id);
+	ret = zynqmp_pm_clock_disable(clk_id);
 
 	if (ret)
 		pr_warn_once("%s() clock disable failed for %s, ret = %d\n",
@@ -79,9 +77,8 @@ static int zynqmp_clk_gate_is_enabled(struct clk_hw *hw)
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 clk_id = gate->clk_id;
 	int state, ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_getstate(clk_id, &state);
+	ret = zynqmp_pm_clock_getstate(clk_id, &state);
 	if (ret) {
 		pr_warn_once("%s() clock get state failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
diff --git a/drivers/clk/zynqmp/clk-mux-zynqmp.c b/drivers/clk/zynqmp/clk-mux-zynqmp.c
index 0af8f74c5fa5..06194149be83 100644
--- a/drivers/clk/zynqmp/clk-mux-zynqmp.c
+++ b/drivers/clk/zynqmp/clk-mux-zynqmp.c
@@ -47,9 +47,8 @@ static u8 zynqmp_clk_mux_get_parent(struct clk_hw *hw)
 	u32 clk_id = mux->clk_id;
 	u32 val;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_getparent(clk_id, &val);
+	ret = zynqmp_pm_clock_getparent(clk_id, &val);
 
 	if (ret)
 		pr_warn_once("%s() getparent failed for clock: %s, ret = %d\n",
@@ -71,9 +70,8 @@ static int zynqmp_clk_mux_set_parent(struct clk_hw *hw, u8 index)
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 clk_id = mux->clk_id;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_setparent(clk_id, index);
+	ret = zynqmp_pm_clock_setparent(clk_id, index);
 
 	if (ret)
 		pr_warn_once("%s() set parent failed for clock: %s, ret = %d\n",
diff --git a/drivers/clk/zynqmp/clk-zynqmp.h b/drivers/clk/zynqmp/clk-zynqmp.h
index fec9a15c8786..5beeb41b29fa 100644
--- a/drivers/clk/zynqmp/clk-zynqmp.h
+++ b/drivers/clk/zynqmp/clk-zynqmp.h
@@ -30,6 +30,7 @@ struct clock_topology {
 	u32 type;
 	u32 flag;
 	u32 type_flag;
+	u8 custom_type_flag;
 };
 
 struct clk_hw *zynqmp_clk_register_pll(const char *name, u32 clk_id,
diff --git a/drivers/clk/zynqmp/clkc.c b/drivers/clk/zynqmp/clkc.c
index 10e89f23880b..db8d0d7161ce 100644
--- a/drivers/clk/zynqmp/clkc.c
+++ b/drivers/clk/zynqmp/clkc.c
@@ -84,6 +84,7 @@ struct name_resp {
 
 struct topology_resp {
 #define CLK_TOPOLOGY_TYPE		GENMASK(3, 0)
+#define CLK_TOPOLOGY_CUSTOM_TYPE_FLAGS	GENMASK(7, 4)
 #define CLK_TOPOLOGY_FLAGS		GENMASK(23, 8)
 #define CLK_TOPOLOGY_TYPE_FLAGS		GENMASK(31, 24)
 	u32 topology[CLK_GET_TOPOLOGY_RESP_WORDS];
@@ -134,7 +135,6 @@ static struct clk_hw *(* const clk_topology[]) (const char *name, u32 clk_id,
 static struct zynqmp_clock *clock;
 static struct clk_hw_onecell_data *zynqmp_data;
 static unsigned int clock_max_idx;
-static const struct zynqmp_eemi_ops *eemi_ops;
 
 /**
  * zynqmp_is_valid_clock() - Check whether clock is valid or not
@@ -206,7 +206,7 @@ static int zynqmp_pm_clock_get_num_clocks(u32 *nclocks)
 
 	qdata.qid = PM_QID_CLOCK_GET_NUM_CLOCKS;
 
-	ret = eemi_ops->query_data(qdata, ret_payload);
+	ret = zynqmp_pm_query_data(qdata, ret_payload);
 	*nclocks = ret_payload[1];
 
 	return ret;
@@ -231,7 +231,7 @@ static int zynqmp_pm_clock_get_name(u32 clock_id,
 	qdata.qid = PM_QID_CLOCK_GET_NAME;
 	qdata.arg1 = clock_id;
 
-	eemi_ops->query_data(qdata, ret_payload);
+	zynqmp_pm_query_data(qdata, ret_payload);
 	memcpy(response, ret_payload, sizeof(*response));
 
 	return 0;
@@ -265,7 +265,7 @@ static int zynqmp_pm_clock_get_topology(u32 clock_id, u32 index,
 	qdata.arg1 = clock_id;
 	qdata.arg2 = index;
 
-	ret = eemi_ops->query_data(qdata, ret_payload);
+	ret = zynqmp_pm_query_data(qdata, ret_payload);
 	memcpy(response, &ret_payload[1], sizeof(*response));
 
 	return ret;
@@ -296,7 +296,7 @@ struct clk_hw *zynqmp_clk_register_fixed_factor(const char *name, u32 clk_id,
 	qdata.qid = PM_QID_CLOCK_GET_FIXEDFACTOR_PARAMS;
 	qdata.arg1 = clk_id;
 
-	ret = eemi_ops->query_data(qdata, ret_payload);
+	ret = zynqmp_pm_query_data(qdata, ret_payload);
 	if (ret)
 		return ERR_PTR(ret);
 
@@ -339,7 +339,7 @@ static int zynqmp_pm_clock_get_parents(u32 clock_id, u32 index,
 	qdata.arg1 = clock_id;
 	qdata.arg2 = index;
 
-	ret = eemi_ops->query_data(qdata, ret_payload);
+	ret = zynqmp_pm_query_data(qdata, ret_payload);
 	memcpy(response, &ret_payload[1], sizeof(*response));
 
 	return ret;
@@ -364,7 +364,7 @@ static int zynqmp_pm_clock_get_attributes(u32 clock_id,
 	qdata.qid = PM_QID_CLOCK_GET_ATTRIBUTES;
 	qdata.arg1 = clock_id;
 
-	ret = eemi_ops->query_data(qdata, ret_payload);
+	ret = zynqmp_pm_query_data(qdata, ret_payload);
 	memcpy(response, &ret_payload[1], sizeof(*response));
 
 	return ret;
@@ -396,6 +396,9 @@ static int __zynqmp_clock_get_topology(struct clock_topology *topology,
 		topology[*nnodes].type_flag =
 				FIELD_GET(CLK_TOPOLOGY_TYPE_FLAGS,
 					  response->topology[i]);
+		topology[*nnodes].custom_type_flag =
+			FIELD_GET(CLK_TOPOLOGY_CUSTOM_TYPE_FLAGS,
+				  response->topology[i]);
 		(*nnodes)++;
 	}
 
@@ -558,7 +561,7 @@ static struct clk_hw *zynqmp_register_clk_topology(int clk_id, char *clk_name,
 {
 	int j;
 	u32 num_nodes, clk_dev_id;
-	char *clk_out = NULL;
+	char *clk_out[MAX_NODES];
 	struct clock_topology *nodes;
 	struct clk_hw *hw = NULL;
 
@@ -572,16 +575,16 @@ static struct clk_hw *zynqmp_register_clk_topology(int clk_id, char *clk_name,
 		 * Intermediate clock names are postfixed with type of clock.
 		 */
 		if (j != (num_nodes - 1)) {
-			clk_out = kasprintf(GFP_KERNEL, "%s%s", clk_name,
+			clk_out[j] = kasprintf(GFP_KERNEL, "%s%s", clk_name,
 					    clk_type_postfix[nodes[j].type]);
 		} else {
-			clk_out = kasprintf(GFP_KERNEL, "%s", clk_name);
+			clk_out[j] = kasprintf(GFP_KERNEL, "%s", clk_name);
 		}
 
 		if (!clk_topology[nodes[j].type])
 			continue;
 
-		hw = (*clk_topology[nodes[j].type])(clk_out, clk_dev_id,
+		hw = (*clk_topology[nodes[j].type])(clk_out[j], clk_dev_id,
 						    parent_names,
 						    num_parents,
 						    &nodes[j]);
@@ -590,9 +593,12 @@ static struct clk_hw *zynqmp_register_clk_topology(int clk_id, char *clk_name,
 				     __func__,  clk_dev_id, clk_name,
 				     PTR_ERR(hw));
 
-		parent_names[0] = clk_out;
+		parent_names[0] = clk_out[j];
 	}
-	kfree(clk_out);
+
+	for (j = 0; j < num_nodes; j++)
+		kfree(clk_out[j]);
+
 	return hw;
 }
 
@@ -663,6 +669,11 @@ static void zynqmp_get_clock_info(void)
 			continue;
 
 		clock[i].valid = FIELD_GET(CLK_ATTR_VALID, attr.attr[0]);
+		/* skip query for Invalid clock */
+		ret = zynqmp_is_valid_clock(i);
+		if (ret != CLK_ATTR_VALID)
+			continue;
+
 		clock[i].type = FIELD_GET(CLK_ATTR_TYPE, attr.attr[0]) ?
 			CLK_TYPE_EXTERNAL : CLK_TYPE_OUTPUT;
 
@@ -738,10 +749,6 @@ static int zynqmp_clock_probe(struct platform_device *pdev)
 	int ret;
 	struct device *dev = &pdev->dev;
 
-	eemi_ops = zynqmp_pm_get_eemi_ops();
-	if (IS_ERR(eemi_ops))
-		return PTR_ERR(eemi_ops);
-
 	ret = zynqmp_clk_setup(dev->of_node);
 
 	return ret;
diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c
index 4be2cc76aa2e..66da02b83d39 100644
--- a/drivers/clk/zynqmp/divider.c
+++ b/drivers/clk/zynqmp/divider.c
@@ -25,7 +25,8 @@
 #define to_zynqmp_clk_divider(_hw)		\
 	container_of(_hw, struct zynqmp_clk_divider, hw)
 
-#define CLK_FRAC	BIT(13) /* has a fractional parent */
+#define CLK_FRAC		BIT(13) /* has a fractional parent */
+#define CUSTOM_FLAG_CLK_FRAC	BIT(0) /* has a fractional parent in custom type flag */
 
 /**
  * struct zynqmp_clk_divider - adjustable divider clock
@@ -83,9 +84,8 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw,
 	u32 div_type = divider->div_type;
 	u32 div, value;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_getdivider(clk_id, &div);
+	ret = zynqmp_pm_clock_getdivider(clk_id, &div);
 
 	if (ret)
 		pr_warn_once("%s() get divider failed for %s, ret = %d\n",
@@ -111,23 +111,30 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw,
 
 static void zynqmp_get_divider2_val(struct clk_hw *hw,
 				    unsigned long rate,
-				    unsigned long parent_rate,
 				    struct zynqmp_clk_divider *divider,
 				    int *bestdiv)
 {
 	int div1;
 	int div2;
 	long error = LONG_MAX;
-	struct clk_hw *parent_hw = clk_hw_get_parent(hw);
-	struct zynqmp_clk_divider *pdivider = to_zynqmp_clk_divider(parent_hw);
+	unsigned long div1_prate;
+	struct clk_hw *div1_parent_hw;
+	struct clk_hw *div2_parent_hw = clk_hw_get_parent(hw);
+	struct zynqmp_clk_divider *pdivider =
+				to_zynqmp_clk_divider(div2_parent_hw);
 
 	if (!pdivider)
 		return;
 
+	div1_parent_hw = clk_hw_get_parent(div2_parent_hw);
+	if (!div1_parent_hw)
+		return;
+
+	div1_prate = clk_hw_get_rate(div1_parent_hw);
 	*bestdiv = 1;
 	for (div1 = 1; div1 <= pdivider->max_div;) {
 		for (div2 = 1; div2 <= divider->max_div;) {
-			long new_error = ((parent_rate / div1) / div2) - rate;
+			long new_error = ((div1_prate / div1) / div2) - rate;
 
 			if (abs(new_error) < abs(error)) {
 				*bestdiv = div2;
@@ -163,11 +170,10 @@ static long zynqmp_clk_divider_round_rate(struct clk_hw *hw,
 	u32 div_type = divider->div_type;
 	u32 bestdiv;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
 	/* if read only, just return current value */
 	if (divider->flags & CLK_DIVIDER_READ_ONLY) {
-		ret = eemi_ops->clock_getdivider(clk_id, &bestdiv);
+		ret = zynqmp_pm_clock_getdivider(clk_id, &bestdiv);
 
 		if (ret)
 			pr_warn_once("%s() get divider failed for %s, ret = %d\n",
@@ -192,11 +198,13 @@ static long zynqmp_clk_divider_round_rate(struct clk_hw *hw,
 	 */
 	if (div_type == TYPE_DIV2 &&
 	    (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT)) {
-		zynqmp_get_divider2_val(hw, rate, *prate, divider, &bestdiv);
+		zynqmp_get_divider2_val(hw, rate, divider, &bestdiv);
 	}
 
 	if ((clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && divider->is_frac)
 		bestdiv = rate % *prate ? 1 : bestdiv;
+
+	bestdiv = min_t(u32, bestdiv, divider->max_div);
 	*prate = rate * bestdiv;
 
 	return rate;
@@ -219,7 +227,6 @@ static int zynqmp_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 	u32 div_type = divider->div_type;
 	u32 value, div;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
 	value = zynqmp_divider_get_val(parent_rate, rate, divider->flags);
 	if (div_type == TYPE_DIV1) {
@@ -233,7 +240,7 @@ static int zynqmp_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 	if (divider->flags & CLK_DIVIDER_POWER_OF_TWO)
 		div = __ffs(div);
 
-	ret = eemi_ops->clock_setdivider(clk_id, div);
+	ret = zynqmp_pm_clock_setdivider(clk_id, div);
 
 	if (ret)
 		pr_warn_once("%s() set divider failed for %s, ret = %d\n",
@@ -256,9 +263,8 @@ static const struct clk_ops zynqmp_clk_divider_ops = {
  * Return: Maximum divisor of a clock if query data is successful
  *	   U16_MAX in case of query data is not success
  */
-u32 zynqmp_clk_get_max_divisor(u32 clk_id, u32 type)
+static u32 zynqmp_clk_get_max_divisor(u32 clk_id, u32 type)
 {
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 	struct zynqmp_pm_query_data qdata = {0};
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -266,7 +272,7 @@ u32 zynqmp_clk_get_max_divisor(u32 clk_id, u32 type)
 	qdata.qid = PM_QID_CLOCK_GET_MAX_DIVISOR;
 	qdata.arg1 = clk_id;
 	qdata.arg2 = type;
-	ret = eemi_ops->query_data(qdata, ret_payload);
+	ret = zynqmp_pm_query_data(qdata, ret_payload);
 	/*
 	 * To maintain backward compatibility return maximum possible value
 	 * (0xFFFF) if query for max divisor is not successful.
@@ -311,7 +317,8 @@ struct clk_hw *zynqmp_clk_register_divider(const char *name,
 	init.num_parents = 1;
 
 	/* struct clk_divider assignments */
-	div->is_frac = !!(nodes->flag & CLK_FRAC);
+	div->is_frac = !!((nodes->flag & CLK_FRAC) |
+			  (nodes->custom_type_flag & CUSTOM_FLAG_CLK_FRAC));
 	div->flags = nodes->type_flag;
 	div->hw.init = &init;
 	div->clk_id = clk_id;
diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c
index 89b599530105..92f449ed38e5 100644
--- a/drivers/clk/zynqmp/pll.c
+++ b/drivers/clk/zynqmp/pll.c
@@ -50,10 +50,8 @@ static inline enum pll_mode zynqmp_pll_get_mode(struct clk_hw *hw)
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->ioctl(0, IOCTL_GET_PLL_FRAC_MODE, clk_id, 0,
-			      ret_payload);
+	ret = zynqmp_pm_get_pll_frac_mode(clk_id, ret_payload);
 	if (ret)
 		pr_warn_once("%s() PLL get frac mode failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
@@ -73,14 +71,13 @@ static inline void zynqmp_pll_set_mode(struct clk_hw *hw, bool on)
 	const char *clk_name = clk_hw_get_name(hw);
 	int ret;
 	u32 mode;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
 	if (on)
 		mode = PLL_MODE_FRAC;
 	else
 		mode = PLL_MODE_INT;
 
-	ret = eemi_ops->ioctl(0, IOCTL_SET_PLL_FRAC_MODE, clk_id, mode, NULL);
+	ret = zynqmp_pm_set_pll_frac_mode(clk_id, mode);
 	if (ret)
 		pr_warn_once("%s() PLL set frac mode failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
@@ -139,17 +136,15 @@ static unsigned long zynqmp_pll_recalc_rate(struct clk_hw *hw,
 	unsigned long rate, frac;
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_getdivider(clk_id, &fbdiv);
+	ret = zynqmp_pm_clock_getdivider(clk_id, &fbdiv);
 	if (ret)
 		pr_warn_once("%s() get divider failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
 
 	rate =  parent_rate * fbdiv;
 	if (zynqmp_pll_get_mode(hw) == PLL_MODE_FRAC) {
-		eemi_ops->ioctl(0, IOCTL_GET_PLL_FRAC_DATA, clk_id, 0,
-				ret_payload);
+		zynqmp_pm_get_pll_frac_data(clk_id, ret_payload);
 		data = ret_payload[1];
 		frac = (parent_rate * data) / FRAC_DIV;
 		rate = rate + frac;
@@ -177,7 +172,6 @@ static int zynqmp_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 	u32 fbdiv;
 	long rate_div, frac, m, f;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
 	if (zynqmp_pll_get_mode(hw) == PLL_MODE_FRAC) {
 		rate_div = (rate * FRAC_DIV) / parent_rate;
@@ -187,21 +181,21 @@ static int zynqmp_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 		rate = parent_rate * m;
 		frac = (parent_rate * f) / FRAC_DIV;
 
-		ret = eemi_ops->clock_setdivider(clk_id, m);
+		ret = zynqmp_pm_clock_setdivider(clk_id, m);
 		if (ret == -EUSERS)
 			WARN(1, "More than allowed devices are using the %s, which is forbidden\n",
 			     clk_name);
 		else if (ret)
 			pr_warn_once("%s() set divider failed for %s, ret = %d\n",
 				     __func__, clk_name, ret);
-		eemi_ops->ioctl(0, IOCTL_SET_PLL_FRAC_DATA, clk_id, f, NULL);
+		zynqmp_pm_set_pll_frac_data(clk_id, f);
 
 		return rate + frac;
 	}
 
 	fbdiv = DIV_ROUND_CLOSEST(rate, parent_rate);
 	fbdiv = clamp_t(u32, fbdiv, PLL_FBDIV_MIN, PLL_FBDIV_MAX);
-	ret = eemi_ops->clock_setdivider(clk_id, fbdiv);
+	ret = zynqmp_pm_clock_setdivider(clk_id, fbdiv);
 	if (ret)
 		pr_warn_once("%s() set divider failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
@@ -222,9 +216,8 @@ static int zynqmp_pll_is_enabled(struct clk_hw *hw)
 	u32 clk_id = clk->clk_id;
 	unsigned int state;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
-	ret = eemi_ops->clock_getstate(clk_id, &state);
+	ret = zynqmp_pm_clock_getstate(clk_id, &state);
 	if (ret) {
 		pr_warn_once("%s() clock get state failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
@@ -246,12 +239,11 @@ static int zynqmp_pll_enable(struct clk_hw *hw)
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 clk_id = clk->clk_id;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
 	if (zynqmp_pll_is_enabled(hw))
 		return 0;
 
-	ret = eemi_ops->clock_enable(clk_id);
+	ret = zynqmp_pm_clock_enable(clk_id);
 	if (ret)
 		pr_warn_once("%s() clock enable failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
@@ -269,12 +261,11 @@ static void zynqmp_pll_disable(struct clk_hw *hw)
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 clk_id = clk->clk_id;
 	int ret;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 
 	if (!zynqmp_pll_is_enabled(hw))
 		return;
 
-	ret = eemi_ops->clock_disable(clk_id);
+	ret = zynqmp_pm_clock_disable(clk_id);
 	if (ret)
 		pr_warn_once("%s() clock disable failed for %s, ret = %d\n",
 			     __func__, clk_name, ret);
diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c
index c4f15c4068c0..9de1dabfb126 100644
--- a/drivers/clocksource/timer-riscv.c
+++ b/drivers/clocksource/timer-riscv.c
@@ -12,8 +12,11 @@
 #include <linux/cpu.h>
 #include <linux/delay.h>
 #include <linux/irq.h>
+#include <linux/irqdomain.h>
 #include <linux/sched_clock.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/interrupt.h>
+#include <linux/of_irq.h>
 #include <asm/smp.h>
 #include <asm/sbi.h>
 
@@ -39,6 +42,7 @@ static int riscv_clock_next_event(unsigned long delta,
 	return 0;
 }
 
+static unsigned int riscv_clock_event_irq;
 static DEFINE_PER_CPU(struct clock_event_device, riscv_clock_event) = {
 	.name			= "riscv_timer_clockevent",
 	.features		= CLOCK_EVT_FEAT_ONESHOT,
@@ -74,30 +78,36 @@ static int riscv_timer_starting_cpu(unsigned int cpu)
 	struct clock_event_device *ce = per_cpu_ptr(&riscv_clock_event, cpu);
 
 	ce->cpumask = cpumask_of(cpu);
+	ce->irq = riscv_clock_event_irq;
 	clockevents_config_and_register(ce, riscv_timebase, 100, 0x7fffffff);
 
-	csr_set(CSR_IE, IE_TIE);
+	enable_percpu_irq(riscv_clock_event_irq,
+			  irq_get_trigger_type(riscv_clock_event_irq));
 	return 0;
 }
 
 static int riscv_timer_dying_cpu(unsigned int cpu)
 {
-	csr_clear(CSR_IE, IE_TIE);
+	disable_percpu_irq(riscv_clock_event_irq);
 	return 0;
 }
 
 /* called directly from the low-level interrupt handler */
-void riscv_timer_interrupt(void)
+static irqreturn_t riscv_timer_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *evdev = this_cpu_ptr(&riscv_clock_event);
 
 	csr_clear(CSR_IE, IE_TIE);
 	evdev->event_handler(evdev);
+
+	return IRQ_HANDLED;
 }
 
 static int __init riscv_timer_init_dt(struct device_node *n)
 {
 	int cpuid, hartid, error;
+	struct device_node *child;
+	struct irq_domain *domain;
 
 	hartid = riscv_of_processor_hartid(n);
 	if (hartid < 0) {
@@ -115,6 +125,25 @@ static int __init riscv_timer_init_dt(struct device_node *n)
 	if (cpuid != smp_processor_id())
 		return 0;
 
+	domain = NULL;
+	child = of_get_compatible_child(n, "riscv,cpu-intc");
+	if (!child) {
+		pr_err("Failed to find INTC node [%pOF]\n", n);
+		return -ENODEV;
+	}
+	domain = irq_find_host(child);
+	of_node_put(child);
+	if (!domain) {
+		pr_err("Failed to find IRQ domain for node [%pOF]\n", n);
+		return -ENODEV;
+	}
+
+	riscv_clock_event_irq = irq_create_mapping(domain, RV_IRQ_TIMER);
+	if (!riscv_clock_event_irq) {
+		pr_err("Failed to map timer interrupt for node [%pOF]\n", n);
+		return -ENODEV;
+	}
+
 	pr_info("%s: Registering clocksource cpuid [%d] hartid [%d]\n",
 	       __func__, cpuid, hartid);
 	error = clocksource_register_hz(&riscv_clocksource, riscv_timebase);
@@ -126,6 +155,14 @@ static int __init riscv_timer_init_dt(struct device_node *n)
 
 	sched_clock_register(riscv_sched_clock, 64, riscv_timebase);
 
+	error = request_percpu_irq(riscv_clock_event_irq,
+				    riscv_timer_interrupt,
+				    "riscv-timer", &riscv_clock_event);
+	if (error) {
+		pr_err("registering percpu irq failed [%d]\n", error);
+		return error;
+	}
+
 	error = cpuhp_setup_state(CPUHP_AP_RISCV_TIMER_STARTING,
 			 "clockevents/riscv/timer:starting",
 			 riscv_timer_starting_cpu, riscv_timer_dying_cpu);
diff --git a/drivers/connector/Kconfig b/drivers/connector/Kconfig
index ba1f3f421cc6..0c2d2aa82d8c 100644
--- a/drivers/connector/Kconfig
+++ b/drivers/connector/Kconfig
@@ -3,7 +3,7 @@
 menuconfig CONNECTOR
 	tristate "Connector - unified userspace <-> kernelspace linker"
 	depends on NET
-	---help---
+	help
 	  This is unified userspace <-> kernelspace connector working on top
 	  of the netlink socket protocol.
 
@@ -16,7 +16,7 @@ config PROC_EVENTS
 	bool "Report process events to userspace"
 	depends on CONNECTOR=y
 	default y
-	---help---
+	help
 	  Provide a connector that reports process events to userspace. Send
 	  events such as fork, exec, id change (uid, gid, suid, etc), and exit.
 
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index 9481292981f0..c6cbfc8baf72 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -295,11 +295,11 @@ config ARM_TANGO_CPUFREQ
 	default y
 
 config ARM_TEGRA20_CPUFREQ
-	tristate "Tegra20 CPUFreq support"
-	depends on ARCH_TEGRA
+	tristate "Tegra20/30 CPUFreq support"
+	depends on ARCH_TEGRA && CPUFREQ_DT
 	default y
 	help
-	  This adds the CPUFreq driver support for Tegra20 SOCs.
+	  This adds the CPUFreq driver support for Tegra20/30 SOCs.
 
 config ARM_TEGRA124_CPUFREQ
 	bool "Tegra124 CPUFreq support"
diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86
index bc58a0809d11..399526289320 100644
--- a/drivers/cpufreq/Kconfig.x86
+++ b/drivers/cpufreq/Kconfig.x86
@@ -75,7 +75,7 @@ config X86_SFI_CPUFREQ
 config ELAN_CPUFREQ
 	tristate "AMD Elan SC400 and SC410"
 	depends on MELAN
-	---help---
+	help
 	  This adds the CPUFreq driver for AMD Elan SC400 and SC410
 	  processors.
 
@@ -90,7 +90,7 @@ config ELAN_CPUFREQ
 config SC520_CPUFREQ
 	tristate "AMD Elan SC520"
 	depends on MELAN
-	---help---
+	help
 	  This adds the CPUFreq driver for AMD Elan SC520 processor.
 
 	  For details, take a look at <file:Documentation/cpu-freq/>.
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 289e8ce3fd13..429e5a36c08a 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -126,12 +126,12 @@ static void boost_set_msr_each(void *p_en)
 	boost_set_msr(enable);
 }
 
-static int set_boost(int val)
+static int set_boost(struct cpufreq_policy *policy, int val)
 {
-	get_online_cpus();
-	on_each_cpu(boost_set_msr_each, (void *)(long)val, 1);
-	put_online_cpus();
-	pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis");
+	on_each_cpu_mask(policy->cpus, boost_set_msr_each,
+			 (void *)(long)val, 1);
+	pr_debug("CPU %*pbl: Core Boosting %sabled.\n",
+		 cpumask_pr_args(policy->cpus), val ? "en" : "dis");
 
 	return 0;
 }
@@ -162,7 +162,9 @@ static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf,
 	if (ret || val > 1)
 		return -EINVAL;
 
-	set_boost(val);
+	get_online_cpus();
+	set_boost(policy, val);
+	put_online_cpus();
 
 	return count;
 }
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index bda0b2406fba..257d726a4456 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -37,6 +37,7 @@
  * requested etc.
  */
 static struct cppc_cpudata **all_cpu_data;
+static bool boost_supported;
 
 struct cppc_workaround_oem_info {
 	char oem_id[ACPI_OEM_ID_SIZE + 1];
@@ -310,7 +311,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	 * Section 8.4.7.1.1.5 of ACPI 6.1 spec)
 	 */
 	policy->min = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.lowest_nonlinear_perf);
-	policy->max = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.highest_perf);
+	policy->max = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.nominal_perf);
 
 	/*
 	 * Set cpuinfo.min_freq to Lowest to make the full range of performance
@@ -318,7 +319,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	 * nonlinear perf
 	 */
 	policy->cpuinfo.min_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.lowest_perf);
-	policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.highest_perf);
+	policy->cpuinfo.max_freq = cppc_cpufreq_perf_to_khz(cpu, cpu->perf_caps.nominal_perf);
 
 	policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu_num);
 	policy->shared_type = cpu->shared_type;
@@ -343,6 +344,13 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 
 	cpu->cur_policy = policy;
 
+	/*
+	 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
+	 * is supported.
+	 */
+	if (cpu->perf_caps.highest_perf > cpu->perf_caps.nominal_perf)
+		boost_supported = true;
+
 	/* Set policy->cur to max now. The governors will adjust later. */
 	policy->cur = cppc_cpufreq_perf_to_khz(cpu,
 					cpu->perf_caps.highest_perf);
@@ -410,6 +418,32 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpunum)
 	return cppc_get_rate_from_fbctrs(cpu, fb_ctrs_t0, fb_ctrs_t1);
 }
 
+static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
+{
+	struct cppc_cpudata *cpudata;
+	int ret;
+
+	if (!boost_supported) {
+		pr_err("BOOST not supported by CPU or firmware\n");
+		return -EINVAL;
+	}
+
+	cpudata = all_cpu_data[policy->cpu];
+	if (state)
+		policy->max = cppc_cpufreq_perf_to_khz(cpudata,
+					cpudata->perf_caps.highest_perf);
+	else
+		policy->max = cppc_cpufreq_perf_to_khz(cpudata,
+					cpudata->perf_caps.nominal_perf);
+	policy->cpuinfo.max_freq = policy->max;
+
+	ret = freq_qos_update_request(policy->max_freq_req, policy->max);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
 static struct cpufreq_driver cppc_cpufreq_driver = {
 	.flags = CPUFREQ_CONST_LOOPS,
 	.verify = cppc_verify_policy,
@@ -417,6 +451,7 @@ static struct cpufreq_driver cppc_cpufreq_driver = {
 	.get = cppc_cpufreq_get_rate,
 	.init = cppc_cpufreq_cpu_init,
 	.stop_cpu = cppc_cpufreq_stop_cpu,
+	.set_boost = cppc_cpufreq_set_boost,
 	.name = "cppc_cpufreq",
 };
 
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 26fe8dfb9ce6..79742bbd221f 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -121,6 +121,10 @@ static int resources_available(void)
 
 	clk_put(cpu_clk);
 
+	ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL);
+	if (ret)
+		return ret;
+
 	name = find_supply_name(cpu_dev);
 	/* Platform doesn't require regulator */
 	if (!name)
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index d03f250f68e4..0128de3603df 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -2532,34 +2532,29 @@ EXPORT_SYMBOL_GPL(cpufreq_update_limits);
 /*********************************************************************
  *               BOOST						     *
  *********************************************************************/
-static int cpufreq_boost_set_sw(int state)
+static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
 {
-	struct cpufreq_policy *policy;
-
-	for_each_active_policy(policy) {
-		int ret;
+	int ret;
 
-		if (!policy->freq_table)
-			return -ENXIO;
+	if (!policy->freq_table)
+		return -ENXIO;
 
-		ret = cpufreq_frequency_table_cpuinfo(policy,
-						      policy->freq_table);
-		if (ret) {
-			pr_err("%s: Policy frequency update failed\n",
-			       __func__);
-			return ret;
-		}
-
-		ret = freq_qos_update_request(policy->max_freq_req, policy->max);
-		if (ret < 0)
-			return ret;
+	ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
+	if (ret) {
+		pr_err("%s: Policy frequency update failed\n", __func__);
+		return ret;
 	}
 
+	ret = freq_qos_update_request(policy->max_freq_req, policy->max);
+	if (ret < 0)
+		return ret;
+
 	return 0;
 }
 
 int cpufreq_boost_trigger_state(int state)
 {
+	struct cpufreq_policy *policy;
 	unsigned long flags;
 	int ret = 0;
 
@@ -2570,15 +2565,25 @@ int cpufreq_boost_trigger_state(int state)
 	cpufreq_driver->boost_enabled = state;
 	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 
-	ret = cpufreq_driver->set_boost(state);
-	if (ret) {
-		write_lock_irqsave(&cpufreq_driver_lock, flags);
-		cpufreq_driver->boost_enabled = !state;
-		write_unlock_irqrestore(&cpufreq_driver_lock, flags);
-
-		pr_err("%s: Cannot %s BOOST\n",
-		       __func__, state ? "enable" : "disable");
+	get_online_cpus();
+	for_each_active_policy(policy) {
+		ret = cpufreq_driver->set_boost(policy, state);
+		if (ret)
+			goto err_reset_state;
 	}
+	put_online_cpus();
+
+	return 0;
+
+err_reset_state:
+	put_online_cpus();
+
+	write_lock_irqsave(&cpufreq_driver_lock, flags);
+	cpufreq_driver->boost_enabled = !state;
+	write_unlock_irqrestore(&cpufreq_driver_lock, flags);
+
+	pr_err("%s: Cannot %s BOOST\n",
+	       __func__, state ? "enable" : "disable");
 
 	return ret;
 }
diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c
index 2e233ad72758..3d2f143748ef 100644
--- a/drivers/cpufreq/tegra186-cpufreq.c
+++ b/drivers/cpufreq/tegra186-cpufreq.c
@@ -93,7 +93,8 @@ static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy,
 
 static struct cpufreq_driver tegra186_cpufreq_driver = {
 	.name = "tegra186",
-	.flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY,
+	.flags = CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+			CPUFREQ_NEED_INITIAL_FREQ_CHECK,
 	.verify = cpufreq_generic_frequency_table_verify,
 	.target_index = tegra186_cpufreq_set_target,
 	.init = tegra186_cpufreq_init,
diff --git a/drivers/cpufreq/tegra20-cpufreq.c b/drivers/cpufreq/tegra20-cpufreq.c
index f84ecd22f488..8c893043953e 100644
--- a/drivers/cpufreq/tegra20-cpufreq.c
+++ b/drivers/cpufreq/tegra20-cpufreq.c
@@ -7,201 +7,96 @@
  *	Based on arch/arm/plat-omap/cpu-omap.c, (C) 2005 Nokia Corporation
  */
 
-#include <linux/clk.h>
-#include <linux/cpufreq.h>
+#include <linux/bits.h>
+#include <linux/cpu.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/pm_opp.h>
 #include <linux/types.h>
 
-static struct cpufreq_frequency_table freq_table[] = {
-	{ .frequency = 216000 },
-	{ .frequency = 312000 },
-	{ .frequency = 456000 },
-	{ .frequency = 608000 },
-	{ .frequency = 760000 },
-	{ .frequency = 816000 },
-	{ .frequency = 912000 },
-	{ .frequency = 1000000 },
-	{ .frequency = CPUFREQ_TABLE_END },
-};
-
-struct tegra20_cpufreq {
-	struct device *dev;
-	struct cpufreq_driver driver;
-	struct clk *cpu_clk;
-	struct clk *pll_x_clk;
-	struct clk *pll_p_clk;
-	bool pll_x_prepared;
-};
+#include <soc/tegra/common.h>
+#include <soc/tegra/fuse.h>
 
-static unsigned int tegra_get_intermediate(struct cpufreq_policy *policy,
-					   unsigned int index)
+static bool cpu0_node_has_opp_v2_prop(void)
 {
-	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
-	unsigned int ifreq = clk_get_rate(cpufreq->pll_p_clk) / 1000;
-
-	/*
-	 * Don't switch to intermediate freq if:
-	 * - we are already at it, i.e. policy->cur == ifreq
-	 * - index corresponds to ifreq
-	 */
-	if (freq_table[index].frequency == ifreq || policy->cur == ifreq)
-		return 0;
-
-	return ifreq;
-}
+	struct device_node *np = of_cpu_device_node_get(0);
+	bool ret = false;
 
-static int tegra_target_intermediate(struct cpufreq_policy *policy,
-				     unsigned int index)
-{
-	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
-	int ret;
-
-	/*
-	 * Take an extra reference to the main pll so it doesn't turn
-	 * off when we move the cpu off of it as enabling it again while we
-	 * switch to it from tegra_target() would take additional time.
-	 *
-	 * When target-freq is equal to intermediate freq we don't need to
-	 * switch to an intermediate freq and so this routine isn't called.
-	 * Also, we wouldn't be using pll_x anymore and must not take extra
-	 * reference to it, as it can be disabled now to save some power.
-	 */
-	clk_prepare_enable(cpufreq->pll_x_clk);
-
-	ret = clk_set_parent(cpufreq->cpu_clk, cpufreq->pll_p_clk);
-	if (ret)
-		clk_disable_unprepare(cpufreq->pll_x_clk);
-	else
-		cpufreq->pll_x_prepared = true;
+	if (of_get_property(np, "operating-points-v2", NULL))
+		ret = true;
 
+	of_node_put(np);
 	return ret;
 }
 
-static int tegra_target(struct cpufreq_policy *policy, unsigned int index)
-{
-	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
-	unsigned long rate = freq_table[index].frequency;
-	unsigned int ifreq = clk_get_rate(cpufreq->pll_p_clk) / 1000;
-	int ret;
-
-	/*
-	 * target freq == pll_p, don't need to take extra reference to pll_x_clk
-	 * as it isn't used anymore.
-	 */
-	if (rate == ifreq)
-		return clk_set_parent(cpufreq->cpu_clk, cpufreq->pll_p_clk);
-
-	ret = clk_set_rate(cpufreq->pll_x_clk, rate * 1000);
-	/* Restore to earlier frequency on error, i.e. pll_x */
-	if (ret)
-		dev_err(cpufreq->dev, "Failed to change pll_x to %lu\n", rate);
-
-	ret = clk_set_parent(cpufreq->cpu_clk, cpufreq->pll_x_clk);
-	/* This shouldn't fail while changing or restoring */
-	WARN_ON(ret);
-
-	/*
-	 * Drop count to pll_x clock only if we switched to intermediate freq
-	 * earlier while transitioning to a target frequency.
-	 */
-	if (cpufreq->pll_x_prepared) {
-		clk_disable_unprepare(cpufreq->pll_x_clk);
-		cpufreq->pll_x_prepared = false;
-	}
-
-	return ret;
-}
-
-static int tegra_cpu_init(struct cpufreq_policy *policy)
-{
-	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
-
-	clk_prepare_enable(cpufreq->cpu_clk);
-
-	/* FIXME: what's the actual transition time? */
-	cpufreq_generic_init(policy, freq_table, 300 * 1000);
-	policy->clk = cpufreq->cpu_clk;
-	policy->suspend_freq = freq_table[0].frequency;
-	return 0;
-}
-
-static int tegra_cpu_exit(struct cpufreq_policy *policy)
-{
-	struct tegra20_cpufreq *cpufreq = cpufreq_get_driver_data();
-
-	clk_disable_unprepare(cpufreq->cpu_clk);
-	return 0;
-}
-
 static int tegra20_cpufreq_probe(struct platform_device *pdev)
 {
-	struct tegra20_cpufreq *cpufreq;
+	struct platform_device *cpufreq_dt;
+	struct opp_table *opp_table;
+	struct device *cpu_dev;
+	u32 versions[2];
 	int err;
 
-	cpufreq = devm_kzalloc(&pdev->dev, sizeof(*cpufreq), GFP_KERNEL);
-	if (!cpufreq)
-		return -ENOMEM;
+	if (!cpu0_node_has_opp_v2_prop()) {
+		dev_err(&pdev->dev, "operating points not found\n");
+		dev_err(&pdev->dev, "please update your device tree\n");
+		return -ENODEV;
+	}
+
+	if (of_machine_is_compatible("nvidia,tegra20")) {
+		versions[0] = BIT(tegra_sku_info.cpu_process_id);
+		versions[1] = BIT(tegra_sku_info.soc_speedo_id);
+	} else {
+		versions[0] = BIT(tegra_sku_info.cpu_process_id);
+		versions[1] = BIT(tegra_sku_info.cpu_speedo_id);
+	}
+
+	dev_info(&pdev->dev, "hardware version 0x%x 0x%x\n",
+		 versions[0], versions[1]);
 
-	cpufreq->cpu_clk = clk_get_sys(NULL, "cclk");
-	if (IS_ERR(cpufreq->cpu_clk))
-		return PTR_ERR(cpufreq->cpu_clk);
+	cpu_dev = get_cpu_device(0);
+	if (WARN_ON(!cpu_dev))
+		return -ENODEV;
 
-	cpufreq->pll_x_clk = clk_get_sys(NULL, "pll_x");
-	if (IS_ERR(cpufreq->pll_x_clk)) {
-		err = PTR_ERR(cpufreq->pll_x_clk);
-		goto put_cpu;
+	opp_table = dev_pm_opp_set_supported_hw(cpu_dev, versions, 2);
+	err = PTR_ERR_OR_ZERO(opp_table);
+	if (err) {
+		dev_err(&pdev->dev, "failed to set supported hw: %d\n", err);
+		return err;
 	}
 
-	cpufreq->pll_p_clk = clk_get_sys(NULL, "pll_p");
-	if (IS_ERR(cpufreq->pll_p_clk)) {
-		err = PTR_ERR(cpufreq->pll_p_clk);
-		goto put_pll_x;
+	cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+	err = PTR_ERR_OR_ZERO(cpufreq_dt);
+	if (err) {
+		dev_err(&pdev->dev,
+			"failed to create cpufreq-dt device: %d\n", err);
+		goto err_put_supported_hw;
 	}
 
-	cpufreq->dev = &pdev->dev;
-	cpufreq->driver.get = cpufreq_generic_get;
-	cpufreq->driver.attr = cpufreq_generic_attr;
-	cpufreq->driver.init = tegra_cpu_init;
-	cpufreq->driver.exit = tegra_cpu_exit;
-	cpufreq->driver.flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK;
-	cpufreq->driver.verify = cpufreq_generic_frequency_table_verify;
-	cpufreq->driver.suspend = cpufreq_generic_suspend;
-	cpufreq->driver.driver_data = cpufreq;
-	cpufreq->driver.target_index = tegra_target;
-	cpufreq->driver.get_intermediate = tegra_get_intermediate;
-	cpufreq->driver.target_intermediate = tegra_target_intermediate;
-	snprintf(cpufreq->driver.name, CPUFREQ_NAME_LEN, "tegra");
-
-	err = cpufreq_register_driver(&cpufreq->driver);
-	if (err)
-		goto put_pll_p;
-
-	platform_set_drvdata(pdev, cpufreq);
+	platform_set_drvdata(pdev, cpufreq_dt);
 
 	return 0;
 
-put_pll_p:
-	clk_put(cpufreq->pll_p_clk);
-put_pll_x:
-	clk_put(cpufreq->pll_x_clk);
-put_cpu:
-	clk_put(cpufreq->cpu_clk);
+err_put_supported_hw:
+	dev_pm_opp_put_supported_hw(opp_table);
 
 	return err;
 }
 
 static int tegra20_cpufreq_remove(struct platform_device *pdev)
 {
-	struct tegra20_cpufreq *cpufreq = platform_get_drvdata(pdev);
+	struct platform_device *cpufreq_dt;
+	struct opp_table *opp_table;
 
-	cpufreq_unregister_driver(&cpufreq->driver);
+	cpufreq_dt = platform_get_drvdata(pdev);
+	platform_device_unregister(cpufreq_dt);
 
-	clk_put(cpufreq->pll_p_clk);
-	clk_put(cpufreq->pll_x_clk);
-	clk_put(cpufreq->cpu_clk);
+	opp_table = dev_pm_opp_get_opp_table(get_cpu_device(0));
+	dev_pm_opp_put_supported_hw(opp_table);
+	dev_pm_opp_put_opp_table(opp_table);
 
 	return 0;
 }
diff --git a/drivers/cpuidle/cpuidle-arm.c b/drivers/cpuidle/cpuidle-arm.c
index 9e5156d39627..8c758920d699 100644
--- a/drivers/cpuidle/cpuidle-arm.c
+++ b/drivers/cpuidle/cpuidle-arm.c
@@ -8,6 +8,7 @@
 
 #define pr_fmt(fmt) "CPUidle arm: " fmt
 
+#include <linux/cpu_cooling.h>
 #include <linux/cpuidle.h>
 #include <linux/cpumask.h>
 #include <linux/cpu_pm.h>
@@ -124,6 +125,8 @@ static int __init arm_idle_init_cpu(int cpu)
 	if (ret)
 		goto out_kfree_drv;
 
+	cpuidle_cooling_register(drv);
+
 	return 0;
 
 out_kfree_drv:
diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
index d0fb585073c6..3806f911b61c 100644
--- a/drivers/cpuidle/cpuidle-psci.c
+++ b/drivers/cpuidle/cpuidle-psci.c
@@ -9,6 +9,7 @@
 #define pr_fmt(fmt) "CPUidle PSCI: " fmt
 
 #include <linux/cpuhotplug.h>
+#include <linux/cpu_cooling.h>
 #include <linux/cpuidle.h>
 #include <linux/cpumask.h>
 #include <linux/cpu_pm.h>
@@ -319,6 +320,8 @@ static int __init psci_idle_init_cpu(int cpu)
 	if (ret)
 		goto out_kfree_drv;
 
+	cpuidle_cooling_register(drv);
+
 	return 0;
 
 out_kfree_drv:
diff --git a/drivers/cpuidle/cpuidle-pseries.c b/drivers/cpuidle/cpuidle-pseries.c
index 74c247972bb3..6513ef2af66a 100644
--- a/drivers/cpuidle/cpuidle-pseries.c
+++ b/drivers/cpuidle/cpuidle-pseries.c
@@ -19,6 +19,7 @@
 #include <asm/machdep.h>
 #include <asm/firmware.h>
 #include <asm/runlatch.h>
+#include <asm/idle.h>
 #include <asm/plpar_wrappers.h>
 
 struct cpuidle_driver pseries_idle_driver = {
@@ -31,39 +32,15 @@ static struct cpuidle_state *cpuidle_state_table __read_mostly;
 static u64 snooze_timeout __read_mostly;
 static bool snooze_timeout_en __read_mostly;
 
-static inline void idle_loop_prolog(unsigned long *in_purr)
-{
-	ppc64_runlatch_off();
-	*in_purr = mfspr(SPRN_PURR);
-	/*
-	 * Indicate to the HV that we are idle. Now would be
-	 * a good time to find other work to dispatch.
-	 */
-	get_lppaca()->idle = 1;
-}
-
-static inline void idle_loop_epilog(unsigned long in_purr)
-{
-	u64 wait_cycles;
-
-	wait_cycles = be64_to_cpu(get_lppaca()->wait_state_cycles);
-	wait_cycles += mfspr(SPRN_PURR) - in_purr;
-	get_lppaca()->wait_state_cycles = cpu_to_be64(wait_cycles);
-	get_lppaca()->idle = 0;
-
-	ppc64_runlatch_on();
-}
-
 static int snooze_loop(struct cpuidle_device *dev,
 			struct cpuidle_driver *drv,
 			int index)
 {
-	unsigned long in_purr;
 	u64 snooze_exit_time;
 
 	set_thread_flag(TIF_POLLING_NRFLAG);
 
-	idle_loop_prolog(&in_purr);
+	pseries_idle_prolog();
 	local_irq_enable();
 	snooze_exit_time = get_tb() + snooze_timeout;
 
@@ -87,7 +64,7 @@ static int snooze_loop(struct cpuidle_device *dev,
 
 	local_irq_disable();
 
-	idle_loop_epilog(in_purr);
+	pseries_idle_epilog();
 
 	return index;
 }
@@ -113,9 +90,8 @@ static int dedicated_cede_loop(struct cpuidle_device *dev,
 				struct cpuidle_driver *drv,
 				int index)
 {
-	unsigned long in_purr;
 
-	idle_loop_prolog(&in_purr);
+	pseries_idle_prolog();
 	get_lppaca()->donate_dedicated_cpu = 1;
 
 	HMT_medium();
@@ -124,7 +100,7 @@ static int dedicated_cede_loop(struct cpuidle_device *dev,
 	local_irq_disable();
 	get_lppaca()->donate_dedicated_cpu = 0;
 
-	idle_loop_epilog(in_purr);
+	pseries_idle_epilog();
 
 	return index;
 }
@@ -133,9 +109,8 @@ static int shared_cede_loop(struct cpuidle_device *dev,
 			struct cpuidle_driver *drv,
 			int index)
 {
-	unsigned long in_purr;
 
-	idle_loop_prolog(&in_purr);
+	pseries_idle_prolog();
 
 	/*
 	 * Yield the processor to the hypervisor.  We return if
@@ -147,7 +122,7 @@ static int shared_cede_loop(struct cpuidle_device *dev,
 	check_and_cede_processor();
 
 	local_irq_disable();
-	idle_loop_epilog(in_purr);
+	pseries_idle_epilog();
 
 	return index;
 }
diff --git a/drivers/cpuidle/cpuidle-tegra.c b/drivers/cpuidle/cpuidle-tegra.c
index 313b0290e97b..150045849d78 100644
--- a/drivers/cpuidle/cpuidle-tegra.c
+++ b/drivers/cpuidle/cpuidle-tegra.c
@@ -365,7 +365,6 @@ static int tegra_cpuidle_probe(struct platform_device *pdev)
 		break;
 
 	case TEGRA30:
-		tegra_cpuidle_disable_state(TEGRA_CC6);
 		break;
 
 	case TEGRA114:
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 2c887e4d005a..802b9ada4e9e 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -3,7 +3,7 @@
 menuconfig CRYPTO_HW
 	bool "Hardware crypto devices"
 	default y
-	---help---
+	help
 	  Say Y here to get to see options for hardware crypto devices and
 	  processors. This option alone does not add any kernel code.
 
@@ -334,7 +334,7 @@ config HW_RANDOM_PPC4XX
 	bool "PowerPC 4xx generic true random number generator support"
 	depends on CRYPTO_DEV_PPC4XX && HW_RANDOM
 	default y
-	---help---
+	help
 	 This option provides the kernel-side support for the TRNG hardware
 	 found in the security function of some PowerPC 4xx SoCs.
 
@@ -420,7 +420,7 @@ config CRYPTO_DEV_EXYNOS_RNG
 	depends on ARCH_EXYNOS || COMPILE_TEST
 	depends on HAS_IOMEM
 	select CRYPTO_RNG
-	---help---
+	help
 	  This driver provides kernel-side support through the
 	  cryptographic API for the pseudo random number generator hardware
 	  found on Exynos SoCs.
@@ -597,7 +597,7 @@ source "drivers/crypto/marvell/Kconfig"
 config CRYPTO_DEV_CAVIUM_ZIP
 	tristate "Cavium ZIP driver"
 	depends on PCI && 64BIT && (ARM64 || COMPILE_TEST)
-	---help---
+	help
 	  Select this option if you want to enable compression/decompression
 	  acceleration on Cavium's ARM based SoCs
 
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index 788c6607078b..cee2a2713038 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -278,7 +278,7 @@ static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
 
 struct nitrox_device *nitrox_get_first_device(void)
 {
-	struct nitrox_device *ndev = NULL;
+	struct nitrox_device *ndev;
 
 	mutex_lock(&devlist_lock);
 	list_for_each_entry(ndev, &ndevlist, list) {
@@ -286,7 +286,7 @@ struct nitrox_device *nitrox_get_first_device(void)
 			break;
 	}
 	mutex_unlock(&devlist_lock);
-	if (!ndev)
+	if (&ndev->list == &ndevlist)
 		return NULL;
 
 	refcount_inc(&ndev->refcnt);
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index f2756836093f..2984fdf51e85 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -8,7 +8,7 @@ config CRYPTO_DEV_CHELSIO
 	select CRYPTO_SHA512
 	select CRYPTO_AUTHENC
 	select CRYPTO_GF128MUL
-	---help---
+	help
 	  The Chelsio Crypto Co-processor driver for T6 adapters.
 
 	  For general information about Chelsio and our products, visit
@@ -29,7 +29,7 @@ config CHELSIO_IPSEC_INLINE
 	depends on XFRM_OFFLOAD
 	depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD
 	default n
-	---help---
+	help
 	  Enable support for IPSec Tx Inline.
 
 config CRYPTO_DEV_CHELSIO_TLS
@@ -37,7 +37,7 @@ config CRYPTO_DEV_CHELSIO_TLS
 	depends on CHELSIO_T4
 	depends on TLS_TOE
 	select CRYPTO_DEV_CHELSIO
-	---help---
+	help
 	  Support Chelsio Inline TLS with Chelsio crypto accelerator.
 
 	  To compile this driver as a module, choose M here: the module
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index f26a7a15551a..4c2553672b6f 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -2590,11 +2590,22 @@ int chcr_aead_dma_map(struct device *dev,
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	unsigned int authsize = crypto_aead_authsize(tfm);
-	int dst_size;
+	int src_len, dst_len;
 
-	dst_size = req->assoclen + req->cryptlen + (op_type ?
-				0 : authsize);
-	if (!req->cryptlen || !dst_size)
+	/* calculate and handle src and dst sg length separately
+	 * for inplace and out-of place operations
+	 */
+	if (req->src == req->dst) {
+		src_len = req->assoclen + req->cryptlen + (op_type ?
+							0 : authsize);
+		dst_len = src_len;
+	} else {
+		src_len = req->assoclen + req->cryptlen;
+		dst_len = req->assoclen + req->cryptlen + (op_type ?
+							-authsize : authsize);
+	}
+
+	if (!req->cryptlen || !src_len || !dst_len)
 		return 0;
 	reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
 					DMA_BIDIRECTIONAL);
@@ -2606,20 +2617,23 @@ int chcr_aead_dma_map(struct device *dev,
 		reqctx->b0_dma = 0;
 	if (req->src == req->dst) {
 		error = dma_map_sg(dev, req->src,
-				sg_nents_for_len(req->src, dst_size),
+				sg_nents_for_len(req->src, src_len),
 					DMA_BIDIRECTIONAL);
 		if (!error)
 			goto err;
 	} else {
-		error = dma_map_sg(dev, req->src, sg_nents(req->src),
+		error = dma_map_sg(dev, req->src,
+				   sg_nents_for_len(req->src, src_len),
 				   DMA_TO_DEVICE);
 		if (!error)
 			goto err;
-		error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
+		error = dma_map_sg(dev, req->dst,
+				   sg_nents_for_len(req->dst, dst_len),
 				   DMA_FROM_DEVICE);
 		if (!error) {
-			dma_unmap_sg(dev, req->src, sg_nents(req->src),
-				   DMA_TO_DEVICE);
+			dma_unmap_sg(dev, req->src,
+				     sg_nents_for_len(req->src, src_len),
+				     DMA_TO_DEVICE);
 			goto err;
 		}
 	}
@@ -2637,24 +2651,37 @@ void chcr_aead_dma_unmap(struct device *dev,
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	unsigned int authsize = crypto_aead_authsize(tfm);
-	int dst_size;
+	int src_len, dst_len;
 
-	dst_size = req->assoclen + req->cryptlen + (op_type ?
-					0 : authsize);
-	if (!req->cryptlen || !dst_size)
+	/* calculate and handle src and dst sg length separately
+	 * for inplace and out-of place operations
+	 */
+	if (req->src == req->dst) {
+		src_len = req->assoclen + req->cryptlen + (op_type ?
+							0 : authsize);
+		dst_len = src_len;
+	} else {
+		src_len = req->assoclen + req->cryptlen;
+		dst_len = req->assoclen + req->cryptlen + (op_type ?
+						-authsize : authsize);
+	}
+
+	if (!req->cryptlen || !src_len || !dst_len)
 		return;
 
 	dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
 					DMA_BIDIRECTIONAL);
 	if (req->src == req->dst) {
 		dma_unmap_sg(dev, req->src,
-			     sg_nents_for_len(req->src, dst_size),
+			     sg_nents_for_len(req->src, src_len),
 			     DMA_BIDIRECTIONAL);
 	} else {
-		dma_unmap_sg(dev, req->src, sg_nents(req->src),
-				   DMA_TO_DEVICE);
-		dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
-				   DMA_FROM_DEVICE);
+		dma_unmap_sg(dev, req->src,
+			     sg_nents_for_len(req->src, src_len),
+			     DMA_TO_DEVICE);
+		dma_unmap_sg(dev, req->dst,
+			     sg_nents_for_len(req->dst, dst_len),
+			     DMA_FROM_DEVICE);
 	}
 }
 
@@ -4364,22 +4391,32 @@ static int chcr_unregister_alg(void)
 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
 		switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
 		case CRYPTO_ALG_TYPE_SKCIPHER:
-			if (driver_algs[i].is_registered)
+			if (driver_algs[i].is_registered && refcount_read(
+			    &driver_algs[i].alg.skcipher.base.cra_refcnt)
+			    == 1) {
 				crypto_unregister_skcipher(
 						&driver_algs[i].alg.skcipher);
+				driver_algs[i].is_registered = 0;
+			}
 			break;
 		case CRYPTO_ALG_TYPE_AEAD:
-			if (driver_algs[i].is_registered)
+			if (driver_algs[i].is_registered && refcount_read(
+			    &driver_algs[i].alg.aead.base.cra_refcnt) == 1) {
 				crypto_unregister_aead(
 						&driver_algs[i].alg.aead);
+				driver_algs[i].is_registered = 0;
+			}
 			break;
 		case CRYPTO_ALG_TYPE_AHASH:
-			if (driver_algs[i].is_registered)
+			if (driver_algs[i].is_registered && refcount_read(
+			    &driver_algs[i].alg.hash.halg.base.cra_refcnt)
+			    == 1) {
 				crypto_unregister_ahash(
 						&driver_algs[i].alg.hash);
+				driver_algs[i].is_registered = 0;
+			}
 			break;
 		}
-		driver_algs[i].is_registered = 0;
 	}
 	return 0;
 }
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index f58c2b5c7fc5..d4f6e010dc79 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -389,10 +389,6 @@ static inline void copy_hash_init_values(char *key, int digestsize)
 	}
 }
 
-static const u8 sgl_lengths[20] = {
-	0, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 13, 13, 14, 15
-};
-
 /* Number of len fields(8) * size of one addr field */
 #define PHYSDSGL_MAX_LEN_SIZE 16
 
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index b3fdbdc25acb..31e427e273f8 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -223,7 +223,7 @@ struct chcr_authenc_ctx {
 
 struct __aead_ctx {
 	struct chcr_gcm_ctx gcm[0];
-	struct chcr_authenc_ctx authenc[0];
+	struct chcr_authenc_ctx authenc[];
 };
 
 struct chcr_aead_ctx {
@@ -235,7 +235,7 @@ struct chcr_aead_ctx {
 	u8 nonce[4];
 	u16 hmac_ctrl;
 	u16 mayverify;
-	struct	__aead_ctx ctx[0];
+	struct	__aead_ctx ctx[];
 };
 
 struct hmac_ctx {
@@ -247,7 +247,7 @@ struct hmac_ctx {
 struct __crypto_ctx {
 	struct hmac_ctx hmacctx[0];
 	struct ablk_ctx ablkctx[0];
-	struct chcr_aead_ctx aeadctx[0];
+	struct chcr_aead_ctx aeadctx[];
 };
 
 struct chcr_context {
@@ -257,7 +257,7 @@ struct chcr_context {
 	unsigned int  ntxq;
 	unsigned int  nrxq;
 	struct completion cbc_aes_aio_done;
-	struct __crypto_ctx crypto_ctx[0];
+	struct __crypto_ctx crypto_ctx[];
 };
 
 struct chcr_hctx_per_wr {
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c
index 9a642c79a657..f200fae6f7cb 100644
--- a/drivers/crypto/chelsio/chtls/chtls_cm.c
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -93,8 +93,10 @@ static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
 					    struct sock *sk)
 {
 	struct net_device *ndev = cdev->ports[0];
+#if IS_ENABLED(CONFIG_IPV6)
 	struct net_device *temp;
 	int addr_type;
+#endif
 
 	switch (sk->sk_family) {
 	case PF_INET:
@@ -102,19 +104,21 @@ static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
 			return ndev;
 		ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
 		break;
+#if IS_ENABLED(CONFIG_IPV6)
 	case PF_INET6:
 		addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
 		if (likely(addr_type == IPV6_ADDR_ANY))
 			return ndev;
 
-	for_each_netdev_rcu(&init_net, temp) {
-		if (ipv6_chk_addr(&init_net, (struct in6_addr *)
-				  &sk->sk_v6_rcv_saddr, temp, 1)) {
-			ndev = temp;
-			break;
+		for_each_netdev_rcu(&init_net, temp) {
+			if (ipv6_chk_addr(&init_net, (struct in6_addr *)
+					  &sk->sk_v6_rcv_saddr, temp, 1)) {
+				ndev = temp;
+				break;
+			}
 		}
-	}
 	break;
+#endif
 	default:
 		return NULL;
 	}
@@ -476,8 +480,10 @@ void chtls_destroy_sock(struct sock *sk)
 	csk->cdev = NULL;
 	if (sk->sk_family == AF_INET)
 		sk->sk_prot = &tcp_prot;
+#if IS_ENABLED(CONFIG_IPV6)
 	else
 		sk->sk_prot = &tcpv6_prot;
+#endif
 	sk->sk_prot->destroy(sk);
 }
 
@@ -629,14 +635,15 @@ static void chtls_reset_synq(struct listen_ctx *listen_ctx)
 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 {
 	struct net_device *ndev;
+#if IS_ENABLED(CONFIG_IPV6)
+	bool clip_valid = false;
+#endif
 	struct listen_ctx *ctx;
 	struct adapter *adap;
 	struct port_info *pi;
-	bool clip_valid;
+	int ret = 0;
 	int stid;
-	int ret;
 
-	clip_valid = false;
 	rcu_read_lock();
 	ndev = chtls_find_netdev(cdev, sk);
 	rcu_read_unlock();
@@ -674,6 +681,7 @@ int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 					  inet_sk(sk)->inet_rcv_saddr,
 					  inet_sk(sk)->inet_sport, 0,
 					  cdev->lldi->rxq_ids[0]);
+#if IS_ENABLED(CONFIG_IPV6)
 	} else {
 		int addr_type;
 
@@ -689,6 +697,7 @@ int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 					   &sk->sk_v6_rcv_saddr,
 					   inet_sk(sk)->inet_sport,
 					   cdev->lldi->rxq_ids[0]);
+#endif
 	}
 	if (ret > 0)
 		ret = net_xmit_errno(ret);
@@ -696,8 +705,10 @@ int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 		goto del_hash;
 	return 0;
 del_hash:
+#if IS_ENABLED(CONFIG_IPV6)
 	if (clip_valid)
 		cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);
+#endif
 	listen_hash_del(cdev, sk);
 free_stid:
 	cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
@@ -711,8 +722,6 @@ free_ctx:
 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
 {
 	struct listen_ctx *listen_ctx;
-	struct chtls_sock *csk;
-	int addr_type = 0;
 	int stid;
 
 	stid = listen_hash_del(cdev, sk);
@@ -725,7 +734,11 @@ void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
 	cxgb4_remove_server(cdev->lldi->ports[0], stid,
 			    cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);
 
+#if IS_ENABLED(CONFIG_IPV6)
 	if (sk->sk_family == PF_INET6) {
+		struct chtls_sock *csk;
+		int addr_type = 0;
+
 		csk = rcu_dereference_sk_user_data(sk);
 		addr_type = ipv6_addr_type((const struct in6_addr *)
 					  &sk->sk_v6_rcv_saddr);
@@ -733,6 +746,7 @@ void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
 			cxgb4_clip_release(csk->egress_dev, (const u32 *)
 					   &sk->sk_v6_rcv_saddr, 1);
 	}
+#endif
 	chtls_disconnect_acceptq(sk);
 }
 
@@ -941,9 +955,11 @@ static unsigned int chtls_select_mss(const struct chtls_sock *csk,
 	tp = tcp_sk(sk);
 	tcpoptsz = 0;
 
+#if IS_ENABLED(CONFIG_IPV6)
 	if (sk->sk_family == AF_INET6)
 		iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
 	else
+#endif
 		iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
 	if (req->tcpopt.tstamp)
 		tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
@@ -1091,13 +1107,13 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
 				    const struct cpl_pass_accept_req *req,
 				    struct chtls_dev *cdev)
 {
+	struct neighbour *n = NULL;
 	struct inet_sock *newinet;
 	const struct iphdr *iph;
 	struct tls_context *ctx;
 	struct net_device *ndev;
 	struct chtls_sock *csk;
 	struct dst_entry *dst;
-	struct neighbour *n;
 	struct tcp_sock *tp;
 	struct sock *newsk;
 	u16 port_id;
@@ -1115,6 +1131,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
 			goto free_sk;
 
 		n = dst_neigh_lookup(dst, &iph->saddr);
+#if IS_ENABLED(CONFIG_IPV6)
 	} else {
 		const struct ipv6hdr *ip6h;
 		struct flowi6 fl6;
@@ -1131,6 +1148,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
 		if (IS_ERR(dst))
 			goto free_sk;
 		n = dst_neigh_lookup(dst, &ip6h->saddr);
+#endif
 	}
 	if (!n)
 		goto free_sk;
@@ -1158,6 +1176,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
 		newinet->inet_daddr = iph->saddr;
 		newinet->inet_rcv_saddr = iph->daddr;
 		newinet->inet_saddr = iph->daddr;
+#if IS_ENABLED(CONFIG_IPV6)
 	} else {
 		struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;
 		struct inet_request_sock *treq = inet_rsk(oreq);
@@ -1175,6 +1194,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
 		newinet->inet_opt = NULL;
 		newinet->inet_daddr = LOOPBACK4_IPV6;
 		newinet->inet_saddr = LOOPBACK4_IPV6;
+#endif
 	}
 
 	oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
@@ -1337,10 +1357,12 @@ static void chtls_pass_accept_request(struct sock *sk,
 	if (iph->version == 0x4) {
 		chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
 		ip_dsfield = ipv4_get_dsfield(iph);
+#if IS_ENABLED(CONFIG_IPV6)
 	} else {
 		inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
 		inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
 		ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));
+#endif
 	}
 	if (req->tcpopt.wsf <= 14 &&
 	    sock_net(sk)->ipv4.sysctl_tcp_window_scaling) {
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c
index 7dfffdde9593..d98b89d0fa6e 100644
--- a/drivers/crypto/chelsio/chtls/chtls_main.c
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -608,9 +608,11 @@ static void __init chtls_init_ulp_ops(void)
 	chtls_cpl_prot.recvmsg		= chtls_recvmsg;
 	chtls_cpl_prot.setsockopt	= chtls_setsockopt;
 	chtls_cpl_prot.getsockopt	= chtls_getsockopt;
+#if IS_ENABLED(CONFIG_IPV6)
 	chtls_cpl_protv6		= chtls_cpl_prot;
 	chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6,
 			   &tcpv6_prot, PF_INET6);
+#endif
 }
 
 static int __init chtls_register(void)
diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile
index 015155da59c2..bc89a20e5d9d 100644
--- a/drivers/crypto/nx/Makefile
+++ b/drivers/crypto/nx/Makefile
@@ -15,4 +15,4 @@ obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compres
 obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o
 nx-compress-objs := nx-842.o
 nx-compress-pseries-objs := nx-842-pseries.o
-nx-compress-powernv-objs := nx-842-powernv.o
+nx-compress-powernv-objs := nx-common-powernv.o
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
index c037a2403b82..13c65deda8e9 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * Driver for IBM PowerNV 842 compression accelerator
+ * Driver for IBM PowerNV compression accelerator
  *
  * Copyright (C) 2015 Dan Streetman, IBM Corp
  */
@@ -20,7 +20,7 @@
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
-MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
+MODULE_DESCRIPTION("H/W Compression driver for IBM PowerNV processors");
 MODULE_ALIAS_CRYPTO("842");
 MODULE_ALIAS_CRYPTO("842-nx");
 
@@ -40,9 +40,9 @@ struct nx842_workmem {
 	char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
 } __packed __aligned(WORKMEM_ALIGN);
 
-struct nx842_coproc {
+struct nx_coproc {
 	unsigned int chip_id;
-	unsigned int ct;
+	unsigned int ct;	/* Can be 842 or GZIP high/normal*/
 	unsigned int ci;	/* Coprocessor instance, used with icswx */
 	struct {
 		struct vas_window *rxwin;
@@ -58,9 +58,16 @@ struct nx842_coproc {
 static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
 
 /* no cpu hotplug on powernv, so this list never changes after init */
-static LIST_HEAD(nx842_coprocs);
+static LIST_HEAD(nx_coprocs);
 static unsigned int nx842_ct;	/* used in icswx function */
 
+/*
+ * Using same values as in skiboot or coprocessor type representing
+ * in NX workbook.
+ */
+#define NX_CT_GZIP	(2)	/* on P9 and later */
+#define NX_CT_842	(3)
+
 static int (*nx842_powernv_exec)(const unsigned char *in,
 				unsigned int inlen, unsigned char *out,
 				unsigned int *outlenp, void *workmem, int fc);
@@ -666,15 +673,15 @@ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
 				      wmem, CCW_FC_842_DECOMP_CRC);
 }
 
-static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
+static inline void nx_add_coprocs_list(struct nx_coproc *coproc,
 					int chipid)
 {
 	coproc->chip_id = chipid;
 	INIT_LIST_HEAD(&coproc->list);
-	list_add(&coproc->list, &nx842_coprocs);
+	list_add(&coproc->list, &nx_coprocs);
 }
 
-static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
+static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc)
 {
 	struct vas_window *txwin = NULL;
 	struct vas_tx_win_attr txattr;
@@ -685,7 +692,6 @@ static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
 	 */
 	vas_init_tx_win_attr(&txattr, coproc->ct);
 	txattr.lpid = 0;	/* lpid is 0 for kernel requests */
-	txattr.pid = 0;		/* pid is 0 for kernel requests */
 
 	/*
 	 * Open a VAS send window which is used to send request to NX.
@@ -704,9 +710,9 @@ static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
  * cpu_txwin is used in copy/paste operation for each compression /
  * decompression request.
  */
-static int nx842_open_percpu_txwins(void)
+static int nx_open_percpu_txwins(void)
 {
-	struct nx842_coproc *coproc, *n;
+	struct nx_coproc *coproc, *n;
 	unsigned int i, chip_id;
 
 	for_each_possible_cpu(i) {
@@ -714,17 +720,18 @@ static int nx842_open_percpu_txwins(void)
 
 		chip_id = cpu_to_chip_id(i);
 
-		list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+		list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
 			/*
 			 * Kernel requests use only high priority FIFOs. So
 			 * open send windows for these FIFOs.
+			 * GZIP is not supported in kernel right now.
 			 */
 
 			if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
 				continue;
 
 			if (coproc->chip_id == chip_id) {
-				txwin = nx842_alloc_txwin(coproc);
+				txwin = nx_alloc_txwin(coproc);
 				if (IS_ERR(txwin))
 					return PTR_ERR(txwin);
 
@@ -743,13 +750,28 @@ static int nx842_open_percpu_txwins(void)
 	return 0;
 }
 
+static int __init nx_set_ct(struct nx_coproc *coproc, const char *priority,
+				int high, int normal)
+{
+	if (!strcmp(priority, "High"))
+		coproc->ct = high;
+	else if (!strcmp(priority, "Normal"))
+		coproc->ct = normal;
+	else {
+		pr_err("Invalid RxFIFO priority value\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
-					int vasid, int *ct)
+					int vasid, int type, int *ct)
 {
 	struct vas_window *rxwin = NULL;
 	struct vas_rx_win_attr rxattr;
-	struct nx842_coproc *coproc;
 	u32 lpid, pid, tid, fifo_size;
+	struct nx_coproc *coproc;
 	u64 rx_fifo;
 	const char *priority;
 	int ret;
@@ -794,15 +816,15 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
 	if (!coproc)
 		return -ENOMEM;
 
-	if (!strcmp(priority, "High"))
-		coproc->ct = VAS_COP_TYPE_842_HIPRI;
-	else if (!strcmp(priority, "Normal"))
-		coproc->ct = VAS_COP_TYPE_842;
-	else {
-		pr_err("Invalid RxFIFO priority value\n");
-		ret =  -EINVAL;
+	if (type == NX_CT_842)
+		ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI,
+			VAS_COP_TYPE_842);
+	else if (type == NX_CT_GZIP)
+		ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_GZIP_HIPRI,
+				VAS_COP_TYPE_GZIP);
+
+	if (ret)
 		goto err_out;
-	}
 
 	vas_init_rx_win_attr(&rxattr, coproc->ct);
 	rxattr.rx_fifo = (void *)rx_fifo;
@@ -830,7 +852,7 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
 
 	coproc->vas.rxwin = rxwin;
 	coproc->vas.id = vasid;
-	nx842_add_coprocs_list(coproc, chip_id);
+	nx_add_coprocs_list(coproc, chip_id);
 
 	/*
 	 * (lpid, pid, tid) combination has to be unique for each
@@ -848,13 +870,47 @@ err_out:
 	return ret;
 }
 
+static int __init nx_coproc_init(int chip_id, int ct_842, int ct_gzip)
+{
+	int ret = 0;
+
+	if (opal_check_token(OPAL_NX_COPROC_INIT)) {
+		ret = opal_nx_coproc_init(chip_id, ct_842);
+
+		if (!ret)
+			ret = opal_nx_coproc_init(chip_id, ct_gzip);
+
+		if (ret) {
+			ret = opal_error_code(ret);
+			pr_err("Failed to initialize NX for chip(%d): %d\n",
+				chip_id, ret);
+		}
+	} else
+		pr_warn("Firmware doesn't support NX initialization\n");
+
+	return ret;
+}
+
+static int __init find_nx_device_tree(struct device_node *dn, int chip_id,
+					int vasid, int type, char *devname,
+					int *ct)
+{
+	int ret = 0;
+
+	if (of_device_is_compatible(dn, devname)) {
+		ret  = vas_cfg_coproc_info(dn, chip_id, vasid, type, ct);
+		if (ret)
+			of_node_put(dn);
+	}
+
+	return ret;
+}
 
-static int __init nx842_powernv_probe_vas(struct device_node *pn)
+static int __init nx_powernv_probe_vas(struct device_node *pn)
 {
-	struct device_node *dn;
 	int chip_id, vasid, ret = 0;
-	int nx_fifo_found = 0;
-	int uninitialized_var(ct);
+	int ct_842 = 0, ct_gzip = 0;
+	struct device_node *dn;
 
 	chip_id = of_get_ibm_chip_id(pn);
 	if (chip_id < 0) {
@@ -869,40 +925,33 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn)
 	}
 
 	for_each_child_of_node(pn, dn) {
-		if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
-			ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct);
-			if (ret) {
-				of_node_put(dn);
-				return ret;
-			}
-			nx_fifo_found++;
-		}
+		ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842,
+					"ibm,p9-nx-842", &ct_842);
+
+		if (!ret)
+			ret = find_nx_device_tree(dn, chip_id, vasid,
+				NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip);
+
+		if (ret)
+			return ret;
 	}
 
-	if (!nx_fifo_found) {
-		pr_err("NX842 FIFO nodes are missing\n");
+	if (!ct_842 || !ct_gzip) {
+		pr_err("NX FIFO nodes are missing\n");
 		return -EINVAL;
 	}
 
 	/*
 	 * Initialize NX instance for both high and normal priority FIFOs.
 	 */
-	if (opal_check_token(OPAL_NX_COPROC_INIT)) {
-		ret = opal_nx_coproc_init(chip_id, ct);
-		if (ret) {
-			pr_err("Failed to initialize NX for chip(%d): %d\n",
-				chip_id, ret);
-			ret = opal_error_code(ret);
-		}
-	} else
-		pr_warn("Firmware doesn't support NX initialization\n");
+	ret = nx_coproc_init(chip_id, ct_842, ct_gzip);
 
 	return ret;
 }
 
 static int __init nx842_powernv_probe(struct device_node *dn)
 {
-	struct nx842_coproc *coproc;
+	struct nx_coproc *coproc;
 	unsigned int ct, ci;
 	int chip_id;
 
@@ -922,13 +971,13 @@ static int __init nx842_powernv_probe(struct device_node *dn)
 		return -EINVAL;
 	}
 
-	coproc = kmalloc(sizeof(*coproc), GFP_KERNEL);
+	coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
 	if (!coproc)
 		return -ENOMEM;
 
 	coproc->ct = ct;
 	coproc->ci = ci;
-	nx842_add_coprocs_list(coproc, chip_id);
+	nx_add_coprocs_list(coproc, chip_id);
 
 	pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
 
@@ -941,9 +990,9 @@ static int __init nx842_powernv_probe(struct device_node *dn)
 	return 0;
 }
 
-static void nx842_delete_coprocs(void)
+static void nx_delete_coprocs(void)
 {
-	struct nx842_coproc *coproc, *n;
+	struct nx_coproc *coproc, *n;
 	struct vas_window *txwin;
 	int i;
 
@@ -955,10 +1004,10 @@ static void nx842_delete_coprocs(void)
 		if (txwin)
 			vas_win_close(txwin);
 
-		per_cpu(cpu_txwin, i) = 0;
+		per_cpu(cpu_txwin, i) = NULL;
 	}
 
-	list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+	list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
 		if (coproc->vas.rxwin)
 			vas_win_close(coproc->vas.rxwin);
 
@@ -1002,7 +1051,7 @@ static struct crypto_alg nx842_powernv_alg = {
 	.coa_decompress		= nx842_crypto_decompress } }
 };
 
-static __init int nx842_powernv_init(void)
+static __init int nx_compress_powernv_init(void)
 {
 	struct device_node *dn;
 	int ret;
@@ -1017,15 +1066,15 @@ static __init int nx842_powernv_init(void)
 	BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
 
 	for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
-		ret = nx842_powernv_probe_vas(dn);
+		ret = nx_powernv_probe_vas(dn);
 		if (ret) {
-			nx842_delete_coprocs();
+			nx_delete_coprocs();
 			of_node_put(dn);
 			return ret;
 		}
 	}
 
-	if (list_empty(&nx842_coprocs)) {
+	if (list_empty(&nx_coprocs)) {
 		for_each_compatible_node(dn, NULL, "ibm,power-nx")
 			nx842_powernv_probe(dn);
 
@@ -1034,9 +1083,25 @@ static __init int nx842_powernv_init(void)
 
 		nx842_powernv_exec = nx842_exec_icswx;
 	} else {
-		ret = nx842_open_percpu_txwins();
+		/*
+		 * Register VAS user space API for NX GZIP so
+		 * that user space can use GZIP engine.
+		 * Using high FIFO priority for kernel requests and
+		 * normal FIFO priority is assigned for userspace.
+		 * 842 compression is supported only in kernel.
+		 */
+		ret = vas_register_coproc_api(THIS_MODULE, VAS_COP_TYPE_GZIP,
+						"nx-gzip");
+
+		/*
+		 * GZIP is not supported in kernel right now.
+		 * So open tx windows only for 842.
+		 */
+		if (!ret)
+			ret = nx_open_percpu_txwins();
+
 		if (ret) {
-			nx842_delete_coprocs();
+			nx_delete_coprocs();
 			return ret;
 		}
 
@@ -1045,18 +1110,27 @@ static __init int nx842_powernv_init(void)
 
 	ret = crypto_register_alg(&nx842_powernv_alg);
 	if (ret) {
-		nx842_delete_coprocs();
+		nx_delete_coprocs();
 		return ret;
 	}
 
 	return 0;
 }
-module_init(nx842_powernv_init);
+module_init(nx_compress_powernv_init);
 
-static void __exit nx842_powernv_exit(void)
+static void __exit nx_compress_powernv_exit(void)
 {
+	/*
+	 * GZIP engine is supported only in power9 or later and nx842_ct
+	 * is used on power8 (icswx).
+	 * VAS API for NX GZIP is registered during init for user space
+	 * use. So delete this API use for GZIP engine.
+	 */
+	if (!nx842_ct)
+		vas_unregister_coproc_api();
+
 	crypto_unregister_alg(&nx842_powernv_alg);
 
-	nx842_delete_coprocs();
+	nx_delete_coprocs();
 }
-module_exit(nx842_powernv_exit);
+module_exit(nx_compress_powernv_exit);
diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c
index 32dc00dc570b..9f937bdc53a7 100644
--- a/drivers/crypto/omap-aes-gcm.c
+++ b/drivers/crypto/omap-aes-gcm.c
@@ -77,7 +77,6 @@ static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
 		tag = (u8 *)rctx->auth_tag;
 		for (i = 0; i < dd->authsize; i++) {
 			if (tag[i]) {
-				dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
 				ret = -EBADMSG;
 			}
 		}
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 824ddf2a66ff..b5aff20c5900 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -1269,13 +1269,17 @@ static int omap_aes_remove(struct platform_device *pdev)
 	spin_unlock(&list_lock);
 
 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
+		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
 			crypto_unregister_skcipher(
 					&dd->pdata->algs_info[i].algs_list[j]);
+			dd->pdata->algs_info[i].registered--;
+		}
 
-	for (i = dd->pdata->aead_algs_info->size - 1; i >= 0; i--) {
+	for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
 		aalg = &dd->pdata->aead_algs_info->algs_list[i];
 		crypto_unregister_aead(aalg);
+		dd->pdata->aead_algs_info->registered--;
+
 	}
 
 	crypto_engine_exit(dd->engine);
diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c
index cc88b7362bc2..94b2dba90f0d 100644
--- a/drivers/crypto/omap-crypto.c
+++ b/drivers/crypto/omap-crypto.c
@@ -178,11 +178,17 @@ static void omap_crypto_copy_data(struct scatterlist *src,
 		amt = min(src->length - srco, dst->length - dsto);
 		amt = min(len, amt);
 
-		srcb = sg_virt(src) + srco;
-		dstb = sg_virt(dst) + dsto;
+		srcb = kmap_atomic(sg_page(src)) + srco + src->offset;
+		dstb = kmap_atomic(sg_page(dst)) + dsto + dst->offset;
 
 		memcpy(dstb, srcb, amt);
 
+		if (!PageSlab(sg_page(dst)))
+			flush_kernel_dcache_page(sg_page(dst));
+
+		kunmap_atomic(srcb);
+		kunmap_atomic(dstb);
+
 		srco += amt;
 		dsto += amt;
 		len -= amt;
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 063ad5d03f33..82691a057d2a 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -168,8 +168,6 @@ struct omap_sham_hmac_ctx {
 };
 
 struct omap_sham_ctx {
-	struct omap_sham_dev	*dd;
-
 	unsigned long		flags;
 
 	/* fallback stuff */
@@ -750,8 +748,17 @@ static int omap_sham_align_sgs(struct scatterlist *sg,
 	int offset = rctx->offset;
 	int bufcnt = rctx->bufcnt;
 
-	if (!sg || !sg->length || !nbytes)
+	if (!sg || !sg->length || !nbytes) {
+		if (bufcnt) {
+			bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs;
+			sg_init_table(rctx->sgl, 1);
+			sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt);
+			rctx->sg = rctx->sgl;
+			rctx->sg_len = 1;
+		}
+
 		return 0;
+	}
 
 	new_len = nbytes;
 
@@ -895,7 +902,7 @@ static int omap_sham_prepare_request(struct ahash_request *req, bool update)
 	if (hash_later < 0)
 		hash_later = 0;
 
-	if (hash_later) {
+	if (hash_later && hash_later <= rctx->buflen) {
 		scatterwalk_map_and_copy(rctx->buffer,
 					 req->src,
 					 req->nbytes - hash_later,
@@ -925,27 +932,35 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
 	return 0;
 }
 
+struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx)
+{
+	struct omap_sham_dev *dd;
+
+	if (ctx->dd)
+		return ctx->dd;
+
+	spin_lock_bh(&sham.lock);
+	dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list);
+	list_move_tail(&dd->list, &sham.dev_list);
+	ctx->dd = dd;
+	spin_unlock_bh(&sham.lock);
+
+	return dd;
+}
+
 static int omap_sham_init(struct ahash_request *req)
 {
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
-	struct omap_sham_dev *dd = NULL, *tmp;
+	struct omap_sham_dev *dd;
 	int bs = 0;
 
-	spin_lock_bh(&sham.lock);
-	if (!tctx->dd) {
-		list_for_each_entry(tmp, &sham.dev_list, list) {
-			dd = tmp;
-			break;
-		}
-		tctx->dd = dd;
-	} else {
-		dd = tctx->dd;
-	}
-	spin_unlock_bh(&sham.lock);
+	ctx->dd = NULL;
 
-	ctx->dd = dd;
+	dd = omap_sham_find_dev(ctx);
+	if (!dd)
+		return -ENODEV;
 
 	ctx->flags = 0;
 
@@ -1215,8 +1230,7 @@ err1:
 static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
 {
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
-	struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
-	struct omap_sham_dev *dd = tctx->dd;
+	struct omap_sham_dev *dd = ctx->dd;
 
 	ctx->op = op;
 
@@ -1226,7 +1240,7 @@ static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
 static int omap_sham_update(struct ahash_request *req)
 {
 	struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
-	struct omap_sham_dev *dd = ctx->dd;
+	struct omap_sham_dev *dd = omap_sham_find_dev(ctx);
 
 	if (!req->nbytes)
 		return 0;
@@ -1319,21 +1333,8 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
 	struct omap_sham_hmac_ctx *bctx = tctx->base;
 	int bs = crypto_shash_blocksize(bctx->shash);
 	int ds = crypto_shash_digestsize(bctx->shash);
-	struct omap_sham_dev *dd = NULL, *tmp;
 	int err, i;
 
-	spin_lock_bh(&sham.lock);
-	if (!tctx->dd) {
-		list_for_each_entry(tmp, &sham.dev_list, list) {
-			dd = tmp;
-			break;
-		}
-		tctx->dd = dd;
-	} else {
-		dd = tctx->dd;
-	}
-	spin_unlock_bh(&sham.lock);
-
 	err = crypto_shash_setkey(tctx->fallback, key, keylen);
 	if (err)
 		return err;
@@ -1350,7 +1351,7 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
 
 	memset(bctx->ipad + keylen, 0, bs - keylen);
 
-	if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
+	if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) {
 		memcpy(bctx->opad, bctx->ipad, bs);
 
 		for (i = 0; i < bs; i++) {
@@ -1571,7 +1572,8 @@ static struct ahash_alg algs_sha224_sha256[] = {
 		.cra_name		= "sha224",
 		.cra_driver_name	= "omap-sha224",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA224_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
@@ -1592,7 +1594,8 @@ static struct ahash_alg algs_sha224_sha256[] = {
 		.cra_name		= "sha256",
 		.cra_driver_name	= "omap-sha256",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA256_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
@@ -1614,7 +1617,8 @@ static struct ahash_alg algs_sha224_sha256[] = {
 		.cra_name		= "hmac(sha224)",
 		.cra_driver_name	= "omap-hmac-sha224",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA224_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
@@ -1637,7 +1641,8 @@ static struct ahash_alg algs_sha224_sha256[] = {
 		.cra_name		= "hmac(sha256)",
 		.cra_driver_name	= "omap-hmac-sha256",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA256_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
@@ -1662,7 +1667,8 @@ static struct ahash_alg algs_sha384_sha512[] = {
 		.cra_name		= "sha384",
 		.cra_driver_name	= "omap-sha384",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA384_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
@@ -1683,7 +1689,8 @@ static struct ahash_alg algs_sha384_sha512[] = {
 		.cra_name		= "sha512",
 		.cra_driver_name	= "omap-sha512",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA512_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx),
@@ -1705,7 +1712,8 @@ static struct ahash_alg algs_sha384_sha512[] = {
 		.cra_name		= "hmac(sha384)",
 		.cra_driver_name	= "omap-hmac-sha384",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA384_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
@@ -1728,7 +1736,8 @@ static struct ahash_alg algs_sha384_sha512[] = {
 		.cra_name		= "hmac(sha512)",
 		.cra_driver_name	= "omap-hmac-sha512",
 		.cra_priority		= 400,
-		.cra_flags		= CRYPTO_ALG_ASYNC |
+		.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC |
 						CRYPTO_ALG_NEED_FALLBACK,
 		.cra_blocksize		= SHA512_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct omap_sham_ctx) +
@@ -2154,6 +2163,7 @@ static int omap_sham_probe(struct platform_device *pdev)
 	}
 
 	dd->flags |= dd->pdata->flags;
+	sham.flags |= dd->pdata->flags;
 
 	pm_runtime_use_autosuspend(dev);
 	pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
@@ -2181,6 +2191,9 @@ static int omap_sham_probe(struct platform_device *pdev)
 	spin_unlock(&sham.lock);
 
 	for (i = 0; i < dd->pdata->algs_info_size; i++) {
+		if (dd->pdata->algs_info[i].registered)
+			break;
+
 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
 			struct ahash_alg *alg;
 
@@ -2232,9 +2245,11 @@ static int omap_sham_remove(struct platform_device *pdev)
 	list_del(&dd->list);
 	spin_unlock(&sham.lock);
 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
+		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
 			crypto_unregister_ahash(
 					&dd->pdata->algs_info[i].algs_list[j]);
+			dd->pdata->algs_info[i].registered--;
+		}
 	tasklet_kill(&dd->done_task);
 	pm_runtime_disable(&pdev->dev);
 
diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c
index fd045e64972a..cb8a6ea2a4bc 100644
--- a/drivers/crypto/virtio/virtio_crypto_algs.c
+++ b/drivers/crypto/virtio/virtio_crypto_algs.c
@@ -350,13 +350,18 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
 	int err;
 	unsigned long flags;
 	struct scatterlist outhdr, iv_sg, status_sg, **sgs;
-	int i;
 	u64 dst_len;
 	unsigned int num_out = 0, num_in = 0;
 	int sg_total;
 	uint8_t *iv;
+	struct scatterlist *sg;
 
 	src_nents = sg_nents_for_len(req->src, req->cryptlen);
+	if (src_nents < 0) {
+		pr_err("Invalid number of src SG.\n");
+		return src_nents;
+	}
+
 	dst_nents = sg_nents(req->dst);
 
 	pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n",
@@ -402,6 +407,7 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
 		goto free;
 	}
 
+	dst_len = min_t(unsigned int, req->cryptlen, dst_len);
 	pr_debug("virtio_crypto: src_len: %u, dst_len: %llu\n",
 			req->cryptlen, dst_len);
 
@@ -442,12 +448,12 @@ __virtio_crypto_skcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
 	vc_sym_req->iv = iv;
 
 	/* Source data */
-	for (i = 0; i < src_nents; i++)
-		sgs[num_out++] = &req->src[i];
+	for (sg = req->src; src_nents; sg = sg_next(sg), src_nents--)
+		sgs[num_out++] = sg;
 
 	/* Destination data */
-	for (i = 0; i < dst_nents; i++)
-		sgs[num_out + num_in++] = &req->dst[i];
+	for (sg = req->dst; sg; sg = sg_next(sg))
+		sgs[num_out + num_in++] = sg;
 
 	/* Status */
 	sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status));
@@ -577,10 +583,11 @@ static void virtio_crypto_skcipher_finalize_req(
 		scatterwalk_map_and_copy(req->iv, req->dst,
 					 req->cryptlen - AES_BLOCK_SIZE,
 					 AES_BLOCK_SIZE, 0);
-	crypto_finalize_skcipher_request(vc_sym_req->base.dataq->engine,
-					   req, err);
 	kzfree(vc_sym_req->iv);
 	virtcrypto_clear_request(&vc_sym_req->base);
+
+	crypto_finalize_skcipher_request(vc_sym_req->base.dataq->engine,
+					   req, err);
 }
 
 static struct virtio_crypto_algo virtio_crypto_algs[] = { {
diff --git a/drivers/crypto/xilinx/zynqmp-aes-gcm.c b/drivers/crypto/xilinx/zynqmp-aes-gcm.c
index 09f7f468eef8..cd11558893cd 100644
--- a/drivers/crypto/xilinx/zynqmp-aes-gcm.c
+++ b/drivers/crypto/xilinx/zynqmp-aes-gcm.c
@@ -46,7 +46,6 @@ struct zynqmp_aead_drv_ctx {
 	} alg;
 	struct device *dev;
 	struct crypto_engine *engine;
-	const struct zynqmp_eemi_ops *eemi_ops;
 };
 
 struct zynqmp_aead_hw_req {
@@ -80,21 +79,15 @@ static int zynqmp_aes_aead_cipher(struct aead_request *req)
 	struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
 	struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req);
 	struct device *dev = tfm_ctx->dev;
-	struct aead_alg *alg = crypto_aead_alg(aead);
-	struct zynqmp_aead_drv_ctx *drv_ctx;
 	struct zynqmp_aead_hw_req *hwreq;
 	dma_addr_t dma_addr_data, dma_addr_hw_req;
 	unsigned int data_size;
 	unsigned int status;
+	int ret;
 	size_t dma_size;
 	char *kbuf;
 	int err;
 
-	drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead);
-
-	if (!drv_ctx->eemi_ops->aes)
-		return -ENOTSUPP;
-
 	if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY)
 		dma_size = req->cryptlen + ZYNQMP_AES_KEY_SIZE
 			   + GCM_AES_IV_SIZE;
@@ -136,9 +129,12 @@ static int zynqmp_aes_aead_cipher(struct aead_request *req)
 		hwreq->key = 0;
 	}
 
-	drv_ctx->eemi_ops->aes(dma_addr_hw_req, &status);
+	ret = zynqmp_pm_aes_engine(dma_addr_hw_req, &status);
 
-	if (status) {
+	if (ret) {
+		dev_err(dev, "ERROR: AES PM API failed\n");
+		err = ret;
+	} else if (status) {
 		switch (status) {
 		case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR:
 			dev_err(dev, "ERROR: Gcm Tag mismatch\n");
@@ -388,12 +384,6 @@ static int zynqmp_aes_aead_probe(struct platform_device *pdev)
 	else
 		return -ENODEV;
 
-	aes_drv_ctx.eemi_ops = zynqmp_pm_get_eemi_ops();
-	if (IS_ERR(aes_drv_ctx.eemi_ops)) {
-		dev_err(dev, "Failed to get ZynqMP EEMI interface\n");
-		return PTR_ERR(aes_drv_ctx.eemi_ops);
-	}
-
 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(ZYNQMP_DMA_BIT_MASK));
 	if (err < 0) {
 		dev_err(dev, "No usable DMA configuration\n");
diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h
index 3107ce80e809..16850d5388ab 100644
--- a/drivers/dax/dax-private.h
+++ b/drivers/dax/dax-private.h
@@ -44,6 +44,7 @@ struct dax_region {
  * @dev - device core
  * @pgmap - pgmap for memmap setup / lifetime (driver owned)
  * @dax_mem_res: physical address range of hotadded DAX memory
+ * @dax_mem_name: name for hotadded DAX memory via add_memory_driver_managed()
  */
 struct dev_dax {
 	struct dax_region *region;
diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
index 1e678bdf5aed..275aa5f87399 100644
--- a/drivers/dax/kmem.c
+++ b/drivers/dax/kmem.c
@@ -14,6 +14,11 @@
 #include "dax-private.h"
 #include "bus.h"
 
+/* Memory resource name used for add_memory_driver_managed(). */
+static const char *kmem_name;
+/* Set if any memory will remain added when the driver will be unloaded. */
+static bool any_hotremove_failed;
+
 int dev_dax_kmem_probe(struct device *dev)
 {
 	struct dev_dax *dev_dax = to_dev_dax(dev);
@@ -70,7 +75,12 @@ int dev_dax_kmem_probe(struct device *dev)
 	 */
 	new_res->flags = IORESOURCE_SYSTEM_RAM;
 
-	rc = add_memory(numa_node, new_res->start, resource_size(new_res));
+	/*
+	 * Ensure that future kexec'd kernels will not treat this as RAM
+	 * automatically.
+	 */
+	rc = add_memory_driver_managed(numa_node, new_res->start,
+				       resource_size(new_res), kmem_name);
 	if (rc) {
 		release_resource(new_res);
 		kfree(new_res);
@@ -100,6 +110,7 @@ static int dev_dax_kmem_remove(struct device *dev)
 	 */
 	rc = remove_memory(dev_dax->target_node, kmem_start, kmem_size);
 	if (rc) {
+		any_hotremove_failed = true;
 		dev_err(dev,
 			"DAX region %pR cannot be hotremoved until the next reboot\n",
 			res);
@@ -124,6 +135,7 @@ static int dev_dax_kmem_remove(struct device *dev)
 	 * permanently pinned as reserved by the unreleased
 	 * request_mem_region().
 	 */
+	any_hotremove_failed = true;
 	return 0;
 }
 #endif /* CONFIG_MEMORY_HOTREMOVE */
@@ -137,12 +149,24 @@ static struct dax_device_driver device_dax_kmem_driver = {
 
 static int __init dax_kmem_init(void)
 {
-	return dax_driver_register(&device_dax_kmem_driver);
+	int rc;
+
+	/* Resource name is permanently allocated if any hotremove fails. */
+	kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL);
+	if (!kmem_name)
+		return -ENOMEM;
+
+	rc = dax_driver_register(&device_dax_kmem_driver);
+	if (rc)
+		kfree_const(kmem_name);
+	return rc;
 }
 
 static void __exit dax_kmem_exit(void)
 {
 	dax_driver_unregister(&device_dax_kmem_driver);
+	if (!any_hotremove_failed)
+		kfree_const(kmem_name);
 }
 
 MODULE_AUTHOR("Intel Corporation");
diff --git a/drivers/dca/dca-sysfs.c b/drivers/dca/dca-sysfs.c
index eb25627b059d..21ebd0af268b 100644
--- a/drivers/dca/dca-sysfs.c
+++ b/drivers/dca/dca-sysfs.c
@@ -24,9 +24,7 @@ int dca_sysfs_add_req(struct dca_provider *dca, struct device *dev, int slot)
 
 	cd = device_create(dca_class, dca->cd, MKDEV(0, slot + 1), NULL,
 			   "requester%d", req_count++);
-	if (IS_ERR(cd))
-		return PTR_ERR(cd);
-	return 0;
+	return PTR_ERR_OR_ZERO(cd);
 }
 
 void dca_sysfs_remove_req(struct dca_provider *dca, int slot)
diff --git a/drivers/dio/dio.c b/drivers/dio/dio.c
index c9aa15fb86a9..193b40e7aec0 100644
--- a/drivers/dio/dio.c
+++ b/drivers/dio/dio.c
@@ -135,7 +135,8 @@ int __init dio_find(int deviceid)
 		else
 			va = ioremap(pa, PAGE_SIZE);
 
-                if (probe_kernel_read(&i, (unsigned char *)va + DIO_IDOFF, 1)) {
+		if (copy_from_kernel_nofault(&i,
+				(unsigned char *)va + DIO_IDOFF, 1)) {
 			if (scode >= DIOII_SCBASE)
 				iounmap(va);
                         continue;             /* no board present at that select code */
@@ -208,7 +209,8 @@ static int __init dio_init(void)
 		else
 			va = ioremap(pa, PAGE_SIZE);
 
-                if (probe_kernel_read(&i, (unsigned char *)va + DIO_IDOFF, 1)) {
+		if (copy_from_kernel_nofault(&i,
+				(unsigned char *)va + DIO_IDOFF, 1)) {
 			if (scode >= DIOII_SCBASE)
 				iounmap(va);
                         continue;              /* no board present at that select code */
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index 9626673f1d83..4f8224a6ac95 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -5,7 +5,7 @@ config SYNC_FILE
 	bool "Explicit Synchronization Framework"
 	default n
 	select DMA_SHARED_BUFFER
-	---help---
+	help
 	  The Sync File Framework adds explicit synchronization via
 	  userspace. It enables send/receive 'struct dma_fence' objects to/from
 	  userspace via Sync File fds for synchronization between drivers via
@@ -22,7 +22,7 @@ config SW_SYNC
 	default n
 	depends on SYNC_FILE
 	depends on DEBUG_FS
-	---help---
+	help
 	  A sync object driver that uses a 32bit counter to coordinate
 	  synchronization.  Useful when there is no hardware primitive backing
 	  the synchronization.
diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c
index 4264e64788c4..b45f8514dc82 100644
--- a/drivers/dma-buf/dma-resv.c
+++ b/drivers/dma-buf/dma-resv.c
@@ -34,6 +34,7 @@
 
 #include <linux/dma-resv.h>
 #include <linux/export.h>
+#include <linux/mm.h>
 #include <linux/sched/mm.h>
 
 /**
@@ -109,7 +110,7 @@ static int __init dma_resv_lockdep(void)
 
 	dma_resv_init(&obj);
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	ww_acquire_init(&ctx, &reservation_ww_class);
 	ret = dma_resv_lock(&obj, &ctx);
 	if (ret == -EDEADLK)
@@ -118,7 +119,7 @@ static int __init dma_resv_lockdep(void)
 	fs_reclaim_release(GFP_KERNEL);
 	ww_mutex_unlock(&obj.lock);
 	ww_acquire_fini(&ctx);
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	
 	mmput(mm);
 
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 023db6883d05..de41d7928bff 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -106,7 +106,7 @@ config AXI_DMAC
 	select REGMAP_MMIO
 	help
 	  Enable support for the Analog Devices AXI-DMAC peripheral. This DMA
-	  controller is often used in Analog Device's reference designs for FPGA
+	  controller is often used in Analog Devices' reference designs for FPGA
 	  platforms.
 
 config BCM_SBA_RAID
@@ -198,7 +198,7 @@ config FSL_DMA
 	depends on FSL_SOC
 	select DMA_ENGINE
 	select ASYNC_TX_ENABLE_CHANNEL_SWITCH
-	---help---
+	help
 	  Enable support for the Freescale Elo series DMA controllers.
 	  The Elo is the DMA controller on some mpc82xx and mpc83xx parts, the
 	  EloPlus is on mpc85xx and mpc86xx and Pxxx parts, and the Elo3 is on
@@ -233,7 +233,7 @@ config FSL_RAID
 	depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH
 	select DMA_ENGINE
 	select DMA_ENGINE_RAID
-	---help---
+	help
 	  Enable support for Freescale RAID Engine. RAID Engine is
 	  available on some QorIQ SoCs (like P5020/P5040). It has
 	  the capability to offload memcpy, xor and pq computation
@@ -395,12 +395,10 @@ config MMP_TDMA
 	bool "MMP Two-Channel DMA support"
 	depends on ARCH_MMP || COMPILE_TEST
 	select DMA_ENGINE
-	select MMP_SRAM if ARCH_MMP
 	select GENERIC_ALLOCATOR
 	help
 	  Support the MMP Two-Channel DMA engine.
 	  This engine used for MMP Audio DMA and pxa910 SQU.
-	  It needs sram driver under mach-mmp.
 
 config MOXART_DMA
 	tristate "MOXART DMA support"
@@ -416,7 +414,7 @@ config MPC512X_DMA
 	tristate "Freescale MPC512x built-in DMA engine support"
 	depends on PPC_MPC512x || PPC_MPC831x
 	select DMA_ENGINE
-	---help---
+	help
 	  Enable support for the Freescale MPC512x built-in DMA engine.
 
 config MV_XOR
@@ -425,7 +423,7 @@ config MV_XOR
 	select DMA_ENGINE
 	select DMA_ENGINE_RAID
 	select ASYNC_TX_ENABLE_CHANNEL_SWITCH
-	---help---
+	help
 	  Enable support for the Marvell XOR engine.
 
 config MV_XOR_V2
@@ -435,7 +433,7 @@ config MV_XOR_V2
 	select DMA_ENGINE_RAID
 	select ASYNC_TX_ENABLE_CHANNEL_SWITCH
 	select GENERIC_MSI_IRQ_DOMAIN
-	---help---
+	help
 	  Enable support for the Marvell version 2 XOR engine.
 
 	  This engine provides acceleration for copy, XOR and RAID6
diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h
index 397692e937b3..80fc2fe8c77e 100644
--- a/drivers/dma/at_hdmac_regs.h
+++ b/drivers/dma/at_hdmac_regs.h
@@ -331,7 +331,7 @@ struct at_dma {
 	struct dma_pool		*dma_desc_pool;
 	struct dma_pool		*memset_pool;
 	/* AT THE END channels table */
-	struct at_dma_chan	chan[0];
+	struct at_dma_chan	chan[];
 };
 
 #define	dma_readl(atdma, name) \
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index bb0eaf38b594..fd92f048c491 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -212,7 +212,7 @@ struct at_xdmac {
 	struct clk		*clk;
 	u32			save_gim;
 	struct dma_pool		*at_xdmac_desc_pool;
-	struct at_xdmac_chan	chan[0];
+	struct at_xdmac_chan	chan[];
 };
 
 
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index d31076d9ef25..2b06a7a8629d 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -53,6 +53,8 @@
 #include <linux/mempool.h>
 #include <linux/numa.h>
 
+#include "dmaengine.h"
+
 static DEFINE_MUTEX(dma_list_mutex);
 static DEFINE_IDA(dma_ida);
 static LIST_HEAD(dma_device_list);
@@ -145,9 +147,9 @@ static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { }
 
 /**
  * dev_to_dma_chan - convert a device pointer to its sysfs container object
- * @dev - device node
+ * @dev:	device node
  *
- * Must be called under dma_list_mutex
+ * Must be called under dma_list_mutex.
  */
 static struct dma_chan *dev_to_dma_chan(struct device *dev)
 {
@@ -243,22 +245,18 @@ static struct class dma_devclass = {
 
 /* --- client and device registration --- */
 
-/**
- * dma_cap_mask_all - enable iteration over all operation types
- */
+/* enable iteration over all operation types */
 static dma_cap_mask_t dma_cap_mask_all;
 
 /**
- * dma_chan_tbl_ent - tracks channel allocations per core/operation
- * @chan - associated channel for this entry
+ * struct dma_chan_tbl_ent - tracks channel allocations per core/operation
+ * @chan:	associated channel for this entry
  */
 struct dma_chan_tbl_ent {
 	struct dma_chan *chan;
 };
 
-/**
- * channel_table - percpu lookup table for memory-to-memory offload providers
- */
+/* percpu lookup table for memory-to-memory offload providers */
 static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];
 
 static int __init dma_channel_table_init(void)
@@ -295,8 +293,11 @@ static int __init dma_channel_table_init(void)
 arch_initcall(dma_channel_table_init);
 
 /**
- * dma_chan_is_local - returns true if the channel is in the same numa-node as
- *	the cpu
+ * dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU
+ * @chan:	DMA channel to test
+ * @cpu:	CPU index which the channel should be close to
+ *
+ * Returns true if the channel is in the same NUMA-node as the CPU.
  */
 static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
 {
@@ -306,14 +307,14 @@ static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
 }
 
 /**
- * min_chan - returns the channel with min count and in the same numa-node as
- *	the cpu
- * @cap: capability to match
- * @cpu: cpu index which the channel should be close to
+ * min_chan - finds the channel with min count and in the same NUMA-node as the CPU
+ * @cap:	capability to match
+ * @cpu:	CPU index which the channel should be close to
  *
- * If some channels are close to the given cpu, the one with the lowest
- * reference count is returned. Otherwise, cpu is ignored and only the
+ * If some channels are close to the given CPU, the one with the lowest
+ * reference count is returned. Otherwise, CPU is ignored and only the
  * reference count is taken into account.
+ *
  * Must be called under dma_list_mutex.
  */
 static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
@@ -351,10 +352,11 @@ static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
 /**
  * dma_channel_rebalance - redistribute the available channels
  *
- * Optimize for cpu isolation (each cpu gets a dedicated channel for an
- * operation type) in the SMP case,  and operation isolation (avoid
- * multi-tasking channels) in the non-SMP case.  Must be called under
- * dma_list_mutex.
+ * Optimize for CPU isolation (each CPU gets a dedicated channel for an
+ * operation type) in the SMP case, and operation isolation (avoid
+ * multi-tasking channels) in the non-SMP case.
+ *
+ * Must be called under dma_list_mutex.
  */
 static void dma_channel_rebalance(void)
 {
@@ -404,9 +406,9 @@ static struct module *dma_chan_to_owner(struct dma_chan *chan)
 
 /**
  * balance_ref_count - catch up the channel reference count
- * @chan - channel to balance ->client_count versus dmaengine_ref_count
+ * @chan:	channel to balance ->client_count versus dmaengine_ref_count
  *
- * balance_ref_count must be called under dma_list_mutex
+ * Must be called under dma_list_mutex.
  */
 static void balance_ref_count(struct dma_chan *chan)
 {
@@ -436,10 +438,10 @@ static void dma_device_put(struct dma_device *device)
 }
 
 /**
- * dma_chan_get - try to grab a dma channel's parent driver module
- * @chan - channel to grab
+ * dma_chan_get - try to grab a DMA channel's parent driver module
+ * @chan:	channel to grab
  *
- * Must be called under dma_list_mutex
+ * Must be called under dma_list_mutex.
  */
 static int dma_chan_get(struct dma_chan *chan)
 {
@@ -483,10 +485,10 @@ module_put_out:
 }
 
 /**
- * dma_chan_put - drop a reference to a dma channel's parent driver module
- * @chan - channel to release
+ * dma_chan_put - drop a reference to a DMA channel's parent driver module
+ * @chan:	channel to release
  *
- * Must be called under dma_list_mutex
+ * Must be called under dma_list_mutex.
  */
 static void dma_chan_put(struct dma_chan *chan)
 {
@@ -537,7 +539,7 @@ EXPORT_SYMBOL(dma_sync_wait);
 
 /**
  * dma_find_channel - find a channel to carry out the operation
- * @tx_type: transaction type
+ * @tx_type:	transaction type
  */
 struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
 {
@@ -677,7 +679,7 @@ static struct dma_chan *find_candidate(struct dma_device *device,
 
 /**
  * dma_get_slave_channel - try to get specific channel exclusively
- * @chan: target channel
+ * @chan:	target channel
  */
 struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
 {
@@ -731,10 +733,10 @@ EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);
 
 /**
  * __dma_request_channel - try to allocate an exclusive channel
- * @mask: capabilities that the channel must satisfy
- * @fn: optional callback to disposition available channels
- * @fn_param: opaque parameter to pass to dma_filter_fn
- * @np: device node to look for DMA channels
+ * @mask:	capabilities that the channel must satisfy
+ * @fn:		optional callback to disposition available channels
+ * @fn_param:	opaque parameter to pass to dma_filter_fn()
+ * @np:		device node to look for DMA channels
  *
  * Returns pointer to appropriate DMA channel on success or NULL.
  */
@@ -877,7 +879,7 @@ EXPORT_SYMBOL_GPL(dma_request_slave_channel);
 
 /**
  * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
- * @mask: capabilities that the channel must satisfy
+ * @mask:	capabilities that the channel must satisfy
  *
  * Returns pointer to appropriate DMA channel on success or an error pointer.
  */
@@ -968,7 +970,7 @@ void dmaengine_get(void)
 EXPORT_SYMBOL(dmaengine_get);
 
 /**
- * dmaengine_put - let dma drivers be removed when ref_count == 0
+ * dmaengine_put - let DMA drivers be removed when ref_count == 0
  */
 void dmaengine_put(void)
 {
@@ -1132,7 +1134,7 @@ EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);
 
 /**
  * dma_async_device_register - registers DMA devices found
- * @device: &dma_device
+ * @device:	pointer to &struct dma_device
  *
  * After calling this routine the structure should not be freed except in the
  * device_release() callback which will be called after
@@ -1304,7 +1306,7 @@ EXPORT_SYMBOL(dma_async_device_register);
 
 /**
  * dma_async_device_unregister - unregister a DMA device
- * @device: &dma_device
+ * @device:	pointer to &struct dma_device
  *
  * This routine is called by dma driver exit routines, dmaengine holds module
  * references to prevent it being called while channels are in use.
@@ -1341,7 +1343,7 @@ static void dmam_device_release(struct device *dev, void *res)
 
 /**
  * dmaenginem_async_device_register - registers DMA devices found
- * @device: &dma_device
+ * @device:	pointer to &struct dma_device
  *
  * The operation is managed and will be undone on driver detach.
  */
@@ -1578,8 +1580,9 @@ int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
 }
 EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);
 
-/* dma_wait_for_async_tx - spin wait for a transaction to complete
- * @tx: in-flight transaction to wait on
+/**
+ * dma_wait_for_async_tx - spin wait for a transaction to complete
+ * @tx:		in-flight transaction to wait on
  */
 enum dma_status
 dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
@@ -1602,9 +1605,12 @@ dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
 }
 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
 
-/* dma_run_dependencies - helper routine for dma drivers to process
- *	(start) dependent operations on their target channel
- * @tx: transaction with dependencies
+/**
+ * dma_run_dependencies - process dependent operations on the target channel
+ * @tx:		transaction with dependencies
+ *
+ * Helper routine for DMA drivers to process (start) dependent operations
+ * on their target channel.
  */
 void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
 {
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index 0425984db118..b175229a4b01 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -60,9 +60,9 @@ MODULE_PARM_DESC(pq_sources,
 		"Number of p+q source buffers (default: 3)");
 
 static int timeout = 3000;
-module_param(timeout, uint, S_IRUGO | S_IWUSR);
+module_param(timeout, int, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
-		 "Pass 0xFFFFFFFF (4294967295) for maximum timeout");
+		 "Pass -1 for infinite timeout");
 
 static bool noverify;
 module_param(noverify, bool, S_IRUGO | S_IWUSR);
@@ -72,10 +72,6 @@ static bool norandom;
 module_param(norandom, bool, 0644);
 MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
 
-static bool polled;
-module_param(polled, bool, S_IRUGO | S_IWUSR);
-MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
-
 static bool verbose;
 module_param(verbose, bool, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
@@ -88,6 +84,10 @@ static unsigned int transfer_size;
 module_param(transfer_size, uint, 0644);
 MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
 
+static bool polled;
+module_param(polled, bool, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
+
 /**
  * struct dmatest_params - test parameters.
  * @buf_size:		size of the memcpy test buffer
@@ -98,7 +98,12 @@ MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default
  * @iterations:		iterations before stopping test
  * @xor_sources:	number of xor source buffers
  * @pq_sources:		number of p+q source buffers
- * @timeout:		transfer timeout in msec, 0 - 0xFFFFFFFF (4294967295)
+ * @timeout:		transfer timeout in msec, -1 for infinite timeout
+ * @noverify:		disable data verification
+ * @norandom:		disable random offset setup
+ * @alignment:		custom data address alignment taken as 2^alignment
+ * @transfer_size:	custom transfer size in bytes
+ * @polled:		use polling for completion instead of interrupts
  */
 struct dmatest_params {
 	unsigned int	buf_size;
@@ -109,7 +114,7 @@ struct dmatest_params {
 	unsigned int	iterations;
 	unsigned int	xor_sources;
 	unsigned int	pq_sources;
-	unsigned int	timeout;
+	int		timeout;
 	bool		noverify;
 	bool		norandom;
 	int		alignment;
@@ -120,7 +125,10 @@ struct dmatest_params {
 /**
  * struct dmatest_info - test information.
  * @params:		test parameters
+ * @channels:		channels under test
+ * @nr_channels:	number of channels under test
  * @lock:		access protection to the fields of this structure
+ * @did_init:		module has been initialized completely
  */
 static struct dmatest_info {
 	/* Test parameters */
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index ff392c01bad1..ed430ad9b3dd 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -13,8 +13,9 @@
 #include <linux/dmaengine.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
+#include <linux/irq.h>
 #include <linux/dma/edma.h>
-#include <linux/pci.h>
+#include <linux/dma-mapping.h>
 
 #include "dw-edma-core.h"
 #include "dw-edma-v0-core.h"
@@ -322,7 +323,7 @@ static struct dma_async_tx_descriptor *
 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 {
 	struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
-	enum dma_transfer_direction direction = xfer->direction;
+	enum dma_transfer_direction dir = xfer->direction;
 	phys_addr_t src_addr, dst_addr;
 	struct scatterlist *sg = NULL;
 	struct dw_edma_chunk *chunk;
@@ -331,10 +332,26 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 	u32 cnt;
 	int i;
 
-	if ((direction == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) ||
-	    (direction == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ))
+	if (!chan->configured)
 		return NULL;
 
+	switch (chan->config.direction) {
+	case DMA_DEV_TO_MEM: /* local dma */
+		if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ)
+			break;
+		return NULL;
+	case DMA_MEM_TO_DEV: /* local dma */
+		if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE)
+			break;
+		return NULL;
+	default: /* remote dma */
+		if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ)
+			break;
+		if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE)
+			break;
+		return NULL;
+	}
+
 	if (xfer->cyclic) {
 		if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
 			return NULL;
@@ -343,9 +360,6 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 			return NULL;
 	}
 
-	if (!chan->configured)
-		return NULL;
-
 	desc = dw_edma_alloc_desc(chan);
 	if (unlikely(!desc))
 		goto err_alloc;
@@ -386,7 +400,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 		chunk->ll_region.sz += burst->sz;
 		desc->alloc_sz += burst->sz;
 
-		if (direction == DMA_DEV_TO_MEM) {
+		if (chan->dir == EDMA_DIR_WRITE) {
 			burst->sar = src_addr;
 			if (xfer->cyclic) {
 				burst->dar = xfer->xfer.cyclic.paddr;
@@ -773,6 +787,7 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip,
 	u32 rd_mask = 1;
 	int i, err = 0;
 	u32 ch_cnt;
+	int irq;
 
 	ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
 
@@ -781,16 +796,16 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip,
 
 	if (dw->nr_irqs == 1) {
 		/* Common IRQ shared among all channels */
-		err = request_irq(pci_irq_vector(to_pci_dev(dev), 0),
-				  dw_edma_interrupt_common,
+		irq = dw->ops->irq_vector(dev, 0);
+		err = request_irq(irq, dw_edma_interrupt_common,
 				  IRQF_SHARED, dw->name, &dw->irq[0]);
 		if (err) {
 			dw->nr_irqs = 0;
 			return err;
 		}
 
-		get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), 0),
-				   &dw->irq[0].msi);
+		if (irq_get_msi_desc(irq))
+			get_cached_msi_msg(irq, &dw->irq[0].msi);
 	} else {
 		/* Distribute IRQs equally among all channels */
 		int tmp = dw->nr_irqs;
@@ -804,7 +819,8 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip,
 		dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
 
 		for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
-			err = request_irq(pci_irq_vector(to_pci_dev(dev), i),
+			irq = dw->ops->irq_vector(dev, i);
+			err = request_irq(irq,
 					  i < *wr_alloc ?
 						dw_edma_interrupt_write :
 						dw_edma_interrupt_read,
@@ -815,8 +831,8 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip,
 				return err;
 			}
 
-			get_cached_msi_msg(pci_irq_vector(to_pci_dev(dev), i),
-					   &dw->irq[i].msi);
+			if (irq_get_msi_desc(irq))
+				get_cached_msi_msg(irq, &dw->irq[i].msi);
 		}
 
 		dw->nr_irqs = i;
@@ -827,12 +843,23 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip,
 
 int dw_edma_probe(struct dw_edma_chip *chip)
 {
-	struct device *dev = chip->dev;
-	struct dw_edma *dw = chip->dw;
+	struct device *dev;
+	struct dw_edma *dw;
 	u32 wr_alloc = 0;
 	u32 rd_alloc = 0;
 	int i, err;
 
+	if (!chip)
+		return -EINVAL;
+
+	dev = chip->dev;
+	if (!dev)
+		return -EINVAL;
+
+	dw = chip->dw;
+	if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector)
+		return -EINVAL;
+
 	raw_spin_lock_init(&dw->lock);
 
 	/* Find out how many write channels are supported by hardware */
@@ -884,7 +911,7 @@ int dw_edma_probe(struct dw_edma_chip *chip)
 
 err_irq_free:
 	for (i = (dw->nr_irqs - 1); i >= 0; i--)
-		free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]);
+		free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]);
 
 	dw->nr_irqs = 0;
 
@@ -904,7 +931,7 @@ int dw_edma_remove(struct dw_edma_chip *chip)
 
 	/* Free irqs */
 	for (i = (dw->nr_irqs - 1); i >= 0; i--)
-		free_irq(pci_irq_vector(to_pci_dev(dev), i), &dw->irq[i]);
+		free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]);
 
 	/* Power management */
 	pm_runtime_disable(dev);
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index 4e5f9f6e901b..31fc50d31792 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -103,6 +103,10 @@ struct dw_edma_irq {
 	struct dw_edma			*dw;
 };
 
+struct dw_edma_core_ops {
+	int	(*irq_vector)(struct device *dev, unsigned int nr);
+};
+
 struct dw_edma {
 	char				name[20];
 
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index dc85f55e1bb8..1eafc602e17e 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -54,6 +54,15 @@ static const struct dw_edma_pcie_data snps_edda_data = {
 	.irqs				= 1,
 };
 
+static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr)
+{
+	return pci_irq_vector(to_pci_dev(dev), nr);
+}
+
+static const struct dw_edma_core_ops dw_edma_pcie_core_ops = {
+	.irq_vector = dw_edma_pcie_irq_vector,
+};
+
 static int dw_edma_pcie_probe(struct pci_dev *pdev,
 			      const struct pci_device_id *pid)
 {
@@ -151,6 +160,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 	dw->version = pdata->version;
 	dw->mode = pdata->mode;
 	dw->nr_irqs = nr_irqs;
+	dw->ops = &dw_edma_pcie_core_ops;
 
 	/* Debug info */
 	pci_dbg(pdev, "Version:\t%u\n", dw->version);
diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c
index 3999827970ab..052dae5d6ddd 100644
--- a/drivers/dma/idxd/sysfs.c
+++ b/drivers/dma/idxd/sysfs.c
@@ -1092,6 +1092,16 @@ static const struct attribute_group *idxd_wq_attribute_groups[] = {
 };
 
 /* IDXD device attribs */
+static ssize_t version_show(struct device *dev, struct device_attribute *attr,
+			    char *buf)
+{
+	struct idxd_device *idxd =
+		container_of(dev, struct idxd_device, conf_dev);
+
+	return sprintf(buf, "%#x\n", idxd->hw.version);
+}
+static DEVICE_ATTR_RO(version);
+
 static ssize_t max_work_queues_size_show(struct device *dev,
 					 struct device_attribute *attr,
 					 char *buf)
@@ -1313,6 +1323,7 @@ static ssize_t cdev_major_show(struct device *dev,
 static DEVICE_ATTR_RO(cdev_major);
 
 static struct attribute *idxd_device_attributes[] = {
+	&dev_attr_version.attr,
 	&dev_attr_max_groups.attr,
 	&dev_attr_max_work_queues.attr,
 	&dev_attr_max_work_queues_size.attr,
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 4d4477df4ede..91774039ae5d 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -2063,7 +2063,7 @@ static int sdma_probe(struct platform_device *pdev)
 
 	/* initially no scripts available */
 	saddr_arr = (s32 *)sdma->script_addrs;
-	for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++)
+	for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++)
 		saddr_arr[i] = -EINVAL;
 
 	dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 18c011e57592..8ad0ad861c86 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -332,8 +332,8 @@ ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
 	u8 *pos;
 	off_t offs;
 
-	chunk = idx / IOAT_DESCS_PER_2M;
-	idx &= (IOAT_DESCS_PER_2M - 1);
+	chunk = idx / IOAT_DESCS_PER_CHUNK;
+	idx &= (IOAT_DESCS_PER_CHUNK - 1);
 	offs = idx * IOAT_DESC_SZ;
 	pos = (u8 *)ioat_chan->descs[chunk].virt + offs;
 	phys = ioat_chan->descs[chunk].hw + offs;
@@ -370,7 +370,8 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
 	if (!ring)
 		return NULL;
 
-	ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M;
+	chunks = (total_descs * IOAT_DESC_SZ) / IOAT_CHUNK_SIZE;
+	ioat_chan->desc_chunks = chunks;
 
 	for (i = 0; i < chunks; i++) {
 		struct ioat_descs *descs = &ioat_chan->descs[i];
@@ -382,8 +383,9 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
 
 			for (idx = 0; idx < i; idx++) {
 				descs = &ioat_chan->descs[idx];
-				dma_free_coherent(to_dev(ioat_chan), SZ_2M,
-						  descs->virt, descs->hw);
+				dma_free_coherent(to_dev(ioat_chan),
+						IOAT_CHUNK_SIZE,
+						descs->virt, descs->hw);
 				descs->virt = NULL;
 				descs->hw = 0;
 			}
@@ -404,7 +406,7 @@ ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
 
 			for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
 				dma_free_coherent(to_dev(ioat_chan),
-						  SZ_2M,
+						  IOAT_CHUNK_SIZE,
 						  ioat_chan->descs[idx].virt,
 						  ioat_chan->descs[idx].hw);
 				ioat_chan->descs[idx].virt = NULL;
@@ -867,6 +869,23 @@ static void check_active(struct ioatdma_chan *ioat_chan)
 		mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
 }
 
+static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan)
+{
+	spin_lock_bh(&ioat_chan->prep_lock);
+	set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
+	spin_unlock_bh(&ioat_chan->prep_lock);
+
+	ioat_abort_descs(ioat_chan);
+	dev_warn(to_dev(ioat_chan), "Reset channel...\n");
+	ioat_reset_hw(ioat_chan);
+	dev_warn(to_dev(ioat_chan), "Restart channel...\n");
+	ioat_restart_channel(ioat_chan);
+
+	spin_lock_bh(&ioat_chan->prep_lock);
+	clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
+	spin_unlock_bh(&ioat_chan->prep_lock);
+}
+
 void ioat_timer_event(struct timer_list *t)
 {
 	struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer);
@@ -889,19 +908,7 @@ void ioat_timer_event(struct timer_list *t)
 
 		if (test_bit(IOAT_RUN, &ioat_chan->state)) {
 			spin_lock_bh(&ioat_chan->cleanup_lock);
-			spin_lock_bh(&ioat_chan->prep_lock);
-			set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
-			spin_unlock_bh(&ioat_chan->prep_lock);
-
-			ioat_abort_descs(ioat_chan);
-			dev_warn(to_dev(ioat_chan), "Reset channel...\n");
-			ioat_reset_hw(ioat_chan);
-			dev_warn(to_dev(ioat_chan), "Restart channel...\n");
-			ioat_restart_channel(ioat_chan);
-
-			spin_lock_bh(&ioat_chan->prep_lock);
-			clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
-			spin_unlock_bh(&ioat_chan->prep_lock);
+			ioat_reboot_chan(ioat_chan);
 			spin_unlock_bh(&ioat_chan->cleanup_lock);
 		}
 
@@ -915,17 +922,23 @@ void ioat_timer_event(struct timer_list *t)
 		spin_lock_bh(&ioat_chan->prep_lock);
 		check_active(ioat_chan);
 		spin_unlock_bh(&ioat_chan->prep_lock);
-		spin_unlock_bh(&ioat_chan->cleanup_lock);
-		return;
+		goto unlock_out;
+	}
+
+	/* handle the missed cleanup case */
+	if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) {
+		/* timer restarted in ioat_cleanup_preamble
+		 * and IOAT_COMPLETION_ACK cleared
+		 */
+		__cleanup(ioat_chan, phys_complete);
+		goto unlock_out;
 	}
 
 	/* if we haven't made progress and we have already
 	 * acknowledged a pending completion once, then be more
 	 * forceful with a restart
 	 */
-	if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
-		__cleanup(ioat_chan, phys_complete);
-	else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
+	if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
 		u32 chanerr;
 
 		chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
@@ -937,25 +950,23 @@ void ioat_timer_event(struct timer_list *t)
 		dev_dbg(to_dev(ioat_chan), "Active descriptors: %d\n",
 			ioat_ring_active(ioat_chan));
 
-		spin_lock_bh(&ioat_chan->prep_lock);
-		set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
-		spin_unlock_bh(&ioat_chan->prep_lock);
+		ioat_reboot_chan(ioat_chan);
 
-		ioat_abort_descs(ioat_chan);
-		dev_warn(to_dev(ioat_chan), "Resetting channel...\n");
-		ioat_reset_hw(ioat_chan);
-		dev_warn(to_dev(ioat_chan), "Restarting channel...\n");
-		ioat_restart_channel(ioat_chan);
+		goto unlock_out;
+	}
 
+	/* handle missed issue pending case */
+	if (ioat_ring_pending(ioat_chan)) {
+		dev_warn(to_dev(ioat_chan),
+			"Completion timeout with pending descriptors\n");
 		spin_lock_bh(&ioat_chan->prep_lock);
-		clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);
+		__ioat_issue_pending(ioat_chan);
 		spin_unlock_bh(&ioat_chan->prep_lock);
-		spin_unlock_bh(&ioat_chan->cleanup_lock);
-		return;
-	} else
-		set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
+	}
 
+	set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
 	mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
+unlock_out:
 	spin_unlock_bh(&ioat_chan->cleanup_lock);
 }
 
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index b8e8e0b9693c..e6b622e1ba92 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -81,6 +81,11 @@ struct ioatdma_device {
 	u32 msixpba;
 };
 
+#define IOAT_MAX_ORDER 16
+#define IOAT_MAX_DESCS (1 << IOAT_MAX_ORDER)
+#define IOAT_CHUNK_SIZE (SZ_512K)
+#define IOAT_DESCS_PER_CHUNK (IOAT_CHUNK_SIZE / IOAT_DESC_SZ)
+
 struct ioat_descs {
 	void *virt;
 	dma_addr_t hw;
@@ -128,7 +133,7 @@ struct ioatdma_chan {
 	u16 produce;
 	struct ioat_ring_ent **ring;
 	spinlock_t prep_lock;
-	struct ioat_descs descs[2];
+	struct ioat_descs descs[IOAT_MAX_DESCS / IOAT_DESCS_PER_CHUNK];
 	int desc_chunks;
 	int intr_coalesce;
 	int prev_intr_coalesce;
@@ -301,9 +306,6 @@ static inline bool is_ioat_bug(unsigned long err)
 	return !!err;
 }
 
-#define IOAT_MAX_ORDER 16
-#define IOAT_MAX_DESCS 65536
-#define IOAT_DESCS_PER_2M 32768
 
 static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)
 {
diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c
index 60e9afbb896c..58d13564f88b 100644
--- a/drivers/dma/ioat/init.c
+++ b/drivers/dma/ioat/init.c
@@ -651,7 +651,7 @@ static void ioat_free_chan_resources(struct dma_chan *c)
 	}
 
 	for (i = 0; i < ioat_chan->desc_chunks; i++) {
-		dma_free_coherent(to_dev(ioat_chan), SZ_2M,
+		dma_free_coherent(to_dev(ioat_chan), IOAT_CHUNK_SIZE,
 				  ioat_chan->descs[i].virt,
 				  ioat_chan->descs[i].hw);
 		ioat_chan->descs[i].virt = NULL;
diff --git a/drivers/dma/mediatek/Kconfig b/drivers/dma/mediatek/Kconfig
index 1ad63ddc292d..7a46a5455957 100644
--- a/drivers/dma/mediatek/Kconfig
+++ b/drivers/dma/mediatek/Kconfig
@@ -5,7 +5,7 @@ config MTK_HSDMA
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
-	---help---
+	help
 	  Enable support for High-Speed DMA controller on MediaTek
 	  SoCs.
 
diff --git a/drivers/dma/milbeaut-hdmac.c b/drivers/dma/milbeaut-hdmac.c
index 8853d442430b..a8cfb59f6efe 100644
--- a/drivers/dma/milbeaut-hdmac.c
+++ b/drivers/dma/milbeaut-hdmac.c
@@ -77,7 +77,7 @@ struct milbeaut_hdmac_device {
 	struct dma_device ddev;
 	struct clk *clk;
 	void __iomem *reg_base;
-	struct milbeaut_hdmac_chan channels[0];
+	struct milbeaut_hdmac_chan channels[];
 };
 
 static struct milbeaut_hdmac_chan *
diff --git a/drivers/dma/milbeaut-xdmac.c b/drivers/dma/milbeaut-xdmac.c
index ab3d2f395378..85a597228fb0 100644
--- a/drivers/dma/milbeaut-xdmac.c
+++ b/drivers/dma/milbeaut-xdmac.c
@@ -74,7 +74,7 @@ struct milbeaut_xdmac_chan {
 struct milbeaut_xdmac_device {
 	struct dma_device ddev;
 	void __iomem *reg_base;
-	struct milbeaut_xdmac_chan channels[0];
+	struct milbeaut_xdmac_chan channels[];
 };
 
 static struct milbeaut_xdmac_chan *
diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c
index d683232d7fea..dbc6a48424fa 100644
--- a/drivers/dma/mmp_tdma.c
+++ b/drivers/dma/mmp_tdma.c
@@ -235,7 +235,7 @@ static int mmp_tdma_config_chan(struct dma_chan *chan)
 			tdcr |= TDCR_BURSTSZ_128B;
 			break;
 		default:
-			dev_err(tdmac->dev, "mmp_tdma: unknown burst size.\n");
+			dev_err(tdmac->dev, "unknown burst size.\n");
 			return -EINVAL;
 		}
 
@@ -250,7 +250,7 @@ static int mmp_tdma_config_chan(struct dma_chan *chan)
 			tdcr |= TDCR_SSZ_32_BITS;
 			break;
 		default:
-			dev_err(tdmac->dev, "mmp_tdma: unknown bus size.\n");
+			dev_err(tdmac->dev, "unknown bus size.\n");
 			return -EINVAL;
 		}
 	} else if (tdmac->type == PXA910_SQU) {
@@ -276,7 +276,7 @@ static int mmp_tdma_config_chan(struct dma_chan *chan)
 			tdcr |= TDCR_BURSTSZ_SQU_32B;
 			break;
 		default:
-			dev_err(tdmac->dev, "mmp_tdma: unknown burst size.\n");
+			dev_err(tdmac->dev, "unknown burst size.\n");
 			return -EINVAL;
 		}
 	}
@@ -429,8 +429,15 @@ static struct dma_async_tx_descriptor *mmp_tdma_prep_dma_cyclic(
 	int num_periods = buf_len / period_len;
 	int i = 0, buf = 0;
 
-	if (tdmac->status != DMA_COMPLETE)
+	if (!is_slave_direction(direction)) {
+		dev_err(tdmac->dev, "unsupported transfer direction\n");
 		return NULL;
+	}
+
+	if (tdmac->status != DMA_COMPLETE) {
+		dev_err(tdmac->dev, "controller busy");
+		return NULL;
+	}
 
 	if (period_len > TDMA_MAX_XFER_BYTES) {
 		dev_err(tdmac->dev,
@@ -704,6 +711,17 @@ static int mmp_tdma_probe(struct platform_device *pdev)
 	tdev->device.device_terminate_all = mmp_tdma_terminate_all;
 	tdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
 
+	tdev->device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
+	if (type == MMP_AUD_TDMA) {
+		tdev->device.max_burst = SZ_128;
+		tdev->device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+		tdev->device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+	} else if (type == PXA910_SQU) {
+		tdev->device.max_burst = SZ_32;
+	}
+	tdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+	tdev->device.descriptor_reuse = true;
+
 	dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
 	platform_set_drvdata(pdev, tdev);
 
diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c
index e04499c1f27f..347146a6e1d0 100644
--- a/drivers/dma/moxart-dma.c
+++ b/drivers/dma/moxart-dma.c
@@ -127,7 +127,7 @@ struct moxart_desc {
 	unsigned int			dma_cycles;
 	struct virt_dma_desc		vd;
 	uint8_t				es;
-	struct moxart_sg		sg[0];
+	struct moxart_sg		sg[];
 };
 
 struct moxart_chan {
@@ -568,7 +568,7 @@ static int moxart_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct device_node *node = dev->of_node;
 	struct resource *res;
-	static void __iomem *dma_base_addr;
+	void __iomem *dma_base_addr;
 	int ret, i;
 	unsigned int irq;
 	struct moxart_chan *ch;
diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig
index 1d189438aeb0..3bcb689162c6 100644
--- a/drivers/dma/qcom/Kconfig
+++ b/drivers/dma/qcom/Kconfig
@@ -4,7 +4,7 @@ config QCOM_BAM_DMA
 	depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM)
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
-	---help---
+	help
 	  Enable support for the QCOM BAM DMA controller.  This controller
 	  provides DMA capabilities for a variety of on-chip devices.
 
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index ef73f65224b1..5a08dd0d3388 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -74,7 +74,7 @@ struct bam_async_desc {
 	struct list_head desc_node;
 	enum dma_transfer_direction dir;
 	size_t length;
-	struct bam_desc_hw desc[0];
+	struct bam_desc_hw desc[];
 };
 
 enum bam_reg {
diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c
index 411f91fde734..0a6d3ea08c78 100644
--- a/drivers/dma/qcom/hidma.c
+++ b/drivers/dma/qcom/hidma.c
@@ -550,7 +550,7 @@ static void hidma_free_chan_resources(struct dma_chan *dmach)
 		kfree(mdesc);
 	}
 
-	mchan->allocated = 0;
+	mchan->allocated = false;
 	spin_unlock_irqrestore(&mchan->lock, irqflags);
 }
 
@@ -897,7 +897,6 @@ uninit:
 	if (msi)
 		hidma_free_msis(dmadev);
 
-	hidma_debug_uninit(dmadev);
 	hidma_ll_uninit(dmadev->lldev);
 dmafree:
 	if (dmadev)
diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c
index 6d0bec947636..5c118c7e02bd 100644
--- a/drivers/dma/sf-pdma/sf-pdma.c
+++ b/drivers/dma/sf-pdma/sf-pdma.c
@@ -506,11 +506,11 @@ static int sf_pdma_probe(struct platform_device *pdev)
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	pdma->membase = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(pdma->membase))
-		goto ERR_MEMBASE;
+		return PTR_ERR(pdma->membase);
 
 	ret = sf_pdma_irq_init(pdev, pdma);
 	if (ret)
-		goto ERR_INITIRQ;
+		return ret;
 
 	sf_pdma_setup_chans(pdma);
 
@@ -544,24 +544,13 @@ static int sf_pdma_probe(struct platform_device *pdev)
 			 "Failed to set DMA mask. Fall back to default.\n");
 
 	ret = dma_async_device_register(&pdma->dma_dev);
-	if (ret)
-		goto ERR_REG_DMADEVICE;
+	if (ret) {
+		dev_err(&pdev->dev,
+			"Can't register SiFive Platform DMA. (%d)\n", ret);
+		return ret;
+	}
 
 	return 0;
-
-ERR_MEMBASE:
-	devm_kfree(&pdev->dev, pdma);
-	return PTR_ERR(pdma->membase);
-
-ERR_INITIRQ:
-	devm_kfree(&pdev->dev, pdma);
-	return ret;
-
-ERR_REG_DMADEVICE:
-	devm_kfree(&pdev->dev, pdma);
-	dev_err(&pdev->dev,
-		"Can't register SiFive Platform DMA. (%d)\n", ret);
-	return ret;
 }
 
 static int sf_pdma_remove(struct platform_device *pdev)
diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
index 0ddbaa4b4f0b..96ad1b3d24c6 100644
--- a/drivers/dma/stm32-dma.c
+++ b/drivers/dma/stm32-dma.c
@@ -117,6 +117,7 @@
 #define STM32_DMA_FIFO_THRESHOLD_HALFFULL		0x01
 #define STM32_DMA_FIFO_THRESHOLD_3QUARTERSFULL		0x02
 #define STM32_DMA_FIFO_THRESHOLD_FULL			0x03
+#define STM32_DMA_FIFO_THRESHOLD_NONE			0x04
 
 #define STM32_DMA_MAX_DATA_ITEMS	0xffff
 /*
@@ -136,6 +137,9 @@
 /* DMA Features */
 #define STM32_DMA_THRESHOLD_FTR_MASK	GENMASK(1, 0)
 #define STM32_DMA_THRESHOLD_FTR_GET(n)	((n) & STM32_DMA_THRESHOLD_FTR_MASK)
+#define STM32_DMA_DIRECT_MODE_MASK	BIT(2)
+#define STM32_DMA_DIRECT_MODE_GET(n)	(((n) & STM32_DMA_DIRECT_MODE_MASK) \
+					 >> 2)
 
 enum stm32_dma_width {
 	STM32_DMA_BYTE,
@@ -281,6 +285,9 @@ static bool stm32_dma_fifo_threshold_is_allowed(u32 burst, u32 threshold,
 {
 	u32 remaining;
 
+	if (threshold == STM32_DMA_FIFO_THRESHOLD_NONE)
+		return false;
+
 	if (width != DMA_SLAVE_BUSWIDTH_UNDEFINED) {
 		if (burst != 0) {
 			/*
@@ -302,6 +309,10 @@ static bool stm32_dma_fifo_threshold_is_allowed(u32 burst, u32 threshold,
 
 static bool stm32_dma_is_burst_possible(u32 buf_len, u32 threshold)
 {
+	/* If FIFO direct mode, burst is not possible */
+	if (threshold == STM32_DMA_FIFO_THRESHOLD_NONE)
+		return false;
+
 	/*
 	 * Buffer or period length has to be aligned on FIFO depth.
 	 * Otherwise bytes may be stuck within FIFO at buffer or period
@@ -657,6 +668,12 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
 				dev_dbg(chan2dev(chan), "FIFO over/underrun\n");
 		}
 	}
+	if (status & STM32_DMA_DMEI) {
+		stm32_dma_irq_clear(chan, STM32_DMA_DMEI);
+		status &= ~STM32_DMA_DMEI;
+		if (sfcr & STM32_DMA_SCR_DMEIE)
+			dev_dbg(chan2dev(chan), "Direct mode overrun\n");
+	}
 	if (status) {
 		stm32_dma_irq_clear(chan, status);
 		dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
@@ -692,13 +709,13 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 	int src_bus_width, dst_bus_width;
 	int src_burst_size, dst_burst_size;
 	u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst;
-	u32 dma_scr, threshold;
+	u32 dma_scr, fifoth;
 
 	src_addr_width = chan->dma_sconfig.src_addr_width;
 	dst_addr_width = chan->dma_sconfig.dst_addr_width;
 	src_maxburst = chan->dma_sconfig.src_maxburst;
 	dst_maxburst = chan->dma_sconfig.dst_maxburst;
-	threshold = chan->threshold;
+	fifoth = chan->threshold;
 
 	switch (direction) {
 	case DMA_MEM_TO_DEV:
@@ -710,7 +727,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 		/* Set device burst size */
 		dst_best_burst = stm32_dma_get_best_burst(buf_len,
 							  dst_maxburst,
-							  threshold,
+							  fifoth,
 							  dst_addr_width);
 
 		dst_burst_size = stm32_dma_get_burst(chan, dst_best_burst);
@@ -718,7 +735,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 			return dst_burst_size;
 
 		/* Set memory data size */
-		src_addr_width = stm32_dma_get_max_width(buf_len, threshold);
+		src_addr_width = stm32_dma_get_max_width(buf_len, fifoth);
 		chan->mem_width = src_addr_width;
 		src_bus_width = stm32_dma_get_width(chan, src_addr_width);
 		if (src_bus_width < 0)
@@ -728,7 +745,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 		src_maxburst = STM32_DMA_MAX_BURST;
 		src_best_burst = stm32_dma_get_best_burst(buf_len,
 							  src_maxburst,
-							  threshold,
+							  fifoth,
 							  src_addr_width);
 		src_burst_size = stm32_dma_get_burst(chan, src_best_burst);
 		if (src_burst_size < 0)
@@ -742,7 +759,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 
 		/* Set FIFO threshold */
 		chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK;
-		chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(threshold);
+		if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE)
+			chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth);
 
 		/* Set peripheral address */
 		chan->chan_reg.dma_spar = chan->dma_sconfig.dst_addr;
@@ -758,7 +776,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 		/* Set device burst size */
 		src_best_burst = stm32_dma_get_best_burst(buf_len,
 							  src_maxburst,
-							  threshold,
+							  fifoth,
 							  src_addr_width);
 		chan->mem_burst = src_best_burst;
 		src_burst_size = stm32_dma_get_burst(chan, src_best_burst);
@@ -766,7 +784,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 			return src_burst_size;
 
 		/* Set memory data size */
-		dst_addr_width = stm32_dma_get_max_width(buf_len, threshold);
+		dst_addr_width = stm32_dma_get_max_width(buf_len, fifoth);
 		chan->mem_width = dst_addr_width;
 		dst_bus_width = stm32_dma_get_width(chan, dst_addr_width);
 		if (dst_bus_width < 0)
@@ -776,7 +794,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 		dst_maxburst = STM32_DMA_MAX_BURST;
 		dst_best_burst = stm32_dma_get_best_burst(buf_len,
 							  dst_maxburst,
-							  threshold,
+							  fifoth,
 							  dst_addr_width);
 		chan->mem_burst = dst_best_burst;
 		dst_burst_size = stm32_dma_get_burst(chan, dst_best_burst);
@@ -791,7 +809,8 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 
 		/* Set FIFO threshold */
 		chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK;
-		chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(threshold);
+		if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE)
+			chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth);
 
 		/* Set peripheral address */
 		chan->chan_reg.dma_spar = chan->dma_sconfig.src_addr;
@@ -1216,6 +1235,8 @@ static void stm32_dma_set_config(struct stm32_dma_chan *chan,
 	chan->chan_reg.dma_scr |= STM32_DMA_SCR_TEIE | STM32_DMA_SCR_TCIE;
 
 	chan->threshold = STM32_DMA_THRESHOLD_FTR_GET(cfg->features);
+	if (STM32_DMA_DIRECT_MODE_GET(cfg->features))
+		chan->threshold = STM32_DMA_FIFO_THRESHOLD_NONE;
 }
 
 static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec,
diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c
index b9f0d9636620..55fc7400f717 100644
--- a/drivers/dma/tegra20-apb-dma.c
+++ b/drivers/dma/tegra20-apb-dma.c
@@ -225,7 +225,7 @@ struct tegra_dma {
 	u32				global_pause_count;
 
 	/* Last member of the structure */
-	struct tegra_dma_channel channels[0];
+	struct tegra_dma_channel channels[];
 };
 
 static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val)
diff --git a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig
index f76e06651f80..79618fac119a 100644
--- a/drivers/dma/ti/Kconfig
+++ b/drivers/dma/ti/Kconfig
@@ -36,7 +36,7 @@ config DMA_OMAP
 
 config TI_K3_UDMA
 	bool "Texas Instruments UDMA support"
-	depends on ARCH_K3 || COMPILE_TEST
+	depends on ARCH_K3
 	depends on TI_SCI_PROTOCOL
 	depends on TI_SCI_INTA_IRQCHIP
 	select DMA_ENGINE
@@ -49,7 +49,7 @@ config TI_K3_UDMA
 
 config TI_K3_UDMA_GLUE_LAYER
 	bool "Texas Instruments UDMA Glue layer for non DMAengine users"
-	depends on ARCH_K3 || COMPILE_TEST
+	depends on ARCH_K3
 	depends on TI_K3_UDMA
 	help
 	  Say y here to support the K3 NAVSS DMA glue interface
diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c
index c4a5c170c1f9..35d81bd857f1 100644
--- a/drivers/dma/ti/edma.c
+++ b/drivers/dma/ti/edma.c
@@ -211,7 +211,7 @@ struct edma_desc {
 	u32				residue;
 	u32				residue_stat;
 
-	struct edma_pset		pset[0];
+	struct edma_pset		pset[];
 };
 
 struct edma_cc;
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index a90e154b0ae0..c91e2dc1bb72 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -170,7 +170,7 @@ struct udma_desc {
 	void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */
 
 	unsigned int hwdesc_count;
-	struct udma_hwdesc hwdesc[0];
+	struct udma_hwdesc hwdesc[];
 };
 
 enum udma_chan_state {
@@ -231,7 +231,6 @@ struct udma_chan {
 	struct udma_tx_drain tx_drain;
 
 	u32 bcnt; /* number of bytes completed since the start of the channel */
-	u32 in_ring_cnt; /* number of descriptors in flight */
 
 	/* Channel configuration parameters */
 	struct udma_chan_config config;
@@ -574,7 +573,6 @@ static int udma_push_to_ring(struct udma_chan *uc, int idx)
 	struct udma_desc *d = uc->desc;
 	struct k3_ring *ring = NULL;
 	dma_addr_t paddr;
-	int ret;
 
 	switch (uc->config.dir) {
 	case DMA_DEV_TO_MEM:
@@ -598,11 +596,7 @@ static int udma_push_to_ring(struct udma_chan *uc, int idx)
 		udma_sync_for_device(uc, idx);
 	}
 
-	ret = k3_ringacc_ring_push(ring, &paddr);
-	if (!ret)
-		uc->in_ring_cnt++;
-
-	return ret;
+	return k3_ringacc_ring_push(ring, &paddr);
 }
 
 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
@@ -655,9 +649,6 @@ static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
 						d->hwdesc[0].cppi5_desc_size,
 						DMA_FROM_DEVICE);
 		rmb(); /* Ensure that reads are not moved before this point */
-
-		if (!ret)
-			uc->in_ring_cnt--;
 	}
 
 	return ret;
@@ -697,8 +688,6 @@ static void udma_reset_rings(struct udma_chan *uc)
 		udma_desc_free(&uc->terminated_desc->vd);
 		uc->terminated_desc = NULL;
 	}
-
-	uc->in_ring_cnt = 0;
 }
 
 static void udma_reset_counters(struct udma_chan *uc)
@@ -1073,9 +1062,6 @@ static irqreturn_t udma_ring_irq_handler(int irq, void *data)
 
 	/* Teardown completion message */
 	if (cppi5_desc_is_tdcm(paddr)) {
-		/* Compensate our internal pop/push counter */
-		uc->in_ring_cnt++;
-
 		complete_all(&uc->teardown_completed);
 
 		if (uc->terminated_desc) {
@@ -1291,10 +1277,8 @@ static int udma_get_tchan(struct udma_chan *uc)
 	}
 
 	uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1);
-	if (IS_ERR(uc->tchan))
-		return PTR_ERR(uc->tchan);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(uc->tchan);
 }
 
 static int udma_get_rchan(struct udma_chan *uc)
@@ -1308,10 +1292,8 @@ static int udma_get_rchan(struct udma_chan *uc)
 	}
 
 	uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1);
-	if (IS_ERR(uc->rchan))
-		return PTR_ERR(uc->rchan);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(uc->rchan);
 }
 
 static int udma_get_chan_pair(struct udma_chan *uc)
@@ -1373,10 +1355,8 @@ static int udma_get_rflow(struct udma_chan *uc, int flow_id)
 	}
 
 	uc->rflow = __udma_get_rflow(ud, flow_id);
-	if (IS_ERR(uc->rflow))
-		return PTR_ERR(uc->rflow);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(uc->rflow);
 }
 
 static void udma_put_rchan(struct udma_chan *uc)
@@ -1870,6 +1850,7 @@ static int udma_alloc_chan_resources(struct dma_chan *chan)
 		udma_stop(uc);
 		if (udma_is_chan_running(uc)) {
 			dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
+			ret = -EBUSY;
 			goto err_res_free;
 		}
 	}
@@ -3189,7 +3170,7 @@ static struct udma_match_data am654_main_data = {
 
 static struct udma_match_data am654_mcu_data = {
 	.psil_base = 0x6000,
-	.enable_memcpy_support = true, /* TEST: DMA domains */
+	.enable_memcpy_support = false,
 	.statictr_z_mask = GENMASK(11, 0),
 	.rchan_oes_offset = 0x2000,
 	.tpl_levels = 2,
@@ -3471,6 +3452,9 @@ static int udma_setup_rx_flush(struct udma_dev *ud)
 	tr_req->icnt0 = rx_flush->buffer_size;
 	tr_req->icnt1 = 1;
 
+	dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
+				   hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
+
 	/* Set up descriptor to be used for packet mode */
 	hwdesc = &rx_flush->hwdescs[1];
 	hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index 39382694fdfc..68e48bf54d78 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -88,7 +88,7 @@ struct timb_dma {
 	struct dma_device	dma;
 	void __iomem		*membase;
 	struct tasklet_struct	tasklet;
-	struct timb_dma_chan	channels[0];
+	struct timb_dma_chan	channels[];
 };
 
 static struct device *chan2dev(struct dma_chan *chan)
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index fe2eb892a1bd..7b6ec3014ba2 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -44,7 +44,7 @@ config EDAC_DECODE_MCE
 	tristate "Decode MCEs in human-readable form (only on AMD for now)"
 	depends on CPU_SUP_AMD && X86_MCE_AMD
 	default y
-	---help---
+	help
 	  Enable this option if you want to decode Machine Check Exceptions
 	  occurring on your machine in human-readable form.
 
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 9cf7cc1f3f72..ef90070a9194 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -4,9 +4,6 @@
 
 static struct edac_pci_ctl_info *pci_ctl;
 
-static int report_gart_errors;
-module_param(report_gart_errors, int, 0644);
-
 /*
  * Set by command line parameter. If BIOS has enabled the ECC, this override is
  * cleared to prevent re-enabling the hardware by this driver.
@@ -2319,6 +2316,16 @@ static struct amd64_family_type family_types[] = {
 			.dbam_to_cs		= f17_addr_mask_to_cs_size,
 		}
 	},
+	[F17_M60H_CPUS] = {
+		.ctl_name = "F17h_M60h",
+		.f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0,
+		.f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6,
+		.max_mcs = 2,
+		.ops = {
+			.early_channel_count	= f17_early_channel_count,
+			.dbam_to_cs		= f17_addr_mask_to_cs_size,
+		}
+	},
 	[F17_M70H_CPUS] = {
 		.ctl_name = "F17h_M70h",
 		.f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0,
@@ -3357,6 +3364,10 @@ static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
 			fam_type = &family_types[F17_M30H_CPUS];
 			pvt->ops = &family_types[F17_M30H_CPUS].ops;
 			break;
+		} else if (pvt->model >= 0x60 && pvt->model <= 0x6f) {
+			fam_type = &family_types[F17_M60H_CPUS];
+			pvt->ops = &family_types[F17_M60H_CPUS].ops;
+			break;
 		} else if (pvt->model >= 0x70 && pvt->model <= 0x7f) {
 			fam_type = &family_types[F17_M70H_CPUS];
 			pvt->ops = &family_types[F17_M70H_CPUS].ops;
@@ -3681,9 +3692,6 @@ static int __init amd64_edac_init(void)
 	}
 
 	/* register stuff with EDAC MCE */
-	if (report_gart_errors)
-		amd_report_gart_errors(true);
-
 	if (boot_cpu_data.x86 >= 0x17)
 		amd_register_ecc_decoder(decode_umc_error);
 	else
@@ -3718,8 +3726,6 @@ static void __exit amd64_edac_exit(void)
 		edac_pci_release_generic_ctl(pci_ctl);
 
 	/* unregister from EDAC MCE */
-	amd_report_gart_errors(false);
-
 	if (boot_cpu_data.x86 >= 0x17)
 		amd_unregister_ecc_decoder(decode_umc_error);
 	else
diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h
index abbf3c274d74..52b5d03eeba0 100644
--- a/drivers/edac/amd64_edac.h
+++ b/drivers/edac/amd64_edac.h
@@ -120,6 +120,8 @@
 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F6 0x15ee
 #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F0 0x1490
 #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F6 0x1496
+#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F0 0x1448
+#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F6 0x144e
 #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F0 0x1440
 #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F6 0x1446
 #define PCI_DEVICE_ID_AMD_19H_DF_F0	0x1650
@@ -293,6 +295,7 @@ enum amd_families {
 	F17_CPUS,
 	F17_M10H_CPUS,
 	F17_M30H_CPUS,
+	F17_M60H_CPUS,
 	F17_M70H_CPUS,
 	F19_CPUS,
 	NUM_FAMILIES,
diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c
index 75ede27bdf6a..5813e931f2f0 100644
--- a/drivers/edac/edac_mc.c
+++ b/drivers/edac/edac_mc.c
@@ -43,8 +43,6 @@
 int edac_op_state = EDAC_OPSTATE_INVAL;
 EXPORT_SYMBOL_GPL(edac_op_state);
 
-static int edac_report = EDAC_REPORTING_ENABLED;
-
 /* lock to memory controller's control array */
 static DEFINE_MUTEX(mem_ctls_mutex);
 static LIST_HEAD(mc_devices);
@@ -60,65 +58,6 @@ static struct mem_ctl_info *error_desc_to_mci(struct edac_raw_error_desc *e)
 	return container_of(e, struct mem_ctl_info, error_desc);
 }
 
-int edac_get_report_status(void)
-{
-	return edac_report;
-}
-EXPORT_SYMBOL_GPL(edac_get_report_status);
-
-void edac_set_report_status(int new)
-{
-	if (new == EDAC_REPORTING_ENABLED ||
-	    new == EDAC_REPORTING_DISABLED ||
-	    new == EDAC_REPORTING_FORCE)
-		edac_report = new;
-}
-EXPORT_SYMBOL_GPL(edac_set_report_status);
-
-static int edac_report_set(const char *str, const struct kernel_param *kp)
-{
-	if (!str)
-		return -EINVAL;
-
-	if (!strncmp(str, "on", 2))
-		edac_report = EDAC_REPORTING_ENABLED;
-	else if (!strncmp(str, "off", 3))
-		edac_report = EDAC_REPORTING_DISABLED;
-	else if (!strncmp(str, "force", 5))
-		edac_report = EDAC_REPORTING_FORCE;
-
-	return 0;
-}
-
-static int edac_report_get(char *buffer, const struct kernel_param *kp)
-{
-	int ret = 0;
-
-	switch (edac_report) {
-	case EDAC_REPORTING_ENABLED:
-		ret = sprintf(buffer, "on");
-		break;
-	case EDAC_REPORTING_DISABLED:
-		ret = sprintf(buffer, "off");
-		break;
-	case EDAC_REPORTING_FORCE:
-		ret = sprintf(buffer, "force");
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
-static const struct kernel_param_ops edac_report_ops = {
-	.set = edac_report_set,
-	.get = edac_report_get,
-};
-
-module_param_cb(edac_report, &edac_report_ops, &edac_report, 0644);
-
 unsigned int edac_dimm_info_location(struct dimm_info *dimm, char *buf,
 				     unsigned int len)
 {
diff --git a/drivers/edac/i7core_edac.c b/drivers/edac/i7core_edac.c
index b3135b208f9a..5860ca41185c 100644
--- a/drivers/edac/i7core_edac.c
+++ b/drivers/edac/i7core_edac.c
@@ -1815,7 +1815,7 @@ static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
 	struct mem_ctl_info *mci;
 
 	i7_dev = get_i7core_dev(mce->socketid);
-	if (!i7_dev)
+	if (!i7_dev || (mce->kflags & MCE_HANDLED_CEC))
 		return NOTIFY_DONE;
 
 	mci = i7_dev->mci;
@@ -1834,7 +1834,8 @@ static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
 	i7core_check_error(mci, mce);
 
 	/* Advise mcelog that the errors were handled */
-	return NOTIFY_STOP;
+	mce->kflags |= MCE_HANDLED_EDAC;
+	return NOTIFY_OK;
 }
 
 static struct notifier_block i7_mce_dec = {
diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c
index 8874b7722b2f..2b5401db56ad 100644
--- a/drivers/edac/mce_amd.c
+++ b/drivers/edac/mce_amd.c
@@ -10,15 +10,8 @@ static struct amd_decoder_ops fam_ops;
 
 static u8 xec_mask	 = 0xf;
 
-static bool report_gart_errors;
 static void (*decode_dram_ecc)(int node_id, struct mce *m);
 
-void amd_report_gart_errors(bool v)
-{
-	report_gart_errors = v;
-}
-EXPORT_SYMBOL_GPL(amd_report_gart_errors);
-
 void amd_register_ecc_decoder(void (*f)(int, struct mce *))
 {
 	decode_dram_ecc = f;
@@ -1030,20 +1023,6 @@ static inline void amd_decode_err_code(u16 ec)
 	pr_cont("\n");
 }
 
-/*
- * Filter out unwanted MCE signatures here.
- */
-static bool ignore_mce(struct mce *m)
-{
-	/*
-	 * NB GART TLB error reporting is disabled by default.
-	 */
-	if (m->bank == 4 && XEC(m->status, 0x1f) == 0x5 && !report_gart_errors)
-		return true;
-
-	return false;
-}
-
 static const char *decode_error_status(struct mce *m)
 {
 	if (m->status & MCI_STATUS_UC) {
@@ -1067,8 +1046,8 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 	unsigned int fam = x86_family(m->cpuid);
 	int ecc;
 
-	if (ignore_mce(m))
-		return NOTIFY_STOP;
+	if (m->kflags & MCE_HANDLED_CEC)
+		return NOTIFY_DONE;
 
 	pr_emerg(HW_ERR "%s\n", decode_error_status(m));
 
@@ -1170,7 +1149,8 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
  err_code:
 	amd_decode_err_code(m->status & 0xffff);
 
-	return NOTIFY_STOP;
+	m->kflags |= MCE_HANDLED_EDAC;
+	return NOTIFY_OK;
 }
 
 static struct notifier_block amd_mce_dec_nb = {
diff --git a/drivers/edac/mce_amd.h b/drivers/edac/mce_amd.h
index 4e9c5e596c6c..4811b18d9606 100644
--- a/drivers/edac/mce_amd.h
+++ b/drivers/edac/mce_amd.h
@@ -7,7 +7,6 @@
 #include <asm/mce.h>
 
 #define EC(x)				((x) & 0xffff)
-#define XEC(x, mask)			(((x) >> 16) & mask)
 
 #define LOW_SYNDROME(x)			(((x) >> 15) & 0xff)
 #define HIGH_SYNDROME(x)		(((x) >> 24) & 0xff)
@@ -77,7 +76,6 @@ struct amd_decoder_ops {
 	bool (*mc2_mce)(u16, u8);
 };
 
-void amd_report_gart_errors(bool);
 void amd_register_ecc_decoder(void (*f)(int, struct mce *));
 void amd_unregister_ecc_decoder(void (*f)(int, struct mce *));
 
diff --git a/drivers/edac/pnd2_edac.c b/drivers/edac/pnd2_edac.c
index bc47328eb485..c1f2e6deb021 100644
--- a/drivers/edac/pnd2_edac.c
+++ b/drivers/edac/pnd2_edac.c
@@ -1396,11 +1396,8 @@ static int pnd2_mce_check_error(struct notifier_block *nb, unsigned long val, vo
 	struct dram_addr daddr;
 	char *type;
 
-	if (edac_get_report_status() == EDAC_REPORTING_DISABLED)
-		return NOTIFY_DONE;
-
 	mci = pnd2_mci;
-	if (!mci)
+	if (!mci || (mce->kflags & MCE_HANDLED_CEC))
 		return NOTIFY_DONE;
 
 	/*
@@ -1429,7 +1426,8 @@ static int pnd2_mce_check_error(struct notifier_block *nb, unsigned long val, vo
 	pnd2_mce_output_error(mci, mce, &daddr);
 
 	/* Advice mcelog that the error were handled */
-	return NOTIFY_STOP;
+	mce->kflags |= MCE_HANDLED_EDAC;
+	return NOTIFY_OK;
 }
 
 static struct notifier_block pnd2_mce_dec = {
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index 7d51c82be62b..d414698ca324 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -3134,7 +3134,7 @@ static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val,
 	struct mem_ctl_info *mci;
 	char *type;
 
-	if (edac_get_report_status() == EDAC_REPORTING_DISABLED)
+	if (mce->kflags & MCE_HANDLED_CEC)
 		return NOTIFY_DONE;
 
 	/*
@@ -3183,7 +3183,8 @@ static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val,
 	sbridge_mce_output_error(mci, mce);
 
 	/* Advice mcelog that the error were handled */
-	return NOTIFY_STOP;
+	mce->kflags |= MCE_HANDLED_EDAC;
+	return NOTIFY_OK;
 }
 
 static struct notifier_block sbridge_mce_dec = {
@@ -3523,8 +3524,6 @@ static int __init sbridge_init(void)
 
 	if (rc >= 0) {
 		mce_register_decode_chain(&sbridge_mce_dec);
-		if (edac_get_report_status() == EDAC_REPORTING_DISABLED)
-			sbridge_printk(KERN_WARNING, "Loading driver, error reporting disabled.\n");
 		return 0;
 	}
 
diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c
index 46be1a77bd1d..6d8d6dc626bf 100644
--- a/drivers/edac/skx_common.c
+++ b/drivers/edac/skx_common.c
@@ -573,7 +573,7 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val,
 	struct mem_ctl_info *mci;
 	char *type;
 
-	if (edac_get_report_status() == EDAC_REPORTING_DISABLED)
+	if (mce->kflags & MCE_HANDLED_CEC)
 		return NOTIFY_DONE;
 
 	/* ignore unless this is memory related with an address */
@@ -615,6 +615,7 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val,
 
 	skx_mce_output_error(mci, mce, &res);
 
+	mce->kflags |= MCE_HANDLED_EDAC;
 	return NOTIFY_DONE;
 }
 
diff --git a/drivers/eisa/Kconfig b/drivers/eisa/Kconfig
index ffc894b4d782..c8bbf90209f5 100644
--- a/drivers/eisa/Kconfig
+++ b/drivers/eisa/Kconfig
@@ -9,7 +9,7 @@ config HAVE_EISA
 menuconfig EISA
 	bool "EISA support"
 	depends on HAVE_EISA
-	---help---
+	help
 	  The Extended Industry Standard Architecture (EISA) bus was
 	  developed as an open alternative to the IBM MicroChannel bus.
 
@@ -26,7 +26,7 @@ config EISA_VLB_PRIMING
 	bool "Vesa Local Bus priming"
 	depends on X86 && EISA
 	default n
-	---help---
+	help
 	  Activate this option if your system contains a Vesa Local
 	  Bus (VLB) card that identify itself as an EISA card (such as
 	  the Adaptec AHA-284x).
@@ -37,7 +37,7 @@ config EISA_PCI_EISA
 	bool "Generic PCI/EISA bridge"
 	depends on !PARISC && PCI && EISA
 	default y
-	---help---
+	help
 	  Activate this option if your system contains a PCI to EISA
 	  bridge. If your system have both PCI and EISA slots, you
 	  certainly need this option.
@@ -51,7 +51,7 @@ config EISA_VIRTUAL_ROOT
 	bool "EISA virtual root device"
 	depends on EISA && (ALPHA || X86)
 	default y
-	---help---
+	help
 	  Activate this option if your system only have EISA bus
 	  (no PCI slots). The Alpha Jensen is an example of such
 	  a system.
@@ -62,7 +62,7 @@ config EISA_NAMES
 	bool "EISA device name database"
 	depends on EISA
 	default y
-	---help---
+	help
 	  By default, the kernel contains a database of all known EISA
 	  device names to make the information in sysfs comprehensible
 	  to the user. This database increases size of the kernel
diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c
index ad02dc6747a4..0317b614b680 100644
--- a/drivers/extcon/extcon-adc-jack.c
+++ b/drivers/extcon/extcon-adc-jack.c
@@ -124,7 +124,7 @@ static int adc_jack_probe(struct platform_device *pdev)
 	for (i = 0; data->adc_conditions[i].id != EXTCON_NONE; i++);
 	data->num_conditions = i;
 
-	data->chan = iio_channel_get(&pdev->dev, pdata->consumer_channel);
+	data->chan = devm_iio_channel_get(&pdev->dev, pdata->consumer_channel);
 	if (IS_ERR(data->chan))
 		return PTR_ERR(data->chan);
 
@@ -164,7 +164,6 @@ static int adc_jack_remove(struct platform_device *pdev)
 
 	free_irq(data->irq, data);
 	cancel_work_sync(&data->handler.work);
-	iio_channel_release(data->chan);
 
 	return 0;
 }
diff --git a/drivers/extcon/extcon-arizona.c b/drivers/extcon/extcon-arizona.c
index 7401733db08b..aae82db542a5 100644
--- a/drivers/extcon/extcon-arizona.c
+++ b/drivers/extcon/extcon-arizona.c
@@ -1460,7 +1460,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
 	if (!info->input) {
 		dev_err(arizona->dev, "Can't allocate input dev\n");
 		ret = -ENOMEM;
-		goto err_register;
+		return ret;
 	}
 
 	info->input->name = "Headset";
@@ -1492,7 +1492,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
 		if (ret != 0) {
 			dev_err(arizona->dev, "Failed to request GPIO%d: %d\n",
 				pdata->micd_pol_gpio, ret);
-			goto err_register;
+			return ret;
 		}
 
 		info->micd_pol_gpio = gpio_to_desc(pdata->micd_pol_gpio);
@@ -1515,7 +1515,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
 			dev_err(arizona->dev,
 				"Failed to get microphone polarity GPIO: %d\n",
 				ret);
-			goto err_register;
+			return ret;
 		}
 	}
 
@@ -1672,7 +1672,7 @@ static int arizona_extcon_probe(struct platform_device *pdev)
 	if (ret != 0) {
 		dev_err(&pdev->dev, "Failed to get JACKDET rise IRQ: %d\n",
 			ret);
-		goto err_gpio;
+		goto err_pm;
 	}
 
 	ret = arizona_set_irq_wake(arizona, jack_irq_rise, 1);
@@ -1721,14 +1721,14 @@ static int arizona_extcon_probe(struct platform_device *pdev)
 		dev_warn(arizona->dev, "Failed to set MICVDD to bypass: %d\n",
 			 ret);
 
-	pm_runtime_put(&pdev->dev);
-
 	ret = input_register_device(info->input);
 	if (ret) {
 		dev_err(&pdev->dev, "Can't register input device: %d\n", ret);
 		goto err_hpdet;
 	}
 
+	pm_runtime_put(&pdev->dev);
+
 	return 0;
 
 err_hpdet:
@@ -1743,10 +1743,11 @@ err_rise_wake:
 	arizona_set_irq_wake(arizona, jack_irq_rise, 0);
 err_rise:
 	arizona_free_irq(arizona, jack_irq_rise, info);
+err_pm:
+	pm_runtime_put(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
 err_gpio:
 	gpiod_put(info->micd_pol_gpio);
-err_register:
-	pm_runtime_disable(&pdev->dev);
 	return ret;
 }
 
diff --git a/drivers/extcon/extcon-max14577.c b/drivers/extcon/extcon-max14577.c
index 32f663436e6e..cc47d626095c 100644
--- a/drivers/extcon/extcon-max14577.c
+++ b/drivers/extcon/extcon-max14577.c
@@ -782,9 +782,19 @@ static const struct platform_device_id max14577_muic_id[] = {
 };
 MODULE_DEVICE_TABLE(platform, max14577_muic_id);
 
+static const struct of_device_id of_max14577_muic_dt_match[] = {
+	{ .compatible = "maxim,max14577-muic",
+	  .data = (void *)MAXIM_DEVICE_TYPE_MAX14577, },
+	{ .compatible = "maxim,max77836-muic",
+	  .data = (void *)MAXIM_DEVICE_TYPE_MAX77836, },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, of_max14577_muic_dt_match);
+
 static struct platform_driver max14577_muic_driver = {
 	.driver		= {
 		.name	= "max14577-muic",
+		.of_match_table = of_max14577_muic_dt_match,
 	},
 	.probe		= max14577_muic_probe,
 	.remove		= max14577_muic_remove,
diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c
index 2dfbfec572f9..0a6438cbb3f3 100644
--- a/drivers/extcon/extcon.c
+++ b/drivers/extcon/extcon.c
@@ -900,7 +900,7 @@ int extcon_register_notifier(struct extcon_dev *edev, unsigned int id,
 			     struct notifier_block *nb)
 {
 	unsigned long flags;
-	int ret, idx = -EINVAL;
+	int ret, idx;
 
 	if (!edev || !nb)
 		return -EINVAL;
diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c
index c7ea4f2d5ca6..fb6c651214f3 100644
--- a/drivers/firewire/core-cdev.c
+++ b/drivers/firewire/core-cdev.c
@@ -117,7 +117,7 @@ struct inbound_transaction_resource {
 struct descriptor_resource {
 	struct client_resource resource;
 	struct fw_descriptor descriptor;
-	u32 data[0];
+	u32 data[];
 };
 
 struct iso_resource {
diff --git a/drivers/firewire/core-transaction.c b/drivers/firewire/core-transaction.c
index 404a035f104d..439d918bbaaf 100644
--- a/drivers/firewire/core-transaction.c
+++ b/drivers/firewire/core-transaction.c
@@ -620,7 +620,7 @@ struct fw_request {
 	u32 request_header[4];
 	int ack;
 	u32 length;
-	u32 data[0];
+	u32 data[];
 };
 
 static void free_response_callback(struct fw_packet *packet,
diff --git a/drivers/firewire/core.h b/drivers/firewire/core.h
index 4b0e4ee655a1..71d5f16f311c 100644
--- a/drivers/firewire/core.h
+++ b/drivers/firewire/core.h
@@ -191,7 +191,7 @@ struct fw_node {
 	/* Upper layer specific data. */
 	void *data;
 
-	struct fw_node *ports[0];
+	struct fw_node *ports[];
 };
 
 static inline struct fw_node *fw_node_get(struct fw_node *node)
diff --git a/drivers/firewire/nosy.c b/drivers/firewire/nosy.c
index 6ca2f5ab6c57..5fd6a60b6741 100644
--- a/drivers/firewire/nosy.c
+++ b/drivers/firewire/nosy.c
@@ -52,7 +52,7 @@ struct pcl {
 
 struct packet {
 	unsigned int length;
-	char data[0];
+	char data[];
 };
 
 struct packet_buffer {
diff --git a/drivers/firewire/ohci.c b/drivers/firewire/ohci.c
index 33269316f111..54fdc39cd0bc 100644
--- a/drivers/firewire/ohci.c
+++ b/drivers/firewire/ohci.c
@@ -111,7 +111,7 @@ struct descriptor_buffer {
 	dma_addr_t buffer_bus;
 	size_t buffer_size;
 	size_t used;
-	struct descriptor buffer[0];
+	struct descriptor buffer[];
 };
 
 struct context {
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index 4843e94713a4..fbd785dd0513 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -178,8 +178,9 @@ config ISCSI_IBFT
 	  Otherwise, say N.
 
 config RASPBERRYPI_FIRMWARE
-	tristate "Raspberry Pi Firmware Driver"
+	bool "Raspberry Pi Firmware Driver"
 	depends on BCM2835_MBOX
+	default USB_PCI
 	help
 	  This option enables support for communicating with the firmware on the
 	  Raspberry Pi.
diff --git a/drivers/firmware/arm_scmi/Makefile b/drivers/firmware/arm_scmi/Makefile
index 6694d0d908d6..1cad32b38b29 100644
--- a/drivers/firmware/arm_scmi/Makefile
+++ b/drivers/firmware/arm_scmi/Makefile
@@ -2,6 +2,8 @@
 obj-y	= scmi-bus.o scmi-driver.o scmi-protocols.o scmi-transport.o
 scmi-bus-y = bus.o
 scmi-driver-y = driver.o
-scmi-transport-y = mailbox.o shmem.o
+scmi-transport-y = shmem.o
+scmi-transport-$(CONFIG_MAILBOX) += mailbox.o
+scmi-transport-$(CONFIG_ARM_PSCI_FW) += smc.o
 scmi-protocols-y = base.o clock.o perf.o power.o reset.o sensors.o
 obj-$(CONFIG_ARM_SCMI_POWER_DOMAIN) += scmi_pm_domain.o
diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c
index f804e8af6521..ce7d9203e41b 100644
--- a/drivers/firmware/arm_scmi/base.c
+++ b/drivers/firmware/arm_scmi/base.c
@@ -14,6 +14,13 @@ enum scmi_base_protocol_cmd {
 	BASE_DISCOVER_LIST_PROTOCOLS = 0x6,
 	BASE_DISCOVER_AGENT = 0x7,
 	BASE_NOTIFY_ERRORS = 0x8,
+	BASE_SET_DEVICE_PERMISSIONS = 0x9,
+	BASE_SET_PROTOCOL_PERMISSIONS = 0xa,
+	BASE_RESET_AGENT_CONFIGURATION = 0xb,
+};
+
+enum scmi_base_protocol_notify {
+	BASE_ERROR_EVENT = 0x0,
 };
 
 struct scmi_msg_resp_base_attributes {
diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h
index 5ac06469b01c..31fe5a22a011 100644
--- a/drivers/firmware/arm_scmi/common.h
+++ b/drivers/firmware/arm_scmi/common.h
@@ -178,6 +178,8 @@ struct scmi_chan_info {
  * @send_message: Callback to send a message
  * @mark_txdone: Callback to mark tx as done
  * @fetch_response: Callback to fetch response
+ * @fetch_notification: Callback to fetch notification
+ * @clear_channel: Callback to clear a channel
  * @poll_done: Callback to poll transfer status
  */
 struct scmi_transport_ops {
@@ -190,6 +192,9 @@ struct scmi_transport_ops {
 	void (*mark_txdone)(struct scmi_chan_info *cinfo, int ret);
 	void (*fetch_response)(struct scmi_chan_info *cinfo,
 			       struct scmi_xfer *xfer);
+	void (*fetch_notification)(struct scmi_chan_info *cinfo,
+				   size_t max_len, struct scmi_xfer *xfer);
+	void (*clear_channel)(struct scmi_chan_info *cinfo);
 	bool (*poll_done)(struct scmi_chan_info *cinfo, struct scmi_xfer *xfer);
 };
 
@@ -210,6 +215,9 @@ struct scmi_desc {
 };
 
 extern const struct scmi_desc scmi_mailbox_desc;
+#ifdef CONFIG_HAVE_ARM_SMCCC
+extern const struct scmi_desc scmi_smc_desc;
+#endif
 
 void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr);
 void scmi_free_channel(struct scmi_chan_info *cinfo, struct idr *idr, int id);
@@ -222,5 +230,8 @@ void shmem_tx_prepare(struct scmi_shared_mem __iomem *shmem,
 u32 shmem_read_header(struct scmi_shared_mem __iomem *shmem);
 void shmem_fetch_response(struct scmi_shared_mem __iomem *shmem,
 			  struct scmi_xfer *xfer);
+void shmem_fetch_notification(struct scmi_shared_mem __iomem *shmem,
+			      size_t max_len, struct scmi_xfer *xfer);
+void shmem_clear_channel(struct scmi_shared_mem __iomem *shmem);
 bool shmem_poll_done(struct scmi_shared_mem __iomem *shmem,
 		     struct scmi_xfer *xfer);
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index dbec767222e9..7483cacf63f9 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -76,6 +76,7 @@ struct scmi_xfers_info {
  *	implementation version and (sub-)vendor identification.
  * @handle: Instance of SCMI handle to send to clients
  * @tx_minfo: Universal Transmit Message management info
+ * @rx_minfo: Universal Receive Message management info
  * @tx_idr: IDR object to map protocol id to Tx channel info pointer
  * @rx_idr: IDR object to map protocol id to Rx channel info pointer
  * @protocols_imp: List of protocols implemented, currently maximum of
@@ -89,6 +90,7 @@ struct scmi_info {
 	struct scmi_revision_info version;
 	struct scmi_handle handle;
 	struct scmi_xfers_info tx_minfo;
+	struct scmi_xfers_info rx_minfo;
 	struct idr tx_idr;
 	struct idr rx_idr;
 	u8 *protocols_imp;
@@ -200,37 +202,66 @@ __scmi_xfer_put(struct scmi_xfers_info *minfo, struct scmi_xfer *xfer)
 	spin_unlock_irqrestore(&minfo->xfer_lock, flags);
 }
 
-/**
- * scmi_rx_callback() - callback for receiving messages
- *
- * @cinfo: SCMI channel info
- * @msg_hdr: Message header
- *
- * Processes one received message to appropriate transfer information and
- * signals completion of the transfer.
- *
- * NOTE: This function will be invoked in IRQ context, hence should be
- * as optimal as possible.
- */
-void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr)
+static void scmi_handle_notification(struct scmi_chan_info *cinfo, u32 msg_hdr)
 {
-	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);
-	struct scmi_xfers_info *minfo = &info->tx_minfo;
-	u16 xfer_id = MSG_XTRACT_TOKEN(msg_hdr);
-	u8 msg_type = MSG_XTRACT_TYPE(msg_hdr);
-	struct device *dev = cinfo->dev;
 	struct scmi_xfer *xfer;
+	struct device *dev = cinfo->dev;
+	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);
+	struct scmi_xfers_info *minfo = &info->rx_minfo;
 
-	if (msg_type == MSG_TYPE_NOTIFICATION)
-		return; /* Notifications not yet supported */
+	xfer = scmi_xfer_get(cinfo->handle, minfo);
+	if (IS_ERR(xfer)) {
+		dev_err(dev, "failed to get free message slot (%ld)\n",
+			PTR_ERR(xfer));
+		info->desc->ops->clear_channel(cinfo);
+		return;
+	}
+
+	unpack_scmi_header(msg_hdr, &xfer->hdr);
+	scmi_dump_header_dbg(dev, &xfer->hdr);
+	info->desc->ops->fetch_notification(cinfo, info->desc->max_msg_size,
+					    xfer);
+
+	trace_scmi_rx_done(xfer->transfer_id, xfer->hdr.id,
+			   xfer->hdr.protocol_id, xfer->hdr.seq,
+			   MSG_TYPE_NOTIFICATION);
+
+	__scmi_xfer_put(minfo, xfer);
+
+	info->desc->ops->clear_channel(cinfo);
+}
+
+static void scmi_handle_response(struct scmi_chan_info *cinfo,
+				 u16 xfer_id, u8 msg_type)
+{
+	struct scmi_xfer *xfer;
+	struct device *dev = cinfo->dev;
+	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);
+	struct scmi_xfers_info *minfo = &info->tx_minfo;
 
 	/* Are we even expecting this? */
 	if (!test_bit(xfer_id, minfo->xfer_alloc_table)) {
 		dev_err(dev, "message for %d is not expected!\n", xfer_id);
+		info->desc->ops->clear_channel(cinfo);
 		return;
 	}
 
 	xfer = &minfo->xfer_block[xfer_id];
+	/*
+	 * Even if a response was indeed expected on this slot at this point,
+	 * a buggy platform could wrongly reply feeding us an unexpected
+	 * delayed response we're not prepared to handle: bail-out safely
+	 * blaming firmware.
+	 */
+	if (unlikely(msg_type == MSG_TYPE_DELAYED_RESP && !xfer->async_done)) {
+		dev_err(dev,
+			"Delayed Response for %d not expected! Buggy F/W ?\n",
+			xfer_id);
+		info->desc->ops->clear_channel(cinfo);
+		/* It was unexpected, so nobody will clear the xfer if not us */
+		__scmi_xfer_put(minfo, xfer);
+		return;
+	}
 
 	scmi_dump_header_dbg(dev, &xfer->hdr);
 
@@ -240,10 +271,43 @@ void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr)
 			   xfer->hdr.protocol_id, xfer->hdr.seq,
 			   msg_type);
 
-	if (msg_type == MSG_TYPE_DELAYED_RESP)
+	if (msg_type == MSG_TYPE_DELAYED_RESP) {
+		info->desc->ops->clear_channel(cinfo);
 		complete(xfer->async_done);
-	else
+	} else {
 		complete(&xfer->done);
+	}
+}
+
+/**
+ * scmi_rx_callback() - callback for receiving messages
+ *
+ * @cinfo: SCMI channel info
+ * @msg_hdr: Message header
+ *
+ * Processes one received message to appropriate transfer information and
+ * signals completion of the transfer.
+ *
+ * NOTE: This function will be invoked in IRQ context, hence should be
+ * as optimal as possible.
+ */
+void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr)
+{
+	u16 xfer_id = MSG_XTRACT_TOKEN(msg_hdr);
+	u8 msg_type = MSG_XTRACT_TYPE(msg_hdr);
+
+	switch (msg_type) {
+	case MSG_TYPE_NOTIFICATION:
+		scmi_handle_notification(cinfo, msg_hdr);
+		break;
+	case MSG_TYPE_COMMAND:
+	case MSG_TYPE_DELAYED_RESP:
+		scmi_handle_response(cinfo, xfer_id, msg_type);
+		break;
+	default:
+		WARN_ONCE(1, "received unknown msg_type:%d\n", msg_type);
+		break;
+	}
 }
 
 /**
@@ -525,13 +589,13 @@ int scmi_handle_put(const struct scmi_handle *handle)
 	return 0;
 }
 
-static int scmi_xfer_info_init(struct scmi_info *sinfo)
+static int __scmi_xfer_info_init(struct scmi_info *sinfo,
+				 struct scmi_xfers_info *info)
 {
 	int i;
 	struct scmi_xfer *xfer;
 	struct device *dev = sinfo->dev;
 	const struct scmi_desc *desc = sinfo->desc;
-	struct scmi_xfers_info *info = &sinfo->tx_minfo;
 
 	/* Pre-allocated messages, no more than what hdr.seq can support */
 	if (WARN_ON(desc->max_msg >= MSG_TOKEN_MAX)) {
@@ -566,6 +630,16 @@ static int scmi_xfer_info_init(struct scmi_info *sinfo)
 	return 0;
 }
 
+static int scmi_xfer_info_init(struct scmi_info *sinfo)
+{
+	int ret = __scmi_xfer_info_init(sinfo, &sinfo->tx_minfo);
+
+	if (!ret && idr_find(&sinfo->rx_idr, SCMI_PROTOCOL_BASE))
+		ret = __scmi_xfer_info_init(sinfo, &sinfo->rx_minfo);
+
+	return ret;
+}
+
 static int scmi_chan_setup(struct scmi_info *info, struct device *dev,
 			   int prot_id, bool tx)
 {
@@ -699,10 +773,6 @@ static int scmi_probe(struct platform_device *pdev)
 	info->desc = desc;
 	INIT_LIST_HEAD(&info->node);
 
-	ret = scmi_xfer_info_init(info);
-	if (ret)
-		return ret;
-
 	platform_set_drvdata(pdev, info);
 	idr_init(&info->tx_idr);
 	idr_init(&info->rx_idr);
@@ -715,6 +785,10 @@ static int scmi_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	ret = scmi_xfer_info_init(info);
+	if (ret)
+		return ret;
+
 	ret = scmi_base_protocol_init(handle);
 	if (ret) {
 		dev_err(dev, "unable to communicate with SCMI(%d)\n", ret);
@@ -827,6 +901,9 @@ ATTRIBUTE_GROUPS(versions);
 /* Each compatible listed below must have descriptor associated with it */
 static const struct of_device_id scmi_of_match[] = {
 	{ .compatible = "arm,scmi", .data = &scmi_mailbox_desc },
+#ifdef CONFIG_ARM_PSCI_FW
+	{ .compatible = "arm,scmi-smc", .data = &scmi_smc_desc},
+#endif
 	{ /* Sentinel */ },
 };
 
diff --git a/drivers/firmware/arm_scmi/mailbox.c b/drivers/firmware/arm_scmi/mailbox.c
index 73077bbc4ad9..6998dc86b5ce 100644
--- a/drivers/firmware/arm_scmi/mailbox.c
+++ b/drivers/firmware/arm_scmi/mailbox.c
@@ -158,6 +158,21 @@ static void mailbox_fetch_response(struct scmi_chan_info *cinfo,
 	shmem_fetch_response(smbox->shmem, xfer);
 }
 
+static void mailbox_fetch_notification(struct scmi_chan_info *cinfo,
+				       size_t max_len, struct scmi_xfer *xfer)
+{
+	struct scmi_mailbox *smbox = cinfo->transport_info;
+
+	shmem_fetch_notification(smbox->shmem, max_len, xfer);
+}
+
+static void mailbox_clear_channel(struct scmi_chan_info *cinfo)
+{
+	struct scmi_mailbox *smbox = cinfo->transport_info;
+
+	shmem_clear_channel(smbox->shmem);
+}
+
 static bool
 mailbox_poll_done(struct scmi_chan_info *cinfo, struct scmi_xfer *xfer)
 {
@@ -173,6 +188,8 @@ static struct scmi_transport_ops scmi_mailbox_ops = {
 	.send_message = mailbox_send_message,
 	.mark_txdone = mailbox_mark_txdone,
 	.fetch_response = mailbox_fetch_response,
+	.fetch_notification = mailbox_fetch_notification,
+	.clear_channel = mailbox_clear_channel,
 	.poll_done = mailbox_poll_done,
 };
 
diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c
index 34f3a917dd8d..eadc171e254b 100644
--- a/drivers/firmware/arm_scmi/perf.c
+++ b/drivers/firmware/arm_scmi/perf.c
@@ -27,6 +27,11 @@ enum scmi_performance_protocol_cmd {
 	PERF_DESCRIBE_FASTCHANNEL = 0xb,
 };
 
+enum scmi_performance_protocol_notify {
+	PERFORMANCE_LIMITS_CHANGED = 0x0,
+	PERFORMANCE_LEVEL_CHANGED = 0x1,
+};
+
 struct scmi_opp {
 	u32 perf;
 	u32 power;
diff --git a/drivers/firmware/arm_scmi/power.c b/drivers/firmware/arm_scmi/power.c
index 214886ce84f1..cf7f0312381b 100644
--- a/drivers/firmware/arm_scmi/power.c
+++ b/drivers/firmware/arm_scmi/power.c
@@ -12,6 +12,12 @@ enum scmi_power_protocol_cmd {
 	POWER_STATE_SET = 0x4,
 	POWER_STATE_GET = 0x5,
 	POWER_STATE_NOTIFY = 0x6,
+	POWER_STATE_CHANGE_REQUESTED_NOTIFY = 0x7,
+};
+
+enum scmi_power_protocol_notify {
+	POWER_STATE_CHANGED = 0x0,
+	POWER_STATE_CHANGE_REQUESTED = 0x1,
 };
 
 struct scmi_msg_resp_power_attributes {
diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c
index eba61b9c1f53..db1b1ab303da 100644
--- a/drivers/firmware/arm_scmi/sensors.c
+++ b/drivers/firmware/arm_scmi/sensors.c
@@ -14,6 +14,10 @@ enum scmi_sensor_protocol_cmd {
 	SENSOR_READING_GET = 0x6,
 };
 
+enum scmi_sensor_protocol_notify {
+	SENSOR_TRIP_POINT_EVENT = 0x0,
+};
+
 struct scmi_msg_resp_sensor_attributes {
 	__le16 num_sensors;
 	u8 max_requests;
diff --git a/drivers/firmware/arm_scmi/shmem.c b/drivers/firmware/arm_scmi/shmem.c
index e1e816e0018c..0e3eaea5d852 100644
--- a/drivers/firmware/arm_scmi/shmem.c
+++ b/drivers/firmware/arm_scmi/shmem.c
@@ -67,6 +67,21 @@ void shmem_fetch_response(struct scmi_shared_mem __iomem *shmem,
 	memcpy_fromio(xfer->rx.buf, shmem->msg_payload + 4, xfer->rx.len);
 }
 
+void shmem_fetch_notification(struct scmi_shared_mem __iomem *shmem,
+			      size_t max_len, struct scmi_xfer *xfer)
+{
+	/* Skip only the length of header in shmem area i.e 4 bytes */
+	xfer->rx.len = min_t(size_t, max_len, ioread32(&shmem->length) - 4);
+
+	/* Take a copy to the rx buffer.. */
+	memcpy_fromio(xfer->rx.buf, shmem->msg_payload, xfer->rx.len);
+}
+
+void shmem_clear_channel(struct scmi_shared_mem __iomem *shmem)
+{
+	iowrite32(SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE, &shmem->channel_status);
+}
+
 bool shmem_poll_done(struct scmi_shared_mem __iomem *shmem,
 		     struct scmi_xfer *xfer)
 {
diff --git a/drivers/firmware/arm_scmi/smc.c b/drivers/firmware/arm_scmi/smc.c
new file mode 100644
index 000000000000..49bc4b0e8428
--- /dev/null
+++ b/drivers/firmware/arm_scmi/smc.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * System Control and Management Interface (SCMI) Message SMC/HVC
+ * Transport driver
+ *
+ * Copyright 2020 NXP
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+
+#include "common.h"
+
+/**
+ * struct scmi_smc - Structure representing a SCMI smc transport
+ *
+ * @cinfo: SCMI channel info
+ * @shmem: Transmit/Receive shared memory area
+ * @func_id: smc/hvc call function id
+ */
+
+struct scmi_smc {
+	struct scmi_chan_info *cinfo;
+	struct scmi_shared_mem __iomem *shmem;
+	struct mutex shmem_lock;
+	u32 func_id;
+};
+
+static bool smc_chan_available(struct device *dev, int idx)
+{
+	struct device_node *np = of_parse_phandle(dev->of_node, "shmem", 0);
+	if (!np)
+		return false;
+
+	of_node_put(np);
+	return true;
+}
+
+static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
+			  bool tx)
+{
+	struct device *cdev = cinfo->dev;
+	struct scmi_smc *scmi_info;
+	resource_size_t size;
+	struct resource res;
+	struct device_node *np;
+	u32 func_id;
+	int ret;
+
+	if (!tx)
+		return -ENODEV;
+
+	scmi_info = devm_kzalloc(dev, sizeof(*scmi_info), GFP_KERNEL);
+	if (!scmi_info)
+		return -ENOMEM;
+
+	np = of_parse_phandle(cdev->of_node, "shmem", 0);
+	ret = of_address_to_resource(np, 0, &res);
+	of_node_put(np);
+	if (ret) {
+		dev_err(cdev, "failed to get SCMI Tx shared memory\n");
+		return ret;
+	}
+
+	size = resource_size(&res);
+	scmi_info->shmem = devm_ioremap(dev, res.start, size);
+	if (!scmi_info->shmem) {
+		dev_err(dev, "failed to ioremap SCMI Tx shared memory\n");
+		return -EADDRNOTAVAIL;
+	}
+
+	ret = of_property_read_u32(dev->of_node, "arm,smc-id", &func_id);
+	if (ret < 0)
+		return ret;
+
+	scmi_info->func_id = func_id;
+	scmi_info->cinfo = cinfo;
+	mutex_init(&scmi_info->shmem_lock);
+	cinfo->transport_info = scmi_info;
+
+	return 0;
+}
+
+static int smc_chan_free(int id, void *p, void *data)
+{
+	struct scmi_chan_info *cinfo = p;
+	struct scmi_smc *scmi_info = cinfo->transport_info;
+
+	cinfo->transport_info = NULL;
+	scmi_info->cinfo = NULL;
+
+	scmi_free_channel(cinfo, data, id);
+
+	return 0;
+}
+
+static int smc_send_message(struct scmi_chan_info *cinfo,
+			    struct scmi_xfer *xfer)
+{
+	struct scmi_smc *scmi_info = cinfo->transport_info;
+	struct arm_smccc_res res;
+
+	mutex_lock(&scmi_info->shmem_lock);
+
+	shmem_tx_prepare(scmi_info->shmem, xfer);
+
+	arm_smccc_1_1_invoke(scmi_info->func_id, 0, 0, 0, 0, 0, 0, 0, &res);
+	scmi_rx_callback(scmi_info->cinfo, shmem_read_header(scmi_info->shmem));
+
+	mutex_unlock(&scmi_info->shmem_lock);
+
+	/* Only SMCCC_RET_NOT_SUPPORTED is valid error code */
+	if (res.a0)
+		return -EOPNOTSUPP;
+	return 0;
+}
+
+static void smc_fetch_response(struct scmi_chan_info *cinfo,
+			       struct scmi_xfer *xfer)
+{
+	struct scmi_smc *scmi_info = cinfo->transport_info;
+
+	shmem_fetch_response(scmi_info->shmem, xfer);
+}
+
+static bool
+smc_poll_done(struct scmi_chan_info *cinfo, struct scmi_xfer *xfer)
+{
+	struct scmi_smc *scmi_info = cinfo->transport_info;
+
+	return shmem_poll_done(scmi_info->shmem, xfer);
+}
+
+static struct scmi_transport_ops scmi_smc_ops = {
+	.chan_available = smc_chan_available,
+	.chan_setup = smc_chan_setup,
+	.chan_free = smc_chan_free,
+	.send_message = smc_send_message,
+	.fetch_response = smc_fetch_response,
+	.poll_done = smc_poll_done,
+};
+
+const struct scmi_desc scmi_smc_desc = {
+	.ops = &scmi_smc_ops,
+	.max_rx_timeout_ms = 30,
+	.max_msg = 1,
+	.max_msg_size = 128,
+};
diff --git a/drivers/firmware/dmi-id.c b/drivers/firmware/dmi-id.c
index ff39f64f2aae..86d71b0212b1 100644
--- a/drivers/firmware/dmi-id.c
+++ b/drivers/firmware/dmi-id.c
@@ -42,6 +42,8 @@ DEFINE_DMI_ATTR_WITH_SHOW(bios_vendor,		0444, DMI_BIOS_VENDOR);
 DEFINE_DMI_ATTR_WITH_SHOW(bios_version,		0444, DMI_BIOS_VERSION);
 DEFINE_DMI_ATTR_WITH_SHOW(bios_date,		0444, DMI_BIOS_DATE);
 DEFINE_DMI_ATTR_WITH_SHOW(sys_vendor,		0444, DMI_SYS_VENDOR);
+DEFINE_DMI_ATTR_WITH_SHOW(bios_release,		0444, DMI_BIOS_RELEASE);
+DEFINE_DMI_ATTR_WITH_SHOW(ec_firmware_release,	0444, DMI_EC_FIRMWARE_RELEASE);
 DEFINE_DMI_ATTR_WITH_SHOW(product_name,		0444, DMI_PRODUCT_NAME);
 DEFINE_DMI_ATTR_WITH_SHOW(product_version,	0444, DMI_PRODUCT_VERSION);
 DEFINE_DMI_ATTR_WITH_SHOW(product_serial,	0400, DMI_PRODUCT_SERIAL);
@@ -78,6 +80,8 @@ static ssize_t get_modalias(char *buffer, size_t buffer_size)
 		{ "bvn", DMI_BIOS_VENDOR },
 		{ "bvr", DMI_BIOS_VERSION },
 		{ "bd",  DMI_BIOS_DATE },
+		{ "br",  DMI_BIOS_RELEASE },
+		{ "efr", DMI_EC_FIRMWARE_RELEASE },
 		{ "svn", DMI_SYS_VENDOR },
 		{ "pn",  DMI_PRODUCT_NAME },
 		{ "pvr", DMI_PRODUCT_VERSION },
@@ -187,6 +191,8 @@ static void __init dmi_id_init_attr_table(void)
 	ADD_DMI_ATTR(bios_vendor,       DMI_BIOS_VENDOR);
 	ADD_DMI_ATTR(bios_version,      DMI_BIOS_VERSION);
 	ADD_DMI_ATTR(bios_date,         DMI_BIOS_DATE);
+	ADD_DMI_ATTR(bios_release,      DMI_BIOS_RELEASE);
+	ADD_DMI_ATTR(ec_firmware_release, DMI_EC_FIRMWARE_RELEASE);
 	ADD_DMI_ATTR(sys_vendor,        DMI_SYS_VENDOR);
 	ADD_DMI_ATTR(product_name,      DMI_PRODUCT_NAME);
 	ADD_DMI_ATTR(product_version,   DMI_PRODUCT_VERSION);
diff --git a/drivers/firmware/dmi-sysfs.c b/drivers/firmware/dmi-sysfs.c
index b6180023eba7..8b8127fa8955 100644
--- a/drivers/firmware/dmi-sysfs.c
+++ b/drivers/firmware/dmi-sysfs.c
@@ -262,7 +262,7 @@ struct dmi_system_event_log {
 	u8	header_format;
 	u8	type_descriptors_supported_count;
 	u8	per_log_type_descriptor_length;
-	u8	supported_log_type_descriptos[0];
+	u8	supported_log_type_descriptos[];
 } __packed;
 
 #define DMI_SYSFS_SEL_FIELD(_field) \
diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c
index f59163cb7cba..5066d1f1d687 100644
--- a/drivers/firmware/dmi_scan.c
+++ b/drivers/firmware/dmi_scan.c
@@ -186,6 +186,34 @@ static void __init dmi_save_ident(const struct dmi_header *dm, int slot,
 	dmi_ident[slot] = p;
 }
 
+static void __init dmi_save_release(const struct dmi_header *dm, int slot,
+		int index)
+{
+	const u8 *minor, *major;
+	char *s;
+
+	/* If the table doesn't have the field, let's return */
+	if (dmi_ident[slot] || dm->length < index)
+		return;
+
+	minor = (u8 *) dm + index;
+	major = (u8 *) dm + index - 1;
+
+	/* As per the spec, if the system doesn't support this field,
+	 * the value is FF
+	 */
+	if (*major == 0xFF && *minor == 0xFF)
+		return;
+
+	s = dmi_alloc(8);
+	if (!s)
+		return;
+
+	sprintf(s, "%u.%u", *major, *minor);
+
+	dmi_ident[slot] = s;
+}
+
 static void __init dmi_save_uuid(const struct dmi_header *dm, int slot,
 		int index)
 {
@@ -444,6 +472,8 @@ static void __init dmi_decode(const struct dmi_header *dm, void *dummy)
 		dmi_save_ident(dm, DMI_BIOS_VENDOR, 4);
 		dmi_save_ident(dm, DMI_BIOS_VERSION, 5);
 		dmi_save_ident(dm, DMI_BIOS_DATE, 8);
+		dmi_save_release(dm, DMI_BIOS_RELEASE, 21);
+		dmi_save_release(dm, DMI_EC_FIRMWARE_RELEASE, 23);
 		break;
 	case 1:		/* System Information */
 		dmi_save_ident(dm, DMI_SYS_VENDOR, 4);
diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig
index 6b38f9e5d203..e6fc022bc87e 100644
--- a/drivers/firmware/efi/Kconfig
+++ b/drivers/firmware/efi/Kconfig
@@ -139,7 +139,7 @@ config EFI_BOOTLOADER_CONTROL
 	tristate "EFI Bootloader Control"
 	depends on EFI_VARS
 	default n
-	---help---
+	help
 	  This module installs a reboot hook, such that if reboot() is
 	  invoked with a string argument NNN, "NNN" is copied to the
 	  "LoaderEntryOneShot" EFI variable, to be read by the
diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c
index b876373f2297..3359ae2adf24 100644
--- a/drivers/firmware/efi/arm-runtime.c
+++ b/drivers/firmware/efi/arm-runtime.c
@@ -18,12 +18,12 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/pgtable.h>
 
 #include <asm/cacheflush.h>
 #include <asm/efi.h>
 #include <asm/mmu.h>
 #include <asm/pgalloc.h>
-#include <asm/pgtable.h>
 
 #if defined(CONFIG_PTDUMP_DEBUGFS) && defined(CONFIG_ARM64)
 #include <asm/ptdump.h>
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index 9357d6b6e87c..7f1657b6c30d 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -54,7 +54,7 @@ struct mm_struct efi_mm = {
 	.mm_rb			= RB_ROOT,
 	.mm_users		= ATOMIC_INIT(2),
 	.mm_count		= ATOMIC_INIT(1),
-	.mmap_sem		= __RWSEM_INITIALIZER(efi_mm.mmap_sem),
+	MMAP_LOCK_INITIALIZER(efi_mm)
 	.page_table_lock	= __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock),
 	.mmlist			= LIST_HEAD_INIT(efi_mm.mmlist),
 	.cpu_bitmap		= { [BITS_TO_LONGS(NR_CPUS)] = 0},
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index cce4a7436052..75daaf20374e 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -37,7 +37,9 @@ KBUILD_CFLAGS			:= $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \
 KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
 
 GCOV_PROFILE			:= n
+# Sanitizer runtimes are unavailable and cannot be linked here.
 KASAN_SANITIZE			:= n
+KCSAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
 OBJECT_FILES_NON_STANDARD	:= y
 
diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c
index fd7f0fbec07e..d17e4d6ac9bc 100644
--- a/drivers/firmware/google/memconsole-coreboot.c
+++ b/drivers/firmware/google/memconsole-coreboot.c
@@ -21,7 +21,7 @@
 struct cbmem_cons {
 	u32 size_dont_access_after_boot;
 	u32 cursor;
-	u8  body[0];
+	u8  body[];
 } __packed;
 
 #define CURSOR_MASK ((1 << 28) - 1)
diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c
index db0812263d46..d23c5c69ab52 100644
--- a/drivers/firmware/google/vpd.c
+++ b/drivers/firmware/google/vpd.c
@@ -32,7 +32,7 @@ struct vpd_cbmem {
 	u32 version;
 	u32 ro_size;
 	u32 rw_size;
-	u8  blob[0];
+	u8  blob[];
 };
 
 struct vpd_section {
diff --git a/drivers/firmware/imx/imx-scu.c b/drivers/firmware/imx/imx-scu.c
index f71eaa5bf52d..2ab048222fe9 100644
--- a/drivers/firmware/imx/imx-scu.c
+++ b/drivers/firmware/imx/imx-scu.c
@@ -8,7 +8,6 @@
  */
 
 #include <linux/err.h>
-#include <linux/firmware/imx/types.h>
 #include <linux/firmware/imx/ipc.h>
 #include <linux/firmware/imx/sci.h>
 #include <linux/interrupt.h>
@@ -38,6 +37,7 @@ struct imx_sc_ipc {
 	struct device *dev;
 	struct mutex lock;
 	struct completion done;
+	bool fast_ipc;
 
 	/* temporarily store the SCU msg */
 	u32 *msg;
@@ -115,6 +115,7 @@ static void imx_scu_rx_callback(struct mbox_client *c, void *msg)
 	struct imx_sc_ipc *sc_ipc = sc_chan->sc_ipc;
 	struct imx_sc_rpc_msg *hdr;
 	u32 *data = msg;
+	int i;
 
 	if (!sc_ipc->msg) {
 		dev_warn(sc_ipc->dev, "unexpected rx idx %d 0x%08x, ignore!\n",
@@ -122,6 +123,19 @@ static void imx_scu_rx_callback(struct mbox_client *c, void *msg)
 		return;
 	}
 
+	if (sc_ipc->fast_ipc) {
+		hdr = msg;
+		sc_ipc->rx_size = hdr->size;
+		sc_ipc->msg[0] = *data++;
+
+		for (i = 1; i < sc_ipc->rx_size; i++)
+			sc_ipc->msg[i] = *data++;
+
+		complete(&sc_ipc->done);
+
+		return;
+	}
+
 	if (sc_chan->idx == 0) {
 		hdr = msg;
 		sc_ipc->rx_size = hdr->size;
@@ -143,20 +157,22 @@ static void imx_scu_rx_callback(struct mbox_client *c, void *msg)
 
 static int imx_scu_ipc_write(struct imx_sc_ipc *sc_ipc, void *msg)
 {
-	struct imx_sc_rpc_msg *hdr = msg;
+	struct imx_sc_rpc_msg hdr = *(struct imx_sc_rpc_msg *)msg;
 	struct imx_sc_chan *sc_chan;
 	u32 *data = msg;
 	int ret;
+	int size;
 	int i;
 
 	/* Check size */
-	if (hdr->size > IMX_SC_RPC_MAX_MSG)
+	if (hdr.size > IMX_SC_RPC_MAX_MSG)
 		return -EINVAL;
 
-	dev_dbg(sc_ipc->dev, "RPC SVC %u FUNC %u SIZE %u\n", hdr->svc,
-		hdr->func, hdr->size);
+	dev_dbg(sc_ipc->dev, "RPC SVC %u FUNC %u SIZE %u\n", hdr.svc,
+		hdr.func, hdr.size);
 
-	for (i = 0; i < hdr->size; i++) {
+	size = sc_ipc->fast_ipc ? 1 : hdr.size;
+	for (i = 0; i < size; i++) {
 		sc_chan = &sc_ipc->chans[i % 4];
 
 		/*
@@ -168,8 +184,10 @@ static int imx_scu_ipc_write(struct imx_sc_ipc *sc_ipc, void *msg)
 		 * Wait for tx_done before every send to ensure that no
 		 * queueing happens at the mailbox channel level.
 		 */
-		wait_for_completion(&sc_chan->tx_done);
-		reinit_completion(&sc_chan->tx_done);
+		if (!sc_ipc->fast_ipc) {
+			wait_for_completion(&sc_chan->tx_done);
+			reinit_completion(&sc_chan->tx_done);
+		}
 
 		ret = mbox_send_message(sc_chan->ch, &data[i]);
 		if (ret < 0)
@@ -246,6 +264,8 @@ static int imx_scu_probe(struct platform_device *pdev)
 	struct imx_sc_chan *sc_chan;
 	struct mbox_client *cl;
 	char *chan_name;
+	struct of_phandle_args args;
+	int num_channel;
 	int ret;
 	int i;
 
@@ -253,11 +273,20 @@ static int imx_scu_probe(struct platform_device *pdev)
 	if (!sc_ipc)
 		return -ENOMEM;
 
-	for (i = 0; i < SCU_MU_CHAN_NUM; i++) {
-		if (i < 4)
+	ret = of_parse_phandle_with_args(pdev->dev.of_node, "mboxes",
+					 "#mbox-cells", 0, &args);
+	if (ret)
+		return ret;
+
+	sc_ipc->fast_ipc = of_device_is_compatible(args.np, "fsl,imx8-mu-scu");
+
+	num_channel = sc_ipc->fast_ipc ? 2 : SCU_MU_CHAN_NUM;
+	for (i = 0; i < num_channel; i++) {
+		if (i < num_channel / 2)
 			chan_name = kasprintf(GFP_KERNEL, "tx%d", i);
 		else
-			chan_name = kasprintf(GFP_KERNEL, "rx%d", i - 4);
+			chan_name = kasprintf(GFP_KERNEL, "rx%d",
+					      i - num_channel / 2);
 
 		if (!chan_name)
 			return -ENOMEM;
@@ -269,19 +298,22 @@ static int imx_scu_probe(struct platform_device *pdev)
 		cl->knows_txdone = true;
 		cl->rx_callback = imx_scu_rx_callback;
 
-		/* Initial tx_done completion as "done" */
-		cl->tx_done = imx_scu_tx_done;
-		init_completion(&sc_chan->tx_done);
-		complete(&sc_chan->tx_done);
+		if (!sc_ipc->fast_ipc) {
+			/* Initial tx_done completion as "done" */
+			cl->tx_done = imx_scu_tx_done;
+			init_completion(&sc_chan->tx_done);
+			complete(&sc_chan->tx_done);
+		}
 
 		sc_chan->sc_ipc = sc_ipc;
-		sc_chan->idx = i % 4;
+		sc_chan->idx = i % (num_channel / 2);
 		sc_chan->ch = mbox_request_channel_byname(cl, chan_name);
 		if (IS_ERR(sc_chan->ch)) {
 			ret = PTR_ERR(sc_chan->ch);
 			if (ret != -EPROBE_DEFER)
 				dev_err(dev, "Failed to request mbox chan %s ret %d\n",
 					chan_name, ret);
+			kfree(chan_name);
 			return ret;
 		}
 
diff --git a/drivers/firmware/iscsi_ibft.c b/drivers/firmware/iscsi_ibft.c
index 96758b71a8db..7127a04bca19 100644
--- a/drivers/firmware/iscsi_ibft.c
+++ b/drivers/firmware/iscsi_ibft.c
@@ -104,7 +104,7 @@ struct ibft_control {
 	u16 tgt0_off;
 	u16 nic1_off;
 	u16 tgt1_off;
-	u16 expansion[0];
+	u16 expansion[];
 } __attribute__((__packed__));
 
 struct ibft_initiator {
diff --git a/drivers/firmware/pcdp.h b/drivers/firmware/pcdp.h
index ce75d1da9e84..e02540571c52 100644
--- a/drivers/firmware/pcdp.h
+++ b/drivers/firmware/pcdp.h
@@ -103,6 +103,6 @@ struct pcdp {
 	u8			creator_id[4];
 	u32			creator_rev;
 	u32			num_uarts;
-	struct pcdp_uart	uart[0];	/* actual size is num_uarts */
+	struct pcdp_uart	uart[];	/* actual size is num_uarts */
 	/* remainder of table is pcdp_device structures */
 } __attribute__((packed));
diff --git a/drivers/firmware/qcom_scm-legacy.c b/drivers/firmware/qcom_scm-legacy.c
index 8532e7c78ef7..eba6b60bfb61 100644
--- a/drivers/firmware/qcom_scm-legacy.c
+++ b/drivers/firmware/qcom_scm-legacy.c
@@ -56,7 +56,7 @@ struct scm_legacy_command {
 	__le32 buf_offset;
 	__le32 resp_hdr_offset;
 	__le32 id;
-	__le32 buf[0];
+	__le32 buf[];
 };
 
 /**
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index 059bb0fbae9e..0e7233a20f34 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -6,7 +6,6 @@
 #include <linux/init.h>
 #include <linux/cpumask.h>
 #include <linux/export.h>
-#include <linux/dma-direct.h>
 #include <linux/dma-mapping.h>
 #include <linux/module.h>
 #include <linux/types.h>
@@ -806,8 +805,7 @@ int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
 	struct qcom_scm_mem_map_info *mem_to_map;
 	phys_addr_t mem_to_map_phys;
 	phys_addr_t dest_phys;
-	phys_addr_t ptr_phys;
-	dma_addr_t ptr_dma;
+	dma_addr_t ptr_phys;
 	size_t mem_to_map_sz;
 	size_t dest_sz;
 	size_t src_sz;
@@ -824,10 +822,9 @@ int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
 	ptr_sz = ALIGN(src_sz, SZ_64) + ALIGN(mem_to_map_sz, SZ_64) +
 			ALIGN(dest_sz, SZ_64);
 
-	ptr = dma_alloc_coherent(__scm->dev, ptr_sz, &ptr_dma, GFP_KERNEL);
+	ptr = dma_alloc_coherent(__scm->dev, ptr_sz, &ptr_phys, GFP_KERNEL);
 	if (!ptr)
 		return -ENOMEM;
-	ptr_phys = dma_to_phys(__scm->dev, ptr_dma);
 
 	/* Fill source vmid detail */
 	src = ptr;
@@ -855,7 +852,7 @@ int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
 
 	ret = __qcom_scm_assign_mem(__scm->dev, mem_to_map_phys, mem_to_map_sz,
 				    ptr_phys, src_sz, dest_phys, dest_sz);
-	dma_free_coherent(__scm->dev, ptr_sz, ptr, ptr_dma);
+	dma_free_coherent(__scm->dev, ptr_sz, ptr, ptr_phys);
 	if (ret) {
 		dev_err(__scm->dev,
 			"Assign memory protection call failed %d\n", ret);
@@ -943,7 +940,7 @@ bool qcom_scm_hdcp_available(void)
 
 	qcom_scm_clk_disable();
 
-	return ret > 0 ? true : false;
+	return ret > 0;
 }
 EXPORT_SYMBOL(qcom_scm_hdcp_available);
 
diff --git a/drivers/firmware/raspberrypi.c b/drivers/firmware/raspberrypi.c
index a3e85186f8e6..ef8098856a47 100644
--- a/drivers/firmware/raspberrypi.c
+++ b/drivers/firmware/raspberrypi.c
@@ -12,6 +12,8 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
 #include <soc/bcm2835/raspberrypi-firmware.h>
 
 #define MBOX_MSG(chan, data28)		(((data28) & ~0xf) | ((chan) & 0xf))
@@ -19,6 +21,8 @@
 #define MBOX_DATA28(msg)		((msg) & ~0xf)
 #define MBOX_CHAN_PROPERTY		8
 
+#define VL805_PCI_CONFIG_VERSION_OFFSET		0x50
+
 static struct platform_device *rpi_hwmon;
 static struct platform_device *rpi_clk;
 
@@ -280,6 +284,63 @@ struct rpi_firmware *rpi_firmware_get(struct device_node *firmware_node)
 }
 EXPORT_SYMBOL_GPL(rpi_firmware_get);
 
+/*
+ * The Raspberry Pi 4 gets its USB functionality from VL805, a PCIe chip that
+ * implements xHCI. After a PCI reset, VL805's firmware may either be loaded
+ * directly from an EEPROM or, if not present, by the SoC's co-processor,
+ * VideoCore. RPi4's VideoCore OS contains both the non public firmware load
+ * logic and the VL805 firmware blob. This function triggers the aforementioned
+ * process.
+ */
+int rpi_firmware_init_vl805(struct pci_dev *pdev)
+{
+	struct device_node *fw_np;
+	struct rpi_firmware *fw;
+	u32 dev_addr, version;
+	int ret;
+
+	fw_np = of_find_compatible_node(NULL, NULL,
+					"raspberrypi,bcm2835-firmware");
+	if (!fw_np)
+		return 0;
+
+	fw = rpi_firmware_get(fw_np);
+	of_node_put(fw_np);
+	if (!fw)
+		return -ENODEV;
+
+	/*
+	 * Make sure we don't trigger a firmware load unnecessarily.
+	 *
+	 * If something went wrong with PCI, this whole exercise would be
+	 * futile as VideoCore expects from us a configured PCI bus. Just take
+	 * the faulty version (likely ~0) and let xHCI's registration fail
+	 * further down the line.
+	 */
+	pci_read_config_dword(pdev, VL805_PCI_CONFIG_VERSION_OFFSET, &version);
+	if (version)
+		goto exit;
+
+	dev_addr = pdev->bus->number << 20 | PCI_SLOT(pdev->devfn) << 15 |
+		   PCI_FUNC(pdev->devfn) << 12;
+
+	ret = rpi_firmware_property(fw, RPI_FIRMWARE_NOTIFY_XHCI_RESET,
+				    &dev_addr, sizeof(dev_addr));
+	if (ret)
+		return ret;
+
+	/* Wait for vl805 to startup */
+	usleep_range(200, 1000);
+
+	pci_read_config_dword(pdev, VL805_PCI_CONFIG_VERSION_OFFSET,
+			      &version);
+exit:
+	pci_info(pdev, "VL805 firmware version %08x\n", version);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rpi_firmware_init_vl805);
+
 static const struct of_device_id rpi_firmware_of_match[] = {
 	{ .compatible = "raspberrypi,bcm2835-firmware", },
 	{},
diff --git a/drivers/firmware/stratix10-rsu.c b/drivers/firmware/stratix10-rsu.c
index f8533338b018..4379475c99ed 100644
--- a/drivers/firmware/stratix10-rsu.c
+++ b/drivers/firmware/stratix10-rsu.c
@@ -72,7 +72,7 @@ static void rsu_status_callback(struct stratix10_svc_client *client,
 	struct stratix10_rsu_priv *priv = client->priv;
 	struct arm_smccc_res *res = (struct arm_smccc_res *)data->kaddr1;
 
-	if (data->status == BIT(SVC_STATUS_RSU_OK)) {
+	if (data->status == BIT(SVC_STATUS_OK)) {
 		priv->status.version = FIELD_GET(RSU_VERSION_MASK,
 						 res->a2);
 		priv->status.state = FIELD_GET(RSU_STATE_MASK, res->a2);
@@ -108,9 +108,9 @@ static void rsu_command_callback(struct stratix10_svc_client *client,
 {
 	struct stratix10_rsu_priv *priv = client->priv;
 
-	if (data->status == BIT(SVC_STATUS_RSU_NO_SUPPORT))
+	if (data->status == BIT(SVC_STATUS_NO_SUPPORT))
 		dev_warn(client->dev, "Secure FW doesn't support notify\n");
-	else if (data->status == BIT(SVC_STATUS_RSU_ERROR))
+	else if (data->status == BIT(SVC_STATUS_ERROR))
 		dev_err(client->dev, "Failure, returned status is %lu\n",
 			BIT(data->status));
 
@@ -133,9 +133,9 @@ static void rsu_retry_callback(struct stratix10_svc_client *client,
 	struct stratix10_rsu_priv *priv = client->priv;
 	unsigned int *counter = (unsigned int *)data->kaddr1;
 
-	if (data->status == BIT(SVC_STATUS_RSU_OK))
+	if (data->status == BIT(SVC_STATUS_OK))
 		priv->retry_counter = *counter;
-	else if (data->status == BIT(SVC_STATUS_RSU_NO_SUPPORT))
+	else if (data->status == BIT(SVC_STATUS_NO_SUPPORT))
 		dev_warn(client->dev, "Secure FW doesn't support retry\n");
 	else
 		dev_err(client->dev, "Failed to get retry counter %lu\n",
diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c
index d5f0769f3761..e0db8dbfc9d1 100644
--- a/drivers/firmware/stratix10-svc.c
+++ b/drivers/firmware/stratix10-svc.c
@@ -214,7 +214,7 @@ static void svc_thread_cmd_data_claim(struct stratix10_svc_controller *ctrl,
 				complete(&ctrl->complete_status);
 				break;
 			}
-			cb_data->status = BIT(SVC_STATUS_RECONFIG_BUFFER_DONE);
+			cb_data->status = BIT(SVC_STATUS_BUFFER_DONE);
 			cb_data->kaddr1 = svc_pa_to_va(res.a1);
 			cb_data->kaddr2 = (res.a2) ?
 					  svc_pa_to_va(res.a2) : NULL;
@@ -227,7 +227,7 @@ static void svc_thread_cmd_data_claim(struct stratix10_svc_controller *ctrl,
 				 __func__);
 		}
 	} while (res.a0 == INTEL_SIP_SMC_STATUS_OK ||
-		 res.a0 == INTEL_SIP_SMC_FPGA_CONFIG_STATUS_BUSY ||
+		 res.a0 == INTEL_SIP_SMC_STATUS_BUSY ||
 		 wait_for_completion_timeout(&ctrl->complete_status, timeout));
 }
 
@@ -250,7 +250,7 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl,
 	cb_data->kaddr1 = NULL;
 	cb_data->kaddr2 = NULL;
 	cb_data->kaddr3 = NULL;
-	cb_data->status = BIT(SVC_STATUS_RECONFIG_ERROR);
+	cb_data->status = BIT(SVC_STATUS_ERROR);
 
 	pr_debug("%s: polling config status\n", __func__);
 
@@ -259,7 +259,7 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl,
 		ctrl->invoke_fn(INTEL_SIP_SMC_FPGA_CONFIG_ISDONE,
 				0, 0, 0, 0, 0, 0, 0, &res);
 		if ((res.a0 == INTEL_SIP_SMC_STATUS_OK) ||
-		    (res.a0 == INTEL_SIP_SMC_FPGA_CONFIG_STATUS_ERROR))
+		    (res.a0 == INTEL_SIP_SMC_STATUS_ERROR))
 			break;
 
 		/*
@@ -271,7 +271,7 @@ static void svc_thread_cmd_config_status(struct stratix10_svc_controller *ctrl,
 	}
 
 	if (res.a0 == INTEL_SIP_SMC_STATUS_OK && count_in_sec)
-		cb_data->status = BIT(SVC_STATUS_RECONFIG_COMPLETED);
+		cb_data->status = BIT(SVC_STATUS_COMPLETED);
 
 	p_data->chan->scl->receive_cb(p_data->chan->scl, cb_data);
 }
@@ -294,24 +294,18 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data,
 
 	switch (p_data->command) {
 	case COMMAND_RECONFIG:
-		cb_data->status = BIT(SVC_STATUS_RECONFIG_REQUEST_OK);
+	case COMMAND_RSU_UPDATE:
+	case COMMAND_RSU_NOTIFY:
+		cb_data->status = BIT(SVC_STATUS_OK);
 		break;
 	case COMMAND_RECONFIG_DATA_SUBMIT:
-		cb_data->status = BIT(SVC_STATUS_RECONFIG_BUFFER_SUBMITTED);
-		break;
-	case COMMAND_NOOP:
-		cb_data->status = BIT(SVC_STATUS_RECONFIG_BUFFER_SUBMITTED);
-		cb_data->kaddr1 = svc_pa_to_va(res.a1);
+		cb_data->status = BIT(SVC_STATUS_BUFFER_SUBMITTED);
 		break;
 	case COMMAND_RECONFIG_STATUS:
-		cb_data->status = BIT(SVC_STATUS_RECONFIG_COMPLETED);
-		break;
-	case COMMAND_RSU_UPDATE:
-	case COMMAND_RSU_NOTIFY:
-		cb_data->status = BIT(SVC_STATUS_RSU_OK);
+		cb_data->status = BIT(SVC_STATUS_COMPLETED);
 		break;
 	case COMMAND_RSU_RETRY:
-		cb_data->status = BIT(SVC_STATUS_RSU_OK);
+		cb_data->status = BIT(SVC_STATUS_OK);
 		cb_data->kaddr1 = &res.a1;
 		break;
 	default:
@@ -430,9 +424,9 @@ static int svc_normal_to_secure_thread(void *data)
 
 		if (pdata->command == COMMAND_RSU_STATUS) {
 			if (res.a0 == INTEL_SIP_SMC_RSU_ERROR)
-				cbdata->status = BIT(SVC_STATUS_RSU_ERROR);
+				cbdata->status = BIT(SVC_STATUS_ERROR);
 			else
-				cbdata->status = BIT(SVC_STATUS_RSU_OK);
+				cbdata->status = BIT(SVC_STATUS_OK);
 
 			cbdata->kaddr1 = &res;
 			cbdata->kaddr2 = NULL;
@@ -445,7 +439,7 @@ static int svc_normal_to_secure_thread(void *data)
 		case INTEL_SIP_SMC_STATUS_OK:
 			svc_thread_recv_status_ok(pdata, cbdata, res);
 			break;
-		case INTEL_SIP_SMC_FPGA_CONFIG_STATUS_BUSY:
+		case INTEL_SIP_SMC_STATUS_BUSY:
 			switch (pdata->command) {
 			case COMMAND_RECONFIG_DATA_SUBMIT:
 				svc_thread_cmd_data_claim(ctrl,
@@ -460,33 +454,13 @@ static int svc_normal_to_secure_thread(void *data)
 				break;
 			}
 			break;
-		case INTEL_SIP_SMC_FPGA_CONFIG_STATUS_REJECTED:
+		case INTEL_SIP_SMC_STATUS_REJECTED:
 			pr_debug("%s: STATUS_REJECTED\n", __func__);
 			break;
-		case INTEL_SIP_SMC_FPGA_CONFIG_STATUS_ERROR:
+		case INTEL_SIP_SMC_STATUS_ERROR:
 		case INTEL_SIP_SMC_RSU_ERROR:
 			pr_err("%s: STATUS_ERROR\n", __func__);
-			switch (pdata->command) {
-			/* for FPGA mgr */
-			case COMMAND_RECONFIG_DATA_CLAIM:
-			case COMMAND_RECONFIG:
-			case COMMAND_RECONFIG_DATA_SUBMIT:
-			case COMMAND_RECONFIG_STATUS:
-				cbdata->status =
-					BIT(SVC_STATUS_RECONFIG_ERROR);
-				break;
-
-			/* for RSU */
-			case COMMAND_RSU_STATUS:
-			case COMMAND_RSU_UPDATE:
-			case COMMAND_RSU_NOTIFY:
-			case COMMAND_RSU_RETRY:
-				cbdata->status =
-					BIT(SVC_STATUS_RSU_ERROR);
-				break;
-			}
-
-			cbdata->status = BIT(SVC_STATUS_RECONFIG_ERROR);
+			cbdata->status = BIT(SVC_STATUS_ERROR);
 			cbdata->kaddr1 = NULL;
 			cbdata->kaddr2 = NULL;
 			cbdata->kaddr3 = NULL;
@@ -502,7 +476,7 @@ static int svc_normal_to_secure_thread(void *data)
 			if ((pdata->command == COMMAND_RSU_RETRY) ||
 				(pdata->command == COMMAND_RSU_NOTIFY)) {
 				cbdata->status =
-					BIT(SVC_STATUS_RSU_NO_SUPPORT);
+					BIT(SVC_STATUS_NO_SUPPORT);
 				cbdata->kaddr1 = NULL;
 				cbdata->kaddr2 = NULL;
 				cbdata->kaddr3 = NULL;
diff --git a/drivers/firmware/tegra/bpmp-tegra186.c b/drivers/firmware/tegra/bpmp-tegra186.c
index ea308751635f..63ab21d89c2c 100644
--- a/drivers/firmware/tegra/bpmp-tegra186.c
+++ b/drivers/firmware/tegra/bpmp-tegra186.c
@@ -176,7 +176,7 @@ static int tegra186_bpmp_init(struct tegra_bpmp *bpmp)
 	priv->tx.pool = of_gen_pool_get(bpmp->dev->of_node, "shmem", 0);
 	if (!priv->tx.pool) {
 		dev_err(bpmp->dev, "TX shmem pool not found\n");
-		return -ENOMEM;
+		return -EPROBE_DEFER;
 	}
 
 	priv->tx.virt = gen_pool_dma_alloc(priv->tx.pool, 4096, &priv->tx.phys);
@@ -188,7 +188,7 @@ static int tegra186_bpmp_init(struct tegra_bpmp *bpmp)
 	priv->rx.pool = of_gen_pool_get(bpmp->dev->of_node, "shmem", 1);
 	if (!priv->rx.pool) {
 		dev_err(bpmp->dev, "RX shmem pool not found\n");
-		err = -ENOMEM;
+		err = -EPROBE_DEFER;
 		goto free_tx;
 	}
 
diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c
index 6741fcda0c37..fe6702df24bf 100644
--- a/drivers/firmware/tegra/bpmp.c
+++ b/drivers/firmware/tegra/bpmp.c
@@ -6,6 +6,7 @@
 #include <linux/clk/tegra.h>
 #include <linux/genalloc.h>
 #include <linux/mailbox_client.h>
+#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
@@ -869,12 +870,8 @@ static struct platform_driver tegra_bpmp_driver = {
 		.name = "tegra-bpmp",
 		.of_match_table = tegra_bpmp_match,
 		.pm = &tegra_bpmp_pm_ops,
+		.suppress_bind_attrs = true,
 	},
 	.probe = tegra_bpmp_probe,
 };
-
-static int __init tegra_bpmp_init(void)
-{
-	return platform_driver_register(&tegra_bpmp_driver);
-}
-core_initcall(tegra_bpmp_init);
+builtin_platform_driver(tegra_bpmp_driver);
diff --git a/drivers/firmware/trusted_foundations.c b/drivers/firmware/trusted_foundations.c
index fc544e19b0a1..1389fa9418a7 100644
--- a/drivers/firmware/trusted_foundations.c
+++ b/drivers/firmware/trusted_foundations.c
@@ -19,6 +19,7 @@
 
 #define TF_CACHE_ENABLE		1
 #define TF_CACHE_DISABLE	2
+#define TF_CACHE_REENABLE	4
 
 #define TF_SET_CPU_BOOT_ADDR_SMC 0xfffff200
 
@@ -29,6 +30,7 @@
 #define TF_CPU_PM_S1		0xffffffe4
 #define TF_CPU_PM_S1_NOFLUSH_L2	0xffffffe7
 
+static unsigned long tf_idle_mode = TF_PM_MODE_NONE;
 static unsigned long cpu_boot_addr;
 
 static void tf_generic_smc(u32 type, u32 arg1, u32 arg2)
@@ -85,25 +87,40 @@ static int tf_prepare_idle(unsigned long mode)
 			       cpu_boot_addr);
 		break;
 
+	case TF_PM_MODE_NONE:
+		break;
+
 	default:
 		return -EINVAL;
 	}
 
+	tf_idle_mode = mode;
+
 	return 0;
 }
 
 #ifdef CONFIG_CACHE_L2X0
 static void tf_cache_write_sec(unsigned long val, unsigned int reg)
 {
-	u32 l2x0_way_mask = 0xff;
+	u32 enable_op, l2x0_way_mask = 0xff;
 
 	switch (reg) {
 	case L2X0_CTRL:
 		if (l2x0_saved_regs.aux_ctrl & L310_AUX_CTRL_ASSOCIATIVITY_16)
 			l2x0_way_mask = 0xffff;
 
+		switch (tf_idle_mode) {
+		case TF_PM_MODE_LP2:
+			enable_op = TF_CACHE_REENABLE;
+			break;
+
+		default:
+			enable_op = TF_CACHE_ENABLE;
+			break;
+		}
+
 		if (val == L2X0_CTRL_EN)
-			tf_generic_smc(TF_CACHE_MAINT, TF_CACHE_ENABLE,
+			tf_generic_smc(TF_CACHE_MAINT, enable_op,
 				       l2x0_saved_regs.aux_ctrl);
 		else
 			tf_generic_smc(TF_CACHE_MAINT, TF_CACHE_DISABLE,
diff --git a/drivers/firmware/xilinx/zynqmp-debug.c b/drivers/firmware/xilinx/zynqmp-debug.c
index 43bc6cfdab45..99606b34975e 100644
--- a/drivers/firmware/xilinx/zynqmp-debug.c
+++ b/drivers/firmware/xilinx/zynqmp-debug.c
@@ -85,14 +85,13 @@ static int get_pm_api_id(char *pm_api_req, u32 *pm_id)
 
 static int process_api_request(u32 pm_id, u64 *pm_api_arg, u32 *pm_api_ret)
 {
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 	u32 pm_api_version;
 	int ret;
 	struct zynqmp_pm_query_data qdata = {0};
 
 	switch (pm_id) {
 	case PM_GET_API_VERSION:
-		ret = eemi_ops->get_api_version(&pm_api_version);
+		ret = zynqmp_pm_get_api_version(&pm_api_version);
 		sprintf(debugfs_buf, "PM-API Version = %d.%d\n",
 			pm_api_version >> 16, pm_api_version & 0xffff);
 		break;
@@ -102,7 +101,7 @@ static int process_api_request(u32 pm_id, u64 *pm_api_arg, u32 *pm_api_ret)
 		qdata.arg2 = pm_api_arg[2];
 		qdata.arg3 = pm_api_arg[3];
 
-		ret = eemi_ops->query_data(qdata, pm_api_ret);
+		ret = zynqmp_pm_query_data(qdata, pm_api_ret);
 		if (ret)
 			break;
 
diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index 41b65164a367..8d1ff2454e2e 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -2,7 +2,7 @@
 /*
  * Xilinx Zynq MPSoC Firmware layer
  *
- *  Copyright (C) 2014-2018 Xilinx, Inc.
+ *  Copyright (C) 2014-2020 Xilinx, Inc.
  *
  *  Michal Simek <michal.simek@xilinx.com>
  *  Davorin Mista <davorin.mista@aggios.com>
@@ -24,8 +24,6 @@
 #include <linux/firmware/xlnx-zynqmp.h>
 #include "zynqmp-debug.h"
 
-static const struct zynqmp_eemi_ops *eemi_ops_tbl;
-
 static bool feature_check_enabled;
 static u32 zynqmp_pm_features[PM_API_MAX];
 
@@ -219,7 +217,7 @@ static u32 pm_tz_version;
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_get_api_version(u32 *version)
+int zynqmp_pm_get_api_version(u32 *version)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -237,6 +235,7 @@ static int zynqmp_pm_get_api_version(u32 *version)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_api_version);
 
 /**
  * zynqmp_pm_get_chipid - Get silicon ID registers
@@ -246,7 +245,7 @@ static int zynqmp_pm_get_api_version(u32 *version)
  * Return:      Returns the status of the operation and the idcode and version
  *              registers in @idcode and @version.
  */
-static int zynqmp_pm_get_chipid(u32 *idcode, u32 *version)
+int zynqmp_pm_get_chipid(u32 *idcode, u32 *version)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -260,6 +259,7 @@ static int zynqmp_pm_get_chipid(u32 *idcode, u32 *version)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_chipid);
 
 /**
  * zynqmp_pm_get_trustzone_version() - Get secure trustzone firmware version
@@ -324,7 +324,7 @@ static int get_set_conduit_method(struct device_node *np)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
+int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
 {
 	int ret;
 
@@ -338,6 +338,7 @@ static int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
 	 */
 	return qdata.qid == PM_QID_CLOCK_GET_NAME ? 0 : ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_query_data);
 
 /**
  * zynqmp_pm_clock_enable() - Enable the clock for given id
@@ -348,10 +349,11 @@ static int zynqmp_pm_query_data(struct zynqmp_pm_query_data qdata, u32 *out)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_enable(u32 clock_id)
+int zynqmp_pm_clock_enable(u32 clock_id)
 {
 	return zynqmp_pm_invoke_fn(PM_CLOCK_ENABLE, clock_id, 0, 0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_enable);
 
 /**
  * zynqmp_pm_clock_disable() - Disable the clock for given id
@@ -362,10 +364,11 @@ static int zynqmp_pm_clock_enable(u32 clock_id)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_disable(u32 clock_id)
+int zynqmp_pm_clock_disable(u32 clock_id)
 {
 	return zynqmp_pm_invoke_fn(PM_CLOCK_DISABLE, clock_id, 0, 0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_disable);
 
 /**
  * zynqmp_pm_clock_getstate() - Get the clock state for given id
@@ -377,7 +380,7 @@ static int zynqmp_pm_clock_disable(u32 clock_id)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
+int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -388,6 +391,7 @@ static int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getstate);
 
 /**
  * zynqmp_pm_clock_setdivider() - Set the clock divider for given id
@@ -399,11 +403,12 @@ static int zynqmp_pm_clock_getstate(u32 clock_id, u32 *state)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_setdivider(u32 clock_id, u32 divider)
+int zynqmp_pm_clock_setdivider(u32 clock_id, u32 divider)
 {
 	return zynqmp_pm_invoke_fn(PM_CLOCK_SETDIVIDER, clock_id, divider,
 				   0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_setdivider);
 
 /**
  * zynqmp_pm_clock_getdivider() - Get the clock divider for given id
@@ -415,7 +420,7 @@ static int zynqmp_pm_clock_setdivider(u32 clock_id, u32 divider)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
+int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -426,6 +431,7 @@ static int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getdivider);
 
 /**
  * zynqmp_pm_clock_setrate() - Set the clock rate for given id
@@ -436,13 +442,14 @@ static int zynqmp_pm_clock_getdivider(u32 clock_id, u32 *divider)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate)
+int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate)
 {
 	return zynqmp_pm_invoke_fn(PM_CLOCK_SETRATE, clock_id,
 				   lower_32_bits(rate),
 				   upper_32_bits(rate),
 				   0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_setrate);
 
 /**
  * zynqmp_pm_clock_getrate() - Get the clock rate for given id
@@ -454,7 +461,7 @@ static int zynqmp_pm_clock_setrate(u32 clock_id, u64 rate)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
+int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -465,6 +472,7 @@ static int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getrate);
 
 /**
  * zynqmp_pm_clock_setparent() - Set the clock parent for given id
@@ -475,11 +483,12 @@ static int zynqmp_pm_clock_getrate(u32 clock_id, u64 *rate)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id)
+int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id)
 {
 	return zynqmp_pm_invoke_fn(PM_CLOCK_SETPARENT, clock_id,
 				   parent_id, 0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_setparent);
 
 /**
  * zynqmp_pm_clock_getparent() - Get the clock parent for given id
@@ -491,7 +500,7 @@ static int zynqmp_pm_clock_setparent(u32 clock_id, u32 parent_id)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_clock_getparent(u32 clock_id, u32 *parent_id)
+int zynqmp_pm_clock_getparent(u32 clock_id, u32 *parent_id)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -502,48 +511,191 @@ static int zynqmp_pm_clock_getparent(u32 clock_id, u32 *parent_id)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_clock_getparent);
 
 /**
- * zynqmp_is_valid_ioctl() - Check whether IOCTL ID is valid or not
- * @ioctl_id:	IOCTL ID
+ * zynqmp_pm_set_pll_frac_mode() - PM API for set PLL mode
+ *
+ * @clk_id:	PLL clock ID
+ * @mode:	PLL mode (PLL_MODE_FRAC/PLL_MODE_INT)
+ *
+ * This function sets PLL mode
  *
- * Return: 1 if IOCTL is valid else 0
+ * Return: Returns status, either success or error+reason
  */
-static inline int zynqmp_is_valid_ioctl(u32 ioctl_id)
+int zynqmp_pm_set_pll_frac_mode(u32 clk_id, u32 mode)
 {
-	switch (ioctl_id) {
-	case IOCTL_SD_DLL_RESET:
-	case IOCTL_SET_SD_TAPDELAY:
-	case IOCTL_SET_PLL_FRAC_MODE:
-	case IOCTL_GET_PLL_FRAC_MODE:
-	case IOCTL_SET_PLL_FRAC_DATA:
-	case IOCTL_GET_PLL_FRAC_DATA:
-		return 1;
-	default:
-		return 0;
-	}
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_SET_PLL_FRAC_MODE,
+				   clk_id, mode, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_pll_frac_mode);
 
 /**
- * zynqmp_pm_ioctl() - PM IOCTL API for device control and configs
- * @node_id:	Node ID of the device
- * @ioctl_id:	ID of the requested IOCTL
- * @arg1:	Argument 1 to requested IOCTL call
- * @arg2:	Argument 2 to requested IOCTL call
- * @out:	Returned output value
+ * zynqmp_pm_get_pll_frac_mode() - PM API for get PLL mode
+ *
+ * @clk_id:	PLL clock ID
+ * @mode:	PLL mode
  *
- * This function calls IOCTL to firmware for device control and configuration.
+ * This function return current PLL mode
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_ioctl(u32 node_id, u32 ioctl_id, u32 arg1, u32 arg2,
-			   u32 *out)
+int zynqmp_pm_get_pll_frac_mode(u32 clk_id, u32 *mode)
 {
-	if (!zynqmp_is_valid_ioctl(ioctl_id))
-		return -EINVAL;
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_GET_PLL_FRAC_MODE,
+				   clk_id, 0, mode);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_pll_frac_mode);
+
+/**
+ * zynqmp_pm_set_pll_frac_data() - PM API for setting pll fraction data
+ *
+ * @clk_id:	PLL clock ID
+ * @data:	fraction data
+ *
+ * This function sets fraction data.
+ * It is valid for fraction mode only.
+ *
+ * Return: Returns status, either success or error+reason
+ */
+int zynqmp_pm_set_pll_frac_data(u32 clk_id, u32 data)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_SET_PLL_FRAC_DATA,
+				   clk_id, data, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_pll_frac_data);
+
+/**
+ * zynqmp_pm_get_pll_frac_data() - PM API for getting pll fraction data
+ *
+ * @clk_id:	PLL clock ID
+ * @data:	fraction data
+ *
+ * This function returns fraction data value.
+ *
+ * Return: Returns status, either success or error+reason
+ */
+int zynqmp_pm_get_pll_frac_data(u32 clk_id, u32 *data)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_GET_PLL_FRAC_DATA,
+				   clk_id, 0, data);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_pll_frac_data);
+
+/**
+ * zynqmp_pm_set_sd_tapdelay() -  Set tap delay for the SD device
+ *
+ * @node_id	Node ID of the device
+ * @type	Type of tap delay to set (input/output)
+ * @value	Value to set fot the tap delay
+ *
+ * This function sets input/output tap delay for the SD device.
+ *
+ * @return	Returns status, either success or error+reason
+ */
+int zynqmp_pm_set_sd_tapdelay(u32 node_id, u32 type, u32 value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, node_id, IOCTL_SET_SD_TAPDELAY,
+				   type, value, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_sd_tapdelay);
+
+/**
+ * zynqmp_pm_sd_dll_reset() - Reset DLL logic
+ *
+ * @node_id	Node ID of the device
+ * @type	Reset type
+ *
+ * This function resets DLL logic for the SD device.
+ *
+ * @return	Returns status, either success or error+reason
+ */
+int zynqmp_pm_sd_dll_reset(u32 node_id, u32 type)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, node_id, IOCTL_SET_SD_TAPDELAY,
+				   type, 0, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_sd_dll_reset);
+
+/**
+ * zynqmp_pm_write_ggs() - PM API for writing global general storage (ggs)
+ * @index	GGS register index
+ * @value	Register value to be written
+ *
+ * This function writes value to GGS register.
+ *
+ * @return      Returns status, either success or error+reason
+ */
+int zynqmp_pm_write_ggs(u32 index, u32 value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_WRITE_GGS,
+				   index, value, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_write_ggs);
+
+/**
+ * zynqmp_pm_write_ggs() - PM API for reading global general storage (ggs)
+ * @index	GGS register index
+ * @value	Register value to be written
+ *
+ * This function returns GGS register value.
+ *
+ * @return      Returns status, either success or error+reason
+ */
+int zynqmp_pm_read_ggs(u32 index, u32 *value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_READ_GGS,
+				   index, 0, value);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_read_ggs);
+
+/**
+ * zynqmp_pm_write_pggs() - PM API for writing persistent global general
+ *			     storage (pggs)
+ * @index	PGGS register index
+ * @value	Register value to be written
+ *
+ * This function writes value to PGGS register.
+ *
+ * @return      Returns status, either success or error+reason
+ */
+int zynqmp_pm_write_pggs(u32 index, u32 value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_WRITE_PGGS, index, value,
+				   NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_write_pggs);
+
+/**
+ * zynqmp_pm_write_pggs() - PM API for reading persistent global general
+ *			     storage (pggs)
+ * @index	PGGS register index
+ * @value	Register value to be written
+ *
+ * This function returns PGGS register value.
+ *
+ * @return      Returns status, either success or error+reason
+ */
+int zynqmp_pm_read_pggs(u32 index, u32 *value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_READ_PGGS, index, 0,
+				   value);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_read_pggs);
 
-	return zynqmp_pm_invoke_fn(PM_IOCTL, node_id, ioctl_id,
-				   arg1, arg2, out);
+/**
+ * zynqmp_pm_set_boot_health_status() - PM API for setting healthy boot status
+ * @value	Status value to be written
+ *
+ * This function sets healthy bit value to indicate boot health status
+ * to firmware.
+ *
+ * @return      Returns status, either success or error+reason
+ */
+int zynqmp_pm_set_boot_health_status(u32 value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, 0, IOCTL_SET_BOOT_HEALTH_STATUS,
+				   value, 0, NULL);
 }
 
 /**
@@ -554,12 +706,13 @@ static int zynqmp_pm_ioctl(u32 node_id, u32 ioctl_id, u32 arg1, u32 arg2,
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
-				  const enum zynqmp_pm_reset_action assert_flag)
+int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
+			   const enum zynqmp_pm_reset_action assert_flag)
 {
 	return zynqmp_pm_invoke_fn(PM_RESET_ASSERT, reset, assert_flag,
 				   0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_reset_assert);
 
 /**
  * zynqmp_pm_reset_get_status - Get status of the reset
@@ -568,8 +721,7 @@ static int zynqmp_pm_reset_assert(const enum zynqmp_pm_reset reset,
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset,
-				      u32 *status)
+int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset, u32 *status)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -583,6 +735,7 @@ static int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset,
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_reset_get_status);
 
 /**
  * zynqmp_pm_fpga_load - Perform the fpga load
@@ -597,12 +750,12 @@ static int zynqmp_pm_reset_get_status(const enum zynqmp_pm_reset reset,
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_fpga_load(const u64 address, const u32 size,
-			       const u32 flags)
+int zynqmp_pm_fpga_load(const u64 address, const u32 size, const u32 flags)
 {
 	return zynqmp_pm_invoke_fn(PM_FPGA_LOAD, lower_32_bits(address),
 				   upper_32_bits(address), size, flags, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_load);
 
 /**
  * zynqmp_pm_fpga_get_status - Read value from PCAP status register
@@ -613,7 +766,7 @@ static int zynqmp_pm_fpga_load(const u64 address, const u32 size,
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_fpga_get_status(u32 *value)
+int zynqmp_pm_fpga_get_status(u32 *value)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -626,6 +779,7 @@ static int zynqmp_pm_fpga_get_status(u32 *value)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_fpga_get_status);
 
 /**
  * zynqmp_pm_init_finalize() - PM call to inform firmware that the caller
@@ -636,10 +790,11 @@ static int zynqmp_pm_fpga_get_status(u32 *value)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_init_finalize(void)
+int zynqmp_pm_init_finalize(void)
 {
 	return zynqmp_pm_invoke_fn(PM_PM_INIT_FINALIZE, 0, 0, 0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_init_finalize);
 
 /**
  * zynqmp_pm_set_suspend_mode()	- Set system suspend mode
@@ -649,10 +804,11 @@ static int zynqmp_pm_init_finalize(void)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_set_suspend_mode(u32 mode)
+int zynqmp_pm_set_suspend_mode(u32 mode)
 {
 	return zynqmp_pm_invoke_fn(PM_SET_SUSPEND_MODE, mode, 0, 0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_suspend_mode);
 
 /**
  * zynqmp_pm_request_node() - Request a node with specific capabilities
@@ -666,13 +822,13 @@ static int zynqmp_pm_set_suspend_mode(u32 mode)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_request_node(const u32 node, const u32 capabilities,
-				  const u32 qos,
-				  const enum zynqmp_pm_request_ack ack)
+int zynqmp_pm_request_node(const u32 node, const u32 capabilities,
+			   const u32 qos, const enum zynqmp_pm_request_ack ack)
 {
 	return zynqmp_pm_invoke_fn(PM_REQUEST_NODE, node, capabilities,
 				   qos, ack, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_request_node);
 
 /**
  * zynqmp_pm_release_node() - Release a node
@@ -684,10 +840,11 @@ static int zynqmp_pm_request_node(const u32 node, const u32 capabilities,
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_release_node(const u32 node)
+int zynqmp_pm_release_node(const u32 node)
 {
 	return zynqmp_pm_invoke_fn(PM_RELEASE_NODE, node, 0, 0, 0, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_release_node);
 
 /**
  * zynqmp_pm_set_requirement() - PM call to set requirement for PM slaves
@@ -701,13 +858,14 @@ static int zynqmp_pm_release_node(const u32 node)
  *
  * Return: Returns status, either success or error+reason
  */
-static int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities,
-				     const u32 qos,
-				     const enum zynqmp_pm_request_ack ack)
+int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities,
+			      const u32 qos,
+			      const enum zynqmp_pm_request_ack ack)
 {
 	return zynqmp_pm_invoke_fn(PM_SET_REQUIREMENT, node, capabilities,
 				   qos, ack, NULL);
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_requirement);
 
 /**
  * zynqmp_pm_aes - Access AES hardware to encrypt/decrypt the data using
@@ -717,7 +875,7 @@ static int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities,
  *
  * Return:	Returns status, either success or error code.
  */
-static int zynqmp_pm_aes_engine(const u64 address, u32 *out)
+int zynqmp_pm_aes_engine(const u64 address, u32 *out)
 {
 	u32 ret_payload[PAYLOAD_ARG_CNT];
 	int ret;
@@ -732,47 +890,304 @@ static int zynqmp_pm_aes_engine(const u64 address, u32 *out)
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(zynqmp_pm_aes_engine);
 
-static const struct zynqmp_eemi_ops eemi_ops = {
-	.get_api_version = zynqmp_pm_get_api_version,
-	.get_chipid = zynqmp_pm_get_chipid,
-	.query_data = zynqmp_pm_query_data,
-	.clock_enable = zynqmp_pm_clock_enable,
-	.clock_disable = zynqmp_pm_clock_disable,
-	.clock_getstate = zynqmp_pm_clock_getstate,
-	.clock_setdivider = zynqmp_pm_clock_setdivider,
-	.clock_getdivider = zynqmp_pm_clock_getdivider,
-	.clock_setrate = zynqmp_pm_clock_setrate,
-	.clock_getrate = zynqmp_pm_clock_getrate,
-	.clock_setparent = zynqmp_pm_clock_setparent,
-	.clock_getparent = zynqmp_pm_clock_getparent,
-	.ioctl = zynqmp_pm_ioctl,
-	.reset_assert = zynqmp_pm_reset_assert,
-	.reset_get_status = zynqmp_pm_reset_get_status,
-	.init_finalize = zynqmp_pm_init_finalize,
-	.set_suspend_mode = zynqmp_pm_set_suspend_mode,
-	.request_node = zynqmp_pm_request_node,
-	.release_node = zynqmp_pm_release_node,
-	.set_requirement = zynqmp_pm_set_requirement,
-	.fpga_load = zynqmp_pm_fpga_load,
-	.fpga_get_status = zynqmp_pm_fpga_get_status,
-	.aes = zynqmp_pm_aes_engine,
+/**
+ * zynqmp_pm_system_shutdown - PM call to request a system shutdown or restart
+ * @type:	Shutdown or restart? 0 for shutdown, 1 for restart
+ * @subtype:	Specifies which system should be restarted or shut down
+ *
+ * Return:	Returns status, either success or error+reason
+ */
+int zynqmp_pm_system_shutdown(const u32 type, const u32 subtype)
+{
+	return zynqmp_pm_invoke_fn(PM_SYSTEM_SHUTDOWN, type, subtype,
+				   0, 0, NULL);
+}
+
+/**
+ * struct zynqmp_pm_shutdown_scope - Struct for shutdown scope
+ * @subtype:	Shutdown subtype
+ * @name:	Matching string for scope argument
+ *
+ * This struct encapsulates mapping between shutdown scope ID and string.
+ */
+struct zynqmp_pm_shutdown_scope {
+	const enum zynqmp_pm_shutdown_subtype subtype;
+	const char *name;
+};
+
+static struct zynqmp_pm_shutdown_scope shutdown_scopes[] = {
+	[ZYNQMP_PM_SHUTDOWN_SUBTYPE_SUBSYSTEM] = {
+		.subtype = ZYNQMP_PM_SHUTDOWN_SUBTYPE_SUBSYSTEM,
+		.name = "subsystem",
+	},
+	[ZYNQMP_PM_SHUTDOWN_SUBTYPE_PS_ONLY] = {
+		.subtype = ZYNQMP_PM_SHUTDOWN_SUBTYPE_PS_ONLY,
+		.name = "ps_only",
+	},
+	[ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM] = {
+		.subtype = ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM,
+		.name = "system",
+	},
 };
 
+static struct zynqmp_pm_shutdown_scope *selected_scope =
+		&shutdown_scopes[ZYNQMP_PM_SHUTDOWN_SUBTYPE_SYSTEM];
+
 /**
- * zynqmp_pm_get_eemi_ops - Get eemi ops functions
+ * zynqmp_pm_is_shutdown_scope_valid - Check if shutdown scope string is valid
+ * @scope_string:	Shutdown scope string
  *
- * Return: Pointer of eemi_ops structure
+ * Return:		Return pointer to matching shutdown scope struct from
+ *			array of available options in system if string is valid,
+ *			otherwise returns NULL.
  */
-const struct zynqmp_eemi_ops *zynqmp_pm_get_eemi_ops(void)
+static struct zynqmp_pm_shutdown_scope*
+		zynqmp_pm_is_shutdown_scope_valid(const char *scope_string)
+{
+	int count;
+
+	for (count = 0; count < ARRAY_SIZE(shutdown_scopes); count++)
+		if (sysfs_streq(scope_string, shutdown_scopes[count].name))
+			return &shutdown_scopes[count];
+
+	return NULL;
+}
+
+static ssize_t shutdown_scope_show(struct device *device,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(shutdown_scopes); i++) {
+		if (&shutdown_scopes[i] == selected_scope) {
+			strcat(buf, "[");
+			strcat(buf, shutdown_scopes[i].name);
+			strcat(buf, "]");
+		} else {
+			strcat(buf, shutdown_scopes[i].name);
+		}
+		strcat(buf, " ");
+	}
+	strcat(buf, "\n");
+
+	return strlen(buf);
+}
+
+static ssize_t shutdown_scope_store(struct device *device,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int ret;
+	struct zynqmp_pm_shutdown_scope *scope;
+
+	scope = zynqmp_pm_is_shutdown_scope_valid(buf);
+	if (!scope)
+		return -EINVAL;
+
+	ret = zynqmp_pm_system_shutdown(ZYNQMP_PM_SHUTDOWN_TYPE_SETSCOPE_ONLY,
+					scope->subtype);
+	if (ret) {
+		pr_err("unable to set shutdown scope %s\n", buf);
+		return ret;
+	}
+
+	selected_scope = scope;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(shutdown_scope);
+
+static ssize_t health_status_store(struct device *device,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	int ret;
+	unsigned int value;
+
+	ret = kstrtouint(buf, 10, &value);
+	if (ret)
+		return ret;
+
+	ret = zynqmp_pm_set_boot_health_status(value);
+	if (ret) {
+		dev_err(device, "unable to set healthy bit value to %u\n",
+			value);
+		return ret;
+	}
+
+	return count;
+}
+
+static DEVICE_ATTR_WO(health_status);
+
+static ssize_t ggs_show(struct device *device,
+			struct device_attribute *attr,
+			char *buf,
+			u32 reg)
+{
+	int ret;
+	u32 ret_payload[PAYLOAD_ARG_CNT];
+
+	ret = zynqmp_pm_read_ggs(reg, ret_payload);
+	if (ret)
+		return ret;
+
+	return sprintf(buf, "0x%x\n", ret_payload[1]);
+}
+
+static ssize_t ggs_store(struct device *device,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count,
+			 u32 reg)
+{
+	long value;
+	int ret;
+
+	if (reg >= GSS_NUM_REGS)
+		return -EINVAL;
+
+	ret = kstrtol(buf, 16, &value);
+	if (ret) {
+		count = -EFAULT;
+		goto err;
+	}
+
+	ret = zynqmp_pm_write_ggs(reg, value);
+	if (ret)
+		count = -EFAULT;
+err:
+	return count;
+}
+
+/* GGS register show functions */
+#define GGS0_SHOW(N)						\
+	ssize_t ggs##N##_show(struct device *device,		\
+			      struct device_attribute *attr,	\
+			      char *buf)			\
+	{							\
+		return ggs_show(device, attr, buf, N);		\
+	}
+
+static GGS0_SHOW(0);
+static GGS0_SHOW(1);
+static GGS0_SHOW(2);
+static GGS0_SHOW(3);
+
+/* GGS register store function */
+#define GGS0_STORE(N)						\
+	ssize_t ggs##N##_store(struct device *device,		\
+			       struct device_attribute *attr,	\
+			       const char *buf,			\
+			       size_t count)			\
+	{							\
+		return ggs_store(device, attr, buf, count, N);	\
+	}
+
+static GGS0_STORE(0);
+static GGS0_STORE(1);
+static GGS0_STORE(2);
+static GGS0_STORE(3);
+
+static ssize_t pggs_show(struct device *device,
+			 struct device_attribute *attr,
+			 char *buf,
+			 u32 reg)
+{
+	int ret;
+	u32 ret_payload[PAYLOAD_ARG_CNT];
+
+	ret = zynqmp_pm_read_pggs(reg, ret_payload);
+	if (ret)
+		return ret;
+
+	return sprintf(buf, "0x%x\n", ret_payload[1]);
+}
+
+static ssize_t pggs_store(struct device *device,
+			  struct device_attribute *attr,
+			  const char *buf, size_t count,
+			  u32 reg)
 {
-	if (eemi_ops_tbl)
-		return eemi_ops_tbl;
-	else
-		return ERR_PTR(-EPROBE_DEFER);
+	long value;
+	int ret;
+
+	if (reg >= GSS_NUM_REGS)
+		return -EINVAL;
 
+	ret = kstrtol(buf, 16, &value);
+	if (ret) {
+		count = -EFAULT;
+		goto err;
+	}
+
+	ret = zynqmp_pm_write_pggs(reg, value);
+	if (ret)
+		count = -EFAULT;
+
+err:
+	return count;
 }
-EXPORT_SYMBOL_GPL(zynqmp_pm_get_eemi_ops);
+
+#define PGGS0_SHOW(N)						\
+	ssize_t pggs##N##_show(struct device *device,		\
+			       struct device_attribute *attr,	\
+			       char *buf)			\
+	{							\
+		return pggs_show(device, attr, buf, N);		\
+	}
+
+#define PGGS0_STORE(N)						\
+	ssize_t pggs##N##_store(struct device *device,		\
+				struct device_attribute *attr,	\
+				const char *buf,		\
+				size_t count)			\
+	{							\
+		return pggs_store(device, attr, buf, count, N);	\
+	}
+
+/* PGGS register show functions */
+static PGGS0_SHOW(0);
+static PGGS0_SHOW(1);
+static PGGS0_SHOW(2);
+static PGGS0_SHOW(3);
+
+/* PGGS register store functions */
+static PGGS0_STORE(0);
+static PGGS0_STORE(1);
+static PGGS0_STORE(2);
+static PGGS0_STORE(3);
+
+/* GGS register attributes */
+static DEVICE_ATTR_RW(ggs0);
+static DEVICE_ATTR_RW(ggs1);
+static DEVICE_ATTR_RW(ggs2);
+static DEVICE_ATTR_RW(ggs3);
+
+/* PGGS register attributes */
+static DEVICE_ATTR_RW(pggs0);
+static DEVICE_ATTR_RW(pggs1);
+static DEVICE_ATTR_RW(pggs2);
+static DEVICE_ATTR_RW(pggs3);
+
+static struct attribute *zynqmp_firmware_attrs[] = {
+	&dev_attr_ggs0.attr,
+	&dev_attr_ggs1.attr,
+	&dev_attr_ggs2.attr,
+	&dev_attr_ggs3.attr,
+	&dev_attr_pggs0.attr,
+	&dev_attr_pggs1.attr,
+	&dev_attr_pggs2.attr,
+	&dev_attr_pggs3.attr,
+	&dev_attr_shutdown_scope.attr,
+	&dev_attr_health_status.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(zynqmp_firmware);
 
 static int zynqmp_firmware_probe(struct platform_device *pdev)
 {
@@ -820,11 +1235,6 @@ static int zynqmp_firmware_probe(struct platform_device *pdev)
 	pr_info("%s Trustzone version v%d.%d\n", __func__,
 		pm_tz_version >> 16, pm_tz_version & 0xFFFF);
 
-	/* Assign eemi_ops_table */
-	eemi_ops_tbl = &eemi_ops;
-
-	zynqmp_pm_api_debugfs_init();
-
 	ret = mfd_add_devices(&pdev->dev, PLATFORM_DEVID_NONE, firmware_devs,
 			      ARRAY_SIZE(firmware_devs), NULL, 0, NULL);
 	if (ret) {
@@ -832,6 +1242,8 @@ static int zynqmp_firmware_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	zynqmp_pm_api_debugfs_init();
+
 	return of_platform_populate(dev->of_node, NULL, NULL, dev);
 }
 
@@ -854,6 +1266,7 @@ static struct platform_driver zynqmp_firmware_driver = {
 	.driver = {
 		.name = "zynqmp_firmware",
 		.of_match_table = zynqmp_firmware_of_match,
+		.dev_groups = zynqmp_firmware_groups,
 	},
 	.probe = zynqmp_firmware_probe,
 	.remove = zynqmp_firmware_remove,
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index 72380e1d31c7..b2408a710662 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -156,7 +156,7 @@ config FPGA_DFL
 
 config FPGA_DFL_FME
 	tristate "FPGA DFL FME Driver"
-	depends on FPGA_DFL && HWMON
+	depends on FPGA_DFL && HWMON && PERF_EVENTS
 	help
 	  The FPGA Management Engine (FME) is a feature device implemented
 	  under Device Feature List (DFL) framework. Select this option to
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index 4865b74b00a4..d8e21dfc6778 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -40,6 +40,7 @@ obj-$(CONFIG_FPGA_DFL_FME_REGION)	+= dfl-fme-region.o
 obj-$(CONFIG_FPGA_DFL_AFU)		+= dfl-afu.o
 
 dfl-fme-objs := dfl-fme-main.o dfl-fme-pr.o dfl-fme-error.o
+dfl-fme-objs += dfl-fme-perf.o
 dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o
 dfl-afu-objs += dfl-afu-error.o
 
diff --git a/drivers/fpga/dfl-afu-dma-region.c b/drivers/fpga/dfl-afu-dma-region.c
index d902acb36d14..02d8cbad1ae2 100644
--- a/drivers/fpga/dfl-afu-dma-region.c
+++ b/drivers/fpga/dfl-afu-dma-region.c
@@ -61,10 +61,10 @@ static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
 				     region->pages);
 	if (pinned < 0) {
 		ret = pinned;
-		goto put_pages;
+		goto free_pages;
 	} else if (pinned != npages) {
 		ret = -EFAULT;
-		goto free_pages;
+		goto put_pages;
 	}
 
 	dev_dbg(dev, "%d pages pinned\n", pinned);
diff --git a/drivers/fpga/dfl-afu-main.c b/drivers/fpga/dfl-afu-main.c
index 65437b6a6842..b0c31789a909 100644
--- a/drivers/fpga/dfl-afu-main.c
+++ b/drivers/fpga/dfl-afu-main.c
@@ -561,14 +561,16 @@ static int afu_open(struct inode *inode, struct file *filp)
 	if (WARN_ON(!pdata))
 		return -ENODEV;
 
-	ret = dfl_feature_dev_use_begin(pdata);
-	if (ret)
-		return ret;
-
-	dev_dbg(&fdev->dev, "Device File Open\n");
-	filp->private_data = fdev;
+	mutex_lock(&pdata->lock);
+	ret = dfl_feature_dev_use_begin(pdata, filp->f_flags & O_EXCL);
+	if (!ret) {
+		dev_dbg(&fdev->dev, "Device File Opened %d Times\n",
+			dfl_feature_dev_use_count(pdata));
+		filp->private_data = fdev;
+	}
+	mutex_unlock(&pdata->lock);
 
-	return 0;
+	return ret;
 }
 
 static int afu_release(struct inode *inode, struct file *filp)
@@ -581,12 +583,14 @@ static int afu_release(struct inode *inode, struct file *filp)
 	pdata = dev_get_platdata(&pdev->dev);
 
 	mutex_lock(&pdata->lock);
-	__port_reset(pdev);
-	afu_dma_region_destroy(pdata);
-	mutex_unlock(&pdata->lock);
-
 	dfl_feature_dev_use_end(pdata);
 
+	if (!dfl_feature_dev_use_count(pdata)) {
+		__port_reset(pdev);
+		afu_dma_region_destroy(pdata);
+	}
+	mutex_unlock(&pdata->lock);
+
 	return 0;
 }
 
@@ -746,6 +750,12 @@ static long afu_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	return -EINVAL;
 }
 
+static const struct vm_operations_struct afu_vma_ops = {
+#ifdef CONFIG_HAVE_IOREMAP_PROT
+	.access = generic_access_phys,
+#endif
+};
+
 static int afu_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct platform_device *pdev = filp->private_data;
@@ -775,6 +785,9 @@ static int afu_mmap(struct file *filp, struct vm_area_struct *vma)
 	    !(region.flags & DFL_PORT_REGION_WRITE))
 		return -EPERM;
 
+	/* Support debug access to the mapping */
+	vma->vm_ops = &afu_vma_ops;
+
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
 	return remap_pfn_range(vma, vma->vm_start,
diff --git a/drivers/fpga/dfl-fme-main.c b/drivers/fpga/dfl-fme-main.c
index 1d4690c99268..fc210d4e1863 100644
--- a/drivers/fpga/dfl-fme-main.c
+++ b/drivers/fpga/dfl-fme-main.c
@@ -580,6 +580,10 @@ static struct dfl_feature_driver fme_feature_drvs[] = {
 		.ops = &fme_power_mgmt_ops,
 	},
 	{
+		.id_table = fme_perf_id_table,
+		.ops = &fme_perf_ops,
+	},
+	{
 		.ops = NULL,
 	},
 };
@@ -600,14 +604,16 @@ static int fme_open(struct inode *inode, struct file *filp)
 	if (WARN_ON(!pdata))
 		return -ENODEV;
 
-	ret = dfl_feature_dev_use_begin(pdata);
-	if (ret)
-		return ret;
-
-	dev_dbg(&fdev->dev, "Device File Open\n");
-	filp->private_data = pdata;
+	mutex_lock(&pdata->lock);
+	ret = dfl_feature_dev_use_begin(pdata, filp->f_flags & O_EXCL);
+	if (!ret) {
+		dev_dbg(&fdev->dev, "Device File Opened %d Times\n",
+			dfl_feature_dev_use_count(pdata));
+		filp->private_data = pdata;
+	}
+	mutex_unlock(&pdata->lock);
 
-	return 0;
+	return ret;
 }
 
 static int fme_release(struct inode *inode, struct file *filp)
@@ -616,7 +622,10 @@ static int fme_release(struct inode *inode, struct file *filp)
 	struct platform_device *pdev = pdata->dev;
 
 	dev_dbg(&pdev->dev, "Device File Release\n");
+
+	mutex_lock(&pdata->lock);
 	dfl_feature_dev_use_end(pdata);
+	mutex_unlock(&pdata->lock);
 
 	return 0;
 }
diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c
new file mode 100644
index 000000000000..6ce1ed222ea4
--- /dev/null
+++ b/drivers/fpga/dfl-fme-perf.c
@@ -0,0 +1,1020 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for FPGA Management Engine (FME) Global Performance Reporting
+ *
+ * Copyright 2019 Intel Corporation, Inc.
+ *
+ * Authors:
+ *   Kang Luwei <luwei.kang@intel.com>
+ *   Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *   Wu Hao <hao.wu@intel.com>
+ *   Xu Yilun <yilun.xu@intel.com>
+ *   Joseph Grecco <joe.grecco@intel.com>
+ *   Enno Luebbers <enno.luebbers@intel.com>
+ *   Tim Whisonant <tim.whisonant@intel.com>
+ *   Ananda Ravuri <ananda.ravuri@intel.com>
+ *   Mitchel, Henry <henry.mitchel@intel.com>
+ */
+
+#include <linux/perf_event.h>
+#include "dfl.h"
+#include "dfl-fme.h"
+
+/*
+ * Performance Counter Registers for Cache.
+ *
+ * Cache Events are listed below as CACHE_EVNT_*.
+ */
+#define CACHE_CTRL			0x8
+#define CACHE_RESET_CNTR		BIT_ULL(0)
+#define CACHE_FREEZE_CNTR		BIT_ULL(8)
+#define CACHE_CTRL_EVNT			GENMASK_ULL(19, 16)
+#define CACHE_EVNT_RD_HIT		0x0
+#define CACHE_EVNT_WR_HIT		0x1
+#define CACHE_EVNT_RD_MISS		0x2
+#define CACHE_EVNT_WR_MISS		0x3
+#define CACHE_EVNT_RSVD			0x4
+#define CACHE_EVNT_HOLD_REQ		0x5
+#define CACHE_EVNT_DATA_WR_PORT_CONTEN	0x6
+#define CACHE_EVNT_TAG_WR_PORT_CONTEN	0x7
+#define CACHE_EVNT_TX_REQ_STALL		0x8
+#define CACHE_EVNT_RX_REQ_STALL		0x9
+#define CACHE_EVNT_EVICTIONS		0xa
+#define CACHE_EVNT_MAX			CACHE_EVNT_EVICTIONS
+#define CACHE_CHANNEL_SEL		BIT_ULL(20)
+#define CACHE_CHANNEL_RD		0
+#define CACHE_CHANNEL_WR		1
+#define CACHE_CNTR0			0x10
+#define CACHE_CNTR1			0x18
+#define CACHE_CNTR_EVNT_CNTR		GENMASK_ULL(47, 0)
+#define CACHE_CNTR_EVNT			GENMASK_ULL(63, 60)
+
+/*
+ * Performance Counter Registers for Fabric.
+ *
+ * Fabric Events are listed below as FAB_EVNT_*
+ */
+#define FAB_CTRL			0x20
+#define FAB_RESET_CNTR			BIT_ULL(0)
+#define FAB_FREEZE_CNTR			BIT_ULL(8)
+#define FAB_CTRL_EVNT			GENMASK_ULL(19, 16)
+#define FAB_EVNT_PCIE0_RD		0x0
+#define FAB_EVNT_PCIE0_WR		0x1
+#define FAB_EVNT_PCIE1_RD		0x2
+#define FAB_EVNT_PCIE1_WR		0x3
+#define FAB_EVNT_UPI_RD			0x4
+#define FAB_EVNT_UPI_WR			0x5
+#define FAB_EVNT_MMIO_RD		0x6
+#define FAB_EVNT_MMIO_WR		0x7
+#define FAB_EVNT_MAX			FAB_EVNT_MMIO_WR
+#define FAB_PORT_ID			GENMASK_ULL(21, 20)
+#define FAB_PORT_FILTER			BIT_ULL(23)
+#define FAB_PORT_FILTER_DISABLE		0
+#define FAB_PORT_FILTER_ENABLE		1
+#define FAB_CNTR			0x28
+#define FAB_CNTR_EVNT_CNTR		GENMASK_ULL(59, 0)
+#define FAB_CNTR_EVNT			GENMASK_ULL(63, 60)
+
+/*
+ * Performance Counter Registers for Clock.
+ *
+ * Clock Counter can't be reset or frozen by SW.
+ */
+#define CLK_CNTR			0x30
+#define BASIC_EVNT_CLK			0x0
+#define BASIC_EVNT_MAX			BASIC_EVNT_CLK
+
+/*
+ * Performance Counter Registers for IOMMU / VT-D.
+ *
+ * VT-D Events are listed below as VTD_EVNT_* and VTD_SIP_EVNT_*
+ */
+#define VTD_CTRL			0x38
+#define VTD_RESET_CNTR			BIT_ULL(0)
+#define VTD_FREEZE_CNTR			BIT_ULL(8)
+#define VTD_CTRL_EVNT			GENMASK_ULL(19, 16)
+#define VTD_EVNT_AFU_MEM_RD_TRANS	0x0
+#define VTD_EVNT_AFU_MEM_WR_TRANS	0x1
+#define VTD_EVNT_AFU_DEVTLB_RD_HIT	0x2
+#define VTD_EVNT_AFU_DEVTLB_WR_HIT	0x3
+#define VTD_EVNT_DEVTLB_4K_FILL		0x4
+#define VTD_EVNT_DEVTLB_2M_FILL		0x5
+#define VTD_EVNT_DEVTLB_1G_FILL		0x6
+#define VTD_EVNT_MAX			VTD_EVNT_DEVTLB_1G_FILL
+#define VTD_CNTR			0x40
+#define VTD_CNTR_EVNT_CNTR		GENMASK_ULL(47, 0)
+#define VTD_CNTR_EVNT			GENMASK_ULL(63, 60)
+
+#define VTD_SIP_CTRL			0x48
+#define VTD_SIP_RESET_CNTR		BIT_ULL(0)
+#define VTD_SIP_FREEZE_CNTR		BIT_ULL(8)
+#define VTD_SIP_CTRL_EVNT		GENMASK_ULL(19, 16)
+#define VTD_SIP_EVNT_IOTLB_4K_HIT	0x0
+#define VTD_SIP_EVNT_IOTLB_2M_HIT	0x1
+#define VTD_SIP_EVNT_IOTLB_1G_HIT	0x2
+#define VTD_SIP_EVNT_SLPWC_L3_HIT	0x3
+#define VTD_SIP_EVNT_SLPWC_L4_HIT	0x4
+#define VTD_SIP_EVNT_RCC_HIT		0x5
+#define VTD_SIP_EVNT_IOTLB_4K_MISS	0x6
+#define VTD_SIP_EVNT_IOTLB_2M_MISS	0x7
+#define VTD_SIP_EVNT_IOTLB_1G_MISS	0x8
+#define VTD_SIP_EVNT_SLPWC_L3_MISS	0x9
+#define VTD_SIP_EVNT_SLPWC_L4_MISS	0xa
+#define VTD_SIP_EVNT_RCC_MISS		0xb
+#define VTD_SIP_EVNT_MAX		VTD_SIP_EVNT_SLPWC_L4_MISS
+#define VTD_SIP_CNTR			0X50
+#define VTD_SIP_CNTR_EVNT_CNTR		GENMASK_ULL(47, 0)
+#define VTD_SIP_CNTR_EVNT		GENMASK_ULL(63, 60)
+
+#define PERF_TIMEOUT			30
+
+#define PERF_MAX_PORT_NUM		1U
+
+/**
+ * struct fme_perf_priv - priv data structure for fme perf driver
+ *
+ * @dev: parent device.
+ * @ioaddr: mapped base address of mmio region.
+ * @pmu: pmu data structure for fme perf counters.
+ * @id: id of this fme performance report private feature.
+ * @fab_users: current user number on fabric counters.
+ * @fab_port_id: used to indicate current working mode of fabric counters.
+ * @fab_lock: lock to protect fabric counters working mode.
+ * @cpu: active CPU to which the PMU is bound for accesses.
+ * @cpuhp_node: node for CPU hotplug notifier link.
+ * @cpuhp_state: state for CPU hotplug notification;
+ */
+struct fme_perf_priv {
+	struct device *dev;
+	void __iomem *ioaddr;
+	struct pmu pmu;
+	u64 id;
+
+	u32 fab_users;
+	u32 fab_port_id;
+	spinlock_t fab_lock;
+
+	unsigned int cpu;
+	struct hlist_node node;
+	enum cpuhp_state cpuhp_state;
+};
+
+/**
+ * struct fme_perf_event_ops - callbacks for fme perf events
+ *
+ * @event_init: callback invoked during event init.
+ * @event_destroy: callback invoked during event destroy.
+ * @read_counter: callback to read hardware counters.
+ */
+struct fme_perf_event_ops {
+	int (*event_init)(struct fme_perf_priv *priv, u32 event, u32 portid);
+	void (*event_destroy)(struct fme_perf_priv *priv, u32 event,
+			      u32 portid);
+	u64 (*read_counter)(struct fme_perf_priv *priv, u32 event, u32 portid);
+};
+
+#define to_fme_perf_priv(_pmu)	container_of(_pmu, struct fme_perf_priv, pmu)
+
+static ssize_t cpumask_show(struct device *dev,
+			    struct device_attribute *attr, char *buf)
+{
+	struct pmu *pmu = dev_get_drvdata(dev);
+	struct fme_perf_priv *priv;
+
+	priv = to_fme_perf_priv(pmu);
+
+	return cpumap_print_to_pagebuf(true, buf, cpumask_of(priv->cpu));
+}
+static DEVICE_ATTR_RO(cpumask);
+
+static struct attribute *fme_perf_cpumask_attrs[] = {
+	&dev_attr_cpumask.attr,
+	NULL,
+};
+
+static struct attribute_group fme_perf_cpumask_group = {
+	.attrs = fme_perf_cpumask_attrs,
+};
+
+#define FME_EVENT_MASK		GENMASK_ULL(11, 0)
+#define FME_EVENT_SHIFT		0
+#define FME_EVTYPE_MASK		GENMASK_ULL(15, 12)
+#define FME_EVTYPE_SHIFT	12
+#define FME_EVTYPE_BASIC	0
+#define FME_EVTYPE_CACHE	1
+#define FME_EVTYPE_FABRIC	2
+#define FME_EVTYPE_VTD		3
+#define FME_EVTYPE_VTD_SIP	4
+#define FME_EVTYPE_MAX		FME_EVTYPE_VTD_SIP
+#define FME_PORTID_MASK		GENMASK_ULL(23, 16)
+#define FME_PORTID_SHIFT	16
+#define FME_PORTID_ROOT		(0xffU)
+
+#define get_event(_config)	FIELD_GET(FME_EVENT_MASK, _config)
+#define get_evtype(_config)	FIELD_GET(FME_EVTYPE_MASK, _config)
+#define get_portid(_config)	FIELD_GET(FME_PORTID_MASK, _config)
+
+PMU_FORMAT_ATTR(event,		"config:0-11");
+PMU_FORMAT_ATTR(evtype,		"config:12-15");
+PMU_FORMAT_ATTR(portid,		"config:16-23");
+
+static struct attribute *fme_perf_format_attrs[] = {
+	&format_attr_event.attr,
+	&format_attr_evtype.attr,
+	&format_attr_portid.attr,
+	NULL,
+};
+
+static struct attribute_group fme_perf_format_group = {
+	.name = "format",
+	.attrs = fme_perf_format_attrs,
+};
+
+/*
+ * There are no default events, but we need to create
+ * "events" group (with empty attrs) before updating
+ * it with detected events (using pmu->attr_update).
+ */
+static struct attribute *fme_perf_events_attrs_empty[] = {
+	NULL,
+};
+
+static struct attribute_group fme_perf_events_group = {
+	.name = "events",
+	.attrs = fme_perf_events_attrs_empty,
+};
+
+static const struct attribute_group *fme_perf_groups[] = {
+	&fme_perf_format_group,
+	&fme_perf_cpumask_group,
+	&fme_perf_events_group,
+	NULL,
+};
+
+static bool is_portid_root(u32 portid)
+{
+	return portid == FME_PORTID_ROOT;
+}
+
+static bool is_portid_port(u32 portid)
+{
+	return portid < PERF_MAX_PORT_NUM;
+}
+
+static bool is_portid_root_or_port(u32 portid)
+{
+	return is_portid_root(portid) || is_portid_port(portid);
+}
+
+static u64 fme_read_perf_cntr_reg(void __iomem *addr)
+{
+	u32 low;
+	u64 v;
+
+	/*
+	 * For 64bit counter registers, the counter may increases and carries
+	 * out of bit [31] between 2 32bit reads. So add extra reads to help
+	 * to prevent this issue. This only happens in platforms which don't
+	 * support 64bit read - readq is split into 2 readl.
+	 */
+	do {
+		v = readq(addr);
+		low = readl(addr);
+	} while (((u32)v) > low);
+
+	return v;
+}
+
+static int basic_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+	if (event <= BASIC_EVNT_MAX && is_portid_root(portid))
+		return 0;
+
+	return -EINVAL;
+}
+
+static u64 basic_read_event_counter(struct fme_perf_priv *priv,
+				    u32 event, u32 portid)
+{
+	void __iomem *base = priv->ioaddr;
+
+	return fme_read_perf_cntr_reg(base + CLK_CNTR);
+}
+
+static int cache_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+	if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
+	    event <= CACHE_EVNT_MAX && is_portid_root(portid))
+		return 0;
+
+	return -EINVAL;
+}
+
+static u64 cache_read_event_counter(struct fme_perf_priv *priv,
+				    u32 event, u32 portid)
+{
+	void __iomem *base = priv->ioaddr;
+	u64 v, count;
+	u8 channel;
+
+	if (event == CACHE_EVNT_WR_HIT || event == CACHE_EVNT_WR_MISS ||
+	    event == CACHE_EVNT_DATA_WR_PORT_CONTEN ||
+	    event == CACHE_EVNT_TAG_WR_PORT_CONTEN)
+		channel = CACHE_CHANNEL_WR;
+	else
+		channel = CACHE_CHANNEL_RD;
+
+	/* set channel access type and cache event code. */
+	v = readq(base + CACHE_CTRL);
+	v &= ~(CACHE_CHANNEL_SEL | CACHE_CTRL_EVNT);
+	v |= FIELD_PREP(CACHE_CHANNEL_SEL, channel);
+	v |= FIELD_PREP(CACHE_CTRL_EVNT, event);
+	writeq(v, base + CACHE_CTRL);
+
+	if (readq_poll_timeout_atomic(base + CACHE_CNTR0, v,
+				      FIELD_GET(CACHE_CNTR_EVNT, v) == event,
+				      1, PERF_TIMEOUT)) {
+		dev_err(priv->dev, "timeout, unmatched cache event code in counter register.\n");
+		return 0;
+	}
+
+	v = fme_read_perf_cntr_reg(base + CACHE_CNTR0);
+	count = FIELD_GET(CACHE_CNTR_EVNT_CNTR, v);
+	v = fme_read_perf_cntr_reg(base + CACHE_CNTR1);
+	count += FIELD_GET(CACHE_CNTR_EVNT_CNTR, v);
+
+	return count;
+}
+
+static bool is_fabric_event_supported(struct fme_perf_priv *priv, u32 event,
+				      u32 portid)
+{
+	if (event > FAB_EVNT_MAX || !is_portid_root_or_port(portid))
+		return false;
+
+	if (priv->id == FME_FEATURE_ID_GLOBAL_DPERF &&
+	    (event == FAB_EVNT_PCIE1_RD || event == FAB_EVNT_UPI_RD ||
+	     event == FAB_EVNT_PCIE1_WR || event == FAB_EVNT_UPI_WR))
+		return false;
+
+	return true;
+}
+
+static int fabric_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+	void __iomem *base = priv->ioaddr;
+	int ret = 0;
+	u64 v;
+
+	if (!is_fabric_event_supported(priv, event, portid))
+		return -EINVAL;
+
+	/*
+	 * as fabric counter set only can be in either overall or port mode.
+	 * In overall mode, it counts overall data for FPGA, and in port mode,
+	 * it is configured to monitor on one individual port.
+	 *
+	 * so every time, a new event is initialized, driver checks
+	 * current working mode and if someone is using this counter set.
+	 */
+	spin_lock(&priv->fab_lock);
+	if (priv->fab_users && priv->fab_port_id != portid) {
+		dev_dbg(priv->dev, "conflict fabric event monitoring mode.\n");
+		ret = -EOPNOTSUPP;
+		goto exit;
+	}
+
+	priv->fab_users++;
+
+	/*
+	 * skip if current working mode matches, otherwise change the working
+	 * mode per input port_id, to monitor overall data or another port.
+	 */
+	if (priv->fab_port_id == portid)
+		goto exit;
+
+	priv->fab_port_id = portid;
+
+	v = readq(base + FAB_CTRL);
+	v &= ~(FAB_PORT_FILTER | FAB_PORT_ID);
+
+	if (is_portid_root(portid)) {
+		v |= FIELD_PREP(FAB_PORT_FILTER, FAB_PORT_FILTER_DISABLE);
+	} else {
+		v |= FIELD_PREP(FAB_PORT_FILTER, FAB_PORT_FILTER_ENABLE);
+		v |= FIELD_PREP(FAB_PORT_ID, portid);
+	}
+	writeq(v, base + FAB_CTRL);
+
+exit:
+	spin_unlock(&priv->fab_lock);
+	return ret;
+}
+
+static void fabric_event_destroy(struct fme_perf_priv *priv, u32 event,
+				 u32 portid)
+{
+	spin_lock(&priv->fab_lock);
+	priv->fab_users--;
+	spin_unlock(&priv->fab_lock);
+}
+
+static u64 fabric_read_event_counter(struct fme_perf_priv *priv, u32 event,
+				     u32 portid)
+{
+	void __iomem *base = priv->ioaddr;
+	u64 v;
+
+	v = readq(base + FAB_CTRL);
+	v &= ~FAB_CTRL_EVNT;
+	v |= FIELD_PREP(FAB_CTRL_EVNT, event);
+	writeq(v, base + FAB_CTRL);
+
+	if (readq_poll_timeout_atomic(base + FAB_CNTR, v,
+				      FIELD_GET(FAB_CNTR_EVNT, v) == event,
+				      1, PERF_TIMEOUT)) {
+		dev_err(priv->dev, "timeout, unmatched fab event code in counter register.\n");
+		return 0;
+	}
+
+	v = fme_read_perf_cntr_reg(base + FAB_CNTR);
+	return FIELD_GET(FAB_CNTR_EVNT_CNTR, v);
+}
+
+static int vtd_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+	if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
+	    event <= VTD_EVNT_MAX && is_portid_port(portid))
+		return 0;
+
+	return -EINVAL;
+}
+
+static u64 vtd_read_event_counter(struct fme_perf_priv *priv, u32 event,
+				  u32 portid)
+{
+	void __iomem *base = priv->ioaddr;
+	u64 v;
+
+	event += (portid * (VTD_EVNT_MAX + 1));
+
+	v = readq(base + VTD_CTRL);
+	v &= ~VTD_CTRL_EVNT;
+	v |= FIELD_PREP(VTD_CTRL_EVNT, event);
+	writeq(v, base + VTD_CTRL);
+
+	if (readq_poll_timeout_atomic(base + VTD_CNTR, v,
+				      FIELD_GET(VTD_CNTR_EVNT, v) == event,
+				      1, PERF_TIMEOUT)) {
+		dev_err(priv->dev, "timeout, unmatched vtd event code in counter register.\n");
+		return 0;
+	}
+
+	v = fme_read_perf_cntr_reg(base + VTD_CNTR);
+	return FIELD_GET(VTD_CNTR_EVNT_CNTR, v);
+}
+
+static int vtd_sip_event_init(struct fme_perf_priv *priv, u32 event, u32 portid)
+{
+	if (priv->id == FME_FEATURE_ID_GLOBAL_IPERF &&
+	    event <= VTD_SIP_EVNT_MAX && is_portid_root(portid))
+		return 0;
+
+	return -EINVAL;
+}
+
+static u64 vtd_sip_read_event_counter(struct fme_perf_priv *priv, u32 event,
+				      u32 portid)
+{
+	void __iomem *base = priv->ioaddr;
+	u64 v;
+
+	v = readq(base + VTD_SIP_CTRL);
+	v &= ~VTD_SIP_CTRL_EVNT;
+	v |= FIELD_PREP(VTD_SIP_CTRL_EVNT, event);
+	writeq(v, base + VTD_SIP_CTRL);
+
+	if (readq_poll_timeout_atomic(base + VTD_SIP_CNTR, v,
+				      FIELD_GET(VTD_SIP_CNTR_EVNT, v) == event,
+				      1, PERF_TIMEOUT)) {
+		dev_err(priv->dev, "timeout, unmatched vtd sip event code in counter register\n");
+		return 0;
+	}
+
+	v = fme_read_perf_cntr_reg(base + VTD_SIP_CNTR);
+	return FIELD_GET(VTD_SIP_CNTR_EVNT_CNTR, v);
+}
+
+static struct fme_perf_event_ops fme_perf_event_ops[] = {
+	[FME_EVTYPE_BASIC]	= {.event_init = basic_event_init,
+				   .read_counter = basic_read_event_counter,},
+	[FME_EVTYPE_CACHE]	= {.event_init = cache_event_init,
+				   .read_counter = cache_read_event_counter,},
+	[FME_EVTYPE_FABRIC]	= {.event_init = fabric_event_init,
+				   .event_destroy = fabric_event_destroy,
+				   .read_counter = fabric_read_event_counter,},
+	[FME_EVTYPE_VTD]	= {.event_init = vtd_event_init,
+				   .read_counter = vtd_read_event_counter,},
+	[FME_EVTYPE_VTD_SIP]	= {.event_init = vtd_sip_event_init,
+				   .read_counter = vtd_sip_read_event_counter,},
+};
+
+static ssize_t fme_perf_event_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct dev_ext_attribute *eattr;
+	unsigned long config;
+	char *ptr = buf;
+
+	eattr = container_of(attr, struct dev_ext_attribute, attr);
+	config = (unsigned long)eattr->var;
+
+	ptr += sprintf(ptr, "event=0x%02x", (unsigned int)get_event(config));
+	ptr += sprintf(ptr, ",evtype=0x%02x", (unsigned int)get_evtype(config));
+
+	if (is_portid_root(get_portid(config)))
+		ptr += sprintf(ptr, ",portid=0x%02x\n", FME_PORTID_ROOT);
+	else
+		ptr += sprintf(ptr, ",portid=?\n");
+
+	return (ssize_t)(ptr - buf);
+}
+
+#define FME_EVENT_ATTR(_name) \
+	__ATTR(_name, 0444, fme_perf_event_show, NULL)
+
+#define FME_PORT_EVENT_CONFIG(_event, _type)				\
+	(void *)((((_event) << FME_EVENT_SHIFT) & FME_EVENT_MASK) |	\
+		(((_type) << FME_EVTYPE_SHIFT) & FME_EVTYPE_MASK))
+
+#define FME_EVENT_CONFIG(_event, _type)					\
+	(void *)((((_event) << FME_EVENT_SHIFT) & FME_EVENT_MASK) |	\
+		(((_type) << FME_EVTYPE_SHIFT) & FME_EVTYPE_MASK) |	\
+		(FME_PORTID_ROOT << FME_PORTID_SHIFT))
+
+/* FME Perf Basic Events */
+#define FME_EVENT_BASIC(_name, _event)					\
+static struct dev_ext_attribute fme_perf_event_##_name = {		\
+	.attr = FME_EVENT_ATTR(_name),					\
+	.var = FME_EVENT_CONFIG(_event, FME_EVTYPE_BASIC),		\
+}
+
+FME_EVENT_BASIC(clock, BASIC_EVNT_CLK);
+
+static struct attribute *fme_perf_basic_events_attrs[] = {
+	&fme_perf_event_clock.attr.attr,
+	NULL,
+};
+
+static const struct attribute_group fme_perf_basic_events_group = {
+	.name = "events",
+	.attrs = fme_perf_basic_events_attrs,
+};
+
+/* FME Perf Cache Events */
+#define FME_EVENT_CACHE(_name, _event)					\
+static struct dev_ext_attribute fme_perf_event_cache_##_name = {	\
+	.attr = FME_EVENT_ATTR(cache_##_name),				\
+	.var = FME_EVENT_CONFIG(_event, FME_EVTYPE_CACHE),		\
+}
+
+FME_EVENT_CACHE(read_hit,     CACHE_EVNT_RD_HIT);
+FME_EVENT_CACHE(read_miss,    CACHE_EVNT_RD_MISS);
+FME_EVENT_CACHE(write_hit,    CACHE_EVNT_WR_HIT);
+FME_EVENT_CACHE(write_miss,   CACHE_EVNT_WR_MISS);
+FME_EVENT_CACHE(hold_request, CACHE_EVNT_HOLD_REQ);
+FME_EVENT_CACHE(tx_req_stall, CACHE_EVNT_TX_REQ_STALL);
+FME_EVENT_CACHE(rx_req_stall, CACHE_EVNT_RX_REQ_STALL);
+FME_EVENT_CACHE(eviction,     CACHE_EVNT_EVICTIONS);
+FME_EVENT_CACHE(data_write_port_contention, CACHE_EVNT_DATA_WR_PORT_CONTEN);
+FME_EVENT_CACHE(tag_write_port_contention,  CACHE_EVNT_TAG_WR_PORT_CONTEN);
+
+static struct attribute *fme_perf_cache_events_attrs[] = {
+	&fme_perf_event_cache_read_hit.attr.attr,
+	&fme_perf_event_cache_read_miss.attr.attr,
+	&fme_perf_event_cache_write_hit.attr.attr,
+	&fme_perf_event_cache_write_miss.attr.attr,
+	&fme_perf_event_cache_hold_request.attr.attr,
+	&fme_perf_event_cache_tx_req_stall.attr.attr,
+	&fme_perf_event_cache_rx_req_stall.attr.attr,
+	&fme_perf_event_cache_eviction.attr.attr,
+	&fme_perf_event_cache_data_write_port_contention.attr.attr,
+	&fme_perf_event_cache_tag_write_port_contention.attr.attr,
+	NULL,
+};
+
+static umode_t fme_perf_events_visible(struct kobject *kobj,
+				       struct attribute *attr, int n)
+{
+	struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
+	struct fme_perf_priv *priv = to_fme_perf_priv(pmu);
+
+	return (priv->id == FME_FEATURE_ID_GLOBAL_IPERF) ? attr->mode : 0;
+}
+
+static const struct attribute_group fme_perf_cache_events_group = {
+	.name = "events",
+	.attrs = fme_perf_cache_events_attrs,
+	.is_visible = fme_perf_events_visible,
+};
+
+/* FME Perf Fabric Events */
+#define FME_EVENT_FABRIC(_name, _event)					\
+static struct dev_ext_attribute fme_perf_event_fab_##_name = {		\
+	.attr = FME_EVENT_ATTR(fab_##_name),				\
+	.var = FME_EVENT_CONFIG(_event, FME_EVTYPE_FABRIC),		\
+}
+
+#define FME_EVENT_FABRIC_PORT(_name, _event)				\
+static struct dev_ext_attribute fme_perf_event_fab_port_##_name = {	\
+	.attr = FME_EVENT_ATTR(fab_port_##_name),			\
+	.var = FME_PORT_EVENT_CONFIG(_event, FME_EVTYPE_FABRIC),	\
+}
+
+FME_EVENT_FABRIC(pcie0_read,  FAB_EVNT_PCIE0_RD);
+FME_EVENT_FABRIC(pcie0_write, FAB_EVNT_PCIE0_WR);
+FME_EVENT_FABRIC(pcie1_read,  FAB_EVNT_PCIE1_RD);
+FME_EVENT_FABRIC(pcie1_write, FAB_EVNT_PCIE1_WR);
+FME_EVENT_FABRIC(upi_read,    FAB_EVNT_UPI_RD);
+FME_EVENT_FABRIC(upi_write,   FAB_EVNT_UPI_WR);
+FME_EVENT_FABRIC(mmio_read,   FAB_EVNT_MMIO_RD);
+FME_EVENT_FABRIC(mmio_write,  FAB_EVNT_MMIO_WR);
+
+FME_EVENT_FABRIC_PORT(pcie0_read,  FAB_EVNT_PCIE0_RD);
+FME_EVENT_FABRIC_PORT(pcie0_write, FAB_EVNT_PCIE0_WR);
+FME_EVENT_FABRIC_PORT(pcie1_read,  FAB_EVNT_PCIE1_RD);
+FME_EVENT_FABRIC_PORT(pcie1_write, FAB_EVNT_PCIE1_WR);
+FME_EVENT_FABRIC_PORT(upi_read,    FAB_EVNT_UPI_RD);
+FME_EVENT_FABRIC_PORT(upi_write,   FAB_EVNT_UPI_WR);
+FME_EVENT_FABRIC_PORT(mmio_read,   FAB_EVNT_MMIO_RD);
+FME_EVENT_FABRIC_PORT(mmio_write,  FAB_EVNT_MMIO_WR);
+
+static struct attribute *fme_perf_fabric_events_attrs[] = {
+	&fme_perf_event_fab_pcie0_read.attr.attr,
+	&fme_perf_event_fab_pcie0_write.attr.attr,
+	&fme_perf_event_fab_pcie1_read.attr.attr,
+	&fme_perf_event_fab_pcie1_write.attr.attr,
+	&fme_perf_event_fab_upi_read.attr.attr,
+	&fme_perf_event_fab_upi_write.attr.attr,
+	&fme_perf_event_fab_mmio_read.attr.attr,
+	&fme_perf_event_fab_mmio_write.attr.attr,
+	&fme_perf_event_fab_port_pcie0_read.attr.attr,
+	&fme_perf_event_fab_port_pcie0_write.attr.attr,
+	&fme_perf_event_fab_port_pcie1_read.attr.attr,
+	&fme_perf_event_fab_port_pcie1_write.attr.attr,
+	&fme_perf_event_fab_port_upi_read.attr.attr,
+	&fme_perf_event_fab_port_upi_write.attr.attr,
+	&fme_perf_event_fab_port_mmio_read.attr.attr,
+	&fme_perf_event_fab_port_mmio_write.attr.attr,
+	NULL,
+};
+
+static umode_t fme_perf_fabric_events_visible(struct kobject *kobj,
+					      struct attribute *attr, int n)
+{
+	struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
+	struct fme_perf_priv *priv = to_fme_perf_priv(pmu);
+	struct dev_ext_attribute *eattr;
+	unsigned long var;
+
+	eattr = container_of(attr, struct dev_ext_attribute, attr.attr);
+	var = (unsigned long)eattr->var;
+
+	if (is_fabric_event_supported(priv, get_event(var), get_portid(var)))
+		return attr->mode;
+
+	return 0;
+}
+
+static const struct attribute_group fme_perf_fabric_events_group = {
+	.name = "events",
+	.attrs = fme_perf_fabric_events_attrs,
+	.is_visible = fme_perf_fabric_events_visible,
+};
+
+/* FME Perf VTD Events */
+#define FME_EVENT_VTD_PORT(_name, _event)				\
+static struct dev_ext_attribute fme_perf_event_vtd_port_##_name = {	\
+	.attr = FME_EVENT_ATTR(vtd_port_##_name),			\
+	.var = FME_PORT_EVENT_CONFIG(_event, FME_EVTYPE_VTD),		\
+}
+
+FME_EVENT_VTD_PORT(read_transaction,  VTD_EVNT_AFU_MEM_RD_TRANS);
+FME_EVENT_VTD_PORT(write_transaction, VTD_EVNT_AFU_MEM_WR_TRANS);
+FME_EVENT_VTD_PORT(devtlb_read_hit,   VTD_EVNT_AFU_DEVTLB_RD_HIT);
+FME_EVENT_VTD_PORT(devtlb_write_hit,  VTD_EVNT_AFU_DEVTLB_WR_HIT);
+FME_EVENT_VTD_PORT(devtlb_4k_fill,    VTD_EVNT_DEVTLB_4K_FILL);
+FME_EVENT_VTD_PORT(devtlb_2m_fill,    VTD_EVNT_DEVTLB_2M_FILL);
+FME_EVENT_VTD_PORT(devtlb_1g_fill,    VTD_EVNT_DEVTLB_1G_FILL);
+
+static struct attribute *fme_perf_vtd_events_attrs[] = {
+	&fme_perf_event_vtd_port_read_transaction.attr.attr,
+	&fme_perf_event_vtd_port_write_transaction.attr.attr,
+	&fme_perf_event_vtd_port_devtlb_read_hit.attr.attr,
+	&fme_perf_event_vtd_port_devtlb_write_hit.attr.attr,
+	&fme_perf_event_vtd_port_devtlb_4k_fill.attr.attr,
+	&fme_perf_event_vtd_port_devtlb_2m_fill.attr.attr,
+	&fme_perf_event_vtd_port_devtlb_1g_fill.attr.attr,
+	NULL,
+};
+
+static const struct attribute_group fme_perf_vtd_events_group = {
+	.name = "events",
+	.attrs = fme_perf_vtd_events_attrs,
+	.is_visible = fme_perf_events_visible,
+};
+
+/* FME Perf VTD SIP Events */
+#define FME_EVENT_VTD_SIP(_name, _event)				\
+static struct dev_ext_attribute fme_perf_event_vtd_sip_##_name = {	\
+	.attr = FME_EVENT_ATTR(vtd_sip_##_name),			\
+	.var = FME_EVENT_CONFIG(_event, FME_EVTYPE_VTD_SIP),		\
+}
+
+FME_EVENT_VTD_SIP(iotlb_4k_hit,  VTD_SIP_EVNT_IOTLB_4K_HIT);
+FME_EVENT_VTD_SIP(iotlb_2m_hit,  VTD_SIP_EVNT_IOTLB_2M_HIT);
+FME_EVENT_VTD_SIP(iotlb_1g_hit,  VTD_SIP_EVNT_IOTLB_1G_HIT);
+FME_EVENT_VTD_SIP(slpwc_l3_hit,  VTD_SIP_EVNT_SLPWC_L3_HIT);
+FME_EVENT_VTD_SIP(slpwc_l4_hit,  VTD_SIP_EVNT_SLPWC_L4_HIT);
+FME_EVENT_VTD_SIP(rcc_hit,       VTD_SIP_EVNT_RCC_HIT);
+FME_EVENT_VTD_SIP(iotlb_4k_miss, VTD_SIP_EVNT_IOTLB_4K_MISS);
+FME_EVENT_VTD_SIP(iotlb_2m_miss, VTD_SIP_EVNT_IOTLB_2M_MISS);
+FME_EVENT_VTD_SIP(iotlb_1g_miss, VTD_SIP_EVNT_IOTLB_1G_MISS);
+FME_EVENT_VTD_SIP(slpwc_l3_miss, VTD_SIP_EVNT_SLPWC_L3_MISS);
+FME_EVENT_VTD_SIP(slpwc_l4_miss, VTD_SIP_EVNT_SLPWC_L4_MISS);
+FME_EVENT_VTD_SIP(rcc_miss,      VTD_SIP_EVNT_RCC_MISS);
+
+static struct attribute *fme_perf_vtd_sip_events_attrs[] = {
+	&fme_perf_event_vtd_sip_iotlb_4k_hit.attr.attr,
+	&fme_perf_event_vtd_sip_iotlb_2m_hit.attr.attr,
+	&fme_perf_event_vtd_sip_iotlb_1g_hit.attr.attr,
+	&fme_perf_event_vtd_sip_slpwc_l3_hit.attr.attr,
+	&fme_perf_event_vtd_sip_slpwc_l4_hit.attr.attr,
+	&fme_perf_event_vtd_sip_rcc_hit.attr.attr,
+	&fme_perf_event_vtd_sip_iotlb_4k_miss.attr.attr,
+	&fme_perf_event_vtd_sip_iotlb_2m_miss.attr.attr,
+	&fme_perf_event_vtd_sip_iotlb_1g_miss.attr.attr,
+	&fme_perf_event_vtd_sip_slpwc_l3_miss.attr.attr,
+	&fme_perf_event_vtd_sip_slpwc_l4_miss.attr.attr,
+	&fme_perf_event_vtd_sip_rcc_miss.attr.attr,
+	NULL,
+};
+
+static const struct attribute_group fme_perf_vtd_sip_events_group = {
+	.name = "events",
+	.attrs = fme_perf_vtd_sip_events_attrs,
+	.is_visible = fme_perf_events_visible,
+};
+
+static const struct attribute_group *fme_perf_events_groups[] = {
+	&fme_perf_basic_events_group,
+	&fme_perf_cache_events_group,
+	&fme_perf_fabric_events_group,
+	&fme_perf_vtd_events_group,
+	&fme_perf_vtd_sip_events_group,
+	NULL,
+};
+
+static struct fme_perf_event_ops *get_event_ops(u32 evtype)
+{
+	if (evtype > FME_EVTYPE_MAX)
+		return NULL;
+
+	return &fme_perf_event_ops[evtype];
+}
+
+static void fme_perf_event_destroy(struct perf_event *event)
+{
+	struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
+	struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+
+	if (ops->event_destroy)
+		ops->event_destroy(priv, event->hw.idx, event->hw.config_base);
+}
+
+static int fme_perf_event_init(struct perf_event *event)
+{
+	struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	struct fme_perf_event_ops *ops;
+	u32 eventid, evtype, portid;
+
+	/* test the event attr type check for PMU enumeration */
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	/*
+	 * fme counters are shared across all cores.
+	 * Therefore, it does not support per-process mode.
+	 * Also, it does not support event sampling mode.
+	 */
+	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+		return -EINVAL;
+
+	if (event->cpu < 0)
+		return -EINVAL;
+
+	if (event->cpu != priv->cpu)
+		return -EINVAL;
+
+	eventid = get_event(event->attr.config);
+	portid = get_portid(event->attr.config);
+	evtype = get_evtype(event->attr.config);
+	if (evtype > FME_EVTYPE_MAX)
+		return -EINVAL;
+
+	hwc->event_base = evtype;
+	hwc->idx = (int)eventid;
+	hwc->config_base = portid;
+
+	event->destroy = fme_perf_event_destroy;
+
+	dev_dbg(priv->dev, "%s event=0x%x, evtype=0x%x, portid=0x%x,\n",
+		__func__, eventid, evtype, portid);
+
+	ops = get_event_ops(evtype);
+	if (ops->event_init)
+		return ops->event_init(priv, eventid, portid);
+
+	return 0;
+}
+
+static void fme_perf_event_update(struct perf_event *event)
+{
+	struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
+	struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	u64 now, prev, delta;
+
+	now = ops->read_counter(priv, (u32)hwc->idx, hwc->config_base);
+	prev = local64_read(&hwc->prev_count);
+	delta = now - prev;
+
+	local64_add(delta, &event->count);
+}
+
+static void fme_perf_event_start(struct perf_event *event, int flags)
+{
+	struct fme_perf_event_ops *ops = get_event_ops(event->hw.event_base);
+	struct fme_perf_priv *priv = to_fme_perf_priv(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	u64 count;
+
+	count = ops->read_counter(priv, (u32)hwc->idx, hwc->config_base);
+	local64_set(&hwc->prev_count, count);
+}
+
+static void fme_perf_event_stop(struct perf_event *event, int flags)
+{
+	fme_perf_event_update(event);
+}
+
+static int fme_perf_event_add(struct perf_event *event, int flags)
+{
+	if (flags & PERF_EF_START)
+		fme_perf_event_start(event, flags);
+
+	return 0;
+}
+
+static void fme_perf_event_del(struct perf_event *event, int flags)
+{
+	fme_perf_event_stop(event, PERF_EF_UPDATE);
+}
+
+static void fme_perf_event_read(struct perf_event *event)
+{
+	fme_perf_event_update(event);
+}
+
+static void fme_perf_setup_hardware(struct fme_perf_priv *priv)
+{
+	void __iomem *base = priv->ioaddr;
+	u64 v;
+
+	/* read and save current working mode for fabric counters */
+	v = readq(base + FAB_CTRL);
+
+	if (FIELD_GET(FAB_PORT_FILTER, v) == FAB_PORT_FILTER_DISABLE)
+		priv->fab_port_id = FME_PORTID_ROOT;
+	else
+		priv->fab_port_id = FIELD_GET(FAB_PORT_ID, v);
+}
+
+static int fme_perf_pmu_register(struct platform_device *pdev,
+				 struct fme_perf_priv *priv)
+{
+	struct pmu *pmu = &priv->pmu;
+	char *name;
+	int ret;
+
+	spin_lock_init(&priv->fab_lock);
+
+	fme_perf_setup_hardware(priv);
+
+	pmu->task_ctx_nr =	perf_invalid_context;
+	pmu->attr_groups =	fme_perf_groups;
+	pmu->attr_update =	fme_perf_events_groups;
+	pmu->event_init =	fme_perf_event_init;
+	pmu->add =		fme_perf_event_add;
+	pmu->del =		fme_perf_event_del;
+	pmu->start =		fme_perf_event_start;
+	pmu->stop =		fme_perf_event_stop;
+	pmu->read =		fme_perf_event_read;
+	pmu->capabilities =	PERF_PMU_CAP_NO_INTERRUPT |
+				PERF_PMU_CAP_NO_EXCLUDE;
+
+	name = devm_kasprintf(priv->dev, GFP_KERNEL, "dfl_fme%d", pdev->id);
+
+	ret = perf_pmu_register(pmu, name, -1);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void fme_perf_pmu_unregister(struct fme_perf_priv *priv)
+{
+	perf_pmu_unregister(&priv->pmu);
+}
+
+static int fme_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+	struct fme_perf_priv *priv;
+	int target;
+
+	priv = hlist_entry_safe(node, struct fme_perf_priv, node);
+
+	if (cpu != priv->cpu)
+		return 0;
+
+	target = cpumask_any_but(cpu_online_mask, cpu);
+	if (target >= nr_cpu_ids)
+		return 0;
+
+	priv->cpu = target;
+	return 0;
+}
+
+static int fme_perf_init(struct platform_device *pdev,
+			 struct dfl_feature *feature)
+{
+	struct fme_perf_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = &pdev->dev;
+	priv->ioaddr = feature->ioaddr;
+	priv->id = feature->id;
+	priv->cpu = raw_smp_processor_id();
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+				      "perf/fpga/dfl_fme:online",
+				      NULL, fme_perf_offline_cpu);
+	if (ret < 0)
+		return ret;
+
+	priv->cpuhp_state = ret;
+
+	/* Register the pmu instance for cpu hotplug */
+	ret = cpuhp_state_add_instance_nocalls(priv->cpuhp_state, &priv->node);
+	if (ret)
+		goto cpuhp_instance_err;
+
+	ret = fme_perf_pmu_register(pdev, priv);
+	if (ret)
+		goto pmu_register_err;
+
+	feature->priv = priv;
+	return 0;
+
+pmu_register_err:
+	cpuhp_state_remove_instance_nocalls(priv->cpuhp_state, &priv->node);
+cpuhp_instance_err:
+	cpuhp_remove_multi_state(priv->cpuhp_state);
+	return ret;
+}
+
+static void fme_perf_uinit(struct platform_device *pdev,
+			   struct dfl_feature *feature)
+{
+	struct fme_perf_priv *priv = feature->priv;
+
+	fme_perf_pmu_unregister(priv);
+	cpuhp_state_remove_instance_nocalls(priv->cpuhp_state, &priv->node);
+	cpuhp_remove_multi_state(priv->cpuhp_state);
+}
+
+const struct dfl_feature_id fme_perf_id_table[] = {
+	{.id = FME_FEATURE_ID_GLOBAL_IPERF,},
+	{.id = FME_FEATURE_ID_GLOBAL_DPERF,},
+	{0,}
+};
+
+const struct dfl_feature_ops fme_perf_ops = {
+	.init = fme_perf_init,
+	.uinit = fme_perf_uinit,
+};
diff --git a/drivers/fpga/dfl-fme.h b/drivers/fpga/dfl-fme.h
index 6685c8ef965b..4195dd68193e 100644
--- a/drivers/fpga/dfl-fme.h
+++ b/drivers/fpga/dfl-fme.h
@@ -38,5 +38,7 @@ extern const struct dfl_feature_id fme_pr_mgmt_id_table[];
 extern const struct dfl_feature_ops fme_global_err_ops;
 extern const struct dfl_feature_id fme_global_err_id_table[];
 extern const struct attribute_group fme_global_err_group;
+extern const struct dfl_feature_ops fme_perf_ops;
+extern const struct dfl_feature_id fme_perf_id_table[];
 
 #endif /* __DFL_FME_H */
diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c
index 96a2b8274a33..990994874bf1 100644
--- a/drivers/fpga/dfl.c
+++ b/drivers/fpga/dfl.c
@@ -1079,6 +1079,7 @@ static int __init dfl_fpga_init(void)
  */
 int dfl_fpga_cdev_release_port(struct dfl_fpga_cdev *cdev, int port_id)
 {
+	struct dfl_feature_platform_data *pdata;
 	struct platform_device *port_pdev;
 	int ret = -ENODEV;
 
@@ -1093,7 +1094,11 @@ int dfl_fpga_cdev_release_port(struct dfl_fpga_cdev *cdev, int port_id)
 		goto put_dev_exit;
 	}
 
-	ret = dfl_feature_dev_use_begin(dev_get_platdata(&port_pdev->dev));
+	pdata = dev_get_platdata(&port_pdev->dev);
+
+	mutex_lock(&pdata->lock);
+	ret = dfl_feature_dev_use_begin(pdata, true);
+	mutex_unlock(&pdata->lock);
 	if (ret)
 		goto put_dev_exit;
 
@@ -1120,6 +1125,7 @@ EXPORT_SYMBOL_GPL(dfl_fpga_cdev_release_port);
  */
 int dfl_fpga_cdev_assign_port(struct dfl_fpga_cdev *cdev, int port_id)
 {
+	struct dfl_feature_platform_data *pdata;
 	struct platform_device *port_pdev;
 	int ret = -ENODEV;
 
@@ -1138,7 +1144,12 @@ int dfl_fpga_cdev_assign_port(struct dfl_fpga_cdev *cdev, int port_id)
 	if (ret)
 		goto put_dev_exit;
 
-	dfl_feature_dev_use_end(dev_get_platdata(&port_pdev->dev));
+	pdata = dev_get_platdata(&port_pdev->dev);
+
+	mutex_lock(&pdata->lock);
+	dfl_feature_dev_use_end(pdata);
+	mutex_unlock(&pdata->lock);
+
 	cdev->released_port_num--;
 put_dev_exit:
 	put_device(&port_pdev->dev);
diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h
index 9f0e656de720..2f5d3052e36e 100644
--- a/drivers/fpga/dfl.h
+++ b/drivers/fpga/dfl.h
@@ -197,16 +197,16 @@ struct dfl_feature_driver {
  *		    feature dev (platform device)'s reources.
  * @ioaddr: mapped mmio resource address.
  * @ops: ops of this sub feature.
+ * @priv: priv data of this feature.
  */
 struct dfl_feature {
 	u64 id;
 	int resource_index;
 	void __iomem *ioaddr;
 	const struct dfl_feature_ops *ops;
+	void *priv;
 };
 
-#define DEV_STATUS_IN_USE	0
-
 #define FEATURE_DEV_ID_UNUSED	(-1)
 
 /**
@@ -219,8 +219,9 @@ struct dfl_feature {
  * @dfl_cdev: ptr to container device.
  * @id: id used for this feature device.
  * @disable_count: count for port disable.
+ * @excl_open: set on feature device exclusive open.
+ * @open_count: count for feature device open.
  * @num: number for sub features.
- * @dev_status: dev status (e.g. DEV_STATUS_IN_USE).
  * @private: ptr to feature dev private data.
  * @features: sub features of this feature dev.
  */
@@ -232,26 +233,46 @@ struct dfl_feature_platform_data {
 	struct dfl_fpga_cdev *dfl_cdev;
 	int id;
 	unsigned int disable_count;
-	unsigned long dev_status;
+	bool excl_open;
+	int open_count;
 	void *private;
 	int num;
-	struct dfl_feature features[0];
+	struct dfl_feature features[];
 };
 
 static inline
-int dfl_feature_dev_use_begin(struct dfl_feature_platform_data *pdata)
+int dfl_feature_dev_use_begin(struct dfl_feature_platform_data *pdata,
+			      bool excl)
 {
-	/* Test and set IN_USE flags to ensure file is exclusively used */
-	if (test_and_set_bit_lock(DEV_STATUS_IN_USE, &pdata->dev_status))
+	if (pdata->excl_open)
 		return -EBUSY;
 
+	if (excl) {
+		if (pdata->open_count)
+			return -EBUSY;
+
+		pdata->excl_open = true;
+	}
+	pdata->open_count++;
+
 	return 0;
 }
 
 static inline
 void dfl_feature_dev_use_end(struct dfl_feature_platform_data *pdata)
 {
-	clear_bit_unlock(DEV_STATUS_IN_USE, &pdata->dev_status);
+	pdata->excl_open = false;
+
+	if (WARN_ON(pdata->open_count <= 0))
+		return;
+
+	pdata->open_count--;
+}
+
+static inline
+int dfl_feature_dev_use_count(struct dfl_feature_platform_data *pdata)
+{
+	return pdata->open_count;
 }
 
 static inline
diff --git a/drivers/fpga/ice40-spi.c b/drivers/fpga/ice40-spi.c
index 56e112e14a10..8d689fea0dab 100644
--- a/drivers/fpga/ice40-spi.c
+++ b/drivers/fpga/ice40-spi.c
@@ -46,10 +46,16 @@ static int ice40_fpga_ops_write_init(struct fpga_manager *mgr,
 	struct spi_message message;
 	struct spi_transfer assert_cs_then_reset_delay = {
 		.cs_change   = 1,
-		.delay_usecs = ICE40_SPI_RESET_DELAY
+		.delay = {
+			.value = ICE40_SPI_RESET_DELAY,
+			.unit = SPI_DELAY_UNIT_USECS
+		}
 	};
 	struct spi_transfer housekeeping_delay_then_release_cs = {
-		.delay_usecs = ICE40_SPI_HOUSEKEEPING_DELAY
+		.delay = {
+			.value = ICE40_SPI_HOUSEKEEPING_DELAY,
+			.unit = SPI_DELAY_UNIT_USECS
+		}
 	};
 	int ret;
 
diff --git a/drivers/fpga/machxo2-spi.c b/drivers/fpga/machxo2-spi.c
index 4d8a87641587..b316369156fe 100644
--- a/drivers/fpga/machxo2-spi.c
+++ b/drivers/fpga/machxo2-spi.c
@@ -157,7 +157,8 @@ static int machxo2_cleanup(struct fpga_manager *mgr)
 	spi_message_init(&msg);
 	tx[1].tx_buf = &refresh;
 	tx[1].len = sizeof(refresh);
-	tx[1].delay_usecs = MACHXO2_REFRESH_USEC;
+	tx[1].delay.value = MACHXO2_REFRESH_USEC;
+	tx[1].delay.unit = SPI_DELAY_UNIT_USECS;
 	spi_message_add_tail(&tx[1], &msg);
 	ret = spi_sync(spi, &msg);
 	if (ret)
@@ -208,7 +209,8 @@ static int machxo2_write_init(struct fpga_manager *mgr,
 	spi_message_init(&msg);
 	tx[0].tx_buf = &enable;
 	tx[0].len = sizeof(enable);
-	tx[0].delay_usecs = MACHXO2_LOW_DELAY_USEC;
+	tx[0].delay.value = MACHXO2_LOW_DELAY_USEC;
+	tx[0].delay.unit = SPI_DELAY_UNIT_USECS;
 	spi_message_add_tail(&tx[0], &msg);
 
 	tx[1].tx_buf = &erase;
@@ -269,7 +271,8 @@ static int machxo2_write(struct fpga_manager *mgr, const char *buf,
 		spi_message_init(&msg);
 		tx.tx_buf = payload;
 		tx.len = MACHXO2_BUF_SIZE;
-		tx.delay_usecs = MACHXO2_HIGH_DELAY_USEC;
+		tx.delay.value = MACHXO2_HIGH_DELAY_USEC;
+		tx.delay.unit = SPI_DELAY_UNIT_USECS;
 		spi_message_add_tail(&tx, &msg);
 		ret = spi_sync(spi, &msg);
 		if (ret) {
@@ -317,7 +320,8 @@ static int machxo2_write_complete(struct fpga_manager *mgr,
 		spi_message_init(&msg);
 		tx[1].tx_buf = &refresh;
 		tx[1].len = sizeof(refresh);
-		tx[1].delay_usecs = MACHXO2_REFRESH_USEC;
+		tx[1].delay.value = MACHXO2_REFRESH_USEC;
+		tx[1].delay.unit = SPI_DELAY_UNIT_USECS;
 		spi_message_add_tail(&tx[1], &msg);
 		ret = spi_sync(spi, &msg);
 		if (ret)
diff --git a/drivers/fpga/stratix10-soc.c b/drivers/fpga/stratix10-soc.c
index 215d33789c74..44b7c569d4dc 100644
--- a/drivers/fpga/stratix10-soc.c
+++ b/drivers/fpga/stratix10-soc.c
@@ -154,11 +154,11 @@ static void s10_receive_callback(struct stratix10_svc_client *client,
 	 * Here we set status bits as we receive them.  Elsewhere, we always use
 	 * test_and_clear_bit() to check status in priv->status
 	 */
-	for (i = 0; i <= SVC_STATUS_RECONFIG_ERROR; i++)
+	for (i = 0; i <= SVC_STATUS_ERROR; i++)
 		if (status & (1 << i))
 			set_bit(i, &priv->status);
 
-	if (status & BIT(SVC_STATUS_RECONFIG_BUFFER_DONE)) {
+	if (status & BIT(SVC_STATUS_BUFFER_DONE)) {
 		s10_unlock_bufs(priv, data->kaddr1);
 		s10_unlock_bufs(priv, data->kaddr2);
 		s10_unlock_bufs(priv, data->kaddr3);
@@ -209,8 +209,7 @@ static int s10_ops_write_init(struct fpga_manager *mgr,
 	}
 
 	ret = 0;
-	if (!test_and_clear_bit(SVC_STATUS_RECONFIG_REQUEST_OK,
-				&priv->status)) {
+	if (!test_and_clear_bit(SVC_STATUS_OK, &priv->status)) {
 		ret = -ETIMEDOUT;
 		goto init_done;
 	}
@@ -323,17 +322,15 @@ static int s10_ops_write(struct fpga_manager *mgr, const char *buf,
 				&priv->status_return_completion,
 				S10_BUFFER_TIMEOUT);
 
-		if (test_and_clear_bit(SVC_STATUS_RECONFIG_BUFFER_DONE,
-				       &priv->status) ||
-		    test_and_clear_bit(SVC_STATUS_RECONFIG_BUFFER_SUBMITTED,
+		if (test_and_clear_bit(SVC_STATUS_BUFFER_DONE, &priv->status) ||
+		    test_and_clear_bit(SVC_STATUS_BUFFER_SUBMITTED,
 				       &priv->status)) {
 			ret = 0;
 			continue;
 		}
 
-		if (test_and_clear_bit(SVC_STATUS_RECONFIG_ERROR,
-				       &priv->status)) {
-			dev_err(dev, "ERROR - giving up - SVC_STATUS_RECONFIG_ERROR\n");
+		if (test_and_clear_bit(SVC_STATUS_ERROR, &priv->status)) {
+			dev_err(dev, "ERROR - giving up - SVC_STATUS_ERROR\n");
 			ret = -EFAULT;
 			break;
 		}
@@ -393,13 +390,11 @@ static int s10_ops_write_complete(struct fpga_manager *mgr,
 		timeout = ret;
 		ret = 0;
 
-		if (test_and_clear_bit(SVC_STATUS_RECONFIG_COMPLETED,
-				       &priv->status))
+		if (test_and_clear_bit(SVC_STATUS_COMPLETED, &priv->status))
 			break;
 
-		if (test_and_clear_bit(SVC_STATUS_RECONFIG_ERROR,
-				       &priv->status)) {
-			dev_err(dev, "ERROR - giving up - SVC_STATUS_RECONFIG_ERROR\n");
+		if (test_and_clear_bit(SVC_STATUS_ERROR, &priv->status)) {
+			dev_err(dev, "ERROR - giving up - SVC_STATUS_ERROR\n");
 			ret = -EFAULT;
 			break;
 		}
@@ -482,7 +477,8 @@ static int s10_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id s10_of_match[] = {
-	{ .compatible = "intel,stratix10-soc-fpga-mgr", },
+	{.compatible = "intel,stratix10-soc-fpga-mgr"},
+	{.compatible = "intel,agilex-soc-fpga-mgr"},
 	{},
 };
 
diff --git a/drivers/fpga/zynqmp-fpga.c b/drivers/fpga/zynqmp-fpga.c
index b8a88d21d038..4a1139e05280 100644
--- a/drivers/fpga/zynqmp-fpga.c
+++ b/drivers/fpga/zynqmp-fpga.c
@@ -40,16 +40,12 @@ static int zynqmp_fpga_ops_write_init(struct fpga_manager *mgr,
 static int zynqmp_fpga_ops_write(struct fpga_manager *mgr,
 				 const char *buf, size_t size)
 {
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
 	struct zynqmp_fpga_priv *priv;
 	dma_addr_t dma_addr;
 	u32 eemi_flags = 0;
 	char *kbuf;
 	int ret;
 
-	if (IS_ERR_OR_NULL(eemi_ops) || !eemi_ops->fpga_load)
-		return -ENXIO;
-
 	priv = mgr->priv;
 
 	kbuf = dma_alloc_coherent(priv->dev, size, &dma_addr, GFP_KERNEL);
@@ -63,7 +59,7 @@ static int zynqmp_fpga_ops_write(struct fpga_manager *mgr,
 	if (priv->flags & FPGA_MGR_PARTIAL_RECONFIG)
 		eemi_flags |= XILINX_ZYNQMP_PM_FPGA_PARTIAL;
 
-	ret = eemi_ops->fpga_load(dma_addr, size, eemi_flags);
+	ret = zynqmp_pm_fpga_load(dma_addr, size, eemi_flags);
 
 	dma_free_coherent(priv->dev, size, kbuf, dma_addr);
 
@@ -78,13 +74,9 @@ static int zynqmp_fpga_ops_write_complete(struct fpga_manager *mgr,
 
 static enum fpga_mgr_states zynqmp_fpga_ops_state(struct fpga_manager *mgr)
 {
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_pm_get_eemi_ops();
-	u32 status;
-
-	if (IS_ERR_OR_NULL(eemi_ops) || !eemi_ops->fpga_get_status)
-		return FPGA_MGR_STATE_UNKNOWN;
+	u32 status = 0;
 
-	eemi_ops->fpga_get_status(&status);
+	zynqmp_pm_fpga_get_status(&status);
 	if (status & IXR_FPGA_DONE_MASK)
 		return FPGA_MGR_STATE_OPERATING;
 
diff --git a/drivers/fsi/Kconfig b/drivers/fsi/Kconfig
index 4cc0e630ab79..e6668a869913 100644
--- a/drivers/fsi/Kconfig
+++ b/drivers/fsi/Kconfig
@@ -7,7 +7,7 @@ menuconfig FSI
 	tristate "FSI support"
 	depends on OF
 	select CRC4
-	---help---
+	help
 	  FSI - the FRU Support Interface - is a simple bus for low-level
 	  access to POWER-based hardware.
 
@@ -16,7 +16,7 @@ if FSI
 config FSI_NEW_DEV_NODE
 	bool "Create '/dev/fsi' directory for char devices"
 	default n
-	---help---
+	help
 	This option causes char devices created for FSI devices to be
 	located under a common /dev/fsi/ directory. Set to N unless your
 	userspace has been updated to handle the new location.
@@ -32,12 +32,12 @@ config FSI_MASTER_GPIO
 	tristate "GPIO-based FSI master"
 	depends on GPIOLIB
 	select CRC4
-	---help---
+	help
 	This option enables a FSI master driver using GPIO lines.
 
 config FSI_MASTER_HUB
 	tristate "FSI hub master"
-	---help---
+	help
 	This option enables a FSI hub master driver.  Hub is a type of FSI
 	master that is connected to the upstream master via a slave.  Hubs
 	allow chaining of FSI links to an arbitrary depth.  This allows for
@@ -48,7 +48,7 @@ config FSI_MASTER_AST_CF
 	depends on GPIOLIB
 	depends on GPIO_ASPEED
 	select GENERIC_ALLOCATOR
-	---help---
+	help
 	This option enables a FSI master using the AST2400 and AST2500 GPIO
 	lines driven by the internal ColdFire coprocessor. This requires
 	the corresponding machine specific ColdFire firmware to be available.
@@ -64,13 +64,13 @@ config FSI_MASTER_ASPEED
 
 config FSI_SCOM
 	tristate "SCOM FSI client device driver"
-	---help---
+	help
 	This option enables an FSI based SCOM device driver.
 
 config FSI_SBEFIFO
 	tristate "SBEFIFO FSI client device driver"
 	depends on OF_ADDRESS
-	---help---
+	help
 	This option enables an FSI based SBEFIFO device driver. The SBEFIFO is
 	a pipe-like FSI device for communicating with the self boot engine
 	(SBE) on POWER processors.
@@ -78,7 +78,7 @@ config FSI_SBEFIFO
 config FSI_OCC
 	tristate "OCC SBEFIFO client device driver"
 	depends on FSI_SBEFIFO
-	---help---
+	help
 	This option enables an SBEFIFO based On-Chip Controller (OCC) device
 	driver. The OCC is a device embedded on a POWER processor that collects
 	and aggregates sensor data from the processor and system. The OCC can
diff --git a/drivers/gnss/Kconfig b/drivers/gnss/Kconfig
index a0404ce155a6..bd12e3d57baa 100644
--- a/drivers/gnss/Kconfig
+++ b/drivers/gnss/Kconfig
@@ -5,7 +5,7 @@
 
 menuconfig GNSS
 	tristate "GNSS receiver support"
-	---help---
+	help
 	  Say Y here if you have a GNSS receiver (e.g. a GPS receiver).
 
 	  To compile this driver as a module, choose M here: the module will
@@ -32,7 +32,7 @@ config GNSS_MTK_SERIAL
 config GNSS_SIRF_SERIAL
 	tristate "SiRFstar GNSS receiver support"
 	depends on SERIAL_DEV_BUS
-	---help---
+	help
 	  Say Y here if you have a SiRFstar-based GNSS receiver which uses a
 	  serial interface.
 
@@ -45,7 +45,7 @@ config GNSS_UBX_SERIAL
 	tristate "u-blox GNSS receiver support"
 	depends on SERIAL_DEV_BUS
 	select GNSS_SERIAL
-	---help---
+	help
 	  Say Y here if you have a u-blox GNSS receiver which uses a serial
 	  interface.
 
diff --git a/drivers/gnss/serial.h b/drivers/gnss/serial.h
index 980ffdc86c2a..621953f7821d 100644
--- a/drivers/gnss/serial.h
+++ b/drivers/gnss/serial.h
@@ -16,7 +16,7 @@ struct gnss_serial {
 	struct gnss_device *gdev;
 	speed_t	speed;
 	const struct gnss_serial_ops *ops;
-	unsigned long drvdata[0];
+	unsigned long drvdata[];
 };
 
 enum gnss_serial_pm_state {
diff --git a/drivers/gnss/sirf.c b/drivers/gnss/sirf.c
index effed3a8d398..2ecb1d3e8eeb 100644
--- a/drivers/gnss/sirf.c
+++ b/drivers/gnss/sirf.c
@@ -439,14 +439,18 @@ static int sirf_probe(struct serdev_device *serdev)
 
 	data->on_off = devm_gpiod_get_optional(dev, "sirf,onoff",
 			GPIOD_OUT_LOW);
-	if (IS_ERR(data->on_off))
+	if (IS_ERR(data->on_off)) {
+		ret = PTR_ERR(data->on_off);
 		goto err_put_device;
+	}
 
 	if (data->on_off) {
 		data->wakeup = devm_gpiod_get_optional(dev, "sirf,wakeup",
 				GPIOD_IN);
-		if (IS_ERR(data->wakeup))
+		if (IS_ERR(data->wakeup)) {
+			ret = PTR_ERR(data->wakeup);
 			goto err_put_device;
+		}
 
 		ret = regulator_enable(data->vcc);
 		if (ret)
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index 1b96169d84f7..c6b5c65c8405 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -73,6 +73,10 @@ config GPIO_GENERIC
 	depends on HAS_IOMEM # Only for IOMEM drivers
 	tristate
 
+config GPIO_REGMAP
+	depends on REGMAP
+	tristate
+
 # put drivers in the right section, in alphabetical order
 
 # This symbol is selected by both I2C and SPI expanders
@@ -422,7 +426,7 @@ config GPIO_OMAP
 	  Say yes here to enable GPIO support for TI OMAP SoCs.
 
 config GPIO_PL061
-	bool "PrimeCell PL061 GPIO support"
+	tristate "PrimeCell PL061 GPIO support"
 	depends on ARM_AMBA
 	select IRQ_DOMAIN
 	select GPIOLIB_IRQCHIP
@@ -439,7 +443,7 @@ config GPIO_PMIC_EIC_SPRD
 
 config GPIO_PXA
 	bool "PXA GPIO support"
-	depends on ARCH_PXA || ARCH_MMP
+	depends on ARCH_PXA || ARCH_MMP || COMPILE_TEST
 	help
 	  Say yes here to support the PXA GPIO device
 
@@ -638,7 +642,7 @@ config GPIO_XGENE
 
 config GPIO_XGENE_SB
 	tristate "APM X-Gene GPIO standby controller support"
-	depends on ARCH_XGENE && OF_GPIO
+	depends on (ARCH_XGENE || COMPILE_TEST)
 	select GPIO_GENERIC
 	select GPIOLIB_IRQCHIP
 	select IRQ_DOMAIN_HIERARCHY
@@ -952,7 +956,7 @@ config GPIO_PCA953X
 
 config GPIO_PCA953X_IRQ
 	bool "Interrupt controller support for PCA953x"
-	depends on GPIO_PCA953X=y
+	depends on GPIO_PCA953X
 	select GPIOLIB_IRQCHIP
 	help
 	  Say yes here to enable the pca953x to be used as an interrupt
@@ -1232,7 +1236,7 @@ config GPIO_TC3589X
 config GPIO_TIMBERDALE
 	bool "Support for timberdale GPIO IP"
 	depends on MFD_TIMBERDALE
-	---help---
+	help
 	Add support for the GPIO IP in the timberdale FPGA.
 
 config GPIO_TPS65086
@@ -1541,6 +1545,18 @@ config GPIO_VIPERBOARD
 
 endmenu
 
+config GPIO_AGGREGATOR
+	tristate "GPIO Aggregator"
+	help
+	  Say yes here to enable the GPIO Aggregator, which provides a way to
+	  aggregate existing GPIO lines into a new virtual GPIO chip.
+	  This can serve the following purposes:
+	    - Assign permissions for a collection of GPIO lines to a user,
+	    - Export a collection of GPIO lines to a virtual machine,
+	    - Provide a generic driver for a GPIO-operated device in an
+	      industrial control context, to be operated from userspace using
+	      the GPIO chardev interface.
+
 config GPIO_MOCKUP
 	tristate "GPIO Testing Driver"
 	select IRQ_SIM
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index b2cfc21a97f3..1e4894e0bf0f 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_GPIO_SYSFS)	+= gpiolib-sysfs.o
 obj-$(CONFIG_GPIO_ACPI)		+= gpiolib-acpi.o
 
 # Device drivers. Generally keep list sorted alphabetically
+obj-$(CONFIG_GPIO_REGMAP)	+= gpio-regmap.o
 obj-$(CONFIG_GPIO_GENERIC)	+= gpio-generic.o
 
 # directly supported by gpio-generic
@@ -25,6 +26,7 @@ obj-$(CONFIG_GPIO_74XX_MMIO)		+= gpio-74xx-mmio.o
 obj-$(CONFIG_GPIO_ADNP)			+= gpio-adnp.o
 obj-$(CONFIG_GPIO_ADP5520)		+= gpio-adp5520.o
 obj-$(CONFIG_GPIO_ADP5588)		+= gpio-adp5588.o
+obj-$(CONFIG_GPIO_AGGREGATOR)		+= gpio-aggregator.o
 obj-$(CONFIG_GPIO_ALTERA_A10SR)		+= gpio-altera-a10sr.o
 obj-$(CONFIG_GPIO_ALTERA)  		+= gpio-altera.o
 obj-$(CONFIG_GPIO_AMD8111)		+= gpio-amd8111.o
diff --git a/drivers/gpio/TODO b/drivers/gpio/TODO
index 3a44e6ae52bd..b989c9352da2 100644
--- a/drivers/gpio/TODO
+++ b/drivers/gpio/TODO
@@ -99,6 +99,10 @@ similar and probe a proper driver in the gpiolib subsystem.
 In some cases it makes sense to create a GPIO chip from the local driver
 for a few GPIOs. Those should stay where they are.
 
+At the same time it makes sense to get rid of code duplication in existing or
+new coming drivers. For example, gpio-ml-ioh should be incorporated into
+gpio-pch. In similar way gpio-intel-mid into gpio-pxa.
+
 
 Generic MMIO GPIO
 
diff --git a/drivers/gpio/gpio-aggregator.c b/drivers/gpio/gpio-aggregator.c
new file mode 100644
index 000000000000..9b0adbdddbfc
--- /dev/null
+++ b/drivers/gpio/gpio-aggregator.c
@@ -0,0 +1,568 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// GPIO Aggregator
+//
+// Copyright (C) 2019-2020 Glider bv
+
+#define DRV_NAME       "gpio-aggregator"
+#define pr_fmt(fmt)	DRV_NAME ": " fmt
+
+#include <linux/bitmap.h>
+#include <linux/bitops.h>
+#include <linux/ctype.h>
+#include <linux/gpio/consumer.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/machine.h>
+#include <linux/idr.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/overflow.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+
+
+/*
+ * GPIO Aggregator sysfs interface
+ */
+
+struct gpio_aggregator {
+	struct gpiod_lookup_table *lookups;
+	struct platform_device *pdev;
+	char args[];
+};
+
+static DEFINE_MUTEX(gpio_aggregator_lock);	/* protects idr */
+static DEFINE_IDR(gpio_aggregator_idr);
+
+static char *get_arg(char **args)
+{
+	char *start = *args, *end;
+
+	start = skip_spaces(start);
+	if (!*start)
+		return NULL;
+
+	if (*start == '"') {
+		/* Quoted arg */
+		end = strchr(++start, '"');
+		if (!end)
+			return ERR_PTR(-EINVAL);
+	} else {
+		/* Unquoted arg */
+		for (end = start; *end && !isspace(*end); end++) ;
+	}
+
+	if (*end)
+		*end++ = '\0';
+
+	*args = end;
+	return start;
+}
+
+static bool isrange(const char *s)
+{
+	size_t n;
+
+	if (IS_ERR_OR_NULL(s))
+		return false;
+
+	while (1) {
+		n = strspn(s, "0123456789");
+		if (!n)
+			return false;
+
+		s += n;
+
+		switch (*s++) {
+		case '\0':
+			return true;
+
+		case '-':
+		case ',':
+			break;
+
+		default:
+			return false;
+		}
+	}
+}
+
+static int aggr_add_gpio(struct gpio_aggregator *aggr, const char *key,
+			 int hwnum, unsigned int *n)
+{
+	struct gpiod_lookup_table *lookups;
+
+	lookups = krealloc(aggr->lookups, struct_size(lookups, table, *n + 2),
+			   GFP_KERNEL);
+	if (!lookups)
+		return -ENOMEM;
+
+	lookups->table[*n] =
+		(struct gpiod_lookup)GPIO_LOOKUP_IDX(key, hwnum, NULL, *n, 0);
+
+	(*n)++;
+	memset(&lookups->table[*n], 0, sizeof(lookups->table[*n]));
+
+	aggr->lookups = lookups;
+	return 0;
+}
+
+static int aggr_parse(struct gpio_aggregator *aggr)
+{
+	unsigned int first_index, last_index, i, n = 0;
+	char *name, *offsets, *first, *last, *next;
+	char *args = aggr->args;
+	int error;
+
+	for (name = get_arg(&args), offsets = get_arg(&args); name;
+	     offsets = get_arg(&args)) {
+		if (IS_ERR(name)) {
+			pr_err("Cannot get GPIO specifier: %pe\n", name);
+			return PTR_ERR(name);
+		}
+
+		if (!isrange(offsets)) {
+			/* Named GPIO line */
+			error = aggr_add_gpio(aggr, name, U16_MAX, &n);
+			if (error)
+				return error;
+
+			name = offsets;
+			continue;
+		}
+
+		/* GPIO chip + offset(s) */
+		for (first = offsets; *first; first = next) {
+			next = strchrnul(first, ',');
+			if (*next)
+				*next++ = '\0';
+
+			last = strchr(first, '-');
+			if (last)
+				*last++ = '\0';
+
+			if (kstrtouint(first, 10, &first_index)) {
+				pr_err("Cannot parse GPIO index %s\n", first);
+				return -EINVAL;
+			}
+
+			if (!last) {
+				last_index = first_index;
+			} else if (kstrtouint(last, 10, &last_index)) {
+				pr_err("Cannot parse GPIO index %s\n", last);
+				return -EINVAL;
+			}
+
+			for (i = first_index; i <= last_index; i++) {
+				error = aggr_add_gpio(aggr, name, i, &n);
+				if (error)
+					return error;
+			}
+		}
+
+		name = get_arg(&args);
+	}
+
+	if (!n) {
+		pr_err("No GPIOs specified\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static ssize_t new_device_store(struct device_driver *driver, const char *buf,
+				size_t count)
+{
+	struct gpio_aggregator *aggr;
+	struct platform_device *pdev;
+	int res, id;
+
+	/* kernfs guarantees string termination, so count + 1 is safe */
+	aggr = kzalloc(sizeof(*aggr) + count + 1, GFP_KERNEL);
+	if (!aggr)
+		return -ENOMEM;
+
+	memcpy(aggr->args, buf, count + 1);
+
+	aggr->lookups = kzalloc(struct_size(aggr->lookups, table, 1),
+				GFP_KERNEL);
+	if (!aggr->lookups) {
+		res = -ENOMEM;
+		goto free_ga;
+	}
+
+	mutex_lock(&gpio_aggregator_lock);
+	id = idr_alloc(&gpio_aggregator_idr, aggr, 0, 0, GFP_KERNEL);
+	mutex_unlock(&gpio_aggregator_lock);
+
+	if (id < 0) {
+		res = id;
+		goto free_table;
+	}
+
+	aggr->lookups->dev_id = kasprintf(GFP_KERNEL, "%s.%d", DRV_NAME, id);
+	if (!aggr->lookups->dev_id) {
+		res = -ENOMEM;
+		goto remove_idr;
+	}
+
+	res = aggr_parse(aggr);
+	if (res)
+		goto free_dev_id;
+
+	gpiod_add_lookup_table(aggr->lookups);
+
+	pdev = platform_device_register_simple(DRV_NAME, id, NULL, 0);
+	if (IS_ERR(pdev)) {
+		res = PTR_ERR(pdev);
+		goto remove_table;
+	}
+
+	aggr->pdev = pdev;
+	return count;
+
+remove_table:
+	gpiod_remove_lookup_table(aggr->lookups);
+free_dev_id:
+	kfree(aggr->lookups->dev_id);
+remove_idr:
+	mutex_lock(&gpio_aggregator_lock);
+	idr_remove(&gpio_aggregator_idr, id);
+	mutex_unlock(&gpio_aggregator_lock);
+free_table:
+	kfree(aggr->lookups);
+free_ga:
+	kfree(aggr);
+	return res;
+}
+
+static DRIVER_ATTR_WO(new_device);
+
+static void gpio_aggregator_free(struct gpio_aggregator *aggr)
+{
+	platform_device_unregister(aggr->pdev);
+	gpiod_remove_lookup_table(aggr->lookups);
+	kfree(aggr->lookups->dev_id);
+	kfree(aggr->lookups);
+	kfree(aggr);
+}
+
+static ssize_t delete_device_store(struct device_driver *driver,
+				   const char *buf, size_t count)
+{
+	struct gpio_aggregator *aggr;
+	unsigned int id;
+	int error;
+
+	if (!str_has_prefix(buf, DRV_NAME "."))
+		return -EINVAL;
+
+	error = kstrtouint(buf + strlen(DRV_NAME "."), 10, &id);
+	if (error)
+		return error;
+
+	mutex_lock(&gpio_aggregator_lock);
+	aggr = idr_remove(&gpio_aggregator_idr, id);
+	mutex_unlock(&gpio_aggregator_lock);
+	if (!aggr)
+		return -ENOENT;
+
+	gpio_aggregator_free(aggr);
+	return count;
+}
+static DRIVER_ATTR_WO(delete_device);
+
+static struct attribute *gpio_aggregator_attrs[] = {
+	&driver_attr_new_device.attr,
+	&driver_attr_delete_device.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(gpio_aggregator);
+
+static int __exit gpio_aggregator_idr_remove(int id, void *p, void *data)
+{
+	gpio_aggregator_free(p);
+	return 0;
+}
+
+static void __exit gpio_aggregator_remove_all(void)
+{
+	mutex_lock(&gpio_aggregator_lock);
+	idr_for_each(&gpio_aggregator_idr, gpio_aggregator_idr_remove, NULL);
+	idr_destroy(&gpio_aggregator_idr);
+	mutex_unlock(&gpio_aggregator_lock);
+}
+
+
+/*
+ *  GPIO Forwarder
+ */
+
+struct gpiochip_fwd {
+	struct gpio_chip chip;
+	struct gpio_desc **descs;
+	union {
+		struct mutex mlock;	/* protects tmp[] if can_sleep */
+		spinlock_t slock;	/* protects tmp[] if !can_sleep */
+	};
+	unsigned long tmp[];		/* values and descs for multiple ops */
+};
+
+static int gpio_fwd_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+	return gpiod_get_direction(fwd->descs[offset]);
+}
+
+static int gpio_fwd_direction_input(struct gpio_chip *chip, unsigned int offset)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+	return gpiod_direction_input(fwd->descs[offset]);
+}
+
+static int gpio_fwd_direction_output(struct gpio_chip *chip,
+				     unsigned int offset, int value)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+	return gpiod_direction_output(fwd->descs[offset], value);
+}
+
+static int gpio_fwd_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+	return gpiod_get_value(fwd->descs[offset]);
+}
+
+static int gpio_fwd_get_multiple(struct gpio_chip *chip, unsigned long *mask,
+				 unsigned long *bits)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+	unsigned long *values, flags = 0;
+	struct gpio_desc **descs;
+	unsigned int i, j = 0;
+	int error;
+
+	if (chip->can_sleep)
+		mutex_lock(&fwd->mlock);
+	else
+		spin_lock_irqsave(&fwd->slock, flags);
+
+	/* Both values bitmap and desc pointers are stored in tmp[] */
+	values = &fwd->tmp[0];
+	descs = (void *)&fwd->tmp[BITS_TO_LONGS(fwd->chip.ngpio)];
+
+	bitmap_clear(values, 0, fwd->chip.ngpio);
+	for_each_set_bit(i, mask, fwd->chip.ngpio)
+		descs[j++] = fwd->descs[i];
+
+	error = gpiod_get_array_value(j, descs, NULL, values);
+	if (!error) {
+		j = 0;
+		for_each_set_bit(i, mask, fwd->chip.ngpio)
+			__assign_bit(i, bits, test_bit(j++, values));
+	}
+
+	if (chip->can_sleep)
+		mutex_unlock(&fwd->mlock);
+	else
+		spin_unlock_irqrestore(&fwd->slock, flags);
+
+	return error;
+}
+
+static void gpio_fwd_set(struct gpio_chip *chip, unsigned int offset, int value)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+	gpiod_set_value(fwd->descs[offset], value);
+}
+
+static void gpio_fwd_set_multiple(struct gpio_chip *chip, unsigned long *mask,
+				  unsigned long *bits)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+	unsigned long *values, flags = 0;
+	struct gpio_desc **descs;
+	unsigned int i, j = 0;
+
+	if (chip->can_sleep)
+		mutex_lock(&fwd->mlock);
+	else
+		spin_lock_irqsave(&fwd->slock, flags);
+
+	/* Both values bitmap and desc pointers are stored in tmp[] */
+	values = &fwd->tmp[0];
+	descs = (void *)&fwd->tmp[BITS_TO_LONGS(fwd->chip.ngpio)];
+
+	for_each_set_bit(i, mask, fwd->chip.ngpio) {
+		__assign_bit(j, values, test_bit(i, bits));
+		descs[j++] = fwd->descs[i];
+	}
+
+	gpiod_set_array_value(j, descs, NULL, values);
+
+	if (chip->can_sleep)
+		mutex_unlock(&fwd->mlock);
+	else
+		spin_unlock_irqrestore(&fwd->slock, flags);
+}
+
+static int gpio_fwd_set_config(struct gpio_chip *chip, unsigned int offset,
+			       unsigned long config)
+{
+	struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+
+	return gpiod_set_config(fwd->descs[offset], config);
+}
+
+/**
+ * gpiochip_fwd_create() - Create a new GPIO forwarder
+ * @dev: Parent device pointer
+ * @ngpios: Number of GPIOs in the forwarder.
+ * @descs: Array containing the GPIO descriptors to forward to.
+ *         This array must contain @ngpios entries, and must not be deallocated
+ *         before the forwarder has been destroyed again.
+ *
+ * This function creates a new gpiochip, which forwards all GPIO operations to
+ * the passed GPIO descriptors.
+ *
+ * Return: An opaque object pointer, or an ERR_PTR()-encoded negative error
+ *         code on failure.
+ */
+static struct gpiochip_fwd *gpiochip_fwd_create(struct device *dev,
+						unsigned int ngpios,
+						struct gpio_desc *descs[])
+{
+	const char *label = dev_name(dev);
+	struct gpiochip_fwd *fwd;
+	struct gpio_chip *chip;
+	unsigned int i;
+	int error;
+
+	fwd = devm_kzalloc(dev, struct_size(fwd, tmp,
+			   BITS_TO_LONGS(ngpios) + ngpios), GFP_KERNEL);
+	if (!fwd)
+		return ERR_PTR(-ENOMEM);
+
+	chip = &fwd->chip;
+
+	/*
+	 * If any of the GPIO lines are sleeping, then the entire forwarder
+	 * will be sleeping.
+	 * If any of the chips support .set_config(), then the forwarder will
+	 * support setting configs.
+	 */
+	for (i = 0; i < ngpios; i++) {
+		struct gpio_chip *parent = gpiod_to_chip(descs[i]);
+
+		dev_dbg(dev, "%u => gpio-%d\n", i, desc_to_gpio(descs[i]));
+
+		if (gpiod_cansleep(descs[i]))
+			chip->can_sleep = true;
+		if (parent && parent->set_config)
+			chip->set_config = gpio_fwd_set_config;
+	}
+
+	chip->label = label;
+	chip->parent = dev;
+	chip->owner = THIS_MODULE;
+	chip->get_direction = gpio_fwd_get_direction;
+	chip->direction_input = gpio_fwd_direction_input;
+	chip->direction_output = gpio_fwd_direction_output;
+	chip->get = gpio_fwd_get;
+	chip->get_multiple = gpio_fwd_get_multiple;
+	chip->set = gpio_fwd_set;
+	chip->set_multiple = gpio_fwd_set_multiple;
+	chip->base = -1;
+	chip->ngpio = ngpios;
+	fwd->descs = descs;
+
+	if (chip->can_sleep)
+		mutex_init(&fwd->mlock);
+	else
+		spin_lock_init(&fwd->slock);
+
+	error = devm_gpiochip_add_data(dev, chip, fwd);
+	if (error)
+		return ERR_PTR(error);
+
+	return fwd;
+}
+
+
+/*
+ *  GPIO Aggregator platform device
+ */
+
+static int gpio_aggregator_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct gpio_desc **descs;
+	struct gpiochip_fwd *fwd;
+	int i, n;
+
+	n = gpiod_count(dev, NULL);
+	if (n < 0)
+		return n;
+
+	descs = devm_kmalloc_array(dev, n, sizeof(*descs), GFP_KERNEL);
+	if (!descs)
+		return -ENOMEM;
+
+	for (i = 0; i < n; i++) {
+		descs[i] = devm_gpiod_get_index(dev, NULL, i, GPIOD_ASIS);
+		if (IS_ERR(descs[i]))
+			return PTR_ERR(descs[i]);
+	}
+
+	fwd = gpiochip_fwd_create(dev, n, descs);
+	if (IS_ERR(fwd))
+		return PTR_ERR(fwd);
+
+	platform_set_drvdata(pdev, fwd);
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id gpio_aggregator_dt_ids[] = {
+	/*
+	 * Add GPIO-operated devices controlled from userspace below,
+	 * or use "driver_override" in sysfs
+	 */
+	{},
+};
+MODULE_DEVICE_TABLE(of, gpio_aggregator_dt_ids);
+#endif
+
+static struct platform_driver gpio_aggregator_driver = {
+	.probe = gpio_aggregator_probe,
+	.driver = {
+		.name = DRV_NAME,
+		.groups = gpio_aggregator_groups,
+		.of_match_table = of_match_ptr(gpio_aggregator_dt_ids),
+	},
+};
+
+static int __init gpio_aggregator_init(void)
+{
+	return platform_driver_register(&gpio_aggregator_driver);
+}
+module_init(gpio_aggregator_init);
+
+static void __exit gpio_aggregator_exit(void)
+{
+	gpio_aggregator_remove_all();
+	platform_driver_unregister(&gpio_aggregator_driver);
+}
+module_exit(gpio_aggregator_exit);
+
+MODULE_AUTHOR("Geert Uytterhoeven <geert+renesas@glider.be>");
+MODULE_DESCRIPTION("GPIO Aggregator");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c
index 92e127e74813..1d8d55bd63aa 100644
--- a/drivers/gpio/gpio-dwapb.c
+++ b/drivers/gpio/gpio-dwapb.c
@@ -49,7 +49,9 @@
 #define GPIO_EXT_PORTC		0x58
 #define GPIO_EXT_PORTD		0x5c
 
+#define DWAPB_DRIVER_NAME	"gpio-dwapb"
 #define DWAPB_MAX_PORTS		4
+
 #define GPIO_EXT_PORT_STRIDE	0x04 /* register stride 32 bits */
 #define GPIO_SWPORT_DR_STRIDE	0x0c /* register stride 3*32 bits */
 #define GPIO_SWPORT_DDR_STRIDE	0x0c /* register stride 3*32 bits */
@@ -62,6 +64,8 @@
 #define GPIO_INTSTATUS_V2	0x3c
 #define GPIO_PORTA_EOI_V2	0x40
 
+#define DWAPB_NR_CLOCKS		2
+
 struct dwapb_gpio;
 
 #ifdef CONFIG_PM_SLEEP
@@ -97,7 +101,7 @@ struct dwapb_gpio {
 	struct irq_domain	*domain;
 	unsigned int		flags;
 	struct reset_control	*rst;
-	struct clk		*clk;
+	struct clk_bulk_data	clks[DWAPB_NR_CLOCKS];
 };
 
 static inline u32 gpio_reg_v2_convert(unsigned int offset)
@@ -189,22 +193,21 @@ static void dwapb_toggle_trigger(struct dwapb_gpio *gpio, unsigned int offs)
 
 static u32 dwapb_do_irq(struct dwapb_gpio *gpio)
 {
-	u32 irq_status = dwapb_read(gpio, GPIO_INTSTATUS);
-	u32 ret = irq_status;
+	unsigned long irq_status;
+	irq_hw_number_t hwirq;
 
-	while (irq_status) {
-		int hwirq = fls(irq_status) - 1;
+	irq_status = dwapb_read(gpio, GPIO_INTSTATUS);
+	for_each_set_bit(hwirq, &irq_status, 32) {
 		int gpio_irq = irq_find_mapping(gpio->domain, hwirq);
+		u32 irq_type = irq_get_trigger_type(gpio_irq);
 
 		generic_handle_irq(gpio_irq);
-		irq_status &= ~BIT(hwirq);
 
-		if ((irq_get_trigger_type(gpio_irq) & IRQ_TYPE_SENSE_MASK)
-			== IRQ_TYPE_EDGE_BOTH)
+		if ((irq_type & IRQ_TYPE_SENSE_MASK) == IRQ_TYPE_EDGE_BOTH)
 			dwapb_toggle_trigger(gpio, hwirq);
 	}
 
-	return ret;
+	return irq_status;
 }
 
 static void dwapb_irq_handler(struct irq_desc *desc)
@@ -212,10 +215,9 @@ static void dwapb_irq_handler(struct irq_desc *desc)
 	struct dwapb_gpio *gpio = irq_desc_get_handler_data(desc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 
+	chained_irq_enter(chip, desc);
 	dwapb_do_irq(gpio);
-
-	if (chip->irq_eoi)
-		chip->irq_eoi(irq_desc_get_irq_data(desc));
+	chained_irq_exit(chip, desc);
 }
 
 static void dwapb_irq_enable(struct irq_data *d)
@@ -228,7 +230,7 @@ static void dwapb_irq_enable(struct irq_data *d)
 
 	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	val = dwapb_read(gpio, GPIO_INTEN);
-	val |= BIT(d->hwirq);
+	val |= BIT(irqd_to_hwirq(d));
 	dwapb_write(gpio, GPIO_INTEN, val);
 	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
@@ -243,46 +245,20 @@ static void dwapb_irq_disable(struct irq_data *d)
 
 	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	val = dwapb_read(gpio, GPIO_INTEN);
-	val &= ~BIT(d->hwirq);
+	val &= ~BIT(irqd_to_hwirq(d));
 	dwapb_write(gpio, GPIO_INTEN, val);
 	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 }
 
-static int dwapb_irq_reqres(struct irq_data *d)
-{
-	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
-	struct dwapb_gpio *gpio = igc->private;
-	struct gpio_chip *gc = &gpio->ports[0].gc;
-	int ret;
-
-	ret = gpiochip_lock_as_irq(gc, irqd_to_hwirq(d));
-	if (ret) {
-		dev_err(gpio->dev, "unable to lock HW IRQ %lu for IRQ\n",
-			irqd_to_hwirq(d));
-		return ret;
-	}
-	return 0;
-}
-
-static void dwapb_irq_relres(struct irq_data *d)
-{
-	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
-	struct dwapb_gpio *gpio = igc->private;
-	struct gpio_chip *gc = &gpio->ports[0].gc;
-
-	gpiochip_unlock_as_irq(gc, irqd_to_hwirq(d));
-}
-
 static int dwapb_irq_set_type(struct irq_data *d, u32 type)
 {
 	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
 	struct dwapb_gpio *gpio = igc->private;
 	struct gpio_chip *gc = &gpio->ports[0].gc;
-	int bit = d->hwirq;
+	irq_hw_number_t bit = irqd_to_hwirq(d);
 	unsigned long level, polarity, flags;
 
-	if (type & ~(IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING |
-		     IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW))
+	if (type & ~IRQ_TYPE_SENSE_MASK)
 		return -EINVAL;
 
 	spin_lock_irqsave(&gc->bgpio_lock, flags);
@@ -328,11 +304,12 @@ static int dwapb_irq_set_wake(struct irq_data *d, unsigned int enable)
 	struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
 	struct dwapb_gpio *gpio = igc->private;
 	struct dwapb_context *ctx = gpio->ports[0].ctx;
+	irq_hw_number_t bit = irqd_to_hwirq(d);
 
 	if (enable)
-		ctx->wake_en |= BIT(d->hwirq);
+		ctx->wake_en |= BIT(bit);
 	else
-		ctx->wake_en &= ~BIT(d->hwirq);
+		ctx->wake_en &= ~BIT(bit);
 
 	return 0;
 }
@@ -350,9 +327,10 @@ static int dwapb_gpio_set_debounce(struct gpio_chip *gc,
 
 	val_deb = dwapb_read(gpio, GPIO_PORTA_DEBOUNCE);
 	if (debounce)
-		dwapb_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb | mask);
+		val_deb |= mask;
 	else
-		dwapb_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb & ~mask);
+		val_deb &= ~mask;
+	dwapb_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb);
 
 	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
@@ -373,12 +351,7 @@ static int dwapb_gpio_set_config(struct gpio_chip *gc, unsigned offset,
 
 static irqreturn_t dwapb_irq_handler_mfd(int irq, void *dev_id)
 {
-	u32 worked;
-	struct dwapb_gpio *gpio = dev_id;
-
-	worked = dwapb_do_irq(gpio);
-
-	return worked ? IRQ_HANDLED : IRQ_NONE;
+	return IRQ_RETVAL(dwapb_do_irq(dev_id));
 }
 
 static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
@@ -388,17 +361,23 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 	struct gpio_chip *gc = &port->gc;
 	struct fwnode_handle  *fwnode = pp->fwnode;
 	struct irq_chip_generic	*irq_gc = NULL;
-	unsigned int hwirq, ngpio = gc->ngpio;
+	unsigned int ngpio = gc->ngpio;
 	struct irq_chip_type *ct;
+	irq_hw_number_t hwirq;
 	int err, i;
 
+	if (memchr_inv(pp->irq, 0, sizeof(pp->irq)) == NULL) {
+		dev_warn(gpio->dev, "no IRQ for port%d\n", pp->idx);
+		return;
+	}
+
 	gpio->domain = irq_domain_create_linear(fwnode, ngpio,
 						 &irq_generic_chip_ops, gpio);
 	if (!gpio->domain)
 		return;
 
 	err = irq_alloc_domain_generic_chips(gpio->domain, ngpio, 2,
-					     "gpio-dwapb", handle_level_irq,
+					     DWAPB_DRIVER_NAME, handle_bad_irq,
 					     IRQ_NOREQUEST, 0,
 					     IRQ_GC_INIT_NESTED_LOCK);
 	if (err) {
@@ -426,8 +405,6 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 		ct->chip.irq_set_type = dwapb_irq_set_type;
 		ct->chip.irq_enable = dwapb_irq_enable;
 		ct->chip.irq_disable = dwapb_irq_disable;
-		ct->chip.irq_request_resources = dwapb_irq_reqres;
-		ct->chip.irq_release_resources = dwapb_irq_relres;
 #ifdef CONFIG_PM_SLEEP
 		ct->chip.irq_set_wake = dwapb_irq_set_wake;
 #endif
@@ -437,6 +414,7 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 	}
 
 	irq_gc->chip_types[0].type = IRQ_TYPE_LEVEL_MASK;
+	irq_gc->chip_types[0].handler = handle_level_irq;
 	irq_gc->chip_types[1].type = IRQ_TYPE_EDGE_BOTH;
 	irq_gc->chip_types[1].handler = handle_edge_irq;
 
@@ -444,7 +422,7 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 		int i;
 
 		for (i = 0; i < pp->ngpio; i++) {
-			if (pp->irq[i] >= 0)
+			if (pp->irq[i])
 				irq_set_chained_handler_and_data(pp->irq[i],
 						dwapb_irq_handler, gpio);
 		}
@@ -455,7 +433,7 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 		 */
 		err = devm_request_irq(gpio->dev, pp->irq[0],
 				       dwapb_irq_handler_mfd,
-				       IRQF_SHARED, "gpio-dwapb-mfd", gpio);
+				       IRQF_SHARED, DWAPB_DRIVER_NAME, gpio);
 		if (err) {
 			dev_err(gpio->dev, "error requesting IRQ\n");
 			irq_domain_remove(gpio->domain);
@@ -464,7 +442,7 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
 		}
 	}
 
-	for (hwirq = 0 ; hwirq < ngpio ; hwirq++)
+	for (hwirq = 0; hwirq < ngpio; hwirq++)
 		irq_create_mapping(gpio->domain, hwirq);
 
 	port->gc.to_irq = dwapb_gpio_to_irq;
@@ -480,7 +458,7 @@ static void dwapb_irq_teardown(struct dwapb_gpio *gpio)
 	if (!gpio->domain)
 		return;
 
-	for (hwirq = 0 ; hwirq < ngpio ; hwirq++)
+	for (hwirq = 0; hwirq < ngpio; hwirq++)
 		irq_dispose_mapping(irq_find_mapping(gpio->domain, hwirq));
 
 	irq_domain_remove(gpio->domain);
@@ -505,10 +483,9 @@ static int dwapb_gpio_add_port(struct dwapb_gpio *gpio,
 		return -ENOMEM;
 #endif
 
-	dat = gpio->regs + GPIO_EXT_PORTA + (pp->idx * GPIO_EXT_PORT_STRIDE);
-	set = gpio->regs + GPIO_SWPORTA_DR + (pp->idx * GPIO_SWPORT_DR_STRIDE);
-	dirout = gpio->regs + GPIO_SWPORTA_DDR +
-		(pp->idx * GPIO_SWPORT_DDR_STRIDE);
+	dat = gpio->regs + GPIO_EXT_PORTA + pp->idx * GPIO_EXT_PORT_STRIDE;
+	set = gpio->regs + GPIO_SWPORTA_DR + pp->idx * GPIO_SWPORT_DR_STRIDE;
+	dirout = gpio->regs + GPIO_SWPORTA_DDR + pp->idx * GPIO_SWPORT_DDR_STRIDE;
 
 	/* This registers 32 GPIO lines per port */
 	err = bgpio_init(&port->gc, gpio->dev, 4, dat, set, NULL, dirout,
@@ -529,40 +506,66 @@ static int dwapb_gpio_add_port(struct dwapb_gpio *gpio,
 	if (pp->idx == 0)
 		port->gc.set_config = dwapb_gpio_set_config;
 
-	if (pp->has_irq)
+	/* Only port A can provide interrupts in all configurations of the IP */
+	if (pp->idx == 0)
 		dwapb_configure_irqs(gpio, port, pp);
 
 	err = gpiochip_add_data(&port->gc, port);
-	if (err)
+	if (err) {
 		dev_err(gpio->dev, "failed to register gpiochip for port%d\n",
 			port->idx);
-	else
-		port->is_registered = true;
+		return err;
+	}
 
 	/* Add GPIO-signaled ACPI event support */
-	if (pp->has_irq)
-		acpi_gpiochip_request_interrupts(&port->gc);
+	acpi_gpiochip_request_interrupts(&port->gc);
 
-	return err;
+	port->is_registered = true;
+
+	return 0;
 }
 
 static void dwapb_gpio_unregister(struct dwapb_gpio *gpio)
 {
 	unsigned int m;
 
-	for (m = 0; m < gpio->nr_ports; ++m)
-		if (gpio->ports[m].is_registered)
-			gpiochip_remove(&gpio->ports[m].gc);
+	for (m = 0; m < gpio->nr_ports; ++m) {
+		struct dwapb_gpio_port *port = &gpio->ports[m];
+
+		if (!port->is_registered)
+			continue;
+
+		acpi_gpiochip_free_interrupts(&port->gc);
+		gpiochip_remove(&port->gc);
+	}
+}
+
+static void dwapb_get_irq(struct device *dev, struct fwnode_handle *fwnode,
+			  struct dwapb_port_property *pp)
+{
+	struct device_node *np = NULL;
+	int irq = -ENXIO, j;
+
+	if (fwnode_property_read_bool(fwnode, "interrupt-controller"))
+		np = to_of_node(fwnode);
+
+	for (j = 0; j < pp->ngpio; j++) {
+		if (np)
+			irq = of_irq_get(np, j);
+		else if (has_acpi_companion(dev))
+			irq = platform_get_irq_optional(to_platform_device(dev), j);
+		if (irq > 0)
+			pp->irq[j] = irq;
+	}
 }
 
-static struct dwapb_platform_data *
-dwapb_gpio_get_pdata(struct device *dev)
+static struct dwapb_platform_data *dwapb_gpio_get_pdata(struct device *dev)
 {
 	struct fwnode_handle *fwnode;
 	struct dwapb_platform_data *pdata;
 	struct dwapb_port_property *pp;
 	int nports;
-	int i, j;
+	int i;
 
 	nports = device_get_child_node_count(dev);
 	if (nports == 0)
@@ -580,8 +583,6 @@ dwapb_gpio_get_pdata(struct device *dev)
 
 	i = 0;
 	device_for_each_child_node(dev, fwnode)  {
-		struct device_node *np = NULL;
-
 		pp = &pdata->properties[i++];
 		pp->fwnode = fwnode;
 
@@ -593,8 +594,7 @@ dwapb_gpio_get_pdata(struct device *dev)
 			return ERR_PTR(-EINVAL);
 		}
 
-		if (fwnode_property_read_u32(fwnode, "snps,nr-gpios",
-					 &pp->ngpio)) {
+		if (fwnode_property_read_u32(fwnode, "snps,nr-gpios", &pp->ngpio)) {
 			dev_info(dev,
 				 "failed to get number of gpios for port%d\n",
 				 i);
@@ -608,28 +608,8 @@ dwapb_gpio_get_pdata(struct device *dev)
 		 * Only port A can provide interrupts in all configurations of
 		 * the IP.
 		 */
-		if (pp->idx != 0)
-			continue;
-
-		if (dev->of_node && fwnode_property_read_bool(fwnode,
-						  "interrupt-controller")) {
-			np = to_of_node(fwnode);
-		}
-
-		for (j = 0; j < pp->ngpio; j++) {
-			pp->irq[j] = -ENXIO;
-
-			if (np)
-				pp->irq[j] = of_irq_get(np, j);
-			else if (has_acpi_companion(dev))
-				pp->irq[j] = platform_get_irq(to_platform_device(dev), j);
-
-			if (pp->irq[j] >= 0)
-				pp->has_irq = true;
-		}
-
-		if (!pp->has_irq)
-			dev_warn(dev, "no irq for port%d\n", pp->idx);
+		if (pp->idx == 0)
+			dwapb_get_irq(dev, fwnode, pp);
 	}
 
 	return pdata;
@@ -689,29 +669,24 @@ static int dwapb_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(gpio->regs))
 		return PTR_ERR(gpio->regs);
 
-	/* Optional bus clock */
-	gpio->clk = devm_clk_get(&pdev->dev, "bus");
-	if (!IS_ERR(gpio->clk)) {
-		err = clk_prepare_enable(gpio->clk);
-		if (err) {
-			dev_info(&pdev->dev, "Cannot enable clock\n");
-			return err;
-		}
+	/* Optional bus and debounce clocks */
+	gpio->clks[0].id = "bus";
+	gpio->clks[1].id = "db";
+	err = devm_clk_bulk_get_optional(&pdev->dev, DWAPB_NR_CLOCKS,
+					 gpio->clks);
+	if (err) {
+		dev_err(&pdev->dev, "Cannot get APB/Debounce clocks\n");
+		return err;
 	}
 
-	gpio->flags = 0;
-	if (dev->of_node) {
-		gpio->flags = (uintptr_t)of_device_get_match_data(dev);
-	} else if (has_acpi_companion(dev)) {
-		const struct acpi_device_id *acpi_id;
-
-		acpi_id = acpi_match_device(dwapb_acpi_match, dev);
-		if (acpi_id) {
-			if (acpi_id->driver_data)
-				gpio->flags = acpi_id->driver_data;
-		}
+	err = clk_bulk_prepare_enable(DWAPB_NR_CLOCKS, gpio->clks);
+	if (err) {
+		dev_err(&pdev->dev, "Cannot enable APB/Debounce clocks\n");
+		return err;
 	}
 
+	gpio->flags = (uintptr_t)device_get_match_data(dev);
+
 	for (i = 0; i < gpio->nr_ports; i++) {
 		err = dwapb_gpio_add_port(gpio, &pdata->properties[i], i);
 		if (err)
@@ -724,7 +699,7 @@ static int dwapb_gpio_probe(struct platform_device *pdev)
 out_unregister:
 	dwapb_gpio_unregister(gpio);
 	dwapb_irq_teardown(gpio);
-	clk_disable_unprepare(gpio->clk);
+	clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks);
 
 	return err;
 }
@@ -736,7 +711,7 @@ static int dwapb_gpio_remove(struct platform_device *pdev)
 	dwapb_gpio_unregister(gpio);
 	dwapb_irq_teardown(gpio);
 	reset_control_assert(gpio->rst);
-	clk_disable_unprepare(gpio->clk);
+	clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks);
 
 	return 0;
 }
@@ -755,8 +730,6 @@ static int dwapb_gpio_suspend(struct device *dev)
 		unsigned int idx = gpio->ports[i].idx;
 		struct dwapb_context *ctx = gpio->ports[i].ctx;
 
-		BUG_ON(!ctx);
-
 		offset = GPIO_SWPORTA_DDR + idx * GPIO_SWPORT_DDR_STRIDE;
 		ctx->dir = dwapb_read(gpio, offset);
 
@@ -775,13 +748,12 @@ static int dwapb_gpio_suspend(struct device *dev)
 			ctx->int_deb	= dwapb_read(gpio, GPIO_PORTA_DEBOUNCE);
 
 			/* Mask out interrupts */
-			dwapb_write(gpio, GPIO_INTMASK,
-				    0xffffffff & ~ctx->wake_en);
+			dwapb_write(gpio, GPIO_INTMASK, ~ctx->wake_en);
 		}
 	}
 	spin_unlock_irqrestore(&gc->bgpio_lock, flags);
 
-	clk_disable_unprepare(gpio->clk);
+	clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks);
 
 	return 0;
 }
@@ -791,10 +763,13 @@ static int dwapb_gpio_resume(struct device *dev)
 	struct dwapb_gpio *gpio = dev_get_drvdata(dev);
 	struct gpio_chip *gc	= &gpio->ports[0].gc;
 	unsigned long flags;
-	int i;
+	int i, err;
 
-	if (!IS_ERR(gpio->clk))
-		clk_prepare_enable(gpio->clk);
+	err = clk_bulk_prepare_enable(DWAPB_NR_CLOCKS, gpio->clks);
+	if (err) {
+		dev_err(gpio->dev, "Cannot reenable APB/Debounce clocks\n");
+		return err;
+	}
 
 	spin_lock_irqsave(&gc->bgpio_lock, flags);
 	for (i = 0; i < gpio->nr_ports; i++) {
@@ -802,8 +777,6 @@ static int dwapb_gpio_resume(struct device *dev)
 		unsigned int idx = gpio->ports[i].idx;
 		struct dwapb_context *ctx = gpio->ports[i].ctx;
 
-		BUG_ON(!ctx);
-
 		offset = GPIO_SWPORTA_DR + idx * GPIO_SWPORT_DR_STRIDE;
 		dwapb_write(gpio, offset, ctx->data);
 
@@ -836,10 +809,10 @@ static SIMPLE_DEV_PM_OPS(dwapb_gpio_pm_ops, dwapb_gpio_suspend,
 
 static struct platform_driver dwapb_gpio_driver = {
 	.driver		= {
-		.name	= "gpio-dwapb",
+		.name	= DWAPB_DRIVER_NAME,
 		.pm	= &dwapb_gpio_pm_ops,
-		.of_match_table = of_match_ptr(dwapb_of_match),
-		.acpi_match_table = ACPI_PTR(dwapb_acpi_match),
+		.of_match_table = dwapb_of_match,
+		.acpi_match_table = dwapb_acpi_match,
 	},
 	.probe		= dwapb_gpio_probe,
 	.remove		= dwapb_gpio_remove,
@@ -850,3 +823,4 @@ module_platform_driver(dwapb_gpio_driver);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Jamie Iles");
 MODULE_DESCRIPTION("Synopsys DesignWare APB GPIO driver");
+MODULE_ALIAS("platform:" DWAPB_DRIVER_NAME);
diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c
index cadd02993539..18a3147f5a42 100644
--- a/drivers/gpio/gpio-f7188x.c
+++ b/drivers/gpio/gpio-f7188x.c
@@ -36,9 +36,19 @@
 #define SIO_F71889A_ID		0x1005	/* F71889A chipset ID */
 #define SIO_F81866_ID		0x1010	/* F81866 chipset ID */
 #define SIO_F81804_ID		0x1502  /* F81804 chipset ID, same for f81966 */
-
-
-enum chips { f71869, f71869a, f71882fg, f71889a, f71889f, f81866, f81804 };
+#define SIO_F81865_ID		0x0704	/* F81865 chipset ID */
+
+
+enum chips {
+	f71869,
+	f71869a,
+	f71882fg,
+	f71889a,
+	f71889f,
+	f81866,
+	f81804,
+	f81865,
+};
 
 static const char * const f7188x_names[] = {
 	"f71869",
@@ -48,6 +58,7 @@ static const char * const f7188x_names[] = {
 	"f71889f",
 	"f81866",
 	"f81804",
+	"f81865",
 };
 
 struct f7188x_sio {
@@ -233,6 +244,15 @@ static struct f7188x_gpio_bank f81804_gpio_bank[] = {
 	F7188X_GPIO_BANK(90, 8, 0x98),
 };
 
+static struct f7188x_gpio_bank f81865_gpio_bank[] = {
+	F7188X_GPIO_BANK(0, 8, 0xF0),
+	F7188X_GPIO_BANK(10, 8, 0xE0),
+	F7188X_GPIO_BANK(20, 8, 0xD0),
+	F7188X_GPIO_BANK(30, 8, 0xC0),
+	F7188X_GPIO_BANK(40, 8, 0xB0),
+	F7188X_GPIO_BANK(50, 8, 0xA0),
+	F7188X_GPIO_BANK(60, 5, 0x90),
+};
 
 static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 {
@@ -425,6 +445,10 @@ static int f7188x_gpio_probe(struct platform_device *pdev)
 		data->nr_bank = ARRAY_SIZE(f81804_gpio_bank);
 		data->bank = f81804_gpio_bank;
 		break;
+	case f81865:
+		data->nr_bank = ARRAY_SIZE(f81865_gpio_bank);
+		data->bank = f81865_gpio_bank;
+		break;
 	default:
 		return -ENODEV;
 	}
@@ -490,6 +514,9 @@ static int __init f7188x_find(int addr, struct f7188x_sio *sio)
 	case SIO_F81804_ID:
 		sio->type = f81804;
 		break;
+	case SIO_F81865_ID:
+		sio->type = f81865;
+		break;
 	default:
 		pr_info(DRVNAME ": Unsupported Fintek device 0x%04x\n", devid);
 		goto err;
diff --git a/drivers/gpio/gpio-ftgpio010.c b/drivers/gpio/gpio-ftgpio010.c
index fbddb1662428..4031164780f7 100644
--- a/drivers/gpio/gpio-ftgpio010.c
+++ b/drivers/gpio/gpio-ftgpio010.c
@@ -193,7 +193,7 @@ static int ftgpio_gpio_set_config(struct gpio_chip *gc, unsigned int offset,
 	if (val == deb_div) {
 		/*
 		 * The debounce timer happens to already be set to the
-		 * desireable value, what a coincidence! We can just enable
+		 * desirable value, what a coincidence! We can just enable
 		 * debounce on this GPIO line and return. This happens more
 		 * often than you think, for example when all GPIO keys
 		 * on a system are requesting the same debounce interval.
diff --git a/drivers/gpio/gpio-ich.c b/drivers/gpio/gpio-ich.c
index 2f086d0aa1f4..9960bb8b0f5b 100644
--- a/drivers/gpio/gpio-ich.c
+++ b/drivers/gpio/gpio-ich.c
@@ -89,7 +89,7 @@ static struct {
 	struct device *dev;
 	struct gpio_chip chip;
 	struct resource *gpio_base;	/* GPIO IO base */
-	struct resource *pm_base;	/* Power Mangagment IO base */
+	struct resource *pm_base;	/* Power Management IO base */
 	struct ichx_desc *desc;	/* Pointer to chipset-specific description */
 	u32 orig_gpio_ctrl;	/* Orig CTRL value, used to restore on exit */
 	u8 use_gpio;		/* Which GPIO groups are usable */
diff --git a/drivers/gpio/gpio-max730x.c b/drivers/gpio/gpio-max730x.c
index 1e1935c51096..b8c1fe20f49a 100644
--- a/drivers/gpio/gpio-max730x.c
+++ b/drivers/gpio/gpio-max730x.c
@@ -47,7 +47,7 @@
 
 static int max7301_direction_input(struct gpio_chip *chip, unsigned offset)
 {
-	struct max7301 *ts = gpiochip_get_data(chip);
+	struct max7301 *ts = container_of(chip, struct max7301, chip);
 	u8 *config;
 	u8 offset_bits, pin_config;
 	int ret;
@@ -89,7 +89,7 @@ static int __max7301_set(struct max7301 *ts, unsigned offset, int value)
 static int max7301_direction_output(struct gpio_chip *chip, unsigned offset,
 				    int value)
 {
-	struct max7301 *ts = gpiochip_get_data(chip);
+	struct max7301 *ts = container_of(chip, struct max7301, chip);
 	u8 *config;
 	u8 offset_bits;
 	int ret;
@@ -189,10 +189,6 @@ int __max730x_probe(struct max7301 *ts)
 	ts->chip.parent = dev;
 	ts->chip.owner = THIS_MODULE;
 
-	ret = gpiochip_add_data(&ts->chip, ts);
-	if (ret)
-		goto exit_destroy;
-
 	/*
 	 * initialize pullups according to platform data and cache the
 	 * register values for later use.
@@ -214,7 +210,9 @@ int __max730x_probe(struct max7301 *ts)
 		}
 	}
 
-	return ret;
+	ret = gpiochip_add_data(&ts->chip, ts);
+	if (!ret)
+		return ret;
 
 exit_destroy:
 	mutex_destroy(&ts->lock);
diff --git a/drivers/gpio/gpio-mb86s7x.c b/drivers/gpio/gpio-mb86s7x.c
index 501e89548f53..37c5363e391e 100644
--- a/drivers/gpio/gpio-mb86s7x.c
+++ b/drivers/gpio/gpio-mb86s7x.c
@@ -145,7 +145,9 @@ static int mb86s70_gpio_to_irq(struct gpio_chip *gc, unsigned int offset)
 
 	for (index = 0;; index++) {
 		irq = platform_get_irq(to_platform_device(gc->parent), index);
-		if (irq <= 0)
+		if (irq < 0)
+			return irq;
+		if (irq == 0)
 			break;
 		if (irq_get_irq_data(irq)->hwirq == offset)
 			return irq;
@@ -168,15 +170,13 @@ static int mb86s70_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(gchip->base))
 		return PTR_ERR(gchip->base);
 
-	if (!has_acpi_companion(&pdev->dev)) {
-		gchip->clk = devm_clk_get(&pdev->dev, NULL);
-		if (IS_ERR(gchip->clk))
-			return PTR_ERR(gchip->clk);
+	gchip->clk = devm_clk_get_optional(&pdev->dev, NULL);
+	if (IS_ERR(gchip->clk))
+		return PTR_ERR(gchip->clk);
 
-		ret = clk_prepare_enable(gchip->clk);
-		if (ret)
-			return ret;
-	}
+	ret = clk_prepare_enable(gchip->clk);
+	if (ret)
+		return ret;
 
 	spin_lock_init(&gchip->lock);
 
@@ -186,15 +186,13 @@ static int mb86s70_gpio_probe(struct platform_device *pdev)
 	gchip->gc.free = mb86s70_gpio_free;
 	gchip->gc.get = mb86s70_gpio_get;
 	gchip->gc.set = mb86s70_gpio_set;
+	gchip->gc.to_irq = mb86s70_gpio_to_irq;
 	gchip->gc.label = dev_name(&pdev->dev);
 	gchip->gc.ngpio = 32;
 	gchip->gc.owner = THIS_MODULE;
 	gchip->gc.parent = &pdev->dev;
 	gchip->gc.base = -1;
 
-	if (has_acpi_companion(&pdev->dev))
-		gchip->gc.to_irq = mb86s70_gpio_to_irq;
-
 	ret = gpiochip_add_data(&gchip->gc, gchip);
 	if (ret) {
 		dev_err(&pdev->dev, "couldn't register gpio driver\n");
@@ -202,8 +200,7 @@ static int mb86s70_gpio_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	if (has_acpi_companion(&pdev->dev))
-		acpi_gpiochip_request_interrupts(&gchip->gc);
+	acpi_gpiochip_request_interrupts(&gchip->gc);
 
 	return 0;
 }
@@ -212,8 +209,7 @@ static int mb86s70_gpio_remove(struct platform_device *pdev)
 {
 	struct mb86s70_gpio_chip *gchip = platform_get_drvdata(pdev);
 
-	if (has_acpi_companion(&pdev->dev))
-		acpi_gpiochip_free_interrupts(&gchip->gc);
+	acpi_gpiochip_free_interrupts(&gchip->gc);
 	gpiochip_remove(&gchip->gc);
 	clk_disable_unprepare(gchip->clk);
 
diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c
index 48918a016cd8..706687fab634 100644
--- a/drivers/gpio/gpio-merrifield.c
+++ b/drivers/gpio/gpio-merrifield.c
@@ -443,8 +443,8 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 
 	base = pcim_iomap_table(pdev)[1];
 
-	irq_base = readl(base);
-	gpio_base = readl(sizeof(u32) + base);
+	irq_base = readl(base + 0 * sizeof(u32));
+	gpio_base = readl(base + 1 * sizeof(u32));
 
 	/* Release the IO mapping, since we already get the info from BAR1 */
 	pcim_iounmap_regions(pdev, BIT(1));
@@ -473,6 +473,10 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 
 	raw_spin_lock_init(&priv->lock);
 
+	retval = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
+	if (retval < 0)
+		return retval;
+
 	girq = &priv->chip.irq;
 	girq->chip = &mrfld_irqchip;
 	girq->init_hw = mrfld_irq_init_hw;
@@ -482,7 +486,7 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 				     sizeof(*girq->parents), GFP_KERNEL);
 	if (!girq->parents)
 		return -ENOMEM;
-	girq->parents[0] = pdev->irq;
+	girq->parents[0] = pci_irq_vector(pdev, 0);
 	girq->first = irq_base;
 	girq->default_type = IRQ_TYPE_NONE;
 	girq->handler = handle_bad_irq;
diff --git a/drivers/gpio/gpio-mlxbf2.c b/drivers/gpio/gpio-mlxbf2.c
index da570e63589d..94d5efce1721 100644
--- a/drivers/gpio/gpio-mlxbf2.c
+++ b/drivers/gpio/gpio-mlxbf2.c
@@ -14,7 +14,6 @@
 #include <linux/resource.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
-#include <linux/version.h>
 
 /*
  * There are 3 YU GPIO blocks:
@@ -110,8 +109,8 @@ static int mlxbf2_gpio_get_lock_res(struct platform_device *pdev)
 	}
 
 	yu_arm_gpio_lock_param.io = devm_ioremap(dev, res->start, size);
-	if (IS_ERR(yu_arm_gpio_lock_param.io))
-		ret = PTR_ERR(yu_arm_gpio_lock_param.io);
+	if (!yu_arm_gpio_lock_param.io)
+		ret = -ENOMEM;
 
 exit:
 	mutex_unlock(yu_arm_gpio_lock_param.lock);
diff --git a/drivers/gpio/gpio-mm-lantiq.c b/drivers/gpio/gpio-mm-lantiq.c
index f460d71b0c92..538e31fe8903 100644
--- a/drivers/gpio/gpio-mm-lantiq.c
+++ b/drivers/gpio/gpio-mm-lantiq.c
@@ -36,7 +36,7 @@ struct ltq_mm {
  * @chip:     Pointer to our private data structure.
  *
  * Write the shadow value to the EBU to set the gpios. We need to set the
- * global EBU lock to make sure that PCI/MTD dont break.
+ * global EBU lock to make sure that PCI/MTD don't break.
  */
 static void ltq_mm_apply(struct ltq_mm *chip)
 {
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 4269ea9a817e..1fca8dd7824f 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -306,37 +306,39 @@ static const struct regmap_config pca953x_i2c_regmap = {
 	.writeable_reg = pca953x_writeable_register,
 	.volatile_reg = pca953x_volatile_register,
 
+	.disable_locking = true,
 	.cache_type = REGCACHE_RBTREE,
-	/* REVISIT: should be 0x7f but some 24 bit chips use REG_ADDR_AI */
-	.max_register = 0xff,
+	.max_register = 0x7f,
 };
 
-static u8 pca953x_recalc_addr(struct pca953x_chip *chip, int reg, int off,
-			      bool write, bool addrinc)
+static const struct regmap_config pca953x_ai_i2c_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.read_flag_mask = REG_ADDR_AI,
+	.write_flag_mask = REG_ADDR_AI,
+
+	.readable_reg = pca953x_readable_register,
+	.writeable_reg = pca953x_writeable_register,
+	.volatile_reg = pca953x_volatile_register,
+
+	.cache_type = REGCACHE_RBTREE,
+	.max_register = 0x7f,
+};
+
+static u8 pca953x_recalc_addr(struct pca953x_chip *chip, int reg, int off)
 {
 	int bank_shift = pca953x_bank_shift(chip);
 	int addr = (reg & PCAL_GPIO_MASK) << bank_shift;
 	int pinctrl = (reg & PCAL_PINCTRL_MASK) << 1;
 	u8 regaddr = pinctrl | addr | (off / BANK_SZ);
 
-	/* Single byte read doesn't need AI bit set. */
-	if (!addrinc)
-		return regaddr;
-
-	/* Chips with 24 and more GPIOs always support Auto Increment */
-	if (write && NBANK(chip) > 2)
-		regaddr |= REG_ADDR_AI;
-
-	/* PCA9575 needs address-increment on multi-byte writes */
-	if (PCA_CHIP_TYPE(chip->driver_data) == PCA957X_TYPE)
-		regaddr |= REG_ADDR_AI;
-
 	return regaddr;
 }
 
 static int pca953x_write_regs(struct pca953x_chip *chip, int reg, unsigned long *val)
 {
-	u8 regaddr = pca953x_recalc_addr(chip, reg, 0, true, true);
+	u8 regaddr = pca953x_recalc_addr(chip, reg, 0);
 	u8 value[MAX_BANK];
 	int i, ret;
 
@@ -354,7 +356,7 @@ static int pca953x_write_regs(struct pca953x_chip *chip, int reg, unsigned long
 
 static int pca953x_read_regs(struct pca953x_chip *chip, int reg, unsigned long *val)
 {
-	u8 regaddr = pca953x_recalc_addr(chip, reg, 0, false, true);
+	u8 regaddr = pca953x_recalc_addr(chip, reg, 0);
 	u8 value[MAX_BANK];
 	int i, ret;
 
@@ -373,8 +375,7 @@ static int pca953x_read_regs(struct pca953x_chip *chip, int reg, unsigned long *
 static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
-	u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off,
-					true, false);
+	u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off);
 	u8 bit = BIT(off % BANK_SZ);
 	int ret;
 
@@ -388,10 +389,8 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc,
 		unsigned off, int val)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
-	u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off,
-					true, false);
-	u8 outreg = pca953x_recalc_addr(chip, chip->regs->output, off,
-					true, false);
+	u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off);
+	u8 outreg = pca953x_recalc_addr(chip, chip->regs->output, off);
 	u8 bit = BIT(off % BANK_SZ);
 	int ret;
 
@@ -411,8 +410,7 @@ exit:
 static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
-	u8 inreg = pca953x_recalc_addr(chip, chip->regs->input, off,
-				       true, false);
+	u8 inreg = pca953x_recalc_addr(chip, chip->regs->input, off);
 	u8 bit = BIT(off % BANK_SZ);
 	u32 reg_val;
 	int ret;
@@ -436,8 +434,7 @@ static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off)
 static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
-	u8 outreg = pca953x_recalc_addr(chip, chip->regs->output, off,
-					true, false);
+	u8 outreg = pca953x_recalc_addr(chip, chip->regs->output, off);
 	u8 bit = BIT(off % BANK_SZ);
 
 	mutex_lock(&chip->i2c_lock);
@@ -448,8 +445,7 @@ static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val)
 static int pca953x_gpio_get_direction(struct gpio_chip *gc, unsigned off)
 {
 	struct pca953x_chip *chip = gpiochip_get_data(gc);
-	u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off,
-					true, false);
+	u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off);
 	u8 bit = BIT(off % BANK_SZ);
 	u32 reg_val;
 	int ret;
@@ -466,6 +462,23 @@ static int pca953x_gpio_get_direction(struct gpio_chip *gc, unsigned off)
 	return GPIO_LINE_DIRECTION_OUT;
 }
 
+static int pca953x_gpio_get_multiple(struct gpio_chip *gc,
+				     unsigned long *mask, unsigned long *bits)
+{
+	struct pca953x_chip *chip = gpiochip_get_data(gc);
+	DECLARE_BITMAP(reg_val, MAX_LINE);
+	int ret;
+
+	mutex_lock(&chip->i2c_lock);
+	ret = pca953x_read_regs(chip, chip->regs->input, reg_val);
+	mutex_unlock(&chip->i2c_lock);
+	if (ret)
+		return ret;
+
+	bitmap_replace(bits, bits, reg_val, mask, gc->ngpio);
+	return 0;
+}
+
 static void pca953x_gpio_set_multiple(struct gpio_chip *gc,
 				      unsigned long *mask, unsigned long *bits)
 {
@@ -489,10 +502,8 @@ static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip,
 					 unsigned int offset,
 					 unsigned long config)
 {
-	u8 pull_en_reg = pca953x_recalc_addr(chip, PCAL953X_PULL_EN, offset,
-					     true, false);
-	u8 pull_sel_reg = pca953x_recalc_addr(chip, PCAL953X_PULL_SEL, offset,
-					      true, false);
+	u8 pull_en_reg = pca953x_recalc_addr(chip, PCAL953X_PULL_EN, offset);
+	u8 pull_sel_reg = pca953x_recalc_addr(chip, PCAL953X_PULL_SEL, offset);
 	u8 bit = BIT(offset % BANK_SZ);
 	int ret;
 
@@ -551,6 +562,7 @@ static void pca953x_setup_gpio(struct pca953x_chip *chip, int gpios)
 	gc->get = pca953x_gpio_get_value;
 	gc->set = pca953x_gpio_set_value;
 	gc->get_direction = pca953x_gpio_get_direction;
+	gc->get_multiple = pca953x_gpio_get_multiple;
 	gc->set_multiple = pca953x_gpio_set_multiple;
 	gc->set_config = pca953x_gpio_set_config;
 	gc->can_sleep = true;
@@ -863,6 +875,7 @@ static int pca953x_probe(struct i2c_client *client,
 	int ret;
 	u32 invert = 0;
 	struct regulator *reg;
+	const struct regmap_config *regmap_config;
 
 	chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL);
 	if (chip == NULL)
@@ -925,7 +938,17 @@ static int pca953x_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, chip);
 
-	chip->regmap = devm_regmap_init_i2c(client, &pca953x_i2c_regmap);
+	pca953x_setup_gpio(chip, chip->driver_data & PCA_GPIO_MASK);
+
+	if (NBANK(chip) > 2 || PCA_CHIP_TYPE(chip->driver_data) == PCA957X_TYPE) {
+		dev_info(&client->dev, "using AI\n");
+		regmap_config = &pca953x_ai_i2c_regmap;
+	} else {
+		dev_info(&client->dev, "using no AI\n");
+		regmap_config = &pca953x_i2c_regmap;
+	}
+
+	chip->regmap = devm_regmap_init_i2c(client, regmap_config);
 	if (IS_ERR(chip->regmap)) {
 		ret = PTR_ERR(chip->regmap);
 		goto err_exit;
@@ -956,7 +979,6 @@ static int pca953x_probe(struct i2c_client *client,
 	/* initialize cached registers from their original values.
 	 * we can't share this chip with another i2c master.
 	 */
-	pca953x_setup_gpio(chip, chip->driver_data & PCA_GPIO_MASK);
 
 	if (PCA_CHIP_TYPE(chip->driver_data) == PCA953X_TYPE) {
 		chip->regs = &pca953x_regs;
@@ -1154,7 +1176,7 @@ static struct i2c_driver pca953x_driver = {
 		.name	= "pca953x",
 		.pm	= &pca953x_pm_ops,
 		.of_match_table = pca953x_dt_ids,
-		.acpi_match_table = ACPI_PTR(pca953x_acpi_ids),
+		.acpi_match_table = pca953x_acpi_ids,
 	},
 	.probe		= pca953x_probe,
 	.remove		= pca953x_remove,
diff --git a/drivers/gpio/gpio-pch.c b/drivers/gpio/gpio-pch.c
index 3f3d9a94b709..e96d28bf43b4 100644
--- a/drivers/gpio/gpio-pch.c
+++ b/drivers/gpio/gpio-pch.c
@@ -2,6 +2,7 @@
 /*
  * Copyright (C) 2011 LAPIS Semiconductor Co., Ltd.
  */
+#include <linux/bits.h>
 #include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
@@ -11,11 +12,11 @@
 #include <linux/slab.h>
 
 #define PCH_EDGE_FALLING	0
-#define PCH_EDGE_RISING		BIT(0)
-#define PCH_LEVEL_L		BIT(1)
-#define PCH_LEVEL_H		(BIT(0) | BIT(1))
-#define PCH_EDGE_BOTH		BIT(2)
-#define PCH_IM_MASK		(BIT(0) | BIT(1) | BIT(2))
+#define PCH_EDGE_RISING		1
+#define PCH_LEVEL_L		2
+#define PCH_LEVEL_H		3
+#define PCH_EDGE_BOTH		4
+#define PCH_IM_MASK		GENMASK(2, 0)
 
 #define PCH_IRQ_BASE		24
 
@@ -103,9 +104,9 @@ static void pch_gpio_set(struct gpio_chip *gpio, unsigned nr, int val)
 	spin_lock_irqsave(&chip->spinlock, flags);
 	reg_val = ioread32(&chip->reg->po);
 	if (val)
-		reg_val |= (1 << nr);
+		reg_val |= BIT(nr);
 	else
-		reg_val &= ~(1 << nr);
+		reg_val &= ~BIT(nr);
 
 	iowrite32(reg_val, &chip->reg->po);
 	spin_unlock_irqrestore(&chip->spinlock, flags);
@@ -115,7 +116,7 @@ static int pch_gpio_get(struct gpio_chip *gpio, unsigned nr)
 {
 	struct pch_gpio *chip =	gpiochip_get_data(gpio);
 
-	return (ioread32(&chip->reg->pi) >> nr) & 1;
+	return !!(ioread32(&chip->reg->pi) & BIT(nr));
 }
 
 static int pch_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
@@ -130,13 +131,14 @@ static int pch_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 
 	reg_val = ioread32(&chip->reg->po);
 	if (val)
-		reg_val |= (1 << nr);
+		reg_val |= BIT(nr);
 	else
-		reg_val &= ~(1 << nr);
+		reg_val &= ~BIT(nr);
 	iowrite32(reg_val, &chip->reg->po);
 
-	pm = ioread32(&chip->reg->pm) & ((1 << gpio_pins[chip->ioh]) - 1);
-	pm |= (1 << nr);
+	pm = ioread32(&chip->reg->pm);
+	pm &= BIT(gpio_pins[chip->ioh]) - 1;
+	pm |= BIT(nr);
 	iowrite32(pm, &chip->reg->pm);
 
 	spin_unlock_irqrestore(&chip->spinlock, flags);
@@ -151,8 +153,9 @@ static int pch_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 	unsigned long flags;
 
 	spin_lock_irqsave(&chip->spinlock, flags);
-	pm = ioread32(&chip->reg->pm) & ((1 << gpio_pins[chip->ioh]) - 1);
-	pm &= ~(1 << nr);
+	pm = ioread32(&chip->reg->pm);
+	pm &= BIT(gpio_pins[chip->ioh]) - 1;
+	pm &= ~BIT(nr);
 	iowrite32(pm, &chip->reg->pm);
 	spin_unlock_irqrestore(&chip->spinlock, flags);
 
@@ -226,17 +229,15 @@ static int pch_irq_type(struct irq_data *d, unsigned int type)
 	int ch, irq = d->irq;
 
 	ch = irq - chip->irq_base;
-	if (irq <= chip->irq_base + 7) {
+	if (irq < chip->irq_base + 8) {
 		im_reg = &chip->reg->im0;
-		im_pos = ch;
+		im_pos = ch - 0;
 	} else {
 		im_reg = &chip->reg->im1;
 		im_pos = ch - 8;
 	}
 	dev_dbg(chip->dev, "irq=%d type=%d ch=%d pos=%d\n", irq, type, ch, im_pos);
 
-	spin_lock_irqsave(&chip->spinlock, flags);
-
 	switch (type) {
 	case IRQ_TYPE_EDGE_RISING:
 		val = PCH_EDGE_RISING;
@@ -254,20 +255,21 @@ static int pch_irq_type(struct irq_data *d, unsigned int type)
 		val = PCH_LEVEL_L;
 		break;
 	default:
-		goto unlock;
+		return 0;
 	}
 
+	spin_lock_irqsave(&chip->spinlock, flags);
+
 	/* Set interrupt mode */
 	im = ioread32(im_reg) & ~(PCH_IM_MASK << (im_pos * 4));
 	iowrite32(im | (val << (im_pos * 4)), im_reg);
 
 	/* And the handler */
-	if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH))
+	if (type & IRQ_TYPE_LEVEL_MASK)
 		irq_set_handler_locked(d, handle_level_irq);
-	else if (type & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING))
+	else if (type & IRQ_TYPE_EDGE_BOTH)
 		irq_set_handler_locked(d, handle_edge_irq);
 
-unlock:
 	spin_unlock_irqrestore(&chip->spinlock, flags);
 	return 0;
 }
@@ -277,7 +279,7 @@ static void pch_irq_unmask(struct irq_data *d)
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct pch_gpio *chip = gc->private;
 
-	iowrite32(1 << (d->irq - chip->irq_base), &chip->reg->imaskclr);
+	iowrite32(BIT(d->irq - chip->irq_base), &chip->reg->imaskclr);
 }
 
 static void pch_irq_mask(struct irq_data *d)
@@ -285,7 +287,7 @@ static void pch_irq_mask(struct irq_data *d)
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct pch_gpio *chip = gc->private;
 
-	iowrite32(1 << (d->irq - chip->irq_base), &chip->reg->imask);
+	iowrite32(BIT(d->irq - chip->irq_base), &chip->reg->imask);
 }
 
 static void pch_irq_ack(struct irq_data *d)
@@ -293,21 +295,22 @@ static void pch_irq_ack(struct irq_data *d)
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct pch_gpio *chip = gc->private;
 
-	iowrite32(1 << (d->irq - chip->irq_base), &chip->reg->iclr);
+	iowrite32(BIT(d->irq - chip->irq_base), &chip->reg->iclr);
 }
 
 static irqreturn_t pch_gpio_handler(int irq, void *dev_id)
 {
 	struct pch_gpio *chip = dev_id;
 	unsigned long reg_val = ioread32(&chip->reg->istatus);
-	int i, ret = IRQ_NONE;
+	int i;
+
+	dev_dbg(chip->dev, "irq=%d  status=0x%lx\n", irq, reg_val);
 
-	for_each_set_bit(i, &reg_val, gpio_pins[chip->ioh]) {
-		dev_dbg(chip->dev, "[%d]:irq=%d  status=0x%lx\n", i, irq, reg_val);
+	reg_val &= BIT(gpio_pins[chip->ioh]) - 1;
+	for_each_set_bit(i, &reg_val, gpio_pins[chip->ioh])
 		generic_handle_irq(chip->irq_base + i);
-		ret = IRQ_HANDLED;
-	}
-	return ret;
+
+	return IRQ_RETVAL(reg_val);
 }
 
 static int pch_gpio_alloc_generic_chip(struct pch_gpio *chip,
@@ -344,7 +347,6 @@ static int pch_gpio_probe(struct pci_dev *pdev,
 	s32 ret;
 	struct pch_gpio *chip;
 	int irq_base;
-	u32 msk;
 
 	chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL);
 	if (chip == NULL)
@@ -357,7 +359,7 @@ static int pch_gpio_probe(struct pci_dev *pdev,
 		return ret;
 	}
 
-	ret = pcim_iomap_regions(pdev, 1 << 1, KBUILD_MODNAME);
+	ret = pcim_iomap_regions(pdev, BIT(1), KBUILD_MODNAME);
 	if (ret) {
 		dev_err(&pdev->dev, "pci_request_regions FAILED-%d", ret);
 		return ret;
@@ -393,9 +395,8 @@ static int pch_gpio_probe(struct pci_dev *pdev,
 	chip->irq_base = irq_base;
 
 	/* Mask all interrupts, but enable them */
-	msk = (1 << gpio_pins[chip->ioh]) - 1;
-	iowrite32(msk, &chip->reg->imask);
-	iowrite32(msk, &chip->reg->ien);
+	iowrite32(BIT(gpio_pins[chip->ioh]) - 1, &chip->reg->imask);
+	iowrite32(BIT(gpio_pins[chip->ioh]) - 1, &chip->reg->ien);
 
 	ret = devm_request_irq(&pdev->dev, pdev->irq, pch_gpio_handler,
 			       IRQF_SHARED, KBUILD_MODNAME, chip);
diff --git a/drivers/gpio/gpio-pl061.c b/drivers/gpio/gpio-pl061.c
index e241fb884c12..f1b53dd1df1a 100644
--- a/drivers/gpio/gpio-pl061.c
+++ b/drivers/gpio/gpio-pl061.c
@@ -16,6 +16,7 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
+#include <linux/module.h>
 #include <linux/bitops.h>
 #include <linux/gpio/driver.h>
 #include <linux/device.h>
@@ -408,6 +409,7 @@ static const struct amba_id pl061_ids[] = {
 	},
 	{ 0, 0 },
 };
+MODULE_DEVICE_TABLE(amba, pl061_ids);
 
 static struct amba_driver pl061_gpio_driver = {
 	.drv = {
@@ -419,9 +421,6 @@ static struct amba_driver pl061_gpio_driver = {
 	.id_table	= pl061_ids,
 	.probe		= pl061_probe,
 };
+module_amba_driver(pl061_gpio_driver);
 
-static int __init pl061_gpio_init(void)
-{
-	return amba_driver_register(&pl061_gpio_driver);
-}
-device_initcall(pl061_gpio_init);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index 7284473c9fe3..eac1582c70da 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -250,8 +250,10 @@ static int gpio_rcar_request(struct gpio_chip *chip, unsigned offset)
 	int error;
 
 	error = pm_runtime_get_sync(p->dev);
-	if (error < 0)
+	if (error < 0) {
+		pm_runtime_put(p->dev);
 		return error;
+	}
 
 	error = pinctrl_gpio_request(chip->base + offset);
 	if (error)
diff --git a/drivers/gpio/gpio-regmap.c b/drivers/gpio/gpio-regmap.c
new file mode 100644
index 000000000000..5412cb3b0b2a
--- /dev/null
+++ b/drivers/gpio/gpio-regmap.c
@@ -0,0 +1,349 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * regmap based generic GPIO driver
+ *
+ * Copyright 2020 Michael Walle <michael@walle.cc>
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+struct gpio_regmap {
+	struct device *parent;
+	struct regmap *regmap;
+	struct gpio_chip gpio_chip;
+
+	int reg_stride;
+	int ngpio_per_reg;
+	unsigned int reg_dat_base;
+	unsigned int reg_set_base;
+	unsigned int reg_clr_base;
+	unsigned int reg_dir_in_base;
+	unsigned int reg_dir_out_base;
+
+	int (*reg_mask_xlate)(struct gpio_regmap *gpio, unsigned int base,
+			      unsigned int offset, unsigned int *reg,
+			      unsigned int *mask);
+
+	void *driver_data;
+};
+
+static unsigned int gpio_regmap_addr(unsigned int addr)
+{
+	if (addr == GPIO_REGMAP_ADDR_ZERO)
+		return 0;
+
+	return addr;
+}
+
+static int gpio_regmap_simple_xlate(struct gpio_regmap *gpio,
+				    unsigned int base, unsigned int offset,
+				    unsigned int *reg, unsigned int *mask)
+{
+	unsigned int line = offset % gpio->ngpio_per_reg;
+	unsigned int stride = offset / gpio->ngpio_per_reg;
+
+	*reg = base + stride * gpio->reg_stride;
+	*mask = BIT(line);
+
+	return 0;
+}
+
+static int gpio_regmap_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct gpio_regmap *gpio = gpiochip_get_data(chip);
+	unsigned int base, val, reg, mask;
+	int ret;
+
+	/* we might not have an output register if we are input only */
+	if (gpio->reg_dat_base)
+		base = gpio_regmap_addr(gpio->reg_dat_base);
+	else
+		base = gpio_regmap_addr(gpio->reg_set_base);
+
+	ret = gpio->reg_mask_xlate(gpio, base, offset, &reg, &mask);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(gpio->regmap, reg, &val);
+	if (ret)
+		return ret;
+
+	return !!(val & mask);
+}
+
+static void gpio_regmap_set(struct gpio_chip *chip, unsigned int offset,
+			    int val)
+{
+	struct gpio_regmap *gpio = gpiochip_get_data(chip);
+	unsigned int base = gpio_regmap_addr(gpio->reg_set_base);
+	unsigned int reg, mask;
+
+	gpio->reg_mask_xlate(gpio, base, offset, &reg, &mask);
+	if (val)
+		regmap_update_bits(gpio->regmap, reg, mask, mask);
+	else
+		regmap_update_bits(gpio->regmap, reg, mask, 0);
+}
+
+static void gpio_regmap_set_with_clear(struct gpio_chip *chip,
+				       unsigned int offset, int val)
+{
+	struct gpio_regmap *gpio = gpiochip_get_data(chip);
+	unsigned int base, reg, mask;
+
+	if (val)
+		base = gpio_regmap_addr(gpio->reg_set_base);
+	else
+		base = gpio_regmap_addr(gpio->reg_clr_base);
+
+	gpio->reg_mask_xlate(gpio, base, offset, &reg, &mask);
+	regmap_write(gpio->regmap, reg, mask);
+}
+
+static int gpio_regmap_get_direction(struct gpio_chip *chip,
+				     unsigned int offset)
+{
+	struct gpio_regmap *gpio = gpiochip_get_data(chip);
+	unsigned int base, val, reg, mask;
+	int invert, ret;
+
+	if (gpio->reg_dir_out_base) {
+		base = gpio_regmap_addr(gpio->reg_dir_out_base);
+		invert = 0;
+	} else if (gpio->reg_dir_in_base) {
+		base = gpio_regmap_addr(gpio->reg_dir_in_base);
+		invert = 1;
+	} else {
+		return -EOPNOTSUPP;
+	}
+
+	ret = gpio->reg_mask_xlate(gpio, base, offset, &reg, &mask);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(gpio->regmap, reg, &val);
+	if (ret)
+		return ret;
+
+	if (!!(val & mask) ^ invert)
+		return GPIO_LINE_DIRECTION_OUT;
+	else
+		return GPIO_LINE_DIRECTION_IN;
+}
+
+static int gpio_regmap_set_direction(struct gpio_chip *chip,
+				     unsigned int offset, bool output)
+{
+	struct gpio_regmap *gpio = gpiochip_get_data(chip);
+	unsigned int base, val, reg, mask;
+	int invert, ret;
+
+	if (gpio->reg_dir_out_base) {
+		base = gpio_regmap_addr(gpio->reg_dir_out_base);
+		invert = 0;
+	} else if (gpio->reg_dir_in_base) {
+		base = gpio_regmap_addr(gpio->reg_dir_in_base);
+		invert = 1;
+	} else {
+		return -EOPNOTSUPP;
+	}
+
+	ret = gpio->reg_mask_xlate(gpio, base, offset, &reg, &mask);
+	if (ret)
+		return ret;
+
+	if (invert)
+		val = output ? 0 : mask;
+	else
+		val = output ? mask : 0;
+
+	return regmap_update_bits(gpio->regmap, reg, mask, val);
+}
+
+static int gpio_regmap_direction_input(struct gpio_chip *chip,
+				       unsigned int offset)
+{
+	return gpio_regmap_set_direction(chip, offset, false);
+}
+
+static int gpio_regmap_direction_output(struct gpio_chip *chip,
+					unsigned int offset, int value)
+{
+	gpio_regmap_set(chip, offset, value);
+
+	return gpio_regmap_set_direction(chip, offset, true);
+}
+
+void gpio_regmap_set_drvdata(struct gpio_regmap *gpio, void *data)
+{
+	gpio->driver_data = data;
+}
+EXPORT_SYMBOL_GPL(gpio_regmap_set_drvdata);
+
+void *gpio_regmap_get_drvdata(struct gpio_regmap *gpio)
+{
+	return gpio->driver_data;
+}
+EXPORT_SYMBOL_GPL(gpio_regmap_get_drvdata);
+
+/**
+ * gpio_regmap_register() - Register a generic regmap GPIO controller
+ * @config: configuration for gpio_regmap
+ *
+ * Return: A pointer to the registered gpio_regmap or ERR_PTR error value.
+ */
+struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config)
+{
+	struct gpio_regmap *gpio;
+	struct gpio_chip *chip;
+	int ret;
+
+	if (!config->parent)
+		return ERR_PTR(-EINVAL);
+
+	if (!config->ngpio)
+		return ERR_PTR(-EINVAL);
+
+	/* we need at least one */
+	if (!config->reg_dat_base && !config->reg_set_base)
+		return ERR_PTR(-EINVAL);
+
+	/* if we have a direction register we need both input and output */
+	if ((config->reg_dir_out_base || config->reg_dir_in_base) &&
+	    (!config->reg_dat_base || !config->reg_set_base))
+		return ERR_PTR(-EINVAL);
+
+	/* we don't support having both registers simultaneously for now */
+	if (config->reg_dir_out_base && config->reg_dir_in_base)
+		return ERR_PTR(-EINVAL);
+
+	gpio = kzalloc(sizeof(*gpio), GFP_KERNEL);
+	if (!gpio)
+		return ERR_PTR(-ENOMEM);
+
+	gpio->parent = config->parent;
+	gpio->regmap = config->regmap;
+	gpio->ngpio_per_reg = config->ngpio_per_reg;
+	gpio->reg_stride = config->reg_stride;
+	gpio->reg_mask_xlate = config->reg_mask_xlate;
+	gpio->reg_dat_base = config->reg_dat_base;
+	gpio->reg_set_base = config->reg_set_base;
+	gpio->reg_clr_base = config->reg_clr_base;
+	gpio->reg_dir_in_base = config->reg_dir_in_base;
+	gpio->reg_dir_out_base = config->reg_dir_out_base;
+
+	/* if not set, assume there is only one register */
+	if (!gpio->ngpio_per_reg)
+		gpio->ngpio_per_reg = config->ngpio;
+
+	/* if not set, assume they are consecutive */
+	if (!gpio->reg_stride)
+		gpio->reg_stride = 1;
+
+	if (!gpio->reg_mask_xlate)
+		gpio->reg_mask_xlate = gpio_regmap_simple_xlate;
+
+	chip = &gpio->gpio_chip;
+	chip->parent = config->parent;
+	chip->base = -1;
+	chip->ngpio = config->ngpio;
+	chip->names = config->names;
+	chip->label = config->label ?: dev_name(config->parent);
+
+	/*
+	 * If our regmap is fast_io we should probably set can_sleep to false.
+	 * Right now, the regmap doesn't save this property, nor is there any
+	 * access function for it.
+	 * The only regmap type which uses fast_io is regmap-mmio. For now,
+	 * assume a safe default of true here.
+	 */
+	chip->can_sleep = true;
+
+	chip->get = gpio_regmap_get;
+	if (gpio->reg_set_base && gpio->reg_clr_base)
+		chip->set = gpio_regmap_set_with_clear;
+	else if (gpio->reg_set_base)
+		chip->set = gpio_regmap_set;
+
+	if (gpio->reg_dir_in_base || gpio->reg_dir_out_base) {
+		chip->get_direction = gpio_regmap_get_direction;
+		chip->direction_input = gpio_regmap_direction_input;
+		chip->direction_output = gpio_regmap_direction_output;
+	}
+
+	ret = gpiochip_add_data(chip, gpio);
+	if (ret < 0)
+		goto err_free_gpio;
+
+	if (config->irq_domain) {
+		ret = gpiochip_irqchip_add_domain(chip, config->irq_domain);
+		if (ret)
+			goto err_remove_gpiochip;
+	}
+
+	return gpio;
+
+err_remove_gpiochip:
+	gpiochip_remove(chip);
+err_free_gpio:
+	kfree(gpio);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(gpio_regmap_register);
+
+/**
+ * gpio_regmap_unregister() - Unregister a generic regmap GPIO controller
+ * @gpio: gpio_regmap device to unregister
+ */
+void gpio_regmap_unregister(struct gpio_regmap *gpio)
+{
+	gpiochip_remove(&gpio->gpio_chip);
+	kfree(gpio);
+}
+EXPORT_SYMBOL_GPL(gpio_regmap_unregister);
+
+static void devm_gpio_regmap_unregister(struct device *dev, void *res)
+{
+	gpio_regmap_unregister(*(struct gpio_regmap **)res);
+}
+
+/**
+ * devm_gpio_regmap_register() - resource managed gpio_regmap_register()
+ * @dev: device that is registering this GPIO device
+ * @config: configuration for gpio_regmap
+ *
+ * Managed gpio_regmap_register(). For generic regmap GPIO device registered by
+ * this function, gpio_regmap_unregister() is automatically called on driver
+ * detach. See gpio_regmap_register() for more information.
+ *
+ * Return: A pointer to the registered gpio_regmap or ERR_PTR error value.
+ */
+struct gpio_regmap *devm_gpio_regmap_register(struct device *dev,
+					      const struct gpio_regmap_config *config)
+{
+	struct gpio_regmap **ptr, *gpio;
+
+	ptr = devres_alloc(devm_gpio_regmap_unregister, sizeof(*ptr),
+			   GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	gpio = gpio_regmap_register(config);
+	if (!IS_ERR(gpio)) {
+		*ptr = gpio;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
+	}
+
+	return gpio;
+}
+EXPORT_SYMBOL_GPL(devm_gpio_regmap_register);
+
+MODULE_AUTHOR("Michael Walle <michael@walle.cc>");
+MODULE_DESCRIPTION("GPIO generic regmap driver core");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-tegra186.c b/drivers/gpio/gpio-tegra186.c
index 79b553dc39a3..178e9128ded0 100644
--- a/drivers/gpio/gpio-tegra186.c
+++ b/drivers/gpio/gpio-tegra186.c
@@ -894,6 +894,7 @@ static const struct of_device_id tegra186_gpio_of_match[] = {
 		/* sentinel */
 	}
 };
+MODULE_DEVICE_TABLE(of, tegra186_gpio_of_match);
 
 static struct platform_driver tegra186_gpio_driver = {
 	.driver = {
diff --git a/drivers/gpio/gpio-xgene-sb.c b/drivers/gpio/gpio-xgene-sb.c
index 25d86441666e..a809609ee957 100644
--- a/drivers/gpio/gpio-xgene-sb.c
+++ b/drivers/gpio/gpio-xgene-sb.c
@@ -10,8 +10,8 @@
 
 #include <linux/module.h>
 #include <linux/io.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/of_gpio.h>
 #include <linux/gpio/driver.h>
 #include <linux/acpi.h>
 
@@ -122,7 +122,7 @@ static int xgene_gpio_sb_to_irq(struct gpio_chip *gc, u32 gpio)
 	fwspec.fwnode = gc->parent->fwnode;
 	fwspec.param_count = 2;
 	fwspec.param[0] = GPIO_TO_HWIRQ(priv, gpio);
-	fwspec.param[1] = IRQ_TYPE_NONE;
+	fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
 	return irq_create_fwspec_mapping(&fwspec);
 }
 
@@ -290,10 +290,8 @@ static int xgene_gpio_sb_probe(struct platform_device *pdev)
 
 	dev_info(&pdev->dev, "X-Gene GPIO Standby driver registered\n");
 
-	if (priv->nirq > 0) {
-		/* Register interrupt handlers for gpio signaled acpi events */
-		acpi_gpiochip_request_interrupts(&priv->gc);
-	}
+	/* Register interrupt handlers for GPIO signaled ACPI Events */
+	acpi_gpiochip_request_interrupts(&priv->gc);
 
 	return ret;
 }
@@ -302,9 +300,7 @@ static int xgene_gpio_sb_remove(struct platform_device *pdev)
 {
 	struct xgene_gpio_sb *priv = platform_get_drvdata(pdev);
 
-	if (priv->nirq > 0) {
-		acpi_gpiochip_free_interrupts(&priv->gc);
-	}
+	acpi_gpiochip_free_interrupts(&priv->gc);
 
 	irq_domain_remove(priv->irq_domain);
 
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index 0017367e94ee..9276051663da 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -1353,7 +1353,7 @@ int acpi_gpio_count(struct device *dev, const char *con_id)
 }
 
 /* Run deferred acpi_gpiochip_request_irqs() */
-static int acpi_gpio_handle_deferred_request_irqs(void)
+static int __init acpi_gpio_handle_deferred_request_irqs(void)
 {
 	struct acpi_gpio_chip *acpi_gpio, *tmp;
 
@@ -1371,7 +1371,7 @@ static int acpi_gpio_handle_deferred_request_irqs(void)
 /* We must use _sync so that this runs after the first deferred_probe run */
 late_initcall_sync(acpi_gpio_handle_deferred_request_irqs);
 
-static const struct dmi_system_id gpiolib_acpi_quirks[] = {
+static const struct dmi_system_id gpiolib_acpi_quirks[] __initconst = {
 	{
 		/*
 		 * The Minix Neo Z83-4 has a micro-USB-B id-pin handler for
@@ -1455,7 +1455,7 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] = {
 	{} /* Terminating entry */
 };
 
-static int acpi_gpio_setup_params(void)
+static int __init acpi_gpio_setup_params(void)
 {
 	const struct acpi_gpiolib_dmi_quirk *quirk = NULL;
 	const struct dmi_system_id *id;
diff --git a/drivers/gpio/gpiolib-devprop.c b/drivers/gpio/gpiolib-devprop.c
index 53781b253986..26741032fa9e 100644
--- a/drivers/gpio/gpiolib-devprop.c
+++ b/drivers/gpio/gpiolib-devprop.c
@@ -37,8 +37,11 @@ void devprop_gpiochip_set_names(struct gpio_chip *chip,
 	if (count < 0)
 		return;
 
-	if (count > gdev->ngpio)
+	if (count > gdev->ngpio) {
+		dev_warn(&gdev->dev, "gpio-line-names is length %d but should be at most length %d",
+			 count, gdev->ngpio);
 		count = gdev->ngpio;
+	}
 
 	names = kcalloc(count, sizeof(*names), GFP_KERNEL);
 	if (!names)
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index ccc449df3792..219eb0054233 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -344,6 +344,12 @@ struct gpio_desc *gpiod_get_from_of_node(struct device_node *node,
 	if (transitory)
 		lflags |= GPIO_TRANSITORY;
 
+	if (flags & OF_GPIO_PULL_UP)
+		lflags |= GPIO_PULL_UP;
+
+	if (flags & OF_GPIO_PULL_DOWN)
+		lflags |= GPIO_PULL_DOWN;
+
 	ret = gpiod_configure_flags(desc, propname, lflags, dflags);
 	if (ret < 0) {
 		gpiod_put(desc);
@@ -460,6 +466,24 @@ static struct gpio_desc *of_find_arizona_gpio(struct device *dev,
 	return of_get_named_gpiod_flags(dev->of_node, con_id, 0, of_flags);
 }
 
+static struct gpio_desc *of_find_usb_gpio(struct device *dev,
+					  const char *con_id,
+					  enum of_gpio_flags *of_flags)
+{
+	/*
+	 * Currently this USB quirk is only for the Fairchild FUSB302 host which is using
+	 * an undocumented DT GPIO line named "fcs,int_n" without the compulsory "-gpios"
+	 * suffix.
+	 */
+	if (!IS_ENABLED(CONFIG_TYPEC_FUSB302))
+		return ERR_PTR(-ENOENT);
+
+	if (!con_id || strcmp(con_id, "fcs,int_n"))
+		return ERR_PTR(-ENOENT);
+
+	return of_get_named_gpiod_flags(dev->of_node, con_id, 0, of_flags);
+}
+
 struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
 			       unsigned int idx, unsigned long *flags)
 {
@@ -504,6 +528,9 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
 	if (PTR_ERR(desc) == -ENOENT)
 		desc = of_find_arizona_gpio(dev, con_id, &of_flags);
 
+	if (PTR_ERR(desc) == -ENOENT)
+		desc = of_find_usb_gpio(dev, con_id, &of_flags);
+
 	if (IS_ERR(desc))
 		return desc;
 
@@ -585,6 +612,10 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np,
 		*lflags |= GPIO_ACTIVE_LOW;
 	if (xlate_flags & OF_GPIO_TRANSITORY)
 		*lflags |= GPIO_TRANSITORY;
+	if (xlate_flags & OF_GPIO_PULL_UP)
+		*lflags |= GPIO_PULL_UP;
+	if (xlate_flags & OF_GPIO_PULL_DOWN)
+		*lflags |= GPIO_PULL_DOWN;
 
 	if (of_property_read_bool(np, "input"))
 		*dflags |= GPIOD_IN;
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index c14f0784274a..4fa075d49fbc 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -296,6 +296,9 @@ static int gpiodev_add_to_list(struct gpio_device *gdev)
 
 /*
  * Convert a GPIO name to its descriptor
+ * Note that there is no guarantee that GPIO names are globally unique!
+ * Hence this function will return, if it exists, a reference to the first GPIO
+ * line found that matches the given name.
  */
 static struct gpio_desc *gpio_name_to_desc(const char * const name)
 {
@@ -329,10 +332,12 @@ static struct gpio_desc *gpio_name_to_desc(const char * const name)
 }
 
 /*
- * Takes the names from gc->names and checks if they are all unique. If they
- * are, they are assigned to their gpio descriptors.
+ * Take the names from gc->names and assign them to their GPIO descriptors.
+ * Warn if a name is already used for a GPIO line on a different GPIO chip.
  *
- * Warning if one of the names is already used for a different GPIO.
+ * Note that:
+ *   1. Non-unique names are still accepted,
+ *   2. Name collisions within the same GPIO chip are not reported.
  */
 static int gpiochip_set_desc_names(struct gpio_chip *gc)
 {
@@ -1267,8 +1272,7 @@ static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		if (copy_to_user(ip, &chipinfo, sizeof(chipinfo)))
 			return -EFAULT;
 		return 0;
-	} else if (cmd == GPIO_GET_LINEINFO_IOCTL ||
-		   cmd == GPIO_GET_LINEINFO_WATCH_IOCTL) {
+	} else if (cmd == GPIO_GET_LINEINFO_IOCTL) {
 		struct gpioline_info lineinfo;
 
 		if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
@@ -1280,23 +1284,37 @@ static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 		hwgpio = gpio_chip_hwgpio(desc);
 
-		if (cmd == GPIO_GET_LINEINFO_WATCH_IOCTL &&
-		    test_bit(hwgpio, priv->watched_lines))
-			return -EBUSY;
-
 		gpio_desc_to_lineinfo(desc, &lineinfo);
 
 		if (copy_to_user(ip, &lineinfo, sizeof(lineinfo)))
 			return -EFAULT;
-
-		if (cmd == GPIO_GET_LINEINFO_WATCH_IOCTL)
-			set_bit(hwgpio, priv->watched_lines);
-
 		return 0;
 	} else if (cmd == GPIO_GET_LINEHANDLE_IOCTL) {
 		return linehandle_create(gdev, ip);
 	} else if (cmd == GPIO_GET_LINEEVENT_IOCTL) {
 		return lineevent_create(gdev, ip);
+	} else if (cmd == GPIO_GET_LINEINFO_WATCH_IOCTL) {
+		struct gpioline_info lineinfo;
+
+		if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
+			return -EFAULT;
+
+		desc = gpiochip_get_desc(gc, lineinfo.line_offset);
+		if (IS_ERR(desc))
+			return PTR_ERR(desc);
+
+		hwgpio = gpio_chip_hwgpio(desc);
+
+		if (test_bit(hwgpio, priv->watched_lines))
+			return -EBUSY;
+
+		gpio_desc_to_lineinfo(desc, &lineinfo);
+
+		if (copy_to_user(ip, &lineinfo, sizeof(lineinfo)))
+			return -EFAULT;
+
+		set_bit(hwgpio, priv->watched_lines);
+		return 0;
 	} else if (cmd == GPIO_GET_LINEINFO_UNWATCH_IOCTL) {
 		if (copy_from_user(&offset, ip, sizeof(offset)))
 			return -EFAULT;
@@ -1538,9 +1556,8 @@ static int gpiochip_setup_dev(struct gpio_device *gdev)
 
 	/* From this point, the .release() function cleans up gpio_device */
 	gdev->dev.release = gpiodevice_release;
-	pr_debug("%s: registered GPIOs %d to %d on device: %s (%s)\n",
-		 __func__, gdev->base, gdev->base + gdev->ngpio - 1,
-		 dev_name(&gdev->dev), gdev->chip->label ? : "generic");
+	dev_dbg(&gdev->dev, "registered GPIOs %d to %d on %s\n", gdev->base,
+		gdev->base + gdev->ngpio - 1, gdev->chip->label ? : "generic");
 
 	return 0;
 
@@ -1556,8 +1573,8 @@ static void gpiochip_machine_hog(struct gpio_chip *gc, struct gpiod_hog *hog)
 
 	desc = gpiochip_get_desc(gc, hog->chip_hwnum);
 	if (IS_ERR(desc)) {
-		pr_err("%s: unable to get GPIO desc: %ld\n",
-		       __func__, PTR_ERR(desc));
+		chip_err(gc, "%s: unable to get GPIO desc: %ld\n", __func__,
+			 PTR_ERR(desc));
 		return;
 	}
 
@@ -1566,8 +1583,8 @@ static void gpiochip_machine_hog(struct gpio_chip *gc, struct gpiod_hog *hog)
 
 	rv = gpiod_hog(desc, hog->line_name, hog->lflags, hog->dflags);
 	if (rv)
-		pr_err("%s: unable to hog GPIO line (%s:%u): %d\n",
-		       __func__, gc->label, hog->chip_hwnum, rv);
+		gpiod_err(desc, "%s: unable to hog GPIO line (%s:%u): %d\n",
+			  __func__, gc->label, hog->chip_hwnum, rv);
 }
 
 static void machine_gpiochip_add(struct gpio_chip *gc)
@@ -1592,8 +1609,8 @@ static void gpiochip_setup_devs(void)
 	list_for_each_entry(gdev, &gpio_devices, list) {
 		ret = gpiochip_setup_dev(gdev);
 		if (ret)
-			pr_err("%s: Failed to initialize gpio device (%d)\n",
-			       dev_name(&gdev->dev), ret);
+			dev_err(&gdev->dev,
+				"Failed to initialize gpio device (%d)\n", ret);
 	}
 }
 
@@ -2461,32 +2478,37 @@ static void gpiochip_irq_relres(struct irq_data *d)
 	gpiochip_relres_irq(gc, d->hwirq);
 }
 
+static void gpiochip_irq_mask(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	if (gc->irq.irq_mask)
+		gc->irq.irq_mask(d);
+	gpiochip_disable_irq(gc, d->hwirq);
+}
+
+static void gpiochip_irq_unmask(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+
+	gpiochip_enable_irq(gc, d->hwirq);
+	if (gc->irq.irq_unmask)
+		gc->irq.irq_unmask(d);
+}
+
 static void gpiochip_irq_enable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 
 	gpiochip_enable_irq(gc, d->hwirq);
-	if (gc->irq.irq_enable)
-		gc->irq.irq_enable(d);
-	else
-		gc->irq.chip->irq_unmask(d);
+	gc->irq.irq_enable(d);
 }
 
 static void gpiochip_irq_disable(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 
-	/*
-	 * Since we override .irq_disable() we need to mimic the
-	 * behaviour of __irq_disable() in irq/chip.c.
-	 * First call .irq_disable() if it exists, else mimic the
-	 * behaviour of mask_irq() which calls .irq_mask() if
-	 * it exists.
-	 */
-	if (gc->irq.irq_disable)
-		gc->irq.irq_disable(d);
-	else if (gc->irq.chip->irq_mask)
-		gc->irq.chip->irq_mask(d);
+	gc->irq.irq_disable(d);
 	gpiochip_disable_irq(gc, d->hwirq);
 }
 
@@ -2511,10 +2533,22 @@ static void gpiochip_set_irq_hooks(struct gpio_chip *gc)
 			  "detected irqchip that is shared with multiple gpiochips: please fix the driver.\n");
 		return;
 	}
-	gc->irq.irq_enable = irqchip->irq_enable;
-	gc->irq.irq_disable = irqchip->irq_disable;
-	irqchip->irq_enable = gpiochip_irq_enable;
-	irqchip->irq_disable = gpiochip_irq_disable;
+
+	if (irqchip->irq_disable) {
+		gc->irq.irq_disable = irqchip->irq_disable;
+		irqchip->irq_disable = gpiochip_irq_disable;
+	} else {
+		gc->irq.irq_mask = irqchip->irq_mask;
+		irqchip->irq_mask = gpiochip_irq_mask;
+	}
+
+	if (irqchip->irq_enable) {
+		gc->irq.irq_enable = irqchip->irq_enable;
+		irqchip->irq_enable = gpiochip_irq_enable;
+	} else {
+		gc->irq.irq_unmask = irqchip->irq_unmask;
+		irqchip->irq_unmask = gpiochip_irq_unmask;
+	}
 }
 
 /**
@@ -2702,7 +2736,7 @@ int gpiochip_irqchip_add_key(struct gpio_chip *gc,
 		return -EINVAL;
 
 	if (!gc->parent) {
-		pr_err("missing gpiochip .dev parent pointer\n");
+		chip_err(gc, "missing gpiochip .dev parent pointer\n");
 		return -EINVAL;
 	}
 	gc->irq.threaded = threaded;
@@ -2752,6 +2786,26 @@ int gpiochip_irqchip_add_key(struct gpio_chip *gc,
 }
 EXPORT_SYMBOL_GPL(gpiochip_irqchip_add_key);
 
+/**
+ * gpiochip_irqchip_add_domain() - adds an irqdomain to a gpiochip
+ * @gc: the gpiochip to add the irqchip to
+ * @domain: the irqdomain to add to the gpiochip
+ *
+ * This function adds an IRQ domain to the gpiochip.
+ */
+int gpiochip_irqchip_add_domain(struct gpio_chip *gc,
+				struct irq_domain *domain)
+{
+	if (!domain)
+		return -EINVAL;
+
+	gc->to_irq = gpiochip_to_irq;
+	gc->irq.domain = domain;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gpiochip_irqchip_add_domain);
+
 #else /* CONFIG_GPIOLIB_IRQCHIP */
 
 static inline int gpiochip_add_irqchip(struct gpio_chip *gc,
@@ -4653,7 +4707,7 @@ static struct gpio_desc *gpiod_find(struct device *dev, const char *con_id,
 	if (!table)
 		return desc;
 
-	for (p = &table->table[0]; p->chip_label; p++) {
+	for (p = &table->table[0]; p->key; p++) {
 		struct gpio_chip *gc;
 
 		/* idx must always match exactly */
@@ -4664,18 +4718,30 @@ static struct gpio_desc *gpiod_find(struct device *dev, const char *con_id,
 		if (p->con_id && (!con_id || strcmp(p->con_id, con_id)))
 			continue;
 
-		gc = find_chip_by_name(p->chip_label);
+		if (p->chip_hwnum == U16_MAX) {
+			desc = gpio_name_to_desc(p->key);
+			if (desc) {
+				*flags = p->flags;
+				return desc;
+			}
+
+			dev_warn(dev, "cannot find GPIO line %s, deferring\n",
+				 p->key);
+			return ERR_PTR(-EPROBE_DEFER);
+		}
+
+		gc = find_chip_by_name(p->key);
 
 		if (!gc) {
 			/*
 			 * As the lookup table indicates a chip with
-			 * p->chip_label should exist, assume it may
+			 * p->key should exist, assume it may
 			 * still appear later and let the interested
 			 * consumer be probed again or let the Deferred
 			 * Probe infrastructure handle the error.
 			 */
 			dev_warn(dev, "cannot find GPIO chip %s, deferring\n",
-				 p->chip_label);
+				 p->key);
 			return ERR_PTR(-EPROBE_DEFER);
 		}
 
@@ -4706,7 +4772,7 @@ static int platform_gpio_count(struct device *dev, const char *con_id)
 	if (!table)
 		return -ENOENT;
 
-	for (p = &table->table[0]; p->chip_label; p++) {
+	for (p = &table->table[0]; p->key; p++) {
 		if ((con_id && p->con_id && !strcmp(con_id, p->con_id)) ||
 		    (!con_id && !p->con_id))
 			count++;
@@ -4877,7 +4943,7 @@ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
 
 	/* No particular flag request, return here... */
 	if (!(dflags & GPIOD_FLAGS_BIT_DIR_SET)) {
-		pr_debug("no flags found for %s\n", con_id);
+		gpiod_dbg(desc, "no flags found for %s\n", con_id);
 		return 0;
 	}
 
@@ -5108,8 +5174,7 @@ int gpiod_hog(struct gpio_desc *desc, const char *name,
 	/* Mark GPIO as hogged so it can be identified and removed later */
 	set_bit(FLAG_IS_HOGGED, &desc->flags);
 
-	pr_info("GPIO line %d (%s) hogged as %s%s\n",
-		desc_to_gpio(desc), name,
+	gpiod_info(desc, "hogged as %s%s\n",
 		(dflags & GPIOD_FLAGS_BIT_DIR_OUT) ? "output" : "input",
 		(dflags & GPIOD_FLAGS_BIT_DIR_OUT) ?
 		  (dflags & GPIOD_FLAGS_BIT_DIR_VAL) ? "/high" : "/low" : "");
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index 853ce681b4a4..9ed242316414 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -81,8 +81,7 @@ struct gpio_array {
 	unsigned long		invert_mask[];
 };
 
-struct gpio_desc *gpiochip_get_desc(struct gpio_chip *chip,
-				    unsigned int hwnum);
+struct gpio_desc *gpiochip_get_desc(struct gpio_chip *gc, unsigned int hwnum);
 int gpiod_get_array_value_complex(bool raw, bool can_sleep,
 				  unsigned int array_size,
 				  struct gpio_desc **desc_array,
@@ -163,18 +162,18 @@ static inline int gpio_chip_hwgpio(const struct gpio_desc *desc)
 
 /* With chip prefix */
 
-#define chip_emerg(chip, fmt, ...)					\
-	dev_emerg(&chip->gpiodev->dev, "(%s): " fmt, chip->label, ##__VA_ARGS__)
-#define chip_crit(chip, fmt, ...)					\
-	dev_crit(&chip->gpiodev->dev, "(%s): " fmt, chip->label, ##__VA_ARGS__)
-#define chip_err(chip, fmt, ...)					\
-	dev_err(&chip->gpiodev->dev, "(%s): " fmt, chip->label, ##__VA_ARGS__)
-#define chip_warn(chip, fmt, ...)					\
-	dev_warn(&chip->gpiodev->dev, "(%s): " fmt, chip->label, ##__VA_ARGS__)
-#define chip_info(chip, fmt, ...)					\
-	dev_info(&chip->gpiodev->dev, "(%s): " fmt, chip->label, ##__VA_ARGS__)
-#define chip_dbg(chip, fmt, ...)					\
-	dev_dbg(&chip->gpiodev->dev, "(%s): " fmt, chip->label, ##__VA_ARGS__)
+#define chip_emerg(gc, fmt, ...)					\
+	dev_emerg(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__)
+#define chip_crit(gc, fmt, ...)					\
+	dev_crit(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__)
+#define chip_err(gc, fmt, ...)					\
+	dev_err(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__)
+#define chip_warn(gc, fmt, ...)					\
+	dev_warn(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__)
+#define chip_info(gc, fmt, ...)					\
+	dev_info(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__)
+#define chip_dbg(gc, fmt, ...)					\
+	dev_dbg(&gc->gpiodev->dev, "(%s): " fmt, gc->label, ##__VA_ARGS__)
 
 #ifdef CONFIG_GPIO_SYSFS
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 3f2b695cf19e..ffe149aafc39 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -27,6 +27,7 @@
 
 #include <linux/types.h>
 #include <linux/mm.h>
+#include <linux/kthread.h>
 #include <linux/workqueue.h>
 #include <kgd_kfd_interface.h>
 #include <drm/ttm/ttm_execbuf_util.h>
@@ -186,7 +187,7 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
  * disabled. The memory must be pinned and mapped to the hardware when
  * this is called in hqd_load functions, so it should never fault in
  * the first place. This resolves a circular lock dependency involving
- * four locks, including the DQM lock and mmap_sem.
+ * four locks, including the DQM lock and mmap_lock.
  */
 #define read_user_wptr(mmptr, wptr, dst)				\
 	({								\
@@ -195,10 +196,10 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
 			pagefault_disable();				\
 			if ((mmptr) == current->mm) {			\
 				valid = !get_user((dst), (wptr));	\
-			} else if (current->mm == NULL) {		\
-				use_mm(mmptr);				\
+			} else if (current->flags & PF_KTHREAD) {	\
+				kthread_use_mm(mmptr);			\
 				valid = !get_user((dst), (wptr));	\
-				unuse_mm(mmptr);			\
+				kthread_unuse_mm(mmptr);		\
 			}						\
 			pagefault_enable();				\
 		}							\
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
index 6529caca88fe..35d4a5ab0228 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
@@ -22,7 +22,6 @@
 #include <linux/module.h>
 #include <linux/fdtable.h>
 #include <linux/uaccess.h>
-#include <linux/mmu_context.h>
 #include <linux/firmware.h>
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
index 691c89705bcd..bf927f432506 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
@@ -19,7 +19,6 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  */
-#include <linux/mmu_context.h>
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
 #include "gc/gc_10_1_0_offset.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index 0b7e78748540..744366c7ee85 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -20,8 +20,6 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <linux/mmu_context.h>
-
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
 #include "cikd.h"
@@ -237,7 +235,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 			     CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
 	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
 
-	/* read_user_ptr may take the mm->mmap_sem.
+	/* read_user_ptr may take the mm->mmap_lock.
 	 * release srbm_mutex to avoid circular dependency between
 	 * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
 	 */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index ccd635b812b5..feab4cc6e836 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -20,8 +20,6 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <linux/mmu_context.h>
-
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
 #include "gfx_v8_0.h"
@@ -224,7 +222,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 			     CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
 	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
 
-	/* read_user_ptr may take the mm->mmap_sem.
+	/* read_user_ptr may take the mm->mmap_lock.
 	 * release srbm_mutex to avoid circular dependency between
 	 * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex.
 	 */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index df841c2ac5e7..c7fd0c47b254 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -19,8 +19,6 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  */
-#include <linux/mmu_context.h>
-
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
 #include "gc/gc_9_0_offset.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 68e6e1bc8f3a..b91b5171270f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1393,9 +1393,9 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 	 * concurrently and the queues are actually stopped
 	 */
 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
-		down_write(&current->mm->mmap_sem);
+		mmap_write_lock(current->mm);
 		is_invalid_userptr = atomic_read(&mem->invalid);
-		up_write(&current->mm->mmap_sem);
+		mmap_write_unlock(current->mm);
 	}
 
 	mutex_lock(&mem->lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index d7646cbce346..16596a9ccabe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -163,6 +163,9 @@ static ssize_t amdgpu_get_power_dpm_state(struct device *dev,
 	enum amd_pm_state_type pm;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -196,6 +199,9 @@ static ssize_t amdgpu_set_power_dpm_state(struct device *dev,
 	enum amd_pm_state_type  state;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (strncmp("battery", buf, strlen("battery")) == 0)
 		state = POWER_STATE_TYPE_BATTERY;
 	else if (strncmp("balanced", buf, strlen("balanced")) == 0)
@@ -297,6 +303,9 @@ static ssize_t amdgpu_get_power_dpm_force_performance_level(struct device *dev,
 	enum amd_dpm_forced_level level = 0xff;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -334,6 +343,9 @@ static ssize_t amdgpu_set_power_dpm_force_performance_level(struct device *dev,
 	enum amd_dpm_forced_level current_level = 0xff;
 	int ret = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (strncmp("low", buf, strlen("low")) == 0) {
 		level = AMD_DPM_FORCED_LEVEL_LOW;
 	} else if (strncmp("high", buf, strlen("high")) == 0) {
@@ -433,6 +445,9 @@ static ssize_t amdgpu_get_pp_num_states(struct device *dev,
 	struct pp_states_info data;
 	int i, buf_len, ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -472,6 +487,9 @@ static ssize_t amdgpu_get_pp_cur_state(struct device *dev,
 	enum amd_pm_state_type pm = 0;
 	int i = 0, ret = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -508,6 +526,9 @@ static ssize_t amdgpu_get_pp_force_state(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (adev->pp_force_state_enabled)
 		return amdgpu_get_pp_cur_state(dev, attr, buf);
 	else
@@ -525,6 +546,9 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
 	unsigned long idx;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (strlen(buf) == 1)
 		adev->pp_force_state_enabled = false;
 	else if (is_support_sw_smu(adev))
@@ -580,6 +604,9 @@ static ssize_t amdgpu_get_pp_table(struct device *dev,
 	char *table = NULL;
 	int size, ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -619,6 +646,9 @@ static ssize_t amdgpu_set_pp_table(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	int ret = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -666,7 +696,7 @@ static ssize_t amdgpu_set_pp_table(struct device *dev,
  * default power levels, write "r" (reset) to the file to reset them.
  *
  *
- * < For Vega20 >
+ * < For Vega20 and newer ASICs >
  *
  * Reading the file will display:
  *
@@ -721,6 +751,9 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev,
 	const char delimiter[3] = {' ', '\n', '\0'};
 	uint32_t type;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (count > 127)
 		return -EINVAL;
 
@@ -810,6 +843,9 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -859,6 +895,9 @@ static ssize_t amdgpu_set_pp_features(struct device *dev,
 	uint64_t featuremask;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = kstrtou64(buf, 0, &featuremask);
 	if (ret)
 		return -EINVAL;
@@ -899,6 +938,9 @@ static ssize_t amdgpu_get_pp_features(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -955,6 +997,9 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1018,6 +1063,9 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
 	int ret;
 	uint32_t mask = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = amdgpu_read_mask(buf, count, &mask);
 	if (ret)
 		return ret;
@@ -1049,6 +1097,9 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1076,6 +1127,9 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
 	uint32_t mask = 0;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = amdgpu_read_mask(buf, count, &mask);
 	if (ret)
 		return ret;
@@ -1107,6 +1161,9 @@ static ssize_t amdgpu_get_pp_dpm_socclk(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1134,6 +1191,9 @@ static ssize_t amdgpu_set_pp_dpm_socclk(struct device *dev,
 	int ret;
 	uint32_t mask = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = amdgpu_read_mask(buf, count, &mask);
 	if (ret)
 		return ret;
@@ -1167,6 +1227,9 @@ static ssize_t amdgpu_get_pp_dpm_fclk(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1194,6 +1257,9 @@ static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev,
 	int ret;
 	uint32_t mask = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = amdgpu_read_mask(buf, count, &mask);
 	if (ret)
 		return ret;
@@ -1227,6 +1293,9 @@ static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1254,6 +1323,9 @@ static ssize_t amdgpu_set_pp_dpm_dcefclk(struct device *dev,
 	int ret;
 	uint32_t mask = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = amdgpu_read_mask(buf, count, &mask);
 	if (ret)
 		return ret;
@@ -1287,6 +1359,9 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1314,6 +1389,9 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
 	int ret;
 	uint32_t mask = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = amdgpu_read_mask(buf, count, &mask);
 	if (ret)
 		return ret;
@@ -1347,6 +1425,9 @@ static ssize_t amdgpu_get_pp_sclk_od(struct device *dev,
 	uint32_t value = 0;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1372,6 +1453,9 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
 	int ret;
 	long int value;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
@@ -1410,6 +1494,9 @@ static ssize_t amdgpu_get_pp_mclk_od(struct device *dev,
 	uint32_t value = 0;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1435,6 +1522,9 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
 	int ret;
 	long int value;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = kstrtol(buf, 0, &value);
 
 	if (ret)
@@ -1493,6 +1583,9 @@ static ssize_t amdgpu_get_pp_power_profile_mode(struct device *dev,
 	ssize_t size;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -1528,6 +1621,9 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev,
 	long int profile_mode = 0;
 	const char delimiter[3] = {' ', '\n', '\0'};
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	tmp[0] = *(buf);
 	tmp[1] = '\0';
 	ret = kstrtol(tmp, 0, &profile_mode);
@@ -1572,7 +1668,7 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev,
 }
 
 /**
- * DOC: busy_percent
+ * DOC: gpu_busy_percent
  *
  * The amdgpu driver provides a sysfs API for reading how busy the GPU
  * is as a percentage.  The file gpu_busy_percent is used for this.
@@ -1587,6 +1683,9 @@ static ssize_t amdgpu_get_gpu_busy_percent(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	int r, value, size = sizeof(value);
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(ddev->dev);
 	if (r < 0)
 		return r;
@@ -1620,6 +1719,9 @@ static ssize_t amdgpu_get_mem_busy_percent(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	int r, value, size = sizeof(value);
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(ddev->dev);
 	if (r < 0)
 		return r;
@@ -1658,6 +1760,9 @@ static ssize_t amdgpu_get_pcie_bw(struct device *dev,
 	uint64_t count0 = 0, count1 = 0;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (adev->flags & AMD_IS_APU)
 		return -ENODATA;
 
@@ -1694,6 +1799,9 @@ static ssize_t amdgpu_get_unique_id(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (adev->unique_id)
 		return snprintf(buf, PAGE_SIZE, "%016llx\n", adev->unique_id);
 
@@ -1888,6 +1996,9 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
 	int channel = to_sensor_dev_attr(attr)->index;
 	int r, temp = 0, size = sizeof(temp);
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (channel >= PP_TEMP_MAX)
 		return -EINVAL;
 
@@ -2019,6 +2130,9 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev,
 	u32 pwm_mode = 0;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(adev->ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -2050,6 +2164,9 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev,
 	int err, ret;
 	int value;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	err = kstrtoint(buf, 10, &value);
 	if (err)
 		return err;
@@ -2099,6 +2216,9 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev,
 	u32 value;
 	u32 pwm_mode;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	err = pm_runtime_get_sync(adev->ddev->dev);
 	if (err < 0)
 		return err;
@@ -2148,6 +2268,9 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev,
 	int err;
 	u32 speed = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	err = pm_runtime_get_sync(adev->ddev->dev);
 	if (err < 0)
 		return err;
@@ -2178,6 +2301,9 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev,
 	int err;
 	u32 speed = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	err = pm_runtime_get_sync(adev->ddev->dev);
 	if (err < 0)
 		return err;
@@ -2207,6 +2333,9 @@ static ssize_t amdgpu_hwmon_get_fan1_min(struct device *dev,
 	u32 size = sizeof(min_rpm);
 	int r;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -2232,6 +2361,9 @@ static ssize_t amdgpu_hwmon_get_fan1_max(struct device *dev,
 	u32 size = sizeof(max_rpm);
 	int r;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -2256,6 +2388,9 @@ static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev,
 	int err;
 	u32 rpm = 0;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	err = pm_runtime_get_sync(adev->ddev->dev);
 	if (err < 0)
 		return err;
@@ -2285,6 +2420,9 @@ static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev,
 	u32 value;
 	u32 pwm_mode;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	err = pm_runtime_get_sync(adev->ddev->dev);
 	if (err < 0)
 		return err;
@@ -2331,6 +2469,9 @@ static ssize_t amdgpu_hwmon_get_fan1_enable(struct device *dev,
 	u32 pwm_mode = 0;
 	int ret;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	ret = pm_runtime_get_sync(adev->ddev->dev);
 	if (ret < 0)
 		return ret;
@@ -2363,6 +2504,9 @@ static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev,
 	int value;
 	u32 pwm_mode;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	err = kstrtoint(buf, 10, &value);
 	if (err)
 		return err;
@@ -2403,6 +2547,9 @@ static ssize_t amdgpu_hwmon_show_vddgfx(struct device *dev,
 	u32 vddgfx;
 	int r, size = sizeof(vddgfx);
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -2435,6 +2582,9 @@ static ssize_t amdgpu_hwmon_show_vddnb(struct device *dev,
 	u32 vddnb;
 	int r, size = sizeof(vddnb);
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	/* only APUs have vddnb */
 	if  (!(adev->flags & AMD_IS_APU))
 		return -EINVAL;
@@ -2472,6 +2622,9 @@ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev,
 	int r, size = sizeof(u32);
 	unsigned uw;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -2508,6 +2661,9 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev,
 	ssize_t size;
 	int r;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -2537,6 +2693,9 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev,
 	ssize_t size;
 	int r;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -2567,6 +2726,9 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev,
 	int err;
 	u32 value;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	if (amdgpu_sriov_vf(adev))
 		return -EINVAL;
 
@@ -2605,6 +2767,9 @@ static ssize_t amdgpu_hwmon_show_sclk(struct device *dev,
 	uint32_t sclk;
 	int r, size = sizeof(sclk);
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -2637,6 +2802,9 @@ static ssize_t amdgpu_hwmon_show_mclk(struct device *dev,
 	uint32_t mclk;
 	int r, size = sizeof(mclk);
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(adev->ddev->dev);
 	if (r < 0)
 		return r;
@@ -3497,6 +3665,9 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
 	u32 flags = 0;
 	int r;
 
+	if (adev->in_gpu_reset)
+		return -EPERM;
+
 	r = pm_runtime_get_sync(dev->dev);
 	if (r < 0)
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 9cbecd5ba814..e59c01a83dac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -910,7 +910,7 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
 		goto out_free_ranges;
 	}
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	vma = find_vma(mm, start);
 	if (unlikely(!vma || start < vma->vm_start)) {
 		r = -EFAULT;
@@ -921,15 +921,15 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages)
 		r = -EPERM;
 		goto out_unlock;
 	}
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
 
 retry:
 	range->notifier_seq = mmu_interval_read_begin(&bo->notifier);
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	r = hmm_range_fault(range);
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	if (unlikely(r)) {
 		/*
 		 * FIXME: This timeout should encompass the retry from
@@ -954,7 +954,7 @@ retry:
 	return 0;
 
 out_unlock:
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 out_free_pfns:
 	kvfree(range->hmm_pfns);
 out_free_ranges:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 15476fca8fa6..a9583b95fcc1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -901,7 +901,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 
 	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	vma = find_vma(mm, address);
 
 	memory_exception_data.gpu_id = dev->id;
@@ -924,7 +924,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 			memory_exception_data.failure.NoExecute = 0;
 	}
 
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	mmput(mm);
 
 	pr_debug("notpresent %d, noexecute %d, readonly %d\n",
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index f0587d94294d..fee60921fccf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -40,6 +40,7 @@
 #include <drm/drm_file.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_device.h>
+#include <drm/drm_ioctl.h>
 #include <kgd_kfd_interface.h>
 #include <linux/swap.h>
 
@@ -1076,7 +1077,7 @@ static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
 #if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
 	struct drm_device *ddev = kfd->ddev;
 
-	return devcgroup_check_permission(DEVCG_DEV_CHAR, ddev->driver->major,
+	return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
 					  ddev->render->index,
 					  DEVCG_ACC_WRITE | DEVCG_ACC_READ);
 #else
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index bdba0bfd6df1..7ced9f87be97 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -1356,7 +1356,7 @@ static int dm_late_init(void *handle)
 	unsigned int linear_lut[16];
 	int i;
 	struct dmcu *dmcu = NULL;
-	bool ret = false;
+	bool ret;
 
 	if (!adev->dm.fw_dmcu)
 		return detect_mst_link_for_all_connectors(adev->ddev);
@@ -1377,13 +1377,10 @@ static int dm_late_init(void *handle)
 	 */
 	params.min_abm_backlight = 0x28F;
 
-	/* todo will enable for navi10 */
-	if (adev->asic_type <= CHIP_RAVEN) {
-		ret = dmcu_load_iram(dmcu, params);
+	ret = dmcu_load_iram(dmcu, params);
 
-		if (!ret)
-			return -EINVAL;
-	}
+	if (!ret)
+		return -EINVAL;
 
 	return detect_mst_link_for_all_connectors(adev->ddev);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 45cfb7c45566..6f93a6ca4cf0 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -1016,9 +1016,17 @@ static void program_timing_sync(
 			}
 		}
 
-		/* set first pipe with plane as master */
+		/* set first unblanked pipe as master */
 		for (j = 0; j < group_size; j++) {
-			if (pipe_set[j]->plane_state) {
+			bool is_blanked;
+
+			if (pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked)
+				is_blanked =
+					pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked(pipe_set[j]->stream_res.opp);
+			else
+				is_blanked =
+					pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg);
+			if (!is_blanked) {
 				if (j == 0)
 					break;
 
@@ -1039,9 +1047,17 @@ static void program_timing_sync(
 				status->timing_sync_info.master = false;
 
 		}
-		/* remove any other pipes with plane as they have already been synced */
+		/* remove any other unblanked pipes as they have already been synced */
 		for (j = j + 1; j < group_size; j++) {
-			if (pipe_set[j]->plane_state) {
+			bool is_blanked;
+
+			if (pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked)
+				is_blanked =
+					pipe_set[j]->stream_res.opp->funcs->dpg_is_blanked(pipe_set[j]->stream_res.opp);
+			else
+				is_blanked =
+					pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg);
+			if (!is_blanked) {
 				group_size--;
 				pipe_set[j] = pipe_set[group_size];
 				j--;
@@ -2522,6 +2538,12 @@ void dc_commit_updates_for_stream(struct dc *dc,
 
 	copy_stream_update_to_stream(dc, context, stream, stream_update);
 
+	if (!dc->res_pool->funcs->validate_bandwidth(dc, context, false)) {
+		DC_ERROR("Mode validation failed for stream update!\n");
+		dc_release_state(context);
+		return;
+	}
+
 	commit_planes_for_stream(
 				dc,
 				srf_updates,
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/Makefile b/drivers/gpu/drm/amd/display/dc/dsc/Makefile
index 3f66868df171..ea29cf95d470 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dsc/Makefile
@@ -28,8 +28,6 @@ endif
 endif
 
 CFLAGS_$(AMDDALPATH)/dc/dsc/rc_calc.o := $(dsc_ccflags)
-CFLAGS_$(AMDDALPATH)/dc/dsc/rc_calc_dpi.o := $(dsc_ccflags)
-CFLAGS_$(AMDDALPATH)/dc/dsc/dc_dsc.o := $(dsc_ccflags)
 
 DSC = dc_dsc.o rc_calc.o rc_calc_dpi.o
 
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
index 0ea6662a1563..0c7f247bb7de 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
@@ -22,10 +22,12 @@
  * Author: AMD
  */
 
+#include <drm/drm_dsc.h>
 #include "dc_hw_types.h"
 #include "dsc.h"
 #include <drm/drm_dp_helper.h>
 #include "dc.h"
+#include "rc_calc.h"
 
 /* This module's internal functions */
 
@@ -304,22 +306,6 @@ static inline uint32_t dsc_div_by_10_round_up(uint32_t value)
 	return (value + 9) / 10;
 }
 
-static inline uint32_t calc_dsc_bpp_x16(uint32_t stream_bandwidth_kbps, uint32_t pix_clk_100hz, uint32_t bpp_increment_div)
-{
-	uint32_t dsc_target_bpp_x16;
-	float f_dsc_target_bpp;
-	float f_stream_bandwidth_100bps = stream_bandwidth_kbps * 10.0f;
-	uint32_t precision = bpp_increment_div; // bpp_increment_div is actually precision
-
-	f_dsc_target_bpp = f_stream_bandwidth_100bps / pix_clk_100hz;
-
-	// Round down to the nearest precision stop to bring it into DSC spec range
-	dsc_target_bpp_x16 = (uint32_t)(f_dsc_target_bpp * precision);
-	dsc_target_bpp_x16 = (dsc_target_bpp_x16 * 16) / precision;
-
-	return dsc_target_bpp_x16;
-}
-
 /* Get DSC bandwidth range based on [min_bpp, max_bpp] target bitrate range, and timing's pixel clock
  * and uncompressed bandwidth.
  */
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
index 03ae15946c6d..667afbc260f9 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c
@@ -23,6 +23,7 @@
  * Authors: AMD
  *
  */
+#include <drm/drm_dsc.h>
 
 #include "os_types.h"
 #include "rc_calc.h"
@@ -40,7 +41,8 @@
 	break
 
 
-void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc, enum max_min max_min, float bpp)
+static void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc,
+		       enum max_min max_min, float bpp)
 {
 	int mode = MODE_SELECT(444, 422, 420);
 	int sel = table_hash(mode, bpc, max_min);
@@ -85,7 +87,7 @@ void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc, enum ma
 	memcpy(qps, table[index].qps, sizeof(qp_set));
 }
 
-double dsc_roundf(double num)
+static double dsc_roundf(double num)
 {
 	if (num < 0.0)
 		num = num - 0.5;
@@ -95,7 +97,7 @@ double dsc_roundf(double num)
 	return (int)(num);
 }
 
-double dsc_ceil(double num)
+static double dsc_ceil(double num)
 {
 	double retval = (int)num;
 
@@ -105,7 +107,7 @@ double dsc_ceil(double num)
 	return (int)retval;
 }
 
-void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp)
+static void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp)
 {
 	int   *p = ofs;
 
@@ -160,7 +162,7 @@ void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp)
 	}
 }
 
-int median3(int a, int b, int c)
+static int median3(int a, int b, int c)
 {
 	if (a > b)
 		swap(a, b);
@@ -172,13 +174,25 @@ int median3(int a, int b, int c)
 	return b;
 }
 
-void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_comp bpc, float bpp, int slice_width, int slice_height, int minor_version)
+static void _do_calc_rc_params(struct rc_params *rc, enum colour_mode cm,
+			       enum bits_per_comp bpc, u8 drm_bpp,
+			       bool is_navite_422_or_420,
+			       int slice_width, int slice_height,
+			       int minor_version)
 {
+	float bpp;
 	float bpp_group;
 	float initial_xmit_delay_factor;
 	int padding_pixels;
 	int i;
 
+	bpp = ((float)drm_bpp / 16.0);
+	/* in native_422 or native_420 modes, the bits_per_pixel is double the
+	 * target bpp (the latter is what calc_rc_params expects)
+	 */
+	if (is_navite_422_or_420)
+		bpp /= 2.0;
+
 	rc->rc_quant_incr_limit0 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
 	rc->rc_quant_incr_limit1 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
 
@@ -251,3 +265,128 @@ void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_com
 	rc->rc_buf_thresh[13] = 8064;
 }
 
+static u32 _do_bytes_per_pixel_calc(int slice_width, u8 drm_bpp,
+				    bool is_navite_422_or_420)
+{
+	float bpp;
+	u32 bytes_per_pixel;
+	double d_bytes_per_pixel;
+
+	bpp = ((float)drm_bpp / 16.0);
+	d_bytes_per_pixel = dsc_ceil(bpp * slice_width / 8.0) / slice_width;
+	// TODO: Make sure the formula for calculating this is precise (ceiling
+	// vs. floor, and at what point they should be applied)
+	if (is_navite_422_or_420)
+		d_bytes_per_pixel /= 2;
+
+	bytes_per_pixel = (u32)dsc_ceil(d_bytes_per_pixel * 0x10000000);
+
+	return bytes_per_pixel;
+}
+
+static u32 _do_calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
+				u32 bpp_increment_div)
+{
+	u32 dsc_target_bpp_x16;
+	float f_dsc_target_bpp;
+	float f_stream_bandwidth_100bps;
+	// bpp_increment_div is actually precision
+	u32 precision = bpp_increment_div;
+
+	f_stream_bandwidth_100bps = stream_bandwidth_kbps * 10.0f;
+	f_dsc_target_bpp = f_stream_bandwidth_100bps / pix_clk_100hz;
+
+	// Round down to the nearest precision stop to bring it into DSC spec
+	// range
+	dsc_target_bpp_x16 = (u32)(f_dsc_target_bpp * precision);
+	dsc_target_bpp_x16 = (dsc_target_bpp_x16 * 16) / precision;
+
+	return dsc_target_bpp_x16;
+}
+
+/**
+ * calc_rc_params - reads the user's cmdline mode
+ * @rc: DC internal DSC parameters
+ * @pps: DRM struct with all required DSC values
+ *
+ * This function expects a drm_dsc_config data struct with all the required DSC
+ * values previously filled out by our driver and based on this information it
+ * computes some of the DSC values.
+ *
+ * @note This calculation requires float point operation, most of it executes
+ * under kernel_fpu_{begin,end}.
+ */
+void calc_rc_params(struct rc_params *rc, const struct drm_dsc_config *pps)
+{
+	enum colour_mode mode;
+	enum bits_per_comp bpc;
+	bool is_navite_422_or_420;
+	u8 drm_bpp = pps->bits_per_pixel;
+	int slice_width  = pps->slice_width;
+	int slice_height = pps->slice_height;
+
+	mode = pps->convert_rgb ? CM_RGB : (pps->simple_422  ? CM_444 :
+					   (pps->native_422  ? CM_422 :
+					    pps->native_420  ? CM_420 : CM_444));
+	bpc = (pps->bits_per_component == 8) ? BPC_8 : (pps->bits_per_component == 10)
+					     ? BPC_10 : BPC_12;
+
+	is_navite_422_or_420 = pps->native_422 || pps->native_420;
+
+	DC_FP_START();
+	_do_calc_rc_params(rc, mode, bpc, drm_bpp, is_navite_422_or_420,
+			   slice_width, slice_height,
+			   pps->dsc_version_minor);
+	DC_FP_END();
+}
+
+/**
+ * calc_dsc_bytes_per_pixel - calculate bytes per pixel
+ * @pps: DRM struct with all required DSC values
+ *
+ * Based on the information inside drm_dsc_config, this function calculates the
+ * total of bytes per pixel.
+ *
+ * @note This calculation requires float point operation, most of it executes
+ * under kernel_fpu_{begin,end}.
+ *
+ * Return:
+ * Return the number of bytes per pixel
+ */
+u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps)
+
+{
+	u32 ret;
+	u8 drm_bpp = pps->bits_per_pixel;
+	int slice_width  = pps->slice_width;
+	bool is_navite_422_or_420 = pps->native_422 || pps->native_420;
+
+	DC_FP_START();
+	ret = _do_bytes_per_pixel_calc(slice_width, drm_bpp,
+				       is_navite_422_or_420);
+	DC_FP_END();
+	return ret;
+}
+
+/**
+ * calc_dsc_bpp_x16 - retrieve the dsc bits per pixel
+ * @stream_bandwidth_kbps:
+ * @pix_clk_100hz:
+ * @bpp_increment_div:
+ *
+ * Calculate the total of bits per pixel for DSC configuration.
+ *
+ * @note This calculation requires float point operation, most of it executes
+ * under kernel_fpu_{begin,end}.
+ */
+u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
+		     u32 bpp_increment_div)
+{
+	u32 dsc_bpp;
+
+	DC_FP_START();
+	dsc_bpp =  _do_calc_dsc_bpp_x16(stream_bandwidth_kbps, pix_clk_100hz,
+					bpp_increment_div);
+	DC_FP_END();
+	return dsc_bpp;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
index b6b1f09c2009..21723fa6561e 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h
@@ -77,7 +77,10 @@ struct qp_entry {
 
 typedef struct qp_entry qp_table[];
 
-void calc_rc_params(struct rc_params *rc, enum colour_mode cm, enum bits_per_comp bpc, float bpp, int slice_width, int slice_height, int minor_version);
+void calc_rc_params(struct rc_params *rc, const struct drm_dsc_config *pps);
+u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps);
+u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
+		     u32 bpp_increment_div);
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
index 1f6e63b71456..ef830aded5b1 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
@@ -27,8 +27,6 @@
 #include "dscc_types.h"
 #include "rc_calc.h"
 
-double dsc_ceil(double num);
-
 static void copy_pps_fields(struct drm_dsc_config *to, const struct drm_dsc_config *from)
 {
 	to->line_buf_depth           = from->line_buf_depth;
@@ -100,34 +98,13 @@ static void copy_rc_to_cfg(struct drm_dsc_config *dsc_cfg, const struct rc_param
 
 int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps, struct dsc_parameters *dsc_params)
 {
-	enum colour_mode  mode = pps->convert_rgb ? CM_RGB :
-							(pps->simple_422  ? CM_444 :
-							(pps->native_422  ? CM_422 :
-							pps->native_420  ? CM_420 : CM_444));
-	enum bits_per_comp bpc = (pps->bits_per_component == 8) ? BPC_8 :
-							(pps->bits_per_component == 10) ? BPC_10 : BPC_12;
-	float            bpp = ((float) pps->bits_per_pixel / 16.0);
-	int              slice_width  = pps->slice_width;
-	int              slice_height = pps->slice_height;
 	int              ret;
 	struct rc_params rc;
 	struct drm_dsc_config   dsc_cfg;
 
-	double d_bytes_per_pixel = dsc_ceil(bpp * slice_width / 8.0) / slice_width;
-
-	// TODO: Make sure the formula for calculating this is precise (ceiling vs. floor, and at what point they should be applied)
-	if (pps->native_422 || pps->native_420)
-		d_bytes_per_pixel /= 2;
-
-	dsc_params->bytes_per_pixel = (uint32_t)dsc_ceil(d_bytes_per_pixel * 0x10000000);
-
-	/* in native_422 or native_420 modes, the bits_per_pixel is double the target bpp
-	 * (the latter is what calc_rc_params expects)
-	 */
-	if (pps->native_422 || pps->native_420)
-		bpp /= 2.0;
+	dsc_params->bytes_per_pixel = calc_dsc_bytes_per_pixel(pps);
 
-	calc_rc_params(&rc, mode, bpc, bpp, slice_width, slice_height, pps->dsc_version_minor);
+	calc_rc_params(&rc, pps);
 	dsc_params->pps = *pps;
 	dsc_params->pps.initial_scale_value = 8 * rc.rc_model_size / (rc.rc_model_size - rc.initial_fullness_offset);
 
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index 9431b48aecb4..bcfe34ef8c28 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -843,7 +843,7 @@ static bool build_regamma(struct pwl_float_data_ex *rgb_regamma,
 	pow_buffer_ptr = -1; // reset back to no optimize
 	ret = true;
 release:
-	kfree(coeff);
+	kvfree(coeff);
 	return ret;
 }
 
@@ -1777,7 +1777,7 @@ bool calculate_user_regamma_ramp(struct dc_transfer_func *output_tf,
 
 	kfree(rgb_regamma);
 rgb_regamma_alloc_fail:
-	kvfree(rgb_user);
+	kfree(rgb_user);
 rgb_user_alloc_fail:
 	return ret;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c
index ae0361e225bb..aa76c2cea747 100644
--- a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c
+++ b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c
@@ -1561,6 +1561,7 @@ static int smu_v11_0_irq_process(struct amdgpu_device *adev,
 	 * events for SMCToHost interrupt.
 	 */
 	uint32_t ctxid = entry->src_data[0];
+	uint32_t data;
 
 	if (client_id == SOC15_IH_CLIENTID_THM) {
 		switch (src_id) {
@@ -1590,6 +1591,11 @@ static int smu_v11_0_irq_process(struct amdgpu_device *adev,
 		orderly_poweroff(true);
 	} else if (client_id == SOC15_IH_CLIENTID_MP1) {
 		if (src_id == 0xfe) {
+			/* ACK SMUToHost interrupt */
+			data = RREG32_SOC15(MP1, 0, mmMP1_SMN_IH_SW_INT_CTRL);
+			data = REG_SET_FIELD(data, MP1_SMN_IH_SW_INT_CTRL, INT_ACK, 1);
+			WREG32_SOC15(MP1, 0, mmMP1_SMN_IH_SW_INT_CTRL, data);
+
 			switch (ctxid) {
 			case 0x3:
 				dev_dbg(adev->dev, "Switched to AC mode!\n");
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
index 85e5b1ed22c2..56923a96b450 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
@@ -239,7 +239,7 @@ static void ci_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
 
 	switch (dev_id) {
 	case 0x67BA:
-	case 0x66B1:
+	case 0x67B1:
 		smu_data->power_tune_defaults = &defaults_hawaii_pro;
 		break;
 	case 0x67B8:
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 7d39b858c9f1..3a3a511670c9 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -226,6 +226,7 @@ static void ast_set_vbios_color_reg(struct ast_private *ast,
 	case 3:
 	case 4:
 		color_index = TrueCModeIndex;
+		break;
 	default:
 		return;
 	}
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 04f876e985de..43271c21d3fc 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -62,7 +62,7 @@ config DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW
 	depends on OF
 	select DRM_KMS_HELPER
 	select DRM_PANEL
-	---help---
+	help
 	  This is a driver for the display bridges of
 	  GE B850v3 that convert dual channel LVDS
 	  to DP++. This is used with the i.MX6 imx-ldb
@@ -89,7 +89,7 @@ config DRM_NXP_PTN3460
 	depends on OF
 	select DRM_KMS_HELPER
 	select DRM_PANEL
-	---help---
+	help
 	  NXP PTN3460 eDP-LVDS bridge chip driver.
 
 config DRM_PARADE_PS8622
@@ -98,7 +98,7 @@ config DRM_PARADE_PS8622
 	select DRM_PANEL
 	select DRM_KMS_HELPER
 	select BACKLIGHT_CLASS_DEVICE
-	---help---
+	help
 	  Parade eDP-LVDS bridge chip driver.
 
 config DRM_PARADE_PS8640
@@ -128,13 +128,13 @@ config DRM_SII902X
 	select REGMAP_I2C
 	select I2C_MUX
 	select SND_SOC_HDMI_CODEC if SND_SOC
-	---help---
+	help
 	  Silicon Image sii902x bridge chip driver.
 
 config DRM_SII9234
 	tristate "Silicon Image SII9234 HDMI/MHL bridge"
 	depends on OF
-	---help---
+	help
 	  Say Y here if you want support for the MHL interface.
 	  It is an I2C driver, that detects connection of MHL bridge
 	  and starts encapsulation of HDMI signal.
@@ -150,7 +150,7 @@ config DRM_SIMPLE_BRIDGE
 config DRM_THINE_THC63LVD1024
 	tristate "Thine THC63LVD1024 LVDS decoder bridge"
 	depends on OF
-	---help---
+	help
 	  Thine THC63LVD1024 LVDS/parallel converter driver.
 
 config DRM_TOSHIBA_TC358764
@@ -168,7 +168,7 @@ config DRM_TOSHIBA_TC358767
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
 	select DRM_PANEL
-	---help---
+	help
 	  Toshiba TC358767 eDP bridge chip driver.
 
 config DRM_TOSHIBA_TC358768
@@ -185,7 +185,7 @@ config DRM_TI_TFP410
 	tristate "TI TFP410 DVI/HDMI bridge"
 	depends on OF
 	select DRM_KMS_HELPER
-	---help---
+	help
 	  Texas Instruments TFP410 DVI/HDMI Transmitter driver
 
 config DRM_TI_SN65DSI86
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index b1099e1251a2..d877ddc6dc57 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -27,6 +27,7 @@
 #include <drm/drm_print.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_file.h>
+#include <drm/drm_sysfs.h>
 
 #include <linux/uaccess.h>
 
@@ -523,6 +524,10 @@ int drm_connector_register(struct drm_connector *connector)
 	drm_mode_object_register(connector->dev, &connector->base);
 
 	connector->registration_state = DRM_CONNECTOR_REGISTERED;
+
+	/* Let userspace know we have a new connector */
+	drm_sysfs_hotplug_event(connector->dev);
+
 	goto unlock;
 
 err_debugfs:
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 02fc24026872..170aa7689110 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -307,13 +307,13 @@ static void drm_fb_helper_sysrq(int dummy1)
 	schedule_work(&drm_fb_helper_restore_work);
 }
 
-static struct sysrq_key_op sysrq_drm_fb_helper_restore_op = {
+static const struct sysrq_key_op sysrq_drm_fb_helper_restore_op = {
 	.handler = drm_fb_helper_sysrq,
 	.help_msg = "force-fb(V)",
 	.action_msg = "Restore framebuffer console",
 };
 #else
-static struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { };
+static const struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { };
 #endif
 
 static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode)
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 939f0032aab1..f0336c804639 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -291,9 +291,6 @@ int drm_sysfs_connector_add(struct drm_connector *connector)
 		return PTR_ERR(connector->kdev);
 	}
 
-	/* Let userspace know we have a new connector */
-	drm_sysfs_hotplug_event(dev);
-
 	if (connector->ddc)
 		return sysfs_create_link(&connector->kdev->kobj,
 				 &connector->ddc->dev.kobj, "ddc");
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index 56197ae0b2f9..4391e242356d 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -37,6 +37,7 @@
 #include <linux/pci.h>
 #include <linux/seq_file.h>
 #include <linux/vmalloc.h>
+#include <linux/pgtable.h>
 
 #if defined(__ia64__)
 #include <linux/efi.h>
@@ -44,7 +45,6 @@
 #endif
 #include <linux/mem_encrypt.h>
 
-#include <asm/pgtable.h>
 
 #include <drm/drm_agpsupport.h>
 #include <drm/drm_device.h>
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index dc9ef302f517..701f3995f621 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -661,7 +661,7 @@ static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj)
 	struct etnaviv_gem_userptr *userptr = &etnaviv_obj->userptr;
 	int ret, pinned = 0, npages = etnaviv_obj->base.size >> PAGE_SHIFT;
 
-	might_lock_read(&current->mm->mmap_sem);
+	might_lock_read(&current->mm->mmap_lock);
 
 	if (userptr->mm != current->mm)
 		return -EPERM;
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index aa22465bb56e..0575a1eea2a1 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -2579,14 +2579,14 @@ static void icl_ddi_vswing_sequence(struct intel_encoder *encoder,
 
 static void
 tgl_dkl_phy_ddi_vswing_sequence(struct intel_encoder *encoder, int link_clock,
-				u32 level)
+				u32 level, enum intel_output_type type)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum tc_port tc_port = intel_port_to_tc(dev_priv, encoder->port);
 	const struct tgl_dkl_phy_ddi_buf_trans *ddi_translations;
 	u32 n_entries, val, ln, dpcnt_mask, dpcnt_val;
 
-	if (encoder->type == INTEL_OUTPUT_HDMI) {
+	if (type == INTEL_OUTPUT_HDMI) {
 		n_entries = ARRAY_SIZE(tgl_dkl_phy_hdmi_ddi_trans);
 		ddi_translations = tgl_dkl_phy_hdmi_ddi_trans;
 	} else {
@@ -2638,7 +2638,7 @@ static void tgl_ddi_vswing_sequence(struct intel_encoder *encoder,
 	if (intel_phy_is_combo(dev_priv, phy))
 		icl_combo_phy_ddi_vswing_sequence(encoder, level, type);
 	else
-		tgl_dkl_phy_ddi_vswing_sequence(encoder, link_clock, level);
+		tgl_dkl_phy_ddi_vswing_sequence(encoder, link_clock, level, type);
 }
 
 static u32 translate_signal_level(struct intel_dp *intel_dp, int signal_levels)
@@ -2987,7 +2987,7 @@ icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port,
 		ln1 = intel_de_read(dev_priv, MG_DP_MODE(1, tc_port));
 	}
 
-	ln0 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X1_MODE);
+	ln0 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE);
 	ln1 &= ~(MG_DP_MODE_CFG_DP_X1_MODE | MG_DP_MODE_CFG_DP_X2_MODE);
 
 	/* DPPATC */
@@ -3472,7 +3472,9 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state,
 					  INTEL_OUTPUT_DP_MST);
 	enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
 
-	intel_dp_set_infoframes(encoder, false, old_crtc_state, old_conn_state);
+	if (!is_mst)
+		intel_dp_set_infoframes(encoder, false,
+					old_crtc_state, old_conn_state);
 
 	/*
 	 * Power down sink before disabling the port, otherwise we end
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 40d42dcff0b7..ed9e53c373a7 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -5206,6 +5206,9 @@ void intel_read_dp_sdp(struct intel_encoder *encoder,
 		       struct intel_crtc_state *crtc_state,
 		       unsigned int type)
 {
+	if (encoder->type != INTEL_OUTPUT_DDI)
+		return;
+
 	switch (type) {
 	case DP_SDP_VSC:
 		intel_read_dp_vsc_sdp(encoder, crtc_state,
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index d18b406f2a7d..f29e51ce489c 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -397,6 +397,14 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 	 */
 	drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port,
 				     false);
+
+	/*
+	 * BSpec 4287: disable DIP after the transcoder is disabled and before
+	 * the transcoder clock select is set to none.
+	 */
+	if (last_mst_stream)
+		intel_dp_set_infoframes(&intel_dig_port->base, false,
+					old_crtc_state, NULL);
 	/*
 	 * From TGL spec: "If multi-stream slave transcoder: Configure
 	 * Transcoder Clock Select to direct no clock to the transcoder"
diff --git a/drivers/gpu/drm/i915/display/intel_global_state.c b/drivers/gpu/drm/i915/display/intel_global_state.c
index 212d4ee68205..7a19215ad844 100644
--- a/drivers/gpu/drm/i915/display/intel_global_state.c
+++ b/drivers/gpu/drm/i915/display/intel_global_state.c
@@ -10,6 +10,28 @@
 #include "intel_display_types.h"
 #include "intel_global_state.h"
 
+static void __intel_atomic_global_state_free(struct kref *kref)
+{
+	struct intel_global_state *obj_state =
+		container_of(kref, struct intel_global_state, ref);
+	struct intel_global_obj *obj = obj_state->obj;
+
+	obj->funcs->atomic_destroy_state(obj, obj_state);
+}
+
+static void intel_atomic_global_state_put(struct intel_global_state *obj_state)
+{
+	kref_put(&obj_state->ref, __intel_atomic_global_state_free);
+}
+
+static struct intel_global_state *
+intel_atomic_global_state_get(struct intel_global_state *obj_state)
+{
+	kref_get(&obj_state->ref);
+
+	return obj_state;
+}
+
 void intel_atomic_global_obj_init(struct drm_i915_private *dev_priv,
 				  struct intel_global_obj *obj,
 				  struct intel_global_state *state,
@@ -17,6 +39,10 @@ void intel_atomic_global_obj_init(struct drm_i915_private *dev_priv,
 {
 	memset(obj, 0, sizeof(*obj));
 
+	state->obj = obj;
+
+	kref_init(&state->ref);
+
 	obj->state = state;
 	obj->funcs = funcs;
 	list_add_tail(&obj->head, &dev_priv->global_obj_list);
@@ -28,7 +54,9 @@ void intel_atomic_global_obj_cleanup(struct drm_i915_private *dev_priv)
 
 	list_for_each_entry_safe(obj, next, &dev_priv->global_obj_list, head) {
 		list_del(&obj->head);
-		obj->funcs->atomic_destroy_state(obj, obj->state);
+
+		drm_WARN_ON(&dev_priv->drm, kref_read(&obj->state->ref) != 1);
+		intel_atomic_global_state_put(obj->state);
 	}
 }
 
@@ -97,10 +125,14 @@ intel_atomic_get_global_obj_state(struct intel_atomic_state *state,
 	if (!obj_state)
 		return ERR_PTR(-ENOMEM);
 
+	obj_state->obj = obj;
 	obj_state->changed = false;
 
+	kref_init(&obj_state->ref);
+
 	state->global_objs[index].state = obj_state;
-	state->global_objs[index].old_state = obj->state;
+	state->global_objs[index].old_state =
+		intel_atomic_global_state_get(obj->state);
 	state->global_objs[index].new_state = obj_state;
 	state->global_objs[index].ptr = obj;
 	obj_state->state = state;
@@ -163,7 +195,9 @@ void intel_atomic_swap_global_state(struct intel_atomic_state *state)
 		new_obj_state->state = NULL;
 
 		state->global_objs[i].state = old_obj_state;
-		obj->state = new_obj_state;
+
+		intel_atomic_global_state_put(obj->state);
+		obj->state = intel_atomic_global_state_get(new_obj_state);
 	}
 }
 
@@ -172,10 +206,9 @@ void intel_atomic_clear_global_state(struct intel_atomic_state *state)
 	int i;
 
 	for (i = 0; i < state->num_global_objs; i++) {
-		struct intel_global_obj *obj = state->global_objs[i].ptr;
+		intel_atomic_global_state_put(state->global_objs[i].old_state);
+		intel_atomic_global_state_put(state->global_objs[i].new_state);
 
-		obj->funcs->atomic_destroy_state(obj,
-						 state->global_objs[i].state);
 		state->global_objs[i].ptr = NULL;
 		state->global_objs[i].state = NULL;
 		state->global_objs[i].old_state = NULL;
diff --git a/drivers/gpu/drm/i915/display/intel_global_state.h b/drivers/gpu/drm/i915/display/intel_global_state.h
index e6163a469029..1f16fa3073c9 100644
--- a/drivers/gpu/drm/i915/display/intel_global_state.h
+++ b/drivers/gpu/drm/i915/display/intel_global_state.h
@@ -6,6 +6,7 @@
 #ifndef __INTEL_GLOBAL_STATE_H__
 #define __INTEL_GLOBAL_STATE_H__
 
+#include <linux/kref.h>
 #include <linux/list.h>
 
 struct drm_i915_private;
@@ -54,7 +55,9 @@ struct intel_global_obj {
 		for_each_if(obj)
 
 struct intel_global_state {
+	struct intel_global_obj *obj;
 	struct intel_atomic_state *state;
+	struct kref ref;
 	bool changed;
 };
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c
index 900ea8b7fc8f..30c229fcb404 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c
@@ -230,7 +230,7 @@ static void intel_context_set_gem(struct intel_context *ce,
 		ce->timeline = intel_timeline_get(ctx->timeline);
 
 	if (ctx->sched.priority >= I915_PRIORITY_NORMAL &&
-	    intel_engine_has_semaphores(ce->engine))
+	    intel_engine_has_timeslices(ce->engine))
 		__set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags);
 }
 
@@ -1921,11 +1921,6 @@ get_engines(struct i915_gem_context *ctx,
 	}
 
 	user = u64_to_user_ptr(args->value);
-	if (!access_ok(user, size)) {
-		err = -EFAULT;
-		goto err_free;
-	}
-
 	if (put_user(0, &user->extensions)) {
 		err = -EFAULT;
 		goto err_free;
@@ -1969,7 +1964,7 @@ static int __apply_priority(struct intel_context *ce, void *arg)
 {
 	struct i915_gem_context *ctx = arg;
 
-	if (!intel_engine_has_semaphores(ce->engine))
+	if (!intel_engine_has_timeslices(ce->engine))
 		return 0;
 
 	if (ctx->sched.priority >= I915_PRIORITY_NORMAL)
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index 3ce185670ca4..db8eb1c6afe9 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -1988,6 +1988,38 @@ static const struct dma_fence_work_ops eb_parse_ops = {
 	.release = __eb_parse_release,
 };
 
+static inline int
+__parser_mark_active(struct i915_vma *vma,
+		     struct intel_timeline *tl,
+		     struct dma_fence *fence)
+{
+	struct intel_gt_buffer_pool_node *node = vma->private;
+
+	return i915_active_ref(&node->active, tl, fence);
+}
+
+static int
+parser_mark_active(struct eb_parse_work *pw, struct intel_timeline *tl)
+{
+	int err;
+
+	mutex_lock(&tl->mutex);
+
+	err = __parser_mark_active(pw->shadow, tl, &pw->base.dma);
+	if (err)
+		goto unlock;
+
+	if (pw->trampoline) {
+		err = __parser_mark_active(pw->trampoline, tl, &pw->base.dma);
+		if (err)
+			goto unlock;
+	}
+
+unlock:
+	mutex_unlock(&tl->mutex);
+	return err;
+}
+
 static int eb_parse_pipeline(struct i915_execbuffer *eb,
 			     struct i915_vma *shadow,
 			     struct i915_vma *trampoline)
@@ -2022,20 +2054,25 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb,
 	pw->shadow = shadow;
 	pw->trampoline = trampoline;
 
+	/* Mark active refs early for this worker, in case we get interrupted */
+	err = parser_mark_active(pw, eb->context->timeline);
+	if (err)
+		goto err_commit;
+
 	err = dma_resv_lock_interruptible(pw->batch->resv, NULL);
 	if (err)
-		goto err_trampoline;
+		goto err_commit;
 
 	err = dma_resv_reserve_shared(pw->batch->resv, 1);
 	if (err)
-		goto err_batch_unlock;
+		goto err_commit_unlock;
 
 	/* Wait for all writes (and relocs) into the batch to complete */
 	err = i915_sw_fence_await_reservation(&pw->base.chain,
 					      pw->batch->resv, NULL, false,
 					      0, I915_FENCE_GFP);
 	if (err < 0)
-		goto err_batch_unlock;
+		goto err_commit_unlock;
 
 	/* Keep the batch alive and unwritten as we parse */
 	dma_resv_add_shared_fence(pw->batch->resv, &pw->base.dma);
@@ -2050,11 +2087,13 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb,
 	dma_fence_work_commit_imm(&pw->base);
 	return 0;
 
-err_batch_unlock:
+err_commit_unlock:
 	dma_resv_unlock(pw->batch->resv);
-err_trampoline:
-	if (trampoline)
-		i915_active_release(&trampoline->active);
+err_commit:
+	i915_sw_fence_set_error_once(&pw->base.chain, err);
+	dma_fence_work_commit_imm(&pw->base);
+	return err;
+
 err_shadow:
 	i915_active_release(&shadow->active);
 err_batch:
@@ -2100,6 +2139,7 @@ static int eb_parse(struct i915_execbuffer *eb)
 		goto err;
 	}
 	i915_gem_object_set_readonly(shadow->obj);
+	shadow->private = pool;
 
 	trampoline = NULL;
 	if (CMDPARSER_USES_GGTT(eb->i915)) {
@@ -2113,6 +2153,7 @@ static int eb_parse(struct i915_execbuffer *eb)
 			shadow = trampoline;
 			goto err_shadow;
 		}
+		shadow->private = pool;
 
 		eb->batch_flags |= I915_DISPATCH_SECURE;
 	}
@@ -2129,7 +2170,6 @@ static int eb_parse(struct i915_execbuffer *eb)
 	eb->trampoline = trampoline;
 	eb->batch_start_offset = 0;
 
-	shadow->private = pool;
 	return 0;
 
 err_trampoline:
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 70f5f82da288..fe45bd4d63a5 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -93,7 +93,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 		struct mm_struct *mm = current->mm;
 		struct vm_area_struct *vma;
 
-		if (down_write_killable(&mm->mmap_sem)) {
+		if (mmap_write_lock_killable(mm)) {
 			addr = -EINTR;
 			goto err;
 		}
@@ -103,7 +103,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
 		else
 			addr = -ENOMEM;
-		up_write(&mm->mmap_sem);
+		mmap_write_unlock(mm);
 		if (IS_ERR_VALUE(addr))
 			goto err;
 	}
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 5d5d7eef3f43..7aff3514d97a 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -39,7 +39,6 @@ static int shmem_get_pages(struct drm_i915_gem_object *obj)
 	unsigned long last_pfn = 0;	/* suppress gcc warning */
 	unsigned int max_segment = i915_sg_segment_size();
 	unsigned int sg_page_sizes;
-	struct pagevec pvec;
 	gfp_t noreclaim;
 	int ret;
 
@@ -192,13 +191,17 @@ err_sg:
 	sg_mark_end(sg);
 err_pages:
 	mapping_clear_unevictable(mapping);
-	pagevec_init(&pvec);
-	for_each_sgt_page(page, sgt_iter, st) {
-		if (!pagevec_add(&pvec, page))
+	if (sg != st->sgl) {
+		struct pagevec pvec;
+
+		pagevec_init(&pvec);
+		for_each_sgt_page(page, sgt_iter, st) {
+			if (!pagevec_add(&pvec, page))
+				check_release_pagevec(&pvec);
+		}
+		if (pagevec_count(&pvec))
 			check_release_pagevec(&pvec);
 	}
-	if (pagevec_count(&pvec))
-		check_release_pagevec(&pvec);
 	sg_free_table(st);
 	kfree(st);
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
index 33776b3f3fa5..c31a6744daee 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
@@ -200,10 +200,10 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm)
 	if (IS_ERR(mn))
 		err = PTR_ERR(mn);
 
-	down_write(&mm->mm->mmap_sem);
+	mmap_write_lock(mm->mm);
 	mutex_lock(&mm->i915->mm_lock);
 	if (mm->mn == NULL && !err) {
-		/* Protected by mmap_sem (write-lock) */
+		/* Protected by mmap_lock (write-lock) */
 		err = __mmu_notifier_register(&mn->mn, mm->mm);
 		if (!err) {
 			/* Protected by mm_lock */
@@ -217,7 +217,7 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm)
 		err = 0;
 	}
 	mutex_unlock(&mm->i915->mm_lock);
-	up_write(&mm->mm->mmap_sem);
+	mmap_write_unlock(mm->mm);
 
 	if (mn && !IS_ERR(mn))
 		kfree(mn);
@@ -468,7 +468,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 		if (mmget_not_zero(mm)) {
 			while (pinned < npages) {
 				if (!locked) {
-					down_read(&mm->mmap_sem);
+					mmap_read_lock(mm);
 					locked = 1;
 				}
 				ret = pin_user_pages_remote
@@ -483,7 +483,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 				pinned += ret;
 			}
 			if (locked)
-				up_read(&mm->mmap_sem);
+				mmap_read_unlock(mm);
 			mmput(mm);
 		}
 	}
@@ -522,8 +522,8 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj)
 
 	/* Spawn a worker so that we can acquire the
 	 * user pages without holding our mutex. Access
-	 * to the user pages requires mmap_sem, and we have
-	 * a strict lock ordering of mmap_sem, struct_mutex -
+	 * to the user pages requires mmap_lock, and we have
+	 * a strict lock ordering of mmap_lock, struct_mutex -
 	 * we already hold struct_mutex here and so cannot
 	 * call gup without encountering a lock inversion.
 	 *
diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c
index 74ddb49b2941..e4aece20bc80 100644
--- a/drivers/gpu/drm/i915/gt/intel_context.c
+++ b/drivers/gpu/drm/i915/gt/intel_context.c
@@ -97,8 +97,6 @@ int __intel_context_do_pin(struct intel_context *ce)
 {
 	int err;
 
-	GEM_BUG_ON(intel_context_is_closed(ce));
-
 	if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
 		err = intel_context_alloc_state(ce);
 		if (err)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index da5b61085257..8691eb61e185 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -646,7 +646,7 @@ static int engine_setup_common(struct intel_engine_cs *engine)
 struct measure_breadcrumb {
 	struct i915_request rq;
 	struct intel_ring ring;
-	u32 cs[1024];
+	u32 cs[2048];
 };
 
 static int measure_breadcrumb_dw(struct intel_context *ce)
@@ -668,6 +668,8 @@ static int measure_breadcrumb_dw(struct intel_context *ce)
 
 	frame->ring.vaddr = frame->cs;
 	frame->ring.size = sizeof(frame->cs);
+	frame->ring.wrap =
+		BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size);
 	frame->ring.effective_size = frame->ring.size;
 	intel_ring_update_space(&frame->ring);
 	frame->rq.ring = &frame->ring;
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index 87e6c5bdd2dc..7c3d8ef4a47c 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -1134,6 +1134,13 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
 			list_move(&rq->sched.link, pl);
 			set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
 
+			/* Check in case we rollback so far we wrap [size/2] */
+			if (intel_ring_direction(rq->ring,
+						 intel_ring_wrap(rq->ring,
+								 rq->tail),
+						 rq->ring->tail) > 0)
+				rq->context->lrc.desc |= CTX_DESC_FORCE_RESTORE;
+
 			active = rq;
 		} else {
 			struct intel_engine_cs *owner = rq->context->engine;
@@ -1498,8 +1505,9 @@ static u64 execlists_update_context(struct i915_request *rq)
 	 * HW has a tendency to ignore us rewinding the TAIL to the end of
 	 * an earlier request.
 	 */
+	GEM_BUG_ON(ce->lrc_reg_state[CTX_RING_TAIL] != rq->ring->tail);
+	prev = rq->ring->tail;
 	tail = intel_ring_set_tail(rq->ring, rq->tail);
-	prev = ce->lrc_reg_state[CTX_RING_TAIL];
 	if (unlikely(intel_ring_direction(rq->ring, tail, prev) <= 0))
 		desc |= CTX_DESC_FORCE_RESTORE;
 	ce->lrc_reg_state[CTX_RING_TAIL] = tail;
@@ -1895,7 +1903,8 @@ static void defer_active(struct intel_engine_cs *engine)
 
 static bool
 need_timeslice(const struct intel_engine_cs *engine,
-	       const struct i915_request *rq)
+	       const struct i915_request *rq,
+	       const struct rb_node *rb)
 {
 	int hint;
 
@@ -1903,9 +1912,28 @@ need_timeslice(const struct intel_engine_cs *engine,
 		return false;
 
 	hint = engine->execlists.queue_priority_hint;
+
+	if (rb) {
+		const struct virtual_engine *ve =
+			rb_entry(rb, typeof(*ve), nodes[engine->id].rb);
+		const struct intel_engine_cs *inflight =
+			intel_context_inflight(&ve->context);
+
+		if (!inflight || inflight == engine) {
+			struct i915_request *next;
+
+			rcu_read_lock();
+			next = READ_ONCE(ve->request);
+			if (next)
+				hint = max(hint, rq_prio(next));
+			rcu_read_unlock();
+		}
+	}
+
 	if (!list_is_last(&rq->sched.link, &engine->active.requests))
 		hint = max(hint, rq_prio(list_next_entry(rq, sched.link)));
 
+	GEM_BUG_ON(hint >= I915_PRIORITY_UNPREEMPTABLE);
 	return hint >= effective_prio(rq);
 }
 
@@ -1977,10 +2005,9 @@ static void set_timeslice(struct intel_engine_cs *engine)
 	set_timer_ms(&engine->execlists.timer, duration);
 }
 
-static void start_timeslice(struct intel_engine_cs *engine)
+static void start_timeslice(struct intel_engine_cs *engine, int prio)
 {
 	struct intel_engine_execlists *execlists = &engine->execlists;
-	const int prio = queue_prio(execlists);
 	unsigned long duration;
 
 	if (!intel_engine_has_timeslices(engine))
@@ -2140,7 +2167,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 			__unwind_incomplete_requests(engine);
 
 			last = NULL;
-		} else if (need_timeslice(engine, last) &&
+		} else if (need_timeslice(engine, last, rb) &&
 			   timeslice_expired(execlists, last)) {
 			if (i915_request_completed(last)) {
 				tasklet_hi_schedule(&execlists->tasklet);
@@ -2188,7 +2215,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 				 * Even if ELSP[1] is occupied and not worthy
 				 * of timeslices, our queue might be.
 				 */
-				start_timeslice(engine);
+				start_timeslice(engine, queue_prio(execlists));
 				return;
 			}
 		}
@@ -2223,7 +2250,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 
 			if (last && !can_merge_rq(last, rq)) {
 				spin_unlock(&ve->base.active.lock);
-				start_timeslice(engine);
+				start_timeslice(engine, rq_prio(rq));
 				return; /* leave this for another sibling */
 			}
 
@@ -4739,6 +4766,14 @@ static int gen12_emit_flush(struct i915_request *request, u32 mode)
 	return 0;
 }
 
+static void assert_request_valid(struct i915_request *rq)
+{
+	struct intel_ring *ring __maybe_unused = rq->ring;
+
+	/* Can we unwind this request without appearing to go forwards? */
+	GEM_BUG_ON(intel_ring_direction(ring, rq->wa_tail, rq->head) <= 0);
+}
+
 /*
  * Reserve space for 2 NOOPs at the end of each request to be
  * used as a workaround for not being allowed to do lite
@@ -4751,6 +4786,9 @@ static u32 *gen8_emit_wa_tail(struct i915_request *request, u32 *cs)
 	*cs++ = MI_NOOP;
 	request->wa_tail = intel_ring_offset(request, cs);
 
+	/* Check that entire request is less than half the ring */
+	assert_request_valid(request);
+
 	return cs;
 }
 
diff --git a/drivers/gpu/drm/i915/gt/intel_ring.c b/drivers/gpu/drm/i915/gt/intel_ring.c
index 8cda1b7e17ba..bdb324167ef3 100644
--- a/drivers/gpu/drm/i915/gt/intel_ring.c
+++ b/drivers/gpu/drm/i915/gt/intel_ring.c
@@ -315,3 +315,7 @@ int intel_ring_cacheline_align(struct i915_request *rq)
 	GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
 	return 0;
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftest_ring.c"
+#endif
diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c
index 90a2b9e399b0..85d2bef51524 100644
--- a/drivers/gpu/drm/i915/gt/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c
@@ -179,6 +179,12 @@ wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set)
 }
 
 static void
+wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr)
+{
+	wa_write_masked_or(wal, reg, clr, 0);
+}
+
+static void
 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val)
 {
 	wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val);
@@ -687,6 +693,227 @@ int intel_engine_emit_ctx_wa(struct i915_request *rq)
 }
 
 static void
+gen4_gt_workarounds_init(struct drm_i915_private *i915,
+			 struct i915_wa_list *wal)
+{
+	/* WaDisable_RenderCache_OperationalFlush:gen4,ilk */
+	wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
+}
+
+static void
+g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
+{
+	gen4_gt_workarounds_init(i915, wal);
+
+	/* WaDisableRenderCachePipelinedFlush:g4x,ilk */
+	wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE);
+}
+
+static void
+ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
+{
+	g4x_gt_workarounds_init(i915, wal);
+
+	wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED);
+}
+
+static void
+snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
+{
+	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
+	wa_masked_en(wal,
+		     _3D_CHICKEN,
+		     _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB);
+
+	/* WaDisable_RenderCache_OperationalFlush:snb */
+	wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE);
+
+	/*
+	 * BSpec recommends 8x4 when MSAA is used,
+	 * however in practice 16x4 seems fastest.
+	 *
+	 * Note that PS/WM thread counts depend on the WIZ hashing
+	 * disable bit, which we don't touch here, but it's good
+	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+	 */
+	wa_add(wal,
+	       GEN6_GT_MODE, 0,
+	       _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
+	       GEN6_WIZ_HASHING_16x4);
+
+	wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB);
+
+	wa_masked_en(wal,
+		     _3D_CHICKEN3,
+		     /* WaStripsFansDisableFastClipPerformanceFix:snb */
+		     _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL |
+		     /*
+		      * Bspec says:
+		      * "This bit must be set if 3DSTATE_CLIP clip mode is set
+		      * to normal and 3DSTATE_SF number of SF output attributes
+		      * is more than 16."
+		      */
+		   _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH);
+}
+
+static void
+ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
+{
+	/* WaDisableEarlyCull:ivb */
+	wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
+
+	/* WaDisablePSDDualDispatchEnable:ivb */
+	if (IS_IVB_GT1(i915))
+		wa_masked_en(wal,
+			     GEN7_HALF_SLICE_CHICKEN1,
+			     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
+
+	/* WaDisable_RenderCache_OperationalFlush:ivb */
+	wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
+
+	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
+	wa_masked_dis(wal,
+		      GEN7_COMMON_SLICE_CHICKEN1,
+		      GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
+
+	/* WaApplyL3ControlAndL3ChickenMode:ivb */
+	wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
+	wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
+
+	/* WaForceL3Serialization:ivb */
+	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
+
+	/*
+	 * WaVSThreadDispatchOverride:ivb,vlv
+	 *
+	 * This actually overrides the dispatch
+	 * mode for all thread types.
+	 */
+	wa_write_masked_or(wal, GEN7_FF_THREAD_MODE,
+			   GEN7_FF_SCHED_MASK,
+			   GEN7_FF_TS_SCHED_HW |
+			   GEN7_FF_VS_SCHED_HW |
+			   GEN7_FF_DS_SCHED_HW);
+
+	if (0) { /* causes HiZ corruption on ivb:gt1 */
+		/* enable HiZ Raw Stall Optimization */
+		wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
+	}
+
+	/* WaDisable4x2SubspanOptimization:ivb */
+	wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
+
+	/*
+	 * BSpec recommends 8x4 when MSAA is used,
+	 * however in practice 16x4 seems fastest.
+	 *
+	 * Note that PS/WM thread counts depend on the WIZ hashing
+	 * disable bit, which we don't touch here, but it's good
+	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+	 */
+	wa_add(wal, GEN7_GT_MODE, 0,
+	       _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
+	       GEN6_WIZ_HASHING_16x4);
+}
+
+static void
+vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
+{
+	/* WaDisableEarlyCull:vlv */
+	wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL);
+
+	/* WaPsdDispatchEnable:vlv */
+	/* WaDisablePSDDualDispatchEnable:vlv */
+	wa_masked_en(wal,
+		     GEN7_HALF_SLICE_CHICKEN1,
+		     GEN7_MAX_PS_THREAD_DEP |
+		     GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE);
+
+	/* WaDisable_RenderCache_OperationalFlush:vlv */
+	wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE);
+
+	/* WaForceL3Serialization:vlv */
+	wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE);
+
+	/*
+	 * WaVSThreadDispatchOverride:ivb,vlv
+	 *
+	 * This actually overrides the dispatch
+	 * mode for all thread types.
+	 */
+	wa_write_masked_or(wal,
+			   GEN7_FF_THREAD_MODE,
+			   GEN7_FF_SCHED_MASK,
+			   GEN7_FF_TS_SCHED_HW |
+			   GEN7_FF_VS_SCHED_HW |
+			   GEN7_FF_DS_SCHED_HW);
+
+	/*
+	 * BSpec says this must be set, even though
+	 * WaDisable4x2SubspanOptimization isn't listed for VLV.
+	 */
+	wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
+
+	/*
+	 * BSpec recommends 8x4 when MSAA is used,
+	 * however in practice 16x4 seems fastest.
+	 *
+	 * Note that PS/WM thread counts depend on the WIZ hashing
+	 * disable bit, which we don't touch here, but it's good
+	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+	 */
+	wa_add(wal, GEN7_GT_MODE, 0,
+	       _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
+	       GEN6_WIZ_HASHING_16x4);
+
+	/*
+	 * WaIncreaseL3CreditsForVLVB0:vlv
+	 * This is the hardware default actually.
+	 */
+	wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
+}
+
+static void
+hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
+{
+	/* L3 caching of data atomics doesn't work -- disable it. */
+	wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
+
+	wa_add(wal,
+	       HSW_ROW_CHICKEN3, 0,
+	       _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),
+		0 /* XXX does this reg exist? */);
+
+	/* WaVSRefCountFullforceMissDisable:hsw */
+	wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME);
+
+	wa_masked_dis(wal,
+		      CACHE_MODE_0_GEN7,
+		      /* WaDisable_RenderCache_OperationalFlush:hsw */
+		      RC_OP_FLUSH_ENABLE |
+		      /* enable HiZ Raw Stall Optimization */
+		      HIZ_RAW_STALL_OPT_DISABLE);
+
+	/* WaDisable4x2SubspanOptimization:hsw */
+	wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
+
+	/*
+	 * BSpec recommends 8x4 when MSAA is used,
+	 * however in practice 16x4 seems fastest.
+	 *
+	 * Note that PS/WM thread counts depend on the WIZ hashing
+	 * disable bit, which we don't touch here, but it's good
+	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+	 */
+	wa_add(wal, GEN7_GT_MODE, 0,
+	       _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4),
+	       GEN6_WIZ_HASHING_16x4);
+
+	/* WaSampleCChickenBitEnable:hsw */
+	wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE);
+}
+
+static void
 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 {
 	/* WaDisableKillLogic:bxt,skl,kbl */
@@ -963,6 +1190,20 @@ gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
 		bxt_gt_workarounds_init(i915, wal);
 	else if (IS_SKYLAKE(i915))
 		skl_gt_workarounds_init(i915, wal);
+	else if (IS_HASWELL(i915))
+		hsw_gt_workarounds_init(i915, wal);
+	else if (IS_VALLEYVIEW(i915))
+		vlv_gt_workarounds_init(i915, wal);
+	else if (IS_IVYBRIDGE(i915))
+		ivb_gt_workarounds_init(i915, wal);
+	else if (IS_GEN(i915, 6))
+		snb_gt_workarounds_init(i915, wal);
+	else if (IS_GEN(i915, 5))
+		ilk_gt_workarounds_init(i915, wal);
+	else if (IS_G4X(i915))
+		g4x_gt_workarounds_init(i915, wal);
+	else if (IS_GEN(i915, 4))
+		gen4_gt_workarounds_init(i915, wal);
 	else if (INTEL_GEN(i915) <= 8)
 		return;
 	else
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index 2b2efff6e19d..4aa4cc917d8b 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -310,22 +310,20 @@ static bool wait_until_running(struct hang *h, struct i915_request *rq)
 			  1000));
 }
 
-static void engine_heartbeat_disable(struct intel_engine_cs *engine,
-				     unsigned long *saved)
+static void engine_heartbeat_disable(struct intel_engine_cs *engine)
 {
-	*saved = engine->props.heartbeat_interval_ms;
 	engine->props.heartbeat_interval_ms = 0;
 
 	intel_engine_pm_get(engine);
 	intel_engine_park_heartbeat(engine);
 }
 
-static void engine_heartbeat_enable(struct intel_engine_cs *engine,
-				    unsigned long saved)
+static void engine_heartbeat_enable(struct intel_engine_cs *engine)
 {
 	intel_engine_pm_put(engine);
 
-	engine->props.heartbeat_interval_ms = saved;
+	engine->props.heartbeat_interval_ms =
+		engine->defaults.heartbeat_interval_ms;
 }
 
 static int igt_hang_sanitycheck(void *arg)
@@ -473,7 +471,6 @@ static int igt_reset_nop_engine(void *arg)
 	for_each_engine(engine, gt, id) {
 		unsigned int reset_count, reset_engine_count, count;
 		struct intel_context *ce;
-		unsigned long heartbeat;
 		IGT_TIMEOUT(end_time);
 		int err;
 
@@ -485,7 +482,7 @@ static int igt_reset_nop_engine(void *arg)
 		reset_engine_count = i915_reset_engine_count(global, engine);
 		count = 0;
 
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 		do {
 			int i;
@@ -529,7 +526,7 @@ static int igt_reset_nop_engine(void *arg)
 			}
 		} while (time_before(jiffies, end_time));
 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 
 		pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
 
@@ -564,7 +561,6 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active)
 
 	for_each_engine(engine, gt, id) {
 		unsigned int reset_count, reset_engine_count;
-		unsigned long heartbeat;
 		IGT_TIMEOUT(end_time);
 
 		if (active && !intel_engine_can_store_dword(engine))
@@ -580,7 +576,7 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active)
 		reset_count = i915_reset_count(global);
 		reset_engine_count = i915_reset_engine_count(global, engine);
 
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 		do {
 			if (active) {
@@ -632,7 +628,7 @@ static int __igt_reset_engine(struct intel_gt *gt, bool active)
 			}
 		} while (time_before(jiffies, end_time));
 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 
 		if (err)
 			break;
@@ -789,7 +785,6 @@ static int __igt_reset_engines(struct intel_gt *gt,
 		struct active_engine threads[I915_NUM_ENGINES] = {};
 		unsigned long device = i915_reset_count(global);
 		unsigned long count = 0, reported;
-		unsigned long heartbeat;
 		IGT_TIMEOUT(end_time);
 
 		if (flags & TEST_ACTIVE &&
@@ -832,7 +827,7 @@ static int __igt_reset_engines(struct intel_gt *gt,
 
 		yield(); /* start all threads before we begin */
 
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 		set_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
 		do {
 			struct i915_request *rq = NULL;
@@ -906,7 +901,7 @@ static int __igt_reset_engines(struct intel_gt *gt,
 			}
 		} while (time_before(jiffies, end_time));
 		clear_bit(I915_RESET_ENGINE + id, &gt->reset.flags);
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 
 		pr_info("i915_reset_engine(%s:%s): %lu resets\n",
 			engine->name, test_name, count);
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index 824f99c4cc7c..924bc01ef526 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -51,22 +51,20 @@ static struct i915_vma *create_scratch(struct intel_gt *gt)
 	return vma;
 }
 
-static void engine_heartbeat_disable(struct intel_engine_cs *engine,
-				     unsigned long *saved)
+static void engine_heartbeat_disable(struct intel_engine_cs *engine)
 {
-	*saved = engine->props.heartbeat_interval_ms;
 	engine->props.heartbeat_interval_ms = 0;
 
 	intel_engine_pm_get(engine);
 	intel_engine_park_heartbeat(engine);
 }
 
-static void engine_heartbeat_enable(struct intel_engine_cs *engine,
-				    unsigned long saved)
+static void engine_heartbeat_enable(struct intel_engine_cs *engine)
 {
 	intel_engine_pm_put(engine);
 
-	engine->props.heartbeat_interval_ms = saved;
+	engine->props.heartbeat_interval_ms =
+		engine->defaults.heartbeat_interval_ms;
 }
 
 static bool is_active(struct i915_request *rq)
@@ -224,7 +222,6 @@ static int live_unlite_restore(struct intel_gt *gt, int prio)
 		struct intel_context *ce[2] = {};
 		struct i915_request *rq[2];
 		struct igt_live_test t;
-		unsigned long saved;
 		int n;
 
 		if (prio && !intel_engine_has_preemption(engine))
@@ -237,7 +234,7 @@ static int live_unlite_restore(struct intel_gt *gt, int prio)
 			err = -EIO;
 			break;
 		}
-		engine_heartbeat_disable(engine, &saved);
+		engine_heartbeat_disable(engine);
 
 		for (n = 0; n < ARRAY_SIZE(ce); n++) {
 			struct intel_context *tmp;
@@ -345,7 +342,7 @@ err_ce:
 			intel_context_put(ce[n]);
 		}
 
-		engine_heartbeat_enable(engine, saved);
+		engine_heartbeat_enable(engine);
 		if (igt_live_test_end(&t))
 			err = -EIO;
 		if (err)
@@ -466,7 +463,6 @@ static int live_hold_reset(void *arg)
 
 	for_each_engine(engine, gt, id) {
 		struct intel_context *ce;
-		unsigned long heartbeat;
 		struct i915_request *rq;
 
 		ce = intel_context_create(engine);
@@ -475,7 +471,7 @@ static int live_hold_reset(void *arg)
 			break;
 		}
 
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 
 		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
 		if (IS_ERR(rq)) {
@@ -535,7 +531,7 @@ static int live_hold_reset(void *arg)
 		i915_request_put(rq);
 
 out:
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 		intel_context_put(ce);
 		if (err)
 			break;
@@ -580,10 +576,9 @@ static int live_error_interrupt(void *arg)
 
 	for_each_engine(engine, gt, id) {
 		const struct error_phase *p;
-		unsigned long heartbeat;
 		int err = 0;
 
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 
 		for (p = phases; p->error[0] != GOOD; p++) {
 			struct i915_request *client[ARRAY_SIZE(phases->error)];
@@ -682,7 +677,7 @@ out:
 			}
 		}
 
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 		if (err) {
 			intel_gt_set_wedged(gt);
 			return err;
@@ -828,7 +823,7 @@ slice_semaphore_queue(struct intel_engine_cs *outer,
 		}
 	}
 
-	err = release_queue(outer, vma, n, INT_MAX);
+	err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);
 	if (err)
 		goto out;
 
@@ -895,16 +890,14 @@ static int live_timeslice_preempt(void *arg)
 		enum intel_engine_id id;
 
 		for_each_engine(engine, gt, id) {
-			unsigned long saved;
-
 			if (!intel_engine_has_preemption(engine))
 				continue;
 
 			memset(vaddr, 0, PAGE_SIZE);
 
-			engine_heartbeat_disable(engine, &saved);
+			engine_heartbeat_disable(engine);
 			err = slice_semaphore_queue(engine, vma, count);
-			engine_heartbeat_enable(engine, saved);
+			engine_heartbeat_enable(engine);
 			if (err)
 				goto err_pin;
 
@@ -1009,7 +1002,6 @@ static int live_timeslice_rewind(void *arg)
 		enum { X = 1, Z, Y };
 		struct i915_request *rq[3] = {};
 		struct intel_context *ce;
-		unsigned long heartbeat;
 		unsigned long timeslice;
 		int i, err = 0;
 		u32 *slot;
@@ -1028,7 +1020,7 @@ static int live_timeslice_rewind(void *arg)
 		 * Expect execution/evaluation order XZY
 		 */
 
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
 
 		slot = memset32(engine->status_page.addr + 1000, 0, 4);
@@ -1122,7 +1114,7 @@ err:
 		wmb();
 
 		engine->props.timeslice_duration_ms = timeslice;
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 		for (i = 0; i < 3; i++)
 			i915_request_put(rq[i]);
 		if (igt_flush_test(gt->i915))
@@ -1202,12 +1194,11 @@ static int live_timeslice_queue(void *arg)
 			.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
 		};
 		struct i915_request *rq, *nop;
-		unsigned long saved;
 
 		if (!intel_engine_has_preemption(engine))
 			continue;
 
-		engine_heartbeat_disable(engine, &saved);
+		engine_heartbeat_disable(engine);
 		memset(vaddr, 0, PAGE_SIZE);
 
 		/* ELSP[0]: semaphore wait */
@@ -1284,7 +1275,7 @@ static int live_timeslice_queue(void *arg)
 err_rq:
 		i915_request_put(rq);
 err_heartbeat:
-		engine_heartbeat_enable(engine, saved);
+		engine_heartbeat_enable(engine);
 		if (err)
 			break;
 	}
@@ -1298,6 +1289,121 @@ err_obj:
 	return err;
 }
 
+static int live_timeslice_nopreempt(void *arg)
+{
+	struct intel_gt *gt = arg;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	struct igt_spinner spin;
+	int err = 0;
+
+	/*
+	 * We should not timeslice into a request that is marked with
+	 * I915_REQUEST_NOPREEMPT.
+	 */
+	if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
+		return 0;
+
+	if (igt_spinner_init(&spin, gt))
+		return -ENOMEM;
+
+	for_each_engine(engine, gt, id) {
+		struct intel_context *ce;
+		struct i915_request *rq;
+		unsigned long timeslice;
+
+		if (!intel_engine_has_preemption(engine))
+			continue;
+
+		ce = intel_context_create(engine);
+		if (IS_ERR(ce)) {
+			err = PTR_ERR(ce);
+			break;
+		}
+
+		engine_heartbeat_disable(engine);
+		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
+
+		/* Create an unpreemptible spinner */
+
+		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
+		intel_context_put(ce);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto out_heartbeat;
+		}
+
+		i915_request_get(rq);
+		i915_request_add(rq);
+
+		if (!igt_wait_for_spinner(&spin, rq)) {
+			i915_request_put(rq);
+			err = -ETIME;
+			goto out_spin;
+		}
+
+		set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);
+		i915_request_put(rq);
+
+		/* Followed by a maximum priority barrier (heartbeat) */
+
+		ce = intel_context_create(engine);
+		if (IS_ERR(ce)) {
+			err = PTR_ERR(rq);
+			goto out_spin;
+		}
+
+		rq = intel_context_create_request(ce);
+		intel_context_put(ce);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto out_spin;
+		}
+
+		rq->sched.attr.priority = I915_PRIORITY_BARRIER;
+		i915_request_get(rq);
+		i915_request_add(rq);
+
+		/*
+		 * Wait until the barrier is in ELSP, and we know timeslicing
+		 * will have been activated.
+		 */
+		if (wait_for_submit(engine, rq, HZ / 2)) {
+			i915_request_put(rq);
+			err = -ETIME;
+			goto out_spin;
+		}
+
+		/*
+		 * Since the ELSP[0] request is unpreemptible, it should not
+		 * allow the maximum priority barrier through. Wait long
+		 * enough to see if it is timesliced in by mistake.
+		 */
+		if (i915_request_wait(rq, 0, timeslice_threshold(engine)) >= 0) {
+			pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",
+			       engine->name);
+			err = -EINVAL;
+		}
+		i915_request_put(rq);
+
+out_spin:
+		igt_spinner_end(&spin);
+out_heartbeat:
+		xchg(&engine->props.timeslice_duration_ms, timeslice);
+		engine_heartbeat_enable(engine);
+		if (err)
+			break;
+
+		if (igt_flush_test(gt->i915)) {
+			err = -EIO;
+			break;
+		}
+	}
+
+	igt_spinner_fini(&spin);
+	return err;
+}
+
 static int live_busywait_preempt(void *arg)
 {
 	struct intel_gt *gt = arg;
@@ -4153,7 +4259,6 @@ static int reset_virtual_engine(struct intel_gt *gt,
 {
 	struct intel_engine_cs *engine;
 	struct intel_context *ve;
-	unsigned long *heartbeat;
 	struct igt_spinner spin;
 	struct i915_request *rq;
 	unsigned int n;
@@ -4165,15 +4270,9 @@ static int reset_virtual_engine(struct intel_gt *gt,
 	 * descendents are not executed while the capture is in progress.
 	 */
 
-	heartbeat = kmalloc_array(nsibling, sizeof(*heartbeat), GFP_KERNEL);
-	if (!heartbeat)
+	if (igt_spinner_init(&spin, gt))
 		return -ENOMEM;
 
-	if (igt_spinner_init(&spin, gt)) {
-		err = -ENOMEM;
-		goto out_free;
-	}
-
 	ve = intel_execlists_create_virtual(siblings, nsibling);
 	if (IS_ERR(ve)) {
 		err = PTR_ERR(ve);
@@ -4181,7 +4280,7 @@ static int reset_virtual_engine(struct intel_gt *gt,
 	}
 
 	for (n = 0; n < nsibling; n++)
-		engine_heartbeat_disable(siblings[n], &heartbeat[n]);
+		engine_heartbeat_disable(siblings[n]);
 
 	rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
 	if (IS_ERR(rq)) {
@@ -4252,13 +4351,11 @@ out_rq:
 	i915_request_put(rq);
 out_heartbeat:
 	for (n = 0; n < nsibling; n++)
-		engine_heartbeat_enable(siblings[n], heartbeat[n]);
+		engine_heartbeat_enable(siblings[n]);
 
 	intel_context_put(ve);
 out_spin:
 	igt_spinner_fini(&spin);
-out_free:
-	kfree(heartbeat);
 	return err;
 }
 
@@ -4314,6 +4411,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(live_timeslice_preempt),
 		SUBTEST(live_timeslice_rewind),
 		SUBTEST(live_timeslice_queue),
+		SUBTEST(live_timeslice_nopreempt),
 		SUBTEST(live_busywait_preempt),
 		SUBTEST(live_preempt),
 		SUBTEST(live_late_preempt),
@@ -4932,9 +5030,7 @@ static int live_lrc_gpr(void *arg)
 		return PTR_ERR(scratch);
 
 	for_each_engine(engine, gt, id) {
-		unsigned long heartbeat;
-
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 
 		err = __live_lrc_gpr(engine, scratch, false);
 		if (err)
@@ -4945,7 +5041,7 @@ static int live_lrc_gpr(void *arg)
 			goto err;
 
 err:
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 		if (igt_flush_test(gt->i915))
 			err = -EIO;
 		if (err)
@@ -5092,10 +5188,9 @@ static int live_lrc_timestamp(void *arg)
 	 */
 
 	for_each_engine(data.engine, gt, id) {
-		unsigned long heartbeat;
 		int i, err = 0;
 
-		engine_heartbeat_disable(data.engine, &heartbeat);
+		engine_heartbeat_disable(data.engine);
 
 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 			struct intel_context *tmp;
@@ -5128,7 +5223,7 @@ static int live_lrc_timestamp(void *arg)
 		}
 
 err:
-		engine_heartbeat_enable(data.engine, heartbeat);
+		engine_heartbeat_enable(data.engine);
 		for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
 			if (!data.ce[i])
 				break;
diff --git a/drivers/gpu/drm/i915/gt/selftest_mocs.c b/drivers/gpu/drm/i915/gt/selftest_mocs.c
index 8831ffee2061..63f87d8608c3 100644
--- a/drivers/gpu/drm/i915/gt/selftest_mocs.c
+++ b/drivers/gpu/drm/i915/gt/selftest_mocs.c
@@ -18,6 +18,20 @@ struct live_mocs {
 	void *vaddr;
 };
 
+static struct intel_context *mocs_context_create(struct intel_engine_cs *engine)
+{
+	struct intel_context *ce;
+
+	ce = intel_context_create(engine);
+	if (IS_ERR(ce))
+		return ce;
+
+	/* We build large requests to read the registers from the ring */
+	ce->ring = __intel_context_ring_size(SZ_16K);
+
+	return ce;
+}
+
 static int request_add_sync(struct i915_request *rq, int err)
 {
 	i915_request_get(rq);
@@ -301,7 +315,7 @@ static int live_mocs_clean(void *arg)
 	for_each_engine(engine, gt, id) {
 		struct intel_context *ce;
 
-		ce = intel_context_create(engine);
+		ce = mocs_context_create(engine);
 		if (IS_ERR(ce)) {
 			err = PTR_ERR(ce);
 			break;
@@ -395,7 +409,7 @@ static int live_mocs_reset(void *arg)
 	for_each_engine(engine, gt, id) {
 		struct intel_context *ce;
 
-		ce = intel_context_create(engine);
+		ce = mocs_context_create(engine);
 		if (IS_ERR(ce)) {
 			err = PTR_ERR(ce);
 			break;
diff --git a/drivers/gpu/drm/i915/gt/selftest_ring.c b/drivers/gpu/drm/i915/gt/selftest_ring.c
new file mode 100644
index 000000000000..2a8c534dc125
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/selftest_ring.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright © 2020 Intel Corporation
+ */
+
+static struct intel_ring *mock_ring(unsigned long sz)
+{
+	struct intel_ring *ring;
+
+	ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL);
+	if (!ring)
+		return NULL;
+
+	kref_init(&ring->ref);
+	ring->size = sz;
+	ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(sz);
+	ring->effective_size = sz;
+	ring->vaddr = (void *)(ring + 1);
+	atomic_set(&ring->pin_count, 1);
+
+	intel_ring_update_space(ring);
+
+	return ring;
+}
+
+static void mock_ring_free(struct intel_ring *ring)
+{
+	kfree(ring);
+}
+
+static int check_ring_direction(struct intel_ring *ring,
+				u32 next, u32 prev,
+				int expected)
+{
+	int result;
+
+	result = intel_ring_direction(ring, next, prev);
+	if (result < 0)
+		result = -1;
+	else if (result > 0)
+		result = 1;
+
+	if (result != expected) {
+		pr_err("intel_ring_direction(%u, %u):%d != %d\n",
+		       next, prev, result, expected);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int check_ring_step(struct intel_ring *ring, u32 x, u32 step)
+{
+	u32 prev = x, next = intel_ring_wrap(ring, x + step);
+	int err = 0;
+
+	err |= check_ring_direction(ring, next, next,  0);
+	err |= check_ring_direction(ring, prev, prev,  0);
+	err |= check_ring_direction(ring, next, prev,  1);
+	err |= check_ring_direction(ring, prev, next, -1);
+
+	return err;
+}
+
+static int check_ring_offset(struct intel_ring *ring, u32 x, u32 step)
+{
+	int err = 0;
+
+	err |= check_ring_step(ring, x, step);
+	err |= check_ring_step(ring, intel_ring_wrap(ring, x + 1), step);
+	err |= check_ring_step(ring, intel_ring_wrap(ring, x - 1), step);
+
+	return err;
+}
+
+static int igt_ring_direction(void *dummy)
+{
+	struct intel_ring *ring;
+	unsigned int half = 2048;
+	int step, err = 0;
+
+	ring = mock_ring(2 * half);
+	if (!ring)
+		return -ENOMEM;
+
+	GEM_BUG_ON(ring->size != 2 * half);
+
+	/* Precision of wrap detection is limited to ring->size / 2 */
+	for (step = 1; step < half; step <<= 1) {
+		err |= check_ring_offset(ring, 0, step);
+		err |= check_ring_offset(ring, half, step);
+	}
+	err |= check_ring_step(ring, 0, half - 64);
+
+	/* And check unwrapped handling for good measure */
+	err |= check_ring_offset(ring, 0, 2 * half + 64);
+	err |= check_ring_offset(ring, 3 * half, 1);
+
+	mock_ring_free(ring);
+	return err;
+}
+
+int intel_ring_mock_selftests(void)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(igt_ring_direction),
+	};
+
+	return i915_subtests(tests, NULL);
+}
diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c
index 6275d69aa9cc..5049c3dd08a6 100644
--- a/drivers/gpu/drm/i915/gt/selftest_rps.c
+++ b/drivers/gpu/drm/i915/gt/selftest_rps.c
@@ -20,24 +20,20 @@
 /* Try to isolate the impact of cstates from determing frequency response */
 #define CPU_LATENCY 0 /* -1 to disable pm_qos, 0 to disable cstates */
 
-static unsigned long engine_heartbeat_disable(struct intel_engine_cs *engine)
+static void engine_heartbeat_disable(struct intel_engine_cs *engine)
 {
-	unsigned long old;
-
-	old = fetch_and_zero(&engine->props.heartbeat_interval_ms);
+	engine->props.heartbeat_interval_ms = 0;
 
 	intel_engine_pm_get(engine);
 	intel_engine_park_heartbeat(engine);
-
-	return old;
 }
 
-static void engine_heartbeat_enable(struct intel_engine_cs *engine,
-				    unsigned long saved)
+static void engine_heartbeat_enable(struct intel_engine_cs *engine)
 {
 	intel_engine_pm_put(engine);
 
-	engine->props.heartbeat_interval_ms = saved;
+	engine->props.heartbeat_interval_ms =
+		engine->defaults.heartbeat_interval_ms;
 }
 
 static void dummy_rps_work(struct work_struct *wrk)
@@ -246,7 +242,6 @@ int live_rps_clock_interval(void *arg)
 	intel_gt_check_clock_frequency(gt);
 
 	for_each_engine(engine, gt, id) {
-		unsigned long saved_heartbeat;
 		struct i915_request *rq;
 		u32 cycles;
 		u64 dt;
@@ -254,13 +249,13 @@ int live_rps_clock_interval(void *arg)
 		if (!intel_engine_can_store_dword(engine))
 			continue;
 
-		saved_heartbeat = engine_heartbeat_disable(engine);
+		engine_heartbeat_disable(engine);
 
 		rq = igt_spinner_create_request(&spin,
 						engine->kernel_context,
 						MI_NOOP);
 		if (IS_ERR(rq)) {
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			err = PTR_ERR(rq);
 			break;
 		}
@@ -271,7 +266,7 @@ int live_rps_clock_interval(void *arg)
 			pr_err("%s: RPS spinner did not start\n",
 			       engine->name);
 			igt_spinner_end(&spin);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			intel_gt_set_wedged(engine->gt);
 			err = -EIO;
 			break;
@@ -327,7 +322,7 @@ int live_rps_clock_interval(void *arg)
 		intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 
 		igt_spinner_end(&spin);
-		engine_heartbeat_enable(engine, saved_heartbeat);
+		engine_heartbeat_enable(engine);
 
 		if (err == 0) {
 			u64 time = intel_gt_pm_interval_to_ns(gt, cycles);
@@ -405,7 +400,6 @@ int live_rps_control(void *arg)
 
 	intel_gt_pm_get(gt);
 	for_each_engine(engine, gt, id) {
-		unsigned long saved_heartbeat;
 		struct i915_request *rq;
 		ktime_t min_dt, max_dt;
 		int f, limit;
@@ -414,7 +408,7 @@ int live_rps_control(void *arg)
 		if (!intel_engine_can_store_dword(engine))
 			continue;
 
-		saved_heartbeat = engine_heartbeat_disable(engine);
+		engine_heartbeat_disable(engine);
 
 		rq = igt_spinner_create_request(&spin,
 						engine->kernel_context,
@@ -430,7 +424,7 @@ int live_rps_control(void *arg)
 			pr_err("%s: RPS spinner did not start\n",
 			       engine->name);
 			igt_spinner_end(&spin);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			intel_gt_set_wedged(engine->gt);
 			err = -EIO;
 			break;
@@ -440,7 +434,7 @@ int live_rps_control(void *arg)
 			pr_err("%s: could not set minimum frequency [%x], only %x!\n",
 			       engine->name, rps->min_freq, read_cagf(rps));
 			igt_spinner_end(&spin);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			show_pstate_limits(rps);
 			err = -EINVAL;
 			break;
@@ -457,7 +451,7 @@ int live_rps_control(void *arg)
 			pr_err("%s: could not restore minimum frequency [%x], only %x!\n",
 			       engine->name, rps->min_freq, read_cagf(rps));
 			igt_spinner_end(&spin);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			show_pstate_limits(rps);
 			err = -EINVAL;
 			break;
@@ -472,7 +466,7 @@ int live_rps_control(void *arg)
 		min_dt = ktime_sub(ktime_get(), min_dt);
 
 		igt_spinner_end(&spin);
-		engine_heartbeat_enable(engine, saved_heartbeat);
+		engine_heartbeat_enable(engine);
 
 		pr_info("%s: range:[%x:%uMHz, %x:%uMHz] limit:[%x:%uMHz], %x:%x response %lluns:%lluns\n",
 			engine->name,
@@ -635,7 +629,6 @@ int live_rps_frequency_cs(void *arg)
 	rps->work.func = dummy_rps_work;
 
 	for_each_engine(engine, gt, id) {
-		unsigned long saved_heartbeat;
 		struct i915_request *rq;
 		struct i915_vma *vma;
 		u32 *cancel, *cntr;
@@ -644,14 +637,14 @@ int live_rps_frequency_cs(void *arg)
 			int freq;
 		} min, max;
 
-		saved_heartbeat = engine_heartbeat_disable(engine);
+		engine_heartbeat_disable(engine);
 
 		vma = create_spin_counter(engine,
 					  engine->kernel_context->vm, false,
 					  &cancel, &cntr);
 		if (IS_ERR(vma)) {
 			err = PTR_ERR(vma);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			break;
 		}
 
@@ -732,7 +725,7 @@ err_vma:
 		i915_vma_unpin(vma);
 		i915_vma_put(vma);
 
-		engine_heartbeat_enable(engine, saved_heartbeat);
+		engine_heartbeat_enable(engine);
 		if (igt_flush_test(gt->i915))
 			err = -EIO;
 		if (err)
@@ -778,7 +771,6 @@ int live_rps_frequency_srm(void *arg)
 	rps->work.func = dummy_rps_work;
 
 	for_each_engine(engine, gt, id) {
-		unsigned long saved_heartbeat;
 		struct i915_request *rq;
 		struct i915_vma *vma;
 		u32 *cancel, *cntr;
@@ -787,14 +779,14 @@ int live_rps_frequency_srm(void *arg)
 			int freq;
 		} min, max;
 
-		saved_heartbeat = engine_heartbeat_disable(engine);
+		engine_heartbeat_disable(engine);
 
 		vma = create_spin_counter(engine,
 					  engine->kernel_context->vm, true,
 					  &cancel, &cntr);
 		if (IS_ERR(vma)) {
 			err = PTR_ERR(vma);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			break;
 		}
 
@@ -874,7 +866,7 @@ err_vma:
 		i915_vma_unpin(vma);
 		i915_vma_put(vma);
 
-		engine_heartbeat_enable(engine, saved_heartbeat);
+		engine_heartbeat_enable(engine);
 		if (igt_flush_test(gt->i915))
 			err = -EIO;
 		if (err)
@@ -1066,16 +1058,14 @@ int live_rps_interrupt(void *arg)
 	for_each_engine(engine, gt, id) {
 		/* Keep the engine busy with a spinner; expect an UP! */
 		if (pm_events & GEN6_PM_RP_UP_THRESHOLD) {
-			unsigned long saved_heartbeat;
-
 			intel_gt_pm_wait_for_idle(engine->gt);
 			GEM_BUG_ON(intel_rps_is_active(rps));
 
-			saved_heartbeat = engine_heartbeat_disable(engine);
+			engine_heartbeat_disable(engine);
 
 			err = __rps_up_interrupt(rps, engine, &spin);
 
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			if (err)
 				goto out;
 
@@ -1084,15 +1074,13 @@ int live_rps_interrupt(void *arg)
 
 		/* Keep the engine awake but idle and check for DOWN */
 		if (pm_events & GEN6_PM_RP_DOWN_THRESHOLD) {
-			unsigned long saved_heartbeat;
-
-			saved_heartbeat = engine_heartbeat_disable(engine);
+			engine_heartbeat_disable(engine);
 			intel_rc6_disable(&gt->rc6);
 
 			err = __rps_down_interrupt(rps, engine);
 
 			intel_rc6_enable(&gt->rc6);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			if (err)
 				goto out;
 		}
@@ -1168,7 +1156,6 @@ int live_rps_power(void *arg)
 	rps->work.func = dummy_rps_work;
 
 	for_each_engine(engine, gt, id) {
-		unsigned long saved_heartbeat;
 		struct i915_request *rq;
 		struct {
 			u64 power;
@@ -1178,13 +1165,13 @@ int live_rps_power(void *arg)
 		if (!intel_engine_can_store_dword(engine))
 			continue;
 
-		saved_heartbeat = engine_heartbeat_disable(engine);
+		engine_heartbeat_disable(engine);
 
 		rq = igt_spinner_create_request(&spin,
 						engine->kernel_context,
 						MI_NOOP);
 		if (IS_ERR(rq)) {
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			err = PTR_ERR(rq);
 			break;
 		}
@@ -1195,7 +1182,7 @@ int live_rps_power(void *arg)
 			pr_err("%s: RPS spinner did not start\n",
 			       engine->name);
 			igt_spinner_end(&spin);
-			engine_heartbeat_enable(engine, saved_heartbeat);
+			engine_heartbeat_enable(engine);
 			intel_gt_set_wedged(engine->gt);
 			err = -EIO;
 			break;
@@ -1208,7 +1195,7 @@ int live_rps_power(void *arg)
 		min.power = measure_power_at(rps, &min.freq);
 
 		igt_spinner_end(&spin);
-		engine_heartbeat_enable(engine, saved_heartbeat);
+		engine_heartbeat_enable(engine);
 
 		pr_info("%s: min:%llumW @ %uMHz, max:%llumW @ %uMHz\n",
 			engine->name,
diff --git a/drivers/gpu/drm/i915/gt/selftest_timeline.c b/drivers/gpu/drm/i915/gt/selftest_timeline.c
index c2578a0f2f14..ef1c35073dc0 100644
--- a/drivers/gpu/drm/i915/gt/selftest_timeline.c
+++ b/drivers/gpu/drm/i915/gt/selftest_timeline.c
@@ -751,22 +751,20 @@ out_free:
 	return err;
 }
 
-static void engine_heartbeat_disable(struct intel_engine_cs *engine,
-				     unsigned long *saved)
+static void engine_heartbeat_disable(struct intel_engine_cs *engine)
 {
-	*saved = engine->props.heartbeat_interval_ms;
 	engine->props.heartbeat_interval_ms = 0;
 
 	intel_engine_pm_get(engine);
 	intel_engine_park_heartbeat(engine);
 }
 
-static void engine_heartbeat_enable(struct intel_engine_cs *engine,
-				    unsigned long saved)
+static void engine_heartbeat_enable(struct intel_engine_cs *engine)
 {
 	intel_engine_pm_put(engine);
 
-	engine->props.heartbeat_interval_ms = saved;
+	engine->props.heartbeat_interval_ms =
+		engine->defaults.heartbeat_interval_ms;
 }
 
 static int live_hwsp_rollover_kernel(void *arg)
@@ -785,10 +783,9 @@ static int live_hwsp_rollover_kernel(void *arg)
 		struct intel_context *ce = engine->kernel_context;
 		struct intel_timeline *tl = ce->timeline;
 		struct i915_request *rq[3] = {};
-		unsigned long heartbeat;
 		int i;
 
-		engine_heartbeat_disable(engine, &heartbeat);
+		engine_heartbeat_disable(engine);
 		if (intel_gt_wait_for_idle(gt, HZ / 2)) {
 			err = -EIO;
 			goto out;
@@ -839,7 +836,7 @@ static int live_hwsp_rollover_kernel(void *arg)
 out:
 		for (i = 0; i < ARRAY_SIZE(rq); i++)
 			i915_request_put(rq[i]);
-		engine_heartbeat_enable(engine, heartbeat);
+		engine_heartbeat_enable(engine);
 		if (err)
 			break;
 	}
diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c
index 5ed323254ee1..32785463ec9e 100644
--- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c
@@ -623,6 +623,8 @@ err_request:
 				err = -EINVAL;
 				goto out_unpin;
 			}
+		} else {
+			rsvd = 0;
 		}
 
 		expect = results[0];
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
index eee530453aa6..ad8a9df49f29 100644
--- a/drivers/gpu/drm/i915/gvt/kvmgt.c
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -31,7 +31,7 @@
 #include <linux/init.h>
 #include <linux/device.h>
 #include <linux/mm.h>
-#include <linux/mmu_context.h>
+#include <linux/kthread.h>
 #include <linux/sched/mm.h>
 #include <linux/types.h>
 #include <linux/list.h>
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 1d5ff88078bd..7d361623ff67 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -124,7 +124,7 @@ int intel_gvt_init_vgpu_types(struct intel_gvt *gvt)
 	 */
 	low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE;
 	high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE;
-	num_types = sizeof(vgpu_types) / sizeof(vgpu_types[0]);
+	num_types = ARRAY_SIZE(vgpu_types);
 
 	gvt->types = kcalloc(num_types, sizeof(struct intel_vgpu_type),
 			     GFP_KERNEL);
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 189b573d02be..372354d33f55 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -572,6 +572,9 @@ struct drm_i915_reg_descriptor {
 #define REG32(_reg, ...) \
 	{ .addr = (_reg), __VA_ARGS__ }
 
+#define REG32_IDX(_reg, idx) \
+	{ .addr = _reg(idx) }
+
 /*
  * Convenience macro for adding 64-bit registers.
  *
@@ -669,6 +672,7 @@ static const struct drm_i915_reg_descriptor gen9_blt_regs[] = {
 	REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),
 	REG32(BCS_SWCTRL),
 	REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),
+	REG32_IDX(RING_CTX_TIMESTAMP, BLT_RING_BASE),
 	REG64_IDX(BCS_GPR, 0),
 	REG64_IDX(BCS_GPR, 1),
 	REG64_IDX(BCS_GPR, 2),
diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c
index 8e45ca3d2ede..55b97c3a3dde 100644
--- a/drivers/gpu/drm/i915/i915_ioc32.c
+++ b/drivers/gpu/drm/i915/i915_ioc32.c
@@ -47,20 +47,16 @@ static int compat_i915_getparam(struct file *file, unsigned int cmd,
 				unsigned long arg)
 {
 	struct drm_i915_getparam32 req32;
-	drm_i915_getparam_t __user *request;
+	struct drm_i915_getparam req;
 
 	if (copy_from_user(&req32, (void __user *)arg, sizeof(req32)))
 		return -EFAULT;
 
-	request = compat_alloc_user_space(sizeof(*request));
-	if (!access_ok(request, sizeof(*request)) ||
-	    __put_user(req32.param, &request->param) ||
-	    __put_user((void __user *)(unsigned long)req32.value,
-		       &request->value))
-		return -EFAULT;
+	req.param = req32.param;
+	req.value = compat_ptr(req32.value);
 
-	return drm_ioctl(file, DRM_IOCTL_I915_GETPARAM,
-			 (unsigned long)request);
+	return drm_ioctl_kernel(file, i915_getparam_ioctl, &req,
+				DRM_RENDER_ALLOW);
 }
 
 static drm_ioctl_compat_t *i915_compat_ioctls[] = {
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 4dc601dffc08..284cf078135a 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -3125,6 +3125,7 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
 
 	val = I915_READ(GEN11_DE_HPD_IMR);
 	val &= ~hotplug_irqs;
+	val |= ~enabled_irqs & hotplug_irqs;
 	I915_WRITE(GEN11_DE_HPD_IMR, val);
 	POSTING_READ(GEN11_DE_HPD_IMR);
 
diff --git a/drivers/gpu/drm/i915/i915_mm.c b/drivers/gpu/drm/i915/i915_mm.c
index b6376b25ef63..43039dc8c607 100644
--- a/drivers/gpu/drm/i915/i915_mm.c
+++ b/drivers/gpu/drm/i915/i915_mm.c
@@ -25,7 +25,6 @@
 #include <linux/mm.h>
 #include <linux/io-mapping.h>
 
-#include <asm/pgtable.h>
 
 #include "i915_drv.h"
 
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index add00ec1f787..02559da61e6e 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -65,7 +65,7 @@ i915_param_named_unsafe(vbt_sdvo_panel_type, int, 0400,
 	"Override/Ignore selection of SDVO panel mode in the VBT "
 	"(-2=ignore, -1=auto [default], index in VBT BIOS table)");
 
-i915_param_named_unsafe(reset, int, 0600,
+i915_param_named_unsafe(reset, uint, 0600,
 	"Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])");
 
 i915_param_named_unsafe(vbt_firmware, charp, 0400,
@@ -173,7 +173,7 @@ i915_param_named(enable_gvt, bool, 0400,
 #endif
 
 #if IS_ENABLED(CONFIG_DRM_I915_UNSTABLE_FAKE_LMEM)
-i915_param_named_unsafe(fake_lmem_start, ulong, 0600,
+i915_param_named_unsafe(fake_lmem_start, ulong, 0400,
 	"Fake LMEM start offset (default: 0)");
 #endif
 
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 45323732f099..4f21bfffbf0e 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -64,7 +64,7 @@ struct drm_printer;
 	param(int, mmio_debug, -IS_ENABLED(CONFIG_DRM_I915_DEBUG_MMIO), 0600) \
 	param(int, edp_vswing, 0, 0400) \
 	param(unsigned int, reset, 3, 0600) \
-	param(unsigned int, inject_probe_failure, 0, 0600) \
+	param(unsigned int, inject_probe_failure, 0, 0) \
 	param(int, fastboot, -1, 0600) \
 	param(int, enable_dpcd_backlight, -1, 0600) \
 	param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 75c60c2afb7e..25329b7600c9 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -3676,7 +3676,7 @@ static int read_properties_unlocked(struct i915_perf *perf,
  * buffered data written by the GPU besides periodic OA metrics.
  *
  * Note we copy the properties from userspace outside of the i915 perf
- * mutex to avoid an awkward lockdep with mmap_sem.
+ * mutex to avoid an awkward lockdep with mmap_lock.
  *
  * Most of the implementation details are handled by
  * i915_perf_open_ioctl_locked() after taking the &perf->lock
@@ -3896,9 +3896,6 @@ static struct i915_oa_reg *alloc_oa_regs(struct i915_perf *perf,
 	if (!n_regs)
 		return NULL;
 
-	if (!access_ok(regs, n_regs * sizeof(u32) * 2))
-		return ERR_PTR(-EFAULT);
-
 	/* No is_valid function means we're not allowing any register to be programmed. */
 	GEM_BUG_ON(!is_valid);
 	if (!is_valid)
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
index e991a707bdb7..962ded9ce73f 100644
--- a/drivers/gpu/drm/i915/i915_pmu.c
+++ b/drivers/gpu/drm/i915/i915_pmu.c
@@ -269,12 +269,48 @@ static bool exclusive_mmio_access(const struct drm_i915_private *i915)
 	return IS_GEN(i915, 7);
 }
 
+static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns)
+{
+	struct intel_engine_pmu *pmu = &engine->pmu;
+	bool busy;
+	u32 val;
+
+	val = ENGINE_READ_FW(engine, RING_CTL);
+	if (val == 0) /* powerwell off => engine idle */
+		return;
+
+	if (val & RING_WAIT)
+		add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
+	if (val & RING_WAIT_SEMAPHORE)
+		add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);
+
+	/* No need to sample when busy stats are supported. */
+	if (intel_engine_supports_stats(engine))
+		return;
+
+	/*
+	 * While waiting on a semaphore or event, MI_MODE reports the
+	 * ring as idle. However, previously using the seqno, and with
+	 * execlists sampling, we account for the ring waiting as the
+	 * engine being busy. Therefore, we record the sample as being
+	 * busy if either waiting or !idle.
+	 */
+	busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
+	if (!busy) {
+		val = ENGINE_READ_FW(engine, RING_MI_MODE);
+		busy = !(val & MODE_IDLE);
+	}
+	if (busy)
+		add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
+}
+
 static void
 engines_sample(struct intel_gt *gt, unsigned int period_ns)
 {
 	struct drm_i915_private *i915 = gt->i915;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
+	unsigned long flags;
 
 	if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 		return;
@@ -283,53 +319,17 @@ engines_sample(struct intel_gt *gt, unsigned int period_ns)
 		return;
 
 	for_each_engine(engine, gt, id) {
-		struct intel_engine_pmu *pmu = &engine->pmu;
-		spinlock_t *mmio_lock;
-		unsigned long flags;
-		bool busy;
-		u32 val;
-
 		if (!intel_engine_pm_get_if_awake(engine))
 			continue;
 
-		mmio_lock = NULL;
-		if (exclusive_mmio_access(i915))
-			mmio_lock = &engine->uncore->lock;
-
-		if (unlikely(mmio_lock))
-			spin_lock_irqsave(mmio_lock, flags);
-
-		val = ENGINE_READ_FW(engine, RING_CTL);
-		if (val == 0) /* powerwell off => engine idle */
-			goto skip;
-
-		if (val & RING_WAIT)
-			add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
-		if (val & RING_WAIT_SEMAPHORE)
-			add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);
-
-		/* No need to sample when busy stats are supported. */
-		if (intel_engine_supports_stats(engine))
-			goto skip;
-
-		/*
-		 * While waiting on a semaphore or event, MI_MODE reports the
-		 * ring as idle. However, previously using the seqno, and with
-		 * execlists sampling, we account for the ring waiting as the
-		 * engine being busy. Therefore, we record the sample as being
-		 * busy if either waiting or !idle.
-		 */
-		busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
-		if (!busy) {
-			val = ENGINE_READ_FW(engine, RING_MI_MODE);
-			busy = !(val & MODE_IDLE);
+		if (exclusive_mmio_access(i915)) {
+			spin_lock_irqsave(&engine->uncore->lock, flags);
+			engine_sample(engine, period_ns);
+			spin_unlock_irqrestore(&engine->uncore->lock, flags);
+		} else {
+			engine_sample(engine, period_ns);
 		}
-		if (busy)
-			add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
 
-skip:
-		if (unlikely(mmio_lock))
-			spin_unlock_irqrestore(mmio_lock, flags);
 		intel_engine_pm_put_async(engine);
 	}
 }
diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h
index 5003a71113cb..8aa7866ec6b6 100644
--- a/drivers/gpu/drm/i915/i915_priolist_types.h
+++ b/drivers/gpu/drm/i915/i915_priolist_types.h
@@ -42,7 +42,7 @@ enum {
  * active request.
  */
 #define I915_PRIORITY_UNPREEMPTABLE INT_MAX
-#define I915_PRIORITY_BARRIER INT_MAX
+#define I915_PRIORITY_BARRIER (I915_PRIORITY_UNPREEMPTABLE - 1)
 
 struct i915_priolist {
 	struct list_head requests[I915_PRIORITY_COUNT];
diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c
index ef25ce6e395e..e75c528ebbe0 100644
--- a/drivers/gpu/drm/i915/i915_query.c
+++ b/drivers/gpu/drm/i915/i915_query.c
@@ -25,10 +25,6 @@ static int copy_query_item(void *query_hdr, size_t query_sz,
 			   query_sz))
 		return -EFAULT;
 
-	if (!access_ok(u64_to_user_ptr(query_item->data_ptr),
-		       total_length))
-		return -EFAULT;
-
 	return 0;
 }
 
@@ -72,20 +68,20 @@ static int query_topology_info(struct drm_i915_private *dev_priv,
 	topo.eu_offset = slice_length + subslice_length;
 	topo.eu_stride = sseu->eu_stride;
 
-	if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr),
+	if (copy_to_user(u64_to_user_ptr(query_item->data_ptr),
 			   &topo, sizeof(topo)))
 		return -EFAULT;
 
-	if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo)),
+	if (copy_to_user(u64_to_user_ptr(query_item->data_ptr + sizeof(topo)),
 			   &sseu->slice_mask, slice_length))
 		return -EFAULT;
 
-	if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr +
+	if (copy_to_user(u64_to_user_ptr(query_item->data_ptr +
 					   sizeof(topo) + slice_length),
 			   sseu->subslice_mask, subslice_length))
 		return -EFAULT;
 
-	if (__copy_to_user(u64_to_user_ptr(query_item->data_ptr +
+	if (copy_to_user(u64_to_user_ptr(query_item->data_ptr +
 					   sizeof(topo) +
 					   slice_length + subslice_length),
 			   sseu->eu_mask, eu_length))
@@ -131,14 +127,14 @@ query_engine_info(struct drm_i915_private *i915,
 		info.engine.engine_instance = engine->uabi_instance;
 		info.capabilities = engine->uabi_capabilities;
 
-		if (__copy_to_user(info_ptr, &info, sizeof(info)))
+		if (copy_to_user(info_ptr, &info, sizeof(info)))
 			return -EFAULT;
 
 		query.num_engines++;
 		info_ptr++;
 	}
 
-	if (__copy_to_user(query_ptr, &query, sizeof(query)))
+	if (copy_to_user(query_ptr, &query, sizeof(query)))
 		return -EFAULT;
 
 	return len;
@@ -158,10 +154,6 @@ static int can_copy_perf_config_registers_or_number(u32 user_n_regs,
 	if (user_n_regs < kernel_n_regs)
 		return -EINVAL;
 
-	if (!access_ok(u64_to_user_ptr(user_regs_ptr),
-		       2 * sizeof(u32) * kernel_n_regs))
-		return -EFAULT;
-
 	return 0;
 }
 
@@ -170,6 +162,7 @@ static int copy_perf_config_registers_or_number(const struct i915_oa_reg *kernel
 						u64 user_regs_ptr,
 						u32 *user_n_regs)
 {
+	u32 __user *p = u64_to_user_ptr(user_regs_ptr);
 	u32 r;
 
 	if (*user_n_regs == 0) {
@@ -179,25 +172,19 @@ static int copy_perf_config_registers_or_number(const struct i915_oa_reg *kernel
 
 	*user_n_regs = kernel_n_regs;
 
-	for (r = 0; r < kernel_n_regs; r++) {
-		u32 __user *user_reg_ptr =
-			u64_to_user_ptr(user_regs_ptr + sizeof(u32) * r * 2);
-		u32 __user *user_val_ptr =
-			u64_to_user_ptr(user_regs_ptr + sizeof(u32) * r * 2 +
-					sizeof(u32));
-		int ret;
-
-		ret = __put_user(i915_mmio_reg_offset(kernel_regs[r].addr),
-				 user_reg_ptr);
-		if (ret)
-			return -EFAULT;
+	if (!user_write_access_begin(p, 2 * sizeof(u32) * kernel_n_regs))
+		return -EFAULT;
 
-		ret = __put_user(kernel_regs[r].value, user_val_ptr);
-		if (ret)
-			return -EFAULT;
+	for (r = 0; r < kernel_n_regs; r++, p += 2) {
+		unsafe_put_user(i915_mmio_reg_offset(kernel_regs[r].addr),
+				p, Efault);
+		unsafe_put_user(kernel_regs[r].value, p + 1, Efault);
 	}
-
+	user_write_access_end();
 	return 0;
+Efault:
+	user_write_access_end();
+	return -EFAULT;
 }
 
 static int query_perf_config_data(struct drm_i915_private *i915,
@@ -233,10 +220,7 @@ static int query_perf_config_data(struct drm_i915_private *i915,
 		return -EINVAL;
 	}
 
-	if (!access_ok(user_query_config_ptr, total_size))
-		return -EFAULT;
-
-	if (__get_user(flags, &user_query_config_ptr->flags))
+	if (get_user(flags, &user_query_config_ptr->flags))
 		return -EFAULT;
 
 	if (flags != 0)
@@ -249,7 +233,7 @@ static int query_perf_config_data(struct drm_i915_private *i915,
 		BUILD_BUG_ON(sizeof(user_query_config_ptr->uuid) >= sizeof(uuid));
 
 		memset(&uuid, 0, sizeof(uuid));
-		if (__copy_from_user(uuid, user_query_config_ptr->uuid,
+		if (copy_from_user(uuid, user_query_config_ptr->uuid,
 				     sizeof(user_query_config_ptr->uuid)))
 			return -EFAULT;
 
@@ -263,7 +247,7 @@ static int query_perf_config_data(struct drm_i915_private *i915,
 		}
 		rcu_read_unlock();
 	} else {
-		if (__get_user(config_id, &user_query_config_ptr->config))
+		if (get_user(config_id, &user_query_config_ptr->config))
 			return -EFAULT;
 
 		oa_config = i915_perf_get_oa_config(perf, config_id);
@@ -271,8 +255,7 @@ static int query_perf_config_data(struct drm_i915_private *i915,
 	if (!oa_config)
 		return -ENOENT;
 
-	if (__copy_from_user(&user_config, user_config_ptr,
-			     sizeof(user_config))) {
+	if (copy_from_user(&user_config, user_config_ptr, sizeof(user_config))) {
 		ret = -EFAULT;
 		goto out;
 	}
@@ -318,8 +301,7 @@ static int query_perf_config_data(struct drm_i915_private *i915,
 
 	memcpy(user_config.uuid, oa_config->uuid, sizeof(user_config.uuid));
 
-	if (__copy_to_user(user_config_ptr, &user_config,
-			   sizeof(user_config))) {
+	if (copy_to_user(user_config_ptr, &user_config, sizeof(user_config))) {
 		ret = -EFAULT;
 		goto out;
 	}
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 6c076a24eb82..06cd1d28a176 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -186,7 +186,7 @@ typedef struct {
 
 #define INVALID_MMIO_REG _MMIO(0)
 
-static inline u32 i915_mmio_reg_offset(i915_reg_t reg)
+static __always_inline u32 i915_mmio_reg_offset(i915_reg_t reg)
 {
 	return reg.reg;
 }
@@ -7896,7 +7896,7 @@ enum {
 
 /* GEN7 chicken */
 #define GEN7_COMMON_SLICE_CHICKEN1		_MMIO(0x7010)
-  #define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC	((1 << 10) | (1 << 26))
+  #define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC	(1 << 10)
   #define GEN9_RHWO_OPTIMIZATION_DISABLE	(1 << 14)
 
 #define COMMON_SLICE_CHICKEN2					_MMIO(0x7014)
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index 526c1e9acbd5..def62100e666 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -121,8 +121,39 @@ static void i915_fence_release(struct dma_fence *fence)
 	i915_sw_fence_fini(&rq->submit);
 	i915_sw_fence_fini(&rq->semaphore);
 
-	/* Keep one request on each engine for reserved use under mempressure */
-	if (!cmpxchg(&rq->engine->request_pool, NULL, rq))
+	/*
+	 * Keep one request on each engine for reserved use under mempressure
+	 *
+	 * We do not hold a reference to the engine here and so have to be
+	 * very careful in what rq->engine we poke. The virtual engine is
+	 * referenced via the rq->context and we released that ref during
+	 * i915_request_retire(), ergo we must not dereference a virtual
+	 * engine here. Not that we would want to, as the only consumer of
+	 * the reserved engine->request_pool is the power management parking,
+	 * which must-not-fail, and that is only run on the physical engines.
+	 *
+	 * Since the request must have been executed to be have completed,
+	 * we know that it will have been processed by the HW and will
+	 * not be unsubmitted again, so rq->engine and rq->execution_mask
+	 * at this point is stable. rq->execution_mask will be a single
+	 * bit if the last and _only_ engine it could execution on was a
+	 * physical engine, if it's multiple bits then it started on and
+	 * could still be on a virtual engine. Thus if the mask is not a
+	 * power-of-two we assume that rq->engine may still be a virtual
+	 * engine and so a dangling invalid pointer that we cannot dereference
+	 *
+	 * For example, consider the flow of a bonded request through a virtual
+	 * engine. The request is created with a wide engine mask (all engines
+	 * that we might execute on). On processing the bond, the request mask
+	 * is reduced to one or more engines. If the request is subsequently
+	 * bound to a single engine, it will then be constrained to only
+	 * execute on that engine and never returned to the virtual engine
+	 * after timeslicing away, see __unwind_incomplete_requests(). Thus we
+	 * know that if the rq->execution_mask is a single bit, rq->engine
+	 * can be a physical engine with the exact corresponding mask.
+	 */
+	if (is_power_of_2(rq->execution_mask) &&
+	    !cmpxchg(&rq->engine->request_pool, NULL, rq))
 		return;
 
 	kmem_cache_free(global.slab_requests, rq);
@@ -326,6 +357,53 @@ void i915_request_retire_upto(struct i915_request *rq)
 	} while (i915_request_retire(tmp) && tmp != rq);
 }
 
+static struct i915_request * const *
+__engine_active(struct intel_engine_cs *engine)
+{
+	return READ_ONCE(engine->execlists.active);
+}
+
+static bool __request_in_flight(const struct i915_request *signal)
+{
+	struct i915_request * const *port, *rq;
+	bool inflight = false;
+
+	if (!i915_request_is_ready(signal))
+		return false;
+
+	/*
+	 * Even if we have unwound the request, it may still be on
+	 * the GPU (preempt-to-busy). If that request is inside an
+	 * unpreemptible critical section, it will not be removed. Some
+	 * GPU functions may even be stuck waiting for the paired request
+	 * (__await_execution) to be submitted and cannot be preempted
+	 * until the bond is executing.
+	 *
+	 * As we know that there are always preemption points between
+	 * requests, we know that only the currently executing request
+	 * may be still active even though we have cleared the flag.
+	 * However, we can't rely on our tracking of ELSP[0] to known
+	 * which request is currently active and so maybe stuck, as
+	 * the tracking maybe an event behind. Instead assume that
+	 * if the context is still inflight, then it is still active
+	 * even if the active flag has been cleared.
+	 */
+	if (!intel_context_inflight(signal->context))
+		return false;
+
+	rcu_read_lock();
+	for (port = __engine_active(signal->engine); (rq = *port); port++) {
+		if (rq->context == signal->context) {
+			inflight = i915_seqno_passed(rq->fence.seqno,
+						     signal->fence.seqno);
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return inflight;
+}
+
 static int
 __await_execution(struct i915_request *rq,
 		  struct i915_request *signal,
@@ -356,7 +434,7 @@ __await_execution(struct i915_request *rq,
 	}
 
 	spin_lock_irq(&signal->lock);
-	if (i915_request_is_active(signal)) {
+	if (i915_request_is_active(signal) || __request_in_flight(signal)) {
 		if (hook) {
 			hook(rq, &signal->fence);
 			i915_request_put(signal);
@@ -1022,37 +1100,91 @@ await_fence:
 					     I915_FENCE_GFP);
 }
 
+static bool intel_timeline_sync_has_start(struct intel_timeline *tl,
+					  struct dma_fence *fence)
+{
+	return __intel_timeline_sync_is_later(tl,
+					      fence->context,
+					      fence->seqno - 1);
+}
+
+static int intel_timeline_sync_set_start(struct intel_timeline *tl,
+					 const struct dma_fence *fence)
+{
+	return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);
+}
+
 static int
-i915_request_await_request(struct i915_request *to, struct i915_request *from)
+__i915_request_await_execution(struct i915_request *to,
+			       struct i915_request *from,
+			       void (*hook)(struct i915_request *rq,
+					    struct dma_fence *signal))
 {
-	int ret;
+	int err;
 
-	GEM_BUG_ON(to == from);
-	GEM_BUG_ON(to->timeline == from->timeline);
+	GEM_BUG_ON(intel_context_is_barrier(from->context));
 
-	if (i915_request_completed(from)) {
-		i915_sw_fence_set_error_once(&to->submit, from->fence.error);
+	/* Submit both requests at the same time */
+	err = __await_execution(to, from, hook, I915_FENCE_GFP);
+	if (err)
+		return err;
+
+	/* Squash repeated depenendices to the same timelines */
+	if (intel_timeline_sync_has_start(i915_request_timeline(to),
+					  &from->fence))
 		return 0;
+
+	/*
+	 * Wait until the start of this request.
+	 *
+	 * The execution cb fires when we submit the request to HW. But in
+	 * many cases this may be long before the request itself is ready to
+	 * run (consider that we submit 2 requests for the same context, where
+	 * the request of interest is behind an indefinite spinner). So we hook
+	 * up to both to reduce our queues and keep the execution lag minimised
+	 * in the worst case, though we hope that the await_start is elided.
+	 */
+	err = i915_request_await_start(to, from);
+	if (err < 0)
+		return err;
+
+	/*
+	 * Ensure both start together [after all semaphores in signal]
+	 *
+	 * Now that we are queued to the HW at roughly the same time (thanks
+	 * to the execute cb) and are ready to run at roughly the same time
+	 * (thanks to the await start), our signaler may still be indefinitely
+	 * delayed by waiting on a semaphore from a remote engine. If our
+	 * signaler depends on a semaphore, so indirectly do we, and we do not
+	 * want to start our payload until our signaler also starts theirs.
+	 * So we wait.
+	 *
+	 * However, there is also a second condition for which we need to wait
+	 * for the precise start of the signaler. Consider that the signaler
+	 * was submitted in a chain of requests following another context
+	 * (with just an ordinary intra-engine fence dependency between the
+	 * two). In this case the signaler is queued to HW, but not for
+	 * immediate execution, and so we must wait until it reaches the
+	 * active slot.
+	 */
+	if (intel_engine_has_semaphores(to->engine) &&
+	    !i915_request_has_initial_breadcrumb(to)) {
+		err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);
+		if (err < 0)
+			return err;
 	}
 
+	/* Couple the dependency tree for PI on this exposed to->fence */
 	if (to->engine->schedule) {
-		ret = i915_sched_node_add_dependency(&to->sched,
+		err = i915_sched_node_add_dependency(&to->sched,
 						     &from->sched,
-						     I915_DEPENDENCY_EXTERNAL);
-		if (ret < 0)
-			return ret;
+						     I915_DEPENDENCY_WEAK);
+		if (err < 0)
+			return err;
 	}
 
-	if (to->engine == from->engine)
-		ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
-						       &from->submit,
-						       I915_FENCE_GFP);
-	else
-		ret = emit_semaphore_wait(to, from, I915_FENCE_GFP);
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	return intel_timeline_sync_set_start(i915_request_timeline(to),
+					     &from->fence);
 }
 
 static void mark_external(struct i915_request *rq)
@@ -1105,23 +1237,20 @@ i915_request_await_external(struct i915_request *rq, struct dma_fence *fence)
 }
 
 int
-i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
+i915_request_await_execution(struct i915_request *rq,
+			     struct dma_fence *fence,
+			     void (*hook)(struct i915_request *rq,
+					  struct dma_fence *signal))
 {
 	struct dma_fence **child = &fence;
 	unsigned int nchild = 1;
 	int ret;
 
-	/*
-	 * Note that if the fence-array was created in signal-on-any mode,
-	 * we should *not* decompose it into its individual fences. However,
-	 * we don't currently store which mode the fence-array is operating
-	 * in. Fortunately, the only user of signal-on-any is private to
-	 * amdgpu and we should not see any incoming fence-array from
-	 * sync-file being in signal-on-any mode.
-	 */
 	if (dma_fence_is_array(fence)) {
 		struct dma_fence_array *array = to_dma_fence_array(fence);
 
+		/* XXX Error for signal-on-any fence arrays */
+
 		child = array->fences;
 		nchild = array->num_fences;
 		GEM_BUG_ON(!nchild);
@@ -1134,138 +1263,95 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
 			continue;
 		}
 
-		/*
-		 * Requests on the same timeline are explicitly ordered, along
-		 * with their dependencies, by i915_request_add() which ensures
-		 * that requests are submitted in-order through each ring.
-		 */
 		if (fence->context == rq->fence.context)
 			continue;
 
-		/* Squash repeated waits to the same timelines */
-		if (fence->context &&
-		    intel_timeline_sync_is_later(i915_request_timeline(rq),
-						 fence))
-			continue;
+		/*
+		 * We don't squash repeated fence dependencies here as we
+		 * want to run our callback in all cases.
+		 */
 
 		if (dma_fence_is_i915(fence))
-			ret = i915_request_await_request(rq, to_request(fence));
+			ret = __i915_request_await_execution(rq,
+							     to_request(fence),
+							     hook);
 		else
 			ret = i915_request_await_external(rq, fence);
 		if (ret < 0)
 			return ret;
-
-		/* Record the latest fence used against each timeline */
-		if (fence->context)
-			intel_timeline_sync_set(i915_request_timeline(rq),
-						fence);
 	} while (--nchild);
 
 	return 0;
 }
 
-static bool intel_timeline_sync_has_start(struct intel_timeline *tl,
-					  struct dma_fence *fence)
-{
-	return __intel_timeline_sync_is_later(tl,
-					      fence->context,
-					      fence->seqno - 1);
-}
-
-static int intel_timeline_sync_set_start(struct intel_timeline *tl,
-					 const struct dma_fence *fence)
+static int
+await_request_submit(struct i915_request *to, struct i915_request *from)
 {
-	return __intel_timeline_sync_set(tl, fence->context, fence->seqno - 1);
+	/*
+	 * If we are waiting on a virtual engine, then it may be
+	 * constrained to execute on a single engine *prior* to submission.
+	 * When it is submitted, it will be first submitted to the virtual
+	 * engine and then passed to the physical engine. We cannot allow
+	 * the waiter to be submitted immediately to the physical engine
+	 * as it may then bypass the virtual request.
+	 */
+	if (to->engine == READ_ONCE(from->engine))
+		return i915_sw_fence_await_sw_fence_gfp(&to->submit,
+							&from->submit,
+							I915_FENCE_GFP);
+	else
+		return __i915_request_await_execution(to, from, NULL);
 }
 
 static int
-__i915_request_await_execution(struct i915_request *to,
-			       struct i915_request *from,
-			       void (*hook)(struct i915_request *rq,
-					    struct dma_fence *signal))
+i915_request_await_request(struct i915_request *to, struct i915_request *from)
 {
-	int err;
-
-	GEM_BUG_ON(intel_context_is_barrier(from->context));
+	int ret;
 
-	/* Submit both requests at the same time */
-	err = __await_execution(to, from, hook, I915_FENCE_GFP);
-	if (err)
-		return err;
+	GEM_BUG_ON(to == from);
+	GEM_BUG_ON(to->timeline == from->timeline);
 
-	/* Squash repeated depenendices to the same timelines */
-	if (intel_timeline_sync_has_start(i915_request_timeline(to),
-					  &from->fence))
+	if (i915_request_completed(from)) {
+		i915_sw_fence_set_error_once(&to->submit, from->fence.error);
 		return 0;
-
-	/*
-	 * Wait until the start of this request.
-	 *
-	 * The execution cb fires when we submit the request to HW. But in
-	 * many cases this may be long before the request itself is ready to
-	 * run (consider that we submit 2 requests for the same context, where
-	 * the request of interest is behind an indefinite spinner). So we hook
-	 * up to both to reduce our queues and keep the execution lag minimised
-	 * in the worst case, though we hope that the await_start is elided.
-	 */
-	err = i915_request_await_start(to, from);
-	if (err < 0)
-		return err;
-
-	/*
-	 * Ensure both start together [after all semaphores in signal]
-	 *
-	 * Now that we are queued to the HW at roughly the same time (thanks
-	 * to the execute cb) and are ready to run at roughly the same time
-	 * (thanks to the await start), our signaler may still be indefinitely
-	 * delayed by waiting on a semaphore from a remote engine. If our
-	 * signaler depends on a semaphore, so indirectly do we, and we do not
-	 * want to start our payload until our signaler also starts theirs.
-	 * So we wait.
-	 *
-	 * However, there is also a second condition for which we need to wait
-	 * for the precise start of the signaler. Consider that the signaler
-	 * was submitted in a chain of requests following another context
-	 * (with just an ordinary intra-engine fence dependency between the
-	 * two). In this case the signaler is queued to HW, but not for
-	 * immediate execution, and so we must wait until it reaches the
-	 * active slot.
-	 */
-	if (intel_engine_has_semaphores(to->engine) &&
-	    !i915_request_has_initial_breadcrumb(to)) {
-		err = __emit_semaphore_wait(to, from, from->fence.seqno - 1);
-		if (err < 0)
-			return err;
 	}
 
-	/* Couple the dependency tree for PI on this exposed to->fence */
 	if (to->engine->schedule) {
-		err = i915_sched_node_add_dependency(&to->sched,
+		ret = i915_sched_node_add_dependency(&to->sched,
 						     &from->sched,
-						     I915_DEPENDENCY_WEAK);
-		if (err < 0)
-			return err;
+						     I915_DEPENDENCY_EXTERNAL);
+		if (ret < 0)
+			return ret;
 	}
 
-	return intel_timeline_sync_set_start(i915_request_timeline(to),
-					     &from->fence);
+	if (is_power_of_2(to->execution_mask | READ_ONCE(from->execution_mask)))
+		ret = await_request_submit(to, from);
+	else
+		ret = emit_semaphore_wait(to, from, I915_FENCE_GFP);
+	if (ret < 0)
+		return ret;
+
+	return 0;
 }
 
 int
-i915_request_await_execution(struct i915_request *rq,
-			     struct dma_fence *fence,
-			     void (*hook)(struct i915_request *rq,
-					  struct dma_fence *signal))
+i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence)
 {
 	struct dma_fence **child = &fence;
 	unsigned int nchild = 1;
 	int ret;
 
+	/*
+	 * Note that if the fence-array was created in signal-on-any mode,
+	 * we should *not* decompose it into its individual fences. However,
+	 * we don't currently store which mode the fence-array is operating
+	 * in. Fortunately, the only user of signal-on-any is private to
+	 * amdgpu and we should not see any incoming fence-array from
+	 * sync-file being in signal-on-any mode.
+	 */
 	if (dma_fence_is_array(fence)) {
 		struct dma_fence_array *array = to_dma_fence_array(fence);
 
-		/* XXX Error for signal-on-any fence arrays */
-
 		child = array->fences;
 		nchild = array->num_fences;
 		GEM_BUG_ON(!nchild);
@@ -1278,22 +1364,31 @@ i915_request_await_execution(struct i915_request *rq,
 			continue;
 		}
 
+		/*
+		 * Requests on the same timeline are explicitly ordered, along
+		 * with their dependencies, by i915_request_add() which ensures
+		 * that requests are submitted in-order through each ring.
+		 */
 		if (fence->context == rq->fence.context)
 			continue;
 
-		/*
-		 * We don't squash repeated fence dependencies here as we
-		 * want to run our callback in all cases.
-		 */
+		/* Squash repeated waits to the same timelines */
+		if (fence->context &&
+		    intel_timeline_sync_is_later(i915_request_timeline(rq),
+						 fence))
+			continue;
 
 		if (dma_fence_is_i915(fence))
-			ret = __i915_request_await_execution(rq,
-							     to_request(fence),
-							     hook);
+			ret = i915_request_await_request(rq, to_request(fence));
 		else
 			ret = i915_request_await_external(rq, fence);
 		if (ret < 0)
 			return ret;
+
+		/* Record the latest fence used against each timeline */
+		if (fence->context)
+			intel_timeline_sync_set(i915_request_timeline(rq),
+						fence);
 	} while (--nchild);
 
 	return 0;
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index f4ea318781f0..cbb880b10c65 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -209,14 +209,6 @@ static void kick_submission(struct intel_engine_cs *engine,
 	if (!inflight)
 		goto unlock;
 
-	ENGINE_TRACE(engine,
-		     "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
-		     prio,
-		     rq->fence.context, rq->fence.seqno,
-		     inflight->fence.context, inflight->fence.seqno,
-		     inflight->sched.attr.priority);
-	engine->execlists.queue_priority_hint = prio;
-
 	/*
 	 * If we are already the currently executing context, don't
 	 * bother evaluating if we should preempt ourselves.
@@ -224,6 +216,14 @@ static void kick_submission(struct intel_engine_cs *engine,
 	if (inflight->context == rq->context)
 		goto unlock;
 
+	ENGINE_TRACE(engine,
+		     "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n",
+		     prio,
+		     rq->fence.context, rq->fence.seqno,
+		     inflight->fence.context, inflight->fence.seqno,
+		     inflight->sched.attr.priority);
+
+	engine->execlists.queue_priority_hint = prio;
 	if (need_preempt(prio, rq_prio(inflight)))
 		tasklet_hi_schedule(&engine->execlists.tasklet);
 
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 696491d71a1d..07f663cd2d1c 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -6830,16 +6830,6 @@ static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
 	I915_WRITE(ILK_DISPLAY_CHICKEN2,
 		   I915_READ(ILK_DISPLAY_CHICKEN2) |
 		   ILK_ELPIN_409_SELECT);
-	I915_WRITE(_3D_CHICKEN2,
-		   _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
-		   _3D_CHICKEN2_WM_READ_PIPELINED);
-
-	/* WaDisableRenderCachePipelinedFlush:ilk */
-	I915_WRITE(CACHE_MODE_0,
-		   _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
-
-	/* WaDisable_RenderCache_OperationalFlush:ilk */
-	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
 
 	g4x_disable_trickle_feed(dev_priv);
 
@@ -6902,27 +6892,6 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
 		   I915_READ(ILK_DISPLAY_CHICKEN2) |
 		   ILK_ELPIN_409_SELECT);
 
-	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
-	I915_WRITE(_3D_CHICKEN,
-		   _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
-
-	/* WaDisable_RenderCache_OperationalFlush:snb */
-	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
-
-	/*
-	 * BSpec recoomends 8x4 when MSAA is used,
-	 * however in practice 16x4 seems fastest.
-	 *
-	 * Note that PS/WM thread counts depend on the WIZ hashing
-	 * disable bit, which we don't touch here, but it's good
-	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
-	 */
-	I915_WRITE(GEN6_GT_MODE,
-		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
-
-	I915_WRITE(CACHE_MODE_0,
-		   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
-
 	I915_WRITE(GEN6_UCGCTL1,
 		   I915_READ(GEN6_UCGCTL1) |
 		   GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
@@ -6945,18 +6914,6 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
 		   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
 		   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 
-	/* WaStripsFansDisableFastClipPerformanceFix:snb */
-	I915_WRITE(_3D_CHICKEN3,
-		   _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
-
-	/*
-	 * Bspec says:
-	 * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
-	 * 3DSTATE_SF number of SF output attributes is more than 16."
-	 */
-	I915_WRITE(_3D_CHICKEN3,
-		   _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
-
 	/*
 	 * According to the spec the following bits should be
 	 * set in order to enable memory self-refresh and fbc:
@@ -6986,24 +6943,6 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
 	gen6_check_mch_setup(dev_priv);
 }
 
-static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
-{
-	u32 reg = I915_READ(GEN7_FF_THREAD_MODE);
-
-	/*
-	 * WaVSThreadDispatchOverride:ivb,vlv
-	 *
-	 * This actually overrides the dispatch
-	 * mode for all thread types.
-	 */
-	reg &= ~GEN7_FF_SCHED_MASK;
-	reg |= GEN7_FF_TS_SCHED_HW;
-	reg |= GEN7_FF_VS_SCHED_HW;
-	reg |= GEN7_FF_DS_SCHED_HW;
-
-	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
-}
-
 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
 {
 	/*
@@ -7230,45 +7169,10 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
 
 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	/* L3 caching of data atomics doesn't work -- disable it. */
-	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
-	I915_WRITE(HSW_ROW_CHICKEN3,
-		   _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
-
 	/* This is required by WaCatErrorRejectionIssue:hsw */
 	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-			I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
-			GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
-
-	/* WaVSRefCountFullforceMissDisable:hsw */
-	I915_WRITE(GEN7_FF_THREAD_MODE,
-		   I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
-
-	/* WaDisable_RenderCache_OperationalFlush:hsw */
-	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
-
-	/* enable HiZ Raw Stall Optimization */
-	I915_WRITE(CACHE_MODE_0_GEN7,
-		   _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
-
-	/* WaDisable4x2SubspanOptimization:hsw */
-	I915_WRITE(CACHE_MODE_1,
-		   _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
-
-	/*
-	 * BSpec recommends 8x4 when MSAA is used,
-	 * however in practice 16x4 seems fastest.
-	 *
-	 * Note that PS/WM thread counts depend on the WIZ hashing
-	 * disable bit, which we don't touch here, but it's good
-	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
-	 */
-	I915_WRITE(GEN7_GT_MODE,
-		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
-
-	/* WaSampleCChickenBitEnable:hsw */
-	I915_WRITE(HALF_SLICE_CHICKEN3,
-		   _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
+		   I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
+		   GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
 	/* WaSwitchSolVfFArbitrationPriority:hsw */
 	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
@@ -7282,32 +7186,11 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
 
 	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 
-	/* WaDisableEarlyCull:ivb */
-	I915_WRITE(_3D_CHICKEN3,
-		   _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
-
 	/* WaDisableBackToBackFlipFix:ivb */
 	I915_WRITE(IVB_CHICKEN3,
 		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
-	/* WaDisablePSDDualDispatchEnable:ivb */
-	if (IS_IVB_GT1(dev_priv))
-		I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
-			   _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
-
-	/* WaDisable_RenderCache_OperationalFlush:ivb */
-	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
-
-	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
-	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
-		   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
-
-	/* WaApplyL3ControlAndL3ChickenMode:ivb */
-	I915_WRITE(GEN7_L3CNTLREG1,
-			GEN7_WA_FOR_GEN7_L3_CONTROL);
-	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
-		   GEN7_WA_L3_CHICKEN_MODE);
 	if (IS_IVB_GT1(dev_priv))
 		I915_WRITE(GEN7_ROW_CHICKEN2,
 			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
@@ -7319,10 +7202,6 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
 			   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 	}
 
-	/* WaForceL3Serialization:ivb */
-	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
-		   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
-
 	/*
 	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 	 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
@@ -7337,29 +7216,6 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
 
 	g4x_disable_trickle_feed(dev_priv);
 
-	gen7_setup_fixed_func_scheduler(dev_priv);
-
-	if (0) { /* causes HiZ corruption on ivb:gt1 */
-		/* enable HiZ Raw Stall Optimization */
-		I915_WRITE(CACHE_MODE_0_GEN7,
-			   _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
-	}
-
-	/* WaDisable4x2SubspanOptimization:ivb */
-	I915_WRITE(CACHE_MODE_1,
-		   _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
-
-	/*
-	 * BSpec recommends 8x4 when MSAA is used,
-	 * however in practice 16x4 seems fastest.
-	 *
-	 * Note that PS/WM thread counts depend on the WIZ hashing
-	 * disable bit, which we don't touch here, but it's good
-	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
-	 */
-	I915_WRITE(GEN7_GT_MODE,
-		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
-
 	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 	snpcr &= ~GEN6_MBC_SNPCR_MASK;
 	snpcr |= GEN6_MBC_SNPCR_MED;
@@ -7373,28 +7229,11 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
 
 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	/* WaDisableEarlyCull:vlv */
-	I915_WRITE(_3D_CHICKEN3,
-		   _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
-
 	/* WaDisableBackToBackFlipFix:vlv */
 	I915_WRITE(IVB_CHICKEN3,
 		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
-	/* WaPsdDispatchEnable:vlv */
-	/* WaDisablePSDDualDispatchEnable:vlv */
-	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
-		   _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
-				      GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
-
-	/* WaDisable_RenderCache_OperationalFlush:vlv */
-	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
-
-	/* WaForceL3Serialization:vlv */
-	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
-		   ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
-
 	/* WaDisableDopClockGating:vlv */
 	I915_WRITE(GEN7_ROW_CHICKEN2,
 		   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
@@ -7404,8 +7243,6 @@ static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
 		   I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
 		   GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-	gen7_setup_fixed_func_scheduler(dev_priv);
-
 	/*
 	 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 	 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
@@ -7420,30 +7257,6 @@ static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
 		   I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 
 	/*
-	 * BSpec says this must be set, even though
-	 * WaDisable4x2SubspanOptimization isn't listed for VLV.
-	 */
-	I915_WRITE(CACHE_MODE_1,
-		   _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
-
-	/*
-	 * BSpec recommends 8x4 when MSAA is used,
-	 * however in practice 16x4 seems fastest.
-	 *
-	 * Note that PS/WM thread counts depend on the WIZ hashing
-	 * disable bit, which we don't touch here, but it's good
-	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
-	 */
-	I915_WRITE(GEN7_GT_MODE,
-		   _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
-
-	/*
-	 * WaIncreaseL3CreditsForVLVB0:vlv
-	 * This is the hardware default actually.
-	 */
-	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
-
-	/*
 	 * WaDisableVLVClockGating_VBIIssue:vlv
 	 * Disable clock gating on th GCFG unit to prevent a delay
 	 * in the reporting of vblank events.
@@ -7495,13 +7308,6 @@ static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
 		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
 
-	/* WaDisableRenderCachePipelinedFlush */
-	I915_WRITE(CACHE_MODE_0,
-		   _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
-
-	/* WaDisable_RenderCache_OperationalFlush:g4x */
-	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
-
 	g4x_disable_trickle_feed(dev_priv);
 }
 
@@ -7517,11 +7323,6 @@ static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
 	intel_uncore_write(uncore,
 			   MI_ARB_STATE,
 			   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
-
-	/* WaDisable_RenderCache_OperationalFlush:gen4 */
-	intel_uncore_write(uncore,
-			   CACHE_MODE_0,
-			   _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
 }
 
 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -7534,9 +7335,6 @@ static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
 	I915_WRITE(RENCLK_GATE_D2, 0);
 	I915_WRITE(MI_ARB_STATE,
 		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
-
-	/* WaDisable_RenderCache_OperationalFlush:gen4 */
-	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
 }
 
 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
index 6a2be7d0dd95..6090ce35226b 100644
--- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
@@ -21,6 +21,7 @@ selftest(fence, i915_sw_fence_mock_selftests)
 selftest(scatterlist, scatterlist_mock_selftests)
 selftest(syncmap, i915_syncmap_mock_selftests)
 selftest(uncore, intel_uncore_mock_selftests)
+selftest(ring, intel_ring_mock_selftests)
 selftest(engine, intel_engine_cs_mock_selftests)
 selftest(timelines, intel_timeline_mock_selftests)
 selftest(requests, i915_request_mock_selftests)
diff --git a/drivers/gpu/drm/mediatek/Kconfig b/drivers/gpu/drm/mediatek/Kconfig
index fa5ffc4fe823..c420f5a3d33b 100644
--- a/drivers/gpu/drm/mediatek/Kconfig
+++ b/drivers/gpu/drm/mediatek/Kconfig
@@ -11,6 +11,7 @@ config DRM_MEDIATEK
 	select DRM_MIPI_DSI
 	select DRM_PANEL
 	select MEMORY
+	select MTK_MMSYS
 	select MTK_SMI
 	select VIDEOMODE_HELPERS
 	help
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_color.c b/drivers/gpu/drm/mediatek/mtk_disp_color.c
index 6fb0d6983a4a..3ae9c810845b 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_color.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_color.c
@@ -119,7 +119,10 @@ static int mtk_disp_color_probe(struct platform_device *pdev)
 	ret = mtk_ddp_comp_init(dev, dev->of_node, &priv->ddp_comp, comp_id,
 				&mtk_disp_color_funcs);
 	if (ret) {
-		dev_err(dev, "Failed to initialize component: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to initialize component: %d\n",
+				ret);
+
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
index 891d80c73e04..28651bc579bc 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
@@ -386,7 +386,10 @@ static int mtk_disp_ovl_probe(struct platform_device *pdev)
 	ret = mtk_ddp_comp_init(dev, dev->of_node, &priv->ddp_comp, comp_id,
 				&mtk_disp_ovl_funcs);
 	if (ret) {
-		dev_err(dev, "Failed to initialize component: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to initialize component: %d\n",
+				ret);
+
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
index 0cb848d64206..e04319fedf46 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
@@ -294,7 +294,10 @@ static int mtk_disp_rdma_probe(struct platform_device *pdev)
 	ret = mtk_ddp_comp_init(dev, dev->of_node, &priv->ddp_comp, comp_id,
 				&mtk_disp_rdma_funcs);
 	if (ret) {
-		dev_err(dev, "Failed to initialize component: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to initialize component: %d\n",
+				ret);
+
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c
index 945c3ac92998..d4f0fb7ad312 100644
--- a/drivers/gpu/drm/mediatek/mtk_dpi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dpi.c
@@ -739,21 +739,27 @@ static int mtk_dpi_probe(struct platform_device *pdev)
 	dpi->engine_clk = devm_clk_get(dev, "engine");
 	if (IS_ERR(dpi->engine_clk)) {
 		ret = PTR_ERR(dpi->engine_clk);
-		dev_err(dev, "Failed to get engine clock: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get engine clock: %d\n", ret);
+
 		return ret;
 	}
 
 	dpi->pixel_clk = devm_clk_get(dev, "pixel");
 	if (IS_ERR(dpi->pixel_clk)) {
 		ret = PTR_ERR(dpi->pixel_clk);
-		dev_err(dev, "Failed to get pixel clock: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get pixel clock: %d\n", ret);
+
 		return ret;
 	}
 
 	dpi->tvd_clk = devm_clk_get(dev, "pll");
 	if (IS_ERR(dpi->tvd_clk)) {
 		ret = PTR_ERR(dpi->tvd_clk);
-		dev_err(dev, "Failed to get tvdpll clock: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get tvdpll clock: %d\n", ret);
+
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
index fe85e487e477..fe46c4bac64d 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
@@ -6,6 +6,7 @@
 #include <linux/clk.h>
 #include <linux/pm_runtime.h>
 #include <linux/soc/mediatek/mtk-cmdq.h>
+#include <linux/soc/mediatek/mtk-mmsys.h>
 
 #include <asm/barrier.h>
 #include <soc/mediatek/smi.h>
@@ -28,7 +29,7 @@
  * @enabled: records whether crtc_enable succeeded
  * @planes: array of 4 drm_plane structures, one for each overlay plane
  * @pending_planes: whether any plane has pending changes to be applied
- * @config_regs: memory mapped mmsys configuration register space
+ * @mmsys_dev: pointer to the mmsys device for configuration registers
  * @mutex: handle to one of the ten disp_mutex streams
  * @ddp_comp_nr: number of components in ddp_comp
  * @ddp_comp: array of pointers the mtk_ddp_comp structures used by this crtc
@@ -50,7 +51,7 @@ struct mtk_drm_crtc {
 	u32				cmdq_event;
 #endif
 
-	void __iomem			*config_regs;
+	struct device			*mmsys_dev;
 	struct mtk_disp_mutex		*mutex;
 	unsigned int			ddp_comp_nr;
 	struct mtk_ddp_comp		**ddp_comp;
@@ -300,9 +301,9 @@ static int mtk_crtc_ddp_hw_init(struct mtk_drm_crtc *mtk_crtc)
 
 	DRM_DEBUG_DRIVER("mediatek_ddp_ddp_path_setup\n");
 	for (i = 0; i < mtk_crtc->ddp_comp_nr - 1; i++) {
-		mtk_ddp_add_comp_to_path(mtk_crtc->config_regs,
-					 mtk_crtc->ddp_comp[i]->id,
-					 mtk_crtc->ddp_comp[i + 1]->id);
+		mtk_mmsys_ddp_connect(mtk_crtc->mmsys_dev,
+				      mtk_crtc->ddp_comp[i]->id,
+				      mtk_crtc->ddp_comp[i + 1]->id);
 		mtk_disp_mutex_add_comp(mtk_crtc->mutex,
 					mtk_crtc->ddp_comp[i]->id);
 	}
@@ -360,9 +361,9 @@ static void mtk_crtc_ddp_hw_fini(struct mtk_drm_crtc *mtk_crtc)
 					   mtk_crtc->ddp_comp[i]->id);
 	mtk_disp_mutex_disable(mtk_crtc->mutex);
 	for (i = 0; i < mtk_crtc->ddp_comp_nr - 1; i++) {
-		mtk_ddp_remove_comp_from_path(mtk_crtc->config_regs,
-					      mtk_crtc->ddp_comp[i]->id,
-					      mtk_crtc->ddp_comp[i + 1]->id);
+		mtk_mmsys_ddp_disconnect(mtk_crtc->mmsys_dev,
+					 mtk_crtc->ddp_comp[i]->id,
+					 mtk_crtc->ddp_comp[i + 1]->id);
 		mtk_disp_mutex_remove_comp(mtk_crtc->mutex,
 					   mtk_crtc->ddp_comp[i]->id);
 	}
@@ -766,7 +767,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 	if (!mtk_crtc)
 		return -ENOMEM;
 
-	mtk_crtc->config_regs = priv->config_regs;
+	mtk_crtc->mmsys_dev = priv->mmsys_dev;
 	mtk_crtc->ddp_comp_nr = path_len;
 	mtk_crtc->ddp_comp = devm_kmalloc_array(dev, mtk_crtc->ddp_comp_nr,
 						sizeof(*mtk_crtc->ddp_comp),
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp.c
index 13035c906035..014c1bbe1df2 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp.c
@@ -13,26 +13,6 @@
 #include "mtk_drm_ddp.h"
 #include "mtk_drm_ddp_comp.h"
 
-#define DISP_REG_CONFIG_DISP_OVL0_MOUT_EN	0x040
-#define DISP_REG_CONFIG_DISP_OVL1_MOUT_EN	0x044
-#define DISP_REG_CONFIG_DISP_OD_MOUT_EN		0x048
-#define DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN	0x04c
-#define DISP_REG_CONFIG_DISP_UFOE_MOUT_EN	0x050
-#define DISP_REG_CONFIG_DISP_COLOR0_SEL_IN	0x084
-#define DISP_REG_CONFIG_DISP_COLOR1_SEL_IN	0x088
-#define DISP_REG_CONFIG_DSIE_SEL_IN		0x0a4
-#define DISP_REG_CONFIG_DSIO_SEL_IN		0x0a8
-#define DISP_REG_CONFIG_DPI_SEL_IN		0x0ac
-#define DISP_REG_CONFIG_DISP_RDMA2_SOUT		0x0b8
-#define DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN	0x0c4
-#define DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN	0x0c8
-#define DISP_REG_CONFIG_MMSYS_CG_CON0		0x100
-
-#define DISP_REG_CONFIG_DISP_OVL_MOUT_EN	0x030
-#define DISP_REG_CONFIG_OUT_SEL			0x04c
-#define DISP_REG_CONFIG_DSI_SEL			0x050
-#define DISP_REG_CONFIG_DPI_SEL			0x064
-
 #define MT2701_DISP_MUTEX0_MOD0			0x2c
 #define MT2701_DISP_MUTEX0_SOF0			0x30
 
@@ -94,48 +74,6 @@
 #define MUTEX_SOF_DSI2			5
 #define MUTEX_SOF_DSI3			6
 
-#define OVL0_MOUT_EN_COLOR0		0x1
-#define OD_MOUT_EN_RDMA0		0x1
-#define OD1_MOUT_EN_RDMA1		BIT(16)
-#define UFOE_MOUT_EN_DSI0		0x1
-#define COLOR0_SEL_IN_OVL0		0x1
-#define OVL1_MOUT_EN_COLOR1		0x1
-#define GAMMA_MOUT_EN_RDMA1		0x1
-#define RDMA0_SOUT_DPI0			0x2
-#define RDMA0_SOUT_DPI1			0x3
-#define RDMA0_SOUT_DSI1			0x1
-#define RDMA0_SOUT_DSI2			0x4
-#define RDMA0_SOUT_DSI3			0x5
-#define RDMA1_SOUT_DPI0			0x2
-#define RDMA1_SOUT_DPI1			0x3
-#define RDMA1_SOUT_DSI1			0x1
-#define RDMA1_SOUT_DSI2			0x4
-#define RDMA1_SOUT_DSI3			0x5
-#define RDMA2_SOUT_DPI0			0x2
-#define RDMA2_SOUT_DPI1			0x3
-#define RDMA2_SOUT_DSI1			0x1
-#define RDMA2_SOUT_DSI2			0x4
-#define RDMA2_SOUT_DSI3			0x5
-#define DPI0_SEL_IN_RDMA1		0x1
-#define DPI0_SEL_IN_RDMA2		0x3
-#define DPI1_SEL_IN_RDMA1		(0x1 << 8)
-#define DPI1_SEL_IN_RDMA2		(0x3 << 8)
-#define DSI0_SEL_IN_RDMA1		0x1
-#define DSI0_SEL_IN_RDMA2		0x4
-#define DSI1_SEL_IN_RDMA1		0x1
-#define DSI1_SEL_IN_RDMA2		0x4
-#define DSI2_SEL_IN_RDMA1		(0x1 << 16)
-#define DSI2_SEL_IN_RDMA2		(0x4 << 16)
-#define DSI3_SEL_IN_RDMA1		(0x1 << 16)
-#define DSI3_SEL_IN_RDMA2		(0x4 << 16)
-#define COLOR1_SEL_IN_OVL1		0x1
-
-#define OVL_MOUT_EN_RDMA		0x1
-#define BLS_TO_DSI_RDMA1_TO_DPI1	0x8
-#define BLS_TO_DPI_RDMA1_TO_DSI		0x2
-#define DSI_SEL_IN_BLS			0x0
-#define DPI_SEL_IN_BLS			0x0
-#define DSI_SEL_IN_RDMA			0x1
 
 struct mtk_disp_mutex {
 	int id;
@@ -246,200 +184,6 @@ static const struct mtk_ddp_data mt8173_ddp_driver_data = {
 	.mutex_sof_reg = MT2701_DISP_MUTEX0_SOF0,
 };
 
-static unsigned int mtk_ddp_mout_en(enum mtk_ddp_comp_id cur,
-				    enum mtk_ddp_comp_id next,
-				    unsigned int *addr)
-{
-	unsigned int value;
-
-	if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_COLOR0) {
-		*addr = DISP_REG_CONFIG_DISP_OVL0_MOUT_EN;
-		value = OVL0_MOUT_EN_COLOR0;
-	} else if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_RDMA0) {
-		*addr = DISP_REG_CONFIG_DISP_OVL_MOUT_EN;
-		value = OVL_MOUT_EN_RDMA;
-	} else if (cur == DDP_COMPONENT_OD0 && next == DDP_COMPONENT_RDMA0) {
-		*addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN;
-		value = OD_MOUT_EN_RDMA0;
-	} else if (cur == DDP_COMPONENT_UFOE && next == DDP_COMPONENT_DSI0) {
-		*addr = DISP_REG_CONFIG_DISP_UFOE_MOUT_EN;
-		value = UFOE_MOUT_EN_DSI0;
-	} else if (cur == DDP_COMPONENT_OVL1 && next == DDP_COMPONENT_COLOR1) {
-		*addr = DISP_REG_CONFIG_DISP_OVL1_MOUT_EN;
-		value = OVL1_MOUT_EN_COLOR1;
-	} else if (cur == DDP_COMPONENT_GAMMA && next == DDP_COMPONENT_RDMA1) {
-		*addr = DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN;
-		value = GAMMA_MOUT_EN_RDMA1;
-	} else if (cur == DDP_COMPONENT_OD1 && next == DDP_COMPONENT_RDMA1) {
-		*addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN;
-		value = OD1_MOUT_EN_RDMA1;
-	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DPI0) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
-		value = RDMA0_SOUT_DPI0;
-	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DPI1) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
-		value = RDMA0_SOUT_DPI1;
-	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI1) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
-		value = RDMA0_SOUT_DSI1;
-	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI2) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
-		value = RDMA0_SOUT_DSI2;
-	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI3) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
-		value = RDMA0_SOUT_DSI3;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
-		value = RDMA1_SOUT_DSI1;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
-		value = RDMA1_SOUT_DSI2;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
-		value = RDMA1_SOUT_DSI3;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
-		value = RDMA1_SOUT_DPI0;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
-		value = RDMA1_SOUT_DPI1;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
-		value = RDMA2_SOUT_DPI0;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
-		value = RDMA2_SOUT_DPI1;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
-		value = RDMA2_SOUT_DSI1;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
-		value = RDMA2_SOUT_DSI2;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) {
-		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
-		value = RDMA2_SOUT_DSI3;
-	} else {
-		value = 0;
-	}
-
-	return value;
-}
-
-static unsigned int mtk_ddp_sel_in(enum mtk_ddp_comp_id cur,
-				   enum mtk_ddp_comp_id next,
-				   unsigned int *addr)
-{
-	unsigned int value;
-
-	if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_COLOR0) {
-		*addr = DISP_REG_CONFIG_DISP_COLOR0_SEL_IN;
-		value = COLOR0_SEL_IN_OVL0;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) {
-		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
-		value = DPI0_SEL_IN_RDMA1;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) {
-		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
-		value = DPI1_SEL_IN_RDMA1;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI0) {
-		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
-		value = DSI0_SEL_IN_RDMA1;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) {
-		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
-		value = DSI1_SEL_IN_RDMA1;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) {
-		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
-		value = DSI2_SEL_IN_RDMA1;
-	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) {
-		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
-		value = DSI3_SEL_IN_RDMA1;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) {
-		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
-		value = DPI0_SEL_IN_RDMA2;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) {
-		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
-		value = DPI1_SEL_IN_RDMA2;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI0) {
-		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
-		value = DSI0_SEL_IN_RDMA2;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) {
-		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
-		value = DSI1_SEL_IN_RDMA2;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) {
-		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
-		value = DSI2_SEL_IN_RDMA2;
-	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) {
-		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
-		value = DSI3_SEL_IN_RDMA2;
-	} else if (cur == DDP_COMPONENT_OVL1 && next == DDP_COMPONENT_COLOR1) {
-		*addr = DISP_REG_CONFIG_DISP_COLOR1_SEL_IN;
-		value = COLOR1_SEL_IN_OVL1;
-	} else if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DSI0) {
-		*addr = DISP_REG_CONFIG_DSI_SEL;
-		value = DSI_SEL_IN_BLS;
-	} else {
-		value = 0;
-	}
-
-	return value;
-}
-
-static void mtk_ddp_sout_sel(void __iomem *config_regs,
-			     enum mtk_ddp_comp_id cur,
-			     enum mtk_ddp_comp_id next)
-{
-	if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DSI0) {
-		writel_relaxed(BLS_TO_DSI_RDMA1_TO_DPI1,
-			       config_regs + DISP_REG_CONFIG_OUT_SEL);
-	} else if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DPI0) {
-		writel_relaxed(BLS_TO_DPI_RDMA1_TO_DSI,
-			       config_regs + DISP_REG_CONFIG_OUT_SEL);
-		writel_relaxed(DSI_SEL_IN_RDMA,
-			       config_regs + DISP_REG_CONFIG_DSI_SEL);
-		writel_relaxed(DPI_SEL_IN_BLS,
-			       config_regs + DISP_REG_CONFIG_DPI_SEL);
-	}
-}
-
-void mtk_ddp_add_comp_to_path(void __iomem *config_regs,
-			      enum mtk_ddp_comp_id cur,
-			      enum mtk_ddp_comp_id next)
-{
-	unsigned int addr, value, reg;
-
-	value = mtk_ddp_mout_en(cur, next, &addr);
-	if (value) {
-		reg = readl_relaxed(config_regs + addr) | value;
-		writel_relaxed(reg, config_regs + addr);
-	}
-
-	mtk_ddp_sout_sel(config_regs, cur, next);
-
-	value = mtk_ddp_sel_in(cur, next, &addr);
-	if (value) {
-		reg = readl_relaxed(config_regs + addr) | value;
-		writel_relaxed(reg, config_regs + addr);
-	}
-}
-
-void mtk_ddp_remove_comp_from_path(void __iomem *config_regs,
-				   enum mtk_ddp_comp_id cur,
-				   enum mtk_ddp_comp_id next)
-{
-	unsigned int addr, value, reg;
-
-	value = mtk_ddp_mout_en(cur, next, &addr);
-	if (value) {
-		reg = readl_relaxed(config_regs + addr) & ~value;
-		writel_relaxed(reg, config_regs + addr);
-	}
-
-	value = mtk_ddp_sel_in(cur, next, &addr);
-	if (value) {
-		reg = readl_relaxed(config_regs + addr) & ~value;
-		writel_relaxed(reg, config_regs + addr);
-	}
-}
-
 struct mtk_disp_mutex *mtk_disp_mutex_get(struct device *dev, unsigned int id)
 {
 	struct mtk_ddp *ddp = dev_get_drvdata(dev);
@@ -628,7 +372,8 @@ static int mtk_ddp_probe(struct platform_device *pdev)
 	if (!ddp->data->no_clk) {
 		ddp->clk = devm_clk_get(dev, NULL);
 		if (IS_ERR(ddp->clk)) {
-			dev_err(dev, "Failed to get clock\n");
+			if (PTR_ERR(ddp->clk) != -EPROBE_DEFER)
+				dev_err(dev, "Failed to get clock\n");
 			return PTR_ERR(ddp->clk);
 		}
 	}
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp.h b/drivers/gpu/drm/mediatek/mtk_drm_ddp.h
index 827be424a148..6b691a57be4a 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp.h
@@ -12,13 +12,6 @@ struct regmap;
 struct device;
 struct mtk_disp_mutex;
 
-void mtk_ddp_add_comp_to_path(void __iomem *config_regs,
-			      enum mtk_ddp_comp_id cur,
-			      enum mtk_ddp_comp_id next);
-void mtk_ddp_remove_comp_from_path(void __iomem *config_regs,
-				   enum mtk_ddp_comp_id cur,
-				   enum mtk_ddp_comp_id next);
-
 struct mtk_disp_mutex *mtk_disp_mutex_get(struct device *dev, unsigned int id);
 int mtk_disp_mutex_prepare(struct mtk_disp_mutex *mutex);
 void mtk_disp_mutex_add_comp(struct mtk_disp_mutex *mutex,
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index ce570283b55f..6bd369434d9d 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -10,6 +10,7 @@
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/pm_runtime.h>
+#include <linux/soc/mediatek/mtk-mmsys.h>
 #include <linux/dma-mapping.h>
 
 #include <drm/drm_atomic.h>
@@ -418,11 +419,22 @@ static const struct of_device_id mtk_ddp_comp_dt_ids[] = {
 	{ }
 };
 
+static const struct of_device_id mtk_drm_of_ids[] = {
+	{ .compatible = "mediatek,mt2701-mmsys",
+	  .data = &mt2701_mmsys_driver_data},
+	{ .compatible = "mediatek,mt2712-mmsys",
+	  .data = &mt2712_mmsys_driver_data},
+	{ .compatible = "mediatek,mt8173-mmsys",
+	  .data = &mt8173_mmsys_driver_data},
+	{ }
+};
+
 static int mtk_drm_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct device_node *phandle = dev->parent->of_node;
+	const struct of_device_id *of_id;
 	struct mtk_drm_private *private;
-	struct resource *mem;
 	struct device_node *node;
 	struct component_match *match = NULL;
 	int ret;
@@ -433,18 +445,20 @@ static int mtk_drm_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	private->data = of_device_get_match_data(dev);
-
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	private->config_regs = devm_ioremap_resource(dev, mem);
-	if (IS_ERR(private->config_regs)) {
-		ret = PTR_ERR(private->config_regs);
-		dev_err(dev, "Failed to ioremap mmsys-config resource: %d\n",
-			ret);
-		return ret;
+	private->mmsys_dev = dev->parent;
+	if (!private->mmsys_dev) {
+		dev_err(dev, "Failed to get MMSYS device\n");
+		return -ENODEV;
 	}
 
+	of_id = of_match_node(mtk_drm_of_ids, phandle);
+	if (!of_id)
+		return -ENODEV;
+
+	private->data = of_id->data;
+
 	/* Iterate over sibling DISP function blocks */
-	for_each_child_of_node(dev->of_node->parent, node) {
+	for_each_child_of_node(phandle->parent, node) {
 		const struct of_device_id *of_id;
 		enum mtk_ddp_comp_type comp_type;
 		int comp_id;
@@ -578,22 +592,11 @@ static int mtk_drm_sys_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(mtk_drm_pm_ops, mtk_drm_sys_suspend,
 			 mtk_drm_sys_resume);
 
-static const struct of_device_id mtk_drm_of_ids[] = {
-	{ .compatible = "mediatek,mt2701-mmsys",
-	  .data = &mt2701_mmsys_driver_data},
-	{ .compatible = "mediatek,mt2712-mmsys",
-	  .data = &mt2712_mmsys_driver_data},
-	{ .compatible = "mediatek,mt8173-mmsys",
-	  .data = &mt8173_mmsys_driver_data},
-	{ }
-};
-
 static struct platform_driver mtk_drm_platform_driver = {
 	.probe	= mtk_drm_probe,
 	.remove	= mtk_drm_remove,
 	.driver	= {
 		.name	= "mediatek-drm",
-		.of_match_table = mtk_drm_of_ids,
 		.pm     = &mtk_drm_pm_ops,
 	},
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
index 17bc99b9f5d4..b5be63e53176 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
@@ -39,7 +39,7 @@ struct mtk_drm_private {
 
 	struct device_node *mutex_node;
 	struct device *mutex_dev;
-	void __iomem *config_regs;
+	struct device *mmsys_dev;
 	struct device_node *comp_node[DDP_COMPONENT_ID_MAX];
 	struct mtk_ddp_comp *ddp_comp[DDP_COMPONENT_ID_MAX];
 	const struct mtk_mmsys_driver_data *data;
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index a9a25087112f..270bf22c98fe 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -1186,14 +1186,18 @@ static int mtk_dsi_probe(struct platform_device *pdev)
 	dsi->engine_clk = devm_clk_get(dev, "engine");
 	if (IS_ERR(dsi->engine_clk)) {
 		ret = PTR_ERR(dsi->engine_clk);
-		dev_err(dev, "Failed to get engine clock: %d\n", ret);
+
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get engine clock: %d\n", ret);
 		goto err_unregister_host;
 	}
 
 	dsi->digital_clk = devm_clk_get(dev, "digital");
 	if (IS_ERR(dsi->digital_clk)) {
 		ret = PTR_ERR(dsi->digital_clk);
-		dev_err(dev, "Failed to get digital clock: %d\n", ret);
+
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get digital clock: %d\n", ret);
 		goto err_unregister_host;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index 7bc086ec74f7..5feb760617cb 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -1470,7 +1470,9 @@ static int mtk_hdmi_dt_parse_pdata(struct mtk_hdmi *hdmi,
 
 	ret = mtk_hdmi_get_all_clk(hdmi, np);
 	if (ret) {
-		dev_err(dev, "Failed to get clocks: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get clocks: %d\n", ret);
+
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 1579cf0d828f..42f8aae28b31 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -65,6 +65,7 @@ msm-y := \
 	disp/dpu1/dpu_hw_lm.o \
 	disp/dpu1/dpu_hw_pingpong.o \
 	disp/dpu1/dpu_hw_sspp.o \
+	disp/dpu1/dpu_hw_dspp.o \
 	disp/dpu1/dpu_hw_top.o \
 	disp/dpu1/dpu_hw_util.o \
 	disp/dpu1/dpu_hw_vbif.o \
diff --git a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c
index 1f83bc18d500..60f6472a3e58 100644
--- a/drivers/gpu/drm/msm/adreno/a2xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a2xx_gpu.c
@@ -401,6 +401,21 @@ static struct msm_gpu_state *a2xx_gpu_state_get(struct msm_gpu *gpu)
 	return state;
 }
 
+static struct msm_gem_address_space *
+a2xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
+{
+	struct msm_mmu *mmu = msm_gpummu_new(&pdev->dev, gpu);
+	struct msm_gem_address_space *aspace;
+
+	aspace = msm_gem_address_space_create(mmu, "gpu", SZ_16M,
+		SZ_16M + 0xfff * SZ_64K);
+
+	if (IS_ERR(aspace) && !IS_ERR(mmu))
+		mmu->funcs->destroy(mmu);
+
+	return aspace;
+}
+
 /* Register offset defines for A2XX - copy of A3XX */
 static const unsigned int a2xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
 	REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_AXXX_CP_RB_BASE),
@@ -429,6 +444,7 @@ static const struct adreno_gpu_funcs funcs = {
 #endif
 		.gpu_state_get = a2xx_gpu_state_get,
 		.gpu_state_put = adreno_gpu_state_put,
+		.create_address_space = a2xx_create_address_space,
 	},
 };
 
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index b67f88872726..0a5ea9f56cb8 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -441,6 +441,7 @@ static const struct adreno_gpu_funcs funcs = {
 #endif
 		.gpu_state_get = a3xx_gpu_state_get,
 		.gpu_state_put = adreno_gpu_state_put,
+		.create_address_space = adreno_iommu_create_address_space,
 	},
 };
 
diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
index 253d8d85daad..b9b26b2bf9c5 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
@@ -66,19 +66,22 @@ static void a4xx_enable_hwcg(struct msm_gpu *gpu)
 		}
 	}
 
-	for (i = 0; i < 4; i++) {
-		gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL_MARB_CCU(i),
-				0x00000922);
-	}
+	/* No CCU for A405 */
+	if (!adreno_is_a405(adreno_gpu)) {
+		for (i = 0; i < 4; i++) {
+			gpu_write(gpu, REG_A4XX_RBBM_CLOCK_CTL_MARB_CCU(i),
+					0x00000922);
+		}
 
-	for (i = 0; i < 4; i++) {
-		gpu_write(gpu, REG_A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU(i),
-				0x00000000);
-	}
+		for (i = 0; i < 4; i++) {
+			gpu_write(gpu, REG_A4XX_RBBM_CLOCK_HYST_RB_MARB_CCU(i),
+					0x00000000);
+		}
 
-	for (i = 0; i < 4; i++) {
-		gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1(i),
-				0x00000001);
+		for (i = 0; i < 4; i++) {
+			gpu_write(gpu, REG_A4XX_RBBM_CLOCK_DELAY_RB_MARB_CCU_L1(i),
+					0x00000001);
+		}
 	}
 
 	gpu_write(gpu, REG_A4XX_RBBM_CLOCK_MODE_GPC, 0x02222222);
@@ -137,7 +140,9 @@ static int a4xx_hw_init(struct msm_gpu *gpu)
 	uint32_t *ptr, len;
 	int i, ret;
 
-	if (adreno_is_a420(adreno_gpu)) {
+	if (adreno_is_a405(adreno_gpu)) {
+		gpu_write(gpu, REG_A4XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
+	} else if (adreno_is_a420(adreno_gpu)) {
 		gpu_write(gpu, REG_A4XX_VBIF_ABIT_SORT, 0x0001001F);
 		gpu_write(gpu, REG_A4XX_VBIF_ABIT_SORT_CONF, 0x000000A4);
 		gpu_write(gpu, REG_A4XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000001);
@@ -440,6 +445,52 @@ static const unsigned int a4xx_registers[] = {
 	~0 /* sentinel */
 };
 
+static const unsigned int a405_registers[] = {
+	/* RBBM */
+	0x0000, 0x0002, 0x0004, 0x0021, 0x0023, 0x0024, 0x0026, 0x0026,
+	0x0028, 0x002B, 0x002E, 0x0034, 0x0037, 0x0044, 0x0047, 0x0066,
+	0x0068, 0x0095, 0x009C, 0x0170, 0x0174, 0x01AF,
+	/* CP */
+	0x0200, 0x0233, 0x0240, 0x0250, 0x04C0, 0x04DD, 0x0500, 0x050B,
+	0x0578, 0x058F,
+	/* VSC */
+	0x0C00, 0x0C03, 0x0C08, 0x0C41, 0x0C50, 0x0C51,
+	/* GRAS */
+	0x0C80, 0x0C81, 0x0C88, 0x0C8F,
+	/* RB */
+	0x0CC0, 0x0CC0, 0x0CC4, 0x0CD2,
+	/* PC */
+	0x0D00, 0x0D0C, 0x0D10, 0x0D17, 0x0D20, 0x0D23,
+	/* VFD */
+	0x0E40, 0x0E4A,
+	/* VPC */
+	0x0E60, 0x0E61, 0x0E63, 0x0E68,
+	/* UCHE */
+	0x0E80, 0x0E84, 0x0E88, 0x0E95,
+	/* GRAS CTX 0 */
+	0x2000, 0x2004, 0x2008, 0x2067, 0x2070, 0x2078, 0x207B, 0x216E,
+	/* PC CTX 0 */
+	0x21C0, 0x21C6, 0x21D0, 0x21D0, 0x21D9, 0x21D9, 0x21E5, 0x21E7,
+	/* VFD CTX 0 */
+	0x2200, 0x2204, 0x2208, 0x22A9,
+	/* GRAS CTX 1 */
+	0x2400, 0x2404, 0x2408, 0x2467, 0x2470, 0x2478, 0x247B, 0x256E,
+	/* PC CTX 1 */
+	0x25C0, 0x25C6, 0x25D0, 0x25D0, 0x25D9, 0x25D9, 0x25E5, 0x25E7,
+	/* VFD CTX 1 */
+	0x2600, 0x2604, 0x2608, 0x26A9,
+	/* VBIF version 0x20050000*/
+	0x3000, 0x3007, 0x302C, 0x302C, 0x3030, 0x3030, 0x3034, 0x3036,
+	0x3038, 0x3038, 0x303C, 0x303D, 0x3040, 0x3040, 0x3049, 0x3049,
+	0x3058, 0x3058, 0x305B, 0x3061, 0x3064, 0x3068, 0x306C, 0x306D,
+	0x3080, 0x3088, 0x308B, 0x308C, 0x3090, 0x3094, 0x3098, 0x3098,
+	0x309C, 0x309C, 0x30C0, 0x30C0, 0x30C8, 0x30C8, 0x30D0, 0x30D0,
+	0x30D8, 0x30D8, 0x30E0, 0x30E0, 0x3100, 0x3100, 0x3108, 0x3108,
+	0x3110, 0x3110, 0x3118, 0x3118, 0x3120, 0x3120, 0x3124, 0x3125,
+	0x3129, 0x3129, 0x340C, 0x340C, 0x3410, 0x3410,
+	~0 /* sentinel */
+};
+
 static struct msm_gpu_state *a4xx_gpu_state_get(struct msm_gpu *gpu)
 {
 	struct msm_gpu_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
@@ -532,6 +583,7 @@ static const struct adreno_gpu_funcs funcs = {
 #endif
 		.gpu_state_get = a4xx_gpu_state_get,
 		.gpu_state_put = adreno_gpu_state_put,
+		.create_address_space = adreno_iommu_create_address_space,
 	},
 	.get_timestamp = a4xx_get_timestamp,
 };
@@ -563,13 +615,14 @@ struct msm_gpu *a4xx_gpu_init(struct drm_device *dev)
 	gpu->perfcntrs = NULL;
 	gpu->num_perfcntrs = 0;
 
-	adreno_gpu->registers = a4xx_registers;
-	adreno_gpu->reg_offsets = a4xx_register_offsets;
-
 	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
 	if (ret)
 		goto fail;
 
+	adreno_gpu->registers = adreno_is_a405(adreno_gpu) ? a405_registers :
+							     a4xx_registers;
+	adreno_gpu->reg_offsets = a4xx_register_offsets;
+
 	/* if needed, allocate gmem: */
 	if (adreno_is_a4xx(adreno_gpu)) {
 		ret = adreno_gpu_ocmem_init(dev->dev, adreno_gpu,
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index 724024a2243a..d95970a73fb4 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -1404,6 +1404,10 @@ static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu)
 {
 	u64 busy_cycles, busy_time;
 
+	/* Only read the gpu busy if the hardware is already active */
+	if (pm_runtime_get_if_in_use(&gpu->pdev->dev) == 0)
+		return 0;
+
 	busy_cycles = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO,
 			REG_A5XX_RBBM_PERFCTR_RBBM_0_HI);
 
@@ -1412,6 +1416,8 @@ static unsigned long a5xx_gpu_busy(struct msm_gpu *gpu)
 
 	gpu->devfreq.busy_cycles = busy_cycles;
 
+	pm_runtime_put(&gpu->pdev->dev);
+
 	if (WARN_ON(busy_time > ~0LU))
 		return ~0LU;
 
@@ -1439,6 +1445,7 @@ static const struct adreno_gpu_funcs funcs = {
 		.gpu_busy = a5xx_gpu_busy,
 		.gpu_state_get = a5xx_gpu_state_get,
 		.gpu_state_put = a5xx_gpu_state_put,
+		.create_address_space = adreno_iommu_create_address_space,
 	},
 	.get_timestamp = a5xx_get_timestamp,
 };
diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
index ed78fee2a262..47840b73cdda 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
@@ -1047,6 +1047,8 @@ enum a6xx_tex_type {
 
 #define REG_A6XX_CP_MISC_CNTL					0x00000840
 
+#define REG_A6XX_CP_APRIV_CNTL					0x00000844
+
 #define REG_A6XX_CP_ROQ_THRESHOLDS_1				0x000008c1
 
 #define REG_A6XX_CP_ROQ_THRESHOLDS_2				0x000008c2
@@ -1764,6 +1766,8 @@ static inline uint32_t A6XX_CP_PROTECT_REG_MASK_LEN(uint32_t val)
 
 #define REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL			0x00000010
 
+#define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL			0x00000011
+
 #define REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL			0x0000001f
 
 #define REG_A6XX_RBBM_INT_CLEAR_CMD				0x00000037
@@ -2418,6 +2422,16 @@ static inline uint32_t A6XX_UCHE_CLIENT_PF_PERFSEL(uint32_t val)
 
 #define REG_A6XX_TPL1_NC_MODE_CNTL				0x0000b604
 
+#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0			0x0000b608
+
+#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1			0x0000b609
+
+#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2			0x0000b60a
+
+#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3			0x0000b60b
+
+#define REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4			0x0000b60c
+
 #define REG_A6XX_TPL1_PERFCTR_TP_SEL_0				0x0000b610
 
 #define REG_A6XX_TPL1_PERFCTR_TP_SEL_1				0x0000b611
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index c4e71abbdd53..096be97ce9f9 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -2,14 +2,16 @@
 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
 
 #include <linux/clk.h>
-#include <linux/dma-mapping.h>
 #include <linux/interconnect.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_opp.h>
 #include <soc/qcom/cmd-db.h>
+#include <drm/drm_gem.h>
 
 #include "a6xx_gpu.h"
 #include "a6xx_gmu.xml.h"
+#include "msm_gem.h"
+#include "msm_mmu.h"
 
 static void a6xx_gmu_fault(struct a6xx_gmu *gmu)
 {
@@ -127,8 +129,6 @@ static void __a6xx_gmu_set_freq(struct a6xx_gmu *gmu, int index)
 	if (ret)
 		dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
 
-	gmu->freq = gmu->gpu_freqs[index];
-
 	/*
 	 * Eventually we will want to scale the path vote with the frequency but
 	 * for now leave it at max so that the performance is nominal.
@@ -151,8 +151,21 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, unsigned long freq)
 			break;
 
 	gmu->current_perf_index = perf_index;
+	gmu->freq = gmu->gpu_freqs[perf_index];
+
+	/*
+	 * This can get called from devfreq while the hardware is idle. Don't
+	 * bring up the power if it isn't already active
+	 */
+	if (pm_runtime_get_if_in_use(gmu->dev) == 0)
+		return;
 
-	__a6xx_gmu_set_freq(gmu, perf_index);
+	if (gmu->legacy)
+		__a6xx_gmu_set_freq(gmu, perf_index);
+	else
+		a6xx_hfi_set_freq(gmu, perf_index);
+
+	pm_runtime_put(gmu->dev);
 }
 
 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
@@ -196,6 +209,12 @@ static int a6xx_gmu_start(struct a6xx_gmu *gmu)
 	u32 val;
 
 	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
+
+	/* Set the log wptr index
+	 * note: downstream saves the value in poweroff and restores it here
+	 */
+	gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0);
+
 	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0);
 
 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val,
@@ -232,8 +251,13 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
 
 	switch (state) {
 	case GMU_OOB_GPU_SET:
-		request = GMU_OOB_GPU_SET_REQUEST;
-		ack = GMU_OOB_GPU_SET_ACK;
+		if (gmu->legacy) {
+			request = GMU_OOB_GPU_SET_REQUEST;
+			ack = GMU_OOB_GPU_SET_ACK;
+		} else {
+			request = GMU_OOB_GPU_SET_REQUEST_NEW;
+			ack = GMU_OOB_GPU_SET_ACK_NEW;
+		}
 		name = "GPU_SET";
 		break;
 	case GMU_OOB_BOOT_SLUMBER:
@@ -272,6 +296,13 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
 /* Clear a pending OOB state in the GMU */
 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
 {
+	if (!gmu->legacy) {
+		WARN_ON(state != GMU_OOB_GPU_SET);
+		gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
+			1 << GMU_OOB_GPU_SET_CLEAR_NEW);
+		return;
+	}
+
 	switch (state) {
 	case GMU_OOB_GPU_SET:
 		gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
@@ -294,6 +325,9 @@ static int a6xx_sptprac_enable(struct a6xx_gmu *gmu)
 	int ret;
 	u32 val;
 
+	if (!gmu->legacy)
+		return 0;
+
 	gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000);
 
 	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
@@ -313,6 +347,9 @@ static void a6xx_sptprac_disable(struct a6xx_gmu *gmu)
 	u32 val;
 	int ret;
 
+	if (!gmu->legacy)
+		return;
+
 	/* Make sure retention is on */
 	gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11));
 
@@ -356,6 +393,11 @@ static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu)
 	if (gmu->idle_level < GMU_IDLE_STATE_SPTP)
 		a6xx_sptprac_disable(gmu);
 
+	if (!gmu->legacy) {
+		ret = a6xx_hfi_send_prep_slumber(gmu);
+		goto out;
+	}
+
 	/* Tell the GMU to get ready to slumber */
 	gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1);
 
@@ -371,6 +413,7 @@ static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu)
 		}
 	}
 
+out:
 	/* Put fence into allow mode */
 	gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
 	return ret;
@@ -392,7 +435,7 @@ static int a6xx_rpmh_start(struct a6xx_gmu *gmu)
 		return ret;
 	}
 
-	ret = gmu_poll_timeout(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
+	ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
 		!val, 100, 10000);
 
 	if (ret) {
@@ -418,7 +461,7 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
 
 	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1);
 
-	ret = gmu_poll_timeout(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
+	ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
 		val, val & (1 << 16), 100, 10000);
 	if (ret)
 		DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n");
@@ -441,32 +484,48 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
 	struct platform_device *pdev = to_platform_device(gmu->dev);
 	void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
 	void __iomem *seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq");
+	uint32_t pdc_address_offset;
 
 	if (!pdcptr || !seqptr)
 		goto err;
 
+	if (adreno_is_a618(adreno_gpu) || adreno_is_a640(adreno_gpu))
+		pdc_address_offset = 0x30090;
+	else if (adreno_is_a650(adreno_gpu))
+		pdc_address_offset = 0x300a0;
+	else
+		pdc_address_offset = 0x30080;
+
 	/* Disable SDE clock gating */
-	gmu_write(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
+	gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
 
 	/* Setup RSC PDC handshake for sleep and wakeup */
-	gmu_write(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
-	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
-	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
-	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
-	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
-	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000);
-	gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
-	gmu_write(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
-	gmu_write(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
-	gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
-	gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
+	gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
 
 	/* Load RSC sequencer uCode for sleep and wakeup */
-	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
-	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
-	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
-	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
-	gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
+	if (adreno_is_a650(adreno_gpu)) {
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xeaaae5a0);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xe1a1ebab);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e0a581);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xecac82e2);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020edad);
+	} else {
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
+		gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
+	}
 
 	/* Load PDC sequencer uCode for power up and power down sequence */
 	pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
@@ -487,10 +546,7 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
 
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
-	if (adreno_is_a618(adreno_gpu))
-		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30090);
-	else
-		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30080);
+	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, pdc_address_offset);
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
 
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
@@ -502,17 +558,12 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
 
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
-	if (adreno_is_a618(adreno_gpu))
+	if (adreno_is_a618(adreno_gpu) || adreno_is_a650(adreno_gpu))
 		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x2);
 	else
 		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
-
-
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
-	if (adreno_is_a618(adreno_gpu))
-		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30090);
-	else
-		pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30080);
+	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, pdc_address_offset);
 	pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
 
 	/* Setup GPU PDC */
@@ -542,6 +593,8 @@ static void a6xx_gmu_power_config(struct a6xx_gmu *gmu)
 {
 	/* Disable GMU WB/RB buffer */
 	gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1);
+	gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1);
+	gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1);
 
 	gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400);
 
@@ -571,14 +624,95 @@ static void a6xx_gmu_power_config(struct a6xx_gmu *gmu)
 		A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE);
 }
 
+struct block_header {
+	u32 addr;
+	u32 size;
+	u32 type;
+	u32 value;
+	u32 data[];
+};
+
+/* this should be a general kernel helper */
+static int in_range(u32 addr, u32 start, u32 size)
+{
+	return addr >= start && addr < start + size;
+}
+
+static bool fw_block_mem(struct a6xx_gmu_bo *bo, const struct block_header *blk)
+{
+	if (!in_range(blk->addr, bo->iova, bo->size))
+		return false;
+
+	memcpy(bo->virt + blk->addr - bo->iova, blk->data, blk->size);
+	return true;
+}
+
+static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu)
+{
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	const struct firmware *fw_image = adreno_gpu->fw[ADRENO_FW_GMU];
+	const struct block_header *blk;
+	u32 reg_offset;
+
+	u32 itcm_base = 0x00000000;
+	u32 dtcm_base = 0x00040000;
+
+	if (adreno_is_a650(adreno_gpu))
+		dtcm_base = 0x10004000;
+
+	if (gmu->legacy) {
+		/* Sanity check the size of the firmware that was loaded */
+		if (fw_image->size > 0x8000) {
+			DRM_DEV_ERROR(gmu->dev,
+				"GMU firmware is bigger than the available region\n");
+			return -EINVAL;
+		}
+
+		gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START,
+			       (u32*) fw_image->data, fw_image->size);
+		return 0;
+	}
+
+
+	for (blk = (const struct block_header *) fw_image->data;
+	     (const u8*) blk < fw_image->data + fw_image->size;
+	     blk = (const struct block_header *) &blk->data[blk->size >> 2]) {
+		if (blk->size == 0)
+			continue;
+
+		if (in_range(blk->addr, itcm_base, SZ_16K)) {
+			reg_offset = (blk->addr - itcm_base) >> 2;
+			gmu_write_bulk(gmu,
+				REG_A6XX_GMU_CM3_ITCM_START + reg_offset,
+				blk->data, blk->size);
+		} else if (in_range(blk->addr, dtcm_base, SZ_16K)) {
+			reg_offset = (blk->addr - dtcm_base) >> 2;
+			gmu_write_bulk(gmu,
+				REG_A6XX_GMU_CM3_DTCM_START + reg_offset,
+				blk->data, blk->size);
+		} else if (!fw_block_mem(&gmu->icache, blk) &&
+			   !fw_block_mem(&gmu->dcache, blk) &&
+			   !fw_block_mem(&gmu->dummy, blk)) {
+			DRM_DEV_ERROR(gmu->dev,
+				"failed to match fw block (addr=%.8x size=%d data[0]=%.8x)\n",
+				blk->addr, blk->size, blk->data[0]);
+		}
+	}
+
+	return 0;
+}
+
 static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 {
 	static bool rpmh_init;
 	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
-	int i, ret;
+	int ret;
 	u32 chipid;
-	u32 *image;
+
+	if (adreno_is_a650(adreno_gpu))
+		gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1);
 
 	if (state == GMU_WARM_BOOT) {
 		ret = a6xx_rpmh_start(gmu);
@@ -589,13 +723,6 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 			"GMU firmware is not loaded\n"))
 			return -ENOENT;
 
-		/* Sanity check the size of the firmware that was loaded */
-		if (adreno_gpu->fw[ADRENO_FW_GMU]->size > 0x8000) {
-			DRM_DEV_ERROR(gmu->dev,
-				"GMU firmware is bigger than the available region\n");
-			return -EINVAL;
-		}
-
 		/* Turn on register retention */
 		gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
 
@@ -609,18 +736,16 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 				return ret;
 		}
 
-		image = (u32 *) adreno_gpu->fw[ADRENO_FW_GMU]->data;
-
-		for (i = 0; i < adreno_gpu->fw[ADRENO_FW_GMU]->size >> 2; i++)
-			gmu_write(gmu, REG_A6XX_GMU_CM3_ITCM_START + i,
-				image[i]);
+		ret = a6xx_gmu_fw_load(gmu);
+		if (ret)
+			return ret;
 	}
 
 	gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0);
 	gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02);
 
 	/* Write the iova of the HFI table */
-	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi->iova);
+	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova);
 	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1);
 
 	gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0,
@@ -633,6 +758,9 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 
 	gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid);
 
+	gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG,
+		  gmu->log.iova | (gmu->log.size / SZ_4K - 1));
+
 	/* Set up the lowest idle level on the GMU */
 	a6xx_gmu_power_config(gmu);
 
@@ -640,9 +768,11 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 	if (ret)
 		return ret;
 
-	ret = a6xx_gmu_gfx_rail_on(gmu);
-	if (ret)
-		return ret;
+	if (gmu->legacy) {
+		ret = a6xx_gmu_gfx_rail_on(gmu);
+		if (ret)
+			return ret;
+	}
 
 	/* Enable SPTP_PC if the CPU is responsible for it */
 	if (gmu->idle_level < GMU_IDLE_STATE_SPTP) {
@@ -683,13 +813,13 @@ static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu)
 	u32 val;
 
 	/* Make sure there are no outstanding RPMh votes */
-	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val,
+	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val,
 		(val & 1), 100, 10000);
-	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val,
+	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val,
 		(val & 1), 100, 10000);
-	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val,
+	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val,
 		(val & 1), 100, 10000);
-	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val,
+	gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val,
 		(val & 1), 100, 1000);
 }
 
@@ -744,6 +874,13 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
 	status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
 		GMU_WARM_BOOT : GMU_COLD_BOOT;
 
+	/*
+	 * Warm boot path does not work on newer GPUs
+	 * Presumably this is because icache/dcache regions must be restored
+	 */
+	if (!gmu->legacy)
+		status = GMU_COLD_BOOT;
+
 	ret = a6xx_gmu_fw_start(gmu, status);
 	if (ret)
 		goto out;
@@ -761,7 +898,10 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
 	enable_irq(gmu->hfi_irq);
 
 	/* Set the GPU to the current freq */
-	__a6xx_gmu_set_freq(gmu, gmu->current_perf_index);
+	if (gmu->legacy)
+		__a6xx_gmu_set_freq(gmu, gmu->current_perf_index);
+	else
+		a6xx_hfi_set_freq(gmu, gmu->current_perf_index);
 
 	/*
 	 * "enable" the GX power domain which won't actually do anything but it
@@ -919,34 +1059,75 @@ int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
 	return 0;
 }
 
-static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo)
+static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu)
 {
-	if (IS_ERR_OR_NULL(bo))
-		return;
-
-	dma_free_wc(gmu->dev, bo->size, bo->virt, bo->iova);
-	kfree(bo);
+	msm_gem_kernel_put(gmu->hfi.obj, gmu->aspace, false);
+	msm_gem_kernel_put(gmu->debug.obj, gmu->aspace, false);
+	msm_gem_kernel_put(gmu->icache.obj, gmu->aspace, false);
+	msm_gem_kernel_put(gmu->dcache.obj, gmu->aspace, false);
+	msm_gem_kernel_put(gmu->dummy.obj, gmu->aspace, false);
+	msm_gem_kernel_put(gmu->log.obj, gmu->aspace, false);
+
+	gmu->aspace->mmu->funcs->detach(gmu->aspace->mmu);
+	msm_gem_address_space_put(gmu->aspace);
 }
 
-static struct a6xx_gmu_bo *a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu,
-		size_t size)
+static int a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo,
+		size_t size, u64 iova)
 {
-	struct a6xx_gmu_bo *bo;
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
+	struct drm_device *dev = a6xx_gpu->base.base.dev;
+	uint32_t flags = MSM_BO_WC;
+	u64 range_start, range_end;
+	int ret;
 
-	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
-	if (!bo)
-		return ERR_PTR(-ENOMEM);
+	size = PAGE_ALIGN(size);
+	if (!iova) {
+		/* no fixed address - use GMU's uncached range */
+		range_start = 0x60000000 + PAGE_SIZE; /* skip dummy page */
+		range_end = 0x80000000;
+	} else {
+		/* range for fixed address */
+		range_start = iova;
+		range_end = iova + size;
+		/* use IOMMU_PRIV for icache/dcache */
+		flags |= MSM_BO_MAP_PRIV;
+	}
 
-	bo->size = PAGE_ALIGN(size);
+	bo->obj = msm_gem_new(dev, size, flags);
+	if (IS_ERR(bo->obj))
+		return PTR_ERR(bo->obj);
 
-	bo->virt = dma_alloc_wc(gmu->dev, bo->size, &bo->iova, GFP_KERNEL);
+	ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->aspace, &bo->iova,
+		range_start >> PAGE_SHIFT, range_end >> PAGE_SHIFT);
+	if (ret) {
+		drm_gem_object_put(bo->obj);
+		return ret;
+	}
+
+	bo->virt = msm_gem_get_vaddr(bo->obj);
+	bo->size = size;
+
+	return 0;
+}
+
+static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu)
+{
+	struct iommu_domain *domain;
+	struct msm_mmu *mmu;
 
-	if (!bo->virt) {
-		kfree(bo);
-		return ERR_PTR(-ENOMEM);
+	domain = iommu_domain_alloc(&platform_bus_type);
+	if (!domain)
+		return -ENODEV;
+
+	mmu = msm_iommu_new(gmu->dev, domain);
+	gmu->aspace = msm_gem_address_space_create(mmu, "gmu", 0x0, 0x7fffffff);
+	if (IS_ERR(gmu->aspace)) {
+		iommu_domain_free(domain);
+		return PTR_ERR(gmu->aspace);
 	}
 
-	return bo;
+	return 0;
 }
 
 /* Return the 'arc-level' for the given frequency */
@@ -1011,8 +1192,8 @@ static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes,
 
 		if (j == pri_count) {
 			DRM_DEV_ERROR(dev,
-				"Level %u not found in in the RPMh list\n",
-					level);
+				      "Level %u not found in the RPMh list\n",
+				      level);
 			DRM_DEV_ERROR(dev, "Available levels:\n");
 			for (j = 0; j < pri_count; j++)
 				DRM_DEV_ERROR(dev, "  %u\n", pri[j]);
@@ -1190,6 +1371,7 @@ static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev,
 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
 {
 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	struct platform_device *pdev = to_platform_device(gmu->dev);
 
 	if (!gmu->initialized)
 		return;
@@ -1202,9 +1384,12 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
 	}
 
 	iounmap(gmu->mmio);
+	if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
+		iounmap(gmu->rscc);
 	gmu->mmio = NULL;
+	gmu->rscc = NULL;
 
-	a6xx_gmu_memory_free(gmu, gmu->hfi);
+	a6xx_gmu_memory_free(gmu);
 
 	free_irq(gmu->gmu_irq, gmu);
 	free_irq(gmu->hfi_irq, gmu);
@@ -1217,6 +1402,7 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
 
 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 {
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
 	struct platform_device *pdev = of_find_device_by_node(node);
 	int ret;
@@ -1226,15 +1412,7 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 
 	gmu->dev = &pdev->dev;
 
-	/* Pass force_dma false to require the DT to set the dma region */
-	ret = of_dma_configure(gmu->dev, node, false);
-	if (ret)
-		return ret;
-
-	/* Set the mask after the of_dma_configure() */
-	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(31));
-	if (ret)
-		return ret;
+	of_dma_configure(gmu->dev, node, true);
 
 	/* Fow now, don't do anything fancy until we get our feet under us */
 	gmu->idle_level = GMU_IDLE_STATE_ACTIVE;
@@ -1246,20 +1424,64 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 	if (ret)
 		goto err_put_device;
 
+	ret = a6xx_gmu_memory_probe(gmu);
+	if (ret)
+		goto err_put_device;
+
+	/* Allocate memory for the GMU dummy page */
+	ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, SZ_4K, 0x60000000);
+	if (ret)
+		goto err_memory;
+
+	if (adreno_is_a650(adreno_gpu)) {
+		ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
+			SZ_16M - SZ_16K, 0x04000);
+		if (ret)
+			goto err_memory;
+	} else if (adreno_is_a640(adreno_gpu)) {
+		ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
+			SZ_256K - SZ_16K, 0x04000);
+		if (ret)
+			goto err_memory;
+
+		ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache,
+			SZ_256K - SZ_16K, 0x44000);
+		if (ret)
+			goto err_memory;
+	} else {
+		/* HFI v1, has sptprac */
+		gmu->legacy = true;
+
+		/* Allocate memory for the GMU debug region */
+		ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0);
+		if (ret)
+			goto err_memory;
+	}
+
 	/* Allocate memory for for the HFI queues */
-	gmu->hfi = a6xx_gmu_memory_alloc(gmu, SZ_16K);
-	if (IS_ERR(gmu->hfi))
+	ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0);
+	if (ret)
 		goto err_memory;
 
-	/* Allocate memory for the GMU debug region */
-	gmu->debug = a6xx_gmu_memory_alloc(gmu, SZ_16K);
-	if (IS_ERR(gmu->debug))
+	/* Allocate memory for the GMU log region */
+	ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_4K, 0);
+	if (ret)
 		goto err_memory;
 
 	/* Map the GMU registers */
 	gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
-	if (IS_ERR(gmu->mmio))
+	if (IS_ERR(gmu->mmio)) {
+		ret = PTR_ERR(gmu->mmio);
 		goto err_memory;
+	}
+
+	if (adreno_is_a650(adreno_gpu)) {
+		gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc");
+		if (IS_ERR(gmu->rscc))
+			goto err_mmio;
+	} else {
+		gmu->rscc = gmu->mmio + 0x23000;
+	}
 
 	/* Get the HFI and GMU interrupts */
 	gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
@@ -1286,13 +1508,15 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 
 err_mmio:
 	iounmap(gmu->mmio);
+	if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
+		iounmap(gmu->rscc);
 	free_irq(gmu->gmu_irq, gmu);
 	free_irq(gmu->hfi_irq, gmu);
-err_memory:
-	a6xx_gmu_memory_free(gmu, gmu->hfi);
 
 	ret = -ENODEV;
 
+err_memory:
+	a6xx_gmu_memory_free(gmu);
 err_put_device:
 	/* Drop reference taken in of_find_device_by_node */
 	put_device(gmu->dev);
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index 4af65a36d5ca..47df4745db50 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
@@ -10,9 +10,10 @@
 #include "a6xx_hfi.h"
 
 struct a6xx_gmu_bo {
+	struct drm_gem_object *obj;
 	void *virt;
 	size_t size;
-	dma_addr_t iova;
+	u64 iova;
 };
 
 /*
@@ -43,7 +44,10 @@ struct a6xx_gmu_bo {
 struct a6xx_gmu {
 	struct device *dev;
 
+	struct msm_gem_address_space *aspace;
+
 	void * __iomem mmio;
+	void * __iomem rscc;
 
 	int hfi_irq;
 	int gmu_irq;
@@ -52,8 +56,12 @@ struct a6xx_gmu {
 
 	int idle_level;
 
-	struct a6xx_gmu_bo *hfi;
-	struct a6xx_gmu_bo *debug;
+	struct a6xx_gmu_bo hfi;
+	struct a6xx_gmu_bo debug;
+	struct a6xx_gmu_bo icache;
+	struct a6xx_gmu_bo dcache;
+	struct a6xx_gmu_bo dummy;
+	struct a6xx_gmu_bo log;
 
 	int nr_clocks;
 	struct clk_bulk_data *clocks;
@@ -76,6 +84,7 @@ struct a6xx_gmu {
 
 	bool initialized;
 	bool hung;
+	bool legacy; /* a618 or a630 */
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
@@ -88,6 +97,13 @@ static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value)
 	return msm_writel(value, gmu->mmio + (offset << 2));
 }
 
+static inline void
+gmu_write_bulk(struct a6xx_gmu *gmu, u32 offset, const u32 *data, u32 size)
+{
+	memcpy_toio(gmu->mmio + (offset << 2), data, size);
+	wmb();
+}
+
 static inline void gmu_rmw(struct a6xx_gmu *gmu, u32 reg, u32 mask, u32 or)
 {
 	u32 val = gmu_read(gmu, reg);
@@ -111,6 +127,15 @@ static inline u64 gmu_read64(struct a6xx_gmu *gmu, u32 lo, u32 hi)
 	readl_poll_timeout((gmu)->mmio + ((addr) << 2), val, cond, \
 		interval, timeout)
 
+static inline void gmu_write_rscc(struct a6xx_gmu *gmu, u32 offset, u32 value)
+{
+	return msm_writel(value, gmu->rscc + (offset << 2));
+}
+
+#define gmu_poll_timeout_rscc(gmu, addr, val, cond, interval, timeout) \
+	readl_poll_timeout((gmu)->rscc + ((addr) << 2), val, cond, \
+		interval, timeout)
+
 /*
  * These are the available OOB (out of band requests) to the GMU where "out of
  * band" means that the CPU talks to the GMU directly and not through HFI.
@@ -156,10 +181,16 @@ enum a6xx_gmu_oob_state {
 #define GMU_OOB_GPU_SET_ACK	24
 #define GMU_OOB_GPU_SET_CLEAR	24
 
+#define GMU_OOB_GPU_SET_REQUEST_NEW	30
+#define GMU_OOB_GPU_SET_ACK_NEW		31
+#define GMU_OOB_GPU_SET_CLEAR_NEW	31
+
 
 void a6xx_hfi_init(struct a6xx_gmu *gmu);
 int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state);
 void a6xx_hfi_stop(struct a6xx_gmu *gmu);
+int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu);
+int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, int index);
 
 bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu);
 bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu);
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
index 1cc1c135236b..176ae94d9fe6 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h
@@ -101,6 +101,10 @@ static inline uint32_t A6XX_HFI_IRQ_OOB_MASK(uint32_t val)
 
 #define REG_A6XX_GMU_DCVS_RETURN				0x000023ff
 
+#define REG_A6XX_GMU_ICACHE_CONFIG				0x00004c00
+
+#define REG_A6XX_GMU_DCACHE_CONFIG				0x00004c01
+
 #define REG_A6XX_GMU_SYS_BUS_CONFIG				0x00004c0f
 
 #define REG_A6XX_GMU_CM3_SYSRESET				0x00005000
@@ -199,6 +203,12 @@ static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val)
 
 #define REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE		0x000050ec
 
+#define REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF			0x000050f0
+
+#define REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG			0x00005100
+
+#define REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP			0x00005101
+
 #define REG_A6XX_GMU_BOOT_KMD_LM_HANDSHAKE			0x000051f0
 
 #define REG_A6XX_GMU_LLM_GLM_SLEEP_CTRL				0x00005157
@@ -330,8 +340,6 @@ static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val)
 
 #define REG_A6XX_GMU_AO_SPARE_CNTL				0x00009316
 
-#define REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0			0x00008c04
-
 #define REG_A6XX_GMU_RSCC_CONTROL_REQ				0x00009307
 
 #define REG_A6XX_GMU_RSCC_CONTROL_ACK				0x00009308
@@ -344,39 +352,41 @@ static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val)
 
 #define REG_A6XX_GPU_CC_GX_DOMAIN_MISC				0x00009d42
 
-#define REG_A6XX_RSCC_PDC_SEQ_START_ADDR			0x00008c08
+#define REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0			0x00000004
+
+#define REG_A6XX_RSCC_PDC_SEQ_START_ADDR			0x00000008
 
-#define REG_A6XX_RSCC_PDC_MATCH_VALUE_LO			0x00008c09
+#define REG_A6XX_RSCC_PDC_MATCH_VALUE_LO			0x00000009
 
-#define REG_A6XX_RSCC_PDC_MATCH_VALUE_HI			0x00008c0a
+#define REG_A6XX_RSCC_PDC_MATCH_VALUE_HI			0x0000000a
 
-#define REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0				0x00008c0b
+#define REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0				0x0000000b
 
-#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR			0x00008c0d
+#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR			0x0000000d
 
-#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA			0x00008c0e
+#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA			0x0000000e
 
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0		0x00008c82
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0		0x00000082
 
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0		0x00008c83
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0		0x00000083
 
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0			0x00008c89
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0			0x00000089
 
-#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0		0x00008c8c
+#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0		0x0000008c
 
-#define REG_A6XX_RSCC_OVERRIDE_START_ADDR			0x00008d00
+#define REG_A6XX_RSCC_OVERRIDE_START_ADDR			0x00000100
 
-#define REG_A6XX_RSCC_SEQ_BUSY_DRV0				0x00008d01
+#define REG_A6XX_RSCC_SEQ_BUSY_DRV0				0x00000101
 
-#define REG_A6XX_RSCC_SEQ_MEM_0_DRV0				0x00008d80
+#define REG_A6XX_RSCC_SEQ_MEM_0_DRV0				0x00000180
 
-#define REG_A6XX_RSCC_TCS0_DRV0_STATUS				0x00008f46
+#define REG_A6XX_RSCC_TCS0_DRV0_STATUS				0x00000346
 
-#define REG_A6XX_RSCC_TCS1_DRV0_STATUS				0x000090ae
+#define REG_A6XX_RSCC_TCS1_DRV0_STATUS				0x000003ee
 
-#define REG_A6XX_RSCC_TCS2_DRV0_STATUS				0x00009216
+#define REG_A6XX_RSCC_TCS2_DRV0_STATUS				0x00000496
 
-#define REG_A6XX_RSCC_TCS3_DRV0_STATUS				0x0000937e
+#define REG_A6XX_RSCC_TCS3_DRV0_STATUS				0x0000053e
 
 
 #endif /* A6XX_GMU_XML */
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index 68af24150de5..a1589e040c57 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -414,7 +414,17 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
 		a6xx_set_hwcg(gpu, true);
 
 	/* VBIF/GBIF start*/
-	gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
+	if (adreno_is_a640(adreno_gpu) || adreno_is_a650(adreno_gpu)) {
+		gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620);
+		gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620);
+		gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620);
+		gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
+		gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
+		gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x3);
+	} else {
+		gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
+	}
+
 	if (adreno_is_a630(adreno_gpu))
 		gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
 
@@ -429,25 +439,35 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000);
 	gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);
 
-	/* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
-	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,
-		REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000);
+	if (!adreno_is_a650(adreno_gpu)) {
+		/* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
+		gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO,
+			REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000);
 
-	gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
-		REG_A6XX_UCHE_GMEM_RANGE_MAX_HI,
-		0x00100000 + adreno_gpu->gmem - 1);
+		gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO,
+			REG_A6XX_UCHE_GMEM_RANGE_MAX_HI,
+			0x00100000 + adreno_gpu->gmem - 1);
+	}
 
 	gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
 	gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
 
-	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
+	if (adreno_is_a640(adreno_gpu) || adreno_is_a650(adreno_gpu))
+		gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140);
+	else
+		gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
 	gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
 
 	/* Setting the mem pool size */
 	gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
 
 	/* Setting the primFifo thresholds default values */
-	gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11));
+	if (adreno_is_a650(adreno_gpu))
+		gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300000);
+	else if (adreno_is_a640(adreno_gpu))
+		gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200000);
+	else
+		gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11));
 
 	/* Set the AHB default slave response to "ERROR" */
 	gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
@@ -471,6 +491,19 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
 
 	gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1);
 
+	/* Set weights for bicubic filtering */
+	if (adreno_is_a650(adreno_gpu)) {
+		gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0);
+		gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1,
+			0x3fe05ff4);
+		gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2,
+			0x3fa0ebee);
+		gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3,
+			0x3f5193ed);
+		gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4,
+			0x3f0243f0);
+	}
+
 	/* Protect registers from the CP */
 	gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003);
 
@@ -508,6 +541,11 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
 			A6XX_PROTECT_RDONLY(0x980, 0x4));
 	gpu_write(gpu, REG_A6XX_CP_PROTECT(25), A6XX_PROTECT_RW(0xa630, 0x0));
 
+	if (adreno_is_a650(adreno_gpu)) {
+		gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
+			(1 << 6) | (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1));
+	}
+
 	/* Enable interrupts */
 	gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK);
 
@@ -566,8 +604,10 @@ out:
 	 */
 	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
 
-	/* Take the GMU out of its special boot mode */
-	a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
+	if (a6xx_gpu->gmu.legacy) {
+		/* Take the GMU out of its special boot mode */
+		a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
+	}
 
 	return ret;
 }
@@ -810,6 +850,11 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 	u64 busy_cycles, busy_time;
 
+
+	/* Only read the gpu busy if the hardware is already active */
+	if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0)
+		return 0;
+
 	busy_cycles = gmu_read64(&a6xx_gpu->gmu,
 			REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L,
 			REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
@@ -819,6 +864,8 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
 
 	gpu->devfreq.busy_cycles = busy_cycles;
 
+	pm_runtime_put(a6xx_gpu->gmu.dev);
+
 	if (WARN_ON(busy_time > ~0LU))
 		return ~0LU;
 
@@ -846,6 +893,7 @@ static const struct adreno_gpu_funcs funcs = {
 #if defined(CONFIG_DRM_MSM_GPU_STATE)
 		.gpu_state_get = a6xx_gpu_state_get,
 		.gpu_state_put = a6xx_gpu_state_put,
+		.create_address_space = adreno_iommu_create_address_space,
 #endif
 	},
 	.get_timestamp = a6xx_get_timestamp,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
index e450e0b97211..9921e632f1ca 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
@@ -17,10 +17,14 @@ static const char * const a6xx_hfi_msg_id[] = {
 	HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE),
 	HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE),
 	HFI_MSG_ID(HFI_H2F_MSG_TEST),
+	HFI_MSG_ID(HFI_H2F_MSG_START),
+	HFI_MSG_ID(HFI_H2F_MSG_CORE_FW_START),
+	HFI_MSG_ID(HFI_H2F_MSG_GX_BW_PERF_VOTE),
+	HFI_MSG_ID(HFI_H2F_MSG_PREPARE_SLUMBER),
 };
 
-static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data,
-		u32 dwords)
+static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu,
+	struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
 {
 	struct a6xx_hfi_queue_header *header = queue->header;
 	u32 i, hdr, index = header->read_index;
@@ -48,6 +52,9 @@ static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data,
 		index = (index + 1) % header->size;
 	}
 
+	if (!gmu->legacy)
+		index = ALIGN(index, 4) % header->size;
+
 	header->read_index = index;
 	return HFI_HEADER_SIZE(hdr);
 }
@@ -73,6 +80,12 @@ static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu,
 		index = (index + 1) % header->size;
 	}
 
+	/* Cookify any non used data at the end of the write buffer */
+	if (!gmu->legacy) {
+		for (; index % 4; index = (index + 1) % header->size)
+			queue->data[index] = 0xfafafafa;
+	}
+
 	header->write_index = index;
 	spin_unlock(&queue->lock);
 
@@ -106,7 +119,7 @@ static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum,
 		struct a6xx_hfi_msg_response resp;
 
 		/* Get the next packet */
-		ret = a6xx_hfi_queue_read(queue, (u32 *) &resp,
+		ret = a6xx_hfi_queue_read(gmu, queue, (u32 *) &resp,
 			sizeof(resp) >> 2);
 
 		/* If the queue is empty our response never made it */
@@ -176,8 +189,8 @@ static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state)
 {
 	struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };
 
-	msg.dbg_buffer_addr = (u32) gmu->debug->iova;
-	msg.dbg_buffer_size = (u32) gmu->debug->size;
+	msg.dbg_buffer_addr = (u32) gmu->debug.iova;
+	msg.dbg_buffer_size = (u32) gmu->debug.size;
 	msg.boot_state = boot_state;
 
 	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg),
@@ -195,6 +208,28 @@ static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version)
 		version, sizeof(*version));
 }
 
+static int a6xx_hfi_send_perf_table_v1(struct a6xx_gmu *gmu)
+{
+	struct a6xx_hfi_msg_perf_table_v1 msg = { 0 };
+	int i;
+
+	msg.num_gpu_levels = gmu->nr_gpu_freqs;
+	msg.num_gmu_levels = gmu->nr_gmu_freqs;
+
+	for (i = 0; i < gmu->nr_gpu_freqs; i++) {
+		msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
+		msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
+	}
+
+	for (i = 0; i < gmu->nr_gmu_freqs; i++) {
+		msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
+		msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
+	}
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
+		NULL, 0);
+}
+
 static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
 {
 	struct a6xx_hfi_msg_perf_table msg = { 0 };
@@ -205,6 +240,7 @@ static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
 
 	for (i = 0; i < gmu->nr_gpu_freqs; i++) {
 		msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
+		msg.gx_votes[i].acd = 0xffffffff;
 		msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
 	}
 
@@ -306,7 +342,45 @@ static int a6xx_hfi_send_test(struct a6xx_gmu *gmu)
 		NULL, 0);
 }
 
-int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
+static int a6xx_hfi_send_start(struct a6xx_gmu *gmu)
+{
+	struct a6xx_hfi_msg_start msg = { 0 };
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_START, &msg, sizeof(msg),
+		NULL, 0);
+}
+
+static int a6xx_hfi_send_core_fw_start(struct a6xx_gmu *gmu)
+{
+	struct a6xx_hfi_msg_core_fw_start msg = { 0 };
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_CORE_FW_START, &msg,
+		sizeof(msg), NULL, 0);
+}
+
+int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, int index)
+{
+	struct a6xx_hfi_gx_bw_perf_vote_cmd msg = { 0 };
+
+	msg.ack_type = 1; /* blocking */
+	msg.freq = index;
+	msg.bw = 0; /* TODO: bus scaling */
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_GX_BW_PERF_VOTE, &msg,
+		sizeof(msg), NULL, 0);
+}
+
+int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu)
+{
+	struct a6xx_hfi_prep_slumber_cmd msg = { 0 };
+
+	/* TODO: should freq and bw fields be non-zero ? */
+
+	return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PREPARE_SLUMBER, &msg,
+		sizeof(msg), NULL, 0);
+}
+
+static int a6xx_hfi_start_v1(struct a6xx_gmu *gmu, int boot_state)
 {
 	int ret;
 
@@ -324,7 +398,7 @@ int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
 	 * the GMU firmware
 	 */
 
-	ret = a6xx_hfi_send_perf_table(gmu);
+	ret = a6xx_hfi_send_perf_table_v1(gmu);
 	if (ret)
 		return ret;
 
@@ -341,6 +415,37 @@ int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
 	return 0;
 }
 
+int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
+{
+	int ret;
+
+	if (gmu->legacy)
+		return a6xx_hfi_start_v1(gmu, boot_state);
+
+
+	ret = a6xx_hfi_send_perf_table(gmu);
+	if (ret)
+		return ret;
+
+	ret = a6xx_hfi_send_bw_table(gmu);
+	if (ret)
+		return ret;
+
+	ret = a6xx_hfi_send_core_fw_start(gmu);
+	if (ret)
+		return ret;
+
+	/*
+	 * Downstream driver sends this in its "a6xx_hw_init" equivalent,
+	 * but seems to be no harm in sending it here
+	 */
+	ret = a6xx_hfi_send_start(gmu);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 void a6xx_hfi_stop(struct a6xx_gmu *gmu)
 {
 	int i;
@@ -385,7 +490,7 @@ static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue,
 
 void a6xx_hfi_init(struct a6xx_gmu *gmu)
 {
-	struct a6xx_gmu_bo *hfi = gmu->hfi;
+	struct a6xx_gmu_bo *hfi = &gmu->hfi;
 	struct a6xx_hfi_queue_table_header *table = hfi->virt;
 	struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table);
 	u64 offset;
@@ -415,5 +520,5 @@ void a6xx_hfi_init(struct a6xx_gmu *gmu)
 	/* GMU response queue */
 	offset += SZ_4K;
 	a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset,
-		hfi->iova + offset, 4);
+		hfi->iova + offset, gmu->legacy ? 4 : 1);
 }
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.h b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h
index 60d1319fa44f..2bd670ca42d6 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h
@@ -51,7 +51,8 @@ struct a6xx_hfi_queue {
 /* HFI message types */
 
 #define HFI_MSG_CMD 0
-#define HFI_MSG_ACK 2
+#define HFI_MSG_ACK 1
+#define HFI_MSG_ACK_V1 2
 
 #define HFI_F2H_MSG_ACK 126
 
@@ -94,7 +95,13 @@ struct perf_level {
 	u32 freq;
 };
 
-struct a6xx_hfi_msg_perf_table {
+struct perf_gx_level {
+	u32 vote;
+	u32 acd;
+	u32 freq;
+};
+
+struct a6xx_hfi_msg_perf_table_v1 {
 	u32 header;
 	u32 num_gpu_levels;
 	u32 num_gmu_levels;
@@ -103,6 +110,15 @@ struct a6xx_hfi_msg_perf_table {
 	struct perf_level cx_votes[4];
 };
 
+struct a6xx_hfi_msg_perf_table {
+	u32 header;
+	u32 num_gpu_levels;
+	u32 num_gmu_levels;
+
+	struct perf_gx_level gx_votes[16];
+	struct perf_level cx_votes[4];
+};
+
 #define HFI_H2F_MSG_BW_TABLE 3
 
 struct a6xx_hfi_msg_bw_table {
@@ -124,4 +140,34 @@ struct a6xx_hfi_msg_test {
 	u32 header;
 };
 
+#define HFI_H2F_MSG_START 10
+
+struct a6xx_hfi_msg_start {
+	u32 header;
+};
+
+#define HFI_H2F_MSG_CORE_FW_START 14
+
+struct a6xx_hfi_msg_core_fw_start {
+	u32 header;
+	u32 handle;
+};
+
+#define HFI_H2F_MSG_GX_BW_PERF_VOTE 30
+
+struct a6xx_hfi_gx_bw_perf_vote_cmd {
+	u32 header;
+	u32 ack_type;
+	u32 freq;
+	u32 bw;
+};
+
+#define HFI_H2F_MSG_PREPARE_SLUMBER 33
+
+struct a6xx_hfi_prep_slumber_cmd {
+	u32 header;
+	u32 bw;
+	u32 freq;
+};
+
 #endif
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index cb3a6e597d76..7732f03d9e3a 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -93,6 +93,17 @@ static const struct adreno_info gpulist[] = {
 		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init  = a3xx_gpu_init,
 	}, {
+		.rev   = ADRENO_REV(4, 0, 5, ANY_ID),
+		.revn  = 405,
+		.name  = "A405",
+		.fw = {
+			[ADRENO_FW_PM4] = "a420_pm4.fw",
+			[ADRENO_FW_PFP] = "a420_pfp.fw",
+		},
+		.gmem  = SZ_256K,
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
+		.init  = a4xx_gpu_init,
+	}, {
 		.rev   = ADRENO_REV(4, 2, 0, ANY_ID),
 		.revn  = 420,
 		.name  = "A420",
@@ -189,6 +200,30 @@ static const struct adreno_info gpulist[] = {
 		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init = a6xx_gpu_init,
 		.zapfw = "a630_zap.mdt",
+	}, {
+		.rev = ADRENO_REV(6, 4, 0, ANY_ID),
+		.revn = 640,
+		.name = "A640",
+		.fw = {
+			[ADRENO_FW_SQE] = "a630_sqe.fw",
+			[ADRENO_FW_GMU] = "a640_gmu.bin",
+		},
+		.gmem = SZ_1M,
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
+		.init = a6xx_gpu_init,
+		.zapfw = "a640_zap.mdt",
+	}, {
+		.rev = ADRENO_REV(6, 5, 0, ANY_ID),
+		.revn = 650,
+		.name = "A650",
+		.fw = {
+			[ADRENO_FW_SQE] = "a650_sqe.fw",
+			[ADRENO_FW_GMU] = "a650_gmu.bin",
+		},
+		.gmem = SZ_1M + SZ_128K,
+		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
+		.init = a6xx_gpu_init,
+		.zapfw = "a650_zap.mdt",
 	},
 };
 
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 1d5c43c22269..89673c7ed473 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -185,6 +185,23 @@ int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid)
 	return zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw, pasid);
 }
 
+struct msm_gem_address_space *
+adreno_iommu_create_address_space(struct msm_gpu *gpu,
+		struct platform_device *pdev)
+{
+	struct iommu_domain *iommu = iommu_domain_alloc(&platform_bus_type);
+	struct msm_mmu *mmu = msm_iommu_new(&pdev->dev, iommu);
+	struct msm_gem_address_space *aspace;
+
+	aspace = msm_gem_address_space_create(mmu, "gpu", SZ_16M,
+		0xfffffff);
+
+	if (IS_ERR(aspace) && !IS_ERR(mmu))
+		mmu->funcs->destroy(mmu);
+
+	return aspace;
+}
+
 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -197,7 +214,7 @@ int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
 		*value = adreno_gpu->gmem;
 		return 0;
 	case MSM_PARAM_GMEM_BASE:
-		*value = 0x100000;
+		*value = !adreno_is_a650(adreno_gpu) ? 0x100000 : 0;
 		return 0;
 	case MSM_PARAM_CHIP_ID:
 		*value = adreno_gpu->rev.patchid |
@@ -459,7 +476,7 @@ void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 				break;
 			/* fall-thru */
 		case MSM_SUBMIT_CMD_BUF:
-			OUT_PKT3(ring, adreno_is_a430(adreno_gpu) ?
+			OUT_PKT3(ring, adreno_is_a4xx(adreno_gpu) ?
 				CP_INDIRECT_BUFFER_PFE : CP_INDIRECT_BUFFER_PFD, 2);
 			OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
 			OUT_RING(ring, submit->cmd[i].size);
@@ -988,12 +1005,6 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 
 	adreno_gpu_config.ioname = "kgsl_3d0_reg_memory";
 
-	adreno_gpu_config.va_start = SZ_16M;
-	adreno_gpu_config.va_end = 0xffffffff;
-	/* maximum range of a2xx mmu */
-	if (adreno_is_a2xx(adreno_gpu))
-		adreno_gpu_config.va_end = SZ_16M + 0xfff * SZ_64K;
-
 	adreno_gpu_config.nr_rings = nr_rings;
 
 	adreno_get_pwrlevels(&pdev->dev, gpu);
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index 9ff4e550e7bd..2f5d2c3acc3a 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -202,6 +202,11 @@ static inline bool adreno_is_a4xx(struct adreno_gpu *gpu)
 	return (gpu->revn >= 400) && (gpu->revn < 500);
 }
 
+static inline int adreno_is_a405(struct adreno_gpu *gpu)
+{
+	return gpu->revn == 405;
+}
+
 static inline int adreno_is_a420(struct adreno_gpu *gpu)
 {
 	return gpu->revn == 420;
@@ -237,6 +242,16 @@ static inline int adreno_is_a630(struct adreno_gpu *gpu)
        return gpu->revn == 630;
 }
 
+static inline int adreno_is_a640(struct adreno_gpu *gpu)
+{
+       return gpu->revn == 640;
+}
+
+static inline int adreno_is_a650(struct adreno_gpu *gpu)
+{
+       return gpu->revn == 650;
+}
+
 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value);
 const struct firmware *adreno_request_fw(struct adreno_gpu *adreno_gpu,
 		const char *fwname);
@@ -273,6 +288,14 @@ int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state);
 int adreno_gpu_state_put(struct msm_gpu_state *state);
 
 /*
+ * Common helper function to initialize the default address space for arm-smmu
+ * attached targets
+ */
+struct msm_gem_address_space *
+adreno_iommu_create_address_space(struct msm_gpu *gpu,
+		struct platform_device *pdev);
+
+/*
  * For a5xx and a6xx targets load the zap shader that is used to pull the GPU
  * out of secure mode
  */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
index 11f2bebe3869..7c230f719ad3 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c
@@ -36,22 +36,6 @@ static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
 	return to_dpu_kms(priv->kms);
 }
 
-static bool _dpu_core_video_mode_intf_connected(struct drm_crtc *crtc)
-{
-	struct drm_crtc *tmp_crtc;
-
-	drm_for_each_crtc(tmp_crtc, crtc->dev) {
-		if ((dpu_crtc_get_intf_mode(tmp_crtc) == INTF_MODE_VIDEO) &&
-				tmp_crtc->enabled) {
-			DPU_DEBUG("video interface connected crtc:%d\n",
-				tmp_crtc->base.id);
-			return true;
-		}
-	}
-
-	return false;
-}
-
 static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms,
 		struct drm_crtc *crtc,
 		struct drm_crtc_state *state,
@@ -94,7 +78,6 @@ int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
 	u32 bw, threshold;
 	u64 bw_sum_of_intfs = 0;
 	enum dpu_crtc_client_type curr_client_type;
-	bool is_video_mode;
 	struct dpu_crtc_state *dpu_cstate;
 	struct drm_crtc *tmp_crtc;
 	struct dpu_kms *kms;
@@ -144,11 +127,7 @@ int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
 		bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000);
 		DPU_DEBUG("calculated bandwidth=%uk\n", bw);
 
-		is_video_mode = dpu_crtc_get_intf_mode(crtc) == INTF_MODE_VIDEO;
-		threshold = (is_video_mode ||
-			_dpu_core_video_mode_intf_connected(crtc)) ?
-			kms->catalog->perf.max_bw_low :
-			kms->catalog->perf.max_bw_high;
+		threshold = kms->catalog->perf.max_bw_high;
 
 		DPU_DEBUG("final threshold bw limit = %d\n", threshold);
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
index 17448505a9b5..e15b42a780e0 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -9,6 +9,7 @@
 #include <linux/sort.h>
 #include <linux/debugfs.h>
 #include <linux/ktime.h>
+#include <linux/bits.h>
 
 #include <drm/drm_crtc.h>
 #include <drm/drm_flip_work.h>
@@ -20,6 +21,7 @@
 #include "dpu_kms.h"
 #include "dpu_hw_lm.h"
 #include "dpu_hw_ctl.h"
+#include "dpu_hw_dspp.h"
 #include "dpu_crtc.h"
 #include "dpu_plane.h"
 #include "dpu_encoder.h"
@@ -40,6 +42,9 @@
 /* timeout in ms waiting for frame done */
 #define DPU_CRTC_FRAME_DONE_TIMEOUT_MS	60
 
+#define	CONVERT_S3_15(val) \
+	(((((u64)val) & ~BIT_ULL(63)) >> 17) & GENMASK_ULL(17, 0))
+
 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
 {
 	struct msm_drm_private *priv = crtc->dev->dev_private;
@@ -88,11 +93,9 @@ static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
 
 static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
 {
-	struct dpu_crtc *dpu_crtc;
 	struct dpu_crtc_state *crtc_state;
 	int lm_idx, lm_horiz_position;
 
-	dpu_crtc = to_dpu_crtc(crtc);
 	crtc_state = to_dpu_crtc_state(crtc->state);
 
 	lm_horiz_position = 0;
@@ -422,6 +425,74 @@ static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc,
 	drm_mode_debug_printmodeline(adj_mode);
 }
 
+static void _dpu_crtc_get_pcc_coeff(struct drm_crtc_state *state,
+		struct dpu_hw_pcc_cfg *cfg)
+{
+	struct drm_color_ctm *ctm;
+
+	memset(cfg, 0, sizeof(struct dpu_hw_pcc_cfg));
+
+	ctm = (struct drm_color_ctm *)state->ctm->data;
+
+	if (!ctm)
+		return;
+
+	cfg->r.r = CONVERT_S3_15(ctm->matrix[0]);
+	cfg->g.r = CONVERT_S3_15(ctm->matrix[1]);
+	cfg->b.r = CONVERT_S3_15(ctm->matrix[2]);
+
+	cfg->r.g = CONVERT_S3_15(ctm->matrix[3]);
+	cfg->g.g = CONVERT_S3_15(ctm->matrix[4]);
+	cfg->b.g = CONVERT_S3_15(ctm->matrix[5]);
+
+	cfg->r.b = CONVERT_S3_15(ctm->matrix[6]);
+	cfg->g.b = CONVERT_S3_15(ctm->matrix[7]);
+	cfg->b.b = CONVERT_S3_15(ctm->matrix[8]);
+}
+
+static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
+{
+	struct drm_crtc_state *state = crtc->state;
+	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
+	struct dpu_crtc_mixer *mixer = cstate->mixers;
+	struct dpu_hw_pcc_cfg cfg;
+	struct dpu_hw_ctl *ctl;
+	struct dpu_hw_mixer *lm;
+	struct dpu_hw_dspp *dspp;
+	int i;
+
+
+	if (!state->color_mgmt_changed)
+		return;
+
+	for (i = 0; i < cstate->num_mixers; i++) {
+		ctl = mixer[i].lm_ctl;
+		lm = mixer[i].hw_lm;
+		dspp = mixer[i].hw_dspp;
+
+		if (!dspp || !dspp->ops.setup_pcc)
+			continue;
+
+		if (!state->ctm) {
+			dspp->ops.setup_pcc(dspp, NULL);
+		} else {
+			_dpu_crtc_get_pcc_coeff(state, &cfg);
+			dspp->ops.setup_pcc(dspp, &cfg);
+		}
+
+		mixer[i].flush_mask |= ctl->ops.get_bitmask_dspp(ctl,
+			mixer[i].hw_dspp->idx);
+
+		/* stage config flush mask */
+		ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask);
+
+		DPU_DEBUG("lm %d, ctl %d, flush mask 0x%x\n",
+			mixer[i].hw_lm->idx - DSPP_0,
+			ctl->idx - CTL_0,
+			mixer[i].flush_mask);
+	}
+}
+
 static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 		struct drm_crtc_state *old_state)
 {
@@ -430,7 +501,6 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 	struct drm_encoder *encoder;
 	struct drm_device *dev;
 	unsigned long flags;
-	struct dpu_crtc_smmu_state_data *smmu_state;
 
 	if (!crtc) {
 		DPU_ERROR("invalid crtc\n");
@@ -448,7 +518,6 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 	dpu_crtc = to_dpu_crtc(crtc);
 	cstate = to_dpu_crtc_state(crtc->state);
 	dev = crtc->dev;
-	smmu_state = &dpu_crtc->smmu_state;
 
 	_dpu_crtc_setup_lm_bounds(crtc, crtc->state);
 
@@ -475,6 +544,8 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 
 	_dpu_crtc_blend_setup(crtc);
 
+	_dpu_crtc_setup_cp_blocks(crtc);
+
 	/*
 	 * PP_DONE irq is only used by command mode for now.
 	 * It is better to request pending before FLUSH and START trigger
@@ -491,7 +562,6 @@ static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
 	struct drm_device *dev;
 	struct drm_plane *plane;
 	struct msm_drm_private *priv;
-	struct msm_drm_thread *event_thread;
 	unsigned long flags;
 	struct dpu_crtc_state *cstate;
 
@@ -513,8 +583,6 @@ static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
 		return;
 	}
 
-	event_thread = &priv->event_thread[crtc->index];
-
 	if (dpu_crtc->event) {
 		DPU_DEBUG("already received dpu_crtc->event\n");
 	} else {
@@ -567,7 +635,6 @@ static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
 static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
 		struct drm_crtc_state *state)
 {
-	struct dpu_crtc *dpu_crtc;
 	struct dpu_crtc_state *cstate;
 
 	if (!crtc || !state) {
@@ -575,7 +642,6 @@ static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
 		return;
 	}
 
-	dpu_crtc = to_dpu_crtc(crtc);
 	cstate = to_dpu_crtc_state(state);
 
 	DPU_DEBUG("crtc%d\n", crtc->base.id);
@@ -662,11 +728,9 @@ static void dpu_crtc_reset(struct drm_crtc *crtc)
 /**
  * dpu_crtc_duplicate_state - state duplicate hook
  * @crtc: Pointer to drm crtc structure
- * @Returns: Pointer to new drm_crtc_state structure
  */
 static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
 {
-	struct dpu_crtc *dpu_crtc;
 	struct dpu_crtc_state *cstate, *old_cstate;
 
 	if (!crtc || !crtc->state) {
@@ -674,7 +738,6 @@ static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
 		return NULL;
 	}
 
-	dpu_crtc = to_dpu_crtc(crtc);
 	old_cstate = to_dpu_crtc_state(crtc->state);
 	cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
 	if (!cstate) {
@@ -693,9 +756,7 @@ static void dpu_crtc_disable(struct drm_crtc *crtc,
 {
 	struct dpu_crtc *dpu_crtc;
 	struct dpu_crtc_state *cstate;
-	struct drm_display_mode *mode;
 	struct drm_encoder *encoder;
-	struct msm_drm_private *priv;
 	unsigned long flags;
 	bool release_bandwidth = false;
 
@@ -705,8 +766,6 @@ static void dpu_crtc_disable(struct drm_crtc *crtc,
 	}
 	dpu_crtc = to_dpu_crtc(crtc);
 	cstate = to_dpu_crtc_state(crtc->state);
-	mode = &cstate->base.adjusted_mode;
-	priv = crtc->dev->dev_private;
 
 	DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
 
@@ -768,14 +827,12 @@ static void dpu_crtc_enable(struct drm_crtc *crtc,
 {
 	struct dpu_crtc *dpu_crtc;
 	struct drm_encoder *encoder;
-	struct msm_drm_private *priv;
 	bool request_bandwidth;
 
 	if (!crtc) {
 		DPU_ERROR("invalid crtc\n");
 		return;
 	}
-	priv = crtc->dev->dev_private;
 
 	pm_runtime_get_sync(crtc->dev->dev);
 
@@ -1319,6 +1376,8 @@ struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
 
 	drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
 
+	drm_crtc_enable_color_mgmt(crtc, 0, true, 0);
+
 	/* save user friendly CRTC name for later */
 	snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
index 5174e86124cc..cec3474340e8 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
@@ -73,12 +73,14 @@ struct dpu_crtc_smmu_state_data {
  * struct dpu_crtc_mixer: stores the map for each virtual pipeline in the CRTC
  * @hw_lm:	LM HW Driver context
  * @lm_ctl:	CTL Path HW driver context
+ * @lm_dspp:	DSPP HW driver context
  * @mixer_op_mode:	mixer blending operation mode
  * @flush_mask:	mixer flush mask for ctl, mixer and pipe
  */
 struct dpu_crtc_mixer {
 	struct dpu_hw_mixer *hw_lm;
 	struct dpu_hw_ctl *lm_ctl;
+	struct dpu_hw_dspp *hw_dspp;
 	u32 mixer_op_mode;
 	u32 flush_mask;
 };
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
index a1b79ee2bd9d..63976dcd2ac8 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
@@ -20,6 +20,7 @@
 #include "dpu_hw_catalog.h"
 #include "dpu_hw_intf.h"
 #include "dpu_hw_ctl.h"
+#include "dpu_hw_dspp.h"
 #include "dpu_formats.h"
 #include "dpu_encoder_phys.h"
 #include "dpu_crtc.h"
@@ -536,6 +537,7 @@ static struct msm_display_topology dpu_encoder_get_topology(
 	 * 1 LM, 1 INTF
 	 * 2 LM, 1 INTF (stream merge to support high resolution interfaces)
 	 *
+	 * Adding color blocks only to primary interface
 	 */
 	if (intf_count == 2)
 		topology.num_lm = 2;
@@ -544,6 +546,9 @@ static struct msm_display_topology dpu_encoder_get_topology(
 	else
 		topology.num_lm = (mode->hdisplay > MAX_HDISPLAY_SPLIT) ? 2 : 1;
 
+	if (dpu_enc->disp_info.intf_type == DRM_MODE_ENCODER_DSI)
+		topology.num_dspp = topology.num_lm;
+
 	topology.num_enc = 0;
 	topology.num_intf = intf_count;
 
@@ -959,7 +964,8 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 	struct dpu_hw_blk *hw_pp[MAX_CHANNELS_PER_ENC];
 	struct dpu_hw_blk *hw_ctl[MAX_CHANNELS_PER_ENC];
 	struct dpu_hw_blk *hw_lm[MAX_CHANNELS_PER_ENC];
-	int num_lm, num_ctl, num_pp;
+	struct dpu_hw_blk *hw_dspp[MAX_CHANNELS_PER_ENC] = { NULL };
+	int num_lm, num_ctl, num_pp, num_dspp;
 	int i, j;
 
 	if (!drm_enc) {
@@ -1008,6 +1014,9 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 		drm_enc->base.id, DPU_HW_BLK_CTL, hw_ctl, ARRAY_SIZE(hw_ctl));
 	num_lm = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
 		drm_enc->base.id, DPU_HW_BLK_LM, hw_lm, ARRAY_SIZE(hw_lm));
+	num_dspp = dpu_rm_get_assigned_resources(&dpu_kms->rm, global_state,
+		drm_enc->base.id, DPU_HW_BLK_DSPP, hw_dspp,
+		ARRAY_SIZE(hw_dspp));
 
 	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++)
 		dpu_enc->hw_pp[i] = i < num_pp ? to_dpu_hw_pingpong(hw_pp[i])
@@ -1020,6 +1029,7 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 
 		cstate->mixers[i].hw_lm = to_dpu_hw_mixer(hw_lm[i]);
 		cstate->mixers[i].lm_ctl = to_dpu_hw_ctl(hw_ctl[ctl_idx]);
+		cstate->mixers[i].hw_dspp = to_dpu_hw_dspp(hw_dspp[i]);
 	}
 
 	cstate->num_mixers = num_lm;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
index c567917541e8..29d4fde3172b 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
@@ -41,6 +41,8 @@
 #define PINGPONG_SDM845_SPLIT_MASK \
 	(PINGPONG_SDM845_MASK | BIT(DPU_PINGPONG_TE2))
 
+#define DSPP_SC7180_MASK BIT(DPU_DSPP_PCC)
+
 #define DEFAULT_PIXEL_RAM_SIZE		(50 * 1024)
 #define DEFAULT_DPU_LINE_WIDTH		2048
 #define DEFAULT_DPU_OUTPUT_LINE_WIDTH	2560
@@ -291,29 +293,30 @@ static const struct dpu_lm_sub_blks sdm845_lm_sblk = {
 	},
 };
 
-#define LM_BLK(_name, _id, _base, _fmask, _sblk, _pp, _lmpair) \
+#define LM_BLK(_name, _id, _base, _fmask, _sblk, _pp, _lmpair, _dspp) \
 	{ \
 	.name = _name, .id = _id, \
 	.base = _base, .len = 0x320, \
 	.features = _fmask, \
 	.sblk = _sblk, \
 	.pingpong = _pp, \
-	.lm_pair_mask = (1 << _lmpair) \
+	.lm_pair_mask = (1 << _lmpair), \
+	.dspp = _dspp \
 	}
 
 static const struct dpu_lm_cfg sdm845_lm[] = {
 	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_0, LM_1),
+		&sdm845_lm_sblk, PINGPONG_0, LM_1, 0),
 	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_1, LM_0),
+		&sdm845_lm_sblk, PINGPONG_1, LM_0, 0),
 	LM_BLK("lm_2", LM_2, 0x46000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_2, LM_5),
+		&sdm845_lm_sblk, PINGPONG_2, LM_5, 0),
 	LM_BLK("lm_3", LM_3, 0x0, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_MAX, 0),
+		&sdm845_lm_sblk, PINGPONG_MAX, 0, 0),
 	LM_BLK("lm_4", LM_4, 0x0, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_MAX, 0),
+		&sdm845_lm_sblk, PINGPONG_MAX, 0, 0),
 	LM_BLK("lm_5", LM_5, 0x49000, MIXER_SDM845_MASK,
-		&sdm845_lm_sblk, PINGPONG_3, LM_2),
+		&sdm845_lm_sblk, PINGPONG_3, LM_2, 0),
 };
 
 /* SC7180 */
@@ -328,11 +331,30 @@ static const struct dpu_lm_sub_blks sc7180_lm_sblk = {
 
 static const struct dpu_lm_cfg sc7180_lm[] = {
 	LM_BLK("lm_0", LM_0, 0x44000, MIXER_SC7180_MASK,
-		&sc7180_lm_sblk, PINGPONG_0, LM_1),
+		&sc7180_lm_sblk, PINGPONG_0, LM_1, DSPP_0),
 	LM_BLK("lm_1", LM_1, 0x45000, MIXER_SC7180_MASK,
-		&sc7180_lm_sblk, PINGPONG_1, LM_0),
+		&sc7180_lm_sblk, PINGPONG_1, LM_0, 0),
+};
+
+/*************************************************************
+ * DSPP sub blocks config
+ *************************************************************/
+static const struct dpu_dspp_sub_blks sc7180_dspp_sblk = {
+	.pcc = {.id = DPU_DSPP_PCC, .base = 0x1700,
+		.len = 0x90, .version = 0x10000},
 };
 
+#define DSPP_BLK(_name, _id, _base) \
+		{\
+		.name = _name, .id = _id, \
+		.base = _base, .len = 0x1800, \
+		.features = DSPP_SC7180_MASK, \
+		.sblk = &sc7180_dspp_sblk \
+		}
+
+static const struct dpu_dspp_cfg sc7180_dspp[] = {
+	DSPP_BLK("dspp_0", DSPP_0, 0x54000),
+};
 /*************************************************************
  * PINGPONG sub blocks config
  *************************************************************/
@@ -515,8 +537,8 @@ static const struct dpu_perf_cfg sdm845_perf_data = {
 };
 
 static const struct dpu_perf_cfg sc7180_perf_data = {
-	.max_bw_low = 3900000,
-	.max_bw_high = 5500000,
+	.max_bw_low = 6800000,
+	.max_bw_high = 6800000,
 	.min_core_ib = 2400000,
 	.min_llcc_ib = 800000,
 	.min_dram_ib = 800000,
@@ -587,6 +609,8 @@ static void sc7180_cfg_init(struct dpu_mdss_cfg *dpu_cfg)
 		.sspp = sc7180_sspp,
 		.mixer_count = ARRAY_SIZE(sc7180_lm),
 		.mixer = sc7180_lm,
+		.dspp_count = ARRAY_SIZE(sc7180_dspp),
+		.dspp = sc7180_dspp,
 		.pingpong_count = ARRAY_SIZE(sc7180_pp),
 		.pingpong = sc7180_pp,
 		.intf_count = ARRAY_SIZE(sc7180_intf),
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
index 09df7d87dd43..f7de43838c69 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
@@ -146,6 +146,17 @@ enum {
 };
 
 /**
+ * DSPP sub-blocks
+ * @DPU_DSPP_PCC             Panel color correction block
+ * @DPU_DSPP_GC              Gamma correction block
+ */
+enum {
+	DPU_DSPP_PCC = 0x1,
+	DPU_DSPP_GC,
+	DPU_DSPP_MAX
+};
+
+/**
  * PINGPONG sub-blocks
  * @DPU_PINGPONG_TE         Tear check block
  * @DPU_PINGPONG_TE2        Additional tear check block for split pipes
@@ -377,6 +388,16 @@ struct dpu_lm_sub_blks {
 	struct dpu_pp_blk gc;
 };
 
+/**
+ * struct dpu_dspp_sub_blks: Information of DSPP block
+ * @gc : gamma correction block
+ * @pcc: pixel color correction block
+ */
+struct dpu_dspp_sub_blks {
+	struct dpu_pp_blk gc;
+	struct dpu_pp_blk pcc;
+};
+
 struct dpu_pingpong_sub_blks {
 	struct dpu_pp_blk te;
 	struct dpu_pp_blk te2;
@@ -471,10 +492,24 @@ struct dpu_lm_cfg {
 	DPU_HW_BLK_INFO;
 	const struct dpu_lm_sub_blks *sblk;
 	u32 pingpong;
+	u32 dspp;
 	unsigned long lm_pair_mask;
 };
 
 /**
+ * struct dpu_dspp_cfg - information of DSPP blocks
+ * @id                 enum identifying this block
+ * @base               register offset of this block
+ * @features           bit mask identifying sub-blocks/features
+ *                     supported by this block
+ * @sblk               sub-blocks information
+ */
+struct dpu_dspp_cfg  {
+	DPU_HW_BLK_INFO;
+	const struct dpu_dspp_sub_blks *sblk;
+};
+
+/**
  * struct dpu_pingpong_cfg - information of PING-PONG blocks
  * @id                 enum identifying this block
  * @base               register offset of this block
@@ -688,6 +723,9 @@ struct dpu_mdss_cfg {
 
 	u32 ad_count;
 
+	u32 dspp_count;
+	const struct dpu_dspp_cfg *dspp;
+
 	/* Add additional block data structures here */
 
 	struct dpu_perf_cfg perf;
@@ -716,6 +754,7 @@ struct dpu_mdss_hw_cfg_handler {
 #define BLK_PINGPONG(s) ((s)->pingpong)
 #define BLK_INTF(s) ((s)->intf)
 #define BLK_AD(s) ((s)->ad)
+#define BLK_DSPP(s) ((s)->dspp)
 
 /**
  * dpu_hw_catalog_init - dpu hardware catalog init API retrieves
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
index 831e5f7a9b7f..613ae8f0cfcd 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
@@ -272,6 +272,31 @@ static int dpu_hw_ctl_active_get_bitmask_intf(struct dpu_hw_ctl *ctx,
 	return 0;
 }
 
+static uint32_t dpu_hw_ctl_get_bitmask_dspp(struct dpu_hw_ctl *ctx,
+	enum dpu_dspp dspp)
+{
+	uint32_t flushbits = 0;
+
+	switch (dspp) {
+	case DSPP_0:
+		flushbits = BIT(13);
+		break;
+	case DSPP_1:
+		flushbits = BIT(14);
+		break;
+	case DSPP_2:
+		flushbits = BIT(15);
+		break;
+	case DSPP_3:
+		flushbits = BIT(21);
+		break;
+	default:
+		return 0;
+	}
+
+	return flushbits;
+}
+
 static u32 dpu_hw_ctl_poll_reset_status(struct dpu_hw_ctl *ctx, u32 timeout_us)
 {
 	struct dpu_hw_blk_reg_map *c = &ctx->hw;
@@ -548,6 +573,7 @@ static void _setup_ctl_ops(struct dpu_hw_ctl_ops *ops,
 	ops->setup_blendstage = dpu_hw_ctl_setup_blendstage;
 	ops->get_bitmask_sspp = dpu_hw_ctl_get_bitmask_sspp;
 	ops->get_bitmask_mixer = dpu_hw_ctl_get_bitmask_mixer;
+	ops->get_bitmask_dspp = dpu_hw_ctl_get_bitmask_dspp;
 };
 
 static struct dpu_hw_blk_ops dpu_hw_ops;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
index 09e1263c72e2..ec579b470a80 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
@@ -139,6 +139,9 @@ struct dpu_hw_ctl_ops {
 	uint32_t (*get_bitmask_mixer)(struct dpu_hw_ctl *ctx,
 		enum dpu_lm blk);
 
+	uint32_t (*get_bitmask_dspp)(struct dpu_hw_ctl *ctx,
+		enum dpu_dspp blk);
+
 	/**
 	 * Query the value of the intf flush mask
 	 * No effect on hardware
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c
new file mode 100644
index 000000000000..a7a24539921f
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.c
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ */
+
+#include "dpu_hwio.h"
+#include "dpu_hw_catalog.h"
+#include "dpu_hw_lm.h"
+#include "dpu_hw_dspp.h"
+#include "dpu_kms.h"
+
+
+/* DSPP_PCC */
+#define PCC_EN BIT(0)
+#define PCC_DIS 0
+#define PCC_RED_R_OFF 0x10
+#define PCC_RED_G_OFF 0x1C
+#define PCC_RED_B_OFF 0x28
+#define PCC_GREEN_R_OFF 0x14
+#define PCC_GREEN_G_OFF 0x20
+#define PCC_GREEN_B_OFF 0x2C
+#define PCC_BLUE_R_OFF 0x18
+#define PCC_BLUE_G_OFF 0x24
+#define PCC_BLUE_B_OFF 0x30
+
+static void dpu_setup_dspp_pcc(struct dpu_hw_dspp *ctx,
+		struct dpu_hw_pcc_cfg *cfg)
+{
+
+	u32 base = ctx->cap->sblk->pcc.base;
+
+	if (!ctx || !base) {
+		DRM_ERROR("invalid ctx %pK pcc base 0x%x\n", ctx, base);
+		return;
+	}
+
+	if (!cfg) {
+		DRM_DEBUG_DRIVER("disable pcc feature\n");
+		DPU_REG_WRITE(&ctx->hw, base, PCC_DIS);
+		return;
+	}
+
+	DPU_REG_WRITE(&ctx->hw, base + PCC_RED_R_OFF, cfg->r.r);
+	DPU_REG_WRITE(&ctx->hw, base + PCC_RED_G_OFF, cfg->r.g);
+	DPU_REG_WRITE(&ctx->hw, base + PCC_RED_B_OFF, cfg->r.b);
+
+	DPU_REG_WRITE(&ctx->hw, base + PCC_GREEN_R_OFF, cfg->g.r);
+	DPU_REG_WRITE(&ctx->hw, base + PCC_GREEN_G_OFF, cfg->g.g);
+	DPU_REG_WRITE(&ctx->hw, base + PCC_GREEN_B_OFF, cfg->g.b);
+
+	DPU_REG_WRITE(&ctx->hw, base + PCC_BLUE_R_OFF, cfg->b.r);
+	DPU_REG_WRITE(&ctx->hw, base + PCC_BLUE_G_OFF, cfg->b.g);
+	DPU_REG_WRITE(&ctx->hw, base + PCC_BLUE_B_OFF, cfg->b.b);
+
+	DPU_REG_WRITE(&ctx->hw, base, PCC_EN);
+}
+
+static void _setup_dspp_ops(struct dpu_hw_dspp *c,
+		unsigned long features)
+{
+	if (test_bit(DPU_DSPP_PCC, &features) &&
+		IS_SC7180_TARGET(c->hw.hwversion))
+		c->ops.setup_pcc = dpu_setup_dspp_pcc;
+}
+
+static const struct dpu_dspp_cfg *_dspp_offset(enum dpu_dspp dspp,
+		const struct dpu_mdss_cfg *m,
+		void __iomem *addr,
+		struct dpu_hw_blk_reg_map *b)
+{
+	int i;
+
+	if (!m || !addr || !b)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < m->dspp_count; i++) {
+		if (dspp == m->dspp[i].id) {
+			b->base_off = addr;
+			b->blk_off = m->dspp[i].base;
+			b->length = m->dspp[i].len;
+			b->hwversion = m->hwversion;
+			b->log_mask = DPU_DBG_MASK_DSPP;
+			return &m->dspp[i];
+		}
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static struct dpu_hw_blk_ops dpu_hw_ops;
+
+struct dpu_hw_dspp *dpu_hw_dspp_init(enum dpu_dspp idx,
+			void __iomem *addr,
+			const struct dpu_mdss_cfg *m)
+{
+	struct dpu_hw_dspp *c;
+	const struct dpu_dspp_cfg *cfg;
+
+	if (!addr || !m)
+		return ERR_PTR(-EINVAL);
+
+	c = kzalloc(sizeof(*c), GFP_KERNEL);
+	if (!c)
+		return ERR_PTR(-ENOMEM);
+
+	cfg = _dspp_offset(idx, m, addr, &c->hw);
+	if (IS_ERR_OR_NULL(cfg)) {
+		kfree(c);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* Assign ops */
+	c->idx = idx;
+	c->cap = cfg;
+	_setup_dspp_ops(c, c->cap->features);
+
+	dpu_hw_blk_init(&c->base, DPU_HW_BLK_DSPP, idx, &dpu_hw_ops);
+
+	return c;
+}
+
+void dpu_hw_dspp_destroy(struct dpu_hw_dspp *dspp)
+{
+	if (dspp)
+		dpu_hw_blk_destroy(&dspp->base);
+
+	kfree(dspp);
+}
+
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h
new file mode 100644
index 000000000000..7fa189cfcb06
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dspp.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DPU_HW_DSPP_H
+#define _DPU_HW_DSPP_H
+
+#include "dpu_hw_blk.h"
+
+struct dpu_hw_dspp;
+
+/**
+ * struct dpu_hw_pcc_coeff - PCC coefficient structure for each color
+ *                            component.
+ * @r: red coefficient.
+ * @g: green coefficient.
+ * @b: blue coefficient.
+ */
+
+struct dpu_hw_pcc_coeff {
+	__u32 r;
+	__u32 g;
+	__u32 b;
+};
+
+/**
+ * struct dpu_hw_pcc - pcc feature structure
+ * @r: red coefficients.
+ * @g: green coefficients.
+ * @b: blue coefficients.
+ */
+struct dpu_hw_pcc_cfg {
+	struct dpu_hw_pcc_coeff r;
+	struct dpu_hw_pcc_coeff g;
+	struct dpu_hw_pcc_coeff b;
+};
+
+/**
+ * struct dpu_hw_dspp_ops - interface to the dspp hardware driver functions
+ * Caller must call the init function to get the dspp context for each dspp
+ * Assumption is these functions will be called after clocks are enabled
+ */
+struct dpu_hw_dspp_ops {
+	/**
+	 * setup_pcc - setup dspp pcc
+	 * @ctx: Pointer to dspp context
+	 * @cfg: Pointer to configuration
+	 */
+	void (*setup_pcc)(struct dpu_hw_dspp *ctx, struct dpu_hw_pcc_cfg *cfg);
+
+};
+
+/**
+ * struct dpu_hw_dspp - dspp description
+ * @base: Hardware block base structure
+ * @hw: Block hardware details
+ * @idx: DSPP index
+ * @cap: Pointer to layer_cfg
+ * @ops: Pointer to operations possible for this DSPP
+ */
+struct dpu_hw_dspp {
+	struct dpu_hw_blk base;
+	struct dpu_hw_blk_reg_map hw;
+
+	/* dspp */
+	int idx;
+	const struct dpu_dspp_cfg *cap;
+
+	/* Ops */
+	struct dpu_hw_dspp_ops ops;
+};
+
+/**
+ * dpu_hw_dspp - convert base object dpu_hw_base to container
+ * @hw: Pointer to base hardware block
+ * return: Pointer to hardware block container
+ */
+static inline struct dpu_hw_dspp *to_dpu_hw_dspp(struct dpu_hw_blk *hw)
+{
+	return container_of(hw, struct dpu_hw_dspp, base);
+}
+
+/**
+ * dpu_hw_dspp_init - initializes the dspp hw driver object.
+ * should be called once before accessing every dspp.
+ * @idx:  DSPP index for which driver object is required
+ * @addr: Mapped register io address of MDP
+ * @Return: pointer to structure or ERR_PTR
+ */
+struct dpu_hw_dspp *dpu_hw_dspp_init(enum dpu_dspp idx,
+	void __iomem *addr, const struct dpu_mdss_cfg *m);
+
+/**
+ * dpu_hw_dspp_destroy(): Destroys DSPP driver context
+ * @dspp: Pointer to DSPP driver context
+ */
+void dpu_hw_dspp_destroy(struct dpu_hw_dspp *dspp);
+
+#endif /*_DPU_HW_DSPP_H */
+
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
index 686882132bf6..402dc5832361 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
@@ -95,6 +95,7 @@ enum dpu_hw_blk_type {
 	DPU_HW_BLK_PINGPONG,
 	DPU_HW_BLK_INTF,
 	DPU_HW_BLK_WB,
+	DPU_HW_BLK_DSPP,
 	DPU_HW_BLK_MAX,
 };
 
@@ -425,5 +426,6 @@ struct dpu_mdss_color {
 #define DPU_DBG_MASK_TOP      (1 << 7)
 #define DPU_DBG_MASK_VBIF     (1 << 8)
 #define DPU_DBG_MASK_ROT      (1 << 9)
+#define DPU_DBG_MASK_DSPP     (1 << 10)
 
 #endif  /* _DPU_HW_MDSS_H */
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
index ce19f1d39367..b8615d4fe8a3 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
@@ -772,29 +772,21 @@ static int _dpu_kms_mmu_init(struct dpu_kms *dpu_kms)
 {
 	struct iommu_domain *domain;
 	struct msm_gem_address_space *aspace;
-	int ret;
+	struct msm_mmu *mmu;
 
 	domain = iommu_domain_alloc(&platform_bus_type);
 	if (!domain)
 		return 0;
 
-	domain->geometry.aperture_start = 0x1000;
-	domain->geometry.aperture_end = 0xffffffff;
+	mmu = msm_iommu_new(dpu_kms->dev->dev, domain);
+	aspace = msm_gem_address_space_create(mmu, "dpu1",
+		0x1000, 0xfffffff);
 
-	aspace = msm_gem_address_space_create(dpu_kms->dev->dev,
-			domain, "dpu1");
 	if (IS_ERR(aspace)) {
-		iommu_domain_free(domain);
+		mmu->funcs->destroy(mmu);
 		return PTR_ERR(aspace);
 	}
 
-	ret = aspace->mmu->funcs->attach(aspace->mmu);
-	if (ret) {
-		DPU_ERROR("failed to attach iommu %d\n", ret);
-		msm_gem_address_space_put(aspace);
-		return ret;
-	}
-
 	dpu_kms->base.aspace = aspace;
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
index 9aba2910d83a..a3b122bfb676 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
@@ -158,6 +158,7 @@ struct dpu_global_state {
 	uint32_t mixer_to_enc_id[LM_MAX - LM_0];
 	uint32_t ctl_to_enc_id[CTL_MAX - CTL_0];
 	uint32_t intf_to_enc_id[INTF_MAX - INTF_0];
+	uint32_t dspp_to_enc_id[DSPP_MAX - DSPP_0];
 };
 
 struct dpu_global_state
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
index 9b62451b01ee..9b2b5044e8e0 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.c
@@ -9,6 +9,7 @@
 #include "dpu_hw_ctl.h"
 #include "dpu_hw_pingpong.h"
 #include "dpu_hw_intf.h"
+#include "dpu_hw_dspp.h"
 #include "dpu_encoder.h"
 #include "dpu_trace.h"
 
@@ -174,6 +175,23 @@ int dpu_rm_init(struct dpu_rm *rm,
 		rm->ctl_blks[ctl->id - CTL_0] = &hw->base;
 	}
 
+	for (i = 0; i < cat->dspp_count; i++) {
+		struct dpu_hw_dspp *hw;
+		const struct dpu_dspp_cfg *dspp = &cat->dspp[i];
+
+		if (dspp->id < DSPP_0 || dspp->id >= DSPP_MAX) {
+			DPU_ERROR("skip dspp %d with invalid id\n", dspp->id);
+			continue;
+		}
+		hw = dpu_hw_dspp_init(dspp->id, mmio, cat);
+		if (IS_ERR_OR_NULL(hw)) {
+			rc = PTR_ERR(hw);
+			DPU_ERROR("failed dspp object creation: err %d\n", rc);
+			goto fail;
+		}
+		rm->dspp_blks[dspp->id - DSPP_0] = &hw->base;
+	}
+
 	return 0;
 
 fail:
@@ -222,12 +240,17 @@ static bool _dpu_rm_check_lm_peer(struct dpu_rm *rm, int primary_idx,
  *      if lm, and all other hardwired blocks connected to the lm (pp) is
  *      available and appropriate
  * @pp_idx: output parameter, index of pingpong block attached to the layer
- *      mixer in rm->pongpong_blks[].
+ *      mixer in rm->pingpong_blks[].
+ * @dspp_idx: output parameter, index of dspp block attached to the layer
+ *      mixer in rm->dspp_blks[].
+ * @reqs: input parameter, rm requirements for HW blocks needed in the
+ *      datapath.
  * @Return: true if lm matches all requirements, false otherwise
  */
 static bool _dpu_rm_check_lm_and_get_connected_blks(struct dpu_rm *rm,
 		struct dpu_global_state *global_state,
-		uint32_t enc_id, int lm_idx, int *pp_idx)
+		uint32_t enc_id, int lm_idx, int *pp_idx, int *dspp_idx,
+		struct dpu_rm_requirements *reqs)
 {
 	const struct dpu_lm_cfg *lm_cfg;
 	int idx;
@@ -251,6 +274,23 @@ static bool _dpu_rm_check_lm_and_get_connected_blks(struct dpu_rm *rm,
 		return false;
 	}
 	*pp_idx = idx;
+
+	if (!reqs->topology.num_dspp)
+		return true;
+
+	idx = lm_cfg->dspp - DSPP_0;
+	if (idx < 0 || idx >= ARRAY_SIZE(rm->dspp_blks)) {
+		DPU_ERROR("failed to get dspp on lm %d\n", lm_cfg->dspp);
+		return false;
+	}
+
+	if (reserved_by_other(global_state->dspp_to_enc_id, idx, enc_id)) {
+		DPU_DEBUG("lm %d dspp %d already reserved\n", lm_cfg->id,
+				lm_cfg->dspp);
+		return false;
+	}
+	*dspp_idx = idx;
+
 	return true;
 }
 
@@ -262,6 +302,7 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm,
 {
 	int lm_idx[MAX_BLOCKS];
 	int pp_idx[MAX_BLOCKS];
+	int dspp_idx[MAX_BLOCKS] = {0};
 	int i, j, lm_count = 0;
 
 	if (!reqs->topology.num_lm) {
@@ -279,7 +320,8 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm,
 		lm_idx[lm_count] = i;
 
 		if (!_dpu_rm_check_lm_and_get_connected_blks(rm, global_state,
-				enc_id, i, &pp_idx[lm_count])) {
+				enc_id, i, &pp_idx[lm_count],
+				&dspp_idx[lm_count], reqs)) {
 			continue;
 		}
 
@@ -299,7 +341,8 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm,
 
 			if (!_dpu_rm_check_lm_and_get_connected_blks(rm,
 					global_state, enc_id, j,
-					&pp_idx[lm_count])) {
+					&pp_idx[lm_count], &dspp_idx[lm_count],
+					reqs)) {
 				continue;
 			}
 
@@ -316,6 +359,8 @@ static int _dpu_rm_reserve_lms(struct dpu_rm *rm,
 	for (i = 0; i < lm_count; i++) {
 		global_state->mixer_to_enc_id[lm_idx[i]] = enc_id;
 		global_state->pingpong_to_enc_id[pp_idx[i]] = enc_id;
+		global_state->dspp_to_enc_id[dspp_idx[i]] =
+			reqs->topology.num_dspp ? enc_id : 0;
 
 		trace_dpu_rm_reserve_lms(lm_idx[i] + LM_0, enc_id,
 					 pp_idx[i] + PINGPONG_0);
@@ -560,6 +605,11 @@ int dpu_rm_get_assigned_resources(struct dpu_rm *rm,
 		hw_to_enc_id = global_state->intf_to_enc_id;
 		max_blks = ARRAY_SIZE(rm->intf_blks);
 		break;
+	case DPU_HW_BLK_DSPP:
+		hw_blks = rm->dspp_blks;
+		hw_to_enc_id = global_state->dspp_to_enc_id;
+		max_blks = ARRAY_SIZE(rm->dspp_blks);
+		break;
 	default:
 		DPU_ERROR("blk type %d not managed by rm\n", type);
 		return 0;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
index 6d2b04f306f0..08726bb1063a 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_rm.h
@@ -19,6 +19,7 @@ struct dpu_global_state;
  * @mixer_blks: array of layer mixer hardware resources
  * @ctl_blks: array of ctl hardware resources
  * @intf_blks: array of intf hardware resources
+ * @dspp_blks: array of dspp hardware resources
  * @lm_max_width: cached layer mixer maximum width
  * @rm_lock: resource manager mutex
  */
@@ -27,6 +28,7 @@ struct dpu_rm {
 	struct dpu_hw_blk *mixer_blks[LM_MAX - LM_0];
 	struct dpu_hw_blk *ctl_blks[CTL_MAX - CTL_0];
 	struct dpu_hw_blk *intf_blks[INTF_MAX - INTF_0];
+	struct dpu_hw_blk *dspp_blks[DSPP_MAX - DSPP_0];
 
 	uint32_t lm_max_width;
 };
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
index dda05436f716..08897184b1d9 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
@@ -510,18 +510,20 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev)
 	mdelay(16);
 
 	if (config->iommu) {
-		aspace = msm_gem_address_space_create(&pdev->dev,
-				config->iommu, "mdp4");
+		struct msm_mmu *mmu = msm_iommu_new(&pdev->dev,
+			config->iommu);
+
+		aspace  = msm_gem_address_space_create(mmu,
+			"mdp4", 0x1000, 0xffffffff);
+
 		if (IS_ERR(aspace)) {
+			if (!IS_ERR(mmu))
+				mmu->funcs->destroy(mmu);
 			ret = PTR_ERR(aspace);
 			goto fail;
 		}
 
 		kms->aspace = aspace;
-
-		ret = aspace->mmu->funcs->attach(aspace->mmu);
-		if (ret)
-			goto fail;
 	} else {
 		DRM_DEV_INFO(dev->dev, "no iommu, fallback to phys "
 				"contig buffers for scanout\n");
@@ -569,10 +571,6 @@ static struct mdp4_platform_config *mdp4_get_config(struct platform_device *dev)
 	/* TODO: Chips that aren't apq8064 have a 200 Mhz max_clk */
 	config.max_clk = 266667000;
 	config.iommu = iommu_domain_alloc(&platform_bus_type);
-	if (config.iommu) {
-		config.iommu->geometry.aperture_start = 0x1000;
-		config.iommu->geometry.aperture_end = 0xffffffff;
-	}
 
 	return &config;
 }
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c
index e3c4c250238b..25a13a2a57a9 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cfg.c
@@ -342,6 +342,81 @@ static const struct mdp5_cfg_hw msm8x16_config = {
 	.max_clk = 320000000,
 };
 
+static const struct mdp5_cfg_hw msm8x36_config = {
+	.name = "msm8x36",
+	.mdp = {
+		.count = 1,
+		.base = { 0x0 },
+		.caps = MDP_CAP_SMP |
+			0,
+	},
+	.smp = {
+		.mmb_count = 8,
+		.mmb_size = 10240,
+		.clients = {
+			[SSPP_VIG0] = 1, [SSPP_DMA0] = 4,
+			[SSPP_RGB0] = 7, [SSPP_RGB1] = 8,
+		},
+	},
+	.ctl = {
+		.count = 3,
+		.base = { 0x01000, 0x01200, 0x01400 },
+		.flush_hw_mask = 0x4003ffff,
+	},
+	.pipe_vig = {
+		.count = 1,
+		.base = { 0x04000 },
+		.caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
+				MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC |
+				MDP_PIPE_CAP_DECIMATION,
+	},
+	.pipe_rgb = {
+		.count = 2,
+		.base = { 0x14000, 0x16000 },
+		.caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
+				MDP_PIPE_CAP_DECIMATION,
+	},
+	.pipe_dma = {
+		.count = 1,
+		.base = { 0x24000 },
+		.caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP,
+	},
+	.lm = {
+		.count = 2,
+		.base = { 0x44000, 0x47000 },
+		.instances = {
+				{ .id = 0, .pp = 0, .dspp = 0,
+				  .caps = MDP_LM_CAP_DISPLAY, },
+				{ .id = 1, .pp = -1, .dspp = -1,
+				  .caps = MDP_LM_CAP_WB, },
+				},
+		.nb_stages = 8,
+		.max_width = 2560,
+		.max_height = 0xFFFF,
+	},
+	.pp = {
+		.count = 1,
+		.base = { 0x70000 },
+	},
+	.ad = {
+		.count = 1,
+		.base = { 0x78000 },
+	},
+	.dspp = {
+		.count = 1,
+		.base = { 0x54000 },
+	},
+	.intf = {
+		.base = { 0x00000, 0x6a800, 0x6b000 },
+		.connect = {
+			[0] = INTF_DISABLED,
+			[1] = INTF_DSI,
+			[2] = INTF_DSI,
+		},
+	},
+	.max_clk = 366670000,
+};
+
 static const struct mdp5_cfg_hw msm8x94_config = {
 	.name = "msm8x94",
 	.mdp = {
@@ -840,6 +915,7 @@ static const struct mdp5_cfg_handler cfg_handlers_v1[] = {
 	{ .revision = 2, .config = { .hw = &msm8x74v2_config } },
 	{ .revision = 3, .config = { .hw = &apq8084_config } },
 	{ .revision = 6, .config = { .hw = &msm8x16_config } },
+	{ .revision = 8, .config = { .hw = &msm8x36_config } },
 	{ .revision = 9, .config = { .hw = &msm8x94_config } },
 	{ .revision = 7, .config = { .hw = &msm8x96_config } },
 	{ .revision = 11, .config = { .hw = &msm8x76_config } },
@@ -941,10 +1017,6 @@ static struct mdp5_cfg_platform *mdp5_get_config(struct platform_device *dev)
 	static struct mdp5_cfg_platform config = {};
 
 	config.iommu = iommu_domain_alloc(&platform_bus_type);
-	if (config.iommu) {
-		config.iommu->geometry.aperture_start = 0x1000;
-		config.iommu->geometry.aperture_end = 0xffffffff;
-	}
 
 	return &config;
 }
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
index 998bef1190a3..b5fed67c4651 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
@@ -959,7 +959,7 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
 	if (!ctl)
 		return -EINVAL;
 
-	/* don't support LM cursors when we we have source split enabled */
+	/* don't support LM cursors when we have source split enabled */
 	if (mdp5_cstate->pipeline.r_mixer)
 		return -EINVAL;
 
@@ -1030,7 +1030,7 @@ static int mdp5_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
 		return -EINVAL;
 	}
 
-	/* don't support LM cursors when we we have source split enabled */
+	/* don't support LM cursors when we have source split enabled */
 	if (mdp5_cstate->pipeline.r_mixer)
 		return -EINVAL;
 
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
index c902c6503675..19ec48695ffb 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
@@ -624,25 +624,25 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 	mdelay(16);
 
 	if (config->platform.iommu) {
+		struct msm_mmu *mmu;
+
 		iommu_dev = &pdev->dev;
 		if (!dev_iommu_fwspec_get(iommu_dev))
 			iommu_dev = iommu_dev->parent;
 
-		aspace = msm_gem_address_space_create(iommu_dev,
-				config->platform.iommu, "mdp5");
+		mmu = msm_iommu_new(iommu_dev, config->platform.iommu);
+
+		aspace = msm_gem_address_space_create(mmu, "mdp5",
+			0x1000, 0xffffffff);
+
 		if (IS_ERR(aspace)) {
+			if (!IS_ERR(mmu))
+				mmu->funcs->destroy(mmu);
 			ret = PTR_ERR(aspace);
 			goto fail;
 		}
 
 		kms->aspace = aspace;
-
-		ret = aspace->mmu->funcs->attach(aspace->mmu);
-		if (ret) {
-			DRM_DEV_ERROR(&pdev->dev, "failed to attach iommu: %d\n",
-				ret);
-			goto fail;
-		}
 	} else {
 		DRM_DEV_INFO(&pdev->dev,
 			 "no iommu, fallback to phys contig buffers for scanout\n");
@@ -935,7 +935,8 @@ static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
 
 	return 0;
 fail:
-	mdp5_destroy(pdev);
+	if (mdp5_kms)
+		mdp5_destroy(pdev);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 29295dee2a2e..f6ce40bf3699 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -37,9 +37,10 @@
  * - 1.4.0 - softpin, MSM_RELOC_BO_DUMP, and GEM_INFO support to set/get
  *           GEM object's debug name
  * - 1.5.0 - Add SUBMITQUERY_QUERY ioctl
+ * - 1.6.0 - Syncobj support
  */
 #define MSM_VERSION_MAJOR	1
-#define MSM_VERSION_MINOR	5
+#define MSM_VERSION_MINOR	6
 #define MSM_VERSION_PATCHLEVEL	0
 
 static const struct drm_mode_config_funcs mode_config_funcs = {
@@ -1002,7 +1003,8 @@ static struct drm_driver msm_driver = {
 	.driver_features    = DRIVER_GEM |
 				DRIVER_RENDER |
 				DRIVER_ATOMIC |
-				DRIVER_MODESET,
+				DRIVER_MODESET |
+				DRIVER_SYNCOBJ,
 	.open               = msm_open,
 	.postclose           = msm_postclose,
 	.lastclose          = drm_fb_helper_lastclose,
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 194d900a460e..e2d6a6056418 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -105,6 +105,7 @@ struct msm_display_topology {
 	u32 num_lm;
 	u32 num_enc;
 	u32 num_intf;
+	u32 num_dspp;
 };
 
 /**
@@ -236,7 +237,8 @@ int msm_crtc_enable_vblank(struct drm_crtc *crtc);
 void msm_crtc_disable_vblank(struct drm_crtc *crtc);
 
 int msm_gem_init_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, int npages);
+		struct msm_gem_vma *vma, int npages,
+		u64 range_start, u64 range_end);
 void msm_gem_purge_vma(struct msm_gem_address_space *aspace,
 		struct msm_gem_vma *vma);
 void msm_gem_unmap_vma(struct msm_gem_address_space *aspace,
@@ -250,12 +252,8 @@ void msm_gem_close_vma(struct msm_gem_address_space *aspace,
 void msm_gem_address_space_put(struct msm_gem_address_space *aspace);
 
 struct msm_gem_address_space *
-msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
-		const char *name);
-
-struct msm_gem_address_space *
-msm_gem_address_space_create_a2xx(struct device *dev, struct msm_gpu *gpu,
-		const char *name, uint64_t va_start, uint64_t va_end);
+msm_gem_address_space_create(struct msm_mmu *mmu, const char *name,
+		u64 va_start, u64 size);
 
 int msm_register_mmu(struct drm_device *dev, struct msm_mmu *mmu);
 void msm_unregister_mmu(struct drm_device *dev, struct msm_mmu *mmu);
@@ -276,6 +274,9 @@ vm_fault_t msm_gem_fault(struct vm_fault *vmf);
 uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj);
 int msm_gem_get_iova(struct drm_gem_object *obj,
 		struct msm_gem_address_space *aspace, uint64_t *iova);
+int msm_gem_get_and_pin_iova_range(struct drm_gem_object *obj,
+		struct msm_gem_address_space *aspace, uint64_t *iova,
+		u64 range_start, u64 range_end);
 int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
 		struct msm_gem_address_space *aspace, uint64_t *iova);
 uint64_t msm_gem_iova(struct drm_gem_object *obj,
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 5a6a79fbc9d6..6277fde13df9 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -389,7 +389,8 @@ put_iova(struct drm_gem_object *obj)
 }
 
 static int msm_gem_get_iova_locked(struct drm_gem_object *obj,
-		struct msm_gem_address_space *aspace, uint64_t *iova)
+		struct msm_gem_address_space *aspace, uint64_t *iova,
+		u64 range_start, u64 range_end)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct msm_gem_vma *vma;
@@ -404,7 +405,8 @@ static int msm_gem_get_iova_locked(struct drm_gem_object *obj,
 		if (IS_ERR(vma))
 			return PTR_ERR(vma);
 
-		ret = msm_gem_init_vma(aspace, vma, obj->size >> PAGE_SHIFT);
+		ret = msm_gem_init_vma(aspace, vma, obj->size >> PAGE_SHIFT,
+			range_start, range_end);
 		if (ret) {
 			del_vma(vma);
 			return ret;
@@ -426,6 +428,9 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj,
 	if (!(msm_obj->flags & MSM_BO_GPU_READONLY))
 		prot |= IOMMU_WRITE;
 
+	if (msm_obj->flags & MSM_BO_MAP_PRIV)
+		prot |= IOMMU_PRIV;
+
 	WARN_ON(!mutex_is_locked(&msm_obj->lock));
 
 	if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED))
@@ -443,9 +448,13 @@ static int msm_gem_pin_iova(struct drm_gem_object *obj,
 			msm_obj->sgt, obj->size >> PAGE_SHIFT);
 }
 
-/* get iova and pin it. Should have a matching put */
-int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
-		struct msm_gem_address_space *aspace, uint64_t *iova)
+/*
+ * get iova and pin it. Should have a matching put
+ * limits iova to specified range (in pages)
+ */
+int msm_gem_get_and_pin_iova_range(struct drm_gem_object *obj,
+		struct msm_gem_address_space *aspace, uint64_t *iova,
+		u64 range_start, u64 range_end)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	u64 local;
@@ -453,7 +462,8 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
 
 	mutex_lock(&msm_obj->lock);
 
-	ret = msm_gem_get_iova_locked(obj, aspace, &local);
+	ret = msm_gem_get_iova_locked(obj, aspace, &local,
+		range_start, range_end);
 
 	if (!ret)
 		ret = msm_gem_pin_iova(obj, aspace);
@@ -465,6 +475,13 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
 	return ret;
 }
 
+/* get iova and pin it. Should have a matching put */
+int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
+		struct msm_gem_address_space *aspace, uint64_t *iova)
+{
+	return msm_gem_get_and_pin_iova_range(obj, aspace, iova, 0, U64_MAX);
+}
+
 /*
  * Get an iova but don't pin it. Doesn't need a put because iovas are currently
  * valid for the life of the object
@@ -476,7 +493,7 @@ int msm_gem_get_iova(struct drm_gem_object *obj,
 	int ret;
 
 	mutex_lock(&msm_obj->lock);
-	ret = msm_gem_get_iova_locked(obj, aspace, iova);
+	ret = msm_gem_get_iova_locked(obj, aspace, iova, 0, U64_MAX);
 	mutex_unlock(&msm_obj->lock);
 
 	return ret;
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index 30584eaf8cc8..972490b14ba5 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -13,6 +13,7 @@
 
 /* Additional internal-use only BO flags: */
 #define MSM_BO_STOLEN        0x10000000    /* try to use stolen/splash memory */
+#define MSM_BO_MAP_PRIV      0x20000000    /* use IOMMU_PRIV when mapping */
 
 struct msm_gem_address_space {
 	const char *name;
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 385d4965a8d0..6630aa817505 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -8,7 +8,9 @@
 #include <linux/sync_file.h>
 #include <linux/uaccess.h>
 
+#include <drm/drm_drv.h>
 #include <drm/drm_file.h>
+#include <drm/drm_syncobj.h>
 
 #include "msm_drv.h"
 #include "msm_gpu.h"
@@ -391,6 +393,186 @@ static void submit_cleanup(struct msm_gem_submit *submit)
 	}
 }
 
+
+struct msm_submit_post_dep {
+	struct drm_syncobj *syncobj;
+	uint64_t point;
+	struct dma_fence_chain *chain;
+};
+
+static struct drm_syncobj **msm_wait_deps(struct drm_device *dev,
+                                          struct drm_file *file,
+                                          uint64_t in_syncobjs_addr,
+                                          uint32_t nr_in_syncobjs,
+                                          size_t syncobj_stride,
+                                          struct msm_ringbuffer *ring)
+{
+	struct drm_syncobj **syncobjs = NULL;
+	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
+	int ret = 0;
+	uint32_t i, j;
+
+	syncobjs = kcalloc(nr_in_syncobjs, sizeof(*syncobjs),
+	                   GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
+	if (!syncobjs)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < nr_in_syncobjs; ++i) {
+		uint64_t address = in_syncobjs_addr + i * syncobj_stride;
+		struct dma_fence *fence;
+
+		if (copy_from_user(&syncobj_desc,
+			           u64_to_user_ptr(address),
+			           min(syncobj_stride, sizeof(syncobj_desc)))) {
+			ret = -EFAULT;
+			break;
+		}
+
+		if (syncobj_desc.point &&
+		    !drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) {
+			ret = -EOPNOTSUPP;
+			break;
+		}
+
+		if (syncobj_desc.flags & ~MSM_SUBMIT_SYNCOBJ_FLAGS) {
+			ret = -EINVAL;
+			break;
+		}
+
+		ret = drm_syncobj_find_fence(file, syncobj_desc.handle,
+		                             syncobj_desc.point, 0, &fence);
+		if (ret)
+			break;
+
+		if (!dma_fence_match_context(fence, ring->fctx->context))
+			ret = dma_fence_wait(fence, true);
+
+		dma_fence_put(fence);
+		if (ret)
+			break;
+
+		if (syncobj_desc.flags & MSM_SUBMIT_SYNCOBJ_RESET) {
+			syncobjs[i] =
+				drm_syncobj_find(file, syncobj_desc.handle);
+			if (!syncobjs[i]) {
+				ret = -EINVAL;
+				break;
+			}
+		}
+	}
+
+	if (ret) {
+		for (j = 0; j <= i; ++j) {
+			if (syncobjs[j])
+				drm_syncobj_put(syncobjs[j]);
+		}
+		kfree(syncobjs);
+		return ERR_PTR(ret);
+	}
+	return syncobjs;
+}
+
+static void msm_reset_syncobjs(struct drm_syncobj **syncobjs,
+                               uint32_t nr_syncobjs)
+{
+	uint32_t i;
+
+	for (i = 0; syncobjs && i < nr_syncobjs; ++i) {
+		if (syncobjs[i])
+			drm_syncobj_replace_fence(syncobjs[i], NULL);
+	}
+}
+
+static struct msm_submit_post_dep *msm_parse_post_deps(struct drm_device *dev,
+                                                       struct drm_file *file,
+                                                       uint64_t syncobjs_addr,
+                                                       uint32_t nr_syncobjs,
+                                                       size_t syncobj_stride)
+{
+	struct msm_submit_post_dep *post_deps;
+	struct drm_msm_gem_submit_syncobj syncobj_desc = {0};
+	int ret = 0;
+	uint32_t i, j;
+
+	post_deps = kmalloc_array(nr_syncobjs, sizeof(*post_deps),
+	                          GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
+	if (!post_deps)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < nr_syncobjs; ++i) {
+		uint64_t address = syncobjs_addr + i * syncobj_stride;
+
+		if (copy_from_user(&syncobj_desc,
+			           u64_to_user_ptr(address),
+			           min(syncobj_stride, sizeof(syncobj_desc)))) {
+			ret = -EFAULT;
+			break;
+		}
+
+		post_deps[i].point = syncobj_desc.point;
+		post_deps[i].chain = NULL;
+
+		if (syncobj_desc.flags) {
+			ret = -EINVAL;
+			break;
+		}
+
+		if (syncobj_desc.point) {
+			if (!drm_core_check_feature(dev,
+			                            DRIVER_SYNCOBJ_TIMELINE)) {
+				ret = -EOPNOTSUPP;
+				break;
+			}
+
+			post_deps[i].chain =
+				kmalloc(sizeof(*post_deps[i].chain),
+				        GFP_KERNEL);
+			if (!post_deps[i].chain) {
+				ret = -ENOMEM;
+				break;
+			}
+		}
+
+		post_deps[i].syncobj =
+			drm_syncobj_find(file, syncobj_desc.handle);
+		if (!post_deps[i].syncobj) {
+			ret = -EINVAL;
+			break;
+		}
+	}
+
+	if (ret) {
+		for (j = 0; j <= i; ++j) {
+			kfree(post_deps[j].chain);
+			if (post_deps[j].syncobj)
+				drm_syncobj_put(post_deps[j].syncobj);
+		}
+
+		kfree(post_deps);
+		return ERR_PTR(ret);
+	}
+
+	return post_deps;
+}
+
+static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
+                                  uint32_t count, struct dma_fence *fence)
+{
+	uint32_t i;
+
+	for (i = 0; post_deps && i < count; ++i) {
+		if (post_deps[i].chain) {
+			drm_syncobj_add_point(post_deps[i].syncobj,
+			                      post_deps[i].chain,
+			                      fence, post_deps[i].point);
+			post_deps[i].chain = NULL;
+		} else {
+			drm_syncobj_replace_fence(post_deps[i].syncobj,
+			                          fence);
+		}
+	}
+}
+
 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		struct drm_file *file)
 {
@@ -403,6 +585,8 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	struct sync_file *sync_file = NULL;
 	struct msm_gpu_submitqueue *queue;
 	struct msm_ringbuffer *ring;
+	struct msm_submit_post_dep *post_deps = NULL;
+	struct drm_syncobj **syncobjs_to_reset = NULL;
 	int out_fence_fd = -1;
 	struct pid *pid = get_pid(task_pid(current));
 	bool has_ww_ticket = false;
@@ -411,6 +595,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	if (!gpu)
 		return -ENXIO;
 
+	if (args->pad)
+		return -EINVAL;
+
 	/* for now, we just have 3d pipe.. eventually this would need to
 	 * be more clever to dispatch to appropriate gpu module:
 	 */
@@ -458,9 +645,29 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 			return ret;
 	}
 
+	if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
+		syncobjs_to_reset = msm_wait_deps(dev, file,
+		                                  args->in_syncobjs,
+		                                  args->nr_in_syncobjs,
+		                                  args->syncobj_stride, ring);
+		if (IS_ERR(syncobjs_to_reset))
+			return PTR_ERR(syncobjs_to_reset);
+	}
+
+	if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
+		post_deps = msm_parse_post_deps(dev, file,
+		                                args->out_syncobjs,
+		                                args->nr_out_syncobjs,
+		                                args->syncobj_stride);
+		if (IS_ERR(post_deps)) {
+			ret = PTR_ERR(post_deps);
+			goto out_post_unlock;
+		}
+	}
+
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
-		return ret;
+		goto out_post_unlock;
 
 	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
@@ -587,6 +794,11 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		args->fence_fd = out_fence_fd;
 	}
 
+	msm_reset_syncobjs(syncobjs_to_reset, args->nr_in_syncobjs);
+	msm_process_post_deps(post_deps, args->nr_out_syncobjs,
+	                      submit->fence);
+
+
 out:
 	submit_cleanup(submit);
 	if (has_ww_ticket)
@@ -597,5 +809,23 @@ out_unlock:
 	if (ret && (out_fence_fd >= 0))
 		put_unused_fd(out_fence_fd);
 	mutex_unlock(&dev->struct_mutex);
+
+out_post_unlock:
+	if (!IS_ERR_OR_NULL(post_deps)) {
+		for (i = 0; i < args->nr_out_syncobjs; ++i) {
+			kfree(post_deps[i].chain);
+			drm_syncobj_put(post_deps[i].syncobj);
+		}
+		kfree(post_deps);
+	}
+
+	if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
+		for (i = 0; i < args->nr_in_syncobjs; ++i) {
+			if (syncobjs_to_reset[i])
+				drm_syncobj_put(syncobjs_to_reset[i]);
+		}
+		kfree(syncobjs_to_reset);
+	}
+
 	return ret;
 }
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
index 1af5354bcd46..5f6a11211b64 100644
--- a/drivers/gpu/drm/msm/msm_gem_vma.c
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -103,7 +103,8 @@ void msm_gem_close_vma(struct msm_gem_address_space *aspace,
 
 /* Initialize a new vma and allocate an iova for it */
 int msm_gem_init_vma(struct msm_gem_address_space *aspace,
-		struct msm_gem_vma *vma, int npages)
+		struct msm_gem_vma *vma, int npages,
+		u64 range_start, u64 range_end)
 {
 	int ret;
 
@@ -111,7 +112,8 @@ int msm_gem_init_vma(struct msm_gem_address_space *aspace,
 		return -EBUSY;
 
 	spin_lock(&aspace->lock);
-	ret = drm_mm_insert_node(&aspace->mm, &vma->node, npages);
+	ret = drm_mm_insert_node_in_range(&aspace->mm, &vma->node, npages, 0,
+		0, range_start, range_end, 0);
 	spin_unlock(&aspace->lock);
 
 	if (ret)
@@ -125,37 +127,14 @@ int msm_gem_init_vma(struct msm_gem_address_space *aspace,
 	return 0;
 }
 
-
 struct msm_gem_address_space *
-msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
-		const char *name)
+msm_gem_address_space_create(struct msm_mmu *mmu, const char *name,
+		u64 va_start, u64 size)
 {
 	struct msm_gem_address_space *aspace;
-	u64 size = domain->geometry.aperture_end -
-		domain->geometry.aperture_start;
-
-	aspace = kzalloc(sizeof(*aspace), GFP_KERNEL);
-	if (!aspace)
-		return ERR_PTR(-ENOMEM);
-
-	spin_lock_init(&aspace->lock);
-	aspace->name = name;
-	aspace->mmu = msm_iommu_new(dev, domain);
-
-	drm_mm_init(&aspace->mm, (domain->geometry.aperture_start >> PAGE_SHIFT),
-		size >> PAGE_SHIFT);
 
-	kref_init(&aspace->kref);
-
-	return aspace;
-}
-
-struct msm_gem_address_space *
-msm_gem_address_space_create_a2xx(struct device *dev, struct msm_gpu *gpu,
-		const char *name, uint64_t va_start, uint64_t va_end)
-{
-	struct msm_gem_address_space *aspace;
-	u64 size = va_end - va_start;
+	if (IS_ERR(mmu))
+		return ERR_CAST(mmu);
 
 	aspace = kzalloc(sizeof(*aspace), GFP_KERNEL);
 	if (!aspace)
@@ -163,10 +142,9 @@ msm_gem_address_space_create_a2xx(struct device *dev, struct msm_gpu *gpu,
 
 	spin_lock_init(&aspace->lock);
 	aspace->name = name;
-	aspace->mmu = msm_gpummu_new(dev, gpu);
+	aspace->mmu = mmu;
 
-	drm_mm_init(&aspace->mm, (va_start >> PAGE_SHIFT),
-		size >> PAGE_SHIFT);
+	drm_mm_init(&aspace->mm, va_start >> PAGE_SHIFT, size >> PAGE_SHIFT);
 
 	kref_init(&aspace->kref);
 
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 615c5cda5389..a22d30622306 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -821,51 +821,6 @@ static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu)
 	return 0;
 }
 
-static struct msm_gem_address_space *
-msm_gpu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev,
-		uint64_t va_start, uint64_t va_end)
-{
-	struct msm_gem_address_space *aspace;
-	int ret;
-
-	/*
-	 * Setup IOMMU.. eventually we will (I think) do this once per context
-	 * and have separate page tables per context.  For now, to keep things
-	 * simple and to get something working, just use a single address space:
-	 */
-	if (!adreno_is_a2xx(to_adreno_gpu(gpu))) {
-		struct iommu_domain *iommu = iommu_domain_alloc(&platform_bus_type);
-		if (!iommu)
-			return NULL;
-
-		iommu->geometry.aperture_start = va_start;
-		iommu->geometry.aperture_end = va_end;
-
-		DRM_DEV_INFO(gpu->dev->dev, "%s: using IOMMU\n", gpu->name);
-
-		aspace = msm_gem_address_space_create(&pdev->dev, iommu, "gpu");
-		if (IS_ERR(aspace))
-			iommu_domain_free(iommu);
-	} else {
-		aspace = msm_gem_address_space_create_a2xx(&pdev->dev, gpu, "gpu",
-			va_start, va_end);
-	}
-
-	if (IS_ERR(aspace)) {
-		DRM_DEV_ERROR(gpu->dev->dev, "failed to init mmu: %ld\n",
-			PTR_ERR(aspace));
-		return ERR_CAST(aspace);
-	}
-
-	ret = aspace->mmu->funcs->attach(aspace->mmu);
-	if (ret) {
-		msm_gem_address_space_put(aspace);
-		return ERR_PTR(ret);
-	}
-
-	return aspace;
-}
-
 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
 		const char *name, struct msm_gpu_config *config)
@@ -938,8 +893,8 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 
 	msm_devfreq_init(gpu);
 
-	gpu->aspace = msm_gpu_create_address_space(gpu, pdev,
-		config->va_start, config->va_end);
+
+	gpu->aspace = gpu->funcs->create_address_space(gpu, pdev);
 
 	if (gpu->aspace == NULL)
 		DRM_DEV_INFO(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index 6ccae4ba905c..429cb40f7931 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -21,8 +21,6 @@ struct msm_gpu_state;
 
 struct msm_gpu_config {
 	const char *ioname;
-	uint64_t va_start;
-	uint64_t va_end;
 	unsigned int nr_rings;
 };
 
@@ -64,6 +62,8 @@ struct msm_gpu_funcs {
 	int (*gpu_state_put)(struct msm_gpu_state *state);
 	unsigned long (*gpu_get_freq)(struct msm_gpu *gpu);
 	void (*gpu_set_freq)(struct msm_gpu *gpu, unsigned long freq);
+	struct msm_gem_address_space *(*create_address_space)
+		(struct msm_gpu *gpu, struct platform_device *pdev);
 };
 
 struct msm_gpu {
diff --git a/drivers/gpu/drm/msm/msm_gpummu.c b/drivers/gpu/drm/msm/msm_gpummu.c
index 34980d8eb7ad..310a31b05faa 100644
--- a/drivers/gpu/drm/msm/msm_gpummu.c
+++ b/drivers/gpu/drm/msm/msm_gpummu.c
@@ -21,17 +21,12 @@ struct msm_gpummu {
 #define GPUMMU_PAGE_SIZE SZ_4K
 #define TABLE_SIZE (sizeof(uint32_t) * GPUMMU_VA_RANGE / GPUMMU_PAGE_SIZE)
 
-static int msm_gpummu_attach(struct msm_mmu *mmu)
-{
-	return 0;
-}
-
 static void msm_gpummu_detach(struct msm_mmu *mmu)
 {
 }
 
 static int msm_gpummu_map(struct msm_mmu *mmu, uint64_t iova,
-		struct sg_table *sgt, unsigned len, int prot)
+		struct sg_table *sgt, size_t len, int prot)
 {
 	struct msm_gpummu *gpummu = to_msm_gpummu(mmu);
 	unsigned idx = (iova - GPUMMU_VA_START) / GPUMMU_PAGE_SIZE;
@@ -59,7 +54,7 @@ static int msm_gpummu_map(struct msm_mmu *mmu, uint64_t iova,
 	return 0;
 }
 
-static int msm_gpummu_unmap(struct msm_mmu *mmu, uint64_t iova, unsigned len)
+static int msm_gpummu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
 {
 	struct msm_gpummu *gpummu = to_msm_gpummu(mmu);
 	unsigned idx = (iova - GPUMMU_VA_START) / GPUMMU_PAGE_SIZE;
@@ -85,7 +80,6 @@ static void msm_gpummu_destroy(struct msm_mmu *mmu)
 }
 
 static const struct msm_mmu_funcs funcs = {
-		.attach = msm_gpummu_attach,
 		.detach = msm_gpummu_detach,
 		.map = msm_gpummu_map,
 		.unmap = msm_gpummu_unmap,
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index ad58cfe5998e..3a381a9674c9 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -23,13 +23,6 @@ static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
 	return 0;
 }
 
-static int msm_iommu_attach(struct msm_mmu *mmu)
-{
-	struct msm_iommu *iommu = to_msm_iommu(mmu);
-
-	return iommu_attach_device(iommu->domain, mmu->dev);
-}
-
 static void msm_iommu_detach(struct msm_mmu *mmu)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
@@ -38,7 +31,7 @@ static void msm_iommu_detach(struct msm_mmu *mmu)
 }
 
 static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
-		struct sg_table *sgt, unsigned len, int prot)
+		struct sg_table *sgt, size_t len, int prot)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 	size_t ret;
@@ -49,7 +42,7 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
 	return (ret == len) ? 0 : -EINVAL;
 }
 
-static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, unsigned len)
+static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, size_t len)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 
@@ -66,7 +59,6 @@ static void msm_iommu_destroy(struct msm_mmu *mmu)
 }
 
 static const struct msm_mmu_funcs funcs = {
-		.attach = msm_iommu_attach,
 		.detach = msm_iommu_detach,
 		.map = msm_iommu_map,
 		.unmap = msm_iommu_unmap,
@@ -76,6 +68,10 @@ static const struct msm_mmu_funcs funcs = {
 struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain)
 {
 	struct msm_iommu *iommu;
+	int ret;
+
+	if (!domain)
+		return ERR_PTR(-ENODEV);
 
 	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 	if (!iommu)
@@ -85,5 +81,11 @@ struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain)
 	msm_mmu_init(&iommu->base, dev, &funcs);
 	iommu_set_fault_handler(domain, msm_fault_handler, iommu);
 
+	ret = iommu_attach_device(iommu->domain, dev);
+	if (ret) {
+		kfree(iommu);
+		return ERR_PTR(ret);
+	}
+
 	return &iommu->base;
 }
diff --git a/drivers/gpu/drm/msm/msm_mmu.h b/drivers/gpu/drm/msm/msm_mmu.h
index 67a623f14319..3a534ee59bf6 100644
--- a/drivers/gpu/drm/msm/msm_mmu.h
+++ b/drivers/gpu/drm/msm/msm_mmu.h
@@ -10,11 +10,10 @@
 #include <linux/iommu.h>
 
 struct msm_mmu_funcs {
-	int (*attach)(struct msm_mmu *mmu);
 	void (*detach)(struct msm_mmu *mmu);
 	int (*map)(struct msm_mmu *mmu, uint64_t iova, struct sg_table *sgt,
-			unsigned len, int prot);
-	int (*unmap)(struct msm_mmu *mmu, uint64_t iova, unsigned len);
+			size_t len, int prot);
+	int (*unmap)(struct msm_mmu *mmu, uint64_t iova, size_t len);
 	void (*destroy)(struct msm_mmu *mmu);
 };
 
diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c
index 732f65df5c4f..fea30e7aa9e8 100644
--- a/drivers/gpu/drm/msm/msm_rd.c
+++ b/drivers/gpu/drm/msm/msm_rd.c
@@ -29,8 +29,6 @@
  * or shader programs (if not emitted inline in cmdstream).
  */
 
-#ifdef CONFIG_DEBUG_FS
-
 #include <linux/circ_buf.h>
 #include <linux/debugfs.h>
 #include <linux/kfifo.h>
@@ -47,6 +45,8 @@ bool rd_full = false;
 MODULE_PARM_DESC(rd_full, "If true, $debugfs/.../rd will snapshot all buffer contents");
 module_param_named(rd_full, rd_full, bool, 0600);
 
+#ifdef CONFIG_DEBUG_FS
+
 enum rd_sect_type {
 	RD_NONE,
 	RD_TEST,       /* ascii text */
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index 7622490d8602..d472942102f5 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -277,7 +277,7 @@ nv50_outp_release(struct nouveau_encoder *nv_encoder)
 }
 
 static int
-nv50_outp_acquire(struct nouveau_encoder *nv_encoder)
+nv50_outp_acquire(struct nouveau_encoder *nv_encoder, bool hda)
 {
 	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
 	struct nv50_disp *disp = nv50_disp(drm->dev);
@@ -289,6 +289,7 @@ nv50_outp_acquire(struct nouveau_encoder *nv_encoder)
 		.base.method = NV50_DISP_MTHD_V1_ACQUIRE,
 		.base.hasht  = nv_encoder->dcb->hasht,
 		.base.hashm  = nv_encoder->dcb->hashm,
+		.info.hda = hda,
 	};
 	int ret;
 
@@ -393,7 +394,7 @@ nv50_dac_enable(struct drm_encoder *encoder)
 	struct nv50_head_atom *asyh = nv50_head_atom(nv_crtc->base.state);
 	struct nv50_core *core = nv50_disp(encoder->dev)->core;
 
-	nv50_outp_acquire(nv_encoder);
+	nv50_outp_acquire(nv_encoder, false);
 
 	core->func->dac->ctrl(core, nv_encoder->or, 1 << nv_crtc->index, asyh);
 	asyh->or.depth = 0;
@@ -510,7 +511,7 @@ nv50_audio_component_get_eld(struct device *kdev, int port, int dev_id,
 		if (!nv_connector || !nv_crtc || nv_encoder->or != port ||
 		    nv_crtc->index != dev_id)
 			continue;
-		*enabled = drm_detect_monitor_audio(nv_connector->edid);
+		*enabled = nv_encoder->audio;
 		if (*enabled) {
 			ret = drm_eld_size(nv_connector->base.eld);
 			memcpy(buf, nv_connector->base.eld,
@@ -600,6 +601,7 @@ nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
 				(0x0100 << nv_crtc->index),
 	};
 
+	nv_encoder->audio = false;
 	nvif_mthd(&disp->disp->object, 0, &args, sizeof(args));
 
 	nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or,
@@ -636,6 +638,7 @@ nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
 
 	nvif_mthd(&disp->disp->object, 0, &args,
 		  sizeof(args.base) + drm_eld_size(args.data));
+	nv_encoder->audio = true;
 
 	nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or,
 					nv_crtc->index);
@@ -966,7 +969,7 @@ nv50_msto_enable(struct drm_encoder *encoder)
 		DRM_DEBUG_KMS("Failed to allocate VCPI\n");
 
 	if (!mstm->links++)
-		nv50_outp_acquire(mstm->outp);
+		nv50_outp_acquire(mstm->outp, false /*XXX: MST audio.*/);
 
 	if (mstm->outp->link & 1)
 		proto = 0x8;
@@ -1560,12 +1563,18 @@ nv50_sor_enable(struct drm_encoder *encoder)
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_connector *nv_connector;
 	struct nvbios *bios = &drm->vbios;
+	bool hda = false;
 	u8 proto = 0xf;
 	u8 depth = 0x0;
 
 	nv_connector = nouveau_encoder_connector_get(nv_encoder);
 	nv_encoder->crtc = encoder->crtc;
-	nv50_outp_acquire(nv_encoder);
+
+	if ((disp->disp->object.oclass == GT214_DISP ||
+	     disp->disp->object.oclass >= GF110_DISP) &&
+	    drm_detect_monitor_audio(nv_connector->edid))
+		hda = true;
+	nv50_outp_acquire(nv_encoder, hda);
 
 	switch (nv_encoder->dcb->type) {
 	case DCB_OUTPUT_TMDS:
@@ -1775,7 +1784,7 @@ nv50_pior_enable(struct drm_encoder *encoder)
 	u8 owner = 1 << nv_crtc->index;
 	u8 proto;
 
-	nv50_outp_acquire(nv_encoder);
+	nv50_outp_acquire(nv_encoder, false);
 
 	switch (asyh->or.bpc) {
 	case 10: asyh->or.depth = 0x6; break;
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
index e25ead56052c..99b9b681736d 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
@@ -192,6 +192,8 @@ nv50_wndw_atomic_check_release(struct nv50_wndw *wndw,
 	wndw->func->release(wndw, asyw, asyh);
 	asyw->ntfy.handle = 0;
 	asyw->sema.handle = 0;
+	asyw->xlut.handle = 0;
+	memset(asyw->image.handle, 0x00, sizeof(asyw->image.handle));
 }
 
 static int
@@ -519,7 +521,8 @@ nv50_wndw_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state)
 			return PTR_ERR(ctxdma);
 		}
 
-		asyw->image.handle[0] = ctxdma->object.handle;
+		if (asyw->visible)
+			asyw->image.handle[0] = ctxdma->object.handle;
 	}
 
 	asyw->state.fence = dma_resv_get_excl_rcu(nvbo->bo.base.resv);
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
index 38bf4f38e869..53800fb46582 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h
@@ -46,7 +46,8 @@ struct nv50_disp_acquire_v0 {
 	__u8  version;
 	__u8  or;
 	__u8  link;
-	__u8  pad03[5];
+	__u8  hda;
+	__u8  pad04[4];
 };
 
 struct nv50_disp_dac_load_v0 {
diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h
index de51733b0476..a72c412ac8b1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_encoder.h
+++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h
@@ -52,6 +52,7 @@ struct nouveau_encoder {
 	 * actually programmed on the hw, not the proposed crtc */
 	struct drm_crtc *crtc;
 	u32 ctrl;
+	bool audio;
 
 	struct drm_display_mode mode;
 	int last_dpms;
diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c
index 22f054f7ee3e..ba9f9359c30e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_svm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_svm.c
@@ -175,10 +175,10 @@ nouveau_svmm_bind(struct drm_device *dev, void *data,
 	 */
 
 	mm = get_task_mm(current);
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 
 	if (!cli->svm.svmm) {
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 		return -EINVAL;
 	}
 
@@ -205,7 +205,7 @@ nouveau_svmm_bind(struct drm_device *dev, void *data,
 	 */
 	args->result = 0;
 
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	mmput(mm);
 
 	return 0;
@@ -355,7 +355,7 @@ nouveau_svmm_init(struct drm_device *dev, void *data,
 	if (ret)
 		goto out_free;
 
-	down_write(&current->mm->mmap_sem);
+	mmap_write_lock(current->mm);
 	svmm->notifier.ops = &nouveau_mn_ops;
 	ret = __mmu_notifier_register(&svmm->notifier, current->mm);
 	if (ret)
@@ -364,12 +364,12 @@ nouveau_svmm_init(struct drm_device *dev, void *data,
 
 	cli->svm.svmm = svmm;
 	cli->svm.cli = cli;
-	up_write(&current->mm->mmap_sem);
+	mmap_write_unlock(current->mm);
 	mutex_unlock(&cli->mutex);
 	return 0;
 
 out_mm_unlock:
-	up_write(&current->mm->mmap_sem);
+	mmap_write_unlock(current->mm);
 out_free:
 	mutex_unlock(&cli->mutex);
 	kfree(svmm);
@@ -571,9 +571,9 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm,
 			return -EBUSY;
 
 		range.notifier_seq = mmu_interval_read_begin(range.notifier);
-		down_read(&mm->mmap_sem);
+		mmap_read_lock(mm);
 		ret = hmm_range_fault(&range);
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 		if (ret) {
 			/*
 			 * FIXME: the input PFN_REQ flags are destroyed on
@@ -705,18 +705,18 @@ nouveau_svm_fault(struct nvif_notify *notify)
 		/* Intersect fault window with the CPU VMA, cancelling
 		 * the fault if the address is invalid.
 		 */
-		down_read(&mm->mmap_sem);
+		mmap_read_lock(mm);
 		vma = find_vma_intersection(mm, start, limit);
 		if (!vma) {
 			SVMM_ERR(svmm, "wndw %016llx-%016llx", start, limit);
-			up_read(&mm->mmap_sem);
+			mmap_read_unlock(mm);
 			mmput(mm);
 			nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]);
 			continue;
 		}
 		start = max_t(u64, start, vma->vm_start);
 		limit = min_t(u64, limit, vma->vm_end);
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 		SVMM_DBG(svmm, "wndw %016llx-%016llx", start, limit);
 
 		if (buffer->fault[fi]->addr != start) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
index 571687ba85b8..cf075311cdd2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
@@ -39,6 +39,7 @@ nvkm-y += nvkm/engine/disp/sorgf119.o
 nvkm-y += nvkm/engine/disp/sorgk104.o
 nvkm-y += nvkm/engine/disp/sorgm107.o
 nvkm-y += nvkm/engine/disp/sorgm200.o
+nvkm-y += nvkm/engine/disp/sorgp100.o
 nvkm-y += nvkm/engine/disp/sorgv100.o
 nvkm-y += nvkm/engine/disp/sortu102.o
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
index fd6216684f6d..8471de3f3b61 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
@@ -36,7 +36,7 @@ gp100_disp = {
 	.super = gf119_disp_super,
 	.root = &gp100_disp_root_oclass,
 	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
-	.sor = { .cnt = gf119_sor_cnt, .new = gm200_sor_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gp100_sor_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
index 3468ddec1270..a3779c5046ea 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp102.c
@@ -63,7 +63,7 @@ gp102_disp = {
 	.super = gf119_disp_super,
 	.root = &gp102_disp_root_oclass,
 	.head = { .cnt = gf119_head_cnt, .new = gf119_head_new },
-	.sor = { .cnt = gf119_sor_cnt, .new = gm200_sor_new },
+	.sor = { .cnt = gf119_sor_cnt, .new = gp100_sor_new },
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c
index 9b16a08eb4d9..bf6d41fb0c9f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/hdmigm200.c
@@ -27,10 +27,10 @@ void
 gm200_hdmi_scdc(struct nvkm_ior *ior, int head, u8 scdc)
 {
 	struct nvkm_device *device = ior->disp->engine.subdev.device;
-	const u32 hoff = head * 0x800;
+	const u32 soff = nv50_ior_base(ior);
 	const u32 ctrl = scdc & 0x3;
 
-	nvkm_mask(device, 0x61c5bc + hoff, 0x00000003, ctrl);
+	nvkm_mask(device, 0x61c5bc + soff, 0x00000003, ctrl);
 
 	ior->tmds.high_speed = !!(scdc & 0x2);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
index c1d7a36e4d3c..1a200a9ba4e4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h
@@ -201,6 +201,7 @@ int gf119_sor_new(struct nvkm_disp *, int);
 int gk104_sor_new(struct nvkm_disp *, int);
 int gm107_sor_new(struct nvkm_disp *, int);
 int gm200_sor_new(struct nvkm_disp *, int);
+int gp100_sor_new(struct nvkm_disp *, int);
 
 int gv100_sor_cnt(struct nvkm_disp *, unsigned long *);
 int gv100_sor_new(struct nvkm_disp *, int);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c
index c62030c96fba..dcf08249374a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.c
@@ -111,8 +111,44 @@ nvkm_outp_acquire_ior(struct nvkm_outp *outp, u8 user, struct nvkm_ior *ior)
 	return 0;
 }
 
+static inline int
+nvkm_outp_acquire_hda(struct nvkm_outp *outp, enum nvkm_ior_type type,
+		      u8 user, bool hda)
+{
+	struct nvkm_ior *ior;
+
+	/* First preference is to reuse the OR that is currently armed
+	 * on HW, if any, in order to prevent unnecessary switching.
+	 */
+	list_for_each_entry(ior, &outp->disp->ior, head) {
+		if (!ior->identity && !!ior->func->hda.hpd == hda &&
+		    !ior->asy.outp && ior->arm.outp == outp)
+			return nvkm_outp_acquire_ior(outp, user, ior);
+	}
+
+	/* Failing that, a completely unused OR is the next best thing. */
+	list_for_each_entry(ior, &outp->disp->ior, head) {
+		if (!ior->identity && !!ior->func->hda.hpd == hda &&
+		    !ior->asy.outp && ior->type == type && !ior->arm.outp &&
+		    (ior->func->route.set || ior->id == __ffs(outp->info.or)))
+			return nvkm_outp_acquire_ior(outp, user, ior);
+	}
+
+	/* Last resort is to assign an OR that's already active on HW,
+	 * but will be released during the next modeset.
+	 */
+	list_for_each_entry(ior, &outp->disp->ior, head) {
+		if (!ior->identity && !!ior->func->hda.hpd == hda &&
+		    !ior->asy.outp && ior->type == type &&
+		    (ior->func->route.set || ior->id == __ffs(outp->info.or)))
+			return nvkm_outp_acquire_ior(outp, user, ior);
+	}
+
+	return -ENOSPC;
+}
+
 int
-nvkm_outp_acquire(struct nvkm_outp *outp, u8 user)
+nvkm_outp_acquire(struct nvkm_outp *outp, u8 user, bool hda)
 {
 	struct nvkm_ior *ior = outp->ior;
 	enum nvkm_ior_proto proto;
@@ -137,32 +173,25 @@ nvkm_outp_acquire(struct nvkm_outp *outp, u8 user)
 		return nvkm_outp_acquire_ior(outp, user, ior);
 	}
 
-	/* First preference is to reuse the OR that is currently armed
-	 * on HW, if any, in order to prevent unnecessary switching.
+	/* If we don't need HDA, first try to acquire an OR that doesn't
+	 * support it to leave free the ones that do.
 	 */
-	list_for_each_entry(ior, &outp->disp->ior, head) {
-		if (!ior->identity && !ior->asy.outp && ior->arm.outp == outp)
-			return nvkm_outp_acquire_ior(outp, user, ior);
-	}
+	if (!hda) {
+		if (!nvkm_outp_acquire_hda(outp, type, user, false))
+			return 0;
 
-	/* Failing that, a completely unused OR is the next best thing. */
-	list_for_each_entry(ior, &outp->disp->ior, head) {
-		if (!ior->identity &&
-		    !ior->asy.outp && ior->type == type && !ior->arm.outp &&
-		    (ior->func->route.set || ior->id == __ffs(outp->info.or)))
-			return nvkm_outp_acquire_ior(outp, user, ior);
+		/* Use a HDA-supporting SOR anyway. */
+		return nvkm_outp_acquire_hda(outp, type, user, true);
 	}
 
-	/* Last resort is to assign an OR that's already active on HW,
-	 * but will be released during the next modeset.
-	 */
-	list_for_each_entry(ior, &outp->disp->ior, head) {
-		if (!ior->identity && !ior->asy.outp && ior->type == type &&
-		    (ior->func->route.set || ior->id == __ffs(outp->info.or)))
-			return nvkm_outp_acquire_ior(outp, user, ior);
-	}
+	/* We want HDA, try to acquire an OR that supports it. */
+	if (!nvkm_outp_acquire_hda(outp, type, user, true))
+		return 0;
 
-	return -ENOSPC;
+	/* There weren't any free ORs that support HDA, grab one that
+	 * doesn't and at least allow display to work still.
+	 */
+	return nvkm_outp_acquire_hda(outp, type, user, false);
 }
 
 void
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h
index 721b068b87ef..ee028d30cfe7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h
@@ -32,7 +32,7 @@ int nvkm_outp_new(struct nvkm_disp *, int index, struct dcb_output *,
 void nvkm_outp_del(struct nvkm_outp **);
 void nvkm_outp_init(struct nvkm_outp *);
 void nvkm_outp_fini(struct nvkm_outp *);
-int nvkm_outp_acquire(struct nvkm_outp *, u8 user);
+int nvkm_outp_acquire(struct nvkm_outp *, u8 user, bool hda);
 void nvkm_outp_release(struct nvkm_outp *, u8 user);
 void nvkm_outp_route(struct nvkm_disp *);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
index a7672ef17d3b..fb5de44e4b8d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c
@@ -99,7 +99,7 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 		} *args = data;
 		int ret = -ENOSYS;
 		if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
-			ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER);
+			ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, args->v0.hda);
 			if (ret == 0) {
 				args->v0.or = outp->ior->id;
 				args->v0.link = outp->ior->asy.link;
@@ -119,7 +119,7 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 		if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {
 			if (args->v0.data & 0xfff00000)
 				return -EINVAL;
-			ret = nvkm_outp_acquire(outp, NVKM_OUTP_PRIV);
+			ret = nvkm_outp_acquire(outp, NVKM_OUTP_PRIV, false);
 			if (ret)
 				return ret;
 			ret = outp->ior->func->sense(outp->ior, args->v0.data);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
index cf2075db742a..4dd7f382968e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c
@@ -89,7 +89,7 @@ gm200_sor_route_get(struct nvkm_outp *outp, int *link)
 }
 
 static const struct nvkm_ior_func
-gm200_sor = {
+gm200_sor_hda = {
 	.route = {
 		.get = gm200_sor_route_get,
 		.set = gm200_sor_route_set,
@@ -119,8 +119,42 @@ gm200_sor = {
 	},
 };
 
+static const struct nvkm_ior_func
+gm200_sor = {
+	.route = {
+		.get = gm200_sor_route_get,
+		.set = gm200_sor_route_set,
+	},
+	.state = gf119_sor_state,
+	.power = nv50_sor_power,
+	.clock = gf119_sor_clock,
+	.hdmi = {
+		.ctrl = gk104_hdmi_ctrl,
+		.scdc = gm200_hdmi_scdc,
+	},
+	.dp = {
+		.lanes = { 0, 1, 2, 3 },
+		.links = gf119_sor_dp_links,
+		.power = g94_sor_dp_power,
+		.pattern = gm107_sor_dp_pattern,
+		.drive = gm200_sor_dp_drive,
+		.vcpi = gf119_sor_dp_vcpi,
+		.audio = gf119_sor_dp_audio,
+		.audio_sym = gf119_sor_dp_audio_sym,
+		.watermark = gf119_sor_dp_watermark,
+	},
+};
+
 int
 gm200_sor_new(struct nvkm_disp *disp, int id)
 {
+	struct nvkm_device *device = disp->engine.subdev.device;
+	u32 hda;
+
+	if (!((hda = nvkm_rd32(device, 0x08a15c)) & 0x40000000))
+		hda = nvkm_rd32(device, 0x101034);
+
+	if (hda & BIT(id))
+		return nvkm_ior_new_(&gm200_sor_hda, disp, SOR, id);
 	return nvkm_ior_new_(&gm200_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgp100.c
new file mode 100644
index 000000000000..c54f88317a07
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgp100.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "ior.h"
+
+static const struct nvkm_ior_func
+gp100_sor_hda = {
+	.route = {
+		.get = gm200_sor_route_get,
+		.set = gm200_sor_route_set,
+	},
+	.state = gf119_sor_state,
+	.power = nv50_sor_power,
+	.clock = gf119_sor_clock,
+	.hdmi = {
+		.ctrl = gk104_hdmi_ctrl,
+		.scdc = gm200_hdmi_scdc,
+	},
+	.dp = {
+		.lanes = { 0, 1, 2, 3 },
+		.links = gf119_sor_dp_links,
+		.power = g94_sor_dp_power,
+		.pattern = gm107_sor_dp_pattern,
+		.drive = gm200_sor_dp_drive,
+		.vcpi = gf119_sor_dp_vcpi,
+		.audio = gf119_sor_dp_audio,
+		.audio_sym = gf119_sor_dp_audio_sym,
+		.watermark = gf119_sor_dp_watermark,
+	},
+	.hda = {
+		.hpd = gf119_hda_hpd,
+		.eld = gf119_hda_eld,
+		.device_entry = gf119_hda_device_entry,
+	},
+};
+
+static const struct nvkm_ior_func
+gp100_sor = {
+	.route = {
+		.get = gm200_sor_route_get,
+		.set = gm200_sor_route_set,
+	},
+	.state = gf119_sor_state,
+	.power = nv50_sor_power,
+	.clock = gf119_sor_clock,
+	.hdmi = {
+		.ctrl = gk104_hdmi_ctrl,
+		.scdc = gm200_hdmi_scdc,
+	},
+	.dp = {
+		.lanes = { 0, 1, 2, 3 },
+		.links = gf119_sor_dp_links,
+		.power = g94_sor_dp_power,
+		.pattern = gm107_sor_dp_pattern,
+		.drive = gm200_sor_dp_drive,
+		.vcpi = gf119_sor_dp_vcpi,
+		.audio = gf119_sor_dp_audio,
+		.audio_sym = gf119_sor_dp_audio_sym,
+		.watermark = gf119_sor_dp_watermark,
+	},
+};
+
+int
+gp100_sor_new(struct nvkm_disp *disp, int id)
+{
+	struct nvkm_device *device = disp->engine.subdev.device;
+	u32 hda;
+
+	if (!((hda = nvkm_rd32(device, 0x08a15c)) & 0x40000000))
+		hda = nvkm_rd32(device, 0x10ebb0) >> 8;
+
+	if (hda & BIT(id))
+		return nvkm_ior_new_(&gp100_sor_hda, disp, SOR, id);
+	return nvkm_ior_new_(&gp100_sor, disp, SOR, id);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c
index d11a0dff10c6..4441187e8ec9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgv100.c
@@ -78,7 +78,7 @@ gv100_sor_state(struct nvkm_ior *sor, struct nvkm_ior_state *state)
 }
 
 static const struct nvkm_ior_func
-gv100_sor = {
+gv100_sor_hda = {
 	.route = {
 		.get = gm200_sor_route_get,
 		.set = gm200_sor_route_set,
@@ -107,9 +107,42 @@ gv100_sor = {
 	},
 };
 
+static const struct nvkm_ior_func
+gv100_sor = {
+	.route = {
+		.get = gm200_sor_route_get,
+		.set = gm200_sor_route_set,
+	},
+	.state = gv100_sor_state,
+	.power = nv50_sor_power,
+	.clock = gf119_sor_clock,
+	.hdmi = {
+		.ctrl = gv100_hdmi_ctrl,
+		.scdc = gm200_hdmi_scdc,
+	},
+	.dp = {
+		.lanes = { 0, 1, 2, 3 },
+		.links = gf119_sor_dp_links,
+		.power = g94_sor_dp_power,
+		.pattern = gm107_sor_dp_pattern,
+		.drive = gm200_sor_dp_drive,
+		.audio = gv100_sor_dp_audio,
+		.audio_sym = gv100_sor_dp_audio_sym,
+		.watermark = gv100_sor_dp_watermark,
+	},
+};
+
 int
 gv100_sor_new(struct nvkm_disp *disp, int id)
 {
+	struct nvkm_device *device = disp->engine.subdev.device;
+	u32 hda;
+
+	if (!((hda = nvkm_rd32(device, 0x08a15c)) & 0x40000000))
+		hda = nvkm_rd32(device, 0x118fb0) >> 8;
+
+	if (hda & BIT(id))
+		return nvkm_ior_new_(&gv100_sor_hda, disp, SOR, id);
 	return nvkm_ior_new_(&gv100_sor, disp, SOR, id);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c
index fa6d74251237..59865a934c4b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sortu102.c
@@ -62,7 +62,7 @@ tu102_sor_dp_links(struct nvkm_ior *sor, struct nvkm_i2c_aux *aux)
 }
 
 static const struct nvkm_ior_func
-tu102_sor = {
+tu102_sor_hda = {
 	.route = {
 		.get = gm200_sor_route_get,
 		.set = gm200_sor_route_set,
@@ -92,8 +92,38 @@ tu102_sor = {
 	},
 };
 
+static const struct nvkm_ior_func
+tu102_sor = {
+	.route = {
+		.get = gm200_sor_route_get,
+		.set = gm200_sor_route_set,
+	},
+	.state = gv100_sor_state,
+	.power = nv50_sor_power,
+	.clock = gf119_sor_clock,
+	.hdmi = {
+		.ctrl = gv100_hdmi_ctrl,
+		.scdc = gm200_hdmi_scdc,
+	},
+	.dp = {
+		.lanes = { 0, 1, 2, 3 },
+		.links = tu102_sor_dp_links,
+		.power = g94_sor_dp_power,
+		.pattern = gm107_sor_dp_pattern,
+		.drive = gm200_sor_dp_drive,
+		.vcpi = tu102_sor_dp_vcpi,
+		.audio = gv100_sor_dp_audio,
+		.audio_sym = gv100_sor_dp_audio_sym,
+		.watermark = gv100_sor_dp_watermark,
+	},
+};
+
 int
 tu102_sor_new(struct nvkm_disp *disp, int id)
 {
+	struct nvkm_device *device = disp->engine.subdev.device;
+	u32 hda = nvkm_rd32(device, 0x08a15c);
+	if (hda & BIT(id))
+		return nvkm_ior_new_(&tu102_sor_hda, disp, SOR, id);
 	return nvkm_ior_new_(&tu102_sor, disp, SOR, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
index ec330d791d15..e56880f3e3bd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
@@ -352,7 +352,7 @@ gk20a_gr_load(struct gf100_gr *gr, int ver, const struct gf100_gr_fwif *fwif)
 
 static const struct gf100_gr_fwif
 gk20a_gr_fwif[] = {
-	{ -1, gk20a_gr_load, &gk20a_gr },
+	{ 0, gk20a_gr_load, &gk20a_gr },
 	{}
 };
 
diff --git a/drivers/gpu/drm/omapdrm/dss/Kconfig b/drivers/gpu/drm/omapdrm/dss/Kconfig
index 72ae79c0c9b4..2658c521b702 100644
--- a/drivers/gpu/drm/omapdrm/dss/Kconfig
+++ b/drivers/gpu/drm/omapdrm/dss/Kconfig
@@ -71,7 +71,7 @@ config OMAP4_DSS_HDMI_CEC
 	depends on OMAP4_DSS_HDMI
 	select CEC_CORE
 	default y
-	---help---
+	help
 	  When selected the HDMI transmitter will support the CEC feature.
 
 config OMAP5_DSS_HDMI
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 0d0ab8e0ff3b..cc31d187042e 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -196,12 +196,12 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
 		p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm,
 					      &p->validated);
 	if (need_mmap_lock)
-		down_read(&current->mm->mmap_sem);
+		mmap_read_lock(current->mm);
 
 	r = radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring);
 
 	if (need_mmap_lock)
-		up_read(&current->mm->mmap_sem);
+		mmap_read_unlock(current->mm);
 
 	return r;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index 068c3e5da173..3c8f570a20ee 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -342,17 +342,17 @@ int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data,
 	}
 
 	if (args->flags & RADEON_GEM_USERPTR_VALIDATE) {
-		down_read(&current->mm->mmap_sem);
+		mmap_read_lock(current->mm);
 		r = radeon_bo_reserve(bo, true);
 		if (r) {
-			up_read(&current->mm->mmap_sem);
+			mmap_read_unlock(current->mm);
 			goto release_object;
 		}
 
 		radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_GTT);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		radeon_bo_unreserve(bo);
-		up_read(&current->mm->mmap_sem);
+		mmap_read_unlock(current->mm);
 		if (r)
 			goto release_object;
 	}
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi.h b/drivers/gpu/drm/sun4i/sun4i_hdmi.h
index 7ad3f06c127e..00ca35f07ba5 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi.h
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi.h
@@ -148,7 +148,7 @@
 #define SUN4I_HDMI_DDC_CMD_IMPLICIT_WRITE	3
 
 #define SUN4I_HDMI_DDC_CLK_REG		0x528
-#define SUN4I_HDMI_DDC_CLK_M(m)			(((m) & 0x7) << 3)
+#define SUN4I_HDMI_DDC_CLK_M(m)			(((m) & 0xf) << 3)
 #define SUN4I_HDMI_DDC_CLK_N(n)			((n) & 0x7)
 
 #define SUN4I_HDMI_DDC_LINE_CTRL_REG	0x540
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c
index 2ff780114106..12430b9d4e93 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c
@@ -33,7 +33,7 @@ static unsigned long sun4i_ddc_calc_divider(unsigned long rate,
 	unsigned long best_rate = 0;
 	u8 best_m = 0, best_n = 0, _m, _n;
 
-	for (_m = 0; _m < 8; _m++) {
+	for (_m = 0; _m < 16; _m++) {
 		for (_n = 0; _n < 8; _n++) {
 			unsigned long tmp_rate;
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 52d2b71f1588..f09b096ba4fd 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -257,54 +257,6 @@ static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 	return 0;
 }
 
-#ifdef CONFIG_X86
-#define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot)
-#define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr)
-#else
-#define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0,  __prot)
-#define __ttm_kunmap_atomic(__addr) vunmap(__addr)
-#endif
-
-
-/**
- * ttm_kmap_atomic_prot - Efficient kernel map of a single page with
- * specified page protection.
- *
- * @page: The page to map.
- * @prot: The page protection.
- *
- * This function maps a TTM page using the kmap_atomic api if available,
- * otherwise falls back to vmap. The user must make sure that the
- * specified page does not have an aliased mapping with a different caching
- * policy unless the architecture explicitly allows it. Also mapping and
- * unmapping using this api must be correctly nested. Unmapping should
- * occur in the reverse order of mapping.
- */
-void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot)
-{
-	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
-		return kmap_atomic(page);
-	else
-		return __ttm_kmap_atomic_prot(page, prot);
-}
-EXPORT_SYMBOL(ttm_kmap_atomic_prot);
-
-/**
- * ttm_kunmap_atomic_prot - Unmap a page that was mapped using
- * ttm_kmap_atomic_prot.
- *
- * @addr: The virtual address from the map.
- * @prot: The page protection.
- */
-void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot)
-{
-	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
-		kunmap_atomic(addr);
-	else
-		__ttm_kunmap_atomic(addr);
-}
-EXPORT_SYMBOL(ttm_kunmap_atomic_prot);
-
 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 				unsigned long page,
 				pgprot_t prot)
@@ -316,13 +268,13 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 		return -ENOMEM;
 
 	src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
-	dst = ttm_kmap_atomic_prot(d, prot);
+	dst = kmap_atomic_prot(d, prot);
 	if (!dst)
 		return -ENOMEM;
 
 	memcpy_fromio(dst, src, PAGE_SIZE);
 
-	ttm_kunmap_atomic_prot(dst, prot);
+	kunmap_atomic(dst);
 
 	return 0;
 }
@@ -338,13 +290,13 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 		return -ENOMEM;
 
 	dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
-	src = ttm_kmap_atomic_prot(s, prot);
+	src = kmap_atomic_prot(s, prot);
 	if (!src)
 		return -ENOMEM;
 
 	memcpy_toio(dst, src, PAGE_SIZE);
 
-	ttm_kunmap_atomic_prot(src, prot);
+	kunmap_atomic(src);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 0ad30b112982..a43aa7275f12 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -58,7 +58,7 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 		goto out_clear;
 
 	/*
-	 * If possible, avoid waiting for GPU with mmap_sem
+	 * If possible, avoid waiting for GPU with mmap_lock
 	 * held.  We only do this if the fault allows retry and this
 	 * is the first attempt.
 	 */
@@ -68,7 +68,7 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 			goto out_unlock;
 
 		ttm_bo_get(bo);
-		up_read(&vmf->vma->vm_mm->mmap_sem);
+		mmap_read_unlock(vmf->vma->vm_mm);
 		(void) dma_fence_wait(bo->moving, true);
 		dma_resv_unlock(bo->base.resv);
 		ttm_bo_put(bo);
@@ -131,20 +131,20 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,
 {
 	/*
 	 * Work around locking order reversal in fault / nopfn
-	 * between mmap_sem and bo_reserve: Perform a trylock operation
+	 * between mmap_lock and bo_reserve: Perform a trylock operation
 	 * for reserve, and if it fails, retry the fault after waiting
 	 * for the buffer to become unreserved.
 	 */
 	if (unlikely(!dma_resv_trylock(bo->base.resv))) {
 		/*
 		 * If the fault allows retry and this is the first
-		 * fault attempt, we try to release the mmap_sem
+		 * fault attempt, we try to release the mmap_lock
 		 * before waiting
 		 */
 		if (fault_flag_allow_retry_first(vmf->flags)) {
 			if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
 				ttm_bo_get(bo);
-				up_read(&vmf->vma->vm_mm->mmap_sem);
+				mmap_read_unlock(vmf->vma->vm_mm);
 				if (!dma_resv_lock_interruptible(bo->base.resv,
 								 NULL))
 					dma_resv_unlock(bo->base.resv);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c
index bb46ca0c458f..1629427d5734 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c
@@ -27,6 +27,7 @@
  **************************************************************************/
 
 #include "vmwgfx_drv.h"
+#include <linux/highmem.h>
 
 /*
  * Template that implements find_first_diff() for a generic
@@ -374,12 +375,12 @@ static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
 		copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset);
 
 		if (unmap_src) {
-			ttm_kunmap_atomic_prot(d->src_addr, d->src_prot);
+			kunmap_atomic(d->src_addr);
 			d->src_addr = NULL;
 		}
 
 		if (unmap_dst) {
-			ttm_kunmap_atomic_prot(d->dst_addr, d->dst_prot);
+			kunmap_atomic(d->dst_addr);
 			d->dst_addr = NULL;
 		}
 
@@ -388,8 +389,8 @@ static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
 				return -EINVAL;
 
 			d->dst_addr =
-				ttm_kmap_atomic_prot(d->dst_pages[dst_page],
-						     d->dst_prot);
+				kmap_atomic_prot(d->dst_pages[dst_page],
+						 d->dst_prot);
 			if (!d->dst_addr)
 				return -ENOMEM;
 
@@ -401,8 +402,8 @@ static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
 				return -EINVAL;
 
 			d->src_addr =
-				ttm_kmap_atomic_prot(d->src_pages[src_page],
-						     d->src_prot);
+				kmap_atomic_prot(d->src_pages[src_page],
+						 d->src_prot);
 			if (!d->src_addr)
 				return -ENOMEM;
 
@@ -499,9 +500,9 @@ int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
 	}
 out:
 	if (d.src_addr)
-		ttm_kunmap_atomic_prot(d.src_addr, d.src_prot);
+		kunmap_atomic(d.src_addr);
 	if (d.dst_addr)
-		ttm_kunmap_atomic_prot(d.dst_addr, d.dst_prot);
+		kunmap_atomic(d.dst_addr);
 
 	return ret;
 }
diff --git a/drivers/greybus/Kconfig b/drivers/greybus/Kconfig
index b84fcaf8b105..78ba3c3083d5 100644
--- a/drivers/greybus/Kconfig
+++ b/drivers/greybus/Kconfig
@@ -2,8 +2,8 @@
 menuconfig GREYBUS
 	tristate "Greybus support"
 	depends on SYSFS
-	---help---
-	  This option enables the Greybus driver core.  Greybus is an
+	help
+	  This option enables the Greybus driver core.  Greybus is a
 	  hardware protocol that was designed to provide Unipro with a
 	  sane application layer.  It was originally designed for the
 	  ARA project, a module phone system, but has shown up in other
@@ -12,7 +12,7 @@ menuconfig GREYBUS
 
 	  Say Y here to enable support for these types of drivers.
 
-	  To compile this code as a module, chose M here: the module
+	  To compile this code as a module, choose M here: the module
 	  will be called greybus.ko
 
 if GREYBUS
@@ -20,12 +20,12 @@ if GREYBUS
 config GREYBUS_ES2
 	tristate "Greybus ES3 USB host controller"
 	depends on USB
-	---help---
+	help
 	  Select this option if you have a Toshiba ES3 USB device that
 	  acts as a Greybus "host controller".  This device is a bridge
 	  from a USB device to a Unipro network.
 
-	  To compile this code as a module, chose M here: the module
+	  To compile this code as a module, choose M here: the module
 	  will be called gb-es2.ko
 
 endif	# GREYBUS
diff --git a/drivers/greybus/arpc.h b/drivers/greybus/arpc.h
index c8b83c5cfa79..b9ea81b55b29 100644
--- a/drivers/greybus/arpc.h
+++ b/drivers/greybus/arpc.h
@@ -21,7 +21,7 @@ struct arpc_request_message {
 	__le16	id;		/* RPC unique id */
 	__le16	size;		/* Size in bytes of header + payload */
 	__u8	type;		/* RPC type */
-	__u8	data[0];	/* ARPC data */
+	__u8	data[];	/* ARPC data */
 } __packed;
 
 struct arpc_response_message {
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index 34f07371716d..45e87dc59d4e 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -9,7 +9,7 @@ config HID
 	tristate "HID bus support"
 	depends on INPUT
 	default y
-	---help---
+	help
 	  A human interface device (HID) is a type of computer device that
 	  interacts directly with and takes input from humans. The term "HID"
 	  most commonly used to refer to the USB-HID specification, but other
@@ -31,7 +31,7 @@ config HID_BATTERY_STRENGTH
 	depends on HID
 	select POWER_SUPPLY
 	default n
-	---help---
+	help
 	This option adds support of reporting battery strength (for HID devices
 	that support this feature) through power_supply class so that userspace
 	tools, such as upower, can display it.
@@ -39,10 +39,10 @@ config HID_BATTERY_STRENGTH
 config HIDRAW
 	bool "/dev/hidraw raw HID device support"
 	depends on HID
-	---help---
+	help
 	Say Y here if you want to support HID devices (from the USB
 	specification standpoint) that aren't strictly user interface
-	devices, like monitor controls and Uninterruptable Power Supplies.
+	devices, like monitor controls and Uninterruptible Power Supplies.
 
 	This module supports these devices separately using a separate
 	event interface on /dev/hidraw.
@@ -59,7 +59,7 @@ config UHID
 	tristate "User-space I/O driver support for HID subsystem"
 	depends on HID
 	default n
-	---help---
+	help
 	Say Y here if you want to provide HID I/O Drivers from user-space.
 	This allows to write I/O drivers in user-space and feed the data from
 	the device into the kernel. The kernel parses the HID reports, loads the
@@ -80,7 +80,7 @@ config HID_GENERIC
 	tristate "Generic HID driver"
 	depends on HID
 	default HID
-	---help---
+	help
 	Support for generic devices on the HID bus. This includes most
 	keyboards and mice, joysticks, tablets and digitizers.
 
@@ -96,13 +96,13 @@ config HID_A4TECH
 	tristate "A4 tech mice"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for A4 tech X5 and WOP-35 / Trust 450L mice.
 
 config HID_ACCUTOUCH
 	tristate "Accutouch touch device"
 	depends on USB_HID
-	---help---
+	help
 	  This selects a driver for the Accutouch 2216 touch controller.
 
 	  The driver works around a problem in the reported device capabilities
@@ -114,14 +114,14 @@ config HID_ACCUTOUCH
 config HID_ACRUX
 	tristate "ACRUX game controller support"
 	depends on HID
-	---help---
+	help
 	Say Y here if you want to enable support for ACRUX game controllers.
 
 config HID_ACRUX_FF
 	bool "ACRUX force feedback support"
 	depends on HID_ACRUX
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you want to enable force feedback support for ACRUX
 	game controllers.
 
@@ -129,7 +129,7 @@ config HID_APPLE
 	tristate "Apple {i,Power,Mac}Books"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for some Apple devices which less or more break
 	HID specification.
 
@@ -139,7 +139,7 @@ config HID_APPLE
 config HID_APPLEIR
 	tristate "Apple infrared receiver"
 	depends on (USB_HID)
-	---help---
+	help
 	Support for Apple infrared remote control. All the Apple computers from
 	  2005 onwards include such a port, except the unibody Macbook (2009),
 	  and Mac Pros. This receiver is also used in the Apple TV set-top box
@@ -149,10 +149,11 @@ config HID_APPLEIR
 
 config HID_ASUS
 	tristate "Asus"
+	depends on USB_HID
 	depends on LEDS_CLASS
 	depends on ASUS_WMI || ASUS_WMI=n
 	select POWER_SUPPLY
-	---help---
+	help
 	Support for Asus notebook built-in keyboard and touchpad via i2c, and
 	the Asus Republic of Gamers laptop keyboard special keys.
 
@@ -165,21 +166,21 @@ config HID_ASUS
 config HID_AUREAL
 	tristate "Aureal"
 	depends on HID
-	---help---
+	help
 	Support for Aureal Cy se W-01RN Remote Controller and other Aureal derived remotes.
 
 config HID_BELKIN
 	tristate "Belkin Flip KVM and Wireless keyboard"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for Belkin Flip KVM and Wireless keyboard.
 
 config HID_BETOP_FF
 	tristate "Betop Production Inc. force feedback support"
 	depends on USB_HID
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you want to enable force feedback support for devices by
 	BETOP Production Ltd.
 	Currently the following devices are known to be supported:
@@ -201,20 +202,20 @@ config HID_CHERRY
 	tristate "Cherry Cymotion keyboard"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for Cherry Cymotion keyboard.
 
 config HID_CHICONY
 	tristate "Chicony devices"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for Chicony Tactical pad and special keys on Chicony keyboards.
 
 config HID_CORSAIR
 	tristate "Corsair devices"
 	depends on HID && USB && LEDS_CLASS
-	---help---
+	help
 	Support for Corsair devices that are not fully compliant with the
 	HID standard.
 
@@ -246,7 +247,7 @@ config HID_PRODIKEYS
 	tristate "Prodikeys PC-MIDI Keyboard support"
 	depends on HID && SND
 	select SND_RAWMIDI
-	---help---
+	help
 	Support for Prodikeys PC-MIDI Keyboard device support.
 	Say Y here to enable support for this device.
 	- Prodikeys PC-MIDI keyboard.
@@ -260,14 +261,14 @@ config HID_PRODIKEYS
 config HID_CMEDIA
 	tristate "CMedia CM6533 HID audio jack controls"
 	depends on HID
-	---help---
+	help
 	Support for CMedia CM6533 HID audio jack controls.
 
 config HID_CP2112
 	tristate "Silicon Labs CP2112 HID USB-to-SMBus Bridge support"
 	depends on USB_HID && HIDRAW && I2C && GPIOLIB
 	select GPIOLIB_IRQCHIP
-	---help---
+	help
 	Support for Silicon Labs CP2112 HID USB to SMBus Master Bridge.
 	This is a HID device driver which registers as an i2c adapter
 	and gpiochip to expose these functions of the CP2112. The
@@ -286,13 +287,13 @@ config HID_CYPRESS
 	tristate "Cypress mouse and barcode readers"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for cypress mouse and barcode readers.
 
 config HID_DRAGONRISE
 	tristate "DragonRise Inc. game controller"
 	depends on HID
-	---help---
+	help
 	Say Y here if you have DragonRise Inc. game controllers.
 	These might be branded as:
 	- Tesun USB-703
@@ -304,7 +305,7 @@ config DRAGONRISE_FF
 	bool "DragonRise Inc. force feedback"
 	depends on HID_DRAGONRISE
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you want to enable force feedback support for DragonRise Inc.
 	game controllers.
 
@@ -312,7 +313,7 @@ config HID_EMS_FF
 	tristate "EMS Production Inc. force feedback support"
 	depends on HID
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you want to enable force feedback support for devices by
 	EMS Production Ltd.
 	Currently the following devices are known to be supported:
@@ -321,7 +322,7 @@ config HID_EMS_FF
 config HID_ELAN
 	tristate "ELAN USB Touchpad Support"
 	depends on LEDS_CLASS && USB_HID
-	---help---
+	help
 	Say Y to enable support for the USB ELAN touchpad
 	Currently the following devices are known to be supported:
 	 - HP Pavilion X2 10-p0XX.
@@ -329,7 +330,7 @@ config HID_ELAN
 config HID_ELECOM
 	tristate "ELECOM HID devices"
 	depends on HID
-	---help---
+	help
 	Support for ELECOM devices:
 	  - BM084 Bluetooth Mouse
 	  - EX-G Trackballs (M-XT3DRBK, M-XT3URBK)
@@ -339,7 +340,7 @@ config HID_ELECOM
 config HID_ELO
 	tristate "ELO USB 4000/4500 touchscreen"
 	depends on USB_HID
-	---help---
+	help
 	Support for the ELO USB 4000/4500 touchscreens. Note that this is for
 	different devices than those handled by CONFIG_TOUCHSCREEN_USB_ELO.
 
@@ -347,19 +348,19 @@ config HID_EZKEY
 	tristate "Ezkey BTC 8193 keyboard"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for Ezkey BTC 8193 keyboard.
 
 config HID_GEMBIRD
 	tristate "Gembird Joypad"
 	depends on HID
-	---help---
+	help
 	Support for Gembird JPD-DualForce 2.
 
 config HID_GFRM
 	tristate "Google Fiber TV Box remote control support"
 	depends on HID
-	---help---
+	help
 	Support for Google Fiber TV Box remote controls
 
 config HID_GLORIOUS
@@ -372,7 +373,7 @@ config HID_GLORIOUS
 config HID_HOLTEK
 	tristate "Holtek HID devices"
 	depends on USB_HID
-	---help---
+	help
 	Support for Holtek based devices:
 	  - Holtek On Line Grip based game controller
 	  - Trust GXT 18 Gaming Keyboard
@@ -386,20 +387,20 @@ config HOLTEK_FF
 	bool "Holtek On Line Grip force feedback support"
 	depends on HID_HOLTEK
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	  Say Y here if you have a Holtek On Line Grip based game controller
 	  and want to have force feedback support for it.
 
 config HID_GOOGLE_HAMMER
 	tristate "Google Hammer Keyboard"
 	depends on USB_HID && LEDS_CLASS && CROS_EC
-	---help---
+	help
 	Say Y here if you have a Google Hammer device.
 
 config HID_GT683R
 	tristate "MSI GT68xR LED support"
 	depends on LEDS_CLASS && USB_HID
-	---help---
+	help
 	Say Y here if you want to enable support for the three MSI GT68xR LEDs
 
 	This driver support following modes:
@@ -413,7 +414,7 @@ config HID_GT683R
 config HID_KEYTOUCH
 	tristate "Keytouch HID devices"
 	depends on HID
-	---help---
+	help
 	Support for Keytouch HID devices not fully compliant with
 	the specification. Currently supported:
 		- Keytouch IEC 60945
@@ -421,7 +422,7 @@ config HID_KEYTOUCH
 config HID_KYE
 	tristate "KYE/Genius devices"
 	depends on HID
-	---help---
+	help
 	Support for KYE/Genius devices not fully compliant with HID standard:
 	- Ergo Mouse
 	- EasyPen i405X tablet
@@ -431,13 +432,13 @@ config HID_KYE
 config HID_UCLOGIC
 	tristate "UC-Logic"
 	depends on USB_HID
-	---help---
+	help
 	Support for UC-Logic and Huion tablets.
 
 config HID_WALTOP
 	tristate "Waltop"
 	depends on HID
-	---help---
+	help
 	Support for Waltop tablets.
 
 config HID_VIEWSONIC
@@ -449,13 +450,13 @@ config HID_VIEWSONIC
 config HID_GYRATION
 	tristate "Gyration remote control"
 	depends on HID
-	---help---
+	help
 	Support for Gyration remote control.
 
 config HID_ICADE
 	tristate "ION iCade arcade controller"
 	depends on HID
-	---help---
+	help
 	Support for the ION iCade arcade controller to work as a joystick.
 
 	To compile this driver as a module, choose M here: the
@@ -465,13 +466,13 @@ config HID_ITE
 	tristate "ITE devices"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for ITE devices not fully compliant with HID standard.
 
 config HID_JABRA
 	tristate "Jabra USB HID Driver"
 	depends on HID
-	---help---
+	help
 	Support for Jabra USB HID devices.
 
 	Prevents mapping of vendor defined HID usages to input events. Without
@@ -482,27 +483,27 @@ config HID_JABRA
 config HID_TWINHAN
 	tristate "Twinhan IR remote control"
 	depends on HID
-	---help---
+	help
 	Support for Twinhan IR remote control.
 
 config HID_KENSINGTON
 	tristate "Kensington Slimblade Trackball"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for Kensington Slimblade Trackball.
 
 config HID_LCPOWER
 	tristate "LC-Power"
 	depends on HID
-	---help---
+	help
 	Support for LC-Power RC1000MCE RF remote control.
 
 config HID_LED
 	tristate "Simple RGB LED support"
 	depends on HID
 	depends on LEDS_CLASS
-	---help---
+	help
 	Support for simple RGB LED devices. Currently supported are:
 	- Riso Kagaku Webmail Notifier
 	- Dream Cheeky Webmail Notifier and Friends Alert
@@ -518,7 +519,7 @@ config HID_LENOVO
 	depends on HID
 	select NEW_LEDS
 	select LEDS_CLASS
-	---help---
+	help
 	Support for IBM/Lenovo devices that are not fully compliant with HID standard.
 
 	Say Y if you want support for horizontal scrolling of the IBM/Lenovo
@@ -534,18 +535,18 @@ config HID_LOGITECH
 	depends on HID
 	depends on LEDS_CLASS
 	default !EXPERT
-	---help---
+	help
 	Support for Logitech devices that are not fully compliant with HID standard.
 
 config HID_LOGITECH_DJ
-	tristate "Logitech Unifying receivers full support"
+	tristate "Logitech receivers full support"
 	depends on USB_HID
 	depends on HIDRAW
 	depends on HID_LOGITECH
 	select HID_LOGITECH_HIDPP
-	---help---
-	Say Y if you want support for Logitech Unifying receivers and devices.
-	Unifying receivers are capable of pairing up to 6 Logitech compliant
+	help
+	Say Y if you want support for Logitech receivers and devices.
+	Logitech receivers are capable of pairing multiple Logitech compliant
 	devices to the same receiver. Without this driver it will be handled by
 	generic USB_HID driver and all incoming events will be multiplexed
 	into a single mouse and a single keyboard device.
@@ -554,7 +555,7 @@ config HID_LOGITECH_HIDPP
 	tristate "Logitech HID++ devices support"
 	depends on HID_LOGITECH
 	select POWER_SUPPLY
-	---help---
+	help
 	Support for Logitech devices relyingon the HID++ Logitech specification
 
 	Say Y if you want support for Logitech devices relying on the HID++
@@ -620,7 +621,7 @@ config LOGIWHEELS_FF
 config HID_MAGICMOUSE
 	tristate "Apple Magic Mouse/Trackpad multi-touch support"
 	depends on HID
-	---help---
+	help
 	Support for the Apple Magic Mouse/Trackpad multi-touch.
 
 	Say Y here if you want support for the multi-touch features of the
@@ -629,7 +630,7 @@ config HID_MAGICMOUSE
 config HID_MALTRON
 	tristate "Maltron L90 keyboard"
 	depends on HID
-	---help---
+	help
 	Adds support for the volume up, volume down, mute, and play/pause buttons
 	of the Maltron L90 keyboard.
 
@@ -637,7 +638,7 @@ config HID_MAYFLASH
 	tristate "Mayflash game controller adapter force feedback"
 	depends on HID
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you have HJZ Mayflash PS3 game controller adapters
 	and want to enable force feedback support.
 
@@ -645,7 +646,7 @@ config HID_REDRAGON
 	tristate "Redragon keyboards"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
     Support for Redragon keyboards that need fix-ups to work properly.
 
 config HID_MICROSOFT
@@ -653,20 +654,20 @@ config HID_MICROSOFT
 	depends on HID
 	default !EXPERT
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Support for Microsoft devices that are not fully compliant with HID standard.
 
 config HID_MONTEREY
 	tristate "Monterey Genius KB29E keyboard"
 	depends on HID
 	default !EXPERT
-	---help---
+	help
 	Support for Monterey Genius KB29E.
 
 config HID_MULTITOUCH
 	tristate "HID Multitouch panels"
 	depends on HID
-	---help---
+	help
 	  Generic support for HID multitouch panels.
 
 	  Say Y here if you have one of the following devices:
@@ -711,20 +712,20 @@ config HID_MULTITOUCH
 
 config HID_NTI
 	tristate "NTI keyboard adapters"
-	---help---
+	help
 	Support for the "extra" Sun keyboard keys on keyboards attached
 	through Network Technologies USB-SUN keyboard adapters.
 
 config HID_NTRIG
 	tristate "N-Trig touch screen"
 	depends on USB_HID
-	---help---
+	help
 	Support for N-Trig touch screen.
 
 config HID_ORTEK
 	tristate "Ortek PKB-1700/WKB-2000/Skycable wireless keyboard and mouse trackpad"
 	depends on HID
-	---help---
+	help
 	There are certain devices which have LogicalMaximum wrong in the keyboard
 	usage page of their report descriptor. The most prevailing ones so far
 	are manufactured by Ortek, thus the name of the driver. Currently
@@ -737,7 +738,7 @@ config HID_ORTEK
 config HID_PANTHERLORD
 	tristate "Pantherlord/GreenAsia game controller"
 	depends on HID
-	---help---
+	help
 	  Say Y here if you have a PantherLord/GreenAsia based game controller
 	  or adapter.
 
@@ -745,14 +746,14 @@ config PANTHERLORD_FF
 	bool "Pantherlord force feedback support"
 	depends on HID_PANTHERLORD
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	  Say Y here if you have a PantherLord/GreenAsia based game controller
 	  or adapter and want to enable force feedback support for it.
 
 config HID_PENMOUNT
 	tristate "Penmount touch device"
 	depends on USB_HID
-	---help---
+	help
 	  This selects a driver for the PenMount 6000 touch controller.
 
 	  The driver works around a problem in the report descript allowing
@@ -763,13 +764,13 @@ config HID_PENMOUNT
 config HID_PETALYNX
 	tristate "Petalynx Maxter remote control"
 	depends on HID
-	---help---
+	help
 	Support for Petalynx Maxter remote control.
 
 config HID_PICOLCD
 	tristate "PicoLCD (graphic version)"
 	depends on HID
-	---help---
+	help
 	  This provides support for Minibox PicoLCD devices, currently
 	  only the graphical ones are supported.
 
@@ -795,7 +796,7 @@ config HID_PICOLCD_FB
 	select FB_SYS_COPYAREA
 	select FB_SYS_IMAGEBLIT
 	select FB_SYS_FOPS
-	---help---
+	help
 	  Provide access to PicoLCD's 256x64 monochrome display via a
 	  framebuffer device.
 
@@ -804,7 +805,7 @@ config HID_PICOLCD_BACKLIGHT
 	default !EXPERT
 	depends on HID_PICOLCD
 	depends on HID_PICOLCD=BACKLIGHT_CLASS_DEVICE || BACKLIGHT_CLASS_DEVICE=y
-	---help---
+	help
 	  Provide access to PicoLCD's backlight control via backlight
 	  class.
 
@@ -813,7 +814,7 @@ config HID_PICOLCD_LCD
 	default !EXPERT
 	depends on HID_PICOLCD
 	depends on HID_PICOLCD=LCD_CLASS_DEVICE || LCD_CLASS_DEVICE=y
-	---help---
+	help
 	  Provide access to PicoLCD's LCD contrast via lcd class.
 
 config HID_PICOLCD_LEDS
@@ -821,7 +822,7 @@ config HID_PICOLCD_LEDS
 	default !EXPERT
 	depends on HID_PICOLCD
 	depends on HID_PICOLCD=LEDS_CLASS || LEDS_CLASS=y
-	---help---
+	help
 	  Provide access to PicoLCD's GPO pins via leds class.
 
 config HID_PICOLCD_CIR
@@ -829,13 +830,13 @@ config HID_PICOLCD_CIR
 	default !EXPERT
 	depends on HID_PICOLCD
 	depends on HID_PICOLCD=RC_CORE || RC_CORE=y
-	---help---
+	help
 	  Provide access to PicoLCD's CIR interface via remote control (LIRC).
 
 config HID_PLANTRONICS
 	tristate "Plantronics USB HID Driver"
 	depends on HID
-	---help---
+	help
 	  Provides HID support for Plantronics USB audio devices.
 	  Correctly maps vendor unique volume up/down HID usages to
 	  KEY_VOLUMEUP and KEY_VOLUMEDOWN events and prevents core mapping
@@ -846,21 +847,21 @@ config HID_PLANTRONICS
 config HID_PRIMAX
 	tristate "Primax non-fully HID-compliant devices"
 	depends on HID
-	---help---
+	help
 	Support for Primax devices that are not fully compliant with the
 	HID standard.
 
 config HID_RETRODE
 	tristate "Retrode 2 USB adapter for vintage video games"
 	depends on USB_HID
-	---help---
+	help
 	Support for
 	  * Retrode 2 cartridge and controller adapter
 
 config HID_ROCCAT
 	tristate "Roccat device support"
 	depends on USB_HID
-	---help---
+	help
 	Support for Roccat devices.
 	Say Y here if you have a Roccat mouse or keyboard and want
 	support for its special functionalities.
@@ -868,7 +869,7 @@ config HID_ROCCAT
 config HID_SAITEK
 	tristate "Saitek (Mad Catz) non-fully HID-compliant devices"
 	depends on HID
-	---help---
+	help
 	Support for Saitek devices that are not fully compliant with the
 	HID standard.
 
@@ -880,7 +881,7 @@ config HID_SAITEK
 config HID_SAMSUNG
 	tristate "Samsung InfraRed remote control or keyboards"
 	depends on HID
-	---help---
+	help
 	Support for Samsung InfraRed remote control or keyboards.
 
 config HID_SONY
@@ -889,7 +890,7 @@ config HID_SONY
 	depends on NEW_LEDS
 	depends on LEDS_CLASS
 	select POWER_SUPPLY
-	---help---
+	help
 	Support for
 
 	  * Sony PS3 6-axis controllers
@@ -902,21 +903,21 @@ config SONY_FF
 	bool "Sony PS2/3/4 accessories force feedback support" 
 	depends on HID_SONY
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you have a Sony PS2/3/4 accessory and want to enable
 	force feedback support for it.
 
 config HID_SPEEDLINK
 	tristate "Speedlink VAD Cezanne mouse support"
 	depends on HID
-	---help---
+	help
 	Support for Speedlink Vicious and Divine Cezanne mouse.
 
 config HID_STEAM
 	tristate "Steam Controller support"
 	depends on HID
 	select POWER_SUPPLY
-	---help---
+	help
 	Say Y here if you have a Steam Controller if you want to use it
 	without running the Steam Client. It supports both the wired and
 	the wireless adaptor.
@@ -924,13 +925,13 @@ config HID_STEAM
 config HID_STEELSERIES
 	tristate "Steelseries SRW-S1 steering wheel support"
 	depends on HID
-	---help---
+	help
 	Support for Steelseries SRW-S1 steering wheel
 
 config HID_SUNPLUS
 	tristate "Sunplus wireless desktop"
 	depends on HID
-	---help---
+	help
 	Support for Sunplus wireless desktop.
 
 config HID_RMI
@@ -941,7 +942,7 @@ config HID_RMI
 	select RMI4_F11
 	select RMI4_F12
 	select RMI4_F30
-	---help---
+	help
 	Support for Synaptics RMI4 touchpads.
 	Say Y here if you have a Synaptics RMI4 touchpads over i2c-hid or usbhid
 	and want support for its special functionalities.
@@ -949,7 +950,7 @@ config HID_RMI
 config HID_GREENASIA
 	tristate "GreenAsia (Product ID 0x12) game controller support"
 	depends on HID
-	---help---
+	help
 	  Say Y here if you have a GreenAsia (Product ID 0x12) based game
 	  controller or adapter.
 
@@ -957,7 +958,7 @@ config GREENASIA_FF
 	bool "GreenAsia (Product ID 0x12) force feedback support"
 	depends on HID_GREENASIA
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you have a GreenAsia (Product ID 0x12) based game controller
 	(like MANTA Warrior MM816 and SpeedLink Strike2 SL-6635) or adapter
 	and want to enable force feedback support for it.
@@ -965,13 +966,13 @@ config GREENASIA_FF
 config HID_HYPERV_MOUSE
 	tristate "Microsoft Hyper-V mouse driver"
 	depends on HYPERV
-	---help---
+	help
 	Select this option to enable the Hyper-V mouse driver.
 
 config HID_SMARTJOYPLUS
 	tristate "SmartJoy PLUS PS2/USB adapter support"
 	depends on HID
-	---help---
+	help
 	Support for SmartJoy PLUS PS2/USB adapter, Super Dual Box,
 	Super Joy Box 3 Pro, Super Dual Box Pro, and Super Joy Box 5 Pro.
 
@@ -982,20 +983,20 @@ config SMARTJOYPLUS_FF
 	bool "SmartJoy PLUS PS2/USB adapter force feedback support"
 	depends on HID_SMARTJOYPLUS
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Say Y here if you have a SmartJoy PLUS PS2/USB adapter and want to
 	enable force feedback support for it.
 
 config HID_TIVO
 	tristate "TiVo Slide Bluetooth remote control support"
 	depends on HID
-	---help---
+	help
 	Say Y if you have a TiVo Slide Bluetooth remote control.
 
 config HID_TOPSEED
 	tristate "TopSeed Cyberlink, BTC Emprex, Conceptronic remote control support"
 	depends on HID
-	---help---
+	help
 	Say Y if you have a TopSeed Cyberlink or BTC Emprex or Conceptronic
 	CLLRCMCE remote control.
 
@@ -1004,7 +1005,7 @@ config HID_THINGM
 	depends on HID
 	depends on LEDS_CLASS
 	select HID_LED
-	---help---
+	help
 	Support for the ThingM blink(1) USB RGB LED. This driver has been
 	merged into the generic hid led driver. Config symbol HID_THINGM
 	just selects HID_LED and will be removed soon.
@@ -1012,7 +1013,7 @@ config HID_THINGM
 config HID_THRUSTMASTER
 	tristate "ThrustMaster devices support"
 	depends on HID
-	---help---
+	help
 	  Say Y here if you have a THRUSTMASTER FireStore Dual Power 2 or
 	  a THRUSTMASTER Ferrari GT Rumble Wheel.
 
@@ -1020,7 +1021,7 @@ config THRUSTMASTER_FF
 	bool "ThrustMaster devices force feedback support"
 	depends on HID_THRUSTMASTER
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	  Say Y here if you have a THRUSTMASTER FireStore Dual Power 2 or 3,
 	  a THRUSTMASTER Dual Trigger 3-in-1 or a THRUSTMASTER Ferrari GT
 	  Rumble Force or Force Feedback Wheel.
@@ -1028,7 +1029,7 @@ config THRUSTMASTER_FF
 config HID_UDRAW_PS3
 	tristate "THQ PS3 uDraw tablet"
 	depends on HID
-	---help---
+	help
 	  Say Y here if you want to use the THQ uDraw gaming tablet for
 	  the PS3.
 
@@ -1068,7 +1069,7 @@ config HID_WIIMOTE
 	depends on LEDS_CLASS
 	select POWER_SUPPLY
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	Support for Nintendo Wii and Wii U Bluetooth peripherals. Supported
 	devices are the Wii Remote and its extension devices, but also devices
 	based on the Wii Remote like the Wii U Pro Controller or the
@@ -1090,7 +1091,7 @@ config HID_WIIMOTE
 config HID_XINMO
 	tristate "Xin-Mo non-fully compliant devices"
 	depends on HID
-	---help---
+	help
 	Support for Xin-Mo devices that are not fully compliant with the HID
 	standard. Currently only supports the Xin-Mo Dual Arcade. Say Y here
 	if you have a Xin-Mo Dual Arcade controller.
@@ -1098,21 +1099,21 @@ config HID_XINMO
 config HID_ZEROPLUS
 	tristate "Zeroplus based game controller support"
 	depends on HID
-	---help---
+	help
 	  Say Y here if you have a Zeroplus based game controller.
 
 config ZEROPLUS_FF
 	bool "Zeroplus based game controller force feedback support"
 	depends on HID_ZEROPLUS
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	  Say Y here if you have a Zeroplus based game controller and want
 	  to have force feedback support for it.
 
 config HID_ZYDACRON
 	tristate "Zydacron remote control support"
 	depends on HID
-	---help---
+	help
 	Support for Zydacron remote control.
 
 config HID_SENSOR_HUB
@@ -1120,7 +1121,7 @@ config HID_SENSOR_HUB
 	depends on HID && HAS_IOMEM
 	select MFD_CORE
 	default n
-	---help---
+	help
 	  Support for HID Sensor framework. This creates a MFD instance
 	  for a sensor hub and identifies all the sensors connected to it.
 	  Each sensor is registered as a MFD cell, so that sensor specific
@@ -1133,21 +1134,21 @@ config HID_SENSOR_CUSTOM_SENSOR
 	tristate "HID Sensors hub custom sensor support"
 	depends on HID_SENSOR_HUB
 	default n
-	---help---
+	help
 	  HID Sensor hub specification allows definition of some custom and
 	  generic sensors. Unlike other HID sensors, they can't be exported
 	  via Linux IIO because of custom fields. This is up to the manufacturer
 	  to decide how to interpret these special sensor ids and process in
 	  the user space. Currently some manufacturers are using these ids for
 	  sensor calibration and debugging other sensors. Manufacturers
-	  should't use these special custom sensor ids to export any of the
+	  shouldn't use these special custom sensor ids to export any of the
 	  standard sensors.
 	  Select this config option for custom/generic sensor support.
 
 config HID_ALPS
 	tristate "Alps HID device support"
 	depends on HID
-	---help---
+	help
 	Support for Alps I2C HID touchpads and StickPointer.
 	Say Y here if you have a Alps touchpads over i2c-hid or usbhid
 	and want support for its special functionalities.
@@ -1156,7 +1157,7 @@ config HID_MCP2221
 	tristate "Microchip MCP2221 HID USB-to-I2C/SMbus host support"
 	depends on USB_HID && I2C
 	depends on GPIOLIB
-	---help---
+	help
 	Provides I2C and SMBUS host adapter functionality over USB-HID
 	through MCP2221 device.
 
diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c
index b2ad319a74b9..6f1fe7248d81 100644
--- a/drivers/hid/hid-alps.c
+++ b/drivers/hid/hid-alps.c
@@ -387,8 +387,7 @@ static int u1_raw_event(struct alps_dev *hdata, u8 *data, int size)
 				input_report_abs(hdata->input,
 					ABS_MT_PRESSURE, z);
 			} else {
-				input_mt_report_slot_state(hdata->input,
-					MT_TOOL_FINGER, 0);
+				input_mt_report_slot_inactive(hdata->input);
 			}
 		}
 
diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c
index d732d1d10caf..359bdfbe3701 100644
--- a/drivers/hid/hid-apple.c
+++ b/drivers/hid/hid-apple.c
@@ -51,6 +51,12 @@ MODULE_PARM_DESC(swap_opt_cmd, "Swap the Option (\"Alt\") and Command (\"Flag\")
 		"(For people who want to keep Windows PC keyboard muscle memory. "
 		"[0] = as-is, Mac layout. 1 = swapped, Windows layout.)");
 
+static unsigned int swap_fn_leftctrl;
+module_param(swap_fn_leftctrl, uint, 0644);
+MODULE_PARM_DESC(swap_fn_leftctrl, "Swap the Fn and left Control keys. "
+		"(For people who want to keep PC keyboard muscle memory. "
+		"[0] = as-is, Mac layout, 1 = swapped, PC layout)");
+
 struct apple_sc {
 	unsigned long quirks;
 	unsigned int fn_on;
@@ -162,6 +168,11 @@ static const struct apple_key_translation swapped_option_cmd_keys[] = {
 	{ }
 };
 
+static const struct apple_key_translation swapped_fn_leftctrl_keys[] = {
+	{ KEY_FN, KEY_LEFTCTRL },
+	{ }
+};
+
 static const struct apple_key_translation *apple_find_translation(
 		const struct apple_key_translation *table, u16 from)
 {
@@ -183,9 +194,11 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input,
 	bool do_translate;
 	u16 code = 0;
 
-	if (usage->code == KEY_FN) {
+	u16 fn_keycode = (swap_fn_leftctrl) ? (KEY_LEFTCTRL) : (KEY_FN);
+
+	if (usage->code == fn_keycode) {
 		asc->fn_on = !!value;
-		input_event(input, usage->type, usage->code, value);
+		input_event(input, usage->type, KEY_FN, value);
 		return 1;
 	}
 
@@ -270,6 +283,14 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input,
 		}
 	}
 
+	if (swap_fn_leftctrl) {
+		trans = apple_find_translation(swapped_fn_leftctrl_keys, usage->code);
+		if (trans) {
+			input_event(input, usage->type, trans->to, value);
+			return 1;
+		}
+	}
+
 	return 0;
 }
 
@@ -333,6 +354,11 @@ static void apple_setup_input(struct input_dev *input)
 
 	for (trans = apple_iso_keyboard; trans->from; trans++)
 		set_bit(trans->to, input->keybit);
+
+	if (swap_fn_leftctrl) {
+		for (trans = swapped_fn_leftctrl_keys; trans->from; trans++)
+			set_bit(trans->to, input->keybit);
+	}
 }
 
 static int apple_input_mapping(struct hid_device *hdev, struct hid_input *hi,
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index e6e4c841fb06..c183caf89d49 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -40,7 +40,9 @@ MODULE_AUTHOR("Frederik Wenigwieser <frederik.wenigwieser@gmail.com>");
 MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad");
 
 #define T100_TPAD_INTF 2
+#define MEDION_E1239T_TPAD_INTF 1
 
+#define E1239T_TP_TOGGLE_REPORT_ID 0x05
 #define T100CHI_MOUSE_REPORT_ID 0x06
 #define FEATURE_REPORT_ID 0x0d
 #define INPUT_REPORT_ID 0x5d
@@ -77,6 +79,7 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad");
 #define QUIRK_G752_KEYBOARD		BIT(8)
 #define QUIRK_T101HA_DOCK		BIT(9)
 #define QUIRK_T90CHI			BIT(10)
+#define QUIRK_MEDION_E1239T		BIT(11)
 
 #define I2C_KEYBOARD_QUIRKS			(QUIRK_FIX_NOTEBOOK_REPORT | \
 						 QUIRK_NO_INIT_REPORTS | \
@@ -102,12 +105,14 @@ struct asus_touchpad_info {
 	int res_y;
 	int contact_size;
 	int max_contacts;
+	int report_size;
 };
 
 struct asus_drvdata {
 	unsigned long quirks;
 	struct hid_device *hdev;
 	struct input_dev *input;
+	struct input_dev *tp_kbd_input;
 	struct asus_kbd_leds *kbd_backlight;
 	const struct asus_touchpad_info *tp;
 	bool enable_backlight;
@@ -126,6 +131,7 @@ static const struct asus_touchpad_info asus_i2c_tp = {
 	.max_y = 1758,
 	.contact_size = 5,
 	.max_contacts = 5,
+	.report_size = 28 /* 2 byte header + 5 * 5 + 1 byte footer */,
 };
 
 static const struct asus_touchpad_info asus_t100ta_tp = {
@@ -135,6 +141,7 @@ static const struct asus_touchpad_info asus_t100ta_tp = {
 	.res_y = 27, /* units/mm */
 	.contact_size = 5,
 	.max_contacts = 5,
+	.report_size = 28 /* 2 byte header + 5 * 5 + 1 byte footer */,
 };
 
 static const struct asus_touchpad_info asus_t100ha_tp = {
@@ -144,6 +151,7 @@ static const struct asus_touchpad_info asus_t100ha_tp = {
 	.res_y = 29, /* units/mm */
 	.contact_size = 5,
 	.max_contacts = 5,
+	.report_size = 28 /* 2 byte header + 5 * 5 + 1 byte footer */,
 };
 
 static const struct asus_touchpad_info asus_t200ta_tp = {
@@ -153,6 +161,7 @@ static const struct asus_touchpad_info asus_t200ta_tp = {
 	.res_y = 28, /* units/mm */
 	.contact_size = 5,
 	.max_contacts = 5,
+	.report_size = 28 /* 2 byte header + 5 * 5 + 1 byte footer */,
 };
 
 static const struct asus_touchpad_info asus_t100chi_tp = {
@@ -162,6 +171,17 @@ static const struct asus_touchpad_info asus_t100chi_tp = {
 	.res_y = 29, /* units/mm */
 	.contact_size = 3,
 	.max_contacts = 4,
+	.report_size = 15 /* 2 byte header + 3 * 4 + 1 byte footer */,
+};
+
+static const struct asus_touchpad_info medion_e1239t_tp = {
+	.max_x = 2640,
+	.max_y = 1380,
+	.res_x = 29, /* units/mm */
+	.res_y = 28, /* units/mm */
+	.contact_size = 5,
+	.max_contacts = 5,
+	.report_size = 32 /* 2 byte header + 5 * 5 + 5 byte footer */,
 };
 
 static void asus_report_contact_down(struct asus_drvdata *drvdat,
@@ -229,7 +249,7 @@ static int asus_report_input(struct asus_drvdata *drvdat, u8 *data, int size)
 	int i, toolType = MT_TOOL_FINGER;
 	u8 *contactData = data + 2;
 
-	if (size != 3 + drvdat->tp->contact_size * drvdat->tp->max_contacts)
+	if (size != drvdat->tp->report_size)
 		return 0;
 
 	for (i = 0; i < drvdat->tp->max_contacts; i++) {
@@ -257,6 +277,34 @@ static int asus_report_input(struct asus_drvdata *drvdat, u8 *data, int size)
 	return 1;
 }
 
+static int asus_e1239t_event(struct asus_drvdata *drvdat, u8 *data, int size)
+{
+	if (size != 3)
+		return 0;
+
+	/* Handle broken mute key which only sends press events */
+	if (!drvdat->tp &&
+	    data[0] == 0x02 && data[1] == 0xe2 && data[2] == 0x00) {
+		input_report_key(drvdat->input, KEY_MUTE, 1);
+		input_sync(drvdat->input);
+		input_report_key(drvdat->input, KEY_MUTE, 0);
+		input_sync(drvdat->input);
+		return 1;
+	}
+
+	/* Handle custom touchpad toggle key which only sends press events */
+	if (drvdat->tp_kbd_input &&
+	    data[0] == 0x05 && data[1] == 0x02 && data[2] == 0x28) {
+		input_report_key(drvdat->tp_kbd_input, KEY_F21, 1);
+		input_sync(drvdat->tp_kbd_input);
+		input_report_key(drvdat->tp_kbd_input, KEY_F21, 0);
+		input_sync(drvdat->tp_kbd_input);
+		return 1;
+	}
+
+	return 0;
+}
+
 static int asus_event(struct hid_device *hdev, struct hid_field *field,
 		      struct hid_usage *usage, __s32 value)
 {
@@ -281,6 +329,9 @@ static int asus_raw_event(struct hid_device *hdev,
 	if (drvdata->tp && data[0] == INPUT_REPORT_ID)
 		return asus_report_input(drvdata, data, size);
 
+	if (drvdata->quirks & QUIRK_MEDION_E1239T)
+		return asus_e1239t_event(drvdata, data, size);
+
 	return 0;
 }
 
@@ -615,6 +666,21 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi)
 	    hi->report->id != T100CHI_MOUSE_REPORT_ID)
 		return 0;
 
+	/* Handle MULTI_INPUT on E1239T mouse/touchpad USB interface */
+	if (drvdata->tp && (drvdata->quirks & QUIRK_MEDION_E1239T)) {
+		switch (hi->report->id) {
+		case E1239T_TP_TOGGLE_REPORT_ID:
+			input_set_capability(input, EV_KEY, KEY_F21);
+			input->name = "Asus Touchpad Keys";
+			drvdata->tp_kbd_input = input;
+			return 0;
+		case INPUT_REPORT_ID:
+			break; /* Touchpad report, handled below */
+		default:
+			return 0; /* Ignore other reports */
+		}
+	}
+
 	if (drvdata->tp) {
 		int ret;
 
@@ -677,24 +743,16 @@ static int asus_input_mapping(struct hid_device *hdev,
 	 * This avoids a bunch of non-functional hid_input devices getting
 	 * created because of the T100CHI using HID_QUIRK_MULTI_INPUT.
 	 */
-	if (drvdata->quirks & (QUIRK_T100CHI | QUIRK_T90CHI)) {
-		if (field->application == (HID_UP_GENDESK | 0x0080) ||
-		    usage->hid == (HID_UP_GENDEVCTRLS | 0x0024) ||
-		    usage->hid == (HID_UP_GENDEVCTRLS | 0x0025) ||
-		    usage->hid == (HID_UP_GENDEVCTRLS | 0x0026))
-			return -1;
-		/*
-		 * We use the hid_input for the mouse report for the touchpad,
-		 * keep the left button, to avoid the core removing it.
-		 */
-		if (field->application == HID_GD_MOUSE &&
-		    usage->hid != (HID_UP_BUTTON | 1))
-			return -1;
-	}
+	if ((drvdata->quirks & (QUIRK_T100CHI | QUIRK_T90CHI)) &&
+	    (field->application == (HID_UP_GENDESK | 0x0080) ||
+	     field->application == HID_GD_MOUSE ||
+	     usage->hid == (HID_UP_GENDEVCTRLS | 0x0024) ||
+	     usage->hid == (HID_UP_GENDEVCTRLS | 0x0025) ||
+	     usage->hid == (HID_UP_GENDEVCTRLS | 0x0026)))
+		return -1;
 
 	/* ASUS-specific keyboard hotkeys */
 	if ((usage->hid & HID_USAGE_PAGE) == 0xff310000) {
-		set_bit(EV_REP, hi->input->evbit);
 		switch (usage->hid & HID_USAGE) {
 		case 0x10: asus_map_key_clear(KEY_BRIGHTNESSDOWN);	break;
 		case 0x20: asus_map_key_clear(KEY_BRIGHTNESSUP);		break;
@@ -737,11 +795,11 @@ static int asus_input_mapping(struct hid_device *hdev,
 		if (drvdata->quirks & QUIRK_USE_KBD_BACKLIGHT)
 			drvdata->enable_backlight = true;
 
+		set_bit(EV_REP, hi->input->evbit);
 		return 1;
 	}
 
 	if ((usage->hid & HID_USAGE_PAGE) == HID_UP_MSVENDOR) {
-		set_bit(EV_REP, hi->input->evbit);
 		switch (usage->hid & HID_USAGE) {
 		case 0xff01: asus_map_key_clear(BTN_1);	break;
 		case 0xff02: asus_map_key_clear(BTN_2);	break;
@@ -764,6 +822,7 @@ static int asus_input_mapping(struct hid_device *hdev,
 			return 0;
 		}
 
+		set_bit(EV_REP, hi->input->evbit);
 		return 1;
 	}
 
@@ -782,6 +841,16 @@ static int asus_input_mapping(struct hid_device *hdev,
 		}
 	}
 
+	/*
+	 * The mute button is broken and only sends press events, we
+	 * deal with this in our raw_event handler, so do not map it.
+	 */
+	if ((drvdata->quirks & QUIRK_MEDION_E1239T) &&
+	    usage->hid == (HID_UP_CONSUMER | 0xe2)) {
+		input_set_capability(hi->input, EV_KEY, KEY_MUTE);
+		return -1;
+	}
+
 	return 0;
 }
 
@@ -849,7 +918,8 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	if (drvdata->quirks & QUIRK_IS_MULTITOUCH)
 		drvdata->tp = &asus_i2c_tp;
 
-	if (drvdata->quirks & QUIRK_T100_KEYBOARD) {
+	if ((drvdata->quirks & QUIRK_T100_KEYBOARD) &&
+	    hid_is_using_ll_driver(hdev, &usb_hid_driver)) {
 		struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
 
 		if (intf->altsetting->desc.bInterfaceNumber == T100_TPAD_INTF) {
@@ -877,6 +947,19 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		drvdata->tp = &asus_t100chi_tp;
 	}
 
+	if ((drvdata->quirks & QUIRK_MEDION_E1239T) &&
+	    hid_is_using_ll_driver(hdev, &usb_hid_driver)) {
+		struct usb_host_interface *alt =
+			to_usb_interface(hdev->dev.parent)->altsetting;
+
+		if (alt->desc.bInterfaceNumber == MEDION_E1239T_TPAD_INTF) {
+			/* For separate input-devs for tp and tp toggle key */
+			hdev->quirks |= HID_QUIRK_MULTI_INPUT;
+			drvdata->quirks |= QUIRK_SKIP_INPUT_MAPPING;
+			drvdata->tp = &medion_e1239t_tp;
+		}
+	}
+
 	if (drvdata->quirks & QUIRK_NO_INIT_REPORTS)
 		hdev->quirks |= HID_QUIRK_NO_INIT_REPORTS;
 
@@ -1056,7 +1139,8 @@ static const struct hid_device_id asus_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_JESS, USB_DEVICE_ID_ASUS_MD_5112) },
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ASUSTEK,
 		USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD), QUIRK_T100CHI },
-
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ITE, USB_DEVICE_ID_ITE_MEDION_E1239T),
+		QUIRK_MEDION_E1239T },
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, asus_devices);
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 1c71a1aa76b2..874fc3791f3b 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -76,12 +76,9 @@
 
 #define USB_VENDOR_ID_ALPS_JP		0x044E
 #define HID_DEVICE_ID_ALPS_U1_DUAL	0x120B
-#define HID_DEVICE_ID_ALPS_U1_DUAL_PTP	0x121F
-#define HID_DEVICE_ID_ALPS_U1_DUAL_3BTN_PTP	0x1220
 #define HID_DEVICE_ID_ALPS_U1		0x1215
 #define HID_DEVICE_ID_ALPS_U1_UNICORN_LEGACY         0x121E
 #define HID_DEVICE_ID_ALPS_T4_BTNLESS	0x120C
-#define HID_DEVICE_ID_ALPS_1222		0x1222
 
 #define USB_VENDOR_ID_AMI		0x046b
 #define USB_DEVICE_ID_AMI_VIRT_KEYBOARD_AND_MOUSE	0xff10
@@ -281,9 +278,6 @@
 
 #define USB_VENDOR_ID_CIDC		0x1677
 
-#define I2C_VENDOR_ID_CIRQUE		0x0488
-#define I2C_PRODUCT_ID_CIRQUE_121F	0x121F
-
 #define USB_VENDOR_ID_CJTOUCH		0x24b8
 #define USB_DEVICE_ID_CJTOUCH_MULTI_TOUCH_0020	0x0020
 #define USB_DEVICE_ID_CJTOUCH_MULTI_TOUCH_0040	0x0040
@@ -640,6 +634,7 @@
 #define I2C_DEVICE_ID_ITE_LENOVO_LEGION_Y720	0x837a
 #define USB_DEVICE_ID_ITE_LENOVO_YOGA900	0x8396
 #define USB_DEVICE_ID_ITE8595		0x8595
+#define USB_DEVICE_ID_ITE_MEDION_E1239T	0xce50
 
 #define USB_VENDOR_ID_JABRA		0x0b0e
 #define USB_DEVICE_ID_JABRA_SPEAK_410	0x0412
@@ -730,8 +725,6 @@
 #define USB_DEVICE_ID_LENOVO_SCROLLPOINT_OPTICAL	0x6049
 #define USB_DEVICE_ID_LENOVO_TPPRODOCK	0x6067
 #define USB_DEVICE_ID_LENOVO_X1_COVER	0x6085
-#define USB_DEVICE_ID_LENOVO_X1_TAB	0x60a3
-#define USB_DEVICE_ID_LENOVO_X1_TAB3	0x60b5
 #define USB_DEVICE_ID_LENOVO_PIXART_USB_MOUSE_608D	0x608d
 
 #define USB_VENDOR_ID_LG		0x1fd2
@@ -1157,6 +1150,9 @@
 #define USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8882	0x8882
 #define USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8883	0x8883
 
+#define USB_VENDOR_ID_TRUST             0x145f
+#define USB_DEVICE_ID_TRUST_PANORA_TABLET   0x0212
+
 #define USB_VENDOR_ID_TURBOX		0x062a
 #define USB_DEVICE_ID_TURBOX_KEYBOARD	0x0201
 #define USB_DEVICE_ID_ASUS_MD_5110	0x5110
diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c
index ed9b1c1f460d..48dff5d6b605 100644
--- a/drivers/hid/hid-logitech-dj.c
+++ b/drivers/hid/hid-logitech-dj.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- *  HID driver for Logitech Unifying receivers
+ *  HID driver for Logitech receivers
  *
  *  Copyright (c) 2011 Logitech
  */
@@ -701,7 +701,7 @@ static void logi_dj_recv_add_djhid_device(struct dj_receiver_dev *djrcv_dev,
 			type_str, dj_hiddev->product);
 	} else {
 		snprintf(dj_hiddev->name, sizeof(dj_hiddev->name),
-			"Logitech Unifying Device. Wireless PID:%04x",
+			"Logitech Wireless Device PID:%04x",
 			dj_hiddev->product);
 	}
 
diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c
index 094f4f1b6555..1e1cf8eae649 100644
--- a/drivers/hid/hid-logitech-hidpp.c
+++ b/drivers/hid/hid-logitech-hidpp.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- *  HIDPP protocol for Logitech Unifying receivers
+ *  HIDPP protocol for Logitech receivers
  *
  *  Copyright (c) 2011 Logitech (c)
  *  Copyright (c) 2012-2013 Google (c)
diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c
index d958475f8c81..e1b93ce32e01 100644
--- a/drivers/hid/hid-mcp2221.c
+++ b/drivers/hid/hid-mcp2221.c
@@ -15,6 +15,7 @@
 #include <linux/hid.h>
 #include <linux/hidraw.h>
 #include <linux/i2c.h>
+#include <linux/gpio/driver.h>
 #include "hid-ids.h"
 
 /* Commands codes in a raw output report */
@@ -27,6 +28,8 @@ enum {
 	MCP2221_I2C_PARAM_OR_STATUS	= 0x10,
 	MCP2221_I2C_SET_SPEED = 0x20,
 	MCP2221_I2C_CANCEL = 0x10,
+	MCP2221_GPIO_SET = 0x50,
+	MCP2221_GPIO_GET = 0x51,
 };
 
 /* Response codes in a raw input report */
@@ -42,6 +45,8 @@ enum {
 	MCP2221_I2C_WRADDRL_SEND = 0x21,
 	MCP2221_I2C_ADDR_NACK = 0x25,
 	MCP2221_I2C_READ_COMPL = 0x55,
+	MCP2221_ALT_F_NOT_GPIOV = 0xEE,
+	MCP2221_ALT_F_NOT_GPIOD = 0xEF,
 };
 
 /*
@@ -59,6 +64,9 @@ struct mcp2221 {
 	int rxbuf_idx;
 	int status;
 	u8 cur_i2c_clk_div;
+	struct gpio_chip *gc;
+	u8 gp_idx;
+	u8 gpio_dir;
 };
 
 /*
@@ -526,6 +534,110 @@ static const struct i2c_algorithm mcp_i2c_algo = {
 	.functionality = mcp_i2c_func,
 };
 
+static int mcp_gpio_get(struct gpio_chip *gc,
+				unsigned int offset)
+{
+	int ret;
+	struct mcp2221 *mcp = gpiochip_get_data(gc);
+
+	mcp->txbuf[0] = MCP2221_GPIO_GET;
+
+	mcp->gp_idx = (offset + 1) * 2;
+
+	mutex_lock(&mcp->lock);
+	ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1);
+	mutex_unlock(&mcp->lock);
+
+	return ret;
+}
+
+static void mcp_gpio_set(struct gpio_chip *gc,
+				unsigned int offset, int value)
+{
+	struct mcp2221 *mcp = gpiochip_get_data(gc);
+
+	memset(mcp->txbuf, 0, 18);
+	mcp->txbuf[0] = MCP2221_GPIO_SET;
+
+	mcp->gp_idx = ((offset + 1) * 4) - 1;
+
+	mcp->txbuf[mcp->gp_idx - 1] = 1;
+	mcp->txbuf[mcp->gp_idx] = !!value;
+
+	mutex_lock(&mcp->lock);
+	mcp_send_data_req_status(mcp, mcp->txbuf, 18);
+	mutex_unlock(&mcp->lock);
+}
+
+static int mcp_gpio_dir_set(struct mcp2221 *mcp,
+				unsigned int offset, u8 val)
+{
+	memset(mcp->txbuf, 0, 18);
+	mcp->txbuf[0] = MCP2221_GPIO_SET;
+
+	mcp->gp_idx = (offset + 1) * 5;
+
+	mcp->txbuf[mcp->gp_idx - 1] = 1;
+	mcp->txbuf[mcp->gp_idx] = val;
+
+	return mcp_send_data_req_status(mcp, mcp->txbuf, 18);
+}
+
+static int mcp_gpio_direction_input(struct gpio_chip *gc,
+				unsigned int offset)
+{
+	int ret;
+	struct mcp2221 *mcp = gpiochip_get_data(gc);
+
+	mutex_lock(&mcp->lock);
+	ret = mcp_gpio_dir_set(mcp, offset, 0);
+	mutex_unlock(&mcp->lock);
+
+	return ret;
+}
+
+static int mcp_gpio_direction_output(struct gpio_chip *gc,
+				unsigned int offset, int value)
+{
+	int ret;
+	struct mcp2221 *mcp = gpiochip_get_data(gc);
+
+	mutex_lock(&mcp->lock);
+	ret = mcp_gpio_dir_set(mcp, offset, 1);
+	mutex_unlock(&mcp->lock);
+
+	/* Can't configure as output, bailout early */
+	if (ret)
+		return ret;
+
+	mcp_gpio_set(gc, offset, value);
+
+	return 0;
+}
+
+static int mcp_gpio_get_direction(struct gpio_chip *gc,
+				unsigned int offset)
+{
+	int ret;
+	struct mcp2221 *mcp = gpiochip_get_data(gc);
+
+	mcp->txbuf[0] = MCP2221_GPIO_GET;
+
+	mcp->gp_idx = (offset + 1) * 2;
+
+	mutex_lock(&mcp->lock);
+	ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1);
+	mutex_unlock(&mcp->lock);
+
+	if (ret)
+		return ret;
+
+	if (mcp->gpio_dir)
+		return GPIO_LINE_DIRECTION_IN;
+
+	return GPIO_LINE_DIRECTION_OUT;
+}
+
 /* Gives current state of i2c engine inside mcp2221 */
 static int mcp_get_i2c_eng_state(struct mcp2221 *mcp,
 				u8 *data, u8 idx)
@@ -638,6 +750,39 @@ static int mcp2221_raw_event(struct hid_device *hdev,
 		complete(&mcp->wait_in_report);
 		break;
 
+	case MCP2221_GPIO_GET:
+		switch (data[1]) {
+		case MCP2221_SUCCESS:
+			if ((data[mcp->gp_idx] == MCP2221_ALT_F_NOT_GPIOV) ||
+				(data[mcp->gp_idx + 1] == MCP2221_ALT_F_NOT_GPIOD)) {
+				mcp->status = -ENOENT;
+			} else {
+				mcp->status = !!data[mcp->gp_idx];
+				mcp->gpio_dir = !!data[mcp->gp_idx + 1];
+			}
+			break;
+		default:
+			mcp->status = -EAGAIN;
+		}
+		complete(&mcp->wait_in_report);
+		break;
+
+	case MCP2221_GPIO_SET:
+		switch (data[1]) {
+		case MCP2221_SUCCESS:
+			if ((data[mcp->gp_idx] == MCP2221_ALT_F_NOT_GPIOV) ||
+				(data[mcp->gp_idx - 1] == MCP2221_ALT_F_NOT_GPIOV)) {
+				mcp->status = -ENOENT;
+			} else {
+				mcp->status = 0;
+			}
+			break;
+		default:
+			mcp->status = -EAGAIN;
+		}
+		complete(&mcp->wait_in_report);
+		break;
+
 	default:
 		mcp->status = -EIO;
 		complete(&mcp->wait_in_report);
@@ -702,8 +847,32 @@ static int mcp2221_probe(struct hid_device *hdev,
 	}
 	i2c_set_adapdata(&mcp->adapter, mcp);
 
+	/* Setup GPIO chip */
+	mcp->gc = devm_kzalloc(&hdev->dev, sizeof(*mcp->gc), GFP_KERNEL);
+	if (!mcp->gc) {
+		ret = -ENOMEM;
+		goto err_gc;
+	}
+
+	mcp->gc->label = "mcp2221_gpio";
+	mcp->gc->direction_input = mcp_gpio_direction_input;
+	mcp->gc->direction_output = mcp_gpio_direction_output;
+	mcp->gc->get_direction = mcp_gpio_get_direction;
+	mcp->gc->set = mcp_gpio_set;
+	mcp->gc->get = mcp_gpio_get;
+	mcp->gc->ngpio = 4;
+	mcp->gc->base = -1;
+	mcp->gc->can_sleep = 1;
+	mcp->gc->parent = &hdev->dev;
+
+	ret = devm_gpiochip_add_data(&hdev->dev, mcp->gc, mcp);
+	if (ret)
+		goto err_gc;
+
 	return 0;
 
+err_gc:
+	i2c_del_adapter(&mcp->adapter);
 err_i2c:
 	hid_hw_close(mcp->hdev);
 err_hstop:
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index 03c720b47306..3f94b4954225 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -69,6 +69,7 @@ MODULE_LICENSE("GPL");
 #define MT_QUIRK_ASUS_CUSTOM_UP		BIT(17)
 #define MT_QUIRK_WIN8_PTP_BUTTONS	BIT(18)
 #define MT_QUIRK_SEPARATE_APP_REPORT	BIT(19)
+#define MT_QUIRK_FORCE_MULTI_INPUT	BIT(20)
 
 #define MT_INPUTMODE_TOUCHSCREEN	0x02
 #define MT_INPUTMODE_TOUCHPAD		0x03
@@ -188,7 +189,8 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app);
 /* reserved					0x0011 */
 #define MT_CLS_WIN_8				0x0012
 #define MT_CLS_EXPORT_ALL_INPUTS		0x0013
-#define MT_CLS_WIN_8_DUAL			0x0014
+/* reserved					0x0014 */
+#define MT_CLS_WIN_8_FORCE_MULTI_INPUT		0x0015
 
 /* vendor specific classes */
 #define MT_CLS_3M				0x0101
@@ -272,12 +274,14 @@ static const struct mt_class mt_classes[] = {
 		.quirks = MT_QUIRK_ALWAYS_VALID |
 			MT_QUIRK_CONTACT_CNT_ACCURATE,
 		.export_all_inputs = true },
-	{ .name = MT_CLS_WIN_8_DUAL,
+	{ .name = MT_CLS_WIN_8_FORCE_MULTI_INPUT,
 		.quirks = MT_QUIRK_ALWAYS_VALID |
 			MT_QUIRK_IGNORE_DUPLICATES |
 			MT_QUIRK_HOVERING |
 			MT_QUIRK_CONTACT_CNT_ACCURATE |
-			MT_QUIRK_WIN8_PTP_BUTTONS,
+			MT_QUIRK_STICKY_FINGERS |
+			MT_QUIRK_WIN8_PTP_BUTTONS |
+			MT_QUIRK_FORCE_MULTI_INPUT,
 		.export_all_inputs = true },
 
 	/*
@@ -754,8 +758,7 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 			MT_STORE_FIELD(inrange_state);
 			return 1;
 		case HID_DG_CONFIDENCE:
-			if ((cls->name == MT_CLS_WIN_8 ||
-				cls->name == MT_CLS_WIN_8_DUAL) &&
+			if (cls->name == MT_CLS_WIN_8 &&
 				(field->application == HID_DG_TOUCHPAD ||
 				 field->application == HID_DG_TOUCHSCREEN))
 				app->quirks |= MT_QUIRK_CONFIDENCE;
@@ -896,7 +899,7 @@ static void mt_release_pending_palms(struct mt_device *td,
 		clear_bit(slotnum, app->pending_palm_slots);
 
 		input_mt_slot(input, slotnum);
-		input_mt_report_slot_state(input, MT_TOOL_PALM, false);
+		input_mt_report_slot_inactive(input);
 
 		need_sync = true;
 	}
@@ -1640,9 +1643,7 @@ static void mt_release_contacts(struct hid_device *hid)
 		if (mt) {
 			for (i = 0; i < mt->num_slots; i++) {
 				input_mt_slot(input_dev, i);
-				input_mt_report_slot_state(input_dev,
-							   MT_TOOL_FINGER,
-							   false);
+				input_mt_report_slot_inactive(input_dev);
 			}
 			input_mt_sync_frame(input_dev);
 			input_sync(input_dev);
@@ -1714,6 +1715,11 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	if (id->group != HID_GROUP_MULTITOUCH_WIN_8)
 		hdev->quirks |= HID_QUIRK_MULTI_INPUT;
 
+	if (mtclass->quirks & MT_QUIRK_FORCE_MULTI_INPUT) {
+		hdev->quirks &= ~HID_QUIRK_INPUT_PER_APP;
+		hdev->quirks |= HID_QUIRK_MULTI_INPUT;
+	}
+
 	timer_setup(&td->release_timer, mt_expired_timeout, 0);
 
 	ret = hid_parse(hdev);
@@ -1786,32 +1792,6 @@ static const struct hid_device_id mt_devices[] = {
 		MT_USB_DEVICE(USB_VENDOR_ID_3M,
 			USB_DEVICE_ID_3M3266) },
 
-	/* Alps devices */
-	{ .driver_data = MT_CLS_WIN_8_DUAL,
-		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
-			USB_VENDOR_ID_ALPS_JP,
-			HID_DEVICE_ID_ALPS_U1_DUAL_PTP) },
-	{ .driver_data = MT_CLS_WIN_8_DUAL,
-		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
-			USB_VENDOR_ID_ALPS_JP,
-			HID_DEVICE_ID_ALPS_U1_DUAL_3BTN_PTP) },
-	{ .driver_data = MT_CLS_WIN_8_DUAL,
-		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
-			USB_VENDOR_ID_ALPS_JP,
-			HID_DEVICE_ID_ALPS_1222) },
-
-	/* Lenovo X1 TAB Gen 2 */
-	{ .driver_data = MT_CLS_WIN_8_DUAL,
-		HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8,
-			   USB_VENDOR_ID_LENOVO,
-			   USB_DEVICE_ID_LENOVO_X1_TAB) },
-
-	/* Lenovo X1 TAB Gen 3 */
-	{ .driver_data = MT_CLS_WIN_8_DUAL,
-		HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8,
-			   USB_VENDOR_ID_LENOVO,
-			   USB_DEVICE_ID_LENOVO_X1_TAB3) },
-
 	/* Anton devices */
 	{ .driver_data = MT_CLS_EXPORT_ALL_INPUTS,
 		MT_USB_DEVICE(USB_VENDOR_ID_ANTON,
@@ -1846,12 +1826,6 @@ static const struct hid_device_id mt_devices[] = {
 		MT_USB_DEVICE(USB_VENDOR_ID_CHUNGHWAT,
 			USB_DEVICE_ID_CHUNGHWAT_MULTITOUCH) },
 
-	/* Cirque devices */
-	{ .driver_data = MT_CLS_WIN_8_DUAL,
-		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
-			I2C_VENDOR_ID_CIRQUE,
-			I2C_PRODUCT_ID_CIRQUE_121F) },
-
 	/* CJTouch panels */
 	{ .driver_data = MT_CLS_NSMU,
 		MT_USB_DEVICE(USB_VENDOR_ID_CJTOUCH,
@@ -1926,6 +1900,11 @@ static const struct hid_device_id mt_devices[] = {
 		MT_USB_DEVICE(USB_VENDOR_ID_DWAV,
 			USB_DEVICE_ID_DWAV_EGALAX_MULTITOUCH_C002) },
 
+	/* Elan devices */
+	{ .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT,
+		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
+			USB_VENDOR_ID_ELAN, 0x313a) },
+
 	/* Elitegroup panel */
 	{ .driver_data = MT_CLS_SERIAL,
 		MT_USB_DEVICE(USB_VENDOR_ID_ELITEGROUP,
@@ -2056,6 +2035,11 @@ static const struct hid_device_id mt_devices[] = {
 		MT_USB_DEVICE(USB_VENDOR_ID_STANTUM_STM,
 			USB_DEVICE_ID_MTP_STM)},
 
+	/* Synaptics devices */
+	{ .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT,
+		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
+			USB_VENDOR_ID_SYNAPTICS, 0xce08) },
+
 	/* TopSeed panels */
 	{ .driver_data = MT_CLS_TOPSEED,
 		MT_USB_DEVICE(USB_VENDOR_ID_TOPSEED2,
diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
index e4cb543de0cd..ca8b5c261c7c 100644
--- a/drivers/hid/hid-quirks.c
+++ b/drivers/hid/hid-quirks.c
@@ -168,6 +168,7 @@ static const struct hid_device_id hid_quirks[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_TOUCHPACK, USB_DEVICE_ID_TOUCHPACK_RTS), HID_QUIRK_MULTI_INPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_TPV, USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8882), HID_QUIRK_NOGET },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_TPV, USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8883), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TRUST, USB_DEVICE_ID_TRUST_PANORA_TABLET), HID_QUIRK_MULTI_INPUT | HID_QUIRK_HIDINPUT_FORCE },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_TURBOX_KEYBOARD), HID_QUIRK_NOGET },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_KNA5), HID_QUIRK_MULTI_INPUT },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWA60), HID_QUIRK_MULTI_INPUT },
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index 4c6ed6ef31f1..2f073f536070 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -867,6 +867,23 @@ static u8 *sony_report_fixup(struct hid_device *hdev, u8 *rdesc,
 	if (sc->quirks & PS3REMOTE)
 		return ps3remote_fixup(hdev, rdesc, rsize);
 
+	/*
+	 * Some knock-off USB dongles incorrectly report their button count
+	 * as 13 instead of 16 causing three non-functional buttons.
+	 */
+	if ((sc->quirks & SIXAXIS_CONTROLLER_USB) && *rsize >= 45 &&
+		/* Report Count (13) */
+		rdesc[23] == 0x95 && rdesc[24] == 0x0D &&
+		/* Usage Maximum (13) */
+		rdesc[37] == 0x29 && rdesc[38] == 0x0D &&
+		/* Report Count (3) */
+		rdesc[43] == 0x95 && rdesc[44] == 0x03) {
+		hid_info(hdev, "Fixing up USB dongle report descriptor\n");
+		rdesc[24] = 0x10;
+		rdesc[38] = 0x10;
+		rdesc[44] = 0x00;
+	}
+
 	return rdesc;
 }
 
diff --git a/drivers/hid/i2c-hid/Kconfig b/drivers/hid/i2c-hid/Kconfig
index 0e2ae47f38a7..c4e5dfeab2bd 100644
--- a/drivers/hid/i2c-hid/Kconfig
+++ b/drivers/hid/i2c-hid/Kconfig
@@ -7,7 +7,7 @@ config I2C_HID
 	default n
 	depends on I2C && INPUT
 	select HID
-	---help---
+	help
 	  Say Y here if you use a keyboard, a touchpad, a touchscreen, or any
 	  other HID based devices which is connected to your computer via I2C.
 
diff --git a/drivers/hid/i2c-hid/i2c-hid-dmi-quirks.c b/drivers/hid/i2c-hid/i2c-hid-dmi-quirks.c
index a66f08041a1a..ec142bc8c1da 100644
--- a/drivers/hid/i2c-hid/i2c-hid-dmi-quirks.c
+++ b/drivers/hid/i2c-hid/i2c-hid-dmi-quirks.c
@@ -389,6 +389,14 @@ static const struct dmi_system_id i2c_hid_dmi_desc_override_table[] = {
 		},
 		.driver_data = (void *)&sipodev_desc
 	},
+	{
+		.ident = "Schneider SCL142ALM",
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "SCHNEIDER"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "SCL142ALM"),
+		},
+		.driver_data = (void *)&sipodev_desc
+	},
 	{ }	/* Terminate list */
 };
 
diff --git a/drivers/hid/intel-ish-hid/ishtp-fw-loader.c b/drivers/hid/intel-ish-hid/ishtp-fw-loader.c
index aa2dbed30fc3..6cf59fd26ad7 100644
--- a/drivers/hid/intel-ish-hid/ishtp-fw-loader.c
+++ b/drivers/hid/intel-ish-hid/ishtp-fw-loader.c
@@ -480,6 +480,7 @@ static int ish_query_loader_prop(struct ishtp_cl_data *client_data,
 			    sizeof(ldr_xfer_query_resp));
 	if (rv < 0) {
 		client_data->flag_retry = true;
+		*fw_info = (struct shim_fw_info){};
 		return rv;
 	}
 
@@ -489,6 +490,7 @@ static int ish_query_loader_prop(struct ishtp_cl_data *client_data,
 			"data size %d is not equal to size of loader_xfer_query_response %zu\n",
 			rv, sizeof(struct loader_xfer_query_response));
 		client_data->flag_retry = true;
+		*fw_info = (struct shim_fw_info){};
 		return -EMSGSIZE;
 	}
 
diff --git a/drivers/hid/usbhid/Kconfig b/drivers/hid/usbhid/Kconfig
index b5f3a3c4149e..dcf3a235870f 100644
--- a/drivers/hid/usbhid/Kconfig
+++ b/drivers/hid/usbhid/Kconfig
@@ -7,7 +7,7 @@ config USB_HID
 	default y
 	depends on USB && INPUT
 	select HID
-	---help---
+	help
 	  Say Y here if you want to connect USB keyboards,
 	  mice, joysticks, graphic tablets, or any other HID based devices
 	  to your computer via USB, as well as Uninterruptible Power Supply
@@ -51,7 +51,7 @@ menu "USB HID Boot Protocol drivers"
 config USB_KBD
 	tristate "USB HIDBP Keyboard (simple Boot) support"
 	depends on USB && INPUT
-	---help---
+	help
 	  Say Y here only if you are absolutely sure that you don't want
 	  to use the generic HID driver for your USB keyboard and prefer
 	  to use the keyboard in its limited Boot Protocol mode instead.
@@ -67,7 +67,7 @@ config USB_KBD
 config USB_MOUSE
 	tristate "USB HIDBP Mouse (simple Boot) support"
 	depends on USB && INPUT
-	---help---
+	help
 	  Say Y here only if you are absolutely sure that you don't want
 	  to use the generic HID driver for your USB mouse and prefer
 	  to use the mouse in its limited Boot Protocol mode instead.
diff --git a/drivers/hsi/Kconfig b/drivers/hsi/Kconfig
index bcddb06ef038..42da4c235508 100644
--- a/drivers/hsi/Kconfig
+++ b/drivers/hsi/Kconfig
@@ -4,7 +4,7 @@
 #
 menuconfig HSI
 	tristate "HSI support"
-	---help---
+	help
 	  The "High speed synchronous Serial Interface" is
 	  synchronous serial interface used mainly to connect
 	  application engines and cellular modems.
diff --git a/drivers/hsi/clients/Kconfig b/drivers/hsi/clients/Kconfig
index 3c423a27f0ae..80f60cb28c98 100644
--- a/drivers/hsi/clients/Kconfig
+++ b/drivers/hsi/clients/Kconfig
@@ -35,7 +35,7 @@ config SSI_PROTOCOL
 config HSI_CHAR
 	tristate "HSI/SSI character driver"
 	depends on HSI
-	---help---
+	help
 	  If you say Y here, you will enable the HSI/SSI character driver.
 	  This driver provides a simple character device interface for
 	  serial communication with the cellular modem over HSI/SSI bus.
diff --git a/drivers/hsi/controllers/Kconfig b/drivers/hsi/controllers/Kconfig
index 3ad4a5a9ab05..03ca5b558824 100644
--- a/drivers/hsi/controllers/Kconfig
+++ b/drivers/hsi/controllers/Kconfig
@@ -8,7 +8,7 @@ config OMAP_SSI
 	tristate "OMAP SSI hardware driver"
 	depends on HSI && OF && ARM && COMMON_CLK
 	depends on ARCH_OMAP3 || COMPILE_TEST
-	---help---
+	help
 	  SSI is a legacy version of HSI. It is usually used to connect
 	  an application engine with a cellular modem.
 	  If you say Y here, you will enable the OMAP SSI hardware driver.
diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c
index 9915578533bb..8f12995ec133 100644
--- a/drivers/hwmon/k10temp.c
+++ b/drivers/hwmon/k10temp.c
@@ -632,6 +632,7 @@ static const struct pci_device_id k10temp_id_table[] = {
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_DF_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F3) },
+	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) },
 	{ PCI_VDEVICE(HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) },
 	{}
diff --git a/drivers/hwtracing/coresight/Makefile b/drivers/hwtracing/coresight/Makefile
index 0e3e72f0f510..19497d1d92bf 100644
--- a/drivers/hwtracing/coresight/Makefile
+++ b/drivers/hwtracing/coresight/Makefile
@@ -2,7 +2,8 @@
 #
 # Makefile for CoreSight drivers.
 #
-obj-$(CONFIG_CORESIGHT) += coresight.o coresight-etm-perf.o coresight-platform.o
+obj-$(CONFIG_CORESIGHT) += coresight.o coresight-etm-perf.o \
+			   coresight-platform.o coresight-sysfs.o
 obj-$(CONFIG_CORESIGHT_LINK_AND_SINK_TMC) += coresight-tmc.o \
 					     coresight-tmc-etf.o \
 					     coresight-tmc-etr.o
diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c
index 2fdaeec80ee5..98f830c6ed50 100644
--- a/drivers/hwtracing/coresight/coresight-cti-platform.c
+++ b/drivers/hwtracing/coresight/coresight-cti-platform.c
@@ -2,11 +2,17 @@
 /*
  * Copyright (c) 2019, The Linaro Limited. All rights reserved.
  */
+#include <linux/coresight.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/of.h>
+#include <linux/property.h>
+#include <linux/slab.h>
 
 #include <dt-bindings/arm/coresight-cti-dt.h>
-#include <linux/of.h>
 
 #include "coresight-cti.h"
+#include "coresight-priv.h"
 
 /* Number of CTI signals in the v8 architecturally defined connection */
 #define NR_V8PE_IN_SIGS		2
@@ -429,8 +435,7 @@ static int cti_plat_create_impdef_connections(struct device *dev,
 }
 
 /* get the hardware configuration & connection data. */
-int cti_plat_get_hw_data(struct device *dev,
-			 struct cti_drvdata *drvdata)
+static int cti_plat_get_hw_data(struct device *dev, struct cti_drvdata *drvdata)
 {
 	int rc = 0;
 	struct cti_device *cti_dev = &drvdata->ctidev;
diff --git a/drivers/hwtracing/coresight/coresight-cti-sysfs.c b/drivers/hwtracing/coresight/coresight-cti-sysfs.c
index 1f8fb7c15e80..392757f3a019 100644
--- a/drivers/hwtracing/coresight/coresight-cti-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-cti-sysfs.c
@@ -4,7 +4,13 @@
  * Author: Mike Leach <mike.leach@linaro.org>
  */
 
+#include <linux/atomic.h>
 #include <linux/coresight.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/sysfs.h>
 
 #include "coresight-cti.h"
 
@@ -1036,8 +1042,8 @@ static int cti_create_con_sysfs_attr(struct device *dev,
 				     enum cti_conn_attr_type attr_type,
 				     int attr_idx)
 {
-	struct dev_ext_attribute *eattr = 0;
-	char *name = 0;
+	struct dev_ext_attribute *eattr;
+	char *name;
 
 	eattr = devm_kzalloc(dev, sizeof(struct dev_ext_attribute),
 				    GFP_KERNEL);
@@ -1139,7 +1145,7 @@ static int cti_create_con_attr_set(struct device *dev, int con_idx,
 }
 
 /* create the array of group pointers for the CTI sysfs groups */
-int cti_create_cons_groups(struct device *dev, struct cti_device *ctidev)
+static int cti_create_cons_groups(struct device *dev, struct cti_device *ctidev)
 {
 	int nr_groups;
 
@@ -1156,8 +1162,8 @@ int cti_create_cons_groups(struct device *dev, struct cti_device *ctidev)
 int cti_create_cons_sysfs(struct device *dev, struct cti_drvdata *drvdata)
 {
 	struct cti_device *ctidev = &drvdata->ctidev;
-	int err = 0, con_idx = 0, i;
-	struct cti_trig_con *tc = NULL;
+	int err, con_idx = 0, i;
+	struct cti_trig_con *tc;
 
 	err = cti_create_cons_groups(dev, ctidev);
 	if (err)
diff --git a/drivers/hwtracing/coresight/coresight-cti.c b/drivers/hwtracing/coresight/coresight-cti.c
index aa6e0249bd70..40387d58c8e7 100644
--- a/drivers/hwtracing/coresight/coresight-cti.c
+++ b/drivers/hwtracing/coresight/coresight-cti.c
@@ -4,7 +4,22 @@
  * Author: Mike Leach <mike.leach@linaro.org>
  */
 
+#include <linux/amba/bus.h>
+#include <linux/atomic.h>
+#include <linux/bits.h>
+#include <linux/coresight.h>
+#include <linux/cpu_pm.h>
+#include <linux/cpuhotplug.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
 #include <linux/property.h>
+#include <linux/spinlock.h>
+
+#include "coresight-priv.h"
 #include "coresight-cti.h"
 
 /**
@@ -19,7 +34,7 @@
  */
 
 /* net of CTI devices connected via CTM */
-LIST_HEAD(ect_net);
+static LIST_HEAD(ect_net);
 
 /* protect the list */
 static DEFINE_MUTEX(ect_mutex);
@@ -27,6 +42,12 @@ static DEFINE_MUTEX(ect_mutex);
 #define csdev_to_cti_drvdata(csdev)	\
 	dev_get_drvdata(csdev->dev.parent)
 
+/* power management handling */
+static int nr_cti_cpu;
+
+/* quick lookup list for CPU bound CTIs when power handling */
+static struct cti_drvdata *cti_cpu_drvdata[NR_CPUS];
+
 /*
  * CTI naming. CTI bound to cores will have the name cti_cpu<N> where
  * N is the CPU ID. System CTIs will have the name cti_sys<I> where I
@@ -116,6 +137,35 @@ cti_err_not_enabled:
 	return rc;
 }
 
+/* re-enable CTI on CPU when using CPU hotplug */
+static void cti_cpuhp_enable_hw(struct cti_drvdata *drvdata)
+{
+	struct cti_config *config = &drvdata->config;
+	struct device *dev = &drvdata->csdev->dev;
+
+	pm_runtime_get_sync(dev->parent);
+	spin_lock(&drvdata->spinlock);
+	config->hw_powered = true;
+
+	/* no need to do anything if no enable request */
+	if (!atomic_read(&drvdata->config.enable_req_count))
+		goto cti_hp_not_enabled;
+
+	/* try to claim the device */
+	if (coresight_claim_device(drvdata->base))
+		goto cti_hp_not_enabled;
+
+	cti_write_all_hw_regs(drvdata);
+	config->hw_enabled = true;
+	spin_unlock(&drvdata->spinlock);
+	return;
+
+	/* did not re-enable due to no claim / no request */
+cti_hp_not_enabled:
+	spin_unlock(&drvdata->spinlock);
+	pm_runtime_put(dev->parent);
+}
+
 /* disable hardware */
 static int cti_disable_hw(struct cti_drvdata *drvdata)
 {
@@ -442,6 +492,34 @@ int cti_channel_setop(struct device *dev, enum cti_chan_set_op op,
 	return err;
 }
 
+static bool cti_add_sysfs_link(struct cti_drvdata *drvdata,
+			       struct cti_trig_con *tc)
+{
+	struct coresight_sysfs_link link_info;
+	int link_err = 0;
+
+	link_info.orig = drvdata->csdev;
+	link_info.orig_name = tc->con_dev_name;
+	link_info.target = tc->con_dev;
+	link_info.target_name = dev_name(&drvdata->csdev->dev);
+
+	link_err = coresight_add_sysfs_link(&link_info);
+	if (link_err)
+		dev_warn(&drvdata->csdev->dev,
+			 "Failed to set CTI sysfs link %s<=>%s\n",
+			 link_info.orig_name, link_info.target_name);
+	return !link_err;
+}
+
+static void cti_remove_sysfs_link(struct cti_trig_con *tc)
+{
+	struct coresight_sysfs_link link_info;
+
+	link_info.orig_name = tc->con_dev_name;
+	link_info.target = tc->con_dev;
+	coresight_remove_sysfs_link(&link_info);
+}
+
 /*
  * Look for a matching connection device name in the list of connections.
  * If found then swap in the csdev name, set trig con association pointer
@@ -452,6 +530,8 @@ cti_match_fixup_csdev(struct cti_device *ctidev, const char *node_name,
 		      struct coresight_device *csdev)
 {
 	struct cti_trig_con *tc;
+	struct cti_drvdata *drvdata = container_of(ctidev, struct cti_drvdata,
+						   ctidev);
 
 	list_for_each_entry(tc, &ctidev->trig_cons, node) {
 		if (tc->con_dev_name) {
@@ -459,7 +539,12 @@ cti_match_fixup_csdev(struct cti_device *ctidev, const char *node_name,
 				/* match: so swap in csdev name & dev */
 				tc->con_dev_name = dev_name(&csdev->dev);
 				tc->con_dev = csdev;
-				return true;
+				/* try to set sysfs link */
+				if (cti_add_sysfs_link(drvdata, tc))
+					return true;
+				/* link failed - remove CTI reference */
+				tc->con_dev = NULL;
+				break;
 			}
 		}
 	}
@@ -522,6 +607,7 @@ void cti_remove_assoc_from_csdev(struct coresight_device *csdev)
 		ctidev = &ctidrv->ctidev;
 		list_for_each_entry(tc, &ctidev->trig_cons, node) {
 			if (tc->con_dev == csdev->ect_dev) {
+				cti_remove_sysfs_link(tc);
 				tc->con_dev = NULL;
 				break;
 			}
@@ -543,10 +629,16 @@ static void cti_update_conn_xrefs(struct cti_drvdata *drvdata)
 	struct cti_device *ctidev = &drvdata->ctidev;
 
 	list_for_each_entry(tc, &ctidev->trig_cons, node) {
-		if (tc->con_dev)
-			/* set tc->con_dev->ect_dev */
-			coresight_set_assoc_ectdev_mutex(tc->con_dev,
+		if (tc->con_dev) {
+			/* if we can set the sysfs link */
+			if (cti_add_sysfs_link(drvdata, tc))
+				/* set the CTI/csdev association */
+				coresight_set_assoc_ectdev_mutex(tc->con_dev,
 							 drvdata->csdev);
+			else
+				/* otherwise remove reference from CTI */
+				tc->con_dev = NULL;
+		}
 	}
 }
 
@@ -559,7 +651,113 @@ static void cti_remove_conn_xrefs(struct cti_drvdata *drvdata)
 		if (tc->con_dev) {
 			coresight_set_assoc_ectdev_mutex(tc->con_dev,
 							 NULL);
+			cti_remove_sysfs_link(tc);
+			tc->con_dev = NULL;
+		}
+	}
+}
+
+/** cti PM callbacks **/
+static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
+			     void *v)
+{
+	struct cti_drvdata *drvdata;
+	unsigned int cpu = smp_processor_id();
+	int notify_res = NOTIFY_OK;
+
+	if (!cti_cpu_drvdata[cpu])
+		return NOTIFY_OK;
+
+	drvdata = cti_cpu_drvdata[cpu];
+
+	if (WARN_ON_ONCE(drvdata->ctidev.cpu != cpu))
+		return NOTIFY_BAD;
+
+	spin_lock(&drvdata->spinlock);
+
+	switch (cmd) {
+	case CPU_PM_ENTER:
+		/* CTI regs all static - we have a copy & nothing to save */
+		drvdata->config.hw_powered = false;
+		if (drvdata->config.hw_enabled)
+			coresight_disclaim_device(drvdata->base);
+		break;
+
+	case CPU_PM_ENTER_FAILED:
+		drvdata->config.hw_powered = true;
+		if (drvdata->config.hw_enabled) {
+			if (coresight_claim_device(drvdata->base))
+				drvdata->config.hw_enabled = false;
+		}
+		break;
+
+	case CPU_PM_EXIT:
+		/* write hardware registers to re-enable. */
+		drvdata->config.hw_powered = true;
+		drvdata->config.hw_enabled = false;
+
+		/* check enable reference count to enable HW */
+		if (atomic_read(&drvdata->config.enable_req_count)) {
+			/* check we can claim the device as we re-power */
+			if (coresight_claim_device(drvdata->base))
+				goto cti_notify_exit;
+
+			drvdata->config.hw_enabled = true;
+			cti_write_all_hw_regs(drvdata);
+		}
+		break;
+
+	default:
+		notify_res = NOTIFY_DONE;
+		break;
+	}
+
+cti_notify_exit:
+	spin_unlock(&drvdata->spinlock);
+	return notify_res;
+}
+
+static struct notifier_block cti_cpu_pm_nb = {
+	.notifier_call = cti_cpu_pm_notify,
+};
+
+/* CPU HP handlers */
+static int cti_starting_cpu(unsigned int cpu)
+{
+	struct cti_drvdata *drvdata = cti_cpu_drvdata[cpu];
+
+	if (!drvdata)
+		return 0;
+
+	cti_cpuhp_enable_hw(drvdata);
+	return 0;
+}
+
+static int cti_dying_cpu(unsigned int cpu)
+{
+	struct cti_drvdata *drvdata = cti_cpu_drvdata[cpu];
+
+	if (!drvdata)
+		return 0;
+
+	spin_lock(&drvdata->spinlock);
+	drvdata->config.hw_powered = false;
+	coresight_disclaim_device(drvdata->base);
+	spin_unlock(&drvdata->spinlock);
+	return 0;
+}
+
+/* release PM registrations */
+static void cti_pm_release(struct cti_drvdata *drvdata)
+{
+	if (drvdata->ctidev.cpu >= 0) {
+		if (--nr_cti_cpu == 0) {
+			cpu_pm_unregister_notifier(&cti_cpu_pm_nb);
+
+			cpuhp_remove_state_nocalls(
+				CPUHP_AP_ARM_CORESIGHT_CTI_STARTING);
 		}
+		cti_cpu_drvdata[drvdata->ctidev.cpu] = NULL;
 	}
 }
 
@@ -578,12 +776,12 @@ int cti_disable(struct coresight_device *csdev)
 	return cti_disable_hw(drvdata);
 }
 
-const struct coresight_ops_ect cti_ops_ect = {
+static const struct coresight_ops_ect cti_ops_ect = {
 	.enable = cti_enable,
 	.disable = cti_disable,
 };
 
-const struct coresight_ops cti_ops = {
+static const struct coresight_ops cti_ops = {
 	.ect_ops = &cti_ops_ect,
 };
 
@@ -598,6 +796,7 @@ static void cti_device_release(struct device *dev)
 
 	mutex_lock(&ect_mutex);
 	cti_remove_conn_xrefs(drvdata);
+	cti_pm_release(drvdata);
 
 	/* remove from the list */
 	list_for_each_entry_safe(ect_item, ect_tmp, &ect_net, node) {
@@ -673,6 +872,24 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
 		goto err_out;
 	}
 
+	/* setup CPU power management handling for CPU bound CTI devices. */
+	if (drvdata->ctidev.cpu >= 0) {
+		cti_cpu_drvdata[drvdata->ctidev.cpu] = drvdata;
+		if (!nr_cti_cpu++) {
+			cpus_read_lock();
+			ret = cpuhp_setup_state_nocalls_cpuslocked(
+				CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
+				"arm/coresight_cti:starting",
+				cti_starting_cpu, cti_dying_cpu);
+
+			if (!ret)
+				ret = cpu_pm_register_notifier(&cti_cpu_pm_nb);
+			cpus_read_unlock();
+			if (ret)
+				goto err_out;
+		}
+	}
+
 	/* create dynamic attributes for connections */
 	ret = cti_create_cons_sysfs(dev, drvdata);
 	if (ret) {
@@ -711,6 +928,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
 	return 0;
 
 err_out:
+	cti_pm_release(drvdata);
 	return ret;
 }
 
diff --git a/drivers/hwtracing/coresight/coresight-cti.h b/drivers/hwtracing/coresight/coresight-cti.h
index 004df3ab9dd0..acf7b545e6b9 100644
--- a/drivers/hwtracing/coresight/coresight-cti.h
+++ b/drivers/hwtracing/coresight/coresight-cti.h
@@ -7,8 +7,14 @@
 #ifndef _CORESIGHT_CORESIGHT_CTI_H
 #define _CORESIGHT_CORESIGHT_CTI_H
 
-#include <asm/local.h>
+#include <linux/coresight.h>
+#include <linux/device.h>
+#include <linux/fwnode.h>
+#include <linux/list.h>
 #include <linux/spinlock.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+
 #include "coresight-priv.h"
 
 /*
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index 3810290e6d07..03e3f2590191 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -717,7 +717,7 @@ static const struct attribute_group coresight_etb_mgmt_group = {
 	.name = "mgmt",
 };
 
-const struct attribute_group *coresight_etb_groups[] = {
+static const struct attribute_group *coresight_etb_groups[] = {
 	&coresight_etb_group,
 	&coresight_etb_mgmt_group,
 	NULL,
diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c
index e2cb6873c3f2..bf22dcfd3327 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x.c
@@ -504,7 +504,7 @@ static int etm_enable_perf(struct coresight_device *csdev,
 static int etm_enable_sysfs(struct coresight_device *csdev)
 {
 	struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
-	struct etm_enable_arg arg = { 0 };
+	struct etm_enable_arg arg = { };
 	int ret;
 
 	spin_lock(&drvdata->spinlock);
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
index ce41482431f9..b673e738bc9a 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
@@ -205,7 +205,7 @@ static ssize_t reset_store(struct device *dev,
 	 * started state. ARM recommends start-stop logic is set before
 	 * each trace run.
 	 */
-	config->vinst_ctrl |= BIT(0);
+	config->vinst_ctrl = BIT(0);
 	if (drvdata->nr_addr_cmp == true) {
 		config->mode |= ETM_MODE_VIEWINST_STARTSTOP;
 		/* SSSTATUS, bit[9] */
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
index a90d757f7043..747afc875f91 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x.c
@@ -412,7 +412,7 @@ out:
 static int etm4_enable_sysfs(struct coresight_device *csdev)
 {
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
-	struct etm4_enable_arg arg = { 0 };
+	struct etm4_enable_arg arg = { };
 	int ret;
 
 	spin_lock(&drvdata->spinlock);
@@ -791,7 +791,7 @@ static void etm4_set_default_config(struct etmv4_config *config)
 	config->ts_ctrl = 0x0;
 
 	/* TRCVICTLR::EVENT = 0x01, select the always on logic */
-	config->vinst_ctrl |= BIT(0);
+	config->vinst_ctrl = BIT(0);
 }
 
 static u64 etm4_get_ns_access_type(struct etmv4_config *config)
@@ -894,17 +894,8 @@ static void etm4_set_start_stop_filter(struct etmv4_config *config,
 
 static void etm4_set_default_filter(struct etmv4_config *config)
 {
-	u64 start, stop;
-
-	/*
-	 * Configure address range comparator '0' to encompass all
-	 * possible addresses.
-	 */
-	start = 0x0;
-	stop = ~0x0;
-
-	etm4_set_comparator_filter(config, start, stop,
-				   ETM_DEFAULT_ADDR_COMP);
+	/* Trace everything 'default' filter achieved by no filtering */
+	config->viiectlr = 0x0;
 
 	/*
 	 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
@@ -925,11 +916,9 @@ static void etm4_set_default(struct etmv4_config *config)
 	/*
 	 * Make default initialisation trace everything
 	 *
-	 * Select the "always true" resource selector on the
-	 * "Enablign Event" line and configure address range comparator
-	 * '0' to trace all the possible address range.  From there
-	 * configure the "include/exclude" engine to include address
-	 * range comparator '0'.
+	 * This is done by a minimum default config sufficient to enable
+	 * full instruction trace - with a default filter for trace all
+	 * achieved by having no filtering.
 	 */
 	etm4_set_default_config(config);
 	etm4_set_default_filter(config);
@@ -1527,6 +1516,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
 	return 0;
 
 err_arch_supported:
+	etmdrvdata[drvdata->cpu] = NULL;
 	if (--etm4_count == 0) {
 		etm4_cpu_pm_unregister();
 
@@ -1552,10 +1542,13 @@ static const struct amba_id etm4_ids[] = {
 	CS_AMBA_ID(0x000bb95a),			/* Cortex-A72 */
 	CS_AMBA_ID(0x000bb959),			/* Cortex-A73 */
 	CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */
+	CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */
 	CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */
 	CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */
-	CS_AMBA_ID(0x000bb802),			/* Qualcomm Kryo 385 Cortex-A55 */
-	CS_AMBA_ID(0x000bb803),			/* Qualcomm Kryo 385 Cortex-A75 */
+	CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */
+	CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */
+	CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */
+	CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */
 	CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */
 	{},
 };
diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c
index 43418a2126ff..e4912abda3aa 100644
--- a/drivers/hwtracing/coresight/coresight-platform.c
+++ b/drivers/hwtracing/coresight/coresight-platform.c
@@ -87,6 +87,7 @@ static void of_coresight_get_ports_legacy(const struct device_node *node,
 					  int *nr_inport, int *nr_outport)
 {
 	struct device_node *ep = NULL;
+	struct of_endpoint endpoint;
 	int in = 0, out = 0;
 
 	do {
@@ -94,10 +95,16 @@ static void of_coresight_get_ports_legacy(const struct device_node *node,
 		if (!ep)
 			break;
 
-		if (of_coresight_legacy_ep_is_input(ep))
-			in++;
-		else
-			out++;
+		if (of_graph_parse_endpoint(ep, &endpoint))
+			continue;
+
+		if (of_coresight_legacy_ep_is_input(ep)) {
+			in = (endpoint.port + 1 > in) ?
+				endpoint.port + 1 : in;
+		} else {
+			out = (endpoint.port + 1) > out ?
+				endpoint.port + 1 : out;
+		}
 
 	} while (ep);
 
@@ -137,9 +144,16 @@ of_coresight_count_ports(struct device_node *port_parent)
 {
 	int i = 0;
 	struct device_node *ep = NULL;
+	struct of_endpoint endpoint;
+
+	while ((ep = of_graph_get_next_endpoint(port_parent, ep))) {
+		/* Defer error handling to parsing */
+		if (of_graph_parse_endpoint(ep, &endpoint))
+			continue;
+		if (endpoint.port + 1 > i)
+			i = endpoint.port + 1;
+	}
 
-	while ((ep = of_graph_get_next_endpoint(port_parent, ep)))
-		i++;
 	return i;
 }
 
@@ -191,14 +205,12 @@ static int of_coresight_get_cpu(struct device *dev)
  * Parses the local port, remote device name and the remote port.
  *
  * Returns :
- *	 1	- If the parsing is successful and a connection record
- *		  was created for an output connection.
  *	 0	- If the parsing completed without any fatal errors.
  *	-Errno	- Fatal error, abort the scanning.
  */
 static int of_coresight_parse_endpoint(struct device *dev,
 				       struct device_node *ep,
-				       struct coresight_connection *conn)
+				       struct coresight_platform_data *pdata)
 {
 	int ret = 0;
 	struct of_endpoint endpoint, rendpoint;
@@ -206,6 +218,7 @@ static int of_coresight_parse_endpoint(struct device *dev,
 	struct device_node *rep = NULL;
 	struct device *rdev = NULL;
 	struct fwnode_handle *rdev_fwnode;
+	struct coresight_connection *conn;
 
 	do {
 		/* Parse the local port details */
@@ -232,6 +245,13 @@ static int of_coresight_parse_endpoint(struct device *dev,
 			break;
 		}
 
+		conn = &pdata->conns[endpoint.port];
+		if (conn->child_fwnode) {
+			dev_warn(dev, "Duplicate output port %d\n",
+				 endpoint.port);
+			ret = -EINVAL;
+			break;
+		}
 		conn->outport = endpoint.port;
 		/*
 		 * Hold the refcount to the target device. This could be
@@ -244,7 +264,6 @@ static int of_coresight_parse_endpoint(struct device *dev,
 		conn->child_fwnode = fwnode_handle_get(rdev_fwnode);
 		conn->child_port = rendpoint.port;
 		/* Connection record updated */
-		ret = 1;
 	} while (0);
 
 	of_node_put(rparent);
@@ -258,7 +277,6 @@ static int of_get_coresight_platform_data(struct device *dev,
 					  struct coresight_platform_data *pdata)
 {
 	int ret = 0;
-	struct coresight_connection *conn;
 	struct device_node *ep = NULL;
 	const struct device_node *parent = NULL;
 	bool legacy_binding = false;
@@ -287,8 +305,6 @@ static int of_get_coresight_platform_data(struct device *dev,
 		dev_warn_once(dev, "Uses obsolete Coresight DT bindings\n");
 	}
 
-	conn = pdata->conns;
-
 	/* Iterate through each output port to discover topology */
 	while ((ep = of_graph_get_next_endpoint(parent, ep))) {
 		/*
@@ -300,15 +316,9 @@ static int of_get_coresight_platform_data(struct device *dev,
 		if (legacy_binding && of_coresight_legacy_ep_is_input(ep))
 			continue;
 
-		ret = of_coresight_parse_endpoint(dev, ep, conn);
-		switch (ret) {
-		case 1:
-			conn++;		/* Fall through */
-		case 0:
-			break;
-		default:
+		ret = of_coresight_parse_endpoint(dev, ep, pdata);
+		if (ret)
 			return ret;
-		}
 	}
 
 	return 0;
@@ -501,7 +511,7 @@ static inline bool acpi_validate_dsd_graph(const union acpi_object *graph)
 }
 
 /* acpi_get_dsd_graph	- Find the _DSD Graph property for the given device. */
-const union acpi_object *
+static const union acpi_object *
 acpi_get_dsd_graph(struct acpi_device *adev)
 {
 	int i;
@@ -564,7 +574,7 @@ acpi_validate_coresight_graph(const union acpi_object *cs_graph)
  * Returns the pointer to the CoreSight Graph Package when found. Otherwise
  * returns NULL.
  */
-const union acpi_object *
+static const union acpi_object *
 acpi_get_coresight_graph(struct acpi_device *adev)
 {
 	const union acpi_object *graph_list, *graph;
@@ -647,6 +657,16 @@ static int acpi_coresight_parse_link(struct acpi_device *adev,
 		 *    coresight_remove_match().
 		 */
 		conn->child_fwnode = fwnode_handle_get(&r_adev->fwnode);
+	} else if (dir == ACPI_CORESIGHT_LINK_SLAVE) {
+		/*
+		 * We are only interested in the port number
+		 * for the input ports at this component.
+		 * Store the port number in child_port.
+		 */
+		conn->child_port = fields[0].integer.value;
+	} else {
+		/* Invalid direction */
+		return -EINVAL;
 	}
 
 	return dir;
@@ -692,10 +712,20 @@ static int acpi_coresight_parse_graph(struct acpi_device *adev,
 			return dir;
 
 		if (dir == ACPI_CORESIGHT_LINK_MASTER) {
-			pdata->nr_outport++;
+			if (ptr->outport > pdata->nr_outport)
+				pdata->nr_outport = ptr->outport;
 			ptr++;
 		} else {
-			pdata->nr_inport++;
+			WARN_ON(pdata->nr_inport == ptr->child_port);
+			/*
+			 * We do not track input port connections for a device.
+			 * However we need the highest port number described,
+			 * which can be recorded now and reuse this connection
+			 * record for an output connection. Hence, do not move
+			 * the ptr for input connections
+			 */
+			if (ptr->child_port > pdata->nr_inport)
+				pdata->nr_inport = ptr->child_port;
 		}
 	}
 
@@ -704,8 +734,13 @@ static int acpi_coresight_parse_graph(struct acpi_device *adev,
 		return rc;
 
 	/* Copy the connection information to the final location */
-	for (i = 0; i < pdata->nr_outport; i++)
-		pdata->conns[i] = conns[i];
+	for (i = 0; conns + i < ptr; i++) {
+		int port = conns[i].outport;
+
+		/* Duplicate output port */
+		WARN_ON(pdata->conns[port].child_fwnode);
+		pdata->conns[port] = conns[i];
+	}
 
 	devm_kfree(&adev->dev, conns);
 	return 0;
@@ -822,7 +857,7 @@ coresight_get_platform_data(struct device *dev)
 error:
 	if (!IS_ERR_OR_NULL(pdata))
 		/* Cleanup the connection information */
-		coresight_release_platform_data(pdata);
+		coresight_release_platform_data(NULL, pdata);
 	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(coresight_get_platform_data);
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index 890f9a5c97c6..36c943ae94d5 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -153,6 +153,15 @@ struct coresight_device *coresight_get_sink_by_id(u32 id);
 struct list_head *coresight_build_path(struct coresight_device *csdev,
 				       struct coresight_device *sink);
 void coresight_release_path(struct list_head *path);
+int coresight_add_sysfs_link(struct coresight_sysfs_link *info);
+void coresight_remove_sysfs_link(struct coresight_sysfs_link *info);
+int coresight_create_conns_sysfs_group(struct coresight_device *csdev);
+void coresight_remove_conns_sysfs_group(struct coresight_device *csdev);
+int coresight_make_links(struct coresight_device *orig,
+			 struct coresight_connection *conn,
+			 struct coresight_device *target);
+void coresight_remove_links(struct coresight_device *orig,
+			    struct coresight_connection *conn);
 
 #ifdef CONFIG_CORESIGHT_SOURCE_ETM3X
 extern int etm_readl_cp14(u32 off, unsigned int *val);
@@ -206,12 +215,16 @@ cti_remove_assoc_from_csdev(struct coresight_device *csdev) {}
 /* extract the data value from a UCI structure given amba_id pointer. */
 static inline void *coresight_get_uci_data(const struct amba_id *id)
 {
-	if (id->data)
-		return ((struct amba_cs_uci_id *)(id->data))->data;
-	return 0;
+	struct amba_cs_uci_id *uci_id = id->data;
+
+	if (!uci_id)
+		return NULL;
+
+	return uci_id->data;
 }
 
-void coresight_release_platform_data(struct coresight_platform_data *pdata);
+void coresight_release_platform_data(struct coresight_device *csdev,
+				     struct coresight_platform_data *pdata);
 struct coresight_device *
 coresight_find_csdev_by_fwnode(struct fwnode_handle *r_fwnode);
 void coresight_set_assoc_ectdev_mutex(struct coresight_device *csdev,
diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c
new file mode 100644
index 000000000000..82afeaf2ccc4
--- /dev/null
+++ b/drivers/hwtracing/coresight/coresight-sysfs.c
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019, Linaro Limited, All rights reserved.
+ * Author: Mike Leach <mike.leach@linaro.org>
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+
+#include "coresight-priv.h"
+
+/*
+ * Connections group - links attribute.
+ * Count of created links between coresight components in the group.
+ */
+static ssize_t nr_links_show(struct device *dev,
+			     struct device_attribute *attr,
+			     char *buf)
+{
+	struct coresight_device *csdev = to_coresight_device(dev);
+
+	return sprintf(buf, "%d\n", csdev->nr_links);
+}
+static DEVICE_ATTR_RO(nr_links);
+
+static struct attribute *coresight_conns_attrs[] = {
+	&dev_attr_nr_links.attr,
+	NULL,
+};
+
+static struct attribute_group coresight_conns_group = {
+	.attrs = coresight_conns_attrs,
+	.name = "connections",
+};
+
+/*
+ * Create connections group for CoreSight devices.
+ * This group will then be used to collate the sysfs links between
+ * devices.
+ */
+int coresight_create_conns_sysfs_group(struct coresight_device *csdev)
+{
+	int ret = 0;
+
+	if (!csdev)
+		return -EINVAL;
+
+	ret = sysfs_create_group(&csdev->dev.kobj, &coresight_conns_group);
+	if (ret)
+		return ret;
+
+	csdev->has_conns_grp = true;
+	return ret;
+}
+
+void coresight_remove_conns_sysfs_group(struct coresight_device *csdev)
+{
+	if (!csdev)
+		return;
+
+	if (csdev->has_conns_grp) {
+		sysfs_remove_group(&csdev->dev.kobj, &coresight_conns_group);
+		csdev->has_conns_grp = false;
+	}
+}
+
+int coresight_add_sysfs_link(struct coresight_sysfs_link *info)
+{
+	int ret = 0;
+
+	if (!info)
+		return -EINVAL;
+	if (!info->orig || !info->target ||
+	    !info->orig_name || !info->target_name)
+		return -EINVAL;
+	if (!info->orig->has_conns_grp || !info->target->has_conns_grp)
+		return -EINVAL;
+
+	/* first link orig->target */
+	ret = sysfs_add_link_to_group(&info->orig->dev.kobj,
+				      coresight_conns_group.name,
+				      &info->target->dev.kobj,
+				      info->orig_name);
+	if (ret)
+		return ret;
+
+	/* second link target->orig */
+	ret = sysfs_add_link_to_group(&info->target->dev.kobj,
+				      coresight_conns_group.name,
+				      &info->orig->dev.kobj,
+				      info->target_name);
+
+	/* error in second link - remove first - otherwise inc counts */
+	if (ret) {
+		sysfs_remove_link_from_group(&info->orig->dev.kobj,
+					     coresight_conns_group.name,
+					     info->orig_name);
+	} else {
+		info->orig->nr_links++;
+		info->target->nr_links++;
+	}
+
+	return ret;
+}
+
+void coresight_remove_sysfs_link(struct coresight_sysfs_link *info)
+{
+	if (!info)
+		return;
+	if (!info->orig || !info->target ||
+	    !info->orig_name || !info->target_name)
+		return;
+
+	sysfs_remove_link_from_group(&info->orig->dev.kobj,
+				     coresight_conns_group.name,
+				     info->orig_name);
+
+	sysfs_remove_link_from_group(&info->target->dev.kobj,
+				     coresight_conns_group.name,
+				     info->target_name);
+
+	info->orig->nr_links--;
+	info->target->nr_links--;
+}
+
+/*
+ * coresight_make_links: Make a link for a connection from a @orig
+ * device to @target, represented by @conn.
+ *
+ *   e.g, for devOrig[output_X] -> devTarget[input_Y] is represented
+ *   as two symbolic links :
+ *
+ *	/sys/.../devOrig/out:X	-> /sys/.../devTarget/
+ *	/sys/.../devTarget/in:Y	-> /sys/.../devOrig/
+ *
+ * The link names are allocated for a device where it appears. i.e, the
+ * "out" link on the master and "in" link on the slave device.
+ * The link info is stored in the connection record for avoiding
+ * the reconstruction of names for removal.
+ */
+int coresight_make_links(struct coresight_device *orig,
+			 struct coresight_connection *conn,
+			 struct coresight_device *target)
+{
+	int ret = -ENOMEM;
+	char *outs = NULL, *ins = NULL;
+	struct coresight_sysfs_link *link = NULL;
+
+	do {
+		outs = devm_kasprintf(&orig->dev, GFP_KERNEL,
+				      "out:%d", conn->outport);
+		if (!outs)
+			break;
+		ins = devm_kasprintf(&target->dev, GFP_KERNEL,
+				     "in:%d", conn->child_port);
+		if (!ins)
+			break;
+		link = devm_kzalloc(&orig->dev,
+				    sizeof(struct coresight_sysfs_link),
+				    GFP_KERNEL);
+		if (!link)
+			break;
+
+		link->orig = orig;
+		link->target = target;
+		link->orig_name = outs;
+		link->target_name = ins;
+
+		ret = coresight_add_sysfs_link(link);
+		if (ret)
+			break;
+
+		conn->link = link;
+
+		/*
+		 * Install the device connection. This also indicates that
+		 * the links are operational on both ends.
+		 */
+		conn->child_dev = target;
+		return 0;
+	} while (0);
+
+	return ret;
+}
+
+/*
+ * coresight_remove_links: Remove the sysfs links for a given connection @conn,
+ * from @orig device to @target device. See coresight_make_links() for more
+ * details.
+ */
+void coresight_remove_links(struct coresight_device *orig,
+			    struct coresight_connection *conn)
+{
+	if (!orig || !conn->link)
+		return;
+
+	coresight_remove_sysfs_link(conn->link);
+
+	devm_kfree(&conn->child_dev->dev, conn->link->target_name);
+	devm_kfree(&orig->dev, conn->link->orig_name);
+	devm_kfree(&orig->dev, conn->link);
+	conn->link = NULL;
+	conn->child_dev = NULL;
+}
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index d0cc3985b72a..36cce2bfb744 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -596,13 +596,6 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 		goto out;
 	}
 
-	/* There is no point in reading a TMC in HW FIFO mode */
-	mode = readl_relaxed(drvdata->base + TMC_MODE);
-	if (mode != TMC_MODE_CIRCULAR_BUFFER) {
-		ret = -EINVAL;
-		goto out;
-	}
-
 	/* Don't interfere if operated from Perf */
 	if (drvdata->mode == CS_MODE_PERF) {
 		ret = -EINVAL;
@@ -616,8 +609,15 @@ int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 	}
 
 	/* Disable the TMC if need be */
-	if (drvdata->mode == CS_MODE_SYSFS)
+	if (drvdata->mode == CS_MODE_SYSFS) {
+		/* There is no point in reading a TMC in HW FIFO mode */
+		mode = readl_relaxed(drvdata->base + TMC_MODE);
+		if (mode != TMC_MODE_CIRCULAR_BUFFER) {
+			ret = -EINVAL;
+			goto out;
+		}
 		__tmc_etb_disable_hw(drvdata);
+	}
 
 	drvdata->reading = true;
 out:
diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c
index 1cf82fa58289..39fba1d16e6e 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.c
+++ b/drivers/hwtracing/coresight/coresight-tmc.c
@@ -361,7 +361,7 @@ static const struct attribute_group coresight_tmc_mgmt_group = {
 	.name = "mgmt",
 };
 
-const struct attribute_group *coresight_tmc_groups[] = {
+static const struct attribute_group *coresight_tmc_groups[] = {
 	&coresight_tmc_group,
 	&coresight_tmc_mgmt_group,
 	NULL,
diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c
index c71553c09f8e..f3efbb3b2b4d 100644
--- a/drivers/hwtracing/coresight/coresight.c
+++ b/drivers/hwtracing/coresight/coresight.c
@@ -1031,7 +1031,7 @@ static void coresight_device_release(struct device *dev)
 
 static int coresight_orphan_match(struct device *dev, void *data)
 {
-	int i;
+	int i, ret = 0;
 	bool still_orphan = false;
 	struct coresight_device *csdev, *i_csdev;
 	struct coresight_connection *conn;
@@ -1053,49 +1053,62 @@ static int coresight_orphan_match(struct device *dev, void *data)
 	for (i = 0; i < i_csdev->pdata->nr_outport; i++) {
 		conn = &i_csdev->pdata->conns[i];
 
+		/* Skip the port if FW doesn't describe it */
+		if (!conn->child_fwnode)
+			continue;
 		/* We have found at least one orphan connection */
 		if (conn->child_dev == NULL) {
 			/* Does it match this newly added device? */
-			if (conn->child_fwnode == csdev->dev.fwnode)
-				conn->child_dev = csdev;
-			else
+			if (conn->child_fwnode == csdev->dev.fwnode) {
+				ret = coresight_make_links(i_csdev,
+							   conn, csdev);
+				if (ret)
+					return ret;
+			} else {
 				/* This component still has an orphan */
 				still_orphan = true;
+			}
 		}
 	}
 
 	i_csdev->orphan = still_orphan;
 
 	/*
-	 * Returning '0' ensures that all known component on the
-	 * bus will be checked.
+	 * Returning '0' in case we didn't encounter any error,
+	 * ensures that all known component on the bus will be checked.
 	 */
 	return 0;
 }
 
-static void coresight_fixup_orphan_conns(struct coresight_device *csdev)
+static int coresight_fixup_orphan_conns(struct coresight_device *csdev)
 {
-	/*
-	 * No need to check for a return value as orphan connection(s)
-	 * are hooked-up with each newly added component.
-	 */
-	bus_for_each_dev(&coresight_bustype, NULL,
+	return bus_for_each_dev(&coresight_bustype, NULL,
 			 csdev, coresight_orphan_match);
 }
 
 
-static void coresight_fixup_device_conns(struct coresight_device *csdev)
+static int coresight_fixup_device_conns(struct coresight_device *csdev)
 {
-	int i;
+	int i, ret = 0;
 
 	for (i = 0; i < csdev->pdata->nr_outport; i++) {
 		struct coresight_connection *conn = &csdev->pdata->conns[i];
 
+		if (!conn->child_fwnode)
+			continue;
 		conn->child_dev =
 			coresight_find_csdev_by_fwnode(conn->child_fwnode);
-		if (!conn->child_dev)
+		if (conn->child_dev) {
+			ret = coresight_make_links(csdev, conn,
+						   conn->child_dev);
+			if (ret)
+				break;
+		} else {
 			csdev->orphan = true;
+		}
 	}
+
+	return 0;
 }
 
 static int coresight_remove_match(struct device *dev, void *data)
@@ -1118,12 +1131,12 @@ static int coresight_remove_match(struct device *dev, void *data)
 	for (i = 0; i < iterator->pdata->nr_outport; i++) {
 		conn = &iterator->pdata->conns[i];
 
-		if (conn->child_dev == NULL)
+		if (conn->child_dev == NULL || conn->child_fwnode == NULL)
 			continue;
 
 		if (csdev->dev.fwnode == conn->child_fwnode) {
 			iterator->orphan = true;
-			conn->child_dev = NULL;
+			coresight_remove_links(iterator, conn);
 			/*
 			 * Drop the reference to the handle for the remote
 			 * device acquired in parsing the connections from
@@ -1213,16 +1226,27 @@ postcore_initcall(coresight_init);
  * coresight_release_platform_data: Release references to the devices connected
  * to the output port of this device.
  */
-void coresight_release_platform_data(struct coresight_platform_data *pdata)
+void coresight_release_platform_data(struct coresight_device *csdev,
+				     struct coresight_platform_data *pdata)
 {
 	int i;
+	struct coresight_connection *conns = pdata->conns;
 
 	for (i = 0; i < pdata->nr_outport; i++) {
-		if (pdata->conns[i].child_fwnode) {
-			fwnode_handle_put(pdata->conns[i].child_fwnode);
+		/* If we have made the links, remove them now */
+		if (csdev && conns[i].child_dev)
+			coresight_remove_links(csdev, &conns[i]);
+		/*
+		 * Drop the refcount and clear the handle as this device
+		 * is going away
+		 */
+		if (conns[i].child_fwnode) {
+			fwnode_handle_put(conns[i].child_fwnode);
 			pdata->conns[i].child_fwnode = NULL;
 		}
 	}
+	if (csdev)
+		coresight_remove_conns_sysfs_group(csdev);
 }
 
 struct coresight_device *coresight_register(struct coresight_desc *desc)
@@ -1304,11 +1328,19 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 
 	mutex_lock(&coresight_mutex);
 
-	coresight_fixup_device_conns(csdev);
-	coresight_fixup_orphan_conns(csdev);
-	cti_add_assoc_to_csdev(csdev);
+	ret = coresight_create_conns_sysfs_group(csdev);
+	if (!ret)
+		ret = coresight_fixup_device_conns(csdev);
+	if (!ret)
+		ret = coresight_fixup_orphan_conns(csdev);
+	if (!ret)
+		cti_add_assoc_to_csdev(csdev);
 
 	mutex_unlock(&coresight_mutex);
+	if (ret) {
+		coresight_unregister(csdev);
+		return ERR_PTR(ret);
+	}
 
 	return csdev;
 
@@ -1316,7 +1348,7 @@ err_free_csdev:
 	kfree(csdev);
 err_out:
 	/* Cleanup the connection information */
-	coresight_release_platform_data(desc->pdata);
+	coresight_release_platform_data(NULL, desc->pdata);
 	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(coresight_register);
@@ -1326,7 +1358,7 @@ void coresight_unregister(struct coresight_device *csdev)
 	etm_perf_del_symlink_sink(csdev);
 	/* Remove references of that device in the topology */
 	coresight_remove_conns(csdev);
-	coresight_release_platform_data(csdev->pdata);
+	coresight_release_platform_data(csdev, csdev->pdata);
 	device_unregister(&csdev->dev);
 }
 EXPORT_SYMBOL_GPL(coresight_unregister);
diff --git a/drivers/hwtracing/stm/policy.c b/drivers/hwtracing/stm/policy.c
index 4f932a419752..603b4a9969d3 100644
--- a/drivers/hwtracing/stm/policy.c
+++ b/drivers/hwtracing/stm/policy.c
@@ -34,7 +34,7 @@ struct stp_policy_node {
 	unsigned int		first_channel;
 	unsigned int		last_channel;
 	/* this is the one that's exposed to the attributes */
-	unsigned char		priv[0];
+	unsigned char		priv[];
 };
 
 void *stp_policy_node_priv(struct stp_policy_node *pn)
diff --git a/drivers/hwtracing/stm/stm.h b/drivers/hwtracing/stm/stm.h
index 3569439d53bb..a9be49fc7a6b 100644
--- a/drivers/hwtracing/stm/stm.h
+++ b/drivers/hwtracing/stm/stm.h
@@ -23,7 +23,7 @@ void *stp_policy_node_priv(struct stp_policy_node *pn);
 
 struct stp_master {
 	unsigned int	nr_free;
-	unsigned long	chan_map[0];
+	unsigned long	chan_map[];
 };
 
 struct stm_device {
@@ -42,7 +42,7 @@ struct stm_device {
 	const struct config_item_type		*pdrv_node_type;
 	/* master allocation */
 	spinlock_t		mc_lock;
-	struct stp_master	*masters[0];
+	struct stp_master	*masters[];
 };
 
 #define to_stm_device(_d)				\
diff --git a/drivers/i2c/Kconfig b/drivers/i2c/Kconfig
index 1474e57ecafc..ef39c83aaf33 100644
--- a/drivers/i2c/Kconfig
+++ b/drivers/i2c/Kconfig
@@ -9,7 +9,7 @@ config I2C
 	tristate "I2C support"
 	select RT_MUTEXES
 	select IRQ_DOMAIN
-	---help---
+	help
 	  I2C (pronounce: I-squared-C) is a slow serial bus protocol used in
 	  many micro controller applications and developed by Philips.  SMBus,
 	  or System Management Bus is a subset of the I2C protocol.  More
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 2ddca08f8a76..735bf31a3fdf 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -475,8 +475,8 @@ config I2C_BCM_KONA
 
 config I2C_BRCMSTB
 	tristate "BRCM Settop/DSL I2C controller"
-	depends on ARCH_BRCMSTB || BMIPS_GENERIC || ARCH_BCM_63XX || \
-		   COMPILE_TEST
+	depends on ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC || \
+		   ARCH_BCM_63XX || COMPILE_TEST
 	default y
 	help
 	  If you say yes to this option, support will be included for the
@@ -526,22 +526,12 @@ config I2C_DAVINCI
 
 config I2C_DESIGNWARE_CORE
 	tristate
-
-config I2C_DESIGNWARE_PLATFORM
-	tristate "Synopsys DesignWare Platform"
-	select I2C_DESIGNWARE_CORE
-	depends on (ACPI && COMMON_CLK) || !ACPI
-	help
-	  If you say yes to this option, support will be included for the
-	  Synopsys DesignWare I2C adapter.
-
-	  This driver can also be built as a module.  If so, the module
-	  will be called i2c-designware-platform.
+	select REGMAP
 
 config I2C_DESIGNWARE_SLAVE
 	bool "Synopsys DesignWare Slave"
+	depends on I2C_DESIGNWARE_CORE
 	select I2C_SLAVE
-	depends on I2C_DESIGNWARE_PLATFORM
 	help
 	  If you say yes to this option, support will be included for the
 	  Synopsys DesignWare I2C slave adapter.
@@ -549,20 +539,22 @@ config I2C_DESIGNWARE_SLAVE
 	  This is not a standalone module, this module compiles together with
 	  i2c-designware-core.
 
-config I2C_DESIGNWARE_PCI
-	tristate "Synopsys DesignWare PCI"
-	depends on PCI
+config I2C_DESIGNWARE_PLATFORM
+	tristate "Synopsys DesignWare Platform"
+	depends on (ACPI && COMMON_CLK) || !ACPI
 	select I2C_DESIGNWARE_CORE
+	select MFD_SYSCON if MIPS_BAIKAL_T1
 	help
 	  If you say yes to this option, support will be included for the
-	  Synopsys DesignWare I2C adapter. Only master mode is supported.
+	  Synopsys DesignWare I2C adapter.
 
 	  This driver can also be built as a module.  If so, the module
-	  will be called i2c-designware-pci.
+	  will be called i2c-designware-platform.
 
 config I2C_DESIGNWARE_BAYTRAIL
 	bool "Intel Baytrail I2C semaphore support"
 	depends on ACPI
+	depends on I2C_DESIGNWARE_PLATFORM
 	depends on (I2C_DESIGNWARE_PLATFORM=m && IOSF_MBI) || \
 		   (I2C_DESIGNWARE_PLATFORM=y && IOSF_MBI=y)
 	help
@@ -572,6 +564,17 @@ config I2C_DESIGNWARE_BAYTRAIL
 	  the platform firmware controlling it. You should say Y if running on
 	  a BayTrail system using the AXP288.
 
+config I2C_DESIGNWARE_PCI
+	tristate "Synopsys DesignWare PCI"
+	depends on PCI
+	select I2C_DESIGNWARE_CORE
+	help
+	  If you say yes to this option, support will be included for the
+	  Synopsys DesignWare I2C adapter. Only master mode is supported.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called i2c-designware-pci.
+
 config I2C_DIGICOLOR
 	tristate "Conexant Digicolor I2C driver"
 	depends on ARCH_DIGICOLOR || COMPILE_TEST
@@ -791,6 +794,15 @@ config I2C_NOMADIK
 	  I2C interface from ST-Ericsson's Nomadik and Ux500 architectures,
 	  as well as the STA2X11 PCIe I/O HUB.
 
+config I2C_NPCM7XX
+	tristate "Nuvoton I2C Controller"
+	depends on ARCH_NPCM7XX || COMPILE_TEST
+	help
+	  If you say yes to this option, support will be included for the
+	  Nuvoton I2C controller, which is available on the NPCM7xx BMC
+	  controller.
+	  Driver can also support slave mode (select I2C_SLAVE).
+
 config I2C_OCORES
 	tristate "OpenCores I2C Controller"
 	help
@@ -885,6 +897,16 @@ config I2C_PXA_SLAVE
 	  is necessary for systems where the PXA may be a target on the
 	  I2C bus.
 
+config I2C_QCOM_CCI
+	tristate "Qualcomm Camera Control Interface"
+	depends on ARCH_QCOM || COMPILE_TEST
+	help
+	  If you say yes to this option, support will be included for the
+	  built-in camera control interface on the Qualcomm SoCs.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called i2c-qcom-cci.
+
 config I2C_QCOM_GENI
 	tristate "Qualcomm Technologies Inc.'s GENI based I2C controller"
 	depends on ARCH_QCOM || COMPILE_TEST
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index 25d60889713c..306d5dc3f417 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -48,16 +48,15 @@ obj-$(CONFIG_I2C_CADENCE)	+= i2c-cadence.o
 obj-$(CONFIG_I2C_CBUS_GPIO)	+= i2c-cbus-gpio.o
 obj-$(CONFIG_I2C_CPM)		+= i2c-cpm.o
 obj-$(CONFIG_I2C_DAVINCI)	+= i2c-davinci.o
-obj-$(CONFIG_I2C_DESIGNWARE_CORE)	+= i2c-designware-core.o
-i2c-designware-core-objs := i2c-designware-common.o i2c-designware-master.o
-ifeq ($(CONFIG_I2C_DESIGNWARE_SLAVE),y)
-i2c-designware-core-objs += i2c-designware-slave.o
-endif
-obj-$(CONFIG_I2C_DESIGNWARE_PLATFORM)	+= i2c-designware-platform.o
-i2c-designware-platform-objs := i2c-designware-platdrv.o
+obj-$(CONFIG_I2C_DESIGNWARE_CORE)			+= i2c-designware-core.o
+i2c-designware-core-y					:= i2c-designware-common.o
+i2c-designware-core-y					+= i2c-designware-master.o
+i2c-designware-core-$(CONFIG_I2C_DESIGNWARE_SLAVE) 	+= i2c-designware-slave.o
+obj-$(CONFIG_I2C_DESIGNWARE_PLATFORM)			+= i2c-designware-platform.o
+i2c-designware-platform-y 				:= i2c-designware-platdrv.o
 i2c-designware-platform-$(CONFIG_I2C_DESIGNWARE_BAYTRAIL) += i2c-designware-baytrail.o
-obj-$(CONFIG_I2C_DESIGNWARE_PCI)	+= i2c-designware-pci.o
-i2c-designware-pci-objs := i2c-designware-pcidrv.o
+obj-$(CONFIG_I2C_DESIGNWARE_PCI)			+= i2c-designware-pci.o
+i2c-designware-pci-y					:= i2c-designware-pcidrv.o
 obj-$(CONFIG_I2C_DIGICOLOR)	+= i2c-digicolor.o
 obj-$(CONFIG_I2C_EFM32)		+= i2c-efm32.o
 obj-$(CONFIG_I2C_EG20T)		+= i2c-eg20t.o
@@ -81,6 +80,7 @@ obj-$(CONFIG_I2C_MT7621)	+= i2c-mt7621.o
 obj-$(CONFIG_I2C_MV64XXX)	+= i2c-mv64xxx.o
 obj-$(CONFIG_I2C_MXS)		+= i2c-mxs.o
 obj-$(CONFIG_I2C_NOMADIK)	+= i2c-nomadik.o
+obj-$(CONFIG_I2C_NPCM7XX)	+= i2c-npcm7xx.o
 obj-$(CONFIG_I2C_OCORES)	+= i2c-ocores.o
 obj-$(CONFIG_I2C_OMAP)		+= i2c-omap.o
 obj-$(CONFIG_I2C_OWL)		+= i2c-owl.o
@@ -91,6 +91,7 @@ obj-$(CONFIG_I2C_PNX)		+= i2c-pnx.o
 obj-$(CONFIG_I2C_PUV3)		+= i2c-puv3.o
 obj-$(CONFIG_I2C_PXA)		+= i2c-pxa.o
 obj-$(CONFIG_I2C_PXA_PCI)	+= i2c-pxa-pci.o
+obj-$(CONFIG_I2C_QCOM_CCI)	+= i2c-qcom-cci.o
 obj-$(CONFIG_I2C_QCOM_GENI)	+= i2c-qcom-geni.o
 obj-$(CONFIG_I2C_QUP)		+= i2c-qup.o
 obj-$(CONFIG_I2C_RIIC)		+= i2c-riic.o
diff --git a/drivers/i2c/busses/i2c-altera.c b/drivers/i2c/busses/i2c-altera.c
index 16ddc26c00e6..7d62cbda6e06 100644
--- a/drivers/i2c/busses/i2c-altera.c
+++ b/drivers/i2c/busses/i2c-altera.c
@@ -69,7 +69,6 @@
  * @fifo_size: size of the FIFO passed in.
  * @isr_mask: cached copy of local ISR enables.
  * @isr_status: cached copy of local ISR status.
- * @lock: spinlock for IRQ synchronization.
  * @isr_mutex: mutex for IRQ thread.
  */
 struct altr_i2c_dev {
@@ -86,18 +85,14 @@ struct altr_i2c_dev {
 	u32 fifo_size;
 	u32 isr_mask;
 	u32 isr_status;
-	spinlock_t lock;	/* IRQ synchronization */
 	struct mutex isr_mutex;
 };
 
 static void
 altr_i2c_int_enable(struct altr_i2c_dev *idev, u32 mask, bool enable)
 {
-	unsigned long flags;
 	u32 int_en;
 
-	spin_lock_irqsave(&idev->lock, flags);
-
 	int_en = readl(idev->base + ALTR_I2C_ISER);
 	if (enable)
 		idev->isr_mask = int_en | mask;
@@ -105,8 +100,6 @@ altr_i2c_int_enable(struct altr_i2c_dev *idev, u32 mask, bool enable)
 		idev->isr_mask = int_en & ~mask;
 
 	writel(idev->isr_mask, idev->base + ALTR_I2C_ISER);
-
-	spin_unlock_irqrestore(&idev->lock, flags);
 }
 
 static void altr_i2c_int_clear(struct altr_i2c_dev *idev, u32 mask)
@@ -346,6 +339,7 @@ static int altr_i2c_xfer_msg(struct altr_i2c_dev *idev, struct i2c_msg *msg)
 
 	time_left = wait_for_completion_timeout(&idev->msg_complete,
 						ALTR_I2C_XFER_TIMEOUT);
+	mutex_lock(&idev->isr_mutex);
 	altr_i2c_int_enable(idev, imask, false);
 
 	value = readl(idev->base + ALTR_I2C_STATUS) & ALTR_I2C_STAT_CORE;
@@ -358,6 +352,7 @@ static int altr_i2c_xfer_msg(struct altr_i2c_dev *idev, struct i2c_msg *msg)
 	}
 
 	altr_i2c_core_disable(idev);
+	mutex_unlock(&idev->isr_mutex);
 
 	return idev->msg_err;
 }
@@ -389,23 +384,19 @@ static const struct i2c_algorithm altr_i2c_algo = {
 static int altr_i2c_probe(struct platform_device *pdev)
 {
 	struct altr_i2c_dev *idev = NULL;
-	struct resource *res;
 	int irq, ret;
 
 	idev = devm_kzalloc(&pdev->dev, sizeof(*idev), GFP_KERNEL);
 	if (!idev)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	idev->base = devm_ioremap_resource(&pdev->dev, res);
+	idev->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(idev->base))
 		return PTR_ERR(idev->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "missing interrupt resource\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	idev->i2c_clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(idev->i2c_clk)) {
@@ -415,7 +406,6 @@ static int altr_i2c_probe(struct platform_device *pdev)
 
 	idev->dev = &pdev->dev;
 	init_completion(&idev->msg_complete);
-	spin_lock_init(&idev->lock);
 	mutex_init(&idev->isr_mutex);
 
 	ret = device_property_read_u32(idev->dev, "fifo-size",
@@ -453,7 +443,9 @@ static int altr_i2c_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	mutex_lock(&idev->isr_mutex);
 	altr_i2c_init(idev);
+	mutex_unlock(&idev->isr_mutex);
 
 	i2c_set_adapdata(&idev->adapter, idev);
 	strlcpy(idev->adapter.name, pdev->name, sizeof(idev->adapter.name));
diff --git a/drivers/i2c/busses/i2c-at91-core.c b/drivers/i2c/busses/i2c-at91-core.c
index 3da1a8acecb5..e14edd236108 100644
--- a/drivers/i2c/busses/i2c-at91-core.c
+++ b/drivers/i2c/busses/i2c-at91-core.c
@@ -131,6 +131,7 @@ static struct at91_twi_pdata sama5d2_config = {
 	.has_dig_filtr = true,
 	.has_adv_dig_filtr = true,
 	.has_ana_filtr = true,
+	.has_clear_cmd = false,	/* due to errata, CLEAR cmd is not working */
 };
 
 static struct at91_twi_pdata sam9x60_config = {
@@ -142,6 +143,7 @@ static struct at91_twi_pdata sam9x60_config = {
 	.has_dig_filtr = true,
 	.has_adv_dig_filtr = true,
 	.has_ana_filtr = true,
+	.has_clear_cmd = true,
 };
 
 static const struct of_device_id atmel_twi_dt_ids[] = {
diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c
index 37b96ac9dfae..363d540a8345 100644
--- a/drivers/i2c/busses/i2c-at91-master.c
+++ b/drivers/i2c/busses/i2c-at91-master.c
@@ -480,7 +480,6 @@ static int at91_do_twi_transfer(struct at91_twi_dev *dev)
 	unsigned long time_left;
 	bool has_unre_flag = dev->pdata->has_unre_flag;
 	bool has_alt_cmd = dev->pdata->has_alt_cmd;
-	struct i2c_bus_recovery_info *rinfo = &dev->rinfo;
 
 	/*
 	 * WARNING: the TXCOMP bit in the Status Register is NOT a clear on
@@ -641,11 +640,12 @@ error:
 			       AT91_TWI_THRCLR | AT91_TWI_LOCKCLR);
 	}
 
-	if (rinfo->get_sda && !(rinfo->get_sda(&dev->adapter))) {
-		dev_dbg(dev->dev,
-			"SDA is down; clear bus using gpio\n");
-		i2c_recover_bus(&dev->adapter);
-	}
+	/*
+	 * some faulty I2C slave devices might hold SDA down;
+	 * we can send a bus clear command, hoping that the pins will be
+	 * released
+	 */
+	i2c_recover_bus(&dev->adapter);
 
 	return ret;
 }
@@ -830,7 +830,7 @@ static void at91_unprepare_twi_recovery(struct i2c_adapter *adap)
 	pinctrl_select_state(dev->pinctrl, dev->pinctrl_pins_default);
 }
 
-static int at91_init_twi_recovery_info(struct platform_device *pdev,
+static int at91_init_twi_recovery_gpio(struct platform_device *pdev,
 				       struct at91_twi_dev *dev)
 {
 	struct i2c_bus_recovery_info *rinfo = &dev->rinfo;
@@ -894,6 +894,41 @@ static int at91_init_twi_recovery_info(struct platform_device *pdev,
 	return 0;
 }
 
+static int at91_twi_recover_bus_cmd(struct i2c_adapter *adap)
+{
+	struct at91_twi_dev *dev = i2c_get_adapdata(adap);
+
+	dev->transfer_status |= at91_twi_read(dev, AT91_TWI_SR);
+	if (!(dev->transfer_status & AT91_TWI_SDA)) {
+		dev_dbg(dev->dev, "SDA is down; sending bus clear command\n");
+		if (dev->use_alt_cmd) {
+			unsigned int acr;
+
+			acr = at91_twi_read(dev, AT91_TWI_ACR);
+			acr &= ~AT91_TWI_ACR_DATAL_MASK;
+			at91_twi_write(dev, AT91_TWI_ACR, acr);
+		}
+		at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_CLEAR);
+	}
+
+	return 0;
+}
+
+static int at91_init_twi_recovery_info(struct platform_device *pdev,
+				       struct at91_twi_dev *dev)
+{
+	struct i2c_bus_recovery_info *rinfo = &dev->rinfo;
+	bool has_clear_cmd = dev->pdata->has_clear_cmd;
+
+	if (!has_clear_cmd)
+		return at91_init_twi_recovery_gpio(pdev, dev);
+
+	rinfo->recover_bus = at91_twi_recover_bus_cmd;
+	dev->adapter.bus_recovery_info = rinfo;
+
+	return 0;
+}
+
 int at91_twi_probe_master(struct platform_device *pdev,
 			  u32 phy_addr, struct at91_twi_dev *dev)
 {
diff --git a/drivers/i2c/busses/i2c-at91.h b/drivers/i2c/busses/i2c-at91.h
index f57a6cab96b4..7e7b4955ca7f 100644
--- a/drivers/i2c/busses/i2c-at91.h
+++ b/drivers/i2c/busses/i2c-at91.h
@@ -36,6 +36,7 @@
 #define	AT91_TWI_SVDIS		BIT(5)	/* Slave Transfer Disable */
 #define	AT91_TWI_QUICK		BIT(6)	/* SMBus quick command */
 #define	AT91_TWI_SWRST		BIT(7)	/* Software Reset */
+#define	AT91_TWI_CLEAR		BIT(15) /* Bus clear command */
 #define	AT91_TWI_ACMEN		BIT(16) /* Alternative Command Mode Enable */
 #define	AT91_TWI_ACMDIS		BIT(17) /* Alternative Command Mode Disable */
 #define	AT91_TWI_THRCLR		BIT(24) /* Transmit Holding Register Clear */
@@ -69,6 +70,8 @@
 #define	AT91_TWI_NACK		BIT(8)	/* Not Acknowledged */
 #define	AT91_TWI_EOSACC		BIT(11)	/* End Of Slave Access */
 #define	AT91_TWI_LOCK		BIT(23) /* TWI Lock due to Frame Errors */
+#define	AT91_TWI_SCL		BIT(24) /* TWI SCL status */
+#define	AT91_TWI_SDA		BIT(25) /* TWI SDA status */
 
 #define	AT91_TWI_INT_MASK \
 	(AT91_TWI_TXCOMP | AT91_TWI_RXRDY | AT91_TWI_TXRDY | AT91_TWI_NACK \
@@ -81,7 +84,8 @@
 #define	AT91_TWI_THR		0x0034	/* Transmit Holding Register */
 
 #define	AT91_TWI_ACR		0x0040	/* Alternative Command Register */
-#define	AT91_TWI_ACR_DATAL(len)	((len) & 0xff)
+#define	AT91_TWI_ACR_DATAL_MASK	GENMASK(15, 0)
+#define	AT91_TWI_ACR_DATAL(len)	((len) & AT91_TWI_ACR_DATAL_MASK)
 #define	AT91_TWI_ACR_DIR	BIT(8)
 
 #define AT91_TWI_FILTR		0x0044
@@ -118,6 +122,7 @@ struct at91_twi_pdata {
 	bool has_dig_filtr;
 	bool has_adv_dig_filtr;
 	bool has_ana_filtr;
+	bool has_clear_cmd;
 	struct at_dma_slave dma_slave;
 };
 
diff --git a/drivers/i2c/busses/i2c-axxia.c b/drivers/i2c/busses/i2c-axxia.c
index be3681d08a8d..5294b73beca8 100644
--- a/drivers/i2c/busses/i2c-axxia.c
+++ b/drivers/i2c/busses/i2c-axxia.c
@@ -734,7 +734,6 @@ static int axxia_i2c_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct axxia_i2c_dev *idev = NULL;
-	struct resource *res;
 	void __iomem *base;
 	int ret = 0;
 
@@ -742,16 +741,13 @@ static int axxia_i2c_probe(struct platform_device *pdev)
 	if (!idev)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(&pdev->dev, res);
+	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
 	idev->irq = platform_get_irq(pdev, 0);
-	if (idev->irq < 0) {
-		dev_err(&pdev->dev, "missing interrupt resource\n");
+	if (idev->irq < 0)
 		return idev->irq;
-	}
 
 	idev->i2c_clk = devm_clk_get(&pdev->dev, "i2c");
 	if (IS_ERR(idev->i2c_clk)) {
diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c
index d091a12596ad..8a3c98866fb7 100644
--- a/drivers/i2c/busses/i2c-bcm-iproc.c
+++ b/drivers/i2c/busses/i2c-bcm-iproc.c
@@ -79,6 +79,7 @@
 #define M_CMD_STATUS_RX_FIFO_FULL    0x6
 #define M_CMD_PROTOCOL_SHIFT         9
 #define M_CMD_PROTOCOL_MASK          0xf
+#define M_CMD_PROTOCOL_QUICK         0x0
 #define M_CMD_PROTOCOL_BLK_WR        0x7
 #define M_CMD_PROTOCOL_BLK_RD        0x8
 #define M_CMD_PROTOCOL_PROCESS       0xa
@@ -768,7 +769,11 @@ static int bcm_iproc_i2c_xfer_internal(struct bcm_iproc_i2c_dev *iproc_i2c,
 	 * number of bytes to read
 	 */
 	val = BIT(M_CMD_START_BUSY_SHIFT);
-	if (msg->flags & I2C_M_RD) {
+
+	if (msg->len == 0) {
+		/* SMBUS QUICK Command (Read/Write) */
+		val |= (M_CMD_PROTOCOL_QUICK << M_CMD_PROTOCOL_SHIFT);
+	} else if (msg->flags & I2C_M_RD) {
 		u32 protocol;
 
 		iproc_i2c->rx_bytes = 0;
@@ -830,8 +835,7 @@ static uint32_t bcm_iproc_i2c_functionality(struct i2c_adapter *adap)
 {
 	u32 val;
 
-	/* We do not support the SMBUS Quick command */
-	val = I2C_FUNC_I2C | (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK);
+	val = I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 
 	if (adap->algo->reg_slave)
 		val |= I2C_FUNC_SLAVE;
diff --git a/drivers/i2c/busses/i2c-bcm-kona.c b/drivers/i2c/busses/i2c-bcm-kona.c
index 572aebbb254e..ed5e1275ae46 100644
--- a/drivers/i2c/busses/i2c-bcm-kona.c
+++ b/drivers/i2c/busses/i2c-bcm-kona.c
@@ -750,7 +750,6 @@ static int bcm_kona_i2c_probe(struct platform_device *pdev)
 	int rc = 0;
 	struct bcm_kona_i2c_dev *dev;
 	struct i2c_adapter *adap;
-	struct resource *iomem;
 
 	/* Allocate memory for private data structure */
 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
@@ -762,8 +761,7 @@ static int bcm_kona_i2c_probe(struct platform_device *pdev)
 	init_completion(&dev->done);
 
 	/* Map hardware registers */
-	iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	dev->base = devm_ioremap_resource(dev->device, iomem);
+	dev->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(dev->base))
 		return -ENOMEM;
 
@@ -823,8 +821,7 @@ static int bcm_kona_i2c_probe(struct platform_device *pdev)
 	/* Get the interrupt number */
 	dev->irq = platform_get_irq(pdev, 0);
 	if (dev->irq < 0) {
-		dev_err(dev->device, "no irq resource\n");
-		rc = -ENODEV;
+		rc = dev->irq;
 		goto probe_disable_clk;
 	}
 
diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c
index 169a2836922d..d4e0a0f6732a 100644
--- a/drivers/i2c/busses/i2c-brcmstb.c
+++ b/drivers/i2c/busses/i2c-brcmstb.c
@@ -647,20 +647,22 @@ static int brcmstb_i2c_probe(struct platform_device *pdev)
 		int_name = NULL;
 
 	/* Get the interrupt number */
-	dev->irq = platform_get_irq(pdev, 0);
+	dev->irq = platform_get_irq_optional(pdev, 0);
 
 	/* disable the bsc interrupt line */
 	brcmstb_i2c_enable_disable_irq(dev, INT_DISABLE);
 
 	/* register the ISR handler */
-	rc = devm_request_irq(&pdev->dev, dev->irq, brcmstb_i2c_isr,
-			      IRQF_SHARED,
-			      int_name ? int_name : pdev->name,
-			      dev);
-
-	if (rc) {
-		dev_dbg(dev->device, "falling back to polling mode");
-		dev->irq = -1;
+	if (dev->irq >= 0) {
+		rc = devm_request_irq(&pdev->dev, dev->irq, brcmstb_i2c_isr,
+				      IRQF_SHARED,
+				      int_name ? int_name : pdev->name,
+				      dev);
+
+		if (rc) {
+			dev_dbg(dev->device, "falling back to polling mode");
+			dev->irq = -1;
+		}
 	}
 
 	if (of_property_read_u32(dev->device->of_node,
diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c
index 89d58f7d2a25..4b72398af505 100644
--- a/drivers/i2c/busses/i2c-cadence.c
+++ b/drivers/i2c/busses/i2c-cadence.c
@@ -23,6 +23,7 @@
 #define CDNS_I2C_ISR_OFFSET		0x10 /* IRQ Status Register, RW */
 #define CDNS_I2C_XFER_SIZE_OFFSET	0x14 /* Transfer Size Register, RW */
 #define CDNS_I2C_TIME_OUT_OFFSET	0x1C /* Time Out Register, RW */
+#define CDNS_I2C_IMR_OFFSET		0x20 /* IRQ Mask Register, RO */
 #define CDNS_I2C_IER_OFFSET		0x24 /* IRQ Enable Register, WO */
 #define CDNS_I2C_IDR_OFFSET		0x28 /* IRQ Disable Register, WO */
 
@@ -40,9 +41,17 @@
 #define CDNS_I2C_CR_DIVB_SHIFT		8
 #define CDNS_I2C_CR_DIVB_MASK		(0x3f << CDNS_I2C_CR_DIVB_SHIFT)
 
+#define CDNS_I2C_CR_MASTER_EN_MASK	(CDNS_I2C_CR_NEA | \
+					 CDNS_I2C_CR_ACK_EN | \
+					 CDNS_I2C_CR_MS)
+
+#define CDNS_I2C_CR_SLAVE_EN_MASK	~CDNS_I2C_CR_MASTER_EN_MASK
+
 /* Status Register Bit mask definitions */
 #define CDNS_I2C_SR_BA		BIT(8)
+#define CDNS_I2C_SR_TXDV	BIT(6)
 #define CDNS_I2C_SR_RXDV	BIT(5)
+#define CDNS_I2C_SR_RXRW	BIT(3)
 
 /*
  * I2C Address Register Bit mask definitions
@@ -91,6 +100,14 @@
 					 CDNS_I2C_IXR_DATA | \
 					 CDNS_I2C_IXR_COMP)
 
+#define CDNS_I2C_IXR_SLAVE_INTR_MASK	(CDNS_I2C_IXR_RX_UNF | \
+					 CDNS_I2C_IXR_TX_OVF | \
+					 CDNS_I2C_IXR_RX_OVF | \
+					 CDNS_I2C_IXR_TO | \
+					 CDNS_I2C_IXR_NACK | \
+					 CDNS_I2C_IXR_DATA | \
+					 CDNS_I2C_IXR_COMP)
+
 #define CDNS_I2C_TIMEOUT		msecs_to_jiffies(1000)
 /* timeout for pm runtime autosuspend */
 #define CNDS_I2C_PM_TIMEOUT		1000	/* ms */
@@ -114,6 +131,32 @@
 #define cdns_i2c_readreg(offset)       readl_relaxed(id->membase + offset)
 #define cdns_i2c_writereg(val, offset) writel_relaxed(val, id->membase + offset)
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+/**
+ * enum cdns_i2c_mode - I2C Controller current operating mode
+ *
+ * @CDNS_I2C_MODE_SLAVE:       I2C controller operating in slave mode
+ * @CDNS_I2C_MODE_MASTER:      I2C Controller operating in master mode
+ */
+enum cdns_i2c_mode {
+	CDNS_I2C_MODE_SLAVE,
+	CDNS_I2C_MODE_MASTER,
+};
+
+/**
+ * enum cdns_i2c_slave_mode - Slave state when I2C is operating in slave mode
+ *
+ * @CDNS_I2C_SLAVE_STATE_IDLE: I2C slave idle
+ * @CDNS_I2C_SLAVE_STATE_SEND: I2C slave sending data to master
+ * @CDNS_I2C_SLAVE_STATE_RECV: I2C slave receiving data from master
+ */
+enum cdns_i2c_slave_state {
+	CDNS_I2C_SLAVE_STATE_IDLE,
+	CDNS_I2C_SLAVE_STATE_SEND,
+	CDNS_I2C_SLAVE_STATE_RECV,
+};
+#endif
+
 /**
  * struct cdns_i2c - I2C device private data structure
  *
@@ -135,6 +178,10 @@
  * @clk:		Pointer to struct clk
  * @clk_rate_change_nb:	Notifier block for clock rate changes
  * @quirks:		flag for broken hold bit usage in r1p10
+ * @ctrl_reg_diva_divb: value of fields DIV_A and DIV_B from CR register
+ * @slave:		Registered slave instance.
+ * @dev_mode:		I2C operating role(master/slave).
+ * @slave_state:	I2C Slave state(idle/read/write).
  */
 struct cdns_i2c {
 	struct device		*dev;
@@ -155,6 +202,12 @@ struct cdns_i2c {
 	struct clk *clk;
 	struct notifier_block clk_rate_change_nb;
 	u32 quirks;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u16 ctrl_reg_diva_divb;
+	struct i2c_client *slave;
+	enum cdns_i2c_mode dev_mode;
+	enum cdns_i2c_slave_state slave_state;
+#endif
 };
 
 struct cdns_platform_data {
@@ -183,17 +236,155 @@ static inline bool cdns_is_holdquirk(struct cdns_i2c *id, bool hold_wrkaround)
 		(id->curr_recv_count == CDNS_I2C_FIFO_DEPTH + 1));
 }
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+static void cdns_i2c_set_mode(enum cdns_i2c_mode mode, struct cdns_i2c *id)
+{
+	/* Disable all interrupts */
+	cdns_i2c_writereg(CDNS_I2C_IXR_ALL_INTR_MASK, CDNS_I2C_IDR_OFFSET);
+
+	/* Clear FIFO and transfer size */
+	cdns_i2c_writereg(CDNS_I2C_CR_CLR_FIFO, CDNS_I2C_CR_OFFSET);
+
+	/* Update device mode and state */
+	id->dev_mode = mode;
+	id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE;
+
+	switch (mode) {
+	case CDNS_I2C_MODE_MASTER:
+		/* Enable i2c master */
+		cdns_i2c_writereg(id->ctrl_reg_diva_divb |
+				  CDNS_I2C_CR_MASTER_EN_MASK,
+				  CDNS_I2C_CR_OFFSET);
+		/*
+		 * This delay is needed to give the IP some time to switch to
+		 * the master mode. With lower values(like 110 us) i2cdetect
+		 * will not detect any slave and without this delay, the IP will
+		 * trigger a timeout interrupt.
+		 */
+		usleep_range(115, 125);
+		break;
+	case CDNS_I2C_MODE_SLAVE:
+		/* Enable i2c slave */
+		cdns_i2c_writereg(id->ctrl_reg_diva_divb &
+				  CDNS_I2C_CR_SLAVE_EN_MASK,
+				  CDNS_I2C_CR_OFFSET);
+
+		/* Setting slave address */
+		cdns_i2c_writereg(id->slave->addr & CDNS_I2C_ADDR_MASK,
+				  CDNS_I2C_ADDR_OFFSET);
+
+		/* Enable slave send/receive interrupts */
+		cdns_i2c_writereg(CDNS_I2C_IXR_SLAVE_INTR_MASK,
+				  CDNS_I2C_IER_OFFSET);
+		break;
+	}
+}
+
+static void cdns_i2c_slave_rcv_data(struct cdns_i2c *id)
+{
+	u8 bytes;
+	unsigned char data;
+
+	/* Prepare backend for data reception */
+	if (id->slave_state == CDNS_I2C_SLAVE_STATE_IDLE) {
+		id->slave_state = CDNS_I2C_SLAVE_STATE_RECV;
+		i2c_slave_event(id->slave, I2C_SLAVE_WRITE_REQUESTED, NULL);
+	}
+
+	/* Fetch number of bytes to receive */
+	bytes = cdns_i2c_readreg(CDNS_I2C_XFER_SIZE_OFFSET);
+
+	/* Read data and send to backend */
+	while (bytes--) {
+		data = cdns_i2c_readreg(CDNS_I2C_DATA_OFFSET);
+		i2c_slave_event(id->slave, I2C_SLAVE_WRITE_RECEIVED, &data);
+	}
+}
+
+static void cdns_i2c_slave_send_data(struct cdns_i2c *id)
+{
+	u8 data;
+
+	/* Prepare backend for data transmission */
+	if (id->slave_state == CDNS_I2C_SLAVE_STATE_IDLE) {
+		id->slave_state = CDNS_I2C_SLAVE_STATE_SEND;
+		i2c_slave_event(id->slave, I2C_SLAVE_READ_REQUESTED, &data);
+	} else {
+		i2c_slave_event(id->slave, I2C_SLAVE_READ_PROCESSED, &data);
+	}
+
+	/* Send data over bus */
+	cdns_i2c_writereg(data, CDNS_I2C_DATA_OFFSET);
+}
+
 /**
- * cdns_i2c_isr - Interrupt handler for the I2C device
- * @irq:	irq number for the I2C device
- * @ptr:	void pointer to cdns_i2c structure
+ * cdns_i2c_slave_isr - Interrupt handler for the I2C device in slave role
+ * @ptr:       Pointer to I2C device private data
+ *
+ * This function handles the data interrupt and transfer complete interrupt of
+ * the I2C device in slave role.
+ *
+ * Return: IRQ_HANDLED always
+ */
+static irqreturn_t cdns_i2c_slave_isr(void *ptr)
+{
+	struct cdns_i2c *id = ptr;
+	unsigned int isr_status, i2c_status;
+
+	/* Fetch the interrupt status */
+	isr_status = cdns_i2c_readreg(CDNS_I2C_ISR_OFFSET);
+	cdns_i2c_writereg(isr_status, CDNS_I2C_ISR_OFFSET);
+
+	/* Ignore masked interrupts */
+	isr_status &= ~cdns_i2c_readreg(CDNS_I2C_IMR_OFFSET);
+
+	/* Fetch transfer mode (send/receive) */
+	i2c_status = cdns_i2c_readreg(CDNS_I2C_SR_OFFSET);
+
+	/* Handle data send/receive */
+	if (i2c_status & CDNS_I2C_SR_RXRW) {
+		/* Send data to master */
+		if (isr_status & CDNS_I2C_IXR_DATA)
+			cdns_i2c_slave_send_data(id);
+
+		if (isr_status & CDNS_I2C_IXR_COMP) {
+			id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE;
+			i2c_slave_event(id->slave, I2C_SLAVE_STOP, NULL);
+		}
+	} else {
+		/* Receive data from master */
+		if (isr_status & CDNS_I2C_IXR_DATA)
+			cdns_i2c_slave_rcv_data(id);
+
+		if (isr_status & CDNS_I2C_IXR_COMP) {
+			cdns_i2c_slave_rcv_data(id);
+			id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE;
+			i2c_slave_event(id->slave, I2C_SLAVE_STOP, NULL);
+		}
+	}
+
+	/* Master indicated xfer stop or fifo underflow/overflow */
+	if (isr_status & (CDNS_I2C_IXR_NACK | CDNS_I2C_IXR_RX_OVF |
+			  CDNS_I2C_IXR_RX_UNF | CDNS_I2C_IXR_TX_OVF)) {
+		id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE;
+		i2c_slave_event(id->slave, I2C_SLAVE_STOP, NULL);
+		cdns_i2c_writereg(CDNS_I2C_CR_CLR_FIFO, CDNS_I2C_CR_OFFSET);
+	}
+
+	return IRQ_HANDLED;
+}
+#endif
+
+/**
+ * cdns_i2c_master_isr - Interrupt handler for the I2C device in master role
+ * @ptr:       Pointer to I2C device private data
  *
  * This function handles the data interrupt, transfer complete interrupt and
- * the error interrupts of the I2C device.
+ * the error interrupts of the I2C device in master role.
  *
  * Return: IRQ_HANDLED always
  */
-static irqreturn_t cdns_i2c_isr(int irq, void *ptr)
+static irqreturn_t cdns_i2c_master_isr(void *ptr)
 {
 	unsigned int isr_status, avail_bytes, updatetx;
 	unsigned int bytes_to_send;
@@ -358,6 +549,27 @@ static irqreturn_t cdns_i2c_isr(int irq, void *ptr)
 }
 
 /**
+ * cdns_i2c_isr - Interrupt handler for the I2C device
+ * @irq:	irq number for the I2C device
+ * @ptr:	void pointer to cdns_i2c structure
+ *
+ * This function passes the control to slave/master based on current role of
+ * i2c controller.
+ *
+ * Return: IRQ_HANDLED always
+ */
+static irqreturn_t cdns_i2c_isr(int irq, void *ptr)
+{
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	struct cdns_i2c *id = ptr;
+
+	if (id->dev_mode == CDNS_I2C_MODE_SLAVE)
+		return cdns_i2c_slave_isr(ptr);
+#endif
+	return cdns_i2c_master_isr(ptr);
+}
+
+/**
  * cdns_i2c_mrecv - Prepare and start a master receive operation
  * @id:		pointer to the i2c device structure
  */
@@ -577,10 +789,28 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 	u32 reg;
 	struct cdns_i2c *id = adap->algo_data;
 	bool hold_quirk;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	bool change_role = false;
+#endif
 
 	ret = pm_runtime_get_sync(id->dev);
 	if (ret < 0)
 		return ret;
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	/* Check i2c operating mode and switch if possible */
+	if (id->dev_mode == CDNS_I2C_MODE_SLAVE) {
+		if (id->slave_state != CDNS_I2C_SLAVE_STATE_IDLE)
+			return -EAGAIN;
+
+		/* Set mode to master */
+		cdns_i2c_set_mode(CDNS_I2C_MODE_MASTER, id);
+
+		/* Mark flag to change role once xfer is completed */
+		change_role = true;
+	}
+#endif
+
 	/* Check if the bus is free */
 	if (cdns_i2c_readreg(CDNS_I2C_SR_OFFSET) & CDNS_I2C_SR_BA) {
 		ret = -EAGAIN;
@@ -639,7 +869,15 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 	}
 
 	ret = num;
+
 out:
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	/* Switch i2c mode to slave */
+	if (change_role)
+		cdns_i2c_set_mode(CDNS_I2C_MODE_SLAVE, id);
+#endif
+
 	pm_runtime_mark_last_busy(id->dev);
 	pm_runtime_put_autosuspend(id->dev);
 	return ret;
@@ -653,14 +891,67 @@ out:
  */
 static u32 cdns_i2c_func(struct i2c_adapter *adap)
 {
-	return I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR |
-		(I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) |
-		I2C_FUNC_SMBUS_BLOCK_DATA;
+	u32 func = I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR |
+			(I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) |
+			I2C_FUNC_SMBUS_BLOCK_DATA;
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	func |= I2C_FUNC_SLAVE;
+#endif
+
+	return func;
 }
 
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+static int cdns_reg_slave(struct i2c_client *slave)
+{
+	int ret;
+	struct cdns_i2c *id = container_of(slave->adapter, struct cdns_i2c,
+									adap);
+
+	if (id->slave)
+		return -EBUSY;
+
+	if (slave->flags & I2C_CLIENT_TEN)
+		return -EAFNOSUPPORT;
+
+	ret = pm_runtime_get_sync(id->dev);
+	if (ret < 0)
+		return ret;
+
+	/* Store slave information */
+	id->slave = slave;
+
+	/* Enable I2C slave */
+	cdns_i2c_set_mode(CDNS_I2C_MODE_SLAVE, id);
+
+	return 0;
+}
+
+static int cdns_unreg_slave(struct i2c_client *slave)
+{
+	struct cdns_i2c *id = container_of(slave->adapter, struct cdns_i2c,
+									adap);
+
+	pm_runtime_put(id->dev);
+
+	/* Remove slave information */
+	id->slave = NULL;
+
+	/* Enable I2C master */
+	cdns_i2c_set_mode(CDNS_I2C_MODE_MASTER, id);
+
+	return 0;
+}
+#endif
+
 static const struct i2c_algorithm cdns_i2c_algo = {
 	.master_xfer	= cdns_i2c_master_xfer,
 	.functionality	= cdns_i2c_func,
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	.reg_slave	= cdns_reg_slave,
+	.unreg_slave	= cdns_unreg_slave,
+#endif
 };
 
 /**
@@ -755,7 +1046,10 @@ static int cdns_i2c_setclk(unsigned long clk_in, struct cdns_i2c *id)
 	ctrl_reg |= ((div_a << CDNS_I2C_CR_DIVA_SHIFT) |
 			(div_b << CDNS_I2C_CR_DIVB_SHIFT));
 	cdns_i2c_writereg(ctrl_reg, CDNS_I2C_CR_OFFSET);
-
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	id->ctrl_reg_diva_divb = ctrl_reg & (CDNS_I2C_CR_DIVA_MASK |
+				 CDNS_I2C_CR_DIVB_MASK);
+#endif
 	return 0;
 }
 
@@ -906,8 +1200,7 @@ static int cdns_i2c_probe(struct platform_device *pdev)
 		id->quirks = data->quirks;
 	}
 
-	r_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	id->membase = devm_ioremap_resource(&pdev->dev, r_mem);
+	id->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &r_mem);
 	if (IS_ERR(id->membase))
 		return PTR_ERR(id->membase);
 
@@ -949,8 +1242,12 @@ static int cdns_i2c_probe(struct platform_device *pdev)
 	if (ret || (id->i2c_clk > I2C_MAX_FAST_MODE_FREQ))
 		id->i2c_clk = I2C_MAX_STANDARD_MODE_FREQ;
 
-	cdns_i2c_writereg(CDNS_I2C_CR_ACK_EN | CDNS_I2C_CR_NEA | CDNS_I2C_CR_MS,
-			  CDNS_I2C_CR_OFFSET);
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	/* Set initial mode to master */
+	id->dev_mode = CDNS_I2C_MODE_MASTER;
+	id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE;
+#endif
+	cdns_i2c_writereg(CDNS_I2C_CR_MASTER_EN_MASK, CDNS_I2C_CR_OFFSET);
 
 	ret = cdns_i2c_setclk(id->input_clk, id);
 	if (ret) {
diff --git a/drivers/i2c/busses/i2c-cht-wc.c b/drivers/i2c/busses/i2c-cht-wc.c
index 35e55feda763..f80d79e973cd 100644
--- a/drivers/i2c/busses/i2c-cht-wc.c
+++ b/drivers/i2c/busses/i2c-cht-wc.c
@@ -314,10 +314,8 @@ static int cht_wc_i2c_adap_i2c_probe(struct platform_device *pdev)
 	int ret, reg, irq;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "Error missing irq resource\n");
-		return -EINVAL;
-	}
+	if (irq < 0)
+		return irq;
 
 	adap = devm_kzalloc(&pdev->dev, sizeof(*adap), GFP_KERNEL);
 	if (!adap)
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index e3ceb256a380..232a7679b69b 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -761,7 +761,6 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 {
 	struct davinci_i2c_dev *dev;
 	struct i2c_adapter *adap;
-	struct resource *mem;
 	struct i2c_bus_recovery_info *rinfo;
 	int r, irq;
 
@@ -814,8 +813,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 	if (IS_ERR(dev->clk))
 		return PTR_ERR(dev->clk);
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	dev->base = devm_ioremap_resource(&pdev->dev, mem);
+	dev->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(dev->base)) {
 		return PTR_ERR(dev->base);
 	}
diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c
index c70c6fc09ee3..e3a8640db7da 100644
--- a/drivers/i2c/busses/i2c-designware-common.c
+++ b/drivers/i2c/busses/i2c-designware-common.c
@@ -8,17 +8,22 @@
  * Copyright (C) 2007 MontaVista Software Inc.
  * Copyright (C) 2009 Provigent Ltd.
  */
+#include <linux/acpi.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/export.h>
-#include <linux/errno.h>
+#include <linux/device.h>
 #include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/export.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/regmap.h>
 #include <linux/swab.h>
+#include <linux/types.h>
 
 #include "i2c-designware-core.h"
 
@@ -53,69 +58,267 @@ static char *abort_sources[] = {
 		"incorrect slave-transmitter mode configuration",
 };
 
-u32 dw_readl(struct dw_i2c_dev *dev, int offset)
+static int dw_reg_read(void *context, unsigned int reg, unsigned int *val)
 {
-	u32 value;
+	struct dw_i2c_dev *dev = context;
 
-	if (dev->flags & ACCESS_16BIT)
-		value = readw_relaxed(dev->base + offset) |
-			(readw_relaxed(dev->base + offset + 2) << 16);
-	else
-		value = readl_relaxed(dev->base + offset);
+	*val = readl_relaxed(dev->base + reg);
 
-	if (dev->flags & ACCESS_SWAP)
-		return swab32(value);
-	else
-		return value;
+	return 0;
 }
 
-void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset)
+static int dw_reg_write(void *context, unsigned int reg, unsigned int val)
 {
-	if (dev->flags & ACCESS_SWAP)
-		b = swab32(b);
-
-	if (dev->flags & ACCESS_16BIT) {
-		writew_relaxed((u16)b, dev->base + offset);
-		writew_relaxed((u16)(b >> 16), dev->base + offset + 2);
-	} else {
-		writel_relaxed(b, dev->base + offset);
-	}
+	struct dw_i2c_dev *dev = context;
+
+	writel_relaxed(val, dev->base + reg);
+
+	return 0;
+}
+
+static int dw_reg_read_swab(void *context, unsigned int reg, unsigned int *val)
+{
+	struct dw_i2c_dev *dev = context;
+
+	*val = swab32(readl_relaxed(dev->base + reg));
+
+	return 0;
+}
+
+static int dw_reg_write_swab(void *context, unsigned int reg, unsigned int val)
+{
+	struct dw_i2c_dev *dev = context;
+
+	writel_relaxed(swab32(val), dev->base + reg);
+
+	return 0;
+}
+
+static int dw_reg_read_word(void *context, unsigned int reg, unsigned int *val)
+{
+	struct dw_i2c_dev *dev = context;
+
+	*val = readw_relaxed(dev->base + reg) |
+		(readw_relaxed(dev->base + reg + 2) << 16);
+
+	return 0;
+}
+
+static int dw_reg_write_word(void *context, unsigned int reg, unsigned int val)
+{
+	struct dw_i2c_dev *dev = context;
+
+	writew_relaxed(val, dev->base + reg);
+	writew_relaxed(val >> 16, dev->base + reg + 2);
+
+	return 0;
 }
 
 /**
- * i2c_dw_set_reg_access() - Set register access flags
+ * i2c_dw_init_regmap() - Initialize registers map
  * @dev: device private data
  *
- * Autodetects needed register access mode and sets access flags accordingly.
- * This must be called before doing any other register access.
+ * Autodetects needed register access mode and creates the regmap with
+ * corresponding read/write callbacks. This must be called before doing any
+ * other register access.
  */
-int i2c_dw_set_reg_access(struct dw_i2c_dev *dev)
+int i2c_dw_init_regmap(struct dw_i2c_dev *dev)
 {
+	struct regmap_config map_cfg = {
+		.reg_bits = 32,
+		.val_bits = 32,
+		.reg_stride = 4,
+		.disable_locking = true,
+		.reg_read = dw_reg_read,
+		.reg_write = dw_reg_write,
+		.max_register = DW_IC_COMP_TYPE,
+	};
 	u32 reg;
 	int ret;
 
+	/*
+	 * Skip detecting the registers map configuration if the regmap has
+	 * already been provided by a higher code.
+	 */
+	if (dev->map)
+		return 0;
+
 	ret = i2c_dw_acquire_lock(dev);
 	if (ret)
 		return ret;
 
-	reg = dw_readl(dev, DW_IC_COMP_TYPE);
+	reg = readl(dev->base + DW_IC_COMP_TYPE);
 	i2c_dw_release_lock(dev);
 
 	if (reg == swab32(DW_IC_COMP_TYPE_VALUE)) {
-		/* Configure register endianness access */
-		dev->flags |= ACCESS_SWAP;
+		map_cfg.reg_read = dw_reg_read_swab;
+		map_cfg.reg_write = dw_reg_write_swab;
 	} else if (reg == (DW_IC_COMP_TYPE_VALUE & 0x0000ffff)) {
-		/* Configure register access mode 16bit */
-		dev->flags |= ACCESS_16BIT;
+		map_cfg.reg_read = dw_reg_read_word;
+		map_cfg.reg_write = dw_reg_write_word;
 	} else if (reg != DW_IC_COMP_TYPE_VALUE) {
 		dev_err(dev->dev,
 			"Unknown Synopsys component type: 0x%08x\n", reg);
 		return -ENODEV;
 	}
 
+	/*
+	 * Note we'll check the return value of the regmap IO accessors only
+	 * at the probe stage. The rest of the code won't do this because
+	 * basically we have MMIO-based regmap so non of the read/write methods
+	 * can fail.
+	 */
+	dev->map = devm_regmap_init(dev->dev, NULL, dev, &map_cfg);
+	if (IS_ERR(dev->map)) {
+		dev_err(dev->dev, "Failed to init the registers map\n");
+		return PTR_ERR(dev->map);
+	}
+
 	return 0;
 }
 
+static const u32 supported_speeds[] = {
+	I2C_MAX_HIGH_SPEED_MODE_FREQ,
+	I2C_MAX_FAST_MODE_PLUS_FREQ,
+	I2C_MAX_FAST_MODE_FREQ,
+	I2C_MAX_STANDARD_MODE_FREQ,
+};
+
+int i2c_dw_validate_speed(struct dw_i2c_dev *dev)
+{
+	struct i2c_timings *t = &dev->timings;
+	unsigned int i;
+
+	/*
+	 * Only standard mode at 100kHz, fast mode at 400kHz,
+	 * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported.
+	 */
+	for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) {
+		if (t->bus_freq_hz == supported_speeds[i])
+			return 0;
+	}
+
+	dev_err(dev->dev,
+		"%d Hz is unsupported, only 100kHz, 400kHz, 1MHz and 3.4MHz are supported\n",
+		t->bus_freq_hz);
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL_GPL(i2c_dw_validate_speed);
+
+#ifdef CONFIG_ACPI
+
+#include <linux/dmi.h>
+
+/*
+ * The HCNT/LCNT information coming from ACPI should be the most accurate
+ * for given platform. However, some systems get it wrong. On such systems
+ * we get better results by calculating those based on the input clock.
+ */
+static const struct dmi_system_id i2c_dw_no_acpi_params[] = {
+	{
+		.ident = "Dell Inspiron 7348",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 7348"),
+		},
+	},
+	{}
+};
+
+static void i2c_dw_acpi_params(struct device *device, char method[],
+			       u16 *hcnt, u16 *lcnt, u32 *sda_hold)
+{
+	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER };
+	acpi_handle handle = ACPI_HANDLE(device);
+	union acpi_object *obj;
+
+	if (dmi_check_system(i2c_dw_no_acpi_params))
+		return;
+
+	if (ACPI_FAILURE(acpi_evaluate_object(handle, method, NULL, &buf)))
+		return;
+
+	obj = (union acpi_object *)buf.pointer;
+	if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 3) {
+		const union acpi_object *objs = obj->package.elements;
+
+		*hcnt = (u16)objs[0].integer.value;
+		*lcnt = (u16)objs[1].integer.value;
+		*sda_hold = (u32)objs[2].integer.value;
+	}
+
+	kfree(buf.pointer);
+}
+
+int i2c_dw_acpi_configure(struct device *device)
+{
+	struct dw_i2c_dev *dev = dev_get_drvdata(device);
+	struct i2c_timings *t = &dev->timings;
+	u32 ss_ht = 0, fp_ht = 0, hs_ht = 0, fs_ht = 0;
+
+	/*
+	 * Try to get SDA hold time and *CNT values from an ACPI method for
+	 * selected speed modes.
+	 */
+	i2c_dw_acpi_params(device, "SSCN", &dev->ss_hcnt, &dev->ss_lcnt, &ss_ht);
+	i2c_dw_acpi_params(device, "FPCN", &dev->fp_hcnt, &dev->fp_lcnt, &fp_ht);
+	i2c_dw_acpi_params(device, "HSCN", &dev->hs_hcnt, &dev->hs_lcnt, &hs_ht);
+	i2c_dw_acpi_params(device, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt, &fs_ht);
+
+	switch (t->bus_freq_hz) {
+	case I2C_MAX_STANDARD_MODE_FREQ:
+		dev->sda_hold_time = ss_ht;
+		break;
+	case I2C_MAX_FAST_MODE_PLUS_FREQ:
+		dev->sda_hold_time = fp_ht;
+		break;
+	case I2C_MAX_HIGH_SPEED_MODE_FREQ:
+		dev->sda_hold_time = hs_ht;
+		break;
+	case I2C_MAX_FAST_MODE_FREQ:
+	default:
+		dev->sda_hold_time = fs_ht;
+		break;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(i2c_dw_acpi_configure);
+
+void i2c_dw_acpi_adjust_bus_speed(struct device *device)
+{
+	struct dw_i2c_dev *dev = dev_get_drvdata(device);
+	struct i2c_timings *t = &dev->timings;
+	u32 acpi_speed;
+	int i;
+
+	acpi_speed = i2c_acpi_find_bus_speed(device);
+	/*
+	 * Some DSTDs use a non standard speed, round down to the lowest
+	 * standard speed.
+	 */
+	for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) {
+		if (acpi_speed >= supported_speeds[i])
+			break;
+	}
+	acpi_speed = i < ARRAY_SIZE(supported_speeds) ? supported_speeds[i] : 0;
+
+	/*
+	 * Find bus speed from the "clock-frequency" device property, ACPI
+	 * or by using fast mode if neither is set.
+	 */
+	if (acpi_speed && t->bus_freq_hz)
+		t->bus_freq_hz = min(t->bus_freq_hz, acpi_speed);
+	else if (acpi_speed || t->bus_freq_hz)
+		t->bus_freq_hz = max(t->bus_freq_hz, acpi_speed);
+	else
+		t->bus_freq_hz = I2C_MAX_FAST_MODE_FREQ;
+}
+EXPORT_SYMBOL_GPL(i2c_dw_acpi_adjust_bus_speed);
+
+#endif	/* CONFIG_ACPI */
+
 u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset)
 {
 	/*
@@ -181,11 +384,17 @@ int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev)
 		return ret;
 
 	/* Configure SDA Hold Time if required */
-	reg = dw_readl(dev, DW_IC_COMP_VERSION);
+	ret = regmap_read(dev->map, DW_IC_COMP_VERSION, &reg);
+	if (ret)
+		goto err_release_lock;
+
 	if (reg >= DW_IC_SDA_HOLD_MIN_VERS) {
 		if (!dev->sda_hold_time) {
 			/* Keep previous hold time setting if no one set it */
-			dev->sda_hold_time = dw_readl(dev, DW_IC_SDA_HOLD);
+			ret = regmap_read(dev->map, DW_IC_SDA_HOLD,
+					  &dev->sda_hold_time);
+			if (ret)
+				goto err_release_lock;
 		}
 
 		/*
@@ -209,14 +418,16 @@ int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev)
 		dev->sda_hold_time = 0;
 	}
 
+err_release_lock:
 	i2c_dw_release_lock(dev);
 
-	return 0;
+	return ret;
 }
 
 void __i2c_dw_disable(struct dw_i2c_dev *dev)
 {
 	int timeout = 100;
+	u32 status;
 
 	do {
 		__i2c_dw_disable_nowait(dev);
@@ -224,7 +435,8 @@ void __i2c_dw_disable(struct dw_i2c_dev *dev)
 		 * The enable status register may be unimplemented, but
 		 * in that case this test reads zero and exits the loop.
 		 */
-		if ((dw_readl(dev, DW_IC_ENABLE_STATUS) & 1) == 0)
+		regmap_read(dev->map, DW_IC_ENABLE_STATUS, &status);
+		if ((status & 1) == 0)
 			return;
 
 		/*
@@ -303,22 +515,23 @@ void i2c_dw_release_lock(struct dw_i2c_dev *dev)
  */
 int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev)
 {
-	int timeout = TIMEOUT;
+	u32 status;
+	int ret;
 
-	while (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) {
-		if (timeout <= 0) {
-			dev_warn(dev->dev, "timeout waiting for bus ready\n");
-			i2c_recover_bus(&dev->adapter);
+	ret = regmap_read_poll_timeout(dev->map, DW_IC_STATUS, status,
+				       !(status & DW_IC_STATUS_ACTIVITY),
+				       1100, 20000);
+	if (ret) {
+		dev_warn(dev->dev, "timeout waiting for bus ready\n");
 
-			if (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY)
-				return -ETIMEDOUT;
-			return 0;
-		}
-		timeout--;
-		usleep_range(1000, 1100);
+		i2c_recover_bus(&dev->adapter);
+
+		regmap_read(dev->map, DW_IC_STATUS, &status);
+		if (!(status & DW_IC_STATUS_ACTIVITY))
+			ret = 0;
 	}
 
-	return 0;
+	return ret;
 }
 
 int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev)
@@ -344,15 +557,19 @@ int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev)
 		return -EIO;
 }
 
-void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev)
+int i2c_dw_set_fifo_size(struct dw_i2c_dev *dev)
 {
 	u32 param, tx_fifo_depth, rx_fifo_depth;
+	int ret;
 
 	/*
 	 * Try to detect the FIFO depth if not set by interface driver,
 	 * the depth could be from 2 to 256 from HW spec.
 	 */
-	param = dw_readl(dev, DW_IC_COMP_PARAM_1);
+	ret = regmap_read(dev->map, DW_IC_COMP_PARAM_1, &param);
+	if (ret)
+		return ret;
+
 	tx_fifo_depth = ((param >> 16) & 0xff) + 1;
 	rx_fifo_depth = ((param >> 8)  & 0xff) + 1;
 	if (!dev->tx_fifo_depth) {
@@ -364,6 +581,8 @@ void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev)
 		dev->rx_fifo_depth = min_t(u32, dev->rx_fifo_depth,
 				rx_fifo_depth);
 	}
+
+	return 0;
 }
 
 u32 i2c_dw_func(struct i2c_adapter *adap)
@@ -375,17 +594,19 @@ u32 i2c_dw_func(struct i2c_adapter *adap)
 
 void i2c_dw_disable(struct dw_i2c_dev *dev)
 {
+	u32 dummy;
+
 	/* Disable controller */
 	__i2c_dw_disable(dev);
 
 	/* Disable all interrupts */
-	dw_writel(dev, 0, DW_IC_INTR_MASK);
-	dw_readl(dev, DW_IC_CLR_INTR);
+	regmap_write(dev->map, DW_IC_INTR_MASK, 0);
+	regmap_read(dev->map, DW_IC_CLR_INTR, &dummy);
 }
 
 void i2c_dw_disable_int(struct dw_i2c_dev *dev)
 {
-	dw_writel(dev, 0, DW_IC_INTR_MASK);
+	regmap_write(dev->map, DW_IC_INTR_MASK, 0);
 }
 
 MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter core");
diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h
index b220ad64c38d..556673a1f61b 100644
--- a/drivers/i2c/busses/i2c-designware-core.h
+++ b/drivers/i2c/busses/i2c-designware-core.h
@@ -9,7 +9,14 @@
  * Copyright (C) 2009 Provigent Ltd.
  */
 
+#include <linux/bits.h>
+#include <linux/compiler_types.h>
+#include <linux/completion.h>
+#include <linux/dev_printk.h>
+#include <linux/errno.h>
 #include <linux/i2c.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
 
 #define DW_IC_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C |			\
 					I2C_FUNC_SMBUS_BYTE |		\
@@ -120,8 +127,6 @@
 #define STATUS_WRITE_IN_PROGRESS	0x1
 #define STATUS_READ_IN_PROGRESS		0x2
 
-#define TIMEOUT			20 /* ms */
-
 /*
  * operation modes
  */
@@ -170,11 +175,17 @@
 					 DW_IC_TX_ABRT_TXDATA_NOACK | \
 					 DW_IC_TX_ABRT_GCALL_NOACK)
 
+struct clk;
+struct device;
+struct reset_control;
 
 /**
  * struct dw_i2c_dev - private i2c-designware data
  * @dev: driver model device node
+ * @map: IO registers map
+ * @sysmap: System controller registers map
  * @base: IO registers pointer
+ * @ext: Extended IO registers pointer
  * @cmd_complete: tx completion indicator
  * @clk: input reference clock
  * @pclk: clock required to access the registers
@@ -224,6 +235,8 @@
  */
 struct dw_i2c_dev {
 	struct device		*dev;
+	struct regmap		*map;
+	struct regmap		*sysmap;
 	void __iomem		*base;
 	void __iomem		*ext;
 	struct completion	cmd_complete;
@@ -232,7 +245,6 @@ struct dw_i2c_dev {
 	struct reset_control	*rst;
 	struct i2c_client		*slave;
 	u32			(*get_clk_rate_khz) (struct dw_i2c_dev *dev);
-	struct dw_pci_controller *controller;
 	int			cmd_err;
 	struct i2c_msg		*msgs;
 	int			msgs_num;
@@ -276,18 +288,14 @@ struct dw_i2c_dev {
 	bool			suspended;
 };
 
-#define ACCESS_SWAP		0x00000001
-#define ACCESS_16BIT		0x00000002
-#define ACCESS_INTR_MASK	0x00000004
-#define ACCESS_NO_IRQ_SUSPEND	0x00000008
+#define ACCESS_INTR_MASK	0x00000001
+#define ACCESS_NO_IRQ_SUSPEND	0x00000002
 
-#define MODEL_CHERRYTRAIL	0x00000100
-#define MODEL_MSCC_OCELOT	0x00000200
+#define MODEL_MSCC_OCELOT	0x00000100
+#define MODEL_BAIKAL_BT1	0x00000200
 #define MODEL_MASK		0x00000f00
 
-u32 dw_readl(struct dw_i2c_dev *dev, int offset);
-void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset);
-int i2c_dw_set_reg_access(struct dw_i2c_dev *dev);
+int i2c_dw_init_regmap(struct dw_i2c_dev *dev);
 u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset);
 u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset);
 int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev);
@@ -297,32 +305,67 @@ int i2c_dw_acquire_lock(struct dw_i2c_dev *dev);
 void i2c_dw_release_lock(struct dw_i2c_dev *dev);
 int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev);
 int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev);
-void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev);
+int i2c_dw_set_fifo_size(struct dw_i2c_dev *dev);
 u32 i2c_dw_func(struct i2c_adapter *adap);
 void i2c_dw_disable(struct dw_i2c_dev *dev);
 void i2c_dw_disable_int(struct dw_i2c_dev *dev);
 
 static inline void __i2c_dw_enable(struct dw_i2c_dev *dev)
 {
-	dw_writel(dev, 1, DW_IC_ENABLE);
+	regmap_write(dev->map, DW_IC_ENABLE, 1);
 }
 
 static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev)
 {
-	dw_writel(dev, 0, DW_IC_ENABLE);
+	regmap_write(dev->map, DW_IC_ENABLE, 0);
 }
 
 void __i2c_dw_disable(struct dw_i2c_dev *dev);
 
-extern int i2c_dw_probe(struct dw_i2c_dev *dev);
+extern void i2c_dw_configure_master(struct dw_i2c_dev *dev);
+extern int i2c_dw_probe_master(struct dw_i2c_dev *dev);
+
 #if IS_ENABLED(CONFIG_I2C_DESIGNWARE_SLAVE)
+extern void i2c_dw_configure_slave(struct dw_i2c_dev *dev);
 extern int i2c_dw_probe_slave(struct dw_i2c_dev *dev);
 #else
+static inline void i2c_dw_configure_slave(struct dw_i2c_dev *dev) { }
 static inline int i2c_dw_probe_slave(struct dw_i2c_dev *dev) { return -EINVAL; }
 #endif
 
+static inline int i2c_dw_probe(struct dw_i2c_dev *dev)
+{
+	switch (dev->mode) {
+	case DW_IC_SLAVE:
+		return i2c_dw_probe_slave(dev);
+	case DW_IC_MASTER:
+		return i2c_dw_probe_master(dev);
+	default:
+		dev_err(dev->dev, "Wrong operation mode: %d\n", dev->mode);
+		return -EINVAL;
+	}
+}
+
+static inline void i2c_dw_configure(struct dw_i2c_dev *dev)
+{
+	if (i2c_detect_slave_mode(dev->dev))
+		i2c_dw_configure_slave(dev);
+	else
+		i2c_dw_configure_master(dev);
+}
+
 #if IS_ENABLED(CONFIG_I2C_DESIGNWARE_BAYTRAIL)
 extern int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev);
 #else
 static inline int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev) { return 0; }
 #endif
+
+int i2c_dw_validate_speed(struct dw_i2c_dev *dev);
+
+#if IS_ENABLED(CONFIG_ACPI)
+int i2c_dw_acpi_configure(struct device *device);
+void i2c_dw_acpi_adjust_bus_speed(struct device *device);
+#else
+static inline int i2c_dw_acpi_configure(struct device *device) { return -ENODEV; }
+static inline void i2c_dw_acpi_adjust_bus_speed(struct device *device) {}
+#endif
diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c
index 3a58eef20936..d6425ad6e6a3 100644
--- a/drivers/i2c/busses/i2c-designware-master.c
+++ b/drivers/i2c/busses/i2c-designware-master.c
@@ -18,6 +18,7 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/regmap.h>
 #include <linux/reset.h>
 
 #include "i2c-designware-core.h"
@@ -25,11 +26,11 @@
 static void i2c_dw_configure_fifo_master(struct dw_i2c_dev *dev)
 {
 	/* Configure Tx/Rx FIFO threshold levels */
-	dw_writel(dev, dev->tx_fifo_depth / 2, DW_IC_TX_TL);
-	dw_writel(dev, 0, DW_IC_RX_TL);
+	regmap_write(dev->map, DW_IC_TX_TL, dev->tx_fifo_depth / 2);
+	regmap_write(dev->map, DW_IC_RX_TL, 0);
 
 	/* Configure the I2C master */
-	dw_writel(dev, dev->master_cfg, DW_IC_CON);
+	regmap_write(dev->map, DW_IC_CON, dev->master_cfg);
 }
 
 static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev)
@@ -44,8 +45,11 @@ static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev)
 	ret = i2c_dw_acquire_lock(dev);
 	if (ret)
 		return ret;
-	comp_param1 = dw_readl(dev, DW_IC_COMP_PARAM_1);
+
+	ret = regmap_read(dev->map, DW_IC_COMP_PARAM_1, &comp_param1);
 	i2c_dw_release_lock(dev);
+	if (ret)
+		return ret;
 
 	/* Set standard and fast speed dividers for high/low periods */
 	sda_falling_time = t->sda_fall_ns ?: 300; /* ns */
@@ -76,14 +80,27 @@ static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev)
 	 */
 	if (t->bus_freq_hz == 1000000) {
 		/*
-		 * Check are fast mode plus parameters available and use
-		 * fast mode if not.
+		 * Check are Fast Mode Plus parameters available. Calculate
+		 * SCL timing parameters for Fast Mode Plus if not set.
 		 */
 		if (dev->fp_hcnt && dev->fp_lcnt) {
 			dev->fs_hcnt = dev->fp_hcnt;
 			dev->fs_lcnt = dev->fp_lcnt;
-			fp_str = " Plus";
+		} else {
+			ic_clk = i2c_dw_clk_rate(dev);
+			dev->fs_hcnt =
+				i2c_dw_scl_hcnt(ic_clk,
+						260,	/* tHIGH = 260 ns */
+						sda_falling_time,
+						0,	/* DW default */
+						0);	/* No offset */
+			dev->fs_lcnt =
+				i2c_dw_scl_lcnt(ic_clk,
+						500,	/* tLOW = 500 ns */
+						scl_falling_time,
+						0);	/* No offset */
 		}
+		fp_str = " Plus";
 	}
 	/*
 	 * Calculate SCL timing parameters for fast mode if not set. They are
@@ -116,10 +133,22 @@ static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev)
 			dev->master_cfg |= DW_IC_CON_SPEED_FAST;
 			dev->hs_hcnt = 0;
 			dev->hs_lcnt = 0;
-		} else if (dev->hs_hcnt && dev->hs_lcnt) {
-			dev_dbg(dev->dev, "High Speed Mode HCNT:LCNT = %d:%d\n",
-				dev->hs_hcnt, dev->hs_lcnt);
+		} else if (!dev->hs_hcnt || !dev->hs_lcnt) {
+			ic_clk = i2c_dw_clk_rate(dev);
+			dev->hs_hcnt =
+				i2c_dw_scl_hcnt(ic_clk,
+						160,	/* tHIGH = 160 ns */
+						sda_falling_time,
+						0,	/* DW default */
+						0);	/* No offset */
+			dev->hs_lcnt =
+				i2c_dw_scl_lcnt(ic_clk,
+						320,	/* tLOW = 320 ns */
+						scl_falling_time,
+						0);	/* No offset */
 		}
+		dev_dbg(dev->dev, "High Speed Mode HCNT:LCNT = %d:%d\n",
+			dev->hs_hcnt, dev->hs_lcnt);
 	}
 
 	ret = i2c_dw_set_sda_hold(dev);
@@ -162,22 +191,22 @@ static int i2c_dw_init_master(struct dw_i2c_dev *dev)
 	__i2c_dw_disable(dev);
 
 	/* Write standard speed timing parameters */
-	dw_writel(dev, dev->ss_hcnt, DW_IC_SS_SCL_HCNT);
-	dw_writel(dev, dev->ss_lcnt, DW_IC_SS_SCL_LCNT);
+	regmap_write(dev->map, DW_IC_SS_SCL_HCNT, dev->ss_hcnt);
+	regmap_write(dev->map, DW_IC_SS_SCL_LCNT, dev->ss_lcnt);
 
 	/* Write fast mode/fast mode plus timing parameters */
-	dw_writel(dev, dev->fs_hcnt, DW_IC_FS_SCL_HCNT);
-	dw_writel(dev, dev->fs_lcnt, DW_IC_FS_SCL_LCNT);
+	regmap_write(dev->map, DW_IC_FS_SCL_HCNT, dev->fs_hcnt);
+	regmap_write(dev->map, DW_IC_FS_SCL_LCNT, dev->fs_lcnt);
 
 	/* Write high speed timing parameters if supported */
 	if (dev->hs_hcnt && dev->hs_lcnt) {
-		dw_writel(dev, dev->hs_hcnt, DW_IC_HS_SCL_HCNT);
-		dw_writel(dev, dev->hs_lcnt, DW_IC_HS_SCL_LCNT);
+		regmap_write(dev->map, DW_IC_HS_SCL_HCNT, dev->hs_hcnt);
+		regmap_write(dev->map, DW_IC_HS_SCL_LCNT, dev->hs_lcnt);
 	}
 
 	/* Write SDA hold time if supported */
 	if (dev->sda_hold_time)
-		dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD);
+		regmap_write(dev->map, DW_IC_SDA_HOLD, dev->sda_hold_time);
 
 	i2c_dw_configure_fifo_master(dev);
 	i2c_dw_release_lock(dev);
@@ -188,15 +217,15 @@ static int i2c_dw_init_master(struct dw_i2c_dev *dev)
 static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 {
 	struct i2c_msg *msgs = dev->msgs;
-	u32 ic_con, ic_tar = 0;
+	u32 ic_con = 0, ic_tar = 0;
+	u32 dummy;
 
 	/* Disable the adapter */
 	__i2c_dw_disable(dev);
 
 	/* If the slave address is ten bit address, enable 10BITADDR */
-	ic_con = dw_readl(dev, DW_IC_CON);
 	if (msgs[dev->msg_write_idx].flags & I2C_M_TEN) {
-		ic_con |= DW_IC_CON_10BITADDR_MASTER;
+		ic_con = DW_IC_CON_10BITADDR_MASTER;
 		/*
 		 * If I2C_DYNAMIC_TAR_UPDATE is set, the 10-bit addressing
 		 * mode has to be enabled via bit 12 of IC_TAR register.
@@ -204,17 +233,17 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 		 * detected from registers.
 		 */
 		ic_tar = DW_IC_TAR_10BITADDR_MASTER;
-	} else {
-		ic_con &= ~DW_IC_CON_10BITADDR_MASTER;
 	}
 
-	dw_writel(dev, ic_con, DW_IC_CON);
+	regmap_update_bits(dev->map, DW_IC_CON, DW_IC_CON_10BITADDR_MASTER,
+			   ic_con);
 
 	/*
 	 * Set the slave (target) address and enable 10-bit addressing mode
 	 * if applicable.
 	 */
-	dw_writel(dev, msgs[dev->msg_write_idx].addr | ic_tar, DW_IC_TAR);
+	regmap_write(dev->map, DW_IC_TAR,
+		     msgs[dev->msg_write_idx].addr | ic_tar);
 
 	/* Enforce disabled interrupts (due to HW issues) */
 	i2c_dw_disable_int(dev);
@@ -223,11 +252,11 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 	__i2c_dw_enable(dev);
 
 	/* Dummy read to avoid the register getting stuck on Bay Trail */
-	dw_readl(dev, DW_IC_ENABLE_STATUS);
+	regmap_read(dev->map, DW_IC_ENABLE_STATUS, &dummy);
 
 	/* Clear and enable interrupts */
-	dw_readl(dev, DW_IC_CLR_INTR);
-	dw_writel(dev, DW_IC_INTR_MASTER_MASK, DW_IC_INTR_MASK);
+	regmap_read(dev->map, DW_IC_CLR_INTR, &dummy);
+	regmap_write(dev->map, DW_IC_INTR_MASK, DW_IC_INTR_MASTER_MASK);
 }
 
 /*
@@ -246,6 +275,7 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 	u32 buf_len = dev->tx_buf_len;
 	u8 *buf = dev->tx_buf;
 	bool need_restart = false;
+	unsigned int flr;
 
 	intr_mask = DW_IC_INTR_MASTER_MASK;
 
@@ -278,8 +308,11 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 				need_restart = true;
 		}
 
-		tx_limit = dev->tx_fifo_depth - dw_readl(dev, DW_IC_TXFLR);
-		rx_limit = dev->rx_fifo_depth - dw_readl(dev, DW_IC_RXFLR);
+		regmap_read(dev->map, DW_IC_TXFLR, &flr);
+		tx_limit = dev->tx_fifo_depth - flr;
+
+		regmap_read(dev->map, DW_IC_RXFLR, &flr);
+		rx_limit = dev->rx_fifo_depth - flr;
 
 		while (buf_len > 0 && tx_limit > 0 && rx_limit > 0) {
 			u32 cmd = 0;
@@ -312,11 +345,14 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 				if (dev->rx_outstanding >= dev->rx_fifo_depth)
 					break;
 
-				dw_writel(dev, cmd | 0x100, DW_IC_DATA_CMD);
+				regmap_write(dev->map, DW_IC_DATA_CMD,
+					     cmd | 0x100);
 				rx_limit--;
 				dev->rx_outstanding++;
-			} else
-				dw_writel(dev, cmd | *buf++, DW_IC_DATA_CMD);
+			} else {
+				regmap_write(dev->map, DW_IC_DATA_CMD,
+					     cmd | *buf++);
+			}
 			tx_limit--; buf_len--;
 		}
 
@@ -346,7 +382,7 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev)
 	if (dev->msg_err)
 		intr_mask = 0;
 
-	dw_writel(dev, intr_mask,  DW_IC_INTR_MASK);
+	regmap_write(dev->map,  DW_IC_INTR_MASK, intr_mask);
 }
 
 static u8
@@ -371,10 +407,10 @@ static void
 i2c_dw_read(struct dw_i2c_dev *dev)
 {
 	struct i2c_msg *msgs = dev->msgs;
-	int rx_valid;
+	unsigned int rx_valid;
 
 	for (; dev->msg_read_idx < dev->msgs_num; dev->msg_read_idx++) {
-		u32 len;
+		u32 len, tmp;
 		u8 *buf;
 
 		if (!(msgs[dev->msg_read_idx].flags & I2C_M_RD))
@@ -388,18 +424,18 @@ i2c_dw_read(struct dw_i2c_dev *dev)
 			buf = dev->rx_buf;
 		}
 
-		rx_valid = dw_readl(dev, DW_IC_RXFLR);
+		regmap_read(dev->map, DW_IC_RXFLR, &rx_valid);
 
 		for (; len > 0 && rx_valid > 0; len--, rx_valid--) {
 			u32 flags = msgs[dev->msg_read_idx].flags;
 
-			*buf = dw_readl(dev, DW_IC_DATA_CMD);
+			regmap_read(dev->map, DW_IC_DATA_CMD, &tmp);
 			/* Ensure length byte is a valid value */
 			if (flags & I2C_M_RECV_LEN &&
-				*buf <= I2C_SMBUS_BLOCK_MAX && *buf > 0) {
-				len = i2c_dw_recv_len(dev, *buf);
+			    tmp <= I2C_SMBUS_BLOCK_MAX && tmp > 0) {
+				len = i2c_dw_recv_len(dev, tmp);
 			}
-			buf++;
+			*buf++ = tmp;
 			dev->rx_outstanding--;
 		}
 
@@ -517,7 +553,7 @@ static const struct i2c_adapter_quirks i2c_dw_quirks = {
 
 static u32 i2c_dw_read_clear_intrbits(struct dw_i2c_dev *dev)
 {
-	u32 stat;
+	u32 stat, dummy;
 
 	/*
 	 * The IC_INTR_STAT register just indicates "enabled" interrupts.
@@ -525,47 +561,47 @@ static u32 i2c_dw_read_clear_intrbits(struct dw_i2c_dev *dev)
 	 * in the IC_RAW_INTR_STAT register.
 	 *
 	 * That is,
-	 *   stat = dw_readl(IC_INTR_STAT);
+	 *   stat = readl(IC_INTR_STAT);
 	 * equals to,
-	 *   stat = dw_readl(IC_RAW_INTR_STAT) & dw_readl(IC_INTR_MASK);
+	 *   stat = readl(IC_RAW_INTR_STAT) & readl(IC_INTR_MASK);
 	 *
 	 * The raw version might be useful for debugging purposes.
 	 */
-	stat = dw_readl(dev, DW_IC_INTR_STAT);
+	regmap_read(dev->map, DW_IC_INTR_STAT, &stat);
 
 	/*
 	 * Do not use the IC_CLR_INTR register to clear interrupts, or
 	 * you'll miss some interrupts, triggered during the period from
-	 * dw_readl(IC_INTR_STAT) to dw_readl(IC_CLR_INTR).
+	 * readl(IC_INTR_STAT) to readl(IC_CLR_INTR).
 	 *
 	 * Instead, use the separately-prepared IC_CLR_* registers.
 	 */
 	if (stat & DW_IC_INTR_RX_UNDER)
-		dw_readl(dev, DW_IC_CLR_RX_UNDER);
+		regmap_read(dev->map, DW_IC_CLR_RX_UNDER, &dummy);
 	if (stat & DW_IC_INTR_RX_OVER)
-		dw_readl(dev, DW_IC_CLR_RX_OVER);
+		regmap_read(dev->map, DW_IC_CLR_RX_OVER, &dummy);
 	if (stat & DW_IC_INTR_TX_OVER)
-		dw_readl(dev, DW_IC_CLR_TX_OVER);
+		regmap_read(dev->map, DW_IC_CLR_TX_OVER, &dummy);
 	if (stat & DW_IC_INTR_RD_REQ)
-		dw_readl(dev, DW_IC_CLR_RD_REQ);
+		regmap_read(dev->map, DW_IC_CLR_RD_REQ, &dummy);
 	if (stat & DW_IC_INTR_TX_ABRT) {
 		/*
 		 * The IC_TX_ABRT_SOURCE register is cleared whenever
 		 * the IC_CLR_TX_ABRT is read.  Preserve it beforehand.
 		 */
-		dev->abort_source = dw_readl(dev, DW_IC_TX_ABRT_SOURCE);
-		dw_readl(dev, DW_IC_CLR_TX_ABRT);
+		regmap_read(dev->map, DW_IC_TX_ABRT_SOURCE, &dev->abort_source);
+		regmap_read(dev->map, DW_IC_CLR_TX_ABRT, &dummy);
 	}
 	if (stat & DW_IC_INTR_RX_DONE)
-		dw_readl(dev, DW_IC_CLR_RX_DONE);
+		regmap_read(dev->map, DW_IC_CLR_RX_DONE, &dummy);
 	if (stat & DW_IC_INTR_ACTIVITY)
-		dw_readl(dev, DW_IC_CLR_ACTIVITY);
+		regmap_read(dev->map, DW_IC_CLR_ACTIVITY, &dummy);
 	if (stat & DW_IC_INTR_STOP_DET)
-		dw_readl(dev, DW_IC_CLR_STOP_DET);
+		regmap_read(dev->map, DW_IC_CLR_STOP_DET, &dummy);
 	if (stat & DW_IC_INTR_START_DET)
-		dw_readl(dev, DW_IC_CLR_START_DET);
+		regmap_read(dev->map, DW_IC_CLR_START_DET, &dummy);
 	if (stat & DW_IC_INTR_GEN_CALL)
-		dw_readl(dev, DW_IC_CLR_GEN_CALL);
+		regmap_read(dev->map, DW_IC_CLR_GEN_CALL, &dummy);
 
 	return stat;
 }
@@ -587,7 +623,7 @@ static int i2c_dw_irq_handler_master(struct dw_i2c_dev *dev)
 		 * Anytime TX_ABRT is set, the contents of the tx/rx
 		 * buffers are flushed. Make sure to skip them.
 		 */
-		dw_writel(dev, 0, DW_IC_INTR_MASK);
+		regmap_write(dev->map, DW_IC_INTR_MASK, 0);
 		goto tx_aborted;
 	}
 
@@ -608,9 +644,9 @@ tx_aborted:
 		complete(&dev->cmd_complete);
 	else if (unlikely(dev->flags & ACCESS_INTR_MASK)) {
 		/* Workaround to trigger pending interrupt */
-		stat = dw_readl(dev, DW_IC_INTR_MASK);
+		regmap_read(dev->map, DW_IC_INTR_MASK, &stat);
 		i2c_dw_disable_int(dev);
-		dw_writel(dev, stat, DW_IC_INTR_MASK);
+		regmap_write(dev->map, DW_IC_INTR_MASK, stat);
 	}
 
 	return 0;
@@ -621,8 +657,8 @@ static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id)
 	struct dw_i2c_dev *dev = dev_id;
 	u32 stat, enabled;
 
-	enabled = dw_readl(dev, DW_IC_ENABLE);
-	stat = dw_readl(dev, DW_IC_RAW_INTR_STAT);
+	regmap_read(dev->map, DW_IC_ENABLE, &enabled);
+	regmap_read(dev->map, DW_IC_RAW_INTR_STAT, &stat);
 	dev_dbg(dev->dev, "enabled=%#x stat=%#x\n", enabled, stat);
 	if (!enabled || !(stat & ~DW_IC_INTR_ACTIVITY))
 		return IRQ_NONE;
@@ -632,6 +668,30 @@ static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+void i2c_dw_configure_master(struct dw_i2c_dev *dev)
+{
+	struct i2c_timings *t = &dev->timings;
+
+	dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY;
+
+	dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE |
+			  DW_IC_CON_RESTART_EN;
+
+	dev->mode = DW_IC_MASTER;
+
+	switch (t->bus_freq_hz) {
+	case I2C_MAX_STANDARD_MODE_FREQ:
+		dev->master_cfg |= DW_IC_CON_SPEED_STD;
+		break;
+	case I2C_MAX_HIGH_SPEED_MODE_FREQ:
+		dev->master_cfg |= DW_IC_CON_SPEED_HIGH;
+		break;
+	default:
+		dev->master_cfg |= DW_IC_CON_SPEED_FAST;
+	}
+}
+EXPORT_SYMBOL_GPL(i2c_dw_configure_master);
+
 static void i2c_dw_prepare_recovery(struct i2c_adapter *adap)
 {
 	struct dw_i2c_dev *dev = i2c_get_adapdata(adap);
@@ -678,7 +738,7 @@ static int i2c_dw_init_recovery_info(struct dw_i2c_dev *dev)
 	return 0;
 }
 
-int i2c_dw_probe(struct dw_i2c_dev *dev)
+int i2c_dw_probe_master(struct dw_i2c_dev *dev)
 {
 	struct i2c_adapter *adap = &dev->adapter;
 	unsigned long irq_flags;
@@ -690,7 +750,7 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 	dev->disable = i2c_dw_disable;
 	dev->disable_int = i2c_dw_disable_int;
 
-	ret = i2c_dw_set_reg_access(dev);
+	ret = i2c_dw_init_regmap(dev);
 	if (ret)
 		return ret;
 
@@ -698,7 +758,9 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 	if (ret)
 		return ret;
 
-	i2c_dw_set_fifo_size(dev);
+	ret = i2c_dw_set_fifo_size(dev);
+	if (ret)
+		return ret;
 
 	ret = dev->init(dev);
 	if (ret)
@@ -745,7 +807,7 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(i2c_dw_probe);
+EXPORT_SYMBOL_GPL(i2c_dw_probe_master);
 
 MODULE_DESCRIPTION("Synopsys DesignWare I2C bus master adapter");
 MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c
index 7a0b65b5b5b5..947c096f86e3 100644
--- a/drivers/i2c/busses/i2c-designware-pcidrv.c
+++ b/drivers/i2c/busses/i2c-designware-pcidrv.c
@@ -46,20 +46,12 @@ struct dw_scl_sda_cfg {
 
 struct dw_pci_controller {
 	u32 bus_num;
-	u32 bus_cfg;
-	u32 tx_fifo_depth;
-	u32 rx_fifo_depth;
-	u32 clk_khz;
-	u32 functionality;
 	u32 flags;
 	struct dw_scl_sda_cfg *scl_sda_cfg;
 	int (*setup)(struct pci_dev *pdev, struct dw_pci_controller *c);
+	u32 (*get_clk_rate_khz)(struct dw_i2c_dev *dev);
 };
 
-#define INTEL_MID_STD_CFG  (DW_IC_CON_MASTER |			\
-				DW_IC_CON_SLAVE_DISABLE |	\
-				DW_IC_CON_RESTART_EN)
-
 /* Merrifield HCNT/LCNT/SDA hold time */
 static struct dw_scl_sda_cfg mrfld_config = {
 	.ss_hcnt = 0x2f8,
@@ -86,12 +78,18 @@ static struct dw_scl_sda_cfg hsw_config = {
 	.sda_hold = 0x9,
 };
 
+static u32 mfld_get_clk_rate_khz(struct dw_i2c_dev *dev)
+{
+	return 25000;
+}
+
 static int mfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c)
 {
+	struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev);
+
 	switch (pdev->device) {
 	case 0x0817:
-		c->bus_cfg &= ~DW_IC_CON_SPEED_MASK;
-		c->bus_cfg |= DW_IC_CON_SPEED_STD;
+		dev->timings.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ;
 		/* fall through */
 	case 0x0818:
 	case 0x0819:
@@ -125,57 +123,37 @@ static int mrfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c)
 	return -ENODEV;
 }
 
+static u32 ehl_get_clk_rate_khz(struct dw_i2c_dev *dev)
+{
+	return 100000;
+}
+
 static struct dw_pci_controller dw_pci_controllers[] = {
 	[medfield] = {
 		.bus_num = -1,
-		.bus_cfg   = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
-		.tx_fifo_depth = 32,
-		.rx_fifo_depth = 32,
-		.functionality = I2C_FUNC_10BIT_ADDR,
-		.clk_khz      = 25000,
 		.setup = mfld_setup,
+		.get_clk_rate_khz = mfld_get_clk_rate_khz,
 	},
 	[merrifield] = {
 		.bus_num = -1,
-		.bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
-		.tx_fifo_depth = 64,
-		.rx_fifo_depth = 64,
-		.functionality = I2C_FUNC_10BIT_ADDR,
 		.scl_sda_cfg = &mrfld_config,
 		.setup = mrfld_setup,
 	},
 	[baytrail] = {
 		.bus_num = -1,
-		.bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
-		.tx_fifo_depth = 32,
-		.rx_fifo_depth = 32,
-		.functionality = I2C_FUNC_10BIT_ADDR,
 		.scl_sda_cfg = &byt_config,
 	},
 	[haswell] = {
 		.bus_num = -1,
-		.bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
-		.tx_fifo_depth = 32,
-		.rx_fifo_depth = 32,
-		.functionality = I2C_FUNC_10BIT_ADDR,
 		.scl_sda_cfg = &hsw_config,
 	},
 	[cherrytrail] = {
 		.bus_num = -1,
-		.bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
-		.tx_fifo_depth = 32,
-		.rx_fifo_depth = 32,
-		.functionality = I2C_FUNC_10BIT_ADDR,
-		.flags = MODEL_CHERRYTRAIL,
 		.scl_sda_cfg = &byt_config,
 	},
 	[elkhartlake] = {
 		.bus_num = -1,
-		.bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST,
-		.tx_fifo_depth = 32,
-		.rx_fifo_depth = 32,
-		.functionality = I2C_FUNC_10BIT_ADDR,
-		.clk_khz = 100000,
+		.get_clk_rate_khz = ehl_get_clk_rate_khz,
 	},
 };
 
@@ -205,11 +183,6 @@ static int i2c_dw_pci_resume(struct device *dev)
 static UNIVERSAL_DEV_PM_OPS(i2c_dw_pm_ops, i2c_dw_pci_suspend,
 			    i2c_dw_pci_resume, NULL);
 
-static u32 i2c_dw_get_clk_rate_khz(struct dw_i2c_dev *dev)
-{
-	return dev->controller->clk_khz;
-}
-
 static int i2c_dw_pci_probe(struct pci_dev *pdev,
 			    const struct pci_device_id *id)
 {
@@ -250,14 +223,15 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev,
 	if (r < 0)
 		return r;
 
-	dev->clk = NULL;
-	dev->controller = controller;
-	dev->get_clk_rate_khz = i2c_dw_get_clk_rate_khz;
+	dev->get_clk_rate_khz = controller->get_clk_rate_khz;
+	dev->timings.bus_freq_hz = I2C_MAX_FAST_MODE_FREQ;
 	dev->base = pcim_iomap_table(pdev)[0];
 	dev->dev = &pdev->dev;
 	dev->irq = pci_irq_vector(pdev, 0);
 	dev->flags |= controller->flags;
 
+	pci_set_drvdata(pdev, dev);
+
 	if (controller->setup) {
 		r = controller->setup(pdev, controller);
 		if (r) {
@@ -266,10 +240,19 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev,
 		}
 	}
 
-	dev->functionality = controller->functionality |
-				DW_IC_DEFAULT_FUNCTIONALITY;
+	i2c_dw_acpi_adjust_bus_speed(&pdev->dev);
+
+	if (has_acpi_companion(&pdev->dev))
+		i2c_dw_acpi_configure(&pdev->dev);
+
+	r = i2c_dw_validate_speed(dev);
+	if (r) {
+		pci_free_irq_vectors(pdev);
+		return r;
+	}
+
+	i2c_dw_configure(dev);
 
-	dev->master_cfg = controller->bus_cfg;
 	if (controller->scl_sda_cfg) {
 		cfg = controller->scl_sda_cfg;
 		dev->ss_hcnt = cfg->ss_hcnt;
@@ -279,11 +262,6 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev,
 		dev->sda_hold_time = cfg->sda_hold;
 	}
 
-	pci_set_drvdata(pdev, dev);
-
-	dev->tx_fifo_depth = controller->tx_fifo_depth;
-	dev->rx_fifo_depth = controller->rx_fifo_depth;
-
 	adap = &dev->adapter;
 	adap->owner = THIS_MODULE;
 	adap->class = 0;
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index c429d664f655..0de4e302fc6a 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -12,13 +12,13 @@
 #include <linux/clk-provider.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/dmi.h>
 #include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_data/i2c-designware.h>
@@ -26,6 +26,7 @@
 #include <linux/pm.h>
 #include <linux/pm_runtime.h>
 #include <linux/property.h>
+#include <linux/regmap.h>
 #include <linux/reset.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
@@ -39,91 +40,13 @@ static u32 i2c_dw_get_clk_rate_khz(struct dw_i2c_dev *dev)
 }
 
 #ifdef CONFIG_ACPI
-/*
- * The HCNT/LCNT information coming from ACPI should be the most accurate
- * for given platform. However, some systems get it wrong. On such systems
- * we get better results by calculating those based on the input clock.
- */
-static const struct dmi_system_id dw_i2c_no_acpi_params[] = {
-	{
-		.ident = "Dell Inspiron 7348",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 7348"),
-		},
-	},
-	{ }
-};
-
-static void dw_i2c_acpi_params(struct platform_device *pdev, char method[],
-			       u16 *hcnt, u16 *lcnt, u32 *sda_hold)
-{
-	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER };
-	acpi_handle handle = ACPI_HANDLE(&pdev->dev);
-	union acpi_object *obj;
-
-	if (dmi_check_system(dw_i2c_no_acpi_params))
-		return;
-
-	if (ACPI_FAILURE(acpi_evaluate_object(handle, method, NULL, &buf)))
-		return;
-
-	obj = (union acpi_object *)buf.pointer;
-	if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 3) {
-		const union acpi_object *objs = obj->package.elements;
-
-		*hcnt = (u16)objs[0].integer.value;
-		*lcnt = (u16)objs[1].integer.value;
-		*sda_hold = (u32)objs[2].integer.value;
-	}
-
-	kfree(buf.pointer);
-}
-
-static int dw_i2c_acpi_configure(struct platform_device *pdev)
-{
-	struct dw_i2c_dev *dev = platform_get_drvdata(pdev);
-	struct i2c_timings *t = &dev->timings;
-	u32 ss_ht = 0, fp_ht = 0, hs_ht = 0, fs_ht = 0;
-
-	dev->tx_fifo_depth = 32;
-	dev->rx_fifo_depth = 32;
-
-	/*
-	 * Try to get SDA hold time and *CNT values from an ACPI method for
-	 * selected speed modes.
-	 */
-	dw_i2c_acpi_params(pdev, "SSCN", &dev->ss_hcnt, &dev->ss_lcnt, &ss_ht);
-	dw_i2c_acpi_params(pdev, "FPCN", &dev->fp_hcnt, &dev->fp_lcnt, &fp_ht);
-	dw_i2c_acpi_params(pdev, "HSCN", &dev->hs_hcnt, &dev->hs_lcnt, &hs_ht);
-	dw_i2c_acpi_params(pdev, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt, &fs_ht);
-
-	switch (t->bus_freq_hz) {
-	case I2C_MAX_STANDARD_MODE_FREQ:
-		dev->sda_hold_time = ss_ht;
-		break;
-	case I2C_MAX_FAST_MODE_PLUS_FREQ:
-		dev->sda_hold_time = fp_ht;
-		break;
-	case I2C_MAX_HIGH_SPEED_MODE_FREQ:
-		dev->sda_hold_time = hs_ht;
-		break;
-	case I2C_MAX_FAST_MODE_FREQ:
-	default:
-		dev->sda_hold_time = fs_ht;
-		break;
-	}
-
-	return 0;
-}
-
 static const struct acpi_device_id dw_i2c_acpi_match[] = {
 	{ "INT33C2", 0 },
 	{ "INT33C3", 0 },
 	{ "INT3432", 0 },
 	{ "INT3433", 0 },
 	{ "80860F41", ACCESS_NO_IRQ_SUSPEND },
-	{ "808622C1", ACCESS_NO_IRQ_SUSPEND | MODEL_CHERRYTRAIL },
+	{ "808622C1", ACCESS_NO_IRQ_SUSPEND },
 	{ "AMD0010", ACCESS_INTR_MASK },
 	{ "AMDI0010", ACCESS_INTR_MASK },
 	{ "AMDI0510", 0 },
@@ -134,14 +57,66 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = {
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, dw_i2c_acpi_match);
-#else
-static inline int dw_i2c_acpi_configure(struct platform_device *pdev)
-{
-	return -ENODEV;
-}
 #endif
 
 #ifdef CONFIG_OF
+#define BT1_I2C_CTL			0x100
+#define BT1_I2C_CTL_ADDR_MASK		GENMASK(7, 0)
+#define BT1_I2C_CTL_WR			BIT(8)
+#define BT1_I2C_CTL_GO			BIT(31)
+#define BT1_I2C_DI			0x104
+#define BT1_I2C_DO			0x108
+
+static int bt1_i2c_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct dw_i2c_dev *dev = context;
+	int ret;
+
+	/*
+	 * Note these methods shouldn't ever fail because the system controller
+	 * registers are memory mapped. We check the return value just in case.
+	 */
+	ret = regmap_write(dev->sysmap, BT1_I2C_CTL,
+			   BT1_I2C_CTL_GO | (reg & BT1_I2C_CTL_ADDR_MASK));
+	if (ret)
+		return ret;
+
+	return regmap_read(dev->sysmap, BT1_I2C_DO, val);
+}
+
+static int bt1_i2c_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct dw_i2c_dev *dev = context;
+	int ret;
+
+	ret = regmap_write(dev->sysmap, BT1_I2C_DI, val);
+	if (ret)
+		return ret;
+
+	return regmap_write(dev->sysmap, BT1_I2C_CTL,
+		BT1_I2C_CTL_GO | BT1_I2C_CTL_WR | (reg & BT1_I2C_CTL_ADDR_MASK));
+}
+
+static struct regmap_config bt1_i2c_cfg = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.fast_io = true,
+	.reg_read = bt1_i2c_read,
+	.reg_write = bt1_i2c_write,
+	.max_register = DW_IC_COMP_TYPE,
+};
+
+static int bt1_i2c_request_regs(struct dw_i2c_dev *dev)
+{
+	dev->sysmap = syscon_node_to_regmap(dev->dev->of_node->parent);
+	if (IS_ERR(dev->sysmap))
+		return PTR_ERR(dev->sysmap);
+
+	dev->map = devm_regmap_init(dev->dev, NULL, dev, &bt1_i2c_cfg);
+	return PTR_ERR_OR_ZERO(dev->map);
+}
+
 #define MSCC_ICPU_CFG_TWI_DELAY		0x0
 #define MSCC_ICPU_CFG_TWI_DELAY_ENABLE	BIT(0)
 #define MSCC_ICPU_CFG_TWI_SPIKE_FILTER	0x4
@@ -157,12 +132,10 @@ static int mscc_twi_set_sda_hold_time(struct dw_i2c_dev *dev)
 static int dw_i2c_of_configure(struct platform_device *pdev)
 {
 	struct dw_i2c_dev *dev = platform_get_drvdata(pdev);
-	struct resource *mem;
 
 	switch (dev->flags & MODEL_MASK) {
 	case MODEL_MSCC_OCELOT:
-		mem = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-		dev->ext = devm_ioremap_resource(&pdev->dev, mem);
+		dev->ext = devm_platform_ioremap_resource(pdev, 1);
 		if (!IS_ERR(dev->ext))
 			dev->set_sda_hold_time = mscc_twi_set_sda_hold_time;
 		break;
@@ -176,48 +149,21 @@ static int dw_i2c_of_configure(struct platform_device *pdev)
 static const struct of_device_id dw_i2c_of_match[] = {
 	{ .compatible = "snps,designware-i2c", },
 	{ .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT },
+	{ .compatible = "baikal,bt1-sys-i2c", .data = (void *)MODEL_BAIKAL_BT1 },
 	{},
 };
 MODULE_DEVICE_TABLE(of, dw_i2c_of_match);
 #else
-static inline int dw_i2c_of_configure(struct platform_device *pdev)
+static int bt1_i2c_request_regs(struct dw_i2c_dev *dev)
 {
 	return -ENODEV;
 }
-#endif
-
-static void i2c_dw_configure_master(struct dw_i2c_dev *dev)
-{
-	struct i2c_timings *t = &dev->timings;
-
-	dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY;
-
-	dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE |
-			  DW_IC_CON_RESTART_EN;
-
-	dev->mode = DW_IC_MASTER;
-
-	switch (t->bus_freq_hz) {
-	case I2C_MAX_STANDARD_MODE_FREQ:
-		dev->master_cfg |= DW_IC_CON_SPEED_STD;
-		break;
-	case I2C_MAX_HIGH_SPEED_MODE_FREQ:
-		dev->master_cfg |= DW_IC_CON_SPEED_HIGH;
-		break;
-	default:
-		dev->master_cfg |= DW_IC_CON_SPEED_FAST;
-	}
-}
 
-static void i2c_dw_configure_slave(struct dw_i2c_dev *dev)
+static inline int dw_i2c_of_configure(struct platform_device *pdev)
 {
-	dev->functionality = I2C_FUNC_SLAVE | DW_IC_DEFAULT_FUNCTIONALITY;
-
-	dev->slave_cfg = DW_IC_CON_RX_FIFO_FULL_HLD_CTRL |
-			 DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED;
-
-	dev->mode = DW_IC_SLAVE;
+	return -ENODEV;
 }
+#endif
 
 static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev)
 {
@@ -227,12 +173,23 @@ static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev)
 		pm_runtime_put_noidle(dev->dev);
 }
 
-static const u32 supported_speeds[] = {
-	I2C_MAX_HIGH_SPEED_MODE_FREQ,
-	I2C_MAX_FAST_MODE_PLUS_FREQ,
-	I2C_MAX_FAST_MODE_FREQ,
-	I2C_MAX_STANDARD_MODE_FREQ,
-};
+static int dw_i2c_plat_request_regs(struct dw_i2c_dev *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev->dev);
+	int ret;
+
+	switch (dev->flags & MODEL_MASK) {
+	case MODEL_BAIKAL_BT1:
+		ret = bt1_i2c_request_regs(dev);
+		break;
+	default:
+		dev->base = devm_platform_ioremap_resource(pdev, 0);
+		ret = PTR_ERR_OR_ZERO(dev->base);
+		break;
+	}
+
+	return ret;
+}
 
 static int dw_i2c_plat_probe(struct platform_device *pdev)
 {
@@ -240,9 +197,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	struct i2c_adapter *adap;
 	struct dw_i2c_dev *dev;
 	struct i2c_timings *t;
-	u32 acpi_speed;
-	struct resource *mem;
-	int i, irq, ret;
+	int irq, ret;
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
@@ -252,15 +207,15 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	if (!dev)
 		return -ENOMEM;
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	dev->base = devm_ioremap_resource(&pdev->dev, mem);
-	if (IS_ERR(dev->base))
-		return PTR_ERR(dev->base);
-
+	dev->flags = (uintptr_t)device_get_match_data(&pdev->dev);
 	dev->dev = &pdev->dev;
 	dev->irq = irq;
 	platform_set_drvdata(pdev, dev);
 
+	ret = dw_i2c_plat_request_regs(dev);
+	if (ret)
+		return ret;
+
 	dev->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(dev->rst))
 		return PTR_ERR(dev->rst);
@@ -273,60 +228,23 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	else
 		i2c_parse_fw_timings(&pdev->dev, t, false);
 
-	acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev);
-	/*
-	 * Some DSTDs use a non standard speed, round down to the lowest
-	 * standard speed.
-	 */
-	for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) {
-		if (acpi_speed >= supported_speeds[i])
-			break;
-	}
-	acpi_speed = i < ARRAY_SIZE(supported_speeds) ? supported_speeds[i] : 0;
-
-	/*
-	 * Find bus speed from the "clock-frequency" device property, ACPI
-	 * or by using fast mode if neither is set.
-	 */
-	if (acpi_speed && t->bus_freq_hz)
-		t->bus_freq_hz = min(t->bus_freq_hz, acpi_speed);
-	else if (acpi_speed || t->bus_freq_hz)
-		t->bus_freq_hz = max(t->bus_freq_hz, acpi_speed);
-	else
-		t->bus_freq_hz = I2C_MAX_FAST_MODE_FREQ;
-
-	dev->flags |= (uintptr_t)device_get_match_data(&pdev->dev);
+	i2c_dw_acpi_adjust_bus_speed(&pdev->dev);
 
 	if (pdev->dev.of_node)
 		dw_i2c_of_configure(pdev);
 
 	if (has_acpi_companion(&pdev->dev))
-		dw_i2c_acpi_configure(pdev);
+		i2c_dw_acpi_configure(&pdev->dev);
 
-	/*
-	 * Only standard mode at 100kHz, fast mode at 400kHz,
-	 * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported.
-	 */
-	for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) {
-		if (t->bus_freq_hz == supported_speeds[i])
-			break;
-	}
-	if (i == ARRAY_SIZE(supported_speeds)) {
-		dev_err(&pdev->dev,
-			"%d Hz is unsupported, only 100kHz, 400kHz, 1MHz and 3.4MHz are supported\n",
-			t->bus_freq_hz);
-		ret = -EINVAL;
+	ret = i2c_dw_validate_speed(dev);
+	if (ret)
 		goto exit_reset;
-	}
 
 	ret = i2c_dw_probe_lock_support(dev);
 	if (ret)
 		goto exit_reset;
 
-	if (i2c_detect_slave_mode(&pdev->dev))
-		i2c_dw_configure_slave(dev);
-	else
-		i2c_dw_configure_master(dev);
+	i2c_dw_configure(dev);
 
 	/* Optional interface clock */
 	dev->pclk = devm_clk_get_optional(&pdev->dev, "pclk");
@@ -377,11 +295,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 
 	pm_runtime_enable(&pdev->dev);
 
-	if (dev->mode == DW_IC_SLAVE)
-		ret = i2c_dw_probe_slave(dev);
-	else
-		ret = i2c_dw_probe(dev);
-
+	ret = i2c_dw_probe(dev);
 	if (ret)
 		goto exit_probe;
 
diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c
index f5ecf76c0d02..44974b53a626 100644
--- a/drivers/i2c/busses/i2c-designware-slave.c
+++ b/drivers/i2c/busses/i2c-designware-slave.c
@@ -14,18 +14,19 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/regmap.h>
 
 #include "i2c-designware-core.h"
 
 static void i2c_dw_configure_fifo_slave(struct dw_i2c_dev *dev)
 {
 	/* Configure Tx/Rx FIFO threshold levels. */
-	dw_writel(dev, 0, DW_IC_TX_TL);
-	dw_writel(dev, 0, DW_IC_RX_TL);
+	regmap_write(dev->map, DW_IC_TX_TL, 0);
+	regmap_write(dev->map, DW_IC_RX_TL, 0);
 
 	/* Configure the I2C slave. */
-	dw_writel(dev, dev->slave_cfg, DW_IC_CON);
-	dw_writel(dev, DW_IC_INTR_SLAVE_MASK, DW_IC_INTR_MASK);
+	regmap_write(dev->map, DW_IC_CON, dev->slave_cfg);
+	regmap_write(dev->map, DW_IC_INTR_MASK, DW_IC_INTR_SLAVE_MASK);
 }
 
 /**
@@ -49,7 +50,7 @@ static int i2c_dw_init_slave(struct dw_i2c_dev *dev)
 
 	/* Write SDA hold time if supported */
 	if (dev->sda_hold_time)
-		dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD);
+		regmap_write(dev->map, DW_IC_SDA_HOLD, dev->sda_hold_time);
 
 	i2c_dw_configure_fifo_slave(dev);
 	i2c_dw_release_lock(dev);
@@ -72,7 +73,7 @@ static int i2c_dw_reg_slave(struct i2c_client *slave)
 	 * the address to which the DW_apb_i2c responds.
 	 */
 	__i2c_dw_disable_nowait(dev);
-	dw_writel(dev, slave->addr, DW_IC_SAR);
+	regmap_write(dev->map, DW_IC_SAR, slave->addr);
 	dev->slave = slave;
 
 	__i2c_dw_enable(dev);
@@ -103,7 +104,7 @@ static int i2c_dw_unreg_slave(struct i2c_client *slave)
 
 static u32 i2c_dw_read_clear_intrbits_slave(struct dw_i2c_dev *dev)
 {
-	u32 stat;
+	u32 stat, dummy;
 
 	/*
 	 * The IC_INTR_STAT register just indicates "enabled" interrupts.
@@ -111,39 +112,39 @@ static u32 i2c_dw_read_clear_intrbits_slave(struct dw_i2c_dev *dev)
 	 * in the IC_RAW_INTR_STAT register.
 	 *
 	 * That is,
-	 *   stat = dw_readl(IC_INTR_STAT);
+	 *   stat = readl(IC_INTR_STAT);
 	 * equals to,
-	 *   stat = dw_readl(IC_RAW_INTR_STAT) & dw_readl(IC_INTR_MASK);
+	 *   stat = readl(IC_RAW_INTR_STAT) & readl(IC_INTR_MASK);
 	 *
 	 * The raw version might be useful for debugging purposes.
 	 */
-	stat = dw_readl(dev, DW_IC_INTR_STAT);
+	regmap_read(dev->map, DW_IC_INTR_STAT, &stat);
 
 	/*
 	 * Do not use the IC_CLR_INTR register to clear interrupts, or
 	 * you'll miss some interrupts, triggered during the period from
-	 * dw_readl(IC_INTR_STAT) to dw_readl(IC_CLR_INTR).
+	 * readl(IC_INTR_STAT) to readl(IC_CLR_INTR).
 	 *
 	 * Instead, use the separately-prepared IC_CLR_* registers.
 	 */
 	if (stat & DW_IC_INTR_TX_ABRT)
-		dw_readl(dev, DW_IC_CLR_TX_ABRT);
+		regmap_read(dev->map, DW_IC_CLR_TX_ABRT, &dummy);
 	if (stat & DW_IC_INTR_RX_UNDER)
-		dw_readl(dev, DW_IC_CLR_RX_UNDER);
+		regmap_read(dev->map, DW_IC_CLR_RX_UNDER, &dummy);
 	if (stat & DW_IC_INTR_RX_OVER)
-		dw_readl(dev, DW_IC_CLR_RX_OVER);
+		regmap_read(dev->map, DW_IC_CLR_RX_OVER, &dummy);
 	if (stat & DW_IC_INTR_TX_OVER)
-		dw_readl(dev, DW_IC_CLR_TX_OVER);
+		regmap_read(dev->map, DW_IC_CLR_TX_OVER, &dummy);
 	if (stat & DW_IC_INTR_RX_DONE)
-		dw_readl(dev, DW_IC_CLR_RX_DONE);
+		regmap_read(dev->map, DW_IC_CLR_RX_DONE, &dummy);
 	if (stat & DW_IC_INTR_ACTIVITY)
-		dw_readl(dev, DW_IC_CLR_ACTIVITY);
+		regmap_read(dev->map, DW_IC_CLR_ACTIVITY, &dummy);
 	if (stat & DW_IC_INTR_STOP_DET)
-		dw_readl(dev, DW_IC_CLR_STOP_DET);
+		regmap_read(dev->map, DW_IC_CLR_STOP_DET, &dummy);
 	if (stat & DW_IC_INTR_START_DET)
-		dw_readl(dev, DW_IC_CLR_START_DET);
+		regmap_read(dev->map, DW_IC_CLR_START_DET, &dummy);
 	if (stat & DW_IC_INTR_GEN_CALL)
-		dw_readl(dev, DW_IC_CLR_GEN_CALL);
+		regmap_read(dev->map, DW_IC_CLR_GEN_CALL, &dummy);
 
 	return stat;
 }
@@ -155,14 +156,14 @@ static u32 i2c_dw_read_clear_intrbits_slave(struct dw_i2c_dev *dev)
 
 static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev)
 {
-	u32 raw_stat, stat, enabled;
-	u8 val, slave_activity;
+	u32 raw_stat, stat, enabled, tmp;
+	u8 val = 0, slave_activity;
 
-	stat = dw_readl(dev, DW_IC_INTR_STAT);
-	enabled = dw_readl(dev, DW_IC_ENABLE);
-	raw_stat = dw_readl(dev, DW_IC_RAW_INTR_STAT);
-	slave_activity = ((dw_readl(dev, DW_IC_STATUS) &
-		DW_IC_STATUS_SLAVE_ACTIVITY) >> 6);
+	regmap_read(dev->map, DW_IC_INTR_STAT, &stat);
+	regmap_read(dev->map, DW_IC_ENABLE, &enabled);
+	regmap_read(dev->map, DW_IC_RAW_INTR_STAT, &raw_stat);
+	regmap_read(dev->map, DW_IC_STATUS, &tmp);
+	slave_activity = ((tmp & DW_IC_STATUS_SLAVE_ACTIVITY) >> 6);
 
 	if (!enabled || !(raw_stat & ~DW_IC_INTR_ACTIVITY) || !dev->slave)
 		return 0;
@@ -177,7 +178,8 @@ static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev)
 	if (stat & DW_IC_INTR_RD_REQ) {
 		if (slave_activity) {
 			if (stat & DW_IC_INTR_RX_FULL) {
-				val = dw_readl(dev, DW_IC_DATA_CMD);
+				regmap_read(dev->map, DW_IC_DATA_CMD, &tmp);
+				val = tmp;
 
 				if (!i2c_slave_event(dev->slave,
 						     I2C_SLAVE_WRITE_RECEIVED,
@@ -185,24 +187,24 @@ static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev)
 					dev_vdbg(dev->dev, "Byte %X acked!",
 						 val);
 				}
-				dw_readl(dev, DW_IC_CLR_RD_REQ);
+				regmap_read(dev->map, DW_IC_CLR_RD_REQ, &tmp);
 				stat = i2c_dw_read_clear_intrbits_slave(dev);
 			} else {
-				dw_readl(dev, DW_IC_CLR_RD_REQ);
-				dw_readl(dev, DW_IC_CLR_RX_UNDER);
+				regmap_read(dev->map, DW_IC_CLR_RD_REQ, &tmp);
+				regmap_read(dev->map, DW_IC_CLR_RX_UNDER, &tmp);
 				stat = i2c_dw_read_clear_intrbits_slave(dev);
 			}
 			if (!i2c_slave_event(dev->slave,
 					     I2C_SLAVE_READ_REQUESTED,
 					     &val))
-				dw_writel(dev, val, DW_IC_DATA_CMD);
+				regmap_write(dev->map, DW_IC_DATA_CMD, val);
 		}
 	}
 
 	if (stat & DW_IC_INTR_RX_DONE) {
 		if (!i2c_slave_event(dev->slave, I2C_SLAVE_READ_PROCESSED,
 				     &val))
-			dw_readl(dev, DW_IC_CLR_RX_DONE);
+			regmap_read(dev->map, DW_IC_CLR_RX_DONE, &tmp);
 
 		i2c_slave_event(dev->slave, I2C_SLAVE_STOP, &val);
 		stat = i2c_dw_read_clear_intrbits_slave(dev);
@@ -210,7 +212,8 @@ static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev)
 	}
 
 	if (stat & DW_IC_INTR_RX_FULL) {
-		val = dw_readl(dev, DW_IC_DATA_CMD);
+		regmap_read(dev->map, DW_IC_DATA_CMD, &tmp);
+		val = tmp;
 		if (!i2c_slave_event(dev->slave, I2C_SLAVE_WRITE_RECEIVED,
 				     &val))
 			dev_vdbg(dev->dev, "Byte %X acked!", val);
@@ -241,6 +244,17 @@ static const struct i2c_algorithm i2c_dw_algo = {
 	.unreg_slave = i2c_dw_unreg_slave,
 };
 
+void i2c_dw_configure_slave(struct dw_i2c_dev *dev)
+{
+	dev->functionality = I2C_FUNC_SLAVE | DW_IC_DEFAULT_FUNCTIONALITY;
+
+	dev->slave_cfg = DW_IC_CON_RX_FIFO_FULL_HLD_CTRL |
+			 DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED;
+
+	dev->mode = DW_IC_SLAVE;
+}
+EXPORT_SYMBOL_GPL(i2c_dw_configure_slave);
+
 int i2c_dw_probe_slave(struct dw_i2c_dev *dev)
 {
 	struct i2c_adapter *adap = &dev->adapter;
@@ -252,7 +266,7 @@ int i2c_dw_probe_slave(struct dw_i2c_dev *dev)
 	dev->disable = i2c_dw_disable;
 	dev->disable_int = i2c_dw_disable_int;
 
-	ret = i2c_dw_set_reg_access(dev);
+	ret = i2c_dw_init_regmap(dev);
 	if (ret)
 		return ret;
 
@@ -260,7 +274,9 @@ int i2c_dw_probe_slave(struct dw_i2c_dev *dev)
 	if (ret)
 		return ret;
 
-	i2c_dw_set_fifo_size(dev);
+	ret = i2c_dw_set_fifo_size(dev);
+	if (ret)
+		return ret;
 
 	ret = dev->init(dev);
 	if (ret)
diff --git a/drivers/i2c/busses/i2c-digicolor.c b/drivers/i2c/busses/i2c-digicolor.c
index 056a5c4f0833..332f00437479 100644
--- a/drivers/i2c/busses/i2c-digicolor.c
+++ b/drivers/i2c/busses/i2c-digicolor.c
@@ -290,7 +290,6 @@ static int dc_i2c_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct dc_i2c *i2c;
-	struct resource *r;
 	int ret = 0, irq;
 
 	i2c = devm_kzalloc(&pdev->dev, sizeof(struct dc_i2c), GFP_KERNEL);
@@ -311,8 +310,7 @@ static int dc_i2c_probe(struct platform_device *pdev)
 	if (IS_ERR(i2c->clk))
 		return PTR_ERR(i2c->clk);
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->regs = devm_ioremap_resource(&pdev->dev, r);
+	i2c->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->regs))
 		return PTR_ERR(i2c->regs);
 
diff --git a/drivers/i2c/busses/i2c-efm32.c b/drivers/i2c/busses/i2c-efm32.c
index 18cca8f56da8..838ce0947191 100644
--- a/drivers/i2c/busses/i2c-efm32.c
+++ b/drivers/i2c/busses/i2c-efm32.c
@@ -312,9 +312,6 @@ static int efm32_i2c_probe(struct platform_device *pdev)
 	int ret;
 	u32 clkdiv;
 
-	if (!np)
-		return -EINVAL;
-
 	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
 	if (!ddata)
 		return -ENOMEM;
@@ -352,7 +349,6 @@ static int efm32_i2c_probe(struct platform_device *pdev)
 
 	ret = platform_get_irq(pdev, 0);
 	if (ret <= 0) {
-		dev_err(&pdev->dev, "failed to get irq (%d)\n", ret);
 		if (!ret)
 			ret = -EINVAL;
 		return ret;
diff --git a/drivers/i2c/busses/i2c-emev2.c b/drivers/i2c/busses/i2c-emev2.c
index 959d4912ec0d..1a319352e51b 100644
--- a/drivers/i2c/busses/i2c-emev2.c
+++ b/drivers/i2c/busses/i2c-emev2.c
@@ -361,15 +361,13 @@ static const struct i2c_algorithm em_i2c_algo = {
 static int em_i2c_probe(struct platform_device *pdev)
 {
 	struct em_i2c_device *priv;
-	struct resource *r;
 	int ret;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->base = devm_ioremap_resource(&pdev->dev, r);
+	priv->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->base))
 		return PTR_ERR(priv->base);
 
diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c
index 527030953ba1..6ce3ec03b595 100644
--- a/drivers/i2c/busses/i2c-exynos5.c
+++ b/drivers/i2c/busses/i2c-exynos5.c
@@ -736,7 +736,6 @@ static int exynos5_i2c_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct exynos5_i2c *i2c;
-	struct resource *mem;
 	int ret;
 
 	i2c = devm_kzalloc(&pdev->dev, sizeof(struct exynos5_i2c), GFP_KERNEL);
@@ -762,8 +761,7 @@ static int exynos5_i2c_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->regs = devm_ioremap_resource(&pdev->dev, mem);
+	i2c->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->regs)) {
 		ret = PTR_ERR(i2c->regs);
 		goto err_clk;
@@ -879,6 +877,6 @@ static struct platform_driver exynos5_i2c_driver = {
 module_platform_driver(exynos5_i2c_driver);
 
 MODULE_DESCRIPTION("Exynos5 HS-I2C Bus driver");
-MODULE_AUTHOR("Naveen Krishna Chatradhi, <ch.naveen@samsung.com>");
-MODULE_AUTHOR("Taekgyun Ko, <taeggyun.ko@samsung.com>");
+MODULE_AUTHOR("Naveen Krishna Chatradhi <ch.naveen@samsung.com>");
+MODULE_AUTHOR("Taekgyun Ko <taeggyun.ko@samsung.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c
index 6610304b6dc6..ab15b1ec2ab3 100644
--- a/drivers/i2c/busses/i2c-hix5hd2.c
+++ b/drivers/i2c/busses/i2c-hix5hd2.c
@@ -388,7 +388,6 @@ static int hix5hd2_i2c_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct hix5hd2_i2c_priv *priv;
-	struct resource *mem;
 	unsigned int freq;
 	int irq, ret;
 
@@ -409,8 +408,7 @@ static int hix5hd2_i2c_probe(struct platform_device *pdev)
 		}
 	}
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->regs = devm_ioremap_resource(&pdev->dev, mem);
+	priv->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->regs))
 		return PTR_ERR(priv->regs);
 
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index a9c03f5c3482..fea644921a76 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -1318,6 +1318,12 @@ static void i801_probe_optional_slaves(struct i801_priv *priv)
 
 	if (is_dell_system_with_lis3lv02d())
 		register_dell_lis3lv02d_i2c_device(priv);
+
+	/* Instantiate SPD EEPROMs unless the SMBus is multiplexed */
+#if IS_ENABLED(CONFIG_I2C_MUX_GPIO)
+	if (!priv->mux_drvdata)
+#endif
+		i2c_register_spd(&priv->adapter);
 }
 #else
 static void __init input_apanel_init(void) {}
@@ -1439,9 +1445,9 @@ static int i801_add_mux(struct i801_priv *priv)
 		return -ENOMEM;
 	lookup->dev_id = "i2c-mux-gpio";
 	for (i = 0; i < mux_config->n_gpios; i++) {
-		lookup->table[i].chip_label = mux_config->gpio_chip;
-		lookup->table[i].chip_hwnum = mux_config->gpios[i];
-		lookup->table[i].con_id = "mux";
+		lookup->table[i] = (struct gpiod_lookup)
+			GPIO_LOOKUP(mux_config->gpio_chip,
+				    mux_config->gpios[i], "mux", 0);
 	}
 	gpiod_add_lookup_table(lookup);
 	priv->lookup = lookup;
diff --git a/drivers/i2c/busses/i2c-icy.c b/drivers/i2c/busses/i2c-icy.c
index 271470f4d8a9..66c9923fc766 100644
--- a/drivers/i2c/busses/i2c-icy.c
+++ b/drivers/i2c/busses/i2c-icy.c
@@ -43,6 +43,7 @@
 #include <linux/i2c.h>
 #include <linux/i2c-algo-pcf.h>
 
+#include <asm/amigahw.h>
 #include <asm/amigaints.h>
 #include <linux/zorro.h>
 
diff --git a/drivers/i2c/busses/i2c-img-scb.c b/drivers/i2c/busses/i2c-img-scb.c
index 422097a31c95..98a89301ed2a 100644
--- a/drivers/i2c/busses/i2c-img-scb.c
+++ b/drivers/i2c/busses/i2c-img-scb.c
@@ -1330,7 +1330,6 @@ static int img_i2c_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
 	struct img_i2c *i2c;
-	struct resource *res;
 	int irq, ret;
 	u32 val;
 
@@ -1338,16 +1337,13 @@ static int img_i2c_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->base = devm_ioremap_resource(&pdev->dev, res);
+	i2c->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->base))
 		return PTR_ERR(i2c->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "can't get irq number\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	i2c->sys_clk = devm_clk_get(&pdev->dev, "sys");
 	if (IS_ERR(i2c->sys_clk)) {
diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c
index 94743ba581fe..9db6ccded5e9 100644
--- a/drivers/i2c/busses/i2c-imx-lpi2c.c
+++ b/drivers/i2c/busses/i2c-imx-lpi2c.c
@@ -551,10 +551,8 @@ static int lpi2c_imx_probe(struct platform_device *pdev)
 		return PTR_ERR(lpi2c_imx->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "can't get irq number\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	lpi2c_imx->adapter.owner	= THIS_MODULE;
 	lpi2c_imx->adapter.algo		= &lpi2c_imx_algo;
diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c
index b426fc956938..ba831df6661e 100644
--- a/drivers/i2c/busses/i2c-jz4780.c
+++ b/drivers/i2c/busses/i2c-jz4780.c
@@ -763,7 +763,6 @@ static int jz4780_i2c_probe(struct platform_device *pdev)
 	int ret = 0;
 	unsigned int clk_freq = 0;
 	unsigned short tmp;
-	struct resource *r;
 	struct jz4780_i2c *i2c;
 
 	i2c = devm_kzalloc(&pdev->dev, sizeof(struct jz4780_i2c), GFP_KERNEL);
@@ -787,8 +786,7 @@ static int jz4780_i2c_probe(struct platform_device *pdev)
 	init_completion(&i2c->trans_waitq);
 	spin_lock_init(&i2c->lock);
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->iomem = devm_ioremap_resource(&pdev->dev, r);
+	i2c->iomem = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->iomem))
 		return PTR_ERR(i2c->iomem);
 
diff --git a/drivers/i2c/busses/i2c-lpc2k.c b/drivers/i2c/busses/i2c-lpc2k.c
index 13b0c12e2dba..4e30c5267142 100644
--- a/drivers/i2c/busses/i2c-lpc2k.c
+++ b/drivers/i2c/busses/i2c-lpc2k.c
@@ -346,7 +346,6 @@ static const struct i2c_algorithm i2c_lpc2k_algorithm = {
 static int i2c_lpc2k_probe(struct platform_device *pdev)
 {
 	struct lpc2k_i2c *i2c;
-	struct resource *res;
 	u32 bus_clk_rate;
 	u32 scl_high;
 	u32 clkrate;
@@ -356,16 +355,13 @@ static int i2c_lpc2k_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->base = devm_ioremap_resource(&pdev->dev, res);
+	i2c->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->base))
 		return PTR_ERR(i2c->base);
 
 	i2c->irq = platform_get_irq(pdev, 0);
-	if (i2c->irq < 0) {
-		dev_err(&pdev->dev, "can't get interrupt resource\n");
+	if (i2c->irq < 0)
 		return i2c->irq;
-	}
 
 	init_waitqueue_head(&i2c->wait);
 
diff --git a/drivers/i2c/busses/i2c-meson.c b/drivers/i2c/busses/i2c-meson.c
index 06b3bed78421..c5dec572fc48 100644
--- a/drivers/i2c/busses/i2c-meson.c
+++ b/drivers/i2c/busses/i2c-meson.c
@@ -397,7 +397,6 @@ static int meson_i2c_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct meson_i2c *i2c;
-	struct resource *mem;
 	struct i2c_timings timings;
 	int irq, ret = 0;
 
@@ -422,16 +421,13 @@ static int meson_i2c_probe(struct platform_device *pdev)
 		return PTR_ERR(i2c->clk);
 	}
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->regs = devm_ioremap_resource(&pdev->dev, mem);
+	i2c->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->regs))
 		return PTR_ERR(i2c->regs);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "can't find IRQ\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	ret = devm_request_irq(&pdev->dev, irq, meson_i2c_irq, 0, NULL, i2c);
 	if (ret < 0) {
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index 0ca6c38a15eb..deef69e56906 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -40,12 +40,11 @@
 #define I2C_SOFT_RST			0x0001
 #define I2C_FIFO_ADDR_CLR		0x0001
 #define I2C_DELAY_LEN			0x0002
-#define I2C_ST_START_CON		0x8001
-#define I2C_FS_START_CON		0x1800
 #define I2C_TIME_CLR_VALUE		0x0000
 #define I2C_TIME_DEFAULT_VALUE		0x0003
 #define I2C_WRRD_TRANAC_VALUE		0x0002
 #define I2C_RD_TRANAC_VALUE		0x0001
+#define I2C_SCL_MIS_COMP_VALUE		0x0000
 
 #define I2C_DMA_CON_TX			0x0000
 #define I2C_DMA_CON_RX			0x0001
@@ -55,10 +54,13 @@
 #define I2C_DMA_HARD_RST		0x0002
 #define I2C_DMA_4G_MODE			0x0001
 
-#define I2C_DEFAULT_CLK_DIV		5
 #define MAX_SAMPLE_CNT_DIV		8
 #define MAX_STEP_CNT_DIV		64
+#define MAX_CLOCK_DIV			256
 #define MAX_HS_STEP_CNT_DIV		8
+#define I2C_STANDARD_MODE_BUFFER	(1000 / 2)
+#define I2C_FAST_MODE_BUFFER		(300 / 2)
+#define I2C_FAST_MODE_PLUS_BUFFER	(20 / 2)
 
 #define I2C_CONTROL_RS                  (0x1 << 1)
 #define I2C_CONTROL_DMA_EN              (0x1 << 2)
@@ -123,6 +125,12 @@ enum I2C_REGS_OFFSET {
 	OFFSET_TRANSFER_LEN_AUX,
 	OFFSET_CLOCK_DIV,
 	OFFSET_LTIMING,
+	OFFSET_SCL_HIGH_LOW_RATIO,
+	OFFSET_HS_SCL_HIGH_LOW_RATIO,
+	OFFSET_SCL_MIS_COMP_POINT,
+	OFFSET_STA_STO_AC_TIMING,
+	OFFSET_HS_STA_STO_AC_TIMING,
+	OFFSET_SDA_TIMING,
 };
 
 static const u16 mt_i2c_regs_v1[] = {
@@ -150,6 +158,12 @@ static const u16 mt_i2c_regs_v1[] = {
 	[OFFSET_DEBUGCTRL] = 0x68,
 	[OFFSET_TRANSFER_LEN_AUX] = 0x6c,
 	[OFFSET_CLOCK_DIV] = 0x70,
+	[OFFSET_SCL_HIGH_LOW_RATIO] = 0x74,
+	[OFFSET_HS_SCL_HIGH_LOW_RATIO] = 0x78,
+	[OFFSET_SCL_MIS_COMP_POINT] = 0x7C,
+	[OFFSET_STA_STO_AC_TIMING] = 0x80,
+	[OFFSET_HS_STA_STO_AC_TIMING] = 0x84,
+	[OFFSET_SDA_TIMING] = 0x88,
 };
 
 static const u16 mt_i2c_regs_v2[] = {
@@ -168,9 +182,11 @@ static const u16 mt_i2c_regs_v2[] = {
 	[OFFSET_HS] = 0x30,
 	[OFFSET_IO_CONFIG] = 0x34,
 	[OFFSET_FIFO_ADDR_CLR] = 0x38,
+	[OFFSET_SDA_TIMING] = 0x3c,
 	[OFFSET_TRANSFER_LEN_AUX] = 0x44,
 	[OFFSET_CLOCK_DIV] = 0x48,
 	[OFFSET_SOFTRESET] = 0x50,
+	[OFFSET_SCL_MIS_COMP_POINT] = 0x90,
 	[OFFSET_DEBUGSTAT] = 0xe0,
 	[OFFSET_DEBUGCTRL] = 0xe8,
 	[OFFSET_FIFO_STAT] = 0xf4,
@@ -191,6 +207,19 @@ struct mtk_i2c_compatible {
 	unsigned char ltiming_adjust: 1;
 };
 
+struct mtk_i2c_ac_timing {
+	u16 htiming;
+	u16 ltiming;
+	u16 hs;
+	u16 ext;
+	u16 inter_clk_div;
+	u16 scl_hl_ratio;
+	u16 hs_scl_hl_ratio;
+	u16 sta_stop;
+	u16 hs_sta_stop;
+	u16 sda_timing;
+};
+
 struct mtk_i2c {
 	struct i2c_adapter adap;	/* i2c host adapter */
 	struct device *dev;
@@ -215,9 +244,46 @@ struct mtk_i2c {
 	u16 ltiming_reg;
 	unsigned char auto_restart;
 	bool ignore_restart_irq;
+	struct mtk_i2c_ac_timing ac_timing;
 	const struct mtk_i2c_compatible *dev_comp;
 };
 
+/**
+ * struct i2c_spec_values:
+ * min_low_ns: min LOW period of the SCL clock
+ * min_su_sta_ns: min set-up time for a repeated START condition
+ * max_hd_dat_ns: max data hold time
+ * min_su_dat_ns: min data set-up time
+ */
+struct i2c_spec_values {
+	unsigned int min_low_ns;
+	unsigned int min_high_ns;
+	unsigned int min_su_sta_ns;
+	unsigned int max_hd_dat_ns;
+	unsigned int min_su_dat_ns;
+};
+
+static const struct i2c_spec_values standard_mode_spec = {
+	.min_low_ns = 4700 + I2C_STANDARD_MODE_BUFFER,
+	.min_su_sta_ns = 4700 + I2C_STANDARD_MODE_BUFFER,
+	.max_hd_dat_ns = 3450 - I2C_STANDARD_MODE_BUFFER,
+	.min_su_dat_ns = 250 + I2C_STANDARD_MODE_BUFFER,
+};
+
+static const struct i2c_spec_values fast_mode_spec = {
+	.min_low_ns = 1300 + I2C_FAST_MODE_BUFFER,
+	.min_su_sta_ns = 600 + I2C_FAST_MODE_BUFFER,
+	.max_hd_dat_ns = 900 - I2C_FAST_MODE_BUFFER,
+	.min_su_dat_ns = 100 + I2C_FAST_MODE_BUFFER,
+};
+
+static const struct i2c_spec_values fast_mode_plus_spec = {
+	.min_low_ns = 500 + I2C_FAST_MODE_PLUS_BUFFER,
+	.min_su_sta_ns = 260 + I2C_FAST_MODE_PLUS_BUFFER,
+	.max_hd_dat_ns = 400 - I2C_FAST_MODE_PLUS_BUFFER,
+	.min_su_dat_ns = 50 + I2C_FAST_MODE_PLUS_BUFFER,
+};
+
 static const struct i2c_adapter_quirks mt6577_i2c_quirks = {
 	.flags = I2C_AQ_COMB_WRITE_THEN_READ,
 	.max_num_msgs = 1,
@@ -397,14 +463,38 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c)
 	if (i2c->dev_comp->dcm)
 		mtk_i2c_writew(i2c, I2C_DCM_DISABLE, OFFSET_DCM_EN);
 
-	if (i2c->dev_comp->timing_adjust)
-		mtk_i2c_writew(i2c, I2C_DEFAULT_CLK_DIV - 1, OFFSET_CLOCK_DIV);
-
 	mtk_i2c_writew(i2c, i2c->timing_reg, OFFSET_TIMING);
 	mtk_i2c_writew(i2c, i2c->high_speed_reg, OFFSET_HS);
 	if (i2c->dev_comp->ltiming_adjust)
 		mtk_i2c_writew(i2c, i2c->ltiming_reg, OFFSET_LTIMING);
 
+	if (i2c->dev_comp->timing_adjust) {
+		mtk_i2c_writew(i2c, i2c->ac_timing.ext, OFFSET_EXT_CONF);
+		mtk_i2c_writew(i2c, i2c->ac_timing.inter_clk_div,
+			       OFFSET_CLOCK_DIV);
+		mtk_i2c_writew(i2c, I2C_SCL_MIS_COMP_VALUE,
+			       OFFSET_SCL_MIS_COMP_POINT);
+		mtk_i2c_writew(i2c, i2c->ac_timing.sda_timing,
+			       OFFSET_SDA_TIMING);
+
+		if (i2c->dev_comp->ltiming_adjust) {
+			mtk_i2c_writew(i2c, i2c->ac_timing.htiming,
+				       OFFSET_TIMING);
+			mtk_i2c_writew(i2c, i2c->ac_timing.hs, OFFSET_HS);
+			mtk_i2c_writew(i2c, i2c->ac_timing.ltiming,
+				       OFFSET_LTIMING);
+		} else {
+			mtk_i2c_writew(i2c, i2c->ac_timing.scl_hl_ratio,
+				       OFFSET_SCL_HIGH_LOW_RATIO);
+			mtk_i2c_writew(i2c, i2c->ac_timing.hs_scl_hl_ratio,
+				       OFFSET_HS_SCL_HIGH_LOW_RATIO);
+			mtk_i2c_writew(i2c, i2c->ac_timing.sta_stop,
+				       OFFSET_STA_STO_AC_TIMING);
+			mtk_i2c_writew(i2c, i2c->ac_timing.hs_sta_stop,
+				       OFFSET_HS_STA_STO_AC_TIMING);
+		}
+	}
+
 	/* If use i2c pin from PMIC mt6397 side, need set PATH_DIR first */
 	if (i2c->have_pmic)
 		mtk_i2c_writew(i2c, I2C_CONTROL_WRAPPER, OFFSET_PATH_DIR);
@@ -422,6 +512,126 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c)
 	writel(I2C_DMA_CLR_FLAG, i2c->pdmabase + OFFSET_RST);
 }
 
+static const struct i2c_spec_values *mtk_i2c_get_spec(unsigned int speed)
+{
+	if (speed <= I2C_MAX_STANDARD_MODE_FREQ)
+		return &standard_mode_spec;
+	else if (speed <= I2C_MAX_FAST_MODE_FREQ)
+		return &fast_mode_spec;
+	else
+		return &fast_mode_plus_spec;
+}
+
+static int mtk_i2c_max_step_cnt(unsigned int target_speed)
+{
+	if (target_speed > I2C_MAX_FAST_MODE_FREQ)
+		return MAX_HS_STEP_CNT_DIV;
+	else
+		return MAX_STEP_CNT_DIV;
+}
+
+/*
+ * Check and Calculate i2c ac-timing
+ *
+ * Hardware design:
+ * sample_ns = (1000000000 * (sample_cnt + 1)) / clk_src
+ * xxx_cnt_div =  spec->min_xxx_ns / sample_ns
+ *
+ * Sample_ns is rounded down for xxx_cnt_div would be greater
+ * than the smallest spec.
+ * The sda_timing is chosen as the middle value between
+ * the largest and smallest.
+ */
+static int mtk_i2c_check_ac_timing(struct mtk_i2c *i2c,
+				   unsigned int clk_src,
+				   unsigned int check_speed,
+				   unsigned int step_cnt,
+				   unsigned int sample_cnt)
+{
+	const struct i2c_spec_values *spec;
+	unsigned int su_sta_cnt, low_cnt, high_cnt, max_step_cnt;
+	unsigned int sda_max, sda_min, clk_ns, max_sta_cnt = 0x3f;
+	unsigned int sample_ns = div_u64(1000000000ULL * (sample_cnt + 1),
+					 clk_src);
+
+	if (!i2c->dev_comp->timing_adjust)
+		return 0;
+
+	if (i2c->dev_comp->ltiming_adjust)
+		max_sta_cnt = 0x100;
+
+	spec = mtk_i2c_get_spec(check_speed);
+
+	if (i2c->dev_comp->ltiming_adjust)
+		clk_ns = 1000000000 / clk_src;
+	else
+		clk_ns = sample_ns / 2;
+
+	su_sta_cnt = DIV_ROUND_UP(spec->min_su_sta_ns, clk_ns);
+	if (su_sta_cnt > max_sta_cnt)
+		return -1;
+
+	low_cnt = DIV_ROUND_UP(spec->min_low_ns, sample_ns);
+	max_step_cnt = mtk_i2c_max_step_cnt(check_speed);
+	if ((2 * step_cnt) > low_cnt && low_cnt < max_step_cnt) {
+		if (low_cnt > step_cnt) {
+			high_cnt = 2 * step_cnt - low_cnt;
+		} else {
+			high_cnt = step_cnt;
+			low_cnt = step_cnt;
+		}
+	} else {
+		return -2;
+	}
+
+	sda_max = spec->max_hd_dat_ns / sample_ns;
+	if (sda_max > low_cnt)
+		sda_max = 0;
+
+	sda_min = DIV_ROUND_UP(spec->min_su_dat_ns, sample_ns);
+	if (sda_min < low_cnt)
+		sda_min = 0;
+
+	if (sda_min > sda_max)
+		return -3;
+
+	if (check_speed > I2C_MAX_FAST_MODE_FREQ) {
+		if (i2c->dev_comp->ltiming_adjust) {
+			i2c->ac_timing.hs = I2C_TIME_DEFAULT_VALUE |
+				(sample_cnt << 12) | (high_cnt << 8);
+			i2c->ac_timing.ltiming &= ~GENMASK(15, 9);
+			i2c->ac_timing.ltiming |= (sample_cnt << 12) |
+				(low_cnt << 9);
+			i2c->ac_timing.ext &= ~GENMASK(7, 1);
+			i2c->ac_timing.ext |= (su_sta_cnt << 1) | (1 << 0);
+		} else {
+			i2c->ac_timing.hs_scl_hl_ratio = (1 << 12) |
+				(high_cnt << 6) | low_cnt;
+			i2c->ac_timing.hs_sta_stop = (su_sta_cnt << 8) |
+				su_sta_cnt;
+		}
+		i2c->ac_timing.sda_timing &= ~GENMASK(11, 6);
+		i2c->ac_timing.sda_timing |= (1 << 12) |
+			((sda_max + sda_min) / 2) << 6;
+	} else {
+		if (i2c->dev_comp->ltiming_adjust) {
+			i2c->ac_timing.htiming = (sample_cnt << 8) | (high_cnt);
+			i2c->ac_timing.ltiming = (sample_cnt << 6) | (low_cnt);
+			i2c->ac_timing.ext = (su_sta_cnt << 8) | (1 << 0);
+		} else {
+			i2c->ac_timing.scl_hl_ratio = (1 << 12) |
+				(high_cnt << 6) | low_cnt;
+			i2c->ac_timing.sta_stop = (su_sta_cnt << 8) |
+				su_sta_cnt;
+		}
+
+		i2c->ac_timing.sda_timing = (1 << 12) |
+			(sda_max + sda_min) / 2;
+	}
+
+	return 0;
+}
+
 /*
  * Calculate i2c port speed
  *
@@ -446,15 +656,12 @@ static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src,
 	unsigned int opt_div;
 	unsigned int best_mul;
 	unsigned int cnt_mul;
+	int ret = -EINVAL;
 
 	if (target_speed > I2C_MAX_FAST_MODE_PLUS_FREQ)
 		target_speed = I2C_MAX_FAST_MODE_PLUS_FREQ;
 
-	if (target_speed > I2C_MAX_FAST_MODE_FREQ)
-		max_step_cnt = MAX_HS_STEP_CNT_DIV;
-	else
-		max_step_cnt = MAX_STEP_CNT_DIV;
-
+	max_step_cnt = mtk_i2c_max_step_cnt(target_speed);
 	base_step_cnt = max_step_cnt;
 	/* Find the best combination */
 	opt_div = DIV_ROUND_UP(clk_src >> 1, target_speed);
@@ -473,6 +680,11 @@ static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src,
 			continue;
 
 		if (cnt_mul < best_mul) {
+			ret = mtk_i2c_check_ac_timing(i2c, clk_src,
+				target_speed, step_cnt - 1, sample_cnt - 1);
+			if (ret)
+				continue;
+
 			best_mul = cnt_mul;
 			base_sample_cnt = sample_cnt;
 			base_step_cnt = step_cnt;
@@ -481,6 +693,9 @@ static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src,
 		}
 	}
 
+	if (ret)
+		return -EINVAL;
+
 	sample_cnt = base_sample_cnt;
 	step_cnt = base_step_cnt;
 
@@ -506,47 +721,68 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk)
 	unsigned int l_step_cnt;
 	unsigned int l_sample_cnt;
 	unsigned int target_speed;
+	unsigned int clk_div;
+	unsigned int max_clk_div;
 	int ret;
 
-	clk_src = parent_clk / i2c->clk_src_div;
 	target_speed = i2c->speed_hz;
+	parent_clk /= i2c->clk_src_div;
 
-	if (target_speed > I2C_MAX_FAST_MODE_FREQ) {
-		/* Set master code speed register */
-		ret = mtk_i2c_calculate_speed(i2c, clk_src, I2C_MAX_FAST_MODE_FREQ,
-					      &l_step_cnt, &l_sample_cnt);
-		if (ret < 0)
-			return ret;
-
-		i2c->timing_reg = (l_sample_cnt << 8) | l_step_cnt;
-
-		/* Set the high speed mode register */
-		ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed,
-					      &step_cnt, &sample_cnt);
-		if (ret < 0)
-			return ret;
-
-		i2c->high_speed_reg = I2C_TIME_DEFAULT_VALUE |
-			(sample_cnt << 12) | (step_cnt << 8);
+	if (i2c->dev_comp->timing_adjust)
+		max_clk_div = MAX_CLOCK_DIV;
+	else
+		max_clk_div = 1;
+
+	for (clk_div = 1; clk_div <= max_clk_div; clk_div++) {
+		clk_src = parent_clk / clk_div;
+
+		if (target_speed > I2C_MAX_FAST_MODE_FREQ) {
+			/* Set master code speed register */
+			ret = mtk_i2c_calculate_speed(i2c, clk_src,
+						      I2C_MAX_FAST_MODE_FREQ,
+						      &l_step_cnt,
+						      &l_sample_cnt);
+			if (ret < 0)
+				continue;
+
+			i2c->timing_reg = (l_sample_cnt << 8) | l_step_cnt;
+
+			/* Set the high speed mode register */
+			ret = mtk_i2c_calculate_speed(i2c, clk_src,
+						      target_speed, &step_cnt,
+						      &sample_cnt);
+			if (ret < 0)
+				continue;
+
+			i2c->high_speed_reg = I2C_TIME_DEFAULT_VALUE |
+					(sample_cnt << 12) | (step_cnt << 8);
+
+			if (i2c->dev_comp->ltiming_adjust)
+				i2c->ltiming_reg =
+					(l_sample_cnt << 6) | l_step_cnt |
+					(sample_cnt << 12) | (step_cnt << 9);
+		} else {
+			ret = mtk_i2c_calculate_speed(i2c, clk_src,
+						      target_speed, &l_step_cnt,
+						      &l_sample_cnt);
+			if (ret < 0)
+				continue;
 
-		if (i2c->dev_comp->ltiming_adjust)
-			i2c->ltiming_reg = (l_sample_cnt << 6) | l_step_cnt |
-					   (sample_cnt << 12) | (step_cnt << 9);
-	} else {
-		ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed,
-					      &step_cnt, &sample_cnt);
-		if (ret < 0)
-			return ret;
+			i2c->timing_reg = (l_sample_cnt << 8) | l_step_cnt;
 
-		i2c->timing_reg = (sample_cnt << 8) | step_cnt;
+			/* Disable the high speed transaction */
+			i2c->high_speed_reg = I2C_TIME_CLR_VALUE;
 
-		/* Disable the high speed transaction */
-		i2c->high_speed_reg = I2C_TIME_CLR_VALUE;
+			if (i2c->dev_comp->ltiming_adjust)
+				i2c->ltiming_reg =
+					(l_sample_cnt << 6) | l_step_cnt;
+		}
 
-		if (i2c->dev_comp->ltiming_adjust)
-			i2c->ltiming_reg = (sample_cnt << 6) | step_cnt;
+		break;
 	}
 
+	i2c->ac_timing.inter_clk_div = clk_div - 1;
+
 	return 0;
 }
 
@@ -586,12 +822,6 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs,
 
 	mtk_i2c_writew(i2c, control_reg, OFFSET_CONTROL);
 
-	/* set start condition */
-	if (i2c->speed_hz <= I2C_MAX_STANDARD_MODE_FREQ)
-		mtk_i2c_writew(i2c, I2C_ST_START_CON, OFFSET_EXT_CONF);
-	else
-		mtk_i2c_writew(i2c, I2C_FS_START_CON, OFFSET_EXT_CONF);
-
 	addr_reg = i2c_8bit_addr_from_msg(msgs);
 	mtk_i2c_writew(i2c, addr_reg, OFFSET_SLAVE_ADDR);
 
@@ -948,9 +1178,6 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 	if (ret)
 		return -EINVAL;
 
-	if (i2c->dev_comp->timing_adjust)
-		i2c->clk_src_div *= I2C_DEFAULT_CLK_DIV;
-
 	if (i2c->have_pmic && !i2c->dev_comp->pmic_i2c)
 		return -EINVAL;
 
diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c
index 9b8f1d8552ea..829b8c98ae51 100644
--- a/drivers/i2c/busses/i2c-mv64xxx.c
+++ b/drivers/i2c/busses/i2c-mv64xxx.c
@@ -877,7 +877,6 @@ mv64xxx_i2c_probe(struct platform_device *pd)
 {
 	struct mv64xxx_i2c_data		*drv_data;
 	struct mv64xxx_i2c_pdata	*pdata = dev_get_platdata(&pd->dev);
-	struct resource	*r;
 	int	rc;
 
 	if ((!pdata && !pd->dev.of_node))
@@ -888,8 +887,7 @@ mv64xxx_i2c_probe(struct platform_device *pd)
 	if (!drv_data)
 		return -ENOMEM;
 
-	r = platform_get_resource(pd, IORESOURCE_MEM, 0);
-	drv_data->reg_base = devm_ioremap_resource(&pd->dev, r);
+	drv_data->reg_base = devm_platform_ioremap_resource(pd, 0);
 	if (IS_ERR(drv_data->reg_base))
 		return PTR_ERR(drv_data->reg_base);
 
diff --git a/drivers/i2c/busses/i2c-npcm7xx.c b/drivers/i2c/busses/i2c-npcm7xx.c
new file mode 100644
index 000000000000..75f07138a6fa
--- /dev/null
+++ b/drivers/i2c/busses/i2c-npcm7xx.c
@@ -0,0 +1,2342 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Nuvoton NPCM7xx I2C Controller driver
+ *
+ * Copyright (C) 2020 Nuvoton Technologies tali.perry@nuvoton.com
+ */
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/debugfs.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+enum i2c_mode {
+	I2C_MASTER,
+	I2C_SLAVE,
+};
+
+/*
+ * External I2C Interface driver xfer indication values, which indicate status
+ * of the bus.
+ */
+enum i2c_state_ind {
+	I2C_NO_STATUS_IND = 0,
+	I2C_SLAVE_RCV_IND,
+	I2C_SLAVE_XMIT_IND,
+	I2C_SLAVE_XMIT_MISSING_DATA_IND,
+	I2C_SLAVE_RESTART_IND,
+	I2C_SLAVE_DONE_IND,
+	I2C_MASTER_DONE_IND,
+	I2C_NACK_IND,
+	I2C_BUS_ERR_IND,
+	I2C_WAKE_UP_IND,
+	I2C_BLOCK_BYTES_ERR_IND,
+	I2C_SLAVE_RCV_MISSING_DATA_IND,
+};
+
+/*
+ * Operation type values (used to define the operation currently running)
+ * module is interrupt driven, on each interrupt the current operation is
+ * checked to see if the module is currently reading or writing.
+ */
+enum i2c_oper {
+	I2C_NO_OPER = 0,
+	I2C_WRITE_OPER,
+	I2C_READ_OPER,
+};
+
+/* I2C Bank (module had 2 banks of registers) */
+enum i2c_bank {
+	I2C_BANK_0 = 0,
+	I2C_BANK_1,
+};
+
+/* Internal I2C states values (for the I2C module state machine). */
+enum i2c_state {
+	I2C_DISABLE = 0,
+	I2C_IDLE,
+	I2C_MASTER_START,
+	I2C_SLAVE_MATCH,
+	I2C_OPER_STARTED,
+	I2C_STOP_PENDING,
+};
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+/* Module supports setting multiple own slave addresses */
+enum i2c_addr {
+	I2C_SLAVE_ADDR1 = 0,
+	I2C_SLAVE_ADDR2,
+	I2C_SLAVE_ADDR3,
+	I2C_SLAVE_ADDR4,
+	I2C_SLAVE_ADDR5,
+	I2C_SLAVE_ADDR6,
+	I2C_SLAVE_ADDR7,
+	I2C_SLAVE_ADDR8,
+	I2C_SLAVE_ADDR9,
+	I2C_SLAVE_ADDR10,
+	I2C_GC_ADDR,
+	I2C_ARP_ADDR,
+};
+#endif
+
+/* init register and default value required to enable module */
+#define NPCM_I2CSEGCTL			0xE4
+#define NPCM_I2CSEGCTL_INIT_VAL		0x0333F000
+
+/* Common regs */
+#define NPCM_I2CSDA			0x00
+#define NPCM_I2CST			0x02
+#define NPCM_I2CCST			0x04
+#define NPCM_I2CCTL1			0x06
+#define NPCM_I2CADDR1			0x08
+#define NPCM_I2CCTL2			0x0A
+#define NPCM_I2CADDR2			0x0C
+#define NPCM_I2CCTL3			0x0E
+#define NPCM_I2CCST2			0x18
+#define NPCM_I2CCST3			0x19
+#define I2C_VER				0x1F
+
+/*BANK0 regs*/
+#define NPCM_I2CADDR3			0x10
+#define NPCM_I2CADDR7			0x11
+#define NPCM_I2CADDR4			0x12
+#define NPCM_I2CADDR8			0x13
+#define NPCM_I2CADDR5			0x14
+#define NPCM_I2CADDR9			0x15
+#define NPCM_I2CADDR6			0x16
+#define NPCM_I2CADDR10			0x17
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+/*
+ * npcm_i2caddr array:
+ * The module supports having multiple own slave addresses.
+ * Since the addr regs are sprinkled all over the address space,
+ * use this array to get the address or each register.
+ */
+#define I2C_NUM_OWN_ADDR 10
+static const int npcm_i2caddr[I2C_NUM_OWN_ADDR] = {
+	NPCM_I2CADDR1, NPCM_I2CADDR2, NPCM_I2CADDR3, NPCM_I2CADDR4,
+	NPCM_I2CADDR5, NPCM_I2CADDR6, NPCM_I2CADDR7, NPCM_I2CADDR8,
+	NPCM_I2CADDR9, NPCM_I2CADDR10,
+};
+#endif
+
+#define NPCM_I2CCTL4			0x1A
+#define NPCM_I2CCTL5			0x1B
+#define NPCM_I2CSCLLT			0x1C /* SCL Low Time */
+#define NPCM_I2CFIF_CTL			0x1D /* FIFO Control */
+#define NPCM_I2CSCLHT			0x1E /* SCL High Time */
+
+/* BANK 1 regs */
+#define NPCM_I2CFIF_CTS			0x10 /* Both FIFOs Control and Status */
+#define NPCM_I2CTXF_CTL			0x12 /* Tx-FIFO Control */
+#define NPCM_I2CT_OUT			0x14 /* Bus T.O. */
+#define NPCM_I2CPEC			0x16 /* PEC Data */
+#define NPCM_I2CTXF_STS			0x1A /* Tx-FIFO Status */
+#define NPCM_I2CRXF_STS			0x1C /* Rx-FIFO Status */
+#define NPCM_I2CRXF_CTL			0x1E /* Rx-FIFO Control */
+
+/* NPCM_I2CST reg fields */
+#define NPCM_I2CST_XMIT			BIT(0)
+#define NPCM_I2CST_MASTER		BIT(1)
+#define NPCM_I2CST_NMATCH		BIT(2)
+#define NPCM_I2CST_STASTR		BIT(3)
+#define NPCM_I2CST_NEGACK		BIT(4)
+#define NPCM_I2CST_BER			BIT(5)
+#define NPCM_I2CST_SDAST		BIT(6)
+#define NPCM_I2CST_SLVSTP		BIT(7)
+
+/* NPCM_I2CCST reg fields */
+#define NPCM_I2CCST_BUSY		BIT(0)
+#define NPCM_I2CCST_BB			BIT(1)
+#define NPCM_I2CCST_MATCH		BIT(2)
+#define NPCM_I2CCST_GCMATCH		BIT(3)
+#define NPCM_I2CCST_TSDA		BIT(4)
+#define NPCM_I2CCST_TGSCL		BIT(5)
+#define NPCM_I2CCST_MATCHAF		BIT(6)
+#define NPCM_I2CCST_ARPMATCH		BIT(7)
+
+/* NPCM_I2CCTL1 reg fields */
+#define NPCM_I2CCTL1_START		BIT(0)
+#define NPCM_I2CCTL1_STOP		BIT(1)
+#define NPCM_I2CCTL1_INTEN		BIT(2)
+#define NPCM_I2CCTL1_EOBINTE		BIT(3)
+#define NPCM_I2CCTL1_ACK		BIT(4)
+#define NPCM_I2CCTL1_GCMEN		BIT(5)
+#define NPCM_I2CCTL1_NMINTE		BIT(6)
+#define NPCM_I2CCTL1_STASTRE		BIT(7)
+
+/* RW1S fields (inside a RW reg): */
+#define NPCM_I2CCTL1_RWS   \
+	(NPCM_I2CCTL1_START | NPCM_I2CCTL1_STOP | NPCM_I2CCTL1_ACK)
+
+/* npcm_i2caddr reg fields */
+#define NPCM_I2CADDR_A			GENMASK(6, 0)
+#define NPCM_I2CADDR_SAEN		BIT(7)
+
+/* NPCM_I2CCTL2 reg fields */
+#define I2CCTL2_ENABLE			BIT(0)
+#define I2CCTL2_SCLFRQ6_0		GENMASK(7, 1)
+
+/* NPCM_I2CCTL3 reg fields */
+#define I2CCTL3_SCLFRQ8_7		GENMASK(1, 0)
+#define I2CCTL3_ARPMEN			BIT(2)
+#define I2CCTL3_IDL_START		BIT(3)
+#define I2CCTL3_400K_MODE		BIT(4)
+#define I2CCTL3_BNK_SEL			BIT(5)
+#define I2CCTL3_SDA_LVL			BIT(6)
+#define I2CCTL3_SCL_LVL			BIT(7)
+
+/* NPCM_I2CCST2 reg fields */
+#define NPCM_I2CCST2_MATCHA1F		BIT(0)
+#define NPCM_I2CCST2_MATCHA2F		BIT(1)
+#define NPCM_I2CCST2_MATCHA3F		BIT(2)
+#define NPCM_I2CCST2_MATCHA4F		BIT(3)
+#define NPCM_I2CCST2_MATCHA5F		BIT(4)
+#define NPCM_I2CCST2_MATCHA6F		BIT(5)
+#define NPCM_I2CCST2_MATCHA7F		BIT(5)
+#define NPCM_I2CCST2_INTSTS		BIT(7)
+
+/* NPCM_I2CCST3 reg fields */
+#define NPCM_I2CCST3_MATCHA8F		BIT(0)
+#define NPCM_I2CCST3_MATCHA9F		BIT(1)
+#define NPCM_I2CCST3_MATCHA10F		BIT(2)
+#define NPCM_I2CCST3_EO_BUSY		BIT(7)
+
+/* NPCM_I2CCTL4 reg fields */
+#define I2CCTL4_HLDT			GENMASK(5, 0)
+#define I2CCTL4_LVL_WE			BIT(7)
+
+/* NPCM_I2CCTL5 reg fields */
+#define I2CCTL5_DBNCT			GENMASK(3, 0)
+
+/* NPCM_I2CFIF_CTS reg fields */
+#define NPCM_I2CFIF_CTS_RXF_TXE		BIT(1)
+#define NPCM_I2CFIF_CTS_RFTE_IE		BIT(3)
+#define NPCM_I2CFIF_CTS_CLR_FIFO	BIT(6)
+#define NPCM_I2CFIF_CTS_SLVRSTR		BIT(7)
+
+/* NPCM_I2CTXF_CTL reg fields */
+#define NPCM_I2CTXF_CTL_TX_THR		GENMASK(4, 0)
+#define NPCM_I2CTXF_CTL_THR_TXIE	BIT(6)
+
+/* NPCM_I2CT_OUT reg fields */
+#define NPCM_I2CT_OUT_TO_CKDIV		GENMASK(5, 0)
+#define NPCM_I2CT_OUT_T_OUTIE		BIT(6)
+#define NPCM_I2CT_OUT_T_OUTST		BIT(7)
+
+/* NPCM_I2CTXF_STS reg fields */
+#define NPCM_I2CTXF_STS_TX_BYTES	GENMASK(4, 0)
+#define NPCM_I2CTXF_STS_TX_THST		BIT(6)
+
+/* NPCM_I2CRXF_STS reg fields */
+#define NPCM_I2CRXF_STS_RX_BYTES	GENMASK(4, 0)
+#define NPCM_I2CRXF_STS_RX_THST		BIT(6)
+
+/* NPCM_I2CFIF_CTL reg fields */
+#define NPCM_I2CFIF_CTL_FIFO_EN		BIT(4)
+
+/* NPCM_I2CRXF_CTL reg fields */
+#define NPCM_I2CRXF_CTL_RX_THR		GENMASK(4, 0)
+#define NPCM_I2CRXF_CTL_LAST_PEC	BIT(5)
+#define NPCM_I2CRXF_CTL_THR_RXIE	BIT(6)
+
+#define I2C_HW_FIFO_SIZE		16
+
+/* I2C_VER reg fields */
+#define I2C_VER_VERSION			GENMASK(6, 0)
+#define I2C_VER_FIFO_EN			BIT(7)
+
+/* stall/stuck timeout in us */
+#define DEFAULT_STALL_COUNT		25
+
+/* SCLFRQ field position */
+#define SCLFRQ_0_TO_6			GENMASK(6, 0)
+#define SCLFRQ_7_TO_8			GENMASK(8, 7)
+
+/* supported clk settings. values in Hz. */
+#define I2C_FREQ_MIN_HZ			10000
+#define I2C_FREQ_MAX_HZ			I2C_MAX_FAST_MODE_PLUS_FREQ
+
+/* Status of one I2C module */
+struct npcm_i2c {
+	struct i2c_adapter adap;
+	struct device *dev;
+	unsigned char __iomem *reg;
+	spinlock_t lock;   /* IRQ synchronization */
+	struct completion cmd_complete;
+	int cmd_err;
+	struct i2c_msg *msgs;
+	int msgs_num;
+	int num;
+	u32 apb_clk;
+	struct i2c_bus_recovery_info rinfo;
+	enum i2c_state state;
+	enum i2c_oper operation;
+	enum i2c_mode master_or_slave;
+	enum i2c_state_ind stop_ind;
+	u8 dest_addr;
+	u8 *rd_buf;
+	u16 rd_size;
+	u16 rd_ind;
+	u8 *wr_buf;
+	u16 wr_size;
+	u16 wr_ind;
+	bool fifo_use;
+	u16 PEC_mask; /* PEC bit mask per slave address */
+	bool PEC_use;
+	bool read_block_use;
+	unsigned long int_time_stamp;
+	unsigned long bus_freq; /* in Hz */
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u8 own_slave_addr;
+	struct i2c_client *slave;
+	int slv_rd_size;
+	int slv_rd_ind;
+	int slv_wr_size;
+	int slv_wr_ind;
+	u8 slv_rd_buf[I2C_HW_FIFO_SIZE];
+	u8 slv_wr_buf[I2C_HW_FIFO_SIZE];
+#endif
+	struct dentry *debugfs; /* debugfs device directory */
+	u64 ber_cnt;
+	u64 rec_succ_cnt;
+	u64 rec_fail_cnt;
+	u64 nack_cnt;
+	u64 timeout_cnt;
+};
+
+static inline void npcm_i2c_select_bank(struct npcm_i2c *bus,
+					enum i2c_bank bank)
+{
+	u8 i2cctl3 = ioread8(bus->reg + NPCM_I2CCTL3);
+
+	if (bank == I2C_BANK_0)
+		i2cctl3 = i2cctl3 & ~I2CCTL3_BNK_SEL;
+	else
+		i2cctl3 = i2cctl3 | I2CCTL3_BNK_SEL;
+	iowrite8(i2cctl3, bus->reg + NPCM_I2CCTL3);
+}
+
+static void npcm_i2c_init_params(struct npcm_i2c *bus)
+{
+	bus->stop_ind = I2C_NO_STATUS_IND;
+	bus->rd_size = 0;
+	bus->wr_size = 0;
+	bus->rd_ind = 0;
+	bus->wr_ind = 0;
+	bus->read_block_use = false;
+	bus->int_time_stamp = 0;
+	bus->PEC_use = false;
+	bus->PEC_mask = 0;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (bus->slave)
+		bus->master_or_slave = I2C_SLAVE;
+#endif
+}
+
+static inline void npcm_i2c_wr_byte(struct npcm_i2c *bus, u8 data)
+{
+	iowrite8(data, bus->reg + NPCM_I2CSDA);
+}
+
+static inline u8 npcm_i2c_rd_byte(struct npcm_i2c *bus)
+{
+	return ioread8(bus->reg + NPCM_I2CSDA);
+}
+
+static int npcm_i2c_get_SCL(struct i2c_adapter *_adap)
+{
+	struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap);
+
+	return !!(I2CCTL3_SCL_LVL & ioread32(bus->reg + NPCM_I2CCTL3));
+}
+
+static int npcm_i2c_get_SDA(struct i2c_adapter *_adap)
+{
+	struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap);
+
+	return !!(I2CCTL3_SDA_LVL & ioread32(bus->reg + NPCM_I2CCTL3));
+}
+
+static inline u16 npcm_i2c_get_index(struct npcm_i2c *bus)
+{
+	if (bus->operation == I2C_READ_OPER)
+		return bus->rd_ind;
+	if (bus->operation == I2C_WRITE_OPER)
+		return bus->wr_ind;
+	return 0;
+}
+
+/* quick protocol (just address) */
+static inline bool npcm_i2c_is_quick(struct npcm_i2c *bus)
+{
+	return bus->wr_size == 0 && bus->rd_size == 0;
+}
+
+static void npcm_i2c_disable(struct npcm_i2c *bus)
+{
+	u8 i2cctl2;
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	int i;
+
+	/* select bank 0 for I2C addresses */
+	npcm_i2c_select_bank(bus, I2C_BANK_0);
+
+	/* Slave addresses removal */
+	for (i = I2C_SLAVE_ADDR1; i < I2C_NUM_OWN_ADDR; i++)
+		iowrite8(0, bus->reg + npcm_i2caddr[i]);
+
+	npcm_i2c_select_bank(bus, I2C_BANK_1);
+#endif
+	/* Disable module */
+	i2cctl2 = ioread8(bus->reg + NPCM_I2CCTL2);
+	i2cctl2 = i2cctl2 & ~I2CCTL2_ENABLE;
+	iowrite8(i2cctl2, bus->reg + NPCM_I2CCTL2);
+
+	bus->state = I2C_DISABLE;
+}
+
+static void npcm_i2c_enable(struct npcm_i2c *bus)
+{
+	u8 i2cctl2 = ioread8(bus->reg + NPCM_I2CCTL2);
+
+	i2cctl2 = i2cctl2 | I2CCTL2_ENABLE;
+	iowrite8(i2cctl2, bus->reg + NPCM_I2CCTL2);
+	bus->state = I2C_IDLE;
+}
+
+/* enable\disable end of busy (EOB) interrupts */
+static inline void npcm_i2c_eob_int(struct npcm_i2c *bus, bool enable)
+{
+	u8 val;
+
+	/* Clear EO_BUSY pending bit: */
+	val = ioread8(bus->reg + NPCM_I2CCST3);
+	val = val | NPCM_I2CCST3_EO_BUSY;
+	iowrite8(val, bus->reg + NPCM_I2CCST3);
+
+	val = ioread8(bus->reg + NPCM_I2CCTL1);
+	val &= ~NPCM_I2CCTL1_RWS;
+	if (enable)
+		val |= NPCM_I2CCTL1_EOBINTE;
+	else
+		val &= ~NPCM_I2CCTL1_EOBINTE;
+	iowrite8(val, bus->reg + NPCM_I2CCTL1);
+}
+
+static inline bool npcm_i2c_tx_fifo_empty(struct npcm_i2c *bus)
+{
+	u8 tx_fifo_sts;
+
+	tx_fifo_sts = ioread8(bus->reg + NPCM_I2CTXF_STS);
+	/* check if TX FIFO is not empty */
+	if ((tx_fifo_sts & NPCM_I2CTXF_STS_TX_BYTES) == 0)
+		return false;
+
+	/* check if TX FIFO status bit is set: */
+	return !!FIELD_GET(NPCM_I2CTXF_STS_TX_THST, tx_fifo_sts);
+}
+
+static inline bool npcm_i2c_rx_fifo_full(struct npcm_i2c *bus)
+{
+	u8 rx_fifo_sts;
+
+	rx_fifo_sts = ioread8(bus->reg + NPCM_I2CRXF_STS);
+	/* check if RX FIFO is not empty: */
+	if ((rx_fifo_sts & NPCM_I2CRXF_STS_RX_BYTES) == 0)
+		return false;
+
+	/* check if rx fifo full status is set: */
+	return !!FIELD_GET(NPCM_I2CRXF_STS_RX_THST, rx_fifo_sts);
+}
+
+static inline void npcm_i2c_clear_fifo_int(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	val = ioread8(bus->reg + NPCM_I2CFIF_CTS);
+	val = (val & NPCM_I2CFIF_CTS_SLVRSTR) | NPCM_I2CFIF_CTS_RXF_TXE;
+	iowrite8(val, bus->reg + NPCM_I2CFIF_CTS);
+}
+
+static inline void npcm_i2c_clear_tx_fifo(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	val = ioread8(bus->reg + NPCM_I2CTXF_STS);
+	val = val | NPCM_I2CTXF_STS_TX_THST;
+	iowrite8(val, bus->reg + NPCM_I2CTXF_STS);
+}
+
+static inline void npcm_i2c_clear_rx_fifo(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	val = ioread8(bus->reg + NPCM_I2CRXF_STS);
+	val = val | NPCM_I2CRXF_STS_RX_THST;
+	iowrite8(val, bus->reg + NPCM_I2CRXF_STS);
+}
+
+static void npcm_i2c_int_enable(struct npcm_i2c *bus, bool enable)
+{
+	u8 val;
+
+	val = ioread8(bus->reg + NPCM_I2CCTL1);
+	val &= ~NPCM_I2CCTL1_RWS;
+	if (enable)
+		val |= NPCM_I2CCTL1_INTEN;
+	else
+		val &= ~NPCM_I2CCTL1_INTEN;
+	iowrite8(val, bus->reg + NPCM_I2CCTL1);
+}
+
+static inline void npcm_i2c_master_start(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	val = ioread8(bus->reg + NPCM_I2CCTL1);
+	val &= ~(NPCM_I2CCTL1_STOP | NPCM_I2CCTL1_ACK);
+	val |= NPCM_I2CCTL1_START;
+	iowrite8(val, bus->reg + NPCM_I2CCTL1);
+}
+
+static inline void npcm_i2c_master_stop(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	/*
+	 * override HW issue: I2C may fail to supply stop condition in Master
+	 * Write operation.
+	 * Need to delay at least 5 us from the last int, before issueing a stop
+	 */
+	udelay(10); /* function called from interrupt, can't sleep */
+	val = ioread8(bus->reg + NPCM_I2CCTL1);
+	val &= ~(NPCM_I2CCTL1_START | NPCM_I2CCTL1_ACK);
+	val |= NPCM_I2CCTL1_STOP;
+	iowrite8(val, bus->reg + NPCM_I2CCTL1);
+
+	if (!bus->fifo_use)
+		return;
+
+	npcm_i2c_select_bank(bus, I2C_BANK_1);
+
+	if (bus->operation == I2C_READ_OPER)
+		npcm_i2c_clear_rx_fifo(bus);
+	else
+		npcm_i2c_clear_tx_fifo(bus);
+	npcm_i2c_clear_fifo_int(bus);
+	iowrite8(0, bus->reg + NPCM_I2CTXF_CTL);
+}
+
+static inline void npcm_i2c_stall_after_start(struct npcm_i2c *bus, bool stall)
+{
+	u8 val;
+
+	val = ioread8(bus->reg + NPCM_I2CCTL1);
+	val &= ~NPCM_I2CCTL1_RWS;
+	if (stall)
+		val |= NPCM_I2CCTL1_STASTRE;
+	else
+		val &= ~NPCM_I2CCTL1_STASTRE;
+	iowrite8(val, bus->reg + NPCM_I2CCTL1);
+}
+
+static inline void npcm_i2c_nack(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	val = ioread8(bus->reg + NPCM_I2CCTL1);
+	val &= ~(NPCM_I2CCTL1_STOP | NPCM_I2CCTL1_START);
+	val |= NPCM_I2CCTL1_ACK;
+	iowrite8(val, bus->reg + NPCM_I2CCTL1);
+}
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+static void npcm_i2c_slave_int_enable(struct npcm_i2c *bus, bool enable)
+{
+	u8 i2cctl1;
+
+	/* enable interrupt on slave match: */
+	i2cctl1 = ioread8(bus->reg + NPCM_I2CCTL1);
+	i2cctl1 &= ~NPCM_I2CCTL1_RWS;
+	if (enable)
+		i2cctl1 |= NPCM_I2CCTL1_NMINTE;
+	else
+		i2cctl1 &= ~NPCM_I2CCTL1_NMINTE;
+	iowrite8(i2cctl1, bus->reg + NPCM_I2CCTL1);
+}
+
+static int npcm_i2c_slave_enable(struct npcm_i2c *bus, enum i2c_addr addr_type,
+				 u8 addr, bool enable)
+{
+	u8 i2cctl1;
+	u8 i2cctl3;
+	u8 sa_reg;
+
+	sa_reg = (addr & 0x7F) | FIELD_PREP(NPCM_I2CADDR_SAEN, enable);
+	if (addr_type == I2C_GC_ADDR) {
+		i2cctl1 = ioread8(bus->reg + NPCM_I2CCTL1);
+		if (enable)
+			i2cctl1 |= NPCM_I2CCTL1_GCMEN;
+		else
+			i2cctl1 &= ~NPCM_I2CCTL1_GCMEN;
+		iowrite8(i2cctl1, bus->reg + NPCM_I2CCTL1);
+		return 0;
+	}
+	if (addr_type == I2C_ARP_ADDR) {
+		i2cctl3 = ioread8(bus->reg + NPCM_I2CCTL3);
+		if (enable)
+			i2cctl3 |= I2CCTL3_ARPMEN;
+		else
+			i2cctl3 &= ~I2CCTL3_ARPMEN;
+		iowrite8(i2cctl3, bus->reg + NPCM_I2CCTL3);
+		return 0;
+	}
+	if (addr_type >= I2C_ARP_ADDR)
+		return -EFAULT;
+	/* select bank 0 for address 3 to 10 */
+	if (addr_type > I2C_SLAVE_ADDR2)
+		npcm_i2c_select_bank(bus, I2C_BANK_0);
+	/* Set and enable the address */
+	iowrite8(sa_reg, bus->reg + npcm_i2caddr[addr_type]);
+	npcm_i2c_slave_int_enable(bus, enable);
+	if (addr_type > I2C_SLAVE_ADDR2)
+		npcm_i2c_select_bank(bus, I2C_BANK_1);
+	return 0;
+}
+#endif
+
+static void npcm_i2c_reset(struct npcm_i2c *bus)
+{
+	/*
+	 * Save I2CCTL1 relevant bits. It is being cleared when the module
+	 *  is disabled.
+	 */
+	u8 i2cctl1;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	u8 addr;
+#endif
+
+	i2cctl1 = ioread8(bus->reg + NPCM_I2CCTL1);
+
+	npcm_i2c_disable(bus);
+	npcm_i2c_enable(bus);
+
+	/* Restore NPCM_I2CCTL1 Status */
+	i2cctl1 &= ~NPCM_I2CCTL1_RWS;
+	iowrite8(i2cctl1, bus->reg + NPCM_I2CCTL1);
+
+	/* Clear BB (BUS BUSY) bit */
+	iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST);
+	iowrite8(0xFF, bus->reg + NPCM_I2CST);
+
+	/* Clear EOB bit */
+	iowrite8(NPCM_I2CCST3_EO_BUSY, bus->reg + NPCM_I2CCST3);
+
+	/* Clear all fifo bits: */
+	iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, bus->reg + NPCM_I2CFIF_CTS);
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (bus->slave) {
+		addr = bus->slave->addr;
+		npcm_i2c_slave_enable(bus, I2C_SLAVE_ADDR1, addr, true);
+	}
+#endif
+
+	bus->state = I2C_IDLE;
+}
+
+static inline bool npcm_i2c_is_master(struct npcm_i2c *bus)
+{
+	return !!FIELD_GET(NPCM_I2CST_MASTER, ioread8(bus->reg + NPCM_I2CST));
+}
+
+static void npcm_i2c_callback(struct npcm_i2c *bus,
+			      enum i2c_state_ind op_status, u16 info)
+{
+	struct i2c_msg *msgs;
+	int msgs_num;
+
+	msgs = bus->msgs;
+	msgs_num = bus->msgs_num;
+	/*
+	 * check that transaction was not timed-out, and msgs still
+	 * holds a valid value.
+	 */
+	if (!msgs)
+		return;
+
+	if (completion_done(&bus->cmd_complete))
+		return;
+
+	switch (op_status) {
+	case I2C_MASTER_DONE_IND:
+		bus->cmd_err = bus->msgs_num;
+		fallthrough;
+	case I2C_BLOCK_BYTES_ERR_IND:
+		/* Master tx finished and all transmit bytes were sent */
+		if (bus->msgs) {
+			if (msgs[0].flags & I2C_M_RD)
+				msgs[0].len = info;
+			else if (msgs_num == 2 &&
+				 msgs[1].flags & I2C_M_RD)
+				msgs[1].len = info;
+		}
+		if (completion_done(&bus->cmd_complete) == false)
+			complete(&bus->cmd_complete);
+	break;
+
+	case I2C_NACK_IND:
+		/* MASTER transmit got a NACK before tx all bytes */
+		bus->cmd_err = -ENXIO;
+		if (bus->master_or_slave == I2C_MASTER)
+			complete(&bus->cmd_complete);
+
+		break;
+	case I2C_BUS_ERR_IND:
+		/* Bus error */
+		bus->cmd_err = -EAGAIN;
+		if (bus->master_or_slave == I2C_MASTER)
+			complete(&bus->cmd_complete);
+
+		break;
+	case I2C_WAKE_UP_IND:
+		/* I2C wake up */
+		break;
+	default:
+		break;
+	}
+
+	bus->operation = I2C_NO_OPER;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (bus->slave)
+		bus->master_or_slave = I2C_SLAVE;
+#endif
+}
+
+static u8 npcm_i2c_fifo_usage(struct npcm_i2c *bus)
+{
+	if (bus->operation == I2C_WRITE_OPER)
+		return FIELD_GET(NPCM_I2CTXF_STS_TX_BYTES,
+				 ioread8(bus->reg + NPCM_I2CTXF_STS));
+	if (bus->operation == I2C_READ_OPER)
+		return FIELD_GET(NPCM_I2CRXF_STS_RX_BYTES,
+				 ioread8(bus->reg + NPCM_I2CRXF_STS));
+	return 0;
+}
+
+static void npcm_i2c_write_to_fifo_master(struct npcm_i2c *bus, u16 max_bytes)
+{
+	u8 size_free_fifo;
+
+	/*
+	 * Fill the FIFO, while the FIFO is not full and there are more bytes
+	 * to write
+	 */
+	size_free_fifo = I2C_HW_FIFO_SIZE - npcm_i2c_fifo_usage(bus);
+	while (max_bytes-- && size_free_fifo) {
+		if (bus->wr_ind < bus->wr_size)
+			npcm_i2c_wr_byte(bus, bus->wr_buf[bus->wr_ind++]);
+		else
+			npcm_i2c_wr_byte(bus, 0xFF);
+		size_free_fifo = I2C_HW_FIFO_SIZE - npcm_i2c_fifo_usage(bus);
+	}
+}
+
+/*
+ * npcm_i2c_set_fifo:
+ * configure the FIFO before using it. If nread is -1 RX FIFO will not be
+ * configured. same for nwrite
+ */
+static void npcm_i2c_set_fifo(struct npcm_i2c *bus, int nread, int nwrite)
+{
+	u8 rxf_ctl = 0;
+
+	if (!bus->fifo_use)
+		return;
+	npcm_i2c_select_bank(bus, I2C_BANK_1);
+	npcm_i2c_clear_tx_fifo(bus);
+	npcm_i2c_clear_rx_fifo(bus);
+
+	/* configure RX FIFO */
+	if (nread > 0) {
+		rxf_ctl = min_t(int, nread, I2C_HW_FIFO_SIZE);
+
+		/* set LAST bit. if LAST is set next FIFO packet is nacked */
+		if (nread <= I2C_HW_FIFO_SIZE)
+			rxf_ctl |= NPCM_I2CRXF_CTL_LAST_PEC;
+
+		/*
+		 * if we are about to read the first byte in blk rd mode,
+		 * don't NACK it. If slave returns zero size HW can't NACK
+		 * it immidiattly, it will read extra byte and then NACK.
+		 */
+		if (bus->rd_ind == 0 && bus->read_block_use) {
+			/* set fifo to read one byte, no last: */
+			rxf_ctl = 1;
+		}
+
+		/* set fifo size: */
+		iowrite8(rxf_ctl, bus->reg + NPCM_I2CRXF_CTL);
+	}
+
+	/* configure TX FIFO */
+	if (nwrite > 0) {
+		if (nwrite > I2C_HW_FIFO_SIZE)
+			/* data to send is more then FIFO size. */
+			iowrite8(I2C_HW_FIFO_SIZE, bus->reg + NPCM_I2CTXF_CTL);
+		else
+			iowrite8(nwrite, bus->reg + NPCM_I2CTXF_CTL);
+
+		npcm_i2c_clear_tx_fifo(bus);
+	}
+}
+
+static void npcm_i2c_read_fifo(struct npcm_i2c *bus, u8 bytes_in_fifo)
+{
+	u8 data;
+
+	while (bytes_in_fifo--) {
+		data = npcm_i2c_rd_byte(bus);
+		if (bus->rd_ind < bus->rd_size)
+			bus->rd_buf[bus->rd_ind++] = data;
+	}
+}
+
+static inline void npcm_i2c_clear_master_status(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	/* Clear NEGACK, STASTR and BER bits */
+	val = NPCM_I2CST_BER | NPCM_I2CST_NEGACK | NPCM_I2CST_STASTR;
+	iowrite8(val, bus->reg + NPCM_I2CST);
+}
+
+static void npcm_i2c_master_abort(struct npcm_i2c *bus)
+{
+	/* Only current master is allowed to issue a stop condition */
+	if (!npcm_i2c_is_master(bus))
+		return;
+
+	npcm_i2c_eob_int(bus, true);
+	npcm_i2c_master_stop(bus);
+	npcm_i2c_clear_master_status(bus);
+}
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+static u8 npcm_i2c_get_slave_addr(struct npcm_i2c *bus, enum i2c_addr addr_type)
+{
+	u8 slave_add;
+
+	/* select bank 0 for address 3 to 10 */
+	if (addr_type > I2C_SLAVE_ADDR2)
+		npcm_i2c_select_bank(bus, I2C_BANK_0);
+
+	slave_add = ioread8(bus->reg + npcm_i2caddr[(int)addr_type]);
+
+	if (addr_type > I2C_SLAVE_ADDR2)
+		npcm_i2c_select_bank(bus, I2C_BANK_1);
+
+	return slave_add;
+}
+
+static int npcm_i2c_remove_slave_addr(struct npcm_i2c *bus, u8 slave_add)
+{
+	int i;
+
+	/* Set the enable bit */
+	slave_add |= 0x80;
+	npcm_i2c_select_bank(bus, I2C_BANK_0);
+	for (i = I2C_SLAVE_ADDR1; i < I2C_NUM_OWN_ADDR; i++) {
+		if (ioread8(bus->reg + npcm_i2caddr[i]) == slave_add)
+			iowrite8(0, bus->reg + npcm_i2caddr[i]);
+	}
+	npcm_i2c_select_bank(bus, I2C_BANK_1);
+	return 0;
+}
+
+static void npcm_i2c_write_fifo_slave(struct npcm_i2c *bus, u16 max_bytes)
+{
+	/*
+	 * Fill the FIFO, while the FIFO is not full and there are more bytes
+	 * to write
+	 */
+	npcm_i2c_clear_fifo_int(bus);
+	npcm_i2c_clear_tx_fifo(bus);
+	iowrite8(0, bus->reg + NPCM_I2CTXF_CTL);
+	while (max_bytes-- && I2C_HW_FIFO_SIZE != npcm_i2c_fifo_usage(bus)) {
+		if (bus->slv_wr_size <= 0)
+			break;
+		bus->slv_wr_ind = bus->slv_wr_ind % I2C_HW_FIFO_SIZE;
+		npcm_i2c_wr_byte(bus, bus->slv_wr_buf[bus->slv_wr_ind]);
+		bus->slv_wr_ind++;
+		bus->slv_wr_ind = bus->slv_wr_ind % I2C_HW_FIFO_SIZE;
+		bus->slv_wr_size--;
+	}
+}
+
+static void npcm_i2c_read_fifo_slave(struct npcm_i2c *bus, u8 bytes_in_fifo)
+{
+	u8 data;
+
+	if (!bus->slave)
+		return;
+
+	while (bytes_in_fifo--) {
+		data = npcm_i2c_rd_byte(bus);
+
+		bus->slv_rd_ind = bus->slv_rd_ind % I2C_HW_FIFO_SIZE;
+		bus->slv_rd_buf[bus->slv_rd_ind] = data;
+		bus->slv_rd_ind++;
+
+		/* 1st byte is length in block protocol: */
+		if (bus->slv_rd_ind == 1 && bus->read_block_use)
+			bus->slv_rd_size = data + bus->PEC_use + 1;
+	}
+}
+
+static int npcm_i2c_slave_get_wr_buf(struct npcm_i2c *bus)
+{
+	int i;
+	u8 value;
+	int ind;
+	int ret = bus->slv_wr_ind;
+
+	/* fill a cyclic buffer */
+	for (i = 0; i < I2C_HW_FIFO_SIZE; i++) {
+		if (bus->slv_wr_size >= I2C_HW_FIFO_SIZE)
+			break;
+		i2c_slave_event(bus->slave, I2C_SLAVE_READ_REQUESTED, &value);
+		ind = (bus->slv_wr_ind + bus->slv_wr_size) % I2C_HW_FIFO_SIZE;
+		bus->slv_wr_buf[ind] = value;
+		bus->slv_wr_size++;
+		i2c_slave_event(bus->slave, I2C_SLAVE_READ_PROCESSED, &value);
+	}
+	return I2C_HW_FIFO_SIZE - ret;
+}
+
+static void npcm_i2c_slave_send_rd_buf(struct npcm_i2c *bus)
+{
+	int i;
+
+	for (i = 0; i < bus->slv_rd_ind; i++)
+		i2c_slave_event(bus->slave, I2C_SLAVE_WRITE_RECEIVED,
+				&bus->slv_rd_buf[i]);
+	/*
+	 * once we send bytes up, need to reset the counter of the wr buf
+	 * got data from master (new offset in device), ignore wr fifo:
+	 */
+	if (bus->slv_rd_ind) {
+		bus->slv_wr_size = 0;
+		bus->slv_wr_ind = 0;
+	}
+
+	bus->slv_rd_ind = 0;
+	bus->slv_rd_size = bus->adap.quirks->max_read_len;
+
+	npcm_i2c_clear_fifo_int(bus);
+	npcm_i2c_clear_rx_fifo(bus);
+}
+
+static void npcm_i2c_slave_receive(struct npcm_i2c *bus, u16 nread,
+				   u8 *read_data)
+{
+	bus->state = I2C_OPER_STARTED;
+	bus->operation = I2C_READ_OPER;
+	bus->slv_rd_size = nread;
+	bus->slv_rd_ind = 0;
+
+	iowrite8(0, bus->reg + NPCM_I2CTXF_CTL);
+	iowrite8(I2C_HW_FIFO_SIZE, bus->reg + NPCM_I2CRXF_CTL);
+	npcm_i2c_clear_tx_fifo(bus);
+	npcm_i2c_clear_rx_fifo(bus);
+}
+
+static void npcm_i2c_slave_xmit(struct npcm_i2c *bus, u16 nwrite,
+				u8 *write_data)
+{
+	if (nwrite == 0)
+		return;
+
+	bus->state = I2C_OPER_STARTED;
+	bus->operation = I2C_WRITE_OPER;
+
+	/* get the next buffer */
+	npcm_i2c_slave_get_wr_buf(bus);
+	npcm_i2c_write_fifo_slave(bus, nwrite);
+}
+
+/*
+ * npcm_i2c_slave_wr_buf_sync:
+ * currently slave IF only supports single byte operations.
+ * in order to utilyze the npcm HW FIFO, the driver will ask for 16 bytes
+ * at a time, pack them in buffer, and then transmit them all together
+ * to the FIFO and onward to the bus.
+ * NACK on read will be once reached to bus->adap->quirks->max_read_len.
+ * sending a NACK wherever the backend requests for it is not supported.
+ * the next two functions allow reading to local buffer before writing it all
+ * to the HW FIFO.
+ */
+static void npcm_i2c_slave_wr_buf_sync(struct npcm_i2c *bus)
+{
+	int left_in_fifo;
+
+	left_in_fifo = FIELD_GET(NPCM_I2CTXF_STS_TX_BYTES,
+				 ioread8(bus->reg + NPCM_I2CTXF_STS));
+
+	/* fifo already full: */
+	if (left_in_fifo >= I2C_HW_FIFO_SIZE ||
+	    bus->slv_wr_size >= I2C_HW_FIFO_SIZE)
+		return;
+
+	/* update the wr fifo index back to the untransmitted bytes: */
+	bus->slv_wr_ind = bus->slv_wr_ind - left_in_fifo;
+	bus->slv_wr_size = bus->slv_wr_size + left_in_fifo;
+
+	if (bus->slv_wr_ind < 0)
+		bus->slv_wr_ind += I2C_HW_FIFO_SIZE;
+}
+
+static void npcm_i2c_slave_rd_wr(struct npcm_i2c *bus)
+{
+	if (NPCM_I2CST_XMIT & ioread8(bus->reg + NPCM_I2CST)) {
+		/*
+		 * Slave got an address match with direction bit 1 so it should
+		 * transmit data. Write till the master will NACK
+		 */
+		bus->operation = I2C_WRITE_OPER;
+		npcm_i2c_slave_xmit(bus, bus->adap.quirks->max_write_len,
+				    bus->slv_wr_buf);
+	} else {
+		/*
+		 * Slave got an address match with direction bit 0 so it should
+		 * receive data.
+		 * this module does not support saying no to bytes.
+		 * it will always ACK.
+		 */
+		bus->operation = I2C_READ_OPER;
+		npcm_i2c_read_fifo_slave(bus, npcm_i2c_fifo_usage(bus));
+		bus->stop_ind = I2C_SLAVE_RCV_IND;
+		npcm_i2c_slave_send_rd_buf(bus);
+		npcm_i2c_slave_receive(bus, bus->adap.quirks->max_read_len,
+				       bus->slv_rd_buf);
+	}
+}
+
+static irqreturn_t npcm_i2c_int_slave_handler(struct npcm_i2c *bus)
+{
+	u8 val;
+	irqreturn_t ret = IRQ_NONE;
+	u8 i2cst = ioread8(bus->reg + NPCM_I2CST);
+
+	/* Slave: A NACK has occurred */
+	if (NPCM_I2CST_NEGACK & i2cst) {
+		bus->stop_ind = I2C_NACK_IND;
+		npcm_i2c_slave_wr_buf_sync(bus);
+		if (bus->fifo_use)
+			/* clear the FIFO */
+			iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO,
+				 bus->reg + NPCM_I2CFIF_CTS);
+
+		/* In slave write, NACK is OK, otherwise it is a problem */
+		bus->stop_ind = I2C_NO_STATUS_IND;
+		bus->operation = I2C_NO_OPER;
+		bus->own_slave_addr = 0xFF;
+
+		/*
+		 * Slave has to wait for STOP to decide this is the end
+		 * of the transaction. tx is not yet considered as done
+		 */
+		iowrite8(NPCM_I2CST_NEGACK, bus->reg + NPCM_I2CST);
+
+		ret = IRQ_HANDLED;
+	}
+
+	/* Slave mode: a Bus Error (BER) has been identified */
+	if (NPCM_I2CST_BER & i2cst) {
+		/*
+		 * Check whether bus arbitration or Start or Stop during data
+		 * xfer bus arbitration problem should not result in recovery
+		 */
+		bus->stop_ind = I2C_BUS_ERR_IND;
+
+		/* wait for bus busy before clear fifo */
+		iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, bus->reg + NPCM_I2CFIF_CTS);
+
+		bus->state = I2C_IDLE;
+
+		/*
+		 * in BER case we might get 2 interrupts: one for slave one for
+		 * master ( for a channel which is master\slave switching)
+		 */
+		if (completion_done(&bus->cmd_complete) == false) {
+			bus->cmd_err = -EIO;
+			complete(&bus->cmd_complete);
+		}
+		bus->own_slave_addr = 0xFF;
+		iowrite8(NPCM_I2CST_BER, bus->reg + NPCM_I2CST);
+		ret = IRQ_HANDLED;
+	}
+
+	/* A Slave Stop Condition has been identified */
+	if (NPCM_I2CST_SLVSTP & i2cst) {
+		u8 bytes_in_fifo = npcm_i2c_fifo_usage(bus);
+
+		bus->stop_ind = I2C_SLAVE_DONE_IND;
+
+		if (bus->operation == I2C_READ_OPER)
+			npcm_i2c_read_fifo_slave(bus, bytes_in_fifo);
+
+		/* if the buffer is empty nothing will be sent */
+		npcm_i2c_slave_send_rd_buf(bus);
+
+		/* Slave done transmitting or receiving */
+		bus->stop_ind = I2C_NO_STATUS_IND;
+
+		/*
+		 * Note, just because we got here, it doesn't mean we through
+		 * away the wr buffer.
+		 * we keep it until the next received offset.
+		 */
+		bus->operation = I2C_NO_OPER;
+		bus->own_slave_addr = 0xFF;
+		i2c_slave_event(bus->slave, I2C_SLAVE_STOP, 0);
+		iowrite8(NPCM_I2CST_SLVSTP, bus->reg + NPCM_I2CST);
+		if (bus->fifo_use) {
+			npcm_i2c_clear_fifo_int(bus);
+			npcm_i2c_clear_rx_fifo(bus);
+			npcm_i2c_clear_tx_fifo(bus);
+
+			iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO,
+				 bus->reg + NPCM_I2CFIF_CTS);
+		}
+		bus->state = I2C_IDLE;
+		ret = IRQ_HANDLED;
+	}
+
+	/* restart condition occurred and Rx-FIFO was not empty */
+	if (bus->fifo_use && FIELD_GET(NPCM_I2CFIF_CTS_SLVRSTR,
+				       ioread8(bus->reg + NPCM_I2CFIF_CTS))) {
+		bus->stop_ind = I2C_SLAVE_RESTART_IND;
+		bus->master_or_slave = I2C_SLAVE;
+		if (bus->operation == I2C_READ_OPER)
+			npcm_i2c_read_fifo_slave(bus, npcm_i2c_fifo_usage(bus));
+		bus->operation = I2C_WRITE_OPER;
+		iowrite8(0, bus->reg + NPCM_I2CRXF_CTL);
+		val = NPCM_I2CFIF_CTS_CLR_FIFO | NPCM_I2CFIF_CTS_SLVRSTR |
+		      NPCM_I2CFIF_CTS_RXF_TXE;
+		iowrite8(val, bus->reg + NPCM_I2CFIF_CTS);
+		npcm_i2c_slave_rd_wr(bus);
+		ret = IRQ_HANDLED;
+	}
+
+	/* A Slave Address Match has been identified */
+	if (NPCM_I2CST_NMATCH & i2cst) {
+		u8 info = 0;
+
+		/* Address match automatically implies slave mode */
+		bus->master_or_slave = I2C_SLAVE;
+		npcm_i2c_clear_fifo_int(bus);
+		npcm_i2c_clear_rx_fifo(bus);
+		npcm_i2c_clear_tx_fifo(bus);
+		iowrite8(0, bus->reg + NPCM_I2CTXF_CTL);
+		iowrite8(I2C_HW_FIFO_SIZE, bus->reg + NPCM_I2CRXF_CTL);
+		if (NPCM_I2CST_XMIT & i2cst) {
+			bus->operation = I2C_WRITE_OPER;
+		} else {
+			i2c_slave_event(bus->slave, I2C_SLAVE_WRITE_REQUESTED,
+					&info);
+			bus->operation = I2C_READ_OPER;
+		}
+		if (bus->own_slave_addr == 0xFF) {
+			/* Check which type of address match */
+			val = ioread8(bus->reg + NPCM_I2CCST);
+			if (NPCM_I2CCST_MATCH & val) {
+				u16 addr;
+				enum i2c_addr eaddr;
+				u8 i2ccst2;
+				u8 i2ccst3;
+
+				i2ccst3 = ioread8(bus->reg + NPCM_I2CCST3);
+				i2ccst2 = ioread8(bus->reg + NPCM_I2CCST2);
+
+				/*
+				 * the i2c module can response to 10 own SA.
+				 * check which one was addressed by the master.
+				 * repond to the first one.
+				 */
+				addr = ((i2ccst3 & 0x07) << 7) |
+					(i2ccst2 & 0x7F);
+				info = ffs(addr);
+				eaddr = (enum i2c_addr)info;
+				addr = npcm_i2c_get_slave_addr(bus, eaddr);
+				addr &= 0x7F;
+				bus->own_slave_addr = addr;
+				if (bus->PEC_mask & BIT(info))
+					bus->PEC_use = true;
+				else
+					bus->PEC_use = false;
+			} else {
+				if (NPCM_I2CCST_GCMATCH & val)
+					bus->own_slave_addr = 0;
+				if (NPCM_I2CCST_ARPMATCH & val)
+					bus->own_slave_addr = 0x61;
+			}
+		} else {
+			/*
+			 *  Slave match can happen in two options:
+			 *  1. Start, SA, read (slave read without further ado)
+			 *  2. Start, SA, read, data, restart, SA, read,  ...
+			 *     (slave read in fragmented mode)
+			 *  3. Start, SA, write, data, restart, SA, read, ..
+			 *     (regular write-read mode)
+			 */
+			if ((bus->state == I2C_OPER_STARTED &&
+			     bus->operation == I2C_READ_OPER &&
+			     bus->stop_ind == I2C_SLAVE_XMIT_IND) ||
+			     bus->stop_ind == I2C_SLAVE_RCV_IND) {
+				/* slave tx after slave rx w/o STOP */
+				bus->stop_ind = I2C_SLAVE_RESTART_IND;
+			}
+		}
+
+		if (NPCM_I2CST_XMIT & i2cst)
+			bus->stop_ind = I2C_SLAVE_XMIT_IND;
+		else
+			bus->stop_ind = I2C_SLAVE_RCV_IND;
+		bus->state = I2C_SLAVE_MATCH;
+		npcm_i2c_slave_rd_wr(bus);
+		iowrite8(NPCM_I2CST_NMATCH, bus->reg + NPCM_I2CST);
+		ret = IRQ_HANDLED;
+	}
+
+	/* Slave SDA status is set - tx or rx */
+	if ((NPCM_I2CST_SDAST & i2cst) ||
+	    (bus->fifo_use &&
+	    (npcm_i2c_tx_fifo_empty(bus) || npcm_i2c_rx_fifo_full(bus)))) {
+		npcm_i2c_slave_rd_wr(bus);
+		iowrite8(NPCM_I2CST_SDAST, bus->reg + NPCM_I2CST);
+		ret = IRQ_HANDLED;
+	} /* SDAST */
+
+	return ret;
+}
+
+static int npcm_i2c_reg_slave(struct i2c_client *client)
+{
+	unsigned long lock_flags;
+	struct npcm_i2c *bus = i2c_get_adapdata(client->adapter);
+
+	bus->slave = client;
+
+	if (!bus->slave)
+		return -EINVAL;
+
+	if (client->flags & I2C_CLIENT_TEN)
+		return -EAFNOSUPPORT;
+
+	spin_lock_irqsave(&bus->lock, lock_flags);
+
+	npcm_i2c_init_params(bus);
+	bus->slv_rd_size = 0;
+	bus->slv_wr_size = 0;
+	bus->slv_rd_ind = 0;
+	bus->slv_wr_ind = 0;
+	if (client->flags & I2C_CLIENT_PEC)
+		bus->PEC_use = true;
+
+	dev_info(bus->dev, "i2c%d register slave SA=0x%x, PEC=%d\n", bus->num,
+		 client->addr, bus->PEC_use);
+
+	npcm_i2c_slave_enable(bus, I2C_SLAVE_ADDR1, client->addr, true);
+	npcm_i2c_clear_fifo_int(bus);
+	npcm_i2c_clear_rx_fifo(bus);
+	npcm_i2c_clear_tx_fifo(bus);
+	npcm_i2c_slave_int_enable(bus, true);
+
+	spin_unlock_irqrestore(&bus->lock, lock_flags);
+	return 0;
+}
+
+static int npcm_i2c_unreg_slave(struct i2c_client *client)
+{
+	struct npcm_i2c *bus = client->adapter->algo_data;
+	unsigned long lock_flags;
+
+	spin_lock_irqsave(&bus->lock, lock_flags);
+	if (!bus->slave) {
+		spin_unlock_irqrestore(&bus->lock, lock_flags);
+		return -EINVAL;
+	}
+	npcm_i2c_slave_int_enable(bus, false);
+	npcm_i2c_remove_slave_addr(bus, client->addr);
+	bus->slave = NULL;
+	spin_unlock_irqrestore(&bus->lock, lock_flags);
+	return 0;
+}
+#endif /* CONFIG_I2C_SLAVE */
+
+static void npcm_i2c_master_fifo_read(struct npcm_i2c *bus)
+{
+	int rcount;
+	int fifo_bytes;
+	enum i2c_state_ind ind = I2C_MASTER_DONE_IND;
+
+	fifo_bytes = npcm_i2c_fifo_usage(bus);
+	rcount = bus->rd_size - bus->rd_ind;
+
+	/*
+	 * In order not to change the RX_TRH during transaction (we found that
+	 * this might be problematic if it takes too much time to read the FIFO)
+	 * we read the data in the following way. If the number of bytes to
+	 * read == FIFO Size + C (where C < FIFO Size)then first read C bytes
+	 * and in the next int we read rest of the data.
+	 */
+	if (rcount < (2 * I2C_HW_FIFO_SIZE) && rcount > I2C_HW_FIFO_SIZE)
+		fifo_bytes = rcount - I2C_HW_FIFO_SIZE;
+
+	if (rcount <= fifo_bytes) {
+		/* last bytes are about to be read - end of tx */
+		bus->state = I2C_STOP_PENDING;
+		bus->stop_ind = ind;
+		npcm_i2c_eob_int(bus, true);
+		/* Stop should be set before reading last byte. */
+		npcm_i2c_master_stop(bus);
+		npcm_i2c_read_fifo(bus, fifo_bytes);
+	} else {
+		npcm_i2c_read_fifo(bus, fifo_bytes);
+		rcount = bus->rd_size - bus->rd_ind;
+		npcm_i2c_set_fifo(bus, rcount, -1);
+	}
+}
+
+static void npcm_i2c_irq_master_handler_write(struct npcm_i2c *bus)
+{
+	u16 wcount;
+
+	if (bus->fifo_use)
+		npcm_i2c_clear_tx_fifo(bus); /* clear the TX fifo status bit */
+
+	/* Master write operation - last byte handling */
+	if (bus->wr_ind == bus->wr_size) {
+		if (bus->fifo_use && npcm_i2c_fifo_usage(bus) > 0)
+			/*
+			 * No more bytes to send (to add to the FIFO),
+			 * however the FIFO is not empty yet. It is
+			 * still in the middle of tx. Currently there's nothing
+			 * to do except for waiting to the end of the tx
+			 * We will get an int when the FIFO will get empty.
+			 */
+			return;
+
+		if (bus->rd_size == 0) {
+			/* all bytes have been written, in wr only operation */
+			npcm_i2c_eob_int(bus, true);
+			bus->state = I2C_STOP_PENDING;
+			bus->stop_ind = I2C_MASTER_DONE_IND;
+			npcm_i2c_master_stop(bus);
+			/* Clear SDA Status bit (by writing dummy byte) */
+			npcm_i2c_wr_byte(bus, 0xFF);
+
+		} else {
+			/* last write-byte written on previous int - restart */
+			npcm_i2c_set_fifo(bus, bus->rd_size, -1);
+			/* Generate repeated start upon next write to SDA */
+			npcm_i2c_master_start(bus);
+
+			/*
+			 * Receiving one byte only - stall after successful
+			 * completion of send address byte. If we NACK here, and
+			 * slave doesn't ACK the address, we might
+			 * unintentionally NACK the next multi-byte read.
+			 */
+			if (bus->rd_size == 1)
+				npcm_i2c_stall_after_start(bus, true);
+
+			/* Next int will occur on read */
+			bus->operation = I2C_READ_OPER;
+			/* send the slave address in read direction */
+			npcm_i2c_wr_byte(bus, bus->dest_addr | 0x1);
+		}
+	} else {
+		/* write next byte not last byte and not slave address */
+		if (!bus->fifo_use || bus->wr_size == 1) {
+			npcm_i2c_wr_byte(bus, bus->wr_buf[bus->wr_ind++]);
+		} else {
+			wcount = bus->wr_size - bus->wr_ind;
+			npcm_i2c_set_fifo(bus, -1, wcount);
+			if (wcount)
+				npcm_i2c_write_to_fifo_master(bus, wcount);
+		}
+	}
+}
+
+static void npcm_i2c_irq_master_handler_read(struct npcm_i2c *bus)
+{
+	u16 block_extra_bytes_size;
+	u8 data;
+
+	/* added bytes to the packet: */
+	block_extra_bytes_size = bus->read_block_use + bus->PEC_use;
+
+	/*
+	 * Perform master read, distinguishing between last byte and the rest of
+	 * the bytes. The last byte should be read when the clock is stopped
+	 */
+	if (bus->rd_ind == 0) { /* first byte handling: */
+		if (bus->read_block_use) {
+			/* first byte in block protocol is the size: */
+			data = npcm_i2c_rd_byte(bus);
+			data = clamp_val(data, 1, I2C_SMBUS_BLOCK_MAX);
+			bus->rd_size = data + block_extra_bytes_size;
+			bus->rd_buf[bus->rd_ind++] = data;
+
+			/* clear RX FIFO interrupt status: */
+			if (bus->fifo_use) {
+				data = ioread8(bus->reg + NPCM_I2CFIF_CTS);
+				data = data | NPCM_I2CFIF_CTS_RXF_TXE;
+				iowrite8(data, bus->reg + NPCM_I2CFIF_CTS);
+			}
+
+			npcm_i2c_set_fifo(bus, bus->rd_size - 1, -1);
+			npcm_i2c_stall_after_start(bus, false);
+		} else {
+			npcm_i2c_clear_tx_fifo(bus);
+			npcm_i2c_master_fifo_read(bus);
+		}
+	} else {
+		if (bus->rd_size == block_extra_bytes_size &&
+		    bus->read_block_use) {
+			bus->state = I2C_STOP_PENDING;
+			bus->stop_ind = I2C_BLOCK_BYTES_ERR_IND;
+			bus->cmd_err = -EIO;
+			npcm_i2c_eob_int(bus, true);
+			npcm_i2c_master_stop(bus);
+			npcm_i2c_read_fifo(bus, npcm_i2c_fifo_usage(bus));
+		} else {
+			npcm_i2c_master_fifo_read(bus);
+		}
+	}
+}
+
+static void npcm_i2c_irq_handle_nmatch(struct npcm_i2c *bus)
+{
+	iowrite8(NPCM_I2CST_NMATCH, bus->reg + NPCM_I2CST);
+	npcm_i2c_nack(bus);
+	bus->stop_ind = I2C_BUS_ERR_IND;
+	npcm_i2c_callback(bus, bus->stop_ind, npcm_i2c_get_index(bus));
+}
+
+/* A NACK has occurred */
+static void npcm_i2c_irq_handle_nack(struct npcm_i2c *bus)
+{
+	u8 val;
+
+	if (bus->nack_cnt < ULLONG_MAX)
+		bus->nack_cnt++;
+
+	if (bus->fifo_use) {
+		/*
+		 * if there are still untransmitted bytes in TX FIFO
+		 * reduce them from wr_ind
+		 */
+		if (bus->operation == I2C_WRITE_OPER)
+			bus->wr_ind -= npcm_i2c_fifo_usage(bus);
+
+		/* clear the FIFO */
+		iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, bus->reg + NPCM_I2CFIF_CTS);
+	}
+
+	/* In master write operation, got unexpected NACK */
+	bus->stop_ind = I2C_NACK_IND;
+	/* Only current master is allowed to issue Stop Condition */
+	if (npcm_i2c_is_master(bus)) {
+		/* stopping in the middle */
+		npcm_i2c_eob_int(bus, false);
+		npcm_i2c_master_stop(bus);
+
+		/*
+		 * The bus is released from stall only after the SW clears
+		 * NEGACK bit. Then a Stop condition is sent.
+		 */
+		npcm_i2c_clear_master_status(bus);
+		readx_poll_timeout_atomic(ioread8, bus->reg + NPCM_I2CCST, val,
+					  !(val & NPCM_I2CCST_BUSY), 10, 200);
+	}
+	bus->state = I2C_IDLE;
+
+	/*
+	 * In Master mode, NACK should be cleared only after STOP.
+	 * In such case, the bus is released from stall only after the
+	 * software clears NACK bit. Then a Stop condition is sent.
+	 */
+	npcm_i2c_callback(bus, bus->stop_ind, bus->wr_ind);
+}
+
+	/* Master mode: a Bus Error has been identified */
+static void npcm_i2c_irq_handle_ber(struct npcm_i2c *bus)
+{
+	if (bus->ber_cnt < ULLONG_MAX)
+		bus->ber_cnt++;
+	bus->stop_ind = I2C_BUS_ERR_IND;
+	if (npcm_i2c_is_master(bus)) {
+		npcm_i2c_master_abort(bus);
+	} else {
+		npcm_i2c_clear_master_status(bus);
+
+		/* Clear BB (BUS BUSY) bit */
+		iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST);
+
+		bus->cmd_err = -EAGAIN;
+		npcm_i2c_callback(bus, bus->stop_ind, npcm_i2c_get_index(bus));
+	}
+	bus->state = I2C_IDLE;
+}
+
+	/* EOB: a master End Of Busy (meaning STOP completed) */
+static void npcm_i2c_irq_handle_eob(struct npcm_i2c *bus)
+{
+	npcm_i2c_eob_int(bus, false);
+	bus->state = I2C_IDLE;
+	npcm_i2c_callback(bus, bus->stop_ind, bus->rd_ind);
+}
+
+/* Address sent and requested stall occurred (Master mode) */
+static void npcm_i2c_irq_handle_stall_after_start(struct npcm_i2c *bus)
+{
+	if (npcm_i2c_is_quick(bus)) {
+		bus->state = I2C_STOP_PENDING;
+		bus->stop_ind = I2C_MASTER_DONE_IND;
+		npcm_i2c_eob_int(bus, true);
+		npcm_i2c_master_stop(bus);
+	} else if ((bus->rd_size == 1) && !bus->read_block_use) {
+		/*
+		 * Receiving one byte only - set NACK after ensuring
+		 * slave ACKed the address byte.
+		 */
+		npcm_i2c_nack(bus);
+	}
+
+	/* Reset stall-after-address-byte */
+	npcm_i2c_stall_after_start(bus, false);
+
+	/* Clear stall only after setting STOP */
+	iowrite8(NPCM_I2CST_STASTR, bus->reg + NPCM_I2CST);
+}
+
+/* SDA status is set - TX or RX, master */
+static void npcm_i2c_irq_handle_sda(struct npcm_i2c *bus, u8 i2cst)
+{
+	u8 fif_cts;
+
+	if (!npcm_i2c_is_master(bus))
+		return;
+
+	if (bus->state == I2C_IDLE) {
+		bus->stop_ind = I2C_WAKE_UP_IND;
+
+		if (npcm_i2c_is_quick(bus) || bus->read_block_use)
+			/*
+			 * Need to stall after successful
+			 * completion of sending address byte
+			 */
+			npcm_i2c_stall_after_start(bus, true);
+		else
+			npcm_i2c_stall_after_start(bus, false);
+
+		/*
+		 * Receiving one byte only - stall after successful completion
+		 * of sending address byte If we NACK here, and slave doesn't
+		 * ACK the address, we might unintentionally NACK the next
+		 * multi-byte read
+		 */
+		if (bus->wr_size == 0 && bus->rd_size == 1)
+			npcm_i2c_stall_after_start(bus, true);
+
+		/* Initiate I2C master tx */
+
+		/* select bank 1 for FIFO regs */
+		npcm_i2c_select_bank(bus, I2C_BANK_1);
+
+		fif_cts = ioread8(bus->reg + NPCM_I2CFIF_CTS);
+		fif_cts = fif_cts & ~NPCM_I2CFIF_CTS_SLVRSTR;
+
+		/* clear FIFO and relevant status bits. */
+		fif_cts = fif_cts | NPCM_I2CFIF_CTS_CLR_FIFO;
+		iowrite8(fif_cts, bus->reg + NPCM_I2CFIF_CTS);
+
+		/* re-enable */
+		fif_cts = fif_cts | NPCM_I2CFIF_CTS_RXF_TXE;
+		iowrite8(fif_cts, bus->reg + NPCM_I2CFIF_CTS);
+
+		/*
+		 * Configure the FIFO threshold:
+		 * according to the needed # of bytes to read.
+		 * Note: due to HW limitation can't config the rx fifo before it
+		 * got and ACK on the restart. LAST bit will not be reset unless
+		 * RX completed. It will stay set on the next tx.
+		 */
+		if (bus->wr_size)
+			npcm_i2c_set_fifo(bus, -1, bus->wr_size);
+		else
+			npcm_i2c_set_fifo(bus, bus->rd_size, -1);
+
+		bus->state = I2C_OPER_STARTED;
+
+		if (npcm_i2c_is_quick(bus) || bus->wr_size)
+			npcm_i2c_wr_byte(bus, bus->dest_addr);
+		else
+			npcm_i2c_wr_byte(bus, bus->dest_addr | BIT(0));
+	/* SDA interrupt, after start\restart */
+	} else {
+		if (NPCM_I2CST_XMIT & i2cst) {
+			bus->operation = I2C_WRITE_OPER;
+			npcm_i2c_irq_master_handler_write(bus);
+		} else {
+			bus->operation = I2C_READ_OPER;
+			npcm_i2c_irq_master_handler_read(bus);
+		}
+	}
+}
+
+static int npcm_i2c_int_master_handler(struct npcm_i2c *bus)
+{
+	u8 i2cst;
+	int ret = -EIO;
+
+	i2cst = ioread8(bus->reg + NPCM_I2CST);
+
+	if (FIELD_GET(NPCM_I2CST_NMATCH, i2cst)) {
+		npcm_i2c_irq_handle_nmatch(bus);
+		return 0;
+	}
+	/* A NACK has occurred */
+	if (FIELD_GET(NPCM_I2CST_NEGACK, i2cst)) {
+		npcm_i2c_irq_handle_nack(bus);
+		return 0;
+	}
+
+	/* Master mode: a Bus Error has been identified */
+	if (FIELD_GET(NPCM_I2CST_BER, i2cst)) {
+		npcm_i2c_irq_handle_ber(bus);
+		return 0;
+	}
+
+	/* EOB: a master End Of Busy (meaning STOP completed) */
+	if ((FIELD_GET(NPCM_I2CCTL1_EOBINTE,
+		       ioread8(bus->reg + NPCM_I2CCTL1)) == 1) &&
+	    (FIELD_GET(NPCM_I2CCST3_EO_BUSY,
+		       ioread8(bus->reg + NPCM_I2CCST3)))) {
+		npcm_i2c_irq_handle_eob(bus);
+		return 0;
+	}
+
+	/* Address sent and requested stall occurred (Master mode) */
+	if (FIELD_GET(NPCM_I2CST_STASTR, i2cst)) {
+		npcm_i2c_irq_handle_stall_after_start(bus);
+		ret = 0;
+	}
+
+	/* SDA status is set - TX or RX, master */
+	if (FIELD_GET(NPCM_I2CST_SDAST, i2cst) ||
+	    (bus->fifo_use &&
+	    (npcm_i2c_tx_fifo_empty(bus) || npcm_i2c_rx_fifo_full(bus)))) {
+		npcm_i2c_irq_handle_sda(bus, i2cst);
+		ret = 0;
+	}
+
+	return ret;
+}
+
+/* recovery using TGCLK functionality of the module */
+static int npcm_i2c_recovery_tgclk(struct i2c_adapter *_adap)
+{
+	u8               val;
+	u8               fif_cts;
+	bool             done = false;
+	int              status = -ENOTRECOVERABLE;
+	struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap);
+	/* Allow 3 bytes (27 toggles) to be read from the slave: */
+	int              iter = 27;
+
+	if ((npcm_i2c_get_SDA(_adap) == 1) && (npcm_i2c_get_SCL(_adap) == 1)) {
+		dev_dbg(bus->dev, "bus%d recovery skipped, bus not stuck",
+			bus->num);
+		npcm_i2c_reset(bus);
+		return status;
+	}
+
+	npcm_i2c_int_enable(bus, false);
+	npcm_i2c_disable(bus);
+	npcm_i2c_enable(bus);
+	iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST);
+	npcm_i2c_clear_tx_fifo(bus);
+	npcm_i2c_clear_rx_fifo(bus);
+	iowrite8(0, bus->reg + NPCM_I2CRXF_CTL);
+	iowrite8(0, bus->reg + NPCM_I2CTXF_CTL);
+	npcm_i2c_stall_after_start(bus, false);
+
+	/* select bank 1 for FIFO regs */
+	npcm_i2c_select_bank(bus, I2C_BANK_1);
+
+	/* clear FIFO and relevant status bits. */
+	fif_cts = ioread8(bus->reg + NPCM_I2CFIF_CTS);
+	fif_cts &= ~NPCM_I2CFIF_CTS_SLVRSTR;
+	fif_cts |= NPCM_I2CFIF_CTS_CLR_FIFO;
+	iowrite8(fif_cts, bus->reg + NPCM_I2CFIF_CTS);
+	npcm_i2c_set_fifo(bus, -1, 0);
+
+	/* Repeat the following sequence until SDA is released */
+	do {
+		/* Issue a single SCL toggle */
+		iowrite8(NPCM_I2CCST_TGSCL, bus->reg + NPCM_I2CCST);
+		usleep_range(20, 30);
+		/* If SDA line is inactive (high), stop */
+		if (npcm_i2c_get_SDA(_adap)) {
+			done = true;
+			status = 0;
+		}
+	} while (!done && iter--);
+
+	/* If SDA line is released: send start-addr-stop, to re-sync. */
+	if (npcm_i2c_get_SDA(_adap)) {
+		/* Send an address byte in write direction: */
+		npcm_i2c_wr_byte(bus, bus->dest_addr);
+		npcm_i2c_master_start(bus);
+		/* Wait until START condition is sent */
+		status = readx_poll_timeout(npcm_i2c_get_SCL, _adap, val, !val,
+					    20, 200);
+		/* If START condition was sent */
+		if (npcm_i2c_is_master(bus) > 0) {
+			usleep_range(20, 30);
+			npcm_i2c_master_stop(bus);
+			usleep_range(200, 500);
+		}
+	}
+	npcm_i2c_reset(bus);
+	npcm_i2c_int_enable(bus, true);
+
+	if ((npcm_i2c_get_SDA(_adap) == 1) && (npcm_i2c_get_SCL(_adap) == 1))
+		status = 0;
+	else
+		status = -ENOTRECOVERABLE;
+	if (status) {
+		if (bus->rec_fail_cnt < ULLONG_MAX)
+			bus->rec_fail_cnt++;
+	} else {
+		if (bus->rec_succ_cnt < ULLONG_MAX)
+			bus->rec_succ_cnt++;
+	}
+	return status;
+}
+
+/* recovery using bit banging functionality of the module */
+static void npcm_i2c_recovery_init(struct i2c_adapter *_adap)
+{
+	struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap);
+	struct i2c_bus_recovery_info *rinfo = &bus->rinfo;
+
+	rinfo->recover_bus = npcm_i2c_recovery_tgclk;
+
+	/*
+	 * npcm i2c HW allows direct reading of SCL and SDA.
+	 * However, it does not support setting SCL and SDA directly.
+	 * The recovery function can togle SCL when SDA is low (but not set)
+	 * Getter functions used internally, and can be used externaly.
+	 */
+	rinfo->get_scl = npcm_i2c_get_SCL;
+	rinfo->get_sda = npcm_i2c_get_SDA;
+	_adap->bus_recovery_info = rinfo;
+}
+
+/* SCLFRQ min/max field values */
+#define SCLFRQ_MIN  10
+#define SCLFRQ_MAX  511
+#define clk_coef(freq, mul)	DIV_ROUND_UP((freq) * (mul), 1000000)
+
+/*
+ * npcm_i2c_init_clk: init HW timing parameters.
+ * NPCM7XX i2c module timing parameters are depenent on module core clk (APB)
+ * and bus frequency.
+ * 100kHz bus requires tSCL = 4 * SCLFRQ * tCLK. LT and HT are simetric.
+ * 400kHz bus requires assymetric HT and LT. A different equation is recomended
+ * by the HW designer, given core clock range (equations in comments below).
+ *
+ */
+static int npcm_i2c_init_clk(struct npcm_i2c *bus, u32 bus_freq_hz)
+{
+	u32  k1 = 0;
+	u32  k2 = 0;
+	u8   dbnct = 0;
+	u32  sclfrq = 0;
+	u8   hldt = 7;
+	u8   fast_mode = 0;
+	u32  src_clk_khz;
+	u32  bus_freq_khz;
+
+	src_clk_khz = bus->apb_clk / 1000;
+	bus_freq_khz = bus_freq_hz / 1000;
+	bus->bus_freq = bus_freq_hz;
+
+	/* 100KHz and below: */
+	if (bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ) {
+		sclfrq = src_clk_khz / (bus_freq_khz * 4);
+
+		if (sclfrq < SCLFRQ_MIN || sclfrq > SCLFRQ_MAX)
+			return -EDOM;
+
+		if (src_clk_khz >= 40000)
+			hldt = 17;
+		else if (src_clk_khz >= 12500)
+			hldt = 15;
+		else
+			hldt = 7;
+	}
+
+	/* 400KHz: */
+	else if (bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) {
+		sclfrq = 0;
+		fast_mode = I2CCTL3_400K_MODE;
+
+		if (src_clk_khz < 7500)
+			/* 400KHZ cannot be supported for core clock < 7.5MHz */
+			return -EDOM;
+
+		else if (src_clk_khz >= 50000) {
+			k1 = 80;
+			k2 = 48;
+			hldt = 12;
+			dbnct = 7;
+		}
+
+		/* Master or Slave with frequency > 25MHz */
+		else if (src_clk_khz > 25000) {
+			hldt = clk_coef(src_clk_khz, 300) + 7;
+			k1 = clk_coef(src_clk_khz, 1600);
+			k2 = clk_coef(src_clk_khz, 900);
+		}
+	}
+
+	/* 1MHz: */
+	else if (bus_freq_hz <= I2C_MAX_FAST_MODE_PLUS_FREQ) {
+		sclfrq = 0;
+		fast_mode = I2CCTL3_400K_MODE;
+
+		/* 1MHZ cannot be supported for core clock < 24 MHz */
+		if (src_clk_khz < 24000)
+			return -EDOM;
+
+		k1 = clk_coef(src_clk_khz, 620);
+		k2 = clk_coef(src_clk_khz, 380);
+
+		/* Core clk > 40 MHz */
+		if (src_clk_khz > 40000) {
+			/*
+			 * Set HLDT:
+			 * SDA hold time:  (HLDT-7) * T(CLK) >= 120
+			 * HLDT = 120/T(CLK) + 7 = 120 * FREQ(CLK) + 7
+			 */
+			hldt = clk_coef(src_clk_khz, 120) + 7;
+		} else {
+			hldt = 7;
+			dbnct = 2;
+		}
+	}
+
+	/* Frequency larger than 1 MHz is not supported */
+	else
+		return -EINVAL;
+
+	if (bus_freq_hz >= I2C_MAX_FAST_MODE_FREQ) {
+		k1 = round_up(k1, 2);
+		k2 = round_up(k2 + 1, 2);
+		if (k1 < SCLFRQ_MIN || k1 > SCLFRQ_MAX ||
+		    k2 < SCLFRQ_MIN || k2 > SCLFRQ_MAX)
+			return -EDOM;
+	}
+
+	/* write sclfrq value. bits [6:0] are in I2CCTL2 reg */
+	iowrite8(FIELD_PREP(I2CCTL2_SCLFRQ6_0, sclfrq & 0x7F),
+		 bus->reg + NPCM_I2CCTL2);
+
+	/* bits [8:7] are in I2CCTL3 reg */
+	iowrite8(fast_mode | FIELD_PREP(I2CCTL3_SCLFRQ8_7, (sclfrq >> 7) & 0x3),
+		 bus->reg + NPCM_I2CCTL3);
+
+	/* Select Bank 0 to access NPCM_I2CCTL4/NPCM_I2CCTL5 */
+	npcm_i2c_select_bank(bus, I2C_BANK_0);
+
+	if (bus_freq_hz >= I2C_MAX_FAST_MODE_FREQ) {
+		/*
+		 * Set SCL Low/High Time:
+		 * k1 = 2 * SCLLT7-0 -> Low Time  = k1 / 2
+		 * k2 = 2 * SCLLT7-0 -> High Time = k2 / 2
+		 */
+		iowrite8(k1 / 2, bus->reg + NPCM_I2CSCLLT);
+		iowrite8(k2 / 2, bus->reg + NPCM_I2CSCLHT);
+
+		iowrite8(dbnct, bus->reg + NPCM_I2CCTL5);
+	}
+
+	iowrite8(hldt, bus->reg + NPCM_I2CCTL4);
+
+	/* Return to Bank 1, and stay there by default: */
+	npcm_i2c_select_bank(bus, I2C_BANK_1);
+
+	return 0;
+}
+
+static int npcm_i2c_init_module(struct npcm_i2c *bus, enum i2c_mode mode,
+				u32 bus_freq_hz)
+{
+	u8 val;
+	int ret;
+
+	/* Check whether module already enabled or frequency is out of bounds */
+	if ((bus->state != I2C_DISABLE && bus->state != I2C_IDLE) ||
+	    bus_freq_hz < I2C_FREQ_MIN_HZ || bus_freq_hz > I2C_FREQ_MAX_HZ)
+		return -EINVAL;
+
+	npcm_i2c_disable(bus);
+
+	/* Configure FIFO mode : */
+	if (FIELD_GET(I2C_VER_FIFO_EN, ioread8(bus->reg + I2C_VER))) {
+		bus->fifo_use = true;
+		npcm_i2c_select_bank(bus, I2C_BANK_0);
+		val = ioread8(bus->reg + NPCM_I2CFIF_CTL);
+		val |= NPCM_I2CFIF_CTL_FIFO_EN;
+		iowrite8(val, bus->reg + NPCM_I2CFIF_CTL);
+		npcm_i2c_select_bank(bus, I2C_BANK_1);
+	} else {
+		bus->fifo_use = false;
+	}
+
+	/* Configure I2C module clock frequency */
+	ret = npcm_i2c_init_clk(bus, bus_freq_hz);
+	if (ret) {
+		dev_err(bus->dev, "npcm_i2c_init_clk failed\n");
+		return ret;
+	}
+
+	/* Enable module (before configuring CTL1) */
+	npcm_i2c_enable(bus);
+	bus->state = I2C_IDLE;
+	val = ioread8(bus->reg + NPCM_I2CCTL1);
+	val = (val | NPCM_I2CCTL1_NMINTE) & ~NPCM_I2CCTL1_RWS;
+	iowrite8(val, bus->reg + NPCM_I2CCTL1);
+
+	npcm_i2c_int_enable(bus, true);
+
+	npcm_i2c_reset(bus);
+
+	return 0;
+}
+
+static int __npcm_i2c_init(struct npcm_i2c *bus, struct platform_device *pdev)
+{
+	u32 clk_freq_hz;
+	int ret;
+
+	/* Initialize the internal data structures */
+	bus->state = I2C_DISABLE;
+	bus->master_or_slave = I2C_SLAVE;
+	bus->int_time_stamp = 0;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	bus->slave = NULL;
+#endif
+
+	ret = device_property_read_u32(&pdev->dev, "clock-frequency",
+				       &clk_freq_hz);
+	if (ret) {
+		dev_info(&pdev->dev, "Could not read clock-frequency property");
+		clk_freq_hz = I2C_MAX_STANDARD_MODE_FREQ;
+	}
+
+	ret = npcm_i2c_init_module(bus, I2C_MASTER, clk_freq_hz);
+	if (ret) {
+		dev_err(&pdev->dev, "npcm_i2c_init_module failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static irqreturn_t npcm_i2c_bus_irq(int irq, void *dev_id)
+{
+	struct npcm_i2c *bus = dev_id;
+
+	if (npcm_i2c_is_master(bus))
+		bus->master_or_slave = I2C_MASTER;
+
+	if (bus->master_or_slave == I2C_MASTER) {
+		bus->int_time_stamp = jiffies;
+		if (!npcm_i2c_int_master_handler(bus))
+			return IRQ_HANDLED;
+	}
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	if (bus->slave) {
+		bus->master_or_slave = I2C_SLAVE;
+		return npcm_i2c_int_slave_handler(bus);
+	}
+#endif
+	return IRQ_NONE;
+}
+
+static bool npcm_i2c_master_start_xmit(struct npcm_i2c *bus,
+				       u8 slave_addr, u16 nwrite, u16 nread,
+				       u8 *write_data, u8 *read_data,
+				       bool use_PEC, bool use_read_block)
+{
+	if (bus->state != I2C_IDLE) {
+		bus->cmd_err = -EBUSY;
+		return false;
+	}
+	bus->dest_addr = slave_addr << 1;
+	bus->wr_buf = write_data;
+	bus->wr_size = nwrite;
+	bus->wr_ind = 0;
+	bus->rd_buf = read_data;
+	bus->rd_size = nread;
+	bus->rd_ind = 0;
+	bus->PEC_use = 0;
+
+	/* for tx PEC is appended to buffer from i2c IF. PEC flag is ignored */
+	if (nread)
+		bus->PEC_use = use_PEC;
+
+	bus->read_block_use = use_read_block;
+	if (nread && !nwrite)
+		bus->operation = I2C_READ_OPER;
+	else
+		bus->operation = I2C_WRITE_OPER;
+	if (bus->fifo_use) {
+		u8 i2cfif_cts;
+
+		npcm_i2c_select_bank(bus, I2C_BANK_1);
+		/* clear FIFO and relevant status bits. */
+		i2cfif_cts = ioread8(bus->reg + NPCM_I2CFIF_CTS);
+		i2cfif_cts &= ~NPCM_I2CFIF_CTS_SLVRSTR;
+		i2cfif_cts |= NPCM_I2CFIF_CTS_CLR_FIFO;
+		iowrite8(i2cfif_cts, bus->reg + NPCM_I2CFIF_CTS);
+	}
+
+	bus->state = I2C_IDLE;
+	npcm_i2c_stall_after_start(bus, true);
+	npcm_i2c_master_start(bus);
+	return true;
+}
+
+static int npcm_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
+				int num)
+{
+	struct npcm_i2c *bus = container_of(adap, struct npcm_i2c, adap);
+	struct i2c_msg *msg0, *msg1;
+	unsigned long time_left, flags;
+	u16 nwrite, nread;
+	u8 *write_data, *read_data;
+	u8 slave_addr;
+	int timeout;
+	int ret = 0;
+	bool read_block = false;
+	bool read_PEC = false;
+	u8 bus_busy;
+	unsigned long timeout_usec;
+
+	if (bus->state == I2C_DISABLE) {
+		dev_err(bus->dev, "I2C%d module is disabled", bus->num);
+		return -EINVAL;
+	}
+
+	msg0 = &msgs[0];
+	slave_addr = msg0->addr;
+	if (msg0->flags & I2C_M_RD) { /* read */
+		nwrite = 0;
+		write_data = NULL;
+		read_data = msg0->buf;
+		if (msg0->flags & I2C_M_RECV_LEN) {
+			nread = 1;
+			read_block = true;
+			if (msg0->flags & I2C_CLIENT_PEC)
+				read_PEC = true;
+		} else {
+			nread = msg0->len;
+		}
+	} else { /* write */
+		nwrite = msg0->len;
+		write_data = msg0->buf;
+		nread = 0;
+		read_data = NULL;
+		if (num == 2) {
+			msg1 = &msgs[1];
+			read_data = msg1->buf;
+			if (msg1->flags & I2C_M_RECV_LEN) {
+				nread = 1;
+				read_block = true;
+				if (msg1->flags & I2C_CLIENT_PEC)
+					read_PEC = true;
+			} else {
+				nread = msg1->len;
+				read_block = false;
+			}
+		}
+	}
+
+	/* Adaptive TimeOut: astimated time in usec + 100% margin */
+	timeout_usec = (2 * 10000 / bus->bus_freq) * (2 + nread + nwrite);
+	timeout = max(msecs_to_jiffies(35), usecs_to_jiffies(timeout_usec));
+	if (nwrite >= 32 * 1024 || nread >= 32 * 1024) {
+		dev_err(bus->dev, "i2c%d buffer too big\n", bus->num);
+		return -EINVAL;
+	}
+
+	time_left = jiffies + msecs_to_jiffies(DEFAULT_STALL_COUNT) + 1;
+	do {
+		/*
+		 * we must clear slave address immediately when the bus is not
+		 * busy, so we spinlock it, but we don't keep the lock for the
+		 * entire while since it is too long.
+		 */
+		spin_lock_irqsave(&bus->lock, flags);
+		bus_busy = ioread8(bus->reg + NPCM_I2CCST) & NPCM_I2CCST_BB;
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+		if (!bus_busy && bus->slave)
+			iowrite8((bus->slave->addr & 0x7F),
+				 bus->reg + NPCM_I2CADDR1);
+#endif
+		spin_unlock_irqrestore(&bus->lock, flags);
+
+	} while (time_is_after_jiffies(time_left) && bus_busy);
+
+	if (bus_busy) {
+		iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST);
+		npcm_i2c_reset(bus);
+		i2c_recover_bus(adap);
+		return -EAGAIN;
+	}
+
+	npcm_i2c_init_params(bus);
+	bus->dest_addr = slave_addr;
+	bus->msgs = msgs;
+	bus->msgs_num = num;
+	bus->cmd_err = 0;
+	bus->read_block_use = read_block;
+
+	reinit_completion(&bus->cmd_complete);
+	if (!npcm_i2c_master_start_xmit(bus, slave_addr, nwrite, nread,
+					write_data, read_data, read_PEC,
+					read_block))
+		ret = -EBUSY;
+
+	if (ret != -EBUSY) {
+		time_left = wait_for_completion_timeout(&bus->cmd_complete,
+							timeout);
+
+		if (time_left == 0) {
+			if (bus->timeout_cnt < ULLONG_MAX)
+				bus->timeout_cnt++;
+			if (bus->master_or_slave == I2C_MASTER) {
+				i2c_recover_bus(adap);
+				bus->cmd_err = -EIO;
+				bus->state = I2C_IDLE;
+			}
+		}
+	}
+	ret = bus->cmd_err;
+
+	/* if there was BER, check if need to recover the bus: */
+	if (bus->cmd_err == -EAGAIN)
+		ret = i2c_recover_bus(adap);
+
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	/* reenable slave if it was enabled */
+	if (bus->slave)
+		iowrite8((bus->slave->addr & 0x7F) | NPCM_I2CADDR_SAEN,
+			 bus->reg + NPCM_I2CADDR1);
+#endif
+	return bus->cmd_err;
+}
+
+static u32 npcm_i2c_functionality(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C |
+	       I2C_FUNC_SMBUS_EMUL |
+	       I2C_FUNC_SMBUS_BLOCK_DATA |
+	       I2C_FUNC_SMBUS_PEC |
+	       I2C_FUNC_SLAVE;
+}
+
+static const struct i2c_adapter_quirks npcm_i2c_quirks = {
+	.max_read_len = 32768,
+	.max_write_len = 32768,
+	.flags = I2C_AQ_COMB_WRITE_THEN_READ,
+};
+
+static const struct i2c_algorithm npcm_i2c_algo = {
+	.master_xfer = npcm_i2c_master_xfer,
+	.functionality = npcm_i2c_functionality,
+#if IS_ENABLED(CONFIG_I2C_SLAVE)
+	.reg_slave	= npcm_i2c_reg_slave,
+	.unreg_slave	= npcm_i2c_unreg_slave,
+#endif
+};
+
+/* i2c debugfs directory: used to keep health monitor of i2c devices */
+static struct dentry *npcm_i2c_debugfs_dir;
+
+static void npcm_i2c_init_debugfs(struct platform_device *pdev,
+				  struct npcm_i2c *bus)
+{
+	struct dentry *d;
+
+	if (!npcm_i2c_debugfs_dir)
+		return;
+	d = debugfs_create_dir(dev_name(&pdev->dev), npcm_i2c_debugfs_dir);
+	if (IS_ERR_OR_NULL(d))
+		return;
+	debugfs_create_u64("ber_cnt", 0444, d, &bus->ber_cnt);
+	debugfs_create_u64("nack_cnt", 0444, d, &bus->nack_cnt);
+	debugfs_create_u64("rec_succ_cnt", 0444, d, &bus->rec_succ_cnt);
+	debugfs_create_u64("rec_fail_cnt", 0444, d, &bus->rec_fail_cnt);
+	debugfs_create_u64("timeout_cnt", 0444, d, &bus->timeout_cnt);
+
+	bus->debugfs = d;
+}
+
+static int npcm_i2c_probe_bus(struct platform_device *pdev)
+{
+	struct npcm_i2c *bus;
+	struct i2c_adapter *adap;
+	struct clk *i2c_clk;
+	static struct regmap *gcr_regmap;
+	static struct regmap *clk_regmap;
+	int irq;
+	int ret;
+
+	bus = devm_kzalloc(&pdev->dev, sizeof(*bus), GFP_KERNEL);
+	if (!bus)
+		return -ENOMEM;
+
+	bus->dev = &pdev->dev;
+
+	bus->num = of_alias_get_id(pdev->dev.of_node, "i2c");
+	/* core clk must be acquired to calculate module timing settings */
+	i2c_clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(i2c_clk))
+		return PTR_ERR(i2c_clk);
+	bus->apb_clk = clk_get_rate(i2c_clk);
+
+	gcr_regmap = syscon_regmap_lookup_by_compatible("nuvoton,npcm750-gcr");
+	if (IS_ERR(gcr_regmap))
+		return PTR_ERR(gcr_regmap);
+	regmap_write(gcr_regmap, NPCM_I2CSEGCTL, NPCM_I2CSEGCTL_INIT_VAL);
+
+	clk_regmap = syscon_regmap_lookup_by_compatible("nuvoton,npcm750-clk");
+	if (IS_ERR(clk_regmap))
+		return PTR_ERR(clk_regmap);
+
+	bus->reg = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(bus->reg))
+		return PTR_ERR(bus->reg);
+
+	spin_lock_init(&bus->lock);
+	init_completion(&bus->cmd_complete);
+
+	adap = &bus->adap;
+	adap->owner = THIS_MODULE;
+	adap->retries = 3;
+	adap->timeout = HZ;
+	adap->algo = &npcm_i2c_algo;
+	adap->quirks = &npcm_i2c_quirks;
+	adap->algo_data = bus;
+	adap->dev.parent = &pdev->dev;
+	adap->dev.of_node = pdev->dev.of_node;
+	adap->nr = pdev->id;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	ret = devm_request_irq(bus->dev, irq, npcm_i2c_bus_irq, 0,
+			       dev_name(bus->dev), bus);
+	if (ret)
+		return ret;
+
+	ret = __npcm_i2c_init(bus, pdev);
+	if (ret)
+		return ret;
+
+	npcm_i2c_recovery_init(adap);
+
+	i2c_set_adapdata(adap, bus);
+
+	snprintf(bus->adap.name, sizeof(bus->adap.name), "npcm_i2c_%d",
+		 bus->num);
+	ret = i2c_add_numbered_adapter(&bus->adap);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, bus);
+	npcm_i2c_init_debugfs(pdev, bus);
+	return 0;
+}
+
+static int npcm_i2c_remove_bus(struct platform_device *pdev)
+{
+	unsigned long lock_flags;
+	struct npcm_i2c *bus = platform_get_drvdata(pdev);
+
+	debugfs_remove_recursive(bus->debugfs);
+	spin_lock_irqsave(&bus->lock, lock_flags);
+	npcm_i2c_disable(bus);
+	spin_unlock_irqrestore(&bus->lock, lock_flags);
+	i2c_del_adapter(&bus->adap);
+	return 0;
+}
+
+static const struct of_device_id npcm_i2c_bus_of_table[] = {
+	{ .compatible = "nuvoton,npcm750-i2c", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, npcm_i2c_bus_of_table);
+
+static struct platform_driver npcm_i2c_bus_driver = {
+	.probe = npcm_i2c_probe_bus,
+	.remove = npcm_i2c_remove_bus,
+	.driver = {
+		.name = "nuvoton-i2c",
+		.of_match_table = npcm_i2c_bus_of_table,
+	}
+};
+
+static int __init npcm_i2c_init(void)
+{
+	npcm_i2c_debugfs_dir = debugfs_create_dir("npcm_i2c", NULL);
+	platform_driver_register(&npcm_i2c_bus_driver);
+	return 0;
+}
+module_init(npcm_i2c_init);
+
+static void __exit npcm_i2c_exit(void)
+{
+	platform_driver_unregister(&npcm_i2c_bus_driver);
+	debugfs_remove_recursive(npcm_i2c_debugfs_dir);
+}
+module_exit(npcm_i2c_exit);
+
+MODULE_AUTHOR("Avi Fishman <avi.fishman@gmail.com>");
+MODULE_AUTHOR("Tali Perry <tali.perry@nuvoton.com>");
+MODULE_AUTHOR("Tyrone Ting <kfting@nuvoton.com>");
+MODULE_DESCRIPTION("Nuvoton I2C Bus Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/i2c/busses/i2c-nvidia-gpu.c b/drivers/i2c/busses/i2c-nvidia-gpu.c
index f5d25ce00f03..f480105000b8 100644
--- a/drivers/i2c/busses/i2c-nvidia-gpu.c
+++ b/drivers/i2c/busses/i2c-nvidia-gpu.c
@@ -277,10 +277,7 @@ static int gpu_populate_client(struct gpu_i2c_dev *i2cd, int irq)
 	i2cd->gpu_ccgx_ucsi->irq = irq;
 	i2cd->gpu_ccgx_ucsi->properties = ccgx_props;
 	i2cd->ccgx_client = i2c_new_client_device(&i2cd->adapter, i2cd->gpu_ccgx_ucsi);
-	if (IS_ERR(i2cd->ccgx_client))
-		return PTR_ERR(i2cd->ccgx_client);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(i2cd->ccgx_client);
 }
 
 static int gpu_i2c_probe(struct pci_dev *pdev, const struct pci_device_id *id)
diff --git a/drivers/i2c/busses/i2c-octeon-platdrv.c b/drivers/i2c/busses/i2c-octeon-platdrv.c
index 64bda83e65ac..0c227963c8d6 100644
--- a/drivers/i2c/busses/i2c-octeon-platdrv.c
+++ b/drivers/i2c/busses/i2c-octeon-platdrv.c
@@ -136,7 +136,6 @@ static int octeon_i2c_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
 	int irq, result = 0, hlc_irq = 0;
-	struct resource *res_mem;
 	struct octeon_i2c *i2c;
 	bool cn78xx_style;
 
@@ -167,8 +166,7 @@ static int octeon_i2c_probe(struct platform_device *pdev)
 	i2c->roff.twsi_int = 0x10;
 	i2c->roff.sw_twsi_ext = 0x18;
 
-	res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->twsi_base = devm_ioremap_resource(&pdev->dev, res_mem);
+	i2c->twsi_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->twsi_base)) {
 		result = PTR_ERR(i2c->twsi_base);
 		goto out;
diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
index 71b4637c86b7..175c590b93b7 100644
--- a/drivers/i2c/busses/i2c-omap.c
+++ b/drivers/i2c/busses/i2c-omap.c
@@ -1365,10 +1365,8 @@ omap_i2c_probe(struct platform_device *pdev)
 	u16 minor, major;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "no irq resource?\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	omap = devm_kzalloc(&pdev->dev, sizeof(struct omap_i2c_dev), GFP_KERNEL);
 	if (!omap)
diff --git a/drivers/i2c/busses/i2c-owl.c b/drivers/i2c/busses/i2c-owl.c
index 3ab8be62c581..672f1f239bd6 100644
--- a/drivers/i2c/busses/i2c-owl.c
+++ b/drivers/i2c/busses/i2c-owl.c
@@ -396,23 +396,19 @@ static int owl_i2c_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct owl_i2c_dev *i2c_dev;
-	struct resource *res;
 	int ret, irq;
 
 	i2c_dev = devm_kzalloc(dev, sizeof(*i2c_dev), GFP_KERNEL);
 	if (!i2c_dev)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c_dev->base = devm_ioremap_resource(dev, res);
+	i2c_dev->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c_dev->base))
 		return PTR_ERR(i2c_dev->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(dev, "failed to get IRQ number\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	if (of_property_read_u32(dev->of_node, "clock-frequency",
 				 &i2c_dev->bus_freq))
diff --git a/drivers/i2c/busses/i2c-pca-platform.c b/drivers/i2c/busses/i2c-pca-platform.c
index 635dd697ac0b..546426a470cc 100644
--- a/drivers/i2c/busses/i2c-pca-platform.c
+++ b/drivers/i2c/busses/i2c-pca-platform.c
@@ -149,8 +149,7 @@ static int i2c_pca_pf_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->reg_base = devm_ioremap_resource(&pdev->dev, res);
+	i2c->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(i2c->reg_base))
 		return PTR_ERR(i2c->reg_base);
 
diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c
index 30ded6422e7b..69740a4ff1db 100644
--- a/drivers/i2c/busses/i2c-piix4.c
+++ b/drivers/i2c/busses/i2c-piix4.c
@@ -977,7 +977,8 @@ static int piix4_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	}
 
 	if (dev->vendor == PCI_VENDOR_ID_AMD &&
-	    dev->device == PCI_DEVICE_ID_AMD_HUDSON2_SMBUS) {
+	    (dev->device == PCI_DEVICE_ID_AMD_HUDSON2_SMBUS ||
+	     dev->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS)) {
 		retval = piix4_setup_sb800(dev, id, 1);
 	}
 
diff --git a/drivers/i2c/busses/i2c-pnx.c b/drivers/i2c/busses/i2c-pnx.c
index 686c06f31625..5d7207c10f1d 100644
--- a/drivers/i2c/busses/i2c-pnx.c
+++ b/drivers/i2c/busses/i2c-pnx.c
@@ -720,7 +720,6 @@ static int i2c_pnx_probe(struct platform_device *pdev)
 
 	alg_data->irq = platform_get_irq(pdev, 0);
 	if (alg_data->irq < 0) {
-		dev_err(&pdev->dev, "Failed to get IRQ from platform resource\n");
 		ret = alg_data->irq;
 		goto out_clock;
 	}
diff --git a/drivers/i2c/busses/i2c-powermac.c b/drivers/i2c/busses/i2c-powermac.c
index d565714c1f13..3e38e114948b 100644
--- a/drivers/i2c/busses/i2c-powermac.c
+++ b/drivers/i2c/busses/i2c-powermac.c
@@ -207,18 +207,18 @@ static u32 i2c_powermac_get_addr(struct i2c_adapter *adap,
 					   struct pmac_i2c_bus *bus,
 					   struct device_node *node)
 {
-	const __be32 *prop;
-	int len;
+	u32 prop;
+	int ret;
 
 	/* First check for valid "reg" */
-	prop = of_get_property(node, "reg", &len);
-	if (prop && (len >= sizeof(int)))
-		return (be32_to_cpup(prop) & 0xff) >> 1;
+	ret = of_property_read_u32(node, "reg", &prop);
+	if (ret == 0)
+		return (prop & 0xff) >> 1;
 
 	/* Then check old-style "i2c-address" */
-	prop = of_get_property(node, "i2c-address", &len);
-	if (prop && (len >= sizeof(int)))
-		return (be32_to_cpup(prop) & 0xff) >> 1;
+	ret = of_property_read_u32(node, "i2c-address", &prop);
+	if (ret == 0)
+		return (prop & 0xff) >> 1;
 
 	/* Now handle some devices with missing "reg" properties */
 	if (of_node_name_eq(node, "cereal"))
@@ -315,7 +315,7 @@ static void i2c_powermac_register_devices(struct i2c_adapter *adap,
 {
 	struct i2c_client *newdev;
 	struct device_node *node;
-	bool found_onyx = 0;
+	bool found_onyx = false;
 
 	/*
 	 * In some cases we end up with the via-pmu node itself, in this
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index 466e4f681d7a..35ca2c02c9b9 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -16,25 +16,120 @@
  *    Dec 2004: Added support for PXA27x and slave device probing [Liam Girdwood]
  *    Feb 2005: Rework slave mode handling [RMK]
  */
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/i2c.h>
-#include <linux/init.h>
-#include <linux/time.h>
-#include <linux/sched.h>
+#include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/err.h>
 #include <linux/errno.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
 #include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
-#include <linux/err.h>
-#include <linux/clk.h>
-#include <linux/slab.h>
-#include <linux/io.h>
 #include <linux/platform_data/i2c-pxa.h>
+#include <linux/slab.h>
+
+/* I2C register field definitions */
+#define IBMR_SDAS	(1 << 0)
+#define IBMR_SCLS	(1 << 1)
+
+#define ICR_START	(1 << 0)	   /* start bit */
+#define ICR_STOP	(1 << 1)	   /* stop bit */
+#define ICR_ACKNAK	(1 << 2)	   /* send ACK(0) or NAK(1) */
+#define ICR_TB		(1 << 3)	   /* transfer byte bit */
+#define ICR_MA		(1 << 4)	   /* master abort */
+#define ICR_SCLE	(1 << 5)	   /* master clock enable */
+#define ICR_IUE		(1 << 6)	   /* unit enable */
+#define ICR_GCD		(1 << 7)	   /* general call disable */
+#define ICR_ITEIE	(1 << 8)	   /* enable tx interrupts */
+#define ICR_IRFIE	(1 << 9)	   /* enable rx interrupts */
+#define ICR_BEIE	(1 << 10)	   /* enable bus error ints */
+#define ICR_SSDIE	(1 << 11)	   /* slave STOP detected int enable */
+#define ICR_ALDIE	(1 << 12)	   /* enable arbitration interrupt */
+#define ICR_SADIE	(1 << 13)	   /* slave address detected int enable */
+#define ICR_UR		(1 << 14)	   /* unit reset */
+#define ICR_FM		(1 << 15)	   /* fast mode */
+#define ICR_HS		(1 << 16)	   /* High Speed mode */
+#define ICR_A3700_FM	(1 << 16)	   /* fast mode for armada-3700 */
+#define ICR_A3700_HS	(1 << 17)	   /* high speed mode for armada-3700 */
+#define ICR_GPIOEN	(1 << 19)	   /* enable GPIO mode for SCL in HS */
+
+#define ISR_RWM		(1 << 0)	   /* read/write mode */
+#define ISR_ACKNAK	(1 << 1)	   /* ack/nak status */
+#define ISR_UB		(1 << 2)	   /* unit busy */
+#define ISR_IBB		(1 << 3)	   /* bus busy */
+#define ISR_SSD		(1 << 4)	   /* slave stop detected */
+#define ISR_ALD		(1 << 5)	   /* arbitration loss detected */
+#define ISR_ITE		(1 << 6)	   /* tx buffer empty */
+#define ISR_IRF		(1 << 7)	   /* rx buffer full */
+#define ISR_GCAD	(1 << 8)	   /* general call address detected */
+#define ISR_SAD		(1 << 9)	   /* slave address detected */
+#define ISR_BED		(1 << 10)	   /* bus error no ACK/NAK */
+
+#define ILCR_SLV_SHIFT		0
+#define ILCR_SLV_MASK		(0x1FF << ILCR_SLV_SHIFT)
+#define ILCR_FLV_SHIFT		9
+#define ILCR_FLV_MASK		(0x1FF << ILCR_FLV_SHIFT)
+#define ILCR_HLVL_SHIFT		18
+#define ILCR_HLVL_MASK		(0x1FF << ILCR_HLVL_SHIFT)
+#define ILCR_HLVH_SHIFT		27
+#define ILCR_HLVH_MASK		(0x1F << ILCR_HLVH_SHIFT)
+
+#define IWCR_CNT_SHIFT		0
+#define IWCR_CNT_MASK		(0x1F << IWCR_CNT_SHIFT)
+#define IWCR_HS_CNT1_SHIFT	5
+#define IWCR_HS_CNT1_MASK	(0x1F << IWCR_HS_CNT1_SHIFT)
+#define IWCR_HS_CNT2_SHIFT	10
+#define IWCR_HS_CNT2_MASK	(0x1F << IWCR_HS_CNT2_SHIFT)
+
+/* need a longer timeout if we're dealing with the fact we may well be
+ * looking at a multi-master environment
+ */
+#define DEF_TIMEOUT             32
+
+#define NO_SLAVE		(-ENXIO)
+#define BUS_ERROR               (-EREMOTEIO)
+#define XFER_NAKED              (-ECONNREFUSED)
+#define I2C_RETRY               (-2000) /* an error has occurred retry transmit */
+
+/* ICR initialize bit values
+ *
+ * 15 FM     0 (100 kHz operation)
+ * 14 UR     0 (No unit reset)
+ * 13 SADIE  0 (Disables the unit from interrupting on slave addresses
+ *              matching its slave address)
+ * 12 ALDIE  0 (Disables the unit from interrupt when it loses arbitration
+ *              in master mode)
+ * 11 SSDIE  0 (Disables interrupts from a slave stop detected, in slave mode)
+ * 10 BEIE   1 (Enable interrupts from detected bus errors, no ACK sent)
+ *  9 IRFIE  1 (Enable interrupts from full buffer received)
+ *  8 ITEIE  1 (Enables the I2C unit to interrupt when transmit buffer empty)
+ *  7 GCD    1 (Disables i2c unit response to general call messages as a slave)
+ *  6 IUE    0 (Disable unit until we change settings)
+ *  5 SCLE   1 (Enables the i2c clock output for master mode (drives SCL)
+ *  4 MA     0 (Only send stop with the ICR stop bit)
+ *  3 TB     0 (We are not transmitting a byte initially)
+ *  2 ACKNAK 0 (Send an ACK after the unit receives a byte)
+ *  1 STOP   0 (Do not send a STOP)
+ *  0 START  0 (Do not send a START)
+ */
+#define I2C_ICR_INIT	(ICR_BEIE | ICR_IRFIE | ICR_ITEIE | ICR_GCD | ICR_SCLE)
 
-#include <asm/irq.h>
+/* I2C status register init values
+ *
+ * 10 BED    1 (Clear bus error detected)
+ *  9 SAD    1 (Clear slave address detected)
+ *  7 IRF    1 (Clear IDBR Receive Full)
+ *  6 ITE    1 (Clear IDBR Transmit Empty)
+ *  5 ALD    1 (Clear Arbitration Loss Detected)
+ *  4 SSD    1 (Clear Slave Stop Detected)
+ */
+#define I2C_ISR_INIT	0x7FF  /* status register init */
 
 struct pxa_reg_layout {
 	u32 ibmr;
@@ -56,12 +151,7 @@ enum pxa_i2c_types {
 	REGS_A3700,
 };
 
-#define ICR_BUSMODE_FM	(1 << 16)	   /* shifted fast mode for armada-3700 */
-#define ICR_BUSMODE_HS	(1 << 17)	   /* shifted high speed mode for armada-3700 */
-
-/*
- * I2C registers definitions
- */
+/* I2C register layout definitions */
 static struct pxa_reg_layout pxa_reg_layout[] = {
 	[REGS_PXA2XX] = {
 		.ibmr =	0x00,
@@ -69,6 +159,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = {
 		.icr =	0x10,
 		.isr =	0x18,
 		.isar =	0x20,
+		.fm = ICR_FM,
+		.hs = ICR_HS,
 	},
 	[REGS_PXA3XX] = {
 		.ibmr =	0x00,
@@ -76,6 +168,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = {
 		.icr =	0x08,
 		.isr =	0x0c,
 		.isar =	0x10,
+		.fm = ICR_FM,
+		.hs = ICR_HS,
 	},
 	[REGS_CE4100] = {
 		.ibmr =	0x14,
@@ -83,6 +177,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = {
 		.icr =	0x00,
 		.isr =	0x04,
 		/* no isar register */
+		.fm = ICR_FM,
+		.hs = ICR_HS,
 	},
 	[REGS_PXA910] = {
 		.ibmr = 0x00,
@@ -92,6 +188,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = {
 		.isar = 0x20,
 		.ilcr = 0x28,
 		.iwcr = 0x30,
+		.fm = ICR_FM,
+		.hs = ICR_HS,
 	},
 	[REGS_A3700] = {
 		.ibmr =	0x00,
@@ -99,11 +197,20 @@ static struct pxa_reg_layout pxa_reg_layout[] = {
 		.icr =	0x08,
 		.isr =	0x0c,
 		.isar =	0x10,
-		.fm = ICR_BUSMODE_FM,
-		.hs = ICR_BUSMODE_HS,
+		.fm = ICR_A3700_FM,
+		.hs = ICR_A3700_HS,
 	},
 };
 
+static const struct of_device_id i2c_pxa_dt_ids[] = {
+	{ .compatible = "mrvl,pxa-i2c", .data = (void *)REGS_PXA2XX },
+	{ .compatible = "mrvl,pwri2c", .data = (void *)REGS_PXA3XX },
+	{ .compatible = "mrvl,mmp-twsi", .data = (void *)REGS_PXA910 },
+	{ .compatible = "marvell,armada-3700-i2c", .data = (void *)REGS_A3700 },
+	{}
+};
+MODULE_DEVICE_TABLE(of, i2c_pxa_dt_ids);
+
 static const struct platform_device_id i2c_pxa_id_table[] = {
 	{ "pxa2xx-i2c",		REGS_PXA2XX },
 	{ "pxa3xx-pwri2c",	REGS_PXA3XX },
@@ -114,58 +221,6 @@ static const struct platform_device_id i2c_pxa_id_table[] = {
 };
 MODULE_DEVICE_TABLE(platform, i2c_pxa_id_table);
 
-/*
- * I2C bit definitions
- */
-
-#define ICR_START	(1 << 0)	   /* start bit */
-#define ICR_STOP	(1 << 1)	   /* stop bit */
-#define ICR_ACKNAK	(1 << 2)	   /* send ACK(0) or NAK(1) */
-#define ICR_TB		(1 << 3)	   /* transfer byte bit */
-#define ICR_MA		(1 << 4)	   /* master abort */
-#define ICR_SCLE	(1 << 5)	   /* master clock enable */
-#define ICR_IUE		(1 << 6)	   /* unit enable */
-#define ICR_GCD		(1 << 7)	   /* general call disable */
-#define ICR_ITEIE	(1 << 8)	   /* enable tx interrupts */
-#define ICR_IRFIE	(1 << 9)	   /* enable rx interrupts */
-#define ICR_BEIE	(1 << 10)	   /* enable bus error ints */
-#define ICR_SSDIE	(1 << 11)	   /* slave STOP detected int enable */
-#define ICR_ALDIE	(1 << 12)	   /* enable arbitration interrupt */
-#define ICR_SADIE	(1 << 13)	   /* slave address detected int enable */
-#define ICR_UR		(1 << 14)	   /* unit reset */
-#define ICR_FM		(1 << 15)	   /* fast mode */
-#define ICR_HS		(1 << 16)	   /* High Speed mode */
-#define ICR_GPIOEN	(1 << 19)	   /* enable GPIO mode for SCL in HS */
-
-#define ISR_RWM		(1 << 0)	   /* read/write mode */
-#define ISR_ACKNAK	(1 << 1)	   /* ack/nak status */
-#define ISR_UB		(1 << 2)	   /* unit busy */
-#define ISR_IBB		(1 << 3)	   /* bus busy */
-#define ISR_SSD		(1 << 4)	   /* slave stop detected */
-#define ISR_ALD		(1 << 5)	   /* arbitration loss detected */
-#define ISR_ITE		(1 << 6)	   /* tx buffer empty */
-#define ISR_IRF		(1 << 7)	   /* rx buffer full */
-#define ISR_GCAD	(1 << 8)	   /* general call address detected */
-#define ISR_SAD		(1 << 9)	   /* slave address detected */
-#define ISR_BED		(1 << 10)	   /* bus error no ACK/NAK */
-
-/* bit field shift & mask */
-#define ILCR_SLV_SHIFT		0
-#define ILCR_SLV_MASK		(0x1FF << ILCR_SLV_SHIFT)
-#define ILCR_FLV_SHIFT		9
-#define ILCR_FLV_MASK		(0x1FF << ILCR_FLV_SHIFT)
-#define ILCR_HLVL_SHIFT		18
-#define ILCR_HLVL_MASK		(0x1FF << ILCR_HLVL_SHIFT)
-#define ILCR_HLVH_SHIFT		27
-#define ILCR_HLVH_MASK		(0x1F << ILCR_HLVH_SHIFT)
-
-#define IWCR_CNT_SHIFT		0
-#define IWCR_CNT_MASK		(0x1F << IWCR_CNT_SHIFT)
-#define IWCR_HS_CNT1_SHIFT	5
-#define IWCR_HS_CNT1_MASK	(0x1F << IWCR_HS_CNT1_SHIFT)
-#define IWCR_HS_CNT2_SHIFT	10
-#define IWCR_HS_CNT2_MASK	(0x1F << IWCR_HS_CNT2_SHIFT)
-
 struct pxa_i2c {
 	spinlock_t		lock;
 	wait_queue_head_t	wait;
@@ -207,6 +262,11 @@ struct pxa_i2c {
 	bool			highmode_enter;
 	u32			fm_mask;
 	u32			hs_mask;
+
+	struct i2c_bus_recovery_info recovery;
+	struct pinctrl		*pinctrl;
+	struct pinctrl_state	*pinctrl_default;
+	struct pinctrl_state	*pinctrl_recovery;
 };
 
 #define _IBMR(i2c)	((i2c)->reg_ibmr)
@@ -234,13 +294,14 @@ struct bits {
 static inline void
 decode_bits(const char *prefix, const struct bits *bits, int num, u32 val)
 {
-	printk("%s %08x: ", prefix, val);
+	printk("%s %08x:", prefix, val);
 	while (num--) {
 		const char *str = val & bits->mask ? bits->set : bits->unset;
 		if (str)
-			printk("%s ", str);
+			pr_cont(" %s", str);
 		bits++;
 	}
+	pr_cont("\n");
 }
 
 static const struct bits isr_bits[] = {
@@ -260,7 +321,6 @@ static const struct bits isr_bits[] = {
 static void decode_ISR(unsigned int val)
 {
 	decode_bits(KERN_DEBUG "ISR", isr_bits, ARRAY_SIZE(isr_bits), val);
-	printk("\n");
 }
 
 static const struct bits icr_bits[] = {
@@ -285,7 +345,6 @@ static const struct bits icr_bits[] = {
 static void decode_ICR(unsigned int val)
 {
 	decode_bits(KERN_DEBUG "ICR", icr_bits, ARRAY_SIZE(icr_bits), val);
-	printk("\n");
 }
 #endif
 
@@ -311,11 +370,10 @@ static void i2c_pxa_scream_blue_murder(struct pxa_i2c *i2c, const char *why)
 	dev_err(dev, "IBMR: %08x IDBR: %08x ICR: %08x ISR: %08x\n",
 		readl(_IBMR(i2c)), readl(_IDBR(i2c)), readl(_ICR(i2c)),
 		readl(_ISR(i2c)));
-	dev_dbg(dev, "log: ");
+	dev_err(dev, "log:");
 	for (i = 0; i < i2c->irqlogidx; i++)
-		pr_debug("[%08x:%08x] ", i2c->isrlog[i], i2c->icrlog[i]);
-
-	pr_debug("\n");
+		pr_cont(" [%03x:%05x]", i2c->isrlog[i], i2c->icrlog[i]);
+	pr_cont("\n");
 }
 
 #else /* ifdef DEBUG */
@@ -330,7 +388,6 @@ static void i2c_pxa_scream_blue_murder(struct pxa_i2c *i2c, const char *why)
 #endif /* ifdef DEBUG / else */
 
 static void i2c_pxa_master_complete(struct pxa_i2c *i2c, int ret);
-static irqreturn_t i2c_pxa_handler(int this_irq, void *dev_id);
 
 static inline int i2c_pxa_is_slavemode(struct pxa_i2c *i2c)
 {
@@ -346,7 +403,7 @@ static void i2c_pxa_abort(struct pxa_i2c *i2c)
 		return;
 	}
 
-	while ((i > 0) && (readl(_IBMR(i2c)) & 0x1) == 0) {
+	while ((i > 0) && (readl(_IBMR(i2c)) & IBMR_SDAS) == 0) {
 		unsigned long icr = readl(_ICR(i2c));
 
 		icr &= ~ICR_START;
@@ -367,19 +424,26 @@ static void i2c_pxa_abort(struct pxa_i2c *i2c)
 static int i2c_pxa_wait_bus_not_busy(struct pxa_i2c *i2c)
 {
 	int timeout = DEF_TIMEOUT;
+	u32 isr;
+
+	while (1) {
+		isr = readl(_ISR(i2c));
+		if (!(isr & (ISR_IBB | ISR_UB)))
+			return 0;
 
-	while (timeout-- && readl(_ISR(i2c)) & (ISR_IBB | ISR_UB)) {
-		if ((readl(_ISR(i2c)) & ISR_SAD) != 0)
+		if (isr & ISR_SAD)
 			timeout += 4;
 
+		if (!timeout--)
+			break;
+
 		msleep(2);
 		show_state(i2c);
 	}
 
-	if (timeout < 0)
-		show_state(i2c);
+	show_state(i2c);
 
-	return timeout < 0 ? I2C_RETRY : 0;
+	return I2C_RETRY;
 }
 
 static int i2c_pxa_wait_master(struct pxa_i2c *i2c)
@@ -401,7 +465,8 @@ static int i2c_pxa_wait_master(struct pxa_i2c *i2c)
 		 * quick check of the i2c lines themselves to ensure they've
 		 * gone high...
 		 */
-		if ((readl(_ISR(i2c)) & (ISR_UB | ISR_IBB)) == 0 && readl(_IBMR(i2c)) == 3) {
+		if ((readl(_ISR(i2c)) & (ISR_UB | ISR_IBB)) == 0 &&
+		    readl(_IBMR(i2c)) == (IBMR_SCLS | IBMR_SDAS)) {
 			if (i2c_debug > 0)
 				dev_dbg(&i2c->adap.dev, "%s: done\n", __func__);
 			return 1;
@@ -501,13 +566,8 @@ static void i2c_pxa_set_slave(struct pxa_i2c *i2c, int errcode)
 #define i2c_pxa_set_slave(i2c, err)	do { } while (0)
 #endif
 
-static void i2c_pxa_reset(struct pxa_i2c *i2c)
+static void i2c_pxa_do_reset(struct pxa_i2c *i2c)
 {
-	pr_debug("Resetting I2C Controller Unit\n");
-
-	/* abort any transfer currently under way */
-	i2c_pxa_abort(i2c);
-
 	/* reset according to 9.8 */
 	writel(ICR_UR, _ICR(i2c));
 	writel(I2C_ISR_INIT, _ISR(i2c));
@@ -526,12 +586,25 @@ static void i2c_pxa_reset(struct pxa_i2c *i2c)
 #endif
 
 	i2c_pxa_set_slave(i2c, 0);
+}
 
+static void i2c_pxa_enable(struct pxa_i2c *i2c)
+{
 	/* enable unit */
 	writel(readl(_ICR(i2c)) | ICR_IUE, _ICR(i2c));
 	udelay(100);
 }
 
+static void i2c_pxa_reset(struct pxa_i2c *i2c)
+{
+	pr_debug("Resetting I2C Controller Unit\n");
+
+	/* abort any transfer currently under way */
+	i2c_pxa_abort(i2c);
+	i2c_pxa_do_reset(i2c);
+	i2c_pxa_enable(i2c);
+}
+
 
 #ifdef CONFIG_I2C_PXA_SLAVE
 /*
@@ -596,7 +669,7 @@ static void i2c_pxa_slave_start(struct pxa_i2c *i2c, u32 isr)
 	timeout = 0x10000;
 
 	while (1) {
-		if ((readl(_IBMR(i2c)) & 2) == 2)
+		if ((readl(_IBMR(i2c)) & IBMR_SCLS) == IBMR_SCLS)
 			break;
 
 		timeout--;
@@ -691,7 +764,7 @@ static void i2c_pxa_slave_start(struct pxa_i2c *i2c, u32 isr)
 	timeout = 0x10000;
 
 	while (1) {
-		if ((readl(_IBMR(i2c)) & 2) == 2)
+		if ((readl(_IBMR(i2c)) & IBMR_SCLS) == IBMR_SCLS)
 			break;
 
 		timeout--;
@@ -716,16 +789,6 @@ static void i2c_pxa_slave_stop(struct pxa_i2c *i2c)
  * PXA I2C Master mode
  */
 
-static inline unsigned int i2c_pxa_addr_byte(struct i2c_msg *msg)
-{
-	unsigned int addr = (msg->addr & 0x7f) << 1;
-
-	if (msg->flags & I2C_M_RD)
-		addr |= 1;
-
-	return addr;
-}
-
 static inline void i2c_pxa_start_message(struct pxa_i2c *i2c)
 {
 	u32 icr;
@@ -733,8 +796,8 @@ static inline void i2c_pxa_start_message(struct pxa_i2c *i2c)
 	/*
 	 * Step 1: target slave address into IDBR
 	 */
-	writel(i2c_pxa_addr_byte(i2c->msg), _IDBR(i2c));
-	i2c->req_slave_addr = i2c_pxa_addr_byte(i2c->msg);
+	i2c->req_slave_addr = i2c_8bit_addr_from_msg(i2c->msg);
+	writel(i2c->req_slave_addr, _IDBR(i2c));
 
 	/*
 	 * Step 2: initiate the write.
@@ -747,42 +810,12 @@ static inline void i2c_pxa_stop_message(struct pxa_i2c *i2c)
 {
 	u32 icr;
 
-	/*
-	 * Clear the STOP and ACK flags
-	 */
+	/* Clear the START, STOP, ACK, TB and MA flags */
 	icr = readl(_ICR(i2c));
-	icr &= ~(ICR_STOP | ICR_ACKNAK);
+	icr &= ~(ICR_START | ICR_STOP | ICR_ACKNAK | ICR_TB | ICR_MA);
 	writel(icr, _ICR(i2c));
 }
 
-static int i2c_pxa_pio_set_master(struct pxa_i2c *i2c)
-{
-	/* make timeout the same as for interrupt based functions */
-	long timeout = 2 * DEF_TIMEOUT;
-
-	/*
-	 * Wait for the bus to become free.
-	 */
-	while (timeout-- && readl(_ISR(i2c)) & (ISR_IBB | ISR_UB)) {
-		udelay(1000);
-		show_state(i2c);
-	}
-
-	if (timeout < 0) {
-		show_state(i2c);
-		dev_err(&i2c->adap.dev,
-			"i2c_pxa: timeout waiting for bus free\n");
-		return I2C_RETRY;
-	}
-
-	/*
-	 * Set master mode.
-	 */
-	writel(readl(_ICR(i2c)) | ICR_SCLE, _ICR(i2c));
-
-	return 0;
-}
-
 /*
  * PXA I2C send master code
  * 1. Load master code to IDBR and send it.
@@ -811,140 +844,6 @@ static int i2c_pxa_send_mastercode(struct pxa_i2c *i2c)
 	return (timeout == 0) ? I2C_RETRY : 0;
 }
 
-static int i2c_pxa_do_pio_xfer(struct pxa_i2c *i2c,
-			       struct i2c_msg *msg, int num)
-{
-	unsigned long timeout = 500000; /* 5 seconds */
-	int ret = 0;
-
-	ret = i2c_pxa_pio_set_master(i2c);
-	if (ret)
-		goto out;
-
-	i2c->msg = msg;
-	i2c->msg_num = num;
-	i2c->msg_idx = 0;
-	i2c->msg_ptr = 0;
-	i2c->irqlogidx = 0;
-
-	i2c_pxa_start_message(i2c);
-
-	while (i2c->msg_num > 0 && --timeout) {
-		i2c_pxa_handler(0, i2c);
-		udelay(10);
-	}
-
-	i2c_pxa_stop_message(i2c);
-
-	/*
-	 * We place the return code in i2c->msg_idx.
-	 */
-	ret = i2c->msg_idx;
-
-out:
-	if (timeout == 0) {
-		i2c_pxa_scream_blue_murder(i2c, "timeout");
-		ret = I2C_RETRY;
-	}
-
-	return ret;
-}
-
-/*
- * We are protected by the adapter bus mutex.
- */
-static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num)
-{
-	long timeout;
-	int ret;
-
-	/*
-	 * Wait for the bus to become free.
-	 */
-	ret = i2c_pxa_wait_bus_not_busy(i2c);
-	if (ret) {
-		dev_err(&i2c->adap.dev, "i2c_pxa: timeout waiting for bus free\n");
-		goto out;
-	}
-
-	/*
-	 * Set master mode.
-	 */
-	ret = i2c_pxa_set_master(i2c);
-	if (ret) {
-		dev_err(&i2c->adap.dev, "i2c_pxa_set_master: error %d\n", ret);
-		goto out;
-	}
-
-	if (i2c->high_mode) {
-		ret = i2c_pxa_send_mastercode(i2c);
-		if (ret) {
-			dev_err(&i2c->adap.dev, "i2c_pxa_send_mastercode timeout\n");
-			goto out;
-			}
-	}
-
-	spin_lock_irq(&i2c->lock);
-
-	i2c->msg = msg;
-	i2c->msg_num = num;
-	i2c->msg_idx = 0;
-	i2c->msg_ptr = 0;
-	i2c->irqlogidx = 0;
-
-	i2c_pxa_start_message(i2c);
-
-	spin_unlock_irq(&i2c->lock);
-
-	/*
-	 * The rest of the processing occurs in the interrupt handler.
-	 */
-	timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5);
-	i2c_pxa_stop_message(i2c);
-
-	/*
-	 * We place the return code in i2c->msg_idx.
-	 */
-	ret = i2c->msg_idx;
-
-	if (!timeout && i2c->msg_num) {
-		i2c_pxa_scream_blue_murder(i2c, "timeout");
-		ret = I2C_RETRY;
-	}
-
- out:
-	return ret;
-}
-
-static int i2c_pxa_pio_xfer(struct i2c_adapter *adap,
-			    struct i2c_msg msgs[], int num)
-{
-	struct pxa_i2c *i2c = adap->algo_data;
-	int ret, i;
-
-	/* If the I2C controller is disabled we need to reset it
-	  (probably due to a suspend/resume destroying state). We do
-	  this here as we can then avoid worrying about resuming the
-	  controller before its users. */
-	if (!(readl(_ICR(i2c)) & ICR_IUE))
-		i2c_pxa_reset(i2c);
-
-	for (i = adap->retries; i >= 0; i--) {
-		ret = i2c_pxa_do_pio_xfer(i2c, msgs, num);
-		if (ret != I2C_RETRY)
-			goto out;
-
-		if (i2c_debug)
-			dev_dbg(&adap->dev, "Retrying transmission\n");
-		udelay(100);
-	}
-	i2c_pxa_scream_blue_murder(i2c, "exhausted retries");
-	ret = -EREMOTEIO;
- out:
-	i2c_pxa_set_slave(i2c, ret);
-	return ret;
-}
-
 /*
  * i2c_pxa_master_complete - complete the message and wake up.
  */
@@ -996,7 +895,7 @@ static void i2c_pxa_irq_txempty(struct pxa_i2c *i2c, u32 isr)
 		 */
 		if (isr & ISR_ACKNAK) {
 			if (i2c->msg_ptr == 0 && i2c->msg_idx == 0)
-				ret = I2C_RETRY;
+				ret = NO_SLAVE;
 			else
 				ret = XFER_NAKED;
 		}
@@ -1047,8 +946,8 @@ static void i2c_pxa_irq_txempty(struct pxa_i2c *i2c, u32 isr)
 		/*
 		 * Write the next address.
 		 */
-		writel(i2c_pxa_addr_byte(i2c->msg), _IDBR(i2c));
-		i2c->req_slave_addr = i2c_pxa_addr_byte(i2c->msg);
+		i2c->req_slave_addr = i2c_8bit_addr_from_msg(i2c->msg);
+		writel(i2c->req_slave_addr, _IDBR(i2c));
 
 		/*
 		 * And trigger a repeated start, and send the byte.
@@ -1056,14 +955,8 @@ static void i2c_pxa_irq_txempty(struct pxa_i2c *i2c, u32 isr)
 		icr &= ~ICR_ALDIE;
 		icr |= ICR_START | ICR_TB;
 	} else {
-		if (i2c->msg->len == 0) {
-			/*
-			 * Device probes have a message length of zero
-			 * and need the bus to be reset before it can
-			 * be used again.
-			 */
-			i2c_pxa_reset(i2c);
-		}
+		if (i2c->msg->len == 0)
+			icr |= ICR_MA;
 		i2c_pxa_master_complete(i2c, 0);
 	}
 
@@ -1151,28 +1044,108 @@ static irqreturn_t i2c_pxa_handler(int this_irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+/*
+ * We are protected by the adapter bus mutex.
+ */
+static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num)
+{
+	long timeout;
+	int ret;
+
+	/*
+	 * Wait for the bus to become free.
+	 */
+	ret = i2c_pxa_wait_bus_not_busy(i2c);
+	if (ret) {
+		dev_err(&i2c->adap.dev, "i2c_pxa: timeout waiting for bus free\n");
+		i2c_recover_bus(&i2c->adap);
+		goto out;
+	}
+
+	/*
+	 * Set master mode.
+	 */
+	ret = i2c_pxa_set_master(i2c);
+	if (ret) {
+		dev_err(&i2c->adap.dev, "i2c_pxa_set_master: error %d\n", ret);
+		goto out;
+	}
+
+	if (i2c->high_mode) {
+		ret = i2c_pxa_send_mastercode(i2c);
+		if (ret) {
+			dev_err(&i2c->adap.dev, "i2c_pxa_send_mastercode timeout\n");
+			goto out;
+			}
+	}
+
+	spin_lock_irq(&i2c->lock);
+
+	i2c->msg = msg;
+	i2c->msg_num = num;
+	i2c->msg_idx = 0;
+	i2c->msg_ptr = 0;
+	i2c->irqlogidx = 0;
+
+	i2c_pxa_start_message(i2c);
+
+	spin_unlock_irq(&i2c->lock);
+
+	/*
+	 * The rest of the processing occurs in the interrupt handler.
+	 */
+	timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5);
+	i2c_pxa_stop_message(i2c);
+
+	/*
+	 * We place the return code in i2c->msg_idx.
+	 */
+	ret = i2c->msg_idx;
+
+	if (!timeout && i2c->msg_num) {
+		i2c_pxa_scream_blue_murder(i2c, "timeout with active message");
+		i2c_recover_bus(&i2c->adap);
+		ret = I2C_RETRY;
+	}
 
-static int i2c_pxa_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+ out:
+	return ret;
+}
+
+static int i2c_pxa_internal_xfer(struct pxa_i2c *i2c,
+				 struct i2c_msg *msgs, int num,
+				 int (*xfer)(struct pxa_i2c *,
+					     struct i2c_msg *, int num))
 {
-	struct pxa_i2c *i2c = adap->algo_data;
 	int ret, i;
 
-	for (i = adap->retries; i >= 0; i--) {
-		ret = i2c_pxa_do_xfer(i2c, msgs, num);
-		if (ret != I2C_RETRY)
+	for (i = 0; ; ) {
+		ret = xfer(i2c, msgs, num);
+		if (ret != I2C_RETRY && ret != NO_SLAVE)
 			goto out;
+		if (++i >= i2c->adap.retries)
+			break;
 
 		if (i2c_debug)
-			dev_dbg(&adap->dev, "Retrying transmission\n");
+			dev_dbg(&i2c->adap.dev, "Retrying transmission\n");
 		udelay(100);
 	}
-	i2c_pxa_scream_blue_murder(i2c, "exhausted retries");
+	if (ret != NO_SLAVE)
+		i2c_pxa_scream_blue_murder(i2c, "exhausted retries");
 	ret = -EREMOTEIO;
  out:
 	i2c_pxa_set_slave(i2c, ret);
 	return ret;
 }
 
+static int i2c_pxa_xfer(struct i2c_adapter *adap,
+			struct i2c_msg msgs[], int num)
+{
+	struct pxa_i2c *i2c = adap->algo_data;
+
+	return i2c_pxa_internal_xfer(i2c, msgs, num, i2c_pxa_do_xfer);
+}
+
 static u32 i2c_pxa_functionality(struct i2c_adapter *adap)
 {
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL |
@@ -1188,6 +1161,87 @@ static const struct i2c_algorithm i2c_pxa_algorithm = {
 #endif
 };
 
+/* Non-interrupt mode support */
+static int i2c_pxa_pio_set_master(struct pxa_i2c *i2c)
+{
+	/* make timeout the same as for interrupt based functions */
+	long timeout = 2 * DEF_TIMEOUT;
+
+	/*
+	 * Wait for the bus to become free.
+	 */
+	while (timeout-- && readl(_ISR(i2c)) & (ISR_IBB | ISR_UB))
+		udelay(1000);
+
+	if (timeout < 0) {
+		show_state(i2c);
+		dev_err(&i2c->adap.dev,
+			"i2c_pxa: timeout waiting for bus free (set_master)\n");
+		return I2C_RETRY;
+	}
+
+	/*
+	 * Set master mode.
+	 */
+	writel(readl(_ICR(i2c)) | ICR_SCLE, _ICR(i2c));
+
+	return 0;
+}
+
+static int i2c_pxa_do_pio_xfer(struct pxa_i2c *i2c,
+			       struct i2c_msg *msg, int num)
+{
+	unsigned long timeout = 500000; /* 5 seconds */
+	int ret = 0;
+
+	ret = i2c_pxa_pio_set_master(i2c);
+	if (ret)
+		goto out;
+
+	i2c->msg = msg;
+	i2c->msg_num = num;
+	i2c->msg_idx = 0;
+	i2c->msg_ptr = 0;
+	i2c->irqlogidx = 0;
+
+	i2c_pxa_start_message(i2c);
+
+	while (i2c->msg_num > 0 && --timeout) {
+		i2c_pxa_handler(0, i2c);
+		udelay(10);
+	}
+
+	i2c_pxa_stop_message(i2c);
+
+	/*
+	 * We place the return code in i2c->msg_idx.
+	 */
+	ret = i2c->msg_idx;
+
+out:
+	if (timeout == 0) {
+		i2c_pxa_scream_blue_murder(i2c, "timeout (do_pio_xfer)");
+		ret = I2C_RETRY;
+	}
+
+	return ret;
+}
+
+static int i2c_pxa_pio_xfer(struct i2c_adapter *adap,
+			    struct i2c_msg msgs[], int num)
+{
+	struct pxa_i2c *i2c = adap->algo_data;
+
+	/* If the I2C controller is disabled we need to reset it
+	  (probably due to a suspend/resume destroying state). We do
+	  this here as we can then avoid worrying about resuming the
+	  controller before its users. */
+	if (!(readl(_ICR(i2c)) & ICR_IUE))
+		i2c_pxa_reset(i2c);
+
+	return i2c_pxa_internal_xfer(i2c, msgs, num, i2c_pxa_do_pio_xfer);
+}
+
 static const struct i2c_algorithm i2c_pxa_pio_algorithm = {
 	.master_xfer	= i2c_pxa_pio_xfer,
 	.functionality	= i2c_pxa_functionality,
@@ -1197,15 +1251,6 @@ static const struct i2c_algorithm i2c_pxa_pio_algorithm = {
 #endif
 };
 
-static const struct of_device_id i2c_pxa_dt_ids[] = {
-	{ .compatible = "mrvl,pxa-i2c", .data = (void *)REGS_PXA2XX },
-	{ .compatible = "mrvl,pwri2c", .data = (void *)REGS_PXA3XX },
-	{ .compatible = "mrvl,mmp-twsi", .data = (void *)REGS_PXA910 },
-	{ .compatible = "marvell,armada-3700-i2c", .data = (void *)REGS_A3700 },
-	{}
-};
-MODULE_DEVICE_TABLE(of, i2c_pxa_dt_ids);
-
 static int i2c_pxa_probe_dt(struct platform_device *pdev, struct pxa_i2c *i2c,
 			    enum pxa_i2c_types *i2c_types)
 {
@@ -1249,6 +1294,131 @@ static int i2c_pxa_probe_pdata(struct platform_device *pdev,
 	return 0;
 }
 
+static void i2c_pxa_prepare_recovery(struct i2c_adapter *adap)
+{
+	struct pxa_i2c *i2c = adap->algo_data;
+	u32 ibmr = readl(_IBMR(i2c));
+
+	/*
+	 * Program the GPIOs to reflect the current I2C bus state while
+	 * we transition to recovery; this avoids glitching the bus.
+	 */
+	gpiod_set_value(i2c->recovery.scl_gpiod, ibmr & IBMR_SCLS);
+	gpiod_set_value(i2c->recovery.sda_gpiod, ibmr & IBMR_SDAS);
+
+	WARN_ON(pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_recovery));
+}
+
+static void i2c_pxa_unprepare_recovery(struct i2c_adapter *adap)
+{
+	struct pxa_i2c *i2c = adap->algo_data;
+	u32 isr;
+
+	/*
+	 * The bus should now be free. Clear up the I2C controller before
+	 * handing control of the bus back to avoid the bus changing state.
+	 */
+	isr = readl(_ISR(i2c));
+	if (isr & (ISR_UB | ISR_IBB)) {
+		dev_dbg(&i2c->adap.dev,
+			"recovery: resetting controller, ISR=0x%08x\n", isr);
+		i2c_pxa_do_reset(i2c);
+	}
+
+	WARN_ON(pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_default));
+
+	dev_dbg(&i2c->adap.dev, "recovery: IBMR 0x%08x ISR 0x%08x\n",
+	        readl(_IBMR(i2c)), readl(_ISR(i2c)));
+
+	i2c_pxa_enable(i2c);
+}
+
+static int i2c_pxa_init_recovery(struct pxa_i2c *i2c)
+{
+	struct i2c_bus_recovery_info *bri = &i2c->recovery;
+	struct device *dev = i2c->adap.dev.parent;
+
+	/*
+	 * When slave mode is enabled, we are not the only master on the bus.
+	 * Bus recovery can only be performed when we are the master, which
+	 * we can't be certain of. Therefore, when slave mode is enabled, do
+	 * not configure bus recovery.
+	 */
+	if (IS_ENABLED(CONFIG_I2C_PXA_SLAVE))
+		return 0;
+
+	i2c->pinctrl = devm_pinctrl_get(dev);
+	if (PTR_ERR(i2c->pinctrl) == -ENODEV)
+		i2c->pinctrl = NULL;
+	if (IS_ERR(i2c->pinctrl))
+		return PTR_ERR(i2c->pinctrl);
+
+	if (!i2c->pinctrl)
+		return 0;
+
+	i2c->pinctrl_default = pinctrl_lookup_state(i2c->pinctrl,
+						    PINCTRL_STATE_DEFAULT);
+	i2c->pinctrl_recovery = pinctrl_lookup_state(i2c->pinctrl, "recovery");
+
+	if (IS_ERR(i2c->pinctrl_default) || IS_ERR(i2c->pinctrl_recovery)) {
+		dev_info(dev, "missing pinmux recovery information: %ld %ld\n",
+			 PTR_ERR(i2c->pinctrl_default),
+			 PTR_ERR(i2c->pinctrl_recovery));
+		return 0;
+	}
+
+	/*
+	 * Claiming GPIOs can influence the pinmux state, and may glitch the
+	 * I2C bus. Do this carefully.
+	 */
+	bri->scl_gpiod = devm_gpiod_get(dev, "scl", GPIOD_OUT_HIGH_OPEN_DRAIN);
+	if (bri->scl_gpiod == ERR_PTR(-EPROBE_DEFER))
+		return -EPROBE_DEFER;
+	if (IS_ERR(bri->scl_gpiod)) {
+		dev_info(dev, "missing scl gpio recovery information: %pe\n",
+			 bri->scl_gpiod);
+		return 0;
+	}
+
+	/*
+	 * We have SCL. Pull SCL low and wait a bit so that SDA glitches
+	 * have no effect.
+	 */
+	gpiod_direction_output(bri->scl_gpiod, 0);
+	udelay(10);
+	bri->sda_gpiod = devm_gpiod_get(dev, "sda", GPIOD_OUT_HIGH_OPEN_DRAIN);
+
+	/* Wait a bit in case of a SDA glitch, and then release SCL. */
+	udelay(10);
+	gpiod_direction_output(bri->scl_gpiod, 1);
+
+	if (bri->sda_gpiod == ERR_PTR(-EPROBE_DEFER))
+		return -EPROBE_DEFER;
+
+	if (IS_ERR(bri->sda_gpiod)) {
+		dev_info(dev, "missing sda gpio recovery information: %pe\n",
+			 bri->sda_gpiod);
+		return 0;
+	}
+
+	bri->prepare_recovery = i2c_pxa_prepare_recovery;
+	bri->unprepare_recovery = i2c_pxa_unprepare_recovery;
+	bri->recover_bus = i2c_generic_scl_recovery;
+
+	i2c->adap.bus_recovery_info = bri;
+
+	/*
+	 * Claiming GPIOs can change the pinmux state, which confuses the
+	 * pinctrl since pinctrl's idea of the current setting is unaffected
+	 * by the pinmux change caused by claiming the GPIO. Work around that
+	 * by switching pinctrl to the GPIO state here. We do it this way to
+	 * avoid glitching the I2C bus.
+	 */
+	pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_recovery);
+
+	return pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_default);
+}
+
 static int i2c_pxa_probe(struct platform_device *dev)
 {
 	struct i2c_pxa_platform_data *plat = dev_get_platdata(&dev->dev);
@@ -1261,19 +1431,28 @@ static int i2c_pxa_probe(struct platform_device *dev)
 	if (!i2c)
 		return -ENOMEM;
 
+	/* Default adapter num to device id; i2c_pxa_probe_dt can override. */
+	i2c->adap.nr = dev->id;
+	i2c->adap.owner   = THIS_MODULE;
+	i2c->adap.retries = 5;
+	i2c->adap.algo_data = i2c;
+	i2c->adap.dev.parent = &dev->dev;
+#ifdef CONFIG_OF
+	i2c->adap.dev.of_node = dev->dev.of_node;
+#endif
+
 	res = platform_get_resource(dev, IORESOURCE_MEM, 0);
 	i2c->reg_base = devm_ioremap_resource(&dev->dev, res);
 	if (IS_ERR(i2c->reg_base))
 		return PTR_ERR(i2c->reg_base);
 
 	irq = platform_get_irq(dev, 0);
-	if (irq < 0) {
-		dev_err(&dev->dev, "no irq resource: %d\n", irq);
+	if (irq < 0)
 		return irq;
-	}
 
-	/* Default adapter num to device id; i2c_pxa_probe_dt can override. */
-	i2c->adap.nr = dev->id;
+	ret = i2c_pxa_init_recovery(i2c);
+	if (ret)
+		return ret;
 
 	ret = i2c_pxa_probe_dt(dev, i2c, &i2c_type);
 	if (ret > 0)
@@ -1281,9 +1460,6 @@ static int i2c_pxa_probe(struct platform_device *dev)
 	if (ret < 0)
 		return ret;
 
-	i2c->adap.owner   = THIS_MODULE;
-	i2c->adap.retries = 5;
-
 	spin_lock_init(&i2c->lock);
 	init_waitqueue_head(&i2c->wait);
 
@@ -1299,8 +1475,8 @@ static int i2c_pxa_probe(struct platform_device *dev)
 	i2c->reg_idbr = i2c->reg_base + pxa_reg_layout[i2c_type].idbr;
 	i2c->reg_icr = i2c->reg_base + pxa_reg_layout[i2c_type].icr;
 	i2c->reg_isr = i2c->reg_base + pxa_reg_layout[i2c_type].isr;
-	i2c->fm_mask = pxa_reg_layout[i2c_type].fm ? : ICR_FM;
-	i2c->hs_mask = pxa_reg_layout[i2c_type].hs ? : ICR_HS;
+	i2c->fm_mask = pxa_reg_layout[i2c_type].fm;
+	i2c->hs_mask = pxa_reg_layout[i2c_type].hs;
 
 	if (i2c_type != REGS_CE4100)
 		i2c->reg_isar = i2c->reg_base + pxa_reg_layout[i2c_type].isar;
@@ -1349,12 +1525,6 @@ static int i2c_pxa_probe(struct platform_device *dev)
 
 	i2c_pxa_reset(i2c);
 
-	i2c->adap.algo_data = i2c;
-	i2c->adap.dev.parent = &dev->dev;
-#ifdef CONFIG_OF
-	i2c->adap.dev.of_node = dev->dev.of_node;
-#endif
-
 	ret = i2c_add_numbered_adapter(&i2c->adap);
 	if (ret < 0)
 		goto ereqirq;
diff --git a/drivers/i2c/busses/i2c-qcom-cci.c b/drivers/i2c/busses/i2c-qcom-cci.c
new file mode 100644
index 000000000000..f13735beca58
--- /dev/null
+++ b/drivers/i2c/busses/i2c-qcom-cci.c
@@ -0,0 +1,791 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
+// Copyright (c) 2017-20 Linaro Limited.
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/i2c.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#define CCI_HW_VERSION				0x0
+#define CCI_RESET_CMD				0x004
+#define CCI_RESET_CMD_MASK			0x0f73f3f7
+#define CCI_RESET_CMD_M0_MASK			0x000003f1
+#define CCI_RESET_CMD_M1_MASK			0x0003f001
+#define CCI_QUEUE_START				0x008
+#define CCI_HALT_REQ				0x034
+#define CCI_HALT_REQ_I2C_M0_Q0Q1		BIT(0)
+#define CCI_HALT_REQ_I2C_M1_Q0Q1		BIT(1)
+
+#define CCI_I2C_Mm_SCL_CTL(m)			(0x100 + 0x100 * (m))
+#define CCI_I2C_Mm_SDA_CTL_0(m)			(0x104 + 0x100 * (m))
+#define CCI_I2C_Mm_SDA_CTL_1(m)			(0x108 + 0x100 * (m))
+#define CCI_I2C_Mm_SDA_CTL_2(m)			(0x10c + 0x100 * (m))
+#define CCI_I2C_Mm_MISC_CTL(m)			(0x110 + 0x100 * (m))
+
+#define CCI_I2C_Mm_READ_DATA(m)			(0x118 + 0x100 * (m))
+#define CCI_I2C_Mm_READ_BUF_LEVEL(m)		(0x11c + 0x100 * (m))
+#define CCI_I2C_Mm_Qn_EXEC_WORD_CNT(m, n)	(0x300 + 0x200 * (m) + 0x100 * (n))
+#define CCI_I2C_Mm_Qn_CUR_WORD_CNT(m, n)	(0x304 + 0x200 * (m) + 0x100 * (n))
+#define CCI_I2C_Mm_Qn_CUR_CMD(m, n)		(0x308 + 0x200 * (m) + 0x100 * (n))
+#define CCI_I2C_Mm_Qn_REPORT_STATUS(m, n)	(0x30c + 0x200 * (m) + 0x100 * (n))
+#define CCI_I2C_Mm_Qn_LOAD_DATA(m, n)		(0x310 + 0x200 * (m) + 0x100 * (n))
+
+#define CCI_IRQ_GLOBAL_CLEAR_CMD		0xc00
+#define CCI_IRQ_MASK_0				0xc04
+#define CCI_IRQ_MASK_0_I2C_M0_RD_DONE		BIT(0)
+#define CCI_IRQ_MASK_0_I2C_M0_Q0_REPORT		BIT(4)
+#define CCI_IRQ_MASK_0_I2C_M0_Q1_REPORT		BIT(8)
+#define CCI_IRQ_MASK_0_I2C_M1_RD_DONE		BIT(12)
+#define CCI_IRQ_MASK_0_I2C_M1_Q0_REPORT		BIT(16)
+#define CCI_IRQ_MASK_0_I2C_M1_Q1_REPORT		BIT(20)
+#define CCI_IRQ_MASK_0_RST_DONE_ACK		BIT(24)
+#define CCI_IRQ_MASK_0_I2C_M0_Q0Q1_HALT_ACK	BIT(25)
+#define CCI_IRQ_MASK_0_I2C_M1_Q0Q1_HALT_ACK	BIT(26)
+#define CCI_IRQ_MASK_0_I2C_M0_ERROR		0x18000ee6
+#define CCI_IRQ_MASK_0_I2C_M1_ERROR		0x60ee6000
+#define CCI_IRQ_CLEAR_0				0xc08
+#define CCI_IRQ_STATUS_0			0xc0c
+#define CCI_IRQ_STATUS_0_I2C_M0_RD_DONE		BIT(0)
+#define CCI_IRQ_STATUS_0_I2C_M0_Q0_REPORT	BIT(4)
+#define CCI_IRQ_STATUS_0_I2C_M0_Q1_REPORT	BIT(8)
+#define CCI_IRQ_STATUS_0_I2C_M1_RD_DONE		BIT(12)
+#define CCI_IRQ_STATUS_0_I2C_M1_Q0_REPORT	BIT(16)
+#define CCI_IRQ_STATUS_0_I2C_M1_Q1_REPORT	BIT(20)
+#define CCI_IRQ_STATUS_0_RST_DONE_ACK		BIT(24)
+#define CCI_IRQ_STATUS_0_I2C_M0_Q0Q1_HALT_ACK	BIT(25)
+#define CCI_IRQ_STATUS_0_I2C_M1_Q0Q1_HALT_ACK	BIT(26)
+#define CCI_IRQ_STATUS_0_I2C_M0_Q0_NACK_ERR	BIT(27)
+#define CCI_IRQ_STATUS_0_I2C_M0_Q1_NACK_ERR	BIT(28)
+#define CCI_IRQ_STATUS_0_I2C_M1_Q0_NACK_ERR	BIT(29)
+#define CCI_IRQ_STATUS_0_I2C_M1_Q1_NACK_ERR	BIT(30)
+#define CCI_IRQ_STATUS_0_I2C_M0_ERROR		0x18000ee6
+#define CCI_IRQ_STATUS_0_I2C_M1_ERROR		0x60ee6000
+
+#define CCI_TIMEOUT	(msecs_to_jiffies(100))
+#define NUM_MASTERS	2
+#define NUM_QUEUES	2
+
+/* Max number of resources + 1 for a NULL terminator */
+#define CCI_RES_MAX	6
+
+#define CCI_I2C_SET_PARAM	1
+#define CCI_I2C_REPORT		8
+#define CCI_I2C_WRITE		9
+#define CCI_I2C_READ		10
+
+#define CCI_I2C_REPORT_IRQ_EN	BIT(8)
+
+enum {
+	I2C_MODE_STANDARD,
+	I2C_MODE_FAST,
+	I2C_MODE_FAST_PLUS,
+};
+
+enum cci_i2c_queue_t {
+	QUEUE_0,
+	QUEUE_1
+};
+
+struct hw_params {
+	u16 thigh; /* HIGH period of the SCL clock in clock ticks */
+	u16 tlow; /* LOW period of the SCL clock */
+	u16 tsu_sto; /* set-up time for STOP condition */
+	u16 tsu_sta; /* set-up time for a repeated START condition */
+	u16 thd_dat; /* data hold time */
+	u16 thd_sta; /* hold time (repeated) START condition */
+	u16 tbuf; /* bus free time between a STOP and START condition */
+	u8 scl_stretch_en;
+	u16 trdhld;
+	u16 tsp; /* pulse width of spikes suppressed by the input filter */
+};
+
+struct cci;
+
+struct cci_master {
+	struct i2c_adapter adap;
+	u16 master;
+	u8 mode;
+	int status;
+	struct completion irq_complete;
+	struct cci *cci;
+};
+
+struct cci_data {
+	unsigned int num_masters;
+	struct i2c_adapter_quirks quirks;
+	u16 queue_size[NUM_QUEUES];
+	unsigned long cci_clk_rate;
+	struct hw_params params[3];
+};
+
+struct cci {
+	struct device *dev;
+	void __iomem *base;
+	unsigned int irq;
+	const struct cci_data *data;
+	struct clk_bulk_data *clocks;
+	int nclocks;
+	struct cci_master master[NUM_MASTERS];
+};
+
+static irqreturn_t cci_isr(int irq, void *dev)
+{
+	struct cci *cci = dev;
+	u32 val, reset = 0;
+	int ret = IRQ_NONE;
+
+	val = readl(cci->base + CCI_IRQ_STATUS_0);
+	writel(val, cci->base + CCI_IRQ_CLEAR_0);
+	writel(0x1, cci->base + CCI_IRQ_GLOBAL_CLEAR_CMD);
+
+	if (val & CCI_IRQ_STATUS_0_RST_DONE_ACK) {
+		complete(&cci->master[0].irq_complete);
+		if (cci->master[1].master)
+			complete(&cci->master[1].irq_complete);
+		ret = IRQ_HANDLED;
+	}
+
+	if (val & CCI_IRQ_STATUS_0_I2C_M0_RD_DONE ||
+			val & CCI_IRQ_STATUS_0_I2C_M0_Q0_REPORT ||
+			val & CCI_IRQ_STATUS_0_I2C_M0_Q1_REPORT) {
+		cci->master[0].status = 0;
+		complete(&cci->master[0].irq_complete);
+		ret = IRQ_HANDLED;
+	}
+
+	if (val & CCI_IRQ_STATUS_0_I2C_M1_RD_DONE ||
+			val & CCI_IRQ_STATUS_0_I2C_M1_Q0_REPORT ||
+			val & CCI_IRQ_STATUS_0_I2C_M1_Q1_REPORT) {
+		cci->master[1].status = 0;
+		complete(&cci->master[1].irq_complete);
+		ret = IRQ_HANDLED;
+	}
+
+	if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M0_Q0Q1_HALT_ACK)) {
+		reset = CCI_RESET_CMD_M0_MASK;
+		ret = IRQ_HANDLED;
+	}
+
+	if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M1_Q0Q1_HALT_ACK)) {
+		reset = CCI_RESET_CMD_M1_MASK;
+		ret = IRQ_HANDLED;
+	}
+
+	if (unlikely(reset))
+		writel(reset, cci->base + CCI_RESET_CMD);
+
+	if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M0_ERROR)) {
+		if (val & CCI_IRQ_STATUS_0_I2C_M0_Q0_NACK_ERR ||
+			val & CCI_IRQ_STATUS_0_I2C_M0_Q1_NACK_ERR)
+			cci->master[0].status = -ENXIO;
+		else
+			cci->master[0].status = -EIO;
+
+		writel(CCI_HALT_REQ_I2C_M0_Q0Q1, cci->base + CCI_HALT_REQ);
+		ret = IRQ_HANDLED;
+	}
+
+	if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M1_ERROR)) {
+		if (val & CCI_IRQ_STATUS_0_I2C_M1_Q0_NACK_ERR ||
+			val & CCI_IRQ_STATUS_0_I2C_M1_Q1_NACK_ERR)
+			cci->master[0].status = -ENXIO;
+		else
+			cci->master[0].status = -EIO;
+
+		writel(CCI_HALT_REQ_I2C_M1_Q0Q1, cci->base + CCI_HALT_REQ);
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+static int cci_halt(struct cci *cci, u8 master_num)
+{
+	struct cci_master *master;
+	u32 val;
+
+	if (master_num >= cci->data->num_masters) {
+		dev_err(cci->dev, "Unsupported master idx (%u)\n", master_num);
+		return -EINVAL;
+	}
+
+	val = BIT(master_num);
+	master = &cci->master[master_num];
+
+	reinit_completion(&master->irq_complete);
+	writel(val, cci->base + CCI_HALT_REQ);
+
+	if (!wait_for_completion_timeout(&master->irq_complete, CCI_TIMEOUT)) {
+		dev_err(cci->dev, "CCI halt timeout\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int cci_reset(struct cci *cci)
+{
+	/*
+	 * we reset the whole controller, here and for implicity use
+	 * master[0].xxx for waiting on it.
+	 */
+	reinit_completion(&cci->master[0].irq_complete);
+	writel(CCI_RESET_CMD_MASK, cci->base + CCI_RESET_CMD);
+
+	if (!wait_for_completion_timeout(&cci->master[0].irq_complete,
+					 CCI_TIMEOUT)) {
+		dev_err(cci->dev, "CCI reset timeout\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int cci_init(struct cci *cci)
+{
+	u32 val = CCI_IRQ_MASK_0_I2C_M0_RD_DONE |
+			CCI_IRQ_MASK_0_I2C_M0_Q0_REPORT |
+			CCI_IRQ_MASK_0_I2C_M0_Q1_REPORT |
+			CCI_IRQ_MASK_0_I2C_M1_RD_DONE |
+			CCI_IRQ_MASK_0_I2C_M1_Q0_REPORT |
+			CCI_IRQ_MASK_0_I2C_M1_Q1_REPORT |
+			CCI_IRQ_MASK_0_RST_DONE_ACK |
+			CCI_IRQ_MASK_0_I2C_M0_Q0Q1_HALT_ACK |
+			CCI_IRQ_MASK_0_I2C_M1_Q0Q1_HALT_ACK |
+			CCI_IRQ_MASK_0_I2C_M0_ERROR |
+			CCI_IRQ_MASK_0_I2C_M1_ERROR;
+	int i;
+
+	writel(val, cci->base + CCI_IRQ_MASK_0);
+
+	for (i = 0; i < cci->data->num_masters; i++) {
+		int mode = cci->master[i].mode;
+		const struct hw_params *hw;
+
+		if (!cci->master[i].cci)
+			continue;
+
+		hw = &cci->data->params[mode];
+
+		val = hw->thigh << 16 | hw->tlow;
+		writel(val, cci->base + CCI_I2C_Mm_SCL_CTL(i));
+
+		val = hw->tsu_sto << 16 | hw->tsu_sta;
+		writel(val, cci->base + CCI_I2C_Mm_SDA_CTL_0(i));
+
+		val = hw->thd_dat << 16 | hw->thd_sta;
+		writel(val, cci->base + CCI_I2C_Mm_SDA_CTL_1(i));
+
+		val = hw->tbuf;
+		writel(val, cci->base + CCI_I2C_Mm_SDA_CTL_2(i));
+
+		val = hw->scl_stretch_en << 8 | hw->trdhld << 4 | hw->tsp;
+		writel(val, cci->base + CCI_I2C_Mm_MISC_CTL(i));
+	}
+
+	return 0;
+}
+
+static int cci_run_queue(struct cci *cci, u8 master, u8 queue)
+{
+	u32 val;
+
+	val = readl(cci->base + CCI_I2C_Mm_Qn_CUR_WORD_CNT(master, queue));
+	writel(val, cci->base + CCI_I2C_Mm_Qn_EXEC_WORD_CNT(master, queue));
+
+	reinit_completion(&cci->master[master].irq_complete);
+	val = BIT(master * 2 + queue);
+	writel(val, cci->base + CCI_QUEUE_START);
+
+	if (!wait_for_completion_timeout(&cci->master[master].irq_complete,
+					 CCI_TIMEOUT)) {
+		dev_err(cci->dev, "master %d queue %d timeout\n",
+			master, queue);
+		cci_reset(cci);
+		cci_init(cci);
+		return -ETIMEDOUT;
+	}
+
+	return cci->master[master].status;
+}
+
+static int cci_validate_queue(struct cci *cci, u8 master, u8 queue)
+{
+	u32 val;
+
+	val = readl(cci->base + CCI_I2C_Mm_Qn_CUR_WORD_CNT(master, queue));
+	if (val == cci->data->queue_size[queue])
+		return -EINVAL;
+
+	if (!val)
+		return 0;
+
+	val = CCI_I2C_REPORT | CCI_I2C_REPORT_IRQ_EN;
+	writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue));
+
+	return cci_run_queue(cci, master, queue);
+}
+
+static int cci_i2c_read(struct cci *cci, u16 master,
+			u16 addr, u8 *buf, u16 len)
+{
+	u32 val, words_read, words_exp;
+	u8 queue = QUEUE_1;
+	int i, index = 0, ret;
+	bool first = true;
+
+	/*
+	 * Call validate queue to make sure queue is empty before starting.
+	 * This is to avoid overflow / underflow of queue.
+	 */
+	ret = cci_validate_queue(cci, master, queue);
+	if (ret < 0)
+		return ret;
+
+	val = CCI_I2C_SET_PARAM | (addr & 0x7f) << 4;
+	writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue));
+
+	val = CCI_I2C_READ | len << 4;
+	writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue));
+
+	ret = cci_run_queue(cci, master, queue);
+	if (ret < 0)
+		return ret;
+
+	words_read = readl(cci->base + CCI_I2C_Mm_READ_BUF_LEVEL(master));
+	words_exp = len / 4 + 1;
+	if (words_read != words_exp) {
+		dev_err(cci->dev, "words read = %d, words expected = %d\n",
+			words_read, words_exp);
+		return -EIO;
+	}
+
+	do {
+		val = readl(cci->base + CCI_I2C_Mm_READ_DATA(master));
+
+		for (i = 0; i < 4 && index < len; i++) {
+			if (first) {
+				/* The LS byte of this register represents the
+				 * first byte read from the slave during a read
+				 * access.
+				 */
+				first = false;
+				continue;
+			}
+			buf[index++] = (val >> (i * 8)) & 0xff;
+		}
+	} while (--words_read);
+
+	return 0;
+}
+
+static int cci_i2c_write(struct cci *cci, u16 master,
+			 u16 addr, u8 *buf, u16 len)
+{
+	u8 queue = QUEUE_0;
+	u8 load[12] = { 0 };
+	int i = 0, j, ret;
+	u32 val;
+
+	/*
+	 * Call validate queue to make sure queue is empty before starting.
+	 * This is to avoid overflow / underflow of queue.
+	 */
+	ret = cci_validate_queue(cci, master, queue);
+	if (ret < 0)
+		return ret;
+
+	val = CCI_I2C_SET_PARAM | (addr & 0x7f) << 4;
+	writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue));
+
+	load[i++] = CCI_I2C_WRITE | len << 4;
+
+	for (j = 0; j < len; j++)
+		load[i++] = buf[j];
+
+	for (j = 0; j < i; j += 4) {
+		val = load[j];
+		val |= load[j + 1] << 8;
+		val |= load[j + 2] << 16;
+		val |= load[j + 3] << 24;
+		writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue));
+	}
+
+	val = CCI_I2C_REPORT | CCI_I2C_REPORT_IRQ_EN;
+	writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue));
+
+	return cci_run_queue(cci, master, queue);
+}
+
+static int cci_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+{
+	struct cci_master *cci_master = i2c_get_adapdata(adap);
+	struct cci *cci = cci_master->cci;
+	int i, ret;
+
+	ret = pm_runtime_get_sync(cci->dev);
+	if (ret < 0)
+		goto err;
+
+	for (i = 0; i < num; i++) {
+		if (msgs[i].flags & I2C_M_RD)
+			ret = cci_i2c_read(cci, cci_master->master,
+					   msgs[i].addr, msgs[i].buf,
+					   msgs[i].len);
+		else
+			ret = cci_i2c_write(cci, cci_master->master,
+					    msgs[i].addr, msgs[i].buf,
+					    msgs[i].len);
+
+		if (ret < 0)
+			break;
+	}
+
+	if (!ret)
+		ret = num;
+
+err:
+	pm_runtime_mark_last_busy(cci->dev);
+	pm_runtime_put_autosuspend(cci->dev);
+
+	return ret;
+}
+
+static u32 cci_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm cci_algo = {
+	.master_xfer	= cci_xfer,
+	.functionality	= cci_func,
+};
+
+static int cci_enable_clocks(struct cci *cci)
+{
+	return clk_bulk_prepare_enable(cci->nclocks, cci->clocks);
+}
+
+static void cci_disable_clocks(struct cci *cci)
+{
+	clk_bulk_disable_unprepare(cci->nclocks, cci->clocks);
+}
+
+static int __maybe_unused cci_suspend_runtime(struct device *dev)
+{
+	struct cci *cci = dev_get_drvdata(dev);
+
+	cci_disable_clocks(cci);
+	return 0;
+}
+
+static int __maybe_unused cci_resume_runtime(struct device *dev)
+{
+	struct cci *cci = dev_get_drvdata(dev);
+	int ret;
+
+	ret = cci_enable_clocks(cci);
+	if (ret)
+		return ret;
+
+	cci_init(cci);
+	return 0;
+}
+
+static int __maybe_unused cci_suspend(struct device *dev)
+{
+	if (!pm_runtime_suspended(dev))
+		return cci_suspend_runtime(dev);
+
+	return 0;
+}
+
+static int __maybe_unused cci_resume(struct device *dev)
+{
+	cci_resume_runtime(dev);
+	pm_runtime_mark_last_busy(dev);
+	pm_request_autosuspend(dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops qcom_cci_pm = {
+	SET_SYSTEM_SLEEP_PM_OPS(cci_suspend, cci_resume)
+	SET_RUNTIME_PM_OPS(cci_suspend_runtime, cci_resume_runtime, NULL)
+};
+
+static int cci_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	unsigned long cci_clk_rate = 0;
+	struct device_node *child;
+	struct resource *r;
+	struct cci *cci;
+	int ret, i;
+	u32 val;
+
+	cci = devm_kzalloc(dev, sizeof(*cci), GFP_KERNEL);
+	if (!cci)
+		return -ENOMEM;
+
+	cci->dev = dev;
+	platform_set_drvdata(pdev, cci);
+	cci->data = device_get_match_data(dev);
+	if (!cci->data)
+		return -ENOENT;
+
+	for_each_available_child_of_node(dev->of_node, child) {
+		u32 idx;
+
+		ret = of_property_read_u32(child, "reg", &idx);
+		if (ret) {
+			dev_err(dev, "%pOF invalid 'reg' property", child);
+			continue;
+		}
+
+		if (idx >= cci->data->num_masters) {
+			dev_err(dev, "%pOF invalid 'reg' value: %u (max is %u)",
+				child, idx, cci->data->num_masters - 1);
+			continue;
+		}
+
+		cci->master[idx].adap.quirks = &cci->data->quirks;
+		cci->master[idx].adap.algo = &cci_algo;
+		cci->master[idx].adap.dev.parent = dev;
+		cci->master[idx].adap.dev.of_node = child;
+		cci->master[idx].master = idx;
+		cci->master[idx].cci = cci;
+
+		i2c_set_adapdata(&cci->master[idx].adap, &cci->master[idx]);
+		snprintf(cci->master[idx].adap.name,
+			 sizeof(cci->master[idx].adap.name), "Qualcomm-CCI");
+
+		cci->master[idx].mode = I2C_MODE_STANDARD;
+		ret = of_property_read_u32(child, "clock-frequency", &val);
+		if (!ret) {
+			if (val == 400000)
+				cci->master[idx].mode = I2C_MODE_FAST;
+			else if (val == 1000000)
+				cci->master[idx].mode = I2C_MODE_FAST_PLUS;
+		}
+
+		init_completion(&cci->master[idx].irq_complete);
+	}
+
+	/* Memory */
+
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	cci->base = devm_ioremap_resource(dev, r);
+	if (IS_ERR(cci->base))
+		return PTR_ERR(cci->base);
+
+	/* Clocks */
+
+	ret = devm_clk_bulk_get_all(dev, &cci->clocks);
+	if (ret < 1) {
+		dev_err(dev, "failed to get clocks %d\n", ret);
+		return ret;
+	}
+	cci->nclocks = ret;
+
+	/* Retrieve CCI clock rate */
+	for (i = 0; i < cci->nclocks; i++) {
+		if (!strcmp(cci->clocks[i].id, "cci")) {
+			cci_clk_rate = clk_get_rate(cci->clocks[i].clk);
+			break;
+		}
+	}
+
+	if (cci_clk_rate != cci->data->cci_clk_rate) {
+		/* cci clock set by the bootloader or via assigned clock rate
+		 * in DT.
+		 */
+		dev_warn(dev, "Found %lu cci clk rate while %lu was expected\n",
+			 cci_clk_rate, cci->data->cci_clk_rate);
+	}
+
+	ret = cci_enable_clocks(cci);
+	if (ret < 0)
+		return ret;
+
+	/* Interrupt */
+
+	ret = platform_get_irq(pdev, 0);
+	if (ret < 0)
+		goto disable_clocks;
+	cci->irq = ret;
+
+	ret = devm_request_irq(dev, cci->irq, cci_isr, 0, dev_name(dev), cci);
+	if (ret < 0) {
+		dev_err(dev, "request_irq failed, ret: %d\n", ret);
+		goto disable_clocks;
+	}
+
+	val = readl(cci->base + CCI_HW_VERSION);
+	dev_dbg(dev, "CCI HW version = 0x%08x", val);
+
+	ret = cci_reset(cci);
+	if (ret < 0)
+		goto error;
+
+	ret = cci_init(cci);
+	if (ret < 0)
+		goto error;
+
+	for (i = 0; i < cci->data->num_masters; i++) {
+		if (!cci->master[i].cci)
+			continue;
+
+		ret = i2c_add_adapter(&cci->master[i].adap);
+		if (ret < 0)
+			goto error_i2c;
+	}
+
+	pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	return 0;
+
+error_i2c:
+	for (; i >= 0; i--) {
+		if (cci->master[i].cci)
+			i2c_del_adapter(&cci->master[i].adap);
+	}
+error:
+	disable_irq(cci->irq);
+disable_clocks:
+	cci_disable_clocks(cci);
+
+	return ret;
+}
+
+static int cci_remove(struct platform_device *pdev)
+{
+	struct cci *cci = platform_get_drvdata(pdev);
+	int i;
+
+	for (i = 0; i < cci->data->num_masters; i++) {
+		if (cci->master[i].cci)
+			i2c_del_adapter(&cci->master[i].adap);
+		cci_halt(cci, i);
+	}
+
+	disable_irq(cci->irq);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_set_suspended(&pdev->dev);
+
+	return 0;
+}
+
+static const struct cci_data cci_v1_data = {
+	.num_masters = 1,
+	.queue_size = { 64, 16 },
+	.quirks = {
+		.max_write_len = 10,
+		.max_read_len = 12,
+	},
+	.cci_clk_rate =  19200000,
+	.params[I2C_MODE_STANDARD] = {
+		.thigh = 78,
+		.tlow = 114,
+		.tsu_sto = 28,
+		.tsu_sta = 28,
+		.thd_dat = 10,
+		.thd_sta = 77,
+		.tbuf = 118,
+		.scl_stretch_en = 0,
+		.trdhld = 6,
+		.tsp = 1
+	},
+	.params[I2C_MODE_FAST] = {
+		.thigh = 20,
+		.tlow = 28,
+		.tsu_sto = 21,
+		.tsu_sta = 21,
+		.thd_dat = 13,
+		.thd_sta = 18,
+		.tbuf = 32,
+		.scl_stretch_en = 0,
+		.trdhld = 6,
+		.tsp = 3
+	},
+};
+
+static const struct cci_data cci_v2_data = {
+	.num_masters = 2,
+	.queue_size = { 64, 16 },
+	.quirks = {
+		.max_write_len = 11,
+		.max_read_len = 12,
+	},
+	.cci_clk_rate =  37500000,
+	.params[I2C_MODE_STANDARD] = {
+		.thigh = 201,
+		.tlow = 174,
+		.tsu_sto = 204,
+		.tsu_sta = 231,
+		.thd_dat = 22,
+		.thd_sta = 162,
+		.tbuf = 227,
+		.scl_stretch_en = 0,
+		.trdhld = 6,
+		.tsp = 3
+	},
+	.params[I2C_MODE_FAST] = {
+		.thigh = 38,
+		.tlow = 56,
+		.tsu_sto = 40,
+		.tsu_sta = 40,
+		.thd_dat = 22,
+		.thd_sta = 35,
+		.tbuf = 62,
+		.scl_stretch_en = 0,
+		.trdhld = 6,
+		.tsp = 3
+	},
+	.params[I2C_MODE_FAST_PLUS] = {
+		.thigh = 16,
+		.tlow = 22,
+		.tsu_sto = 17,
+		.tsu_sta = 18,
+		.thd_dat = 16,
+		.thd_sta = 15,
+		.tbuf = 24,
+		.scl_stretch_en = 0,
+		.trdhld = 3,
+		.tsp = 3
+	},
+};
+
+static const struct of_device_id cci_dt_match[] = {
+	{ .compatible = "qcom,msm8916-cci", .data = &cci_v1_data},
+	{ .compatible = "qcom,msm8996-cci", .data = &cci_v2_data},
+	{ .compatible = "qcom,sdm845-cci", .data = &cci_v2_data},
+	{}
+};
+MODULE_DEVICE_TABLE(of, cci_dt_match);
+
+static struct platform_driver qcom_cci_driver = {
+	.probe  = cci_probe,
+	.remove = cci_remove,
+	.driver = {
+		.name = "i2c-qcom-cci",
+		.of_match_table = cci_dt_match,
+		.pm = &qcom_cci_pm,
+	},
+};
+
+module_platform_driver(qcom_cci_driver);
+
+MODULE_DESCRIPTION("Qualcomm Camera Control Interface driver");
+MODULE_AUTHOR("Todor Tomov <todor.tomov@linaro.org>");
+MODULE_AUTHOR("Loic Poulain <loic.poulain@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c
index 748872a9b0fc..fbc04b60cfd1 100644
--- a/drivers/i2c/busses/i2c-qup.c
+++ b/drivers/i2c/busses/i2c-qup.c
@@ -956,10 +956,8 @@ static void qup_i2c_conf_v1(struct qup_i2c_dev *qup)
 	u32 qup_config = I2C_MINI_CORE | I2C_N_VAL;
 	u32 io_mode = QUP_REPACK_EN;
 
-	blk->is_tx_blk_mode =
-		blk->total_tx_len > qup->out_fifo_sz ? true : false;
-	blk->is_rx_blk_mode =
-		blk->total_rx_len > qup->in_fifo_sz ? true : false;
+	blk->is_tx_blk_mode = blk->total_tx_len > qup->out_fifo_sz;
+	blk->is_rx_blk_mode = blk->total_rx_len > qup->in_fifo_sz;
 
 	if (blk->is_tx_blk_mode) {
 		io_mode |= QUP_OUTPUT_BLK_MODE;
@@ -1528,9 +1526,9 @@ qup_i2c_determine_mode_v2(struct qup_i2c_dev *qup,
 		qup->use_dma = true;
 	} else {
 		qup->blk.is_tx_blk_mode = max_tx_len > qup->out_fifo_sz -
-			QUP_MAX_TAGS_LEN ? true : false;
+			QUP_MAX_TAGS_LEN;
 		qup->blk.is_rx_blk_mode = max_rx_len > qup->in_fifo_sz -
-			READ_RX_TAGS_LEN ? true : false;
+			READ_RX_TAGS_LEN;
 	}
 
 	return 0;
@@ -1660,7 +1658,6 @@ static int qup_i2c_probe(struct platform_device *pdev)
 	static const int blk_sizes[] = {4, 16, 32};
 	struct qup_i2c_dev *qup;
 	unsigned long one_bit_t;
-	struct resource *res;
 	u32 io_mode, hw_ver, size;
 	int ret, fs_div, hs_div;
 	u32 src_clk_freq = DEFAULT_SRC_CLK;
@@ -1757,16 +1754,13 @@ nodma:
 		return -EINVAL;
 	}
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	qup->base = devm_ioremap_resource(qup->dev, res);
+	qup->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(qup->base))
 		return PTR_ERR(qup->base);
 
 	qup->irq = platform_get_irq(pdev, 0);
-	if (qup->irq < 0) {
-		dev_err(qup->dev, "No IRQ defined\n");
+	if (qup->irq < 0)
 		return qup->irq;
-	}
 
 	if (has_acpi_companion(qup->dev)) {
 		ret = device_property_read_u32(qup->dev,
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index 3b5397aa4ca6..a45c4bf1ec01 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -938,9 +938,7 @@ static int rcar_i2c_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->clk);
 	}
 
-	priv->res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
-	priv->io = devm_ioremap_resource(dev, priv->res);
+	priv->io = devm_platform_get_and_ioremap_resource(pdev, 0, &priv->res);
 	if (IS_ERR(priv->io))
 		return PTR_ERR(priv->io);
 
diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
index 73272d4296bb..bc698240c4aa 100644
--- a/drivers/i2c/busses/i2c-rk3x.c
+++ b/drivers/i2c/busses/i2c-rk3x.c
@@ -1193,7 +1193,6 @@ static int rk3x_i2c_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 	const struct of_device_id *match;
 	struct rk3x_i2c *i2c;
-	struct resource *mem;
 	int ret = 0;
 	int bus_nr;
 	u32 value;
@@ -1223,8 +1222,7 @@ static int rk3x_i2c_probe(struct platform_device *pdev)
 	spin_lock_init(&i2c->lock);
 	init_waitqueue_head(&i2c->wait);
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->regs = devm_ioremap_resource(&pdev->dev, mem);
+	i2c->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->regs))
 		return PTR_ERR(i2c->regs);
 
@@ -1262,10 +1260,8 @@ static int rk3x_i2c_probe(struct platform_device *pdev)
 
 	/* IRQ setup */
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "cannot find rk3x IRQ\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	ret = devm_request_irq(&pdev->dev, irq, rk3x_i2c_irq,
 			       0, dev_name(&pdev->dev), i2c);
diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c
index 5a5638e1daa1..3eafe0eb3e4c 100644
--- a/drivers/i2c/busses/i2c-s3c2410.c
+++ b/drivers/i2c/busses/i2c-s3c2410.c
@@ -435,8 +435,7 @@ static int i2c_s3c_irq_nextbyte(struct s3c24xx_i2c *i2c, unsigned long iicstat)
 		 * fall through to the write state, as we will need to
 		 * send a byte as well
 		 */
-		/* Fall through */
-
+		fallthrough;
 	case STATE_WRITE:
 		/*
 		 * we are writing data to the device... check for the
@@ -1267,5 +1266,5 @@ static void __exit i2c_adap_s3c_exit(void)
 module_exit(i2c_adap_s3c_exit);
 
 MODULE_DESCRIPTION("S3C24XX I2C Bus driver");
-MODULE_AUTHOR("Ben Dooks, <ben@simtec.co.uk>");
+MODULE_AUTHOR("Ben Dooks <ben@simtec.co.uk>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c
index d83ca4028fa0..2cca1b21e26e 100644
--- a/drivers/i2c/busses/i2c-sh_mobile.c
+++ b/drivers/i2c/busses/i2c-sh_mobile.c
@@ -366,7 +366,6 @@ static int sh_mobile_i2c_isr_tx(struct sh_mobile_i2c_data *pd)
 
 static int sh_mobile_i2c_isr_rx(struct sh_mobile_i2c_data *pd)
 {
-	unsigned char data;
 	int real_pos;
 
 	/* switch from TX (address) to RX (data) adds two interrupts */
@@ -387,13 +386,11 @@ static int sh_mobile_i2c_isr_rx(struct sh_mobile_i2c_data *pd)
 		if (real_pos < 0)
 			i2c_op(pd, OP_RX_STOP);
 		else
-			data = i2c_op(pd, OP_RX_STOP_DATA);
+			pd->msg->buf[real_pos] = i2c_op(pd, OP_RX_STOP_DATA);
 	} else if (real_pos >= 0) {
-		data = i2c_op(pd, OP_RX);
+		pd->msg->buf[real_pos] = i2c_op(pd, OP_RX);
 	}
 
-	if (real_pos >= 0)
-		pd->msg->buf[real_pos] = data;
  done:
 	pd->pos++;
 	return pd->pos == (pd->msg->len + 2);
diff --git a/drivers/i2c/busses/i2c-sirf.c b/drivers/i2c/busses/i2c-sirf.c
index a459e00c6851..d7f72ec331e8 100644
--- a/drivers/i2c/busses/i2c-sirf.c
+++ b/drivers/i2c/busses/i2c-sirf.c
@@ -271,7 +271,6 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev)
 {
 	struct sirfsoc_i2c *siic;
 	struct i2c_adapter *adap;
-	struct resource *mem_res;
 	struct clk *clk;
 	int bitrate;
 	int ctrl_speed;
@@ -309,8 +308,7 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev)
 	adap = &siic->adapter;
 	adap->class = I2C_CLASS_DEPRECATED;
 
-	mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	siic->base = devm_ioremap_resource(&pdev->dev, mem_res);
+	siic->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(siic->base)) {
 		err = PTR_ERR(siic->base);
 		goto out;
diff --git a/drivers/i2c/busses/i2c-sprd.c b/drivers/i2c/busses/i2c-sprd.c
index 123a42bfe3b1..19cda6742423 100644
--- a/drivers/i2c/busses/i2c-sprd.c
+++ b/drivers/i2c/busses/i2c-sprd.c
@@ -492,10 +492,8 @@ static int sprd_i2c_probe(struct platform_device *pdev)
 		return PTR_ERR(i2c_dev->base);
 
 	i2c_dev->irq = platform_get_irq(pdev, 0);
-	if (i2c_dev->irq < 0) {
-		dev_err(&pdev->dev, "failed to get irq resource\n");
+	if (i2c_dev->irq < 0)
 		return i2c_dev->irq;
-	}
 
 	i2c_set_adapdata(&i2c_dev->adap, i2c_dev);
 	init_completion(&i2c_dev->complete);
diff --git a/drivers/i2c/busses/i2c-stm32.c b/drivers/i2c/busses/i2c-stm32.c
index 1da347e6a358..3f69a3bb6119 100644
--- a/drivers/i2c/busses/i2c-stm32.c
+++ b/drivers/i2c/busses/i2c-stm32.c
@@ -25,8 +25,9 @@ struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev,
 	/* Request and configure I2C TX dma channel */
 	dma->chan_tx = dma_request_chan(dev, "tx");
 	if (IS_ERR(dma->chan_tx)) {
-		dev_dbg(dev, "can't request DMA tx channel\n");
 		ret = PTR_ERR(dma->chan_tx);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "can't request DMA tx channel\n");
 		goto fail_al;
 	}
 
@@ -44,8 +45,10 @@ struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev,
 	/* Request and configure I2C RX dma channel */
 	dma->chan_rx = dma_request_chan(dev, "rx");
 	if (IS_ERR(dma->chan_rx)) {
-		dev_err(dev, "can't request DMA rx channel\n");
 		ret = PTR_ERR(dma->chan_rx);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "can't request DMA rx channel\n");
+
 		goto fail_tx;
 	}
 
@@ -73,7 +76,8 @@ fail_tx:
 	dma_release_channel(dma->chan_tx);
 fail_al:
 	devm_kfree(dev, dma);
-	dev_info(dev, "can't use DMA\n");
+	if (ret != -EPROBE_DEFER)
+		dev_info(dev, "can't use DMA\n");
 
 	return ERR_PTR(ret);
 }
diff --git a/drivers/i2c/busses/i2c-stm32f4.c b/drivers/i2c/busses/i2c-stm32f4.c
index d6a69dfcac3f..48e269284369 100644
--- a/drivers/i2c/busses/i2c-stm32f4.c
+++ b/drivers/i2c/busses/i2c-stm32f4.c
@@ -797,8 +797,10 @@ static int stm32f4_i2c_probe(struct platform_device *pdev)
 
 	rst = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(rst)) {
-		dev_err(&pdev->dev, "Error: Missing controller reset\n");
 		ret = PTR_ERR(rst);
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Error: Missing reset ctrl\n");
+
 		goto clk_free;
 	}
 	reset_control_assert(rst);
diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c
index 330ffed011e0..bff3479fe122 100644
--- a/drivers/i2c/busses/i2c-stm32f7.c
+++ b/drivers/i2c/busses/i2c-stm32f7.c
@@ -189,8 +189,6 @@ struct stm32f7_i2c_regs {
 /**
  * struct stm32f7_i2c_spec - private i2c specification timing
  * @rate: I2C bus speed (Hz)
- * @rate_min: 80% of I2C bus speed (Hz)
- * @rate_max: 100% of I2C bus speed (Hz)
  * @fall_max: Max fall time of both SDA and SCL signals (ns)
  * @rise_max: Max rise time of both SDA and SCL signals (ns)
  * @hddat_min: Min data hold time (ns)
@@ -201,8 +199,6 @@ struct stm32f7_i2c_regs {
  */
 struct stm32f7_i2c_spec {
 	u32 rate;
-	u32 rate_min;
-	u32 rate_max;
 	u32 fall_max;
 	u32 rise_max;
 	u32 hddat_min;
@@ -214,7 +210,6 @@ struct stm32f7_i2c_spec {
 
 /**
  * struct stm32f7_i2c_setup - private I2C timing setup parameters
- * @speed: I2C speed mode (standard, Fast Plus)
  * @speed_freq: I2C speed frequency  (Hz)
  * @clock_src: I2C clock source frequency (Hz)
  * @rise_time: Rise time (ns)
@@ -224,7 +219,6 @@ struct stm32f7_i2c_spec {
  * @fmp_clr_offset: Fast Mode Plus clear register offset from set register
  */
 struct stm32f7_i2c_setup {
-	enum stm32_i2c_speed speed;
 	u32 speed_freq;
 	u32 clock_src;
 	u32 rise_time;
@@ -287,7 +281,7 @@ struct stm32f7_i2c_msg {
  * @base: virtual memory area
  * @complete: completion of I2C message
  * @clk: hw i2c clock
- * @speed: I2C clock frequency of the controller. Standard, Fast or Fast+
+ * @bus_rate: I2C clock frequency of the controller
  * @msg: Pointer to data to be written
  * @msg_num: number of I2C messages to be executed
  * @msg_id: message identifiant
@@ -314,7 +308,7 @@ struct stm32f7_i2c_dev {
 	void __iomem *base;
 	struct completion complete;
 	struct clk *clk;
-	int speed;
+	unsigned int bus_rate;
 	struct i2c_msg *msg;
 	unsigned int msg_num;
 	unsigned int msg_id;
@@ -342,11 +336,9 @@ struct stm32f7_i2c_dev {
  * Table10. Characteristics of the SDA and SCL bus lines for Standard, Fast,
  * and Fast-mode Plus I2C-bus devices
  */
-static struct stm32f7_i2c_spec i2c_specs[] = {
-	[STM32_I2C_SPEED_STANDARD] = {
+static struct stm32f7_i2c_spec stm32f7_i2c_specs[] = {
+	{
 		.rate = I2C_MAX_STANDARD_MODE_FREQ,
-		.rate_min = I2C_MAX_STANDARD_MODE_FREQ * 8 / 10,	/* 80% */
-		.rate_max = I2C_MAX_STANDARD_MODE_FREQ,
 		.fall_max = 300,
 		.rise_max = 1000,
 		.hddat_min = 0,
@@ -355,10 +347,8 @@ static struct stm32f7_i2c_spec i2c_specs[] = {
 		.l_min = 4700,
 		.h_min = 4000,
 	},
-	[STM32_I2C_SPEED_FAST] = {
+	{
 		.rate = I2C_MAX_FAST_MODE_FREQ,
-		.rate_min = I2C_MAX_FAST_MODE_FREQ * 8 / 10,		/* 80% */
-		.rate_max = I2C_MAX_FAST_MODE_FREQ,
 		.fall_max = 300,
 		.rise_max = 300,
 		.hddat_min = 0,
@@ -367,10 +357,8 @@ static struct stm32f7_i2c_spec i2c_specs[] = {
 		.l_min = 1300,
 		.h_min = 600,
 	},
-	[STM32_I2C_SPEED_FAST_PLUS] = {
+	{
 		.rate = I2C_MAX_FAST_MODE_PLUS_FREQ,
-		.rate_min = I2C_MAX_FAST_MODE_PLUS_FREQ * 8 / 10,	/* 80% */
-		.rate_max = I2C_MAX_FAST_MODE_PLUS_FREQ,
 		.fall_max = 100,
 		.rise_max = 120,
 		.hddat_min = 0,
@@ -411,10 +399,23 @@ static void stm32f7_i2c_disable_irq(struct stm32f7_i2c_dev *i2c_dev, u32 mask)
 	stm32f7_i2c_clr_bits(i2c_dev->base + STM32F7_I2C_CR1, mask);
 }
 
+static struct stm32f7_i2c_spec *stm32f7_get_specs(u32 rate)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(stm32f7_i2c_specs); i++)
+		if (rate <= stm32f7_i2c_specs[i].rate)
+			return &stm32f7_i2c_specs[i];
+
+	return ERR_PTR(-EINVAL);
+}
+
+#define	RATE_MIN(rate)	((rate) * 8 / 10)
 static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 				      struct stm32f7_i2c_setup *setup,
 				      struct stm32f7_i2c_timings *output)
 {
+	struct stm32f7_i2c_spec *specs;
 	u32 p_prev = STM32F7_PRESC_MAX;
 	u32 i2cclk = DIV_ROUND_CLOSEST(NSEC_PER_SEC,
 				       setup->clock_src);
@@ -432,18 +433,19 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 	u16 p, l, a, h;
 	int ret = 0;
 
-	if (setup->speed >= STM32_I2C_SPEED_END) {
-		dev_err(i2c_dev->dev, "speed out of bound {%d/%d}\n",
-			setup->speed, STM32_I2C_SPEED_END - 1);
+	specs = stm32f7_get_specs(setup->speed_freq);
+	if (specs == ERR_PTR(-EINVAL)) {
+		dev_err(i2c_dev->dev, "speed out of bound {%d}\n",
+			setup->speed_freq);
 		return -EINVAL;
 	}
 
-	if ((setup->rise_time > i2c_specs[setup->speed].rise_max) ||
-	    (setup->fall_time > i2c_specs[setup->speed].fall_max)) {
+	if ((setup->rise_time > specs->rise_max) ||
+	    (setup->fall_time > specs->fall_max)) {
 		dev_err(i2c_dev->dev,
 			"timings out of bound Rise{%d>%d}/Fall{%d>%d}\n",
-			setup->rise_time, i2c_specs[setup->speed].rise_max,
-			setup->fall_time, i2c_specs[setup->speed].fall_max);
+			setup->rise_time, specs->rise_max,
+			setup->fall_time, specs->fall_max);
 		return -EINVAL;
 	}
 
@@ -454,12 +456,6 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 		return -EINVAL;
 	}
 
-	if (setup->speed_freq > i2c_specs[setup->speed].rate) {
-		dev_err(i2c_dev->dev, "ERROR: Freq {%d/%d}\n",
-			setup->speed_freq, i2c_specs[setup->speed].rate);
-		return -EINVAL;
-	}
-
 	/*  Analog and Digital Filters */
 	af_delay_min =
 		(setup->analog_filter ?
@@ -469,13 +465,13 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 		 STM32F7_I2C_ANALOG_FILTER_DELAY_MAX : 0);
 	dnf_delay = setup->dnf * i2cclk;
 
-	sdadel_min = i2c_specs[setup->speed].hddat_min + setup->fall_time -
+	sdadel_min = specs->hddat_min + setup->fall_time -
 		af_delay_min - (setup->dnf + 3) * i2cclk;
 
-	sdadel_max = i2c_specs[setup->speed].vddat_max - setup->rise_time -
+	sdadel_max = specs->vddat_max - setup->rise_time -
 		af_delay_max - (setup->dnf + 4) * i2cclk;
 
-	scldel_min = setup->rise_time + i2c_specs[setup->speed].sudat_min;
+	scldel_min = setup->rise_time + specs->sudat_min;
 
 	if (sdadel_min < 0)
 		sdadel_min = 0;
@@ -530,8 +526,8 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 
 	tsync = af_delay_min + dnf_delay + (2 * i2cclk);
 	s = NULL;
-	clk_max = NSEC_PER_SEC / i2c_specs[setup->speed].rate_min;
-	clk_min = NSEC_PER_SEC / i2c_specs[setup->speed].rate_max;
+	clk_max = NSEC_PER_SEC / RATE_MIN(setup->speed_freq);
+	clk_min = NSEC_PER_SEC / setup->speed_freq;
 
 	/*
 	 * Among Prescaler possibilities discovered above figures out SCL Low
@@ -549,7 +545,7 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 		for (l = 0; l < STM32F7_SCLL_MAX; l++) {
 			u32 tscl_l = (l + 1) * prescaler + tsync;
 
-			if ((tscl_l < i2c_specs[setup->speed].l_min) ||
+			if ((tscl_l < specs->l_min) ||
 			    (i2cclk >=
 			     ((tscl_l - af_delay_min - dnf_delay) / 4))) {
 				continue;
@@ -561,7 +557,7 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 					setup->rise_time + setup->fall_time;
 
 				if ((tscl >= clk_min) && (tscl <= clk_max) &&
-				    (tscl_h >= i2c_specs[setup->speed].h_min) &&
+				    (tscl_h >= specs->h_min) &&
 				    (i2cclk < tscl_h)) {
 					int clk_error = tscl - i2cbus;
 
@@ -607,6 +603,17 @@ exit:
 	return ret;
 }
 
+static u32 stm32f7_get_lower_rate(u32 rate)
+{
+	int i = ARRAY_SIZE(stm32f7_i2c_specs);
+
+	while (--i)
+		if (stm32f7_i2c_specs[i].rate < rate)
+			break;
+
+	return stm32f7_i2c_specs[i].rate;
+}
+
 static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev,
 				    struct stm32f7_i2c_setup *setup)
 {
@@ -619,18 +626,15 @@ static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev,
 
 	i2c_parse_fw_timings(i2c_dev->dev, t, false);
 
-	if (t->bus_freq_hz >= I2C_MAX_FAST_MODE_PLUS_FREQ)
-		i2c_dev->speed = STM32_I2C_SPEED_FAST_PLUS;
-	else if (t->bus_freq_hz >= I2C_MAX_FAST_MODE_FREQ)
-		i2c_dev->speed = STM32_I2C_SPEED_FAST;
-	else
-		i2c_dev->speed = STM32_I2C_SPEED_STANDARD;
+	if (t->bus_freq_hz > I2C_MAX_FAST_MODE_PLUS_FREQ) {
+		dev_err(i2c_dev->dev, "Invalid bus speed (%i>%i)\n",
+			t->bus_freq_hz, I2C_MAX_FAST_MODE_PLUS_FREQ);
+		return -EINVAL;
+	}
 
+	setup->speed_freq = t->bus_freq_hz;
 	i2c_dev->setup.rise_time = t->scl_rise_ns;
 	i2c_dev->setup.fall_time = t->scl_fall_ns;
-
-	setup->speed = i2c_dev->speed;
-	setup->speed_freq = i2c_specs[setup->speed].rate;
 	setup->clock_src = clk_get_rate(i2c_dev->clk);
 
 	if (!setup->clock_src) {
@@ -644,17 +648,13 @@ static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev,
 		if (ret) {
 			dev_err(i2c_dev->dev,
 				"failed to compute I2C timings.\n");
-			if (i2c_dev->speed > STM32_I2C_SPEED_STANDARD) {
-				i2c_dev->speed--;
-				setup->speed = i2c_dev->speed;
-				setup->speed_freq =
-					i2c_specs[setup->speed].rate;
-				dev_warn(i2c_dev->dev,
-					 "downgrade I2C Speed Freq to (%i)\n",
-					 i2c_specs[setup->speed].rate);
-			} else {
+			if (setup->speed_freq <= I2C_MAX_STANDARD_MODE_FREQ)
 				break;
-			}
+			setup->speed_freq =
+				stm32f7_get_lower_rate(setup->speed_freq);
+			dev_warn(i2c_dev->dev,
+				 "downgrade I2C Speed Freq to (%i)\n",
+				 setup->speed_freq);
 		}
 	} while (ret);
 
@@ -663,13 +663,15 @@ static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev,
 		return ret;
 	}
 
-	dev_dbg(i2c_dev->dev, "I2C Speed(%i), Freq(%i), Clk Source(%i)\n",
-		setup->speed, setup->speed_freq, setup->clock_src);
+	dev_dbg(i2c_dev->dev, "I2C Speed(%i), Clk Source(%i)\n",
+		setup->speed_freq, setup->clock_src);
 	dev_dbg(i2c_dev->dev, "I2C Rise(%i) and Fall(%i) Time\n",
 		setup->rise_time, setup->fall_time);
 	dev_dbg(i2c_dev->dev, "I2C Analog Filter(%s), DNF(%i)\n",
 		(setup->analog_filter ? "On" : "Off"), setup->dnf);
 
+	i2c_dev->bus_rate = setup->speed_freq;
+
 	return 0;
 }
 
@@ -1462,7 +1464,8 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
 
 	/* NACK received */
 	if (status & STM32F7_I2C_ISR_NACKF) {
-		dev_dbg(i2c_dev->dev, "<%s>: Receive NACK\n", __func__);
+		dev_dbg(i2c_dev->dev, "<%s>: Receive NACK (addr %x)\n",
+			__func__, f7_msg->addr);
 		writel_relaxed(STM32F7_I2C_ICR_NACKCF, base + STM32F7_I2C_ICR);
 		f7_msg->result = -ENXIO;
 	}
@@ -1866,7 +1869,7 @@ static int stm32f7_i2c_write_fm_plus_bits(struct stm32f7_i2c_dev *i2c_dev,
 {
 	int ret;
 
-	if (i2c_dev->speed != STM32_I2C_SPEED_FAST_PLUS ||
+	if (i2c_dev->bus_rate <= I2C_MAX_FAST_MODE_FREQ ||
 	    IS_ERR_OR_NULL(i2c_dev->regmap))
 		/* Optional */
 		return 0;
@@ -1940,8 +1943,7 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 	if (!i2c_dev)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c_dev->base = devm_ioremap_resource(&pdev->dev, res);
+	i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(i2c_dev->base))
 		return PTR_ERR(i2c_dev->base);
 	phy_addr = (dma_addr_t)res->start;
@@ -1967,7 +1969,8 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 
 	i2c_dev->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(i2c_dev->clk)) {
-		dev_err(&pdev->dev, "Error: Missing controller clock\n");
+		if (PTR_ERR(i2c_dev->clk) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Failed to get controller clock\n");
 		return PTR_ERR(i2c_dev->clk);
 	}
 
@@ -1979,8 +1982,10 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 
 	rst = devm_reset_control_get(&pdev->dev, NULL);
 	if (IS_ERR(rst)) {
-		dev_err(&pdev->dev, "Error: Missing controller reset\n");
 		ret = PTR_ERR(rst);
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Error: Missing reset ctrl\n");
+
 		goto clk_free;
 	}
 	reset_control_assert(rst);
@@ -2020,7 +2025,8 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 	if (ret)
 		goto clk_free;
 
-	if (i2c_dev->speed == STM32_I2C_SPEED_FAST_PLUS) {
+	/* Setup Fast mode plus if necessary */
+	if (i2c_dev->bus_rate > I2C_MAX_FAST_MODE_FREQ) {
 		ret = stm32f7_i2c_setup_fm_plus_bits(pdev, i2c_dev);
 		if (ret)
 			goto clk_free;
diff --git a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c
index ba6b60caa45e..64d739baf480 100644
--- a/drivers/i2c/busses/i2c-stu300.c
+++ b/drivers/i2c/busses/i2c-stu300.c
@@ -860,7 +860,6 @@ static int stu300_probe(struct platform_device *pdev)
 {
 	struct stu300_dev *dev;
 	struct i2c_adapter *adap;
-	struct resource *res;
 	int bus_nr;
 	int ret = 0;
 
@@ -876,8 +875,7 @@ static int stu300_probe(struct platform_device *pdev)
 	}
 
 	dev->pdev = pdev;
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	dev->virtbase = devm_ioremap_resource(&pdev->dev, res);
+	dev->virtbase = devm_platform_ioremap_resource(pdev, 0);
 	dev_dbg(&pdev->dev, "initialize bus device I2C%d on virtual "
 		"base %p\n", bus_nr, dev->virtbase);
 	if (IS_ERR(dev->virtbase))
diff --git a/drivers/i2c/busses/i2c-sun6i-p2wi.c b/drivers/i2c/busses/i2c-sun6i-p2wi.c
index e5293f0b3318..2f6f6468214d 100644
--- a/drivers/i2c/busses/i2c-sun6i-p2wi.c
+++ b/drivers/i2c/busses/i2c-sun6i-p2wi.c
@@ -187,7 +187,6 @@ static int p2wi_probe(struct platform_device *pdev)
 	struct device_node *childnp;
 	unsigned long parent_clk_freq;
 	u32 clk_freq = I2C_MAX_STANDARD_MODE_FREQ;
-	struct resource *r;
 	struct p2wi *p2wi;
 	u32 slave_addr;
 	int clk_div;
@@ -231,17 +230,14 @@ static int p2wi_probe(struct platform_device *pdev)
 		p2wi->slave_addr = slave_addr;
 	}
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	p2wi->regs = devm_ioremap_resource(dev, r);
+	p2wi->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(p2wi->regs))
 		return PTR_ERR(p2wi->regs);
 
 	strlcpy(p2wi->adapter.name, pdev->name, sizeof(p2wi->adapter.name));
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(dev, "failed to retrieve irq: %d\n", irq);
+	if (irq < 0)
 		return irq;
-	}
 
 	p2wi->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(p2wi->clk)) {
diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c
index 9099d0a67ace..c9a3dba6a75d 100644
--- a/drivers/i2c/busses/i2c-synquacer.c
+++ b/drivers/i2c/busses/i2c-synquacer.c
@@ -536,7 +536,6 @@ static const struct i2c_adapter synquacer_i2c_ops = {
 static int synquacer_i2c_probe(struct platform_device *pdev)
 {
 	struct synquacer_i2c *i2c;
-	struct resource *r;
 	u32 bus_speed;
 	int ret;
 
@@ -574,16 +573,13 @@ static int synquacer_i2c_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	i2c->base = devm_ioremap_resource(&pdev->dev, r);
+	i2c->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(i2c->base))
 		return PTR_ERR(i2c->base);
 
 	i2c->irq = platform_get_irq(pdev, 0);
-	if (i2c->irq < 0) {
-		dev_err(&pdev->dev, "no IRQ resource found\n");
+	if (i2c->irq < 0)
 		return -ENODEV;
-	}
 
 	ret = devm_request_irq(&pdev->dev, i2c->irq, synquacer_i2c_isr,
 			       0, dev_name(&pdev->dev), i2c);
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index 4c4d17ddc96b..15772964a05f 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -6,6 +6,7 @@
  * Author: Colin Cross <ccross@android.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/dmaengine.h>
@@ -29,16 +30,17 @@
 #define BYTES_PER_FIFO_WORD 4
 
 #define I2C_CNFG				0x000
-#define I2C_CNFG_DEBOUNCE_CNT_SHIFT		12
+#define I2C_CNFG_DEBOUNCE_CNT			GENMASK(14, 12)
 #define I2C_CNFG_PACKET_MODE_EN			BIT(10)
 #define I2C_CNFG_NEW_MASTER_FSM			BIT(11)
 #define I2C_CNFG_MULTI_MASTER_MODE		BIT(17)
-#define I2C_STATUS				0x01C
+#define I2C_STATUS				0x01c
 #define I2C_SL_CNFG				0x020
 #define I2C_SL_CNFG_NACK			BIT(1)
 #define I2C_SL_CNFG_NEWSL			BIT(2)
 #define I2C_SL_ADDR1				0x02c
 #define I2C_SL_ADDR2				0x030
+#define I2C_TLOW_SEXT				0x034
 #define I2C_TX_FIFO				0x050
 #define I2C_RX_FIFO				0x054
 #define I2C_PACKET_TRANSFER_STATUS		0x058
@@ -48,10 +50,8 @@
 #define I2C_FIFO_CONTROL_TX_TRIG(x)		(((x) - 1) << 5)
 #define I2C_FIFO_CONTROL_RX_TRIG(x)		(((x) - 1) << 2)
 #define I2C_FIFO_STATUS				0x060
-#define I2C_FIFO_STATUS_TX_MASK			0xF0
-#define I2C_FIFO_STATUS_TX_SHIFT		4
-#define I2C_FIFO_STATUS_RX_MASK			0x0F
-#define I2C_FIFO_STATUS_RX_SHIFT		0
+#define I2C_FIFO_STATUS_TX			GENMASK(7, 4)
+#define I2C_FIFO_STATUS_RX			GENMASK(3, 0)
 #define I2C_INT_MASK				0x064
 #define I2C_INT_STATUS				0x068
 #define I2C_INT_BUS_CLR_DONE			BIT(11)
@@ -61,7 +61,8 @@
 #define I2C_INT_TX_FIFO_DATA_REQ		BIT(1)
 #define I2C_INT_RX_FIFO_DATA_REQ		BIT(0)
 #define I2C_CLK_DIVISOR				0x06c
-#define I2C_CLK_DIVISOR_STD_FAST_MODE_SHIFT	16
+#define I2C_CLK_DIVISOR_STD_FAST_MODE		GENMASK(31, 16)
+#define I2C_CLK_DIVISOR_HSMODE			GENMASK(15, 0)
 
 #define DVC_CTRL_REG1				0x000
 #define DVC_CTRL_REG1_INTR_EN			BIT(10)
@@ -77,10 +78,11 @@
 #define I2C_ERR_UNKNOWN_INTERRUPT		BIT(2)
 #define I2C_ERR_RX_BUFFER_OVERFLOW		BIT(3)
 
-#define PACKET_HEADER0_HEADER_SIZE_SHIFT	28
-#define PACKET_HEADER0_PACKET_ID_SHIFT		16
-#define PACKET_HEADER0_CONT_ID_SHIFT		12
-#define PACKET_HEADER0_PROTOCOL_I2C		BIT(4)
+#define PACKET_HEADER0_HEADER_SIZE		GENMASK(29, 28)
+#define PACKET_HEADER0_PACKET_ID		GENMASK(23, 16)
+#define PACKET_HEADER0_CONT_ID			GENMASK(15, 12)
+#define PACKET_HEADER0_PROTOCOL			GENMASK(7, 4)
+#define PACKET_HEADER0_PROTOCOL_I2C		1
 
 #define I2C_HEADER_CONT_ON_NAK			BIT(21)
 #define I2C_HEADER_READ				BIT(19)
@@ -91,21 +93,35 @@
 #define I2C_HEADER_SLAVE_ADDR_SHIFT		1
 
 #define I2C_BUS_CLEAR_CNFG			0x084
-#define I2C_BC_SCLK_THRESHOLD			9
-#define I2C_BC_SCLK_THRESHOLD_SHIFT		16
+#define I2C_BC_SCLK_THRESHOLD			GENMASK(23, 16)
 #define I2C_BC_STOP_COND			BIT(2)
 #define I2C_BC_TERMINATE			BIT(1)
 #define I2C_BC_ENABLE				BIT(0)
 #define I2C_BUS_CLEAR_STATUS			0x088
 #define I2C_BC_STATUS				BIT(0)
 
-#define I2C_CONFIG_LOAD				0x08C
+#define I2C_CONFIG_LOAD				0x08c
 #define I2C_MSTR_CONFIG_LOAD			BIT(0)
 
 #define I2C_CLKEN_OVERRIDE			0x090
 #define I2C_MST_CORE_CLKEN_OVR			BIT(0)
 
-#define I2C_CONFIG_LOAD_TIMEOUT			1000000
+#define I2C_INTERFACE_TIMING_0			0x094
+#define  I2C_INTERFACE_TIMING_THIGH		GENMASK(13, 8)
+#define  I2C_INTERFACE_TIMING_TLOW		GENMASK(5, 0)
+#define I2C_INTERFACE_TIMING_1			0x098
+#define  I2C_INTERFACE_TIMING_TBUF		GENMASK(29, 24)
+#define  I2C_INTERFACE_TIMING_TSU_STO		GENMASK(21, 16)
+#define  I2C_INTERFACE_TIMING_THD_STA		GENMASK(13, 8)
+#define  I2C_INTERFACE_TIMING_TSU_STA		GENMASK(5, 0)
+
+#define I2C_HS_INTERFACE_TIMING_0		0x09c
+#define  I2C_HS_INTERFACE_TIMING_THIGH		GENMASK(13, 8)
+#define  I2C_HS_INTERFACE_TIMING_TLOW		GENMASK(5, 0)
+#define I2C_HS_INTERFACE_TIMING_1		0x0a0
+#define  I2C_HS_INTERFACE_TIMING_TSU_STO	GENMASK(21, 16)
+#define  I2C_HS_INTERFACE_TIMING_THD_STA	GENMASK(13, 8)
+#define  I2C_HS_INTERFACE_TIMING_TSU_STA	GENMASK(5, 0)
 
 #define I2C_MST_FIFO_CONTROL			0x0b4
 #define I2C_MST_FIFO_CONTROL_RX_FLUSH		BIT(0)
@@ -114,14 +130,11 @@
 #define I2C_MST_FIFO_CONTROL_TX_TRIG(x)		(((x) - 1) << 16)
 
 #define I2C_MST_FIFO_STATUS			0x0b8
-#define I2C_MST_FIFO_STATUS_RX_MASK		0xff
-#define I2C_MST_FIFO_STATUS_RX_SHIFT		0
-#define I2C_MST_FIFO_STATUS_TX_MASK		0xff0000
-#define I2C_MST_FIFO_STATUS_TX_SHIFT		16
+#define I2C_MST_FIFO_STATUS_TX			GENMASK(23, 16)
+#define I2C_MST_FIFO_STATUS_RX			GENMASK(7, 0)
 
-#define I2C_INTERFACE_TIMING_0			0x94
-#define I2C_THIGH_SHIFT				8
-#define I2C_INTERFACE_TIMING_1			0x98
+/* configuration load timeout in microseconds */
+#define I2C_CONFIG_LOAD_TIMEOUT			1000000
 
 /* Packet header size in bytes */
 #define I2C_PACKET_HEADER_SIZE			12
@@ -230,6 +243,7 @@ struct tegra_i2c_hw_feature {
  * @cont_id: I2C controller ID, used for packet header
  * @irq: IRQ number of transfer complete interrupt
  * @is_dvc: identifies the DVC I2C controller, has a different register layout
+ * @is_vi: identifies the VI I2C controller, has a different register layout
  * @msg_complete: transfer completion notifier
  * @msg_err: error code for completed message
  * @msg_buf: pointer to current message data
@@ -253,12 +267,14 @@ struct tegra_i2c_dev {
 	struct i2c_adapter adapter;
 	struct clk *div_clk;
 	struct clk *fast_clk;
+	struct clk *slow_clk;
 	struct reset_control *rst;
 	void __iomem *base;
 	phys_addr_t base_phys;
 	int cont_id;
 	int irq;
 	int is_dvc;
+	bool is_vi;
 	struct completion msg_complete;
 	int msg_err;
 	u8 *msg_buf;
@@ -297,6 +313,8 @@ static unsigned long tegra_i2c_reg_addr(struct tegra_i2c_dev *i2c_dev,
 {
 	if (i2c_dev->is_dvc)
 		reg += (reg >= I2C_TX_FIFO) ? 0x10 : 0x40;
+	else if (i2c_dev->is_vi)
+		reg = 0xc00 + (reg << 2);
 	return reg;
 }
 
@@ -495,12 +513,10 @@ static int tegra_i2c_empty_rx_fifo(struct tegra_i2c_dev *i2c_dev)
 
 	if (i2c_dev->hw->has_mst_fifo) {
 		val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS);
-		rx_fifo_avail = (val & I2C_MST_FIFO_STATUS_RX_MASK) >>
-			I2C_MST_FIFO_STATUS_RX_SHIFT;
+		rx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_RX, val);
 	} else {
 		val = i2c_readl(i2c_dev, I2C_FIFO_STATUS);
-		rx_fifo_avail = (val & I2C_FIFO_STATUS_RX_MASK) >>
-			I2C_FIFO_STATUS_RX_SHIFT;
+		rx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_RX, val);
 	}
 
 	/* Rounds down to not include partial word at the end of buf */
@@ -551,12 +567,10 @@ static int tegra_i2c_fill_tx_fifo(struct tegra_i2c_dev *i2c_dev)
 
 	if (i2c_dev->hw->has_mst_fifo) {
 		val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS);
-		tx_fifo_avail = (val & I2C_MST_FIFO_STATUS_TX_MASK) >>
-			I2C_MST_FIFO_STATUS_TX_SHIFT;
+		tx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_TX, val);
 	} else {
 		val = i2c_readl(i2c_dev, I2C_FIFO_STATUS);
-		tx_fifo_avail = (val & I2C_FIFO_STATUS_TX_MASK) >>
-			I2C_FIFO_STATUS_TX_SHIFT;
+		tx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_TX, val);
 	}
 
 	/* Rounds down to not include partial word at the end of buf */
@@ -650,6 +664,14 @@ static int __maybe_unused tegra_i2c_runtime_resume(struct device *dev)
 		}
 	}
 
+	if (i2c_dev->slow_clk) {
+		ret = clk_enable(i2c_dev->slow_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to enable slow clock: %d\n", ret);
+			return ret;
+		}
+	}
+
 	ret = clk_enable(i2c_dev->div_clk);
 	if (ret < 0) {
 		dev_err(i2c_dev->dev,
@@ -666,6 +688,10 @@ static int __maybe_unused tegra_i2c_runtime_suspend(struct device *dev)
 	struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev);
 
 	clk_disable(i2c_dev->div_clk);
+
+	if (i2c_dev->slow_clk)
+		clk_disable(i2c_dev->slow_clk);
+
 	if (!i2c_dev->hw->has_single_clk_source)
 		clk_disable(i2c_dev->fast_clk);
 
@@ -703,6 +729,35 @@ static int tegra_i2c_wait_for_config_load(struct tegra_i2c_dev *i2c_dev)
 	return 0;
 }
 
+static void tegra_i2c_vi_init(struct tegra_i2c_dev *i2c_dev)
+{
+	u32 value;
+
+	value = FIELD_PREP(I2C_INTERFACE_TIMING_THIGH, 2) |
+		FIELD_PREP(I2C_INTERFACE_TIMING_TLOW, 4);
+	i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_0);
+
+	value = FIELD_PREP(I2C_INTERFACE_TIMING_TBUF, 4) |
+		FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STO, 7) |
+		FIELD_PREP(I2C_INTERFACE_TIMING_THD_STA, 4) |
+		FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STA, 4);
+	i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_1);
+
+	value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_THIGH, 3) |
+		FIELD_PREP(I2C_HS_INTERFACE_TIMING_TLOW, 8);
+	i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_0);
+
+	value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STO, 11) |
+		FIELD_PREP(I2C_HS_INTERFACE_TIMING_THD_STA, 11) |
+		FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STA, 11);
+	i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_1);
+
+	value = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND;
+	i2c_writel(i2c_dev, value, I2C_BUS_CLEAR_CNFG);
+
+	i2c_writel(i2c_dev, 0x0, I2C_TLOW_SEXT);
+}
+
 static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit)
 {
 	u32 val;
@@ -719,7 +774,7 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit)
 		tegra_dvc_init(i2c_dev);
 
 	val = I2C_CNFG_NEW_MASTER_FSM | I2C_CNFG_PACKET_MODE_EN |
-		(0x2 << I2C_CNFG_DEBOUNCE_CNT_SHIFT);
+	      FIELD_PREP(I2C_CNFG_DEBOUNCE_CNT, 2);
 
 	if (i2c_dev->hw->has_multi_master_mode)
 		val |= I2C_CNFG_MULTI_MASTER_MODE;
@@ -727,10 +782,14 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit)
 	i2c_writel(i2c_dev, val, I2C_CNFG);
 	i2c_writel(i2c_dev, 0, I2C_INT_MASK);
 
+	if (i2c_dev->is_vi)
+		tegra_i2c_vi_init(i2c_dev);
+
 	/* Make sure clock divisor programmed correctly */
-	clk_divisor = i2c_dev->hw->clk_divisor_hs_mode;
-	clk_divisor |= i2c_dev->clk_divisor_non_hs_mode <<
-					I2C_CLK_DIVISOR_STD_FAST_MODE_SHIFT;
+	clk_divisor = FIELD_PREP(I2C_CLK_DIVISOR_HSMODE,
+				 i2c_dev->hw->clk_divisor_hs_mode) |
+		      FIELD_PREP(I2C_CLK_DIVISOR_STD_FAST_MODE,
+				 i2c_dev->clk_divisor_non_hs_mode);
 	i2c_writel(i2c_dev, clk_divisor, I2C_CLK_DIVISOR);
 
 	if (i2c_dev->bus_clk_rate > I2C_MAX_STANDARD_MODE_FREQ &&
@@ -745,7 +804,8 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit)
 	}
 
 	if (i2c_dev->hw->has_interface_timing_reg) {
-		val = (thigh << I2C_THIGH_SHIFT) | tlow;
+		val = FIELD_PREP(I2C_INTERFACE_TIMING_THIGH, thigh) |
+		      FIELD_PREP(I2C_INTERFACE_TIMING_TLOW, tlow);
 		i2c_writel(i2c_dev, val, I2C_INTERFACE_TIMING_0);
 	}
 
@@ -768,7 +828,7 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit)
 		}
 	}
 
-	if (!i2c_dev->is_dvc) {
+	if (!i2c_dev->is_dvc && !i2c_dev->is_vi) {
 		u32 sl_cfg = i2c_readl(i2c_dev, I2C_SL_CNFG);
 
 		sl_cfg |= I2C_SL_CNFG_NACK | I2C_SL_CNFG_NEWSL;
@@ -996,14 +1056,13 @@ tegra_i2c_poll_completion_timeout(struct tegra_i2c_dev *i2c_dev,
 	do {
 		u32 status = i2c_readl(i2c_dev, I2C_INT_STATUS);
 
-		if (status) {
+		if (status)
 			tegra_i2c_isr(i2c_dev->irq, i2c_dev);
 
-			if (completion_done(complete)) {
-				s64 delta = ktime_ms_delta(ktimeout, ktime);
+		if (completion_done(complete)) {
+			s64 delta = ktime_ms_delta(ktimeout, ktime);
 
-				return msecs_to_jiffies(delta) ?: 1;
-			}
+			return msecs_to_jiffies(delta) ?: 1;
 		}
 
 		ktime = ktime_get();
@@ -1030,14 +1089,18 @@ tegra_i2c_wait_completion_timeout(struct tegra_i2c_dev *i2c_dev,
 		disable_irq(i2c_dev->irq);
 
 		/*
-		 * There is a chance that completion may happen after IRQ
-		 * synchronization, which is done by disable_irq().
+		 * Under some rare circumstances (like running KASAN +
+		 * NFS root) CPU, which handles interrupt, may stuck in
+		 * uninterruptible state for a significant time.  In this
+		 * case we will get timeout if I2C transfer is running on
+		 * a sibling CPU, despite of IRQ being raised.
+		 *
+		 * In order to handle this rare condition, the IRQ status
+		 * needs to be checked after timeout.
 		 */
-		if (ret == 0 && completion_done(complete)) {
-			dev_warn(i2c_dev->dev,
-				 "completion done after timeout\n");
-			ret = 1;
-		}
+		if (ret == 0)
+			ret = tegra_i2c_poll_completion_timeout(i2c_dev,
+								complete, 0);
 	}
 
 	return ret;
@@ -1051,8 +1114,8 @@ static int tegra_i2c_issue_bus_clear(struct i2c_adapter *adap)
 	u32 reg;
 
 	reinit_completion(&i2c_dev->msg_complete);
-	reg = (I2C_BC_SCLK_THRESHOLD << I2C_BC_SCLK_THRESHOLD_SHIFT) |
-	      I2C_BC_STOP_COND | I2C_BC_TERMINATE;
+	reg = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND |
+	      I2C_BC_TERMINATE;
 	i2c_writel(i2c_dev, reg, I2C_BUS_CLEAR_CNFG);
 	if (i2c_dev->hw->has_config_load_reg) {
 		err = tegra_i2c_wait_for_config_load(i2c_dev);
@@ -1145,10 +1208,11 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev,
 		}
 	}
 
-	packet_header = (0 << PACKET_HEADER0_HEADER_SIZE_SHIFT) |
-			PACKET_HEADER0_PROTOCOL_I2C |
-			(i2c_dev->cont_id << PACKET_HEADER0_CONT_ID_SHIFT) |
-			(1 << PACKET_HEADER0_PACKET_ID_SHIFT);
+	packet_header = FIELD_PREP(PACKET_HEADER0_HEADER_SIZE, 0) |
+			FIELD_PREP(PACKET_HEADER0_PROTOCOL,
+				   PACKET_HEADER0_PROTOCOL_I2C) |
+			FIELD_PREP(PACKET_HEADER0_CONT_ID, i2c_dev->cont_id) |
+			FIELD_PREP(PACKET_HEADER0_PACKET_ID, 1);
 	if (dma && !i2c_dev->msg_read)
 		*buffer++ = packet_header;
 	else
@@ -1216,6 +1280,15 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev,
 		time_left = tegra_i2c_wait_completion_timeout(
 				i2c_dev, &i2c_dev->dma_complete, xfer_time);
 
+		/*
+		 * Synchronize DMA first, since dmaengine_terminate_sync()
+		 * performs synchronization after the transfer's termination
+		 * and we want to get a completion if transfer succeeded.
+		 */
+		dmaengine_synchronize(i2c_dev->msg_read ?
+				      i2c_dev->rx_dma_chan :
+				      i2c_dev->tx_dma_chan);
+
 		dmaengine_terminate_sync(i2c_dev->msg_read ?
 					 i2c_dev->rx_dma_chan :
 					 i2c_dev->tx_dma_chan);
@@ -1544,6 +1617,7 @@ static const struct tegra_i2c_hw_feature tegra194_i2c_hw = {
 static const struct of_device_id tegra_i2c_of_match[] = {
 	{ .compatible = "nvidia,tegra194-i2c", .data = &tegra194_i2c_hw, },
 	{ .compatible = "nvidia,tegra186-i2c", .data = &tegra186_i2c_hw, },
+	{ .compatible = "nvidia,tegra210-i2c-vi", .data = &tegra210_i2c_hw, },
 	{ .compatible = "nvidia,tegra210-i2c", .data = &tegra210_i2c_hw, },
 	{ .compatible = "nvidia,tegra124-i2c", .data = &tegra124_i2c_hw, },
 	{ .compatible = "nvidia,tegra114-i2c", .data = &tegra114_i2c_hw, },
@@ -1556,6 +1630,7 @@ MODULE_DEVICE_TABLE(of, tegra_i2c_of_match);
 
 static int tegra_i2c_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
 	struct tegra_i2c_dev *i2c_dev;
 	struct resource *res;
 	struct clk *div_clk;
@@ -1611,6 +1686,8 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 	i2c_dev->hw = of_device_get_match_data(&pdev->dev);
 	i2c_dev->is_dvc = of_device_is_compatible(pdev->dev.of_node,
 						  "nvidia,tegra20-i2c-dvc");
+	i2c_dev->is_vi = of_device_is_compatible(dev->of_node,
+						 "nvidia,tegra210-i2c-vi");
 	i2c_dev->adapter.quirks = i2c_dev->hw->quirks;
 	i2c_dev->dma_buf_size = i2c_dev->adapter.quirks->max_write_len +
 				I2C_PACKET_HEADER_SIZE;
@@ -1626,6 +1703,17 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 		i2c_dev->fast_clk = fast_clk;
 	}
 
+	if (i2c_dev->is_vi) {
+		i2c_dev->slow_clk = devm_clk_get(dev, "slow");
+		if (IS_ERR(i2c_dev->slow_clk)) {
+			if (PTR_ERR(i2c_dev->slow_clk) != -EPROBE_DEFER)
+				dev_err(dev, "failed to get slow clock: %ld\n",
+					PTR_ERR(i2c_dev->slow_clk));
+
+			return PTR_ERR(i2c_dev->slow_clk);
+		}
+	}
+
 	platform_set_drvdata(pdev, i2c_dev);
 
 	if (!i2c_dev->hw->has_single_clk_source) {
@@ -1636,6 +1724,14 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 		}
 	}
 
+	if (i2c_dev->slow_clk) {
+		ret = clk_prepare(i2c_dev->slow_clk);
+		if (ret < 0) {
+			dev_err(dev, "failed to prepare slow clock: %d\n", ret);
+			goto unprepare_fast_clk;
+		}
+	}
+
 	if (i2c_dev->bus_clk_rate > I2C_MAX_FAST_MODE_FREQ &&
 	    i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_PLUS_FREQ)
 		i2c_dev->clk_divisor_non_hs_mode =
@@ -1651,7 +1747,7 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 	ret = clk_prepare(i2c_dev->div_clk);
 	if (ret < 0) {
 		dev_err(i2c_dev->dev, "Clock prepare failed %d\n", ret);
-		goto unprepare_fast_clk;
+		goto unprepare_slow_clk;
 	}
 
 	pm_runtime_irq_safe(&pdev->dev);
@@ -1694,8 +1790,8 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 
 	irq_set_status_flags(i2c_dev->irq, IRQ_NOAUTOEN);
 
-	ret = devm_request_irq(&pdev->dev, i2c_dev->irq,
-			       tegra_i2c_isr, 0, dev_name(&pdev->dev), i2c_dev);
+	ret = devm_request_irq(&pdev->dev, i2c_dev->irq, tegra_i2c_isr,
+			       IRQF_NO_SUSPEND, dev_name(&pdev->dev), i2c_dev);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to request irq %i\n", i2c_dev->irq);
 		goto release_dma;
@@ -1738,6 +1834,10 @@ disable_rpm:
 unprepare_div_clk:
 	clk_unprepare(i2c_dev->div_clk);
 
+unprepare_slow_clk:
+	if (i2c_dev->is_vi)
+		clk_unprepare(i2c_dev->slow_clk);
+
 unprepare_fast_clk:
 	if (!i2c_dev->hw->has_single_clk_source)
 		clk_unprepare(i2c_dev->fast_clk);
@@ -1759,6 +1859,10 @@ static int tegra_i2c_remove(struct platform_device *pdev)
 		tegra_i2c_runtime_suspend(&pdev->dev);
 
 	clk_unprepare(i2c_dev->div_clk);
+
+	if (i2c_dev->slow_clk)
+		clk_unprepare(i2c_dev->slow_clk);
+
 	if (!i2c_dev->hw->has_single_clk_source)
 		clk_unprepare(i2c_dev->fast_clk);
 
@@ -1769,15 +1873,14 @@ static int tegra_i2c_remove(struct platform_device *pdev)
 static int __maybe_unused tegra_i2c_suspend(struct device *dev)
 {
 	struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev);
-	int err;
+	int err = 0;
 
 	i2c_mark_adapter_suspended(&i2c_dev->adapter);
 
-	err = pm_runtime_force_suspend(dev);
-	if (err < 0)
-		return err;
+	if (!pm_runtime_status_suspended(dev))
+		err = tegra_i2c_runtime_suspend(dev);
 
-	return 0;
+	return err;
 }
 
 static int __maybe_unused tegra_i2c_resume(struct device *dev)
@@ -1785,6 +1888,10 @@ static int __maybe_unused tegra_i2c_resume(struct device *dev)
 	struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev);
 	int err;
 
+	/*
+	 * We need to ensure that clocks are enabled so that registers can be
+	 * restored in tegra_i2c_init().
+	 */
 	err = tegra_i2c_runtime_resume(dev);
 	if (err)
 		return err;
@@ -1793,13 +1900,16 @@ static int __maybe_unused tegra_i2c_resume(struct device *dev)
 	if (err)
 		return err;
 
-	err = tegra_i2c_runtime_suspend(dev);
-	if (err)
-		return err;
-
-	err = pm_runtime_force_resume(dev);
-	if (err < 0)
-		return err;
+	/*
+	 * In case we are runtime suspended, disable clocks again so that we
+	 * don't unbalance the clock reference counts during the next runtime
+	 * resume transition.
+	 */
+	if (pm_runtime_status_suspended(dev)) {
+		err = tegra_i2c_runtime_suspend(dev);
+		if (err)
+			return err;
+	}
 
 	i2c_mark_adapter_resumed(&i2c_dev->adapter);
 
diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c
index 2b258d54d68c..cb4666c54a23 100644
--- a/drivers/i2c/busses/i2c-uniphier-f.c
+++ b/drivers/i2c/busses/i2c-uniphier-f.c
@@ -529,10 +529,8 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->membase);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(dev, "failed to get IRQ number\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed))
 		bus_speed = I2C_MAX_STANDARD_MODE_FREQ;
diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c
index 668b1fa2b0ef..ee00a44bf4c7 100644
--- a/drivers/i2c/busses/i2c-uniphier.c
+++ b/drivers/i2c/busses/i2c-uniphier.c
@@ -324,10 +324,8 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->membase);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(dev, "failed to get IRQ number\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed))
 		bus_speed = I2C_MAX_STANDARD_MODE_FREQ;
diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c
index 391c878a7cdc..f2241cedf5d3 100644
--- a/drivers/i2c/busses/i2c-xlp9xx.c
+++ b/drivers/i2c/busses/i2c-xlp9xx.c
@@ -506,23 +506,19 @@ static int xlp9xx_i2c_smbus_setup(struct xlp9xx_i2c_dev *priv,
 static int xlp9xx_i2c_probe(struct platform_device *pdev)
 {
 	struct xlp9xx_i2c_dev *priv;
-	struct resource *res;
 	int err = 0;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->base = devm_ioremap_resource(&pdev->dev, res);
+	priv->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->base))
 		return PTR_ERR(priv->base);
 
 	priv->irq = platform_get_irq(pdev, 0);
-	if (priv->irq <= 0) {
-		dev_err(&pdev->dev, "invalid irq!\n");
+	if (priv->irq <= 0)
 		return priv->irq;
-	}
 	/* SMBAlert irq */
 	priv->alert_data.irq = platform_get_irq(pdev, 1);
 	if (priv->alert_data.irq <= 0)
diff --git a/drivers/i2c/busses/i2c-xlr.c b/drivers/i2c/busses/i2c-xlr.c
index 282f161a8b08..126d1393e548 100644
--- a/drivers/i2c/busses/i2c-xlr.c
+++ b/drivers/i2c/busses/i2c-xlr.c
@@ -362,7 +362,6 @@ static int xlr_i2c_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
 	struct xlr_i2c_private  *priv;
-	struct resource *res;
 	struct clk *clk;
 	unsigned long clk_rate;
 	unsigned long clk_div;
@@ -380,8 +379,7 @@ static int xlr_i2c_probe(struct platform_device *pdev)
 	else
 		priv->cfg = &xlr_i2c_config_default;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->iobase = devm_ioremap_resource(&pdev->dev, res);
+	priv->iobase = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->iobase))
 		return PTR_ERR(priv->iobase);
 
diff --git a/drivers/i2c/busses/i2c-zx2967.c b/drivers/i2c/busses/i2c-zx2967.c
index 5f3318559b8d..8db9519695a6 100644
--- a/drivers/i2c/busses/i2c-zx2967.c
+++ b/drivers/i2c/busses/i2c-zx2967.c
@@ -502,7 +502,6 @@ static int zx2967_i2c_probe(struct platform_device *pdev)
 {
 	struct zx2967_i2c *i2c;
 	void __iomem *reg_base;
-	struct resource *res;
 	struct clk *clk;
 	int ret;
 
@@ -510,8 +509,7 @@ static int zx2967_i2c_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	reg_base = devm_ioremap_resource(&pdev->dev, res);
+	reg_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(reg_base))
 		return PTR_ERR(reg_base);
 
diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c
index c8f42f2037cb..2ade99b105b9 100644
--- a/drivers/i2c/i2c-core-acpi.c
+++ b/drivers/i2c/i2c-core-acpi.c
@@ -468,16 +468,12 @@ struct notifier_block i2c_acpi_notifier = {
 struct i2c_client *i2c_acpi_new_device(struct device *dev, int index,
 				       struct i2c_board_info *info)
 {
+	struct acpi_device *adev = ACPI_COMPANION(dev);
 	struct i2c_acpi_lookup lookup;
 	struct i2c_adapter *adapter;
-	struct acpi_device *adev;
 	LIST_HEAD(resource_list);
 	int ret;
 
-	adev = ACPI_COMPANION(dev);
-	if (!adev)
-		return ERR_PTR(-EINVAL);
-
 	memset(&lookup, 0, sizeof(lookup));
 	lookup.info = info;
 	lookup.device_handle = acpi_device_handle(adev);
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index 1f1442dfcad7..d1f278f73011 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -1598,6 +1598,18 @@ void i2c_del_adapter(struct i2c_adapter *adap)
 }
 EXPORT_SYMBOL(i2c_del_adapter);
 
+static void i2c_parse_timing(struct device *dev, char *prop_name, u32 *cur_val_p,
+			    u32 def_val, bool use_def)
+{
+	int ret;
+
+	ret = device_property_read_u32(dev, prop_name, cur_val_p);
+	if (ret && use_def)
+		*cur_val_p = def_val;
+
+	dev_dbg(dev, "%s: %u\n", prop_name, *cur_val_p);
+}
+
 /**
  * i2c_parse_fw_timings - get I2C related timing parameters from firmware
  * @dev: The device to scan for I2C timing properties
@@ -1616,49 +1628,28 @@ EXPORT_SYMBOL(i2c_del_adapter);
  */
 void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults)
 {
-	int ret;
-
-	ret = device_property_read_u32(dev, "clock-frequency", &t->bus_freq_hz);
-	if (ret && use_defaults)
-		t->bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ;
-
-	ret = device_property_read_u32(dev, "i2c-scl-rising-time-ns", &t->scl_rise_ns);
-	if (ret && use_defaults) {
-		if (t->bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ)
-			t->scl_rise_ns = 1000;
-		else if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ)
-			t->scl_rise_ns = 300;
-		else
-			t->scl_rise_ns = 120;
-	}
-
-	ret = device_property_read_u32(dev, "i2c-scl-falling-time-ns", &t->scl_fall_ns);
-	if (ret && use_defaults) {
-		if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ)
-			t->scl_fall_ns = 300;
-		else
-			t->scl_fall_ns = 120;
-	}
-
-	ret = device_property_read_u32(dev, "i2c-scl-internal-delay-ns", &t->scl_int_delay_ns);
-	if (ret && use_defaults)
-		t->scl_int_delay_ns = 0;
+	bool u = use_defaults;
+	u32 d;
 
-	ret = device_property_read_u32(dev, "i2c-sda-falling-time-ns", &t->sda_fall_ns);
-	if (ret && use_defaults)
-		t->sda_fall_ns = t->scl_fall_ns;
+	i2c_parse_timing(dev, "clock-frequency", &t->bus_freq_hz,
+			 I2C_MAX_STANDARD_MODE_FREQ, u);
 
-	ret = device_property_read_u32(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns);
-	if (ret && use_defaults)
-		t->sda_hold_ns = 0;
+	d = t->bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ ? 1000 :
+	    t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ ? 300 : 120;
+	i2c_parse_timing(dev, "i2c-scl-rising-time-ns", &t->scl_rise_ns, d, u);
 
-	ret = device_property_read_u32(dev, "i2c-digital-filter-width-ns", &t->digital_filter_width_ns);
-	if (ret && use_defaults)
-		t->digital_filter_width_ns = 0;
+	d = t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ ? 300 : 120;
+	i2c_parse_timing(dev, "i2c-scl-falling-time-ns", &t->scl_fall_ns, d, u);
 
-	ret = device_property_read_u32(dev, "i2c-analog-filter-cutoff-frequency", &t->analog_filter_cutoff_freq_hz);
-	if (ret && use_defaults)
-		t->analog_filter_cutoff_freq_hz = 0;
+	i2c_parse_timing(dev, "i2c-scl-internal-delay-ns",
+			 &t->scl_int_delay_ns, 0, u);
+	i2c_parse_timing(dev, "i2c-sda-falling-time-ns", &t->sda_fall_ns,
+			 t->scl_fall_ns, u);
+	i2c_parse_timing(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns, 0, u);
+	i2c_parse_timing(dev, "i2c-digital-filter-width-ns",
+			 &t->digital_filter_width_ns, 0, u);
+	i2c_parse_timing(dev, "i2c-analog-filter-cutoff-frequency",
+			 &t->analog_filter_cutoff_freq_hz, 0, u);
 }
 EXPORT_SYMBOL_GPL(i2c_parse_fw_timings);
 
@@ -2195,7 +2186,6 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver)
 	const unsigned short *address_list;
 	struct i2c_client *temp_client;
 	int i, err = 0;
-	int adap_id = i2c_adapter_id(adapter);
 
 	address_list = driver->address_list;
 	if (!driver->detect || !address_list)
@@ -2223,7 +2213,7 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver)
 	for (i = 0; address_list[i] != I2C_CLIENT_END; i += 1) {
 		dev_dbg(&adapter->dev,
 			"found normal entry for adapter %d, addr 0x%02x\n",
-			adap_id, address_list[i]);
+			i2c_adapter_id(adapter), address_list[i]);
 		temp_client->addr = address_list[i];
 		err = i2c_detect_address(temp_client, driver);
 		if (unlikely(err))
diff --git a/drivers/i2c/i2c-core.h b/drivers/i2c/i2c-core.h
index 517d98be68d2..94ff1693b391 100644
--- a/drivers/i2c/i2c-core.h
+++ b/drivers/i2c/i2c-core.h
@@ -23,9 +23,9 @@ int i2c_dev_irq_from_resources(const struct resource *resources,
 			       unsigned int num_resources);
 
 /*
- * We only allow atomic transfers for very late communication, e.g. to send
- * the powerdown command to a PMIC. Atomic transfers are a corner case and not
- * for generic use!
+ * We only allow atomic transfers for very late communication, e.g. to access a
+ * PMIC when powering down. Atomic transfers are a corner case and not for
+ * generic use!
  */
 static inline bool i2c_in_atomic_xfer_mode(void)
 {
diff --git a/drivers/i2c/i2c-slave-eeprom.c b/drivers/i2c/i2c-slave-eeprom.c
index cb415b10642f..593f2fd39d17 100644
--- a/drivers/i2c/i2c-slave-eeprom.c
+++ b/drivers/i2c/i2c-slave-eeprom.c
@@ -5,10 +5,9 @@
  * Copyright (C) 2014 by Wolfram Sang, Sang Engineering <wsa@sang-engineering.com>
  * Copyright (C) 2014 by Renesas Electronics Corporation
  *
- * Because most IP blocks can only detect one I2C slave address anyhow, this
- * driver does not support simulating EEPROM types which take more than one
- * address. It is prepared to simulate bigger EEPROMs with an internal 16 bit
- * pointer, yet implementation is deferred until the need actually arises.
+ * Because most slave IP cores can only detect one I2C slave address anyhow,
+ * this driver does not support simulating EEPROM types which take more than
+ * one address.
  */
 
 /*
@@ -18,6 +17,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/firmware.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/module.h>
@@ -40,7 +40,7 @@ struct eeprom_data {
 #define I2C_SLAVE_BYTELEN GENMASK(15, 0)
 #define I2C_SLAVE_FLAG_ADDR16 BIT(16)
 #define I2C_SLAVE_FLAG_RO BIT(17)
-#define I2C_SLAVE_DEVICE_MAGIC(_len, _flags) ((_flags) | (_len))
+#define I2C_SLAVE_DEVICE_MAGIC(_len, _flags) ((_flags) | ((_len) - 1))
 
 static int i2c_slave_eeprom_slave_cb(struct i2c_client *client,
 				     enum i2c_slave_event event, u8 *val)
@@ -120,24 +120,47 @@ static ssize_t i2c_slave_eeprom_bin_write(struct file *filp, struct kobject *kob
 	return count;
 }
 
+static int i2c_slave_init_eeprom_data(struct eeprom_data *eeprom, struct i2c_client *client,
+				      unsigned int size)
+{
+	const struct firmware *fw;
+	const char *eeprom_data;
+	int ret = device_property_read_string(&client->dev, "firmware-name", &eeprom_data);
+
+	if (!ret) {
+		ret = request_firmware_into_buf(&fw, eeprom_data, &client->dev,
+						eeprom->buffer, size);
+		if (ret)
+			return ret;
+		release_firmware(fw);
+	} else {
+		/* An empty eeprom typically has all bits set to 1 */
+		memset(eeprom->buffer, 0xff, size);
+	}
+	return 0;
+}
+
 static int i2c_slave_eeprom_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
 	struct eeprom_data *eeprom;
 	int ret;
-	unsigned int size = FIELD_GET(I2C_SLAVE_BYTELEN, id->driver_data);
+	unsigned int size = FIELD_GET(I2C_SLAVE_BYTELEN, id->driver_data) + 1;
 	unsigned int flag_addr16 = FIELD_GET(I2C_SLAVE_FLAG_ADDR16, id->driver_data);
 
 	eeprom = devm_kzalloc(&client->dev, sizeof(struct eeprom_data) + size, GFP_KERNEL);
 	if (!eeprom)
 		return -ENOMEM;
 
-	eeprom->idx_write_cnt = 0;
 	eeprom->num_address_bytes = flag_addr16 ? 2 : 1;
 	eeprom->address_mask = size - 1;
 	eeprom->read_only = FIELD_GET(I2C_SLAVE_FLAG_RO, id->driver_data);
 	spin_lock_init(&eeprom->buffer_lock);
 	i2c_set_clientdata(client, eeprom);
 
+	ret = i2c_slave_init_eeprom_data(eeprom, client, size);
+	if (ret)
+		return ret;
+
 	sysfs_bin_attr_init(&eeprom->bin);
 	eeprom->bin.attr.name = "slave-eeprom";
 	eeprom->bin.attr.mode = S_IRUSR | S_IWUSR;
@@ -175,6 +198,8 @@ static const struct i2c_device_id i2c_slave_eeprom_id[] = {
 	{ "slave-24c32ro", I2C_SLAVE_DEVICE_MAGIC(32768 / 8, I2C_SLAVE_FLAG_ADDR16 | I2C_SLAVE_FLAG_RO) },
 	{ "slave-24c64", I2C_SLAVE_DEVICE_MAGIC(65536 / 8, I2C_SLAVE_FLAG_ADDR16) },
 	{ "slave-24c64ro", I2C_SLAVE_DEVICE_MAGIC(65536 / 8, I2C_SLAVE_FLAG_ADDR16 | I2C_SLAVE_FLAG_RO) },
+	{ "slave-24c512", I2C_SLAVE_DEVICE_MAGIC(524288 / 8, I2C_SLAVE_FLAG_ADDR16) },
+	{ "slave-24c512ro", I2C_SLAVE_DEVICE_MAGIC(524288 / 8, I2C_SLAVE_FLAG_ADDR16 | I2C_SLAVE_FLAG_RO) },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, i2c_slave_eeprom_id);
diff --git a/drivers/i2c/i2c-smbus.c b/drivers/i2c/i2c-smbus.c
index 809bcf8387d0..dc0108287ccf 100644
--- a/drivers/i2c/i2c-smbus.c
+++ b/drivers/i2c/i2c-smbus.c
@@ -3,10 +3,11 @@
  * i2c-smbus.c - SMBus extensions to the I2C protocol
  *
  * Copyright (C) 2008 David Brownell
- * Copyright (C) 2010 Jean Delvare <jdelvare@suse.de>
+ * Copyright (C) 2010-2019 Jean Delvare <jdelvare@suse.de>
  */
 
 #include <linux/device.h>
+#include <linux/dmi.h>
 #include <linux/i2c.h>
 #include <linux/i2c-smbus.h>
 #include <linux/interrupt.h>
@@ -196,6 +197,107 @@ EXPORT_SYMBOL_GPL(i2c_handle_smbus_alert);
 
 module_i2c_driver(smbalert_driver);
 
+/*
+ * SPD is not part of SMBus but we include it here for convenience as the
+ * target systems are the same.
+ * Restrictions to automatic SPD instantiation:
+ *  - Only works if all filled slots have the same memory type
+ *  - Only works for DDR2, DDR3 and DDR4 for now
+ *  - Only works on systems with 1 to 4 memory slots
+ */
+#if IS_ENABLED(CONFIG_DMI)
+void i2c_register_spd(struct i2c_adapter *adap)
+{
+	int n, slot_count = 0, dimm_count = 0;
+	u16 handle;
+	u8 common_mem_type = 0x0, mem_type;
+	u64 mem_size;
+	const char *name;
+
+	while ((handle = dmi_memdev_handle(slot_count)) != 0xffff) {
+		slot_count++;
+
+		/* Skip empty slots */
+		mem_size = dmi_memdev_size(handle);
+		if (!mem_size)
+			continue;
+
+		/* Skip undefined memory type */
+		mem_type = dmi_memdev_type(handle);
+		if (mem_type <= 0x02)		/* Invalid, Other, Unknown */
+			continue;
+
+		if (!common_mem_type) {
+			/* First filled slot */
+			common_mem_type = mem_type;
+		} else {
+			/* Check that all filled slots have the same type */
+			if (mem_type != common_mem_type) {
+				dev_warn(&adap->dev,
+					 "Different memory types mixed, not instantiating SPD\n");
+				return;
+			}
+		}
+		dimm_count++;
+	}
+
+	/* No useful DMI data, bail out */
+	if (!dimm_count)
+		return;
+
+	dev_info(&adap->dev, "%d/%d memory slots populated (from DMI)\n",
+		 dimm_count, slot_count);
+
+	if (slot_count > 4) {
+		dev_warn(&adap->dev,
+			 "Systems with more than 4 memory slots not supported yet, not instantiating SPD\n");
+		return;
+	}
+
+	switch (common_mem_type) {
+	case 0x13:	/* DDR2 */
+	case 0x18:	/* DDR3 */
+	case 0x1C:	/* LPDDR2 */
+	case 0x1D:	/* LPDDR3 */
+		name = "spd";
+		break;
+	case 0x1A:	/* DDR4 */
+	case 0x1E:	/* LPDDR4 */
+		name = "ee1004";
+		break;
+	default:
+		dev_info(&adap->dev,
+			 "Memory type 0x%02x not supported yet, not instantiating SPD\n",
+			 common_mem_type);
+		return;
+	}
+
+	/*
+	 * We don't know in which slots the memory modules are. We could
+	 * try to guess from the slot names, but that would be rather complex
+	 * and unreliable, so better probe all possible addresses until we
+	 * have found all memory modules.
+	 */
+	for (n = 0; n < slot_count && dimm_count; n++) {
+		struct i2c_board_info info;
+		unsigned short addr_list[2];
+
+		memset(&info, 0, sizeof(struct i2c_board_info));
+		strlcpy(info.type, name, I2C_NAME_SIZE);
+		addr_list[0] = 0x50 + n;
+		addr_list[1] = I2C_CLIENT_END;
+
+		if (!IS_ERR(i2c_new_scanned_device(adap, &info, addr_list, NULL))) {
+			dev_info(&adap->dev,
+				 "Successfully instantiated SPD at 0x%hx\n",
+				 addr_list[0]);
+			dimm_count--;
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(i2c_register_spd);
+#endif
+
 MODULE_AUTHOR("Jean Delvare <jdelvare@suse.de>");
 MODULE_DESCRIPTION("SMBus protocol extensions support");
 MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index a0d926ae3f86..4ad665757dd8 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -1,10 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * I2C multiplexer
  *
  * Copyright (c) 2008-2009 Rodolfo Giometti <giometti@linux.it>
  * Copyright (c) 2008-2009 Eurotech S.p.A. <info@eurotech.it>
  *
- * This module supports the PCA954x and PCA954x series of I2C multiplexer/switch
+ * This module supports the PCA954x and PCA984x series of I2C multiplexer/switch
  * chips made by NXP Semiconductors.
  * This includes the:
  *	 PCA9540, PCA9542, PCA9543, PCA9544, PCA9545, PCA9546, PCA9547,
@@ -29,10 +30,6 @@
  *	i2c-virtual_cb.c from Brian Kuschak <bkuschak@yahoo.com>
  * and
  *	pca9540.c from Jean Delvare <jdelvare@suse.de>.
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/device.h>
@@ -43,10 +40,8 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
-#include <linux/of_irq.h>
 #include <linux/pm.h>
+#include <linux/property.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <dt-bindings/mux/mux.h>
@@ -198,7 +193,6 @@ static const struct i2c_device_id pca954x_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, pca954x_id);
 
-#ifdef CONFIG_OF
 static const struct of_device_id pca954x_of_match[] = {
 	{ .compatible = "nxp,pca9540", .data = &chips[pca_9540] },
 	{ .compatible = "nxp,pca9542", .data = &chips[pca_9542] },
@@ -215,7 +209,6 @@ static const struct of_device_id pca954x_of_match[] = {
 	{}
 };
 MODULE_DEVICE_TABLE(of, pca954x_of_match);
-#endif
 
 /* Write to mux register. Don't use i2c_transfer()/i2c_smbus_xfer()
    for this as they will try to lock adapter a second time */
@@ -327,21 +320,18 @@ static DEVICE_ATTR_RW(idle_state);
 static irqreturn_t pca954x_irq_handler(int irq, void *dev_id)
 {
 	struct pca954x *data = dev_id;
-	unsigned int child_irq;
-	int ret, i, handled = 0;
+	unsigned long pending;
+	int ret, i;
 
 	ret = i2c_smbus_read_byte(data->client);
 	if (ret < 0)
 		return IRQ_NONE;
 
-	for (i = 0; i < data->chip->nchans; i++) {
-		if (ret & BIT(PCA954X_IRQ_OFFSET + i)) {
-			child_irq = irq_linear_revmap(data->irq, i);
-			handle_nested_irq(child_irq);
-			handled++;
-		}
-	}
-	return handled ? IRQ_HANDLED : IRQ_NONE;
+	pending = (ret >> PCA954X_IRQ_OFFSET) & (BIT(data->chip->nchans) - 1);
+	for_each_set_bit(i, &pending, data->chip->nchans)
+		handle_nested_irq(irq_linear_revmap(data->irq, i));
+
+	return IRQ_RETVAL(pending);
 }
 
 static int pca954x_irq_set_type(struct irq_data *idata, unsigned int type)
@@ -390,11 +380,8 @@ static int pca954x_irq_setup(struct i2c_mux_core *muxc)
 static void pca954x_cleanup(struct i2c_mux_core *muxc)
 {
 	struct pca954x *data = i2c_mux_priv(muxc);
-	struct i2c_client *client = data->client;
 	int c, irq;
 
-	device_remove_file(&client->dev, &dev_attr_idle_state);
-
 	if (data->irq) {
 		for (c = 0; c < data->chip->nchans; c++) {
 			irq = irq_find_mapping(data->irq, c);
@@ -429,7 +416,6 @@ static int pca954x_probe(struct i2c_client *client,
 {
 	struct i2c_adapter *adap = client->adapter;
 	struct device *dev = &client->dev;
-	struct device_node *np = dev->of_node;
 	struct gpio_desc *gpio;
 	struct i2c_mux_core *muxc;
 	struct pca954x *data;
@@ -459,7 +445,7 @@ static int pca954x_probe(struct i2c_client *client,
 		udelay(1);
 	}
 
-	data->chip = of_device_get_match_data(dev);
+	data->chip = device_get_match_data(dev);
 	if (!data->chip)
 		data->chip = &chips[id->driver_data];
 
@@ -481,8 +467,8 @@ static int pca954x_probe(struct i2c_client *client,
 	}
 
 	data->idle_state = MUX_IDLE_AS_IS;
-	if (of_property_read_u32(np, "idle-state", &data->idle_state)) {
-		if (np && of_property_read_bool(np, "i2c-mux-idle-disconnect"))
+	if (device_property_read_u32(dev, "idle-state", &data->idle_state)) {
+		if (device_property_read_bool(dev, "i2c-mux-idle-disconnect"))
 			data->idle_state = MUX_IDLE_DISCONNECT;
 	}
 
@@ -539,6 +525,8 @@ static int pca954x_remove(struct i2c_client *client)
 {
 	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
 
+	device_remove_file(&client->dev, &dev_attr_idle_state);
+
 	pca954x_cleanup(muxc);
 	return 0;
 }
@@ -565,7 +553,7 @@ static struct i2c_driver pca954x_driver = {
 	.driver		= {
 		.name	= "pca954x",
 		.pm	= &pca954x_pm,
-		.of_match_table = of_match_ptr(pca954x_of_match),
+		.of_match_table = pca954x_of_match,
 	},
 	.probe		= pca954x_probe,
 	.remove		= pca954x_remove,
diff --git a/drivers/ide/Kconfig b/drivers/ide/Kconfig
index 593f149c2910..973ed4b684ce 100644
--- a/drivers/ide/Kconfig
+++ b/drivers/ide/Kconfig
@@ -12,7 +12,7 @@ menuconfig IDE
 	depends on HAVE_IDE
 	depends on BLOCK
 	select BLK_SCSI_REQUEST
-	---help---
+	help
 	  If you say Y here, your kernel will be able to manage ATA/(E)IDE and
 	  ATAPI units. The most common cases are IDE hard drives and ATAPI
 	  CD-ROM drives.
@@ -49,7 +49,7 @@ config IDE_LEGACY
 config BLK_DEV_IDE_SATA
 	bool "Support for SATA (deprecated; conflicts with libata SATA driver)"
 	default n
-	---help---
+	help
 	  There are two drivers for Serial ATA controllers.
 
 	  The main driver, "libata", uses the SCSI subsystem
@@ -121,7 +121,7 @@ config BLK_DEV_IDECD
 	depends on BLK_DEV
 	select IDE_ATAPI
 	select CDROM
-	---help---
+	help
 	  If you have a CD-ROM drive using the ATAPI protocol, say Y. ATAPI is
 	  a newer protocol used by IDE CD-ROM and TAPE drives, similar to the
 	  SCSI protocol. Most new CD-ROM drives use ATAPI, including the
@@ -172,7 +172,7 @@ config BLK_DEV_IDETAPE
 config BLK_DEV_IDEACPI
 	bool "IDE ACPI support"
 	depends on ACPI
-	---help---
+	help
 	  Implement ACPI support for generic IDE devices. On modern
 	  machines ACPI support is required to properly handle ACPI S3 states.
 
@@ -232,7 +232,7 @@ config BLK_DEV_CMD640
 	tristate "CMD640 chipset bugfix/support"
 	depends on X86
 	select IDE_TIMINGS
-	---help---
+	help
 	  The CMD-Technologies CMD640 IDE chip is used on many common 486 and
 	  Pentium motherboards, usually in combination with a "Neptune" or
 	  "SiS" chipset. Unfortunately, it has a number of rather nasty
diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig
index 5d91a6dda894..1080637ca40e 100644
--- a/drivers/iio/accel/Kconfig
+++ b/drivers/iio/accel/Kconfig
@@ -89,13 +89,13 @@ config ADXL372_I2C
 	  module will be called adxl372_i2c.
 
 config BMA180
-	tristate "Bosch BMA180/BMA25x 3-Axis Accelerometer Driver"
-	depends on I2C
+	tristate "Bosch BMA023/BMA1x0/BMA25x 3-Axis Accelerometer Driver"
+	depends on I2C && INPUT_BMA150=n
 	select IIO_BUFFER
 	select IIO_TRIGGERED_BUFFER
 	help
-	  Say Y here if you want to build a driver for the Bosch BMA180 or
-	  BMA25x triaxial acceleration sensor.
+	  Say Y here if you want to build a driver for the Bosch BMA023, BMA150
+	  BMA180, SMB380, or BMA25x triaxial acceleration sensor.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called bma180.
@@ -238,7 +238,7 @@ config IIO_ST_ACCEL_3AXIS
 	  Say yes here to build support for STMicroelectronics accelerometers:
 	  LSM303DLH, LSM303DLHC, LIS3DH, LSM330D, LSM330DL, LSM330DLC,
 	  LIS331DLH, LSM303DL, LSM303DLM, LSM330, LIS2DH12, H3LIS331DL,
-	  LNG2DM, LIS3DE, LIS2DE12
+	  LNG2DM, LIS3DE, LIS2DE12, LIS2HH12
 
 	  This driver can also be built as a module. If so, these modules
 	  will be created:
diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c
index fcd91d5f05fd..265722e4b13f 100644
--- a/drivers/iio/accel/bma180.c
+++ b/drivers/iio/accel/bma180.c
@@ -7,6 +7,7 @@
  * Support for BMA250 (c) Peter Meerwald <pmeerw@pmeerw.net>
  *
  * SPI is not supported by driver
+ * BMA023/BMA150/SMB380: 7-bit I2C slave address 0x38
  * BMA180: 7-bit I2C slave address 0x40 or 0x41
  * BMA250: 7-bit I2C slave address 0x18 or 0x19
  * BMA254: 7-bit I2C slave address 0x18 or 0x19
@@ -33,6 +34,8 @@
 #define BMA180_IRQ_NAME "bma180_event"
 
 enum chip_ids {
+	BMA023,
+	BMA150,
 	BMA180,
 	BMA250,
 	BMA254,
@@ -48,7 +51,7 @@ struct bma180_part_info {
 	unsigned int num_scales;
 	const int *bw_table;
 	unsigned int num_bw;
-	int center_temp;
+	int temp_offset;
 
 	u8 int_reset_reg, int_reset_mask;
 	u8 sleep_reg, sleep_mask;
@@ -57,13 +60,25 @@ struct bma180_part_info {
 	u8 power_reg, power_mask, lowpower_val;
 	u8 int_enable_reg, int_enable_mask;
 	u8 int_map_reg, int_enable_dataready_int1_mask;
-	u8 softreset_reg;
+	u8 softreset_reg, softreset_val;
 
 	int (*chip_config)(struct bma180_data *data);
 	void (*chip_disable)(struct bma180_data *data);
 };
 
 /* Register set */
+#define BMA023_CTRL_REG0	0x0a
+#define BMA023_CTRL_REG1	0x0b
+#define BMA023_CTRL_REG2	0x14
+#define BMA023_CTRL_REG3	0x15
+
+#define BMA023_RANGE_MASK	GENMASK(4, 3) /* Range of accel values */
+#define BMA023_BW_MASK		GENMASK(2, 0) /* Accel bandwidth */
+#define BMA023_SLEEP		BIT(0)
+#define BMA023_INT_RESET_MASK	BIT(6)
+#define BMA023_NEW_DATA_INT	BIT(5) /* Intr every new accel data is ready */
+#define BMA023_RESET_VAL	BIT(1)
+
 #define BMA180_CHIP_ID		0x00 /* Need to distinguish BMA180 from other */
 #define BMA180_ACC_X_LSB	0x02 /* First of 6 registers of accel data */
 #define BMA180_TEMP		0x08
@@ -94,6 +109,7 @@ struct bma180_part_info {
 /* We have to write this value in reset register to do soft reset */
 #define BMA180_RESET_VAL	0xb6
 
+#define BMA023_ID_REG_VAL	0x02
 #define BMA180_ID_REG_VAL	0x03
 #define BMA250_ID_REG_VAL	0x03
 #define BMA254_ID_REG_VAL	0xfa /* 250 decimal */
@@ -156,6 +172,9 @@ enum bma180_chan {
 	TEMP
 };
 
+static int bma023_bw_table[] = { 25, 50, 100, 190, 375, 750, 1500 }; /* Hz */
+static int bma023_scale_table[] = { 2452, 4903, 9709, };
+
 static int bma180_bw_table[] = { 10, 20, 40, 75, 150, 300 }; /* Hz */
 static int bma180_scale_table[] = { 1275, 1863, 2452, 3727, 4903, 9709, 19417 };
 
@@ -319,7 +338,8 @@ static int bma180_set_pmode(struct bma180_data *data, bool mode)
 static int bma180_soft_reset(struct bma180_data *data)
 {
 	int ret = i2c_smbus_write_byte_data(data->client,
-		data->part_info->softreset_reg, BMA180_RESET_VAL);
+		data->part_info->softreset_reg,
+		data->part_info->softreset_val);
 
 	if (ret)
 		dev_err(&data->client->dev, "failed to reset the chip\n");
@@ -349,11 +369,28 @@ static int bma180_chip_init(struct bma180_data *data)
 	 */
 	msleep(20);
 
-	ret = bma180_set_new_data_intr_state(data, false);
+	return bma180_set_new_data_intr_state(data, false);
+}
+
+static int bma023_chip_config(struct bma180_data *data)
+{
+	int ret = bma180_chip_init(data);
+
 	if (ret)
-		return ret;
+		goto err;
+
+	ret = bma180_set_bw(data, 50); /* 50 Hz */
+	if (ret)
+		goto err;
+	ret = bma180_set_scale(data, 2452); /* 2 G */
+	if (ret)
+		goto err;
 
-	return bma180_set_pmode(data, false);
+	return 0;
+
+err:
+	dev_err(&data->client->dev, "failed to config the chip\n");
+	return ret;
 }
 
 static int bma180_chip_config(struct bma180_data *data)
@@ -362,6 +399,9 @@ static int bma180_chip_config(struct bma180_data *data)
 
 	if (ret)
 		goto err;
+	ret = bma180_set_pmode(data, false);
+	if (ret)
+		goto err;
 	ret = bma180_set_bits(data, BMA180_CTRL_REG0, BMA180_DIS_WAKE_UP, 1);
 	if (ret)
 		goto err;
@@ -391,6 +431,9 @@ static int bma25x_chip_config(struct bma180_data *data)
 
 	if (ret)
 		goto err;
+	ret = bma180_set_pmode(data, false);
+	if (ret)
+		goto err;
 	ret = bma180_set_bw(data, 16); /* 16 Hz */
 	if (ret)
 		goto err;
@@ -413,6 +456,17 @@ err:
 	return ret;
 }
 
+static void bma023_chip_disable(struct bma180_data *data)
+{
+	if (bma180_set_sleep_state(data, true))
+		goto err;
+
+	return;
+
+err:
+	dev_err(&data->client->dev, "failed to disable the chip\n");
+}
+
 static void bma180_chip_disable(struct bma180_data *data)
 {
 	if (bma180_set_new_data_intr_state(data, false))
@@ -512,8 +566,12 @@ static int bma180_read_raw(struct iio_dev *indio_dev,
 		iio_device_release_direct_mode(indio_dev);
 		if (ret < 0)
 			return ret;
-		*val = sign_extend32(ret >> chan->scan_type.shift,
-			chan->scan_type.realbits - 1);
+		if (chan->scan_type.sign == 's') {
+			*val = sign_extend32(ret >> chan->scan_type.shift,
+				chan->scan_type.realbits - 1);
+		} else {
+			*val = ret;
+		}
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY:
 		*val = data->bw;
@@ -531,7 +589,7 @@ static int bma180_read_raw(struct iio_dev *indio_dev,
 			return -EINVAL;
 		}
 	case IIO_CHAN_INFO_OFFSET:
-		*val = data->part_info->center_temp;
+		*val = data->part_info->temp_offset;
 		return IIO_VAL_INT;
 	default:
 		return -EINVAL;
@@ -609,6 +667,11 @@ static const struct iio_enum bma180_power_mode_enum = {
 	.set = bma180_set_power_mode,
 };
 
+static const struct iio_chan_spec_ext_info bma023_ext_info[] = {
+	IIO_MOUNT_MATRIX(IIO_SHARED_BY_DIR, bma180_accel_get_mount_matrix),
+	{ }
+};
+
 static const struct iio_chan_spec_ext_info bma180_ext_info[] = {
 	IIO_ENUM("power_mode", true, &bma180_power_mode_enum),
 	IIO_ENUM_AVAILABLE("power_mode", &bma180_power_mode_enum),
@@ -616,6 +679,35 @@ static const struct iio_chan_spec_ext_info bma180_ext_info[] = {
 	{ }
 };
 
+#define BMA023_ACC_CHANNEL(_axis, _bits) {				\
+	.type = IIO_ACCEL,						\
+	.modified = 1,							\
+	.channel2 = IIO_MOD_##_axis,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),			\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |		\
+		BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY),	\
+	.scan_index = AXIS_##_axis,					\
+	.scan_type = {							\
+		.sign = 's',						\
+		.realbits = _bits,					\
+		.storagebits = 16,					\
+		.shift = 16 - _bits,					\
+	},								\
+	.ext_info = bma023_ext_info,					\
+}
+
+#define BMA150_TEMP_CHANNEL {						\
+	.type = IIO_TEMP,						\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |			\
+		BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_OFFSET),	\
+	.scan_index = TEMP,						\
+	.scan_type = {							\
+		.sign = 'u',						\
+		.realbits = 8,						\
+		.storagebits = 16,					\
+	},								\
+}
+
 #define BMA180_ACC_CHANNEL(_axis, _bits) {				\
 	.type = IIO_ACCEL,						\
 	.modified = 1,							\
@@ -645,6 +737,21 @@ static const struct iio_chan_spec_ext_info bma180_ext_info[] = {
 	},								\
 }
 
+static const struct iio_chan_spec bma023_channels[] = {
+	BMA023_ACC_CHANNEL(X, 10),
+	BMA023_ACC_CHANNEL(Y, 10),
+	BMA023_ACC_CHANNEL(Z, 10),
+	IIO_CHAN_SOFT_TIMESTAMP(4),
+};
+
+static const struct iio_chan_spec bma150_channels[] = {
+	BMA023_ACC_CHANNEL(X, 10),
+	BMA023_ACC_CHANNEL(Y, 10),
+	BMA023_ACC_CHANNEL(Z, 10),
+	BMA150_TEMP_CHANNEL,
+	IIO_CHAN_SOFT_TIMESTAMP(4),
+};
+
 static const struct iio_chan_spec bma180_channels[] = {
 	BMA180_ACC_CHANNEL(X, 14),
 	BMA180_ACC_CHANNEL(Y, 14),
@@ -670,6 +777,63 @@ static const struct iio_chan_spec bma254_channels[] = {
 };
 
 static const struct bma180_part_info bma180_part_info[] = {
+	[BMA023] = {
+		.chip_id = BMA023_ID_REG_VAL,
+		.channels = bma023_channels,
+		.num_channels = ARRAY_SIZE(bma023_channels),
+		.scale_table = bma023_scale_table,
+		.num_scales = ARRAY_SIZE(bma023_scale_table),
+		.bw_table = bma023_bw_table,
+		.num_bw = ARRAY_SIZE(bma023_bw_table),
+		/* No temperature channel */
+		.temp_offset = 0,
+		.int_reset_reg = BMA023_CTRL_REG0,
+		.int_reset_mask = BMA023_INT_RESET_MASK,
+		.sleep_reg = BMA023_CTRL_REG0,
+		.sleep_mask = BMA023_SLEEP,
+		.bw_reg = BMA023_CTRL_REG2,
+		.bw_mask = BMA023_BW_MASK,
+		.scale_reg = BMA023_CTRL_REG2,
+		.scale_mask = BMA023_RANGE_MASK,
+		/* No power mode on bma023 */
+		.power_reg = 0,
+		.power_mask = 0,
+		.lowpower_val = 0,
+		.int_enable_reg = BMA023_CTRL_REG3,
+		.int_enable_mask = BMA023_NEW_DATA_INT,
+		.softreset_reg = BMA023_CTRL_REG0,
+		.softreset_val = BMA023_RESET_VAL,
+		.chip_config = bma023_chip_config,
+		.chip_disable = bma023_chip_disable,
+	},
+	[BMA150] = {
+		.chip_id = BMA023_ID_REG_VAL,
+		.channels = bma150_channels,
+		.num_channels = ARRAY_SIZE(bma150_channels),
+		.scale_table = bma023_scale_table,
+		.num_scales = ARRAY_SIZE(bma023_scale_table),
+		.bw_table = bma023_bw_table,
+		.num_bw = ARRAY_SIZE(bma023_bw_table),
+		.temp_offset = -60, /* 0 LSB @ -30 degree C */
+		.int_reset_reg = BMA023_CTRL_REG0,
+		.int_reset_mask = BMA023_INT_RESET_MASK,
+		.sleep_reg = BMA023_CTRL_REG0,
+		.sleep_mask = BMA023_SLEEP,
+		.bw_reg = BMA023_CTRL_REG2,
+		.bw_mask = BMA023_BW_MASK,
+		.scale_reg = BMA023_CTRL_REG2,
+		.scale_mask = BMA023_RANGE_MASK,
+		/* No power mode on bma150 */
+		.power_reg = 0,
+		.power_mask = 0,
+		.lowpower_val = 0,
+		.int_enable_reg = BMA023_CTRL_REG3,
+		.int_enable_mask = BMA023_NEW_DATA_INT,
+		.softreset_reg = BMA023_CTRL_REG0,
+		.softreset_val = BMA023_RESET_VAL,
+		.chip_config = bma023_chip_config,
+		.chip_disable = bma023_chip_disable,
+	},
 	[BMA180] = {
 		.chip_id = BMA180_ID_REG_VAL,
 		.channels = bma180_channels,
@@ -678,7 +842,7 @@ static const struct bma180_part_info bma180_part_info[] = {
 		.num_scales = ARRAY_SIZE(bma180_scale_table),
 		.bw_table = bma180_bw_table,
 		.num_bw = ARRAY_SIZE(bma180_bw_table),
-		.center_temp = 48, /* 0 LSB @ 24 degree C */
+		.temp_offset = 48, /* 0 LSB @ 24 degree C */
 		.int_reset_reg = BMA180_CTRL_REG0,
 		.int_reset_mask = BMA180_RESET_INT,
 		.sleep_reg = BMA180_CTRL_REG0,
@@ -693,6 +857,7 @@ static const struct bma180_part_info bma180_part_info[] = {
 		.int_enable_reg = BMA180_CTRL_REG3,
 		.int_enable_mask = BMA180_NEW_DATA_INT,
 		.softreset_reg = BMA180_RESET,
+		.softreset_val = BMA180_RESET_VAL,
 		.chip_config = bma180_chip_config,
 		.chip_disable = bma180_chip_disable,
 	},
@@ -704,7 +869,7 @@ static const struct bma180_part_info bma180_part_info[] = {
 		.num_scales = ARRAY_SIZE(bma25x_scale_table),
 		.bw_table = bma25x_bw_table,
 		.num_bw = ARRAY_SIZE(bma25x_bw_table),
-		.center_temp = 48, /* 0 LSB @ 24 degree C */
+		.temp_offset = 48, /* 0 LSB @ 24 degree C */
 		.int_reset_reg = BMA250_INT_RESET_REG,
 		.int_reset_mask = BMA250_INT_RESET_MASK,
 		.sleep_reg = BMA250_POWER_REG,
@@ -721,6 +886,7 @@ static const struct bma180_part_info bma180_part_info[] = {
 		.int_map_reg = BMA250_INT_MAP_REG,
 		.int_enable_dataready_int1_mask = BMA250_INT1_DATA_MASK,
 		.softreset_reg = BMA250_RESET_REG,
+		.softreset_val = BMA180_RESET_VAL,
 		.chip_config = bma25x_chip_config,
 		.chip_disable = bma25x_chip_disable,
 	},
@@ -732,7 +898,7 @@ static const struct bma180_part_info bma180_part_info[] = {
 		.num_scales = ARRAY_SIZE(bma25x_scale_table),
 		.bw_table = bma25x_bw_table,
 		.num_bw = ARRAY_SIZE(bma25x_bw_table),
-		.center_temp = 46, /* 0 LSB @ 23 degree C */
+		.temp_offset = 46, /* 0 LSB @ 23 degree C */
 		.int_reset_reg = BMA254_INT_RESET_REG,
 		.int_reset_mask = BMA254_INT_RESET_MASK,
 		.sleep_reg = BMA254_POWER_REG,
@@ -749,6 +915,7 @@ static const struct bma180_part_info bma180_part_info[] = {
 		.int_map_reg = BMA254_INT_MAP_REG,
 		.int_enable_dataready_int1_mask = BMA254_INT1_DATA_MASK,
 		.softreset_reg = BMA254_RESET_REG,
+		.softreset_val = BMA180_RESET_VAL,
 		.chip_config = bma25x_chip_config,
 		.chip_disable = bma25x_chip_disable,
 	},
@@ -990,9 +1157,12 @@ static SIMPLE_DEV_PM_OPS(bma180_pm_ops, bma180_suspend, bma180_resume);
 #endif
 
 static const struct i2c_device_id bma180_ids[] = {
+	{ "bma023", BMA023 },
+	{ "bma150", BMA150 },
 	{ "bma180", BMA180 },
 	{ "bma250", BMA250 },
 	{ "bma254", BMA254 },
+	{ "smb380", BMA150 },
 	{ }
 };
 
@@ -1000,6 +1170,14 @@ MODULE_DEVICE_TABLE(i2c, bma180_ids);
 
 static const struct of_device_id bma180_of_match[] = {
 	{
+		.compatible = "bosch,bma023",
+		.data = (void *)BMA023
+	},
+	{
+		.compatible = "bosch,bma150",
+		.data = (void *)BMA150
+	},
+	{
 		.compatible = "bosch,bma180",
 		.data = (void *)BMA180
 	},
@@ -1011,6 +1189,10 @@ static const struct of_device_id bma180_of_match[] = {
 		.compatible = "bosch,bma254",
 		.data = (void *)BMA254
 	},
+	{
+		.compatible = "bosch,smb380",
+		.data = (void *)BMA150
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, bma180_of_match);
@@ -1030,5 +1212,5 @@ module_i2c_driver(bma180_driver);
 
 MODULE_AUTHOR("Kravchenko Oleksandr <x0199363@ti.com>");
 MODULE_AUTHOR("Texas Instruments, Inc.");
-MODULE_DESCRIPTION("Bosch BMA180/BMA25x triaxial acceleration sensor");
+MODULE_DESCRIPTION("Bosch BMA023/BMA1x0/BMA25x triaxial acceleration sensor");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/accel/dmard06.c b/drivers/iio/accel/dmard06.c
index 2bf210fa4ba6..ef89bded7390 100644
--- a/drivers/iio/accel/dmard06.c
+++ b/drivers/iio/accel/dmard06.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/i2c.h>
 #include <linux/iio/iio.h>
 
@@ -226,7 +227,7 @@ static struct i2c_driver dmard06_driver = {
 	.id_table = dmard06_id,
 	.driver = {
 		.name = DMARD06_DRV_NAME,
-		.of_match_table = of_match_ptr(dmard06_of_match),
+		.of_match_table = dmard06_of_match,
 		.pm = DMARD06_PM_OPS,
 	},
 };
diff --git a/drivers/iio/accel/hid-sensor-accel-3d.c b/drivers/iio/accel/hid-sensor-accel-3d.c
index 0d9e2def2b25..0ec0533448bc 100644
--- a/drivers/iio/accel/hid-sensor-accel-3d.c
+++ b/drivers/iio/accel/hid-sensor-accel-3d.c
@@ -14,8 +14,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 enum accel_3d_channel {
@@ -391,18 +389,13 @@ static int hid_accel_3d_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		goto error_free_dev_mem;
-	}
 	atomic_set(&accel_state->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 					&accel_state->common_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		goto error_free_dev_mem;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -426,9 +419,7 @@ static int hid_accel_3d_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&accel_state->common_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &accel_state->common_attributes);
 error_free_dev_mem:
 	kfree(indio_dev->channels);
 	return ret;
@@ -443,8 +434,7 @@ static int hid_accel_3d_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&accel_state->common_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &accel_state->common_attributes);
 	kfree(indio_dev->channels);
 
 	return 0;
diff --git a/drivers/iio/accel/kxsd9-i2c.c b/drivers/iio/accel/kxsd9-i2c.c
index 38411e1c155b..b580d605f848 100644
--- a/drivers/iio/accel/kxsd9-i2c.c
+++ b/drivers/iio/accel/kxsd9-i2c.c
@@ -2,6 +2,7 @@
 #include <linux/device.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
 #include <linux/delay.h>
@@ -21,8 +22,8 @@ static int kxsd9_i2c_probe(struct i2c_client *i2c,
 
 	regmap = devm_regmap_init_i2c(i2c, &config);
 	if (IS_ERR(regmap)) {
-		dev_err(&i2c->dev, "Failed to register i2c regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&i2c->dev, "Failed to register i2c regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 
@@ -36,15 +37,11 @@ static int kxsd9_i2c_remove(struct i2c_client *client)
 	return kxsd9_common_remove(&client->dev);
 }
 
-#ifdef CONFIG_OF
 static const struct of_device_id kxsd9_of_match[] = {
 	{ .compatible = "kionix,kxsd9", },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, kxsd9_of_match);
-#else
-#define kxsd9_of_match NULL
-#endif
 
 static const struct i2c_device_id kxsd9_i2c_id[] = {
 	{"kxsd9", 0},
@@ -55,7 +52,7 @@ MODULE_DEVICE_TABLE(i2c, kxsd9_i2c_id);
 static struct i2c_driver kxsd9_i2c_driver = {
 	.driver = {
 		.name	= "kxsd9",
-		.of_match_table = of_match_ptr(kxsd9_of_match),
+		.of_match_table = kxsd9_of_match,
 		.pm = &kxsd9_dev_pm_ops,
 	},
 	.probe		= kxsd9_i2c_probe,
diff --git a/drivers/iio/accel/mxc4005.c b/drivers/iio/accel/mxc4005.c
index 3d5bea651923..9d07642c0de1 100644
--- a/drivers/iio/accel/mxc4005.c
+++ b/drivers/iio/accel/mxc4005.c
@@ -135,7 +135,7 @@ static int mxc4005_read_xyz(struct mxc4005_data *data)
 	int ret;
 
 	ret = regmap_bulk_read(data->regmap, MXC4005_REG_XOUT_UPPER,
-			       (u8 *) data->buffer, sizeof(data->buffer));
+			       data->buffer, sizeof(data->buffer));
 	if (ret < 0) {
 		dev_err(data->dev, "failed to read axes\n");
 		return ret;
@@ -150,7 +150,7 @@ static int mxc4005_read_axis(struct mxc4005_data *data,
 	__be16 reg;
 	int ret;
 
-	ret = regmap_bulk_read(data->regmap, addr, (u8 *) &reg, sizeof(reg));
+	ret = regmap_bulk_read(data->regmap, addr, &reg, sizeof(reg));
 	if (ret < 0) {
 		dev_err(data->dev, "failed to read reg %02x\n", addr);
 		return ret;
diff --git a/drivers/iio/accel/st_accel.h b/drivers/iio/accel/st_accel.h
index 5b13e293cade..5d356288e001 100644
--- a/drivers/iio/accel/st_accel.h
+++ b/drivers/iio/accel/st_accel.h
@@ -35,6 +35,7 @@ enum st_accel_type {
 	LIS2DW12,
 	LIS3DHH,
 	LIS2DE12,
+	LIS2HH12,
 	ST_ACCEL_MAX,
 };
 
@@ -59,6 +60,7 @@ enum st_accel_type {
 #define LIS3DHH_ACCEL_DEV_NAME		"lis3dhh"
 #define LIS3DE_ACCEL_DEV_NAME		"lis3de"
 #define LIS2DE12_ACCEL_DEV_NAME		"lis2de12"
+#define LIS2HH12_ACCEL_DEV_NAME		"lis2hh12"
 
 /**
 * struct st_sensors_platform_data - default accel platform data
diff --git a/drivers/iio/accel/st_accel_buffer.c b/drivers/iio/accel/st_accel_buffer.c
index 9f2b40474b8e..b5c814ef1637 100644
--- a/drivers/iio/accel/st_accel_buffer.c
+++ b/drivers/iio/accel/st_accel_buffer.c
@@ -37,8 +37,7 @@ static int st_accel_buffer_postenable(struct iio_dev *indio_dev)
 	if (err < 0)
 		return err;
 
-	err = st_sensors_set_axis_enable(indio_dev,
-					 (u8)indio_dev->active_scan_mask[0]);
+	err = st_sensors_set_axis_enable(indio_dev, indio_dev->active_scan_mask[0]);
 	if (err < 0)
 		goto st_accel_buffer_predisable;
 
diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c
index 7320275c7e56..43c50167d220 100644
--- a/drivers/iio/accel/st_accel_core.c
+++ b/drivers/iio/accel/st_accel_core.c
@@ -904,6 +904,83 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		.multi_read_bit = true,
 		.bootime = 2,
 	},
+	{
+		.wai = 0x41,
+		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
+		.sensors_supported = {
+			[0] = LIS2HH12_ACCEL_DEV_NAME,
+		},
+		.ch = (struct iio_chan_spec *)st_accel_16bit_channels,
+		.odr = {
+			.addr = 0x20,
+			.mask = 0x70,
+			.odr_avl = {
+				{ .hz = 10, .value = 0x01, },
+				{ .hz = 50, .value = 0x02, },
+				{ .hz = 100, .value = 0x03, },
+				{ .hz = 200, .value = 0x04, },
+				{ .hz = 400, .value = 0x05, },
+				{ .hz = 800, .value = 0x06, },
+			},
+		},
+		.pw = {
+			.addr = 0x20,
+			.mask = 0x70,
+			.value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE,
+		},
+		.enable_axis = {
+			.addr = ST_SENSORS_DEFAULT_AXIS_ADDR,
+			.mask = ST_SENSORS_DEFAULT_AXIS_MASK,
+		},
+		.fs = {
+			.addr = 0x23,
+			.mask = 0x30,
+			.fs_avl = {
+				[0] = {
+					.num = ST_ACCEL_FS_AVL_2G,
+					.value = 0x00,
+					.gain = IIO_G_TO_M_S_2(61),
+				},
+				[1] = {
+					.num = ST_ACCEL_FS_AVL_4G,
+					.value = 0x02,
+					.gain = IIO_G_TO_M_S_2(122),
+				},
+				[2] = {
+					.num = ST_ACCEL_FS_AVL_8G,
+					.value = 0x03,
+					.gain = IIO_G_TO_M_S_2(244),
+				},
+			},
+		},
+		.bdu = {
+			.addr = 0x20,
+			.mask = 0x08,
+		},
+		.drdy_irq = {
+			.int1 = {
+				.addr = 0x22,
+				.mask = 0x01,
+			},
+			.int2 = {
+				.addr = 0x25,
+				.mask = 0x01,
+			},
+			.addr_ihl = 0x24,
+			.mask_ihl = 0x02,
+			.stat_drdy = {
+				.addr = ST_SENSORS_DEFAULT_STAT_ADDR,
+				.mask = 0x07,
+			},
+		},
+		.sim = {
+			.addr = 0x23,
+			.value = BIT(0),
+		},
+		.multi_read_bit = true,
+		.bootime = 2,
+	},
+
 };
 
 static int st_accel_read_raw(struct iio_dev *indio_dev,
@@ -1170,8 +1247,7 @@ EXPORT_SYMBOL(st_accel_get_settings);
 int st_accel_common_probe(struct iio_dev *indio_dev)
 {
 	struct st_sensor_data *adata = iio_priv(indio_dev);
-	struct st_sensors_platform_data *pdata =
-		(struct st_sensors_platform_data *)adata->dev->platform_data;
+	struct st_sensors_platform_data *pdata = dev_get_platdata(adata->dev);
 	struct iio_chan_spec *channels;
 	size_t channels_size;
 	int err;
@@ -1204,8 +1280,7 @@ int st_accel_common_probe(struct iio_dev *indio_dev)
 			 "failed to apply ACPI orientation data: %d\n", err);
 
 	indio_dev->channels = channels;
-	adata->current_fullscale = (struct st_sensor_fullscale_avl *)
-					&adata->sensor_settings->fs.fs_avl[0];
+	adata->current_fullscale = &adata->sensor_settings->fs.fs_avl[0];
 	adata->odr = adata->sensor_settings->odr.odr_avl[0].hz;
 
 	if (!pdata)
diff --git a/drivers/iio/accel/st_accel_i2c.c b/drivers/iio/accel/st_accel_i2c.c
index 6b283be26ebc..360e16f2cadb 100644
--- a/drivers/iio/accel/st_accel_i2c.c
+++ b/drivers/iio/accel/st_accel_i2c.c
@@ -104,6 +104,10 @@ static const struct of_device_id st_accel_of_match[] = {
 		.compatible = "st,lis2de12",
 		.data = LIS2DE12_ACCEL_DEV_NAME,
 	},
+	{
+		.compatible = "st,lis2hh12",
+		.data = LIS2HH12_ACCEL_DEV_NAME,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_accel_of_match);
@@ -138,6 +142,7 @@ static const struct i2c_device_id st_accel_id_table[] = {
 	{ LIS2DW12_ACCEL_DEV_NAME },
 	{ LIS3DE_ACCEL_DEV_NAME },
 	{ LIS2DE12_ACCEL_DEV_NAME },
+	{ LIS2HH12_ACCEL_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, st_accel_id_table);
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 12bb8b7ca1ff..ff3569635ce0 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -246,6 +246,41 @@ config AD799X
 	  To compile this driver as a module, choose M here: the module will be
 	  called ad799x.
 
+config AD9467
+	tristate "Analog Devices AD9467 High Speed ADC driver"
+	depends on SPI
+	select ADI_AXI_ADC
+	help
+	  Say yes here to build support for Analog Devices:
+	  * AD9467 16-Bit, 200 MSPS/250 MSPS Analog-to-Digital Converter
+
+	  The driver requires the assistance of the AXI ADC IP core to operate,
+	  since SPI is used for configuration only, while data has to be
+	  streamed into memory via DMA.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ad9467.
+
+config ADI_AXI_ADC
+	tristate "Analog Devices Generic AXI ADC IP core driver"
+	select IIO_BUFFER
+	select IIO_BUFFER_HW_CONSUMER
+	select IIO_BUFFER_DMAENGINE
+	help
+	  Say yes here to build support for Analog Devices Generic
+	  AXI ADC IP core. The IP core is used for interfacing with
+	  analog-to-digital (ADC) converters that require either a high-speed
+	  serial interface (JESD204B/C) or a source synchronous parallel
+	  interface (LVDS/CMOS).
+	  Typically (for such devices) SPI will be used for configuration only,
+	  while this IP core handles the streaming of data into memory via DMA.
+
+	  Link: https://wiki.analog.com/resources/fpga/docs/axi_adc_ip
+	  If unsure, say N (but it's safe to say "Y").
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called adi-axi-adc.
+
 config ASPEED_ADC
 	tristate "Aspeed ADC"
 	depends on ARCH_ASPEED || COMPILE_TEST
@@ -595,6 +630,16 @@ config MAX1118
 	  To compile this driver as a module, choose M here: the module will be
 	  called max1118.
 
+config MAX1241
+	tristate "Maxim max1241 ADC driver"
+	depends on SPI_MASTER
+	help
+	  Say yes here to build support for Maxim max1241 12-bit, single-channel
+	  ADC.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called max1241.
+
 config MAX1363
 	tristate "Maxim max1363 ADC driver"
 	depends on I2C
@@ -692,6 +737,16 @@ config MESON_SARADC
 	  To compile this driver as a module, choose M here: the
 	  module will be called meson_saradc.
 
+config MP2629_ADC
+	tristate "Monolithic MP2629 ADC driver"
+	depends on MFD_MP2629
+	help
+	  Say yes to have support for battery charger IC MP2629 ADC device
+	  accessed over I2C.
+
+	  This driver provides ADC conversion of system, input power supply
+	  and battery voltage & current information.
+
 config NAU7802
 	tristate "Nuvoton NAU7802 ADC driver"
 	depends on I2C
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index 637807861112..90f94ada7b30 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -26,6 +26,8 @@ obj-$(CONFIG_AD7793) += ad7793.o
 obj-$(CONFIG_AD7887) += ad7887.o
 obj-$(CONFIG_AD7949) += ad7949.o
 obj-$(CONFIG_AD799X) += ad799x.o
+obj-$(CONFIG_AD9467) += ad9467.o
+obj-$(CONFIG_ADI_AXI_ADC) += adi-axi-adc.o
 obj-$(CONFIG_ASPEED_ADC) += aspeed_adc.o
 obj-$(CONFIG_AT91_ADC) += at91_adc.o
 obj-$(CONFIG_AT91_SAMA5D2_ADC) += at91-sama5d2_adc.o
@@ -57,6 +59,7 @@ obj-$(CONFIG_LTC2497) += ltc2497.o ltc2497-core.o
 obj-$(CONFIG_MAX1027) += max1027.o
 obj-$(CONFIG_MAX11100) += max11100.o
 obj-$(CONFIG_MAX1118) += max1118.o
+obj-$(CONFIG_MAX1241) += max1241.o
 obj-$(CONFIG_MAX1363) += max1363.o
 obj-$(CONFIG_MAX9611) += max9611.o
 obj-$(CONFIG_MCP320X) += mcp320x.o
@@ -65,6 +68,7 @@ obj-$(CONFIG_MCP3911) += mcp3911.o
 obj-$(CONFIG_MEDIATEK_MT6577_AUXADC) += mt6577_auxadc.o
 obj-$(CONFIG_MEN_Z188_ADC) += men_z188_adc.o
 obj-$(CONFIG_MESON_SARADC) += meson_saradc.o
+obj-$(CONFIG_MP2629_ADC) += mp2629_adc.o
 obj-$(CONFIG_MXS_LRADC_ADC) += mxs-lradc-adc.o
 obj-$(CONFIG_NAU7802) += nau7802.o
 obj-$(CONFIG_NPCM_ADC) += npcm_adc.o
diff --git a/drivers/iio/adc/ad7476.c b/drivers/iio/adc/ad7476.c
index 76747488044b..4e816d714ad2 100644
--- a/drivers/iio/adc/ad7476.c
+++ b/drivers/iio/adc/ad7476.c
@@ -12,9 +12,11 @@
 #include <linux/sysfs.h>
 #include <linux/spi/spi.h>
 #include <linux/regulator/consumer.h>
+#include <linux/gpio/consumer.h>
 #include <linux/err.h>
 #include <linux/module.h>
 #include <linux/bitops.h>
+#include <linux/delay.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
@@ -27,6 +29,8 @@ struct ad7476_state;
 struct ad7476_chip_info {
 	unsigned int			int_vref_uv;
 	struct iio_chan_spec		channel[2];
+	/* channels used when convst gpio is defined */
+	struct iio_chan_spec		convst_channel[2];
 	void (*reset)(struct ad7476_state *);
 };
 
@@ -34,6 +38,7 @@ struct ad7476_state {
 	struct spi_device		*spi;
 	const struct ad7476_chip_info	*chip_info;
 	struct regulator		*reg;
+	struct gpio_desc		*convst_gpio;
 	struct spi_transfer		xfer;
 	struct spi_message		msg;
 	/*
@@ -64,6 +69,17 @@ enum ad7476_supported_device_ids {
 	ID_ADS7868,
 };
 
+static void ad7091_convst(struct ad7476_state *st)
+{
+	if (!st->convst_gpio)
+		return;
+
+	gpiod_set_value(st->convst_gpio, 0);
+	udelay(1); /* CONVST pulse width: 10 ns min */
+	gpiod_set_value(st->convst_gpio, 1);
+	udelay(1); /* Conversion time: 650 ns max */
+}
+
 static irqreturn_t ad7476_trigger_handler(int irq, void  *p)
 {
 	struct iio_poll_func *pf = p;
@@ -71,6 +87,8 @@ static irqreturn_t ad7476_trigger_handler(int irq, void  *p)
 	struct ad7476_state *st = iio_priv(indio_dev);
 	int b_sent;
 
+	ad7091_convst(st);
+
 	b_sent = spi_sync(st->spi, &st->msg);
 	if (b_sent < 0)
 		goto done;
@@ -93,6 +111,8 @@ static int ad7476_scan_direct(struct ad7476_state *st)
 {
 	int ret;
 
+	ad7091_convst(st);
+
 	ret = spi_sync(st->spi, &st->msg);
 	if (ret)
 		return ret;
@@ -160,6 +180,8 @@ static int ad7476_read_raw(struct iio_dev *indio_dev,
 #define AD7940_CHAN(bits) _AD7476_CHAN((bits), 15 - (bits), \
 		BIT(IIO_CHAN_INFO_RAW))
 #define AD7091R_CHAN(bits) _AD7476_CHAN((bits), 16 - (bits), 0)
+#define AD7091R_CONVST_CHAN(bits) _AD7476_CHAN((bits), 16 - (bits), \
+		BIT(IIO_CHAN_INFO_RAW))
 #define ADS786X_CHAN(bits) _AD7476_CHAN((bits), 12 - (bits), \
 		BIT(IIO_CHAN_INFO_RAW))
 
@@ -167,6 +189,8 @@ static const struct ad7476_chip_info ad7476_chip_info_tbl[] = {
 	[ID_AD7091R] = {
 		.channel[0] = AD7091R_CHAN(12),
 		.channel[1] = IIO_CHAN_SOFT_TIMESTAMP(1),
+		.convst_channel[0] = AD7091R_CONVST_CHAN(12),
+		.convst_channel[1] = IIO_CHAN_SOFT_TIMESTAMP(1),
 		.reset = ad7091_reset,
 	},
 	[ID_AD7276] = {
@@ -232,6 +256,13 @@ static const struct iio_info ad7476_info = {
 	.read_raw = &ad7476_read_raw,
 };
 
+static void ad7476_reg_disable(void *data)
+{
+	struct ad7476_state *st = data;
+
+	regulator_disable(st->reg);
+}
+
 static int ad7476_probe(struct spi_device *spi)
 {
 	struct ad7476_state *st;
@@ -254,6 +285,17 @@ static int ad7476_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
+	ret = devm_add_action_or_reset(&spi->dev, ad7476_reg_disable,
+				       st);
+	if (ret)
+		return ret;
+
+	st->convst_gpio = devm_gpiod_get_optional(&spi->dev,
+						  "adi,conversion-start",
+						  GPIOD_OUT_LOW);
+	if (IS_ERR(st->convst_gpio))
+		return PTR_ERR(st->convst_gpio);
+
 	spi_set_drvdata(spi, indio_dev);
 
 	st->spi = spi;
@@ -266,6 +308,9 @@ static int ad7476_probe(struct spi_device *spi)
 	indio_dev->channels = st->chip_info->channel;
 	indio_dev->num_channels = 2;
 	indio_dev->info = &ad7476_info;
+
+	if (st->convst_gpio)
+		indio_dev->channels = st->chip_info->convst_channel;
 	/* Setup default message */
 
 	st->xfer.rx_buf = &st->data;
@@ -295,19 +340,8 @@ error_disable_reg:
 	return ret;
 }
 
-static int ad7476_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct ad7476_state *st = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
-	iio_triggered_buffer_cleanup(indio_dev);
-	regulator_disable(st->reg);
-
-	return 0;
-}
-
 static const struct spi_device_id ad7476_id[] = {
+	{"ad7091", ID_AD7091R},
 	{"ad7091r", ID_AD7091R},
 	{"ad7273", ID_AD7277},
 	{"ad7274", ID_AD7276},
@@ -343,7 +377,6 @@ static struct spi_driver ad7476_driver = {
 		.name	= "ad7476",
 	},
 	.probe		= ad7476_probe,
-	.remove		= ad7476_remove,
 	.id_table	= ad7476_id,
 };
 module_spi_driver(ad7476_driver);
diff --git a/drivers/iio/adc/ad7780.c b/drivers/iio/adc/ad7780.c
index 291c1a898129..f47606ebbbbe 100644
--- a/drivers/iio/adc/ad7780.c
+++ b/drivers/iio/adc/ad7780.c
@@ -206,10 +206,29 @@ static const struct ad_sigma_delta_info ad7780_sigma_delta_info = {
 	.irq_flags = IRQF_TRIGGER_LOW,
 };
 
-#define AD7780_CHANNEL(bits, wordsize) \
-	AD_SD_CHANNEL(1, 0, 0, bits, 32, (wordsize) - (bits))
-#define AD7170_CHANNEL(bits, wordsize) \
-	AD_SD_CHANNEL_NO_SAMP_FREQ(1, 0, 0, bits, 32, (wordsize) - (bits))
+#define _AD7780_CHANNEL(_bits, _wordsize, _mask_all)		\
+{								\
+	.type = IIO_VOLTAGE,					\
+	.indexed = 1,						\
+	.channel = 0,						\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
+		BIT(IIO_CHAN_INFO_OFFSET),			\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+	.info_mask_shared_by_all = _mask_all,			\
+	.scan_index = 1,					\
+	.scan_type = {						\
+		.sign = 'u',					\
+		.realbits = (_bits),				\
+		.storagebits = 32,				\
+		.shift = (_wordsize) - (_bits),			\
+		.endianness = IIO_BE,				\
+	},							\
+}
+
+#define AD7780_CHANNEL(_bits, _wordsize)	\
+	_AD7780_CHANNEL(_bits, _wordsize, BIT(IIO_CHAN_INFO_SAMP_FREQ))
+#define AD7170_CHANNEL(_bits, _wordsize)	\
+	_AD7780_CHANNEL(_bits, _wordsize, 0)
 
 static const struct ad7780_chip_info ad7780_chip_info_tbl[] = {
 	[ID_AD7170] = {
diff --git a/drivers/iio/adc/ad7791.c b/drivers/iio/adc/ad7791.c
index abb239392631..48432b6f6002 100644
--- a/drivers/iio/adc/ad7791.c
+++ b/drivers/iio/adc/ad7791.c
@@ -64,25 +64,73 @@
 #define AD7791_MODE_SEL_MASK		(0x3 << 6)
 #define AD7791_MODE_SEL(x)		((x) << 6)
 
+#define __AD7991_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+	_storagebits, _shift, _extend_name, _type, _mask_all) \
+	{ \
+		.type = (_type), \
+		.differential = (_channel2 == -1 ? 0 : 1), \
+		.indexed = 1, \
+		.channel = (_channel1), \
+		.channel2 = (_channel2), \
+		.address = (_address), \
+		.extend_name = (_extend_name), \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			BIT(IIO_CHAN_INFO_OFFSET), \
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+		.info_mask_shared_by_all = _mask_all, \
+		.scan_index = (_si), \
+		.scan_type = { \
+			.sign = 'u', \
+			.realbits = (_bits), \
+			.storagebits = (_storagebits), \
+			.shift = (_shift), \
+			.endianness = IIO_BE, \
+		}, \
+	}
+
+#define AD7991_SHORTED_CHANNEL(_si, _channel, _address, _bits, \
+	_storagebits, _shift) \
+	__AD7991_CHANNEL(_si, _channel, _channel, _address, _bits, \
+		_storagebits, _shift, "shorted", IIO_VOLTAGE, \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7991_CHANNEL(_si, _channel, _address, _bits, \
+	_storagebits, _shift) \
+	__AD7991_CHANNEL(_si, _channel, -1, _address, _bits, \
+		_storagebits, _shift, NULL, IIO_VOLTAGE, \
+		 BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7991_DIFF_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+	_storagebits, _shift) \
+	__AD7991_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+		_storagebits, _shift, NULL, IIO_VOLTAGE, \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7991_SUPPLY_CHANNEL(_si, _channel, _address, _bits, _storagebits, \
+	_shift) \
+	__AD7991_CHANNEL(_si, _channel, -1, _address, _bits, \
+		_storagebits, _shift, "supply", IIO_VOLTAGE, \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
 #define DECLARE_AD7787_CHANNELS(name, bits, storagebits) \
 const struct iio_chan_spec name[] = { \
-	AD_SD_DIFF_CHANNEL(0, 0, 0, AD7791_CH_AIN1P_AIN1N, \
+	AD7991_DIFF_CHANNEL(0, 0, 0, AD7791_CH_AIN1P_AIN1N, \
 		(bits), (storagebits), 0), \
-	AD_SD_CHANNEL(1, 1, AD7791_CH_AIN2, (bits), (storagebits), 0), \
-	AD_SD_SHORTED_CHANNEL(2, 0, AD7791_CH_AIN1N_AIN1N, \
+	AD7991_CHANNEL(1, 1, AD7791_CH_AIN2, (bits), (storagebits), 0), \
+	AD7991_SHORTED_CHANNEL(2, 0, AD7791_CH_AIN1N_AIN1N, \
 		(bits), (storagebits), 0), \
-	AD_SD_SUPPLY_CHANNEL(3, 2, AD7791_CH_AVDD_MONITOR,  \
+	AD7991_SUPPLY_CHANNEL(3, 2, AD7791_CH_AVDD_MONITOR,  \
 		(bits), (storagebits), 0), \
 	IIO_CHAN_SOFT_TIMESTAMP(4), \
 }
 
 #define DECLARE_AD7791_CHANNELS(name, bits, storagebits) \
 const struct iio_chan_spec name[] = { \
-	AD_SD_DIFF_CHANNEL(0, 0, 0, AD7791_CH_AIN1P_AIN1N, \
+	AD7991_DIFF_CHANNEL(0, 0, 0, AD7791_CH_AIN1P_AIN1N, \
 		(bits), (storagebits), 0), \
-	AD_SD_SHORTED_CHANNEL(1, 0, AD7791_CH_AIN1N_AIN1N, \
+	AD7991_SHORTED_CHANNEL(1, 0, AD7791_CH_AIN1N_AIN1N, \
 		(bits), (storagebits), 0), \
-	AD_SD_SUPPLY_CHANNEL(2, 1, AD7791_CH_AVDD_MONITOR, \
+	AD7991_SUPPLY_CHANNEL(2, 1, AD7791_CH_AVDD_MONITOR, \
 		(bits), (storagebits), 0), \
 	IIO_CHAN_SOFT_TIMESTAMP(3), \
 }
@@ -444,5 +492,5 @@ static struct spi_driver ad7791_driver = {
 module_spi_driver(ad7791_driver);
 
 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
-MODULE_DESCRIPTION("Analog Device AD7787/AD7788/AD7789/AD7790/AD7791 ADC driver");
+MODULE_DESCRIPTION("Analog Devices AD7787/AD7788/AD7789/AD7790/AD7791 ADC driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/ad7793.c b/drivers/iio/adc/ad7793.c
index e5691e330323..808485f42415 100644
--- a/drivers/iio/adc/ad7793.c
+++ b/drivers/iio/adc/ad7793.c
@@ -354,29 +354,28 @@ static IIO_CONST_ATTR_SAMP_FREQ_AVAIL(
 static IIO_CONST_ATTR_NAMED(sampling_frequency_available_ad7797,
 	sampling_frequency_available, "123 62 50 33 17 16 12 10 8 6 4");
 
-static ssize_t ad7793_show_scale_available(struct device *dev,
-			struct device_attribute *attr, char *buf)
+static int ad7793_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type, int *length,
+			     long mask)
 {
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad7793_state *st = iio_priv(indio_dev);
-	int i, len = 0;
 
-	for (i = 0; i < ARRAY_SIZE(st->scale_avail); i++)
-		len += sprintf(buf + len, "%d.%09u ", st->scale_avail[i][0],
-			       st->scale_avail[i][1]);
-
-	len += sprintf(buf + len, "\n");
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		*vals = (int *)st->scale_avail;
+		*type = IIO_VAL_INT_PLUS_NANO;
+		/* Values are stored in a 2D matrix  */
+		*length = ARRAY_SIZE(st->scale_avail) * 2;
 
-	return len;
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
 }
 
-static IIO_DEVICE_ATTR_NAMED(in_m_in_scale_available,
-		in_voltage-voltage_scale_available, S_IRUGO,
-		ad7793_show_scale_available, NULL, 0);
-
 static struct attribute *ad7793_attributes[] = {
 	&iio_const_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_dev_attr_in_m_in_scale_available.dev_attr.attr,
 	NULL
 };
 
@@ -534,6 +533,7 @@ static const struct iio_info ad7793_info = {
 	.read_raw = &ad7793_read_raw,
 	.write_raw = &ad7793_write_raw,
 	.write_raw_get_fmt = &ad7793_write_raw_get_fmt,
+	.read_avail = ad7793_read_avail,
 	.attrs = &ad7793_attribute_group,
 	.validate_trigger = ad_sd_validate_trigger,
 };
@@ -546,47 +546,113 @@ static const struct iio_info ad7797_info = {
 	.validate_trigger = ad_sd_validate_trigger,
 };
 
+#define __AD7793_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+	_storagebits, _shift, _extend_name, _type, _mask_type_av, _mask_all) \
+	{ \
+		.type = (_type), \
+		.differential = (_channel2 == -1 ? 0 : 1), \
+		.indexed = 1, \
+		.channel = (_channel1), \
+		.channel2 = (_channel2), \
+		.address = (_address), \
+		.extend_name = (_extend_name), \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			BIT(IIO_CHAN_INFO_OFFSET), \
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+		.info_mask_shared_by_type_available = (_mask_type_av), \
+		.info_mask_shared_by_all = _mask_all, \
+		.scan_index = (_si), \
+		.scan_type = { \
+			.sign = 'u', \
+			.realbits = (_bits), \
+			.storagebits = (_storagebits), \
+			.shift = (_shift), \
+			.endianness = IIO_BE, \
+		}, \
+	}
+
+#define AD7793_DIFF_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+	_storagebits, _shift) \
+	__AD7793_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+		_storagebits, _shift, NULL, IIO_VOLTAGE, \
+		BIT(IIO_CHAN_INFO_SCALE), \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7793_SHORTED_CHANNEL(_si, _channel, _address, _bits, \
+	_storagebits, _shift) \
+	__AD7793_CHANNEL(_si, _channel, _channel, _address, _bits, \
+		_storagebits, _shift, "shorted", IIO_VOLTAGE, \
+		BIT(IIO_CHAN_INFO_SCALE), \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7793_TEMP_CHANNEL(_si, _address, _bits, _storagebits, _shift) \
+	__AD7793_CHANNEL(_si, 0, -1, _address, _bits, \
+		_storagebits, _shift, NULL, IIO_TEMP, \
+		0, \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7793_SUPPLY_CHANNEL(_si, _channel, _address, _bits, _storagebits, \
+	_shift) \
+	__AD7793_CHANNEL(_si, _channel, -1, _address, _bits, \
+		_storagebits, _shift, "supply", IIO_VOLTAGE, \
+		0, \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7797_DIFF_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+	_storagebits, _shift) \
+	__AD7793_CHANNEL(_si, _channel1, _channel2, _address, _bits, \
+		_storagebits, _shift, NULL, IIO_VOLTAGE, \
+		0, \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
+#define AD7797_SHORTED_CHANNEL(_si, _channel, _address, _bits, \
+	_storagebits, _shift) \
+	__AD7793_CHANNEL(_si, _channel, _channel, _address, _bits, \
+		_storagebits, _shift, "shorted", IIO_VOLTAGE, \
+		0, \
+		BIT(IIO_CHAN_INFO_SAMP_FREQ))
+
 #define DECLARE_AD7793_CHANNELS(_name, _b, _sb, _s) \
 const struct iio_chan_spec _name##_channels[] = { \
-	AD_SD_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), (_s)), \
-	AD_SD_DIFF_CHANNEL(1, 1, 1, AD7793_CH_AIN2P_AIN2M, (_b), (_sb), (_s)), \
-	AD_SD_DIFF_CHANNEL(2, 2, 2, AD7793_CH_AIN3P_AIN3M, (_b), (_sb), (_s)), \
-	AD_SD_SHORTED_CHANNEL(3, 0, AD7793_CH_AIN1M_AIN1M, (_b), (_sb), (_s)), \
-	AD_SD_TEMP_CHANNEL(4, AD7793_CH_TEMP, (_b), (_sb), (_s)), \
-	AD_SD_SUPPLY_CHANNEL(5, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), (_s)), \
+	AD7793_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), (_s)), \
+	AD7793_DIFF_CHANNEL(1, 1, 1, AD7793_CH_AIN2P_AIN2M, (_b), (_sb), (_s)), \
+	AD7793_DIFF_CHANNEL(2, 2, 2, AD7793_CH_AIN3P_AIN3M, (_b), (_sb), (_s)), \
+	AD7793_SHORTED_CHANNEL(3, 0, AD7793_CH_AIN1M_AIN1M, (_b), (_sb), (_s)), \
+	AD7793_TEMP_CHANNEL(4, AD7793_CH_TEMP, (_b), (_sb), (_s)), \
+	AD7793_SUPPLY_CHANNEL(5, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), (_s)), \
 	IIO_CHAN_SOFT_TIMESTAMP(6), \
 }
 
 #define DECLARE_AD7795_CHANNELS(_name, _b, _sb) \
 const struct iio_chan_spec _name##_channels[] = { \
-	AD_SD_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), 0), \
-	AD_SD_DIFF_CHANNEL(1, 1, 1, AD7793_CH_AIN2P_AIN2M, (_b), (_sb), 0), \
-	AD_SD_DIFF_CHANNEL(2, 2, 2, AD7793_CH_AIN3P_AIN3M, (_b), (_sb), 0), \
-	AD_SD_DIFF_CHANNEL(3, 3, 3, AD7795_CH_AIN4P_AIN4M, (_b), (_sb), 0), \
-	AD_SD_DIFF_CHANNEL(4, 4, 4, AD7795_CH_AIN5P_AIN5M, (_b), (_sb), 0), \
-	AD_SD_DIFF_CHANNEL(5, 5, 5, AD7795_CH_AIN6P_AIN6M, (_b), (_sb), 0), \
-	AD_SD_SHORTED_CHANNEL(6, 0, AD7795_CH_AIN1M_AIN1M, (_b), (_sb), 0), \
-	AD_SD_TEMP_CHANNEL(7, AD7793_CH_TEMP, (_b), (_sb), 0), \
-	AD_SD_SUPPLY_CHANNEL(8, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(1, 1, 1, AD7793_CH_AIN2P_AIN2M, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(2, 2, 2, AD7793_CH_AIN3P_AIN3M, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(3, 3, 3, AD7795_CH_AIN4P_AIN4M, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(4, 4, 4, AD7795_CH_AIN5P_AIN5M, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(5, 5, 5, AD7795_CH_AIN6P_AIN6M, (_b), (_sb), 0), \
+	AD7793_SHORTED_CHANNEL(6, 0, AD7795_CH_AIN1M_AIN1M, (_b), (_sb), 0), \
+	AD7793_TEMP_CHANNEL(7, AD7793_CH_TEMP, (_b), (_sb), 0), \
+	AD7793_SUPPLY_CHANNEL(8, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), 0), \
 	IIO_CHAN_SOFT_TIMESTAMP(9), \
 }
 
 #define DECLARE_AD7797_CHANNELS(_name, _b, _sb) \
 const struct iio_chan_spec _name##_channels[] = { \
-	AD_SD_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), 0), \
-	AD_SD_SHORTED_CHANNEL(1, 0, AD7793_CH_AIN1M_AIN1M, (_b), (_sb), 0), \
-	AD_SD_TEMP_CHANNEL(2, AD7793_CH_TEMP, (_b), (_sb), 0), \
-	AD_SD_SUPPLY_CHANNEL(3, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), 0), \
+	AD7797_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), 0), \
+	AD7797_SHORTED_CHANNEL(1, 0, AD7793_CH_AIN1M_AIN1M, (_b), (_sb), 0), \
+	AD7793_TEMP_CHANNEL(2, AD7793_CH_TEMP, (_b), (_sb), 0), \
+	AD7793_SUPPLY_CHANNEL(3, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), 0), \
 	IIO_CHAN_SOFT_TIMESTAMP(4), \
 }
 
 #define DECLARE_AD7799_CHANNELS(_name, _b, _sb) \
 const struct iio_chan_spec _name##_channels[] = { \
-	AD_SD_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), 0), \
-	AD_SD_DIFF_CHANNEL(1, 1, 1, AD7793_CH_AIN2P_AIN2M, (_b), (_sb), 0), \
-	AD_SD_DIFF_CHANNEL(2, 2, 2, AD7793_CH_AIN3P_AIN3M, (_b), (_sb), 0), \
-	AD_SD_SHORTED_CHANNEL(3, 0, AD7793_CH_AIN1M_AIN1M, (_b), (_sb), 0), \
-	AD_SD_SUPPLY_CHANNEL(4, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(0, 0, 0, AD7793_CH_AIN1P_AIN1M, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(1, 1, 1, AD7793_CH_AIN2P_AIN2M, (_b), (_sb), 0), \
+	AD7793_DIFF_CHANNEL(2, 2, 2, AD7793_CH_AIN3P_AIN3M, (_b), (_sb), 0), \
+	AD7793_SHORTED_CHANNEL(3, 0, AD7793_CH_AIN1M_AIN1M, (_b), (_sb), 0), \
+	AD7793_SUPPLY_CHANNEL(4, 3, AD7793_CH_AVDD_MONITOR, (_b), (_sb), 0), \
 	IIO_CHAN_SOFT_TIMESTAMP(5), \
 }
 
diff --git a/drivers/iio/adc/ad9467.c b/drivers/iio/adc/ad9467.c
new file mode 100644
index 000000000000..1e8fd83b9bc2
--- /dev/null
+++ b/drivers/iio/adc/ad9467.c
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Analog Devices AD9467 SPI ADC driver
+ *
+ * Copyright 2012-2020 Analog Devices Inc.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/of_device.h>
+
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+#include <linux/clk.h>
+
+#include <linux/iio/adc/adi-axi-adc.h>
+
+/*
+ * ADI High-Speed ADC common spi interface registers
+ * See Application-Note AN-877:
+ *   https://www.analog.com/media/en/technical-documentation/application-notes/AN-877.pdf
+ */
+
+#define AN877_ADC_REG_CHIP_PORT_CONF		0x00
+#define AN877_ADC_REG_CHIP_ID			0x01
+#define AN877_ADC_REG_CHIP_GRADE		0x02
+#define AN877_ADC_REG_CHAN_INDEX		0x05
+#define AN877_ADC_REG_TRANSFER			0xFF
+#define AN877_ADC_REG_MODES			0x08
+#define AN877_ADC_REG_TEST_IO			0x0D
+#define AN877_ADC_REG_ADC_INPUT			0x0F
+#define AN877_ADC_REG_OFFSET			0x10
+#define AN877_ADC_REG_OUTPUT_MODE		0x14
+#define AN877_ADC_REG_OUTPUT_ADJUST		0x15
+#define AN877_ADC_REG_OUTPUT_PHASE		0x16
+#define AN877_ADC_REG_OUTPUT_DELAY		0x17
+#define AN877_ADC_REG_VREF			0x18
+#define AN877_ADC_REG_ANALOG_INPUT		0x2C
+
+/* AN877_ADC_REG_TEST_IO */
+#define AN877_ADC_TESTMODE_OFF			0x0
+#define AN877_ADC_TESTMODE_MIDSCALE_SHORT	0x1
+#define AN877_ADC_TESTMODE_POS_FULLSCALE	0x2
+#define AN877_ADC_TESTMODE_NEG_FULLSCALE	0x3
+#define AN877_ADC_TESTMODE_ALT_CHECKERBOARD	0x4
+#define AN877_ADC_TESTMODE_PN23_SEQ		0x5
+#define AN877_ADC_TESTMODE_PN9_SEQ		0x6
+#define AN877_ADC_TESTMODE_ONE_ZERO_TOGGLE	0x7
+#define AN877_ADC_TESTMODE_USER			0x8
+#define AN877_ADC_TESTMODE_BIT_TOGGLE		0x9
+#define AN877_ADC_TESTMODE_SYNC			0xA
+#define AN877_ADC_TESTMODE_ONE_BIT_HIGH		0xB
+#define AN877_ADC_TESTMODE_MIXED_BIT_FREQUENCY	0xC
+#define AN877_ADC_TESTMODE_RAMP			0xF
+
+/* AN877_ADC_REG_TRANSFER */
+#define AN877_ADC_TRANSFER_SYNC			0x1
+
+/* AN877_ADC_REG_OUTPUT_MODE */
+#define AN877_ADC_OUTPUT_MODE_OFFSET_BINARY	0x0
+#define AN877_ADC_OUTPUT_MODE_TWOS_COMPLEMENT	0x1
+#define AN877_ADC_OUTPUT_MODE_GRAY_CODE		0x2
+
+/* AN877_ADC_REG_OUTPUT_PHASE */
+#define AN877_ADC_OUTPUT_EVEN_ODD_MODE_EN	0x20
+#define AN877_ADC_INVERT_DCO_CLK		0x80
+
+/* AN877_ADC_REG_OUTPUT_DELAY */
+#define AN877_ADC_DCO_DELAY_ENABLE		0x80
+
+/*
+ * Analog Devices AD9467 16-Bit, 200/250 MSPS ADC
+ */
+
+#define CHIPID_AD9467			0x50
+#define AD9467_DEF_OUTPUT_MODE		0x08
+#define AD9467_REG_VREF_MASK		0x0F
+
+enum {
+	ID_AD9467,
+};
+
+struct ad9467_state {
+	struct spi_device		*spi;
+	struct clk			*clk;
+	unsigned int			output_mode;
+
+	struct gpio_desc		*pwrdown_gpio;
+	struct gpio_desc		*reset_gpio;
+};
+
+static int ad9467_spi_read(struct spi_device *spi, unsigned int reg)
+{
+	unsigned char tbuf[2], rbuf[1];
+	int ret;
+
+	tbuf[0] = 0x80 | (reg >> 8);
+	tbuf[1] = reg & 0xFF;
+
+	ret = spi_write_then_read(spi,
+				  tbuf, ARRAY_SIZE(tbuf),
+				  rbuf, ARRAY_SIZE(rbuf));
+
+	if (ret < 0)
+		return ret;
+
+	return rbuf[0];
+}
+
+static int ad9467_spi_write(struct spi_device *spi, unsigned int reg,
+			    unsigned int val)
+{
+	unsigned char buf[3];
+
+	buf[0] = reg >> 8;
+	buf[1] = reg & 0xFF;
+	buf[2] = val;
+
+	return spi_write(spi, buf, ARRAY_SIZE(buf));
+}
+
+static int ad9467_reg_access(struct adi_axi_adc_conv *conv, unsigned int reg,
+			     unsigned int writeval, unsigned int *readval)
+{
+	struct ad9467_state *st = adi_axi_adc_conv_priv(conv);
+	struct spi_device *spi = st->spi;
+	int ret;
+
+	if (readval == NULL) {
+		ret = ad9467_spi_write(spi, reg, writeval);
+		ad9467_spi_write(spi, AN877_ADC_REG_TRANSFER,
+				 AN877_ADC_TRANSFER_SYNC);
+		return ret;
+	}
+
+	ret = ad9467_spi_read(spi, reg);
+	if (ret < 0)
+		return ret;
+	*readval = ret;
+
+	return 0;
+}
+
+static const unsigned int ad9467_scale_table[][2] = {
+	{2000, 0}, {2100, 6}, {2200, 7},
+	{2300, 8}, {2400, 9}, {2500, 10},
+};
+
+static void __ad9467_get_scale(struct adi_axi_adc_conv *conv, int index,
+			       unsigned int *val, unsigned int *val2)
+{
+	const struct adi_axi_adc_chip_info *info = conv->chip_info;
+	const struct iio_chan_spec *chan = &info->channels[0];
+	unsigned int tmp;
+
+	tmp = (info->scale_table[index][0] * 1000000ULL) >>
+			chan->scan_type.realbits;
+	*val = tmp / 1000000;
+	*val2 = tmp % 1000000;
+}
+
+#define AD9467_CHAN(_chan, _si, _bits, _sign)				\
+{									\
+	.type = IIO_VOLTAGE,						\
+	.indexed = 1,							\
+	.channel = _chan,						\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |		\
+		BIT(IIO_CHAN_INFO_SAMP_FREQ),				\
+	.scan_index = _si,						\
+	.scan_type = {							\
+		.sign = _sign,						\
+		.realbits = _bits,					\
+		.storagebits = 16,					\
+	},								\
+}
+
+static const struct iio_chan_spec ad9467_channels[] = {
+	AD9467_CHAN(0, 0, 16, 'S'),
+};
+
+static const struct adi_axi_adc_chip_info ad9467_chip_tbl[] = {
+	[ID_AD9467] = {
+		.id = CHIPID_AD9467,
+		.max_rate = 250000000UL,
+		.scale_table = ad9467_scale_table,
+		.num_scales = ARRAY_SIZE(ad9467_scale_table),
+		.channels = ad9467_channels,
+		.num_channels = ARRAY_SIZE(ad9467_channels),
+	},
+};
+
+static int ad9467_get_scale(struct adi_axi_adc_conv *conv, int *val, int *val2)
+{
+	const struct adi_axi_adc_chip_info *info = conv->chip_info;
+	struct ad9467_state *st = adi_axi_adc_conv_priv(conv);
+	unsigned int i, vref_val, vref_mask;
+
+	vref_val = ad9467_spi_read(st->spi, AN877_ADC_REG_VREF);
+
+	switch (info->id) {
+	case CHIPID_AD9467:
+		vref_mask = AD9467_REG_VREF_MASK;
+		break;
+	default:
+		vref_mask = 0xFFFF;
+		break;
+	}
+
+	vref_val &= vref_mask;
+
+	for (i = 0; i < info->num_scales; i++) {
+		if (vref_val == info->scale_table[i][1])
+			break;
+	}
+
+	if (i == info->num_scales)
+		return -ERANGE;
+
+	__ad9467_get_scale(conv, i, val, val2);
+
+	return IIO_VAL_INT_PLUS_MICRO;
+}
+
+static int ad9467_set_scale(struct adi_axi_adc_conv *conv, int val, int val2)
+{
+	const struct adi_axi_adc_chip_info *info = conv->chip_info;
+	struct ad9467_state *st = adi_axi_adc_conv_priv(conv);
+	unsigned int scale_val[2];
+	unsigned int i;
+
+	if (val != 0)
+		return -EINVAL;
+
+	for (i = 0; i < info->num_scales; i++) {
+		__ad9467_get_scale(conv, i, &scale_val[0], &scale_val[1]);
+		if (scale_val[0] != val || scale_val[1] != val2)
+			continue;
+
+		ad9467_spi_write(st->spi, AN877_ADC_REG_VREF,
+				 info->scale_table[i][1]);
+		ad9467_spi_write(st->spi, AN877_ADC_REG_TRANSFER,
+				 AN877_ADC_TRANSFER_SYNC);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int ad9467_read_raw(struct adi_axi_adc_conv *conv,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2, long m)
+{
+	struct ad9467_state *st = adi_axi_adc_conv_priv(conv);
+
+	switch (m) {
+	case IIO_CHAN_INFO_SCALE:
+		return ad9467_get_scale(conv, val, val2);
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = clk_get_rate(st->clk);
+
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad9467_write_raw(struct adi_axi_adc_conv *conv,
+			    struct iio_chan_spec const *chan,
+			    int val, int val2, long mask)
+{
+	const struct adi_axi_adc_chip_info *info = conv->chip_info;
+	struct ad9467_state *st = adi_axi_adc_conv_priv(conv);
+	long r_clk;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		return ad9467_set_scale(conv, val, val2);
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		r_clk = clk_round_rate(st->clk, val);
+		if (r_clk < 0 || r_clk > info->max_rate) {
+			dev_warn(&st->spi->dev,
+				 "Error setting ADC sample rate %ld", r_clk);
+			return -EINVAL;
+		}
+
+		return clk_set_rate(st->clk, r_clk);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad9467_outputmode_set(struct spi_device *spi, unsigned int mode)
+{
+	int ret;
+
+	ret = ad9467_spi_write(spi, AN877_ADC_REG_OUTPUT_MODE, mode);
+	if (ret < 0)
+		return ret;
+
+	return ad9467_spi_write(spi, AN877_ADC_REG_TRANSFER,
+				AN877_ADC_TRANSFER_SYNC);
+}
+
+static int ad9467_preenable_setup(struct adi_axi_adc_conv *conv)
+{
+	struct ad9467_state *st = adi_axi_adc_conv_priv(conv);
+
+	return ad9467_outputmode_set(st->spi, st->output_mode);
+}
+
+static int ad9467_setup(struct ad9467_state *st, unsigned int chip_id)
+{
+	switch (chip_id) {
+	case CHIPID_AD9467:
+		st->output_mode = AD9467_DEF_OUTPUT_MODE |
+				  AN877_ADC_OUTPUT_MODE_TWOS_COMPLEMENT;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static void ad9467_clk_disable(void *data)
+{
+	struct ad9467_state *st = data;
+
+	clk_disable_unprepare(st->clk);
+}
+
+static int ad9467_probe(struct spi_device *spi)
+{
+	const struct adi_axi_adc_chip_info *info;
+	struct adi_axi_adc_conv *conv;
+	struct ad9467_state *st;
+	unsigned int id;
+	int ret;
+
+	info = of_device_get_match_data(&spi->dev);
+	if (!info)
+		return -ENODEV;
+
+	conv = devm_adi_axi_adc_conv_register(&spi->dev, sizeof(*st));
+	if (IS_ERR(conv))
+		return PTR_ERR(conv);
+
+	st = adi_axi_adc_conv_priv(conv);
+	st->spi = spi;
+
+	st->clk = devm_clk_get(&spi->dev, "adc-clk");
+	if (IS_ERR(st->clk))
+		return PTR_ERR(st->clk);
+
+	ret = clk_prepare_enable(st->clk);
+	if (ret < 0)
+		return ret;
+
+	ret = devm_add_action_or_reset(&spi->dev, ad9467_clk_disable, st);
+	if (ret)
+		return ret;
+
+	st->pwrdown_gpio = devm_gpiod_get_optional(&spi->dev, "powerdown",
+						   GPIOD_OUT_LOW);
+	if (IS_ERR(st->pwrdown_gpio))
+		return PTR_ERR(st->pwrdown_gpio);
+
+	st->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset",
+						 GPIOD_OUT_LOW);
+	if (IS_ERR(st->reset_gpio))
+		return PTR_ERR(st->reset_gpio);
+
+	if (st->reset_gpio) {
+		udelay(1);
+		ret = gpiod_direction_output(st->reset_gpio, 1);
+		if (ret)
+			return ret;
+		mdelay(10);
+	}
+
+	spi_set_drvdata(spi, st);
+
+	conv->chip_info = info;
+
+	id = ad9467_spi_read(spi, AN877_ADC_REG_CHIP_ID);
+	if (id != conv->chip_info->id) {
+		dev_err(&spi->dev, "Unrecognized CHIP_ID 0x%X\n", id);
+		return -ENODEV;
+	}
+
+	conv->reg_access = ad9467_reg_access;
+	conv->write_raw = ad9467_write_raw;
+	conv->read_raw = ad9467_read_raw;
+	conv->preenable_setup = ad9467_preenable_setup;
+
+	return ad9467_setup(st, id);
+}
+
+static const struct of_device_id ad9467_of_match[] = {
+	{ .compatible = "adi,ad9467", .data = &ad9467_chip_tbl[ID_AD9467], },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ad9467_of_match);
+
+static struct spi_driver ad9467_driver = {
+	.driver = {
+		.name = "ad9467",
+		.of_match_table = ad9467_of_match,
+	},
+	.probe = ad9467_probe,
+};
+module_spi_driver(ad9467_driver);
+
+MODULE_AUTHOR("Michael Hennerich <michael.hennerich@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD9467 ADC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/ad_sigma_delta.c b/drivers/iio/adc/ad_sigma_delta.c
index 8115b6de1d6c..dd3d54b3bc8b 100644
--- a/drivers/iio/adc/ad_sigma_delta.c
+++ b/drivers/iio/adc/ad_sigma_delta.c
@@ -70,9 +70,7 @@ int ad_sd_write_reg(struct ad_sigma_delta *sigma_delta, unsigned int reg,
 
 	switch (size) {
 	case 3:
-		data[1] = val >> 16;
-		data[2] = val >> 8;
-		data[3] = val;
+		put_unaligned_be24(val, &data[1]);
 		break;
 	case 2:
 		put_unaligned_be16(val, &data[1]);
@@ -157,9 +155,7 @@ int ad_sd_read_reg(struct ad_sigma_delta *sigma_delta,
 		*val = get_unaligned_be32(sigma_delta->data);
 		break;
 	case 3:
-		*val = (sigma_delta->data[0] << 16) |
-			(sigma_delta->data[1] << 8) |
-			sigma_delta->data[2];
+		*val = get_unaligned_be24(&sigma_delta->data[0]);
 		break;
 	case 2:
 		*val = get_unaligned_be16(sigma_delta->data);
diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c
new file mode 100644
index 000000000000..c24c8da99eb4
--- /dev/null
+++ b/drivers/iio/adc/adi-axi-adc.c
@@ -0,0 +1,482 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Analog Devices Generic AXI ADC IP core
+ * Link: https://wiki.analog.com/resources/fpga/docs/axi_adc_ip
+ *
+ * Copyright 2012-2020 Analog Devices Inc.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/buffer-dmaengine.h>
+
+#include <linux/fpga/adi-axi-common.h>
+#include <linux/iio/adc/adi-axi-adc.h>
+
+/**
+ * Register definitions:
+ *   https://wiki.analog.com/resources/fpga/docs/axi_adc_ip#register_map
+ */
+
+/* ADC controls */
+
+#define ADI_AXI_REG_RSTN			0x0040
+#define   ADI_AXI_REG_RSTN_CE_N			BIT(2)
+#define   ADI_AXI_REG_RSTN_MMCM_RSTN		BIT(1)
+#define   ADI_AXI_REG_RSTN_RSTN			BIT(0)
+
+/* ADC Channel controls */
+
+#define ADI_AXI_REG_CHAN_CTRL(c)		(0x0400 + (c) * 0x40)
+#define   ADI_AXI_REG_CHAN_CTRL_LB_OWR		BIT(11)
+#define   ADI_AXI_REG_CHAN_CTRL_PN_SEL_OWR	BIT(10)
+#define   ADI_AXI_REG_CHAN_CTRL_IQCOR_EN	BIT(9)
+#define   ADI_AXI_REG_CHAN_CTRL_DCFILT_EN	BIT(8)
+#define   ADI_AXI_REG_CHAN_CTRL_FMT_SIGNEXT	BIT(6)
+#define   ADI_AXI_REG_CHAN_CTRL_FMT_TYPE	BIT(5)
+#define   ADI_AXI_REG_CHAN_CTRL_FMT_EN		BIT(4)
+#define   ADI_AXI_REG_CHAN_CTRL_PN_TYPE_OWR	BIT(1)
+#define   ADI_AXI_REG_CHAN_CTRL_ENABLE		BIT(0)
+
+#define ADI_AXI_REG_CHAN_CTRL_DEFAULTS		\
+	(ADI_AXI_REG_CHAN_CTRL_FMT_SIGNEXT |	\
+	 ADI_AXI_REG_CHAN_CTRL_FMT_EN |		\
+	 ADI_AXI_REG_CHAN_CTRL_ENABLE)
+
+struct adi_axi_adc_core_info {
+	unsigned int				version;
+};
+
+struct adi_axi_adc_state {
+	struct mutex				lock;
+
+	struct adi_axi_adc_client		*client;
+	void __iomem				*regs;
+};
+
+struct adi_axi_adc_client {
+	struct list_head			entry;
+	struct adi_axi_adc_conv			conv;
+	struct adi_axi_adc_state		*state;
+	struct device				*dev;
+	const struct adi_axi_adc_core_info	*info;
+};
+
+static LIST_HEAD(registered_clients);
+static DEFINE_MUTEX(registered_clients_lock);
+
+static struct adi_axi_adc_client *conv_to_client(struct adi_axi_adc_conv *conv)
+{
+	return container_of(conv, struct adi_axi_adc_client, conv);
+}
+
+void *adi_axi_adc_conv_priv(struct adi_axi_adc_conv *conv)
+{
+	struct adi_axi_adc_client *cl = conv_to_client(conv);
+
+	return (char *)cl + ALIGN(sizeof(struct adi_axi_adc_client), IIO_ALIGN);
+}
+EXPORT_SYMBOL_GPL(adi_axi_adc_conv_priv);
+
+static void adi_axi_adc_write(struct adi_axi_adc_state *st,
+			      unsigned int reg,
+			      unsigned int val)
+{
+	iowrite32(val, st->regs + reg);
+}
+
+static unsigned int adi_axi_adc_read(struct adi_axi_adc_state *st,
+				     unsigned int reg)
+{
+	return ioread32(st->regs + reg);
+}
+
+static int adi_axi_adc_config_dma_buffer(struct device *dev,
+					 struct iio_dev *indio_dev)
+{
+	struct iio_buffer *buffer;
+	const char *dma_name;
+
+	if (!device_property_present(dev, "dmas"))
+		return 0;
+
+	if (device_property_read_string(dev, "dma-names", &dma_name))
+		dma_name = "rx";
+
+	buffer = devm_iio_dmaengine_buffer_alloc(indio_dev->dev.parent,
+						 dma_name);
+	if (IS_ERR(buffer))
+		return PTR_ERR(buffer);
+
+	indio_dev->modes |= INDIO_BUFFER_HARDWARE;
+	iio_device_attach_buffer(indio_dev, buffer);
+
+	return 0;
+}
+
+static int adi_axi_adc_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct adi_axi_adc_state *st = iio_priv(indio_dev);
+	struct adi_axi_adc_conv *conv = &st->client->conv;
+
+	if (!conv->read_raw)
+		return -EOPNOTSUPP;
+
+	return conv->read_raw(conv, chan, val, val2, mask);
+}
+
+static int adi_axi_adc_write_raw(struct iio_dev *indio_dev,
+				 struct iio_chan_spec const *chan,
+				 int val, int val2, long mask)
+{
+	struct adi_axi_adc_state *st = iio_priv(indio_dev);
+	struct adi_axi_adc_conv *conv = &st->client->conv;
+
+	if (!conv->write_raw)
+		return -EOPNOTSUPP;
+
+	return conv->write_raw(conv, chan, val, val2, mask);
+}
+
+static int adi_axi_adc_update_scan_mode(struct iio_dev *indio_dev,
+					const unsigned long *scan_mask)
+{
+	struct adi_axi_adc_state *st = iio_priv(indio_dev);
+	struct adi_axi_adc_conv *conv = &st->client->conv;
+	unsigned int i, ctrl;
+
+	for (i = 0; i < conv->chip_info->num_channels; i++) {
+		ctrl = adi_axi_adc_read(st, ADI_AXI_REG_CHAN_CTRL(i));
+
+		if (test_bit(i, scan_mask))
+			ctrl |= ADI_AXI_REG_CHAN_CTRL_ENABLE;
+		else
+			ctrl &= ~ADI_AXI_REG_CHAN_CTRL_ENABLE;
+
+		adi_axi_adc_write(st, ADI_AXI_REG_CHAN_CTRL(i), ctrl);
+	}
+
+	return 0;
+}
+
+static struct adi_axi_adc_conv *adi_axi_adc_conv_register(struct device *dev,
+							  size_t sizeof_priv)
+{
+	struct adi_axi_adc_client *cl;
+	size_t alloc_size;
+
+	alloc_size = ALIGN(sizeof(struct adi_axi_adc_client), IIO_ALIGN);
+	if (sizeof_priv)
+		alloc_size += ALIGN(sizeof_priv, IIO_ALIGN);
+
+	cl = kzalloc(alloc_size, GFP_KERNEL);
+	if (!cl)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_lock(&registered_clients_lock);
+
+	cl->dev = get_device(dev);
+
+	list_add_tail(&cl->entry, &registered_clients);
+
+	mutex_unlock(&registered_clients_lock);
+
+	return &cl->conv;
+}
+
+static void adi_axi_adc_conv_unregister(struct adi_axi_adc_conv *conv)
+{
+	struct adi_axi_adc_client *cl = conv_to_client(conv);
+
+	mutex_lock(&registered_clients_lock);
+
+	list_del(&cl->entry);
+	put_device(cl->dev);
+
+	mutex_unlock(&registered_clients_lock);
+
+	kfree(cl);
+}
+
+static void devm_adi_axi_adc_conv_release(struct device *dev, void *res)
+{
+	adi_axi_adc_conv_unregister(*(struct adi_axi_adc_conv **)res);
+}
+
+struct adi_axi_adc_conv *devm_adi_axi_adc_conv_register(struct device *dev,
+							size_t sizeof_priv)
+{
+	struct adi_axi_adc_conv **ptr, *conv;
+
+	ptr = devres_alloc(devm_adi_axi_adc_conv_release, sizeof(*ptr),
+			   GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	conv = adi_axi_adc_conv_register(dev, sizeof_priv);
+	if (IS_ERR(conv)) {
+		devres_free(ptr);
+		return ERR_CAST(conv);
+	}
+
+	*ptr = conv;
+	devres_add(dev, ptr);
+
+	return conv;
+}
+EXPORT_SYMBOL_GPL(devm_adi_axi_adc_conv_register);
+
+static ssize_t in_voltage_scale_available_show(struct device *dev,
+					       struct device_attribute *attr,
+					       char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct adi_axi_adc_state *st = iio_priv(indio_dev);
+	struct adi_axi_adc_conv *conv = &st->client->conv;
+	size_t len = 0;
+	int i;
+
+	for (i = 0; i < conv->chip_info->num_scales; i++) {
+		const unsigned int *s = conv->chip_info->scale_table[i];
+
+		len += scnprintf(buf + len, PAGE_SIZE - len,
+				 "%u.%06u ", s[0], s[1]);
+	}
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+static IIO_DEVICE_ATTR_RO(in_voltage_scale_available, 0);
+
+enum {
+	ADI_AXI_ATTR_SCALE_AVAIL,
+};
+
+#define ADI_AXI_ATTR(_en_, _file_)			\
+	[ADI_AXI_ATTR_##_en_] = &iio_dev_attr_##_file_.dev_attr.attr
+
+static struct attribute *adi_axi_adc_attributes[] = {
+	ADI_AXI_ATTR(SCALE_AVAIL, in_voltage_scale_available),
+	NULL
+};
+
+static umode_t axi_adc_attr_is_visible(struct kobject *kobj,
+				       struct attribute *attr, int n)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct adi_axi_adc_state *st = iio_priv(indio_dev);
+	struct adi_axi_adc_conv *conv = &st->client->conv;
+
+	switch (n) {
+	case ADI_AXI_ATTR_SCALE_AVAIL:
+		if (!conv->chip_info->num_scales)
+			return 0;
+		return attr->mode;
+	default:
+		return attr->mode;
+	}
+}
+
+static const struct attribute_group adi_axi_adc_attribute_group = {
+	.attrs = adi_axi_adc_attributes,
+	.is_visible = axi_adc_attr_is_visible,
+};
+
+static const struct iio_info adi_axi_adc_info = {
+	.read_raw = &adi_axi_adc_read_raw,
+	.write_raw = &adi_axi_adc_write_raw,
+	.attrs = &adi_axi_adc_attribute_group,
+	.update_scan_mode = &adi_axi_adc_update_scan_mode,
+};
+
+static const struct adi_axi_adc_core_info adi_axi_adc_10_0_a_info = {
+	.version = ADI_AXI_PCORE_VER(10, 0, 'a'),
+};
+
+static struct adi_axi_adc_client *adi_axi_adc_attach_client(struct device *dev)
+{
+	const struct adi_axi_adc_core_info *info;
+	struct adi_axi_adc_client *cl;
+	struct device_node *cln;
+
+	info = of_device_get_match_data(dev);
+	if (!info)
+		return ERR_PTR(-ENODEV);
+
+	cln = of_parse_phandle(dev->of_node, "adi,adc-dev", 0);
+	if (!cln) {
+		dev_err(dev, "No 'adi,adc-dev' node defined\n");
+		return ERR_PTR(-ENODEV);
+	}
+
+	mutex_lock(&registered_clients_lock);
+
+	list_for_each_entry(cl, &registered_clients, entry) {
+		if (!cl->dev)
+			continue;
+
+		if (cl->dev->of_node != cln)
+			continue;
+
+		if (!try_module_get(dev->driver->owner)) {
+			mutex_unlock(&registered_clients_lock);
+			return ERR_PTR(-ENODEV);
+		}
+
+		get_device(dev);
+		cl->info = info;
+		mutex_unlock(&registered_clients_lock);
+		return cl;
+	}
+
+	mutex_unlock(&registered_clients_lock);
+
+	return ERR_PTR(-EPROBE_DEFER);
+}
+
+static int adi_axi_adc_setup_channels(struct device *dev,
+				      struct adi_axi_adc_state *st)
+{
+	struct adi_axi_adc_conv *conv = &st->client->conv;
+	int i, ret;
+
+	if (conv->preenable_setup) {
+		ret = conv->preenable_setup(conv);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < conv->chip_info->num_channels; i++) {
+		adi_axi_adc_write(st, ADI_AXI_REG_CHAN_CTRL(i),
+				  ADI_AXI_REG_CHAN_CTRL_DEFAULTS);
+	}
+
+	return 0;
+}
+
+static void axi_adc_reset(struct adi_axi_adc_state *st)
+{
+	adi_axi_adc_write(st, ADI_AXI_REG_RSTN, 0);
+	mdelay(10);
+	adi_axi_adc_write(st, ADI_AXI_REG_RSTN, ADI_AXI_REG_RSTN_MMCM_RSTN);
+	mdelay(10);
+	adi_axi_adc_write(st, ADI_AXI_REG_RSTN,
+			  ADI_AXI_REG_RSTN_RSTN | ADI_AXI_REG_RSTN_MMCM_RSTN);
+}
+
+static void adi_axi_adc_cleanup(void *data)
+{
+	struct adi_axi_adc_client *cl = data;
+
+	put_device(cl->dev);
+	module_put(cl->dev->driver->owner);
+}
+
+static int adi_axi_adc_probe(struct platform_device *pdev)
+{
+	struct adi_axi_adc_conv *conv;
+	struct iio_dev *indio_dev;
+	struct adi_axi_adc_client *cl;
+	struct adi_axi_adc_state *st;
+	unsigned int ver;
+	int ret;
+
+	cl = adi_axi_adc_attach_client(&pdev->dev);
+	if (IS_ERR(cl))
+		return PTR_ERR(cl);
+
+	ret = devm_add_action_or_reset(&pdev->dev, adi_axi_adc_cleanup, cl);
+	if (ret)
+		return ret;
+
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*st));
+	if (indio_dev == NULL)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	st->client = cl;
+	cl->state = st;
+	mutex_init(&st->lock);
+
+	st->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(st->regs))
+		return PTR_ERR(st->regs);
+
+	conv = &st->client->conv;
+
+	axi_adc_reset(st);
+
+	ver = adi_axi_adc_read(st, ADI_AXI_REG_VERSION);
+
+	if (cl->info->version > ver) {
+		dev_err(&pdev->dev,
+			"IP core version is too old. Expected %d.%.2d.%c, Reported %d.%.2d.%c\n",
+			ADI_AXI_PCORE_VER_MAJOR(cl->info->version),
+			ADI_AXI_PCORE_VER_MINOR(cl->info->version),
+			ADI_AXI_PCORE_VER_PATCH(cl->info->version),
+			ADI_AXI_PCORE_VER_MAJOR(ver),
+			ADI_AXI_PCORE_VER_MINOR(ver),
+			ADI_AXI_PCORE_VER_PATCH(ver));
+		return -ENODEV;
+	}
+
+	indio_dev->info = &adi_axi_adc_info;
+	indio_dev->dev.parent = &pdev->dev;
+	indio_dev->name = "adi-axi-adc";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->num_channels = conv->chip_info->num_channels;
+	indio_dev->channels = conv->chip_info->channels;
+
+	ret = adi_axi_adc_config_dma_buffer(&pdev->dev, indio_dev);
+	if (ret)
+		return ret;
+
+	ret = adi_axi_adc_setup_channels(&pdev->dev, st);
+	if (ret)
+		return ret;
+
+	ret = devm_iio_device_register(&pdev->dev, indio_dev);
+	if (ret)
+		return ret;
+
+	dev_info(&pdev->dev, "AXI ADC IP core (%d.%.2d.%c) probed\n",
+		 ADI_AXI_PCORE_VER_MAJOR(ver),
+		 ADI_AXI_PCORE_VER_MINOR(ver),
+		 ADI_AXI_PCORE_VER_PATCH(ver));
+
+	return 0;
+}
+
+/* Match table for of_platform binding */
+static const struct of_device_id adi_axi_adc_of_match[] = {
+	{ .compatible = "adi,axi-adc-10.0.a", .data = &adi_axi_adc_10_0_a_info },
+	{ /* end of list */ }
+};
+MODULE_DEVICE_TABLE(of, adi_axi_adc_of_match);
+
+static struct platform_driver adi_axi_adc_driver = {
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = adi_axi_adc_of_match,
+	},
+	.probe = adi_axi_adc_probe,
+};
+module_platform_driver(adi_axi_adc_driver);
+
+MODULE_AUTHOR("Michael Hennerich <michael.hennerich@analog.com>");
+MODULE_DESCRIPTION("Analog Devices Generic AXI ADC IP core driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index 9d96f7d08b95..9abbbdcc7420 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -8,6 +8,7 @@
 
 #include <linux/bitops.h>
 #include <linux/clk.h>
+#include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
 #include <linux/interrupt.h>
@@ -100,6 +101,8 @@
 #define AT91_SAMA5D2_IER_YRDY   BIT(21)
 /* Interrupt Enable Register - TS pressure measurement ready */
 #define AT91_SAMA5D2_IER_PRDY   BIT(22)
+/* Interrupt Enable Register - Data ready */
+#define AT91_SAMA5D2_IER_DRDY   BIT(24)
 /* Interrupt Enable Register - general overrun error */
 #define AT91_SAMA5D2_IER_GOVRE BIT(25)
 /* Interrupt Enable Register - Pen detect */
@@ -486,6 +489,21 @@ static inline int at91_adc_of_xlate(struct iio_dev *indio_dev,
 	return at91_adc_chan_xlate(indio_dev, iiospec->args[0]);
 }
 
+static unsigned int at91_adc_active_scan_mask_to_reg(struct iio_dev *indio_dev)
+{
+	u32 mask = 0;
+	u8 bit;
+
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->num_channels) {
+		struct iio_chan_spec const *chan =
+			 at91_adc_chan_get(indio_dev, bit);
+		mask |= BIT(chan->channel);
+	}
+
+	return mask & GENMASK(11, 0);
+}
+
 static void at91_adc_config_emr(struct at91_adc_state *st)
 {
 	/* configure the extended mode register */
@@ -710,7 +728,6 @@ static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state)
 	struct iio_dev *indio = iio_trigger_get_drvdata(trig);
 	struct at91_adc_state *st = iio_priv(indio);
 	u32 status = at91_adc_readl(st, AT91_SAMA5D2_TRGR);
-	u8 bit;
 
 	/* clear TRGMOD */
 	status &= ~AT91_SAMA5D2_TRGR_TRGMOD_MASK;
@@ -721,50 +738,6 @@ static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state)
 	/* set/unset hw trigger */
 	at91_adc_writel(st, AT91_SAMA5D2_TRGR, status);
 
-	for_each_set_bit(bit, indio->active_scan_mask, indio->num_channels) {
-		struct iio_chan_spec const *chan = at91_adc_chan_get(indio, bit);
-		u32 cor;
-
-		if (!chan)
-			continue;
-		/* these channel types cannot be handled by this trigger */
-		if (chan->type == IIO_POSITIONRELATIVE ||
-		    chan->type == IIO_PRESSURE)
-			continue;
-
-		if (state) {
-			cor = at91_adc_readl(st, AT91_SAMA5D2_COR);
-
-			if (chan->differential)
-				cor |= (BIT(chan->channel) |
-					BIT(chan->channel2)) <<
-					AT91_SAMA5D2_COR_DIFF_OFFSET;
-			else
-				cor &= ~(BIT(chan->channel) <<
-				       AT91_SAMA5D2_COR_DIFF_OFFSET);
-
-			at91_adc_writel(st, AT91_SAMA5D2_COR, cor);
-		}
-
-		if (state) {
-			at91_adc_writel(st, AT91_SAMA5D2_CHER,
-					BIT(chan->channel));
-			/* enable irq only if not using DMA */
-			if (!st->dma_st.dma_chan) {
-				at91_adc_writel(st, AT91_SAMA5D2_IER,
-						BIT(chan->channel));
-			}
-		} else {
-			/* disable irq only if not using DMA */
-			if (!st->dma_st.dma_chan) {
-				at91_adc_writel(st, AT91_SAMA5D2_IDR,
-						BIT(chan->channel));
-			}
-			at91_adc_writel(st, AT91_SAMA5D2_CHDR,
-					BIT(chan->channel));
-		}
-	}
-
 	return 0;
 }
 
@@ -781,6 +754,7 @@ static int at91_adc_reenable_trigger(struct iio_trigger *trig)
 
 	/* Needed to ACK the DRDY interruption */
 	at91_adc_readl(st, AT91_SAMA5D2_LCDR);
+
 	return 0;
 }
 
@@ -888,18 +862,37 @@ static int at91_adc_dma_start(struct iio_dev *indio_dev)
 	return 0;
 }
 
-static int at91_adc_buffer_postenable(struct iio_dev *indio_dev)
+static bool at91_adc_buffer_check_use_irq(struct iio_dev *indio,
+					  struct at91_adc_state *st)
+{
+	/* if using DMA, we do not use our own IRQ (we use DMA-controller) */
+	if (st->dma_st.dma_chan)
+		return false;
+	/* if the trigger is not ours, then it has its own IRQ */
+	if (iio_trigger_validate_own_device(indio->trig, indio))
+		return false;
+	return true;
+}
+
+static bool at91_adc_current_chan_is_touch(struct iio_dev *indio_dev)
+{
+	struct at91_adc_state *st = iio_priv(indio_dev);
+
+	return !!bitmap_subset(indio_dev->active_scan_mask,
+			       &st->touch_st.channels_bitmask,
+			       AT91_SAMA5D2_MAX_CHAN_IDX + 1);
+}
+
+static int at91_adc_buffer_preenable(struct iio_dev *indio_dev)
 {
 	int ret;
+	u8 bit;
 	struct at91_adc_state *st = iio_priv(indio_dev);
 
 	/* check if we are enabling triggered buffer or the touchscreen */
-	if (bitmap_subset(indio_dev->active_scan_mask,
-			  &st->touch_st.channels_bitmask,
-			  AT91_SAMA5D2_MAX_CHAN_IDX + 1)) {
-		/* touchscreen enabling */
+	if (at91_adc_current_chan_is_touch(indio_dev))
 		return at91_adc_configure_touch(st, true);
-	}
+
 	/* if we are not in triggered mode, we cannot enable the buffer. */
 	if (!(indio_dev->currentmode & INDIO_ALL_TRIGGERED_MODES))
 		return -EINVAL;
@@ -911,41 +904,65 @@ static int at91_adc_buffer_postenable(struct iio_dev *indio_dev)
 		return ret;
 	}
 
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->num_channels) {
+		struct iio_chan_spec const *chan =
+					at91_adc_chan_get(indio_dev, bit);
+		u32 cor;
+
+		if (!chan)
+			continue;
+		/* these channel types cannot be handled by this trigger */
+		if (chan->type == IIO_POSITIONRELATIVE ||
+		    chan->type == IIO_PRESSURE)
+			continue;
+
+		cor = at91_adc_readl(st, AT91_SAMA5D2_COR);
+
+		if (chan->differential)
+			cor |= (BIT(chan->channel) | BIT(chan->channel2)) <<
+				AT91_SAMA5D2_COR_DIFF_OFFSET;
+		else
+			cor &= ~(BIT(chan->channel) <<
+			       AT91_SAMA5D2_COR_DIFF_OFFSET);
+
+		at91_adc_writel(st, AT91_SAMA5D2_COR, cor);
+
+		at91_adc_writel(st, AT91_SAMA5D2_CHER, BIT(chan->channel));
+	}
+
+	if (at91_adc_buffer_check_use_irq(indio_dev, st))
+		at91_adc_writel(st, AT91_SAMA5D2_IER, AT91_SAMA5D2_IER_DRDY);
+
+	return 0;
+}
+
+static int at91_adc_buffer_postenable(struct iio_dev *indio_dev)
+{
+	if (at91_adc_current_chan_is_touch(indio_dev))
+		return 0;
+
 	return iio_triggered_buffer_postenable(indio_dev);
 }
 
-static int at91_adc_buffer_predisable(struct iio_dev *indio_dev)
+static int at91_adc_buffer_postdisable(struct iio_dev *indio_dev)
 {
 	struct at91_adc_state *st = iio_priv(indio_dev);
-	int ret;
 	u8 bit;
 
 	/* check if we are disabling triggered buffer or the touchscreen */
-	if (bitmap_subset(indio_dev->active_scan_mask,
-			  &st->touch_st.channels_bitmask,
-			  AT91_SAMA5D2_MAX_CHAN_IDX + 1)) {
-		/* touchscreen disable */
+	if (at91_adc_current_chan_is_touch(indio_dev))
 		return at91_adc_configure_touch(st, false);
-	}
+
 	/* if we are not in triggered mode, nothing to do here */
 	if (!(indio_dev->currentmode & INDIO_ALL_TRIGGERED_MODES))
 		return -EINVAL;
 
-	/* continue with the triggered buffer */
-	ret = iio_triggered_buffer_predisable(indio_dev);
-	if (ret < 0)
-		dev_err(&indio_dev->dev, "buffer predisable failed\n");
-
-	if (!st->dma_st.dma_chan)
-		return ret;
-
-	/* if we are using DMA we must clear registers and end DMA */
-	dmaengine_terminate_sync(st->dma_st.dma_chan);
-
 	/*
-	 * For each enabled channel we must read the last converted value
+	 * For each enable channel we must disable it in hardware.
+	 * In the case of DMA, we must read the last converted value
 	 * to clear EOC status and not get a possible interrupt later.
-	 * This value is being read by DMA from LCDR anyway
+	 * This value is being read by DMA from LCDR anyway, so it's not lost.
 	 */
 	for_each_set_bit(bit, indio_dev->active_scan_mask,
 			 indio_dev->num_channels) {
@@ -958,16 +975,37 @@ static int at91_adc_buffer_predisable(struct iio_dev *indio_dev)
 		if (chan->type == IIO_POSITIONRELATIVE ||
 		    chan->type == IIO_PRESSURE)
 			continue;
+
+		at91_adc_writel(st, AT91_SAMA5D2_CHDR, BIT(chan->channel));
+
 		if (st->dma_st.dma_chan)
 			at91_adc_readl(st, chan->address);
 	}
 
+	if (at91_adc_buffer_check_use_irq(indio_dev, st))
+		at91_adc_writel(st, AT91_SAMA5D2_IDR, AT91_SAMA5D2_IER_DRDY);
+
 	/* read overflow register to clear possible overflow status */
 	at91_adc_readl(st, AT91_SAMA5D2_OVER);
-	return ret;
+
+	/* if we are using DMA we must clear registers and end DMA */
+	if (st->dma_st.dma_chan)
+		dmaengine_terminate_sync(st->dma_st.dma_chan);
+
+	return 0;
+}
+
+static int at91_adc_buffer_predisable(struct iio_dev *indio_dev)
+{
+	if (at91_adc_current_chan_is_touch(indio_dev))
+		return 0;
+
+	return iio_triggered_buffer_predisable(indio_dev);
 }
 
 static const struct iio_buffer_setup_ops at91_buffer_setup_ops = {
+	.preenable = &at91_adc_buffer_preenable,
+	.postdisable = &at91_adc_buffer_postdisable,
 	.postenable = &at91_adc_buffer_postenable,
 	.predisable = &at91_adc_buffer_predisable,
 };
@@ -1015,6 +1053,22 @@ static void at91_adc_trigger_handler_nodma(struct iio_dev *indio_dev,
 	int i = 0;
 	int val;
 	u8 bit;
+	u32 mask = at91_adc_active_scan_mask_to_reg(indio_dev);
+	unsigned int timeout = 50;
+
+	/*
+	 * Check if the conversion is ready. If not, wait a little bit, and
+	 * in case of timeout exit with an error.
+	 */
+	while ((at91_adc_readl(st, AT91_SAMA5D2_ISR) & mask) != mask &&
+	       timeout) {
+		usleep_range(50, 100);
+		timeout--;
+	}
+
+	/* Cannot read data, not ready. Continue without reporting data */
+	if (!timeout)
+		return;
 
 	for_each_set_bit(bit, indio_dev->active_scan_mask,
 			 indio_dev->num_channels) {
@@ -1102,6 +1156,13 @@ static irqreturn_t at91_adc_trigger_handler(int irq, void *p)
 	struct iio_dev *indio_dev = pf->indio_dev;
 	struct at91_adc_state *st = iio_priv(indio_dev);
 
+	/*
+	 * If it's not our trigger, start a conversion now, as we are
+	 * actually polling the trigger now.
+	 */
+	if (iio_trigger_validate_own_device(indio_dev->trig, indio_dev))
+		at91_adc_writel(st, AT91_SAMA5D2_CR, AT91_SAMA5D2_CR_START);
+
 	if (st->dma_st.dma_chan)
 		at91_adc_trigger_handler_dma(indio_dev);
 	else
@@ -1114,20 +1175,9 @@ static irqreturn_t at91_adc_trigger_handler(int irq, void *p)
 
 static int at91_adc_buffer_init(struct iio_dev *indio)
 {
-	struct at91_adc_state *st = iio_priv(indio);
-
-	if (st->selected_trig->hw_trig) {
-		return devm_iio_triggered_buffer_setup(&indio->dev, indio,
-			&iio_pollfunc_store_time,
-			&at91_adc_trigger_handler, &at91_buffer_setup_ops);
-	}
-	/*
-	 * we need to prepare the buffer ops in case we will get
-	 * another buffer attached (like a callback buffer for the touchscreen)
-	 */
-	indio->setup_ops = &at91_buffer_setup_ops;
-
-	return 0;
+	return devm_iio_triggered_buffer_setup(&indio->dev, indio,
+		&iio_pollfunc_store_time,
+		&at91_adc_trigger_handler, &at91_buffer_setup_ops);
 }
 
 static unsigned at91_adc_startup_time(unsigned startup_time_min,
@@ -1281,7 +1331,8 @@ static irqreturn_t at91_adc_interrupt(int irq, void *private)
 		status = at91_adc_readl(st, AT91_SAMA5D2_XPOSR);
 		status = at91_adc_readl(st, AT91_SAMA5D2_YPOSR);
 		status = at91_adc_readl(st, AT91_SAMA5D2_PRESSR);
-	} else if (iio_buffer_enabled(indio) && !st->dma_st.dma_chan) {
+	} else if (iio_buffer_enabled(indio) &&
+		   (status & AT91_SAMA5D2_IER_DRDY)) {
 		/* triggered buffer without DMA */
 		disable_irq_nosync(irq);
 		iio_trigger_poll(indio->trig);
@@ -1901,14 +1952,10 @@ static __maybe_unused int at91_adc_resume(struct device *dev)
 		return 0;
 
 	/* check if we are enabling triggered buffer or the touchscreen */
-	if (bitmap_subset(indio_dev->active_scan_mask,
-			  &st->touch_st.channels_bitmask,
-			  AT91_SAMA5D2_MAX_CHAN_IDX + 1)) {
-		/* touchscreen enabling */
+	if (at91_adc_current_chan_is_touch(indio_dev))
 		return at91_adc_configure_touch(st, true);
-	} else {
+	else
 		return at91_adc_configure_trigger(st->trig, true);
-	}
 
 	/* not needed but more explicit */
 	return 0;
diff --git a/drivers/iio/adc/at91_adc.c b/drivers/iio/adc/at91_adc.c
index abe99856c823..0368b6dc6d60 100644
--- a/drivers/iio/adc/at91_adc.c
+++ b/drivers/iio/adc/at91_adc.c
@@ -1152,7 +1152,6 @@ static int at91_adc_probe(struct platform_device *pdev)
 	int ret;
 	struct iio_dev *idev;
 	struct at91_adc_state *st;
-	struct resource *res;
 	u32 reg;
 
 	idev = devm_iio_device_alloc(&pdev->dev, sizeof(struct at91_adc_state));
@@ -1182,9 +1181,7 @@ static int at91_adc_probe(struct platform_device *pdev)
 	if (st->irq < 0)
 		return -ENODEV;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-
-	st->reg_base = devm_ioremap_resource(&pdev->dev, res);
+	st->reg_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(st->reg_base))
 		return PTR_ERR(st->reg_base);
 
diff --git a/drivers/iio/adc/exynos_adc.c b/drivers/iio/adc/exynos_adc.c
index 22131a677445..6bda4f4d89fe 100644
--- a/drivers/iio/adc/exynos_adc.c
+++ b/drivers/iio/adc/exynos_adc.c
@@ -449,9 +449,6 @@ static void exynos_adc_exynos7_init_hw(struct exynos_adc *info)
 {
 	u32 con1, con2;
 
-	if (info->data->needs_adc_phy)
-		regmap_write(info->pmu_map, info->data->phy_offset, 1);
-
 	con1 = ADC_V2_CON1_SOFT_RESET;
 	writel(con1, ADC_V2_CON1(info->regs));
 
@@ -531,8 +528,19 @@ static int exynos_read_raw(struct iio_dev *indio_dev,
 	unsigned long timeout;
 	int ret;
 
-	if (mask != IIO_CHAN_INFO_RAW)
+	if (mask == IIO_CHAN_INFO_SCALE) {
+		ret = regulator_get_voltage(info->vdd);
+		if (ret < 0)
+			return ret;
+
+		/* Regulator voltage is in uV, but need mV */
+		*val = ret / 1000;
+		*val2 = info->data->mask;
+
+		return IIO_VAL_FRACTIONAL;
+	} else if (mask != IIO_CHAN_INFO_RAW) {
 		return -EINVAL;
+	}
 
 	mutex_lock(&indio_dev->mlock);
 	reinit_completion(&info->completion);
@@ -683,6 +691,7 @@ static const struct iio_info exynos_adc_iio_info = {
 	.channel = _index,				\
 	.address = _index,				\
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),	\
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SCALE),	\
 	.datasheet_name = _id,				\
 }
 
diff --git a/drivers/iio/adc/fsl-imx25-gcq.c b/drivers/iio/adc/fsl-imx25-gcq.c
index fa71489195c6..b0a4dc88ba9b 100644
--- a/drivers/iio/adc/fsl-imx25-gcq.c
+++ b/drivers/iio/adc/fsl-imx25-gcq.c
@@ -294,7 +294,6 @@ static int mx25_gcq_probe(struct platform_device *pdev)
 	struct mx25_gcq_priv *priv;
 	struct mx25_tsadc *tsadc = dev_get_drvdata(pdev->dev.parent);
 	struct device *dev = &pdev->dev;
-	struct resource *res;
 	void __iomem *mem;
 	int ret;
 	int i;
@@ -305,8 +304,7 @@ static int mx25_gcq_probe(struct platform_device *pdev)
 
 	priv = iio_priv(indio_dev);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	mem = devm_ioremap_resource(dev, res);
+	mem = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(mem))
 		return PTR_ERR(mem);
 
diff --git a/drivers/iio/adc/intel_mrfld_adc.c b/drivers/iio/adc/intel_mrfld_adc.c
index c35a1beb817c..a6d2e1f27e76 100644
--- a/drivers/iio/adc/intel_mrfld_adc.c
+++ b/drivers/iio/adc/intel_mrfld_adc.c
@@ -75,7 +75,7 @@ static int mrfld_adc_single_conv(struct iio_dev *indio_dev,
 	struct regmap *regmap = adc->regmap;
 	unsigned int req;
 	long timeout;
-	u8 buf[2];
+	__be16 value;
 	int ret;
 
 	reinit_completion(&adc->completion);
@@ -105,11 +105,11 @@ static int mrfld_adc_single_conv(struct iio_dev *indio_dev,
 		goto done;
 	}
 
-	ret = regmap_bulk_read(regmap, chan->address, buf, 2);
+	ret = regmap_bulk_read(regmap, chan->address, &value, sizeof(value));
 	if (ret)
 		goto done;
 
-	*result = get_unaligned_be16(buf);
+	*result = be16_to_cpu(value);
 	ret = IIO_VAL_INT;
 
 done:
diff --git a/drivers/iio/adc/max1241.c b/drivers/iio/adc/max1241.c
new file mode 100644
index 000000000000..541939c7abca
--- /dev/null
+++ b/drivers/iio/adc/max1241.c
@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * MAX1241 low-power, 12-bit serial ADC
+ *
+ * Datasheet: https://datasheets.maximintegrated.com/en/ds/MAX1240-MAX1241.pdf
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/iio/iio.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spi/spi.h>
+
+#define MAX1241_VAL_MASK GENMASK(11, 0)
+#define MAX1241_SHUTDOWN_DELAY_USEC 4
+
+enum max1241_id {
+	max1241,
+};
+
+struct max1241 {
+	struct spi_device *spi;
+	struct mutex lock;
+	struct regulator *vdd;
+	struct regulator *vref;
+	struct gpio_desc *shutdown;
+
+	__be16 data ____cacheline_aligned;
+};
+
+static const struct iio_chan_spec max1241_channels[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.channel = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				BIT(IIO_CHAN_INFO_SCALE),
+	},
+};
+
+static int max1241_read(struct max1241 *adc)
+{
+	struct spi_transfer xfers[] = {
+		/*
+		 * Begin conversion by bringing /CS low for at least
+		 * tconv us.
+		 */
+		{
+			.len = 0,
+			.delay.value = 8,
+			.delay.unit = SPI_DELAY_UNIT_USECS,
+		},
+		/*
+		 * Then read two bytes of data in our RX buffer.
+		 */
+		{
+			.rx_buf = &adc->data,
+			.len = 2,
+		},
+	};
+
+	return spi_sync_transfer(adc->spi, xfers, ARRAY_SIZE(xfers));
+}
+
+static int max1241_read_raw(struct iio_dev *indio_dev,
+			struct iio_chan_spec const *chan,
+			int *val, int *val2, long mask)
+{
+	int ret, vref_uV;
+	struct max1241 *adc = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&adc->lock);
+
+		if (adc->shutdown) {
+			gpiod_set_value(adc->shutdown, 0);
+			udelay(MAX1241_SHUTDOWN_DELAY_USEC);
+			ret = max1241_read(adc);
+			gpiod_set_value(adc->shutdown, 1);
+		} else
+			ret = max1241_read(adc);
+
+		if (ret) {
+			mutex_unlock(&adc->lock);
+			return ret;
+		}
+
+		*val = (be16_to_cpu(adc->data) >> 3) & MAX1241_VAL_MASK;
+
+		mutex_unlock(&adc->lock);
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		vref_uV = regulator_get_voltage(adc->vref);
+
+		if (vref_uV < 0)
+			return vref_uV;
+
+		*val = vref_uV / 1000;
+		*val2 = 12;
+
+		return IIO_VAL_FRACTIONAL_LOG2;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info max1241_info = {
+	.read_raw = max1241_read_raw,
+};
+
+static void max1241_disable_vdd_action(void *data)
+{
+	struct max1241 *adc = data;
+	struct device *dev = &adc->spi->dev;
+	int err;
+
+	err = regulator_disable(adc->vdd);
+	if (err)
+		dev_err(dev, "could not disable vdd regulator.\n");
+}
+
+static void max1241_disable_vref_action(void *data)
+{
+	struct max1241 *adc = data;
+	struct device *dev = &adc->spi->dev;
+	int err;
+
+	err = regulator_disable(adc->vref);
+	if (err)
+		dev_err(dev, "could not disable vref regulator.\n");
+}
+
+static int max1241_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct iio_dev *indio_dev;
+	struct max1241 *adc;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*adc));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	adc = iio_priv(indio_dev);
+	adc->spi = spi;
+	mutex_init(&adc->lock);
+
+	spi_set_drvdata(spi, indio_dev);
+
+	adc->vdd = devm_regulator_get(dev, "vdd");
+	if (IS_ERR(adc->vdd)) {
+		dev_err(dev, "failed to get vdd regulator\n");
+		return PTR_ERR(adc->vdd);
+	}
+
+	ret = regulator_enable(adc->vdd);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, max1241_disable_vdd_action, adc);
+	if (ret) {
+		dev_err(dev, "could not set up vdd regulator cleanup action\n");
+		return ret;
+	}
+
+	adc->vref = devm_regulator_get(dev, "vref");
+	if (IS_ERR(adc->vref)) {
+		dev_err(dev, "failed to get vref regulator\n");
+		return PTR_ERR(adc->vref);
+	}
+
+	ret = regulator_enable(adc->vref);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, max1241_disable_vref_action, adc);
+	if (ret) {
+		dev_err(dev, "could not set up vref regulator cleanup action\n");
+		return ret;
+	}
+
+	adc->shutdown = devm_gpiod_get_optional(dev, "shutdown",
+						GPIOD_OUT_HIGH);
+	if (IS_ERR(adc->shutdown))
+		return PTR_ERR(adc->shutdown);
+
+	if (adc->shutdown)
+		dev_dbg(dev, "shutdown pin passed, low-power mode enabled");
+	else
+		dev_dbg(dev, "no shutdown pin passed, low-power mode disabled");
+
+	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->dev.parent = dev;
+	indio_dev->info = &max1241_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = max1241_channels;
+	indio_dev->num_channels = ARRAY_SIZE(max1241_channels);
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct spi_device_id max1241_id[] = {
+	{ "max1241", max1241 },
+	{}
+};
+
+static const struct of_device_id max1241_dt_ids[] = {
+	{ .compatible = "maxim,max1241" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, max1241_dt_ids);
+
+static struct spi_driver max1241_spi_driver = {
+	.driver = {
+		.name = "max1241",
+		.of_match_table = max1241_dt_ids,
+	},
+	.probe = max1241_probe,
+	.id_table = max1241_id,
+};
+module_spi_driver(max1241_spi_driver);
+
+MODULE_AUTHOR("Alexandru Lazar <alazar@startmail.com>");
+MODULE_DESCRIPTION("MAX1241 ADC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/max1363.c b/drivers/iio/adc/max1363.c
index 5c2cc61b666e..9d92017c79b2 100644
--- a/drivers/iio/adc/max1363.c
+++ b/drivers/iio/adc/max1363.c
@@ -150,6 +150,7 @@ struct max1363_chip_info {
  * @current_mode:	the scan mode of this chip
  * @requestedmask:	a valid requested set of channels
  * @reg:		supply regulator
+ * @lock		lock to ensure state is consistent
  * @monitor_on:		whether monitor mode is enabled
  * @monitor_speed:	parameter corresponding to device monitor speed setting
  * @mask_high:		bitmask for enabled high thresholds
@@ -169,6 +170,7 @@ struct max1363_state {
 	const struct max1363_mode	*current_mode;
 	u32				requestedmask;
 	struct regulator		*reg;
+	struct mutex			lock;
 
 	/* Using monitor modes and buffer at the same time is
 	   currently not supported */
@@ -364,7 +366,11 @@ static int max1363_read_single_chan(struct iio_dev *indio_dev,
 	struct max1363_state *st = iio_priv(indio_dev);
 	struct i2c_client *client = st->client;
 
-	mutex_lock(&indio_dev->mlock);
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
+	mutex_lock(&st->lock);
+
 	/*
 	 * If monitor mode is enabled, the method for reading a single
 	 * channel will have to be rather different and has not yet
@@ -372,7 +378,7 @@ static int max1363_read_single_chan(struct iio_dev *indio_dev,
 	 *
 	 * Also, cannot read directly if buffered capture enabled.
 	 */
-	if (st->monitor_on || iio_buffer_enabled(indio_dev)) {
+	if (st->monitor_on) {
 		ret = -EBUSY;
 		goto error_ret;
 	}
@@ -404,8 +410,10 @@ static int max1363_read_single_chan(struct iio_dev *indio_dev,
 		data = rxbuf[0];
 	}
 	*val = data;
+
 error_ret:
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
+	iio_device_release_direct_mode(indio_dev);
 	return ret;
 
 }
@@ -705,9 +713,9 @@ static ssize_t max1363_monitor_store_freq(struct device *dev,
 	if (!found)
 		return -EINVAL;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	st->monitor_speed = i;
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return 0;
 }
@@ -810,12 +818,12 @@ static int max1363_read_event_config(struct iio_dev *indio_dev,
 	int val;
 	int number = chan->channel;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	if (dir == IIO_EV_DIR_FALLING)
 		val = (1 << number) & st->mask_low;
 	else
 		val = (1 << number) & st->mask_high;
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return val;
 }
@@ -962,7 +970,11 @@ static int max1363_write_event_config(struct iio_dev *indio_dev,
 	u16 unifiedmask;
 	int number = chan->channel;
 
-	mutex_lock(&indio_dev->mlock);
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
+	mutex_lock(&st->lock);
+
 	unifiedmask = st->mask_low | st->mask_high;
 	if (dir == IIO_EV_DIR_FALLING) {
 
@@ -989,7 +1001,8 @@ static int max1363_write_event_config(struct iio_dev *indio_dev,
 
 	max1363_monitor_mode_update(st, !!(st->mask_high | st->mask_low));
 error_ret:
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
+	iio_device_release_direct_mode(indio_dev);
 
 	return ret;
 }
@@ -1587,6 +1600,7 @@ static int max1363_probe(struct i2c_client *client,
 
 	st = iio_priv(indio_dev);
 
+	mutex_init(&st->lock);
 	st->reg = devm_regulator_get(&client->dev, "vcc");
 	if (IS_ERR(st->reg)) {
 		ret = PTR_ERR(st->reg);
diff --git a/drivers/iio/adc/mcp3422.c b/drivers/iio/adc/mcp3422.c
index ea24d7c58b12..d86c0b5d80a3 100644
--- a/drivers/iio/adc/mcp3422.c
+++ b/drivers/iio/adc/mcp3422.c
@@ -19,6 +19,7 @@
 #include <linux/delay.h>
 #include <linux/sysfs.h>
 #include <linux/of.h>
+#include <asm/unaligned.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
@@ -117,11 +118,11 @@ static int mcp3422_read(struct mcp3422 *adc, int *value, u8 *config)
 
 	if (sample_rate == MCP3422_SRATE_3) {
 		ret = i2c_master_recv(adc->i2c, buf, 4);
-		temp = buf[0] << 16 | buf[1] << 8 | buf[2];
+		temp = get_unaligned_be24(&buf[0]);
 		*config = buf[3];
 	} else {
 		ret = i2c_master_recv(adc->i2c, buf, 3);
-		temp = buf[0] << 8 | buf[1];
+		temp = get_unaligned_be16(&buf[0]);
 		*config = buf[2];
 	}
 
diff --git a/drivers/iio/adc/mp2629_adc.c b/drivers/iio/adc/mp2629_adc.c
new file mode 100644
index 000000000000..331a9a728217
--- /dev/null
+++ b/drivers/iio/adc/mp2629_adc.c
@@ -0,0 +1,208 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * MP2629 Driver for ADC
+ *
+ * Copyright 2020 Monolithic Power Systems, Inc
+ *
+ * Author: Saravanan Sekar <sravanhome@gmail.com>
+ */
+
+#include <linux/iio/driver.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/machine.h>
+#include <linux/mfd/mp2629.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define	MP2629_REG_ADC_CTRL		0x03
+#define	MP2629_REG_BATT_VOLT		0x0e
+#define	MP2629_REG_SYSTEM_VOLT		0x0f
+#define	MP2629_REG_INPUT_VOLT		0x11
+#define	MP2629_REG_BATT_CURRENT		0x12
+#define	MP2629_REG_INPUT_CURRENT	0x13
+
+#define	MP2629_ADC_START		BIT(7)
+#define	MP2629_ADC_CONTINUOUS		BIT(6)
+
+#define MP2629_MAP(_mp, _mpc) IIO_MAP(#_mp, "mp2629_charger", "mp2629-"_mpc)
+
+#define MP2629_ADC_CHAN(_ch, _type) {				\
+	.type = _type,						\
+	.indexed = 1,						\
+	.datasheet_name = #_ch,					\
+	.channel = MP2629_ ## _ch,				\
+	.address = MP2629_REG_ ## _ch,				\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
+}
+
+struct mp2629_adc {
+	struct regmap *regmap;
+	struct device *dev;
+};
+
+static struct iio_chan_spec mp2629_channels[] = {
+	MP2629_ADC_CHAN(BATT_VOLT, IIO_VOLTAGE),
+	MP2629_ADC_CHAN(SYSTEM_VOLT, IIO_VOLTAGE),
+	MP2629_ADC_CHAN(INPUT_VOLT, IIO_VOLTAGE),
+	MP2629_ADC_CHAN(BATT_CURRENT, IIO_CURRENT),
+	MP2629_ADC_CHAN(INPUT_CURRENT, IIO_CURRENT)
+};
+
+static struct iio_map mp2629_adc_maps[] = {
+	MP2629_MAP(BATT_VOLT, "batt-volt"),
+	MP2629_MAP(SYSTEM_VOLT, "system-volt"),
+	MP2629_MAP(INPUT_VOLT, "input-volt"),
+	MP2629_MAP(BATT_CURRENT, "batt-current"),
+	MP2629_MAP(INPUT_CURRENT, "input-current")
+};
+
+static int mp2629_read_raw(struct iio_dev *indio_dev,
+			struct iio_chan_spec const *chan,
+			int *val, int *val2, long mask)
+{
+	struct mp2629_adc *info = iio_priv(indio_dev);
+	unsigned int rval;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = regmap_read(info->regmap, chan->address, &rval);
+		if (ret)
+			return ret;
+
+		if (chan->address == MP2629_INPUT_VOLT)
+			rval &= GENMASK(6, 0);
+		*val = rval;
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->channel) {
+		case MP2629_BATT_VOLT:
+		case MP2629_SYSTEM_VOLT:
+			*val = 20;
+			return IIO_VAL_INT;
+
+		case MP2629_INPUT_VOLT:
+			*val = 60;
+			return IIO_VAL_INT;
+
+		case MP2629_BATT_CURRENT:
+			*val = 175;
+			*val2 = 10;
+			return IIO_VAL_FRACTIONAL;
+
+		case MP2629_INPUT_CURRENT:
+			*val = 133;
+			*val2 = 10;
+			return IIO_VAL_FRACTIONAL;
+
+		default:
+			return -EINVAL;
+		}
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info mp2629_adc_info = {
+	.read_raw = &mp2629_read_raw,
+};
+
+static int mp2629_adc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mp2629_data *ddata = dev_get_drvdata(dev->parent);
+	struct mp2629_adc *info;
+	struct iio_dev *indio_dev;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*info));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	info = iio_priv(indio_dev);
+	info->regmap = ddata->regmap;
+	info->dev = dev;
+	platform_set_drvdata(pdev, indio_dev);
+
+	ret = regmap_update_bits(info->regmap, MP2629_REG_ADC_CTRL,
+				MP2629_ADC_START | MP2629_ADC_CONTINUOUS,
+				MP2629_ADC_START | MP2629_ADC_CONTINUOUS);
+	if (ret) {
+		dev_err(dev, "adc enable fail: %d\n", ret);
+		return ret;
+	}
+
+	ret = iio_map_array_register(indio_dev, mp2629_adc_maps);
+	if (ret) {
+		dev_err(dev, "IIO maps register fail: %d\n", ret);
+		goto fail_disable;
+	}
+
+	indio_dev->name = "mp2629-adc";
+	indio_dev->dev.parent = dev;
+	indio_dev->channels = mp2629_channels;
+	indio_dev->num_channels = ARRAY_SIZE(mp2629_channels);
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->info = &mp2629_adc_info;
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(dev, "IIO device register fail: %d\n", ret);
+		goto fail_map_unregister;
+	}
+
+	return 0;
+
+fail_map_unregister:
+	iio_map_array_unregister(indio_dev);
+
+fail_disable:
+	regmap_update_bits(info->regmap, MP2629_REG_ADC_CTRL,
+					 MP2629_ADC_CONTINUOUS, 0);
+	regmap_update_bits(info->regmap, MP2629_REG_ADC_CTRL,
+					 MP2629_ADC_START, 0);
+
+	return ret;
+}
+
+static int mp2629_adc_remove(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct mp2629_adc *info = iio_priv(indio_dev);
+
+	iio_device_unregister(indio_dev);
+
+	iio_map_array_unregister(indio_dev);
+
+	regmap_update_bits(info->regmap, MP2629_REG_ADC_CTRL,
+					 MP2629_ADC_CONTINUOUS, 0);
+	regmap_update_bits(info->regmap, MP2629_REG_ADC_CTRL,
+					 MP2629_ADC_START, 0);
+
+	return 0;
+}
+
+static const struct of_device_id mp2629_adc_of_match[] = {
+	{ .compatible = "mps,mp2629_adc"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, mp2629_adc_of_match);
+
+static struct platform_driver mp2629_adc_driver = {
+	.driver = {
+		.name = "mp2629_adc",
+		.of_match_table = mp2629_adc_of_match,
+	},
+	.probe		= mp2629_adc_probe,
+	.remove		= mp2629_adc_remove,
+};
+module_platform_driver(mp2629_adc_driver);
+
+MODULE_AUTHOR("Saravanan Sekar <sravanhome@gmail.com>");
+MODULE_DESCRIPTION("MP2629 ADC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c
index 2df88d2b880a..0e2068ec068b 100644
--- a/drivers/iio/adc/stm32-adc-core.c
+++ b/drivers/iio/adc/stm32-adc-core.c
@@ -65,12 +65,14 @@ struct stm32_adc_priv;
  * @clk_sel:	clock selection routine
  * @max_clk_rate_hz: maximum analog clock rate (Hz, from datasheet)
  * @has_syscfg: SYSCFG capability flags
+ * @num_irqs:	number of interrupt lines
  */
 struct stm32_adc_priv_cfg {
 	const struct stm32_adc_common_regs *regs;
 	int (*clk_sel)(struct platform_device *, struct stm32_adc_priv *);
 	u32 max_clk_rate_hz;
 	unsigned int has_syscfg;
+	unsigned int num_irqs;
 };
 
 /**
@@ -375,21 +377,15 @@ static int stm32_adc_irq_probe(struct platform_device *pdev,
 	struct device_node *np = pdev->dev.of_node;
 	unsigned int i;
 
-	for (i = 0; i < STM32_ADC_MAX_ADCS; i++) {
+	/*
+	 * Interrupt(s) must be provided, depending on the compatible:
+	 * - stm32f4/h7 shares a common interrupt line.
+	 * - stm32mp1, has one line per ADC
+	 */
+	for (i = 0; i < priv->cfg->num_irqs; i++) {
 		priv->irq[i] = platform_get_irq(pdev, i);
-		if (priv->irq[i] < 0) {
-			/*
-			 * At least one interrupt must be provided, make others
-			 * optional:
-			 * - stm32f4/h7 shares a common interrupt.
-			 * - stm32mp1, has one line per ADC (either for ADC1,
-			 *   ADC2 or both).
-			 */
-			if (i && priv->irq[i] == -ENXIO)
-				continue;
-
+		if (priv->irq[i] < 0)
 			return priv->irq[i];
-		}
 	}
 
 	priv->domain = irq_domain_add_simple(np, STM32_ADC_MAX_ADCS, 0,
@@ -400,9 +396,7 @@ static int stm32_adc_irq_probe(struct platform_device *pdev,
 		return -ENOMEM;
 	}
 
-	for (i = 0; i < STM32_ADC_MAX_ADCS; i++) {
-		if (priv->irq[i] < 0)
-			continue;
+	for (i = 0; i < priv->cfg->num_irqs; i++) {
 		irq_set_chained_handler(priv->irq[i], stm32_adc_irq_handler);
 		irq_set_handler_data(priv->irq[i], priv);
 	}
@@ -420,11 +414,8 @@ static void stm32_adc_irq_remove(struct platform_device *pdev,
 		irq_dispose_mapping(irq_find_mapping(priv->domain, hwirq));
 	irq_domain_remove(priv->domain);
 
-	for (i = 0; i < STM32_ADC_MAX_ADCS; i++) {
-		if (priv->irq[i] < 0)
-			continue;
+	for (i = 0; i < priv->cfg->num_irqs; i++)
 		irq_set_chained_handler(priv->irq[i], NULL);
-	}
 }
 
 static int stm32_adc_core_switches_supply_en(struct stm32_adc_priv *priv,
@@ -817,6 +808,7 @@ static const struct stm32_adc_priv_cfg stm32f4_adc_priv_cfg = {
 	.regs = &stm32f4_adc_common_regs,
 	.clk_sel = stm32f4_adc_clk_sel,
 	.max_clk_rate_hz = 36000000,
+	.num_irqs = 1,
 };
 
 static const struct stm32_adc_priv_cfg stm32h7_adc_priv_cfg = {
@@ -824,6 +816,7 @@ static const struct stm32_adc_priv_cfg stm32h7_adc_priv_cfg = {
 	.clk_sel = stm32h7_adc_clk_sel,
 	.max_clk_rate_hz = 36000000,
 	.has_syscfg = HAS_VBOOSTER,
+	.num_irqs = 1,
 };
 
 static const struct stm32_adc_priv_cfg stm32mp1_adc_priv_cfg = {
@@ -831,6 +824,7 @@ static const struct stm32_adc_priv_cfg stm32mp1_adc_priv_cfg = {
 	.clk_sel = stm32h7_adc_clk_sel,
 	.max_clk_rate_hz = 40000000,
 	.has_syscfg = HAS_VBOOSTER | HAS_ANASWVDD,
+	.num_irqs = 2,
 };
 
 static const struct of_device_id stm32_adc_of_match[] = {
diff --git a/drivers/iio/adc/sun4i-gpadc-iio.c b/drivers/iio/adc/sun4i-gpadc-iio.c
index 176e1cb4abb1..0f2c1738a90d 100644
--- a/drivers/iio/adc/sun4i-gpadc-iio.c
+++ b/drivers/iio/adc/sun4i-gpadc-iio.c
@@ -496,7 +496,6 @@ static int sun4i_gpadc_probe_dt(struct platform_device *pdev,
 				struct iio_dev *indio_dev)
 {
 	struct sun4i_gpadc_iio *info = iio_priv(indio_dev);
-	struct resource *mem;
 	void __iomem *base;
 	int ret;
 
@@ -508,8 +507,7 @@ static int sun4i_gpadc_probe_dt(struct platform_device *pdev,
 	indio_dev->num_channels = ARRAY_SIZE(sun8i_a33_gpadc_channels);
 	indio_dev->channels = sun8i_a33_gpadc_channels;
 
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(&pdev->dev, mem);
+	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
diff --git a/drivers/iio/adc/ti-ads124s08.c b/drivers/iio/adc/ti-ads124s08.c
index 552c2be8d87a..f1ee3b1e2827 100644
--- a/drivers/iio/adc/ti-ads124s08.c
+++ b/drivers/iio/adc/ti-ads124s08.c
@@ -22,6 +22,8 @@
 #include <linux/iio/triggered_buffer.h>
 #include <linux/iio/sysfs.h>
 
+#include <asm/unaligned.h>
+
 /* Commands */
 #define ADS124S08_CMD_NOP	0x00
 #define ADS124S08_CMD_WAKEUP	0x02
@@ -188,7 +190,6 @@ static int ads124s_read(struct iio_dev *indio_dev, unsigned int chan)
 {
 	struct ads124s_private *priv = iio_priv(indio_dev);
 	int ret;
-	u32 tmp;
 	struct spi_transfer t[] = {
 		{
 			.tx_buf = &priv->data[0],
@@ -208,9 +209,7 @@ static int ads124s_read(struct iio_dev *indio_dev, unsigned int chan)
 	if (ret < 0)
 		return ret;
 
-	tmp = priv->data[2] << 16 | priv->data[3] << 8 | priv->data[4];
-
-	return tmp;
+	return get_unaligned_be24(&priv->data[2]);
 }
 
 static int ads124s_read_raw(struct iio_dev *indio_dev,
diff --git a/drivers/iio/adc/xilinx-xadc-core.c b/drivers/iio/adc/xilinx-xadc-core.c
index 6fd06e4eff73..d7fecab9252e 100644
--- a/drivers/iio/adc/xilinx-xadc-core.c
+++ b/drivers/iio/adc/xilinx-xadc-core.c
@@ -3,7 +3,7 @@
  * Xilinx XADC driver
  *
  * Copyright 2013-2014 Analog Devices Inc.
- *  Author: Lars-Peter Clauen <lars@metafoo.de>
+ *  Author: Lars-Peter Clausen <lars@metafoo.de>
  *
  * Documentation for the parts can be found at:
  *  - XADC hardmacro: Xilinx UG480
@@ -663,7 +663,7 @@ static int xadc_trigger_set_state(struct iio_trigger *trigger, bool state)
 	mutex_lock(&xadc->mutex);
 
 	if (state) {
-		/* Only one of the two triggers can be active at the a time. */
+		/* Only one of the two triggers can be active at a time. */
 		if (xadc->trigger != NULL) {
 			ret = -EBUSY;
 			goto err_out;
diff --git a/drivers/iio/adc/xilinx-xadc-events.c b/drivers/iio/adc/xilinx-xadc-events.c
index dbfd5da290a4..2357f585720a 100644
--- a/drivers/iio/adc/xilinx-xadc-events.c
+++ b/drivers/iio/adc/xilinx-xadc-events.c
@@ -3,7 +3,7 @@
  * Xilinx XADC driver
  *
  * Copyright 2013 Analog Devices Inc.
- *  Author: Lars-Peter Clauen <lars@metafoo.de>
+ *  Author: Lars-Peter Clausen <lars@metafoo.de>
  */
 
 #include <linux/iio/events.h>
diff --git a/drivers/iio/adc/xilinx-xadc.h b/drivers/iio/adc/xilinx-xadc.h
index 4017f18b0a4f..25abed9c0285 100644
--- a/drivers/iio/adc/xilinx-xadc.h
+++ b/drivers/iio/adc/xilinx-xadc.h
@@ -3,7 +3,7 @@
  * Xilinx XADC driver
  *
  * Copyright 2013 Analog Devices Inc.
- *  Author: Lars-Peter Clauen <lars@metafoo.de>
+ *  Author: Lars-Peter Clausen <lars@metafoo.de>
  */
 
 #ifndef __IIO_XILINX_XADC__
diff --git a/drivers/iio/buffer/industrialio-buffer-dma.c b/drivers/iio/buffer/industrialio-buffer-dma.c
index a74bd9c0587c..d348af8b9705 100644
--- a/drivers/iio/buffer/industrialio-buffer-dma.c
+++ b/drivers/iio/buffer/industrialio-buffer-dma.c
@@ -12,7 +12,6 @@
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/poll.h>
-#include <linux/iio/buffer.h>
 #include <linux/iio/buffer_impl.h>
 #include <linux/iio/buffer-dma.h>
 #include <linux/dma-mapping.h>
diff --git a/drivers/iio/buffer/industrialio-buffer-dmaengine.c b/drivers/iio/buffer/industrialio-buffer-dmaengine.c
index b129693af0fd..6dedf12b69a4 100644
--- a/drivers/iio/buffer/industrialio-buffer-dmaengine.c
+++ b/drivers/iio/buffer/industrialio-buffer-dmaengine.c
@@ -134,7 +134,7 @@ static ssize_t iio_dmaengine_buffer_get_length_align(struct device *dev,
 	struct dmaengine_buffer *dmaengine_buffer =
 		iio_buffer_to_dmaengine_buffer(indio_dev->buffer);
 
-	return sprintf(buf, "%u\n", dmaengine_buffer->align);
+	return sprintf(buf, "%zu\n", dmaengine_buffer->align);
 }
 
 static IIO_DEVICE_ATTR(length_align_bytes, 0444,
@@ -229,6 +229,45 @@ void iio_dmaengine_buffer_free(struct iio_buffer *buffer)
 }
 EXPORT_SYMBOL_GPL(iio_dmaengine_buffer_free);
 
+static void __devm_iio_dmaengine_buffer_free(struct device *dev, void *res)
+{
+	iio_dmaengine_buffer_free(*(struct iio_buffer **)res);
+}
+
+/**
+ * devm_iio_dmaengine_buffer_alloc() - Resource-managed iio_dmaengine_buffer_alloc()
+ * @dev: Parent device for the buffer
+ * @channel: DMA channel name, typically "rx".
+ *
+ * This allocates a new IIO buffer which internally uses the DMAengine framework
+ * to perform its transfers. The parent device will be used to request the DMA
+ * channel.
+ *
+ * The buffer will be automatically de-allocated once the device gets destroyed.
+ */
+struct iio_buffer *devm_iio_dmaengine_buffer_alloc(struct device *dev,
+	const char *channel)
+{
+	struct iio_buffer **bufferp, *buffer;
+
+	bufferp = devres_alloc(__devm_iio_dmaengine_buffer_free,
+			       sizeof(*bufferp), GFP_KERNEL);
+	if (!bufferp)
+		return ERR_PTR(-ENOMEM);
+
+	buffer = iio_dmaengine_buffer_alloc(dev, channel);
+	if (IS_ERR(buffer)) {
+		devres_free(bufferp);
+		return buffer;
+	}
+
+	*bufferp = buffer;
+	devres_add(dev, bufferp);
+
+	return buffer;
+}
+EXPORT_SYMBOL_GPL(devm_iio_dmaengine_buffer_alloc);
+
 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
 MODULE_DESCRIPTION("DMA buffer for the IIO framework");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/buffer/industrialio-hw-consumer.c b/drivers/iio/buffer/industrialio-hw-consumer.c
index 95165697d8ae..f2d27788f666 100644
--- a/drivers/iio/buffer/industrialio-hw-consumer.c
+++ b/drivers/iio/buffer/industrialio-hw-consumer.c
@@ -142,17 +142,6 @@ static void devm_iio_hw_consumer_release(struct device *dev, void *res)
 	iio_hw_consumer_free(*(struct iio_hw_consumer **)res);
 }
 
-static int devm_iio_hw_consumer_match(struct device *dev, void *res, void *data)
-{
-	struct iio_hw_consumer **r = res;
-
-	if (!r || !*r) {
-		WARN_ON(!r || !*r);
-		return 0;
-	}
-	return *r == data;
-}
-
 /**
  * devm_iio_hw_consumer_alloc - Resource-managed iio_hw_consumer_alloc()
  * @dev: Pointer to consumer device.
@@ -160,9 +149,6 @@ static int devm_iio_hw_consumer_match(struct device *dev, void *res, void *data)
  * Managed iio_hw_consumer_alloc. iio_hw_consumer allocated with this function
  * is automatically freed on driver detach.
  *
- * If an iio_hw_consumer allocated with this function needs to be freed
- * separately, devm_iio_hw_consumer_free() must be used.
- *
  * returns pointer to allocated iio_hw_consumer on success, NULL on failure.
  */
 struct iio_hw_consumer *devm_iio_hw_consumer_alloc(struct device *dev)
@@ -187,23 +173,6 @@ struct iio_hw_consumer *devm_iio_hw_consumer_alloc(struct device *dev)
 EXPORT_SYMBOL_GPL(devm_iio_hw_consumer_alloc);
 
 /**
- * devm_iio_hw_consumer_free - Resource-managed iio_hw_consumer_free()
- * @dev: Pointer to consumer device.
- * @hwc: iio_hw_consumer to free.
- *
- * Free iio_hw_consumer allocated with devm_iio_hw_consumer_alloc().
- */
-void devm_iio_hw_consumer_free(struct device *dev, struct iio_hw_consumer *hwc)
-{
-	int rc;
-
-	rc = devres_release(dev, devm_iio_hw_consumer_release,
-			    devm_iio_hw_consumer_match, hwc);
-	WARN_ON(rc);
-}
-EXPORT_SYMBOL_GPL(devm_iio_hw_consumer_free);
-
-/**
  * iio_hw_consumer_enable() - Enable IIO hardware consumer
  * @hwc: iio_hw_consumer to enable.
  *
diff --git a/drivers/iio/buffer/industrialio-triggered-buffer.c b/drivers/iio/buffer/industrialio-triggered-buffer.c
index cb322b2f09cd..e8046c1ecd6b 100644
--- a/drivers/iio/buffer/industrialio-triggered-buffer.c
+++ b/drivers/iio/buffer/industrialio-triggered-buffer.c
@@ -126,17 +126,6 @@ int devm_iio_triggered_buffer_setup(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_iio_triggered_buffer_setup);
 
-void devm_iio_triggered_buffer_cleanup(struct device *dev,
-				       struct iio_dev *indio_dev)
-{
-	int rc;
-
-	rc = devres_release(dev, devm_iio_triggered_buffer_clean,
-			    devm_iio_device_match, indio_dev);
-	WARN_ON(rc);
-}
-EXPORT_SYMBOL_GPL(devm_iio_triggered_buffer_cleanup);
-
 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
 MODULE_DESCRIPTION("IIO helper functions for setting up triggered buffers");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/buffer/kfifo_buf.c b/drivers/iio/buffer/kfifo_buf.c
index 3150f8ab984b..1359abed3b31 100644
--- a/drivers/iio/buffer/kfifo_buf.c
+++ b/drivers/iio/buffer/kfifo_buf.c
@@ -179,16 +179,6 @@ static void devm_iio_kfifo_release(struct device *dev, void *res)
 	iio_kfifo_free(*(struct iio_buffer **)res);
 }
 
-static int devm_iio_kfifo_match(struct device *dev, void *res, void *data)
-{
-	struct iio_buffer **r = res;
-
-	if (WARN_ON(!r || !*r))
-		return 0;
-
-	return *r == data;
-}
-
 /**
  * devm_iio_fifo_allocate - Resource-managed iio_kfifo_allocate()
  * @dev:		Device to allocate kfifo buffer for
@@ -216,16 +206,4 @@ struct iio_buffer *devm_iio_kfifo_allocate(struct device *dev)
 }
 EXPORT_SYMBOL(devm_iio_kfifo_allocate);
 
-/**
- * devm_iio_fifo_free - Resource-managed iio_kfifo_free()
- * @dev:		Device the buffer belongs to
- * @r:			The buffer associated with the device
- */
-void devm_iio_kfifo_free(struct device *dev, struct iio_buffer *r)
-{
-	WARN_ON(devres_release(dev, devm_iio_kfifo_release,
-			       devm_iio_kfifo_match, r));
-}
-EXPORT_SYMBOL(devm_iio_kfifo_free);
-
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/chemical/Kconfig b/drivers/iio/chemical/Kconfig
index a7e65a59bf42..7f21afd73b1c 100644
--- a/drivers/iio/chemical/Kconfig
+++ b/drivers/iio/chemical/Kconfig
@@ -22,6 +22,17 @@ config ATLAS_PH_SENSOR
 	  To compile this driver as module, choose M here: the
 	  module will be called atlas-ph-sensor.
 
+config ATLAS_EZO_SENSOR
+	tristate "Atlas Scientific EZO sensors"
+	depends on I2C
+	help
+	  Say Y here to build I2C interface support for the following
+	  Atlas Scientific EZO sensors
+	    * CO2 EZO Sensor
+
+	  To compile this driver as module, choose M here: the
+	  module will be called atlas-ezo-sensor.
+
 config BME680
 	tristate "Bosch Sensortec BME680 sensor driver"
 	depends on (I2C || SPI)
diff --git a/drivers/iio/chemical/Makefile b/drivers/iio/chemical/Makefile
index 33d3a595dda9..aba4167db745 100644
--- a/drivers/iio/chemical/Makefile
+++ b/drivers/iio/chemical/Makefile
@@ -5,6 +5,7 @@
 
 # When adding new entries keep the list in alphabetical order
 obj-$(CONFIG_ATLAS_PH_SENSOR)	+= atlas-sensor.o
+obj-$(CONFIG_ATLAS_EZO_SENSOR)	+= atlas-ezo-sensor.o
 obj-$(CONFIG_BME680) += bme680_core.o
 obj-$(CONFIG_BME680_I2C) += bme680_i2c.o
 obj-$(CONFIG_BME680_SPI) += bme680_spi.o
diff --git a/drivers/iio/chemical/atlas-ezo-sensor.c b/drivers/iio/chemical/atlas-ezo-sensor.c
new file mode 100644
index 000000000000..f5a6d8ec6d4d
--- /dev/null
+++ b/drivers/iio/chemical/atlas-ezo-sensor.c
@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * atlas-ezo-sensor.c - Support for Atlas Scientific EZO sensors
+ *
+ * Copyright (C) 2020 Konsulko Group
+ * Author: Matt Ranostay <matt.ranostay@konsulko.com>
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/mutex.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/of_device.h>
+#include <linux/iio/iio.h>
+
+#define ATLAS_EZO_DRV_NAME		"atlas-ezo-sensor"
+#define ATLAS_CO2_INT_TIME_IN_MS	950
+
+enum {
+	ATLAS_CO2_EZO,
+};
+
+struct atlas_ezo_device {
+	const struct iio_chan_spec *channels;
+	int num_channels;
+	int delay;
+};
+
+struct atlas_ezo_data {
+	struct i2c_client *client;
+	struct atlas_ezo_device *chip;
+
+	/* lock to avoid multiple concurrent read calls */
+	struct mutex lock;
+
+	u8 buffer[8];
+};
+
+static const struct iio_chan_spec atlas_co2_ezo_channels[] = {
+	{
+		.type = IIO_CONCENTRATION,
+		.modified = 1,
+		.channel2 = IIO_MOD_CO2,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 32,
+			.storagebits = 32,
+			.endianness = IIO_CPU,
+		},
+	},
+};
+
+static struct atlas_ezo_device atlas_ezo_devices[] = {
+	[ATLAS_CO2_EZO] = {
+		.channels = atlas_co2_ezo_channels,
+		.num_channels = 1,
+		.delay = ATLAS_CO2_INT_TIME_IN_MS,
+	},
+};
+
+static int atlas_ezo_read_raw(struct iio_dev *indio_dev,
+			  struct iio_chan_spec const *chan,
+			  int *val, int *val2, long mask)
+{
+	struct atlas_ezo_data *data = iio_priv(indio_dev);
+	struct i2c_client *client = data->client;
+	int ret = 0;
+
+	if (chan->type != IIO_CONCENTRATION)
+		return -EINVAL;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW: {
+		long tmp;
+
+		mutex_lock(&data->lock);
+
+		tmp = i2c_smbus_write_byte(client, 'R');
+
+		if (tmp < 0) {
+			mutex_unlock(&data->lock);
+			return tmp;
+		}
+
+		msleep(data->chip->delay);
+
+		tmp = i2c_master_recv(client, data->buffer, sizeof(data->buffer));
+
+		if (tmp < 0 || data->buffer[0] != 1) {
+			mutex_unlock(&data->lock);
+			return -EBUSY;
+		}
+
+		ret = kstrtol(data->buffer + 1, 10, &tmp);
+
+		*val = tmp;
+
+		mutex_unlock(&data->lock);
+
+		return ret ? ret : IIO_VAL_INT;
+	}
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		*val2 = 100; /* 0.0001 */
+		return IIO_VAL_INT_PLUS_MICRO;
+	}
+
+	return ret;
+}
+
+static const struct iio_info atlas_info = {
+	.read_raw = atlas_ezo_read_raw,
+};
+
+static const struct i2c_device_id atlas_ezo_id[] = {
+	{ "atlas-co2-ezo", ATLAS_CO2_EZO },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, atlas_ezo_id);
+
+static const struct of_device_id atlas_ezo_dt_ids[] = {
+	{ .compatible = "atlas,co2-ezo", .data = (void *)ATLAS_CO2_EZO, },
+	{}
+};
+MODULE_DEVICE_TABLE(of, atlas_ezo_dt_ids);
+
+static int atlas_ezo_probe(struct i2c_client *client,
+		       const struct i2c_device_id *id)
+{
+	struct atlas_ezo_data *data;
+	struct atlas_ezo_device *chip;
+	const struct of_device_id *of_id;
+	struct iio_dev *indio_dev;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	of_id = of_match_device(atlas_ezo_dt_ids, &client->dev);
+	if (!of_id)
+		chip = &atlas_ezo_devices[id->driver_data];
+	else
+		chip = &atlas_ezo_devices[(unsigned long)of_id->data];
+
+	indio_dev->info = &atlas_info;
+	indio_dev->name = ATLAS_EZO_DRV_NAME;
+	indio_dev->channels = chip->channels;
+	indio_dev->num_channels = chip->num_channels;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->dev.parent = &client->dev;
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+	data->chip = chip;
+	mutex_init(&data->lock);
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+};
+
+static struct i2c_driver atlas_ezo_driver = {
+	.driver = {
+		.name	= ATLAS_EZO_DRV_NAME,
+		.of_match_table	= atlas_ezo_dt_ids,
+	},
+	.probe		= atlas_ezo_probe,
+	.id_table	= atlas_ezo_id,
+};
+module_i2c_driver(atlas_ezo_driver);
+
+MODULE_AUTHOR("Matt Ranostay <matt.ranostay@konsulko.com>");
+MODULE_DESCRIPTION("Atlas Scientific EZO sensors");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/chemical/atlas-sensor.c b/drivers/iio/chemical/atlas-sensor.c
index 7b199ce16ecf..78a27e36bf32 100644
--- a/drivers/iio/chemical/atlas-sensor.c
+++ b/drivers/iio/chemical/atlas-sensor.c
@@ -53,6 +53,8 @@
 #define ATLAS_REG_DO_CALIB_STATUS_PRESSURE	BIT(0)
 #define ATLAS_REG_DO_CALIB_STATUS_DO		BIT(1)
 
+#define ATLAS_REG_RTD_DATA		0x0e
+
 #define ATLAS_REG_PH_TEMP_DATA		0x0e
 #define ATLAS_REG_PH_DATA		0x16
 
@@ -72,12 +74,14 @@
 #define ATLAS_EC_INT_TIME_IN_MS		650
 #define ATLAS_ORP_INT_TIME_IN_MS	450
 #define ATLAS_DO_INT_TIME_IN_MS		450
+#define ATLAS_RTD_INT_TIME_IN_MS	450
 
 enum {
 	ATLAS_PH_SM,
 	ATLAS_EC_SM,
 	ATLAS_ORP_SM,
 	ATLAS_DO_SM,
+	ATLAS_RTD_SM,
 };
 
 struct atlas_data {
@@ -218,6 +222,22 @@ static const struct iio_chan_spec atlas_do_channels[] = {
 	},
 };
 
+static const struct iio_chan_spec atlas_rtd_channels[] = {
+	{
+		.type = IIO_TEMP,
+		.address = ATLAS_REG_RTD_DATA,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 32,
+			.storagebits = 32,
+			.endianness = IIO_BE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
 static int atlas_check_ph_calibration(struct atlas_data *data)
 {
 	struct device *dev = &data->client->dev;
@@ -362,6 +382,12 @@ static struct atlas_device atlas_devices[] = {
 				.calibration = &atlas_check_do_calibration,
 				.delay = ATLAS_DO_INT_TIME_IN_MS,
 	},
+	[ATLAS_RTD_SM] = {
+				.channels = atlas_rtd_channels,
+				.num_channels = 2,
+				.data_reg = ATLAS_REG_RTD_DATA,
+				.delay = ATLAS_RTD_INT_TIME_IN_MS,
+	},
 };
 
 static int atlas_set_powermode(struct atlas_data *data, int on)
@@ -438,8 +464,7 @@ static irqreturn_t atlas_trigger_handler(int irq, void *private)
 	int ret;
 
 	ret = regmap_bulk_read(data->regmap, data->chip->data_reg,
-			      (u8 *) &data->buffer,
-			      sizeof(__be32) * channels);
+			      &data->buffer, sizeof(__be32) * channels);
 
 	if (!ret)
 		iio_push_to_buffers_with_timestamp(indio_dev, data->buffer,
@@ -475,7 +500,7 @@ static int atlas_read_measurement(struct atlas_data *data, int reg, __be32 *val)
 	if (suspended)
 		msleep(data->chip->delay);
 
-	ret = regmap_bulk_read(data->regmap, reg, (u8 *) val, sizeof(*val));
+	ret = regmap_bulk_read(data->regmap, reg, val, sizeof(*val));
 
 	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
@@ -490,6 +515,7 @@ static int atlas_read_raw(struct iio_dev *indio_dev,
 	struct atlas_data *data = iio_priv(indio_dev);
 
 	switch (mask) {
+	case IIO_CHAN_INFO_PROCESSED:
 	case IIO_CHAN_INFO_RAW: {
 		int ret;
 		__be32 reg;
@@ -497,7 +523,7 @@ static int atlas_read_raw(struct iio_dev *indio_dev,
 		switch (chan->type) {
 		case IIO_TEMP:
 			ret = regmap_bulk_read(data->regmap, chan->address,
-					      (u8 *) &reg, sizeof(reg));
+					       &reg, sizeof(reg));
 			break;
 		case IIO_PH:
 		case IIO_CONCENTRATION:
@@ -578,6 +604,7 @@ static const struct i2c_device_id atlas_id[] = {
 	{ "atlas-ec-sm", ATLAS_EC_SM},
 	{ "atlas-orp-sm", ATLAS_ORP_SM},
 	{ "atlas-do-sm", ATLAS_DO_SM},
+	{ "atlas-rtd-sm", ATLAS_RTD_SM},
 	{}
 };
 MODULE_DEVICE_TABLE(i2c, atlas_id);
@@ -587,6 +614,7 @@ static const struct of_device_id atlas_dt_ids[] = {
 	{ .compatible = "atlas,ec-sm", .data = (void *)ATLAS_EC_SM, },
 	{ .compatible = "atlas,orp-sm", .data = (void *)ATLAS_ORP_SM, },
 	{ .compatible = "atlas,do-sm", .data = (void *)ATLAS_DO_SM, },
+	{ .compatible = "atlas,rtd-sm", .data = (void *)ATLAS_RTD_SM, },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, atlas_dt_ids);
diff --git a/drivers/iio/chemical/bme680_core.c b/drivers/iio/chemical/bme680_core.c
index ccde4c65ff93..13773e01699b 100644
--- a/drivers/iio/chemical/bme680_core.c
+++ b/drivers/iio/chemical/bme680_core.c
@@ -114,14 +114,16 @@ static int bme680_read_calib(struct bme680_data *data,
 	__le16 buf;
 
 	/* Temperature related coefficients */
-	ret = regmap_bulk_read(data->regmap, BME680_T1_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_T1_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_T1_LSB_REG\n");
 		return ret;
 	}
 	calib->par_t1 = le16_to_cpu(buf);
 
-	ret = regmap_bulk_read(data->regmap, BME680_T2_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_T2_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_T2_LSB_REG\n");
 		return ret;
@@ -136,14 +138,16 @@ static int bme680_read_calib(struct bme680_data *data,
 	calib->par_t3 = tmp;
 
 	/* Pressure related coefficients */
-	ret = regmap_bulk_read(data->regmap, BME680_P1_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_P1_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_P1_LSB_REG\n");
 		return ret;
 	}
 	calib->par_p1 = le16_to_cpu(buf);
 
-	ret = regmap_bulk_read(data->regmap, BME680_P2_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_P2_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_P2_LSB_REG\n");
 		return ret;
@@ -157,14 +161,16 @@ static int bme680_read_calib(struct bme680_data *data,
 	}
 	calib->par_p3 = tmp;
 
-	ret = regmap_bulk_read(data->regmap, BME680_P4_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_P4_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_P4_LSB_REG\n");
 		return ret;
 	}
 	calib->par_p4 = le16_to_cpu(buf);
 
-	ret = regmap_bulk_read(data->regmap, BME680_P5_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_P5_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_P5_LSB_REG\n");
 		return ret;
@@ -185,14 +191,16 @@ static int bme680_read_calib(struct bme680_data *data,
 	}
 	calib->par_p7 = tmp;
 
-	ret = regmap_bulk_read(data->regmap, BME680_P8_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_P8_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_P8_LSB_REG\n");
 		return ret;
 	}
 	calib->par_p8 = le16_to_cpu(buf);
 
-	ret = regmap_bulk_read(data->regmap, BME680_P9_LSB_REG, (u8 *) &buf, 2);
+	ret = regmap_bulk_read(data->regmap, BME680_P9_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_P9_LSB_REG\n");
 		return ret;
@@ -276,8 +284,8 @@ static int bme680_read_calib(struct bme680_data *data,
 	}
 	calib->par_gh1 = tmp;
 
-	ret = regmap_bulk_read(data->regmap, BME680_GH2_LSB_REG, (u8 *) &buf,
-			       2);
+	ret = regmap_bulk_read(data->regmap, BME680_GH2_LSB_REG,
+			       &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(dev, "failed to read BME680_GH2_LSB_REG\n");
 		return ret;
@@ -615,7 +623,7 @@ static int bme680_read_temp(struct bme680_data *data, int *val)
 		return ret;
 
 	ret = regmap_bulk_read(data->regmap, BME680_REG_TEMP_MSB,
-			       (u8 *) &tmp, 3);
+			       &tmp, 3);
 	if (ret < 0) {
 		dev_err(dev, "failed to read temperature\n");
 		return ret;
@@ -656,7 +664,7 @@ static int bme680_read_press(struct bme680_data *data,
 		return ret;
 
 	ret = regmap_bulk_read(data->regmap, BME680_REG_PRESS_MSB,
-			       (u8 *) &tmp, 3);
+			       &tmp, 3);
 	if (ret < 0) {
 		dev_err(dev, "failed to read pressure\n");
 		return ret;
@@ -689,7 +697,7 @@ static int bme680_read_humid(struct bme680_data *data,
 		return ret;
 
 	ret = regmap_bulk_read(data->regmap, BM6880_REG_HUMIDITY_MSB,
-			       (u8 *) &tmp, 2);
+			       &tmp, sizeof(tmp));
 	if (ret < 0) {
 		dev_err(dev, "failed to read humidity\n");
 		return ret;
@@ -754,7 +762,7 @@ static int bme680_read_gas(struct bme680_data *data,
 	}
 
 	ret = regmap_bulk_read(data->regmap, BME680_REG_GAS_MSB,
-			       (u8 *) &tmp, 2);
+			       &tmp, sizeof(tmp));
 	if (ret < 0) {
 		dev_err(dev, "failed to read gas resistance\n");
 		return ret;
diff --git a/drivers/iio/chemical/ccs811.c b/drivers/iio/chemical/ccs811.c
index 2ebdfc35bcda..3ecd633f9ed3 100644
--- a/drivers/iio/chemical/ccs811.c
+++ b/drivers/iio/chemical/ccs811.c
@@ -16,6 +16,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/buffer.h>
@@ -36,6 +37,7 @@
 #define CCS811_ERR		0xE0
 /* Used to transition from boot to application mode */
 #define CCS811_APP_START	0xF4
+#define CCS811_SW_RESET		0xFF
 
 /* Status register flags */
 #define CCS811_STATUS_ERROR		BIT(0)
@@ -74,6 +76,7 @@ struct ccs811_data {
 	struct mutex lock; /* Protect readings */
 	struct ccs811_reading buffer;
 	struct iio_trigger *drdy_trig;
+	struct gpio_desc *wakeup_gpio;
 	bool drdy_trig_on;
 };
 
@@ -166,10 +169,25 @@ static int ccs811_setup(struct i2c_client *client)
 					 CCS811_MODE_IAQ_1SEC);
 }
 
+static void ccs811_set_wakeup(struct ccs811_data *data, bool enable)
+{
+	if (!data->wakeup_gpio)
+		return;
+
+	gpiod_set_value(data->wakeup_gpio, enable);
+
+	if (enable)
+		usleep_range(50, 60);
+	else
+		usleep_range(20, 30);
+}
+
 static int ccs811_get_measurement(struct ccs811_data *data)
 {
 	int ret, tries = 11;
 
+	ccs811_set_wakeup(data, true);
+
 	/* Maximum waiting time: 1s, as measurements are made every second */
 	while (tries-- > 0) {
 		ret = i2c_smbus_read_byte_data(data->client, CCS811_STATUS);
@@ -183,9 +201,12 @@ static int ccs811_get_measurement(struct ccs811_data *data)
 	if (!(ret & CCS811_STATUS_DATA_READY))
 		return -EIO;
 
-	return i2c_smbus_read_i2c_block_data(data->client,
+	ret = i2c_smbus_read_i2c_block_data(data->client,
 					    CCS811_ALG_RESULT_DATA, 8,
 					    (char *)&data->buffer);
+	ccs811_set_wakeup(data, false);
+
+	return ret;
 }
 
 static int ccs811_read_raw(struct iio_dev *indio_dev,
@@ -336,6 +357,45 @@ static irqreturn_t ccs811_data_rdy_trigger_poll(int irq, void *private)
 	return IRQ_HANDLED;
 }
 
+static int ccs811_reset(struct i2c_client *client)
+{
+	struct gpio_desc *reset_gpio;
+	int ret;
+
+	reset_gpio = devm_gpiod_get_optional(&client->dev, "reset",
+					     GPIOD_OUT_LOW);
+	if (IS_ERR(reset_gpio))
+		return PTR_ERR(reset_gpio);
+
+	/* Try to reset using nRESET pin if available else do SW reset */
+	if (reset_gpio) {
+		gpiod_set_value(reset_gpio, 1);
+		usleep_range(20, 30);
+		gpiod_set_value(reset_gpio, 0);
+	} else {
+		/*
+		 * As per the datasheet, this sequence of values needs to be
+		 * written to the SW_RESET register for triggering the soft
+		 * reset in the device and placing it in boot mode.
+		 */
+		static const u8 reset_seq[] = {
+			0x11, 0xE5, 0x72, 0x8A,
+		};
+
+		ret = i2c_smbus_write_i2c_block_data(client, CCS811_SW_RESET,
+					     sizeof(reset_seq), reset_seq);
+		if (ret < 0) {
+			dev_err(&client->dev, "Failed to reset sensor\n");
+			return ret;
+		}
+	}
+
+	/* tSTART delay required after reset */
+	usleep_range(1000, 2000);
+
+	return 0;
+}
+
 static int ccs811_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
@@ -348,36 +408,59 @@ static int ccs811_probe(struct i2c_client *client,
 				     | I2C_FUNC_SMBUS_READ_I2C_BLOCK))
 		return -EOPNOTSUPP;
 
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	i2c_set_clientdata(client, indio_dev);
+	data->client = client;
+
+	data->wakeup_gpio = devm_gpiod_get_optional(&client->dev, "wakeup",
+						    GPIOD_OUT_HIGH);
+	if (IS_ERR(data->wakeup_gpio))
+		return PTR_ERR(data->wakeup_gpio);
+
+	ccs811_set_wakeup(data, true);
+
+	ret = ccs811_reset(client);
+	if (ret) {
+		ccs811_set_wakeup(data, false);
+		return ret;
+	}
+
 	/* Check hardware id (should be 0x81 for this family of devices) */
 	ret = i2c_smbus_read_byte_data(client, CCS811_HW_ID);
-	if (ret < 0)
+	if (ret < 0) {
+		ccs811_set_wakeup(data, false);
 		return ret;
+	}
 
 	if (ret != CCS811_HW_ID_VALUE) {
 		dev_err(&client->dev, "hardware id doesn't match CCS81x\n");
+		ccs811_set_wakeup(data, false);
 		return -ENODEV;
 	}
 
 	ret = i2c_smbus_read_byte_data(client, CCS811_HW_VERSION);
-	if (ret < 0)
+	if (ret < 0) {
+		ccs811_set_wakeup(data, false);
 		return ret;
+	}
 
 	if ((ret & CCS811_HW_VERSION_MASK) != CCS811_HW_VERSION_VALUE) {
 		dev_err(&client->dev, "no CCS811 sensor\n");
+		ccs811_set_wakeup(data, false);
 		return -ENODEV;
 	}
 
-	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
-	if (!indio_dev)
-		return -ENOMEM;
-
 	ret = ccs811_setup(client);
-	if (ret < 0)
+	if (ret < 0) {
+		ccs811_set_wakeup(data, false);
 		return ret;
+	}
 
-	data = iio_priv(indio_dev);
-	i2c_set_clientdata(client, indio_dev);
-	data->client = client;
+	ccs811_set_wakeup(data, false);
 
 	mutex_init(&data->lock);
 
@@ -466,9 +549,16 @@ static const struct i2c_device_id ccs811_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, ccs811_id);
 
+static const struct of_device_id ccs811_dt_ids[] = {
+	{ .compatible = "ams,ccs811" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ccs811_dt_ids);
+
 static struct i2c_driver ccs811_driver = {
 	.driver = {
 		.name = "ccs811",
+		.of_match_table = ccs811_dt_ids,
 	},
 	.probe = ccs811_probe,
 	.remove = ccs811_remove,
diff --git a/drivers/iio/chemical/pms7003.c b/drivers/iio/chemical/pms7003.c
index 23c9ab252470..07bb90d72434 100644
--- a/drivers/iio/chemical/pms7003.c
+++ b/drivers/iio/chemical/pms7003.c
@@ -73,6 +73,11 @@ struct pms7003_state {
 	struct pms7003_frame frame;
 	struct completion frame_ready;
 	struct mutex lock; /* must be held whenever state gets touched */
+	/* Used to construct scan to push to the IIO buffer */
+	struct {
+		u16 data[3]; /* PM1, PM2P5, PM10 */
+		s64 ts;
+	} scan;
 };
 
 static int pms7003_do_cmd(struct pms7003_state *state, enum pms7003_cmd cmd)
@@ -104,7 +109,6 @@ static irqreturn_t pms7003_trigger_handler(int irq, void *p)
 	struct iio_dev *indio_dev = pf->indio_dev;
 	struct pms7003_state *state = iio_priv(indio_dev);
 	struct pms7003_frame *frame = &state->frame;
-	u16 data[3 + 1 + 4]; /* PM1, PM2P5, PM10, padding, timestamp */
 	int ret;
 
 	mutex_lock(&state->lock);
@@ -114,12 +118,15 @@ static irqreturn_t pms7003_trigger_handler(int irq, void *p)
 		goto err;
 	}
 
-	data[PM1] = pms7003_get_pm(frame->data + PMS7003_PM1_OFFSET);
-	data[PM2P5] = pms7003_get_pm(frame->data + PMS7003_PM2P5_OFFSET);
-	data[PM10] = pms7003_get_pm(frame->data + PMS7003_PM10_OFFSET);
+	state->scan.data[PM1] =
+		pms7003_get_pm(frame->data + PMS7003_PM1_OFFSET);
+	state->scan.data[PM2P5] =
+		pms7003_get_pm(frame->data + PMS7003_PM2P5_OFFSET);
+	state->scan.data[PM10] =
+		pms7003_get_pm(frame->data + PMS7003_PM10_OFFSET);
 	mutex_unlock(&state->lock);
 
-	iio_push_to_buffers_with_timestamp(indio_dev, data,
+	iio_push_to_buffers_with_timestamp(indio_dev, &state->scan,
 					   iio_get_time_ns(indio_dev));
 err:
 	iio_trigger_notify_done(indio_dev->trig);
diff --git a/drivers/iio/chemical/sps30.c b/drivers/iio/chemical/sps30.c
index acb9f8ecbb3d..a88c1fb875a0 100644
--- a/drivers/iio/chemical/sps30.c
+++ b/drivers/iio/chemical/sps30.c
@@ -230,15 +230,18 @@ static irqreturn_t sps30_trigger_handler(int irq, void *p)
 	struct iio_dev *indio_dev = pf->indio_dev;
 	struct sps30_state *state = iio_priv(indio_dev);
 	int ret;
-	s32 data[4 + 2]; /* PM1, PM2P5, PM4, PM10, timestamp */
+	struct {
+		s32 data[4]; /* PM1, PM2P5, PM4, PM10 */
+		s64 ts;
+	} scan;
 
 	mutex_lock(&state->lock);
-	ret = sps30_do_meas(state, data, 4);
+	ret = sps30_do_meas(state, scan.data, ARRAY_SIZE(scan.data));
 	mutex_unlock(&state->lock);
 	if (ret)
 		goto err;
 
-	iio_push_to_buffers_with_timestamp(indio_dev, data,
+	iio_push_to_buffers_with_timestamp(indio_dev, &scan,
 					   iio_get_time_ns(indio_dev));
 err:
 	iio_trigger_notify_done(indio_dev->trig);
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
index 906d87780419..ff375790b7e8 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
@@ -13,6 +13,8 @@
 #include <linux/hid-sensor-hub.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/trigger.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
 #include <linux/iio/buffer.h>
 #include <linux/iio/sysfs.h>
 #include "hid-sensor-trigger.h"
@@ -222,7 +224,8 @@ static int hid_sensor_data_rdy_trigger_set_state(struct iio_trigger *trig,
 	return hid_sensor_power_state(iio_trigger_get_drvdata(trig), state);
 }
 
-void hid_sensor_remove_trigger(struct hid_sensor_common *attrb)
+void hid_sensor_remove_trigger(struct iio_dev *indio_dev,
+			       struct hid_sensor_common *attrb)
 {
 	if (atomic_read(&attrb->runtime_pm_enable))
 		pm_runtime_disable(&attrb->pdev->dev);
@@ -233,6 +236,7 @@ void hid_sensor_remove_trigger(struct hid_sensor_common *attrb)
 	cancel_work_sync(&attrb->work);
 	iio_trigger_unregister(attrb->trigger);
 	iio_trigger_free(attrb->trigger);
+	iio_triggered_buffer_cleanup(indio_dev);
 }
 EXPORT_SYMBOL(hid_sensor_remove_trigger);
 
@@ -246,11 +250,18 @@ int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 	int ret;
 	struct iio_trigger *trig;
 
+	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
+					 NULL, NULL);
+	if (ret) {
+		dev_err(&indio_dev->dev, "Triggered Buffer Setup Failed\n");
+		return ret;
+	}
+
 	trig = iio_trigger_alloc("%s-dev%d", name, indio_dev->id);
 	if (trig == NULL) {
 		dev_err(&indio_dev->dev, "Trigger Allocate Failed\n");
 		ret = -ENOMEM;
-		goto error_ret;
+		goto error_triggered_buffer_cleanup;
 	}
 
 	trig->dev.parent = indio_dev->dev.parent;
@@ -284,7 +295,8 @@ error_unreg_trigger:
 	iio_trigger_unregister(trig);
 error_free_trig:
 	iio_trigger_free(trig);
-error_ret:
+error_triggered_buffer_cleanup:
+	iio_triggered_buffer_cleanup(indio_dev);
 	return ret;
 }
 EXPORT_SYMBOL(hid_sensor_setup_trigger);
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.h b/drivers/iio/common/hid-sensors/hid-sensor-trigger.h
index f47b940ff170..bb45cc89e551 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.h
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.h
@@ -13,7 +13,8 @@ extern const struct dev_pm_ops hid_sensor_pm_ops;
 
 int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 				struct hid_sensor_common *attrb);
-void hid_sensor_remove_trigger(struct hid_sensor_common *attrb);
+void hid_sensor_remove_trigger(struct iio_dev *indio_dev,
+			       struct hid_sensor_common *attrb);
 int hid_sensor_power_state(struct hid_sensor_common *st, bool state);
 
 #endif
diff --git a/drivers/iio/common/st_sensors/st_sensors_core.c b/drivers/iio/common/st_sensors/st_sensors_core.c
index 13bdfbbf5f71..7a69c1be7393 100644
--- a/drivers/iio/common/st_sensors/st_sensors_core.c
+++ b/drivers/iio/common/st_sensors/st_sensors_core.c
@@ -20,11 +20,6 @@
 
 #include "st_sensors_core.h"
 
-static inline u32 st_sensors_get_unaligned_le24(const u8 *p)
-{
-	return (s32)((p[0] | p[1] << 8 | p[2] << 16) << 8) >> 8;
-}
-
 int st_sensors_write_data_with_mask(struct iio_dev *indio_dev,
 				    u8 reg_addr, u8 mask, u8 data)
 {
@@ -150,8 +145,7 @@ static int st_sensors_set_fullscale(struct iio_dev *indio_dev, unsigned int fs)
 	if (err < 0)
 		goto st_accel_set_fullscale_error;
 
-	sdata->current_fullscale = (struct st_sensor_fullscale_avl *)
-					&sdata->sensor_settings->fs.fs_avl[i];
+	sdata->current_fullscale = &sdata->sensor_settings->fs.fs_avl[i];
 	return err;
 
 st_accel_set_fullscale_error:
@@ -278,8 +272,7 @@ static int st_sensors_set_drdy_int_pin(struct iio_dev *indio_dev,
 	    !sdata->sensor_settings->drdy_irq.int2.addr) {
 		if (pdata->drdy_int_pin)
 			dev_info(&indio_dev->dev,
-				 "DRDY on pin INT%d specified, but sensor "
-				 "does not support interrupts\n",
+				 "DRDY on pin INT%d specified, but sensor does not support interrupts\n",
 				 pdata->drdy_int_pin);
 		return 0;
 	}
@@ -545,7 +538,7 @@ static int st_sensors_read_axis_data(struct iio_dev *indio_dev,
 	else if (byte_for_channel == 2)
 		*data = (s16)get_unaligned_le16(outdata);
 	else if (byte_for_channel == 3)
-		*data = (s32)st_sensors_get_unaligned_le24(outdata);
+		*data = (s32)sign_extend32(get_unaligned_le24(outdata), 23);
 
 st_sensors_free_memory:
 	kfree(outdata);
diff --git a/drivers/iio/common/st_sensors/st_sensors_i2c.c b/drivers/iio/common/st_sensors/st_sensors_i2c.c
index 286830fb5d35..b400560bac93 100644
--- a/drivers/iio/common/st_sensors/st_sensors_i2c.c
+++ b/drivers/iio/common/st_sensors/st_sensors_i2c.c
@@ -49,8 +49,8 @@ int st_sensors_i2c_configure(struct iio_dev *indio_dev,
 
 	sdata->regmap = devm_regmap_init_i2c(client, config);
 	if (IS_ERR(sdata->regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap (%d)\n",
-			(int)PTR_ERR(sdata->regmap));
+		dev_err(&client->dev, "Failed to register i2c regmap (%ld)\n",
+			PTR_ERR(sdata->regmap));
 		return PTR_ERR(sdata->regmap);
 	}
 
diff --git a/drivers/iio/common/st_sensors/st_sensors_spi.c b/drivers/iio/common/st_sensors/st_sensors_spi.c
index 1275fb0eda31..ee70515bb89f 100644
--- a/drivers/iio/common/st_sensors/st_sensors_spi.c
+++ b/drivers/iio/common/st_sensors/st_sensors_spi.c
@@ -44,7 +44,7 @@ static bool st_sensors_is_spi_3_wire(struct spi_device *spi)
 	if (device_property_read_bool(dev, "spi-3wire"))
 		return true;
 
-	pdata = (struct st_sensors_platform_data *)dev->platform_data;
+	pdata = dev_get_platdata(dev);
 	if (pdata && pdata->spi_3wire)
 		return true;
 
@@ -101,8 +101,8 @@ int st_sensors_spi_configure(struct iio_dev *indio_dev,
 
 	sdata->regmap = devm_regmap_init_spi(spi, config);
 	if (IS_ERR(sdata->regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap (%d)\n",
-			(int)PTR_ERR(sdata->regmap));
+		dev_err(&spi->dev, "Failed to register spi regmap (%ld)\n",
+			PTR_ERR(sdata->regmap));
 		return PTR_ERR(sdata->regmap);
 	}
 
diff --git a/drivers/iio/common/st_sensors/st_sensors_trigger.c b/drivers/iio/common/st_sensors/st_sensors_trigger.c
index e817537cdfb5..0507283bd4c1 100644
--- a/drivers/iio/common/st_sensors/st_sensors_trigger.c
+++ b/drivers/iio/common/st_sensors/st_sensors_trigger.c
@@ -44,8 +44,7 @@ static int st_sensors_new_samples_available(struct iio_dev *indio_dev,
 			  sdata->sensor_settings->drdy_irq.stat_drdy.addr,
 			  &status);
 	if (ret < 0) {
-		dev_err(sdata->dev,
-			"error checking samples available\n");
+		dev_err(sdata->dev, "error checking samples available\n");
 		return ret;
 	}
 
@@ -148,9 +147,7 @@ int st_sensors_allocate_trigger(struct iio_dev *indio_dev,
 	case IRQF_TRIGGER_LOW:
 		if (!sdata->sensor_settings->drdy_irq.addr_ihl) {
 			dev_err(&indio_dev->dev,
-				"falling/low specified for IRQ "
-				"but hardware supports only rising/high: "
-				"will request rising/high\n");
+				"falling/low specified for IRQ but hardware supports only rising/high: will request rising/high\n");
 			if (irq_trig == IRQF_TRIGGER_FALLING)
 				irq_trig = IRQF_TRIGGER_RISING;
 			if (irq_trig == IRQF_TRIGGER_LOW)
@@ -163,8 +160,7 @@ int st_sensors_allocate_trigger(struct iio_dev *indio_dev,
 			if (err < 0)
 				goto iio_trigger_free;
 			dev_info(&indio_dev->dev,
-				 "interrupts on the falling edge or "
-				 "active low level\n");
+				 "interrupts on the falling edge or active low level\n");
 		}
 		break;
 	case IRQF_TRIGGER_RISING:
@@ -178,8 +174,7 @@ int st_sensors_allocate_trigger(struct iio_dev *indio_dev,
 	default:
 		/* This is the most preferred mode, if possible */
 		dev_err(&indio_dev->dev,
-			"unsupported IRQ trigger specified (%lx), enforce "
-			"rising edge\n", irq_trig);
+			"unsupported IRQ trigger specified (%lx), enforce rising edge\n", irq_trig);
 		irq_trig = IRQF_TRIGGER_RISING;
 	}
 
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index 93744011b63f..dae8d27e772d 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -279,12 +279,12 @@ config LTC1660
 	  module will be called ltc1660.
 
 config LTC2632
-	tristate "Linear Technology LTC2632-12/10/8 and LTC2636-12/10/8 DAC spi driver"
+	tristate "Linear Technology LTC2632-12/10/8 and similar DAC spi driver"
 	depends on SPI
 	help
 	  Say yes here to build support for Linear Technology
-	  LTC2632-12, LTC2632-10, LTC2632-8, LTC2636-12, LTC2636-10 and
-	  LTC2636-8 converters (DAC).
+	  LTC2632, LTC2634 and LTC2636 DAC resolution 12/10/8 bit
+	  low 0-2.5V and high 0-4.096V range converters.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called ltc2632.
@@ -325,7 +325,7 @@ config MAX5821
 config MCP4725
 	tristate "MCP4725/6 DAC driver"
 	depends on I2C
-	---help---
+	help
 	  Say Y here if you want to build a driver for the Microchip
 	  MCP 4725/6 12-bit digital-to-analog converter (DAC) with I2C
 	  interface.
diff --git a/drivers/iio/dac/ad5360.c b/drivers/iio/dac/ad5360.c
index 2ac428b957e3..3e0c9e84e8da 100644
--- a/drivers/iio/dac/ad5360.c
+++ b/drivers/iio/dac/ad5360.c
@@ -67,6 +67,7 @@ struct ad5360_chip_info {
  * @chip_info:		chip model specific constants, available modes etc
  * @vref_reg:		vref supply regulators
  * @ctrl:		control register cache
+ * @lock		lock to protect the data buffer during SPI ops
  * @data:		spi transfer buffers
  */
 
@@ -75,6 +76,7 @@ struct ad5360_state {
 	const struct ad5360_chip_info	*chip_info;
 	struct regulator_bulk_data	vref_reg[3];
 	unsigned int			ctrl;
+	struct mutex			lock;
 
 	/*
 	 * DMA (thus cache coherency maintenance) requires the
@@ -205,10 +207,11 @@ static int ad5360_write(struct iio_dev *indio_dev, unsigned int cmd,
 	unsigned int addr, unsigned int val, unsigned int shift)
 {
 	int ret;
+	struct ad5360_state *st = iio_priv(indio_dev);
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	ret = ad5360_write_unlocked(indio_dev, cmd, addr, val, shift);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -229,7 +232,7 @@ static int ad5360_read(struct iio_dev *indio_dev, unsigned int type,
 		},
 	};
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	st->data[0].d32 = cpu_to_be32(AD5360_CMD(AD5360_CMD_SPECIAL_FUNCTION) |
 		AD5360_ADDR(AD5360_REG_SF_READBACK) |
@@ -240,7 +243,7 @@ static int ad5360_read(struct iio_dev *indio_dev, unsigned int type,
 	if (ret >= 0)
 		ret = be32_to_cpu(st->data[1].d32) & 0xffff;
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -261,7 +264,7 @@ static int ad5360_update_ctrl(struct iio_dev *indio_dev, unsigned int set,
 	struct ad5360_state *st = iio_priv(indio_dev);
 	unsigned int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	st->ctrl |= set;
 	st->ctrl &= ~clr;
@@ -269,7 +272,7 @@ static int ad5360_update_ctrl(struct iio_dev *indio_dev, unsigned int set,
 	ret = ad5360_write_unlocked(indio_dev, AD5360_CMD_SPECIAL_FUNCTION,
 			AD5360_REG_SF_CTRL, st->ctrl, 0);
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -479,6 +482,8 @@ static int ad5360_probe(struct spi_device *spi)
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->num_channels = st->chip_info->num_channels;
 
+	mutex_init(&st->lock);
+
 	ret = ad5360_alloc_channels(indio_dev);
 	if (ret) {
 		dev_err(&spi->dev, "Failed to allocate channel spec: %d\n", ret);
diff --git a/drivers/iio/dac/ad5380.c b/drivers/iio/dac/ad5380.c
index 2ebe08326048..b37e5675f716 100644
--- a/drivers/iio/dac/ad5380.c
+++ b/drivers/iio/dac/ad5380.c
@@ -51,6 +51,7 @@ struct ad5380_chip_info {
  * @vref_reg:		vref supply regulator
  * @vref:		actual reference voltage used in uA
  * @pwr_down:		whether the chip is currently in power down mode
+ * @lock		lock to protect the data buffer during regmap ops
  */
 
 struct ad5380_state {
@@ -59,6 +60,7 @@ struct ad5380_state {
 	struct regulator		*vref_reg;
 	int				vref;
 	bool				pwr_down;
+	struct mutex			lock;
 };
 
 enum ad5380_type {
@@ -98,7 +100,7 @@ static ssize_t ad5380_write_dac_powerdown(struct iio_dev *indio_dev,
 	if (ret)
 		return ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	if (pwr_down)
 		ret = regmap_write(st->regmap, AD5380_REG_SF_PWR_DOWN, 0);
@@ -107,7 +109,7 @@ static ssize_t ad5380_write_dac_powerdown(struct iio_dev *indio_dev,
 
 	st->pwr_down = pwr_down;
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret ? ret : len;
 }
@@ -390,6 +392,8 @@ static int ad5380_probe(struct device *dev, struct regmap *regmap,
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->num_channels = st->chip_info->num_channels;
 
+	mutex_init(&st->lock);
+
 	ret = ad5380_alloc_channels(indio_dev);
 	if (ret) {
 		dev_err(dev, "Failed to allocate channel spec: %d\n", ret);
diff --git a/drivers/iio/dac/ad5421.c b/drivers/iio/dac/ad5421.c
index 63063e85cd0a..fec27764cea8 100644
--- a/drivers/iio/dac/ad5421.c
+++ b/drivers/iio/dac/ad5421.c
@@ -62,12 +62,14 @@
  * @current_range:	current range which the device is configured for
  * @data:		spi transfer buffers
  * @fault_mask:		software masking of events
+ * @lock	lock to protect the data buffer during SPI ops
  */
 struct ad5421_state {
 	struct spi_device		*spi;
 	unsigned int			ctrl;
 	enum ad5421_current_range	current_range;
 	unsigned int			fault_mask;
+	struct mutex			lock;
 
 	/*
 	 * DMA (thus cache coherency maintenance) requires the
@@ -142,11 +144,12 @@ static int ad5421_write_unlocked(struct iio_dev *indio_dev,
 static int ad5421_write(struct iio_dev *indio_dev, unsigned int reg,
 	unsigned int val)
 {
+	struct ad5421_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	ret = ad5421_write_unlocked(indio_dev, reg, val);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -166,7 +169,7 @@ static int ad5421_read(struct iio_dev *indio_dev, unsigned int reg)
 		},
 	};
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	st->data[0].d32 = cpu_to_be32((1 << 23) | (reg << 16));
 
@@ -174,7 +177,7 @@ static int ad5421_read(struct iio_dev *indio_dev, unsigned int reg)
 	if (ret >= 0)
 		ret = be32_to_cpu(st->data[1].d32) & 0xffff;
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -185,14 +188,14 @@ static int ad5421_update_ctrl(struct iio_dev *indio_dev, unsigned int set,
 	struct ad5421_state *st = iio_priv(indio_dev);
 	unsigned int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	st->ctrl &= ~clr;
 	st->ctrl |= set;
 
 	ret = ad5421_write_unlocked(indio_dev, AD5421_REG_CTRL, st->ctrl);
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -400,12 +403,12 @@ static int ad5421_write_event_config(struct iio_dev *indio_dev,
 		return -EINVAL;
 	}
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	if (state)
 		st->fault_mask |= mask;
 	else
 		st->fault_mask &= ~mask;
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return 0;
 }
@@ -491,6 +494,8 @@ static int ad5421_probe(struct spi_device *spi)
 	indio_dev->channels = ad5421_channels;
 	indio_dev->num_channels = ARRAY_SIZE(ad5421_channels);
 
+	mutex_init(&st->lock);
+
 	st->ctrl = AD5421_CTRL_WATCHDOG_DISABLE |
 			AD5421_CTRL_AUTO_FAULT_READBACK;
 
diff --git a/drivers/iio/dac/ad5446.c b/drivers/iio/dac/ad5446.c
index 61c670f7fc5f..8f8afc8999bc 100644
--- a/drivers/iio/dac/ad5446.c
+++ b/drivers/iio/dac/ad5446.c
@@ -21,6 +21,8 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
+#include <asm/unaligned.h>
+
 #define MODE_PWRDWN_1k		0x1
 #define MODE_PWRDWN_100k	0x2
 #define MODE_PWRDWN_TRISTATE	0x3
@@ -31,6 +33,7 @@
  * @chip_info:		chip model specific constants, available modes etc
  * @reg:		supply regulator
  * @vref_mv:		actual reference voltage used
+ * @lock		lock to protect the data buffer during write ops
  */
 
 struct ad5446_state {
@@ -41,6 +44,7 @@ struct ad5446_state {
 	unsigned			cached_val;
 	unsigned			pwr_down_mode;
 	unsigned			pwr_down;
+	struct mutex			lock;
 };
 
 /**
@@ -110,7 +114,7 @@ static ssize_t ad5446_write_dac_powerdown(struct iio_dev *indio_dev,
 	if (ret)
 		return ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	st->pwr_down = powerdown;
 
 	if (st->pwr_down) {
@@ -121,7 +125,7 @@ static ssize_t ad5446_write_dac_powerdown(struct iio_dev *indio_dev,
 	}
 
 	ret = st->chip_info->write(st, val);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret ? ret : len;
 }
@@ -195,11 +199,11 @@ static int ad5446_write_raw(struct iio_dev *indio_dev,
 			return -EINVAL;
 
 		val <<= chan->scan_type.shift;
-		mutex_lock(&indio_dev->mlock);
+		mutex_lock(&st->lock);
 		st->cached_val = val;
 		if (!st->pwr_down)
 			ret = st->chip_info->write(st, val);
-		mutex_unlock(&indio_dev->mlock);
+		mutex_unlock(&st->lock);
 		break;
 	default:
 		ret = -EINVAL;
@@ -254,6 +258,8 @@ static int ad5446_probe(struct device *dev, const char *name,
 	indio_dev->channels = &st->chip_info->channel;
 	indio_dev->num_channels = 1;
 
+	mutex_init(&st->lock);
+
 	st->pwr_down_mode = MODE_PWRDWN_1k;
 
 	if (st->chip_info->int_vref_mv)
@@ -302,9 +308,7 @@ static int ad5660_write(struct ad5446_state *st, unsigned val)
 	struct spi_device *spi = to_spi_device(st->dev);
 	uint8_t data[3];
 
-	data[0] = (val >> 16) & 0xFF;
-	data[1] = (val >> 8) & 0xFF;
-	data[2] = val & 0xFF;
+	put_unaligned_be24(val, &data[0]);
 
 	return spi_write(spi, data, sizeof(data));
 }
diff --git a/drivers/iio/dac/ad5449.c b/drivers/iio/dac/ad5449.c
index fed3ebaccac4..d739b10e5236 100644
--- a/drivers/iio/dac/ad5449.c
+++ b/drivers/iio/dac/ad5449.c
@@ -56,11 +56,13 @@ struct ad5449_chip_info {
  * @has_sdo:		whether the SDO line is connected
  * @dac_cache:		Cache for the DAC values
  * @data:		spi transfer buffers
+ * @lock		lock to protect the data buffer during SPI ops
  */
 struct ad5449 {
 	struct spi_device		*spi;
 	const struct ad5449_chip_info	*chip_info;
 	struct regulator_bulk_data	vref_reg[AD5449_MAX_VREFS];
+	struct mutex			lock;
 
 	bool has_sdo;
 	uint16_t dac_cache[AD5449_MAX_CHANNELS];
@@ -87,10 +89,10 @@ static int ad5449_write(struct iio_dev *indio_dev, unsigned int addr,
 	struct ad5449 *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	st->data[0] = cpu_to_be16((addr << 12) | val);
 	ret = spi_write(st->spi, st->data, 2);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -112,7 +114,7 @@ static int ad5449_read(struct iio_dev *indio_dev, unsigned int addr,
 		},
 	};
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	st->data[0] = cpu_to_be16(addr << 12);
 	st->data[1] = cpu_to_be16(AD5449_CMD_NOOP);
 
@@ -123,7 +125,7 @@ static int ad5449_read(struct iio_dev *indio_dev, unsigned int addr,
 	*val = be16_to_cpu(st->data[1]);
 
 out_unlock:
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 	return ret;
 }
 
@@ -302,6 +304,8 @@ static int ad5449_spi_probe(struct spi_device *spi)
 	indio_dev->channels = st->chip_info->channels;
 	indio_dev->num_channels = st->chip_info->num_channels;
 
+	mutex_init(&st->lock);
+
 	if (st->chip_info->has_ctrl) {
 		unsigned int ctrl = 0x00;
 		if (pdata) {
diff --git a/drivers/iio/dac/ad5592r-base.c b/drivers/iio/dac/ad5592r-base.c
index e2110113e884..410e90e5f75f 100644
--- a/drivers/iio/dac/ad5592r-base.c
+++ b/drivers/iio/dac/ad5592r-base.c
@@ -156,7 +156,6 @@ static void ad5592r_gpio_cleanup(struct ad5592r_state *st)
 static int ad5592r_reset(struct ad5592r_state *st)
 {
 	struct gpio_desc *gpio;
-	struct iio_dev *iio_dev = iio_priv_to_dev(st);
 
 	gpio = devm_gpiod_get_optional(st->dev, "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(gpio))
@@ -166,10 +165,10 @@ static int ad5592r_reset(struct ad5592r_state *st)
 		udelay(1);
 		gpiod_set_value(gpio, 1);
 	} else {
-		mutex_lock(&iio_dev->mlock);
+		mutex_lock(&st->lock);
 		/* Writing this magic value resets the device */
 		st->ops->reg_write(st, AD5592R_REG_RESET, 0xdac);
-		mutex_unlock(&iio_dev->mlock);
+		mutex_unlock(&st->lock);
 	}
 
 	udelay(250);
@@ -197,7 +196,6 @@ static int ad5592r_set_channel_modes(struct ad5592r_state *st)
 	const struct ad5592r_rw_ops *ops = st->ops;
 	int ret;
 	unsigned i;
-	struct iio_dev *iio_dev = iio_priv_to_dev(st);
 	u8 pulldown = 0, tristate = 0, dac = 0, adc = 0;
 	u16 read_back;
 
@@ -247,7 +245,7 @@ static int ad5592r_set_channel_modes(struct ad5592r_state *st)
 		}
 	}
 
-	mutex_lock(&iio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	/* Pull down unused pins to GND */
 	ret = ops->reg_write(st, AD5592R_REG_PULLDOWN, pulldown);
@@ -285,7 +283,7 @@ static int ad5592r_set_channel_modes(struct ad5592r_state *st)
 		ret = -EIO;
 
 err_unlock:
-	mutex_unlock(&iio_dev->mlock);
+	mutex_unlock(&st->lock);
 	return ret;
 }
 
@@ -314,11 +312,11 @@ static int ad5592r_write_raw(struct iio_dev *iio_dev,
 		if (!chan->output)
 			return -EINVAL;
 
-		mutex_lock(&iio_dev->mlock);
+		mutex_lock(&st->lock);
 		ret = st->ops->write_dac(st, chan->channel, val);
 		if (!ret)
 			st->cached_dac[chan->channel] = val;
-		mutex_unlock(&iio_dev->mlock);
+		mutex_unlock(&st->lock);
 		return ret;
 	case IIO_CHAN_INFO_SCALE:
 		if (chan->type == IIO_VOLTAGE) {
@@ -333,12 +331,12 @@ static int ad5592r_write_raw(struct iio_dev *iio_dev,
 			else
 				return -EINVAL;
 
-			mutex_lock(&iio_dev->mlock);
+			mutex_lock(&st->lock);
 
 			ret = st->ops->reg_read(st, AD5592R_REG_CTRL,
 						&st->cached_gp_ctrl);
 			if (ret < 0) {
-				mutex_unlock(&iio_dev->mlock);
+				mutex_unlock(&st->lock);
 				return ret;
 			}
 
@@ -360,7 +358,7 @@ static int ad5592r_write_raw(struct iio_dev *iio_dev,
 
 			ret = st->ops->reg_write(st, AD5592R_REG_CTRL,
 						 st->cached_gp_ctrl);
-			mutex_unlock(&iio_dev->mlock);
+			mutex_unlock(&st->lock);
 
 			return ret;
 		}
@@ -382,7 +380,7 @@ static int ad5592r_read_raw(struct iio_dev *iio_dev,
 
 	switch (m) {
 	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&iio_dev->mlock);
+		mutex_lock(&st->lock);
 
 		if (!chan->output) {
 			ret = st->ops->read_adc(st, chan->channel, &read_val);
@@ -419,7 +417,7 @@ static int ad5592r_read_raw(struct iio_dev *iio_dev,
 		} else {
 			int mult;
 
-			mutex_lock(&iio_dev->mlock);
+			mutex_lock(&st->lock);
 
 			if (chan->output)
 				mult = !!(st->cached_gp_ctrl &
@@ -437,7 +435,7 @@ static int ad5592r_read_raw(struct iio_dev *iio_dev,
 	case IIO_CHAN_INFO_OFFSET:
 		ret = ad5592r_get_vref(st);
 
-		mutex_lock(&iio_dev->mlock);
+		mutex_lock(&st->lock);
 
 		if (st->cached_gp_ctrl & AD5592R_REG_CTRL_ADC_RANGE)
 			*val = (-34365 * 25) / ret;
@@ -450,7 +448,7 @@ static int ad5592r_read_raw(struct iio_dev *iio_dev,
 	}
 
 unlock:
-	mutex_unlock(&iio_dev->mlock);
+	mutex_unlock(&st->lock);
 	return ret;
 }
 
@@ -625,6 +623,8 @@ int ad5592r_probe(struct device *dev, const char *name,
 	iio_dev->info = &ad5592r_info;
 	iio_dev->modes = INDIO_DIRECT_MODE;
 
+	mutex_init(&st->lock);
+
 	ad5592r_init_scales(st, ad5592r_get_vref(st));
 
 	ret = ad5592r_reset(st);
diff --git a/drivers/iio/dac/ad5592r-base.h b/drivers/iio/dac/ad5592r-base.h
index 4774e4cd9c11..23dac2f1ff8a 100644
--- a/drivers/iio/dac/ad5592r-base.h
+++ b/drivers/iio/dac/ad5592r-base.h
@@ -52,6 +52,7 @@ struct ad5592r_state {
 	struct regulator *reg;
 	struct gpio_chip gpiochip;
 	struct mutex gpio_lock;	/* Protect cached gpio_out, gpio_val, etc. */
+	struct mutex lock;
 	unsigned int num_channels;
 	const struct ad5592r_rw_ops *ops;
 	int scale_avail[2][2];
diff --git a/drivers/iio/dac/ad5592r.c b/drivers/iio/dac/ad5592r.c
index 34ba059a77da..49308ad13c4b 100644
--- a/drivers/iio/dac/ad5592r.c
+++ b/drivers/iio/dac/ad5592r.c
@@ -98,7 +98,7 @@ static int ad5592r_reg_read(struct ad5592r_state *st, u8 reg, u16 *value)
 	return 0;
 }
 
-static int ad5593r_gpio_read(struct ad5592r_state *st, u8 *value)
+static int ad5592r_gpio_read(struct ad5592r_state *st, u8 *value)
 {
 	int ret;
 
@@ -121,7 +121,7 @@ static const struct ad5592r_rw_ops ad5592r_rw_ops = {
 	.read_adc = ad5592r_read_adc,
 	.reg_write = ad5592r_reg_write,
 	.reg_read = ad5592r_reg_read,
-	.gpio_read = ad5593r_gpio_read,
+	.gpio_read = ad5592r_gpio_read,
 };
 
 static int ad5592r_spi_probe(struct spi_device *spi)
diff --git a/drivers/iio/dac/ad5593r.c b/drivers/iio/dac/ad5593r.c
index 44ea3b8117d0..1fbe9c019c7f 100644
--- a/drivers/iio/dac/ad5593r.c
+++ b/drivers/iio/dac/ad5593r.c
@@ -134,5 +134,5 @@ static struct i2c_driver ad5593r_driver = {
 module_i2c_driver(ad5593r_driver);
 
 MODULE_AUTHOR("Paul Cercueil <paul.cercueil@analog.com>");
-MODULE_DESCRIPTION("Analog Devices AD5592R multi-channel converters");
+MODULE_DESCRIPTION("Analog Devices AD5593R multi-channel converters");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/dac/ad5624r_spi.c b/drivers/iio/dac/ad5624r_spi.c
index e6c022e1dc1c..2015a5df840c 100644
--- a/drivers/iio/dac/ad5624r_spi.c
+++ b/drivers/iio/dac/ad5624r_spi.c
@@ -18,6 +18,8 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
+#include <asm/unaligned.h>
+
 #include "ad5624r.h"
 
 static int ad5624r_spi_write(struct spi_device *spi,
@@ -35,11 +37,9 @@ static int ad5624r_spi_write(struct spi_device *spi,
 	 * for the AD5664R, AD5644R, and AD5624R, respectively.
 	 */
 	data = (0 << 22) | (cmd << 19) | (addr << 16) | (val << shift);
-	msg[0] = data >> 16;
-	msg[1] = data >> 8;
-	msg[2] = data;
+	put_unaligned_be24(data, &msg[0]);
 
-	return spi_write(spi, msg, 3);
+	return spi_write(spi, msg, sizeof(msg));
 }
 
 static int ad5624r_read_raw(struct iio_dev *indio_dev,
diff --git a/drivers/iio/dac/ad5686.c b/drivers/iio/dac/ad5686.c
index e06b29c565b9..8dd67da0a7da 100644
--- a/drivers/iio/dac/ad5686.c
+++ b/drivers/iio/dac/ad5686.c
@@ -127,9 +127,9 @@ static int ad5686_read_raw(struct iio_dev *indio_dev,
 
 	switch (m) {
 	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&indio_dev->mlock);
+		mutex_lock(&st->lock);
 		ret = st->read(st, chan->address);
-		mutex_unlock(&indio_dev->mlock);
+		mutex_unlock(&st->lock);
 		if (ret < 0)
 			return ret;
 		*val = (ret >> chan->scan_type.shift) &
@@ -157,12 +157,12 @@ static int ad5686_write_raw(struct iio_dev *indio_dev,
 		if (val > (1 << chan->scan_type.realbits) || val < 0)
 			return -EINVAL;
 
-		mutex_lock(&indio_dev->mlock);
+		mutex_lock(&st->lock);
 		ret = st->write(st,
 				AD5686_CMD_WRITE_INPUT_N_UPDATE_N,
 				chan->address,
 				val << chan->scan_type.shift);
-		mutex_unlock(&indio_dev->mlock);
+		mutex_unlock(&st->lock);
 		break;
 	default:
 		ret = -EINVAL;
@@ -468,6 +468,8 @@ int ad5686_probe(struct device *dev,
 	indio_dev->channels = st->chip_info->channels;
 	indio_dev->num_channels = st->chip_info->num_channels;
 
+	mutex_init(&st->lock);
+
 	switch (st->chip_info->regmap_type) {
 	case AD5310_REGMAP:
 		cmd = AD5686_CMD_CONTROL_REG;
diff --git a/drivers/iio/dac/ad5686.h b/drivers/iio/dac/ad5686.h
index 70a779939ddb..52009b5eef88 100644
--- a/drivers/iio/dac/ad5686.h
+++ b/drivers/iio/dac/ad5686.h
@@ -117,6 +117,7 @@ struct ad5686_chip_info {
  * @pwr_down_mask:	power down mask
  * @pwr_down_mode:	current power down mode
  * @use_internal_vref:	set to true if the internal reference voltage is used
+ * @lock		lock to protect the data buffer during regmap ops
  * @data:		spi transfer buffers
  */
 
@@ -130,6 +131,7 @@ struct ad5686_state {
 	ad5686_write_func		write;
 	ad5686_read_func		read;
 	bool				use_internal_vref;
+	struct mutex			lock;
 
 	/*
 	 * DMA (thus cache coherency maintenance) requires the
diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c
index 388ddd14bfd0..7723bd313fc6 100644
--- a/drivers/iio/dac/ad5755.c
+++ b/drivers/iio/dac/ad5755.c
@@ -82,6 +82,7 @@ struct ad5755_chip_info {
  * @pwr_down:	bitmask which contains  hether a channel is powered down or not
  * @ctrl:	software shadow of the channel ctrl registers
  * @channels:	iio channel spec for the device
+ * @lock	lock to protect the data buffer during SPI ops
  * @data:	spi transfer buffers
  */
 struct ad5755_state {
@@ -90,6 +91,7 @@ struct ad5755_state {
 	unsigned int			pwr_down;
 	unsigned int			ctrl[AD5755_NUM_CHANNELS];
 	struct iio_chan_spec		channels[AD5755_NUM_CHANNELS];
+	struct mutex			lock;
 
 	/*
 	 * DMA (thus cache coherency maintenance) requires the
@@ -174,11 +176,12 @@ static int ad5755_write_ctrl_unlocked(struct iio_dev *indio_dev,
 static int ad5755_write(struct iio_dev *indio_dev, unsigned int reg,
 	unsigned int val)
 {
+	struct ad5755_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	ret = ad5755_write_unlocked(indio_dev, reg, val);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -186,11 +189,12 @@ static int ad5755_write(struct iio_dev *indio_dev, unsigned int reg,
 static int ad5755_write_ctrl(struct iio_dev *indio_dev, unsigned int channel,
 	unsigned int reg, unsigned int val)
 {
+	struct ad5755_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	ret = ad5755_write_ctrl_unlocked(indio_dev, channel, reg, val);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -211,7 +215,7 @@ static int ad5755_read(struct iio_dev *indio_dev, unsigned int addr)
 		},
 	};
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	st->data[0].d32 = cpu_to_be32(AD5755_READ_FLAG | (addr << 16));
 	st->data[1].d32 = cpu_to_be32(AD5755_NOOP);
@@ -220,7 +224,7 @@ static int ad5755_read(struct iio_dev *indio_dev, unsigned int addr)
 	if (ret >= 0)
 		ret = be32_to_cpu(st->data[1].d32) & 0xffff;
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -246,7 +250,7 @@ static int ad5755_set_channel_pwr_down(struct iio_dev *indio_dev,
 	struct ad5755_state *st = iio_priv(indio_dev);
 	unsigned int mask = BIT(channel);
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	if ((bool)(st->pwr_down & mask) == pwr_down)
 		goto out_unlock;
@@ -266,7 +270,7 @@ static int ad5755_set_channel_pwr_down(struct iio_dev *indio_dev,
 	}
 
 out_unlock:
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return 0;
 }
@@ -746,6 +750,8 @@ static int ad5755_probe(struct spi_device *spi)
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->num_channels = AD5755_NUM_CHANNELS;
 
+	mutex_init(&st->lock);
+
 	if (spi->dev.of_node)
 		pdata = ad5755_parse_dt(&spi->dev);
 	else
diff --git a/drivers/iio/dac/ad5761.c b/drivers/iio/dac/ad5761.c
index 7468fbd11684..67c4fa75c6f1 100644
--- a/drivers/iio/dac/ad5761.c
+++ b/drivers/iio/dac/ad5761.c
@@ -57,11 +57,13 @@ enum ad5761_supported_device_ids {
  * @use_intref:		true when the internal voltage reference is used
  * @vref:		actual voltage reference in mVolts
  * @range:		output range mode used
+ * @lock		lock to protect the data buffer during SPI ops
  * @data:		cache aligned spi buffer
  */
 struct ad5761_state {
 	struct spi_device		*spi;
 	struct regulator		*vref_reg;
+	struct mutex			lock;
 
 	bool use_intref;
 	int vref;
@@ -124,9 +126,9 @@ static int ad5761_spi_write(struct iio_dev *indio_dev, u8 addr, u16 val)
 	struct ad5761_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	ret = _ad5761_spi_write(st, addr, val);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -163,9 +165,9 @@ static int ad5761_spi_read(struct iio_dev *indio_dev, u8 addr, u16 *val)
 	struct ad5761_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	ret = _ad5761_spi_read(st, addr, val);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -368,6 +370,8 @@ static int ad5761_probe(struct spi_device *spi)
 	if (pdata)
 		voltage_range = pdata->voltage_range;
 
+	mutex_init(&st->lock);
+
 	ret = ad5761_spi_set_range(st, voltage_range);
 	if (ret)
 		goto disable_regulator_err;
diff --git a/drivers/iio/dac/ad5764.c b/drivers/iio/dac/ad5764.c
index f7ab211604a1..5b0f0fe354f6 100644
--- a/drivers/iio/dac/ad5764.c
+++ b/drivers/iio/dac/ad5764.c
@@ -46,6 +46,7 @@ struct ad5764_chip_info {
  * @spi:		spi_device
  * @chip_info:		chip info
  * @vref_reg:		vref supply regulators
+ * @lock		lock to protect the data buffer during SPI ops
  * @data:		spi transfer buffers
  */
 
@@ -53,6 +54,7 @@ struct ad5764_state {
 	struct spi_device		*spi;
 	const struct ad5764_chip_info	*chip_info;
 	struct regulator_bulk_data	vref_reg[2];
+	struct mutex			lock;
 
 	/*
 	 * DMA (thus cache coherency maintenance) requires the
@@ -126,11 +128,11 @@ static int ad5764_write(struct iio_dev *indio_dev, unsigned int reg,
 	struct ad5764_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 	st->data[0].d32 = cpu_to_be32((reg << 16) | val);
 
 	ret = spi_write(st->spi, &st->data[0].d8[1], 3);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -151,7 +153,7 @@ static int ad5764_read(struct iio_dev *indio_dev, unsigned int reg,
 		},
 	};
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&st->lock);
 
 	st->data[0].d32 = cpu_to_be32((1 << 23) | (reg << 16));
 
@@ -159,7 +161,7 @@ static int ad5764_read(struct iio_dev *indio_dev, unsigned int reg,
 	if (ret >= 0)
 		*val = be32_to_cpu(st->data[1].d32) & 0xffff;
 
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&st->lock);
 
 	return ret;
 }
@@ -295,6 +297,8 @@ static int ad5764_probe(struct spi_device *spi)
 	indio_dev->num_channels = AD5764_NUM_CHANNELS;
 	indio_dev->channels = st->chip_info->channels;
 
+	mutex_init(&st->lock);
+
 	if (st->chip_info->int_vref == 0) {
 		st->vref_reg[0].supply = "vrefAB";
 		st->vref_reg[1].supply = "vrefCD";
diff --git a/drivers/iio/dac/ltc2632.c b/drivers/iio/dac/ltc2632.c
index 7adc91056aa1..f891311f05cf 100644
--- a/drivers/iio/dac/ltc2632.c
+++ b/drivers/iio/dac/ltc2632.c
@@ -12,6 +12,8 @@
 #include <linux/iio/iio.h>
 #include <linux/regulator/consumer.h>
 
+#include <asm/unaligned.h>
+
 #define LTC2632_CMD_WRITE_INPUT_N               0x0
 #define LTC2632_CMD_UPDATE_DAC_N                0x1
 #define LTC2632_CMD_WRITE_INPUT_N_UPDATE_ALL    0x2
@@ -24,6 +26,7 @@
 /**
  * struct ltc2632_chip_info - chip specific information
  * @channels:		channel spec for the DAC
+ * @num_channels:	DAC channel count of the chip
  * @vref_mv:		internal reference voltage
  */
 struct ltc2632_chip_info {
@@ -53,6 +56,12 @@ enum ltc2632_supported_device_ids {
 	ID_LTC2632H12,
 	ID_LTC2632H10,
 	ID_LTC2632H8,
+	ID_LTC2634L12,
+	ID_LTC2634L10,
+	ID_LTC2634L8,
+	ID_LTC2634H12,
+	ID_LTC2634H10,
+	ID_LTC2634H8,
 	ID_LTC2636L12,
 	ID_LTC2636L10,
 	ID_LTC2636L8,
@@ -75,9 +84,7 @@ static int ltc2632_spi_write(struct spi_device *spi,
 	 * 10-, 8-bit input code followed by 4, 6, or 8 don't care bits.
 	 */
 	data = (cmd << 20) | (addr << 16) | (val << shift);
-	msg[0] = data >> 16;
-	msg[1] = data >> 8;
-	msg[2] = data;
+	put_unaligned_be24(data, &msg[0]);
 
 	return spi_write(spi, msg, sizeof(msg));
 }
@@ -235,6 +242,36 @@ static const struct ltc2632_chip_info ltc2632_chip_info_tbl[] = {
 		.num_channels	= 2,
 		.vref_mv	= 4096,
 	},
+	[ID_LTC2634L12] = {
+		.channels	= ltc2632x12_channels,
+		.num_channels	= 4,
+		.vref_mv	= 2500,
+	},
+	[ID_LTC2634L10] = {
+		.channels	= ltc2632x10_channels,
+		.num_channels	= 4,
+		.vref_mv	= 2500,
+	},
+	[ID_LTC2634L8] =  {
+		.channels	= ltc2632x8_channels,
+		.num_channels	= 4,
+		.vref_mv	= 2500,
+	},
+	[ID_LTC2634H12] = {
+		.channels	= ltc2632x12_channels,
+		.num_channels	= 4,
+		.vref_mv	= 4096,
+	},
+	[ID_LTC2634H10] = {
+		.channels	= ltc2632x10_channels,
+		.num_channels	= 4,
+		.vref_mv	= 4096,
+	},
+	[ID_LTC2634H8] =  {
+		.channels	= ltc2632x8_channels,
+		.num_channels	= 4,
+		.vref_mv	= 4096,
+	},
 	[ID_LTC2636L12] = {
 		.channels	= ltc2632x12_channels,
 		.num_channels	= 8,
@@ -356,6 +393,12 @@ static const struct spi_device_id ltc2632_id[] = {
 	{ "ltc2632-h12", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2632H12] },
 	{ "ltc2632-h10", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2632H10] },
 	{ "ltc2632-h8", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2632H8] },
+	{ "ltc2634-l12", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2634L12] },
+	{ "ltc2634-l10", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2634L10] },
+	{ "ltc2634-l8", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2634L8] },
+	{ "ltc2634-h12", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2634H12] },
+	{ "ltc2634-h10", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2634H10] },
+	{ "ltc2634-h8", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2634H8] },
 	{ "ltc2636-l12", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2636L12] },
 	{ "ltc2636-l10", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2636L10] },
 	{ "ltc2636-l8", (kernel_ulong_t)&ltc2632_chip_info_tbl[ID_LTC2636L8] },
@@ -386,6 +429,24 @@ static const struct of_device_id ltc2632_of_match[] = {
 		.compatible = "lltc,ltc2632-h8",
 		.data = &ltc2632_chip_info_tbl[ID_LTC2632H8]
 	}, {
+		.compatible = "lltc,ltc2634-l12",
+		.data = &ltc2632_chip_info_tbl[ID_LTC2634L12]
+	}, {
+		.compatible = "lltc,ltc2634-l10",
+		.data = &ltc2632_chip_info_tbl[ID_LTC2634L10]
+	}, {
+		.compatible = "lltc,ltc2634-l8",
+		.data = &ltc2632_chip_info_tbl[ID_LTC2634L8]
+	}, {
+		.compatible = "lltc,ltc2634-h12",
+		.data = &ltc2632_chip_info_tbl[ID_LTC2634H12]
+	}, {
+		.compatible = "lltc,ltc2634-h10",
+		.data = &ltc2632_chip_info_tbl[ID_LTC2634H10]
+	}, {
+		.compatible = "lltc,ltc2634-h8",
+		.data = &ltc2632_chip_info_tbl[ID_LTC2634H8]
+	}, {
 		.compatible = "lltc,ltc2636-l12",
 		.data = &ltc2632_chip_info_tbl[ID_LTC2636L12]
 	}, {
diff --git a/drivers/iio/dac/vf610_dac.c b/drivers/iio/dac/vf610_dac.c
index 7f1e9317c3f3..9417a4a3e22a 100644
--- a/drivers/iio/dac/vf610_dac.c
+++ b/drivers/iio/dac/vf610_dac.c
@@ -36,6 +36,7 @@ struct vf610_dac {
 	struct device *dev;
 	enum vf610_conversion_mode_sel conv_mode;
 	void __iomem *regs;
+	struct mutex lock;
 };
 
 static void vf610_dac_init(struct vf610_dac *info)
@@ -64,7 +65,7 @@ static int vf610_set_conversion_mode(struct iio_dev *indio_dev,
 	struct vf610_dac *info = iio_priv(indio_dev);
 	int val;
 
-	mutex_lock(&indio_dev->mlock);
+	mutex_lock(&info->lock);
 	info->conv_mode = mode;
 	val = readl(info->regs + VF610_DACx_STATCTRL);
 	if (mode)
@@ -72,7 +73,7 @@ static int vf610_set_conversion_mode(struct iio_dev *indio_dev,
 	else
 		val &= ~VF610_DAC_LPEN;
 	writel(val, info->regs + VF610_DACx_STATCTRL);
-	mutex_unlock(&indio_dev->mlock);
+	mutex_unlock(&info->lock);
 
 	return 0;
 }
@@ -147,9 +148,9 @@ static int vf610_write_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&indio_dev->mlock);
+		mutex_lock(&info->lock);
 		writel(VF610_DAC_DAT0(val), info->regs);
-		mutex_unlock(&indio_dev->mlock);
+		mutex_unlock(&info->lock);
 		return 0;
 
 	default:
@@ -205,6 +206,8 @@ static int vf610_dac_probe(struct platform_device *pdev)
 	indio_dev->channels = vf610_dac_iio_channels;
 	indio_dev->num_channels = ARRAY_SIZE(vf610_dac_iio_channels);
 
+	mutex_init(&info->lock);
+
 	ret = clk_prepare_enable(info->clk);
 	if (ret) {
 		dev_err(&pdev->dev,
diff --git a/drivers/iio/gyro/Kconfig b/drivers/iio/gyro/Kconfig
index 7eaf77707b0b..6daeddf37f60 100644
--- a/drivers/iio/gyro/Kconfig
+++ b/drivers/iio/gyro/Kconfig
@@ -61,7 +61,7 @@ config BMG160
 	help
 	  Say yes here to build support for BOSCH BMG160 Tri-axis Gyro Sensor
 	  driver connected via I2C or SPI. This driver also supports BMI055
-	  gyroscope.
+	  and BMI088 gyroscope.
 
 	  This driver can also be built as a module.  If so, the module
 	  will be called bmg160_i2c or bmg160_spi.
diff --git a/drivers/iio/gyro/adis16130.c b/drivers/iio/gyro/adis16130.c
index 79e63c8a2ea8..2a9ec08ec561 100644
--- a/drivers/iio/gyro/adis16130.c
+++ b/drivers/iio/gyro/adis16130.c
@@ -12,6 +12,8 @@
 
 #include <linux/iio/iio.h>
 
+#include <asm/unaligned.h>
+
 #define ADIS16130_CON         0x0
 #define ADIS16130_CON_RD      (1 << 6)
 #define ADIS16130_IOP         0x1
@@ -59,7 +61,7 @@ static int adis16130_spi_read(struct iio_dev *indio_dev, u8 reg_addr, u32 *val)
 
 	ret = spi_sync_transfer(st->us, &xfer, 1);
 	if (ret == 0)
-		*val = (st->buf[1] << 16) | (st->buf[2] << 8) | st->buf[3];
+		*val = get_unaligned_be24(&st->buf[1]);
 	mutex_unlock(&st->buf_lock);
 
 	return ret;
diff --git a/drivers/iio/gyro/adis16136.c b/drivers/iio/gyro/adis16136.c
index a4c967a5fc5c..afdc57af475d 100644
--- a/drivers/iio/gyro/adis16136.c
+++ b/drivers/iio/gyro/adis16136.c
@@ -148,16 +148,14 @@ DEFINE_DEBUGFS_ATTRIBUTE(adis16136_flash_count_fops,
 static int adis16136_debugfs_init(struct iio_dev *indio_dev)
 {
 	struct adis16136 *adis16136 = iio_priv(indio_dev);
+	struct dentry *d = iio_get_debugfs_dentry(indio_dev);
 
 	debugfs_create_file_unsafe("serial_number", 0400,
-		indio_dev->debugfs_dentry, adis16136,
-		&adis16136_serial_fops);
+		d, adis16136, &adis16136_serial_fops);
 	debugfs_create_file_unsafe("product_id", 0400,
-		indio_dev->debugfs_dentry,
-		adis16136, &adis16136_product_id_fops);
+		d, adis16136, &adis16136_product_id_fops);
 	debugfs_create_file_unsafe("flash_count", 0400,
-		indio_dev->debugfs_dentry,
-		adis16136, &adis16136_flash_count_fops);
+		d, adis16136, &adis16136_flash_count_fops);
 
 	return 0;
 }
diff --git a/drivers/iio/gyro/bmg160_i2c.c b/drivers/iio/gyro/bmg160_i2c.c
index 4fc9c6a3321f..b3fa46bd02cb 100644
--- a/drivers/iio/gyro/bmg160_i2c.c
+++ b/drivers/iio/gyro/bmg160_i2c.c
@@ -21,8 +21,8 @@ static int bmg160_i2c_probe(struct i2c_client *client,
 
 	regmap = devm_regmap_init_i2c(client, &bmg160_regmap_i2c_conf);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&client->dev, "Failed to register i2c regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 
@@ -42,6 +42,7 @@ static int bmg160_i2c_remove(struct i2c_client *client)
 static const struct acpi_device_id bmg160_acpi_match[] = {
 	{"BMG0160", 0},
 	{"BMI055B", 0},
+	{"BMI088B", 0},
 	{},
 };
 
@@ -50,6 +51,7 @@ MODULE_DEVICE_TABLE(acpi, bmg160_acpi_match);
 static const struct i2c_device_id bmg160_i2c_id[] = {
 	{"bmg160", 0},
 	{"bmi055_gyro", 0},
+	{"bmi088_gyro", 0},
 	{}
 };
 
diff --git a/drivers/iio/gyro/bmg160_spi.c b/drivers/iio/gyro/bmg160_spi.c
index 182a59c42507..745962e1e423 100644
--- a/drivers/iio/gyro/bmg160_spi.c
+++ b/drivers/iio/gyro/bmg160_spi.c
@@ -19,8 +19,8 @@ static int bmg160_spi_probe(struct spi_device *spi)
 
 	regmap = devm_regmap_init_spi(spi, &bmg160_regmap_spi_conf);
 	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&spi->dev, "Failed to register spi regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 
@@ -37,6 +37,7 @@ static int bmg160_spi_remove(struct spi_device *spi)
 static const struct spi_device_id bmg160_spi_id[] = {
 	{"bmg160", 0},
 	{"bmi055_gyro", 0},
+	{"bmi088_gyro", 0},
 	{}
 };
 
diff --git a/drivers/iio/gyro/hid-sensor-gyro-3d.c b/drivers/iio/gyro/hid-sensor-gyro-3d.c
index 08cacbbf31e6..7f382aae1dfd 100644
--- a/drivers/iio/gyro/hid-sensor-gyro-3d.c
+++ b/drivers/iio/gyro/hid-sensor-gyro-3d.c
@@ -14,8 +14,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 enum gyro_3d_channel {
@@ -326,18 +324,13 @@ static int hid_gyro_3d_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		goto error_free_dev_mem;
-	}
 	atomic_set(&gyro_state->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 					&gyro_state->common_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		goto error_free_dev_mem;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -361,9 +354,7 @@ static int hid_gyro_3d_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&gyro_state->common_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &gyro_state->common_attributes);
 error_free_dev_mem:
 	kfree(indio_dev->channels);
 	return ret;
@@ -378,8 +369,7 @@ static int hid_gyro_3d_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_GYRO_3D);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&gyro_state->common_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &gyro_state->common_attributes);
 	kfree(indio_dev->channels);
 
 	return 0;
diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c
index afa8018b9238..ef5bcbc4b45b 100644
--- a/drivers/iio/gyro/mpu3050-i2c.c
+++ b/drivers/iio/gyro/mpu3050-i2c.c
@@ -51,8 +51,8 @@ static int mpu3050_i2c_probe(struct i2c_client *client,
 
 	regmap = devm_regmap_init_i2c(client, &mpu3050_i2c_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&client->dev, "Failed to register i2c regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 
diff --git a/drivers/iio/gyro/st_gyro_buffer.c b/drivers/iio/gyro/st_gyro_buffer.c
index 7465ad62391c..9c92ff7a82be 100644
--- a/drivers/iio/gyro/st_gyro_buffer.c
+++ b/drivers/iio/gyro/st_gyro_buffer.c
@@ -37,8 +37,7 @@ static int st_gyro_buffer_postenable(struct iio_dev *indio_dev)
 	if (err < 0)
 		return err;
 
-	err = st_sensors_set_axis_enable(indio_dev,
-					 (u8)indio_dev->active_scan_mask[0]);
+	err = st_sensors_set_axis_enable(indio_dev, indio_dev->active_scan_mask[0]);
 	if (err < 0)
 		goto st_gyro_buffer_predisable;
 
diff --git a/drivers/iio/gyro/st_gyro_core.c b/drivers/iio/gyro/st_gyro_core.c
index 26c50b24bc08..c8aa051995d3 100644
--- a/drivers/iio/gyro/st_gyro_core.c
+++ b/drivers/iio/gyro/st_gyro_core.c
@@ -460,6 +460,7 @@ EXPORT_SYMBOL(st_gyro_get_settings);
 int st_gyro_common_probe(struct iio_dev *indio_dev)
 {
 	struct st_sensor_data *gdata = iio_priv(indio_dev);
+	struct st_sensors_platform_data *pdata;
 	int err;
 
 	indio_dev->modes = INDIO_DIRECT_MODE;
@@ -477,12 +478,12 @@ int st_gyro_common_probe(struct iio_dev *indio_dev)
 	indio_dev->channels = gdata->sensor_settings->ch;
 	indio_dev->num_channels = ST_SENSORS_NUMBER_ALL_CHANNELS;
 
-	gdata->current_fullscale = (struct st_sensor_fullscale_avl *)
-					&gdata->sensor_settings->fs.fs_avl[0];
+	gdata->current_fullscale = &gdata->sensor_settings->fs.fs_avl[0];
 	gdata->odr = gdata->sensor_settings->odr.odr_avl[0].hz;
 
-	err = st_sensors_init_sensor(indio_dev,
-				(struct st_sensors_platform_data *)&gyro_pdata);
+	pdata = (struct st_sensors_platform_data *)&gyro_pdata;
+
+	err = st_sensors_init_sensor(indio_dev, pdata);
 	if (err < 0)
 		goto st_gyro_power_off;
 
diff --git a/drivers/iio/health/afe4403.c b/drivers/iio/health/afe4403.c
index dc22dc363a99..e9f87e42ff4f 100644
--- a/drivers/iio/health/afe4403.c
+++ b/drivers/iio/health/afe4403.c
@@ -23,6 +23,8 @@
 #include <linux/iio/triggered_buffer.h>
 #include <linux/iio/trigger_consumer.h>
 
+#include <asm/unaligned.h>
+
 #include "afe440x.h"
 
 #define AFE4403_DRIVER_NAME		"afe4403"
@@ -220,13 +222,11 @@ static int afe4403_read(struct afe4403_data *afe, unsigned int reg, u32 *val)
 	if (ret)
 		return ret;
 
-	ret = spi_write_then_read(afe->spi, &reg, 1, rx, 3);
+	ret = spi_write_then_read(afe->spi, &reg, 1, rx, sizeof(rx));
 	if (ret)
 		return ret;
 
-	*val = (rx[0] << 16) |
-		(rx[1] << 8) |
-		(rx[2]);
+	*val = get_unaligned_be24(&rx[0]);
 
 	/* Disable reading from the device */
 	tx[3] = AFE440X_CONTROL0_WRITE;
@@ -322,13 +322,11 @@ static irqreturn_t afe4403_trigger_handler(int irq, void *private)
 			 indio_dev->masklength) {
 		ret = spi_write_then_read(afe->spi,
 					  &afe4403_channel_values[bit], 1,
-					  rx, 3);
+					  rx, sizeof(rx));
 		if (ret)
 			goto err;
 
-		buffer[i++] = (rx[0] << 16) |
-				(rx[1] << 8) |
-				(rx[2]);
+		buffer[i++] = get_unaligned_be24(&rx[0]);
 	}
 
 	/* Disable reading from the device */
diff --git a/drivers/iio/health/max30100.c b/drivers/iio/health/max30100.c
index 84010501762d..546fc37ad75d 100644
--- a/drivers/iio/health/max30100.c
+++ b/drivers/iio/health/max30100.c
@@ -16,7 +16,7 @@
 #include <linux/irq.h>
 #include <linux/i2c.h>
 #include <linux/mutex.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/buffer.h>
@@ -267,11 +267,10 @@ static int max30100_get_current_idx(unsigned int val, int *reg)
 static int max30100_led_init(struct max30100_data *data)
 {
 	struct device *dev = &data->client->dev;
-	struct device_node *np = dev->of_node;
 	unsigned int val[2];
 	int reg, ret;
 
-	ret = of_property_read_u32_array(np, "maxim,led-current-microamp",
+	ret = device_property_read_u32_array(dev, "maxim,led-current-microamp",
 					(unsigned int *) &val, 2);
 	if (ret) {
 		/* Default to 24 mA RED LED, 50 mA IR LED */
@@ -502,7 +501,7 @@ MODULE_DEVICE_TABLE(of, max30100_dt_ids);
 static struct i2c_driver max30100_driver = {
 	.driver = {
 		.name	= MAX30100_DRV_NAME,
-		.of_match_table	= of_match_ptr(max30100_dt_ids),
+		.of_match_table	= max30100_dt_ids,
 	},
 	.probe		= max30100_probe,
 	.remove		= max30100_remove,
diff --git a/drivers/iio/humidity/hid-sensor-humidity.c b/drivers/iio/humidity/hid-sensor-humidity.c
index c99b54b0568d..d2318c4aab0f 100644
--- a/drivers/iio/humidity/hid-sensor-humidity.c
+++ b/drivers/iio/humidity/hid-sensor-humidity.c
@@ -7,8 +7,6 @@
 #include <linux/hid-sensor-hub.h>
 #include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
-#include <linux/iio/triggered_buffer.h>
-#include <linux/iio/trigger_consumer.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 
@@ -233,12 +231,8 @@ static int hid_humidity_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = devm_iio_triggered_buffer_setup(&pdev->dev, indio_dev,
-					&iio_pollfunc_store_time, NULL, NULL);
-	if (ret)
-		return ret;
-
 	atomic_set(&humid_st->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 				&humid_st->common_attributes);
 	if (ret)
@@ -261,7 +255,7 @@ static int hid_humidity_probe(struct platform_device *pdev)
 error_remove_callback:
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_HUMIDITY);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&humid_st->common_attributes);
+	hid_sensor_remove_trigger(indio_dev, &humid_st->common_attributes);
 	return ret;
 }
 
@@ -274,7 +268,7 @@ static int hid_humidity_remove(struct platform_device *pdev)
 
 	iio_device_unregister(indio_dev);
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_HUMIDITY);
-	hid_sensor_remove_trigger(&humid_st->common_attributes);
+	hid_sensor_remove_trigger(indio_dev, &humid_st->common_attributes);
 
 	return 0;
 }
diff --git a/drivers/iio/humidity/hts221_buffer.c b/drivers/iio/humidity/hts221_buffer.c
index 81d50a861c22..9fb3f33614d4 100644
--- a/drivers/iio/humidity/hts221_buffer.c
+++ b/drivers/iio/humidity/hts221_buffer.c
@@ -74,10 +74,9 @@ static irqreturn_t hts221_trigger_handler_thread(int irq, void *private)
 
 int hts221_allocate_trigger(struct hts221_hw *hw)
 {
+	struct st_sensors_platform_data *pdata = dev_get_platdata(hw->dev);
 	struct iio_dev *iio_dev = iio_priv_to_dev(hw);
 	bool irq_active_low = false, open_drain = false;
-	struct device_node *np = hw->dev->of_node;
-	struct st_sensors_platform_data *pdata;
 	unsigned long irq_type;
 	int err;
 
@@ -106,8 +105,7 @@ int hts221_allocate_trigger(struct hts221_hw *hw)
 	if (err < 0)
 		return err;
 
-	pdata = (struct st_sensors_platform_data *)hw->dev->platform_data;
-	if ((np && of_property_read_bool(np, "drive-open-drain")) ||
+	if (device_property_read_bool(hw->dev, "drive-open-drain") ||
 	    (pdata && pdata->open_drain)) {
 		irq_type |= IRQF_SHARED;
 		open_drain = true;
diff --git a/drivers/iio/humidity/hts221_i2c.c b/drivers/iio/humidity/hts221_i2c.c
index 4272b7030c44..cab39c4756f8 100644
--- a/drivers/iio/humidity/hts221_i2c.c
+++ b/drivers/iio/humidity/hts221_i2c.c
@@ -32,8 +32,8 @@ static int hts221_i2c_probe(struct i2c_client *client,
 
 	regmap = devm_regmap_init_i2c(client, &hts221_i2c_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&client->dev, "Failed to register i2c regmap %ld\n",
+			PTR_ERR(regmap));
 		return PTR_ERR(regmap);
 	}
 
@@ -63,7 +63,7 @@ static struct i2c_driver hts221_driver = {
 	.driver = {
 		.name = "hts221_i2c",
 		.pm = &hts221_pm_ops,
-		.of_match_table = of_match_ptr(hts221_i2c_of_match),
+		.of_match_table = hts221_i2c_of_match,
 		.acpi_match_table = ACPI_PTR(hts221_acpi_match),
 	},
 	.probe = hts221_i2c_probe,
diff --git a/drivers/iio/humidity/hts221_spi.c b/drivers/iio/humidity/hts221_spi.c
index 055dba8897d2..729e86e433b1 100644
--- a/drivers/iio/humidity/hts221_spi.c
+++ b/drivers/iio/humidity/hts221_spi.c
@@ -31,8 +31,8 @@ static int hts221_spi_probe(struct spi_device *spi)
 
 	regmap = devm_regmap_init_spi(spi, &hts221_spi_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&spi->dev, "Failed to register spi regmap %ld\n",
+			PTR_ERR(regmap));
 		return PTR_ERR(regmap);
 	}
 
@@ -56,7 +56,7 @@ static struct spi_driver hts221_driver = {
 	.driver = {
 		.name = "hts221_spi",
 		.pm = &hts221_pm_ops,
-		.of_match_table = of_match_ptr(hts221_spi_of_match),
+		.of_match_table = hts221_spi_of_match,
 	},
 	.probe = hts221_spi_probe,
 	.id_table = hts221_spi_id_table,
diff --git a/drivers/iio/imu/Kconfig b/drivers/iio/imu/Kconfig
index 60bb1029e759..fc4123d518bc 100644
--- a/drivers/iio/imu/Kconfig
+++ b/drivers/iio/imu/Kconfig
@@ -29,6 +29,19 @@ config ADIS16460
 	  To compile this driver as a module, choose M here: the module will be
 	  called adis16460.
 
+config ADIS16475
+	tristate "Analog Devices ADIS16475 and similar IMU driver"
+	depends on SPI
+	select IIO_ADIS_LIB
+	select IIO_ADIS_LIB_BUFFER if IIO_BUFFER
+	help
+	  Say yes here to build support for Analog Devices ADIS16470, ADIS16475,
+	  ADIS16477, ADIS16465, ADIS16467, ADIS16500, ADIS16505, ADIS16507 inertial
+	  sensors.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called adis16475.
+
 config ADIS16480
 	tristate "Analog Devices ADIS16480 and similar IMU driver"
 	depends on SPI
diff --git a/drivers/iio/imu/Makefile b/drivers/iio/imu/Makefile
index 5237fd4bc384..88b2c4555230 100644
--- a/drivers/iio/imu/Makefile
+++ b/drivers/iio/imu/Makefile
@@ -6,6 +6,7 @@
 # When adding new entries keep the list in alphabetical order
 obj-$(CONFIG_ADIS16400) += adis16400.o
 obj-$(CONFIG_ADIS16460) += adis16460.o
+obj-$(CONFIG_ADIS16475) += adis16475.o
 obj-$(CONFIG_ADIS16480) += adis16480.o
 
 adis_lib-y += adis.o
diff --git a/drivers/iio/imu/adis.c b/drivers/iio/imu/adis.c
index a8afd01de4f3..c539dfa3b8d3 100644
--- a/drivers/iio/imu/adis.c
+++ b/drivers/iio/imu/adis.c
@@ -223,6 +223,31 @@ int __adis_read_reg(struct adis *adis, unsigned int reg,
 	return ret;
 }
 EXPORT_SYMBOL_GPL(__adis_read_reg);
+/**
+ * __adis_update_bits_base() - ADIS Update bits function - Unlocked version
+ * @adis: The adis device
+ * @reg: The address of the lower of the two registers
+ * @mask: Bitmask to change
+ * @val: Value to be written
+ * @size: Size of the register to update
+ *
+ * Updates the desired bits of @reg in accordance with @mask and @val.
+ */
+int __adis_update_bits_base(struct adis *adis, unsigned int reg, const u32 mask,
+			    const u32 val, u8 size)
+{
+	int ret;
+	u32 __val;
+
+	ret = __adis_read_reg(adis, reg, &__val, size);
+	if (ret)
+		return ret;
+
+	__val = (__val & ~mask) | (val & mask);
+
+	return __adis_write_reg(adis, reg, __val, size);
+}
+EXPORT_SYMBOL_GPL(__adis_update_bits_base);
 
 #ifdef CONFIG_DEBUG_FS
 
@@ -419,7 +444,7 @@ int __adis_initial_startup(struct adis *adis)
 
 	if (prod_id != adis->data->prod_id)
 		dev_warn(&adis->spi->dev,
-			 "Device ID(%u) and product ID(%u) do not match.",
+			 "Device ID(%u) and product ID(%u) do not match.\n",
 			 adis->data->prod_id, prod_id);
 
 	return 0;
diff --git a/drivers/iio/imu/adis16400.c b/drivers/iio/imu/adis16400.c
index 05e70c1c4835..229f2ff98469 100644
--- a/drivers/iio/imu/adis16400.c
+++ b/drivers/iio/imu/adis16400.c
@@ -258,7 +258,7 @@ static int adis16400_show_product_id(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16400_product_id_fops,
+DEFINE_DEBUGFS_ATTRIBUTE(adis16400_product_id_fops,
 	adis16400_show_product_id, NULL, "%lld\n");
 
 static int adis16400_show_flash_count(void *arg, u64 *val)
@@ -275,23 +275,22 @@ static int adis16400_show_flash_count(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16400_flash_count_fops,
+DEFINE_DEBUGFS_ATTRIBUTE(adis16400_flash_count_fops,
 	adis16400_show_flash_count, NULL, "%lld\n");
 
 static int adis16400_debugfs_init(struct iio_dev *indio_dev)
 {
 	struct adis16400_state *st = iio_priv(indio_dev);
+	struct dentry *d = iio_get_debugfs_dentry(indio_dev);
 
 	if (st->variant->flags & ADIS16400_HAS_SERIAL_NUMBER)
-		debugfs_create_file("serial_number", 0400,
-			indio_dev->debugfs_dentry, st,
-			&adis16400_serial_number_fops);
+		debugfs_create_file_unsafe("serial_number", 0400,
+				d, st, &adis16400_serial_number_fops);
 	if (st->variant->flags & ADIS16400_HAS_PROD_ID)
-		debugfs_create_file("product_id", 0400,
-			indio_dev->debugfs_dentry, st,
-			&adis16400_product_id_fops);
-	debugfs_create_file("flash_count", 0400, indio_dev->debugfs_dentry,
-		st, &adis16400_flash_count_fops);
+		debugfs_create_file_unsafe("product_id", 0400,
+				d, st, &adis16400_product_id_fops);
+	debugfs_create_file_unsafe("flash_count", 0400,
+			d, st, &adis16400_flash_count_fops);
 
 	return 0;
 }
@@ -1194,7 +1193,7 @@ static int adis16400_probe(struct spi_device *spi)
 		indio_dev->available_scan_masks = st->avail_scan_mask;
 		st->adis.burst = &adis16400_burst;
 		if (st->variant->flags & ADIS16400_BURST_DIAG_STAT)
-			st->adis.burst->extra_len = sizeof(u16);
+			st->adis.burst_extra_len = sizeof(u16);
 	}
 
 	adis16400_data = &st->variant->adis_data;
diff --git a/drivers/iio/imu/adis16460.c b/drivers/iio/imu/adis16460.c
index 0027683d0256..ad20c488a3ba 100644
--- a/drivers/iio/imu/adis16460.c
+++ b/drivers/iio/imu/adis16460.c
@@ -87,8 +87,8 @@ static int adis16460_show_serial_number(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16460_serial_number_fops,
-	adis16460_show_serial_number, NULL, "0x%.4llx\n");
+DEFINE_DEBUGFS_ATTRIBUTE(adis16460_serial_number_fops,
+		adis16460_show_serial_number, NULL, "0x%.4llx\n");
 
 static int adis16460_show_product_id(void *arg, u64 *val)
 {
@@ -105,8 +105,8 @@ static int adis16460_show_product_id(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16460_product_id_fops,
-	adis16460_show_product_id, NULL, "%llu\n");
+DEFINE_DEBUGFS_ATTRIBUTE(adis16460_product_id_fops,
+		adis16460_show_product_id, NULL, "%llu\n");
 
 static int adis16460_show_flash_count(void *arg, u64 *val)
 {
@@ -123,19 +123,20 @@ static int adis16460_show_flash_count(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16460_flash_count_fops,
-	adis16460_show_flash_count, NULL, "%lld\n");
+DEFINE_DEBUGFS_ATTRIBUTE(adis16460_flash_count_fops,
+		adis16460_show_flash_count, NULL, "%lld\n");
 
 static int adis16460_debugfs_init(struct iio_dev *indio_dev)
 {
 	struct adis16460 *adis16460 = iio_priv(indio_dev);
-
-	debugfs_create_file("serial_number", 0400, indio_dev->debugfs_dentry,
-		adis16460, &adis16460_serial_number_fops);
-	debugfs_create_file("product_id", 0400, indio_dev->debugfs_dentry,
-		adis16460, &adis16460_product_id_fops);
-	debugfs_create_file("flash_count", 0400, indio_dev->debugfs_dentry,
-		adis16460, &adis16460_flash_count_fops);
+	struct dentry *d = iio_get_debugfs_dentry(indio_dev);
+
+	debugfs_create_file_unsafe("serial_number", 0400,
+			d, adis16460, &adis16460_serial_number_fops);
+	debugfs_create_file_unsafe("product_id", 0400,
+			d, adis16460, &adis16460_product_id_fops);
+	debugfs_create_file_unsafe("flash_count", 0400,
+			d, adis16460, &adis16460_flash_count_fops);
 
 	return 0;
 }
diff --git a/drivers/iio/imu/adis16475.c b/drivers/iio/imu/adis16475.c
new file mode 100644
index 000000000000..c6dac4fc67a1
--- /dev/null
+++ b/drivers/iio/imu/adis16475.c
@@ -0,0 +1,1338 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ADIS16475 IMU driver
+ *
+ * Copyright 2019 Analog Devices Inc.
+ */
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/imu/adis.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
+#include <linux/spi/spi.h>
+
+#define ADIS16475_REG_DIAG_STAT		0x02
+#define ADIS16475_REG_X_GYRO_L		0x04
+#define ADIS16475_REG_Y_GYRO_L		0x08
+#define ADIS16475_REG_Z_GYRO_L		0x0C
+#define ADIS16475_REG_X_ACCEL_L		0x10
+#define ADIS16475_REG_Y_ACCEL_L		0x14
+#define ADIS16475_REG_Z_ACCEL_L		0x18
+#define ADIS16475_REG_TEMP_OUT		0x1c
+#define ADIS16475_REG_X_GYRO_BIAS_L	0x40
+#define ADIS16475_REG_Y_GYRO_BIAS_L	0x44
+#define ADIS16475_REG_Z_GYRO_BIAS_L	0x48
+#define ADIS16475_REG_X_ACCEL_BIAS_L	0x4c
+#define ADIS16475_REG_Y_ACCEL_BIAS_L	0x50
+#define ADIS16475_REG_Z_ACCEL_BIAS_L	0x54
+#define ADIS16475_REG_FILT_CTRL		0x5c
+#define ADIS16475_FILT_CTRL_MASK	GENMASK(2, 0)
+#define ADIS16475_FILT_CTRL(x)		FIELD_PREP(ADIS16475_FILT_CTRL_MASK, x)
+#define ADIS16475_REG_MSG_CTRL		0x60
+#define ADIS16475_MSG_CTRL_DR_POL_MASK	BIT(0)
+#define ADIS16475_MSG_CTRL_DR_POL(x) \
+				FIELD_PREP(ADIS16475_MSG_CTRL_DR_POL_MASK, x)
+#define ADIS16475_SYNC_MODE_MASK	GENMASK(4, 2)
+#define ADIS16475_SYNC_MODE(x)		FIELD_PREP(ADIS16475_SYNC_MODE_MASK, x)
+#define ADIS16475_REG_UP_SCALE		0x62
+#define ADIS16475_REG_DEC_RATE		0x64
+#define ADIS16475_REG_GLOB_CMD		0x68
+#define ADIS16475_REG_FIRM_REV		0x6c
+#define ADIS16475_REG_FIRM_DM		0x6e
+#define ADIS16475_REG_FIRM_Y		0x70
+#define ADIS16475_REG_PROD_ID		0x72
+#define ADIS16475_REG_SERIAL_NUM	0x74
+#define ADIS16475_REG_FLASH_CNT		0x7c
+#define ADIS16500_BURST32_MASK		BIT(9)
+#define ADIS16500_BURST32(x)		FIELD_PREP(ADIS16500_BURST32_MASK, x)
+/* number of data elements in burst mode */
+#define ADIS16475_BURST32_MAX_DATA	32
+#define ADIS16475_BURST_MAX_DATA	20
+#define ADIS16475_MAX_SCAN_DATA		20
+/* spi max speed in brust mode */
+#define ADIS16475_BURST_MAX_SPEED	1000000
+#define ADIS16475_LSB_DEC_MASK		BIT(0)
+#define ADIS16475_LSB_FIR_MASK		BIT(1)
+
+enum {
+	ADIS16475_SYNC_DIRECT = 1,
+	ADIS16475_SYNC_SCALED,
+	ADIS16475_SYNC_OUTPUT,
+	ADIS16475_SYNC_PULSE = 5,
+};
+
+struct adis16475_sync {
+	u16 sync_mode;
+	u16 min_rate;
+	u16 max_rate;
+};
+
+struct adis16475_chip_info {
+	const struct iio_chan_spec *channels;
+	const struct adis16475_sync *sync;
+	const struct adis_data adis_data;
+	const char *name;
+	u32 num_channels;
+	u32 gyro_max_val;
+	u32 gyro_max_scale;
+	u32 accel_max_val;
+	u32 accel_max_scale;
+	u32 temp_scale;
+	u32 int_clk;
+	u16 max_dec;
+	u8 num_sync;
+	bool has_burst32;
+};
+
+struct adis16475 {
+	const struct adis16475_chip_info *info;
+	struct adis adis;
+	u32 clk_freq;
+	bool burst32;
+	unsigned long lsb_flag;
+	/* Alignment needed for the timestamp */
+	__be16 data[ADIS16475_MAX_SCAN_DATA] __aligned(8);
+};
+
+enum {
+	ADIS16475_SCAN_GYRO_X,
+	ADIS16475_SCAN_GYRO_Y,
+	ADIS16475_SCAN_GYRO_Z,
+	ADIS16475_SCAN_ACCEL_X,
+	ADIS16475_SCAN_ACCEL_Y,
+	ADIS16475_SCAN_ACCEL_Z,
+	ADIS16475_SCAN_TEMP,
+	ADIS16475_SCAN_DIAG_S_FLAGS,
+	ADIS16475_SCAN_CRC_FAILURE,
+};
+
+#ifdef CONFIG_DEBUG_FS
+static ssize_t adis16475_show_firmware_revision(struct file *file,
+						char __user *userbuf,
+						size_t count, loff_t *ppos)
+{
+	struct adis16475 *st = file->private_data;
+	char buf[7];
+	size_t len;
+	u16 rev;
+	int ret;
+
+	ret = adis_read_reg_16(&st->adis, ADIS16475_REG_FIRM_REV, &rev);
+	if (ret)
+		return ret;
+
+	len = scnprintf(buf, sizeof(buf), "%x.%x\n", rev >> 8, rev & 0xff);
+
+	return simple_read_from_buffer(userbuf, count, ppos, buf, len);
+}
+
+static const struct file_operations adis16475_firmware_revision_fops = {
+	.open = simple_open,
+	.read = adis16475_show_firmware_revision,
+	.llseek = default_llseek,
+	.owner = THIS_MODULE,
+};
+
+static ssize_t adis16475_show_firmware_date(struct file *file,
+					    char __user *userbuf,
+					    size_t count, loff_t *ppos)
+{
+	struct adis16475 *st = file->private_data;
+	u16 md, year;
+	char buf[12];
+	size_t len;
+	int ret;
+
+	ret = adis_read_reg_16(&st->adis, ADIS16475_REG_FIRM_Y, &year);
+	if (ret)
+		return ret;
+
+	ret = adis_read_reg_16(&st->adis, ADIS16475_REG_FIRM_DM, &md);
+	if (ret)
+		return ret;
+
+	len = snprintf(buf, sizeof(buf), "%.2x-%.2x-%.4x\n", md >> 8, md & 0xff,
+		       year);
+
+	return simple_read_from_buffer(userbuf, count, ppos, buf, len);
+}
+
+static const struct file_operations adis16475_firmware_date_fops = {
+	.open = simple_open,
+	.read = adis16475_show_firmware_date,
+	.llseek = default_llseek,
+	.owner = THIS_MODULE,
+};
+
+static int adis16475_show_serial_number(void *arg, u64 *val)
+{
+	struct adis16475 *st = arg;
+	u16 serial;
+	int ret;
+
+	ret = adis_read_reg_16(&st->adis, ADIS16475_REG_SERIAL_NUM, &serial);
+	if (ret)
+		return ret;
+
+	*val = serial;
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(adis16475_serial_number_fops,
+			 adis16475_show_serial_number, NULL, "0x%.4llx\n");
+
+static int adis16475_show_product_id(void *arg, u64 *val)
+{
+	struct adis16475 *st = arg;
+	u16 prod_id;
+	int ret;
+
+	ret = adis_read_reg_16(&st->adis, ADIS16475_REG_PROD_ID, &prod_id);
+	if (ret)
+		return ret;
+
+	*val = prod_id;
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(adis16475_product_id_fops,
+			 adis16475_show_product_id, NULL, "%llu\n");
+
+static int adis16475_show_flash_count(void *arg, u64 *val)
+{
+	struct adis16475 *st = arg;
+	u32 flash_count;
+	int ret;
+
+	ret = adis_read_reg_32(&st->adis, ADIS16475_REG_FLASH_CNT,
+			       &flash_count);
+	if (ret)
+		return ret;
+
+	*val = flash_count;
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(adis16475_flash_count_fops,
+			 adis16475_show_flash_count, NULL, "%lld\n");
+
+static void adis16475_debugfs_init(struct iio_dev *indio_dev)
+{
+	struct adis16475 *st = iio_priv(indio_dev);
+	struct dentry *d = iio_get_debugfs_dentry(indio_dev);
+
+	debugfs_create_file_unsafe("serial_number", 0400,
+				   d, st, &adis16475_serial_number_fops);
+	debugfs_create_file_unsafe("product_id", 0400,
+				   d, st, &adis16475_product_id_fops);
+	debugfs_create_file_unsafe("flash_count", 0400,
+				   d, st, &adis16475_flash_count_fops);
+	debugfs_create_file("firmware_revision", 0400,
+			    d, st, &adis16475_firmware_revision_fops);
+	debugfs_create_file("firmware_date", 0400, d,
+			    st, &adis16475_firmware_date_fops);
+}
+#else
+static void adis16475_debugfs_init(struct iio_dev *indio_dev)
+{
+}
+#endif
+
+static int adis16475_get_freq(struct adis16475 *st, u32 *freq)
+{
+	int ret;
+	u16 dec;
+
+	ret = adis_read_reg_16(&st->adis, ADIS16475_REG_DEC_RATE, &dec);
+	if (ret)
+		return -EINVAL;
+
+	*freq = DIV_ROUND_CLOSEST(st->clk_freq, dec + 1);
+
+	return 0;
+}
+
+static int adis16475_set_freq(struct adis16475 *st, const u32 freq)
+{
+	u16 dec;
+	int ret;
+
+	if (!freq)
+		return -EINVAL;
+
+	dec = DIV_ROUND_CLOSEST(st->clk_freq, freq);
+
+	if (dec)
+		dec--;
+
+	if (dec > st->info->max_dec)
+		dec = st->info->max_dec;
+
+	ret = adis_write_reg_16(&st->adis, ADIS16475_REG_DEC_RATE, dec);
+	if (ret)
+		return ret;
+
+	/*
+	 * If decimation is used, then gyro and accel data will have meaningful
+	 * bits on the LSB registers. This info is used on the trigger handler.
+	 */
+	assign_bit(ADIS16475_LSB_DEC_MASK, &st->lsb_flag, dec);
+
+	return 0;
+}
+
+/* The values are approximated. */
+static const u32 adis16475_3db_freqs[] = {
+	[0] = 720, /* Filter disabled, full BW (~720Hz) */
+	[1] = 360,
+	[2] = 164,
+	[3] = 80,
+	[4] = 40,
+	[5] = 20,
+	[6] = 10,
+};
+
+static int adis16475_get_filter(struct adis16475 *st, u32 *filter)
+{
+	u16 filter_sz;
+	int ret;
+	const int mask = ADIS16475_FILT_CTRL_MASK;
+
+	ret = adis_read_reg_16(&st->adis, ADIS16475_REG_FILT_CTRL, &filter_sz);
+	if (ret)
+		return ret;
+
+	*filter = adis16475_3db_freqs[filter_sz & mask];
+
+	return 0;
+}
+
+static int adis16475_set_filter(struct adis16475 *st, const u32 filter)
+{
+	int i = ARRAY_SIZE(adis16475_3db_freqs);
+	int ret;
+
+	while (--i) {
+		if (adis16475_3db_freqs[i] >= filter)
+			break;
+	}
+
+	ret = adis_write_reg_16(&st->adis, ADIS16475_REG_FILT_CTRL,
+				ADIS16475_FILT_CTRL(i));
+	if (ret)
+		return ret;
+
+	/*
+	 * If FIR is used, then gyro and accel data will have meaningful
+	 * bits on the LSB registers. This info is used on the trigger handler.
+	 */
+	assign_bit(ADIS16475_LSB_FIR_MASK, &st->lsb_flag, i);
+
+	return 0;
+}
+
+static const u32 adis16475_calib_regs[] = {
+	[ADIS16475_SCAN_GYRO_X] = ADIS16475_REG_X_GYRO_BIAS_L,
+	[ADIS16475_SCAN_GYRO_Y] = ADIS16475_REG_Y_GYRO_BIAS_L,
+	[ADIS16475_SCAN_GYRO_Z] = ADIS16475_REG_Z_GYRO_BIAS_L,
+	[ADIS16475_SCAN_ACCEL_X] = ADIS16475_REG_X_ACCEL_BIAS_L,
+	[ADIS16475_SCAN_ACCEL_Y] = ADIS16475_REG_Y_ACCEL_BIAS_L,
+	[ADIS16475_SCAN_ACCEL_Z] = ADIS16475_REG_Z_ACCEL_BIAS_L,
+};
+
+static int adis16475_read_raw(struct iio_dev *indio_dev,
+			      const struct iio_chan_spec *chan,
+			      int *val, int *val2, long info)
+{
+	struct adis16475 *st = iio_priv(indio_dev);
+	int ret;
+	u32 tmp;
+
+	switch (info) {
+	case IIO_CHAN_INFO_RAW:
+		return adis_single_conversion(indio_dev, chan, 0, val);
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_ANGL_VEL:
+			*val = st->info->gyro_max_val;
+			*val2 = st->info->gyro_max_scale;
+			return IIO_VAL_FRACTIONAL;
+		case IIO_ACCEL:
+			*val = st->info->accel_max_val;
+			*val2 = st->info->accel_max_scale;
+			return IIO_VAL_FRACTIONAL;
+		case IIO_TEMP:
+			*val = st->info->temp_scale;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_CALIBBIAS:
+		ret = adis_read_reg_32(&st->adis,
+				       adis16475_calib_regs[chan->scan_index],
+				       val);
+		if (ret)
+			return ret;
+
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY:
+		ret = adis16475_get_filter(st, val);
+		if (ret)
+			return ret;
+
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		ret = adis16475_get_freq(st, &tmp);
+		if (ret)
+			return ret;
+
+		*val = tmp / 1000;
+		*val2 = (tmp % 1000) * 1000;
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int adis16475_write_raw(struct iio_dev *indio_dev,
+			       const struct iio_chan_spec *chan,
+			       int val, int val2, long info)
+{
+	struct adis16475 *st = iio_priv(indio_dev);
+	u32 tmp;
+
+	switch (info) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		tmp = val * 1000 + val2 / 1000;
+		return adis16475_set_freq(st, tmp);
+	case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY:
+		return adis16475_set_filter(st, val);
+	case IIO_CHAN_INFO_CALIBBIAS:
+		return adis_write_reg_32(&st->adis,
+					 adis16475_calib_regs[chan->scan_index],
+					 val);
+	default:
+		return -EINVAL;
+	}
+}
+
+#define ADIS16475_MOD_CHAN(_type, _mod, _address, _si, _r_bits, _s_bits) \
+	{ \
+		.type = (_type), \
+		.modified = 1, \
+		.channel2 = (_mod), \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			BIT(IIO_CHAN_INFO_CALIBBIAS), \
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ) | \
+			BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), \
+		.address = (_address), \
+		.scan_index = (_si), \
+		.scan_type = { \
+			.sign = 's', \
+			.realbits = (_r_bits), \
+			.storagebits = (_s_bits), \
+			.endianness = IIO_BE, \
+		}, \
+	}
+
+#define ADIS16475_GYRO_CHANNEL(_mod) \
+	ADIS16475_MOD_CHAN(IIO_ANGL_VEL, IIO_MOD_ ## _mod, \
+			   ADIS16475_REG_ ## _mod ## _GYRO_L, \
+			   ADIS16475_SCAN_GYRO_ ## _mod, 32, 32)
+
+#define ADIS16475_ACCEL_CHANNEL(_mod) \
+	ADIS16475_MOD_CHAN(IIO_ACCEL, IIO_MOD_ ## _mod, \
+			   ADIS16475_REG_ ## _mod ## _ACCEL_L, \
+			   ADIS16475_SCAN_ACCEL_ ## _mod, 32, 32)
+
+#define ADIS16475_TEMP_CHANNEL() { \
+		.type = IIO_TEMP, \
+		.indexed = 1, \
+		.channel = 0, \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			BIT(IIO_CHAN_INFO_SCALE), \
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ) | \
+			BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), \
+		.address = ADIS16475_REG_TEMP_OUT, \
+		.scan_index = ADIS16475_SCAN_TEMP, \
+		.scan_type = { \
+			.sign = 's', \
+			.realbits = 16, \
+			.storagebits = 16, \
+			.endianness = IIO_BE, \
+		}, \
+	}
+
+static const struct iio_chan_spec adis16475_channels[] = {
+	ADIS16475_GYRO_CHANNEL(X),
+	ADIS16475_GYRO_CHANNEL(Y),
+	ADIS16475_GYRO_CHANNEL(Z),
+	ADIS16475_ACCEL_CHANNEL(X),
+	ADIS16475_ACCEL_CHANNEL(Y),
+	ADIS16475_ACCEL_CHANNEL(Z),
+	ADIS16475_TEMP_CHANNEL(),
+	IIO_CHAN_SOFT_TIMESTAMP(7)
+};
+
+enum adis16475_variant {
+	ADIS16470,
+	ADIS16475_1,
+	ADIS16475_2,
+	ADIS16475_3,
+	ADIS16477_1,
+	ADIS16477_2,
+	ADIS16477_3,
+	ADIS16465_1,
+	ADIS16465_2,
+	ADIS16465_3,
+	ADIS16467_1,
+	ADIS16467_2,
+	ADIS16467_3,
+	ADIS16500,
+	ADIS16505_1,
+	ADIS16505_2,
+	ADIS16505_3,
+	ADIS16507_1,
+	ADIS16507_2,
+	ADIS16507_3,
+};
+
+enum {
+	ADIS16475_DIAG_STAT_DATA_PATH = 1,
+	ADIS16475_DIAG_STAT_FLASH_MEM,
+	ADIS16475_DIAG_STAT_SPI,
+	ADIS16475_DIAG_STAT_STANDBY,
+	ADIS16475_DIAG_STAT_SENSOR,
+	ADIS16475_DIAG_STAT_MEMORY,
+	ADIS16475_DIAG_STAT_CLK,
+};
+
+static const char * const adis16475_status_error_msgs[] = {
+	[ADIS16475_DIAG_STAT_DATA_PATH] = "Data Path Overrun",
+	[ADIS16475_DIAG_STAT_FLASH_MEM] = "Flash memory update failure",
+	[ADIS16475_DIAG_STAT_SPI] = "SPI communication error",
+	[ADIS16475_DIAG_STAT_STANDBY] = "Standby mode",
+	[ADIS16475_DIAG_STAT_SENSOR] = "Sensor failure",
+	[ADIS16475_DIAG_STAT_MEMORY] = "Memory failure",
+	[ADIS16475_DIAG_STAT_CLK] = "Clock error",
+};
+
+static int adis16475_enable_irq(struct adis *adis, bool enable)
+{
+	/*
+	 * There is no way to gate the data-ready signal internally inside the
+	 * ADIS16475. We can only control it's polarity...
+	 */
+	if (enable)
+		enable_irq(adis->spi->irq);
+	else
+		disable_irq(adis->spi->irq);
+
+	return 0;
+}
+
+#define ADIS16475_DATA(_prod_id, _timeouts)				\
+{									\
+	.msc_ctrl_reg = ADIS16475_REG_MSG_CTRL,				\
+	.glob_cmd_reg = ADIS16475_REG_GLOB_CMD,				\
+	.diag_stat_reg = ADIS16475_REG_DIAG_STAT,			\
+	.prod_id_reg = ADIS16475_REG_PROD_ID,				\
+	.prod_id = (_prod_id),						\
+	.self_test_mask = BIT(2),					\
+	.self_test_reg = ADIS16475_REG_GLOB_CMD,			\
+	.cs_change_delay = 16,						\
+	.read_delay = 5,						\
+	.write_delay = 5,						\
+	.status_error_msgs = adis16475_status_error_msgs,		\
+	.status_error_mask = BIT(ADIS16475_DIAG_STAT_DATA_PATH) |	\
+		BIT(ADIS16475_DIAG_STAT_FLASH_MEM) |			\
+		BIT(ADIS16475_DIAG_STAT_SPI) |				\
+		BIT(ADIS16475_DIAG_STAT_STANDBY) |			\
+		BIT(ADIS16475_DIAG_STAT_SENSOR) |			\
+		BIT(ADIS16475_DIAG_STAT_MEMORY) |			\
+		BIT(ADIS16475_DIAG_STAT_CLK),				\
+	.enable_irq = adis16475_enable_irq,				\
+	.timeouts = (_timeouts),					\
+}
+
+static const struct adis16475_sync adis16475_sync_mode[] = {
+	{ ADIS16475_SYNC_OUTPUT },
+	{ ADIS16475_SYNC_DIRECT, 1900, 2100 },
+	{ ADIS16475_SYNC_SCALED, 1, 128 },
+	{ ADIS16475_SYNC_PULSE, 1000, 2100 },
+};
+
+static const struct adis_timeout adis16475_timeouts = {
+	.reset_ms = 200,
+	.sw_reset_ms = 200,
+	.self_test_ms = 20,
+};
+
+static const struct adis_timeout adis1650x_timeouts = {
+	.reset_ms = 260,
+	.sw_reset_ms = 260,
+	.self_test_ms = 30,
+};
+
+static const struct adis16475_chip_info adis16475_chip_info[] = {
+	[ADIS16470] = {
+		.name = "adis16470",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(800 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16470, &adis16475_timeouts),
+	},
+	[ADIS16475_1] = {
+		.name = "adis16475-1",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(160 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(4000 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16475, &adis16475_timeouts),
+	},
+	[ADIS16475_2] = {
+		.name = "adis16475-2",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(40 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(4000 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16475, &adis16475_timeouts),
+	},
+	[ADIS16475_3] = {
+		.name = "adis16475-3",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(4000 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16475, &adis16475_timeouts),
+	},
+	[ADIS16477_1] = {
+		.name = "adis16477-1",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(160 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(800 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16477, &adis16475_timeouts),
+	},
+	[ADIS16477_2] = {
+		.name = "adis16477-2",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(40 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(800 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16477, &adis16475_timeouts),
+	},
+	[ADIS16477_3] = {
+		.name = "adis16477-3",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(800 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16477, &adis16475_timeouts),
+	},
+	[ADIS16465_1] = {
+		.name = "adis16465-1",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(160 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(4000 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16465, &adis16475_timeouts),
+	},
+	[ADIS16465_2] = {
+		.name = "adis16465-2",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(40 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(4000 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16465, &adis16475_timeouts),
+	},
+	[ADIS16465_3] = {
+		.name = "adis16465-3",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(4000 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16465, &adis16475_timeouts),
+	},
+	[ADIS16467_1] = {
+		.name = "adis16467-1",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(160 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(800 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16467, &adis16475_timeouts),
+	},
+	[ADIS16467_2] = {
+		.name = "adis16467-2",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(40 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(800 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16467, &adis16475_timeouts),
+	},
+	[ADIS16467_3] = {
+		.name = "adis16467-3",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 1,
+		.accel_max_scale = IIO_M_S_2_TO_G(800 << 16),
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode),
+		.adis_data = ADIS16475_DATA(16467, &adis16475_timeouts),
+	},
+	[ADIS16500] = {
+		.name = "adis16500",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 392,
+		.accel_max_scale = 32000 << 16,
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		/* pulse sync not supported */
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode) - 1,
+		.has_burst32 = true,
+		.adis_data = ADIS16475_DATA(16500, &adis1650x_timeouts),
+	},
+	[ADIS16505_1] = {
+		.name = "adis16505-1",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(160 << 16),
+		.accel_max_val = 78,
+		.accel_max_scale = 32000 << 16,
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		/* pulse sync not supported */
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode) - 1,
+		.has_burst32 = true,
+		.adis_data = ADIS16475_DATA(16505, &adis1650x_timeouts),
+	},
+	[ADIS16505_2] = {
+		.name = "adis16505-2",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(40 << 16),
+		.accel_max_val = 78,
+		.accel_max_scale = 32000 << 16,
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		/* pulse sync not supported */
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode) - 1,
+		.has_burst32 = true,
+		.adis_data = ADIS16475_DATA(16505, &adis1650x_timeouts),
+	},
+	[ADIS16505_3] = {
+		.name = "adis16505-3",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 78,
+		.accel_max_scale = 32000 << 16,
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		/* pulse sync not supported */
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode) - 1,
+		.has_burst32 = true,
+		.adis_data = ADIS16475_DATA(16505, &adis1650x_timeouts),
+	},
+	[ADIS16507_1] = {
+		.name = "adis16507-1",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(160 << 16),
+		.accel_max_val = 392,
+		.accel_max_scale = 32000 << 16,
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		/* pulse sync not supported */
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode) - 1,
+		.has_burst32 = true,
+		.adis_data = ADIS16475_DATA(16507, &adis1650x_timeouts),
+	},
+	[ADIS16507_2] = {
+		.name = "adis16507-2",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(40 << 16),
+		.accel_max_val = 392,
+		.accel_max_scale = 32000 << 16,
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		/* pulse sync not supported */
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode) - 1,
+		.has_burst32 = true,
+		.adis_data = ADIS16475_DATA(16507, &adis1650x_timeouts),
+	},
+	[ADIS16507_3] = {
+		.name = "adis16507-3",
+		.num_channels = ARRAY_SIZE(adis16475_channels),
+		.channels = adis16475_channels,
+		.gyro_max_val = 1,
+		.gyro_max_scale = IIO_RAD_TO_DEGREE(10 << 16),
+		.accel_max_val = 392,
+		.accel_max_scale = 32000 << 16,
+		.temp_scale = 100,
+		.int_clk = 2000,
+		.max_dec = 1999,
+		.sync = adis16475_sync_mode,
+		/* pulse sync not supported */
+		.num_sync = ARRAY_SIZE(adis16475_sync_mode) - 1,
+		.has_burst32 = true,
+		.adis_data = ADIS16475_DATA(16507, &adis1650x_timeouts),
+	},
+};
+
+static const struct iio_info adis16475_info = {
+	.read_raw = &adis16475_read_raw,
+	.write_raw = &adis16475_write_raw,
+	.update_scan_mode = adis_update_scan_mode,
+	.debugfs_reg_access = adis_debugfs_reg_access,
+};
+
+static struct adis_burst adis16475_burst = {
+	.en = true,
+	.reg_cmd = ADIS16475_REG_GLOB_CMD,
+	/*
+	 * adis_update_scan_mode_burst() sets the burst length in respect with
+	 * the number of channels and allocates 16 bits for each. However,
+	 * adis1647x devices also need space for DIAG_STAT, DATA_CNTR or
+	 * TIME_STAMP (depending on the clock mode but for us these bytes are
+	 * don't care...) and CRC.
+	 */
+	.extra_len = 3 * sizeof(u16),
+	.burst_max_len = ADIS16475_BURST32_MAX_DATA,
+};
+
+static bool adis16475_validate_crc(const u8 *buffer, u16 crc,
+				   const bool burst32)
+{
+	int i;
+	/* extra 6 elements for low gyro and accel */
+	const u16 sz = burst32 ? ADIS16475_BURST32_MAX_DATA :
+		ADIS16475_BURST_MAX_DATA;
+
+	for (i = 0; i < sz - 2; i++)
+		crc -= buffer[i];
+
+	return crc == 0;
+}
+
+static void adis16475_burst32_check(struct adis16475 *st)
+{
+	int ret;
+	struct adis *adis = &st->adis;
+
+	if (!st->info->has_burst32)
+		return;
+
+	if (st->lsb_flag && !st->burst32) {
+		const u16 en = ADIS16500_BURST32(1);
+
+		ret = __adis_update_bits(&st->adis, ADIS16475_REG_MSG_CTRL,
+					 ADIS16500_BURST32_MASK, en);
+		if (ret)
+			return;
+
+		st->burst32 = true;
+
+		/*
+		 * In 32-bit mode we need extra 2 bytes for all gyro
+		 * and accel channels.
+		 */
+		adis->burst_extra_len = 6 * sizeof(u16);
+		adis->xfer[1].len += 6 * sizeof(u16);
+		dev_dbg(&adis->spi->dev, "Enable burst32 mode, xfer:%d",
+			adis->xfer[1].len);
+
+	} else if (!st->lsb_flag && st->burst32) {
+		const u16 en = ADIS16500_BURST32(0);
+
+		ret = __adis_update_bits(&st->adis, ADIS16475_REG_MSG_CTRL,
+					 ADIS16500_BURST32_MASK, en);
+		if (ret)
+			return;
+
+		st->burst32 = false;
+
+		/* Remove the extra bits */
+		adis->burst_extra_len = 0;
+		adis->xfer[1].len -= 6 * sizeof(u16);
+		dev_dbg(&adis->spi->dev, "Disable burst32 mode, xfer:%d\n",
+			adis->xfer[1].len);
+	}
+}
+
+static irqreturn_t adis16475_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct adis16475 *st = iio_priv(indio_dev);
+	struct adis *adis = &st->adis;
+	int ret, bit, i = 0;
+	__be16 *buffer;
+	u16 crc;
+	bool valid;
+	/* offset until the first element after gyro and accel */
+	const u8 offset = st->burst32 ? 13 : 7;
+	const u32 cached_spi_speed_hz = adis->spi->max_speed_hz;
+
+	adis->spi->max_speed_hz = ADIS16475_BURST_MAX_SPEED;
+
+	ret = spi_sync(adis->spi, &adis->msg);
+	if (ret)
+		return ret;
+
+	adis->spi->max_speed_hz = cached_spi_speed_hz;
+	buffer = adis->buffer;
+
+	crc = be16_to_cpu(buffer[offset + 2]);
+	valid = adis16475_validate_crc(adis->buffer, crc, st->burst32);
+	if (!valid) {
+		dev_err(&adis->spi->dev, "Invalid crc\n");
+		goto check_burst32;
+	}
+
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->masklength) {
+		/*
+		 * When burst mode is used, system flags is the first data
+		 * channel in the sequence, but the scan index is 7.
+		 */
+		switch (bit) {
+		case ADIS16475_SCAN_TEMP:
+			st->data[i++] = buffer[offset];
+			break;
+		case ADIS16475_SCAN_GYRO_X ... ADIS16475_SCAN_ACCEL_Z:
+			/*
+			 * The first 2 bytes on the received data are the
+			 * DIAG_STAT reg, hence the +1 offset here...
+			 */
+			if (st->burst32) {
+				/* upper 16 */
+				st->data[i++] = buffer[bit * 2 + 2];
+				/* lower 16 */
+				st->data[i++] = buffer[bit * 2 + 1];
+			} else {
+				st->data[i++] = buffer[bit + 1];
+				/*
+				 * Don't bother in doing the manual read if the
+				 * device supports burst32. burst32 will be
+				 * enabled in the next call to
+				 * adis16475_burst32_check()...
+				 */
+				if (st->lsb_flag && !st->info->has_burst32) {
+					u16 val = 0;
+					const u32 reg = ADIS16475_REG_X_GYRO_L +
+						bit * 4;
+
+					adis_read_reg_16(adis, reg, &val);
+					st->data[i++] = cpu_to_be16(val);
+				} else {
+					/* lower not used */
+					st->data[i++] = 0;
+				}
+			}
+			break;
+		}
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, st->data, pf->timestamp);
+check_burst32:
+	/*
+	 * We only check the burst mode at the end of the current capture since
+	 * it takes a full data ready cycle for the device to update the burst
+	 * array.
+	 */
+	adis16475_burst32_check(st);
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static void adis16475_disable_clk(void *data)
+{
+	clk_disable_unprepare((struct clk *)data);
+}
+
+static int adis16475_config_sync_mode(struct adis16475 *st)
+{
+	int ret;
+	struct device *dev = &st->adis.spi->dev;
+	const struct adis16475_sync *sync;
+	u32 sync_mode;
+
+	/* default to internal clk */
+	st->clk_freq = st->info->int_clk * 1000;
+
+	ret = device_property_read_u32(dev, "adi,sync-mode", &sync_mode);
+	if (ret)
+		return 0;
+
+	if (sync_mode >= st->info->num_sync) {
+		dev_err(dev, "Invalid sync mode: %u for %s\n", sync_mode,
+			st->info->name);
+		return -EINVAL;
+	}
+
+	sync = &st->info->sync[sync_mode];
+
+	/* All the other modes require external input signal */
+	if (sync->sync_mode != ADIS16475_SYNC_OUTPUT) {
+		struct clk *clk = devm_clk_get(dev, NULL);
+
+		if (IS_ERR(clk))
+			return PTR_ERR(clk);
+
+		ret = clk_prepare_enable(clk);
+		if (ret)
+			return ret;
+
+		ret = devm_add_action_or_reset(dev, adis16475_disable_clk, clk);
+		if (ret)
+			return ret;
+
+		st->clk_freq = clk_get_rate(clk);
+		if (st->clk_freq < sync->min_rate ||
+		    st->clk_freq > sync->max_rate) {
+			dev_err(dev,
+				"Clk rate:%u not in a valid range:[%u %u]\n",
+				st->clk_freq, sync->min_rate, sync->max_rate);
+			return -EINVAL;
+		}
+
+		if (sync->sync_mode == ADIS16475_SYNC_SCALED) {
+			u16 up_scale;
+			u32 scaled_out_freq = 0;
+			/*
+			 * If we are in scaled mode, we must have an up_scale.
+			 * In scaled mode the allowable input clock range is
+			 * 1 Hz to 128 Hz, and the allowable output range is
+			 * 1900 to 2100 Hz. Hence, a scale must be given to
+			 * get the allowable output.
+			 */
+			ret = device_property_read_u32(dev,
+						       "adi,scaled-output-hz",
+						       &scaled_out_freq);
+			if (ret) {
+				dev_err(dev, "adi,scaled-output-hz must be given when in scaled sync mode");
+				return -EINVAL;
+			} else if (scaled_out_freq < 1900 ||
+				   scaled_out_freq > 2100) {
+				dev_err(dev, "Invalid value: %u for adi,scaled-output-hz",
+					scaled_out_freq);
+				return -EINVAL;
+			}
+
+			up_scale = DIV_ROUND_CLOSEST(scaled_out_freq,
+						     st->clk_freq);
+
+			ret = __adis_write_reg_16(&st->adis,
+						  ADIS16475_REG_UP_SCALE,
+						  up_scale);
+			if (ret)
+				return ret;
+
+			st->clk_freq = scaled_out_freq;
+		}
+
+		st->clk_freq *= 1000;
+	}
+	/*
+	 * Keep in mind that the mask for the clk modes in adis1650*
+	 * chips is different (1100 instead of 11100). However, we
+	 * are not configuring BIT(4) in these chips and the default
+	 * value is 0, so we are fine in doing the below operations.
+	 * I'm keeping this for simplicity and avoiding extra variables
+	 * in chip_info.
+	 */
+	ret = __adis_update_bits(&st->adis, ADIS16475_REG_MSG_CTRL,
+				 ADIS16475_SYNC_MODE_MASK, sync->sync_mode);
+	if (ret)
+		return ret;
+
+	usleep_range(250, 260);
+
+	return 0;
+}
+
+static int adis16475_config_irq_pin(struct adis16475 *st)
+{
+	int ret;
+	struct irq_data *desc;
+	u32 irq_type;
+	u16 val = 0;
+	u8 polarity;
+	struct spi_device *spi = st->adis.spi;
+
+	desc = irq_get_irq_data(spi->irq);
+	if (!desc) {
+		dev_err(&spi->dev, "Could not find IRQ %d\n", spi->irq);
+		return -EINVAL;
+	}
+	/*
+	 * It is possible to configure the data ready polarity. Furthermore, we
+	 * need to update the adis struct if we want data ready as active low.
+	 */
+	irq_type = irqd_get_trigger_type(desc);
+	if (irq_type == IRQ_TYPE_EDGE_RISING) {
+		polarity = 1;
+		st->adis.irq_flag = IRQF_TRIGGER_RISING;
+	} else if (irq_type == IRQ_TYPE_EDGE_FALLING) {
+		polarity = 0;
+		st->adis.irq_flag = IRQF_TRIGGER_FALLING;
+	} else {
+		dev_err(&spi->dev, "Invalid interrupt type 0x%x specified\n",
+			irq_type);
+		return -EINVAL;
+	}
+
+	val = ADIS16475_MSG_CTRL_DR_POL(polarity);
+	ret = __adis_update_bits(&st->adis, ADIS16475_REG_MSG_CTRL,
+				 ADIS16475_MSG_CTRL_DR_POL_MASK, val);
+	if (ret)
+		return ret;
+	/*
+	 * There is a delay writing to any bits written to the MSC_CTRL
+	 * register. It should not be bigger than 200us, so 250 should be more
+	 * than enough!
+	 */
+	usleep_range(250, 260);
+
+	return 0;
+}
+
+static const struct of_device_id adis16475_of_match[] = {
+	{ .compatible = "adi,adis16470",
+		.data = &adis16475_chip_info[ADIS16470] },
+	{ .compatible = "adi,adis16475-1",
+		.data = &adis16475_chip_info[ADIS16475_1] },
+	{ .compatible = "adi,adis16475-2",
+		.data = &adis16475_chip_info[ADIS16475_2] },
+	{ .compatible = "adi,adis16475-3",
+		.data = &adis16475_chip_info[ADIS16475_3] },
+	{ .compatible = "adi,adis16477-1",
+		.data = &adis16475_chip_info[ADIS16477_1] },
+	{ .compatible = "adi,adis16477-2",
+		.data = &adis16475_chip_info[ADIS16477_2] },
+	{ .compatible = "adi,adis16477-3",
+		.data = &adis16475_chip_info[ADIS16477_3] },
+	{ .compatible = "adi,adis16465-1",
+		.data = &adis16475_chip_info[ADIS16465_1] },
+	{ .compatible = "adi,adis16465-2",
+		.data = &adis16475_chip_info[ADIS16465_2] },
+	{ .compatible = "adi,adis16465-3",
+		.data = &adis16475_chip_info[ADIS16465_3] },
+	{ .compatible = "adi,adis16467-1",
+		.data = &adis16475_chip_info[ADIS16467_1] },
+	{ .compatible = "adi,adis16467-2",
+		.data = &adis16475_chip_info[ADIS16467_2] },
+	{ .compatible = "adi,adis16467-3",
+		.data = &adis16475_chip_info[ADIS16467_3] },
+	{ .compatible = "adi,adis16500",
+		.data = &adis16475_chip_info[ADIS16500] },
+	{ .compatible = "adi,adis16505-1",
+		.data = &adis16475_chip_info[ADIS16505_1] },
+	{ .compatible = "adi,adis16505-2",
+		.data = &adis16475_chip_info[ADIS16505_2] },
+	{ .compatible = "adi,adis16505-3",
+		.data = &adis16475_chip_info[ADIS16505_3] },
+	{ .compatible = "adi,adis16507-1",
+		.data = &adis16475_chip_info[ADIS16507_1] },
+	{ .compatible = "adi,adis16507-2",
+		.data = &adis16475_chip_info[ADIS16507_2] },
+	{ .compatible = "adi,adis16507-3",
+		.data = &adis16475_chip_info[ADIS16507_3] },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, adis16475_of_match);
+
+static int adis16475_probe(struct spi_device *spi)
+{
+	struct iio_dev *indio_dev;
+	struct adis16475 *st;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	spi_set_drvdata(spi, indio_dev);
+	st->adis.burst = &adis16475_burst;
+
+	st->info = device_get_match_data(&spi->dev);
+	if (!st->info)
+		return -EINVAL;
+
+	ret = adis_init(&st->adis, indio_dev, spi, &st->info->adis_data);
+	if (ret)
+		return ret;
+
+	indio_dev->dev.parent = &spi->dev;
+	indio_dev->name = st->info->name;
+	indio_dev->channels = st->info->channels;
+	indio_dev->num_channels = st->info->num_channels;
+	indio_dev->info = &adis16475_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	ret = __adis_initial_startup(&st->adis);
+	if (ret)
+		return ret;
+
+	ret = adis16475_config_irq_pin(st);
+	if (ret)
+		return ret;
+
+	ret = adis16475_config_sync_mode(st);
+	if (ret)
+		return ret;
+
+	ret = devm_adis_setup_buffer_and_trigger(&st->adis, indio_dev,
+						 adis16475_trigger_handler);
+	if (ret)
+		return ret;
+
+	adis16475_enable_irq(&st->adis, false);
+
+	ret = devm_iio_device_register(&spi->dev, indio_dev);
+	if (ret)
+		return ret;
+
+	adis16475_debugfs_init(indio_dev);
+
+	return 0;
+}
+
+static struct spi_driver adis16475_driver = {
+	.driver = {
+		.name = "adis16475",
+		.of_match_table = adis16475_of_match,
+	},
+	.probe = adis16475_probe,
+};
+module_spi_driver(adis16475_driver);
+
+MODULE_AUTHOR("Nuno Sa <nuno.sa@analog.com>");
+MODULE_DESCRIPTION("Analog Devices ADIS16475 IMU driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c
index cfae0e4476e7..6a471eee110e 100644
--- a/drivers/iio/imu/adis16480.c
+++ b/drivers/iio/imu/adis16480.c
@@ -284,22 +284,18 @@ DEFINE_DEBUGFS_ATTRIBUTE(adis16480_flash_count_fops,
 static int adis16480_debugfs_init(struct iio_dev *indio_dev)
 {
 	struct adis16480 *adis16480 = iio_priv(indio_dev);
+	struct dentry *d = iio_get_debugfs_dentry(indio_dev);
 
 	debugfs_create_file_unsafe("firmware_revision", 0400,
-		indio_dev->debugfs_dentry, adis16480,
-		&adis16480_firmware_revision_fops);
+		d, adis16480, &adis16480_firmware_revision_fops);
 	debugfs_create_file_unsafe("firmware_date", 0400,
-		indio_dev->debugfs_dentry, adis16480,
-		&adis16480_firmware_date_fops);
+		d, adis16480, &adis16480_firmware_date_fops);
 	debugfs_create_file_unsafe("serial_number", 0400,
-		indio_dev->debugfs_dentry, adis16480,
-		&adis16480_serial_number_fops);
+		d, adis16480, &adis16480_serial_number_fops);
 	debugfs_create_file_unsafe("product_id", 0400,
-		indio_dev->debugfs_dentry, adis16480,
-		&adis16480_product_id_fops);
+		d, adis16480, &adis16480_product_id_fops);
 	debugfs_create_file_unsafe("flash_count", 0400,
-		indio_dev->debugfs_dentry, adis16480,
-		&adis16480_flash_count_fops);
+		d, adis16480, &adis16480_flash_count_fops);
 
 	return 0;
 }
diff --git a/drivers/iio/imu/adis_buffer.c b/drivers/iio/imu/adis_buffer.c
index 04e5e2a0fd6b..5b4225ee09b9 100644
--- a/drivers/iio/imu/adis_buffer.c
+++ b/drivers/iio/imu/adis_buffer.c
@@ -23,25 +23,30 @@ static int adis_update_scan_mode_burst(struct iio_dev *indio_dev,
 	const unsigned long *scan_mask)
 {
 	struct adis *adis = iio_device_get_drvdata(indio_dev);
-	unsigned int burst_length;
+	unsigned int burst_length, burst_max_length;
 	u8 *tx;
 
 	/* All but the timestamp channel */
 	burst_length = (indio_dev->num_channels - 1) * sizeof(u16);
-	burst_length += adis->burst->extra_len;
+	burst_length += adis->burst->extra_len + adis->burst_extra_len;
+
+	if (adis->burst->burst_max_len)
+		burst_max_length = adis->burst->burst_max_len;
+	else
+		burst_max_length = burst_length;
 
 	adis->xfer = kcalloc(2, sizeof(*adis->xfer), GFP_KERNEL);
 	if (!adis->xfer)
 		return -ENOMEM;
 
-	adis->buffer = kzalloc(burst_length + sizeof(u16), GFP_KERNEL);
+	adis->buffer = kzalloc(burst_max_length + sizeof(u16), GFP_KERNEL);
 	if (!adis->buffer) {
 		kfree(adis->xfer);
 		adis->xfer = NULL;
 		return -ENOMEM;
 	}
 
-	tx = adis->buffer + burst_length;
+	tx = adis->buffer + burst_max_length;
 	tx[0] = ADIS_READ_REG(adis->burst->reg_cmd);
 	tx[1] = 0;
 
@@ -156,6 +161,14 @@ static irqreturn_t adis_trigger_handler(int irq, void *p)
 	return IRQ_HANDLED;
 }
 
+static void adis_buffer_cleanup(void *arg)
+{
+	struct adis *adis = arg;
+
+	kfree(adis->buffer);
+	kfree(adis->xfer);
+}
+
 /**
  * adis_setup_buffer_and_trigger() - Sets up buffer and trigger for the adis device
  * @adis: The adis device.
@@ -199,6 +212,43 @@ error_buffer_cleanup:
 EXPORT_SYMBOL_GPL(adis_setup_buffer_and_trigger);
 
 /**
+ * devm_adis_setup_buffer_and_trigger() - Sets up buffer and trigger for
+ *					  the managed adis device
+ * @adis: The adis device
+ * @indio_dev: The IIO device
+ * @trigger_handler: Optional trigger handler, may be NULL.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ *
+ * This function perfoms exactly the same as adis_setup_buffer_and_trigger()
+ */
+int
+devm_adis_setup_buffer_and_trigger(struct adis *adis, struct iio_dev *indio_dev,
+				   irq_handler_t trigger_handler)
+{
+	int ret;
+
+	if (!trigger_handler)
+		trigger_handler = adis_trigger_handler;
+
+	ret = devm_iio_triggered_buffer_setup(&adis->spi->dev, indio_dev,
+					      &iio_pollfunc_store_time,
+					      trigger_handler, NULL);
+	if (ret)
+		return ret;
+
+	if (adis->spi->irq) {
+		ret = devm_adis_probe_trigger(adis, indio_dev);
+		if (ret)
+			return ret;
+	}
+
+	return devm_add_action_or_reset(&adis->spi->dev, adis_buffer_cleanup,
+					adis);
+}
+EXPORT_SYMBOL_GPL(devm_adis_setup_buffer_and_trigger);
+
+/**
  * adis_cleanup_buffer_and_trigger() - Free buffer and trigger resources
  * @adis: The adis device.
  * @indio_dev: The IIO device.
diff --git a/drivers/iio/imu/adis_trigger.c b/drivers/iio/imu/adis_trigger.c
index 8b9cd02c0f9f..8afe71947c00 100644
--- a/drivers/iio/imu/adis_trigger.c
+++ b/drivers/iio/imu/adis_trigger.c
@@ -27,6 +27,34 @@ static const struct iio_trigger_ops adis_trigger_ops = {
 	.set_trigger_state = &adis_data_rdy_trigger_set_state,
 };
 
+static void adis_trigger_setup(struct adis *adis)
+{
+	adis->trig->dev.parent = &adis->spi->dev;
+	adis->trig->ops = &adis_trigger_ops;
+	iio_trigger_set_drvdata(adis->trig, adis);
+}
+
+static int adis_validate_irq_flag(struct adis *adis)
+{
+	/*
+	 * Typically this devices have data ready either on the rising edge or
+	 * on the falling edge of the data ready pin. This checks enforces that
+	 * one of those is set in the drivers... It defaults to
+	 * IRQF_TRIGGER_RISING for backward compatibility wiht devices that
+	 * don't support changing the pin polarity.
+	 */
+	if (!adis->irq_flag) {
+		adis->irq_flag = IRQF_TRIGGER_RISING;
+		return 0;
+	} else if (adis->irq_flag != IRQF_TRIGGER_RISING &&
+		   adis->irq_flag != IRQF_TRIGGER_FALLING) {
+		dev_err(&adis->spi->dev, "Invalid IRQ mask: %08lx\n",
+			adis->irq_flag);
+		return -EINVAL;
+	}
+
+	return 0;
+}
 /**
  * adis_probe_trigger() - Sets up trigger for a adis device
  * @adis: The adis device
@@ -45,13 +73,15 @@ int adis_probe_trigger(struct adis *adis, struct iio_dev *indio_dev)
 	if (adis->trig == NULL)
 		return -ENOMEM;
 
-	adis->trig->dev.parent = &adis->spi->dev;
-	adis->trig->ops = &adis_trigger_ops;
-	iio_trigger_set_drvdata(adis->trig, adis);
+	adis_trigger_setup(adis);
+
+	ret = adis_validate_irq_flag(adis);
+	if (ret)
+		return ret;
 
 	ret = request_irq(adis->spi->irq,
 			  &iio_trigger_generic_data_rdy_poll,
-			  IRQF_TRIGGER_RISING,
+			  adis->irq_flag,
 			  indio_dev->name,
 			  adis->trig);
 	if (ret)
@@ -74,6 +104,40 @@ error_free_trig:
 EXPORT_SYMBOL_GPL(adis_probe_trigger);
 
 /**
+ * devm_adis_probe_trigger() - Sets up trigger for a managed adis device
+ * @adis: The adis device
+ * @indio_dev: The IIO device
+ *
+ * Returns 0 on success or a negative error code
+ */
+int devm_adis_probe_trigger(struct adis *adis, struct iio_dev *indio_dev)
+{
+	int ret;
+
+	adis->trig = devm_iio_trigger_alloc(&adis->spi->dev, "%s-dev%d",
+					    indio_dev->name, indio_dev->id);
+	if (!adis->trig)
+		return -ENOMEM;
+
+	adis_trigger_setup(adis);
+
+	ret = adis_validate_irq_flag(adis);
+	if (ret)
+		return ret;
+
+	ret = devm_request_irq(&adis->spi->dev, adis->spi->irq,
+			       &iio_trigger_generic_data_rdy_poll,
+			       adis->irq_flag,
+			       indio_dev->name,
+			       adis->trig);
+	if (ret)
+		return ret;
+
+	return devm_iio_trigger_register(&adis->spi->dev, adis->trig);
+}
+EXPORT_SYMBOL_GPL(devm_adis_probe_trigger);
+
+/**
  * adis_remove_trigger() - Remove trigger for a adis devices
  * @adis: The adis device
  *
diff --git a/drivers/iio/imu/bmi160/bmi160_i2c.c b/drivers/iio/imu/bmi160/bmi160_i2c.c
index e36f5e82d400..26398614eddf 100644
--- a/drivers/iio/imu/bmi160/bmi160_i2c.c
+++ b/drivers/iio/imu/bmi160/bmi160_i2c.c
@@ -24,8 +24,8 @@ static int bmi160_i2c_probe(struct i2c_client *client,
 
 	regmap = devm_regmap_init_i2c(client, &bmi160_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&client->dev, "Failed to register i2c regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 
diff --git a/drivers/iio/imu/bmi160/bmi160_spi.c b/drivers/iio/imu/bmi160/bmi160_spi.c
index c19e3df35559..61389b41c6d9 100644
--- a/drivers/iio/imu/bmi160/bmi160_spi.c
+++ b/drivers/iio/imu/bmi160/bmi160_spi.c
@@ -20,8 +20,8 @@ static int bmi160_spi_probe(struct spi_device *spi)
 
 	regmap = devm_regmap_init_spi(spi, &bmi160_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&spi->dev, "Failed to register spi regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 	return bmi160_core_probe(&spi->dev, regmap, id->name, true);
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c
index 2f8560ba4572..c27d06035c8b 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c
@@ -135,6 +135,7 @@ int inv_mpu_acpi_create_mux_client(struct i2c_client *client)
 	st->mux_client = NULL;
 	if (ACPI_HANDLE(&client->dev)) {
 		struct i2c_board_info info;
+		struct i2c_client *mux_client;
 		struct acpi_device *adev;
 		int ret = -1;
 
@@ -172,9 +173,10 @@ int inv_mpu_acpi_create_mux_client(struct i2c_client *client)
 			} else
 				return 0; /* no secondary addr, which is OK */
 		}
-		st->mux_client = i2c_new_device(st->muxc->adapter[0], &info);
-		if (!st->mux_client)
-			return -ENODEV;
+		mux_client = i2c_new_client_device(st->muxc->adapter[0], &info);
+		if (IS_ERR(mux_client))
+			return PTR_ERR(mux_client);
+		st->mux_client = mux_client;
 	}
 
 	return 0;
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
index 0b8d2f7a0165..4d604fe842e5 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
@@ -526,7 +526,7 @@ static int inv_mpu6050_sensor_set(struct inv_mpu6050_state  *st, int reg,
 	__be16 d = cpu_to_be16(val);
 
 	ind = (axis - IIO_MOD_X) * 2;
-	result = regmap_bulk_write(st->map, reg + ind, (u8 *)&d, 2);
+	result = regmap_bulk_write(st->map, reg + ind, &d, sizeof(d));
 	if (result)
 		return -EINVAL;
 
@@ -540,7 +540,7 @@ static int inv_mpu6050_sensor_show(struct inv_mpu6050_state  *st, int reg,
 	__be16 d;
 
 	ind = (axis - IIO_MOD_X) * 2;
-	result = regmap_bulk_read(st->map, reg + ind, (u8 *)&d, 2);
+	result = regmap_bulk_read(st->map, reg + ind, &d, sizeof(d));
 	if (result)
 		return -EINVAL;
 	*val = (short)be16_to_cpup(&d);
@@ -1248,12 +1248,31 @@ static const struct attribute_group inv_attribute_group = {
 	.attrs = inv_attributes
 };
 
+static int inv_mpu6050_reg_access(struct iio_dev *indio_dev,
+				  unsigned int reg,
+				  unsigned int writeval,
+				  unsigned int *readval)
+{
+	struct inv_mpu6050_state *st = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&st->lock);
+	if (readval)
+		ret = regmap_read(st->map, reg, readval);
+	else
+		ret = regmap_write(st->map, reg, writeval);
+	mutex_unlock(&st->lock);
+
+	return ret;
+}
+
 static const struct iio_info mpu_info = {
 	.read_raw = &inv_mpu6050_read_raw,
 	.write_raw = &inv_mpu6050_write_raw,
 	.write_raw_get_fmt = &inv_write_raw_get_fmt,
 	.attrs = &inv_attribute_group,
 	.validate_trigger = inv_mpu6050_validate_trigger,
+	.debugfs_reg_access = &inv_mpu6050_reg_access,
 };
 
 /**
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
index 6993d3b87bb0..28cfae1e61cf 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
@@ -122,8 +122,8 @@ static int inv_mpu_probe(struct i2c_client *client,
 
 	regmap = devm_regmap_init_i2c(client, &inv_mpu_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&client->dev, "Failed to register i2c regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
index 673b198e6368..6f968ce687e1 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_spi.c
@@ -53,8 +53,8 @@ static int inv_mpu_probe(struct spi_device *spi)
 
 	regmap = devm_regmap_init_spi(spi, &inv_mpu_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&spi->dev, "Failed to register spi regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
index 41cb20cb3809..b56df409ed0f 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
@@ -111,7 +111,7 @@ struct st_lsm6dsx_odr {
 	u8 val;
 };
 
-#define ST_LSM6DSX_ODR_LIST_SIZE	6
+#define ST_LSM6DSX_ODR_LIST_SIZE	8
 struct st_lsm6dsx_odr_table_entry {
 	struct st_lsm6dsx_reg reg;
 
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
index 4426524b59f2..0b776cb91928 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
@@ -27,7 +27,8 @@
  *   - FIFO size: 4KB
  *
  * - LSM6DSO/LSM6DSOX/ASM330LHH/LSM6DSR/ISM330DHCX:
- *   - Accelerometer/Gyroscope supported ODR [Hz]: 13, 26, 52, 104, 208, 416
+ *   - Accelerometer/Gyroscope supported ODR [Hz]: 13, 26, 52, 104, 208, 416,
+ *     833
  *   - Accelerometer supported full-scale [g]: +-2/+-4/+-8/+-16
  *   - Gyroscope supported full-scale [dps]: +-125/+-245/+-500/+-1000/+-2000
  *   - FIFO size: 3KB
@@ -791,7 +792,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.odr_avl[3] = { 104000, 0x04 },
 				.odr_avl[4] = { 208000, 0x05 },
 				.odr_avl[5] = { 416000, 0x06 },
-				.odr_len = 6,
+				.odr_avl[6] = { 833000, 0x07 },
+				.odr_len = 7,
 			},
 			[ST_LSM6DSX_ID_GYRO] = {
 				.reg = {
@@ -804,7 +806,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.odr_avl[3] = { 104000, 0x04 },
 				.odr_avl[4] = { 208000, 0x05 },
 				.odr_avl[5] = { 416000, 0x06 },
-				.odr_len = 6,
+				.odr_avl[6] = { 833000, 0x07 },
+				.odr_len = 7,
 			},
 		},
 		.fs_table = {
@@ -994,7 +997,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.odr_avl[3] = { 104000, 0x04 },
 				.odr_avl[4] = { 208000, 0x05 },
 				.odr_avl[5] = { 416000, 0x06 },
-				.odr_len = 6,
+				.odr_avl[6] = { 833000, 0x07 },
+				.odr_len = 7,
 			},
 			[ST_LSM6DSX_ID_GYRO] = {
 				.reg = {
@@ -1007,7 +1011,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.odr_avl[3] = { 104000, 0x04 },
 				.odr_avl[4] = { 208000, 0x05 },
 				.odr_avl[5] = { 416000, 0x06 },
-				.odr_len = 6,
+				.odr_avl[6] = { 833000, 0x07 },
+				.odr_len = 7,
 			},
 		},
 		.fs_table = {
@@ -1171,7 +1176,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.odr_avl[3] = { 104000, 0x04 },
 				.odr_avl[4] = { 208000, 0x05 },
 				.odr_avl[5] = { 416000, 0x06 },
-				.odr_len = 6,
+				.odr_avl[6] = { 833000, 0x07 },
+				.odr_len = 7,
 			},
 			[ST_LSM6DSX_ID_GYRO] = {
 				.reg = {
@@ -1184,7 +1190,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				.odr_avl[3] = { 104000, 0x04 },
 				.odr_avl[4] = { 208000, 0x05 },
 				.odr_avl[5] = { 416000, 0x06 },
-				.odr_len = 6,
+				.odr_avl[6] = { 833000, 0x07 },
+				.odr_len = 7,
 			},
 		},
 		.fs_table = {
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c
index 1cf98195f84d..c1f83fe0d8da 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c
@@ -88,6 +88,69 @@ static const struct st_lsm6dsx_ext_dev_settings st_lsm6dsx_ext_dev_table[] = {
 			.len = 6,
 		},
 	},
+	/* LIS3MDL */
+	{
+		.i2c_addr = { 0x1e },
+		.wai = {
+			.addr = 0x0f,
+			.val = 0x3d,
+		},
+		.id = ST_LSM6DSX_ID_MAGN,
+		.odr_table = {
+			.reg = {
+				.addr = 0x20,
+				.mask = GENMASK(4, 2),
+			},
+			.odr_avl[0] = {  1000, 0x0 },
+			.odr_avl[1] = {  2000, 0x1 },
+			.odr_avl[2] = {  3000, 0x2 },
+			.odr_avl[3] = {  5000, 0x3 },
+			.odr_avl[4] = { 10000, 0x4 },
+			.odr_avl[5] = { 20000, 0x5 },
+			.odr_avl[6] = { 40000, 0x6 },
+			.odr_avl[7] = { 80000, 0x7 },
+			.odr_len = 8,
+		},
+		.fs_table = {
+			.reg = {
+				.addr = 0x21,
+				.mask = GENMASK(6, 5),
+			},
+			.fs_avl[0] = {
+				.gain = 146,
+				.val = 0x00,
+			}, /* 4000 uG/LSB */
+			.fs_avl[1] = {
+				.gain = 292,
+				.val = 0x01,
+			}, /* 8000 uG/LSB */
+			.fs_avl[2] = {
+				.gain = 438,
+				.val = 0x02,
+			}, /* 12000 uG/LSB */
+			.fs_avl[3] = {
+				.gain = 584,
+				.val = 0x03,
+			}, /* 16000 uG/LSB */
+			.fs_len = 4,
+		},
+		.pwr_table = {
+			.reg = {
+				.addr = 0x22,
+				.mask = GENMASK(1, 0),
+			},
+			.off_val = 0x2,
+			.on_val = 0x0,
+		},
+		.bdu = {
+			.addr = 0x24,
+			.mask = BIT(6),
+		},
+		.out = {
+			.addr = 0x28,
+			.len = 6,
+		},
+	},
 };
 
 static void st_lsm6dsx_shub_wait_complete(struct st_lsm6dsx_hw *hw)
@@ -519,6 +582,36 @@ st_lsm6dsx_shub_read_raw(struct iio_dev *iio_dev,
 }
 
 static int
+st_lsm6dsx_shub_set_full_scale(struct st_lsm6dsx_sensor *sensor,
+			       u32 gain)
+{
+	const struct st_lsm6dsx_fs_table_entry *fs_table;
+	int i, err;
+
+	fs_table = &sensor->ext_info.settings->fs_table;
+	if (!fs_table->reg.addr)
+		return -ENOTSUPP;
+
+	for (i = 0; i < fs_table->fs_len; i++) {
+		if (fs_table->fs_avl[i].gain == gain)
+			break;
+	}
+
+	if (i == fs_table->fs_len)
+		return -EINVAL;
+
+	err = st_lsm6dsx_shub_write_with_mask(sensor, fs_table->reg.addr,
+					      fs_table->reg.mask,
+					      fs_table->fs_avl[i].val);
+	if (err < 0)
+		return err;
+
+	sensor->gain = gain;
+
+	return 0;
+}
+
+static int
 st_lsm6dsx_shub_write_raw(struct iio_dev *iio_dev,
 			  struct iio_chan_spec const *chan,
 			  int val, int val2, long mask)
@@ -554,6 +647,9 @@ st_lsm6dsx_shub_write_raw(struct iio_dev *iio_dev,
 		}
 		break;
 	}
+	case IIO_CHAN_INFO_SCALE:
+		err = st_lsm6dsx_shub_set_full_scale(sensor, val2);
+		break;
 	default:
 		err = -EINVAL;
 		break;
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index 4ada5592aa2b..9fa238c0a7d4 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -189,10 +189,12 @@ __poll_t iio_buffer_poll(struct file *filp,
  */
 void iio_buffer_wakeup_poll(struct iio_dev *indio_dev)
 {
-	if (!indio_dev->buffer)
+	struct iio_buffer *buffer = indio_dev->buffer;
+
+	if (!buffer)
 		return;
 
-	wake_up(&indio_dev->buffer->pollq);
+	wake_up(&buffer->pollq);
 }
 
 void iio_buffer_init(struct iio_buffer *buffer)
@@ -262,10 +264,11 @@ static ssize_t iio_scan_el_show(struct device *dev,
 {
 	int ret;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct iio_buffer *buffer = indio_dev->buffer;
 
 	/* Ensure ret is 0 or 1. */
 	ret = !!test_bit(to_iio_dev_attr(attr)->address,
-		       indio_dev->buffer->scan_mask);
+		       buffer->scan_mask);
 
 	return sprintf(buf, "%d\n", ret);
 }
@@ -316,8 +319,7 @@ static int iio_scan_mask_set(struct iio_dev *indio_dev,
 	const unsigned long *mask;
 	unsigned long *trialmask;
 
-	trialmask = kcalloc(BITS_TO_LONGS(indio_dev->masklength),
-			    sizeof(*trialmask), GFP_KERNEL);
+	trialmask = bitmap_zalloc(indio_dev->masklength, GFP_KERNEL);
 	if (trialmask == NULL)
 		return -ENOMEM;
 	if (!indio_dev->masklength) {
@@ -382,7 +384,7 @@ static ssize_t iio_scan_el_store(struct device *dev,
 	if (ret < 0)
 		return ret;
 	mutex_lock(&indio_dev->mlock);
-	if (iio_buffer_is_active(indio_dev->buffer)) {
+	if (iio_buffer_is_active(buffer)) {
 		ret = -EBUSY;
 		goto error_ret;
 	}
@@ -411,7 +413,9 @@ static ssize_t iio_scan_el_ts_show(struct device *dev,
 				   char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	return sprintf(buf, "%d\n", indio_dev->buffer->scan_timestamp);
+	struct iio_buffer *buffer = indio_dev->buffer;
+
+	return sprintf(buf, "%d\n", buffer->scan_timestamp);
 }
 
 static ssize_t iio_scan_el_ts_store(struct device *dev,
@@ -421,6 +425,7 @@ static ssize_t iio_scan_el_ts_store(struct device *dev,
 {
 	int ret;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct iio_buffer *buffer = indio_dev->buffer;
 	bool state;
 
 	ret = strtobool(buf, &state);
@@ -428,11 +433,11 @@ static ssize_t iio_scan_el_ts_store(struct device *dev,
 		return ret;
 
 	mutex_lock(&indio_dev->mlock);
-	if (iio_buffer_is_active(indio_dev->buffer)) {
+	if (iio_buffer_is_active(buffer)) {
 		ret = -EBUSY;
 		goto error_ret;
 	}
-	indio_dev->buffer->scan_timestamp = state;
+	buffer->scan_timestamp = state;
 error_ret:
 	mutex_unlock(&indio_dev->mlock);
 
@@ -440,10 +445,10 @@ error_ret:
 }
 
 static int iio_buffer_add_channel_sysfs(struct iio_dev *indio_dev,
+					struct iio_buffer *buffer,
 					const struct iio_chan_spec *chan)
 {
 	int ret, attrcount = 0;
-	struct iio_buffer *buffer = indio_dev->buffer;
 
 	ret = __iio_add_chan_devattr("index",
 				     chan,
@@ -519,7 +524,7 @@ static ssize_t iio_buffer_write_length(struct device *dev,
 		return len;
 
 	mutex_lock(&indio_dev->mlock);
-	if (iio_buffer_is_active(indio_dev->buffer)) {
+	if (iio_buffer_is_active(buffer)) {
 		ret = -EBUSY;
 	} else {
 		buffer->access->set_length(buffer, val);
@@ -540,7 +545,9 @@ static ssize_t iio_buffer_show_enable(struct device *dev,
 				      char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	return sprintf(buf, "%d\n", iio_buffer_is_active(indio_dev->buffer));
+	struct iio_buffer *buffer = indio_dev->buffer;
+
+	return sprintf(buf, "%d\n", iio_buffer_is_active(buffer));
 }
 
 static unsigned int iio_storage_bytes_for_si(struct iio_dev *indio_dev,
@@ -687,6 +694,13 @@ static int iio_verify_update(struct iio_dev *indio_dev,
 	bool scan_timestamp;
 	unsigned int modes;
 
+	if (insert_buffer &&
+	    bitmap_empty(insert_buffer->scan_mask, indio_dev->masklength)) {
+		dev_dbg(&indio_dev->dev,
+			"At least one scan element must be enabled first\n");
+		return -EINVAL;
+	}
+
 	memset(config, 0, sizeof(*config));
 	config->watermark = ~0;
 
@@ -913,6 +927,7 @@ static int iio_enable_buffers(struct iio_dev *indio_dev,
 	indio_dev->active_scan_mask = config->scan_mask;
 	indio_dev->scan_timestamp = config->scan_timestamp;
 	indio_dev->scan_bytes = config->scan_bytes;
+	indio_dev->currentmode = config->mode;
 
 	iio_update_demux(indio_dev);
 
@@ -948,8 +963,6 @@ static int iio_enable_buffers(struct iio_dev *indio_dev,
 			goto err_disable_buffers;
 	}
 
-	indio_dev->currentmode = config->mode;
-
 	if (indio_dev->setup_ops->postenable) {
 		ret = indio_dev->setup_ops->postenable(indio_dev);
 		if (ret) {
@@ -966,10 +979,10 @@ err_disable_buffers:
 					     buffer_list)
 		iio_buffer_disable(buffer, indio_dev);
 err_run_postdisable:
-	indio_dev->currentmode = INDIO_DIRECT_MODE;
 	if (indio_dev->setup_ops->postdisable)
 		indio_dev->setup_ops->postdisable(indio_dev);
 err_undo_config:
+	indio_dev->currentmode = INDIO_DIRECT_MODE;
 	indio_dev->active_scan_mask = NULL;
 
 	return ret;
@@ -1004,8 +1017,6 @@ static int iio_disable_buffers(struct iio_dev *indio_dev)
 			ret = ret2;
 	}
 
-	indio_dev->currentmode = INDIO_DIRECT_MODE;
-
 	if (indio_dev->setup_ops->postdisable) {
 		ret2 = indio_dev->setup_ops->postdisable(indio_dev);
 		if (ret2 && !ret)
@@ -1014,6 +1025,7 @@ static int iio_disable_buffers(struct iio_dev *indio_dev)
 
 	iio_free_scan_mask(indio_dev, indio_dev->active_scan_mask);
 	indio_dev->active_scan_mask = NULL;
+	indio_dev->currentmode = INDIO_DIRECT_MODE;
 
 	return ret;
 }
@@ -1123,6 +1135,7 @@ static ssize_t iio_buffer_store_enable(struct device *dev,
 	int ret;
 	bool requested_state;
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct iio_buffer *buffer = indio_dev->buffer;
 	bool inlist;
 
 	ret = strtobool(buf, &requested_state);
@@ -1132,17 +1145,15 @@ static ssize_t iio_buffer_store_enable(struct device *dev,
 	mutex_lock(&indio_dev->mlock);
 
 	/* Find out if it is in the list */
-	inlist = iio_buffer_is_active(indio_dev->buffer);
+	inlist = iio_buffer_is_active(buffer);
 	/* Already in desired state */
 	if (inlist == requested_state)
 		goto done;
 
 	if (requested_state)
-		ret = __iio_update_buffers(indio_dev,
-					 indio_dev->buffer, NULL);
+		ret = __iio_update_buffers(indio_dev, buffer, NULL);
 	else
-		ret = __iio_update_buffers(indio_dev,
-					 NULL, indio_dev->buffer);
+		ret = __iio_update_buffers(indio_dev, NULL, buffer);
 
 done:
 	mutex_unlock(&indio_dev->mlock);
@@ -1184,7 +1195,7 @@ static ssize_t iio_buffer_store_watermark(struct device *dev,
 		goto out;
 	}
 
-	if (iio_buffer_is_active(indio_dev->buffer)) {
+	if (iio_buffer_is_active(buffer)) {
 		ret = -EBUSY;
 		goto out;
 	}
@@ -1201,11 +1212,9 @@ static ssize_t iio_dma_show_data_available(struct device *dev,
 						char *buf)
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	size_t bytes;
-
-	bytes = iio_buffer_data_available(indio_dev->buffer);
+	struct iio_buffer *buffer = indio_dev->buffer;
 
-	return sprintf(buf, "%zu\n", bytes);
+	return sprintf(buf, "%zu\n", iio_buffer_data_available(buffer));
 }
 
 static DEVICE_ATTR(length, S_IRUGO | S_IWUSR, iio_buffer_read_length,
@@ -1233,7 +1242,7 @@ int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
 	struct iio_dev_attr *p;
 	struct attribute **attr;
 	struct iio_buffer *buffer = indio_dev->buffer;
-	int ret, i, attrn, attrcount, attrcount_orig = 0;
+	int ret, i, attrn, attrcount;
 	const struct iio_chan_spec *channels;
 
 	channels = indio_dev->channels;
@@ -1277,12 +1286,7 @@ int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
 
 	indio_dev->groups[indio_dev->groupcounter++] = &buffer->buffer_group;
 
-	if (buffer->scan_el_attrs != NULL) {
-		attr = buffer->scan_el_attrs->attrs;
-		while (*attr++ != NULL)
-			attrcount_orig++;
-	}
-	attrcount = attrcount_orig;
+	attrcount = 0;
 	INIT_LIST_HEAD(&buffer->scan_el_dev_attr_list);
 	channels = indio_dev->channels;
 	if (channels) {
@@ -1291,7 +1295,7 @@ int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
 			if (channels[i].scan_index < 0)
 				continue;
 
-			ret = iio_buffer_add_channel_sysfs(indio_dev,
+			ret = iio_buffer_add_channel_sysfs(indio_dev, buffer,
 							 &channels[i]);
 			if (ret < 0)
 				goto error_cleanup_dynamic;
@@ -1319,10 +1323,7 @@ int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
 		ret = -ENOMEM;
 		goto error_free_scan_mask;
 	}
-	if (buffer->scan_el_attrs)
-		memcpy(buffer->scan_el_group.attrs, buffer->scan_el_attrs,
-		       sizeof(buffer->scan_el_group.attrs[0])*attrcount_orig);
-	attrn = attrcount_orig;
+	attrn = 0;
 
 	list_for_each_entry(p, &buffer->scan_el_dev_attr_list, l)
 		buffer->scan_el_group.attrs[attrn++] = &p->dev_attr.attr;
@@ -1334,20 +1335,22 @@ error_free_scan_mask:
 	bitmap_free(buffer->scan_mask);
 error_cleanup_dynamic:
 	iio_free_chan_devattr_list(&buffer->scan_el_dev_attr_list);
-	kfree(indio_dev->buffer->buffer_group.attrs);
+	kfree(buffer->buffer_group.attrs);
 
 	return ret;
 }
 
 void iio_buffer_free_sysfs_and_mask(struct iio_dev *indio_dev)
 {
-	if (!indio_dev->buffer)
+	struct iio_buffer *buffer = indio_dev->buffer;
+
+	if (!buffer)
 		return;
 
-	bitmap_free(indio_dev->buffer->scan_mask);
-	kfree(indio_dev->buffer->buffer_group.attrs);
-	kfree(indio_dev->buffer->scan_el_group.attrs);
-	iio_free_chan_devattr_list(&indio_dev->buffer->scan_el_dev_attr_list);
+	bitmap_free(buffer->scan_mask);
+	kfree(buffer->buffer_group.attrs);
+	kfree(buffer->scan_el_group.attrs);
+	iio_free_chan_devattr_list(&buffer->scan_el_dev_attr_list);
 }
 
 /**
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index 24f7bbff4938..1527f01a44f1 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -572,46 +572,46 @@ static ssize_t __iio_format_value(char *buf, size_t len, unsigned int type,
 
 	switch (type) {
 	case IIO_VAL_INT:
-		return snprintf(buf, len, "%d", vals[0]);
+		return scnprintf(buf, len, "%d", vals[0]);
 	case IIO_VAL_INT_PLUS_MICRO_DB:
 		scale_db = true;
 		/* fall through */
 	case IIO_VAL_INT_PLUS_MICRO:
 		if (vals[1] < 0)
-			return snprintf(buf, len, "-%d.%06u%s", abs(vals[0]),
+			return scnprintf(buf, len, "-%d.%06u%s", abs(vals[0]),
 					-vals[1], scale_db ? " dB" : "");
 		else
-			return snprintf(buf, len, "%d.%06u%s", vals[0], vals[1],
+			return scnprintf(buf, len, "%d.%06u%s", vals[0], vals[1],
 					scale_db ? " dB" : "");
 	case IIO_VAL_INT_PLUS_NANO:
 		if (vals[1] < 0)
-			return snprintf(buf, len, "-%d.%09u", abs(vals[0]),
+			return scnprintf(buf, len, "-%d.%09u", abs(vals[0]),
 					-vals[1]);
 		else
-			return snprintf(buf, len, "%d.%09u", vals[0], vals[1]);
+			return scnprintf(buf, len, "%d.%09u", vals[0], vals[1]);
 	case IIO_VAL_FRACTIONAL:
 		tmp = div_s64((s64)vals[0] * 1000000000LL, vals[1]);
 		tmp1 = vals[1];
 		tmp0 = (int)div_s64_rem(tmp, 1000000000, &tmp1);
-		return snprintf(buf, len, "%d.%09u", tmp0, abs(tmp1));
+		return scnprintf(buf, len, "%d.%09u", tmp0, abs(tmp1));
 	case IIO_VAL_FRACTIONAL_LOG2:
 		tmp = shift_right((s64)vals[0] * 1000000000LL, vals[1]);
 		tmp0 = (int)div_s64_rem(tmp, 1000000000LL, &tmp1);
-		return snprintf(buf, len, "%d.%09u", tmp0, abs(tmp1));
+		return scnprintf(buf, len, "%d.%09u", tmp0, abs(tmp1));
 	case IIO_VAL_INT_MULTIPLE:
 	{
 		int i;
 		int l = 0;
 
 		for (i = 0; i < size; ++i) {
-			l += snprintf(&buf[l], len - l, "%d ", vals[i]);
+			l += scnprintf(&buf[l], len - l, "%d ", vals[i]);
 			if (l >= len)
 				break;
 		}
 		return l;
 	}
 	case IIO_VAL_CHAR:
-		return snprintf(buf, len, "%c", (char)vals[0]);
+		return scnprintf(buf, len, "%c", (char)vals[0]);
 	default:
 		return 0;
 	}
@@ -682,10 +682,10 @@ static ssize_t iio_format_avail_list(char *buf, const int *vals,
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
 			if (i < length - 1)
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						" ");
 			else
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						"\n");
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
@@ -698,10 +698,10 @@ static ssize_t iio_format_avail_list(char *buf, const int *vals,
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
 			if (i < length / 2 - 1)
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						" ");
 			else
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						"\n");
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
@@ -725,10 +725,10 @@ static ssize_t iio_format_avail_range(char *buf, const int *vals, int type)
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
 			if (i < 2)
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						" ");
 			else
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						"]\n");
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
@@ -741,10 +741,10 @@ static ssize_t iio_format_avail_range(char *buf, const int *vals, int type)
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
 			if (i < 2)
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						" ");
 			else
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						"]\n");
 			if (len >= PAGE_SIZE)
 				return -EFBIG;
@@ -1507,27 +1507,27 @@ struct iio_dev *iio_device_alloc(int sizeof_priv)
 	alloc_size += IIO_ALIGN - 1;
 
 	dev = kzalloc(alloc_size, GFP_KERNEL);
+	if (!dev)
+		return NULL;
 
-	if (dev) {
-		dev->dev.groups = dev->groups;
-		dev->dev.type = &iio_device_type;
-		dev->dev.bus = &iio_bus_type;
-		device_initialize(&dev->dev);
-		dev_set_drvdata(&dev->dev, (void *)dev);
-		mutex_init(&dev->mlock);
-		mutex_init(&dev->info_exist_lock);
-		INIT_LIST_HEAD(&dev->channel_attr_list);
-
-		dev->id = ida_simple_get(&iio_ida, 0, 0, GFP_KERNEL);
-		if (dev->id < 0) {
-			/* cannot use a dev_err as the name isn't available */
-			pr_err("failed to get device id\n");
-			kfree(dev);
-			return NULL;
-		}
-		dev_set_name(&dev->dev, "iio:device%d", dev->id);
-		INIT_LIST_HEAD(&dev->buffer_list);
+	dev->dev.groups = dev->groups;
+	dev->dev.type = &iio_device_type;
+	dev->dev.bus = &iio_bus_type;
+	device_initialize(&dev->dev);
+	dev_set_drvdata(&dev->dev, (void *)dev);
+	mutex_init(&dev->mlock);
+	mutex_init(&dev->info_exist_lock);
+	INIT_LIST_HEAD(&dev->channel_attr_list);
+
+	dev->id = ida_simple_get(&iio_ida, 0, 0, GFP_KERNEL);
+	if (dev->id < 0) {
+		/* cannot use a dev_err as the name isn't available */
+		pr_err("failed to get device id\n");
+		kfree(dev);
+		return NULL;
 	}
+	dev_set_name(&dev->dev, "iio:device%d", dev->id);
+	INIT_LIST_HEAD(&dev->buffer_list);
 
 	return dev;
 }
@@ -1549,17 +1549,6 @@ static void devm_iio_device_release(struct device *dev, void *res)
 	iio_device_free(*(struct iio_dev **)res);
 }
 
-int devm_iio_device_match(struct device *dev, void *res, void *data)
-{
-	struct iio_dev **r = res;
-	if (!r || !*r) {
-		WARN_ON(!r || !*r);
-		return 0;
-	}
-	return *r == data;
-}
-EXPORT_SYMBOL_GPL(devm_iio_device_match);
-
 /**
  * devm_iio_device_alloc - Resource-managed iio_device_alloc()
  * @dev:		Device to allocate iio_dev for
@@ -1568,9 +1557,6 @@ EXPORT_SYMBOL_GPL(devm_iio_device_match);
  * Managed iio_device_alloc. iio_dev allocated with this function is
  * automatically freed on driver detach.
  *
- * If an iio_dev allocated with this function needs to be freed separately,
- * devm_iio_device_free() must be used.
- *
  * RETURNS:
  * Pointer to allocated iio_dev on success, NULL on failure.
  */
@@ -1596,23 +1582,6 @@ struct iio_dev *devm_iio_device_alloc(struct device *dev, int sizeof_priv)
 EXPORT_SYMBOL_GPL(devm_iio_device_alloc);
 
 /**
- * devm_iio_device_free - Resource-managed iio_device_free()
- * @dev:		Device this iio_dev belongs to
- * @iio_dev:		the iio_dev associated with the device
- *
- * Free iio_dev allocated with devm_iio_device_alloc().
- */
-void devm_iio_device_free(struct device *dev, struct iio_dev *iio_dev)
-{
-	int rc;
-
-	rc = devres_release(dev, devm_iio_device_release,
-			    devm_iio_device_match, iio_dev);
-	WARN_ON(rc);
-}
-EXPORT_SYMBOL_GPL(devm_iio_device_free);
-
-/**
  * iio_chrdev_open() - chrdev file open for buffer access and ioctls
  * @inode:	Inode structure for identifying the device in the file system
  * @filp:	File structure for iio device used to keep and later access
@@ -1714,6 +1683,9 @@ int __iio_device_register(struct iio_dev *indio_dev, struct module *this_mod)
 {
 	int ret;
 
+	if (!indio_dev->info)
+		return -EINVAL;
+
 	indio_dev->driver_module = this_mod;
 	/* If the calling driver did not initialize of_node, do it here */
 	if (!indio_dev->dev.of_node && indio_dev->dev.parent)
@@ -1726,9 +1698,6 @@ int __iio_device_register(struct iio_dev *indio_dev, struct module *this_mod)
 	if (ret < 0)
 		return ret;
 
-	if (!indio_dev->info)
-		return -EINVAL;
-
 	/* configure elements for the chrdev */
 	indio_dev->dev.devt = MKDEV(MAJOR(iio_devt), indio_dev->id);
 
@@ -1834,23 +1803,6 @@ int __devm_iio_device_register(struct device *dev, struct iio_dev *indio_dev,
 EXPORT_SYMBOL_GPL(__devm_iio_device_register);
 
 /**
- * devm_iio_device_unregister - Resource-managed iio_device_unregister()
- * @dev:	Device this iio_dev belongs to
- * @indio_dev:	the iio_dev associated with the device
- *
- * Unregister iio_dev registered with devm_iio_device_register().
- */
-void devm_iio_device_unregister(struct device *dev, struct iio_dev *indio_dev)
-{
-	int rc;
-
-	rc = devres_release(dev, devm_iio_device_unreg,
-			    devm_iio_device_match, indio_dev);
-	WARN_ON(rc);
-}
-EXPORT_SYMBOL_GPL(devm_iio_device_unregister);
-
-/**
  * iio_device_claim_direct_mode - Keep device in direct mode
  * @indio_dev:	the iio_dev associated with the device
  *
diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c
index 3908a9a90035..53d1931f6be8 100644
--- a/drivers/iio/industrialio-trigger.c
+++ b/drivers/iio/industrialio-trigger.c
@@ -585,18 +585,6 @@ static void devm_iio_trigger_release(struct device *dev, void *res)
 	iio_trigger_free(*(struct iio_trigger **)res);
 }
 
-static int devm_iio_trigger_match(struct device *dev, void *res, void *data)
-{
-	struct iio_trigger **r = res;
-
-	if (!r || !*r) {
-		WARN_ON(!r || !*r);
-		return 0;
-	}
-
-	return *r == data;
-}
-
 /**
  * devm_iio_trigger_alloc - Resource-managed iio_trigger_alloc()
  * @dev:		Device to allocate iio_trigger for
@@ -608,9 +596,6 @@ static int devm_iio_trigger_match(struct device *dev, void *res, void *data)
  * Managed iio_trigger_alloc.  iio_trigger allocated with this function is
  * automatically freed on driver detach.
  *
- * If an iio_trigger allocated with this function needs to be freed separately,
- * devm_iio_trigger_free() must be used.
- *
  * RETURNS:
  * Pointer to allocated iio_trigger on success, NULL on failure.
  */
@@ -640,23 +625,6 @@ struct iio_trigger *devm_iio_trigger_alloc(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_iio_trigger_alloc);
 
-/**
- * devm_iio_trigger_free - Resource-managed iio_trigger_free()
- * @dev:		Device this iio_dev belongs to
- * @iio_trig:		the iio_trigger associated with the device
- *
- * Free iio_trigger allocated with devm_iio_trigger_alloc().
- */
-void devm_iio_trigger_free(struct device *dev, struct iio_trigger *iio_trig)
-{
-	int rc;
-
-	rc = devres_release(dev, devm_iio_trigger_release,
-			    devm_iio_trigger_match, iio_trig);
-	WARN_ON(rc);
-}
-EXPORT_SYMBOL_GPL(devm_iio_trigger_free);
-
 static void devm_iio_trigger_unreg(struct device *dev, void *res)
 {
 	iio_trigger_unregister(*(struct iio_trigger **)res);
@@ -673,9 +641,6 @@ static void devm_iio_trigger_unreg(struct device *dev, void *res)
  * calls iio_trigger_register() internally. Refer to that function for more
  * information.
  *
- * If an iio_trigger registered with this function needs to be unregistered
- * separately, devm_iio_trigger_unregister() must be used.
- *
  * RETURNS:
  * 0 on success, negative error number on failure.
  */
@@ -701,24 +666,6 @@ int __devm_iio_trigger_register(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(__devm_iio_trigger_register);
 
-/**
- * devm_iio_trigger_unregister - Resource-managed iio_trigger_unregister()
- * @dev:	device this iio_trigger belongs to
- * @trig_info:	the trigger associated with the device
- *
- * Unregister trigger registered with devm_iio_trigger_register().
- */
-void devm_iio_trigger_unregister(struct device *dev,
-				 struct iio_trigger *trig_info)
-{
-	int rc;
-
-	rc = devres_release(dev, devm_iio_trigger_unreg, devm_iio_trigger_match,
-			    trig_info);
-	WARN_ON(rc);
-}
-EXPORT_SYMBOL_GPL(devm_iio_trigger_unregister);
-
 bool iio_trigger_using_own(struct iio_dev *indio_dev)
 {
 	return indio_dev->trig->attached_own_device;
diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
index 5a8351c9a426..ede99e0d5371 100644
--- a/drivers/iio/inkern.c
+++ b/drivers/iio/inkern.c
@@ -360,18 +360,6 @@ static void devm_iio_channel_free(struct device *dev, void *res)
 	iio_channel_release(channel);
 }
 
-static int devm_iio_channel_match(struct device *dev, void *res, void *data)
-{
-	struct iio_channel **r = res;
-
-	if (!r || !*r) {
-		WARN_ON(!r || !*r);
-		return 0;
-	}
-
-	return *r == data;
-}
-
 struct iio_channel *devm_iio_channel_get(struct device *dev,
 					 const char *channel_name)
 {
@@ -394,13 +382,6 @@ struct iio_channel *devm_iio_channel_get(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_iio_channel_get);
 
-void devm_iio_channel_release(struct device *dev, struct iio_channel *channel)
-{
-	WARN_ON(devres_release(dev, devm_iio_channel_free,
-			       devm_iio_channel_match, channel));
-}
-EXPORT_SYMBOL_GPL(devm_iio_channel_release);
-
 struct iio_channel *iio_channel_get_all(struct device *dev)
 {
 	const char *name;
@@ -514,14 +495,6 @@ struct iio_channel *devm_iio_channel_get_all(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(devm_iio_channel_get_all);
 
-void devm_iio_channel_release_all(struct device *dev,
-				  struct iio_channel *channels)
-{
-	WARN_ON(devres_release(dev, devm_iio_channel_free_all,
-			       devm_iio_channel_match, channels));
-}
-EXPORT_SYMBOL_GPL(devm_iio_channel_release_all);
-
 static int iio_channel_read(struct iio_channel *chan, int *val, int *val2,
 	enum iio_chan_info_enum info)
 {
diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig
index b27719cefcf9..182bd18c4bb2 100644
--- a/drivers/iio/light/Kconfig
+++ b/drivers/iio/light/Kconfig
@@ -516,6 +516,8 @@ config US5182D
 
 config VCNL4000
 	tristate "VCNL4000/4010/4020/4200 combined ALS and proximity sensor"
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	depends on I2C
 	help
 	  Say Y here if you want to build a driver for the Vishay VCNL4000,
diff --git a/drivers/iio/light/bh1780.c b/drivers/iio/light/bh1780.c
index a8361006dcd9..03f2d8d123c4 100644
--- a/drivers/iio/light/bh1780.c
+++ b/drivers/iio/light/bh1780.c
@@ -13,7 +13,7 @@
 #include <linux/platform_device.h>
 #include <linux/delay.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/mod_devicetable.h>
 #include <linux/pm_runtime.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
@@ -273,13 +273,11 @@ static const struct i2c_device_id bh1780_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, bh1780_id);
 
-#ifdef CONFIG_OF
 static const struct of_device_id of_bh1780_match[] = {
 	{ .compatible = "rohm,bh1780gli", },
 	{},
 };
 MODULE_DEVICE_TABLE(of, of_bh1780_match);
-#endif
 
 static struct i2c_driver bh1780_driver = {
 	.probe		= bh1780_probe,
@@ -288,7 +286,7 @@ static struct i2c_driver bh1780_driver = {
 	.driver = {
 		.name = "bh1780",
 		.pm = &bh1780_dev_pm_ops,
-		.of_match_table = of_match_ptr(of_bh1780_match),
+		.of_match_table = of_bh1780_match,
 	},
 };
 
diff --git a/drivers/iio/light/cm32181.c b/drivers/iio/light/cm32181.c
index 5f4fb5674fa0..160eb3f99795 100644
--- a/drivers/iio/light/cm32181.c
+++ b/drivers/iio/light/cm32181.c
@@ -4,11 +4,13 @@
  * Author: Kevin Tsai <ktsai@capellamicro.com>
  */
 
+#include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/interrupt.h>
 #include <linux/regulator/consumer.h>
 #include <linux/iio/iio.h>
@@ -18,17 +20,24 @@
 
 /* Registers Address */
 #define CM32181_REG_ADDR_CMD		0x00
+#define CM32181_REG_ADDR_WH		0x01
+#define CM32181_REG_ADDR_WL		0x02
+#define CM32181_REG_ADDR_TEST		0x03
 #define CM32181_REG_ADDR_ALS		0x04
 #define CM32181_REG_ADDR_STATUS		0x06
 #define CM32181_REG_ADDR_ID		0x07
 
 /* Number of Configurable Registers */
-#define CM32181_CONF_REG_NUM		0x01
+#define CM32181_CONF_REG_NUM		4
 
 /* CMD register */
-#define CM32181_CMD_ALS_ENABLE		0x00
-#define CM32181_CMD_ALS_DISABLE		0x01
-#define CM32181_CMD_ALS_INT_EN		0x02
+#define CM32181_CMD_ALS_DISABLE		BIT(0)
+#define CM32181_CMD_ALS_INT_EN		BIT(1)
+#define CM32181_CMD_ALS_THRES_WINDOW	BIT(2)
+
+#define CM32181_CMD_ALS_PERS_SHIFT	4
+#define CM32181_CMD_ALS_PERS_MASK	(0x03 << CM32181_CMD_ALS_PERS_SHIFT)
+#define CM32181_CMD_ALS_PERS_DEFAULT	(0x01 << CM32181_CMD_ALS_PERS_SHIFT)
 
 #define CM32181_CMD_ALS_IT_SHIFT	6
 #define CM32181_CMD_ALS_IT_MASK		(0x0F << CM32181_CMD_ALS_IT_SHIFT)
@@ -38,27 +47,133 @@
 #define CM32181_CMD_ALS_SM_MASK		(0x03 << CM32181_CMD_ALS_SM_SHIFT)
 #define CM32181_CMD_ALS_SM_DEFAULT	(0x01 << CM32181_CMD_ALS_SM_SHIFT)
 
-#define CM32181_MLUX_PER_BIT		5	/* ALS_SM=01 IT=800ms */
-#define CM32181_MLUX_PER_BIT_BASE_IT	800000	/* Based on IT=800ms */
-#define	CM32181_CALIBSCALE_DEFAULT	1000
-#define CM32181_CALIBSCALE_RESOLUTION	1000
-#define MLUX_PER_LUX			1000
+#define CM32181_LUX_PER_BIT		500	/* ALS_SM=01 IT=800ms */
+#define CM32181_LUX_PER_BIT_RESOLUTION	100000
+#define CM32181_LUX_PER_BIT_BASE_IT	800000	/* Based on IT=800ms */
+#define CM32181_CALIBSCALE_DEFAULT	100000
+#define CM32181_CALIBSCALE_RESOLUTION	100000
 
-static const u8 cm32181_reg[CM32181_CONF_REG_NUM] = {
-	CM32181_REG_ADDR_CMD,
-};
+#define SMBUS_ALERT_RESPONSE_ADDRESS	0x0c
+
+/* CPM0 Index 0: device-id (3218 or 32181), 1: Unknown, 2: init_regs_bitmap */
+#define CPM0_REGS_BITMAP		2
+#define CPM0_HEADER_SIZE		3
 
-static const int als_it_bits[] = {12, 8, 0, 1, 2, 3};
-static const int als_it_value[] = {25000, 50000, 100000, 200000, 400000,
-	800000};
+/* CPM1 Index 0: lux_per_bit, 1: calibscale, 2: resolution (100000) */
+#define CPM1_LUX_PER_BIT		0
+#define CPM1_CALIBSCALE			1
+#define CPM1_SIZE			3
+
+/* CM3218 Family */
+static const int cm3218_als_it_bits[] = { 0, 1, 2, 3 };
+static const int cm3218_als_it_values[] = { 100000, 200000, 400000, 800000 };
+
+/* CM32181 Family */
+static const int cm32181_als_it_bits[] = { 12, 8, 0, 1, 2, 3 };
+static const int cm32181_als_it_values[] = {
+	25000, 50000, 100000, 200000, 400000, 800000
+};
 
 struct cm32181_chip {
 	struct i2c_client *client;
+	struct device *dev;
 	struct mutex lock;
 	u16 conf_regs[CM32181_CONF_REG_NUM];
+	unsigned long init_regs_bitmap;
 	int calibscale;
+	int lux_per_bit;
+	int lux_per_bit_base_it;
+	int num_als_it;
+	const int *als_it_bits;
+	const int *als_it_values;
 };
 
+static int cm32181_read_als_it(struct cm32181_chip *cm32181, int *val2);
+
+#ifdef CONFIG_ACPI
+/**
+ * cm32181_acpi_get_cpm() - Get CPM object from ACPI
+ * @client	pointer of struct i2c_client.
+ * @obj_name	pointer of ACPI object name.
+ * @count	maximum size of return array.
+ * @vals	pointer of array for return elements.
+ *
+ * Convert ACPI CPM table to array.
+ *
+ * Return: -ENODEV for fail.  Otherwise is number of elements.
+ */
+static int cm32181_acpi_get_cpm(struct device *dev, char *obj_name,
+				u64 *values, int count)
+{
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *cpm, *elem;
+	acpi_handle handle;
+	acpi_status status;
+	int i;
+
+	handle = ACPI_HANDLE(dev);
+	if (!handle)
+		return -ENODEV;
+
+	status = acpi_evaluate_object(handle, obj_name, NULL, &buffer);
+	if (ACPI_FAILURE(status)) {
+		dev_err(dev, "object %s not found\n", obj_name);
+		return -ENODEV;
+	}
+
+	cpm = buffer.pointer;
+	if (cpm->package.count > count)
+		dev_warn(dev, "%s table contains %u values, only using first %d values\n",
+			 obj_name, cpm->package.count, count);
+
+	count = min_t(int, cpm->package.count, count);
+	for (i = 0; i < count; i++) {
+		elem = &(cpm->package.elements[i]);
+		values[i] = elem->integer.value;
+	}
+
+	kfree(buffer.pointer);
+
+	return count;
+}
+
+static void cm32181_acpi_parse_cpm_tables(struct cm32181_chip *cm32181)
+{
+	u64 vals[CPM0_HEADER_SIZE + CM32181_CONF_REG_NUM];
+	struct device *dev = cm32181->dev;
+	int i, count;
+
+	count = cm32181_acpi_get_cpm(dev, "CPM0", vals, ARRAY_SIZE(vals));
+	if (count <= CPM0_HEADER_SIZE)
+		return;
+
+	count -= CPM0_HEADER_SIZE;
+
+	cm32181->init_regs_bitmap = vals[CPM0_REGS_BITMAP];
+	cm32181->init_regs_bitmap &= GENMASK(count - 1, 0);
+	for_each_set_bit(i, &cm32181->init_regs_bitmap, count)
+		cm32181->conf_regs[i] =	vals[CPM0_HEADER_SIZE + i];
+
+	count = cm32181_acpi_get_cpm(dev, "CPM1", vals, ARRAY_SIZE(vals));
+	if (count != CPM1_SIZE)
+		return;
+
+	cm32181->lux_per_bit = vals[CPM1_LUX_PER_BIT];
+
+	/* Check for uncalibrated devices */
+	if (vals[CPM1_CALIBSCALE] == CM32181_CALIBSCALE_DEFAULT)
+		return;
+
+	cm32181->calibscale = vals[CPM1_CALIBSCALE];
+	/* CPM1 lux_per_bit is for the current it value */
+	cm32181_read_als_it(cm32181, &cm32181->lux_per_bit_base_it);
+}
+#else
+static void cm32181_acpi_parse_cpm_tables(struct cm32181_chip *cm32181)
+{
+}
+#endif /* CONFIG_ACPI */
+
 /**
  * cm32181_reg_init() - Initialize CM32181 registers
  * @cm32181:	pointer of struct cm32181.
@@ -78,18 +193,37 @@ static int cm32181_reg_init(struct cm32181_chip *cm32181)
 		return ret;
 
 	/* check device ID */
-	if ((ret & 0xFF) != 0x81)
+	switch (ret & 0xFF) {
+	case 0x18: /* CM3218 */
+		cm32181->num_als_it = ARRAY_SIZE(cm3218_als_it_bits);
+		cm32181->als_it_bits = cm3218_als_it_bits;
+		cm32181->als_it_values = cm3218_als_it_values;
+		break;
+	case 0x81: /* CM32181 */
+	case 0x82: /* CM32182, fully compat. with CM32181 */
+		cm32181->num_als_it = ARRAY_SIZE(cm32181_als_it_bits);
+		cm32181->als_it_bits = cm32181_als_it_bits;
+		cm32181->als_it_values = cm32181_als_it_values;
+		break;
+	default:
 		return -ENODEV;
+	}
 
 	/* Default Values */
-	cm32181->conf_regs[CM32181_REG_ADDR_CMD] = CM32181_CMD_ALS_ENABLE |
+	cm32181->conf_regs[CM32181_REG_ADDR_CMD] =
 			CM32181_CMD_ALS_IT_DEFAULT | CM32181_CMD_ALS_SM_DEFAULT;
+	cm32181->init_regs_bitmap = BIT(CM32181_REG_ADDR_CMD);
 	cm32181->calibscale = CM32181_CALIBSCALE_DEFAULT;
+	cm32181->lux_per_bit = CM32181_LUX_PER_BIT;
+	cm32181->lux_per_bit_base_it = CM32181_LUX_PER_BIT_BASE_IT;
+
+	if (ACPI_HANDLE(cm32181->dev))
+		cm32181_acpi_parse_cpm_tables(cm32181);
 
 	/* Initialize registers*/
-	for (i = 0; i < CM32181_CONF_REG_NUM; i++) {
-		ret = i2c_smbus_write_word_data(client, cm32181_reg[i],
-			cm32181->conf_regs[i]);
+	for_each_set_bit(i, &cm32181->init_regs_bitmap, CM32181_CONF_REG_NUM) {
+		ret = i2c_smbus_write_word_data(client, i,
+						cm32181->conf_regs[i]);
 		if (ret < 0)
 			return ret;
 	}
@@ -102,7 +236,7 @@ static int cm32181_reg_init(struct cm32181_chip *cm32181)
  *  @cm32181:	pointer of struct cm32181
  *  @val2:	pointer of int to load the als_it value.
  *
- *  Report the current integartion time by millisecond.
+ *  Report the current integration time in milliseconds.
  *
  *  Return: IIO_VAL_INT_PLUS_MICRO for success, otherwise -EINVAL.
  */
@@ -114,9 +248,9 @@ static int cm32181_read_als_it(struct cm32181_chip *cm32181, int *val2)
 	als_it = cm32181->conf_regs[CM32181_REG_ADDR_CMD];
 	als_it &= CM32181_CMD_ALS_IT_MASK;
 	als_it >>= CM32181_CMD_ALS_IT_SHIFT;
-	for (i = 0; i < ARRAY_SIZE(als_it_bits); i++) {
-		if (als_it == als_it_bits[i]) {
-			*val2 = als_it_value[i];
+	for (i = 0; i < cm32181->num_als_it; i++) {
+		if (als_it == cm32181->als_it_bits[i]) {
+			*val2 = cm32181->als_it_values[i];
 			return IIO_VAL_INT_PLUS_MICRO;
 		}
 	}
@@ -139,14 +273,14 @@ static int cm32181_write_als_it(struct cm32181_chip *cm32181, int val)
 	u16 als_it;
 	int ret, i, n;
 
-	n = ARRAY_SIZE(als_it_value);
+	n = cm32181->num_als_it;
 	for (i = 0; i < n; i++)
-		if (val <= als_it_value[i])
+		if (val <= cm32181->als_it_values[i])
 			break;
 	if (i >= n)
 		i = n - 1;
 
-	als_it = als_it_bits[i];
+	als_it = cm32181->als_it_bits[i];
 	als_it <<= CM32181_CMD_ALS_IT_SHIFT;
 
 	mutex_lock(&cm32181->lock);
@@ -175,15 +309,15 @@ static int cm32181_get_lux(struct cm32181_chip *cm32181)
 	struct i2c_client *client = cm32181->client;
 	int ret;
 	int als_it;
-	unsigned long lux;
+	u64 lux;
 
 	ret = cm32181_read_als_it(cm32181, &als_it);
 	if (ret < 0)
 		return -EINVAL;
 
-	lux = CM32181_MLUX_PER_BIT;
-	lux *= CM32181_MLUX_PER_BIT_BASE_IT;
-	lux /= als_it;
+	lux = cm32181->lux_per_bit;
+	lux *= cm32181->lux_per_bit_base_it;
+	lux = div_u64(lux, als_it);
 
 	ret = i2c_smbus_read_word_data(client, CM32181_REG_ADDR_ALS);
 	if (ret < 0)
@@ -191,8 +325,8 @@ static int cm32181_get_lux(struct cm32181_chip *cm32181)
 
 	lux *= ret;
 	lux *= cm32181->calibscale;
-	lux /= CM32181_CALIBSCALE_RESOLUTION;
-	lux /= MLUX_PER_LUX;
+	lux = div_u64(lux, CM32181_CALIBSCALE_RESOLUTION);
+	lux = div_u64(lux, CM32181_LUX_PER_BIT_RESOLUTION);
 
 	if (lux > 0xFFFF)
 		lux = 0xFFFF;
@@ -258,11 +392,12 @@ static int cm32181_write_raw(struct iio_dev *indio_dev,
 static ssize_t cm32181_get_it_available(struct device *dev,
 			struct device_attribute *attr, char *buf)
 {
+	struct cm32181_chip *cm32181 = iio_priv(dev_to_iio_dev(dev));
 	int i, n, len;
 
-	n = ARRAY_SIZE(als_it_value);
+	n = cm32181->num_als_it;
 	for (i = 0, len = 0; i < n; i++)
-		len += sprintf(buf + len, "0.%06u ", als_it_value[i]);
+		len += sprintf(buf + len, "0.%06u ", cm32181->als_it_values[i]);
 	return len + sprintf(buf + len, "\n");
 }
 
@@ -294,70 +429,86 @@ static const struct iio_info cm32181_info = {
 	.attrs			= &cm32181_attribute_group,
 };
 
-static int cm32181_probe(struct i2c_client *client,
-			const struct i2c_device_id *id)
+static int cm32181_probe(struct i2c_client *client)
 {
+	struct device *dev = &client->dev;
 	struct cm32181_chip *cm32181;
 	struct iio_dev *indio_dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*cm32181));
-	if (!indio_dev) {
-		dev_err(&client->dev, "devm_iio_device_alloc failed\n");
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*cm32181));
+	if (!indio_dev)
 		return -ENOMEM;
+
+	/*
+	 * Some ACPI systems list 2 I2C resources for the CM3218 sensor, the
+	 * SMBus Alert Response Address (ARA, 0x0c) and the actual I2C address.
+	 * Detect this and take the following step to deal with it:
+	 * 1. When a SMBus Alert capable sensor has an Alert asserted, it will
+	 *    not respond on its actual I2C address. Read a byte from the ARA
+	 *    to clear any pending Alerts.
+	 * 2. Create a "dummy" client for the actual I2C address and
+	 *    use that client to communicate with the sensor.
+	 */
+	if (ACPI_HANDLE(dev) && client->addr == SMBUS_ALERT_RESPONSE_ADDRESS) {
+		struct i2c_board_info board_info = { .type = "dummy" };
+
+		i2c_smbus_read_byte(client);
+
+		client = i2c_acpi_new_device(dev, 1, &board_info);
+		if (IS_ERR(client))
+			return PTR_ERR(client);
 	}
 
 	cm32181 = iio_priv(indio_dev);
-	i2c_set_clientdata(client, indio_dev);
 	cm32181->client = client;
+	cm32181->dev = dev;
 
 	mutex_init(&cm32181->lock);
-	indio_dev->dev.parent = &client->dev;
+	indio_dev->dev.parent = dev;
 	indio_dev->channels = cm32181_channels;
 	indio_dev->num_channels = ARRAY_SIZE(cm32181_channels);
 	indio_dev->info = &cm32181_info;
-	indio_dev->name = id->name;
+	indio_dev->name = dev_name(dev);
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
 	ret = cm32181_reg_init(cm32181);
 	if (ret) {
-		dev_err(&client->dev,
-			"%s: register init failed\n",
-			__func__);
+		dev_err(dev, "%s: register init failed\n", __func__);
 		return ret;
 	}
 
-	ret = devm_iio_device_register(&client->dev, indio_dev);
+	ret = devm_iio_device_register(dev, indio_dev);
 	if (ret) {
-		dev_err(&client->dev,
-			"%s: regist device failed\n",
-			__func__);
+		dev_err(dev, "%s: regist device failed\n", __func__);
 		return ret;
 	}
 
 	return 0;
 }
 
-static const struct i2c_device_id cm32181_id[] = {
-	{ "cm32181", 0 },
-	{ }
-};
-
-MODULE_DEVICE_TABLE(i2c, cm32181_id);
-
 static const struct of_device_id cm32181_of_match[] = {
+	{ .compatible = "capella,cm3218" },
 	{ .compatible = "capella,cm32181" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, cm32181_of_match);
 
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id cm32181_acpi_match[] = {
+	{ "CPLM3218", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, cm32181_acpi_match);
+#endif
+
 static struct i2c_driver cm32181_driver = {
 	.driver = {
 		.name	= "cm32181",
-		.of_match_table = of_match_ptr(cm32181_of_match),
+		.acpi_match_table = ACPI_PTR(cm32181_acpi_match),
+		.of_match_table = cm32181_of_match,
 	},
-	.id_table       = cm32181_id,
-	.probe		= cm32181_probe,
+	.probe_new	= cm32181_probe,
 };
 
 module_i2c_driver(cm32181_driver);
diff --git a/drivers/iio/light/cm3232.c b/drivers/iio/light/cm3232.c
index cd3cfb7d02bd..867200825686 100644
--- a/drivers/iio/light/cm3232.c
+++ b/drivers/iio/light/cm3232.c
@@ -10,6 +10,7 @@
 
 #include <linux/i2c.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/init.h>
@@ -418,7 +419,7 @@ MODULE_DEVICE_TABLE(of, cm3232_of_match);
 static struct i2c_driver cm3232_driver = {
 	.driver = {
 		.name	= "cm3232",
-		.of_match_table = of_match_ptr(cm3232_of_match),
+		.of_match_table = cm3232_of_match,
 #ifdef CONFIG_PM_SLEEP
 		.pm	= &cm3232_pm_ops,
 #endif
diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c
index b7ef16b28280..7a2679bdc987 100644
--- a/drivers/iio/light/gp2ap002.c
+++ b/drivers/iio/light/gp2ap002.c
@@ -158,6 +158,9 @@ static irqreturn_t gp2ap002_prox_irq(int irq, void *d)
 	int val;
 	int ret;
 
+	if (!gp2ap002->enabled)
+		goto err_retrig;
+
 	ret = regmap_read(gp2ap002->map, GP2AP002_PROX, &val);
 	if (ret) {
 		dev_err(gp2ap002->dev, "error reading proximity\n");
@@ -247,6 +250,8 @@ static int gp2ap002_read_raw(struct iio_dev *indio_dev,
 	struct gp2ap002 *gp2ap002 = iio_priv(indio_dev);
 	int ret;
 
+	pm_runtime_get_sync(gp2ap002->dev);
+
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
 		switch (chan->type) {
@@ -255,13 +260,21 @@ static int gp2ap002_read_raw(struct iio_dev *indio_dev,
 			if (ret < 0)
 				return ret;
 			*val = ret;
-			return IIO_VAL_INT;
+			ret = IIO_VAL_INT;
+			goto out;
 		default:
-			return -EINVAL;
+			ret = -EINVAL;
+			goto out;
 		}
 	default:
-		return -EINVAL;
+		ret = -EINVAL;
 	}
+
+out:
+	pm_runtime_mark_last_busy(gp2ap002->dev);
+	pm_runtime_put_autosuspend(gp2ap002->dev);
+
+	return ret;
 }
 
 static int gp2ap002_init(struct gp2ap002 *gp2ap002)
diff --git a/drivers/iio/light/gp2ap020a00f.c b/drivers/iio/light/gp2ap020a00f.c
index 7fbbce0d4bc7..070d4cd0cf54 100644
--- a/drivers/iio/light/gp2ap020a00f.c
+++ b/drivers/iio/light/gp2ap020a00f.c
@@ -38,8 +38,8 @@
 #include <linux/irq.h>
 #include <linux/irq_work.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
-#include <linux/of.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
@@ -1617,18 +1617,16 @@ static const struct i2c_device_id gp2ap020a00f_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, gp2ap020a00f_id);
 
-#ifdef CONFIG_OF
 static const struct of_device_id gp2ap020a00f_of_match[] = {
 	{ .compatible = "sharp,gp2ap020a00f" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, gp2ap020a00f_of_match);
-#endif
 
 static struct i2c_driver gp2ap020a00f_driver = {
 	.driver = {
 		.name	= GP2A_I2C_NAME,
-		.of_match_table = of_match_ptr(gp2ap020a00f_of_match),
+		.of_match_table = gp2ap020a00f_of_match,
 	},
 	.probe		= gp2ap020a00f_probe,
 	.remove		= gp2ap020a00f_remove,
diff --git a/drivers/iio/light/hid-sensor-als.c b/drivers/iio/light/hid-sensor-als.c
index b6cd299517d1..81fa2a422797 100644
--- a/drivers/iio/light/hid-sensor-als.c
+++ b/drivers/iio/light/hid-sensor-als.c
@@ -14,8 +14,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 enum {
@@ -308,18 +306,13 @@ static int hid_als_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		goto error_free_dev_mem;
-	}
 	atomic_set(&als_state->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 				&als_state->common_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		goto error_free_dev_mem;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -343,9 +336,7 @@ static int hid_als_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&als_state->common_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &als_state->common_attributes);
 error_free_dev_mem:
 	kfree(indio_dev->channels);
 	return ret;
@@ -360,8 +351,7 @@ static int hid_als_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_ALS);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&als_state->common_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &als_state->common_attributes);
 	kfree(indio_dev->channels);
 
 	return 0;
diff --git a/drivers/iio/light/hid-sensor-prox.c b/drivers/iio/light/hid-sensor-prox.c
index 7e1030af9ba3..e9c04df07344 100644
--- a/drivers/iio/light/hid-sensor-prox.c
+++ b/drivers/iio/light/hid-sensor-prox.c
@@ -14,8 +14,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 #define CHANNEL_SCAN_INDEX_PRESENCE 0
@@ -286,18 +284,13 @@ static int hid_prox_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		goto error_free_dev_mem;
-	}
 	atomic_set(&prox_state->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 				&prox_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		goto error_free_dev_mem;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -321,9 +314,7 @@ static int hid_prox_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&prox_state->common_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &prox_state->common_attributes);
 error_free_dev_mem:
 	kfree(indio_dev->channels);
 	return ret;
@@ -338,8 +329,7 @@ static int hid_prox_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_PROX);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&prox_state->common_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &prox_state->common_attributes);
 	kfree(indio_dev->channels);
 
 	return 0;
diff --git a/drivers/iio/light/isl29125.c b/drivers/iio/light/isl29125.c
index e37894f0ae0b..95611f5eff01 100644
--- a/drivers/iio/light/isl29125.c
+++ b/drivers/iio/light/isl29125.c
@@ -213,13 +213,24 @@ static const struct iio_info isl29125_info = {
 	.attrs = &isl29125_attribute_group,
 };
 
-static int isl29125_buffer_preenable(struct iio_dev *indio_dev)
+static int isl29125_buffer_postenable(struct iio_dev *indio_dev)
 {
 	struct isl29125_data *data = iio_priv(indio_dev);
+	int err;
+
+	err = iio_triggered_buffer_postenable(indio_dev);
+	if (err)
+		return err;
 
 	data->conf1 |= ISL29125_MODE_RGB;
-	return i2c_smbus_write_byte_data(data->client, ISL29125_CONF1,
+	err = i2c_smbus_write_byte_data(data->client, ISL29125_CONF1,
 		data->conf1);
+	if (err) {
+		iio_triggered_buffer_predisable(indio_dev);
+		return err;
+	}
+
+	return 0;
 }
 
 static int isl29125_buffer_predisable(struct iio_dev *indio_dev)
@@ -227,19 +238,18 @@ static int isl29125_buffer_predisable(struct iio_dev *indio_dev)
 	struct isl29125_data *data = iio_priv(indio_dev);
 	int ret;
 
-	ret = iio_triggered_buffer_predisable(indio_dev);
-	if (ret < 0)
-		return ret;
-
 	data->conf1 &= ~ISL29125_MODE_MASK;
 	data->conf1 |= ISL29125_MODE_PD;
-	return i2c_smbus_write_byte_data(data->client, ISL29125_CONF1,
+	ret = i2c_smbus_write_byte_data(data->client, ISL29125_CONF1,
 		data->conf1);
+
+	iio_triggered_buffer_predisable(indio_dev);
+
+	return ret;
 }
 
 static const struct iio_buffer_setup_ops isl29125_buffer_setup_ops = {
-	.preenable = isl29125_buffer_preenable,
-	.postenable = &iio_triggered_buffer_postenable,
+	.postenable = isl29125_buffer_postenable,
 	.predisable = isl29125_buffer_predisable,
 };
 
diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c
index 71f99d2a22c1..5a3fcb127cd2 100644
--- a/drivers/iio/light/ltr501.c
+++ b/drivers/iio/light/ltr501.c
@@ -101,12 +101,12 @@ struct ltr501_gain {
 	int uscale;
 };
 
-static struct ltr501_gain ltr501_als_gain_tbl[] = {
+static const struct ltr501_gain ltr501_als_gain_tbl[] = {
 	{1, 0},
 	{0, 5000},
 };
 
-static struct ltr501_gain ltr559_als_gain_tbl[] = {
+static const struct ltr501_gain ltr559_als_gain_tbl[] = {
 	{1, 0},
 	{0, 500000},
 	{0, 250000},
@@ -117,14 +117,14 @@ static struct ltr501_gain ltr559_als_gain_tbl[] = {
 	{0, 10000},
 };
 
-static struct ltr501_gain ltr501_ps_gain_tbl[] = {
+static const struct ltr501_gain ltr501_ps_gain_tbl[] = {
 	{1, 0},
 	{0, 250000},
 	{0, 125000},
 	{0, 62500},
 };
 
-static struct ltr501_gain ltr559_ps_gain_tbl[] = {
+static const struct ltr501_gain ltr559_ps_gain_tbl[] = {
 	{0, 62500}, /* x16 gain */
 	{0, 31250}, /* x32 gain */
 	{0, 15625}, /* bits X1 are for x64 gain */
@@ -133,9 +133,9 @@ static struct ltr501_gain ltr559_ps_gain_tbl[] = {
 
 struct ltr501_chip_info {
 	u8 partid;
-	struct ltr501_gain *als_gain;
+	const struct ltr501_gain *als_gain;
 	int als_gain_tbl_size;
-	struct ltr501_gain *ps_gain;
+	const struct ltr501_gain *ps_gain;
 	int ps_gain_tbl_size;
 	u8 als_mode_active;
 	u8 als_gain_mask;
@@ -192,7 +192,7 @@ static int ltr501_match_samp_freq(const struct ltr501_samp_table *tab,
 	return -EINVAL;
 }
 
-static int ltr501_als_read_samp_freq(struct ltr501_data *data,
+static int ltr501_als_read_samp_freq(const struct ltr501_data *data,
 				     int *val, int *val2)
 {
 	int ret, i;
@@ -210,7 +210,7 @@ static int ltr501_als_read_samp_freq(struct ltr501_data *data,
 	return IIO_VAL_INT_PLUS_MICRO;
 }
 
-static int ltr501_ps_read_samp_freq(struct ltr501_data *data,
+static int ltr501_ps_read_samp_freq(const struct ltr501_data *data,
 				    int *val, int *val2)
 {
 	int ret, i;
@@ -266,7 +266,7 @@ static int ltr501_ps_write_samp_freq(struct ltr501_data *data,
 	return ret;
 }
 
-static int ltr501_als_read_samp_period(struct ltr501_data *data, int *val)
+static int ltr501_als_read_samp_period(const struct ltr501_data *data, int *val)
 {
 	int ret, i;
 
@@ -282,7 +282,7 @@ static int ltr501_als_read_samp_period(struct ltr501_data *data, int *val)
 	return IIO_VAL_INT;
 }
 
-static int ltr501_ps_read_samp_period(struct ltr501_data *data, int *val)
+static int ltr501_ps_read_samp_period(const struct ltr501_data *data, int *val)
 {
 	int ret, i;
 
@@ -321,7 +321,7 @@ static unsigned long ltr501_calculate_lux(u16 vis_data, u16 ir_data)
 	return lux / 1000;
 }
 
-static int ltr501_drdy(struct ltr501_data *data, u8 drdy_mask)
+static int ltr501_drdy(const struct ltr501_data *data, u8 drdy_mask)
 {
 	int tries = 100;
 	int ret, status;
@@ -373,7 +373,8 @@ static int ltr501_set_it_time(struct ltr501_data *data, int it)
 }
 
 /* read int time in micro seconds */
-static int ltr501_read_it_time(struct ltr501_data *data, int *val, int *val2)
+static int ltr501_read_it_time(const struct ltr501_data *data,
+			       int *val, int *val2)
 {
 	int ret, index;
 
@@ -391,7 +392,7 @@ static int ltr501_read_it_time(struct ltr501_data *data, int *val, int *val2)
 	return IIO_VAL_INT_PLUS_MICRO;
 }
 
-static int ltr501_read_als(struct ltr501_data *data, __le16 buf[2])
+static int ltr501_read_als(const struct ltr501_data *data, __le16 buf[2])
 {
 	int ret;
 
@@ -403,7 +404,7 @@ static int ltr501_read_als(struct ltr501_data *data, __le16 buf[2])
 				buf, 2 * sizeof(__le16));
 }
 
-static int ltr501_read_ps(struct ltr501_data *data)
+static int ltr501_read_ps(const struct ltr501_data *data)
 {
 	int ret, status;
 
@@ -419,7 +420,7 @@ static int ltr501_read_ps(struct ltr501_data *data)
 	return status;
 }
 
-static int ltr501_read_intr_prst(struct ltr501_data *data,
+static int ltr501_read_intr_prst(const struct ltr501_data *data,
 				 enum iio_chan_type type,
 				 int *val2)
 {
@@ -716,7 +717,7 @@ static int ltr501_read_raw(struct iio_dev *indio_dev,
 	return -EINVAL;
 }
 
-static int ltr501_get_gain_index(struct ltr501_gain *gain, int size,
+static int ltr501_get_gain_index(const struct ltr501_gain *gain, int size,
 				 int val, int val2)
 {
 	int i;
@@ -848,14 +849,14 @@ static int ltr501_write_raw(struct iio_dev *indio_dev,
 	return ret;
 }
 
-static int ltr501_read_thresh(struct iio_dev *indio_dev,
+static int ltr501_read_thresh(const struct iio_dev *indio_dev,
 			      const struct iio_chan_spec *chan,
 			      enum iio_event_type type,
 			      enum iio_event_direction dir,
 			      enum iio_event_info info,
 			      int *val, int *val2)
 {
-	struct ltr501_data *data = iio_priv(indio_dev);
+	const struct ltr501_data *data = iio_priv(indio_dev);
 	int ret, thresh_data;
 
 	switch (chan->type) {
@@ -1263,7 +1264,7 @@ static irqreturn_t ltr501_trigger_handler(int irq, void *p)
 
 	if (mask & LTR501_STATUS_ALS_RDY) {
 		ret = regmap_bulk_read(data->regmap, LTR501_ALS_DATA1,
-				       (u8 *)als_buf, sizeof(als_buf));
+				       als_buf, sizeof(als_buf));
 		if (ret < 0)
 			return ret;
 		if (test_bit(0, indio_dev->active_scan_mask))
@@ -1359,7 +1360,7 @@ static bool ltr501_is_volatile_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-static struct regmap_config ltr501_regmap_config = {
+static const struct regmap_config ltr501_regmap_config = {
 	.name =  LTR501_REGMAP_NAME,
 	.reg_bits = 8,
 	.val_bits = 8,
diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c
index 92004a2563ea..82abfa57b59c 100644
--- a/drivers/iio/light/opt3001.c
+++ b/drivers/iio/light/opt3001.c
@@ -16,6 +16,7 @@
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/types.h>
@@ -844,7 +845,7 @@ static struct i2c_driver opt3001_driver = {
 
 	.driver = {
 		.name = "opt3001",
-		.of_match_table = of_match_ptr(opt3001_of_match),
+		.of_match_table = opt3001_of_match,
 	},
 };
 
diff --git a/drivers/iio/light/si1133.c b/drivers/iio/light/si1133.c
index 9174ab928880..c1adab2a50fd 100644
--- a/drivers/iio/light/si1133.c
+++ b/drivers/iio/light/si1133.c
@@ -17,6 +17,8 @@
 
 #include <linux/util_macros.h>
 
+#include <asm/unaligned.h>
+
 #define SI1133_REG_PART_ID		0x00
 #define SI1133_REG_REV_ID		0x01
 #define SI1133_REG_MFR_ID		0x02
@@ -104,8 +106,6 @@
 #define SI1133_LUX_BUFFER_SIZE		9
 #define SI1133_MEASURE_BUFFER_SIZE	3
 
-#define SI1133_SIGN_BIT_INDEX 23
-
 static const int si1133_scale_available[] = {
 	1, 2, 4, 8, 16, 32, 64, 128};
 
@@ -633,8 +633,7 @@ static int si1133_measure(struct si1133_data *data,
 	if (err)
 		return err;
 
-	*val = sign_extend32((buffer[0] << 16) | (buffer[1] << 8) | buffer[2],
-			     SI1133_SIGN_BIT_INDEX);
+	*val = sign_extend32(get_unaligned_be24(&buffer[0]), 23);
 
 	return err;
 }
@@ -723,16 +722,11 @@ static int si1133_get_lux(struct si1133_data *data, int *val)
 	if (err)
 		return err;
 
-	high_vis =
-		sign_extend32((buffer[0] << 16) | (buffer[1] << 8) | buffer[2],
-			      SI1133_SIGN_BIT_INDEX);
+	high_vis = sign_extend32(get_unaligned_be24(&buffer[0]), 23);
 
-	low_vis =
-		sign_extend32((buffer[3] << 16) | (buffer[4] << 8) | buffer[5],
-			      SI1133_SIGN_BIT_INDEX);
+	low_vis = sign_extend32(get_unaligned_be24(&buffer[3]), 23);
 
-	ir = sign_extend32((buffer[6] << 16) | (buffer[7] << 8) | buffer[8],
-			   SI1133_SIGN_BIT_INDEX);
+	ir = sign_extend32(get_unaligned_be24(&buffer[6]), 23);
 
 	if (high_vis > SI1133_ADC_THRESHOLD || ir > SI1133_ADC_THRESHOLD)
 		lux = si1133_calc_polynomial(high_vis, ir,
diff --git a/drivers/iio/light/st_uvis25_i2c.c b/drivers/iio/light/st_uvis25_i2c.c
index 4889bbeb0c73..98cd49eefe45 100644
--- a/drivers/iio/light/st_uvis25_i2c.c
+++ b/drivers/iio/light/st_uvis25_i2c.c
@@ -9,6 +9,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/regmap.h>
@@ -31,8 +32,8 @@ static int st_uvis25_i2c_probe(struct i2c_client *client,
 
 	regmap = devm_regmap_init_i2c(client, &st_uvis25_i2c_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&client->dev, "Failed to register i2c regmap %ld\n",
+			PTR_ERR(regmap));
 		return PTR_ERR(regmap);
 	}
 
@@ -55,7 +56,7 @@ static struct i2c_driver st_uvis25_driver = {
 	.driver = {
 		.name = "st_uvis25_i2c",
 		.pm = &st_uvis25_pm_ops,
-		.of_match_table = of_match_ptr(st_uvis25_i2c_of_match),
+		.of_match_table = st_uvis25_i2c_of_match,
 	},
 	.probe = st_uvis25_i2c_probe,
 	.id_table = st_uvis25_i2c_id_table,
diff --git a/drivers/iio/light/st_uvis25_spi.c b/drivers/iio/light/st_uvis25_spi.c
index a9ceae4f58b3..af9d94d12787 100644
--- a/drivers/iio/light/st_uvis25_spi.c
+++ b/drivers/iio/light/st_uvis25_spi.c
@@ -9,6 +9,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/spi/spi.h>
 #include <linux/slab.h>
 #include <linux/regmap.h>
@@ -31,8 +32,8 @@ static int st_uvis25_spi_probe(struct spi_device *spi)
 
 	regmap = devm_regmap_init_spi(spi, &st_uvis25_spi_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&spi->dev, "Failed to register spi regmap %ld\n",
+			PTR_ERR(regmap));
 		return PTR_ERR(regmap);
 	}
 
@@ -55,7 +56,7 @@ static struct spi_driver st_uvis25_driver = {
 	.driver = {
 		.name = "st_uvis25_spi",
 		.pm = &st_uvis25_pm_ops,
-		.of_match_table = of_match_ptr(st_uvis25_spi_of_match),
+		.of_match_table = st_uvis25_spi_of_match,
 	},
 	.probe = st_uvis25_spi_probe,
 	.id_table = st_uvis25_spi_id_table,
diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c
index d8c40a83097d..27a5c28aac7f 100644
--- a/drivers/iio/light/tsl2563.c
+++ b/drivers/iio/light/tsl2563.c
@@ -69,7 +69,7 @@
 #define TSL2563_TIMING_GAIN16	0x10
 #define TSL2563_TIMING_GAIN1	0x00
 
-#define TSL2563_INT_DISBLED	0x00
+#define TSL2563_INT_DISABLED	0x00
 #define TSL2563_INT_LEVEL	0x10
 #define TSL2563_INT_PERSIST(n)	((n) & 0x0F)
 
diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c
index be37fcbd4654..9fbde9b71b63 100644
--- a/drivers/iio/light/tsl2772.c
+++ b/drivers/iio/light/tsl2772.c
@@ -932,7 +932,7 @@ static ssize_t in_illuminance0_target_input_show(struct device *dev,
 {
 	struct tsl2772_chip *chip = iio_priv(dev_to_iio_dev(dev));
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", chip->settings.als_cal_target);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", chip->settings.als_cal_target);
 }
 
 static ssize_t in_illuminance0_target_input_store(struct device *dev,
@@ -986,7 +986,7 @@ static ssize_t in_illuminance0_lux_table_show(struct device *dev,
 	int offset = 0;
 
 	while (i < TSL2772_MAX_LUX_TABLE_SIZE) {
-		offset += snprintf(buf + offset, PAGE_SIZE, "%u,%u,",
+		offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%u,%u,",
 			chip->tsl2772_device_lux[i].ch0,
 			chip->tsl2772_device_lux[i].ch1);
 		if (chip->tsl2772_device_lux[i].ch0 == 0) {
@@ -1000,7 +1000,7 @@ static ssize_t in_illuminance0_lux_table_show(struct device *dev,
 		i++;
 	}
 
-	offset += snprintf(buf + offset, PAGE_SIZE, "\n");
+	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "\n");
 	return offset;
 }
 
diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c
index ec803c1e81df..2a4b3d331055 100644
--- a/drivers/iio/light/vcnl4000.c
+++ b/drivers/iio/light/vcnl4000.c
@@ -5,6 +5,7 @@
  *
  * Copyright 2012 Peter Meerwald <pmeerw@pmeerw.net>
  * Copyright 2019 Pursim SPC
+ * Copyright 2020 Mathieu Othacehe <m.othacehe@gmail.com>
  *
  * IIO driver for:
  *   VCNL4000/10/20 (7-bit I2C slave address 0x13)
@@ -13,9 +14,7 @@
  *
  * TODO:
  *   allow to adjust IR current
- *   proximity threshold and event handling
- *   periodic ALS/proximity measurement (VCNL4010/20)
- *   interrupts (VCNL4010/20/40, VCNL4200)
+ *   interrupts (VCNL4040, VCNL4200)
  */
 
 #include <linux/module.h>
@@ -23,9 +22,15 @@
 #include <linux/err.h>
 #include <linux/delay.h>
 #include <linux/pm_runtime.h>
+#include <linux/interrupt.h>
 
+#include <linux/iio/buffer.h>
+#include <linux/iio/events.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 
 #define VCNL4000_DRV_NAME "vcnl4000"
 #define VCNL4000_PROD_ID	0x01
@@ -35,14 +40,22 @@
 
 #define VCNL4000_COMMAND	0x80 /* Command register */
 #define VCNL4000_PROD_REV	0x81 /* Product ID and Revision ID */
+#define VCNL4010_PROX_RATE      0x82 /* Proximity rate */
 #define VCNL4000_LED_CURRENT	0x83 /* IR LED current for proximity mode */
 #define VCNL4000_AL_PARAM	0x84 /* Ambient light parameter register */
+#define VCNL4010_ALS_PARAM      0x84 /* ALS rate */
 #define VCNL4000_AL_RESULT_HI	0x85 /* Ambient light result register, MSB */
 #define VCNL4000_AL_RESULT_LO	0x86 /* Ambient light result register, LSB */
 #define VCNL4000_PS_RESULT_HI	0x87 /* Proximity result register, MSB */
 #define VCNL4000_PS_RESULT_LO	0x88 /* Proximity result register, LSB */
 #define VCNL4000_PS_MEAS_FREQ	0x89 /* Proximity test signal frequency */
+#define VCNL4010_INT_CTRL	0x89 /* Interrupt control */
 #define VCNL4000_PS_MOD_ADJ	0x8a /* Proximity modulator timing adjustment */
+#define VCNL4010_LOW_THR_HI     0x8a /* Low threshold, MSB */
+#define VCNL4010_LOW_THR_LO     0x8b /* Low threshold, LSB */
+#define VCNL4010_HIGH_THR_HI    0x8c /* High threshold, MSB */
+#define VCNL4010_HIGH_THR_LO    0x8d /* High threshold, LSB */
+#define VCNL4010_ISR		0x8e /* Interrupt status */
 
 #define VCNL4200_AL_CONF	0x00 /* Ambient light configuration */
 #define VCNL4200_PS_CONF1	0x03 /* Proximity configuration */
@@ -57,6 +70,36 @@
 #define VCNL4000_PS_RDY		BIT(5) /* proximity data ready? */
 #define VCNL4000_AL_OD		BIT(4) /* start on-demand ALS measurement */
 #define VCNL4000_PS_OD		BIT(3) /* start on-demand proximity measurement */
+#define VCNL4000_ALS_EN		BIT(2) /* start ALS measurement */
+#define VCNL4000_PROX_EN	BIT(1) /* start proximity measurement */
+#define VCNL4000_SELF_TIMED_EN	BIT(0) /* start self-timed measurement */
+
+/* Bit masks for interrupt registers. */
+#define VCNL4010_INT_THR_SEL	BIT(0) /* Select threshold interrupt source */
+#define VCNL4010_INT_THR_EN	BIT(1) /* Threshold interrupt type */
+#define VCNL4010_INT_ALS_EN	BIT(2) /* Enable on ALS data ready */
+#define VCNL4010_INT_PROX_EN	BIT(3) /* Enable on proximity data ready */
+
+#define VCNL4010_INT_THR_HIGH	0 /* High threshold exceeded */
+#define VCNL4010_INT_THR_LOW	1 /* Low threshold exceeded */
+#define VCNL4010_INT_ALS	2 /* ALS data ready */
+#define VCNL4010_INT_PROXIMITY	3 /* Proximity data ready */
+
+#define VCNL4010_INT_THR \
+	(BIT(VCNL4010_INT_THR_LOW) | BIT(VCNL4010_INT_THR_HIGH))
+#define VCNL4010_INT_DRDY \
+	(BIT(VCNL4010_INT_PROXIMITY) | BIT(VCNL4010_INT_ALS))
+
+static const int vcnl4010_prox_sampling_frequency[][2] = {
+	{1, 950000},
+	{3, 906250},
+	{7, 812500},
+	{16, 625000},
+	{31, 250000},
+	{62, 500000},
+	{125, 0},
+	{250, 0},
+};
 
 #define VCNL4000_SLEEP_DELAY_MS	2000 /* before we enter pm_runtime_suspend */
 
@@ -83,10 +126,15 @@ struct vcnl4000_data {
 	struct mutex vcnl4000_lock;
 	struct vcnl4200_channel vcnl4200_al;
 	struct vcnl4200_channel vcnl4200_ps;
+	uint32_t near_level;
 };
 
 struct vcnl4000_chip_spec {
 	const char *prod;
+	struct iio_chan_spec const *channels;
+	const int num_channels;
+	const struct iio_info *info;
+	bool irq_support;
 	int (*init)(struct vcnl4000_data *data);
 	int (*measure_light)(struct vcnl4000_data *data, int *val);
 	int (*measure_proximity)(struct vcnl4000_data *data, int *val);
@@ -215,11 +263,31 @@ static int vcnl4200_init(struct vcnl4000_data *data)
 	return 0;
 };
 
+static int vcnl4000_read_data(struct vcnl4000_data *data, u8 data_reg, int *val)
+{
+	s32 ret;
+
+	ret = i2c_smbus_read_word_swapped(data->client, data_reg);
+	if (ret < 0)
+		return ret;
+
+	*val = ret;
+	return 0;
+}
+
+static int vcnl4000_write_data(struct vcnl4000_data *data, u8 data_reg, int val)
+{
+	if (val > U16_MAX)
+		return -ERANGE;
+
+	return i2c_smbus_write_word_swapped(data->client, data_reg, val);
+}
+
+
 static int vcnl4000_measure(struct vcnl4000_data *data, u8 req_mask,
 				u8 rdy_mask, u8 data_reg, int *val)
 {
 	int tries = 20;
-	__be16 buf;
 	int ret;
 
 	mutex_lock(&data->vcnl4000_lock);
@@ -246,13 +314,11 @@ static int vcnl4000_measure(struct vcnl4000_data *data, u8 req_mask,
 		goto fail;
 	}
 
-	ret = i2c_smbus_read_i2c_block_data(data->client,
-		data_reg, sizeof(buf), (u8 *) &buf);
+	ret = vcnl4000_read_data(data, data_reg, val);
 	if (ret < 0)
 		goto fail;
 
 	mutex_unlock(&data->vcnl4000_lock);
-	*val = be16_to_cpu(buf);
 
 	return 0;
 
@@ -312,47 +378,34 @@ static int vcnl4200_measure_proximity(struct vcnl4000_data *data, int *val)
 	return vcnl4200_measure(data, &data->vcnl4200_ps, val);
 }
 
-static const struct vcnl4000_chip_spec vcnl4000_chip_spec_cfg[] = {
-	[VCNL4000] = {
-		.prod = "VCNL4000",
-		.init = vcnl4000_init,
-		.measure_light = vcnl4000_measure_light,
-		.measure_proximity = vcnl4000_measure_proximity,
-		.set_power_state = vcnl4000_set_power_state,
-	},
-	[VCNL4010] = {
-		.prod = "VCNL4010/4020",
-		.init = vcnl4000_init,
-		.measure_light = vcnl4000_measure_light,
-		.measure_proximity = vcnl4000_measure_proximity,
-		.set_power_state = vcnl4000_set_power_state,
-	},
-	[VCNL4040] = {
-		.prod = "VCNL4040",
-		.init = vcnl4200_init,
-		.measure_light = vcnl4200_measure_light,
-		.measure_proximity = vcnl4200_measure_proximity,
-		.set_power_state = vcnl4200_set_power_state,
-	},
-	[VCNL4200] = {
-		.prod = "VCNL4200",
-		.init = vcnl4200_init,
-		.measure_light = vcnl4200_measure_light,
-		.measure_proximity = vcnl4200_measure_proximity,
-		.set_power_state = vcnl4200_set_power_state,
-	},
-};
+static int vcnl4010_read_proxy_samp_freq(struct vcnl4000_data *data, int *val,
+					 int *val2)
+{
+	int ret;
 
-static const struct iio_chan_spec vcnl4000_channels[] = {
-	{
-		.type = IIO_LIGHT,
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-			BIT(IIO_CHAN_INFO_SCALE),
-	}, {
-		.type = IIO_PROXIMITY,
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
-	}
-};
+	ret = i2c_smbus_read_byte_data(data->client, VCNL4010_PROX_RATE);
+	if (ret < 0)
+		return ret;
+
+	if (ret >= ARRAY_SIZE(vcnl4010_prox_sampling_frequency))
+		return -EINVAL;
+
+	*val = vcnl4010_prox_sampling_frequency[ret][0];
+	*val2 = vcnl4010_prox_sampling_frequency[ret][1];
+
+	return 0;
+}
+
+static bool vcnl4010_is_in_periodic_mode(struct vcnl4000_data *data)
+{
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(data->client, VCNL4000_COMMAND);
+	if (ret < 0)
+		return false;
+
+	return !!(ret & VCNL4000_SELF_TIMED_EN);
+}
 
 static int vcnl4000_set_pm_runtime_state(struct vcnl4000_data *data, bool on)
 {
@@ -412,10 +465,571 @@ static int vcnl4000_read_raw(struct iio_dev *indio_dev,
 	}
 }
 
+static int vcnl4010_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
+{
+	int ret;
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+	case IIO_CHAN_INFO_SCALE:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		/* Protect against event capture. */
+		if (vcnl4010_is_in_periodic_mode(data)) {
+			ret = -EBUSY;
+		} else {
+			ret = vcnl4000_read_raw(indio_dev, chan, val, val2,
+						mask);
+		}
+
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		switch (chan->type) {
+		case IIO_PROXIMITY:
+			ret = vcnl4010_read_proxy_samp_freq(data, val, val2);
+			if (ret < 0)
+				return ret;
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int vcnl4010_read_avail(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       const int **vals, int *type, int *length,
+			       long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*vals = (int *)vcnl4010_prox_sampling_frequency;
+		*type = IIO_VAL_INT_PLUS_MICRO;
+		*length = 2 * ARRAY_SIZE(vcnl4010_prox_sampling_frequency);
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int vcnl4010_write_proxy_samp_freq(struct vcnl4000_data *data, int val,
+					  int val2)
+{
+	unsigned int i;
+	int index = -1;
+
+	for (i = 0; i < ARRAY_SIZE(vcnl4010_prox_sampling_frequency); i++) {
+		if (val == vcnl4010_prox_sampling_frequency[i][0] &&
+		    val2 == vcnl4010_prox_sampling_frequency[i][1]) {
+			index = i;
+			break;
+		}
+	}
+
+	if (index < 0)
+		return -EINVAL;
+
+	return i2c_smbus_write_byte_data(data->client, VCNL4010_PROX_RATE,
+					 index);
+}
+
+static int vcnl4010_write_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long mask)
+{
+	int ret;
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret)
+		return ret;
+
+	/* Protect against event capture. */
+	if (vcnl4010_is_in_periodic_mode(data)) {
+		ret = -EBUSY;
+		goto end;
+	}
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		switch (chan->type) {
+		case IIO_PROXIMITY:
+			ret = vcnl4010_write_proxy_samp_freq(data, val, val2);
+			goto end;
+		default:
+			ret = -EINVAL;
+			goto end;
+		}
+	default:
+		ret = -EINVAL;
+		goto end;
+	}
+
+end:
+	iio_device_release_direct_mode(indio_dev);
+	return ret;
+}
+
+static int vcnl4010_read_event(struct iio_dev *indio_dev,
+			       const struct iio_chan_spec *chan,
+			       enum iio_event_type type,
+			       enum iio_event_direction dir,
+			       enum iio_event_info info,
+			       int *val, int *val2)
+{
+	int ret;
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		switch (dir) {
+		case IIO_EV_DIR_RISING:
+			ret = vcnl4000_read_data(data, VCNL4010_HIGH_THR_HI,
+						 val);
+			if (ret < 0)
+				return ret;
+			return IIO_VAL_INT;
+		case IIO_EV_DIR_FALLING:
+			ret = vcnl4000_read_data(data, VCNL4010_LOW_THR_HI,
+						 val);
+			if (ret < 0)
+				return ret;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int vcnl4010_write_event(struct iio_dev *indio_dev,
+				const struct iio_chan_spec *chan,
+				enum iio_event_type type,
+				enum iio_event_direction dir,
+				enum iio_event_info info,
+				int val, int val2)
+{
+	int ret;
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		switch (dir) {
+		case IIO_EV_DIR_RISING:
+			ret = vcnl4000_write_data(data, VCNL4010_HIGH_THR_HI,
+						  val);
+			if (ret < 0)
+				return ret;
+			return IIO_VAL_INT;
+		case IIO_EV_DIR_FALLING:
+			ret = vcnl4000_write_data(data, VCNL4010_LOW_THR_HI,
+						  val);
+			if (ret < 0)
+				return ret;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static bool vcnl4010_is_thr_enabled(struct vcnl4000_data *data)
+{
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(data->client, VCNL4010_INT_CTRL);
+	if (ret < 0)
+		return false;
+
+	return !!(ret & VCNL4010_INT_THR_EN);
+}
+
+static int vcnl4010_read_event_config(struct iio_dev *indio_dev,
+				      const struct iio_chan_spec *chan,
+				      enum iio_event_type type,
+				      enum iio_event_direction dir)
+{
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+
+	switch (chan->type) {
+	case IIO_PROXIMITY:
+		return vcnl4010_is_thr_enabled(data);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int vcnl4010_config_threshold(struct iio_dev *indio_dev, bool state)
+{
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+	int ret;
+	int icr;
+	int command;
+
+	if (state) {
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		/* Enable periodic measurement of proximity data. */
+		command = VCNL4000_SELF_TIMED_EN | VCNL4000_PROX_EN;
+
+		/*
+		 * Enable interrupts on threshold, for proximity data by
+		 * default.
+		 */
+		icr = VCNL4010_INT_THR_EN;
+	} else {
+		if (!vcnl4010_is_thr_enabled(data))
+			return 0;
+
+		command = 0;
+		icr = 0;
+	}
+
+	ret = i2c_smbus_write_byte_data(data->client, VCNL4000_COMMAND,
+					command);
+	if (ret < 0)
+		goto end;
+
+	ret = i2c_smbus_write_byte_data(data->client, VCNL4010_INT_CTRL, icr);
+
+end:
+	if (state)
+		iio_device_release_direct_mode(indio_dev);
+
+	return ret;
+}
+
+static int vcnl4010_write_event_config(struct iio_dev *indio_dev,
+				       const struct iio_chan_spec *chan,
+				       enum iio_event_type type,
+				       enum iio_event_direction dir,
+				       int state)
+{
+	switch (chan->type) {
+	case IIO_PROXIMITY:
+		return vcnl4010_config_threshold(indio_dev, state);
+	default:
+		return -EINVAL;
+	}
+}
+
+static ssize_t vcnl4000_read_near_level(struct iio_dev *indio_dev,
+					uintptr_t priv,
+					const struct iio_chan_spec *chan,
+					char *buf)
+{
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+
+	return sprintf(buf, "%u\n", data->near_level);
+}
+
+static const struct iio_chan_spec_ext_info vcnl4000_ext_info[] = {
+	{
+		.name = "nearlevel",
+		.shared = IIO_SEPARATE,
+		.read = vcnl4000_read_near_level,
+	},
+	{ /* sentinel */ }
+};
+
+static const struct iio_event_spec vcnl4000_event_spec[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_FALLING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_EITHER,
+		.mask_separate = BIT(IIO_EV_INFO_ENABLE),
+	}
+};
+
+static const struct iio_chan_spec vcnl4000_channels[] = {
+	{
+		.type = IIO_LIGHT,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			BIT(IIO_CHAN_INFO_SCALE),
+	}, {
+		.type = IIO_PROXIMITY,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.ext_info = vcnl4000_ext_info,
+	}
+};
+
+static const struct iio_chan_spec vcnl4010_channels[] = {
+	{
+		.type = IIO_LIGHT,
+		.scan_index = -1,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			BIT(IIO_CHAN_INFO_SCALE),
+	}, {
+		.type = IIO_PROXIMITY,
+		.scan_index = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.event_spec = vcnl4000_event_spec,
+		.num_event_specs = ARRAY_SIZE(vcnl4000_event_spec),
+		.ext_info = vcnl4000_ext_info,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_CPU,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
 static const struct iio_info vcnl4000_info = {
 	.read_raw = vcnl4000_read_raw,
 };
 
+static const struct iio_info vcnl4010_info = {
+	.read_raw = vcnl4010_read_raw,
+	.read_avail = vcnl4010_read_avail,
+	.write_raw = vcnl4010_write_raw,
+	.read_event_value = vcnl4010_read_event,
+	.write_event_value = vcnl4010_write_event,
+	.read_event_config = vcnl4010_read_event_config,
+	.write_event_config = vcnl4010_write_event_config,
+};
+
+static const struct vcnl4000_chip_spec vcnl4000_chip_spec_cfg[] = {
+	[VCNL4000] = {
+		.prod = "VCNL4000",
+		.init = vcnl4000_init,
+		.measure_light = vcnl4000_measure_light,
+		.measure_proximity = vcnl4000_measure_proximity,
+		.set_power_state = vcnl4000_set_power_state,
+		.channels = vcnl4000_channels,
+		.num_channels = ARRAY_SIZE(vcnl4000_channels),
+		.info = &vcnl4000_info,
+		.irq_support = false,
+	},
+	[VCNL4010] = {
+		.prod = "VCNL4010/4020",
+		.init = vcnl4000_init,
+		.measure_light = vcnl4000_measure_light,
+		.measure_proximity = vcnl4000_measure_proximity,
+		.set_power_state = vcnl4000_set_power_state,
+		.channels = vcnl4010_channels,
+		.num_channels = ARRAY_SIZE(vcnl4010_channels),
+		.info = &vcnl4010_info,
+		.irq_support = true,
+	},
+	[VCNL4040] = {
+		.prod = "VCNL4040",
+		.init = vcnl4200_init,
+		.measure_light = vcnl4200_measure_light,
+		.measure_proximity = vcnl4200_measure_proximity,
+		.set_power_state = vcnl4200_set_power_state,
+		.channels = vcnl4000_channels,
+		.num_channels = ARRAY_SIZE(vcnl4000_channels),
+		.info = &vcnl4000_info,
+		.irq_support = false,
+	},
+	[VCNL4200] = {
+		.prod = "VCNL4200",
+		.init = vcnl4200_init,
+		.measure_light = vcnl4200_measure_light,
+		.measure_proximity = vcnl4200_measure_proximity,
+		.set_power_state = vcnl4200_set_power_state,
+		.channels = vcnl4000_channels,
+		.num_channels = ARRAY_SIZE(vcnl4000_channels),
+		.info = &vcnl4000_info,
+		.irq_support = false,
+	},
+};
+
+static irqreturn_t vcnl4010_irq_thread(int irq, void *p)
+{
+	struct iio_dev *indio_dev = p;
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+	unsigned long isr;
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(data->client, VCNL4010_ISR);
+	if (ret < 0)
+		goto end;
+
+	isr = ret;
+
+	if (isr & VCNL4010_INT_THR) {
+		if (test_bit(VCNL4010_INT_THR_LOW, &isr)) {
+			iio_push_event(indio_dev,
+				       IIO_UNMOD_EVENT_CODE(
+					       IIO_PROXIMITY,
+					       1,
+					       IIO_EV_TYPE_THRESH,
+					       IIO_EV_DIR_FALLING),
+				       iio_get_time_ns(indio_dev));
+		}
+
+		if (test_bit(VCNL4010_INT_THR_HIGH, &isr)) {
+			iio_push_event(indio_dev,
+				       IIO_UNMOD_EVENT_CODE(
+					       IIO_PROXIMITY,
+					       1,
+					       IIO_EV_TYPE_THRESH,
+					       IIO_EV_DIR_RISING),
+				       iio_get_time_ns(indio_dev));
+		}
+
+		i2c_smbus_write_byte_data(data->client, VCNL4010_ISR,
+					  isr & VCNL4010_INT_THR);
+	}
+
+	if (isr & VCNL4010_INT_DRDY && iio_buffer_enabled(indio_dev))
+		iio_trigger_poll_chained(indio_dev->trig);
+
+end:
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t vcnl4010_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+	const unsigned long *active_scan_mask = indio_dev->active_scan_mask;
+	u16 buffer[8] = {0}; /* 1x16-bit + ts */
+	bool data_read = false;
+	unsigned long isr;
+	int val = 0;
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(data->client, VCNL4010_ISR);
+	if (ret < 0)
+		goto end;
+
+	isr = ret;
+
+	if (test_bit(0, active_scan_mask)) {
+		if (test_bit(VCNL4010_INT_PROXIMITY, &isr)) {
+			ret = vcnl4000_read_data(data,
+						 VCNL4000_PS_RESULT_HI,
+						 &val);
+			if (ret < 0)
+				goto end;
+
+			buffer[0] = val;
+			data_read = true;
+		}
+	}
+
+	ret = i2c_smbus_write_byte_data(data->client, VCNL4010_ISR,
+					isr & VCNL4010_INT_DRDY);
+	if (ret < 0)
+		goto end;
+
+	if (!data_read)
+		goto end;
+
+	iio_push_to_buffers_with_timestamp(indio_dev, buffer,
+					   iio_get_time_ns(indio_dev));
+
+end:
+	iio_trigger_notify_done(indio_dev->trig);
+	return IRQ_HANDLED;
+}
+
+static int vcnl4010_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+	int ret;
+	int cmd;
+
+	ret = iio_triggered_buffer_postenable(indio_dev);
+	if (ret)
+		return ret;
+
+	/* Do not enable the buffer if we are already capturing events. */
+	if (vcnl4010_is_in_periodic_mode(data)) {
+		ret = -EBUSY;
+		goto end;
+	}
+
+	ret = i2c_smbus_write_byte_data(data->client, VCNL4010_INT_CTRL,
+					VCNL4010_INT_PROX_EN);
+	if (ret < 0)
+		goto end;
+
+	cmd = VCNL4000_SELF_TIMED_EN | VCNL4000_PROX_EN;
+	ret = i2c_smbus_write_byte_data(data->client, VCNL4000_COMMAND, cmd);
+	if (ret < 0)
+		goto end;
+
+	return 0;
+end:
+	iio_triggered_buffer_predisable(indio_dev);
+
+	return ret;
+}
+
+static int vcnl4010_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+	int ret, ret_disable;
+
+	ret = i2c_smbus_write_byte_data(data->client, VCNL4010_INT_CTRL, 0);
+	if (ret < 0)
+		goto end;
+
+	ret = i2c_smbus_write_byte_data(data->client, VCNL4000_COMMAND, 0);
+
+end:
+	ret_disable = iio_triggered_buffer_predisable(indio_dev);
+	if (ret == 0)
+		ret = ret_disable;
+
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops vcnl4010_buffer_ops = {
+	.postenable = &vcnl4010_buffer_postenable,
+	.predisable = &vcnl4010_buffer_predisable,
+};
+
+static const struct iio_trigger_ops vcnl4010_trigger_ops = {
+	.validate_device = iio_trigger_validate_own_device,
+};
+
+static int vcnl4010_probe_trigger(struct iio_dev *indio_dev)
+{
+	struct vcnl4000_data *data = iio_priv(indio_dev);
+	struct i2c_client *client = data->client;
+	struct iio_trigger *trigger;
+
+	trigger = devm_iio_trigger_alloc(&client->dev, "%s-dev%d",
+					 indio_dev->name, indio_dev->id);
+	if (!trigger)
+		return -ENOMEM;
+
+	trigger->dev.parent = &client->dev;
+	trigger->ops = &vcnl4010_trigger_ops;
+	iio_trigger_set_drvdata(trigger, indio_dev);
+
+	return devm_iio_trigger_register(&client->dev, trigger);
+}
+
 static int vcnl4000_probe(struct i2c_client *client,
 			  const struct i2c_device_id *id)
 {
@@ -440,13 +1054,44 @@ static int vcnl4000_probe(struct i2c_client *client,
 	dev_dbg(&client->dev, "%s Ambient light/proximity sensor, Rev: %02x\n",
 		data->chip_spec->prod, data->rev);
 
+	if (device_property_read_u32(&client->dev, "proximity-near-level",
+				     &data->near_level))
+		data->near_level = 0;
+
 	indio_dev->dev.parent = &client->dev;
-	indio_dev->info = &vcnl4000_info;
-	indio_dev->channels = vcnl4000_channels;
-	indio_dev->num_channels = ARRAY_SIZE(vcnl4000_channels);
+	indio_dev->info = data->chip_spec->info;
+	indio_dev->channels = data->chip_spec->channels;
+	indio_dev->num_channels = data->chip_spec->num_channels;
 	indio_dev->name = VCNL4000_DRV_NAME;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
+	if (client->irq && data->chip_spec->irq_support) {
+		ret = devm_iio_triggered_buffer_setup(&client->dev, indio_dev,
+						      NULL,
+						      vcnl4010_trigger_handler,
+						      &vcnl4010_buffer_ops);
+		if (ret < 0) {
+			dev_err(&client->dev,
+				"unable to setup iio triggered buffer\n");
+			return ret;
+		}
+
+		ret = devm_request_threaded_irq(&client->dev, client->irq,
+						NULL, vcnl4010_irq_thread,
+						IRQF_TRIGGER_FALLING |
+						IRQF_ONESHOT,
+						"vcnl4010_irq",
+						indio_dev);
+		if (ret < 0) {
+			dev_err(&client->dev, "irq request failed\n");
+			return ret;
+		}
+
+		ret = vcnl4010_probe_trigger(indio_dev);
+		if (ret < 0)
+			return ret;
+	}
+
 	ret = pm_runtime_set_active(&client->dev);
 	if (ret < 0)
 		goto fail_poweroff;
@@ -540,5 +1185,6 @@ static struct i2c_driver vcnl4000_driver = {
 module_i2c_driver(vcnl4000_driver);
 
 MODULE_AUTHOR("Peter Meerwald <pmeerw@pmeerw.net>");
+MODULE_AUTHOR("Mathieu Othacehe <m.othacehe@gmail.com>");
 MODULE_DESCRIPTION("Vishay VCNL4000 proximity/ambient light sensor driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/light/vl6180.c b/drivers/iio/light/vl6180.c
index d9533a76b8f6..ed7b02765b97 100644
--- a/drivers/iio/light/vl6180.c
+++ b/drivers/iio/light/vl6180.c
@@ -16,6 +16,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 #include <linux/err.h>
@@ -537,7 +538,7 @@ MODULE_DEVICE_TABLE(i2c, vl6180_id);
 static struct i2c_driver vl6180_driver = {
 	.driver = {
 		.name   = VL6180_DRV_NAME,
-		.of_match_table = of_match_ptr(vl6180_of_match),
+		.of_match_table = vl6180_of_match,
 	},
 	.probe  = vl6180_probe,
 	.id_table = vl6180_id,
diff --git a/drivers/iio/light/zopt2201.c b/drivers/iio/light/zopt2201.c
index 5f54f39e7a4c..80ae530720cd 100644
--- a/drivers/iio/light/zopt2201.c
+++ b/drivers/iio/light/zopt2201.c
@@ -19,6 +19,8 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
+#include <asm/unaligned.h>
+
 #define ZOPT2201_DRV_NAME "zopt2201"
 
 /* Registers */
@@ -219,7 +221,7 @@ static int zopt2201_read(struct zopt2201_data *data, u8 reg)
 		goto fail;
 	mutex_unlock(&data->lock);
 
-	return (buf[2] << 16) | (buf[1] << 8) | buf[0];
+	return get_unaligned_le24(&buf[0]);
 
 fail:
 	mutex_unlock(&data->lock);
diff --git a/drivers/iio/magnetometer/ak8974.c b/drivers/iio/magnetometer/ak8974.c
index d32996702110..810fdfd37c88 100644
--- a/drivers/iio/magnetometer/ak8974.c
+++ b/drivers/iio/magnetometer/ak8974.c
@@ -49,6 +49,7 @@
 #define AK8974_WHOAMI_VALUE_AMI306 0x46
 #define AK8974_WHOAMI_VALUE_AMI305 0x47
 #define AK8974_WHOAMI_VALUE_AK8974 0x48
+#define AK8974_WHOAMI_VALUE_HSCDTD008A 0x49
 
 #define AK8974_DATA_X		0x10
 #define AK8974_DATA_Y		0x12
@@ -140,6 +141,12 @@
 #define AK8974_INT_CTRL_PULSE	BIT(1) /* 0 = latched; 1 = pulse (50 usec) */
 #define AK8974_INT_CTRL_RESDEF	(AK8974_INT_CTRL_XYZEN | AK8974_INT_CTRL_POL)
 
+/* HSCDTD008A-specific control register */
+#define HSCDTD008A_CTRL4	0x1E
+#define HSCDTD008A_CTRL4_MMD	BIT(7)	/* must be set to 1 */
+#define HSCDTD008A_CTRL4_RANGE	BIT(4)	/* 0 = 14-bit output; 1 = 15-bit output */
+#define HSCDTD008A_CTRL4_RESDEF	(HSCDTD008A_CTRL4_MMD | HSCDTD008A_CTRL4_RANGE)
+
 /* The AMI305 has elaborate FW version and serial number registers */
 #define AMI305_VER		0xE8
 #define AMI305_SN		0xEA
@@ -241,10 +248,17 @@ static int ak8974_reset(struct ak8974 *ak8974)
 	ret = regmap_write(ak8974->map, AK8974_CTRL3, AK8974_CTRL3_RESDEF);
 	if (ret)
 		return ret;
-	ret = regmap_write(ak8974->map, AK8974_INT_CTRL,
-			   AK8974_INT_CTRL_RESDEF);
-	if (ret)
-		return ret;
+	if (ak8974->variant != AK8974_WHOAMI_VALUE_HSCDTD008A) {
+		ret = regmap_write(ak8974->map, AK8974_INT_CTRL,
+				   AK8974_INT_CTRL_RESDEF);
+		if (ret)
+			return ret;
+	} else {
+		ret = regmap_write(ak8974->map, HSCDTD008A_CTRL4,
+				   HSCDTD008A_CTRL4_RESDEF);
+		if (ret)
+			return ret;
+	}
 
 	/* After reset, power off is default state */
 	return ak8974_set_power(ak8974, AK8974_PWR_OFF);
@@ -267,6 +281,8 @@ static int ak8974_configure(struct ak8974 *ak8974)
 		if (ret)
 			return ret;
 	}
+	if (ak8974->variant == AK8974_WHOAMI_VALUE_HSCDTD008A)
+		return 0;
 	ret = regmap_write(ak8974->map, AK8974_INT_CTRL, AK8974_INT_CTRL_POL);
 	if (ret)
 		return ret;
@@ -495,6 +511,10 @@ static int ak8974_detect(struct ak8974 *ak8974)
 		name = "ak8974";
 		dev_info(&ak8974->i2c->dev, "detected AK8974\n");
 		break;
+	case AK8974_WHOAMI_VALUE_HSCDTD008A:
+		name = "hscdtd008a";
+		dev_info(&ak8974->i2c->dev, "detected hscdtd008a\n");
+		break;
 	default:
 		dev_err(&ak8974->i2c->dev, "unsupported device (%02x) ",
 			whoami);
@@ -534,47 +554,103 @@ static int ak8974_detect(struct ak8974 *ak8974)
 	return 0;
 }
 
+static int ak8974_measure_channel(struct ak8974 *ak8974, unsigned long address,
+				  int *val)
+{
+	__le16 hw_values[3];
+	int ret;
+
+	pm_runtime_get_sync(&ak8974->i2c->dev);
+	mutex_lock(&ak8974->lock);
+
+	/*
+	 * We read all axes and discard all but one, for optimized
+	 * reading, use the triggered buffer.
+	 */
+	ret = ak8974_trigmeas(ak8974);
+	if (ret)
+		goto out_unlock;
+	ret = ak8974_getresult(ak8974, hw_values);
+	if (ret)
+		goto out_unlock;
+	/*
+	 * This explicit cast to (s16) is necessary as the measurement
+	 * is done in 2's complement with positive and negative values.
+	 * The follwing assignment to *val will then convert the signed
+	 * s16 value to a signed int value.
+	 */
+	*val = (s16)le16_to_cpu(hw_values[address]);
+out_unlock:
+	mutex_unlock(&ak8974->lock);
+	pm_runtime_mark_last_busy(&ak8974->i2c->dev);
+	pm_runtime_put_autosuspend(&ak8974->i2c->dev);
+
+	return ret;
+}
+
 static int ak8974_read_raw(struct iio_dev *indio_dev,
 			   struct iio_chan_spec const *chan,
 			   int *val, int *val2,
 			   long mask)
 {
 	struct ak8974 *ak8974 = iio_priv(indio_dev);
-	__le16 hw_values[3];
-	int ret = -EINVAL;
-
-	pm_runtime_get_sync(&ak8974->i2c->dev);
-	mutex_lock(&ak8974->lock);
+	int ret;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
 		if (chan->address > 2) {
 			dev_err(&ak8974->i2c->dev, "faulty channel address\n");
-			ret = -EIO;
-			goto out_unlock;
+			return -EIO;
 		}
-		ret = ak8974_trigmeas(ak8974);
-		if (ret)
-			goto out_unlock;
-		ret = ak8974_getresult(ak8974, hw_values);
+		ret = ak8974_measure_channel(ak8974, chan->address, val);
 		if (ret)
-			goto out_unlock;
-
-		/*
-		 * We read all axes and discard all but one, for optimized
-		 * reading, use the triggered buffer.
-		 */
-		*val = (s16)le16_to_cpu(hw_values[chan->address]);
-
-		ret = IIO_VAL_INT;
+			return ret;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		switch (ak8974->variant) {
+		case AK8974_WHOAMI_VALUE_AMI306:
+		case AK8974_WHOAMI_VALUE_AMI305:
+			/*
+			 * The datasheet for AMI305 and AMI306, page 6
+			 * specifies the range of the sensor to be
+			 * +/- 12 Gauss.
+			 */
+			*val = 12;
+			/*
+			 * 12 bits are used, +/- 2^11
+			 * [ -2048 .. 2047 ] (manual page 20)
+			 * [ 0xf800 .. 0x07ff ]
+			 */
+			*val2 = 11;
+			return IIO_VAL_FRACTIONAL_LOG2;
+		case AK8974_WHOAMI_VALUE_HSCDTD008A:
+			/*
+			 * The datasheet for HSCDTF008A, page 3 specifies the
+			 * range of the sensor as +/- 2.4 mT per axis, which
+			 * corresponds to +/- 2400 uT = +/- 24 Gauss.
+			 */
+			*val = 24;
+			/*
+			 * 15 bits are used (set up in CTRL4), +/- 2^14
+			 * [ -16384 .. 16383 ] (manual page 24)
+			 * [ 0xc000 .. 0x3fff ]
+			 */
+			*val2 = 14;
+			return IIO_VAL_FRACTIONAL_LOG2;
+		default:
+			/* GUESSING +/- 12 Gauss */
+			*val = 12;
+			/* GUESSING 12 bits ADC +/- 2^11 */
+			*val2 = 11;
+			return IIO_VAL_FRACTIONAL_LOG2;
+		}
+		break;
+	default:
+		/* Unknown request */
+		break;
 	}
 
- out_unlock:
-	mutex_unlock(&ak8974->lock);
-	pm_runtime_mark_last_busy(&ak8974->i2c->dev);
-	pm_runtime_put_autosuspend(&ak8974->i2c->dev);
-
-	return ret;
+	return -EINVAL;
 }
 
 static void ak8974_fill_buffer(struct iio_dev *indio_dev)
@@ -631,27 +707,44 @@ static const struct iio_chan_spec_ext_info ak8974_ext_info[] = {
 	{ },
 };
 
-#define AK8974_AXIS_CHANNEL(axis, index)				\
+#define AK8974_AXIS_CHANNEL(axis, index, bits)				\
 	{								\
 		.type = IIO_MAGN,					\
 		.modified = 1,						\
 		.channel2 = IIO_MOD_##axis,				\
-		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
+			BIT(IIO_CHAN_INFO_SCALE),			\
 		.ext_info = ak8974_ext_info,				\
 		.address = index,					\
 		.scan_index = index,					\
 		.scan_type = {						\
 			.sign = 's',					\
-			.realbits = 16,					\
+			.realbits = bits,				\
 			.storagebits = 16,				\
 			.endianness = IIO_LE				\
 		},							\
 	}
 
-static const struct iio_chan_spec ak8974_channels[] = {
-	AK8974_AXIS_CHANNEL(X, 0),
-	AK8974_AXIS_CHANNEL(Y, 1),
-	AK8974_AXIS_CHANNEL(Z, 2),
+/*
+ * We have no datasheet for the AK8974 but we guess that its
+ * ADC is 12 bits. The AMI305 and AMI306 certainly has 12bit
+ * ADC.
+ */
+static const struct iio_chan_spec ak8974_12_bits_channels[] = {
+	AK8974_AXIS_CHANNEL(X, 0, 12),
+	AK8974_AXIS_CHANNEL(Y, 1, 12),
+	AK8974_AXIS_CHANNEL(Z, 2, 12),
+	IIO_CHAN_SOFT_TIMESTAMP(3),
+};
+
+/*
+ * The HSCDTD008A has 15 bits resolution the way we set it up
+ * in CTRL4.
+ */
+static const struct iio_chan_spec ak8974_15_bits_channels[] = {
+	AK8974_AXIS_CHANNEL(X, 0, 15),
+	AK8974_AXIS_CHANNEL(Y, 1, 15),
+	AK8974_AXIS_CHANNEL(Z, 2, 15),
 	IIO_CHAN_SOFT_TIMESTAMP(3),
 };
 
@@ -674,18 +767,18 @@ static bool ak8974_writeable_reg(struct device *dev, unsigned int reg)
 	case AK8974_INT_CTRL:
 	case AK8974_INT_THRES:
 	case AK8974_INT_THRES + 1:
+		return true;
 	case AK8974_PRESET:
 	case AK8974_PRESET + 1:
-		return true;
+		return ak8974->variant != AK8974_WHOAMI_VALUE_HSCDTD008A;
 	case AK8974_OFFSET_X:
 	case AK8974_OFFSET_X + 1:
 	case AK8974_OFFSET_Y:
 	case AK8974_OFFSET_Y + 1:
 	case AK8974_OFFSET_Z:
 	case AK8974_OFFSET_Z + 1:
-		if (ak8974->variant == AK8974_WHOAMI_VALUE_AK8974)
-			return true;
-		return false;
+		return ak8974->variant == AK8974_WHOAMI_VALUE_AK8974 ||
+		       ak8974->variant == AK8974_WHOAMI_VALUE_HSCDTD008A;
 	case AMI305_OFFSET_X:
 	case AMI305_OFFSET_X + 1:
 	case AMI305_OFFSET_Y:
@@ -746,7 +839,12 @@ static int ak8974_probe(struct i2c_client *i2c,
 				      ARRAY_SIZE(ak8974->regs),
 				      ak8974->regs);
 	if (ret < 0) {
-		dev_err(&i2c->dev, "cannot get regulators\n");
+		if (ret != -EPROBE_DEFER)
+			dev_err(&i2c->dev, "cannot get regulators: %d\n", ret);
+		else
+			dev_dbg(&i2c->dev,
+				"regulators unavailable, deferring probe\n");
+
 		return ret;
 	}
 
@@ -795,8 +893,21 @@ static int ak8974_probe(struct i2c_client *i2c,
 	pm_runtime_put(&i2c->dev);
 
 	indio_dev->dev.parent = &i2c->dev;
-	indio_dev->channels = ak8974_channels;
-	indio_dev->num_channels = ARRAY_SIZE(ak8974_channels);
+	switch (ak8974->variant) {
+	case AK8974_WHOAMI_VALUE_AMI306:
+	case AK8974_WHOAMI_VALUE_AMI305:
+		indio_dev->channels = ak8974_12_bits_channels;
+		indio_dev->num_channels = ARRAY_SIZE(ak8974_12_bits_channels);
+		break;
+	case AK8974_WHOAMI_VALUE_HSCDTD008A:
+		indio_dev->channels = ak8974_15_bits_channels;
+		indio_dev->num_channels = ARRAY_SIZE(ak8974_15_bits_channels);
+		break;
+	default:
+		indio_dev->channels = ak8974_12_bits_channels;
+		indio_dev->num_channels = ARRAY_SIZE(ak8974_12_bits_channels);
+		break;
+	}
 	indio_dev->info = &ak8974_info;
 	indio_dev->available_scan_masks = ak8974_scan_masks;
 	indio_dev->modes = INDIO_DIRECT_MODE;
@@ -926,12 +1037,14 @@ static const struct i2c_device_id ak8974_id[] = {
 	{"ami305", 0 },
 	{"ami306", 0 },
 	{"ak8974", 0 },
+	{"hscdtd008a", 0 },
 	{}
 };
 MODULE_DEVICE_TABLE(i2c, ak8974_id);
 
 static const struct of_device_id ak8974_of_match[] = {
 	{ .compatible = "asahi-kasei,ak8974", },
+	{ .compatible = "alps,hscdtd008a", },
 	{}
 };
 MODULE_DEVICE_TABLE(of, ak8974_of_match);
diff --git a/drivers/iio/magnetometer/bmc150_magn_spi.c b/drivers/iio/magnetometer/bmc150_magn_spi.c
index ed9be0490d77..c6ed3ea8460a 100644
--- a/drivers/iio/magnetometer/bmc150_magn_spi.c
+++ b/drivers/iio/magnetometer/bmc150_magn_spi.c
@@ -22,8 +22,8 @@ static int bmc150_magn_spi_probe(struct spi_device *spi)
 
 	regmap = devm_regmap_init_spi(spi, &bmc150_magn_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
-			(int)PTR_ERR(regmap));
+		dev_err(&spi->dev, "Failed to register spi regmap: %pe\n",
+			regmap);
 		return PTR_ERR(regmap);
 	}
 	return bmc150_magn_probe(&spi->dev, regmap, spi->irq, id->name);
diff --git a/drivers/iio/magnetometer/hid-sensor-magn-3d.c b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
index 25e60b233e08..0c09daf87794 100644
--- a/drivers/iio/magnetometer/hid-sensor-magn-3d.c
+++ b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
@@ -14,8 +14,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 enum magn_3d_channel {
@@ -519,18 +517,13 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		return ret;
-	}
 	atomic_set(&magn_state->magn_flux_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 					&magn_state->magn_flux_attributes);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		return ret;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -554,9 +547,7 @@ static int hid_magn_3d_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&magn_state->magn_flux_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &magn_state->magn_flux_attributes);
 	return ret;
 }
 
@@ -569,8 +560,7 @@ static int hid_magn_3d_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_COMPASS_3D);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&magn_state->magn_flux_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &magn_state->magn_flux_attributes);
 
 	return 0;
 }
diff --git a/drivers/iio/magnetometer/mmc35240.c b/drivers/iio/magnetometer/mmc35240.c
index 425cdd07b4e5..1787d656d009 100644
--- a/drivers/iio/magnetometer/mmc35240.c
+++ b/drivers/iio/magnetometer/mmc35240.c
@@ -239,7 +239,7 @@ static int mmc35240_init(struct mmc35240_data *data)
 		return ret;
 
 	ret = regmap_bulk_read(data->regmap, MMC35240_OTP_START_ADDR,
-			       (u8 *)otp_data, sizeof(otp_data));
+			       otp_data, sizeof(otp_data));
 	if (ret < 0)
 		return ret;
 
@@ -295,7 +295,7 @@ static int mmc35240_read_measurement(struct mmc35240_data *data, __le16 buf[3])
 	if (ret < 0)
 		return ret;
 
-	return regmap_bulk_read(data->regmap, MMC35240_REG_XOUT_L, (u8 *)buf,
+	return regmap_bulk_read(data->regmap, MMC35240_REG_XOUT_L, buf,
 				3 * sizeof(__le16));
 }
 
diff --git a/drivers/iio/magnetometer/rm3100-core.c b/drivers/iio/magnetometer/rm3100-core.c
index 7c20918d8108..43a2e420c9c4 100644
--- a/drivers/iio/magnetometer/rm3100-core.c
+++ b/drivers/iio/magnetometer/rm3100-core.c
@@ -22,6 +22,8 @@
 #include <linux/iio/triggered_buffer.h>
 #include <linux/iio/trigger_consumer.h>
 
+#include <asm/unaligned.h>
+
 #include "rm3100.h"
 
 /* Cycle Count Registers. */
@@ -223,8 +225,7 @@ static int rm3100_read_mag(struct rm3100_data *data, int idx, int *val)
 		goto unlock_return;
 	mutex_unlock(&data->lock);
 
-	*val = sign_extend32((buffer[0] << 16) | (buffer[1] << 8) | buffer[2],
-			     23);
+	*val = sign_extend32(get_unaligned_be24(&buffer[0]), 23);
 
 	return IIO_VAL_INT;
 
diff --git a/drivers/iio/magnetometer/st_magn_core.c b/drivers/iio/magnetometer/st_magn_core.c
index e68184a93a6d..79de721e6015 100644
--- a/drivers/iio/magnetometer/st_magn_core.c
+++ b/drivers/iio/magnetometer/st_magn_core.c
@@ -506,8 +506,7 @@ int st_magn_common_probe(struct iio_dev *indio_dev)
 	indio_dev->channels = mdata->sensor_settings->ch;
 	indio_dev->num_channels = ST_SENSORS_NUMBER_ALL_CHANNELS;
 
-	mdata->current_fullscale = (struct st_sensor_fullscale_avl *)
-					&mdata->sensor_settings->fs.fs_avl[0];
+	mdata->current_fullscale = &mdata->sensor_settings->fs.fs_avl[0];
 	mdata->odr = mdata->sensor_settings->odr.odr_avl[0].hz;
 
 	err = st_sensors_init_sensor(indio_dev, NULL);
diff --git a/drivers/iio/orientation/hid-sensor-incl-3d.c b/drivers/iio/orientation/hid-sensor-incl-3d.c
index 00af68764cda..6aac8bea233a 100644
--- a/drivers/iio/orientation/hid-sensor-incl-3d.c
+++ b/drivers/iio/orientation/hid-sensor-incl-3d.c
@@ -15,8 +15,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 enum incl_3d_channel {
@@ -346,18 +344,13 @@ static int hid_incl_3d_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		goto error_free_dev_mem;
-	}
 	atomic_set(&incl_state->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 					&incl_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		goto error_free_dev_mem;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -382,9 +375,7 @@ static int hid_incl_3d_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&incl_state->common_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &incl_state->common_attributes);
 error_free_dev_mem:
 	kfree(indio_dev->channels);
 	return ret;
@@ -399,8 +390,7 @@ static int hid_incl_3d_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_INCLINOMETER_3D);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&incl_state->common_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &incl_state->common_attributes);
 	kfree(indio_dev->channels);
 
 	return 0;
diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c
index 64ae7d04a200..b99f41240e3e 100644
--- a/drivers/iio/orientation/hid-sensor-rotation.c
+++ b/drivers/iio/orientation/hid-sensor-rotation.c
@@ -14,8 +14,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 struct dev_rot_state {
@@ -288,18 +286,13 @@ static int hid_dev_rot_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		return ret;
-	}
 	atomic_set(&rot_state->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 					&rot_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		return ret;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -323,9 +316,7 @@ static int hid_dev_rot_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&rot_state->common_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes);
 	return ret;
 }
 
@@ -338,8 +329,7 @@ static int hid_dev_rot_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, hsdev->usage);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&rot_state->common_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes);
 
 	return 0;
 }
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index 29c209cc1108..126a56d31b6e 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -271,6 +271,8 @@ static u32 bmp280_compensate_humidity(struct bmp280_data *data,
 		+ (s32)2097152) * calib->H2 + 8192) >> 14);
 	var -= ((((var >> 15) * (var >> 15)) >> 7) * (s32)calib->H1) >> 4;
 
+	var = clamp_val(var, 0, 419430400);
+
 	return var >> 12;
 };
 
@@ -337,8 +339,7 @@ static int bmp280_read_temp(struct bmp280_data *data,
 	__be32 tmp = 0;
 	s32 adc_temp, comp_temp;
 
-	ret = regmap_bulk_read(data->regmap, BMP280_REG_TEMP_MSB,
-			       (u8 *) &tmp, 3);
+	ret = regmap_bulk_read(data->regmap, BMP280_REG_TEMP_MSB, &tmp, 3);
 	if (ret < 0) {
 		dev_err(data->dev, "failed to read temperature\n");
 		return ret;
@@ -377,8 +378,7 @@ static int bmp280_read_press(struct bmp280_data *data,
 	if (ret < 0)
 		return ret;
 
-	ret = regmap_bulk_read(data->regmap, BMP280_REG_PRESS_MSB,
-			       (u8 *) &tmp, 3);
+	ret = regmap_bulk_read(data->regmap, BMP280_REG_PRESS_MSB, &tmp, 3);
 	if (ret < 0) {
 		dev_err(data->dev, "failed to read pressure\n");
 		return ret;
@@ -400,8 +400,8 @@ static int bmp280_read_press(struct bmp280_data *data,
 
 static int bmp280_read_humid(struct bmp280_data *data, int *val, int *val2)
 {
+	__be16 tmp;
 	int ret;
-	__be16 tmp = 0;
 	s32 adc_humidity;
 	u32 comp_humidity;
 
@@ -410,8 +410,7 @@ static int bmp280_read_humid(struct bmp280_data *data, int *val, int *val2)
 	if (ret < 0)
 		return ret;
 
-	ret = regmap_bulk_read(data->regmap, BMP280_REG_HUMIDITY_MSB,
-			       (u8 *) &tmp, 2);
+	ret = regmap_bulk_read(data->regmap, BMP280_REG_HUMIDITY_MSB, &tmp, 2);
 	if (ret < 0) {
 		dev_err(data->dev, "failed to read humidity\n");
 		return ret;
@@ -575,57 +574,38 @@ static int bmp280_write_raw(struct iio_dev *indio_dev,
 	return ret;
 }
 
-static ssize_t bmp280_show_avail(char *buf, const int *vals, const int n)
+static int bmp280_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type, int *length,
+			     long mask)
 {
-	size_t len = 0;
-	int i;
-
-	for (i = 0; i < n; i++)
-		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ", vals[i]);
-
-	buf[len - 1] = '\n';
-
-	return len;
-}
-
-static ssize_t bmp280_show_temp_oversampling_avail(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	struct bmp280_data *data = iio_priv(dev_to_iio_dev(dev));
-
-	return bmp280_show_avail(buf, data->chip_info->oversampling_temp_avail,
-				 data->chip_info->num_oversampling_temp_avail);
-}
-
-static ssize_t bmp280_show_press_oversampling_avail(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	struct bmp280_data *data = iio_priv(dev_to_iio_dev(dev));
+	struct bmp280_data *data = iio_priv(indio_dev);
 
-	return bmp280_show_avail(buf, data->chip_info->oversampling_press_avail,
-				 data->chip_info->num_oversampling_press_avail);
+	switch (mask) {
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		switch (chan->type) {
+		case IIO_PRESSURE:
+			*vals = data->chip_info->oversampling_press_avail;
+			*length = data->chip_info->num_oversampling_press_avail;
+			break;
+		case IIO_TEMP:
+			*vals = data->chip_info->oversampling_temp_avail;
+			*length = data->chip_info->num_oversampling_temp_avail;
+			break;
+		default:
+			return -EINVAL;
+		}
+		*type = IIO_VAL_INT;
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
 }
 
-static IIO_DEVICE_ATTR(in_temp_oversampling_ratio_available,
-	S_IRUGO, bmp280_show_temp_oversampling_avail, NULL, 0);
-
-static IIO_DEVICE_ATTR(in_pressure_oversampling_ratio_available,
-	S_IRUGO, bmp280_show_press_oversampling_avail, NULL, 0);
-
-static struct attribute *bmp280_attributes[] = {
-	&iio_dev_attr_in_temp_oversampling_ratio_available.dev_attr.attr,
-	&iio_dev_attr_in_pressure_oversampling_ratio_available.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group bmp280_attrs_group = {
-	.attrs = bmp280_attributes,
-};
-
 static const struct iio_info bmp280_info = {
 	.read_raw = &bmp280_read_raw,
+	.read_avail = &bmp280_read_avail,
 	.write_raw = &bmp280_write_raw,
-	.attrs = &bmp280_attrs_group,
 };
 
 static int bmp280_chip_config(struct bmp280_data *data)
@@ -713,7 +693,7 @@ static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas)
 	unsigned int ctrl;
 
 	if (data->use_eoc)
-		init_completion(&data->done);
+		reinit_completion(&data->done);
 
 	ret = regmap_write(data->regmap, BMP280_REG_CTRL_MEAS, ctrl_meas);
 	if (ret)
@@ -752,14 +732,14 @@ static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas)
 
 static int bmp180_read_adc_temp(struct bmp280_data *data, int *val)
 {
+	__be16 tmp;
 	int ret;
-	__be16 tmp = 0;
 
 	ret = bmp180_measure(data, BMP180_MEAS_TEMP);
 	if (ret)
 		return ret;
 
-	ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, (u8 *)&tmp, 2);
+	ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, &tmp, 2);
 	if (ret)
 		return ret;
 
@@ -856,7 +836,7 @@ static int bmp180_read_adc_press(struct bmp280_data *data, int *val)
 	if (ret)
 		return ret;
 
-	ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, (u8 *)&tmp, 3);
+	ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, &tmp, 3);
 	if (ret)
 		return ret;
 
@@ -965,10 +945,12 @@ static int bmp085_fetch_eoc_irq(struct device *dev,
 
 	irq_trig = irqd_get_trigger_type(irq_get_irq_data(irq));
 	if (irq_trig != IRQF_TRIGGER_RISING) {
-		dev_err(dev, "non-rising trigger given for EOC interrupt, "
-			"trying to enforce it\n");
+		dev_err(dev, "non-rising trigger given for EOC interrupt, trying to enforce it\n");
 		irq_trig = IRQF_TRIGGER_RISING;
 	}
+
+	init_completion(&data->done);
+
 	ret = devm_request_threaded_irq(dev,
 			irq,
 			bmp085_eoc_irq,
@@ -1082,9 +1064,9 @@ int bmp280_common_probe(struct device *dev,
 	usleep_range(data->start_up_time, data->start_up_time + 100);
 
 	/* Bring chip out of reset if there is an assigned GPIO line */
-	gpiod = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	/* Deassert the signal */
-	if (!IS_ERR(gpiod)) {
+	if (gpiod) {
 		dev_info(dev, "release reset\n");
 		gpiod_set_value(gpiod, 0);
 	}
diff --git a/drivers/iio/pressure/hid-sensor-press.c b/drivers/iio/pressure/hid-sensor-press.c
index 953235052155..5e6663f757ae 100644
--- a/drivers/iio/pressure/hid-sensor-press.c
+++ b/drivers/iio/pressure/hid-sensor-press.c
@@ -14,8 +14,6 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger_consumer.h>
-#include <linux/iio/triggered_buffer.h>
 #include "../common/hid-sensors/hid-sensor-trigger.h"
 
 #define CHANNEL_SCAN_INDEX_PRESSURE 0
@@ -290,18 +288,13 @@ static int hid_press_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = iio_triggered_buffer_setup(indio_dev, &iio_pollfunc_store_time,
-		NULL, NULL);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to initialize trigger buffer\n");
-		goto error_free_dev_mem;
-	}
 	atomic_set(&press_state->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 				&press_state->common_attributes);
 	if (ret) {
 		dev_err(&pdev->dev, "trigger setup failed\n");
-		goto error_unreg_buffer_funcs;
+		goto error_free_dev_mem;
 	}
 
 	ret = iio_device_register(indio_dev);
@@ -325,9 +318,7 @@ static int hid_press_probe(struct platform_device *pdev)
 error_iio_unreg:
 	iio_device_unregister(indio_dev);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&press_state->common_attributes);
-error_unreg_buffer_funcs:
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &press_state->common_attributes);
 error_free_dev_mem:
 	kfree(indio_dev->channels);
 	return ret;
@@ -342,8 +333,7 @@ static int hid_press_remove(struct platform_device *pdev)
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_PRESSURE);
 	iio_device_unregister(indio_dev);
-	hid_sensor_remove_trigger(&press_state->common_attributes);
-	iio_triggered_buffer_cleanup(indio_dev);
+	hid_sensor_remove_trigger(indio_dev, &press_state->common_attributes);
 	kfree(indio_dev->channels);
 
 	return 0;
diff --git a/drivers/iio/pressure/hp206c.c b/drivers/iio/pressure/hp206c.c
index 3ac3632e7242..1f931f5b7a65 100644
--- a/drivers/iio/pressure/hp206c.c
+++ b/drivers/iio/pressure/hp206c.c
@@ -18,6 +18,8 @@
 #include <linux/util_macros.h>
 #include <linux/acpi.h>
 
+#include <asm/unaligned.h>
+
 /* I2C commands: */
 #define HP206C_CMD_SOFT_RST	0x06
 
@@ -93,12 +95,12 @@ static int hp206c_read_20bit(struct i2c_client *client, u8 cmd)
 	int ret;
 	u8 values[3];
 
-	ret = i2c_smbus_read_i2c_block_data(client, cmd, 3, values);
+	ret = i2c_smbus_read_i2c_block_data(client, cmd, sizeof(values), values);
 	if (ret < 0)
 		return ret;
-	if (ret != 3)
+	if (ret != sizeof(values))
 		return -EIO;
-	return ((values[0] & 0xF) << 16) | (values[1] << 8) | (values[2]);
+	return get_unaligned_be24(&values[0]) & GENMASK(19, 0);
 }
 
 /* Spin for max 160ms until DEV_RDY is 1, or return error. */
diff --git a/drivers/iio/pressure/ms5611_i2c.c b/drivers/iio/pressure/ms5611_i2c.c
index 8089c59adce5..072c106dd66d 100644
--- a/drivers/iio/pressure/ms5611_i2c.c
+++ b/drivers/iio/pressure/ms5611_i2c.c
@@ -16,6 +16,8 @@
 #include <linux/module.h>
 #include <linux/of_device.h>
 
+#include <asm/unaligned.h>
+
 #include "ms5611.h"
 
 static int ms5611_i2c_reset(struct device *dev)
@@ -50,7 +52,7 @@ static int ms5611_i2c_read_adc(struct ms5611_state *st, s32 *val)
 	if (ret < 0)
 		return ret;
 
-	*val = (buf[0] << 16) | (buf[1] << 8) | buf[2];
+	*val = get_unaligned_be24(&buf[0]);
 
 	return 0;
 }
diff --git a/drivers/iio/pressure/ms5611_spi.c b/drivers/iio/pressure/ms5611_spi.c
index b463eaa799ab..4799aa57135e 100644
--- a/drivers/iio/pressure/ms5611_spi.c
+++ b/drivers/iio/pressure/ms5611_spi.c
@@ -11,6 +11,8 @@
 #include <linux/spi/spi.h>
 #include <linux/of_device.h>
 
+#include <asm/unaligned.h>
+
 #include "ms5611.h"
 
 static int ms5611_spi_reset(struct device *dev)
@@ -45,7 +47,7 @@ static int ms5611_spi_read_adc(struct device *dev, s32 *val)
 	if (ret < 0)
 		return ret;
 
-	*val = (buf[0] << 16) | (buf[1] << 8) | buf[2];
+	*val = get_unaligned_be24(&buf[0]);
 
 	return 0;
 }
diff --git a/drivers/iio/pressure/st_pressure_core.c b/drivers/iio/pressure/st_pressure_core.c
index bd972cec4830..789a2928504a 100644
--- a/drivers/iio/pressure/st_pressure_core.c
+++ b/drivers/iio/pressure/st_pressure_core.c
@@ -683,8 +683,7 @@ EXPORT_SYMBOL(st_press_get_settings);
 int st_press_common_probe(struct iio_dev *indio_dev)
 {
 	struct st_sensor_data *press_data = iio_priv(indio_dev);
-	struct st_sensors_platform_data *pdata =
-		(struct st_sensors_platform_data *)press_data->dev->platform_data;
+	struct st_sensors_platform_data *pdata = dev_get_platdata(press_data->dev);
 	int err;
 
 	indio_dev->modes = INDIO_DIRECT_MODE;
@@ -708,9 +707,7 @@ int st_press_common_probe(struct iio_dev *indio_dev)
 	indio_dev->channels = press_data->sensor_settings->ch;
 	indio_dev->num_channels = press_data->sensor_settings->num_ch;
 
-	press_data->current_fullscale =
-		(struct st_sensor_fullscale_avl *)
-			&press_data->sensor_settings->fs.fs_avl[0];
+	press_data->current_fullscale = &press_data->sensor_settings->fs.fs_avl[0];
 
 	press_data->odr = press_data->sensor_settings->odr.odr_avl[0].hz;
 
diff --git a/drivers/iio/pressure/zpa2326.c b/drivers/iio/pressure/zpa2326.c
index 99dfe33ee402..37fe851f89af 100644
--- a/drivers/iio/pressure/zpa2326.c
+++ b/drivers/iio/pressure/zpa2326.c
@@ -64,6 +64,7 @@
 #include <linux/iio/trigger.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
+#include <asm/unaligned.h>
 #include "zpa2326.h"
 
 /* 200 ms should be enough for the longest conversion time in one-shot mode. */
@@ -1005,22 +1006,20 @@ static int zpa2326_fetch_raw_sample(const struct iio_dev *indio_dev,
 	struct regmap *regs = ((struct zpa2326_private *)
 			       iio_priv(indio_dev))->regmap;
 	int            err;
+	u8             v[3];
 
 	switch (type) {
 	case IIO_PRESSURE:
 		zpa2326_dbg(indio_dev, "fetching raw pressure sample");
 
-		err = regmap_bulk_read(regs, ZPA2326_PRESS_OUT_XL_REG, value,
-				       3);
+		err = regmap_bulk_read(regs, ZPA2326_PRESS_OUT_XL_REG, v, sizeof(v));
 		if (err) {
 			zpa2326_warn(indio_dev, "failed to fetch pressure (%d)",
 				     err);
 			return err;
 		}
 
-		/* Pressure is a 24 bits wide little-endian unsigned int. */
-		*value = (((u8 *)value)[2] << 16) | (((u8 *)value)[1] << 8) |
-			 ((u8 *)value)[0];
+		*value = get_unaligned_le24(&v[0]);
 
 		return IIO_VAL_INT;
 
diff --git a/drivers/iio/proximity/Kconfig b/drivers/iio/proximity/Kconfig
index 37606d400805..12672a0e89ed 100644
--- a/drivers/iio/proximity/Kconfig
+++ b/drivers/iio/proximity/Kconfig
@@ -101,6 +101,19 @@ config SRF04
 	  To compile this driver as a module, choose M here: the
 	  module will be called srf04.
 
+config SX9310
+	tristate "SX9310/SX9311 Semtech proximity sensor"
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	select REGMAP_I2C
+	depends on I2C
+	help
+	  Say Y here to build a driver for Semtech's SX9310/SX9311 capacitive
+	  proximity/button sensor.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called sx9310.
+
 config SX9500
 	tristate "SX9500 Semtech proximity sensor"
 	select IIO_BUFFER
@@ -127,6 +140,17 @@ config SRF08
 	  To compile this driver as a module, choose M here: the
 	  module will be called srf08.
 
+config VCNL3020
+	tristate "VCNL3020 proximity sensor"
+	select REGMAP_I2C
+	depends on I2C
+	help
+	  Say Y here if you want to build a driver for the Vishay VCNL3020
+	  proximity sensor.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called vcnl3020.
+
 config VL53L0X_I2C
 	tristate "STMicroelectronics VL53L0X ToF ranger sensor (I2C)"
 	depends on I2C
diff --git a/drivers/iio/proximity/Makefile b/drivers/iio/proximity/Makefile
index c591b019304e..9c1aca1a8b79 100644
--- a/drivers/iio/proximity/Makefile
+++ b/drivers/iio/proximity/Makefile
@@ -12,6 +12,8 @@ obj-$(CONFIG_PING)		+= ping.o
 obj-$(CONFIG_RFD77402)		+= rfd77402.o
 obj-$(CONFIG_SRF04)		+= srf04.o
 obj-$(CONFIG_SRF08)		+= srf08.o
+obj-$(CONFIG_SX9310)		+= sx9310.o
 obj-$(CONFIG_SX9500)		+= sx9500.o
+obj-$(CONFIG_VCNL3020)		+= vcnl3020.o
 obj-$(CONFIG_VL53L0X_I2C)	+= vl53l0x-i2c.o
 
diff --git a/drivers/iio/proximity/ping.c b/drivers/iio/proximity/ping.c
index 12b893c5b0ee..2e99eeb27f2e 100644
--- a/drivers/iio/proximity/ping.c
+++ b/drivers/iio/proximity/ping.c
@@ -89,14 +89,14 @@ static irqreturn_t ping_handle_irq(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int ping_read(struct ping_data *data)
+static int ping_read(struct iio_dev *indio_dev)
 {
+	struct ping_data *data = iio_priv(indio_dev);
 	int ret;
 	ktime_t ktime_dt;
 	s64 dt_ns;
 	u32 time_ns, distance_mm;
 	struct platform_device *pdev = to_platform_device(data->dev);
-	struct iio_dev *indio_dev = iio_priv_to_dev(data);
 
 	/*
 	 * just one read-echo-cycle can take place at a time
@@ -228,7 +228,6 @@ static int ping_read_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *channel, int *val,
 			    int *val2, long info)
 {
-	struct ping_data *data = iio_priv(indio_dev);
 	int ret;
 
 	if (channel->type != IIO_DISTANCE)
@@ -236,7 +235,7 @@ static int ping_read_raw(struct iio_dev *indio_dev,
 
 	switch (info) {
 	case IIO_CHAN_INFO_RAW:
-		ret = ping_read(data);
+		ret = ping_read(indio_dev);
 		if (ret < 0)
 			return ret;
 		*val = ret;
diff --git a/drivers/iio/proximity/sx9310.c b/drivers/iio/proximity/sx9310.c
new file mode 100644
index 000000000000..d161f3061e35
--- /dev/null
+++ b/drivers/iio/proximity/sx9310.c
@@ -0,0 +1,1069 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Google LLC.
+ *
+ * Driver for Semtech's SX9310/SX9311 capacitive proximity/button solution.
+ * Based on SX9500 driver and Semtech driver using the input framework
+ * <https://my.syncplicity.com/share/teouwsim8niiaud/
+ *          linux-driver-SX9310_NoSmartHSensing>.
+ * Reworked April 2019 by Evan Green <evgreen@chromium.org>
+ * and January 2020 by Daniel Campello <campello@chromium.org>
+ */
+
+#include <linux/acpi.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pm.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include <linux/iio/buffer.h>
+#include <linux/iio/events.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
+
+/* Register definitions. */
+#define SX9310_REG_IRQ_SRC				0x00
+#define SX9310_REG_STAT0				0x01
+#define SX9310_REG_STAT1				0x02
+#define SX9310_REG_IRQ_MSK				0x03
+#define   SX9310_CONVDONE_IRQ				BIT(3)
+#define   SX9310_FAR_IRQ				BIT(5)
+#define   SX9310_CLOSE_IRQ				BIT(6)
+#define   SX9310_EVENT_IRQ				(SX9310_FAR_IRQ | \
+							 SX9310_CLOSE_IRQ)
+#define SX9310_REG_IRQ_FUNC				0x04
+
+#define SX9310_REG_PROX_CTRL0				0x10
+#define   SX9310_REG_PROX_CTRL0_PROXSTAT2		0x10
+#define   SX9310_REG_PROX_CTRL0_EN_MASK			0x0F
+#define SX9310_REG_PROX_CTRL1				0x11
+#define SX9310_REG_PROX_CTRL2				0x12
+#define   SX9310_REG_PROX_CTRL2_COMBMODE_ALL		0x80
+#define   SX9310_REG_PROX_CTRL2_SHIELDEN_DYNAMIC	0x04
+#define SX9310_REG_PROX_CTRL3				0x13
+#define   SX9310_REG_PROX_CTRL3_GAIN0_X8		0x0c
+#define   SX9310_REG_PROX_CTRL3_GAIN12_X4		0x02
+#define SX9310_REG_PROX_CTRL4				0x14
+#define   SX9310_REG_PROX_CTRL4_RESOLUTION_FINEST	0x07
+#define SX9310_REG_PROX_CTRL5				0x15
+#define   SX9310_REG_PROX_CTRL5_RANGE_SMALL		0xc0
+#define   SX9310_REG_PROX_CTRL5_STARTUPSENS_CS1		0x04
+#define   SX9310_REG_PROX_CTRL5_RAWFILT_1P25		0x02
+#define SX9310_REG_PROX_CTRL6				0x16
+#define   SX9310_REG_PROX_CTRL6_COMP_COMMON		0x20
+#define SX9310_REG_PROX_CTRL7				0x17
+#define   SX9310_REG_PROX_CTRL7_AVGNEGFILT_2		0x08
+#define   SX9310_REG_PROX_CTRL7_AVGPOSFILT_512		0x05
+#define SX9310_REG_PROX_CTRL8				0x18
+#define SX9310_REG_PROX_CTRL9				0x19
+#define   SX9310_REG_PROX_CTRL8_9_PTHRESH12_28		0x40
+#define   SX9310_REG_PROX_CTRL8_9_PTHRESH_96		0x88
+#define   SX9310_REG_PROX_CTRL8_9_BODYTHRESH_900	0x03
+#define   SX9310_REG_PROX_CTRL8_9_BODYTHRESH_1500	0x05
+#define SX9310_REG_PROX_CTRL10				0x1a
+#define   SX9310_REG_PROX_CTRL10_HYST_6PCT		0x10
+#define   SX9310_REG_PROX_CTRL10_CLOSE_DEBOUNCE_8	0x12
+#define   SX9310_REG_PROX_CTRL10_FAR_DEBOUNCE_8		0x03
+#define SX9310_REG_PROX_CTRL11				0x1b
+#define SX9310_REG_PROX_CTRL12				0x1c
+#define SX9310_REG_PROX_CTRL13				0x1d
+#define SX9310_REG_PROX_CTRL14				0x1e
+#define SX9310_REG_PROX_CTRL15				0x1f
+#define SX9310_REG_PROX_CTRL16				0x20
+#define SX9310_REG_PROX_CTRL17				0x21
+#define SX9310_REG_PROX_CTRL18				0x22
+#define SX9310_REG_PROX_CTRL19				0x23
+#define SX9310_REG_SAR_CTRL0				0x2a
+#define   SX9310_REG_SAR_CTRL0_SARDEB_4_SAMPLES		0x40
+#define   SX9310_REG_SAR_CTRL0_SARHYST_8		0x10
+#define SX9310_REG_SAR_CTRL1				0x2b
+/* Each increment of the slope register is 0.0078125. */
+#define   SX9310_REG_SAR_CTRL1_SLOPE(_hnslope)		(_hnslope / 78125)
+#define SX9310_REG_SAR_CTRL2				0x2c
+#define   SX9310_REG_SAR_CTRL2_SAROFFSET_DEFAULT	0x3c
+
+#define SX9310_REG_SENSOR_SEL				0x30
+
+#define SX9310_REG_USE_MSB				0x31
+#define SX9310_REG_USE_LSB				0x32
+
+#define SX9310_REG_AVG_MSB				0x33
+#define SX9310_REG_AVG_LSB				0x34
+
+#define SX9310_REG_DIFF_MSB				0x35
+#define SX9310_REG_DIFF_LSB				0x36
+
+#define SX9310_REG_OFFSET_MSB				0x37
+#define SX9310_REG_OFFSET_LSB				0x38
+
+#define SX9310_REG_SAR_MSB				0x39
+#define SX9310_REG_SAR_LSB				0x3a
+
+#define SX9310_REG_I2CADDR				0x40
+#define SX9310_REG_PAUSE				0x41
+#define SX9310_REG_WHOAMI				0x42
+#define   SX9310_WHOAMI_VALUE				0x01
+#define   SX9311_WHOAMI_VALUE				0x02
+
+#define SX9310_REG_RESET				0x7f
+#define   SX9310_SOFT_RESET				0xde
+
+#define SX9310_SCAN_PERIOD_MASK				GENMASK(7, 4)
+#define SX9310_SCAN_PERIOD_SHIFT			4
+
+#define SX9310_COMPSTAT_MASK				GENMASK(3, 0)
+
+/* 4 hardware channels, as defined in STAT0: COMB, CS2, CS1 and CS0. */
+#define SX9310_NUM_CHANNELS				4
+#define SX9310_CHAN_ENABLED_MASK			GENMASK(3, 0)
+
+struct sx9310_data {
+	/* Serialize access to registers and channel configuration */
+	struct mutex mutex;
+	struct i2c_client *client;
+	struct iio_trigger *trig;
+	struct regmap *regmap;
+	/*
+	 * Last reading of the proximity status for each channel.
+	 * We only send an event to user space when this changes.
+	 */
+	bool prox_stat[SX9310_NUM_CHANNELS];
+	bool trigger_enabled;
+	__be16 buffer[SX9310_NUM_CHANNELS +
+		      4]; /* 64-bit data + 64-bit timestamp */
+	/* Remember enabled channels and sample rate during suspend. */
+	unsigned int suspend_ctrl0;
+	struct completion completion;
+	unsigned int chan_read, chan_event;
+	int channel_users[SX9310_NUM_CHANNELS];
+	int whoami;
+};
+
+static const struct iio_event_spec sx9310_events[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_EITHER,
+		.mask_separate = BIT(IIO_EV_INFO_ENABLE),
+	},
+};
+
+#define SX9310_NAMED_CHANNEL(idx, name)					 \
+	{								 \
+		.type = IIO_PROXIMITY,					 \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		 \
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
+		.indexed = 1,						 \
+		.channel = idx,						 \
+		.extend_name = name,					 \
+		.address = SX9310_REG_DIFF_MSB,				 \
+		.event_spec = sx9310_events,				 \
+		.num_event_specs = ARRAY_SIZE(sx9310_events),		 \
+		.scan_index = idx,					 \
+		.scan_type = {						 \
+			.sign = 's',					 \
+			.realbits = 12,					 \
+			.storagebits = 16,				 \
+			.endianness = IIO_BE,				 \
+		},							 \
+	}
+#define SX9310_CHANNEL(idx) SX9310_NAMED_CHANNEL(idx, NULL)
+
+static const struct iio_chan_spec sx9310_channels[] = {
+	SX9310_CHANNEL(0),			/* CS0 */
+	SX9310_CHANNEL(1),			/* CS1 */
+	SX9310_CHANNEL(2),			/* CS2 */
+	SX9310_NAMED_CHANNEL(3, "comb"),	/* COMB */
+
+	IIO_CHAN_SOFT_TIMESTAMP(4),
+};
+
+/*
+ * Each entry contains the integer part (val) and the fractional part, in micro
+ * seconds. It conforms to the IIO output IIO_VAL_INT_PLUS_MICRO.
+ */
+static const struct {
+	int val;
+	int val2;
+} sx9310_samp_freq_table[] = {
+	{ 500, 0 }, /* 0000: Min (no idle time) */
+	{ 66, 666666 }, /* 0001: 15 ms */
+	{ 33, 333333 }, /* 0010: 30 ms (Typ.) */
+	{ 22, 222222 }, /* 0011: 45 ms */
+	{ 16, 666666 }, /* 0100: 60 ms */
+	{ 11, 111111 }, /* 0101: 90 ms */
+	{ 8, 333333 }, /* 0110: 120 ms */
+	{ 5, 0 }, /* 0111: 200 ms */
+	{ 2, 500000 }, /* 1000: 400 ms */
+	{ 1, 666666 }, /* 1001: 600 ms */
+	{ 1, 250000 }, /* 1010: 800 ms */
+	{ 1, 0 }, /* 1011: 1 s */
+	{ 0, 500000 }, /* 1100: 2 s */
+	{ 0, 333333 }, /* 1101: 3 s */
+	{ 0, 250000 }, /* 1110: 4 s */
+	{ 0, 200000 }, /* 1111: 5 s */
+};
+static const unsigned int sx9310_scan_period_table[] = {
+	2,   15,  30,  45,   60,   90,	 120,  200,
+	400, 600, 800, 1000, 2000, 3000, 4000, 5000,
+};
+
+static ssize_t sx9310_show_samp_freq_avail(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	size_t len = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sx9310_samp_freq_table); i++)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "%d.%d ",
+				 sx9310_samp_freq_table[i].val,
+				 sx9310_samp_freq_table[i].val2);
+	buf[len - 1] = '\n';
+	return len;
+}
+static IIO_DEV_ATTR_SAMP_FREQ_AVAIL(sx9310_show_samp_freq_avail);
+
+static const struct regmap_range sx9310_writable_reg_ranges[] = {
+	regmap_reg_range(SX9310_REG_IRQ_MSK, SX9310_REG_IRQ_FUNC),
+	regmap_reg_range(SX9310_REG_PROX_CTRL0, SX9310_REG_PROX_CTRL19),
+	regmap_reg_range(SX9310_REG_SAR_CTRL0, SX9310_REG_SAR_CTRL2),
+	regmap_reg_range(SX9310_REG_SENSOR_SEL, SX9310_REG_SENSOR_SEL),
+	regmap_reg_range(SX9310_REG_OFFSET_MSB, SX9310_REG_OFFSET_LSB),
+	regmap_reg_range(SX9310_REG_PAUSE, SX9310_REG_PAUSE),
+	regmap_reg_range(SX9310_REG_RESET, SX9310_REG_RESET),
+};
+
+static const struct regmap_access_table sx9310_writeable_regs = {
+	.yes_ranges = sx9310_writable_reg_ranges,
+	.n_yes_ranges = ARRAY_SIZE(sx9310_writable_reg_ranges),
+};
+
+static const struct regmap_range sx9310_readable_reg_ranges[] = {
+	regmap_reg_range(SX9310_REG_IRQ_SRC, SX9310_REG_IRQ_FUNC),
+	regmap_reg_range(SX9310_REG_PROX_CTRL0, SX9310_REG_PROX_CTRL19),
+	regmap_reg_range(SX9310_REG_SAR_CTRL0, SX9310_REG_SAR_CTRL2),
+	regmap_reg_range(SX9310_REG_SENSOR_SEL, SX9310_REG_SAR_LSB),
+	regmap_reg_range(SX9310_REG_I2CADDR, SX9310_REG_WHOAMI),
+	regmap_reg_range(SX9310_REG_RESET, SX9310_REG_RESET),
+};
+
+static const struct regmap_access_table sx9310_readable_regs = {
+	.yes_ranges = sx9310_readable_reg_ranges,
+	.n_yes_ranges = ARRAY_SIZE(sx9310_readable_reg_ranges),
+};
+
+static const struct regmap_range sx9310_volatile_reg_ranges[] = {
+	regmap_reg_range(SX9310_REG_IRQ_SRC, SX9310_REG_STAT1),
+	regmap_reg_range(SX9310_REG_USE_MSB, SX9310_REG_DIFF_LSB),
+	regmap_reg_range(SX9310_REG_SAR_MSB, SX9310_REG_SAR_LSB),
+	regmap_reg_range(SX9310_REG_RESET, SX9310_REG_RESET),
+};
+
+static const struct regmap_access_table sx9310_volatile_regs = {
+	.yes_ranges = sx9310_volatile_reg_ranges,
+	.n_yes_ranges = ARRAY_SIZE(sx9310_volatile_reg_ranges),
+};
+
+static const struct regmap_config sx9310_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.max_register = SX9310_REG_RESET,
+	.cache_type = REGCACHE_RBTREE,
+
+	.wr_table = &sx9310_writeable_regs,
+	.rd_table = &sx9310_readable_regs,
+	.volatile_table = &sx9310_volatile_regs,
+};
+
+static int sx9310_update_chan_en(struct sx9310_data *data,
+				 unsigned int chan_read,
+				 unsigned int chan_event)
+{
+	int ret;
+
+	if ((data->chan_read | data->chan_event) != (chan_read | chan_event)) {
+		ret = regmap_update_bits(data->regmap, SX9310_REG_PROX_CTRL0,
+					 SX9310_CHAN_ENABLED_MASK,
+					 chan_read | chan_event);
+		if (ret)
+			return ret;
+	}
+	data->chan_read = chan_read;
+	data->chan_event = chan_event;
+	return 0;
+}
+
+static int sx9310_get_read_channel(struct sx9310_data *data, int channel)
+{
+	return sx9310_update_chan_en(data, data->chan_read | BIT(channel),
+				     data->chan_event);
+}
+
+static int sx9310_put_read_channel(struct sx9310_data *data, int channel)
+{
+	return sx9310_update_chan_en(data, data->chan_read & ~BIT(channel),
+				     data->chan_event);
+}
+
+static int sx9310_get_event_channel(struct sx9310_data *data, int channel)
+{
+	return sx9310_update_chan_en(data, data->chan_read,
+				     data->chan_event | BIT(channel));
+}
+
+static int sx9310_put_event_channel(struct sx9310_data *data, int channel)
+{
+	return sx9310_update_chan_en(data, data->chan_read,
+				     data->chan_event & ~BIT(channel));
+}
+
+static int sx9310_enable_irq(struct sx9310_data *data, unsigned int irq)
+{
+	return regmap_update_bits(data->regmap, SX9310_REG_IRQ_MSK, irq, irq);
+}
+
+static int sx9310_disable_irq(struct sx9310_data *data, unsigned int irq)
+{
+	return regmap_update_bits(data->regmap, SX9310_REG_IRQ_MSK, irq, 0);
+}
+
+static int sx9310_read_prox_data(struct sx9310_data *data,
+				 const struct iio_chan_spec *chan, __be16 *val)
+{
+	int ret;
+
+	ret = regmap_write(data->regmap, SX9310_REG_SENSOR_SEL, chan->channel);
+	if (ret < 0)
+		return ret;
+
+	return regmap_bulk_read(data->regmap, chan->address, val, 2);
+}
+
+/*
+ * If we have no interrupt support, we have to wait for a scan period
+ * after enabling a channel to get a result.
+ */
+static int sx9310_wait_for_sample(struct sx9310_data *data)
+{
+	int ret;
+	unsigned int val;
+
+	ret = regmap_read(data->regmap, SX9310_REG_PROX_CTRL0, &val);
+	if (ret < 0)
+		return ret;
+
+	val = (val & SX9310_SCAN_PERIOD_MASK) >> SX9310_SCAN_PERIOD_SHIFT;
+
+	msleep(sx9310_scan_period_table[val]);
+
+	return 0;
+}
+
+static int sx9310_read_proximity(struct sx9310_data *data,
+				 const struct iio_chan_spec *chan, int *val)
+{
+	int ret = 0;
+	__be16 rawval;
+
+	mutex_lock(&data->mutex);
+
+	ret = sx9310_get_read_channel(data, chan->channel);
+	if (ret < 0)
+		goto out;
+
+	ret = sx9310_enable_irq(data, SX9310_CONVDONE_IRQ);
+	if (ret < 0)
+		goto out_put_channel;
+
+	mutex_unlock(&data->mutex);
+
+	if (data->client->irq > 0) {
+		ret = wait_for_completion_interruptible(&data->completion);
+		reinit_completion(&data->completion);
+	} else {
+		ret = sx9310_wait_for_sample(data);
+	}
+
+	mutex_lock(&data->mutex);
+
+	if (ret < 0)
+		goto out_disable_irq;
+
+	ret = sx9310_read_prox_data(data, chan, &rawval);
+	if (ret < 0)
+		goto out_disable_irq;
+
+	*val = sign_extend32(be16_to_cpu(rawval),
+			     (chan->address == SX9310_REG_DIFF_MSB ? 11 : 15));
+
+	ret = sx9310_disable_irq(data, SX9310_CONVDONE_IRQ);
+	if (ret < 0)
+		goto out_put_channel;
+
+	ret = sx9310_put_read_channel(data, chan->channel);
+	if (ret < 0)
+		goto out;
+
+	mutex_unlock(&data->mutex);
+
+	return IIO_VAL_INT;
+
+out_disable_irq:
+	sx9310_disable_irq(data, SX9310_CONVDONE_IRQ);
+out_put_channel:
+	sx9310_put_read_channel(data, chan->channel);
+out:
+	mutex_unlock(&data->mutex);
+
+	return ret;
+}
+
+static int sx9310_read_samp_freq(struct sx9310_data *data, int *val, int *val2)
+{
+	unsigned int regval;
+	int ret = regmap_read(data->regmap, SX9310_REG_PROX_CTRL0, &regval);
+
+	if (ret < 0)
+		return ret;
+
+	regval = (regval & SX9310_SCAN_PERIOD_MASK) >> SX9310_SCAN_PERIOD_SHIFT;
+	*val = sx9310_samp_freq_table[regval].val;
+	*val2 = sx9310_samp_freq_table[regval].val2;
+
+	return IIO_VAL_INT_PLUS_MICRO;
+}
+
+static int sx9310_read_raw(struct iio_dev *indio_dev,
+			   const struct iio_chan_spec *chan, int *val,
+			   int *val2, long mask)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+	int ret;
+
+	if (chan->type != IIO_PROXIMITY)
+		return -EINVAL;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		ret = sx9310_read_proximity(data, chan, val);
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return sx9310_read_samp_freq(data, val, val2);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int sx9310_set_samp_freq(struct sx9310_data *data, int val, int val2)
+{
+	int i, ret;
+
+	for (i = 0; i < ARRAY_SIZE(sx9310_samp_freq_table); i++)
+		if (val == sx9310_samp_freq_table[i].val &&
+		    val2 == sx9310_samp_freq_table[i].val2)
+			break;
+
+	if (i == ARRAY_SIZE(sx9310_samp_freq_table))
+		return -EINVAL;
+
+	mutex_lock(&data->mutex);
+
+	ret = regmap_update_bits(data->regmap, SX9310_REG_PROX_CTRL0,
+				 SX9310_SCAN_PERIOD_MASK,
+				 i << SX9310_SCAN_PERIOD_SHIFT);
+
+	mutex_unlock(&data->mutex);
+
+	return ret;
+}
+
+static int sx9310_write_raw(struct iio_dev *indio_dev,
+			    const struct iio_chan_spec *chan, int val, int val2,
+			    long mask)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+
+	if (chan->type != IIO_PROXIMITY)
+		return -EINVAL;
+
+	if (mask != IIO_CHAN_INFO_SAMP_FREQ)
+		return -EINVAL;
+
+	return sx9310_set_samp_freq(data, val, val2);
+}
+
+static irqreturn_t sx9310_irq_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct sx9310_data *data = iio_priv(indio_dev);
+
+	if (data->trigger_enabled)
+		iio_trigger_poll(data->trig);
+
+	/*
+	 * Even if no event is enabled, we need to wake the thread to
+	 * clear the interrupt state by reading SX9310_REG_IRQ_SRC.  It
+	 * is not possible to do that here because regmap_read takes a
+	 * mutex.
+	 */
+	return IRQ_WAKE_THREAD;
+}
+
+static void sx9310_push_events(struct iio_dev *indio_dev)
+{
+	int ret;
+	unsigned int val, chan;
+	struct sx9310_data *data = iio_priv(indio_dev);
+	s64 timestamp = iio_get_time_ns(indio_dev);
+
+	/* Read proximity state on all channels */
+	ret = regmap_read(data->regmap, SX9310_REG_STAT0, &val);
+	if (ret < 0) {
+		dev_err(&data->client->dev, "i2c transfer error in irq\n");
+		return;
+	}
+
+	for (chan = 0; chan < SX9310_NUM_CHANNELS; chan++) {
+		int dir;
+		u64 ev;
+		bool new_prox = val & BIT(chan);
+
+		if (!(data->chan_event & BIT(chan)))
+			continue;
+		if (new_prox == data->prox_stat[chan])
+			/* No change on this channel. */
+			continue;
+
+		dir = new_prox ? IIO_EV_DIR_FALLING : IIO_EV_DIR_RISING;
+		ev = IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, chan,
+					  IIO_EV_TYPE_THRESH, dir);
+
+		iio_push_event(indio_dev, ev, timestamp);
+		data->prox_stat[chan] = new_prox;
+	}
+}
+
+static irqreturn_t sx9310_irq_thread_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct sx9310_data *data = iio_priv(indio_dev);
+	int ret;
+	unsigned int val;
+
+	mutex_lock(&data->mutex);
+
+	ret = regmap_read(data->regmap, SX9310_REG_IRQ_SRC, &val);
+	if (ret < 0) {
+		dev_err(&data->client->dev, "i2c transfer error in irq\n");
+		goto out;
+	}
+
+	if (val & SX9310_EVENT_IRQ)
+		sx9310_push_events(indio_dev);
+
+	if (val & SX9310_CONVDONE_IRQ)
+		complete(&data->completion);
+
+out:
+	mutex_unlock(&data->mutex);
+
+	return IRQ_HANDLED;
+}
+
+static int sx9310_read_event_config(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+
+	return !!(data->chan_event & BIT(chan->channel));
+}
+
+static int sx9310_write_event_config(struct iio_dev *indio_dev,
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir, int state)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+	int ret;
+
+	/* If the state hasn't changed, there's nothing to do. */
+	if (!!(data->chan_event & BIT(chan->channel)) == state)
+		return 0;
+
+	mutex_lock(&data->mutex);
+	if (state) {
+		ret = sx9310_get_event_channel(data, chan->channel);
+		if (ret < 0)
+			goto out_unlock;
+		if (!(data->chan_event & ~BIT(chan->channel))) {
+			ret = sx9310_enable_irq(data, SX9310_EVENT_IRQ);
+			if (ret < 0)
+				sx9310_put_event_channel(data, chan->channel);
+		}
+	} else {
+		ret = sx9310_put_event_channel(data, chan->channel);
+		if (ret < 0)
+			goto out_unlock;
+		if (!data->chan_event) {
+			ret = sx9310_disable_irq(data, SX9310_EVENT_IRQ);
+			if (ret < 0)
+				sx9310_get_event_channel(data, chan->channel);
+		}
+	}
+
+out_unlock:
+	mutex_unlock(&data->mutex);
+	return ret;
+}
+
+static struct attribute *sx9310_attributes[] = {
+	&iio_dev_attr_sampling_frequency_available.dev_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group sx9310_attribute_group = {
+	.attrs = sx9310_attributes,
+};
+
+static const struct iio_info sx9310_info = {
+	.attrs = &sx9310_attribute_group,
+	.read_raw = sx9310_read_raw,
+	.write_raw = sx9310_write_raw,
+	.read_event_config = sx9310_read_event_config,
+	.write_event_config = sx9310_write_event_config,
+};
+
+static int sx9310_set_trigger_state(struct iio_trigger *trig, bool state)
+{
+	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig);
+	struct sx9310_data *data = iio_priv(indio_dev);
+	int ret = 0;
+
+	mutex_lock(&data->mutex);
+
+	if (state)
+		ret = sx9310_enable_irq(data, SX9310_CONVDONE_IRQ);
+	else if (!data->chan_read)
+		ret = sx9310_disable_irq(data, SX9310_CONVDONE_IRQ);
+	if (ret < 0)
+		goto out;
+
+	data->trigger_enabled = state;
+
+out:
+	mutex_unlock(&data->mutex);
+
+	return ret;
+}
+
+static const struct iio_trigger_ops sx9310_trigger_ops = {
+	.set_trigger_state = sx9310_set_trigger_state,
+};
+
+static irqreturn_t sx9310_trigger_handler(int irq, void *private)
+{
+	struct iio_poll_func *pf = private;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct sx9310_data *data = iio_priv(indio_dev);
+	__be16 val;
+	int bit, ret, i = 0;
+
+	mutex_lock(&data->mutex);
+
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->masklength) {
+		ret = sx9310_read_prox_data(data, &indio_dev->channels[bit],
+					    &val);
+		if (ret < 0)
+			goto out;
+
+		data->buffer[i++] = val;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, data->buffer,
+					   pf->timestamp);
+
+out:
+	mutex_unlock(&data->mutex);
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int sx9310_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+	unsigned int channels = 0;
+	int bit, ret;
+
+	mutex_lock(&data->mutex);
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->masklength)
+		channels |= BIT(indio_dev->channels[bit].channel);
+
+	ret = sx9310_update_chan_en(data, channels, data->chan_event);
+	mutex_unlock(&data->mutex);
+	return ret;
+}
+
+static int sx9310_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&data->mutex);
+	ret = sx9310_update_chan_en(data, 0, data->chan_event);
+	mutex_unlock(&data->mutex);
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops sx9310_buffer_setup_ops = {
+	.preenable = sx9310_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = iio_triggered_buffer_predisable,
+	.postdisable = sx9310_buffer_postdisable,
+};
+
+struct sx9310_reg_default {
+	u8 reg;
+	u8 def;
+};
+
+#define SX_INIT(_reg, _def)			\
+	{					\
+		.reg = SX9310_REG_##_reg,	\
+		.def = _def,			\
+	}
+
+static const struct sx9310_reg_default sx9310_default_regs[] = {
+	SX_INIT(IRQ_MSK, 0x00),
+	SX_INIT(IRQ_FUNC, 0x00),
+	/*
+	 * The lower 4 bits should not be set as it enable sensors measurements.
+	 * Turning the detection on before the configuration values are set to
+	 * good values can cause the device to return erroneous readings.
+	 */
+	SX_INIT(PROX_CTRL0, SX9310_REG_PROX_CTRL0_PROXSTAT2),
+	SX_INIT(PROX_CTRL1, 0x00),
+	SX_INIT(PROX_CTRL2, SX9310_REG_PROX_CTRL2_COMBMODE_ALL |
+			    SX9310_REG_PROX_CTRL2_SHIELDEN_DYNAMIC),
+	SX_INIT(PROX_CTRL3, SX9310_REG_PROX_CTRL3_GAIN0_X8 |
+			    SX9310_REG_PROX_CTRL3_GAIN12_X4),
+	SX_INIT(PROX_CTRL4, SX9310_REG_PROX_CTRL4_RESOLUTION_FINEST),
+	SX_INIT(PROX_CTRL5, SX9310_REG_PROX_CTRL5_RANGE_SMALL |
+			    SX9310_REG_PROX_CTRL5_STARTUPSENS_CS1 |
+			    SX9310_REG_PROX_CTRL5_RAWFILT_1P25),
+	SX_INIT(PROX_CTRL6, SX9310_REG_PROX_CTRL6_COMP_COMMON),
+	SX_INIT(PROX_CTRL7, SX9310_REG_PROX_CTRL7_AVGNEGFILT_2 |
+			    SX9310_REG_PROX_CTRL7_AVGPOSFILT_512),
+	SX_INIT(PROX_CTRL8, SX9310_REG_PROX_CTRL8_9_PTHRESH_96 |
+			    SX9310_REG_PROX_CTRL8_9_BODYTHRESH_1500),
+	SX_INIT(PROX_CTRL9, SX9310_REG_PROX_CTRL8_9_PTHRESH12_28 |
+			    SX9310_REG_PROX_CTRL8_9_BODYTHRESH_900),
+	SX_INIT(PROX_CTRL10, SX9310_REG_PROX_CTRL10_HYST_6PCT |
+			     SX9310_REG_PROX_CTRL10_CLOSE_DEBOUNCE_8 |
+			     SX9310_REG_PROX_CTRL10_FAR_DEBOUNCE_8),
+	SX_INIT(PROX_CTRL11, 0x00),
+	SX_INIT(PROX_CTRL12, 0x00),
+	SX_INIT(PROX_CTRL13, 0x00),
+	SX_INIT(PROX_CTRL14, 0x00),
+	SX_INIT(PROX_CTRL15, 0x00),
+	SX_INIT(PROX_CTRL16, 0x00),
+	SX_INIT(PROX_CTRL17, 0x00),
+	SX_INIT(PROX_CTRL18, 0x00),
+	SX_INIT(PROX_CTRL19, 0x00),
+	SX_INIT(SAR_CTRL0, SX9310_REG_SAR_CTRL0_SARDEB_4_SAMPLES |
+			   SX9310_REG_SAR_CTRL0_SARHYST_8),
+	SX_INIT(SAR_CTRL1, SX9310_REG_SAR_CTRL1_SLOPE(10781250)),
+	SX_INIT(SAR_CTRL2, SX9310_REG_SAR_CTRL2_SAROFFSET_DEFAULT),
+};
+
+/* Activate all channels and perform an initial compensation. */
+static int sx9310_init_compensation(struct iio_dev *indio_dev)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+	int i, ret;
+	unsigned int val;
+	unsigned int ctrl0;
+
+	ret = regmap_read(data->regmap, SX9310_REG_PROX_CTRL0, &ctrl0);
+	if (ret < 0)
+		return ret;
+
+	/* run the compensation phase on all channels */
+	ret = regmap_write(data->regmap, SX9310_REG_PROX_CTRL0,
+			   ctrl0 | SX9310_REG_PROX_CTRL0_EN_MASK);
+	if (ret < 0)
+		return ret;
+
+	for (i = 100; i >= 0; i--) {
+		msleep(20);
+		ret = regmap_read(data->regmap, SX9310_REG_STAT1, &val);
+		if (ret < 0)
+			goto out;
+		if (!(val & SX9310_COMPSTAT_MASK))
+			break;
+	}
+
+	if (i < 0) {
+		dev_err(&data->client->dev,
+			"initial compensation timed out: 0x%02x", val);
+		ret = -ETIMEDOUT;
+	}
+
+out:
+	regmap_write(data->regmap, SX9310_REG_PROX_CTRL0, ctrl0);
+	return ret;
+}
+
+static int sx9310_init_device(struct iio_dev *indio_dev)
+{
+	struct sx9310_data *data = iio_priv(indio_dev);
+	const struct sx9310_reg_default *initval;
+	int ret;
+	unsigned int i, val;
+
+	ret = regmap_write(data->regmap, SX9310_REG_RESET, SX9310_SOFT_RESET);
+	if (ret < 0)
+		return ret;
+
+	usleep_range(1000, 2000); /* power-up time is ~1ms. */
+
+	/* Clear reset interrupt state by reading SX9310_REG_IRQ_SRC. */
+	ret = regmap_read(data->regmap, SX9310_REG_IRQ_SRC, &val);
+	if (ret < 0)
+		return ret;
+
+	/* Program some sane defaults. */
+	for (i = 0; i < ARRAY_SIZE(sx9310_default_regs); i++) {
+		initval = &sx9310_default_regs[i];
+		ret = regmap_write(data->regmap, initval->reg, initval->def);
+		if (ret < 0)
+			return ret;
+	}
+
+	return sx9310_init_compensation(indio_dev);
+}
+
+static int sx9310_set_indio_dev_name(struct device *dev,
+				     struct iio_dev *indio_dev,
+				     const struct i2c_device_id *id, int whoami)
+{
+	const struct acpi_device_id *acpi_id;
+
+	/* id will be NULL when enumerated via ACPI */
+	if (id) {
+		if (id->driver_data != whoami)
+			dev_err(dev, "WHOAMI does not match i2c_device_id: %s",
+				id->name);
+	} else if (ACPI_HANDLE(dev)) {
+		acpi_id = acpi_match_device(dev->driver->acpi_match_table, dev);
+		if (!acpi_id)
+			return -ENODEV;
+		if (acpi_id->driver_data != whoami)
+			dev_err(dev, "WHOAMI does not match acpi_device_id: %s",
+				acpi_id->id);
+	} else
+		return -ENODEV;
+
+	switch (whoami) {
+	case SX9310_WHOAMI_VALUE:
+		indio_dev->name = "sx9310";
+		break;
+	case SX9311_WHOAMI_VALUE:
+		indio_dev->name = "sx9311";
+		break;
+	default:
+		dev_err(dev, "unexpected WHOAMI response: %u", whoami);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int sx9310_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct iio_dev *indio_dev;
+	struct sx9310_data *data;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (indio_dev == NULL)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+	mutex_init(&data->mutex);
+	init_completion(&data->completion);
+
+	data->regmap = devm_regmap_init_i2c(client, &sx9310_regmap_config);
+	if (IS_ERR(data->regmap))
+		return PTR_ERR(data->regmap);
+
+	ret = regmap_read(data->regmap, SX9310_REG_WHOAMI, &data->whoami);
+	if (ret < 0) {
+		dev_err(&client->dev, "error in reading WHOAMI register: %d",
+			ret);
+		return ret;
+	}
+
+	ret = sx9310_set_indio_dev_name(&client->dev, indio_dev, id,
+					data->whoami);
+	if (ret < 0)
+		return ret;
+
+	ACPI_COMPANION_SET(&indio_dev->dev, ACPI_COMPANION(&client->dev));
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->channels = sx9310_channels;
+	indio_dev->num_channels = ARRAY_SIZE(sx9310_channels);
+	indio_dev->info = &sx9310_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	i2c_set_clientdata(client, indio_dev);
+
+	ret = sx9310_init_device(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	if (client->irq) {
+		ret = devm_request_threaded_irq(&client->dev, client->irq,
+						sx9310_irq_handler,
+						sx9310_irq_thread_handler,
+						IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+						"sx9310_event", indio_dev);
+		if (ret < 0)
+			return ret;
+
+		data->trig =
+			devm_iio_trigger_alloc(&client->dev, "%s-dev%d",
+					       indio_dev->name, indio_dev->id);
+		if (!data->trig)
+			return -ENOMEM;
+
+		data->trig->dev.parent = &client->dev;
+		data->trig->ops = &sx9310_trigger_ops;
+		iio_trigger_set_drvdata(data->trig, indio_dev);
+
+		ret = devm_iio_trigger_register(&client->dev, data->trig);
+		if (ret)
+			return ret;
+	}
+
+	ret = devm_iio_triggered_buffer_setup(&client->dev, indio_dev,
+					      iio_pollfunc_store_time,
+					      sx9310_trigger_handler,
+					      &sx9310_buffer_setup_ops);
+	if (ret < 0)
+		return ret;
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static int __maybe_unused sx9310_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
+	struct sx9310_data *data = iio_priv(indio_dev);
+	u8 ctrl0;
+	int ret;
+
+	disable_irq_nosync(data->client->irq);
+
+	mutex_lock(&data->mutex);
+	ret = regmap_read(data->regmap, SX9310_REG_PROX_CTRL0,
+			  &data->suspend_ctrl0);
+
+	if (ret)
+		goto out;
+
+	ctrl0 = data->suspend_ctrl0 & ~SX9310_REG_PROX_CTRL0_EN_MASK;
+	ret = regmap_write(data->regmap, SX9310_REG_PROX_CTRL0, ctrl0);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(data->regmap, SX9310_REG_PAUSE, 0);
+
+out:
+	mutex_unlock(&data->mutex);
+	return ret;
+}
+
+static int __maybe_unused sx9310_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
+	struct sx9310_data *data = iio_priv(indio_dev);
+	int ret;
+
+	mutex_lock(&data->mutex);
+	ret = regmap_write(data->regmap, SX9310_REG_PAUSE, 1);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(data->regmap, SX9310_REG_PROX_CTRL0,
+			   data->suspend_ctrl0);
+
+out:
+	mutex_unlock(&data->mutex);
+
+	enable_irq(data->client->irq);
+
+	return ret;
+}
+
+static const struct dev_pm_ops sx9310_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(sx9310_suspend, sx9310_resume)
+};
+
+static const struct acpi_device_id sx9310_acpi_match[] = {
+	{ "STH9310", SX9310_WHOAMI_VALUE },
+	{ "STH9311", SX9311_WHOAMI_VALUE },
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, sx9310_acpi_match);
+
+static const struct of_device_id sx9310_of_match[] = {
+	{ .compatible = "semtech,sx9310" },
+	{ .compatible = "semtech,sx9311" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, sx9310_of_match);
+
+static const struct i2c_device_id sx9310_id[] = {
+	{ "sx9310", SX9310_WHOAMI_VALUE },
+	{ "sx9311", SX9311_WHOAMI_VALUE },
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, sx9310_id);
+
+static struct i2c_driver sx9310_driver = {
+	.driver = {
+		.name	= "sx9310",
+		.acpi_match_table = ACPI_PTR(sx9310_acpi_match),
+		.of_match_table = of_match_ptr(sx9310_of_match),
+		.pm = &sx9310_pm_ops,
+	},
+	.probe		= sx9310_probe,
+	.id_table	= sx9310_id,
+};
+module_i2c_driver(sx9310_driver);
+
+MODULE_AUTHOR("Gwendal Grignou <gwendal@chromium.org>");
+MODULE_AUTHOR("Daniel Campello <campello@chromium.org>");
+MODULE_DESCRIPTION("Driver for Semtech SX9310/SX9311 proximity sensor");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/proximity/vcnl3020.c b/drivers/iio/proximity/vcnl3020.c
new file mode 100644
index 000000000000..9ff1a164c2e6
--- /dev/null
+++ b/drivers/iio/proximity/vcnl3020.c
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Support for Vishay VCNL3020 proximity sensor on i2c bus.
+ * Based on Vishay VCNL4000 driver code.
+ *
+ * TODO: interrupts.
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/regmap.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+#define VCNL3020_PROD_ID	0x21
+
+#define VCNL_COMMAND		0x80 /* Command register */
+#define VCNL_PROD_REV		0x81 /* Product ID and Revision ID */
+#define VCNL_PROXIMITY_RATE	0x82 /* Rate of Proximity Measurement */
+#define VCNL_LED_CURRENT	0x83 /* IR LED current for proximity mode */
+#define VCNL_PS_RESULT_HI	0x87 /* Proximity result register, MSB */
+#define VCNL_PS_RESULT_LO	0x88 /* Proximity result register, LSB */
+#define VCNL_PS_ICR		0x89 /* Interrupt Control Register */
+#define VCNL_PS_LO_THR_HI	0x8a /* High byte of low threshold value */
+#define VCNL_PS_LO_THR_LO	0x8b /* Low byte of low threshold value */
+#define VCNL_PS_HI_THR_HI	0x8c /* High byte of high threshold value */
+#define VCNL_PS_HI_THR_LO	0x8d /* Low byte of high threshold value */
+#define VCNL_ISR		0x8e /* Interrupt Status Register */
+#define VCNL_PS_MOD_ADJ		0x8f /* Proximity Modulator Timing Adjustment */
+
+/* Bit masks for COMMAND register */
+#define VCNL_PS_RDY		BIT(5) /* proximity data ready? */
+#define VCNL_PS_OD		BIT(3) /* start on-demand proximity
+					* measurement
+					*/
+
+#define VCNL_ON_DEMAND_TIMEOUT_US	100000
+#define VCNL_POLL_US			20000
+
+/**
+ * struct vcnl3020_data - vcnl3020 specific data.
+ * @regmap:	device register map.
+ * @dev:	vcnl3020 device.
+ * @rev:	revision id.
+ * @lock:	lock for protecting access to device hardware registers.
+ */
+struct vcnl3020_data {
+	struct regmap *regmap;
+	struct device *dev;
+	u8 rev;
+	struct mutex lock;
+};
+
+/**
+ * struct vcnl3020_property - vcnl3020 property.
+ * @name:	property name.
+ * @reg:	i2c register offset.
+ * @conversion_func:	conversion function.
+ */
+struct vcnl3020_property {
+	const char *name;
+	u32 reg;
+	u32 (*conversion_func)(u32 *val);
+};
+
+static u32 microamp_to_reg(u32 *val)
+{
+	/*
+	 * An example of conversion from uA to reg val:
+	 * 200000 uA == 200 mA == 20
+	 */
+	return *val /= 10000;
+};
+
+static struct vcnl3020_property vcnl3020_led_current_property = {
+	.name = "vishay,led-current-microamp",
+	.reg = VCNL_LED_CURRENT,
+	.conversion_func = microamp_to_reg,
+};
+
+static int vcnl3020_get_and_apply_property(struct vcnl3020_data *data,
+					   struct vcnl3020_property prop)
+{
+	int rc;
+	u32 val;
+
+	rc = device_property_read_u32(data->dev, prop.name, &val);
+	if (rc)
+		return 0;
+
+	if (prop.conversion_func)
+		prop.conversion_func(&val);
+
+	rc = regmap_write(data->regmap, prop.reg, val);
+	if (rc) {
+		dev_err(data->dev, "Error (%d) setting property (%s)\n",
+			rc, prop.name);
+	}
+
+	return rc;
+}
+
+static int vcnl3020_init(struct vcnl3020_data *data)
+{
+	int rc;
+	unsigned int reg;
+
+	rc = regmap_read(data->regmap, VCNL_PROD_REV, &reg);
+	if (rc) {
+		dev_err(data->dev,
+			"Error (%d) reading product revision\n", rc);
+		return rc;
+	}
+
+	if (reg != VCNL3020_PROD_ID) {
+		dev_err(data->dev,
+			"Product id (%x) did not match vcnl3020 (%x)\n", reg,
+			VCNL3020_PROD_ID);
+		return -ENODEV;
+	}
+
+	data->rev = reg;
+	mutex_init(&data->lock);
+
+	return vcnl3020_get_and_apply_property(data,
+					       vcnl3020_led_current_property);
+};
+
+static int vcnl3020_measure_proximity(struct vcnl3020_data *data, int *val)
+{
+	int rc;
+	unsigned int reg;
+	__be16 res;
+
+	mutex_lock(&data->lock);
+
+	rc = regmap_write(data->regmap, VCNL_COMMAND, VCNL_PS_OD);
+	if (rc)
+		goto err_unlock;
+
+	/* wait for data to become ready */
+	rc = regmap_read_poll_timeout(data->regmap, VCNL_COMMAND, reg,
+				      reg & VCNL_PS_RDY, VCNL_POLL_US,
+				      VCNL_ON_DEMAND_TIMEOUT_US);
+	if (rc) {
+		dev_err(data->dev,
+			"Error (%d) reading vcnl3020 command register\n", rc);
+		goto err_unlock;
+	}
+
+	/* high & low result bytes read */
+	rc = regmap_bulk_read(data->regmap, VCNL_PS_RESULT_HI, &res,
+			      sizeof(res));
+	if (rc)
+		goto err_unlock;
+
+	*val = be16_to_cpu(res);
+
+err_unlock:
+	mutex_unlock(&data->lock);
+
+	return rc;
+}
+
+static const struct iio_chan_spec vcnl3020_channels[] = {
+	{
+		.type = IIO_PROXIMITY,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+	},
+};
+
+static int vcnl3020_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan, int *val,
+			     int *val2, long mask)
+{
+	int rc;
+	struct vcnl3020_data *data = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		rc = vcnl3020_measure_proximity(data, val);
+		if (rc)
+			return rc;
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info vcnl3020_info = {
+	.read_raw = vcnl3020_read_raw,
+};
+
+static const struct regmap_config vcnl3020_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 8,
+	.max_register	= VCNL_PS_MOD_ADJ,
+};
+
+static int vcnl3020_probe(struct i2c_client *client)
+{
+	struct vcnl3020_data *data;
+	struct iio_dev *indio_dev;
+	struct regmap *regmap;
+	int rc;
+
+	regmap = devm_regmap_init_i2c(client, &vcnl3020_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&client->dev, "regmap_init failed\n");
+		return PTR_ERR(regmap);
+	}
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	i2c_set_clientdata(client, indio_dev);
+	data->regmap = regmap;
+	data->dev = &client->dev;
+
+	rc = vcnl3020_init(data);
+	if (rc)
+		return rc;
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->info = &vcnl3020_info;
+	indio_dev->channels = vcnl3020_channels;
+	indio_dev->num_channels = ARRAY_SIZE(vcnl3020_channels);
+	indio_dev->name = "vcnl3020";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct of_device_id vcnl3020_of_match[] = {
+	{
+		.compatible = "vishay,vcnl3020",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, vcnl3020_of_match);
+
+static struct i2c_driver vcnl3020_driver = {
+	.driver = {
+		.name   = "vcnl3020",
+		.of_match_table = vcnl3020_of_match,
+	},
+	.probe_new  = vcnl3020_probe,
+};
+module_i2c_driver(vcnl3020_driver);
+
+MODULE_AUTHOR("Ivan Mikhaylov <i.mikhaylov@yadro.com>");
+MODULE_DESCRIPTION("Vishay VCNL3020 proximity sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/temperature/hid-sensor-temperature.c b/drivers/iio/temperature/hid-sensor-temperature.c
index eda55b9c1e9b..8d1f434f109d 100644
--- a/drivers/iio/temperature/hid-sensor-temperature.c
+++ b/drivers/iio/temperature/hid-sensor-temperature.c
@@ -7,8 +7,6 @@
 #include <linux/hid-sensor-hub.h>
 #include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
-#include <linux/iio/triggered_buffer.h>
-#include <linux/iio/trigger_consumer.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 
@@ -230,12 +228,8 @@ static int hid_temperature_probe(struct platform_device *pdev)
 	indio_dev->name = name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = devm_iio_triggered_buffer_setup(&pdev->dev, indio_dev,
-					&iio_pollfunc_store_time, NULL, NULL);
-	if (ret)
-		return ret;
-
 	atomic_set(&temp_st->common_attributes.data_ready, 0);
+
 	ret = hid_sensor_setup_trigger(indio_dev, name,
 				&temp_st->common_attributes);
 	if (ret)
@@ -258,7 +252,7 @@ static int hid_temperature_probe(struct platform_device *pdev)
 error_remove_callback:
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_TEMPERATURE);
 error_remove_trigger:
-	hid_sensor_remove_trigger(&temp_st->common_attributes);
+	hid_sensor_remove_trigger(indio_dev, &temp_st->common_attributes);
 	return ret;
 }
 
@@ -270,7 +264,7 @@ static int hid_temperature_remove(struct platform_device *pdev)
 	struct temperature_state *temp_st = iio_priv(indio_dev);
 
 	sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_TEMPERATURE);
-	hid_sensor_remove_trigger(&temp_st->common_attributes);
+	hid_sensor_remove_trigger(indio_dev, &temp_st->common_attributes);
 
 	return 0;
 }
diff --git a/drivers/iio/temperature/ltc2983.c b/drivers/iio/temperature/ltc2983.c
index d39c0d6b77f1..8976e8d59826 100644
--- a/drivers/iio/temperature/ltc2983.c
+++ b/drivers/iio/temperature/ltc2983.c
@@ -390,8 +390,8 @@ static struct ltc2983_custom_sensor *__ltc2983_custom_sensor_new(
 	 * For custom steinhart, the full u32 is taken. For all the others
 	 * the MSB is discarded.
 	 */
-	const u8 n_size = (is_steinhart == true) ? 4 : 3;
-	const u8 e_size = (is_steinhart == true) ? sizeof(u32) : sizeof(u64);
+	const u8 n_size = is_steinhart ? 4 : 3;
+	const u8 e_size = is_steinhart ? sizeof(u32) : sizeof(u64);
 
 	n_entries = of_property_count_elems_of_size(np, propname, e_size);
 	/* n_entries must be an even number */
diff --git a/drivers/iio/temperature/max31856.c b/drivers/iio/temperature/max31856.c
index b4cb21ab2e85..b4c49a5d3685 100644
--- a/drivers/iio/temperature/max31856.c
+++ b/drivers/iio/temperature/max31856.c
@@ -14,6 +14,7 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/util_macros.h>
+#include <asm/unaligned.h>
 #include <dt-bindings/iio/temperature/thermocouple.h>
 /*
  * The MSB of the register value determines whether the following byte will
@@ -168,7 +169,7 @@ static int max31856_thermocouple_read(struct max31856_data *data,
 		if (ret)
 			return ret;
 		/* Skip last 5 dead bits of LTCBL */
-		*val = (reg_val[0] << 16 | reg_val[1] << 8 | reg_val[2]) >> 5;
+		*val = get_unaligned_be24(&reg_val[0]) >> 5;
 		/* Check 7th bit of LTCBH reg. value for sign*/
 		if (reg_val[0] & 0x80)
 			*val -= 0x80000;
@@ -185,7 +186,7 @@ static int max31856_thermocouple_read(struct max31856_data *data,
 		/* Get Cold Junction Temp. offset register value */
 		offset_cjto = reg_val[0];
 		/* Get CJTH and CJTL value and skip last 2 dead bits of CJTL */
-		*val = (reg_val[1] << 8 | reg_val[2]) >> 2;
+		*val = get_unaligned_be16(&reg_val[1]) >> 2;
 		/* As per datasheet add offset into CJTH and CJTL */
 		*val += offset_cjto;
 		/* Check 7th bit of CJTH reg. value for sign */
diff --git a/drivers/iio/trigger/iio-trig-hrtimer.c b/drivers/iio/trigger/iio-trig-hrtimer.c
index a5e670726717..f59bf8d58586 100644
--- a/drivers/iio/trigger/iio-trig-hrtimer.c
+++ b/drivers/iio/trigger/iio-trig-hrtimer.c
@@ -4,7 +4,7 @@
  *
  * Copyright (C) Intuitive Aerial AB
  * Written by Marten Svanfeldt, marten@intuitiveaerial.com
- * Copyright (C) 2012, Analog Device Inc.
+ * Copyright (C) 2012, Analog Devices Inc.
  *	Author: Lars-Peter Clausen <lars@metafoo.de>
  * Copyright (C) 2015, Intel Corporation
  */
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index ade86388434f..a83f9eb86bfe 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -8,7 +8,7 @@ menuconfig INFINIBAND
 	depends on !ALPHA
 	select IRQ_POLL
 	select DIMLIB
-	---help---
+	help
 	  Core support for InfiniBand (IB).  Make sure to also select
 	  any protocols you wish to use as well as drivers for your
 	  InfiniBand hardware.
@@ -18,7 +18,7 @@ if INFINIBAND
 config INFINIBAND_USER_MAD
 	tristate "InfiniBand userspace MAD support"
 	depends on INFINIBAND
-	---help---
+	help
 	  Userspace InfiniBand Management Datagram (MAD) support.  This
 	  is the kernel side of the userspace MAD support, which allows
 	  userspace processes to send and receive MADs. You will also
@@ -28,7 +28,7 @@ config INFINIBAND_USER_MAD
 config INFINIBAND_USER_ACCESS
 	tristate "InfiniBand userspace access (verbs and CM)"
 	depends on MMU
-	---help---
+	help
 	  Userspace InfiniBand access support.  This enables the
 	  kernel side of userspace verbs and the userspace
 	  communication manager (CM).  This allows userspace processes
@@ -40,7 +40,7 @@ config INFINIBAND_USER_ACCESS
 config INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI
 	bool "Allow experimental legacy verbs in new ioctl uAPI  (EXPERIMENTAL)"
 	depends on INFINIBAND_USER_ACCESS
-	---help---
+	help
 	  IOCTL based uAPI support for Infiniband is enabled by default for
 	  new verbs only. This allows userspace to invoke the IOCTL based uAPI
 	  for current legacy verbs too.
@@ -57,7 +57,7 @@ config INFINIBAND_ON_DEMAND_PAGING
 	select MMU_NOTIFIER
 	select INTERVAL_TREE
 	default y
-	---help---
+	help
 	  On demand paging support for the InfiniBand subsystem.
 	  Together with driver support this allows registration of
 	  memory regions without pinning their pages, fetching the
@@ -67,7 +67,7 @@ config INFINIBAND_ADDR_TRANS
 	bool "RDMA/CM"
 	depends on INFINIBAND
 	default y
-	---help---
+	help
 	  Support for RDMA communication manager (CM).
 	  This allows for a generic connection abstraction over RDMA.
 
@@ -75,7 +75,7 @@ config INFINIBAND_ADDR_TRANS_CONFIGFS
 	bool
 	depends on INFINIBAND_ADDR_TRANS && CONFIGFS_FS && !(INFINIBAND=y && CONFIGFS_FS=m)
 	default y
-	---help---
+	help
 	  ConfigFS support for RDMA communication manager (CM).
 	  This allows the user to config the default GID type that the CM
 	  uses for each device, when initiaing new connections.
@@ -107,6 +107,7 @@ source "drivers/infiniband/ulp/srpt/Kconfig"
 
 source "drivers/infiniband/ulp/iser/Kconfig"
 source "drivers/infiniband/ulp/isert/Kconfig"
+source "drivers/infiniband/ulp/rtrs/Kconfig"
 
 source "drivers/infiniband/ulp/opa_vnic/Kconfig"
 
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index d1b14887960e..24cb71a16a28 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -8,11 +8,11 @@ obj-$(CONFIG_INFINIBAND_USER_MAD) +=	ib_umad.o
 obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y)
 
 ib_core-y :=			packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
-				device.o fmr_pool.o cache.o netlink.o \
+				device.o cache.o netlink.o \
 				roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
 				multicast.o mad.o smi.o agent.o mad_rmpp.o \
 				nldev.o restrack.o counters.o ib_core_uverbs.o \
-				trace.o
+				trace.o lag.o
 
 ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o
 ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o
@@ -36,6 +36,9 @@ ib_uverbs-y :=			uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
 				uverbs_std_types_flow_action.o uverbs_std_types_dm.o \
 				uverbs_std_types_mr.o uverbs_std_types_counters.o \
 				uverbs_uapi.o uverbs_std_types_device.o \
-				uverbs_std_types_async_fd.o
+				uverbs_std_types_async_fd.o \
+				uverbs_std_types_srq.o \
+				uverbs_std_types_wq.o \
+				uverbs_std_types_qp.o
 ib_uverbs-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
 ib_uverbs-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 1753a9801b70..3a98439bba83 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -371,6 +371,8 @@ static int fetch_ha(const struct dst_entry *dst, struct rdma_dev_addr *dev_addr,
 		(const void *)&dst_in6->sin6_addr;
 	sa_family_t family = dst_in->sa_family;
 
+	might_sleep();
+
 	/* If we have a gateway in IB mode then it must be an IB network */
 	if (has_gateway(dst, family) && dev_addr->network == RDMA_NETWORK_IB)
 		return ib_nl_fetch_ha(dev_addr, daddr, seq, family);
@@ -727,6 +729,8 @@ int roce_resolve_route_from_path(struct sa_path_rec *rec,
 	struct rdma_dev_addr dev_addr = {};
 	int ret;
 
+	might_sleep();
+
 	if (rec->roce.route_resolved)
 		return 0;
 
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 17f14e0eafe4..9ce787e37e22 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -66,6 +66,8 @@ static const char * const ibcm_rej_reason_strs[] = {
 	[IB_CM_REJ_INVALID_CLASS_VERSION]	= "invalid class version",
 	[IB_CM_REJ_INVALID_FLOW_LABEL]		= "invalid flow label",
 	[IB_CM_REJ_INVALID_ALT_FLOW_LABEL]	= "invalid alt flow label",
+	[IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
+		"vendor option is not supported",
 };
 
 const char *__attribute_const__ ibcm_reject_msg(int reason)
@@ -81,8 +83,11 @@ const char *__attribute_const__ ibcm_reject_msg(int reason)
 EXPORT_SYMBOL(ibcm_reject_msg);
 
 struct cm_id_private;
-static void cm_add_one(struct ib_device *device);
+struct cm_work;
+static int cm_add_one(struct ib_device *device);
 static void cm_remove_one(struct ib_device *device, void *client_data);
+static void cm_process_work(struct cm_id_private *cm_id_priv,
+			    struct cm_work *work);
 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
 				   struct ib_cm_sidr_rep_param *param);
 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
@@ -287,6 +292,8 @@ struct cm_id_private {
 
 	struct list_head work_list;
 	atomic_t work_count;
+
+	struct rdma_ucm_ece ece;
 };
 
 static void cm_work_handler(struct work_struct *work);
@@ -474,24 +481,19 @@ static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 				       grh, &av->ah_attr);
 }
 
-static int add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,
-				  struct cm_av *av,
-				  struct cm_port *port)
+static void add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,
+				   struct cm_av *av, struct cm_port *port)
 {
 	unsigned long flags;
-	int ret = 0;
 
 	spin_lock_irqsave(&cm.lock, flags);
-
 	if (&cm_id_priv->av == av)
 		list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
 	else if (&cm_id_priv->alt_av == av)
 		list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
 	else
-		ret = -EINVAL;
-
+		WARN_ON(true);
 	spin_unlock_irqrestore(&cm.lock, flags);
-	return ret;
 }
 
 static struct cm_port *
@@ -572,12 +574,7 @@ static int cm_init_av_by_path(struct sa_path_rec *path,
 		return ret;
 
 	av->timeout = path->packet_life_time + 1;
-
-	ret = add_cm_id_to_port_list(cm_id_priv, av, port);
-	if (ret) {
-		rdma_destroy_ah_attr(&new_ah_attr);
-		return ret;
-	}
+	add_cm_id_to_port_list(cm_id_priv, av, port);
 	rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
 	return 0;
 }
@@ -587,11 +584,6 @@ static u32 cm_local_id(__be32 local_id)
 	return (__force u32) (local_id ^ cm.random_id_operand);
 }
 
-static void cm_free_id(__be32 local_id)
-{
-	xa_erase_irq(&cm.local_id_table, cm_local_id(local_id));
-}
-
 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
 {
 	struct cm_id_private *cm_id_priv;
@@ -698,9 +690,10 @@ static struct cm_id_private * cm_find_listen(struct ib_device *device,
 		cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
 		if ((cm_id_priv->id.service_mask & service_id) ==
 		     cm_id_priv->id.service_id &&
-		    (cm_id_priv->id.device == device))
+		    (cm_id_priv->id.device == device)) {
+			refcount_inc(&cm_id_priv->refcount);
 			return cm_id_priv;
-
+		}
 		if (device < cm_id_priv->id.device)
 			node = node->rb_left;
 		else if (device > cm_id_priv->id.device)
@@ -745,12 +738,14 @@ static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
 	return NULL;
 }
 
-static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
-						   __be32 remote_id)
+static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
+					       __be32 remote_id)
 {
 	struct rb_node *node = cm.remote_id_table.rb_node;
 	struct cm_timewait_info *timewait_info;
+	struct cm_id_private *res = NULL;
 
+	spin_lock_irq(&cm.lock);
 	while (node) {
 		timewait_info = rb_entry(node, struct cm_timewait_info,
 					 remote_id_node);
@@ -762,10 +757,14 @@ static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
 			node = node->rb_left;
 		else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
 			node = node->rb_right;
-		else
-			return timewait_info;
+		else {
+			res = cm_acquire_id(timewait_info->work.local_id,
+					     timewait_info->work.remote_id);
+			break;
+		}
 	}
-	return NULL;
+	spin_unlock_irq(&cm.lock);
+	return res;
 }
 
 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
@@ -917,6 +916,35 @@ static void cm_free_work(struct cm_work *work)
 	kfree(work);
 }
 
+static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
+				 struct cm_work *work)
+{
+	bool immediate;
+
+	/*
+	 * To deliver the event to the user callback we have the drop the
+	 * spinlock, however, we need to ensure that the user callback is single
+	 * threaded and receives events in the temporal order. If there are
+	 * already events being processed then thread new events onto a list,
+	 * the thread currently processing will pick them up.
+	 */
+	immediate = atomic_inc_and_test(&cm_id_priv->work_count);
+	if (!immediate) {
+		list_add_tail(&work->list, &cm_id_priv->work_list);
+		/*
+		 * This routine always consumes incoming reference. Once queued
+		 * to the work_list then a reference is held by the thread
+		 * currently running cm_process_work() and this reference is not
+		 * needed.
+		 */
+		cm_deref_id(cm_id_priv);
+	}
+	spin_unlock_irq(&cm_id_priv->lock);
+
+	if (immediate)
+		cm_process_work(cm_id_priv, work);
+}
+
 static inline int cm_convert_to_ms(int iba_time)
 {
 	/* approximate conversion to ms from 4.096us x 2^iba_time */
@@ -942,8 +970,10 @@ static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
 	return min(31, ack_timeout);
 }
 
-static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
+static void cm_remove_remote(struct cm_id_private *cm_id_priv)
 {
+	struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
+
 	if (timewait_info->inserted_remote_id) {
 		rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
 		timewait_info->inserted_remote_id = 0;
@@ -982,7 +1012,7 @@ static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 		return;
 
 	spin_lock_irqsave(&cm.lock, flags);
-	cm_cleanup_timewait(cm_id_priv->timewait_info);
+	cm_remove_remote(cm_id_priv);
 	list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
 	spin_unlock_irqrestore(&cm.lock, flags);
 
@@ -1001,6 +1031,11 @@ static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 				   msecs_to_jiffies(wait_time));
 	spin_unlock_irqrestore(&cm.lock, flags);
 
+	/*
+	 * The timewait_info is converted into a work and gets freed during
+	 * cm_free_work() in cm_timewait_handler().
+	 */
+	BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
 	cm_id_priv->timewait_info = NULL;
 }
 
@@ -1013,7 +1048,7 @@ static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
 	cm_id_priv->id.state = IB_CM_IDLE;
 	if (cm_id_priv->timewait_info) {
 		spin_lock_irqsave(&cm.lock, flags);
-		cm_cleanup_timewait(cm_id_priv->timewait_info);
+		cm_remove_remote(cm_id_priv);
 		spin_unlock_irqrestore(&cm.lock, flags);
 		kfree(cm_id_priv->timewait_info);
 		cm_id_priv->timewait_info = NULL;
@@ -1076,7 +1111,9 @@ retest:
 	case IB_CM_REP_SENT:
 	case IB_CM_MRA_REP_RCVD:
 		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
-		/* Fall through */
+		cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
+				   0, NULL, 0);
+		goto retest;
 	case IB_CM_MRA_REQ_SENT:
 	case IB_CM_REP_RCVD:
 	case IB_CM_MRA_REP_SENT:
@@ -1101,7 +1138,7 @@ retest:
 	case IB_CM_TIMEWAIT:
 		/*
 		 * The cm_acquire_id in cm_timewait_handler will stop working
-		 * once we do cm_free_id() below, so just move to idle here for
+		 * once we do xa_erase below, so just move to idle here for
 		 * consistency.
 		 */
 		cm_id->state = IB_CM_IDLE;
@@ -1114,7 +1151,7 @@ retest:
 	spin_lock(&cm.lock);
 	/* Required for cleanup paths related cm_req_handler() */
 	if (cm_id_priv->timewait_info) {
-		cm_cleanup_timewait(cm_id_priv->timewait_info);
+		cm_remove_remote(cm_id_priv);
 		kfree(cm_id_priv->timewait_info);
 		cm_id_priv->timewait_info = NULL;
 	}
@@ -1131,7 +1168,7 @@ retest:
 	spin_unlock(&cm.lock);
 	spin_unlock_irq(&cm_id_priv->lock);
 
-	cm_free_id(cm_id->local_id);
+	xa_erase_irq(&cm.local_id_table, cm_local_id(cm_id->local_id));
 	cm_deref_id(cm_id_priv);
 	wait_for_completion(&cm_id_priv->comp);
 	while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
@@ -1287,6 +1324,13 @@ static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
 	hdr->tid	   = tid;
 }
 
+static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
+				  __be64 tid, u32 attr_mod)
+{
+	cm_format_mad_hdr(hdr, attr_id, tid);
+	hdr->attr_mod = cpu_to_be32(attr_mod);
+}
+
 static void cm_format_req(struct cm_req_msg *req_msg,
 			  struct cm_id_private *cm_id_priv,
 			  struct ib_cm_req_param *param)
@@ -1299,8 +1343,8 @@ static void cm_format_req(struct cm_req_msg *req_msg,
 		pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
 					      pri_path->opa.slid);
 
-	cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
-			  cm_form_tid(cm_id_priv));
+	cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
+			      cm_form_tid(cm_id_priv), param->ece.attr_mod);
 
 	IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
 		be32_to_cpu(cm_id_priv->id.local_id));
@@ -1423,6 +1467,7 @@ static void cm_format_req(struct cm_req_msg *req_msg,
 			cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
 				       alt_path->packet_life_time));
 	}
+	IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
 
 	if (param->private_data && param->private_data_len)
 		IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
@@ -1779,6 +1824,9 @@ static void cm_format_req_event(struct cm_work *work,
 	param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
 	param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
 	param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
+	param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
+	param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
+
 	work->cm_event.private_data =
 		IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
 }
@@ -1927,7 +1975,6 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,
 	struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
 	struct cm_timewait_info *timewait_info;
 	struct cm_req_msg *req_msg;
-	struct ib_cm_id *cm_id;
 
 	req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
 
@@ -1948,7 +1995,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,
 	/* Check for stale connections. */
 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
 	if (timewait_info) {
-		cm_cleanup_timewait(cm_id_priv->timewait_info);
+		cm_remove_remote(cm_id_priv);
 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
 					   timewait_info->work.remote_id);
 
@@ -1957,8 +2004,7 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,
 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
 			     NULL, 0);
 		if (cur_cm_id_priv) {
-			cm_id = &cur_cm_id_priv->id;
-			ib_send_cm_dreq(cm_id, NULL, 0);
+			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
 			cm_deref_id(cur_cm_id_priv);
 		}
 		return NULL;
@@ -1969,14 +2015,13 @@ static struct cm_id_private * cm_match_req(struct cm_work *work,
 		cm_id_priv->id.device,
 		cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
 	if (!listen_cm_id_priv) {
-		cm_cleanup_timewait(cm_id_priv->timewait_info);
+		cm_remove_remote(cm_id_priv);
 		spin_unlock_irq(&cm.lock);
 		cm_issue_rej(work->port, work->mad_recv_wc,
 			     IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
 			     NULL, 0);
 		return NULL;
 	}
-	refcount_inc(&listen_cm_id_priv->refcount);
 	spin_unlock_irq(&cm.lock);
 	return listen_cm_id_priv;
 }
@@ -2153,9 +2198,7 @@ static int cm_req_handler(struct cm_work *work)
 
 	/* Refcount belongs to the event, pairs with cm_process_work() */
 	refcount_inc(&cm_id_priv->refcount);
-	atomic_inc(&cm_id_priv->work_count);
-	spin_unlock_irq(&cm_id_priv->lock);
-	cm_process_work(cm_id_priv, work);
+	cm_queue_work_unlock(cm_id_priv, work);
 	/*
 	 * Since this ID was just created and was not made visible to other MAD
 	 * handlers until the cm_finalize_id() above we know that the
@@ -2176,7 +2219,8 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg,
 			  struct cm_id_private *cm_id_priv,
 			  struct ib_cm_rep_param *param)
 {
-	cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
+	cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
+			      param->ece.attr_mod);
 	IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
 		be32_to_cpu(cm_id_priv->id.local_id));
 	IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
@@ -2203,6 +2247,10 @@ static void cm_format_rep(struct cm_rep_msg *rep_msg,
 		IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
 	}
 
+	IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
+	IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
+	IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
+
 	if (param->private_data && param->private_data_len)
 		IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
 			    param->private_data_len);
@@ -2350,6 +2398,11 @@ static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
 	param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
 	param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
 	param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
+	param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
+	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
+	param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
+	param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
+
 	work->cm_event.private_data =
 		IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
 }
@@ -2404,7 +2457,6 @@ static int cm_rep_handler(struct cm_work *work)
 	struct cm_rep_msg *rep_msg;
 	int ret;
 	struct cm_id_private *cur_cm_id_priv;
-	struct ib_cm_id *cm_id;
 	struct cm_timewait_info *timewait_info;
 
 	rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
@@ -2454,9 +2506,7 @@ static int cm_rep_handler(struct cm_work *work)
 	/* Check for a stale connection. */
 	timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
 	if (timewait_info) {
-		rb_erase(&cm_id_priv->timewait_info->remote_id_node,
-			 &cm.remote_id_table);
-		cm_id_priv->timewait_info->inserted_remote_id = 0;
+		cm_remove_remote(cm_id_priv);
 		cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
 					   timewait_info->work.remote_id);
 
@@ -2472,8 +2522,7 @@ static int cm_rep_handler(struct cm_work *work)
 			IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
 
 		if (cur_cm_id_priv) {
-			cm_id = &cur_cm_id_priv->id;
-			ib_send_cm_dreq(cm_id, NULL, 0);
+			ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
 			cm_deref_id(cur_cm_id_priv);
 		}
 
@@ -2501,15 +2550,7 @@ static int cm_rep_handler(struct cm_work *work)
 				       cm_id_priv->alt_av.timeout - 1);
 
 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 
 error:
@@ -2520,7 +2561,6 @@ error:
 static int cm_establish_handler(struct cm_work *work)
 {
 	struct cm_id_private *cm_id_priv;
-	int ret;
 
 	/* See comment in cm_establish about lookup. */
 	cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
@@ -2534,15 +2574,7 @@ static int cm_establish_handler(struct cm_work *work)
 	}
 
 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 out:
 	cm_deref_id(cm_id_priv);
@@ -2553,7 +2585,6 @@ static int cm_rtu_handler(struct cm_work *work)
 {
 	struct cm_id_private *cm_id_priv;
 	struct cm_rtu_msg *rtu_msg;
-	int ret;
 
 	rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
 	cm_id_priv = cm_acquire_id(
@@ -2576,15 +2607,7 @@ static int cm_rtu_handler(struct cm_work *work)
 	cm_id_priv->id.state = IB_CM_ESTABLISHED;
 
 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 out:
 	cm_deref_id(cm_id_priv);
@@ -2777,7 +2800,6 @@ static int cm_dreq_handler(struct cm_work *work)
 	struct cm_id_private *cm_id_priv;
 	struct cm_dreq_msg *dreq_msg;
 	struct ib_mad_send_buf *msg = NULL;
-	int ret;
 
 	dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
 	cm_id_priv = cm_acquire_id(
@@ -2842,15 +2864,7 @@ static int cm_dreq_handler(struct cm_work *work)
 	}
 	cm_id_priv->id.state = IB_CM_DREQ_RCVD;
 	cm_id_priv->tid = dreq_msg->hdr.tid;
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 
 unlock:	spin_unlock_irq(&cm_id_priv->lock);
@@ -2862,7 +2876,6 @@ static int cm_drep_handler(struct cm_work *work)
 {
 	struct cm_id_private *cm_id_priv;
 	struct cm_drep_msg *drep_msg;
-	int ret;
 
 	drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
 	cm_id_priv = cm_acquire_id(
@@ -2883,15 +2896,7 @@ static int cm_drep_handler(struct cm_work *work)
 	cm_enter_timewait(cm_id_priv);
 
 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 out:
 	cm_deref_id(cm_id_priv);
@@ -2987,24 +2992,15 @@ static void cm_format_rej_event(struct cm_work *work)
 
 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
 {
-	struct cm_timewait_info *timewait_info;
 	struct cm_id_private *cm_id_priv;
 	__be32 remote_id;
 
 	remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
 
 	if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
-		spin_lock_irq(&cm.lock);
-		timewait_info = cm_find_remote_id(
+		cm_id_priv = cm_find_remote_id(
 			*((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
 			remote_id);
-		if (!timewait_info) {
-			spin_unlock_irq(&cm.lock);
-			return NULL;
-		}
-		cm_id_priv =
-			cm_acquire_id(timewait_info->work.local_id, remote_id);
-		spin_unlock_irq(&cm.lock);
 	} else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
 		   CM_MSG_RESPONSE_REQ)
 		cm_id_priv = cm_acquire_id(
@@ -3022,7 +3018,6 @@ static int cm_rej_handler(struct cm_work *work)
 {
 	struct cm_id_private *cm_id_priv;
 	struct cm_rej_msg *rej_msg;
-	int ret;
 
 	rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
 	cm_id_priv = cm_acquire_rejected_id(rej_msg);
@@ -3068,19 +3063,10 @@ static int cm_rej_handler(struct cm_work *work)
 			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
 			 cm_id_priv->id.state);
 		spin_unlock_irq(&cm_id_priv->lock);
-		ret = -EINVAL;
 		goto out;
 	}
 
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 out:
 	cm_deref_id(cm_id_priv);
@@ -3190,7 +3176,7 @@ static int cm_mra_handler(struct cm_work *work)
 {
 	struct cm_id_private *cm_id_priv;
 	struct cm_mra_msg *mra_msg;
-	int timeout, ret;
+	int timeout;
 
 	mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
 	cm_id_priv = cm_acquire_mraed_id(mra_msg);
@@ -3250,15 +3236,7 @@ static int cm_mra_handler(struct cm_work *work)
 
 	cm_id_priv->msg->context[1] = (void *) (unsigned long)
 				      cm_id_priv->id.state;
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 out:
 	spin_unlock_irq(&cm_id_priv->lock);
@@ -3393,15 +3371,7 @@ static int cm_lap_handler(struct cm_work *work)
 
 	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
 	cm_id_priv->tid = lap_msg->hdr.tid;
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 
 unlock:	spin_unlock_irq(&cm_id_priv->lock);
@@ -3413,7 +3383,6 @@ static int cm_apr_handler(struct cm_work *work)
 {
 	struct cm_id_private *cm_id_priv;
 	struct cm_apr_msg *apr_msg;
-	int ret;
 
 	/* Currently Alternate path messages are not supported for
 	 * RoCE link layer.
@@ -3448,16 +3417,7 @@ static int cm_apr_handler(struct cm_work *work)
 	cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
 	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 	cm_id_priv->msg = NULL;
-
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 out:
 	cm_deref_id(cm_id_priv);
@@ -3468,7 +3428,6 @@ static int cm_timewait_handler(struct cm_work *work)
 {
 	struct cm_timewait_info *timewait_info;
 	struct cm_id_private *cm_id_priv;
-	int ret;
 
 	timewait_info = container_of(work, struct cm_timewait_info, work);
 	spin_lock_irq(&cm.lock);
@@ -3487,15 +3446,7 @@ static int cm_timewait_handler(struct cm_work *work)
 		goto out;
 	}
 	cm_id_priv->id.state = IB_CM_IDLE;
-	ret = atomic_inc_and_test(&cm_id_priv->work_count);
-	if (!ret)
-		list_add_tail(&work->list, &cm_id_priv->work_list);
-	spin_unlock_irq(&cm_id_priv->lock);
-
-	if (ret)
-		cm_process_work(cm_id_priv, work);
-	else
-		cm_deref_id(cm_id_priv);
+	cm_queue_work_unlock(cm_id_priv, work);
 	return 0;
 out:
 	cm_deref_id(cm_id_priv);
@@ -3642,7 +3593,6 @@ static int cm_sidr_req_handler(struct cm_work *work)
 					    .status = IB_SIDR_UNSUPPORTED });
 		goto out; /* No match. */
 	}
-	refcount_inc(&listen_cm_id_priv->refcount);
 	spin_unlock_irq(&cm.lock);
 
 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
@@ -3674,8 +3624,8 @@ static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
 			       struct cm_id_private *cm_id_priv,
 			       struct ib_cm_sidr_rep_param *param)
 {
-	cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
-			  cm_id_priv->tid);
+	cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
+			      cm_id_priv->tid, param->ece.attr_mod);
 	IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
 		be32_to_cpu(cm_id_priv->id.remote_id));
 	IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
@@ -3683,6 +3633,10 @@ static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
 	IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
 		be64_to_cpu(cm_id_priv->id.service_id));
 	IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
+	IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
+		param->ece.vendor_id & 0xFF);
+	IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
+		(param->ece.vendor_id >> 8) & 0xFF);
 
 	if (param->info && param->info_length)
 		IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
@@ -4384,7 +4338,7 @@ static void cm_remove_port_fs(struct cm_port *port)
 
 }
 
-static void cm_add_one(struct ib_device *ib_device)
+static int cm_add_one(struct ib_device *ib_device)
 {
 	struct cm_device *cm_dev;
 	struct cm_port *port;
@@ -4403,7 +4357,7 @@ static void cm_add_one(struct ib_device *ib_device)
 	cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
 			 GFP_KERNEL);
 	if (!cm_dev)
-		return;
+		return -ENOMEM;
 
 	cm_dev->ib_device = ib_device;
 	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
@@ -4415,8 +4369,10 @@ static void cm_add_one(struct ib_device *ib_device)
 			continue;
 
 		port = kzalloc(sizeof *port, GFP_KERNEL);
-		if (!port)
+		if (!port) {
+			ret = -ENOMEM;
 			goto error1;
+		}
 
 		cm_dev->port[i-1] = port;
 		port->cm_dev = cm_dev;
@@ -4437,8 +4393,10 @@ static void cm_add_one(struct ib_device *ib_device)
 							cm_recv_handler,
 							port,
 							0);
-		if (IS_ERR(port->mad_agent))
+		if (IS_ERR(port->mad_agent)) {
+			ret = PTR_ERR(port->mad_agent);
 			goto error2;
+		}
 
 		ret = ib_modify_port(ib_device, i, 0, &port_modify);
 		if (ret)
@@ -4447,15 +4405,17 @@ static void cm_add_one(struct ib_device *ib_device)
 		count++;
 	}
 
-	if (!count)
+	if (!count) {
+		ret = -EOPNOTSUPP;
 		goto free;
+	}
 
 	ib_set_client_data(ib_device, &cm_client, cm_dev);
 
 	write_lock_irqsave(&cm.device_lock, flags);
 	list_add_tail(&cm_dev->list, &cm.device_list);
 	write_unlock_irqrestore(&cm.device_lock, flags);
-	return;
+	return 0;
 
 error3:
 	ib_unregister_mad_agent(port->mad_agent);
@@ -4477,6 +4437,7 @@ error1:
 	}
 free:
 	kfree(cm_dev);
+	return ret;
 }
 
 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
@@ -4491,9 +4452,6 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
 	unsigned long flags;
 	int i;
 
-	if (!cm_dev)
-		return;
-
 	write_lock_irqsave(&cm.device_lock, flags);
 	list_del(&cm_dev->list);
 	write_unlock_irqrestore(&cm.device_lock, flags);
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 26e6f7df247b..3d7cc9f0f3d4 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -91,7 +91,13 @@ const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
 }
 EXPORT_SYMBOL(rdma_reject_msg);
 
-bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
+/**
+ * rdma_is_consumer_reject - return true if the consumer rejected the connect
+ *                           request.
+ * @id: Communication identifier that received the REJECT event.
+ * @reason: Value returned in the REJECT event status field.
+ */
+static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
 {
 	if (rdma_ib_or_roce(id->device, id->port_num))
 		return reason == IB_CM_REJ_CONSUMER_DEFINED;
@@ -102,7 +108,6 @@ bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
 	WARN_ON_ONCE(1);
 	return false;
 }
-EXPORT_SYMBOL(rdma_is_consumer_reject);
 
 const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
 				      struct rdma_cm_event *ev, u8 *data_len)
@@ -148,7 +153,7 @@ struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)
 }
 EXPORT_SYMBOL(rdma_res_to_id);
 
-static void cma_add_one(struct ib_device *device);
+static int cma_add_one(struct ib_device *device);
 static void cma_remove_one(struct ib_device *device, void *client_data);
 
 static struct ib_client cma_client = {
@@ -479,6 +484,7 @@ static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
 		rdma_restrack_kadd(&id_priv->res);
 	else
 		rdma_restrack_uadd(&id_priv->res);
+	trace_cm_id_attach(id_priv, cma_dev->device);
 }
 
 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
@@ -883,7 +889,6 @@ struct rdma_cm_id *__rdma_create_id(struct net *net,
 	id_priv->id.route.addr.dev_addr.net = get_net(net);
 	id_priv->seq_num &= 0x00ffffff;
 
-	trace_cm_id_create(id_priv);
 	return &id_priv->id;
 }
 EXPORT_SYMBOL(__rdma_create_id);
@@ -1906,6 +1911,9 @@ static void cma_set_rep_event_data(struct rdma_cm_event *event,
 	event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
 	event->param.conn.srq = rep_data->srq;
 	event->param.conn.qp_num = rep_data->remote_qpn;
+
+	event->ece.vendor_id = rep_data->ece.vendor_id;
+	event->ece.attr_mod = rep_data->ece.attr_mod;
 }
 
 static int cma_cm_event_handler(struct rdma_id_private *id_priv,
@@ -2124,6 +2132,9 @@ static void cma_set_req_event_data(struct rdma_cm_event *event,
 	event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
 	event->param.conn.srq = req_data->srq;
 	event->param.conn.qp_num = req_data->remote_qpn;
+
+	event->ece.vendor_id = req_data->ece.vendor_id;
+	event->ece.attr_mod = req_data->ece.attr_mod;
 }
 
 static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,
@@ -2904,6 +2915,24 @@ static int iboe_tos_to_sl(struct net_device *ndev, int tos)
 		return 0;
 }
 
+static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv)
+{
+	struct sockaddr_in6 *addr6;
+	u16 dport, sport;
+	u32 hash, fl;
+
+	addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv);
+	fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK;
+	if ((cma_family(id_priv) != AF_INET6) || !fl) {
+		dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv)));
+		sport = be16_to_cpu(cma_port(cma_src_addr(id_priv)));
+		hash = (u32)sport * 31 + dport;
+		fl = hash & IB_GRH_FLOWLABEL_MASK;
+	}
+
+	return cpu_to_be32(fl);
+}
+
 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 {
 	struct rdma_route *route = &id_priv->id.route;
@@ -2970,6 +2999,11 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 		goto err2;
 	}
 
+	if (rdma_protocol_roce_udp_encap(id_priv->id.device,
+					 id_priv->id.port_num))
+		route->path_rec->flow_label =
+			cma_get_roce_udp_flow_label(id_priv);
+
 	cma_init_resolve_route_work(work, id_priv);
 	queue_work(cma_wq, &work->work);
 
@@ -3919,6 +3953,8 @@ static int cma_connect_ib(struct rdma_id_private *id_priv,
 	req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
 	req.max_cm_retries = CMA_MAX_CM_RETRIES;
 	req.srq = id_priv->srq ? 1 : 0;
+	req.ece.vendor_id = id_priv->ece.vendor_id;
+	req.ece.attr_mod = id_priv->ece.attr_mod;
 
 	trace_cm_send_req(id_priv);
 	ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
@@ -4008,6 +4044,27 @@ err:
 }
 EXPORT_SYMBOL(rdma_connect);
 
+/**
+ * rdma_connect_ece - Initiate an active connection request with ECE data.
+ * @id: Connection identifier to connect.
+ * @conn_param: Connection information used for connected QPs.
+ * @ece: ECE parameters
+ *
+ * See rdma_connect() explanation.
+ */
+int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
+		     struct rdma_ucm_ece *ece)
+{
+	struct rdma_id_private *id_priv =
+		container_of(id, struct rdma_id_private, id);
+
+	id_priv->ece.vendor_id = ece->vendor_id;
+	id_priv->ece.attr_mod = ece->attr_mod;
+
+	return rdma_connect(id, conn_param);
+}
+EXPORT_SYMBOL(rdma_connect_ece);
+
 static int cma_accept_ib(struct rdma_id_private *id_priv,
 			 struct rdma_conn_param *conn_param)
 {
@@ -4033,6 +4090,8 @@ static int cma_accept_ib(struct rdma_id_private *id_priv,
 	rep.flow_control = conn_param->flow_control;
 	rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
 	rep.srq = id_priv->srq ? 1 : 0;
+	rep.ece.vendor_id = id_priv->ece.vendor_id;
+	rep.ece.attr_mod = id_priv->ece.attr_mod;
 
 	trace_cm_send_rep(id_priv);
 	ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
@@ -4080,7 +4139,11 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
 			return ret;
 		rep.qp_num = id_priv->qp_num;
 		rep.qkey = id_priv->qkey;
+
+		rep.ece.vendor_id = id_priv->ece.vendor_id;
+		rep.ece.attr_mod = id_priv->ece.attr_mod;
 	}
+
 	rep.private_data = private_data;
 	rep.private_data_len = private_data_len;
 
@@ -4133,11 +4196,24 @@ int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
 	return 0;
 reject:
 	cma_modify_qp_err(id_priv);
-	rdma_reject(id, NULL, 0);
+	rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
 	return ret;
 }
 EXPORT_SYMBOL(__rdma_accept);
 
+int __rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
+		      const char *caller, struct rdma_ucm_ece *ece)
+{
+	struct rdma_id_private *id_priv =
+		container_of(id, struct rdma_id_private, id);
+
+	id_priv->ece.vendor_id = ece->vendor_id;
+	id_priv->ece.attr_mod = ece->attr_mod;
+
+	return __rdma_accept(id, conn_param, caller);
+}
+EXPORT_SYMBOL(__rdma_accept_ece);
+
 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
 {
 	struct rdma_id_private *id_priv;
@@ -4160,7 +4236,7 @@ int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
 EXPORT_SYMBOL(rdma_notify);
 
 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
-		u8 private_data_len)
+		u8 private_data_len, u8 reason)
 {
 	struct rdma_id_private *id_priv;
 	int ret;
@@ -4175,9 +4251,8 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
 						private_data, private_data_len);
 		} else {
 			trace_cm_send_rej(id_priv);
-			ret = ib_send_cm_rej(id_priv->cm_id.ib,
-					     IB_CM_REJ_CONSUMER_DEFINED, NULL,
-					     0, private_data, private_data_len);
+			ret = ib_send_cm_rej(id_priv->cm_id.ib, reason, NULL, 0,
+					     private_data, private_data_len);
 		}
 	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
 		ret = iw_cm_reject(id_priv->cm_id.iw,
@@ -4633,29 +4708,34 @@ static struct notifier_block cma_nb = {
 	.notifier_call = cma_netdev_callback
 };
 
-static void cma_add_one(struct ib_device *device)
+static int cma_add_one(struct ib_device *device)
 {
 	struct cma_device *cma_dev;
 	struct rdma_id_private *id_priv;
 	unsigned int i;
 	unsigned long supported_gids = 0;
+	int ret;
 
 	cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
 	if (!cma_dev)
-		return;
+		return -ENOMEM;
 
 	cma_dev->device = device;
 	cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
 					    sizeof(*cma_dev->default_gid_type),
 					    GFP_KERNEL);
-	if (!cma_dev->default_gid_type)
+	if (!cma_dev->default_gid_type) {
+		ret = -ENOMEM;
 		goto free_cma_dev;
+	}
 
 	cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
 					    sizeof(*cma_dev->default_roce_tos),
 					    GFP_KERNEL);
-	if (!cma_dev->default_roce_tos)
+	if (!cma_dev->default_roce_tos) {
+		ret = -ENOMEM;
 		goto free_gid_type;
+	}
 
 	rdma_for_each_port (device, i) {
 		supported_gids = roce_gid_type_mask_support(device, i);
@@ -4681,15 +4761,14 @@ static void cma_add_one(struct ib_device *device)
 	mutex_unlock(&lock);
 
 	trace_cm_add_one(device);
-	return;
+	return 0;
 
 free_gid_type:
 	kfree(cma_dev->default_gid_type);
 
 free_cma_dev:
 	kfree(cma_dev);
-
-	return;
+	return ret;
 }
 
 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
@@ -4751,9 +4830,6 @@ static void cma_remove_one(struct ib_device *device, void *client_data)
 
 	trace_cm_remove_one(device);
 
-	if (!cma_dev)
-		return;
-
 	mutex_lock(&lock);
 	list_del(&cma_dev->list);
 	mutex_unlock(&lock);
diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c
index c672a4978bfd..3c1e2ca564fe 100644
--- a/drivers/infiniband/core/cma_configfs.c
+++ b/drivers/infiniband/core/cma_configfs.c
@@ -322,8 +322,21 @@ fail:
 	return ERR_PTR(err);
 }
 
+static void drop_cma_dev(struct config_group *cgroup, struct config_item *item)
+{
+	struct config_group *group =
+		container_of(item, struct config_group, cg_item);
+	struct cma_dev_group *cma_dev_group =
+		container_of(group, struct cma_dev_group, device_group);
+
+	configfs_remove_default_groups(&cma_dev_group->ports_group);
+	configfs_remove_default_groups(&cma_dev_group->device_group);
+	config_item_put(item);
+}
+
 static struct configfs_group_operations cma_subsys_group_ops = {
 	.make_group	= make_cma_dev,
+	.drop_item	= drop_cma_dev,
 };
 
 static const struct config_item_type cma_subsys_type = {
diff --git a/drivers/infiniband/core/cma_priv.h b/drivers/infiniband/core/cma_priv.h
index 5edcf44a9307..caece96ebcf5 100644
--- a/drivers/infiniband/core/cma_priv.h
+++ b/drivers/infiniband/core/cma_priv.h
@@ -95,6 +95,7 @@ struct rdma_id_private {
 	 * Internal to RDMA/core, don't use in the drivers
 	 */
 	struct rdma_restrack_entry     res;
+	struct rdma_ucm_ece ece;
 };
 
 #if IS_ENABLED(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS)
diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h
index 81e36bf13159..e6e20c36c538 100644
--- a/drivers/infiniband/core/cma_trace.h
+++ b/drivers/infiniband/core/cma_trace.h
@@ -103,23 +103,33 @@ DEFINE_CMA_FSM_EVENT(sent_drep);
 DEFINE_CMA_FSM_EVENT(sent_dreq);
 DEFINE_CMA_FSM_EVENT(id_destroy);
 
-TRACE_EVENT(cm_id_create,
+TRACE_EVENT(cm_id_attach,
 	TP_PROTO(
-		const struct rdma_id_private *id_priv
+		const struct rdma_id_private *id_priv,
+		const struct ib_device *device
 	),
 
-	TP_ARGS(id_priv),
+	TP_ARGS(id_priv, device),
 
 	TP_STRUCT__entry(
 		__field(u32, cm_id)
+		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
+		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
+		__string(devname, device->name)
 	),
 
 	TP_fast_assign(
 		__entry->cm_id = id_priv->res.id;
+		memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
+		       sizeof(struct sockaddr_in6));
+		memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
+		       sizeof(struct sockaddr_in6));
+		__assign_str(devname, device->name);
 	),
 
-	TP_printk("cm.id=%u",
-		__entry->cm_id
+	TP_printk("cm.id=%u src=%pISpc dst=%pISpc device=%s",
+		__entry->cm_id, __entry->srcaddr, __entry->dstaddr,
+		__get_str(devname)
 	)
 );
 
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index cf42acca4a3a..a1e6a67b2c4a 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -414,4 +414,7 @@ void rdma_umap_priv_init(struct rdma_umap_priv *priv,
 			 struct vm_area_struct *vma,
 			 struct rdma_user_mmap_entry *entry);
 
+void ib_cq_pool_init(struct ib_device *dev);
+void ib_cq_pool_destroy(struct ib_device *dev);
+
 #endif /* _CORE_PRIV_H */
diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index 4f25b2400694..655795bfa0ee 100644
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -7,7 +7,11 @@
 #include <linux/slab.h>
 #include <rdma/ib_verbs.h>
 
+#include "core_priv.h"
+
 #include <trace/events/rdma_core.h>
+/* Max size for shared CQ, may require tuning */
+#define IB_MAX_SHARED_CQ_SZ		4096U
 
 /* # of WCs to poll for with a single call to ib_poll_cq */
 #define IB_POLL_BATCH			16
@@ -218,6 +222,7 @@ struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,
 	cq->cq_context = private;
 	cq->poll_ctx = poll_ctx;
 	atomic_set(&cq->usecnt, 0);
+	cq->comp_vector = comp_vector;
 
 	cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);
 	if (!cq->wc)
@@ -309,6 +314,8 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
 {
 	if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
 		return;
+	if (WARN_ON_ONCE(cq->cqe_used))
+		return;
 
 	switch (cq->poll_ctx) {
 	case IB_POLL_DIRECT:
@@ -334,3 +341,169 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
 	kfree(cq);
 }
 EXPORT_SYMBOL(ib_free_cq_user);
+
+void ib_cq_pool_init(struct ib_device *dev)
+{
+	unsigned int i;
+
+	spin_lock_init(&dev->cq_pools_lock);
+	for (i = 0; i < ARRAY_SIZE(dev->cq_pools); i++)
+		INIT_LIST_HEAD(&dev->cq_pools[i]);
+}
+
+void ib_cq_pool_destroy(struct ib_device *dev)
+{
+	struct ib_cq *cq, *n;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dev->cq_pools); i++) {
+		list_for_each_entry_safe(cq, n, &dev->cq_pools[i],
+					 pool_entry) {
+			WARN_ON(cq->cqe_used);
+			cq->shared = false;
+			ib_free_cq(cq);
+		}
+	}
+}
+
+static int ib_alloc_cqs(struct ib_device *dev, unsigned int nr_cqes,
+			enum ib_poll_context poll_ctx)
+{
+	LIST_HEAD(tmp_list);
+	unsigned int nr_cqs, i;
+	struct ib_cq *cq;
+	int ret;
+
+	if (poll_ctx > IB_POLL_LAST_POOL_TYPE) {
+		WARN_ON_ONCE(poll_ctx > IB_POLL_LAST_POOL_TYPE);
+		return -EINVAL;
+	}
+
+	/*
+	 * Allocate at least as many CQEs as requested, and otherwise
+	 * a reasonable batch size so that we can share CQs between
+	 * multiple users instead of allocating a larger number of CQs.
+	 */
+	nr_cqes = min_t(unsigned int, dev->attrs.max_cqe,
+			max(nr_cqes, IB_MAX_SHARED_CQ_SZ));
+	nr_cqs = min_t(unsigned int, dev->num_comp_vectors, num_online_cpus());
+	for (i = 0; i < nr_cqs; i++) {
+		cq = ib_alloc_cq(dev, NULL, nr_cqes, i, poll_ctx);
+		if (IS_ERR(cq)) {
+			ret = PTR_ERR(cq);
+			goto out_free_cqs;
+		}
+		cq->shared = true;
+		list_add_tail(&cq->pool_entry, &tmp_list);
+	}
+
+	spin_lock_irq(&dev->cq_pools_lock);
+	list_splice(&tmp_list, &dev->cq_pools[poll_ctx]);
+	spin_unlock_irq(&dev->cq_pools_lock);
+
+	return 0;
+
+out_free_cqs:
+	list_for_each_entry(cq, &tmp_list, pool_entry) {
+		cq->shared = false;
+		ib_free_cq(cq);
+	}
+	return ret;
+}
+
+/**
+ * ib_cq_pool_get() - Find the least used completion queue that matches
+ *   a given cpu hint (or least used for wild card affinity) and fits
+ *   nr_cqe.
+ * @dev: rdma device
+ * @nr_cqe: number of needed cqe entries
+ * @comp_vector_hint: completion vector hint (-1) for the driver to assign
+ *   a comp vector based on internal counter
+ * @poll_ctx: cq polling context
+ *
+ * Finds a cq that satisfies @comp_vector_hint and @nr_cqe requirements and
+ * claim entries in it for us.  In case there is no available cq, allocate
+ * a new cq with the requirements and add it to the device pool.
+ * IB_POLL_DIRECT cannot be used for shared cqs so it is not a valid value
+ * for @poll_ctx.
+ */
+struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,
+			     int comp_vector_hint,
+			     enum ib_poll_context poll_ctx)
+{
+	static unsigned int default_comp_vector;
+	unsigned int vector, num_comp_vectors;
+	struct ib_cq *cq, *found = NULL;
+	int ret;
+
+	if (poll_ctx > IB_POLL_LAST_POOL_TYPE) {
+		WARN_ON_ONCE(poll_ctx > IB_POLL_LAST_POOL_TYPE);
+		return ERR_PTR(-EINVAL);
+	}
+
+	num_comp_vectors =
+		min_t(unsigned int, dev->num_comp_vectors, num_online_cpus());
+	/* Project the affinty to the device completion vector range */
+	if (comp_vector_hint < 0) {
+		comp_vector_hint =
+			(READ_ONCE(default_comp_vector) + 1) % num_comp_vectors;
+		WRITE_ONCE(default_comp_vector, comp_vector_hint);
+	}
+	vector = comp_vector_hint % num_comp_vectors;
+
+	/*
+	 * Find the least used CQ with correct affinity and
+	 * enough free CQ entries
+	 */
+	while (!found) {
+		spin_lock_irq(&dev->cq_pools_lock);
+		list_for_each_entry(cq, &dev->cq_pools[poll_ctx],
+				    pool_entry) {
+			/*
+			 * Check to see if we have found a CQ with the
+			 * correct completion vector
+			 */
+			if (vector != cq->comp_vector)
+				continue;
+			if (cq->cqe_used + nr_cqe > cq->cqe)
+				continue;
+			found = cq;
+			break;
+		}
+
+		if (found) {
+			found->cqe_used += nr_cqe;
+			spin_unlock_irq(&dev->cq_pools_lock);
+
+			return found;
+		}
+		spin_unlock_irq(&dev->cq_pools_lock);
+
+		/*
+		 * Didn't find a match or ran out of CQs in the device
+		 * pool, allocate a new array of CQs.
+		 */
+		ret = ib_alloc_cqs(dev, nr_cqe, poll_ctx);
+		if (ret)
+			return ERR_PTR(ret);
+	}
+
+	return found;
+}
+EXPORT_SYMBOL(ib_cq_pool_get);
+
+/**
+ * ib_cq_pool_put - Return a CQ taken from a shared pool.
+ * @cq: The CQ to return.
+ * @nr_cqe: The max number of cqes that the user had requested.
+ */
+void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe)
+{
+	if (WARN_ON_ONCE(nr_cqe > cq->cqe_used))
+		return;
+
+	spin_lock_irq(&cq->device->cq_pools_lock);
+	cq->cqe_used -= nr_cqe;
+	spin_unlock_irq(&cq->device->cq_pools_lock);
+}
+EXPORT_SYMBOL(ib_cq_pool_put);
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index d0b3d35ad3e4..905a2beaf885 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -677,8 +677,20 @@ static int add_client_context(struct ib_device *device,
 	if (ret)
 		goto out;
 	downgrade_write(&device->client_data_rwsem);
-	if (client->add)
-		client->add(device);
+	if (client->add) {
+		if (client->add(device)) {
+			/*
+			 * If a client fails to add then the error code is
+			 * ignored, but we won't call any more ops on this
+			 * client.
+			 */
+			xa_erase(&device->client_data, client->client_id);
+			up_read(&device->client_data_rwsem);
+			ib_device_put(device);
+			ib_client_put(client);
+			return 0;
+		}
+	}
 
 	/* Readers shall not see a client until add has been completed */
 	xa_set_mark(&device->client_data, client->client_id,
@@ -1381,6 +1393,7 @@ int ib_register_device(struct ib_device *device, const char *name)
 		goto dev_cleanup;
 	}
 
+	ib_cq_pool_init(device);
 	ret = enable_device_and_get(device);
 	dev_set_uevent_suppress(&device->dev, false);
 	/* Mark for userspace that device is ready */
@@ -1435,6 +1448,7 @@ static void __ib_unregister_device(struct ib_device *ib_dev)
 		goto out;
 
 	disable_device(ib_dev);
+	ib_cq_pool_destroy(ib_dev);
 
 	/* Expedite removing unregistered pointers from the hash table */
 	free_netdevs(ib_dev);
@@ -2557,7 +2571,6 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, add_gid);
 	SET_DEVICE_OP(dev_ops, advise_mr);
 	SET_DEVICE_OP(dev_ops, alloc_dm);
-	SET_DEVICE_OP(dev_ops, alloc_fmr);
 	SET_DEVICE_OP(dev_ops, alloc_hw_stats);
 	SET_DEVICE_OP(dev_ops, alloc_mr);
 	SET_DEVICE_OP(dev_ops, alloc_mr_integrity);
@@ -2584,7 +2597,6 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, create_wq);
 	SET_DEVICE_OP(dev_ops, dealloc_dm);
 	SET_DEVICE_OP(dev_ops, dealloc_driver);
-	SET_DEVICE_OP(dev_ops, dealloc_fmr);
 	SET_DEVICE_OP(dev_ops, dealloc_mw);
 	SET_DEVICE_OP(dev_ops, dealloc_pd);
 	SET_DEVICE_OP(dev_ops, dealloc_ucontext);
@@ -2628,7 +2640,6 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, iw_rem_ref);
 	SET_DEVICE_OP(dev_ops, map_mr_sg);
 	SET_DEVICE_OP(dev_ops, map_mr_sg_pi);
-	SET_DEVICE_OP(dev_ops, map_phys_fmr);
 	SET_DEVICE_OP(dev_ops, mmap);
 	SET_DEVICE_OP(dev_ops, mmap_free);
 	SET_DEVICE_OP(dev_ops, modify_ah);
@@ -2662,7 +2673,6 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, resize_cq);
 	SET_DEVICE_OP(dev_ops, set_vf_guid);
 	SET_DEVICE_OP(dev_ops, set_vf_link_state);
-	SET_DEVICE_OP(dev_ops, unmap_fmr);
 
 	SET_OBJ_SIZE(dev_ops, ib_ah);
 	SET_OBJ_SIZE(dev_ops, ib_cq);
diff --git a/drivers/infiniband/core/fmr_pool.c b/drivers/infiniband/core/fmr_pool.c
deleted file mode 100644
index e08aec427027..000000000000
--- a/drivers/infiniband/core/fmr_pool.c
+++ /dev/null
@@ -1,494 +0,0 @@
-/*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
- * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/errno.h>
-#include <linux/spinlock.h>
-#include <linux/export.h>
-#include <linux/slab.h>
-#include <linux/jhash.h>
-#include <linux/kthread.h>
-
-#include <rdma/ib_fmr_pool.h>
-
-#include "core_priv.h"
-
-#define PFX "fmr_pool: "
-
-enum {
-	IB_FMR_MAX_REMAPS = 32,
-
-	IB_FMR_HASH_BITS  = 8,
-	IB_FMR_HASH_SIZE  = 1 << IB_FMR_HASH_BITS,
-	IB_FMR_HASH_MASK  = IB_FMR_HASH_SIZE - 1
-};
-
-/*
- * If an FMR is not in use, then the list member will point to either
- * its pool's free_list (if the FMR can be mapped again; that is,
- * remap_count < pool->max_remaps) or its pool's dirty_list (if the
- * FMR needs to be unmapped before being remapped).  In either of
- * these cases it is a bug if the ref_count is not 0.  In other words,
- * if ref_count is > 0, then the list member must not be linked into
- * either free_list or dirty_list.
- *
- * The cache_node member is used to link the FMR into a cache bucket
- * (if caching is enabled).  This is independent of the reference
- * count of the FMR.  When a valid FMR is released, its ref_count is
- * decremented, and if ref_count reaches 0, the FMR is placed in
- * either free_list or dirty_list as appropriate.  However, it is not
- * removed from the cache and may be "revived" if a call to
- * ib_fmr_register_physical() occurs before the FMR is remapped.  In
- * this case we just increment the ref_count and remove the FMR from
- * free_list/dirty_list.
- *
- * Before we remap an FMR from free_list, we remove it from the cache
- * (to prevent another user from obtaining a stale FMR).  When an FMR
- * is released, we add it to the tail of the free list, so that our
- * cache eviction policy is "least recently used."
- *
- * All manipulation of ref_count, list and cache_node is protected by
- * pool_lock to maintain consistency.
- */
-
-struct ib_fmr_pool {
-	spinlock_t                pool_lock;
-
-	int                       pool_size;
-	int                       max_pages;
-	int			  max_remaps;
-	int                       dirty_watermark;
-	int                       dirty_len;
-	struct list_head          free_list;
-	struct list_head          dirty_list;
-	struct hlist_head        *cache_bucket;
-
-	void                     (*flush_function)(struct ib_fmr_pool *pool,
-						   void *              arg);
-	void                     *flush_arg;
-
-	struct kthread_worker	  *worker;
-	struct kthread_work	  work;
-
-	atomic_t                  req_ser;
-	atomic_t                  flush_ser;
-
-	wait_queue_head_t         force_wait;
-};
-
-static inline u32 ib_fmr_hash(u64 first_page)
-{
-	return jhash_2words((u32) first_page, (u32) (first_page >> 32), 0) &
-		(IB_FMR_HASH_SIZE - 1);
-}
-
-/* Caller must hold pool_lock */
-static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,
-						      u64 *page_list,
-						      int  page_list_len,
-						      u64  io_virtual_address)
-{
-	struct hlist_head *bucket;
-	struct ib_pool_fmr *fmr;
-
-	if (!pool->cache_bucket)
-		return NULL;
-
-	bucket = pool->cache_bucket + ib_fmr_hash(*page_list);
-
-	hlist_for_each_entry(fmr, bucket, cache_node)
-		if (io_virtual_address == fmr->io_virtual_address &&
-		    page_list_len      == fmr->page_list_len      &&
-		    !memcmp(page_list, fmr->page_list,
-			    page_list_len * sizeof *page_list))
-			return fmr;
-
-	return NULL;
-}
-
-static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
-{
-	int                 ret;
-	struct ib_pool_fmr *fmr;
-	LIST_HEAD(unmap_list);
-	LIST_HEAD(fmr_list);
-
-	spin_lock_irq(&pool->pool_lock);
-
-	list_for_each_entry(fmr, &pool->dirty_list, list) {
-		hlist_del_init(&fmr->cache_node);
-		fmr->remap_count = 0;
-		list_add_tail(&fmr->fmr->list, &fmr_list);
-	}
-
-	list_splice_init(&pool->dirty_list, &unmap_list);
-	pool->dirty_len = 0;
-
-	spin_unlock_irq(&pool->pool_lock);
-
-	if (list_empty(&unmap_list)) {
-		return;
-	}
-
-	ret = ib_unmap_fmr(&fmr_list);
-	if (ret)
-		pr_warn(PFX "ib_unmap_fmr returned %d\n", ret);
-
-	spin_lock_irq(&pool->pool_lock);
-	list_splice(&unmap_list, &pool->free_list);
-	spin_unlock_irq(&pool->pool_lock);
-}
-
-static void ib_fmr_cleanup_func(struct kthread_work *work)
-{
-	struct ib_fmr_pool *pool = container_of(work, struct ib_fmr_pool, work);
-
-	ib_fmr_batch_release(pool);
-	atomic_inc(&pool->flush_ser);
-	wake_up_interruptible(&pool->force_wait);
-
-	if (pool->flush_function)
-		pool->flush_function(pool, pool->flush_arg);
-
-	if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0)
-		kthread_queue_work(pool->worker, &pool->work);
-}
-
-/**
- * ib_create_fmr_pool - Create an FMR pool
- * @pd:Protection domain for FMRs
- * @params:FMR pool parameters
- *
- * Create a pool of FMRs.  Return value is pointer to new pool or
- * error code if creation failed.
- */
-struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd             *pd,
-				       struct ib_fmr_pool_param *params)
-{
-	struct ib_device   *device;
-	struct ib_fmr_pool *pool;
-	int i;
-	int ret;
-	int max_remaps;
-
-	if (!params)
-		return ERR_PTR(-EINVAL);
-
-	device = pd->device;
-	if (!device->ops.alloc_fmr    || !device->ops.dealloc_fmr  ||
-	    !device->ops.map_phys_fmr || !device->ops.unmap_fmr) {
-		dev_info(&device->dev, "Device does not support FMRs\n");
-		return ERR_PTR(-ENOSYS);
-	}
-
-	if (!device->attrs.max_map_per_fmr)
-		max_remaps = IB_FMR_MAX_REMAPS;
-	else
-		max_remaps = device->attrs.max_map_per_fmr;
-
-	pool = kmalloc(sizeof *pool, GFP_KERNEL);
-	if (!pool)
-		return ERR_PTR(-ENOMEM);
-
-	pool->cache_bucket   = NULL;
-	pool->flush_function = params->flush_function;
-	pool->flush_arg      = params->flush_arg;
-
-	INIT_LIST_HEAD(&pool->free_list);
-	INIT_LIST_HEAD(&pool->dirty_list);
-
-	if (params->cache) {
-		pool->cache_bucket =
-			kmalloc_array(IB_FMR_HASH_SIZE,
-				      sizeof(*pool->cache_bucket),
-				      GFP_KERNEL);
-		if (!pool->cache_bucket) {
-			ret = -ENOMEM;
-			goto out_free_pool;
-		}
-
-		for (i = 0; i < IB_FMR_HASH_SIZE; ++i)
-			INIT_HLIST_HEAD(pool->cache_bucket + i);
-	}
-
-	pool->pool_size       = 0;
-	pool->max_pages       = params->max_pages_per_fmr;
-	pool->max_remaps      = max_remaps;
-	pool->dirty_watermark = params->dirty_watermark;
-	pool->dirty_len       = 0;
-	spin_lock_init(&pool->pool_lock);
-	atomic_set(&pool->req_ser,   0);
-	atomic_set(&pool->flush_ser, 0);
-	init_waitqueue_head(&pool->force_wait);
-
-	pool->worker =
-		kthread_create_worker(0, "ib_fmr(%s)", dev_name(&device->dev));
-	if (IS_ERR(pool->worker)) {
-		pr_warn(PFX "couldn't start cleanup kthread worker\n");
-		ret = PTR_ERR(pool->worker);
-		goto out_free_pool;
-	}
-	kthread_init_work(&pool->work, ib_fmr_cleanup_func);
-
-	{
-		struct ib_pool_fmr *fmr;
-		struct ib_fmr_attr fmr_attr = {
-			.max_pages  = params->max_pages_per_fmr,
-			.max_maps   = pool->max_remaps,
-			.page_shift = params->page_shift
-		};
-		int bytes_per_fmr = sizeof *fmr;
-
-		if (pool->cache_bucket)
-			bytes_per_fmr += params->max_pages_per_fmr * sizeof (u64);
-
-		for (i = 0; i < params->pool_size; ++i) {
-			fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);
-			if (!fmr)
-				goto out_fail;
-
-			fmr->pool             = pool;
-			fmr->remap_count      = 0;
-			fmr->ref_count        = 0;
-			INIT_HLIST_NODE(&fmr->cache_node);
-
-			fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr);
-			if (IS_ERR(fmr->fmr)) {
-				pr_warn(PFX "fmr_create failed for FMR %d\n",
-					i);
-				kfree(fmr);
-				goto out_fail;
-			}
-
-			list_add_tail(&fmr->list, &pool->free_list);
-			++pool->pool_size;
-		}
-	}
-
-	return pool;
-
- out_free_pool:
-	kfree(pool->cache_bucket);
-	kfree(pool);
-
-	return ERR_PTR(ret);
-
- out_fail:
-	ib_destroy_fmr_pool(pool);
-
-	return ERR_PTR(-ENOMEM);
-}
-EXPORT_SYMBOL(ib_create_fmr_pool);
-
-/**
- * ib_destroy_fmr_pool - Free FMR pool
- * @pool:FMR pool to free
- *
- * Destroy an FMR pool and free all associated resources.
- */
-void ib_destroy_fmr_pool(struct ib_fmr_pool *pool)
-{
-	struct ib_pool_fmr *fmr;
-	struct ib_pool_fmr *tmp;
-	LIST_HEAD(fmr_list);
-	int                 i;
-
-	kthread_destroy_worker(pool->worker);
-	ib_fmr_batch_release(pool);
-
-	i = 0;
-	list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {
-		if (fmr->remap_count) {
-			INIT_LIST_HEAD(&fmr_list);
-			list_add_tail(&fmr->fmr->list, &fmr_list);
-			ib_unmap_fmr(&fmr_list);
-		}
-		ib_dealloc_fmr(fmr->fmr);
-		list_del(&fmr->list);
-		kfree(fmr);
-		++i;
-	}
-
-	if (i < pool->pool_size)
-		pr_warn(PFX "pool still has %d regions registered\n",
-			pool->pool_size - i);
-
-	kfree(pool->cache_bucket);
-	kfree(pool);
-}
-EXPORT_SYMBOL(ib_destroy_fmr_pool);
-
-/**
- * ib_flush_fmr_pool - Invalidate all unmapped FMRs
- * @pool:FMR pool to flush
- *
- * Ensure that all unmapped FMRs are fully invalidated.
- */
-int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
-{
-	int serial;
-	struct ib_pool_fmr *fmr, *next;
-
-	/*
-	 * The free_list holds FMRs that may have been used
-	 * but have not been remapped enough times to be dirty.
-	 * Put them on the dirty list now so that the cleanup
-	 * thread will reap them too.
-	 */
-	spin_lock_irq(&pool->pool_lock);
-	list_for_each_entry_safe(fmr, next, &pool->free_list, list) {
-		if (fmr->remap_count > 0)
-			list_move(&fmr->list, &pool->dirty_list);
-	}
-	spin_unlock_irq(&pool->pool_lock);
-
-	serial = atomic_inc_return(&pool->req_ser);
-	kthread_queue_work(pool->worker, &pool->work);
-
-	if (wait_event_interruptible(pool->force_wait,
-				     atomic_read(&pool->flush_ser) - serial >= 0))
-		return -EINTR;
-
-	return 0;
-}
-EXPORT_SYMBOL(ib_flush_fmr_pool);
-
-/**
- * ib_fmr_pool_map_phys - Map an FMR from an FMR pool.
- * @pool_handle: FMR pool to allocate FMR from
- * @page_list: List of pages to map
- * @list_len: Number of pages in @page_list
- * @io_virtual_address: I/O virtual address for new FMR
- */
-struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
-					 u64                *page_list,
-					 int                 list_len,
-					 u64                 io_virtual_address)
-{
-	struct ib_fmr_pool *pool = pool_handle;
-	struct ib_pool_fmr *fmr;
-	unsigned long       flags;
-	int                 result;
-
-	if (list_len < 1 || list_len > pool->max_pages)
-		return ERR_PTR(-EINVAL);
-
-	spin_lock_irqsave(&pool->pool_lock, flags);
-	fmr = ib_fmr_cache_lookup(pool,
-				  page_list,
-				  list_len,
-				  io_virtual_address);
-	if (fmr) {
-		/* found in cache */
-		++fmr->ref_count;
-		if (fmr->ref_count == 1) {
-			list_del(&fmr->list);
-		}
-
-		spin_unlock_irqrestore(&pool->pool_lock, flags);
-
-		return fmr;
-	}
-
-	if (list_empty(&pool->free_list)) {
-		spin_unlock_irqrestore(&pool->pool_lock, flags);
-		return ERR_PTR(-EAGAIN);
-	}
-
-	fmr = list_entry(pool->free_list.next, struct ib_pool_fmr, list);
-	list_del(&fmr->list);
-	hlist_del_init(&fmr->cache_node);
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
-
-	result = ib_map_phys_fmr(fmr->fmr, page_list, list_len,
-				 io_virtual_address);
-
-	if (result) {
-		spin_lock_irqsave(&pool->pool_lock, flags);
-		list_add(&fmr->list, &pool->free_list);
-		spin_unlock_irqrestore(&pool->pool_lock, flags);
-
-		pr_warn(PFX "fmr_map returns %d\n", result);
-
-		return ERR_PTR(result);
-	}
-
-	++fmr->remap_count;
-	fmr->ref_count = 1;
-
-	if (pool->cache_bucket) {
-		fmr->io_virtual_address = io_virtual_address;
-		fmr->page_list_len      = list_len;
-		memcpy(fmr->page_list, page_list, list_len * sizeof(*page_list));
-
-		spin_lock_irqsave(&pool->pool_lock, flags);
-		hlist_add_head(&fmr->cache_node,
-			       pool->cache_bucket + ib_fmr_hash(fmr->page_list[0]));
-		spin_unlock_irqrestore(&pool->pool_lock, flags);
-	}
-
-	return fmr;
-}
-EXPORT_SYMBOL(ib_fmr_pool_map_phys);
-
-/**
- * ib_fmr_pool_unmap - Unmap FMR
- * @fmr:FMR to unmap
- *
- * Unmap an FMR.  The FMR mapping may remain valid until the FMR is
- * reused (or until ib_flush_fmr_pool() is called).
- */
-void ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)
-{
-	struct ib_fmr_pool *pool;
-	unsigned long flags;
-
-	pool = fmr->pool;
-
-	spin_lock_irqsave(&pool->pool_lock, flags);
-
-	--fmr->ref_count;
-	if (!fmr->ref_count) {
-		if (fmr->remap_count < pool->max_remaps) {
-			list_add_tail(&fmr->list, &pool->free_list);
-		} else {
-			list_add_tail(&fmr->list, &pool->dirty_list);
-			if (++pool->dirty_len >= pool->dirty_watermark) {
-				atomic_inc(&pool->req_ser);
-				kthread_queue_work(pool->worker, &pool->work);
-			}
-		}
-	}
-
-	spin_unlock_irqrestore(&pool->pool_lock, flags);
-}
-EXPORT_SYMBOL(ib_fmr_pool_unmap);
diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c
new file mode 100644
index 000000000000..7063e41eaf26
--- /dev/null
+++ b/drivers/infiniband/core/lag.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2020 Mellanox Technologies. All rights reserved.
+ */
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_cache.h>
+#include <rdma/lag.h>
+
+static struct sk_buff *rdma_build_skb(struct ib_device *device,
+				      struct net_device *netdev,
+				      struct rdma_ah_attr *ah_attr,
+				      gfp_t flags)
+{
+	struct ipv6hdr *ip6h;
+	struct sk_buff *skb;
+	struct ethhdr *eth;
+	struct iphdr *iph;
+	struct udphdr *uh;
+	u8 smac[ETH_ALEN];
+	bool is_ipv4;
+	int hdr_len;
+
+	is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw);
+	hdr_len = ETH_HLEN + sizeof(struct udphdr) + LL_RESERVED_SPACE(netdev);
+	hdr_len += is_ipv4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr);
+
+	skb = alloc_skb(hdr_len, flags);
+	if (!skb)
+		return NULL;
+
+	skb->dev = netdev;
+	skb_reserve(skb, hdr_len);
+	skb_push(skb, sizeof(struct udphdr));
+	skb_reset_transport_header(skb);
+	uh = udp_hdr(skb);
+	uh->source =
+		htons(rdma_flow_label_to_udp_sport(ah_attr->grh.flow_label));
+	uh->dest = htons(ROCE_V2_UDP_DPORT);
+	uh->len = htons(sizeof(struct udphdr));
+
+	if (is_ipv4) {
+		skb_push(skb, sizeof(struct iphdr));
+		skb_reset_network_header(skb);
+		iph = ip_hdr(skb);
+		iph->frag_off = 0;
+		iph->version = 4;
+		iph->protocol = IPPROTO_UDP;
+		iph->ihl = 0x5;
+		iph->tot_len = htons(sizeof(struct udphdr) + sizeof(struct
+								    iphdr));
+		memcpy(&iph->saddr, ah_attr->grh.sgid_attr->gid.raw + 12,
+		       sizeof(struct in_addr));
+		memcpy(&iph->daddr, ah_attr->grh.dgid.raw + 12,
+		       sizeof(struct in_addr));
+	} else {
+		skb_push(skb, sizeof(struct ipv6hdr));
+		skb_reset_network_header(skb);
+		ip6h = ipv6_hdr(skb);
+		ip6h->version = 6;
+		ip6h->nexthdr = IPPROTO_UDP;
+		memcpy(&ip6h->flow_lbl, &ah_attr->grh.flow_label,
+		       sizeof(*ip6h->flow_lbl));
+		memcpy(&ip6h->saddr, ah_attr->grh.sgid_attr->gid.raw,
+		       sizeof(struct in6_addr));
+		memcpy(&ip6h->daddr, ah_attr->grh.dgid.raw,
+		       sizeof(struct in6_addr));
+	}
+
+	skb_push(skb, sizeof(struct ethhdr));
+	skb_reset_mac_header(skb);
+	eth = eth_hdr(skb);
+	skb->protocol = eth->h_proto = htons(is_ipv4 ? ETH_P_IP : ETH_P_IPV6);
+	rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, NULL, smac);
+	memcpy(eth->h_source, smac, ETH_ALEN);
+	memcpy(eth->h_dest, ah_attr->roce.dmac, ETH_ALEN);
+
+	return skb;
+}
+
+static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device,
+						  struct net_device *master,
+						  struct rdma_ah_attr *ah_attr,
+						  gfp_t flags)
+{
+	struct net_device *slave;
+	struct sk_buff *skb;
+
+	skb = rdma_build_skb(device, master, ah_attr, flags);
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	rcu_read_lock();
+	slave = netdev_get_xmit_slave(master, skb,
+				      !!(device->lag_flags &
+					 RDMA_LAG_FLAGS_HASH_ALL_SLAVES));
+	if (slave)
+		dev_hold(slave);
+	rcu_read_unlock();
+	kfree_skb(skb);
+	return slave;
+}
+
+void rdma_lag_put_ah_roce_slave(struct net_device *xmit_slave)
+{
+	if (xmit_slave)
+		dev_put(xmit_slave);
+}
+
+struct net_device *rdma_lag_get_ah_roce_slave(struct ib_device *device,
+					      struct rdma_ah_attr *ah_attr,
+					      gfp_t flags)
+{
+	struct net_device *slave = NULL;
+	struct net_device *master;
+
+	if (!(ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE &&
+	      ah_attr->grh.sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP &&
+	      ah_attr->grh.flow_label))
+		return NULL;
+
+	rcu_read_lock();
+	master = rdma_read_gid_attr_ndev_rcu(ah_attr->grh.sgid_attr);
+	if (IS_ERR(master)) {
+		rcu_read_unlock();
+		return master;
+	}
+	dev_hold(master);
+	rcu_read_unlock();
+
+	if (!netif_is_bond_master(master))
+		goto put;
+
+	slave = rdma_get_xmit_slave_udp(device, master, ah_attr, flags);
+put:
+	dev_put(master);
+	return slave;
+}
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index c54db13fa9b0..186e0d652e8b 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -85,7 +85,6 @@ MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests
 module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
 MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
 
-/* Client ID 0 is used for snoop-only clients */
 static DEFINE_XARRAY_ALLOC1(ib_mad_clients);
 static u32 ib_mad_client_next;
 static struct list_head ib_mad_port_list;
@@ -483,141 +482,12 @@ error1:
 }
 EXPORT_SYMBOL(ib_register_mad_agent);
 
-static inline int is_snooping_sends(int mad_snoop_flags)
-{
-	return (mad_snoop_flags &
-		(/*IB_MAD_SNOOP_POSTED_SENDS |
-		 IB_MAD_SNOOP_RMPP_SENDS |*/
-		 IB_MAD_SNOOP_SEND_COMPLETIONS /*|
-		 IB_MAD_SNOOP_RMPP_SEND_COMPLETIONS*/));
-}
-
-static inline int is_snooping_recvs(int mad_snoop_flags)
-{
-	return (mad_snoop_flags &
-		(IB_MAD_SNOOP_RECVS /*|
-		 IB_MAD_SNOOP_RMPP_RECVS*/));
-}
-
-static int register_snoop_agent(struct ib_mad_qp_info *qp_info,
-				struct ib_mad_snoop_private *mad_snoop_priv)
-{
-	struct ib_mad_snoop_private **new_snoop_table;
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&qp_info->snoop_lock, flags);
-	/* Check for empty slot in array. */
-	for (i = 0; i < qp_info->snoop_table_size; i++)
-		if (!qp_info->snoop_table[i])
-			break;
-
-	if (i == qp_info->snoop_table_size) {
-		/* Grow table. */
-		new_snoop_table = krealloc(qp_info->snoop_table,
-					   sizeof mad_snoop_priv *
-					   (qp_info->snoop_table_size + 1),
-					   GFP_ATOMIC);
-		if (!new_snoop_table) {
-			i = -ENOMEM;
-			goto out;
-		}
-
-		qp_info->snoop_table = new_snoop_table;
-		qp_info->snoop_table_size++;
-	}
-	qp_info->snoop_table[i] = mad_snoop_priv;
-	atomic_inc(&qp_info->snoop_count);
-out:
-	spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
-	return i;
-}
-
-struct ib_mad_agent *ib_register_mad_snoop(struct ib_device *device,
-					   u8 port_num,
-					   enum ib_qp_type qp_type,
-					   int mad_snoop_flags,
-					   ib_mad_snoop_handler snoop_handler,
-					   ib_mad_recv_handler recv_handler,
-					   void *context)
-{
-	struct ib_mad_port_private *port_priv;
-	struct ib_mad_agent *ret;
-	struct ib_mad_snoop_private *mad_snoop_priv;
-	int qpn;
-	int err;
-
-	/* Validate parameters */
-	if ((is_snooping_sends(mad_snoop_flags) && !snoop_handler) ||
-	    (is_snooping_recvs(mad_snoop_flags) && !recv_handler)) {
-		ret = ERR_PTR(-EINVAL);
-		goto error1;
-	}
-	qpn = get_spl_qp_index(qp_type);
-	if (qpn == -1) {
-		ret = ERR_PTR(-EINVAL);
-		goto error1;
-	}
-	port_priv = ib_get_mad_port(device, port_num);
-	if (!port_priv) {
-		ret = ERR_PTR(-ENODEV);
-		goto error1;
-	}
-	/* Allocate structures */
-	mad_snoop_priv = kzalloc(sizeof *mad_snoop_priv, GFP_KERNEL);
-	if (!mad_snoop_priv) {
-		ret = ERR_PTR(-ENOMEM);
-		goto error1;
-	}
-
-	/* Now, fill in the various structures */
-	mad_snoop_priv->qp_info = &port_priv->qp_info[qpn];
-	mad_snoop_priv->agent.device = device;
-	mad_snoop_priv->agent.recv_handler = recv_handler;
-	mad_snoop_priv->agent.snoop_handler = snoop_handler;
-	mad_snoop_priv->agent.context = context;
-	mad_snoop_priv->agent.qp = port_priv->qp_info[qpn].qp;
-	mad_snoop_priv->agent.port_num = port_num;
-	mad_snoop_priv->mad_snoop_flags = mad_snoop_flags;
-	init_completion(&mad_snoop_priv->comp);
-
-	err = ib_mad_agent_security_setup(&mad_snoop_priv->agent, qp_type);
-	if (err) {
-		ret = ERR_PTR(err);
-		goto error2;
-	}
-
-	mad_snoop_priv->snoop_index = register_snoop_agent(
-						&port_priv->qp_info[qpn],
-						mad_snoop_priv);
-	if (mad_snoop_priv->snoop_index < 0) {
-		ret = ERR_PTR(mad_snoop_priv->snoop_index);
-		goto error3;
-	}
-
-	atomic_set(&mad_snoop_priv->refcount, 1);
-	return &mad_snoop_priv->agent;
-error3:
-	ib_mad_agent_security_cleanup(&mad_snoop_priv->agent);
-error2:
-	kfree(mad_snoop_priv);
-error1:
-	return ret;
-}
-EXPORT_SYMBOL(ib_register_mad_snoop);
-
 static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 {
 	if (atomic_dec_and_test(&mad_agent_priv->refcount))
 		complete(&mad_agent_priv->comp);
 }
 
-static inline void deref_snoop_agent(struct ib_mad_snoop_private *mad_snoop_priv)
-{
-	if (atomic_dec_and_test(&mad_snoop_priv->refcount))
-		complete(&mad_snoop_priv->comp);
-}
-
 static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 {
 	struct ib_mad_port_private *port_priv;
@@ -650,25 +520,6 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 	kfree_rcu(mad_agent_priv, rcu);
 }
 
-static void unregister_mad_snoop(struct ib_mad_snoop_private *mad_snoop_priv)
-{
-	struct ib_mad_qp_info *qp_info;
-	unsigned long flags;
-
-	qp_info = mad_snoop_priv->qp_info;
-	spin_lock_irqsave(&qp_info->snoop_lock, flags);
-	qp_info->snoop_table[mad_snoop_priv->snoop_index] = NULL;
-	atomic_dec(&qp_info->snoop_count);
-	spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
-
-	deref_snoop_agent(mad_snoop_priv);
-	wait_for_completion(&mad_snoop_priv->comp);
-
-	ib_mad_agent_security_cleanup(&mad_snoop_priv->agent);
-
-	kfree(mad_snoop_priv);
-}
-
 /*
  * ib_unregister_mad_agent - Unregisters a client from using MAD services
  *
@@ -677,20 +528,11 @@ static void unregister_mad_snoop(struct ib_mad_snoop_private *mad_snoop_priv)
 void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent)
 {
 	struct ib_mad_agent_private *mad_agent_priv;
-	struct ib_mad_snoop_private *mad_snoop_priv;
-
-	/* If the TID is zero, the agent can only snoop. */
-	if (mad_agent->hi_tid) {
-		mad_agent_priv = container_of(mad_agent,
-					      struct ib_mad_agent_private,
-					      agent);
-		unregister_mad_agent(mad_agent_priv);
-	} else {
-		mad_snoop_priv = container_of(mad_agent,
-					      struct ib_mad_snoop_private,
-					      agent);
-		unregister_mad_snoop(mad_snoop_priv);
-	}
+
+	mad_agent_priv = container_of(mad_agent,
+				      struct ib_mad_agent_private,
+				      agent);
+	unregister_mad_agent(mad_agent_priv);
 }
 EXPORT_SYMBOL(ib_unregister_mad_agent);
 
@@ -706,57 +548,6 @@ static void dequeue_mad(struct ib_mad_list_head *mad_list)
 	spin_unlock_irqrestore(&mad_queue->lock, flags);
 }
 
-static void snoop_send(struct ib_mad_qp_info *qp_info,
-		       struct ib_mad_send_buf *send_buf,
-		       struct ib_mad_send_wc *mad_send_wc,
-		       int mad_snoop_flags)
-{
-	struct ib_mad_snoop_private *mad_snoop_priv;
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&qp_info->snoop_lock, flags);
-	for (i = 0; i < qp_info->snoop_table_size; i++) {
-		mad_snoop_priv = qp_info->snoop_table[i];
-		if (!mad_snoop_priv ||
-		    !(mad_snoop_priv->mad_snoop_flags & mad_snoop_flags))
-			continue;
-
-		atomic_inc(&mad_snoop_priv->refcount);
-		spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
-		mad_snoop_priv->agent.snoop_handler(&mad_snoop_priv->agent,
-						    send_buf, mad_send_wc);
-		deref_snoop_agent(mad_snoop_priv);
-		spin_lock_irqsave(&qp_info->snoop_lock, flags);
-	}
-	spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
-}
-
-static void snoop_recv(struct ib_mad_qp_info *qp_info,
-		       struct ib_mad_recv_wc *mad_recv_wc,
-		       int mad_snoop_flags)
-{
-	struct ib_mad_snoop_private *mad_snoop_priv;
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&qp_info->snoop_lock, flags);
-	for (i = 0; i < qp_info->snoop_table_size; i++) {
-		mad_snoop_priv = qp_info->snoop_table[i];
-		if (!mad_snoop_priv ||
-		    !(mad_snoop_priv->mad_snoop_flags & mad_snoop_flags))
-			continue;
-
-		atomic_inc(&mad_snoop_priv->refcount);
-		spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
-		mad_snoop_priv->agent.recv_handler(&mad_snoop_priv->agent, NULL,
-						   mad_recv_wc);
-		deref_snoop_agent(mad_snoop_priv);
-		spin_lock_irqsave(&qp_info->snoop_lock, flags);
-	}
-	spin_unlock_irqrestore(&qp_info->snoop_lock, flags);
-}
-
 static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid,
 		u16 pkey_index, u8 port_num, struct ib_wc *wc)
 {
@@ -2289,9 +2080,6 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 	recv->header.recv_wc.recv_buf.mad = (struct ib_mad *)recv->mad;
 	recv->header.recv_wc.recv_buf.grh = &recv->grh;
 
-	if (atomic_read(&qp_info->snoop_count))
-		snoop_recv(qp_info, &recv->header.recv_wc, IB_MAD_SNOOP_RECVS);
-
 	/* Validate MAD */
 	if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa))
 		goto out;
@@ -2538,9 +2326,6 @@ retry:
 	mad_send_wc.send_buf = &mad_send_wr->send_buf;
 	mad_send_wc.status = wc->status;
 	mad_send_wc.vendor_err = wc->vendor_err;
-	if (atomic_read(&qp_info->snoop_count))
-		snoop_send(qp_info, &mad_send_wr->send_buf, &mad_send_wc,
-			   IB_MAD_SNOOP_SEND_COMPLETIONS);
 	ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
 
 	if (queued_send_wr) {
@@ -2782,10 +2567,6 @@ static void local_completions(struct work_struct *work)
 			local->mad_priv->header.recv_wc.recv_buf.grh = NULL;
 			local->mad_priv->header.recv_wc.recv_buf.mad =
 						(struct ib_mad *)local->mad_priv->mad;
-			if (atomic_read(&recv_mad_agent->qp_info->snoop_count))
-				snoop_recv(recv_mad_agent->qp_info,
-					  &local->mad_priv->header.recv_wc,
-					   IB_MAD_SNOOP_RECVS);
 			recv_mad_agent->agent.recv_handler(
 						&recv_mad_agent->agent,
 						&local->mad_send_wr->send_buf,
@@ -2800,10 +2581,6 @@ local_send_completion:
 		mad_send_wc.status = IB_WC_SUCCESS;
 		mad_send_wc.vendor_err = 0;
 		mad_send_wc.send_buf = &local->mad_send_wr->send_buf;
-		if (atomic_read(&mad_agent_priv->qp_info->snoop_count))
-			snoop_send(mad_agent_priv->qp_info,
-				   &local->mad_send_wr->send_buf,
-				   &mad_send_wc, IB_MAD_SNOOP_SEND_COMPLETIONS);
 		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
 						   &mad_send_wc);
 
@@ -3119,10 +2896,6 @@ static void init_mad_qp(struct ib_mad_port_private *port_priv,
 	init_mad_queue(qp_info, &qp_info->send_queue);
 	init_mad_queue(qp_info, &qp_info->recv_queue);
 	INIT_LIST_HEAD(&qp_info->overflow_list);
-	spin_lock_init(&qp_info->snoop_lock);
-	qp_info->snoop_table = NULL;
-	qp_info->snoop_table_size = 0;
-	atomic_set(&qp_info->snoop_count, 0);
 }
 
 static int create_mad_qp(struct ib_mad_qp_info *qp_info,
@@ -3166,7 +2939,6 @@ static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)
 		return;
 
 	ib_destroy_qp(qp_info->qp);
-	kfree(qp_info->snoop_table);
 }
 
 /*
@@ -3304,9 +3076,11 @@ static int ib_mad_port_close(struct ib_device *device, int port_num)
 	return 0;
 }
 
-static void ib_mad_init_device(struct ib_device *device)
+static int ib_mad_init_device(struct ib_device *device)
 {
 	int start, i;
+	unsigned int count = 0;
+	int ret;
 
 	start = rdma_start_port(device);
 
@@ -3314,17 +3088,23 @@ static void ib_mad_init_device(struct ib_device *device)
 		if (!rdma_cap_ib_mad(device, i))
 			continue;
 
-		if (ib_mad_port_open(device, i)) {
+		ret = ib_mad_port_open(device, i);
+		if (ret) {
 			dev_err(&device->dev, "Couldn't open port %d\n", i);
 			goto error;
 		}
-		if (ib_agent_port_open(device, i)) {
+		ret = ib_agent_port_open(device, i);
+		if (ret) {
 			dev_err(&device->dev,
 				"Couldn't open port %d for agents\n", i);
 			goto error_agent;
 		}
+		count++;
 	}
-	return;
+	if (!count)
+		return -EOPNOTSUPP;
+
+	return 0;
 
 error_agent:
 	if (ib_mad_port_close(device, i))
@@ -3341,6 +3121,7 @@ error:
 		if (ib_mad_port_close(device, i))
 			dev_err(&device->dev, "Couldn't close port %d\n", i);
 	}
+	return ret;
 }
 
 static void ib_mad_remove_device(struct ib_device *device, void *client_data)
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index 9c2d8b7f1af9..740f03ecc05d 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -42,7 +42,7 @@
 #include <rdma/ib_cache.h>
 #include "sa.h"
 
-static void mcast_add_one(struct ib_device *device);
+static int mcast_add_one(struct ib_device *device);
 static void mcast_remove_one(struct ib_device *device, void *client_data);
 
 static struct ib_client mcast_client = {
@@ -815,7 +815,7 @@ static void mcast_event_handler(struct ib_event_handler *handler,
 	}
 }
 
-static void mcast_add_one(struct ib_device *device)
+static int mcast_add_one(struct ib_device *device)
 {
 	struct mcast_device *dev;
 	struct mcast_port *port;
@@ -825,7 +825,7 @@ static void mcast_add_one(struct ib_device *device)
 	dev = kmalloc(struct_size(dev, port, device->phys_port_cnt),
 		      GFP_KERNEL);
 	if (!dev)
-		return;
+		return -ENOMEM;
 
 	dev->start_port = rdma_start_port(device);
 	dev->end_port = rdma_end_port(device);
@@ -845,7 +845,7 @@ static void mcast_add_one(struct ib_device *device)
 
 	if (!count) {
 		kfree(dev);
-		return;
+		return -EOPNOTSUPP;
 	}
 
 	dev->device = device;
@@ -853,6 +853,7 @@ static void mcast_add_one(struct ib_device *device)
 
 	INIT_IB_EVENT_HANDLER(&dev->event_handler, device, mcast_event_handler);
 	ib_register_event_handler(&dev->event_handler);
+	return 0;
 }
 
 static void mcast_remove_one(struct ib_device *device, void *client_data)
@@ -861,9 +862,6 @@ static void mcast_remove_one(struct ib_device *device, void *client_data)
 	struct mcast_port *port;
 	int i;
 
-	if (!dev)
-		return;
-
 	ib_unregister_event_handler(&dev->event_handler);
 	flush_workqueue(mcast_wq);
 
diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c
index e0a5e897e4b1..38de4942c682 100644
--- a/drivers/infiniband/core/rdma_core.c
+++ b/drivers/infiniband/core/rdma_core.c
@@ -130,6 +130,17 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj,
 	lockdep_assert_held(&ufile->hw_destroy_rwsem);
 	assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE);
 
+	if (reason == RDMA_REMOVE_ABORT_HWOBJ) {
+		reason = RDMA_REMOVE_ABORT;
+		ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason,
+								attrs);
+		/*
+		 * Drivers are not permitted to ignore RDMA_REMOVE_ABORT, see
+		 * ib_is_destroy_retryable, cleanup_retryable == false here.
+		 */
+		WARN_ON(ret);
+	}
+
 	if (reason == RDMA_REMOVE_ABORT) {
 		WARN_ON(!list_empty(&uobj->list));
 		WARN_ON(!uobj->context);
@@ -653,11 +664,15 @@ void rdma_alloc_commit_uobject(struct ib_uobject *uobj,
  * object and anything else connected to uobj before calling this.
  */
 void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
-			      struct uverbs_attr_bundle *attrs)
+			      struct uverbs_attr_bundle *attrs,
+			      bool hw_obj_valid)
 {
 	struct ib_uverbs_file *ufile = uobj->ufile;
 
-	uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);
+	uverbs_destroy_uobject(uobj,
+			       hw_obj_valid ? RDMA_REMOVE_ABORT_HWOBJ :
+					      RDMA_REMOVE_ABORT,
+			       attrs);
 
 	/* Matches the down_read in rdma_alloc_begin_uobject */
 	up_read(&ufile->hw_destroy_rwsem);
@@ -927,8 +942,8 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
 }
 
 void uverbs_finalize_object(struct ib_uobject *uobj,
-			    enum uverbs_obj_access access, bool commit,
-			    struct uverbs_attr_bundle *attrs)
+			    enum uverbs_obj_access access, bool hw_obj_valid,
+			    bool commit, struct uverbs_attr_bundle *attrs)
 {
 	/*
 	 * refcounts should be handled at the object level and not at the
@@ -951,7 +966,7 @@ void uverbs_finalize_object(struct ib_uobject *uobj,
 		if (commit)
 			rdma_alloc_commit_uobject(uobj, attrs);
 		else
-			rdma_alloc_abort_uobject(uobj, attrs);
+			rdma_alloc_abort_uobject(uobj, attrs, hw_obj_valid);
 		break;
 	default:
 		WARN_ON(true);
diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h
index 33978e0f1262..33706dad6c0f 100644
--- a/drivers/infiniband/core/rdma_core.h
+++ b/drivers/infiniband/core/rdma_core.h
@@ -64,8 +64,8 @@ uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
 			     s64 id, struct uverbs_attr_bundle *attrs);
 
 void uverbs_finalize_object(struct ib_uobject *uobj,
-			    enum uverbs_obj_access access, bool commit,
-			    struct uverbs_attr_bundle *attrs);
+			    enum uverbs_obj_access access, bool hw_obj_valid,
+			    bool commit, struct uverbs_attr_bundle *attrs);
 
 int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx);
 
@@ -159,6 +159,9 @@ extern const struct uapi_definition uverbs_def_obj_dm[];
 extern const struct uapi_definition uverbs_def_obj_flow_action[];
 extern const struct uapi_definition uverbs_def_obj_intf[];
 extern const struct uapi_definition uverbs_def_obj_mr[];
+extern const struct uapi_definition uverbs_def_obj_qp[];
+extern const struct uapi_definition uverbs_def_obj_srq[];
+extern const struct uapi_definition uverbs_def_obj_wq[];
 extern const struct uapi_definition uverbs_def_write_intf[];
 
 static inline const struct uverbs_api_write_method *
diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c
index 557efbf29197..614cff89fc71 100644
--- a/drivers/infiniband/core/rw.c
+++ b/drivers/infiniband/core/rw.c
@@ -129,7 +129,7 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 						    qp->integrity_en);
 	int i, j, ret = 0, count = 0;
 
-	ctx->nr_ops = (sg_cnt + pages_per_mr - 1) / pages_per_mr;
+	ctx->nr_ops = DIV_ROUND_UP(sg_cnt, pages_per_mr);
 	ctx->reg = kcalloc(ctx->nr_ops, sizeof(*ctx->reg), GFP_KERNEL);
 	if (!ctx->reg) {
 		ret = -ENOMEM;
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index 74e0058fcf9e..a2ed09a3c714 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -174,7 +174,7 @@ static const struct nla_policy ib_nl_policy[LS_NLA_TYPE_MAX] = {
 };
 
 
-static void ib_sa_add_one(struct ib_device *device);
+static int ib_sa_add_one(struct ib_device *device);
 static void ib_sa_remove_one(struct ib_device *device, void *client_data);
 
 static struct ib_client sa_client = {
@@ -190,7 +190,7 @@ static u32 tid;
 
 #define PATH_REC_FIELD(field) \
 	.struct_offset_bytes = offsetof(struct sa_path_rec, field),	\
-	.struct_size_bytes   = sizeof((struct sa_path_rec *)0)->field,	\
+	.struct_size_bytes   = sizeof_field(struct sa_path_rec, field),	\
 	.field_name          = "sa_path_rec:" #field
 
 static const struct ib_field path_rec_table[] = {
@@ -292,7 +292,7 @@ static const struct ib_field path_rec_table[] = {
 	.struct_offset_bytes = \
 		offsetof(struct sa_path_rec, field), \
 	.struct_size_bytes   = \
-		sizeof((struct sa_path_rec *)0)->field,	\
+		sizeof_field(struct sa_path_rec, field),	\
 	.field_name          = "sa_path_rec:" #field
 
 static const struct ib_field opa_path_rec_table[] = {
@@ -420,7 +420,7 @@ static const struct ib_field opa_path_rec_table[] = {
 
 #define MCMEMBER_REC_FIELD(field) \
 	.struct_offset_bytes = offsetof(struct ib_sa_mcmember_rec, field),	\
-	.struct_size_bytes   = sizeof ((struct ib_sa_mcmember_rec *) 0)->field,	\
+	.struct_size_bytes   = sizeof_field(struct ib_sa_mcmember_rec, field),	\
 	.field_name          = "sa_mcmember_rec:" #field
 
 static const struct ib_field mcmember_rec_table[] = {
@@ -504,7 +504,7 @@ static const struct ib_field mcmember_rec_table[] = {
 
 #define SERVICE_REC_FIELD(field) \
 	.struct_offset_bytes = offsetof(struct ib_sa_service_rec, field),	\
-	.struct_size_bytes   = sizeof ((struct ib_sa_service_rec *) 0)->field,	\
+	.struct_size_bytes   = sizeof_field(struct ib_sa_service_rec, field),	\
 	.field_name          = "sa_service_rec:" #field
 
 static const struct ib_field service_rec_table[] = {
@@ -552,7 +552,7 @@ static const struct ib_field service_rec_table[] = {
 
 #define CLASSPORTINFO_REC_FIELD(field) \
 	.struct_offset_bytes = offsetof(struct ib_class_port_info, field),	\
-	.struct_size_bytes   = sizeof((struct ib_class_port_info *)0)->field,	\
+	.struct_size_bytes   = sizeof_field(struct ib_class_port_info, field),	\
 	.field_name          = "ib_class_port_info:" #field
 
 static const struct ib_field ib_classport_info_rec_table[] = {
@@ -630,7 +630,7 @@ static const struct ib_field ib_classport_info_rec_table[] = {
 	.struct_offset_bytes =\
 		offsetof(struct opa_class_port_info, field),	\
 	.struct_size_bytes   = \
-		sizeof((struct opa_class_port_info *)0)->field,	\
+		sizeof_field(struct opa_class_port_info, field),	\
 	.field_name          = "opa_class_port_info:" #field
 
 static const struct ib_field opa_classport_info_rec_table[] = {
@@ -710,7 +710,7 @@ static const struct ib_field opa_classport_info_rec_table[] = {
 
 #define GUIDINFO_REC_FIELD(field) \
 	.struct_offset_bytes = offsetof(struct ib_sa_guidinfo_rec, field),	\
-	.struct_size_bytes   = sizeof((struct ib_sa_guidinfo_rec *) 0)->field,	\
+	.struct_size_bytes   = sizeof_field(struct ib_sa_guidinfo_rec, field),	\
 	.field_name          = "sa_guidinfo_rec:" #field
 
 static const struct ib_field guidinfo_rec_table[] = {
@@ -1412,17 +1412,13 @@ void ib_sa_pack_path(struct sa_path_rec *rec, void *attribute)
 EXPORT_SYMBOL(ib_sa_pack_path);
 
 static bool ib_sa_opa_pathrecord_support(struct ib_sa_client *client,
-					 struct ib_device *device,
+					 struct ib_sa_device *sa_dev,
 					 u8 port_num)
 {
-	struct ib_sa_device *sa_dev = ib_get_client_data(device, &sa_client);
 	struct ib_sa_port *port;
 	unsigned long flags;
 	bool ret = false;
 
-	if (!sa_dev)
-		return ret;
-
 	port = &sa_dev->port[port_num - sa_dev->start_port];
 	spin_lock_irqsave(&port->classport_lock, flags);
 	if (!port->classport_info.valid)
@@ -1450,8 +1446,8 @@ enum opa_pr_supported {
  * query is possible.
  */
 static int opa_pr_query_possible(struct ib_sa_client *client,
-				 struct ib_device *device,
-				 u8 port_num,
+				 struct ib_sa_device *sa_dev,
+				 struct ib_device *device, u8 port_num,
 				 struct sa_path_rec *rec)
 {
 	struct ib_port_attr port_attr;
@@ -1459,7 +1455,7 @@ static int opa_pr_query_possible(struct ib_sa_client *client,
 	if (ib_query_port(device, port_num, &port_attr))
 		return PR_NOT_SUPPORTED;
 
-	if (ib_sa_opa_pathrecord_support(client, device, port_num))
+	if (ib_sa_opa_pathrecord_support(client, sa_dev, port_num))
 		return PR_OPA_SUPPORTED;
 
 	if (port_attr.lid >= be16_to_cpu(IB_MULTICAST_LID_BASE))
@@ -1574,7 +1570,8 @@ int ib_sa_path_rec_get(struct ib_sa_client *client,
 
 	query->sa_query.port     = port;
 	if (rec->rec_type == SA_PATH_REC_TYPE_OPA) {
-		status = opa_pr_query_possible(client, device, port_num, rec);
+		status = opa_pr_query_possible(client, sa_dev, device, port_num,
+					       rec);
 		if (status == PR_NOT_SUPPORTED) {
 			ret = -EINVAL;
 			goto err1;
@@ -2325,18 +2322,19 @@ static void ib_sa_event(struct ib_event_handler *handler,
 	}
 }
 
-static void ib_sa_add_one(struct ib_device *device)
+static int ib_sa_add_one(struct ib_device *device)
 {
 	struct ib_sa_device *sa_dev;
 	int s, e, i;
 	int count = 0;
+	int ret;
 
 	s = rdma_start_port(device);
 	e = rdma_end_port(device);
 
 	sa_dev = kzalloc(struct_size(sa_dev, port, e - s + 1), GFP_KERNEL);
 	if (!sa_dev)
-		return;
+		return -ENOMEM;
 
 	sa_dev->start_port = s;
 	sa_dev->end_port   = e;
@@ -2356,8 +2354,10 @@ static void ib_sa_add_one(struct ib_device *device)
 			ib_register_mad_agent(device, i + s, IB_QPT_GSI,
 					      NULL, 0, send_handler,
 					      recv_handler, sa_dev, 0);
-		if (IS_ERR(sa_dev->port[i].agent))
+		if (IS_ERR(sa_dev->port[i].agent)) {
+			ret = PTR_ERR(sa_dev->port[i].agent);
 			goto err;
+		}
 
 		INIT_WORK(&sa_dev->port[i].update_task, update_sm_ah);
 		INIT_DELAYED_WORK(&sa_dev->port[i].ib_cpi_work,
@@ -2366,8 +2366,10 @@ static void ib_sa_add_one(struct ib_device *device)
 		count++;
 	}
 
-	if (!count)
+	if (!count) {
+		ret = -EOPNOTSUPP;
 		goto free;
+	}
 
 	ib_set_client_data(device, &sa_client, sa_dev);
 
@@ -2386,7 +2388,7 @@ static void ib_sa_add_one(struct ib_device *device)
 			update_sm_ah(&sa_dev->port[i].update_task);
 	}
 
-	return;
+	return 0;
 
 err:
 	while (--i >= 0) {
@@ -2395,7 +2397,7 @@ err:
 	}
 free:
 	kfree(sa_dev);
-	return;
+	return ret;
 }
 
 static void ib_sa_remove_one(struct ib_device *device, void *client_data)
@@ -2403,9 +2405,6 @@ static void ib_sa_remove_one(struct ib_device *device, void *client_data)
 	struct ib_sa_device *sa_dev = client_data;
 	int i;
 
-	if (!sa_dev)
-		return;
-
 	ib_unregister_event_handler(&sa_dev->event_handler);
 	flush_workqueue(ib_wq);
 
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index 087682e6969e..defe9cd4c5ee 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -1058,8 +1058,7 @@ static int add_port(struct ib_core_device *coredev, int port_num)
 				   coredev->ports_kobj,
 				   "%d", port_num);
 	if (ret) {
-		kfree(p);
-		return ret;
+		goto err_put;
 	}
 
 	p->gid_attr_group = kzalloc(sizeof(*p->gid_attr_group), GFP_KERNEL);
@@ -1072,8 +1071,7 @@ static int add_port(struct ib_core_device *coredev, int port_num)
 	ret = kobject_init_and_add(&p->gid_attr_group->kobj, &gid_attr_type,
 				   &p->kobj, "gid_attrs");
 	if (ret) {
-		kfree(p->gid_attr_group);
-		goto err_put;
+		goto err_put_gid_attrs;
 	}
 
 	if (device->ops.process_mad && is_full_dev) {
@@ -1404,8 +1402,10 @@ int ib_port_register_module_stat(struct ib_device *device, u8 port_num,
 
 		ret = kobject_init_and_add(kobj, ktype, &port->kobj, "%s",
 					   name);
-		if (ret)
+		if (ret) {
+			kobject_put(kobj);
 			return ret;
+		}
 	}
 
 	return 0;
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 16b6cf57fa85..5b87eee8ccc8 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -52,6 +52,7 @@
 #include <rdma/rdma_cm_ib.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib.h>
+#include <rdma/ib_cm.h>
 #include <rdma/rdma_netlink.h>
 #include "core_priv.h"
 
@@ -360,6 +361,9 @@ static int ucma_event_handler(struct rdma_cm_id *cm_id,
 		ucma_copy_conn_event(&uevent->resp.param.conn,
 				     &event->param.conn);
 
+	uevent->resp.ece.vendor_id = event->ece.vendor_id;
+	uevent->resp.ece.attr_mod = event->ece.attr_mod;
+
 	if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) {
 		if (!ctx->backlog) {
 			ret = -ENOMEM;
@@ -404,7 +408,8 @@ static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf,
 	 * Old 32 bit user space does not send the 4 byte padding in the
 	 * reserved field. We don't care, allow it to keep working.
 	 */
-	if (out_len < sizeof(uevent->resp) - sizeof(uevent->resp.reserved))
+	if (out_len < sizeof(uevent->resp) - sizeof(uevent->resp.reserved) -
+			      sizeof(uevent->resp.ece))
 		return -ENOSPC;
 
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
@@ -845,7 +850,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,
 	struct sockaddr *addr;
 	int ret = 0;
 
-	if (out_len < sizeof(resp))
+	if (out_len < offsetof(struct rdma_ucm_query_route_resp, ibdev_index))
 		return -ENOSPC;
 
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
@@ -869,6 +874,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,
 		goto out;
 
 	resp.node_guid = (__force __u64) ctx->cm_id->device->node_guid;
+	resp.ibdev_index = ctx->cm_id->device->index;
 	resp.port_num = ctx->cm_id->port_num;
 
 	if (rdma_cap_ib_sa(ctx->cm_id->device, ctx->cm_id->port_num))
@@ -880,8 +886,8 @@ static ssize_t ucma_query_route(struct ucma_file *file,
 
 out:
 	mutex_unlock(&ctx->mutex);
-	if (copy_to_user(u64_to_user_ptr(cmd.response),
-			 &resp, sizeof(resp)))
+	if (copy_to_user(u64_to_user_ptr(cmd.response), &resp,
+			 min_t(size_t, out_len, sizeof(resp))))
 		ret = -EFAULT;
 
 	ucma_put_ctx(ctx);
@@ -895,6 +901,7 @@ static void ucma_query_device_addr(struct rdma_cm_id *cm_id,
 		return;
 
 	resp->node_guid = (__force __u64) cm_id->device->node_guid;
+	resp->ibdev_index = cm_id->device->index;
 	resp->port_num = cm_id->port_num;
 	resp->pkey = (__force __u16) cpu_to_be16(
 		     ib_addr_get_pkey(&cm_id->route.addr.dev_addr));
@@ -907,7 +914,7 @@ static ssize_t ucma_query_addr(struct ucma_context *ctx,
 	struct sockaddr *addr;
 	int ret = 0;
 
-	if (out_len < sizeof(resp))
+	if (out_len < offsetof(struct rdma_ucm_query_addr_resp, ibdev_index))
 		return -ENOSPC;
 
 	memset(&resp, 0, sizeof resp);
@@ -922,7 +929,7 @@ static ssize_t ucma_query_addr(struct ucma_context *ctx,
 
 	ucma_query_device_addr(ctx->cm_id, &resp);
 
-	if (copy_to_user(response, &resp, sizeof(resp)))
+	if (copy_to_user(response, &resp, min_t(size_t, out_len, sizeof(resp))))
 		ret = -EFAULT;
 
 	return ret;
@@ -974,7 +981,7 @@ static ssize_t ucma_query_gid(struct ucma_context *ctx,
 	struct sockaddr_ib *addr;
 	int ret = 0;
 
-	if (out_len < sizeof(resp))
+	if (out_len < offsetof(struct rdma_ucm_query_addr_resp, ibdev_index))
 		return -ENOSPC;
 
 	memset(&resp, 0, sizeof resp);
@@ -1007,7 +1014,7 @@ static ssize_t ucma_query_gid(struct ucma_context *ctx,
 						    &ctx->cm_id->route.addr.dst_addr);
 	}
 
-	if (copy_to_user(response, &resp, sizeof(resp)))
+	if (copy_to_user(response, &resp, min_t(size_t, out_len, sizeof(resp))))
 		ret = -EFAULT;
 
 	return ret;
@@ -1070,12 +1077,15 @@ static void ucma_copy_conn_param(struct rdma_cm_id *id,
 static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,
 			    int in_len, int out_len)
 {
-	struct rdma_ucm_connect cmd;
 	struct rdma_conn_param conn_param;
+	struct rdma_ucm_ece ece = {};
+	struct rdma_ucm_connect cmd;
 	struct ucma_context *ctx;
+	size_t in_size;
 	int ret;
 
-	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+	in_size = min_t(size_t, in_len, sizeof(cmd));
+	if (copy_from_user(&cmd, inbuf, in_size))
 		return -EFAULT;
 
 	if (!cmd.conn_param.valid)
@@ -1086,8 +1096,13 @@ static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,
 		return PTR_ERR(ctx);
 
 	ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param);
+	if (offsetofend(typeof(cmd), ece) <= in_size) {
+		ece.vendor_id = cmd.ece.vendor_id;
+		ece.attr_mod = cmd.ece.attr_mod;
+	}
+
 	mutex_lock(&ctx->mutex);
-	ret = rdma_connect(ctx->cm_id, &conn_param);
+	ret = rdma_connect_ece(ctx->cm_id, &conn_param, &ece);
 	mutex_unlock(&ctx->mutex);
 	ucma_put_ctx(ctx);
 	return ret;
@@ -1121,28 +1136,36 @@ static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf,
 {
 	struct rdma_ucm_accept cmd;
 	struct rdma_conn_param conn_param;
+	struct rdma_ucm_ece ece = {};
 	struct ucma_context *ctx;
+	size_t in_size;
 	int ret;
 
-	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
+	in_size = min_t(size_t, in_len, sizeof(cmd));
+	if (copy_from_user(&cmd, inbuf, in_size))
 		return -EFAULT;
 
 	ctx = ucma_get_ctx_dev(file, cmd.id);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
+	if (offsetofend(typeof(cmd), ece) <= in_size) {
+		ece.vendor_id = cmd.ece.vendor_id;
+		ece.attr_mod = cmd.ece.attr_mod;
+	}
+
 	if (cmd.conn_param.valid) {
 		ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param);
 		mutex_lock(&file->mut);
 		mutex_lock(&ctx->mutex);
-		ret = __rdma_accept(ctx->cm_id, &conn_param, NULL);
+		ret = __rdma_accept_ece(ctx->cm_id, &conn_param, NULL, &ece);
 		mutex_unlock(&ctx->mutex);
 		if (!ret)
 			ctx->uid = cmd.uid;
 		mutex_unlock(&file->mut);
 	} else {
 		mutex_lock(&ctx->mutex);
-		ret = __rdma_accept(ctx->cm_id, NULL, NULL);
+		ret = __rdma_accept_ece(ctx->cm_id, NULL, NULL, &ece);
 		mutex_unlock(&ctx->mutex);
 	}
 	ucma_put_ctx(ctx);
@@ -1159,12 +1182,24 @@ static ssize_t ucma_reject(struct ucma_file *file, const char __user *inbuf,
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
+	if (!cmd.reason)
+		cmd.reason = IB_CM_REJ_CONSUMER_DEFINED;
+
+	switch (cmd.reason) {
+	case IB_CM_REJ_CONSUMER_DEFINED:
+	case IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED:
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	ctx = ucma_get_ctx_dev(file, cmd.id);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
 	mutex_lock(&ctx->mutex);
-	ret = rdma_reject(ctx->cm_id, cmd.private_data, cmd.private_data_len);
+	ret = rdma_reject(ctx->cm_id, cmd.private_data, cmd.private_data_len,
+			  cmd.reason);
 	mutex_unlock(&ctx->mutex);
 	ucma_put_ctx(ctx);
 	return ret;
diff --git a/drivers/infiniband/core/ud_header.c b/drivers/infiniband/core/ud_header.c
index 29a45d2f8898..d65d541b9a25 100644
--- a/drivers/infiniband/core/ud_header.c
+++ b/drivers/infiniband/core/ud_header.c
@@ -41,7 +41,7 @@
 
 #define STRUCT_FIELD(header, field) \
 	.struct_offset_bytes = offsetof(struct ib_unpacked_ ## header, field),      \
-	.struct_size_bytes   = sizeof ((struct ib_unpacked_ ## header *) 0)->field, \
+	.struct_size_bytes   = sizeof_field(struct ib_unpacked_ ## header, field), \
 	.field_name          = #header ":" #field
 
 static const struct ib_field lrh_table[]  = {
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index 3b1e627d9a8d..ccd28405451c 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -429,7 +429,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
 				ALIGN(bcnt, PAGE_SIZE) / PAGE_SIZE,
 				PAGE_SIZE / sizeof(struct page *));
 
-		down_read(&owning_mm->mmap_sem);
+		mmap_read_lock(owning_mm);
 		/*
 		 * Note: this might result in redundent page getting. We can
 		 * avoid this by checking dma_list to be 0 before calling
@@ -440,7 +440,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
 		npages = get_user_pages_remote(owning_process, owning_mm,
 				user_virt, gup_num_pages,
 				flags, local_page_list, NULL, NULL);
-		up_read(&owning_mm->mmap_sem);
+		mmap_read_unlock(owning_mm);
 
 		if (npages < 0) {
 			if (npages != -EAGAIN)
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index da229eab5903..b0d0b522cc76 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -142,7 +142,7 @@ static dev_t dynamic_issm_dev;
 
 static DEFINE_IDA(umad_ida);
 
-static void ib_umad_add_one(struct ib_device *device);
+static int ib_umad_add_one(struct ib_device *device);
 static void ib_umad_remove_one(struct ib_device *device, void *client_data);
 
 static void ib_umad_dev_free(struct kref *kref)
@@ -1352,37 +1352,41 @@ static void ib_umad_kill_port(struct ib_umad_port *port)
 	put_device(&port->dev);
 }
 
-static void ib_umad_add_one(struct ib_device *device)
+static int ib_umad_add_one(struct ib_device *device)
 {
 	struct ib_umad_device *umad_dev;
 	int s, e, i;
 	int count = 0;
+	int ret;
 
 	s = rdma_start_port(device);
 	e = rdma_end_port(device);
 
 	umad_dev = kzalloc(struct_size(umad_dev, ports, e - s + 1), GFP_KERNEL);
 	if (!umad_dev)
-		return;
+		return -ENOMEM;
 
 	kref_init(&umad_dev->kref);
 	for (i = s; i <= e; ++i) {
 		if (!rdma_cap_ib_mad(device, i))
 			continue;
 
-		if (ib_umad_init_port(device, i, umad_dev,
-				      &umad_dev->ports[i - s]))
+		ret = ib_umad_init_port(device, i, umad_dev,
+					&umad_dev->ports[i - s]);
+		if (ret)
 			goto err;
 
 		count++;
 	}
 
-	if (!count)
+	if (!count) {
+		ret = -EOPNOTSUPP;
 		goto free;
+	}
 
 	ib_set_client_data(device, &umad_client, umad_dev);
 
-	return;
+	return 0;
 
 err:
 	while (--i >= s) {
@@ -1394,6 +1398,7 @@ err:
 free:
 	/* balances kref_init */
 	ib_umad_dev_put(umad_dev);
+	return ret;
 }
 
 static void ib_umad_remove_one(struct ib_device *device, void *client_data)
@@ -1401,9 +1406,6 @@ static void ib_umad_remove_one(struct ib_device *device, void *client_data)
 	struct ib_umad_device *umad_dev = client_data;
 	unsigned int i;
 
-	if (!umad_dev)
-		return;
-
 	rdma_for_each_port (device, i) {
 		if (rdma_cap_ib_mad(device, i))
 			ib_umad_kill_port(
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index 3d189c7ee59e..53a10479958b 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -142,7 +142,7 @@ struct ib_uverbs_file {
 	 * ucontext_lock held
 	 */
 	struct ib_ucontext		       *ucontext;
-	struct ib_uverbs_async_event_file      *async_file;
+	struct ib_uverbs_async_event_file      *default_async_file;
 	struct list_head			list;
 
 	/*
@@ -180,6 +180,7 @@ struct ib_uverbs_mcast_entry {
 
 struct ib_uevent_object {
 	struct ib_uobject	uobject;
+	struct ib_uverbs_async_event_file *event_file;
 	/* List member for ib_uverbs_async_event_file list */
 	struct list_head	event_list;
 	u32			events_reported;
@@ -296,6 +297,24 @@ static inline u32 make_port_cap_flags(const struct ib_port_attr *attr)
 	return res;
 }
 
+static inline struct ib_uverbs_async_event_file *
+ib_uverbs_get_async_event(struct uverbs_attr_bundle *attrs,
+			  u16 id)
+{
+	struct ib_uobject *async_ev_file_uobj;
+	struct ib_uverbs_async_event_file *async_ev_file;
+
+	async_ev_file_uobj = uverbs_attr_get_uobject(attrs, id);
+	if (IS_ERR(async_ev_file_uobj))
+		async_ev_file = READ_ONCE(attrs->ufile->default_async_file);
+	else
+		async_ev_file = container_of(async_ev_file_uobj,
+				       struct ib_uverbs_async_event_file,
+				       uobj);
+	if (async_ev_file)
+		uverbs_uobject_get(&async_ev_file->uobj);
+	return async_ev_file;
+}
 
 void copy_port_attr_to_resp(struct ib_port_attr *attr,
 			    struct ib_uverbs_query_port_resp *resp,
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 060b4ebbd2ba..b48b3f6e632d 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -311,7 +311,7 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
 	return 0;
 
 err_uobj:
-	rdma_alloc_abort_uobject(uobj, attrs);
+	rdma_alloc_abort_uobject(uobj, attrs, false);
 err_ucontext:
 	kfree(attrs->context);
 	attrs->context = NULL;
@@ -356,8 +356,6 @@ static void copy_query_dev_fields(struct ib_ucontext *ucontext,
 	resp->max_mcast_qp_attach	= attr->max_mcast_qp_attach;
 	resp->max_total_mcast_qp_attach	= attr->max_total_mcast_qp_attach;
 	resp->max_ah			= attr->max_ah;
-	resp->max_fmr			= attr->max_fmr;
-	resp->max_map_per_fmr		= attr->max_map_per_fmr;
 	resp->max_srq			= attr->max_srq;
 	resp->max_srq_wr		= attr->max_srq_wr;
 	resp->max_srq_sge		= attr->max_srq_sge;
@@ -1051,6 +1049,10 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
 		goto err_free;
 
 	obj->uevent.uobject.object = cq;
+	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
+	if (obj->uevent.event_file)
+		uverbs_uobject_get(&obj->uevent.event_file->uobj);
+
 	memset(&resp, 0, sizeof resp);
 	resp.base.cq_handle = obj->uevent.uobject.id;
 	resp.base.cqe       = cq->cqe;
@@ -1067,6 +1069,8 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
 	return obj;
 
 err_cb:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
 	ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
 	cq = NULL;
 err_free:
@@ -1460,6 +1464,9 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
 	}
 
 	obj->uevent.uobject.object = qp;
+	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
+	if (obj->uevent.event_file)
+		uverbs_uobject_get(&obj->uevent.event_file->uobj);
 
 	memset(&resp, 0, sizeof resp);
 	resp.base.qpn             = qp->qp_num;
@@ -1473,7 +1480,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
 
 	ret = uverbs_response(attrs, &resp, sizeof(resp));
 	if (ret)
-		goto err_cb;
+		goto err_uevent;
 
 	if (xrcd) {
 		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
@@ -1498,6 +1505,9 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
 
 	rdma_alloc_commit_uobject(&obj->uevent.uobject, attrs);
 	return 0;
+err_uevent:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
 err_cb:
 	ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs));
 
@@ -2954,11 +2964,11 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
 	wq_init_attr.cq = cq;
 	wq_init_attr.max_sge = cmd.max_sge;
 	wq_init_attr.max_wr = cmd.max_wr;
-	wq_init_attr.wq_context = attrs->ufile;
 	wq_init_attr.wq_type = cmd.wq_type;
 	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
 	wq_init_attr.create_flags = cmd.create_flags;
 	INIT_LIST_HEAD(&obj->uevent.event_list);
+	obj->uevent.uobject.user_handle = cmd.user_handle;
 
 	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
 	if (IS_ERR(wq)) {
@@ -2972,12 +2982,12 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
 	wq->cq = cq;
 	wq->pd = pd;
 	wq->device = pd->device;
-	wq->wq_context = wq_init_attr.wq_context;
 	atomic_set(&wq->usecnt, 0);
 	atomic_inc(&pd->usecnt);
 	atomic_inc(&cq->usecnt);
-	wq->uobject = obj;
-	obj->uevent.uobject.object = wq;
+	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
+	if (obj->uevent.event_file)
+		uverbs_uobject_get(&obj->uevent.event_file->uobj);
 
 	memset(&resp, 0, sizeof(resp));
 	resp.wq_handle = obj->uevent.uobject.id;
@@ -2996,6 +3006,8 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
 	return 0;
 
 err_copy:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
 	ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs));
 err_put_cq:
 	rdma_lookup_put_uobject(&cq->uobject->uevent.uobject,
@@ -3441,46 +3453,25 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
 	}
 
 	attr.event_handler  = ib_uverbs_srq_event_handler;
-	attr.srq_context    = attrs->ufile;
 	attr.srq_type       = cmd->srq_type;
 	attr.attr.max_wr    = cmd->max_wr;
 	attr.attr.max_sge   = cmd->max_sge;
 	attr.attr.srq_limit = cmd->srq_limit;
 
 	INIT_LIST_HEAD(&obj->uevent.event_list);
+	obj->uevent.uobject.user_handle = cmd->user_handle;
 
-	srq = rdma_zalloc_drv_obj(ib_dev, ib_srq);
-	if (!srq) {
-		ret = -ENOMEM;
-		goto err_put;
-	}
-
-	srq->device        = pd->device;
-	srq->pd            = pd;
-	srq->srq_type	   = cmd->srq_type;
-	srq->uobject       = obj;
-	srq->event_handler = attr.event_handler;
-	srq->srq_context   = attr.srq_context;
-
-	ret = pd->device->ops.create_srq(srq, &attr, udata);
-	if (ret)
-		goto err_free;
-
-	if (ib_srq_has_cq(cmd->srq_type)) {
-		srq->ext.cq       = attr.ext.cq;
-		atomic_inc(&attr.ext.cq->usecnt);
-	}
-
-	if (cmd->srq_type == IB_SRQT_XRC) {
-		srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
-		atomic_inc(&attr.ext.xrc.xrcd->usecnt);
+	srq = ib_create_srq_user(pd, &attr, obj, udata);
+	if (IS_ERR(srq)) {
+		ret = PTR_ERR(srq);
+		goto err_put_pd;
 	}
 
-	atomic_inc(&pd->usecnt);
-	atomic_set(&srq->usecnt, 0);
-
 	obj->uevent.uobject.object = srq;
 	obj->uevent.uobject.user_handle = cmd->user_handle;
+	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
+	if (obj->uevent.event_file)
+		uverbs_uobject_get(&obj->uevent.event_file->uobj);
 
 	memset(&resp, 0, sizeof resp);
 	resp.srq_handle = obj->uevent.uobject.id;
@@ -3505,14 +3496,11 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
 	return 0;
 
 err_copy:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
 	ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs));
-	/* It was released in ib_destroy_srq_user */
-	srq = NULL;
-err_free:
-	kfree(srq);
-err_put:
+err_put_pd:
 	uobj_put_obj_read(pd);
-
 err_put_cq:
 	if (ib_srq_has_cq(cmd->srq_type))
 		rdma_lookup_put_uobject(&attr.ext.cq->uobject->uevent.uobject,
@@ -3751,7 +3739,7 @@ static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
 #define UAPI_DEF_WRITE_IO(req, resp)                                           \
 	.write.has_resp = 1 +                                                  \
 			  BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) +    \
-			  BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) !=    \
+			  BUILD_BUG_ON_ZERO(sizeof_field(req, response) !=    \
 					    sizeof(u64)),                      \
 	.write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
 
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
index 538affbc517e..2d882c02387c 100644
--- a/drivers/infiniband/core/uverbs_ioctl.c
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -58,6 +58,7 @@ struct bundle_priv {
 
 	DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
 	DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
+	DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
 
 	/*
 	 * Must be last. bundle ends in a flex array which overlaps
@@ -136,7 +137,7 @@ EXPORT_SYMBOL(_uverbs_alloc);
 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
 				   u16 len)
 {
-	if (uattr->len > sizeof(((struct ib_uverbs_attr *)0)->data))
+	if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
 		return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
 					    uattr->len - len);
 
@@ -230,7 +231,8 @@ static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
 
 	for (i = 0; i != attr->len; i++)
 		uverbs_finalize_object(attr->uobjects[i],
-				       spec->u2.objs_arr.access, commit, attrs);
+				       spec->u2.objs_arr.access, false, commit,
+				       attrs);
 }
 
 static int uverbs_process_attr(struct bundle_priv *pbundle,
@@ -502,7 +504,9 @@ static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
 
 		uverbs_finalize_object(
 			attr->obj_attr.uobject,
-			attr->obj_attr.attr_elm->spec.u.obj.access, commit,
+			attr->obj_attr.attr_elm->spec.u.obj.access,
+			test_bit(i, pbundle->uobj_hw_obj_valid),
+			commit,
 			&pbundle->bundle);
 	}
 
@@ -590,6 +594,8 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
 	       sizeof(pbundle->bundle.attr_present));
 	memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
 	memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
+	memset(pbundle->uobj_hw_obj_valid, 0,
+	       sizeof(pbundle->uobj_hw_obj_valid));
 
 	ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
 	bundle_destroy(pbundle, ret == 0);
@@ -784,3 +790,15 @@ int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
 	}
 	return uverbs_copy_to(bundle, idx, from, size);
 }
+
+/* Once called an abort will call through to the type's destroy_hw() */
+void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
+				 u16 idx)
+{
+	struct bundle_priv *pbundle =
+		container_of(bundle, struct bundle_priv, bundle);
+
+	__set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
+		  pbundle->uobj_hw_obj_valid);
+}
+EXPORT_SYMBOL(uverbs_finalize_uobj_create);
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 1bab8de14757..69e4755cc04b 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -75,7 +75,7 @@ static dev_t dynamic_uverbs_dev;
 static struct class *uverbs_class;
 
 static DEFINE_IDA(uverbs_ida);
-static void ib_uverbs_add_one(struct ib_device *device);
+static int ib_uverbs_add_one(struct ib_device *device);
 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);
 
 /*
@@ -146,8 +146,7 @@ void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file,
 
 void ib_uverbs_release_uevent(struct ib_uevent_object *uobj)
 {
-	struct ib_uverbs_async_event_file *async_file =
-		READ_ONCE(uobj->uobject.ufile->async_file);
+	struct ib_uverbs_async_event_file *async_file = uobj->event_file;
 	struct ib_uverbs_event *evt, *tmp;
 
 	if (!async_file)
@@ -159,6 +158,7 @@ void ib_uverbs_release_uevent(struct ib_uevent_object *uobj)
 		kfree(evt);
 	}
 	spin_unlock_irq(&async_file->ev_queue.lock);
+	uverbs_uobject_put(&async_file->uobj);
 }
 
 void ib_uverbs_detach_umcast(struct ib_qp *qp,
@@ -197,8 +197,8 @@ void ib_uverbs_release_file(struct kref *ref)
 	if (atomic_dec_and_test(&file->device->refcount))
 		ib_uverbs_comp_dev(file->device);
 
-	if (file->async_file)
-		uverbs_uobject_put(&file->async_file->uobj);
+	if (file->default_async_file)
+		uverbs_uobject_put(&file->default_async_file->uobj);
 	put_device(&file->device->dev);
 
 	if (file->disassociate_page)
@@ -296,6 +296,8 @@ static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,
 	spin_lock_irq(&ev_queue->lock);
 	if (!list_empty(&ev_queue->event_list))
 		pollflags = EPOLLIN | EPOLLRDNORM;
+	else if (ev_queue->is_closed)
+		pollflags = EPOLLERR;
 	spin_unlock_irq(&ev_queue->lock);
 
 	return pollflags;
@@ -425,7 +427,7 @@ void ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file,
 static void uverbs_uobj_event(struct ib_uevent_object *eobj,
 			      struct ib_event *event)
 {
-	ib_uverbs_async_handler(READ_ONCE(eobj->uobject.ufile->async_file),
+	ib_uverbs_async_handler(eobj->event_file,
 				eobj->uobject.user_handle, event->event,
 				&eobj->event_list, &eobj->events_reported);
 }
@@ -482,10 +484,10 @@ void ib_uverbs_init_async_event_file(
 
 	/* The first async_event_file becomes the default one for the file. */
 	mutex_lock(&uverbs_file->ucontext_lock);
-	if (!uverbs_file->async_file) {
+	if (!uverbs_file->default_async_file) {
 		/* Pairs with the put in ib_uverbs_release_file */
 		uverbs_uobject_get(&async_file->uobj);
-		smp_store_release(&uverbs_file->async_file, async_file);
+		smp_store_release(&uverbs_file->default_async_file, async_file);
 	}
 	mutex_unlock(&uverbs_file->ucontext_lock);
 
@@ -833,12 +835,12 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 			return;
 
 		/*
-		 * The umap_lock is nested under mmap_sem since it used within
+		 * The umap_lock is nested under mmap_lock since it used within
 		 * the vma_ops callbacks, so we have to clean the list one mm
 		 * at a time to get the lock ordering right. Typically there
 		 * will only be one mm, so no big deal.
 		 */
-		down_read(&mm->mmap_sem);
+		mmap_read_lock(mm);
 		if (!mmget_still_valid(mm))
 			goto skip_mm;
 		mutex_lock(&ufile->umap_lock);
@@ -860,7 +862,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 		}
 		mutex_unlock(&ufile->umap_lock);
 	skip_mm:
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 		mmput(mm);
 	}
 }
@@ -1092,7 +1094,7 @@ static int ib_uverbs_create_uapi(struct ib_device *device,
 	return 0;
 }
 
-static void ib_uverbs_add_one(struct ib_device *device)
+static int ib_uverbs_add_one(struct ib_device *device)
 {
 	int devnum;
 	dev_t base;
@@ -1100,16 +1102,16 @@ static void ib_uverbs_add_one(struct ib_device *device)
 	int ret;
 
 	if (!device->ops.alloc_ucontext)
-		return;
+		return -EOPNOTSUPP;
 
 	uverbs_dev = kzalloc(sizeof(*uverbs_dev), GFP_KERNEL);
 	if (!uverbs_dev)
-		return;
+		return -ENOMEM;
 
 	ret = init_srcu_struct(&uverbs_dev->disassociate_srcu);
 	if (ret) {
 		kfree(uverbs_dev);
-		return;
+		return -ENOMEM;
 	}
 
 	device_initialize(&uverbs_dev->dev);
@@ -1129,15 +1131,18 @@ static void ib_uverbs_add_one(struct ib_device *device)
 
 	devnum = ida_alloc_max(&uverbs_ida, IB_UVERBS_MAX_DEVICES - 1,
 			       GFP_KERNEL);
-	if (devnum < 0)
+	if (devnum < 0) {
+		ret = -ENOMEM;
 		goto err;
+	}
 	uverbs_dev->devnum = devnum;
 	if (devnum >= IB_UVERBS_NUM_FIXED_MINOR)
 		base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR;
 	else
 		base = IB_UVERBS_BASE_DEV + devnum;
 
-	if (ib_uverbs_create_uapi(device, uverbs_dev))
+	ret = ib_uverbs_create_uapi(device, uverbs_dev);
+	if (ret)
 		goto err_uapi;
 
 	uverbs_dev->dev.devt = base;
@@ -1152,7 +1157,7 @@ static void ib_uverbs_add_one(struct ib_device *device)
 		goto err_uapi;
 
 	ib_set_client_data(device, &uverbs_client, uverbs_dev);
-	return;
+	return 0;
 
 err_uapi:
 	ida_free(&uverbs_ida, devnum);
@@ -1161,7 +1166,7 @@ err:
 		ib_uverbs_comp_dev(uverbs_dev);
 	wait_for_completion(&uverbs_dev->comp);
 	put_device(&uverbs_dev->dev);
-	return;
+	return ret;
 }
 
 static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
@@ -1201,9 +1206,6 @@ static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
 	struct ib_uverbs_device *uverbs_dev = client_data;
 	int wait_clients = 1;
 
-	if (!uverbs_dev)
-		return;
-
 	cdev_device_del(&uverbs_dev->cdev, &uverbs_dev->dev);
 	ida_free(&uverbs_ida, uverbs_dev->devnum);
 
diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c
index 3abfc63225cb..08c39cfb1bd9 100644
--- a/drivers/infiniband/core/uverbs_std_types.c
+++ b/drivers/infiniband/core/uverbs_std_types.c
@@ -75,40 +75,6 @@ static int uverbs_free_mw(struct ib_uobject *uobject,
 	return uverbs_dealloc_mw((struct ib_mw *)uobject->object);
 }
 
-static int uverbs_free_qp(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why,
-			  struct uverbs_attr_bundle *attrs)
-{
-	struct ib_qp *qp = uobject->object;
-	struct ib_uqp_object *uqp =
-		container_of(uobject, struct ib_uqp_object, uevent.uobject);
-	int ret;
-
-	/*
-	 * If this is a user triggered destroy then do not allow destruction
-	 * until the user cleans up all the mcast bindings. Unlike in other
-	 * places we forcibly clean up the mcast attachments for !DESTROY
-	 * because the mcast attaches are not ubojects and will not be
-	 * destroyed by anything else during cleanup processing.
-	 */
-	if (why == RDMA_REMOVE_DESTROY) {
-		if (!list_empty(&uqp->mcast_list))
-			return -EBUSY;
-	} else if (qp == qp->real_qp) {
-		ib_uverbs_detach_umcast(qp, uqp);
-	}
-
-	ret = ib_destroy_qp_user(qp, &attrs->driver_udata);
-	if (ib_is_destroy_retryable(ret, why, uobject))
-		return ret;
-
-	if (uqp->uxrcd)
-		atomic_dec(&uqp->uxrcd->refcnt);
-
-	ib_uverbs_release_uevent(&uqp->uevent);
-	return ret;
-}
-
 static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject,
 				   enum rdma_remove_reason why,
 				   struct uverbs_attr_bundle *attrs)
@@ -125,48 +91,6 @@ static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject,
 	return ret;
 }
 
-static int uverbs_free_wq(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why,
-			  struct uverbs_attr_bundle *attrs)
-{
-	struct ib_wq *wq = uobject->object;
-	struct ib_uwq_object *uwq =
-		container_of(uobject, struct ib_uwq_object, uevent.uobject);
-	int ret;
-
-	ret = ib_destroy_wq(wq, &attrs->driver_udata);
-	if (ib_is_destroy_retryable(ret, why, uobject))
-		return ret;
-
-	ib_uverbs_release_uevent(&uwq->uevent);
-	return ret;
-}
-
-static int uverbs_free_srq(struct ib_uobject *uobject,
-			   enum rdma_remove_reason why,
-			   struct uverbs_attr_bundle *attrs)
-{
-	struct ib_srq *srq = uobject->object;
-	struct ib_uevent_object *uevent =
-		container_of(uobject, struct ib_uevent_object, uobject);
-	enum ib_srq_type  srq_type = srq->srq_type;
-	int ret;
-
-	ret = ib_destroy_srq_user(srq, &attrs->driver_udata);
-	if (ib_is_destroy_retryable(ret, why, uobject))
-		return ret;
-
-	if (srq_type == IB_SRQT_XRC) {
-		struct ib_usrq_object *us =
-			container_of(uevent, struct ib_usrq_object, uevent);
-
-		atomic_dec(&us->uxrcd->refcnt);
-	}
-
-	ib_uverbs_release_uevent(uevent);
-	return ret;
-}
-
 static int uverbs_free_xrcd(struct ib_uobject *uobject,
 			    enum rdma_remove_reason why,
 			    struct uverbs_attr_bundle *attrs)
@@ -252,10 +176,6 @@ DECLARE_UVERBS_NAMED_OBJECT(
 			     "[infinibandevent]",
 			     O_RDONLY));
 
-DECLARE_UVERBS_NAMED_OBJECT(
-	UVERBS_OBJECT_QP,
-	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp));
-
 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
 	UVERBS_METHOD_MW_DESTROY,
 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_MW_HANDLE,
@@ -267,11 +187,6 @@ DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_MW,
 			    UVERBS_TYPE_ALLOC_IDR(uverbs_free_mw),
 			    &UVERBS_METHOD(UVERBS_METHOD_MW_DESTROY));
 
-DECLARE_UVERBS_NAMED_OBJECT(
-	UVERBS_OBJECT_SRQ,
-	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object),
-				 uverbs_free_srq));
-
 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
 	UVERBS_METHOD_AH_DESTROY,
 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_AH_HANDLE,
@@ -296,10 +211,6 @@ DECLARE_UVERBS_NAMED_OBJECT(
 				 uverbs_free_flow),
 			    &UVERBS_METHOD(UVERBS_METHOD_FLOW_DESTROY));
 
-DECLARE_UVERBS_NAMED_OBJECT(
-	UVERBS_OBJECT_WQ,
-	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), uverbs_free_wq));
-
 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
 	UVERBS_METHOD_RWQ_IND_TBL_DESTROY,
 	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_RWQ_IND_TBL_HANDLE,
@@ -340,18 +251,12 @@ const struct uapi_definition uverbs_def_obj_intf[] = {
 				      UAPI_DEF_OBJ_NEEDS_FN(dealloc_pd)),
 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_COMP_CHANNEL,
 				      UAPI_DEF_OBJ_NEEDS_FN(dealloc_pd)),
-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_QP,
-				      UAPI_DEF_OBJ_NEEDS_FN(destroy_qp)),
 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_AH,
 				      UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),
 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MW,
 				      UAPI_DEF_OBJ_NEEDS_FN(dealloc_mw)),
-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_SRQ,
-				      UAPI_DEF_OBJ_NEEDS_FN(destroy_srq)),
 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_FLOW,
 				      UAPI_DEF_OBJ_NEEDS_FN(destroy_flow)),
-	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_WQ,
-				      UAPI_DEF_OBJ_NEEDS_FN(destroy_wq)),
 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
 		UVERBS_OBJECT_RWQ_IND_TBL,
 		UAPI_DEF_OBJ_NEEDS_FN(destroy_rwq_ind_table)),
diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c
index da4110a0eea2..5dce2c7cc323 100644
--- a/drivers/infiniband/core/uverbs_std_types_cq.c
+++ b/drivers/infiniband/core/uverbs_std_types_cq.c
@@ -100,6 +100,9 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 		uverbs_uobject_get(ev_file_uobj);
 	}
 
+	obj->uevent.event_file = ib_uverbs_get_async_event(
+		attrs, UVERBS_ATTR_CREATE_CQ_EVENT_FD);
+
 	if (attr.comp_vector >= attrs->ufile->device->num_comp_vectors) {
 		ret = -EINVAL;
 		goto err_event_file;
@@ -129,19 +132,17 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 	obj->uevent.uobject.object = cq;
 	obj->uevent.uobject.user_handle = user_handle;
 	rdma_restrack_uadd(&cq->res);
+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE);
 
 	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &cq->cqe,
 			     sizeof(cq->cqe));
-	if (ret)
-		goto err_cq;
+	return ret;
 
-	return 0;
-err_cq:
-	ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
-	cq = NULL;
 err_free:
 	kfree(cq);
 err_event_file:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
 	if (ev_file)
 		uverbs_uobject_put(ev_file_uobj);
 	return ret;
@@ -171,6 +172,10 @@ DECLARE_UVERBS_NAMED_METHOD(
 	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_CQ_RESP_CQE,
 			    UVERBS_ATTR_TYPE(u32),
 			    UA_MANDATORY),
+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_CQ_EVENT_FD,
+		       UVERBS_OBJECT_ASYNC_EVENT,
+		       UVERBS_ACCESS_READ,
+		       UA_OPTIONAL),
 	UVERBS_ATTR_UHW());
 
 static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)(
diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c
index c1286a52dc84..a2722ef8496e 100644
--- a/drivers/infiniband/core/uverbs_std_types_mr.c
+++ b/drivers/infiniband/core/uverbs_std_types_mr.c
@@ -136,21 +136,15 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)(
 
 	uobj->object = mr;
 
+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_REG_DM_MR_HANDLE);
+
 	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_LKEY, &mr->lkey,
 			     sizeof(mr->lkey));
 	if (ret)
-		goto err_dereg;
+		return ret;
 
 	ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_RKEY,
 			     &mr->rkey, sizeof(mr->rkey));
-	if (ret)
-		goto err_dereg;
-
-	return 0;
-
-err_dereg:
-	ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs));
-
 	return ret;
 }
 
diff --git a/drivers/infiniband/core/uverbs_std_types_qp.c b/drivers/infiniband/core/uverbs_std_types_qp.c
new file mode 100644
index 000000000000..3bf8dcdfe7eb
--- /dev/null
+++ b/drivers/infiniband/core/uverbs_std_types_qp.c
@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved.
+ */
+
+#include <rdma/uverbs_std_types.h>
+#include "rdma_core.h"
+#include "uverbs.h"
+#include "core_priv.h"
+
+static int uverbs_free_qp(struct ib_uobject *uobject,
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
+{
+	struct ib_qp *qp = uobject->object;
+	struct ib_uqp_object *uqp =
+		container_of(uobject, struct ib_uqp_object, uevent.uobject);
+	int ret;
+
+	/*
+	 * If this is a user triggered destroy then do not allow destruction
+	 * until the user cleans up all the mcast bindings. Unlike in other
+	 * places we forcibly clean up the mcast attachments for !DESTROY
+	 * because the mcast attaches are not ubojects and will not be
+	 * destroyed by anything else during cleanup processing.
+	 */
+	if (why == RDMA_REMOVE_DESTROY) {
+		if (!list_empty(&uqp->mcast_list))
+			return -EBUSY;
+	} else if (qp == qp->real_qp) {
+		ib_uverbs_detach_umcast(qp, uqp);
+	}
+
+	ret = ib_destroy_qp_user(qp, &attrs->driver_udata);
+	if (ib_is_destroy_retryable(ret, why, uobject))
+		return ret;
+
+	if (uqp->uxrcd)
+		atomic_dec(&uqp->uxrcd->refcnt);
+
+	ib_uverbs_release_uevent(&uqp->uevent);
+	return ret;
+}
+
+static int check_creation_flags(enum ib_qp_type qp_type,
+				u32 create_flags)
+{
+	create_flags &= ~IB_UVERBS_QP_CREATE_SQ_SIG_ALL;
+
+	if (!create_flags || qp_type == IB_QPT_DRIVER)
+		return 0;
+
+	if (qp_type != IB_QPT_RAW_PACKET && qp_type != IB_QPT_UD)
+		return -EINVAL;
+
+	if ((create_flags & IB_UVERBS_QP_CREATE_SCATTER_FCS ||
+	     create_flags & IB_UVERBS_QP_CREATE_CVLAN_STRIPPING) &&
+	     qp_type != IB_QPT_RAW_PACKET)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void set_caps(struct ib_qp_init_attr *attr,
+		     struct ib_uverbs_qp_cap *cap, bool req)
+{
+	if (req) {
+		attr->cap.max_send_wr = cap->max_send_wr;
+		attr->cap.max_recv_wr = cap->max_recv_wr;
+		attr->cap.max_send_sge = cap->max_send_sge;
+		attr->cap.max_recv_sge = cap->max_recv_sge;
+		attr->cap.max_inline_data = cap->max_inline_data;
+	} else {
+		cap->max_send_wr = attr->cap.max_send_wr;
+		cap->max_recv_wr = attr->cap.max_recv_wr;
+		cap->max_send_sge = attr->cap.max_send_sge;
+		cap->max_recv_sge = attr->cap.max_recv_sge;
+		cap->max_inline_data = attr->cap.max_inline_data;
+	}
+}
+
+static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)(
+	struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uqp_object *obj = container_of(
+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_QP_HANDLE),
+		typeof(*obj), uevent.uobject);
+	struct ib_qp_init_attr attr = {};
+	struct ib_uverbs_qp_cap cap = {};
+	struct ib_rwq_ind_table *rwq_ind_tbl = NULL;
+	struct ib_qp *qp;
+	struct ib_pd *pd = NULL;
+	struct ib_srq *srq = NULL;
+	struct ib_cq *recv_cq = NULL;
+	struct ib_cq *send_cq = NULL;
+	struct ib_xrcd *xrcd = NULL;
+	struct ib_uobject *xrcd_uobj = NULL;
+	struct ib_device *device;
+	u64 user_handle;
+	int ret;
+
+	ret = uverbs_copy_from_or_zero(&cap, attrs,
+			       UVERBS_ATTR_CREATE_QP_CAP);
+	if (!ret)
+		ret = uverbs_copy_from(&user_handle, attrs,
+				       UVERBS_ATTR_CREATE_QP_USER_HANDLE);
+	if (!ret)
+		ret = uverbs_get_const(&attr.qp_type, attrs,
+				       UVERBS_ATTR_CREATE_QP_TYPE);
+	if (ret)
+		return ret;
+
+	switch (attr.qp_type) {
+	case IB_QPT_XRC_TGT:
+		if (uverbs_attr_is_valid(attrs,
+				UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE) ||
+		    uverbs_attr_is_valid(attrs,
+				UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE) ||
+		    uverbs_attr_is_valid(attrs,
+				UVERBS_ATTR_CREATE_QP_PD_HANDLE) ||
+		    uverbs_attr_is_valid(attrs,
+				UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE))
+			return -EINVAL;
+
+		xrcd_uobj = uverbs_attr_get_uobject(attrs,
+					UVERBS_ATTR_CREATE_QP_XRCD_HANDLE);
+		if (IS_ERR(xrcd_uobj))
+			return PTR_ERR(xrcd_uobj);
+
+		xrcd = (struct ib_xrcd *)xrcd_uobj->object;
+		if (!xrcd)
+			return -EINVAL;
+		device = xrcd->device;
+		break;
+	case IB_UVERBS_QPT_RAW_PACKET:
+		if (!capable(CAP_NET_RAW))
+			return -EPERM;
+		fallthrough;
+	case IB_UVERBS_QPT_RC:
+	case IB_UVERBS_QPT_UC:
+	case IB_UVERBS_QPT_UD:
+	case IB_UVERBS_QPT_XRC_INI:
+	case IB_UVERBS_QPT_DRIVER:
+		if (uverbs_attr_is_valid(attrs,
+					 UVERBS_ATTR_CREATE_QP_XRCD_HANDLE) ||
+		   (uverbs_attr_is_valid(attrs,
+					 UVERBS_ATTR_CREATE_QP_SRQ_HANDLE) &&
+			attr.qp_type == IB_QPT_XRC_INI))
+			return -EINVAL;
+
+		pd = uverbs_attr_get_obj(attrs,
+					 UVERBS_ATTR_CREATE_QP_PD_HANDLE);
+		if (IS_ERR(pd))
+			return PTR_ERR(pd);
+
+		rwq_ind_tbl = uverbs_attr_get_obj(attrs,
+			UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE);
+		if (!IS_ERR(rwq_ind_tbl)) {
+			if (cap.max_recv_wr || cap.max_recv_sge ||
+			    uverbs_attr_is_valid(attrs,
+				UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE) ||
+			    uverbs_attr_is_valid(attrs,
+					UVERBS_ATTR_CREATE_QP_SRQ_HANDLE))
+				return -EINVAL;
+
+			/* send_cq is optinal */
+			if (cap.max_send_wr) {
+				send_cq = uverbs_attr_get_obj(attrs,
+					UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE);
+				if (IS_ERR(send_cq))
+					return PTR_ERR(send_cq);
+			}
+			attr.rwq_ind_tbl = rwq_ind_tbl;
+		} else {
+			send_cq = uverbs_attr_get_obj(attrs,
+					UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE);
+			if (IS_ERR(send_cq))
+				return PTR_ERR(send_cq);
+
+			if (attr.qp_type != IB_QPT_XRC_INI) {
+				recv_cq = uverbs_attr_get_obj(attrs,
+					UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE);
+				if (IS_ERR(recv_cq))
+					return PTR_ERR(recv_cq);
+			}
+		}
+
+		device = pd->device;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = uverbs_get_flags32(&attr.create_flags, attrs,
+			 UVERBS_ATTR_CREATE_QP_FLAGS,
+			 IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
+			 IB_UVERBS_QP_CREATE_SCATTER_FCS |
+			 IB_UVERBS_QP_CREATE_CVLAN_STRIPPING |
+			 IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING |
+			 IB_UVERBS_QP_CREATE_SQ_SIG_ALL);
+	if (ret)
+		return ret;
+
+	ret = check_creation_flags(attr.qp_type, attr.create_flags);
+	if (ret)
+		return ret;
+
+	if (uverbs_attr_is_valid(attrs,
+			UVERBS_ATTR_CREATE_QP_SOURCE_QPN)) {
+		ret = uverbs_copy_from(&attr.source_qpn, attrs,
+				       UVERBS_ATTR_CREATE_QP_SOURCE_QPN);
+		if (ret)
+			return ret;
+		attr.create_flags |= IB_QP_CREATE_SOURCE_QPN;
+	}
+
+	srq = uverbs_attr_get_obj(attrs,
+				  UVERBS_ATTR_CREATE_QP_SRQ_HANDLE);
+	if (!IS_ERR(srq)) {
+		if ((srq->srq_type == IB_SRQT_XRC &&
+			attr.qp_type != IB_QPT_XRC_TGT) ||
+		    (srq->srq_type != IB_SRQT_XRC &&
+			attr.qp_type == IB_QPT_XRC_TGT))
+			return -EINVAL;
+		attr.srq = srq;
+	}
+
+	obj->uevent.event_file = ib_uverbs_get_async_event(attrs,
+					UVERBS_ATTR_CREATE_QP_EVENT_FD);
+	INIT_LIST_HEAD(&obj->uevent.event_list);
+	INIT_LIST_HEAD(&obj->mcast_list);
+	obj->uevent.uobject.user_handle = user_handle;
+	attr.event_handler = ib_uverbs_qp_event_handler;
+	attr.send_cq = send_cq;
+	attr.recv_cq = recv_cq;
+	attr.xrcd = xrcd;
+	if (attr.create_flags & IB_UVERBS_QP_CREATE_SQ_SIG_ALL) {
+		/* This creation bit is uverbs one, need to mask before
+		 * calling drivers. It was added to prevent an extra user attr
+		 * only for that when using ioctl.
+		 */
+		attr.create_flags &= ~IB_UVERBS_QP_CREATE_SQ_SIG_ALL;
+		attr.sq_sig_type = IB_SIGNAL_ALL_WR;
+	} else {
+		attr.sq_sig_type = IB_SIGNAL_REQ_WR;
+	}
+
+	set_caps(&attr, &cap, true);
+	mutex_init(&obj->mcast_lock);
+
+	if (attr.qp_type == IB_QPT_XRC_TGT)
+		qp = ib_create_qp(pd, &attr);
+	else
+		qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata,
+				   obj);
+
+	if (IS_ERR(qp)) {
+		ret = PTR_ERR(qp);
+		goto err_put;
+	}
+
+	if (attr.qp_type != IB_QPT_XRC_TGT) {
+		atomic_inc(&pd->usecnt);
+		if (attr.send_cq)
+			atomic_inc(&attr.send_cq->usecnt);
+		if (attr.recv_cq)
+			atomic_inc(&attr.recv_cq->usecnt);
+		if (attr.srq)
+			atomic_inc(&attr.srq->usecnt);
+		if (attr.rwq_ind_tbl)
+			atomic_inc(&attr.rwq_ind_tbl->usecnt);
+	} else {
+		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
+					  uobject);
+		atomic_inc(&obj->uxrcd->refcnt);
+		/* It is done in _ib_create_qp for other QP types */
+		qp->uobject = obj;
+	}
+
+	obj->uevent.uobject.object = qp;
+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_QP_HANDLE);
+
+	if (attr.qp_type != IB_QPT_XRC_TGT) {
+		ret = ib_create_qp_security(qp, device);
+		if (ret)
+			return ret;
+	}
+
+	set_caps(&attr, &cap, false);
+	ret = uverbs_copy_to_struct_or_zero(attrs,
+					UVERBS_ATTR_CREATE_QP_RESP_CAP, &cap,
+					sizeof(cap));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_QP_RESP_QP_NUM,
+			     &qp->qp_num,
+			     sizeof(qp->qp_num));
+
+	return ret;
+err_put:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
+	return ret;
+};
+
+DECLARE_UVERBS_NAMED_METHOD(
+	UVERBS_METHOD_QP_CREATE,
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_HANDLE,
+			UVERBS_OBJECT_QP,
+			UVERBS_ACCESS_NEW,
+			UA_MANDATORY),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_XRCD_HANDLE,
+			UVERBS_OBJECT_XRCD,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_PD_HANDLE,
+			UVERBS_OBJECT_PD,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_SRQ_HANDLE,
+			UVERBS_OBJECT_SRQ,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_SEND_CQ_HANDLE,
+			UVERBS_OBJECT_CQ,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_RECV_CQ_HANDLE,
+			UVERBS_OBJECT_CQ,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_QP_IND_TABLE_HANDLE,
+			UVERBS_OBJECT_RWQ_IND_TBL,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_QP_USER_HANDLE,
+			   UVERBS_ATTR_TYPE(u64),
+			   UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_QP_CAP,
+			   UVERBS_ATTR_STRUCT(struct ib_uverbs_qp_cap,
+					      max_inline_data),
+			   UA_MANDATORY),
+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_CREATE_QP_TYPE,
+			     enum ib_uverbs_qp_type,
+			     UA_MANDATORY),
+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_CREATE_QP_FLAGS,
+			     enum ib_uverbs_qp_create_flags,
+			     UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_QP_SOURCE_QPN,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_OPTIONAL),
+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_QP_EVENT_FD,
+		       UVERBS_OBJECT_ASYNC_EVENT,
+		       UVERBS_ACCESS_READ,
+		       UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_QP_RESP_CAP,
+			    UVERBS_ATTR_STRUCT(struct ib_uverbs_qp_cap,
+					       max_inline_data),
+			   UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_QP_RESP_QP_NUM,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_MANDATORY),
+	UVERBS_ATTR_UHW());
+
+static int UVERBS_HANDLER(UVERBS_METHOD_QP_DESTROY)(
+	struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *uobj =
+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_QP_HANDLE);
+	struct ib_uqp_object *obj =
+		container_of(uobj, struct ib_uqp_object, uevent.uobject);
+	struct ib_uverbs_destroy_qp_resp resp = {
+		.events_reported = obj->uevent.events_reported
+	};
+
+	return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_QP_RESP, &resp,
+			      sizeof(resp));
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+	UVERBS_METHOD_QP_DESTROY,
+	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_QP_HANDLE,
+			UVERBS_OBJECT_QP,
+			UVERBS_ACCESS_DESTROY,
+			UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_QP_RESP,
+			    UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_qp_resp),
+			    UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(
+	UVERBS_OBJECT_QP,
+	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp),
+	&UVERBS_METHOD(UVERBS_METHOD_QP_CREATE),
+	&UVERBS_METHOD(UVERBS_METHOD_QP_DESTROY));
+
+const struct uapi_definition uverbs_def_obj_qp[] = {
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_QP,
+				      UAPI_DEF_OBJ_NEEDS_FN(destroy_qp)),
+	{}
+};
diff --git a/drivers/infiniband/core/uverbs_std_types_srq.c b/drivers/infiniband/core/uverbs_std_types_srq.c
new file mode 100644
index 000000000000..c0ecbba26bf4
--- /dev/null
+++ b/drivers/infiniband/core/uverbs_std_types_srq.c
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved.
+ */
+
+#include <rdma/uverbs_std_types.h>
+#include "rdma_core.h"
+#include "uverbs.h"
+
+static int uverbs_free_srq(struct ib_uobject *uobject,
+		    enum rdma_remove_reason why,
+		    struct uverbs_attr_bundle *attrs)
+{
+	struct ib_srq *srq = uobject->object;
+	struct ib_uevent_object *uevent =
+		container_of(uobject, struct ib_uevent_object, uobject);
+	enum ib_srq_type srq_type = srq->srq_type;
+	int ret;
+
+	ret = ib_destroy_srq_user(srq, &attrs->driver_udata);
+	if (ib_is_destroy_retryable(ret, why, uobject))
+		return ret;
+
+	if (srq_type == IB_SRQT_XRC) {
+		struct ib_usrq_object *us =
+			container_of(uobject, struct ib_usrq_object,
+				     uevent.uobject);
+
+		atomic_dec(&us->uxrcd->refcnt);
+	}
+
+	ib_uverbs_release_uevent(uevent);
+	return ret;
+}
+
+static int UVERBS_HANDLER(UVERBS_METHOD_SRQ_CREATE)(
+	struct uverbs_attr_bundle *attrs)
+{
+	struct ib_usrq_object *obj = container_of(
+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_SRQ_HANDLE),
+		typeof(*obj), uevent.uobject);
+	struct ib_pd *pd =
+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_CREATE_SRQ_PD_HANDLE);
+	struct ib_srq_init_attr attr = {};
+	struct ib_uobject *xrcd_uobj;
+	struct ib_srq *srq;
+	u64 user_handle;
+	int ret;
+
+	ret = uverbs_copy_from(&attr.attr.max_sge, attrs,
+			       UVERBS_ATTR_CREATE_SRQ_MAX_SGE);
+	if (!ret)
+		ret = uverbs_copy_from(&attr.attr.max_wr, attrs,
+				       UVERBS_ATTR_CREATE_SRQ_MAX_WR);
+	if (!ret)
+		ret = uverbs_copy_from(&attr.attr.srq_limit, attrs,
+				       UVERBS_ATTR_CREATE_SRQ_LIMIT);
+	if (!ret)
+		ret = uverbs_copy_from(&user_handle, attrs,
+				       UVERBS_ATTR_CREATE_SRQ_USER_HANDLE);
+	if (!ret)
+		ret = uverbs_get_const(&attr.srq_type, attrs,
+				       UVERBS_ATTR_CREATE_SRQ_TYPE);
+	if (ret)
+		return ret;
+
+	if (ib_srq_has_cq(attr.srq_type)) {
+		attr.ext.cq = uverbs_attr_get_obj(attrs,
+					UVERBS_ATTR_CREATE_SRQ_CQ_HANDLE);
+		if (IS_ERR(attr.ext.cq))
+			return PTR_ERR(attr.ext.cq);
+	}
+
+	switch (attr.srq_type) {
+	case IB_UVERBS_SRQT_XRC:
+		xrcd_uobj = uverbs_attr_get_uobject(attrs,
+					UVERBS_ATTR_CREATE_SRQ_XRCD_HANDLE);
+		if (IS_ERR(xrcd_uobj))
+			return PTR_ERR(xrcd_uobj);
+
+		attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
+		if (!attr.ext.xrc.xrcd)
+			return -EINVAL;
+		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
+					  uobject);
+		atomic_inc(&obj->uxrcd->refcnt);
+		break;
+	case IB_UVERBS_SRQT_TM:
+		ret = uverbs_copy_from(&attr.ext.tag_matching.max_num_tags,
+				       attrs,
+				       UVERBS_ATTR_CREATE_SRQ_MAX_NUM_TAGS);
+		if (ret)
+			return ret;
+		break;
+	case IB_UVERBS_SRQT_BASIC:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	obj->uevent.event_file = ib_uverbs_get_async_event(attrs,
+					UVERBS_ATTR_CREATE_SRQ_EVENT_FD);
+	INIT_LIST_HEAD(&obj->uevent.event_list);
+	attr.event_handler = ib_uverbs_srq_event_handler;
+	obj->uevent.uobject.user_handle = user_handle;
+
+	srq = ib_create_srq_user(pd, &attr, obj, &attrs->driver_udata);
+	if (IS_ERR(srq)) {
+		ret = PTR_ERR(srq);
+		goto err;
+	}
+
+	obj->uevent.uobject.object = srq;
+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_SRQ_HANDLE);
+
+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_SRQ_RESP_MAX_WR,
+			     &attr.attr.max_wr,
+			     sizeof(attr.attr.max_wr));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_SRQ_RESP_MAX_SGE,
+			     &attr.attr.max_sge,
+			     sizeof(attr.attr.max_sge));
+	if (ret)
+		return ret;
+
+	if (attr.srq_type == IB_SRQT_XRC) {
+		ret = uverbs_copy_to(attrs,
+				     UVERBS_ATTR_CREATE_SRQ_RESP_SRQ_NUM,
+				     &srq->ext.xrc.srq_num,
+				     sizeof(srq->ext.xrc.srq_num));
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+err:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
+	if (attr.srq_type == IB_SRQT_XRC)
+		atomic_dec(&obj->uxrcd->refcnt);
+	return ret;
+};
+
+DECLARE_UVERBS_NAMED_METHOD(
+	UVERBS_METHOD_SRQ_CREATE,
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_HANDLE,
+			UVERBS_OBJECT_SRQ,
+			UVERBS_ACCESS_NEW,
+			UA_MANDATORY),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_PD_HANDLE,
+			UVERBS_OBJECT_PD,
+			UVERBS_ACCESS_READ,
+			UA_MANDATORY),
+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_CREATE_SRQ_TYPE,
+			     enum ib_uverbs_srq_type,
+			     UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_USER_HANDLE,
+			   UVERBS_ATTR_TYPE(u64),
+			   UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_MAX_WR,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_MAX_SGE,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_LIMIT,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_MANDATORY),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_XRCD_HANDLE,
+			UVERBS_OBJECT_XRCD,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_SRQ_CQ_HANDLE,
+			UVERBS_OBJECT_CQ,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_SRQ_MAX_NUM_TAGS,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_OPTIONAL),
+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_SRQ_EVENT_FD,
+		       UVERBS_OBJECT_ASYNC_EVENT,
+		       UVERBS_ACCESS_READ,
+		       UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_SRQ_RESP_MAX_WR,
+			    UVERBS_ATTR_TYPE(u32),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_SRQ_RESP_MAX_SGE,
+			    UVERBS_ATTR_TYPE(u32),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_SRQ_RESP_SRQ_NUM,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_OPTIONAL),
+	UVERBS_ATTR_UHW());
+
+static int UVERBS_HANDLER(UVERBS_METHOD_SRQ_DESTROY)(
+	struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *uobj =
+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_SRQ_HANDLE);
+	struct ib_usrq_object *obj =
+		container_of(uobj, struct ib_usrq_object, uevent.uobject);
+	struct ib_uverbs_destroy_srq_resp resp = {
+		.events_reported = obj->uevent.events_reported
+	};
+
+	return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_SRQ_RESP, &resp,
+			      sizeof(resp));
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+	UVERBS_METHOD_SRQ_DESTROY,
+	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_SRQ_HANDLE,
+			UVERBS_OBJECT_SRQ,
+			UVERBS_ACCESS_DESTROY,
+			UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_SRQ_RESP,
+			    UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_srq_resp),
+			    UA_MANDATORY));
+
+DECLARE_UVERBS_NAMED_OBJECT(
+	UVERBS_OBJECT_SRQ,
+	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object),
+				 uverbs_free_srq),
+	&UVERBS_METHOD(UVERBS_METHOD_SRQ_CREATE),
+	&UVERBS_METHOD(UVERBS_METHOD_SRQ_DESTROY)
+);
+
+const struct uapi_definition uverbs_def_obj_srq[] = {
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_SRQ,
+				      UAPI_DEF_OBJ_NEEDS_FN(destroy_srq)),
+	{}
+};
diff --git a/drivers/infiniband/core/uverbs_std_types_wq.c b/drivers/infiniband/core/uverbs_std_types_wq.c
new file mode 100644
index 000000000000..cad842ede077
--- /dev/null
+++ b/drivers/infiniband/core/uverbs_std_types_wq.c
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved.
+ */
+
+#include <rdma/uverbs_std_types.h>
+#include "rdma_core.h"
+#include "uverbs.h"
+
+static int uverbs_free_wq(struct ib_uobject *uobject,
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
+{
+	struct ib_wq *wq = uobject->object;
+	struct ib_uwq_object *uwq =
+		container_of(uobject, struct ib_uwq_object, uevent.uobject);
+	int ret;
+
+	ret = ib_destroy_wq(wq, &attrs->driver_udata);
+	if (ib_is_destroy_retryable(ret, why, uobject))
+		return ret;
+
+	ib_uverbs_release_uevent(&uwq->uevent);
+	return ret;
+}
+
+static int UVERBS_HANDLER(UVERBS_METHOD_WQ_CREATE)(
+	struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uwq_object *obj = container_of(
+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_WQ_HANDLE),
+		typeof(*obj), uevent.uobject);
+	struct ib_pd *pd =
+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_CREATE_WQ_PD_HANDLE);
+	struct ib_cq *cq =
+		uverbs_attr_get_obj(attrs, UVERBS_ATTR_CREATE_WQ_CQ_HANDLE);
+	struct ib_wq_init_attr wq_init_attr = {};
+	struct ib_wq *wq;
+	u64 user_handle;
+	int ret;
+
+	ret = uverbs_get_flags32(&wq_init_attr.create_flags, attrs,
+				 UVERBS_ATTR_CREATE_WQ_FLAGS,
+				 IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING |
+				 IB_UVERBS_WQ_FLAGS_SCATTER_FCS |
+				 IB_UVERBS_WQ_FLAGS_DELAY_DROP |
+				 IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING);
+	if (!ret)
+		ret = uverbs_copy_from(&wq_init_attr.max_sge, attrs,
+			       UVERBS_ATTR_CREATE_WQ_MAX_SGE);
+	if (!ret)
+		ret = uverbs_copy_from(&wq_init_attr.max_wr, attrs,
+				       UVERBS_ATTR_CREATE_WQ_MAX_WR);
+	if (!ret)
+		ret = uverbs_copy_from(&user_handle, attrs,
+				       UVERBS_ATTR_CREATE_WQ_USER_HANDLE);
+	if (!ret)
+		ret = uverbs_get_const(&wq_init_attr.wq_type, attrs,
+				       UVERBS_ATTR_CREATE_WQ_TYPE);
+	if (ret)
+		return ret;
+
+	if (wq_init_attr.wq_type != IB_WQT_RQ)
+		return -EINVAL;
+
+	obj->uevent.event_file = ib_uverbs_get_async_event(attrs,
+					UVERBS_ATTR_CREATE_WQ_EVENT_FD);
+	obj->uevent.uobject.user_handle = user_handle;
+	INIT_LIST_HEAD(&obj->uevent.event_list);
+	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
+	wq_init_attr.wq_context = attrs->ufile;
+	wq_init_attr.cq = cq;
+
+	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
+	if (IS_ERR(wq)) {
+		ret = PTR_ERR(wq);
+		goto err;
+	}
+
+	obj->uevent.uobject.object = wq;
+	wq->wq_type = wq_init_attr.wq_type;
+	wq->cq = cq;
+	wq->pd = pd;
+	wq->device = pd->device;
+	wq->wq_context = wq_init_attr.wq_context;
+	atomic_set(&wq->usecnt, 0);
+	atomic_inc(&pd->usecnt);
+	atomic_inc(&cq->usecnt);
+	wq->uobject = obj;
+	uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_CREATE_WQ_HANDLE);
+
+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_WQ_RESP_MAX_WR,
+			     &wq_init_attr.max_wr,
+			     sizeof(wq_init_attr.max_wr));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_WQ_RESP_MAX_SGE,
+			     &wq_init_attr.max_sge,
+			     sizeof(wq_init_attr.max_sge));
+	if (ret)
+		return ret;
+
+	ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_WQ_RESP_WQ_NUM,
+			     &wq->wq_num,
+			     sizeof(wq->wq_num));
+	return ret;
+
+err:
+	if (obj->uevent.event_file)
+		uverbs_uobject_put(&obj->uevent.event_file->uobj);
+	return ret;
+};
+
+DECLARE_UVERBS_NAMED_METHOD(
+	UVERBS_METHOD_WQ_CREATE,
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_HANDLE,
+			UVERBS_OBJECT_WQ,
+			UVERBS_ACCESS_NEW,
+			UA_MANDATORY),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_PD_HANDLE,
+			UVERBS_OBJECT_PD,
+			UVERBS_ACCESS_READ,
+			UA_MANDATORY),
+	UVERBS_ATTR_CONST_IN(UVERBS_ATTR_CREATE_WQ_TYPE,
+			     enum ib_wq_type,
+			     UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_WQ_USER_HANDLE,
+			   UVERBS_ATTR_TYPE(u64),
+			   UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_WQ_MAX_WR,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_MANDATORY),
+	UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_WQ_MAX_SGE,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_MANDATORY),
+	UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_CREATE_WQ_FLAGS,
+			     enum ib_uverbs_wq_flags,
+			     UA_MANDATORY),
+	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_CQ_HANDLE,
+			UVERBS_OBJECT_CQ,
+			UVERBS_ACCESS_READ,
+			UA_OPTIONAL),
+	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_WQ_EVENT_FD,
+		       UVERBS_OBJECT_ASYNC_EVENT,
+		       UVERBS_ACCESS_READ,
+		       UA_OPTIONAL),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_WQ_RESP_MAX_WR,
+			    UVERBS_ATTR_TYPE(u32),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_WQ_RESP_MAX_SGE,
+			    UVERBS_ATTR_TYPE(u32),
+			    UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_WQ_RESP_WQ_NUM,
+			   UVERBS_ATTR_TYPE(u32),
+			   UA_OPTIONAL),
+	UVERBS_ATTR_UHW());
+
+static int UVERBS_HANDLER(UVERBS_METHOD_WQ_DESTROY)(
+	struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uobject *uobj =
+		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_WQ_HANDLE);
+	struct ib_uwq_object *obj =
+		container_of(uobj, struct ib_uwq_object, uevent.uobject);
+
+	return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_WQ_RESP,
+			      &obj->uevent.events_reported,
+			      sizeof(obj->uevent.events_reported));
+}
+
+DECLARE_UVERBS_NAMED_METHOD(
+	UVERBS_METHOD_WQ_DESTROY,
+	UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_WQ_HANDLE,
+			UVERBS_OBJECT_WQ,
+			UVERBS_ACCESS_DESTROY,
+			UA_MANDATORY),
+	UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_WQ_RESP,
+			    UVERBS_ATTR_TYPE(u32),
+			    UA_MANDATORY));
+
+
+DECLARE_UVERBS_NAMED_OBJECT(
+	UVERBS_OBJECT_WQ,
+	UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), uverbs_free_wq),
+	&UVERBS_METHOD(UVERBS_METHOD_WQ_CREATE),
+	&UVERBS_METHOD(UVERBS_METHOD_WQ_DESTROY)
+);
+
+const struct uapi_definition uverbs_def_obj_wq[] = {
+	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_WQ,
+				      UAPI_DEF_OBJ_NEEDS_FN(destroy_wq)),
+	{}
+};
diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c
index 3f121ac31e0a..5addc8fae3f3 100644
--- a/drivers/infiniband/core/uverbs_uapi.c
+++ b/drivers/infiniband/core/uverbs_uapi.c
@@ -634,6 +634,9 @@ static const struct uapi_definition uverbs_core_api[] = {
 	UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
 	UAPI_DEF_CHAIN(uverbs_def_obj_intf),
 	UAPI_DEF_CHAIN(uverbs_def_obj_mr),
+	UAPI_DEF_CHAIN(uverbs_def_obj_qp),
+	UAPI_DEF_CHAIN(uverbs_def_obj_srq),
+	UAPI_DEF_CHAIN(uverbs_def_obj_wq),
 	UAPI_DEF_CHAIN(uverbs_def_write_intf),
 	{},
 };
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 56a71337112c..53d6505c0c7b 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -50,6 +50,7 @@
 #include <rdma/ib_cache.h>
 #include <rdma/ib_addr.h>
 #include <rdma/rw.h>
+#include <rdma/lag.h>
 
 #include "core_priv.h"
 #include <trace/events/rdma_core.h>
@@ -500,8 +501,10 @@ rdma_update_sgid_attr(struct rdma_ah_attr *ah_attr,
 static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
 				     struct rdma_ah_attr *ah_attr,
 				     u32 flags,
-				     struct ib_udata *udata)
+				     struct ib_udata *udata,
+				     struct net_device *xmit_slave)
 {
+	struct rdma_ah_init_attr init_attr = {};
 	struct ib_device *device = pd->device;
 	struct ib_ah *ah;
 	int ret;
@@ -521,8 +524,11 @@ static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
 	ah->pd = pd;
 	ah->type = ah_attr->type;
 	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
+	init_attr.ah_attr = ah_attr;
+	init_attr.flags = flags;
+	init_attr.xmit_slave = xmit_slave;
 
-	ret = device->ops.create_ah(ah, ah_attr, flags, udata);
+	ret = device->ops.create_ah(ah, &init_attr, udata);
 	if (ret) {
 		kfree(ah);
 		return ERR_PTR(ret);
@@ -547,15 +553,22 @@ struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
 			     u32 flags)
 {
 	const struct ib_gid_attr *old_sgid_attr;
+	struct net_device *slave;
 	struct ib_ah *ah;
 	int ret;
 
 	ret = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
 	if (ret)
 		return ERR_PTR(ret);
-
-	ah = _rdma_create_ah(pd, ah_attr, flags, NULL);
-
+	slave = rdma_lag_get_ah_roce_slave(pd->device, ah_attr,
+					   (flags & RDMA_CREATE_AH_SLEEPABLE) ?
+					   GFP_KERNEL : GFP_ATOMIC);
+	if (IS_ERR(slave)) {
+		rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
+		return (void *)slave;
+	}
+	ah = _rdma_create_ah(pd, ah_attr, flags, NULL, slave);
+	rdma_lag_put_ah_roce_slave(slave);
 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
 	return ah;
 }
@@ -594,7 +607,8 @@ struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
 		}
 	}
 
-	ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE, udata);
+	ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE,
+			     udata, NULL);
 
 out:
 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
@@ -967,15 +981,29 @@ EXPORT_SYMBOL(rdma_destroy_ah_user);
 
 /* Shared receive queues */
 
-struct ib_srq *ib_create_srq(struct ib_pd *pd,
-			     struct ib_srq_init_attr *srq_init_attr)
+/**
+ * ib_create_srq_user - Creates a SRQ associated with the specified protection
+ *   domain.
+ * @pd: The protection domain associated with the SRQ.
+ * @srq_init_attr: A list of initial attributes required to create the
+ *   SRQ.  If SRQ creation succeeds, then the attributes are updated to
+ *   the actual capabilities of the created SRQ.
+ * @uobject - uobject pointer if this is not a kernel SRQ
+ * @udata - udata pointer if this is not a kernel SRQ
+ *
+ * srq_attr->max_wr and srq_attr->max_sge are read the determine the
+ * requested size of the SRQ, and set to the actual values allocated
+ * on return.  If ib_create_srq() succeeds, then max_wr and max_sge
+ * will always be at least as large as the requested values.
+ */
+struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
+				  struct ib_srq_init_attr *srq_init_attr,
+				  struct ib_usrq_object *uobject,
+				  struct ib_udata *udata)
 {
 	struct ib_srq *srq;
 	int ret;
 
-	if (!pd->device->ops.create_srq)
-		return ERR_PTR(-EOPNOTSUPP);
-
 	srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
 	if (!srq)
 		return ERR_PTR(-ENOMEM);
@@ -985,6 +1013,7 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,
 	srq->event_handler = srq_init_attr->event_handler;
 	srq->srq_context = srq_init_attr->srq_context;
 	srq->srq_type = srq_init_attr->srq_type;
+	srq->uobject = uobject;
 
 	if (ib_srq_has_cq(srq->srq_type)) {
 		srq->ext.cq = srq_init_attr->ext.cq;
@@ -996,7 +1025,7 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,
 	}
 	atomic_inc(&pd->usecnt);
 
-	ret = pd->device->ops.create_srq(srq, srq_init_attr, NULL);
+	ret = pd->device->ops.create_srq(srq, srq_init_attr, udata);
 	if (ret) {
 		atomic_dec(&srq->pd->usecnt);
 		if (srq->srq_type == IB_SRQT_XRC)
@@ -1009,7 +1038,7 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,
 
 	return srq;
 }
-EXPORT_SYMBOL(ib_create_srq);
+EXPORT_SYMBOL(ib_create_srq_user);
 
 int ib_modify_srq(struct ib_srq *srq,
 		  struct ib_srq_attr *srq_attr,
@@ -1633,11 +1662,35 @@ static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
 	const struct ib_gid_attr *old_sgid_attr_alt_av;
 	int ret;
 
+	attr->xmit_slave = NULL;
 	if (attr_mask & IB_QP_AV) {
 		ret = rdma_fill_sgid_attr(qp->device, &attr->ah_attr,
 					  &old_sgid_attr_av);
 		if (ret)
 			return ret;
+
+		if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
+		    is_qp_type_connected(qp)) {
+			struct net_device *slave;
+
+			/*
+			 * If the user provided the qp_attr then we have to
+			 * resolve it. Kerne users have to provide already
+			 * resolved rdma_ah_attr's.
+			 */
+			if (udata) {
+				ret = ib_resolve_eth_dmac(qp->device,
+							  &attr->ah_attr);
+				if (ret)
+					goto out_av;
+			}
+			slave = rdma_lag_get_ah_roce_slave(qp->device,
+							   &attr->ah_attr,
+							   GFP_KERNEL);
+			if (IS_ERR(slave))
+				goto out_av;
+			attr->xmit_slave = slave;
+		}
 	}
 	if (attr_mask & IB_QP_ALT_PATH) {
 		/*
@@ -1664,18 +1717,6 @@ static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
 		}
 	}
 
-	/*
-	 * If the user provided the qp_attr then we have to resolve it. Kernel
-	 * users have to provide already resolved rdma_ah_attr's
-	 */
-	if (udata && (attr_mask & IB_QP_AV) &&
-	    attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
-	    is_qp_type_connected(qp)) {
-		ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr);
-		if (ret)
-			goto out;
-	}
-
 	if (rdma_ib_or_roce(qp->device, port)) {
 		if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
 			dev_warn(&qp->device->dev,
@@ -1717,8 +1758,10 @@ out:
 	if (attr_mask & IB_QP_ALT_PATH)
 		rdma_unfill_sgid_attr(&attr->alt_ah_attr, old_sgid_attr_alt_av);
 out_av:
-	if (attr_mask & IB_QP_AV)
+	if (attr_mask & IB_QP_AV) {
+		rdma_lag_put_ah_roce_slave(attr->xmit_slave);
 		rdma_unfill_sgid_attr(&attr->ah_attr, old_sgid_attr_av);
+	}
 	return ret;
 }
 
@@ -1962,6 +2005,9 @@ EXPORT_SYMBOL(__ib_create_cq);
 
 int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 {
+	if (cq->shared)
+		return -EOPNOTSUPP;
+
 	return cq->device->ops.modify_cq ?
 		cq->device->ops.modify_cq(cq, cq_count,
 					  cq_period) : -EOPNOTSUPP;
@@ -1970,6 +2016,9 @@ EXPORT_SYMBOL(rdma_set_cq_moderation);
 
 int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
 {
+	if (WARN_ON_ONCE(cq->shared))
+		return -EOPNOTSUPP;
+
 	if (atomic_read(&cq->usecnt))
 		return -EBUSY;
 
@@ -1982,6 +2031,9 @@ EXPORT_SYMBOL(ib_destroy_cq_user);
 
 int ib_resize_cq(struct ib_cq *cq, int cqe)
 {
+	if (cq->shared)
+		return -EOPNOTSUPP;
+
 	return cq->device->ops.resize_cq ?
 		cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
 }
@@ -2160,54 +2212,6 @@ out:
 }
 EXPORT_SYMBOL(ib_alloc_mr_integrity);
 
-/* "Fast" memory regions */
-
-struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
-			    int mr_access_flags,
-			    struct ib_fmr_attr *fmr_attr)
-{
-	struct ib_fmr *fmr;
-
-	if (!pd->device->ops.alloc_fmr)
-		return ERR_PTR(-EOPNOTSUPP);
-
-	fmr = pd->device->ops.alloc_fmr(pd, mr_access_flags, fmr_attr);
-	if (!IS_ERR(fmr)) {
-		fmr->device = pd->device;
-		fmr->pd     = pd;
-		atomic_inc(&pd->usecnt);
-	}
-
-	return fmr;
-}
-EXPORT_SYMBOL(ib_alloc_fmr);
-
-int ib_unmap_fmr(struct list_head *fmr_list)
-{
-	struct ib_fmr *fmr;
-
-	if (list_empty(fmr_list))
-		return 0;
-
-	fmr = list_entry(fmr_list->next, struct ib_fmr, list);
-	return fmr->device->ops.unmap_fmr(fmr_list);
-}
-EXPORT_SYMBOL(ib_unmap_fmr);
-
-int ib_dealloc_fmr(struct ib_fmr *fmr)
-{
-	struct ib_pd *pd;
-	int ret;
-
-	pd = fmr->pd;
-	ret = fmr->device->ops.dealloc_fmr(fmr);
-	if (!ret)
-		atomic_dec(&pd->usecnt);
-
-	return ret;
-}
-EXPORT_SYMBOL(ib_dealloc_fmr);
-
 /* Multicast groups */
 
 static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
@@ -2574,6 +2578,7 @@ EXPORT_SYMBOL(ib_map_mr_sg_pi);
  * @page_size:     page vector desired page size
  *
  * Constraints:
+ *
  * - The first sg element is allowed to have an offset.
  * - Each sg element must either be aligned to page_size or virtually
  *   contiguous to the previous element. In case an sg element has a
@@ -2607,10 +2612,12 @@ EXPORT_SYMBOL(ib_map_mr_sg);
  * @mr:            memory region
  * @sgl:           dma mapped scatterlist
  * @sg_nents:      number of entries in sg
- * @sg_offset_p:   IN:  start offset in bytes into sg
- *                 OUT: offset in bytes for element n of the sg of the first
+ * @sg_offset_p:   ==== =======================================================
+ *                 IN   start offset in bytes into sg
+ *                 OUT  offset in bytes for element n of the sg of the first
  *                      byte that has not been processed where n is the return
  *                      value of this function.
+ *                 ==== =======================================================
  * @set_page:      driver page assignment function pointer
  *
  * Core service helper for drivers to convert the largest
diff --git a/drivers/infiniband/hw/bnxt_re/Kconfig b/drivers/infiniband/hw/bnxt_re/Kconfig
index b83f1cc38c52..0feac5132ce1 100644
--- a/drivers/infiniband/hw/bnxt_re/Kconfig
+++ b/drivers/infiniband/hw/bnxt_re/Kconfig
@@ -5,7 +5,7 @@ config INFINIBAND_BNXT_RE
 	depends on ETHERNET && NETDEVICES && PCI && INET && DCB
 	select NET_VENDOR_BROADCOM
 	select BNXT
-	---help---
+	help
 	  This driver supports Broadcom NetXtreme-E 10/25/40/50 gigabit
 	  RoCE HCAs.  To compile this driver as a module, choose M here:
 	  the module will be called bnxt_re.
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 95f6d493d1b9..8b6ad5cddfce 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -177,9 +177,6 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 	ib_attr->max_total_mcast_qp_attach = 0;
 	ib_attr->max_ah = dev_attr->max_ah;
 
-	ib_attr->max_fmr = 0;
-	ib_attr->max_map_per_fmr = 0;
-
 	ib_attr->max_srq = dev_attr->max_srq;
 	ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
 	ib_attr->max_srq_sge = dev_attr->max_srq_sges;
@@ -631,11 +628,12 @@ static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
 	return nw_type;
 }
 
-int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,
-		      u32 flags, struct ib_udata *udata)
+int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
+		      struct ib_udata *udata)
 {
 	struct ib_pd *ib_pd = ib_ah->pd;
 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 	struct bnxt_re_dev *rdev = pd->rdev;
 	const struct ib_gid_attr *sgid_attr;
@@ -673,7 +671,8 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,
 
 	memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
 	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
-				  !(flags & RDMA_CREATE_AH_SLEEPABLE));
+				  !(init_attr->flags &
+				    RDMA_CREATE_AH_SLEEPABLE));
 	if (rc) {
 		ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
 		return rc;
@@ -856,7 +855,7 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 		return -EFAULT;
 
-	bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
+	bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
 	/* Consider mapping PSN search memory only for RC QPs. */
 	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
 		psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
@@ -879,7 +878,7 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 	qplib_qp->qp_handle = ureq.qp_handle;
 
 	if (!qp->qplib_qp.srq) {
-		bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
+		bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
 		bytes = PAGE_ALIGN(bytes);
 		umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes,
 				   IB_ACCESS_LOCAL_WRITE);
@@ -976,6 +975,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 	qp->qplib_qp.sig_type = true;
 
 	/* Shadow QP SQ depth should be same as QP1 RQ depth */
+	qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size();
 	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
 	qp->qplib_qp.sq.max_sge = 2;
 	/* Q full delta can be 1 since it is internal QP */
@@ -986,6 +986,7 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 	qp->qplib_qp.scq = qp1_qp->scq;
 	qp->qplib_qp.rcq = qp1_qp->rcq;
 
+	qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size();
 	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
 	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
 	/* Q full delta can be 1 since it is internal QP */
@@ -1021,10 +1022,12 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
 	struct bnxt_qplib_dev_attr *dev_attr;
 	struct bnxt_qplib_qp *qplqp;
 	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_q *rq;
 	int entries;
 
 	rdev = qp->rdev;
 	qplqp = &qp->qplib_qp;
+	rq = &qplqp->rq;
 	dev_attr = &rdev->dev_attr;
 
 	if (init_attr->srq) {
@@ -1036,23 +1039,21 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
 			return -EINVAL;
 		}
 		qplqp->srq = &srq->qplib_srq;
-		qplqp->rq.max_wqe = 0;
+		rq->max_wqe = 0;
 	} else {
+		rq->wqe_size = bnxt_re_get_rwqe_size();
 		/* Allocate 1 more than what's provided so posting max doesn't
 		 * mean empty.
 		 */
 		entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
-		qplqp->rq.max_wqe = min_t(u32, entries,
-					  dev_attr->max_qp_wqes + 1);
-
-		qplqp->rq.q_full_delta = qplqp->rq.max_wqe -
-					 init_attr->cap.max_recv_wr;
-		qplqp->rq.max_sge = init_attr->cap.max_recv_sge;
-		if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
-			qplqp->rq.max_sge = dev_attr->max_qp_sges;
+		rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
+		rq->q_full_delta = rq->max_wqe - init_attr->cap.max_recv_wr;
+		rq->max_sge = init_attr->cap.max_recv_sge;
+		if (rq->max_sge > dev_attr->max_qp_sges)
+			rq->max_sge = dev_attr->max_qp_sges;
 	}
-	qplqp->rq.sg_info.pgsize = PAGE_SIZE;
-	qplqp->rq.sg_info.pgshft = PAGE_SHIFT;
+	rq->sg_info.pgsize = PAGE_SIZE;
+	rq->sg_info.pgshft = PAGE_SHIFT;
 
 	return 0;
 }
@@ -1080,15 +1081,18 @@ static void bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
 	struct bnxt_qplib_dev_attr *dev_attr;
 	struct bnxt_qplib_qp *qplqp;
 	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_q *sq;
 	int entries;
 
 	rdev = qp->rdev;
 	qplqp = &qp->qplib_qp;
+	sq = &qplqp->sq;
 	dev_attr = &rdev->dev_attr;
 
-	qplqp->sq.max_sge = init_attr->cap.max_send_sge;
-	if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
-		qplqp->sq.max_sge = dev_attr->max_qp_sges;
+	sq->wqe_size = bnxt_re_get_swqe_size();
+	sq->max_sge = init_attr->cap.max_send_sge;
+	if (sq->max_sge > dev_attr->max_qp_sges)
+		sq->max_sge = dev_attr->max_qp_sges;
 	/*
 	 * Change the SQ depth if user has requested minimum using
 	 * configfs. Only supported for kernel consumers
@@ -1096,9 +1100,9 @@ static void bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
 	entries = init_attr->cap.max_send_wr;
 	/* Allocate 128 + 1 more than what's provided */
 	entries = roundup_pow_of_two(entries + BNXT_QPLIB_RESERVED_QP_WRS + 1);
-	qplqp->sq.max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes +
-			BNXT_QPLIB_RESERVED_QP_WRS + 1);
-	qplqp->sq.q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1;
+	sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes +
+			    BNXT_QPLIB_RESERVED_QP_WRS + 1);
+	sq->q_full_delta = BNXT_QPLIB_RESERVED_QP_WRS + 1;
 	/*
 	 * Reserving one slot for Phantom WQE. Application can
 	 * post one extra entry in this case. But allowing this to avoid
@@ -1511,7 +1515,7 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
 	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 		return -EFAULT;
 
-	bytes = (qplib_srq->max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
+	bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
 	bytes = PAGE_ALIGN(bytes);
 	umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
 			   IB_ACCESS_LOCAL_WRITE);
@@ -1534,15 +1538,20 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
 		       struct ib_srq_init_attr *srq_init_attr,
 		       struct ib_udata *udata)
 {
-	struct ib_pd *ib_pd = ib_srq->pd;
-	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
-	struct bnxt_re_dev *rdev = pd->rdev;
-	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
-	struct bnxt_re_srq *srq =
-		container_of(ib_srq, struct bnxt_re_srq, ib_srq);
+	struct bnxt_qplib_dev_attr *dev_attr;
 	struct bnxt_qplib_nq *nq = NULL;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_srq *srq;
+	struct bnxt_re_pd *pd;
+	struct ib_pd *ib_pd;
 	int rc, entries;
 
+	ib_pd = ib_srq->pd;
+	pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
+	rdev = pd->rdev;
+	dev_attr = &rdev->dev_attr;
+	srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
+
 	if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
 		ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
 		rc = -EINVAL;
@@ -1563,8 +1572,9 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
 	entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
 	if (entries > dev_attr->max_srq_wqes + 1)
 		entries = dev_attr->max_srq_wqes + 1;
-
 	srq->qplib_srq.max_wqe = entries;
+
+	srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size();
 	srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
 	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
 	srq->srq_limit = srq_init_attr->attr.srq_limit;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 23d972da5652..e5fbbeba6d28 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -122,12 +122,6 @@ struct bnxt_re_frpl {
 	u64				*page_list;
 };
 
-struct bnxt_re_fmr {
-	struct bnxt_re_dev	*rdev;
-	struct ib_fmr		ib_fmr;
-	struct bnxt_qplib_mrw	qplib_fmr;
-};
-
 struct bnxt_re_mw {
 	struct bnxt_re_dev	*rdev;
 	struct ib_mw		ib_mw;
@@ -142,6 +136,16 @@ struct bnxt_re_ucontext {
 	spinlock_t		sh_lock;	/* protect shpg */
 };
 
+static inline u16 bnxt_re_get_swqe_size(void)
+{
+	return sizeof(struct sq_send);
+}
+
+static inline u16 bnxt_re_get_rwqe_size(void)
+{
+	return sizeof(struct rq_wqe);
+}
+
 int bnxt_re_query_device(struct ib_device *ibdev,
 			 struct ib_device_attr *ib_attr,
 			 struct ib_udata *udata);
@@ -160,7 +164,7 @@ enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
 					    u8 port_num);
 int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
 void bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
-int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
 		      struct ib_udata *udata);
 int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
 int bnxt_re_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 899a5d2c100e..c5e29577cd43 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -300,12 +300,12 @@ static void bnxt_qplib_service_nq(unsigned long data)
 {
 	struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
 	struct bnxt_qplib_hwq *hwq = &nq->hwq;
-	struct nq_base *nqe, **nq_ptr;
-	struct bnxt_qplib_cq *cq;
-	int num_cqne_processed = 0;
 	int num_srqne_processed = 0;
+	int num_cqne_processed = 0;
+	struct bnxt_qplib_cq *cq;
 	int budget = nq->budget;
 	u32 sw_cons, raw_cons;
+	struct nq_base *nqe;
 	uintptr_t q_handle;
 	u16 type;
 
@@ -314,8 +314,7 @@ static void bnxt_qplib_service_nq(unsigned long data)
 	raw_cons = hwq->cons;
 	while (budget--) {
 		sw_cons = HWQ_CMP(raw_cons, hwq);
-		nq_ptr = (struct nq_base **)hwq->pbl_ptr;
-		nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
+		nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL);
 		if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
 			break;
 
@@ -392,13 +391,11 @@ static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
 {
 	struct bnxt_qplib_nq *nq = dev_instance;
 	struct bnxt_qplib_hwq *hwq = &nq->hwq;
-	struct nq_base **nq_ptr;
 	u32 sw_cons;
 
 	/* Prefetch the NQ element */
 	sw_cons = HWQ_CMP(hwq->cons, hwq);
-	nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
-	prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
+	prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));
 
 	/* Fan out to CPU affinitized kthreads? */
 	tasklet_schedule(&nq->nq_tasklet);
@@ -612,12 +609,13 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
 	struct cmdq_create_srq req;
 	struct bnxt_qplib_pbl *pbl;
 	u16 cmd_flags = 0;
+	u16 pg_sz_lvl;
 	int rc, idx;
 
 	hwq_attr.res = res;
 	hwq_attr.sginfo = &srq->sg_info;
 	hwq_attr.depth = srq->max_wqe;
-	hwq_attr.stride = BNXT_QPLIB_MAX_RQE_ENTRY_SIZE;
+	hwq_attr.stride = srq->wqe_size;
 	hwq_attr.type = HWQ_TYPE_QUEUE;
 	rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
 	if (rc)
@@ -638,22 +636,11 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
 
 	req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
 	pbl = &srq->hwq.pbl[PBL_LVL_0];
-	req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &
-				      CMDQ_CREATE_SRQ_LVL_MASK) <<
-				      CMDQ_CREATE_SRQ_LVL_SFT) |
-				      (pbl->pg_size == ROCE_PG_SIZE_4K ?
-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :
-				       pbl->pg_size == ROCE_PG_SIZE_8K ?
-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :
-				       pbl->pg_size == ROCE_PG_SIZE_64K ?
-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :
-				       pbl->pg_size == ROCE_PG_SIZE_2M ?
-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :
-				       pbl->pg_size == ROCE_PG_SIZE_8M ?
-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :
-				       pbl->pg_size == ROCE_PG_SIZE_1G ?
-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :
-				       CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));
+	pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) <<
+		     CMDQ_CREATE_SRQ_PG_SIZE_SFT);
+	pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) <<
+		      CMDQ_CREATE_SRQ_LVL_SFT;
+	req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
 	req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
 	req.pd_id = cpu_to_le32(srq->pd->id);
 	req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
@@ -740,7 +727,7 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 			     struct bnxt_qplib_swqe *wqe)
 {
 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
-	struct rq_wqe *srqe, **srqe_ptr;
+	struct rq_wqe *srqe;
 	struct sq_sge *hw_sge;
 	u32 sw_prod, sw_cons, count = 0;
 	int i, rc = 0, next;
@@ -758,9 +745,8 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 	spin_unlock(&srq_hwq->lock);
 
 	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
-	srqe_ptr = (struct rq_wqe **)srq_hwq->pbl_ptr;
-	srqe = &srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
-	memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
+	srqe = bnxt_qplib_get_qe(srq_hwq, sw_prod, NULL);
+	memset(srqe, 0, srq->wqe_size);
 	/* Calculate wqe_size16 and data_len */
 	for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
 	     i < wqe->num_sge; i++, hw_sge++) {
@@ -809,6 +795,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	struct bnxt_qplib_pbl *pbl;
 	u16 cmd_flags = 0;
 	u32 qp_flags = 0;
+	u8 pg_sz_lvl;
 	int rc;
 
 	RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
@@ -822,7 +809,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	hwq_attr.res = res;
 	hwq_attr.sginfo = &sq->sg_info;
 	hwq_attr.depth = sq->max_wqe;
-	hwq_attr.stride = BNXT_QPLIB_MAX_SQE_ENTRY_SIZE;
+	hwq_attr.stride = sq->wqe_size;
 	hwq_attr.type = HWQ_TYPE_QUEUE;
 	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
 	if (rc)
@@ -835,33 +822,18 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	}
 	pbl = &sq->hwq.pbl[PBL_LVL_0];
 	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-	req.sq_pg_size_sq_lvl =
-		((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
-				<<  CMDQ_CREATE_QP1_SQ_LVL_SFT) |
-		(pbl->pg_size == ROCE_PG_SIZE_4K ?
-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
-		 pbl->pg_size == ROCE_PG_SIZE_8K ?
-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
-		 pbl->pg_size == ROCE_PG_SIZE_64K ?
-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
-		 pbl->pg_size == ROCE_PG_SIZE_2M ?
-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
-		 pbl->pg_size == ROCE_PG_SIZE_8M ?
-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
-		 pbl->pg_size == ROCE_PG_SIZE_1G ?
-				CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
-		 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
+	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
+		     CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT);
+	pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK);
+	req.sq_pg_size_sq_lvl = pg_sz_lvl;
 
 	if (qp->scq)
 		req.scq_cid = cpu_to_le32(qp->scq->id);
-
-	qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
-
 	/* RQ */
 	if (rq->max_wqe) {
 		hwq_attr.res = res;
 		hwq_attr.sginfo = &rq->sg_info;
-		hwq_attr.stride = BNXT_QPLIB_MAX_RQE_ENTRY_SIZE;
+		hwq_attr.stride = rq->wqe_size;
 		hwq_attr.depth = qp->rq.max_wqe;
 		hwq_attr.type = HWQ_TYPE_QUEUE;
 		rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
@@ -876,32 +848,20 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 		}
 		pbl = &rq->hwq.pbl[PBL_LVL_0];
 		req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-		req.rq_pg_size_rq_lvl =
-			((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
-			 CMDQ_CREATE_QP1_RQ_LVL_SFT) |
-				(pbl->pg_size == ROCE_PG_SIZE_4K ?
-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
-				 pbl->pg_size == ROCE_PG_SIZE_8K ?
-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
-				 pbl->pg_size == ROCE_PG_SIZE_64K ?
-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
-				 pbl->pg_size == ROCE_PG_SIZE_2M ?
-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
-				 pbl->pg_size == ROCE_PG_SIZE_8M ?
-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
-				 pbl->pg_size == ROCE_PG_SIZE_1G ?
-					CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
-				 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
+		pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
+			     CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT);
+		pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK);
+		req.rq_pg_size_rq_lvl = pg_sz_lvl;
 		if (qp->rcq)
 			req.rcq_cid = cpu_to_le32(qp->rcq->id);
 	}
-
 	/* Header buffer - allow hdr_buf pass in */
 	rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
 	if (rc) {
 		rc = -ENOMEM;
 		goto fail;
 	}
+	qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
 	req.qp_flags = cpu_to_le32(qp_flags);
 	req.sq_size = cpu_to_le32(sq->hwq.max_elements);
 	req.rq_size = cpu_to_le32(rq->hwq.max_elements);
@@ -948,23 +908,47 @@ exit:
 	return rc;
 }
 
+static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
+{
+	struct bnxt_qplib_hwq *hwq;
+	struct bnxt_qplib_q *sq;
+	u64 fpsne, psne, psn_pg;
+	u16 indx_pad = 0, indx;
+	u16 pg_num, pg_indx;
+	u64 *page;
+
+	sq = &qp->sq;
+	hwq = &sq->hwq;
+
+	fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->max_elements, &psn_pg);
+	if (!IS_ALIGNED(fpsne, PAGE_SIZE))
+		indx_pad = ALIGN(fpsne, PAGE_SIZE) / size;
+
+	page = (u64 *)psn_pg;
+	for (indx = 0; indx < hwq->max_elements; indx++) {
+		pg_num = (indx + indx_pad) / (PAGE_SIZE / size);
+		pg_indx = (indx + indx_pad) % (PAGE_SIZE / size);
+		psne = page[pg_num] + pg_indx * size;
+		sq->swq[indx].psn_ext = (struct sq_psn_search_ext *)psne;
+		sq->swq[indx].psn_search = (struct sq_psn_search *)psne;
+	}
+}
+
 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct bnxt_qplib_hwq_attr hwq_attr = {};
-	unsigned long int psn_search, poff = 0;
 	struct bnxt_qplib_sg_info sginfo = {};
-	struct sq_psn_search **psn_search_ptr;
 	struct bnxt_qplib_q *sq = &qp->sq;
 	struct bnxt_qplib_q *rq = &qp->rq;
-	int i, rc, req_size, psn_sz = 0;
-	struct sq_send **hw_sq_send_ptr;
 	struct creq_create_qp_resp resp;
+	int rc, req_size, psn_sz = 0;
 	struct bnxt_qplib_hwq *xrrq;
 	u16 cmd_flags = 0, max_ssge;
-	struct cmdq_create_qp req;
 	struct bnxt_qplib_pbl *pbl;
+	struct cmdq_create_qp req;
 	u32 qp_flags = 0;
+	u8 pg_sz_lvl;
 	u16 max_rsge;
 
 	RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
@@ -983,7 +967,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	hwq_attr.res = res;
 	hwq_attr.sginfo = &sq->sg_info;
-	hwq_attr.stride = BNXT_QPLIB_MAX_SQE_ENTRY_SIZE;
+	hwq_attr.stride = sq->wqe_size;
 	hwq_attr.depth = sq->max_wqe;
 	hwq_attr.aux_stride = psn_sz;
 	hwq_attr.aux_depth = hwq_attr.depth;
@@ -997,64 +981,25 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 		rc = -ENOMEM;
 		goto fail_sq;
 	}
-	hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
-	if (psn_sz) {
-		psn_search_ptr = (struct sq_psn_search **)
-				  &hw_sq_send_ptr[get_sqe_pg
-					(sq->hwq.max_elements)];
-		psn_search = (unsigned long int)
-			      &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
-			      [get_sqe_idx(sq->hwq.max_elements)];
-		if (psn_search & ~PAGE_MASK) {
-			/* If the psn_search does not start on a page boundary,
-			 * then calculate the offset
-			 */
-			poff = (psn_search & ~PAGE_MASK) /
-				BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
-		}
-		for (i = 0; i < sq->hwq.max_elements; i++) {
-			sq->swq[i].psn_search =
-				&psn_search_ptr[get_psne_pg(i + poff)]
-					       [get_psne_idx(i + poff)];
-			/*psns_ext will be used only for P5 chips. */
-			sq->swq[i].psn_ext =
-				(struct sq_psn_search_ext *)
-				&psn_search_ptr[get_psne_pg(i + poff)]
-					       [get_psne_idx(i + poff)];
-		}
-	}
+
+	if (psn_sz)
+		bnxt_qplib_init_psn_ptr(qp, psn_sz);
+
 	pbl = &sq->hwq.pbl[PBL_LVL_0];
 	req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-	req.sq_pg_size_sq_lvl =
-		((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
-				 <<  CMDQ_CREATE_QP_SQ_LVL_SFT) |
-		(pbl->pg_size == ROCE_PG_SIZE_4K ?
-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
-		 pbl->pg_size == ROCE_PG_SIZE_8K ?
-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
-		 pbl->pg_size == ROCE_PG_SIZE_64K ?
-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
-		 pbl->pg_size == ROCE_PG_SIZE_2M ?
-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
-		 pbl->pg_size == ROCE_PG_SIZE_8M ?
-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
-		 pbl->pg_size == ROCE_PG_SIZE_1G ?
-				CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
-		 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
+	pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) <<
+		     CMDQ_CREATE_QP_SQ_PG_SIZE_SFT);
+	pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK);
+	req.sq_pg_size_sq_lvl = pg_sz_lvl;
 
 	if (qp->scq)
 		req.scq_cid = cpu_to_le32(qp->scq->id);
 
-	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
-	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
-	if (qp->sig_type)
-		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
-
 	/* RQ */
 	if (rq->max_wqe) {
 		hwq_attr.res = res;
 		hwq_attr.sginfo = &rq->sg_info;
-		hwq_attr.stride = BNXT_QPLIB_MAX_RQE_ENTRY_SIZE;
+		hwq_attr.stride = rq->wqe_size;
 		hwq_attr.depth = rq->max_wqe;
 		hwq_attr.aux_stride = 0;
 		hwq_attr.aux_depth = 0;
@@ -1071,22 +1016,10 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 		}
 		pbl = &rq->hwq.pbl[PBL_LVL_0];
 		req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-		req.rq_pg_size_rq_lvl =
-			((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
-			 CMDQ_CREATE_QP_RQ_LVL_SFT) |
-				(pbl->pg_size == ROCE_PG_SIZE_4K ?
-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
-				 pbl->pg_size == ROCE_PG_SIZE_8K ?
-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
-				 pbl->pg_size == ROCE_PG_SIZE_64K ?
-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
-				 pbl->pg_size == ROCE_PG_SIZE_2M ?
-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
-				 pbl->pg_size == ROCE_PG_SIZE_8M ?
-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
-				 pbl->pg_size == ROCE_PG_SIZE_1G ?
-					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
-				 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
+		pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) <<
+			     CMDQ_CREATE_QP_RQ_PG_SIZE_SFT);
+		pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK);
+		req.rq_pg_size_rq_lvl = pg_sz_lvl;
 	} else {
 		/* SRQ */
 		if (qp->srq) {
@@ -1097,7 +1030,13 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	if (qp->rcq)
 		req.rcq_cid = cpu_to_le32(qp->rcq->id);
+
+	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
+	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
+	if (qp->sig_type)
+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
 	req.qp_flags = cpu_to_le32(qp_flags);
+
 	req.sq_size = cpu_to_le32(sq->hwq.max_elements);
 	req.rq_size = cpu_to_le32(rq->hwq.max_elements);
 	qp->sq_hdr_buf = NULL;
@@ -1483,12 +1422,11 @@ bail:
 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
 {
 	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
-	struct cq_base *hw_cqe, **hw_cqe_ptr;
+	struct cq_base *hw_cqe;
 	int i;
 
 	for (i = 0; i < cq_hwq->max_elements; i++) {
-		hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
-		hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
+		hw_cqe = bnxt_qplib_get_qe(cq_hwq, i, NULL);
 		if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
 			continue;
 		/*
@@ -1615,6 +1553,34 @@ void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
 	return NULL;
 }
 
+static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
+				       struct bnxt_qplib_swqe *wqe,
+				       struct bnxt_qplib_swq *swq)
+{
+	struct sq_psn_search_ext *psns_ext;
+	struct sq_psn_search *psns;
+	u32 flg_npsn;
+	u32 op_spsn;
+
+	psns = swq->psn_search;
+	psns_ext = swq->psn_ext;
+
+	op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
+		    SQ_PSN_SEARCH_START_PSN_MASK);
+	op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
+		     SQ_PSN_SEARCH_OPCODE_MASK);
+	flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
+		     SQ_PSN_SEARCH_NEXT_PSN_MASK);
+
+	if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
+		psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
+		psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
+	} else {
+		psns->opcode_start_psn = cpu_to_le32(op_spsn);
+		psns->flags_next_psn = cpu_to_le32(flg_npsn);
+	}
+}
+
 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
 {
 	struct bnxt_qplib_q *sq = &qp->sq;
@@ -1625,16 +1591,16 @@ void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
 			 struct bnxt_qplib_swqe *wqe)
 {
+	struct bnxt_qplib_nq_work *nq_work = NULL;
+	int i, rc = 0, data_len = 0, pkt_num = 0;
 	struct bnxt_qplib_q *sq = &qp->sq;
+	struct sq_send *hw_sq_send_hdr;
 	struct bnxt_qplib_swq *swq;
-	struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
-	struct sq_sge *hw_sge;
-	struct bnxt_qplib_nq_work *nq_work = NULL;
 	bool sch_handler = false;
-	u32 sw_prod;
+	struct sq_sge *hw_sge;
 	u8 wqe_size16;
-	int i, rc = 0, data_len = 0, pkt_num = 0;
 	__le32 temp32;
+	u32 sw_prod;
 
 	if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
 		if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
@@ -1663,11 +1629,8 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
 		swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
 	swq->start_psn = sq->psn & BTH_PSN_MASK;
 
-	hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
-	hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
-					[get_sqe_idx(sw_prod)];
-
-	memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
+	hw_sq_send_hdr = bnxt_qplib_get_qe(&sq->hwq, sw_prod, NULL);
+	memset(hw_sq_send_hdr, 0, sq->wqe_size);
 
 	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
 		/* Copy the inline data */
@@ -1854,28 +1817,8 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
 		goto done;
 	}
 	swq->next_psn = sq->psn & BTH_PSN_MASK;
-	if (swq->psn_search) {
-		u32 opcd_spsn;
-		u32 flg_npsn;
-
-		opcd_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
-			      SQ_PSN_SEARCH_START_PSN_MASK);
-		opcd_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
-			       SQ_PSN_SEARCH_OPCODE_MASK);
-		flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
-			     SQ_PSN_SEARCH_NEXT_PSN_MASK);
-		if (bnxt_qplib_is_chip_gen_p5(qp->cctx)) {
-			swq->psn_ext->opcode_start_psn =
-						cpu_to_le32(opcd_spsn);
-			swq->psn_ext->flags_next_psn =
-						cpu_to_le32(flg_npsn);
-		} else {
-			swq->psn_search->opcode_start_psn =
-						cpu_to_le32(opcd_spsn);
-			swq->psn_search->flags_next_psn =
-						cpu_to_le32(flg_npsn);
-		}
-	}
+	if (qp->type == CMDQ_CREATE_QP_TYPE_RC)
+		bnxt_qplib_fill_psn_search(qp, wqe, swq);
 queue_err:
 	if (sch_handler) {
 		/* Store the ULP info in the software structures */
@@ -1918,13 +1861,13 @@ void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
 			 struct bnxt_qplib_swqe *wqe)
 {
-	struct bnxt_qplib_q *rq = &qp->rq;
-	struct rq_wqe *rqe, **rqe_ptr;
-	struct sq_sge *hw_sge;
 	struct bnxt_qplib_nq_work *nq_work = NULL;
+	struct bnxt_qplib_q *rq = &qp->rq;
 	bool sch_handler = false;
-	u32 sw_prod;
+	struct sq_sge *hw_sge;
+	struct rq_wqe *rqe;
 	int i, rc = 0;
+	u32 sw_prod;
 
 	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
 		sch_handler = true;
@@ -1941,10 +1884,8 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
 	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
 	rq->swq[sw_prod].wr_id = wqe->wr_id;
 
-	rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
-	rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
-
-	memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
+	rqe = bnxt_qplib_get_qe(&rq->hwq, sw_prod, NULL);
+	memset(rqe, 0, rq->wqe_size);
 
 	/* Calculate wqe_size16 and data_len */
 	for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
@@ -1997,9 +1938,10 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct bnxt_qplib_hwq_attr hwq_attr = {};
 	struct creq_create_cq_resp resp;
-	struct cmdq_create_cq req;
 	struct bnxt_qplib_pbl *pbl;
+	struct cmdq_create_cq req;
 	u16 cmd_flags = 0;
+	u32 pg_sz_lvl;
 	int rc;
 
 	hwq_attr.res = res;
@@ -2020,22 +1962,13 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 	}
 	req.dpi = cpu_to_le32(cq->dpi->dpi);
 	req.cq_handle = cpu_to_le64(cq->cq_handle);
-
 	req.cq_size = cpu_to_le32(cq->hwq.max_elements);
 	pbl = &cq->hwq.pbl[PBL_LVL_0];
-	req.pg_size_lvl = cpu_to_le32(
-	    ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
-						CMDQ_CREATE_CQ_LVL_SFT) |
-	    (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
-	     pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
-	     pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
-	     pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
-	     pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
-	     pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
-	     CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
-
+	pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) <<
+		     CMDQ_CREATE_CQ_PG_SIZE_SFT);
+	pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK);
+	req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
 	req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
-
 	req.cq_fco_cnq_id = cpu_to_le32(
 			(cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
 			 CMDQ_CREATE_CQ_CNQ_ID_SFT);
@@ -2194,13 +2127,13 @@ void bnxt_qplib_mark_qp_error(void *qp_handle)
 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
 		     u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
 {
-	struct bnxt_qplib_q *sq = &qp->sq;
-	struct bnxt_qplib_swq *swq;
 	u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
-	struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;
+	struct bnxt_qplib_q *sq = &qp->sq;
 	struct cq_req *peek_req_hwcqe;
 	struct bnxt_qplib_qp *peek_qp;
 	struct bnxt_qplib_q *peek_sq;
+	struct bnxt_qplib_swq *swq;
+	struct cq_base *peek_hwcqe;
 	int i, rc = 0;
 
 	/* Normal mode */
@@ -2230,9 +2163,8 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
 		i = cq->hwq.max_elements;
 		while (i--) {
 			peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
-			peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
-			peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
-						     [CQE_IDX(peek_sw_cq_cons)];
+			peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
+						       peek_sw_cq_cons, NULL);
 			/* If the next hwcqe is VALID */
 			if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
 					  cq->hwq.max_elements)) {
@@ -2294,11 +2226,11 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
 				     struct bnxt_qplib_cqe **pcqe, int *budget,
 				     u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
 {
-	struct bnxt_qplib_qp *qp;
-	struct bnxt_qplib_q *sq;
-	struct bnxt_qplib_cqe *cqe;
 	u32 sw_sq_cons, cqe_sq_cons;
 	struct bnxt_qplib_swq *swq;
+	struct bnxt_qplib_cqe *cqe;
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_q *sq;
 	int rc = 0;
 
 	qp = (struct bnxt_qplib_qp *)((unsigned long)
@@ -2408,10 +2340,10 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 					struct bnxt_qplib_cqe **pcqe,
 					int *budget)
 {
-	struct bnxt_qplib_qp *qp;
-	struct bnxt_qplib_q *rq;
 	struct bnxt_qplib_srq *srq;
 	struct bnxt_qplib_cqe *cqe;
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_q *rq;
 	u32 wr_id_idx;
 	int rc = 0;
 
@@ -2483,10 +2415,10 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 					struct bnxt_qplib_cqe **pcqe,
 					int *budget)
 {
-	struct bnxt_qplib_qp *qp;
-	struct bnxt_qplib_q *rq;
 	struct bnxt_qplib_srq *srq;
 	struct bnxt_qplib_cqe *cqe;
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_q *rq;
 	u32 wr_id_idx;
 	int rc = 0;
 
@@ -2561,15 +2493,13 @@ done:
 
 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
 {
-	struct cq_base *hw_cqe, **hw_cqe_ptr;
+	struct cq_base *hw_cqe;
 	u32 sw_cons, raw_cons;
 	bool rc = true;
 
 	raw_cons = cq->hwq.cons;
 	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
-	hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
-	hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
-
+	hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
 	 /* Check for Valid bit. If the CQE is valid, return false */
 	rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
 	return rc;
@@ -2813,7 +2743,7 @@ int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 		       int num_cqes, struct bnxt_qplib_qp **lib_qp)
 {
-	struct cq_base *hw_cqe, **hw_cqe_ptr;
+	struct cq_base *hw_cqe;
 	u32 sw_cons, raw_cons;
 	int budget, rc = 0;
 
@@ -2822,8 +2752,7 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 
 	while (budget) {
 		sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
-		hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
-		hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
+		hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
 
 		/* Check for Valid bit */
 		if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 7edb70b6bb16..568ca390322c 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -45,6 +45,7 @@ struct bnxt_qplib_srq {
 	struct bnxt_qplib_db_info	dbinfo;
 	u64				srq_handle;
 	u32				id;
+	u16				wqe_size;
 	u32				max_wqe;
 	u32				max_sge;
 	u32				threshold;
@@ -65,38 +66,7 @@ struct bnxt_qplib_sge {
 	u32				size;
 };
 
-#define BNXT_QPLIB_MAX_SQE_ENTRY_SIZE	sizeof(struct sq_send)
-
-#define SQE_CNT_PER_PG		(PAGE_SIZE / BNXT_QPLIB_MAX_SQE_ENTRY_SIZE)
-#define SQE_MAX_IDX_PER_PG	(SQE_CNT_PER_PG - 1)
-
-static inline u32 get_sqe_pg(u32 val)
-{
-	return ((val & ~SQE_MAX_IDX_PER_PG) / SQE_CNT_PER_PG);
-}
-
-static inline u32 get_sqe_idx(u32 val)
-{
-	return (val & SQE_MAX_IDX_PER_PG);
-}
-
-#define BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE	sizeof(struct sq_psn_search)
-
-#define PSNE_CNT_PER_PG		(PAGE_SIZE / BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE)
-#define PSNE_MAX_IDX_PER_PG	(PSNE_CNT_PER_PG - 1)
-
-static inline u32 get_psne_pg(u32 val)
-{
-	return ((val & ~PSNE_MAX_IDX_PER_PG) / PSNE_CNT_PER_PG);
-}
-
-static inline u32 get_psne_idx(u32 val)
-{
-	return (val & PSNE_MAX_IDX_PER_PG);
-}
-
 #define BNXT_QPLIB_QP_MAX_SGL	6
-
 struct bnxt_qplib_swq {
 	u64				wr_id;
 	int				next_idx;
@@ -226,19 +196,13 @@ struct bnxt_qplib_swqe {
 	};
 };
 
-#define BNXT_QPLIB_MAX_RQE_ENTRY_SIZE	sizeof(struct rq_wqe)
-
-#define RQE_CNT_PER_PG		(PAGE_SIZE / BNXT_QPLIB_MAX_RQE_ENTRY_SIZE)
-#define RQE_MAX_IDX_PER_PG	(RQE_CNT_PER_PG - 1)
-#define RQE_PG(x)		(((x) & ~RQE_MAX_IDX_PER_PG) / RQE_CNT_PER_PG)
-#define RQE_IDX(x)		((x) & RQE_MAX_IDX_PER_PG)
-
 struct bnxt_qplib_q {
 	struct bnxt_qplib_hwq		hwq;
 	struct bnxt_qplib_swq		*swq;
 	struct bnxt_qplib_db_info	dbinfo;
 	struct bnxt_qplib_sg_info	sg_info;
 	u32				max_wqe;
+	u16				wqe_size;
 	u16				q_full_delta;
 	u16				max_sge;
 	u32				psn;
@@ -256,7 +220,7 @@ struct bnxt_qplib_qp {
 	struct bnxt_qplib_dpi		*dpi;
 	struct bnxt_qplib_chip_ctx	*cctx;
 	u64				qp_handle;
-#define        BNXT_QPLIB_QP_ID_INVALID        0xFFFFFFFF
+#define BNXT_QPLIB_QP_ID_INVALID        0xFFFFFFFF
 	u32				id;
 	u8				type;
 	u8				sig_type;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index f01e864bb611..4e211162acee 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -89,10 +89,9 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
 			  struct creq_base *resp, void *sb, u8 is_block)
 {
 	struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
-	struct bnxt_qplib_cmdqe *cmdqe, **hwq_ptr;
 	struct bnxt_qplib_hwq *hwq = &cmdq->hwq;
 	struct bnxt_qplib_crsqe *crsqe;
-	u32 cmdq_depth = rcfw->cmdq_depth;
+	struct bnxt_qplib_cmdqe *cmdqe;
 	u32 sw_prod, cmdq_prod;
 	struct pci_dev *pdev;
 	unsigned long flags;
@@ -163,13 +162,11 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
 				  BNXT_QPLIB_CMDQE_UNITS;
 	}
 
-	hwq_ptr = (struct bnxt_qplib_cmdqe **)hwq->pbl_ptr;
 	preq = (u8 *)req;
 	do {
 		/* Locate the next cmdq slot */
 		sw_prod = HWQ_CMP(hwq->prod, hwq);
-		cmdqe = &hwq_ptr[get_cmdq_pg(sw_prod, cmdq_depth)]
-				[get_cmdq_idx(sw_prod, cmdq_depth)];
+		cmdqe = bnxt_qplib_get_qe(hwq, sw_prod, NULL);
 		if (!cmdqe) {
 			dev_err(&pdev->dev,
 				"RCFW request failed with no cmdqe!\n");
@@ -378,7 +375,7 @@ static void bnxt_qplib_service_creq(unsigned long data)
 	struct bnxt_qplib_creq_ctx *creq = &rcfw->creq;
 	u32 type, budget = CREQ_ENTRY_POLL_BUDGET;
 	struct bnxt_qplib_hwq *hwq = &creq->hwq;
-	struct creq_base *creqe, **hwq_ptr;
+	struct creq_base *creqe;
 	u32 sw_cons, raw_cons;
 	unsigned long flags;
 
@@ -387,8 +384,7 @@ static void bnxt_qplib_service_creq(unsigned long data)
 	raw_cons = hwq->cons;
 	while (budget > 0) {
 		sw_cons = HWQ_CMP(raw_cons, hwq);
-		hwq_ptr = (struct creq_base **)hwq->pbl_ptr;
-		creqe = &hwq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)];
+		creqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL);
 		if (!CREQ_CMP_VALID(creqe, raw_cons, hwq->max_elements))
 			break;
 		/* The valid test of the entry must be done first before
@@ -434,7 +430,6 @@ static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
 {
 	struct bnxt_qplib_rcfw *rcfw = dev_instance;
 	struct bnxt_qplib_creq_ctx *creq;
-	struct creq_base **creq_ptr;
 	struct bnxt_qplib_hwq *hwq;
 	u32 sw_cons;
 
@@ -442,8 +437,7 @@ static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
 	hwq = &creq->hwq;
 	/* Prefetch the CREQ element */
 	sw_cons = HWQ_CMP(hwq->cons, hwq);
-	creq_ptr = (struct creq_base **)creq->hwq.pbl_ptr;
-	prefetch(&creq_ptr[get_creq_pg(sw_cons)][get_creq_idx(sw_cons)]);
+	prefetch(bnxt_qplib_get_qe(hwq, sw_cons, NULL));
 
 	tasklet_schedule(&creq->creq_tasklet);
 
@@ -468,29 +462,13 @@ int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
 	return 0;
 }
 
-static int __get_pbl_pg_idx(struct bnxt_qplib_pbl *pbl)
-{
-	return (pbl->pg_size == ROCE_PG_SIZE_4K ?
-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K :
-		pbl->pg_size == ROCE_PG_SIZE_8K ?
-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8K :
-		pbl->pg_size == ROCE_PG_SIZE_64K ?
-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_64K :
-		pbl->pg_size == ROCE_PG_SIZE_2M ?
-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_2M :
-		pbl->pg_size == ROCE_PG_SIZE_8M ?
-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_8M :
-		pbl->pg_size == ROCE_PG_SIZE_1G ?
-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_1G :
-				      CMDQ_INITIALIZE_FW_QPC_PG_SIZE_PG_4K);
-}
-
 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
 			 struct bnxt_qplib_ctx *ctx, int is_virtfn)
 {
-	struct cmdq_initialize_fw req;
 	struct creq_initialize_fw_resp resp;
-	u16 cmd_flags = 0, level;
+	struct cmdq_initialize_fw req;
+	u16 cmd_flags = 0;
+	u8 pgsz, lvl;
 	int rc;
 
 	RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags);
@@ -511,32 +489,30 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
 	if (bnxt_qplib_is_chip_gen_p5(rcfw->res->cctx))
 		goto config_vf_res;
 
-	level = ctx->qpc_tbl.level;
-	req.qpc_pg_size_qpc_lvl = (level << CMDQ_INITIALIZE_FW_QPC_LVL_SFT) |
-				__get_pbl_pg_idx(&ctx->qpc_tbl.pbl[level]);
-	level = ctx->mrw_tbl.level;
-	req.mrw_pg_size_mrw_lvl = (level << CMDQ_INITIALIZE_FW_MRW_LVL_SFT) |
-				__get_pbl_pg_idx(&ctx->mrw_tbl.pbl[level]);
-	level = ctx->srqc_tbl.level;
-	req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |
-				__get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]);
-	level = ctx->cq_tbl.level;
-	req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |
-				__get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]);
-	level = ctx->srqc_tbl.level;
-	req.srq_pg_size_srq_lvl = (level << CMDQ_INITIALIZE_FW_SRQ_LVL_SFT) |
-				__get_pbl_pg_idx(&ctx->srqc_tbl.pbl[level]);
-	level = ctx->cq_tbl.level;
-	req.cq_pg_size_cq_lvl = (level << CMDQ_INITIALIZE_FW_CQ_LVL_SFT) |
-				__get_pbl_pg_idx(&ctx->cq_tbl.pbl[level]);
-	level = ctx->tim_tbl.level;
-	req.tim_pg_size_tim_lvl = (level << CMDQ_INITIALIZE_FW_TIM_LVL_SFT) |
-				  __get_pbl_pg_idx(&ctx->tim_tbl.pbl[level]);
-	level = ctx->tqm_ctx.pde.level;
-	req.tqm_pg_size_tqm_lvl =
-		(level << CMDQ_INITIALIZE_FW_TQM_LVL_SFT) |
-		 __get_pbl_pg_idx(&ctx->tqm_ctx.pde.pbl[level]);
-
+	lvl = ctx->qpc_tbl.level;
+	pgsz = bnxt_qplib_base_pg_size(&ctx->qpc_tbl);
+	req.qpc_pg_size_qpc_lvl = (pgsz << CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+				   lvl;
+	lvl = ctx->mrw_tbl.level;
+	pgsz = bnxt_qplib_base_pg_size(&ctx->mrw_tbl);
+	req.mrw_pg_size_mrw_lvl = (pgsz << CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+				   lvl;
+	lvl = ctx->srqc_tbl.level;
+	pgsz = bnxt_qplib_base_pg_size(&ctx->srqc_tbl);
+	req.srq_pg_size_srq_lvl = (pgsz << CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+				   lvl;
+	lvl = ctx->cq_tbl.level;
+	pgsz = bnxt_qplib_base_pg_size(&ctx->cq_tbl);
+	req.cq_pg_size_cq_lvl = (pgsz << CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+				 lvl;
+	lvl = ctx->tim_tbl.level;
+	pgsz = bnxt_qplib_base_pg_size(&ctx->tim_tbl);
+	req.tim_pg_size_tim_lvl = (pgsz << CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+				   lvl;
+	lvl = ctx->tqm_ctx.pde.level;
+	pgsz = bnxt_qplib_base_pg_size(&ctx->tqm_ctx.pde);
+	req.tqm_pg_size_tqm_lvl = (pgsz << CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+				   lvl;
 	req.qpc_page_dir =
 		cpu_to_le64(ctx->qpc_tbl.pbl[PBL_LVL_0].pg_map_arr[0]);
 	req.mrw_page_dir =
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
index 411fce3493b6..157387636d00 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
@@ -87,12 +87,6 @@ static inline u32 bnxt_qplib_cmdqe_page_size(u32 depth)
 	return (bnxt_qplib_cmdqe_npages(depth) * PAGE_SIZE);
 }
 
-static inline u32 bnxt_qplib_cmdqe_cnt_per_pg(u32 depth)
-{
-	return (bnxt_qplib_cmdqe_page_size(depth) /
-		 BNXT_QPLIB_CMDQE_UNITS);
-}
-
 /* Set the cmd_size to a factor of CMDQE unit */
 static inline void bnxt_qplib_set_cmd_slots(struct cmdq_base *req)
 {
@@ -100,30 +94,12 @@ static inline void bnxt_qplib_set_cmd_slots(struct cmdq_base *req)
 			 BNXT_QPLIB_CMDQE_UNITS;
 }
 
-#define MAX_CMDQ_IDX(depth)		((depth) - 1)
-
-static inline u32 bnxt_qplib_max_cmdq_idx_per_pg(u32 depth)
-{
-	return (bnxt_qplib_cmdqe_cnt_per_pg(depth) - 1);
-}
-
 #define RCFW_MAX_COOKIE_VALUE		0x7FFF
 #define RCFW_CMD_IS_BLOCKING		0x8000
 #define RCFW_BLOCKED_CMD_WAIT_COUNT	0x4E20
 
 #define HWRM_VERSION_RCFW_CMDQ_DEPTH_CHECK 0x1000900020011ULL
 
-static inline u32 get_cmdq_pg(u32 val, u32 depth)
-{
-	return (val & ~(bnxt_qplib_max_cmdq_idx_per_pg(depth))) /
-		(bnxt_qplib_cmdqe_cnt_per_pg(depth));
-}
-
-static inline u32 get_cmdq_idx(u32 val, u32 depth)
-{
-	return val & (bnxt_qplib_max_cmdq_idx_per_pg(depth));
-}
-
 /* Crsq buf is 1024-Byte */
 struct bnxt_qplib_crsbe {
 	u8			data[1024];
@@ -133,76 +109,9 @@ struct bnxt_qplib_crsbe {
 /* Allocate 1 per QP for async error notification for now */
 #define BNXT_QPLIB_CREQE_MAX_CNT	(64 * 1024)
 #define BNXT_QPLIB_CREQE_UNITS		16	/* 16-Bytes per prod unit */
-#define BNXT_QPLIB_CREQE_CNT_PER_PG	(PAGE_SIZE / BNXT_QPLIB_CREQE_UNITS)
-
-#define MAX_CREQ_IDX			(BNXT_QPLIB_CREQE_MAX_CNT - 1)
-#define MAX_CREQ_IDX_PER_PG		(BNXT_QPLIB_CREQE_CNT_PER_PG - 1)
-
-static inline u32 get_creq_pg(u32 val)
-{
-	return (val & ~MAX_CREQ_IDX_PER_PG) / BNXT_QPLIB_CREQE_CNT_PER_PG;
-}
-
-static inline u32 get_creq_idx(u32 val)
-{
-	return val & MAX_CREQ_IDX_PER_PG;
-}
-
-#define BNXT_QPLIB_CREQE_PER_PG	(PAGE_SIZE / sizeof(struct creq_base))
-
 #define CREQ_CMP_VALID(hdr, raw_cons, cp_bit)			\
 	(!!((hdr)->v & CREQ_BASE_V) ==				\
 	   !((raw_cons) & (cp_bit)))
-
-#define CREQ_DB_KEY_CP			(0x2 << CMPL_DOORBELL_KEY_SFT)
-#define CREQ_DB_IDX_VALID		CMPL_DOORBELL_IDX_VALID
-#define CREQ_DB_IRQ_DIS			CMPL_DOORBELL_MASK
-#define CREQ_DB_CP_FLAGS_REARM		(CREQ_DB_KEY_CP |	\
-					 CREQ_DB_IDX_VALID)
-#define CREQ_DB_CP_FLAGS		(CREQ_DB_KEY_CP |	\
-					 CREQ_DB_IDX_VALID |	\
-					 CREQ_DB_IRQ_DIS)
-
-static inline void bnxt_qplib_ring_creq_db64(void __iomem *db, u32 index,
-					     u32 xid, bool arm)
-{
-	u64 val = 0;
-
-	val = xid & DBC_DBC_XID_MASK;
-	val |= DBC_DBC_PATH_ROCE;
-	val |= arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;
-	val <<= 32;
-	val |= index & DBC_DBC_INDEX_MASK;
-
-	writeq(val, db);
-}
-
-static inline void bnxt_qplib_ring_creq_db_rearm(void __iomem *db, u32 raw_cons,
-						 u32 max_elements, u32 xid,
-						 bool gen_p5)
-{
-	u32 index = raw_cons & (max_elements - 1);
-
-	if (gen_p5)
-		bnxt_qplib_ring_creq_db64(db, index, xid, true);
-	else
-		writel(CREQ_DB_CP_FLAGS_REARM | (index & DBC_DBC32_XID_MASK),
-		       db);
-}
-
-static inline void bnxt_qplib_ring_creq_db(void __iomem *db, u32 raw_cons,
-					   u32 max_elements, u32 xid,
-					   bool gen_p5)
-{
-	u32 index = raw_cons & (max_elements - 1);
-
-	if (gen_p5)
-		bnxt_qplib_ring_creq_db64(db, index, xid, true);
-	else
-		writel(CREQ_DB_CP_FLAGS | (index & DBC_DBC32_XID_MASK),
-		       db);
-}
-
 #define CREQ_ENTRY_POLL_BUDGET		0x100
 
 /* HWQ */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index cab1adf1fed9..7efa6e5dce62 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -347,6 +347,7 @@ done:
 	hwq->depth = hwq_attr->depth;
 	hwq->max_elements = depth;
 	hwq->element_size = stride;
+	hwq->qe_ppg = pg_size / stride;
 	/* For direct access to the elements */
 	lvl = hwq->level;
 	if (hwq_attr->sginfo->nopte && hwq->level)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index 95b645dbbc2d..c29cbd3a2d7b 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -80,6 +80,15 @@ enum bnxt_qplib_pbl_lvl {
 #define ROCE_PG_SIZE_8M		(8 * 1024 * 1024)
 #define ROCE_PG_SIZE_1G		(1024 * 1024 * 1024)
 
+enum bnxt_qplib_hwrm_pg_size {
+	BNXT_QPLIB_HWRM_PG_SIZE_4K	= 0,
+	BNXT_QPLIB_HWRM_PG_SIZE_8K	= 1,
+	BNXT_QPLIB_HWRM_PG_SIZE_64K	= 2,
+	BNXT_QPLIB_HWRM_PG_SIZE_2M	= 3,
+	BNXT_QPLIB_HWRM_PG_SIZE_8M	= 4,
+	BNXT_QPLIB_HWRM_PG_SIZE_1G	= 5,
+};
+
 struct bnxt_qplib_reg_desc {
 	u8		bar_id;
 	resource_size_t	bar_base;
@@ -126,6 +135,7 @@ struct bnxt_qplib_hwq {
 	u32				max_elements;
 	u32				depth;
 	u16				element_size;	/* Size of each entry */
+	u16				qe_ppg;	/* queue entry per page */
 
 	u32				prod;		/* raw */
 	u32				cons;		/* raw */
@@ -263,6 +273,49 @@ static inline u8 bnxt_qplib_get_ring_type(struct bnxt_qplib_chip_ctx *cctx)
 	       RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL;
 }
 
+static inline u8 bnxt_qplib_base_pg_size(struct bnxt_qplib_hwq *hwq)
+{
+	u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+	struct bnxt_qplib_pbl *pbl;
+
+	pbl = &hwq->pbl[PBL_LVL_0];
+	switch (pbl->pg_size) {
+	case ROCE_PG_SIZE_4K:
+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+		break;
+	case ROCE_PG_SIZE_8K:
+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K;
+		break;
+	case ROCE_PG_SIZE_64K:
+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K;
+		break;
+	case ROCE_PG_SIZE_2M:
+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M;
+		break;
+	case ROCE_PG_SIZE_8M:
+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M;
+		break;
+	case ROCE_PG_SIZE_1G:
+		pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G;
+		break;
+	default:
+		break;
+	}
+
+	return pg_size;
+}
+
+static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq,
+				      u32 indx, u64 *pg)
+{
+	u32 pg_num, pg_idx;
+
+	pg_num = (indx / hwq->qe_ppg);
+	pg_idx = (indx % hwq->qe_ppg);
+	if (pg)
+		*pg = (u64)&hwq->pbl_ptr[pg_num];
+	return (void *)(hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx);
+}
 
 #define to_bnxt_qplib(ptr, type, member)	\
 	container_of(ptr, type, member)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index 66954ff6a2f2..4cd475ea97a2 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -132,9 +132,6 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 	attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp);
 	attr->max_ah = le32_to_cpu(sb->max_ah);
 
-	attr->max_fmr = le32_to_cpu(sb->max_fmr);
-	attr->max_map_per_fmr = sb->max_map_per_fmr;
-
 	attr->max_srq = le16_to_cpu(sb->max_srq);
 	attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
 	attr->max_srq_sges = sb->max_srq_sge;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index 13d9432d5ce2..6404f0da1051 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -64,8 +64,6 @@ struct bnxt_qplib_dev_attr {
 	u32				max_mw;
 	u32				max_raw_ethy_qp;
 	u32				max_ah;
-	u32				max_fmr;
-	u32				max_map_per_fmr;
 	u32				max_srq;
 	u32				max_srq_wqes;
 	u32				max_srq_sges;
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index e4b09e7c2175..6f00f07420b7 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -210,6 +210,20 @@ struct sq_send {
 	__le32 data[24];
 };
 
+/* sq_send_hdr (size:256b/32B) */
+struct sq_send_hdr {
+	u8	wqe_type;
+	u8	flags;
+	u8	wqe_size;
+	u8	reserved8_1;
+	__le32	inv_key_or_imm_data;
+	__le32	length;
+	__le32	q_key;
+	__le32	dst_qp;
+	__le32	avid;
+	__le64	reserved64;
+};
+
 /* Send Raw Ethernet and QP1 SQ WQE (40 bytes) */
 struct sq_send_raweth_qp1 {
 	u8 wqe_type;
@@ -265,6 +279,21 @@ struct sq_send_raweth_qp1 {
 	__le32 data[24];
 };
 
+/* sq_send_raweth_qp1_hdr (size:256b/32B) */
+struct sq_send_raweth_qp1_hdr {
+	u8	wqe_type;
+	u8	flags;
+	u8	wqe_size;
+	u8	reserved8;
+	__le16	lflags;
+	__le16	cfa_action;
+	__le32	length;
+	__le32	reserved32_1;
+	__le32	cfa_meta;
+	__le32	reserved32_2;
+	__le64	reserved64;
+};
+
 /* RDMA SQ WQE (40 bytes) */
 struct sq_rdma {
 	u8 wqe_type;
@@ -288,6 +317,20 @@ struct sq_rdma {
 	__le32 data[24];
 };
 
+/* sq_rdma_hdr (size:256b/32B) */
+struct sq_rdma_hdr {
+	u8	wqe_type;
+	u8	flags;
+	u8	wqe_size;
+	u8	reserved8;
+	__le32	imm_data;
+	__le32	length;
+	__le32	reserved32_1;
+	__le64	remote_va;
+	__le32	remote_key;
+	__le32	reserved32_2;
+};
+
 /* Atomic SQ WQE (40 bytes) */
 struct sq_atomic {
 	u8 wqe_type;
@@ -307,6 +350,17 @@ struct sq_atomic {
 	__le32 data[24];
 };
 
+/* sq_atomic_hdr (size:256b/32B) */
+struct sq_atomic_hdr {
+	u8	wqe_type;
+	u8	flags;
+	__le16	reserved16;
+	__le32	remote_key;
+	__le64	remote_va;
+	__le64	swap_data;
+	__le64	cmp_data;
+};
+
 /* Local Invalidate SQ WQE (40 bytes) */
 struct sq_localinvalidate {
 	u8 wqe_type;
@@ -324,6 +378,16 @@ struct sq_localinvalidate {
 	__le32 data[24];
 };
 
+/* sq_localinvalidate_hdr (size:256b/32B) */
+struct sq_localinvalidate_hdr {
+	u8	wqe_type;
+	u8	flags;
+	__le16	reserved16;
+	__le32	inv_l_key;
+	__le64	reserved64;
+	u8	reserved128[16];
+};
+
 /* FR-PMR SQ WQE (40 bytes) */
 struct sq_fr_pmr {
 	u8 wqe_type;
@@ -380,6 +444,21 @@ struct sq_fr_pmr {
 	__le32 data[24];
 };
 
+/* sq_fr_pmr_hdr (size:256b/32B) */
+struct sq_fr_pmr_hdr {
+	u8	wqe_type;
+	u8	flags;
+	u8	access_cntl;
+	u8	zero_based_page_size_log;
+	__le32	l_key;
+	u8	length[5];
+	u8	reserved8_1;
+	u8	reserved8_2;
+	u8	numlevels_pbl_page_size_log;
+	__le64	pblptr;
+	__le64	va;
+};
+
 /* Bind SQ WQE (40 bytes) */
 struct sq_bind {
 	u8 wqe_type;
@@ -417,6 +496,22 @@ struct sq_bind {
 	#define SQ_BIND_DATA_SFT				    0
 };
 
+/* sq_bind_hdr (size:256b/32B) */
+struct sq_bind_hdr {
+	u8	wqe_type;
+	u8	flags;
+	u8	access_cntl;
+	u8	reserved8_1;
+	u8	mw_type_zero_based;
+	u8	reserved8_2;
+	__le16	reserved16;
+	__le32	parent_l_key;
+	__le32	l_key;
+	__le64	va;
+	u8	length[5];
+	u8	reserved24[3];
+};
+
 /* RQ/SRQ WQE Structures */
 /* RQ/SRQ WQE (40 bytes) */
 struct rq_wqe {
@@ -435,6 +530,17 @@ struct rq_wqe {
 	__le32 data[24];
 };
 
+/* rq_wqe_hdr (size:256b/32B) */
+struct rq_wqe_hdr {
+	u8	wqe_type;
+	u8	flags;
+	u8	wqe_size;
+	u8	reserved8;
+	__le32	reserved32;
+	__le32	wr_id[2];
+	u8	reserved128[16];
+};
+
 /* CQ CQE Structures */
 /* Base CQE (32 bytes) */
 struct cq_base {
diff --git a/drivers/infiniband/hw/cxgb4/Kconfig b/drivers/infiniband/hw/cxgb4/Kconfig
index b49e8d4c3854..9e2b2c348afd 100644
--- a/drivers/infiniband/hw/cxgb4/Kconfig
+++ b/drivers/infiniband/hw/cxgb4/Kconfig
@@ -5,7 +5,7 @@ config INFINIBAND_CXGB4
 	depends on INFINIBAND_ADDR_TRANS
 	select CHELSIO_LIB
 	select GENERIC_ALLOCATOR
-	---help---
+	help
 	  This is an iWARP/RDMA driver for the Chelsio T4 and T5
 	  1GbE, 10GbE adapters and T5 40GbE adapter.
 
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index 599340c1f0b8..541dbcf22d0e 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -953,6 +953,7 @@ void c4iw_dealloc(struct uld_ctx *ctx)
 static void c4iw_remove(struct uld_ctx *ctx)
 {
 	pr_debug("c4iw_dev %p\n", ctx->dev);
+	debugfs_remove_recursive(ctx->dev->debugfs_root);
 	c4iw_unregister_device(ctx->dev);
 	c4iw_dealloc(ctx);
 }
diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h
index aa7396a1588a..1889dd172a25 100644
--- a/drivers/infiniband/hw/efa/efa.h
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
@@ -153,8 +154,7 @@ int efa_mmap(struct ib_ucontext *ibucontext,
 	     struct vm_area_struct *vma);
 void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
 int efa_create_ah(struct ib_ah *ibah,
-		  struct rdma_ah_attr *ah_attr,
-		  u32 flags,
+		  struct rdma_ah_init_attr *init_attr,
 		  struct ib_udata *udata);
 void efa_destroy_ah(struct ib_ah *ibah, u32 flags);
 int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index 96b104ab5415..bef2bd291054 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -37,7 +37,7 @@ enum efa_admin_aq_feature_id {
 	EFA_ADMIN_NETWORK_ATTR                      = 3,
 	EFA_ADMIN_QUEUE_ATTR                        = 4,
 	EFA_ADMIN_HW_HINTS                          = 5,
-	EFA_ADMIN_FEATURES_OPCODE_NUM               = 8,
+	EFA_ADMIN_HOST_INFO                         = 6,
 };
 
 /* QP transport type */
@@ -799,6 +799,54 @@ struct efa_admin_mmio_req_read_less_resp {
 	u32 reg_val;
 };
 
+enum efa_admin_os_type {
+	EFA_ADMIN_OS_LINUX                          = 0,
+};
+
+struct efa_admin_host_info {
+	/* OS distribution string format */
+	u8 os_dist_str[128];
+
+	/* Defined in enum efa_admin_os_type */
+	u32 os_type;
+
+	/* Kernel version string format */
+	u8 kernel_ver_str[32];
+
+	/* Kernel version numeric format */
+	u32 kernel_ver;
+
+	/*
+	 * 7:0 : driver_module_type
+	 * 15:8 : driver_sub_minor
+	 * 23:16 : driver_minor
+	 * 31:24 : driver_major
+	 */
+	u32 driver_ver;
+
+	/*
+	 * Device's Bus, Device and Function
+	 * 2:0 : function
+	 * 7:3 : device
+	 * 15:8 : bus
+	 */
+	u16 bdf;
+
+	/*
+	 * Spec version
+	 * 7:0 : spec_minor
+	 * 15:8 : spec_major
+	 */
+	u16 spec_ver;
+
+	/*
+	 * 0 : intree - Intree driver
+	 * 1 : gdr - GPUDirect RDMA supported
+	 * 31:2 : reserved2
+	 */
+	u32 flags;
+};
+
 /* create_qp_cmd */
 #define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK                BIT(0)
 #define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK                BIT(1)
@@ -820,4 +868,17 @@ struct efa_admin_mmio_req_read_less_resp {
 /* feature_device_attr_desc */
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_READ_MASK   BIT(0)
 
+/* host_info */
+#define EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE_MASK         GENMASK(7, 0)
+#define EFA_ADMIN_HOST_INFO_DRIVER_SUB_MINOR_MASK           GENMASK(15, 8)
+#define EFA_ADMIN_HOST_INFO_DRIVER_MINOR_MASK               GENMASK(23, 16)
+#define EFA_ADMIN_HOST_INFO_DRIVER_MAJOR_MASK               GENMASK(31, 24)
+#define EFA_ADMIN_HOST_INFO_FUNCTION_MASK                   GENMASK(2, 0)
+#define EFA_ADMIN_HOST_INFO_DEVICE_MASK                     GENMASK(7, 3)
+#define EFA_ADMIN_HOST_INFO_BUS_MASK                        GENMASK(15, 8)
+#define EFA_ADMIN_HOST_INFO_SPEC_MINOR_MASK                 GENMASK(7, 0)
+#define EFA_ADMIN_HOST_INFO_SPEC_MAJOR_MASK                 GENMASK(15, 8)
+#define EFA_ADMIN_HOST_INFO_INTREE_MASK                     BIT(0)
+#define EFA_ADMIN_HOST_INFO_GDR_MASK                        BIT(1)
+
 #endif /* _EFA_ADMIN_CMDS_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
index 7fce69f5568f..336bc2c57bb1 100644
--- a/drivers/infiniband/hw/efa/efa_com.c
+++ b/drivers/infiniband/hw/efa/efa_com.c
@@ -631,17 +631,20 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
 			cmd->aq_common_descriptor.opcode, PTR_ERR(comp_ctx));
 
 		up(&aq->avail_cmds);
+		atomic64_inc(&aq->stats.cmd_err);
 		return PTR_ERR(comp_ctx);
 	}
 
 	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);
-	if (err)
+	if (err) {
 		ibdev_err_ratelimited(
 			aq->efa_dev,
 			"Failed to process command %s (opcode %u) comp_status %d err %d\n",
 			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
 			cmd->aq_common_descriptor.opcode, comp_ctx->comp_status,
 			err);
+		atomic64_inc(&aq->stats.cmd_err);
+	}
 
 	up(&aq->avail_cmds);
 
diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h
index c67dd8109d1c..5e4c88877ddb 100644
--- a/drivers/infiniband/hw/efa/efa_com.h
+++ b/drivers/infiniband/hw/efa/efa_com.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_COM_H_
@@ -47,6 +47,7 @@ struct efa_com_admin_sq {
 struct efa_com_stats_admin {
 	atomic64_t submitted_cmd;
 	atomic64_t completed_cmd;
+	atomic64_t cmd_err;
 	atomic64_t no_completion;
 };
 
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index eea5574a62e8..fabd8df2e78f 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -351,7 +351,7 @@ int efa_com_destroy_ah(struct efa_com_dev *edev,
 	return 0;
 }
 
-static bool
+bool
 efa_com_check_supported_feature_id(struct efa_com_dev *edev,
 				   enum efa_admin_aq_feature_id feature_id)
 {
@@ -388,7 +388,7 @@ static int efa_com_get_feature_ex(struct efa_com_dev *edev,
 
 	if (control_buff_size)
 		EFA_SET(&get_cmd.aq_common_descriptor.flags,
-			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT, 1);
+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA, 1);
 
 	efa_com_set_dma_addr(control_buf_dma_addr,
 			     &get_cmd.control_buffer.address.mem_addr_high,
@@ -517,12 +517,12 @@ int efa_com_get_hw_hints(struct efa_com_dev *edev,
 	return 0;
 }
 
-static int efa_com_set_feature_ex(struct efa_com_dev *edev,
-				  struct efa_admin_set_feature_resp *set_resp,
-				  struct efa_admin_set_feature_cmd *set_cmd,
-				  enum efa_admin_aq_feature_id feature_id,
-				  dma_addr_t control_buf_dma_addr,
-				  u32 control_buff_size)
+int efa_com_set_feature_ex(struct efa_com_dev *edev,
+			   struct efa_admin_set_feature_resp *set_resp,
+			   struct efa_admin_set_feature_cmd *set_cmd,
+			   enum efa_admin_aq_feature_id feature_id,
+			   dma_addr_t control_buf_dma_addr,
+			   u32 control_buff_size)
 {
 	struct efa_com_admin_queue *aq;
 	int err;
@@ -540,7 +540,7 @@ static int efa_com_set_feature_ex(struct efa_com_dev *edev,
 	if (control_buff_size) {
 		set_cmd->aq_common_descriptor.flags = 0;
 		EFA_SET(&set_cmd->aq_common_descriptor.flags,
-			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT, 1);
+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA, 1);
 		efa_com_set_dma_addr(control_buf_dma_addr,
 				     &set_cmd->control_buffer.address.mem_addr_high,
 				     &set_cmd->control_buffer.address.mem_addr_low);
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index 31db5a0cbd5b..41ce4a476ee6 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_COM_CMD_H_
@@ -270,6 +270,15 @@ int efa_com_get_device_attr(struct efa_com_dev *edev,
 			    struct efa_com_get_device_attr_result *result);
 int efa_com_get_hw_hints(struct efa_com_dev *edev,
 			 struct efa_com_get_hw_hints_result *result);
+bool
+efa_com_check_supported_feature_id(struct efa_com_dev *edev,
+				   enum efa_admin_aq_feature_id feature_id);
+int efa_com_set_feature_ex(struct efa_com_dev *edev,
+			   struct efa_admin_set_feature_resp *set_resp,
+			   struct efa_admin_set_feature_cmd *set_cmd,
+			   enum efa_admin_aq_feature_id feature_id,
+			   dma_addr_t control_buf_dma_addr,
+			   u32 control_buff_size);
 int efa_com_set_aenq_config(struct efa_com_dev *edev, u32 groups);
 int efa_com_alloc_pd(struct efa_com_dev *edev,
 		     struct efa_com_alloc_pd_result *result);
diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index faf3ff1bca2a..82145574c928 100644
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -1,10 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/utsname.h>
+#include <linux/version.h>
 
 #include <rdma/ib_user_verbs.h>
 
@@ -187,6 +189,52 @@ static void efa_stats_init(struct efa_dev *dev)
 		atomic64_set(s, 0);
 }
 
+static void efa_set_host_info(struct efa_dev *dev)
+{
+	struct efa_admin_set_feature_resp resp = {};
+	struct efa_admin_set_feature_cmd cmd = {};
+	struct efa_admin_host_info *hinf;
+	u32 bufsz = sizeof(*hinf);
+	dma_addr_t hinf_dma;
+
+	if (!efa_com_check_supported_feature_id(&dev->edev,
+						EFA_ADMIN_HOST_INFO))
+		return;
+
+	/* Failures in host info set shall not disturb probe */
+	hinf = dma_alloc_coherent(&dev->pdev->dev, bufsz, &hinf_dma,
+				  GFP_KERNEL);
+	if (!hinf)
+		return;
+
+	strlcpy(hinf->os_dist_str, utsname()->release,
+		min(sizeof(hinf->os_dist_str), sizeof(utsname()->release)));
+	hinf->os_type = EFA_ADMIN_OS_LINUX;
+	strlcpy(hinf->kernel_ver_str, utsname()->version,
+		min(sizeof(hinf->kernel_ver_str), sizeof(utsname()->version)));
+	hinf->kernel_ver = LINUX_VERSION_CODE;
+	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MAJOR, 0);
+	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MINOR, 0);
+	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_SUB_MINOR, 0);
+	EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE, 0);
+	EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_BUS, dev->pdev->bus->number);
+	EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_DEVICE,
+		PCI_SLOT(dev->pdev->devfn));
+	EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_FUNCTION,
+		PCI_FUNC(dev->pdev->devfn));
+	EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MAJOR,
+		EFA_COMMON_SPEC_VERSION_MAJOR);
+	EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MINOR,
+		EFA_COMMON_SPEC_VERSION_MINOR);
+	EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_INTREE, 1);
+	EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_GDR, 0);
+
+	efa_com_set_feature_ex(&dev->edev, &resp, &cmd, EFA_ADMIN_HOST_INFO,
+			       hinf_dma, bufsz);
+
+	dma_free_coherent(&dev->pdev->dev, bufsz, hinf, hinf_dma);
+}
+
 static const struct ib_device_ops efa_dev_ops = {
 	.owner = THIS_MODULE,
 	.driver_id = RDMA_DRIVER_EFA,
@@ -251,6 +299,8 @@ static int efa_ib_device_add(struct efa_dev *dev)
 	if (err)
 		goto err_release_doorbell_bar;
 
+	efa_set_host_info(dev);
+
 	dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED;
 	dev->ibdev.phys_port_cnt = 1;
 	dev->ibdev.num_comp_vectors = 1;
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 5c57098a4aee..08313f7c73bc 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -37,13 +37,16 @@ struct efa_user_mmap_entry {
 	op(EFA_RX_DROPS, "rx_drops") \
 	op(EFA_SUBMITTED_CMDS, "submitted_cmds") \
 	op(EFA_COMPLETED_CMDS, "completed_cmds") \
+	op(EFA_CMDS_ERR, "cmds_err") \
 	op(EFA_NO_COMPLETION_CMDS, "no_completion_cmds") \
 	op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \
 	op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \
 	op(EFA_CREATE_QP_ERR, "create_qp_err") \
+	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1568,6 +1571,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1603,12 +1607,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;
 	}
 
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
+	}
 
 	rdma_user_mmap_entry_put(rdma_entry);
 	return err;
@@ -1639,10 +1645,10 @@ static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah)
 }
 
 int efa_create_ah(struct ib_ah *ibah,
-		  struct rdma_ah_attr *ah_attr,
-		  u32 flags,
+		  struct rdma_ah_init_attr *init_attr,
 		  struct ib_udata *udata)
 {
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 	struct efa_dev *dev = to_edev(ibah->device);
 	struct efa_com_create_ah_params params = {};
 	struct efa_ibv_create_ah_resp resp = {};
@@ -1650,7 +1656,7 @@ int efa_create_ah(struct ib_ah *ibah,
 	struct efa_ah *ah = to_eah(ibah);
 	int err;
 
-	if (!(flags & RDMA_CREATE_AH_SLEEPABLE)) {
+	if (!(init_attr->flags & RDMA_CREATE_AH_SLEEPABLE)) {
 		ibdev_dbg(&dev->ibdev,
 			  "Create address handle is not supported in atomic context\n");
 		err = -EOPNOTSUPP;
@@ -1747,15 +1753,18 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
 	as = &dev->edev.aq.stats;
 	stats->value[EFA_SUBMITTED_CMDS] = atomic64_read(&as->submitted_cmd);
 	stats->value[EFA_COMPLETED_CMDS] = atomic64_read(&as->completed_cmd);
+	stats->value[EFA_CMDS_ERR] = atomic64_read(&as->cmd_err);
 	stats->value[EFA_NO_COMPLETION_CMDS] = atomic64_read(&as->no_completion);
 
 	s = &dev->stats;
 	stats->value[EFA_KEEP_ALIVE_RCVD] = atomic64_read(&s->keep_alive_rcvd);
 	stats->value[EFA_ALLOC_PD_ERR] = atomic64_read(&s->sw_stats.alloc_pd_err);
 	stats->value[EFA_CREATE_QP_ERR] = atomic64_read(&s->sw_stats.create_qp_err);
+	stats->value[EFA_CREATE_CQ_ERR] = atomic64_read(&s->sw_stats.create_cq_err);
 	stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->sw_stats.reg_mr_err);
 	stats->value[EFA_ALLOC_UCONTEXT_ERR] = atomic64_read(&s->sw_stats.alloc_ucontext_err);
 	stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->sw_stats.create_ah_err);
+	stats->value[EFA_MMAP_ERR] = atomic64_read(&s->sw_stats.mmap_err);
 
 	return ARRAY_SIZE(efa_stats_names);
 }
diff --git a/drivers/infiniband/hw/hfi1/Kconfig b/drivers/infiniband/hw/hfi1/Kconfig
index 0653f4f7b26c..519866b30a13 100644
--- a/drivers/infiniband/hw/hfi1/Kconfig
+++ b/drivers/infiniband/hw/hfi1/Kconfig
@@ -5,19 +5,19 @@ config INFINIBAND_HFI1
 	select MMU_NOTIFIER
 	select CRC32
 	select I2C_ALGOBIT
-	---help---
+	help
 	This is a low-level driver for Intel OPA Gen1 adapter.
 config HFI1_DEBUG_SDMA_ORDER
 	bool "HFI1 SDMA Order debug"
 	depends on INFINIBAND_HFI1
 	default n
-	---help---
+	help
 	This is a debug flag to test for out of order
 	sdma completions for unit testing
 config SDMA_VERBOSITY
 	bool "Config SDMA Verbosity"
 	depends on INFINIBAND_HFI1
 	default n
-	---help---
+	help
 	This is a configuration flag to enable verbose
 	SDMA debug
diff --git a/drivers/infiniband/hw/hfi1/Makefile b/drivers/infiniband/hw/hfi1/Makefile
index 0405d26d0833..2e89ec10efed 100644
--- a/drivers/infiniband/hw/hfi1/Makefile
+++ b/drivers/infiniband/hw/hfi1/Makefile
@@ -22,9 +22,13 @@ hfi1-y := \
 	init.o \
 	intr.o \
 	iowait.o \
+	ipoib_main.o \
+	ipoib_rx.o \
+	ipoib_tx.o \
 	mad.o \
 	mmu_rb.o \
 	msix.o \
+	netdev_rx.o \
 	opfn.o \
 	pcie.o \
 	pio.o \
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c
index 1aeea5d65c01..2a91b8d95e12 100644
--- a/drivers/infiniband/hw/hfi1/affinity.c
+++ b/drivers/infiniband/hw/hfi1/affinity.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -64,6 +64,7 @@ struct hfi1_affinity_node_list node_affinity = {
 static const char * const irq_type_names[] = {
 	"SDMA",
 	"RCVCTXT",
+	"NETDEVCTXT",
 	"GENERAL",
 	"OTHER",
 };
@@ -915,6 +916,11 @@ static int get_irq_affinity(struct hfi1_devdata *dd,
 			set = &entry->rcv_intr;
 		scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
 		break;
+	case IRQ_NETDEVCTXT:
+		rcd = (struct hfi1_ctxtdata *)msix->arg;
+		set = &entry->def_intr;
+		scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
+		break;
 	default:
 		dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
 		return -EINVAL;
@@ -987,6 +993,10 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
 		if (rcd->ctxt != HFI1_CTRL_CTXT)
 			set = &entry->rcv_intr;
 		break;
+	case IRQ_NETDEVCTXT:
+		rcd = (struct hfi1_ctxtdata *)msix->arg;
+		set = &entry->def_intr;
+		break;
 	default:
 		mutex_unlock(&node_affinity.lock);
 		return;
diff --git a/drivers/infiniband/hw/hfi1/affinity.h b/drivers/infiniband/hw/hfi1/affinity.h
index 6a7e6ea4e426..f94ed5d7c7a3 100644
--- a/drivers/infiniband/hw/hfi1/affinity.h
+++ b/drivers/infiniband/hw/hfi1/affinity.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -52,6 +52,7 @@
 enum irq_type {
 	IRQ_SDMA,
 	IRQ_RCVCTXT,
+	IRQ_NETDEVCTXT,
 	IRQ_GENERAL,
 	IRQ_OTHER
 };
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index e0b1238d31df..15f9c635f292 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -66,10 +66,7 @@
 #include "affinity.h"
 #include "debugfs.h"
 #include "fault.h"
-
-uint kdeth_qp;
-module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
-MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
+#include "netdev.h"
 
 uint num_vls = HFI1_MAX_VLS_SUPPORTED;
 module_param(num_vls, uint, S_IRUGO);
@@ -128,13 +125,15 @@ struct flag_table {
 
 /*
  * RSM instance allocation
- *   0 - Verbs
- *   1 - User Fecn Handling
- *   2 - Vnic
+ *   0 - User Fecn Handling
+ *   1 - Vnic
+ *   2 - AIP
+ *   3 - Verbs
  */
-#define RSM_INS_VERBS             0
-#define RSM_INS_FECN              1
-#define RSM_INS_VNIC              2
+#define RSM_INS_FECN              0
+#define RSM_INS_VNIC              1
+#define RSM_INS_AIP               2
+#define RSM_INS_VERBS             3
 
 /* Bit offset into the GUID which carries HFI id information */
 #define GUID_HFI_INDEX_SHIFT     39
@@ -175,6 +174,25 @@ struct flag_table {
 /* QPN[m+n:1] QW 1, OFFSET 1 */
 #define QPN_SELECT_OFFSET      ((1ull << QW_SHIFT) | (1ull))
 
+/* RSM fields for AIP */
+/* LRH.BTH above is reused for this rule */
+
+/* BTH.DESTQP: QW 1, OFFSET 16 for match */
+#define BTH_DESTQP_QW           1ull
+#define BTH_DESTQP_BIT_OFFSET   16ull
+#define BTH_DESTQP_OFFSET(off) ((BTH_DESTQP_QW << QW_SHIFT) | (off))
+#define BTH_DESTQP_MATCH_OFFSET BTH_DESTQP_OFFSET(BTH_DESTQP_BIT_OFFSET)
+#define BTH_DESTQP_MASK         0xFFull
+#define BTH_DESTQP_VALUE        0x81ull
+
+/* DETH.SQPN: QW 1 Offset 56 for select */
+/* We use 8 most significant Soure QPN bits as entropy fpr AIP */
+#define DETH_AIP_SQPN_QW 3ull
+#define DETH_AIP_SQPN_BIT_OFFSET 56ull
+#define DETH_AIP_SQPN_OFFSET(off) ((DETH_AIP_SQPN_QW << QW_SHIFT) | (off))
+#define DETH_AIP_SQPN_SELECT_OFFSET \
+	DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET)
+
 /* RSM fields for Vnic */
 /* L2_TYPE: QW 0, OFFSET 61 - for match */
 #define L2_TYPE_QW             0ull
@@ -8463,6 +8481,49 @@ static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd)
 	local_irq_restore(flags);
 }
 
+/**
+ * hfi1_netdev_rx_napi - napi poll function to move eoi inline
+ * @napi - pointer to napi object
+ * @budget - netdev budget
+ */
+int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget)
+{
+	struct hfi1_netdev_rxq *rxq = container_of(napi,
+			struct hfi1_netdev_rxq, napi);
+	struct hfi1_ctxtdata *rcd = rxq->rcd;
+	int work_done = 0;
+
+	work_done = rcd->do_interrupt(rcd, budget);
+
+	if (work_done < budget) {
+		napi_complete_done(napi, work_done);
+		hfi1_rcd_eoi_intr(rcd);
+	}
+
+	return work_done;
+}
+
+/* Receive packet napi handler for netdevs VNIC and AIP  */
+irqreturn_t receive_context_interrupt_napi(int irq, void *data)
+{
+	struct hfi1_ctxtdata *rcd = data;
+
+	receive_interrupt_common(rcd);
+
+	if (likely(rcd->napi)) {
+		if (likely(napi_schedule_prep(rcd->napi)))
+			__napi_schedule_irqoff(rcd->napi);
+		else
+			__hfi1_rcd_eoi_intr(rcd);
+	} else {
+		WARN_ONCE(1, "Napi IRQ handler without napi set up ctxt=%d\n",
+			  rcd->ctxt);
+		__hfi1_rcd_eoi_intr(rcd);
+	}
+
+	return IRQ_HANDLED;
+}
+
 /*
  * Receive packet IRQ handler.  This routine expects to be on its own IRQ.
  * This routine will try to handle packets immediately (latency), but if
@@ -13330,13 +13391,12 @@ static int set_up_interrupts(struct hfi1_devdata *dd)
  *                             in array of contexts
  *	freectxts  - number of free user contexts
  *	num_send_contexts - number of PIO send contexts being used
- *	num_vnic_contexts - number of contexts reserved for VNIC
+ *	num_netdev_contexts - number of contexts reserved for netdev
  */
 static int set_up_context_variables(struct hfi1_devdata *dd)
 {
 	unsigned long num_kernel_contexts;
-	u16 num_vnic_contexts = HFI1_NUM_VNIC_CTXT;
-	int total_contexts;
+	u16 num_netdev_contexts;
 	int ret;
 	unsigned ngroups;
 	int rmt_count;
@@ -13373,13 +13433,6 @@ static int set_up_context_variables(struct hfi1_devdata *dd)
 		num_kernel_contexts = send_contexts - num_vls - 1;
 	}
 
-	/* Accommodate VNIC contexts if possible */
-	if ((num_kernel_contexts + num_vnic_contexts) > rcv_contexts) {
-		dd_dev_err(dd, "No receive contexts available for VNIC\n");
-		num_vnic_contexts = 0;
-	}
-	total_contexts = num_kernel_contexts + num_vnic_contexts;
-
 	/*
 	 * User contexts:
 	 *	- default to 1 user context per real (non-HT) CPU core if
@@ -13392,28 +13445,32 @@ static int set_up_context_variables(struct hfi1_devdata *dd)
 	/*
 	 * Adjust the counts given a global max.
 	 */
-	if (total_contexts + n_usr_ctxts > rcv_contexts) {
+	if (num_kernel_contexts + n_usr_ctxts > rcv_contexts) {
 		dd_dev_err(dd,
-			   "Reducing # user receive contexts to: %d, from %u\n",
-			   rcv_contexts - total_contexts,
+			   "Reducing # user receive contexts to: %u, from %u\n",
+			   (u32)(rcv_contexts - num_kernel_contexts),
 			   n_usr_ctxts);
 		/* recalculate */
-		n_usr_ctxts = rcv_contexts - total_contexts;
+		n_usr_ctxts = rcv_contexts - num_kernel_contexts;
 	}
 
+	num_netdev_contexts =
+		hfi1_num_netdev_contexts(dd, rcv_contexts -
+					 (num_kernel_contexts + n_usr_ctxts),
+					 &node_affinity.real_cpu_mask);
 	/*
 	 * The RMT entries are currently allocated as shown below:
 	 * 1. QOS (0 to 128 entries);
 	 * 2. FECN (num_kernel_context - 1 + num_user_contexts +
-	 *    num_vnic_contexts);
-	 * 3. VNIC (num_vnic_contexts).
-	 * It should be noted that FECN oversubscribe num_vnic_contexts
-	 * entries of RMT because both VNIC and PSM could allocate any receive
+	 *    num_netdev_contexts);
+	 * 3. netdev (num_netdev_contexts).
+	 * It should be noted that FECN oversubscribe num_netdev_contexts
+	 * entries of RMT because both netdev and PSM could allocate any receive
 	 * context between dd->first_dyn_alloc_text and dd->num_rcv_contexts,
 	 * and PSM FECN must reserve an RMT entry for each possible PSM receive
 	 * context.
 	 */
-	rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_vnic_contexts * 2);
+	rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_netdev_contexts * 2);
 	if (HFI1_CAP_IS_KSET(TID_RDMA))
 		rmt_count += num_kernel_contexts - 1;
 	if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) {
@@ -13426,21 +13483,20 @@ static int set_up_context_variables(struct hfi1_devdata *dd)
 		n_usr_ctxts = user_rmt_reduced;
 	}
 
-	total_contexts += n_usr_ctxts;
-
-	/* the first N are kernel contexts, the rest are user/vnic contexts */
-	dd->num_rcv_contexts = total_contexts;
+	/* the first N are kernel contexts, the rest are user/netdev contexts */
+	dd->num_rcv_contexts =
+		num_kernel_contexts + n_usr_ctxts + num_netdev_contexts;
 	dd->n_krcv_queues = num_kernel_contexts;
 	dd->first_dyn_alloc_ctxt = num_kernel_contexts;
-	dd->num_vnic_contexts = num_vnic_contexts;
+	dd->num_netdev_contexts = num_netdev_contexts;
 	dd->num_user_contexts = n_usr_ctxts;
 	dd->freectxts = n_usr_ctxts;
 	dd_dev_info(dd,
-		    "rcv contexts: chip %d, used %d (kernel %d, vnic %u, user %u)\n",
+		    "rcv contexts: chip %d, used %d (kernel %d, netdev %u, user %u)\n",
 		    rcv_contexts,
 		    (int)dd->num_rcv_contexts,
 		    (int)dd->n_krcv_queues,
-		    dd->num_vnic_contexts,
+		    dd->num_netdev_contexts,
 		    dd->num_user_contexts);
 
 	/*
@@ -14119,21 +14175,12 @@ static void init_early_variables(struct hfi1_devdata *dd)
 
 static void init_kdeth_qp(struct hfi1_devdata *dd)
 {
-	/* user changed the KDETH_QP */
-	if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
-		/* out of range or illegal value */
-		dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
-		kdeth_qp = 0;
-	}
-	if (kdeth_qp == 0)	/* not set, or failed range check */
-		kdeth_qp = DEFAULT_KDETH_QP;
-
 	write_csr(dd, SEND_BTH_QP,
-		  (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) <<
+		  (RVT_KDETH_QP_PREFIX & SEND_BTH_QP_KDETH_QP_MASK) <<
 		  SEND_BTH_QP_KDETH_QP_SHIFT);
 
 	write_csr(dd, RCV_BTH_QP,
-		  (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) <<
+		  (RVT_KDETH_QP_PREFIX & RCV_BTH_QP_KDETH_QP_MASK) <<
 		  RCV_BTH_QP_KDETH_QP_SHIFT);
 }
 
@@ -14249,6 +14296,12 @@ static void complete_rsm_map_table(struct hfi1_devdata *dd,
 	}
 }
 
+/* Is a receive side mapping rule */
+static bool has_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)
+{
+	return read_csr(dd, RCV_RSM_CFG + (8 * rule_index)) != 0;
+}
+
 /*
  * Add a receive side mapping rule.
  */
@@ -14485,77 +14538,138 @@ static void init_fecn_handling(struct hfi1_devdata *dd,
 	rmt->used += total_cnt;
 }
 
-/* Initialize RSM for VNIC */
-void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)
+static inline bool hfi1_is_rmt_full(int start, int spare)
+{
+	return (start + spare) > NUM_MAP_ENTRIES;
+}
+
+static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd)
 {
 	u8 i, j;
 	u8 ctx_id = 0;
 	u64 reg;
 	u32 regoff;
-	struct rsm_rule_data rrd;
+	int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
+	int ctxt_count = hfi1_netdev_ctxt_count(dd);
 
-	if (hfi1_vnic_is_rsm_full(dd, NUM_VNIC_MAP_ENTRIES)) {
-		dd_dev_err(dd, "Vnic RSM disabled, rmt entries used = %d\n",
-			   dd->vnic.rmt_start);
-		return;
+	/* We already have contexts mapped in RMT */
+	if (has_rsm_rule(dd, RSM_INS_VNIC) || has_rsm_rule(dd, RSM_INS_AIP)) {
+		dd_dev_info(dd, "Contexts are already mapped in RMT\n");
+		return true;
+	}
+
+	if (hfi1_is_rmt_full(rmt_start, NUM_NETDEV_MAP_ENTRIES)) {
+		dd_dev_err(dd, "Not enough RMT entries used = %d\n",
+			   rmt_start);
+		return false;
 	}
 
-	dev_dbg(&(dd)->pcidev->dev, "Vnic rsm start = %d, end %d\n",
-		dd->vnic.rmt_start,
-		dd->vnic.rmt_start + NUM_VNIC_MAP_ENTRIES);
+	dev_dbg(&(dd)->pcidev->dev, "RMT start = %d, end %d\n",
+		rmt_start,
+		rmt_start + NUM_NETDEV_MAP_ENTRIES);
 
 	/* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */
-	regoff = RCV_RSM_MAP_TABLE + (dd->vnic.rmt_start / 8) * 8;
+	regoff = RCV_RSM_MAP_TABLE + (rmt_start / 8) * 8;
 	reg = read_csr(dd, regoff);
-	for (i = 0; i < NUM_VNIC_MAP_ENTRIES; i++) {
-		/* Update map register with vnic context */
-		j = (dd->vnic.rmt_start + i) % 8;
+	for (i = 0; i < NUM_NETDEV_MAP_ENTRIES; i++) {
+		/* Update map register with netdev context */
+		j = (rmt_start + i) % 8;
 		reg &= ~(0xffllu << (j * 8));
-		reg |= (u64)dd->vnic.ctxt[ctx_id++]->ctxt << (j * 8);
-		/* Wrap up vnic ctx index */
-		ctx_id %= dd->vnic.num_ctxt;
+		reg |= (u64)hfi1_netdev_get_ctxt(dd, ctx_id++)->ctxt << (j * 8);
+		/* Wrap up netdev ctx index */
+		ctx_id %= ctxt_count;
 		/* Write back map register */
-		if (j == 7 || ((i + 1) == NUM_VNIC_MAP_ENTRIES)) {
+		if (j == 7 || ((i + 1) == NUM_NETDEV_MAP_ENTRIES)) {
 			dev_dbg(&(dd)->pcidev->dev,
-				"Vnic rsm map reg[%d] =0x%llx\n",
+				"RMT[%d] =0x%llx\n",
 				regoff - RCV_RSM_MAP_TABLE, reg);
 
 			write_csr(dd, regoff, reg);
 			regoff += 8;
-			if (i < (NUM_VNIC_MAP_ENTRIES - 1))
+			if (i < (NUM_NETDEV_MAP_ENTRIES - 1))
 				reg = read_csr(dd, regoff);
 		}
 	}
 
-	/* Add rule for vnic */
-	rrd.offset = dd->vnic.rmt_start;
-	rrd.pkt_type = 4;
-	/* Match 16B packets */
-	rrd.field1_off = L2_TYPE_MATCH_OFFSET;
-	rrd.mask1 = L2_TYPE_MASK;
-	rrd.value1 = L2_16B_VALUE;
-	/* Match ETH L4 packets */
-	rrd.field2_off = L4_TYPE_MATCH_OFFSET;
-	rrd.mask2 = L4_16B_TYPE_MASK;
-	rrd.value2 = L4_16B_ETH_VALUE;
-	/* Calc context from veswid and entropy */
-	rrd.index1_off = L4_16B_HDR_VESWID_OFFSET;
-	rrd.index1_width = ilog2(NUM_VNIC_MAP_ENTRIES);
-	rrd.index2_off = L2_16B_ENTROPY_OFFSET;
-	rrd.index2_width = ilog2(NUM_VNIC_MAP_ENTRIES);
-	add_rsm_rule(dd, RSM_INS_VNIC, &rrd);
-
-	/* Enable RSM if not already enabled */
+	return true;
+}
+
+static void hfi1_enable_rsm_rule(struct hfi1_devdata *dd,
+				 int rule, struct rsm_rule_data *rrd)
+{
+	if (!hfi1_netdev_update_rmt(dd)) {
+		dd_dev_err(dd, "Failed to update RMT for RSM%d rule\n", rule);
+		return;
+	}
+
+	add_rsm_rule(dd, rule, rrd);
 	add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
 }
 
+void hfi1_init_aip_rsm(struct hfi1_devdata *dd)
+{
+	/*
+	 * go through with the initialisation only if this rule actually doesn't
+	 * exist yet
+	 */
+	if (atomic_fetch_inc(&dd->ipoib_rsm_usr_num) == 0) {
+		int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
+		struct rsm_rule_data rrd = {
+			.offset = rmt_start,
+			.pkt_type = IB_PACKET_TYPE,
+			.field1_off = LRH_BTH_MATCH_OFFSET,
+			.mask1 = LRH_BTH_MASK,
+			.value1 = LRH_BTH_VALUE,
+			.field2_off = BTH_DESTQP_MATCH_OFFSET,
+			.mask2 = BTH_DESTQP_MASK,
+			.value2 = BTH_DESTQP_VALUE,
+			.index1_off = DETH_AIP_SQPN_SELECT_OFFSET +
+					ilog2(NUM_NETDEV_MAP_ENTRIES),
+			.index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),
+			.index2_off = DETH_AIP_SQPN_SELECT_OFFSET,
+			.index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)
+		};
+
+		hfi1_enable_rsm_rule(dd, RSM_INS_AIP, &rrd);
+	}
+}
+
+/* Initialize RSM for VNIC */
+void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)
+{
+	int rmt_start = hfi1_netdev_get_free_rmt_idx(dd);
+	struct rsm_rule_data rrd = {
+		/* Add rule for vnic */
+		.offset = rmt_start,
+		.pkt_type = 4,
+		/* Match 16B packets */
+		.field1_off = L2_TYPE_MATCH_OFFSET,
+		.mask1 = L2_TYPE_MASK,
+		.value1 = L2_16B_VALUE,
+		/* Match ETH L4 packets */
+		.field2_off = L4_TYPE_MATCH_OFFSET,
+		.mask2 = L4_16B_TYPE_MASK,
+		.value2 = L4_16B_ETH_VALUE,
+		/* Calc context from veswid and entropy */
+		.index1_off = L4_16B_HDR_VESWID_OFFSET,
+		.index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES),
+		.index2_off = L2_16B_ENTROPY_OFFSET,
+		.index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES)
+	};
+
+	hfi1_enable_rsm_rule(dd, RSM_INS_VNIC, &rrd);
+}
+
 void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)
 {
 	clear_rsm_rule(dd, RSM_INS_VNIC);
+}
 
-	/* Disable RSM if used only by vnic */
-	if (dd->vnic.rmt_start == 0)
-		clear_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
+void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd)
+{
+	/* only actually clear the rule if it's the last user asking to do so */
+	if (atomic_fetch_add_unless(&dd->ipoib_rsm_usr_num, -1, 0) == 1)
+		clear_rsm_rule(dd, RSM_INS_AIP);
 }
 
 static int init_rxe(struct hfi1_devdata *dd)
@@ -14574,8 +14688,8 @@ static int init_rxe(struct hfi1_devdata *dd)
 	init_qos(dd, rmt);
 	init_fecn_handling(dd, rmt);
 	complete_rsm_map_table(dd, rmt);
-	/* record number of used rsm map entries for vnic */
-	dd->vnic.rmt_start = rmt->used;
+	/* record number of used rsm map entries for netdev */
+	hfi1_netdev_set_free_rmt_idx(dd, rmt->used);
 	kfree(rmt);
 
 	/*
@@ -15129,6 +15243,10 @@ int hfi1_init_dd(struct hfi1_devdata *dd)
 		 (dd->revision >> CCE_REVISION_SW_SHIFT)
 		    & CCE_REVISION_SW_MASK);
 
+	/* alloc netdev data */
+	if (hfi1_netdev_alloc(dd))
+		goto bail_cleanup;
+
 	ret = set_up_context_variables(dd);
 	if (ret)
 		goto bail_cleanup;
@@ -15229,6 +15347,7 @@ bail_clear_intr:
 	hfi1_comp_vectors_clean_up(dd);
 	msix_clean_up_interrupts(dd);
 bail_cleanup:
+	hfi1_netdev_free(dd);
 	hfi1_pcie_ddcleanup(dd);
 bail_free:
 	hfi1_free_devdata(dd);
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index 725509261016..2c6f2de74d4d 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -1,7 +1,7 @@
 #ifndef _CHIP_H
 #define _CHIP_H
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -1447,6 +1447,7 @@ irqreturn_t general_interrupt(int irq, void *data);
 irqreturn_t sdma_interrupt(int irq, void *data);
 irqreturn_t receive_context_interrupt(int irq, void *data);
 irqreturn_t receive_context_thread(int irq, void *data);
+irqreturn_t receive_context_interrupt_napi(int irq, void *data);
 
 int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set);
 void init_qsfp_int(struct hfi1_devdata *dd);
@@ -1455,6 +1456,8 @@ void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr);
 void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr);
 void reset_interrupts(struct hfi1_devdata *dd);
 u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx);
+void hfi1_init_aip_rsm(struct hfi1_devdata *dd);
+void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd);
 
 /*
  * Interrupt source table.
diff --git a/drivers/infiniband/hw/hfi1/common.h b/drivers/infiniband/hw/hfi1/common.h
index 40a1ff0c8a8e..ff423e546b80 100644
--- a/drivers/infiniband/hw/hfi1/common.h
+++ b/drivers/infiniband/hw/hfi1/common.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -72,13 +72,6 @@
  * compilation unit
  */
 
-/*
- * If a packet's QP[23:16] bits match this value, then it is
- * a PSM packet and the hardware will expect a KDETH header
- * following the BTH.
- */
-#define DEFAULT_KDETH_QP 0x80
-
 /* driver/hw feature set bitmask */
 #define HFI1_CAP_USER_SHIFT      24
 #define HFI1_CAP_MASK            ((1UL << HFI1_CAP_USER_SHIFT) - 1)
@@ -149,7 +142,8 @@
 				   HFI1_CAP_NO_INTEGRITY |		\
 				   HFI1_CAP_PKEY_CHECK |		\
 				   HFI1_CAP_TID_RDMA |			\
-				   HFI1_CAP_OPFN) <<			\
+				   HFI1_CAP_OPFN |			\
+				   HFI1_CAP_AIP) <<			\
 				  HFI1_CAP_USER_SHIFT)
 /*
  * Set of capabilities that need to be enabled for kernel context in
@@ -166,6 +160,7 @@
 				 HFI1_CAP_PKEY_CHECK |			\
 				 HFI1_CAP_MULTI_PKT_EGR |		\
 				 HFI1_CAP_EXTENDED_PSN |		\
+				 HFI1_CAP_AIP |				\
 				 ((HFI1_CAP_HDRSUPP |			\
 				   HFI1_CAP_MULTI_PKT_EGR |		\
 				   HFI1_CAP_STATIC_RATE_CTRL |		\
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index 049d15befe58..a40701a6e1b6 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015-2018 Intel Corporation.
+ * Copyright(c) 2015-2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -54,6 +54,7 @@
 #include <linux/module.h>
 #include <linux/prefetch.h>
 #include <rdma/ib_verbs.h>
+#include <linux/etherdevice.h>
 
 #include "hfi.h"
 #include "trace.h"
@@ -63,6 +64,9 @@
 #include "vnic.h"
 #include "fault.h"
 
+#include "ipoib.h"
+#include "netdev.h"
+
 #undef pr_fmt
 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
 
@@ -748,6 +752,39 @@ static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
 	return ret;
 }
 
+static void process_rcv_packet_napi(struct hfi1_packet *packet)
+{
+	packet->etype = rhf_rcv_type(packet->rhf);
+
+	/* total length */
+	packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
+	/* retrieve eager buffer details */
+	packet->etail = rhf_egr_index(packet->rhf);
+	packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
+				  &packet->updegr);
+	/*
+	 * Prefetch the contents of the eager buffer.  It is
+	 * OK to send a negative length to prefetch_range().
+	 * The +2 is the size of the RHF.
+	 */
+	prefetch_range(packet->ebuf,
+		       packet->tlen - ((packet->rcd->rcvhdrqentsize -
+				       (rhf_hdrq_offset(packet->rhf)
+					+ 2)) * 4));
+
+	packet->rcd->rhf_rcv_function_map[packet->etype](packet);
+	packet->numpkt++;
+
+	/* Set up for the next packet */
+	packet->rhqoff += packet->rsize;
+	if (packet->rhqoff >= packet->maxcnt)
+		packet->rhqoff = 0;
+
+	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
+				      packet->rcd->rhf_offset;
+	packet->rhf = rhf_to_cpu(packet->rhf_addr);
+}
+
 static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
 {
 	int ret;
@@ -827,6 +864,36 @@ static inline void finish_packet(struct hfi1_packet *packet)
 }
 
 /*
+ * handle_receive_interrupt_napi_fp - receive a packet
+ * @rcd: the context
+ * @budget: polling budget
+ *
+ * Called from interrupt handler for receive interrupt.
+ * This is the fast path interrupt handler
+ * when executing napi soft irq environment.
+ */
+int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget)
+{
+	struct hfi1_packet packet;
+
+	init_packet(rcd, &packet);
+	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
+		goto bail;
+
+	while (packet.numpkt < budget) {
+		process_rcv_packet_napi(&packet);
+		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
+			break;
+
+		process_rcv_update(0, &packet);
+	}
+	hfi1_set_rcd_head(rcd, packet.rhqoff);
+bail:
+	finish_packet(&packet);
+	return packet.numpkt;
+}
+
+/*
  * Handle receive interrupts when using the no dma rtail option.
  */
 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
@@ -1074,6 +1141,63 @@ bail:
 }
 
 /*
+ * handle_receive_interrupt_napi_sp - receive a packet
+ * @rcd: the context
+ * @budget: polling budget
+ *
+ * Called from interrupt handler for errors or receive interrupt.
+ * This is the slow path interrupt handler
+ * when executing napi soft irq environment.
+ */
+int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget)
+{
+	struct hfi1_devdata *dd = rcd->dd;
+	int last = RCV_PKT_OK;
+	bool needset = true;
+	struct hfi1_packet packet;
+
+	init_packet(rcd, &packet);
+	if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
+		goto bail;
+
+	while (last != RCV_PKT_DONE && packet.numpkt < budget) {
+		if (hfi1_need_drop(dd)) {
+			/* On to the next packet */
+			packet.rhqoff += packet.rsize;
+			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
+					  packet.rhqoff +
+					  rcd->rhf_offset;
+			packet.rhf = rhf_to_cpu(packet.rhf_addr);
+
+		} else {
+			if (set_armed_to_active(&packet))
+				goto bail;
+			process_rcv_packet_napi(&packet);
+		}
+
+		if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
+			last = RCV_PKT_DONE;
+
+		if (needset) {
+			needset = false;
+			set_all_fastpath(dd, rcd);
+		}
+
+		process_rcv_update(last, &packet);
+	}
+
+	hfi1_set_rcd_head(rcd, packet.rhqoff);
+
+bail:
+	/*
+	 * Always write head at end, and setup rcv interrupt, even
+	 * if no packets were processed.
+	 */
+	finish_packet(&packet);
+	return packet.numpkt;
+}
+
+/*
  * We may discover in the interrupt that the hardware link state has
  * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),
  * and we need to update the driver's notion of the link state.  We cannot
@@ -1550,6 +1674,82 @@ void handle_eflags(struct hfi1_packet *packet)
 		show_eflags_errs(packet);
 }
 
+static void hfi1_ipoib_ib_rcv(struct hfi1_packet *packet)
+{
+	struct hfi1_ibport *ibp;
+	struct net_device *netdev;
+	struct hfi1_ctxtdata *rcd = packet->rcd;
+	struct napi_struct *napi = rcd->napi;
+	struct sk_buff *skb;
+	struct hfi1_netdev_rxq *rxq = container_of(napi,
+			struct hfi1_netdev_rxq, napi);
+	u32 extra_bytes;
+	u32 tlen, qpnum;
+	bool do_work, do_cnp;
+	struct hfi1_ipoib_dev_priv *priv;
+
+	trace_hfi1_rcvhdr(packet);
+
+	hfi1_setup_ib_header(packet);
+
+	packet->ohdr = &((struct ib_header *)packet->hdr)->u.oth;
+	packet->grh = NULL;
+
+	if (unlikely(rhf_err_flags(packet->rhf))) {
+		handle_eflags(packet);
+		return;
+	}
+
+	qpnum = ib_bth_get_qpn(packet->ohdr);
+	netdev = hfi1_netdev_get_data(rcd->dd, qpnum);
+	if (!netdev)
+		goto drop_no_nd;
+
+	trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));
+	trace_ctxt_rsm_hist(rcd->ctxt);
+
+	/* handle congestion notifications */
+	do_work = hfi1_may_ecn(packet);
+	if (unlikely(do_work)) {
+		do_cnp = (packet->opcode != IB_OPCODE_CNP);
+		(void)hfi1_process_ecn_slowpath(hfi1_ipoib_priv(netdev)->qp,
+						 packet, do_cnp);
+	}
+
+	/*
+	 * We have split point after last byte of DETH
+	 * lets strip padding and CRC and ICRC.
+	 * tlen is whole packet len so we need to
+	 * subtract header size as well.
+	 */
+	tlen = packet->tlen;
+	extra_bytes = ib_bth_get_pad(packet->ohdr) + (SIZE_OF_CRC << 2) +
+			packet->hlen;
+	if (unlikely(tlen < extra_bytes))
+		goto drop;
+
+	tlen -= extra_bytes;
+
+	skb = hfi1_ipoib_prepare_skb(rxq, tlen, packet->ebuf);
+	if (unlikely(!skb))
+		goto drop;
+
+	priv = hfi1_ipoib_priv(netdev);
+	hfi1_ipoib_update_rx_netstats(priv, 1, skb->len);
+
+	skb->dev = netdev;
+	skb->pkt_type = PACKET_HOST;
+	netif_receive_skb(skb);
+
+	return;
+
+drop:
+	++netdev->stats.rx_dropped;
+drop_no_nd:
+	ibp = rcd_to_iport(packet->rcd);
+	++ibp->rvp.n_pkt_drops;
+}
+
 /*
  * The following functions are called by the interrupt handler. They are type
  * specific handlers for each packet type.
@@ -1572,28 +1772,10 @@ static void process_receive_ib(struct hfi1_packet *packet)
 	hfi1_ib_rcv(packet);
 }
 
-static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet)
-{
-	/* Packet received in VNIC context via RSM */
-	if (packet->rcd->is_vnic)
-		return true;
-
-	if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) &&
-	    (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR))
-		return true;
-
-	return false;
-}
-
 static void process_receive_bypass(struct hfi1_packet *packet)
 {
 	struct hfi1_devdata *dd = packet->rcd->dd;
 
-	if (hfi1_is_vnic_packet(packet)) {
-		hfi1_vnic_bypass_rcv(packet);
-		return;
-	}
-
 	if (hfi1_setup_bypass_packet(packet))
 		return;
 
@@ -1757,3 +1939,14 @@ const rhf_rcv_function_ptr normal_rhf_rcv_functions[] = {
 	[RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
 	[RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
 };
+
+const rhf_rcv_function_ptr netdev_rhf_rcv_functions[] = {
+	[RHF_RCV_TYPE_EXPECTED] = process_receive_invalid,
+	[RHF_RCV_TYPE_EAGER] = process_receive_invalid,
+	[RHF_RCV_TYPE_IB] = hfi1_ipoib_ib_rcv,
+	[RHF_RCV_TYPE_ERROR] = process_receive_error,
+	[RHF_RCV_TYPE_BYPASS] = hfi1_vnic_bypass_rcv,
+	[RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
+	[RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
+	[RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
+};
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index e7fdd70c6e78..8ca51e43cf53 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015-2017 Intel Corporation.
+ * Copyright(c) 2015-2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -1264,7 +1264,7 @@ static int get_base_info(struct hfi1_filedata *fd, unsigned long arg, u32 len)
 	memset(&binfo, 0, sizeof(binfo));
 	binfo.hw_version = dd->revision;
 	binfo.sw_version = HFI1_KERN_SWVERSION;
-	binfo.bthqp = kdeth_qp;
+	binfo.bthqp = RVT_KDETH_QP_PREFIX;
 	binfo.jkey = uctxt->jkey;
 	/*
 	 * If more than 64 contexts are enabled the allocated credit
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index b06c2594105a..b4c6bff60a4e 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -1,7 +1,7 @@
 #ifndef _HFI1_KERNEL_H
 #define _HFI1_KERNEL_H
 /*
- * Copyright(c) 2015-2018 Intel Corporation.
+ * Copyright(c) 2015-2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -233,6 +233,8 @@ struct hfi1_ctxtdata {
 	intr_handler fast_handler;
 	/** slow handler */
 	intr_handler slow_handler;
+	/* napi pointer assiociated with netdev */
+	struct napi_struct *napi;
 	/* verbs rx_stats per rcd */
 	struct hfi1_opcode_stats_perctx *opstats;
 	/* clear interrupt mask */
@@ -383,11 +385,11 @@ struct hfi1_packet {
 	u32 rhqoff;
 	u32 dlid;
 	u32 slid;
+	int numpkt;
 	u16 tlen;
 	s16 etail;
 	u16 pkey;
 	u8 hlen;
-	u8 numpkt;
 	u8 rsize;
 	u8 updegr;
 	u8 etype;
@@ -985,7 +987,7 @@ typedef void (*hfi1_make_req)(struct rvt_qp *qp,
 			      struct hfi1_pkt_state *ps,
 			      struct rvt_swqe *wqe);
 extern const rhf_rcv_function_ptr normal_rhf_rcv_functions[];
-
+extern const rhf_rcv_function_ptr netdev_rhf_rcv_functions[];
 
 /* return values for the RHF receive functions */
 #define RHF_RCV_CONTINUE  0	/* keep going */
@@ -1045,23 +1047,10 @@ struct hfi1_asic_data {
 #define NUM_MAP_ENTRIES	 256
 #define NUM_MAP_REGS      32
 
-/*
- * Number of VNIC contexts used. Ensure it is less than or equal to
- * max queues supported by VNIC (HFI1_VNIC_MAX_QUEUE).
- */
-#define HFI1_NUM_VNIC_CTXT   8
-
-/* Number of VNIC RSM entries */
-#define NUM_VNIC_MAP_ENTRIES 8
-
 /* Virtual NIC information */
 struct hfi1_vnic_data {
-	struct hfi1_ctxtdata *ctxt[HFI1_NUM_VNIC_CTXT];
 	struct kmem_cache *txreq_cache;
-	struct xarray vesws;
 	u8 num_vports;
-	u8 rmt_start;
-	u8 num_ctxt;
 };
 
 struct hfi1_vnic_vport_info;
@@ -1167,8 +1156,8 @@ struct hfi1_devdata {
 	u64 z_send_schedule;
 
 	u64 __percpu *send_schedule;
-	/* number of reserved contexts for VNIC usage */
-	u16 num_vnic_contexts;
+	/* number of reserved contexts for netdev usage */
+	u16 num_netdev_contexts;
 	/* number of receive contexts in use by the driver */
 	u32 num_rcv_contexts;
 	/* number of pio send contexts in use by the driver */
@@ -1417,12 +1406,12 @@ struct hfi1_devdata {
 	struct hfi1_vnic_data vnic;
 	/* Lock to protect IRQ SRC register access */
 	spinlock_t irq_src_lock;
-};
+	int vnic_num_vports;
+	struct net_device *dummy_netdev;
 
-static inline bool hfi1_vnic_is_rsm_full(struct hfi1_devdata *dd, int spare)
-{
-	return (dd->vnic.rmt_start + spare) > NUM_MAP_ENTRIES;
-}
+	/* Keeps track of IPoIB RSM rule users */
+	atomic_t ipoib_rsm_usr_num;
+};
 
 /* 8051 firmware version helper */
 #define dc8051_ver(a, b, c) ((a) << 16 | (b) << 8 | (c))
@@ -1500,6 +1489,8 @@ struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt);
 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread);
 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread);
 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread);
+int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget);
+int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget);
 void set_all_slowpath(struct hfi1_devdata *dd);
 
 extern const struct pci_device_id hfi1_pci_tbl[];
@@ -2250,7 +2241,6 @@ extern int num_user_contexts;
 extern unsigned long n_krcvqs;
 extern uint krcvqs[];
 extern int krcvqsset;
-extern uint kdeth_qp;
 extern uint loopback;
 extern uint quick_linkup;
 extern uint rcv_intr_timeout;
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index 3759d9233a1c..5eed4360695f 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -69,6 +69,7 @@
 #include "affinity.h"
 #include "vnic.h"
 #include "exp_rcv.h"
+#include "netdev.h"
 
 #undef pr_fmt
 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
@@ -374,6 +375,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
 		rcd->numa_id = numa;
 		rcd->rcv_array_groups = dd->rcv_entries.ngroups;
 		rcd->rhf_rcv_function_map = normal_rhf_rcv_functions;
+		rcd->msix_intr = CCE_NUM_MSIX_VECTORS;
 
 		mutex_init(&rcd->exp_mutex);
 		spin_lock_init(&rcd->exp_lock);
@@ -1316,6 +1318,7 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
 		goto bail;
 	}
 
+	atomic_set(&dd->ipoib_rsm_usr_num, 0);
 	return dd;
 
 bail:
@@ -1663,9 +1666,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* do the generic initialization */
 	initfail = hfi1_init(dd, 0);
 
-	/* setup vnic */
-	hfi1_vnic_setup(dd);
-
 	ret = hfi1_register_ib_device(dd);
 
 	/*
@@ -1704,7 +1704,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 			hfi1_device_remove(dd);
 		if (!ret)
 			hfi1_unregister_ib_device(dd);
-		hfi1_vnic_cleanup(dd);
 		postinit_cleanup(dd);
 		if (initfail)
 			ret = initfail;
@@ -1749,8 +1748,8 @@ static void remove_one(struct pci_dev *pdev)
 	/* unregister from IB core */
 	hfi1_unregister_ib_device(dd);
 
-	/* cleanup vnic */
-	hfi1_vnic_cleanup(dd);
+	/* free netdev data */
+	hfi1_netdev_free(dd);
 
 	/*
 	 * Disable the IB link, disable interrupts on the device,
diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h
new file mode 100644
index 000000000000..185c9b02c974
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/ipoib.h
@@ -0,0 +1,171 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * Copyright(c) 2020 Intel Corporation.
+ *
+ */
+
+/*
+ * This file contains HFI1 support for IPOIB functionality
+ */
+
+#ifndef HFI1_IPOIB_H
+#define HFI1_IPOIB_H
+
+#include <linux/types.h>
+#include <linux/stddef.h>
+#include <linux/atomic.h>
+#include <linux/netdevice.h>
+#include <linux/slab.h>
+#include <linux/skbuff.h>
+#include <linux/list.h>
+#include <linux/if_infiniband.h>
+
+#include "hfi.h"
+#include "iowait.h"
+#include "netdev.h"
+
+#include <rdma/ib_verbs.h>
+
+#define HFI1_IPOIB_ENTROPY_SHIFT   24
+
+#define HFI1_IPOIB_TXREQ_NAME_LEN   32
+
+#define HFI1_IPOIB_PSEUDO_LEN 20
+#define HFI1_IPOIB_ENCAP_LEN 4
+
+struct hfi1_ipoib_dev_priv;
+
+union hfi1_ipoib_flow {
+	u16 as_int;
+	struct {
+		u8 tx_queue;
+		u8 sc5;
+	} __attribute__((__packed__));
+};
+
+/**
+ * struct hfi1_ipoib_circ_buf - List of items to be processed
+ * @items: ring of items
+ * @head: ring head
+ * @tail: ring tail
+ * @max_items: max items + 1 that the ring can contain
+ * @producer_lock: producer sync lock
+ * @consumer_lock: consumer sync lock
+ */
+struct hfi1_ipoib_circ_buf {
+	void **items;
+	unsigned long head;
+	unsigned long tail;
+	unsigned long max_items;
+	spinlock_t producer_lock; /* head sync lock */
+	spinlock_t consumer_lock; /* tail sync lock */
+};
+
+/**
+ * struct hfi1_ipoib_txq - IPOIB per Tx queue information
+ * @priv: private pointer
+ * @sde: sdma engine
+ * @tx_list: tx request list
+ * @sent_txreqs: count of txreqs posted to sdma
+ * @flow: tracks when list needs to be flushed for a flow change
+ * @q_idx: ipoib Tx queue index
+ * @pkts_sent: indicator packets have been sent from this queue
+ * @wait: iowait structure
+ * @complete_txreqs: count of txreqs completed by sdma
+ * @napi: pointer to tx napi interface
+ * @tx_ring: ring of ipoib txreqs to be reaped by napi callback
+ */
+struct hfi1_ipoib_txq {
+	struct hfi1_ipoib_dev_priv *priv;
+	struct sdma_engine *sde;
+	struct list_head tx_list;
+	u64 sent_txreqs;
+	union hfi1_ipoib_flow flow;
+	u8 q_idx;
+	bool pkts_sent;
+	struct iowait wait;
+
+	atomic64_t ____cacheline_aligned_in_smp complete_txreqs;
+	struct napi_struct *napi;
+	struct hfi1_ipoib_circ_buf tx_ring;
+};
+
+struct hfi1_ipoib_dev_priv {
+	struct hfi1_devdata *dd;
+	struct net_device   *netdev;
+	struct ib_device    *device;
+	struct hfi1_ipoib_txq *txqs;
+	struct kmem_cache *txreq_cache;
+	struct napi_struct *tx_napis;
+	u16 pkey;
+	u16 pkey_index;
+	u32 qkey;
+	u8 port_num;
+
+	const struct net_device_ops *netdev_ops;
+	struct rvt_qp *qp;
+	struct pcpu_sw_netstats __percpu *netstats;
+};
+
+/* hfi1 ipoib rdma netdev's private data structure */
+struct hfi1_ipoib_rdma_netdev {
+	struct rdma_netdev rn;  /* keep this first */
+	/* followed by device private data */
+	struct hfi1_ipoib_dev_priv dev_priv;
+};
+
+static inline struct hfi1_ipoib_dev_priv *
+hfi1_ipoib_priv(const struct net_device *dev)
+{
+	return &((struct hfi1_ipoib_rdma_netdev *)netdev_priv(dev))->dev_priv;
+}
+
+static inline void
+hfi1_ipoib_update_rx_netstats(struct hfi1_ipoib_dev_priv *priv,
+			      u64 packets,
+			      u64 bytes)
+{
+	struct pcpu_sw_netstats *netstats = this_cpu_ptr(priv->netstats);
+
+	u64_stats_update_begin(&netstats->syncp);
+	netstats->rx_packets += packets;
+	netstats->rx_bytes += bytes;
+	u64_stats_update_end(&netstats->syncp);
+}
+
+static inline void
+hfi1_ipoib_update_tx_netstats(struct hfi1_ipoib_dev_priv *priv,
+			      u64 packets,
+			      u64 bytes)
+{
+	struct pcpu_sw_netstats *netstats = this_cpu_ptr(priv->netstats);
+
+	u64_stats_update_begin(&netstats->syncp);
+	netstats->tx_packets += packets;
+	netstats->tx_bytes += bytes;
+	u64_stats_update_end(&netstats->syncp);
+}
+
+int hfi1_ipoib_send_dma(struct net_device *dev,
+			struct sk_buff *skb,
+			struct ib_ah *address,
+			u32 dqpn);
+
+int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv);
+void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv);
+
+int hfi1_ipoib_rxq_init(struct net_device *dev);
+void hfi1_ipoib_rxq_deinit(struct net_device *dev);
+
+void hfi1_ipoib_napi_tx_enable(struct net_device *dev);
+void hfi1_ipoib_napi_tx_disable(struct net_device *dev);
+
+struct sk_buff *hfi1_ipoib_prepare_skb(struct hfi1_netdev_rxq *rxq,
+				       int size, void *data);
+
+int hfi1_ipoib_rn_get_params(struct ib_device *device,
+			     u8 port_num,
+			     enum rdma_netdev_t type,
+			     struct rdma_netdev_alloc_params *params);
+
+#endif /* _IPOIB_H */
diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c
new file mode 100644
index 000000000000..014351ebbefa
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/ipoib_main.c
@@ -0,0 +1,309 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright(c) 2020 Intel Corporation.
+ *
+ */
+
+/*
+ * This file contains HFI1 support for ipoib functionality
+ */
+
+#include "ipoib.h"
+#include "hfi.h"
+
+static u32 qpn_from_mac(u8 *mac_arr)
+{
+	return (u32)mac_arr[1] << 16 | mac_arr[2] << 8 | mac_arr[3];
+}
+
+static int hfi1_ipoib_dev_init(struct net_device *dev)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	int ret;
+
+	priv->netstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+
+	ret = priv->netdev_ops->ndo_init(dev);
+	if (ret)
+		return ret;
+
+	ret = hfi1_netdev_add_data(priv->dd,
+				   qpn_from_mac(priv->netdev->dev_addr),
+				   dev);
+	if (ret < 0) {
+		priv->netdev_ops->ndo_uninit(dev);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void hfi1_ipoib_dev_uninit(struct net_device *dev)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+
+	hfi1_netdev_remove_data(priv->dd, qpn_from_mac(priv->netdev->dev_addr));
+
+	priv->netdev_ops->ndo_uninit(dev);
+}
+
+static int hfi1_ipoib_dev_open(struct net_device *dev)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	int ret;
+
+	ret = priv->netdev_ops->ndo_open(dev);
+	if (!ret) {
+		struct hfi1_ibport *ibp = to_iport(priv->device,
+						   priv->port_num);
+		struct rvt_qp *qp;
+		u32 qpn = qpn_from_mac(priv->netdev->dev_addr);
+
+		rcu_read_lock();
+		qp = rvt_lookup_qpn(ib_to_rvt(priv->device), &ibp->rvp, qpn);
+		if (!qp) {
+			rcu_read_unlock();
+			priv->netdev_ops->ndo_stop(dev);
+			return -EINVAL;
+		}
+		rvt_get_qp(qp);
+		priv->qp = qp;
+		rcu_read_unlock();
+
+		hfi1_netdev_enable_queues(priv->dd);
+		hfi1_ipoib_napi_tx_enable(dev);
+	}
+
+	return ret;
+}
+
+static int hfi1_ipoib_dev_stop(struct net_device *dev)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+
+	if (!priv->qp)
+		return 0;
+
+	hfi1_ipoib_napi_tx_disable(dev);
+	hfi1_netdev_disable_queues(priv->dd);
+
+	rvt_put_qp(priv->qp);
+	priv->qp = NULL;
+
+	return priv->netdev_ops->ndo_stop(dev);
+}
+
+static void hfi1_ipoib_dev_get_stats64(struct net_device *dev,
+				       struct rtnl_link_stats64 *storage)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	u64 rx_packets = 0ull;
+	u64 rx_bytes = 0ull;
+	u64 tx_packets = 0ull;
+	u64 tx_bytes = 0ull;
+	int i;
+
+	netdev_stats_to_stats64(storage, &dev->stats);
+
+	for_each_possible_cpu(i) {
+		const struct pcpu_sw_netstats *stats;
+		unsigned int start;
+		u64 trx_packets;
+		u64 trx_bytes;
+		u64 ttx_packets;
+		u64 ttx_bytes;
+
+		stats = per_cpu_ptr(priv->netstats, i);
+		do {
+			start = u64_stats_fetch_begin_irq(&stats->syncp);
+			trx_packets = stats->rx_packets;
+			trx_bytes = stats->rx_bytes;
+			ttx_packets = stats->tx_packets;
+			ttx_bytes = stats->tx_bytes;
+		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
+
+		rx_packets += trx_packets;
+		rx_bytes += trx_bytes;
+		tx_packets += ttx_packets;
+		tx_bytes += ttx_bytes;
+	}
+
+	storage->rx_packets += rx_packets;
+	storage->rx_bytes += rx_bytes;
+	storage->tx_packets += tx_packets;
+	storage->tx_bytes += tx_bytes;
+}
+
+static const struct net_device_ops hfi1_ipoib_netdev_ops = {
+	.ndo_init         = hfi1_ipoib_dev_init,
+	.ndo_uninit       = hfi1_ipoib_dev_uninit,
+	.ndo_open         = hfi1_ipoib_dev_open,
+	.ndo_stop         = hfi1_ipoib_dev_stop,
+	.ndo_get_stats64  = hfi1_ipoib_dev_get_stats64,
+};
+
+static int hfi1_ipoib_send(struct net_device *dev,
+			   struct sk_buff *skb,
+			   struct ib_ah *address,
+			   u32 dqpn)
+{
+	return hfi1_ipoib_send_dma(dev, skb, address, dqpn);
+}
+
+static int hfi1_ipoib_mcast_attach(struct net_device *dev,
+				   struct ib_device *device,
+				   union ib_gid *mgid,
+				   u16 mlid,
+				   int set_qkey,
+				   u32 qkey)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	u32 qpn = (u32)qpn_from_mac(priv->netdev->dev_addr);
+	struct hfi1_ibport *ibp = to_iport(priv->device, priv->port_num);
+	struct rvt_qp *qp;
+	int ret = -EINVAL;
+
+	rcu_read_lock();
+
+	qp = rvt_lookup_qpn(ib_to_rvt(priv->device), &ibp->rvp, qpn);
+	if (qp) {
+		rvt_get_qp(qp);
+		rcu_read_unlock();
+		if (set_qkey)
+			priv->qkey = qkey;
+
+		/* attach QP to multicast group */
+		ret = ib_attach_mcast(&qp->ibqp, mgid, mlid);
+		rvt_put_qp(qp);
+	} else {
+		rcu_read_unlock();
+	}
+
+	return ret;
+}
+
+static int hfi1_ipoib_mcast_detach(struct net_device *dev,
+				   struct ib_device *device,
+				   union ib_gid *mgid,
+				   u16 mlid)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	u32 qpn = (u32)qpn_from_mac(priv->netdev->dev_addr);
+	struct hfi1_ibport *ibp = to_iport(priv->device, priv->port_num);
+	struct rvt_qp *qp;
+	int ret = -EINVAL;
+
+	rcu_read_lock();
+
+	qp = rvt_lookup_qpn(ib_to_rvt(priv->device), &ibp->rvp, qpn);
+	if (qp) {
+		rvt_get_qp(qp);
+		rcu_read_unlock();
+		ret = ib_detach_mcast(&qp->ibqp, mgid, mlid);
+		rvt_put_qp(qp);
+	} else {
+		rcu_read_unlock();
+	}
+	return ret;
+}
+
+static void hfi1_ipoib_netdev_dtor(struct net_device *dev)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+
+	hfi1_ipoib_txreq_deinit(priv);
+	hfi1_ipoib_rxq_deinit(priv->netdev);
+
+	free_percpu(priv->netstats);
+}
+
+static void hfi1_ipoib_free_rdma_netdev(struct net_device *dev)
+{
+	hfi1_ipoib_netdev_dtor(dev);
+	free_netdev(dev);
+}
+
+static void hfi1_ipoib_set_id(struct net_device *dev, int id)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+
+	priv->pkey_index = (u16)id;
+	ib_query_pkey(priv->device,
+		      priv->port_num,
+		      priv->pkey_index,
+		      &priv->pkey);
+}
+
+static int hfi1_ipoib_setup_rn(struct ib_device *device,
+			       u8 port_num,
+			       struct net_device *netdev,
+			       void *param)
+{
+	struct hfi1_devdata *dd = dd_from_ibdev(device);
+	struct rdma_netdev *rn = netdev_priv(netdev);
+	struct hfi1_ipoib_dev_priv *priv;
+	int rc;
+
+	rn->send = hfi1_ipoib_send;
+	rn->attach_mcast = hfi1_ipoib_mcast_attach;
+	rn->detach_mcast = hfi1_ipoib_mcast_detach;
+	rn->set_id = hfi1_ipoib_set_id;
+	rn->hca = device;
+	rn->port_num = port_num;
+	rn->mtu = netdev->mtu;
+
+	priv = hfi1_ipoib_priv(netdev);
+	priv->dd = dd;
+	priv->netdev = netdev;
+	priv->device = device;
+	priv->port_num = port_num;
+	priv->netdev_ops = netdev->netdev_ops;
+
+	netdev->netdev_ops = &hfi1_ipoib_netdev_ops;
+
+	ib_query_pkey(device, port_num, priv->pkey_index, &priv->pkey);
+
+	rc = hfi1_ipoib_txreq_init(priv);
+	if (rc) {
+		dd_dev_err(dd, "IPoIB netdev TX init - failed(%d)\n", rc);
+		hfi1_ipoib_free_rdma_netdev(netdev);
+		return rc;
+	}
+
+	rc = hfi1_ipoib_rxq_init(netdev);
+	if (rc) {
+		dd_dev_err(dd, "IPoIB netdev RX init - failed(%d)\n", rc);
+		hfi1_ipoib_free_rdma_netdev(netdev);
+		return rc;
+	}
+
+	netdev->priv_destructor = hfi1_ipoib_netdev_dtor;
+	netdev->needs_free_netdev = true;
+
+	return 0;
+}
+
+int hfi1_ipoib_rn_get_params(struct ib_device *device,
+			     u8 port_num,
+			     enum rdma_netdev_t type,
+			     struct rdma_netdev_alloc_params *params)
+{
+	struct hfi1_devdata *dd = dd_from_ibdev(device);
+
+	if (type != RDMA_NETDEV_IPOIB)
+		return -EOPNOTSUPP;
+
+	if (!HFI1_CAP_IS_KSET(AIP) || !dd->num_netdev_contexts)
+		return -EOPNOTSUPP;
+
+	if (!port_num || port_num > dd->num_pports)
+		return -EINVAL;
+
+	params->sizeof_priv = sizeof(struct hfi1_ipoib_rdma_netdev);
+	params->txqs = dd->num_sdma;
+	params->rxqs = dd->num_netdev_contexts;
+	params->param = NULL;
+	params->initialize_rdma_netdev = hfi1_ipoib_setup_rn;
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/hfi1/ipoib_rx.c b/drivers/infiniband/hw/hfi1/ipoib_rx.c
new file mode 100644
index 000000000000..3afa7545242c
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/ipoib_rx.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright(c) 2020 Intel Corporation.
+ *
+ */
+
+#include "netdev.h"
+#include "ipoib.h"
+
+#define HFI1_IPOIB_SKB_PAD ((NET_SKB_PAD) + (NET_IP_ALIGN))
+
+static void copy_ipoib_buf(struct sk_buff *skb, void *data, int size)
+{
+	void *dst_data;
+
+	skb_checksum_none_assert(skb);
+	skb->protocol = *((__be16 *)data);
+
+	dst_data = skb_put(skb, size);
+	memcpy(dst_data, data, size);
+	skb->mac_header = HFI1_IPOIB_PSEUDO_LEN;
+	skb_pull(skb, HFI1_IPOIB_ENCAP_LEN);
+}
+
+static struct sk_buff *prepare_frag_skb(struct napi_struct *napi, int size)
+{
+	struct sk_buff *skb;
+	int skb_size = SKB_DATA_ALIGN(size + HFI1_IPOIB_SKB_PAD);
+	void *frag;
+
+	skb_size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	skb_size = SKB_DATA_ALIGN(skb_size);
+	frag = napi_alloc_frag(skb_size);
+
+	if (unlikely(!frag))
+		return napi_alloc_skb(napi, size);
+
+	skb = build_skb(frag, skb_size);
+
+	if (unlikely(!skb)) {
+		skb_free_frag(frag);
+		return NULL;
+	}
+
+	skb_reserve(skb, HFI1_IPOIB_SKB_PAD);
+	return skb;
+}
+
+struct sk_buff *hfi1_ipoib_prepare_skb(struct hfi1_netdev_rxq *rxq,
+				       int size, void *data)
+{
+	struct napi_struct *napi = &rxq->napi;
+	int skb_size = size + HFI1_IPOIB_ENCAP_LEN;
+	struct sk_buff *skb;
+
+	/*
+	 * For smaller(4k + skb overhead) allocations we will go using
+	 * napi cache. Otherwise we will try to use napi frag cache.
+	 */
+	if (size <= SKB_WITH_OVERHEAD(PAGE_SIZE))
+		skb = napi_alloc_skb(napi, skb_size);
+	else
+		skb = prepare_frag_skb(napi, skb_size);
+
+	if (unlikely(!skb))
+		return NULL;
+
+	copy_ipoib_buf(skb, data, size);
+
+	return skb;
+}
+
+int hfi1_ipoib_rxq_init(struct net_device *netdev)
+{
+	struct hfi1_ipoib_dev_priv *ipoib_priv = hfi1_ipoib_priv(netdev);
+	struct hfi1_devdata *dd = ipoib_priv->dd;
+	int ret;
+
+	ret = hfi1_netdev_rx_init(dd);
+	if (ret)
+		return ret;
+
+	hfi1_init_aip_rsm(dd);
+
+	return ret;
+}
+
+void hfi1_ipoib_rxq_deinit(struct net_device *netdev)
+{
+	struct hfi1_ipoib_dev_priv *ipoib_priv = hfi1_ipoib_priv(netdev);
+	struct hfi1_devdata *dd = ipoib_priv->dd;
+
+	hfi1_deinit_aip_rsm(dd);
+	hfi1_netdev_rx_destroy(dd);
+}
diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
new file mode 100644
index 000000000000..883cb9d48022
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -0,0 +1,828 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright(c) 2020 Intel Corporation.
+ *
+ */
+
+/*
+ * This file contains HFI1 support for IPOIB SDMA functionality
+ */
+
+#include <linux/log2.h>
+#include <linux/circ_buf.h>
+
+#include "sdma.h"
+#include "verbs.h"
+#include "trace_ibhdrs.h"
+#include "ipoib.h"
+
+/* Add a convenience helper */
+#define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
+#define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
+#define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
+
+/**
+ * struct ipoib_txreq - IPOIB transmit descriptor
+ * @txreq: sdma transmit request
+ * @sdma_hdr: 9b ib headers
+ * @sdma_status: status returned by sdma engine
+ * @priv: ipoib netdev private data
+ * @txq: txq on which skb was output
+ * @skb: skb to send
+ */
+struct ipoib_txreq {
+	struct sdma_txreq           txreq;
+	struct hfi1_sdma_header     sdma_hdr;
+	int                         sdma_status;
+	struct hfi1_ipoib_dev_priv *priv;
+	struct hfi1_ipoib_txq      *txq;
+	struct sk_buff             *skb;
+};
+
+struct ipoib_txparms {
+	struct hfi1_devdata        *dd;
+	struct rdma_ah_attr        *ah_attr;
+	struct hfi1_ibport         *ibp;
+	struct hfi1_ipoib_txq      *txq;
+	union hfi1_ipoib_flow       flow;
+	u32                         dqpn;
+	u8                          hdr_dwords;
+	u8                          entropy;
+};
+
+static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)
+{
+	return sent - completed;
+}
+
+static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
+{
+	if (unlikely(hfi1_ipoib_txreqs(++txq->sent_txreqs,
+				       atomic64_read(&txq->complete_txreqs)) >=
+	    min_t(unsigned int, txq->priv->netdev->tx_queue_len,
+		  txq->tx_ring.max_items - 1)))
+		netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
+}
+
+static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
+{
+	struct net_device *dev = txq->priv->netdev;
+
+	/* If the queue is already running just return */
+	if (likely(!__netif_subqueue_stopped(dev, txq->q_idx)))
+		return;
+
+	/* If shutting down just return as queue state is irrelevant */
+	if (unlikely(dev->reg_state != NETREG_REGISTERED))
+		return;
+
+	/*
+	 * When the queue has been drained to less than half full it will be
+	 * restarted.
+	 * The size of the txreq ring is fixed at initialization.
+	 * The tx queue len can be adjusted upward while the interface is
+	 * running.
+	 * The tx queue len can be large enough to overflow the txreq_ring.
+	 * Use the minimum of the current tx_queue_len or the rings max txreqs
+	 * to protect against ring overflow.
+	 */
+	if (hfi1_ipoib_txreqs(txq->sent_txreqs,
+			      atomic64_read(&txq->complete_txreqs))
+	    < min_t(unsigned int, dev->tx_queue_len,
+		    txq->tx_ring.max_items) >> 1)
+		netif_wake_subqueue(dev, txq->q_idx);
+}
+
+static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
+{
+	struct hfi1_ipoib_dev_priv *priv = tx->priv;
+
+	if (likely(!tx->sdma_status)) {
+		hfi1_ipoib_update_tx_netstats(priv, 1, tx->skb->len);
+	} else {
+		++priv->netdev->stats.tx_errors;
+		dd_dev_warn(priv->dd,
+			    "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",
+			    __func__, tx->sdma_status,
+			    le64_to_cpu(tx->sdma_hdr.pbc), tx->txq->q_idx,
+			    tx->txq->sde->this_idx);
+	}
+
+	napi_consume_skb(tx->skb, budget);
+	sdma_txclean(priv->dd, &tx->txreq);
+	kmem_cache_free(priv->txreq_cache, tx);
+}
+
+static int hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq, int budget)
+{
+	struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
+	unsigned long head;
+	unsigned long tail;
+	unsigned int max_tx;
+	int work_done;
+	int tx_count;
+
+	spin_lock_bh(&tx_ring->consumer_lock);
+
+	/* Read index before reading contents at that index. */
+	head = smp_load_acquire(&tx_ring->head);
+	tail = tx_ring->tail;
+	max_tx = tx_ring->max_items;
+
+	work_done = min_t(int, CIRC_CNT(head, tail, max_tx), budget);
+
+	for (tx_count = work_done; tx_count; tx_count--) {
+		hfi1_ipoib_free_tx(tx_ring->items[tail], budget);
+		tail = CIRC_NEXT(tail, max_tx);
+	}
+
+	atomic64_add(work_done, &txq->complete_txreqs);
+
+	/* Finished freeing tx items so store the tail value. */
+	smp_store_release(&tx_ring->tail, tail);
+
+	spin_unlock_bh(&tx_ring->consumer_lock);
+
+	hfi1_ipoib_check_queue_stopped(txq);
+
+	return work_done;
+}
+
+static int hfi1_ipoib_process_tx_ring(struct napi_struct *napi, int budget)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(napi->dev);
+	struct hfi1_ipoib_txq *txq = &priv->txqs[napi - priv->tx_napis];
+
+	int work_done = hfi1_ipoib_drain_tx_ring(txq, budget);
+
+	if (work_done < budget)
+		napi_complete_done(napi, work_done);
+
+	return work_done;
+}
+
+static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx)
+{
+	struct hfi1_ipoib_circ_buf *tx_ring = &tx->txq->tx_ring;
+	unsigned long head;
+	unsigned long tail;
+	size_t max_tx;
+
+	spin_lock(&tx_ring->producer_lock);
+
+	head = tx_ring->head;
+	tail = READ_ONCE(tx_ring->tail);
+	max_tx = tx_ring->max_items;
+
+	if (likely(CIRC_SPACE(head, tail, max_tx))) {
+		tx_ring->items[head] = tx;
+
+		/* Finish storing txreq before incrementing head. */
+		smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx));
+		napi_schedule(tx->txq->napi);
+	} else {
+		struct hfi1_ipoib_txq *txq = tx->txq;
+		struct hfi1_ipoib_dev_priv *priv = tx->priv;
+
+		/* Ring was full */
+		hfi1_ipoib_free_tx(tx, 0);
+		atomic64_inc(&txq->complete_txreqs);
+		dd_dev_dbg(priv->dd, "txq %d full.\n", txq->q_idx);
+	}
+
+	spin_unlock(&tx_ring->producer_lock);
+}
+
+static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)
+{
+	struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);
+
+	tx->sdma_status = status;
+
+	hfi1_ipoib_add_tx(tx);
+}
+
+static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
+					struct ipoib_txparms *txp)
+{
+	struct hfi1_devdata *dd = txp->dd;
+	struct sdma_txreq *txreq = &tx->txreq;
+	struct sk_buff *skb = tx->skb;
+	int ret = 0;
+	int i;
+
+	if (skb_headlen(skb)) {
+		ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb));
+		if (unlikely(ret))
+			return ret;
+	}
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+		ret = sdma_txadd_page(dd,
+				      txreq,
+				      skb_frag_page(frag),
+				      frag->bv_offset,
+				      skb_frag_size(frag));
+		if (unlikely(ret))
+			break;
+	}
+
+	return ret;
+}
+
+static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,
+				    struct ipoib_txparms *txp)
+{
+	struct hfi1_devdata *dd = txp->dd;
+	struct sdma_txreq *txreq = &tx->txreq;
+	struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
+	u16 pkt_bytes =
+		sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len;
+	int ret;
+
+	ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);
+	if (unlikely(ret))
+		return ret;
+
+	/* add pbc + headers */
+	ret = sdma_txadd_kvaddr(dd,
+				txreq,
+				sdma_hdr,
+				sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2));
+	if (unlikely(ret))
+		return ret;
+
+	/* add the ulp payload */
+	return hfi1_ipoib_build_ulp_payload(tx, txp);
+}
+
+static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,
+					   struct ipoib_txparms *txp)
+{
+	struct hfi1_ipoib_dev_priv *priv = tx->priv;
+	struct hfi1_sdma_header *sdma_hdr = &tx->sdma_hdr;
+	struct sk_buff *skb = tx->skb;
+	struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp);
+	struct rdma_ah_attr *ah_attr = txp->ah_attr;
+	struct ib_other_headers *ohdr;
+	struct ib_grh *grh;
+	u16 dwords;
+	u16 slid;
+	u16 dlid;
+	u16 lrh0;
+	u32 bth0;
+	u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 |
+			 priv->netdev->dev_addr[2] << 8 |
+			 priv->netdev->dev_addr[3]);
+	u16 payload_dwords;
+	u8 pad_cnt;
+
+	pad_cnt = -skb->len & 3;
+
+	/* Includes ICRC */
+	payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC;
+
+	/* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
+	txp->hdr_dwords = 7;
+
+	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
+		grh = &sdma_hdr->hdr.ibh.u.l.grh;
+		txp->hdr_dwords +=
+			hfi1_make_grh(txp->ibp,
+				      grh,
+				      rdma_ah_read_grh(ah_attr),
+				      txp->hdr_dwords - LRH_9B_DWORDS,
+				      payload_dwords);
+		lrh0 = HFI1_LRH_GRH;
+		ohdr = &sdma_hdr->hdr.ibh.u.l.oth;
+	} else {
+		lrh0 = HFI1_LRH_BTH;
+		ohdr = &sdma_hdr->hdr.ibh.u.oth;
+	}
+
+	lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
+	lrh0 |= (txp->flow.sc5 & 0xf) << 12;
+
+	dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
+	if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
+		slid = be16_to_cpu(IB_LID_PERMISSIVE);
+	} else {
+		u16 lid = (u16)ppd->lid;
+
+		if (lid) {
+			lid |= rdma_ah_get_path_bits(ah_attr) &
+				((1 << ppd->lmc) - 1);
+			slid = lid;
+		} else {
+			slid = be16_to_cpu(IB_LID_PERMISSIVE);
+		}
+	}
+
+	/* Includes ICRC */
+	dwords = txp->hdr_dwords + payload_dwords;
+
+	/* Build the lrh */
+	sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B;
+	hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid);
+
+	/* Build the bth */
+	bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey;
+
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(txp->dqpn);
+	ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->sent_txreqs));
+
+	/* Build the deth */
+	ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey);
+	ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy <<
+					  HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);
+
+	/* Construct the pbc. */
+	sdma_hdr->pbc =
+		cpu_to_le64(create_pbc(ppd,
+				       ib_is_sc5(txp->flow.sc5) <<
+							      PBC_DC_INFO_SHIFT,
+				       0,
+				       sc_to_vlt(priv->dd, txp->flow.sc5),
+				       dwords - SIZE_OF_CRC +
+						(sizeof(sdma_hdr->pbc) >> 2)));
+}
+
+static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
+						      struct sk_buff *skb,
+						      struct ipoib_txparms *txp)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	struct ipoib_txreq *tx;
+	int ret;
+
+	tx = kmem_cache_alloc_node(priv->txreq_cache,
+				   GFP_ATOMIC,
+				   priv->dd->node);
+	if (unlikely(!tx))
+		return ERR_PTR(-ENOMEM);
+
+	/* so that we can test if the sdma decriptors are there */
+	tx->txreq.num_desc = 0;
+	tx->priv = priv;
+	tx->txq = txp->txq;
+	tx->skb = skb;
+
+	hfi1_ipoib_build_ib_tx_headers(tx, txp);
+
+	ret = hfi1_ipoib_build_tx_desc(tx, txp);
+	if (likely(!ret)) {
+		if (txp->txq->flow.as_int != txp->flow.as_int) {
+			txp->txq->flow.tx_queue = txp->flow.tx_queue;
+			txp->txq->flow.sc5 = txp->flow.sc5;
+			txp->txq->sde =
+				sdma_select_engine_sc(priv->dd,
+						      txp->flow.tx_queue,
+						      txp->flow.sc5);
+		}
+
+		return tx;
+	}
+
+	sdma_txclean(priv->dd, &tx->txreq);
+	kmem_cache_free(priv->txreq_cache, tx);
+
+	return ERR_PTR(ret);
+}
+
+static int hfi1_ipoib_submit_tx_list(struct net_device *dev,
+				     struct hfi1_ipoib_txq *txq)
+{
+	int ret;
+	u16 count_out;
+
+	ret = sdma_send_txlist(txq->sde,
+			       iowait_get_ib_work(&txq->wait),
+			       &txq->tx_list,
+			       &count_out);
+	if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)
+		return ret;
+
+	dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret);
+
+	return ret;
+}
+
+static int hfi1_ipoib_flush_tx_list(struct net_device *dev,
+				    struct hfi1_ipoib_txq *txq)
+{
+	int ret = 0;
+
+	if (!list_empty(&txq->tx_list)) {
+		/* Flush the current list */
+		ret = hfi1_ipoib_submit_tx_list(dev, txq);
+
+		if (unlikely(ret))
+			if (ret != -EBUSY)
+				++dev->stats.tx_carrier_errors;
+	}
+
+	return ret;
+}
+
+static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,
+				struct ipoib_txreq *tx)
+{
+	int ret;
+
+	ret = sdma_send_txreq(txq->sde,
+			      iowait_get_ib_work(&txq->wait),
+			      &tx->txreq,
+			      txq->pkts_sent);
+	if (likely(!ret)) {
+		txq->pkts_sent = true;
+		iowait_starve_clear(txq->pkts_sent, &txq->wait);
+	}
+
+	return ret;
+}
+
+static int hfi1_ipoib_send_dma_single(struct net_device *dev,
+				      struct sk_buff *skb,
+				      struct ipoib_txparms *txp)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	struct hfi1_ipoib_txq *txq = txp->txq;
+	struct ipoib_txreq *tx;
+	int ret;
+
+	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
+	if (IS_ERR(tx)) {
+		int ret = PTR_ERR(tx);
+
+		dev_kfree_skb_any(skb);
+
+		if (ret == -ENOMEM)
+			++dev->stats.tx_errors;
+		else
+			++dev->stats.tx_carrier_errors;
+
+		return NETDEV_TX_OK;
+	}
+
+	ret = hfi1_ipoib_submit_tx(txq, tx);
+	if (likely(!ret)) {
+		trace_sdma_output_ibhdr(tx->priv->dd,
+					&tx->sdma_hdr.hdr,
+					ib_is_sc5(txp->flow.sc5));
+		hfi1_ipoib_check_queue_depth(txq);
+		return NETDEV_TX_OK;
+	}
+
+	txq->pkts_sent = false;
+
+	if (ret == -EBUSY) {
+		list_add_tail(&tx->txreq.list, &txq->tx_list);
+
+		trace_sdma_output_ibhdr(tx->priv->dd,
+					&tx->sdma_hdr.hdr,
+					ib_is_sc5(txp->flow.sc5));
+		hfi1_ipoib_check_queue_depth(txq);
+		return NETDEV_TX_OK;
+	}
+
+	if (ret == -ECOMM) {
+		hfi1_ipoib_check_queue_depth(txq);
+		return NETDEV_TX_OK;
+	}
+
+	sdma_txclean(priv->dd, &tx->txreq);
+	dev_kfree_skb_any(skb);
+	kmem_cache_free(priv->txreq_cache, tx);
+	++dev->stats.tx_carrier_errors;
+
+	return NETDEV_TX_OK;
+}
+
+static int hfi1_ipoib_send_dma_list(struct net_device *dev,
+				    struct sk_buff *skb,
+				    struct ipoib_txparms *txp)
+{
+	struct hfi1_ipoib_txq *txq = txp->txq;
+	struct ipoib_txreq *tx;
+
+	/* Has the flow change ? */
+	if (txq->flow.as_int != txp->flow.as_int)
+		(void)hfi1_ipoib_flush_tx_list(dev, txq);
+
+	tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
+	if (IS_ERR(tx)) {
+		int ret = PTR_ERR(tx);
+
+		dev_kfree_skb_any(skb);
+
+		if (ret == -ENOMEM)
+			++dev->stats.tx_errors;
+		else
+			++dev->stats.tx_carrier_errors;
+
+		return NETDEV_TX_OK;
+	}
+
+	list_add_tail(&tx->txreq.list, &txq->tx_list);
+
+	hfi1_ipoib_check_queue_depth(txq);
+
+	trace_sdma_output_ibhdr(tx->priv->dd,
+				&tx->sdma_hdr.hdr,
+				ib_is_sc5(txp->flow.sc5));
+
+	if (!netdev_xmit_more())
+		(void)hfi1_ipoib_flush_tx_list(dev, txq);
+
+	return NETDEV_TX_OK;
+}
+
+static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
+{
+	if (skb_transport_header_was_set(skb)) {
+		u8 *hdr = (u8 *)skb_transport_header(skb);
+
+		return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);
+	}
+
+	return (u8)skb_get_queue_mapping(skb);
+}
+
+int hfi1_ipoib_send_dma(struct net_device *dev,
+			struct sk_buff *skb,
+			struct ib_ah *address,
+			u32 dqpn)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	struct ipoib_txparms txp;
+	struct rdma_netdev *rn = netdev_priv(dev);
+
+	if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) {
+		dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n",
+			    skb->len,
+			    rn->mtu + HFI1_IPOIB_ENCAP_LEN);
+		++dev->stats.tx_dropped;
+		++dev->stats.tx_errors;
+		dev_kfree_skb_any(skb);
+		return NETDEV_TX_OK;
+	}
+
+	txp.dd = priv->dd;
+	txp.ah_attr = &ibah_to_rvtah(address)->attr;
+	txp.ibp = to_iport(priv->device, priv->port_num);
+	txp.txq = &priv->txqs[skb_get_queue_mapping(skb)];
+	txp.dqpn = dqpn;
+	txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];
+	txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);
+	txp.entropy = hfi1_ipoib_calc_entropy(skb);
+
+	if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list))
+		return hfi1_ipoib_send_dma_list(dev, skb, &txp);
+
+	return hfi1_ipoib_send_dma_single(dev, skb,  &txp);
+}
+
+/*
+ * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function
+ *
+ * This function gets called from sdma_send_txreq() when there are not enough
+ * sdma descriptors available to send the packet. It adds Tx queue's wait
+ * structure to sdma engine's dmawait list to be woken up when descriptors
+ * become available.
+ */
+static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
+				 struct iowait_work *wait,
+				 struct sdma_txreq *txreq,
+				 uint seq,
+				 bool pkts_sent)
+{
+	struct hfi1_ipoib_txq *txq =
+		container_of(wait->iow, struct hfi1_ipoib_txq, wait);
+
+	write_seqlock(&sde->waitlock);
+
+	if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) {
+		if (sdma_progress(sde, seq, txreq)) {
+			write_sequnlock(&sde->waitlock);
+			return -EAGAIN;
+		}
+
+		netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
+
+		if (list_empty(&txq->wait.list))
+			iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
+
+		write_sequnlock(&sde->waitlock);
+		return -EBUSY;
+	}
+
+	write_sequnlock(&sde->waitlock);
+	return -EINVAL;
+}
+
+/*
+ * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function
+ *
+ * This function gets called when SDMA descriptors becomes available and Tx
+ * queue's wait structure was previously added to sdma engine's dmawait list.
+ */
+static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
+{
+	struct hfi1_ipoib_txq *txq =
+		container_of(wait, struct hfi1_ipoib_txq, wait);
+
+	if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
+		iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
+}
+
+static void hfi1_ipoib_flush_txq(struct work_struct *work)
+{
+	struct iowait_work *ioww =
+		container_of(work, struct iowait_work, iowork);
+	struct iowait *wait = iowait_ioww_to_iow(ioww);
+	struct hfi1_ipoib_txq *txq =
+		container_of(wait, struct hfi1_ipoib_txq, wait);
+	struct net_device *dev = txq->priv->netdev;
+
+	if (likely(dev->reg_state == NETREG_REGISTERED) &&
+	    likely(__netif_subqueue_stopped(dev, txq->q_idx)) &&
+	    likely(!hfi1_ipoib_flush_tx_list(dev, txq)))
+		netif_wake_subqueue(dev, txq->q_idx);
+}
+
+int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
+{
+	struct net_device *dev = priv->netdev;
+	char buf[HFI1_IPOIB_TXREQ_NAME_LEN];
+	unsigned long tx_ring_size;
+	int i;
+
+	/*
+	 * Ring holds 1 less than tx_ring_size
+	 * Round up to next power of 2 in order to hold at least tx_queue_len
+	 */
+	tx_ring_size = roundup_pow_of_two((unsigned long)dev->tx_queue_len + 1);
+
+	snprintf(buf, sizeof(buf), "hfi1_%u_ipoib_txreq_cache", priv->dd->unit);
+	priv->txreq_cache = kmem_cache_create(buf,
+					      sizeof(struct ipoib_txreq),
+					      0,
+					      0,
+					      NULL);
+	if (!priv->txreq_cache)
+		return -ENOMEM;
+
+	priv->tx_napis = kcalloc_node(dev->num_tx_queues,
+				      sizeof(struct napi_struct),
+				      GFP_ATOMIC,
+				      priv->dd->node);
+	if (!priv->tx_napis)
+		goto free_txreq_cache;
+
+	priv->txqs = kcalloc_node(dev->num_tx_queues,
+				  sizeof(struct hfi1_ipoib_txq),
+				  GFP_ATOMIC,
+				  priv->dd->node);
+	if (!priv->txqs)
+		goto free_tx_napis;
+
+	for (i = 0; i < dev->num_tx_queues; i++) {
+		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
+
+		iowait_init(&txq->wait,
+			    0,
+			    hfi1_ipoib_flush_txq,
+			    NULL,
+			    hfi1_ipoib_sdma_sleep,
+			    hfi1_ipoib_sdma_wakeup,
+			    NULL,
+			    NULL);
+		txq->priv = priv;
+		txq->sde = NULL;
+		INIT_LIST_HEAD(&txq->tx_list);
+		atomic64_set(&txq->complete_txreqs, 0);
+		txq->q_idx = i;
+		txq->flow.tx_queue = 0xff;
+		txq->flow.sc5 = 0xff;
+		txq->pkts_sent = false;
+
+		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
+					     priv->dd->node);
+
+		txq->tx_ring.items =
+			vzalloc_node(array_size(tx_ring_size,
+						sizeof(struct ipoib_txreq)),
+				     priv->dd->node);
+		if (!txq->tx_ring.items)
+			goto free_txqs;
+
+		spin_lock_init(&txq->tx_ring.producer_lock);
+		spin_lock_init(&txq->tx_ring.consumer_lock);
+		txq->tx_ring.max_items = tx_ring_size;
+
+		txq->napi = &priv->tx_napis[i];
+		netif_tx_napi_add(dev, txq->napi,
+				  hfi1_ipoib_process_tx_ring,
+				  NAPI_POLL_WEIGHT);
+	}
+
+	return 0;
+
+free_txqs:
+	for (i--; i >= 0; i--) {
+		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
+
+		netif_napi_del(txq->napi);
+		vfree(txq->tx_ring.items);
+	}
+
+	kfree(priv->txqs);
+	priv->txqs = NULL;
+
+free_tx_napis:
+	kfree(priv->tx_napis);
+	priv->tx_napis = NULL;
+
+free_txreq_cache:
+	kmem_cache_destroy(priv->txreq_cache);
+	priv->txreq_cache = NULL;
+	return -ENOMEM;
+}
+
+static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)
+{
+	struct sdma_txreq *txreq;
+	struct sdma_txreq *txreq_tmp;
+	atomic64_t *complete_txreqs = &txq->complete_txreqs;
+
+	list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) {
+		struct ipoib_txreq *tx =
+			container_of(txreq, struct ipoib_txreq, txreq);
+
+		list_del(&txreq->list);
+		sdma_txclean(txq->priv->dd, &tx->txreq);
+		dev_kfree_skb_any(tx->skb);
+		kmem_cache_free(txq->priv->txreq_cache, tx);
+		atomic64_inc(complete_txreqs);
+	}
+
+	if (hfi1_ipoib_txreqs(txq->sent_txreqs, atomic64_read(complete_txreqs)))
+		dd_dev_warn(txq->priv->dd,
+			    "txq %d not empty found %llu requests\n",
+			    txq->q_idx,
+			    hfi1_ipoib_txreqs(txq->sent_txreqs,
+					      atomic64_read(complete_txreqs)));
+}
+
+void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
+{
+	int i;
+
+	for (i = 0; i < priv->netdev->num_tx_queues; i++) {
+		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
+
+		iowait_cancel_work(&txq->wait);
+		iowait_sdma_drain(&txq->wait);
+		hfi1_ipoib_drain_tx_list(txq);
+		netif_napi_del(txq->napi);
+		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
+		vfree(txq->tx_ring.items);
+	}
+
+	kfree(priv->txqs);
+	priv->txqs = NULL;
+
+	kfree(priv->tx_napis);
+	priv->tx_napis = NULL;
+
+	kmem_cache_destroy(priv->txreq_cache);
+	priv->txreq_cache = NULL;
+}
+
+void hfi1_ipoib_napi_tx_enable(struct net_device *dev)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	int i;
+
+	for (i = 0; i < dev->num_tx_queues; i++) {
+		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
+
+		napi_enable(txq->napi);
+	}
+}
+
+void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	int i;
+
+	for (i = 0; i < dev->num_tx_queues; i++) {
+		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
+
+		napi_disable(txq->napi);
+		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
+	}
+}
diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c
index 14d2a90964c3..24ca17b77b72 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.c
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.c
@@ -333,7 +333,7 @@ static void do_remove(struct mmu_rb_handler *handler,
 
 /*
  * Work queue function to remove all nodes that have been queued up to
- * be removed.  The key feature is that mm->mmap_sem is not being held
+ * be removed.  The key feature is that mm->mmap_lock is not being held
  * and the remove callback can sleep while taking it, if needed.
  */
 static void handle_remove(struct work_struct *work)
diff --git a/drivers/infiniband/hw/hfi1/msix.c b/drivers/infiniband/hw/hfi1/msix.c
index db82db497b2c..d61ee853d215 100644
--- a/drivers/infiniband/hw/hfi1/msix.c
+++ b/drivers/infiniband/hw/hfi1/msix.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
 /*
- * Copyright(c) 2018 Intel Corporation.
+ * Copyright(c) 2018 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -49,6 +49,7 @@
 #include "hfi.h"
 #include "affinity.h"
 #include "sdma.h"
+#include "netdev.h"
 
 /**
  * msix_initialize() - Calculate, request and configure MSIx IRQs
@@ -69,7 +70,7 @@ int msix_initialize(struct hfi1_devdata *dd)
 	 *	one for each VNIC context
 	 *      ...any new IRQs should be added here.
 	 */
-	total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_vnic_contexts;
+	total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts;
 
 	if (total >= CCE_NUM_MSIX_VECTORS)
 		return -EINVAL;
@@ -140,7 +141,7 @@ static int msix_request_irq(struct hfi1_devdata *dd, void *arg,
 	ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name);
 	if (ret) {
 		dd_dev_err(dd,
-			   "%s: request for IRQ %d failed, MSIx %lu, err %d\n",
+			   "%s: request for IRQ %d failed, MSIx %lx, err %d\n",
 			   name, irq, nr, ret);
 		spin_lock(&dd->msix_info.msix_lock);
 		__clear_bit(nr, dd->msix_info.in_use_msix);
@@ -160,7 +161,7 @@ static int msix_request_irq(struct hfi1_devdata *dd, void *arg,
 	/* This is a request, so a failure is not fatal */
 	ret = hfi1_get_irq_affinity(dd, me);
 	if (ret)
-		dd_dev_err(dd, "unable to pin IRQ %d\n", ret);
+		dd_dev_err(dd, "%s: unable to pin IRQ %d\n", name, ret);
 
 	return nr;
 }
@@ -171,7 +172,8 @@ static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd,
 				       const char *name)
 {
 	int nr = msix_request_irq(rcd->dd, rcd, handler, thread,
-				  IRQ_RCVCTXT, name);
+				  rcd->is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT,
+				  name);
 	if (nr < 0)
 		return nr;
 
@@ -204,6 +206,21 @@ int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)
 }
 
 /**
+ * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs
+ * for netdev context
+ * @rcd: valid netdev contexti
+ */
+int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd)
+{
+	char name[MAX_NAME_SIZE];
+
+	snprintf(name, sizeof(name), DRIVER_NAME "_%d nd kctxt%d",
+		 rcd->dd->unit, rcd->ctxt);
+	return msix_request_rcd_irq_common(rcd, receive_context_interrupt_napi,
+					   NULL, name);
+}
+
+/**
  * msix_request_smda_ira() - Helper for getting SDMA IRQ resources
  * @sde: valid sdma engine
  *
@@ -355,15 +372,16 @@ void msix_clean_up_interrupts(struct hfi1_devdata *dd)
 }
 
 /**
- * msix_vnic_syncrhonize_irq() - Vnic IRQ synchronize
+ * msix_netdev_syncrhonize_irq() - netdev IRQ synchronize
  * @dd: valid devdata
  */
-void msix_vnic_synchronize_irq(struct hfi1_devdata *dd)
+void msix_netdev_synchronize_irq(struct hfi1_devdata *dd)
 {
 	int i;
+	int ctxt_count = hfi1_netdev_ctxt_count(dd);
 
-	for (i = 0; i < dd->vnic.num_ctxt; i++) {
-		struct hfi1_ctxtdata *rcd = dd->vnic.ctxt[i];
+	for (i = 0; i < ctxt_count; i++) {
+		struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, i);
 		struct hfi1_msix_entry *me;
 
 		me = &dd->msix_info.msix_entries[rcd->msix_intr];
diff --git a/drivers/infiniband/hw/hfi1/msix.h b/drivers/infiniband/hw/hfi1/msix.h
index 1a02ab7971c8..e63e944bf0fc 100644
--- a/drivers/infiniband/hw/hfi1/msix.h
+++ b/drivers/infiniband/hw/hfi1/msix.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
 /*
- * Copyright(c) 2018 Intel Corporation.
+ * Copyright(c) 2018 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -59,7 +59,8 @@ int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd);
 int msix_request_sdma_irq(struct sdma_engine *sde);
 void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr);
 
-/* VNIC interface */
-void msix_vnic_synchronize_irq(struct hfi1_devdata *dd);
+/* Netdev interface */
+void msix_netdev_synchronize_irq(struct hfi1_devdata *dd);
+int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd);
 
 #endif
diff --git a/drivers/infiniband/hw/hfi1/netdev.h b/drivers/infiniband/hw/hfi1/netdev.h
new file mode 100644
index 000000000000..947543a3e0c4
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/netdev.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * Copyright(c) 2020 Intel Corporation.
+ *
+ */
+
+#ifndef HFI1_NETDEV_H
+#define HFI1_NETDEV_H
+
+#include "hfi.h"
+
+#include <linux/netdevice.h>
+#include <linux/xarray.h>
+
+/**
+ * struct hfi1_netdev_rxq - Receive Queue for HFI
+ * dummy netdev. Both IPoIB and VNIC netdevices will be working on
+ * top of this device.
+ * @napi: napi object
+ * @priv: ptr to netdev_priv
+ * @rcd:  ptr to receive context data
+ */
+struct hfi1_netdev_rxq {
+	struct napi_struct napi;
+	struct hfi1_netdev_priv *priv;
+	struct hfi1_ctxtdata *rcd;
+};
+
+/*
+ * Number of netdev contexts used. Ensure it is less than or equal to
+ * max queues supported by VNIC (HFI1_VNIC_MAX_QUEUE).
+ */
+#define HFI1_MAX_NETDEV_CTXTS   8
+
+/* Number of NETDEV RSM entries */
+#define NUM_NETDEV_MAP_ENTRIES HFI1_MAX_NETDEV_CTXTS
+
+/**
+ * struct hfi1_netdev_priv: data required to setup and run HFI netdev.
+ * @dd:		hfi1_devdata
+ * @rxq:	pointer to dummy netdev receive queues.
+ * @num_rx_q:	number of receive queues
+ * @rmt_index:	first free index in RMT Array
+ * @msix_start: first free MSI-X interrupt vector.
+ * @dev_tbl:	netdev table for unique identifier VNIC and IPoIb VLANs.
+ * @enabled:	atomic counter of netdevs enabling receive queues.
+ *		When 0 NAPI will be disabled.
+ * @netdevs:	atomic counter of netdevs using dummy netdev.
+ *		When 0 receive queues will be freed.
+ */
+struct hfi1_netdev_priv {
+	struct hfi1_devdata *dd;
+	struct hfi1_netdev_rxq *rxq;
+	int num_rx_q;
+	int rmt_start;
+	struct xarray dev_tbl;
+	/* count of enabled napi polls */
+	atomic_t enabled;
+	/* count of netdevs on top */
+	atomic_t netdevs;
+};
+
+static inline
+struct hfi1_netdev_priv *hfi1_netdev_priv(struct net_device *dev)
+{
+	return (struct hfi1_netdev_priv *)&dev[1];
+}
+
+static inline
+int hfi1_netdev_ctxt_count(struct hfi1_devdata *dd)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	return priv->num_rx_q;
+}
+
+static inline
+struct hfi1_ctxtdata *hfi1_netdev_get_ctxt(struct hfi1_devdata *dd, int ctxt)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	return priv->rxq[ctxt].rcd;
+}
+
+static inline
+int hfi1_netdev_get_free_rmt_idx(struct hfi1_devdata *dd)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	return priv->rmt_start;
+}
+
+static inline
+void hfi1_netdev_set_free_rmt_idx(struct hfi1_devdata *dd, int rmt_idx)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	priv->rmt_start = rmt_idx;
+}
+
+u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
+			     struct cpumask *cpu_mask);
+
+void hfi1_netdev_enable_queues(struct hfi1_devdata *dd);
+void hfi1_netdev_disable_queues(struct hfi1_devdata *dd);
+int hfi1_netdev_rx_init(struct hfi1_devdata *dd);
+int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd);
+int hfi1_netdev_alloc(struct hfi1_devdata *dd);
+void hfi1_netdev_free(struct hfi1_devdata *dd);
+int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data);
+void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id);
+void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id);
+void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id);
+
+/* chip.c  */
+int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget);
+
+#endif /* HFI1_NETDEV_H */
diff --git a/drivers/infiniband/hw/hfi1/netdev_rx.c b/drivers/infiniband/hw/hfi1/netdev_rx.c
new file mode 100644
index 000000000000..63688e85e8da
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/netdev_rx.c
@@ -0,0 +1,481 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright(c) 2020 Intel Corporation.
+ *
+ */
+
+/*
+ * This file contains HFI1 support for netdev RX functionality
+ */
+
+#include "sdma.h"
+#include "verbs.h"
+#include "netdev.h"
+#include "hfi.h"
+
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <rdma/ib_verbs.h>
+
+static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_priv *priv,
+				  struct hfi1_ctxtdata *uctxt)
+{
+	unsigned int rcvctrl_ops;
+	struct hfi1_devdata *dd = priv->dd;
+	int ret;
+
+	uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions;
+	uctxt->do_interrupt = &handle_receive_interrupt_napi_sp;
+
+	/* Now allocate the RcvHdr queue and eager buffers. */
+	ret = hfi1_create_rcvhdrq(dd, uctxt);
+	if (ret)
+		goto done;
+
+	ret = hfi1_setup_eagerbufs(uctxt);
+	if (ret)
+		goto done;
+
+	clear_rcvhdrtail(uctxt);
+
+	rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS;
+	rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS;
+
+	if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))
+		rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
+	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))
+		rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
+	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
+		rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
+	if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
+		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
+
+	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
+done:
+	return ret;
+}
+
+static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd,
+				     struct hfi1_ctxtdata **ctxt)
+{
+	struct hfi1_ctxtdata *uctxt;
+	int ret;
+
+	if (dd->flags & HFI1_FROZEN)
+		return -EIO;
+
+	ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt);
+	if (ret < 0) {
+		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");
+		return -ENOMEM;
+	}
+
+	uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
+		HFI1_CAP_KGET(NODROP_RHQ_FULL) |
+		HFI1_CAP_KGET(NODROP_EGR_FULL) |
+		HFI1_CAP_KGET(DMA_RTAIL);
+	/* Netdev contexts are always NO_RDMA_RTAIL */
+	uctxt->fast_handler = handle_receive_interrupt_napi_fp;
+	uctxt->slow_handler = handle_receive_interrupt_napi_sp;
+	hfi1_set_seq_cnt(uctxt, 1);
+	uctxt->is_vnic = true;
+
+	hfi1_stats.sps_ctxts++;
+
+	dd_dev_info(dd, "created netdev context %d\n", uctxt->ctxt);
+	*ctxt = uctxt;
+
+	return 0;
+}
+
+static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd,
+					struct hfi1_ctxtdata *uctxt)
+{
+	flush_wc();
+
+	/*
+	 * Disable receive context and interrupt available, reset all
+	 * RcvCtxtCtrl bits to default values.
+	 */
+	hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
+		     HFI1_RCVCTRL_TIDFLOW_DIS |
+		     HFI1_RCVCTRL_INTRAVAIL_DIS |
+		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
+		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
+		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);
+
+	if (uctxt->msix_intr != CCE_NUM_MSIX_VECTORS)
+		msix_free_irq(dd, uctxt->msix_intr);
+
+	uctxt->msix_intr = CCE_NUM_MSIX_VECTORS;
+	uctxt->event_flags = 0;
+
+	hfi1_clear_tids(uctxt);
+	hfi1_clear_ctxt_pkey(dd, uctxt);
+
+	hfi1_stats.sps_ctxts--;
+
+	hfi1_free_ctxt(uctxt);
+}
+
+static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv,
+				  struct hfi1_ctxtdata **ctxt)
+{
+	int rc;
+	struct hfi1_devdata *dd = priv->dd;
+
+	rc = hfi1_netdev_allocate_ctxt(dd, ctxt);
+	if (rc) {
+		dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc);
+		return rc;
+	}
+
+	rc = hfi1_netdev_setup_ctxt(priv, *ctxt);
+	if (rc) {
+		dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc);
+		hfi1_netdev_deallocate_ctxt(dd, *ctxt);
+		*ctxt = NULL;
+	}
+
+	return rc;
+}
+
+/**
+ * hfi1_num_netdev_contexts - Count of netdev recv contexts to use.
+ * @dd: device on which to allocate netdev contexts
+ * @available_contexts: count of available receive contexts
+ * @cpu_mask: mask of possible cpus to include for contexts
+ *
+ * Return: count of physical cores on a node or the remaining available recv
+ * contexts for netdev recv context usage up to the maximum of
+ * HFI1_MAX_NETDEV_CTXTS.
+ * A value of 0 can be returned when acceleration is explicitly turned off,
+ * a memory allocation error occurs or when there are no available contexts.
+ *
+ */
+u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
+			     struct cpumask *cpu_mask)
+{
+	cpumask_var_t node_cpu_mask;
+	unsigned int available_cpus;
+
+	if (!HFI1_CAP_IS_KSET(AIP))
+		return 0;
+
+	/* Always give user contexts priority over netdev contexts */
+	if (available_contexts == 0) {
+		dd_dev_info(dd, "No receive contexts available for netdevs.\n");
+		return 0;
+	}
+
+	if (!zalloc_cpumask_var(&node_cpu_mask, GFP_KERNEL)) {
+		dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n");
+		return 0;
+	}
+
+	cpumask_and(node_cpu_mask, cpu_mask,
+		    cpumask_of_node(pcibus_to_node(dd->pcidev->bus)));
+
+	available_cpus = cpumask_weight(node_cpu_mask);
+
+	free_cpumask_var(node_cpu_mask);
+
+	return min3(available_cpus, available_contexts,
+		    (u32)HFI1_MAX_NETDEV_CTXTS);
+}
+
+static int hfi1_netdev_rxq_init(struct net_device *dev)
+{
+	int i;
+	int rc;
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
+	struct hfi1_devdata *dd = priv->dd;
+
+	priv->num_rx_q = dd->num_netdev_contexts;
+	priv->rxq = kcalloc_node(priv->num_rx_q, sizeof(struct hfi1_netdev_rxq),
+				 GFP_KERNEL, dd->node);
+
+	if (!priv->rxq) {
+		dd_dev_err(dd, "Unable to allocate netdev queue data\n");
+		return (-ENOMEM);
+	}
+
+	for (i = 0; i < priv->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+
+		rc = hfi1_netdev_allot_ctxt(priv, &rxq->rcd);
+		if (rc)
+			goto bail_context_irq_failure;
+
+		hfi1_rcd_get(rxq->rcd);
+		rxq->priv = priv;
+		rxq->rcd->napi = &rxq->napi;
+		dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n",
+			    i, rxq->rcd->ctxt);
+		/*
+		 * Disable BUSY_POLL on this NAPI as this is not supported
+		 * right now.
+		 */
+		set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);
+		netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi, 64);
+		rc = msix_netdev_request_rcd_irq(rxq->rcd);
+		if (rc)
+			goto bail_context_irq_failure;
+	}
+
+	return 0;
+
+bail_context_irq_failure:
+	dd_dev_err(dd, "Unable to allot receive context\n");
+	for (; i >= 0; i--) {
+		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+
+		if (rxq->rcd) {
+			hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
+			hfi1_rcd_put(rxq->rcd);
+			rxq->rcd = NULL;
+		}
+	}
+	kfree(priv->rxq);
+	priv->rxq = NULL;
+
+	return rc;
+}
+
+static void hfi1_netdev_rxq_deinit(struct net_device *dev)
+{
+	int i;
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
+	struct hfi1_devdata *dd = priv->dd;
+
+	for (i = 0; i < priv->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+
+		netif_napi_del(&rxq->napi);
+		hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
+		hfi1_rcd_put(rxq->rcd);
+		rxq->rcd = NULL;
+	}
+
+	kfree(priv->rxq);
+	priv->rxq = NULL;
+	priv->num_rx_q = 0;
+}
+
+static void enable_queues(struct hfi1_netdev_priv *priv)
+{
+	int i;
+
+	for (i = 0; i < priv->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+
+		dd_dev_info(priv->dd, "enabling queue %d on context %d\n", i,
+			    rxq->rcd->ctxt);
+		napi_enable(&rxq->napi);
+		hfi1_rcvctrl(priv->dd,
+			     HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB,
+			     rxq->rcd);
+	}
+}
+
+static void disable_queues(struct hfi1_netdev_priv *priv)
+{
+	int i;
+
+	msix_netdev_synchronize_irq(priv->dd);
+
+	for (i = 0; i < priv->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+
+		dd_dev_info(priv->dd, "disabling queue %d on context %d\n", i,
+			    rxq->rcd->ctxt);
+
+		/* wait for napi if it was scheduled */
+		hfi1_rcvctrl(priv->dd,
+			     HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS,
+			     rxq->rcd);
+		napi_synchronize(&rxq->napi);
+		napi_disable(&rxq->napi);
+	}
+}
+
+/**
+ * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time,
+ * it allocates receive queue data and calls netif_napi_add
+ * for each queue.
+ *
+ * @dd: hfi1 dev data
+ */
+int hfi1_netdev_rx_init(struct hfi1_devdata *dd)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	int res;
+
+	if (atomic_fetch_inc(&priv->netdevs))
+		return 0;
+
+	mutex_lock(&hfi1_mutex);
+	init_dummy_netdev(dd->dummy_netdev);
+	res = hfi1_netdev_rxq_init(dd->dummy_netdev);
+	mutex_unlock(&hfi1_mutex);
+	return res;
+}
+
+/**
+ * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0
+ * napi is deleted and receive queses memory is freed.
+ *
+ * @dd: hfi1 dev data
+ */
+int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	/* destroy the RX queues only if it is the last netdev going away */
+	if (atomic_fetch_add_unless(&priv->netdevs, -1, 0) == 1) {
+		mutex_lock(&hfi1_mutex);
+		hfi1_netdev_rxq_deinit(dd->dummy_netdev);
+		mutex_unlock(&hfi1_mutex);
+	}
+
+	return 0;
+}
+
+/**
+ * hfi1_netdev_alloc - Allocates netdev and private data. It is required
+ * because RMT index and MSI-X interrupt can be set only
+ * during driver initialization.
+ *
+ * @dd: hfi1 dev data
+ */
+int hfi1_netdev_alloc(struct hfi1_devdata *dd)
+{
+	struct hfi1_netdev_priv *priv;
+	const int netdev_size = sizeof(*dd->dummy_netdev) +
+		sizeof(struct hfi1_netdev_priv);
+
+	dd_dev_info(dd, "allocating netdev size %d\n", netdev_size);
+	dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node);
+
+	if (!dd->dummy_netdev)
+		return -ENOMEM;
+
+	priv = hfi1_netdev_priv(dd->dummy_netdev);
+	priv->dd = dd;
+	xa_init(&priv->dev_tbl);
+	atomic_set(&priv->enabled, 0);
+	atomic_set(&priv->netdevs, 0);
+
+	return 0;
+}
+
+void hfi1_netdev_free(struct hfi1_devdata *dd)
+{
+	if (dd->dummy_netdev) {
+		dd_dev_info(dd, "hfi1 netdev freed\n");
+		free_netdev(dd->dummy_netdev);
+		dd->dummy_netdev = NULL;
+	}
+}
+
+/**
+ * hfi1_netdev_enable_queues - This is napi enable function.
+ * It enables napi objects associated with queues.
+ * When at least one device has called it it increments atomic counter.
+ * Disable function decrements counter and when it is 0,
+ * calls napi_disable for every queue.
+ *
+ * @dd: hfi1 dev data
+ */
+void hfi1_netdev_enable_queues(struct hfi1_devdata *dd)
+{
+	struct hfi1_netdev_priv *priv;
+
+	if (!dd->dummy_netdev)
+		return;
+
+	priv = hfi1_netdev_priv(dd->dummy_netdev);
+	if (atomic_fetch_inc(&priv->enabled))
+		return;
+
+	mutex_lock(&hfi1_mutex);
+	enable_queues(priv);
+	mutex_unlock(&hfi1_mutex);
+}
+
+void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)
+{
+	struct hfi1_netdev_priv *priv;
+
+	if (!dd->dummy_netdev)
+		return;
+
+	priv = hfi1_netdev_priv(dd->dummy_netdev);
+	if (atomic_dec_if_positive(&priv->enabled))
+		return;
+
+	mutex_lock(&hfi1_mutex);
+	disable_queues(priv);
+	mutex_unlock(&hfi1_mutex);
+}
+
+/**
+ * hfi1_netdev_add_data - Registers data with unique identifier
+ * to be requested later this is needed for VNIC and IPoIB VLANs
+ * implementations.
+ * This call is protected by mutex idr_lock.
+ *
+ * @dd: hfi1 dev data
+ * @id: requested integer id up to INT_MAX
+ * @data: data to be associated with index
+ */
+int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	return xa_insert(&priv->dev_tbl, id, data, GFP_NOWAIT);
+}
+
+/**
+ * hfi1_netdev_remove_data - Removes data with previously given id.
+ * Returns the reference to removed entry.
+ *
+ * @dd: hfi1 dev data
+ * @id: requested integer id up to INT_MAX
+ */
+void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	return xa_erase(&priv->dev_tbl, id);
+}
+
+/**
+ * hfi1_netdev_get_data - Gets data with given id
+ *
+ * @dd: hfi1 dev data
+ * @id: requested integer id up to INT_MAX
+ */
+void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+
+	return xa_load(&priv->dev_tbl, id);
+}
+
+/**
+ * hfi1_netdev_get_first_dat - Gets first entry with greater or equal id.
+ *
+ * @dd: hfi1 dev data
+ * @id: requested integer id up to INT_MAX
+ */
+void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id)
+{
+	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	unsigned long index = *start_id;
+	void *ret;
+
+	ret = xa_find(&priv->dev_tbl, &index, UINT_MAX, XA_PRESENT);
+	*start_id = (int)index;
+	return ret;
+}
diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c
index f8e733aa3bb8..0c2ae9f7b3e8 100644
--- a/drivers/infiniband/hw/hfi1/qp.c
+++ b/drivers/infiniband/hw/hfi1/qp.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2019 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -186,15 +186,6 @@ static void flush_iowait(struct rvt_qp *qp)
 	write_sequnlock_irqrestore(lock, flags);
 }
 
-static inline int opa_mtu_enum_to_int(int mtu)
-{
-	switch (mtu) {
-	case OPA_MTU_8192:  return 8192;
-	case OPA_MTU_10240: return 10240;
-	default:            return -1;
-	}
-}
-
 /**
  * This function is what we would push to the core layer if we wanted to be a
  * "first class citizen".  Instead we hide this here and rely on Verbs ULPs
@@ -202,15 +193,10 @@ static inline int opa_mtu_enum_to_int(int mtu)
  */
 static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu)
 {
-	int val;
-
 	/* Constraining 10KB packets to 8KB packets */
 	if (mtu == (enum ib_mtu)OPA_MTU_10240)
 		mtu = OPA_MTU_8192;
-	val = opa_mtu_enum_to_int((int)mtu);
-	if (val > 0)
-		return val;
-	return ib_mtu_enum_to_int(mtu);
+	return opa_mtu_enum_to_int((enum opa_mtu)mtu);
 }
 
 int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c
index 8a2e0d9351e9..243b4ba0b6f6 100644
--- a/drivers/infiniband/hw/hfi1/tid_rdma.c
+++ b/drivers/infiniband/hw/hfi1/tid_rdma.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
 /*
- * Copyright(c) 2018 Intel Corporation.
+ * Copyright(c) 2018 - 2020 Intel Corporation.
  *
  */
 
@@ -194,7 +194,7 @@ void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 
-	p->qp = (kdeth_qp << 16) | priv->rcd->ctxt;
+	p->qp = (RVT_KDETH_QP_PREFIX << 16) | priv->rcd->ctxt;
 	p->max_len = TID_RDMA_MAX_SEGMENT_SIZE;
 	p->jkey = priv->rcd->jkey;
 	p->max_read = TID_RDMA_MAX_READ_SEGS_PER_REQ;
diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c
index 9a3d236bcc88..b219ea90fd6f 100644
--- a/drivers/infiniband/hw/hfi1/trace.c
+++ b/drivers/infiniband/hw/hfi1/trace.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -47,6 +47,7 @@
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 #include "exp_rcv.h"
+#include "ipoib.h"
 
 static u8 __get_ib_hdr_len(struct ib_header *hdr)
 {
@@ -126,6 +127,7 @@ const char *hfi1_trace_get_packet_l2_str(u8 l2)
 #define RETH_PRN "reth vaddr:0x%.16llx rkey:0x%.8x dlen:0x%.8x"
 #define AETH_PRN "aeth syn:0x%.2x %s msn:0x%.8x"
 #define DETH_PRN "deth qkey:0x%.8x sqpn:0x%.6x"
+#define DETH_ENTROPY_PRN "deth qkey:0x%.8x sqpn:0x%.6x entropy:0x%.2x"
 #define IETH_PRN "ieth rkey:0x%.8x"
 #define ATOMICACKETH_PRN "origdata:%llx"
 #define ATOMICETH_PRN "vaddr:0x%llx rkey:0x%.8x sdata:%llx cdata:%llx"
@@ -444,6 +446,12 @@ const char *parse_everbs_hdrs(
 		break;
 	/* deth */
 	case OP(UD, SEND_ONLY):
+		trace_seq_printf(p, DETH_ENTROPY_PRN,
+				 be32_to_cpu(eh->ud.deth[0]),
+				 be32_to_cpu(eh->ud.deth[1]) & RVT_QPN_MASK,
+				 be32_to_cpu(eh->ud.deth[1]) >>
+					     HFI1_IPOIB_ENTROPY_SHIFT);
+		break;
 	case OP(UD, SEND_ONLY_WITH_IMMEDIATE):
 		trace_seq_printf(p, DETH_PRN,
 				 be32_to_cpu(eh->ud.deth[0]),
@@ -512,6 +520,38 @@ u16 hfi1_trace_get_tid_idx(u32 ent)
 	return EXP_TID_GET(ent, IDX);
 }
 
+struct hfi1_ctxt_hist {
+	atomic_t count;
+	atomic_t data[255];
+};
+
+struct hfi1_ctxt_hist hist = {
+	.count = ATOMIC_INIT(0)
+};
+
+const char *hfi1_trace_print_rsm_hist(struct trace_seq *p, unsigned int ctxt)
+{
+	int i, len = ARRAY_SIZE(hist.data);
+	const char *ret = trace_seq_buffer_ptr(p);
+	unsigned long packet_count = atomic_fetch_inc(&hist.count);
+
+	trace_seq_printf(p, "packet[%lu]", packet_count);
+	for (i = 0; i < len; ++i) {
+		unsigned long val;
+		atomic_t *count = &hist.data[i];
+
+		if (ctxt == i)
+			val = atomic_fetch_inc(count);
+		else
+			val = atomic_read(count);
+
+		if (val)
+			trace_seq_printf(p, "(%d:%lu)", i, val);
+	}
+	trace_seq_putc(p, 0);
+	return ret;
+}
+
 __hfi1_trace_fn(AFFINITY);
 __hfi1_trace_fn(PKT);
 __hfi1_trace_fn(PROC);
diff --git a/drivers/infiniband/hw/hfi1/trace_ctxts.h b/drivers/infiniband/hw/hfi1/trace_ctxts.h
index b5fc5c6cd52f..d8c168dc3ea8 100644
--- a/drivers/infiniband/hw/hfi1/trace_ctxts.h
+++ b/drivers/infiniband/hw/hfi1/trace_ctxts.h
@@ -1,5 +1,5 @@
 /*
-* Copyright(c) 2015, 2016 Intel Corporation.
+* Copyright(c) 2015 - 2020 Intel Corporation.
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
@@ -138,6 +138,15 @@ TRACE_EVENT(hfi1_ctxt_info,
 		      )
 );
 
+const char *hfi1_trace_print_rsm_hist(struct trace_seq *p, unsigned int ctxt);
+TRACE_EVENT(ctxt_rsm_hist,
+	    TP_PROTO(unsigned int ctxt),
+	    TP_ARGS(ctxt),
+	    TP_STRUCT__entry(__field(unsigned int, ctxt)),
+	    TP_fast_assign(__entry->ctxt = ctxt;),
+	    TP_printk("%s", hfi1_trace_print_rsm_hist(p, __entry->ctxt))
+);
+
 #endif /* __HFI1_TRACE_CTXTS_H */
 
 #undef TRACE_INCLUDE_PATH
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 2f6323ad9c59..30865635b449 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015 - 2018 Intel Corporation.
+ * Copyright(c) 2015 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -66,6 +66,7 @@
 #include "vnic.h"
 #include "fault.h"
 #include "affinity.h"
+#include "ipoib.h"
 
 static unsigned int hfi1_lkey_table_size = 16;
 module_param_named(lkey_table_size, hfi1_lkey_table_size, uint,
@@ -1342,7 +1343,7 @@ static void hfi1_fill_device_attr(struct hfi1_devdata *dd)
 			IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_RC_RNR_NAK_GEN |
 			IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_SRQ_RESIZE |
 			IB_DEVICE_MEM_MGT_EXTENSIONS |
-			IB_DEVICE_RDMA_NETDEV_OPA_VNIC;
+			IB_DEVICE_RDMA_NETDEV_OPA;
 	rdi->dparms.props.page_size_cap = PAGE_SIZE;
 	rdi->dparms.props.vendor_id = dd->oui1 << 16 | dd->oui2 << 8 | dd->oui3;
 	rdi->dparms.props.vendor_part_id = dd->pcidev->device;
@@ -1360,7 +1361,6 @@ static void hfi1_fill_device_attr(struct hfi1_devdata *dd)
 	rdi->dparms.props.max_cq = hfi1_max_cqs;
 	rdi->dparms.props.max_ah = hfi1_max_ahs;
 	rdi->dparms.props.max_cqe = hfi1_max_cqes;
-	rdi->dparms.props.max_map_per_fmr = 32767;
 	rdi->dparms.props.max_pd = hfi1_max_pds;
 	rdi->dparms.props.max_qp_rd_atom = HFI1_MAX_RDMA_ATOMIC;
 	rdi->dparms.props.max_qp_init_rd_atom = 255;
@@ -1439,6 +1439,8 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,
 				      4096 : hfi1_max_mtu), IB_MTU_4096);
 	props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu :
 		mtu_to_enum(ppd->ibmtu, IB_MTU_4096);
+	props->phys_mtu = HFI1_CAP_IS_KSET(AIP) ? hfi1_max_mtu :
+				ib_mtu_enum_to_int(props->max_mtu);
 
 	return 0;
 }
@@ -1793,6 +1795,7 @@ static const struct ib_device_ops hfi1_dev_ops = {
 	.modify_device = modify_device,
 	/* keep process mad in the driver */
 	.process_mad = hfi1_process_mad,
+	.rdma_netdev_get_params = hfi1_ipoib_rn_get_params,
 };
 
 /**
@@ -1863,9 +1866,8 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	dd->verbs_dev.rdi.dparms.qpn_start = 0;
 	dd->verbs_dev.rdi.dparms.qpn_inc = 1;
 	dd->verbs_dev.rdi.dparms.qos_shift = dd->qos_shift;
-	dd->verbs_dev.rdi.dparms.qpn_res_start = kdeth_qp << 16;
-	dd->verbs_dev.rdi.dparms.qpn_res_end =
-	dd->verbs_dev.rdi.dparms.qpn_res_start + 65535;
+	dd->verbs_dev.rdi.dparms.qpn_res_start = RVT_KDETH_QP_BASE;
+	dd->verbs_dev.rdi.dparms.qpn_res_end = RVT_AIP_QP_MAX;
 	dd->verbs_dev.rdi.dparms.max_rdma_atomic = HFI1_MAX_RDMA_ATOMIC;
 	dd->verbs_dev.rdi.dparms.psn_mask = PSN_MASK;
 	dd->verbs_dev.rdi.dparms.psn_shift = PSN_SHIFT;
diff --git a/drivers/infiniband/hw/hfi1/vnic.h b/drivers/infiniband/hw/hfi1/vnic.h
index 5ae781514e32..66150a13f374 100644
--- a/drivers/infiniband/hw/hfi1/vnic.h
+++ b/drivers/infiniband/hw/hfi1/vnic.h
@@ -1,7 +1,7 @@
 #ifndef _HFI1_VNIC_H
 #define _HFI1_VNIC_H
 /*
- * Copyright(c) 2017 Intel Corporation.
+ * Copyright(c) 2017 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -69,6 +69,7 @@
 #define HFI1_VNIC_SC_SHIFT      4
 
 #define HFI1_VNIC_MAX_QUEUE 16
+#define HFI1_NUM_VNIC_CTXT 8
 
 /**
  * struct hfi1_vnic_sdma - VNIC per Tx ring SDMA information
@@ -104,7 +105,6 @@ struct hfi1_vnic_rx_queue {
 	struct hfi1_vnic_vport_info *vinfo;
 	struct net_device           *netdev;
 	struct napi_struct           napi;
-	struct sk_buff_head          skbq;
 };
 
 /**
@@ -146,7 +146,6 @@ struct hfi1_vnic_vport_info {
 
 /* vnic hfi1 internal functions */
 void hfi1_vnic_setup(struct hfi1_devdata *dd);
-void hfi1_vnic_cleanup(struct hfi1_devdata *dd);
 int hfi1_vnic_txreq_init(struct hfi1_devdata *dd);
 void hfi1_vnic_txreq_deinit(struct hfi1_devdata *dd);
 
diff --git a/drivers/infiniband/hw/hfi1/vnic_main.c b/drivers/infiniband/hw/hfi1/vnic_main.c
index 6b14581b9965..a90824de0f57 100644
--- a/drivers/infiniband/hw/hfi1/vnic_main.c
+++ b/drivers/infiniband/hw/hfi1/vnic_main.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2017 - 2018 Intel Corporation.
+ * Copyright(c) 2017 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -53,6 +53,7 @@
 #include <linux/if_vlan.h>
 
 #include "vnic.h"
+#include "netdev.h"
 
 #define HFI_TX_TIMEOUT_MS 1000
 
@@ -62,114 +63,6 @@
 
 static DEFINE_SPINLOCK(vport_cntr_lock);
 
-static int setup_vnic_ctxt(struct hfi1_devdata *dd, struct hfi1_ctxtdata *uctxt)
-{
-	unsigned int rcvctrl_ops = 0;
-	int ret;
-
-	uctxt->do_interrupt = &handle_receive_interrupt;
-
-	/* Now allocate the RcvHdr queue and eager buffers. */
-	ret = hfi1_create_rcvhdrq(dd, uctxt);
-	if (ret)
-		goto done;
-
-	ret = hfi1_setup_eagerbufs(uctxt);
-	if (ret)
-		goto done;
-
-	if (hfi1_rcvhdrtail_kvaddr(uctxt))
-		clear_rcvhdrtail(uctxt);
-
-	rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;
-	rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_ENB;
-
-	if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))
-		rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
-	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))
-		rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
-	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
-		rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
-	if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
-		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
-
-	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
-done:
-	return ret;
-}
-
-static int allocate_vnic_ctxt(struct hfi1_devdata *dd,
-			      struct hfi1_ctxtdata **vnic_ctxt)
-{
-	struct hfi1_ctxtdata *uctxt;
-	int ret;
-
-	if (dd->flags & HFI1_FROZEN)
-		return -EIO;
-
-	ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt);
-	if (ret < 0) {
-		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");
-		return -ENOMEM;
-	}
-
-	uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
-			HFI1_CAP_KGET(NODROP_RHQ_FULL) |
-			HFI1_CAP_KGET(NODROP_EGR_FULL) |
-			HFI1_CAP_KGET(DMA_RTAIL);
-	uctxt->seq_cnt = 1;
-	uctxt->is_vnic = true;
-
-	msix_request_rcd_irq(uctxt);
-
-	hfi1_stats.sps_ctxts++;
-	dd_dev_dbg(dd, "created vnic context %d\n", uctxt->ctxt);
-	*vnic_ctxt = uctxt;
-
-	return 0;
-}
-
-static void deallocate_vnic_ctxt(struct hfi1_devdata *dd,
-				 struct hfi1_ctxtdata *uctxt)
-{
-	dd_dev_dbg(dd, "closing vnic context %d\n", uctxt->ctxt);
-	flush_wc();
-
-	/*
-	 * Disable receive context and interrupt available, reset all
-	 * RcvCtxtCtrl bits to default values.
-	 */
-	hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
-		     HFI1_RCVCTRL_TIDFLOW_DIS |
-		     HFI1_RCVCTRL_INTRAVAIL_DIS |
-		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
-		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
-		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);
-
-	/* msix_intr will always be > 0, only clean up if this is true */
-	if (uctxt->msix_intr)
-		msix_free_irq(dd, uctxt->msix_intr);
-
-	uctxt->event_flags = 0;
-
-	hfi1_clear_tids(uctxt);
-	hfi1_clear_ctxt_pkey(dd, uctxt);
-
-	hfi1_stats.sps_ctxts--;
-
-	hfi1_free_ctxt(uctxt);
-}
-
-void hfi1_vnic_setup(struct hfi1_devdata *dd)
-{
-	xa_init(&dd->vnic.vesws);
-}
-
-void hfi1_vnic_cleanup(struct hfi1_devdata *dd)
-{
-	WARN_ON(!xa_empty(&dd->vnic.vesws));
-}
-
 #define SUM_GRP_COUNTERS(stats, qstats, x_grp) do {            \
 		u64 *src64, *dst64;                            \
 		for (src64 = &qstats->x_grp.unicast,           \
@@ -179,6 +72,9 @@ void hfi1_vnic_cleanup(struct hfi1_devdata *dd)
 		}                                              \
 	} while (0)
 
+#define VNIC_MASK (0xFF)
+#define VNIC_ID(val) ((1ull << 24) | ((val) & VNIC_MASK))
+
 /* hfi1_vnic_update_stats - update statistics */
 static void hfi1_vnic_update_stats(struct hfi1_vnic_vport_info *vinfo,
 				   struct opa_vnic_stats *stats)
@@ -454,71 +350,25 @@ static inline int hfi1_vnic_decap_skb(struct hfi1_vnic_rx_queue *rxq,
 	return rc;
 }
 
-static inline struct sk_buff *hfi1_vnic_get_skb(struct hfi1_vnic_rx_queue *rxq)
+static struct hfi1_vnic_vport_info *get_vnic_port(struct hfi1_devdata *dd,
+						  int vesw_id)
 {
-	unsigned char *pad_info;
-	struct sk_buff *skb;
+	int vnic_id = VNIC_ID(vesw_id);
 
-	skb = skb_dequeue(&rxq->skbq);
-	if (unlikely(!skb))
-		return NULL;
-
-	/* remove tail padding and icrc */
-	pad_info = skb->data + skb->len - 1;
-	skb_trim(skb, (skb->len - OPA_VNIC_ICRC_TAIL_LEN -
-		       ((*pad_info) & 0x7)));
-
-	return skb;
+	return hfi1_netdev_get_data(dd, vnic_id);
 }
 
-/* hfi1_vnic_handle_rx - handle skb receive */
-static void hfi1_vnic_handle_rx(struct hfi1_vnic_rx_queue *rxq,
-				int *work_done, int work_to_do)
+static struct hfi1_vnic_vport_info *get_first_vnic_port(struct hfi1_devdata *dd)
 {
-	struct hfi1_vnic_vport_info *vinfo = rxq->vinfo;
-	struct sk_buff *skb;
-	int rc;
-
-	while (1) {
-		if (*work_done >= work_to_do)
-			break;
-
-		skb = hfi1_vnic_get_skb(rxq);
-		if (unlikely(!skb))
-			break;
-
-		rc = hfi1_vnic_decap_skb(rxq, skb);
-		/* update rx counters */
-		hfi1_vnic_update_rx_counters(vinfo, rxq->idx, skb, rc);
-		if (unlikely(rc)) {
-			dev_kfree_skb_any(skb);
-			continue;
-		}
-
-		skb_checksum_none_assert(skb);
-		skb->protocol = eth_type_trans(skb, rxq->netdev);
-
-		napi_gro_receive(&rxq->napi, skb);
-		(*work_done)++;
-	}
-}
-
-/* hfi1_vnic_napi - napi receive polling callback function */
-static int hfi1_vnic_napi(struct napi_struct *napi, int budget)
-{
-	struct hfi1_vnic_rx_queue *rxq = container_of(napi,
-					      struct hfi1_vnic_rx_queue, napi);
-	struct hfi1_vnic_vport_info *vinfo = rxq->vinfo;
-	int work_done = 0;
+	struct hfi1_vnic_vport_info *vinfo;
+	int next_id = VNIC_ID(0);
 
-	v_dbg("napi %d budget %d\n", rxq->idx, budget);
-	hfi1_vnic_handle_rx(rxq, &work_done, budget);
+	vinfo = hfi1_netdev_get_first_data(dd, &next_id);
 
-	v_dbg("napi %d work_done %d\n", rxq->idx, work_done);
-	if (work_done < budget)
-		napi_complete(napi);
+	if (next_id > VNIC_ID(VNIC_MASK))
+		return NULL;
 
-	return work_done;
+	return vinfo;
 }
 
 void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
@@ -527,13 +377,14 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
 	struct hfi1_vnic_vport_info *vinfo = NULL;
 	struct hfi1_vnic_rx_queue *rxq;
 	struct sk_buff *skb;
-	int l4_type, vesw_id = -1;
+	int l4_type, vesw_id = -1, rc;
 	u8 q_idx;
+	unsigned char *pad_info;
 
 	l4_type = hfi1_16B_get_l4(packet->ebuf);
 	if (likely(l4_type == OPA_16B_L4_ETHR)) {
 		vesw_id = HFI1_VNIC_GET_VESWID(packet->ebuf);
-		vinfo = xa_load(&dd->vnic.vesws, vesw_id);
+		vinfo = get_vnic_port(dd, vesw_id);
 
 		/*
 		 * In case of invalid vesw id, count the error on
@@ -541,10 +392,8 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
 		 */
 		if (unlikely(!vinfo)) {
 			struct hfi1_vnic_vport_info *vinfo_tmp;
-			unsigned long index = 0;
 
-			vinfo_tmp = xa_find(&dd->vnic.vesws, &index, ULONG_MAX,
-					XA_PRESENT);
+			vinfo_tmp = get_first_vnic_port(dd);
 			if (vinfo_tmp) {
 				spin_lock(&vport_cntr_lock);
 				vinfo_tmp->stats[0].netstats.rx_nohandler++;
@@ -563,12 +412,6 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
 	rxq = &vinfo->rxq[q_idx];
 	if (unlikely(!netif_oper_up(vinfo->netdev))) {
 		vinfo->stats[q_idx].rx_drop_state++;
-		skb_queue_purge(&rxq->skbq);
-		return;
-	}
-
-	if (unlikely(skb_queue_len(&rxq->skbq) > HFI1_VNIC_RCV_Q_SIZE)) {
-		vinfo->stats[q_idx].netstats.rx_fifo_errors++;
 		return;
 	}
 
@@ -580,62 +423,65 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
 
 	memcpy(skb->data, packet->ebuf, packet->tlen);
 	skb_put(skb, packet->tlen);
-	skb_queue_tail(&rxq->skbq, skb);
 
-	if (napi_schedule_prep(&rxq->napi)) {
-		v_dbg("napi %d scheduling\n", q_idx);
-		__napi_schedule(&rxq->napi);
+	pad_info = skb->data + skb->len - 1;
+	skb_trim(skb, (skb->len - OPA_VNIC_ICRC_TAIL_LEN -
+		       ((*pad_info) & 0x7)));
+
+	rc = hfi1_vnic_decap_skb(rxq, skb);
+
+	/* update rx counters */
+	hfi1_vnic_update_rx_counters(vinfo, rxq->idx, skb, rc);
+	if (unlikely(rc)) {
+		dev_kfree_skb_any(skb);
+		return;
 	}
+
+	skb_checksum_none_assert(skb);
+	skb->protocol = eth_type_trans(skb, rxq->netdev);
+
+	napi_gro_receive(&rxq->napi, skb);
 }
 
 static int hfi1_vnic_up(struct hfi1_vnic_vport_info *vinfo)
 {
 	struct hfi1_devdata *dd = vinfo->dd;
 	struct net_device *netdev = vinfo->netdev;
-	int i, rc;
+	int rc;
 
 	/* ensure virtual eth switch id is valid */
 	if (!vinfo->vesw_id)
 		return -EINVAL;
 
-	rc = xa_insert(&dd->vnic.vesws, vinfo->vesw_id, vinfo, GFP_KERNEL);
+	rc = hfi1_netdev_add_data(dd, VNIC_ID(vinfo->vesw_id), vinfo);
 	if (rc < 0)
 		return rc;
 
-	for (i = 0; i < vinfo->num_rx_q; i++) {
-		struct hfi1_vnic_rx_queue *rxq = &vinfo->rxq[i];
-
-		skb_queue_head_init(&rxq->skbq);
-		napi_enable(&rxq->napi);
-	}
+	rc = hfi1_netdev_rx_init(dd);
+	if (rc)
+		goto err_remove;
 
 	netif_carrier_on(netdev);
 	netif_tx_start_all_queues(netdev);
 	set_bit(HFI1_VNIC_UP, &vinfo->flags);
 
 	return 0;
+
+err_remove:
+	hfi1_netdev_remove_data(dd, VNIC_ID(vinfo->vesw_id));
+	return rc;
 }
 
 static void hfi1_vnic_down(struct hfi1_vnic_vport_info *vinfo)
 {
 	struct hfi1_devdata *dd = vinfo->dd;
-	u8 i;
 
 	clear_bit(HFI1_VNIC_UP, &vinfo->flags);
 	netif_carrier_off(vinfo->netdev);
 	netif_tx_disable(vinfo->netdev);
-	xa_erase(&dd->vnic.vesws, vinfo->vesw_id);
-
-	/* ensure irqs see the change */
-	msix_vnic_synchronize_irq(dd);
+	hfi1_netdev_remove_data(dd, VNIC_ID(vinfo->vesw_id));
 
-	/* remove unread skbs */
-	for (i = 0; i < vinfo->num_rx_q; i++) {
-		struct hfi1_vnic_rx_queue *rxq = &vinfo->rxq[i];
-
-		napi_disable(&rxq->napi);
-		skb_queue_purge(&rxq->skbq);
-	}
+	hfi1_netdev_rx_destroy(dd);
 }
 
 static int hfi1_netdev_open(struct net_device *netdev)
@@ -660,70 +506,31 @@ static int hfi1_netdev_close(struct net_device *netdev)
 	return 0;
 }
 
-static int hfi1_vnic_allot_ctxt(struct hfi1_devdata *dd,
-				struct hfi1_ctxtdata **vnic_ctxt)
-{
-	int rc;
-
-	rc = allocate_vnic_ctxt(dd, vnic_ctxt);
-	if (rc) {
-		dd_dev_err(dd, "vnic ctxt alloc failed %d\n", rc);
-		return rc;
-	}
-
-	rc = setup_vnic_ctxt(dd, *vnic_ctxt);
-	if (rc) {
-		dd_dev_err(dd, "vnic ctxt setup failed %d\n", rc);
-		deallocate_vnic_ctxt(dd, *vnic_ctxt);
-		*vnic_ctxt = NULL;
-	}
-
-	return rc;
-}
-
 static int hfi1_vnic_init(struct hfi1_vnic_vport_info *vinfo)
 {
 	struct hfi1_devdata *dd = vinfo->dd;
-	int i, rc = 0;
+	int rc = 0;
 
 	mutex_lock(&hfi1_mutex);
-	if (!dd->vnic.num_vports) {
+	if (!dd->vnic_num_vports) {
 		rc = hfi1_vnic_txreq_init(dd);
 		if (rc)
 			goto txreq_fail;
 	}
 
-	for (i = dd->vnic.num_ctxt; i < vinfo->num_rx_q; i++) {
-		rc = hfi1_vnic_allot_ctxt(dd, &dd->vnic.ctxt[i]);
-		if (rc)
-			break;
-		hfi1_rcd_get(dd->vnic.ctxt[i]);
-		dd->vnic.ctxt[i]->vnic_q_idx = i;
-	}
-
-	if (i < vinfo->num_rx_q) {
-		/*
-		 * If required amount of contexts is not
-		 * allocated successfully then remaining contexts
-		 * are released.
-		 */
-		while (i-- > dd->vnic.num_ctxt) {
-			deallocate_vnic_ctxt(dd, dd->vnic.ctxt[i]);
-			hfi1_rcd_put(dd->vnic.ctxt[i]);
-			dd->vnic.ctxt[i] = NULL;
-		}
+	rc = hfi1_netdev_rx_init(dd);
+	if (rc) {
+		dd_dev_err(dd, "Unable to initialize netdev contexts\n");
 		goto alloc_fail;
 	}
 
-	if (dd->vnic.num_ctxt != i) {
-		dd->vnic.num_ctxt = i;
-		hfi1_init_vnic_rsm(dd);
-	}
+	hfi1_init_vnic_rsm(dd);
 
-	dd->vnic.num_vports++;
+	dd->vnic_num_vports++;
 	hfi1_vnic_sdma_init(vinfo);
+
 alloc_fail:
-	if (!dd->vnic.num_vports)
+	if (!dd->vnic_num_vports)
 		hfi1_vnic_txreq_deinit(dd);
 txreq_fail:
 	mutex_unlock(&hfi1_mutex);
@@ -733,20 +540,14 @@ txreq_fail:
 static void hfi1_vnic_deinit(struct hfi1_vnic_vport_info *vinfo)
 {
 	struct hfi1_devdata *dd = vinfo->dd;
-	int i;
 
 	mutex_lock(&hfi1_mutex);
-	if (--dd->vnic.num_vports == 0) {
-		for (i = 0; i < dd->vnic.num_ctxt; i++) {
-			deallocate_vnic_ctxt(dd, dd->vnic.ctxt[i]);
-			hfi1_rcd_put(dd->vnic.ctxt[i]);
-			dd->vnic.ctxt[i] = NULL;
-		}
+	if (--dd->vnic_num_vports == 0) {
 		hfi1_deinit_vnic_rsm(dd);
-		dd->vnic.num_ctxt = 0;
 		hfi1_vnic_txreq_deinit(dd);
 	}
 	mutex_unlock(&hfi1_mutex);
+	hfi1_netdev_rx_destroy(dd);
 }
 
 static void hfi1_vnic_set_vesw_id(struct net_device *netdev, int id)
@@ -804,7 +605,7 @@ struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,
 	struct rdma_netdev *rn;
 	int i, size, rc;
 
-	if (!dd->num_vnic_contexts)
+	if (!dd->num_netdev_contexts)
 		return ERR_PTR(-ENOMEM);
 
 	if (!port_num || (port_num > dd->num_pports))
@@ -815,15 +616,16 @@ struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,
 
 	size = sizeof(struct opa_vnic_rdma_netdev) + sizeof(*vinfo);
 	netdev = alloc_netdev_mqs(size, name, name_assign_type, setup,
-				  dd->num_sdma, dd->num_vnic_contexts);
+				  chip_sdma_engines(dd),
+				  dd->num_netdev_contexts);
 	if (!netdev)
 		return ERR_PTR(-ENOMEM);
 
 	rn = netdev_priv(netdev);
 	vinfo = opa_vnic_dev_priv(netdev);
 	vinfo->dd = dd;
-	vinfo->num_tx_q = dd->num_sdma;
-	vinfo->num_rx_q = dd->num_vnic_contexts;
+	vinfo->num_tx_q = chip_sdma_engines(dd);
+	vinfo->num_rx_q = dd->num_netdev_contexts;
 	vinfo->netdev = netdev;
 	rn->free_rdma_netdev = hfi1_vnic_free_rn;
 	rn->set_id = hfi1_vnic_set_vesw_id;
@@ -841,7 +643,6 @@ struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,
 		rxq->idx = i;
 		rxq->vinfo = vinfo;
 		rxq->netdev = netdev;
-		netif_napi_add(netdev, &rxq->napi, hfi1_vnic_napi, 64);
 	}
 
 	rc = hfi1_vnic_init(vinfo);
diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig
index 4921c1e40ccd..18d10ebf900b 100644
--- a/drivers/infiniband/hw/hns/Kconfig
+++ b/drivers/infiniband/hw/hns/Kconfig
@@ -4,7 +4,7 @@ config INFINIBAND_HNS
 	depends on NET_VENDOR_HISILICON
 	depends on ARM64 || (COMPILE_TEST && 64BIT)
 	depends on (HNS_DSAF && HNS_ENET) || HNS3
-	---help---
+	help
 	  This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine
 	  is used in Hisilicon Hip06 and more further ICT SoC based on
 	  platform device.
@@ -15,7 +15,7 @@ config INFINIBAND_HNS_HIP06
 	bool "Hisilicon Hip06 Family RoCE support"
 	depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET
 	depends on INFINIBAND_HNS=m || (HNS_DSAF=y && HNS_ENET=y)
-	---help---
+	help
 	  RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and
 	  Hip07 SoC. These RoCE engines are platform devices.
 
@@ -26,7 +26,7 @@ config INFINIBAND_HNS_HIP08
 	bool "Hisilicon Hip08 Family RoCE support"
 	depends on INFINIBAND_HNS && PCI && HNS3
 	depends on INFINIBAND_HNS=m || HNS3=y
-	---help---
+	help
 	  RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip08 SoC.
 	  The RoCE engine is a PCI device.
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c
index 8a522e14ef62..5b2f9314edd3 100644
--- a/drivers/infiniband/hw/hns/hns_roce_ah.c
+++ b/drivers/infiniband/hw/hns/hns_roce_ah.c
@@ -39,13 +39,14 @@
 #define HNS_ROCE_VLAN_SL_BIT_MASK	7
 #define HNS_ROCE_VLAN_SL_SHIFT		13
 
-int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
-		       u32 flags, struct ib_udata *udata)
+int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
+		       struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device);
 	const struct ib_gid_attr *gid_attr;
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_ah *ah = to_hr_ah(ibah);
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 	u16 vlan_id = 0xffff;
 	bool vlan_en = false;
diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c
index da574c26e063..a522cb2d29ea 100644
--- a/drivers/infiniband/hw/hns/hns_roce_alloc.c
+++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c
@@ -157,84 +157,78 @@ void hns_roce_bitmap_cleanup(struct hns_roce_bitmap *bitmap)
 	kfree(bitmap->table);
 }
 
-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
-		       struct hns_roce_buf *buf)
+void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf)
 {
-	int i;
 	struct device *dev = hr_dev->dev;
+	u32 size = buf->size;
+	int i;
+
+	if (size == 0)
+		return;
 
-	if (buf->nbufs == 1) {
+	buf->size = 0;
+
+	if (hns_roce_buf_is_direct(buf)) {
 		dma_free_coherent(dev, size, buf->direct.buf, buf->direct.map);
 	} else {
-		for (i = 0; i < buf->nbufs; ++i)
+		for (i = 0; i < buf->npages; ++i)
 			if (buf->page_list[i].buf)
 				dma_free_coherent(dev, 1 << buf->page_shift,
 						  buf->page_list[i].buf,
 						  buf->page_list[i].map);
 		kfree(buf->page_list);
+		buf->page_list = NULL;
 	}
 }
 
 int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
 		       struct hns_roce_buf *buf, u32 page_shift)
 {
-	int i = 0;
-	dma_addr_t t;
+	struct hns_roce_buf_list *buf_list;
 	struct device *dev = hr_dev->dev;
-	u32 page_size = 1 << page_shift;
-	u32 order;
+	u32 page_size;
+	int i;
 
-	/* SQ/RQ buf lease than one page, SQ + RQ = 8K */
+	/* The minimum shift of the page accessed by hw is HNS_HW_PAGE_SHIFT */
+	buf->page_shift = max_t(int, HNS_HW_PAGE_SHIFT, page_shift);
+
+	page_size = 1 << buf->page_shift;
+	buf->npages = DIV_ROUND_UP(size, page_size);
+
+	/* required size is not bigger than one trunk size */
 	if (size <= max_direct) {
-		buf->nbufs = 1;
-		/* Npages calculated by page_size */
-		order = get_order(size);
-		if (order <= page_shift - PAGE_SHIFT)
-			order = 0;
-		else
-			order -= page_shift - PAGE_SHIFT;
-		buf->npages = 1 << order;
-		buf->page_shift = page_shift;
-		/* MTT PA must be recorded in 4k alignment, t is 4k aligned */
-		buf->direct.buf = dma_alloc_coherent(dev, size, &t,
+		buf->page_list = NULL;
+		buf->direct.buf = dma_alloc_coherent(dev, size,
+						     &buf->direct.map,
 						     GFP_KERNEL);
 		if (!buf->direct.buf)
 			return -ENOMEM;
-
-		buf->direct.map = t;
-
-		while (t & ((1 << buf->page_shift) - 1)) {
-			--buf->page_shift;
-			buf->npages *= 2;
-		}
 	} else {
-		buf->nbufs = (size + page_size - 1) / page_size;
-		buf->npages = buf->nbufs;
-		buf->page_shift = page_shift;
-		buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list),
-					 GFP_KERNEL);
-
-		if (!buf->page_list)
+		buf_list = kcalloc(buf->npages, sizeof(*buf_list), GFP_KERNEL);
+		if (!buf_list)
 			return -ENOMEM;
 
-		for (i = 0; i < buf->nbufs; ++i) {
-			buf->page_list[i].buf = dma_alloc_coherent(dev,
-								   page_size,
-								   &t,
-								   GFP_KERNEL);
-
-			if (!buf->page_list[i].buf)
-				goto err_free;
+		for (i = 0; i < buf->npages; i++) {
+			buf_list[i].buf = dma_alloc_coherent(dev, page_size,
+							     &buf_list[i].map,
+							     GFP_KERNEL);
+			if (!buf_list[i].buf)
+				break;
+		}
 
-			buf->page_list[i].map = t;
+		if (i != buf->npages && i > 0) {
+			while (i-- > 0)
+				dma_free_coherent(dev, page_size,
+						  buf_list[i].buf,
+						  buf_list[i].map);
+			kfree(buf_list);
+			return -ENOMEM;
 		}
+		buf->page_list = buf_list;
 	}
+	buf->size = size;
 
 	return 0;
-
-err_free:
-	hns_roce_buf_free(hr_dev, size, buf);
-	return -ENOMEM;
 }
 
 int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
@@ -246,33 +240,30 @@ int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
 	end = start + buf_cnt;
 	if (end > buf->npages) {
 		dev_err(hr_dev->dev,
-			"invalid kmem region,offset %d,buf_cnt %d,total %d!\n",
+			"Failed to check kmem bufs, end %d + %d total %d!\n",
 			start, buf_cnt, buf->npages);
 		return -EINVAL;
 	}
 
 	total = 0;
 	for (i = start; i < end; i++)
-		if (buf->nbufs == 1)
-			bufs[total++] = buf->direct.map +
-					((dma_addr_t)i << buf->page_shift);
-		else
-			bufs[total++] = buf->page_list[i].map;
+		bufs[total++] = hns_roce_buf_page(buf, i);
 
 	return total;
 }
 
 int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
 			   int buf_cnt, int start, struct ib_umem *umem,
-			   int page_shift)
+			   unsigned int page_shift)
 {
 	struct ib_block_iter biter;
 	int total = 0;
 	int idx = 0;
 	u64 addr;
 
-	if (page_shift < PAGE_SHIFT) {
-		dev_err(hr_dev->dev, "invalid page shift %d!\n", page_shift);
+	if (page_shift < HNS_HW_PAGE_SHIFT) {
+		dev_err(hr_dev->dev, "Failed to check umem page shift %d!\n",
+			page_shift);
 		return -EINVAL;
 	}
 
@@ -292,49 +283,6 @@ done:
 	return total;
 }
 
-void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum,
-			      int offset, int buf_cnt)
-{
-	if (hopnum == HNS_ROCE_HOP_NUM_0)
-		region->hopnum = 0;
-	else
-		region->hopnum = hopnum;
-
-	region->offset = offset;
-	region->count = buf_cnt;
-}
-
-void hns_roce_free_buf_list(dma_addr_t **bufs, int region_cnt)
-{
-	int i;
-
-	for (i = 0; i < region_cnt; i++) {
-		kfree(bufs[i]);
-		bufs[i] = NULL;
-	}
-}
-
-int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions,
-			    dma_addr_t **bufs, int region_cnt)
-{
-	struct hns_roce_buf_region *r;
-	int i;
-
-	for (i = 0; i < region_cnt; i++) {
-		r = &regions[i];
-		bufs[i] = kcalloc(r->count, sizeof(dma_addr_t), GFP_KERNEL);
-		if (!bufs[i])
-			goto err_alloc;
-	}
-
-	return 0;
-
-err_alloc:
-	hns_roce_free_buf_list(bufs, i);
-
-	return -ENOMEM;
-}
-
 void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev)
 {
 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h
index 8e95a1aa1b4f..f5669ff8cfeb 100644
--- a/drivers/infiniband/hw/hns/hns_roce_common.h
+++ b/drivers/infiniband/hw/hns/hns_roce_common.h
@@ -33,10 +33,6 @@
 #ifndef _HNS_ROCE_COMMON_H
 #define _HNS_ROCE_COMMON_H
 
-#ifndef assert
-#define assert(cond)
-#endif
-
 #define roce_write(dev, reg, val)	writel((val), (dev)->reg_base + (reg))
 #define roce_read(dev, reg)		readl((dev)->reg_base + (reg))
 #define roce_raw_write(value, addr) \
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 5bfb52ffd590..e87d616f7988 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -39,51 +39,40 @@
 #include <rdma/hns-abi.h>
 #include "hns_roce_common.h"
 
-static int hns_roce_alloc_cqc(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_cq *hr_cq)
+static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 {
 	struct hns_roce_cmd_mailbox *mailbox;
-	struct hns_roce_hem_table *mtt_table;
 	struct hns_roce_cq_table *cq_table;
-	struct device *dev = hr_dev->dev;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	u64 mtts[MTT_MIN_COUNT] = { 0 };
 	dma_addr_t dma_handle;
-	u64 *mtts;
 	int ret;
 
-	cq_table = &hr_dev->cq_table;
-
-	/* Get the physical address of cq buf */
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
-		mtt_table = &hr_dev->mr_table.mtt_cqe_table;
-	else
-		mtt_table = &hr_dev->mr_table.mtt_table;
-
-	mtts = hns_roce_table_find(hr_dev, mtt_table, hr_cq->mtt.first_seg,
-				   &dma_handle);
-
-	if (!mtts) {
-		dev_err(dev, "Failed to find mtt for CQ buf.\n");
+	ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts),
+				&dma_handle);
+	if (ret < 1) {
+		ibdev_err(ibdev, "Failed to find CQ mtr\n");
 		return -EINVAL;
 	}
 
+	cq_table = &hr_dev->cq_table;
 	ret = hns_roce_bitmap_alloc(&cq_table->bitmap, &hr_cq->cqn);
 	if (ret) {
-		dev_err(dev, "Num of CQ out of range.\n");
+		ibdev_err(ibdev, "Failed to alloc CQ bitmap, err %d\n", ret);
 		return ret;
 	}
 
 	/* Get CQC memory HEM(Hardware Entry Memory) table */
 	ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn);
 	if (ret) {
-		dev_err(dev,
-			"Get context mem failed(%d) when CQ(0x%lx) alloc.\n",
-			ret, hr_cq->cqn);
+		ibdev_err(ibdev, "Failed to get CQ(0x%lx) context, err %d\n",
+			  hr_cq->cqn, ret);
 		goto err_out;
 	}
 
 	ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL));
 	if (ret) {
-		dev_err(dev, "Failed to xa_store CQ.\n");
+		ibdev_err(ibdev, "Failed to xa_store CQ\n");
 		goto err_put;
 	}
 
@@ -101,9 +90,9 @@ static int hns_roce_alloc_cqc(struct hns_roce_dev *hr_dev,
 			HNS_ROCE_CMD_CREATE_CQC, HNS_ROCE_CMD_TIMEOUT_MSECS);
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
 	if (ret) {
-		dev_err(dev,
-			"Send cmd mailbox failed(%d) when CQ(0x%lx) alloc.\n",
-			ret, hr_cq->cqn);
+		ibdev_err(ibdev,
+			  "Failed to send create cmd for CQ(0x%lx), err %d\n",
+			  hr_cq->cqn, ret);
 		goto err_xa;
 	}
 
@@ -126,7 +115,7 @@ err_out:
 	return ret;
 }
 
-void hns_roce_free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
+static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 {
 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
 	struct device *dev = hr_dev->dev;
@@ -153,190 +142,86 @@ void hns_roce_free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR);
 }
 
-static int get_cq_umem(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
-		       struct hns_roce_ib_create_cq ucmd,
-		       struct ib_udata *udata)
-{
-	struct hns_roce_buf *buf = &hr_cq->buf;
-	struct hns_roce_mtt *mtt = &hr_cq->mtt;
-	struct ib_umem **umem = &hr_cq->umem;
-	u32 npages;
-	int ret;
-
-	*umem = ib_umem_get(&hr_dev->ib_dev, ucmd.buf_addr, buf->size,
-			    IB_ACCESS_LOCAL_WRITE);
-	if (IS_ERR(*umem))
-		return PTR_ERR(*umem);
-
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
-		mtt->mtt_type = MTT_TYPE_CQE;
-	else
-		mtt->mtt_type = MTT_TYPE_WQE;
-
-	npages = DIV_ROUND_UP(ib_umem_page_count(*umem),
-			      1 << hr_dev->caps.cqe_buf_pg_sz);
-	ret = hns_roce_mtt_init(hr_dev, npages, buf->page_shift, mtt);
-	if (ret)
-		goto err_buf;
-
-	ret = hns_roce_ib_umem_write_mtt(hr_dev, mtt, *umem);
-	if (ret)
-		goto err_mtt;
-
-	return 0;
-
-err_mtt:
-	hns_roce_mtt_cleanup(hr_dev, mtt);
-
-err_buf:
-	ib_umem_release(*umem);
-	return ret;
-}
-
-static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
+static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
+			struct ib_udata *udata, unsigned long addr)
 {
-	struct hns_roce_buf *buf = &hr_cq->buf;
-	struct hns_roce_mtt *mtt = &hr_cq->mtt;
-	int ret;
-
-	ret = hns_roce_buf_alloc(hr_dev, buf->size, (1 << buf->page_shift) * 2,
-				 buf, buf->page_shift);
-	if (ret)
-		goto out;
-
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
-		mtt->mtt_type = MTT_TYPE_CQE;
-	else
-		mtt->mtt_type = MTT_TYPE_WQE;
-
-	ret = hns_roce_mtt_init(hr_dev, buf->npages, buf->page_shift, mtt);
-	if (ret)
-		goto err_buf;
-
-	ret = hns_roce_buf_write_mtt(hr_dev, mtt, buf);
-	if (ret)
-		goto err_mtt;
-
-	return 0;
-
-err_mtt:
-	hns_roce_mtt_cleanup(hr_dev, mtt);
-
-err_buf:
-	hns_roce_buf_free(hr_dev, buf->size, buf);
-
-out:
-	return ret;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_buf_attr buf_attr = {};
+	int err;
+
+	buf_attr.page_shift = hr_dev->caps.cqe_buf_pg_sz + HNS_HW_PAGE_SHIFT;
+	buf_attr.region[0].size = hr_cq->cq_depth * hr_dev->caps.cq_entry_sz;
+	buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num;
+	buf_attr.region_count = 1;
+	buf_attr.fixed_page = true;
+
+	err = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
+				  hr_dev->caps.cqe_ba_pg_sz + HNS_HW_PAGE_SHIFT,
+				  udata, addr);
+	if (err)
+		ibdev_err(ibdev, "Failed to alloc CQ mtr, err %d\n", err);
+
+	return err;
 }
 
 static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 {
-	hns_roce_buf_free(hr_dev, hr_cq->buf.size, &hr_cq->buf);
+	hns_roce_mtr_destroy(hr_dev, &hr_cq->mtr);
 }
 
-static int create_user_cq(struct hns_roce_dev *hr_dev,
-			  struct hns_roce_cq *hr_cq,
-			  struct ib_udata *udata,
-			  struct hns_roce_ib_create_cq_resp *resp)
+static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
+		       struct ib_udata *udata, unsigned long addr,
+		       struct hns_roce_ib_create_cq_resp *resp)
 {
-	struct hns_roce_ib_create_cq ucmd;
-	struct device *dev = hr_dev->dev;
-	int ret;
-	struct hns_roce_ucontext *context = rdma_udata_to_drv_context(
-				   udata, struct hns_roce_ucontext, ibucontext);
-
-	if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
-		dev_err(dev, "Failed to copy_from_udata.\n");
-		return -EFAULT;
-	}
+	bool has_db = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB;
+	struct hns_roce_ucontext *uctx;
+	int err;
 
-	/* Get user space address, write it into mtt table */
-	ret = get_cq_umem(hr_dev, hr_cq, ucmd, udata);
-	if (ret) {
-		dev_err(dev, "Failed to get_cq_umem.\n");
-		return ret;
-	}
-
-	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB &&
-	    udata->outlen >= offsetofend(typeof(*resp), cap_flags)) {
-		ret = hns_roce_db_map_user(context, udata, ucmd.db_addr,
-					   &hr_cq->db);
-		if (ret) {
-			dev_err(dev, "cq record doorbell map failed!\n");
-			goto err_mtt;
+	if (udata) {
+		if (has_db &&
+		    udata->outlen >= offsetofend(typeof(*resp), cap_flags)) {
+			uctx = rdma_udata_to_drv_context(udata,
+					struct hns_roce_ucontext, ibucontext);
+			err = hns_roce_db_map_user(uctx, udata, addr,
+						   &hr_cq->db);
+			if (err)
+				return err;
+			hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;
+			resp->cap_flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;
 		}
-		hr_cq->db_en = 1;
-		resp->cap_flags |= HNS_ROCE_SUPPORT_CQ_RECORD_DB;
-	}
-
-	return 0;
-
-err_mtt:
-	hns_roce_mtt_cleanup(hr_dev, &hr_cq->mtt);
-	ib_umem_release(hr_cq->umem);
-
-	return ret;
-}
-
-static int create_kernel_cq(struct hns_roce_dev *hr_dev,
-			    struct hns_roce_cq *hr_cq)
-{
-	struct device *dev = hr_dev->dev;
-	int ret;
-
-	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) {
-		ret = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1);
-		if (ret)
-			return ret;
-
-		hr_cq->set_ci_db = hr_cq->db.db_record;
-		*hr_cq->set_ci_db = 0;
-		hr_cq->db_en = 1;
-	}
-
-	/* Init mtt table and write buff address to mtt table */
-	ret = alloc_cq_buf(hr_dev, hr_cq);
-	if (ret) {
-		dev_err(dev, "Failed to alloc_cq_buf.\n");
-		goto err_db;
+	} else {
+		if (has_db) {
+			err = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1);
+			if (err)
+				return err;
+			hr_cq->set_ci_db = hr_cq->db.db_record;
+			*hr_cq->set_ci_db = 0;
+			hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;
+		}
+		hr_cq->cq_db_l = hr_dev->reg_base + hr_dev->odb_offset +
+				 DB_REG_OFFSET * hr_dev->priv_uar.index;
 	}
 
-	hr_cq->cq_db_l = hr_dev->reg_base + hr_dev->odb_offset +
-			 DB_REG_OFFSET * hr_dev->priv_uar.index;
-
 	return 0;
-
-err_db:
-	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)
-		hns_roce_free_db(hr_dev, &hr_cq->db);
-
-	return ret;
 }
 
-static void destroy_user_cq(struct hns_roce_dev *hr_dev,
-			    struct hns_roce_cq *hr_cq,
-			    struct ib_udata *udata,
-			    struct hns_roce_ib_create_cq_resp *resp)
+static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
+		       struct ib_udata *udata)
 {
-	struct hns_roce_ucontext *context = rdma_udata_to_drv_context(
-				   udata, struct hns_roce_ucontext, ibucontext);
+	struct hns_roce_ucontext *uctx;
 
-	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB &&
-	    udata->outlen >= offsetofend(typeof(*resp), cap_flags))
-		hns_roce_db_unmap_user(context, &hr_cq->db);
-
-	hns_roce_mtt_cleanup(hr_dev, &hr_cq->mtt);
-	ib_umem_release(hr_cq->umem);
-}
-
-static void destroy_kernel_cq(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_cq *hr_cq)
-{
-	hns_roce_mtt_cleanup(hr_dev, &hr_cq->mtt);
-	free_cq_buf(hr_dev, hr_cq);
+	if (!(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB))
+		return;
 
-	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)
+	hr_cq->flags &= ~HNS_ROCE_CQ_FLAG_RECORD_DB;
+	if (udata) {
+		uctx = rdma_udata_to_drv_context(udata,
+						 struct hns_roce_ucontext,
+						 ibucontext);
+		hns_roce_db_unmap_user(uctx, &hr_cq->db);
+	} else {
 		hns_roce_free_db(hr_dev, &hr_cq->db);
+	}
 }
 
 int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
@@ -345,20 +230,21 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_ib_create_cq_resp resp = {};
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
-	struct device *dev = hr_dev->dev;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_ib_create_cq ucmd = {};
 	int vector = attr->comp_vector;
 	u32 cq_entries = attr->cqe;
 	int ret;
 
 	if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) {
-		dev_err(dev, "Create CQ failed. entries=%d, max=%d\n",
-			cq_entries, hr_dev->caps.max_cqes);
+		ibdev_err(ibdev, "Failed to check CQ count %d max=%d\n",
+			  cq_entries, hr_dev->caps.max_cqes);
 		return -EINVAL;
 	}
 
 	if (vector >= hr_dev->caps.num_comp_vectors) {
-		dev_err(dev, "Create CQ failed, vector=%d, max=%d\n",
-			vector, hr_dev->caps.num_comp_vectors);
+		ibdev_err(ibdev, "Failed to check CQ vector=%d max=%d\n",
+			  vector, hr_dev->caps.num_comp_vectors);
 		return -EINVAL;
 	}
 
@@ -367,30 +253,35 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
 	hr_cq->ib_cq.cqe = cq_entries - 1; /* used as cqe index */
 	hr_cq->cq_depth = cq_entries;
 	hr_cq->vector = vector;
-	hr_cq->buf.size = hr_cq->cq_depth * hr_dev->caps.cq_entry_sz;
-	hr_cq->buf.page_shift = PAGE_SHIFT + hr_dev->caps.cqe_buf_pg_sz;
 	spin_lock_init(&hr_cq->lock);
 	INIT_LIST_HEAD(&hr_cq->sq_list);
 	INIT_LIST_HEAD(&hr_cq->rq_list);
 
 	if (udata) {
-		ret = create_user_cq(hr_dev, hr_cq, udata, &resp);
-		if (ret) {
-			dev_err(dev, "Create cq failed in user mode!\n");
-			goto err_cq;
-		}
-	} else {
-		ret = create_kernel_cq(hr_dev, hr_cq);
+		ret = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
 		if (ret) {
-			dev_err(dev, "Create cq failed in kernel mode!\n");
-			goto err_cq;
+			ibdev_err(ibdev, "Failed to copy CQ udata, err %d\n",
+				  ret);
+			return ret;
 		}
 	}
 
-	ret = hns_roce_alloc_cqc(hr_dev, hr_cq);
+	ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr);
+	if (ret) {
+		ibdev_err(ibdev, "Failed to alloc CQ buf, err %d\n", ret);
+		return ret;
+	}
+
+	ret = alloc_cq_db(hr_dev, hr_cq, udata, ucmd.db_addr, &resp);
 	if (ret) {
-		dev_err(dev, "Alloc CQ failed(%d).\n", ret);
-		goto err_dbmap;
+		ibdev_err(ibdev, "Failed to alloc CQ db, err %d\n", ret);
+		goto err_cq_buf;
+	}
+
+	ret = alloc_cqc(hr_dev, hr_cq);
+	if (ret) {
+		ibdev_err(ibdev, "Failed to alloc CQ context, err %d\n", ret);
+		goto err_cq_db;
 	}
 
 	/*
@@ -412,15 +303,11 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
 	return 0;
 
 err_cqc:
-	hns_roce_free_cqc(hr_dev, hr_cq);
-
-err_dbmap:
-	if (udata)
-		destroy_user_cq(hr_dev, hr_cq, udata, &resp);
-	else
-		destroy_kernel_cq(hr_dev, hr_cq);
-
-err_cq:
+	free_cqc(hr_dev, hr_cq);
+err_cq_db:
+	free_cq_db(hr_dev, hr_cq, udata);
+err_cq_buf:
+	free_cq_buf(hr_dev, hr_cq);
 	return ret;
 }
 
@@ -429,28 +316,12 @@ void hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
 
-	if (hr_dev->hw->destroy_cq) {
+	if (hr_dev->hw->destroy_cq)
 		hr_dev->hw->destroy_cq(ib_cq, udata);
-		return;
-	}
-
-	hns_roce_free_cqc(hr_dev, hr_cq);
-	hns_roce_mtt_cleanup(hr_dev, &hr_cq->mtt);
 
-	ib_umem_release(hr_cq->umem);
-	if (udata) {
-		if (hr_cq->db_en == 1)
-			hns_roce_db_unmap_user(rdma_udata_to_drv_context(
-						       udata,
-						       struct hns_roce_ucontext,
-						       ibucontext),
-					       &hr_cq->db);
-	} else {
-		/* Free the buff of stored cq */
-		free_cq_buf(hr_dev, hr_cq);
-		if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB)
-			hns_roce_free_db(hr_dev, &hr_cq->db);
-	}
+	free_cq_buf(hr_dev, hr_cq);
+	free_cq_db(hr_dev, hr_cq, udata);
+	free_cqc(hr_dev, hr_cq);
 }
 
 void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index f6b3cf6b95d6..a77fa6730b2d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -66,6 +66,8 @@
 #define HNS_ROCE_CQE_WCMD_EMPTY_BIT		0x2
 #define HNS_ROCE_MIN_CQE_CNT			16
 
+#define HNS_ROCE_RESERVED_SGE			1
+
 #define HNS_ROCE_MAX_IRQ_NUM			128
 
 #define HNS_ROCE_SGE_IN_WQE			2
@@ -131,12 +133,12 @@ enum {
 };
 
 enum {
-	HNS_ROCE_SUPPORT_RQ_RECORD_DB = 1 << 0,
-	HNS_ROCE_SUPPORT_SQ_RECORD_DB = 1 << 1,
+	HNS_ROCE_QP_CAP_RQ_RECORD_DB = BIT(0),
+	HNS_ROCE_QP_CAP_SQ_RECORD_DB = BIT(1),
 };
 
-enum {
-	HNS_ROCE_SUPPORT_CQ_RECORD_DB = 1 << 0,
+enum hns_roce_cq_flags {
+	HNS_ROCE_CQ_FLAG_RECORD_DB = BIT(0),
 };
 
 enum hns_roce_qp_state {
@@ -209,6 +211,8 @@ enum {
 	HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE	= 0x07,
 };
 
+#define HNS_ROCE_CAP_FLAGS_EX_SHIFT 12
+
 enum {
 	HNS_ROCE_CAP_FLAG_REREG_MR		= BIT(0),
 	HNS_ROCE_CAP_FLAG_ROCE_V1_V2		= BIT(1),
@@ -222,13 +226,6 @@ enum {
 	HNS_ROCE_CAP_FLAG_ATOMIC		= BIT(10),
 };
 
-enum hns_roce_mtt_type {
-	MTT_TYPE_WQE,
-	MTT_TYPE_CQE,
-	MTT_TYPE_SRQWQE,
-	MTT_TYPE_IDX
-};
-
 #define HNS_ROCE_DB_TYPE_COUNT			2
 #define HNS_ROCE_DB_UNIT_SIZE			4
 
@@ -267,9 +264,12 @@ enum {
 #define HNS_ROCE_PORT_DOWN			0
 #define HNS_ROCE_PORT_UP			1
 
-#define HNS_ROCE_MTT_ENTRY_PER_SEG		8
+/* The minimum page size is 4K for hardware */
+#define HNS_HW_PAGE_SHIFT			12
+#define HNS_HW_PAGE_SIZE			(1 << HNS_HW_PAGE_SHIFT)
 
-#define PAGE_ADDR_SHIFT				12
+/* The minimum page count for hardware access page directly. */
+#define HNS_HW_DIRECT_PAGE_COUNT 2
 
 struct hns_roce_uar {
 	u64		pfn;
@@ -300,22 +300,6 @@ struct hns_roce_bitmap {
 	unsigned long		*table;
 };
 
-/* Order bitmap length -- bit num compute formula: 1 << (max_order - order) */
-/* Order = 0: bitmap is biggest, order = max bitmap is least (only a bit) */
-/* Every bit repesent to a partner free/used status in bitmap */
-/*
- * Initial, bits of other bitmap are all 0 except that a bit of max_order is 1
- * Bit = 1 represent to idle and available; bit = 0: not available
- */
-struct hns_roce_buddy {
-	/* Members point to every order level bitmap */
-	unsigned long **bits;
-	/* Represent to avail bits of the order level bitmap */
-	u32            *num_free;
-	int             max_order;
-	spinlock_t      lock;
-};
-
 /* For Hardware Entry Memory */
 struct hns_roce_hem_table {
 	/* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */
@@ -336,13 +320,6 @@ struct hns_roce_hem_table {
 	dma_addr_t	*bt_l0_dma_addr;
 };
 
-struct hns_roce_mtt {
-	unsigned long		first_seg;
-	int			order;
-	int			page_shift;
-	enum hns_roce_mtt_type	mtt_type;
-};
-
 struct hns_roce_buf_region {
 	int offset; /* page offset */
 	u32 count; /* page count */
@@ -357,13 +334,34 @@ struct hns_roce_hem_list {
 	struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL];
 	struct list_head btm_bt; /* link all bottom bt in @mid_bt */
 	dma_addr_t root_ba; /* pointer to the root ba table */
-	int bt_pg_shift;
+};
+
+struct hns_roce_buf_attr {
+	struct {
+		size_t	size;  /* region size */
+		int	hopnum; /* multi-hop addressing hop num */
+	} region[HNS_ROCE_MAX_BT_REGION];
+	int region_count; /* valid region count */
+	unsigned int page_shift;  /* buffer page shift */
+	bool fixed_page; /* decide page shift is fixed-size or maximum size */
+	int user_access; /* umem access flag */
+	bool mtt_only; /* only alloc buffer-required MTT memory */
 };
 
 /* memory translate region */
 struct hns_roce_mtr {
-	struct hns_roce_hem_list hem_list;
-	int buf_pg_shift;
+	struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */
+	struct ib_umem		*umem; /* user space buffer */
+	struct hns_roce_buf	*kmem; /* kernel space buffer */
+	struct {
+		dma_addr_t	root_ba; /* root BA table's address */
+		bool		is_direct; /* addressing without BA table */
+		unsigned int	ba_pg_shift; /* BA table page shift */
+		unsigned int	buf_pg_shift; /* buffer page shift */
+		int		buf_pg_count;  /* buffer page count */
+		struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];
+		unsigned int	region_count;
+	} hem_cfg; /* config for hardware addressing */
 };
 
 struct hns_roce_mw {
@@ -381,43 +379,22 @@ struct hns_roce_mw {
 
 struct hns_roce_mr {
 	struct ib_mr		ibmr;
-	struct ib_umem		*umem;
 	u64			iova; /* MR's virtual orignal addr */
 	u64			size; /* Address range of MR */
 	u32			key; /* Key of MR */
 	u32			pd;   /* PD num of MR */
 	u32			access;	/* Access permission of MR */
-	u32			npages;
 	int			enabled; /* MR's active status */
 	int			type;	/* MR's register type */
-	u64			*pbl_buf;	/* MR's PBL space */
-	dma_addr_t		pbl_dma_addr;	/* MR's PBL space PA */
-	u32			pbl_size;	/* PA number in the PBL */
-	u64			pbl_ba;		/* page table address */
-	u32			l0_chunk_last_num;	/* L0 last number */
-	u32			l1_chunk_last_num;	/* L1 last number */
-	u64			**pbl_bt_l2;	/* PBL BT L2 */
-	u64			**pbl_bt_l1;	/* PBL BT L1 */
-	u64			*pbl_bt_l0;	/* PBL BT L0 */
-	dma_addr_t		*pbl_l2_dma_addr;	/* PBL BT L2 dma addr */
-	dma_addr_t		*pbl_l1_dma_addr;	/* PBL BT L1 dma addr */
-	dma_addr_t		pbl_l0_dma_addr;	/* PBL BT L0 dma addr */
-	u32			pbl_ba_pg_sz;	/* BT chunk page size */
-	u32			pbl_buf_pg_sz;	/* buf chunk page size */
 	u32			pbl_hop_num;	/* multi-hop number */
+	struct hns_roce_mtr	pbl_mtr;
+	u32			npages;
+	dma_addr_t		*page_list;
 };
 
 struct hns_roce_mr_table {
 	struct hns_roce_bitmap		mtpt_bitmap;
-	struct hns_roce_buddy		mtt_buddy;
-	struct hns_roce_hem_table	mtt_table;
 	struct hns_roce_hem_table	mtpt_table;
-	struct hns_roce_buddy		mtt_cqe_buddy;
-	struct hns_roce_hem_table	mtt_cqe_table;
-	struct hns_roce_buddy		mtt_srqwqe_buddy;
-	struct hns_roce_hem_table	mtt_srqwqe_table;
-	struct hns_roce_buddy		mtt_idx_buddy;
-	struct hns_roce_hem_table	mtt_idx_table;
 };
 
 struct hns_roce_wq {
@@ -433,7 +410,7 @@ struct hns_roce_wq {
 };
 
 struct hns_roce_sge {
-	int		sge_cnt;	/* SGE num */
+	unsigned int	sge_cnt;	/* SGE num */
 	int		offset;
 	int		sge_shift;	/* SGE size */
 };
@@ -446,10 +423,9 @@ struct hns_roce_buf_list {
 struct hns_roce_buf {
 	struct hns_roce_buf_list	direct;
 	struct hns_roce_buf_list	*page_list;
-	int				nbufs;
 	u32				npages;
 	u32				size;
-	int				page_shift;
+	unsigned int			page_shift;
 };
 
 struct hns_roce_db_pgdir {
@@ -482,12 +458,10 @@ struct hns_roce_db {
 
 struct hns_roce_cq {
 	struct ib_cq			ib_cq;
-	struct hns_roce_buf		buf;
-	struct hns_roce_mtt		mtt;
+	struct hns_roce_mtr		mtr;
 	struct hns_roce_db		db;
-	u8				db_en;
+	u32				flags;
 	spinlock_t			lock;
-	struct ib_umem			*umem;
 	u32				cq_depth;
 	u32				cons_index;
 	u32				*set_ci_db;
@@ -505,11 +479,8 @@ struct hns_roce_cq {
 };
 
 struct hns_roce_idx_que {
-	struct hns_roce_buf		idx_buf;
-	int				entry_sz;
-	u32				buf_size;
-	struct ib_umem			*umem;
-	struct hns_roce_mtt		mtt;
+	struct hns_roce_mtr		mtr;
+	int				entry_shift;
 	unsigned long			*bitmap;
 };
 
@@ -524,10 +495,9 @@ struct hns_roce_srq {
 	atomic_t		refcount;
 	struct completion	free;
 
-	struct hns_roce_buf	buf;
+	struct hns_roce_mtr	buf_mtr;
+
 	u64		       *wrid;
-	struct ib_umem	       *umem;
-	struct hns_roce_mtt	mtt;
 	struct hns_roce_idx_que idx_que;
 	spinlock_t		lock;
 	int			head;
@@ -656,20 +626,15 @@ struct hns_roce_work {
 
 struct hns_roce_qp {
 	struct ib_qp		ibqp;
-	struct hns_roce_buf	hr_buf;
 	struct hns_roce_wq	rq;
 	struct hns_roce_db	rdb;
 	struct hns_roce_db	sdb;
-	u8			rdb_en;
-	u8			sdb_en;
+	unsigned long		en_flags;
 	u32			doorbell_qpn;
 	u32			sq_signal_bits;
 	struct hns_roce_wq	sq;
 
-	struct ib_umem		*umem;
-	struct hns_roce_mtt	mtt;
 	struct hns_roce_mtr	mtr;
-	int                     wqe_bt_pg_shift;
 
 	u32			buff_size;
 	struct mutex		mutex;
@@ -769,17 +734,11 @@ struct hns_roce_eq {
 	int				over_ignore;
 	int				coalesce;
 	int				arm_st;
-	u64				eqe_ba;
-	int				eqe_ba_pg_sz;
-	int				eqe_buf_pg_sz;
 	int				hop_num;
 	struct hns_roce_mtr		mtr;
-	struct hns_roce_buf		buf;
-	int				eq_max_cnt;
+	u16				eq_max_cnt;
 	int				eq_period;
 	int				shift;
-	dma_addr_t			cur_eqe_ba;
-	dma_addr_t			nxt_eqe_ba;
 	int				event_type;
 	int				sub_type;
 };
@@ -1102,15 +1061,67 @@ static inline struct hns_roce_qp
 	return xa_load(&hr_dev->qp_table_xa, qpn & (hr_dev->caps.num_qps - 1));
 }
 
+static inline bool hns_roce_buf_is_direct(struct hns_roce_buf *buf)
+{
+	if (buf->page_list)
+		return false;
+
+	return true;
+}
+
 static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)
 {
-	u32 page_size = 1 << buf->page_shift;
+	if (hns_roce_buf_is_direct(buf))
+		return (char *)(buf->direct.buf) + (offset & (buf->size - 1));
 
-	if (buf->nbufs == 1)
-		return (char *)(buf->direct.buf) + offset;
+	return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
+	       (offset & ((1 << buf->page_shift) - 1));
+}
+
+static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, int idx)
+{
+	if (hns_roce_buf_is_direct(buf))
+		return buf->direct.map + ((dma_addr_t)idx << buf->page_shift);
 	else
-		return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
-		       (offset & (page_size - 1));
+		return buf->page_list[idx].map;
+}
+
+#define hr_hw_page_align(x)		ALIGN(x, 1 << HNS_HW_PAGE_SHIFT)
+
+static inline u64 to_hr_hw_page_addr(u64 addr)
+{
+	return addr >> HNS_HW_PAGE_SHIFT;
+}
+
+static inline u32 to_hr_hw_page_shift(u32 page_shift)
+{
+	return page_shift - HNS_HW_PAGE_SHIFT;
+}
+
+static inline u32 to_hr_hem_hopnum(u32 hopnum, u32 count)
+{
+	if (count > 0)
+		return hopnum == HNS_ROCE_HOP_NUM_0 ? 0 : hopnum;
+
+	return 0;
+}
+
+static inline u32 to_hr_hem_entries_size(u32 count, u32 buf_shift)
+{
+	return hr_hw_page_align(count << buf_shift);
+}
+
+static inline u32 to_hr_hem_entries_count(u32 count, u32 buf_shift)
+{
+	return hr_hw_page_align(count << buf_shift) >> buf_shift;
+}
+
+static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift)
+{
+	if (!count)
+		return 0;
+
+	return ilog2(to_hr_hem_entries_count(count, buf_shift));
 }
 
 int hns_roce_init_uar_table(struct hns_roce_dev *dev);
@@ -1125,25 +1136,18 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
 int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);
 void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);
 
-int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
-		      struct hns_roce_mtt *mtt);
-void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev,
-			  struct hns_roce_mtt *mtt);
-int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
-			   struct hns_roce_mtt *mtt, struct hns_roce_buf *buf);
-
-void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift,
-		       int buf_pg_shift);
-int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
-			dma_addr_t **bufs, struct hns_roce_buf_region *regions,
-			int region_cnt);
-void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev,
-			  struct hns_roce_mtr *mtr);
-
 /* hns roce hw need current block and next block addr from mtt */
 #define MTT_MIN_COUNT	 2
 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 		      int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr);
+int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+			struct hns_roce_buf_attr *buf_attr,
+			unsigned int page_shift, struct ib_udata *udata,
+			unsigned long user_addr);
+void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
+			  struct hns_roce_mtr *mtr);
+int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+		     dma_addr_t *pages, int page_cnt);
 
 int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);
 int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);
@@ -1171,8 +1175,8 @@ void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
 				unsigned long obj, int cnt,
 				int rr);
 
-int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
-		       u32 flags, struct ib_udata *udata);
+int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
+		       struct ib_udata *udata);
 int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
 void hns_roce_destroy_ah(struct ib_ah *ah, u32 flags);
 
@@ -1200,25 +1204,15 @@ struct ib_mw *hns_roce_alloc_mw(struct ib_pd *pd, enum ib_mw_type,
 				struct ib_udata *udata);
 int hns_roce_dealloc_mw(struct ib_mw *ibmw);
 
-void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size,
-		       struct hns_roce_buf *buf);
+void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf);
 int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
 		       struct hns_roce_buf *buf, u32 page_shift);
 
-int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
-			       struct hns_roce_mtt *mtt, struct ib_umem *umem);
-
-void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum,
-			      int offset, int buf_cnt);
-int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions,
-			    dma_addr_t **bufs, int count);
-void hns_roce_free_buf_list(dma_addr_t **bufs, int count);
-
 int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
 			   int buf_cnt, int start, struct hns_roce_buf *buf);
 int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,
 			   int buf_cnt, int start, struct ib_umem *umem,
-			   int page_shift);
+			   unsigned int page_shift);
 
 int hns_roce_create_srq(struct ib_srq *srq,
 			struct ib_srq_init_attr *srq_init_attr,
@@ -1254,8 +1248,6 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
 		       struct ib_udata *udata);
 
 void hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
-void hns_roce_free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq);
-
 int hns_roce_db_map_user(struct hns_roce_ucontext *context,
 			 struct ib_udata *udata, unsigned long virt,
 			 struct hns_roce_db *db);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
index 263338b90d7a..c8db6f8ae018 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -75,18 +75,6 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
 	case HEM_TYPE_CQC_TIMER:
 		hop_num = hr_dev->caps.cqc_timer_hop_num;
 		break;
-	case HEM_TYPE_CQE:
-		hop_num = hr_dev->caps.cqe_hop_num;
-		break;
-	case HEM_TYPE_MTT:
-		hop_num = hr_dev->caps.mtt_hop_num;
-		break;
-	case HEM_TYPE_SRQWQE:
-		hop_num = hr_dev->caps.srqwqe_hop_num;
-		break;
-	case HEM_TYPE_IDX:
-		hop_num = hr_dev->caps.idx_hop_num;
-		break;
 	default:
 		return false;
 	}
@@ -195,38 +183,6 @@ static int get_hem_table_config(struct hns_roce_dev *hr_dev,
 		mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
 		mhop->hop_num = hr_dev->caps.srqc_hop_num;
 		break;
-	case HEM_TYPE_MTT:
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
-					     + PAGE_SHIFT);
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
-					     + PAGE_SHIFT);
-		mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
-		mhop->hop_num = hr_dev->caps.mtt_hop_num;
-		break;
-	case HEM_TYPE_CQE:
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
-					     + PAGE_SHIFT);
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
-					     + PAGE_SHIFT);
-		mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
-		mhop->hop_num = hr_dev->caps.cqe_hop_num;
-		break;
-	case HEM_TYPE_SRQWQE:
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz
-					    + PAGE_SHIFT);
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
-					    + PAGE_SHIFT);
-		mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
-		mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
-		break;
-	case HEM_TYPE_IDX:
-		mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz
-				       + PAGE_SHIFT);
-		mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
-				       + PAGE_SHIFT);
-		mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
-		mhop->hop_num = hr_dev->caps.idx_hop_num;
-		break;
 	default:
 		dev_err(dev, "Table %d not support multi-hop addressing!\n",
 			type);
@@ -899,57 +855,6 @@ out:
 	return addr;
 }
 
-int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
-			     struct hns_roce_hem_table *table,
-			     unsigned long start, unsigned long end)
-{
-	struct hns_roce_hem_mhop mhop;
-	unsigned long inc = table->table_chunk_size / table->obj_size;
-	unsigned long i = 0;
-	int ret;
-
-	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
-		ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
-		if (ret)
-			goto fail;
-		inc = mhop.bt_chunk_size / table->obj_size;
-	}
-
-	/* Allocate MTT entry memory according to chunk(128K) */
-	for (i = start; i <= end; i += inc) {
-		ret = hns_roce_table_get(hr_dev, table, i);
-		if (ret)
-			goto fail;
-	}
-
-	return 0;
-
-fail:
-	while (i > start) {
-		i -= inc;
-		hns_roce_table_put(hr_dev, table, i);
-	}
-	return ret;
-}
-
-void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_hem_table *table,
-			      unsigned long start, unsigned long end)
-{
-	struct hns_roce_hem_mhop mhop;
-	unsigned long inc = table->table_chunk_size / table->obj_size;
-	unsigned long i;
-
-	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
-		if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
-			return;
-		inc = mhop.bt_chunk_size / table->obj_size;
-	}
-
-	for (i = start; i <= end; i += inc)
-		hns_roce_table_put(hr_dev, table, i);
-}
-
 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 			    struct hns_roce_hem_table *table, u32 type,
 			    unsigned long obj_size, unsigned long nobj,
@@ -1112,12 +1017,6 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
 
 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
 {
-	if ((hr_dev->caps.num_idx_segs))
-		hns_roce_cleanup_hem_table(hr_dev,
-					   &hr_dev->mr_table.mtt_idx_table);
-	if (hr_dev->caps.num_srqwqe_segs)
-		hns_roce_cleanup_hem_table(hr_dev,
-					   &hr_dev->mr_table.mtt_srqwqe_table);
 	if (hr_dev->caps.srqc_entry_sz)
 		hns_roce_cleanup_hem_table(hr_dev,
 					   &hr_dev->srq_table.table);
@@ -1137,10 +1036,6 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
-		hns_roce_cleanup_hem_table(hr_dev,
-					   &hr_dev->mr_table.mtt_cqe_table);
-	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
 }
 
 struct roce_hem_item {
@@ -1505,7 +1400,7 @@ err_exit:
 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
 			      struct hns_roce_hem_list *hem_list,
 			      const struct hns_roce_buf_region *regions,
-			      int region_cnt)
+			      int region_cnt, unsigned int bt_pg_shift)
 {
 	const struct hns_roce_buf_region *r;
 	int ofs, end;
@@ -1519,7 +1414,7 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
 		return -EINVAL;
 	}
 
-	unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN;
+	unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
 	for (i = 0; i < region_cnt; i++) {
 		r = &regions[i];
 		if (!r->count)
@@ -1566,8 +1461,7 @@ void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
 	hem_list->root_ba = 0;
 }
 
-void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
-			    int bt_page_order)
+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
 {
 	int i, j;
 
@@ -1576,8 +1470,6 @@ void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
 	for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
 		for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
 			INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
-
-	hem_list->bt_pg_shift = bt_page_order;
 }
 
 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h
index 3bb8f78fb7b0..b34c940077bb 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.h
@@ -115,12 +115,6 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
 			  struct hns_roce_hem_table *table, unsigned long obj,
 			  dma_addr_t *dma_handle);
-int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
-			     struct hns_roce_hem_table *table,
-			     unsigned long start, unsigned long end);
-void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_hem_table *table,
-			      unsigned long start, unsigned long end);
 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 			    struct hns_roce_hem_table *table, u32 type,
 			    unsigned long obj_size, unsigned long nobj,
@@ -133,14 +127,13 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
 			   struct hns_roce_hem_mhop *mhop);
 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type);
 
-void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
-			    int bt_page_order);
+void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list);
 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
 				   int region_cnt, int unit);
 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
 			      struct hns_roce_hem_list *hem_list,
 			      const struct hns_roce_buf_region *regions,
-			      int region_cnt);
+			      int region_cnt, unsigned int bt_pg_shift);
 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
 			       struct hns_roce_hem_list *hem_list);
 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index 5ff028d77be3..d02207cd30df 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -503,16 +503,13 @@ static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
 static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
 				 u32 ext_sdb_alful)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_db_table *db = &priv->db_table;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_db_table *db;
 	dma_addr_t sdb_dma_addr;
 	__le32 tmp;
 	u32 val;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	db = &priv->db_table;
-
 	/* Configure extend SDB threshold */
 	roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
 	roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
@@ -545,16 +542,13 @@ static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
 static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
 				 u32 ext_odb_alful)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_db_table *db = &priv->db_table;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_db_table *db;
 	dma_addr_t odb_dma_addr;
 	__le32 tmp;
 	u32 val;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	db = &priv->db_table;
-
 	/* Configure extend ODB threshold */
 	roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
 	roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
@@ -583,16 +577,13 @@ static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
 static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
 				u32 odb_ext_mod)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_db_table *db = &priv->db_table;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_db_table *db;
 	dma_addr_t sdb_dma_addr;
 	dma_addr_t odb_dma_addr;
 	int ret = 0;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	db = &priv->db_table;
-
 	db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
 	if (!db->ext_db)
 		return -ENOMEM;
@@ -692,14 +683,14 @@ static struct hns_roce_qp *hns_roce_v1_create_lp_qp(struct hns_roce_dev *hr_dev,
 
 static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_free_mr *free_mr = &priv->free_mr;
 	struct hns_roce_caps *caps = &hr_dev->caps;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
 	struct device *dev = &hr_dev->pdev->dev;
 	struct ib_cq_init_attr cq_init_attr;
-	struct hns_roce_free_mr *free_mr;
 	struct ib_qp_attr attr = { 0 };
-	struct hns_roce_v1_priv *priv;
 	struct hns_roce_qp *hr_qp;
-	struct ib_device *ibdev;
 	struct ib_cq *cq;
 	struct ib_pd *pd;
 	union ib_gid dgid;
@@ -712,14 +703,10 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
 	u8 port = 0;
 	u8 sl;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	free_mr = &priv->free_mr;
-
 	/* Reserved cq for loop qp */
 	cq_init_attr.cqe		= HNS_ROCE_MIN_WQE_NUM * 2;
 	cq_init_attr.comp_vector	= 0;
 
-	ibdev = &hr_dev->ib_dev;
 	cq = rdma_zalloc_drv_obj(ibdev, ib_cq);
 	if (!cq)
 		return -ENOMEM;
@@ -868,16 +855,13 @@ alloc_cq_failed:
 
 static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_free_mr *free_mr = &priv->free_mr;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_free_mr *free_mr;
-	struct hns_roce_v1_priv *priv;
 	struct hns_roce_qp *hr_qp;
 	int ret;
 	int i;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	free_mr = &priv->free_mr;
-
 	for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
 		hr_qp = free_mr->mr_free_qp[i];
 		if (!hr_qp)
@@ -897,18 +881,15 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
 
 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_db_table *db = &priv->db_table;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_db_table *db;
 	u32 sdb_ext_mod;
 	u32 odb_ext_mod;
 	u32 sdb_evt_mod;
 	u32 odb_evt_mod;
 	int ret = 0;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	db = &priv->db_table;
-
 	memset(db, 0, sizeof(*db));
 
 	/* Default DB mode */
@@ -954,15 +935,12 @@ static void hns_roce_v1_recreate_lp_qp_work_fn(struct work_struct *work)
 
 static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev)
 {
-	struct device *dev = &hr_dev->pdev->dev;
+	long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS;
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_free_mr *free_mr = &priv->free_mr;
 	struct hns_roce_recreate_lp_qp_work *lp_qp_work;
-	struct hns_roce_free_mr *free_mr;
-	struct hns_roce_v1_priv *priv;
+	struct device *dev = &hr_dev->pdev->dev;
 	struct completion comp;
-	long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	free_mr = &priv->free_mr;
 
 	lp_qp_work = kzalloc(sizeof(struct hns_roce_recreate_lp_qp_work),
 			     GFP_KERNEL);
@@ -1021,29 +999,21 @@ static int hns_roce_v1_send_lp_wqe(struct hns_roce_qp *hr_qp)
 
 static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
 {
-	struct hns_roce_mr_free_work *mr_work;
-	struct ib_wc wc[HNS_ROCE_V1_RESV_QP];
-	struct hns_roce_free_mr *free_mr;
-	struct hns_roce_cq *mr_free_cq;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_dev *hr_dev;
-	struct hns_roce_mr *hr_mr;
-	struct hns_roce_qp *hr_qp;
-	struct device *dev;
 	unsigned long end =
 		msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
-	int i;
-	int ret;
+	struct hns_roce_mr_free_work *mr_work =
+		container_of(work, struct hns_roce_mr_free_work, work);
+	struct hns_roce_dev *hr_dev = to_hr_dev(mr_work->ib_dev);
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_free_mr *free_mr = &priv->free_mr;
+	struct hns_roce_cq *mr_free_cq = free_mr->mr_free_cq;
+	struct hns_roce_mr *hr_mr = mr_work->mr;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct ib_wc wc[HNS_ROCE_V1_RESV_QP];
+	struct hns_roce_qp *hr_qp;
 	int ne = 0;
-
-	mr_work = container_of(work, struct hns_roce_mr_free_work, work);
-	hr_mr = (struct hns_roce_mr *)mr_work->mr;
-	hr_dev = to_hr_dev(mr_work->ib_dev);
-	dev = &hr_dev->pdev->dev;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	free_mr = &priv->free_mr;
-	mr_free_cq = free_mr->mr_free_cq;
+	int ret;
+	int i;
 
 	for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
 		hr_qp = free_mr->mr_free_qp[i];
@@ -1092,19 +1062,15 @@ free_work:
 static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev,
 				struct hns_roce_mr *mr, struct ib_udata *udata)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_free_mr *free_mr = &priv->free_mr;
+	long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS;
 	struct device *dev = &hr_dev->pdev->dev;
 	struct hns_roce_mr_free_work *mr_work;
-	struct hns_roce_free_mr *free_mr;
-	struct hns_roce_v1_priv *priv;
-	struct completion comp;
-	long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS;
 	unsigned long start = jiffies;
-	int npages;
+	struct completion comp;
 	int ret = 0;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	free_mr = &priv->free_mr;
-
 	if (mr->enabled) {
 		if (hns_roce_hw_destroy_mpt(hr_dev, NULL,
 					    key_to_hw_index(mr->key) &
@@ -1146,17 +1112,9 @@ free_mr:
 	dev_dbg(dev, "Free mr 0x%x use 0x%x us.\n",
 		mr->key, jiffies_to_usecs(jiffies) - jiffies_to_usecs(start));
 
-	if (mr->size != ~0ULL) {
-		npages = ib_umem_page_count(mr->umem);
-		dma_free_coherent(dev, npages * 8, mr->pbl_buf,
-				  mr->pbl_dma_addr);
-	}
-
 	hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
 			     key_to_hw_index(mr->key), 0);
-
-	ib_umem_release(mr->umem);
-
+	hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr);
 	kfree(mr);
 
 	return ret;
@@ -1164,12 +1122,9 @@ free_mr:
 
 static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_db_table *db = &priv->db_table;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_db_table *db;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	db = &priv->db_table;
 
 	if (db->sdb_ext_mod) {
 		dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
@@ -1190,17 +1145,14 @@ static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
 
 static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
 {
-	int ret;
-	u32 val;
-	__le32 tmp;
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_raq_table *raq = &priv->raq_table;
+	struct device *dev = &hr_dev->pdev->dev;
 	int raq_shift = 0;
 	dma_addr_t addr;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_raq_table *raq;
-	struct device *dev = &hr_dev->pdev->dev;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	raq = &priv->raq_table;
+	__le32 tmp;
+	u32 val;
+	int ret;
 
 	raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
 	if (!raq->e_raq_buf)
@@ -1280,12 +1232,9 @@ err_dma_alloc_raq:
 
 static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_raq_table *raq = &priv->raq_table;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_raq_table *raq;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	raq = &priv->raq_table;
 
 	dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
 			  raq->e_raq_buf->map);
@@ -1319,12 +1268,10 @@ static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
 
 static int hns_roce_bt_init(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
 	int ret;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-
 	priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev,
 		HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map,
 		GFP_KERNEL);
@@ -1362,10 +1309,8 @@ err_failed_alloc_mtpt_buf:
 
 static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
 
 	dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 		priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map);
@@ -1379,12 +1324,9 @@ static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
 
 static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_buf_list *tptr_buf;
-	struct hns_roce_v1_priv *priv;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	tptr_buf = &priv->tptr_table.tptr_buf;
 
 	/*
 	 * This buffer will be used for CQ's tptr(tail pointer), also
@@ -1405,12 +1347,9 @@ static int hns_roce_tptr_init(struct hns_roce_dev *hr_dev)
 
 static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_buf_list *tptr_buf;
-	struct hns_roce_v1_priv *priv;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	tptr_buf = &priv->tptr_table.tptr_buf;
 
 	dma_free_coherent(dev, HNS_ROCE_V1_TPTR_BUF_SIZE,
 			  tptr_buf->buf, tptr_buf->map);
@@ -1418,14 +1357,11 @@ static void hns_roce_tptr_free(struct hns_roce_dev *hr_dev)
 
 static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_free_mr *free_mr = &priv->free_mr;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_free_mr *free_mr;
-	struct hns_roce_v1_priv *priv;
 	int ret = 0;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	free_mr = &priv->free_mr;
-
 	free_mr->free_mr_wq = create_singlethread_workqueue("hns_roce_free_mr");
 	if (!free_mr->free_mr_wq) {
 		dev_err(dev, "Create free mr workqueue failed!\n");
@@ -1444,11 +1380,8 @@ static int hns_roce_free_mr_init(struct hns_roce_dev *hr_dev)
 
 static void hns_roce_free_mr_free(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_free_mr *free_mr;
-	struct hns_roce_v1_priv *priv;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	free_mr = &priv->free_mr;
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_free_mr *free_mr = &priv->free_mr;
 
 	flush_workqueue(free_mr->free_mr_wq);
 	destroy_workqueue(free_mr->free_mr_wq);
@@ -1826,9 +1759,12 @@ static void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
 static int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
 				  unsigned long mtpt_idx)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(mr->ibmr.device);
+	u64 pages[HNS_ROCE_MAX_INNER_MTPT_NUM] = { 0 };
+	struct ib_device *ibdev = &hr_dev->ib_dev;
 	struct hns_roce_v1_mpt_entry *mpt_entry;
-	struct sg_dma_page_iter sg_iter;
-	u64 *pages;
+	dma_addr_t pbl_ba;
+	int count;
 	int i;
 
 	/* MPT filled into mailbox buf */
@@ -1878,22 +1814,15 @@ static int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
 	if (mr->type == MR_TYPE_DMA)
 		return 0;
 
-	pages = (u64 *) __get_free_page(GFP_KERNEL);
-	if (!pages)
-		return -ENOMEM;
-
-	i = 0;
-	for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
-		pages[i] = ((u64)sg_page_iter_dma_address(&sg_iter)) >> 12;
-
-		/* Directly record to MTPT table firstly 7 entry */
-		if (i >= HNS_ROCE_MAX_INNER_MTPT_NUM)
-			break;
-		i++;
+	count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
+				  ARRAY_SIZE(pages), &pbl_ba);
+	if (count < 1) {
+		ibdev_err(ibdev, "failed to find PBL mtr, count = %d.", count);
+		return -ENOBUFS;
 	}
 
 	/* Register user mr */
-	for (i = 0; i < HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {
+	for (i = 0; i < count; i++) {
 		switch (i) {
 		case 0:
 			mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
@@ -1959,20 +1888,17 @@ static int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
 		}
 	}
 
-	free_page((unsigned long) pages);
-
-	mpt_entry->pbl_addr_l = cpu_to_le32((u32)(mr->pbl_dma_addr));
-
+	mpt_entry->pbl_addr_l = cpu_to_le32(pbl_ba);
 	roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
-		       MPT_BYTE_12_PBL_ADDR_H_S,
-		       ((u32)(mr->pbl_dma_addr >> 32)));
+		       MPT_BYTE_12_PBL_ADDR_H_S, upper_32_bits(pbl_ba));
 
 	return 0;
 }
 
 static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
 {
-	return hns_roce_buf_offset(&hr_cq->buf, n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
+	return hns_roce_buf_offset(hr_cq->mtr.kmem,
+				   n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
 }
 
 static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
@@ -2066,16 +1992,12 @@ static void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
 				  struct hns_roce_cq *hr_cq, void *mb_buf,
 				  u64 *mtts, dma_addr_t dma_handle)
 {
-	struct hns_roce_cq_context *cq_context = NULL;
-	struct hns_roce_buf_list *tptr_buf;
-	struct hns_roce_v1_priv *priv;
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
+	struct hns_roce_buf_list *tptr_buf = &priv->tptr_table.tptr_buf;
+	struct hns_roce_cq_context *cq_context = mb_buf;
 	dma_addr_t tptr_dma_addr;
 	int offset;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	tptr_buf = &priv->tptr_table.tptr_buf;
-
-	cq_context = mb_buf;
 	memset(cq_context, 0, sizeof(*cq_context));
 
 	/* Get the tptr for this CQ. */
@@ -2416,16 +2338,14 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
 				 struct hns_roce_hem_table *table, int obj,
 				 int step_idx)
 {
+	struct hns_roce_v1_priv *priv = hr_dev->priv;
 	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	unsigned long flags = 0;
 	long end = HW_SYNC_TIMEOUT_MSECS;
 	__le32 bt_cmd_val[2] = {0};
+	unsigned long flags = 0;
 	void __iomem *bt_cmd;
 	u64 bt_ba = 0;
 
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-
 	switch (table->type) {
 	case HEM_TYPE_QPC:
 		bt_ba = priv->bt_table.qpc_buf.map >> 12;
@@ -2479,7 +2399,6 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
 }
 
 static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
-				 struct hns_roce_mtt *mtt,
 				 enum hns_roce_qp_state cur_state,
 				 enum hns_roce_qp_state new_state,
 				 struct hns_roce_qp_context *context,
@@ -2560,6 +2479,29 @@ static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
 	return ret;
 }
 
+static int find_wqe_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
+			u64 *sq_ba, u64 *rq_ba, dma_addr_t *bt_ba)
+{
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	int rq_pa_start;
+	int count;
+
+	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, sq_ba, 1, bt_ba);
+	if (count < 1) {
+		ibdev_err(ibdev, "Failed to find SQ ba\n");
+		return -ENOBUFS;
+	}
+	rq_pa_start = hr_qp->rq.offset >> hr_qp->mtr.hem_cfg.buf_pg_shift;
+	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, rq_pa_start, rq_ba, 1,
+				  NULL);
+	if (!count) {
+		ibdev_err(ibdev, "Failed to find RQ ba\n");
+		return -ENOBUFS;
+	}
+
+	return 0;
+}
+
 static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			     int attr_mask, enum ib_qp_state cur_state,
 			     enum ib_qp_state new_state)
@@ -2567,25 +2509,20 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 	struct hns_roce_sqp_context *context;
-	struct device *dev = &hr_dev->pdev->dev;
 	dma_addr_t dma_handle = 0;
 	u32 __iomem *addr;
-	int rq_pa_start;
+	u64 sq_ba = 0;
+	u64 rq_ba = 0;
 	__le32 tmp;
 	u32 reg_val;
-	u64 *mtts;
 
 	context = kzalloc(sizeof(*context), GFP_KERNEL);
 	if (!context)
 		return -ENOMEM;
 
 	/* Search QP buf's MTTs */
-	mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
-				   hr_qp->mtt.first_seg, &dma_handle);
-	if (!mtts) {
-		dev_err(dev, "qp buf pa find failed\n");
+	if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle))
 		goto out;
-	}
 
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		roce_set_field(context->qp1c_bytes_4,
@@ -2599,11 +2536,11 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 		roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
 			       QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
 
-		context->sq_rq_bt_l = cpu_to_le32((u32)(dma_handle));
+		context->sq_rq_bt_l = cpu_to_le32(dma_handle);
 		roce_set_field(context->qp1c_bytes_12,
 			       QP1C_BYTES_12_SQ_RQ_BT_H_M,
 			       QP1C_BYTES_12_SQ_RQ_BT_H_S,
-			       ((u32)(dma_handle >> 32)));
+			       upper_32_bits(dma_handle));
 
 		roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
 			       QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
@@ -2624,14 +2561,12 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 		roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
 			       QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
 
-		rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
-		context->cur_rq_wqe_ba_l =
-				cpu_to_le32((u32)(mtts[rq_pa_start]));
+		context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba);
 
 		roce_set_field(context->qp1c_bytes_28,
 			       QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
 			       QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
-			       (mtts[rq_pa_start]) >> 32);
+			       upper_32_bits(rq_ba));
 		roce_set_field(context->qp1c_bytes_28,
 			       QP1C_BYTES_28_RQ_CUR_IDX_M,
 			       QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
@@ -2645,12 +2580,12 @@ static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			       QP1C_BYTES_32_TX_CQ_NUM_S,
 			       to_hr_cq(ibqp->send_cq)->cqn);
 
-		context->cur_sq_wqe_ba_l  = cpu_to_le32((u32)mtts[0]);
+		context->cur_sq_wqe_ba_l = cpu_to_le32(sq_ba);
 
 		roce_set_field(context->qp1c_bytes_40,
 			       QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
 			       QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
-			       (mtts[0]) >> 32);
+			       upper_32_bits(sq_ba));
 		roce_set_field(context->qp1c_bytes_40,
 			       QP1C_BYTES_40_SQ_CUR_IDX_M,
 			       QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
@@ -2704,6 +2639,28 @@ out:
 	return -EINVAL;
 }
 
+static bool check_qp_state(enum ib_qp_state cur_state,
+			   enum ib_qp_state new_state)
+{
+	static const bool sm[][IB_QPS_ERR + 1] = {
+		[IB_QPS_RESET] = { [IB_QPS_RESET] = true,
+				   [IB_QPS_INIT] = true },
+		[IB_QPS_INIT] = { [IB_QPS_RESET] = true,
+				  [IB_QPS_INIT] = true,
+				  [IB_QPS_RTR] = true,
+				  [IB_QPS_ERR] = true },
+		[IB_QPS_RTR] = { [IB_QPS_RESET] = true,
+				 [IB_QPS_RTS] = true,
+				 [IB_QPS_ERR] = true },
+		[IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true },
+		[IB_QPS_SQD] = {},
+		[IB_QPS_SQE] = {},
+		[IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
+	};
+
+	return sm[cur_state][new_state];
+}
+
 static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			    int attr_mask, enum ib_qp_state cur_state,
 			    enum ib_qp_state new_state)
@@ -2716,26 +2673,29 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 	dma_addr_t dma_handle_2 = 0;
 	dma_addr_t dma_handle = 0;
 	__le32 doorbell[2] = {0};
-	int rq_pa_start = 0;
 	u64 *mtts_2 = NULL;
 	int ret = -EINVAL;
-	u64 *mtts = NULL;
+	u64 sq_ba = 0;
+	u64 rq_ba = 0;
 	int port;
 	u8 port_num;
 	u8 *dmac;
 	u8 *smac;
 
+	if (!check_qp_state(cur_state, new_state)) {
+		ibdev_err(ibqp->device,
+			  "not support QP(%u) status from %d to %d\n",
+			  ibqp->qp_num, cur_state, new_state);
+		return -EINVAL;
+	}
+
 	context = kzalloc(sizeof(*context), GFP_KERNEL);
 	if (!context)
 		return -ENOMEM;
 
 	/* Search qp buf's mtts */
-	mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
-				   hr_qp->mtt.first_seg, &dma_handle);
-	if (mtts == NULL) {
-		dev_err(dev, "qp buf pa find failed\n");
+	if (find_wqe_mtt(hr_dev, hr_qp, &sq_ba, &rq_ba, &dma_handle))
 		goto out;
-	}
 
 	/* Search IRRL's mtts */
 	mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
@@ -2890,11 +2850,11 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 
 		dmac = (u8 *)attr->ah_attr.roce.dmac;
 
-		context->sq_rq_bt_l = cpu_to_le32((u32)(dma_handle));
+		context->sq_rq_bt_l = cpu_to_le32(dma_handle);
 		roce_set_field(context->qpc_bytes_24,
 			       QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,
 			       QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,
-			       ((u32)(dma_handle >> 32)));
+			       upper_32_bits(dma_handle));
 		roce_set_bit(context->qpc_bytes_24,
 			     QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,
 			     1);
@@ -2993,14 +2953,12 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			       QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,
 			       QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);
 
-		rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
-		context->cur_rq_wqe_ba_l =
-				cpu_to_le32((u32)(mtts[rq_pa_start]));
+		context->cur_rq_wqe_ba_l = cpu_to_le32(rq_ba);
 
 		roce_set_field(context->qpc_bytes_76,
 			QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,
 			QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,
-			mtts[rq_pa_start] >> 32);
+			upper_32_bits(rq_ba));
 		roce_set_field(context->qpc_bytes_76,
 			       QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,
 			       QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);
@@ -3062,8 +3020,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			       QP_CONTEXT_QPC_BYTES_156_SL_S,
 			       rdma_ah_get_sl(&attr->ah_attr));
 		hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
-	} else if (cur_state == IB_QPS_RTR &&
-		new_state == IB_QPS_RTS) {
+	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
 		/* If exist optional param, return error */
 		if ((attr_mask & IB_QP_ALT_PATH) ||
 		    (attr_mask & IB_QP_ACCESS_FLAGS) ||
@@ -3075,12 +3032,12 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			goto out;
 		}
 
-		context->rx_cur_sq_wqe_ba_l = cpu_to_le32((u32)(mtts[0]));
+		context->rx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba);
 
 		roce_set_field(context->qpc_bytes_120,
 			       QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,
 			       QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,
-			       (mtts[0]) >> 32);
+			       upper_32_bits(sq_ba));
 
 		roce_set_field(context->qpc_bytes_124,
 			       QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,
@@ -3223,28 +3180,18 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			       QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,
 			       QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);
 
-		context->tx_cur_sq_wqe_ba_l = cpu_to_le32((u32)(mtts[0]));
+		context->tx_cur_sq_wqe_ba_l = cpu_to_le32(sq_ba);
 
 		roce_set_field(context->qpc_bytes_188,
 			       QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,
 			       QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,
-			       (mtts[0]) >> 32);
+			       upper_32_bits(sq_ba));
 		roce_set_bit(context->qpc_bytes_188,
 			     QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);
 		roce_set_field(context->qpc_bytes_188,
 			       QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,
 			       QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,
 			       0);
-	} else if (!((cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) ||
-		   (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) ||
-		   (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) ||
-		   (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) ||
-		   (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) ||
-		   (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) ||
-		   (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) ||
-		   (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR))) {
-		dev_err(dev, "not support this status migration\n");
-		goto out;
 	}
 
 	/* Every status migrate must change state */
@@ -3253,8 +3200,7 @@ static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 		       QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, new_state);
 
 	/* SW pass context to HW */
-	ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt,
-				    to_hns_roce_state(cur_state),
+	ret = hns_roce_v1_qp_modify(hr_dev, to_hns_roce_state(cur_state),
 				    to_hns_roce_state(new_state), context,
 				    hr_qp);
 	if (ret) {
@@ -3636,8 +3582,6 @@ static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	u32 cqe_cnt_cur;
 	int wait_time = 0;
 
-	hns_roce_free_cqc(hr_dev, hr_cq);
-
 	/*
 	 * Before freeing cq buffer, we need to ensure that the outstanding CQE
 	 * have been written by checking the CQE counter.
@@ -3660,14 +3604,6 @@ static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 		}
 		wait_time++;
 	}
-
-	hns_roce_mtt_cleanup(hr_dev, &hr_cq->mtt);
-
-	ib_umem_release(hr_cq->umem);
-	if (!udata) {
-		/* Free the buff of stored cq */
-		hns_roce_buf_free(hr_dev, hr_cq->buf.size, &hr_cq->buf);
-	}
 }
 
 static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not)
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index c3316672b70e..c597d7281629 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -95,6 +95,7 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 {
 	struct hns_roce_mr *mr = to_hr_mr(wr->mr);
 	struct hns_roce_wqe_frmr_seg *fseg = wqe;
+	u64 pbl_ba;
 
 	/* use ib_access_flags */
 	roce_set_bit(rc_sq_wqe->byte_4, V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
@@ -109,26 +110,27 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 		     wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
 
 	/* Data structure reuse may lead to confusion */
-	rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff);
-	rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32);
+	pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
+	rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
+	rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));
 
 	rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
 	rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
 	rc_sq_wqe->rkey = cpu_to_le32(wr->key);
 	rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
 
-	fseg->pbl_size = cpu_to_le32(mr->pbl_size);
+	fseg->pbl_size = cpu_to_le32(mr->npages);
 	roce_set_field(fseg->mode_buf_pg_sz,
 		       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
 		       V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
-		       mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
 	roce_set_bit(fseg->mode_buf_pg_sz,
 		     V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
 }
 
 static void set_atomic_seg(const struct ib_send_wr *wr, void *wqe,
 			   struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
-			   int valid_num_sge)
+			   unsigned int valid_num_sge)
 {
 	struct hns_roce_wqe_atomic_seg *aseg;
 
@@ -149,56 +151,33 @@ static void set_atomic_seg(const struct ib_send_wr *wr, void *wqe,
 }
 
 static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
-			   unsigned int *sge_ind, int valid_num_sge)
+			   unsigned int *sge_ind, unsigned int valid_num_sge)
 {
 	struct hns_roce_v2_wqe_data_seg *dseg;
-	struct ib_sge *sg;
-	int num_in_wqe = 0;
-	int extend_sge_num;
-	int fi_sge_num;
-	int se_sge_num;
-	int shift;
-	int i;
+	unsigned int cnt = valid_num_sge;
+	struct ib_sge *sge = wr->sg_list;
+	unsigned int idx = *sge_ind;
 
-	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
-		num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
-	extend_sge_num = valid_num_sge - num_in_wqe;
-	sg = wr->sg_list + num_in_wqe;
-	shift = qp->hr_buf.page_shift;
+	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
+		cnt -= HNS_ROCE_SGE_IN_WQE;
+		sge += HNS_ROCE_SGE_IN_WQE;
+	}
 
-	/*
-	 * Check whether wr->num_sge sges are in the same page. If not, we
-	 * should calculate how many sges in the first page and the second
-	 * page.
-	 */
-	dseg = hns_roce_get_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
-	fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
-		      (uintptr_t)dseg) /
-		      sizeof(struct hns_roce_v2_wqe_data_seg);
-	if (extend_sge_num > fi_sge_num) {
-		se_sge_num = extend_sge_num - fi_sge_num;
-		for (i = 0; i < fi_sge_num; i++) {
-			set_data_seg_v2(dseg++, sg + i);
-			(*sge_ind)++;
-		}
-		dseg = hns_roce_get_extend_sge(qp,
-					   (*sge_ind) & (qp->sge.sge_cnt - 1));
-		for (i = 0; i < se_sge_num; i++) {
-			set_data_seg_v2(dseg++, sg + fi_sge_num + i);
-			(*sge_ind)++;
-		}
-	} else {
-		for (i = 0; i < extend_sge_num; i++) {
-			set_data_seg_v2(dseg++, sg + i);
-			(*sge_ind)++;
-		}
+	while (cnt > 0) {
+		dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
+		set_data_seg_v2(dseg, sge);
+		idx++;
+		sge++;
+		cnt--;
 	}
+
+	*sge_ind = idx;
 }
 
 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 			     struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 			     void *wqe, unsigned int *sge_ind,
-			     int valid_num_sge)
+			     unsigned int valid_num_sge)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_v2_wqe_data_seg *dseg = wqe;
@@ -208,15 +187,15 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 	int i;
 
 	if (wr->send_flags & IB_SEND_INLINE && valid_num_sge) {
-		if (le32_to_cpu(rc_sq_wqe->msg_len) >
-		    hr_dev->caps.max_sq_inline) {
+		if (unlikely(le32_to_cpu(rc_sq_wqe->msg_len) >
+			     hr_dev->caps.max_sq_inline)) {
 			ibdev_err(ibdev, "inline len(1-%d)=%d, illegal",
 				  rc_sq_wqe->msg_len,
 				  hr_dev->caps.max_sq_inline);
 			return -EINVAL;
 		}
 
-		if (wr->opcode == IB_WR_RDMA_READ) {
+		if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
 			ibdev_err(ibdev, "Not support inline data!\n");
 			return -EINVAL;
 		}
@@ -230,7 +209,7 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 		roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
 			     1);
 	} else {
-		if (valid_num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
+		if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
 			for (i = 0; i < wr->num_sge; i++) {
 				if (likely(wr->sg_list[i].length)) {
 					set_data_seg_v2(dseg, wr->sg_list + i);
@@ -243,8 +222,8 @@ static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 				     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 				     (*sge_ind) & (qp->sge.sge_cnt - 1));
 
-			for (i = 0; i < wr->num_sge &&
-			     j < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
+			for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE;
+			     i++) {
 				if (likely(wr->sg_list[i].length)) {
 					set_data_seg_v2(dseg, wr->sg_list + i);
 					dseg++;
@@ -290,10 +269,11 @@ static int check_send_valid(struct hns_roce_dev *hr_dev,
 	return 0;
 }
 
-static inline int calc_wr_sge_num(const struct ib_send_wr *wr, u32 *sge_len)
+static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
+				    unsigned int *sge_len)
 {
-	int valid_num = 0;
-	u32 len = 0;
+	unsigned int valid_num = 0;
+	unsigned int len = 0;
 	int i;
 
 	for (i = 0; i < wr->num_sge; i++) {
@@ -424,7 +404,7 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp,
 {
 	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
 	unsigned int curr_idx = *sge_idx;
-	int valid_num_sge;
+	unsigned int valid_num_sge;
 	u32 msg_len = 0;
 	int ret = 0;
 
@@ -521,8 +501,7 @@ static inline void update_sq_db(struct hns_roce_dev *hr_dev,
 		roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
 			       V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
 		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
-			       V2_DB_PARAMETER_IDX_S,
-			       qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
+			       V2_DB_PARAMETER_IDX_S, qp->sq.head);
 		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
 			       V2_DB_PARAMETER_SL_S, qp->sl);
 
@@ -548,7 +527,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 	spin_lock_irqsave(&qp->sq.lock, flags);
 
 	ret = check_send_valid(hr_dev, qp);
-	if (ret) {
+	if (unlikely(ret)) {
 		*bad_wr = wr;
 		nreq = 0;
 		goto out;
@@ -584,7 +563,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 		else if (ibqp->qp_type == IB_QPT_RC)
 			ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 
-		if (ret) {
+		if (unlikely(ret)) {
 			*bad_wr = wr;
 			goto out;
 		}
@@ -634,15 +613,15 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 	spin_lock_irqsave(&hr_qp->rq.lock, flags);
 
 	ret = check_recv_valid(hr_dev, hr_qp);
-	if (ret) {
+	if (unlikely(ret)) {
 		*bad_wr = wr;
 		nreq = 0;
 		goto out;
 	}
 
 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
-		if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
-			hr_qp->ibqp.recv_cq)) {
+		if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
+						  hr_qp->ibqp.recv_cq))) {
 			ret = -ENOMEM;
 			*bad_wr = wr;
 			goto out;
@@ -650,7 +629,7 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 
 		wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
 
-		if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
+		if (unlikely(wr->num_sge >= hr_qp->rq.max_gs)) {
 			ibdev_err(ibdev, "rq:num_sge=%d >= qp->sq.max_gs=%d\n",
 				  wr->num_sge, hr_qp->rq.max_gs);
 			ret = -EINVAL;
@@ -667,13 +646,14 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 			dseg++;
 		}
 
-		if (i < hr_qp->rq.max_gs) {
+		if (wr->num_sge < hr_qp->rq.max_gs) {
 			dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
 			dseg->addr = 0;
+			dseg->len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);
 		}
 
 		/* rq support inline data */
-		if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
+		if (hr_qp->rq_inl_buf.wqe_cnt) {
 			sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list;
 			hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt =
 							       (u32)wr->num_sge;
@@ -715,6 +695,129 @@ out:
 	return ret;
 }
 
+static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
+{
+	return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
+}
+
+static void *get_idx_buf(struct hns_roce_idx_que *idx_que, int n)
+{
+	return hns_roce_buf_offset(idx_que->mtr.kmem,
+				   n << idx_que->entry_shift);
+}
+
+static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
+{
+	/* always called with interrupts disabled. */
+	spin_lock(&srq->lock);
+
+	bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
+	srq->tail++;
+
+	spin_unlock(&srq->lock);
+}
+
+static int find_empty_entry(struct hns_roce_idx_que *idx_que,
+			    unsigned long size)
+{
+	int wqe_idx;
+
+	if (unlikely(bitmap_full(idx_que->bitmap, size)))
+		return -ENOSPC;
+
+	wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
+
+	bitmap_set(idx_que->bitmap, wqe_idx, 1);
+
+	return wqe_idx;
+}
+
+static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
+				     const struct ib_recv_wr *wr,
+				     const struct ib_recv_wr **bad_wr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
+	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
+	struct hns_roce_v2_wqe_data_seg *dseg;
+	struct hns_roce_v2_db srq_db;
+	unsigned long flags;
+	__le32 *srq_idx;
+	int ret = 0;
+	int wqe_idx;
+	void *wqe;
+	int nreq;
+	int ind;
+	int i;
+
+	spin_lock_irqsave(&srq->lock, flags);
+
+	ind = srq->head & (srq->wqe_cnt - 1);
+
+	for (nreq = 0; wr; ++nreq, wr = wr->next) {
+		if (unlikely(wr->num_sge >= srq->max_gs)) {
+			ret = -EINVAL;
+			*bad_wr = wr;
+			break;
+		}
+
+		if (unlikely(srq->head == srq->tail)) {
+			ret = -ENOMEM;
+			*bad_wr = wr;
+			break;
+		}
+
+		wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
+		if (unlikely(wqe_idx < 0)) {
+			ret = -ENOMEM;
+			*bad_wr = wr;
+			break;
+		}
+
+		wqe = get_srq_wqe(srq, wqe_idx);
+		dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
+
+		for (i = 0; i < wr->num_sge; ++i) {
+			dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
+			dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
+			dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
+		}
+
+		if (wr->num_sge < srq->max_gs) {
+			dseg[i].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);
+			dseg[i].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
+			dseg[i].addr = 0;
+		}
+
+		srq_idx = get_idx_buf(&srq->idx_que, ind);
+		*srq_idx = cpu_to_le32(wqe_idx);
+
+		srq->wrid[wqe_idx] = wr->wr_id;
+		ind = (ind + 1) & (srq->wqe_cnt - 1);
+	}
+
+	if (likely(nreq)) {
+		srq->head += nreq;
+
+		/*
+		 * Make sure that descriptors are written before
+		 * doorbell record.
+		 */
+		wmb();
+
+		srq_db.byte_4 =
+			cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
+				    (srq->srqn & V2_DB_BYTE_4_TAG_M));
+		srq_db.parameter =
+			cpu_to_le32(srq->head & V2_DB_PARAMETER_IDX_M);
+
+		hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
+	}
+
+	spin_unlock_irqrestore(&srq->lock, flags);
+
+	return ret;
+}
+
 static int hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
 				      unsigned long instance_stage,
 				      unsigned long reset_stage)
@@ -742,7 +845,7 @@ static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
 					unsigned long instance_stage,
 					unsigned long reset_stage)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hnae3_handle *handle = priv->handle;
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 
@@ -768,7 +871,7 @@ static int hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
 
 static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hnae3_handle *handle = priv->handle;
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 
@@ -785,7 +888,7 @@ static int hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
 
 static int hns_roce_v2_rst_process_cmd(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hnae3_handle *handle = priv->handle;
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 	unsigned long instance_stage;	/* the current instance stage */
@@ -865,7 +968,7 @@ static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
 
 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
 					    &priv->cmq.csq : &priv->cmq.crq;
 
@@ -878,7 +981,7 @@ static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
 
 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
 					    &priv->cmq.csq : &priv->cmq.crq;
 	dma_addr_t dma = ring->desc_dma_addr;
@@ -904,7 +1007,7 @@ static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
 
 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	int ret;
 
 	/* Setup the queue entries for command queue */
@@ -948,7 +1051,7 @@ err_crq:
 
 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 
 	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
 	hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
@@ -970,15 +1073,15 @@ static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
 
 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 	u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 
 	return head == priv->cmq.csq.next_to_use;
 }
 
 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
 	struct hns_roce_cmq_desc *desc;
 	u16 ntc = csq->next_to_clean;
@@ -1003,7 +1106,7 @@ static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
 			       struct hns_roce_cmq_desc *desc, int num)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
 	struct hns_roce_cmq_desc *desc_to_use;
 	bool complete = false;
@@ -1131,7 +1234,7 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
 
 static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hnae3_handle *handle = priv->handle;
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 	unsigned long reset_cnt;
@@ -1151,7 +1254,7 @@ static bool hns_roce_func_clr_chk_rst(struct hns_roce_dev *hr_dev)
 static void hns_roce_func_clr_rst_prc(struct hns_roce_dev *hr_dev, int retval,
 				      int flag)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hnae3_handle *handle = priv->handle;
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 	unsigned long instance_stage;
@@ -1349,34 +1452,26 @@ static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
 static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_pf_timer_res_a *req_a;
-	struct hns_roce_cmq_desc desc[2];
-	int ret, i;
+	struct hns_roce_cmq_desc desc;
+	int ret;
 
-	for (i = 0; i < 2; i++) {
-		hns_roce_cmq_setup_basic_desc(&desc[i],
-					      HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
-					      true);
+	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
+				      true);
 
-		if (i == 0)
-			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
-		else
-			desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
-	}
-
-	ret = hns_roce_cmq_send(hr_dev, desc, 2);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
 	if (ret)
 		return ret;
 
-	req_a = (struct hns_roce_pf_timer_res_a *)desc[0].data;
+	req_a = (struct hns_roce_pf_timer_res_a *)desc.data;
 
 	hr_dev->caps.qpc_timer_bt_num =
-				roce_get_field(req_a->qpc_timer_bt_idx_num,
-					PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
-					PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
+		roce_get_field(req_a->qpc_timer_bt_idx_num,
+			       PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_M,
+			       PF_RES_DATA_1_PF_QPC_TIMER_BT_NUM_S);
 	hr_dev->caps.cqc_timer_bt_num =
-				roce_get_field(req_a->cqc_timer_bt_idx_num,
-					PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
-					PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
+		roce_get_field(req_a->cqc_timer_bt_idx_num,
+			       PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M,
+			       PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S);
 
 	return 0;
 }
@@ -1786,6 +1881,9 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
 	caps->flags = roce_get_field(resp_c->cap_flags_num_pds,
 				     V2_QUERY_PF_CAPS_C_CAP_FLAGS_M,
 				     V2_QUERY_PF_CAPS_C_CAP_FLAGS_S);
+	caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<
+		       HNS_ROCE_CAP_FLAGS_EX_SHIFT;
+
 	caps->num_cqs = 1 << roce_get_field(resp_c->max_gid_num_cqs,
 					    V2_QUERY_PF_CAPS_C_NUM_CQS_M,
 					    V2_QUERY_PF_CAPS_C_NUM_CQS_S);
@@ -1978,11 +2076,6 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
 	hr_dev->vendor_part_id = hr_dev->pci_dev->device;
 	hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
 
-	caps->num_mtt_segs	= HNS_ROCE_V2_MAX_MTT_SEGS;
-	caps->num_cqe_segs	= HNS_ROCE_V2_MAX_CQE_SEGS;
-	caps->num_srqwqe_segs	= HNS_ROCE_V2_MAX_SRQWQE_SEGS;
-	caps->num_idx_segs	= HNS_ROCE_V2_MAX_IDX_SEGS;
-
 	caps->pbl_ba_pg_sz	= HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
 	caps->pbl_buf_pg_sz	= 0;
 	caps->pbl_hop_num	= HNS_ROCE_PBL_HOP_NUM;
@@ -2040,8 +2133,6 @@ static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
 
 	page_num = link_tbl->npages;
 	entry = link_tbl->table.buf;
-	memset(req_a, 0, sizeof(*req_a));
-	memset(req_b, 0, sizeof(*req_b));
 
 	for (i = 0; i < 2; i++) {
 		hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
@@ -2050,39 +2141,30 @@ static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
 			desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
 		else
 			desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
-
-		if (i == 0) {
-			req_a->base_addr_l =
-				cpu_to_le32(link_tbl->table.map & 0xffffffff);
-			req_a->base_addr_h =
-				cpu_to_le32(link_tbl->table.map >> 32);
-			roce_set_field(req_a->depth_pgsz_init_en,
-				       CFG_LLM_QUE_DEPTH_M, CFG_LLM_QUE_DEPTH_S,
-				       link_tbl->npages);
-			roce_set_field(req_a->depth_pgsz_init_en,
-				       CFG_LLM_QUE_PGSZ_M, CFG_LLM_QUE_PGSZ_S,
-				       link_tbl->pg_sz);
-			req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
-			req_a->head_ba_h_nxtptr =
-				cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
-			roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M,
-				       CFG_LLM_HEAD_PTR_S, 0);
-		} else {
-			req_b->tail_ba_l =
-				cpu_to_le32(entry[page_num - 1].blk_ba0);
-			roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M,
-				       CFG_LLM_TAIL_BA_H_S,
-				       entry[page_num - 1].blk_ba1_nxt_ptr &
-					       HNS_ROCE_LINK_TABLE_BA1_M);
-			roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M,
-				       CFG_LLM_TAIL_PTR_S,
-				       (entry[page_num - 2].blk_ba1_nxt_ptr &
-					HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
-					       HNS_ROCE_LINK_TABLE_NXT_PTR_S);
-		}
 	}
+
+	req_a->base_addr_l = cpu_to_le32(link_tbl->table.map & 0xffffffff);
+	req_a->base_addr_h = cpu_to_le32(link_tbl->table.map >> 32);
+	roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_DEPTH_M,
+		       CFG_LLM_QUE_DEPTH_S, link_tbl->npages);
+	roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_QUE_PGSZ_M,
+		       CFG_LLM_QUE_PGSZ_S, link_tbl->pg_sz);
 	roce_set_field(req_a->depth_pgsz_init_en, CFG_LLM_INIT_EN_M,
 		       CFG_LLM_INIT_EN_S, 1);
+	req_a->head_ba_l = cpu_to_le32(entry[0].blk_ba0);
+	req_a->head_ba_h_nxtptr = cpu_to_le32(entry[0].blk_ba1_nxt_ptr);
+	roce_set_field(req_a->head_ptr, CFG_LLM_HEAD_PTR_M, CFG_LLM_HEAD_PTR_S,
+		       0);
+
+	req_b->tail_ba_l = cpu_to_le32(entry[page_num - 1].blk_ba0);
+	roce_set_field(req_b->tail_ba_h, CFG_LLM_TAIL_BA_H_M,
+		       CFG_LLM_TAIL_BA_H_S,
+		       entry[page_num - 1].blk_ba1_nxt_ptr &
+		       HNS_ROCE_LINK_TABLE_BA1_M);
+	roce_set_field(req_b->tail_ptr, CFG_LLM_TAIL_PTR_M, CFG_LLM_TAIL_PTR_S,
+		       (entry[page_num - 2].blk_ba1_nxt_ptr &
+			HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
+			HNS_ROCE_LINK_TABLE_NXT_PTR_S);
 
 	return hns_roce_cmq_send(hr_dev, desc, 2);
 }
@@ -2438,12 +2520,9 @@ static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
 	reg_smac_l = *(u32 *)(&addr[0]);
 	reg_smac_h = *(u16 *)(&addr[4]);
 
-	memset(smac_tb, 0, sizeof(*smac_tb));
-	roce_set_field(smac_tb->tb_idx_rsv,
-		       CFG_SMAC_TB_IDX_M,
+	roce_set_field(smac_tb->tb_idx_rsv, CFG_SMAC_TB_IDX_M,
 		       CFG_SMAC_TB_IDX_S, phy_port);
-	roce_set_field(smac_tb->vf_smac_h_rsv,
-		       CFG_SMAC_TB_VF_SMAC_H_M,
+	roce_set_field(smac_tb->vf_smac_h_rsv, CFG_SMAC_TB_VF_SMAC_H_M,
 		       CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
 	smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
 
@@ -2453,32 +2532,30 @@ static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
 static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
 			struct hns_roce_mr *mr)
 {
-	struct sg_dma_page_iter sg_iter;
-	u64 page_addr;
-	u64 *pages;
-	int i;
+	struct hns_roce_dev *hr_dev = to_hr_dev(mr->ibmr.device);
+	u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	dma_addr_t pbl_ba;
+	int i, count;
 
-	mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
-	mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
-	roce_set_field(mpt_entry->byte_48_mode_ba,
-		       V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
-		       upper_32_bits(mr->pbl_ba >> 3));
+	count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
+				  ARRAY_SIZE(pages), &pbl_ba);
+	if (count < 1) {
+		ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n",
+			  count);
+		return -ENOBUFS;
+	}
 
-	pages = (u64 *)__get_free_page(GFP_KERNEL);
-	if (!pages)
-		return -ENOMEM;
+	/* Aligned to the hardware address access unit */
+	for (i = 0; i < count; i++)
+		pages[i] >>= 6;
 
-	i = 0;
-	for_each_sg_dma_page(mr->umem->sg_head.sgl, &sg_iter, mr->umem->nmap, 0) {
-		page_addr = sg_page_iter_dma_address(&sg_iter);
-		pages[i] = page_addr >> 6;
+	mpt_entry->pbl_size = cpu_to_le32(mr->npages);
+	mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3);
+	roce_set_field(mpt_entry->byte_48_mode_ba,
+		       V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
+		       upper_32_bits(pbl_ba >> 3));
 
-		/* Record the first 2 entry directly to MTPT table */
-		if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
-			goto found;
-		i++;
-	}
-found:
 	mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
 	roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
 		       V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
@@ -2489,9 +2566,7 @@ found:
 	roce_set_field(mpt_entry->byte_64_buf_pa1,
 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
-		       mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
-
-	free_page((unsigned long)pages);
+		       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
 
 	return 0;
 }
@@ -2513,7 +2588,7 @@ static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
 	roce_set_field(mpt_entry->byte_4_pd_hop_st,
 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
-		       mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
 		       V2_MPT_BYTE_4_PD_S, mr->pd);
 
@@ -2599,11 +2674,19 @@ static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
 
 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(mr->ibmr.device);
+	struct ib_device *ibdev = &hr_dev->ib_dev;
 	struct hns_roce_v2_mpt_entry *mpt_entry;
+	dma_addr_t pbl_ba = 0;
 
 	mpt_entry = mb_buf;
 	memset(mpt_entry, 0, sizeof(*mpt_entry));
 
+	if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) {
+		ibdev_err(ibdev, "failed to find frmr mtr.\n");
+		return -ENOBUFS;
+	}
+
 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
 		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
@@ -2611,7 +2694,7 @@ static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
 	roce_set_field(mpt_entry->byte_4_pd_hop_st,
 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
 		       V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
-		       mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
 		       V2_MPT_BYTE_4_PD_S, mr->pd);
 
@@ -2624,17 +2707,17 @@ static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
 	roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
 
-	mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
+	mpt_entry->pbl_size = cpu_to_le32(mr->npages);
 
-	mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
+	mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3));
 	roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
 		       V2_MPT_BYTE_48_PBL_BA_H_S,
-		       upper_32_bits(mr->pbl_ba >> 3));
+		       upper_32_bits(pbl_ba >> 3));
 
 	roce_set_field(mpt_entry->byte_64_buf_pa1,
 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
 		       V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
-		       mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
 
 	return 0;
 }
@@ -2680,7 +2763,8 @@ static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
 
 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
 {
-	return hns_roce_buf_offset(&hr_cq->buf, n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
+	return hns_roce_buf_offset(hr_cq->mtr.kmem,
+				   n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
 }
 
 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
@@ -2692,30 +2776,9 @@ static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
 		!!(n & hr_cq->cq_depth)) ? cqe : NULL;
 }
 
-static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
+static inline void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 ci)
 {
-	return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
-}
-
-static void *get_srq_wqe(struct hns_roce_srq *srq, int n)
-{
-	return hns_roce_buf_offset(&srq->buf, n << srq->wqe_shift);
-}
-
-static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index)
-{
-	/* always called with interrupts disabled. */
-	spin_lock(&srq->lock);
-
-	bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
-	srq->tail++;
-
-	spin_unlock(&srq->lock);
-}
-
-static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
-{
-	*hr_cq->set_ci_db = cons_index & V2_CQ_DB_PARAMETER_CONS_IDX_M;
+	*hr_cq->set_ci_db = ci & V2_CQ_DB_PARAMETER_CONS_IDX_M;
 }
 
 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
@@ -2801,39 +2864,39 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
 	roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
 		       V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
 
-	cq_context->cqe_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
+	cq_context->cqe_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
 
 	roce_set_field(cq_context->byte_16_hop_addr,
 		       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
 		       V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
-		       mtts[0] >> (32 + PAGE_ADDR_SHIFT));
+		       upper_32_bits(to_hr_hw_page_addr(mtts[0])));
 	roce_set_field(cq_context->byte_16_hop_addr,
 		       V2_CQC_BYTE_16_CQE_HOP_NUM_M,
 		       V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
 		       HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
 
-	cq_context->cqe_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
+	cq_context->cqe_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
 	roce_set_field(cq_context->byte_24_pgsz_addr,
 		       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
 		       V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
-		       mtts[1] >> (32 + PAGE_ADDR_SHIFT));
+		       upper_32_bits(to_hr_hw_page_addr(mtts[1])));
 	roce_set_field(cq_context->byte_24_pgsz_addr,
 		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
 		       V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
-		       hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
 	roce_set_field(cq_context->byte_24_pgsz_addr,
 		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
 		       V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
-		       hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));
 
 	cq_context->cqe_ba = cpu_to_le32(dma_handle >> 3);
 
 	roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
 		       V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
 
-	if (hr_cq->db_en)
-		roce_set_bit(cq_context->byte_44_db_record,
-			     V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
+	roce_set_bit(cq_context->byte_44_db_record,
+		     V2_CQC_BYTE_44_DB_RECORD_EN_S,
+		     (hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB) ? 1 : 0);
 
 	roce_set_field(cq_context->byte_44_db_record,
 		       V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
@@ -2873,8 +2936,7 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
 	roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
 		       HNS_ROCE_V2_CQ_DB_NTR);
 	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
-		       V2_CQ_DB_PARAMETER_CONS_IDX_S,
-		       hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
+		       V2_CQ_DB_PARAMETER_CONS_IDX_S, hr_cq->cons_index);
 	roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
 		       V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
 	roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
@@ -2911,7 +2973,7 @@ static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
 		wqe_buf += size;
 	}
 
-	if (data_len) {
+	if (unlikely(data_len)) {
 		wc->status = IB_WC_LOC_LEN_ERR;
 		return -EAGAIN;
 	}
@@ -2968,6 +3030,62 @@ out:
 	return npolled;
 }
 
+static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
+			   struct hns_roce_v2_cqe *cqe, struct ib_wc *wc)
+{
+	static const struct {
+		u32 cqe_status;
+		enum ib_wc_status wc_status;
+	} map[] = {
+		{ HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },
+		{ HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },
+		{ HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
+		{ HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },
+		{ HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
+		{ HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },
+		{ HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
+		{ HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
+		{ HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
+		{ HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
+		{ HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },
+		{ HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,
+		  IB_WC_RETRY_EXC_ERR },
+		{ HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
+		{ HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
+	};
+
+	u32 cqe_status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
+					V2_CQE_BYTE_4_STATUS_S);
+	int i;
+
+	wc->status = IB_WC_GENERAL_ERR;
+	for (i = 0; i < ARRAY_SIZE(map); i++)
+		if (cqe_status == map[i].cqe_status) {
+			wc->status = map[i].wc_status;
+			break;
+		}
+
+	if (likely(wc->status == IB_WC_SUCCESS ||
+		   wc->status == IB_WC_WR_FLUSH_ERR))
+		return;
+
+	ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status);
+	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe,
+		       sizeof(*cqe), false);
+
+	/*
+	 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets
+	 * into errored mode. Hence, as a workaround to this hardware
+	 * limitation, driver needs to assist in flushing. But the flushing
+	 * operation uses mailbox to convey the QP state to the hardware and
+	 * which can sleep due to the mutex protection around the mailbox calls.
+	 * Hence, use the deferred flush for now. Once wc error detected, the
+	 * flushing operation is needed.
+	 */
+	if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
+		init_flush_work(hr_dev, qp);
+}
+
 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 				struct hns_roce_qp **cur_qp, struct ib_wc *wc)
 {
@@ -2979,12 +3097,11 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 	int is_send;
 	u16 wqe_ctr;
 	u32 opcode;
-	u32 status;
 	int qpn;
 	int ret;
 
 	/* Find cqe according to consumer index */
-	cqe = next_cqe_sw_v2(hr_cq);
+	cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
 	if (!cqe)
 		return -EAGAIN;
 
@@ -3009,7 +3126,6 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 		*cur_qp = hr_qp;
 	}
 
-	hr_qp = *cur_qp;
 	wc->qp = &(*cur_qp)->ibqp;
 	wc->vendor_err = 0;
 
@@ -3044,77 +3160,8 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 		++wq->tail;
 	}
 
-	status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
-				V2_CQE_BYTE_4_STATUS_S);
-	switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
-	case HNS_ROCE_CQE_V2_SUCCESS:
-		wc->status = IB_WC_SUCCESS;
-		break;
-	case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
-		wc->status = IB_WC_LOC_LEN_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
-		wc->status = IB_WC_LOC_QP_OP_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
-		wc->status = IB_WC_LOC_PROT_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
-		wc->status = IB_WC_WR_FLUSH_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_MW_BIND_ERR:
-		wc->status = IB_WC_MW_BIND_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
-		wc->status = IB_WC_BAD_RESP_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
-		wc->status = IB_WC_LOC_ACCESS_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
-		wc->status = IB_WC_REM_INV_REQ_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
-		wc->status = IB_WC_REM_ACCESS_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
-		wc->status = IB_WC_REM_OP_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
-		wc->status = IB_WC_RETRY_EXC_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
-		wc->status = IB_WC_RNR_RETRY_EXC_ERR;
-		break;
-	case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
-		wc->status = IB_WC_REM_ABORT_ERR;
-		break;
-	default:
-		wc->status = IB_WC_GENERAL_ERR;
-		break;
-	}
-
-	/*
-	 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state gets
-	 * into errored mode. Hence, as a workaround to this hardware
-	 * limitation, driver needs to assist in flushing. But the flushing
-	 * operation uses mailbox to convey the QP state to the hardware and
-	 * which can sleep due to the mutex protection around the mailbox calls.
-	 * Hence, use the deferred flush for now. Once wc error detected, the
-	 * flushing operation is needed.
-	 */
-	if (wc->status != IB_WC_SUCCESS &&
-	    wc->status != IB_WC_WR_FLUSH_ERR) {
-		ibdev_err(&hr_dev->ib_dev, "error cqe status is: 0x%x\n",
-			  status & HNS_ROCE_V2_CQE_STATUS_MASK);
-
-		if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag))
-			init_flush_work(hr_dev, hr_qp);
-
-		return 0;
-	}
-
-	if (wc->status == IB_WC_WR_FLUSH_ERR)
+	get_cqe_status(hr_dev, *cur_qp, cqe, wc);
+	if (unlikely(wc->status != IB_WC_SUCCESS))
 		return 0;
 
 	if (is_send) {
@@ -3213,7 +3260,7 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 		    opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
 		    (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
 			ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
-			if (ret)
+			if (unlikely(ret))
 				return -EAGAIN;
 		}
 
@@ -3514,29 +3561,18 @@ static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
 			    struct hns_roce_v2_qp_context *context,
 			    struct hns_roce_v2_qp_context *qpc_mask)
 {
-	if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
-		roce_set_field(context->byte_4_sqpn_tst,
-			       V2_QPC_BYTE_4_SGE_SHIFT_M,
-			       V2_QPC_BYTE_4_SGE_SHIFT_S,
-			       ilog2((unsigned int)hr_qp->sge.sge_cnt));
-	else
-		roce_set_field(context->byte_4_sqpn_tst,
-			       V2_QPC_BYTE_4_SGE_SHIFT_M,
-			       V2_QPC_BYTE_4_SGE_SHIFT_S,
-			       hr_qp->sq.max_gs >
-			       HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ?
-			       ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
+	roce_set_field(context->byte_4_sqpn_tst,
+		       V2_QPC_BYTE_4_SGE_SHIFT_M, V2_QPC_BYTE_4_SGE_SHIFT_S,
+		       to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,
+					       hr_qp->sge.sge_shift));
 
 	roce_set_field(context->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
-		       ilog2((unsigned int)hr_qp->sq.wqe_cnt));
+		       ilog2(hr_qp->sq.wqe_cnt));
 
 	roce_set_field(context->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
-		       (hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
-		       hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT ||
-		       hr_qp->ibqp.srq) ? 0 :
-		       ilog2((unsigned int)hr_qp->rq.wqe_cnt));
+		       ilog2(hr_qp->rq.wqe_cnt));
 }
 
 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
@@ -3572,7 +3608,7 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp,
 	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
 		       V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
 
-	if (hr_qp->rdb_en)
+	if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 		roce_set_bit(context->byte_68_rq_db,
 			     V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
 
@@ -3734,30 +3770,19 @@ static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev,
 	return true;
 }
 
-static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
-				 const struct ib_qp_attr *attr, int attr_mask,
-				 struct hns_roce_v2_qp_context *context,
-				 struct hns_roce_v2_qp_context *qpc_mask)
+static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
+			    struct hns_roce_qp *hr_qp,
+			    struct hns_roce_v2_qp_context *context,
+			    struct hns_roce_v2_qp_context *qpc_mask)
 {
-	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
-	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
-	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
-	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct ib_qp *ibqp = &hr_qp->ibqp;
 	u64 mtts[MTT_MIN_COUNT] = { 0 };
-	dma_addr_t dma_handle_3;
-	dma_addr_t dma_handle_2;
 	u64 wqe_sge_ba;
 	u32 page_size;
-	u8 port_num;
-	u64 *mtts_3;
-	u64 *mtts_2;
 	int count;
-	u8 *dmac;
-	u8 *smac;
-	int port;
 
 	/* Search qp buf's mtts */
-	page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
+	page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift;
 	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
 				  hr_qp->rq.offset / page_size, mtts,
 				  MTT_MIN_COUNT, &wqe_sge_ba);
@@ -3765,29 +3790,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 		if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size))
 			return -EINVAL;
 
-	/* Search IRRL's mtts */
-	mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
-				     hr_qp->qpn, &dma_handle_2);
-	if (!mtts_2) {
-		ibdev_err(ibdev, "failed to find QP irrl_table\n");
-		return -EINVAL;
-	}
-
-	/* Search TRRL's mtts */
-	mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
-				     hr_qp->qpn, &dma_handle_3);
-	if (!mtts_3) {
-		ibdev_err(ibdev, "failed to find QP trrl_table\n");
-		return -EINVAL;
-	}
-
-	if (attr_mask & IB_QP_ALT_PATH) {
-		ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error\n",
-			  attr_mask);
-		return -EINVAL;
-	}
-
-	dmac = (u8 *)attr->ah_attr.roce.dmac;
 	context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
 	qpc_mask->wqe_sge_ba = 0;
 
@@ -3804,17 +3806,16 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 
 	roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
 		       V2_QPC_BYTE_12_SQ_HOP_NUM_S,
-		       hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ?
-		       0 : hr_dev->caps.wqe_sq_hop_num);
+		       to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,
+					hr_qp->sq.wqe_cnt));
 	roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
 		       V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
 
 	roce_set_field(context->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_S,
-		       ((ibqp->qp_type == IB_QPT_GSI) ||
-		       hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
-		       hr_dev->caps.wqe_sge_hop_num : 0);
+		       to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,
+					hr_qp->sge.sge_cnt));
 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
@@ -3822,8 +3823,9 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(context->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_RQ_HOP_NUM_M,
 		       V2_QPC_BYTE_20_RQ_HOP_NUM_S,
-		       hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ?
-		       0 : hr_dev->caps.wqe_rq_hop_num);
+		       to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,
+					hr_qp->rq.wqe_cnt));
+
 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_RQ_HOP_NUM_M,
 		       V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
@@ -3831,7 +3833,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(context->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
-		       hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));
 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
@@ -3839,50 +3841,181 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(context->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
-		       hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));
 	roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
 
-	context->rq_cur_blk_addr = cpu_to_le32(mtts[0] >> PAGE_ADDR_SHIFT);
+	context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
 	qpc_mask->rq_cur_blk_addr = 0;
 
 	roce_set_field(context->byte_92_srq_info,
 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
-		       mtts[0] >> (32 + PAGE_ADDR_SHIFT));
+		       upper_32_bits(to_hr_hw_page_addr(mtts[0])));
 	roce_set_field(qpc_mask->byte_92_srq_info,
 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
 		       V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
 
-	context->rq_nxt_blk_addr = cpu_to_le32(mtts[1] >> PAGE_ADDR_SHIFT);
+	context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
 	qpc_mask->rq_nxt_blk_addr = 0;
 
 	roce_set_field(context->byte_104_rq_sge,
 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
-		       mtts[1] >> (32 + PAGE_ADDR_SHIFT));
+		       upper_32_bits(to_hr_hw_page_addr(mtts[1])));
 	roce_set_field(qpc_mask->byte_104_rq_sge,
 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
 
+	roce_set_field(context->byte_84_rq_ci_pi,
+		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
+		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
+	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
+		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
+		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
+
+	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
+		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
+		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
+
+	return 0;
+}
+
+static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
+			    struct hns_roce_qp *hr_qp,
+			    struct hns_roce_v2_qp_context *context,
+			    struct hns_roce_v2_qp_context *qpc_mask)
+{
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	u64 sge_cur_blk = 0;
+	u64 sq_cur_blk = 0;
+	u32 page_size;
+	int count;
+
+	/* search qp buf's mtts */
+	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
+	if (count < 1) {
+		ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n",
+			  hr_qp->qpn);
+		return -EINVAL;
+	}
+	if (hr_qp->sge.sge_cnt > 0) {
+		page_size = 1 << hr_qp->mtr.hem_cfg.buf_pg_shift;
+		count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
+					  hr_qp->sge.offset / page_size,
+					  &sge_cur_blk, 1, NULL);
+		if (count < 1) {
+			ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n",
+				  hr_qp->qpn);
+			return -EINVAL;
+		}
+	}
+
+	/*
+	 * In v2 engine, software pass context and context mask to hardware
+	 * when modifying qp. If software need modify some fields in context,
+	 * we should set all bits of the relevant fields in context mask to
+	 * 0 at the same time, else set them to 0x1.
+	 */
+	context->sq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
+	roce_set_field(context->byte_168_irrl_idx,
+		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
+		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
+		       upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
+	qpc_mask->sq_cur_blk_addr = 0;
+	roce_set_field(qpc_mask->byte_168_irrl_idx,
+		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
+		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
+
+	context->sq_cur_sge_blk_addr =
+		cpu_to_le32(to_hr_hw_page_addr(sge_cur_blk));
+	roce_set_field(context->byte_184_irrl_idx,
+		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
+		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
+		       upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
+	qpc_mask->sq_cur_sge_blk_addr = 0;
+	roce_set_field(qpc_mask->byte_184_irrl_idx,
+		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
+		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
+
+	context->rx_sq_cur_blk_addr =
+		cpu_to_le32(to_hr_hw_page_addr(sq_cur_blk));
+	roce_set_field(context->byte_232_irrl_sge,
+		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
+		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
+		       upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
+	qpc_mask->rx_sq_cur_blk_addr = 0;
+	roce_set_field(qpc_mask->byte_232_irrl_sge,
+		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
+		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
+
+	return 0;
+}
+
+static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
+				 const struct ib_qp_attr *attr, int attr_mask,
+				 struct hns_roce_v2_qp_context *context,
+				 struct hns_roce_v2_qp_context *qpc_mask)
+{
+	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	dma_addr_t trrl_ba;
+	dma_addr_t irrl_ba;
+	u8 port_num;
+	u64 *mtts;
+	u8 *dmac;
+	u8 *smac;
+	int port;
+	int ret;
+
+	ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);
+	if (ret) {
+		ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);
+		return ret;
+	}
+
+	/* Search IRRL's mtts */
+	mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
+				   hr_qp->qpn, &irrl_ba);
+	if (!mtts) {
+		ibdev_err(ibdev, "failed to find qp irrl_table.\n");
+		return -EINVAL;
+	}
+
+	/* Search TRRL's mtts */
+	mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
+				   hr_qp->qpn, &trrl_ba);
+	if (!mtts) {
+		ibdev_err(ibdev, "failed to find qp trrl_table.\n");
+		return -EINVAL;
+	}
+
+	if (attr_mask & IB_QP_ALT_PATH) {
+		ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",
+			  attr_mask);
+		return -EINVAL;
+	}
+
 	roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
-		       V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
+		       V2_QPC_BYTE_132_TRRL_BA_S, trrl_ba >> 4);
 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
 		       V2_QPC_BYTE_132_TRRL_BA_S, 0);
-	context->trrl_ba = cpu_to_le32(dma_handle_3 >> (16 + 4));
+	context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4));
 	qpc_mask->trrl_ba = 0;
 	roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
 		       V2_QPC_BYTE_140_TRRL_BA_S,
-		       (u32)(dma_handle_3 >> (32 + 16 + 4)));
+		       (u32)(trrl_ba >> (32 + 16 + 4)));
 	roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
 		       V2_QPC_BYTE_140_TRRL_BA_S, 0);
 
-	context->irrl_ba = cpu_to_le32(dma_handle_2 >> 6);
+	context->irrl_ba = cpu_to_le32(irrl_ba >> 6);
 	qpc_mask->irrl_ba = 0;
 	roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
 		       V2_QPC_BYTE_208_IRRL_BA_S,
-		       dma_handle_2 >> (32 + 6));
+		       irrl_ba >> (32 + 6));
 	roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
 		       V2_QPC_BYTE_208_IRRL_BA_S, 0);
 
@@ -3897,6 +4030,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
 
 	smac = (u8 *)hr_dev->dev_addr[port];
+	dmac = (u8 *)attr->ah_attr.roce.dmac;
 	/* when dmac equals smac or loop_idc is 1, it should loopback */
 	if (ether_addr_equal_unaligned(dmac, smac) ||
 	    hr_dev->loop_idc == 0x1) {
@@ -3919,6 +4053,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 					 grh->sgid_index));
 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0);
+
 	memcpy(&(context->dmac), dmac, sizeof(u32));
 	roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
 		       V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
@@ -3928,7 +4063,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 
 	/* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */
 	roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
-		       V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
+		       V2_QPC_BYTE_56_LP_PKTN_INI_S,
+		       ilog2(hr_dev->caps.max_sq_inline / IB_MTU_4096));
 	roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
 		       V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
 
@@ -3942,16 +4078,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
 		       V2_QPC_BYTE_24_MTU_S, 0);
 
-	roce_set_field(context->byte_84_rq_ci_pi,
-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
-	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
-		       V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
-
-	roce_set_field(qpc_mask->byte_84_rq_ci_pi,
-		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
-		       V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
 	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
 		     V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
 	roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
@@ -3987,30 +4113,7 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 	struct ib_device *ibdev = &hr_dev->ib_dev;
-	u64 sge_cur_blk = 0;
-	u64 sq_cur_blk = 0;
-	u32 page_size;
-	int count;
-
-	/* Search qp buf's mtts */
-	count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL);
-	if (count < 1) {
-		ibdev_err(ibdev, "failed to find buf pa of QP(0x%lx)\n",
-			  hr_qp->qpn);
-		return -EINVAL;
-	}
-
-	if (hr_qp->sge.offset) {
-		page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
-		count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
-					  hr_qp->sge.offset / page_size,
-					  &sge_cur_blk, 1, NULL);
-		if (count < 1) {
-			ibdev_err(ibdev, "failed to find sge pa of QP(0x%lx)\n",
-				  hr_qp->qpn);
-			return -EINVAL;
-		}
-	}
+	int ret;
 
 	/* Not support alternate path and path migration */
 	if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) {
@@ -4018,48 +4121,11 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 		return -EINVAL;
 	}
 
-	/*
-	 * In v2 engine, software pass context and context mask to hardware
-	 * when modifying qp. If software need modify some fields in context,
-	 * we should set all bits of the relevant fields in context mask to
-	 * 0 at the same time, else set them to 0x1.
-	 */
-	context->sq_cur_blk_addr = cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
-	roce_set_field(context->byte_168_irrl_idx,
-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
-		       sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
-	qpc_mask->sq_cur_blk_addr = 0;
-	roce_set_field(qpc_mask->byte_168_irrl_idx,
-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
-		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
-
-	context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) ||
-		       hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
-		       cpu_to_le32(sge_cur_blk >>
-		       PAGE_ADDR_SHIFT) : 0;
-	roce_set_field(context->byte_184_irrl_idx,
-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
-		       ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs >
-		       HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ?
-		       (sge_cur_blk >>
-		       (32 + PAGE_ADDR_SHIFT)) : 0);
-	qpc_mask->sq_cur_sge_blk_addr = 0;
-	roce_set_field(qpc_mask->byte_184_irrl_idx,
-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
-		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
-
-	context->rx_sq_cur_blk_addr =
-		cpu_to_le32(sq_cur_blk >> PAGE_ADDR_SHIFT);
-	roce_set_field(context->byte_232_irrl_sge,
-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
-		       sq_cur_blk >> (32 + PAGE_ADDR_SHIFT));
-	qpc_mask->rx_sq_cur_blk_addr = 0;
-	roce_set_field(qpc_mask->byte_232_irrl_sge,
-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
-		       V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
+	ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
+	if (ret) {
+		ibdev_err(ibdev, "failed to config sq buf, ret %d\n", ret);
+		return ret;
+	}
 
 	/*
 	 * Set some fields in context to zero, Because the default values
@@ -4108,21 +4174,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 	return 0;
 }
 
-static inline bool hns_roce_v2_check_qp_stat(enum ib_qp_state cur_state,
-					     enum ib_qp_state new_state)
-{
-
-	if ((cur_state != IB_QPS_RESET &&
-	    (new_state == IB_QPS_ERR || new_state == IB_QPS_RESET)) ||
-	    ((cur_state == IB_QPS_RTS || cur_state == IB_QPS_SQD) &&
-	    (new_state == IB_QPS_RTS || new_state == IB_QPS_SQD)) ||
-	    (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS))
-		return true;
-
-	return false;
-
-}
-
 static int hns_roce_v2_set_path(struct ib_qp *ibqp,
 				const struct ib_qp_attr *attr,
 				int attr_mask,
@@ -4226,6 +4277,28 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp,
 	return 0;
 }
 
+static bool check_qp_state(enum ib_qp_state cur_state,
+			   enum ib_qp_state new_state)
+{
+	static const bool sm[][IB_QPS_ERR + 1] = {
+		[IB_QPS_RESET] = { [IB_QPS_RESET] = true,
+				   [IB_QPS_INIT] = true },
+		[IB_QPS_INIT] = { [IB_QPS_RESET] = true,
+				  [IB_QPS_INIT] = true,
+				  [IB_QPS_RTR] = true,
+				  [IB_QPS_ERR] = true },
+		[IB_QPS_RTR] = { [IB_QPS_RESET] = true,
+				 [IB_QPS_RTS] = true,
+				 [IB_QPS_ERR] = true },
+		[IB_QPS_RTS] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true },
+		[IB_QPS_SQD] = {},
+		[IB_QPS_SQE] = {},
+		[IB_QPS_ERR] = { [IB_QPS_RESET] = true, [IB_QPS_ERR] = true }
+	};
+
+	return sm[cur_state][new_state];
+}
+
 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
 				      const struct ib_qp_attr *attr,
 				      int attr_mask,
@@ -4237,6 +4310,11 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	int ret = 0;
 
+	if (!check_qp_state(cur_state, new_state)) {
+		ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");
+		return -EINVAL;
+	}
+
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		memset(qpc_mask, 0, sizeof(*qpc_mask));
 		modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
@@ -4247,23 +4325,11 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
 	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
 		ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
 					    qpc_mask);
-		if (ret)
-			goto out;
 	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
 		ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
 					   qpc_mask);
-		if (ret)
-			goto out;
-	} else if (hns_roce_v2_check_qp_stat(cur_state, new_state)) {
-		/* Nothing */
-		;
-	} else {
-		ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");
-		ret = -EINVAL;
-		goto out;
 	}
 
-out:
 	return ret;
 }
 
@@ -4554,19 +4620,20 @@ out:
 	return ret;
 }
 
-static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
+static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
 {
-	switch (state) {
-	case HNS_ROCE_QP_ST_RST:	return IB_QPS_RESET;
-	case HNS_ROCE_QP_ST_INIT:	return IB_QPS_INIT;
-	case HNS_ROCE_QP_ST_RTR:	return IB_QPS_RTR;
-	case HNS_ROCE_QP_ST_RTS:	return IB_QPS_RTS;
-	case HNS_ROCE_QP_ST_SQ_DRAINING:
-	case HNS_ROCE_QP_ST_SQD:	return IB_QPS_SQD;
-	case HNS_ROCE_QP_ST_SQER:	return IB_QPS_SQE;
-	case HNS_ROCE_QP_ST_ERR:	return IB_QPS_ERR;
-	default:			return -1;
-	}
+	static const enum ib_qp_state map[] = {
+		[HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,
+		[HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,
+		[HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,
+		[HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,
+		[HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,
+		[HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,
+		[HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,
+		[HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD
+	};
+
+	return (state < ARRAY_SIZE(map)) ? map[state] : -1;
 }
 
 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
@@ -4639,7 +4706,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 	qp_attr->path_mig_state = IB_MIG_ARMED;
 	qp_attr->ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;
 	if (hr_qp->ibqp.qp_type == IB_QPT_UD)
-		qp_attr->qkey = V2_QKEY_VAL;
+		qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);
 
 	qp_attr->rq_psn = roce_get_field(context.byte_108_rx_reqepsn,
 					 V2_QPC_BYTE_108_RX_REQ_EPSN_M,
@@ -4838,6 +4905,184 @@ out:
 	return ret;
 }
 
+static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
+				   struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
+				   u32 cqn, void *mb_buf, u64 *mtts_wqe,
+				   u64 *mtts_idx, dma_addr_t dma_handle_wqe,
+				   dma_addr_t dma_handle_idx)
+{
+	struct hns_roce_srq_context *srq_context;
+
+	srq_context = mb_buf;
+	memset(srq_context, 0, sizeof(*srq_context));
+
+	roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
+		       SRQC_BYTE_4_SRQ_ST_S, 1);
+
+	roce_set_field(srq_context->byte_4_srqn_srqst,
+		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
+		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
+		       to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,
+					srq->wqe_cnt));
+	roce_set_field(srq_context->byte_4_srqn_srqst,
+		       SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
+		       ilog2(srq->wqe_cnt));
+
+	roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
+		       SRQC_BYTE_4_SRQN_S, srq->srqn);
+
+	roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
+		       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
+
+	roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
+		       SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
+
+	srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
+
+	roce_set_field(srq_context->byte_24_wqe_bt_ba,
+		       SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
+		       SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
+		       dma_handle_wqe >> 35);
+
+	roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
+		       SRQC_BYTE_28_PD_S, pdn);
+	roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
+		       SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
+		       fls(srq->max_gs - 1));
+
+	srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3);
+	roce_set_field(srq_context->rsv_idx_bt_ba,
+		       SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
+		       SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
+		       dma_handle_idx >> 35);
+
+	srq_context->idx_cur_blk_addr =
+		cpu_to_le32(to_hr_hw_page_addr(mtts_idx[0]));
+	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
+		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
+		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
+		       upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));
+	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
+		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
+		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
+		       to_hr_hem_hopnum(hr_dev->caps.idx_hop_num,
+					srq->wqe_cnt));
+
+	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
+		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
+		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
+		to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.ba_pg_shift));
+	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
+		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
+		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
+		to_hr_hw_page_shift(srq->idx_que.mtr.hem_cfg.buf_pg_shift));
+
+	srq_context->idx_nxt_blk_addr =
+				cpu_to_le32(to_hr_hw_page_addr(mtts_idx[1]));
+	roce_set_field(srq_context->rsv_idxnxtblkaddr,
+		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
+		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
+		       upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));
+	roce_set_field(srq_context->byte_56_xrc_cqn,
+		       SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
+		       cqn);
+	roce_set_field(srq_context->byte_56_xrc_cqn,
+		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
+		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
+		       to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));
+	roce_set_field(srq_context->byte_56_xrc_cqn,
+		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
+		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
+		       to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));
+
+	roce_set_bit(srq_context->db_record_addr_record_en,
+		     SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
+}
+
+static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
+				  struct ib_srq_attr *srq_attr,
+				  enum ib_srq_attr_mask srq_attr_mask,
+				  struct ib_udata *udata)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
+	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
+	struct hns_roce_srq_context *srq_context;
+	struct hns_roce_srq_context *srqc_mask;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	if (srq_attr_mask & IB_SRQ_LIMIT) {
+		if (srq_attr->srq_limit >= srq->wqe_cnt)
+			return -EINVAL;
+
+		mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+		if (IS_ERR(mailbox))
+			return PTR_ERR(mailbox);
+
+		srq_context = mailbox->buf;
+		srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
+
+		memset(srqc_mask, 0xff, sizeof(*srqc_mask));
+
+		roce_set_field(srq_context->byte_8_limit_wl,
+			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,
+			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
+		roce_set_field(srqc_mask->byte_8_limit_wl,
+			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,
+			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
+
+		ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
+					HNS_ROCE_CMD_MODIFY_SRQC,
+					HNS_ROCE_CMD_TIMEOUT_MSECS);
+		hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+		if (ret) {
+			ibdev_err(&hr_dev->ib_dev,
+				  "failed to handle cmd of modifying SRQ, ret = %d.\n",
+				  ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
+	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
+	struct hns_roce_srq_context *srq_context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int limit_wl;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	srq_context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
+				HNS_ROCE_CMD_QUERY_SRQC,
+				HNS_ROCE_CMD_TIMEOUT_MSECS);
+	if (ret) {
+		ibdev_err(&hr_dev->ib_dev,
+			  "failed to process cmd of querying SRQ, ret = %d.\n",
+			  ret);
+		goto out;
+	}
+
+	limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
+				  SRQC_BYTE_8_SRQ_LIMIT_WL_M,
+				  SRQC_BYTE_8_SRQ_LIMIT_WL_S);
+
+	attr->srq_limit = limit_wl;
+	attr->max_wr = srq->wqe_cnt - 1;
+	attr->max_sge = srq->max_gs - HNS_ROCE_RESERVED_SGE;
+
+out:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+	return ret;
+}
+
 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
@@ -4989,24 +5234,14 @@ static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
 	hns_roce_write64(hr_dev, doorbell, eq->doorbell);
 }
 
-static inline void *get_eqe_buf(struct hns_roce_eq *eq, unsigned long offset)
-{
-	u32 buf_chk_sz;
-
-	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
-	if (eq->buf.nbufs == 1)
-		return eq->buf.direct.buf + offset % buf_chk_sz;
-	else
-		return eq->buf.page_list[offset / buf_chk_sz].buf +
-		       offset % buf_chk_sz;
-}
-
 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
 {
 	struct hns_roce_aeqe *aeqe;
 
-	aeqe = get_eqe_buf(eq, (eq->cons_index & (eq->entries - 1)) *
-			   HNS_ROCE_AEQ_ENTRY_SIZE);
+	aeqe = hns_roce_buf_offset(eq->mtr.kmem,
+				   (eq->cons_index & (eq->entries - 1)) *
+				   HNS_ROCE_AEQ_ENTRY_SIZE);
+
 	return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
 		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
 }
@@ -5103,8 +5338,9 @@ static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
 {
 	struct hns_roce_ceqe *ceqe;
 
-	ceqe = get_eqe_buf(eq, (eq->cons_index & (eq->entries - 1)) *
-			   HNS_ROCE_CEQ_ENTRY_SIZE);
+	ceqe = hns_roce_buf_offset(eq->mtr.kmem,
+				   (eq->cons_index & (eq->entries - 1)) *
+				   HNS_ROCE_CEQ_ENTRY_SIZE);
 	return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
 		(!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
 }
@@ -5263,17 +5499,15 @@ static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
 
 static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 {
-	if (!eq->hop_num || eq->hop_num == HNS_ROCE_HOP_NUM_0)
-		hns_roce_mtr_cleanup(hr_dev, &eq->mtr);
-	hns_roce_buf_free(hr_dev, eq->buf.size, &eq->buf);
+	hns_roce_mtr_destroy(hr_dev, &eq->mtr);
 }
 
-static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
-				struct hns_roce_eq *eq,
-				void *mb_buf)
+static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
+		      void *mb_buf)
 {
+	u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
 	struct hns_roce_eq_context *eqc;
-	u64 ba[MTT_MIN_COUNT] = { 0 };
+	u64 bt_ba = 0;
 	int count;
 
 	eqc = mb_buf;
@@ -5281,31 +5515,18 @@ static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
 
 	/* init eqc */
 	eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
-	eq->hop_num = hr_dev->caps.eqe_hop_num;
 	eq->cons_index = 0;
 	eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
 	eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
 	eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
-	eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
-	eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
 	eq->shift = ilog2((unsigned int)eq->entries);
 
-	/* if not muti-hop, eqe buffer only use one trunk */
-	if (!eq->hop_num || eq->hop_num == HNS_ROCE_HOP_NUM_0) {
-		eq->eqe_ba = eq->buf.direct.map;
-		eq->cur_eqe_ba = eq->eqe_ba;
-		if (eq->buf.npages > 1)
-			eq->nxt_eqe_ba = eq->eqe_ba + (1 << eq->eqe_buf_pg_sz);
-		else
-			eq->nxt_eqe_ba = eq->eqe_ba;
-	} else {
-		count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, ba,
-					  MTT_MIN_COUNT, &eq->eqe_ba);
-		eq->cur_eqe_ba = ba[0];
-		if (count > 1)
-			eq->nxt_eqe_ba = ba[1];
-		else
-			eq->nxt_eqe_ba = ba[0];
+	/* if not multi-hop, eqe buffer only use one trunk */
+	count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT,
+				  &bt_ba);
+	if (count < 1) {
+		dev_err(hr_dev->dev, "failed to find EQE mtr\n");
+		return -ENOBUFS;
 	}
 
 	/* set eqc state */
@@ -5339,12 +5560,12 @@ static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
 	/* set eqe_ba_pg_sz */
 	roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BA_PG_SZ_M,
 		       HNS_ROCE_EQC_BA_PG_SZ_S,
-		       eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));
 
 	/* set eqe_buf_pg_sz */
 	roce_set_field(eqc->byte_8, HNS_ROCE_EQC_BUF_PG_SZ_M,
 		       HNS_ROCE_EQC_BUF_PG_SZ_S,
-		       eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
+		       to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));
 
 	/* set eq_producer_idx */
 	roce_set_field(eqc->byte_8, HNS_ROCE_EQC_PROD_INDX_M,
@@ -5363,13 +5584,13 @@ static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
 		       HNS_ROCE_EQC_REPORT_TIMER_S,
 		       HNS_ROCE_EQ_INIT_REPORT_TIMER);
 
-	/* set eqe_ba [34:3] */
+	/* set bt_ba [34:3] */
 	roce_set_field(eqc->eqe_ba0, HNS_ROCE_EQC_EQE_BA_L_M,
-		       HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
+		       HNS_ROCE_EQC_EQE_BA_L_S, bt_ba >> 3);
 
-	/* set eqe_ba [64:35] */
+	/* set bt_ba [64:35] */
 	roce_set_field(eqc->eqe_ba1, HNS_ROCE_EQC_EQE_BA_H_M,
-		       HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
+		       HNS_ROCE_EQC_EQE_BA_H_S, bt_ba >> 35);
 
 	/* set eq shift */
 	roce_set_field(eqc->byte_28, HNS_ROCE_EQC_SHIFT_M, HNS_ROCE_EQC_SHIFT_S,
@@ -5381,15 +5602,15 @@ static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
 
 	/* set cur_eqe_ba [27:12] */
 	roce_set_field(eqc->byte_28, HNS_ROCE_EQC_CUR_EQE_BA_L_M,
-		       HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
+		       HNS_ROCE_EQC_CUR_EQE_BA_L_S, eqe_ba[0] >> 12);
 
 	/* set cur_eqe_ba [59:28] */
 	roce_set_field(eqc->byte_32, HNS_ROCE_EQC_CUR_EQE_BA_M_M,
-		       HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
+		       HNS_ROCE_EQC_CUR_EQE_BA_M_S, eqe_ba[0] >> 28);
 
 	/* set cur_eqe_ba [63:60] */
 	roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CUR_EQE_BA_H_M,
-		       HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
+		       HNS_ROCE_EQC_CUR_EQE_BA_H_S, eqe_ba[0] >> 60);
 
 	/* set eq consumer idx */
 	roce_set_field(eqc->byte_36, HNS_ROCE_EQC_CONS_INDX_M,
@@ -5397,97 +5618,38 @@ static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
 
 	/* set nex_eqe_ba[43:12] */
 	roce_set_field(eqc->nxt_eqe_ba0, HNS_ROCE_EQC_NXT_EQE_BA_L_M,
-		       HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
+		       HNS_ROCE_EQC_NXT_EQE_BA_L_S, eqe_ba[1] >> 12);
 
 	/* set nex_eqe_ba[63:44] */
 	roce_set_field(eqc->nxt_eqe_ba1, HNS_ROCE_EQC_NXT_EQE_BA_H_M,
-		       HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
-}
+		       HNS_ROCE_EQC_NXT_EQE_BA_H_S, eqe_ba[1] >> 44);
 
-static int map_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
-		      u32 page_shift)
-{
-	struct hns_roce_buf_region region = {};
-	dma_addr_t *buf_list = NULL;
-	int ba_num;
-	int ret;
-
-	ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size),
-			      1 << page_shift);
-	hns_roce_init_buf_region(&region, hr_dev->caps.eqe_hop_num, 0, ba_num);
-
-	/* alloc a tmp list for storing eq buf address */
-	ret = hns_roce_alloc_buf_list(&region, &buf_list, 1);
-	if (ret) {
-		dev_err(hr_dev->dev, "alloc eq buf_list error\n");
-		return ret;
-	}
-
-	ba_num = hns_roce_get_kmem_bufs(hr_dev, buf_list, region.count,
-					region.offset, &eq->buf);
-	if (ba_num != region.count) {
-		dev_err(hr_dev->dev, "get eqe buf err,expect %d,ret %d.\n",
-			region.count, ba_num);
-		ret = -ENOBUFS;
-		goto done;
-	}
-
-	hns_roce_mtr_init(&eq->mtr, PAGE_SHIFT + hr_dev->caps.eqe_ba_pg_sz,
-			  page_shift);
-	ret = hns_roce_mtr_attach(hr_dev, &eq->mtr, &buf_list, &region, 1);
-	if (ret)
-		dev_err(hr_dev->dev, "mtr attach error for eqe\n");
-
-	goto done;
-
-	hns_roce_mtr_cleanup(hr_dev, &eq->mtr);
-done:
-	hns_roce_free_buf_list(&buf_list, 1);
-
-	return ret;
+	return 0;
 }
 
 static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 {
-	struct hns_roce_buf *buf = &eq->buf;
-	bool is_mhop = false;
-	u32 page_shift;
-	u32 mhop_num;
-	u32 max_size;
-	int ret;
+	struct hns_roce_buf_attr buf_attr = {};
+	int err;
 
-	page_shift = PAGE_SHIFT + hr_dev->caps.eqe_buf_pg_sz;
-	mhop_num = hr_dev->caps.eqe_hop_num;
-	if (!mhop_num) {
-		max_size = 1 << page_shift;
-		buf->size = max_size;
-	} else if (mhop_num == HNS_ROCE_HOP_NUM_0) {
-		max_size = eq->entries * eq->eqe_size;
-		buf->size = max_size;
-	} else {
-		max_size = 1 << page_shift;
-		buf->size = PAGE_ALIGN(eq->entries * eq->eqe_size);
-		is_mhop = true;
-	}
+	if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)
+		eq->hop_num = 0;
+	else
+		eq->hop_num = hr_dev->caps.eqe_hop_num;
 
-	ret = hns_roce_buf_alloc(hr_dev, buf->size, max_size, buf, page_shift);
-	if (ret) {
-		dev_err(hr_dev->dev, "alloc eq buf error\n");
-		return ret;
-	}
+	buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + HNS_HW_PAGE_SHIFT;
+	buf_attr.region[0].size = eq->entries * eq->eqe_size;
+	buf_attr.region[0].hopnum = eq->hop_num;
+	buf_attr.region_count = 1;
+	buf_attr.fixed_page = true;
 
-	if (is_mhop) {
-		ret = map_eq_buf(hr_dev, eq, page_shift);
-		if (ret) {
-			dev_err(hr_dev->dev, "map roce buf error\n");
-			goto err_alloc;
-		}
-	}
+	err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
+				  hr_dev->caps.eqe_ba_pg_sz +
+				  HNS_HW_PAGE_SHIFT, NULL, 0);
+	if (err)
+		dev_err(hr_dev->dev, "Failed to alloc EQE mtr, err %d\n", err);
 
-	return 0;
-err_alloc:
-	hns_roce_buf_free(hr_dev, buf->size, buf);
-	return ret;
+	return err;
 }
 
 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
@@ -5499,15 +5661,16 @@ static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
 
 	/* Allocate mailbox memory */
 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
-	if (IS_ERR(mailbox))
-		return PTR_ERR(mailbox);
+	if (IS_ERR_OR_NULL(mailbox))
+		return -ENOMEM;
 
 	ret = alloc_eq_buf(hr_dev, eq);
-	if (ret) {
-		ret = -ENOMEM;
+	if (ret)
 		goto free_cmd_mbox;
-	}
-	hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
+
+	ret = config_eqc(hr_dev, eq, mailbox->buf);
+	if (ret)
+		goto err_cmd_mbox;
 
 	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
 				eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
@@ -5731,294 +5894,6 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
 	destroy_workqueue(hr_dev->irq_workq);
 }
 
-static void hns_roce_v2_write_srqc(struct hns_roce_dev *hr_dev,
-				   struct hns_roce_srq *srq, u32 pdn, u16 xrcd,
-				   u32 cqn, void *mb_buf, u64 *mtts_wqe,
-				   u64 *mtts_idx, dma_addr_t dma_handle_wqe,
-				   dma_addr_t dma_handle_idx)
-{
-	struct hns_roce_srq_context *srq_context;
-
-	srq_context = mb_buf;
-	memset(srq_context, 0, sizeof(*srq_context));
-
-	roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQ_ST_M,
-		       SRQC_BYTE_4_SRQ_ST_S, 1);
-
-	roce_set_field(srq_context->byte_4_srqn_srqst,
-		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_M,
-		       SRQC_BYTE_4_SRQ_WQE_HOP_NUM_S,
-		       (hr_dev->caps.srqwqe_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
-		       hr_dev->caps.srqwqe_hop_num));
-	roce_set_field(srq_context->byte_4_srqn_srqst,
-		       SRQC_BYTE_4_SRQ_SHIFT_M, SRQC_BYTE_4_SRQ_SHIFT_S,
-		       ilog2(srq->wqe_cnt));
-
-	roce_set_field(srq_context->byte_4_srqn_srqst, SRQC_BYTE_4_SRQN_M,
-		       SRQC_BYTE_4_SRQN_S, srq->srqn);
-
-	roce_set_field(srq_context->byte_8_limit_wl, SRQC_BYTE_8_SRQ_LIMIT_WL_M,
-		       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
-
-	roce_set_field(srq_context->byte_12_xrcd, SRQC_BYTE_12_SRQ_XRCD_M,
-		       SRQC_BYTE_12_SRQ_XRCD_S, xrcd);
-
-	srq_context->wqe_bt_ba = cpu_to_le32((u32)(dma_handle_wqe >> 3));
-
-	roce_set_field(srq_context->byte_24_wqe_bt_ba,
-		       SRQC_BYTE_24_SRQ_WQE_BT_BA_M,
-		       SRQC_BYTE_24_SRQ_WQE_BT_BA_S,
-		       dma_handle_wqe >> 35);
-
-	roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_PD_M,
-		       SRQC_BYTE_28_PD_S, pdn);
-	roce_set_field(srq_context->byte_28_rqws_pd, SRQC_BYTE_28_RQWS_M,
-		       SRQC_BYTE_28_RQWS_S, srq->max_gs <= 0 ? 0 :
-		       fls(srq->max_gs - 1));
-
-	srq_context->idx_bt_ba = cpu_to_le32(dma_handle_idx >> 3);
-	roce_set_field(srq_context->rsv_idx_bt_ba,
-		       SRQC_BYTE_36_SRQ_IDX_BT_BA_M,
-		       SRQC_BYTE_36_SRQ_IDX_BT_BA_S,
-		       dma_handle_idx >> 35);
-
-	srq_context->idx_cur_blk_addr =
-		cpu_to_le32(mtts_idx[0] >> PAGE_ADDR_SHIFT);
-	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
-		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_M,
-		       SRQC_BYTE_44_SRQ_IDX_CUR_BLK_ADDR_S,
-		       mtts_idx[0] >> (32 + PAGE_ADDR_SHIFT));
-	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
-		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_M,
-		       SRQC_BYTE_44_SRQ_IDX_HOP_NUM_S,
-		       hr_dev->caps.idx_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
-		       hr_dev->caps.idx_hop_num);
-
-	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
-		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_M,
-		       SRQC_BYTE_44_SRQ_IDX_BA_PG_SZ_S,
-		       hr_dev->caps.idx_ba_pg_sz + PG_SHIFT_OFFSET);
-	roce_set_field(srq_context->byte_44_idxbufpgsz_addr,
-		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_M,
-		       SRQC_BYTE_44_SRQ_IDX_BUF_PG_SZ_S,
-		       hr_dev->caps.idx_buf_pg_sz + PG_SHIFT_OFFSET);
-
-	srq_context->idx_nxt_blk_addr =
-		cpu_to_le32(mtts_idx[1] >> PAGE_ADDR_SHIFT);
-	roce_set_field(srq_context->rsv_idxnxtblkaddr,
-		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_M,
-		       SRQC_BYTE_52_SRQ_IDX_NXT_BLK_ADDR_S,
-		       mtts_idx[1] >> (32 + PAGE_ADDR_SHIFT));
-	roce_set_field(srq_context->byte_56_xrc_cqn,
-		       SRQC_BYTE_56_SRQ_XRC_CQN_M, SRQC_BYTE_56_SRQ_XRC_CQN_S,
-		       cqn);
-	roce_set_field(srq_context->byte_56_xrc_cqn,
-		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_M,
-		       SRQC_BYTE_56_SRQ_WQE_BA_PG_SZ_S,
-		       hr_dev->caps.srqwqe_ba_pg_sz + PG_SHIFT_OFFSET);
-	roce_set_field(srq_context->byte_56_xrc_cqn,
-		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_M,
-		       SRQC_BYTE_56_SRQ_WQE_BUF_PG_SZ_S,
-		       hr_dev->caps.srqwqe_buf_pg_sz + PG_SHIFT_OFFSET);
-
-	roce_set_bit(srq_context->db_record_addr_record_en,
-		     SRQC_BYTE_60_SRQ_RECORD_EN_S, 0);
-}
-
-static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
-				  struct ib_srq_attr *srq_attr,
-				  enum ib_srq_attr_mask srq_attr_mask,
-				  struct ib_udata *udata)
-{
-	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
-	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
-	struct hns_roce_srq_context *srq_context;
-	struct hns_roce_srq_context *srqc_mask;
-	struct hns_roce_cmd_mailbox *mailbox;
-	int ret;
-
-	if (srq_attr_mask & IB_SRQ_LIMIT) {
-		if (srq_attr->srq_limit >= srq->wqe_cnt)
-			return -EINVAL;
-
-		mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
-		if (IS_ERR(mailbox))
-			return PTR_ERR(mailbox);
-
-		srq_context = mailbox->buf;
-		srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
-
-		memset(srqc_mask, 0xff, sizeof(*srqc_mask));
-
-		roce_set_field(srq_context->byte_8_limit_wl,
-			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,
-			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, srq_attr->srq_limit);
-		roce_set_field(srqc_mask->byte_8_limit_wl,
-			       SRQC_BYTE_8_SRQ_LIMIT_WL_M,
-			       SRQC_BYTE_8_SRQ_LIMIT_WL_S, 0);
-
-		ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, srq->srqn, 0,
-					HNS_ROCE_CMD_MODIFY_SRQC,
-					HNS_ROCE_CMD_TIMEOUT_MSECS);
-		hns_roce_free_cmd_mailbox(hr_dev, mailbox);
-		if (ret) {
-			ibdev_err(&hr_dev->ib_dev,
-				  "failed to process cmd when modifying SRQ, ret = %d\n",
-				  ret);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
-{
-	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
-	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
-	struct hns_roce_srq_context *srq_context;
-	struct hns_roce_cmd_mailbox *mailbox;
-	int limit_wl;
-	int ret;
-
-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
-	if (IS_ERR(mailbox))
-		return PTR_ERR(mailbox);
-
-	srq_context = mailbox->buf;
-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, srq->srqn, 0,
-				HNS_ROCE_CMD_QUERY_SRQC,
-				HNS_ROCE_CMD_TIMEOUT_MSECS);
-	if (ret) {
-		ibdev_err(&hr_dev->ib_dev,
-			  "failed to process cmd when querying SRQ, ret = %d\n",
-			  ret);
-		goto out;
-	}
-
-	limit_wl = roce_get_field(srq_context->byte_8_limit_wl,
-				  SRQC_BYTE_8_SRQ_LIMIT_WL_M,
-				  SRQC_BYTE_8_SRQ_LIMIT_WL_S);
-
-	attr->srq_limit = limit_wl;
-	attr->max_wr    = srq->wqe_cnt - 1;
-	attr->max_sge   = srq->max_gs;
-
-	memcpy(srq_context, mailbox->buf, sizeof(*srq_context));
-
-out:
-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
-	return ret;
-}
-
-static int find_empty_entry(struct hns_roce_idx_que *idx_que,
-			    unsigned long size)
-{
-	int wqe_idx;
-
-	if (unlikely(bitmap_full(idx_que->bitmap, size)))
-		return -ENOSPC;
-
-	wqe_idx = find_first_zero_bit(idx_que->bitmap, size);
-
-	bitmap_set(idx_que->bitmap, wqe_idx, 1);
-
-	return wqe_idx;
-}
-
-static void fill_idx_queue(struct hns_roce_idx_que *idx_que,
-			   int cur_idx, int wqe_idx)
-{
-	unsigned int *addr;
-
-	addr = (unsigned int *)hns_roce_buf_offset(&idx_que->idx_buf,
-						   cur_idx * idx_que->entry_sz);
-	*addr = wqe_idx;
-}
-
-static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
-				     const struct ib_recv_wr *wr,
-				     const struct ib_recv_wr **bad_wr)
-{
-	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
-	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
-	struct hns_roce_v2_wqe_data_seg *dseg;
-	struct hns_roce_v2_db srq_db;
-	unsigned long flags;
-	int ret = 0;
-	int wqe_idx;
-	void *wqe;
-	int nreq;
-	int ind;
-	int i;
-
-	spin_lock_irqsave(&srq->lock, flags);
-
-	ind = srq->head & (srq->wqe_cnt - 1);
-
-	for (nreq = 0; wr; ++nreq, wr = wr->next) {
-		if (unlikely(wr->num_sge > srq->max_gs)) {
-			ret = -EINVAL;
-			*bad_wr = wr;
-			break;
-		}
-
-		if (unlikely(srq->head == srq->tail)) {
-			ret = -ENOMEM;
-			*bad_wr = wr;
-			break;
-		}
-
-		wqe_idx = find_empty_entry(&srq->idx_que, srq->wqe_cnt);
-		if (wqe_idx < 0) {
-			ret = -ENOMEM;
-			*bad_wr = wr;
-			break;
-		}
-
-		fill_idx_queue(&srq->idx_que, ind, wqe_idx);
-		wqe = get_srq_wqe(srq, wqe_idx);
-		dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
-
-		for (i = 0; i < wr->num_sge; ++i) {
-			dseg[i].len = cpu_to_le32(wr->sg_list[i].length);
-			dseg[i].lkey = cpu_to_le32(wr->sg_list[i].lkey);
-			dseg[i].addr = cpu_to_le64(wr->sg_list[i].addr);
-		}
-
-		if (i < srq->max_gs) {
-			dseg[i].len = 0;
-			dseg[i].lkey = cpu_to_le32(0x100);
-			dseg[i].addr = 0;
-		}
-
-		srq->wrid[wqe_idx] = wr->wr_id;
-		ind = (ind + 1) & (srq->wqe_cnt - 1);
-	}
-
-	if (likely(nreq)) {
-		srq->head += nreq;
-
-		/*
-		 * Make sure that descriptors are written before
-		 * doorbell record.
-		 */
-		wmb();
-
-		srq_db.byte_4 =
-			cpu_to_le32(HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S |
-				    (srq->srqn & V2_DB_BYTE_4_TAG_M));
-		srq_db.parameter = cpu_to_le32(srq->head);
-
-		hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l);
-
-	}
-
-	spin_unlock_irqrestore(&srq->lock, flags);
-
-	return ret;
-}
-
 static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
 	.query_cqc_info = hns_roce_v2_query_cqc_info,
 };
@@ -6161,7 +6036,7 @@ error_failed_kzalloc:
 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
 					   bool reset)
 {
-	struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
+	struct hns_roce_dev *hr_dev = handle->priv;
 
 	if (!hr_dev)
 		return;
@@ -6241,7 +6116,7 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
 	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
 	clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
 
-	hr_dev = (struct hns_roce_dev *)handle->priv;
+	hr_dev = handle->priv;
 	if (!hr_dev)
 		return 0;
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index 82dd9f6f4845..e176b0aaa4ac 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -92,7 +92,9 @@
 #define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ		PAGE_SIZE
 #define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED		0xFFFFF000
 #define HNS_ROCE_V2_MAX_INNER_MTPT_NUM		2
-#define HNS_ROCE_INVALID_LKEY			0x100
+#define HNS_ROCE_INVALID_LKEY			0x0
+#define HNS_ROCE_INVALID_SGE_LENGTH		0x80000000
+
 #define HNS_ROCE_CMQ_TX_TIMEOUT			30000
 #define HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE	2
 #define HNS_ROCE_V2_RSV_QPS			8
@@ -1241,10 +1243,9 @@ struct hns_roce_func_clear {
 };
 
 #define FUNC_CLEAR_RST_FUN_DONE_S 0
-/* Each physical function manages up to 248 virtual functions；
- * it takes up to 100ms for each function to execute clear；
- * if an abnormal reset occurs, it is executed twice at most;
- * so it takes up to 249 * 2 * 100ms.
+/* Each physical function manages up to 248 virtual functions, it takes up to
+ * 100ms for each function to execute clear. If an abnormal reset occurs, it is
+ * executed twice at most, so it takes up to 249 * 2 * 100ms.
  */
 #define HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS	(249 * 2 * 100)
 #define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL	40
@@ -1648,7 +1649,7 @@ struct hns_roce_query_pf_caps_c {
 struct hns_roce_query_pf_caps_d {
 	__le32 wq_hop_num_max_srqs;
 	__le16 srq_depth;
-	__le16 rsv;
+	__le16 cap_flags_ex;
 	__le32 num_ceqs_ceq_depth;
 	__le32 arm_st_aeq_depth;
 	__le32 num_uars_rsv_pds;
@@ -1978,7 +1979,7 @@ int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
 				    void __iomem *dest)
 {
-	struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
 	struct hnae3_handle *handle = priv->handle;
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index d0031d559213..50763cf4fa3d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -233,7 +233,6 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
 	enum ib_mtu mtu;
 	u8 port;
 
-	assert(port_num > 0);
 	port = port_num - 1;
 
 	/* props being zeroed by the caller, avoid zeroing it here */
@@ -579,33 +578,12 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
 	int ret;
 	struct device *dev = hr_dev->dev;
 
-	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtt_table,
-				      HEM_TYPE_MTT, hr_dev->caps.mtt_entry_sz,
-				      hr_dev->caps.num_mtt_segs, 1);
-	if (ret) {
-		dev_err(dev, "Failed to init MTT context memory, aborting.\n");
-		return ret;
-	}
-
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {
-		ret = hns_roce_init_hem_table(hr_dev,
-					      &hr_dev->mr_table.mtt_cqe_table,
-					      HEM_TYPE_CQE,
-					      hr_dev->caps.mtt_entry_sz,
-					      hr_dev->caps.num_cqe_segs, 1);
-		if (ret) {
-			dev_err(dev,
-				"Failed to init CQE context memory, aborting.\n");
-			goto err_unmap_cqe;
-		}
-	}
-
 	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table,
 				      HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz,
 				      hr_dev->caps.num_mtpts, 1);
 	if (ret) {
 		dev_err(dev, "Failed to init MTPT context memory, aborting.\n");
-		goto err_unmap_mtt;
+		return ret;
 	}
 
 	ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table,
@@ -660,32 +638,6 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
 		}
 	}
 
-	if (hr_dev->caps.num_srqwqe_segs) {
-		ret = hns_roce_init_hem_table(hr_dev,
-					     &hr_dev->mr_table.mtt_srqwqe_table,
-					     HEM_TYPE_SRQWQE,
-					     hr_dev->caps.mtt_entry_sz,
-					     hr_dev->caps.num_srqwqe_segs, 1);
-		if (ret) {
-			dev_err(dev,
-				"Failed to init MTT srqwqe memory, aborting.\n");
-			goto err_unmap_srq;
-		}
-	}
-
-	if (hr_dev->caps.num_idx_segs) {
-		ret = hns_roce_init_hem_table(hr_dev,
-					      &hr_dev->mr_table.mtt_idx_table,
-					      HEM_TYPE_IDX,
-					      hr_dev->caps.idx_entry_sz,
-					      hr_dev->caps.num_idx_segs, 1);
-		if (ret) {
-			dev_err(dev,
-				"Failed to init MTT idx memory, aborting.\n");
-			goto err_unmap_srqwqe;
-		}
-	}
-
 	if (hr_dev->caps.sccc_entry_sz) {
 		ret = hns_roce_init_hem_table(hr_dev,
 					      &hr_dev->qp_table.sccc_table,
@@ -695,7 +647,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
 		if (ret) {
 			dev_err(dev,
 				"Failed to init SCC context memory, aborting.\n");
-			goto err_unmap_idx;
+			goto err_unmap_srq;
 		}
 	}
 
@@ -733,17 +685,6 @@ err_unmap_ctx:
 	if (hr_dev->caps.sccc_entry_sz)
 		hns_roce_cleanup_hem_table(hr_dev,
 					   &hr_dev->qp_table.sccc_table);
-
-err_unmap_idx:
-	if (hr_dev->caps.num_idx_segs)
-		hns_roce_cleanup_hem_table(hr_dev,
-					   &hr_dev->mr_table.mtt_idx_table);
-
-err_unmap_srqwqe:
-	if (hr_dev->caps.num_srqwqe_segs)
-		hns_roce_cleanup_hem_table(hr_dev,
-					   &hr_dev->mr_table.mtt_srqwqe_table);
-
 err_unmap_srq:
 	if (hr_dev->caps.srqc_entry_sz)
 		hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table);
@@ -765,14 +706,6 @@ err_unmap_qp:
 err_unmap_dmpt:
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
 
-err_unmap_mtt:
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
-		hns_roce_cleanup_hem_table(hr_dev,
-					   &hr_dev->mr_table.mtt_cqe_table);
-
-err_unmap_cqe:
-	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
-
 	return ret;
 }
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index 176f34692f88..4c0bbb12770d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -66,645 +66,89 @@ int hns_roce_hw_destroy_mpt(struct hns_roce_dev *hr_dev,
 				 HNS_ROCE_CMD_TIMEOUT_MSECS);
 }
 
-static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
-				unsigned long *seg)
+static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
+			u32 pd, u64 iova, u64 size, u32 access)
 {
-	int o;
-	u32 m;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	unsigned long obj = 0;
+	int err;
 
-	spin_lock(&buddy->lock);
-
-	for (o = order; o <= buddy->max_order; ++o) {
-		if (buddy->num_free[o]) {
-			m = 1 << (buddy->max_order - o);
-			*seg = find_first_bit(buddy->bits[o], m);
-			if (*seg < m)
-				goto found;
-		}
-	}
-	spin_unlock(&buddy->lock);
-	return -EINVAL;
-
- found:
-	clear_bit(*seg, buddy->bits[o]);
-	--buddy->num_free[o];
-
-	while (o > order) {
-		--o;
-		*seg <<= 1;
-		set_bit(*seg ^ 1, buddy->bits[o]);
-		++buddy->num_free[o];
-	}
-
-	spin_unlock(&buddy->lock);
-
-	*seg <<= order;
-	return 0;
-}
-
-static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg,
-				int order)
-{
-	seg >>= order;
-
-	spin_lock(&buddy->lock);
-
-	while (test_bit(seg ^ 1, buddy->bits[order])) {
-		clear_bit(seg ^ 1, buddy->bits[order]);
-		--buddy->num_free[order];
-		seg >>= 1;
-		++order;
-	}
-
-	set_bit(seg, buddy->bits[order]);
-	++buddy->num_free[order];
-
-	spin_unlock(&buddy->lock);
-}
-
-static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
-{
-	int i, s;
-
-	buddy->max_order = max_order;
-	spin_lock_init(&buddy->lock);
-	buddy->bits = kcalloc(buddy->max_order + 1,
-			      sizeof(*buddy->bits),
-			      GFP_KERNEL);
-	buddy->num_free = kcalloc(buddy->max_order + 1,
-				  sizeof(*buddy->num_free),
-				  GFP_KERNEL);
-	if (!buddy->bits || !buddy->num_free)
-		goto err_out;
-
-	for (i = 0; i <= buddy->max_order; ++i) {
-		s = BITS_TO_LONGS(1 << (buddy->max_order - i));
-		buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL |
-					 __GFP_NOWARN);
-		if (!buddy->bits[i]) {
-			buddy->bits[i] = vzalloc(array_size(s, sizeof(long)));
-			if (!buddy->bits[i])
-				goto err_out_free;
-		}
-	}
-
-	set_bit(0, buddy->bits[buddy->max_order]);
-	buddy->num_free[buddy->max_order] = 1;
-
-	return 0;
-
-err_out_free:
-	for (i = 0; i <= buddy->max_order; ++i)
-		kvfree(buddy->bits[i]);
-
-err_out:
-	kfree(buddy->bits);
-	kfree(buddy->num_free);
-	return -ENOMEM;
-}
-
-static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)
-{
-	int i;
-
-	for (i = 0; i <= buddy->max_order; ++i)
-		kvfree(buddy->bits[i]);
-
-	kfree(buddy->bits);
-	kfree(buddy->num_free);
-}
-
-static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order,
-				    unsigned long *seg, u32 mtt_type)
-{
-	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
-	struct hns_roce_hem_table *table;
-	struct hns_roce_buddy *buddy;
-	int ret;
-
-	switch (mtt_type) {
-	case MTT_TYPE_WQE:
-		buddy = &mr_table->mtt_buddy;
-		table = &mr_table->mtt_table;
-		break;
-	case MTT_TYPE_CQE:
-		buddy = &mr_table->mtt_cqe_buddy;
-		table = &mr_table->mtt_cqe_table;
-		break;
-	case MTT_TYPE_SRQWQE:
-		buddy = &mr_table->mtt_srqwqe_buddy;
-		table = &mr_table->mtt_srqwqe_table;
-		break;
-	case MTT_TYPE_IDX:
-		buddy = &mr_table->mtt_idx_buddy;
-		table = &mr_table->mtt_idx_table;
-		break;
-	default:
-		dev_err(hr_dev->dev, "Unsupport MTT table type: %d\n",
-			mtt_type);
-		return -EINVAL;
-	}
-
-	ret = hns_roce_buddy_alloc(buddy, order, seg);
-	if (ret)
-		return ret;
-
-	ret = hns_roce_table_get_range(hr_dev, table, *seg,
-				       *seg + (1 << order) - 1);
-	if (ret) {
-		hns_roce_buddy_free(buddy, *seg, order);
-		return ret;
-	}
-
-	return 0;
-}
-
-int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
-		      struct hns_roce_mtt *mtt)
-{
-	int ret;
-	int i;
-
-	/* Page num is zero, correspond to DMA memory register */
-	if (!npages) {
-		mtt->order = -1;
-		mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT;
-		return 0;
-	}
-
-	/* Note: if page_shift is zero, FAST memory register */
-	mtt->page_shift = page_shift;
-
-	/* Compute MTT entry necessary */
-	for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages;
-	     i <<= 1)
-		++mtt->order;
-
-	/* Allocate MTT entry */
-	ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg,
-				       mtt->mtt_type);
-	if (ret)
-		return -ENOMEM;
-
-	return 0;
-}
-
-void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)
-{
-	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
-
-	if (mtt->order < 0)
-		return;
-
-	switch (mtt->mtt_type) {
-	case MTT_TYPE_WQE:
-		hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,
-				    mtt->order);
-		hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,
-					mtt->first_seg,
-					mtt->first_seg + (1 << mtt->order) - 1);
-		break;
-	case MTT_TYPE_CQE:
-		hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,
-				    mtt->order);
-		hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,
-					mtt->first_seg,
-					mtt->first_seg + (1 << mtt->order) - 1);
-		break;
-	case MTT_TYPE_SRQWQE:
-		hns_roce_buddy_free(&mr_table->mtt_srqwqe_buddy, mtt->first_seg,
-				    mtt->order);
-		hns_roce_table_put_range(hr_dev, &mr_table->mtt_srqwqe_table,
-					mtt->first_seg,
-					mtt->first_seg + (1 << mtt->order) - 1);
-		break;
-	case MTT_TYPE_IDX:
-		hns_roce_buddy_free(&mr_table->mtt_idx_buddy, mtt->first_seg,
-				    mtt->order);
-		hns_roce_table_put_range(hr_dev, &mr_table->mtt_idx_table,
-					mtt->first_seg,
-					mtt->first_seg + (1 << mtt->order) - 1);
-		break;
-	default:
-		dev_err(hr_dev->dev,
-			"Unsupport mtt type %d, clean mtt failed\n",
-			mtt->mtt_type);
-		break;
-	}
-}
-
-static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
-			       struct hns_roce_mr *mr, int err_loop_index,
-			       int loop_i, int loop_j)
-{
-	struct device *dev = hr_dev->dev;
-	u32 mhop_num;
-	u32 pbl_bt_sz;
-	u64 bt_idx;
-	int i, j;
-
-	pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
-	mhop_num = hr_dev->caps.pbl_hop_num;
-
-	i = loop_i;
-	if (mhop_num == 3 && err_loop_index == 2) {
-		for (; i >= 0; i--) {
-			dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
-					  mr->pbl_l1_dma_addr[i]);
-
-			for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
-				if (i == loop_i && j >= loop_j)
-					break;
-
-				bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j;
-				dma_free_coherent(dev, pbl_bt_sz,
-						  mr->pbl_bt_l2[bt_idx],
-						  mr->pbl_l2_dma_addr[bt_idx]);
-			}
-		}
-	} else if (mhop_num == 3 && err_loop_index == 1) {
-		for (i -= 1; i >= 0; i--) {
-			dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
-					  mr->pbl_l1_dma_addr[i]);
-
-			for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
-				bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j;
-				dma_free_coherent(dev, pbl_bt_sz,
-						  mr->pbl_bt_l2[bt_idx],
-						  mr->pbl_l2_dma_addr[bt_idx]);
-			}
-		}
-	} else if (mhop_num == 2 && err_loop_index == 1) {
-		for (i -= 1; i >= 0; i--)
-			dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
-					  mr->pbl_l1_dma_addr[i]);
-	} else {
-		dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.",
-			 mhop_num, err_loop_index);
-		return;
-	}
-
-	dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr);
-	mr->pbl_bt_l0 = NULL;
-	mr->pbl_l0_dma_addr = 0;
-}
-static int pbl_1hop_alloc(struct hns_roce_dev *hr_dev, int npages,
-			       struct hns_roce_mr *mr, u32 pbl_bt_sz)
-{
-	struct device *dev = hr_dev->dev;
-
-	if (npages > pbl_bt_sz / 8) {
-		dev_err(dev, "npages %d is larger than buf_pg_sz!",
-			npages);
-		return -EINVAL;
-	}
-	mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
-					 &(mr->pbl_dma_addr),
-					 GFP_KERNEL);
-	if (!mr->pbl_buf)
+	/* Allocate a key for mr from mr_table */
+	err = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &obj);
+	if (err) {
+		ibdev_err(ibdev,
+			  "failed to alloc bitmap for MR key, ret = %d.\n",
+			  err);
 		return -ENOMEM;
-
-	mr->pbl_size = npages;
-	mr->pbl_ba = mr->pbl_dma_addr;
-	mr->pbl_hop_num = 1;
-	mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
-	mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
-	return 0;
-
-}
-
-
-static int pbl_2hop_alloc(struct hns_roce_dev *hr_dev, int npages,
-			       struct hns_roce_mr *mr, u32 pbl_bt_sz)
-{
-	struct device *dev = hr_dev->dev;
-	int npages_allocated;
-	u64 pbl_last_bt_num;
-	u64 pbl_bt_cnt = 0;
-	u64 size;
-	int i;
-
-	pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);
-
-	/* alloc L1 BT */
-	for (i = 0; i < pbl_bt_sz / 8; i++) {
-		if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
-			size = pbl_bt_sz;
-		} else {
-			npages_allocated = i * (pbl_bt_sz / 8);
-			size = (npages - npages_allocated) * 8;
-		}
-		mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size,
-					    &(mr->pbl_l1_dma_addr[i]),
-					    GFP_KERNEL);
-		if (!mr->pbl_bt_l1[i]) {
-			hns_roce_loop_free(hr_dev, mr, 1, i, 0);
-			return -ENOMEM;
-		}
-
-		*(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
-
-		pbl_bt_cnt++;
-		if (pbl_bt_cnt >= pbl_last_bt_num)
-			break;
 	}
 
-	mr->l0_chunk_last_num = i + 1;
-
-	return 0;
-}
-
-static int pbl_3hop_alloc(struct hns_roce_dev *hr_dev, int npages,
-			       struct hns_roce_mr *mr, u32 pbl_bt_sz)
-{
-	struct device *dev = hr_dev->dev;
-	int mr_alloc_done = 0;
-	int npages_allocated;
-	u64 pbl_last_bt_num;
-	u64 pbl_bt_cnt = 0;
-	u64 bt_idx;
-	u64 size;
-	int i;
-	int j = 0;
-
-	pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);
-
-	mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num,
-				      sizeof(*mr->pbl_l2_dma_addr),
-				      GFP_KERNEL);
-	if (!mr->pbl_l2_dma_addr)
-		return -ENOMEM;
-
-	mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num,
-				sizeof(*mr->pbl_bt_l2),
-				GFP_KERNEL);
-	if (!mr->pbl_bt_l2)
-		goto err_kcalloc_bt_l2;
-
-	/* alloc L1, L2 BT */
-	for (i = 0; i < pbl_bt_sz / 8; i++) {
-		mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz,
-					    &(mr->pbl_l1_dma_addr[i]),
-					    GFP_KERNEL);
-		if (!mr->pbl_bt_l1[i]) {
-			hns_roce_loop_free(hr_dev, mr, 1, i, 0);
-			goto err_dma_alloc_l0;
-		}
-
-		*(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
-
-		for (j = 0; j < pbl_bt_sz / 8; j++) {
-			bt_idx = i * pbl_bt_sz / 8 + j;
-
-			if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
-				size = pbl_bt_sz;
-			} else {
-				npages_allocated = bt_idx *
-						   (pbl_bt_sz / 8);
-				size = (npages - npages_allocated) * 8;
-			}
-			mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent(
-				      dev, size,
-				      &(mr->pbl_l2_dma_addr[bt_idx]),
-				      GFP_KERNEL);
-			if (!mr->pbl_bt_l2[bt_idx]) {
-				hns_roce_loop_free(hr_dev, mr, 2, i, j);
-				goto err_dma_alloc_l0;
-			}
-
-			*(mr->pbl_bt_l1[i] + j) =
-					mr->pbl_l2_dma_addr[bt_idx];
-
-			pbl_bt_cnt++;
-			if (pbl_bt_cnt >= pbl_last_bt_num) {
-				mr_alloc_done = 1;
-				break;
-			}
-		}
+	mr->iova = iova;			/* MR va starting addr */
+	mr->size = size;			/* MR addr range */
+	mr->pd = pd;				/* MR num */
+	mr->access = access;			/* MR access permit */
+	mr->enabled = 0;			/* MR active status */
+	mr->key = hw_index_to_key(obj);		/* MR key */
 
-		if (mr_alloc_done)
-			break;
+	err = hns_roce_table_get(hr_dev, &hr_dev->mr_table.mtpt_table, obj);
+	if (err) {
+		ibdev_err(ibdev, "failed to alloc mtpt, ret = %d.\n", err);
+		goto err_free_bitmap;
 	}
 
-	mr->l0_chunk_last_num = i + 1;
-	mr->l1_chunk_last_num = j + 1;
-
-
 	return 0;
-
-err_dma_alloc_l0:
-	kfree(mr->pbl_bt_l2);
-	mr->pbl_bt_l2 = NULL;
-
-err_kcalloc_bt_l2:
-	kfree(mr->pbl_l2_dma_addr);
-	mr->pbl_l2_dma_addr = NULL;
-
-	return -ENOMEM;
+err_free_bitmap:
+	hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, obj, BITMAP_NO_RR);
+	return err;
 }
 
-
-/* PBL multi hop addressing */
-static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages,
-			       struct hns_roce_mr *mr)
+static void free_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
 {
-	struct device *dev = hr_dev->dev;
-	u32 pbl_bt_sz;
-	u32 mhop_num;
-
-	mhop_num = (mr->type == MR_TYPE_FRMR ? 1 : hr_dev->caps.pbl_hop_num);
-	pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
-
-	if (mhop_num == HNS_ROCE_HOP_NUM_0)
-		return 0;
-
-	if (mhop_num == 1)
-		return pbl_1hop_alloc(hr_dev, npages, mr, pbl_bt_sz);
-
-	mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8,
-				      sizeof(*mr->pbl_l1_dma_addr),
-				      GFP_KERNEL);
-	if (!mr->pbl_l1_dma_addr)
-		return -ENOMEM;
-
-	mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1),
-				GFP_KERNEL);
-	if (!mr->pbl_bt_l1)
-		goto err_kcalloc_bt_l1;
-
-	/* alloc L0 BT */
-	mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz,
-					   &(mr->pbl_l0_dma_addr),
-					   GFP_KERNEL);
-	if (!mr->pbl_bt_l0)
-		goto err_kcalloc_l2_dma;
-
-	if (mhop_num == 2) {
-		if (pbl_2hop_alloc(hr_dev, npages, mr, pbl_bt_sz))
-			goto err_kcalloc_l2_dma;
-	}
-
-	if (mhop_num == 3) {
-		if (pbl_3hop_alloc(hr_dev, npages, mr, pbl_bt_sz))
-			goto err_kcalloc_l2_dma;
-	}
-
+	unsigned long obj = key_to_hw_index(mr->key);
 
-	mr->pbl_size = npages;
-	mr->pbl_ba = mr->pbl_l0_dma_addr;
-	mr->pbl_hop_num = hr_dev->caps.pbl_hop_num;
-	mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
-	mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
-
-	return 0;
-
-err_kcalloc_l2_dma:
-	kfree(mr->pbl_bt_l1);
-	mr->pbl_bt_l1 = NULL;
-
-err_kcalloc_bt_l1:
-	kfree(mr->pbl_l1_dma_addr);
-	mr->pbl_l1_dma_addr = NULL;
-
-	return -ENOMEM;
+	hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table, obj);
+	hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, obj, BITMAP_NO_RR);
 }
 
-static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
-			     u64 size, u32 access, int npages,
-			     struct hns_roce_mr *mr)
+static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
+			size_t length, struct ib_udata *udata, u64 start,
+			int access)
 {
-	struct device *dev = hr_dev->dev;
-	unsigned long index = 0;
-	int ret;
-
-	/* Allocate a key for mr from mr_table */
-	ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
-	if (ret)
-		return -ENOMEM;
-
-	mr->iova = iova;			/* MR va starting addr */
-	mr->size = size;			/* MR addr range */
-	mr->pd = pd;				/* MR num */
-	mr->access = access;			/* MR access permit */
-	mr->enabled = 0;			/* MR active status */
-	mr->key = hw_index_to_key(index);	/* MR key */
-
-	if (size == ~0ull) {
-		mr->pbl_buf = NULL;
-		mr->pbl_dma_addr = 0;
-		/* PBL multi-hop addressing parameters */
-		mr->pbl_bt_l2 = NULL;
-		mr->pbl_bt_l1 = NULL;
-		mr->pbl_bt_l0 = NULL;
-		mr->pbl_l2_dma_addr = NULL;
-		mr->pbl_l1_dma_addr = NULL;
-		mr->pbl_l0_dma_addr = 0;
-	} else {
-		if (!hr_dev->caps.pbl_hop_num) {
-			mr->pbl_buf = dma_alloc_coherent(dev,
-							 npages * BA_BYTE_LEN,
-							 &(mr->pbl_dma_addr),
-							 GFP_KERNEL);
-			if (!mr->pbl_buf)
-				return -ENOMEM;
-		} else {
-			ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
-		}
-	}
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	bool is_fast = mr->type == MR_TYPE_FRMR;
+	struct hns_roce_buf_attr buf_attr = {};
+	int err;
+
+	mr->pbl_hop_num = is_fast ? 1 : hr_dev->caps.pbl_hop_num;
+	buf_attr.page_shift = is_fast ? PAGE_SHIFT :
+			      hr_dev->caps.pbl_buf_pg_sz + HNS_HW_PAGE_SHIFT;
+	buf_attr.region[0].size = length;
+	buf_attr.region[0].hopnum = mr->pbl_hop_num;
+	buf_attr.region_count = 1;
+	buf_attr.fixed_page = true;
+	buf_attr.user_access = access;
+	/* fast MR's buffer is alloced before mapping, not at creation */
+	buf_attr.mtt_only = is_fast;
+
+	err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,
+				  hr_dev->caps.pbl_ba_pg_sz + HNS_HW_PAGE_SHIFT,
+				  udata, start);
+	if (err)
+		ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err);
+	else
+		mr->npages = mr->pbl_mtr.hem_cfg.buf_pg_count;
 
-	return ret;
+	return err;
 }
 
-static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
-			       struct hns_roce_mr *mr)
+static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
 {
-	struct device *dev = hr_dev->dev;
-	int npages_allocated;
-	int npages;
-	int i, j;
-	u32 pbl_bt_sz;
-	u32 mhop_num;
-	u64 bt_idx;
-
-	npages = mr->pbl_size;
-	pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
-	mhop_num = (mr->type == MR_TYPE_FRMR) ? 1 : hr_dev->caps.pbl_hop_num;
-
-	if (mhop_num == HNS_ROCE_HOP_NUM_0)
-		return;
-
-	if (mhop_num == 1) {
-		dma_free_coherent(dev, (unsigned int)(npages * BA_BYTE_LEN),
-				  mr->pbl_buf, mr->pbl_dma_addr);
-		return;
-	}
-
-	dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0,
-			  mr->pbl_l0_dma_addr);
-
-	if (mhop_num == 2) {
-		for (i = 0; i < mr->l0_chunk_last_num; i++) {
-			if (i == mr->l0_chunk_last_num - 1) {
-				npages_allocated =
-						i * (pbl_bt_sz / BA_BYTE_LEN);
-
-				dma_free_coherent(dev,
-				      (npages - npages_allocated) * BA_BYTE_LEN,
-				       mr->pbl_bt_l1[i],
-				       mr->pbl_l1_dma_addr[i]);
-
-				break;
-			}
-
-			dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
-					  mr->pbl_l1_dma_addr[i]);
-		}
-	} else if (mhop_num == 3) {
-		for (i = 0; i < mr->l0_chunk_last_num; i++) {
-			dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
-					  mr->pbl_l1_dma_addr[i]);
-
-			for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
-				bt_idx = i * (pbl_bt_sz / BA_BYTE_LEN) + j;
-
-				if ((i == mr->l0_chunk_last_num - 1)
-				    && j == mr->l1_chunk_last_num - 1) {
-					npages_allocated = bt_idx *
-						      (pbl_bt_sz / BA_BYTE_LEN);
-
-					dma_free_coherent(dev,
-					      (npages - npages_allocated) *
-					      BA_BYTE_LEN,
-					      mr->pbl_bt_l2[bt_idx],
-					      mr->pbl_l2_dma_addr[bt_idx]);
-
-					break;
-				}
-
-				dma_free_coherent(dev, pbl_bt_sz,
-						mr->pbl_bt_l2[bt_idx],
-						mr->pbl_l2_dma_addr[bt_idx]);
-			}
-		}
-	}
-
-	kfree(mr->pbl_bt_l1);
-	kfree(mr->pbl_l1_dma_addr);
-	mr->pbl_bt_l1 = NULL;
-	mr->pbl_l1_dma_addr = NULL;
-	if (mhop_num == 3) {
-		kfree(mr->pbl_bt_l2);
-		kfree(mr->pbl_l2_dma_addr);
-		mr->pbl_bt_l2 = NULL;
-		mr->pbl_l2_dma_addr = NULL;
-	}
+	hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr);
 }
 
 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
 			     struct hns_roce_mr *mr)
 {
-	struct device *dev = hr_dev->dev;
-	int npages = 0;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
 	int ret;
 
 	if (mr->enabled) {
@@ -712,27 +156,12 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
 					      key_to_hw_index(mr->key) &
 					      (hr_dev->caps.num_mtpts - 1));
 		if (ret)
-			dev_warn(dev, "DESTROY_MPT failed (%d)\n", ret);
+			ibdev_warn(ibdev, "failed to destroy mpt, ret = %d.\n",
+				   ret);
 	}
 
-	if (mr->size != ~0ULL) {
-		if (mr->type == MR_TYPE_MR)
-			npages = ib_umem_page_count(mr->umem);
-
-		if (!hr_dev->caps.pbl_hop_num)
-			dma_free_coherent(dev,
-					  (unsigned int)(npages * BA_BYTE_LEN),
-					  mr->pbl_buf, mr->pbl_dma_addr);
-		else
-			hns_roce_mhop_free(hr_dev, mr);
-	}
-
-	if (mr->enabled)
-		hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
-				   key_to_hw_index(mr->key));
-
-	hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
-			     key_to_hw_index(mr->key), BITMAP_NO_RR);
+	free_mr_pbl(hr_dev, mr);
+	free_mr_key(hr_dev, mr);
 }
 
 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
@@ -742,18 +171,12 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
 	unsigned long mtpt_idx = key_to_hw_index(mr->key);
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_cmd_mailbox *mailbox;
-	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
-
-	/* Prepare HEM entry memory */
-	ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
-	if (ret)
-		return ret;
 
 	/* Allocate mailbox memory */
 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
 	if (IS_ERR(mailbox)) {
 		ret = PTR_ERR(mailbox);
-		goto err_table;
+		return ret;
 	}
 
 	if (mr->type != MR_TYPE_FRMR)
@@ -780,137 +203,6 @@ static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
 err_page:
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
 
-err_table:
-	hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
-	return ret;
-}
-
-static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev,
-				    struct hns_roce_mtt *mtt, u32 start_index,
-				    u32 npages, u64 *page_list)
-{
-	struct hns_roce_hem_table *table;
-	dma_addr_t dma_handle;
-	__le64 *mtts;
-	u32 bt_page_size;
-	u32 i;
-
-	switch (mtt->mtt_type) {
-	case MTT_TYPE_WQE:
-		table = &hr_dev->mr_table.mtt_table;
-		bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
-		break;
-	case MTT_TYPE_CQE:
-		table = &hr_dev->mr_table.mtt_cqe_table;
-		bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
-		break;
-	case MTT_TYPE_SRQWQE:
-		table = &hr_dev->mr_table.mtt_srqwqe_table;
-		bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
-		break;
-	case MTT_TYPE_IDX:
-		table = &hr_dev->mr_table.mtt_idx_table;
-		bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* All MTTs must fit in the same page */
-	if (start_index / (bt_page_size / sizeof(u64)) !=
-		(start_index + npages - 1) / (bt_page_size / sizeof(u64)))
-		return -EINVAL;
-
-	if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
-		return -EINVAL;
-
-	mtts = hns_roce_table_find(hr_dev, table,
-				mtt->first_seg +
-				start_index / HNS_ROCE_MTT_ENTRY_PER_SEG,
-				&dma_handle);
-	if (!mtts)
-		return -ENOMEM;
-
-	/* Save page addr, low 12 bits : 0 */
-	for (i = 0; i < npages; ++i) {
-		if (!hr_dev->caps.mtt_hop_num)
-			mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT);
-		else
-			mtts[i] = cpu_to_le64(page_list[i]);
-	}
-
-	return 0;
-}
-
-static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_mtt *mtt, u32 start_index,
-			      u32 npages, u64 *page_list)
-{
-	int chunk;
-	int ret;
-	u32 bt_page_size;
-
-	if (mtt->order < 0)
-		return -EINVAL;
-
-	switch (mtt->mtt_type) {
-	case MTT_TYPE_WQE:
-		bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
-		break;
-	case MTT_TYPE_CQE:
-		bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
-		break;
-	case MTT_TYPE_SRQWQE:
-		bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
-		break;
-	case MTT_TYPE_IDX:
-		bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
-		break;
-	default:
-		dev_err(hr_dev->dev,
-			"Unsupport mtt type %d, write mtt failed\n",
-			mtt->mtt_type);
-		return -EINVAL;
-	}
-
-	while (npages > 0) {
-		chunk = min_t(int, bt_page_size / sizeof(u64), npages);
-
-		ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk,
-					       page_list);
-		if (ret)
-			return ret;
-
-		npages -= chunk;
-		start_index += chunk;
-		page_list += chunk;
-	}
-
-	return 0;
-}
-
-int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
-			   struct hns_roce_mtt *mtt, struct hns_roce_buf *buf)
-{
-	u64 *page_list;
-	int ret;
-	u32 i;
-
-	page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL);
-	if (!page_list)
-		return -ENOMEM;
-
-	for (i = 0; i < buf->npages; ++i) {
-		if (buf->nbufs == 1)
-			page_list[i] = buf->direct.map + (i << buf->page_shift);
-		else
-			page_list[i] = buf->page_list[i].map;
-
-	}
-	ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list);
-
-	kfree(page_list);
-
 	return ret;
 }
 
@@ -923,50 +215,6 @@ int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)
 				   hr_dev->caps.num_mtpts,
 				   hr_dev->caps.num_mtpts - 1,
 				   hr_dev->caps.reserved_mrws, 0);
-	if (ret)
-		return ret;
-
-	ret = hns_roce_buddy_init(&mr_table->mtt_buddy,
-				  ilog2(hr_dev->caps.num_mtt_segs));
-	if (ret)
-		goto err_buddy;
-
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {
-		ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy,
-					  ilog2(hr_dev->caps.num_cqe_segs));
-		if (ret)
-			goto err_buddy_cqe;
-	}
-
-	if (hr_dev->caps.num_srqwqe_segs) {
-		ret = hns_roce_buddy_init(&mr_table->mtt_srqwqe_buddy,
-					  ilog2(hr_dev->caps.num_srqwqe_segs));
-		if (ret)
-			goto err_buddy_srqwqe;
-	}
-
-	if (hr_dev->caps.num_idx_segs) {
-		ret = hns_roce_buddy_init(&mr_table->mtt_idx_buddy,
-					  ilog2(hr_dev->caps.num_idx_segs));
-		if (ret)
-			goto err_buddy_idx;
-	}
-
-	return 0;
-
-err_buddy_idx:
-	if (hr_dev->caps.num_srqwqe_segs)
-		hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
-
-err_buddy_srqwqe:
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
-		hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
-
-err_buddy_cqe:
-	hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
-
-err_buddy:
-	hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
 	return ret;
 }
 
@@ -974,30 +222,24 @@ void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
 
-	if (hr_dev->caps.num_idx_segs)
-		hns_roce_buddy_cleanup(&mr_table->mtt_idx_buddy);
-	if (hr_dev->caps.num_srqwqe_segs)
-		hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
-	hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
-	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
-		hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
 	hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
 }
 
 struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
 	struct hns_roce_mr *mr;
 	int ret;
 
-	mr = kmalloc(sizeof(*mr), GFP_KERNEL);
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 	if (mr == NULL)
 		return  ERR_PTR(-ENOMEM);
 
 	mr->type = MR_TYPE_DMA;
 
 	/* Allocate memory region key */
-	ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0,
-				~0ULL, acc, 0, mr);
+	hns_roce_hem_list_init(&mr->pbl_mtr.hem_list);
+	ret = alloc_mr_key(hr_dev, mr, to_hr_pd(pd)->pdn, 0, 0, acc);
 	if (ret)
 		goto err_free;
 
@@ -1006,203 +248,52 @@ struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)
 		goto err_mr;
 
 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
-	mr->umem = NULL;
 
 	return &mr->ibmr;
-
 err_mr:
-	hns_roce_mr_free(to_hr_dev(pd->device), mr);
+	free_mr_key(hr_dev, mr);
 
 err_free:
 	kfree(mr);
 	return ERR_PTR(ret);
 }
 
-int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
-			       struct hns_roce_mtt *mtt, struct ib_umem *umem)
-{
-	struct device *dev = hr_dev->dev;
-	struct sg_dma_page_iter sg_iter;
-	unsigned int order;
-	int npage = 0;
-	int ret = 0;
-	int i;
-	u64 page_addr;
-	u64 *pages;
-	u32 bt_page_size;
-	u32 n;
-
-	switch (mtt->mtt_type) {
-	case MTT_TYPE_WQE:
-		order = hr_dev->caps.mtt_ba_pg_sz;
-		break;
-	case MTT_TYPE_CQE:
-		order = hr_dev->caps.cqe_ba_pg_sz;
-		break;
-	case MTT_TYPE_SRQWQE:
-		order = hr_dev->caps.srqwqe_ba_pg_sz;
-		break;
-	case MTT_TYPE_IDX:
-		order = hr_dev->caps.idx_ba_pg_sz;
-		break;
-	default:
-		dev_err(dev, "Unsupport mtt type %d, write mtt failed\n",
-			mtt->mtt_type);
-		return -EINVAL;
-	}
-
-	bt_page_size = 1 << (order + PAGE_SHIFT);
-
-	pages = (u64 *) __get_free_pages(GFP_KERNEL, order);
-	if (!pages)
-		return -ENOMEM;
-
-	i = n = 0;
-
-	for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
-		page_addr = sg_page_iter_dma_address(&sg_iter);
-		if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) {
-			if (page_addr & ((1 << mtt->page_shift) - 1)) {
-				dev_err(dev,
-					"page_addr is not page_shift %d alignment!\n",
-					mtt->page_shift);
-				ret = -EINVAL;
-				goto out;
-			}
-			pages[i++] = page_addr;
-		}
-		npage++;
-		if (i == bt_page_size / sizeof(u64)) {
-			ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
-			if (ret)
-				goto out;
-			n += i;
-			i = 0;
-		}
-	}
-
-	if (i)
-		ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
-
-out:
-	free_pages((unsigned long) pages, order);
-	return ret;
-}
-
-static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,
-				     struct hns_roce_mr *mr,
-				     struct ib_umem *umem)
-{
-	struct sg_dma_page_iter sg_iter;
-	int i = 0, j = 0;
-	u64 page_addr;
-	u32 pbl_bt_sz;
-
-	if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0)
-		return 0;
-
-	pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
-	for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
-		page_addr = sg_page_iter_dma_address(&sg_iter);
-		if (!hr_dev->caps.pbl_hop_num) {
-			/* for hip06, page addr is aligned to 4K */
-			mr->pbl_buf[i++] = page_addr >> 12;
-		} else if (hr_dev->caps.pbl_hop_num == 1) {
-			mr->pbl_buf[i++] = page_addr;
-		} else {
-			if (hr_dev->caps.pbl_hop_num == 2)
-				mr->pbl_bt_l1[i][j] = page_addr;
-			else if (hr_dev->caps.pbl_hop_num == 3)
-				mr->pbl_bt_l2[i][j] = page_addr;
-
-			j++;
-			if (j >= (pbl_bt_sz / BA_BYTE_LEN)) {
-				i++;
-				j = 0;
-			}
-		}
-	}
-
-	/* Memory barrier */
-	mb();
-
-	return 0;
-}
-
 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				   u64 virt_addr, int access_flags,
 				   struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
-	struct device *dev = hr_dev->dev;
 	struct hns_roce_mr *mr;
-	int bt_size;
 	int ret;
-	int n;
-	int i;
 
-	mr = kmalloc(sizeof(*mr), GFP_KERNEL);
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	mr->umem = ib_umem_get(pd->device, start, length, access_flags);
-	if (IS_ERR(mr->umem)) {
-		ret = PTR_ERR(mr->umem);
-		goto err_free;
-	}
-
-	n = ib_umem_page_count(mr->umem);
-
-	if (!hr_dev->caps.pbl_hop_num) {
-		if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) {
-			dev_err(dev,
-			     " MR len %lld err. MR is limited to 4G at most!\n",
-			     length);
-			ret = -EINVAL;
-			goto err_umem;
-		}
-	} else {
-		u64 pbl_size = 1;
-
-		bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) /
-			  BA_BYTE_LEN;
-		for (i = 0; i < hr_dev->caps.pbl_hop_num; i++)
-			pbl_size *= bt_size;
-		if (n > pbl_size) {
-			dev_err(dev,
-			    " MR len %lld err. MR page num is limited to %lld!\n",
-			    length, pbl_size);
-			ret = -EINVAL;
-			goto err_umem;
-		}
-	}
-
 	mr->type = MR_TYPE_MR;
-
-	ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length,
-				access_flags, n, mr);
+	ret = alloc_mr_key(hr_dev, mr, to_hr_pd(pd)->pdn, virt_addr, length,
+			   access_flags);
 	if (ret)
-		goto err_umem;
+		goto err_alloc_mr;
 
-	ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
+	ret = alloc_mr_pbl(hr_dev, mr, length, udata, start, access_flags);
 	if (ret)
-		goto err_mr;
+		goto err_alloc_key;
 
 	ret = hns_roce_mr_enable(hr_dev, mr);
 	if (ret)
-		goto err_mr;
+		goto err_alloc_pbl;
 
 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
+	mr->ibmr.length = length;
 
 	return &mr->ibmr;
 
-err_mr:
-	hns_roce_mr_free(hr_dev, mr);
-
-err_umem:
-	ib_umem_release(mr->umem);
-
-err_free:
+err_alloc_pbl:
+	free_mr_pbl(hr_dev, mr);
+err_alloc_key:
+	free_mr_key(hr_dev, mr);
+err_alloc_mr:
 	kfree(mr);
 	return ERR_PTR(ret);
 }
@@ -1214,84 +305,36 @@ static int rereg_mr_trans(struct ib_mr *ibmr, int flags,
 			  u32 pdn, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
+	struct ib_device *ibdev = &hr_dev->ib_dev;
 	struct hns_roce_mr *mr = to_hr_mr(ibmr);
-	struct device *dev = hr_dev->dev;
-	int npages;
 	int ret;
 
-	if (mr->size != ~0ULL) {
-		npages = ib_umem_page_count(mr->umem);
-
-		if (hr_dev->caps.pbl_hop_num)
-			hns_roce_mhop_free(hr_dev, mr);
-		else
-			dma_free_coherent(dev, npages * 8,
-					  mr->pbl_buf, mr->pbl_dma_addr);
-	}
-	ib_umem_release(mr->umem);
-
-	mr->umem = ib_umem_get(ibmr->device, start, length, mr_access_flags);
-	if (IS_ERR(mr->umem)) {
-		ret = PTR_ERR(mr->umem);
-		mr->umem = NULL;
-		return -ENOMEM;
-	}
-	npages = ib_umem_page_count(mr->umem);
-
-	if (hr_dev->caps.pbl_hop_num) {
-		ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
-		if (ret)
-			goto release_umem;
-	} else {
-		mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
-						 &(mr->pbl_dma_addr),
-						 GFP_KERNEL);
-		if (!mr->pbl_buf) {
-			ret = -ENOMEM;
-			goto release_umem;
-		}
+	free_mr_pbl(hr_dev, mr);
+	ret = alloc_mr_pbl(hr_dev, mr, length, udata, start, mr_access_flags);
+	if (ret) {
+		ibdev_err(ibdev, "failed to create mr PBL, ret = %d.\n", ret);
+		return ret;
 	}
 
 	ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn,
 					   mr_access_flags, virt_addr,
 					   length, mailbox->buf);
-	if (ret)
-		goto release_umem;
-
-
-	ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
 	if (ret) {
-		if (mr->size != ~0ULL) {
-			npages = ib_umem_page_count(mr->umem);
-
-			if (hr_dev->caps.pbl_hop_num)
-				hns_roce_mhop_free(hr_dev, mr);
-			else
-				dma_free_coherent(dev, npages * 8,
-						  mr->pbl_buf,
-						  mr->pbl_dma_addr);
-		}
-
-		goto release_umem;
+		ibdev_err(ibdev, "failed to write mtpt, ret = %d.\n", ret);
+		free_mr_pbl(hr_dev, mr);
 	}
 
-	return 0;
-
-release_umem:
-	ib_umem_release(mr->umem);
 	return ret;
-
 }
 
-
 int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
 			   u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
 			   struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
+	struct ib_device *ib_dev = &hr_dev->ib_dev;
 	struct hns_roce_mr *mr = to_hr_mr(ibmr);
 	struct hns_roce_cmd_mailbox *mailbox;
-	struct device *dev = hr_dev->dev;
 	unsigned long mtpt_idx;
 	u32 pdn = 0;
 	int ret;
@@ -1312,7 +355,7 @@ int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
 
 	ret = hns_roce_hw_destroy_mpt(hr_dev, NULL, mtpt_idx);
 	if (ret)
-		dev_warn(dev, "DESTROY_MPT failed (%d)\n", ret);
+		ibdev_warn(ib_dev, "failed to destroy MPT, ret = %d.\n", ret);
 
 	mr->enabled = 0;
 
@@ -1336,8 +379,7 @@ int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
 
 	ret = hns_roce_hw_create_mpt(hr_dev, mailbox, mtpt_idx);
 	if (ret) {
-		dev_err(dev, "CREATE_MPT failed (%d)\n", ret);
-		ib_umem_release(mr->umem);
+		ibdev_err(ib_dev, "failed to create MPT, ret = %d.\n", ret);
 		goto free_cmd_mbox;
 	}
 
@@ -1365,8 +407,6 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 		ret = hr_dev->hw->dereg_mr(hr_dev, mr, udata);
 	} else {
 		hns_roce_mr_free(hr_dev, mr);
-
-		ib_umem_release(mr->umem);
 		kfree(mr);
 	}
 
@@ -1380,12 +420,8 @@ struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_mr *mr;
 	u64 length;
-	u32 page_size;
 	int ret;
 
-	page_size = 1 << (hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT);
-	length = max_num_sg * page_size;
-
 	if (mr_type != IB_MR_TYPE_MEM_REG)
 		return ERR_PTR(-EINVAL);
 
@@ -1402,23 +438,28 @@ struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
 	mr->type = MR_TYPE_FRMR;
 
 	/* Allocate memory region key */
-	ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, 0, length,
-				0, max_num_sg, mr);
+	length = max_num_sg * (1 << PAGE_SHIFT);
+	ret = alloc_mr_key(hr_dev, mr, to_hr_pd(pd)->pdn, 0, length, 0);
 	if (ret)
 		goto err_free;
 
+	ret = alloc_mr_pbl(hr_dev, mr, length, NULL, 0, 0);
+	if (ret)
+		goto err_key;
+
 	ret = hns_roce_mr_enable(hr_dev, mr);
 	if (ret)
-		goto err_mr;
+		goto err_pbl;
 
 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
-	mr->umem = NULL;
+	mr->ibmr.length = length;
 
 	return &mr->ibmr;
 
-err_mr:
-	hns_roce_mr_free(to_hr_dev(pd->device), mr);
-
+err_key:
+	free_mr_key(hr_dev, mr);
+err_pbl:
+	free_mr_pbl(hr_dev, mr);
 err_free:
 	kfree(mr);
 	return ERR_PTR(ret);
@@ -1428,19 +469,54 @@ static int hns_roce_set_page(struct ib_mr *ibmr, u64 addr)
 {
 	struct hns_roce_mr *mr = to_hr_mr(ibmr);
 
-	mr->pbl_buf[mr->npages++] = addr;
+	if (likely(mr->npages < mr->pbl_mtr.hem_cfg.buf_pg_count)) {
+		mr->page_list[mr->npages++] = addr;
+		return 0;
+	}
 
-	return 0;
+	return -ENOBUFS;
 }
 
 int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		       unsigned int *sg_offset)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
+	struct ib_device *ibdev = &hr_dev->ib_dev;
 	struct hns_roce_mr *mr = to_hr_mr(ibmr);
+	struct hns_roce_mtr *mtr = &mr->pbl_mtr;
+	int ret = 0;
 
 	mr->npages = 0;
+	mr->page_list = kvcalloc(mr->pbl_mtr.hem_cfg.buf_pg_count,
+				 sizeof(dma_addr_t), GFP_KERNEL);
+	if (!mr->page_list)
+		return ret;
+
+	ret = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page);
+	if (ret < 1) {
+		ibdev_err(ibdev, "failed to store sg pages %d %d, cnt = %d.\n",
+			  mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, ret);
+		goto err_page_list;
+	}
+
+	mtr->hem_cfg.region[0].offset = 0;
+	mtr->hem_cfg.region[0].count = mr->npages;
+	mtr->hem_cfg.region[0].hopnum = mr->pbl_hop_num;
+	mtr->hem_cfg.region_count = 1;
+	ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages);
+	if (ret) {
+		ibdev_err(ibdev, "failed to map sg mtr, ret = %d.\n", ret);
+		ret = 0;
+	} else {
+		mr->pbl_mtr.hem_cfg.buf_pg_shift = ilog2(ibmr->page_size);
+		ret = mr->npages;
+	}
+
+err_page_list:
+	kvfree(mr->page_list);
+	mr->page_list = NULL;
 
-	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page);
+	return ret;
 }
 
 static void hns_roce_mw_free(struct hns_roce_dev *hr_dev,
@@ -1564,32 +640,23 @@ int hns_roce_dealloc_mw(struct ib_mw *ibmw)
 	return 0;
 }
 
-void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift,
-		       int buf_pg_shift)
-{
-	hns_roce_hem_list_init(&mtr->hem_list, bt_pg_shift);
-	mtr->buf_pg_shift = buf_pg_shift;
-}
-
-void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev,
-			  struct hns_roce_mtr *mtr)
-{
-	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
-}
-
-static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_mtr *mtr, dma_addr_t *bufs,
-			      struct hns_roce_buf_region *r)
+static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+			  dma_addr_t *pages, struct hns_roce_buf_region *region)
 {
+	__le64 *mtts;
 	int offset;
 	int count;
 	int npage;
-	u64 *mtts;
+	u64 addr;
 	int end;
 	int i;
 
-	offset = r->offset;
-	end = offset + r->count;
+	/* if hopnum is 0, buffer cannot store BAs, so skip write mtt */
+	if (!region->hopnum)
+		return 0;
+
+	offset = region->offset;
+	end = offset + region->count;
 	npage = 0;
 	while (offset < end) {
 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
@@ -1597,13 +664,13 @@ static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev,
 		if (!mtts)
 			return -ENOBUFS;
 
-		/* Save page addr, low 12 bits : 0 */
 		for (i = 0; i < count; i++) {
 			if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
-				mtts[i] = bufs[npage] >> PAGE_ADDR_SHIFT;
+				addr = to_hr_hw_page_addr(pages[npage]);
 			else
-				mtts[i] = bufs[npage];
+				addr = pages[npage];
 
+			mtts[i] = cpu_to_le64(addr);
 			npage++;
 		}
 		offset += count;
@@ -1612,69 +679,416 @@ static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev,
 	return 0;
 }
 
-int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
-			dma_addr_t **bufs, struct hns_roce_buf_region *regions,
-			int region_cnt)
+static inline bool mtr_has_mtt(struct hns_roce_buf_attr *attr)
 {
-	struct hns_roce_buf_region *r;
-	int ret;
 	int i;
 
-	ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, regions,
-					region_cnt);
-	if (ret)
-		return ret;
+	for (i = 0; i < attr->region_count; i++)
+		if (attr->region[i].hopnum != HNS_ROCE_HOP_NUM_0 &&
+		    attr->region[i].hopnum > 0)
+			return true;
 
-	for (i = 0; i < region_cnt; i++) {
-		r = &regions[i];
-		ret = hns_roce_write_mtr(hr_dev, mtr, bufs[i], r);
+	/* because the mtr only one root base address, when hopnum is 0 means
+	 * root base address equals the first buffer address, thus all alloced
+	 * memory must in a continuous space accessed by direct mode.
+	 */
+	return false;
+}
+
+static inline size_t mtr_bufs_size(struct hns_roce_buf_attr *attr)
+{
+	size_t size = 0;
+	int i;
+
+	for (i = 0; i < attr->region_count; i++)
+		size += attr->region[i].size;
+
+	return size;
+}
+
+static inline int mtr_umem_page_count(struct ib_umem *umem,
+				      unsigned int page_shift)
+{
+	int count = ib_umem_page_count(umem);
+
+	if (page_shift >= PAGE_SHIFT)
+		count >>= page_shift - PAGE_SHIFT;
+	else
+		count <<= PAGE_SHIFT - page_shift;
+
+	return count;
+}
+
+static inline size_t mtr_kmem_direct_size(bool is_direct, size_t alloc_size,
+					  unsigned int page_shift)
+{
+	if (is_direct)
+		return ALIGN(alloc_size, 1 << page_shift);
+	else
+		return HNS_HW_DIRECT_PAGE_COUNT << page_shift;
+}
+
+/*
+ * check the given pages in continuous address space
+ * Returns 0 on success, or the error page num.
+ */
+static inline int mtr_check_direct_pages(dma_addr_t *pages, int page_count,
+					 unsigned int page_shift)
+{
+	size_t page_size = 1 << page_shift;
+	int i;
+
+	for (i = 1; i < page_count; i++)
+		if (pages[i] - pages[i - 1] != page_size)
+			return i;
+
+	return 0;
+}
+
+static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
+{
+	/* release user buffers */
+	if (mtr->umem) {
+		ib_umem_release(mtr->umem);
+		mtr->umem = NULL;
+	}
+
+	/* release kernel buffers */
+	if (mtr->kmem) {
+		hns_roce_buf_free(hr_dev, mtr->kmem);
+		kfree(mtr->kmem);
+		mtr->kmem = NULL;
+	}
+}
+
+static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+			  struct hns_roce_buf_attr *buf_attr, bool is_direct,
+			  struct ib_udata *udata, unsigned long user_addr)
+{
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	unsigned int max_pg_shift = buf_attr->page_shift;
+	unsigned int best_pg_shift = 0;
+	int all_pg_count = 0;
+	size_t direct_size;
+	size_t total_size;
+	unsigned long tmp;
+	int ret = 0;
+
+	total_size = mtr_bufs_size(buf_attr);
+	if (total_size < 1) {
+		ibdev_err(ibdev, "Failed to check mtr size\n");
+		return -EINVAL;
+	}
+
+	if (udata) {
+		mtr->kmem = NULL;
+		mtr->umem = ib_umem_get(ibdev, user_addr, total_size,
+					buf_attr->user_access);
+		if (IS_ERR_OR_NULL(mtr->umem)) {
+			ibdev_err(ibdev, "Failed to get umem, ret %ld\n",
+				  PTR_ERR(mtr->umem));
+			return -ENOMEM;
+		}
+		if (buf_attr->fixed_page) {
+			best_pg_shift = max_pg_shift;
+		} else {
+			tmp = GENMASK(max_pg_shift, 0);
+			ret = ib_umem_find_best_pgsz(mtr->umem, tmp, user_addr);
+			best_pg_shift = (ret <= PAGE_SIZE) ?
+					PAGE_SHIFT : ilog2(ret);
+		}
+		all_pg_count = mtr_umem_page_count(mtr->umem, best_pg_shift);
+		ret = 0;
+	} else {
+		mtr->umem = NULL;
+		mtr->kmem = kzalloc(sizeof(*mtr->kmem), GFP_KERNEL);
+		if (!mtr->kmem) {
+			ibdev_err(ibdev, "Failed to alloc kmem\n");
+			return -ENOMEM;
+		}
+		direct_size = mtr_kmem_direct_size(is_direct, total_size,
+						   max_pg_shift);
+		ret = hns_roce_buf_alloc(hr_dev, total_size, direct_size,
+					 mtr->kmem, max_pg_shift);
 		if (ret) {
-			dev_err(hr_dev->dev,
-				"write mtr[%d/%d] err %d,offset=%d.\n",
-				i, region_cnt, ret,  r->offset);
-			goto err_write;
+			ibdev_err(ibdev, "Failed to alloc kmem, ret %d\n", ret);
+			goto err_alloc_mem;
+		} else {
+			best_pg_shift = max_pg_shift;
+			all_pg_count = mtr->kmem->npages;
 		}
 	}
 
-	return 0;
+	/* must bigger than minimum hardware page shift */
+	if (best_pg_shift < HNS_HW_PAGE_SHIFT || all_pg_count < 1) {
+		ret = -EINVAL;
+		ibdev_err(ibdev, "Failed to check mtr page shift %d count %d\n",
+			  best_pg_shift, all_pg_count);
+		goto err_alloc_mem;
+	}
 
-err_write:
-	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
+	mtr->hem_cfg.buf_pg_shift = best_pg_shift;
+	mtr->hem_cfg.buf_pg_count = all_pg_count;
 
+	return 0;
+err_alloc_mem:
+	mtr_free_bufs(hr_dev, mtr);
 	return ret;
 }
 
+static int mtr_get_pages(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+			 dma_addr_t *pages, int count, unsigned int page_shift)
+{
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	int npage;
+	int err;
+
+	if (mtr->umem)
+		npage = hns_roce_get_umem_bufs(hr_dev, pages, count, 0,
+					       mtr->umem, page_shift);
+	else
+		npage = hns_roce_get_kmem_bufs(hr_dev, pages, count, 0,
+					       mtr->kmem);
+
+	if (mtr->hem_cfg.is_direct && npage > 1) {
+		err = mtr_check_direct_pages(pages, npage, page_shift);
+		if (err) {
+			ibdev_err(ibdev, "Failed to check %s direct page-%d\n",
+				  mtr->umem ? "user" : "kernel", err);
+			npage = err;
+		}
+	}
+
+	return npage;
+}
+
+int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+		     dma_addr_t *pages, int page_cnt)
+{
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_buf_region *r;
+	int err;
+	int i;
+
+	for (i = 0; i < mtr->hem_cfg.region_count; i++) {
+		r = &mtr->hem_cfg.region[i];
+		if (r->offset + r->count > page_cnt) {
+			err = -EINVAL;
+			ibdev_err(ibdev,
+				  "Failed to check mtr%d end %d + %d, max %d\n",
+				  i, r->offset, r->count, page_cnt);
+			return err;
+		}
+
+		err = mtr_map_region(hr_dev, mtr, &pages[r->offset], r);
+		if (err) {
+			ibdev_err(ibdev,
+				  "Failed to map mtr%d offset %d, err %d\n",
+				  i, r->offset, err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 		      int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr)
 {
-	u64 *mtts = mtt_buf;
 	int mtt_count;
 	int total = 0;
-	u64 *addr;
+	__le64 *mtts;
 	int npage;
+	u64 addr;
 	int left;
 
-	if (mtts == NULL || mtt_max < 1)
+	if (!mtt_buf || mtt_max < 1)
 		goto done;
 
+	/* no mtt memory in direct mode, so just return the buffer address */
+	if (mtr->hem_cfg.is_direct) {
+		npage = offset;
+		for (total = 0; total < mtt_max; total++, npage++) {
+			addr = mtr->hem_cfg.root_ba +
+			       (npage << mtr->hem_cfg.buf_pg_shift);
+
+			if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
+				mtt_buf[total] = to_hr_hw_page_addr(addr);
+			else
+				mtt_buf[total] = addr;
+		}
+
+		goto done;
+	}
+
 	left = mtt_max;
 	while (left > 0) {
 		mtt_count = 0;
-		addr = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
+		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
 						  offset + total,
 						  &mtt_count, NULL);
-		if (!addr || !mtt_count)
+		if (!mtts || !mtt_count)
 			goto done;
 
 		npage = min(mtt_count, left);
-		memcpy(&mtts[total], addr, BA_BYTE_LEN * npage);
 		left -= npage;
-		total += npage;
+		for (mtt_count = 0; mtt_count < npage; mtt_count++)
+			mtt_buf[total++] = le64_to_cpu(mtts[mtt_count]);
 	}
 
 done:
 	if (base_addr)
-		*base_addr = mtr->hem_list.root_ba;
+		*base_addr = mtr->hem_cfg.root_ba;
 
 	return total;
 }
+
+/* convert buffer size to page index and page count */
+static unsigned int mtr_init_region(struct hns_roce_buf_attr *attr,
+				    int page_cnt,
+				    struct hns_roce_buf_region *regions,
+				    int region_cnt, unsigned int page_shift)
+{
+	unsigned int page_size = 1 << page_shift;
+	int max_region = attr->region_count;
+	struct hns_roce_buf_region *r;
+	unsigned int i = 0;
+	int page_idx = 0;
+
+	for (; i < region_cnt && i < max_region && page_idx < page_cnt; i++) {
+		r = &regions[i];
+		r->hopnum = attr->region[i].hopnum == HNS_ROCE_HOP_NUM_0 ?
+			    0 : attr->region[i].hopnum;
+		r->offset = page_idx;
+		r->count = DIV_ROUND_UP(attr->region[i].size, page_size);
+		page_idx += r->count;
+	}
+
+	return i;
+}
+
+/**
+ * hns_roce_mtr_create - Create hns memory translate region.
+ *
+ * @mtr: memory translate region
+ * @init_attr: init attribute for creating mtr
+ * @page_shift: page shift for multi-hop base address table
+ * @udata: user space context, if it's NULL, means kernel space
+ * @user_addr: userspace virtual address to start at
+ * @buf_alloced: mtr has private buffer, true means need to alloc
+ */
+int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
+			struct hns_roce_buf_attr *buf_attr,
+			unsigned int page_shift, struct ib_udata *udata,
+			unsigned long user_addr)
+{
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	dma_addr_t *pages = NULL;
+	int region_cnt = 0;
+	int all_pg_cnt;
+	int get_pg_cnt;
+	bool has_mtt;
+	int err = 0;
+
+	has_mtt = mtr_has_mtt(buf_attr);
+	/* if buffer only need mtt, just init the hem cfg */
+	if (buf_attr->mtt_only) {
+		mtr->hem_cfg.buf_pg_shift = buf_attr->page_shift;
+		mtr->hem_cfg.buf_pg_count = mtr_bufs_size(buf_attr) >>
+					    buf_attr->page_shift;
+		mtr->umem = NULL;
+		mtr->kmem = NULL;
+	} else {
+		err = mtr_alloc_bufs(hr_dev, mtr, buf_attr, !has_mtt, udata,
+				     user_addr);
+		if (err) {
+			ibdev_err(ibdev, "Failed to alloc mtr bufs, err %d\n",
+				  err);
+			return err;
+		}
+	}
+
+	/* alloc mtt memory */
+	all_pg_cnt = mtr->hem_cfg.buf_pg_count;
+	hns_roce_hem_list_init(&mtr->hem_list);
+	mtr->hem_cfg.is_direct = !has_mtt;
+	mtr->hem_cfg.ba_pg_shift = page_shift;
+	mtr->hem_cfg.region_count = 0;
+	region_cnt = mtr_init_region(buf_attr, all_pg_cnt,
+				     mtr->hem_cfg.region,
+				     ARRAY_SIZE(mtr->hem_cfg.region),
+				     mtr->hem_cfg.buf_pg_shift);
+	if (region_cnt < 1) {
+		err = -ENOBUFS;
+		ibdev_err(ibdev, "failed to init mtr region %d\n", region_cnt);
+		goto err_alloc_bufs;
+	}
+
+	mtr->hem_cfg.region_count = region_cnt;
+
+	if (has_mtt) {
+		err = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,
+						mtr->hem_cfg.region, region_cnt,
+						page_shift);
+		if (err) {
+			ibdev_err(ibdev, "Failed to request mtr hem, err %d\n",
+				  err);
+			goto err_alloc_bufs;
+		}
+		mtr->hem_cfg.root_ba = mtr->hem_list.root_ba;
+	}
+
+	/* no buffer to map */
+	if (buf_attr->mtt_only)
+		return 0;
+
+	/* alloc a tmp array to store buffer's dma address */
+	pages = kvcalloc(all_pg_cnt, sizeof(dma_addr_t), GFP_KERNEL);
+	if (!pages) {
+		err = -ENOMEM;
+		ibdev_err(ibdev, "Failed to alloc mtr page list %d\n",
+			  all_pg_cnt);
+		goto err_alloc_hem_list;
+	}
+
+	get_pg_cnt = mtr_get_pages(hr_dev, mtr, pages, all_pg_cnt,
+				   mtr->hem_cfg.buf_pg_shift);
+	if (get_pg_cnt != all_pg_cnt) {
+		ibdev_err(ibdev, "Failed to get mtr page %d != %d\n",
+			  get_pg_cnt, all_pg_cnt);
+		err = -ENOBUFS;
+		goto err_alloc_page_list;
+	}
+
+	if (!has_mtt) {
+		mtr->hem_cfg.root_ba = pages[0];
+	} else {
+		/* write buffer's dma address to BA table */
+		err = hns_roce_mtr_map(hr_dev, mtr, pages, all_pg_cnt);
+		if (err) {
+			ibdev_err(ibdev, "Failed to map mtr pages, err %d\n",
+				  err);
+			goto err_alloc_page_list;
+		}
+	}
+
+	/* drop tmp array */
+	kvfree(pages);
+	return 0;
+err_alloc_page_list:
+	kvfree(pages);
+err_alloc_hem_list:
+	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
+err_alloc_bufs:
+	mtr_free_bufs(hr_dev, mtr);
+	return err;
+}
+
+void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
+{
+	/* release multi-hop addressing resource */
+	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
+
+	/* free buffers */
+	mtr_free_bufs(hr_dev, mtr);
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index 6317901c4b4f..a0a47bd66975 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -355,16 +355,16 @@ static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 	hns_roce_bitmap_free_range(&qp_table->bitmap, hr_qp->qpn, 1, BITMAP_RR);
 }
 
-static int set_rq_size(struct hns_roce_dev *hr_dev,
-				struct ib_qp_cap *cap, bool is_user, int has_rq,
-				struct hns_roce_qp *hr_qp)
+static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,
+		       struct hns_roce_qp *hr_qp, int has_rq)
 {
-	u32 max_cnt;
+	u32 cnt;
 
 	/* If srq exist, set zero for relative number of rq */
 	if (!has_rq) {
 		hr_qp->rq.wqe_cnt = 0;
 		hr_qp->rq.max_gs = 0;
+		hr_qp->rq_inl_buf.wqe_cnt = 0;
 		cap->max_recv_wr = 0;
 		cap->max_recv_sge = 0;
 
@@ -379,17 +379,15 @@ static int set_rq_size(struct hns_roce_dev *hr_dev,
 		return -EINVAL;
 	}
 
-	max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes);
-
-	hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt);
-	if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) {
+	cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes));
+	if (cnt > hr_dev->caps.max_wqes) {
 		ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n",
 			  cap->max_recv_wr);
 		return -EINVAL;
 	}
 
-	max_cnt = max(1U, cap->max_recv_sge);
-	hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt);
+	hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) +
+					      HNS_ROCE_RESERVED_SGE);
 
 	if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE)
 		hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz);
@@ -397,8 +395,57 @@ static int set_rq_size(struct hns_roce_dev *hr_dev,
 		hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz *
 					    hr_qp->rq.max_gs);
 
-	cap->max_recv_wr = hr_qp->rq.wqe_cnt;
-	cap->max_recv_sge = hr_qp->rq.max_gs;
+	hr_qp->rq.wqe_cnt = cnt;
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE)
+		hr_qp->rq_inl_buf.wqe_cnt = cnt;
+	else
+		hr_qp->rq_inl_buf.wqe_cnt = 0;
+
+	cap->max_recv_wr = cnt;
+	cap->max_recv_sge = hr_qp->rq.max_gs - HNS_ROCE_RESERVED_SGE;
+
+	return 0;
+}
+
+static int set_extend_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
+				struct hns_roce_qp *hr_qp,
+				struct ib_qp_cap *cap)
+{
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	u32 cnt;
+
+	cnt = max(1U, cap->max_send_sge);
+	if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
+		hr_qp->sq.max_gs = roundup_pow_of_two(cnt);
+		hr_qp->sge.sge_cnt = 0;
+
+		return 0;
+	}
+
+	hr_qp->sq.max_gs = cnt;
+
+	/* UD sqwqe's sge use extend sge */
+	if (hr_qp->ibqp.qp_type == IB_QPT_GSI ||
+	    hr_qp->ibqp.qp_type == IB_QPT_UD) {
+		cnt = roundup_pow_of_two(sq_wqe_cnt * hr_qp->sq.max_gs);
+	} else if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) {
+		cnt = roundup_pow_of_two(sq_wqe_cnt *
+				     (hr_qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE));
+
+		if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_A) {
+			if (cnt > hr_dev->caps.max_extend_sg) {
+				ibdev_err(ibdev,
+					  "failed to check exSGE num, exSGE num = %d.\n",
+					  cnt);
+				return -EINVAL;
+			}
+		}
+	} else {
+		cnt = 0;
+	}
+
+	hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
+	hr_qp->sge.sge_cnt = cnt;
 
 	return 0;
 }
@@ -430,174 +477,79 @@ static int set_user_sq_size(struct hns_roce_dev *hr_dev,
 			    struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp,
 			    struct hns_roce_ib_create_qp *ucmd)
 {
-	u32 ex_sge_num;
-	u32 page_size;
-	u32 max_cnt;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	u32 cnt = 0;
 	int ret;
 
-	if (check_shl_overflow(1, ucmd->log_sq_bb_count, &hr_qp->sq.wqe_cnt) ||
-	    hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes)
+	if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) ||
+	    cnt > hr_dev->caps.max_wqes)
 		return -EINVAL;
 
 	ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
 	if (ret) {
-		ibdev_err(&hr_dev->ib_dev, "Failed to check user SQ size limit\n");
+		ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n",
+			  ret);
 		return ret;
 	}
 
-	hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
-
-	max_cnt = max(1U, cap->max_send_sge);
-	if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
-		hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt);
-	else
-		hr_qp->sq.max_gs = max_cnt;
-
-	if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE)
-		hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
-							(hr_qp->sq.max_gs - 2));
-
-	if (hr_qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE &&
-	    hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_A) {
-		if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) {
-			ibdev_err(&hr_dev->ib_dev,
-				  "Failed to check extended SGE size limit %d\n",
-				  hr_qp->sge.sge_cnt);
-			return -EINVAL;
-		}
-	}
-
-	hr_qp->sge.sge_shift = 4;
-	ex_sge_num = hr_qp->sge.sge_cnt;
+	ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap);
+	if (ret)
+		return ret;
 
-	/* Get buf size, SQ and RQ  are aligned to page_szie */
-	if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
-		hr_qp->buff_size = round_up((hr_qp->rq.wqe_cnt <<
-					     hr_qp->rq.wqe_shift), PAGE_SIZE) +
-				   round_up((hr_qp->sq.wqe_cnt <<
-					     hr_qp->sq.wqe_shift), PAGE_SIZE);
-
-		hr_qp->sq.offset = 0;
-		hr_qp->rq.offset = round_up((hr_qp->sq.wqe_cnt <<
-					     hr_qp->sq.wqe_shift), PAGE_SIZE);
-	} else {
-		page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
-		hr_qp->sge.sge_cnt = ex_sge_num ?
-		   max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0;
-		hr_qp->buff_size = round_up((hr_qp->rq.wqe_cnt <<
-					     hr_qp->rq.wqe_shift), page_size) +
-				   round_up((hr_qp->sge.sge_cnt <<
-					     hr_qp->sge.sge_shift), page_size) +
-				   round_up((hr_qp->sq.wqe_cnt <<
-					     hr_qp->sq.wqe_shift), page_size);
-
-		hr_qp->sq.offset = 0;
-		if (ex_sge_num) {
-			hr_qp->sge.offset = round_up((hr_qp->sq.wqe_cnt <<
-						      hr_qp->sq.wqe_shift),
-						     page_size);
-			hr_qp->rq.offset = hr_qp->sge.offset +
-					   round_up((hr_qp->sge.sge_cnt <<
-						     hr_qp->sge.sge_shift),
-						    page_size);
-		} else {
-			hr_qp->rq.offset = round_up((hr_qp->sq.wqe_cnt <<
-						     hr_qp->sq.wqe_shift),
-						    page_size);
-		}
-	}
+	hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
+	hr_qp->sq.wqe_cnt = cnt;
 
 	return 0;
 }
 
-static int split_wqe_buf_region(struct hns_roce_dev *hr_dev,
-				struct hns_roce_qp *hr_qp,
-				struct hns_roce_buf_region *regions,
-				int region_max, int page_shift)
+static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
+			    struct hns_roce_qp *hr_qp,
+			    struct hns_roce_buf_attr *buf_attr)
 {
-	int page_size = 1 << page_shift;
-	bool is_extend_sge;
-	int region_cnt = 0;
 	int buf_size;
-	int buf_cnt;
+	int idx = 0;
 
-	if (hr_qp->buff_size < 1 || region_max < 1)
-		return region_cnt;
+	hr_qp->buff_size = 0;
 
-	if (hr_qp->sge.sge_cnt > 0)
-		is_extend_sge = true;
-	else
-		is_extend_sge = false;
-
-	/* sq region */
-	if (is_extend_sge)
-		buf_size = hr_qp->sge.offset - hr_qp->sq.offset;
-	else
-		buf_size = hr_qp->rq.offset - hr_qp->sq.offset;
-
-	if (buf_size > 0 && region_cnt < region_max) {
-		buf_cnt = DIV_ROUND_UP(buf_size, page_size);
-		hns_roce_init_buf_region(&regions[region_cnt],
-					 hr_dev->caps.wqe_sq_hop_num,
-					 hr_qp->sq.offset / page_size,
-					 buf_cnt);
-		region_cnt++;
-	}
-
-	/* sge region */
-	if (is_extend_sge) {
-		buf_size = hr_qp->rq.offset - hr_qp->sge.offset;
-		if (buf_size > 0 && region_cnt < region_max) {
-			buf_cnt = DIV_ROUND_UP(buf_size, page_size);
-			hns_roce_init_buf_region(&regions[region_cnt],
-						 hr_dev->caps.wqe_sge_hop_num,
-						 hr_qp->sge.offset / page_size,
-						 buf_cnt);
-			region_cnt++;
-		}
-	}
-
-	/* rq region */
-	buf_size = hr_qp->buff_size - hr_qp->rq.offset;
-	if (buf_size > 0) {
-		buf_cnt = DIV_ROUND_UP(buf_size, page_size);
-		hns_roce_init_buf_region(&regions[region_cnt],
-					 hr_dev->caps.wqe_rq_hop_num,
-					 hr_qp->rq.offset / page_size,
-					 buf_cnt);
-		region_cnt++;
-	}
-
-	return region_cnt;
-}
-
-static int set_extend_sge_param(struct hns_roce_dev *hr_dev,
-				struct hns_roce_qp *hr_qp)
-{
-	struct device *dev = hr_dev->dev;
-
-	if (hr_qp->sq.max_gs > 2) {
-		hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
-				     (hr_qp->sq.max_gs - 2));
-		hr_qp->sge.sge_shift = 4;
-	}
-
-	/* ud sqwqe's sge use extend sge */
-	if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 &&
-	    hr_qp->ibqp.qp_type == IB_QPT_GSI) {
-		hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
-				     hr_qp->sq.max_gs);
-		hr_qp->sge.sge_shift = 4;
-	}
+	/* SQ WQE */
+	hr_qp->sq.offset = 0;
+	buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt,
+					  hr_qp->sq.wqe_shift);
+	if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
+		buf_attr->region[idx].size = buf_size;
+		buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num;
+		idx++;
+		hr_qp->buff_size += buf_size;
+	}
+
+	/* extend SGE WQE in SQ */
+	hr_qp->sge.offset = hr_qp->buff_size;
+	buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt,
+					  hr_qp->sge.sge_shift);
+	if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
+		buf_attr->region[idx].size = buf_size;
+		buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num;
+		idx++;
+		hr_qp->buff_size += buf_size;
+	}
+
+	/* RQ WQE */
+	hr_qp->rq.offset = hr_qp->buff_size;
+	buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt,
+					  hr_qp->rq.wqe_shift);
+	if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
+		buf_attr->region[idx].size = buf_size;
+		buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num;
+		idx++;
+		hr_qp->buff_size += buf_size;
+	}
+
+	if (hr_qp->buff_size < 1)
+		return -EINVAL;
 
-	if (hr_qp->sq.max_gs > 2 &&
-	    hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08_A) {
-		if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) {
-			dev_err(dev, "The extended sge cnt error! sge_cnt=%d\n",
-				hr_qp->sge.sge_cnt);
-			return -EINVAL;
-		}
-	}
+	buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
+	buf_attr->fixed_page = true;
+	buf_attr->region_count = idx;
 
 	return 0;
 }
@@ -605,62 +557,35 @@ static int set_extend_sge_param(struct hns_roce_dev *hr_dev,
 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev,
 			      struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp)
 {
-	u32 page_size;
-	u32 max_cnt;
-	int size;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	u32 cnt;
 	int ret;
 
 	if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes ||
 	    cap->max_send_sge > hr_dev->caps.max_sq_sg ||
 	    cap->max_inline_data > hr_dev->caps.max_sq_inline) {
-		ibdev_err(&hr_dev->ib_dev,
-			  "SQ WR or sge or inline data error!\n");
+		ibdev_err(ibdev,
+			  "failed to check SQ WR, SGE or inline num, ret = %d.\n",
+			  -EINVAL);
 		return -EINVAL;
 	}
 
-	hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
-
-	max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes);
-
-	hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt);
-	if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) {
-		ibdev_err(&hr_dev->ib_dev,
-			  "while setting kernel sq size, sq.wqe_cnt too large\n");
+	cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes));
+	if (cnt > hr_dev->caps.max_wqes) {
+		ibdev_err(ibdev, "failed to check WQE num, WQE num = %d.\n",
+			  cnt);
 		return -EINVAL;
 	}
 
-	/* Get data_seg numbers */
-	max_cnt = max(1U, cap->max_send_sge);
-	if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
-		hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt);
-	else
-		hr_qp->sq.max_gs = max_cnt;
+	hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
+	hr_qp->sq.wqe_cnt = cnt;
 
-	ret = set_extend_sge_param(hr_dev, hr_qp);
-	if (ret) {
-		ibdev_err(&hr_dev->ib_dev, "set extend sge parameters fail\n");
+	ret = set_extend_sge_param(hr_dev, cnt, hr_qp, cap);
+	if (ret)
 		return ret;
-	}
 
-	/* Get buf size, SQ and RQ are aligned to PAGE_SIZE */
-	page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
-	hr_qp->sq.offset = 0;
-	size = round_up(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift, page_size);
-
-	if (hr_dev->hw_rev != HNS_ROCE_HW_VER1 && hr_qp->sge.sge_cnt) {
-		hr_qp->sge.sge_cnt = max(page_size/(1 << hr_qp->sge.sge_shift),
-					 (u32)hr_qp->sge.sge_cnt);
-		hr_qp->sge.offset = size;
-		size += round_up(hr_qp->sge.sge_cnt << hr_qp->sge.sge_shift,
-				 page_size);
-	}
-
-	hr_qp->rq.offset = size;
-	size += round_up((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), page_size);
-	hr_qp->buff_size = size;
-
-	/* Get wr and sge number which send */
-	cap->max_send_wr = hr_qp->sq.wqe_cnt;
+	/* sync the parameters of kernel QP to user's configuration */
+	cap->max_send_wr = cnt;
 	cap->max_send_sge = hr_qp->sq.max_gs;
 
 	/* We don't support inline sends for kernel QPs (yet) */
@@ -691,8 +616,8 @@ static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
 			       struct ib_qp_init_attr *init_attr)
 {
 	u32 max_recv_sge = init_attr->cap.max_recv_sge;
+	u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt;
 	struct hns_roce_rinl_wqe *wqe_list;
-	u32 wqe_cnt = hr_qp->rq.wqe_cnt;
 	int i;
 
 	/* allocate recv inline buf */
@@ -714,7 +639,6 @@ static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
 		wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
 
 	hr_qp->rq_inl_buf.wqe_list = wqe_list;
-	hr_qp->rq_inl_buf.wqe_cnt = wqe_cnt;
 
 	return 0;
 
@@ -727,140 +651,55 @@ err:
 
 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
 {
-	kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
+	if (hr_qp->rq_inl_buf.wqe_list)
+		kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
 	kfree(hr_qp->rq_inl_buf.wqe_list);
 }
 
-static int map_wqe_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
-		       u32 page_shift, bool is_user)
-{
-/* WQE buffer include 3 parts: SQ, extend SGE and RQ. */
-#define HNS_ROCE_WQE_REGION_MAX	 3
-	struct hns_roce_buf_region regions[HNS_ROCE_WQE_REGION_MAX] = {};
-	dma_addr_t *buf_list[HNS_ROCE_WQE_REGION_MAX] = {};
-	struct ib_device *ibdev = &hr_dev->ib_dev;
-	struct hns_roce_buf_region *r;
-	int region_count;
-	int buf_count;
-	int ret;
-	int i;
-
-	region_count = split_wqe_buf_region(hr_dev, hr_qp, regions,
-					    ARRAY_SIZE(regions), page_shift);
-
-	/* alloc a tmp list to store WQE buffers address */
-	ret = hns_roce_alloc_buf_list(regions, buf_list, region_count);
-	if (ret) {
-		ibdev_err(ibdev, "Failed to alloc WQE buffer list\n");
-		return ret;
-	}
-
-	for (i = 0; i < region_count; i++) {
-		r = &regions[i];
-		if (is_user)
-			buf_count = hns_roce_get_umem_bufs(hr_dev, buf_list[i],
-					r->count, r->offset, hr_qp->umem,
-					page_shift);
-		else
-			buf_count = hns_roce_get_kmem_bufs(hr_dev, buf_list[i],
-					r->count, r->offset, &hr_qp->hr_buf);
-
-		if (buf_count != r->count) {
-			ibdev_err(ibdev, "Failed to get %s WQE buf, expect %d = %d.\n",
-				  is_user ? "user" : "kernel",
-				  r->count, buf_count);
-			ret = -ENOBUFS;
-			goto done;
-		}
-	}
-
-	hr_qp->wqe_bt_pg_shift = hr_dev->caps.mtt_ba_pg_sz;
-	hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift,
-			  page_shift);
-	ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, regions,
-				  region_count);
-	if (ret)
-		ibdev_err(ibdev, "Failed to attach WQE's mtr\n");
-
-	goto done;
-
-	hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
-done:
-	hns_roce_free_buf_list(buf_list, region_count);
-
-	return ret;
-}
-
 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 			struct ib_qp_init_attr *init_attr,
 			struct ib_udata *udata, unsigned long addr)
 {
-	u32 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
 	struct ib_device *ibdev = &hr_dev->ib_dev;
-	bool is_rq_buf_inline;
+	struct hns_roce_buf_attr buf_attr = {};
 	int ret;
 
-	is_rq_buf_inline = (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
-			   hns_roce_qp_has_rq(init_attr);
-	if (is_rq_buf_inline) {
+	if (!udata && hr_qp->rq_inl_buf.wqe_cnt) {
 		ret = alloc_rq_inline_buf(hr_qp, init_attr);
 		if (ret) {
-			ibdev_err(ibdev, "Failed to alloc inline RQ buffer\n");
+			ibdev_err(ibdev,
+				  "failed to alloc inline buf, ret = %d.\n",
+				  ret);
 			return ret;
 		}
-	}
-
-	if (udata) {
-		hr_qp->umem = ib_umem_get(ibdev, addr, hr_qp->buff_size, 0);
-		if (IS_ERR(hr_qp->umem)) {
-			ret = PTR_ERR(hr_qp->umem);
-			goto err_inline;
-		}
 	} else {
-		ret = hns_roce_buf_alloc(hr_dev, hr_qp->buff_size,
-					 (1 << page_shift) * 2,
-					 &hr_qp->hr_buf, page_shift);
-		if (ret)
-			goto err_inline;
+		hr_qp->rq_inl_buf.wqe_list = NULL;
 	}
 
-	ret = map_wqe_buf(hr_dev, hr_qp, page_shift, udata);
-	if (ret)
-		goto err_alloc;
+	ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
+	if (ret) {
+		ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
+		goto err_inline;
+	}
+	ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
+				  HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
+				  udata, addr);
+	if (ret) {
+		ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
+		goto err_inline;
+	}
 
 	return 0;
-
 err_inline:
-	if (is_rq_buf_inline)
-		free_rq_inline_buf(hr_qp);
-
-err_alloc:
-	if (udata) {
-		ib_umem_release(hr_qp->umem);
-		hr_qp->umem = NULL;
-	} else {
-		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
-	}
-
-	ibdev_err(ibdev, "Failed to alloc WQE buffer, ret %d.\n", ret);
+	free_rq_inline_buf(hr_qp);
 
 	return ret;
 }
 
 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 {
-	hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
-	if (hr_qp->umem) {
-		ib_umem_release(hr_qp->umem);
-		hr_qp->umem = NULL;
-	}
-
-	if (hr_qp->hr_buf.nbufs > 0)
-		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
-
-	if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
-	     hr_qp->rq.wqe_cnt)
-		free_rq_inline_buf(hr_qp);
+	hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
+	free_rq_inline_buf(hr_qp);
 }
 
 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
@@ -912,8 +751,8 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 					  "Failed to map user SQ doorbell\n");
 				goto err_out;
 			}
-			hr_qp->sdb_en = 1;
-			resp->cap_flags |= HNS_ROCE_SUPPORT_SQ_RECORD_DB;
+			hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
+			resp->cap_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
 		}
 
 		if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) {
@@ -924,8 +763,8 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 					  "Failed to map user RQ doorbell\n");
 				goto err_sdb;
 			}
-			hr_qp->rdb_en = 1;
-			resp->cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB;
+			hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
+			resp->cap_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
 		}
 	} else {
 		/* QP doorbell register address */
@@ -942,13 +781,13 @@ static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 				goto err_out;
 			}
 			*hr_qp->rdb.db_record = 0;
-			hr_qp->rdb_en = 1;
+			hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
 		}
 	}
 
 	return 0;
 err_sdb:
-	if (udata && hr_qp->sdb_en)
+	if (udata && hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
 		hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
 err_out:
 	return ret;
@@ -961,12 +800,12 @@ static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 		udata, struct hns_roce_ucontext, ibucontext);
 
 	if (udata) {
-		if (hr_qp->rdb_en)
+		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 			hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
-		if (hr_qp->sdb_en)
+		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
 			hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
 	} else {
-		if (hr_qp->rdb_en)
+		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 			hns_roce_free_db(hr_dev, &hr_qp->rdb);
 	}
 }
@@ -1003,8 +842,7 @@ err_sq:
 	return ret;
 }
 
-static void free_kernel_wrid(struct hns_roce_dev *hr_dev,
-			     struct hns_roce_qp *hr_qp)
+static void free_kernel_wrid(struct hns_roce_qp *hr_qp)
 {
 	kfree(hr_qp->rq.wrid);
 	kfree(hr_qp->sq.wrid);
@@ -1025,10 +863,11 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 	else
 		hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
 
-	ret = set_rq_size(hr_dev, &init_attr->cap, udata,
-			  hns_roce_qp_has_rq(init_attr), hr_qp);
+	ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp,
+			  hns_roce_qp_has_rq(init_attr));
 	if (ret) {
-		ibdev_err(ibdev, "Failed to set user RQ size\n");
+		ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n",
+			  ret);
 		return ret;
 	}
 
@@ -1156,7 +995,7 @@ err_buf:
 err_db:
 	free_qp_db(hr_dev, hr_qp, udata);
 err_wrid:
-	free_kernel_wrid(hr_dev, hr_qp);
+	free_kernel_wrid(hr_qp);
 	return ret;
 }
 
@@ -1170,7 +1009,7 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 	free_qpc(hr_dev, hr_qp);
 	free_qpn(hr_dev, hr_qp);
 	free_qp_buf(hr_dev, hr_qp);
-	free_kernel_wrid(hr_dev, hr_qp);
+	free_kernel_wrid(hr_qp);
 	free_qp_db(hr_dev, hr_qp, udata);
 
 	kfree(hr_qp);
@@ -1339,10 +1178,10 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 
 	if (ibqp->uobject &&
 	    (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
-		if (hr_qp->sdb_en == 1) {
+		if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) {
 			hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
 
-			if (hr_qp->rdb_en == 1)
+			if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
 				hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
 		} else {
 			ibdev_warn(&hr_dev->ib_dev,
@@ -1431,10 +1270,9 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
 	}
 }
 
-static void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
+static inline void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
 {
-
-	return hns_roce_buf_offset(&hr_qp->hr_buf, offset);
+	return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
 }
 
 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, int n)
@@ -1449,8 +1287,7 @@ void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, int n)
 
 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, int n)
 {
-	return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset +
-					(n << hr_qp->sge.sge_shift));
+	return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift));
 }
 
 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index 5b3dd1a337d4..f40a000e94ee 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -77,56 +77,56 @@ static int hns_roce_hw_destroy_srq(struct hns_roce_dev *dev,
 				 HNS_ROCE_CMD_TIMEOUT_MSECS);
 }
 
-static int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn,
-			      u16 xrcd, struct hns_roce_mtt *hr_mtt,
-			      u64 db_rec_addr, struct hns_roce_srq *srq)
+static int alloc_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
+		      u32 pdn, u32 cqn, u16 xrcd, u64 db_rec_addr)
 {
 	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
 	struct hns_roce_cmd_mailbox *mailbox;
-	dma_addr_t dma_handle_wqe;
-	dma_addr_t dma_handle_idx;
-	u64 *mtts_wqe;
-	u64 *mtts_idx;
+	u64 mtts_wqe[MTT_MIN_COUNT] = { 0 };
+	u64 mtts_idx[MTT_MIN_COUNT] = { 0 };
+	dma_addr_t dma_handle_wqe = 0;
+	dma_addr_t dma_handle_idx = 0;
 	int ret;
 
 	/* Get the physical address of srq buf */
-	mtts_wqe = hns_roce_table_find(hr_dev,
-				       &hr_dev->mr_table.mtt_srqwqe_table,
-				       srq->mtt.first_seg,
-				       &dma_handle_wqe);
-	if (!mtts_wqe) {
-		dev_err(hr_dev->dev, "Failed to find mtt for srq buf.\n");
-		return -EINVAL;
+	ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,
+				ARRAY_SIZE(mtts_wqe), &dma_handle_wqe);
+	if (ret < 1) {
+		ibdev_err(ibdev, "Failed to find mtr for SRQ WQE\n");
+		return -ENOBUFS;
 	}
 
 	/* Get physical address of idx que buf */
-	mtts_idx = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_idx_table,
-				       srq->idx_que.mtt.first_seg,
-				       &dma_handle_idx);
-	if (!mtts_idx) {
-		dev_err(hr_dev->dev,
-			"Failed to find mtt for srq idx queue buf.\n");
-		return -EINVAL;
+	ret = hns_roce_mtr_find(hr_dev, &srq->idx_que.mtr, 0, mtts_idx,
+				ARRAY_SIZE(mtts_idx), &dma_handle_idx);
+	if (ret < 1) {
+		ibdev_err(ibdev, "Failed to find mtr for SRQ idx\n");
+		return -ENOBUFS;
 	}
 
 	ret = hns_roce_bitmap_alloc(&srq_table->bitmap, &srq->srqn);
 	if (ret) {
-		dev_err(hr_dev->dev,
-			"Failed to alloc a bit from srq bitmap.\n");
+		ibdev_err(ibdev, "Failed to alloc SRQ number, err %d\n", ret);
 		return -ENOMEM;
 	}
 
 	ret = hns_roce_table_get(hr_dev, &srq_table->table, srq->srqn);
-	if (ret)
+	if (ret) {
+		ibdev_err(ibdev, "Failed to get SRQC table, err %d\n", ret);
 		goto err_out;
+	}
 
 	ret = xa_err(xa_store(&srq_table->xa, srq->srqn, srq, GFP_KERNEL));
-	if (ret)
+	if (ret) {
+		ibdev_err(ibdev, "Failed to store SRQC, err %d\n", ret);
 		goto err_put;
+	}
 
 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
-	if (IS_ERR(mailbox)) {
-		ret = PTR_ERR(mailbox);
+	if (IS_ERR_OR_NULL(mailbox)) {
+		ret = -ENOMEM;
+		ibdev_err(ibdev, "Failed to alloc mailbox for SRQC\n");
 		goto err_xa;
 	}
 
@@ -136,8 +136,10 @@ static int hns_roce_srq_alloc(struct hns_roce_dev *hr_dev, u32 pdn, u32 cqn,
 
 	ret = hns_roce_hw_create_srq(hr_dev, mailbox, srq->srqn);
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
-	if (ret)
+	if (ret) {
+		ibdev_err(ibdev, "Failed to config SRQC, err %d\n", ret);
 		goto err_xa;
+	}
 
 	atomic_set(&srq->refcount, 1);
 	init_completion(&srq->free);
@@ -154,8 +156,7 @@ err_out:
 	return ret;
 }
 
-static void hns_roce_srq_free(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_srq *srq)
+static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
 {
 	struct hns_roce_srq_table *srq_table = &hr_dev->srq_table;
 	int ret;
@@ -175,187 +176,104 @@ static void hns_roce_srq_free(struct hns_roce_dev *hr_dev,
 	hns_roce_bitmap_free(&srq_table->bitmap, srq->srqn, BITMAP_NO_RR);
 }
 
-static int create_user_srq(struct hns_roce_srq *srq, struct ib_udata *udata,
-			   int srq_buf_size)
+static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
+			 struct ib_udata *udata, unsigned long addr)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
-	struct hns_roce_ib_create_srq  ucmd;
-	struct hns_roce_buf *buf;
-	int ret;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_buf_attr buf_attr = {};
+	int err;
+
+	srq->wqe_shift = ilog2(roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE,
+						      HNS_ROCE_SGE_SIZE *
+						      srq->max_gs)));
+
+	buf_attr.page_shift = hr_dev->caps.srqwqe_buf_pg_sz + HNS_HW_PAGE_SHIFT;
+	buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt,
+							 srq->wqe_shift);
+	buf_attr.region[0].hopnum = hr_dev->caps.srqwqe_hop_num;
+	buf_attr.region_count = 1;
+	buf_attr.fixed_page = true;
+
+	err = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,
+				  hr_dev->caps.srqwqe_ba_pg_sz +
+				  HNS_HW_PAGE_SHIFT, udata, addr);
+	if (err)
+		ibdev_err(ibdev, "Failed to alloc SRQ buf mtr, err %d\n", err);
+
+	return err;
+}
 
-	if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
-		return -EFAULT;
-
-	srq->umem =
-		ib_umem_get(srq->ibsrq.device, ucmd.buf_addr, srq_buf_size, 0);
-	if (IS_ERR(srq->umem))
-		return PTR_ERR(srq->umem);
-
-	buf = &srq->buf;
-	buf->npages = (ib_umem_page_count(srq->umem) +
-		       (1 << hr_dev->caps.srqwqe_buf_pg_sz) - 1) /
-		      (1 << hr_dev->caps.srqwqe_buf_pg_sz);
-	buf->page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
-	ret = hns_roce_mtt_init(hr_dev, buf->npages, buf->page_shift,
-				&srq->mtt);
-	if (ret)
-		goto err_user_buf;
+static void free_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
+{
+	hns_roce_mtr_destroy(hr_dev, &srq->buf_mtr);
+}
 
-	ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->mtt, srq->umem);
-	if (ret)
-		goto err_user_srq_mtt;
-
-	/* config index queue BA */
-	srq->idx_que.umem = ib_umem_get(srq->ibsrq.device, ucmd.que_addr,
-					srq->idx_que.buf_size, 0);
-	if (IS_ERR(srq->idx_que.umem)) {
-		dev_err(hr_dev->dev, "ib_umem_get error for index queue\n");
-		ret = PTR_ERR(srq->idx_que.umem);
-		goto err_user_srq_mtt;
+static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
+			 struct ib_udata *udata, unsigned long addr)
+{
+	struct hns_roce_idx_que *idx_que = &srq->idx_que;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_buf_attr buf_attr = {};
+	int err;
+
+	srq->idx_que.entry_shift = ilog2(HNS_ROCE_IDX_QUE_ENTRY_SZ);
+
+	buf_attr.page_shift = hr_dev->caps.idx_buf_pg_sz + HNS_HW_PAGE_SHIFT;
+	buf_attr.region[0].size = to_hr_hem_entries_size(srq->wqe_cnt,
+					srq->idx_que.entry_shift);
+	buf_attr.region[0].hopnum = hr_dev->caps.idx_hop_num;
+	buf_attr.region_count = 1;
+	buf_attr.fixed_page = true;
+
+	err = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,
+				  hr_dev->caps.idx_ba_pg_sz + HNS_HW_PAGE_SHIFT,
+				  udata, addr);
+	if (err) {
+		ibdev_err(ibdev, "Failed to alloc SRQ idx mtr, err %d\n", err);
+		return err;
 	}
 
-	buf = &srq->idx_que.idx_buf;
-	buf->npages = DIV_ROUND_UP(ib_umem_page_count(srq->idx_que.umem),
-				   1 << hr_dev->caps.idx_buf_pg_sz);
-	buf->page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
-	ret = hns_roce_mtt_init(hr_dev, buf->npages, buf->page_shift,
-				&srq->idx_que.mtt);
-	if (ret) {
-		dev_err(hr_dev->dev, "hns_roce_mtt_init error for idx que\n");
-		goto err_user_idx_mtt;
-	}
+	if (!udata) {
+		idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL);
+		if (!idx_que->bitmap) {
+			ibdev_err(ibdev, "Failed to alloc SRQ idx bitmap\n");
+			err = -ENOMEM;
+			goto err_idx_mtr;
+		}
 
-	ret = hns_roce_ib_umem_write_mtt(hr_dev, &srq->idx_que.mtt,
-					 srq->idx_que.umem);
-	if (ret) {
-		dev_err(hr_dev->dev,
-			"hns_roce_ib_umem_write_mtt error for idx que\n");
-		goto err_user_idx_buf;
 	}
 
 	return 0;
+err_idx_mtr:
+	hns_roce_mtr_destroy(hr_dev, &idx_que->mtr);
 
-err_user_idx_buf:
-	hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
-
-err_user_idx_mtt:
-	ib_umem_release(srq->idx_que.umem);
-
-err_user_srq_mtt:
-	hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
-
-err_user_buf:
-	ib_umem_release(srq->umem);
-
-	return ret;
+	return err;
 }
 
-static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq,
-				   u32 page_shift)
+static void free_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
 	struct hns_roce_idx_que *idx_que = &srq->idx_que;
 
-	idx_que->bitmap = bitmap_zalloc(srq->wqe_cnt, GFP_KERNEL);
-	if (!idx_que->bitmap)
-		return -ENOMEM;
-
-	idx_que->buf_size = srq->idx_que.buf_size;
-
-	if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2,
-			       &idx_que->idx_buf, page_shift)) {
-		bitmap_free(idx_que->bitmap);
-		return -ENOMEM;
-	}
-
-	return 0;
+	bitmap_free(idx_que->bitmap);
+	idx_que->bitmap = NULL;
+	hns_roce_mtr_destroy(hr_dev, &idx_que->mtr);
 }
 
-static int create_kernel_srq(struct hns_roce_srq *srq, int srq_buf_size)
+static int alloc_srq_wrid(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
-	u32 page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
-	int ret;
-
-	if (hns_roce_buf_alloc(hr_dev, srq_buf_size, (1 << page_shift) * 2,
-			       &srq->buf, page_shift))
-		return -ENOMEM;
-
 	srq->head = 0;
 	srq->tail = srq->wqe_cnt - 1;
-
-	ret = hns_roce_mtt_init(hr_dev, srq->buf.npages, srq->buf.page_shift,
-				&srq->mtt);
-	if (ret)
-		goto err_kernel_buf;
-
-	ret = hns_roce_buf_write_mtt(hr_dev, &srq->mtt, &srq->buf);
-	if (ret)
-		goto err_kernel_srq_mtt;
-
-	page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
-	ret = hns_roce_create_idx_que(srq->ibsrq.pd, srq, page_shift);
-	if (ret) {
-		dev_err(hr_dev->dev, "Create idx queue fail(%d)!\n", ret);
-		goto err_kernel_srq_mtt;
-	}
-
-	/* Init mtt table for idx_que */
-	ret = hns_roce_mtt_init(hr_dev, srq->idx_que.idx_buf.npages,
-				srq->idx_que.idx_buf.page_shift,
-				&srq->idx_que.mtt);
-	if (ret)
-		goto err_kernel_create_idx;
-
-	/* Write buffer address into the mtt table */
-	ret = hns_roce_buf_write_mtt(hr_dev, &srq->idx_que.mtt,
-				     &srq->idx_que.idx_buf);
-	if (ret)
-		goto err_kernel_idx_buf;
-
 	srq->wrid = kvmalloc_array(srq->wqe_cnt, sizeof(u64), GFP_KERNEL);
-	if (!srq->wrid) {
-		ret = -ENOMEM;
-		goto err_kernel_idx_buf;
-	}
+	if (!srq->wrid)
+		return -ENOMEM;
 
 	return 0;
-
-err_kernel_idx_buf:
-	hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
-
-err_kernel_create_idx:
-	hns_roce_buf_free(hr_dev, srq->idx_que.buf_size,
-			  &srq->idx_que.idx_buf);
-	kfree(srq->idx_que.bitmap);
-
-err_kernel_srq_mtt:
-	hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
-
-err_kernel_buf:
-	hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf);
-
-	return ret;
-}
-
-static void destroy_user_srq(struct hns_roce_dev *hr_dev,
-			     struct hns_roce_srq *srq)
-{
-	hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
-	ib_umem_release(srq->idx_que.umem);
-	hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
-	ib_umem_release(srq->umem);
 }
 
-static void destroy_kernel_srq(struct hns_roce_dev *hr_dev,
-			       struct hns_roce_srq *srq, int srq_buf_size)
+static void free_srq_wrid(struct hns_roce_srq *srq)
 {
-	kvfree(srq->wrid);
-	hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
-	hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, &srq->idx_que.idx_buf);
-	kfree(srq->idx_que.bitmap);
-	hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
-	hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf);
+	kfree(srq->wrid);
+	srq->wrid = NULL;
 }
 
 int hns_roce_create_srq(struct ib_srq *ib_srq,
@@ -365,8 +283,8 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device);
 	struct hns_roce_ib_create_srq_resp resp = {};
 	struct hns_roce_srq *srq = to_hr_srq(ib_srq);
-	int srq_desc_size;
-	int srq_buf_size;
+	struct ib_device *ibdev = &hr_dev->ib_dev;
+	struct hns_roce_ib_create_srq ucmd = {};
 	int ret = 0;
 	u32 cqn;
 
@@ -379,43 +297,47 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
 	spin_lock_init(&srq->lock);
 
 	srq->wqe_cnt = roundup_pow_of_two(init_attr->attr.max_wr + 1);
-	srq->max_gs = init_attr->attr.max_sge;
-
-	srq_desc_size = roundup_pow_of_two(max(HNS_ROCE_SGE_SIZE,
-					HNS_ROCE_SGE_SIZE * srq->max_gs));
-
-	srq->wqe_shift = ilog2(srq_desc_size);
-
-	srq_buf_size = srq->wqe_cnt * srq_desc_size;
-
-	srq->idx_que.entry_sz = HNS_ROCE_IDX_QUE_ENTRY_SZ;
-	srq->idx_que.buf_size = srq->wqe_cnt * srq->idx_que.entry_sz;
-	srq->mtt.mtt_type = MTT_TYPE_SRQWQE;
-	srq->idx_que.mtt.mtt_type = MTT_TYPE_IDX;
+	srq->max_gs = init_attr->attr.max_sge + HNS_ROCE_RESERVED_SGE;
 
 	if (udata) {
-		ret = create_user_srq(srq, udata, srq_buf_size);
+		ret = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
 		if (ret) {
-			dev_err(hr_dev->dev, "Create user srq failed\n");
-			goto err_srq;
+			ibdev_err(ibdev, "Failed to copy SRQ udata, err %d\n",
+				  ret);
+			return ret;
 		}
-	} else {
-		ret = create_kernel_srq(srq, srq_buf_size);
+	}
+
+	ret = alloc_srq_buf(hr_dev, srq, udata, ucmd.buf_addr);
+	if (ret) {
+		ibdev_err(ibdev, "Failed to alloc SRQ buffer, err %d\n", ret);
+		return ret;
+	}
+
+	ret = alloc_srq_idx(hr_dev, srq, udata, ucmd.que_addr);
+	if (ret) {
+		ibdev_err(ibdev, "Failed to alloc SRQ idx, err %d\n", ret);
+		goto err_buf_alloc;
+	}
+
+	if (!udata) {
+		ret = alloc_srq_wrid(hr_dev, srq);
 		if (ret) {
-			dev_err(hr_dev->dev, "Create kernel srq failed\n");
-			goto err_srq;
+			ibdev_err(ibdev, "Failed to alloc SRQ wrid, err %d\n",
+				  ret);
+			goto err_idx_alloc;
 		}
 	}
 
 	cqn = ib_srq_has_cq(init_attr->srq_type) ?
 	      to_hr_cq(init_attr->ext.cq)->cqn : 0;
-
 	srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG;
 
-	ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(ib_srq->pd)->pdn, cqn, 0,
-				 &srq->mtt, 0, srq);
-	if (ret)
-		goto err_wrid;
+	ret = alloc_srqc(hr_dev, srq, to_hr_pd(ib_srq->pd)->pdn, cqn, 0, 0);
+	if (ret) {
+		ibdev_err(ibdev, "Failed to alloc SRQ context, err %d\n", ret);
+		goto err_wrid_alloc;
+	}
 
 	srq->event = hns_roce_ib_srq_event;
 	resp.srqn = srq->srqn;
@@ -431,15 +353,13 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
 	return 0;
 
 err_srqc_alloc:
-	hns_roce_srq_free(hr_dev, srq);
-
-err_wrid:
-	if (udata)
-		destroy_user_srq(hr_dev, srq);
-	else
-		destroy_kernel_srq(hr_dev, srq, srq_buf_size);
-
-err_srq:
+	free_srqc(hr_dev, srq);
+err_wrid_alloc:
+	free_srq_wrid(srq);
+err_idx_alloc:
+	free_srq_idx(hr_dev, srq);
+err_buf_alloc:
+	free_srq_buf(hr_dev, srq);
 	return ret;
 }
 
@@ -448,18 +368,10 @@ void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
 	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
 
-	hns_roce_srq_free(hr_dev, srq);
-	hns_roce_mtt_cleanup(hr_dev, &srq->mtt);
-
-	if (udata) {
-		hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt);
-	} else {
-		kvfree(srq->wrid);
-		hns_roce_buf_free(hr_dev, srq->wqe_cnt << srq->wqe_shift,
-				  &srq->buf);
-	}
-	ib_umem_release(srq->idx_que.umem);
-	ib_umem_release(srq->umem);
+	free_srqc(hr_dev, srq);
+	free_srq_idx(hr_dev, srq);
+	free_srq_wrid(srq);
+	free_srq_buf(hr_dev, srq);
 }
 
 int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/i40iw/Kconfig b/drivers/infiniband/hw/i40iw/Kconfig
index e4b45f4cd8f8..7476f3b14c39 100644
--- a/drivers/infiniband/hw/i40iw/Kconfig
+++ b/drivers/infiniband/hw/i40iw/Kconfig
@@ -5,5 +5,5 @@ config INFINIBAND_I40IW
 	depends on IPV6 || !IPV6
 	depends on PCI
 	select GENERIC_ALLOCATOR
-	---help---
+	help
 	Intel(R) Ethernet X722 iWARP Driver
diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h
index 3c62c9327a9c..49d92638e0db 100644
--- a/drivers/infiniband/hw/i40iw/i40iw.h
+++ b/drivers/infiniband/hw/i40iw/i40iw.h
@@ -382,15 +382,6 @@ static inline struct i40iw_mr *to_iwmr(struct ib_mr *ibmr)
 }
 
 /**
- * to_iwmr_from_ibfmr - get device memory region
- * @ibfmr: ib fmr
- **/
-static inline struct i40iw_mr *to_iwmr_from_ibfmr(struct ib_fmr *ibfmr)
-{
-	return container_of(ibfmr, struct i40iw_mr, ibfmr);
-}
-
-/**
  * to_iwmw - get device memory window
  * @ibmw: ib memory window
  **/
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 1b6fb1380961..19af29a48c55 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -83,7 +83,6 @@ static int i40iw_query_device(struct ib_device *ibdev,
 	props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE;
 	props->max_qp_init_rd_atom = props->max_qp_rd_atom;
 	props->atomic_cap = IB_ATOMIC_NONE;
-	props->max_map_per_fmr = 1;
 	props->max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR;
 	return 0;
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h
index 3a413752ccc3..331bc21cbcc7 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.h
@@ -89,7 +89,6 @@ struct i40iw_mr {
 	union {
 		struct ib_mr ibmr;
 		struct ib_mw ibmw;
-		struct ib_fmr ibfmr;
 	};
 	struct ib_umem *region;
 	u16 type;
diff --git a/drivers/infiniband/hw/mlx4/Kconfig b/drivers/infiniband/hw/mlx4/Kconfig
index cc7c42fe6499..f30ce9dd080a 100644
--- a/drivers/infiniband/hw/mlx4/Kconfig
+++ b/drivers/infiniband/hw/mlx4/Kconfig
@@ -4,7 +4,7 @@ config MLX4_INFINIBAND
 	depends on NETDEVICES && ETHERNET && PCI && INET
 	select NET_VENDOR_MELLANOX
 	select MLX4_CORE
-	---help---
+	help
 	  This driver provides low-level InfiniBand support for
 	  Mellanox ConnectX PCI Express host channel adapters (HCAs).
 	  This is required to use InfiniBand protocols such as
diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c
index 02a169f8027b..5f8f8d5c0ce0 100644
--- a/drivers/infiniband/hw/mlx4/ah.c
+++ b/drivers/infiniband/hw/mlx4/ah.c
@@ -141,10 +141,11 @@ static int create_iboe_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 	return 0;
 }
 
-int mlx4_ib_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,
-		      u32 flags, struct ib_udata *udata)
-
+int mlx4_ib_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
+		      struct ib_udata *udata)
 {
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
+
 	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
 		if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
 			return -EINVAL;
@@ -167,12 +168,14 @@ int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
 			    int slave_sgid_index, u8 *s_mac, u16 vlan_tag)
 {
 	struct rdma_ah_attr slave_attr = *ah_attr;
+	struct rdma_ah_init_attr init_attr = {};
 	struct mlx4_ib_ah *mah = to_mah(ah);
 	int ret;
 
 	slave_attr.grh.sgid_attr = NULL;
 	slave_attr.grh.sgid_index = slave_sgid_index;
-	ret = mlx4_ib_create_ah(ah, &slave_attr, 0, NULL);
+	init_attr.ah_attr = &slave_attr;
+	ret = mlx4_ib_create_ah(ah, &init_attr, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 275722cec8c6..816d28854a8e 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -558,7 +558,6 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 	props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm;
 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 					   props->max_mcast_grp;
-	props->max_map_per_fmr = dev->dev->caps.max_fmr_maps;
 	props->hca_core_clock = dev->dev->caps.hca_core_clock * 1000UL;
 	props->timestamp_mask = 0xFFFFFFFFFFFFULL;
 	props->max_ah = INT_MAX;
@@ -2600,13 +2599,6 @@ static const struct ib_device_ops mlx4_ib_dev_wq_ops = {
 	.modify_wq = mlx4_ib_modify_wq,
 };
 
-static const struct ib_device_ops mlx4_ib_dev_fmr_ops = {
-	.alloc_fmr = mlx4_ib_fmr_alloc,
-	.dealloc_fmr = mlx4_ib_fmr_dealloc,
-	.map_phys_fmr = mlx4_ib_map_phys_fmr,
-	.unmap_fmr = mlx4_ib_unmap_fmr,
-};
-
 static const struct ib_device_ops mlx4_ib_dev_mw_ops = {
 	.alloc_mw = mlx4_ib_alloc_mw,
 	.dealloc_mw = mlx4_ib_dealloc_mw,
@@ -2724,9 +2716,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 		ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_wq_ops);
 	}
 
-	if (!mlx4_is_slave(ibdev->dev))
-		ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_fmr_ops);
-
 	if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
 	    dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
 		ibdev->ib_dev.uverbs_cmd_mask |=
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index d188573187fa..6f4ea1067095 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -146,11 +146,6 @@ struct mlx4_ib_mw {
 	struct mlx4_mw		mmw;
 };
 
-struct mlx4_ib_fmr {
-	struct ib_fmr           ibfmr;
-	struct mlx4_fmr         mfmr;
-};
-
 #define MAX_REGS_PER_FLOW 2
 
 struct mlx4_flow_reg_id {
@@ -679,11 +674,6 @@ static inline struct mlx4_ib_mw *to_mmw(struct ib_mw *ibmw)
 	return container_of(ibmw, struct mlx4_ib_mw, ibmw);
 }
 
-static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr)
-{
-	return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr);
-}
-
 static inline struct mlx4_ib_flow *to_mflow(struct ib_flow *ibflow)
 {
 	return container_of(ibflow, struct mlx4_ib_flow, ibflow);
@@ -752,7 +742,7 @@ int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
 
-int mlx4_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+int mlx4_ib_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
 		      struct ib_udata *udata);
 int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
 			    int slave_sgid_index, u8 *s_mac, u16 vlan_tag);
@@ -794,12 +784,6 @@ int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 int mlx4_ib_mad_init(struct mlx4_ib_dev *dev);
 void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev);
 
-struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags,
-				  struct ib_fmr_attr *fmr_attr);
-int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages,
-			 u64 iova);
-int mlx4_ib_unmap_fmr(struct list_head *fmr_list);
-int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr);
 int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
 			 struct ib_port_attr *props, int netw_view);
 int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index b0121c90c561..7e0b205c05eb 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -380,7 +380,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start,
 		unsigned long untagged_start = untagged_addr(start);
 		struct vm_area_struct *vma;
 
-		down_read(&current->mm->mmap_sem);
+		mmap_read_lock(current->mm);
 		/*
 		 * FIXME: Ideally this would iterate over all the vmas that
 		 * cover the memory, but for now it requires a single vma to
@@ -395,7 +395,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start,
 			access_flags |= IB_ACCESS_LOCAL_WRITE;
 		}
 
-		up_read(&current->mm->mmap_sem);
+		mmap_read_unlock(current->mm);
 	}
 
 	return ib_umem_get(device, start, length, access_flags);
@@ -698,99 +698,6 @@ err_free:
 	return ERR_PTR(err);
 }
 
-struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc,
-				 struct ib_fmr_attr *fmr_attr)
-{
-	struct mlx4_ib_dev *dev = to_mdev(pd->device);
-	struct mlx4_ib_fmr *fmr;
-	int err = -ENOMEM;
-
-	fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
-	if (!fmr)
-		return ERR_PTR(-ENOMEM);
-
-	err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc),
-			     fmr_attr->max_pages, fmr_attr->max_maps,
-			     fmr_attr->page_shift, &fmr->mfmr);
-	if (err)
-		goto err_free;
-
-	err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr);
-	if (err)
-		goto err_mr;
-
-	fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key;
-
-	return &fmr->ibfmr;
-
-err_mr:
-	(void) mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr);
-
-err_free:
-	kfree(fmr);
-
-	return ERR_PTR(err);
-}
-
-int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
-		      int npages, u64 iova)
-{
-	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
-	struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device);
-
-	return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova,
-				 &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
-}
-
-int mlx4_ib_unmap_fmr(struct list_head *fmr_list)
-{
-	struct ib_fmr *ibfmr;
-	int err;
-	struct mlx4_dev *mdev = NULL;
-
-	list_for_each_entry(ibfmr, fmr_list, list) {
-		if (mdev && to_mdev(ibfmr->device)->dev != mdev)
-			return -EINVAL;
-		mdev = to_mdev(ibfmr->device)->dev;
-	}
-
-	if (!mdev)
-		return 0;
-
-	list_for_each_entry(ibfmr, fmr_list, list) {
-		struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
-
-		mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);
-	}
-
-	/*
-	 * Make sure all MPT status updates are visible before issuing
-	 * SYNC_TPT firmware command.
-	 */
-	wmb();
-
-	err = mlx4_SYNC_TPT(mdev);
-	if (err)
-		pr_warn("SYNC_TPT error %d when "
-		       "unmapping FMRs\n", err);
-
-	return 0;
-}
-
-int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr)
-{
-	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);
-	struct mlx4_ib_dev *dev = to_mdev(ibfmr->device);
-	int err;
-
-	err = mlx4_fmr_free(dev->dev, &ifmr->mfmr);
-
-	if (!err)
-		kfree(ifmr);
-
-	return err;
-}
-
 static int mlx4_set_page(struct ib_mr *ibmr, u64 addr)
 {
 	struct mlx4_ib_mr *mr = to_mmr(ibmr);
diff --git a/drivers/infiniband/hw/mlx5/Kconfig b/drivers/infiniband/hw/mlx5/Kconfig
index ea248def4556..ef1ff42eaec5 100644
--- a/drivers/infiniband/hw/mlx5/Kconfig
+++ b/drivers/infiniband/hw/mlx5/Kconfig
@@ -2,7 +2,7 @@
 config MLX5_INFINIBAND
 	tristate "Mellanox 5th generation network adapters (ConnectX series) support"
 	depends on NETDEVICES && ETHERNET && PCI && MLX5_CORE
-	---help---
+	help
 	  This driver provides low-level InfiniBand support for
 	  Mellanox Connect-IB PCI Express host channel adapters (HCAs).
 	  This is required to use InfiniBand protocols such as
diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile
index 228be05fbaf8..8cca61c671f8 100644
--- a/drivers/infiniband/hw/mlx5/Makefile
+++ b/drivers/infiniband/hw/mlx5/Makefile
@@ -16,7 +16,8 @@ mlx5_ib-y := ah.o \
 	     qpc.o \
 	     restrack.o \
 	     srq.o \
-	     srq_cmd.o
+	     srq_cmd.o \
+	     wr.o
 
 mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
 mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o
diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c
index 80642dd359bc..59e5ec39b447 100644
--- a/drivers/infiniband/hw/mlx5/ah.c
+++ b/drivers/infiniband/hw/mlx5/ah.c
@@ -32,9 +32,28 @@
 
 #include "mlx5_ib.h"
 
+static __be16 mlx5_ah_get_udp_sport(const struct mlx5_ib_dev *dev,
+				  const struct rdma_ah_attr *ah_attr)
+{
+	enum ib_gid_type gid_type = ah_attr->grh.sgid_attr->gid_type;
+	__be16 sport;
+
+	if ((gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
+	    (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) &&
+	    (ah_attr->grh.flow_label & IB_GRH_FLOWLABEL_MASK))
+		sport = cpu_to_be16(
+			rdma_flow_label_to_udp_sport(ah_attr->grh.flow_label));
+	else
+		sport = mlx5_get_roce_udp_sport_min(dev,
+						    ah_attr->grh.sgid_attr);
+
+	return sport;
+}
+
 static void create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,
-			 struct rdma_ah_attr *ah_attr)
+			 struct rdma_ah_init_attr *init_attr)
 {
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 	enum ib_gid_type gid_type;
 
 	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
@@ -51,12 +70,15 @@ static void create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,
 	ah->av.stat_rate_sl = (rdma_ah_get_static_rate(ah_attr) << 4);
 
 	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
+		if (init_attr->xmit_slave)
+			ah->xmit_port =
+				mlx5_lag_get_slave_port(dev->mdev,
+							init_attr->xmit_slave);
 		gid_type = ah_attr->grh.sgid_attr->gid_type;
 
 		memcpy(ah->av.rmac, ah_attr->roce.dmac,
 		       sizeof(ah_attr->roce.dmac));
-		ah->av.udp_sport =
-			mlx5_get_roce_udp_sport(dev, ah_attr->grh.sgid_attr);
+		ah->av.udp_sport = mlx5_ah_get_udp_sport(dev, ah_attr);
 		ah->av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) & 0x7) << 1;
 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
 #define MLX5_ECN_ENABLED BIT(1)
@@ -68,10 +90,11 @@ static void create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,
 	}
 }
 
-int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
-		      u32 flags, struct ib_udata *udata)
+int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
+		      struct ib_udata *udata)
 
 {
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 	struct mlx5_ib_ah *ah = to_mah(ibah);
 	struct mlx5_ib_dev *dev = to_mdev(ibah->device);
 	enum rdma_ah_attr_type ah_type = ah_attr->type;
@@ -97,7 +120,7 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
 			return err;
 	}
 
-	create_ib_ah(dev, ah, ah_attr);
+	create_ib_ah(dev, ah, init_attr);
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c
index a2fcbc49131e..cc24c711e92a 100644
--- a/drivers/infiniband/hw/mlx5/cmd.c
+++ b/drivers/infiniband/hw/mlx5/cmd.c
@@ -1,46 +1,19 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /*
- * Copyright (c) 2017, Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * Copyright (c) 2017-2020, Mellanox Technologies inc. All rights reserved.
  */
 
 #include "cmd.h"
 
 int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey)
 {
-	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {0};
-	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
 	int err;
 
 	MLX5_SET(query_special_contexts_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);
 	if (!err)
 		*mkey = MLX5_GET(query_special_contexts_out, out,
 				 dump_fill_mkey);
@@ -50,12 +23,12 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey)
 int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey)
 {
 	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)]   = {};
+	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {};
 	int err;
 
 	MLX5_SET(query_special_contexts_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec_inout(dev, query_special_contexts, in, out);
 	if (!err)
 		*null_mkey = MLX5_GET(query_special_contexts_out, out,
 				      null_mkey);
@@ -63,23 +36,15 @@ int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey)
 }
 
 int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
-			       void *out, int out_size)
+			       void *out)
 {
-	u32 in[MLX5_ST_SZ_DW(query_cong_params_in)] = { };
+	u32 in[MLX5_ST_SZ_DW(query_cong_params_in)] = {};
 
 	MLX5_SET(query_cong_params_in, in, opcode,
 		 MLX5_CMD_OP_QUERY_CONG_PARAMS);
 	MLX5_SET(query_cong_params_in, in, cong_protocol, cong_point);
 
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size);
-}
-
-int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *dev,
-				void *in, int in_size)
-{
-	u32 out[MLX5_ST_SZ_DW(modify_cong_params_out)] = { };
-
-	return mlx5_cmd_exec(dev, in, in_size, out, sizeof(out));
+	return mlx5_cmd_exec_inout(dev, query_cong_params, in, out);
 }
 
 int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
@@ -133,7 +98,7 @@ int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
 		MLX5_SET64(alloc_memic_in, in, range_start_addr,
 			   hw_start_addr + (page_idx * PAGE_SIZE));
 
-		ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+		ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);
 		if (ret) {
 			spin_lock(&dm->lock);
 			bitmap_clear(dm->memic_alloc_pages,
@@ -162,8 +127,7 @@ void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)
 	struct mlx5_core_dev *dev = dm->dev;
 	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
 	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
-	u32 out[MLX5_ST_SZ_DW(dealloc_memic_out)] = {0};
-	u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {0};
+	u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};
 	u64 start_page_idx;
 	int err;
 
@@ -174,7 +138,7 @@ void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)
 	MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);
 	MLX5_SET(dealloc_memic_in, in, memic_size, length);
 
-	err =  mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	err =  mlx5_cmd_exec_in(dev, dealloc_memic, in);
 	if (err)
 		return;
 
@@ -198,49 +162,46 @@ int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out)
 
 void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)]   = {};
-	u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
 
 	MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
 	MLX5_SET(destroy_tir_in, in, tirn, tirn);
 	MLX5_SET(destroy_tir_in, in, uid, uid);
-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec_in(dev, destroy_tir, in);
 }
 
 void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)]   = {0};
-	u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};
 
 	MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
 	MLX5_SET(destroy_tis_in, in, tisn, tisn);
 	MLX5_SET(destroy_tis_in, in, uid, uid);
-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec_in(dev, destroy_tis, in);
 }
 
 void mlx5_cmd_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u16 uid)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)]   = {};
-	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};
 
 	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
 	MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
 	MLX5_SET(destroy_rqt_in, in, uid, uid);
-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec_in(dev, destroy_rqt, in);
 }
 
 int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn,
 				    u16 uid)
 {
-	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)]   = {0};
-	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
 	int err;
 
 	MLX5_SET(alloc_transport_domain_in, in, opcode,
 		 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
 	MLX5_SET(alloc_transport_domain_in, in, uid, uid);
 
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out);
 	if (!err)
 		*tdn = MLX5_GET(alloc_transport_domain_out, out,
 				transport_domain);
@@ -251,32 +212,29 @@ int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn,
 void mlx5_cmd_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn,
 				       u16 uid)
 {
-	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)]   = {0};
-	u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};
 
 	MLX5_SET(dealloc_transport_domain_in, in, opcode,
 		 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
 	MLX5_SET(dealloc_transport_domain_in, in, uid, uid);
 	MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec_in(dev, dealloc_transport_domain, in);
 }
 
 void mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid)
 {
-	u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)]   = {};
+	u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {};
 
 	MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
 	MLX5_SET(dealloc_pd_in, in, pd, pdn);
 	MLX5_SET(dealloc_pd_in, in, uid, uid);
-	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	mlx5_cmd_exec_in(dev, dealloc_pd, in);
 }
 
 int mlx5_cmd_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,
 			u32 qpn, u16 uid)
 {
-	u32 out[MLX5_ST_SZ_DW(attach_to_mcg_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(attach_to_mcg_in)]   = {};
+	u32 in[MLX5_ST_SZ_DW(attach_to_mcg_in)] = {};
 	void *gid;
 
 	MLX5_SET(attach_to_mcg_in, in, opcode, MLX5_CMD_OP_ATTACH_TO_MCG);
@@ -284,14 +242,13 @@ int mlx5_cmd_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,
 	MLX5_SET(attach_to_mcg_in, in, uid, uid);
 	gid = MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid);
 	memcpy(gid, mgid, sizeof(*mgid));
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	return mlx5_cmd_exec_in(dev, attach_to_mcg, in);
 }
 
 int mlx5_cmd_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,
 			u32 qpn, u16 uid)
 {
-	u32 out[MLX5_ST_SZ_DW(detach_from_mcg_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(detach_from_mcg_in)]   = {};
+	u32 in[MLX5_ST_SZ_DW(detach_from_mcg_in)] = {};
 	void *gid;
 
 	MLX5_SET(detach_from_mcg_in, in, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
@@ -299,18 +256,18 @@ int mlx5_cmd_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,
 	MLX5_SET(detach_from_mcg_in, in, uid, uid);
 	gid = MLX5_ADDR_OF(detach_from_mcg_in, in, multicast_gid);
 	memcpy(gid, mgid, sizeof(*mgid));
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	return mlx5_cmd_exec_in(dev, detach_from_mcg, in);
 }
 
 int mlx5_cmd_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn, u16 uid)
 {
 	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)]   = {};
+	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)] = {};
 	int err;
 
 	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);
 	MLX5_SET(alloc_xrcd_in, in, uid, uid);
-	err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec_inout(dev, alloc_xrcd, in, out);
 	if (!err)
 		*xrcdn = MLX5_GET(alloc_xrcd_out, out, xrcd);
 	return err;
@@ -318,13 +275,12 @@ int mlx5_cmd_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn, u16 uid)
 
 int mlx5_cmd_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn, u16 uid)
 {
-	u32 out[MLX5_ST_SZ_DW(dealloc_xrcd_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)]   = {};
+	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)] = {};
 
 	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
 	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);
 	MLX5_SET(dealloc_xrcd_in, in, uid, uid);
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	return mlx5_cmd_exec_in(dev, dealloc_xrcd, in);
 }
 
 int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,
@@ -350,7 +306,7 @@ int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,
 	data = MLX5_ADDR_OF(mad_ifc_in, in, mad);
 	memcpy(data, inb, MLX5_FLD_SZ_BYTES(mad_ifc_in, mad));
 
-	err = mlx5_cmd_exec(dev, in, inlen, out, outlen);
+	err = mlx5_cmd_exec_inout(dev, mad_ifc, in, out);
 	if (err)
 		goto out;
 
diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h
index 43079b18d9b4..f4d8558db434 100644
--- a/drivers/infiniband/hw/mlx5/cmd.h
+++ b/drivers/infiniband/hw/mlx5/cmd.h
@@ -40,10 +40,8 @@
 int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey);
 int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);
 int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
-			       void *out, int out_size);
+			       void *out);
 int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out);
-int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *mdev,
-				void *in, int in_size);
 int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
 			 u64 length, u32 alignment);
 void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);
diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c
index de4da92b81a6..b9291e482428 100644
--- a/drivers/infiniband/hw/mlx5/cong.c
+++ b/drivers/infiniband/hw/mlx5/cong.c
@@ -290,7 +290,7 @@ static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u8 port_num,
 
 	node = mlx5_ib_param_to_node(offset);
 
-	err = mlx5_cmd_query_cong_params(mdev, node, out, outlen);
+	err = mlx5_cmd_query_cong_params(mdev, node, out);
 	if (err)
 		goto free;
 
@@ -339,7 +339,7 @@ static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u8 port_num,
 	MLX5_SET(field_select_r_roce_rp, field, field_select_r_roce_rp,
 		 attr_mask);
 
-	err = mlx5_cmd_modify_cong_params(mdev, in, inlen);
+	err = mlx5_cmd_exec_in(dev->mdev, modify_cong_params, in);
 	kvfree(in);
 alloc_err:
 	mlx5_ib_put_native_port_mdev(dev, port_num + 1);
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 35b98c2d64d5..9454a66c12cc 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -495,6 +495,10 @@ static u64 devx_get_obj_id(const void *in)
 		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 					MLX5_GET(rst2init_qp_in, in, qpn));
 		break;
+	case MLX5_CMD_OP_INIT2INIT_QP:
+		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
+					MLX5_GET(init2init_qp_in, in, qpn));
+		break;
 	case MLX5_CMD_OP_INIT2RTR_QP:
 		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 					MLX5_GET(init2rtr_qp_in, in, qpn));
@@ -615,7 +619,7 @@ static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
 		enum ib_qp_type	qp_type = qp->ibqp.qp_type;
 
 		if (qp_type == IB_QPT_RAW_PACKET ||
-		    (qp->flags & MLX5_IB_QP_UNDERLAY)) {
+		    (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
 			struct mlx5_ib_raw_packet_qp *raw_packet_qp =
 							 &qp->raw_packet_qp;
 			struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
@@ -820,6 +824,7 @@ static bool devx_is_obj_modify_cmd(const void *in)
 	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 	case MLX5_CMD_OP_RST2INIT_QP:
 	case MLX5_CMD_OP_INIT2RTR_QP:
+	case MLX5_CMD_OP_INIT2INIT_QP:
 	case MLX5_CMD_OP_RTR2RTS_QP:
 	case MLX5_CMD_OP_RTS2RTS_QP:
 	case MLX5_CMD_OP_SQERR2RTS_QP:
@@ -2217,14 +2222,12 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
 	obj->mdev = dev->mdev;
 	uobj->object = obj;
 	devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
-	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
-	if (err)
-		goto err_umem_destroy;
+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
 
-	return 0;
+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id,
+			     sizeof(obj_id));
+	return err;
 
-err_umem_destroy:
-	mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
 err_umem_release:
 	ib_umem_release(obj->umem);
 err_obj_free:
diff --git a/drivers/infiniband/hw/mlx5/flow.c b/drivers/infiniband/hw/mlx5/flow.c
index 3a0601c2052c..216a1108ad34 100644
--- a/drivers/infiniband/hw/mlx5/flow.c
+++ b/drivers/infiniband/hw/mlx5/flow.c
@@ -67,46 +67,41 @@ static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
 	},
 };
 
-#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
-static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
-	struct uverbs_attr_bundle *attrs)
+static int get_dests(struct uverbs_attr_bundle *attrs,
+		     struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
+		     int *dest_type, struct ib_qp **qp, u32 *flags)
 {
-	struct mlx5_flow_context flow_context = {.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG};
-	struct mlx5_ib_flow_handler *flow_handler;
-	struct mlx5_ib_flow_matcher *fs_matcher;
-	struct ib_uobject **arr_flow_actions;
-	struct ib_uflow_resources *uflow_res;
-	struct mlx5_flow_act flow_act = {};
-	void *devx_obj;
-	int dest_id, dest_type;
-	void *cmd_in;
-	int inlen;
 	bool dest_devx, dest_qp;
-	struct ib_qp *qp = NULL;
-	struct ib_uobject *uobj =
-		uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
-	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
-	int len, ret, i;
-	u32 counter_id = 0;
-	u32 *offset_attr;
-	u32 offset = 0;
-
-	if (!capable(CAP_NET_RAW))
-		return -EPERM;
+	void *devx_obj;
+	int err;
 
-	dest_devx =
-		uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
+	dest_devx = uverbs_attr_is_valid(attrs,
+					 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
 	dest_qp = uverbs_attr_is_valid(attrs,
 				       MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
 
-	fs_matcher = uverbs_attr_get_obj(attrs,
-					 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
-	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS &&
-	    ((dest_devx && dest_qp) || (!dest_devx && !dest_qp)))
+	*flags = 0;
+	err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
+				 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
+					 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
+	if (err)
+		return err;
+
+	/* Both flags are not allowed */
+	if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
+	    *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
 		return -EINVAL;
 
-	/* Allow only DEVX object as dest when inserting to FDB */
-	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !dest_devx)
+	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
+		if (dest_devx && (dest_qp || *flags))
+			return -EINVAL;
+		else if (dest_qp && *flags)
+			return -EINVAL;
+	}
+
+	/* Allow only DEVX object, drop as dest for FDB */
+	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx ||
+	     (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
 		return -EINVAL;
 
 	/* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
@@ -114,43 +109,86 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
 	    ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
 		return -EINVAL;
 
+	*qp = NULL;
 	if (dest_devx) {
-		devx_obj = uverbs_attr_get_obj(
-			attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
-		if (IS_ERR(devx_obj))
-			return PTR_ERR(devx_obj);
+		devx_obj =
+			uverbs_attr_get_obj(attrs,
+					    MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
 
 		/* Verify that the given DEVX object is a flow
 		 * steering destination.
 		 */
-		if (!mlx5_ib_devx_is_flow_dest(devx_obj, &dest_id, &dest_type))
+		if (!mlx5_ib_devx_is_flow_dest(devx_obj, dest_id, dest_type))
 			return -EINVAL;
 		/* Allow only flow table as dest when inserting to FDB or RDMA_RX */
 		if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB ||
 		     fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) &&
-		    dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
+		    *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
 			return -EINVAL;
 	} else if (dest_qp) {
 		struct mlx5_ib_qp *mqp;
 
-		qp = uverbs_attr_get_obj(attrs,
-					 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
-		if (IS_ERR(qp))
-			return PTR_ERR(qp);
+		*qp = uverbs_attr_get_obj(attrs,
+					  MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
+		if (IS_ERR(*qp))
+			return PTR_ERR(*qp);
 
-		if (qp->qp_type != IB_QPT_RAW_PACKET)
+		if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
 			return -EINVAL;
 
-		mqp = to_mqp(qp);
-		if (mqp->flags & MLX5_IB_QP_RSS)
-			dest_id = mqp->rss_qp.tirn;
+		mqp = to_mqp(*qp);
+		if (mqp->is_rss)
+			*dest_id = mqp->rss_qp.tirn;
 		else
-			dest_id = mqp->raw_packet_qp.rq.tirn;
-		dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
-	} else {
-		dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
+			*dest_id = mqp->raw_packet_qp.rq.tirn;
+		*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
+	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
+		*dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
 	}
 
+	if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
+	    fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
+		return -EINVAL;
+
+	return 0;
+}
+
+#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
+static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
+	struct uverbs_attr_bundle *attrs)
+{
+	struct mlx5_flow_context flow_context = {.flow_tag =
+		MLX5_FS_DEFAULT_FLOW_TAG};
+	u32 *offset_attr, offset = 0, counter_id = 0;
+	int dest_id, dest_type, inlen, len, ret, i;
+	struct mlx5_ib_flow_handler *flow_handler;
+	struct mlx5_ib_flow_matcher *fs_matcher;
+	struct ib_uobject **arr_flow_actions;
+	struct ib_uflow_resources *uflow_res;
+	struct mlx5_flow_act flow_act = {};
+	struct ib_qp *qp = NULL;
+	void *devx_obj, *cmd_in;
+	struct ib_uobject *uobj;
+	struct mlx5_ib_dev *dev;
+	u32 flags;
+
+	if (!capable(CAP_NET_RAW))
+		return -EPERM;
+
+	fs_matcher = uverbs_attr_get_obj(attrs,
+					 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
+	uobj =  uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
+	dev = mlx5_udata_to_mdev(&attrs->driver_udata);
+
+	if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
+		return -EINVAL;
+
+	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
+
+	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
+
 	len = uverbs_attr_get_uobjs_arr(attrs,
 		MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
 	if (len) {
@@ -180,10 +218,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
 		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
 	}
 
-	if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
-	    fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
-		return -EINVAL;
-
 	cmd_in = uverbs_attr_get_alloced_ptr(
 		attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
 	inlen = uverbs_attr_get_len(attrs,
@@ -629,7 +663,10 @@ DECLARE_UVERBS_NAMED_METHOD(
 	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
 			   UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
 			   UA_OPTIONAL,
-			   UA_ALLOC_AND_COPY));
+			   UA_ALLOC_AND_COPY),
+	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
+			     enum mlx5_ib_create_flow_flags,
+			     UA_OPTIONAL));
 
 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
 	MLX5_IB_METHOD_DESTROY_FLOW,
diff --git a/drivers/infiniband/hw/mlx5/gsi.c b/drivers/infiniband/hw/mlx5/gsi.c
index 1ae6fd95acaa..40d418153891 100644
--- a/drivers/infiniband/hw/mlx5/gsi.c
+++ b/drivers/infiniband/hw/mlx5/gsi.c
@@ -119,17 +119,15 @@ struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
 	struct mlx5_ib_gsi_qp *gsi;
 	struct ib_qp_init_attr hw_init_attr = *init_attr;
 	const u8 port_num = init_attr->port_num;
-	const int num_pkeys = pd->device->attrs.max_pkeys;
-	const int num_qps = mlx5_ib_deth_sqpn_cap(dev) ? num_pkeys : 0;
+	int num_qps = 0;
 	int ret;
 
-	mlx5_ib_dbg(dev, "creating GSI QP\n");
-
-	if (port_num > ARRAY_SIZE(dev->devr.ports) || port_num < 1) {
-		mlx5_ib_warn(dev,
-			     "invalid port number %d during GSI QP creation\n",
-			     port_num);
-		return ERR_PTR(-EINVAL);
+	if (mlx5_ib_deth_sqpn_cap(dev)) {
+		if (MLX5_CAP_GEN(dev->mdev,
+				 port_type) == MLX5_CAP_PORT_TYPE_IB)
+			num_qps = pd->device->attrs.max_pkeys;
+		else if (dev->lag_active)
+			num_qps = MLX5_MAX_PORTS;
 	}
 
 	gsi = kzalloc(sizeof(*gsi), GFP_KERNEL);
@@ -270,7 +268,7 @@ static struct ib_qp *create_gsi_ud_qp(struct mlx5_ib_gsi_qp *gsi)
 }
 
 static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp,
-			 u16 qp_index)
+			 u16 pkey_index)
 {
 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
 	struct ib_qp_attr attr;
@@ -279,7 +277,7 @@ static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp,
 
 	mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY | IB_QP_PORT;
 	attr.qp_state = IB_QPS_INIT;
-	attr.pkey_index = qp_index;
+	attr.pkey_index = pkey_index;
 	attr.qkey = IB_QP1_QKEY;
 	attr.port_num = gsi->port_num;
 	ret = ib_modify_qp(qp, &attr, mask);
@@ -313,12 +311,17 @@ static void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index)
 {
 	struct ib_device *device = gsi->rx_qp->device;
 	struct mlx5_ib_dev *dev = to_mdev(device);
+	int pkey_index = qp_index;
+	struct mlx5_ib_qp *mqp;
 	struct ib_qp *qp;
 	unsigned long flags;
 	u16 pkey;
 	int ret;
 
-	ret = ib_query_pkey(device, gsi->port_num, qp_index, &pkey);
+	if (MLX5_CAP_GEN(dev->mdev,  port_type) != MLX5_CAP_PORT_TYPE_IB)
+		pkey_index = 0;
+
+	ret = ib_query_pkey(device, gsi->port_num, pkey_index, &pkey);
 	if (ret) {
 		mlx5_ib_warn(dev, "unable to read P_Key at port %d, index %d\n",
 			     gsi->port_num, qp_index);
@@ -347,7 +350,10 @@ static void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index)
 		return;
 	}
 
-	ret = modify_to_rts(gsi, qp, qp_index);
+	mqp = to_mqp(qp);
+	if (dev->lag_active)
+		mqp->gsi_lag_port = qp_index + 1;
+	ret = modify_to_rts(gsi, qp, pkey_index);
 	if (ret)
 		goto err_destroy_qp;
 
@@ -466,11 +472,15 @@ static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_gsi_qp *gsi,
 static struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr)
 {
 	struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device);
+	struct mlx5_ib_ah *ah = to_mah(wr->ah);
 	int qp_index = wr->pkey_index;
 
-	if (!mlx5_ib_deth_sqpn_cap(dev))
+	if (!gsi->num_qps)
 		return gsi->rx_qp;
 
+	if (dev->lag_active && ah->xmit_port)
+		qp_index = ah->xmit_port - 1;
+
 	if (qp_index >= gsi->num_qps)
 		return NULL;
 
diff --git a/drivers/infiniband/hw/mlx5/ib_rep.h b/drivers/infiniband/hw/mlx5/ib_rep.h
index 3b6750cba796..5b30d3fa8f8d 100644
--- a/drivers/infiniband/hw/mlx5/ib_rep.h
+++ b/drivers/infiniband/hw/mlx5/ib_rep.h
@@ -9,9 +9,9 @@
 #include <linux/mlx5/eswitch.h>
 #include "mlx5_ib.h"
 
-#ifdef CONFIG_MLX5_ESWITCH
 extern const struct mlx5_ib_profile raw_eth_profile;
 
+#ifdef CONFIG_MLX5_ESWITCH
 u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw);
 struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw,
 					  u16 vport_num);
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 566b42f3fb18..343a8b8361e7 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -53,6 +53,7 @@
 #include <linux/list.h>
 #include <rdma/ib_smi.h>
 #include <rdma/ib_umem.h>
+#include <rdma/lag.h>
 #include <linux/in.h>
 #include <linux/etherdevice.h>
 #include "mlx5_ib.h"
@@ -60,6 +61,7 @@
 #include "cmd.h"
 #include "srq.h"
 #include "qp.h"
+#include "wr.h"
 #include <linux/mlx5/fs_helpers.h>
 #include <linux/mlx5/accel.h>
 #include <rdma/uverbs_std_types.h>
@@ -70,17 +72,10 @@
 #define UVERBS_MODULE_NAME mlx5_ib
 #include <rdma/uverbs_named_ioctl.h>
 
-#define DRIVER_NAME "mlx5_ib"
-#define DRIVER_VERSION "5.0-0"
-
 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
-MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
+MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
 MODULE_LICENSE("Dual BSD/GPL");
 
-static char mlx5_version[] =
-	DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
-	DRIVER_VERSION "\n";
-
 struct mlx5_ib_event_work {
 	struct work_struct	work;
 	union {
@@ -628,8 +623,8 @@ static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
 			     attr->index, NULL, NULL);
 }
 
-__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,
-			       const struct ib_gid_attr *attr)
+__be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
+				   const struct ib_gid_attr *attr)
 {
 	if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
 		return 0;
@@ -1004,7 +999,6 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 					   props->max_mcast_grp;
-	props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
 	props->max_ah = INT_MAX;
 	props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
 	props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
@@ -1964,6 +1958,9 @@ uar_done:
 		resp.response_length += sizeof(resp.dump_fill_mkey);
 	}
 
+	if (MLX5_CAP_GEN(dev->mdev, ece_support))
+		resp.comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
+
 	err = ib_copy_to_udata(udata, &resp, resp.response_length);
 	if (err)
 		goto out_mdev;
@@ -1974,7 +1971,7 @@ uar_done:
 	context->lib_caps = req.lib_caps;
 	print_lib_caps(dev, context->lib_caps);
 
-	if (dev->lag_active) {
+	if (mlx5_ib_lag_should_assign_affinity(dev)) {
 		u8 port = mlx5_core_native_port_num(dev->mdev) - 1;
 
 		atomic_set(&context->tx_port_affinity,
@@ -2561,7 +2558,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	struct mlx5_ib_alloc_pd_resp resp;
 	int err;
 	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
-	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)]   = {};
+	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
 	u16 uid = 0;
 	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
 		udata, struct mlx5_ib_ucontext, ibucontext);
@@ -2569,8 +2566,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	uid = context ? context->devx_uid : 0;
 	MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
 	MLX5_SET(alloc_pd_in, in, uid, uid);
-	err = mlx5_cmd_exec(to_mdev(ibdev)->mdev, in, sizeof(in),
-			    out, sizeof(out));
+	err = mlx5_cmd_exec_inout(to_mdev(ibdev)->mdev, alloc_pd, in, out);
 	if (err)
 		return err;
 
@@ -3944,7 +3940,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 		dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
 	} else {
 		dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
-		if (mqp->flags & MLX5_IB_QP_RSS)
+		if (mqp->is_rss)
 			dst->tir_num = mqp->rss_qp.tirn;
 		else
 			dst->tir_num = mqp->raw_packet_qp.rq.tirn;
@@ -4199,18 +4195,17 @@ mlx5_ib_raw_fs_rule_add(struct mlx5_ib_dev *dev,
 
 	if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) {
 		dst[dst_num].type = dest_type;
-		dst[dst_num].tir_num = dest_id;
+		dst[dst_num++].tir_num = dest_id;
 		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 	} else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) {
 		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
-		dst[dst_num].ft_num = dest_id;
+		dst[dst_num++].ft_num = dest_id;
 		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
-	} else {
-		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
+	} else  if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) {
+		dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
 		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
 	}
 
-	dst_num++;
 
 	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
 		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
@@ -4420,7 +4415,7 @@ static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 	uid = ibqp->pd ?
 		to_mpd(ibqp->pd)->uid : 0;
 
-	if (mqp->flags & MLX5_IB_QP_UNDERLAY) {
+	if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
 		return -EOPNOTSUPP;
 	}
@@ -6194,26 +6189,20 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
 	mmap_offset = mlx5_entry_to_mmap_offset(entry);
 	length = entry->rdma_entry.npages * PAGE_SIZE;
 	uobj->object = entry;
+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
 
 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
 			     &mmap_offset, sizeof(mmap_offset));
 	if (err)
-		goto err;
+		return err;
 
 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
 			     &entry->page_idx, sizeof(entry->page_idx));
 	if (err)
-		goto err;
+		return err;
 
 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
 			     &length, sizeof(length));
-	if (err)
-		goto err;
-
-	return 0;
-
-err:
-	rdma_user_mmap_entry_remove(&entry->rdma_entry);
 	return err;
 }
 
@@ -6327,26 +6316,20 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
 	mmap_offset = mlx5_entry_to_mmap_offset(entry);
 	length = entry->rdma_entry.npages * PAGE_SIZE;
 	uobj->object = entry;
+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
 
 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
 			     &mmap_offset, sizeof(mmap_offset));
 	if (err)
-		goto err;
+		return err;
 
 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
 			     &entry->page_idx, sizeof(entry->page_idx));
 	if (err)
-		goto err;
+		return err;
 
 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
 			     &length, sizeof(length));
-	if (err)
-		goto err;
-
-	return 0;
-
-err:
-	rdma_user_mmap_entry_remove(&entry->rdma_entry);
 	return err;
 }
 
@@ -6540,6 +6523,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	dev->ib_dev.phys_port_cnt	= dev->num_ports;
 	dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_count(mdev);
 	dev->ib_dev.dev.parent		= mdev->device;
+	dev->ib_dev.lag_flags		= RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
 
 	mutex_init(&dev->cap_mask_mutex);
 	INIT_LIST_HEAD(&dev->qp_list);
@@ -6629,8 +6613,8 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
 	.modify_qp = mlx5_ib_modify_qp,
 	.modify_srq = mlx5_ib_modify_srq,
 	.poll_cq = mlx5_ib_poll_cq,
-	.post_recv = mlx5_ib_post_recv,
-	.post_send = mlx5_ib_post_send,
+	.post_recv = mlx5_ib_post_recv_nodrain,
+	.post_send = mlx5_ib_post_send_nodrain,
 	.post_srq_recv = mlx5_ib_post_srq_recv,
 	.process_mad = mlx5_ib_process_mad,
 	.query_ah = mlx5_ib_query_ah,
@@ -7131,6 +7115,8 @@ void *__mlx5_ib_add(struct mlx5_ib_dev *dev,
 	int err;
 	int i;
 
+	dev->profile = profile;
+
 	for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
 		if (profile->stage[i].init) {
 			err = profile->stage[i].init(dev);
@@ -7139,7 +7125,6 @@ void *__mlx5_ib_add(struct mlx5_ib_dev *dev,
 		}
 	}
 
-	dev->profile = profile;
 	dev->ib_active = true;
 
 	return dev;
@@ -7313,8 +7298,6 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	int port_type_cap;
 	int num_ports;
 
-	printk_once(KERN_INFO "%s", mlx5_version);
-
 	if (MLX5_ESWITCH_MANAGER(mdev) &&
 	    mlx5_ib_eswitch_mode(mdev->priv.eswitch) == MLX5_ESWITCH_OFFLOADS) {
 		if (!mlx5_core_mp_enabled(mdev))
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index aaabb8a98eed..5dbe3eb0d9cb 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -337,7 +337,6 @@ struct mlx5_ib_rwq {
 	struct ib_umem		*umem;
 	size_t			buf_size;
 	unsigned int		page_shift;
-	int			create_type;
 	struct mlx5_db		db;
 	u32			user_index;
 	u32			wqe_count;
@@ -346,17 +345,6 @@ struct mlx5_ib_rwq {
 	u32			create_flags; /* Use enum mlx5_ib_wq_flags */
 };
 
-enum {
-	MLX5_QP_USER,
-	MLX5_QP_KERNEL,
-	MLX5_QP_EMPTY
-};
-
-enum {
-	MLX5_WQ_USER,
-	MLX5_WQ_KERNEL
-};
-
 struct mlx5_ib_rwq_ind_table {
 	struct ib_rwq_ind_table ib_rwq_ind_tbl;
 	u32			rqtn;
@@ -443,34 +431,37 @@ struct mlx5_ib_qp {
 	/* serialize qp state modifications
 	 */
 	struct mutex		mutex;
+	/* cached variant of create_flags from struct ib_qp_init_attr */
 	u32			flags;
 	u8			port;
 	u8			state;
-	int			wq_sig;
-	int			scat_cqe;
 	int			max_inline_data;
 	struct mlx5_bf	        bf;
-	int			has_rq;
+	u8			has_rq:1;
+	u8			is_rss:1;
 
 	/* only for user space QPs. For kernel
 	 * we have it from the bf object
 	 */
 	int			bfregn;
 
-	int			create_type;
-
 	struct list_head	qps_list;
 	struct list_head	cq_recv_list;
 	struct list_head	cq_send_list;
 	struct mlx5_rate_limit	rl;
 	u32                     underlay_qpn;
 	u32			flags_en;
-	/* storage for qp sub type when core qp type is IB_QPT_DRIVER */
-	enum ib_qp_type		qp_sub_type;
+	/*
+	 * IB/core doesn't store low-level QP types, so
+	 * store both MLX and IBTA types in the field below.
+	 * IB_QPT_DRIVER will be break to DCI/DCT subtypes.
+	 */
+	enum ib_qp_type		type;
 	/* A flag to indicate if there's a new counter is configured
 	 * but not take effective
 	 */
 	u32                     counter_pending;
+	u16			gsi_lag_port;
 };
 
 struct mlx5_ib_cq_buf {
@@ -481,24 +472,6 @@ struct mlx5_ib_cq_buf {
 	int			nent;
 };
 
-enum mlx5_ib_qp_flags {
-	MLX5_IB_QP_LSO                          = IB_QP_CREATE_IPOIB_UD_LSO,
-	MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK     = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
-	MLX5_IB_QP_CROSS_CHANNEL            = IB_QP_CREATE_CROSS_CHANNEL,
-	MLX5_IB_QP_MANAGED_SEND             = IB_QP_CREATE_MANAGED_SEND,
-	MLX5_IB_QP_MANAGED_RECV             = IB_QP_CREATE_MANAGED_RECV,
-	MLX5_IB_QP_SIGNATURE_HANDLING           = 1 << 5,
-	/* QP uses 1 as its source QP number */
-	MLX5_IB_QP_SQPN_QP1			= 1 << 6,
-	MLX5_IB_QP_CAP_SCATTER_FCS		= 1 << 7,
-	MLX5_IB_QP_RSS				= 1 << 8,
-	MLX5_IB_QP_CVLAN_STRIPPING		= 1 << 9,
-	MLX5_IB_QP_UNDERLAY			= 1 << 10,
-	MLX5_IB_QP_PCI_WRITE_END_PADDING	= 1 << 11,
-	MLX5_IB_QP_TUNNEL_OFFLOAD		= 1 << 12,
-	MLX5_IB_QP_PACKET_BASED_CREDIT		= 1 << 13,
-};
-
 struct mlx5_umr_wr {
 	struct ib_send_wr		wr;
 	u64				virt_addr;
@@ -702,12 +675,6 @@ struct umr_common {
 	struct semaphore	sem;
 };
 
-enum {
-	MLX5_FMR_INVALID,
-	MLX5_FMR_VALID,
-	MLX5_FMR_BUSY,
-};
-
 struct mlx5_cache_ent {
 	struct list_head	head;
 	/* sync access to the cahce entry
@@ -1181,7 +1148,7 @@ void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db)
 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
-int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
 		      struct ib_udata *udata);
 int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
 void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);
@@ -1205,10 +1172,6 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr
 int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
 void mlx5_ib_drain_sq(struct ib_qp *qp);
 void mlx5_ib_drain_rq(struct ib_qp *qp);
-int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
-		      const struct ib_send_wr **bad_wr);
-int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
-		      const struct ib_recv_wr **bad_wr);
 int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
 			size_t buflen, size_t *bc);
 int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
@@ -1284,8 +1247,6 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 			    struct ib_port_attr *props);
 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
 		       struct ib_port_attr *props);
-int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
-void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
 void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
 			unsigned long max_page_shift,
 			int *count, int *shift,
@@ -1383,8 +1344,8 @@ int mlx5_ib_get_vf_guid(struct ib_device *device, int vf, u8 port,
 int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,
 			u64 guid, int type);
 
-__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev,
-			       const struct ib_gid_attr *attr);
+__be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
+				   const struct ib_gid_attr *attr);
 
 void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
 void mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
@@ -1581,4 +1542,11 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
 
 int mlx5_ib_enable_driver(struct ib_device *dev);
 int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
+
+static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)
+{
+	return dev->lag_active ||
+		(MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 &&
+		 MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));
+}
 #endif /* MLX5_IB_H */
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 16af1105cfcf..7d2ec9ee5097 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -447,8 +447,7 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,
 {
 	int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?
 		     pfault->wqe.wq_num : pfault->token;
-	u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { };
-	u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)]   = { };
+	u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = {};
 	int err;
 
 	MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME);
@@ -457,7 +456,7 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,
 	MLX5_SET(page_fault_resume_in, in, wq_number, wq_num);
 	MLX5_SET(page_fault_resume_in, in, error, !!error);
 
-	err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
+	err = mlx5_cmd_exec_in(dev->mdev, page_fault_resume, in);
 	if (err)
 		mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n",
 			    wq_num, err);
@@ -1136,8 +1135,7 @@ static int mlx5_ib_mr_initiator_pfault_handler(
 	if (qp->ibqp.qp_type == IB_QPT_XRC_INI)
 		*wqe += sizeof(struct mlx5_wqe_xrc_seg);
 
-	if (qp->ibqp.qp_type == IB_QPT_UD ||
-	    qp->qp_sub_type == MLX5_IB_QPT_DCI) {
+	if (qp->type == IB_QPT_UD || qp->type == MLX5_IB_QPT_DCI) {
 		av = *wqe;
 		if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV))
 			*wqe += sizeof(struct mlx5_av);
@@ -1190,7 +1188,7 @@ static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev,
 	struct mlx5_ib_wq *wq = &qp->rq;
 	int wqe_size = 1 << wq->wqe_shift;
 
-	if (qp->wq_sig) {
+	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
 		mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n");
 		return -EFAULT;
 	}
diff --git a/drivers/infiniband/hw/mlx5/qos.c b/drivers/infiniband/hw/mlx5/qos.c
index cac878a70edb..dce92554142a 100644
--- a/drivers/infiniband/hw/mlx5/qos.c
+++ b/drivers/infiniband/hw/mlx5/qos.c
@@ -69,17 +69,14 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_PP_OBJ_ALLOC)(
 	if (err)
 		goto err;
 
-	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_PP_OBJ_ALLOC_INDEX,
-			     &pp_entry->index, sizeof(pp_entry->index));
-	if (err)
-		goto clean;
-
 	pp_entry->mdev = dev->mdev;
 	uobj->object = pp_entry;
-	return 0;
+	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_PP_OBJ_ALLOC_HANDLE);
+
+	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_PP_OBJ_ALLOC_INDEX,
+			     &pp_entry->index, sizeof(pp_entry->index));
+	return err;
 
-clean:
-	mlx5_rl_remove_rate_raw(dev->mdev, pp_entry->index);
 err:
 	kfree(pp_entry);
 	return err;
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index d93eec5d3277..81bf6b975e0e 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -40,9 +40,7 @@
 #include "ib_rep.h"
 #include "cmd.h"
 #include "qp.h"
-
-/* not supported currently */
-static int wq_signature;
+#include "wr.h"
 
 enum {
 	MLX5_IB_ACK_REQ_FREQ	= 8,
@@ -55,32 +53,6 @@ enum {
 	MLX5_IB_LINK_TYPE_ETH		= 1
 };
 
-enum {
-	MLX5_IB_SQ_STRIDE	= 6,
-	MLX5_IB_SQ_UMR_INLINE_THRESHOLD = 64,
-};
-
-static const u32 mlx5_ib_opcode[] = {
-	[IB_WR_SEND]				= MLX5_OPCODE_SEND,
-	[IB_WR_LSO]				= MLX5_OPCODE_LSO,
-	[IB_WR_SEND_WITH_IMM]			= MLX5_OPCODE_SEND_IMM,
-	[IB_WR_RDMA_WRITE]			= MLX5_OPCODE_RDMA_WRITE,
-	[IB_WR_RDMA_WRITE_WITH_IMM]		= MLX5_OPCODE_RDMA_WRITE_IMM,
-	[IB_WR_RDMA_READ]			= MLX5_OPCODE_RDMA_READ,
-	[IB_WR_ATOMIC_CMP_AND_SWP]		= MLX5_OPCODE_ATOMIC_CS,
-	[IB_WR_ATOMIC_FETCH_AND_ADD]		= MLX5_OPCODE_ATOMIC_FA,
-	[IB_WR_SEND_WITH_INV]			= MLX5_OPCODE_SEND_INVAL,
-	[IB_WR_LOCAL_INV]			= MLX5_OPCODE_UMR,
-	[IB_WR_REG_MR]				= MLX5_OPCODE_UMR,
-	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= MLX5_OPCODE_ATOMIC_MASKED_CS,
-	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= MLX5_OPCODE_ATOMIC_MASKED_FA,
-	[MLX5_IB_WR_UMR]			= MLX5_OPCODE_UMR,
-};
-
-struct mlx5_wqe_eth_pad {
-	u8 rsvd0[16];
-};
-
 enum raw_qp_set_mask_map {
 	MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID		= 1UL << 0,
 	MLX5_RAW_QP_RATE_LIMIT			= 1UL << 1,
@@ -392,17 +364,26 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
 		cap->max_recv_wr = 0;
 		cap->max_recv_sge = 0;
 	} else {
+		int wq_sig = !!(qp->flags_en & MLX5_QP_FLAG_SIGNATURE);
+
 		if (ucmd) {
 			qp->rq.wqe_cnt = ucmd->rq_wqe_count;
 			if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
 				return -EINVAL;
 			qp->rq.wqe_shift = ucmd->rq_wqe_shift;
-			if ((1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) < qp->wq_sig)
+			if ((1 << qp->rq.wqe_shift) /
+				    sizeof(struct mlx5_wqe_data_seg) <
+			    wq_sig)
 				return -EINVAL;
-			qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
+			qp->rq.max_gs =
+				(1 << qp->rq.wqe_shift) /
+					sizeof(struct mlx5_wqe_data_seg) -
+				wq_sig;
 			qp->rq.max_post = qp->rq.wqe_cnt;
 		} else {
-			wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0;
+			wqe_size =
+				wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :
+					 0;
 			wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
 			wqe_size = roundup_pow_of_two(wqe_size);
 			wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
@@ -416,7 +397,10 @@ static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
 				return -EINVAL;
 			}
 			qp->rq.wqe_shift = ilog2(wqe_size);
-			qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
+			qp->rq.max_gs =
+				(1 << qp->rq.wqe_shift) /
+					sizeof(struct mlx5_wqe_data_seg) -
+				wq_sig;
 			qp->rq.max_post = qp->rq.wqe_cnt;
 		}
 	}
@@ -596,7 +580,7 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev,
 	}
 
 	if (attr->qp_type == IB_QPT_RAW_PACKET ||
-	    qp->flags & MLX5_IB_QP_UNDERLAY) {
+	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 		qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
 	} else {
@@ -751,10 +735,7 @@ static int to_mlx5_st(enum ib_qp_type type)
 	case IB_QPT_SMI:		return MLX5_QP_ST_QP0;
 	case MLX5_IB_QPT_HW_GSI:	return MLX5_QP_ST_QP1;
 	case MLX5_IB_QPT_DCI:		return MLX5_QP_ST_DCI;
-	case IB_QPT_RAW_IPV6:		return MLX5_QP_ST_RAW_IPV6;
-	case IB_QPT_RAW_PACKET:
-	case IB_QPT_RAW_ETHERTYPE:	return MLX5_QP_ST_RAW_ETHERTYPE;
-	case IB_QPT_MAX:
+	case IB_QPT_RAW_PACKET:		return MLX5_QP_ST_RAW_ETHERTYPE;
 	default:		return -EINVAL;
 	}
 }
@@ -891,7 +872,6 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		goto err_umem;
 	}
 
-	rwq->create_type = MLX5_WQ_USER;
 	return 0;
 
 err_umem:
@@ -906,15 +886,14 @@ static int adjust_bfregn(struct mlx5_ib_dev *dev,
 				bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
 }
 
-static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
-			  struct mlx5_ib_qp *qp, struct ib_udata *udata,
-			  struct ib_qp_init_attr *attr,
-			  u32 **in,
-			  struct mlx5_ib_create_qp_resp *resp, int *inlen,
-			  struct mlx5_ib_qp_base *base)
+static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+			   struct mlx5_ib_qp *qp, struct ib_udata *udata,
+			   struct ib_qp_init_attr *attr, u32 **in,
+			   struct mlx5_ib_create_qp_resp *resp, int *inlen,
+			   struct mlx5_ib_qp_base *base,
+			   struct mlx5_ib_create_qp *ucmd)
 {
 	struct mlx5_ib_ucontext *context;
-	struct mlx5_ib_create_qp ucmd;
 	struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
 	int page_shift = 0;
 	int uar_index = 0;
@@ -928,30 +907,24 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	u16 uid;
 	u32 uar_flags;
 
-	err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
-	if (err) {
-		mlx5_ib_dbg(dev, "copy failed\n");
-		return err;
-	}
-
 	context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
 					    ibucontext);
-	uar_flags = ucmd.flags & (MLX5_QP_FLAG_UAR_PAGE_INDEX |
-				  MLX5_QP_FLAG_BFREG_INDEX);
+	uar_flags = qp->flags_en &
+		    (MLX5_QP_FLAG_UAR_PAGE_INDEX | MLX5_QP_FLAG_BFREG_INDEX);
 	switch (uar_flags) {
 	case MLX5_QP_FLAG_UAR_PAGE_INDEX:
-		uar_index = ucmd.bfreg_index;
+		uar_index = ucmd->bfreg_index;
 		bfregn = MLX5_IB_INVALID_BFREG;
 		break;
 	case MLX5_QP_FLAG_BFREG_INDEX:
 		uar_index = bfregn_to_uar_index(dev, &context->bfregi,
-						ucmd.bfreg_index, true);
+						ucmd->bfreg_index, true);
 		if (uar_index < 0)
 			return uar_index;
 		bfregn = MLX5_IB_INVALID_BFREG;
 		break;
 	case 0:
-		if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
+		if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
 			return -EINVAL;
 		bfregn = alloc_bfreg(dev, &context->bfregi);
 		if (bfregn < 0)
@@ -970,12 +943,12 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 
-	err = set_user_buf_size(dev, qp, &ucmd, base, attr);
+	err = set_user_buf_size(dev, qp, ucmd, base, attr);
 	if (err)
 		goto err_bfreg;
 
-	if (ucmd.buf_addr && ubuffer->buf_size) {
-		ubuffer->buf_addr = ucmd.buf_addr;
+	if (ucmd->buf_addr && ubuffer->buf_size) {
+		ubuffer->buf_addr = ucmd->buf_addr;
 		err = mlx5_ib_umem_get(dev, udata, ubuffer->buf_addr,
 				       ubuffer->buf_size, &ubuffer->umem,
 				       &npages, &page_shift, &ncont, &offset);
@@ -993,8 +966,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		goto err_umem;
 	}
 
-	uid = (attr->qp_type != IB_QPT_XRC_TGT &&
-	       attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
+	uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
 	MLX5_SET(create_qp_in, *in, uid, uid);
 	pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
 	if (ubuffer->umem)
@@ -1012,24 +984,14 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		resp->bfreg_index = MLX5_IB_INVALID_BFREG;
 	qp->bfregn = bfregn;
 
-	err = mlx5_ib_db_map_user(context, udata, ucmd.db_addr, &qp->db);
+	err = mlx5_ib_db_map_user(context, udata, ucmd->db_addr, &qp->db);
 	if (err) {
 		mlx5_ib_dbg(dev, "map failed\n");
 		goto err_free;
 	}
 
-	err = ib_copy_to_udata(udata, resp, min(udata->outlen, sizeof(*resp)));
-	if (err) {
-		mlx5_ib_dbg(dev, "copy failed\n");
-		goto err_unmap;
-	}
-	qp->create_type = MLX5_QP_USER;
-
 	return 0;
 
-err_unmap:
-	mlx5_ib_db_unmap_user(context, &qp->db);
-
 err_free:
 	kvfree(*in);
 
@@ -1042,72 +1004,50 @@ err_bfreg:
 	return err;
 }
 
-static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
-			    struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base,
-			    struct ib_udata *udata)
+static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+		       struct mlx5_ib_qp_base *base, struct ib_udata *udata)
 {
-	struct mlx5_ib_ucontext *context =
-		rdma_udata_to_drv_context(
-			udata,
-			struct mlx5_ib_ucontext,
-			ibucontext);
-
-	mlx5_ib_db_unmap_user(context, &qp->db);
-	ib_umem_release(base->ubuffer.umem);
-
-	/*
-	 * Free only the BFREGs which are handled by the kernel.
-	 * BFREGs of UARs allocated dynamically are handled by user.
-	 */
-	if (qp->bfregn != MLX5_IB_INVALID_BFREG)
-		mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
-}
+	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx5_ib_ucontext, ibucontext);
 
-/* get_sq_edge - Get the next nearby edge.
- *
- * An 'edge' is defined as the first following address after the end
- * of the fragment or the SQ. Accordingly, during the WQE construction
- * which repetitively increases the pointer to write the next data, it
- * simply should check if it gets to an edge.
- *
- * @sq - SQ buffer.
- * @idx - Stride index in the SQ buffer.
- *
- * Return:
- *	The new edge.
- */
-static void *get_sq_edge(struct mlx5_ib_wq *sq, u32 idx)
-{
-	void *fragment_end;
+	if (udata) {
+		/* User QP */
+		mlx5_ib_db_unmap_user(context, &qp->db);
+		ib_umem_release(base->ubuffer.umem);
 
-	fragment_end = mlx5_frag_buf_get_wqe
-		(&sq->fbc,
-		 mlx5_frag_buf_get_idx_last_contig_stride(&sq->fbc, idx));
+		/*
+		 * Free only the BFREGs which are handled by the kernel.
+		 * BFREGs of UARs allocated dynamically are handled by user.
+		 */
+		if (qp->bfregn != MLX5_IB_INVALID_BFREG)
+			mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
+		return;
+	}
 
-	return fragment_end + MLX5_SEND_WQE_BB;
+	/* Kernel QP */
+	kvfree(qp->sq.wqe_head);
+	kvfree(qp->sq.w_list);
+	kvfree(qp->sq.wrid);
+	kvfree(qp->sq.wr_data);
+	kvfree(qp->rq.wrid);
+	if (qp->db.db)
+		mlx5_db_free(dev->mdev, &qp->db);
+	if (qp->buf.frags)
+		mlx5_frag_buf_free(dev->mdev, &qp->buf);
 }
 
-static int create_kernel_qp(struct mlx5_ib_dev *dev,
-			    struct ib_qp_init_attr *init_attr,
-			    struct mlx5_ib_qp *qp,
-			    u32 **in, int *inlen,
-			    struct mlx5_ib_qp_base *base)
+static int _create_kernel_qp(struct mlx5_ib_dev *dev,
+			     struct ib_qp_init_attr *init_attr,
+			     struct mlx5_ib_qp *qp, u32 **in, int *inlen,
+			     struct mlx5_ib_qp_base *base)
 {
 	int uar_index;
 	void *qpc;
 	int err;
 
-	if (init_attr->create_flags & ~(IB_QP_CREATE_INTEGRITY_EN |
-					IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
-					IB_QP_CREATE_IPOIB_UD_LSO |
-					IB_QP_CREATE_NETIF_QP |
-					MLX5_IB_QP_CREATE_SQPN_QP1 |
-					MLX5_IB_QP_CREATE_WC_TEST))
-		return -EINVAL;
-
 	if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
 		qp->bf.bfreg = &dev->fp_bfreg;
-	else if (init_attr->create_flags & MLX5_IB_QP_CREATE_WC_TEST)
+	else if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
 		qp->bf.bfreg = &dev->wc_bfreg;
 	else
 		qp->bf.bfreg = &dev->bfreg;
@@ -1167,10 +1107,8 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
 	MLX5_SET(qpc, qpc, fre, 1);
 	MLX5_SET(qpc, qpc, rlky, 1);
 
-	if (init_attr->create_flags & MLX5_IB_QP_CREATE_SQPN_QP1) {
+	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
 		MLX5_SET(qpc, qpc, deth_sqpn, 1);
-		qp->flags |= MLX5_IB_QP_SQPN_QP1;
-	}
 
 	mlx5_fill_page_frag_array(&qp->buf,
 				  (__be64 *)MLX5_ADDR_OF(create_qp_in,
@@ -1198,7 +1136,6 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
 		err = -ENOMEM;
 		goto err_wrid;
 	}
-	qp->create_type = MLX5_QP_KERNEL;
 
 	return 0;
 
@@ -1218,36 +1155,15 @@ err_buf:
 	return err;
 }
 
-static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
-{
-	kvfree(qp->sq.wqe_head);
-	kvfree(qp->sq.w_list);
-	kvfree(qp->sq.wrid);
-	kvfree(qp->sq.wr_data);
-	kvfree(qp->rq.wrid);
-	mlx5_db_free(dev->mdev, &qp->db);
-	mlx5_frag_buf_free(dev->mdev, &qp->buf);
-}
-
 static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
 {
-	if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) ||
-	    (attr->qp_type == MLX5_IB_QPT_DCI) ||
-	    (attr->qp_type == IB_QPT_XRC_INI))
+	if (attr->srq || (qp->type == IB_QPT_XRC_TGT) ||
+	    (qp->type == MLX5_IB_QPT_DCI) || (qp->type == IB_QPT_XRC_INI))
 		return MLX5_SRQ_RQ;
 	else if (!qp->has_rq)
 		return MLX5_ZERO_LEN_RQ;
-	else
-		return MLX5_NON_ZERO_RQ;
-}
-
-static int is_connected(enum ib_qp_type qp_type)
-{
-	if (qp_type == IB_QPT_RC || qp_type == IB_QPT_UC ||
-	    qp_type == MLX5_IB_QPT_DCI)
-		return 1;
 
-	return 0;
+	return MLX5_NON_ZERO_RQ;
 }
 
 static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
@@ -1260,7 +1176,7 @@ static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
 
 	MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
 	MLX5_SET(tisc, tisc, transport_domain, tdn);
-	if (qp->flags & MLX5_IB_QP_UNDERLAY)
+	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
 		MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
 
 	return mlx5_core_create_tis(dev->mdev, in, &sq->tisn);
@@ -1409,7 +1325,7 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
 	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
 	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
 
-	if (mqp->flags & MLX5_IB_QP_CAP_SCATTER_FCS)
+	if (mqp->flags & IB_QP_CREATE_SCATTER_FCS)
 		MLX5_SET(rqc, rqc, scatter_fcs, 1);
 
 	wq = MLX5_ADDR_OF(rqc, rqc, wq);
@@ -1440,13 +1356,6 @@ static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
 	mlx5_core_destroy_rq_tracked(dev, &rq->base.mqp);
 }
 
-static bool tunnel_offload_supported(struct mlx5_core_dev *dev)
-{
-	return  (MLX5_CAP_ETH(dev, tunnel_stateless_vxlan) ||
-		 MLX5_CAP_ETH(dev, tunnel_stateless_gre) ||
-		 MLX5_CAP_ETH(dev, tunnel_stateless_geneve_rx));
-}
-
 static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
 				      struct mlx5_ib_rq *rq,
 				      u32 qp_flags_en,
@@ -1524,6 +1433,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	u16 uid = to_mpd(pd)->uid;
 	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
 
+	if (!qp->sq.wqe_cnt && !qp->rq.wqe_cnt)
+		return -EINVAL;
 	if (qp->sq.wqe_cnt) {
 		err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
 		if (err)
@@ -1547,9 +1458,9 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	if (qp->rq.wqe_cnt) {
 		rq->base.container_mibqp = qp;
 
-		if (qp->flags & MLX5_IB_QP_CVLAN_STRIPPING)
+		if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
 			rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
-		if (qp->flags & MLX5_IB_QP_PCI_WRITE_END_PADDING)
+		if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
 			rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
 		err = create_raw_packet_qp_rq(dev, rq, in, inlen, pd);
 		if (err)
@@ -1584,14 +1495,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 
 	qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
 						     rq->base.mqp.qpn;
-	err = ib_copy_to_udata(udata, resp, min(udata->outlen, sizeof(*resp)));
-	if (err)
-		goto err_destroy_tir;
-
 	return 0;
 
-err_destroy_tir:
-	destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, pd);
 err_destroy_rq:
 	destroy_raw_packet_qp_rq(dev, rq);
 err_destroy_sq:
@@ -1643,14 +1548,27 @@ static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *q
 			     to_mpd(qp->ibqp.pd)->uid);
 }
 
-static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
-				 struct ib_pd *pd,
-				 struct ib_qp_init_attr *init_attr,
-				 struct ib_udata *udata)
+struct mlx5_create_qp_params {
+	struct ib_udata *udata;
+	size_t inlen;
+	size_t outlen;
+	size_t ucmd_size;
+	void *ucmd;
+	u8 is_rss_raw : 1;
+	struct ib_qp_init_attr *attr;
+	u32 uidx;
+	struct mlx5_ib_create_qp_resp resp;
+};
+
+static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+				 struct mlx5_ib_qp *qp,
+				 struct mlx5_create_qp_params *params)
 {
+	struct ib_qp_init_attr *init_attr = params->attr;
+	struct mlx5_ib_create_qp_rss *ucmd = params->ucmd;
+	struct ib_udata *udata = params->udata;
 	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
 		udata, struct mlx5_ib_ucontext, ibucontext);
-	struct mlx5_ib_create_qp_resp resp = {};
 	int inlen;
 	int outlen;
 	int err;
@@ -1660,79 +1578,28 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	void *hfso;
 	u32 selected_fields = 0;
 	u32 outer_l4;
-	size_t min_resp_len;
 	u32 tdn = mucontext->tdn;
-	struct mlx5_ib_create_qp_rss ucmd = {};
-	size_t required_cmd_sz;
 	u8 lb_flag = 0;
 
-	if (init_attr->qp_type != IB_QPT_RAW_PACKET)
-		return -EOPNOTSUPP;
-
-	if (init_attr->create_flags || init_attr->send_cq)
-		return -EINVAL;
-
-	min_resp_len = offsetof(typeof(resp), bfreg_index) + sizeof(resp.bfreg_index);
-	if (udata->outlen < min_resp_len)
-		return -EINVAL;
-
-	required_cmd_sz = offsetof(typeof(ucmd), flags) + sizeof(ucmd.flags);
-	if (udata->inlen < required_cmd_sz) {
-		mlx5_ib_dbg(dev, "invalid inlen\n");
-		return -EINVAL;
-	}
-
-	if (udata->inlen > sizeof(ucmd) &&
-	    !ib_is_udata_cleared(udata, sizeof(ucmd),
-				 udata->inlen - sizeof(ucmd))) {
-		mlx5_ib_dbg(dev, "inlen is not supported\n");
-		return -EOPNOTSUPP;
-	}
-
-	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
-		mlx5_ib_dbg(dev, "copy failed\n");
-		return -EFAULT;
-	}
-
-	if (ucmd.comp_mask) {
+	if (ucmd->comp_mask) {
 		mlx5_ib_dbg(dev, "invalid comp mask\n");
 		return -EOPNOTSUPP;
 	}
 
-	if (ucmd.flags & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |
-			   MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
-			   MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)) {
-		mlx5_ib_dbg(dev, "invalid flags\n");
-		return -EOPNOTSUPP;
-	}
-
-	if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS &&
-	    !tunnel_offload_supported(dev->mdev)) {
-		mlx5_ib_dbg(dev, "tunnel offloads isn't supported\n");
-		return -EOPNOTSUPP;
-	}
-
-	if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
-	    !(ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
+	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
+	    !(ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
 		mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
 		return -EOPNOTSUPP;
 	}
 
-	if (ucmd.flags & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC || dev->is_rep) {
-		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
+	if (dev->is_rep)
 		qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
-	}
 
-	if (ucmd.flags & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC) {
-		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
-		qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC;
-	}
+	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
+		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
 
-	err = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
-	if (err) {
-		mlx5_ib_dbg(dev, "copy failed\n");
-		return -EINVAL;
-	}
+	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
+		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
 
 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
 	outlen = MLX5_ST_SZ_BYTES(create_tir_out);
@@ -1751,29 +1618,29 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 
 	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
 
-	if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
+	if (ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
 		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
 
 	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
 
-	if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER)
+	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER)
 		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
 	else
 		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
 
-	switch (ucmd.rx_hash_function) {
+	switch (ucmd->rx_hash_function) {
 	case MLX5_RX_HASH_FUNC_TOEPLITZ:
 	{
 		void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
 		size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
 
-		if (len != ucmd.rx_key_len) {
+		if (len != ucmd->rx_key_len) {
 			err = -EINVAL;
 			goto err;
 		}
 
 		MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
-		memcpy(rss_key, ucmd.rx_hash_key, len);
+		memcpy(rss_key, ucmd->rx_hash_key, len);
 		break;
 	}
 	default:
@@ -1781,7 +1648,7 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		goto err;
 	}
 
-	if (!ucmd.rx_hash_fields_mask) {
+	if (!ucmd->rx_hash_fields_mask) {
 		/* special case when this TIR serves as steering entry without hashing */
 		if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
 			goto create_tir;
@@ -1789,29 +1656,31 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		goto err;
 	}
 
-	if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
-	     (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
-	     ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
-	     (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
+	if (((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
+	     (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
+	     ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
+	     (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
 		err = -EINVAL;
 		goto err;
 	}
 
 	/* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
-	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
-	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
+	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
+	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
 			 MLX5_L3_PROT_TYPE_IPV4);
-	else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
-		 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
+	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
+		 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
 			 MLX5_L3_PROT_TYPE_IPV6);
 
-	outer_l4 = ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
-		    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) << 0 |
-		   ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
-		    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP)) << 1 |
-		   (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
+	outer_l4 = ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
+		    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
+			   << 0 |
+		   ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
+		    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
+			   << 1 |
+		   (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
 
 	/* Check that only one l4 protocol is set */
 	if (outer_l4 & (outer_l4 - 1)) {
@@ -1820,32 +1689,32 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	}
 
 	/* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
-	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
-	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
+	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
+	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
 			 MLX5_L4_PROT_TYPE_TCP);
-	else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
-		 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
+	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
+		 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
 			 MLX5_L4_PROT_TYPE_UDP);
 
-	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
-	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
+	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
+	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
 		selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
 
-	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
-	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
+	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
+	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
 		selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
 
-	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
-	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
+	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
+	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
 		selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
 
-	if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
-	    (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
+	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
+	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
 		selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
 
-	if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
+	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
 		selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
 
 	MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
@@ -1867,73 +1736,43 @@ create_tir:
 		goto err;
 
 	if (mucontext->devx_uid) {
-		resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
-		resp.tirn = qp->rss_qp.tirn;
+		params->resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
+		params->resp.tirn = qp->rss_qp.tirn;
 		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner)) {
-			resp.tir_icm_addr =
+			params->resp.tir_icm_addr =
 				MLX5_GET(create_tir_out, out, icm_address_31_0);
-			resp.tir_icm_addr |= (u64)MLX5_GET(create_tir_out, out,
-							   icm_address_39_32)
-					     << 32;
-			resp.tir_icm_addr |= (u64)MLX5_GET(create_tir_out, out,
-							   icm_address_63_40)
-					     << 40;
-			resp.comp_mask |=
+			params->resp.tir_icm_addr |=
+				(u64)MLX5_GET(create_tir_out, out,
+					      icm_address_39_32)
+				<< 32;
+			params->resp.tir_icm_addr |=
+				(u64)MLX5_GET(create_tir_out, out,
+					      icm_address_63_40)
+				<< 40;
+			params->resp.comp_mask |=
 				MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
 		}
 	}
 
-	err = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
-	if (err)
-		goto err_copy;
-
 	kvfree(in);
 	/* qpn is reserved for that QP */
 	qp->trans_qp.base.mqp.qpn = 0;
-	qp->flags |= MLX5_IB_QP_RSS;
+	qp->is_rss = true;
 	return 0;
 
-err_copy:
-	mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn, mucontext->devx_uid);
 err:
 	kvfree(in);
 	return err;
 }
 
-static void configure_responder_scat_cqe(struct ib_qp_init_attr *init_attr,
-					 void *qpc)
-{
-	int rcqe_sz;
-
-	if (init_attr->qp_type == MLX5_IB_QPT_DCI)
-		return;
-
-	rcqe_sz = mlx5_ib_get_cqe_size(init_attr->recv_cq);
-
-	if (init_attr->qp_type == MLX5_IB_QPT_DCT) {
-		if (rcqe_sz == 128)
-			MLX5_SET(dctc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
-
-		return;
-	}
-
-	MLX5_SET(qpc, qpc, cs_res,
-		 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
-				  MLX5_RES_SCAT_DATA32_CQE);
-}
-
 static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
 					 struct ib_qp_init_attr *init_attr,
 					 struct mlx5_ib_create_qp *ucmd,
 					 void *qpc)
 {
-	enum ib_qp_type qpt = init_attr->qp_type;
 	int scqe_sz;
 	bool allow_scat_cqe = false;
 
-	if (qpt == IB_QPT_UC || qpt == IB_QPT_UD)
-		return;
-
 	if (ucmd)
 		allow_scat_cqe = ucmd->flags & MLX5_QP_FLAG_ALLOW_SCATTER_CQE;
 
@@ -1998,269 +1837,182 @@ static int get_atomic_mode(struct mlx5_ib_dev *dev,
 	return atomic_mode;
 }
 
-static inline bool check_flags_mask(uint64_t input, uint64_t supported)
-{
-	return (input & ~supported) == 0;
-}
-
-static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
-			    struct ib_qp_init_attr *init_attr,
-			    struct ib_udata *udata, struct mlx5_ib_qp *qp)
+static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+			     struct mlx5_create_qp_params *params)
 {
+	struct mlx5_ib_create_qp *ucmd = params->ucmd;
+	struct ib_qp_init_attr *attr = params->attr;
+	u32 uidx = params->uidx;
 	struct mlx5_ib_resources *devr = &dev->devr;
+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
 	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
 	struct mlx5_core_dev *mdev = dev->mdev;
-	struct mlx5_ib_create_qp_resp resp = {};
-	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
-		udata, struct mlx5_ib_ucontext, ibucontext);
-	struct mlx5_ib_cq *send_cq;
-	struct mlx5_ib_cq *recv_cq;
-	unsigned long flags;
-	u32 uidx = MLX5_IB_DEFAULT_UIDX;
-	struct mlx5_ib_create_qp ucmd;
 	struct mlx5_ib_qp_base *base;
-	int mlx5_st;
+	unsigned long flags;
 	void *qpc;
 	u32 *in;
 	int err;
 
 	mutex_init(&qp->mutex);
-	spin_lock_init(&qp->sq.lock);
-	spin_lock_init(&qp->rq.lock);
 
-	mlx5_st = to_mlx5_st(init_attr->qp_type);
-	if (mlx5_st < 0)
-		return -EINVAL;
+	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
+		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
 
-	if (init_attr->rwq_ind_tbl) {
-		if (!udata)
-			return -ENOSYS;
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
 
-		err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata);
-		return err;
-	}
+	if (MLX5_CAP_GEN(mdev, ece_support))
+		MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
+	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 
-	if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
-		if (!MLX5_CAP_GEN(mdev, block_lb_mc)) {
-			mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n");
-			return -EINVAL;
-		} else {
-			qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;
-		}
-	}
+	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);
+	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
+	MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);
 
-	if (init_attr->create_flags &
-			(IB_QP_CREATE_CROSS_CHANNEL |
-			 IB_QP_CREATE_MANAGED_SEND |
-			 IB_QP_CREATE_MANAGED_RECV)) {
-		if (!MLX5_CAP_GEN(mdev, cd)) {
-			mlx5_ib_dbg(dev, "cross-channel isn't supported\n");
-			return -EINVAL;
-		}
-		if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL)
-			qp->flags |= MLX5_IB_QP_CROSS_CHANNEL;
-		if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND)
-			qp->flags |= MLX5_IB_QP_MANAGED_SEND;
-		if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)
-			qp->flags |= MLX5_IB_QP_MANAGED_RECV;
-	}
-
-	if (init_attr->qp_type == IB_QPT_UD &&
-	    (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO))
-		if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
-			mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n");
-			return -EOPNOTSUPP;
-		}
+	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
+		MLX5_SET(qpc, qpc, block_lb_mc, 1);
+	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
+		MLX5_SET(qpc, qpc, cd_master, 1);
+	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
+		MLX5_SET(qpc, qpc, cd_slave_send, 1);
+	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
+		MLX5_SET(qpc, qpc, cd_slave_receive, 1);
 
-	if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
-		if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
-			mlx5_ib_dbg(dev, "Scatter FCS is supported only for Raw Packet QPs");
-			return -EOPNOTSUPP;
-		}
-		if (!MLX5_CAP_GEN(dev->mdev, eth_net_offloads) ||
-		    !MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
-			mlx5_ib_dbg(dev, "Scatter FCS isn't supported\n");
-			return -EOPNOTSUPP;
-		}
-		qp->flags |= MLX5_IB_QP_CAP_SCATTER_FCS;
-	}
+	MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);
+	MLX5_SET(qpc, qpc, no_sq, 1);
+	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
+	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
+	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
+	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);
+	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
 
-	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
-		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
+	/* 0xffffff means we ask to work with cqe version 0 */
+	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
+		MLX5_SET(qpc, qpc, user_index, uidx);
 
-	if (init_attr->create_flags & IB_QP_CREATE_CVLAN_STRIPPING) {
-		if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
-		      MLX5_CAP_ETH(dev->mdev, vlan_cap)) ||
-		    (init_attr->qp_type != IB_QPT_RAW_PACKET))
-			return -EOPNOTSUPP;
-		qp->flags |= MLX5_IB_QP_CVLAN_STRIPPING;
+	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
+		MLX5_SET(qpc, qpc, end_padding_mode,
+			 MLX5_WQ_END_PAD_MODE_ALIGN);
+		/* Special case to clean flag */
+		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
 	}
 
-	if (udata) {
-		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
-			mlx5_ib_dbg(dev, "copy failed\n");
-			return -EFAULT;
-		}
+	base = &qp->trans_qp.base;
+	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
+	kvfree(in);
+	if (err)
+		return err;
 
-		if (!check_flags_mask(ucmd.flags,
-				      MLX5_QP_FLAG_ALLOW_SCATTER_CQE |
-				      MLX5_QP_FLAG_BFREG_INDEX |
-				      MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE |
-				      MLX5_QP_FLAG_SCATTER_CQE |
-				      MLX5_QP_FLAG_SIGNATURE |
-				      MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC |
-				      MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
-				      MLX5_QP_FLAG_TUNNEL_OFFLOADS |
-				      MLX5_QP_FLAG_UAR_PAGE_INDEX |
-				      MLX5_QP_FLAG_TYPE_DCI |
-				      MLX5_QP_FLAG_TYPE_DCT))
-			return -EINVAL;
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
+	if (MLX5_CAP_GEN(mdev, ece_support))
+		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
 
-		err = get_qp_user_index(ucontext, &ucmd, udata->inlen, &uidx);
-		if (err)
-			return err;
+	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
+	list_add_tail(&qp->qps_list, &dev->qp_list);
+	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
 
-		qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);
-		if (MLX5_CAP_GEN(dev->mdev, sctr_data_cqe))
-			qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);
-		if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS) {
-			if (init_attr->qp_type != IB_QPT_RAW_PACKET ||
-			    !tunnel_offload_supported(mdev)) {
-				mlx5_ib_dbg(dev, "Tunnel offload isn't supported\n");
-				return -EOPNOTSUPP;
-			}
-			qp->flags_en |= MLX5_QP_FLAG_TUNNEL_OFFLOADS;
-		}
+	qp->trans_qp.xrcdn = to_mxrcd(attr->xrcd)->xrcdn;
+	return 0;
+}
 
-		if (ucmd.flags & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC) {
-			if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
-				mlx5_ib_dbg(dev, "Self-LB UC isn't supported\n");
-				return -EOPNOTSUPP;
-			}
-			qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
-		}
+static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+			  struct mlx5_ib_qp *qp,
+			  struct mlx5_create_qp_params *params)
+{
+	struct ib_qp_init_attr *init_attr = params->attr;
+	struct mlx5_ib_create_qp *ucmd = params->ucmd;
+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
+	struct ib_udata *udata = params->udata;
+	u32 uidx = params->uidx;
+	struct mlx5_ib_resources *devr = &dev->devr;
+	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
+	struct mlx5_core_dev *mdev = dev->mdev;
+	struct mlx5_ib_cq *send_cq;
+	struct mlx5_ib_cq *recv_cq;
+	unsigned long flags;
+	struct mlx5_ib_qp_base *base;
+	int mlx5_st;
+	void *qpc;
+	u32 *in;
+	int err;
 
-		if (ucmd.flags & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC) {
-			if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
-				mlx5_ib_dbg(dev, "Self-LB UM isn't supported\n");
-				return -EOPNOTSUPP;
-			}
-			qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC;
-		}
+	mutex_init(&qp->mutex);
+	spin_lock_init(&qp->sq.lock);
+	spin_lock_init(&qp->rq.lock);
 
-		if (ucmd.flags & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE) {
-			if (init_attr->qp_type != IB_QPT_RC ||
-				!MLX5_CAP_GEN(dev->mdev, qp_packet_based)) {
-				mlx5_ib_dbg(dev, "packet based credit mode isn't supported\n");
-				return -EOPNOTSUPP;
-			}
-			qp->flags |= MLX5_IB_QP_PACKET_BASED_CREDIT;
-		}
+	mlx5_st = to_mlx5_st(qp->type);
+	if (mlx5_st < 0)
+		return -EINVAL;
 
-		if (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN) {
-			if (init_attr->qp_type != IB_QPT_UD ||
-			    (MLX5_CAP_GEN(dev->mdev, port_type) !=
-			     MLX5_CAP_PORT_TYPE_IB) ||
-			    !mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS)) {
-				mlx5_ib_dbg(dev, "Source QP option isn't supported\n");
-				return -EOPNOTSUPP;
-			}
+	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
+		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
 
-			qp->flags |= MLX5_IB_QP_UNDERLAY;
-			qp->underlay_qpn = init_attr->source_qpn;
-		}
-	} else {
-		qp->wq_sig = !!wq_signature;
-	}
+	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
+		qp->underlay_qpn = init_attr->source_qpn;
 
 	base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
-		qp->flags & MLX5_IB_QP_UNDERLAY) ?
+		qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
 	       &qp->raw_packet_qp.rq.base :
 	       &qp->trans_qp.base;
 
 	qp->has_rq = qp_has_rq(init_attr);
-	err = set_rq_size(dev, &init_attr->cap, qp->has_rq,
-			  qp, udata ? &ucmd : NULL);
+	err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
 	if (err) {
 		mlx5_ib_dbg(dev, "err %d\n", err);
 		return err;
 	}
 
-	if (pd) {
-		if (udata) {
-			__u32 max_wqes =
-				1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
-			mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count);
-			if (ucmd.rq_wqe_shift != qp->rq.wqe_shift ||
-			    ucmd.rq_wqe_count != qp->rq.wqe_cnt) {
-				mlx5_ib_dbg(dev, "invalid rq params\n");
-				return -EINVAL;
-			}
-			if (ucmd.sq_wqe_count > max_wqes) {
-				mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) > max allowed (%d)\n",
-					    ucmd.sq_wqe_count, max_wqes);
-				return -EINVAL;
-			}
-			if (init_attr->create_flags &
-			    MLX5_IB_QP_CREATE_SQPN_QP1) {
-				mlx5_ib_dbg(dev, "user-space is not allowed to create UD QPs spoofing as QP1\n");
-				return -EINVAL;
-			}
-			err = create_user_qp(dev, pd, qp, udata, init_attr, &in,
-					     &resp, &inlen, base);
-			if (err)
-				mlx5_ib_dbg(dev, "err %d\n", err);
-		} else {
-			err = create_kernel_qp(dev, init_attr, qp, &in, &inlen,
-					       base);
-			if (err)
-				mlx5_ib_dbg(dev, "err %d\n", err);
-		}
+	if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
+	    ucmd->rq_wqe_count != qp->rq.wqe_cnt)
+		return -EINVAL;
 
-		if (err)
-			return err;
-	} else {
-		in = kvzalloc(inlen, GFP_KERNEL);
-		if (!in)
-			return -ENOMEM;
+	if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
+		return -EINVAL;
 
-		qp->create_type = MLX5_QP_EMPTY;
-	}
+	err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
+			      &inlen, base, ucmd);
+	if (err)
+		return err;
 
 	if (is_sqp(init_attr->qp_type))
 		qp->port = init_attr->port_num;
 
+	if (MLX5_CAP_GEN(mdev, ece_support))
+		MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 
 	MLX5_SET(qpc, qpc, st, mlx5_st);
 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
+	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
 
-	if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR)
-		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
-	else
-		MLX5_SET(qpc, qpc, latency_sensitive, 1);
-
-
-	if (qp->wq_sig)
+	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
 		MLX5_SET(qpc, qpc, wq_signature, 1);
 
-	if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
+	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
 		MLX5_SET(qpc, qpc, block_lb_mc, 1);
 
-	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
+	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
 		MLX5_SET(qpc, qpc, cd_master, 1);
-	if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
+	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
 		MLX5_SET(qpc, qpc, cd_slave_send, 1);
-	if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
+	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
 		MLX5_SET(qpc, qpc, cd_slave_receive, 1);
-	if (qp->flags & MLX5_IB_QP_PACKET_BASED_CREDIT)
+	if (qp->flags_en & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)
 		MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);
-	if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
-		configure_responder_scat_cqe(init_attr, qpc);
-		configure_requester_scat_cqe(dev, init_attr,
-					     udata ? &ucmd : NULL,
-					     qpc);
+	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
+	    (init_attr->qp_type == IB_QPT_RC ||
+	     init_attr->qp_type == IB_QPT_UC)) {
+		int rcqe_sz = mlx5_ib_get_cqe_size(init_attr->recv_cq);
+
+		MLX5_SET(qpc, qpc, cs_res,
+			 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
+					  MLX5_RES_SCAT_DATA32_CQE);
 	}
+	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
+	    (qp->type == MLX5_IB_QPT_DCI || qp->type == IB_QPT_RC))
+		configure_requester_scat_cqe(dev, init_attr, ucmd, qpc);
 
 	if (qp->rq.wqe_cnt) {
 		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
@@ -2281,12 +2033,6 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 	/* Set default resources */
 	switch (init_attr->qp_type) {
-	case IB_QPT_XRC_TGT:
-		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
-		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
-		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
-		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr->xrcd)->xrcdn);
-		break;
 	case IB_QPT_XRC_INI:
 		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
 		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
@@ -2314,52 +2060,163 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
 		MLX5_SET(qpc, qpc, user_index, uidx);
 
-	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
-	if (init_attr->qp_type == IB_QPT_UD &&
-	    (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) {
-		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
-		qp->flags |= MLX5_IB_QP_LSO;
+	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING &&
+	    init_attr->qp_type != IB_QPT_RAW_PACKET) {
+		MLX5_SET(qpc, qpc, end_padding_mode,
+			 MLX5_WQ_END_PAD_MODE_ALIGN);
+		/* Special case to clean flag */
+		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
 	}
 
-	if (init_attr->create_flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
-		if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
-			mlx5_ib_dbg(dev, "scatter end padding is not supported\n");
-			err = -EOPNOTSUPP;
-			goto err;
-		} else if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
-			MLX5_SET(qpc, qpc, end_padding_mode,
-				 MLX5_WQ_END_PAD_MODE_ALIGN);
-		} else {
-			qp->flags |= MLX5_IB_QP_PCI_WRITE_END_PADDING;
-		}
+	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
+	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
+		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
+		raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
+		err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
+					   &params->resp);
+	} else
+		err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
+
+	kvfree(in);
+	if (err)
+		goto err_create;
+
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
+	if (MLX5_CAP_GEN(mdev, ece_support))
+		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
+
+	get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
+		&send_cq, &recv_cq);
+	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
+	mlx5_ib_lock_cqs(send_cq, recv_cq);
+	/* Maintain device to QPs access, needed for further handling via reset
+	 * flow
+	 */
+	list_add_tail(&qp->qps_list, &dev->qp_list);
+	/* Maintain CQ to QPs access, needed for further handling via reset flow
+	 */
+	if (send_cq)
+		list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
+	if (recv_cq)
+		list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
+	mlx5_ib_unlock_cqs(send_cq, recv_cq);
+	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
+
+	return 0;
+
+err_create:
+	destroy_qp(dev, qp, base, udata);
+	return err;
+}
+
+static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+			    struct mlx5_ib_qp *qp,
+			    struct mlx5_create_qp_params *params)
+{
+	struct ib_qp_init_attr *attr = params->attr;
+	u32 uidx = params->uidx;
+	struct mlx5_ib_resources *devr = &dev->devr;
+	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
+	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
+	struct mlx5_core_dev *mdev = dev->mdev;
+	struct mlx5_ib_cq *send_cq;
+	struct mlx5_ib_cq *recv_cq;
+	unsigned long flags;
+	struct mlx5_ib_qp_base *base;
+	int mlx5_st;
+	void *qpc;
+	u32 *in;
+	int err;
+
+	mutex_init(&qp->mutex);
+	spin_lock_init(&qp->sq.lock);
+	spin_lock_init(&qp->rq.lock);
+
+	mlx5_st = to_mlx5_st(qp->type);
+	if (mlx5_st < 0)
+		return -EINVAL;
+
+	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
+		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
+
+	base = &qp->trans_qp.base;
+
+	qp->has_rq = qp_has_rq(attr);
+	err = set_rq_size(dev, &attr->cap, qp->has_rq, qp, NULL);
+	if (err) {
+		mlx5_ib_dbg(dev, "err %d\n", err);
+		return err;
 	}
 
-	if (inlen < 0) {
-		err = -EINVAL;
-		goto err;
+	err = _create_kernel_qp(dev, attr, qp, &in, &inlen, base);
+	if (err)
+		return err;
+
+	if (is_sqp(attr->qp_type))
+		qp->port = attr->port_num;
+
+	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
+
+	MLX5_SET(qpc, qpc, st, mlx5_st);
+	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
+
+	if (attr->qp_type != MLX5_IB_QPT_REG_UMR)
+		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
+	else
+		MLX5_SET(qpc, qpc, latency_sensitive, 1);
+
+
+	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
+		MLX5_SET(qpc, qpc, block_lb_mc, 1);
+
+	if (qp->rq.wqe_cnt) {
+		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
+		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
 	}
 
-	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
-	    qp->flags & MLX5_IB_QP_UNDERLAY) {
-		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
-		raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
-		err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
-					   &resp);
+	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));
+
+	if (qp->sq.wqe_cnt)
+		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
+	else
+		MLX5_SET(qpc, qpc, no_sq, 1);
+
+	if (attr->srq) {
+		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
+			 to_msrq(attr->srq)->msrq.srqn);
 	} else {
-		err = mlx5_core_create_qp(dev, &base->mqp, in, inlen);
+		MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
+		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
+			 to_msrq(devr->s1)->msrq.srqn);
 	}
 
-	if (err) {
-		mlx5_ib_dbg(dev, "create qp failed\n");
-		goto err_create;
-	}
+	if (attr->send_cq)
+		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);
+
+	if (attr->recv_cq)
+		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);
+
+	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
+
+	/* 0xffffff means we ask to work with cqe version 0 */
+	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
+		MLX5_SET(qpc, qpc, user_index, uidx);
+
+	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
+	if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
+		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
 
+	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
 	kvfree(in);
+	if (err)
+		goto err_create;
 
 	base->container_mibqp = qp;
 	base->mqp.event = mlx5_ib_qp_event;
 
-	get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq,
+	get_cqs(qp->type, attr->send_cq, attr->recv_cq,
 		&send_cq, &recv_cq);
 	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
 	mlx5_ib_lock_cqs(send_cq, recv_cq);
@@ -2379,13 +2236,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	return 0;
 
 err_create:
-	if (qp->create_type == MLX5_QP_USER)
-		destroy_qp_user(dev, pd, qp, base, udata);
-	else if (qp->create_type == MLX5_QP_KERNEL)
-		destroy_qp_kernel(dev, qp);
-
-err:
-	kvfree(in);
+	destroy_qp(dev, qp, base, NULL);
 	return err;
 }
 
@@ -2447,11 +2298,6 @@ static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *re
 	}
 }
 
-static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp)
-{
-	return to_mpd(qp->ibqp.pd);
-}
-
 static void get_cqs(enum ib_qp_type qp_type,
 		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
 		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
@@ -2472,14 +2318,10 @@ static void get_cqs(enum ib_qp_type qp_type,
 	case IB_QPT_RC:
 	case IB_QPT_UC:
 	case IB_QPT_UD:
-	case IB_QPT_RAW_IPV6:
-	case IB_QPT_RAW_ETHERTYPE:
 	case IB_QPT_RAW_PACKET:
 		*send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
 		*recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
 		break;
-
-	case IB_QPT_MAX:
 	default:
 		*send_cq = NULL;
 		*recv_cq = NULL;
@@ -2505,15 +2347,15 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	}
 
 	base = (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
-		qp->flags & MLX5_IB_QP_UNDERLAY) ?
+		qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
 	       &qp->raw_packet_qp.rq.base :
 	       &qp->trans_qp.base;
 
 	if (qp->state != IB_QPS_RESET) {
 		if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET &&
-		    !(qp->flags & MLX5_IB_QP_UNDERLAY)) {
+		    !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
 			err = mlx5_core_qp_modify(dev, MLX5_CMD_OP_2RST_QP, 0,
-						  NULL, &base->mqp);
+						  NULL, &base->mqp, NULL);
 		} else {
 			struct mlx5_modify_raw_qp_param raw_qp_param = {
 				.operation = MLX5_CMD_OP_2RST_QP
@@ -2539,7 +2381,7 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	if (recv_cq)
 		list_del(&qp->cq_recv_list);
 
-	if (qp->create_type == MLX5_QP_KERNEL) {
+	if (!udata) {
 		__mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
 				   qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
 		if (send_cq != recv_cq)
@@ -2550,7 +2392,7 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
 
 	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
-	    qp->flags & MLX5_IB_QP_UNDERLAY) {
+	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		destroy_raw_packet_qp(dev, qp);
 	} else {
 		err = mlx5_core_destroy_qp(dev, &base->mqp);
@@ -2559,254 +2401,454 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 				     base->mqp.qpn);
 	}
 
-	if (qp->create_type == MLX5_QP_KERNEL)
-		destroy_qp_kernel(dev, qp);
-	else if (qp->create_type == MLX5_QP_USER)
-		destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base, udata);
+	destroy_qp(dev, qp, base, udata);
 }
 
-static const char *ib_qp_type_str(enum ib_qp_type type)
+static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+		      struct mlx5_ib_qp *qp,
+		      struct mlx5_create_qp_params *params)
 {
-	switch (type) {
-	case IB_QPT_SMI:
-		return "IB_QPT_SMI";
-	case IB_QPT_GSI:
-		return "IB_QPT_GSI";
+	struct ib_qp_init_attr *attr = params->attr;
+	struct mlx5_ib_create_qp *ucmd = params->ucmd;
+	u32 uidx = params->uidx;
+	void *dctc;
+
+	qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
+	if (!qp->dct.in)
+		return -ENOMEM;
+
+	MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
+	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
+	MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
+	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
+	MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
+	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
+	MLX5_SET(dctc, dctc, user_index, uidx);
+	if (MLX5_CAP_GEN(dev->mdev, ece_support))
+		MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
+
+	if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) {
+		int rcqe_sz = mlx5_ib_get_cqe_size(attr->recv_cq);
+
+		if (rcqe_sz == 128)
+			MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
+	}
+
+	qp->state = IB_QPS_RESET;
+
+	return 0;
+}
+
+static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
+			 enum ib_qp_type *type)
+{
+	if (attr->qp_type == IB_QPT_DRIVER && !MLX5_CAP_GEN(dev->mdev, dct))
+		goto out;
+
+	switch (attr->qp_type) {
+	case IB_QPT_XRC_TGT:
+	case IB_QPT_XRC_INI:
+		if (!MLX5_CAP_GEN(dev->mdev, xrc))
+			goto out;
+		fallthrough;
 	case IB_QPT_RC:
-		return "IB_QPT_RC";
 	case IB_QPT_UC:
-		return "IB_QPT_UC";
-	case IB_QPT_UD:
-		return "IB_QPT_UD";
-	case IB_QPT_RAW_IPV6:
-		return "IB_QPT_RAW_IPV6";
-	case IB_QPT_RAW_ETHERTYPE:
-		return "IB_QPT_RAW_ETHERTYPE";
-	case IB_QPT_XRC_INI:
-		return "IB_QPT_XRC_INI";
-	case IB_QPT_XRC_TGT:
-		return "IB_QPT_XRC_TGT";
+	case IB_QPT_SMI:
+	case MLX5_IB_QPT_HW_GSI:
+	case IB_QPT_DRIVER:
+	case IB_QPT_GSI:
+		if (dev->profile == &raw_eth_profile)
+			goto out;
 	case IB_QPT_RAW_PACKET:
-		return "IB_QPT_RAW_PACKET";
+	case IB_QPT_UD:
 	case MLX5_IB_QPT_REG_UMR:
-		return "MLX5_IB_QPT_REG_UMR";
-	case IB_QPT_DRIVER:
-		return "IB_QPT_DRIVER";
-	case IB_QPT_MAX:
+		break;
 	default:
-		return "Invalid QP type";
+		goto out;
 	}
+
+	*type = attr->qp_type;
+	return 0;
+
+out:
+	mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr->qp_type);
+	return -EOPNOTSUPP;
 }
 
-static struct ib_qp *mlx5_ib_create_dct(struct ib_pd *pd,
-					struct ib_qp_init_attr *attr,
-					struct mlx5_ib_create_qp *ucmd,
-					struct ib_udata *udata)
+static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+			    struct ib_qp_init_attr *attr,
+			    struct ib_udata *udata)
 {
 	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
 		udata, struct mlx5_ib_ucontext, ibucontext);
-	struct mlx5_ib_qp *qp;
-	int err = 0;
-	u32 uidx = MLX5_IB_DEFAULT_UIDX;
-	void *dctc;
 
-	if (!attr->srq || !attr->recv_cq)
-		return ERR_PTR(-EINVAL);
+	if (!udata) {
+		/* Kernel create_qp callers */
+		if (attr->rwq_ind_tbl)
+			return -EOPNOTSUPP;
 
-	err = get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &uidx);
-	if (err)
-		return ERR_PTR(err);
+		switch (attr->qp_type) {
+		case IB_QPT_RAW_PACKET:
+		case IB_QPT_DRIVER:
+			return -EOPNOTSUPP;
+		default:
+			return 0;
+		}
+	}
 
-	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
-	if (!qp)
-		return ERR_PTR(-ENOMEM);
+	/* Userspace create_qp callers */
+	if (attr->qp_type == IB_QPT_RAW_PACKET && !ucontext->cqe_version) {
+		mlx5_ib_dbg(dev,
+			"Raw Packet QP is only supported for CQE version > 0\n");
+		return -EINVAL;
+	}
 
-	qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
-	if (!qp->dct.in) {
-		err = -ENOMEM;
-		goto err_free;
+	if (attr->qp_type != IB_QPT_RAW_PACKET && attr->rwq_ind_tbl) {
+		mlx5_ib_dbg(dev,
+			    "Wrong QP type %d for the RWQ indirect table\n",
+			    attr->qp_type);
+		return -EINVAL;
 	}
 
-	MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
-	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
-	qp->qp_sub_type = MLX5_IB_QPT_DCT;
-	MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
-	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
-	MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
-	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
-	MLX5_SET(dctc, dctc, user_index, uidx);
+	switch (attr->qp_type) {
+	case IB_QPT_SMI:
+	case MLX5_IB_QPT_HW_GSI:
+	case MLX5_IB_QPT_REG_UMR:
+	case IB_QPT_GSI:
+		mlx5_ib_dbg(dev, "Kernel doesn't support QP type %d\n",
+			    attr->qp_type);
+		return -EINVAL;
+	default:
+		break;
+	}
 
-	if (ucmd->flags & MLX5_QP_FLAG_SCATTER_CQE)
-		configure_responder_scat_cqe(attr, dctc);
+	/*
+	 * We don't need to see this warning, it means that kernel code
+	 * missing ib_pd. Placed here to catch developer's mistakes.
+	 */
+	WARN_ONCE(!pd && attr->qp_type != IB_QPT_XRC_TGT,
+		  "There is a missing PD pointer assignment\n");
+	return 0;
+}
 
-	qp->state = IB_QPS_RESET;
+static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
+				bool cond, struct mlx5_ib_qp *qp)
+{
+	if (!(*flags & flag))
+		return;
 
-	return &qp->ibqp;
-err_free:
-	kfree(qp);
-	return ERR_PTR(err);
+	if (cond) {
+		qp->flags_en |= flag;
+		*flags &= ~flag;
+		return;
+	}
+
+	if (flag == MLX5_QP_FLAG_SCATTER_CQE) {
+		/*
+		 * We don't return error if this flag was provided,
+		 * and mlx5 doesn't have right capability.
+		 */
+		*flags &= ~MLX5_QP_FLAG_SCATTER_CQE;
+		return;
+	}
+	mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);
 }
 
-static int set_mlx_qp_type(struct mlx5_ib_dev *dev,
-			   struct ib_qp_init_attr *init_attr,
-			   struct mlx5_ib_create_qp *ucmd,
-			   struct ib_udata *udata)
+static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+				void *ucmd, struct ib_qp_init_attr *attr)
 {
-	enum { MLX_QP_FLAGS = MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI };
-	int err;
+	struct mlx5_core_dev *mdev = dev->mdev;
+	bool cond;
+	int flags;
 
-	if (!udata)
+	if (attr->rwq_ind_tbl)
+		flags = ((struct mlx5_ib_create_qp_rss *)ucmd)->flags;
+	else
+		flags = ((struct mlx5_ib_create_qp *)ucmd)->flags;
+
+	switch (flags & (MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI)) {
+	case MLX5_QP_FLAG_TYPE_DCI:
+		qp->type = MLX5_IB_QPT_DCI;
+		break;
+	case MLX5_QP_FLAG_TYPE_DCT:
+		qp->type = MLX5_IB_QPT_DCT;
+		break;
+	default:
+		if (qp->type != IB_QPT_DRIVER)
+			break;
+		/*
+		 * It is IB_QPT_DRIVER and or no subtype or
+		 * wrong subtype were provided.
+		 */
 		return -EINVAL;
+	}
 
-	if (udata->inlen < sizeof(*ucmd)) {
-		mlx5_ib_dbg(dev, "create_qp user command is smaller than expected\n");
+	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCI, true, qp);
+	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCT, true, qp);
+
+	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SIGNATURE, true, qp);
+	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SCATTER_CQE,
+			    MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
+
+	if (qp->type == IB_QPT_RAW_PACKET) {
+		cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) ||
+		       MLX5_CAP_ETH(mdev, tunnel_stateless_gre) ||
+		       MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);
+		process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TUNNEL_OFFLOADS,
+				    cond, qp);
+		process_vendor_flag(dev, &flags,
+				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, true,
+				    qp);
+		process_vendor_flag(dev, &flags,
+				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, true,
+				    qp);
+	}
+
+	if (qp->type == IB_QPT_RC)
+		process_vendor_flag(dev, &flags,
+				    MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,
+				    MLX5_CAP_GEN(mdev, qp_packet_based), qp);
+
+	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_BFREG_INDEX, true, qp);
+	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_UAR_PAGE_INDEX, true, qp);
+
+	cond = qp->flags_en & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |
+				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
+				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);
+	if (attr->rwq_ind_tbl && cond) {
+		mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",
+			    cond);
 		return -EINVAL;
 	}
-	err = ib_copy_from_udata(ucmd, udata, sizeof(*ucmd));
-	if (err)
-		return err;
 
-	if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCI) {
-		init_attr->qp_type = MLX5_IB_QPT_DCI;
-	} else {
-		if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCT) {
-			init_attr->qp_type = MLX5_IB_QPT_DCT;
-		} else {
-			mlx5_ib_dbg(dev, "Invalid QP flags\n");
-			return -EINVAL;
-		}
+	if (flags)
+		mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);
+
+	return (flags) ? -EINVAL : 0;
 	}
 
-	if (!MLX5_CAP_GEN(dev->mdev, dct)) {
-		mlx5_ib_dbg(dev, "DC transport is not supported\n");
-		return -EOPNOTSUPP;
+static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
+				bool cond, struct mlx5_ib_qp *qp)
+{
+	if (!(*flags & flag))
+		return;
+
+	if (cond) {
+		qp->flags |= flag;
+		*flags &= ~flag;
+		return;
 	}
 
-	return 0;
+	if (flag == MLX5_IB_QP_CREATE_WC_TEST) {
+		/*
+		 * Special case, if condition didn't meet, it won't be error,
+		 * just different in-kernel flow.
+		 */
+		*flags &= ~MLX5_IB_QP_CREATE_WC_TEST;
+		return;
+	}
+	mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);
 }
 
-struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
-				struct ib_qp_init_attr *verbs_init_attr,
-				struct ib_udata *udata)
+static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+				struct ib_qp_init_attr *attr)
 {
-	struct mlx5_ib_dev *dev;
-	struct mlx5_ib_qp *qp;
-	u16 xrcdn = 0;
-	int err;
-	struct ib_qp_init_attr mlx_init_attr;
-	struct ib_qp_init_attr *init_attr = verbs_init_attr;
-	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
-		udata, struct mlx5_ib_ucontext, ibucontext);
+	enum ib_qp_type qp_type = qp->type;
+	struct mlx5_core_dev *mdev = dev->mdev;
+	int create_flags = attr->create_flags;
+	bool cond;
 
-	if (pd) {
-		dev = to_mdev(pd->device);
+	if (qp->type == IB_QPT_UD && dev->profile == &raw_eth_profile)
+		if (create_flags & ~MLX5_IB_QP_CREATE_WC_TEST)
+			return -EINVAL;
 
-		if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
-			if (!ucontext) {
-				mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n");
-				return ERR_PTR(-EINVAL);
-			} else if (!ucontext->cqe_version) {
-				mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version > 0\n");
-				return ERR_PTR(-EINVAL);
-			}
-		}
-	} else {
-		/* being cautious here */
-		if (init_attr->qp_type != IB_QPT_XRC_TGT &&
-		    init_attr->qp_type != MLX5_IB_QPT_REG_UMR) {
-			pr_warn("%s: no PD for transport %s\n", __func__,
-				ib_qp_type_str(init_attr->qp_type));
-			return ERR_PTR(-EINVAL);
-		}
-		dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device);
+	if (qp_type == MLX5_IB_QPT_DCT)
+		return (create_flags) ? -EINVAL : 0;
+
+	if (qp_type == IB_QPT_RAW_PACKET && attr->rwq_ind_tbl)
+		return (create_flags) ? -EINVAL : 0;
+
+	process_create_flag(dev, &create_flags,
+			    IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
+			    MLX5_CAP_GEN(mdev, block_lb_mc), qp);
+	process_create_flag(dev, &create_flags, IB_QP_CREATE_CROSS_CHANNEL,
+			    MLX5_CAP_GEN(mdev, cd), qp);
+	process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_SEND,
+			    MLX5_CAP_GEN(mdev, cd), qp);
+	process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_RECV,
+			    MLX5_CAP_GEN(mdev, cd), qp);
+
+	if (qp_type == IB_QPT_UD) {
+		process_create_flag(dev, &create_flags,
+				    IB_QP_CREATE_IPOIB_UD_LSO,
+				    MLX5_CAP_GEN(mdev, ipoib_basic_offloads),
+				    qp);
+		cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;
+		process_create_flag(dev, &create_flags, IB_QP_CREATE_SOURCE_QPN,
+				    cond, qp);
+	}
+
+	if (qp_type == IB_QPT_RAW_PACKET) {
+		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
+		       MLX5_CAP_ETH(mdev, scatter_fcs);
+		process_create_flag(dev, &create_flags,
+				    IB_QP_CREATE_SCATTER_FCS, cond, qp);
+
+		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
+		       MLX5_CAP_ETH(mdev, vlan_cap);
+		process_create_flag(dev, &create_flags,
+				    IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);
+	}
+
+	process_create_flag(dev, &create_flags,
+			    IB_QP_CREATE_PCI_WRITE_END_PADDING,
+			    MLX5_CAP_GEN(mdev, end_pad), qp);
+
+	process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_WC_TEST,
+			    qp_type != MLX5_IB_QPT_REG_UMR, qp);
+	process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
+			    true, qp);
+
+	if (create_flags)
+		mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
+			    create_flags);
+
+	return (create_flags) ? -EINVAL : 0;
+}
+
+static int process_udata_size(struct mlx5_ib_dev *dev,
+			      struct mlx5_create_qp_params *params)
+{
+	size_t ucmd = sizeof(struct mlx5_ib_create_qp);
+	struct ib_udata *udata = params->udata;
+	size_t outlen = udata->outlen;
+	size_t inlen = udata->inlen;
+
+	params->outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));
+	params->ucmd_size = ucmd;
+	if (!params->is_rss_raw) {
+		/* User has old rdma-core, which doesn't support ECE */
+		size_t min_inlen =
+			offsetof(struct mlx5_ib_create_qp, ece_options);
+
+		/*
+		 * We will check in check_ucmd_data() that user
+		 * cleared everything after inlen.
+		 */
+		params->inlen = (inlen < min_inlen) ? 0 : min(inlen, ucmd);
+		goto out;
 	}
 
-	if (init_attr->qp_type == IB_QPT_DRIVER) {
-		struct mlx5_ib_create_qp ucmd;
+	/* RSS RAW QP */
+	if (inlen < offsetofend(struct mlx5_ib_create_qp_rss, flags))
+		return -EINVAL;
 
-		init_attr = &mlx_init_attr;
-		memcpy(init_attr, verbs_init_attr, sizeof(*verbs_init_attr));
-		err = set_mlx_qp_type(dev, init_attr, &ucmd, udata);
-		if (err)
-			return ERR_PTR(err);
+	if (outlen < offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))
+		return -EINVAL;
 
-		if (init_attr->qp_type == MLX5_IB_QPT_DCI) {
-			if (init_attr->cap.max_recv_wr ||
-			    init_attr->cap.max_recv_sge) {
-				mlx5_ib_dbg(dev, "DCI QP requires zero size receive queue\n");
-				return ERR_PTR(-EINVAL);
-			}
-		} else {
-			return mlx5_ib_create_dct(pd, init_attr, &ucmd, udata);
-		}
+	ucmd = sizeof(struct mlx5_ib_create_qp_rss);
+	params->ucmd_size = ucmd;
+	if (inlen > ucmd && !ib_is_udata_cleared(udata, ucmd, inlen - ucmd))
+		return -EINVAL;
+
+	params->inlen = min(ucmd, inlen);
+out:
+	if (!params->inlen)
+		mlx5_ib_dbg(dev, "udata is too small\n");
+
+	return (params->inlen) ? 0 : -EINVAL;
+}
+
+static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+		     struct mlx5_ib_qp *qp,
+		     struct mlx5_create_qp_params *params)
+{
+	int err;
+
+	if (params->is_rss_raw) {
+		err = create_rss_raw_qp_tir(dev, pd, qp, params);
+		goto out;
 	}
 
-	switch (init_attr->qp_type) {
-	case IB_QPT_XRC_TGT:
-	case IB_QPT_XRC_INI:
-		if (!MLX5_CAP_GEN(dev->mdev, xrc)) {
-			mlx5_ib_dbg(dev, "XRC not supported\n");
-			return ERR_PTR(-ENOSYS);
-		}
-		init_attr->recv_cq = NULL;
-		if (init_attr->qp_type == IB_QPT_XRC_TGT) {
-			xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
-			init_attr->send_cq = NULL;
-		}
+	if (qp->type == MLX5_IB_QPT_DCT) {
+		err = create_dct(dev, pd, qp, params);
+		goto out;
+	}
 
-		/* fall through */
-	case IB_QPT_RAW_PACKET:
-	case IB_QPT_RC:
-	case IB_QPT_UC:
-	case IB_QPT_UD:
-	case IB_QPT_SMI:
-	case MLX5_IB_QPT_HW_GSI:
-	case MLX5_IB_QPT_REG_UMR:
-	case MLX5_IB_QPT_DCI:
-		qp = kzalloc(sizeof(*qp), GFP_KERNEL);
-		if (!qp)
-			return ERR_PTR(-ENOMEM);
+	if (qp->type == IB_QPT_XRC_TGT) {
+		err = create_xrc_tgt_qp(dev, qp, params);
+		goto out;
+	}
 
-		err = create_qp_common(dev, pd, init_attr, udata, qp);
-		if (err) {
-			mlx5_ib_dbg(dev, "create_qp_common failed\n");
-			kfree(qp);
-			return ERR_PTR(err);
-		}
+	if (params->udata)
+		err = create_user_qp(dev, pd, qp, params);
+	else
+		err = create_kernel_qp(dev, pd, qp, params);
 
-		if (is_qp0(init_attr->qp_type))
-			qp->ibqp.qp_num = 0;
-		else if (is_qp1(init_attr->qp_type))
-			qp->ibqp.qp_num = 1;
-		else
-			qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
+out:
+	if (err) {
+		mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
+		return err;
+	}
 
-		mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",
-			    qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
-			    init_attr->recv_cq ? to_mcq(init_attr->recv_cq)->mcq.cqn : -1,
-			    init_attr->send_cq ? to_mcq(init_attr->send_cq)->mcq.cqn : -1);
+	if (is_qp0(qp->type))
+		qp->ibqp.qp_num = 0;
+	else if (is_qp1(qp->type))
+		qp->ibqp.qp_num = 1;
+	else
+		qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
 
-		qp->trans_qp.xrcdn = xrcdn;
+	mlx5_ib_dbg(dev,
+		"QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",
+		qp->type, qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
+		params->attr->recv_cq ? to_mcq(params->attr->recv_cq)->mcq.cqn :
+					-1,
+		params->attr->send_cq ? to_mcq(params->attr->send_cq)->mcq.cqn :
+					-1,
+		params->resp.ece_options);
 
-		break;
+	return 0;
+}
 
-	case IB_QPT_GSI:
-		return mlx5_ib_gsi_create_qp(pd, init_attr);
+static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+			 struct ib_qp_init_attr *attr)
+{
+	int ret = 0;
 
-	case IB_QPT_RAW_IPV6:
-	case IB_QPT_RAW_ETHERTYPE:
-	case IB_QPT_MAX:
+	switch (qp->type) {
+	case MLX5_IB_QPT_DCT:
+		ret = (!attr->srq || !attr->recv_cq) ? -EINVAL : 0;
+		break;
+	case MLX5_IB_QPT_DCI:
+		ret = (attr->cap.max_recv_wr || attr->cap.max_recv_sge) ?
+			      -EINVAL :
+			      0;
+		break;
+	case IB_QPT_RAW_PACKET:
+		ret = (attr->rwq_ind_tbl && attr->send_cq) ? -EINVAL : 0;
+		break;
 	default:
-		mlx5_ib_dbg(dev, "unsupported qp type %d\n",
-			    init_attr->qp_type);
-		/* Don't support raw QPs */
-		return ERR_PTR(-EOPNOTSUPP);
+		break;
 	}
 
-	if (verbs_init_attr->qp_type == IB_QPT_DRIVER)
-		qp->qp_sub_type = init_attr->qp_type;
+	if (ret)
+		mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp->type);
 
-	return &qp->ibqp;
+	return ret;
+}
+
+static int get_qp_uidx(struct mlx5_ib_qp *qp,
+		       struct mlx5_create_qp_params *params)
+{
+	struct mlx5_ib_create_qp *ucmd = params->ucmd;
+	struct ib_udata *udata = params->udata;
+	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
+		udata, struct mlx5_ib_ucontext, ibucontext);
+
+	if (params->is_rss_raw)
+		return 0;
+
+	return get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &params->uidx);
 }
 
 static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
@@ -2828,6 +2870,150 @@ static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
 	return 0;
 }
 
+static int check_ucmd_data(struct mlx5_ib_dev *dev,
+			   struct mlx5_create_qp_params *params)
+{
+	struct ib_qp_init_attr *attr = params->attr;
+	struct ib_udata *udata = params->udata;
+	size_t size, last;
+	int ret;
+
+	if (params->is_rss_raw)
+		/*
+		 * These QPs don't have "reserved" field in their
+		 * create_qp input struct, so their data is always valid.
+		 */
+		last = sizeof(struct mlx5_ib_create_qp_rss);
+	else
+		/* IB_QPT_RAW_PACKET doesn't have ECE data */
+		switch (attr->qp_type) {
+		case IB_QPT_RAW_PACKET:
+			last = offsetof(struct mlx5_ib_create_qp, ece_options);
+			break;
+		default:
+			last = offsetof(struct mlx5_ib_create_qp, reserved);
+		}
+
+	if (udata->inlen <= last)
+		return 0;
+
+	/*
+	 * User provides different create_qp structures based on the
+	 * flow and we need to know if he cleared memory after our
+	 * struct create_qp ends.
+	 */
+	size = udata->inlen - last;
+	ret = ib_is_udata_cleared(params->udata, last, size);
+	if (!ret)
+		mlx5_ib_dbg(
+			dev,
+			"udata is not cleared, inlen = %lu, ucmd = %lu, last = %lu, size = %lu\n",
+			udata->inlen, params->ucmd_size, last, size);
+	return ret ? 0 : -EINVAL;
+}
+
+struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attr,
+				struct ib_udata *udata)
+{
+	struct mlx5_create_qp_params params = {};
+	struct mlx5_ib_dev *dev;
+	struct mlx5_ib_qp *qp;
+	enum ib_qp_type type;
+	int err;
+
+	dev = pd ? to_mdev(pd->device) :
+		   to_mdev(to_mxrcd(attr->xrcd)->ibxrcd.device);
+
+	err = check_qp_type(dev, attr, &type);
+	if (err)
+		return ERR_PTR(err);
+
+	err = check_valid_flow(dev, pd, attr, udata);
+	if (err)
+		return ERR_PTR(err);
+
+	if (attr->qp_type == IB_QPT_GSI)
+		return mlx5_ib_gsi_create_qp(pd, attr);
+
+	params.udata = udata;
+	params.uidx = MLX5_IB_DEFAULT_UIDX;
+	params.attr = attr;
+	params.is_rss_raw = !!attr->rwq_ind_tbl;
+
+	if (udata) {
+		err = process_udata_size(dev, &params);
+		if (err)
+			return ERR_PTR(err);
+
+		err = check_ucmd_data(dev, &params);
+		if (err)
+			return ERR_PTR(err);
+
+		params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);
+		if (!params.ucmd)
+			return ERR_PTR(-ENOMEM);
+
+		err = ib_copy_from_udata(params.ucmd, udata, params.inlen);
+		if (err)
+			goto free_ucmd;
+	}
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp) {
+		err = -ENOMEM;
+		goto free_ucmd;
+	}
+
+	qp->type = type;
+	if (udata) {
+		err = process_vendor_flags(dev, qp, params.ucmd, attr);
+		if (err)
+			goto free_qp;
+
+		err = get_qp_uidx(qp, &params);
+		if (err)
+			goto free_qp;
+	}
+	err = process_create_flags(dev, qp, attr);
+	if (err)
+		goto free_qp;
+
+	err = check_qp_attr(dev, qp, attr);
+	if (err)
+		goto free_qp;
+
+	err = create_qp(dev, pd, qp, &params);
+	if (err)
+		goto free_qp;
+
+	kfree(params.ucmd);
+	params.ucmd = NULL;
+
+	if (udata)
+		/*
+		 * It is safe to copy response for all user create QP flows,
+		 * including MLX5_IB_QPT_DCT, which doesn't need it.
+		 * In that case, resp will be filled with zeros.
+		 */
+		err = ib_copy_to_udata(udata, &params.resp, params.outlen);
+	if (err)
+		goto destroy_qp;
+
+	return &qp->ibqp;
+
+destroy_qp:
+	if (qp->type == MLX5_IB_QPT_DCT)
+		mlx5_ib_destroy_dct(qp);
+	else
+		destroy_qp_common(dev, qp, udata);
+	qp = NULL;
+free_qp:
+	kfree(qp);
+free_ucmd:
+	kfree(params.ucmd);
+	return ERR_PTR(err);
+}
+
 int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
@@ -2836,7 +3022,7 @@ int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 	if (unlikely(qp->qp_type == IB_QPT_GSI))
 		return mlx5_ib_gsi_destroy_qp(qp);
 
-	if (mqp->qp_sub_type == MLX5_IB_QPT_DCT)
+	if (mqp->type == MLX5_IB_QPT_DCT)
 		return mlx5_ib_destroy_dct(mqp);
 
 	destroy_qp_common(dev, mqp, udata);
@@ -2846,14 +3032,13 @@ int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 	return 0;
 }
 
-static int to_mlx5_access_flags(struct mlx5_ib_qp *qp,
-				const struct ib_qp_attr *attr,
-				int attr_mask, __be32 *hw_access_flags_be)
+static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,
+				const struct ib_qp_attr *attr, int attr_mask,
+				void *qpc)
 {
-	u8 dest_rd_atomic;
-	u32 access_flags, hw_access_flags = 0;
-
 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
+	u8 dest_rd_atomic;
+	u32 access_flags;
 
 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
 		dest_rd_atomic = attr->max_dest_rd_atomic;
@@ -2868,8 +3053,8 @@ static int to_mlx5_access_flags(struct mlx5_ib_qp *qp,
 	if (!dest_rd_atomic)
 		access_flags &= IB_ACCESS_REMOTE_WRITE;
 
-	if (access_flags & IB_ACCESS_REMOTE_READ)
-		hw_access_flags |= MLX5_QP_BIT_RRE;
+	MLX5_SET(qpc, qpc, rre, !!(access_flags & IB_ACCESS_REMOTE_READ));
+
 	if (access_flags & IB_ACCESS_REMOTE_ATOMIC) {
 		int atomic_mode;
 
@@ -2877,15 +3062,11 @@ static int to_mlx5_access_flags(struct mlx5_ib_qp *qp,
 		if (atomic_mode < 0)
 			return -EOPNOTSUPP;
 
-		hw_access_flags |= MLX5_QP_BIT_RAE;
-		hw_access_flags |= atomic_mode << MLX5_ATOMIC_MODE_OFFSET;
+		MLX5_SET(qpc, qpc, rae, 1);
+		MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);
 	}
 
-	if (access_flags & IB_ACCESS_REMOTE_WRITE)
-		hw_access_flags |= MLX5_QP_BIT_RWE;
-
-	*hw_access_flags_be = cpu_to_be32(hw_access_flags);
-
+	MLX5_SET(qpc, qpc, rwe, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
 	return 0;
 }
 
@@ -2965,11 +3146,22 @@ static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
 	return err;
 }
 
+static void mlx5_set_path_udp_sport(void *path, const struct rdma_ah_attr *ah,
+				    u32 lqpn, u32 rqpn)
+
+{
+	u32 fl = ah->grh.flow_label;
+
+	if (!fl)
+		fl = rdma_calc_flow_label(lqpn, rqpn);
+
+	MLX5_SET(ads, path, udp_sport, rdma_flow_label_to_udp_sport(fl));
+}
+
 static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
-			 const struct rdma_ah_attr *ah,
-			 struct mlx5_qp_path *path, u8 port, int attr_mask,
-			 u32 path_flags, const struct ib_qp_attr *attr,
-			 bool alt)
+			 const struct rdma_ah_attr *ah, void *path, u8 port,
+			 int attr_mask, u32 path_flags,
+			 const struct ib_qp_attr *attr, bool alt)
 {
 	const struct ib_global_route *grh = rdma_ah_read_grh(ah);
 	int err;
@@ -2978,8 +3170,8 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	u8 sl = rdma_ah_get_sl(ah);
 
 	if (attr_mask & IB_QP_PKEY_INDEX)
-		path->pkey_index = cpu_to_be16(alt ? attr->alt_pkey_index :
-						     attr->pkey_index);
+		MLX5_SET(ads, path, pkey_index,
+			 alt ? attr->alt_pkey_index : attr->pkey_index);
 
 	if (ah_flags & IB_AH_GRH) {
 		if (grh->sgid_index >=
@@ -2995,45 +3187,49 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		if (!(ah_flags & IB_AH_GRH))
 			return -EINVAL;
 
-		memcpy(path->rmac, ah->roce.dmac, sizeof(ah->roce.dmac));
-		if (qp->ibqp.qp_type == IB_QPT_RC ||
-		    qp->ibqp.qp_type == IB_QPT_UC ||
-		    qp->ibqp.qp_type == IB_QPT_XRC_INI ||
-		    qp->ibqp.qp_type == IB_QPT_XRC_TGT)
-			path->udp_sport =
-				mlx5_get_roce_udp_sport(dev, ah->grh.sgid_attr);
-		path->dci_cfi_prio_sl = (sl & 0x7) << 4;
+		ether_addr_copy(MLX5_ADDR_OF(ads, path, rmac_47_32),
+				ah->roce.dmac);
+		if ((qp->ibqp.qp_type == IB_QPT_RC ||
+		     qp->ibqp.qp_type == IB_QPT_UC ||
+		     qp->ibqp.qp_type == IB_QPT_XRC_INI ||
+		     qp->ibqp.qp_type == IB_QPT_XRC_TGT) &&
+		    (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
+		    (attr_mask & IB_QP_DEST_QPN))
+			mlx5_set_path_udp_sport(path, ah,
+						qp->ibqp.qp_num,
+						attr->dest_qp_num);
+		MLX5_SET(ads, path, eth_prio, sl & 0x7);
 		gid_type = ah->grh.sgid_attr->gid_type;
 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
-			path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f;
+			MLX5_SET(ads, path, dscp, grh->traffic_class >> 2);
 	} else {
-		path->fl_free_ar = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
-		path->fl_free_ar |=
-			(path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0;
-		path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
-		path->grh_mlid = rdma_ah_get_path_bits(ah) & 0x7f;
-		if (ah_flags & IB_AH_GRH)
-			path->grh_mlid	|= 1 << 7;
-		path->dci_cfi_prio_sl = sl & 0xf;
+		MLX5_SET(ads, path, fl, !!(path_flags & MLX5_PATH_FLAG_FL));
+		MLX5_SET(ads, path, free_ar,
+			 !!(path_flags & MLX5_PATH_FLAG_FREE_AR));
+		MLX5_SET(ads, path, rlid, rdma_ah_get_dlid(ah));
+		MLX5_SET(ads, path, mlid, rdma_ah_get_path_bits(ah));
+		MLX5_SET(ads, path, grh, !!(ah_flags & IB_AH_GRH));
+		MLX5_SET(ads, path, sl, sl);
 	}
 
 	if (ah_flags & IB_AH_GRH) {
-		path->mgid_index = grh->sgid_index;
-		path->hop_limit  = grh->hop_limit;
-		path->tclass_flowlabel =
-			cpu_to_be32((grh->traffic_class << 20) |
-				    (grh->flow_label));
-		memcpy(path->rgid, grh->dgid.raw, 16);
+		MLX5_SET(ads, path, src_addr_index, grh->sgid_index);
+		MLX5_SET(ads, path, hop_limit, grh->hop_limit);
+		MLX5_SET(ads, path, tclass, grh->traffic_class);
+		MLX5_SET(ads, path, flow_label, grh->flow_label);
+		memcpy(MLX5_ADDR_OF(ads, path, rgid_rip), grh->dgid.raw,
+		       sizeof(grh->dgid.raw));
 	}
 
 	err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));
 	if (err < 0)
 		return err;
-	path->static_rate = err;
-	path->port = port;
+	MLX5_SET(ads, path, stat_rate, err);
+	MLX5_SET(ads, path, vhca_port_num, port);
 
 	if (attr_mask & IB_QP_TIMEOUT)
-		path->ackto_lt = (alt ? attr->alt_timeout : attr->timeout) << 3;
+		MLX5_SET(ads, path, ack_timeout,
+			 alt ? attr->alt_timeout : attr->timeout);
 
 	if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
 		return modify_raw_packet_eth_prio(dev->mdev,
@@ -3050,10 +3246,12 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
 					  MLX5_QP_OPTPAR_RAE		|
 					  MLX5_QP_OPTPAR_RWE		|
 					  MLX5_QP_OPTPAR_PKEY_INDEX	|
-					  MLX5_QP_OPTPAR_PRI_PORT,
+					  MLX5_QP_OPTPAR_PRI_PORT	|
+					  MLX5_QP_OPTPAR_LAG_TX_AFF,
 			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE		|
 					  MLX5_QP_OPTPAR_PKEY_INDEX	|
-					  MLX5_QP_OPTPAR_PRI_PORT,
+					  MLX5_QP_OPTPAR_PRI_PORT	|
+					  MLX5_QP_OPTPAR_LAG_TX_AFF,
 			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX	|
 					  MLX5_QP_OPTPAR_Q_KEY		|
 					  MLX5_QP_OPTPAR_PRI_PORT,
@@ -3061,17 +3259,20 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
 					  MLX5_QP_OPTPAR_RAE		|
 					  MLX5_QP_OPTPAR_RWE		|
 					  MLX5_QP_OPTPAR_PKEY_INDEX	|
-					  MLX5_QP_OPTPAR_PRI_PORT,
+					  MLX5_QP_OPTPAR_PRI_PORT	|
+					  MLX5_QP_OPTPAR_LAG_TX_AFF,
 		},
 		[MLX5_QP_STATE_RTR] = {
 			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
 					  MLX5_QP_OPTPAR_RRE            |
 					  MLX5_QP_OPTPAR_RAE            |
 					  MLX5_QP_OPTPAR_RWE            |
-					  MLX5_QP_OPTPAR_PKEY_INDEX,
+					  MLX5_QP_OPTPAR_PKEY_INDEX	|
+					  MLX5_QP_OPTPAR_LAG_TX_AFF,
 			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
 					  MLX5_QP_OPTPAR_RWE            |
-					  MLX5_QP_OPTPAR_PKEY_INDEX,
+					  MLX5_QP_OPTPAR_PKEY_INDEX	|
+					  MLX5_QP_OPTPAR_LAG_TX_AFF,
 			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX     |
 					  MLX5_QP_OPTPAR_Q_KEY,
 			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX	|
@@ -3080,7 +3281,8 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
 					  MLX5_QP_OPTPAR_RRE            |
 					  MLX5_QP_OPTPAR_RAE            |
 					  MLX5_QP_OPTPAR_RWE            |
-					  MLX5_QP_OPTPAR_PKEY_INDEX,
+					  MLX5_QP_OPTPAR_PKEY_INDEX	|
+					  MLX5_QP_OPTPAR_LAG_TX_AFF,
 		},
 	},
 	[MLX5_QP_STATE_RTR] = {
@@ -3414,43 +3616,80 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	return 0;
 }
 
-static unsigned int get_tx_affinity(struct mlx5_ib_dev *dev,
-				    struct mlx5_ib_pd *pd,
-				    struct mlx5_ib_qp_base *qp_base,
-				    u8 port_num, struct ib_udata *udata)
+static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
+				       struct ib_udata *udata)
 {
 	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
 		udata, struct mlx5_ib_ucontext, ibucontext);
-	unsigned int tx_port_affinity;
+	u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
+	atomic_t *tx_port_affinity;
+
+	if (ucontext)
+		tx_port_affinity = &ucontext->tx_port_affinity;
+	else
+		tx_port_affinity = &dev->port[port_num].roce.tx_port_affinity;
 
-	if (ucontext) {
-		tx_port_affinity = (unsigned int)atomic_add_return(
-					   1, &ucontext->tx_port_affinity) %
-					   MLX5_MAX_PORTS +
-				   1;
+	return (unsigned int)atomic_add_return(1, tx_port_affinity) %
+		MLX5_MAX_PORTS + 1;
+}
+
+static bool qp_supports_affinity(struct ib_qp *qp)
+{
+	if ((qp->qp_type == IB_QPT_RC) ||
+	    (qp->qp_type == IB_QPT_UD) ||
+	    (qp->qp_type == IB_QPT_UC) ||
+	    (qp->qp_type == IB_QPT_RAW_PACKET) ||
+	    (qp->qp_type == IB_QPT_XRC_INI) ||
+	    (qp->qp_type == IB_QPT_XRC_TGT))
+		return true;
+	return false;
+}
+
+static unsigned int get_tx_affinity(struct ib_qp *qp,
+				    const struct ib_qp_attr *attr,
+				    int attr_mask, u8 init,
+				    struct ib_udata *udata)
+{
+	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
+		udata, struct mlx5_ib_ucontext, ibucontext);
+	struct mlx5_ib_dev *dev = to_mdev(qp->device);
+	struct mlx5_ib_qp *mqp = to_mqp(qp);
+	struct mlx5_ib_qp_base *qp_base;
+	unsigned int tx_affinity;
+
+	if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
+	      qp_supports_affinity(qp)))
+		return 0;
+
+	if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
+		tx_affinity = mqp->gsi_lag_port;
+	else if (init)
+		tx_affinity = get_tx_affinity_rr(dev, udata);
+	else if ((attr_mask & IB_QP_AV) && attr->xmit_slave)
+		tx_affinity =
+			mlx5_lag_get_slave_port(dev->mdev, attr->xmit_slave);
+	else
+		return 0;
+
+	qp_base = &mqp->trans_qp.base;
+	if (ucontext)
 		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",
-				tx_port_affinity, qp_base->mqp.qpn, ucontext);
-	} else {
-		tx_port_affinity =
-			(unsigned int)atomic_add_return(
-				1, &dev->port[port_num].roce.tx_port_affinity) %
-				MLX5_MAX_PORTS +
-			1;
+			    tx_affinity, qp_base->mqp.qpn, ucontext);
+	else
 		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
-				tx_port_affinity, qp_base->mqp.qpn);
-	}
-
-	return tx_port_affinity;
+			    tx_affinity, qp_base->mqp.qpn);
+	return tx_affinity;
 }
 
 static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
 				    struct rdma_counter *counter)
 {
 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
+	u32 in[MLX5_ST_SZ_DW(rts2rts_qp_in)] = {};
 	struct mlx5_ib_qp *mqp = to_mqp(qp);
-	struct mlx5_qp_context context = {};
 	struct mlx5_ib_qp_base *base;
 	u32 set_id;
+	u32 *qpc;
 
 	if (counter)
 		set_id = counter->id;
@@ -3458,11 +3697,15 @@ static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
 		set_id = mlx5_ib_get_counters_id(dev, mqp->port - 1);
 
 	base = &mqp->trans_qp.base;
-	context.qp_counter_set_usr_page &= cpu_to_be32(0xffffff);
-	context.qp_counter_set_usr_page |= cpu_to_be32(set_id << 24);
-	return mlx5_core_qp_modify(dev, MLX5_CMD_OP_RTS2RTS_QP,
-				   MLX5_QP_OPTPAR_COUNTER_SET_ID, &context,
-				   &base->mqp);
+	MLX5_SET(rts2rts_qp_in, in, opcode, MLX5_CMD_OP_RTS2RTS_QP);
+	MLX5_SET(rts2rts_qp_in, in, qpn, base->mqp.qpn);
+	MLX5_SET(rts2rts_qp_in, in, uid, base->mqp.uid);
+	MLX5_SET(rts2rts_qp_in, in, opt_param_mask,
+		 MLX5_QP_OPTPAR_COUNTER_SET_ID);
+
+	qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);
+	MLX5_SET(qpc, qpc, counter_set_id, set_id);
+	return mlx5_cmd_exec_in(dev->mdev, rts2rts_qp, in);
 }
 
 static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
@@ -3470,6 +3713,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 			       enum ib_qp_state cur_state,
 			       enum ib_qp_state new_state,
 			       const struct mlx5_ib_modify_qp *ucmd,
+			       struct mlx5_ib_modify_qp_resp *resp,
 			       struct ib_udata *udata)
 {
 	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
@@ -3513,67 +3757,60 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	struct mlx5_ib_qp *qp = to_mqp(ibqp);
 	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 	struct mlx5_ib_cq *send_cq, *recv_cq;
-	struct mlx5_qp_context *context;
 	struct mlx5_ib_pd *pd;
 	enum mlx5_qp_state mlx5_cur, mlx5_new;
-	enum mlx5_qp_optpar optpar;
+	void *qpc, *pri_path, *alt_path;
+	enum mlx5_qp_optpar optpar = 0;
 	u32 set_id = 0;
 	int mlx5_st;
 	int err;
 	u16 op;
 	u8 tx_affinity = 0;
 
-	mlx5_st = to_mlx5_st(ibqp->qp_type == IB_QPT_DRIVER ?
-			     qp->qp_sub_type : ibqp->qp_type);
+	mlx5_st = to_mlx5_st(qp->type);
 	if (mlx5_st < 0)
 		return -EINVAL;
 
-	context = kzalloc(sizeof(*context), GFP_KERNEL);
-	if (!context)
+	qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
+	if (!qpc)
 		return -ENOMEM;
 
-	pd = get_pd(qp);
-	context->flags = cpu_to_be32(mlx5_st << 16);
+	pd = to_mpd(qp->ibqp.pd);
+	MLX5_SET(qpc, qpc, st, mlx5_st);
 
 	if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
-		context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
+		MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
 	} else {
 		switch (attr->path_mig_state) {
 		case IB_MIG_MIGRATED:
-			context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
+			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
 			break;
 		case IB_MIG_REARM:
-			context->flags |= cpu_to_be32(MLX5_QP_PM_REARM << 11);
+			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);
 			break;
 		case IB_MIG_ARMED:
-			context->flags |= cpu_to_be32(MLX5_QP_PM_ARMED << 11);
+			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);
 			break;
 		}
 	}
 
-	if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
-		if ((ibqp->qp_type == IB_QPT_RC) ||
-		    (ibqp->qp_type == IB_QPT_UD &&
-		     !(qp->flags & MLX5_IB_QP_SQPN_QP1)) ||
-		    (ibqp->qp_type == IB_QPT_UC) ||
-		    (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
-		    (ibqp->qp_type == IB_QPT_XRC_INI) ||
-		    (ibqp->qp_type == IB_QPT_XRC_TGT)) {
-			if (dev->lag_active) {
-				u8 p = mlx5_core_native_port_num(dev->mdev) - 1;
-				tx_affinity = get_tx_affinity(dev, pd, base, p,
-							      udata);
-				context->flags |= cpu_to_be32(tx_affinity << 24);
-			}
-		}
-	}
+	tx_affinity = get_tx_affinity(ibqp, attr, attr_mask,
+				      cur_state == IB_QPS_RESET &&
+				      new_state == IB_QPS_INIT, udata);
+
+	MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);
+	if (tx_affinity && new_state == IB_QPS_RTR &&
+	    MLX5_CAP_GEN(dev->mdev, init2_lag_tx_port_affinity))
+		optpar |= MLX5_QP_OPTPAR_LAG_TX_AFF;
 
 	if (is_sqp(ibqp->qp_type)) {
-		context->mtu_msgmax = (IB_MTU_256 << 5) | 8;
+		MLX5_SET(qpc, qpc, mtu, IB_MTU_256);
+		MLX5_SET(qpc, qpc, log_msg_max, 8);
 	} else if ((ibqp->qp_type == IB_QPT_UD &&
-		    !(qp->flags & MLX5_IB_QP_UNDERLAY)) ||
+		    !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) ||
 		   ibqp->qp_type == MLX5_IB_QPT_REG_UMR) {
-		context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
+		MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);
+		MLX5_SET(qpc, qpc, log_msg_max, 12);
 	} else if (attr_mask & IB_QP_PATH_MTU) {
 		if (attr->path_mtu < IB_MTU_256 ||
 		    attr->path_mtu > IB_MTU_4096) {
@@ -3581,40 +3818,45 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 			err = -EINVAL;
 			goto out;
 		}
-		context->mtu_msgmax = (attr->path_mtu << 5) |
-				      (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg);
+		MLX5_SET(qpc, qpc, mtu, attr->path_mtu);
+		MLX5_SET(qpc, qpc, log_msg_max,
+			 MLX5_CAP_GEN(dev->mdev, log_max_msg));
 	}
 
 	if (attr_mask & IB_QP_DEST_QPN)
-		context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num);
+		MLX5_SET(qpc, qpc, remote_qpn, attr->dest_qp_num);
+
+	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
+	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
 
 	if (attr_mask & IB_QP_PKEY_INDEX)
-		context->pri_path.pkey_index = cpu_to_be16(attr->pkey_index);
+		MLX5_SET(ads, pri_path, pkey_index, attr->pkey_index);
 
 	/* todo implement counter_index functionality */
 
 	if (is_sqp(ibqp->qp_type))
-		context->pri_path.port = qp->port;
+		MLX5_SET(ads, pri_path, vhca_port_num, qp->port);
 
 	if (attr_mask & IB_QP_PORT)
-		context->pri_path.port = attr->port_num;
+		MLX5_SET(ads, pri_path, vhca_port_num, attr->port_num);
 
 	if (attr_mask & IB_QP_AV) {
-		err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path,
-				    attr_mask & IB_QP_PORT ? attr->port_num : qp->port,
+		err = mlx5_set_path(dev, qp, &attr->ah_attr, pri_path,
+				    attr_mask & IB_QP_PORT ? attr->port_num :
+							     qp->port,
 				    attr_mask, 0, attr, false);
 		if (err)
 			goto out;
 	}
 
 	if (attr_mask & IB_QP_TIMEOUT)
-		context->pri_path.ackto_lt |= attr->timeout << 3;
+		MLX5_SET(ads, pri_path, ack_timeout, attr->timeout);
 
 	if (attr_mask & IB_QP_ALT_PATH) {
-		err = mlx5_set_path(dev, qp, &attr->alt_ah_attr,
-				    &context->alt_path,
+		err = mlx5_set_path(dev, qp, &attr->alt_ah_attr, alt_path,
 				    attr->alt_port_num,
-				    attr_mask | IB_QP_PKEY_INDEX | IB_QP_TIMEOUT,
+				    attr_mask | IB_QP_PKEY_INDEX |
+					    IB_QP_TIMEOUT,
 				    0, attr, true);
 		if (err)
 			goto out;
@@ -3623,75 +3865,68 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
 		&send_cq, &recv_cq);
 
-	context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
-	context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0;
-	context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0;
-	context->params1  = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28);
+	MLX5_SET(qpc, qpc, pd, pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
+	if (send_cq)
+		MLX5_SET(qpc, qpc, cqn_snd, send_cq->mcq.cqn);
+	if (recv_cq)
+		MLX5_SET(qpc, qpc, cqn_rcv, recv_cq->mcq.cqn);
+
+	MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);
 
 	if (attr_mask & IB_QP_RNR_RETRY)
-		context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
+		MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry);
 
 	if (attr_mask & IB_QP_RETRY_CNT)
-		context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
+		MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt);
 
-	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
-		if (attr->max_rd_atomic)
-			context->params1 |=
-				cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
-	}
+	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic)
+		MLX5_SET(qpc, qpc, log_sra_max, ilog2(attr->max_rd_atomic));
 
 	if (attr_mask & IB_QP_SQ_PSN)
-		context->next_send_psn = cpu_to_be32(attr->sq_psn);
+		MLX5_SET(qpc, qpc, next_send_psn, attr->sq_psn);
 
-	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
-		if (attr->max_dest_rd_atomic)
-			context->params2 |=
-				cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
-	}
+	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic)
+		MLX5_SET(qpc, qpc, log_rra_max,
+			 ilog2(attr->max_dest_rd_atomic));
 
 	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
-		__be32 access_flags;
-
-		err = to_mlx5_access_flags(qp, attr, attr_mask, &access_flags);
+		err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);
 		if (err)
 			goto out;
-
-		context->params2 |= access_flags;
 	}
 
 	if (attr_mask & IB_QP_MIN_RNR_TIMER)
-		context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
+		MLX5_SET(qpc, qpc, min_rnr_nak, attr->min_rnr_timer);
 
 	if (attr_mask & IB_QP_RQ_PSN)
-		context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
+		MLX5_SET(qpc, qpc, next_rcv_psn, attr->rq_psn);
 
 	if (attr_mask & IB_QP_QKEY)
-		context->qkey = cpu_to_be32(attr->qkey);
+		MLX5_SET(qpc, qpc, q_key, attr->qkey);
 
 	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
-		context->db_rec_addr = cpu_to_be64(qp->db.dma);
+		MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
 
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
 			       qp->port) - 1;
 
 		/* Underlay port should be used - index 0 function per port */
-		if (qp->flags & MLX5_IB_QP_UNDERLAY)
+		if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
 			port_num = 0;
 
 		if (ibqp->counter)
 			set_id = ibqp->counter->id;
 		else
 			set_id = mlx5_ib_get_counters_id(dev, port_num);
-		context->qp_counter_set_usr_page |=
-			cpu_to_be32(set_id << 24);
+		MLX5_SET(qpc, qpc, counter_set_id, set_id);
 	}
 
 	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
-		context->sq_crq_size |= cpu_to_be16(1 << 4);
+		MLX5_SET(qpc, qpc, rlky, 1);
 
-	if (qp->flags & MLX5_IB_QP_SQPN_QP1)
-		context->deth_sqpn = cpu_to_be32(1);
+	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
+		MLX5_SET(qpc, qpc, deth_sqpn, 1);
 
 	mlx5_cur = to_mlx5_state(cur_state);
 	mlx5_new = to_mlx5_state(new_state);
@@ -3703,11 +3938,11 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	}
 
 	op = optab[mlx5_cur][mlx5_new];
-	optpar = ib_mask_to_mlx5_opt(attr_mask);
+	optpar |= ib_mask_to_mlx5_opt(attr_mask);
 	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
 
 	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
-	    qp->flags & MLX5_IB_QP_UNDERLAY) {
+	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		struct mlx5_modify_raw_qp_param raw_qp_param = {};
 
 		raw_qp_param.operation = op;
@@ -3749,7 +3984,15 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 
 		err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
 	} else {
-		err = mlx5_core_qp_modify(dev, op, optpar, context, &base->mqp);
+		if (udata) {
+			/* For the kernel flows, the resp will stay zero */
+			resp->ece_options =
+				MLX5_CAP_GEN(dev->mdev, ece_support) ?
+					ucmd->ece_options : 0;
+			resp->response_length = sizeof(*resp);
+		}
+		err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp,
+					  &resp->ece_options);
 	}
 
 	if (err)
@@ -3796,7 +4039,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	}
 
 out:
-	kfree(context);
+	kfree(qpc);
 	return err;
 }
 
@@ -3854,7 +4097,8 @@ static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new
  * Other transitions and attributes are illegal
  */
 static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
-			      int attr_mask, struct ib_udata *udata)
+			      int attr_mask, struct mlx5_ib_modify_qp *ucmd,
+			      struct ib_udata *udata)
 {
 	struct mlx5_ib_qp *qp = to_mqp(ibqp);
 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
@@ -3870,6 +4114,15 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	new_state = attr->qp_state;
 
 	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
+	if (MLX5_CAP_GEN(dev->mdev, ece_support) && ucmd->ece_options)
+		/*
+		 * DCT doesn't initialize QP till modify command is executed,
+		 * so we need to overwrite previously set ECE field if user
+		 * provided any value except zero, which means not set/not
+		 * valid.
+		 */
+		MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
+
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		u16 set_id;
 
@@ -3902,17 +4155,23 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 
 		set_id = mlx5_ib_get_counters_id(dev, attr->port_num - 1);
 		MLX5_SET(dctc, dctc, counter_set_id, set_id);
-
 	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
 		struct mlx5_ib_modify_qp_resp resp = {};
-		u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {0};
-		u32 min_resp_len = offsetof(typeof(resp), dctn) +
-				   sizeof(resp.dctn);
+		u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {};
+		u32 min_resp_len = offsetofend(typeof(resp), dctn);
 
 		if (udata->outlen < min_resp_len)
 			return -EINVAL;
 		resp.response_length = min_resp_len;
 
+		/*
+		 * If we don't have enough space for the ECE options,
+		 * simply indicate it with resp.response_length.
+		 */
+		resp.response_length = (udata->outlen < sizeof(resp)) ?
+					       min_resp_len :
+					       sizeof(resp);
+
 		required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
 		if (!is_valid_mask(attr_mask, required, 0))
 			return -EINVAL;
@@ -3929,6 +4188,8 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		if (err)
 			return err;
 		resp.dctn = qp->dct.mdct.mqp.qpn;
+		if (MLX5_CAP_GEN(dev->mdev, ece_support))
+			resp.ece_options = MLX5_GET(create_dct_out, out, ece);
 		err = ib_copy_to_udata(udata, &resp, resp.response_length);
 		if (err) {
 			mlx5_core_destroy_dct(dev, &qp->dct.mdct);
@@ -3949,11 +4210,11 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		      int attr_mask, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
+	struct mlx5_ib_modify_qp_resp resp = {};
 	struct mlx5_ib_qp *qp = to_mqp(ibqp);
 	struct mlx5_ib_modify_qp ucmd = {};
 	enum ib_qp_type qp_type;
 	enum ib_qp_state cur_state, new_state;
-	size_t required_cmd_sz;
 	int err = -EINVAL;
 	int port;
 
@@ -3961,9 +4222,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		return -ENOSYS;
 
 	if (udata && udata->inlen) {
-		required_cmd_sz = offsetof(typeof(ucmd), reserved) +
-			sizeof(ucmd.reserved);
-		if (udata->inlen < required_cmd_sz)
+		if (udata->inlen < offsetofend(typeof(ucmd), ece_options))
 			return -EINVAL;
 
 		if (udata->inlen > sizeof(ucmd) &&
@@ -3976,23 +4235,20 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 			return -EFAULT;
 
 		if (ucmd.comp_mask ||
-		    memchr_inv(&ucmd.reserved, 0, sizeof(ucmd.reserved)) ||
 		    memchr_inv(&ucmd.burst_info.reserved, 0,
 			       sizeof(ucmd.burst_info.reserved)))
 			return -EOPNOTSUPP;
+
 	}
 
 	if (unlikely(ibqp->qp_type == IB_QPT_GSI))
 		return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
 
-	if (ibqp->qp_type == IB_QPT_DRIVER)
-		qp_type = qp->qp_sub_type;
-	else
-		qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ?
-			IB_QPT_GSI : ibqp->qp_type;
+	qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ? IB_QPT_GSI :
+								    qp->type;
 
 	if (qp_type == MLX5_IB_QPT_DCT)
-		return mlx5_ib_modify_dct(ibqp, attr, attr_mask, udata);
+		return mlx5_ib_modify_dct(ibqp, attr, attr_mask, &ucmd, udata);
 
 	mutex_lock(&qp->mutex);
 
@@ -4003,7 +4259,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
 	}
 
-	if (qp->flags & MLX5_IB_QP_UNDERLAY) {
+	if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
 			mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
 				    attr_mask);
@@ -4063,1439 +4319,19 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	}
 
 	err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
-				  new_state, &ucmd, udata);
-
-out:
-	mutex_unlock(&qp->mutex);
-	return err;
-}
-
-static void _handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
-				   u32 wqe_sz, void **cur_edge)
-{
-	u32 idx;
-
-	idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);
-	*cur_edge = get_sq_edge(sq, idx);
-
-	*seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
-}
-
-/* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the
- * next nearby edge and get new address translation for current WQE position.
- * @sq - SQ buffer.
- * @seg: Current WQE position (16B aligned).
- * @wqe_sz: Total current WQE size [16B].
- * @cur_edge: Updated current edge.
- */
-static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
-					 u32 wqe_sz, void **cur_edge)
-{
-	if (likely(*seg != *cur_edge))
-		return;
-
-	_handle_post_send_edge(sq, seg, wqe_sz, cur_edge);
-}
-
-/* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's
- * pointers. At the end @seg is aligned to 16B regardless the copied size.
- * @sq - SQ buffer.
- * @cur_edge: Updated current edge.
- * @seg: Current WQE position (16B aligned).
- * @wqe_sz: Total current WQE size [16B].
- * @src: Pointer to copy from.
- * @n: Number of bytes to copy.
- */
-static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,
-				   void **seg, u32 *wqe_sz, const void *src,
-				   size_t n)
-{
-	while (likely(n)) {
-		size_t leftlen = *cur_edge - *seg;
-		size_t copysz = min_t(size_t, leftlen, n);
-		size_t stride;
-
-		memcpy(*seg, src, copysz);
-
-		n -= copysz;
-		src += copysz;
-		stride = !n ? ALIGN(copysz, 16) : copysz;
-		*seg += stride;
-		*wqe_sz += stride >> 4;
-		handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
-	}
-}
-
-static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
-{
-	struct mlx5_ib_cq *cq;
-	unsigned cur;
-
-	cur = wq->head - wq->tail;
-	if (likely(cur + nreq < wq->max_post))
-		return 0;
-
-	cq = to_mcq(ib_cq);
-	spin_lock(&cq->lock);
-	cur = wq->head - wq->tail;
-	spin_unlock(&cq->lock);
-
-	return cur + nreq >= wq->max_post;
-}
-
-static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
-					  u64 remote_addr, u32 rkey)
-{
-	rseg->raddr    = cpu_to_be64(remote_addr);
-	rseg->rkey     = cpu_to_be32(rkey);
-	rseg->reserved = 0;
-}
-
-static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
-			void **seg, int *size, void **cur_edge)
-{
-	struct mlx5_wqe_eth_seg *eseg = *seg;
-
-	memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
-
-	if (wr->send_flags & IB_SEND_IP_CSUM)
-		eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
-				 MLX5_ETH_WQE_L4_CSUM;
-
-	if (wr->opcode == IB_WR_LSO) {
-		struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
-		size_t left, copysz;
-		void *pdata = ud_wr->header;
-		size_t stride;
-
-		left = ud_wr->hlen;
-		eseg->mss = cpu_to_be16(ud_wr->mss);
-		eseg->inline_hdr.sz = cpu_to_be16(left);
-
-		/* memcpy_send_wqe should get a 16B align address. Hence, we
-		 * first copy up to the current edge and then, if needed,
-		 * fall-through to memcpy_send_wqe.
-		 */
-		copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
-			       left);
-		memcpy(eseg->inline_hdr.start, pdata, copysz);
-		stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
-			       sizeof(eseg->inline_hdr.start) + copysz, 16);
-		*size += stride / 16;
-		*seg += stride;
-
-		if (copysz < left) {
-			handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-			left -= copysz;
-			pdata += copysz;
-			memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata,
-					left);
-		}
-
-		return;
-	}
-
-	*seg += sizeof(struct mlx5_wqe_eth_seg);
-	*size += sizeof(struct mlx5_wqe_eth_seg) / 16;
-}
-
-static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
-			     const struct ib_send_wr *wr)
-{
-	memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
-	dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
-	dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
-}
-
-static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
-{
-	dseg->byte_count = cpu_to_be32(sg->length);
-	dseg->lkey       = cpu_to_be32(sg->lkey);
-	dseg->addr       = cpu_to_be64(sg->addr);
-}
-
-static u64 get_xlt_octo(u64 bytes)
-{
-	return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
-	       MLX5_IB_UMR_OCTOWORD;
-}
-
-static __be64 frwr_mkey_mask(bool atomic)
-{
-	u64 result;
-
-	result = MLX5_MKEY_MASK_LEN		|
-		MLX5_MKEY_MASK_PAGE_SIZE	|
-		MLX5_MKEY_MASK_START_ADDR	|
-		MLX5_MKEY_MASK_EN_RINVAL	|
-		MLX5_MKEY_MASK_KEY		|
-		MLX5_MKEY_MASK_LR		|
-		MLX5_MKEY_MASK_LW		|
-		MLX5_MKEY_MASK_RR		|
-		MLX5_MKEY_MASK_RW		|
-		MLX5_MKEY_MASK_SMALL_FENCE	|
-		MLX5_MKEY_MASK_FREE;
-
-	if (atomic)
-		result |= MLX5_MKEY_MASK_A;
-
-	return cpu_to_be64(result);
-}
-
-static __be64 sig_mkey_mask(void)
-{
-	u64 result;
-
-	result = MLX5_MKEY_MASK_LEN		|
-		MLX5_MKEY_MASK_PAGE_SIZE	|
-		MLX5_MKEY_MASK_START_ADDR	|
-		MLX5_MKEY_MASK_EN_SIGERR	|
-		MLX5_MKEY_MASK_EN_RINVAL	|
-		MLX5_MKEY_MASK_KEY		|
-		MLX5_MKEY_MASK_LR		|
-		MLX5_MKEY_MASK_LW		|
-		MLX5_MKEY_MASK_RR		|
-		MLX5_MKEY_MASK_RW		|
-		MLX5_MKEY_MASK_SMALL_FENCE	|
-		MLX5_MKEY_MASK_FREE		|
-		MLX5_MKEY_MASK_BSF_EN;
-
-	return cpu_to_be64(result);
-}
-
-static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
-			    struct mlx5_ib_mr *mr, u8 flags, bool atomic)
-{
-	int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
-
-	memset(umr, 0, sizeof(*umr));
-
-	umr->flags = flags;
-	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
-	umr->mkey_mask = frwr_mkey_mask(atomic);
-}
-
-static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
-{
-	memset(umr, 0, sizeof(*umr));
-	umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
-	umr->flags = MLX5_UMR_INLINE;
-}
-
-static __be64 get_umr_enable_mr_mask(void)
-{
-	u64 result;
-
-	result = MLX5_MKEY_MASK_KEY |
-		 MLX5_MKEY_MASK_FREE;
-
-	return cpu_to_be64(result);
-}
-
-static __be64 get_umr_disable_mr_mask(void)
-{
-	u64 result;
-
-	result = MLX5_MKEY_MASK_FREE;
-
-	return cpu_to_be64(result);
-}
-
-static __be64 get_umr_update_translation_mask(void)
-{
-	u64 result;
-
-	result = MLX5_MKEY_MASK_LEN |
-		 MLX5_MKEY_MASK_PAGE_SIZE |
-		 MLX5_MKEY_MASK_START_ADDR;
-
-	return cpu_to_be64(result);
-}
-
-static __be64 get_umr_update_access_mask(int atomic)
-{
-	u64 result;
-
-	result = MLX5_MKEY_MASK_LR |
-		 MLX5_MKEY_MASK_LW |
-		 MLX5_MKEY_MASK_RR |
-		 MLX5_MKEY_MASK_RW;
-
-	if (atomic)
-		result |= MLX5_MKEY_MASK_A;
-
-	return cpu_to_be64(result);
-}
-
-static __be64 get_umr_update_pd_mask(void)
-{
-	u64 result;
-
-	result = MLX5_MKEY_MASK_PD;
-
-	return cpu_to_be64(result);
-}
-
-static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
-{
-	if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
-	     MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) ||
-	    (mask & MLX5_MKEY_MASK_A &&
-	     MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
-		return -EPERM;
-	return 0;
-}
-
-static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
-			       struct mlx5_wqe_umr_ctrl_seg *umr,
-			       const struct ib_send_wr *wr, int atomic)
-{
-	const struct mlx5_umr_wr *umrwr = umr_wr(wr);
-
-	memset(umr, 0, sizeof(*umr));
-
-	if (!umrwr->ignore_free_state) {
-		if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
-			 /* fail if free */
-			umr->flags = MLX5_UMR_CHECK_FREE;
-		else
-			/* fail if not free */
-			umr->flags = MLX5_UMR_CHECK_NOT_FREE;
-	}
-
-	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
-	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
-		u64 offset = get_xlt_octo(umrwr->offset);
-
-		umr->xlt_offset = cpu_to_be16(offset & 0xffff);
-		umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
-		umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
-	}
-	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
-		umr->mkey_mask |= get_umr_update_translation_mask();
-	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
-		umr->mkey_mask |= get_umr_update_access_mask(atomic);
-		umr->mkey_mask |= get_umr_update_pd_mask();
-	}
-	if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
-		umr->mkey_mask |= get_umr_enable_mr_mask();
-	if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
-		umr->mkey_mask |= get_umr_disable_mr_mask();
-
-	if (!wr->num_sge)
-		umr->flags |= MLX5_UMR_INLINE;
-
-	return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
-}
-
-static u8 get_umr_flags(int acc)
-{
-	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
-	       (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
-	       (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
-	       (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
-		MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
-}
-
-static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
-			     struct mlx5_ib_mr *mr,
-			     u32 key, int access)
-{
-	int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1;
-
-	memset(seg, 0, sizeof(*seg));
-
-	if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
-		seg->log2_page_size = ilog2(mr->ibmr.page_size);
-	else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
-		/* KLMs take twice the size of MTTs */
-		ndescs *= 2;
-
-	seg->flags = get_umr_flags(access) | mr->access_mode;
-	seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
-	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
-	seg->start_addr = cpu_to_be64(mr->ibmr.iova);
-	seg->len = cpu_to_be64(mr->ibmr.length);
-	seg->xlt_oct_size = cpu_to_be32(ndescs);
-}
-
-static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
-{
-	memset(seg, 0, sizeof(*seg));
-	seg->status = MLX5_MKEY_STATUS_FREE;
-}
-
-static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg,
-				 const struct ib_send_wr *wr)
-{
-	const struct mlx5_umr_wr *umrwr = umr_wr(wr);
-
-	memset(seg, 0, sizeof(*seg));
-	if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
-		seg->status = MLX5_MKEY_STATUS_FREE;
-
-	seg->flags = convert_access(umrwr->access_flags);
-	if (umrwr->pd)
-		seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
-	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
-	    !umrwr->length)
-		seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
-
-	seg->start_addr = cpu_to_be64(umrwr->virt_addr);
-	seg->len = cpu_to_be64(umrwr->length);
-	seg->log2_page_size = umrwr->page_shift;
-	seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
-				       mlx5_mkey_variant(umrwr->mkey));
-}
-
-static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
-			     struct mlx5_ib_mr *mr,
-			     struct mlx5_ib_pd *pd)
-{
-	int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs);
-
-	dseg->addr = cpu_to_be64(mr->desc_map);
-	dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
-	dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
-}
-
-static __be32 send_ieth(const struct ib_send_wr *wr)
-{
-	switch (wr->opcode) {
-	case IB_WR_SEND_WITH_IMM:
-	case IB_WR_RDMA_WRITE_WITH_IMM:
-		return wr->ex.imm_data;
-
-	case IB_WR_SEND_WITH_INV:
-		return cpu_to_be32(wr->ex.invalidate_rkey);
-
-	default:
-		return 0;
-	}
-}
-
-static u8 calc_sig(void *wqe, int size)
-{
-	u8 *p = wqe;
-	u8 res = 0;
-	int i;
-
-	for (i = 0; i < size; i++)
-		res ^= p[i];
-
-	return ~res;
-}
-
-static u8 wq_sig(void *wqe)
-{
-	return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
-}
-
-static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
-			    void **wqe, int *wqe_sz, void **cur_edge)
-{
-	struct mlx5_wqe_inline_seg *seg;
-	size_t offset;
-	int inl = 0;
-	int i;
-
-	seg = *wqe;
-	*wqe += sizeof(*seg);
-	offset = sizeof(*seg);
-
-	for (i = 0; i < wr->num_sge; i++) {
-		size_t len  = wr->sg_list[i].length;
-		void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
-
-		inl += len;
-
-		if (unlikely(inl > qp->max_inline_data))
-			return -ENOMEM;
-
-		while (likely(len)) {
-			size_t leftlen;
-			size_t copysz;
+				  new_state, &ucmd, &resp, udata);
 
-			handle_post_send_edge(&qp->sq, wqe,
-					      *wqe_sz + (offset >> 4),
-					      cur_edge);
-
-			leftlen = *cur_edge - *wqe;
-			copysz = min_t(size_t, leftlen, len);
-
-			memcpy(*wqe, addr, copysz);
-			len -= copysz;
-			addr += copysz;
-			*wqe += copysz;
-			offset += copysz;
-		}
-	}
-
-	seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
-
-	*wqe_sz +=  ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
-
-	return 0;
-}
-
-static u16 prot_field_size(enum ib_signature_type type)
-{
-	switch (type) {
-	case IB_SIG_TYPE_T10_DIF:
-		return MLX5_DIF_SIZE;
-	default:
-		return 0;
-	}
-}
-
-static u8 bs_selector(int block_size)
-{
-	switch (block_size) {
-	case 512:	    return 0x1;
-	case 520:	    return 0x2;
-	case 4096:	    return 0x3;
-	case 4160:	    return 0x4;
-	case 1073741824:    return 0x5;
-	default:	    return 0;
-	}
-}
-
-static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
-			      struct mlx5_bsf_inl *inl)
-{
-	/* Valid inline section and allow BSF refresh */
-	inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
-				       MLX5_BSF_REFRESH_DIF);
-	inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
-	inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
-	/* repeating block */
-	inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
-	inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
-			MLX5_DIF_CRC : MLX5_DIF_IPCS;
-
-	if (domain->sig.dif.ref_remap)
-		inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
-
-	if (domain->sig.dif.app_escape) {
-		if (domain->sig.dif.ref_escape)
-			inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
-		else
-			inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
-	}
-
-	inl->dif_app_bitmask_check =
-		cpu_to_be16(domain->sig.dif.apptag_check_mask);
-}
-
-static int mlx5_set_bsf(struct ib_mr *sig_mr,
-			struct ib_sig_attrs *sig_attrs,
-			struct mlx5_bsf *bsf, u32 data_size)
-{
-	struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
-	struct mlx5_bsf_basic *basic = &bsf->basic;
-	struct ib_sig_domain *mem = &sig_attrs->mem;
-	struct ib_sig_domain *wire = &sig_attrs->wire;
-
-	memset(bsf, 0, sizeof(*bsf));
-
-	/* Basic + Extended + Inline */
-	basic->bsf_size_sbs = 1 << 7;
-	/* Input domain check byte mask */
-	basic->check_byte_mask = sig_attrs->check_mask;
-	basic->raw_data_size = cpu_to_be32(data_size);
-
-	/* Memory domain */
-	switch (sig_attrs->mem.sig_type) {
-	case IB_SIG_TYPE_NONE:
-		break;
-	case IB_SIG_TYPE_T10_DIF:
-		basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
-		basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
-		mlx5_fill_inl_bsf(mem, &bsf->m_inl);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* Wire domain */
-	switch (sig_attrs->wire.sig_type) {
-	case IB_SIG_TYPE_NONE:
-		break;
-	case IB_SIG_TYPE_T10_DIF:
-		if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
-		    mem->sig_type == wire->sig_type) {
-			/* Same block structure */
-			basic->bsf_size_sbs |= 1 << 4;
-			if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
-				basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
-			if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
-				basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
-			if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
-				basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
-		} else
-			basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
-
-		basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
-		mlx5_fill_inl_bsf(wire, &bsf->w_inl);
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int set_sig_data_segment(const struct ib_send_wr *send_wr,
-				struct ib_mr *sig_mr,
-				struct ib_sig_attrs *sig_attrs,
-				struct mlx5_ib_qp *qp, void **seg, int *size,
-				void **cur_edge)
-{
-	struct mlx5_bsf *bsf;
-	u32 data_len;
-	u32 data_key;
-	u64 data_va;
-	u32 prot_len = 0;
-	u32 prot_key = 0;
-	u64 prot_va = 0;
-	bool prot = false;
-	int ret;
-	int wqe_size;
-	struct mlx5_ib_mr *mr = to_mmr(sig_mr);
-	struct mlx5_ib_mr *pi_mr = mr->pi_mr;
-
-	data_len = pi_mr->data_length;
-	data_key = pi_mr->ibmr.lkey;
-	data_va = pi_mr->data_iova;
-	if (pi_mr->meta_ndescs) {
-		prot_len = pi_mr->meta_length;
-		prot_key = pi_mr->ibmr.lkey;
-		prot_va = pi_mr->pi_iova;
-		prot = true;
-	}
-
-	if (!prot || (data_key == prot_key && data_va == prot_va &&
-		      data_len == prot_len)) {
-		/**
-		 * Source domain doesn't contain signature information
-		 * or data and protection are interleaved in memory.
-		 * So need construct:
-		 *                  ------------------
-		 *                 |     data_klm     |
-		 *                  ------------------
-		 *                 |       BSF        |
-		 *                  ------------------
-		 **/
-		struct mlx5_klm *data_klm = *seg;
-
-		data_klm->bcount = cpu_to_be32(data_len);
-		data_klm->key = cpu_to_be32(data_key);
-		data_klm->va = cpu_to_be64(data_va);
-		wqe_size = ALIGN(sizeof(*data_klm), 64);
-	} else {
-		/**
-		 * Source domain contains signature information
-		 * So need construct a strided block format:
-		 *               ---------------------------
-		 *              |     stride_block_ctrl     |
-		 *               ---------------------------
-		 *              |          data_klm         |
-		 *               ---------------------------
-		 *              |          prot_klm         |
-		 *               ---------------------------
-		 *              |             BSF           |
-		 *               ---------------------------
-		 **/
-		struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
-		struct mlx5_stride_block_entry *data_sentry;
-		struct mlx5_stride_block_entry *prot_sentry;
-		u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
-		int prot_size;
-
-		sblock_ctrl = *seg;
-		data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
-		prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
-
-		prot_size = prot_field_size(sig_attrs->mem.sig_type);
-		if (!prot_size) {
-			pr_err("Bad block size given: %u\n", block_size);
-			return -EINVAL;
-		}
-		sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
-							    prot_size);
-		sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
-		sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
-		sblock_ctrl->num_entries = cpu_to_be16(2);
-
-		data_sentry->bcount = cpu_to_be16(block_size);
-		data_sentry->key = cpu_to_be32(data_key);
-		data_sentry->va = cpu_to_be64(data_va);
-		data_sentry->stride = cpu_to_be16(block_size);
-
-		prot_sentry->bcount = cpu_to_be16(prot_size);
-		prot_sentry->key = cpu_to_be32(prot_key);
-		prot_sentry->va = cpu_to_be64(prot_va);
-		prot_sentry->stride = cpu_to_be16(prot_size);
-
-		wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
-				 sizeof(*prot_sentry), 64);
-	}
-
-	*seg += wqe_size;
-	*size += wqe_size / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-
-	bsf = *seg;
-	ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
-	if (ret)
-		return -EINVAL;
-
-	*seg += sizeof(*bsf);
-	*size += sizeof(*bsf) / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-
-	return 0;
-}
-
-static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
-				 struct ib_mr *sig_mr, int access_flags,
-				 u32 size, u32 length, u32 pdn)
-{
-	u32 sig_key = sig_mr->rkey;
-	u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
-
-	memset(seg, 0, sizeof(*seg));
-
-	seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
-	seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
-	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
-				    MLX5_MKEY_BSF_EN | pdn);
-	seg->len = cpu_to_be64(length);
-	seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
-	seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
-}
-
-static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
-				u32 size)
-{
-	memset(umr, 0, sizeof(*umr));
-
-	umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
-	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
-	umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
-	umr->mkey_mask = sig_mkey_mask();
-}
-
-static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
-			 struct mlx5_ib_qp *qp, void **seg, int *size,
-			 void **cur_edge)
-{
-	const struct ib_reg_wr *wr = reg_wr(send_wr);
-	struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
-	struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
-	struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
-	u32 pdn = get_pd(qp)->pdn;
-	u32 xlt_size;
-	int region_len, ret;
-
-	if (unlikely(send_wr->num_sge != 0) ||
-	    unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
-	    unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
-	    unlikely(!sig_mr->sig->sig_status_checked))
-		return -EINVAL;
-
-	/* length of the protected region, data + protection */
-	region_len = pi_mr->ibmr.length;
-
-	/**
-	 * KLM octoword size - if protection was provided
-	 * then we use strided block format (3 octowords),
-	 * else we use single KLM (1 octoword)
-	 **/
-	if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
-		xlt_size = 0x30;
-	else
-		xlt_size = sizeof(struct mlx5_klm);
-
-	set_sig_umr_segment(*seg, xlt_size);
-	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
-	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-
-	set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
-			     pdn);
-	*seg += sizeof(struct mlx5_mkey_seg);
-	*size += sizeof(struct mlx5_mkey_seg) / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-
-	ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
-				   cur_edge);
-	if (ret)
-		return ret;
-
-	sig_mr->sig->sig_status_checked = false;
-	return 0;
-}
-
-static int set_psv_wr(struct ib_sig_domain *domain,
-		      u32 psv_idx, void **seg, int *size)
-{
-	struct mlx5_seg_set_psv *psv_seg = *seg;
-
-	memset(psv_seg, 0, sizeof(*psv_seg));
-	psv_seg->psv_num = cpu_to_be32(psv_idx);
-	switch (domain->sig_type) {
-	case IB_SIG_TYPE_NONE:
-		break;
-	case IB_SIG_TYPE_T10_DIF:
-		psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
-						     domain->sig.dif.app_tag);
-		psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
-		break;
-	default:
-		pr_err("Bad signature type (%d) is given.\n",
-		       domain->sig_type);
-		return -EINVAL;
-	}
-
-	*seg += sizeof(*psv_seg);
-	*size += sizeof(*psv_seg) / 16;
-
-	return 0;
-}
-
-static int set_reg_wr(struct mlx5_ib_qp *qp,
-		      const struct ib_reg_wr *wr,
-		      void **seg, int *size, void **cur_edge,
-		      bool check_not_free)
-{
-	struct mlx5_ib_mr *mr = to_mmr(wr->mr);
-	struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
-	struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
-	int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
-	bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
-	bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
-	u8 flags = 0;
-
-	if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) {
-		mlx5_ib_warn(to_mdev(qp->ibqp.device),
-			     "Fast update of %s for MR is disabled\n",
-			     (MLX5_CAP_GEN(dev->mdev,
-					   umr_modify_entity_size_disabled)) ?
-				     "entity size" :
-				     "atomic access");
-		return -EINVAL;
-	}
-
-	if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
-		mlx5_ib_warn(to_mdev(qp->ibqp.device),
-			     "Invalid IB_SEND_INLINE send flag\n");
-		return -EINVAL;
-	}
-
-	if (check_not_free)
-		flags |= MLX5_UMR_CHECK_NOT_FREE;
-	if (umr_inline)
-		flags |= MLX5_UMR_INLINE;
-
-	set_reg_umr_seg(*seg, mr, flags, atomic);
-	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
-	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-
-	set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
-	*seg += sizeof(struct mlx5_mkey_seg);
-	*size += sizeof(struct mlx5_mkey_seg) / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-
-	if (umr_inline) {
-		memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
-				mr_list_size);
-		*size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
-	} else {
-		set_reg_data_seg(*seg, mr, pd);
-		*seg += sizeof(struct mlx5_wqe_data_seg);
-		*size += (sizeof(struct mlx5_wqe_data_seg) / 16);
-	}
-	return 0;
-}
-
-static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
-			void **cur_edge)
-{
-	set_linv_umr_seg(*seg);
-	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
-	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-	set_linv_mkey_seg(*seg);
-	*seg += sizeof(struct mlx5_mkey_seg);
-	*size += sizeof(struct mlx5_mkey_seg) / 16;
-	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
-}
-
-static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
-{
-	__be32 *p = NULL;
-	int i, j;
-
-	pr_debug("dump WQE index %u:\n", idx);
-	for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
-		if ((i & 0xf) == 0) {
-			p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
-			pr_debug("WQBB at %p:\n", (void *)p);
-			j = 0;
-			idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
-		}
-		pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
-			 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
-			 be32_to_cpu(p[j + 3]));
-	}
-}
-
-static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
-		       struct mlx5_wqe_ctrl_seg **ctrl,
-		       const struct ib_send_wr *wr, unsigned int *idx,
-		       int *size, void **cur_edge, int nreq,
-		       bool send_signaled, bool solicited)
-{
-	if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
-		return -ENOMEM;
-
-	*idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
-	*seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
-	*ctrl = *seg;
-	*(uint32_t *)(*seg + 8) = 0;
-	(*ctrl)->imm = send_ieth(wr);
-	(*ctrl)->fm_ce_se = qp->sq_signal_bits |
-		(send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
-		(solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
-
-	*seg += sizeof(**ctrl);
-	*size = sizeof(**ctrl) / 16;
-	*cur_edge = qp->sq.cur_edge;
-
-	return 0;
-}
-
-static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
-		     struct mlx5_wqe_ctrl_seg **ctrl,
-		     const struct ib_send_wr *wr, unsigned *idx,
-		     int *size, void **cur_edge, int nreq)
-{
-	return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
-			   wr->send_flags & IB_SEND_SIGNALED,
-			   wr->send_flags & IB_SEND_SOLICITED);
-}
-
-static void finish_wqe(struct mlx5_ib_qp *qp,
-		       struct mlx5_wqe_ctrl_seg *ctrl,
-		       void *seg, u8 size, void *cur_edge,
-		       unsigned int idx, u64 wr_id, int nreq, u8 fence,
-		       u32 mlx5_opcode)
-{
-	u8 opmod = 0;
-
-	ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
-					     mlx5_opcode | ((u32)opmod << 24));
-	ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
-	ctrl->fm_ce_se |= fence;
-	if (unlikely(qp->wq_sig))
-		ctrl->signature = wq_sig(ctrl);
-
-	qp->sq.wrid[idx] = wr_id;
-	qp->sq.w_list[idx].opcode = mlx5_opcode;
-	qp->sq.wqe_head[idx] = qp->sq.head + nreq;
-	qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
-	qp->sq.w_list[idx].next = qp->sq.cur_post;
-
-	/* We save the edge which was possibly updated during the WQE
-	 * construction, into SQ's cache.
-	 */
-	seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
-	qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
-			  get_sq_edge(&qp->sq, qp->sq.cur_post &
-				      (qp->sq.wqe_cnt - 1)) :
-			  cur_edge;
-}
-
-static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
-			      const struct ib_send_wr **bad_wr, bool drain)
-{
-	struct mlx5_wqe_ctrl_seg *ctrl = NULL;  /* compiler warning */
-	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
-	struct mlx5_core_dev *mdev = dev->mdev;
-	struct ib_reg_wr reg_pi_wr;
-	struct mlx5_ib_qp *qp;
-	struct mlx5_ib_mr *mr;
-	struct mlx5_ib_mr *pi_mr;
-	struct mlx5_ib_mr pa_pi_mr;
-	struct ib_sig_attrs *sig_attrs;
-	struct mlx5_wqe_xrc_seg *xrc;
-	struct mlx5_bf *bf;
-	void *cur_edge;
-	int uninitialized_var(size);
-	unsigned long flags;
-	unsigned idx;
-	int err = 0;
-	int num_sge;
-	void *seg;
-	int nreq;
-	int i;
-	u8 next_fence = 0;
-	u8 fence;
-
-	if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
-		     !drain)) {
-		*bad_wr = wr;
-		return -EIO;
-	}
-
-	if (unlikely(ibqp->qp_type == IB_QPT_GSI))
-		return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
-
-	qp = to_mqp(ibqp);
-	bf = &qp->bf;
-
-	spin_lock_irqsave(&qp->sq.lock, flags);
-
-	for (nreq = 0; wr; nreq++, wr = wr->next) {
-		if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
-			mlx5_ib_warn(dev, "\n");
-			err = -EINVAL;
-			*bad_wr = wr;
-			goto out;
-		}
-
-		num_sge = wr->num_sge;
-		if (unlikely(num_sge > qp->sq.max_gs)) {
-			mlx5_ib_warn(dev, "\n");
-			err = -EINVAL;
-			*bad_wr = wr;
-			goto out;
-		}
-
-		err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
-				nreq);
-		if (err) {
-			mlx5_ib_warn(dev, "\n");
-			err = -ENOMEM;
-			*bad_wr = wr;
-			goto out;
-		}
-
-		if (wr->opcode == IB_WR_REG_MR ||
-		    wr->opcode == IB_WR_REG_MR_INTEGRITY) {
-			fence = dev->umr_fence;
-			next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
-		} else  {
-			if (wr->send_flags & IB_SEND_FENCE) {
-				if (qp->next_fence)
-					fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
-				else
-					fence = MLX5_FENCE_MODE_FENCE;
-			} else {
-				fence = qp->next_fence;
-			}
-		}
-
-		switch (ibqp->qp_type) {
-		case IB_QPT_XRC_INI:
-			xrc = seg;
-			seg += sizeof(*xrc);
-			size += sizeof(*xrc) / 16;
-			/* fall through */
-		case IB_QPT_RC:
-			switch (wr->opcode) {
-			case IB_WR_RDMA_READ:
-			case IB_WR_RDMA_WRITE:
-			case IB_WR_RDMA_WRITE_WITH_IMM:
-				set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
-					      rdma_wr(wr)->rkey);
-				seg += sizeof(struct mlx5_wqe_raddr_seg);
-				size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
-				break;
-
-			case IB_WR_ATOMIC_CMP_AND_SWP:
-			case IB_WR_ATOMIC_FETCH_AND_ADD:
-			case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
-				mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
-				err = -ENOSYS;
-				*bad_wr = wr;
-				goto out;
-
-			case IB_WR_LOCAL_INV:
-				qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
-				ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);
-				set_linv_wr(qp, &seg, &size, &cur_edge);
-				num_sge = 0;
-				break;
-
-			case IB_WR_REG_MR:
-				qp->sq.wr_data[idx] = IB_WR_REG_MR;
-				ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
-				err = set_reg_wr(qp, reg_wr(wr), &seg, &size,
-						 &cur_edge, true);
-				if (err) {
-					*bad_wr = wr;
-					goto out;
-				}
-				num_sge = 0;
-				break;
-
-			case IB_WR_REG_MR_INTEGRITY:
-				qp->sq.wr_data[idx] = IB_WR_REG_MR_INTEGRITY;
-
-				mr = to_mmr(reg_wr(wr)->mr);
-				pi_mr = mr->pi_mr;
-
-				if (pi_mr) {
-					memset(&reg_pi_wr, 0,
-					       sizeof(struct ib_reg_wr));
-
-					reg_pi_wr.mr = &pi_mr->ibmr;
-					reg_pi_wr.access = reg_wr(wr)->access;
-					reg_pi_wr.key = pi_mr->ibmr.rkey;
-
-					ctrl->imm = cpu_to_be32(reg_pi_wr.key);
-					/* UMR for data + prot registration */
-					err = set_reg_wr(qp, &reg_pi_wr, &seg,
-							 &size, &cur_edge,
-							 false);
-					if (err) {
-						*bad_wr = wr;
-						goto out;
-					}
-					finish_wqe(qp, ctrl, seg, size,
-						   cur_edge, idx, wr->wr_id,
-						   nreq, fence,
-						   MLX5_OPCODE_UMR);
-
-					err = begin_wqe(qp, &seg, &ctrl, wr,
-							&idx, &size, &cur_edge,
-							nreq);
-					if (err) {
-						mlx5_ib_warn(dev, "\n");
-						err = -ENOMEM;
-						*bad_wr = wr;
-						goto out;
-					}
-				} else {
-					memset(&pa_pi_mr, 0,
-					       sizeof(struct mlx5_ib_mr));
-					/* No UMR, use local_dma_lkey */
-					pa_pi_mr.ibmr.lkey =
-						mr->ibmr.pd->local_dma_lkey;
-
-					pa_pi_mr.ndescs = mr->ndescs;
-					pa_pi_mr.data_length = mr->data_length;
-					pa_pi_mr.data_iova = mr->data_iova;
-					if (mr->meta_ndescs) {
-						pa_pi_mr.meta_ndescs =
-							mr->meta_ndescs;
-						pa_pi_mr.meta_length =
-							mr->meta_length;
-						pa_pi_mr.pi_iova = mr->pi_iova;
-					}
-
-					pa_pi_mr.ibmr.length = mr->ibmr.length;
-					mr->pi_mr = &pa_pi_mr;
-				}
-				ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
-				/* UMR for sig MR */
-				err = set_pi_umr_wr(wr, qp, &seg, &size,
-						    &cur_edge);
-				if (err) {
-					mlx5_ib_warn(dev, "\n");
-					*bad_wr = wr;
-					goto out;
-				}
-				finish_wqe(qp, ctrl, seg, size, cur_edge, idx,
-					   wr->wr_id, nreq, fence,
-					   MLX5_OPCODE_UMR);
-
-				/*
-				 * SET_PSV WQEs are not signaled and solicited
-				 * on error
-				 */
-				sig_attrs = mr->ibmr.sig_attrs;
-				err = __begin_wqe(qp, &seg, &ctrl, wr, &idx,
-						  &size, &cur_edge, nreq, false,
-						  true);
-				if (err) {
-					mlx5_ib_warn(dev, "\n");
-					err = -ENOMEM;
-					*bad_wr = wr;
-					goto out;
-				}
-				err = set_psv_wr(&sig_attrs->mem,
-						 mr->sig->psv_memory.psv_idx,
-						 &seg, &size);
-				if (err) {
-					mlx5_ib_warn(dev, "\n");
-					*bad_wr = wr;
-					goto out;
-				}
-				finish_wqe(qp, ctrl, seg, size, cur_edge, idx,
-					   wr->wr_id, nreq, next_fence,
-					   MLX5_OPCODE_SET_PSV);
-
-				err = __begin_wqe(qp, &seg, &ctrl, wr, &idx,
-						  &size, &cur_edge, nreq, false,
-						  true);
-				if (err) {
-					mlx5_ib_warn(dev, "\n");
-					err = -ENOMEM;
-					*bad_wr = wr;
-					goto out;
-				}
-				err = set_psv_wr(&sig_attrs->wire,
-						 mr->sig->psv_wire.psv_idx,
-						 &seg, &size);
-				if (err) {
-					mlx5_ib_warn(dev, "\n");
-					*bad_wr = wr;
-					goto out;
-				}
-				finish_wqe(qp, ctrl, seg, size, cur_edge, idx,
-					   wr->wr_id, nreq, next_fence,
-					   MLX5_OPCODE_SET_PSV);
-
-				qp->next_fence =
-					MLX5_FENCE_MODE_INITIATOR_SMALL;
-				num_sge = 0;
-				goto skip_psv;
-
-			default:
-				break;
-			}
-			break;
-
-		case IB_QPT_UC:
-			switch (wr->opcode) {
-			case IB_WR_RDMA_WRITE:
-			case IB_WR_RDMA_WRITE_WITH_IMM:
-				set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
-					      rdma_wr(wr)->rkey);
-				seg  += sizeof(struct mlx5_wqe_raddr_seg);
-				size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
-				break;
-
-			default:
-				break;
-			}
-			break;
-
-		case IB_QPT_SMI:
-			if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
-				mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
-				err = -EPERM;
-				*bad_wr = wr;
-				goto out;
-			}
-			/* fall through */
-		case MLX5_IB_QPT_HW_GSI:
-			set_datagram_seg(seg, wr);
-			seg += sizeof(struct mlx5_wqe_datagram_seg);
-			size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
-			handle_post_send_edge(&qp->sq, &seg, size, &cur_edge);
-
-			break;
-		case IB_QPT_UD:
-			set_datagram_seg(seg, wr);
-			seg += sizeof(struct mlx5_wqe_datagram_seg);
-			size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
-			handle_post_send_edge(&qp->sq, &seg, size, &cur_edge);
-
-			/* handle qp that supports ud offload */
-			if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
-				struct mlx5_wqe_eth_pad *pad;
-
-				pad = seg;
-				memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
-				seg += sizeof(struct mlx5_wqe_eth_pad);
-				size += sizeof(struct mlx5_wqe_eth_pad) / 16;
-				set_eth_seg(wr, qp, &seg, &size, &cur_edge);
-				handle_post_send_edge(&qp->sq, &seg, size,
-						      &cur_edge);
-			}
-			break;
-		case MLX5_IB_QPT_REG_UMR:
-			if (wr->opcode != MLX5_IB_WR_UMR) {
-				err = -EINVAL;
-				mlx5_ib_warn(dev, "bad opcode\n");
-				goto out;
-			}
-			qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
-			ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey);
-			err = set_reg_umr_segment(dev, seg, wr, !!(MLX5_CAP_GEN(mdev, atomic)));
-			if (unlikely(err))
-				goto out;
-			seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
-			size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
-			handle_post_send_edge(&qp->sq, &seg, size, &cur_edge);
-			set_reg_mkey_segment(seg, wr);
-			seg += sizeof(struct mlx5_mkey_seg);
-			size += sizeof(struct mlx5_mkey_seg) / 16;
-			handle_post_send_edge(&qp->sq, &seg, size, &cur_edge);
-			break;
-
-		default:
-			break;
-		}
-
-		if (wr->send_flags & IB_SEND_INLINE && num_sge) {
-			err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
-			if (unlikely(err)) {
-				mlx5_ib_warn(dev, "\n");
-				*bad_wr = wr;
-				goto out;
-			}
-		} else {
-			for (i = 0; i < num_sge; i++) {
-				handle_post_send_edge(&qp->sq, &seg, size,
-						      &cur_edge);
-				if (likely(wr->sg_list[i].length)) {
-					set_data_ptr_seg
-					((struct mlx5_wqe_data_seg *)seg,
-					 wr->sg_list + i);
-					size += sizeof(struct mlx5_wqe_data_seg) / 16;
-					seg += sizeof(struct mlx5_wqe_data_seg);
-				}
-			}
-		}
-
-		qp->next_fence = next_fence;
-		finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq,
-			   fence, mlx5_ib_opcode[wr->opcode]);
-skip_psv:
-		if (0)
-			dump_wqe(qp, idx, size);
-	}
-
-out:
-	if (likely(nreq)) {
-		qp->sq.head += nreq;
-
-		/* Make sure that descriptors are written before
-		 * updating doorbell record and ringing the doorbell
-		 */
-		wmb();
-
-		qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
-
-		/* Make sure doorbell record is visible to the HCA before
-		 * we hit doorbell */
-		wmb();
-
-		mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
-		/* Make sure doorbells don't leak out of SQ spinlock
-		 * and reach the HCA out of order.
-		 */
-		bf->offset ^= bf->buf_size;
-	}
-
-	spin_unlock_irqrestore(&qp->sq.lock, flags);
-
-	return err;
-}
-
-int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
-		      const struct ib_send_wr **bad_wr)
-{
-	return _mlx5_ib_post_send(ibqp, wr, bad_wr, false);
-}
-
-static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size)
-{
-	sig->signature = calc_sig(sig, size);
-}
-
-static int _mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
-		      const struct ib_recv_wr **bad_wr, bool drain)
-{
-	struct mlx5_ib_qp *qp = to_mqp(ibqp);
-	struct mlx5_wqe_data_seg *scat;
-	struct mlx5_rwqe_sig *sig;
-	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
-	struct mlx5_core_dev *mdev = dev->mdev;
-	unsigned long flags;
-	int err = 0;
-	int nreq;
-	int ind;
-	int i;
-
-	if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
-		     !drain)) {
-		*bad_wr = wr;
-		return -EIO;
-	}
-
-	if (unlikely(ibqp->qp_type == IB_QPT_GSI))
-		return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
-
-	spin_lock_irqsave(&qp->rq.lock, flags);
-
-	ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
-
-	for (nreq = 0; wr; nreq++, wr = wr->next) {
-		if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
-			err = -ENOMEM;
-			*bad_wr = wr;
-			goto out;
-		}
-
-		if (unlikely(wr->num_sge > qp->rq.max_gs)) {
-			err = -EINVAL;
-			*bad_wr = wr;
-			goto out;
-		}
-
-		scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
-		if (qp->wq_sig)
-			scat++;
-
-		for (i = 0; i < wr->num_sge; i++)
-			set_data_ptr_seg(scat + i, wr->sg_list + i);
-
-		if (i < qp->rq.max_gs) {
-			scat[i].byte_count = 0;
-			scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
-			scat[i].addr       = 0;
-		}
-
-		if (qp->wq_sig) {
-			sig = (struct mlx5_rwqe_sig *)scat;
-			set_sig_seg(sig, (qp->rq.max_gs + 1) << 2);
-		}
-
-		qp->rq.wrid[ind] = wr->wr_id;
-
-		ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
-	}
+	/* resp.response_length is set in ECE supported flows only */
+	if (!err && resp.response_length &&
+	    udata->outlen >= resp.response_length)
+		/* Return -EFAULT to the user and expect him to destroy QP. */
+		err = ib_copy_to_udata(udata, &resp, resp.response_length);
 
 out:
-	if (likely(nreq)) {
-		qp->rq.head += nreq;
-
-		/* Make sure that descriptors are written before
-		 * doorbell record.
-		 */
-		wmb();
-
-		*qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
-	}
-
-	spin_unlock_irqrestore(&qp->rq.lock, flags);
-
+	mutex_unlock(&qp->mutex);
 	return err;
 }
 
-int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
-		      const struct ib_recv_wr **bad_wr)
-{
-	return _mlx5_ib_post_recv(ibqp, wr, bad_wr, false);
-}
-
 static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
 {
 	switch (mlx5_state) {
@@ -5521,50 +4357,35 @@ static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
 	}
 }
 
-static int to_ib_qp_access_flags(int mlx5_flags)
-{
-	int ib_flags = 0;
-
-	if (mlx5_flags & MLX5_QP_BIT_RRE)
-		ib_flags |= IB_ACCESS_REMOTE_READ;
-	if (mlx5_flags & MLX5_QP_BIT_RWE)
-		ib_flags |= IB_ACCESS_REMOTE_WRITE;
-	if (mlx5_flags & MLX5_QP_BIT_RAE)
-		ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
-
-	return ib_flags;
-}
-
 static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
-			    struct rdma_ah_attr *ah_attr,
-			    struct mlx5_qp_path *path)
+			    struct rdma_ah_attr *ah_attr, void *path)
 {
+	int port = MLX5_GET(ads, path, vhca_port_num);
+	int static_rate;
 
 	memset(ah_attr, 0, sizeof(*ah_attr));
 
-	if (!path->port || path->port > ibdev->num_ports)
+	if (!port || port > ibdev->num_ports)
 		return;
 
-	ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, path->port);
+	ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port);
 
-	rdma_ah_set_port_num(ah_attr, path->port);
-	rdma_ah_set_sl(ah_attr, path->dci_cfi_prio_sl & 0xf);
+	rdma_ah_set_port_num(ah_attr, port);
+	rdma_ah_set_sl(ah_attr, MLX5_GET(ads, path, sl));
 
-	rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
-	rdma_ah_set_path_bits(ah_attr, path->grh_mlid & 0x7f);
-	rdma_ah_set_static_rate(ah_attr,
-				path->static_rate ? path->static_rate - 5 : 0);
+	rdma_ah_set_dlid(ah_attr, MLX5_GET(ads, path, rlid));
+	rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid));
 
-	if (path->grh_mlid & (1 << 7) ||
+	static_rate = MLX5_GET(ads, path, stat_rate);
+	rdma_ah_set_static_rate(ah_attr, static_rate ? static_rate - 5 : 0);
+	if (MLX5_GET(ads, path, grh) ||
 	    ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
-		u32 tc_fl = be32_to_cpu(path->tclass_flowlabel);
-
-		rdma_ah_set_grh(ah_attr, NULL,
-				tc_fl & 0xfffff,
-				path->mgid_index,
-				path->hop_limit,
-				(tc_fl >> 20) & 0xff);
-		rdma_ah_set_dgid_raw(ah_attr, path->rgid);
+		rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label),
+				MLX5_GET(ads, path, src_addr_index),
+				MLX5_GET(ads, path, hop_limit),
+				MLX5_GET(ads, path, tclass));
+		memcpy(ah_attr, MLX5_ADDR_OF(ads, path, rgid_rip),
+		       MLX5_FLD_SZ_BYTES(ads, rgid_rip));
 	}
 }
 
@@ -5686,10 +4507,9 @@ static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			 struct ib_qp_attr *qp_attr)
 {
 	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
-	struct mlx5_qp_context *context;
-	int mlx5_state;
+	void *qpc, *pri_path, *alt_path;
 	u32 *outb;
-	int err = 0;
+	int err;
 
 	outb = kzalloc(outlen, GFP_KERNEL);
 	if (!outb)
@@ -5699,47 +4519,46 @@ static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	if (err)
 		goto out;
 
-	/* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
-	context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);
+	qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);
 
-	mlx5_state = be32_to_cpu(context->flags) >> 28;
+	qp->state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));
+	if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)
+		qp_attr->sq_draining = 1;
 
-	qp->state		     = to_ib_qp_state(mlx5_state);
-	qp_attr->path_mtu	     = context->mtu_msgmax >> 5;
-	qp_attr->path_mig_state	     =
-		to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
-	qp_attr->qkey		     = be32_to_cpu(context->qkey);
-	qp_attr->rq_psn		     = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff;
-	qp_attr->sq_psn		     = be32_to_cpu(context->next_send_psn) & 0xffffff;
-	qp_attr->dest_qp_num	     = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff;
-	qp_attr->qp_access_flags     =
-		to_ib_qp_access_flags(be32_to_cpu(context->params2));
+	qp_attr->path_mtu = MLX5_GET(qpc, qpc, mtu);
+	qp_attr->path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));
+	qp_attr->qkey = MLX5_GET(qpc, qpc, q_key);
+	qp_attr->rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);
+	qp_attr->sq_psn = MLX5_GET(qpc, qpc, next_send_psn);
+	qp_attr->dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);
 
-	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
-		to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
-		to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
-		qp_attr->alt_pkey_index =
-			be16_to_cpu(context->alt_path.pkey_index);
-		qp_attr->alt_port_num	=
-			rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
-	}
+	if (MLX5_GET(qpc, qpc, rre))
+		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ;
+	if (MLX5_GET(qpc, qpc, rwe))
+		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_WRITE;
+	if (MLX5_GET(qpc, qpc, rae))
+		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_ATOMIC;
 
-	qp_attr->pkey_index = be16_to_cpu(context->pri_path.pkey_index);
-	qp_attr->port_num = context->pri_path.port;
+	qp_attr->max_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_sra_max);
+	qp_attr->max_dest_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_rra_max);
+	qp_attr->min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);
+	qp_attr->retry_cnt = MLX5_GET(qpc, qpc, retry_count);
+	qp_attr->rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);
 
-	/* qp_attr->en_sqd_async_notify is only applicable in modify qp */
-	qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING;
+	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
+	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
 
-	qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7);
+	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
+		to_rdma_ah_attr(dev, &qp_attr->ah_attr, pri_path);
+		to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, alt_path);
+		qp_attr->alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);
+		qp_attr->alt_port_num = MLX5_GET(ads, alt_path, vhca_port_num);
+	}
 
-	qp_attr->max_dest_rd_atomic =
-		1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
-	qp_attr->min_rnr_timer	    =
-		(be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
-	qp_attr->timeout	    = context->pri_path.ackto_lt >> 3;
-	qp_attr->retry_cnt	    = (be32_to_cpu(context->params1) >> 16) & 0x7;
-	qp_attr->rnr_retry	    = (be32_to_cpu(context->params1) >> 13) & 0x7;
-	qp_attr->alt_timeout	    = context->alt_path.ackto_lt >> 3;
+	qp_attr->pkey_index = MLX5_GET(ads, pri_path, pkey_index);
+	qp_attr->port_num = MLX5_GET(ads, pri_path, vhca_port_num);
+	qp_attr->timeout = MLX5_GET(ads, pri_path, ack_timeout);
+	qp_attr->alt_timeout = MLX5_GET(ads, alt_path, ack_timeout);
 
 out:
 	kfree(outb);
@@ -5830,14 +4649,14 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
 	memset(qp_attr, 0, sizeof(*qp_attr));
 
-	if (unlikely(qp->qp_sub_type == MLX5_IB_QPT_DCT))
+	if (unlikely(qp->type == MLX5_IB_QPT_DCT))
 		return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
 					    qp_attr_mask, qp_init_attr);
 
 	mutex_lock(&qp->mutex);
 
 	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
-	    qp->flags & MLX5_IB_QP_UNDERLAY) {
+	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
 		if (err)
 			goto out;
@@ -5871,18 +4690,7 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 
 	qp_init_attr->cap	     = qp_attr->cap;
 
-	qp_init_attr->create_flags = 0;
-	if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
-		qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
-
-	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
-		qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL;
-	if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
-		qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND;
-	if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
-		qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV;
-	if (qp->flags & MLX5_IB_QP_SQPN_QP1)
-		qp_init_attr->create_flags |= MLX5_IB_QP_CREATE_SQPN_QP1;
+	qp_init_attr->create_flags = qp->flags;
 
 	qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
 		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
@@ -6543,7 +5351,7 @@ void mlx5_ib_drain_sq(struct ib_qp *qp)
 	sdrain.cqe.done = mlx5_ib_drain_qp_done;
 	init_completion(&sdrain.done);
 
-	ret = _mlx5_ib_post_send(qp, &swr.wr, &bad_swr, true);
+	ret = mlx5_ib_post_send_drain(qp, &swr.wr, &bad_swr);
 	if (ret) {
 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
 		return;
@@ -6573,7 +5381,7 @@ void mlx5_ib_drain_rq(struct ib_qp *qp)
 	rdrain.cqe.done = mlx5_ib_drain_qp_done;
 	init_completion(&rdrain.done);
 
-	ret = _mlx5_ib_post_recv(qp, &rwr, &bad_rwr, true);
+	ret = mlx5_ib_post_recv_drain(qp, &rwr, &bad_rwr);
 	if (ret) {
 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
 		return;
diff --git a/drivers/infiniband/hw/mlx5/qp.h b/drivers/infiniband/hw/mlx5/qp.h
index ad9d76e3e18a..82ea2b94dfa6 100644
--- a/drivers/infiniband/hw/mlx5/qp.h
+++ b/drivers/infiniband/hw/mlx5/qp.h
@@ -13,10 +13,10 @@ void mlx5_cleanup_qp_table(struct mlx5_ib_dev *dev);
 
 int mlx5_core_create_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *qp,
 			 u32 *in, int inlen, u32 *out, int outlen);
-int mlx5_core_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
-			u32 *in, int inlen);
+int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
+		       u32 *in, int inlen, u32 *out);
 int mlx5_core_qp_modify(struct mlx5_ib_dev *dev, u16 opcode, u32 opt_param_mask,
-			void *qpc, struct mlx5_core_qp *qp);
+			void *qpc, struct mlx5_core_qp *qp, u32 *ece);
 int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp);
 int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct);
 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c
index ea62735042f0..c19d91d6dce8 100644
--- a/drivers/infiniband/hw/mlx5/qpc.c
+++ b/drivers/infiniband/hw/mlx5/qpc.c
@@ -236,16 +236,16 @@ err_cmd:
 	return err;
 }
 
-int mlx5_core_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
-			u32 *in, int inlen)
+int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
+		       u32 *in, int inlen, u32 *out)
 {
-	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
 	u32 din[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
 	int err;
 
 	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
 
-	err = mlx5_cmd_exec(dev->mdev, in, inlen, out, sizeof(out));
+	err = mlx5_cmd_exec(dev->mdev, in, inlen, out,
+			    MLX5_ST_SZ_BYTES(create_qp_out));
 	if (err)
 		return err;
 
@@ -341,9 +341,30 @@ static void mbox_free(struct mbox_info *mbox)
 	kfree(mbox->out);
 }
 
+static int get_ece_from_mbox(void *out, u16 opcode)
+{
+	int ece = 0;
+
+	switch (opcode) {
+	case MLX5_CMD_OP_INIT2RTR_QP:
+		ece = MLX5_GET(init2rtr_qp_out, out, ece);
+		break;
+	case MLX5_CMD_OP_RTR2RTS_QP:
+		ece = MLX5_GET(rtr2rts_qp_out, out, ece);
+		break;
+	case MLX5_CMD_OP_RTS2RTS_QP:
+		ece = MLX5_GET(rts2rts_qp_out, out, ece);
+		break;
+	default:
+		break;
+	}
+
+	return ece;
+}
+
 static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,
 				u32 opt_param_mask, void *qpc,
-				struct mbox_info *mbox, u16 uid)
+				struct mbox_info *mbox, u16 uid, u32 ece)
 {
 	mbox->out = NULL;
 	mbox->in = NULL;
@@ -391,18 +412,21 @@ static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,
 			return -ENOMEM;
 		MOD_QP_IN_SET_QPC(init2rtr_qp, mbox->in, opcode, qpn,
 				  opt_param_mask, qpc, uid);
+		MLX5_SET(init2rtr_qp_in, mbox->in, ece, ece);
 		break;
 	case MLX5_CMD_OP_RTR2RTS_QP:
 		if (MBOX_ALLOC(mbox, rtr2rts_qp))
 			return -ENOMEM;
 		MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox->in, opcode, qpn,
 				  opt_param_mask, qpc, uid);
+		MLX5_SET(rtr2rts_qp_in, mbox->in, ece, ece);
 		break;
 	case MLX5_CMD_OP_RTS2RTS_QP:
 		if (MBOX_ALLOC(mbox, rts2rts_qp))
 			return -ENOMEM;
 		MOD_QP_IN_SET_QPC(rts2rts_qp, mbox->in, opcode, qpn,
 				  opt_param_mask, qpc, uid);
+		MLX5_SET(rts2rts_qp_in, mbox->in, ece, ece);
 		break;
 	case MLX5_CMD_OP_SQERR2RTS_QP:
 		if (MBOX_ALLOC(mbox, sqerr2rts_qp))
@@ -423,18 +447,22 @@ static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,
 }
 
 int mlx5_core_qp_modify(struct mlx5_ib_dev *dev, u16 opcode, u32 opt_param_mask,
-			void *qpc, struct mlx5_core_qp *qp)
+			void *qpc, struct mlx5_core_qp *qp, u32 *ece)
 {
 	struct mbox_info mbox;
 	int err;
 
-	err = modify_qp_mbox_alloc(dev->mdev, opcode, qp->qpn,
-				   opt_param_mask, qpc, &mbox, qp->uid);
+	err = modify_qp_mbox_alloc(dev->mdev, opcode, qp->qpn, opt_param_mask,
+				   qpc, &mbox, qp->uid, (ece) ? *ece : 0);
 	if (err)
 		return err;
 
 	err = mlx5_cmd_exec(dev->mdev, mbox.in, mbox.inlen, mbox.out,
 			    mbox.outlen);
+
+	if (ece)
+		*ece = get_ece_from_mbox(mbox.out, opcode);
+
 	mbox_free(&mbox);
 	return err;
 }
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index b1a8a9175040..6d1ff13d2283 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -310,12 +310,18 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,
 	srq->msrq.event = mlx5_ib_srq_event;
 	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
 
-	if (udata)
-		if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) {
+	if (udata) {
+		struct mlx5_ib_create_srq_resp resp = {
+			.srqn = srq->msrq.srqn,
+		};
+
+		if (ib_copy_to_udata(udata, &resp, min(udata->outlen,
+				     sizeof(resp)))) {
 			mlx5_ib_dbg(dev, "copy to user failed\n");
 			err = -EFAULT;
 			goto err_core;
 		}
+	}
 
 	init_attr->attr.max_wr = srq->msrq.max - 1;
 
diff --git a/drivers/infiniband/hw/mlx5/srq_cmd.c b/drivers/infiniband/hw/mlx5/srq_cmd.c
index c851570791af..6f5eadc4d183 100644
--- a/drivers/infiniband/hw/mlx5/srq_cmd.c
+++ b/drivers/infiniband/hw/mlx5/srq_cmd.c
@@ -132,38 +132,33 @@ static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 
 static int destroy_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
 {
-	u32 srq_in[MLX5_ST_SZ_DW(destroy_srq_in)] = {0};
-	u32 srq_out[MLX5_ST_SZ_DW(destroy_srq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_srq_in)] = {};
 
-	MLX5_SET(destroy_srq_in, srq_in, opcode,
-		 MLX5_CMD_OP_DESTROY_SRQ);
-	MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn);
-	MLX5_SET(destroy_srq_in, srq_in, uid, srq->uid);
+	MLX5_SET(destroy_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_SRQ);
+	MLX5_SET(destroy_srq_in, in, srqn, srq->srqn);
+	MLX5_SET(destroy_srq_in, in, uid, srq->uid);
 
-	return mlx5_cmd_exec(dev->mdev, srq_in, sizeof(srq_in), srq_out,
-			     sizeof(srq_out));
+	return mlx5_cmd_exec_in(dev->mdev, destroy_srq, in);
 }
 
 static int arm_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 		       u16 lwm, int is_srq)
 {
-	u32 srq_in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
-	u32 srq_out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
 
-	MLX5_SET(arm_rq_in, srq_in, opcode, MLX5_CMD_OP_ARM_RQ);
-	MLX5_SET(arm_rq_in, srq_in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
-	MLX5_SET(arm_rq_in, srq_in, srq_number, srq->srqn);
-	MLX5_SET(arm_rq_in, srq_in, lwm,      lwm);
-	MLX5_SET(arm_rq_in, srq_in, uid, srq->uid);
+	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
+	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
+	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
+	MLX5_SET(arm_rq_in, in, lwm, lwm);
+	MLX5_SET(arm_rq_in, in, uid, srq->uid);
 
-	return mlx5_cmd_exec(dev->mdev, srq_in, sizeof(srq_in), srq_out,
-			     sizeof(srq_out));
+	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
 }
 
 static int query_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 			 struct mlx5_srq_attr *out)
 {
-	u32 srq_in[MLX5_ST_SZ_DW(query_srq_in)] = {0};
+	u32 in[MLX5_ST_SZ_DW(query_srq_in)] = {};
 	u32 *srq_out;
 	void *srqc;
 	int err;
@@ -172,11 +167,9 @@ static int query_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 	if (!srq_out)
 		return -ENOMEM;
 
-	MLX5_SET(query_srq_in, srq_in, opcode,
-		 MLX5_CMD_OP_QUERY_SRQ);
-	MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn);
-	err = mlx5_cmd_exec(dev->mdev, srq_in, sizeof(srq_in), srq_out,
-			    MLX5_ST_SZ_BYTES(query_srq_out));
+	MLX5_SET(query_srq_in, in, opcode, MLX5_CMD_OP_QUERY_SRQ);
+	MLX5_SET(query_srq_in, in, srqn, srq->srqn);
+	err = mlx5_cmd_exec_inout(dev->mdev, query_srq, in, srq_out);
 	if (err)
 		goto out;
 
@@ -234,39 +227,35 @@ out:
 static int destroy_xrc_srq_cmd(struct mlx5_ib_dev *dev,
 			       struct mlx5_core_srq *srq)
 {
-	u32 xrcsrq_in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)]   = {0};
-	u32 xrcsrq_out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {};
 
-	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, opcode,
-		 MLX5_CMD_OP_DESTROY_XRC_SRQ);
-	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
-	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, uid, srq->uid);
+	MLX5_SET(destroy_xrc_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRC_SRQ);
+	MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, srq->srqn);
+	MLX5_SET(destroy_xrc_srq_in, in, uid, srq->uid);
 
-	return mlx5_cmd_exec(dev->mdev, xrcsrq_in, sizeof(xrcsrq_in),
-			     xrcsrq_out, sizeof(xrcsrq_out));
+	return mlx5_cmd_exec_in(dev->mdev, destroy_xrc_srq, in);
 }
 
 static int arm_xrc_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 			   u16 lwm)
 {
-	u32 xrcsrq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)]   = {0};
-	u32 xrcsrq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {};
 
-	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);
-	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, op_mod,   MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
-	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
-	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, lwm,      lwm);
-	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, uid, srq->uid);
+	MLX5_SET(arm_xrc_srq_in, in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ);
+	MLX5_SET(arm_xrc_srq_in, in, op_mod,
+		 MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
+	MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, srq->srqn);
+	MLX5_SET(arm_xrc_srq_in, in, lwm, lwm);
+	MLX5_SET(arm_xrc_srq_in, in, uid, srq->uid);
 
-	return  mlx5_cmd_exec(dev->mdev, xrcsrq_in, sizeof(xrcsrq_in),
-			      xrcsrq_out, sizeof(xrcsrq_out));
+	return  mlx5_cmd_exec_in(dev->mdev, arm_xrc_srq, in);
 }
 
 static int query_xrc_srq_cmd(struct mlx5_ib_dev *dev,
 			     struct mlx5_core_srq *srq,
 			     struct mlx5_srq_attr *out)
 {
-	u32 xrcsrq_in[MLX5_ST_SZ_DW(query_xrc_srq_in)];
+	u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {};
 	u32 *xrcsrq_out;
 	void *xrc_srqc;
 	int err;
@@ -274,14 +263,11 @@ static int query_xrc_srq_cmd(struct mlx5_ib_dev *dev,
 	xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL);
 	if (!xrcsrq_out)
 		return -ENOMEM;
-	memset(xrcsrq_in, 0, sizeof(xrcsrq_in));
 
-	MLX5_SET(query_xrc_srq_in, xrcsrq_in, opcode,
-		 MLX5_CMD_OP_QUERY_XRC_SRQ);
-	MLX5_SET(query_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);
+	MLX5_SET(query_xrc_srq_in, in, opcode, MLX5_CMD_OP_QUERY_XRC_SRQ);
+	MLX5_SET(query_xrc_srq_in, in, xrc_srqn, srq->srqn);
 
-	err = mlx5_cmd_exec(dev->mdev, xrcsrq_in, sizeof(xrcsrq_in),
-			    xrcsrq_out, MLX5_ST_SZ_BYTES(query_xrc_srq_out));
+	err = mlx5_cmd_exec_inout(dev->mdev, query_xrc_srq, in, xrcsrq_out);
 	if (err)
 		goto out;
 
@@ -341,13 +327,12 @@ out:
 
 static int destroy_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)]   = {};
-	u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {};
 
 	MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
 	MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);
 	MLX5_SET(destroy_rmp_in, in, uid, srq->uid);
-	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
+	return mlx5_cmd_exec_in(dev->mdev, destroy_rmp, in);
 }
 
 static int arm_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
@@ -384,7 +369,7 @@ static int arm_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 	MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
 	MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
 
-	err = mlx5_cmd_exec(dev->mdev, in, inlen, out, outlen);
+	err = mlx5_cmd_exec_inout(dev->mdev, modify_rmp, in, out);
 
 out:
 	kvfree(in);
@@ -414,7 +399,7 @@ static int query_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 
 	MLX5_SET(query_rmp_in, rmp_in, opcode, MLX5_CMD_OP_QUERY_RMP);
 	MLX5_SET(query_rmp_in, rmp_in, rmpn,   srq->srqn);
-	err = mlx5_cmd_exec(dev->mdev, rmp_in, inlen, rmp_out, outlen);
+	err = mlx5_cmd_exec_inout(dev->mdev, query_rmp, rmp_in, rmp_out);
 	if (err)
 		goto out;
 
@@ -477,36 +462,34 @@ static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 
 static int destroy_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
 {
-	u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {0};
-	u32 out[MLX5_ST_SZ_DW(destroy_xrq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {};
 
 	MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);
-	MLX5_SET(destroy_xrq_in, in, xrqn,   srq->srqn);
+	MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);
 	MLX5_SET(destroy_xrq_in, in, uid, srq->uid);
 
-	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
+	return mlx5_cmd_exec_in(dev->mdev, destroy_xrq, in);
 }
 
 static int arm_xrq_cmd(struct mlx5_ib_dev *dev,
 		       struct mlx5_core_srq *srq,
 		       u16 lwm)
 {
-	u32 out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
-	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
+	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
 
-	MLX5_SET(arm_rq_in, in, opcode,     MLX5_CMD_OP_ARM_RQ);
-	MLX5_SET(arm_rq_in, in, op_mod,     MLX5_ARM_RQ_IN_OP_MOD_XRQ);
+	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
+	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ);
 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
-	MLX5_SET(arm_rq_in, in, lwm,	    lwm);
+	MLX5_SET(arm_rq_in, in, lwm, lwm);
 	MLX5_SET(arm_rq_in, in, uid, srq->uid);
 
-	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
+	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
 }
 
 static int query_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 			 struct mlx5_srq_attr *out)
 {
-	u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {0};
+	u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {};
 	u32 *xrq_out;
 	int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);
 	void *xrqc;
@@ -519,7 +502,7 @@ static int query_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 	MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);
 	MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);
 
-	err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), xrq_out, outlen);
+	err = mlx5_cmd_exec_inout(dev->mdev, query_xrq, in, xrq_out);
 	if (err)
 		goto out;
 
diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c
new file mode 100644
index 000000000000..2c6df1c43b55
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/wr.c
@@ -0,0 +1,1504 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
+ */
+
+#include <linux/gfp.h>
+#include <linux/mlx5/qp.h>
+#include <linux/mlx5/driver.h>
+#include "wr.h"
+
+static const u32 mlx5_ib_opcode[] = {
+	[IB_WR_SEND]				= MLX5_OPCODE_SEND,
+	[IB_WR_LSO]				= MLX5_OPCODE_LSO,
+	[IB_WR_SEND_WITH_IMM]			= MLX5_OPCODE_SEND_IMM,
+	[IB_WR_RDMA_WRITE]			= MLX5_OPCODE_RDMA_WRITE,
+	[IB_WR_RDMA_WRITE_WITH_IMM]		= MLX5_OPCODE_RDMA_WRITE_IMM,
+	[IB_WR_RDMA_READ]			= MLX5_OPCODE_RDMA_READ,
+	[IB_WR_ATOMIC_CMP_AND_SWP]		= MLX5_OPCODE_ATOMIC_CS,
+	[IB_WR_ATOMIC_FETCH_AND_ADD]		= MLX5_OPCODE_ATOMIC_FA,
+	[IB_WR_SEND_WITH_INV]			= MLX5_OPCODE_SEND_INVAL,
+	[IB_WR_LOCAL_INV]			= MLX5_OPCODE_UMR,
+	[IB_WR_REG_MR]				= MLX5_OPCODE_UMR,
+	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= MLX5_OPCODE_ATOMIC_MASKED_CS,
+	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= MLX5_OPCODE_ATOMIC_MASKED_FA,
+	[MLX5_IB_WR_UMR]			= MLX5_OPCODE_UMR,
+};
+
+/* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the
+ * next nearby edge and get new address translation for current WQE position.
+ * @sq - SQ buffer.
+ * @seg: Current WQE position (16B aligned).
+ * @wqe_sz: Total current WQE size [16B].
+ * @cur_edge: Updated current edge.
+ */
+static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,
+					 u32 wqe_sz, void **cur_edge)
+{
+	u32 idx;
+
+	if (likely(*seg != *cur_edge))
+		return;
+
+	idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);
+	*cur_edge = get_sq_edge(sq, idx);
+
+	*seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);
+}
+
+/* memcpy_send_wqe - copy data from src to WQE and update the relevant WQ's
+ * pointers. At the end @seg is aligned to 16B regardless the copied size.
+ * @sq - SQ buffer.
+ * @cur_edge: Updated current edge.
+ * @seg: Current WQE position (16B aligned).
+ * @wqe_sz: Total current WQE size [16B].
+ * @src: Pointer to copy from.
+ * @n: Number of bytes to copy.
+ */
+static inline void memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,
+				   void **seg, u32 *wqe_sz, const void *src,
+				   size_t n)
+{
+	while (likely(n)) {
+		size_t leftlen = *cur_edge - *seg;
+		size_t copysz = min_t(size_t, leftlen, n);
+		size_t stride;
+
+		memcpy(*seg, src, copysz);
+
+		n -= copysz;
+		src += copysz;
+		stride = !n ? ALIGN(copysz, 16) : copysz;
+		*seg += stride;
+		*wqe_sz += stride >> 4;
+		handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);
+	}
+}
+
+static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq,
+			    struct ib_cq *ib_cq)
+{
+	struct mlx5_ib_cq *cq;
+	unsigned int cur;
+
+	cur = wq->head - wq->tail;
+	if (likely(cur + nreq < wq->max_post))
+		return 0;
+
+	cq = to_mcq(ib_cq);
+	spin_lock(&cq->lock);
+	cur = wq->head - wq->tail;
+	spin_unlock(&cq->lock);
+
+	return cur + nreq >= wq->max_post;
+}
+
+static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
+					  u64 remote_addr, u32 rkey)
+{
+	rseg->raddr    = cpu_to_be64(remote_addr);
+	rseg->rkey     = cpu_to_be32(rkey);
+	rseg->reserved = 0;
+}
+
+static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp,
+			void **seg, int *size, void **cur_edge)
+{
+	struct mlx5_wqe_eth_seg *eseg = *seg;
+
+	memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
+
+	if (wr->send_flags & IB_SEND_IP_CSUM)
+		eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
+				 MLX5_ETH_WQE_L4_CSUM;
+
+	if (wr->opcode == IB_WR_LSO) {
+		struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
+		size_t left, copysz;
+		void *pdata = ud_wr->header;
+		size_t stride;
+
+		left = ud_wr->hlen;
+		eseg->mss = cpu_to_be16(ud_wr->mss);
+		eseg->inline_hdr.sz = cpu_to_be16(left);
+
+		/* memcpy_send_wqe should get a 16B align address. Hence, we
+		 * first copy up to the current edge and then, if needed,
+		 * continue to memcpy_send_wqe.
+		 */
+		copysz = min_t(u64, *cur_edge - (void *)eseg->inline_hdr.start,
+			       left);
+		memcpy(eseg->inline_hdr.start, pdata, copysz);
+		stride = ALIGN(sizeof(struct mlx5_wqe_eth_seg) -
+			       sizeof(eseg->inline_hdr.start) + copysz, 16);
+		*size += stride / 16;
+		*seg += stride;
+
+		if (copysz < left) {
+			handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+			left -= copysz;
+			pdata += copysz;
+			memcpy_send_wqe(&qp->sq, cur_edge, seg, size, pdata,
+					left);
+		}
+
+		return;
+	}
+
+	*seg += sizeof(struct mlx5_wqe_eth_seg);
+	*size += sizeof(struct mlx5_wqe_eth_seg) / 16;
+}
+
+static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
+			     const struct ib_send_wr *wr)
+{
+	memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
+	dseg->av.dqp_dct =
+		cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
+	dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
+}
+
+static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
+{
+	dseg->byte_count = cpu_to_be32(sg->length);
+	dseg->lkey       = cpu_to_be32(sg->lkey);
+	dseg->addr       = cpu_to_be64(sg->addr);
+}
+
+static u64 get_xlt_octo(u64 bytes)
+{
+	return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
+	       MLX5_IB_UMR_OCTOWORD;
+}
+
+static __be64 frwr_mkey_mask(bool atomic)
+{
+	u64 result;
+
+	result = MLX5_MKEY_MASK_LEN		|
+		MLX5_MKEY_MASK_PAGE_SIZE	|
+		MLX5_MKEY_MASK_START_ADDR	|
+		MLX5_MKEY_MASK_EN_RINVAL	|
+		MLX5_MKEY_MASK_KEY		|
+		MLX5_MKEY_MASK_LR		|
+		MLX5_MKEY_MASK_LW		|
+		MLX5_MKEY_MASK_RR		|
+		MLX5_MKEY_MASK_RW		|
+		MLX5_MKEY_MASK_SMALL_FENCE	|
+		MLX5_MKEY_MASK_FREE;
+
+	if (atomic)
+		result |= MLX5_MKEY_MASK_A;
+
+	return cpu_to_be64(result);
+}
+
+static __be64 sig_mkey_mask(void)
+{
+	u64 result;
+
+	result = MLX5_MKEY_MASK_LEN		|
+		MLX5_MKEY_MASK_PAGE_SIZE	|
+		MLX5_MKEY_MASK_START_ADDR	|
+		MLX5_MKEY_MASK_EN_SIGERR	|
+		MLX5_MKEY_MASK_EN_RINVAL	|
+		MLX5_MKEY_MASK_KEY		|
+		MLX5_MKEY_MASK_LR		|
+		MLX5_MKEY_MASK_LW		|
+		MLX5_MKEY_MASK_RR		|
+		MLX5_MKEY_MASK_RW		|
+		MLX5_MKEY_MASK_SMALL_FENCE	|
+		MLX5_MKEY_MASK_FREE		|
+		MLX5_MKEY_MASK_BSF_EN;
+
+	return cpu_to_be64(result);
+}
+
+static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
+			    struct mlx5_ib_mr *mr, u8 flags, bool atomic)
+{
+	int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
+
+	memset(umr, 0, sizeof(*umr));
+
+	umr->flags = flags;
+	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
+	umr->mkey_mask = frwr_mkey_mask(atomic);
+}
+
+static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
+{
+	memset(umr, 0, sizeof(*umr));
+	umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
+	umr->flags = MLX5_UMR_INLINE;
+}
+
+static __be64 get_umr_enable_mr_mask(void)
+{
+	u64 result;
+
+	result = MLX5_MKEY_MASK_KEY |
+		 MLX5_MKEY_MASK_FREE;
+
+	return cpu_to_be64(result);
+}
+
+static __be64 get_umr_disable_mr_mask(void)
+{
+	u64 result;
+
+	result = MLX5_MKEY_MASK_FREE;
+
+	return cpu_to_be64(result);
+}
+
+static __be64 get_umr_update_translation_mask(void)
+{
+	u64 result;
+
+	result = MLX5_MKEY_MASK_LEN |
+		 MLX5_MKEY_MASK_PAGE_SIZE |
+		 MLX5_MKEY_MASK_START_ADDR;
+
+	return cpu_to_be64(result);
+}
+
+static __be64 get_umr_update_access_mask(int atomic)
+{
+	u64 result;
+
+	result = MLX5_MKEY_MASK_LR |
+		 MLX5_MKEY_MASK_LW |
+		 MLX5_MKEY_MASK_RR |
+		 MLX5_MKEY_MASK_RW;
+
+	if (atomic)
+		result |= MLX5_MKEY_MASK_A;
+
+	return cpu_to_be64(result);
+}
+
+static __be64 get_umr_update_pd_mask(void)
+{
+	u64 result;
+
+	result = MLX5_MKEY_MASK_PD;
+
+	return cpu_to_be64(result);
+}
+
+static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
+{
+	if ((mask & MLX5_MKEY_MASK_PAGE_SIZE &&
+	     MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) ||
+	    (mask & MLX5_MKEY_MASK_A &&
+	     MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)))
+		return -EPERM;
+	return 0;
+}
+
+static int set_reg_umr_segment(struct mlx5_ib_dev *dev,
+			       struct mlx5_wqe_umr_ctrl_seg *umr,
+			       const struct ib_send_wr *wr, int atomic)
+{
+	const struct mlx5_umr_wr *umrwr = umr_wr(wr);
+
+	memset(umr, 0, sizeof(*umr));
+
+	if (!umrwr->ignore_free_state) {
+		if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
+			 /* fail if free */
+			umr->flags = MLX5_UMR_CHECK_FREE;
+		else
+			/* fail if not free */
+			umr->flags = MLX5_UMR_CHECK_NOT_FREE;
+	}
+
+	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
+	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
+		u64 offset = get_xlt_octo(umrwr->offset);
+
+		umr->xlt_offset = cpu_to_be16(offset & 0xffff);
+		umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
+		umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
+	}
+	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
+		umr->mkey_mask |= get_umr_update_translation_mask();
+	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
+		umr->mkey_mask |= get_umr_update_access_mask(atomic);
+		umr->mkey_mask |= get_umr_update_pd_mask();
+	}
+	if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
+		umr->mkey_mask |= get_umr_enable_mr_mask();
+	if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
+		umr->mkey_mask |= get_umr_disable_mr_mask();
+
+	if (!wr->num_sge)
+		umr->flags |= MLX5_UMR_INLINE;
+
+	return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));
+}
+
+static u8 get_umr_flags(int acc)
+{
+	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
+	       (acc & IB_ACCESS_REMOTE_WRITE  ? MLX5_PERM_REMOTE_WRITE : 0) |
+	       (acc & IB_ACCESS_REMOTE_READ   ? MLX5_PERM_REMOTE_READ  : 0) |
+	       (acc & IB_ACCESS_LOCAL_WRITE   ? MLX5_PERM_LOCAL_WRITE  : 0) |
+		MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
+}
+
+static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
+			     struct mlx5_ib_mr *mr,
+			     u32 key, int access)
+{
+	int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1;
+
+	memset(seg, 0, sizeof(*seg));
+
+	if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
+		seg->log2_page_size = ilog2(mr->ibmr.page_size);
+	else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
+		/* KLMs take twice the size of MTTs */
+		ndescs *= 2;
+
+	seg->flags = get_umr_flags(access) | mr->access_mode;
+	seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
+	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
+	seg->start_addr = cpu_to_be64(mr->ibmr.iova);
+	seg->len = cpu_to_be64(mr->ibmr.length);
+	seg->xlt_oct_size = cpu_to_be32(ndescs);
+}
+
+static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
+{
+	memset(seg, 0, sizeof(*seg));
+	seg->status = MLX5_MKEY_STATUS_FREE;
+}
+
+static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg,
+				 const struct ib_send_wr *wr)
+{
+	const struct mlx5_umr_wr *umrwr = umr_wr(wr);
+
+	memset(seg, 0, sizeof(*seg));
+	if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
+		seg->status = MLX5_MKEY_STATUS_FREE;
+
+	seg->flags = convert_access(umrwr->access_flags);
+	if (umrwr->pd)
+		seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
+	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
+	    !umrwr->length)
+		seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
+
+	seg->start_addr = cpu_to_be64(umrwr->virt_addr);
+	seg->len = cpu_to_be64(umrwr->length);
+	seg->log2_page_size = umrwr->page_shift;
+	seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
+				       mlx5_mkey_variant(umrwr->mkey));
+}
+
+static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
+			     struct mlx5_ib_mr *mr,
+			     struct mlx5_ib_pd *pd)
+{
+	int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs);
+
+	dseg->addr = cpu_to_be64(mr->desc_map);
+	dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
+	dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
+}
+
+static __be32 send_ieth(const struct ib_send_wr *wr)
+{
+	switch (wr->opcode) {
+	case IB_WR_SEND_WITH_IMM:
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		return wr->ex.imm_data;
+
+	case IB_WR_SEND_WITH_INV:
+		return cpu_to_be32(wr->ex.invalidate_rkey);
+
+	default:
+		return 0;
+	}
+}
+
+static u8 calc_sig(void *wqe, int size)
+{
+	u8 *p = wqe;
+	u8 res = 0;
+	int i;
+
+	for (i = 0; i < size; i++)
+		res ^= p[i];
+
+	return ~res;
+}
+
+static u8 wq_sig(void *wqe)
+{
+	return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
+}
+
+static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
+			    void **wqe, int *wqe_sz, void **cur_edge)
+{
+	struct mlx5_wqe_inline_seg *seg;
+	size_t offset;
+	int inl = 0;
+	int i;
+
+	seg = *wqe;
+	*wqe += sizeof(*seg);
+	offset = sizeof(*seg);
+
+	for (i = 0; i < wr->num_sge; i++) {
+		size_t len  = wr->sg_list[i].length;
+		void *addr = (void *)(unsigned long)(wr->sg_list[i].addr);
+
+		inl += len;
+
+		if (unlikely(inl > qp->max_inline_data))
+			return -ENOMEM;
+
+		while (likely(len)) {
+			size_t leftlen;
+			size_t copysz;
+
+			handle_post_send_edge(&qp->sq, wqe,
+					      *wqe_sz + (offset >> 4),
+					      cur_edge);
+
+			leftlen = *cur_edge - *wqe;
+			copysz = min_t(size_t, leftlen, len);
+
+			memcpy(*wqe, addr, copysz);
+			len -= copysz;
+			addr += copysz;
+			*wqe += copysz;
+			offset += copysz;
+		}
+	}
+
+	seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
+
+	*wqe_sz +=  ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
+
+	return 0;
+}
+
+static u16 prot_field_size(enum ib_signature_type type)
+{
+	switch (type) {
+	case IB_SIG_TYPE_T10_DIF:
+		return MLX5_DIF_SIZE;
+	default:
+		return 0;
+	}
+}
+
+static u8 bs_selector(int block_size)
+{
+	switch (block_size) {
+	case 512:	    return 0x1;
+	case 520:	    return 0x2;
+	case 4096:	    return 0x3;
+	case 4160:	    return 0x4;
+	case 1073741824:    return 0x5;
+	default:	    return 0;
+	}
+}
+
+static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
+			      struct mlx5_bsf_inl *inl)
+{
+	/* Valid inline section and allow BSF refresh */
+	inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
+				       MLX5_BSF_REFRESH_DIF);
+	inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
+	inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
+	/* repeating block */
+	inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
+	inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
+			MLX5_DIF_CRC : MLX5_DIF_IPCS;
+
+	if (domain->sig.dif.ref_remap)
+		inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
+
+	if (domain->sig.dif.app_escape) {
+		if (domain->sig.dif.ref_escape)
+			inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
+		else
+			inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
+	}
+
+	inl->dif_app_bitmask_check =
+		cpu_to_be16(domain->sig.dif.apptag_check_mask);
+}
+
+static int mlx5_set_bsf(struct ib_mr *sig_mr,
+			struct ib_sig_attrs *sig_attrs,
+			struct mlx5_bsf *bsf, u32 data_size)
+{
+	struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
+	struct mlx5_bsf_basic *basic = &bsf->basic;
+	struct ib_sig_domain *mem = &sig_attrs->mem;
+	struct ib_sig_domain *wire = &sig_attrs->wire;
+
+	memset(bsf, 0, sizeof(*bsf));
+
+	/* Basic + Extended + Inline */
+	basic->bsf_size_sbs = 1 << 7;
+	/* Input domain check byte mask */
+	basic->check_byte_mask = sig_attrs->check_mask;
+	basic->raw_data_size = cpu_to_be32(data_size);
+
+	/* Memory domain */
+	switch (sig_attrs->mem.sig_type) {
+	case IB_SIG_TYPE_NONE:
+		break;
+	case IB_SIG_TYPE_T10_DIF:
+		basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
+		basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
+		mlx5_fill_inl_bsf(mem, &bsf->m_inl);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Wire domain */
+	switch (sig_attrs->wire.sig_type) {
+	case IB_SIG_TYPE_NONE:
+		break;
+	case IB_SIG_TYPE_T10_DIF:
+		if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
+		    mem->sig_type == wire->sig_type) {
+			/* Same block structure */
+			basic->bsf_size_sbs |= 1 << 4;
+			if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
+				basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
+			if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
+				basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
+			if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
+				basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
+		} else
+			basic->wire.bs_selector =
+				bs_selector(wire->sig.dif.pi_interval);
+
+		basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
+		mlx5_fill_inl_bsf(wire, &bsf->w_inl);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+
+static int set_sig_data_segment(const struct ib_send_wr *send_wr,
+				struct ib_mr *sig_mr,
+				struct ib_sig_attrs *sig_attrs,
+				struct mlx5_ib_qp *qp, void **seg, int *size,
+				void **cur_edge)
+{
+	struct mlx5_bsf *bsf;
+	u32 data_len;
+	u32 data_key;
+	u64 data_va;
+	u32 prot_len = 0;
+	u32 prot_key = 0;
+	u64 prot_va = 0;
+	bool prot = false;
+	int ret;
+	int wqe_size;
+	struct mlx5_ib_mr *mr = to_mmr(sig_mr);
+	struct mlx5_ib_mr *pi_mr = mr->pi_mr;
+
+	data_len = pi_mr->data_length;
+	data_key = pi_mr->ibmr.lkey;
+	data_va = pi_mr->data_iova;
+	if (pi_mr->meta_ndescs) {
+		prot_len = pi_mr->meta_length;
+		prot_key = pi_mr->ibmr.lkey;
+		prot_va = pi_mr->pi_iova;
+		prot = true;
+	}
+
+	if (!prot || (data_key == prot_key && data_va == prot_va &&
+		      data_len == prot_len)) {
+		/**
+		 * Source domain doesn't contain signature information
+		 * or data and protection are interleaved in memory.
+		 * So need construct:
+		 *                  ------------------
+		 *                 |     data_klm     |
+		 *                  ------------------
+		 *                 |       BSF        |
+		 *                  ------------------
+		 **/
+		struct mlx5_klm *data_klm = *seg;
+
+		data_klm->bcount = cpu_to_be32(data_len);
+		data_klm->key = cpu_to_be32(data_key);
+		data_klm->va = cpu_to_be64(data_va);
+		wqe_size = ALIGN(sizeof(*data_klm), 64);
+	} else {
+		/**
+		 * Source domain contains signature information
+		 * So need construct a strided block format:
+		 *               ---------------------------
+		 *              |     stride_block_ctrl     |
+		 *               ---------------------------
+		 *              |          data_klm         |
+		 *               ---------------------------
+		 *              |          prot_klm         |
+		 *               ---------------------------
+		 *              |             BSF           |
+		 *               ---------------------------
+		 **/
+		struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
+		struct mlx5_stride_block_entry *data_sentry;
+		struct mlx5_stride_block_entry *prot_sentry;
+		u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
+		int prot_size;
+
+		sblock_ctrl = *seg;
+		data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
+		prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
+
+		prot_size = prot_field_size(sig_attrs->mem.sig_type);
+		if (!prot_size) {
+			pr_err("Bad block size given: %u\n", block_size);
+			return -EINVAL;
+		}
+		sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
+							    prot_size);
+		sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
+		sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
+		sblock_ctrl->num_entries = cpu_to_be16(2);
+
+		data_sentry->bcount = cpu_to_be16(block_size);
+		data_sentry->key = cpu_to_be32(data_key);
+		data_sentry->va = cpu_to_be64(data_va);
+		data_sentry->stride = cpu_to_be16(block_size);
+
+		prot_sentry->bcount = cpu_to_be16(prot_size);
+		prot_sentry->key = cpu_to_be32(prot_key);
+		prot_sentry->va = cpu_to_be64(prot_va);
+		prot_sentry->stride = cpu_to_be16(prot_size);
+
+		wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
+				 sizeof(*prot_sentry), 64);
+	}
+
+	*seg += wqe_size;
+	*size += wqe_size / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+
+	bsf = *seg;
+	ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
+	if (ret)
+		return -EINVAL;
+
+	*seg += sizeof(*bsf);
+	*size += sizeof(*bsf) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+
+	return 0;
+}
+
+static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
+				 struct ib_mr *sig_mr, int access_flags,
+				 u32 size, u32 length, u32 pdn)
+{
+	u32 sig_key = sig_mr->rkey;
+	u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
+
+	memset(seg, 0, sizeof(*seg));
+
+	seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS;
+	seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
+	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
+				    MLX5_MKEY_BSF_EN | pdn);
+	seg->len = cpu_to_be64(length);
+	seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
+	seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
+}
+
+static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
+				u32 size)
+{
+	memset(umr, 0, sizeof(*umr));
+
+	umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
+	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
+	umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
+	umr->mkey_mask = sig_mkey_mask();
+}
+
+static int set_pi_umr_wr(const struct ib_send_wr *send_wr,
+			 struct mlx5_ib_qp *qp, void **seg, int *size,
+			 void **cur_edge)
+{
+	const struct ib_reg_wr *wr = reg_wr(send_wr);
+	struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr);
+	struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr;
+	struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs;
+	u32 pdn = to_mpd(qp->ibqp.pd)->pdn;
+	u32 xlt_size;
+	int region_len, ret;
+
+	if (unlikely(send_wr->num_sge != 0) ||
+	    unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) ||
+	    unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) ||
+	    unlikely(!sig_mr->sig->sig_status_checked))
+		return -EINVAL;
+
+	/* length of the protected region, data + protection */
+	region_len = pi_mr->ibmr.length;
+
+	/**
+	 * KLM octoword size - if protection was provided
+	 * then we use strided block format (3 octowords),
+	 * else we use single KLM (1 octoword)
+	 **/
+	if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE)
+		xlt_size = 0x30;
+	else
+		xlt_size = sizeof(struct mlx5_klm);
+
+	set_sig_umr_segment(*seg, xlt_size);
+	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
+	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+
+	set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len,
+			     pdn);
+	*seg += sizeof(struct mlx5_mkey_seg);
+	*size += sizeof(struct mlx5_mkey_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+
+	ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size,
+				   cur_edge);
+	if (ret)
+		return ret;
+
+	sig_mr->sig->sig_status_checked = false;
+	return 0;
+}
+
+static int set_psv_wr(struct ib_sig_domain *domain,
+		      u32 psv_idx, void **seg, int *size)
+{
+	struct mlx5_seg_set_psv *psv_seg = *seg;
+
+	memset(psv_seg, 0, sizeof(*psv_seg));
+	psv_seg->psv_num = cpu_to_be32(psv_idx);
+	switch (domain->sig_type) {
+	case IB_SIG_TYPE_NONE:
+		break;
+	case IB_SIG_TYPE_T10_DIF:
+		psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
+						     domain->sig.dif.app_tag);
+		psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
+		break;
+	default:
+		pr_err("Bad signature type (%d) is given.\n",
+		       domain->sig_type);
+		return -EINVAL;
+	}
+
+	*seg += sizeof(*psv_seg);
+	*size += sizeof(*psv_seg) / 16;
+
+	return 0;
+}
+
+static int set_reg_wr(struct mlx5_ib_qp *qp,
+		      const struct ib_reg_wr *wr,
+		      void **seg, int *size, void **cur_edge,
+		      bool check_not_free)
+{
+	struct mlx5_ib_mr *mr = to_mmr(wr->mr);
+	struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
+	struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
+	int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size;
+	bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD;
+	bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC;
+	u8 flags = 0;
+
+	if (!mlx5_ib_can_use_umr(dev, atomic, wr->access)) {
+		mlx5_ib_warn(to_mdev(qp->ibqp.device),
+			     "Fast update of %s for MR is disabled\n",
+			     (MLX5_CAP_GEN(dev->mdev,
+					   umr_modify_entity_size_disabled)) ?
+				     "entity size" :
+				     "atomic access");
+		return -EINVAL;
+	}
+
+	if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
+		mlx5_ib_warn(to_mdev(qp->ibqp.device),
+			     "Invalid IB_SEND_INLINE send flag\n");
+		return -EINVAL;
+	}
+
+	if (check_not_free)
+		flags |= MLX5_UMR_CHECK_NOT_FREE;
+	if (umr_inline)
+		flags |= MLX5_UMR_INLINE;
+
+	set_reg_umr_seg(*seg, mr, flags, atomic);
+	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
+	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+
+	set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
+	*seg += sizeof(struct mlx5_mkey_seg);
+	*size += sizeof(struct mlx5_mkey_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+
+	if (umr_inline) {
+		memcpy_send_wqe(&qp->sq, cur_edge, seg, size, mr->descs,
+				mr_list_size);
+		*size = ALIGN(*size, MLX5_SEND_WQE_BB >> 4);
+	} else {
+		set_reg_data_seg(*seg, mr, pd);
+		*seg += sizeof(struct mlx5_wqe_data_seg);
+		*size += (sizeof(struct mlx5_wqe_data_seg) / 16);
+	}
+	return 0;
+}
+
+static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
+			void **cur_edge)
+{
+	set_linv_umr_seg(*seg);
+	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
+	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+	set_linv_mkey_seg(*seg);
+	*seg += sizeof(struct mlx5_mkey_seg);
+	*size += sizeof(struct mlx5_mkey_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+}
+
+static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
+{
+	__be32 *p = NULL;
+	int i, j;
+
+	pr_debug("dump WQE index %u:\n", idx);
+	for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
+		if ((i & 0xf) == 0) {
+			p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
+			pr_debug("WQBB at %p:\n", (void *)p);
+			j = 0;
+			idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
+		}
+		pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
+			 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
+			 be32_to_cpu(p[j + 3]));
+	}
+}
+
+static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
+		       struct mlx5_wqe_ctrl_seg **ctrl,
+		       const struct ib_send_wr *wr, unsigned int *idx,
+		       int *size, void **cur_edge, int nreq,
+		       bool send_signaled, bool solicited)
+{
+	if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
+		return -ENOMEM;
+
+	*idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
+	*seg = mlx5_frag_buf_get_wqe(&qp->sq.fbc, *idx);
+	*ctrl = *seg;
+	*(uint32_t *)(*seg + 8) = 0;
+	(*ctrl)->imm = send_ieth(wr);
+	(*ctrl)->fm_ce_se = qp->sq_signal_bits |
+		(send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
+		(solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
+
+	*seg += sizeof(**ctrl);
+	*size = sizeof(**ctrl) / 16;
+	*cur_edge = qp->sq.cur_edge;
+
+	return 0;
+}
+
+static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
+		     struct mlx5_wqe_ctrl_seg **ctrl,
+		     const struct ib_send_wr *wr, unsigned int *idx, int *size,
+		     void **cur_edge, int nreq)
+{
+	return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
+			   wr->send_flags & IB_SEND_SIGNALED,
+			   wr->send_flags & IB_SEND_SOLICITED);
+}
+
+static void finish_wqe(struct mlx5_ib_qp *qp,
+		       struct mlx5_wqe_ctrl_seg *ctrl,
+		       void *seg, u8 size, void *cur_edge,
+		       unsigned int idx, u64 wr_id, int nreq, u8 fence,
+		       u32 mlx5_opcode)
+{
+	u8 opmod = 0;
+
+	ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
+					     mlx5_opcode | ((u32)opmod << 24));
+	ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
+	ctrl->fm_ce_se |= fence;
+	if (unlikely(qp->flags_en & MLX5_QP_FLAG_SIGNATURE))
+		ctrl->signature = wq_sig(ctrl);
+
+	qp->sq.wrid[idx] = wr_id;
+	qp->sq.w_list[idx].opcode = mlx5_opcode;
+	qp->sq.wqe_head[idx] = qp->sq.head + nreq;
+	qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
+	qp->sq.w_list[idx].next = qp->sq.cur_post;
+
+	/* We save the edge which was possibly updated during the WQE
+	 * construction, into SQ's cache.
+	 */
+	seg = PTR_ALIGN(seg, MLX5_SEND_WQE_BB);
+	qp->sq.cur_edge = (unlikely(seg == cur_edge)) ?
+			  get_sq_edge(&qp->sq, qp->sq.cur_post &
+				      (qp->sq.wqe_cnt - 1)) :
+			  cur_edge;
+}
+
+static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size)
+{
+	set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey);
+	*seg += sizeof(struct mlx5_wqe_raddr_seg);
+	*size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
+}
+
+static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
+			     struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
+			     int *size, void **cur_edge, unsigned int idx)
+{
+	qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
+	(*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey);
+	set_linv_wr(qp, seg, size, cur_edge);
+}
+
+static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
+			 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
+			 void **cur_edge, unsigned int idx)
+{
+	qp->sq.wr_data[idx] = IB_WR_REG_MR;
+	(*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key);
+	return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true);
+}
+
+static int handle_psv(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+		      const struct ib_send_wr *wr,
+		      struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
+		      void **cur_edge, unsigned int *idx, int nreq,
+		      struct ib_sig_domain *domain, u32 psv_index,
+		      u8 next_fence)
+{
+	int err;
+
+	/*
+	 * SET_PSV WQEs are not signaled and solicited on error.
+	 */
+	err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq,
+			  false, true);
+	if (unlikely(err)) {
+		mlx5_ib_warn(dev, "\n");
+		err = -ENOMEM;
+		goto out;
+	}
+	err = set_psv_wr(domain, psv_index, seg, size);
+	if (unlikely(err)) {
+		mlx5_ib_warn(dev, "\n");
+		goto out;
+	}
+	finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
+		   next_fence, MLX5_OPCODE_SET_PSV);
+
+out:
+	return err;
+}
+
+static int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
+				   struct mlx5_ib_qp *qp,
+				   const struct ib_send_wr *wr,
+				   struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
+				   int *size, void **cur_edge,
+				   unsigned int *idx, int nreq, u8 fence,
+				   u8 next_fence)
+{
+	struct mlx5_ib_mr *mr;
+	struct mlx5_ib_mr *pi_mr;
+	struct mlx5_ib_mr pa_pi_mr;
+	struct ib_sig_attrs *sig_attrs;
+	struct ib_reg_wr reg_pi_wr;
+	int err;
+
+	qp->sq.wr_data[*idx] = IB_WR_REG_MR_INTEGRITY;
+
+	mr = to_mmr(reg_wr(wr)->mr);
+	pi_mr = mr->pi_mr;
+
+	if (pi_mr) {
+		memset(&reg_pi_wr, 0,
+		       sizeof(struct ib_reg_wr));
+
+		reg_pi_wr.mr = &pi_mr->ibmr;
+		reg_pi_wr.access = reg_wr(wr)->access;
+		reg_pi_wr.key = pi_mr->ibmr.rkey;
+
+		(*ctrl)->imm = cpu_to_be32(reg_pi_wr.key);
+		/* UMR for data + prot registration */
+		err = set_reg_wr(qp, &reg_pi_wr, seg, size, cur_edge, false);
+		if (unlikely(err))
+			goto out;
+
+		finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id,
+			   nreq, fence, MLX5_OPCODE_UMR);
+
+		err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq);
+		if (unlikely(err)) {
+			mlx5_ib_warn(dev, "\n");
+			err = -ENOMEM;
+			goto out;
+		}
+	} else {
+		memset(&pa_pi_mr, 0, sizeof(struct mlx5_ib_mr));
+		/* No UMR, use local_dma_lkey */
+		pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey;
+		pa_pi_mr.ndescs = mr->ndescs;
+		pa_pi_mr.data_length = mr->data_length;
+		pa_pi_mr.data_iova = mr->data_iova;
+		if (mr->meta_ndescs) {
+			pa_pi_mr.meta_ndescs = mr->meta_ndescs;
+			pa_pi_mr.meta_length = mr->meta_length;
+			pa_pi_mr.pi_iova = mr->pi_iova;
+		}
+
+		pa_pi_mr.ibmr.length = mr->ibmr.length;
+		mr->pi_mr = &pa_pi_mr;
+	}
+	(*ctrl)->imm = cpu_to_be32(mr->ibmr.rkey);
+	/* UMR for sig MR */
+	err = set_pi_umr_wr(wr, qp, seg, size, cur_edge);
+	if (unlikely(err)) {
+		mlx5_ib_warn(dev, "\n");
+		goto out;
+	}
+	finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq,
+		   fence, MLX5_OPCODE_UMR);
+
+	sig_attrs = mr->ibmr.sig_attrs;
+	err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
+			 &sig_attrs->mem, mr->sig->psv_memory.psv_idx,
+			 next_fence);
+	if (unlikely(err))
+		goto out;
+
+	err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq,
+			 &sig_attrs->wire, mr->sig->psv_wire.psv_idx,
+			 next_fence);
+	if (unlikely(err))
+		goto out;
+
+	qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
+
+out:
+	return err;
+}
+
+static int handle_qpt_rc(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+			 const struct ib_send_wr *wr,
+			 struct mlx5_wqe_ctrl_seg **ctrl, void **seg, int *size,
+			 void **cur_edge, unsigned int *idx, int nreq, u8 fence,
+			 u8 next_fence, int *num_sge)
+{
+	int err = 0;
+
+	switch (wr->opcode) {
+	case IB_WR_RDMA_READ:
+	case IB_WR_RDMA_WRITE:
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		handle_rdma_op(wr, seg, size);
+		break;
+
+	case IB_WR_ATOMIC_CMP_AND_SWP:
+	case IB_WR_ATOMIC_FETCH_AND_ADD:
+	case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
+		mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
+		err = -EOPNOTSUPP;
+		goto out;
+
+	case IB_WR_LOCAL_INV:
+		handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx);
+		*num_sge = 0;
+		break;
+
+	case IB_WR_REG_MR:
+		err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx);
+		if (unlikely(err))
+			goto out;
+		*num_sge = 0;
+		break;
+
+	case IB_WR_REG_MR_INTEGRITY:
+		err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size,
+					      cur_edge, idx, nreq, fence,
+					      next_fence);
+		if (unlikely(err))
+			goto out;
+		*num_sge = 0;
+		break;
+
+	default:
+		break;
+	}
+
+out:
+	return err;
+}
+
+static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size)
+{
+	switch (wr->opcode) {
+	case IB_WR_RDMA_WRITE:
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		handle_rdma_op(wr, seg, size);
+		break;
+	default:
+		break;
+	}
+}
+
+static void handle_qpt_hw_gsi(struct mlx5_ib_qp *qp,
+			      const struct ib_send_wr *wr, void **seg,
+			      int *size, void **cur_edge)
+{
+	set_datagram_seg(*seg, wr);
+	*seg += sizeof(struct mlx5_wqe_datagram_seg);
+	*size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+}
+
+static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr,
+			  void **seg, int *size, void **cur_edge)
+{
+	set_datagram_seg(*seg, wr);
+	*seg += sizeof(struct mlx5_wqe_datagram_seg);
+	*size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+
+	/* handle qp that supports ud offload */
+	if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
+		struct mlx5_wqe_eth_pad *pad;
+
+		pad = *seg;
+		memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
+		*seg += sizeof(struct mlx5_wqe_eth_pad);
+		*size += sizeof(struct mlx5_wqe_eth_pad) / 16;
+		set_eth_seg(wr, qp, seg, size, cur_edge);
+		handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+	}
+}
+
+static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+			      const struct ib_send_wr *wr,
+			      struct mlx5_wqe_ctrl_seg **ctrl, void **seg,
+			      int *size, void **cur_edge, unsigned int idx)
+{
+	int err = 0;
+
+	if (unlikely(wr->opcode != MLX5_IB_WR_UMR)) {
+		err = -EINVAL;
+		mlx5_ib_warn(dev, "bad opcode %d\n", wr->opcode);
+		goto out;
+	}
+
+	qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
+	(*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);
+	err = set_reg_umr_segment(dev, *seg, wr,
+				  !!(MLX5_CAP_GEN(dev->mdev, atomic)));
+	if (unlikely(err))
+		goto out;
+	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
+	*size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+	set_reg_mkey_segment(*seg, wr);
+	*seg += sizeof(struct mlx5_mkey_seg);
+	*size += sizeof(struct mlx5_mkey_seg) / 16;
+	handle_post_send_edge(&qp->sq, seg, *size, cur_edge);
+out:
+	return err;
+}
+
+int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+		      const struct ib_send_wr **bad_wr, bool drain)
+{
+	struct mlx5_wqe_ctrl_seg *ctrl = NULL;  /* compiler warning */
+	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
+	struct mlx5_core_dev *mdev = dev->mdev;
+	struct mlx5_ib_qp *qp;
+	struct mlx5_wqe_xrc_seg *xrc;
+	struct mlx5_bf *bf;
+	void *cur_edge;
+	int uninitialized_var(size);
+	unsigned long flags;
+	unsigned int idx;
+	int err = 0;
+	int num_sge;
+	void *seg;
+	int nreq;
+	int i;
+	u8 next_fence = 0;
+	u8 fence;
+
+	if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
+		     !drain)) {
+		*bad_wr = wr;
+		return -EIO;
+	}
+
+	if (unlikely(ibqp->qp_type == IB_QPT_GSI))
+		return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
+
+	qp = to_mqp(ibqp);
+	bf = &qp->bf;
+
+	spin_lock_irqsave(&qp->sq.lock, flags);
+
+	for (nreq = 0; wr; nreq++, wr = wr->next) {
+		if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
+			mlx5_ib_warn(dev, "\n");
+			err = -EINVAL;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		num_sge = wr->num_sge;
+		if (unlikely(num_sge > qp->sq.max_gs)) {
+			mlx5_ib_warn(dev, "\n");
+			err = -EINVAL;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge,
+				nreq);
+		if (err) {
+			mlx5_ib_warn(dev, "\n");
+			err = -ENOMEM;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		if (wr->opcode == IB_WR_REG_MR ||
+		    wr->opcode == IB_WR_REG_MR_INTEGRITY) {
+			fence = dev->umr_fence;
+			next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
+		} else  {
+			if (wr->send_flags & IB_SEND_FENCE) {
+				if (qp->next_fence)
+					fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
+				else
+					fence = MLX5_FENCE_MODE_FENCE;
+			} else {
+				fence = qp->next_fence;
+			}
+		}
+
+		switch (ibqp->qp_type) {
+		case IB_QPT_XRC_INI:
+			xrc = seg;
+			seg += sizeof(*xrc);
+			size += sizeof(*xrc) / 16;
+			fallthrough;
+		case IB_QPT_RC:
+			err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size,
+					    &cur_edge, &idx, nreq, fence,
+					    next_fence, &num_sge);
+			if (unlikely(err)) {
+				*bad_wr = wr;
+				goto out;
+			} else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) {
+				goto skip_psv;
+			}
+			break;
+
+		case IB_QPT_UC:
+			handle_qpt_uc(wr, &seg, &size);
+			break;
+		case IB_QPT_SMI:
+			if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
+				mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
+				err = -EPERM;
+				*bad_wr = wr;
+				goto out;
+			}
+			fallthrough;
+		case MLX5_IB_QPT_HW_GSI:
+			handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge);
+			break;
+		case IB_QPT_UD:
+			handle_qpt_ud(qp, wr, &seg, &size, &cur_edge);
+			break;
+		case MLX5_IB_QPT_REG_UMR:
+			err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg,
+						       &size, &cur_edge, idx);
+			if (unlikely(err))
+				goto out;
+			break;
+
+		default:
+			break;
+		}
+
+		if (wr->send_flags & IB_SEND_INLINE && num_sge) {
+			err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge);
+			if (unlikely(err)) {
+				mlx5_ib_warn(dev, "\n");
+				*bad_wr = wr;
+				goto out;
+			}
+		} else {
+			for (i = 0; i < num_sge; i++) {
+				handle_post_send_edge(&qp->sq, &seg, size,
+						      &cur_edge);
+				if (unlikely(!wr->sg_list[i].length))
+					continue;
+
+				set_data_ptr_seg(
+					(struct mlx5_wqe_data_seg *)seg,
+					wr->sg_list + i);
+				size += sizeof(struct mlx5_wqe_data_seg) / 16;
+				seg += sizeof(struct mlx5_wqe_data_seg);
+			}
+		}
+
+		qp->next_fence = next_fence;
+		finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq,
+			   fence, mlx5_ib_opcode[wr->opcode]);
+skip_psv:
+		if (0)
+			dump_wqe(qp, idx, size);
+	}
+
+out:
+	if (likely(nreq)) {
+		qp->sq.head += nreq;
+
+		/* Make sure that descriptors are written before
+		 * updating doorbell record and ringing the doorbell
+		 */
+		wmb();
+
+		qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
+
+		/* Make sure doorbell record is visible to the HCA before
+		 * we hit doorbell.
+		 */
+		wmb();
+
+		mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset);
+		/* Make sure doorbells don't leak out of SQ spinlock
+		 * and reach the HCA out of order.
+		 */
+		bf->offset ^= bf->buf_size;
+	}
+
+	spin_unlock_irqrestore(&qp->sq.lock, flags);
+
+	return err;
+}
+
+static void set_sig_seg(struct mlx5_rwqe_sig *sig, int max_gs)
+{
+	 sig->signature = calc_sig(sig, (max_gs + 1) << 2);
+}
+
+int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
+		      const struct ib_recv_wr **bad_wr, bool drain)
+{
+	struct mlx5_ib_qp *qp = to_mqp(ibqp);
+	struct mlx5_wqe_data_seg *scat;
+	struct mlx5_rwqe_sig *sig;
+	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
+	struct mlx5_core_dev *mdev = dev->mdev;
+	unsigned long flags;
+	int err = 0;
+	int nreq;
+	int ind;
+	int i;
+
+	if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR &&
+		     !drain)) {
+		*bad_wr = wr;
+		return -EIO;
+	}
+
+	if (unlikely(ibqp->qp_type == IB_QPT_GSI))
+		return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
+
+	spin_lock_irqsave(&qp->rq.lock, flags);
+
+	ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
+
+	for (nreq = 0; wr; nreq++, wr = wr->next) {
+		if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
+			err = -ENOMEM;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		if (unlikely(wr->num_sge > qp->rq.max_gs)) {
+			err = -EINVAL;
+			*bad_wr = wr;
+			goto out;
+		}
+
+		scat = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ind);
+		if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
+			scat++;
+
+		for (i = 0; i < wr->num_sge; i++)
+			set_data_ptr_seg(scat + i, wr->sg_list + i);
+
+		if (i < qp->rq.max_gs) {
+			scat[i].byte_count = 0;
+			scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
+			scat[i].addr       = 0;
+		}
+
+		if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) {
+			sig = (struct mlx5_rwqe_sig *)scat;
+			set_sig_seg(sig, qp->rq.max_gs);
+		}
+
+		qp->rq.wrid[ind] = wr->wr_id;
+
+		ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
+	}
+
+out:
+	if (likely(nreq)) {
+		qp->rq.head += nreq;
+
+		/* Make sure that descriptors are written before
+		 * doorbell record.
+		 */
+		wmb();
+
+		*qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
+	}
+
+	spin_unlock_irqrestore(&qp->rq.lock, flags);
+
+	return err;
+}
diff --git a/drivers/infiniband/hw/mlx5/wr.h b/drivers/infiniband/hw/mlx5/wr.h
new file mode 100644
index 000000000000..4f0057516402
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/wr.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.
+ */
+
+#ifndef _MLX5_IB_WR_H
+#define _MLX5_IB_WR_H
+
+#include "mlx5_ib.h"
+
+enum {
+	MLX5_IB_SQ_UMR_INLINE_THRESHOLD = 64,
+};
+
+struct mlx5_wqe_eth_pad {
+	u8 rsvd0[16];
+};
+
+
+/* get_sq_edge - Get the next nearby edge.
+ *
+ * An 'edge' is defined as the first following address after the end
+ * of the fragment or the SQ. Accordingly, during the WQE construction
+ * which repetitively increases the pointer to write the next data, it
+ * simply should check if it gets to an edge.
+ *
+ * @sq - SQ buffer.
+ * @idx - Stride index in the SQ buffer.
+ *
+ * Return:
+ *	The new edge.
+ */
+static inline void *get_sq_edge(struct mlx5_ib_wq *sq, u32 idx)
+{
+	void *fragment_end;
+
+	fragment_end = mlx5_frag_buf_get_wqe
+		(&sq->fbc,
+		 mlx5_frag_buf_get_idx_last_contig_stride(&sq->fbc, idx));
+
+	return fragment_end + MLX5_SEND_WQE_BB;
+}
+
+int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+		      const struct ib_send_wr **bad_wr, bool drain);
+int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
+		      const struct ib_recv_wr **bad_wr, bool drain);
+
+static inline int mlx5_ib_post_send_nodrain(struct ib_qp *ibqp,
+					    const struct ib_send_wr *wr,
+					    const struct ib_send_wr **bad_wr)
+{
+	return mlx5_ib_post_send(ibqp, wr, bad_wr, false);
+}
+
+static inline int mlx5_ib_post_send_drain(struct ib_qp *ibqp,
+					  const struct ib_send_wr *wr,
+					  const struct ib_send_wr **bad_wr)
+{
+	return mlx5_ib_post_send(ibqp, wr, bad_wr, true);
+}
+
+static inline int mlx5_ib_post_recv_nodrain(struct ib_qp *ibqp,
+					    const struct ib_recv_wr *wr,
+					    const struct ib_recv_wr **bad_wr)
+{
+	return mlx5_ib_post_recv(ibqp, wr, bad_wr, false);
+}
+
+static inline int mlx5_ib_post_recv_drain(struct ib_qp *ibqp,
+					  const struct ib_recv_wr *wr,
+					  const struct ib_recv_wr **bad_wr)
+{
+	return mlx5_ib_post_recv(ibqp, wr, bad_wr, true);
+}
+#endif /* _MLX5_IB_WR_H */
diff --git a/drivers/infiniband/hw/mthca/Kconfig b/drivers/infiniband/hw/mthca/Kconfig
index 66ff527f5928..faa7381f7d63 100644
--- a/drivers/infiniband/hw/mthca/Kconfig
+++ b/drivers/infiniband/hw/mthca/Kconfig
@@ -2,7 +2,7 @@
 config INFINIBAND_MTHCA
 	tristate "Mellanox HCA support"
 	depends on PCI
-	---help---
+	help
 	  This is a low-level driver for Mellanox InfiniHost host
 	  channel adapters (HCAs), including the MT23108 PCI-X HCA
 	  ("Tavor") and the MT25208 PCI Express HCA ("Arbel").
@@ -11,7 +11,7 @@ config INFINIBAND_MTHCA_DEBUG
 	bool "Verbose debugging output" if EXPERT
 	depends on INFINIBAND_MTHCA
 	default y
-	---help---
+	help
 	  This option causes debugging code to be compiled into the
 	  mthca driver.  The output can be turned on via the
 	  debug_level module parameter (which can also be set after
diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h
index 599794c5a78f..7550e9d03dec 100644
--- a/drivers/infiniband/hw/mthca/mthca_dev.h
+++ b/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -478,16 +478,6 @@ int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
 			u32 access, struct mthca_mr *mr);
 void mthca_free_mr(struct mthca_dev *dev,  struct mthca_mr *mr);
 
-int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd,
-		    u32 access, struct mthca_fmr *fmr);
-int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
-			     int list_len, u64 iova);
-void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr);
-int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
-			     int list_len, u64 iova);
-void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr);
-int mthca_free_fmr(struct mthca_dev *dev,  struct mthca_fmr *fmr);
-
 int mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt);
 void mthca_unmap_eq_icm(struct mthca_dev *dev);
 
diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/infiniband/hw/mthca/mthca_mr.c
index 4250b2c18c64..ce0e0867e488 100644
--- a/drivers/infiniband/hw/mthca/mthca_mr.c
+++ b/drivers/infiniband/hw/mthca/mthca_mr.c
@@ -541,7 +541,7 @@ int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
 	return err;
 }
 
-/* Free mr or fmr */
+/* Free mr */
 static void mthca_free_region(struct mthca_dev *dev, u32 lkey)
 {
 	mthca_table_put(dev, dev->mr_table.mpt_table,
@@ -564,266 +564,6 @@ void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr)
 	mthca_free_mtt(dev, mr->mtt);
 }
 
-int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd,
-		    u32 access, struct mthca_fmr *mr)
-{
-	struct mthca_mpt_entry *mpt_entry;
-	struct mthca_mailbox *mailbox;
-	u64 mtt_seg;
-	u32 key, idx;
-	int list_len = mr->attr.max_pages;
-	int err = -ENOMEM;
-	int i;
-
-	if (mr->attr.page_shift < 12 || mr->attr.page_shift >= 32)
-		return -EINVAL;
-
-	/* For Arbel, all MTTs must fit in the same page. */
-	if (mthca_is_memfree(dev) &&
-	    mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE)
-		return -EINVAL;
-
-	mr->maps = 0;
-
-	key = mthca_alloc(&dev->mr_table.mpt_alloc);
-	if (key == -1)
-		return -ENOMEM;
-	key = adjust_key(dev, key);
-
-	idx = key & (dev->limits.num_mpts - 1);
-	mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
-
-	if (mthca_is_memfree(dev)) {
-		err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
-		if (err)
-			goto err_out_mpt_free;
-
-		mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL);
-		BUG_ON(!mr->mem.arbel.mpt);
-	} else
-		mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base +
-			sizeof *(mr->mem.tavor.mpt) * idx;
-
-	mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy);
-	if (IS_ERR(mr->mtt)) {
-		err = PTR_ERR(mr->mtt);
-		goto err_out_table;
-	}
-
-	mtt_seg = mr->mtt->first_seg * dev->limits.mtt_seg_size;
-
-	if (mthca_is_memfree(dev)) {
-		mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table,
-						      mr->mtt->first_seg,
-						      &mr->mem.arbel.dma_handle);
-		BUG_ON(!mr->mem.arbel.mtts);
-	} else
-		mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg;
-
-	mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
-	if (IS_ERR(mailbox)) {
-		err = PTR_ERR(mailbox);
-		goto err_out_free_mtt;
-	}
-
-	mpt_entry = mailbox->buf;
-
-	mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS     |
-				       MTHCA_MPT_FLAG_MIO         |
-				       MTHCA_MPT_FLAG_REGION      |
-				       access);
-
-	mpt_entry->page_size = cpu_to_be32(mr->attr.page_shift - 12);
-	mpt_entry->key       = cpu_to_be32(key);
-	mpt_entry->pd        = cpu_to_be32(pd);
-	memset(&mpt_entry->start, 0,
-	       sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start));
-	mpt_entry->mtt_seg   = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg);
-
-	if (0) {
-		mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
-		for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
-			if (i % 4 == 0)
-				printk("[%02x] ", i * 4);
-			printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i]));
-			if ((i + 1) % 4 == 0)
-				printk("\n");
-		}
-	}
-
-	err = mthca_SW2HW_MPT(dev, mailbox,
-			      key & (dev->limits.num_mpts - 1));
-	if (err) {
-		mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
-		goto err_out_mailbox_free;
-	}
-
-	mthca_free_mailbox(dev, mailbox);
-	return 0;
-
-err_out_mailbox_free:
-	mthca_free_mailbox(dev, mailbox);
-
-err_out_free_mtt:
-	mthca_free_mtt(dev, mr->mtt);
-
-err_out_table:
-	mthca_table_put(dev, dev->mr_table.mpt_table, key);
-
-err_out_mpt_free:
-	mthca_free(&dev->mr_table.mpt_alloc, key);
-	return err;
-}
-
-int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr)
-{
-	if (fmr->maps)
-		return -EBUSY;
-
-	mthca_free_region(dev, fmr->ibmr.lkey);
-	mthca_free_mtt(dev, fmr->mtt);
-
-	return 0;
-}
-
-static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list,
-				  int list_len, u64 iova)
-{
-	int i, page_mask;
-
-	if (list_len > fmr->attr.max_pages)
-		return -EINVAL;
-
-	page_mask = (1 << fmr->attr.page_shift) - 1;
-
-	/* We are getting page lists, so va must be page aligned. */
-	if (iova & page_mask)
-		return -EINVAL;
-
-	/* Trust the user not to pass misaligned data in page_list */
-	if (0)
-		for (i = 0; i < list_len; ++i) {
-			if (page_list[i] & ~page_mask)
-				return -EINVAL;
-		}
-
-	if (fmr->maps >= fmr->attr.max_maps)
-		return -EINVAL;
-
-	return 0;
-}
-
-
-int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
-			     int list_len, u64 iova)
-{
-	struct mthca_fmr *fmr = to_mfmr(ibfmr);
-	struct mthca_dev *dev = to_mdev(ibfmr->device);
-	struct mthca_mpt_entry mpt_entry;
-	u32 key;
-	int i, err;
-
-	err = mthca_check_fmr(fmr, page_list, list_len, iova);
-	if (err)
-		return err;
-
-	++fmr->maps;
-
-	key = tavor_key_to_hw_index(fmr->ibmr.lkey);
-	key += dev->limits.num_mpts;
-	fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
-
-	writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
-
-	for (i = 0; i < list_len; ++i) {
-		__be64 mtt_entry = cpu_to_be64(page_list[i] |
-					       MTHCA_MTT_FLAG_PRESENT);
-		mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i);
-	}
-
-	mpt_entry.lkey   = cpu_to_be32(key);
-	mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
-	mpt_entry.start  = cpu_to_be64(iova);
-
-	__raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key);
-	memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start,
-		    offsetof(struct mthca_mpt_entry, window_count) -
-		    offsetof(struct mthca_mpt_entry, start));
-
-	writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt);
-
-	return 0;
-}
-
-int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
-			     int list_len, u64 iova)
-{
-	struct mthca_fmr *fmr = to_mfmr(ibfmr);
-	struct mthca_dev *dev = to_mdev(ibfmr->device);
-	u32 key;
-	int i, err;
-
-	err = mthca_check_fmr(fmr, page_list, list_len, iova);
-	if (err)
-		return err;
-
-	++fmr->maps;
-
-	key = arbel_key_to_hw_index(fmr->ibmr.lkey);
-	if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
-		key += SINAI_FMR_KEY_INC;
-	else
-		key += dev->limits.num_mpts;
-	fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
-
-	*(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
-
-	wmb();
-
-	dma_sync_single_for_cpu(&dev->pdev->dev, fmr->mem.arbel.dma_handle,
-				list_len * sizeof(u64), DMA_TO_DEVICE);
-
-	for (i = 0; i < list_len; ++i)
-		fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] |
-						     MTHCA_MTT_FLAG_PRESENT);
-
-	dma_sync_single_for_device(&dev->pdev->dev, fmr->mem.arbel.dma_handle,
-				   list_len * sizeof(u64), DMA_TO_DEVICE);
-
-	fmr->mem.arbel.mpt->key    = cpu_to_be32(key);
-	fmr->mem.arbel.mpt->lkey   = cpu_to_be32(key);
-	fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
-	fmr->mem.arbel.mpt->start  = cpu_to_be64(iova);
-
-	wmb();
-
-	*(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW;
-
-	wmb();
-
-	return 0;
-}
-
-void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
-{
-	if (!fmr->maps)
-		return;
-
-	fmr->maps = 0;
-
-	writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
-}
-
-void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
-{
-	if (!fmr->maps)
-		return;
-
-	fmr->maps = 0;
-
-	*(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
-}
-
 int mthca_init_mr_table(struct mthca_dev *dev)
 {
 	phys_addr_t addr;
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 69a3e4f62fb1..9fa2f9164a47 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -118,16 +118,6 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *pr
 	props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 					   props->max_mcast_grp;
-	/*
-	 * If Sinai memory key optimization is being used, then only
-	 * the 8-bit key portion will change.  For other HCAs, the
-	 * unused index bits will also be used for FMR remapping.
-	 */
-	if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
-		props->max_map_per_fmr = 255;
-	else
-		props->max_map_per_fmr =
-			(1 << (32 - ilog2(mdev->limits.num_mpts))) - 1;
 
 	err = 0;
  out:
@@ -388,14 +378,15 @@ static void mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 	mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
 }
 
-static int mthca_ah_create(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
-			   u32 flags, struct ib_udata *udata)
+static int mthca_ah_create(struct ib_ah *ibah,
+			   struct rdma_ah_init_attr *init_attr,
+			   struct ib_udata *udata)
 
 {
 	struct mthca_ah *ah = to_mah(ibah);
 
-	return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd), ah_attr,
-			       ah);
+	return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd),
+			       init_attr->ah_attr, ah);
 }
 
 static void mthca_ah_destroy(struct ib_ah *ah, u32 flags)
@@ -957,69 +948,6 @@ static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata)
 	return 0;
 }
 
-static struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
-				      struct ib_fmr_attr *fmr_attr)
-{
-	struct mthca_fmr *fmr;
-	int err;
-
-	fmr = kmalloc(sizeof *fmr, GFP_KERNEL);
-	if (!fmr)
-		return ERR_PTR(-ENOMEM);
-
-	memcpy(&fmr->attr, fmr_attr, sizeof *fmr_attr);
-	err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num,
-			     convert_access(mr_access_flags), fmr);
-
-	if (err) {
-		kfree(fmr);
-		return ERR_PTR(err);
-	}
-
-	return &fmr->ibmr;
-}
-
-static int mthca_dealloc_fmr(struct ib_fmr *fmr)
-{
-	struct mthca_fmr *mfmr = to_mfmr(fmr);
-	int err;
-
-	err = mthca_free_fmr(to_mdev(fmr->device), mfmr);
-	if (err)
-		return err;
-
-	kfree(mfmr);
-	return 0;
-}
-
-static int mthca_unmap_fmr(struct list_head *fmr_list)
-{
-	struct ib_fmr *fmr;
-	int err;
-	struct mthca_dev *mdev = NULL;
-
-	list_for_each_entry(fmr, fmr_list, list) {
-		if (mdev && to_mdev(fmr->device) != mdev)
-			return -EINVAL;
-		mdev = to_mdev(fmr->device);
-	}
-
-	if (!mdev)
-		return 0;
-
-	if (mthca_is_memfree(mdev)) {
-		list_for_each_entry(fmr, fmr_list, list)
-			mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr));
-
-		wmb();
-	} else
-		list_for_each_entry(fmr, fmr_list, list)
-			mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr));
-
-	err = mthca_SYNC_TPT(mdev);
-	return err;
-}
-
 static ssize_t hw_rev_show(struct device *device,
 			   struct device_attribute *attr, char *buf)
 {
@@ -1203,20 +1131,6 @@ static const struct ib_device_ops mthca_dev_tavor_srq_ops = {
 	INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq),
 };
 
-static const struct ib_device_ops mthca_dev_arbel_fmr_ops = {
-	.alloc_fmr = mthca_alloc_fmr,
-	.dealloc_fmr = mthca_dealloc_fmr,
-	.map_phys_fmr = mthca_arbel_map_phys_fmr,
-	.unmap_fmr = mthca_unmap_fmr,
-};
-
-static const struct ib_device_ops mthca_dev_tavor_fmr_ops = {
-	.alloc_fmr = mthca_alloc_fmr,
-	.dealloc_fmr = mthca_dealloc_fmr,
-	.map_phys_fmr = mthca_tavor_map_phys_fmr,
-	.unmap_fmr = mthca_unmap_fmr,
-};
-
 static const struct ib_device_ops mthca_dev_arbel_ops = {
 	.post_recv = mthca_arbel_post_receive,
 	.post_send = mthca_arbel_post_send,
@@ -1275,15 +1189,6 @@ int mthca_register_device(struct mthca_dev *dev)
 					  &mthca_dev_tavor_srq_ops);
 	}
 
-	if (dev->mthca_flags & MTHCA_FLAG_FMR) {
-		if (mthca_is_memfree(dev))
-			ib_set_device_ops(&dev->ib_dev,
-					  &mthca_dev_arbel_fmr_ops);
-		else
-			ib_set_device_ops(&dev->ib_dev,
-					  &mthca_dev_tavor_fmr_ops);
-	}
-
 	ib_set_device_ops(&dev->ib_dev, &mthca_dev_ops);
 
 	if (mthca_is_memfree(dev))
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.h b/drivers/infiniband/hw/mthca/mthca_provider.h
index 596acc45569b..84c64bff0d92 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.h
+++ b/drivers/infiniband/hw/mthca/mthca_provider.h
@@ -76,24 +76,6 @@ struct mthca_mr {
 	struct mthca_mtt *mtt;
 };
 
-struct mthca_fmr {
-	struct ib_fmr      ibmr;
-	struct ib_fmr_attr attr;
-	struct mthca_mtt  *mtt;
-	int                maps;
-	union {
-		struct {
-			struct mthca_mpt_entry __iomem *mpt;
-			u64 __iomem *mtts;
-		} tavor;
-		struct {
-			struct mthca_mpt_entry *mpt;
-			__be64 *mtts;
-			dma_addr_t dma_handle;
-		} arbel;
-	} mem;
-};
-
 struct mthca_pd {
 	struct ib_pd    ibpd;
 	u32             pd_num;
@@ -301,11 +283,6 @@ static inline struct mthca_ucontext *to_mucontext(struct ib_ucontext *ibucontext
 	return container_of(ibucontext, struct mthca_ucontext, ibucontext);
 }
 
-static inline struct mthca_fmr *to_mfmr(struct ib_fmr *ibmr)
-{
-	return container_of(ibmr, struct mthca_fmr, ibmr);
-}
-
 static inline struct mthca_mr *to_mmr(struct ib_mr *ibmr)
 {
 	return container_of(ibmr, struct mthca_mr, ibmr);
diff --git a/drivers/infiniband/hw/ocrdma/Kconfig b/drivers/infiniband/hw/ocrdma/Kconfig
index dd4ec388208c..54bd70bc4d1a 100644
--- a/drivers/infiniband/hw/ocrdma/Kconfig
+++ b/drivers/infiniband/hw/ocrdma/Kconfig
@@ -4,6 +4,6 @@ config INFINIBAND_OCRDMA
 	depends on ETHERNET && NETDEVICES && PCI && INET && (IPV6 || IPV6=n)
 	select NET_VENDOR_EMULEX
 	select BE2NET
-	---help---
+	help
 	  This driver provides low-level InfiniBand over Ethernet
 	  support for Emulex One Connect host channel adapters (HCAs).
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma.h b/drivers/infiniband/hw/ocrdma/ocrdma.h
index 7baedc74e39d..fcfe0e82197a 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma.h
@@ -98,7 +98,6 @@ struct ocrdma_dev_attr {
 	u64 max_mr_size;
 	u32 max_num_mr_pbl;
 	int max_mw;
-	int max_fmr;
 	int max_map_per_fmr;
 	int max_pages_per_frmr;
 	u16 max_ord_per_qp;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
index 2b7f00ac41b0..6eea02b18968 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
@@ -155,7 +155,7 @@ static inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah *ah,
 	return status;
 }
 
-int ocrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
+int ocrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 		     struct ib_udata *udata)
 {
 	u32 *ahid_addr;
@@ -165,6 +165,7 @@ int ocrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
 	u16 vlan_tag = 0xffff;
 	const struct ib_gid_attr *sgid_attr;
 	struct ocrdma_pd *pd = get_ocrdma_pd(ibah->pd);
+	struct rdma_ah_attr *attr = init_attr->ah_attr;
 	struct ocrdma_dev *dev = get_ocrdma_dev(ibah->device);
 
 	if ((attr->type != RDMA_AH_ATTR_TYPE_ROCE) ||
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
index 9780afcde780..8b73b3489f3a 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
@@ -51,7 +51,7 @@ enum {
 	OCRDMA_AH_L3_TYPE_SHIFT		= 0x1D /* 29 bits */
 };
 
-int ocrdma_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+int ocrdma_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
 		     struct ib_udata *udata);
 void ocrdma_destroy_ah(struct ib_ah *ah, u32 flags);
 int ocrdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index d82d3ec3649e..e07bf0b2209a 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -1190,7 +1190,6 @@ static void ocrdma_get_attr(struct ocrdma_dev *dev,
 	attr->max_mr = rsp->max_mr;
 	attr->max_mr_size = ((u64)rsp->max_mr_size_hi << 32) |
 			      rsp->max_mr_size_lo;
-	attr->max_fmr = 0;
 	attr->max_pages_per_frmr = rsp->max_pages_per_frmr;
 	attr->max_num_mr_pbl = rsp->max_num_mr_pbl;
 	attr->max_cqe = rsp->max_cq_cqes_per_cq &
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 10e343894595..d11c74390a12 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -99,8 +99,6 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
 	attr->max_mw = dev->attr.max_mw;
 	attr->max_pd = dev->attr.max_pd;
 	attr->atomic_cap = 0;
-	attr->max_fmr = 0;
-	attr->max_map_per_fmr = 0;
 	attr->max_qp_rd_atom =
 	    min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
 	attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
diff --git a/drivers/infiniband/hw/qedr/Kconfig b/drivers/infiniband/hw/qedr/Kconfig
index 9c30325e1414..9e31d13b03c3 100644
--- a/drivers/infiniband/hw/qedr/Kconfig
+++ b/drivers/infiniband/hw/qedr/Kconfig
@@ -6,6 +6,6 @@ config INFINIBAND_QEDR
 	select QED_LL2
 	select QED_OOO
 	select QED_RDMA
-	---help---
+	help
 	  This driver provides low-level InfiniBand over Ethernet
 	  support for QLogic QED host channel adapters (HCAs).
diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c
index dcdc85a1ab25..ccaedfd53e49 100644
--- a/drivers/infiniband/hw/qedr/main.c
+++ b/drivers/infiniband/hw/qedr/main.c
@@ -632,7 +632,6 @@ static int qedr_set_device_attr(struct qedr_dev *dev)
 	attr->max_mr_size = qed_attr->max_mr_size;
 	attr->max_cqe = min_t(u64, qed_attr->max_cqe, QEDR_MAX_CQES);
 	attr->max_mw = qed_attr->max_mw;
-	attr->max_fmr = qed_attr->max_fmr;
 	attr->max_mr_mw_fmr_pbl = qed_attr->max_mr_mw_fmr_pbl;
 	attr->max_mr_mw_fmr_size = qed_attr->max_mr_mw_fmr_size;
 	attr->max_pd = qed_attr->max_pd;
diff --git a/drivers/infiniband/hw/qedr/qedr.h b/drivers/infiniband/hw/qedr/qedr.h
index 5488dbd59d3c..fdf90ecb2699 100644
--- a/drivers/infiniband/hw/qedr/qedr.h
+++ b/drivers/infiniband/hw/qedr/qedr.h
@@ -103,7 +103,6 @@ struct qedr_device_attr {
 	u64	max_mr_size;
 	u32	max_cqe;
 	u32	max_mw;
-	u32	max_fmr;
 	u32	max_mr_mw_fmr_pbl;
 	u64	max_mr_mw_fmr_size;
 	u32	max_pd;
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index a5bd3adaf90a..9b9e80266367 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -145,8 +145,6 @@ int qedr_query_device(struct ib_device *ibdev,
 	attr->max_mw = qattr->max_mw;
 	attr->max_pd = qattr->max_pd;
 	attr->atomic_cap = dev->atomic_cap;
-	attr->max_fmr = qattr->max_fmr;
-	attr->max_map_per_fmr = 16;
 	attr->max_qp_init_rd_atom =
 	    1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 	attr->max_qp_rd_atom =
@@ -2750,12 +2748,12 @@ int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 	return 0;
 }
 
-int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
+int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 		   struct ib_udata *udata)
 {
 	struct qedr_ah *ah = get_qedr_ah(ibah);
 
-	rdma_copy_ah_attr(&ah->attr, attr);
+	rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
 
 	return 0;
 }
diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h
index 18027844eb87..5e02387e068d 100644
--- a/drivers/infiniband/hw/qedr/verbs.h
+++ b/drivers/infiniband/hw/qedr/verbs.h
@@ -70,7 +70,7 @@ int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr);
 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
 		       const struct ib_recv_wr **bad_recv_wr);
-int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
+int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 		   struct ib_udata *udata);
 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags);
 
diff --git a/drivers/infiniband/hw/qib/Kconfig b/drivers/infiniband/hw/qib/Kconfig
index 376d19f29346..6c4895777042 100644
--- a/drivers/infiniband/hw/qib/Kconfig
+++ b/drivers/infiniband/hw/qib/Kconfig
@@ -3,7 +3,7 @@ config INFINIBAND_QIB
 	tristate "Intel PCIe HCA support"
 	depends on 64BIT && INFINIBAND_RDMAVT
 	depends on PCI
-	---help---
+	help
 	This is a low-level driver for Intel PCIe QLE InfiniBand host
 	channel adapters.  This driver does not support the Intel
 	HyperTransport card (model QHT7140).
@@ -12,6 +12,6 @@ config INFINIBAND_QIB_DCA
 	bool "QIB DCA support"
 	depends on INFINIBAND_QIB && DCA && SMP && !(INFINIBAND_QIB=y && DCA=m)
 	default y
-	---help---
+	help
 	Setting this enables DCA support on some Intel chip sets
 	with the iba7322 HCA.
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c
index b0144229cf3b..ff87a67dd7b7 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -40,10 +40,10 @@
 #include <linux/highmem.h>
 #include <linux/io.h>
 #include <linux/jiffies.h>
-#include <asm/pgtable.h>
 #include <linux/delay.h>
 #include <linux/export.h>
 #include <linux/uio.h>
+#include <linux/pgtable.h>
 
 #include <rdma/ib.h>
 
diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c
index 91d64dd71a8a..8bcbc884e5b6 100644
--- a/drivers/infiniband/hw/qib/qib_iba7322.c
+++ b/drivers/infiniband/hw/qib/qib_iba7322.c
@@ -2375,7 +2375,6 @@ static int qib_7322_bringup_serdes(struct qib_pportdata *ppd)
 	struct qib_devdata *dd = ppd->dd;
 	u64 val, guid, ibc;
 	unsigned long flags;
-	int ret = 0;
 
 	/*
 	 * SerDes model not in Pd, but still need to
@@ -2510,7 +2509,7 @@ static int qib_7322_bringup_serdes(struct qib_pportdata *ppd)
 		val | ERR_MASK_N(IBStatusChanged));
 
 	/* Always zero until we start messing with SerDes for real */
-	return ret;
+	return 0;
 }
 
 /**
@@ -6875,7 +6874,7 @@ static int init_sdma_7322_regs(struct qib_pportdata *ppd)
 	struct qib_devdata *dd = ppd->dd;
 	unsigned lastbuf, erstbuf;
 	u64 senddmabufmask[3] = { 0 };
-	int n, ret = 0;
+	int n;
 
 	qib_write_kreg_port(ppd, krp_senddmabase, ppd->sdma_descq_phys);
 	qib_sdma_7322_setlengen(ppd);
@@ -6904,7 +6903,7 @@ static int init_sdma_7322_regs(struct qib_pportdata *ppd)
 	qib_write_kreg_port(ppd, krp_senddmabufmask0, senddmabufmask[0]);
 	qib_write_kreg_port(ppd, krp_senddmabufmask1, senddmabufmask[1]);
 	qib_write_kreg_port(ppd, krp_senddmabufmask2, senddmabufmask[2]);
-	return ret;
+	return 0;
 }
 
 /* sdma_lock must be held */
diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c
index 342e3172ca40..4c24e83f3175 100644
--- a/drivers/infiniband/hw/qib/qib_user_pages.c
+++ b/drivers/infiniband/hw/qib/qib_user_pages.c
@@ -106,18 +106,18 @@ int qib_get_user_pages(unsigned long start_page, size_t num_pages,
 		goto bail;
 	}
 
-	down_read(&current->mm->mmap_sem);
+	mmap_read_lock(current->mm);
 	for (got = 0; got < num_pages; got += ret) {
 		ret = pin_user_pages(start_page + got * PAGE_SIZE,
 				     num_pages - got,
 				     FOLL_LONGTERM | FOLL_WRITE | FOLL_FORCE,
 				     p + got, NULL);
 		if (ret < 0) {
-			up_read(&current->mm->mmap_sem);
+			mmap_read_unlock(current->mm);
 			goto bail_release;
 		}
 	}
-	up_read(&current->mm->mmap_sem);
+	mmap_read_unlock(current->mm);
 
 	return 0;
 bail_release:
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index 7508abb6a0fa..7acf9ba5358a 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -1460,7 +1460,6 @@ static void qib_fill_device_attr(struct qib_devdata *dd)
 	rdi->dparms.props.max_cq = ib_qib_max_cqs;
 	rdi->dparms.props.max_cqe = ib_qib_max_cqes;
 	rdi->dparms.props.max_ah = ib_qib_max_ahs;
-	rdi->dparms.props.max_map_per_fmr = 32767;
 	rdi->dparms.props.max_qp_rd_atom = QIB_MAX_RDMA_ATOMIC;
 	rdi->dparms.props.max_qp_init_rd_atom = 255;
 	rdi->dparms.props.max_srq = ib_qib_max_srqs;
diff --git a/drivers/infiniband/hw/usnic/Kconfig b/drivers/infiniband/hw/usnic/Kconfig
index c0847d9e2539..9019599c173b 100644
--- a/drivers/infiniband/hw/usnic/Kconfig
+++ b/drivers/infiniband/hw/usnic/Kconfig
@@ -6,6 +6,6 @@ config INFINIBAND_USNIC
 	select ENIC
 	select NET_VENDOR_CISCO
 	select PCI_IOV
-	---help---
+	help
 	  This is a low-level driver for Cisco's Virtual Interface
 	  Cards (VICs), including the VIC 1240 and 1280 cards.
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index 71f82339446c..b8a77ce11590 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -322,7 +322,6 @@ int usnic_ib_query_device(struct ib_device *ibdev,
 	props->max_mcast_grp = 0;
 	props->max_mcast_qp_attach = 0;
 	props->max_total_mcast_qp_attach = 0;
-	props->max_map_per_fmr = 0;
 	/* Owned by Userspace
 	 * max_qp_wr, max_sge, max_sge_rd, max_cqe */
 	mutex_unlock(&us_ibdev->usdev_lock);
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c
index bd9f944b68fc..760b254ba42d 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom.c
+++ b/drivers/infiniband/hw/usnic/usnic_uiom.c
@@ -123,7 +123,7 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable,
 	npages = PAGE_ALIGN(size + (addr & ~PAGE_MASK)) >> PAGE_SHIFT;
 
 	uiomr->owning_mm = mm = current->mm;
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 
 	locked = atomic64_add_return(npages, &current->mm->pinned_vm);
 	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
@@ -187,7 +187,7 @@ out:
 	} else
 		mmgrab(uiomr->owning_mm);
 
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	free_page((unsigned long) page_list);
 	return ret;
 }
diff --git a/drivers/infiniband/hw/vmw_pvrdma/Kconfig b/drivers/infiniband/hw/vmw_pvrdma/Kconfig
index b99c9f0fc06a..a11e892e5e77 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/Kconfig
+++ b/drivers/infiniband/hw/vmw_pvrdma/Kconfig
@@ -2,7 +2,7 @@
 config INFINIBAND_VMWARE_PVRDMA
 	tristate "VMware Paravirtualized RDMA Driver"
 	depends on NETDEVICES && ETHERNET && PCI && INET && VMXNET3
-	---help---
+	help
 	  This driver provides low-level support for VMware Paravirtual
 	  RDMA adapter. It interacts with the VMXNet3 driver to provide
 	  Ethernet capabilities.
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index faf7ecd7b3fa..ccbded2d26ce 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -509,9 +509,10 @@ void pvrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
  *
  * @return: 0 on success, otherwise errno.
  */
-int pvrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
-		     u32 flags, struct ib_udata *udata)
+int pvrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
+		     struct ib_udata *udata)
 {
+	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 	struct pvrdma_dev *dev = to_vdev(ibah->device);
 	struct pvrdma_ah *ah = to_vah(ibah);
 	const struct ib_global_route *grh;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
index e4a48f5c0c85..267702226f10 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
@@ -414,7 +414,7 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
-int pvrdma_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+int pvrdma_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
 		     struct ib_udata *udata);
 void pvrdma_destroy_ah(struct ib_ah *ah, u32 flags);
 
diff --git a/drivers/infiniband/sw/rdmavt/Kconfig b/drivers/infiniband/sw/rdmavt/Kconfig
index 1f2759c72108..9ef5f5ce1ff6 100644
--- a/drivers/infiniband/sw/rdmavt/Kconfig
+++ b/drivers/infiniband/sw/rdmavt/Kconfig
@@ -4,5 +4,5 @@ config INFINIBAND_RDMAVT
 	depends on X86_64 && ARCH_DMA_ADDR_T_64BIT
 	depends on PCI
 	select DMA_VIRT_OPS
-	---help---
+	help
 	This is a common software verbs provider for RDMA networks.
diff --git a/drivers/infiniband/sw/rdmavt/ah.c b/drivers/infiniband/sw/rdmavt/ah.c
index ee02c6176007..40480add7dd3 100644
--- a/drivers/infiniband/sw/rdmavt/ah.c
+++ b/drivers/infiniband/sw/rdmavt/ah.c
@@ -98,14 +98,14 @@ EXPORT_SYMBOL(rvt_check_ah);
  *
  * Return: 0 on success
  */
-int rvt_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
-		  u32 create_flags, struct ib_udata *udata)
+int rvt_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
+		  struct ib_udata *udata)
 {
 	struct rvt_ah *ah = ibah_to_rvtah(ibah);
 	struct rvt_dev_info *dev = ib_to_rvt(ibah->device);
 	unsigned long flags;
 
-	if (rvt_check_ah(ibah->device, ah_attr))
+	if (rvt_check_ah(ibah->device, init_attr->ah_attr))
 		return -EINVAL;
 
 	spin_lock_irqsave(&dev->n_ahs_lock, flags);
@@ -117,10 +117,11 @@ int rvt_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
 	dev->n_ahs_allocated++;
 	spin_unlock_irqrestore(&dev->n_ahs_lock, flags);
 
-	rdma_copy_ah_attr(&ah->attr, ah_attr);
+	rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
 
 	if (dev->driver_f.notify_new_ah)
-		dev->driver_f.notify_new_ah(ibah->device, ah_attr, ah);
+		dev->driver_f.notify_new_ah(ibah->device,
+					    init_attr->ah_attr, ah);
 
 	return 0;
 }
diff --git a/drivers/infiniband/sw/rdmavt/ah.h b/drivers/infiniband/sw/rdmavt/ah.h
index bbb4d3bdec4e..40b7123fec76 100644
--- a/drivers/infiniband/sw/rdmavt/ah.h
+++ b/drivers/infiniband/sw/rdmavt/ah.h
@@ -50,8 +50,8 @@
 
 #include <rdma/rdma_vt.h>
 
-int rvt_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
-		  u32 create_flags, struct ib_udata *udata);
+int rvt_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,
+		  struct ib_udata *udata);
 void rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags);
 int rvt_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
 int rvt_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index 37853aa3bcf7..f5d0e33cf3d7 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -48,7 +48,6 @@
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
-#include <asm/pgtable.h>
 #include <rdma/uverbs_ioctl.h>
 #include "mmap.h"
 
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 72f6534fbb52..60864e5ca7cb 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -97,7 +97,6 @@ int rvt_driver_mr_init(struct rvt_dev_info *rdi)
 		RCU_INIT_POINTER(rdi->lkey_table.table[i], NULL);
 
 	rdi->dparms.props.max_mr = rdi->lkey_table.max;
-	rdi->dparms.props.max_fmr = rdi->lkey_table.max;
 	return 0;
 }
 
@@ -714,160 +713,6 @@ bail:
 EXPORT_SYMBOL(rvt_invalidate_rkey);
 
 /**
- * rvt_alloc_fmr - allocate a fast memory region
- * @pd: the protection domain for this memory region
- * @mr_access_flags: access flags for this memory region
- * @fmr_attr: fast memory region attributes
- *
- * Return: the memory region on success, otherwise returns an errno.
- */
-struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
-			     struct ib_fmr_attr *fmr_attr)
-{
-	struct rvt_fmr *fmr;
-	int m;
-	struct ib_fmr *ret;
-	int rval = -ENOMEM;
-
-	/* Allocate struct plus pointers to first level page tables. */
-	m = (fmr_attr->max_pages + RVT_SEGSZ - 1) / RVT_SEGSZ;
-	fmr = kzalloc(struct_size(fmr, mr.map, m), GFP_KERNEL);
-	if (!fmr)
-		goto bail;
-
-	rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages,
-				PERCPU_REF_INIT_ATOMIC);
-	if (rval)
-		goto bail;
-
-	/*
-	 * ib_alloc_fmr() will initialize fmr->ibfmr except for lkey &
-	 * rkey.
-	 */
-	rval = rvt_alloc_lkey(&fmr->mr, 0);
-	if (rval)
-		goto bail_mregion;
-	fmr->ibfmr.rkey = fmr->mr.lkey;
-	fmr->ibfmr.lkey = fmr->mr.lkey;
-	/*
-	 * Resources are allocated but no valid mapping (RKEY can't be
-	 * used).
-	 */
-	fmr->mr.access_flags = mr_access_flags;
-	fmr->mr.max_segs = fmr_attr->max_pages;
-	fmr->mr.page_shift = fmr_attr->page_shift;
-
-	ret = &fmr->ibfmr;
-done:
-	return ret;
-
-bail_mregion:
-	rvt_deinit_mregion(&fmr->mr);
-bail:
-	kfree(fmr);
-	ret = ERR_PTR(rval);
-	goto done;
-}
-
-/**
- * rvt_map_phys_fmr - set up a fast memory region
- * @ibfmr: the fast memory region to set up
- * @page_list: the list of pages to associate with the fast memory region
- * @list_len: the number of pages to associate with the fast memory region
- * @iova: the virtual address of the start of the fast memory region
- *
- * This may be called from interrupt context.
- *
- * Return: 0 on success
- */
-
-int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
-		     int list_len, u64 iova)
-{
-	struct rvt_fmr *fmr = to_ifmr(ibfmr);
-	struct rvt_lkey_table *rkt;
-	unsigned long flags;
-	int m, n;
-	unsigned long i;
-	u32 ps;
-	struct rvt_dev_info *rdi = ib_to_rvt(ibfmr->device);
-
-	i = atomic_long_read(&fmr->mr.refcount.count);
-	if (i > 2)
-		return -EBUSY;
-
-	if (list_len > fmr->mr.max_segs)
-		return -EINVAL;
-
-	rkt = &rdi->lkey_table;
-	spin_lock_irqsave(&rkt->lock, flags);
-	fmr->mr.user_base = iova;
-	fmr->mr.iova = iova;
-	ps = 1 << fmr->mr.page_shift;
-	fmr->mr.length = list_len * ps;
-	m = 0;
-	n = 0;
-	for (i = 0; i < list_len; i++) {
-		fmr->mr.map[m]->segs[n].vaddr = (void *)page_list[i];
-		fmr->mr.map[m]->segs[n].length = ps;
-		trace_rvt_mr_fmr_seg(&fmr->mr, m, n, (void *)page_list[i], ps);
-		if (++n == RVT_SEGSZ) {
-			m++;
-			n = 0;
-		}
-	}
-	spin_unlock_irqrestore(&rkt->lock, flags);
-	return 0;
-}
-
-/**
- * rvt_unmap_fmr - unmap fast memory regions
- * @fmr_list: the list of fast memory regions to unmap
- *
- * Return: 0 on success.
- */
-int rvt_unmap_fmr(struct list_head *fmr_list)
-{
-	struct rvt_fmr *fmr;
-	struct rvt_lkey_table *rkt;
-	unsigned long flags;
-	struct rvt_dev_info *rdi;
-
-	list_for_each_entry(fmr, fmr_list, ibfmr.list) {
-		rdi = ib_to_rvt(fmr->ibfmr.device);
-		rkt = &rdi->lkey_table;
-		spin_lock_irqsave(&rkt->lock, flags);
-		fmr->mr.user_base = 0;
-		fmr->mr.iova = 0;
-		fmr->mr.length = 0;
-		spin_unlock_irqrestore(&rkt->lock, flags);
-	}
-	return 0;
-}
-
-/**
- * rvt_dealloc_fmr - deallocate a fast memory region
- * @ibfmr: the fast memory region to deallocate
- *
- * Return: 0 on success.
- */
-int rvt_dealloc_fmr(struct ib_fmr *ibfmr)
-{
-	struct rvt_fmr *fmr = to_ifmr(ibfmr);
-	int ret = 0;
-
-	rvt_free_lkey(&fmr->mr);
-	rvt_put_mr(&fmr->mr); /* will set completion if last */
-	ret = rvt_check_refs(&fmr->mr, __func__);
-	if (ret)
-		goto out;
-	rvt_deinit_mregion(&fmr->mr);
-	kfree(fmr);
-out:
-	return ret;
-}
-
-/**
  * rvt_sge_adjacent - is isge compressible
  * @last_sge: last outgoing SGE written
  * @sge: SGE to check
diff --git a/drivers/infiniband/sw/rdmavt/mr.h b/drivers/infiniband/sw/rdmavt/mr.h
index 2c8d0752e8e3..780fc63af98b 100644
--- a/drivers/infiniband/sw/rdmavt/mr.h
+++ b/drivers/infiniband/sw/rdmavt/mr.h
@@ -49,10 +49,6 @@
  */
 
 #include <rdma/rdma_vt.h>
-struct rvt_fmr {
-	struct ib_fmr ibfmr;
-	struct rvt_mregion mr;        /* must be last */
-};
 
 struct rvt_mr {
 	struct ib_mr ibmr;
@@ -60,11 +56,6 @@ struct rvt_mr {
 	struct rvt_mregion mr;  /* must be last */
 };
 
-static inline struct rvt_fmr *to_ifmr(struct ib_fmr *ibfmr)
-{
-	return container_of(ibfmr, struct rvt_fmr, ibfmr);
-}
-
 static inline struct rvt_mr *to_imr(struct ib_mr *ibmr)
 {
 	return container_of(ibmr, struct rvt_mr, ibmr);
@@ -83,11 +74,5 @@ struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
 			   u32 max_num_sg, struct ib_udata *udata);
 int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
-struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
-			     struct ib_fmr_attr *fmr_attr);
-int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
-		     int list_len, u64 iova);
-int rvt_unmap_fmr(struct list_head *fmr_list);
-int rvt_dealloc_fmr(struct ib_fmr *ibfmr);
 
 #endif          /* DEF_RVTMR_H */
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 500a7ee04c44..511b72809e14 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 - 2019 Intel Corporation.
+ * Copyright(c) 2016 - 2020 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -525,15 +525,18 @@ static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
  * @rdi: rvt device info structure
  * @qpt: queue pair number table pointer
  * @port_num: IB port number, 1 based, comes from core
+ * @exclude_prefix: prefix of special queue pair number being allocated
  *
  * Return: The queue pair number
  */
 static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
-		     enum ib_qp_type type, u8 port_num)
+		     enum ib_qp_type type, u8 port_num, u8 exclude_prefix)
 {
 	u32 i, offset, max_scan, qpn;
 	struct rvt_qpn_map *map;
 	u32 ret;
+	u32 max_qpn = exclude_prefix == RVT_AIP_QP_PREFIX ?
+		RVT_AIP_QPN_MAX : RVT_QPN_MAX;
 
 	if (rdi->driver_f.alloc_qpn)
 		return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num);
@@ -553,7 +556,7 @@ static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
 	}
 
 	qpn = qpt->last + qpt->incr;
-	if (qpn >= RVT_QPN_MAX)
+	if (qpn >= max_qpn)
 		qpn = qpt->incr | ((qpt->last & 1) ^ 1);
 	/* offset carries bit 0 */
 	offset = qpn & RVT_BITS_PER_PAGE_MASK;
@@ -987,6 +990,9 @@ static void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
 {
 	struct rvt_qpn_map *map;
 
+	if ((qpn & RVT_AIP_QP_PREFIX_MASK) == RVT_AIP_QP_BASE)
+		qpn &= RVT_AIP_QP_SUFFIX;
+
 	map = qpt->map + (qpn & RVT_QPN_MASK) / RVT_BITS_PER_PAGE;
 	if (map->page)
 		clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page);
@@ -1074,13 +1080,15 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 	struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device);
 	void *priv = NULL;
 	size_t sqsize;
+	u8 exclude_prefix = 0;
 
 	if (!rdi)
 		return ERR_PTR(-EINVAL);
 
 	if (init_attr->cap.max_send_sge > rdi->dparms.props.max_send_sge ||
 	    init_attr->cap.max_send_wr > rdi->dparms.props.max_qp_wr ||
-	    init_attr->create_flags)
+	    (init_attr->create_flags &&
+	     init_attr->create_flags != IB_QP_CREATE_NETDEV_USE))
 		return ERR_PTR(-EINVAL);
 
 	/* Check receive queue parameters if no SRQ is specified. */
@@ -1199,14 +1207,20 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 			goto bail_driver_priv;
 		}
 
+		if (init_attr->create_flags & IB_QP_CREATE_NETDEV_USE)
+			exclude_prefix = RVT_AIP_QP_PREFIX;
+
 		err = alloc_qpn(rdi, &rdi->qp_dev->qpn_table,
 				init_attr->qp_type,
-				init_attr->port_num);
+				init_attr->port_num,
+				exclude_prefix);
 		if (err < 0) {
 			ret = ERR_PTR(err);
 			goto bail_rq_wq;
 		}
 		qp->ibqp.qp_num = err;
+		if (init_attr->create_flags & IB_QP_CREATE_NETDEV_USE)
+			qp->ibqp.qp_num |= RVT_AIP_QP_BASE;
 		qp->port_num = init_attr->port_num;
 		rvt_init_qp(rdi, qp, init_attr->qp_type);
 		if (rdi->driver_f.qp_priv_init) {
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 72b031ab7092..f904bb34477a 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -378,7 +378,6 @@ enum {
 static const struct ib_device_ops rvt_dev_ops = {
 	.uverbs_abi_ver = RVT_UVERBS_ABI_VERSION,
 
-	.alloc_fmr = rvt_alloc_fmr,
 	.alloc_mr = rvt_alloc_mr,
 	.alloc_pd = rvt_alloc_pd,
 	.alloc_ucontext = rvt_alloc_ucontext,
@@ -387,7 +386,6 @@ static const struct ib_device_ops rvt_dev_ops = {
 	.create_cq = rvt_create_cq,
 	.create_qp = rvt_create_qp,
 	.create_srq = rvt_create_srq,
-	.dealloc_fmr = rvt_dealloc_fmr,
 	.dealloc_pd = rvt_dealloc_pd,
 	.dealloc_ucontext = rvt_dealloc_ucontext,
 	.dereg_mr = rvt_dereg_mr,
@@ -399,7 +397,6 @@ static const struct ib_device_ops rvt_dev_ops = {
 	.get_dma_mr = rvt_get_dma_mr,
 	.get_port_immutable = rvt_get_port_immutable,
 	.map_mr_sg = rvt_map_mr_sg,
-	.map_phys_fmr = rvt_map_phys_fmr,
 	.mmap = rvt_mmap,
 	.modify_ah = rvt_modify_ah,
 	.modify_device = rvt_modify_device,
@@ -420,7 +417,6 @@ static const struct ib_device_ops rvt_dev_ops = {
 	.reg_user_mr = rvt_reg_user_mr,
 	.req_notify_cq = rvt_req_notify_cq,
 	.resize_cq = rvt_resize_cq,
-	.unmap_fmr = rvt_unmap_fmr,
 
 	INIT_RDMA_OBJ_SIZE(ib_ah, rvt_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_cq, rvt_cq, ibcq),
diff --git a/drivers/infiniband/sw/rxe/Kconfig b/drivers/infiniband/sw/rxe/Kconfig
index d9bcfe740588..a0c6c7dfc181 100644
--- a/drivers/infiniband/sw/rxe/Kconfig
+++ b/drivers/infiniband/sw/rxe/Kconfig
@@ -6,7 +6,7 @@ config RDMA_RXE
 	select NET_UDP_TUNNEL
 	select CRYPTO_CRC32
 	select DMA_VIRT_OPS
-	---help---
+	help
 	This driver implements the InfiniBand RDMA transport over
 	the Linux network stack. It enables a system with a
 	standard Ethernet adapter to interoperate with a RoCE
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 4afdd2e20883..5642eefb4ba1 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -77,6 +77,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
 {
 	rxe->max_inline_data			= RXE_MAX_INLINE_DATA;
 
+	rxe->attr.vendor_id			= RXE_VENDOR_ID;
 	rxe->attr.max_mr_size			= RXE_MAX_MR_SIZE;
 	rxe->attr.page_size_cap			= RXE_PAGE_SIZE_CAP;
 	rxe->attr.max_qp			= RXE_MAX_QP;
diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c
index 6a413d73b95d..7887f623f62c 100644
--- a/drivers/infiniband/sw/rxe/rxe_mmap.c
+++ b/drivers/infiniband/sw/rxe/rxe_mmap.c
@@ -35,7 +35,6 @@
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
 #include <linux/errno.h>
-#include <asm/pgtable.h>
 #include <rdma/uverbs_ioctl.h>
 
 #include "rxe.h"
diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
index f59616b02477..99e9d8ba9767 100644
--- a/drivers/infiniband/sw/rxe/rxe_param.h
+++ b/drivers/infiniband/sw/rxe/rxe_param.h
@@ -127,6 +127,9 @@ enum rxe_device_param {
 
 	/* Delay before calling arbiter timer */
 	RXE_NSEC_ARB_TIMER_DELAY	= 200,
+
+	/* IBTA v1.4 A3.3.1 VENDOR INFORMATION section */
+	RXE_VENDOR_ID			= 0XFFFFFF,
 };
 
 /* default/initial rxe port parameters */
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 9dd4bd7aea92..b8a22af724e8 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -195,15 +195,16 @@ static void rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	rxe_drop_ref(pd);
 }
 
-static int rxe_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr,
-			 u32 flags, struct ib_udata *udata)
+static int rxe_create_ah(struct ib_ah *ibah,
+			 struct rdma_ah_init_attr *init_attr,
+			 struct ib_udata *udata)
 
 {
 	int err;
 	struct rxe_dev *rxe = to_rdev(ibah->device);
 	struct rxe_ah *ah = to_rah(ibah);
 
-	err = rxe_av_chk_attr(rxe, attr);
+	err = rxe_av_chk_attr(rxe, init_attr->ah_attr);
 	if (err)
 		return err;
 
@@ -211,7 +212,7 @@ static int rxe_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr,
 	if (err)
 		return err;
 
-	rxe_init_av(attr, &ah->av);
+	rxe_init_av(init_attr->ah_attr, &ah->av);
 	return 0;
 }
 
diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h
index af5e9f8c0fcd..e9753831ac3f 100644
--- a/drivers/infiniband/sw/siw/siw.h
+++ b/drivers/infiniband/sw/siw/siw.h
@@ -30,7 +30,6 @@
 #define SIW_MAX_MR (SIW_MAX_QP * 10)
 #define SIW_MAX_PD SIW_MAX_QP
 #define SIW_MAX_MW 0 /* to be set if MW's are supported */
-#define SIW_MAX_FMR SIW_MAX_MR
 #define SIW_MAX_SRQ SIW_MAX_QP
 #define SIW_MAX_SRQ_WR (SIW_MAX_QP_WR * 10)
 #define SIW_MAX_CONTEXT SIW_MAX_PD
@@ -59,7 +58,6 @@ struct siw_dev_cap {
 	int max_mr;
 	int max_pd;
 	int max_mw;
-	int max_fmr;
 	int max_srq;
 	int max_srq_wr;
 	int max_srq_sge;
@@ -139,7 +137,7 @@ struct siw_pble {
 struct siw_pbl {
 	unsigned int num_buf;
 	unsigned int max_buf;
-	struct siw_pble pbe[1];
+	struct siw_pble pbe[];
 };
 
 /*
diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c
index 5cd40fb9e20c..a0b8cc643c5c 100644
--- a/drivers/infiniband/sw/siw/siw_main.c
+++ b/drivers/infiniband/sw/siw/siw_main.c
@@ -413,7 +413,6 @@ static struct siw_device *siw_device_create(struct net_device *netdev)
 	sdev->attrs.max_mr = SIW_MAX_MR;
 	sdev->attrs.max_pd = SIW_MAX_PD;
 	sdev->attrs.max_mw = SIW_MAX_MW;
-	sdev->attrs.max_fmr = SIW_MAX_FMR;
 	sdev->attrs.max_srq = SIW_MAX_SRQ;
 	sdev->attrs.max_srq_wr = SIW_MAX_SRQ_WR;
 	sdev->attrs.max_srq_sge = SIW_MAX_SGE;
diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c
index e2061dc0b043..34a910cf0edb 100644
--- a/drivers/infiniband/sw/siw/siw_mem.c
+++ b/drivers/infiniband/sw/siw/siw_mem.c
@@ -349,14 +349,11 @@ dma_addr_t siw_pbl_get_buffer(struct siw_pbl *pbl, u64 off, int *len, int *idx)
 struct siw_pbl *siw_pbl_alloc(u32 num_buf)
 {
 	struct siw_pbl *pbl;
-	int buf_size = sizeof(*pbl);
 
 	if (num_buf == 0)
 		return ERR_PTR(-EINVAL);
 
-	buf_size += ((num_buf - 1) * sizeof(struct siw_pble));
-
-	pbl = kzalloc(buf_size, GFP_KERNEL);
+	pbl = kzalloc(struct_size(pbl, pbe, num_buf), GFP_KERNEL);
 	if (!pbl)
 		return ERR_PTR(-ENOMEM);
 
@@ -397,7 +394,7 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable)
 	if (!writable)
 		foll_flags |= FOLL_FORCE;
 
-	down_read(&mm_s->mmap_sem);
+	mmap_read_lock(mm_s);
 
 	mlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 
@@ -441,7 +438,7 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable)
 		num_pages -= got;
 	}
 out_sem_up:
-	up_read(&mm_s->mmap_sem);
+	mmap_read_unlock(mm_s);
 
 	if (rv > 0)
 		return umem;
diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
index aeb842bc7a1e..987e2ba05dbc 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.c
+++ b/drivers/infiniband/sw/siw/siw_verbs.c
@@ -136,7 +136,6 @@ int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
 	attr->max_cq = sdev->attrs.max_cq;
 	attr->max_cqe = sdev->attrs.max_cqe;
 	attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
-	attr->max_fmr = sdev->attrs.max_fmr;
 	attr->max_mr = sdev->attrs.max_mr;
 	attr->max_mw = sdev->attrs.max_mw;
 	attr->max_mr_size = ~0ull;
diff --git a/drivers/infiniband/ulp/Makefile b/drivers/infiniband/ulp/Makefile
index 437813c7b481..4d0004b58377 100644
--- a/drivers/infiniband/ulp/Makefile
+++ b/drivers/infiniband/ulp/Makefile
@@ -5,3 +5,4 @@ obj-$(CONFIG_INFINIBAND_SRPT)		+= srpt/
 obj-$(CONFIG_INFINIBAND_ISER)		+= iser/
 obj-$(CONFIG_INFINIBAND_ISERT)		+= isert/
 obj-$(CONFIG_INFINIBAND_OPA_VNIC)	+= opa_vnic/
+obj-$(CONFIG_INFINIBAND_RTRS)		+= rtrs/
diff --git a/drivers/infiniband/ulp/ipoib/Kconfig b/drivers/infiniband/ulp/ipoib/Kconfig
index 7af68604af77..254e31a90a66 100644
--- a/drivers/infiniband/ulp/ipoib/Kconfig
+++ b/drivers/infiniband/ulp/ipoib/Kconfig
@@ -2,7 +2,7 @@
 config INFINIBAND_IPOIB
 	tristate "IP-over-InfiniBand"
 	depends on NETDEVICES && INET && (IPV6 || IPV6=n)
-	---help---
+	help
 	  Support for the IP-over-InfiniBand protocol (IPoIB). This
 	  transports IP packets over InfiniBand so you can use your IB
 	  device as a fancy NIC.
@@ -13,7 +13,7 @@ config INFINIBAND_IPOIB_CM
 	bool "IP-over-InfiniBand Connected Mode support"
 	depends on INFINIBAND_IPOIB
 	default n
-	---help---
+	help
 	  This option enables support for IPoIB connected mode.  After
 	  enabling this option, you need to switch to connected mode
 	  through /sys/class/net/ibXXX/mode to actually create
@@ -28,7 +28,7 @@ config INFINIBAND_IPOIB_DEBUG
 	bool "IP-over-InfiniBand debugging" if EXPERT
 	depends on INFINIBAND_IPOIB
 	default y
-	---help---
+	help
 	  This option causes debugging code to be compiled into the
 	  IPoIB driver.  The output can be turned on via the
 	  debug_level and mcast_debug_level module parameters (which
@@ -42,7 +42,7 @@ config INFINIBAND_IPOIB_DEBUG
 config INFINIBAND_IPOIB_DEBUG_DATA
 	bool "IP-over-InfiniBand data path debugging"
 	depends on INFINIBAND_IPOIB_DEBUG
-	---help---
+	help
 	  This option compiles debugging code into the data path
 	  of the IPoIB driver.  The output can be turned on via the
 	  data_debug_level module parameter; however, even with output
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index ceec24d45185..3cfb682b91b0 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -86,7 +86,7 @@ struct workqueue_struct *ipoib_workqueue;
 
 struct ib_sa_client ipoib_sa_client;
 
-static void ipoib_add_one(struct ib_device *device);
+static int ipoib_add_one(struct ib_device *device);
 static void ipoib_remove_one(struct ib_device *device, void *client_data);
 static void ipoib_neigh_reclaim(struct rcu_head *rp);
 static struct net_device *ipoib_get_net_dev_by_params(
@@ -479,9 +479,6 @@ static struct net_device *ipoib_get_net_dev_by_params(
 	if (ret)
 		return NULL;
 
-	if (!dev_list)
-		return NULL;
-
 	/* See if we can find a unique device matching the L2 parameters */
 	matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
 						gid, NULL, &net_dev);
@@ -529,6 +526,7 @@ int ipoib_set_mode(struct net_device *dev, const char *buf)
 			   "will cause multicast packet drops\n");
 		netdev_update_features(dev);
 		dev_set_mtu(dev, ipoib_cm_max_mtu(dev));
+		netif_set_real_num_tx_queues(dev, 1);
 		rtnl_unlock();
 		priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
 
@@ -540,6 +538,7 @@ int ipoib_set_mode(struct net_device *dev, const char *buf)
 		clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
 		netdev_update_features(dev);
 		dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
+		netif_set_real_num_tx_queues(dev, dev->num_tx_queues);
 		rtnl_unlock();
 		ipoib_flush_paths(dev);
 		return (!rtnl_trylock()) ? -EBUSY : 0;
@@ -1860,7 +1859,7 @@ static int ipoib_parent_init(struct net_device *ndev)
 			priv->port);
 		return result;
 	}
-	priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu);
+	priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);
 
 	result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
 	if (result) {
@@ -1901,6 +1900,7 @@ static int ipoib_ndo_init(struct net_device *ndev)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(ndev);
 	int rc;
+	struct rdma_netdev *rn = netdev_priv(ndev);
 
 	if (priv->parent) {
 		ipoib_child_init(ndev);
@@ -1913,6 +1913,7 @@ static int ipoib_ndo_init(struct net_device *ndev)
 	/* MTU will be reset when mcast join happens */
 	ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
 	priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
+	rn->mtu = priv->mcast_mtu;
 	ndev->max_mtu = IPOIB_CM_MTU;
 
 	ndev->neigh_priv_len = sizeof(struct ipoib_neigh);
@@ -2074,9 +2075,17 @@ static const struct net_device_ops ipoib_netdev_ops_vf = {
 	.ndo_do_ioctl		 = ipoib_ioctl,
 };
 
+static const struct net_device_ops ipoib_netdev_default_pf = {
+	.ndo_init		 = ipoib_dev_init_default,
+	.ndo_uninit		 = ipoib_dev_uninit_default,
+	.ndo_open		 = ipoib_ib_dev_open_default,
+	.ndo_stop		 = ipoib_ib_dev_stop_default,
+};
+
 void ipoib_setup_common(struct net_device *dev)
 {
 	dev->header_ops		 = &ipoib_header_ops;
+	dev->netdev_ops          = &ipoib_netdev_default_pf;
 
 	ipoib_set_ethtool_ops(dev);
 
@@ -2126,13 +2135,6 @@ static void ipoib_build_priv(struct net_device *dev)
 	INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
 }
 
-static const struct net_device_ops ipoib_netdev_default_pf = {
-	.ndo_init		 = ipoib_dev_init_default,
-	.ndo_uninit		 = ipoib_dev_uninit_default,
-	.ndo_open		 = ipoib_ib_dev_open_default,
-	.ndo_stop		 = ipoib_ib_dev_stop_default,
-};
-
 static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u8 port,
 					     const char *name)
 {
@@ -2170,7 +2172,6 @@ int ipoib_intf_init(struct ib_device *hca, u8 port, const char *name,
 		if (rc != -EOPNOTSUPP)
 			goto out;
 
-		dev->netdev_ops = &ipoib_netdev_default_pf;
 		rn->send = ipoib_send;
 		rn->attach_mcast = ipoib_mcast_attach;
 		rn->detach_mcast = ipoib_mcast_detach;
@@ -2516,7 +2517,7 @@ sysfs_failed:
 	return ERR_PTR(-ENOMEM);
 }
 
-static void ipoib_add_one(struct ib_device *device)
+static int ipoib_add_one(struct ib_device *device)
 {
 	struct list_head *dev_list;
 	struct net_device *dev;
@@ -2526,7 +2527,7 @@ static void ipoib_add_one(struct ib_device *device)
 
 	dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);
 	if (!dev_list)
-		return;
+		return -ENOMEM;
 
 	INIT_LIST_HEAD(dev_list);
 
@@ -2543,10 +2544,11 @@ static void ipoib_add_one(struct ib_device *device)
 
 	if (!count) {
 		kfree(dev_list);
-		return;
+		return -EOPNOTSUPP;
 	}
 
 	ib_set_client_data(device, &ipoib_client, dev_list);
+	return 0;
 }
 
 static void ipoib_remove_one(struct ib_device *device, void *client_data)
@@ -2554,9 +2556,6 @@ static void ipoib_remove_one(struct ib_device *device, void *client_data)
 	struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
 	struct list_head *dev_list = client_data;
 
-	if (!dev_list)
-		return;
-
 	list_for_each_entry_safe(priv, tmp, dev_list, list) {
 		LIST_HEAD(head);
 		ipoib_parent_unregister_pre(priv->dev);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index b9e9562f5034..9bfa514473d5 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -135,12 +135,11 @@ static void ipoib_mcast_free(struct ipoib_mcast *mcast)
 	kfree(mcast);
 }
 
-static struct ipoib_mcast *ipoib_mcast_alloc(struct net_device *dev,
-					     int can_sleep)
+static struct ipoib_mcast *ipoib_mcast_alloc(struct net_device *dev)
 {
 	struct ipoib_mcast *mcast;
 
-	mcast = kzalloc(sizeof(*mcast), can_sleep ? GFP_KERNEL : GFP_ATOMIC);
+	mcast = kzalloc(sizeof(*mcast), GFP_ATOMIC);
 	if (!mcast)
 		return NULL;
 
@@ -218,6 +217,7 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
 	struct rdma_ah_attr av;
 	int ret;
 	int set_qkey = 0;
+	int mtu;
 
 	mcast->mcmember = *mcmember;
 
@@ -240,13 +240,12 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
 		priv->broadcast->mcmember.flow_label = mcmember->flow_label;
 		priv->broadcast->mcmember.hop_limit = mcmember->hop_limit;
 		/* assume if the admin and the mcast are the same both can be changed */
+		mtu = rdma_mtu_enum_to_int(priv->ca,  priv->port,
+					   priv->broadcast->mcmember.mtu);
 		if (priv->mcast_mtu == priv->admin_mtu)
-			priv->admin_mtu =
-			priv->mcast_mtu =
-			IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu));
-		else
-			priv->mcast_mtu =
-			IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu));
+			priv->admin_mtu = IPOIB_UD_MTU(mtu);
+		priv->mcast_mtu = IPOIB_UD_MTU(mtu);
+		rn->mtu = priv->mcast_mtu;
 
 		priv->qkey = be32_to_cpu(priv->broadcast->mcmember.qkey);
 		spin_unlock_irq(&priv->lock);
@@ -599,7 +598,7 @@ void ipoib_mcast_join_task(struct work_struct *work)
 	if (!priv->broadcast) {
 		struct ipoib_mcast *broadcast;
 
-		broadcast = ipoib_mcast_alloc(dev, 0);
+		broadcast = ipoib_mcast_alloc(dev);
 		if (!broadcast) {
 			ipoib_warn(priv, "failed to allocate broadcast group\n");
 			/*
@@ -782,7 +781,7 @@ void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb)
 			ipoib_dbg_mcast(priv, "setting up send only multicast group for %pI6\n",
 					mgid);
 
-			mcast = ipoib_mcast_alloc(dev, 0);
+			mcast = ipoib_mcast_alloc(dev);
 			if (!mcast) {
 				ipoib_warn(priv, "unable to allocate memory "
 					   "for multicast structure\n");
@@ -936,7 +935,7 @@ void ipoib_mcast_restart_task(struct work_struct *work)
 			ipoib_dbg_mcast(priv, "adding multicast entry for mgid %pI6\n",
 					mgid.raw);
 
-			nmcast = ipoib_mcast_alloc(dev, 0);
+			nmcast = ipoib_mcast_alloc(dev);
 			if (!nmcast) {
 				ipoib_warn(priv, "unable to allocate memory for multicast structure\n");
 				continue;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index b69304d28f06..587252fd6f57 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -206,6 +206,9 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
 	if (priv->hca_caps & IB_DEVICE_MANAGED_FLOW_STEERING)
 		init_attr.create_flags |= IB_QP_CREATE_NETIF_QP;
 
+	if (priv->hca_caps & IB_DEVICE_RDMA_NETDEV_OPA)
+		init_attr.create_flags |= IB_QP_CREATE_NETDEV_USE;
+
 	priv->qp = ib_create_qp(priv->pd, &init_attr);
 	if (IS_ERR(priv->qp)) {
 		pr_warn("%s: failed to create QP\n", ca->name);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
index 8ac8e18fbe0c..30865605e098 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
@@ -97,6 +97,7 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv,
 {
 	struct net_device *ndev = priv->dev;
 	int result;
+	struct rdma_netdev *rn = netdev_priv(ndev);
 
 	ASSERT_RTNL();
 
@@ -117,6 +118,8 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv,
 		goto out_early;
 	}
 
+	rn->mtu = priv->mcast_mtu;
+
 	priv->parent = ppriv->dev;
 	priv->pkey = pkey;
 	priv->child_type = type;
diff --git a/drivers/infiniband/ulp/iser/Kconfig b/drivers/infiniband/ulp/iser/Kconfig
index 1d29dffeaff9..3016a0c9a9f0 100644
--- a/drivers/infiniband/ulp/iser/Kconfig
+++ b/drivers/infiniband/ulp/iser/Kconfig
@@ -3,7 +3,7 @@ config INFINIBAND_ISER
 	tristate "iSCSI Extensions for RDMA (iSER)"
 	depends on SCSI && INET && INFINIBAND_ADDR_TRANS
 	select SCSI_ISCSI_ATTRS
-	---help---
+	help
 	  Support for the iSCSI Extensions for RDMA (iSER) Protocol
 	  over InfiniBand. This allows you to access storage devices
 	  that speak iSCSI over iSER over InfiniBand.
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 029c00163442..1d77c7f42e38 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -65,7 +65,6 @@
 #include <linux/in6.h>
 
 #include <rdma/ib_verbs.h>
-#include <rdma/ib_fmr_pool.h>
 #include <rdma/rdma_cm.h>
 
 #define DRV_NAME	"iser"
@@ -313,33 +312,6 @@ struct iser_comp {
 };
 
 /**
- * struct iser_reg_ops - Memory registration operations
- *     per-device registration schemes
- *
- * @alloc_reg_res:     Allocate registration resources
- * @free_reg_res:      Free registration resources
- * @reg_mem:           Register memory buffers
- * @unreg_mem:         Un-register memory buffers
- * @reg_desc_get:      Get a registration descriptor for pool
- * @reg_desc_put:      Get a registration descriptor to pool
- */
-struct iser_reg_ops {
-	int            (*alloc_reg_res)(struct ib_conn *ib_conn,
-					unsigned cmds_max,
-					unsigned int size);
-	void           (*free_reg_res)(struct ib_conn *ib_conn);
-	int            (*reg_mem)(struct iscsi_iser_task *iser_task,
-				  struct iser_data_buf *mem,
-				  struct iser_reg_resources *rsc,
-				  struct iser_mem_reg *reg);
-	void           (*unreg_mem)(struct iscsi_iser_task *iser_task,
-				    enum iser_data_dir cmd_dir);
-	struct iser_fr_desc * (*reg_desc_get)(struct ib_conn *ib_conn);
-	void           (*reg_desc_put)(struct ib_conn *ib_conn,
-				       struct iser_fr_desc *desc);
-};
-
-/**
  * struct iser_device - iSER device handle
  *
  * @ib_device:     RDMA device
@@ -351,8 +323,6 @@ struct iser_reg_ops {
  * @comps_used:    Number of completion contexts used, Min between online
  *                 cpus and device max completion vectors
  * @comps:         Dinamically allocated array of completion handlers
- * @reg_ops:       Registration ops
- * @remote_inv_sup: Remote invalidate is supported on this device
  */
 struct iser_device {
 	struct ib_device             *ib_device;
@@ -362,26 +332,18 @@ struct iser_device {
 	int                          refcount;
 	int			     comps_used;
 	struct iser_comp	     *comps;
-	const struct iser_reg_ops    *reg_ops;
-	bool                         remote_inv_sup;
 };
 
 /**
  * struct iser_reg_resources - Fast registration resources
  *
  * @mr:         memory region
- * @fmr_pool:   pool of fmrs
  * @sig_mr:     signature memory region
- * @page_vec:   fast reg page list used by fmr pool
  * @mr_valid:   is mr valid indicator
  */
 struct iser_reg_resources {
-	union {
-		struct ib_mr             *mr;
-		struct ib_fmr_pool       *fmr_pool;
-	};
+	struct ib_mr                     *mr;
 	struct ib_mr                     *sig_mr;
-	struct iser_page_vec             *page_vec;
 	u8				  mr_valid:1;
 };
 
@@ -403,7 +365,7 @@ struct iser_fr_desc {
  * struct iser_fr_pool - connection fast registration pool
  *
  * @list:                list of fastreg descriptors
- * @lock:                protects fmr/fastreg pool
+ * @lock:                protects fastreg pool
  * @size:                size of the pool
  */
 struct iser_fr_pool {
@@ -518,12 +480,6 @@ struct iscsi_iser_task {
 	struct iser_data_buf         prot[ISER_DIRS_NUM];
 };
 
-struct iser_page_vec {
-	u64 *pages;
-	int npages;
-	struct ib_mr fake_mr;
-};
-
 /**
  * struct iser_global - iSER global context
  *
@@ -548,8 +504,6 @@ extern int iser_pi_guard;
 extern unsigned int iser_max_sectors;
 extern bool iser_always_reg;
 
-int iser_assign_reg_ops(struct iser_device *device);
-
 int iser_send_control(struct iscsi_conn *conn,
 		      struct iscsi_task *task);
 
@@ -591,22 +545,17 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
 				     struct iser_data_buf *mem,
 				     enum iser_data_dir cmd_dir);
 
-int iser_reg_rdma_mem(struct iscsi_iser_task *task,
-		      enum iser_data_dir dir,
-		      bool all_imm);
-void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
-			 enum iser_data_dir dir);
+int iser_reg_mem_fastreg(struct iscsi_iser_task *task,
+			 enum iser_data_dir dir,
+			 bool all_imm);
+void iser_unreg_mem_fastreg(struct iscsi_iser_task *task,
+			    enum iser_data_dir dir);
 
 int  iser_connect(struct iser_conn *iser_conn,
 		  struct sockaddr *src_addr,
 		  struct sockaddr *dst_addr,
 		  int non_blocking);
 
-void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
-			enum iser_data_dir cmd_dir);
-void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
-			    enum iser_data_dir cmd_dir);
-
 int  iser_post_recvl(struct iser_conn *iser_conn);
 int  iser_post_recvm(struct iser_conn *iser_conn, int count);
 int  iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
@@ -625,26 +574,12 @@ int  iser_initialize_task_headers(struct iscsi_task *task,
 			struct iser_tx_desc *tx_desc);
 int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
 			      struct iscsi_session *session);
-int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
-			unsigned cmds_max,
-			unsigned int size);
-void iser_free_fmr_pool(struct ib_conn *ib_conn);
 int iser_alloc_fastreg_pool(struct ib_conn *ib_conn,
 			    unsigned cmds_max,
 			    unsigned int size);
 void iser_free_fastreg_pool(struct ib_conn *ib_conn);
 u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
 			     enum iser_data_dir cmd_dir, sector_t *sector);
-struct iser_fr_desc *
-iser_reg_desc_get_fr(struct ib_conn *ib_conn);
-void
-iser_reg_desc_put_fr(struct ib_conn *ib_conn,
-		     struct iser_fr_desc *desc);
-struct iser_fr_desc *
-iser_reg_desc_get_fmr(struct ib_conn *ib_conn);
-void
-iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
-		      struct iser_fr_desc *desc);
 
 static inline struct iser_conn *
 to_iser_conn(struct ib_conn *ib_conn)
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index 4a7045bb0831..27a6f75a9912 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -72,7 +72,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task)
 			return err;
 	}
 
-	err = iser_reg_rdma_mem(iser_task, ISER_DIR_IN, false);
+	err = iser_reg_mem_fastreg(iser_task, ISER_DIR_IN, false);
 	if (err) {
 		iser_err("Failed to set up Data-IN RDMA\n");
 		return err;
@@ -126,8 +126,8 @@ iser_prepare_write_cmd(struct iscsi_task *task,
 			return err;
 	}
 
-	err = iser_reg_rdma_mem(iser_task, ISER_DIR_OUT,
-				buf_out->data_len == imm_sz);
+	err = iser_reg_mem_fastreg(iser_task, ISER_DIR_OUT,
+				   buf_out->data_len == imm_sz);
 	if (err != 0) {
 		iser_err("Failed to register write cmd RDMA mem\n");
 		return err;
@@ -250,8 +250,8 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
 	iser_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */
 	iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;
 
-	if (device->reg_ops->alloc_reg_res(ib_conn, session->scsi_cmds_max,
-					   iser_conn->pages_per_mr))
+	if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max,
+				    iser_conn->pages_per_mr))
 		goto create_rdma_reg_res_failed;
 
 	if (iser_alloc_login_buf(iser_conn))
@@ -293,7 +293,7 @@ rx_desc_dma_map_failed:
 rx_desc_alloc_fail:
 	iser_free_login_buf(iser_conn);
 alloc_login_buf_fail:
-	device->reg_ops->free_reg_res(ib_conn);
+	iser_free_fastreg_pool(ib_conn);
 create_rdma_reg_res_failed:
 	iser_err("failed allocating rx descriptors / data buffers\n");
 	return -ENOMEM;
@@ -306,8 +306,7 @@ void iser_free_rx_descriptors(struct iser_conn *iser_conn)
 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
 	struct iser_device *device = ib_conn->device;
 
-	if (device->reg_ops->free_reg_res)
-		device->reg_ops->free_reg_res(ib_conn);
+	iser_free_fastreg_pool(ib_conn);
 
 	rx_desc = iser_conn->rx_descs;
 	for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)
@@ -768,7 +767,7 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
 	int prot_count = scsi_prot_sg_count(iser_task->sc);
 
 	if (iser_task->dir[ISER_DIR_IN]) {
-		iser_unreg_rdma_mem(iser_task, ISER_DIR_IN);
+		iser_unreg_mem_fastreg(iser_task, ISER_DIR_IN);
 		iser_dma_unmap_task_data(iser_task,
 					 &iser_task->data[ISER_DIR_IN],
 					 DMA_FROM_DEVICE);
@@ -779,7 +778,7 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
 	}
 
 	if (iser_task->dir[ISER_DIR_OUT]) {
-		iser_unreg_rdma_mem(iser_task, ISER_DIR_OUT);
+		iser_unreg_mem_fastreg(iser_task, ISER_DIR_OUT);
 		iser_dma_unmap_task_data(iser_task,
 					 &iser_task->data[ISER_DIR_OUT],
 					 DMA_TO_DEVICE);
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index 999ef7cdd05e..d4e057fac219 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -38,62 +38,13 @@
 #include <linux/scatterlist.h>
 
 #include "iscsi_iser.h"
-static
-int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
-		      struct iser_data_buf *mem,
-		      struct iser_reg_resources *rsc,
-		      struct iser_mem_reg *mem_reg);
-static
-int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
-		     struct iser_data_buf *mem,
-		     struct iser_reg_resources *rsc,
-		     struct iser_mem_reg *mem_reg);
-
-static const struct iser_reg_ops fastreg_ops = {
-	.alloc_reg_res	= iser_alloc_fastreg_pool,
-	.free_reg_res	= iser_free_fastreg_pool,
-	.reg_mem	= iser_fast_reg_mr,
-	.unreg_mem	= iser_unreg_mem_fastreg,
-	.reg_desc_get	= iser_reg_desc_get_fr,
-	.reg_desc_put	= iser_reg_desc_put_fr,
-};
-
-static const struct iser_reg_ops fmr_ops = {
-	.alloc_reg_res	= iser_alloc_fmr_pool,
-	.free_reg_res	= iser_free_fmr_pool,
-	.reg_mem	= iser_fast_reg_fmr,
-	.unreg_mem	= iser_unreg_mem_fmr,
-	.reg_desc_get	= iser_reg_desc_get_fmr,
-	.reg_desc_put	= iser_reg_desc_put_fmr,
-};
 
 void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc)
 {
 	iser_err_comp(wc, "memreg");
 }
 
-int iser_assign_reg_ops(struct iser_device *device)
-{
-	struct ib_device *ib_dev = device->ib_device;
-
-	/* Assign function handles  - based on FMR support */
-	if (ib_dev->ops.alloc_fmr && ib_dev->ops.dealloc_fmr &&
-	    ib_dev->ops.map_phys_fmr && ib_dev->ops.unmap_fmr) {
-		iser_info("FMR supported, using FMR for registration\n");
-		device->reg_ops = &fmr_ops;
-	} else if (ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
-		iser_info("FastReg supported, using FastReg for registration\n");
-		device->reg_ops = &fastreg_ops;
-		device->remote_inv_sup = iser_always_reg;
-	} else {
-		iser_err("IB device does not support FMRs nor FastRegs, can't register memory\n");
-		return -1;
-	}
-
-	return 0;
-}
-
-struct iser_fr_desc *
+static struct iser_fr_desc *
 iser_reg_desc_get_fr(struct ib_conn *ib_conn)
 {
 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
@@ -109,7 +60,7 @@ iser_reg_desc_get_fr(struct ib_conn *ib_conn)
 	return desc;
 }
 
-void
+static void
 iser_reg_desc_put_fr(struct ib_conn *ib_conn,
 		     struct iser_fr_desc *desc)
 {
@@ -121,44 +72,6 @@ iser_reg_desc_put_fr(struct ib_conn *ib_conn,
 	spin_unlock_irqrestore(&fr_pool->lock, flags);
 }
 
-struct iser_fr_desc *
-iser_reg_desc_get_fmr(struct ib_conn *ib_conn)
-{
-	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
-
-	return list_first_entry(&fr_pool->list,
-				struct iser_fr_desc, list);
-}
-
-void
-iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
-		      struct iser_fr_desc *desc)
-{
-}
-
-static void iser_data_buf_dump(struct iser_data_buf *data,
-			       struct ib_device *ibdev)
-{
-	struct scatterlist *sg;
-	int i;
-
-	for_each_sg(data->sg, sg, data->dma_nents, i)
-		iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p "
-			 "off:0x%x sz:0x%x dma_len:0x%x\n",
-			 i, (unsigned long)sg_dma_address(sg),
-			 sg_page(sg), sg->offset, sg->length, sg_dma_len(sg));
-}
-
-static void iser_dump_page_vec(struct iser_page_vec *page_vec)
-{
-	int i;
-
-	iser_err("page vec npages %d data length %lld\n",
-		 page_vec->npages, page_vec->fake_mr.length);
-	for (i = 0; i < page_vec->npages; i++)
-		iser_err("vec[%d]: %llx\n", i, page_vec->pages[i]);
-}
-
 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
 			    struct iser_data_buf *data,
 			    enum iser_data_dir iser_dir,
@@ -213,84 +126,9 @@ iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
 	return 0;
 }
 
-static int iser_set_page(struct ib_mr *mr, u64 addr)
-{
-	struct iser_page_vec *page_vec =
-		container_of(mr, struct iser_page_vec, fake_mr);
-
-	page_vec->pages[page_vec->npages++] = addr;
-
-	return 0;
-}
-
-static
-int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
-		      struct iser_data_buf *mem,
-		      struct iser_reg_resources *rsc,
-		      struct iser_mem_reg *reg)
-{
-	struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
-	struct iser_device *device = ib_conn->device;
-	struct iser_page_vec *page_vec = rsc->page_vec;
-	struct ib_fmr_pool *fmr_pool = rsc->fmr_pool;
-	struct ib_pool_fmr *fmr;
-	int ret, plen;
-
-	page_vec->npages = 0;
-	page_vec->fake_mr.page_size = SZ_4K;
-	plen = ib_sg_to_pages(&page_vec->fake_mr, mem->sg,
-			      mem->dma_nents, NULL, iser_set_page);
-	if (unlikely(plen < mem->dma_nents)) {
-		iser_err("page vec too short to hold this SG\n");
-		iser_data_buf_dump(mem, device->ib_device);
-		iser_dump_page_vec(page_vec);
-		return -EINVAL;
-	}
-
-	fmr  = ib_fmr_pool_map_phys(fmr_pool, page_vec->pages,
-				    page_vec->npages, page_vec->pages[0]);
-	if (IS_ERR(fmr)) {
-		ret = PTR_ERR(fmr);
-		iser_err("ib_fmr_pool_map_phys failed: %d\n", ret);
-		return ret;
-	}
-
-	reg->sge.lkey = fmr->fmr->lkey;
-	reg->rkey = fmr->fmr->rkey;
-	reg->sge.addr = page_vec->fake_mr.iova;
-	reg->sge.length = page_vec->fake_mr.length;
-	reg->mem_h = fmr;
-
-	iser_dbg("fmr reg: lkey=0x%x, rkey=0x%x, addr=0x%llx,"
-		 " length=0x%x\n", reg->sge.lkey, reg->rkey,
-		 reg->sge.addr, reg->sge.length);
-
-	return 0;
-}
-
-/**
- * Unregister (previosuly registered using FMR) memory.
- * If memory is non-FMR does nothing.
- */
-void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
-			enum iser_data_dir cmd_dir)
-{
-	struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
-
-	if (!reg->mem_h)
-		return;
-
-	iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n", reg->mem_h);
-
-	ib_fmr_pool_unmap((struct ib_pool_fmr *)reg->mem_h);
-
-	reg->mem_h = NULL;
-}
-
 void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
 			    enum iser_data_dir cmd_dir)
 {
-	struct iser_device *device = iser_task->iser_conn->ib_conn.device;
 	struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
 	struct iser_fr_desc *desc;
 	struct ib_mr_status mr_status;
@@ -312,7 +150,7 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
 		ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS,
 				   &mr_status);
 	}
-	device->reg_ops->reg_desc_put(&iser_task->iser_conn->ib_conn, desc);
+	iser_reg_desc_put_fr(&iser_task->iser_conn->ib_conn, reg->mem_h);
 	reg->mem_h = NULL;
 }
 
@@ -509,15 +347,14 @@ iser_reg_data_sg(struct iscsi_iser_task *task,
 	if (use_dma_key)
 		return iser_reg_dma(device, mem, reg);
 
-	return device->reg_ops->reg_mem(task, mem, &desc->rsc, reg);
+	return iser_fast_reg_mr(task, mem, &desc->rsc, reg);
 }
 
-int iser_reg_rdma_mem(struct iscsi_iser_task *task,
-		      enum iser_data_dir dir,
-		      bool all_imm)
+int iser_reg_mem_fastreg(struct iscsi_iser_task *task,
+			 enum iser_data_dir dir,
+			 bool all_imm)
 {
 	struct ib_conn *ib_conn = &task->iser_conn->ib_conn;
-	struct iser_device *device = ib_conn->device;
 	struct iser_data_buf *mem = &task->data[dir];
 	struct iser_mem_reg *reg = &task->rdma_reg[dir];
 	struct iser_fr_desc *desc = NULL;
@@ -528,7 +365,7 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task,
 		      scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL;
 
 	if (!use_dma_key) {
-		desc = device->reg_ops->reg_desc_get(ib_conn);
+		desc = iser_reg_desc_get_fr(ib_conn);
 		reg->mem_h = desc;
 	}
 
@@ -549,15 +386,8 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task,
 
 err_reg:
 	if (desc)
-		device->reg_ops->reg_desc_put(ib_conn, desc);
+		iser_reg_desc_put_fr(ib_conn, desc);
 
 	return err;
 }
 
-void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
-			 enum iser_data_dir dir)
-{
-	struct iser_device *device = task->iser_conn->ib_conn.device;
-
-	device->reg_ops->unreg_mem(task, dir);
-}
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 127887c6c03f..c1f44c41f501 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -68,11 +68,12 @@ static void iser_event_handler(struct ib_event_handler *handler,
 static int iser_create_device_ib_res(struct iser_device *device)
 {
 	struct ib_device *ib_dev = device->ib_device;
-	int ret, i, max_cqe;
+	int i, max_cqe;
 
-	ret = iser_assign_reg_ops(device);
-	if (ret)
-		return ret;
+	if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {
+		iser_err("IB device does not support memory registrations\n");
+		return -1;
+	}
 
 	device->comps_used = min_t(int, num_online_cpus(),
 				 ib_dev->num_comp_vectors);
@@ -147,96 +148,6 @@ static void iser_free_device_ib_res(struct iser_device *device)
 	device->pd = NULL;
 }
 
-/**
- * iser_alloc_fmr_pool - Creates FMR pool and page_vector
- * @ib_conn: connection RDMA resources
- * @cmds_max: max number of SCSI commands for this connection
- * @size: max number of pages per map request
- *
- * Return: 0 on success, or errno code on failure
- */
-int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
-			unsigned cmds_max,
-			unsigned int size)
-{
-	struct iser_device *device = ib_conn->device;
-	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
-	struct iser_page_vec *page_vec;
-	struct iser_fr_desc *desc;
-	struct ib_fmr_pool *fmr_pool;
-	struct ib_fmr_pool_param params;
-	int ret;
-
-	INIT_LIST_HEAD(&fr_pool->list);
-	spin_lock_init(&fr_pool->lock);
-
-	desc = kzalloc(sizeof(*desc), GFP_KERNEL);
-	if (!desc)
-		return -ENOMEM;
-
-	page_vec = kmalloc(sizeof(*page_vec) + (sizeof(u64) * size),
-			   GFP_KERNEL);
-	if (!page_vec) {
-		ret = -ENOMEM;
-		goto err_frpl;
-	}
-
-	page_vec->pages = (u64 *)(page_vec + 1);
-
-	params.page_shift        = ilog2(SZ_4K);
-	params.max_pages_per_fmr = size;
-	/* make the pool size twice the max number of SCSI commands *
-	 * the ML is expected to queue, watermark for unmap at 50%  */
-	params.pool_size	 = cmds_max * 2;
-	params.dirty_watermark	 = cmds_max;
-	params.cache		 = 0;
-	params.flush_function	 = NULL;
-	params.access		 = (IB_ACCESS_LOCAL_WRITE  |
-				    IB_ACCESS_REMOTE_WRITE |
-				    IB_ACCESS_REMOTE_READ);
-
-	fmr_pool = ib_create_fmr_pool(device->pd, &params);
-	if (IS_ERR(fmr_pool)) {
-		ret = PTR_ERR(fmr_pool);
-		iser_err("FMR allocation failed, err %d\n", ret);
-		goto err_fmr;
-	}
-
-	desc->rsc.page_vec = page_vec;
-	desc->rsc.fmr_pool = fmr_pool;
-	list_add(&desc->list, &fr_pool->list);
-
-	return 0;
-
-err_fmr:
-	kfree(page_vec);
-err_frpl:
-	kfree(desc);
-
-	return ret;
-}
-
-/**
- * iser_free_fmr_pool - releases the FMR pool and page vec
- * @ib_conn: connection RDMA resources
- */
-void iser_free_fmr_pool(struct ib_conn *ib_conn)
-{
-	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
-	struct iser_fr_desc *desc;
-
-	desc = list_first_entry(&fr_pool->list,
-				struct iser_fr_desc, list);
-	list_del(&desc->list);
-
-	iser_info("freeing conn %p fmr pool %p\n",
-		  ib_conn, desc->rsc.fmr_pool);
-
-	ib_destroy_fmr_pool(desc->rsc.fmr_pool);
-	kfree(desc->rsc.page_vec);
-	kfree(desc);
-}
-
 static struct iser_fr_desc *
 iser_create_fastreg_desc(struct iser_device *device,
 			 struct ib_pd *pd,
@@ -667,13 +578,12 @@ iser_calc_scsi_params(struct iser_conn *iser_conn,
 	u32 max_num_sg;
 
 	/*
-	 * FRs without SG_GAPS or FMRs can only map up to a (device) page per
-	 * entry, but if the first entry is misaligned we'll end up using two
-	 * entries (head and tail) for a single page worth data, so one
-	 * additional entry is required.
+	 * FRs without SG_GAPS can only map up to a (device) page per entry,
+	 * but if the first entry is misaligned we'll end up using two entries
+	 * (head and tail) for a single page worth data, so one additional
+	 * entry is required.
 	 */
-	if ((attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) &&
-	    (attr->device_cap_flags & IB_DEVICE_SG_GAPS_REG))
+	if (attr->device_cap_flags & IB_DEVICE_SG_GAPS_REG)
 		reserved_mr_pages = 0;
 	else
 		reserved_mr_pages = 1;
@@ -684,14 +594,8 @@ iser_calc_scsi_params(struct iser_conn *iser_conn,
 		max_num_sg = attr->max_fast_reg_page_list_len;
 
 	sg_tablesize = DIV_ROUND_UP(max_sectors * SECTOR_SIZE, SZ_4K);
-	if (attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)
-		sup_sg_tablesize =
-			min_t(
-			 uint, ISCSI_ISER_MAX_SG_TABLESIZE,
-			 max_num_sg - reserved_mr_pages);
-	else
-		sup_sg_tablesize = ISCSI_ISER_MAX_SG_TABLESIZE;
-
+	sup_sg_tablesize = min_t(uint, ISCSI_ISER_MAX_SG_TABLESIZE,
+				 max_num_sg - reserved_mr_pages);
 	iser_conn->scsi_sg_tablesize = min(sg_tablesize, sup_sg_tablesize);
 	iser_conn->pages_per_mr =
 		iser_conn->scsi_sg_tablesize + reserved_mr_pages;
@@ -755,7 +659,7 @@ static void iser_route_handler(struct rdma_cm_id *cma_id)
 	struct iser_cm_hdr req_hdr;
 	struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
-	struct iser_device *device = ib_conn->device;
+	struct ib_device *ib_dev = ib_conn->device->ib_device;
 
 	if (iser_conn->state != ISER_CONN_PENDING)
 		/* bailout */
@@ -766,14 +670,14 @@ static void iser_route_handler(struct rdma_cm_id *cma_id)
 		goto failure;
 
 	memset(&conn_param, 0, sizeof conn_param);
-	conn_param.responder_resources = device->ib_device->attrs.max_qp_rd_atom;
+	conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom;
 	conn_param.initiator_depth     = 1;
 	conn_param.retry_count	       = 7;
 	conn_param.rnr_retry_count     = 6;
 
 	memset(&req_hdr, 0, sizeof(req_hdr));
 	req_hdr.flags = ISER_ZBVA_NOT_SUP;
-	if (!device->remote_inv_sup)
+	if (!iser_always_reg)
 		req_hdr.flags |= ISER_SEND_W_INV_NOT_SUP;
 	conn_param.private_data	= (void *)&req_hdr;
 	conn_param.private_data_len = sizeof(struct iser_cm_hdr);
diff --git a/drivers/infiniband/ulp/isert/Kconfig b/drivers/infiniband/ulp/isert/Kconfig
index 1a3f5ca8354c..798147adbef1 100644
--- a/drivers/infiniband/ulp/isert/Kconfig
+++ b/drivers/infiniband/ulp/isert/Kconfig
@@ -2,5 +2,5 @@
 config INFINIBAND_ISERT
 	tristate "iSCSI Extensions for RDMA (iSER) target support"
 	depends on INET && INFINIBAND_ADDR_TRANS && TARGET_CORE && ISCSI_TARGET
-	---help---
+	help
 	Support for iSCSI Extensions for RDMA (iSER) Target on Infiniband fabrics.
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index a1a035270cab..b7df38ee8ae0 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -15,6 +15,7 @@
 #include <linux/in.h>
 #include <linux/in6.h>
 #include <rdma/ib_verbs.h>
+#include <rdma/ib_cm.h>
 #include <rdma/rdma_cm.h>
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
@@ -502,7 +503,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
 	if (!np->enabled) {
 		spin_unlock_bh(&np->np_thread_lock);
 		isert_dbg("iscsi_np is not enabled, reject connect request\n");
-		return rdma_reject(cma_id, NULL, 0);
+		return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
 	}
 	spin_unlock_bh(&np->np_thread_lock);
 
@@ -553,7 +554,7 @@ out_rsp_dma_map:
 	isert_free_login_buf(isert_conn);
 out:
 	kfree(isert_conn);
-	rdma_reject(cma_id, NULL, 0);
+	rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
 	return ret;
 }
 
diff --git a/drivers/infiniband/ulp/opa_vnic/Kconfig b/drivers/infiniband/ulp/opa_vnic/Kconfig
index a1f266b9c0b2..e84248587187 100644
--- a/drivers/infiniband/ulp/opa_vnic/Kconfig
+++ b/drivers/infiniband/ulp/opa_vnic/Kconfig
@@ -2,7 +2,7 @@
 config INFINIBAND_OPA_VNIC
 	tristate "Intel OPA VNIC support"
 	depends on X86_64 && INFINIBAND
-	---help---
+	help
 	This is Omni-Path (OPA) Virtual Network Interface Controller (VNIC)
 	driver for Ethernet over Omni-Path feature. It implements the HW
 	independent VNIC functionality. It interfaces with Linux stack for
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
index 6e8d650c17c7..874a8eb7638c 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
@@ -113,7 +113,7 @@ struct opa_vnic_vema_port {
 	struct mutex                    lock;
 };
 
-static void opa_vnic_vema_add_one(struct ib_device *device);
+static int opa_vnic_vema_add_one(struct ib_device *device);
 static void opa_vnic_vema_rem_one(struct ib_device *device,
 				  void *client_data);
 
@@ -989,18 +989,18 @@ static void opa_vnic_ctrl_config_dev(struct opa_vnic_ctrl_port *cport, bool en)
  *
  * Allocate the vnic control port and initialize it.
  */
-static void opa_vnic_vema_add_one(struct ib_device *device)
+static int opa_vnic_vema_add_one(struct ib_device *device)
 {
 	struct opa_vnic_ctrl_port *cport;
 	int rc, size = sizeof(*cport);
 
 	if (!rdma_cap_opa_vnic(device))
-		return;
+		return -EOPNOTSUPP;
 
 	size += device->phys_port_cnt * sizeof(struct opa_vnic_vema_port);
 	cport = kzalloc(size, GFP_KERNEL);
 	if (!cport)
-		return;
+		return -ENOMEM;
 
 	cport->num_ports = device->phys_port_cnt;
 	cport->ibdev = device;
@@ -1012,6 +1012,7 @@ static void opa_vnic_vema_add_one(struct ib_device *device)
 
 	ib_set_client_data(device, &opa_vnic_client, cport);
 	opa_vnic_ctrl_config_dev(cport, true);
+	return 0;
 }
 
 /**
@@ -1026,9 +1027,6 @@ static void opa_vnic_vema_rem_one(struct ib_device *device,
 {
 	struct opa_vnic_ctrl_port *cport = client_data;
 
-	if (!cport)
-		return;
-
 	c_info("removing VNIC client\n");
 	opa_vnic_ctrl_config_dev(cport, false);
 	vema_unregister(cport);
diff --git a/drivers/infiniband/ulp/rtrs/Kconfig b/drivers/infiniband/ulp/rtrs/Kconfig
new file mode 100644
index 000000000000..9092b62e6dc8
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/Kconfig
@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+config INFINIBAND_RTRS
+	tristate
+	depends on INFINIBAND_ADDR_TRANS
+
+config INFINIBAND_RTRS_CLIENT
+	tristate "RTRS client module"
+	depends on INFINIBAND_ADDR_TRANS
+	select INFINIBAND_RTRS
+	help
+	  RDMA transport client module.
+
+	  RDMA Transport (RTRS) client implements a reliable transport layer
+	  and also multipathing functionality and that it is intended to be
+	  the base layer for a block storage initiator over RDMA.
+
+config INFINIBAND_RTRS_SERVER
+	tristate "RTRS server module"
+	depends on INFINIBAND_ADDR_TRANS
+	select INFINIBAND_RTRS
+	help
+	  RDMA transport server module.
+
+	  RDMA Transport (RTRS) server module processing connection and IO
+	  requests received from the RTRS client module, it will pass the
+	  IO requests to its user eg. RNBD_server.
diff --git a/drivers/infiniband/ulp/rtrs/Makefile b/drivers/infiniband/ulp/rtrs/Makefile
new file mode 100644
index 000000000000..3898509be270
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/Makefile
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+rtrs-client-y := rtrs-clt.o \
+		  rtrs-clt-stats.o \
+		  rtrs-clt-sysfs.o
+
+rtrs-server-y := rtrs-srv.o \
+		  rtrs-srv-stats.o \
+		  rtrs-srv-sysfs.o
+
+rtrs-core-y := rtrs.o
+
+obj-$(CONFIG_INFINIBAND_RTRS)        += rtrs-core.o
+obj-$(CONFIG_INFINIBAND_RTRS_CLIENT) += rtrs-client.o
+obj-$(CONFIG_INFINIBAND_RTRS_SERVER) += rtrs-server.o
diff --git a/drivers/infiniband/ulp/rtrs/README b/drivers/infiniband/ulp/rtrs/README
new file mode 100644
index 000000000000..5d9ea142e5dd
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/README
@@ -0,0 +1,213 @@
+****************************
+RDMA Transport (RTRS)
+****************************
+
+RTRS (RDMA Transport) is a reliable high speed transport library
+which provides support to establish optimal number of connections
+between client and server machines using RDMA (InfiniBand, RoCE, iWarp)
+transport. It is optimized to transfer (read/write) IO blocks.
+
+In its core interface it follows the BIO semantics of providing the
+possibility to either write data from an sg list to the remote side
+or to request ("read") data transfer from the remote side into a given
+sg list.
+
+RTRS provides I/O fail-over and load-balancing capabilities by using
+multipath I/O (see "add_path" and "mp_policy" configuration entries in
+Documentation/ABI/testing/sysfs-class-rtrs-client).
+
+RTRS is used by the RNBD (RDMA Network Block Device) modules.
+
+==================
+Transport protocol
+==================
+
+Overview
+--------
+An established connection between a client and a server is called rtrs
+session. A session is associated with a set of memory chunks reserved on the
+server side for a given client for rdma transfer. A session
+consists of multiple paths, each representing a separate physical link
+between client and server. Those are used for load balancing and failover.
+Each path consists of as many connections (QPs) as there are cpus on
+the client.
+
+When processing an incoming write or read request, rtrs client uses memory
+chunks reserved for him on the server side. Their number, size and addresses
+need to be exchanged between client and server during the connection
+establishment phase. Apart from the memory related information client needs to
+inform the server about the session name and identify each path and connection
+individually.
+
+On an established session client sends to server write or read messages.
+Server uses immediate field to tell the client which request is being
+acknowledged and for errno. Client uses immediate field to tell the server
+which of the memory chunks has been accessed and at which offset the message
+can be found.
+
+Module parameter always_invalidate is introduced for the security problem
+discussed in LPC RDMA MC 2019. When always_invalidate=Y, on the server side we
+invalidate each rdma buffer before we hand it over to RNBD server and
+then pass it to the block layer. A new rkey is generated and registered for the
+buffer after it returns back from the block layer and RNBD server.
+The new rkey is sent back to the client along with the IO result.
+The procedure is the default behaviour of the driver. This invalidation and
+registration on each IO causes performance drop of up to 20%. A user of the
+driver may choose to load the modules with this mechanism switched off
+(always_invalidate=N), if he understands and can take the risk of a malicious
+client being able to corrupt memory of a server it is connected to. This might
+be a reasonable option in a scenario where all the clients and all the servers
+are located within a secure datacenter.
+
+
+Connection establishment
+------------------------
+
+1. Client starts establishing connections belonging to a path of a session one
+by one via attaching RTRS_MSG_CON_REQ messages to the rdma_connect requests.
+Those include uuid of the session and uuid of the path to be
+established. They are used by the server to find a persisting session/path or
+to create a new one when necessary. The message also contains the protocol
+version and magic for compatibility, total number of connections per session
+(as many as cpus on the client), the id of the current connection and
+the reconnect counter, which is used to resolve the situations where
+client is trying to reconnect a path, while server is still destroying the old
+one.
+
+2. Server accepts the connection requests one by one and attaches
+RTRS_MSG_CONN_RSP messages to the rdma_accept. Apart from magic and
+protocol version, the messages include error code, queue depth supported by
+the server (number of memory chunks which are going to be allocated for that
+session) and the maximum size of one io, RTRS_MSG_NEW_RKEY_F flags is set
+when always_invalidate=Y.
+
+3. After all connections of a path are established client sends to server the
+RTRS_MSG_INFO_REQ message, containing the name of the session. This message
+requests the address information from the server.
+
+4. Server replies to the session info request message with RTRS_MSG_INFO_RSP,
+which contains the addresses and keys of the RDMA buffers allocated for that
+session.
+
+5. Session becomes connected after all paths to be established are connected
+(i.e. steps 1-4 finished for all paths requested for a session)
+
+6. Server and client exchange periodically heartbeat messages (empty rdma
+messages with an immediate field) which are used to detect a crash on remote
+side or network outage in an absence of IO.
+
+7. On any RDMA related error or in the case of a heartbeat timeout, the
+corresponding path is disconnected, all the inflight IO are failed over to a
+healthy path, if any, and the reconnect mechanism is triggered.
+
+CLT                                     SRV
+*for each connection belonging to a path and for each path:
+RTRS_MSG_CON_REQ  ------------------->
+                   <------------------- RTRS_MSG_CON_RSP
+...
+*after all connections are established:
+RTRS_MSG_INFO_REQ ------------------->
+                   <------------------- RTRS_MSG_INFO_RSP
+*heartbeat is started from both sides:
+                   -------------------> [RTRS_HB_MSG_IMM]
+[RTRS_HB_MSG_ACK] <-------------------
+[RTRS_HB_MSG_IMM] <-------------------
+                   -------------------> [RTRS_HB_MSG_ACK]
+
+IO path
+-------
+
+* Write (always_invalidate=N) *
+
+1. When processing a write request client selects one of the memory chunks
+on the server side and rdma writes there the user data, user header and the
+RTRS_MSG_RDMA_WRITE message. Apart from the type (write), the message only
+contains size of the user header. The client tells the server which chunk has
+been accessed and at what offset the RTRS_MSG_RDMA_WRITE can be found by
+using the IMM field.
+
+2. When confirming a write request server sends an "empty" rdma message with
+an immediate field. The 32 bit field is used to specify the outstanding
+inflight IO and for the error code.
+
+CLT                                                          SRV
+usr_data + usr_hdr + rtrs_msg_rdma_write -----------------> [RTRS_IO_REQ_IMM]
+[RTRS_IO_RSP_IMM]                        <----------------- (id + errno)
+
+* Write (always_invalidate=Y) *
+
+1. When processing a write request client selects one of the memory chunks
+on the server side and rdma writes there the user data, user header and the
+RTRS_MSG_RDMA_WRITE message. Apart from the type (write), the message only
+contains size of the user header. The client tells the server which chunk has
+been accessed and at what offset the RTRS_MSG_RDMA_WRITE can be found by
+using the IMM field, Server invalidate rkey associated to the memory chunks
+first, when it finishes, pass the IO to RNBD server module.
+
+2. When confirming a write request server sends an "empty" rdma message with
+an immediate field. The 32 bit field is used to specify the outstanding
+inflight IO and for the error code. The new rkey is sent back using
+SEND_WITH_IMM WR, client When it recived new rkey message, it validates
+the message and finished IO after update rkey for the rbuffer, then post
+back the recv buffer for later use.
+
+CLT                                                          SRV
+usr_data + usr_hdr + rtrs_msg_rdma_write -----------------> [RTRS_IO_REQ_IMM]
+[RTRS_MSG_RKEY_RSP]                     <----------------- (RTRS_MSG_RKEY_RSP)
+[RTRS_IO_RSP_IMM]                        <----------------- (id + errno)
+
+
+* Read (always_invalidate=N)*
+
+1. When processing a read request client selects one of the memory chunks
+on the server side and rdma writes there the user header and the
+RTRS_MSG_RDMA_READ message. This message contains the type (read), size of
+the user header, flags (specifying if memory invalidation is necessary) and the
+list of addresses along with keys for the data to be read into.
+
+2. When confirming a read request server transfers the requested data first,
+attaches an invalidation message if requested and finally an "empty" rdma
+message with an immediate field. The 32 bit field is used to specify the
+outstanding inflight IO and the error code.
+
+CLT                                           SRV
+usr_hdr + rtrs_msg_rdma_read --------------> [RTRS_IO_REQ_IMM]
+[RTRS_IO_RSP_IMM]            <-------------- usr_data + (id + errno)
+or in case client requested invalidation:
+[RTRS_IO_RSP_IMM_W_INV]      <-------------- usr_data + (INV) + (id + errno)
+
+* Read (always_invalidate=Y)*
+
+1. When processing a read request client selects one of the memory chunks
+on the server side and rdma writes there the user header and the
+RTRS_MSG_RDMA_READ message. This message contains the type (read), size of
+the user header, flags (specifying if memory invalidation is necessary) and the
+list of addresses along with keys for the data to be read into.
+Server invalidate rkey associated to the memory chunks first, when it finishes,
+passes the IO to RNBD server module.
+
+2. When confirming a read request server transfers the requested data first,
+attaches an invalidation message if requested and finally an "empty" rdma
+message with an immediate field. The 32 bit field is used to specify the
+outstanding inflight IO and the error code. The new rkey is sent back using
+SEND_WITH_IMM WR, client When it recived new rkey message, it validates
+the message and finished IO after update rkey for the rbuffer, then post
+back the recv buffer for later use.
+
+CLT                                           SRV
+usr_hdr + rtrs_msg_rdma_read --------------> [RTRS_IO_REQ_IMM]
+[RTRS_IO_RSP_IMM]            <-------------- usr_data + (id + errno)
+[RTRS_MSG_RKEY_RSP]	     <----------------- (RTRS_MSG_RKEY_RSP)
+or in case client requested invalidation:
+[RTRS_IO_RSP_IMM_W_INV]      <-------------- usr_data + (INV) + (id + errno)
+=========================================
+Contributors List(in alphabetical order)
+=========================================
+Danil Kipnis <danil.kipnis@profitbricks.com>
+Fabian Holler <mail@fholler.de>
+Guoqing Jiang <guoqing.jiang@cloud.ionos.com>
+Jack Wang <jinpu.wang@profitbricks.com>
+Kleber Souza <kleber.souza@profitbricks.com>
+Lutz Pogrell <lutz.pogrell@cloud.ionos.com>
+Milind Dumbare <Milind.dumbare@gmail.com>
+Roman Penyaev <roman.penyaev@profitbricks.com>
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c
new file mode 100644
index 000000000000..26bbe5d6dff5
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include "rtrs-clt.h"
+
+void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct rtrs_clt_stats *stats = sess->stats;
+	struct rtrs_clt_stats_pcpu *s;
+	int cpu;
+
+	cpu = raw_smp_processor_id();
+	s = this_cpu_ptr(stats->pcpu_stats);
+	if (unlikely(con->cpu != cpu)) {
+		s->cpu_migr.to++;
+
+		/* Careful here, override s pointer */
+		s = per_cpu_ptr(stats->pcpu_stats, con->cpu);
+		atomic_inc(&s->cpu_migr.from);
+	}
+}
+
+void rtrs_clt_inc_failover_cnt(struct rtrs_clt_stats *stats)
+{
+	struct rtrs_clt_stats_pcpu *s;
+
+	s = this_cpu_ptr(stats->pcpu_stats);
+	s->rdma.failover_cnt++;
+}
+
+int rtrs_clt_stats_migration_cnt_to_str(struct rtrs_clt_stats *stats,
+					 char *buf, size_t len)
+{
+	struct rtrs_clt_stats_pcpu *s;
+
+	size_t used;
+	int cpu;
+
+	used = scnprintf(buf, len, "    ");
+	for_each_possible_cpu(cpu)
+		used += scnprintf(buf + used, len - used, " CPU%u", cpu);
+
+	used += scnprintf(buf + used, len - used, "\nfrom:");
+	for_each_possible_cpu(cpu) {
+		s = per_cpu_ptr(stats->pcpu_stats, cpu);
+		used += scnprintf(buf + used, len - used, " %d",
+				  atomic_read(&s->cpu_migr.from));
+	}
+
+	used += scnprintf(buf + used, len - used, "\nto  :");
+	for_each_possible_cpu(cpu) {
+		s = per_cpu_ptr(stats->pcpu_stats, cpu);
+		used += scnprintf(buf + used, len - used, " %d",
+				  s->cpu_migr.to);
+	}
+	used += scnprintf(buf + used, len - used, "\n");
+
+	return used;
+}
+
+int rtrs_clt_stats_reconnects_to_str(struct rtrs_clt_stats *stats, char *buf,
+				      size_t len)
+{
+	return scnprintf(buf, len, "%d %d\n",
+			 stats->reconnects.successful_cnt,
+			 stats->reconnects.fail_cnt);
+}
+
+ssize_t rtrs_clt_stats_rdma_to_str(struct rtrs_clt_stats *stats,
+				    char *page, size_t len)
+{
+	struct rtrs_clt_stats_rdma sum;
+	struct rtrs_clt_stats_rdma *r;
+	int cpu;
+
+	memset(&sum, 0, sizeof(sum));
+
+	for_each_possible_cpu(cpu) {
+		r = &per_cpu_ptr(stats->pcpu_stats, cpu)->rdma;
+
+		sum.dir[READ].cnt	  += r->dir[READ].cnt;
+		sum.dir[READ].size_total  += r->dir[READ].size_total;
+		sum.dir[WRITE].cnt	  += r->dir[WRITE].cnt;
+		sum.dir[WRITE].size_total += r->dir[WRITE].size_total;
+		sum.failover_cnt	  += r->failover_cnt;
+	}
+
+	return scnprintf(page, len, "%llu %llu %llu %llu %u %llu\n",
+			 sum.dir[READ].cnt, sum.dir[READ].size_total,
+			 sum.dir[WRITE].cnt, sum.dir[WRITE].size_total,
+			 atomic_read(&stats->inflight), sum.failover_cnt);
+}
+
+ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *s,
+				 char *page, size_t len)
+{
+	return scnprintf(page, len, "echo 1 to reset all statistics\n");
+}
+
+int rtrs_clt_reset_rdma_stats(struct rtrs_clt_stats *stats, bool enable)
+{
+	struct rtrs_clt_stats_pcpu *s;
+	int cpu;
+
+	if (!enable)
+		return -EINVAL;
+
+	for_each_possible_cpu(cpu) {
+		s = per_cpu_ptr(stats->pcpu_stats, cpu);
+		memset(&s->rdma, 0, sizeof(s->rdma));
+	}
+
+	return 0;
+}
+
+int rtrs_clt_reset_cpu_migr_stats(struct rtrs_clt_stats *stats, bool enable)
+{
+	struct rtrs_clt_stats_pcpu *s;
+	int cpu;
+
+	if (!enable)
+		return -EINVAL;
+
+	for_each_possible_cpu(cpu) {
+		s = per_cpu_ptr(stats->pcpu_stats, cpu);
+		memset(&s->cpu_migr, 0, sizeof(s->cpu_migr));
+	}
+
+	return 0;
+}
+
+int rtrs_clt_reset_reconnects_stat(struct rtrs_clt_stats *stats, bool enable)
+{
+	if (!enable)
+		return -EINVAL;
+
+	memset(&stats->reconnects, 0, sizeof(stats->reconnects));
+
+	return 0;
+}
+
+int rtrs_clt_reset_all_stats(struct rtrs_clt_stats *s, bool enable)
+{
+	if (enable) {
+		rtrs_clt_reset_rdma_stats(s, enable);
+		rtrs_clt_reset_cpu_migr_stats(s, enable);
+		rtrs_clt_reset_reconnects_stat(s, enable);
+		atomic_set(&s->inflight, 0);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static inline void rtrs_clt_update_rdma_stats(struct rtrs_clt_stats *stats,
+					       size_t size, int d)
+{
+	struct rtrs_clt_stats_pcpu *s;
+
+	s = this_cpu_ptr(stats->pcpu_stats);
+	s->rdma.dir[d].cnt++;
+	s->rdma.dir[d].size_total += size;
+}
+
+void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir)
+{
+	struct rtrs_clt_con *con = req->con;
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct rtrs_clt_stats *stats = sess->stats;
+	unsigned int len;
+
+	len = req->usr_len + req->data_len;
+	rtrs_clt_update_rdma_stats(stats, len, dir);
+	if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT)
+		atomic_inc(&stats->inflight);
+}
+
+int rtrs_clt_init_stats(struct rtrs_clt_stats *stats)
+{
+	stats->pcpu_stats = alloc_percpu(typeof(*stats->pcpu_stats));
+	if (!stats->pcpu_stats)
+		return -ENOMEM;
+
+	/*
+	 * successful_cnt will be set to 0 after session
+	 * is established for the first time
+	 */
+	stats->reconnects.successful_cnt = -1;
+
+	return 0;
+}
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c
new file mode 100644
index 000000000000..298b747d0330
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c
@@ -0,0 +1,483 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include "rtrs-pri.h"
+#include "rtrs-clt.h"
+#include "rtrs-log.h"
+
+#define MIN_MAX_RECONN_ATT -1
+#define MAX_MAX_RECONN_ATT 9999
+
+static void rtrs_clt_sess_release(struct kobject *kobj)
+{
+	struct rtrs_clt_sess *sess;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+
+	free_sess(sess);
+}
+
+static struct kobj_type ktype_sess = {
+	.sysfs_ops = &kobj_sysfs_ops,
+	.release = rtrs_clt_sess_release
+};
+
+static void rtrs_clt_sess_stats_release(struct kobject *kobj)
+{
+	struct rtrs_clt_stats *stats;
+
+	stats = container_of(kobj, struct rtrs_clt_stats, kobj_stats);
+
+	free_percpu(stats->pcpu_stats);
+
+	kfree(stats);
+}
+
+static struct kobj_type ktype_stats = {
+	.sysfs_ops = &kobj_sysfs_ops,
+	.release = rtrs_clt_sess_stats_release,
+};
+
+static ssize_t max_reconnect_attempts_show(struct device *dev,
+					   struct device_attribute *attr,
+					   char *page)
+{
+	struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);
+
+	return sprintf(page, "%d\n", rtrs_clt_get_max_reconnect_attempts(clt));
+}
+
+static ssize_t max_reconnect_attempts_store(struct device *dev,
+					    struct device_attribute *attr,
+					    const char *buf,
+					    size_t count)
+{
+	int value;
+	int ret;
+	struct rtrs_clt *clt  = container_of(dev, struct rtrs_clt, dev);
+
+	ret = kstrtoint(buf, 10, &value);
+	if (ret) {
+		rtrs_err(clt, "%s: failed to convert string '%s' to int\n",
+			  attr->attr.name, buf);
+		return ret;
+	}
+	if (value > MAX_MAX_RECONN_ATT ||
+		     value < MIN_MAX_RECONN_ATT) {
+		rtrs_err(clt,
+			  "%s: invalid range (provided: '%s', accepted: min: %d, max: %d)\n",
+			  attr->attr.name, buf, MIN_MAX_RECONN_ATT,
+			  MAX_MAX_RECONN_ATT);
+		return -EINVAL;
+	}
+	rtrs_clt_set_max_reconnect_attempts(clt, value);
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(max_reconnect_attempts);
+
+static ssize_t mpath_policy_show(struct device *dev,
+				 struct device_attribute *attr,
+				 char *page)
+{
+	struct rtrs_clt *clt;
+
+	clt = container_of(dev, struct rtrs_clt, dev);
+
+	switch (clt->mp_policy) {
+	case MP_POLICY_RR:
+		return sprintf(page, "round-robin (RR: %d)\n", clt->mp_policy);
+	case MP_POLICY_MIN_INFLIGHT:
+		return sprintf(page, "min-inflight (MI: %d)\n", clt->mp_policy);
+	default:
+		return sprintf(page, "Unknown (%d)\n", clt->mp_policy);
+	}
+}
+
+static ssize_t mpath_policy_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf,
+				  size_t count)
+{
+	struct rtrs_clt *clt;
+	int value;
+	int ret;
+
+	clt = container_of(dev, struct rtrs_clt, dev);
+
+	ret = kstrtoint(buf, 10, &value);
+	if (!ret && (value == MP_POLICY_RR ||
+		     value == MP_POLICY_MIN_INFLIGHT)) {
+		clt->mp_policy = value;
+		return count;
+	}
+
+	if (!strncasecmp(buf, "round-robin", 11) ||
+	    !strncasecmp(buf, "rr", 2))
+		clt->mp_policy = MP_POLICY_RR;
+	else if (!strncasecmp(buf, "min-inflight", 12) ||
+		 !strncasecmp(buf, "mi", 2))
+		clt->mp_policy = MP_POLICY_MIN_INFLIGHT;
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(mpath_policy);
+
+static ssize_t add_path_show(struct device *dev,
+			     struct device_attribute *attr, char *page)
+{
+	return scnprintf(page, PAGE_SIZE,
+			 "Usage: echo [<source addr>@]<destination addr> > %s\n\n*addr ::= [ ip:<ipv4|ipv6> | gid:<gid> ]\n",
+			 attr->attr.name);
+}
+
+static ssize_t add_path_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	struct sockaddr_storage srcaddr, dstaddr;
+	struct rtrs_addr addr = {
+		.src = &srcaddr,
+		.dst = &dstaddr
+	};
+	struct rtrs_clt *clt;
+	const char *nl;
+	size_t len;
+	int err;
+
+	clt = container_of(dev, struct rtrs_clt, dev);
+
+	nl = strchr(buf, '\n');
+	if (nl)
+		len = nl - buf;
+	else
+		len = count;
+	err = rtrs_addr_to_sockaddr(buf, len, clt->port, &addr);
+	if (err)
+		return -EINVAL;
+
+	err = rtrs_clt_create_path_from_sysfs(clt, &addr);
+	if (err)
+		return err;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(add_path);
+
+static ssize_t rtrs_clt_state_show(struct kobject *kobj,
+				    struct kobj_attribute *attr, char *page)
+{
+	struct rtrs_clt_sess *sess;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+	if (sess->state == RTRS_CLT_CONNECTED)
+		return sprintf(page, "connected\n");
+
+	return sprintf(page, "disconnected\n");
+}
+
+static struct kobj_attribute rtrs_clt_state_attr =
+	__ATTR(state, 0444, rtrs_clt_state_show, NULL);
+
+static ssize_t rtrs_clt_reconnect_show(struct kobject *kobj,
+					struct kobj_attribute *attr,
+					char *page)
+{
+	return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
+			 attr->attr.name);
+}
+
+static ssize_t rtrs_clt_reconnect_store(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct rtrs_clt_sess *sess;
+	int ret;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+	if (!sysfs_streq(buf, "1")) {
+		rtrs_err(sess->clt, "%s: unknown value: '%s'\n",
+			  attr->attr.name, buf);
+		return -EINVAL;
+	}
+	ret = rtrs_clt_reconnect_from_sysfs(sess);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static struct kobj_attribute rtrs_clt_reconnect_attr =
+	__ATTR(reconnect, 0644, rtrs_clt_reconnect_show,
+	       rtrs_clt_reconnect_store);
+
+static ssize_t rtrs_clt_disconnect_show(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 char *page)
+{
+	return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
+			 attr->attr.name);
+}
+
+static ssize_t rtrs_clt_disconnect_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct rtrs_clt_sess *sess;
+	int ret;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+	if (!sysfs_streq(buf, "1")) {
+		rtrs_err(sess->clt, "%s: unknown value: '%s'\n",
+			  attr->attr.name, buf);
+		return -EINVAL;
+	}
+	ret = rtrs_clt_disconnect_from_sysfs(sess);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static struct kobj_attribute rtrs_clt_disconnect_attr =
+	__ATTR(disconnect, 0644, rtrs_clt_disconnect_show,
+	       rtrs_clt_disconnect_store);
+
+static ssize_t rtrs_clt_remove_path_show(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  char *page)
+{
+	return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
+			 attr->attr.name);
+}
+
+static ssize_t rtrs_clt_remove_path_store(struct kobject *kobj,
+					   struct kobj_attribute *attr,
+					   const char *buf, size_t count)
+{
+	struct rtrs_clt_sess *sess;
+	int ret;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+	if (!sysfs_streq(buf, "1")) {
+		rtrs_err(sess->clt, "%s: unknown value: '%s'\n",
+			  attr->attr.name, buf);
+		return -EINVAL;
+	}
+	ret = rtrs_clt_remove_path_from_sysfs(sess, &attr->attr);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static struct kobj_attribute rtrs_clt_remove_path_attr =
+	__ATTR(remove_path, 0644, rtrs_clt_remove_path_show,
+	       rtrs_clt_remove_path_store);
+
+STAT_ATTR(struct rtrs_clt_stats, cpu_migration,
+	  rtrs_clt_stats_migration_cnt_to_str,
+	  rtrs_clt_reset_cpu_migr_stats);
+
+STAT_ATTR(struct rtrs_clt_stats, reconnects,
+	  rtrs_clt_stats_reconnects_to_str,
+	  rtrs_clt_reset_reconnects_stat);
+
+STAT_ATTR(struct rtrs_clt_stats, rdma,
+	  rtrs_clt_stats_rdma_to_str,
+	  rtrs_clt_reset_rdma_stats);
+
+STAT_ATTR(struct rtrs_clt_stats, reset_all,
+	  rtrs_clt_reset_all_help,
+	  rtrs_clt_reset_all_stats);
+
+static struct attribute *rtrs_clt_stats_attrs[] = {
+	&cpu_migration_attr.attr,
+	&reconnects_attr.attr,
+	&rdma_attr.attr,
+	&reset_all_attr.attr,
+	NULL
+};
+
+static struct attribute_group rtrs_clt_stats_attr_group = {
+	.attrs = rtrs_clt_stats_attrs,
+};
+
+static ssize_t rtrs_clt_hca_port_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_clt_sess *sess;
+
+	sess = container_of(kobj, typeof(*sess), kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%u\n", sess->hca_port);
+}
+
+static struct kobj_attribute rtrs_clt_hca_port_attr =
+	__ATTR(hca_port, 0444, rtrs_clt_hca_port_show, NULL);
+
+static ssize_t rtrs_clt_hca_name_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_clt_sess *sess;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%s\n", sess->hca_name);
+}
+
+static struct kobj_attribute rtrs_clt_hca_name_attr =
+	__ATTR(hca_name, 0444, rtrs_clt_hca_name_show, NULL);
+
+static ssize_t rtrs_clt_src_addr_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_clt_sess *sess;
+	int cnt;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+	cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
+			      page, PAGE_SIZE);
+	return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
+}
+
+static struct kobj_attribute rtrs_clt_src_addr_attr =
+	__ATTR(src_addr, 0444, rtrs_clt_src_addr_show, NULL);
+
+static ssize_t rtrs_clt_dst_addr_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_clt_sess *sess;
+	int cnt;
+
+	sess = container_of(kobj, struct rtrs_clt_sess, kobj);
+	cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
+			      page, PAGE_SIZE);
+	return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
+}
+
+static struct kobj_attribute rtrs_clt_dst_addr_attr =
+	__ATTR(dst_addr, 0444, rtrs_clt_dst_addr_show, NULL);
+
+static struct attribute *rtrs_clt_sess_attrs[] = {
+	&rtrs_clt_hca_name_attr.attr,
+	&rtrs_clt_hca_port_attr.attr,
+	&rtrs_clt_src_addr_attr.attr,
+	&rtrs_clt_dst_addr_attr.attr,
+	&rtrs_clt_state_attr.attr,
+	&rtrs_clt_reconnect_attr.attr,
+	&rtrs_clt_disconnect_attr.attr,
+	&rtrs_clt_remove_path_attr.attr,
+	NULL,
+};
+
+static struct attribute_group rtrs_clt_sess_attr_group = {
+	.attrs = rtrs_clt_sess_attrs,
+};
+
+int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt *clt = sess->clt;
+	char str[NAME_MAX];
+	int err, cnt;
+
+	cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
+			      str, sizeof(str));
+	cnt += scnprintf(str + cnt, sizeof(str) - cnt, "@");
+	sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
+			str + cnt, sizeof(str) - cnt);
+
+	err = kobject_init_and_add(&sess->kobj, &ktype_sess, clt->kobj_paths,
+				   "%s", str);
+	if (err) {
+		pr_err("kobject_init_and_add: %d\n", err);
+		return err;
+	}
+	err = sysfs_create_group(&sess->kobj, &rtrs_clt_sess_attr_group);
+	if (err) {
+		pr_err("sysfs_create_group(): %d\n", err);
+		goto put_kobj;
+	}
+	err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats,
+				   &sess->kobj, "stats");
+	if (err) {
+		pr_err("kobject_init_and_add: %d\n", err);
+		goto remove_group;
+	}
+
+	err = sysfs_create_group(&sess->stats->kobj_stats,
+				 &rtrs_clt_stats_attr_group);
+	if (err) {
+		pr_err("failed to create stats sysfs group, err: %d\n", err);
+		goto put_kobj_stats;
+	}
+
+	return 0;
+
+put_kobj_stats:
+	kobject_del(&sess->stats->kobj_stats);
+	kobject_put(&sess->stats->kobj_stats);
+remove_group:
+	sysfs_remove_group(&sess->kobj, &rtrs_clt_sess_attr_group);
+put_kobj:
+	kobject_del(&sess->kobj);
+	kobject_put(&sess->kobj);
+
+	return err;
+}
+
+void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,
+				  const struct attribute *sysfs_self)
+{
+	kobject_del(&sess->stats->kobj_stats);
+	kobject_put(&sess->stats->kobj_stats);
+	if (sysfs_self)
+		sysfs_remove_file_self(&sess->kobj, sysfs_self);
+	kobject_del(&sess->kobj);
+}
+
+static struct attribute *rtrs_clt_attrs[] = {
+	&dev_attr_max_reconnect_attempts.attr,
+	&dev_attr_mpath_policy.attr,
+	&dev_attr_add_path.attr,
+	NULL,
+};
+
+static struct attribute_group rtrs_clt_attr_group = {
+	.attrs = rtrs_clt_attrs,
+};
+
+int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt)
+{
+	return sysfs_create_group(&clt->dev.kobj, &rtrs_clt_attr_group);
+}
+
+void rtrs_clt_destroy_sysfs_root_folders(struct rtrs_clt *clt)
+{
+	if (clt->kobj_paths) {
+		kobject_del(clt->kobj_paths);
+		kobject_put(clt->kobj_paths);
+	}
+}
+
+void rtrs_clt_destroy_sysfs_root_files(struct rtrs_clt *clt)
+{
+	sysfs_remove_group(&clt->dev.kobj, &rtrs_clt_attr_group);
+}
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
new file mode 100644
index 000000000000..564388a85603
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -0,0 +1,2992 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include <linux/module.h>
+#include <linux/rculist.h>
+
+#include "rtrs-clt.h"
+#include "rtrs-log.h"
+
+#define RTRS_CONNECT_TIMEOUT_MS 30000
+/*
+ * Wait a bit before trying to reconnect after a failure
+ * in order to give server time to finish clean up which
+ * leads to "false positives" failed reconnect attempts
+ */
+#define RTRS_RECONNECT_BACKOFF 1000
+
+MODULE_DESCRIPTION("RDMA Transport Client");
+MODULE_LICENSE("GPL");
+
+static const struct rtrs_rdma_dev_pd_ops dev_pd_ops;
+static struct rtrs_rdma_dev_pd dev_pd = {
+	.ops = &dev_pd_ops
+};
+
+static struct workqueue_struct *rtrs_wq;
+static struct class *rtrs_clt_dev_class;
+
+static inline bool rtrs_clt_is_connected(const struct rtrs_clt *clt)
+{
+	struct rtrs_clt_sess *sess;
+	bool connected = false;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(sess, &clt->paths_list, s.entry)
+		connected |= READ_ONCE(sess->state) == RTRS_CLT_CONNECTED;
+	rcu_read_unlock();
+
+	return connected;
+}
+
+static struct rtrs_permit *
+__rtrs_get_permit(struct rtrs_clt *clt, enum rtrs_clt_con_type con_type)
+{
+	size_t max_depth = clt->queue_depth;
+	struct rtrs_permit *permit;
+	int bit;
+
+	/*
+	 * Adapted from null_blk get_tag(). Callers from different cpus may
+	 * grab the same bit, since find_first_zero_bit is not atomic.
+	 * But then the test_and_set_bit_lock will fail for all the
+	 * callers but one, so that they will loop again.
+	 * This way an explicit spinlock is not required.
+	 */
+	do {
+		bit = find_first_zero_bit(clt->permits_map, max_depth);
+		if (unlikely(bit >= max_depth))
+			return NULL;
+	} while (unlikely(test_and_set_bit_lock(bit, clt->permits_map)));
+
+	permit = get_permit(clt, bit);
+	WARN_ON(permit->mem_id != bit);
+	permit->cpu_id = raw_smp_processor_id();
+	permit->con_type = con_type;
+
+	return permit;
+}
+
+static inline void __rtrs_put_permit(struct rtrs_clt *clt,
+				      struct rtrs_permit *permit)
+{
+	clear_bit_unlock(permit->mem_id, clt->permits_map);
+}
+
+/**
+ * rtrs_clt_get_permit() - allocates permit for future RDMA operation
+ * @clt:	Current session
+ * @con_type:	Type of connection to use with the permit
+ * @can_wait:	Wait type
+ *
+ * Description:
+ *    Allocates permit for the following RDMA operation.  Permit is used
+ *    to preallocate all resources and to propagate memory pressure
+ *    up earlier.
+ *
+ * Context:
+ *    Can sleep if @wait == RTRS_TAG_WAIT
+ */
+struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *clt,
+					  enum rtrs_clt_con_type con_type,
+					  int can_wait)
+{
+	struct rtrs_permit *permit;
+	DEFINE_WAIT(wait);
+
+	permit = __rtrs_get_permit(clt, con_type);
+	if (likely(permit) || !can_wait)
+		return permit;
+
+	do {
+		prepare_to_wait(&clt->permits_wait, &wait,
+				TASK_UNINTERRUPTIBLE);
+		permit = __rtrs_get_permit(clt, con_type);
+		if (likely(permit))
+			break;
+
+		io_schedule();
+	} while (1);
+
+	finish_wait(&clt->permits_wait, &wait);
+
+	return permit;
+}
+EXPORT_SYMBOL(rtrs_clt_get_permit);
+
+/**
+ * rtrs_clt_put_permit() - puts allocated permit
+ * @clt:	Current session
+ * @permit:	Permit to be freed
+ *
+ * Context:
+ *    Does not matter
+ */
+void rtrs_clt_put_permit(struct rtrs_clt *clt, struct rtrs_permit *permit)
+{
+	if (WARN_ON(!test_bit(permit->mem_id, clt->permits_map)))
+		return;
+
+	__rtrs_put_permit(clt, permit);
+
+	/*
+	 * rtrs_clt_get_permit() adds itself to the &clt->permits_wait list
+	 * before calling schedule(). So if rtrs_clt_get_permit() is sleeping
+	 * it must have added itself to &clt->permits_wait before
+	 * __rtrs_put_permit() finished.
+	 * Hence it is safe to guard wake_up() with a waitqueue_active() test.
+	 */
+	if (waitqueue_active(&clt->permits_wait))
+		wake_up(&clt->permits_wait);
+}
+EXPORT_SYMBOL(rtrs_clt_put_permit);
+
+void *rtrs_permit_to_pdu(struct rtrs_permit *permit)
+{
+	return permit + 1;
+}
+EXPORT_SYMBOL(rtrs_permit_to_pdu);
+
+/**
+ * rtrs_permit_to_clt_con() - returns RDMA connection pointer by the permit
+ * @sess: client session pointer
+ * @permit: permit for the allocation of the RDMA buffer
+ * Note:
+ *     IO connection starts from 1.
+ *     0 connection is for user messages.
+ */
+static
+struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_sess *sess,
+					    struct rtrs_permit *permit)
+{
+	int id = 0;
+
+	if (likely(permit->con_type == RTRS_IO_CON))
+		id = (permit->cpu_id % (sess->s.con_num - 1)) + 1;
+
+	return to_clt_con(sess->s.con[id]);
+}
+
+/**
+ * __rtrs_clt_change_state() - change the session state through session state
+ * machine.
+ *
+ * @sess: client session to change the state of.
+ * @new_state: state to change to.
+ *
+ * returns true if successful, false if the requested state can not be set.
+ *
+ * Locks:
+ * state_wq lock must be hold.
+ */
+static bool __rtrs_clt_change_state(struct rtrs_clt_sess *sess,
+				     enum rtrs_clt_state new_state)
+{
+	enum rtrs_clt_state old_state;
+	bool changed = false;
+
+	lockdep_assert_held(&sess->state_wq.lock);
+
+	old_state = sess->state;
+	switch (new_state) {
+	case RTRS_CLT_CONNECTING:
+		switch (old_state) {
+		case RTRS_CLT_RECONNECTING:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_CLT_RECONNECTING:
+		switch (old_state) {
+		case RTRS_CLT_CONNECTED:
+		case RTRS_CLT_CONNECTING_ERR:
+		case RTRS_CLT_CLOSED:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_CLT_CONNECTED:
+		switch (old_state) {
+		case RTRS_CLT_CONNECTING:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_CLT_CONNECTING_ERR:
+		switch (old_state) {
+		case RTRS_CLT_CONNECTING:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_CLT_CLOSING:
+		switch (old_state) {
+		case RTRS_CLT_CONNECTING:
+		case RTRS_CLT_CONNECTING_ERR:
+		case RTRS_CLT_RECONNECTING:
+		case RTRS_CLT_CONNECTED:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_CLT_CLOSED:
+		switch (old_state) {
+		case RTRS_CLT_CLOSING:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_CLT_DEAD:
+		switch (old_state) {
+		case RTRS_CLT_CLOSED:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+	if (changed) {
+		sess->state = new_state;
+		wake_up_locked(&sess->state_wq);
+	}
+
+	return changed;
+}
+
+static bool rtrs_clt_change_state_from_to(struct rtrs_clt_sess *sess,
+					   enum rtrs_clt_state old_state,
+					   enum rtrs_clt_state new_state)
+{
+	bool changed = false;
+
+	spin_lock_irq(&sess->state_wq.lock);
+	if (sess->state == old_state)
+		changed = __rtrs_clt_change_state(sess, new_state);
+	spin_unlock_irq(&sess->state_wq.lock);
+
+	return changed;
+}
+
+static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+
+	if (rtrs_clt_change_state_from_to(sess,
+					   RTRS_CLT_CONNECTED,
+					   RTRS_CLT_RECONNECTING)) {
+		struct rtrs_clt *clt = sess->clt;
+		unsigned int delay_ms;
+
+		/*
+		 * Normal scenario, reconnect if we were successfully connected
+		 */
+		delay_ms = clt->reconnect_delay_sec * 1000;
+		queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,
+				   msecs_to_jiffies(delay_ms));
+	} else {
+		/*
+		 * Error can happen just on establishing new connection,
+		 * so notify waiter with error state, waiter is responsible
+		 * for cleaning the rest and reconnect if needed.
+		 */
+		rtrs_clt_change_state_from_to(sess,
+					       RTRS_CLT_CONNECTING,
+					       RTRS_CLT_CONNECTING_ERR);
+	}
+}
+
+static void rtrs_clt_fast_reg_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_clt_con *con = cq->cq_context;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(con->c.sess, "Failed IB_WR_REG_MR: %s\n",
+			  ib_wc_status_msg(wc->status));
+		rtrs_rdma_error_recovery(con);
+	}
+}
+
+static struct ib_cqe fast_reg_cqe = {
+	.done = rtrs_clt_fast_reg_done
+};
+
+static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno,
+			      bool notify, bool can_wait);
+
+static void rtrs_clt_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_clt_io_req *req =
+		container_of(wc->wr_cqe, typeof(*req), inv_cqe);
+	struct rtrs_clt_con *con = cq->cq_context;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(con->c.sess, "Failed IB_WR_LOCAL_INV: %s\n",
+			  ib_wc_status_msg(wc->status));
+		rtrs_rdma_error_recovery(con);
+	}
+	req->need_inv = false;
+	if (likely(req->need_inv_comp))
+		complete(&req->inv_comp);
+	else
+		/* Complete request from INV callback */
+		complete_rdma_req(req, req->inv_errno, true, false);
+}
+
+static int rtrs_inv_rkey(struct rtrs_clt_io_req *req)
+{
+	struct rtrs_clt_con *con = req->con;
+	struct ib_send_wr wr = {
+		.opcode		    = IB_WR_LOCAL_INV,
+		.wr_cqe		    = &req->inv_cqe,
+		.send_flags	    = IB_SEND_SIGNALED,
+		.ex.invalidate_rkey = req->mr->rkey,
+	};
+	req->inv_cqe.done = rtrs_clt_inv_rkey_done;
+
+	return ib_post_send(con->c.qp, &wr, NULL);
+}
+
+static void complete_rdma_req(struct rtrs_clt_io_req *req, int errno,
+			      bool notify, bool can_wait)
+{
+	struct rtrs_clt_con *con = req->con;
+	struct rtrs_clt_sess *sess;
+	int err;
+
+	if (WARN_ON(!req->in_use))
+		return;
+	if (WARN_ON(!req->con))
+		return;
+	sess = to_clt_sess(con->c.sess);
+
+	if (req->sg_cnt) {
+		if (unlikely(req->dir == DMA_FROM_DEVICE && req->need_inv)) {
+			/*
+			 * We are here to invalidate read requests
+			 * ourselves.  In normal scenario server should
+			 * send INV for all read requests, but
+			 * we are here, thus two things could happen:
+			 *
+			 *    1.  this is failover, when errno != 0
+			 *        and can_wait == 1,
+			 *
+			 *    2.  something totally bad happened and
+			 *        server forgot to send INV, so we
+			 *        should do that ourselves.
+			 */
+
+			if (likely(can_wait)) {
+				req->need_inv_comp = true;
+			} else {
+				/* This should be IO path, so always notify */
+				WARN_ON(!notify);
+				/* Save errno for INV callback */
+				req->inv_errno = errno;
+			}
+
+			err = rtrs_inv_rkey(req);
+			if (unlikely(err)) {
+				rtrs_err(con->c.sess, "Send INV WR key=%#x: %d\n",
+					  req->mr->rkey, err);
+			} else if (likely(can_wait)) {
+				wait_for_completion(&req->inv_comp);
+			} else {
+				/*
+				 * Something went wrong, so request will be
+				 * completed from INV callback.
+				 */
+				WARN_ON_ONCE(1);
+
+				return;
+			}
+		}
+		ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist,
+				req->sg_cnt, req->dir);
+	}
+	if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT)
+		atomic_dec(&sess->stats->inflight);
+
+	req->in_use = false;
+	req->con = NULL;
+
+	if (notify)
+		req->conf(req->priv, errno);
+}
+
+static int rtrs_post_send_rdma(struct rtrs_clt_con *con,
+				struct rtrs_clt_io_req *req,
+				struct rtrs_rbuf *rbuf, u32 off,
+				u32 imm, struct ib_send_wr *wr)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	enum ib_send_flags flags;
+	struct ib_sge sge;
+
+	if (unlikely(!req->sg_size)) {
+		rtrs_wrn(con->c.sess,
+			 "Doing RDMA Write failed, no data supplied\n");
+		return -EINVAL;
+	}
+
+	/* user data and user message in the first list element */
+	sge.addr   = req->iu->dma_addr;
+	sge.length = req->sg_size;
+	sge.lkey   = sess->s.dev->ib_pd->local_dma_lkey;
+
+	/*
+	 * From time to time we have to post signalled sends,
+	 * or send queue will fill up and only QP reset can help.
+	 */
+	flags = atomic_inc_return(&con->io_cnt) % sess->queue_depth ?
+			0 : IB_SEND_SIGNALED;
+
+	ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr,
+				      req->sg_size, DMA_TO_DEVICE);
+
+	return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1,
+					    rbuf->rkey, rbuf->addr + off,
+					    imm, flags, wr);
+}
+
+static void process_io_rsp(struct rtrs_clt_sess *sess, u32 msg_id,
+			   s16 errno, bool w_inval)
+{
+	struct rtrs_clt_io_req *req;
+
+	if (WARN_ON(msg_id >= sess->queue_depth))
+		return;
+
+	req = &sess->reqs[msg_id];
+	/* Drop need_inv if server responded with send with invalidation */
+	req->need_inv &= !w_inval;
+	complete_rdma_req(req, errno, true, false);
+}
+
+static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc)
+{
+	struct rtrs_iu *iu;
+	int err;
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+
+	WARN_ON(sess->flags != RTRS_MSG_NEW_RKEY_F);
+	iu = container_of(wc->wr_cqe, struct rtrs_iu,
+			  cqe);
+	err = rtrs_iu_post_recv(&con->c, iu);
+	if (unlikely(err)) {
+		rtrs_err(con->c.sess, "post iu failed %d\n", err);
+		rtrs_rdma_error_recovery(con);
+	}
+}
+
+static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct rtrs_msg_rkey_rsp *msg;
+	u32 imm_type, imm_payload;
+	bool w_inval = false;
+	struct rtrs_iu *iu;
+	u32 buf_id;
+	int err;
+
+	WARN_ON(sess->flags != RTRS_MSG_NEW_RKEY_F);
+
+	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
+
+	if (unlikely(wc->byte_len < sizeof(*msg))) {
+		rtrs_err(con->c.sess, "rkey response is malformed: size %d\n",
+			  wc->byte_len);
+		goto out;
+	}
+	ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr,
+				   iu->size, DMA_FROM_DEVICE);
+	msg = iu->buf;
+	if (unlikely(le16_to_cpu(msg->type) != RTRS_MSG_RKEY_RSP)) {
+		rtrs_err(sess->clt, "rkey response is malformed: type %d\n",
+			  le16_to_cpu(msg->type));
+		goto out;
+	}
+	buf_id = le16_to_cpu(msg->buf_id);
+	if (WARN_ON(buf_id >= sess->queue_depth))
+		goto out;
+
+	rtrs_from_imm(be32_to_cpu(wc->ex.imm_data), &imm_type, &imm_payload);
+	if (likely(imm_type == RTRS_IO_RSP_IMM ||
+		   imm_type == RTRS_IO_RSP_W_INV_IMM)) {
+		u32 msg_id;
+
+		w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM);
+		rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err);
+
+		if (WARN_ON(buf_id != msg_id))
+			goto out;
+		sess->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey);
+		process_io_rsp(sess, msg_id, err, w_inval);
+	}
+	ib_dma_sync_single_for_device(sess->s.dev->ib_dev, iu->dma_addr,
+				      iu->size, DMA_FROM_DEVICE);
+	return rtrs_clt_recv_done(con, wc);
+out:
+	rtrs_rdma_error_recovery(con);
+}
+
+static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc);
+
+static struct ib_cqe io_comp_cqe = {
+	.done = rtrs_clt_rdma_done
+};
+
+/*
+ * Post x2 empty WRs: first is for this RDMA with IMM,
+ * second is for RECV with INV, which happened earlier.
+ */
+static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe)
+{
+	struct ib_recv_wr wr_arr[2], *wr;
+	int i;
+
+	memset(wr_arr, 0, sizeof(wr_arr));
+	for (i = 0; i < ARRAY_SIZE(wr_arr); i++) {
+		wr = &wr_arr[i];
+		wr->wr_cqe  = cqe;
+		if (i)
+			/* Chain backwards */
+			wr->next = &wr_arr[i - 1];
+	}
+
+	return ib_post_recv(con->qp, wr, NULL);
+}
+
+static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_clt_con *con = cq->cq_context;
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	u32 imm_type, imm_payload;
+	bool w_inval = false;
+	int err;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR) {
+			rtrs_err(sess->clt, "RDMA failed: %s\n",
+				  ib_wc_status_msg(wc->status));
+			rtrs_rdma_error_recovery(con);
+		}
+		return;
+	}
+	rtrs_clt_update_wc_stats(con);
+
+	switch (wc->opcode) {
+	case IB_WC_RECV_RDMA_WITH_IMM:
+		/*
+		 * post_recv() RDMA write completions of IO reqs (read/write)
+		 * and hb
+		 */
+		if (WARN_ON(wc->wr_cqe->done != rtrs_clt_rdma_done))
+			return;
+		rtrs_from_imm(be32_to_cpu(wc->ex.imm_data),
+			       &imm_type, &imm_payload);
+		if (likely(imm_type == RTRS_IO_RSP_IMM ||
+			   imm_type == RTRS_IO_RSP_W_INV_IMM)) {
+			u32 msg_id;
+
+			w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM);
+			rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err);
+
+			process_io_rsp(sess, msg_id, err, w_inval);
+		} else if (imm_type == RTRS_HB_MSG_IMM) {
+			WARN_ON(con->c.cid);
+			rtrs_send_hb_ack(&sess->s);
+			if (sess->flags == RTRS_MSG_NEW_RKEY_F)
+				return  rtrs_clt_recv_done(con, wc);
+		} else if (imm_type == RTRS_HB_ACK_IMM) {
+			WARN_ON(con->c.cid);
+			sess->s.hb_missed_cnt = 0;
+			if (sess->flags == RTRS_MSG_NEW_RKEY_F)
+				return  rtrs_clt_recv_done(con, wc);
+		} else {
+			rtrs_wrn(con->c.sess, "Unknown IMM type %u\n",
+				  imm_type);
+		}
+		if (w_inval)
+			/*
+			 * Post x2 empty WRs: first is for this RDMA with IMM,
+			 * second is for RECV with INV, which happened earlier.
+			 */
+			err = rtrs_post_recv_empty_x2(&con->c, &io_comp_cqe);
+		else
+			err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
+		if (unlikely(err)) {
+			rtrs_err(con->c.sess, "rtrs_post_recv_empty(): %d\n",
+				  err);
+			rtrs_rdma_error_recovery(con);
+			break;
+		}
+		break;
+	case IB_WC_RECV:
+		/*
+		 * Key invalidations from server side
+		 */
+		WARN_ON(!(wc->wc_flags & IB_WC_WITH_INVALIDATE ||
+			  wc->wc_flags & IB_WC_WITH_IMM));
+		WARN_ON(wc->wr_cqe->done != rtrs_clt_rdma_done);
+		if (sess->flags == RTRS_MSG_NEW_RKEY_F) {
+			if (wc->wc_flags & IB_WC_WITH_INVALIDATE)
+				return  rtrs_clt_recv_done(con, wc);
+
+			return  rtrs_clt_rkey_rsp_done(con, wc);
+		}
+		break;
+	case IB_WC_RDMA_WRITE:
+		/*
+		 * post_send() RDMA write completions of IO reqs (read/write)
+		 * and hb
+		 */
+		break;
+
+	default:
+		rtrs_wrn(sess->clt, "Unexpected WC type: %d\n", wc->opcode);
+		return;
+	}
+}
+
+static int post_recv_io(struct rtrs_clt_con *con, size_t q_size)
+{
+	int err, i;
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+
+	for (i = 0; i < q_size; i++) {
+		if (sess->flags == RTRS_MSG_NEW_RKEY_F) {
+			struct rtrs_iu *iu = &con->rsp_ius[i];
+
+			err = rtrs_iu_post_recv(&con->c, iu);
+		} else {
+			err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
+		}
+		if (unlikely(err))
+			return err;
+	}
+
+	return 0;
+}
+
+static int post_recv_sess(struct rtrs_clt_sess *sess)
+{
+	size_t q_size = 0;
+	int err, cid;
+
+	for (cid = 0; cid < sess->s.con_num; cid++) {
+		if (cid == 0)
+			q_size = SERVICE_CON_QUEUE_DEPTH;
+		else
+			q_size = sess->queue_depth;
+
+		/*
+		 * x2 for RDMA read responses + FR key invalidations,
+		 * RDMA writes do not require any FR registrations.
+		 */
+		q_size *= 2;
+
+		err = post_recv_io(to_clt_con(sess->s.con[cid]), q_size);
+		if (unlikely(err)) {
+			rtrs_err(sess->clt, "post_recv_io(), err: %d\n", err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+struct path_it {
+	int i;
+	struct list_head skip_list;
+	struct rtrs_clt *clt;
+	struct rtrs_clt_sess *(*next_path)(struct path_it *it);
+};
+
+/**
+ * list_next_or_null_rr_rcu - get next list element in round-robin fashion.
+ * @head:	the head for the list.
+ * @ptr:        the list head to take the next element from.
+ * @type:       the type of the struct this is embedded in.
+ * @memb:       the name of the list_head within the struct.
+ *
+ * Next element returned in round-robin fashion, i.e. head will be skipped,
+ * but if list is observed as empty, NULL will be returned.
+ *
+ * This primitive may safely run concurrently with the _rcu list-mutation
+ * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
+ */
+#define list_next_or_null_rr_rcu(head, ptr, type, memb) \
+({ \
+	list_next_or_null_rcu(head, ptr, type, memb) ?: \
+		list_next_or_null_rcu(head, READ_ONCE((ptr)->next), \
+				      type, memb); \
+})
+
+/**
+ * get_next_path_rr() - Returns path in round-robin fashion.
+ * @it:	the path pointer
+ *
+ * Related to @MP_POLICY_RR
+ *
+ * Locks:
+ *    rcu_read_lock() must be hold.
+ */
+static struct rtrs_clt_sess *get_next_path_rr(struct path_it *it)
+{
+	struct rtrs_clt_sess __rcu **ppcpu_path;
+	struct rtrs_clt_sess *path;
+	struct rtrs_clt *clt;
+
+	clt = it->clt;
+
+	/*
+	 * Here we use two RCU objects: @paths_list and @pcpu_path
+	 * pointer.  See rtrs_clt_remove_path_from_arr() for details
+	 * how that is handled.
+	 */
+
+	ppcpu_path = this_cpu_ptr(clt->pcpu_path);
+	path = rcu_dereference(*ppcpu_path);
+	if (unlikely(!path))
+		path = list_first_or_null_rcu(&clt->paths_list,
+					      typeof(*path), s.entry);
+	else
+		path = list_next_or_null_rr_rcu(&clt->paths_list,
+						&path->s.entry,
+						typeof(*path),
+						s.entry);
+	rcu_assign_pointer(*ppcpu_path, path);
+
+	return path;
+}
+
+/**
+ * get_next_path_min_inflight() - Returns path with minimal inflight count.
+ * @it:	the path pointer
+ *
+ * Related to @MP_POLICY_MIN_INFLIGHT
+ *
+ * Locks:
+ *    rcu_read_lock() must be hold.
+ */
+static struct rtrs_clt_sess *get_next_path_min_inflight(struct path_it *it)
+{
+	struct rtrs_clt_sess *min_path = NULL;
+	struct rtrs_clt *clt = it->clt;
+	struct rtrs_clt_sess *sess;
+	int min_inflight = INT_MAX;
+	int inflight;
+
+	list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) {
+		if (unlikely(!list_empty(raw_cpu_ptr(sess->mp_skip_entry))))
+			continue;
+
+		inflight = atomic_read(&sess->stats->inflight);
+
+		if (inflight < min_inflight) {
+			min_inflight = inflight;
+			min_path = sess;
+		}
+	}
+
+	/*
+	 * add the path to the skip list, so that next time we can get
+	 * a different one
+	 */
+	if (min_path)
+		list_add(raw_cpu_ptr(min_path->mp_skip_entry), &it->skip_list);
+
+	return min_path;
+}
+
+static inline void path_it_init(struct path_it *it, struct rtrs_clt *clt)
+{
+	INIT_LIST_HEAD(&it->skip_list);
+	it->clt = clt;
+	it->i = 0;
+
+	if (clt->mp_policy == MP_POLICY_RR)
+		it->next_path = get_next_path_rr;
+	else
+		it->next_path = get_next_path_min_inflight;
+}
+
+static inline void path_it_deinit(struct path_it *it)
+{
+	struct list_head *skip, *tmp;
+	/*
+	 * The skip_list is used only for the MIN_INFLIGHT policy.
+	 * We need to remove paths from it, so that next IO can insert
+	 * paths (->mp_skip_entry) into a skip_list again.
+	 */
+	list_for_each_safe(skip, tmp, &it->skip_list)
+		list_del_init(skip);
+}
+
+/**
+ * rtrs_clt_init_req() Initialize an rtrs_clt_io_req holding information
+ * about an inflight IO.
+ * The user buffer holding user control message (not data) is copied into
+ * the corresponding buffer of rtrs_iu (req->iu->buf), which later on will
+ * also hold the control message of rtrs.
+ * @req: an io request holding information about IO.
+ * @sess: client session
+ * @conf: conformation callback function to notify upper layer.
+ * @permit: permit for allocation of RDMA remote buffer
+ * @priv: private pointer
+ * @vec: kernel vector containing control message
+ * @usr_len: length of the user message
+ * @sg: scater list for IO data
+ * @sg_cnt: number of scater list entries
+ * @data_len: length of the IO data
+ * @dir: direction of the IO.
+ */
+static void rtrs_clt_init_req(struct rtrs_clt_io_req *req,
+			      struct rtrs_clt_sess *sess,
+			      void (*conf)(void *priv, int errno),
+			      struct rtrs_permit *permit, void *priv,
+			      const struct kvec *vec, size_t usr_len,
+			      struct scatterlist *sg, size_t sg_cnt,
+			      size_t data_len, int dir)
+{
+	struct iov_iter iter;
+	size_t len;
+
+	req->permit = permit;
+	req->in_use = true;
+	req->usr_len = usr_len;
+	req->data_len = data_len;
+	req->sglist = sg;
+	req->sg_cnt = sg_cnt;
+	req->priv = priv;
+	req->dir = dir;
+	req->con = rtrs_permit_to_clt_con(sess, permit);
+	req->conf = conf;
+	req->need_inv = false;
+	req->need_inv_comp = false;
+	req->inv_errno = 0;
+
+	iov_iter_kvec(&iter, READ, vec, 1, usr_len);
+	len = _copy_from_iter(req->iu->buf, usr_len, &iter);
+	WARN_ON(len != usr_len);
+
+	reinit_completion(&req->inv_comp);
+}
+
+static struct rtrs_clt_io_req *
+rtrs_clt_get_req(struct rtrs_clt_sess *sess,
+		 void (*conf)(void *priv, int errno),
+		 struct rtrs_permit *permit, void *priv,
+		 const struct kvec *vec, size_t usr_len,
+		 struct scatterlist *sg, size_t sg_cnt,
+		 size_t data_len, int dir)
+{
+	struct rtrs_clt_io_req *req;
+
+	req = &sess->reqs[permit->mem_id];
+	rtrs_clt_init_req(req, sess, conf, permit, priv, vec, usr_len,
+			   sg, sg_cnt, data_len, dir);
+	return req;
+}
+
+static struct rtrs_clt_io_req *
+rtrs_clt_get_copy_req(struct rtrs_clt_sess *alive_sess,
+		       struct rtrs_clt_io_req *fail_req)
+{
+	struct rtrs_clt_io_req *req;
+	struct kvec vec = {
+		.iov_base = fail_req->iu->buf,
+		.iov_len  = fail_req->usr_len
+	};
+
+	req = &alive_sess->reqs[fail_req->permit->mem_id];
+	rtrs_clt_init_req(req, alive_sess, fail_req->conf, fail_req->permit,
+			   fail_req->priv, &vec, fail_req->usr_len,
+			   fail_req->sglist, fail_req->sg_cnt,
+			   fail_req->data_len, fail_req->dir);
+	return req;
+}
+
+static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con,
+				    struct rtrs_clt_io_req *req,
+				    struct rtrs_rbuf *rbuf,
+				    u32 size, u32 imm)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct ib_sge *sge = req->sge;
+	enum ib_send_flags flags;
+	struct scatterlist *sg;
+	size_t num_sge;
+	int i;
+
+	for_each_sg(req->sglist, sg, req->sg_cnt, i) {
+		sge[i].addr   = sg_dma_address(sg);
+		sge[i].length = sg_dma_len(sg);
+		sge[i].lkey   = sess->s.dev->ib_pd->local_dma_lkey;
+	}
+	sge[i].addr   = req->iu->dma_addr;
+	sge[i].length = size;
+	sge[i].lkey   = sess->s.dev->ib_pd->local_dma_lkey;
+
+	num_sge = 1 + req->sg_cnt;
+
+	/*
+	 * From time to time we have to post signalled sends,
+	 * or send queue will fill up and only QP reset can help.
+	 */
+	flags = atomic_inc_return(&con->io_cnt) % sess->queue_depth ?
+			0 : IB_SEND_SIGNALED;
+
+	ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr,
+				      size, DMA_TO_DEVICE);
+
+	return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, sge, num_sge,
+					    rbuf->rkey, rbuf->addr, imm,
+					    flags, NULL);
+}
+
+static int rtrs_clt_write_req(struct rtrs_clt_io_req *req)
+{
+	struct rtrs_clt_con *con = req->con;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_clt_sess *sess = to_clt_sess(s);
+	struct rtrs_msg_rdma_write *msg;
+
+	struct rtrs_rbuf *rbuf;
+	int ret, count = 0;
+	u32 imm, buf_id;
+
+	const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;
+
+	if (unlikely(tsize > sess->chunk_size)) {
+		rtrs_wrn(s, "Write request failed, size too big %zu > %d\n",
+			  tsize, sess->chunk_size);
+		return -EMSGSIZE;
+	}
+	if (req->sg_cnt) {
+		count = ib_dma_map_sg(sess->s.dev->ib_dev, req->sglist,
+				      req->sg_cnt, req->dir);
+		if (unlikely(!count)) {
+			rtrs_wrn(s, "Write request failed, map failed\n");
+			return -EINVAL;
+		}
+	}
+	/* put rtrs msg after sg and user message */
+	msg = req->iu->buf + req->usr_len;
+	msg->type = cpu_to_le16(RTRS_MSG_WRITE);
+	msg->usr_len = cpu_to_le16(req->usr_len);
+
+	/* rtrs message on server side will be after user data and message */
+	imm = req->permit->mem_off + req->data_len + req->usr_len;
+	imm = rtrs_to_io_req_imm(imm);
+	buf_id = req->permit->mem_id;
+	req->sg_size = tsize;
+	rbuf = &sess->rbufs[buf_id];
+
+	/*
+	 * Update stats now, after request is successfully sent it is not
+	 * safe anymore to touch it.
+	 */
+	rtrs_clt_update_all_stats(req, WRITE);
+
+	ret = rtrs_post_rdma_write_sg(req->con, req, rbuf,
+				       req->usr_len + sizeof(*msg),
+				       imm);
+	if (unlikely(ret)) {
+		rtrs_err(s, "Write request failed: %d\n", ret);
+		if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT)
+			atomic_dec(&sess->stats->inflight);
+		if (req->sg_cnt)
+			ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist,
+					req->sg_cnt, req->dir);
+	}
+
+	return ret;
+}
+
+static int rtrs_map_sg_fr(struct rtrs_clt_io_req *req, size_t count)
+{
+	int nr;
+
+	/* Align the MR to a 4K page size to match the block virt boundary */
+	nr = ib_map_mr_sg(req->mr, req->sglist, count, NULL, SZ_4K);
+	if (nr < 0)
+		return nr;
+	if (unlikely(nr < req->sg_cnt))
+		return -EINVAL;
+	ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
+
+	return nr;
+}
+
+static int rtrs_clt_read_req(struct rtrs_clt_io_req *req)
+{
+	struct rtrs_clt_con *con = req->con;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_clt_sess *sess = to_clt_sess(s);
+	struct rtrs_msg_rdma_read *msg;
+	struct rtrs_ib_dev *dev;
+
+	struct ib_reg_wr rwr;
+	struct ib_send_wr *wr = NULL;
+
+	int ret, count = 0;
+	u32 imm, buf_id;
+
+	const size_t tsize = sizeof(*msg) + req->data_len + req->usr_len;
+
+	s = &sess->s;
+	dev = sess->s.dev;
+
+	if (unlikely(tsize > sess->chunk_size)) {
+		rtrs_wrn(s,
+			  "Read request failed, message size is %zu, bigger than CHUNK_SIZE %d\n",
+			  tsize, sess->chunk_size);
+		return -EMSGSIZE;
+	}
+
+	if (req->sg_cnt) {
+		count = ib_dma_map_sg(dev->ib_dev, req->sglist, req->sg_cnt,
+				      req->dir);
+		if (unlikely(!count)) {
+			rtrs_wrn(s,
+				  "Read request failed, dma map failed\n");
+			return -EINVAL;
+		}
+	}
+	/* put our message into req->buf after user message*/
+	msg = req->iu->buf + req->usr_len;
+	msg->type = cpu_to_le16(RTRS_MSG_READ);
+	msg->usr_len = cpu_to_le16(req->usr_len);
+
+	if (count) {
+		ret = rtrs_map_sg_fr(req, count);
+		if (ret < 0) {
+			rtrs_err_rl(s,
+				     "Read request failed, failed to map  fast reg. data, err: %d\n",
+				     ret);
+			ib_dma_unmap_sg(dev->ib_dev, req->sglist, req->sg_cnt,
+					req->dir);
+			return ret;
+		}
+		rwr = (struct ib_reg_wr) {
+			.wr.opcode = IB_WR_REG_MR,
+			.wr.wr_cqe = &fast_reg_cqe,
+			.mr = req->mr,
+			.key = req->mr->rkey,
+			.access = (IB_ACCESS_LOCAL_WRITE |
+				   IB_ACCESS_REMOTE_WRITE),
+		};
+		wr = &rwr.wr;
+
+		msg->sg_cnt = cpu_to_le16(1);
+		msg->flags = cpu_to_le16(RTRS_MSG_NEED_INVAL_F);
+
+		msg->desc[0].addr = cpu_to_le64(req->mr->iova);
+		msg->desc[0].key = cpu_to_le32(req->mr->rkey);
+		msg->desc[0].len = cpu_to_le32(req->mr->length);
+
+		/* Further invalidation is required */
+		req->need_inv = !!RTRS_MSG_NEED_INVAL_F;
+
+	} else {
+		msg->sg_cnt = 0;
+		msg->flags = 0;
+	}
+	/*
+	 * rtrs message will be after the space reserved for disk data and
+	 * user message
+	 */
+	imm = req->permit->mem_off + req->data_len + req->usr_len;
+	imm = rtrs_to_io_req_imm(imm);
+	buf_id = req->permit->mem_id;
+
+	req->sg_size  = sizeof(*msg);
+	req->sg_size += le16_to_cpu(msg->sg_cnt) * sizeof(struct rtrs_sg_desc);
+	req->sg_size += req->usr_len;
+
+	/*
+	 * Update stats now, after request is successfully sent it is not
+	 * safe anymore to touch it.
+	 */
+	rtrs_clt_update_all_stats(req, READ);
+
+	ret = rtrs_post_send_rdma(req->con, req, &sess->rbufs[buf_id],
+				   req->data_len, imm, wr);
+	if (unlikely(ret)) {
+		rtrs_err(s, "Read request failed: %d\n", ret);
+		if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT)
+			atomic_dec(&sess->stats->inflight);
+		req->need_inv = false;
+		if (req->sg_cnt)
+			ib_dma_unmap_sg(dev->ib_dev, req->sglist,
+					req->sg_cnt, req->dir);
+	}
+
+	return ret;
+}
+
+/**
+ * rtrs_clt_failover_req() Try to find an active path for a failed request
+ * @clt: clt context
+ * @fail_req: a failed io request.
+ */
+static int rtrs_clt_failover_req(struct rtrs_clt *clt,
+				 struct rtrs_clt_io_req *fail_req)
+{
+	struct rtrs_clt_sess *alive_sess;
+	struct rtrs_clt_io_req *req;
+	int err = -ECONNABORTED;
+	struct path_it it;
+
+	rcu_read_lock();
+	for (path_it_init(&it, clt);
+	     (alive_sess = it.next_path(&it)) && it.i < it.clt->paths_num;
+	     it.i++) {
+		if (unlikely(READ_ONCE(alive_sess->state) !=
+			     RTRS_CLT_CONNECTED))
+			continue;
+		req = rtrs_clt_get_copy_req(alive_sess, fail_req);
+		if (req->dir == DMA_TO_DEVICE)
+			err = rtrs_clt_write_req(req);
+		else
+			err = rtrs_clt_read_req(req);
+		if (unlikely(err)) {
+			req->in_use = false;
+			continue;
+		}
+		/* Success path */
+		rtrs_clt_inc_failover_cnt(alive_sess->stats);
+		break;
+	}
+	path_it_deinit(&it);
+	rcu_read_unlock();
+
+	return err;
+}
+
+static void fail_all_outstanding_reqs(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt *clt = sess->clt;
+	struct rtrs_clt_io_req *req;
+	int i, err;
+
+	if (!sess->reqs)
+		return;
+	for (i = 0; i < sess->queue_depth; ++i) {
+		req = &sess->reqs[i];
+		if (!req->in_use)
+			continue;
+
+		/*
+		 * Safely (without notification) complete failed request.
+		 * After completion this request is still useble and can
+		 * be failovered to another path.
+		 */
+		complete_rdma_req(req, -ECONNABORTED, false, true);
+
+		err = rtrs_clt_failover_req(clt, req);
+		if (unlikely(err))
+			/* Failover failed, notify anyway */
+			req->conf(req->priv, err);
+	}
+}
+
+static void free_sess_reqs(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt_io_req *req;
+	int i;
+
+	if (!sess->reqs)
+		return;
+	for (i = 0; i < sess->queue_depth; ++i) {
+		req = &sess->reqs[i];
+		if (req->mr)
+			ib_dereg_mr(req->mr);
+		kfree(req->sge);
+		rtrs_iu_free(req->iu, DMA_TO_DEVICE,
+			      sess->s.dev->ib_dev, 1);
+	}
+	kfree(sess->reqs);
+	sess->reqs = NULL;
+}
+
+static int alloc_sess_reqs(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt_io_req *req;
+	struct rtrs_clt *clt = sess->clt;
+	int i, err = -ENOMEM;
+
+	sess->reqs = kcalloc(sess->queue_depth, sizeof(*sess->reqs),
+			     GFP_KERNEL);
+	if (!sess->reqs)
+		return -ENOMEM;
+
+	for (i = 0; i < sess->queue_depth; ++i) {
+		req = &sess->reqs[i];
+		req->iu = rtrs_iu_alloc(1, sess->max_hdr_size, GFP_KERNEL,
+					 sess->s.dev->ib_dev,
+					 DMA_TO_DEVICE,
+					 rtrs_clt_rdma_done);
+		if (!req->iu)
+			goto out;
+
+		req->sge = kmalloc_array(clt->max_segments + 1,
+					 sizeof(*req->sge), GFP_KERNEL);
+		if (!req->sge)
+			goto out;
+
+		req->mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG,
+				      sess->max_pages_per_mr);
+		if (IS_ERR(req->mr)) {
+			err = PTR_ERR(req->mr);
+			req->mr = NULL;
+			pr_err("Failed to alloc sess->max_pages_per_mr %d\n",
+			       sess->max_pages_per_mr);
+			goto out;
+		}
+
+		init_completion(&req->inv_comp);
+	}
+
+	return 0;
+
+out:
+	free_sess_reqs(sess);
+
+	return err;
+}
+
+static int alloc_permits(struct rtrs_clt *clt)
+{
+	unsigned int chunk_bits;
+	int err, i;
+
+	clt->permits_map = kcalloc(BITS_TO_LONGS(clt->queue_depth),
+				   sizeof(long), GFP_KERNEL);
+	if (!clt->permits_map) {
+		err = -ENOMEM;
+		goto out_err;
+	}
+	clt->permits = kcalloc(clt->queue_depth, permit_size(clt), GFP_KERNEL);
+	if (!clt->permits) {
+		err = -ENOMEM;
+		goto err_map;
+	}
+	chunk_bits = ilog2(clt->queue_depth - 1) + 1;
+	for (i = 0; i < clt->queue_depth; i++) {
+		struct rtrs_permit *permit;
+
+		permit = get_permit(clt, i);
+		permit->mem_id = i;
+		permit->mem_off = i << (MAX_IMM_PAYL_BITS - chunk_bits);
+	}
+
+	return 0;
+
+err_map:
+	kfree(clt->permits_map);
+	clt->permits_map = NULL;
+out_err:
+	return err;
+}
+
+static void free_permits(struct rtrs_clt *clt)
+{
+	kfree(clt->permits_map);
+	clt->permits_map = NULL;
+	kfree(clt->permits);
+	clt->permits = NULL;
+}
+
+static void query_fast_reg_mode(struct rtrs_clt_sess *sess)
+{
+	struct ib_device *ib_dev;
+	u64 max_pages_per_mr;
+	int mr_page_shift;
+
+	ib_dev = sess->s.dev->ib_dev;
+
+	/*
+	 * Use the smallest page size supported by the HCA, down to a
+	 * minimum of 4096 bytes. We're unlikely to build large sglists
+	 * out of smaller entries.
+	 */
+	mr_page_shift      = max(12, ffs(ib_dev->attrs.page_size_cap) - 1);
+	max_pages_per_mr   = ib_dev->attrs.max_mr_size;
+	do_div(max_pages_per_mr, (1ull << mr_page_shift));
+	sess->max_pages_per_mr =
+		min3(sess->max_pages_per_mr, (u32)max_pages_per_mr,
+		     ib_dev->attrs.max_fast_reg_page_list_len);
+	sess->max_send_sge = ib_dev->attrs.max_send_sge;
+}
+
+static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess,
+					   enum rtrs_clt_state new_state,
+					   enum rtrs_clt_state *old_state)
+{
+	bool changed;
+
+	spin_lock_irq(&sess->state_wq.lock);
+	*old_state = sess->state;
+	changed = __rtrs_clt_change_state(sess, new_state);
+	spin_unlock_irq(&sess->state_wq.lock);
+
+	return changed;
+}
+
+static bool rtrs_clt_change_state(struct rtrs_clt_sess *sess,
+				   enum rtrs_clt_state new_state)
+{
+	enum rtrs_clt_state old_state;
+
+	return rtrs_clt_change_state_get_old(sess, new_state, &old_state);
+}
+
+static void rtrs_clt_hb_err_handler(struct rtrs_con *c)
+{
+	struct rtrs_clt_con *con = container_of(c, typeof(*con), c);
+
+	rtrs_rdma_error_recovery(con);
+}
+
+static void rtrs_clt_init_hb(struct rtrs_clt_sess *sess)
+{
+	rtrs_init_hb(&sess->s, &io_comp_cqe,
+		      RTRS_HB_INTERVAL_MS,
+		      RTRS_HB_MISSED_MAX,
+		      rtrs_clt_hb_err_handler,
+		      rtrs_wq);
+}
+
+static void rtrs_clt_start_hb(struct rtrs_clt_sess *sess)
+{
+	rtrs_start_hb(&sess->s);
+}
+
+static void rtrs_clt_stop_hb(struct rtrs_clt_sess *sess)
+{
+	rtrs_stop_hb(&sess->s);
+}
+
+static void rtrs_clt_reconnect_work(struct work_struct *work);
+static void rtrs_clt_close_work(struct work_struct *work);
+
+static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt,
+					 const struct rtrs_addr *path,
+					 size_t con_num, u16 max_segments,
+					 size_t max_segment_size)
+{
+	struct rtrs_clt_sess *sess;
+	int err = -ENOMEM;
+	int cpu;
+
+	sess = kzalloc(sizeof(*sess), GFP_KERNEL);
+	if (!sess)
+		goto err;
+
+	/* Extra connection for user messages */
+	con_num += 1;
+
+	sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL);
+	if (!sess->s.con)
+		goto err_free_sess;
+
+	sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL);
+	if (!sess->stats)
+		goto err_free_con;
+
+	mutex_init(&sess->init_mutex);
+	uuid_gen(&sess->s.uuid);
+	memcpy(&sess->s.dst_addr, path->dst,
+	       rdma_addr_size((struct sockaddr *)path->dst));
+
+	/*
+	 * rdma_resolve_addr() passes src_addr to cma_bind_addr, which
+	 * checks the sa_family to be non-zero. If user passed src_addr=NULL
+	 * the sess->src_addr will contain only zeros, which is then fine.
+	 */
+	if (path->src)
+		memcpy(&sess->s.src_addr, path->src,
+		       rdma_addr_size((struct sockaddr *)path->src));
+	strlcpy(sess->s.sessname, clt->sessname, sizeof(sess->s.sessname));
+	sess->s.con_num = con_num;
+	sess->clt = clt;
+	sess->max_pages_per_mr = max_segments * max_segment_size >> 12;
+	init_waitqueue_head(&sess->state_wq);
+	sess->state = RTRS_CLT_CONNECTING;
+	atomic_set(&sess->connected_cnt, 0);
+	INIT_WORK(&sess->close_work, rtrs_clt_close_work);
+	INIT_DELAYED_WORK(&sess->reconnect_dwork, rtrs_clt_reconnect_work);
+	rtrs_clt_init_hb(sess);
+
+	sess->mp_skip_entry = alloc_percpu(typeof(*sess->mp_skip_entry));
+	if (!sess->mp_skip_entry)
+		goto err_free_stats;
+
+	for_each_possible_cpu(cpu)
+		INIT_LIST_HEAD(per_cpu_ptr(sess->mp_skip_entry, cpu));
+
+	err = rtrs_clt_init_stats(sess->stats);
+	if (err)
+		goto err_free_percpu;
+
+	return sess;
+
+err_free_percpu:
+	free_percpu(sess->mp_skip_entry);
+err_free_stats:
+	kfree(sess->stats);
+err_free_con:
+	kfree(sess->s.con);
+err_free_sess:
+	kfree(sess);
+err:
+	return ERR_PTR(err);
+}
+
+void free_sess(struct rtrs_clt_sess *sess)
+{
+	free_percpu(sess->mp_skip_entry);
+	mutex_destroy(&sess->init_mutex);
+	kfree(sess->s.con);
+	kfree(sess->rbufs);
+	kfree(sess);
+}
+
+static int create_con(struct rtrs_clt_sess *sess, unsigned int cid)
+{
+	struct rtrs_clt_con *con;
+
+	con = kzalloc(sizeof(*con), GFP_KERNEL);
+	if (!con)
+		return -ENOMEM;
+
+	/* Map first two connections to the first CPU */
+	con->cpu  = (cid ? cid - 1 : 0) % nr_cpu_ids;
+	con->c.cid = cid;
+	con->c.sess = &sess->s;
+	atomic_set(&con->io_cnt, 0);
+
+	sess->s.con[cid] = &con->c;
+
+	return 0;
+}
+
+static void destroy_con(struct rtrs_clt_con *con)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+
+	sess->s.con[con->c.cid] = NULL;
+	kfree(con);
+}
+
+static int create_con_cq_qp(struct rtrs_clt_con *con)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	u16 wr_queue_size;
+	int err, cq_vector;
+	struct rtrs_msg_rkey_rsp *rsp;
+
+	/*
+	 * This function can fail, but still destroy_con_cq_qp() should
+	 * be called, this is because create_con_cq_qp() is called on cm
+	 * event path, thus caller/waiter never knows: have we failed before
+	 * create_con_cq_qp() or after.  To solve this dilemma without
+	 * creating any additional flags just allow destroy_con_cq_qp() be
+	 * called many times.
+	 */
+
+	if (con->c.cid == 0) {
+		/*
+		 * One completion for each receive and two for each send
+		 * (send request + registration)
+		 * + 2 for drain and heartbeat
+		 * in case qp gets into error state
+		 */
+		wr_queue_size = SERVICE_CON_QUEUE_DEPTH * 3 + 2;
+		/* We must be the first here */
+		if (WARN_ON(sess->s.dev))
+			return -EINVAL;
+
+		/*
+		 * The whole session uses device from user connection.
+		 * Be careful not to close user connection before ib dev
+		 * is gracefully put.
+		 */
+		sess->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device,
+						       &dev_pd);
+		if (!sess->s.dev) {
+			rtrs_wrn(sess->clt,
+				  "rtrs_ib_dev_find_get_or_add(): no memory\n");
+			return -ENOMEM;
+		}
+		sess->s.dev_ref = 1;
+		query_fast_reg_mode(sess);
+	} else {
+		/*
+		 * Here we assume that session members are correctly set.
+		 * This is always true if user connection (cid == 0) is
+		 * established first.
+		 */
+		if (WARN_ON(!sess->s.dev))
+			return -EINVAL;
+		if (WARN_ON(!sess->queue_depth))
+			return -EINVAL;
+
+		/* Shared between connections */
+		sess->s.dev_ref++;
+		wr_queue_size =
+			min_t(int, sess->s.dev->ib_dev->attrs.max_qp_wr,
+			      /* QD * (REQ + RSP + FR REGS or INVS) + drain */
+			      sess->queue_depth * 3 + 1);
+	}
+	/* alloc iu to recv new rkey reply when server reports flags set */
+	if (sess->flags == RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) {
+		con->rsp_ius = rtrs_iu_alloc(wr_queue_size, sizeof(*rsp),
+					      GFP_KERNEL, sess->s.dev->ib_dev,
+					      DMA_FROM_DEVICE,
+					      rtrs_clt_rdma_done);
+		if (!con->rsp_ius)
+			return -ENOMEM;
+		con->queue_size = wr_queue_size;
+	}
+	cq_vector = con->cpu % sess->s.dev->ib_dev->num_comp_vectors;
+	err = rtrs_cq_qp_create(&sess->s, &con->c, sess->max_send_sge,
+				 cq_vector, wr_queue_size, wr_queue_size,
+				 IB_POLL_SOFTIRQ);
+	/*
+	 * In case of error we do not bother to clean previous allocations,
+	 * since destroy_con_cq_qp() must be called.
+	 */
+	return err;
+}
+
+static void destroy_con_cq_qp(struct rtrs_clt_con *con)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+
+	/*
+	 * Be careful here: destroy_con_cq_qp() can be called even
+	 * create_con_cq_qp() failed, see comments there.
+	 */
+
+	rtrs_cq_qp_destroy(&con->c);
+	if (con->rsp_ius) {
+		rtrs_iu_free(con->rsp_ius, DMA_FROM_DEVICE,
+			      sess->s.dev->ib_dev, con->queue_size);
+		con->rsp_ius = NULL;
+		con->queue_size = 0;
+	}
+	if (sess->s.dev_ref && !--sess->s.dev_ref) {
+		rtrs_ib_dev_put(sess->s.dev);
+		sess->s.dev = NULL;
+	}
+}
+
+static void stop_cm(struct rtrs_clt_con *con)
+{
+	rdma_disconnect(con->c.cm_id);
+	if (con->c.qp)
+		ib_drain_qp(con->c.qp);
+}
+
+static void destroy_cm(struct rtrs_clt_con *con)
+{
+	rdma_destroy_id(con->c.cm_id);
+	con->c.cm_id = NULL;
+}
+
+static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con)
+{
+	struct rtrs_sess *s = con->c.sess;
+	int err;
+
+	err = create_con_cq_qp(con);
+	if (err) {
+		rtrs_err(s, "create_con_cq_qp(), err: %d\n", err);
+		return err;
+	}
+	err = rdma_resolve_route(con->c.cm_id, RTRS_CONNECT_TIMEOUT_MS);
+	if (err) {
+		rtrs_err(s, "Resolving route failed, err: %d\n", err);
+		destroy_con_cq_qp(con);
+	}
+
+	return err;
+}
+
+static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct rtrs_clt *clt = sess->clt;
+	struct rtrs_msg_conn_req msg;
+	struct rdma_conn_param param;
+
+	int err;
+
+	param = (struct rdma_conn_param) {
+		.retry_count = 7,
+		.rnr_retry_count = 7,
+		.private_data = &msg,
+		.private_data_len = sizeof(msg),
+	};
+
+	msg = (struct rtrs_msg_conn_req) {
+		.magic = cpu_to_le16(RTRS_MAGIC),
+		.version = cpu_to_le16(RTRS_PROTO_VER),
+		.cid = cpu_to_le16(con->c.cid),
+		.cid_num = cpu_to_le16(sess->s.con_num),
+		.recon_cnt = cpu_to_le16(sess->s.recon_cnt),
+	};
+	uuid_copy(&msg.sess_uuid, &sess->s.uuid);
+	uuid_copy(&msg.paths_uuid, &clt->paths_uuid);
+
+	err = rdma_connect(con->c.cm_id, &param);
+	if (err)
+		rtrs_err(clt, "rdma_connect(): %d\n", err);
+
+	return err;
+}
+
+static int rtrs_rdma_conn_established(struct rtrs_clt_con *con,
+				       struct rdma_cm_event *ev)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct rtrs_clt *clt = sess->clt;
+	const struct rtrs_msg_conn_rsp *msg;
+	u16 version, queue_depth;
+	int errno;
+	u8 len;
+
+	msg = ev->param.conn.private_data;
+	len = ev->param.conn.private_data_len;
+	if (len < sizeof(*msg)) {
+		rtrs_err(clt, "Invalid RTRS connection response\n");
+		return -ECONNRESET;
+	}
+	if (le16_to_cpu(msg->magic) != RTRS_MAGIC) {
+		rtrs_err(clt, "Invalid RTRS magic\n");
+		return -ECONNRESET;
+	}
+	version = le16_to_cpu(msg->version);
+	if (version >> 8 != RTRS_PROTO_VER_MAJOR) {
+		rtrs_err(clt, "Unsupported major RTRS version: %d, expected %d\n",
+			  version >> 8, RTRS_PROTO_VER_MAJOR);
+		return -ECONNRESET;
+	}
+	errno = le16_to_cpu(msg->errno);
+	if (errno) {
+		rtrs_err(clt, "Invalid RTRS message: errno %d\n",
+			  errno);
+		return -ECONNRESET;
+	}
+	if (con->c.cid == 0) {
+		queue_depth = le16_to_cpu(msg->queue_depth);
+
+		if (queue_depth > MAX_SESS_QUEUE_DEPTH) {
+			rtrs_err(clt, "Invalid RTRS message: queue=%d\n",
+				  queue_depth);
+			return -ECONNRESET;
+		}
+		if (!sess->rbufs || sess->queue_depth < queue_depth) {
+			kfree(sess->rbufs);
+			sess->rbufs = kcalloc(queue_depth, sizeof(*sess->rbufs),
+					      GFP_KERNEL);
+			if (!sess->rbufs)
+				return -ENOMEM;
+		}
+		sess->queue_depth = queue_depth;
+		sess->max_hdr_size = le32_to_cpu(msg->max_hdr_size);
+		sess->max_io_size = le32_to_cpu(msg->max_io_size);
+		sess->flags = le32_to_cpu(msg->flags);
+		sess->chunk_size = sess->max_io_size + sess->max_hdr_size;
+
+		/*
+		 * Global queue depth and IO size is always a minimum.
+		 * If while a reconnection server sends us a value a bit
+		 * higher - client does not care and uses cached minimum.
+		 *
+		 * Since we can have several sessions (paths) restablishing
+		 * connections in parallel, use lock.
+		 */
+		mutex_lock(&clt->paths_mutex);
+		clt->queue_depth = min_not_zero(sess->queue_depth,
+						clt->queue_depth);
+		clt->max_io_size = min_not_zero(sess->max_io_size,
+						clt->max_io_size);
+		mutex_unlock(&clt->paths_mutex);
+
+		/*
+		 * Cache the hca_port and hca_name for sysfs
+		 */
+		sess->hca_port = con->c.cm_id->port_num;
+		scnprintf(sess->hca_name, sizeof(sess->hca_name),
+			  sess->s.dev->ib_dev->name);
+		sess->s.src_addr = con->c.cm_id->route.addr.src_addr;
+	}
+
+	return 0;
+}
+
+static inline void flag_success_on_conn(struct rtrs_clt_con *con)
+{
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+
+	atomic_inc(&sess->connected_cnt);
+	con->cm_err = 1;
+}
+
+static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con,
+				    struct rdma_cm_event *ev)
+{
+	struct rtrs_sess *s = con->c.sess;
+	const struct rtrs_msg_conn_rsp *msg;
+	const char *rej_msg;
+	int status, errno;
+	u8 data_len;
+
+	status = ev->status;
+	rej_msg = rdma_reject_msg(con->c.cm_id, status);
+	msg = rdma_consumer_reject_data(con->c.cm_id, ev, &data_len);
+
+	if (msg && data_len >= sizeof(*msg)) {
+		errno = (int16_t)le16_to_cpu(msg->errno);
+		if (errno == -EBUSY)
+			rtrs_err(s,
+				  "Previous session is still exists on the server, please reconnect later\n");
+		else
+			rtrs_err(s,
+				  "Connect rejected: status %d (%s), rtrs errno %d\n",
+				  status, rej_msg, errno);
+	} else {
+		rtrs_err(s,
+			  "Connect rejected but with malformed message: status %d (%s)\n",
+			  status, rej_msg);
+	}
+
+	return -ECONNRESET;
+}
+
+static void rtrs_clt_close_conns(struct rtrs_clt_sess *sess, bool wait)
+{
+	if (rtrs_clt_change_state(sess, RTRS_CLT_CLOSING))
+		queue_work(rtrs_wq, &sess->close_work);
+	if (wait)
+		flush_work(&sess->close_work);
+}
+
+static inline void flag_error_on_conn(struct rtrs_clt_con *con, int cm_err)
+{
+	if (con->cm_err == 1) {
+		struct rtrs_clt_sess *sess;
+
+		sess = to_clt_sess(con->c.sess);
+		if (atomic_dec_and_test(&sess->connected_cnt))
+
+			wake_up(&sess->state_wq);
+	}
+	con->cm_err = cm_err;
+}
+
+static int rtrs_clt_rdma_cm_handler(struct rdma_cm_id *cm_id,
+				     struct rdma_cm_event *ev)
+{
+	struct rtrs_clt_con *con = cm_id->context;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_clt_sess *sess = to_clt_sess(s);
+	int cm_err = 0;
+
+	switch (ev->event) {
+	case RDMA_CM_EVENT_ADDR_RESOLVED:
+		cm_err = rtrs_rdma_addr_resolved(con);
+		break;
+	case RDMA_CM_EVENT_ROUTE_RESOLVED:
+		cm_err = rtrs_rdma_route_resolved(con);
+		break;
+	case RDMA_CM_EVENT_ESTABLISHED:
+		con->cm_err = rtrs_rdma_conn_established(con, ev);
+		if (likely(!con->cm_err)) {
+			/*
+			 * Report success and wake up. Here we abuse state_wq,
+			 * i.e. wake up without state change, but we set cm_err.
+			 */
+			flag_success_on_conn(con);
+			wake_up(&sess->state_wq);
+			return 0;
+		}
+		break;
+	case RDMA_CM_EVENT_REJECTED:
+		cm_err = rtrs_rdma_conn_rejected(con, ev);
+		break;
+	case RDMA_CM_EVENT_CONNECT_ERROR:
+	case RDMA_CM_EVENT_UNREACHABLE:
+		rtrs_wrn(s, "CM error event %d\n", ev->event);
+		cm_err = -ECONNRESET;
+		break;
+	case RDMA_CM_EVENT_ADDR_ERROR:
+	case RDMA_CM_EVENT_ROUTE_ERROR:
+		cm_err = -EHOSTUNREACH;
+		break;
+	case RDMA_CM_EVENT_DISCONNECTED:
+	case RDMA_CM_EVENT_ADDR_CHANGE:
+	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+		cm_err = -ECONNRESET;
+		break;
+	case RDMA_CM_EVENT_DEVICE_REMOVAL:
+		/*
+		 * Device removal is a special case.  Queue close and return 0.
+		 */
+		rtrs_clt_close_conns(sess, false);
+		return 0;
+	default:
+		rtrs_err(s, "Unexpected RDMA CM event (%d)\n", ev->event);
+		cm_err = -ECONNRESET;
+		break;
+	}
+
+	if (cm_err) {
+		/*
+		 * cm error makes sense only on connection establishing,
+		 * in other cases we rely on normal procedure of reconnecting.
+		 */
+		flag_error_on_conn(con, cm_err);
+		rtrs_rdma_error_recovery(con);
+	}
+
+	return 0;
+}
+
+static int create_cm(struct rtrs_clt_con *con)
+{
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_clt_sess *sess = to_clt_sess(s);
+	struct rdma_cm_id *cm_id;
+	int err;
+
+	cm_id = rdma_create_id(&init_net, rtrs_clt_rdma_cm_handler, con,
+			       sess->s.dst_addr.ss_family == AF_IB ?
+			       RDMA_PS_IB : RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(cm_id)) {
+		err = PTR_ERR(cm_id);
+		rtrs_err(s, "Failed to create CM ID, err: %d\n", err);
+
+		return err;
+	}
+	con->c.cm_id = cm_id;
+	con->cm_err = 0;
+	/* allow the port to be reused */
+	err = rdma_set_reuseaddr(cm_id, 1);
+	if (err != 0) {
+		rtrs_err(s, "Set address reuse failed, err: %d\n", err);
+		goto destroy_cm;
+	}
+	err = rdma_resolve_addr(cm_id, (struct sockaddr *)&sess->s.src_addr,
+				(struct sockaddr *)&sess->s.dst_addr,
+				RTRS_CONNECT_TIMEOUT_MS);
+	if (err) {
+		rtrs_err(s, "Failed to resolve address, err: %d\n", err);
+		goto destroy_cm;
+	}
+	/*
+	 * Combine connection status and session events. This is needed
+	 * for waiting two possible cases: cm_err has something meaningful
+	 * or session state was really changed to error by device removal.
+	 */
+	err = wait_event_interruptible_timeout(
+			sess->state_wq,
+			con->cm_err || sess->state != RTRS_CLT_CONNECTING,
+			msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));
+	if (err == 0 || err == -ERESTARTSYS) {
+		if (err == 0)
+			err = -ETIMEDOUT;
+		/* Timedout or interrupted */
+		goto errr;
+	}
+	if (con->cm_err < 0) {
+		err = con->cm_err;
+		goto errr;
+	}
+	if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTING) {
+		/* Device removal */
+		err = -ECONNABORTED;
+		goto errr;
+	}
+
+	return 0;
+
+errr:
+	stop_cm(con);
+	/* Is safe to call destroy if cq_qp is not inited */
+	destroy_con_cq_qp(con);
+destroy_cm:
+	destroy_cm(con);
+
+	return err;
+}
+
+static void rtrs_clt_sess_up(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt *clt = sess->clt;
+	int up;
+
+	/*
+	 * We can fire RECONNECTED event only when all paths were
+	 * connected on rtrs_clt_open(), then each was disconnected
+	 * and the first one connected again.  That's why this nasty
+	 * game with counter value.
+	 */
+
+	mutex_lock(&clt->paths_ev_mutex);
+	up = ++clt->paths_up;
+	/*
+	 * Here it is safe to access paths num directly since up counter
+	 * is greater than MAX_PATHS_NUM only while rtrs_clt_open() is
+	 * in progress, thus paths removals are impossible.
+	 */
+	if (up > MAX_PATHS_NUM && up == MAX_PATHS_NUM + clt->paths_num)
+		clt->paths_up = clt->paths_num;
+	else if (up == 1)
+		clt->link_ev(clt->priv, RTRS_CLT_LINK_EV_RECONNECTED);
+	mutex_unlock(&clt->paths_ev_mutex);
+
+	/* Mark session as established */
+	sess->established = true;
+	sess->reconnect_attempts = 0;
+	sess->stats->reconnects.successful_cnt++;
+}
+
+static void rtrs_clt_sess_down(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt *clt = sess->clt;
+
+	if (!sess->established)
+		return;
+
+	sess->established = false;
+	mutex_lock(&clt->paths_ev_mutex);
+	WARN_ON(!clt->paths_up);
+	if (--clt->paths_up == 0)
+		clt->link_ev(clt->priv, RTRS_CLT_LINK_EV_DISCONNECTED);
+	mutex_unlock(&clt->paths_ev_mutex);
+}
+
+static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt_con *con;
+	unsigned int cid;
+
+	WARN_ON(READ_ONCE(sess->state) == RTRS_CLT_CONNECTED);
+
+	/*
+	 * Possible race with rtrs_clt_open(), when DEVICE_REMOVAL comes
+	 * exactly in between.  Start destroying after it finishes.
+	 */
+	mutex_lock(&sess->init_mutex);
+	mutex_unlock(&sess->init_mutex);
+
+	/*
+	 * All IO paths must observe !CONNECTED state before we
+	 * free everything.
+	 */
+	synchronize_rcu();
+
+	rtrs_clt_stop_hb(sess);
+
+	/*
+	 * The order it utterly crucial: firstly disconnect and complete all
+	 * rdma requests with error (thus set in_use=false for requests),
+	 * then fail outstanding requests checking in_use for each, and
+	 * eventually notify upper layer about session disconnection.
+	 */
+
+	for (cid = 0; cid < sess->s.con_num; cid++) {
+		if (!sess->s.con[cid])
+			break;
+		con = to_clt_con(sess->s.con[cid]);
+		stop_cm(con);
+	}
+	fail_all_outstanding_reqs(sess);
+	free_sess_reqs(sess);
+	rtrs_clt_sess_down(sess);
+
+	/*
+	 * Wait for graceful shutdown, namely when peer side invokes
+	 * rdma_disconnect(). 'connected_cnt' is decremented only on
+	 * CM events, thus if other side had crashed and hb has detected
+	 * something is wrong, here we will stuck for exactly timeout ms,
+	 * since CM does not fire anything.  That is fine, we are not in
+	 * hurry.
+	 */
+	wait_event_timeout(sess->state_wq, !atomic_read(&sess->connected_cnt),
+			   msecs_to_jiffies(RTRS_CONNECT_TIMEOUT_MS));
+
+	for (cid = 0; cid < sess->s.con_num; cid++) {
+		if (!sess->s.con[cid])
+			break;
+		con = to_clt_con(sess->s.con[cid]);
+		destroy_con_cq_qp(con);
+		destroy_cm(con);
+		destroy_con(con);
+	}
+}
+
+static inline bool xchg_sessions(struct rtrs_clt_sess __rcu **rcu_ppcpu_path,
+				 struct rtrs_clt_sess *sess,
+				 struct rtrs_clt_sess *next)
+{
+	struct rtrs_clt_sess **ppcpu_path;
+
+	/* Call cmpxchg() without sparse warnings */
+	ppcpu_path = (typeof(ppcpu_path))rcu_ppcpu_path;
+	return sess == cmpxchg(ppcpu_path, sess, next);
+}
+
+static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt *clt = sess->clt;
+	struct rtrs_clt_sess *next;
+	bool wait_for_grace = false;
+	int cpu;
+
+	mutex_lock(&clt->paths_mutex);
+	list_del_rcu(&sess->s.entry);
+
+	/* Make sure everybody observes path removal. */
+	synchronize_rcu();
+
+	/*
+	 * At this point nobody sees @sess in the list, but still we have
+	 * dangling pointer @pcpu_path which _can_ point to @sess.  Since
+	 * nobody can observe @sess in the list, we guarantee that IO path
+	 * will not assign @sess to @pcpu_path, i.e. @pcpu_path can be equal
+	 * to @sess, but can never again become @sess.
+	 */
+
+	/*
+	 * Decrement paths number only after grace period, because
+	 * caller of do_each_path() must firstly observe list without
+	 * path and only then decremented paths number.
+	 *
+	 * Otherwise there can be the following situation:
+	 *    o Two paths exist and IO is coming.
+	 *    o One path is removed:
+	 *      CPU#0                          CPU#1
+	 *      do_each_path():                rtrs_clt_remove_path_from_arr():
+	 *          path = get_next_path()
+	 *          ^^^                            list_del_rcu(path)
+	 *          [!CONNECTED path]              clt->paths_num--
+	 *                                              ^^^^^^^^^
+	 *          load clt->paths_num                 from 2 to 1
+	 *                    ^^^^^^^^^
+	 *                    sees 1
+	 *
+	 *      path is observed as !CONNECTED, but do_each_path() loop
+	 *      ends, because expression i < clt->paths_num is false.
+	 */
+	clt->paths_num--;
+
+	/*
+	 * Get @next connection from current @sess which is going to be
+	 * removed.  If @sess is the last element, then @next is NULL.
+	 */
+	rcu_read_lock();
+	next = list_next_or_null_rr_rcu(&clt->paths_list, &sess->s.entry,
+					typeof(*next), s.entry);
+	rcu_read_unlock();
+
+	/*
+	 * @pcpu paths can still point to the path which is going to be
+	 * removed, so change the pointer manually.
+	 */
+	for_each_possible_cpu(cpu) {
+		struct rtrs_clt_sess __rcu **ppcpu_path;
+
+		ppcpu_path = per_cpu_ptr(clt->pcpu_path, cpu);
+		if (rcu_dereference_protected(*ppcpu_path,
+			lockdep_is_held(&clt->paths_mutex)) != sess)
+			/*
+			 * synchronize_rcu() was called just after deleting
+			 * entry from the list, thus IO code path cannot
+			 * change pointer back to the pointer which is going
+			 * to be removed, we are safe here.
+			 */
+			continue;
+
+		/*
+		 * We race with IO code path, which also changes pointer,
+		 * thus we have to be careful not to overwrite it.
+		 */
+		if (xchg_sessions(ppcpu_path, sess, next))
+			/*
+			 * @ppcpu_path was successfully replaced with @next,
+			 * that means that someone could also pick up the
+			 * @sess and dereferencing it right now, so wait for
+			 * a grace period is required.
+			 */
+			wait_for_grace = true;
+	}
+	if (wait_for_grace)
+		synchronize_rcu();
+
+	mutex_unlock(&clt->paths_mutex);
+}
+
+static void rtrs_clt_add_path_to_arr(struct rtrs_clt_sess *sess,
+				      struct rtrs_addr *addr)
+{
+	struct rtrs_clt *clt = sess->clt;
+
+	mutex_lock(&clt->paths_mutex);
+	clt->paths_num++;
+
+	list_add_tail_rcu(&sess->s.entry, &clt->paths_list);
+	mutex_unlock(&clt->paths_mutex);
+}
+
+static void rtrs_clt_close_work(struct work_struct *work)
+{
+	struct rtrs_clt_sess *sess;
+
+	sess = container_of(work, struct rtrs_clt_sess, close_work);
+
+	cancel_delayed_work_sync(&sess->reconnect_dwork);
+	rtrs_clt_stop_and_destroy_conns(sess);
+	rtrs_clt_change_state(sess, RTRS_CLT_CLOSED);
+}
+
+static int init_conns(struct rtrs_clt_sess *sess)
+{
+	unsigned int cid;
+	int err;
+
+	/*
+	 * On every new session connections increase reconnect counter
+	 * to avoid clashes with previous sessions not yet closed
+	 * sessions on a server side.
+	 */
+	sess->s.recon_cnt++;
+
+	/* Establish all RDMA connections  */
+	for (cid = 0; cid < sess->s.con_num; cid++) {
+		err = create_con(sess, cid);
+		if (err)
+			goto destroy;
+
+		err = create_cm(to_clt_con(sess->s.con[cid]));
+		if (err) {
+			destroy_con(to_clt_con(sess->s.con[cid]));
+			goto destroy;
+		}
+	}
+	err = alloc_sess_reqs(sess);
+	if (err)
+		goto destroy;
+
+	rtrs_clt_start_hb(sess);
+
+	return 0;
+
+destroy:
+	while (cid--) {
+		struct rtrs_clt_con *con = to_clt_con(sess->s.con[cid]);
+
+		stop_cm(con);
+		destroy_con_cq_qp(con);
+		destroy_cm(con);
+		destroy_con(con);
+	}
+	/*
+	 * If we've never taken async path and got an error, say,
+	 * doing rdma_resolve_addr(), switch to CONNECTION_ERR state
+	 * manually to keep reconnecting.
+	 */
+	rtrs_clt_change_state(sess, RTRS_CLT_CONNECTING_ERR);
+
+	return err;
+}
+
+static void rtrs_clt_info_req_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_clt_con *con = cq->cq_context;
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct rtrs_iu *iu;
+
+	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
+	rtrs_iu_free(iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(sess->clt, "Sess info request send failed: %s\n",
+			  ib_wc_status_msg(wc->status));
+		rtrs_clt_change_state(sess, RTRS_CLT_CONNECTING_ERR);
+		return;
+	}
+
+	rtrs_clt_update_wc_stats(con);
+}
+
+static int process_info_rsp(struct rtrs_clt_sess *sess,
+			    const struct rtrs_msg_info_rsp *msg)
+{
+	unsigned int sg_cnt, total_len;
+	int i, sgi;
+
+	sg_cnt = le16_to_cpu(msg->sg_cnt);
+	if (unlikely(!sg_cnt))
+		return -EINVAL;
+	/*
+	 * Check if IB immediate data size is enough to hold the mem_id and
+	 * the offset inside the memory chunk.
+	 */
+	if (unlikely((ilog2(sg_cnt - 1) + 1) +
+		     (ilog2(sess->chunk_size - 1) + 1) >
+		     MAX_IMM_PAYL_BITS)) {
+		rtrs_err(sess->clt,
+			  "RDMA immediate size (%db) not enough to encode %d buffers of size %dB\n",
+			  MAX_IMM_PAYL_BITS, sg_cnt, sess->chunk_size);
+		return -EINVAL;
+	}
+	if (unlikely(!sg_cnt || (sess->queue_depth % sg_cnt))) {
+		rtrs_err(sess->clt, "Incorrect sg_cnt %d, is not multiple\n",
+			  sg_cnt);
+		return -EINVAL;
+	}
+	total_len = 0;
+	for (sgi = 0, i = 0; sgi < sg_cnt && i < sess->queue_depth; sgi++) {
+		const struct rtrs_sg_desc *desc = &msg->desc[sgi];
+		u32 len, rkey;
+		u64 addr;
+
+		addr = le64_to_cpu(desc->addr);
+		rkey = le32_to_cpu(desc->key);
+		len  = le32_to_cpu(desc->len);
+
+		total_len += len;
+
+		if (unlikely(!len || (len % sess->chunk_size))) {
+			rtrs_err(sess->clt, "Incorrect [%d].len %d\n", sgi,
+				  len);
+			return -EINVAL;
+		}
+		for ( ; len && i < sess->queue_depth; i++) {
+			sess->rbufs[i].addr = addr;
+			sess->rbufs[i].rkey = rkey;
+
+			len  -= sess->chunk_size;
+			addr += sess->chunk_size;
+		}
+	}
+	/* Sanity check */
+	if (unlikely(sgi != sg_cnt || i != sess->queue_depth)) {
+		rtrs_err(sess->clt, "Incorrect sg vector, not fully mapped\n");
+		return -EINVAL;
+	}
+	if (unlikely(total_len != sess->chunk_size * sess->queue_depth)) {
+		rtrs_err(sess->clt, "Incorrect total_len %d\n", total_len);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_clt_con *con = cq->cq_context;
+	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
+	struct rtrs_msg_info_rsp *msg;
+	enum rtrs_clt_state state;
+	struct rtrs_iu *iu;
+	size_t rx_sz;
+	int err;
+
+	state = RTRS_CLT_CONNECTING_ERR;
+
+	WARN_ON(con->c.cid);
+	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(sess->clt, "Sess info response recv failed: %s\n",
+			  ib_wc_status_msg(wc->status));
+		goto out;
+	}
+	WARN_ON(wc->opcode != IB_WC_RECV);
+
+	if (unlikely(wc->byte_len < sizeof(*msg))) {
+		rtrs_err(sess->clt, "Sess info response is malformed: size %d\n",
+			  wc->byte_len);
+		goto out;
+	}
+	ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr,
+				   iu->size, DMA_FROM_DEVICE);
+	msg = iu->buf;
+	if (unlikely(le16_to_cpu(msg->type) != RTRS_MSG_INFO_RSP)) {
+		rtrs_err(sess->clt, "Sess info response is malformed: type %d\n",
+			  le16_to_cpu(msg->type));
+		goto out;
+	}
+	rx_sz  = sizeof(*msg);
+	rx_sz += sizeof(msg->desc[0]) * le16_to_cpu(msg->sg_cnt);
+	if (unlikely(wc->byte_len < rx_sz)) {
+		rtrs_err(sess->clt, "Sess info response is malformed: size %d\n",
+			  wc->byte_len);
+		goto out;
+	}
+	err = process_info_rsp(sess, msg);
+	if (unlikely(err))
+		goto out;
+
+	err = post_recv_sess(sess);
+	if (unlikely(err))
+		goto out;
+
+	state = RTRS_CLT_CONNECTED;
+
+out:
+	rtrs_clt_update_wc_stats(con);
+	rtrs_iu_free(iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
+	rtrs_clt_change_state(sess, state);
+}
+
+static int rtrs_send_sess_info(struct rtrs_clt_sess *sess)
+{
+	struct rtrs_clt_con *usr_con = to_clt_con(sess->s.con[0]);
+	struct rtrs_msg_info_req *msg;
+	struct rtrs_iu *tx_iu, *rx_iu;
+	size_t rx_sz;
+	int err;
+
+	rx_sz  = sizeof(struct rtrs_msg_info_rsp);
+	rx_sz += sizeof(u64) * MAX_SESS_QUEUE_DEPTH;
+
+	tx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req), GFP_KERNEL,
+			       sess->s.dev->ib_dev, DMA_TO_DEVICE,
+			       rtrs_clt_info_req_done);
+	rx_iu = rtrs_iu_alloc(1, rx_sz, GFP_KERNEL, sess->s.dev->ib_dev,
+			       DMA_FROM_DEVICE, rtrs_clt_info_rsp_done);
+	if (unlikely(!tx_iu || !rx_iu)) {
+		err = -ENOMEM;
+		goto out;
+	}
+	/* Prepare for getting info response */
+	err = rtrs_iu_post_recv(&usr_con->c, rx_iu);
+	if (unlikely(err)) {
+		rtrs_err(sess->clt, "rtrs_iu_post_recv(), err: %d\n", err);
+		goto out;
+	}
+	rx_iu = NULL;
+
+	msg = tx_iu->buf;
+	msg->type = cpu_to_le16(RTRS_MSG_INFO_REQ);
+	memcpy(msg->sessname, sess->s.sessname, sizeof(msg->sessname));
+
+	ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr,
+				      tx_iu->size, DMA_TO_DEVICE);
+
+	/* Send info request */
+	err = rtrs_iu_post_send(&usr_con->c, tx_iu, sizeof(*msg), NULL);
+	if (unlikely(err)) {
+		rtrs_err(sess->clt, "rtrs_iu_post_send(), err: %d\n", err);
+		goto out;
+	}
+	tx_iu = NULL;
+
+	/* Wait for state change */
+	wait_event_interruptible_timeout(sess->state_wq,
+					 sess->state != RTRS_CLT_CONNECTING,
+					 msecs_to_jiffies(
+						 RTRS_CONNECT_TIMEOUT_MS));
+	if (unlikely(READ_ONCE(sess->state) != RTRS_CLT_CONNECTED)) {
+		if (READ_ONCE(sess->state) == RTRS_CLT_CONNECTING_ERR)
+			err = -ECONNRESET;
+		else
+			err = -ETIMEDOUT;
+		goto out;
+	}
+
+out:
+	if (tx_iu)
+		rtrs_iu_free(tx_iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
+	if (rx_iu)
+		rtrs_iu_free(rx_iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
+	if (unlikely(err))
+		/* If we've never taken async path because of malloc problems */
+		rtrs_clt_change_state(sess, RTRS_CLT_CONNECTING_ERR);
+
+	return err;
+}
+
+/**
+ * init_sess() - establishes all session connections and does handshake
+ * @sess: client session.
+ * In case of error full close or reconnect procedure should be taken,
+ * because reconnect or close async works can be started.
+ */
+static int init_sess(struct rtrs_clt_sess *sess)
+{
+	int err;
+
+	mutex_lock(&sess->init_mutex);
+	err = init_conns(sess);
+	if (err) {
+		rtrs_err(sess->clt, "init_conns(), err: %d\n", err);
+		goto out;
+	}
+	err = rtrs_send_sess_info(sess);
+	if (err) {
+		rtrs_err(sess->clt, "rtrs_send_sess_info(), err: %d\n", err);
+		goto out;
+	}
+	rtrs_clt_sess_up(sess);
+out:
+	mutex_unlock(&sess->init_mutex);
+
+	return err;
+}
+
+static void rtrs_clt_reconnect_work(struct work_struct *work)
+{
+	struct rtrs_clt_sess *sess;
+	struct rtrs_clt *clt;
+	unsigned int delay_ms;
+	int err;
+
+	sess = container_of(to_delayed_work(work), struct rtrs_clt_sess,
+			    reconnect_dwork);
+	clt = sess->clt;
+
+	if (READ_ONCE(sess->state) != RTRS_CLT_RECONNECTING)
+		return;
+
+	if (sess->reconnect_attempts >= clt->max_reconnect_attempts) {
+		/* Close a session completely if max attempts is reached */
+		rtrs_clt_close_conns(sess, false);
+		return;
+	}
+	sess->reconnect_attempts++;
+
+	/* Stop everything */
+	rtrs_clt_stop_and_destroy_conns(sess);
+	msleep(RTRS_RECONNECT_BACKOFF);
+	if (rtrs_clt_change_state(sess, RTRS_CLT_CONNECTING)) {
+		err = init_sess(sess);
+		if (err)
+			goto reconnect_again;
+	}
+
+	return;
+
+reconnect_again:
+	if (rtrs_clt_change_state(sess, RTRS_CLT_RECONNECTING)) {
+		sess->stats->reconnects.fail_cnt++;
+		delay_ms = clt->reconnect_delay_sec * 1000;
+		queue_delayed_work(rtrs_wq, &sess->reconnect_dwork,
+				   msecs_to_jiffies(delay_ms));
+	}
+}
+
+static void rtrs_clt_dev_release(struct device *dev)
+{
+	struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);
+
+	kfree(clt);
+}
+
+static struct rtrs_clt *alloc_clt(const char *sessname, size_t paths_num,
+				  u16 port, size_t pdu_sz, void *priv,
+				  void	(*link_ev)(void *priv,
+						   enum rtrs_clt_link_ev ev),
+				  unsigned int max_segments,
+				  size_t max_segment_size,
+				  unsigned int reconnect_delay_sec,
+				  unsigned int max_reconnect_attempts)
+{
+	struct rtrs_clt *clt;
+	int err;
+
+	if (!paths_num || paths_num > MAX_PATHS_NUM)
+		return ERR_PTR(-EINVAL);
+
+	if (strlen(sessname) >= sizeof(clt->sessname))
+		return ERR_PTR(-EINVAL);
+
+	clt = kzalloc(sizeof(*clt), GFP_KERNEL);
+	if (!clt)
+		return ERR_PTR(-ENOMEM);
+
+	clt->pcpu_path = alloc_percpu(typeof(*clt->pcpu_path));
+	if (!clt->pcpu_path) {
+		kfree(clt);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	uuid_gen(&clt->paths_uuid);
+	INIT_LIST_HEAD_RCU(&clt->paths_list);
+	clt->paths_num = paths_num;
+	clt->paths_up = MAX_PATHS_NUM;
+	clt->port = port;
+	clt->pdu_sz = pdu_sz;
+	clt->max_segments = max_segments;
+	clt->max_segment_size = max_segment_size;
+	clt->reconnect_delay_sec = reconnect_delay_sec;
+	clt->max_reconnect_attempts = max_reconnect_attempts;
+	clt->priv = priv;
+	clt->link_ev = link_ev;
+	clt->mp_policy = MP_POLICY_MIN_INFLIGHT;
+	strlcpy(clt->sessname, sessname, sizeof(clt->sessname));
+	init_waitqueue_head(&clt->permits_wait);
+	mutex_init(&clt->paths_ev_mutex);
+	mutex_init(&clt->paths_mutex);
+
+	clt->dev.class = rtrs_clt_dev_class;
+	clt->dev.release = rtrs_clt_dev_release;
+	err = dev_set_name(&clt->dev, "%s", sessname);
+	if (err) {
+		free_percpu(clt->pcpu_path);
+		kfree(clt);
+		return ERR_PTR(err);
+	}
+	/*
+	 * Suppress user space notification until
+	 * sysfs files are created
+	 */
+	dev_set_uevent_suppress(&clt->dev, true);
+	err = device_register(&clt->dev);
+	if (err) {
+		free_percpu(clt->pcpu_path);
+		put_device(&clt->dev);
+		return ERR_PTR(err);
+	}
+
+	clt->kobj_paths = kobject_create_and_add("paths", &clt->dev.kobj);
+	if (!clt->kobj_paths) {
+		free_percpu(clt->pcpu_path);
+		device_unregister(&clt->dev);
+		return NULL;
+	}
+	err = rtrs_clt_create_sysfs_root_files(clt);
+	if (err) {
+		free_percpu(clt->pcpu_path);
+		kobject_del(clt->kobj_paths);
+		kobject_put(clt->kobj_paths);
+		device_unregister(&clt->dev);
+		return ERR_PTR(err);
+	}
+	dev_set_uevent_suppress(&clt->dev, false);
+	kobject_uevent(&clt->dev.kobj, KOBJ_ADD);
+
+	return clt;
+}
+
+static void wait_for_inflight_permits(struct rtrs_clt *clt)
+{
+	if (clt->permits_map) {
+		size_t sz = clt->queue_depth;
+
+		wait_event(clt->permits_wait,
+			   find_first_bit(clt->permits_map, sz) >= sz);
+	}
+}
+
+static void free_clt(struct rtrs_clt *clt)
+{
+	wait_for_inflight_permits(clt);
+	free_permits(clt);
+	free_percpu(clt->pcpu_path);
+	mutex_destroy(&clt->paths_ev_mutex);
+	mutex_destroy(&clt->paths_mutex);
+	/* release callback will free clt in last put */
+	device_unregister(&clt->dev);
+}
+
+/**
+ * rtrs_clt_open() - Open a session to an RTRS server
+ * @ops: holds the link event callback and the private pointer.
+ * @sessname: name of the session
+ * @paths: Paths to be established defined by their src and dst addresses
+ * @paths_num: Number of elements in the @paths array
+ * @port: port to be used by the RTRS session
+ * @pdu_sz: Size of extra payload which can be accessed after permit allocation.
+ * @reconnect_delay_sec: time between reconnect tries
+ * @max_segments: Max. number of segments per IO request
+ * @max_segment_size: Max. size of one segment
+ * @max_reconnect_attempts: Number of times to reconnect on error before giving
+ *			    up, 0 for * disabled, -1 for forever
+ *
+ * Starts session establishment with the rtrs_server. The function can block
+ * up to ~2000ms before it returns.
+ *
+ * Return a valid pointer on success otherwise PTR_ERR.
+ */
+struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
+				 const char *sessname,
+				 const struct rtrs_addr *paths,
+				 size_t paths_num, u16 port,
+				 size_t pdu_sz, u8 reconnect_delay_sec,
+				 u16 max_segments,
+				 size_t max_segment_size,
+				 s16 max_reconnect_attempts)
+{
+	struct rtrs_clt_sess *sess, *tmp;
+	struct rtrs_clt *clt;
+	int err, i;
+
+	clt = alloc_clt(sessname, paths_num, port, pdu_sz, ops->priv,
+			ops->link_ev,
+			max_segments, max_segment_size, reconnect_delay_sec,
+			max_reconnect_attempts);
+	if (IS_ERR(clt)) {
+		err = PTR_ERR(clt);
+		goto out;
+	}
+	for (i = 0; i < paths_num; i++) {
+		struct rtrs_clt_sess *sess;
+
+		sess = alloc_sess(clt, &paths[i], nr_cpu_ids,
+				  max_segments, max_segment_size);
+		if (IS_ERR(sess)) {
+			err = PTR_ERR(sess);
+			goto close_all_sess;
+		}
+		list_add_tail_rcu(&sess->s.entry, &clt->paths_list);
+
+		err = init_sess(sess);
+		if (err) {
+			list_del_rcu(&sess->s.entry);
+			rtrs_clt_close_conns(sess, true);
+			free_sess(sess);
+			goto close_all_sess;
+		}
+
+		err = rtrs_clt_create_sess_files(sess);
+		if (err) {
+			list_del_rcu(&sess->s.entry);
+			rtrs_clt_close_conns(sess, true);
+			free_sess(sess);
+			goto close_all_sess;
+		}
+	}
+	err = alloc_permits(clt);
+	if (err)
+		goto close_all_sess;
+
+	return clt;
+
+close_all_sess:
+	list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) {
+		rtrs_clt_destroy_sess_files(sess, NULL);
+		rtrs_clt_close_conns(sess, true);
+		kobject_put(&sess->kobj);
+	}
+	rtrs_clt_destroy_sysfs_root_files(clt);
+	rtrs_clt_destroy_sysfs_root_folders(clt);
+	free_clt(clt);
+
+out:
+	return ERR_PTR(err);
+}
+EXPORT_SYMBOL(rtrs_clt_open);
+
+/**
+ * rtrs_clt_close() - Close a session
+ * @clt: Session handle. Session is freed upon return.
+ */
+void rtrs_clt_close(struct rtrs_clt *clt)
+{
+	struct rtrs_clt_sess *sess, *tmp;
+
+	/* Firstly forbid sysfs access */
+	rtrs_clt_destroy_sysfs_root_files(clt);
+	rtrs_clt_destroy_sysfs_root_folders(clt);
+
+	/* Now it is safe to iterate over all paths without locks */
+	list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) {
+		rtrs_clt_destroy_sess_files(sess, NULL);
+		rtrs_clt_close_conns(sess, true);
+		kobject_put(&sess->kobj);
+	}
+	free_clt(clt);
+}
+EXPORT_SYMBOL(rtrs_clt_close);
+
+int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess)
+{
+	enum rtrs_clt_state old_state;
+	int err = -EBUSY;
+	bool changed;
+
+	changed = rtrs_clt_change_state_get_old(sess, RTRS_CLT_RECONNECTING,
+						 &old_state);
+	if (changed) {
+		sess->reconnect_attempts = 0;
+		queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, 0);
+	}
+	if (changed || old_state == RTRS_CLT_RECONNECTING) {
+		/*
+		 * flush_delayed_work() queues pending work for immediate
+		 * execution, so do the flush if we have queued something
+		 * right now or work is pending.
+		 */
+		flush_delayed_work(&sess->reconnect_dwork);
+		err = (READ_ONCE(sess->state) ==
+		       RTRS_CLT_CONNECTED ? 0 : -ENOTCONN);
+	}
+
+	return err;
+}
+
+int rtrs_clt_disconnect_from_sysfs(struct rtrs_clt_sess *sess)
+{
+	rtrs_clt_close_conns(sess, true);
+
+	return 0;
+}
+
+int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess,
+				     const struct attribute *sysfs_self)
+{
+	enum rtrs_clt_state old_state;
+	bool changed;
+
+	/*
+	 * Continue stopping path till state was changed to DEAD or
+	 * state was observed as DEAD:
+	 * 1. State was changed to DEAD - we were fast and nobody
+	 *    invoked rtrs_clt_reconnect(), which can again start
+	 *    reconnecting.
+	 * 2. State was observed as DEAD - we have someone in parallel
+	 *    removing the path.
+	 */
+	do {
+		rtrs_clt_close_conns(sess, true);
+		changed = rtrs_clt_change_state_get_old(sess,
+							RTRS_CLT_DEAD,
+							&old_state);
+	} while (!changed && old_state != RTRS_CLT_DEAD);
+
+	if (likely(changed)) {
+		rtrs_clt_destroy_sess_files(sess, sysfs_self);
+		rtrs_clt_remove_path_from_arr(sess);
+		kobject_put(&sess->kobj);
+	}
+
+	return 0;
+}
+
+void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value)
+{
+	clt->max_reconnect_attempts = (unsigned int)value;
+}
+
+int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt)
+{
+	return (int)clt->max_reconnect_attempts;
+}
+
+/**
+ * rtrs_clt_request() - Request data transfer to/from server via RDMA.
+ *
+ * @dir:	READ/WRITE
+ * @ops:	callback function to be called as confirmation, and the pointer.
+ * @clt:	Session
+ * @permit:	Preallocated permit
+ * @vec:	Message that is sent to server together with the request.
+ *		Sum of len of all @vec elements limited to <= IO_MSG_SIZE.
+ *		Since the msg is copied internally it can be allocated on stack.
+ * @nr:		Number of elements in @vec.
+ * @data_len:	length of data sent to/from server
+ * @sg:		Pages to be sent/received to/from server.
+ * @sg_cnt:	Number of elements in the @sg
+ *
+ * Return:
+ * 0:		Success
+ * <0:		Error
+ *
+ * On dir=READ rtrs client will request a data transfer from Server to client.
+ * The data that the server will respond with will be stored in @sg when
+ * the user receives an %RTRS_CLT_RDMA_EV_RDMA_REQUEST_WRITE_COMPL event.
+ * On dir=WRITE rtrs client will rdma write data in sg to server side.
+ */
+int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,
+		     struct rtrs_clt *clt, struct rtrs_permit *permit,
+		      const struct kvec *vec, size_t nr, size_t data_len,
+		      struct scatterlist *sg, unsigned int sg_cnt)
+{
+	struct rtrs_clt_io_req *req;
+	struct rtrs_clt_sess *sess;
+
+	enum dma_data_direction dma_dir;
+	int err = -ECONNABORTED, i;
+	size_t usr_len, hdr_len;
+	struct path_it it;
+
+	/* Get kvec length */
+	for (i = 0, usr_len = 0; i < nr; i++)
+		usr_len += vec[i].iov_len;
+
+	if (dir == READ) {
+		hdr_len = sizeof(struct rtrs_msg_rdma_read) +
+			  sg_cnt * sizeof(struct rtrs_sg_desc);
+		dma_dir = DMA_FROM_DEVICE;
+	} else {
+		hdr_len = sizeof(struct rtrs_msg_rdma_write);
+		dma_dir = DMA_TO_DEVICE;
+	}
+
+	rcu_read_lock();
+	for (path_it_init(&it, clt);
+	     (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) {
+		if (unlikely(READ_ONCE(sess->state) != RTRS_CLT_CONNECTED))
+			continue;
+
+		if (unlikely(usr_len + hdr_len > sess->max_hdr_size)) {
+			rtrs_wrn_rl(sess->clt,
+				     "%s request failed, user message size is %zu and header length %zu, but max size is %u\n",
+				     dir == READ ? "Read" : "Write",
+				     usr_len, hdr_len, sess->max_hdr_size);
+			err = -EMSGSIZE;
+			break;
+		}
+		req = rtrs_clt_get_req(sess, ops->conf_fn, permit, ops->priv,
+				       vec, usr_len, sg, sg_cnt, data_len,
+				       dma_dir);
+		if (dir == READ)
+			err = rtrs_clt_read_req(req);
+		else
+			err = rtrs_clt_write_req(req);
+		if (unlikely(err)) {
+			req->in_use = false;
+			continue;
+		}
+		/* Success path */
+		break;
+	}
+	path_it_deinit(&it);
+	rcu_read_unlock();
+
+	return err;
+}
+EXPORT_SYMBOL(rtrs_clt_request);
+
+/**
+ * rtrs_clt_query() - queries RTRS session attributes
+ *@clt: session pointer
+ *@attr: query results for session attributes.
+ * Returns:
+ *    0 on success
+ *    -ECOMM		no connection to the server
+ */
+int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr)
+{
+	if (!rtrs_clt_is_connected(clt))
+		return -ECOMM;
+
+	attr->queue_depth      = clt->queue_depth;
+	attr->max_io_size      = clt->max_io_size;
+	attr->sess_kobj	       = &clt->dev.kobj;
+	strlcpy(attr->sessname, clt->sessname, sizeof(attr->sessname));
+
+	return 0;
+}
+EXPORT_SYMBOL(rtrs_clt_query);
+
+int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,
+				     struct rtrs_addr *addr)
+{
+	struct rtrs_clt_sess *sess;
+	int err;
+
+	sess = alloc_sess(clt, addr, nr_cpu_ids, clt->max_segments,
+			  clt->max_segment_size);
+	if (IS_ERR(sess))
+		return PTR_ERR(sess);
+
+	/*
+	 * It is totally safe to add path in CONNECTING state: coming
+	 * IO will never grab it.  Also it is very important to add
+	 * path before init, since init fires LINK_CONNECTED event.
+	 */
+	rtrs_clt_add_path_to_arr(sess, addr);
+
+	err = init_sess(sess);
+	if (err)
+		goto close_sess;
+
+	err = rtrs_clt_create_sess_files(sess);
+	if (err)
+		goto close_sess;
+
+	return 0;
+
+close_sess:
+	rtrs_clt_remove_path_from_arr(sess);
+	rtrs_clt_close_conns(sess, true);
+	free_sess(sess);
+
+	return err;
+}
+
+static int rtrs_clt_ib_dev_init(struct rtrs_ib_dev *dev)
+{
+	if (!(dev->ib_dev->attrs.device_cap_flags &
+	      IB_DEVICE_MEM_MGT_EXTENSIONS)) {
+		pr_err("Memory registrations not supported.\n");
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static const struct rtrs_rdma_dev_pd_ops dev_pd_ops = {
+	.init = rtrs_clt_ib_dev_init
+};
+
+static int __init rtrs_client_init(void)
+{
+	rtrs_rdma_dev_pd_init(0, &dev_pd);
+
+	rtrs_clt_dev_class = class_create(THIS_MODULE, "rtrs-client");
+	if (IS_ERR(rtrs_clt_dev_class)) {
+		pr_err("Failed to create rtrs-client dev class\n");
+		return PTR_ERR(rtrs_clt_dev_class);
+	}
+	rtrs_wq = alloc_workqueue("rtrs_client_wq", WQ_MEM_RECLAIM, 0);
+	if (!rtrs_wq) {
+		class_destroy(rtrs_clt_dev_class);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void __exit rtrs_client_exit(void)
+{
+	destroy_workqueue(rtrs_wq);
+	class_destroy(rtrs_clt_dev_class);
+	rtrs_rdma_dev_pd_deinit(&dev_pd);
+}
+
+module_init(rtrs_client_init);
+module_exit(rtrs_client_exit);
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
new file mode 100644
index 000000000000..167acd3c90fc
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
@@ -0,0 +1,252 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#ifndef RTRS_CLT_H
+#define RTRS_CLT_H
+
+#include <linux/device.h>
+#include "rtrs-pri.h"
+
+/**
+ * enum rtrs_clt_state - Client states.
+ */
+enum rtrs_clt_state {
+	RTRS_CLT_CONNECTING,
+	RTRS_CLT_CONNECTING_ERR,
+	RTRS_CLT_RECONNECTING,
+	RTRS_CLT_CONNECTED,
+	RTRS_CLT_CLOSING,
+	RTRS_CLT_CLOSED,
+	RTRS_CLT_DEAD,
+};
+
+enum rtrs_mp_policy {
+	MP_POLICY_RR,
+	MP_POLICY_MIN_INFLIGHT,
+};
+
+/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
+struct rtrs_clt_stats_reconnects {
+	int successful_cnt;
+	int fail_cnt;
+};
+
+/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
+struct rtrs_clt_stats_cpu_migr {
+	atomic_t from;
+	int to;
+};
+
+/* stats for Read and write operation.
+ * see Documentation/ABI/testing/sysfs-class-rtrs-client for details
+ */
+struct rtrs_clt_stats_rdma {
+	struct {
+		u64 cnt;
+		u64 size_total;
+	} dir[2];
+
+	u64 failover_cnt;
+};
+
+struct rtrs_clt_stats_pcpu {
+	struct rtrs_clt_stats_cpu_migr		cpu_migr;
+	struct rtrs_clt_stats_rdma		rdma;
+};
+
+struct rtrs_clt_stats {
+	struct kobject				kobj_stats;
+	struct rtrs_clt_stats_pcpu    __percpu	*pcpu_stats;
+	struct rtrs_clt_stats_reconnects	reconnects;
+	atomic_t				inflight;
+};
+
+struct rtrs_clt_con {
+	struct rtrs_con	c;
+	struct rtrs_iu		*rsp_ius;
+	u32			queue_size;
+	unsigned int		cpu;
+	atomic_t		io_cnt;
+	int			cm_err;
+};
+
+/**
+ * rtrs_permit - permits the memory allocation for future RDMA operation.
+ *		 Combine with irq pinning to keep IO on same CPU.
+ */
+struct rtrs_permit {
+	enum rtrs_clt_con_type con_type;
+	unsigned int cpu_id;
+	unsigned int mem_id;
+	unsigned int mem_off;
+};
+
+/**
+ * rtrs_clt_io_req - describes one inflight IO request
+ */
+struct rtrs_clt_io_req {
+	struct list_head        list;
+	struct rtrs_iu		*iu;
+	struct scatterlist	*sglist; /* list holding user data */
+	unsigned int		sg_cnt;
+	unsigned int		sg_size;
+	unsigned int		data_len;
+	unsigned int		usr_len;
+	void			*priv;
+	bool			in_use;
+	struct rtrs_clt_con	*con;
+	struct rtrs_sg_desc	*desc;
+	struct ib_sge		*sge;
+	struct rtrs_permit	*permit;
+	enum dma_data_direction dir;
+	void			(*conf)(void *priv, int errno);
+	unsigned long		start_jiffies;
+
+	struct ib_mr		*mr;
+	struct ib_cqe		inv_cqe;
+	struct completion	inv_comp;
+	int			inv_errno;
+	bool			need_inv_comp;
+	bool			need_inv;
+};
+
+struct rtrs_rbuf {
+	u64 addr;
+	u32 rkey;
+};
+
+struct rtrs_clt_sess {
+	struct rtrs_sess	s;
+	struct rtrs_clt	*clt;
+	wait_queue_head_t	state_wq;
+	enum rtrs_clt_state	state;
+	atomic_t		connected_cnt;
+	struct mutex		init_mutex;
+	struct rtrs_clt_io_req	*reqs;
+	struct delayed_work	reconnect_dwork;
+	struct work_struct	close_work;
+	unsigned int		reconnect_attempts;
+	bool			established;
+	struct rtrs_rbuf	*rbufs;
+	size_t			max_io_size;
+	u32			max_hdr_size;
+	u32			chunk_size;
+	size_t			queue_depth;
+	u32			max_pages_per_mr;
+	int			max_send_sge;
+	u32			flags;
+	struct kobject		kobj;
+	struct rtrs_clt_stats	*stats;
+	/* cache hca_port and hca_name to display in sysfs */
+	u8			hca_port;
+	char                    hca_name[IB_DEVICE_NAME_MAX];
+	struct list_head __percpu
+				*mp_skip_entry;
+};
+
+struct rtrs_clt {
+	struct list_head	paths_list; /* rcu protected list */
+	size_t			paths_num;
+	struct rtrs_clt_sess
+	__rcu * __percpu	*pcpu_path;
+	uuid_t			paths_uuid;
+	int			paths_up;
+	struct mutex		paths_mutex;
+	struct mutex		paths_ev_mutex;
+	char			sessname[NAME_MAX];
+	u16			port;
+	unsigned int		max_reconnect_attempts;
+	unsigned int		reconnect_delay_sec;
+	unsigned int		max_segments;
+	size_t			max_segment_size;
+	void			*permits;
+	unsigned long		*permits_map;
+	size_t			queue_depth;
+	size_t			max_io_size;
+	wait_queue_head_t	permits_wait;
+	size_t			pdu_sz;
+	void			*priv;
+	void			(*link_ev)(void *priv,
+					   enum rtrs_clt_link_ev ev);
+	struct device		dev;
+	struct kobject		*kobj_paths;
+	enum rtrs_mp_policy	mp_policy;
+};
+
+static inline struct rtrs_clt_con *to_clt_con(struct rtrs_con *c)
+{
+	return container_of(c, struct rtrs_clt_con, c);
+}
+
+static inline struct rtrs_clt_sess *to_clt_sess(struct rtrs_sess *s)
+{
+	return container_of(s, struct rtrs_clt_sess, s);
+}
+
+static inline int permit_size(struct rtrs_clt *clt)
+{
+	return sizeof(struct rtrs_permit) + clt->pdu_sz;
+}
+
+static inline struct rtrs_permit *get_permit(struct rtrs_clt *clt, int idx)
+{
+	return (struct rtrs_permit *)(clt->permits + permit_size(clt) * idx);
+}
+
+int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess);
+int rtrs_clt_disconnect_from_sysfs(struct rtrs_clt_sess *sess);
+int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,
+				     struct rtrs_addr *addr);
+int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess,
+				     const struct attribute *sysfs_self);
+
+void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value);
+int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt);
+void free_sess(struct rtrs_clt_sess *sess);
+
+/* rtrs-clt-stats.c */
+
+int rtrs_clt_init_stats(struct rtrs_clt_stats *stats);
+
+void rtrs_clt_inc_failover_cnt(struct rtrs_clt_stats *s);
+
+void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con);
+void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir);
+
+int rtrs_clt_reset_rdma_lat_distr_stats(struct rtrs_clt_stats *stats,
+					 bool enable);
+ssize_t rtrs_clt_stats_rdma_lat_distr_to_str(struct rtrs_clt_stats *stats,
+					      char *page, size_t len);
+int rtrs_clt_reset_cpu_migr_stats(struct rtrs_clt_stats *stats, bool enable);
+int rtrs_clt_stats_migration_cnt_to_str(struct rtrs_clt_stats *stats, char *buf,
+					 size_t len);
+int rtrs_clt_reset_reconnects_stat(struct rtrs_clt_stats *stats, bool enable);
+int rtrs_clt_stats_reconnects_to_str(struct rtrs_clt_stats *stats, char *buf,
+				      size_t len);
+int rtrs_clt_reset_wc_comp_stats(struct rtrs_clt_stats *stats, bool enable);
+int rtrs_clt_stats_wc_completion_to_str(struct rtrs_clt_stats *stats, char *buf,
+					 size_t len);
+int rtrs_clt_reset_rdma_stats(struct rtrs_clt_stats *stats, bool enable);
+ssize_t rtrs_clt_stats_rdma_to_str(struct rtrs_clt_stats *stats,
+				    char *page, size_t len);
+int rtrs_clt_reset_all_stats(struct rtrs_clt_stats *stats, bool enable);
+ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *stats,
+				 char *page, size_t len);
+
+/* rtrs-clt-sysfs.c */
+
+int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt);
+void rtrs_clt_destroy_sysfs_root_folders(struct rtrs_clt *clt);
+void rtrs_clt_destroy_sysfs_root_files(struct rtrs_clt *clt);
+
+int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess);
+void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,
+				  const struct attribute *sysfs_self);
+
+#endif /* RTRS_CLT_H */
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-log.h b/drivers/infiniband/ulp/rtrs/rtrs-log.h
new file mode 100644
index 000000000000..53c785b992f2
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-log.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#ifndef RTRS_LOG_H
+#define RTRS_LOG_H
+
+#define rtrs_log(fn, obj, fmt, ...)				\
+	fn("<%s>: " fmt, obj->sessname, ##__VA_ARGS__)
+
+#define rtrs_err(obj, fmt, ...)	\
+	rtrs_log(pr_err, obj, fmt, ##__VA_ARGS__)
+#define rtrs_err_rl(obj, fmt, ...)	\
+	rtrs_log(pr_err_ratelimited, obj, fmt, ##__VA_ARGS__)
+#define rtrs_wrn(obj, fmt, ...)	\
+	rtrs_log(pr_warn, obj, fmt, ##__VA_ARGS__)
+#define rtrs_wrn_rl(obj, fmt, ...) \
+	rtrs_log(pr_warn_ratelimited, obj, fmt, ##__VA_ARGS__)
+#define rtrs_info(obj, fmt, ...) \
+	rtrs_log(pr_info, obj, fmt, ##__VA_ARGS__)
+#define rtrs_info_rl(obj, fmt, ...) \
+	rtrs_log(pr_info_ratelimited, obj, fmt, ##__VA_ARGS__)
+
+#endif /* RTRS_LOG_H */
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-pri.h b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
new file mode 100644
index 000000000000..0a93c87ef92b
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-pri.h
@@ -0,0 +1,399 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#ifndef RTRS_PRI_H
+#define RTRS_PRI_H
+
+#include <linux/uuid.h>
+#include <rdma/rdma_cm.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib.h>
+
+#include "rtrs.h"
+
+#define RTRS_PROTO_VER_MAJOR 2
+#define RTRS_PROTO_VER_MINOR 0
+
+#define RTRS_PROTO_VER_STRING __stringify(RTRS_PROTO_VER_MAJOR) "." \
+			       __stringify(RTRS_PROTO_VER_MINOR)
+
+enum rtrs_imm_const {
+	MAX_IMM_TYPE_BITS = 4,
+	MAX_IMM_TYPE_MASK = ((1 << MAX_IMM_TYPE_BITS) - 1),
+	MAX_IMM_PAYL_BITS = 28,
+	MAX_IMM_PAYL_MASK = ((1 << MAX_IMM_PAYL_BITS) - 1),
+};
+
+enum rtrs_imm_type {
+	RTRS_IO_REQ_IMM       = 0, /* client to server */
+	RTRS_IO_RSP_IMM       = 1, /* server to client */
+	RTRS_IO_RSP_W_INV_IMM = 2, /* server to client */
+
+	RTRS_HB_MSG_IMM = 8, /* HB: HeartBeat */
+	RTRS_HB_ACK_IMM = 9,
+
+	RTRS_LAST_IMM,
+};
+
+enum {
+	SERVICE_CON_QUEUE_DEPTH = 512,
+
+	MAX_PATHS_NUM = 128,
+
+	/*
+	 * With the size of struct rtrs_permit allocated on the client, 4K
+	 * is the maximum number of rtrs_permits we can allocate. This number is
+	 * also used on the client to allocate the IU for the user connection
+	 * to receive the RDMA addresses from the server.
+	 */
+	MAX_SESS_QUEUE_DEPTH = 4096,
+
+	RTRS_HB_INTERVAL_MS = 5000,
+	RTRS_HB_MISSED_MAX = 5,
+
+	RTRS_MAGIC = 0x1BBD,
+	RTRS_PROTO_VER = (RTRS_PROTO_VER_MAJOR << 8) | RTRS_PROTO_VER_MINOR,
+};
+
+struct rtrs_ib_dev;
+
+struct rtrs_rdma_dev_pd_ops {
+	struct rtrs_ib_dev *(*alloc)(void);
+	void (*free)(struct rtrs_ib_dev *dev);
+	int (*init)(struct rtrs_ib_dev *dev);
+	void (*deinit)(struct rtrs_ib_dev *dev);
+};
+
+struct rtrs_rdma_dev_pd {
+	struct mutex		mutex;
+	struct list_head	list;
+	enum ib_pd_flags	pd_flags;
+	const struct rtrs_rdma_dev_pd_ops *ops;
+};
+
+struct rtrs_ib_dev {
+	struct ib_device	 *ib_dev;
+	struct ib_pd		 *ib_pd;
+	struct kref		 ref;
+	struct list_head	 entry;
+	struct rtrs_rdma_dev_pd *pool;
+};
+
+struct rtrs_con {
+	struct rtrs_sess	*sess;
+	struct ib_qp		*qp;
+	struct ib_cq		*cq;
+	struct rdma_cm_id	*cm_id;
+	unsigned int		cid;
+};
+
+struct rtrs_sess {
+	struct list_head	entry;
+	struct sockaddr_storage dst_addr;
+	struct sockaddr_storage src_addr;
+	char			sessname[NAME_MAX];
+	uuid_t			uuid;
+	struct rtrs_con	**con;
+	unsigned int		con_num;
+	unsigned int		recon_cnt;
+	struct rtrs_ib_dev	*dev;
+	int			dev_ref;
+	struct ib_cqe		*hb_cqe;
+	void			(*hb_err_handler)(struct rtrs_con *con);
+	struct workqueue_struct *hb_wq;
+	struct delayed_work	hb_dwork;
+	unsigned int		hb_interval_ms;
+	unsigned int		hb_missed_cnt;
+	unsigned int		hb_missed_max;
+};
+
+/* rtrs information unit */
+struct rtrs_iu {
+	struct list_head        list;
+	struct ib_cqe           cqe;
+	dma_addr_t              dma_addr;
+	void                    *buf;
+	size_t                  size;
+	enum dma_data_direction direction;
+};
+
+/**
+ * enum rtrs_msg_types - RTRS message types, see also rtrs/README
+ * @RTRS_MSG_INFO_REQ:		Client additional info request to the server
+ * @RTRS_MSG_INFO_RSP:		Server additional info response to the client
+ * @RTRS_MSG_WRITE:		Client writes data per RDMA to server
+ * @RTRS_MSG_READ:		Client requests data transfer from server
+ * @RTRS_MSG_RKEY_RSP:		Server refreshed rkey for rbuf
+ */
+enum rtrs_msg_types {
+	RTRS_MSG_INFO_REQ,
+	RTRS_MSG_INFO_RSP,
+	RTRS_MSG_WRITE,
+	RTRS_MSG_READ,
+	RTRS_MSG_RKEY_RSP,
+};
+
+/**
+ * enum rtrs_msg_flags - RTRS message flags.
+ * @RTRS_NEED_INVAL:	Send invalidation in response.
+ * @RTRS_MSG_NEW_RKEY_F: Send refreshed rkey in response.
+ */
+enum rtrs_msg_flags {
+	RTRS_MSG_NEED_INVAL_F = 1 << 0,
+	RTRS_MSG_NEW_RKEY_F = 1 << 1,
+};
+
+/**
+ * struct rtrs_sg_desc - RDMA-Buffer entry description
+ * @addr:	Address of RDMA destination buffer
+ * @key:	Authorization rkey to write to the buffer
+ * @len:	Size of the buffer
+ */
+struct rtrs_sg_desc {
+	__le64			addr;
+	__le32			key;
+	__le32			len;
+};
+
+/**
+ * struct rtrs_msg_conn_req - Client connection request to the server
+ * @magic:	   RTRS magic
+ * @version:	   RTRS protocol version
+ * @cid:	   Current connection id
+ * @cid_num:	   Number of connections per session
+ * @recon_cnt:	   Reconnections counter
+ * @sess_uuid:	   UUID of a session (path)
+ * @paths_uuid:	   UUID of a group of sessions (paths)
+ *
+ * NOTE: max size 56 bytes, see man rdma_connect().
+ */
+struct rtrs_msg_conn_req {
+	/* Is set to 0 by cma.c in case of AF_IB, do not touch that.
+	 * see https://www.spinics.net/lists/linux-rdma/msg22397.html
+	 */
+	u8		__cma_version;
+	/* On sender side that should be set to 0, or cma_save_ip_info()
+	 * extract garbage and will fail.
+	 */
+	u8		__ip_version;
+	__le16		magic;
+	__le16		version;
+	__le16		cid;
+	__le16		cid_num;
+	__le16		recon_cnt;
+	uuid_t		sess_uuid;
+	uuid_t		paths_uuid;
+	u8		reserved[12];
+};
+
+/**
+ * struct rtrs_msg_conn_rsp - Server connection response to the client
+ * @magic:	   RTRS magic
+ * @version:	   RTRS protocol version
+ * @errno:	   If rdma_accept() then 0, if rdma_reject() indicates error
+ * @queue_depth:   max inflight messages (queue-depth) in this session
+ * @max_io_size:   max io size server supports
+ * @max_hdr_size:  max msg header size server supports
+ *
+ * NOTE: size is 56 bytes, max possible is 136 bytes, see man rdma_accept().
+ */
+struct rtrs_msg_conn_rsp {
+	__le16		magic;
+	__le16		version;
+	__le16		errno;
+	__le16		queue_depth;
+	__le32		max_io_size;
+	__le32		max_hdr_size;
+	__le32		flags;
+	u8		reserved[36];
+};
+
+/**
+ * struct rtrs_msg_info_req
+ * @type:		@RTRS_MSG_INFO_REQ
+ * @sessname:		Session name chosen by client
+ */
+struct rtrs_msg_info_req {
+	__le16		type;
+	u8		sessname[NAME_MAX];
+	u8		reserved[15];
+};
+
+/**
+ * struct rtrs_msg_info_rsp
+ * @type:		@RTRS_MSG_INFO_RSP
+ * @sg_cnt:		Number of @desc entries
+ * @desc:		RDMA buffers where the client can write to server
+ */
+struct rtrs_msg_info_rsp {
+	__le16		type;
+	__le16          sg_cnt;
+	u8              reserved[4];
+	struct rtrs_sg_desc desc[];
+};
+
+/**
+ * struct rtrs_msg_rkey_rsp
+ * @type:		@RTRS_MSG_RKEY_RSP
+ * @buf_id:		RDMA buf_id of the new rkey
+ * @rkey:		new remote key for RDMA buffers id from server
+ */
+struct rtrs_msg_rkey_rsp {
+	__le16		type;
+	__le16          buf_id;
+	__le32		rkey;
+};
+
+/**
+ * struct rtrs_msg_rdma_read - RDMA data transfer request from client
+ * @type:		always @RTRS_MSG_READ
+ * @usr_len:		length of user payload
+ * @sg_cnt:		number of @desc entries
+ * @desc:		RDMA buffers where the server can write the result to
+ */
+struct rtrs_msg_rdma_read {
+	__le16			type;
+	__le16			usr_len;
+	__le16			flags;
+	__le16			sg_cnt;
+	struct rtrs_sg_desc    desc[];
+};
+
+/**
+ * struct_msg_rdma_write - Message transferred to server with RDMA-Write
+ * @type:		always @RTRS_MSG_WRITE
+ * @usr_len:		length of user payload
+ */
+struct rtrs_msg_rdma_write {
+	__le16			type;
+	__le16			usr_len;
+};
+
+/**
+ * struct_msg_rdma_hdr - header for read or write request
+ * @type:		@RTRS_MSG_WRITE | @RTRS_MSG_READ
+ */
+struct rtrs_msg_rdma_hdr {
+	__le16			type;
+};
+
+/* rtrs.c */
+
+struct rtrs_iu *rtrs_iu_alloc(u32 queue_size, size_t size, gfp_t t,
+			      struct ib_device *dev, enum dma_data_direction,
+			      void (*done)(struct ib_cq *cq, struct ib_wc *wc));
+void rtrs_iu_free(struct rtrs_iu *iu, enum dma_data_direction dir,
+		  struct ib_device *dev, u32 queue_size);
+int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu);
+int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,
+		      struct ib_send_wr *head);
+int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu,
+				struct ib_sge *sge, unsigned int num_sge,
+				u32 rkey, u64 rdma_addr, u32 imm_data,
+				enum ib_send_flags flags,
+				struct ib_send_wr *head);
+
+int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe);
+int rtrs_post_rdma_write_imm_empty(struct rtrs_con *con, struct ib_cqe *cqe,
+				   u32 imm_data, enum ib_send_flags flags,
+				   struct ib_send_wr *head);
+
+int rtrs_cq_qp_create(struct rtrs_sess *rtrs_sess, struct rtrs_con *con,
+		      u32 max_send_sge, int cq_vector, u16 cq_size,
+		      u16 wr_queue_size, enum ib_poll_context poll_ctx);
+void rtrs_cq_qp_destroy(struct rtrs_con *con);
+
+void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe,
+		  unsigned int interval_ms, unsigned int missed_max,
+		  void (*err_handler)(struct rtrs_con *con),
+		  struct workqueue_struct *wq);
+void rtrs_start_hb(struct rtrs_sess *sess);
+void rtrs_stop_hb(struct rtrs_sess *sess);
+void rtrs_send_hb_ack(struct rtrs_sess *sess);
+
+void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
+			   struct rtrs_rdma_dev_pd *pool);
+void rtrs_rdma_dev_pd_deinit(struct rtrs_rdma_dev_pd *pool);
+
+struct rtrs_ib_dev *rtrs_ib_dev_find_or_add(struct ib_device *ib_dev,
+					    struct rtrs_rdma_dev_pd *pool);
+int rtrs_ib_dev_put(struct rtrs_ib_dev *dev);
+
+static inline u32 rtrs_to_imm(u32 type, u32 payload)
+{
+	BUILD_BUG_ON(MAX_IMM_PAYL_BITS + MAX_IMM_TYPE_BITS != 32);
+	BUILD_BUG_ON(RTRS_LAST_IMM > (1<<MAX_IMM_TYPE_BITS));
+	return ((type & MAX_IMM_TYPE_MASK) << MAX_IMM_PAYL_BITS) |
+		(payload & MAX_IMM_PAYL_MASK);
+}
+
+static inline void rtrs_from_imm(u32 imm, u32 *type, u32 *payload)
+{
+	*payload = imm & MAX_IMM_PAYL_MASK;
+	*type = imm >> MAX_IMM_PAYL_BITS;
+}
+
+static inline u32 rtrs_to_io_req_imm(u32 addr)
+{
+	return rtrs_to_imm(RTRS_IO_REQ_IMM, addr);
+}
+
+static inline u32 rtrs_to_io_rsp_imm(u32 msg_id, int errno, bool w_inval)
+{
+	enum rtrs_imm_type type;
+	u32 payload;
+
+	/* 9 bits for errno, 19 bits for msg_id */
+	payload = (abs(errno) & 0x1ff) << 19 | (msg_id & 0x7ffff);
+	type = w_inval ? RTRS_IO_RSP_W_INV_IMM : RTRS_IO_RSP_IMM;
+
+	return rtrs_to_imm(type, payload);
+}
+
+static inline void rtrs_from_io_rsp_imm(u32 payload, u32 *msg_id, int *errno)
+{
+	/* 9 bits for errno, 19 bits for msg_id */
+	*msg_id = payload & 0x7ffff;
+	*errno = -(int)((payload >> 19) & 0x1ff);
+}
+
+#define STAT_STORE_FUNC(type, set_value, reset)				\
+static ssize_t set_value##_store(struct kobject *kobj,			\
+			     struct kobj_attribute *attr,		\
+			     const char *buf, size_t count)		\
+{									\
+	int ret = -EINVAL;						\
+	type *stats = container_of(kobj, type, kobj_stats);		\
+									\
+	if (sysfs_streq(buf, "1"))					\
+		ret = reset(stats, true);			\
+	else if (sysfs_streq(buf, "0"))					\
+		ret = reset(stats, false);			\
+	if (ret)							\
+		return ret;						\
+									\
+	return count;							\
+}
+
+#define STAT_SHOW_FUNC(type, get_value, print)				\
+static ssize_t get_value##_show(struct kobject *kobj,			\
+			   struct kobj_attribute *attr,			\
+			   char *page)					\
+{									\
+	type *stats = container_of(kobj, type, kobj_stats);		\
+									\
+	return print(stats, page, PAGE_SIZE);			\
+}
+
+#define STAT_ATTR(type, stat, print, reset)				\
+STAT_STORE_FUNC(type, stat, reset)					\
+STAT_SHOW_FUNC(type, stat, print)					\
+static struct kobj_attribute stat##_attr = __ATTR_RW(stat)
+
+#endif /* RTRS_PRI_H */
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-stats.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-stats.c
new file mode 100644
index 000000000000..e102b1368d0c
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-stats.c
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include "rtrs-srv.h"
+
+int rtrs_srv_reset_rdma_stats(struct rtrs_srv_stats *stats, bool enable)
+{
+	if (enable) {
+		struct rtrs_srv_stats_rdma_stats *r = &stats->rdma_stats;
+
+		memset(r, 0, sizeof(*r));
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+ssize_t rtrs_srv_stats_rdma_to_str(struct rtrs_srv_stats *stats,
+				    char *page, size_t len)
+{
+	struct rtrs_srv_stats_rdma_stats *r = &stats->rdma_stats;
+	struct rtrs_srv_sess *sess = stats->sess;
+
+	return scnprintf(page, len, "%lld %lld %lld %lld %u\n",
+			 (s64)atomic64_read(&r->dir[READ].cnt),
+			 (s64)atomic64_read(&r->dir[READ].size_total),
+			 (s64)atomic64_read(&r->dir[WRITE].cnt),
+			 (s64)atomic64_read(&r->dir[WRITE].size_total),
+			 atomic_read(&sess->ids_inflight));
+}
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
new file mode 100644
index 000000000000..3d7877534bcc
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include "rtrs-pri.h"
+#include "rtrs-srv.h"
+#include "rtrs-log.h"
+
+static void rtrs_srv_release(struct kobject *kobj)
+{
+	struct rtrs_srv_sess *sess;
+
+	sess = container_of(kobj, struct rtrs_srv_sess, kobj);
+	kfree(sess);
+}
+
+static struct kobj_type ktype = {
+	.sysfs_ops	= &kobj_sysfs_ops,
+	.release	= rtrs_srv_release,
+};
+
+static ssize_t rtrs_srv_disconnect_show(struct kobject *kobj,
+					 struct kobj_attribute *attr,
+					 char *page)
+{
+	return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
+			 attr->attr.name);
+}
+
+static ssize_t rtrs_srv_disconnect_store(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct rtrs_srv_sess *sess;
+	struct rtrs_sess *s;
+	char str[MAXHOSTNAMELEN];
+
+	sess = container_of(kobj, struct rtrs_srv_sess, kobj);
+	s = &sess->s;
+	if (!sysfs_streq(buf, "1")) {
+		rtrs_err(s, "%s: invalid value: '%s'\n",
+			  attr->attr.name, buf);
+		return -EINVAL;
+	}
+
+	sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, str, sizeof(str));
+
+	rtrs_info(s, "disconnect for path %s requested\n", str);
+	close_sess(sess);
+
+	return count;
+}
+
+static struct kobj_attribute rtrs_srv_disconnect_attr =
+	__ATTR(disconnect, 0644,
+	       rtrs_srv_disconnect_show, rtrs_srv_disconnect_store);
+
+static ssize_t rtrs_srv_hca_port_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_srv_sess *sess;
+	struct rtrs_con *usr_con;
+
+	sess = container_of(kobj, typeof(*sess), kobj);
+	usr_con = sess->s.con[0];
+
+	return scnprintf(page, PAGE_SIZE, "%u\n",
+			 usr_con->cm_id->port_num);
+}
+
+static struct kobj_attribute rtrs_srv_hca_port_attr =
+	__ATTR(hca_port, 0444, rtrs_srv_hca_port_show, NULL);
+
+static ssize_t rtrs_srv_hca_name_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_srv_sess *sess;
+
+	sess = container_of(kobj, struct rtrs_srv_sess, kobj);
+
+	return scnprintf(page, PAGE_SIZE, "%s\n",
+			 sess->s.dev->ib_dev->name);
+}
+
+static struct kobj_attribute rtrs_srv_hca_name_attr =
+	__ATTR(hca_name, 0444, rtrs_srv_hca_name_show, NULL);
+
+static ssize_t rtrs_srv_src_addr_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_srv_sess *sess;
+	int cnt;
+
+	sess = container_of(kobj, struct rtrs_srv_sess, kobj);
+	cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
+			      page, PAGE_SIZE);
+	return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
+}
+
+static struct kobj_attribute rtrs_srv_src_addr_attr =
+	__ATTR(src_addr, 0444, rtrs_srv_src_addr_show, NULL);
+
+static ssize_t rtrs_srv_dst_addr_show(struct kobject *kobj,
+				       struct kobj_attribute *attr,
+				       char *page)
+{
+	struct rtrs_srv_sess *sess;
+	int cnt;
+
+	sess = container_of(kobj, struct rtrs_srv_sess, kobj);
+	cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
+			      page, PAGE_SIZE);
+	return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
+}
+
+static struct kobj_attribute rtrs_srv_dst_addr_attr =
+	__ATTR(dst_addr, 0444, rtrs_srv_dst_addr_show, NULL);
+
+static struct attribute *rtrs_srv_sess_attrs[] = {
+	&rtrs_srv_hca_name_attr.attr,
+	&rtrs_srv_hca_port_attr.attr,
+	&rtrs_srv_src_addr_attr.attr,
+	&rtrs_srv_dst_addr_attr.attr,
+	&rtrs_srv_disconnect_attr.attr,
+	NULL,
+};
+
+static struct attribute_group rtrs_srv_sess_attr_group = {
+	.attrs = rtrs_srv_sess_attrs,
+};
+
+STAT_ATTR(struct rtrs_srv_stats, rdma,
+	  rtrs_srv_stats_rdma_to_str,
+	  rtrs_srv_reset_rdma_stats);
+
+static struct attribute *rtrs_srv_stats_attrs[] = {
+	&rdma_attr.attr,
+	NULL,
+};
+
+static struct attribute_group rtrs_srv_stats_attr_group = {
+	.attrs = rtrs_srv_stats_attrs,
+};
+
+static void rtrs_srv_dev_release(struct device *dev)
+{
+	struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev);
+
+	kfree(srv);
+}
+
+static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	int err = 0;
+
+	mutex_lock(&srv->paths_mutex);
+	if (srv->dev_ref++) {
+		/*
+		 * Device needs to be registered only on the first session
+		 */
+		goto unlock;
+	}
+	srv->dev.class = rtrs_dev_class;
+	srv->dev.release = rtrs_srv_dev_release;
+	err = dev_set_name(&srv->dev, "%s", sess->s.sessname);
+	if (err)
+		goto unlock;
+
+	/*
+	 * Suppress user space notification until
+	 * sysfs files are created
+	 */
+	dev_set_uevent_suppress(&srv->dev, true);
+	err = device_register(&srv->dev);
+	if (err) {
+		pr_err("device_register(): %d\n", err);
+		goto put;
+	}
+	srv->kobj_paths = kobject_create_and_add("paths", &srv->dev.kobj);
+	if (!srv->kobj_paths) {
+		err = -ENOMEM;
+		pr_err("kobject_create_and_add(): %d\n", err);
+		device_unregister(&srv->dev);
+		goto unlock;
+	}
+	dev_set_uevent_suppress(&srv->dev, false);
+	kobject_uevent(&srv->dev.kobj, KOBJ_ADD);
+	goto unlock;
+
+put:
+	put_device(&srv->dev);
+unlock:
+	mutex_unlock(&srv->paths_mutex);
+
+	return err;
+}
+
+static void
+rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+
+	mutex_lock(&srv->paths_mutex);
+	if (!--srv->dev_ref) {
+		kobject_del(srv->kobj_paths);
+		kobject_put(srv->kobj_paths);
+		mutex_unlock(&srv->paths_mutex);
+		device_unregister(&srv->dev);
+	} else {
+		mutex_unlock(&srv->paths_mutex);
+	}
+}
+
+static void rtrs_srv_sess_stats_release(struct kobject *kobj)
+{
+	struct rtrs_srv_stats *stats;
+
+	stats = container_of(kobj, struct rtrs_srv_stats, kobj_stats);
+
+	kfree(stats);
+}
+
+static struct kobj_type ktype_stats = {
+	.sysfs_ops = &kobj_sysfs_ops,
+	.release = rtrs_srv_sess_stats_release,
+};
+
+static int rtrs_srv_create_stats_files(struct rtrs_srv_sess *sess)
+{
+	int err;
+	struct rtrs_sess *s = &sess->s;
+
+	err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats,
+				   &sess->kobj, "stats");
+	if (err) {
+		rtrs_err(s, "kobject_init_and_add(): %d\n", err);
+		return err;
+	}
+	err = sysfs_create_group(&sess->stats->kobj_stats,
+				 &rtrs_srv_stats_attr_group);
+	if (err) {
+		rtrs_err(s, "sysfs_create_group(): %d\n", err);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	kobject_del(&sess->stats->kobj_stats);
+	kobject_put(&sess->stats->kobj_stats);
+
+	return err;
+}
+
+int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_sess *s = &sess->s;
+	char str[NAME_MAX];
+	int err, cnt;
+
+	cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
+			      str, sizeof(str));
+	cnt += scnprintf(str + cnt, sizeof(str) - cnt, "@");
+	sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
+			str + cnt, sizeof(str) - cnt);
+
+	err = rtrs_srv_create_once_sysfs_root_folders(sess);
+	if (err)
+		return err;
+
+	err = kobject_init_and_add(&sess->kobj, &ktype, srv->kobj_paths,
+				   "%s", str);
+	if (err) {
+		rtrs_err(s, "kobject_init_and_add(): %d\n", err);
+		goto destroy_root;
+	}
+	err = sysfs_create_group(&sess->kobj, &rtrs_srv_sess_attr_group);
+	if (err) {
+		rtrs_err(s, "sysfs_create_group(): %d\n", err);
+		goto put_kobj;
+	}
+	err = rtrs_srv_create_stats_files(sess);
+	if (err)
+		goto remove_group;
+
+	return 0;
+
+remove_group:
+	sysfs_remove_group(&sess->kobj, &rtrs_srv_sess_attr_group);
+put_kobj:
+	kobject_del(&sess->kobj);
+	kobject_put(&sess->kobj);
+destroy_root:
+	rtrs_srv_destroy_once_sysfs_root_folders(sess);
+
+	return err;
+}
+
+void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess)
+{
+	if (sess->kobj.state_in_sysfs) {
+		kobject_del(&sess->stats->kobj_stats);
+		kobject_put(&sess->stats->kobj_stats);
+		kobject_del(&sess->kobj);
+		kobject_put(&sess->kobj);
+
+		rtrs_srv_destroy_once_sysfs_root_folders(sess);
+	}
+}
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
new file mode 100644
index 000000000000..0d9241f5d9e6
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -0,0 +1,2178 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include <linux/module.h>
+#include <linux/mempool.h>
+
+#include "rtrs-srv.h"
+#include "rtrs-log.h"
+#include <rdma/ib_cm.h>
+
+MODULE_DESCRIPTION("RDMA Transport Server");
+MODULE_LICENSE("GPL");
+
+/* Must be power of 2, see mask from mr->page_size in ib_sg_to_pages() */
+#define DEFAULT_MAX_CHUNK_SIZE (128 << 10)
+#define DEFAULT_SESS_QUEUE_DEPTH 512
+#define MAX_HDR_SIZE PAGE_SIZE
+
+/* We guarantee to serve 10 paths at least */
+#define CHUNK_POOL_SZ 10
+
+static struct rtrs_rdma_dev_pd dev_pd;
+static mempool_t *chunk_pool;
+struct class *rtrs_dev_class;
+
+static int __read_mostly max_chunk_size = DEFAULT_MAX_CHUNK_SIZE;
+static int __read_mostly sess_queue_depth = DEFAULT_SESS_QUEUE_DEPTH;
+
+static bool always_invalidate = true;
+module_param(always_invalidate, bool, 0444);
+MODULE_PARM_DESC(always_invalidate,
+		 "Invalidate memory registration for contiguous memory regions before accessing.");
+
+module_param_named(max_chunk_size, max_chunk_size, int, 0444);
+MODULE_PARM_DESC(max_chunk_size,
+		 "Max size for each IO request, when change the unit is in byte (default: "
+		 __stringify(DEFAULT_MAX_CHUNK_SIZE) "KB)");
+
+module_param_named(sess_queue_depth, sess_queue_depth, int, 0444);
+MODULE_PARM_DESC(sess_queue_depth,
+		 "Number of buffers for pending I/O requests to allocate per session. Maximum: "
+		 __stringify(MAX_SESS_QUEUE_DEPTH) " (default: "
+		 __stringify(DEFAULT_SESS_QUEUE_DEPTH) ")");
+
+static cpumask_t cq_affinity_mask = { CPU_BITS_ALL };
+
+static struct workqueue_struct *rtrs_wq;
+
+static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c)
+{
+	return container_of(c, struct rtrs_srv_con, c);
+}
+
+static inline struct rtrs_srv_sess *to_srv_sess(struct rtrs_sess *s)
+{
+	return container_of(s, struct rtrs_srv_sess, s);
+}
+
+static bool __rtrs_srv_change_state(struct rtrs_srv_sess *sess,
+				     enum rtrs_srv_state new_state)
+{
+	enum rtrs_srv_state old_state;
+	bool changed = false;
+
+	lockdep_assert_held(&sess->state_lock);
+	old_state = sess->state;
+	switch (new_state) {
+	case RTRS_SRV_CONNECTED:
+		switch (old_state) {
+		case RTRS_SRV_CONNECTING:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_SRV_CLOSING:
+		switch (old_state) {
+		case RTRS_SRV_CONNECTING:
+		case RTRS_SRV_CONNECTED:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	case RTRS_SRV_CLOSED:
+		switch (old_state) {
+		case RTRS_SRV_CLOSING:
+			changed = true;
+			fallthrough;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+	if (changed)
+		sess->state = new_state;
+
+	return changed;
+}
+
+static bool rtrs_srv_change_state_get_old(struct rtrs_srv_sess *sess,
+					   enum rtrs_srv_state new_state,
+					   enum rtrs_srv_state *old_state)
+{
+	bool changed;
+
+	spin_lock_irq(&sess->state_lock);
+	*old_state = sess->state;
+	changed = __rtrs_srv_change_state(sess, new_state);
+	spin_unlock_irq(&sess->state_lock);
+
+	return changed;
+}
+
+static bool rtrs_srv_change_state(struct rtrs_srv_sess *sess,
+				   enum rtrs_srv_state new_state)
+{
+	enum rtrs_srv_state old_state;
+
+	return rtrs_srv_change_state_get_old(sess, new_state, &old_state);
+}
+
+static void free_id(struct rtrs_srv_op *id)
+{
+	if (!id)
+		return;
+	kfree(id);
+}
+
+static void rtrs_srv_free_ops_ids(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	int i;
+
+	WARN_ON(atomic_read(&sess->ids_inflight));
+	if (sess->ops_ids) {
+		for (i = 0; i < srv->queue_depth; i++)
+			free_id(sess->ops_ids[i]);
+		kfree(sess->ops_ids);
+		sess->ops_ids = NULL;
+	}
+}
+
+static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc);
+
+static struct ib_cqe io_comp_cqe = {
+	.done = rtrs_srv_rdma_done
+};
+
+static int rtrs_srv_alloc_ops_ids(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_srv_op *id;
+	int i;
+
+	sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids),
+				GFP_KERNEL);
+	if (!sess->ops_ids)
+		goto err;
+
+	for (i = 0; i < srv->queue_depth; ++i) {
+		id = kzalloc(sizeof(*id), GFP_KERNEL);
+		if (!id)
+			goto err;
+
+		sess->ops_ids[i] = id;
+	}
+	init_waitqueue_head(&sess->ids_waitq);
+	atomic_set(&sess->ids_inflight, 0);
+
+	return 0;
+
+err:
+	rtrs_srv_free_ops_ids(sess);
+	return -ENOMEM;
+}
+
+static inline void rtrs_srv_get_ops_ids(struct rtrs_srv_sess *sess)
+{
+	atomic_inc(&sess->ids_inflight);
+}
+
+static inline void rtrs_srv_put_ops_ids(struct rtrs_srv_sess *sess)
+{
+	if (atomic_dec_and_test(&sess->ids_inflight))
+		wake_up(&sess->ids_waitq);
+}
+
+static void rtrs_srv_wait_ops_ids(struct rtrs_srv_sess *sess)
+{
+	wait_event(sess->ids_waitq, !atomic_read(&sess->ids_inflight));
+}
+
+
+static void rtrs_srv_reg_mr_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_srv_con *con = cq->cq_context;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(s, "REG MR failed: %s\n",
+			  ib_wc_status_msg(wc->status));
+		close_sess(sess);
+		return;
+	}
+}
+
+static struct ib_cqe local_reg_cqe = {
+	.done = rtrs_srv_reg_mr_done
+};
+
+static int rdma_write_sg(struct rtrs_srv_op *id)
+{
+	struct rtrs_sess *s = id->con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	dma_addr_t dma_addr = sess->dma_addr[id->msg_id];
+	struct rtrs_srv_mr *srv_mr;
+	struct rtrs_srv *srv = sess->srv;
+	struct ib_send_wr inv_wr, imm_wr;
+	struct ib_rdma_wr *wr = NULL;
+	enum ib_send_flags flags;
+	size_t sg_cnt;
+	int err, offset;
+	bool need_inval;
+	u32 rkey = 0;
+	struct ib_reg_wr rwr;
+	struct ib_sge *plist;
+	struct ib_sge list;
+
+	sg_cnt = le16_to_cpu(id->rd_msg->sg_cnt);
+	need_inval = le16_to_cpu(id->rd_msg->flags) & RTRS_MSG_NEED_INVAL_F;
+	if (unlikely(sg_cnt != 1))
+		return -EINVAL;
+
+	offset = 0;
+
+	wr		= &id->tx_wr;
+	plist		= &id->tx_sg;
+	plist->addr	= dma_addr + offset;
+	plist->length	= le32_to_cpu(id->rd_msg->desc[0].len);
+
+	/* WR will fail with length error
+	 * if this is 0
+	 */
+	if (unlikely(plist->length == 0)) {
+		rtrs_err(s, "Invalid RDMA-Write sg list length 0\n");
+		return -EINVAL;
+	}
+
+	plist->lkey = sess->s.dev->ib_pd->local_dma_lkey;
+	offset += plist->length;
+
+	wr->wr.sg_list	= plist;
+	wr->wr.num_sge	= 1;
+	wr->remote_addr	= le64_to_cpu(id->rd_msg->desc[0].addr);
+	wr->rkey	= le32_to_cpu(id->rd_msg->desc[0].key);
+	if (rkey == 0)
+		rkey = wr->rkey;
+	else
+		/* Only one key is actually used */
+		WARN_ON_ONCE(rkey != wr->rkey);
+
+	wr->wr.opcode = IB_WR_RDMA_WRITE;
+	wr->wr.ex.imm_data = 0;
+	wr->wr.send_flags  = 0;
+
+	if (need_inval && always_invalidate) {
+		wr->wr.next = &rwr.wr;
+		rwr.wr.next = &inv_wr;
+		inv_wr.next = &imm_wr;
+	} else if (always_invalidate) {
+		wr->wr.next = &rwr.wr;
+		rwr.wr.next = &imm_wr;
+	} else if (need_inval) {
+		wr->wr.next = &inv_wr;
+		inv_wr.next = &imm_wr;
+	} else {
+		wr->wr.next = &imm_wr;
+	}
+	/*
+	 * From time to time we have to post signaled sends,
+	 * or send queue will fill up and only QP reset can help.
+	 */
+	flags = (atomic_inc_return(&id->con->wr_cnt) % srv->queue_depth) ?
+		0 : IB_SEND_SIGNALED;
+
+	if (need_inval) {
+		inv_wr.sg_list = NULL;
+		inv_wr.num_sge = 0;
+		inv_wr.opcode = IB_WR_SEND_WITH_INV;
+		inv_wr.send_flags = 0;
+		inv_wr.ex.invalidate_rkey = rkey;
+	}
+
+	imm_wr.next = NULL;
+	if (always_invalidate) {
+		struct rtrs_msg_rkey_rsp *msg;
+
+		srv_mr = &sess->mrs[id->msg_id];
+		rwr.wr.opcode = IB_WR_REG_MR;
+		rwr.wr.num_sge = 0;
+		rwr.mr = srv_mr->mr;
+		rwr.wr.send_flags = 0;
+		rwr.key = srv_mr->mr->rkey;
+		rwr.access = (IB_ACCESS_LOCAL_WRITE |
+			      IB_ACCESS_REMOTE_WRITE);
+		msg = srv_mr->iu->buf;
+		msg->buf_id = cpu_to_le16(id->msg_id);
+		msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP);
+		msg->rkey = cpu_to_le32(srv_mr->mr->rkey);
+
+		list.addr   = srv_mr->iu->dma_addr;
+		list.length = sizeof(*msg);
+		list.lkey   = sess->s.dev->ib_pd->local_dma_lkey;
+		imm_wr.sg_list = &list;
+		imm_wr.num_sge = 1;
+		imm_wr.opcode = IB_WR_SEND_WITH_IMM;
+		ib_dma_sync_single_for_device(sess->s.dev->ib_dev,
+					      srv_mr->iu->dma_addr,
+					      srv_mr->iu->size, DMA_TO_DEVICE);
+	} else {
+		imm_wr.sg_list = NULL;
+		imm_wr.num_sge = 0;
+		imm_wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM;
+	}
+	imm_wr.send_flags = flags;
+	imm_wr.ex.imm_data = cpu_to_be32(rtrs_to_io_rsp_imm(id->msg_id,
+							     0, need_inval));
+
+	imm_wr.wr_cqe   = &io_comp_cqe;
+	ib_dma_sync_single_for_device(sess->s.dev->ib_dev, dma_addr,
+				      offset, DMA_BIDIRECTIONAL);
+
+	err = ib_post_send(id->con->c.qp, &id->tx_wr.wr, NULL);
+	if (unlikely(err))
+		rtrs_err(s,
+			  "Posting RDMA-Write-Request to QP failed, err: %d\n",
+			  err);
+
+	return err;
+}
+
+/**
+ * send_io_resp_imm() - respond to client with empty IMM on failed READ/WRITE
+ *                      requests or on successful WRITE request.
+ * @con:	the connection to send back result
+ * @id:		the id associated with the IO
+ * @errno:	the error number of the IO.
+ *
+ * Return 0 on success, errno otherwise.
+ */
+static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id,
+			    int errno)
+{
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct ib_send_wr inv_wr, imm_wr, *wr = NULL;
+	struct ib_reg_wr rwr;
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_srv_mr *srv_mr;
+	bool need_inval = false;
+	enum ib_send_flags flags;
+	u32 imm;
+	int err;
+
+	if (id->dir == READ) {
+		struct rtrs_msg_rdma_read *rd_msg = id->rd_msg;
+		size_t sg_cnt;
+
+		need_inval = le16_to_cpu(rd_msg->flags) &
+				RTRS_MSG_NEED_INVAL_F;
+		sg_cnt = le16_to_cpu(rd_msg->sg_cnt);
+
+		if (need_inval) {
+			if (likely(sg_cnt)) {
+				inv_wr.sg_list = NULL;
+				inv_wr.num_sge = 0;
+				inv_wr.opcode = IB_WR_SEND_WITH_INV;
+				inv_wr.send_flags = 0;
+				/* Only one key is actually used */
+				inv_wr.ex.invalidate_rkey =
+					le32_to_cpu(rd_msg->desc[0].key);
+			} else {
+				WARN_ON_ONCE(1);
+				need_inval = false;
+			}
+		}
+	}
+
+	if (need_inval && always_invalidate) {
+		wr = &inv_wr;
+		inv_wr.next = &rwr.wr;
+		rwr.wr.next = &imm_wr;
+	} else if (always_invalidate) {
+		wr = &rwr.wr;
+		rwr.wr.next = &imm_wr;
+	} else if (need_inval) {
+		wr = &inv_wr;
+		inv_wr.next = &imm_wr;
+	} else {
+		wr = &imm_wr;
+	}
+	/*
+	 * From time to time we have to post signalled sends,
+	 * or send queue will fill up and only QP reset can help.
+	 */
+	flags = (atomic_inc_return(&con->wr_cnt) % srv->queue_depth) ?
+		0 : IB_SEND_SIGNALED;
+	imm = rtrs_to_io_rsp_imm(id->msg_id, errno, need_inval);
+	imm_wr.next = NULL;
+	if (always_invalidate) {
+		struct ib_sge list;
+		struct rtrs_msg_rkey_rsp *msg;
+
+		srv_mr = &sess->mrs[id->msg_id];
+		rwr.wr.next = &imm_wr;
+		rwr.wr.opcode = IB_WR_REG_MR;
+		rwr.wr.num_sge = 0;
+		rwr.wr.send_flags = 0;
+		rwr.mr = srv_mr->mr;
+		rwr.key = srv_mr->mr->rkey;
+		rwr.access = (IB_ACCESS_LOCAL_WRITE |
+			      IB_ACCESS_REMOTE_WRITE);
+		msg = srv_mr->iu->buf;
+		msg->buf_id = cpu_to_le16(id->msg_id);
+		msg->type = cpu_to_le16(RTRS_MSG_RKEY_RSP);
+		msg->rkey = cpu_to_le32(srv_mr->mr->rkey);
+
+		list.addr   = srv_mr->iu->dma_addr;
+		list.length = sizeof(*msg);
+		list.lkey   = sess->s.dev->ib_pd->local_dma_lkey;
+		imm_wr.sg_list = &list;
+		imm_wr.num_sge = 1;
+		imm_wr.opcode = IB_WR_SEND_WITH_IMM;
+		ib_dma_sync_single_for_device(sess->s.dev->ib_dev,
+					      srv_mr->iu->dma_addr,
+					      srv_mr->iu->size, DMA_TO_DEVICE);
+	} else {
+		imm_wr.sg_list = NULL;
+		imm_wr.num_sge = 0;
+		imm_wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM;
+	}
+	imm_wr.send_flags = flags;
+	imm_wr.wr_cqe   = &io_comp_cqe;
+
+	imm_wr.ex.imm_data = cpu_to_be32(imm);
+
+	err = ib_post_send(id->con->c.qp, wr, NULL);
+	if (unlikely(err))
+		rtrs_err_rl(s, "Posting RDMA-Reply to QP failed, err: %d\n",
+			     err);
+
+	return err;
+}
+
+void close_sess(struct rtrs_srv_sess *sess)
+{
+	enum rtrs_srv_state old_state;
+
+	if (rtrs_srv_change_state_get_old(sess, RTRS_SRV_CLOSING,
+					   &old_state))
+		queue_work(rtrs_wq, &sess->close_work);
+	WARN_ON(sess->state != RTRS_SRV_CLOSING);
+}
+
+static inline const char *rtrs_srv_state_str(enum rtrs_srv_state state)
+{
+	switch (state) {
+	case RTRS_SRV_CONNECTING:
+		return "RTRS_SRV_CONNECTING";
+	case RTRS_SRV_CONNECTED:
+		return "RTRS_SRV_CONNECTED";
+	case RTRS_SRV_CLOSING:
+		return "RTRS_SRV_CLOSING";
+	case RTRS_SRV_CLOSED:
+		return "RTRS_SRV_CLOSED";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+/**
+ * rtrs_srv_resp_rdma() - Finish an RDMA request
+ *
+ * @id:		Internal RTRS operation identifier
+ * @status:	Response Code sent to the other side for this operation.
+ *		0 = success, <=0 error
+ * Context: any
+ *
+ * Finish a RDMA operation. A message is sent to the client and the
+ * corresponding memory areas will be released.
+ */
+bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int status)
+{
+	struct rtrs_srv_sess *sess;
+	struct rtrs_srv_con *con;
+	struct rtrs_sess *s;
+	int err;
+
+	if (WARN_ON(!id))
+		return true;
+
+	con = id->con;
+	s = con->c.sess;
+	sess = to_srv_sess(s);
+
+	id->status = status;
+
+	if (unlikely(sess->state != RTRS_SRV_CONNECTED)) {
+		rtrs_err_rl(s,
+			     "Sending I/O response failed,  session is disconnected, sess state %s\n",
+			     rtrs_srv_state_str(sess->state));
+		goto out;
+	}
+	if (always_invalidate) {
+		struct rtrs_srv_mr *mr = &sess->mrs[id->msg_id];
+
+		ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey));
+	}
+	if (unlikely(atomic_sub_return(1,
+				       &con->sq_wr_avail) < 0)) {
+		pr_err("IB send queue full\n");
+		atomic_add(1, &con->sq_wr_avail);
+		spin_lock(&con->rsp_wr_wait_lock);
+		list_add_tail(&id->wait_list, &con->rsp_wr_wait_list);
+		spin_unlock(&con->rsp_wr_wait_lock);
+		return false;
+	}
+
+	if (status || id->dir == WRITE || !id->rd_msg->sg_cnt)
+		err = send_io_resp_imm(con, id, status);
+	else
+		err = rdma_write_sg(id);
+
+	if (unlikely(err)) {
+		rtrs_err_rl(s, "IO response failed: %d\n", err);
+		close_sess(sess);
+	}
+out:
+	rtrs_srv_put_ops_ids(sess);
+	return true;
+}
+EXPORT_SYMBOL(rtrs_srv_resp_rdma);
+
+/**
+ * rtrs_srv_set_sess_priv() - Set private pointer in rtrs_srv.
+ * @srv:	Session pointer
+ * @priv:	The private pointer that is associated with the session.
+ */
+void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv)
+{
+	srv->priv = priv;
+}
+EXPORT_SYMBOL(rtrs_srv_set_sess_priv);
+
+static void unmap_cont_bufs(struct rtrs_srv_sess *sess)
+{
+	int i;
+
+	for (i = 0; i < sess->mrs_num; i++) {
+		struct rtrs_srv_mr *srv_mr;
+
+		srv_mr = &sess->mrs[i];
+		rtrs_iu_free(srv_mr->iu, DMA_TO_DEVICE,
+			      sess->s.dev->ib_dev, 1);
+		ib_dereg_mr(srv_mr->mr);
+		ib_dma_unmap_sg(sess->s.dev->ib_dev, srv_mr->sgt.sgl,
+				srv_mr->sgt.nents, DMA_BIDIRECTIONAL);
+		sg_free_table(&srv_mr->sgt);
+	}
+	kfree(sess->mrs);
+}
+
+static int map_cont_bufs(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_sess *ss = &sess->s;
+	int i, mri, err, mrs_num;
+	unsigned int chunk_bits;
+	int chunks_per_mr = 1;
+
+	/*
+	 * Here we map queue_depth chunks to MR.  Firstly we have to
+	 * figure out how many chunks can we map per MR.
+	 */
+	if (always_invalidate) {
+		/*
+		 * in order to do invalidate for each chunks of memory, we needs
+		 * more memory regions.
+		 */
+		mrs_num = srv->queue_depth;
+	} else {
+		chunks_per_mr =
+			sess->s.dev->ib_dev->attrs.max_fast_reg_page_list_len;
+		mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr);
+		chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num);
+	}
+
+	sess->mrs = kcalloc(mrs_num, sizeof(*sess->mrs), GFP_KERNEL);
+	if (!sess->mrs)
+		return -ENOMEM;
+
+	sess->mrs_num = mrs_num;
+
+	for (mri = 0; mri < mrs_num; mri++) {
+		struct rtrs_srv_mr *srv_mr = &sess->mrs[mri];
+		struct sg_table *sgt = &srv_mr->sgt;
+		struct scatterlist *s;
+		struct ib_mr *mr;
+		int nr, chunks;
+
+		chunks = chunks_per_mr * mri;
+		if (!always_invalidate)
+			chunks_per_mr = min_t(int, chunks_per_mr,
+					      srv->queue_depth - chunks);
+
+		err = sg_alloc_table(sgt, chunks_per_mr, GFP_KERNEL);
+		if (err)
+			goto err;
+
+		for_each_sg(sgt->sgl, s, chunks_per_mr, i)
+			sg_set_page(s, srv->chunks[chunks + i],
+				    max_chunk_size, 0);
+
+		nr = ib_dma_map_sg(sess->s.dev->ib_dev, sgt->sgl,
+				   sgt->nents, DMA_BIDIRECTIONAL);
+		if (nr < sgt->nents) {
+			err = nr < 0 ? nr : -EINVAL;
+			goto free_sg;
+		}
+		mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG,
+				 sgt->nents);
+		if (IS_ERR(mr)) {
+			err = PTR_ERR(mr);
+			goto unmap_sg;
+		}
+		nr = ib_map_mr_sg(mr, sgt->sgl, sgt->nents,
+				  NULL, max_chunk_size);
+		if (nr < 0 || nr < sgt->nents) {
+			err = nr < 0 ? nr : -EINVAL;
+			goto dereg_mr;
+		}
+
+		if (always_invalidate) {
+			srv_mr->iu = rtrs_iu_alloc(1,
+					sizeof(struct rtrs_msg_rkey_rsp),
+					GFP_KERNEL, sess->s.dev->ib_dev,
+					DMA_TO_DEVICE, rtrs_srv_rdma_done);
+			if (!srv_mr->iu) {
+				err = -ENOMEM;
+				rtrs_err(ss, "rtrs_iu_alloc(), err: %d\n", err);
+				goto free_iu;
+			}
+		}
+		/* Eventually dma addr for each chunk can be cached */
+		for_each_sg(sgt->sgl, s, sgt->orig_nents, i)
+			sess->dma_addr[chunks + i] = sg_dma_address(s);
+
+		ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
+		srv_mr->mr = mr;
+
+		continue;
+err:
+		while (mri--) {
+			srv_mr = &sess->mrs[mri];
+			sgt = &srv_mr->sgt;
+			mr = srv_mr->mr;
+free_iu:
+			rtrs_iu_free(srv_mr->iu, DMA_TO_DEVICE,
+				      sess->s.dev->ib_dev, 1);
+dereg_mr:
+			ib_dereg_mr(mr);
+unmap_sg:
+			ib_dma_unmap_sg(sess->s.dev->ib_dev, sgt->sgl,
+					sgt->nents, DMA_BIDIRECTIONAL);
+free_sg:
+			sg_free_table(sgt);
+		}
+		kfree(sess->mrs);
+
+		return err;
+	}
+
+	chunk_bits = ilog2(srv->queue_depth - 1) + 1;
+	sess->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);
+
+	return 0;
+}
+
+static void rtrs_srv_hb_err_handler(struct rtrs_con *c)
+{
+	close_sess(to_srv_sess(c->sess));
+}
+
+static void rtrs_srv_init_hb(struct rtrs_srv_sess *sess)
+{
+	rtrs_init_hb(&sess->s, &io_comp_cqe,
+		      RTRS_HB_INTERVAL_MS,
+		      RTRS_HB_MISSED_MAX,
+		      rtrs_srv_hb_err_handler,
+		      rtrs_wq);
+}
+
+static void rtrs_srv_start_hb(struct rtrs_srv_sess *sess)
+{
+	rtrs_start_hb(&sess->s);
+}
+
+static void rtrs_srv_stop_hb(struct rtrs_srv_sess *sess)
+{
+	rtrs_stop_hb(&sess->s);
+}
+
+static void rtrs_srv_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_srv_con *con = cq->cq_context;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_iu *iu;
+
+	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
+	rtrs_iu_free(iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(s, "Sess info response send failed: %s\n",
+			  ib_wc_status_msg(wc->status));
+		close_sess(sess);
+		return;
+	}
+	WARN_ON(wc->opcode != IB_WC_SEND);
+}
+
+static void rtrs_srv_sess_up(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_srv_ctx *ctx = srv->ctx;
+	int up;
+
+	mutex_lock(&srv->paths_ev_mutex);
+	up = ++srv->paths_up;
+	if (up == 1)
+		ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_CONNECTED, NULL);
+	mutex_unlock(&srv->paths_ev_mutex);
+
+	/* Mark session as established */
+	sess->established = true;
+}
+
+static void rtrs_srv_sess_down(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_srv_ctx *ctx = srv->ctx;
+
+	if (!sess->established)
+		return;
+
+	sess->established = false;
+	mutex_lock(&srv->paths_ev_mutex);
+	WARN_ON(!srv->paths_up);
+	if (--srv->paths_up == 0)
+		ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_DISCONNECTED, srv->priv);
+	mutex_unlock(&srv->paths_ev_mutex);
+}
+
+static int post_recv_sess(struct rtrs_srv_sess *sess);
+
+static int process_info_req(struct rtrs_srv_con *con,
+			    struct rtrs_msg_info_req *msg)
+{
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct ib_send_wr *reg_wr = NULL;
+	struct rtrs_msg_info_rsp *rsp;
+	struct rtrs_iu *tx_iu;
+	struct ib_reg_wr *rwr;
+	int mri, err;
+	size_t tx_sz;
+
+	err = post_recv_sess(sess);
+	if (unlikely(err)) {
+		rtrs_err(s, "post_recv_sess(), err: %d\n", err);
+		return err;
+	}
+	rwr = kcalloc(sess->mrs_num, sizeof(*rwr), GFP_KERNEL);
+	if (unlikely(!rwr))
+		return -ENOMEM;
+	strlcpy(sess->s.sessname, msg->sessname, sizeof(sess->s.sessname));
+
+	tx_sz  = sizeof(*rsp);
+	tx_sz += sizeof(rsp->desc[0]) * sess->mrs_num;
+	tx_iu = rtrs_iu_alloc(1, tx_sz, GFP_KERNEL, sess->s.dev->ib_dev,
+			       DMA_TO_DEVICE, rtrs_srv_info_rsp_done);
+	if (unlikely(!tx_iu)) {
+		err = -ENOMEM;
+		goto rwr_free;
+	}
+
+	rsp = tx_iu->buf;
+	rsp->type = cpu_to_le16(RTRS_MSG_INFO_RSP);
+	rsp->sg_cnt = cpu_to_le16(sess->mrs_num);
+
+	for (mri = 0; mri < sess->mrs_num; mri++) {
+		struct ib_mr *mr = sess->mrs[mri].mr;
+
+		rsp->desc[mri].addr = cpu_to_le64(mr->iova);
+		rsp->desc[mri].key  = cpu_to_le32(mr->rkey);
+		rsp->desc[mri].len  = cpu_to_le32(mr->length);
+
+		/*
+		 * Fill in reg MR request and chain them *backwards*
+		 */
+		rwr[mri].wr.next = mri ? &rwr[mri - 1].wr : NULL;
+		rwr[mri].wr.opcode = IB_WR_REG_MR;
+		rwr[mri].wr.wr_cqe = &local_reg_cqe;
+		rwr[mri].wr.num_sge = 0;
+		rwr[mri].wr.send_flags = mri ? 0 : IB_SEND_SIGNALED;
+		rwr[mri].mr = mr;
+		rwr[mri].key = mr->rkey;
+		rwr[mri].access = (IB_ACCESS_LOCAL_WRITE |
+				   IB_ACCESS_REMOTE_WRITE);
+		reg_wr = &rwr[mri].wr;
+	}
+
+	err = rtrs_srv_create_sess_files(sess);
+	if (unlikely(err))
+		goto iu_free;
+	kobject_get(&sess->kobj);
+	get_device(&sess->srv->dev);
+	rtrs_srv_change_state(sess, RTRS_SRV_CONNECTED);
+	rtrs_srv_start_hb(sess);
+
+	/*
+	 * We do not account number of established connections at the current
+	 * moment, we rely on the client, which should send info request when
+	 * all connections are successfully established.  Thus, simply notify
+	 * listener with a proper event if we are the first path.
+	 */
+	rtrs_srv_sess_up(sess);
+
+	ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr,
+				      tx_iu->size, DMA_TO_DEVICE);
+
+	/* Send info response */
+	err = rtrs_iu_post_send(&con->c, tx_iu, tx_sz, reg_wr);
+	if (unlikely(err)) {
+		rtrs_err(s, "rtrs_iu_post_send(), err: %d\n", err);
+iu_free:
+		rtrs_iu_free(tx_iu, DMA_TO_DEVICE, sess->s.dev->ib_dev, 1);
+	}
+rwr_free:
+	kfree(rwr);
+
+	return err;
+}
+
+static void rtrs_srv_info_req_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_srv_con *con = cq->cq_context;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_msg_info_req *msg;
+	struct rtrs_iu *iu;
+	int err;
+
+	WARN_ON(con->c.cid);
+
+	iu = container_of(wc->wr_cqe, struct rtrs_iu, cqe);
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(s, "Sess info request receive failed: %s\n",
+			  ib_wc_status_msg(wc->status));
+		goto close;
+	}
+	WARN_ON(wc->opcode != IB_WC_RECV);
+
+	if (unlikely(wc->byte_len < sizeof(*msg))) {
+		rtrs_err(s, "Sess info request is malformed: size %d\n",
+			  wc->byte_len);
+		goto close;
+	}
+	ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr,
+				   iu->size, DMA_FROM_DEVICE);
+	msg = iu->buf;
+	if (unlikely(le16_to_cpu(msg->type) != RTRS_MSG_INFO_REQ)) {
+		rtrs_err(s, "Sess info request is malformed: type %d\n",
+			  le16_to_cpu(msg->type));
+		goto close;
+	}
+	err = process_info_req(con, msg);
+	if (unlikely(err))
+		goto close;
+
+out:
+	rtrs_iu_free(iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
+	return;
+close:
+	close_sess(sess);
+	goto out;
+}
+
+static int post_recv_info_req(struct rtrs_srv_con *con)
+{
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_iu *rx_iu;
+	int err;
+
+	rx_iu = rtrs_iu_alloc(1, sizeof(struct rtrs_msg_info_req),
+			       GFP_KERNEL, sess->s.dev->ib_dev,
+			       DMA_FROM_DEVICE, rtrs_srv_info_req_done);
+	if (unlikely(!rx_iu))
+		return -ENOMEM;
+	/* Prepare for getting info response */
+	err = rtrs_iu_post_recv(&con->c, rx_iu);
+	if (unlikely(err)) {
+		rtrs_err(s, "rtrs_iu_post_recv(), err: %d\n", err);
+		rtrs_iu_free(rx_iu, DMA_FROM_DEVICE, sess->s.dev->ib_dev, 1);
+		return err;
+	}
+
+	return 0;
+}
+
+static int post_recv_io(struct rtrs_srv_con *con, size_t q_size)
+{
+	int i, err;
+
+	for (i = 0; i < q_size; i++) {
+		err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
+		if (unlikely(err))
+			return err;
+	}
+
+	return 0;
+}
+
+static int post_recv_sess(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_sess *s = &sess->s;
+	size_t q_size;
+	int err, cid;
+
+	for (cid = 0; cid < sess->s.con_num; cid++) {
+		if (cid == 0)
+			q_size = SERVICE_CON_QUEUE_DEPTH;
+		else
+			q_size = srv->queue_depth;
+
+		err = post_recv_io(to_srv_con(sess->s.con[cid]), q_size);
+		if (unlikely(err)) {
+			rtrs_err(s, "post_recv_io(), err: %d\n", err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static void process_read(struct rtrs_srv_con *con,
+			 struct rtrs_msg_rdma_read *msg,
+			 u32 buf_id, u32 off)
+{
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_srv_ctx *ctx = srv->ctx;
+	struct rtrs_srv_op *id;
+
+	size_t usr_len, data_len;
+	void *data;
+	int ret;
+
+	if (unlikely(sess->state != RTRS_SRV_CONNECTED)) {
+		rtrs_err_rl(s,
+			     "Processing read request failed,  session is disconnected, sess state %s\n",
+			     rtrs_srv_state_str(sess->state));
+		return;
+	}
+	if (unlikely(msg->sg_cnt != 1 && msg->sg_cnt != 0)) {
+		rtrs_err_rl(s,
+			    "Processing read request failed, invalid message\n");
+		return;
+	}
+	rtrs_srv_get_ops_ids(sess);
+	rtrs_srv_update_rdma_stats(sess->stats, off, READ);
+	id = sess->ops_ids[buf_id];
+	id->con		= con;
+	id->dir		= READ;
+	id->msg_id	= buf_id;
+	id->rd_msg	= msg;
+	usr_len = le16_to_cpu(msg->usr_len);
+	data_len = off - usr_len;
+	data = page_address(srv->chunks[buf_id]);
+	ret = ctx->ops.rdma_ev(srv, srv->priv, id, READ, data, data_len,
+			   data + data_len, usr_len);
+
+	if (unlikely(ret)) {
+		rtrs_err_rl(s,
+			     "Processing read request failed, user module cb reported for msg_id %d, err: %d\n",
+			     buf_id, ret);
+		goto send_err_msg;
+	}
+
+	return;
+
+send_err_msg:
+	ret = send_io_resp_imm(con, id, ret);
+	if (ret < 0) {
+		rtrs_err_rl(s,
+			     "Sending err msg for failed RDMA-Write-Req failed, msg_id %d, err: %d\n",
+			     buf_id, ret);
+		close_sess(sess);
+	}
+	rtrs_srv_put_ops_ids(sess);
+}
+
+static void process_write(struct rtrs_srv_con *con,
+			  struct rtrs_msg_rdma_write *req,
+			  u32 buf_id, u32 off)
+{
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_srv_ctx *ctx = srv->ctx;
+	struct rtrs_srv_op *id;
+
+	size_t data_len, usr_len;
+	void *data;
+	int ret;
+
+	if (unlikely(sess->state != RTRS_SRV_CONNECTED)) {
+		rtrs_err_rl(s,
+			     "Processing write request failed,  session is disconnected, sess state %s\n",
+			     rtrs_srv_state_str(sess->state));
+		return;
+	}
+	rtrs_srv_get_ops_ids(sess);
+	rtrs_srv_update_rdma_stats(sess->stats, off, WRITE);
+	id = sess->ops_ids[buf_id];
+	id->con    = con;
+	id->dir    = WRITE;
+	id->msg_id = buf_id;
+
+	usr_len = le16_to_cpu(req->usr_len);
+	data_len = off - usr_len;
+	data = page_address(srv->chunks[buf_id]);
+	ret = ctx->ops.rdma_ev(srv, srv->priv, id, WRITE, data, data_len,
+			   data + data_len, usr_len);
+	if (unlikely(ret)) {
+		rtrs_err_rl(s,
+			     "Processing write request failed, user module callback reports err: %d\n",
+			     ret);
+		goto send_err_msg;
+	}
+
+	return;
+
+send_err_msg:
+	ret = send_io_resp_imm(con, id, ret);
+	if (ret < 0) {
+		rtrs_err_rl(s,
+			     "Processing write request failed, sending I/O response failed, msg_id %d, err: %d\n",
+			     buf_id, ret);
+		close_sess(sess);
+	}
+	rtrs_srv_put_ops_ids(sess);
+}
+
+static void process_io_req(struct rtrs_srv_con *con, void *msg,
+			   u32 id, u32 off)
+{
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_msg_rdma_hdr *hdr;
+	unsigned int type;
+
+	ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, sess->dma_addr[id],
+				   max_chunk_size, DMA_BIDIRECTIONAL);
+	hdr = msg;
+	type = le16_to_cpu(hdr->type);
+
+	switch (type) {
+	case RTRS_MSG_WRITE:
+		process_write(con, msg, id, off);
+		break;
+	case RTRS_MSG_READ:
+		process_read(con, msg, id, off);
+		break;
+	default:
+		rtrs_err(s,
+			  "Processing I/O request failed, unknown message type received: 0x%02x\n",
+			  type);
+		goto err;
+	}
+
+	return;
+
+err:
+	close_sess(sess);
+}
+
+static void rtrs_srv_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_srv_mr *mr =
+		container_of(wc->wr_cqe, typeof(*mr), inv_cqe);
+	struct rtrs_srv_con *con = cq->cq_context;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_srv *srv = sess->srv;
+	u32 msg_id, off;
+	void *data;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		rtrs_err(s, "Failed IB_WR_LOCAL_INV: %s\n",
+			  ib_wc_status_msg(wc->status));
+		close_sess(sess);
+	}
+	msg_id = mr->msg_id;
+	off = mr->msg_off;
+	data = page_address(srv->chunks[msg_id]) + off;
+	process_io_req(con, data, msg_id, off);
+}
+
+static int rtrs_srv_inv_rkey(struct rtrs_srv_con *con,
+			      struct rtrs_srv_mr *mr)
+{
+	struct ib_send_wr wr = {
+		.opcode		    = IB_WR_LOCAL_INV,
+		.wr_cqe		    = &mr->inv_cqe,
+		.send_flags	    = IB_SEND_SIGNALED,
+		.ex.invalidate_rkey = mr->mr->rkey,
+	};
+	mr->inv_cqe.done = rtrs_srv_inv_rkey_done;
+
+	return ib_post_send(con->c.qp, &wr, NULL);
+}
+
+static void rtrs_rdma_process_wr_wait_list(struct rtrs_srv_con *con)
+{
+	spin_lock(&con->rsp_wr_wait_lock);
+	while (!list_empty(&con->rsp_wr_wait_list)) {
+		struct rtrs_srv_op *id;
+		int ret;
+
+		id = list_entry(con->rsp_wr_wait_list.next,
+				struct rtrs_srv_op, wait_list);
+		list_del(&id->wait_list);
+
+		spin_unlock(&con->rsp_wr_wait_lock);
+		ret = rtrs_srv_resp_rdma(id, id->status);
+		spin_lock(&con->rsp_wr_wait_lock);
+
+		if (!ret) {
+			list_add(&id->wait_list, &con->rsp_wr_wait_list);
+			break;
+		}
+	}
+	spin_unlock(&con->rsp_wr_wait_lock);
+}
+
+static void rtrs_srv_rdma_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+	struct rtrs_srv_con *con = cq->cq_context;
+	struct rtrs_sess *s = con->c.sess;
+	struct rtrs_srv_sess *sess = to_srv_sess(s);
+	struct rtrs_srv *srv = sess->srv;
+	u32 imm_type, imm_payload;
+	int err;
+
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR) {
+			rtrs_err(s,
+				  "%s (wr_cqe: %p, type: %d, vendor_err: 0x%x, len: %u)\n",
+				  ib_wc_status_msg(wc->status), wc->wr_cqe,
+				  wc->opcode, wc->vendor_err, wc->byte_len);
+			close_sess(sess);
+		}
+		return;
+	}
+
+	switch (wc->opcode) {
+	case IB_WC_RECV_RDMA_WITH_IMM:
+		/*
+		 * post_recv() RDMA write completions of IO reqs (read/write)
+		 * and hb
+		 */
+		if (WARN_ON(wc->wr_cqe != &io_comp_cqe))
+			return;
+		err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);
+		if (unlikely(err)) {
+			rtrs_err(s, "rtrs_post_recv(), err: %d\n", err);
+			close_sess(sess);
+			break;
+		}
+		rtrs_from_imm(be32_to_cpu(wc->ex.imm_data),
+			       &imm_type, &imm_payload);
+		if (likely(imm_type == RTRS_IO_REQ_IMM)) {
+			u32 msg_id, off;
+			void *data;
+
+			msg_id = imm_payload >> sess->mem_bits;
+			off = imm_payload & ((1 << sess->mem_bits) - 1);
+			if (unlikely(msg_id >= srv->queue_depth ||
+				     off >= max_chunk_size)) {
+				rtrs_err(s, "Wrong msg_id %u, off %u\n",
+					  msg_id, off);
+				close_sess(sess);
+				return;
+			}
+			if (always_invalidate) {
+				struct rtrs_srv_mr *mr = &sess->mrs[msg_id];
+
+				mr->msg_off = off;
+				mr->msg_id = msg_id;
+				err = rtrs_srv_inv_rkey(con, mr);
+				if (unlikely(err)) {
+					rtrs_err(s, "rtrs_post_recv(), err: %d\n",
+						  err);
+					close_sess(sess);
+					break;
+				}
+			} else {
+				data = page_address(srv->chunks[msg_id]) + off;
+				process_io_req(con, data, msg_id, off);
+			}
+		} else if (imm_type == RTRS_HB_MSG_IMM) {
+			WARN_ON(con->c.cid);
+			rtrs_send_hb_ack(&sess->s);
+		} else if (imm_type == RTRS_HB_ACK_IMM) {
+			WARN_ON(con->c.cid);
+			sess->s.hb_missed_cnt = 0;
+		} else {
+			rtrs_wrn(s, "Unknown IMM type %u\n", imm_type);
+		}
+		break;
+	case IB_WC_RDMA_WRITE:
+	case IB_WC_SEND:
+		/*
+		 * post_send() RDMA write completions of IO reqs (read/write)
+		 * and hb
+		 */
+		atomic_add(srv->queue_depth, &con->sq_wr_avail);
+
+		if (unlikely(!list_empty_careful(&con->rsp_wr_wait_list)))
+			rtrs_rdma_process_wr_wait_list(con);
+
+		break;
+	default:
+		rtrs_wrn(s, "Unexpected WC type: %d\n", wc->opcode);
+		return;
+	}
+}
+
+/**
+ * rtrs_srv_get_sess_name() - Get rtrs_srv peer hostname.
+ * @srv:	Session
+ * @sessname:	Sessname buffer
+ * @len:	Length of sessname buffer
+ */
+int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len)
+{
+	struct rtrs_srv_sess *sess;
+	int err = -ENOTCONN;
+
+	mutex_lock(&srv->paths_mutex);
+	list_for_each_entry(sess, &srv->paths_list, s.entry) {
+		if (sess->state != RTRS_SRV_CONNECTED)
+			continue;
+		strlcpy(sessname, sess->s.sessname,
+		       min_t(size_t, sizeof(sess->s.sessname), len));
+		err = 0;
+		break;
+	}
+	mutex_unlock(&srv->paths_mutex);
+
+	return err;
+}
+EXPORT_SYMBOL(rtrs_srv_get_sess_name);
+
+/**
+ * rtrs_srv_get_sess_qdepth() - Get rtrs_srv qdepth.
+ * @srv:	Session
+ */
+int rtrs_srv_get_queue_depth(struct rtrs_srv *srv)
+{
+	return srv->queue_depth;
+}
+EXPORT_SYMBOL(rtrs_srv_get_queue_depth);
+
+static int find_next_bit_ring(struct rtrs_srv_sess *sess)
+{
+	struct ib_device *ib_dev = sess->s.dev->ib_dev;
+	int v;
+
+	v = cpumask_next(sess->cur_cq_vector, &cq_affinity_mask);
+	if (v >= nr_cpu_ids || v >= ib_dev->num_comp_vectors)
+		v = cpumask_first(&cq_affinity_mask);
+	return v;
+}
+
+static int rtrs_srv_get_next_cq_vector(struct rtrs_srv_sess *sess)
+{
+	sess->cur_cq_vector = find_next_bit_ring(sess);
+
+	return sess->cur_cq_vector;
+}
+
+static struct rtrs_srv *__alloc_srv(struct rtrs_srv_ctx *ctx,
+				     const uuid_t *paths_uuid)
+{
+	struct rtrs_srv *srv;
+	int i;
+
+	srv = kzalloc(sizeof(*srv), GFP_KERNEL);
+	if  (!srv)
+		return NULL;
+
+	refcount_set(&srv->refcount, 1);
+	INIT_LIST_HEAD(&srv->paths_list);
+	mutex_init(&srv->paths_mutex);
+	mutex_init(&srv->paths_ev_mutex);
+	uuid_copy(&srv->paths_uuid, paths_uuid);
+	srv->queue_depth = sess_queue_depth;
+	srv->ctx = ctx;
+
+	srv->chunks = kcalloc(srv->queue_depth, sizeof(*srv->chunks),
+			      GFP_KERNEL);
+	if (!srv->chunks)
+		goto err_free_srv;
+
+	for (i = 0; i < srv->queue_depth; i++) {
+		srv->chunks[i] = mempool_alloc(chunk_pool, GFP_KERNEL);
+		if (!srv->chunks[i])
+			goto err_free_chunks;
+	}
+	list_add(&srv->ctx_list, &ctx->srv_list);
+
+	return srv;
+
+err_free_chunks:
+	while (i--)
+		mempool_free(srv->chunks[i], chunk_pool);
+	kfree(srv->chunks);
+
+err_free_srv:
+	kfree(srv);
+
+	return NULL;
+}
+
+static void free_srv(struct rtrs_srv *srv)
+{
+	int i;
+
+	WARN_ON(refcount_read(&srv->refcount));
+	for (i = 0; i < srv->queue_depth; i++)
+		mempool_free(srv->chunks[i], chunk_pool);
+	kfree(srv->chunks);
+	mutex_destroy(&srv->paths_mutex);
+	mutex_destroy(&srv->paths_ev_mutex);
+	/* last put to release the srv structure */
+	put_device(&srv->dev);
+}
+
+static inline struct rtrs_srv *__find_srv_and_get(struct rtrs_srv_ctx *ctx,
+						   const uuid_t *paths_uuid)
+{
+	struct rtrs_srv *srv;
+
+	list_for_each_entry(srv, &ctx->srv_list, ctx_list) {
+		if (uuid_equal(&srv->paths_uuid, paths_uuid) &&
+		    refcount_inc_not_zero(&srv->refcount))
+			return srv;
+	}
+
+	return NULL;
+}
+
+static struct rtrs_srv *get_or_create_srv(struct rtrs_srv_ctx *ctx,
+					   const uuid_t *paths_uuid)
+{
+	struct rtrs_srv *srv;
+
+	mutex_lock(&ctx->srv_mutex);
+	srv = __find_srv_and_get(ctx, paths_uuid);
+	if (!srv)
+		srv = __alloc_srv(ctx, paths_uuid);
+	mutex_unlock(&ctx->srv_mutex);
+
+	return srv;
+}
+
+static void put_srv(struct rtrs_srv *srv)
+{
+	if (refcount_dec_and_test(&srv->refcount)) {
+		struct rtrs_srv_ctx *ctx = srv->ctx;
+
+		WARN_ON(srv->dev.kobj.state_in_sysfs);
+
+		mutex_lock(&ctx->srv_mutex);
+		list_del(&srv->ctx_list);
+		mutex_unlock(&ctx->srv_mutex);
+		free_srv(srv);
+	}
+}
+
+static void __add_path_to_srv(struct rtrs_srv *srv,
+			      struct rtrs_srv_sess *sess)
+{
+	list_add_tail(&sess->s.entry, &srv->paths_list);
+	srv->paths_num++;
+	WARN_ON(srv->paths_num >= MAX_PATHS_NUM);
+}
+
+static void del_path_from_srv(struct rtrs_srv_sess *sess)
+{
+	struct rtrs_srv *srv = sess->srv;
+
+	if (WARN_ON(!srv))
+		return;
+
+	mutex_lock(&srv->paths_mutex);
+	list_del(&sess->s.entry);
+	WARN_ON(!srv->paths_num);
+	srv->paths_num--;
+	mutex_unlock(&srv->paths_mutex);
+}
+
+/* return true if addresses are the same, error other wise */
+static int sockaddr_cmp(const struct sockaddr *a, const struct sockaddr *b)
+{
+	switch (a->sa_family) {
+	case AF_IB:
+		return memcmp(&((struct sockaddr_ib *)a)->sib_addr,
+			      &((struct sockaddr_ib *)b)->sib_addr,
+			      sizeof(struct ib_addr)) &&
+			(b->sa_family == AF_IB);
+	case AF_INET:
+		return memcmp(&((struct sockaddr_in *)a)->sin_addr,
+			      &((struct sockaddr_in *)b)->sin_addr,
+			      sizeof(struct in_addr)) &&
+			(b->sa_family == AF_INET);
+	case AF_INET6:
+		return memcmp(&((struct sockaddr_in6 *)a)->sin6_addr,
+			      &((struct sockaddr_in6 *)b)->sin6_addr,
+			      sizeof(struct in6_addr)) &&
+			(b->sa_family == AF_INET6);
+	default:
+		return -ENOENT;
+	}
+}
+
+static bool __is_path_w_addr_exists(struct rtrs_srv *srv,
+				    struct rdma_addr *addr)
+{
+	struct rtrs_srv_sess *sess;
+
+	list_for_each_entry(sess, &srv->paths_list, s.entry)
+		if (!sockaddr_cmp((struct sockaddr *)&sess->s.dst_addr,
+				  (struct sockaddr *)&addr->dst_addr) &&
+		    !sockaddr_cmp((struct sockaddr *)&sess->s.src_addr,
+				  (struct sockaddr *)&addr->src_addr))
+			return true;
+
+	return false;
+}
+
+static void free_sess(struct rtrs_srv_sess *sess)
+{
+	if (sess->kobj.state_in_sysfs)
+		kobject_put(&sess->kobj);
+	else
+		kfree(sess);
+}
+
+static void rtrs_srv_close_work(struct work_struct *work)
+{
+	struct rtrs_srv_sess *sess;
+	struct rtrs_srv_con *con;
+	int i;
+
+	sess = container_of(work, typeof(*sess), close_work);
+
+	rtrs_srv_destroy_sess_files(sess);
+	rtrs_srv_stop_hb(sess);
+
+	for (i = 0; i < sess->s.con_num; i++) {
+		if (!sess->s.con[i])
+			continue;
+		con = to_srv_con(sess->s.con[i]);
+		rdma_disconnect(con->c.cm_id);
+		ib_drain_qp(con->c.qp);
+	}
+	/* Wait for all inflights */
+	rtrs_srv_wait_ops_ids(sess);
+
+	/* Notify upper layer if we are the last path */
+	rtrs_srv_sess_down(sess);
+
+	unmap_cont_bufs(sess);
+	rtrs_srv_free_ops_ids(sess);
+
+	for (i = 0; i < sess->s.con_num; i++) {
+		if (!sess->s.con[i])
+			continue;
+		con = to_srv_con(sess->s.con[i]);
+		rtrs_cq_qp_destroy(&con->c);
+		rdma_destroy_id(con->c.cm_id);
+		kfree(con);
+	}
+	rtrs_ib_dev_put(sess->s.dev);
+
+	del_path_from_srv(sess);
+	put_srv(sess->srv);
+	sess->srv = NULL;
+	rtrs_srv_change_state(sess, RTRS_SRV_CLOSED);
+
+	kfree(sess->dma_addr);
+	kfree(sess->s.con);
+	free_sess(sess);
+}
+
+static int rtrs_rdma_do_accept(struct rtrs_srv_sess *sess,
+			       struct rdma_cm_id *cm_id)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_msg_conn_rsp msg;
+	struct rdma_conn_param param;
+	int err;
+
+	param = (struct rdma_conn_param) {
+		.rnr_retry_count = 7,
+		.private_data = &msg,
+		.private_data_len = sizeof(msg),
+	};
+
+	msg = (struct rtrs_msg_conn_rsp) {
+		.magic = cpu_to_le16(RTRS_MAGIC),
+		.version = cpu_to_le16(RTRS_PROTO_VER),
+		.queue_depth = cpu_to_le16(srv->queue_depth),
+		.max_io_size = cpu_to_le32(max_chunk_size - MAX_HDR_SIZE),
+		.max_hdr_size = cpu_to_le32(MAX_HDR_SIZE),
+	};
+
+	if (always_invalidate)
+		msg.flags = cpu_to_le32(RTRS_MSG_NEW_RKEY_F);
+
+	err = rdma_accept(cm_id, &param);
+	if (err)
+		pr_err("rdma_accept(), err: %d\n", err);
+
+	return err;
+}
+
+static int rtrs_rdma_do_reject(struct rdma_cm_id *cm_id, int errno)
+{
+	struct rtrs_msg_conn_rsp msg;
+	int err;
+
+	msg = (struct rtrs_msg_conn_rsp) {
+		.magic = cpu_to_le16(RTRS_MAGIC),
+		.version = cpu_to_le16(RTRS_PROTO_VER),
+		.errno = cpu_to_le16(errno),
+	};
+
+	err = rdma_reject(cm_id, &msg, sizeof(msg), IB_CM_REJ_CONSUMER_DEFINED);
+	if (err)
+		pr_err("rdma_reject(), err: %d\n", err);
+
+	/* Bounce errno back */
+	return errno;
+}
+
+static struct rtrs_srv_sess *
+__find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid)
+{
+	struct rtrs_srv_sess *sess;
+
+	list_for_each_entry(sess, &srv->paths_list, s.entry) {
+		if (uuid_equal(&sess->s.uuid, sess_uuid))
+			return sess;
+	}
+
+	return NULL;
+}
+
+static int create_con(struct rtrs_srv_sess *sess,
+		      struct rdma_cm_id *cm_id,
+		      unsigned int cid)
+{
+	struct rtrs_srv *srv = sess->srv;
+	struct rtrs_sess *s = &sess->s;
+	struct rtrs_srv_con *con;
+
+	u16 cq_size, wr_queue_size;
+	int err, cq_vector;
+
+	con = kzalloc(sizeof(*con), GFP_KERNEL);
+	if (!con) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	spin_lock_init(&con->rsp_wr_wait_lock);
+	INIT_LIST_HEAD(&con->rsp_wr_wait_list);
+	con->c.cm_id = cm_id;
+	con->c.sess = &sess->s;
+	con->c.cid = cid;
+	atomic_set(&con->wr_cnt, 0);
+
+	if (con->c.cid == 0) {
+		/*
+		 * All receive and all send (each requiring invalidate)
+		 * + 2 for drain and heartbeat
+		 */
+		wr_queue_size = SERVICE_CON_QUEUE_DEPTH * 3 + 2;
+		cq_size = wr_queue_size;
+	} else {
+		/*
+		 * If we have all receive requests posted and
+		 * all write requests posted and each read request
+		 * requires an invalidate request + drain
+		 * and qp gets into error state.
+		 */
+		cq_size = srv->queue_depth * 3 + 1;
+		/*
+		 * In theory we might have queue_depth * 32
+		 * outstanding requests if an unsafe global key is used
+		 * and we have queue_depth read requests each consisting
+		 * of 32 different addresses. div 3 for mlx5.
+		 */
+		wr_queue_size = sess->s.dev->ib_dev->attrs.max_qp_wr / 3;
+	}
+	atomic_set(&con->sq_wr_avail, wr_queue_size);
+	cq_vector = rtrs_srv_get_next_cq_vector(sess);
+
+	/* TODO: SOFTIRQ can be faster, but be careful with softirq context */
+	err = rtrs_cq_qp_create(&sess->s, &con->c, 1, cq_vector, cq_size,
+				 wr_queue_size, IB_POLL_WORKQUEUE);
+	if (err) {
+		rtrs_err(s, "rtrs_cq_qp_create(), err: %d\n", err);
+		goto free_con;
+	}
+	if (con->c.cid == 0) {
+		err = post_recv_info_req(con);
+		if (err)
+			goto free_cqqp;
+	}
+	WARN_ON(sess->s.con[cid]);
+	sess->s.con[cid] = &con->c;
+
+	/*
+	 * Change context from server to current connection.  The other
+	 * way is to use cm_id->qp->qp_context, which does not work on OFED.
+	 */
+	cm_id->context = &con->c;
+
+	return 0;
+
+free_cqqp:
+	rtrs_cq_qp_destroy(&con->c);
+free_con:
+	kfree(con);
+
+err:
+	return err;
+}
+
+static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv,
+					   struct rdma_cm_id *cm_id,
+					   unsigned int con_num,
+					   unsigned int recon_cnt,
+					   const uuid_t *uuid)
+{
+	struct rtrs_srv_sess *sess;
+	int err = -ENOMEM;
+
+	if (srv->paths_num >= MAX_PATHS_NUM) {
+		err = -ECONNRESET;
+		goto err;
+	}
+	if (__is_path_w_addr_exists(srv, &cm_id->route.addr)) {
+		err = -EEXIST;
+		pr_err("Path with same addr exists\n");
+		goto err;
+	}
+	sess = kzalloc(sizeof(*sess), GFP_KERNEL);
+	if (!sess)
+		goto err;
+
+	sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL);
+	if (!sess->stats)
+		goto err_free_sess;
+
+	sess->stats->sess = sess;
+
+	sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr),
+				 GFP_KERNEL);
+	if (!sess->dma_addr)
+		goto err_free_stats;
+
+	sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL);
+	if (!sess->s.con)
+		goto err_free_dma_addr;
+
+	sess->state = RTRS_SRV_CONNECTING;
+	sess->srv = srv;
+	sess->cur_cq_vector = -1;
+	sess->s.dst_addr = cm_id->route.addr.dst_addr;
+	sess->s.src_addr = cm_id->route.addr.src_addr;
+	sess->s.con_num = con_num;
+	sess->s.recon_cnt = recon_cnt;
+	uuid_copy(&sess->s.uuid, uuid);
+	spin_lock_init(&sess->state_lock);
+	INIT_WORK(&sess->close_work, rtrs_srv_close_work);
+	rtrs_srv_init_hb(sess);
+
+	sess->s.dev = rtrs_ib_dev_find_or_add(cm_id->device, &dev_pd);
+	if (!sess->s.dev) {
+		err = -ENOMEM;
+		goto err_free_con;
+	}
+	err = map_cont_bufs(sess);
+	if (err)
+		goto err_put_dev;
+
+	err = rtrs_srv_alloc_ops_ids(sess);
+	if (err)
+		goto err_unmap_bufs;
+
+	__add_path_to_srv(srv, sess);
+
+	return sess;
+
+err_unmap_bufs:
+	unmap_cont_bufs(sess);
+err_put_dev:
+	rtrs_ib_dev_put(sess->s.dev);
+err_free_con:
+	kfree(sess->s.con);
+err_free_dma_addr:
+	kfree(sess->dma_addr);
+err_free_stats:
+	kfree(sess->stats);
+err_free_sess:
+	kfree(sess);
+err:
+	return ERR_PTR(err);
+}
+
+static int rtrs_rdma_connect(struct rdma_cm_id *cm_id,
+			      const struct rtrs_msg_conn_req *msg,
+			      size_t len)
+{
+	struct rtrs_srv_ctx *ctx = cm_id->context;
+	struct rtrs_srv_sess *sess;
+	struct rtrs_srv *srv;
+
+	u16 version, con_num, cid;
+	u16 recon_cnt;
+	int err;
+
+	if (len < sizeof(*msg)) {
+		pr_err("Invalid RTRS connection request\n");
+		goto reject_w_econnreset;
+	}
+	if (le16_to_cpu(msg->magic) != RTRS_MAGIC) {
+		pr_err("Invalid RTRS magic\n");
+		goto reject_w_econnreset;
+	}
+	version = le16_to_cpu(msg->version);
+	if (version >> 8 != RTRS_PROTO_VER_MAJOR) {
+		pr_err("Unsupported major RTRS version: %d, expected %d\n",
+		       version >> 8, RTRS_PROTO_VER_MAJOR);
+		goto reject_w_econnreset;
+	}
+	con_num = le16_to_cpu(msg->cid_num);
+	if (con_num > 4096) {
+		/* Sanity check */
+		pr_err("Too many connections requested: %d\n", con_num);
+		goto reject_w_econnreset;
+	}
+	cid = le16_to_cpu(msg->cid);
+	if (cid >= con_num) {
+		/* Sanity check */
+		pr_err("Incorrect cid: %d >= %d\n", cid, con_num);
+		goto reject_w_econnreset;
+	}
+	recon_cnt = le16_to_cpu(msg->recon_cnt);
+	srv = get_or_create_srv(ctx, &msg->paths_uuid);
+	if (!srv) {
+		err = -ENOMEM;
+		goto reject_w_err;
+	}
+	mutex_lock(&srv->paths_mutex);
+	sess = __find_sess(srv, &msg->sess_uuid);
+	if (sess) {
+		struct rtrs_sess *s = &sess->s;
+
+		/* Session already holds a reference */
+		put_srv(srv);
+
+		if (sess->state != RTRS_SRV_CONNECTING) {
+			rtrs_err(s, "Session in wrong state: %s\n",
+				  rtrs_srv_state_str(sess->state));
+			mutex_unlock(&srv->paths_mutex);
+			goto reject_w_econnreset;
+		}
+		/*
+		 * Sanity checks
+		 */
+		if (con_num != s->con_num || cid >= s->con_num) {
+			rtrs_err(s, "Incorrect request: %d, %d\n",
+				  cid, con_num);
+			mutex_unlock(&srv->paths_mutex);
+			goto reject_w_econnreset;
+		}
+		if (s->con[cid]) {
+			rtrs_err(s, "Connection already exists: %d\n",
+				  cid);
+			mutex_unlock(&srv->paths_mutex);
+			goto reject_w_econnreset;
+		}
+	} else {
+		sess = __alloc_sess(srv, cm_id, con_num, recon_cnt,
+				    &msg->sess_uuid);
+		if (IS_ERR(sess)) {
+			mutex_unlock(&srv->paths_mutex);
+			put_srv(srv);
+			err = PTR_ERR(sess);
+			goto reject_w_err;
+		}
+	}
+	err = create_con(sess, cm_id, cid);
+	if (err) {
+		(void)rtrs_rdma_do_reject(cm_id, err);
+		/*
+		 * Since session has other connections we follow normal way
+		 * through workqueue, but still return an error to tell cma.c
+		 * to call rdma_destroy_id() for current connection.
+		 */
+		goto close_and_return_err;
+	}
+	err = rtrs_rdma_do_accept(sess, cm_id);
+	if (err) {
+		(void)rtrs_rdma_do_reject(cm_id, err);
+		/*
+		 * Since current connection was successfully added to the
+		 * session we follow normal way through workqueue to close the
+		 * session, thus return 0 to tell cma.c we call
+		 * rdma_destroy_id() ourselves.
+		 */
+		err = 0;
+		goto close_and_return_err;
+	}
+	mutex_unlock(&srv->paths_mutex);
+
+	return 0;
+
+reject_w_err:
+	return rtrs_rdma_do_reject(cm_id, err);
+
+reject_w_econnreset:
+	return rtrs_rdma_do_reject(cm_id, -ECONNRESET);
+
+close_and_return_err:
+	close_sess(sess);
+	mutex_unlock(&srv->paths_mutex);
+
+	return err;
+}
+
+static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id,
+				     struct rdma_cm_event *ev)
+{
+	struct rtrs_srv_sess *sess = NULL;
+	struct rtrs_sess *s = NULL;
+
+	if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
+		struct rtrs_con *c = cm_id->context;
+
+		s = c->sess;
+		sess = to_srv_sess(s);
+	}
+
+	switch (ev->event) {
+	case RDMA_CM_EVENT_CONNECT_REQUEST:
+		/*
+		 * In case of error cma.c will destroy cm_id,
+		 * see cma_process_remove()
+		 */
+		return rtrs_rdma_connect(cm_id, ev->param.conn.private_data,
+					  ev->param.conn.private_data_len);
+	case RDMA_CM_EVENT_ESTABLISHED:
+		/* Nothing here */
+		break;
+	case RDMA_CM_EVENT_REJECTED:
+	case RDMA_CM_EVENT_CONNECT_ERROR:
+	case RDMA_CM_EVENT_UNREACHABLE:
+		rtrs_err(s, "CM error (CM event: %s, err: %d)\n",
+			  rdma_event_msg(ev->event), ev->status);
+		close_sess(sess);
+		break;
+	case RDMA_CM_EVENT_DISCONNECTED:
+	case RDMA_CM_EVENT_ADDR_CHANGE:
+	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+		close_sess(sess);
+		break;
+	case RDMA_CM_EVENT_DEVICE_REMOVAL:
+		close_sess(sess);
+		break;
+	default:
+		pr_err("Ignoring unexpected CM event %s, err %d\n",
+		       rdma_event_msg(ev->event), ev->status);
+		break;
+	}
+
+	return 0;
+}
+
+static struct rdma_cm_id *rtrs_srv_cm_init(struct rtrs_srv_ctx *ctx,
+					    struct sockaddr *addr,
+					    enum rdma_ucm_port_space ps)
+{
+	struct rdma_cm_id *cm_id;
+	int ret;
+
+	cm_id = rdma_create_id(&init_net, rtrs_srv_rdma_cm_handler,
+			       ctx, ps, IB_QPT_RC);
+	if (IS_ERR(cm_id)) {
+		ret = PTR_ERR(cm_id);
+		pr_err("Creating id for RDMA connection failed, err: %d\n",
+		       ret);
+		goto err_out;
+	}
+	ret = rdma_bind_addr(cm_id, addr);
+	if (ret) {
+		pr_err("Binding RDMA address failed, err: %d\n", ret);
+		goto err_cm;
+	}
+	ret = rdma_listen(cm_id, 64);
+	if (ret) {
+		pr_err("Listening on RDMA connection failed, err: %d\n",
+		       ret);
+		goto err_cm;
+	}
+
+	return cm_id;
+
+err_cm:
+	rdma_destroy_id(cm_id);
+err_out:
+
+	return ERR_PTR(ret);
+}
+
+static int rtrs_srv_rdma_init(struct rtrs_srv_ctx *ctx, u16 port)
+{
+	struct sockaddr_in6 sin = {
+		.sin6_family	= AF_INET6,
+		.sin6_addr	= IN6ADDR_ANY_INIT,
+		.sin6_port	= htons(port),
+	};
+	struct sockaddr_ib sib = {
+		.sib_family			= AF_IB,
+		.sib_sid	= cpu_to_be64(RDMA_IB_IP_PS_IB | port),
+		.sib_sid_mask	= cpu_to_be64(0xffffffffffffffffULL),
+		.sib_pkey	= cpu_to_be16(0xffff),
+	};
+	struct rdma_cm_id *cm_ip, *cm_ib;
+	int ret;
+
+	/*
+	 * We accept both IPoIB and IB connections, so we need to keep
+	 * two cm id's, one for each socket type and port space.
+	 * If the cm initialization of one of the id's fails, we abort
+	 * everything.
+	 */
+	cm_ip = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sin, RDMA_PS_TCP);
+	if (IS_ERR(cm_ip))
+		return PTR_ERR(cm_ip);
+
+	cm_ib = rtrs_srv_cm_init(ctx, (struct sockaddr *)&sib, RDMA_PS_IB);
+	if (IS_ERR(cm_ib)) {
+		ret = PTR_ERR(cm_ib);
+		goto free_cm_ip;
+	}
+
+	ctx->cm_id_ip = cm_ip;
+	ctx->cm_id_ib = cm_ib;
+
+	return 0;
+
+free_cm_ip:
+	rdma_destroy_id(cm_ip);
+
+	return ret;
+}
+
+static struct rtrs_srv_ctx *alloc_srv_ctx(struct rtrs_srv_ops *ops)
+{
+	struct rtrs_srv_ctx *ctx;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return NULL;
+
+	ctx->ops = *ops;
+	mutex_init(&ctx->srv_mutex);
+	INIT_LIST_HEAD(&ctx->srv_list);
+
+	return ctx;
+}
+
+static void free_srv_ctx(struct rtrs_srv_ctx *ctx)
+{
+	WARN_ON(!list_empty(&ctx->srv_list));
+	mutex_destroy(&ctx->srv_mutex);
+	kfree(ctx);
+}
+
+/**
+ * rtrs_srv_open() - open RTRS server context
+ * @ops:		callback functions
+ * @port:               port to listen on
+ *
+ * Creates server context with specified callbacks.
+ *
+ * Return a valid pointer on success otherwise PTR_ERR.
+ */
+struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port)
+{
+	struct rtrs_srv_ctx *ctx;
+	int err;
+
+	ctx = alloc_srv_ctx(ops);
+	if (!ctx)
+		return ERR_PTR(-ENOMEM);
+
+	err = rtrs_srv_rdma_init(ctx, port);
+	if (err) {
+		free_srv_ctx(ctx);
+		return ERR_PTR(err);
+	}
+
+	return ctx;
+}
+EXPORT_SYMBOL(rtrs_srv_open);
+
+static void close_sessions(struct rtrs_srv *srv)
+{
+	struct rtrs_srv_sess *sess;
+
+	mutex_lock(&srv->paths_mutex);
+	list_for_each_entry(sess, &srv->paths_list, s.entry)
+		close_sess(sess);
+	mutex_unlock(&srv->paths_mutex);
+}
+
+static void close_ctx(struct rtrs_srv_ctx *ctx)
+{
+	struct rtrs_srv *srv;
+
+	mutex_lock(&ctx->srv_mutex);
+	list_for_each_entry(srv, &ctx->srv_list, ctx_list)
+		close_sessions(srv);
+	mutex_unlock(&ctx->srv_mutex);
+	flush_workqueue(rtrs_wq);
+}
+
+/**
+ * rtrs_srv_close() - close RTRS server context
+ * @ctx: pointer to server context
+ *
+ * Closes RTRS server context with all client sessions.
+ */
+void rtrs_srv_close(struct rtrs_srv_ctx *ctx)
+{
+	rdma_destroy_id(ctx->cm_id_ip);
+	rdma_destroy_id(ctx->cm_id_ib);
+	close_ctx(ctx);
+	free_srv_ctx(ctx);
+}
+EXPORT_SYMBOL(rtrs_srv_close);
+
+static int check_module_params(void)
+{
+	if (sess_queue_depth < 1 || sess_queue_depth > MAX_SESS_QUEUE_DEPTH) {
+		pr_err("Invalid sess_queue_depth value %d, has to be >= %d, <= %d.\n",
+		       sess_queue_depth, 1, MAX_SESS_QUEUE_DEPTH);
+		return -EINVAL;
+	}
+	if (max_chunk_size < 4096 || !is_power_of_2(max_chunk_size)) {
+		pr_err("Invalid max_chunk_size value %d, has to be >= %d and should be power of two.\n",
+		       max_chunk_size, 4096);
+		return -EINVAL;
+	}
+
+	/*
+	 * Check if IB immediate data size is enough to hold the mem_id and the
+	 * offset inside the memory chunk
+	 */
+	if ((ilog2(sess_queue_depth - 1) + 1) +
+	    (ilog2(max_chunk_size - 1) + 1) > MAX_IMM_PAYL_BITS) {
+		pr_err("RDMA immediate size (%db) not enough to encode %d buffers of size %dB. Reduce 'sess_queue_depth' or 'max_chunk_size' parameters.\n",
+		       MAX_IMM_PAYL_BITS, sess_queue_depth, max_chunk_size);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int __init rtrs_server_init(void)
+{
+	int err;
+
+	pr_info("Loading module %s, proto %s: (max_chunk_size: %d (pure IO %ld, headers %ld) , sess_queue_depth: %d, always_invalidate: %d)\n",
+		KBUILD_MODNAME, RTRS_PROTO_VER_STRING,
+		max_chunk_size, max_chunk_size - MAX_HDR_SIZE, MAX_HDR_SIZE,
+		sess_queue_depth, always_invalidate);
+
+	rtrs_rdma_dev_pd_init(0, &dev_pd);
+
+	err = check_module_params();
+	if (err) {
+		pr_err("Failed to load module, invalid module parameters, err: %d\n",
+		       err);
+		return err;
+	}
+	chunk_pool = mempool_create_page_pool(sess_queue_depth * CHUNK_POOL_SZ,
+					      get_order(max_chunk_size));
+	if (!chunk_pool)
+		return -ENOMEM;
+	rtrs_dev_class = class_create(THIS_MODULE, "rtrs-server");
+	if (IS_ERR(rtrs_dev_class)) {
+		err = PTR_ERR(rtrs_dev_class);
+		goto out_chunk_pool;
+	}
+	rtrs_wq = alloc_workqueue("rtrs_server_wq", WQ_MEM_RECLAIM, 0);
+	if (!rtrs_wq) {
+		err = -ENOMEM;
+		goto out_dev_class;
+	}
+
+	return 0;
+
+out_dev_class:
+	class_destroy(rtrs_dev_class);
+out_chunk_pool:
+	mempool_destroy(chunk_pool);
+
+	return err;
+}
+
+static void __exit rtrs_server_exit(void)
+{
+	destroy_workqueue(rtrs_wq);
+	class_destroy(rtrs_dev_class);
+	mempool_destroy(chunk_pool);
+	rtrs_rdma_dev_pd_deinit(&dev_pd);
+}
+
+module_init(rtrs_server_init);
+module_exit(rtrs_server_exit);
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
new file mode 100644
index 000000000000..dc95b0932f0d
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#ifndef RTRS_SRV_H
+#define RTRS_SRV_H
+
+#include <linux/device.h>
+#include <linux/refcount.h>
+#include "rtrs-pri.h"
+
+/*
+ * enum rtrs_srv_state - Server states.
+ */
+enum rtrs_srv_state {
+	RTRS_SRV_CONNECTING,
+	RTRS_SRV_CONNECTED,
+	RTRS_SRV_CLOSING,
+	RTRS_SRV_CLOSED,
+};
+
+/* stats for Read and write operation.
+ * see Documentation/ABI/testing/sysfs-class-rtrs-server for details
+ */
+struct rtrs_srv_stats_rdma_stats {
+	struct {
+		atomic64_t	cnt;
+		atomic64_t	size_total;
+	} dir[2];
+};
+
+struct rtrs_srv_stats {
+	struct kobject				kobj_stats;
+	struct rtrs_srv_stats_rdma_stats	rdma_stats;
+	struct rtrs_srv_sess			*sess;
+};
+
+struct rtrs_srv_con {
+	struct rtrs_con		c;
+	atomic_t		wr_cnt;
+	atomic_t		sq_wr_avail;
+	struct list_head	rsp_wr_wait_list;
+	spinlock_t		rsp_wr_wait_lock;
+};
+
+/* IO context in rtrs_srv, each io has one */
+struct rtrs_srv_op {
+	struct rtrs_srv_con		*con;
+	u32				msg_id;
+	u8				dir;
+	struct rtrs_msg_rdma_read	*rd_msg;
+	struct ib_rdma_wr		tx_wr;
+	struct ib_sge			tx_sg;
+	struct list_head		wait_list;
+	int				status;
+};
+
+/*
+ * server side memory region context, when always_invalidate=Y, we need
+ * queue_depth of memory regrion to invalidate each memory region.
+ */
+struct rtrs_srv_mr {
+	struct ib_mr	*mr;
+	struct sg_table	sgt;
+	struct ib_cqe	inv_cqe;	/* only for always_invalidate=true */
+	u32		msg_id;		/* only for always_invalidate=true */
+	u32		msg_off;	/* only for always_invalidate=true */
+	struct rtrs_iu	*iu;		/* send buffer for new rkey msg */
+};
+
+struct rtrs_srv_sess {
+	struct rtrs_sess	s;
+	struct rtrs_srv	*srv;
+	struct work_struct	close_work;
+	enum rtrs_srv_state	state;
+	spinlock_t		state_lock;
+	int			cur_cq_vector;
+	struct rtrs_srv_op	**ops_ids;
+	atomic_t		ids_inflight;
+	wait_queue_head_t	ids_waitq;
+	struct rtrs_srv_mr	*mrs;
+	unsigned int		mrs_num;
+	dma_addr_t		*dma_addr;
+	bool			established;
+	unsigned int		mem_bits;
+	struct kobject		kobj;
+	struct rtrs_srv_stats	*stats;
+};
+
+struct rtrs_srv {
+	struct list_head	paths_list;
+	int			paths_up;
+	struct mutex		paths_ev_mutex;
+	size_t			paths_num;
+	struct mutex		paths_mutex;
+	uuid_t			paths_uuid;
+	refcount_t		refcount;
+	struct rtrs_srv_ctx	*ctx;
+	struct list_head	ctx_list;
+	void			*priv;
+	size_t			queue_depth;
+	struct page		**chunks;
+	struct device		dev;
+	unsigned int		dev_ref;
+	struct kobject		*kobj_paths;
+};
+
+struct rtrs_srv_ctx {
+	struct rtrs_srv_ops ops;
+	struct rdma_cm_id *cm_id_ip;
+	struct rdma_cm_id *cm_id_ib;
+	struct mutex srv_mutex;
+	struct list_head srv_list;
+};
+
+extern struct class *rtrs_dev_class;
+
+void close_sess(struct rtrs_srv_sess *sess);
+
+static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s,
+					      size_t size, int d)
+{
+	atomic64_inc(&s->rdma_stats.dir[d].cnt);
+	atomic64_add(size, &s->rdma_stats.dir[d].size_total);
+}
+
+/* functions which are implemented in rtrs-srv-stats.c */
+int rtrs_srv_reset_rdma_stats(struct rtrs_srv_stats *stats, bool enable);
+ssize_t rtrs_srv_stats_rdma_to_str(struct rtrs_srv_stats *stats,
+				    char *page, size_t len);
+int rtrs_srv_reset_wc_completion_stats(struct rtrs_srv_stats *stats,
+					bool enable);
+int rtrs_srv_stats_wc_completion_to_str(struct rtrs_srv_stats *stats, char *buf,
+					 size_t len);
+int rtrs_srv_reset_all_stats(struct rtrs_srv_stats *stats, bool enable);
+ssize_t rtrs_srv_reset_all_help(struct rtrs_srv_stats *stats,
+				 char *page, size_t len);
+
+/* functions which are implemented in rtrs-srv-sysfs.c */
+int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess);
+void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess);
+
+#endif /* RTRS_SRV_H */
diff --git a/drivers/infiniband/ulp/rtrs/rtrs.c b/drivers/infiniband/ulp/rtrs/rtrs.c
new file mode 100644
index 000000000000..ff1093d6e4bc
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs.c
@@ -0,0 +1,612 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#undef pr_fmt
+#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
+
+#include <linux/module.h>
+#include <linux/inet.h>
+
+#include "rtrs-pri.h"
+#include "rtrs-log.h"
+
+MODULE_DESCRIPTION("RDMA Transport Core");
+MODULE_LICENSE("GPL");
+
+struct rtrs_iu *rtrs_iu_alloc(u32 queue_size, size_t size, gfp_t gfp_mask,
+			      struct ib_device *dma_dev,
+			      enum dma_data_direction dir,
+			      void (*done)(struct ib_cq *cq, struct ib_wc *wc))
+{
+	struct rtrs_iu *ius, *iu;
+	int i;
+
+	ius = kcalloc(queue_size, sizeof(*ius), gfp_mask);
+	if (!ius)
+		return NULL;
+	for (i = 0; i < queue_size; i++) {
+		iu = &ius[i];
+		iu->buf = kzalloc(size, gfp_mask);
+		if (!iu->buf)
+			goto err;
+
+		iu->dma_addr = ib_dma_map_single(dma_dev, iu->buf, size, dir);
+		if (ib_dma_mapping_error(dma_dev, iu->dma_addr))
+			goto err;
+
+		iu->cqe.done  = done;
+		iu->size      = size;
+		iu->direction = dir;
+	}
+	return ius;
+err:
+	rtrs_iu_free(ius, dir, dma_dev, i);
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(rtrs_iu_alloc);
+
+void rtrs_iu_free(struct rtrs_iu *ius, enum dma_data_direction dir,
+		   struct ib_device *ibdev, u32 queue_size)
+{
+	struct rtrs_iu *iu;
+	int i;
+
+	if (!ius)
+		return;
+
+	for (i = 0; i < queue_size; i++) {
+		iu = &ius[i];
+		ib_dma_unmap_single(ibdev, iu->dma_addr, iu->size, dir);
+		kfree(iu->buf);
+	}
+	kfree(ius);
+}
+EXPORT_SYMBOL_GPL(rtrs_iu_free);
+
+int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu)
+{
+	struct rtrs_sess *sess = con->sess;
+	struct ib_recv_wr wr;
+	struct ib_sge list;
+
+	list.addr   = iu->dma_addr;
+	list.length = iu->size;
+	list.lkey   = sess->dev->ib_pd->local_dma_lkey;
+
+	if (list.length == 0) {
+		rtrs_wrn(con->sess,
+			  "Posting receive work request failed, sg list is empty\n");
+		return -EINVAL;
+	}
+	wr = (struct ib_recv_wr) {
+		.wr_cqe  = &iu->cqe,
+		.sg_list = &list,
+		.num_sge = 1,
+	};
+
+	return ib_post_recv(con->qp, &wr, NULL);
+}
+EXPORT_SYMBOL_GPL(rtrs_iu_post_recv);
+
+int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe)
+{
+	struct ib_recv_wr wr;
+
+	wr = (struct ib_recv_wr) {
+		.wr_cqe  = cqe,
+	};
+
+	return ib_post_recv(con->qp, &wr, NULL);
+}
+EXPORT_SYMBOL_GPL(rtrs_post_recv_empty);
+
+int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,
+		       struct ib_send_wr *head)
+{
+	struct rtrs_sess *sess = con->sess;
+	struct ib_send_wr wr;
+	struct ib_sge list;
+
+	if (WARN_ON(size == 0))
+		return -EINVAL;
+
+	list.addr   = iu->dma_addr;
+	list.length = size;
+	list.lkey   = sess->dev->ib_pd->local_dma_lkey;
+
+	wr = (struct ib_send_wr) {
+		.wr_cqe     = &iu->cqe,
+		.sg_list    = &list,
+		.num_sge    = 1,
+		.opcode     = IB_WR_SEND,
+		.send_flags = IB_SEND_SIGNALED,
+	};
+
+	if (head) {
+		struct ib_send_wr *tail = head;
+
+		while (tail->next)
+			tail = tail->next;
+		tail->next = &wr;
+	} else {
+		head = &wr;
+	}
+
+	return ib_post_send(con->qp, head, NULL);
+}
+EXPORT_SYMBOL_GPL(rtrs_iu_post_send);
+
+int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu,
+				 struct ib_sge *sge, unsigned int num_sge,
+				 u32 rkey, u64 rdma_addr, u32 imm_data,
+				 enum ib_send_flags flags,
+				 struct ib_send_wr *head)
+{
+	struct ib_rdma_wr wr;
+	int i;
+
+	wr = (struct ib_rdma_wr) {
+		.wr.wr_cqe	  = &iu->cqe,
+		.wr.sg_list	  = sge,
+		.wr.num_sge	  = num_sge,
+		.rkey		  = rkey,
+		.remote_addr	  = rdma_addr,
+		.wr.opcode	  = IB_WR_RDMA_WRITE_WITH_IMM,
+		.wr.ex.imm_data = cpu_to_be32(imm_data),
+		.wr.send_flags  = flags,
+	};
+
+	/*
+	 * If one of the sges has 0 size, the operation will fail with a
+	 * length error
+	 */
+	for (i = 0; i < num_sge; i++)
+		if (WARN_ON(sge[i].length == 0))
+			return -EINVAL;
+
+	if (head) {
+		struct ib_send_wr *tail = head;
+
+		while (tail->next)
+			tail = tail->next;
+		tail->next = &wr.wr;
+	} else {
+		head = &wr.wr;
+	}
+
+	return ib_post_send(con->qp, head, NULL);
+}
+EXPORT_SYMBOL_GPL(rtrs_iu_post_rdma_write_imm);
+
+int rtrs_post_rdma_write_imm_empty(struct rtrs_con *con, struct ib_cqe *cqe,
+				    u32 imm_data, enum ib_send_flags flags,
+				    struct ib_send_wr *head)
+{
+	struct ib_send_wr wr;
+
+	wr = (struct ib_send_wr) {
+		.wr_cqe	= cqe,
+		.send_flags	= flags,
+		.opcode	= IB_WR_RDMA_WRITE_WITH_IMM,
+		.ex.imm_data	= cpu_to_be32(imm_data),
+	};
+
+	if (head) {
+		struct ib_send_wr *tail = head;
+
+		while (tail->next)
+			tail = tail->next;
+		tail->next = &wr;
+	} else {
+		head = &wr;
+	}
+
+	return ib_post_send(con->qp, head, NULL);
+}
+EXPORT_SYMBOL_GPL(rtrs_post_rdma_write_imm_empty);
+
+static void qp_event_handler(struct ib_event *ev, void *ctx)
+{
+	struct rtrs_con *con = ctx;
+
+	switch (ev->event) {
+	case IB_EVENT_COMM_EST:
+		rtrs_info(con->sess, "QP event %s (%d) received\n",
+			   ib_event_msg(ev->event), ev->event);
+		rdma_notify(con->cm_id, IB_EVENT_COMM_EST);
+		break;
+	default:
+		rtrs_info(con->sess, "Unhandled QP event %s (%d) received\n",
+			   ib_event_msg(ev->event), ev->event);
+		break;
+	}
+}
+
+static int create_cq(struct rtrs_con *con, int cq_vector, u16 cq_size,
+		     enum ib_poll_context poll_ctx)
+{
+	struct rdma_cm_id *cm_id = con->cm_id;
+	struct ib_cq *cq;
+
+	cq = ib_alloc_cq(cm_id->device, con, cq_size,
+			 cq_vector, poll_ctx);
+	if (IS_ERR(cq)) {
+		rtrs_err(con->sess, "Creating completion queue failed, errno: %ld\n",
+			  PTR_ERR(cq));
+		return PTR_ERR(cq);
+	}
+	con->cq = cq;
+
+	return 0;
+}
+
+static int create_qp(struct rtrs_con *con, struct ib_pd *pd,
+		     u16 wr_queue_size, u32 max_sge)
+{
+	struct ib_qp_init_attr init_attr = {NULL};
+	struct rdma_cm_id *cm_id = con->cm_id;
+	int ret;
+
+	init_attr.cap.max_send_wr = wr_queue_size;
+	init_attr.cap.max_recv_wr = wr_queue_size;
+	init_attr.cap.max_recv_sge = 1;
+	init_attr.event_handler = qp_event_handler;
+	init_attr.qp_context = con;
+	init_attr.cap.max_send_sge = max_sge;
+
+	init_attr.qp_type = IB_QPT_RC;
+	init_attr.send_cq = con->cq;
+	init_attr.recv_cq = con->cq;
+	init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
+
+	ret = rdma_create_qp(cm_id, pd, &init_attr);
+	if (ret) {
+		rtrs_err(con->sess, "Creating QP failed, err: %d\n", ret);
+		return ret;
+	}
+	con->qp = cm_id->qp;
+
+	return ret;
+}
+
+int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con,
+		       u32 max_send_sge, int cq_vector, u16 cq_size,
+		       u16 wr_queue_size, enum ib_poll_context poll_ctx)
+{
+	int err;
+
+	err = create_cq(con, cq_vector, cq_size, poll_ctx);
+	if (err)
+		return err;
+
+	err = create_qp(con, sess->dev->ib_pd, wr_queue_size, max_send_sge);
+	if (err) {
+		ib_free_cq(con->cq);
+		con->cq = NULL;
+		return err;
+	}
+	con->sess = sess;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rtrs_cq_qp_create);
+
+void rtrs_cq_qp_destroy(struct rtrs_con *con)
+{
+	if (con->qp) {
+		rdma_destroy_qp(con->cm_id);
+		con->qp = NULL;
+	}
+	if (con->cq) {
+		ib_free_cq(con->cq);
+		con->cq = NULL;
+	}
+}
+EXPORT_SYMBOL_GPL(rtrs_cq_qp_destroy);
+
+static void schedule_hb(struct rtrs_sess *sess)
+{
+	queue_delayed_work(sess->hb_wq, &sess->hb_dwork,
+			   msecs_to_jiffies(sess->hb_interval_ms));
+}
+
+void rtrs_send_hb_ack(struct rtrs_sess *sess)
+{
+	struct rtrs_con *usr_con = sess->con[0];
+	u32 imm;
+	int err;
+
+	imm = rtrs_to_imm(RTRS_HB_ACK_IMM, 0);
+	err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm,
+					      IB_SEND_SIGNALED, NULL);
+	if (err) {
+		sess->hb_err_handler(usr_con);
+		return;
+	}
+}
+EXPORT_SYMBOL_GPL(rtrs_send_hb_ack);
+
+static void hb_work(struct work_struct *work)
+{
+	struct rtrs_con *usr_con;
+	struct rtrs_sess *sess;
+	u32 imm;
+	int err;
+
+	sess = container_of(to_delayed_work(work), typeof(*sess), hb_dwork);
+	usr_con = sess->con[0];
+
+	if (sess->hb_missed_cnt > sess->hb_missed_max) {
+		sess->hb_err_handler(usr_con);
+		return;
+	}
+	if (sess->hb_missed_cnt++) {
+		/* Reschedule work without sending hb */
+		schedule_hb(sess);
+		return;
+	}
+	imm = rtrs_to_imm(RTRS_HB_MSG_IMM, 0);
+	err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm,
+					      IB_SEND_SIGNALED, NULL);
+	if (err) {
+		sess->hb_err_handler(usr_con);
+		return;
+	}
+
+	schedule_hb(sess);
+}
+
+void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe,
+		  unsigned int interval_ms, unsigned int missed_max,
+		  void (*err_handler)(struct rtrs_con *con),
+		  struct workqueue_struct *wq)
+{
+	sess->hb_cqe = cqe;
+	sess->hb_interval_ms = interval_ms;
+	sess->hb_err_handler = err_handler;
+	sess->hb_wq = wq;
+	sess->hb_missed_max = missed_max;
+	sess->hb_missed_cnt = 0;
+	INIT_DELAYED_WORK(&sess->hb_dwork, hb_work);
+}
+EXPORT_SYMBOL_GPL(rtrs_init_hb);
+
+void rtrs_start_hb(struct rtrs_sess *sess)
+{
+	schedule_hb(sess);
+}
+EXPORT_SYMBOL_GPL(rtrs_start_hb);
+
+void rtrs_stop_hb(struct rtrs_sess *sess)
+{
+	cancel_delayed_work_sync(&sess->hb_dwork);
+	sess->hb_missed_cnt = 0;
+	sess->hb_missed_max = 0;
+}
+EXPORT_SYMBOL_GPL(rtrs_stop_hb);
+
+static int rtrs_str_gid_to_sockaddr(const char *addr, size_t len,
+				     short port, struct sockaddr_storage *dst)
+{
+	struct sockaddr_ib *dst_ib = (struct sockaddr_ib *)dst;
+	int ret;
+
+	/*
+	 * We can use some of the IPv6 functions since GID is a valid
+	 * IPv6 address format
+	 */
+	ret = in6_pton(addr, len, dst_ib->sib_addr.sib_raw, '\0', NULL);
+	if (ret == 0)
+		return -EINVAL;
+
+	dst_ib->sib_family = AF_IB;
+	/*
+	 * Use the same TCP server port number as the IB service ID
+	 * on the IB port space range
+	 */
+	dst_ib->sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | port);
+	dst_ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
+	dst_ib->sib_pkey = cpu_to_be16(0xffff);
+
+	return 0;
+}
+
+/**
+ * rtrs_str_to_sockaddr() - Convert rtrs address string to sockaddr
+ * @addr:	String representation of an addr (IPv4, IPv6 or IB GID):
+ *              - "ip:192.168.1.1"
+ *              - "ip:fe80::200:5aee:feaa:20a2"
+ *              - "gid:fe80::200:5aee:feaa:20a2"
+ * @len:        String address length
+ * @port:	Destination port
+ * @dst:	Destination sockaddr structure
+ *
+ * Returns 0 if conversion successful. Non-zero on error.
+ */
+static int rtrs_str_to_sockaddr(const char *addr, size_t len,
+				u16 port, struct sockaddr_storage *dst)
+{
+	if (strncmp(addr, "gid:", 4) == 0) {
+		return rtrs_str_gid_to_sockaddr(addr + 4, len - 4, port, dst);
+	} else if (strncmp(addr, "ip:", 3) == 0) {
+		char port_str[8];
+		char *cpy;
+		int err;
+
+		snprintf(port_str, sizeof(port_str), "%u", port);
+		cpy = kstrndup(addr + 3, len - 3, GFP_KERNEL);
+		err = cpy ? inet_pton_with_scope(&init_net, AF_UNSPEC,
+						 cpy, port_str, dst) : -ENOMEM;
+		kfree(cpy);
+
+		return err;
+	}
+	return -EPROTONOSUPPORT;
+}
+
+/**
+ * sockaddr_to_str() - convert sockaddr to a string.
+ * @addr:	the sockadddr structure to be converted.
+ * @buf:	string containing socket addr.
+ * @len:	string length.
+ *
+ * The return value is the number of characters written into buf not
+ * including the trailing '\0'. If len is == 0 the function returns 0..
+ */
+int sockaddr_to_str(const struct sockaddr *addr, char *buf, size_t len)
+{
+
+	switch (addr->sa_family) {
+	case AF_IB:
+		return scnprintf(buf, len, "gid:%pI6",
+			&((struct sockaddr_ib *)addr)->sib_addr.sib_raw);
+	case AF_INET:
+		return scnprintf(buf, len, "ip:%pI4",
+			&((struct sockaddr_in *)addr)->sin_addr);
+	case AF_INET6:
+		return scnprintf(buf, len, "ip:%pI6c",
+			  &((struct sockaddr_in6 *)addr)->sin6_addr);
+	}
+	return scnprintf(buf, len, "<invalid address family>");
+}
+EXPORT_SYMBOL(sockaddr_to_str);
+
+/**
+ * rtrs_addr_to_sockaddr() - convert path string "src,dst" or "src@dst"
+ * to sockaddreses
+ * @str:	string containing source and destination addr of a path
+ *		separated by ',' or '@' I.e. "ip:1.1.1.1,ip:1.1.1.2" or
+ *		"ip:1.1.1.1@ip:1.1.1.2". If str contains only one address it's
+ *		considered to be destination.
+ * @len:	string length
+ * @port:	Destination port number.
+ * @addr:	will be set to the source/destination address or to NULL
+ *		if str doesn't contain any source address.
+ *
+ * Returns zero if conversion successful. Non-zero otherwise.
+ */
+int rtrs_addr_to_sockaddr(const char *str, size_t len, u16 port,
+			  struct rtrs_addr *addr)
+{
+	const char *d;
+
+	d = strchr(str, ',');
+	if (!d)
+		d = strchr(str, '@');
+	if (d) {
+		if (rtrs_str_to_sockaddr(str, d - str, 0, addr->src))
+			return -EINVAL;
+		d += 1;
+		len -= d - str;
+		str  = d;
+
+	} else {
+		addr->src = NULL;
+	}
+	return rtrs_str_to_sockaddr(str, len, port, addr->dst);
+}
+EXPORT_SYMBOL(rtrs_addr_to_sockaddr);
+
+void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
+			    struct rtrs_rdma_dev_pd *pool)
+{
+	WARN_ON(pool->ops && (!pool->ops->alloc ^ !pool->ops->free));
+	INIT_LIST_HEAD(&pool->list);
+	mutex_init(&pool->mutex);
+	pool->pd_flags = pd_flags;
+}
+EXPORT_SYMBOL(rtrs_rdma_dev_pd_init);
+
+void rtrs_rdma_dev_pd_deinit(struct rtrs_rdma_dev_pd *pool)
+{
+	mutex_destroy(&pool->mutex);
+	WARN_ON(!list_empty(&pool->list));
+}
+EXPORT_SYMBOL(rtrs_rdma_dev_pd_deinit);
+
+static void dev_free(struct kref *ref)
+{
+	struct rtrs_rdma_dev_pd *pool;
+	struct rtrs_ib_dev *dev;
+
+	dev = container_of(ref, typeof(*dev), ref);
+	pool = dev->pool;
+
+	mutex_lock(&pool->mutex);
+	list_del(&dev->entry);
+	mutex_unlock(&pool->mutex);
+
+	if (pool->ops && pool->ops->deinit)
+		pool->ops->deinit(dev);
+
+	ib_dealloc_pd(dev->ib_pd);
+
+	if (pool->ops && pool->ops->free)
+		pool->ops->free(dev);
+	else
+		kfree(dev);
+}
+
+int rtrs_ib_dev_put(struct rtrs_ib_dev *dev)
+{
+	return kref_put(&dev->ref, dev_free);
+}
+EXPORT_SYMBOL(rtrs_ib_dev_put);
+
+static int rtrs_ib_dev_get(struct rtrs_ib_dev *dev)
+{
+	return kref_get_unless_zero(&dev->ref);
+}
+
+struct rtrs_ib_dev *
+rtrs_ib_dev_find_or_add(struct ib_device *ib_dev,
+			 struct rtrs_rdma_dev_pd *pool)
+{
+	struct rtrs_ib_dev *dev;
+
+	mutex_lock(&pool->mutex);
+	list_for_each_entry(dev, &pool->list, entry) {
+		if (dev->ib_dev->node_guid == ib_dev->node_guid &&
+		    rtrs_ib_dev_get(dev))
+			goto out_unlock;
+	}
+	mutex_unlock(&pool->mutex);
+	if (pool->ops && pool->ops->alloc)
+		dev = pool->ops->alloc();
+	else
+		dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (IS_ERR_OR_NULL(dev))
+		goto out_err;
+
+	kref_init(&dev->ref);
+	dev->pool = pool;
+	dev->ib_dev = ib_dev;
+	dev->ib_pd = ib_alloc_pd(ib_dev, pool->pd_flags);
+	if (IS_ERR(dev->ib_pd))
+		goto out_free_dev;
+
+	if (pool->ops && pool->ops->init && pool->ops->init(dev))
+		goto out_free_pd;
+
+	mutex_lock(&pool->mutex);
+	list_add(&dev->entry, &pool->list);
+out_unlock:
+	mutex_unlock(&pool->mutex);
+	return dev;
+
+out_free_pd:
+	ib_dealloc_pd(dev->ib_pd);
+out_free_dev:
+	if (pool->ops && pool->ops->free)
+		pool->ops->free(dev);
+	else
+		kfree(dev);
+out_err:
+	return NULL;
+}
+EXPORT_SYMBOL(rtrs_ib_dev_find_or_add);
diff --git a/drivers/infiniband/ulp/rtrs/rtrs.h b/drivers/infiniband/ulp/rtrs/rtrs.h
new file mode 100644
index 000000000000..9af750f4d783
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs.h
@@ -0,0 +1,196 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+#ifndef RTRS_H
+#define RTRS_H
+
+#include <linux/socket.h>
+#include <linux/scatterlist.h>
+
+struct rtrs_permit;
+struct rtrs_clt;
+struct rtrs_srv_ctx;
+struct rtrs_srv;
+struct rtrs_srv_op;
+
+/*
+ * RDMA transport (RTRS) client API
+ */
+
+/**
+ * enum rtrs_clt_link_ev - Events about connectivity state of a client
+ * @RTRS_CLT_LINK_EV_RECONNECTED	Client was reconnected.
+ * @RTRS_CLT_LINK_EV_DISCONNECTED	Client was disconnected.
+ */
+enum rtrs_clt_link_ev {
+	RTRS_CLT_LINK_EV_RECONNECTED,
+	RTRS_CLT_LINK_EV_DISCONNECTED,
+};
+
+/**
+ * Source and destination address of a path to be established
+ */
+struct rtrs_addr {
+	struct sockaddr_storage *src;
+	struct sockaddr_storage *dst;
+};
+
+/**
+ * rtrs_clt_ops - it holds the link event callback and private pointer.
+ * @priv: User supplied private data.
+ * @link_ev: Event notification callback function for connection state changes
+ *	@priv: User supplied data that was passed to rtrs_clt_open()
+ *	@ev: Occurred event
+ */
+struct rtrs_clt_ops {
+	void	*priv;
+	void	(*link_ev)(void *priv, enum rtrs_clt_link_ev ev);
+};
+
+struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
+				 const char *sessname,
+				 const struct rtrs_addr *paths,
+				 size_t path_cnt, u16 port,
+				 size_t pdu_sz, u8 reconnect_delay_sec,
+				 u16 max_segments,
+				 size_t max_segment_size,
+				 s16 max_reconnect_attempts);
+
+void rtrs_clt_close(struct rtrs_clt *sess);
+
+/**
+ * rtrs_permit_to_pdu() - converts rtrs_permit to opaque pdu pointer
+ * @permit: RTRS permit pointer, it associates the memory allocation for future
+ *          RDMA operation.
+ */
+void *rtrs_permit_to_pdu(struct rtrs_permit *permit);
+
+enum {
+	RTRS_PERMIT_NOWAIT = 0,
+	RTRS_PERMIT_WAIT   = 1,
+};
+
+/**
+ * enum rtrs_clt_con_type() type of ib connection to use with a given
+ * rtrs_permit
+ * @ADMIN_CON - use connection reserved for "service" messages
+ * @IO_CON - use a connection reserved for IO
+ */
+enum rtrs_clt_con_type {
+	RTRS_ADMIN_CON,
+	RTRS_IO_CON
+};
+
+struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *sess,
+				    enum rtrs_clt_con_type con_type,
+				    int wait);
+
+void rtrs_clt_put_permit(struct rtrs_clt *sess, struct rtrs_permit *permit);
+
+/**
+ * rtrs_clt_req_ops - it holds the request confirmation callback
+ * and a private pointer.
+ * @priv: User supplied private data.
+ * @conf_fn:	callback function to be called as confirmation
+ *	@priv:	User provided data, passed back with corresponding
+ *		@(conf) confirmation.
+ *	@errno: error number.
+ */
+struct rtrs_clt_req_ops {
+	void	*priv;
+	void	(*conf_fn)(void *priv, int errno);
+};
+
+int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,
+		     struct rtrs_clt *sess, struct rtrs_permit *permit,
+		     const struct kvec *vec, size_t nr, size_t len,
+		     struct scatterlist *sg, unsigned int sg_cnt);
+
+/**
+ * rtrs_attrs - RTRS session attributes
+ */
+struct rtrs_attrs {
+	u32		queue_depth;
+	u32		max_io_size;
+	u8		sessname[NAME_MAX];
+	struct kobject	*sess_kobj;
+};
+
+int rtrs_clt_query(struct rtrs_clt *sess, struct rtrs_attrs *attr);
+
+/*
+ * Here goes RTRS server API
+ */
+
+/**
+ * enum rtrs_srv_link_ev - Server link events
+ * @RTRS_SRV_LINK_EV_CONNECTED:	Connection from client established
+ * @RTRS_SRV_LINK_EV_DISCONNECTED:	Connection was disconnected, all
+ *					connection RTRS resources were freed.
+ */
+enum rtrs_srv_link_ev {
+	RTRS_SRV_LINK_EV_CONNECTED,
+	RTRS_SRV_LINK_EV_DISCONNECTED,
+};
+
+struct rtrs_srv_ops {
+	/**
+	 * rdma_ev():		Event notification for RDMA operations
+	 *			If the callback returns a value != 0, an error
+	 *			message for the data transfer will be sent to
+	 *			the client.
+
+	 *	@sess:		Session
+	 *	@priv:		Private data set by rtrs_srv_set_sess_priv()
+	 *	@id:		internal RTRS operation id
+	 *	@dir:		READ/WRITE
+	 *	@data:		Pointer to (bidirectional) rdma memory area:
+	 *			- in case of %RTRS_SRV_RDMA_EV_RECV contains
+	 *			data sent by the client
+	 *			- in case of %RTRS_SRV_RDMA_EV_WRITE_REQ points
+	 *			to the memory area where the response is to be
+	 *			written to
+	 *	@datalen:	Size of the memory area in @data
+	 *	@usr:		The extra user message sent by the client (%vec)
+	 *	@usrlen:	Size of the user message
+	 */
+	int (*rdma_ev)(struct rtrs_srv *sess, void *priv,
+		       struct rtrs_srv_op *id, int dir,
+		       void *data, size_t datalen, const void *usr,
+		       size_t usrlen);
+	/**
+	 * link_ev():		Events about connectivity state changes
+	 *			If the callback returns != 0 and the event
+	 *			%RTRS_SRV_LINK_EV_CONNECTED the corresponding
+	 *			session will be destroyed.
+	 *	@sess:		Session
+	 *	@ev:		event
+	 *	@priv:		Private data from user if previously set with
+	 *			rtrs_srv_set_sess_priv()
+	 */
+	int (*link_ev)(struct rtrs_srv *sess, enum rtrs_srv_link_ev ev,
+		       void *priv);
+};
+
+struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port);
+
+void rtrs_srv_close(struct rtrs_srv_ctx *ctx);
+
+bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int errno);
+
+void rtrs_srv_set_sess_priv(struct rtrs_srv *sess, void *priv);
+
+int rtrs_srv_get_sess_name(struct rtrs_srv *sess, char *sessname, size_t len);
+
+int rtrs_srv_get_queue_depth(struct rtrs_srv *sess);
+
+int rtrs_addr_to_sockaddr(const char *str, size_t len, u16 port,
+			  struct rtrs_addr *addr);
+
+int sockaddr_to_str(const struct sockaddr *addr, char *buf, size_t len);
+#endif
diff --git a/drivers/infiniband/ulp/srp/Kconfig b/drivers/infiniband/ulp/srp/Kconfig
index 6f5e7b3a3864..67cd63d1399c 100644
--- a/drivers/infiniband/ulp/srp/Kconfig
+++ b/drivers/infiniband/ulp/srp/Kconfig
@@ -3,7 +3,7 @@ config INFINIBAND_SRP
 	tristate "InfiniBand SCSI RDMA Protocol"
 	depends on SCSI && INFINIBAND_ADDR_TRANS
 	select SCSI_SRP_ATTRS
-	---help---
+	help
 	  Support for the SCSI RDMA Protocol over InfiniBand.  This
 	  allows you to access storage devices that speak SRP over
 	  InfiniBand.
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index cd1181c39ed2..d8fcd21ab472 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -71,7 +71,6 @@ static unsigned int srp_sg_tablesize;
 static unsigned int cmd_sg_entries;
 static unsigned int indirect_sg_entries;
 static bool allow_ext_sg;
-static bool prefer_fr = true;
 static bool register_always = true;
 static bool never_register;
 static int topspin_workarounds = 1;
@@ -95,10 +94,6 @@ module_param(topspin_workarounds, int, 0444);
 MODULE_PARM_DESC(topspin_workarounds,
 		 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
 
-module_param(prefer_fr, bool, 0444);
-MODULE_PARM_DESC(prefer_fr,
-"Whether to use fast registration if both FMR and fast registration are supported");
-
 module_param(register_always, bool, 0444);
 MODULE_PARM_DESC(register_always,
 		 "Use memory registration even for contiguous memory regions");
@@ -146,7 +141,7 @@ module_param(ch_count, uint, 0444);
 MODULE_PARM_DESC(ch_count,
 		 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
 
-static void srp_add_one(struct ib_device *device);
+static int srp_add_one(struct ib_device *device);
 static void srp_remove_one(struct ib_device *device, void *client_data);
 static void srp_rename_dev(struct ib_device *device, void *client_data);
 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
@@ -388,24 +383,6 @@ static int srp_new_cm_id(struct srp_rdma_ch *ch)
 		srp_new_ib_cm_id(ch);
 }
 
-static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
-{
-	struct srp_device *dev = target->srp_host->srp_dev;
-	struct ib_fmr_pool_param fmr_param;
-
-	memset(&fmr_param, 0, sizeof(fmr_param));
-	fmr_param.pool_size	    = target->mr_pool_size;
-	fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
-	fmr_param.cache		    = 1;
-	fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
-	fmr_param.page_shift	    = ilog2(dev->mr_page_size);
-	fmr_param.access	    = (IB_ACCESS_LOCAL_WRITE |
-				       IB_ACCESS_REMOTE_WRITE |
-				       IB_ACCESS_REMOTE_READ);
-
-	return ib_create_fmr_pool(dev->pd, &fmr_param);
-}
-
 /**
  * srp_destroy_fr_pool() - free the resources owned by a pool
  * @pool: Fast registration pool to be destroyed.
@@ -556,7 +533,6 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 	struct ib_qp_init_attr *init_attr;
 	struct ib_cq *recv_cq, *send_cq;
 	struct ib_qp *qp;
-	struct ib_fmr_pool *fmr_pool = NULL;
 	struct srp_fr_pool *fr_pool = NULL;
 	const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
 	int ret;
@@ -619,14 +595,6 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 				     "FR pool allocation failed (%d)\n", ret);
 			goto err_qp;
 		}
-	} else if (dev->use_fmr) {
-		fmr_pool = srp_alloc_fmr_pool(target);
-		if (IS_ERR(fmr_pool)) {
-			ret = PTR_ERR(fmr_pool);
-			shost_printk(KERN_WARNING, target->scsi_host, PFX
-				     "FMR pool allocation failed (%d)\n", ret);
-			goto err_qp;
-		}
 	}
 
 	if (ch->qp)
@@ -644,10 +612,6 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 		if (ch->fr_pool)
 			srp_destroy_fr_pool(ch->fr_pool);
 		ch->fr_pool = fr_pool;
-	} else if (dev->use_fmr) {
-		if (ch->fmr_pool)
-			ib_destroy_fmr_pool(ch->fmr_pool);
-		ch->fmr_pool = fmr_pool;
 	}
 
 	kfree(init_attr);
@@ -702,9 +666,6 @@ static void srp_free_ch_ib(struct srp_target_port *target,
 	if (dev->use_fast_reg) {
 		if (ch->fr_pool)
 			srp_destroy_fr_pool(ch->fr_pool);
-	} else if (dev->use_fmr) {
-		if (ch->fmr_pool)
-			ib_destroy_fmr_pool(ch->fmr_pool);
 	}
 
 	srp_destroy_qp(ch);
@@ -1017,12 +978,8 @@ static void srp_free_req_data(struct srp_target_port *target,
 
 	for (i = 0; i < target->req_ring_size; ++i) {
 		req = &ch->req_ring[i];
-		if (dev->use_fast_reg) {
+		if (dev->use_fast_reg)
 			kfree(req->fr_list);
-		} else {
-			kfree(req->fmr_list);
-			kfree(req->map_page);
-		}
 		if (req->indirect_dma_addr) {
 			ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 					    target->indirect_size,
@@ -1056,16 +1013,8 @@ static int srp_alloc_req_data(struct srp_rdma_ch *ch)
 					GFP_KERNEL);
 		if (!mr_list)
 			goto out;
-		if (srp_dev->use_fast_reg) {
+		if (srp_dev->use_fast_reg)
 			req->fr_list = mr_list;
-		} else {
-			req->fmr_list = mr_list;
-			req->map_page = kmalloc_array(srp_dev->max_pages_per_mr,
-						      sizeof(void *),
-						      GFP_KERNEL);
-			if (!req->map_page)
-				goto out;
-		}
 		req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 		if (!req->indirect_desc)
 			goto out;
@@ -1272,11 +1221,6 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd,
 		if (req->nmdesc)
 			srp_fr_pool_put(ch->fr_pool, req->fr_list,
 					req->nmdesc);
-	} else if (dev->use_fmr) {
-		struct ib_pool_fmr **pfmr;
-
-		for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
-			ib_fmr_pool_unmap(*pfmr);
 	}
 
 	ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
@@ -1472,50 +1416,6 @@ static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
 	state->ndesc++;
 }
 
-static int srp_map_finish_fmr(struct srp_map_state *state,
-			      struct srp_rdma_ch *ch)
-{
-	struct srp_target_port *target = ch->target;
-	struct srp_device *dev = target->srp_host->srp_dev;
-	struct ib_pool_fmr *fmr;
-	u64 io_addr = 0;
-
-	if (state->fmr.next >= state->fmr.end) {
-		shost_printk(KERN_ERR, ch->target->scsi_host,
-			     PFX "Out of MRs (mr_per_cmd = %d)\n",
-			     ch->target->mr_per_cmd);
-		return -ENOMEM;
-	}
-
-	WARN_ON_ONCE(!dev->use_fmr);
-
-	if (state->npages == 0)
-		return 0;
-
-	if (state->npages == 1 && target->global_rkey) {
-		srp_map_desc(state, state->base_dma_addr, state->dma_len,
-			     target->global_rkey);
-		goto reset_state;
-	}
-
-	fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
-				   state->npages, io_addr);
-	if (IS_ERR(fmr))
-		return PTR_ERR(fmr);
-
-	*state->fmr.next++ = fmr;
-	state->nmdesc++;
-
-	srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
-		     state->dma_len, fmr->fmr->rkey);
-
-reset_state:
-	state->npages = 0;
-	state->dma_len = 0;
-
-	return 0;
-}
-
 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
 {
 	srp_handle_qp_err(cq, wc, "FAST REG");
@@ -1606,74 +1506,6 @@ static int srp_map_finish_fr(struct srp_map_state *state,
 	return n;
 }
 
-static int srp_map_sg_entry(struct srp_map_state *state,
-			    struct srp_rdma_ch *ch,
-			    struct scatterlist *sg)
-{
-	struct srp_target_port *target = ch->target;
-	struct srp_device *dev = target->srp_host->srp_dev;
-	dma_addr_t dma_addr = sg_dma_address(sg);
-	unsigned int dma_len = sg_dma_len(sg);
-	unsigned int len = 0;
-	int ret;
-
-	WARN_ON_ONCE(!dma_len);
-
-	while (dma_len) {
-		unsigned offset = dma_addr & ~dev->mr_page_mask;
-
-		if (state->npages == dev->max_pages_per_mr ||
-		    (state->npages > 0 && offset != 0)) {
-			ret = srp_map_finish_fmr(state, ch);
-			if (ret)
-				return ret;
-		}
-
-		len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
-
-		if (!state->npages)
-			state->base_dma_addr = dma_addr;
-		state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
-		state->dma_len += len;
-		dma_addr += len;
-		dma_len -= len;
-	}
-
-	/*
-	 * If the end of the MR is not on a page boundary then we need to
-	 * close it out and start a new one -- we can only merge at page
-	 * boundaries.
-	 */
-	ret = 0;
-	if ((dma_addr & ~dev->mr_page_mask) != 0)
-		ret = srp_map_finish_fmr(state, ch);
-	return ret;
-}
-
-static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
-			  struct srp_request *req, struct scatterlist *scat,
-			  int count)
-{
-	struct scatterlist *sg;
-	int i, ret;
-
-	state->pages = req->map_page;
-	state->fmr.next = req->fmr_list;
-	state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
-
-	for_each_sg(scat, sg, count, i) {
-		ret = srp_map_sg_entry(state, ch, sg);
-		if (ret)
-			return ret;
-	}
-
-	ret = srp_map_finish_fmr(state, ch);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
 			 struct srp_request *req, struct scatterlist *scat,
 			 int count)
@@ -1733,7 +1565,6 @@ static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
 	struct srp_device *dev = target->srp_host->srp_dev;
 	struct srp_map_state state;
 	struct srp_direct_buf idb_desc;
-	u64 idb_pages[1];
 	struct scatterlist idb_sg[1];
 	int ret;
 
@@ -1756,14 +1587,6 @@ static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
 		if (ret < 0)
 			return ret;
 		WARN_ON_ONCE(ret < 1);
-	} else if (dev->use_fmr) {
-		state.pages = idb_pages;
-		state.pages[0] = (req->indirect_dma_addr &
-				  dev->mr_page_mask);
-		state.npages = 1;
-		ret = srp_map_finish_fmr(&state, ch);
-		if (ret < 0)
-			return ret;
 	} else {
 		return -EINVAL;
 	}
@@ -1787,9 +1610,6 @@ static void srp_check_mapping(struct srp_map_state *state,
 	if (dev->use_fast_reg)
 		for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
 			mr_len += (*pfr)->mr->length;
-	else if (dev->use_fmr)
-		for (i = 0; i < state->nmdesc; i++)
-			mr_len += be32_to_cpu(req->indirect_desc[i].len);
 	if (desc_len != scsi_bufflen(req->scmnd) ||
 	    mr_len > scsi_bufflen(req->scmnd))
 		pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
@@ -1904,8 +1724,6 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
 	state.desc = req->indirect_desc;
 	if (dev->use_fast_reg)
 		ret = srp_map_sg_fr(&state, ch, req, scat, count);
-	else if (dev->use_fmr)
-		ret = srp_map_sg_fmr(&state, ch, req, scat, count);
 	else
 		ret = srp_map_sg_dma(&state, ch, req, scat, count);
 	req->nmdesc = state.nmdesc;
@@ -3424,6 +3242,7 @@ enum {
 	SRP_OPT_IP_DEST		= 1 << 16,
 	SRP_OPT_TARGET_CAN_QUEUE= 1 << 17,
 	SRP_OPT_MAX_IT_IU_SIZE  = 1 << 18,
+	SRP_OPT_CH_COUNT	= 1 << 19,
 };
 
 static unsigned int srp_opt_mandatory[] = {
@@ -3457,6 +3276,7 @@ static const match_table_t srp_opt_tokens = {
 	{ SRP_OPT_IP_SRC,		"src=%s"		},
 	{ SRP_OPT_IP_DEST,		"dest=%s"		},
 	{ SRP_OPT_MAX_IT_IU_SIZE,	"max_it_iu_size=%d"	},
+	{ SRP_OPT_CH_COUNT,		"ch_count=%u",		},
 	{ SRP_OPT_ERR,			NULL 			}
 };
 
@@ -3758,6 +3578,14 @@ static int srp_parse_options(struct net *net, const char *buf,
 			target->max_it_iu_size = token;
 			break;
 
+		case SRP_OPT_CH_COUNT:
+			if (match_int(args, &token) || token < 1) {
+				pr_warn("bad channel count %s\n", p);
+				goto out;
+			}
+			target->ch_count = token;
+			break;
+
 		default:
 			pr_warn("unknown parameter or missing value '%s' in target creation request\n",
 				p);
@@ -3864,13 +3692,13 @@ static ssize_t srp_create_target(struct device *dev,
 		goto out;
 	}
 
-	if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
+	if (!srp_dev->has_fr && !target->allow_ext_sg &&
 	    target->cmd_sg_cnt < target->sg_tablesize) {
 		pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
 		target->sg_tablesize = target->cmd_sg_cnt;
 	}
 
-	if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
+	if (srp_dev->use_fast_reg) {
 		bool gaps_reg = (ibdev->attrs.device_cap_flags &
 				 IB_DEVICE_SG_GAPS_REG);
 
@@ -3878,12 +3706,12 @@ static ssize_t srp_create_target(struct device *dev,
 				  (ilog2(srp_dev->mr_page_size) - 9);
 		if (!gaps_reg) {
 			/*
-			 * FR and FMR can only map one HCA page per entry. If
-			 * the start address is not aligned on a HCA page
-			 * boundary two entries will be used for the head and
-			 * the tail although these two entries combined
-			 * contain at most one HCA page of data. Hence the "+
-			 * 1" in the calculation below.
+			 * FR can only map one HCA page per entry. If the start
+			 * address is not aligned on a HCA page boundary two
+			 * entries will be used for the head and the tail
+			 * although these two entries combined contain at most
+			 * one HCA page of data. Hence the "+ 1" in the
+			 * calculation below.
 			 *
 			 * The indirect data buffer descriptor is contiguous
 			 * so the memory for that buffer will only be
@@ -3921,11 +3749,13 @@ static ssize_t srp_create_target(struct device *dev,
 		goto out;
 
 	ret = -ENOMEM;
-	target->ch_count = max_t(unsigned, num_online_nodes(),
-				 min(ch_count ? :
-				     min(4 * num_online_nodes(),
-					 ibdev->num_comp_vectors),
-				     num_online_cpus()));
+	if (target->ch_count == 0)
+		target->ch_count =
+			max_t(unsigned int, num_online_nodes(),
+			      min(ch_count ?:
+					  min(4 * num_online_nodes(),
+					      ibdev->num_comp_vectors),
+				  num_online_cpus()));
 	target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
 			     GFP_KERNEL);
 	if (!target->ch)
@@ -4132,7 +3962,7 @@ static void srp_rename_dev(struct ib_device *device, void *client_data)
 	}
 }
 
-static void srp_add_one(struct ib_device *device)
+static int srp_add_one(struct ib_device *device)
 {
 	struct srp_device *srp_dev;
 	struct ib_device_attr *attr = &device->attrs;
@@ -4144,7 +3974,7 @@ static void srp_add_one(struct ib_device *device)
 
 	srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
 	if (!srp_dev)
-		return;
+		return -ENOMEM;
 
 	/*
 	 * Use the smallest page size supported by the HCA, down to a
@@ -4162,23 +3992,15 @@ static void srp_add_one(struct ib_device *device)
 	srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
 					  max_pages_per_mr);
 
-	srp_dev->has_fmr = (device->ops.alloc_fmr &&
-			    device->ops.dealloc_fmr &&
-			    device->ops.map_phys_fmr &&
-			    device->ops.unmap_fmr);
 	srp_dev->has_fr = (attr->device_cap_flags &
 			   IB_DEVICE_MEM_MGT_EXTENSIONS);
-	if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
-		dev_warn(&device->dev, "neither FMR nor FR is supported\n");
-	} else if (!never_register &&
-		   attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
-		srp_dev->use_fast_reg = (srp_dev->has_fr &&
-					 (!srp_dev->has_fmr || prefer_fr));
-		srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
-	}
+	if (!never_register && !srp_dev->has_fr)
+		dev_warn(&device->dev, "FR is not supported\n");
+	else if (!never_register &&
+		 attr->max_mr_size >= 2 * srp_dev->mr_page_size)
+		srp_dev->use_fast_reg = srp_dev->has_fr;
 
-	if (never_register || !register_always ||
-	    (!srp_dev->has_fmr && !srp_dev->has_fr))
+	if (never_register || !register_always || !srp_dev->has_fr)
 		flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
 
 	if (srp_dev->use_fast_reg) {
@@ -4197,8 +4019,12 @@ static void srp_add_one(struct ib_device *device)
 
 	srp_dev->dev = device;
 	srp_dev->pd  = ib_alloc_pd(device, flags);
-	if (IS_ERR(srp_dev->pd))
-		goto free_dev;
+	if (IS_ERR(srp_dev->pd)) {
+		int ret = PTR_ERR(srp_dev->pd);
+
+		kfree(srp_dev);
+		return ret;
+	}
 
 	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
 		srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey;
@@ -4212,10 +4038,7 @@ static void srp_add_one(struct ib_device *device)
 	}
 
 	ib_set_client_data(device, &srp_client, srp_dev);
-	return;
-
-free_dev:
-	kfree(srp_dev);
+	return 0;
 }
 
 static void srp_remove_one(struct ib_device *device, void *client_data)
@@ -4225,8 +4048,6 @@ static void srp_remove_one(struct ib_device *device, void *client_data)
 	struct srp_target_port *target;
 
 	srp_dev = client_data;
-	if (!srp_dev)
-		return;
 
 	list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
 		device_unregister(&host->dev);
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index 6fabcc2faf1f..6818cac0a3b7 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -44,7 +44,6 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_sa.h>
 #include <rdma/ib_cm.h>
-#include <rdma/ib_fmr_pool.h>
 #include <rdma/rdma_cm.h>
 
 enum {
@@ -95,8 +94,7 @@ enum srp_iu_type {
 /*
  * @mr_page_mask: HCA memory registration page mask.
  * @mr_page_size: HCA memory registration page size.
- * @mr_max_size: Maximum size in bytes of a single FMR / FR registration
- *   request.
+ * @mr_max_size: Maximum size in bytes of a single FR registration request.
  */
 struct srp_device {
 	struct list_head	dev_list;
@@ -107,9 +105,7 @@ struct srp_device {
 	int			mr_page_size;
 	int			mr_max_size;
 	int			max_pages_per_mr;
-	bool			has_fmr;
 	bool			has_fr;
-	bool			use_fmr;
 	bool			use_fast_reg;
 };
 
@@ -127,11 +123,7 @@ struct srp_host {
 struct srp_request {
 	struct scsi_cmnd       *scmnd;
 	struct srp_iu	       *cmd;
-	union {
-		struct ib_pool_fmr **fmr_list;
-		struct srp_fr_desc **fr_list;
-	};
-	u64		       *map_page;
+	struct srp_fr_desc     **fr_list;
 	struct srp_direct_buf  *indirect_desc;
 	dma_addr_t		indirect_dma_addr;
 	short			nmdesc;
@@ -155,10 +147,7 @@ struct srp_rdma_ch {
 	struct ib_cq	       *send_cq;
 	struct ib_cq	       *recv_cq;
 	struct ib_qp	       *qp;
-	union {
-		struct ib_fmr_pool     *fmr_pool;
-		struct srp_fr_pool     *fr_pool;
-	};
+	struct srp_fr_pool     *fr_pool;
 	uint32_t		max_it_iu_len;
 	uint32_t		max_ti_iu_len;
 	u8			max_imm_sge;
@@ -319,20 +308,16 @@ struct srp_fr_pool {
  * @pages:	    Array with DMA addresses of pages being considered for
  *		    memory registration.
  * @base_dma_addr:  DMA address of the first page that has not yet been mapped.
- * @dma_len:	    Number of bytes that will be registered with the next
- *		    FMR or FR memory registration call.
+ * @dma_len:	    Number of bytes that will be registered with the next FR
+ *                  memory registration call.
  * @total_len:	    Total number of bytes in the sg-list being mapped.
  * @npages:	    Number of page addresses in the pages[] array.
- * @nmdesc:	    Number of FMR or FR memory descriptors used for mapping.
+ * @nmdesc:	    Number of FR memory descriptors used for mapping.
  * @ndesc:	    Number of SRP buffer descriptors that have been filled in.
  */
 struct srp_map_state {
 	union {
 		struct {
-			struct ib_pool_fmr **next;
-			struct ib_pool_fmr **end;
-		} fmr;
-		struct {
 			struct srp_fr_desc **next;
 			struct srp_fr_desc **end;
 		} fr;
diff --git a/drivers/infiniband/ulp/srpt/Kconfig b/drivers/infiniband/ulp/srpt/Kconfig
index ce7567cea9f6..4b5d9b792cfa 100644
--- a/drivers/infiniband/ulp/srpt/Kconfig
+++ b/drivers/infiniband/ulp/srpt/Kconfig
@@ -2,7 +2,7 @@
 config INFINIBAND_SRPT
 	tristate "InfiniBand SCSI RDMA Protocol target support"
 	depends on INFINIBAND && INFINIBAND_ADDR_TRANS && TARGET_CORE
-	---help---
+	help
 
 	  Support for the SCSI RDMA Protocol (SRP) Target driver. The
 	  SRP protocol is a protocol that allows an initiator to access
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 98552749d71c..ef7fcd3e8e15 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -81,7 +81,7 @@ MODULE_PARM_DESC(srpt_srq_size,
 
 static int srpt_get_u64_x(char *buffer, const struct kernel_param *kp)
 {
-	return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
+	return sprintf(buffer, "0x%016llx\n", *(u64 *)kp->arg);
 }
 module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
 		  0444);
@@ -135,14 +135,11 @@ static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
 static void srpt_event_handler(struct ib_event_handler *handler,
 			       struct ib_event *event)
 {
-	struct srpt_device *sdev;
+	struct srpt_device *sdev =
+		container_of(handler, struct srpt_device, event_handler);
 	struct srpt_port *sport;
 	u8 port_num;
 
-	sdev = ib_get_client_data(event->device, &srpt_client);
-	if (!sdev || sdev->device != event->device)
-		return;
-
 	pr_debug("ASYNC event= %d on device= %s\n", event->event,
 		 dev_name(&sdev->device->dev));
 
@@ -217,8 +214,9 @@ static const char *get_ch_state_name(enum rdma_ch_state s)
  */
 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
 {
-	pr_debug("QP event %d on ch=%p sess_name=%s state=%d\n",
-		 event->event, ch, ch->sess_name, ch->state);
+	pr_debug("QP event %d on ch=%p sess_name=%s-%d state=%s\n",
+		 event->event, ch, ch->sess_name, ch->qp->qp_num,
+		 get_ch_state_name(ch->state));
 
 	switch (event->event) {
 	case IB_EVENT_COMM_EST:
@@ -610,6 +608,11 @@ static int srpt_refresh_port(struct srpt_port *sport)
 			       dev_name(&sport->sdev->device->dev), sport->port,
 			       PTR_ERR(sport->mad_agent));
 			sport->mad_agent = NULL;
+			memset(&port_modify, 0, sizeof(port_modify));
+			port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
+			ib_modify_port(sport->sdev->device, sport->port, 0,
+				       &port_modify);
+
 		}
 	}
 
@@ -633,9 +636,8 @@ static void srpt_unregister_mad_agent(struct srpt_device *sdev)
 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
 		sport = &sdev->port[i - 1];
 		WARN_ON(sport->port != i);
-		if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
-			pr_err("disabling MAD processing failed.\n");
 		if (sport->mad_agent) {
+			ib_modify_port(sdev->device, i, 0, &port_modify);
 			ib_unregister_mad_agent(sport->mad_agent);
 			sport->mad_agent = NULL;
 		}
@@ -1814,18 +1816,13 @@ retry:
 	 */
 	qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
 	qp_init->cap.max_rdma_ctxs = sq_size / 2;
-	qp_init->cap.max_send_sge = min(attrs->max_send_sge,
-					SRPT_MAX_SG_PER_WQE);
-	qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
-					SRPT_MAX_SG_PER_WQE);
+	qp_init->cap.max_send_sge = attrs->max_send_sge;
+	qp_init->cap.max_recv_sge = 1;
 	qp_init->port_num = ch->sport->port;
-	if (sdev->use_srq) {
+	if (sdev->use_srq)
 		qp_init->srq = sdev->srq;
-	} else {
+	else
 		qp_init->cap.max_recv_wr = ch->rq_size;
-		qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
-						SRPT_MAX_SG_PER_WQE);
-	}
 
 	if (ch->using_rdma_cm) {
 		ret = rdma_create_qp(ch->rdma_cm.cm_id, sdev->pd, qp_init);
@@ -1984,8 +1981,8 @@ static void __srpt_close_all_ch(struct srpt_port *sport)
 	list_for_each_entry(nexus, &sport->nexus_list, entry) {
 		list_for_each_entry(ch, &nexus->ch_list, list) {
 			if (srpt_disconnect_ch(ch) >= 0)
-				pr_info("Closing channel %s because target %s_%d has been disabled\n",
-					ch->sess_name,
+				pr_info("Closing channel %s-%d because target %s_%d has been disabled\n",
+					ch->sess_name, ch->qp->qp_num,
 					dev_name(&sport->sdev->device->dev),
 					sport->port);
 			srpt_close_ch(ch);
@@ -2496,7 +2493,8 @@ reject:
 				   SRP_BUF_FORMAT_INDIRECT);
 
 	if (rdma_cm_id)
-		rdma_reject(rdma_cm_id, rej, sizeof(*rej));
+		rdma_reject(rdma_cm_id, rej, sizeof(*rej),
+			    IB_CM_REJ_CONSUMER_DEFINED);
 	else
 		ib_send_cm_rej(ib_cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
 			       rej, sizeof(*rej));
@@ -3104,7 +3102,7 @@ static int srpt_use_srq(struct srpt_device *sdev, bool use_srq)
  * srpt_add_one - InfiniBand device addition callback function
  * @device: Describes a HCA.
  */
-static void srpt_add_one(struct ib_device *device)
+static int srpt_add_one(struct ib_device *device)
 {
 	struct srpt_device *sdev;
 	struct srpt_port *sport;
@@ -3115,14 +3113,16 @@ static void srpt_add_one(struct ib_device *device)
 	sdev = kzalloc(struct_size(sdev, port, device->phys_port_cnt),
 		       GFP_KERNEL);
 	if (!sdev)
-		goto err;
+		return -ENOMEM;
 
 	sdev->device = device;
 	mutex_init(&sdev->sdev_mutex);
 
 	sdev->pd = ib_alloc_pd(device, 0);
-	if (IS_ERR(sdev->pd))
+	if (IS_ERR(sdev->pd)) {
+		ret = PTR_ERR(sdev->pd);
 		goto free_dev;
+	}
 
 	sdev->lkey = sdev->pd->local_dma_lkey;
 
@@ -3138,6 +3138,7 @@ static void srpt_add_one(struct ib_device *device)
 	if (IS_ERR(sdev->cm_id)) {
 		pr_info("ib_create_cm_id() failed: %ld\n",
 			PTR_ERR(sdev->cm_id));
+		ret = PTR_ERR(sdev->cm_id);
 		sdev->cm_id = NULL;
 		if (!rdma_cm_id)
 			goto err_ring;
@@ -3182,7 +3183,8 @@ static void srpt_add_one(struct ib_device *device)
 		mutex_init(&sport->port_gid_id.mutex);
 		INIT_LIST_HEAD(&sport->port_gid_id.tpg_list);
 
-		if (srpt_refresh_port(sport)) {
+		ret = srpt_refresh_port(sport);
+		if (ret) {
 			pr_err("MAD registration failed for %s-%d.\n",
 			       dev_name(&sdev->device->dev), i);
 			goto err_event;
@@ -3193,10 +3195,9 @@ static void srpt_add_one(struct ib_device *device)
 	list_add_tail(&sdev->list, &srpt_dev_list);
 	spin_unlock(&srpt_dev_lock);
 
-out:
 	ib_set_client_data(device, &srpt_client, sdev);
 	pr_debug("added %s.\n", dev_name(&device->dev));
-	return;
+	return 0;
 
 err_event:
 	ib_unregister_event_handler(&sdev->event_handler);
@@ -3208,10 +3209,8 @@ err_ring:
 	ib_dealloc_pd(sdev->pd);
 free_dev:
 	kfree(sdev);
-err:
-	sdev = NULL;
 	pr_info("%s(%s) failed.\n", __func__, dev_name(&device->dev));
-	goto out;
+	return ret;
 }
 
 /**
@@ -3224,12 +3223,6 @@ static void srpt_remove_one(struct ib_device *device, void *client_data)
 	struct srpt_device *sdev = client_data;
 	int i;
 
-	if (!sdev) {
-		pr_info("%s(%s): nothing to do.\n", __func__,
-			dev_name(&device->dev));
-		return;
-	}
-
 	srpt_unregister_mad_agent(sdev);
 
 	ib_unregister_event_handler(&sdev->event_handler);
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index 2e1a69840857..f31c349d07a1 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -105,11 +105,6 @@ enum {
 	SRP_CMD_ACA = 0x4,
 
 	SRPT_DEF_SG_TABLESIZE = 128,
-	/*
-	 * An experimentally determined value that avoids that QP creation
-	 * fails due to "swiotlb buffer is full" on systems using the swiotlb.
-	 */
-	SRPT_MAX_SG_PER_WQE = 16,
 
 	MIN_SRPT_SQ_SIZE = 16,
 	DEF_SRPT_SQ_SIZE = 4096,
diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c
index 0d57e51b8ba1..e494295d1c7b 100644
--- a/drivers/input/evdev.c
+++ b/drivers/input/evdev.c
@@ -282,7 +282,8 @@ static void evdev_pass_values(struct evdev_client *client,
 	spin_unlock(&client->buffer_lock);
 
 	if (wakeup)
-		wake_up_interruptible(&evdev->wait);
+		wake_up_interruptible_poll(&evdev->wait,
+			EPOLLIN | EPOLLOUT | EPOLLRDNORM | EPOLLWRNORM);
 }
 
 /*
@@ -429,7 +430,7 @@ static void evdev_hangup(struct evdev *evdev)
 		kill_fasync(&client->fasync, SIGIO, POLL_HUP);
 	spin_unlock(&evdev->client_lock);
 
-	wake_up_interruptible(&evdev->wait);
+	wake_up_interruptible_poll(&evdev->wait, EPOLLHUP | EPOLLERR);
 }
 
 static int evdev_release(struct inode *inode, struct file *file)
@@ -945,7 +946,7 @@ static int evdev_revoke(struct evdev *evdev, struct evdev_client *client,
 	client->revoked = true;
 	evdev_ungrab(evdev, client);
 	input_flush_device(&evdev->handle, file);
-	wake_up_interruptible(&evdev->wait);
+	wake_up_interruptible_poll(&evdev->wait, EPOLLHUP | EPOLLERR);
 
 	return 0;
 }
diff --git a/drivers/input/gameport/Kconfig b/drivers/input/gameport/Kconfig
index 24acb3b07fed..4761795cb49f 100644
--- a/drivers/input/gameport/Kconfig
+++ b/drivers/input/gameport/Kconfig
@@ -4,7 +4,7 @@
 #
 config GAMEPORT
 	tristate "Gameport support"
-	---help---
+	help
 	  Gameport support is for the standard 15-pin PC gameport. If you
 	  have a joystick, gamepad, gameport card, a soundcard with a gameport
 	  or anything else that uses the gameport, say Y or M here and also to
diff --git a/drivers/input/joystick/Kconfig b/drivers/input/joystick/Kconfig
index 940b744639c7..eb031b7a4866 100644
--- a/drivers/input/joystick/Kconfig
+++ b/drivers/input/joystick/Kconfig
@@ -18,7 +18,7 @@ if INPUT_JOYSTICK
 config JOYSTICK_ANALOG
 	tristate "Classic PC analog joysticks and gamepads"
 	select GAMEPORT
-	---help---
+	help
 	  Say Y here if you have a joystick that connects to the PC
 	  gameport. In addition to the usual PC analog joystick, this driver
 	  supports many extensions, including joysticks with throttle control,
@@ -45,6 +45,7 @@ config JOYSTICK_A3D
 config JOYSTICK_ADI
 	tristate "Logitech ADI digital joysticks and gamepads"
 	select GAMEPORT
+	depends on ADI!=m # avoid module name conflict
 	help
 	  Say Y here if you have a Logitech controller using the ADI
 	  protocol over the PC gameport.
@@ -224,7 +225,7 @@ config JOYSTICK_GAMECON
 	tristate "Multisystem, NES, SNES, N64, PSX joysticks and gamepads"
 	depends on PARPORT
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	  Say Y here if you have a Nintendo Entertainment System gamepad,
 	  Super Nintendo Entertainment System gamepad, Nintendo 64 gamepad,
 	  Sony PlayStation gamepad or a Multisystem -- Atari, Amiga,
@@ -297,13 +298,13 @@ config JOYSTICK_XPAD_FF
 	bool "X-Box gamepad rumble support"
 	depends on JOYSTICK_XPAD && INPUT
 	select INPUT_FF_MEMLESS
-	---help---
+	help
 	  Say Y here if you want to take advantage of xbox 360 rumble features.
 
 config JOYSTICK_XPAD_LEDS
 	bool "LED Support for Xbox360 controller 'BigX' LED"
 	depends on JOYSTICK_XPAD && (LEDS_CLASS=y || LEDS_CLASS=JOYSTICK_XPAD)
-	---help---
+	help
 	  This option enables support for the LED which surrounds the Big X on
 	  XBox 360 controller.
 
diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig
index 28de965a08d5..793ecbbda32c 100644
--- a/drivers/input/keyboard/Kconfig
+++ b/drivers/input/keyboard/Kconfig
@@ -701,7 +701,7 @@ config KEYBOARD_SPEAR
 	  Say Y here if you want to use the SPEAR keyboard.
 
 	  To compile this driver as a module, choose M here: the
-	  module will be called spear-keboard.
+	  module will be called spear-keyboard.
 
 config KEYBOARD_TC3589X
 	tristate "TC3589X Keypad support"
diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c
index 7e3eae54c192..6ec28265771d 100644
--- a/drivers/input/keyboard/atkbd.c
+++ b/drivers/input/keyboard/atkbd.c
@@ -24,6 +24,7 @@
 #include <linux/libps2.h>
 #include <linux/mutex.h>
 #include <linux/dmi.h>
+#include <linux/property.h>
 
 #define DRIVER_DESC	"AT and PS/2 keyboard driver"
 
@@ -63,6 +64,11 @@ static bool atkbd_terminal;
 module_param_named(terminal, atkbd_terminal, bool, 0);
 MODULE_PARM_DESC(terminal, "Enable break codes on an IBM Terminal keyboard connected via AT/PS2");
 
+#define MAX_FUNCTION_ROW_KEYS	24
+
+#define SCANCODE(keymap)	((keymap >> 16) & 0xFFFF)
+#define KEYCODE(keymap)		(keymap & 0xFFFF)
+
 /*
  * Scancode to keycode tables. These are just the default setting, and
  * are loadable via a userland utility.
@@ -230,6 +236,9 @@ struct atkbd {
 
 	/* Serializes reconnect(), attr->set() and event work */
 	struct mutex mutex;
+
+	u32 function_row_physmap[MAX_FUNCTION_ROW_KEYS];
+	int num_function_row_keys;
 };
 
 /*
@@ -283,6 +292,7 @@ static struct device_attribute atkbd_attr_##_name =				\
 	__ATTR(_name, S_IRUGO, atkbd_do_show_##_name, NULL);
 
 ATKBD_DEFINE_RO_ATTR(err_count);
+ATKBD_DEFINE_RO_ATTR(function_row_physmap);
 
 static struct attribute *atkbd_attributes[] = {
 	&atkbd_attr_extra.attr,
@@ -292,11 +302,42 @@ static struct attribute *atkbd_attributes[] = {
 	&atkbd_attr_softrepeat.attr,
 	&atkbd_attr_softraw.attr,
 	&atkbd_attr_err_count.attr,
+	&atkbd_attr_function_row_physmap.attr,
 	NULL
 };
 
+static ssize_t atkbd_show_function_row_physmap(struct atkbd *atkbd, char *buf)
+{
+	ssize_t size = 0;
+	int i;
+
+	if (!atkbd->num_function_row_keys)
+		return 0;
+
+	for (i = 0; i < atkbd->num_function_row_keys; i++)
+		size += scnprintf(buf + size, PAGE_SIZE - size, "%02X ",
+				  atkbd->function_row_physmap[i]);
+	size += scnprintf(buf + size, PAGE_SIZE - size, "\n");
+	return size;
+}
+
+static umode_t atkbd_attr_is_visible(struct kobject *kobj,
+				struct attribute *attr, int i)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct serio *serio = to_serio_port(dev);
+	struct atkbd *atkbd = serio_get_drvdata(serio);
+
+	if (attr == &atkbd_attr_function_row_physmap.attr &&
+	    !atkbd->num_function_row_keys)
+		return 0;
+
+	return attr->mode;
+}
+
 static struct attribute_group atkbd_attribute_group = {
 	.attrs	= atkbd_attributes,
+	.is_visible = atkbd_attr_is_visible,
 };
 
 static const unsigned int xl_table[] = {
@@ -994,6 +1035,39 @@ static unsigned int atkbd_oqo_01plus_scancode_fixup(struct atkbd *atkbd,
 	return code;
 }
 
+static int atkbd_get_keymap_from_fwnode(struct atkbd *atkbd)
+{
+	struct device *dev = &atkbd->ps2dev.serio->dev;
+	int i, n;
+	u32 *ptr;
+	u16 scancode, keycode;
+
+	/* Parse "linux,keymap" property */
+	n = device_property_count_u32(dev, "linux,keymap");
+	if (n <= 0 || n > ATKBD_KEYMAP_SIZE)
+		return -ENXIO;
+
+	ptr = kcalloc(n, sizeof(u32), GFP_KERNEL);
+	if (!ptr)
+		return -ENOMEM;
+
+	if (device_property_read_u32_array(dev, "linux,keymap", ptr, n)) {
+		dev_err(dev, "problem parsing FW keymap property\n");
+		kfree(ptr);
+		return -EINVAL;
+	}
+
+	memset(atkbd->keycode, 0, sizeof(atkbd->keycode));
+	for (i = 0; i < n; i++) {
+		scancode = SCANCODE(ptr[i]);
+		keycode = KEYCODE(ptr[i]);
+		atkbd->keycode[scancode] = keycode;
+	}
+
+	kfree(ptr);
+	return 0;
+}
+
 /*
  * atkbd_set_keycode_table() initializes keyboard's keycode table
  * according to the selected scancode set
@@ -1001,13 +1075,16 @@ static unsigned int atkbd_oqo_01plus_scancode_fixup(struct atkbd *atkbd,
 
 static void atkbd_set_keycode_table(struct atkbd *atkbd)
 {
+	struct device *dev = &atkbd->ps2dev.serio->dev;
 	unsigned int scancode;
 	int i, j;
 
 	memset(atkbd->keycode, 0, sizeof(atkbd->keycode));
 	bitmap_zero(atkbd->force_release_mask, ATKBD_KEYMAP_SIZE);
 
-	if (atkbd->translated) {
+	if (!atkbd_get_keymap_from_fwnode(atkbd)) {
+		dev_dbg(dev, "Using FW keymap\n");
+	} else if (atkbd->translated) {
 		for (i = 0; i < 128; i++) {
 			scancode = atkbd_unxlate_table[i];
 			atkbd->keycode[i] = atkbd_set2_keycode[scancode];
@@ -1121,6 +1198,22 @@ static void atkbd_set_device_attrs(struct atkbd *atkbd)
 	}
 }
 
+static void atkbd_parse_fwnode_data(struct serio *serio)
+{
+	struct atkbd *atkbd = serio_get_drvdata(serio);
+	struct device *dev = &serio->dev;
+	int n;
+
+	/* Parse "function-row-physmap" property */
+	n = device_property_count_u32(dev, "function-row-physmap");
+	if (n > 0 && n <= MAX_FUNCTION_ROW_KEYS &&
+	    !device_property_read_u32_array(dev, "function-row-physmap",
+					    atkbd->function_row_physmap, n)) {
+		atkbd->num_function_row_keys = n;
+		dev_dbg(dev, "FW reported %d function-row key locations\n", n);
+	}
+}
+
 /*
  * atkbd_connect() is called when the serio module finds an interface
  * that isn't handled yet by an appropriate device driver. We check if
@@ -1184,6 +1277,8 @@ static int atkbd_connect(struct serio *serio, struct serio_driver *drv)
 		atkbd->id = 0xab00;
 	}
 
+	atkbd_parse_fwnode_data(serio);
+
 	atkbd_set_keycode_table(atkbd);
 	atkbd_set_device_attrs(atkbd);
 
diff --git a/drivers/input/keyboard/imx_sc_key.c b/drivers/input/keyboard/imx_sc_key.c
index 9f809aeb785c..d18839f1f4f6 100644
--- a/drivers/input/keyboard/imx_sc_key.c
+++ b/drivers/input/keyboard/imx_sc_key.c
@@ -99,6 +99,15 @@ static void imx_sc_check_for_events(struct work_struct *work)
 				      msecs_to_jiffies(REPEAT_INTERVAL));
 }
 
+static void imx_sc_key_action(void *data)
+{
+	struct imx_key_drv_data *priv = data;
+
+	imx_scu_irq_group_enable(SC_IRQ_GROUP_WAKE, SC_IRQ_BUTTON, false);
+	imx_scu_irq_unregister_notifier(&priv->key_notifier);
+	cancel_delayed_work_sync(&priv->check_work);
+}
+
 static int imx_sc_key_probe(struct platform_device *pdev)
 {
 	struct imx_key_drv_data *priv;
@@ -149,27 +158,16 @@ static int imx_sc_key_probe(struct platform_device *pdev)
 		return error;
 	}
 
+	error = devm_add_action_or_reset(&pdev->dev, imx_sc_key_action, &priv);
+	if (error)
+		return error;
+
 	priv->key_notifier.notifier_call = imx_sc_key_notify;
 	error = imx_scu_irq_register_notifier(&priv->key_notifier);
-	if (error) {
-		imx_scu_irq_group_enable(SC_IRQ_GROUP_WAKE, SC_IRQ_BUTTON,
-					 false);
+	if (error)
 		dev_err(&pdev->dev, "failed to register scu notifier\n");
-		return error;
-	}
-
-	return 0;
-}
-
-static int imx_sc_key_remove(struct platform_device *pdev)
-{
-	struct imx_key_drv_data *priv = platform_get_drvdata(pdev);
-
-	imx_scu_irq_group_enable(SC_IRQ_GROUP_WAKE, SC_IRQ_BUTTON, false);
-	imx_scu_irq_unregister_notifier(&priv->key_notifier);
-	cancel_delayed_work_sync(&priv->check_work);
 
-	return 0;
+	return error;
 }
 
 static const struct of_device_id imx_sc_key_ids[] = {
@@ -184,7 +182,6 @@ static struct platform_driver imx_sc_key_driver = {
 		.of_match_table = imx_sc_key_ids,
 	},
 	.probe = imx_sc_key_probe,
-	.remove = imx_sc_key_remove,
 };
 module_platform_driver(imx_sc_key_driver);
 
diff --git a/drivers/input/keyboard/tca6416-keypad.c b/drivers/input/keyboard/tca6416-keypad.c
index 21758767ccf0..9b0f9665dcb0 100644
--- a/drivers/input/keyboard/tca6416-keypad.c
+++ b/drivers/input/keyboard/tca6416-keypad.c
@@ -374,5 +374,5 @@ static void __exit tca6416_keypad_exit(void)
 module_exit(tca6416_keypad_exit);
 
 MODULE_AUTHOR("Sriramakrishnan <srk@ti.com>");
-MODULE_DESCRIPTION("Keypad driver over tca6146 IO expander");
+MODULE_DESCRIPTION("Keypad driver over tca6416 IO expander");
 MODULE_LICENSE("GPL");
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index 7e2e658d551c..362e8a01980c 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -117,16 +117,6 @@ config INPUT_E3X0_BUTTON
 	  To compile this driver as a module, choose M here: the
 	  module will be called e3x0_button.
 
-config INPUT_MSM_VIBRATOR
-	tristate "Qualcomm MSM vibrator driver"
-	select INPUT_FF_MEMLESS
-	help
-	  Support for the vibrator that is found on various Qualcomm MSM
-	  SOCs.
-
-	  To compile this driver as a module, choose M here: the module
-	  will be called msm_vibrator.
-
 config INPUT_PCSPKR
 	tristate "PC Speaker support"
 	depends on PCSPKR_PLATFORM
@@ -265,17 +255,6 @@ config INPUT_APANEL
 	 To compile this driver as a module, choose M here: the module will
 	 be called apanel.
 
-config INPUT_GP2A
-	tristate "Sharp GP2AP002A00F I2C Proximity/Opto sensor driver"
-	depends on I2C
-	depends on GPIOLIB || COMPILE_TEST
-	help
-	  Say Y here if you have a Sharp GP2AP002A00F proximity/als combo-chip
-	  hooked to an I2C bus.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called gp2ap002a00f.
-
 config INPUT_GPIO_BEEPER
 	tristate "Generic GPIO Beeper support"
 	depends on GPIOLIB || COMPILE_TEST
@@ -739,6 +718,17 @@ config INPUT_IMS_PCU
 	  To compile this driver as a module, choose M here: the module will be
 	  called ims_pcu.
 
+config INPUT_IQS269A
+	tristate "Azoteq IQS269A capacitive touch controller"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say Y to enable support for the Azoteq IQS269A capacitive
+	  touch controller.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called iqs269a.
+
 config INPUT_CMA3000
 	tristate "VTI CMA3000 Tri-axis accelerometer"
 	help
diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile
index 8fd187f314bd..a48e5f2d859d 100644
--- a/drivers/input/misc/Makefile
+++ b/drivers/input/misc/Makefile
@@ -33,13 +33,13 @@ obj-$(CONFIG_INPUT_E3X0_BUTTON)		+= e3x0-button.o
 obj-$(CONFIG_INPUT_DRV260X_HAPTICS)	+= drv260x.o
 obj-$(CONFIG_INPUT_DRV2665_HAPTICS)	+= drv2665.o
 obj-$(CONFIG_INPUT_DRV2667_HAPTICS)	+= drv2667.o
-obj-$(CONFIG_INPUT_GP2A)		+= gp2ap002a00f.o
 obj-$(CONFIG_INPUT_GPIO_BEEPER)		+= gpio-beeper.o
 obj-$(CONFIG_INPUT_GPIO_DECODER)	+= gpio_decoder.o
 obj-$(CONFIG_INPUT_GPIO_VIBRA)		+= gpio-vibra.o
 obj-$(CONFIG_INPUT_HISI_POWERKEY)	+= hisi_powerkey.o
 obj-$(CONFIG_HP_SDC_RTC)		+= hp_sdc_rtc.o
 obj-$(CONFIG_INPUT_IMS_PCU)		+= ims-pcu.o
+obj-$(CONFIG_INPUT_IQS269A)		+= iqs269a.o
 obj-$(CONFIG_INPUT_IXP4XX_BEEPER)	+= ixp4xx-beeper.o
 obj-$(CONFIG_INPUT_KEYSPAN_REMOTE)	+= keyspan_remote.o
 obj-$(CONFIG_INPUT_KXTJ9)		+= kxtj9.o
@@ -50,7 +50,6 @@ obj-$(CONFIG_INPUT_MAX8925_ONKEY)	+= max8925_onkey.o
 obj-$(CONFIG_INPUT_MAX8997_HAPTIC)	+= max8997_haptic.o
 obj-$(CONFIG_INPUT_MC13783_PWRBUTTON)	+= mc13783-pwrbutton.o
 obj-$(CONFIG_INPUT_MMA8450)		+= mma8450.o
-obj-$(CONFIG_INPUT_MSM_VIBRATOR)	+= msm-vibrator.o
 obj-$(CONFIG_INPUT_PALMAS_PWRBUTTON)	+= palmas-pwrbutton.o
 obj-$(CONFIG_INPUT_PCAP)		+= pcap_keys.o
 obj-$(CONFIG_INPUT_PCF50633_PMU)	+= pcf50633-input.o
diff --git a/drivers/input/misc/gp2ap002a00f.c b/drivers/input/misc/gp2ap002a00f.c
deleted file mode 100644
index 90abda8eea67..000000000000
--- a/drivers/input/misc/gp2ap002a00f.c
+++ /dev/null
@@ -1,281 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2011 Sony Ericsson Mobile Communications Inc.
- *
- * Author: Courtney Cavin <courtney.cavin@sonyericsson.com>
- * Prepared for up-stream by: Oskar Andero <oskar.andero@sonyericsson.com>
- */
-
-#include <linux/i2c.h>
-#include <linux/irq.h>
-#include <linux/slab.h>
-#include <linux/input.h>
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/gpio.h>
-#include <linux/delay.h>
-#include <linux/input/gp2ap002a00f.h>
-
-struct gp2a_data {
-	struct input_dev *input;
-	const struct gp2a_platform_data *pdata;
-	struct i2c_client *i2c_client;
-};
-
-enum gp2a_addr {
-	GP2A_ADDR_PROX	= 0x0,
-	GP2A_ADDR_GAIN	= 0x1,
-	GP2A_ADDR_HYS	= 0x2,
-	GP2A_ADDR_CYCLE	= 0x3,
-	GP2A_ADDR_OPMOD	= 0x4,
-	GP2A_ADDR_CON	= 0x6
-};
-
-enum gp2a_controls {
-	/* Software Shutdown control: 0 = shutdown, 1 = normal operation */
-	GP2A_CTRL_SSD	= 0x01
-};
-
-static int gp2a_report(struct gp2a_data *dt)
-{
-	int vo = gpio_get_value(dt->pdata->vout_gpio);
-
-	input_report_switch(dt->input, SW_FRONT_PROXIMITY, !vo);
-	input_sync(dt->input);
-
-	return 0;
-}
-
-static irqreturn_t gp2a_irq(int irq, void *handle)
-{
-	struct gp2a_data *dt = handle;
-
-	gp2a_report(dt);
-
-	return IRQ_HANDLED;
-}
-
-static int gp2a_enable(struct gp2a_data *dt)
-{
-	return i2c_smbus_write_byte_data(dt->i2c_client, GP2A_ADDR_OPMOD,
-					 GP2A_CTRL_SSD);
-}
-
-static int gp2a_disable(struct gp2a_data *dt)
-{
-	return i2c_smbus_write_byte_data(dt->i2c_client, GP2A_ADDR_OPMOD,
-					 0x00);
-}
-
-static int gp2a_device_open(struct input_dev *dev)
-{
-	struct gp2a_data *dt = input_get_drvdata(dev);
-	int error;
-
-	error = gp2a_enable(dt);
-	if (error < 0) {
-		dev_err(&dt->i2c_client->dev,
-			"unable to activate, err %d\n", error);
-		return error;
-	}
-
-	gp2a_report(dt);
-
-	return 0;
-}
-
-static void gp2a_device_close(struct input_dev *dev)
-{
-	struct gp2a_data *dt = input_get_drvdata(dev);
-	int error;
-
-	error = gp2a_disable(dt);
-	if (error < 0)
-		dev_err(&dt->i2c_client->dev,
-			"unable to deactivate, err %d\n", error);
-}
-
-static int gp2a_initialize(struct gp2a_data *dt)
-{
-	int error;
-
-	error = i2c_smbus_write_byte_data(dt->i2c_client, GP2A_ADDR_GAIN,
-					  0x08);
-	if (error < 0)
-		return error;
-
-	error = i2c_smbus_write_byte_data(dt->i2c_client, GP2A_ADDR_HYS,
-					  0xc2);
-	if (error < 0)
-		return error;
-
-	error = i2c_smbus_write_byte_data(dt->i2c_client, GP2A_ADDR_CYCLE,
-					  0x04);
-	if (error < 0)
-		return error;
-
-	error = gp2a_disable(dt);
-
-	return error;
-}
-
-static int gp2a_probe(struct i2c_client *client,
-				const struct i2c_device_id *id)
-{
-	const struct gp2a_platform_data *pdata = dev_get_platdata(&client->dev);
-	struct gp2a_data *dt;
-	int error;
-
-	if (!pdata)
-		return -EINVAL;
-
-	if (pdata->hw_setup) {
-		error = pdata->hw_setup(client);
-		if (error < 0)
-			return error;
-	}
-
-	error = gpio_request_one(pdata->vout_gpio, GPIOF_IN, GP2A_I2C_NAME);
-	if (error)
-		goto err_hw_shutdown;
-
-	dt = kzalloc(sizeof(struct gp2a_data), GFP_KERNEL);
-	if (!dt) {
-		error = -ENOMEM;
-		goto err_free_gpio;
-	}
-
-	dt->pdata = pdata;
-	dt->i2c_client = client;
-
-	error = gp2a_initialize(dt);
-	if (error < 0)
-		goto err_free_mem;
-
-	dt->input = input_allocate_device();
-	if (!dt->input) {
-		error = -ENOMEM;
-		goto err_free_mem;
-	}
-
-	input_set_drvdata(dt->input, dt);
-
-	dt->input->open = gp2a_device_open;
-	dt->input->close = gp2a_device_close;
-	dt->input->name = GP2A_I2C_NAME;
-	dt->input->id.bustype = BUS_I2C;
-	dt->input->dev.parent = &client->dev;
-
-	input_set_capability(dt->input, EV_SW, SW_FRONT_PROXIMITY);
-
-	error = request_threaded_irq(client->irq, NULL, gp2a_irq,
-			IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING |
-				IRQF_ONESHOT,
-			GP2A_I2C_NAME, dt);
-	if (error) {
-		dev_err(&client->dev, "irq request failed\n");
-		goto err_free_input_dev;
-	}
-
-	error = input_register_device(dt->input);
-	if (error) {
-		dev_err(&client->dev, "device registration failed\n");
-		goto err_free_irq;
-	}
-
-	device_init_wakeup(&client->dev, pdata->wakeup);
-	i2c_set_clientdata(client, dt);
-
-	return 0;
-
-err_free_irq:
-	free_irq(client->irq, dt);
-err_free_input_dev:
-	input_free_device(dt->input);
-err_free_mem:
-	kfree(dt);
-err_free_gpio:
-	gpio_free(pdata->vout_gpio);
-err_hw_shutdown:
-	if (pdata->hw_shutdown)
-		pdata->hw_shutdown(client);
-	return error;
-}
-
-static int gp2a_remove(struct i2c_client *client)
-{
-	struct gp2a_data *dt = i2c_get_clientdata(client);
-	const struct gp2a_platform_data *pdata = dt->pdata;
-
-	free_irq(client->irq, dt);
-
-	input_unregister_device(dt->input);
-	kfree(dt);
-
-	gpio_free(pdata->vout_gpio);
-
-	if (pdata->hw_shutdown)
-		pdata->hw_shutdown(client);
-
-	return 0;
-}
-
-static int __maybe_unused gp2a_suspend(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct gp2a_data *dt = i2c_get_clientdata(client);
-	int retval = 0;
-
-	if (device_may_wakeup(&client->dev)) {
-		enable_irq_wake(client->irq);
-	} else {
-		mutex_lock(&dt->input->mutex);
-		if (dt->input->users)
-			retval = gp2a_disable(dt);
-		mutex_unlock(&dt->input->mutex);
-	}
-
-	return retval;
-}
-
-static int __maybe_unused gp2a_resume(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct gp2a_data *dt = i2c_get_clientdata(client);
-	int retval = 0;
-
-	if (device_may_wakeup(&client->dev)) {
-		disable_irq_wake(client->irq);
-	} else {
-		mutex_lock(&dt->input->mutex);
-		if (dt->input->users)
-			retval = gp2a_enable(dt);
-		mutex_unlock(&dt->input->mutex);
-	}
-
-	return retval;
-}
-
-static SIMPLE_DEV_PM_OPS(gp2a_pm, gp2a_suspend, gp2a_resume);
-
-static const struct i2c_device_id gp2a_i2c_id[] = {
-	{ GP2A_I2C_NAME, 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, gp2a_i2c_id);
-
-static struct i2c_driver gp2a_i2c_driver = {
-	.driver = {
-		.name	= GP2A_I2C_NAME,
-		.pm	= &gp2a_pm,
-	},
-	.probe		= gp2a_probe,
-	.remove		= gp2a_remove,
-	.id_table	= gp2a_i2c_id,
-};
-
-module_i2c_driver(gp2a_i2c_driver);
-
-MODULE_AUTHOR("Courtney Cavin <courtney.cavin@sonyericsson.com>");
-MODULE_DESCRIPTION("Sharp GP2AP002A00F I2C Proximity/Opto sensor driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/misc/iqs269a.c b/drivers/input/misc/iqs269a.c
new file mode 100644
index 000000000000..6699eb160a0f
--- /dev/null
+++ b/drivers/input/misc/iqs269a.c
@@ -0,0 +1,1833 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Azoteq IQS269A Capacitive Touch Controller
+ *
+ * Copyright (C) 2020 Jeff LaBundy <jeff@labundy.com>
+ *
+ * This driver registers up to 3 input devices: one representing capacitive or
+ * inductive keys as well as Hall-effect switches, and one for each of the two
+ * axial sliders presented by the device.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/input.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define IQS269_VER_INFO				0x00
+#define IQS269_VER_INFO_PROD_NUM		0x4F
+
+#define IQS269_SYS_FLAGS			0x02
+#define IQS269_SYS_FLAGS_SHOW_RESET		BIT(15)
+#define IQS269_SYS_FLAGS_PWR_MODE_MASK		GENMASK(12, 11)
+#define IQS269_SYS_FLAGS_PWR_MODE_SHIFT		11
+#define IQS269_SYS_FLAGS_IN_ATI			BIT(10)
+
+#define IQS269_CHx_COUNTS			0x08
+
+#define IQS269_SLIDER_X				0x30
+
+#define IQS269_CAL_DATA_A			0x35
+#define IQS269_CAL_DATA_A_HALL_BIN_L_MASK	GENMASK(15, 12)
+#define IQS269_CAL_DATA_A_HALL_BIN_L_SHIFT	12
+#define IQS269_CAL_DATA_A_HALL_BIN_R_MASK	GENMASK(11, 8)
+#define IQS269_CAL_DATA_A_HALL_BIN_R_SHIFT	8
+
+#define IQS269_SYS_SETTINGS			0x80
+#define IQS269_SYS_SETTINGS_CLK_DIV		BIT(15)
+#define IQS269_SYS_SETTINGS_ULP_AUTO		BIT(14)
+#define IQS269_SYS_SETTINGS_DIS_AUTO		BIT(13)
+#define IQS269_SYS_SETTINGS_PWR_MODE_MASK	GENMASK(12, 11)
+#define IQS269_SYS_SETTINGS_PWR_MODE_SHIFT	11
+#define IQS269_SYS_SETTINGS_PWR_MODE_MAX	3
+#define IQS269_SYS_SETTINGS_ULP_UPDATE_MASK	GENMASK(10, 8)
+#define IQS269_SYS_SETTINGS_ULP_UPDATE_SHIFT	8
+#define IQS269_SYS_SETTINGS_ULP_UPDATE_MAX	7
+#define IQS269_SYS_SETTINGS_RESEED_OFFSET	BIT(6)
+#define IQS269_SYS_SETTINGS_EVENT_MODE		BIT(5)
+#define IQS269_SYS_SETTINGS_EVENT_MODE_LP	BIT(4)
+#define IQS269_SYS_SETTINGS_REDO_ATI		BIT(2)
+#define IQS269_SYS_SETTINGS_ACK_RESET		BIT(0)
+
+#define IQS269_FILT_STR_LP_LTA_MASK		GENMASK(7, 6)
+#define IQS269_FILT_STR_LP_LTA_SHIFT		6
+#define IQS269_FILT_STR_LP_CNT_MASK		GENMASK(5, 4)
+#define IQS269_FILT_STR_LP_CNT_SHIFT		4
+#define IQS269_FILT_STR_NP_LTA_MASK		GENMASK(3, 2)
+#define IQS269_FILT_STR_NP_LTA_SHIFT		2
+#define IQS269_FILT_STR_NP_CNT_MASK		GENMASK(1, 0)
+#define IQS269_FILT_STR_MAX			3
+
+#define IQS269_EVENT_MASK_SYS			BIT(6)
+#define IQS269_EVENT_MASK_DEEP			BIT(2)
+#define IQS269_EVENT_MASK_TOUCH			BIT(1)
+#define IQS269_EVENT_MASK_PROX			BIT(0)
+
+#define IQS269_RATE_NP_MS_MAX			255
+#define IQS269_RATE_LP_MS_MAX			255
+#define IQS269_RATE_ULP_MS_MAX			4080
+#define IQS269_TIMEOUT_PWR_MS_MAX		130560
+#define IQS269_TIMEOUT_LTA_MS_MAX		130560
+
+#define IQS269_MISC_A_ATI_BAND_DISABLE		BIT(15)
+#define IQS269_MISC_A_ATI_LP_ONLY		BIT(14)
+#define IQS269_MISC_A_ATI_BAND_TIGHTEN		BIT(13)
+#define IQS269_MISC_A_FILT_DISABLE		BIT(12)
+#define IQS269_MISC_A_GPIO3_SELECT_MASK		GENMASK(10, 8)
+#define IQS269_MISC_A_GPIO3_SELECT_SHIFT	8
+#define IQS269_MISC_A_DUAL_DIR			BIT(6)
+#define IQS269_MISC_A_TX_FREQ_MASK		GENMASK(5, 4)
+#define IQS269_MISC_A_TX_FREQ_SHIFT		4
+#define IQS269_MISC_A_TX_FREQ_MAX		3
+#define IQS269_MISC_A_GLOBAL_CAP_SIZE		BIT(0)
+
+#define IQS269_MISC_B_RESEED_UI_SEL_MASK	GENMASK(7, 6)
+#define IQS269_MISC_B_RESEED_UI_SEL_SHIFT	6
+#define IQS269_MISC_B_RESEED_UI_SEL_MAX		3
+#define IQS269_MISC_B_TRACKING_UI_ENABLE	BIT(4)
+#define IQS269_MISC_B_FILT_STR_SLIDER		GENMASK(1, 0)
+
+#define IQS269_CHx_SETTINGS			0x8C
+
+#define IQS269_CHx_ENG_A_MEAS_CAP_SIZE		BIT(15)
+#define IQS269_CHx_ENG_A_RX_GND_INACTIVE	BIT(13)
+#define IQS269_CHx_ENG_A_LOCAL_CAP_SIZE		BIT(12)
+#define IQS269_CHx_ENG_A_ATI_MODE_MASK		GENMASK(9, 8)
+#define IQS269_CHx_ENG_A_ATI_MODE_SHIFT		8
+#define IQS269_CHx_ENG_A_ATI_MODE_MAX		3
+#define IQS269_CHx_ENG_A_INV_LOGIC		BIT(7)
+#define IQS269_CHx_ENG_A_PROJ_BIAS_MASK		GENMASK(6, 5)
+#define IQS269_CHx_ENG_A_PROJ_BIAS_SHIFT	5
+#define IQS269_CHx_ENG_A_PROJ_BIAS_MAX		3
+#define IQS269_CHx_ENG_A_SENSE_MODE_MASK	GENMASK(3, 0)
+#define IQS269_CHx_ENG_A_SENSE_MODE_MAX		15
+
+#define IQS269_CHx_ENG_B_LOCAL_CAP_ENABLE	BIT(13)
+#define IQS269_CHx_ENG_B_SENSE_FREQ_MASK	GENMASK(10, 9)
+#define IQS269_CHx_ENG_B_SENSE_FREQ_SHIFT	9
+#define IQS269_CHx_ENG_B_SENSE_FREQ_MAX		3
+#define IQS269_CHx_ENG_B_STATIC_ENABLE		BIT(8)
+#define IQS269_CHx_ENG_B_ATI_BASE_MASK		GENMASK(7, 6)
+#define IQS269_CHx_ENG_B_ATI_BASE_75		0x00
+#define IQS269_CHx_ENG_B_ATI_BASE_100		0x40
+#define IQS269_CHx_ENG_B_ATI_BASE_150		0x80
+#define IQS269_CHx_ENG_B_ATI_BASE_200		0xC0
+#define IQS269_CHx_ENG_B_ATI_TARGET_MASK	GENMASK(5, 0)
+#define IQS269_CHx_ENG_B_ATI_TARGET_MAX		2016
+
+#define IQS269_CHx_WEIGHT_MAX			255
+#define IQS269_CHx_THRESH_MAX			255
+#define IQS269_CHx_HYST_DEEP_MASK		GENMASK(7, 4)
+#define IQS269_CHx_HYST_DEEP_SHIFT		4
+#define IQS269_CHx_HYST_TOUCH_MASK		GENMASK(3, 0)
+#define IQS269_CHx_HYST_MAX			15
+
+#define IQS269_CHx_HALL_INACTIVE		6
+#define IQS269_CHx_HALL_ACTIVE			7
+
+#define IQS269_HALL_PAD_R			BIT(0)
+#define IQS269_HALL_PAD_L			BIT(1)
+#define IQS269_HALL_PAD_INV			BIT(6)
+
+#define IQS269_HALL_UI				0xF5
+#define IQS269_HALL_UI_ENABLE			BIT(15)
+
+#define IQS269_MAX_REG				0xFF
+
+#define IQS269_NUM_CH				8
+#define IQS269_NUM_SL				2
+
+#define IQS269_ATI_POLL_SLEEP_US		(iqs269->delay_mult * 10000)
+#define IQS269_ATI_POLL_TIMEOUT_US		(iqs269->delay_mult * 500000)
+#define IQS269_ATI_STABLE_DELAY_MS		(iqs269->delay_mult * 150)
+
+#define IQS269_PWR_MODE_POLL_SLEEP_US		IQS269_ATI_POLL_SLEEP_US
+#define IQS269_PWR_MODE_POLL_TIMEOUT_US		IQS269_ATI_POLL_TIMEOUT_US
+
+#define iqs269_irq_wait()			usleep_range(100, 150)
+
+enum iqs269_local_cap_size {
+	IQS269_LOCAL_CAP_SIZE_0,
+	IQS269_LOCAL_CAP_SIZE_GLOBAL_ONLY,
+	IQS269_LOCAL_CAP_SIZE_GLOBAL_0pF5,
+};
+
+enum iqs269_st_offs {
+	IQS269_ST_OFFS_PROX,
+	IQS269_ST_OFFS_DIR,
+	IQS269_ST_OFFS_TOUCH,
+	IQS269_ST_OFFS_DEEP,
+};
+
+enum iqs269_th_offs {
+	IQS269_TH_OFFS_PROX,
+	IQS269_TH_OFFS_TOUCH,
+	IQS269_TH_OFFS_DEEP,
+};
+
+enum iqs269_event_id {
+	IQS269_EVENT_PROX_DN,
+	IQS269_EVENT_PROX_UP,
+	IQS269_EVENT_TOUCH_DN,
+	IQS269_EVENT_TOUCH_UP,
+	IQS269_EVENT_DEEP_DN,
+	IQS269_EVENT_DEEP_UP,
+};
+
+struct iqs269_switch_desc {
+	unsigned int code;
+	bool enabled;
+};
+
+struct iqs269_event_desc {
+	const char *name;
+	enum iqs269_st_offs st_offs;
+	enum iqs269_th_offs th_offs;
+	bool dir_up;
+	u8 mask;
+};
+
+static const struct iqs269_event_desc iqs269_events[] = {
+	[IQS269_EVENT_PROX_DN] = {
+		.name = "event-prox",
+		.st_offs = IQS269_ST_OFFS_PROX,
+		.th_offs = IQS269_TH_OFFS_PROX,
+		.mask = IQS269_EVENT_MASK_PROX,
+	},
+	[IQS269_EVENT_PROX_UP] = {
+		.name = "event-prox-alt",
+		.st_offs = IQS269_ST_OFFS_PROX,
+		.th_offs = IQS269_TH_OFFS_PROX,
+		.dir_up = true,
+		.mask = IQS269_EVENT_MASK_PROX,
+	},
+	[IQS269_EVENT_TOUCH_DN] = {
+		.name = "event-touch",
+		.st_offs = IQS269_ST_OFFS_TOUCH,
+		.th_offs = IQS269_TH_OFFS_TOUCH,
+		.mask = IQS269_EVENT_MASK_TOUCH,
+	},
+	[IQS269_EVENT_TOUCH_UP] = {
+		.name = "event-touch-alt",
+		.st_offs = IQS269_ST_OFFS_TOUCH,
+		.th_offs = IQS269_TH_OFFS_TOUCH,
+		.dir_up = true,
+		.mask = IQS269_EVENT_MASK_TOUCH,
+	},
+	[IQS269_EVENT_DEEP_DN] = {
+		.name = "event-deep",
+		.st_offs = IQS269_ST_OFFS_DEEP,
+		.th_offs = IQS269_TH_OFFS_DEEP,
+		.mask = IQS269_EVENT_MASK_DEEP,
+	},
+	[IQS269_EVENT_DEEP_UP] = {
+		.name = "event-deep-alt",
+		.st_offs = IQS269_ST_OFFS_DEEP,
+		.th_offs = IQS269_TH_OFFS_DEEP,
+		.dir_up = true,
+		.mask = IQS269_EVENT_MASK_DEEP,
+	},
+};
+
+struct iqs269_ver_info {
+	u8 prod_num;
+	u8 sw_num;
+	u8 hw_num;
+	u8 padding;
+} __packed;
+
+struct iqs269_sys_reg {
+	__be16 general;
+	u8 active;
+	u8 filter;
+	u8 reseed;
+	u8 event_mask;
+	u8 rate_np;
+	u8 rate_lp;
+	u8 rate_ulp;
+	u8 timeout_pwr;
+	u8 timeout_rdy;
+	u8 timeout_lta;
+	__be16 misc_a;
+	__be16 misc_b;
+	u8 blocking;
+	u8 padding;
+	u8 slider_select[IQS269_NUM_SL];
+	u8 timeout_tap;
+	u8 timeout_swipe;
+	u8 thresh_swipe;
+	u8 redo_ati;
+} __packed;
+
+struct iqs269_ch_reg {
+	u8 rx_enable;
+	u8 tx_enable;
+	__be16 engine_a;
+	__be16 engine_b;
+	__be16 ati_comp;
+	u8 thresh[3];
+	u8 hyst;
+	u8 assoc_select;
+	u8 assoc_weight;
+} __packed;
+
+struct iqs269_flags {
+	__be16 system;
+	u8 gesture;
+	u8 padding;
+	u8 states[4];
+} __packed;
+
+struct iqs269_private {
+	struct i2c_client *client;
+	struct regmap *regmap;
+	struct mutex lock;
+	struct iqs269_switch_desc switches[ARRAY_SIZE(iqs269_events)];
+	struct iqs269_ch_reg ch_reg[IQS269_NUM_CH];
+	struct iqs269_sys_reg sys_reg;
+	struct input_dev *keypad;
+	struct input_dev *slider[IQS269_NUM_SL];
+	unsigned int keycode[ARRAY_SIZE(iqs269_events) * IQS269_NUM_CH];
+	unsigned int suspend_mode;
+	unsigned int delay_mult;
+	unsigned int ch_num;
+	bool hall_enable;
+	bool ati_current;
+};
+
+static int iqs269_ati_mode_set(struct iqs269_private *iqs269,
+			       unsigned int ch_num, unsigned int mode)
+{
+	u16 engine_a;
+
+	if (ch_num >= IQS269_NUM_CH)
+		return -EINVAL;
+
+	if (mode > IQS269_CHx_ENG_A_ATI_MODE_MAX)
+		return -EINVAL;
+
+	mutex_lock(&iqs269->lock);
+
+	engine_a = be16_to_cpu(iqs269->ch_reg[ch_num].engine_a);
+
+	engine_a &= ~IQS269_CHx_ENG_A_ATI_MODE_MASK;
+	engine_a |= (mode << IQS269_CHx_ENG_A_ATI_MODE_SHIFT);
+
+	iqs269->ch_reg[ch_num].engine_a = cpu_to_be16(engine_a);
+	iqs269->ati_current = false;
+
+	mutex_unlock(&iqs269->lock);
+
+	return 0;
+}
+
+static int iqs269_ati_mode_get(struct iqs269_private *iqs269,
+			       unsigned int ch_num, unsigned int *mode)
+{
+	u16 engine_a;
+
+	if (ch_num >= IQS269_NUM_CH)
+		return -EINVAL;
+
+	mutex_lock(&iqs269->lock);
+	engine_a = be16_to_cpu(iqs269->ch_reg[ch_num].engine_a);
+	mutex_unlock(&iqs269->lock);
+
+	engine_a &= IQS269_CHx_ENG_A_ATI_MODE_MASK;
+	*mode = (engine_a >> IQS269_CHx_ENG_A_ATI_MODE_SHIFT);
+
+	return 0;
+}
+
+static int iqs269_ati_base_set(struct iqs269_private *iqs269,
+			       unsigned int ch_num, unsigned int base)
+{
+	u16 engine_b;
+
+	if (ch_num >= IQS269_NUM_CH)
+		return -EINVAL;
+
+	switch (base) {
+	case 75:
+		base = IQS269_CHx_ENG_B_ATI_BASE_75;
+		break;
+
+	case 100:
+		base = IQS269_CHx_ENG_B_ATI_BASE_100;
+		break;
+
+	case 150:
+		base = IQS269_CHx_ENG_B_ATI_BASE_150;
+		break;
+
+	case 200:
+		base = IQS269_CHx_ENG_B_ATI_BASE_200;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	mutex_lock(&iqs269->lock);
+
+	engine_b = be16_to_cpu(iqs269->ch_reg[ch_num].engine_b);
+
+	engine_b &= ~IQS269_CHx_ENG_B_ATI_BASE_MASK;
+	engine_b |= base;
+
+	iqs269->ch_reg[ch_num].engine_b = cpu_to_be16(engine_b);
+	iqs269->ati_current = false;
+
+	mutex_unlock(&iqs269->lock);
+
+	return 0;
+}
+
+static int iqs269_ati_base_get(struct iqs269_private *iqs269,
+			       unsigned int ch_num, unsigned int *base)
+{
+	u16 engine_b;
+
+	if (ch_num >= IQS269_NUM_CH)
+		return -EINVAL;
+
+	mutex_lock(&iqs269->lock);
+	engine_b = be16_to_cpu(iqs269->ch_reg[ch_num].engine_b);
+	mutex_unlock(&iqs269->lock);
+
+	switch (engine_b & IQS269_CHx_ENG_B_ATI_BASE_MASK) {
+	case IQS269_CHx_ENG_B_ATI_BASE_75:
+		*base = 75;
+		return 0;
+
+	case IQS269_CHx_ENG_B_ATI_BASE_100:
+		*base = 100;
+		return 0;
+
+	case IQS269_CHx_ENG_B_ATI_BASE_150:
+		*base = 150;
+		return 0;
+
+	case IQS269_CHx_ENG_B_ATI_BASE_200:
+		*base = 200;
+		return 0;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int iqs269_ati_target_set(struct iqs269_private *iqs269,
+				 unsigned int ch_num, unsigned int target)
+{
+	u16 engine_b;
+
+	if (ch_num >= IQS269_NUM_CH)
+		return -EINVAL;
+
+	if (target > IQS269_CHx_ENG_B_ATI_TARGET_MAX)
+		return -EINVAL;
+
+	mutex_lock(&iqs269->lock);
+
+	engine_b = be16_to_cpu(iqs269->ch_reg[ch_num].engine_b);
+
+	engine_b &= ~IQS269_CHx_ENG_B_ATI_TARGET_MASK;
+	engine_b |= target / 32;
+
+	iqs269->ch_reg[ch_num].engine_b = cpu_to_be16(engine_b);
+	iqs269->ati_current = false;
+
+	mutex_unlock(&iqs269->lock);
+
+	return 0;
+}
+
+static int iqs269_ati_target_get(struct iqs269_private *iqs269,
+				 unsigned int ch_num, unsigned int *target)
+{
+	u16 engine_b;
+
+	if (ch_num >= IQS269_NUM_CH)
+		return -EINVAL;
+
+	mutex_lock(&iqs269->lock);
+	engine_b = be16_to_cpu(iqs269->ch_reg[ch_num].engine_b);
+	mutex_unlock(&iqs269->lock);
+
+	*target = (engine_b & IQS269_CHx_ENG_B_ATI_TARGET_MASK) * 32;
+
+	return 0;
+}
+
+static int iqs269_parse_mask(const struct fwnode_handle *fwnode,
+			     const char *propname, u8 *mask)
+{
+	unsigned int val[IQS269_NUM_CH];
+	int count, error, i;
+
+	count = fwnode_property_count_u32(fwnode, propname);
+	if (count < 0)
+		return 0;
+
+	if (count > IQS269_NUM_CH)
+		return -EINVAL;
+
+	error = fwnode_property_read_u32_array(fwnode, propname, val, count);
+	if (error)
+		return error;
+
+	*mask = 0;
+
+	for (i = 0; i < count; i++) {
+		if (val[i] >= IQS269_NUM_CH)
+			return -EINVAL;
+
+		*mask |= BIT(val[i]);
+	}
+
+	return 0;
+}
+
+static int iqs269_parse_chan(struct iqs269_private *iqs269,
+			     const struct fwnode_handle *ch_node)
+{
+	struct i2c_client *client = iqs269->client;
+	struct fwnode_handle *ev_node;
+	struct iqs269_ch_reg *ch_reg;
+	u16 engine_a, engine_b;
+	unsigned int reg, val;
+	int error, i;
+
+	error = fwnode_property_read_u32(ch_node, "reg", &reg);
+	if (error) {
+		dev_err(&client->dev, "Failed to read channel number: %d\n",
+			error);
+		return error;
+	} else if (reg >= IQS269_NUM_CH) {
+		dev_err(&client->dev, "Invalid channel number: %u\n", reg);
+		return -EINVAL;
+	}
+
+	iqs269->sys_reg.active |= BIT(reg);
+	if (!fwnode_property_present(ch_node, "azoteq,reseed-disable"))
+		iqs269->sys_reg.reseed |= BIT(reg);
+
+	if (fwnode_property_present(ch_node, "azoteq,blocking-enable"))
+		iqs269->sys_reg.blocking |= BIT(reg);
+
+	if (fwnode_property_present(ch_node, "azoteq,slider0-select"))
+		iqs269->sys_reg.slider_select[0] |= BIT(reg);
+
+	if (fwnode_property_present(ch_node, "azoteq,slider1-select"))
+		iqs269->sys_reg.slider_select[1] |= BIT(reg);
+
+	ch_reg = &iqs269->ch_reg[reg];
+
+	error = regmap_raw_read(iqs269->regmap,
+				IQS269_CHx_SETTINGS + reg * sizeof(*ch_reg) / 2,
+				ch_reg, sizeof(*ch_reg));
+	if (error)
+		return error;
+
+	error = iqs269_parse_mask(ch_node, "azoteq,rx-enable",
+				  &ch_reg->rx_enable);
+	if (error) {
+		dev_err(&client->dev, "Invalid channel %u RX enable mask: %d\n",
+			reg, error);
+		return error;
+	}
+
+	error = iqs269_parse_mask(ch_node, "azoteq,tx-enable",
+				  &ch_reg->tx_enable);
+	if (error) {
+		dev_err(&client->dev, "Invalid channel %u TX enable mask: %d\n",
+			reg, error);
+		return error;
+	}
+
+	engine_a = be16_to_cpu(ch_reg->engine_a);
+	engine_b = be16_to_cpu(ch_reg->engine_b);
+
+	engine_a |= IQS269_CHx_ENG_A_MEAS_CAP_SIZE;
+	if (fwnode_property_present(ch_node, "azoteq,meas-cap-decrease"))
+		engine_a &= ~IQS269_CHx_ENG_A_MEAS_CAP_SIZE;
+
+	engine_a |= IQS269_CHx_ENG_A_RX_GND_INACTIVE;
+	if (fwnode_property_present(ch_node, "azoteq,rx-float-inactive"))
+		engine_a &= ~IQS269_CHx_ENG_A_RX_GND_INACTIVE;
+
+	engine_a &= ~IQS269_CHx_ENG_A_LOCAL_CAP_SIZE;
+	engine_b &= ~IQS269_CHx_ENG_B_LOCAL_CAP_ENABLE;
+	if (!fwnode_property_read_u32(ch_node, "azoteq,local-cap-size", &val)) {
+		switch (val) {
+		case IQS269_LOCAL_CAP_SIZE_0:
+			break;
+
+		case IQS269_LOCAL_CAP_SIZE_GLOBAL_0pF5:
+			engine_a |= IQS269_CHx_ENG_A_LOCAL_CAP_SIZE;
+
+			/* fall through */
+
+		case IQS269_LOCAL_CAP_SIZE_GLOBAL_ONLY:
+			engine_b |= IQS269_CHx_ENG_B_LOCAL_CAP_ENABLE;
+			break;
+
+		default:
+			dev_err(&client->dev,
+				"Invalid channel %u local cap. size: %u\n", reg,
+				val);
+			return -EINVAL;
+		}
+	}
+
+	engine_a &= ~IQS269_CHx_ENG_A_INV_LOGIC;
+	if (fwnode_property_present(ch_node, "azoteq,invert-enable"))
+		engine_a |= IQS269_CHx_ENG_A_INV_LOGIC;
+
+	if (!fwnode_property_read_u32(ch_node, "azoteq,proj-bias", &val)) {
+		if (val > IQS269_CHx_ENG_A_PROJ_BIAS_MAX) {
+			dev_err(&client->dev,
+				"Invalid channel %u bias current: %u\n", reg,
+				val);
+			return -EINVAL;
+		}
+
+		engine_a &= ~IQS269_CHx_ENG_A_PROJ_BIAS_MASK;
+		engine_a |= (val << IQS269_CHx_ENG_A_PROJ_BIAS_SHIFT);
+	}
+
+	if (!fwnode_property_read_u32(ch_node, "azoteq,sense-mode", &val)) {
+		if (val > IQS269_CHx_ENG_A_SENSE_MODE_MAX) {
+			dev_err(&client->dev,
+				"Invalid channel %u sensing mode: %u\n", reg,
+				val);
+			return -EINVAL;
+		}
+
+		engine_a &= ~IQS269_CHx_ENG_A_SENSE_MODE_MASK;
+		engine_a |= val;
+	}
+
+	if (!fwnode_property_read_u32(ch_node, "azoteq,sense-freq", &val)) {
+		if (val > IQS269_CHx_ENG_B_SENSE_FREQ_MAX) {
+			dev_err(&client->dev,
+				"Invalid channel %u sensing frequency: %u\n",
+				reg, val);
+			return -EINVAL;
+		}
+
+		engine_b &= ~IQS269_CHx_ENG_B_SENSE_FREQ_MASK;
+		engine_b |= (val << IQS269_CHx_ENG_B_SENSE_FREQ_SHIFT);
+	}
+
+	engine_b &= ~IQS269_CHx_ENG_B_STATIC_ENABLE;
+	if (fwnode_property_present(ch_node, "azoteq,static-enable"))
+		engine_b |= IQS269_CHx_ENG_B_STATIC_ENABLE;
+
+	ch_reg->engine_a = cpu_to_be16(engine_a);
+	ch_reg->engine_b = cpu_to_be16(engine_b);
+
+	if (!fwnode_property_read_u32(ch_node, "azoteq,ati-mode", &val)) {
+		error = iqs269_ati_mode_set(iqs269, reg, val);
+		if (error) {
+			dev_err(&client->dev,
+				"Invalid channel %u ATI mode: %u\n", reg, val);
+			return error;
+		}
+	}
+
+	if (!fwnode_property_read_u32(ch_node, "azoteq,ati-base", &val)) {
+		error = iqs269_ati_base_set(iqs269, reg, val);
+		if (error) {
+			dev_err(&client->dev,
+				"Invalid channel %u ATI base: %u\n", reg, val);
+			return error;
+		}
+	}
+
+	if (!fwnode_property_read_u32(ch_node, "azoteq,ati-target", &val)) {
+		error = iqs269_ati_target_set(iqs269, reg, val);
+		if (error) {
+			dev_err(&client->dev,
+				"Invalid channel %u ATI target: %u\n", reg,
+				val);
+			return error;
+		}
+	}
+
+	error = iqs269_parse_mask(ch_node, "azoteq,assoc-select",
+				  &ch_reg->assoc_select);
+	if (error) {
+		dev_err(&client->dev, "Invalid channel %u association: %d\n",
+			reg, error);
+		return error;
+	}
+
+	if (!fwnode_property_read_u32(ch_node, "azoteq,assoc-weight", &val)) {
+		if (val > IQS269_CHx_WEIGHT_MAX) {
+			dev_err(&client->dev,
+				"Invalid channel %u associated weight: %u\n",
+				reg, val);
+			return -EINVAL;
+		}
+
+		ch_reg->assoc_weight = val;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(iqs269_events); i++) {
+		ev_node = fwnode_get_named_child_node(ch_node,
+						      iqs269_events[i].name);
+		if (!ev_node)
+			continue;
+
+		if (!fwnode_property_read_u32(ev_node, "azoteq,thresh", &val)) {
+			if (val > IQS269_CHx_THRESH_MAX) {
+				dev_err(&client->dev,
+					"Invalid channel %u threshold: %u\n",
+					reg, val);
+				return -EINVAL;
+			}
+
+			ch_reg->thresh[iqs269_events[i].th_offs] = val;
+		}
+
+		if (!fwnode_property_read_u32(ev_node, "azoteq,hyst", &val)) {
+			u8 *hyst = &ch_reg->hyst;
+
+			if (val > IQS269_CHx_HYST_MAX) {
+				dev_err(&client->dev,
+					"Invalid channel %u hysteresis: %u\n",
+					reg, val);
+				return -EINVAL;
+			}
+
+			if (i == IQS269_EVENT_DEEP_DN ||
+			    i == IQS269_EVENT_DEEP_UP) {
+				*hyst &= ~IQS269_CHx_HYST_DEEP_MASK;
+				*hyst |= (val << IQS269_CHx_HYST_DEEP_SHIFT);
+			} else if (i == IQS269_EVENT_TOUCH_DN ||
+				   i == IQS269_EVENT_TOUCH_UP) {
+				*hyst &= ~IQS269_CHx_HYST_TOUCH_MASK;
+				*hyst |= val;
+			}
+		}
+
+		if (fwnode_property_read_u32(ev_node, "linux,code", &val))
+			continue;
+
+		switch (reg) {
+		case IQS269_CHx_HALL_ACTIVE:
+			if (iqs269->hall_enable) {
+				iqs269->switches[i].code = val;
+				iqs269->switches[i].enabled = true;
+			}
+
+			/* fall through */
+
+		case IQS269_CHx_HALL_INACTIVE:
+			if (iqs269->hall_enable)
+				break;
+
+			/* fall through */
+
+		default:
+			iqs269->keycode[i * IQS269_NUM_CH + reg] = val;
+		}
+
+		iqs269->sys_reg.event_mask &= ~iqs269_events[i].mask;
+	}
+
+	return 0;
+}
+
+static int iqs269_parse_prop(struct iqs269_private *iqs269)
+{
+	struct iqs269_sys_reg *sys_reg = &iqs269->sys_reg;
+	struct i2c_client *client = iqs269->client;
+	struct fwnode_handle *ch_node;
+	u16 general, misc_a, misc_b;
+	unsigned int val;
+	int error;
+
+	iqs269->hall_enable = device_property_present(&client->dev,
+						      "azoteq,hall-enable");
+
+	if (!device_property_read_u32(&client->dev, "azoteq,suspend-mode",
+				      &val)) {
+		if (val > IQS269_SYS_SETTINGS_PWR_MODE_MAX) {
+			dev_err(&client->dev, "Invalid suspend mode: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		iqs269->suspend_mode = val;
+	}
+
+	error = regmap_raw_read(iqs269->regmap, IQS269_SYS_SETTINGS, sys_reg,
+				sizeof(*sys_reg));
+	if (error)
+		return error;
+
+	if (!device_property_read_u32(&client->dev, "azoteq,filt-str-lp-lta",
+				      &val)) {
+		if (val > IQS269_FILT_STR_MAX) {
+			dev_err(&client->dev, "Invalid filter strength: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		sys_reg->filter &= ~IQS269_FILT_STR_LP_LTA_MASK;
+		sys_reg->filter |= (val << IQS269_FILT_STR_LP_LTA_SHIFT);
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,filt-str-lp-cnt",
+				      &val)) {
+		if (val > IQS269_FILT_STR_MAX) {
+			dev_err(&client->dev, "Invalid filter strength: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		sys_reg->filter &= ~IQS269_FILT_STR_LP_CNT_MASK;
+		sys_reg->filter |= (val << IQS269_FILT_STR_LP_CNT_SHIFT);
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,filt-str-np-lta",
+				      &val)) {
+		if (val > IQS269_FILT_STR_MAX) {
+			dev_err(&client->dev, "Invalid filter strength: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		sys_reg->filter &= ~IQS269_FILT_STR_NP_LTA_MASK;
+		sys_reg->filter |= (val << IQS269_FILT_STR_NP_LTA_SHIFT);
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,filt-str-np-cnt",
+				      &val)) {
+		if (val > IQS269_FILT_STR_MAX) {
+			dev_err(&client->dev, "Invalid filter strength: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		sys_reg->filter &= ~IQS269_FILT_STR_NP_CNT_MASK;
+		sys_reg->filter |= val;
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,rate-np-ms",
+				      &val)) {
+		if (val > IQS269_RATE_NP_MS_MAX) {
+			dev_err(&client->dev, "Invalid report rate: %u\n", val);
+			return -EINVAL;
+		}
+
+		sys_reg->rate_np = val;
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,rate-lp-ms",
+				      &val)) {
+		if (val > IQS269_RATE_LP_MS_MAX) {
+			dev_err(&client->dev, "Invalid report rate: %u\n", val);
+			return -EINVAL;
+		}
+
+		sys_reg->rate_lp = val;
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,rate-ulp-ms",
+				      &val)) {
+		if (val > IQS269_RATE_ULP_MS_MAX) {
+			dev_err(&client->dev, "Invalid report rate: %u\n", val);
+			return -EINVAL;
+		}
+
+		sys_reg->rate_ulp = val / 16;
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,timeout-pwr-ms",
+				      &val)) {
+		if (val > IQS269_TIMEOUT_PWR_MS_MAX) {
+			dev_err(&client->dev, "Invalid timeout: %u\n", val);
+			return -EINVAL;
+		}
+
+		sys_reg->timeout_pwr = val / 512;
+	}
+
+	if (!device_property_read_u32(&client->dev, "azoteq,timeout-lta-ms",
+				      &val)) {
+		if (val > IQS269_TIMEOUT_LTA_MS_MAX) {
+			dev_err(&client->dev, "Invalid timeout: %u\n", val);
+			return -EINVAL;
+		}
+
+		sys_reg->timeout_lta = val / 512;
+	}
+
+	misc_a = be16_to_cpu(sys_reg->misc_a);
+	misc_b = be16_to_cpu(sys_reg->misc_b);
+
+	misc_a &= ~IQS269_MISC_A_ATI_BAND_DISABLE;
+	if (device_property_present(&client->dev, "azoteq,ati-band-disable"))
+		misc_a |= IQS269_MISC_A_ATI_BAND_DISABLE;
+
+	misc_a &= ~IQS269_MISC_A_ATI_LP_ONLY;
+	if (device_property_present(&client->dev, "azoteq,ati-lp-only"))
+		misc_a |= IQS269_MISC_A_ATI_LP_ONLY;
+
+	misc_a &= ~IQS269_MISC_A_ATI_BAND_TIGHTEN;
+	if (device_property_present(&client->dev, "azoteq,ati-band-tighten"))
+		misc_a |= IQS269_MISC_A_ATI_BAND_TIGHTEN;
+
+	misc_a &= ~IQS269_MISC_A_FILT_DISABLE;
+	if (device_property_present(&client->dev, "azoteq,filt-disable"))
+		misc_a |= IQS269_MISC_A_FILT_DISABLE;
+
+	if (!device_property_read_u32(&client->dev, "azoteq,gpio3-select",
+				      &val)) {
+		if (val >= IQS269_NUM_CH) {
+			dev_err(&client->dev, "Invalid GPIO3 selection: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		misc_a &= ~IQS269_MISC_A_GPIO3_SELECT_MASK;
+		misc_a |= (val << IQS269_MISC_A_GPIO3_SELECT_SHIFT);
+	}
+
+	misc_a &= ~IQS269_MISC_A_DUAL_DIR;
+	if (device_property_present(&client->dev, "azoteq,dual-direction"))
+		misc_a |= IQS269_MISC_A_DUAL_DIR;
+
+	if (!device_property_read_u32(&client->dev, "azoteq,tx-freq", &val)) {
+		if (val > IQS269_MISC_A_TX_FREQ_MAX) {
+			dev_err(&client->dev,
+				"Invalid excitation frequency: %u\n", val);
+			return -EINVAL;
+		}
+
+		misc_a &= ~IQS269_MISC_A_TX_FREQ_MASK;
+		misc_a |= (val << IQS269_MISC_A_TX_FREQ_SHIFT);
+	}
+
+	misc_a &= ~IQS269_MISC_A_GLOBAL_CAP_SIZE;
+	if (device_property_present(&client->dev, "azoteq,global-cap-increase"))
+		misc_a |= IQS269_MISC_A_GLOBAL_CAP_SIZE;
+
+	if (!device_property_read_u32(&client->dev, "azoteq,reseed-select",
+				      &val)) {
+		if (val > IQS269_MISC_B_RESEED_UI_SEL_MAX) {
+			dev_err(&client->dev, "Invalid reseed selection: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		misc_b &= ~IQS269_MISC_B_RESEED_UI_SEL_MASK;
+		misc_b |= (val << IQS269_MISC_B_RESEED_UI_SEL_SHIFT);
+	}
+
+	misc_b &= ~IQS269_MISC_B_TRACKING_UI_ENABLE;
+	if (device_property_present(&client->dev, "azoteq,tracking-enable"))
+		misc_b |= IQS269_MISC_B_TRACKING_UI_ENABLE;
+
+	if (!device_property_read_u32(&client->dev, "azoteq,filt-str-slider",
+				      &val)) {
+		if (val > IQS269_FILT_STR_MAX) {
+			dev_err(&client->dev, "Invalid filter strength: %u\n",
+				val);
+			return -EINVAL;
+		}
+
+		misc_b &= ~IQS269_MISC_B_FILT_STR_SLIDER;
+		misc_b |= val;
+	}
+
+	sys_reg->misc_a = cpu_to_be16(misc_a);
+	sys_reg->misc_b = cpu_to_be16(misc_b);
+
+	sys_reg->active = 0;
+	sys_reg->reseed = 0;
+
+	sys_reg->blocking = 0;
+
+	sys_reg->slider_select[0] = 0;
+	sys_reg->slider_select[1] = 0;
+
+	sys_reg->event_mask = ~((u8)IQS269_EVENT_MASK_SYS);
+
+	device_for_each_child_node(&client->dev, ch_node) {
+		error = iqs269_parse_chan(iqs269, ch_node);
+		if (error) {
+			fwnode_handle_put(ch_node);
+			return error;
+		}
+	}
+
+	/*
+	 * Volunteer all active channels to participate in ATI when REDO-ATI is
+	 * manually triggered.
+	 */
+	sys_reg->redo_ati = sys_reg->active;
+
+	general = be16_to_cpu(sys_reg->general);
+
+	if (device_property_present(&client->dev, "azoteq,clk-div")) {
+		general |= IQS269_SYS_SETTINGS_CLK_DIV;
+		iqs269->delay_mult = 4;
+	} else {
+		general &= ~IQS269_SYS_SETTINGS_CLK_DIV;
+		iqs269->delay_mult = 1;
+	}
+
+	/*
+	 * Configure the device to automatically switch between normal and low-
+	 * power modes as a function of sensing activity. Ultra-low-power mode,
+	 * if enabled, is reserved for suspend.
+	 */
+	general &= ~IQS269_SYS_SETTINGS_ULP_AUTO;
+	general &= ~IQS269_SYS_SETTINGS_DIS_AUTO;
+	general &= ~IQS269_SYS_SETTINGS_PWR_MODE_MASK;
+
+	if (!device_property_read_u32(&client->dev, "azoteq,ulp-update",
+				      &val)) {
+		if (val > IQS269_SYS_SETTINGS_ULP_UPDATE_MAX) {
+			dev_err(&client->dev, "Invalid update rate: %u\n", val);
+			return -EINVAL;
+		}
+
+		general &= ~IQS269_SYS_SETTINGS_ULP_UPDATE_MASK;
+		general |= (val << IQS269_SYS_SETTINGS_ULP_UPDATE_SHIFT);
+	}
+
+	general &= ~IQS269_SYS_SETTINGS_RESEED_OFFSET;
+	if (device_property_present(&client->dev, "azoteq,reseed-offset"))
+		general |= IQS269_SYS_SETTINGS_RESEED_OFFSET;
+
+	general |= IQS269_SYS_SETTINGS_EVENT_MODE;
+
+	/*
+	 * As per the datasheet, enable streaming during normal-power mode if
+	 * either slider is in use. In that case, the device returns to event
+	 * mode during low-power mode.
+	 */
+	if (sys_reg->slider_select[0] || sys_reg->slider_select[1])
+		general |= IQS269_SYS_SETTINGS_EVENT_MODE_LP;
+
+	general |= IQS269_SYS_SETTINGS_REDO_ATI;
+	general |= IQS269_SYS_SETTINGS_ACK_RESET;
+
+	sys_reg->general = cpu_to_be16(general);
+
+	return 0;
+}
+
+static int iqs269_dev_init(struct iqs269_private *iqs269)
+{
+	struct iqs269_sys_reg *sys_reg = &iqs269->sys_reg;
+	struct iqs269_ch_reg *ch_reg;
+	unsigned int val;
+	int error, i;
+
+	mutex_lock(&iqs269->lock);
+
+	error = regmap_update_bits(iqs269->regmap, IQS269_HALL_UI,
+				   IQS269_HALL_UI_ENABLE,
+				   iqs269->hall_enable ? ~0 : 0);
+	if (error)
+		goto err_mutex;
+
+	for (i = 0; i < IQS269_NUM_CH; i++) {
+		if (!(sys_reg->active & BIT(i)))
+			continue;
+
+		ch_reg = &iqs269->ch_reg[i];
+
+		error = regmap_raw_write(iqs269->regmap,
+					 IQS269_CHx_SETTINGS + i *
+					 sizeof(*ch_reg) / 2, ch_reg,
+					 sizeof(*ch_reg));
+		if (error)
+			goto err_mutex;
+	}
+
+	/*
+	 * The REDO-ATI and ATI channel selection fields must be written in the
+	 * same block write, so every field between registers 0x80 through 0x8B
+	 * (inclusive) must be written as well.
+	 */
+	error = regmap_raw_write(iqs269->regmap, IQS269_SYS_SETTINGS, sys_reg,
+				 sizeof(*sys_reg));
+	if (error)
+		goto err_mutex;
+
+	error = regmap_read_poll_timeout(iqs269->regmap, IQS269_SYS_FLAGS, val,
+					!(val & IQS269_SYS_FLAGS_IN_ATI),
+					 IQS269_ATI_POLL_SLEEP_US,
+					 IQS269_ATI_POLL_TIMEOUT_US);
+	if (error)
+		goto err_mutex;
+
+	msleep(IQS269_ATI_STABLE_DELAY_MS);
+	iqs269->ati_current = true;
+
+err_mutex:
+	mutex_unlock(&iqs269->lock);
+
+	return error;
+}
+
+static int iqs269_input_init(struct iqs269_private *iqs269)
+{
+	struct i2c_client *client = iqs269->client;
+	struct iqs269_flags flags;
+	unsigned int sw_code, keycode;
+	int error, i, j;
+	u8 dir_mask, state;
+
+	iqs269->keypad = devm_input_allocate_device(&client->dev);
+	if (!iqs269->keypad)
+		return -ENOMEM;
+
+	iqs269->keypad->keycodemax = ARRAY_SIZE(iqs269->keycode);
+	iqs269->keypad->keycode = iqs269->keycode;
+	iqs269->keypad->keycodesize = sizeof(*iqs269->keycode);
+
+	iqs269->keypad->name = "iqs269a_keypad";
+	iqs269->keypad->id.bustype = BUS_I2C;
+
+	if (iqs269->hall_enable) {
+		error = regmap_raw_read(iqs269->regmap, IQS269_SYS_FLAGS,
+					&flags, sizeof(flags));
+		if (error) {
+			dev_err(&client->dev,
+				"Failed to read initial status: %d\n", error);
+			return error;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(iqs269_events); i++) {
+		dir_mask = flags.states[IQS269_ST_OFFS_DIR];
+		if (!iqs269_events[i].dir_up)
+			dir_mask = ~dir_mask;
+
+		state = flags.states[iqs269_events[i].st_offs] & dir_mask;
+
+		sw_code = iqs269->switches[i].code;
+
+		for (j = 0; j < IQS269_NUM_CH; j++) {
+			keycode = iqs269->keycode[i * IQS269_NUM_CH + j];
+
+			/*
+			 * Hall-effect sensing repurposes a pair of dedicated
+			 * channels, only one of which reports events.
+			 */
+			switch (j) {
+			case IQS269_CHx_HALL_ACTIVE:
+				if (iqs269->hall_enable &&
+				    iqs269->switches[i].enabled) {
+					input_set_capability(iqs269->keypad,
+							     EV_SW, sw_code);
+					input_report_switch(iqs269->keypad,
+							    sw_code,
+							    state & BIT(j));
+				}
+
+				/* fall through */
+
+			case IQS269_CHx_HALL_INACTIVE:
+				if (iqs269->hall_enable)
+					continue;
+
+				/* fall through */
+
+			default:
+				if (keycode != KEY_RESERVED)
+					input_set_capability(iqs269->keypad,
+							     EV_KEY, keycode);
+			}
+		}
+	}
+
+	input_sync(iqs269->keypad);
+
+	error = input_register_device(iqs269->keypad);
+	if (error) {
+		dev_err(&client->dev, "Failed to register keypad: %d\n", error);
+		return error;
+	}
+
+	for (i = 0; i < IQS269_NUM_SL; i++) {
+		if (!iqs269->sys_reg.slider_select[i])
+			continue;
+
+		iqs269->slider[i] = devm_input_allocate_device(&client->dev);
+		if (!iqs269->slider[i])
+			return -ENOMEM;
+
+		iqs269->slider[i]->name = i ? "iqs269a_slider_1"
+					    : "iqs269a_slider_0";
+		iqs269->slider[i]->id.bustype = BUS_I2C;
+
+		input_set_capability(iqs269->slider[i], EV_KEY, BTN_TOUCH);
+		input_set_abs_params(iqs269->slider[i], ABS_X, 0, 255, 0, 0);
+
+		error = input_register_device(iqs269->slider[i]);
+		if (error) {
+			dev_err(&client->dev,
+				"Failed to register slider %d: %d\n", i, error);
+			return error;
+		}
+	}
+
+	return 0;
+}
+
+static int iqs269_report(struct iqs269_private *iqs269)
+{
+	struct i2c_client *client = iqs269->client;
+	struct iqs269_flags flags;
+	unsigned int sw_code, keycode;
+	int error, i, j;
+	u8 slider_x[IQS269_NUM_SL];
+	u8 dir_mask, state;
+
+	error = regmap_raw_read(iqs269->regmap, IQS269_SYS_FLAGS, &flags,
+				sizeof(flags));
+	if (error) {
+		dev_err(&client->dev, "Failed to read device status: %d\n",
+			error);
+		return error;
+	}
+
+	/*
+	 * The device resets itself if its own watchdog bites, which can happen
+	 * in the event of an I2C communication error. In this case, the device
+	 * asserts a SHOW_RESET interrupt and all registers must be restored.
+	 */
+	if (be16_to_cpu(flags.system) & IQS269_SYS_FLAGS_SHOW_RESET) {
+		dev_err(&client->dev, "Unexpected device reset\n");
+
+		error = iqs269_dev_init(iqs269);
+		if (error)
+			dev_err(&client->dev,
+				"Failed to re-initialize device: %d\n", error);
+
+		return error;
+	}
+
+	error = regmap_raw_read(iqs269->regmap, IQS269_SLIDER_X, slider_x,
+				sizeof(slider_x));
+	if (error) {
+		dev_err(&client->dev, "Failed to read slider position: %d\n",
+			error);
+		return error;
+	}
+
+	for (i = 0; i < IQS269_NUM_SL; i++) {
+		if (!iqs269->sys_reg.slider_select[i])
+			continue;
+
+		/*
+		 * Report BTN_TOUCH if any channel that participates in the
+		 * slider is in a state of touch.
+		 */
+		if (flags.states[IQS269_ST_OFFS_TOUCH] &
+		    iqs269->sys_reg.slider_select[i]) {
+			input_report_key(iqs269->slider[i], BTN_TOUCH, 1);
+			input_report_abs(iqs269->slider[i], ABS_X, slider_x[i]);
+		} else {
+			input_report_key(iqs269->slider[i], BTN_TOUCH, 0);
+		}
+
+		input_sync(iqs269->slider[i]);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(iqs269_events); i++) {
+		dir_mask = flags.states[IQS269_ST_OFFS_DIR];
+		if (!iqs269_events[i].dir_up)
+			dir_mask = ~dir_mask;
+
+		state = flags.states[iqs269_events[i].st_offs] & dir_mask;
+
+		sw_code = iqs269->switches[i].code;
+
+		for (j = 0; j < IQS269_NUM_CH; j++) {
+			keycode = iqs269->keycode[i * IQS269_NUM_CH + j];
+
+			switch (j) {
+			case IQS269_CHx_HALL_ACTIVE:
+				if (iqs269->hall_enable &&
+				    iqs269->switches[i].enabled)
+					input_report_switch(iqs269->keypad,
+							    sw_code,
+							    state & BIT(j));
+
+				/* fall through */
+
+			case IQS269_CHx_HALL_INACTIVE:
+				if (iqs269->hall_enable)
+					continue;
+
+				/* fall through */
+
+			default:
+				input_report_key(iqs269->keypad, keycode,
+						 state & BIT(j));
+			}
+		}
+	}
+
+	input_sync(iqs269->keypad);
+
+	return 0;
+}
+
+static irqreturn_t iqs269_irq(int irq, void *context)
+{
+	struct iqs269_private *iqs269 = context;
+
+	if (iqs269_report(iqs269))
+		return IRQ_NONE;
+
+	/*
+	 * The device does not deassert its interrupt (RDY) pin until shortly
+	 * after receiving an I2C stop condition; the following delay ensures
+	 * the interrupt handler does not return before this time.
+	 */
+	iqs269_irq_wait();
+
+	return IRQ_HANDLED;
+}
+
+static ssize_t counts_show(struct device *dev,
+			   struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	struct i2c_client *client = iqs269->client;
+	__le16 counts;
+	int error;
+
+	if (!iqs269->ati_current || iqs269->hall_enable)
+		return -EPERM;
+
+	/*
+	 * Unsolicited I2C communication prompts the device to assert its RDY
+	 * pin, so disable the interrupt line until the operation is finished
+	 * and RDY has been deasserted.
+	 */
+	disable_irq(client->irq);
+
+	error = regmap_raw_read(iqs269->regmap,
+				IQS269_CHx_COUNTS + iqs269->ch_num * 2,
+				&counts, sizeof(counts));
+
+	iqs269_irq_wait();
+	enable_irq(client->irq);
+
+	if (error)
+		return error;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", le16_to_cpu(counts));
+}
+
+static ssize_t hall_bin_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	struct i2c_client *client = iqs269->client;
+	unsigned int val;
+	int error;
+
+	disable_irq(client->irq);
+
+	error = regmap_read(iqs269->regmap, IQS269_CAL_DATA_A, &val);
+
+	iqs269_irq_wait();
+	enable_irq(client->irq);
+
+	if (error)
+		return error;
+
+	switch (iqs269->ch_reg[IQS269_CHx_HALL_ACTIVE].rx_enable &
+		iqs269->ch_reg[IQS269_CHx_HALL_INACTIVE].rx_enable) {
+	case IQS269_HALL_PAD_R:
+		val &= IQS269_CAL_DATA_A_HALL_BIN_R_MASK;
+		val >>= IQS269_CAL_DATA_A_HALL_BIN_R_SHIFT;
+		break;
+
+	case IQS269_HALL_PAD_L:
+		val &= IQS269_CAL_DATA_A_HALL_BIN_L_MASK;
+		val >>= IQS269_CAL_DATA_A_HALL_BIN_L_SHIFT;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
+}
+
+static ssize_t hall_enable_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", iqs269->hall_enable);
+}
+
+static ssize_t hall_enable_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t count)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = kstrtouint(buf, 10, &val);
+	if (error)
+		return error;
+
+	mutex_lock(&iqs269->lock);
+
+	iqs269->hall_enable = val;
+	iqs269->ati_current = false;
+
+	mutex_unlock(&iqs269->lock);
+
+	return count;
+}
+
+static ssize_t ch_number_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", iqs269->ch_num);
+}
+
+static ssize_t ch_number_store(struct device *dev,
+			       struct device_attribute *attr, const char *buf,
+			       size_t count)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = kstrtouint(buf, 10, &val);
+	if (error)
+		return error;
+
+	if (val >= IQS269_NUM_CH)
+		return -EINVAL;
+
+	iqs269->ch_num = val;
+
+	return count;
+}
+
+static ssize_t rx_enable_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n",
+			 iqs269->ch_reg[iqs269->ch_num].rx_enable);
+}
+
+static ssize_t rx_enable_store(struct device *dev,
+			       struct device_attribute *attr, const char *buf,
+			       size_t count)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = kstrtouint(buf, 10, &val);
+	if (error)
+		return error;
+
+	if (val > 0xFF)
+		return -EINVAL;
+
+	mutex_lock(&iqs269->lock);
+
+	iqs269->ch_reg[iqs269->ch_num].rx_enable = val;
+	iqs269->ati_current = false;
+
+	mutex_unlock(&iqs269->lock);
+
+	return count;
+}
+
+static ssize_t ati_mode_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = iqs269_ati_mode_get(iqs269, iqs269->ch_num, &val);
+	if (error)
+		return error;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
+}
+
+static ssize_t ati_mode_store(struct device *dev,
+			      struct device_attribute *attr, const char *buf,
+			      size_t count)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = kstrtouint(buf, 10, &val);
+	if (error)
+		return error;
+
+	error = iqs269_ati_mode_set(iqs269, iqs269->ch_num, val);
+	if (error)
+		return error;
+
+	return count;
+}
+
+static ssize_t ati_base_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = iqs269_ati_base_get(iqs269, iqs269->ch_num, &val);
+	if (error)
+		return error;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
+}
+
+static ssize_t ati_base_store(struct device *dev,
+			      struct device_attribute *attr, const char *buf,
+			      size_t count)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = kstrtouint(buf, 10, &val);
+	if (error)
+		return error;
+
+	error = iqs269_ati_base_set(iqs269, iqs269->ch_num, val);
+	if (error)
+		return error;
+
+	return count;
+}
+
+static ssize_t ati_target_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = iqs269_ati_target_get(iqs269, iqs269->ch_num, &val);
+	if (error)
+		return error;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", val);
+}
+
+static ssize_t ati_target_store(struct device *dev,
+				struct device_attribute *attr, const char *buf,
+				size_t count)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	unsigned int val;
+	int error;
+
+	error = kstrtouint(buf, 10, &val);
+	if (error)
+		return error;
+
+	error = iqs269_ati_target_set(iqs269, iqs269->ch_num, val);
+	if (error)
+		return error;
+
+	return count;
+}
+
+static ssize_t ati_trigger_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", iqs269->ati_current);
+}
+
+static ssize_t ati_trigger_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t count)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	struct i2c_client *client = iqs269->client;
+	unsigned int val;
+	int error;
+
+	error = kstrtouint(buf, 10, &val);
+	if (error)
+		return error;
+
+	if (!val)
+		return count;
+
+	disable_irq(client->irq);
+
+	error = iqs269_dev_init(iqs269);
+
+	iqs269_irq_wait();
+	enable_irq(client->irq);
+
+	if (error)
+		return error;
+
+	return count;
+}
+
+static DEVICE_ATTR_RO(counts);
+static DEVICE_ATTR_RO(hall_bin);
+static DEVICE_ATTR_RW(hall_enable);
+static DEVICE_ATTR_RW(ch_number);
+static DEVICE_ATTR_RW(rx_enable);
+static DEVICE_ATTR_RW(ati_mode);
+static DEVICE_ATTR_RW(ati_base);
+static DEVICE_ATTR_RW(ati_target);
+static DEVICE_ATTR_RW(ati_trigger);
+
+static struct attribute *iqs269_attrs[] = {
+	&dev_attr_counts.attr,
+	&dev_attr_hall_bin.attr,
+	&dev_attr_hall_enable.attr,
+	&dev_attr_ch_number.attr,
+	&dev_attr_rx_enable.attr,
+	&dev_attr_ati_mode.attr,
+	&dev_attr_ati_base.attr,
+	&dev_attr_ati_target.attr,
+	&dev_attr_ati_trigger.attr,
+	NULL,
+};
+
+static const struct attribute_group iqs269_attr_group = {
+	.attrs = iqs269_attrs,
+};
+
+static const struct regmap_config iqs269_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.max_register = IQS269_MAX_REG,
+};
+
+static int iqs269_probe(struct i2c_client *client)
+{
+	struct iqs269_ver_info ver_info;
+	struct iqs269_private *iqs269;
+	int error;
+
+	iqs269 = devm_kzalloc(&client->dev, sizeof(*iqs269), GFP_KERNEL);
+	if (!iqs269)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, iqs269);
+	iqs269->client = client;
+
+	iqs269->regmap = devm_regmap_init_i2c(client, &iqs269_regmap_config);
+	if (IS_ERR(iqs269->regmap)) {
+		error = PTR_ERR(iqs269->regmap);
+		dev_err(&client->dev, "Failed to initialize register map: %d\n",
+			error);
+		return error;
+	}
+
+	mutex_init(&iqs269->lock);
+
+	error = regmap_raw_read(iqs269->regmap, IQS269_VER_INFO, &ver_info,
+				sizeof(ver_info));
+	if (error)
+		return error;
+
+	if (ver_info.prod_num != IQS269_VER_INFO_PROD_NUM) {
+		dev_err(&client->dev, "Unrecognized product number: 0x%02X\n",
+			ver_info.prod_num);
+		return -EINVAL;
+	}
+
+	error = iqs269_parse_prop(iqs269);
+	if (error)
+		return error;
+
+	error = iqs269_dev_init(iqs269);
+	if (error) {
+		dev_err(&client->dev, "Failed to initialize device: %d\n",
+			error);
+		return error;
+	}
+
+	error = iqs269_input_init(iqs269);
+	if (error)
+		return error;
+
+	error = devm_request_threaded_irq(&client->dev, client->irq,
+					  NULL, iqs269_irq, IRQF_ONESHOT,
+					  client->name, iqs269);
+	if (error) {
+		dev_err(&client->dev, "Failed to request IRQ: %d\n", error);
+		return error;
+	}
+
+	error = devm_device_add_group(&client->dev, &iqs269_attr_group);
+	if (error)
+		dev_err(&client->dev, "Failed to add attributes: %d\n", error);
+
+	return error;
+}
+
+static int __maybe_unused iqs269_suspend(struct device *dev)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	struct i2c_client *client = iqs269->client;
+	unsigned int val;
+	int error;
+
+	if (!iqs269->suspend_mode)
+		return 0;
+
+	disable_irq(client->irq);
+
+	/*
+	 * Automatic power mode switching must be disabled before the device is
+	 * forced into any particular power mode. In this case, the device will
+	 * transition into normal-power mode.
+	 */
+	error = regmap_update_bits(iqs269->regmap, IQS269_SYS_SETTINGS,
+				   IQS269_SYS_SETTINGS_DIS_AUTO, ~0);
+	if (error)
+		goto err_irq;
+
+	/*
+	 * The following check ensures the device has completed its transition
+	 * into normal-power mode before a manual mode switch is performed.
+	 */
+	error = regmap_read_poll_timeout(iqs269->regmap, IQS269_SYS_FLAGS, val,
+					!(val & IQS269_SYS_FLAGS_PWR_MODE_MASK),
+					 IQS269_PWR_MODE_POLL_SLEEP_US,
+					 IQS269_PWR_MODE_POLL_TIMEOUT_US);
+	if (error)
+		goto err_irq;
+
+	error = regmap_update_bits(iqs269->regmap, IQS269_SYS_SETTINGS,
+				   IQS269_SYS_SETTINGS_PWR_MODE_MASK,
+				   iqs269->suspend_mode <<
+				   IQS269_SYS_SETTINGS_PWR_MODE_SHIFT);
+	if (error)
+		goto err_irq;
+
+	/*
+	 * This last check ensures the device has completed its transition into
+	 * the desired power mode to prevent any spurious interrupts from being
+	 * triggered after iqs269_suspend has already returned.
+	 */
+	error = regmap_read_poll_timeout(iqs269->regmap, IQS269_SYS_FLAGS, val,
+					 (val & IQS269_SYS_FLAGS_PWR_MODE_MASK)
+					 == (iqs269->suspend_mode <<
+					     IQS269_SYS_FLAGS_PWR_MODE_SHIFT),
+					 IQS269_PWR_MODE_POLL_SLEEP_US,
+					 IQS269_PWR_MODE_POLL_TIMEOUT_US);
+
+err_irq:
+	iqs269_irq_wait();
+	enable_irq(client->irq);
+
+	return error;
+}
+
+static int __maybe_unused iqs269_resume(struct device *dev)
+{
+	struct iqs269_private *iqs269 = dev_get_drvdata(dev);
+	struct i2c_client *client = iqs269->client;
+	unsigned int val;
+	int error;
+
+	if (!iqs269->suspend_mode)
+		return 0;
+
+	disable_irq(client->irq);
+
+	error = regmap_update_bits(iqs269->regmap, IQS269_SYS_SETTINGS,
+				   IQS269_SYS_SETTINGS_PWR_MODE_MASK, 0);
+	if (error)
+		goto err_irq;
+
+	/*
+	 * This check ensures the device has returned to normal-power mode
+	 * before automatic power mode switching is re-enabled.
+	 */
+	error = regmap_read_poll_timeout(iqs269->regmap, IQS269_SYS_FLAGS, val,
+					!(val & IQS269_SYS_FLAGS_PWR_MODE_MASK),
+					 IQS269_PWR_MODE_POLL_SLEEP_US,
+					 IQS269_PWR_MODE_POLL_TIMEOUT_US);
+	if (error)
+		goto err_irq;
+
+	error = regmap_update_bits(iqs269->regmap, IQS269_SYS_SETTINGS,
+				   IQS269_SYS_SETTINGS_DIS_AUTO, 0);
+	if (error)
+		goto err_irq;
+
+	/*
+	 * This step reports any events that may have been "swallowed" as a
+	 * result of polling PWR_MODE (which automatically acknowledges any
+	 * pending interrupts).
+	 */
+	error = iqs269_report(iqs269);
+
+err_irq:
+	iqs269_irq_wait();
+	enable_irq(client->irq);
+
+	return error;
+}
+
+static SIMPLE_DEV_PM_OPS(iqs269_pm, iqs269_suspend, iqs269_resume);
+
+static const struct of_device_id iqs269_of_match[] = {
+	{ .compatible = "azoteq,iqs269a" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, iqs269_of_match);
+
+static struct i2c_driver iqs269_i2c_driver = {
+	.driver = {
+		.name = "iqs269a",
+		.of_match_table = iqs269_of_match,
+		.pm = &iqs269_pm,
+	},
+	.probe_new = iqs269_probe,
+};
+module_i2c_driver(iqs269_i2c_driver);
+
+MODULE_AUTHOR("Jeff LaBundy <jeff@labundy.com>");
+MODULE_DESCRIPTION("Azoteq IQS269A Capacitive Touch Controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/misc/msm-vibrator.c b/drivers/input/misc/msm-vibrator.c
deleted file mode 100644
index b60f1aaee705..000000000000
--- a/drivers/input/misc/msm-vibrator.c
+++ /dev/null
@@ -1,281 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Qualcomm MSM vibrator driver
- *
- * Copyright (c) 2018 Brian Masney <masneyb@onstation.org>
- *
- * Based on qcom,pwm-vibrator.c from:
- * Copyright (c) 2018 Jonathan Marek <jonathan@marek.ca>
- *
- * Based on msm_pwm_vibrator.c from downstream Android sources:
- * Copyright (C) 2009-2014 LGE, Inc.
- */
-
-#include <linux/clk.h>
-#include <linux/err.h>
-#include <linux/gpio/consumer.h>
-#include <linux/input.h>
-#include <linux/io.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/regulator/consumer.h>
-
-#define REG_CMD_RCGR           0x00
-#define REG_CFG_RCGR           0x04
-#define REG_M                  0x08
-#define REG_N                  0x0C
-#define REG_D                  0x10
-#define REG_CBCR               0x24
-#define MMSS_CC_M_DEFAULT      1
-
-struct msm_vibrator {
-	struct input_dev *input;
-	struct mutex mutex;
-	struct work_struct worker;
-	void __iomem *base;
-	struct regulator *vcc;
-	struct clk *clk;
-	struct gpio_desc *enable_gpio;
-	u16 magnitude;
-	bool enabled;
-};
-
-static void msm_vibrator_write(struct msm_vibrator *vibrator, int offset,
-			       u32 value)
-{
-	writel(value, vibrator->base + offset);
-}
-
-static int msm_vibrator_start(struct msm_vibrator *vibrator)
-{
-	int d_reg_val, ret = 0;
-
-	mutex_lock(&vibrator->mutex);
-
-	if (!vibrator->enabled) {
-		ret = clk_set_rate(vibrator->clk, 24000);
-		if (ret) {
-			dev_err(&vibrator->input->dev,
-				"Failed to set clock rate: %d\n", ret);
-			goto unlock;
-		}
-
-		ret = clk_prepare_enable(vibrator->clk);
-		if (ret) {
-			dev_err(&vibrator->input->dev,
-				"Failed to enable clock: %d\n", ret);
-			goto unlock;
-		}
-
-		ret = regulator_enable(vibrator->vcc);
-		if (ret) {
-			dev_err(&vibrator->input->dev,
-				"Failed to enable regulator: %d\n", ret);
-			clk_disable(vibrator->clk);
-			goto unlock;
-		}
-
-		gpiod_set_value_cansleep(vibrator->enable_gpio, 1);
-
-		vibrator->enabled = true;
-	}
-
-	d_reg_val = 127 - ((126 * vibrator->magnitude) / 0xffff);
-	msm_vibrator_write(vibrator, REG_CFG_RCGR,
-			   (2 << 12) | /* dual edge mode */
-			   (0 << 8) |  /* cxo */
-			   (7 << 0));
-	msm_vibrator_write(vibrator, REG_M, 1);
-	msm_vibrator_write(vibrator, REG_N, 128);
-	msm_vibrator_write(vibrator, REG_D, d_reg_val);
-	msm_vibrator_write(vibrator, REG_CMD_RCGR, 1);
-	msm_vibrator_write(vibrator, REG_CBCR, 1);
-
-unlock:
-	mutex_unlock(&vibrator->mutex);
-
-	return ret;
-}
-
-static void msm_vibrator_stop(struct msm_vibrator *vibrator)
-{
-	mutex_lock(&vibrator->mutex);
-
-	if (vibrator->enabled) {
-		gpiod_set_value_cansleep(vibrator->enable_gpio, 0);
-		regulator_disable(vibrator->vcc);
-		clk_disable(vibrator->clk);
-		vibrator->enabled = false;
-	}
-
-	mutex_unlock(&vibrator->mutex);
-}
-
-static void msm_vibrator_worker(struct work_struct *work)
-{
-	struct msm_vibrator *vibrator = container_of(work,
-						     struct msm_vibrator,
-						     worker);
-
-	if (vibrator->magnitude)
-		msm_vibrator_start(vibrator);
-	else
-		msm_vibrator_stop(vibrator);
-}
-
-static int msm_vibrator_play_effect(struct input_dev *dev, void *data,
-				    struct ff_effect *effect)
-{
-	struct msm_vibrator *vibrator = input_get_drvdata(dev);
-
-	mutex_lock(&vibrator->mutex);
-
-	if (effect->u.rumble.strong_magnitude > 0)
-		vibrator->magnitude = effect->u.rumble.strong_magnitude;
-	else
-		vibrator->magnitude = effect->u.rumble.weak_magnitude;
-
-	mutex_unlock(&vibrator->mutex);
-
-	schedule_work(&vibrator->worker);
-
-	return 0;
-}
-
-static void msm_vibrator_close(struct input_dev *input)
-{
-	struct msm_vibrator *vibrator = input_get_drvdata(input);
-
-	cancel_work_sync(&vibrator->worker);
-	msm_vibrator_stop(vibrator);
-}
-
-static int msm_vibrator_probe(struct platform_device *pdev)
-{
-	struct msm_vibrator *vibrator;
-	struct resource *res;
-	int ret;
-
-	vibrator = devm_kzalloc(&pdev->dev, sizeof(*vibrator), GFP_KERNEL);
-	if (!vibrator)
-		return -ENOMEM;
-
-	vibrator->input = devm_input_allocate_device(&pdev->dev);
-	if (!vibrator->input)
-		return -ENOMEM;
-
-	vibrator->vcc = devm_regulator_get(&pdev->dev, "vcc");
-	if (IS_ERR(vibrator->vcc)) {
-		if (PTR_ERR(vibrator->vcc) != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "Failed to get regulator: %ld\n",
-				PTR_ERR(vibrator->vcc));
-		return PTR_ERR(vibrator->vcc);
-	}
-
-	vibrator->enable_gpio = devm_gpiod_get(&pdev->dev, "enable",
-					       GPIOD_OUT_LOW);
-	if (IS_ERR(vibrator->enable_gpio)) {
-		if (PTR_ERR(vibrator->enable_gpio) != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "Failed to get enable gpio: %ld\n",
-				PTR_ERR(vibrator->enable_gpio));
-		return PTR_ERR(vibrator->enable_gpio);
-	}
-
-	vibrator->clk = devm_clk_get(&pdev->dev, "pwm");
-	if (IS_ERR(vibrator->clk)) {
-		if (PTR_ERR(vibrator->clk) != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "Failed to lookup pwm clock: %ld\n",
-				PTR_ERR(vibrator->clk));
-		return PTR_ERR(vibrator->clk);
-	}
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(&pdev->dev, "Failed to get platform resource\n");
-		return -ENODEV;
-	}
-
-	vibrator->base = devm_ioremap(&pdev->dev, res->start,
-				     resource_size(res));
-	if (!vibrator->base) {
-		dev_err(&pdev->dev, "Failed to iomap resource.\n");
-		return -ENOMEM;
-	}
-
-	vibrator->enabled = false;
-	mutex_init(&vibrator->mutex);
-	INIT_WORK(&vibrator->worker, msm_vibrator_worker);
-
-	vibrator->input->name = "msm-vibrator";
-	vibrator->input->id.bustype = BUS_HOST;
-	vibrator->input->close = msm_vibrator_close;
-
-	input_set_drvdata(vibrator->input, vibrator);
-	input_set_capability(vibrator->input, EV_FF, FF_RUMBLE);
-
-	ret = input_ff_create_memless(vibrator->input, NULL,
-				      msm_vibrator_play_effect);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to create ff memless: %d", ret);
-		return ret;
-	}
-
-	ret = input_register_device(vibrator->input);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to register input device: %d", ret);
-		return ret;
-	}
-
-	platform_set_drvdata(pdev, vibrator);
-
-	return 0;
-}
-
-static int __maybe_unused msm_vibrator_suspend(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct msm_vibrator *vibrator = platform_get_drvdata(pdev);
-
-	cancel_work_sync(&vibrator->worker);
-
-	if (vibrator->enabled)
-		msm_vibrator_stop(vibrator);
-
-	return 0;
-}
-
-static int __maybe_unused msm_vibrator_resume(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct msm_vibrator *vibrator = platform_get_drvdata(pdev);
-
-	if (vibrator->enabled)
-		msm_vibrator_start(vibrator);
-
-	return 0;
-}
-
-static SIMPLE_DEV_PM_OPS(msm_vibrator_pm_ops, msm_vibrator_suspend,
-			 msm_vibrator_resume);
-
-static const struct of_device_id msm_vibrator_of_match[] = {
-	{ .compatible = "qcom,msm8226-vibrator" },
-	{ .compatible = "qcom,msm8974-vibrator" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, msm_vibrator_of_match);
-
-static struct platform_driver msm_vibrator_driver = {
-	.probe	= msm_vibrator_probe,
-	.driver	= {
-		.name = "msm-vibrator",
-		.pm = &msm_vibrator_pm_ops,
-		.of_match_table = of_match_ptr(msm_vibrator_of_match),
-	},
-};
-module_platform_driver(msm_vibrator_driver);
-
-MODULE_AUTHOR("Brian Masney <masneyb@onstation.org>");
-MODULE_DESCRIPTION("Qualcomm MSM vibrator driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/input/misc/xen-kbdfront.c b/drivers/input/misc/xen-kbdfront.c
index 24bc5c5d876f..a1bba722b234 100644
--- a/drivers/input/misc/xen-kbdfront.c
+++ b/drivers/input/misc/xen-kbdfront.c
@@ -146,7 +146,7 @@ static void xenkbd_handle_mt_event(struct xenkbd_info *info,
 		break;
 
 	case XENKBD_MT_EV_UP:
-		input_mt_report_slot_state(info->mtouch, MT_TOOL_FINGER, false);
+		input_mt_report_slot_inactive(info->mtouch);
 		break;
 
 	case XENKBD_MT_EV_SYN:
diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c
index 8719da540383..3f9354baac4b 100644
--- a/drivers/input/mouse/elan_i2c_core.c
+++ b/drivers/input/mouse/elan_i2c_core.c
@@ -938,7 +938,7 @@ static void elan_report_contact(struct elan_tp_data *data,
 		input_report_abs(input, ABS_MT_TOUCH_MINOR, minor);
 	} else {
 		input_mt_slot(input, contact_num);
-		input_mt_report_slot_state(input, MT_TOOL_FINGER, false);
+		input_mt_report_slot_inactive(input);
 	}
 }
 
diff --git a/drivers/input/serio/Kconfig b/drivers/input/serio/Kconfig
index 373a1646019e..0754744b9ce5 100644
--- a/drivers/input/serio/Kconfig
+++ b/drivers/input/serio/Kconfig
@@ -223,7 +223,7 @@ config SERIO_AMS_DELTA
 	tristate "Amstrad Delta (E3) mailboard support"
 	depends on MACH_AMS_DELTA
 	default y
-	---help---
+	help
 	  Say Y here if you have an E3 and want to use its mailboard,
 	  or any standard AT keyboard connected to the mailboard port.
 
diff --git a/drivers/input/serio/hp_sdc.c b/drivers/input/serio/hp_sdc.c
index 654252361653..13eacf6ab431 100644
--- a/drivers/input/serio/hp_sdc.c
+++ b/drivers/input/serio/hp_sdc.c
@@ -1021,7 +1021,7 @@ static int __init hp_sdc_register(void)
 	hp_sdc.base_io	 = (unsigned long) 0xf0428000;
 	hp_sdc.data_io	 = (unsigned long) hp_sdc.base_io + 1;
 	hp_sdc.status_io = (unsigned long) hp_sdc.base_io + 3;
-	if (!probe_kernel_read(&i, (unsigned char *)hp_sdc.data_io, 1))
+	if (!copy_from_kernel_nofault(&i, (unsigned char *)hp_sdc.data_io, 1))
 		hp_sdc.dev = (void *)1;
 	hp_sdc.dev_err   = hp_sdc_init();
 #endif
diff --git a/drivers/input/serio/i8042-ppcio.h b/drivers/input/serio/i8042-ppcio.h
deleted file mode 100644
index 391f94d9e47d..000000000000
--- a/drivers/input/serio/i8042-ppcio.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef _I8042_PPCIO_H
-#define _I8042_PPCIO_H
-
-
-#if defined(CONFIG_WALNUT)
-
-#define I8042_KBD_IRQ 25
-#define I8042_AUX_IRQ 26
-
-#define I8042_KBD_PHYS_DESC "walnutps2/serio0"
-#define I8042_AUX_PHYS_DESC "walnutps2/serio1"
-#define I8042_MUX_PHYS_DESC "walnutps2/serio%d"
-
-extern void *kb_cs;
-extern void *kb_data;
-
-#define I8042_COMMAND_REG (*(int *)kb_cs)
-#define I8042_DATA_REG (*(int *)kb_data)
-
-static inline int i8042_read_data(void)
-{
-	return readb(kb_data);
-}
-
-static inline int i8042_read_status(void)
-{
-	return readb(kb_cs);
-}
-
-static inline void i8042_write_data(int val)
-{
-	writeb(val, kb_data);
-}
-
-static inline void i8042_write_command(int val)
-{
-	writeb(val, kb_cs);
-}
-
-static inline int i8042_platform_init(void)
-{
-	i8042_reset = I8042_RESET_ALWAYS;
-	return 0;
-}
-
-static inline void i8042_platform_exit(void)
-{
-}
-
-#else
-
-#include "i8042-io.h"
-
-#endif
-
-#endif /* _I8042_PPCIO_H */
diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-x86ia64io.h
index 7e048b557462..7b08ff8ddf35 100644
--- a/drivers/input/serio/i8042-x86ia64io.h
+++ b/drivers/input/serio/i8042-x86ia64io.h
@@ -945,6 +945,7 @@ static int i8042_pnp_kbd_probe(struct pnp_dev *dev, const struct pnp_device_id *
 	}
 	i8042_pnp_id_to_string(dev->id, i8042_kbd_firmware_id,
 			       sizeof(i8042_kbd_firmware_id));
+	i8042_kbd_fwnode = dev_fwnode(&dev->dev);
 
 	/* Keyboard ports are always supposed to be wakeup-enabled */
 	device_set_wakeup_enable(&dev->dev, true);
diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c
index 20ff2bed3917..0dddf273afd9 100644
--- a/drivers/input/serio/i8042.c
+++ b/drivers/input/serio/i8042.c
@@ -21,6 +21,7 @@
 #include <linux/i8042.h>
 #include <linux/slab.h>
 #include <linux/suspend.h>
+#include <linux/property.h>
 
 #include <asm/io.h>
 
@@ -124,6 +125,7 @@ MODULE_PARM_DESC(unmask_kbd_data, "Unconditional enable (may reveal sensitive da
 static bool i8042_bypass_aux_irq_test;
 static char i8042_kbd_firmware_id[128];
 static char i8042_aux_firmware_id[128];
+static struct fwnode_handle *i8042_kbd_fwnode;
 
 #include "i8042.h"
 
@@ -1335,6 +1337,7 @@ static int __init i8042_create_kbd_port(void)
 	strlcpy(serio->phys, I8042_KBD_PHYS_DESC, sizeof(serio->phys));
 	strlcpy(serio->firmware_id, i8042_kbd_firmware_id,
 		sizeof(serio->firmware_id));
+	set_primary_fwnode(&serio->dev, i8042_kbd_fwnode);
 
 	port->serio = serio;
 	port->irq = I8042_KBD_IRQ;
diff --git a/drivers/input/serio/i8042.h b/drivers/input/serio/i8042.h
index 38dc27ad3c18..eb376700dfff 100644
--- a/drivers/input/serio/i8042.h
+++ b/drivers/input/serio/i8042.h
@@ -17,8 +17,6 @@
 #include "i8042-ip22io.h"
 #elif defined(CONFIG_SNI_RM)
 #include "i8042-snirm.h"
-#elif defined(CONFIG_PPC)
-#include "i8042-ppcio.h"
 #elif defined(CONFIG_SPARC)
 #include "i8042-sparcio.h"
 #elif defined(CONFIG_X86) || defined(CONFIG_IA64)
diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig
index c071f7c407b6..35c867b2d9a7 100644
--- a/drivers/input/touchscreen/Kconfig
+++ b/drivers/input/touchscreen/Kconfig
@@ -201,6 +201,18 @@ config TOUCHSCREEN_CHIPONE_ICN8505
 	  To compile this driver as a module, choose M here: the
 	  module will be called chipone_icn8505.
 
+config TOUCHSCREEN_CY8CTMA140
+	tristate "cy8ctma140 touchscreen"
+	depends on I2C
+	help
+	  Say Y here if you have a Cypress CY8CTMA140 capacitive
+	  touchscreen also just known as "TMA140"
+
+	  If unsure, say N.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called cy8ctma140.
+
 config TOUCHSCREEN_CY8CTMG110
 	tristate "cy8ctmg110 touchscreen"
 	depends on I2C
diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile
index 94c6162409b3..30d1e1b42492 100644
--- a/drivers/input/touchscreen/Makefile
+++ b/drivers/input/touchscreen/Makefile
@@ -22,6 +22,7 @@ obj-$(CONFIG_TOUCHSCREEN_BU21013)	+= bu21013_ts.o
 obj-$(CONFIG_TOUCHSCREEN_BU21029)	+= bu21029_ts.o
 obj-$(CONFIG_TOUCHSCREEN_CHIPONE_ICN8318)	+= chipone_icn8318.o
 obj-$(CONFIG_TOUCHSCREEN_CHIPONE_ICN8505)	+= chipone_icn8505.o
+obj-$(CONFIG_TOUCHSCREEN_CY8CTMA140)	+= cy8ctma140.o
 obj-$(CONFIG_TOUCHSCREEN_CY8CTMG110)	+= cy8ctmg110_ts.o
 obj-$(CONFIG_TOUCHSCREEN_CYTTSP_CORE)	+= cyttsp_core.o
 obj-$(CONFIG_TOUCHSCREEN_CYTTSP_I2C)	+= cyttsp_i2c.o cyttsp_i2c_common.o
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index ae60442efda0..a2189739e30f 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -822,8 +822,7 @@ static void mxt_proc_t9_message(struct mxt_data *data, u8 *message)
 		 * have happened.
 		 */
 		if (status & MXT_T9_RELEASE) {
-			input_mt_report_slot_state(input_dev,
-						   MT_TOOL_FINGER, 0);
+			input_mt_report_slot_inactive(input_dev);
 			mxt_input_sync(data);
 		}
 
@@ -839,7 +838,7 @@ static void mxt_proc_t9_message(struct mxt_data *data, u8 *message)
 		input_report_abs(input_dev, ABS_MT_TOUCH_MAJOR, area);
 	} else {
 		/* Touch no longer active, close out slot */
-		input_mt_report_slot_state(input_dev, MT_TOOL_FINGER, 0);
+		input_mt_report_slot_inactive(input_dev);
 	}
 
 	data->update_input = true;
@@ -947,7 +946,7 @@ static void mxt_proc_t100_message(struct mxt_data *data, u8 *message)
 		dev_dbg(dev, "[%u] release\n", id);
 
 		/* close out slot */
-		input_mt_report_slot_state(input_dev, 0, 0);
+		input_mt_report_slot_inactive(input_dev);
 	}
 
 	data->update_input = true;
diff --git a/drivers/input/touchscreen/cy8ctma140.c b/drivers/input/touchscreen/cy8ctma140.c
new file mode 100644
index 000000000000..a9be29139cbf
--- /dev/null
+++ b/drivers/input/touchscreen/cy8ctma140.c
@@ -0,0 +1,353 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Driver for Cypress CY8CTMA140 (TMA140) touchscreen
+ * (C) 2020 Linus Walleij <linus.walleij@linaro.org>
+ * (C) 2007 Cypress
+ * (C) 2007 Google, Inc.
+ *
+ * Inspired by the tma140_skomer.c driver in the Samsung GT-S7710 code
+ * drop. The GT-S7710 is codenamed "Skomer", the code also indicates
+ * that the same touchscreen was used in a product called "Lucas".
+ *
+ * The code drop for GT-S7710 also contains a firmware downloader and
+ * 15 (!) versions of the firmware drop from Cypress. But here we assume
+ * the firmware got downloaded to the touchscreen flash successfully and
+ * just use it to read the fingers. The shipped vendor driver does the
+ * same.
+ */
+
+#include <asm/unaligned.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/input.h>
+#include <linux/input/touchscreen.h>
+#include <linux/input/mt.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/i2c.h>
+#include <linux/regulator/consumer.h>
+#include <linux/delay.h>
+
+#define CY8CTMA140_NAME			"cy8ctma140"
+
+#define CY8CTMA140_MAX_FINGERS		4
+
+#define CY8CTMA140_GET_FINGERS		0x00
+#define CY8CTMA140_GET_FW_INFO		0x19
+
+/* This message also fits some bytes for touchkeys, if used */
+#define CY8CTMA140_PACKET_SIZE		31
+
+#define CY8CTMA140_INVALID_BUFFER_BIT	5
+
+struct cy8ctma140 {
+	struct input_dev *input;
+	struct touchscreen_properties props;
+	struct device *dev;
+	struct i2c_client *client;
+	struct regulator_bulk_data regulators[2];
+	u8 prev_fingers;
+	u8 prev_f1id;
+	u8 prev_f2id;
+};
+
+static void cy8ctma140_report(struct cy8ctma140 *ts, u8 *data, int n_fingers)
+{
+	static const u8 contact_offsets[] = { 0x03, 0x09, 0x10, 0x16 };
+	u8 *buf;
+	u16 x, y;
+	u8 w;
+	u8 id;
+	int slot;
+	int i;
+
+	for (i = 0; i < n_fingers; i++) {
+		buf = &data[contact_offsets[i]];
+
+		/*
+		 * Odd contacts have contact ID in the lower nibble of
+		 * the preceding byte, whereas even contacts have it in
+		 * the upper nibble of the following byte.
+		 */
+		id = i % 2 ? buf[-1] & 0x0f : buf[5] >> 4;
+		slot = input_mt_get_slot_by_key(ts->input, id);
+		if (slot < 0)
+			continue;
+
+		x = get_unaligned_be16(buf);
+		y = get_unaligned_be16(buf + 2);
+		w = buf[4];
+
+		dev_dbg(ts->dev, "finger %d: ID %02x (%d, %d) w: %d\n",
+			slot, id, x, y, w);
+
+		input_mt_slot(ts->input, slot);
+		input_mt_report_slot_state(ts->input, MT_TOOL_FINGER, true);
+		touchscreen_report_pos(ts->input, &ts->props, x, y, true);
+		input_report_abs(ts->input, ABS_MT_TOUCH_MAJOR, w);
+	}
+
+	input_mt_sync_frame(ts->input);
+	input_sync(ts->input);
+}
+
+static irqreturn_t cy8ctma140_irq_thread(int irq, void *d)
+{
+	struct cy8ctma140 *ts = d;
+	u8 cmdbuf[] = { CY8CTMA140_GET_FINGERS };
+	u8 buf[CY8CTMA140_PACKET_SIZE];
+	struct i2c_msg msg[] = {
+		{
+			.addr = ts->client->addr,
+			.flags = 0,
+			.len = sizeof(cmdbuf),
+			.buf = cmdbuf,
+		}, {
+			.addr = ts->client->addr,
+			.flags = I2C_M_RD,
+			.len = sizeof(buf),
+			.buf = buf,
+		},
+	};
+	u8 n_fingers;
+	int ret;
+
+	ret = i2c_transfer(ts->client->adapter, msg, ARRAY_SIZE(msg));
+	if (ret != ARRAY_SIZE(msg)) {
+		if (ret < 0)
+			dev_err(ts->dev, "error reading message: %d\n", ret);
+		else
+			dev_err(ts->dev, "wrong number of messages\n");
+		goto out;
+	}
+
+	if (buf[1] & BIT(CY8CTMA140_INVALID_BUFFER_BIT)) {
+		dev_dbg(ts->dev, "invalid event\n");
+		goto out;
+	}
+
+	n_fingers = buf[2] & 0x0f;
+	if (n_fingers > CY8CTMA140_MAX_FINGERS) {
+		dev_err(ts->dev, "unexpected number of fingers: %d\n",
+			n_fingers);
+		goto out;
+	}
+
+	cy8ctma140_report(ts, buf, n_fingers);
+
+out:
+	return IRQ_HANDLED;
+}
+
+static int cy8ctma140_init(struct cy8ctma140 *ts)
+{
+	u8 addr[1];
+	u8 buf[5];
+	int ret;
+
+	addr[0] = CY8CTMA140_GET_FW_INFO;
+	ret = i2c_master_send(ts->client, addr, 1);
+	if (ret < 0) {
+		dev_err(ts->dev, "error sending FW info message\n");
+		return ret;
+	}
+	ret = i2c_master_recv(ts->client, buf, 5);
+	if (ret < 0) {
+		dev_err(ts->dev, "error receiving FW info message\n");
+		return ret;
+	}
+	if (ret != 5) {
+		dev_err(ts->dev, "got only %d bytes\n", ret);
+		return -EIO;
+	}
+
+	dev_dbg(ts->dev, "vendor %c%c, HW ID %.2d, FW ver %.4d\n",
+		buf[0], buf[1], buf[3], buf[4]);
+
+	return 0;
+}
+
+static int cy8ctma140_power_up(struct cy8ctma140 *ts)
+{
+	int error;
+
+	error = regulator_bulk_enable(ARRAY_SIZE(ts->regulators),
+				      ts->regulators);
+	if (error) {
+		dev_err(ts->dev, "failed to enable regulators\n");
+		return error;
+	}
+
+	msleep(250);
+
+	return 0;
+}
+
+static void cy8ctma140_power_down(struct cy8ctma140 *ts)
+{
+	regulator_bulk_disable(ARRAY_SIZE(ts->regulators),
+			       ts->regulators);
+}
+
+/* Called from the registered devm action */
+static void cy8ctma140_power_off_action(void *d)
+{
+	struct cy8ctma140 *ts = d;
+
+	cy8ctma140_power_down(ts);
+}
+
+static int cy8ctma140_probe(struct i2c_client *client,
+			    const struct i2c_device_id *id)
+{
+	struct cy8ctma140 *ts;
+	struct input_dev *input;
+	struct device *dev = &client->dev;
+	int error;
+
+	ts = devm_kzalloc(dev, sizeof(*ts), GFP_KERNEL);
+	if (!ts)
+		return -ENOMEM;
+
+	input = devm_input_allocate_device(dev);
+	if (!input)
+		return -ENOMEM;
+
+	ts->dev = dev;
+	ts->client = client;
+	ts->input = input;
+
+	input_set_capability(input, EV_ABS, ABS_MT_POSITION_X);
+	input_set_capability(input, EV_ABS, ABS_MT_POSITION_Y);
+	/* One byte for width 0..255 so this is the limit */
+	input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0);
+	/*
+	 * This sets up event max/min capabilities and fuzz.
+	 * Some DT properties are compulsory so we do not need
+	 * to provide defaults for X/Y max or pressure max.
+	 *
+	 * We just initialize a very simple MT touchscreen here,
+	 * some devices use the capability of this touchscreen to
+	 * provide touchkeys, and in that case this needs to be
+	 * extended to handle touchkey input.
+	 *
+	 * The firmware takes care of finger tracking and dropping
+	 * invalid ranges.
+	 */
+	touchscreen_parse_properties(input, true, &ts->props);
+	input_abs_set_fuzz(input, ABS_MT_POSITION_X, 0);
+	input_abs_set_fuzz(input, ABS_MT_POSITION_Y, 0);
+
+	error = input_mt_init_slots(input, CY8CTMA140_MAX_FINGERS,
+				  INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED);
+	if (error)
+		return error;
+
+	input->name = CY8CTMA140_NAME;
+	input->id.bustype = BUS_I2C;
+	input_set_drvdata(input, ts);
+
+	/*
+	 * VCPIN is the analog voltage supply
+	 * VDD is the digital voltage supply
+	 * since the voltage range of VDD overlaps that of VCPIN,
+	 * many designs to just supply both with a single voltage
+	 * source of ~3.3 V.
+	 */
+	ts->regulators[0].supply = "vcpin";
+	ts->regulators[1].supply = "vdd";
+	error = devm_regulator_bulk_get(dev, ARRAY_SIZE(ts->regulators),
+				      ts->regulators);
+	if (error) {
+		if (error != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get regulators %d\n",
+				error);
+		return error;
+	}
+
+	error = cy8ctma140_power_up(ts);
+	if (error)
+		return error;
+
+	error = devm_add_action_or_reset(dev, cy8ctma140_power_off_action, ts);
+	if (error) {
+		dev_err(dev, "failed to install power off handler\n");
+		return error;
+	}
+
+	error = devm_request_threaded_irq(dev, client->irq,
+					  NULL, cy8ctma140_irq_thread,
+					  IRQF_ONESHOT, CY8CTMA140_NAME, ts);
+	if (error) {
+		dev_err(dev, "irq %d busy? error %d\n", client->irq, error);
+		return error;
+	}
+
+	error = cy8ctma140_init(ts);
+	if (error)
+		return error;
+
+	error = input_register_device(input);
+	if (error)
+		return error;
+
+	i2c_set_clientdata(client, ts);
+
+	return 0;
+}
+
+static int __maybe_unused cy8ctma140_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct cy8ctma140 *ts = i2c_get_clientdata(client);
+
+	if (!device_may_wakeup(&client->dev))
+		cy8ctma140_power_down(ts);
+
+	return 0;
+}
+
+static int __maybe_unused cy8ctma140_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct cy8ctma140 *ts = i2c_get_clientdata(client);
+	int error;
+
+	if (!device_may_wakeup(&client->dev)) {
+		error = cy8ctma140_power_up(ts);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(cy8ctma140_pm, cy8ctma140_suspend, cy8ctma140_resume);
+
+static const struct i2c_device_id cy8ctma140_idtable[] = {
+	{ CY8CTMA140_NAME, 0 },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(i2c, cy8ctma140_idtable);
+
+static const struct of_device_id cy8ctma140_of_match[] = {
+	{ .compatible = "cypress,cy8ctma140", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, cy8ctma140_of_match);
+
+static struct i2c_driver cy8ctma140_driver = {
+	.driver		= {
+		.name	= CY8CTMA140_NAME,
+		.pm	= &cy8ctma140_pm,
+		.of_match_table = cy8ctma140_of_match,
+	},
+	.id_table	= cy8ctma140_idtable,
+	.probe		= cy8ctma140_probe,
+};
+module_i2c_driver(cy8ctma140_driver);
+
+MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
+MODULE_DESCRIPTION("CY8CTMA140 TouchScreen Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/touchscreen/cyttsp4_core.c b/drivers/input/touchscreen/cyttsp4_core.c
index 6bcffc930384..02a73d9a4def 100644
--- a/drivers/input/touchscreen/cyttsp4_core.c
+++ b/drivers/input/touchscreen/cyttsp4_core.c
@@ -744,8 +744,7 @@ static void cyttsp4_report_slot_liftoff(struct cyttsp4_mt_data *md,
 
 	for (t = 0; t < max_slots; t++) {
 		input_mt_slot(md->input, t);
-		input_mt_report_slot_state(md->input,
-			MT_TOOL_FINGER, false);
+		input_mt_report_slot_inactive(md->input);
 	}
 }
 
@@ -845,7 +844,7 @@ static void cyttsp4_final_sync(struct input_dev *input, int max_slots, int *ids)
 		if (ids[t])
 			continue;
 		input_mt_slot(input, t);
-		input_mt_report_slot_state(input, MT_TOOL_FINGER, false);
+		input_mt_report_slot_inactive(input);
 	}
 
 	input_sync(input);
diff --git a/drivers/input/touchscreen/cyttsp_core.c b/drivers/input/touchscreen/cyttsp_core.c
index 3f5d463dbeed..697aa2c158f7 100644
--- a/drivers/input/touchscreen/cyttsp_core.c
+++ b/drivers/input/touchscreen/cyttsp_core.c
@@ -340,7 +340,7 @@ static void cyttsp_report_tchdata(struct cyttsp *ts)
 			continue;
 
 		input_mt_slot(input, i);
-		input_mt_report_slot_state(input, MT_TOOL_FINGER, false);
+		input_mt_report_slot_inactive(input);
 	}
 
 	input_sync(input);
diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c
index d2587724c52a..3a4f18d3450d 100644
--- a/drivers/input/touchscreen/edt-ft5x06.c
+++ b/drivers/input/touchscreen/edt-ft5x06.c
@@ -38,6 +38,9 @@
 #define WORK_REGISTER_NUM_X		0x33
 #define WORK_REGISTER_NUM_Y		0x34
 
+#define PMOD_REGISTER_ACTIVE		0x00
+#define PMOD_REGISTER_HIBERNATE		0x03
+
 #define M09_REGISTER_THRESHOLD		0x80
 #define M09_REGISTER_GAIN		0x92
 #define M09_REGISTER_OFFSET		0x93
@@ -53,6 +56,7 @@
 
 #define WORK_REGISTER_OPMODE		0x3c
 #define FACTORY_REGISTER_OPMODE		0x01
+#define PMOD_REGISTER_OPMODE		0xa5
 
 #define TOUCH_EVENT_DOWN		0x00
 #define TOUCH_EVENT_UP			0x01
@@ -65,6 +69,12 @@
 #define EDT_RAW_DATA_RETRIES		100
 #define EDT_RAW_DATA_DELAY		1000 /* usec */
 
+enum edt_pmode {
+	EDT_PMODE_NOT_SUPPORTED,
+	EDT_PMODE_HIBERNATE,
+	EDT_PMODE_POWEROFF,
+};
+
 enum edt_ver {
 	EDT_M06,
 	EDT_M09,
@@ -103,6 +113,7 @@ struct edt_ft5x06_ts_data {
 
 	struct mutex mutex;
 	bool factory_mode;
+	enum edt_pmode suspend_mode;
 	int threshold;
 	int gain;
 	int offset;
@@ -527,6 +538,29 @@ static const struct attribute_group edt_ft5x06_attr_group = {
 	.attrs = edt_ft5x06_attrs,
 };
 
+static void edt_ft5x06_restore_reg_parameters(struct edt_ft5x06_ts_data *tsdata)
+{
+	struct edt_reg_addr *reg_addr = &tsdata->reg_addr;
+
+	edt_ft5x06_register_write(tsdata, reg_addr->reg_threshold,
+				  tsdata->threshold);
+	edt_ft5x06_register_write(tsdata, reg_addr->reg_gain,
+				  tsdata->gain);
+	if (reg_addr->reg_offset != NO_REGISTER)
+		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset,
+					  tsdata->offset);
+	if (reg_addr->reg_offset_x != NO_REGISTER)
+		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset_x,
+					  tsdata->offset_x);
+	if (reg_addr->reg_offset_y != NO_REGISTER)
+		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset_y,
+					  tsdata->offset_y);
+	if (reg_addr->reg_report_rate != NO_REGISTER)
+		edt_ft5x06_register_write(tsdata, reg_addr->reg_report_rate,
+				  tsdata->report_rate);
+
+}
+
 #ifdef CONFIG_DEBUG_FS
 static int edt_ft5x06_factory_mode(struct edt_ft5x06_ts_data *tsdata)
 {
@@ -592,7 +626,6 @@ static int edt_ft5x06_work_mode(struct edt_ft5x06_ts_data *tsdata)
 {
 	struct i2c_client *client = tsdata->client;
 	int retries = EDT_SWITCH_MODE_RETRIES;
-	struct edt_reg_addr *reg_addr = &tsdata->reg_addr;
 	int ret;
 	int error;
 
@@ -624,24 +657,7 @@ static int edt_ft5x06_work_mode(struct edt_ft5x06_ts_data *tsdata)
 	kfree(tsdata->raw_buffer);
 	tsdata->raw_buffer = NULL;
 
-	/* restore parameters */
-	edt_ft5x06_register_write(tsdata, reg_addr->reg_threshold,
-				  tsdata->threshold);
-	edt_ft5x06_register_write(tsdata, reg_addr->reg_gain,
-				  tsdata->gain);
-	if (reg_addr->reg_offset != NO_REGISTER)
-		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset,
-					  tsdata->offset);
-	if (reg_addr->reg_offset_x != NO_REGISTER)
-		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset_x,
-					  tsdata->offset_x);
-	if (reg_addr->reg_offset_y != NO_REGISTER)
-		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset_y,
-					  tsdata->offset_y);
-	if (reg_addr->reg_report_rate != NO_REGISTER)
-		edt_ft5x06_register_write(tsdata, reg_addr->reg_report_rate,
-				  tsdata->report_rate);
-
+	edt_ft5x06_restore_reg_parameters(tsdata);
 	enable_irq(client->irq);
 
 	return 0;
@@ -762,9 +778,8 @@ static const struct file_operations debugfs_raw_data_fops = {
 	.read = edt_ft5x06_debugfs_raw_data_read,
 };
 
-static void
-edt_ft5x06_ts_prepare_debugfs(struct edt_ft5x06_ts_data *tsdata,
-			      const char *debugfs_name)
+static void edt_ft5x06_ts_prepare_debugfs(struct edt_ft5x06_ts_data *tsdata,
+					  const char *debugfs_name)
 {
 	tsdata->debug_dir = debugfs_create_dir(debugfs_name, NULL);
 
@@ -777,8 +792,7 @@ edt_ft5x06_ts_prepare_debugfs(struct edt_ft5x06_ts_data *tsdata,
 			    tsdata->debug_dir, tsdata, &debugfs_raw_data_fops);
 }
 
-static void
-edt_ft5x06_ts_teardown_debugfs(struct edt_ft5x06_ts_data *tsdata)
+static void edt_ft5x06_ts_teardown_debugfs(struct edt_ft5x06_ts_data *tsdata)
 {
 	debugfs_remove_recursive(tsdata->debug_dir);
 	kfree(tsdata->raw_buffer);
@@ -786,14 +800,17 @@ edt_ft5x06_ts_teardown_debugfs(struct edt_ft5x06_ts_data *tsdata)
 
 #else
 
-static inline void
-edt_ft5x06_ts_prepare_debugfs(struct edt_ft5x06_ts_data *tsdata,
-			      const char *debugfs_name)
+static int edt_ft5x06_factory_mode(struct edt_ft5x06_ts_data *tsdata)
+{
+	return -ENOSYS;
+}
+
+static void edt_ft5x06_ts_prepare_debugfs(struct edt_ft5x06_ts_data *tsdata,
+					  const char *debugfs_name)
 {
 }
 
-static inline void
-edt_ft5x06_ts_teardown_debugfs(struct edt_ft5x06_ts_data *tsdata)
+static void edt_ft5x06_ts_teardown_debugfs(struct edt_ft5x06_ts_data *tsdata)
 {
 }
 
@@ -938,19 +955,25 @@ static void edt_ft5x06_ts_get_defaults(struct device *dev,
 
 	error = device_property_read_u32(dev, "offset", &val);
 	if (!error) {
-		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset, val);
+		if (reg_addr->reg_offset != NO_REGISTER)
+			edt_ft5x06_register_write(tsdata,
+						  reg_addr->reg_offset, val);
 		tsdata->offset = val;
 	}
 
 	error = device_property_read_u32(dev, "offset-x", &val);
 	if (!error) {
-		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset_x, val);
+		if (reg_addr->reg_offset_x != NO_REGISTER)
+			edt_ft5x06_register_write(tsdata,
+						  reg_addr->reg_offset_x, val);
 		tsdata->offset_x = val;
 	}
 
 	error = device_property_read_u32(dev, "offset-y", &val);
 	if (!error) {
-		edt_ft5x06_register_write(tsdata, reg_addr->reg_offset_y, val);
+		if (reg_addr->reg_offset_y != NO_REGISTER)
+			edt_ft5x06_register_write(tsdata,
+						  reg_addr->reg_offset_y, val);
 		tsdata->offset_y = val;
 	}
 }
@@ -1114,6 +1137,19 @@ static int edt_ft5x06_ts_probe(struct i2c_client *client,
 		return error;
 	}
 
+	/*
+	 * Check which sleep modes we can support. Power-off requieres the
+	 * reset-pin to ensure correct power-down/power-up behaviour. Start with
+	 * the EDT_PMODE_POWEROFF test since this is the deepest possible sleep
+	 * mode.
+	 */
+	if (tsdata->reset_gpio)
+		tsdata->suspend_mode = EDT_PMODE_POWEROFF;
+	else if (tsdata->wake_gpio)
+		tsdata->suspend_mode = EDT_PMODE_HIBERNATE;
+	else
+		tsdata->suspend_mode = EDT_PMODE_NOT_SUPPORTED;
+
 	if (tsdata->wake_gpio) {
 		usleep_range(5000, 6000);
 		gpiod_set_value_cansleep(tsdata->wake_gpio, 1);
@@ -1227,6 +1263,102 @@ static int edt_ft5x06_ts_remove(struct i2c_client *client)
 	return 0;
 }
 
+static int __maybe_unused edt_ft5x06_ts_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct edt_ft5x06_ts_data *tsdata = i2c_get_clientdata(client);
+	struct gpio_desc *reset_gpio = tsdata->reset_gpio;
+	int ret;
+
+	if (device_may_wakeup(dev))
+		return 0;
+
+	if (tsdata->suspend_mode == EDT_PMODE_NOT_SUPPORTED)
+		return 0;
+
+	/* Enter hibernate mode. */
+	ret = edt_ft5x06_register_write(tsdata, PMOD_REGISTER_OPMODE,
+					PMOD_REGISTER_HIBERNATE);
+	if (ret)
+		dev_warn(dev, "Failed to set hibernate mode\n");
+
+	if (tsdata->suspend_mode == EDT_PMODE_HIBERNATE)
+		return 0;
+
+	/*
+	 * Power-off according the datasheet. Cut the power may leaf the irq
+	 * line in an undefined state depending on the host pull resistor
+	 * settings. Disable the irq to avoid adjusting each host till the
+	 * device is back in a full functional state.
+	 */
+	disable_irq(tsdata->client->irq);
+
+	gpiod_set_value_cansleep(reset_gpio, 1);
+	usleep_range(1000, 2000);
+
+	ret = regulator_disable(tsdata->vcc);
+	if (ret)
+		dev_warn(dev, "Failed to disable vcc\n");
+
+	return 0;
+}
+
+static int __maybe_unused edt_ft5x06_ts_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct edt_ft5x06_ts_data *tsdata = i2c_get_clientdata(client);
+	int ret = 0;
+
+	if (device_may_wakeup(dev))
+		return 0;
+
+	if (tsdata->suspend_mode == EDT_PMODE_NOT_SUPPORTED)
+		return 0;
+
+	if (tsdata->suspend_mode == EDT_PMODE_POWEROFF) {
+		struct gpio_desc *reset_gpio = tsdata->reset_gpio;
+
+		/*
+		 * We can't check if the regulator is a dummy or a real
+		 * regulator. So we need to specify the 5ms reset time (T_rst)
+		 * here instead of the 100us T_rtp time. We also need to wait
+		 * 300ms in case it was a real supply and the power was cutted
+		 * of. Toggle the reset pin is also a way to exit the hibernate
+		 * mode.
+		 */
+		gpiod_set_value_cansleep(reset_gpio, 1);
+		usleep_range(5000, 6000);
+
+		ret = regulator_enable(tsdata->vcc);
+		if (ret) {
+			dev_err(dev, "Failed to enable vcc\n");
+			return ret;
+		}
+
+		usleep_range(1000, 2000);
+		gpiod_set_value_cansleep(reset_gpio, 0);
+		msleep(300);
+
+		edt_ft5x06_restore_reg_parameters(tsdata);
+		enable_irq(tsdata->client->irq);
+
+		if (tsdata->factory_mode)
+			ret = edt_ft5x06_factory_mode(tsdata);
+	} else {
+		struct gpio_desc *wake_gpio = tsdata->wake_gpio;
+
+		gpiod_set_value_cansleep(wake_gpio, 0);
+		usleep_range(5000, 6000);
+		gpiod_set_value_cansleep(wake_gpio, 1);
+	}
+
+
+	return ret;
+}
+
+static SIMPLE_DEV_PM_OPS(edt_ft5x06_ts_pm_ops,
+			 edt_ft5x06_ts_suspend, edt_ft5x06_ts_resume);
+
 static const struct edt_i2c_chip_data edt_ft5x06_data = {
 	.max_support_points = 5,
 };
@@ -1265,6 +1397,8 @@ static struct i2c_driver edt_ft5x06_ts_driver = {
 	.driver = {
 		.name = "edt_ft5x06",
 		.of_match_table = edt_ft5x06_of_match,
+		.pm = &edt_ft5x06_ts_pm_ops,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 	.id_table = edt_ft5x06_ts_id,
 	.probe    = edt_ft5x06_ts_probe,
diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c
index 2289f9638116..233cb1085bbd 100644
--- a/drivers/input/touchscreen/elants_i2c.c
+++ b/drivers/input/touchscreen/elants_i2c.c
@@ -33,6 +33,7 @@
 #include <linux/slab.h>
 #include <linux/firmware.h>
 #include <linux/input/mt.h>
+#include <linux/input/touchscreen.h>
 #include <linux/acpi.h>
 #include <linux/of.h>
 #include <linux/gpio/consumer.h>
@@ -89,6 +90,7 @@
 /* FW read command, 0x53 0x?? 0x0, 0x01 */
 #define E_ELAN_INFO_FW_VER	0x00
 #define E_ELAN_INFO_BC_VER	0x10
+#define E_ELAN_INFO_REK		0xE0
 #define E_ELAN_INFO_TEST_VER	0xE0
 #define E_ELAN_INFO_FW_ID	0xF0
 #define E_INFO_OSR		0xD6
@@ -136,6 +138,7 @@ struct elants_data {
 	unsigned int y_res;
 	unsigned int x_max;
 	unsigned int y_max;
+	struct touchscreen_properties prop;
 
 	enum elants_state state;
 	enum elants_iap_mode iap_mode;
@@ -189,7 +192,8 @@ static int elants_i2c_read(struct i2c_client *client, void *data, size_t size)
 
 static int elants_i2c_execute_command(struct i2c_client *client,
 				      const u8 *cmd, size_t cmd_size,
-				      u8 *resp, size_t resp_size)
+				      u8 *resp, size_t resp_size,
+				      int retries, const char *cmd_name)
 {
 	struct i2c_msg msgs[2];
 	int ret;
@@ -209,30 +213,55 @@ static int elants_i2c_execute_command(struct i2c_client *client,
 		break;
 
 	default:
-		dev_err(&client->dev, "%s: invalid command %*ph\n",
-			__func__, (int)cmd_size, cmd);
+		dev_err(&client->dev, "(%s): invalid command: %*ph\n",
+			cmd_name, (int)cmd_size, cmd);
 		return -EINVAL;
 	}
 
-	msgs[0].addr = client->addr;
-	msgs[0].flags = client->flags & I2C_M_TEN;
-	msgs[0].len = cmd_size;
-	msgs[0].buf = (u8 *)cmd;
+	for (;;) {
+		msgs[0].addr = client->addr;
+		msgs[0].flags = client->flags & I2C_M_TEN;
+		msgs[0].len = cmd_size;
+		msgs[0].buf = (u8 *)cmd;
+
+		msgs[1].addr = client->addr;
+		msgs[1].flags = (client->flags & I2C_M_TEN) | I2C_M_RD;
+		msgs[1].flags |= I2C_M_RD;
+		msgs[1].len = resp_size;
+		msgs[1].buf = resp;
+
+		ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
+		if (ret < 0) {
+			if (--retries > 0) {
+				dev_dbg(&client->dev,
+					"(%s) I2C transfer failed: %pe (retrying)\n",
+					cmd_name, ERR_PTR(ret));
+				continue;
+			}
 
-	msgs[1].addr = client->addr;
-	msgs[1].flags = client->flags & I2C_M_TEN;
-	msgs[1].flags |= I2C_M_RD;
-	msgs[1].len = resp_size;
-	msgs[1].buf = resp;
+			dev_err(&client->dev,
+				"(%s) I2C transfer failed: %pe\n",
+				cmd_name, ERR_PTR(ret));
+			return ret;
+		}
 
-	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
-	if (ret < 0)
-		return ret;
+		if (ret != ARRAY_SIZE(msgs) ||
+		    resp[FW_HDR_TYPE] != expected_response) {
+			if (--retries > 0) {
+				dev_dbg(&client->dev,
+					"(%s) unexpected response: %*ph (retrying)\n",
+					cmd_name, ret, resp);
+				continue;
+			}
 
-	if (ret != ARRAY_SIZE(msgs) || resp[FW_HDR_TYPE] != expected_response)
-		return -EIO;
+			dev_err(&client->dev,
+				"(%s) unexpected response: %*ph\n",
+				cmd_name, ret, resp);
+			return -EIO;
+		}
 
-	return 0;
+		return 0;
+	}
 }
 
 static int elants_i2c_calibrate(struct elants_data *ts)
@@ -305,27 +334,21 @@ static u16 elants_i2c_parse_version(u8 *buf)
 static int elants_i2c_query_hw_version(struct elants_data *ts)
 {
 	struct i2c_client *client = ts->client;
-	int error, retry_cnt;
+	int retry_cnt = MAX_RETRIES;
 	const u8 cmd[] = { CMD_HEADER_READ, E_ELAN_INFO_FW_ID, 0x00, 0x01 };
 	u8 resp[HEADER_SIZE];
+	int error;
 
-	for (retry_cnt = 0; retry_cnt < MAX_RETRIES; retry_cnt++) {
+	while (retry_cnt--) {
 		error = elants_i2c_execute_command(client, cmd, sizeof(cmd),
-						   resp, sizeof(resp));
-		if (!error) {
-			ts->hw_version = elants_i2c_parse_version(resp);
-			if (ts->hw_version != 0xffff)
-				return 0;
-		}
-
-		dev_dbg(&client->dev, "read fw id error=%d, buf=%*phC\n",
-			error, (int)sizeof(resp), resp);
-	}
+						   resp, sizeof(resp), 1,
+						   "read fw id");
+		if (error)
+			return error;
 
-	if (error) {
-		dev_err(&client->dev,
-			"Failed to read fw id: %d\n", error);
-		return error;
+		ts->hw_version = elants_i2c_parse_version(resp);
+		if (ts->hw_version != 0xffff)
+			return 0;
 	}
 
 	dev_err(&client->dev, "Invalid fw id: %#04x\n", ts->hw_version);
@@ -336,26 +359,27 @@ static int elants_i2c_query_hw_version(struct elants_data *ts)
 static int elants_i2c_query_fw_version(struct elants_data *ts)
 {
 	struct i2c_client *client = ts->client;
-	int error, retry_cnt;
+	int retry_cnt = MAX_RETRIES;
 	const u8 cmd[] = { CMD_HEADER_READ, E_ELAN_INFO_FW_VER, 0x00, 0x01 };
 	u8 resp[HEADER_SIZE];
+	int error;
 
-	for (retry_cnt = 0; retry_cnt < MAX_RETRIES; retry_cnt++) {
+	while (retry_cnt--) {
 		error = elants_i2c_execute_command(client, cmd, sizeof(cmd),
-						   resp, sizeof(resp));
-		if (!error) {
-			ts->fw_version = elants_i2c_parse_version(resp);
-			if (ts->fw_version != 0x0000 &&
-			    ts->fw_version != 0xffff)
-				return 0;
-		}
+						   resp, sizeof(resp), 1,
+						   "read fw version");
+		if (error)
+			return error;
+
+		ts->fw_version = elants_i2c_parse_version(resp);
+		if (ts->fw_version != 0x0000 && ts->fw_version != 0xffff)
+			return 0;
 
-		dev_dbg(&client->dev, "read fw version error=%d, buf=%*phC\n",
-			error, (int)sizeof(resp), resp);
+		dev_dbg(&client->dev, "(read fw version) resp %*phC\n",
+			(int)sizeof(resp), resp);
 	}
 
-	dev_err(&client->dev,
-		"Failed to read fw version or fw version is invalid\n");
+	dev_err(&client->dev, "Invalid fw ver: %#04x\n", ts->fw_version);
 
 	return -EINVAL;
 }
@@ -363,30 +387,24 @@ static int elants_i2c_query_fw_version(struct elants_data *ts)
 static int elants_i2c_query_test_version(struct elants_data *ts)
 {
 	struct i2c_client *client = ts->client;
-	int error, retry_cnt;
+	int error;
 	u16 version;
 	const u8 cmd[] = { CMD_HEADER_READ, E_ELAN_INFO_TEST_VER, 0x00, 0x01 };
 	u8 resp[HEADER_SIZE];
 
-	for (retry_cnt = 0; retry_cnt < MAX_RETRIES; retry_cnt++) {
-		error = elants_i2c_execute_command(client, cmd, sizeof(cmd),
-						   resp, sizeof(resp));
-		if (!error) {
-			version = elants_i2c_parse_version(resp);
-			ts->test_version = version >> 8;
-			ts->solution_version = version & 0xff;
-
-			return 0;
-		}
-
-		dev_dbg(&client->dev,
-			"read test version error rc=%d, buf=%*phC\n",
-			error, (int)sizeof(resp), resp);
+	error = elants_i2c_execute_command(client, cmd, sizeof(cmd),
+					   resp, sizeof(resp), MAX_RETRIES,
+					   "read test version");
+	if (error) {
+		dev_err(&client->dev, "Failed to read test version\n");
+		return error;
 	}
 
-	dev_err(&client->dev, "Failed to read test version\n");
+	version = elants_i2c_parse_version(resp);
+	ts->test_version = version >> 8;
+	ts->solution_version = version & 0xff;
 
-	return -EINVAL;
+	return 0;
 }
 
 static int elants_i2c_query_bc_version(struct elants_data *ts)
@@ -398,13 +416,10 @@ static int elants_i2c_query_bc_version(struct elants_data *ts)
 	int error;
 
 	error = elants_i2c_execute_command(client, cmd, sizeof(cmd),
-					   resp, sizeof(resp));
-	if (error) {
-		dev_err(&client->dev,
-			"read BC version error=%d, buf=%*phC\n",
-			error, (int)sizeof(resp), resp);
+					   resp, sizeof(resp), 1,
+					   "read BC version");
+	if (error)
 		return error;
-	}
 
 	version = elants_i2c_parse_version(resp);
 	ts->bc_version = version >> 8;
@@ -436,12 +451,10 @@ static int elants_i2c_query_ts_info(struct elants_data *ts)
 	error = elants_i2c_execute_command(client,
 					   get_resolution_cmd,
 					   sizeof(get_resolution_cmd),
-					   resp, sizeof(resp));
-	if (error) {
-		dev_err(&client->dev, "get resolution command failed: %d\n",
-			error);
+					   resp, sizeof(resp), 1,
+					   "get resolution");
+	if (error)
 		return error;
-	}
 
 	rows = resp[2] + resp[6] + resp[10];
 	cols = resp[3] + resp[7] + resp[11];
@@ -449,36 +462,29 @@ static int elants_i2c_query_ts_info(struct elants_data *ts)
 	/* Process mm_to_pixel information */
 	error = elants_i2c_execute_command(client,
 					   get_osr_cmd, sizeof(get_osr_cmd),
-					   resp, sizeof(resp));
-	if (error) {
-		dev_err(&client->dev, "get osr command failed: %d\n",
-			error);
+					   resp, sizeof(resp), 1, "get osr");
+	if (error)
 		return error;
-	}
 
 	osr = resp[3];
 
 	error = elants_i2c_execute_command(client,
 					   get_physical_scan_cmd,
 					   sizeof(get_physical_scan_cmd),
-					   resp, sizeof(resp));
-	if (error) {
-		dev_err(&client->dev, "get physical scan command failed: %d\n",
-			error);
+					   resp, sizeof(resp), 1,
+					   "get physical scan");
+	if (error)
 		return error;
-	}
 
 	phy_x = get_unaligned_be16(&resp[2]);
 
 	error = elants_i2c_execute_command(client,
 					   get_physical_drive_cmd,
 					   sizeof(get_physical_drive_cmd),
-					   resp, sizeof(resp));
-	if (error) {
-		dev_err(&client->dev, "get physical drive command failed: %d\n",
-			error);
+					   resp, sizeof(resp), 1,
+					   "get physical drive");
+	if (error)
 		return error;
-	}
 
 	phy_y = get_unaligned_be16(&resp[2]);
 
@@ -633,11 +639,10 @@ static int elants_i2c_validate_remark_id(struct elants_data *ts,
 
 	/* Compare TS Remark ID and FW Remark ID */
 	error = elants_i2c_execute_command(client, cmd, sizeof(cmd),
-					resp, sizeof(resp));
-	if (error) {
-		dev_err(&client->dev, "failed to query Remark ID: %d\n", error);
+					   resp, sizeof(resp),
+					   1, "read Remark ID");
+	if (error)
 		return error;
-	}
 
 	ts_remark_id = get_unaligned_be16(&resp[3]);
 
@@ -875,8 +880,7 @@ static void elants_i2c_mt_event(struct elants_data *ts, u8 *buf)
 
 			input_mt_slot(input, i);
 			input_mt_report_slot_state(input, tool_type, true);
-			input_event(input, EV_ABS, ABS_MT_POSITION_X, x);
-			input_event(input, EV_ABS, ABS_MT_POSITION_Y, y);
+			touchscreen_report_pos(input, &ts->prop, x, y, true);
 			input_event(input, EV_ABS, ABS_MT_PRESSURE, p);
 			input_event(input, EV_ABS, ABS_MT_TOUCH_MAJOR, w);
 
@@ -1017,7 +1021,7 @@ out:
  */
 static ssize_t calibrate_store(struct device *dev,
 			       struct device_attribute *attr,
-			      const char *buf, size_t count)
+			       const char *buf, size_t count)
 {
 	struct i2c_client *client = to_i2c_client(dev);
 	struct elants_data *ts = i2c_get_clientdata(client);
@@ -1063,8 +1067,28 @@ static ssize_t show_iap_mode(struct device *dev,
 				"Normal" : "Recovery");
 }
 
+static ssize_t show_calibration_count(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	const u8 cmd[] = { CMD_HEADER_READ, E_ELAN_INFO_REK, 0x00, 0x01 };
+	u8 resp[HEADER_SIZE];
+	u16 rek_count;
+	int error;
+
+	error = elants_i2c_execute_command(client, cmd, sizeof(cmd),
+					   resp, sizeof(resp), 1,
+					   "read ReK status");
+	if (error)
+		return sprintf(buf, "%d\n", error);
+
+	rek_count = get_unaligned_be16(&resp[2]);
+	return sprintf(buf, "0x%04x\n", rek_count);
+}
+
 static DEVICE_ATTR_WO(calibrate);
 static DEVICE_ATTR(iap_mode, S_IRUGO, show_iap_mode, NULL);
+static DEVICE_ATTR(calibration_count, S_IRUGO, show_calibration_count, NULL);
 static DEVICE_ATTR(update_fw, S_IWUSR, NULL, write_update_fw);
 
 struct elants_version_attribute {
@@ -1120,6 +1144,7 @@ static struct attribute *elants_attributes[] = {
 	&dev_attr_calibrate.attr,
 	&dev_attr_update_fw.attr,
 	&dev_attr_iap_mode.attr,
+	&dev_attr_calibration_count.attr,
 
 	&elants_ver_attr_fw_version.dattr.attr,
 	&elants_ver_attr_hw_version.dattr.attr,
@@ -1290,25 +1315,7 @@ static int elants_i2c_probe(struct i2c_client *client,
 	ts->input->name = "Elan Touchscreen";
 	ts->input->id.bustype = BUS_I2C;
 
-	__set_bit(BTN_TOUCH, ts->input->keybit);
-	__set_bit(EV_ABS, ts->input->evbit);
-	__set_bit(EV_KEY, ts->input->evbit);
-
-	/* Single touch input params setup */
-	input_set_abs_params(ts->input, ABS_X, 0, ts->x_max, 0, 0);
-	input_set_abs_params(ts->input, ABS_Y, 0, ts->y_max, 0, 0);
-	input_set_abs_params(ts->input, ABS_PRESSURE, 0, 255, 0, 0);
-	input_abs_set_res(ts->input, ABS_X, ts->x_res);
-	input_abs_set_res(ts->input, ABS_Y, ts->y_res);
-
 	/* Multitouch input params setup */
-	error = input_mt_init_slots(ts->input, MAX_CONTACT_NUM,
-				    INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED);
-	if (error) {
-		dev_err(&client->dev,
-			"failed to initialize MT slots: %d\n", error);
-		return error;
-	}
 
 	input_set_abs_params(ts->input, ABS_MT_POSITION_X, 0, ts->x_max, 0, 0);
 	input_set_abs_params(ts->input, ABS_MT_POSITION_Y, 0, ts->y_max, 0, 0);
@@ -1320,6 +1327,16 @@ static int elants_i2c_probe(struct i2c_client *client,
 	input_abs_set_res(ts->input, ABS_MT_POSITION_Y, ts->y_res);
 	input_abs_set_res(ts->input, ABS_MT_TOUCH_MAJOR, 1);
 
+	touchscreen_parse_properties(ts->input, true, &ts->prop);
+
+	error = input_mt_init_slots(ts->input, MAX_CONTACT_NUM,
+				    INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED);
+	if (error) {
+		dev_err(&client->dev,
+			"failed to initialize MT slots: %d\n", error);
+		return error;
+	}
+
 	error = input_register_device(ts->input);
 	if (error) {
 		dev_err(&client->dev,
diff --git a/drivers/input/touchscreen/melfas_mip4.c b/drivers/input/touchscreen/melfas_mip4.c
index 247c3aaba2d8..f67efdd040b2 100644
--- a/drivers/input/touchscreen/melfas_mip4.c
+++ b/drivers/input/touchscreen/melfas_mip4.c
@@ -391,7 +391,7 @@ static void mip4_clear_input(struct mip4_ts *ts)
 	/* Screen */
 	for (i = 0; i < MIP4_MAX_FINGERS; i++) {
 		input_mt_slot(ts->input, i);
-		input_mt_report_slot_state(ts->input, MT_TOOL_FINGER, 0);
+		input_mt_report_slot_inactive(ts->input);
 	}
 
 	/* Keys */
@@ -534,7 +534,7 @@ static void mip4_report_touch(struct mip4_ts *ts, u8 *packet)
 	} else {
 		/* Release event */
 		input_mt_slot(ts->input, id);
-		input_mt_report_slot_state(ts->input, MT_TOOL_FINGER, 0);
+		input_mt_report_slot_inactive(ts->input);
 	}
 
 	input_mt_sync_frame(ts->input);
diff --git a/drivers/input/touchscreen/mms114.c b/drivers/input/touchscreen/mms114.c
index 2ef1adaed9af..1f96657310b7 100644
--- a/drivers/input/touchscreen/mms114.c
+++ b/drivers/input/touchscreen/mms114.c
@@ -54,6 +54,7 @@
 enum mms_type {
 	TYPE_MMS114	= 114,
 	TYPE_MMS152	= 152,
+	TYPE_MMS345L	= 345,
 };
 
 struct mms114_data {
@@ -250,6 +251,15 @@ static int mms114_get_version(struct mms114_data *data)
 	int error;
 
 	switch (data->type) {
+	case TYPE_MMS345L:
+		error = __mms114_read_reg(data, MMS152_FW_REV, 3, buf);
+		if (error)
+			return error;
+
+		dev_info(dev, "TSP FW Rev: bootloader 0x%x / core 0x%x / config 0x%x\n",
+			 buf[0], buf[1], buf[2]);
+		break;
+
 	case TYPE_MMS152:
 		error = __mms114_read_reg(data, MMS152_FW_REV, 3, buf);
 		if (error)
@@ -287,8 +297,8 @@ static int mms114_setup_regs(struct mms114_data *data)
 	if (error < 0)
 		return error;
 
-	/* MMS152 has no configuration or power on registers */
-	if (data->type == TYPE_MMS152)
+	/* Only MMS114 has configuration and power on registers */
+	if (data->type != TYPE_MMS114)
 		return 0;
 
 	error = mms114_set_active(data, true);
@@ -547,7 +557,7 @@ static int __maybe_unused mms114_suspend(struct device *dev)
 	/* Release all touch */
 	for (id = 0; id < MMS114_MAX_TOUCH; id++) {
 		input_mt_slot(input_dev, id);
-		input_mt_report_slot_state(input_dev, MT_TOOL_FINGER, false);
+		input_mt_report_slot_inactive(input_dev);
 	}
 
 	input_mt_report_pointer_emulation(input_dev, true);
@@ -597,6 +607,9 @@ static const struct of_device_id mms114_dt_match[] = {
 	}, {
 		.compatible = "melfas,mms152",
 		.data = (void *)TYPE_MMS152,
+	}, {
+		.compatible = "melfas,mms345l",
+		.data = (void *)TYPE_MMS345L,
 	},
 	{ }
 };
diff --git a/drivers/input/touchscreen/raspberrypi-ts.c b/drivers/input/touchscreen/raspberrypi-ts.c
index 0e2e08f3f433..ef6aaed217cf 100644
--- a/drivers/input/touchscreen/raspberrypi-ts.c
+++ b/drivers/input/touchscreen/raspberrypi-ts.c
@@ -100,7 +100,7 @@ static void rpi_ts_poll(struct input_dev *input)
 	released_ids = ts->known_ids & ~modified_ids;
 	for_each_set_bit(i, &released_ids, RPI_TS_MAX_SUPPORTED_POINTS) {
 		input_mt_slot(input, i);
-		input_mt_report_slot_state(input, MT_TOOL_FINGER, 0);
+		input_mt_report_slot_inactive(input);
 		modified_ids &= ~(BIT(i));
 	}
 	ts->known_ids = modified_ids;
diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c
index b6f95f20f924..b54cc64e4ea6 100644
--- a/drivers/input/touchscreen/stmfts.c
+++ b/drivers/input/touchscreen/stmfts.c
@@ -198,7 +198,7 @@ static void stmfts_report_contact_release(struct stmfts_data *sdata,
 	u8 slot_id = (event[0] & STMFTS_MASK_TOUCH_ID) >> 4;
 
 	input_mt_slot(sdata->input, slot_id);
-	input_mt_report_slot_state(sdata->input, MT_TOOL_FINGER, false);
+	input_mt_report_slot_inactive(sdata->input);
 
 	input_sync(sdata->input);
 }
diff --git a/drivers/interconnect/Kconfig b/drivers/interconnect/Kconfig
index bfa4ca3ab7a9..5b7204ee2eb2 100644
--- a/drivers/interconnect/Kconfig
+++ b/drivers/interconnect/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig INTERCONNECT
-	tristate "On-Chip Interconnect management support"
+	bool "On-Chip Interconnect management support"
 	help
 	  Support for management of the on-chip interconnects.
 
@@ -11,6 +11,7 @@ menuconfig INTERCONNECT
 
 if INTERCONNECT
 
+source "drivers/interconnect/imx/Kconfig"
 source "drivers/interconnect/qcom/Kconfig"
 
 endif
diff --git a/drivers/interconnect/Makefile b/drivers/interconnect/Makefile
index 725029ae7a2c..4825c287ca13 100644
--- a/drivers/interconnect/Makefile
+++ b/drivers/interconnect/Makefile
@@ -4,4 +4,5 @@ CFLAGS_core.o				:= -I$(src)
 icc-core-objs				:= core.o
 
 obj-$(CONFIG_INTERCONNECT)		+= icc-core.o
+obj-$(CONFIG_INTERCONNECT_IMX)		+= imx/
 obj-$(CONFIG_INTERCONNECT_QCOM)		+= qcom/
diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c
index 2c6515e3ecf1..e5f998744501 100644
--- a/drivers/interconnect/core.c
+++ b/drivers/interconnect/core.c
@@ -158,6 +158,7 @@ static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
 		hlist_add_head(&path->reqs[i].req_node, &node->req_list);
 		path->reqs[i].node = node;
 		path->reqs[i].dev = dev;
+		path->reqs[i].enabled = true;
 		/* reference to previous node was saved during path traversal */
 		node = node->reverse;
 	}
@@ -249,9 +250,12 @@ static int aggregate_requests(struct icc_node *node)
 	if (p->pre_aggregate)
 		p->pre_aggregate(node);
 
-	hlist_for_each_entry(r, &node->req_list, req_node)
+	hlist_for_each_entry(r, &node->req_list, req_node) {
+		if (!r->enabled)
+			continue;
 		p->aggregate(node, r->tag, r->avg_bw, r->peak_bw,
 			     &node->avg_bw, &node->peak_bw);
+	}
 
 	return 0;
 }
@@ -350,10 +354,35 @@ static struct icc_node *of_icc_get_from_provider(struct of_phandle_args *spec)
 	return node;
 }
 
+static void devm_icc_release(struct device *dev, void *res)
+{
+	icc_put(*(struct icc_path **)res);
+}
+
+struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
+{
+	struct icc_path **ptr, *path;
+
+	ptr = devres_alloc(devm_icc_release, sizeof(**ptr), GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	path = of_icc_get(dev, name);
+	if (!IS_ERR(path)) {
+		*ptr = path;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
+	}
+
+	return path;
+}
+EXPORT_SYMBOL_GPL(devm_of_icc_get);
+
 /**
- * of_icc_get() - get a path handle from a DT node based on name
+ * of_icc_get_by_index() - get a path handle from a DT node based on index
  * @dev: device pointer for the consumer device
- * @name: interconnect path name
+ * @idx: interconnect path index
  *
  * This function will search for a path between two endpoints and return an
  * icc_path handle on success. Use icc_put() to release constraints when they
@@ -365,13 +394,12 @@ static struct icc_node *of_icc_get_from_provider(struct of_phandle_args *spec)
  * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
  * when the API is disabled or the "interconnects" DT property is missing.
  */
-struct icc_path *of_icc_get(struct device *dev, const char *name)
+struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
 {
-	struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
+	struct icc_path *path;
 	struct icc_node *src_node, *dst_node;
-	struct device_node *np = NULL;
+	struct device_node *np;
 	struct of_phandle_args src_args, dst_args;
-	int idx = 0;
 	int ret;
 
 	if (!dev || !dev->of_node)
@@ -391,12 +419,6 @@ struct icc_path *of_icc_get(struct device *dev, const char *name)
 	 * lets support only global ids and extend this in the future if needed
 	 * without breaking DT compatibility.
 	 */
-	if (name) {
-		idx = of_property_match_string(np, "interconnect-names", name);
-		if (idx < 0)
-			return ERR_PTR(idx);
-	}
-
 	ret = of_parse_phandle_with_args(np, "interconnects",
 					 "#interconnect-cells", idx * 2,
 					 &src_args);
@@ -439,12 +461,8 @@ struct icc_path *of_icc_get(struct device *dev, const char *name)
 		return path;
 	}
 
-	if (name)
-		path->name = kstrdup_const(name, GFP_KERNEL);
-	else
-		path->name = kasprintf(GFP_KERNEL, "%s-%s",
-				       src_node->name, dst_node->name);
-
+	path->name = kasprintf(GFP_KERNEL, "%s-%s",
+			       src_node->name, dst_node->name);
 	if (!path->name) {
 		kfree(path);
 		return ERR_PTR(-ENOMEM);
@@ -452,6 +470,53 @@ struct icc_path *of_icc_get(struct device *dev, const char *name)
 
 	return path;
 }
+EXPORT_SYMBOL_GPL(of_icc_get_by_index);
+
+/**
+ * of_icc_get() - get a path handle from a DT node based on name
+ * @dev: device pointer for the consumer device
+ * @name: interconnect path name
+ *
+ * This function will search for a path between two endpoints and return an
+ * icc_path handle on success. Use icc_put() to release constraints when they
+ * are not needed anymore.
+ * If the interconnect API is disabled, NULL is returned and the consumer
+ * drivers will still build. Drivers are free to handle this specifically,
+ * but they don't have to.
+ *
+ * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
+ * when the API is disabled or the "interconnects" DT property is missing.
+ */
+struct icc_path *of_icc_get(struct device *dev, const char *name)
+{
+	struct device_node *np;
+	int idx = 0;
+
+	if (!dev || !dev->of_node)
+		return ERR_PTR(-ENODEV);
+
+	np = dev->of_node;
+
+	/*
+	 * When the consumer DT node do not have "interconnects" property
+	 * return a NULL path to skip setting constraints.
+	 */
+	if (!of_find_property(np, "interconnects", NULL))
+		return NULL;
+
+	/*
+	 * We use a combination of phandle and specifier for endpoint. For now
+	 * lets support only global ids and extend this in the future if needed
+	 * without breaking DT compatibility.
+	 */
+	if (name) {
+		idx = of_property_match_string(np, "interconnect-names", name);
+		if (idx < 0)
+			return ERR_PTR(idx);
+	}
+
+	return of_icc_get_by_index(dev, idx);
+}
 EXPORT_SYMBOL_GPL(of_icc_get);
 
 /**
@@ -479,6 +544,24 @@ void icc_set_tag(struct icc_path *path, u32 tag)
 EXPORT_SYMBOL_GPL(icc_set_tag);
 
 /**
+ * icc_get_name() - Get name of the icc path
+ * @path: reference to the path returned by icc_get()
+ *
+ * This function is used by an interconnect consumer to get the name of the icc
+ * path.
+ *
+ * Returns a valid pointer on success, or NULL otherwise.
+ */
+const char *icc_get_name(struct icc_path *path)
+{
+	if (!path)
+		return NULL;
+
+	return path->name;
+}
+EXPORT_SYMBOL_GPL(icc_get_name);
+
+/**
  * icc_set_bw() - set bandwidth constraints on an interconnect path
  * @path: reference to the path returned by icc_get()
  * @avg_bw: average bandwidth in kilobytes per second
@@ -546,6 +629,39 @@ int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
 }
 EXPORT_SYMBOL_GPL(icc_set_bw);
 
+static int __icc_enable(struct icc_path *path, bool enable)
+{
+	int i;
+
+	if (!path)
+		return 0;
+
+	if (WARN_ON(IS_ERR(path) || !path->num_nodes))
+		return -EINVAL;
+
+	mutex_lock(&icc_lock);
+
+	for (i = 0; i < path->num_nodes; i++)
+		path->reqs[i].enabled = enable;
+
+	mutex_unlock(&icc_lock);
+
+	return icc_set_bw(path, path->reqs[0].avg_bw,
+			  path->reqs[0].peak_bw);
+}
+
+int icc_enable(struct icc_path *path)
+{
+	return __icc_enable(path, true);
+}
+EXPORT_SYMBOL_GPL(icc_enable);
+
+int icc_disable(struct icc_path *path)
+{
+	return __icc_enable(path, false);
+}
+EXPORT_SYMBOL_GPL(icc_disable);
+
 /**
  * icc_get() - return a handle for path between two endpoints
  * @dev: the device requesting the path
@@ -908,12 +1024,7 @@ static int __init icc_init(void)
 	return 0;
 }
 
-static void __exit icc_exit(void)
-{
-	debugfs_remove_recursive(icc_debugfs_dir);
-}
-module_init(icc_init);
-module_exit(icc_exit);
+device_initcall(icc_init);
 
 MODULE_AUTHOR("Georgi Djakov <georgi.djakov@linaro.org>");
 MODULE_DESCRIPTION("Interconnect Driver Core");
diff --git a/drivers/interconnect/imx/Kconfig b/drivers/interconnect/imx/Kconfig
new file mode 100644
index 000000000000..be2928362bb7
--- /dev/null
+++ b/drivers/interconnect/imx/Kconfig
@@ -0,0 +1,17 @@
+config INTERCONNECT_IMX
+	tristate "i.MX interconnect drivers"
+	depends on ARCH_MXC || COMPILE_TEST
+	help
+	  Generic interconnect drivers for i.MX SOCs
+
+config INTERCONNECT_IMX8MM
+	tristate "i.MX8MM interconnect driver"
+	depends on INTERCONNECT_IMX
+
+config INTERCONNECT_IMX8MN
+	tristate "i.MX8MN interconnect driver"
+	depends on INTERCONNECT_IMX
+
+config INTERCONNECT_IMX8MQ
+	tristate "i.MX8MQ interconnect driver"
+	depends on INTERCONNECT_IMX
diff --git a/drivers/interconnect/imx/Makefile b/drivers/interconnect/imx/Makefile
new file mode 100644
index 000000000000..21fd5233754f
--- /dev/null
+++ b/drivers/interconnect/imx/Makefile
@@ -0,0 +1,9 @@
+imx-interconnect-objs			:= imx.o
+imx8mm-interconnect-objs       		:= imx8mm.o
+imx8mq-interconnect-objs       		:= imx8mq.o
+imx8mn-interconnect-objs       		:= imx8mn.o
+
+obj-$(CONFIG_INTERCONNECT_IMX)		+= imx-interconnect.o
+obj-$(CONFIG_INTERCONNECT_IMX8MM)	+= imx8mm-interconnect.o
+obj-$(CONFIG_INTERCONNECT_IMX8MQ)	+= imx8mq-interconnect.o
+obj-$(CONFIG_INTERCONNECT_IMX8MN)	+= imx8mn-interconnect.o
diff --git a/drivers/interconnect/imx/imx.c b/drivers/interconnect/imx/imx.c
new file mode 100644
index 000000000000..ac420f86008e
--- /dev/null
+++ b/drivers/interconnect/imx/imx.c
@@ -0,0 +1,284 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX SoC
+ *
+ * Copyright (c) 2019, BayLibre
+ * Copyright (c) 2019-2020, NXP
+ * Author: Alexandre Bailon <abailon@baylibre.com>
+ * Author: Leonard Crestez <leonard.crestez@nxp.com>
+ */
+
+#include <linux/device.h>
+#include <linux/interconnect-provider.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_qos.h>
+
+#include "imx.h"
+
+/* private icc_node data */
+struct imx_icc_node {
+	const struct imx_icc_node_desc *desc;
+	struct device *qos_dev;
+	struct dev_pm_qos_request qos_req;
+};
+
+static int imx_icc_node_set(struct icc_node *node)
+{
+	struct device *dev = node->provider->dev;
+	struct imx_icc_node *node_data = node->data;
+	u64 freq;
+
+	if (!node_data->qos_dev)
+		return 0;
+
+	freq = (node->avg_bw + node->peak_bw) * node_data->desc->adj->bw_mul;
+	do_div(freq, node_data->desc->adj->bw_div);
+	dev_dbg(dev, "node %s device %s avg_bw %ukBps peak_bw %ukBps min_freq %llukHz\n",
+		node->name, dev_name(node_data->qos_dev),
+		node->avg_bw, node->peak_bw, freq);
+
+	if (freq > S32_MAX) {
+		dev_err(dev, "%s can't request more than S32_MAX freq\n",
+				node->name);
+		return -ERANGE;
+	}
+
+	dev_pm_qos_update_request(&node_data->qos_req, freq);
+
+	return 0;
+}
+
+static int imx_icc_set(struct icc_node *src, struct icc_node *dst)
+{
+	return imx_icc_node_set(dst);
+}
+
+/* imx_icc_node_destroy() - Destroy an imx icc_node, including private data */
+static void imx_icc_node_destroy(struct icc_node *node)
+{
+	struct imx_icc_node *node_data = node->data;
+	int ret;
+
+	if (dev_pm_qos_request_active(&node_data->qos_req)) {
+		ret = dev_pm_qos_remove_request(&node_data->qos_req);
+		if (ret)
+			dev_warn(node->provider->dev,
+				 "failed to remove qos request for %s\n",
+				 dev_name(node_data->qos_dev));
+	}
+
+	put_device(node_data->qos_dev);
+	icc_node_del(node);
+	icc_node_destroy(node->id);
+}
+
+static int imx_icc_node_init_qos(struct icc_provider *provider,
+				 struct icc_node *node)
+{
+	struct imx_icc_node *node_data = node->data;
+	const struct imx_icc_node_adj_desc *adj = node_data->desc->adj;
+	struct device *dev = provider->dev;
+	struct device_node *dn = NULL;
+	struct platform_device *pdev;
+
+	if (adj->main_noc) {
+		node_data->qos_dev = dev;
+		dev_dbg(dev, "icc node %s[%d] is main noc itself\n",
+			node->name, node->id);
+	} else {
+		dn = of_parse_phandle(dev->of_node, adj->phandle_name, 0);
+		if (!dn) {
+			dev_warn(dev, "Failed to parse %s\n",
+				 adj->phandle_name);
+			return -ENODEV;
+		}
+		/* Allow scaling to be disabled on a per-node basis */
+		if (!dn || !of_device_is_available(dn)) {
+			dev_warn(dev, "Missing property %s, skip scaling %s\n",
+				 adj->phandle_name, node->name);
+			return 0;
+		}
+
+		pdev = of_find_device_by_node(dn);
+		of_node_put(dn);
+		if (!pdev) {
+			dev_warn(dev, "node %s[%d] missing device for %pOF\n",
+				 node->name, node->id, dn);
+			return -EPROBE_DEFER;
+		}
+		node_data->qos_dev = &pdev->dev;
+		dev_dbg(dev, "node %s[%d] has device node %pOF\n",
+			node->name, node->id, dn);
+	}
+
+	return dev_pm_qos_add_request(node_data->qos_dev,
+				      &node_data->qos_req,
+				      DEV_PM_QOS_MIN_FREQUENCY, 0);
+}
+
+static struct icc_node *imx_icc_node_add(struct icc_provider *provider,
+					 const struct imx_icc_node_desc *node_desc)
+{
+	struct device *dev = provider->dev;
+	struct imx_icc_node *node_data;
+	struct icc_node *node;
+	int ret;
+
+	node = icc_node_create(node_desc->id);
+	if (IS_ERR(node)) {
+		dev_err(dev, "failed to create node %d\n", node_desc->id);
+		return node;
+	}
+
+	if (node->data) {
+		dev_err(dev, "already created node %s id=%d\n",
+			node_desc->name, node_desc->id);
+		return ERR_PTR(-EEXIST);
+	}
+
+	node_data = devm_kzalloc(dev, sizeof(*node_data), GFP_KERNEL);
+	if (!node_data) {
+		icc_node_destroy(node->id);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	node->name = node_desc->name;
+	node->data = node_data;
+	node_data->desc = node_desc;
+	icc_node_add(node, provider);
+
+	if (node_desc->adj) {
+		ret = imx_icc_node_init_qos(provider, node);
+		if (ret < 0) {
+			imx_icc_node_destroy(node);
+			return ERR_PTR(ret);
+		}
+	}
+
+	return node;
+}
+
+static void imx_icc_unregister_nodes(struct icc_provider *provider)
+{
+	struct icc_node *node, *tmp;
+
+	list_for_each_entry_safe(node, tmp, &provider->nodes, node_list)
+		imx_icc_node_destroy(node);
+}
+
+static int imx_icc_register_nodes(struct icc_provider *provider,
+				  const struct imx_icc_node_desc *descs,
+				  int count)
+{
+	struct icc_onecell_data *provider_data = provider->data;
+	int ret;
+	int i;
+
+	for (i = 0; i < count; i++) {
+		struct icc_node *node;
+		const struct imx_icc_node_desc *node_desc = &descs[i];
+		size_t j;
+
+		node = imx_icc_node_add(provider, node_desc);
+		if (IS_ERR(node)) {
+			ret = PTR_ERR(node);
+			if (ret != -EPROBE_DEFER)
+				dev_err(provider->dev, "failed to add %s: %d\n",
+					node_desc->name, ret);
+			goto err;
+		}
+		provider_data->nodes[node->id] = node;
+
+		for (j = 0; j < node_desc->num_links; j++) {
+			ret = icc_link_create(node, node_desc->links[j]);
+			if (ret) {
+				dev_err(provider->dev, "failed to link node %d to %d: %d\n",
+					node->id, node_desc->links[j], ret);
+				goto err;
+			}
+		}
+	}
+
+	return 0;
+
+err:
+	imx_icc_unregister_nodes(provider);
+
+	return ret;
+}
+
+static int get_max_node_id(struct imx_icc_node_desc *nodes, int nodes_count)
+{
+	int i, ret = 0;
+
+	for (i = 0; i < nodes_count; ++i)
+		if (nodes[i].id > ret)
+			ret = nodes[i].id;
+
+	return ret;
+}
+
+int imx_icc_register(struct platform_device *pdev,
+		     struct imx_icc_node_desc *nodes, int nodes_count)
+{
+	struct device *dev = &pdev->dev;
+	struct icc_onecell_data *data;
+	struct icc_provider *provider;
+	int max_node_id;
+	int ret;
+
+	/* icc_onecell_data is indexed by node_id, unlike nodes param */
+	max_node_id = get_max_node_id(nodes, nodes_count);
+	data = devm_kzalloc(dev, struct_size(data, nodes, max_node_id),
+			    GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+	data->num_nodes = max_node_id;
+
+	provider = devm_kzalloc(dev, sizeof(*provider), GFP_KERNEL);
+	if (!provider)
+		return -ENOMEM;
+	provider->set = imx_icc_set;
+	provider->aggregate = icc_std_aggregate;
+	provider->xlate = of_icc_xlate_onecell;
+	provider->data = data;
+	provider->dev = dev->parent;
+	platform_set_drvdata(pdev, provider);
+
+	ret = icc_provider_add(provider);
+	if (ret) {
+		dev_err(dev, "error adding interconnect provider: %d\n", ret);
+		return ret;
+	}
+
+	ret = imx_icc_register_nodes(provider, nodes, nodes_count);
+	if (ret)
+		goto provider_del;
+
+	return 0;
+
+provider_del:
+	icc_provider_del(provider);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(imx_icc_register);
+
+int imx_icc_unregister(struct platform_device *pdev)
+{
+	struct icc_provider *provider = platform_get_drvdata(pdev);
+	int ret;
+
+	imx_icc_unregister_nodes(provider);
+
+	ret = icc_provider_del(provider);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(imx_icc_unregister);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/imx/imx.h b/drivers/interconnect/imx/imx.h
new file mode 100644
index 000000000000..75da51076c68
--- /dev/null
+++ b/drivers/interconnect/imx/imx.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Interconnect framework driver for i.MX SoC
+ *
+ * Copyright (c) 2019, BayLibre
+ * Copyright (c) 2019-2020, NXP
+ * Author: Alexandre Bailon <abailon@baylibre.com>
+ * Author: Leonard Crestez <leonard.crestez@nxp.com>
+ */
+#ifndef __DRIVERS_INTERCONNECT_IMX_H
+#define __DRIVERS_INTERCONNECT_IMX_H
+
+#include <linux/kernel.h>
+
+#define IMX_ICC_MAX_LINKS	4
+
+/*
+ * struct imx_icc_node_adj - Describe a dynamic adjustable node
+ */
+struct imx_icc_node_adj_desc {
+	unsigned int bw_mul, bw_div;
+	const char *phandle_name;
+	bool main_noc;
+};
+
+/*
+ * struct imx_icc_node - Describe an interconnect node
+ * @name: name of the node
+ * @id: an unique id to identify the node
+ * @links: an array of slaves' node id
+ * @num_links: number of id defined in links
+ */
+struct imx_icc_node_desc {
+	const char *name;
+	u16 id;
+	u16 links[IMX_ICC_MAX_LINKS];
+	u16 num_links;
+	const struct imx_icc_node_adj_desc *adj;
+};
+
+#define DEFINE_BUS_INTERCONNECT(_name, _id, _adj, ...)			\
+	{								\
+		.id = _id,						\
+		.name = _name,						\
+		.adj = _adj,						\
+		.num_links = ARRAY_SIZE(((int[]){ __VA_ARGS__ })),	\
+		.links = { __VA_ARGS__ },				\
+	}
+
+#define DEFINE_BUS_MASTER(_name, _id, _dest_id)				\
+	DEFINE_BUS_INTERCONNECT(_name, _id, NULL, _dest_id)
+
+#define DEFINE_BUS_SLAVE(_name, _id, _adj)				\
+	DEFINE_BUS_INTERCONNECT(_name, _id, _adj)
+
+int imx_icc_register(struct platform_device *pdev,
+		     struct imx_icc_node_desc *nodes,
+		     int nodes_count);
+int imx_icc_unregister(struct platform_device *pdev);
+
+#endif /* __DRIVERS_INTERCONNECT_IMX_H */
diff --git a/drivers/interconnect/imx/imx8mm.c b/drivers/interconnect/imx/imx8mm.c
new file mode 100644
index 000000000000..1083490bb391
--- /dev/null
+++ b/drivers/interconnect/imx/imx8mm.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX8MM SoC
+ *
+ * Copyright (c) 2019, BayLibre
+ * Copyright (c) 2019-2020, NXP
+ * Author: Alexandre Bailon <abailon@baylibre.com>
+ * Author: Leonard Crestez <leonard.crestez@nxp.com>
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/interconnect/imx8mm.h>
+
+#include "imx.h"
+
+static const struct imx_icc_node_adj_desc imx8mm_dram_adj = {
+	.bw_mul = 1,
+	.bw_div = 16,
+	.phandle_name = "fsl,ddrc",
+};
+
+static const struct imx_icc_node_adj_desc imx8mm_noc_adj = {
+	.bw_mul = 1,
+	.bw_div = 16,
+	.main_noc = true,
+};
+
+/*
+ * Describe bus masters, slaves and connections between them
+ *
+ * This is a simplified subset of the bus diagram, there are several other
+ * PL301 nics which are skipped/merged into PL301_MAIN
+ */
+static struct imx_icc_node_desc nodes[] = {
+	DEFINE_BUS_INTERCONNECT("NOC", IMX8MM_ICN_NOC, &imx8mm_noc_adj,
+			IMX8MM_ICS_DRAM, IMX8MM_ICN_MAIN),
+
+	DEFINE_BUS_SLAVE("DRAM", IMX8MM_ICS_DRAM, &imx8mm_dram_adj),
+	DEFINE_BUS_SLAVE("OCRAM", IMX8MM_ICS_OCRAM, NULL),
+	DEFINE_BUS_MASTER("A53", IMX8MM_ICM_A53, IMX8MM_ICN_NOC),
+
+	/* VPUMIX */
+	DEFINE_BUS_MASTER("VPU H1", IMX8MM_ICM_VPU_H1, IMX8MM_ICN_VIDEO),
+	DEFINE_BUS_MASTER("VPU G1", IMX8MM_ICM_VPU_G1, IMX8MM_ICN_VIDEO),
+	DEFINE_BUS_MASTER("VPU G2", IMX8MM_ICM_VPU_G2, IMX8MM_ICN_VIDEO),
+	DEFINE_BUS_INTERCONNECT("PL301_VIDEO", IMX8MM_ICN_VIDEO, NULL, IMX8MM_ICN_NOC),
+
+	/* GPUMIX */
+	DEFINE_BUS_MASTER("GPU 2D", IMX8MM_ICM_GPU2D, IMX8MM_ICN_GPU),
+	DEFINE_BUS_MASTER("GPU 3D", IMX8MM_ICM_GPU3D, IMX8MM_ICN_GPU),
+	DEFINE_BUS_INTERCONNECT("PL301_GPU", IMX8MM_ICN_GPU, NULL, IMX8MM_ICN_NOC),
+
+	/* DISPLAYMIX */
+	DEFINE_BUS_MASTER("CSI", IMX8MM_ICM_CSI, IMX8MM_ICN_MIPI),
+	DEFINE_BUS_MASTER("LCDIF", IMX8MM_ICM_LCDIF, IMX8MM_ICN_MIPI),
+	DEFINE_BUS_INTERCONNECT("PL301_MIPI", IMX8MM_ICN_MIPI, NULL, IMX8MM_ICN_NOC),
+
+	/* HSIO */
+	DEFINE_BUS_MASTER("USB1", IMX8MM_ICM_USB1, IMX8MM_ICN_HSIO),
+	DEFINE_BUS_MASTER("USB2", IMX8MM_ICM_USB2, IMX8MM_ICN_HSIO),
+	DEFINE_BUS_MASTER("PCIE", IMX8MM_ICM_PCIE, IMX8MM_ICN_HSIO),
+	DEFINE_BUS_INTERCONNECT("PL301_HSIO", IMX8MM_ICN_HSIO, NULL, IMX8MM_ICN_NOC),
+
+	/* Audio */
+	DEFINE_BUS_MASTER("SDMA2", IMX8MM_ICM_SDMA2, IMX8MM_ICN_AUDIO),
+	DEFINE_BUS_MASTER("SDMA3", IMX8MM_ICM_SDMA3, IMX8MM_ICN_AUDIO),
+	DEFINE_BUS_INTERCONNECT("PL301_AUDIO", IMX8MM_ICN_AUDIO, NULL, IMX8MM_ICN_MAIN),
+
+	/* Ethernet */
+	DEFINE_BUS_MASTER("ENET", IMX8MM_ICM_ENET, IMX8MM_ICN_ENET),
+	DEFINE_BUS_INTERCONNECT("PL301_ENET", IMX8MM_ICN_ENET, NULL, IMX8MM_ICN_MAIN),
+
+	/* Other */
+	DEFINE_BUS_MASTER("SDMA1", IMX8MM_ICM_SDMA1, IMX8MM_ICN_MAIN),
+	DEFINE_BUS_MASTER("NAND", IMX8MM_ICM_NAND, IMX8MM_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC1", IMX8MM_ICM_USDHC1, IMX8MM_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC2", IMX8MM_ICM_USDHC2, IMX8MM_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC3", IMX8MM_ICM_USDHC3, IMX8MM_ICN_MAIN),
+	DEFINE_BUS_INTERCONNECT("PL301_MAIN", IMX8MM_ICN_MAIN, NULL,
+			IMX8MM_ICN_NOC, IMX8MM_ICS_OCRAM),
+};
+
+static int imx8mm_icc_probe(struct platform_device *pdev)
+{
+	return imx_icc_register(pdev, nodes, ARRAY_SIZE(nodes));
+}
+
+static int imx8mm_icc_remove(struct platform_device *pdev)
+{
+	return imx_icc_unregister(pdev);
+}
+
+static struct platform_driver imx8mm_icc_driver = {
+	.probe = imx8mm_icc_probe,
+	.remove = imx8mm_icc_remove,
+	.driver = {
+		.name = "imx8mm-interconnect",
+	},
+};
+
+module_platform_driver(imx8mm_icc_driver);
+MODULE_AUTHOR("Alexandre Bailon <abailon@baylibre.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:imx8mm-interconnect");
diff --git a/drivers/interconnect/imx/imx8mn.c b/drivers/interconnect/imx/imx8mn.c
new file mode 100644
index 000000000000..ad97e55fd4e5
--- /dev/null
+++ b/drivers/interconnect/imx/imx8mn.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX8MN SoC
+ *
+ * Copyright (c) 2019-2020, NXP
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/interconnect/imx8mn.h>
+
+#include "imx.h"
+
+static const struct imx_icc_node_adj_desc imx8mn_dram_adj = {
+	.bw_mul = 1,
+	.bw_div = 4,
+	.phandle_name = "fsl,ddrc",
+};
+
+static const struct imx_icc_node_adj_desc imx8mn_noc_adj = {
+	.bw_mul = 1,
+	.bw_div = 4,
+	.main_noc = true,
+};
+
+/*
+ * Describe bus masters, slaves and connections between them
+ *
+ * This is a simplified subset of the bus diagram, there are several other
+ * PL301 nics which are skipped/merged into PL301_MAIN
+ */
+static struct imx_icc_node_desc nodes[] = {
+	DEFINE_BUS_INTERCONNECT("NOC", IMX8MN_ICN_NOC, &imx8mn_noc_adj,
+			IMX8MN_ICS_DRAM, IMX8MN_ICN_MAIN),
+
+	DEFINE_BUS_SLAVE("DRAM", IMX8MN_ICS_DRAM, &imx8mn_dram_adj),
+	DEFINE_BUS_SLAVE("OCRAM", IMX8MN_ICS_OCRAM, NULL),
+	DEFINE_BUS_MASTER("A53", IMX8MN_ICM_A53, IMX8MN_ICN_NOC),
+
+	/* GPUMIX */
+	DEFINE_BUS_MASTER("GPU", IMX8MN_ICM_GPU, IMX8MN_ICN_GPU),
+	DEFINE_BUS_INTERCONNECT("PL301_GPU", IMX8MN_ICN_GPU, NULL, IMX8MN_ICN_NOC),
+
+	/* DISPLAYMIX */
+	DEFINE_BUS_MASTER("CSI1", IMX8MN_ICM_CSI1, IMX8MN_ICN_MIPI),
+	DEFINE_BUS_MASTER("CSI2", IMX8MN_ICM_CSI2, IMX8MN_ICN_MIPI),
+	DEFINE_BUS_MASTER("ISI", IMX8MN_ICM_ISI, IMX8MN_ICN_MIPI),
+	DEFINE_BUS_MASTER("LCDIF", IMX8MN_ICM_LCDIF, IMX8MN_ICN_MIPI),
+	DEFINE_BUS_INTERCONNECT("PL301_MIPI", IMX8MN_ICN_MIPI, NULL, IMX8MN_ICN_NOC),
+
+	/* USB goes straight to NOC */
+	DEFINE_BUS_MASTER("USB", IMX8MN_ICM_USB, IMX8MN_ICN_NOC),
+
+	/* Audio */
+	DEFINE_BUS_MASTER("SDMA2", IMX8MN_ICM_SDMA2, IMX8MN_ICN_AUDIO),
+	DEFINE_BUS_MASTER("SDMA3", IMX8MN_ICM_SDMA3, IMX8MN_ICN_AUDIO),
+	DEFINE_BUS_INTERCONNECT("PL301_AUDIO", IMX8MN_ICN_AUDIO, NULL, IMX8MN_ICN_MAIN),
+
+	/* Ethernet */
+	DEFINE_BUS_MASTER("ENET", IMX8MN_ICM_ENET, IMX8MN_ICN_ENET),
+	DEFINE_BUS_INTERCONNECT("PL301_ENET", IMX8MN_ICN_ENET, NULL, IMX8MN_ICN_MAIN),
+
+	/* Other */
+	DEFINE_BUS_MASTER("SDMA1", IMX8MN_ICM_SDMA1, IMX8MN_ICN_MAIN),
+	DEFINE_BUS_MASTER("NAND", IMX8MN_ICM_NAND, IMX8MN_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC1", IMX8MN_ICM_USDHC1, IMX8MN_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC2", IMX8MN_ICM_USDHC2, IMX8MN_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC3", IMX8MN_ICM_USDHC3, IMX8MN_ICN_MAIN),
+	DEFINE_BUS_INTERCONNECT("PL301_MAIN", IMX8MN_ICN_MAIN, NULL,
+			IMX8MN_ICN_NOC, IMX8MN_ICS_OCRAM),
+};
+
+static int imx8mn_icc_probe(struct platform_device *pdev)
+{
+	return imx_icc_register(pdev, nodes, ARRAY_SIZE(nodes));
+}
+
+static int imx8mn_icc_remove(struct platform_device *pdev)
+{
+	return imx_icc_unregister(pdev);
+}
+
+static struct platform_driver imx8mn_icc_driver = {
+	.probe = imx8mn_icc_probe,
+	.remove = imx8mn_icc_remove,
+	.driver = {
+		.name = "imx8mn-interconnect",
+	},
+};
+
+module_platform_driver(imx8mn_icc_driver);
+MODULE_ALIAS("platform:imx8mn-interconnect");
+MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/imx/imx8mq.c b/drivers/interconnect/imx/imx8mq.c
new file mode 100644
index 000000000000..ba43a15aefec
--- /dev/null
+++ b/drivers/interconnect/imx/imx8mq.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Interconnect framework driver for i.MX8MQ SoC
+ *
+ * Copyright (c) 2019-2020, NXP
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/interconnect/imx8mq.h>
+
+#include "imx.h"
+
+static const struct imx_icc_node_adj_desc imx8mq_dram_adj = {
+	.bw_mul = 1,
+	.bw_div = 4,
+	.phandle_name = "fsl,ddrc",
+};
+
+static const struct imx_icc_node_adj_desc imx8mq_noc_adj = {
+	.bw_mul = 1,
+	.bw_div = 4,
+	.main_noc = true,
+};
+
+/*
+ * Describe bus masters, slaves and connections between them
+ *
+ * This is a simplified subset of the bus diagram, there are several other
+ * PL301 nics which are skipped/merged into PL301_MAIN
+ */
+static struct imx_icc_node_desc nodes[] = {
+	DEFINE_BUS_INTERCONNECT("NOC", IMX8MQ_ICN_NOC, &imx8mq_noc_adj,
+			IMX8MQ_ICS_DRAM, IMX8MQ_ICN_MAIN),
+
+	DEFINE_BUS_SLAVE("DRAM", IMX8MQ_ICS_DRAM, &imx8mq_dram_adj),
+	DEFINE_BUS_SLAVE("OCRAM", IMX8MQ_ICS_OCRAM, NULL),
+	DEFINE_BUS_MASTER("A53", IMX8MQ_ICM_A53, IMX8MQ_ICN_NOC),
+
+	/* VPUMIX */
+	DEFINE_BUS_MASTER("VPU", IMX8MQ_ICM_VPU, IMX8MQ_ICN_VIDEO),
+	DEFINE_BUS_INTERCONNECT("PL301_VIDEO", IMX8MQ_ICN_VIDEO, NULL, IMX8MQ_ICN_NOC),
+
+	/* GPUMIX */
+	DEFINE_BUS_MASTER("GPU", IMX8MQ_ICM_GPU, IMX8MQ_ICN_GPU),
+	DEFINE_BUS_INTERCONNECT("PL301_GPU", IMX8MQ_ICN_GPU, NULL, IMX8MQ_ICN_NOC),
+
+	/* DISPMIX (only for DCSS) */
+	DEFINE_BUS_MASTER("DC", IMX8MQ_ICM_DCSS, IMX8MQ_ICN_DCSS),
+	DEFINE_BUS_INTERCONNECT("PL301_DC", IMX8MQ_ICN_DCSS, NULL, IMX8MQ_ICN_NOC),
+
+	/* USBMIX */
+	DEFINE_BUS_MASTER("USB1", IMX8MQ_ICM_USB1, IMX8MQ_ICN_USB),
+	DEFINE_BUS_MASTER("USB2", IMX8MQ_ICM_USB2, IMX8MQ_ICN_USB),
+	DEFINE_BUS_INTERCONNECT("PL301_USB", IMX8MQ_ICN_USB, NULL, IMX8MQ_ICN_NOC),
+
+	/* PL301_DISPLAY (IPs other than DCSS, inside SUPERMIX) */
+	DEFINE_BUS_MASTER("CSI1", IMX8MQ_ICM_CSI1, IMX8MQ_ICN_DISPLAY),
+	DEFINE_BUS_MASTER("CSI2", IMX8MQ_ICM_CSI2, IMX8MQ_ICN_DISPLAY),
+	DEFINE_BUS_MASTER("LCDIF", IMX8MQ_ICM_LCDIF, IMX8MQ_ICN_DISPLAY),
+	DEFINE_BUS_INTERCONNECT("PL301_DISPLAY", IMX8MQ_ICN_DISPLAY, NULL, IMX8MQ_ICN_MAIN),
+
+	/* AUDIO */
+	DEFINE_BUS_MASTER("SDMA2", IMX8MQ_ICM_SDMA2, IMX8MQ_ICN_AUDIO),
+	DEFINE_BUS_INTERCONNECT("PL301_AUDIO", IMX8MQ_ICN_AUDIO, NULL, IMX8MQ_ICN_DISPLAY),
+
+	/* ENET */
+	DEFINE_BUS_MASTER("ENET", IMX8MQ_ICM_ENET, IMX8MQ_ICN_ENET),
+	DEFINE_BUS_INTERCONNECT("PL301_ENET", IMX8MQ_ICN_ENET, NULL, IMX8MQ_ICN_MAIN),
+
+	/* OTHER */
+	DEFINE_BUS_MASTER("SDMA1", IMX8MQ_ICM_SDMA1, IMX8MQ_ICN_MAIN),
+	DEFINE_BUS_MASTER("NAND", IMX8MQ_ICM_NAND, IMX8MQ_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC1", IMX8MQ_ICM_USDHC1, IMX8MQ_ICN_MAIN),
+	DEFINE_BUS_MASTER("USDHC2", IMX8MQ_ICM_USDHC2, IMX8MQ_ICN_MAIN),
+	DEFINE_BUS_MASTER("PCIE1", IMX8MQ_ICM_PCIE1, IMX8MQ_ICN_MAIN),
+	DEFINE_BUS_MASTER("PCIE2", IMX8MQ_ICM_PCIE2, IMX8MQ_ICN_MAIN),
+	DEFINE_BUS_INTERCONNECT("PL301_MAIN", IMX8MQ_ICN_MAIN, NULL,
+			IMX8MQ_ICN_NOC, IMX8MQ_ICS_OCRAM),
+};
+
+static int imx8mq_icc_probe(struct platform_device *pdev)
+{
+	return imx_icc_register(pdev, nodes, ARRAY_SIZE(nodes));
+}
+
+static int imx8mq_icc_remove(struct platform_device *pdev)
+{
+	return imx_icc_unregister(pdev);
+}
+
+static struct platform_driver imx8mq_icc_driver = {
+	.probe = imx8mq_icc_probe,
+	.remove = imx8mq_icc_remove,
+	.driver = {
+		.name = "imx8mq-interconnect",
+	},
+};
+
+module_platform_driver(imx8mq_icc_driver);
+MODULE_ALIAS("platform:imx8mq-interconnect");
+MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/interconnect/internal.h b/drivers/interconnect/internal.h
index bf18cb7239df..f5f82a5c939e 100644
--- a/drivers/interconnect/internal.h
+++ b/drivers/interconnect/internal.h
@@ -14,6 +14,7 @@
  * @req_node: entry in list of requests for the particular @node
  * @node: the interconnect node to which this constraint applies
  * @dev: reference to the device that sets the constraints
+ * @enabled: indicates whether the path with this request is enabled
  * @tag: path tag (optional)
  * @avg_bw: an integer describing the average bandwidth in kBps
  * @peak_bw: an integer describing the peak bandwidth in kBps
@@ -22,6 +23,7 @@ struct icc_req {
 	struct hlist_node req_node;
 	struct icc_node *node;
 	struct device *dev;
+	bool enabled;
 	u32 tag;
 	u32 avg_bw;
 	u32 peak_bw;
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 2ab07ce17abb..b510f67dfa49 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -15,7 +15,7 @@ menuconfig IOMMU_SUPPORT
 	bool "IOMMU Hardware Support"
 	depends on MMU
 	default y
-	---help---
+	help
 	  Say Y here if you want to compile device drivers for IO Memory
 	  Management Units into the kernel. These devices usually allow to
 	  remap DMA requests and/or remap interrupts from other devices on the
@@ -144,7 +144,7 @@ config AMD_IOMMU
 	select IOMMU_IOVA
 	select IOMMU_DMA
 	depends on X86_64 && PCI && ACPI
-	---help---
+	help
 	  With this option you can enable support for AMD IOMMU hardware in
 	  your system. An IOMMU is a hardware component which provides
 	  remapping of DMA memory accesses from devices. With an AMD IOMMU you
@@ -159,7 +159,7 @@ config AMD_IOMMU_V2
 	tristate "AMD IOMMU Version 2 driver"
 	depends on AMD_IOMMU
 	select MMU_NOTIFIER
-	---help---
+	help
 	  This option enables support for the AMD IOMMUv2 features of the IOMMU
 	  hardware. Select this option if you want to use devices that support
 	  the PCI PRI and PASID interface.
@@ -167,7 +167,7 @@ config AMD_IOMMU_V2
 config AMD_IOMMU_DEBUGFS
 	bool "Enable AMD IOMMU internals in DebugFS"
 	depends on AMD_IOMMU && IOMMU_DEBUGFS
-	---help---
+	help
 	  !!!WARNING!!!  !!!WARNING!!!  !!!WARNING!!!  !!!WARNING!!!
 
 	  DO NOT ENABLE THIS OPTION UNLESS YOU REALLY, -REALLY- KNOW WHAT YOU ARE DOING!!!
@@ -233,7 +233,7 @@ config INTEL_IOMMU_DEFAULT_ON
 config INTEL_IOMMU_BROKEN_GFX_WA
 	bool "Workaround broken graphics drivers (going away soon)"
 	depends on INTEL_IOMMU && BROKEN && X86
-	---help---
+	help
 	  Current Graphics drivers tend to use physical address
 	  for DMA and avoid using DMA APIs. Setting this config
 	  option permits the IOMMU driver to set a unity map for
@@ -244,7 +244,7 @@ config INTEL_IOMMU_BROKEN_GFX_WA
 config INTEL_IOMMU_FLOPPY_WA
 	def_bool y
 	depends on INTEL_IOMMU && X86
-	---help---
+	help
 	  Floppy disk drivers are known to bypass DMA API calls
 	  thereby failing to work when IOMMU is enabled. This
 	  workaround will setup a 1:1 mapping for the first
@@ -266,7 +266,7 @@ config IRQ_REMAP
 	bool "Support for Interrupt Remapping"
 	depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI
 	select DMAR_TABLE
-	---help---
+	help
 	  Supports Interrupt remapping for IO-APIC and MSI devices.
 	  To use x2apic mode in the CPU's which support x2APIC enhancements or
 	  to support platforms with CPU's having > 8 bit APIC ID, say Y.
@@ -277,14 +277,14 @@ config OMAP_IOMMU
 	depends on ARM && MMU || (COMPILE_TEST && (ARM || ARM64 || IA64 || SPARC))
 	depends on ARCH_OMAP2PLUS || COMPILE_TEST
 	select IOMMU_API
-	---help---
+	help
 	  The OMAP3 media platform drivers depend on iommu support,
 	  if you need them say Y here.
 
 config OMAP_IOMMU_DEBUG
 	bool "Export OMAP IOMMU internals in DebugFS"
 	depends on OMAP_IOMMU && DEBUG_FS
-	---help---
+	help
 	  Select this to see extensive information about
 	  the internal state of OMAP IOMMU in debugfs.
 
@@ -303,6 +303,15 @@ config ROCKCHIP_IOMMU
 	  Say Y here if you are using a Rockchip SoC that includes an IOMMU
 	  device.
 
+config SUN50I_IOMMU
+	bool "Allwinner H6 IOMMU Support"
+	depends on ARCH_SUNXI || COMPILE_TEST
+	select ARM_DMA_USE_IOMMU
+	select IOMMU_API
+	select IOMMU_DMA
+	help
+	  Support for the IOMMU introduced in the Allwinner H6 SoCs.
+
 config TEGRA_IOMMU_GART
 	bool "Tegra GART IOMMU Support"
 	depends on ARCH_TEGRA_2x_SOC
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 9f33fdb3bb05..342190196dfb 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -11,24 +11,25 @@ obj-$(CONFIG_IOASID) += ioasid.o
 obj-$(CONFIG_IOMMU_IOVA) += iova.o
 obj-$(CONFIG_OF_IOMMU)	+= of_iommu.o
 obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o
-obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o amd_iommu_quirks.o
-obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o
-obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
+obj-$(CONFIG_AMD_IOMMU) += amd/iommu.o amd/init.o amd/quirks.o
+obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd/debugfs.o
+obj-$(CONFIG_AMD_IOMMU_V2) += amd/iommu_v2.o
 obj-$(CONFIG_ARM_SMMU) += arm_smmu.o
 arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o
 obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
-obj-$(CONFIG_DMAR_TABLE) += dmar.o
-obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o
-obj-$(CONFIG_INTEL_IOMMU) += intel-trace.o
-obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += intel-iommu-debugfs.o
-obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o
+obj-$(CONFIG_DMAR_TABLE) += intel/dmar.o
+obj-$(CONFIG_INTEL_IOMMU) += intel/iommu.o intel/pasid.o
+obj-$(CONFIG_INTEL_IOMMU) += intel/trace.o
+obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += intel/debugfs.o
+obj-$(CONFIG_INTEL_IOMMU_SVM) += intel/svm.o
 obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o
-obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o
+obj-$(CONFIG_IRQ_REMAP) += intel/irq_remapping.o irq_remapping.o
 obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o
 obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o
 obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
 obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
 obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o
+obj-$(CONFIG_SUN50I_IOMMU) += sun50i-iommu.o
 obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o
 obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
 obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd/amd_iommu.h
index 92c2ba6468a0..f892992c8744 100644
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -4,8 +4,10 @@
  * Author: Joerg Roedel <jroedel@suse.de>
  */
 
-#ifndef _ASM_X86_AMD_IOMMU_PROTO_H
-#define _ASM_X86_AMD_IOMMU_PROTO_H
+#ifndef AMD_IOMMU_H
+#define AMD_IOMMU_H
+
+#include <linux/iommu.h>
 
 #include "amd_iommu_types.h"
 
@@ -92,5 +94,15 @@ static inline void *iommu_phys_to_virt(unsigned long paddr)
 }
 
 extern bool translation_pre_enabled(struct amd_iommu *iommu);
-extern struct iommu_dev_data *get_dev_data(struct device *dev);
-#endif /* _ASM_X86_AMD_IOMMU_PROTO_H  */
+extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
+					 struct device *dev);
+extern int __init add_special_device(u8 type, u8 id, u16 *devid,
+				     bool cmd_line);
+
+#ifdef CONFIG_DMI
+void amd_iommu_apply_ivrs_quirks(void);
+#else
+static void amd_iommu_apply_ivrs_quirks(void) { }
+#endif
+
+#endif
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 7a8fdec138bd..30a5d412255a 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -395,10 +395,10 @@
 #define PD_IOMMUV2_MASK		(1UL << 3) /* domain has gcr3 table */
 
 extern bool amd_iommu_dump;
-#define DUMP_printk(format, arg...)					\
-	do {								\
-		if (amd_iommu_dump)						\
-			printk(KERN_INFO "AMD-Vi: " format, ## arg);	\
+#define DUMP_printk(format, arg...)				\
+	do {							\
+		if (amd_iommu_dump)				\
+			pr_info("AMD-Vi: " format, ## arg);	\
 	} while(0);
 
 /* global flag if IOMMUs cache non-present entries */
@@ -645,7 +645,6 @@ struct iommu_dev_data {
 	struct pci_dev *pdev;
 	u16 devid;			  /* PCI Device ID */
 	bool iommu_v2;			  /* Device can make use of IOMMUv2 */
-	bool passthrough;		  /* Device is identity mapped */
 	struct {
 		bool enabled;
 		int qdep;
diff --git a/drivers/iommu/amd_iommu_debugfs.c b/drivers/iommu/amd/debugfs.c
index c6a5c737ef09..545372fcc72f 100644
--- a/drivers/iommu/amd_iommu_debugfs.c
+++ b/drivers/iommu/amd/debugfs.c
@@ -8,10 +8,9 @@
  */
 
 #include <linux/debugfs.h>
-#include <linux/iommu.h>
 #include <linux/pci.h>
-#include "amd_iommu_proto.h"
-#include "amd_iommu_types.h"
+
+#include "amd_iommu.h"
 
 static struct dentry *amd_iommu_debugfs;
 static DEFINE_MUTEX(amd_iommu_debugfs_lock);
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd/init.c
index 5b81fd16f5fa..6ebd4825e320 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd/init.c
@@ -18,7 +18,6 @@
 #include <linux/msi.h>
 #include <linux/amd-iommu.h>
 #include <linux/export.h>
-#include <linux/iommu.h>
 #include <linux/kmemleak.h>
 #include <linux/mem_encrypt.h>
 #include <asm/pci-direct.h>
@@ -32,10 +31,9 @@
 #include <asm/irq_remapping.h>
 
 #include <linux/crash_dump.h>
+
 #include "amd_iommu.h"
-#include "amd_iommu_proto.h"
-#include "amd_iommu_types.h"
-#include "irq_remapping.h"
+#include "../irq_remapping.h"
 
 /*
  * definitions for the ACPI scanning code
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd/iommu.c
index 2883ac389abb..74cca1757172 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -22,7 +22,6 @@
 #include <linux/dma-direct.h>
 #include <linux/dma-iommu.h>
 #include <linux/iommu-helper.h>
-#include <linux/iommu.h>
 #include <linux/delay.h>
 #include <linux/amd-iommu.h>
 #include <linux/notifier.h>
@@ -43,9 +42,8 @@
 #include <asm/gart.h>
 #include <asm/dma.h>
 
-#include "amd_iommu_proto.h"
-#include "amd_iommu_types.h"
-#include "irq_remapping.h"
+#include "amd_iommu.h"
+#include "../irq_remapping.h"
 
 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
 
@@ -71,6 +69,8 @@
  */
 #define AMD_IOMMU_PGSIZES	((~0xFFFUL) & ~(2ULL << 38))
 
+#define DEFAULT_PGTABLE_LEVEL	PAGE_MODE_3_LEVEL
+
 static DEFINE_SPINLOCK(pd_bitmap_lock);
 
 /* List of all available dev_data structures */
@@ -99,7 +99,6 @@ struct iommu_cmd {
 struct kmem_cache *amd_iommu_irq_cache;
 
 static void update_domain(struct protection_domain *domain);
-static int protection_domain_init(struct protection_domain *domain);
 static void detach_device(struct device *dev);
 static void update_and_flush_device_table(struct protection_domain *domain,
 					  struct domain_pgtable *pgtable);
@@ -280,12 +279,6 @@ static struct iommu_dev_data *find_dev_data(u16 devid)
 	return dev_data;
 }
 
-struct iommu_dev_data *get_dev_data(struct device *dev)
-{
-	return dev->archdata.iommu;
-}
-EXPORT_SYMBOL(get_dev_data);
-
 /*
 * Find or create an IOMMU group for a acpihid device.
 */
@@ -314,16 +307,15 @@ static struct iommu_group *acpihid_device_group(struct device *dev)
 static bool pci_iommuv2_capable(struct pci_dev *pdev)
 {
 	static const int caps[] = {
-		PCI_EXT_CAP_ID_ATS,
 		PCI_EXT_CAP_ID_PRI,
 		PCI_EXT_CAP_ID_PASID,
 	};
 	int i, pos;
 
-	if (pci_ats_disabled())
+	if (!pci_ats_supported(pdev))
 		return false;
 
-	for (i = 0; i < 3; ++i) {
+	for (i = 0; i < 2; ++i) {
 		pos = pci_find_ext_capability(pdev, caps[i]);
 		if (pos == 0)
 			return false;
@@ -336,7 +328,7 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
 {
 	struct iommu_dev_data *dev_data;
 
-	dev_data = get_dev_data(&pdev->dev);
+	dev_data = dev_iommu_priv_get(&pdev->dev);
 
 	return dev_data->errata & (1 << erratum) ? true : false;
 }
@@ -349,7 +341,7 @@ static bool check_device(struct device *dev)
 {
 	int devid;
 
-	if (!dev || !dev->dma_mask)
+	if (!dev)
 		return false;
 
 	devid = get_device_id(dev);
@@ -366,32 +358,18 @@ static bool check_device(struct device *dev)
 	return true;
 }
 
-static void init_iommu_group(struct device *dev)
-{
-	struct iommu_group *group;
-
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return;
-
-	iommu_group_put(group);
-}
-
 static int iommu_init_device(struct device *dev)
 {
 	struct iommu_dev_data *dev_data;
-	struct amd_iommu *iommu;
 	int devid;
 
-	if (dev->archdata.iommu)
+	if (dev_iommu_priv_get(dev))
 		return 0;
 
 	devid = get_device_id(dev);
 	if (devid < 0)
 		return devid;
 
-	iommu = amd_iommu_rlookup_table[devid];
-
 	dev_data = find_dev_data(devid);
 	if (!dev_data)
 		return -ENOMEM;
@@ -412,9 +390,7 @@ static int iommu_init_device(struct device *dev)
 		dev_data->iommu_v2 = iommu->is_iommu_v2;
 	}
 
-	dev->archdata.iommu = dev_data;
-
-	iommu_device_link(&iommu->iommu, dev);
+	dev_iommu_priv_set(dev, dev_data);
 
 	return 0;
 }
@@ -433,31 +409,18 @@ static void iommu_ignore_device(struct device *dev)
 	setup_aliases(dev);
 }
 
-static void iommu_uninit_device(struct device *dev)
+static void amd_iommu_uninit_device(struct device *dev)
 {
 	struct iommu_dev_data *dev_data;
-	struct amd_iommu *iommu;
-	int devid;
 
-	devid = get_device_id(dev);
-	if (devid < 0)
-		return;
-
-	iommu = amd_iommu_rlookup_table[devid];
-
-	dev_data = search_dev_data(devid);
+	dev_data = dev_iommu_priv_get(dev);
 	if (!dev_data)
 		return;
 
 	if (dev_data->domain)
 		detach_device(dev);
 
-	iommu_device_unlink(&iommu->iommu, dev);
-
-	iommu_group_remove_device(dev);
-
-	/* Remove dma-ops */
-	dev->dma_ops = NULL;
+	dev_iommu_priv_set(dev, NULL);
 
 	/*
 	 * We keep dev_data around for unplugged devices and reuse it when the
@@ -521,7 +484,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id,
 	pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid),
 					   devid & 0xff);
 	if (pdev)
-		dev_data = get_dev_data(&pdev->dev);
+		dev_data = dev_iommu_priv_get(&pdev->dev);
 
 	if (dev_data && __ratelimit(&dev_data->rs)) {
 		pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n",
@@ -1418,20 +1381,19 @@ static struct page *free_sub_pt(unsigned long root, int mode,
 	return freelist;
 }
 
-static void free_pagetable(struct protection_domain *domain)
+static void free_pagetable(struct domain_pgtable *pgtable)
 {
-	struct domain_pgtable pgtable;
 	struct page *freelist = NULL;
 	unsigned long root;
 
-	amd_iommu_domain_get_pgtable(domain, &pgtable);
-	atomic64_set(&domain->pt_root, 0);
+	if (pgtable->mode == PAGE_MODE_NONE)
+		return;
 
-	BUG_ON(pgtable.mode < PAGE_MODE_NONE ||
-	       pgtable.mode > PAGE_MODE_6_LEVEL);
+	BUG_ON(pgtable->mode < PAGE_MODE_NONE ||
+	       pgtable->mode > PAGE_MODE_6_LEVEL);
 
-	root = (unsigned long)pgtable.root;
-	freelist = free_sub_pt(root, pgtable.mode, freelist);
+	root = (unsigned long)pgtable->root;
+	freelist = free_sub_pt(root, pgtable->mode, freelist);
 
 	free_page_list(freelist);
 }
@@ -1844,70 +1806,6 @@ static void free_gcr3_table(struct protection_domain *domain)
 	free_page((unsigned long)domain->gcr3_tbl);
 }
 
-/*
- * Free a domain, only used if something went wrong in the
- * allocation path and we need to free an already allocated page table
- */
-static void dma_ops_domain_free(struct protection_domain *domain)
-{
-	if (!domain)
-		return;
-
-	iommu_put_dma_cookie(&domain->domain);
-
-	free_pagetable(domain);
-
-	if (domain->id)
-		domain_id_free(domain->id);
-
-	kfree(domain);
-}
-
-/*
- * Allocates a new protection domain usable for the dma_ops functions.
- * It also initializes the page table and the address allocator data
- * structures required for the dma_ops interface
- */
-static struct protection_domain *dma_ops_domain_alloc(void)
-{
-	struct protection_domain *domain;
-	u64 *pt_root, root;
-
-	domain = kzalloc(sizeof(struct protection_domain), GFP_KERNEL);
-	if (!domain)
-		return NULL;
-
-	if (protection_domain_init(domain))
-		goto free_domain;
-
-	pt_root = (void *)get_zeroed_page(GFP_KERNEL);
-	if (!pt_root)
-		goto free_domain;
-
-	root = amd_iommu_domain_encode_pgtable(pt_root, PAGE_MODE_3_LEVEL);
-	atomic64_set(&domain->pt_root, root);
-	domain->flags = PD_DMA_OPS_MASK;
-
-	if (iommu_get_dma_cookie(&domain->domain) == -ENOMEM)
-		goto free_domain;
-
-	return domain;
-
-free_domain:
-	dma_ops_domain_free(domain);
-
-	return NULL;
-}
-
-/*
- * little helper function to check whether a given protection domain is a
- * dma_ops domain
- */
-static bool dma_ops_domain(struct protection_domain *domain)
-{
-	return domain->flags & PD_DMA_OPS_MASK;
-}
-
 static void set_dte_entry(u16 devid, struct protection_domain *domain,
 			  struct domain_pgtable *pgtable,
 			  bool ats, bool ppr)
@@ -2119,14 +2017,14 @@ out_err:
 static int attach_device(struct device *dev,
 			 struct protection_domain *domain)
 {
-	struct pci_dev *pdev;
 	struct iommu_dev_data *dev_data;
+	struct pci_dev *pdev;
 	unsigned long flags;
 	int ret;
 
 	spin_lock_irqsave(&domain->lock, flags);
 
-	dev_data = get_dev_data(dev);
+	dev_data = dev_iommu_priv_get(dev);
 
 	spin_lock(&dev_data->lock);
 
@@ -2139,8 +2037,10 @@ static int attach_device(struct device *dev,
 
 	pdev = to_pci_dev(dev);
 	if (domain->flags & PD_IOMMUV2_MASK) {
+		struct iommu_domain *def_domain = iommu_get_dma_domain(dev);
+
 		ret = -EINVAL;
-		if (!dev_data->passthrough)
+		if (def_domain->type != IOMMU_DOMAIN_IDENTITY)
 			goto out;
 
 		if (dev_data->iommu_v2) {
@@ -2188,7 +2088,7 @@ static void detach_device(struct device *dev)
 	struct iommu_dev_data *dev_data;
 	unsigned long flags;
 
-	dev_data = get_dev_data(dev);
+	dev_data = dev_iommu_priv_get(dev);
 	domain   = dev_data->domain;
 
 	spin_lock_irqsave(&domain->lock, flags);
@@ -2222,68 +2122,60 @@ out:
 	spin_unlock_irqrestore(&domain->lock, flags);
 }
 
-static int amd_iommu_add_device(struct device *dev)
+static struct iommu_device *amd_iommu_probe_device(struct device *dev)
 {
-	struct iommu_dev_data *dev_data;
-	struct iommu_domain *domain;
+	struct iommu_device *iommu_dev;
 	struct amd_iommu *iommu;
 	int ret, devid;
 
-	if (!check_device(dev) || get_dev_data(dev))
-		return 0;
+	if (!check_device(dev))
+		return ERR_PTR(-ENODEV);
 
 	devid = get_device_id(dev);
 	if (devid < 0)
-		return devid;
+		return ERR_PTR(devid);
 
 	iommu = amd_iommu_rlookup_table[devid];
 
+	if (dev_iommu_priv_get(dev))
+		return &iommu->iommu;
+
 	ret = iommu_init_device(dev);
 	if (ret) {
 		if (ret != -ENOTSUPP)
 			dev_err(dev, "Failed to initialize - trying to proceed anyway\n");
-
+		iommu_dev = ERR_PTR(ret);
 		iommu_ignore_device(dev);
-		dev->dma_ops = NULL;
-		goto out;
+	} else {
+		iommu_dev = &iommu->iommu;
 	}
-	init_iommu_group(dev);
 
-	dev_data = get_dev_data(dev);
+	iommu_completion_wait(iommu);
 
-	BUG_ON(!dev_data);
+	return iommu_dev;
+}
 
-	if (dev_data->iommu_v2)
-		iommu_request_dm_for_dev(dev);
+static void amd_iommu_probe_finalize(struct device *dev)
+{
+	struct iommu_domain *domain;
 
 	/* Domains are initialized for this device - have a look what we ended up with */
 	domain = iommu_get_domain_for_dev(dev);
-	if (domain->type == IOMMU_DOMAIN_IDENTITY)
-		dev_data->passthrough = true;
-	else if (domain->type == IOMMU_DOMAIN_DMA)
+	if (domain->type == IOMMU_DOMAIN_DMA)
 		iommu_setup_dma_ops(dev, IOVA_START_PFN << PAGE_SHIFT, 0);
-
-out:
-	iommu_completion_wait(iommu);
-
-	return 0;
 }
 
-static void amd_iommu_remove_device(struct device *dev)
+static void amd_iommu_release_device(struct device *dev)
 {
+	int devid = get_device_id(dev);
 	struct amd_iommu *iommu;
-	int devid;
 
 	if (!check_device(dev))
 		return;
 
-	devid = get_device_id(dev);
-	if (devid < 0)
-		return;
-
 	iommu = amd_iommu_rlookup_table[devid];
 
-	iommu_uninit_device(dev);
+	amd_iommu_uninit_device(dev);
 	iommu_completion_wait(iommu);
 }
 
@@ -2418,27 +2310,46 @@ static void cleanup_domain(struct protection_domain *domain)
 
 static void protection_domain_free(struct protection_domain *domain)
 {
+	struct domain_pgtable pgtable;
+
 	if (!domain)
 		return;
 
 	if (domain->id)
 		domain_id_free(domain->id);
 
+	amd_iommu_domain_get_pgtable(domain, &pgtable);
+	atomic64_set(&domain->pt_root, 0);
+	free_pagetable(&pgtable);
+
 	kfree(domain);
 }
 
-static int protection_domain_init(struct protection_domain *domain)
+static int protection_domain_init(struct protection_domain *domain, int mode)
 {
+	u64 *pt_root = NULL, root;
+
+	BUG_ON(mode < PAGE_MODE_NONE || mode > PAGE_MODE_6_LEVEL);
+
 	spin_lock_init(&domain->lock);
 	domain->id = domain_id_alloc();
 	if (!domain->id)
 		return -ENOMEM;
 	INIT_LIST_HEAD(&domain->dev_list);
 
+	if (mode != PAGE_MODE_NONE) {
+		pt_root = (void *)get_zeroed_page(GFP_KERNEL);
+		if (!pt_root)
+			return -ENOMEM;
+	}
+
+	root = amd_iommu_domain_encode_pgtable(pt_root, mode);
+	atomic64_set(&domain->pt_root, root);
+
 	return 0;
 }
 
-static struct protection_domain *protection_domain_alloc(void)
+static struct protection_domain *protection_domain_alloc(int mode)
 {
 	struct protection_domain *domain;
 
@@ -2446,7 +2357,7 @@ static struct protection_domain *protection_domain_alloc(void)
 	if (!domain)
 		return NULL;
 
-	if (protection_domain_init(domain))
+	if (protection_domain_init(domain, mode))
 		goto out_err;
 
 	return domain;
@@ -2459,54 +2370,35 @@ out_err:
 
 static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
 {
-	struct protection_domain *pdomain;
-	u64 *pt_root, root;
+	struct protection_domain *domain;
+	int mode = DEFAULT_PGTABLE_LEVEL;
 
-	switch (type) {
-	case IOMMU_DOMAIN_UNMANAGED:
-		pdomain = protection_domain_alloc();
-		if (!pdomain)
-			return NULL;
+	if (type == IOMMU_DOMAIN_IDENTITY)
+		mode = PAGE_MODE_NONE;
 
-		pt_root = (void *)get_zeroed_page(GFP_KERNEL);
-		if (!pt_root) {
-			protection_domain_free(pdomain);
-			return NULL;
-		}
+	domain = protection_domain_alloc(mode);
+	if (!domain)
+		return NULL;
 
-		root = amd_iommu_domain_encode_pgtable(pt_root, PAGE_MODE_3_LEVEL);
-		atomic64_set(&pdomain->pt_root, root);
+	domain->domain.geometry.aperture_start = 0;
+	domain->domain.geometry.aperture_end   = ~0ULL;
+	domain->domain.geometry.force_aperture = true;
 
-		pdomain->domain.geometry.aperture_start = 0;
-		pdomain->domain.geometry.aperture_end   = ~0ULL;
-		pdomain->domain.geometry.force_aperture = true;
+	if (type == IOMMU_DOMAIN_DMA &&
+	    iommu_get_dma_cookie(&domain->domain) == -ENOMEM)
+		goto free_domain;
 
-		break;
-	case IOMMU_DOMAIN_DMA:
-		pdomain = dma_ops_domain_alloc();
-		if (!pdomain) {
-			pr_err("Failed to allocate\n");
-			return NULL;
-		}
-		break;
-	case IOMMU_DOMAIN_IDENTITY:
-		pdomain = protection_domain_alloc();
-		if (!pdomain)
-			return NULL;
+	return &domain->domain;
 
-		atomic64_set(&pdomain->pt_root, PAGE_MODE_NONE);
-		break;
-	default:
-		return NULL;
-	}
+free_domain:
+	protection_domain_free(domain);
 
-	return &pdomain->domain;
+	return NULL;
 }
 
 static void amd_iommu_domain_free(struct iommu_domain *dom)
 {
 	struct protection_domain *domain;
-	struct domain_pgtable pgtable;
 
 	domain = to_pdomain(dom);
 
@@ -2518,29 +2410,19 @@ static void amd_iommu_domain_free(struct iommu_domain *dom)
 	if (!dom)
 		return;
 
-	switch (dom->type) {
-	case IOMMU_DOMAIN_DMA:
-		/* Now release the domain */
-		dma_ops_domain_free(domain);
-		break;
-	default:
-		amd_iommu_domain_get_pgtable(domain, &pgtable);
-
-		if (pgtable.mode != PAGE_MODE_NONE)
-			free_pagetable(domain);
+	if (dom->type == IOMMU_DOMAIN_DMA)
+		iommu_put_dma_cookie(&domain->domain);
 
-		if (domain->flags & PD_IOMMUV2_MASK)
-			free_gcr3_table(domain);
+	if (domain->flags & PD_IOMMUV2_MASK)
+		free_gcr3_table(domain);
 
-		protection_domain_free(domain);
-		break;
-	}
+	protection_domain_free(domain);
 }
 
 static void amd_iommu_detach_device(struct iommu_domain *dom,
 				    struct device *dev)
 {
-	struct iommu_dev_data *dev_data = dev->archdata.iommu;
+	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
 	struct amd_iommu *iommu;
 	int devid;
 
@@ -2578,7 +2460,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 	if (!check_device(dev))
 		return -EINVAL;
 
-	dev_data = dev->archdata.iommu;
+	dev_data = dev_iommu_priv_get(dev);
 	dev_data->defer_attach = false;
 
 	iommu = amd_iommu_rlookup_table[dev_data->devid];
@@ -2734,12 +2616,14 @@ static void amd_iommu_get_resv_regions(struct device *dev,
 	list_add_tail(&region->list, head);
 }
 
-static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
-					 struct device *dev)
+bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
+				  struct device *dev)
 {
-	struct iommu_dev_data *dev_data = dev->archdata.iommu;
+	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+
 	return dev_data->defer_attach;
 }
+EXPORT_SYMBOL_GPL(amd_iommu_is_attach_deferred);
 
 static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
 {
@@ -2758,6 +2642,20 @@ static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
 	amd_iommu_flush_iotlb_all(domain);
 }
 
+static int amd_iommu_def_domain_type(struct device *dev)
+{
+	struct iommu_dev_data *dev_data;
+
+	dev_data = dev_iommu_priv_get(dev);
+	if (!dev_data)
+		return 0;
+
+	if (dev_data->iommu_v2)
+		return IOMMU_DOMAIN_IDENTITY;
+
+	return 0;
+}
+
 const struct iommu_ops amd_iommu_ops = {
 	.capable = amd_iommu_capable,
 	.domain_alloc = amd_iommu_domain_alloc,
@@ -2767,8 +2665,9 @@ const struct iommu_ops amd_iommu_ops = {
 	.map = amd_iommu_map,
 	.unmap = amd_iommu_unmap,
 	.iova_to_phys = amd_iommu_iova_to_phys,
-	.add_device = amd_iommu_add_device,
-	.remove_device = amd_iommu_remove_device,
+	.probe_device = amd_iommu_probe_device,
+	.release_device = amd_iommu_release_device,
+	.probe_finalize = amd_iommu_probe_finalize,
 	.device_group = amd_iommu_device_group,
 	.domain_get_attr = amd_iommu_domain_get_attr,
 	.get_resv_regions = amd_iommu_get_resv_regions,
@@ -2777,6 +2676,7 @@ const struct iommu_ops amd_iommu_ops = {
 	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
 	.flush_iotlb_all = amd_iommu_flush_iotlb_all,
 	.iotlb_sync = amd_iommu_iotlb_sync,
+	.def_domain_type = amd_iommu_def_domain_type,
 };
 
 /*****************************************************************************
@@ -2807,7 +2707,6 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom)
 	struct protection_domain *domain = to_pdomain(dom);
 	struct domain_pgtable pgtable;
 	unsigned long flags;
-	u64 pt_root;
 
 	spin_lock_irqsave(&domain->lock, flags);
 
@@ -2815,18 +2714,13 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom)
 	amd_iommu_domain_get_pgtable(domain, &pgtable);
 
 	/* Update data structure */
-	pt_root = amd_iommu_domain_encode_pgtable(NULL, PAGE_MODE_NONE);
-	atomic64_set(&domain->pt_root, pt_root);
+	atomic64_set(&domain->pt_root, 0);
 
 	/* Make changes visible to IOMMUs */
 	update_domain(domain);
 
-	/* Restore old pgtable in domain->ptroot to free page-table */
-	pt_root = amd_iommu_domain_encode_pgtable(pgtable.root, pgtable.mode);
-	atomic64_set(&domain->pt_root, pt_root);
-
 	/* Page-table is not visible to IOMMU anymore, so free it */
-	free_pagetable(domain);
+	free_pagetable(&pgtable);
 
 	spin_unlock_irqrestore(&domain->lock, flags);
 }
@@ -3085,7 +2979,7 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
 	struct amd_iommu *iommu;
 	struct iommu_cmd cmd;
 
-	dev_data = get_dev_data(&pdev->dev);
+	dev_data = dev_iommu_priv_get(&pdev->dev);
 	iommu    = amd_iommu_rlookup_table[dev_data->devid];
 
 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,
@@ -3098,23 +2992,27 @@ EXPORT_SYMBOL(amd_iommu_complete_ppr);
 struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
 {
 	struct protection_domain *pdomain;
-	struct iommu_domain *io_domain;
+	struct iommu_dev_data *dev_data;
 	struct device *dev = &pdev->dev;
+	struct iommu_domain *io_domain;
 
 	if (!check_device(dev))
 		return NULL;
 
-	pdomain = get_dev_data(dev)->domain;
-	if (pdomain == NULL && get_dev_data(dev)->defer_attach) {
-		get_dev_data(dev)->defer_attach = false;
-		io_domain = iommu_get_domain_for_dev(dev);
+	dev_data  = dev_iommu_priv_get(&pdev->dev);
+	pdomain   = dev_data->domain;
+	io_domain = iommu_get_domain_for_dev(dev);
+
+	if (pdomain == NULL && dev_data->defer_attach) {
+		dev_data->defer_attach = false;
 		pdomain = to_pdomain(io_domain);
 		attach_device(dev, pdomain);
 	}
+
 	if (pdomain == NULL)
 		return NULL;
 
-	if (!dma_ops_domain(pdomain))
+	if (io_domain->type != IOMMU_DOMAIN_DMA)
 		return NULL;
 
 	/* Only return IOMMUv2 domains */
@@ -3132,7 +3030,7 @@ void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
 	if (!amd_iommu_v2_supported())
 		return;
 
-	dev_data = get_dev_data(&pdev->dev);
+	dev_data = dev_iommu_priv_get(&pdev->dev);
 	dev_data->errata |= (1 << erratum);
 }
 EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
@@ -3151,11 +3049,8 @@ int amd_iommu_device_info(struct pci_dev *pdev,
 
 	memset(info, 0, sizeof(*info));
 
-	if (!pci_ats_disabled()) {
-		pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
-		if (pos)
-			info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
-	}
+	if (pci_ats_supported(pdev))
+		info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
 
 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
 	if (pos)
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd/iommu_v2.c
index d6d85debd01b..e4b025c5637c 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd/iommu_v2.c
@@ -13,13 +13,11 @@
 #include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/sched/mm.h>
-#include <linux/iommu.h>
 #include <linux/wait.h>
 #include <linux/pci.h>
 #include <linux/gfp.h>
 
-#include "amd_iommu_types.h"
-#include "amd_iommu_proto.h"
+#include "amd_iommu.h"
 
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Joerg Roedel <jroedel@suse.de>");
@@ -487,7 +485,7 @@ static void do_fault(struct work_struct *work)
 		flags |= FAULT_FLAG_WRITE;
 	flags |= FAULT_FLAG_REMOTE;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	vma = find_extend_vma(mm, address);
 	if (!vma || address < vma->vm_start)
 		/* failed to get a vma in the right range */
@@ -499,7 +497,7 @@ static void do_fault(struct work_struct *work)
 
 	ret = handle_mm_fault(vma, address, flags);
 out:
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 
 	if (ret & VM_FAULT_ERROR)
 		/* failed to service fault */
@@ -517,13 +515,12 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
 	struct amd_iommu_fault *iommu_fault;
 	struct pasid_state *pasid_state;
 	struct device_state *dev_state;
+	struct pci_dev *pdev = NULL;
 	unsigned long flags;
 	struct fault *fault;
 	bool finish;
 	u16 tag, devid;
 	int ret;
-	struct iommu_dev_data *dev_data;
-	struct pci_dev *pdev = NULL;
 
 	iommu_fault = data;
 	tag         = iommu_fault->tag & 0x1ff;
@@ -534,12 +531,11 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
 					   devid & 0xff);
 	if (!pdev)
 		return -ENODEV;
-	dev_data = get_dev_data(&pdev->dev);
 
-	/* In kdump kernel pci dev is not initialized yet -> send INVALID */
 	ret = NOTIFY_DONE;
-	if (translation_pre_enabled(amd_iommu_rlookup_table[devid])
-		&& dev_data->defer_attach) {
+
+	/* In kdump kernel pci dev is not initialized yet -> send INVALID */
+	if (amd_iommu_is_attach_deferred(NULL, &pdev->dev)) {
 		amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
 				       PPR_INVALID, tag);
 		goto out;
diff --git a/drivers/iommu/amd_iommu_quirks.c b/drivers/iommu/amd/quirks.c
index 5120ce4fdce3..5120ce4fdce3 100644
--- a/drivers/iommu/amd_iommu_quirks.c
+++ b/drivers/iommu/amd/quirks.c
diff --git a/drivers/iommu/amd_iommu.h b/drivers/iommu/amd_iommu.h
deleted file mode 100644
index 12d540d9b59b..000000000000
--- a/drivers/iommu/amd_iommu.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-
-#ifndef AMD_IOMMU_H
-#define AMD_IOMMU_H
-
-int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line);
-
-#ifdef CONFIG_DMI
-void amd_iommu_apply_ivrs_quirks(void);
-#else
-static void amd_iommu_apply_ivrs_quirks(void) { }
-#endif
-
-#endif
diff --git a/drivers/iommu/arm-smmu-impl.c b/drivers/iommu/arm-smmu-impl.c
index 74d97a886e93..c75b9d957b70 100644
--- a/drivers/iommu/arm-smmu-impl.c
+++ b/drivers/iommu/arm-smmu-impl.c
@@ -150,6 +150,8 @@ static const struct arm_smmu_impl arm_mmu500_impl = {
 
 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
 {
+	const struct device_node *np = smmu->dev->of_node;
+
 	/*
 	 * We will inevitably have to combine model-specific implementation
 	 * quirks with platform-specific integration quirks, but everything
@@ -166,11 +168,11 @@ struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
 		break;
 	}
 
-	if (of_property_read_bool(smmu->dev->of_node,
-				  "calxeda,smmu-secure-config-access"))
+	if (of_property_read_bool(np, "calxeda,smmu-secure-config-access"))
 		smmu->impl = &calxeda_impl;
 
-	if (of_device_is_compatible(smmu->dev->of_node, "qcom,sdm845-smmu-500"))
+	if (of_device_is_compatible(np, "qcom,sdm845-smmu-500") ||
+	    of_device_is_compatible(np, "qcom,sc7180-smmu-500"))
 		return qcom_smmu_impl_init(smmu);
 
 	return smmu;
diff --git a/drivers/iommu/arm-smmu-qcom.c b/drivers/iommu/arm-smmu-qcom.c
index 24c071c1d8b0..cf01d0215a39 100644
--- a/drivers/iommu/arm-smmu-qcom.c
+++ b/drivers/iommu/arm-smmu-qcom.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2019, The Linux Foundation. All rights reserved.
  */
 
+#include <linux/of_device.h>
 #include <linux/qcom_scm.h>
 
 #include "arm-smmu.h"
@@ -11,12 +12,29 @@ struct qcom_smmu {
 	struct arm_smmu_device smmu;
 };
 
+static const struct of_device_id qcom_smmu_client_of_match[] = {
+	{ .compatible = "qcom,adreno" },
+	{ .compatible = "qcom,mdp4" },
+	{ .compatible = "qcom,mdss" },
+	{ .compatible = "qcom,sc7180-mdss" },
+	{ .compatible = "qcom,sc7180-mss-pil" },
+	{ .compatible = "qcom,sdm845-mdss" },
+	{ .compatible = "qcom,sdm845-mss-pil" },
+	{ }
+};
+
+static int qcom_smmu_def_domain_type(struct device *dev)
+{
+	const struct of_device_id *match =
+		of_match_device(qcom_smmu_client_of_match, dev);
+
+	return match ? IOMMU_DOMAIN_IDENTITY : 0;
+}
+
 static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
 {
 	int ret;
 
-	arm_mmu500_reset(smmu);
-
 	/*
 	 * To address performance degradation in non-real time clients,
 	 * such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
@@ -30,8 +48,21 @@ static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
 	return ret;
 }
 
+static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
+{
+	const struct device_node *np = smmu->dev->of_node;
+
+	arm_mmu500_reset(smmu);
+
+	if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
+		return qcom_sdm845_smmu500_reset(smmu);
+
+	return 0;
+}
+
 static const struct arm_smmu_impl qcom_smmu_impl = {
-	.reset = qcom_sdm845_smmu500_reset,
+	.def_domain_type = qcom_smmu_def_domain_type,
+	.reset = qcom_smmu500_reset,
 };
 
 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 82508730feb7..f578677a5c41 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -171,6 +171,8 @@
 #define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
 #define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
 
+#define ARM_SMMU_REG_SZ			0xe00
+
 /* Common MSI config fields */
 #define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
 #define MSI_CFG2_SH			GENMASK(5, 4)
@@ -628,6 +630,7 @@ struct arm_smmu_strtab_cfg {
 struct arm_smmu_device {
 	struct device			*dev;
 	void __iomem			*base;
+	void __iomem			*page1;
 
 #define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
 #define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
@@ -664,7 +667,6 @@ struct arm_smmu_device {
 
 #define ARM_SMMU_MAX_ASIDS		(1 << 16)
 	unsigned int			asid_bits;
-	DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
 
 #define ARM_SMMU_MAX_VMIDS		(1 << 16)
 	unsigned int			vmid_bits;
@@ -724,6 +726,8 @@ struct arm_smmu_option_prop {
 	const char *prop;
 };
 
+static DEFINE_XARRAY_ALLOC1(asid_xa);
+
 static struct arm_smmu_option_prop arm_smmu_options[] = {
 	{ ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
 	{ ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
@@ -733,9 +737,8 @@ static struct arm_smmu_option_prop arm_smmu_options[] = {
 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
 						 struct arm_smmu_device *smmu)
 {
-	if ((offset > SZ_64K) &&
-	    (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
-		offset -= SZ_64K;
+	if (offset > SZ_64K)
+		return smmu->page1 + offset - SZ_64K;
 
 	return smmu->base + offset;
 }
@@ -1763,6 +1766,14 @@ static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
 	cdcfg->cdtab = NULL;
 }
 
+static void arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd)
+{
+	if (!cd->asid)
+		return;
+
+	xa_erase(&asid_xa, cd->asid);
+}
+
 /* Stream table manipulation functions */
 static void
 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
@@ -2448,10 +2459,9 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
 		struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
 
-		if (cfg->cdcfg.cdtab) {
+		if (cfg->cdcfg.cdtab)
 			arm_smmu_free_cd_tables(smmu_domain);
-			arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
-		}
+		arm_smmu_free_asid(&cfg->cd);
 	} else {
 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
 		if (cfg->vmid)
@@ -2466,14 +2476,15 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 				       struct io_pgtable_cfg *pgtbl_cfg)
 {
 	int ret;
-	int asid;
+	u32 asid;
 	struct arm_smmu_device *smmu = smmu_domain->smmu;
 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
 	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 
-	asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
-	if (asid < 0)
-		return asid;
+	ret = xa_alloc(&asid_xa, &asid, &cfg->cd,
+		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
+	if (ret)
+		return ret;
 
 	cfg->s1cdmax = master->ssid_bits;
 
@@ -2506,7 +2517,7 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 out_free_cd_tables:
 	arm_smmu_free_cd_tables(smmu_domain);
 out_free_asid:
-	arm_smmu_bitmap_free(smmu->asid_map, asid);
+	arm_smmu_free_asid(&cfg->cd);
 	return ret;
 }
 
@@ -2652,26 +2663,20 @@ static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
 	}
 }
 
-#ifdef CONFIG_PCI_ATS
 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
 {
-	struct pci_dev *pdev;
+	struct device *dev = master->dev;
 	struct arm_smmu_device *smmu = master->smmu;
-	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 
-	if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
-	    !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
+	if (!(smmu->features & ARM_SMMU_FEAT_ATS))
 		return false;
 
-	pdev = to_pci_dev(master->dev);
-	return !pdev->untrusted && pdev->ats_cap;
-}
-#else
-static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
-{
-	return false;
+	if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
+		return false;
+
+	return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
 }
-#endif
 
 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
 {
@@ -2914,27 +2919,26 @@ static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
 
 static struct iommu_ops arm_smmu_ops;
 
-static int arm_smmu_add_device(struct device *dev)
+static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 {
 	int i, ret;
 	struct arm_smmu_device *smmu;
 	struct arm_smmu_master *master;
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
-	struct iommu_group *group;
 
 	if (!fwspec || fwspec->ops != &arm_smmu_ops)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
-		return -EBUSY;
+		return ERR_PTR(-EBUSY);
 
 	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
 	if (!smmu)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	master = kzalloc(sizeof(*master), GFP_KERNEL);
 	if (!master)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	master->dev = dev;
 	master->smmu = smmu;
@@ -2975,43 +2979,24 @@ static int arm_smmu_add_device(struct device *dev)
 		master->ssid_bits = min_t(u8, master->ssid_bits,
 					  CTXDESC_LINEAR_CDMAX);
 
-	ret = iommu_device_link(&smmu->iommu, dev);
-	if (ret)
-		goto err_disable_pasid;
-
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group)) {
-		ret = PTR_ERR(group);
-		goto err_unlink;
-	}
+	return &smmu->iommu;
 
-	iommu_group_put(group);
-	return 0;
-
-err_unlink:
-	iommu_device_unlink(&smmu->iommu, dev);
-err_disable_pasid:
-	arm_smmu_disable_pasid(master);
 err_free_master:
 	kfree(master);
 	dev_iommu_priv_set(dev, NULL);
-	return ret;
+	return ERR_PTR(ret);
 }
 
-static void arm_smmu_remove_device(struct device *dev)
+static void arm_smmu_release_device(struct device *dev)
 {
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 	struct arm_smmu_master *master;
-	struct arm_smmu_device *smmu;
 
 	if (!fwspec || fwspec->ops != &arm_smmu_ops)
 		return;
 
 	master = dev_iommu_priv_get(dev);
-	smmu = master->smmu;
 	arm_smmu_detach_dev(master);
-	iommu_group_remove_device(dev);
-	iommu_device_unlink(&smmu->iommu, dev);
 	arm_smmu_disable_pasid(master);
 	kfree(master);
 	iommu_fwspec_free(dev);
@@ -3138,8 +3123,8 @@ static struct iommu_ops arm_smmu_ops = {
 	.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
 	.iotlb_sync		= arm_smmu_iotlb_sync,
 	.iova_to_phys		= arm_smmu_iova_to_phys,
-	.add_device		= arm_smmu_add_device,
-	.remove_device		= arm_smmu_remove_device,
+	.probe_device		= arm_smmu_probe_device,
+	.release_device		= arm_smmu_release_device,
 	.device_group		= arm_smmu_device_group,
 	.domain_get_attr	= arm_smmu_domain_get_attr,
 	.domain_set_attr	= arm_smmu_domain_set_attr,
@@ -4021,6 +4006,18 @@ err_reset_pci_ops: __maybe_unused;
 	return err;
 }
 
+static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
+				      resource_size_t size)
+{
+	struct resource res = {
+		.flags = IORESOURCE_MEM,
+		.start = start,
+		.end = start + size - 1,
+	};
+
+	return devm_ioremap_resource(dev, &res);
+}
+
 static int arm_smmu_device_probe(struct platform_device *pdev)
 {
 	int irq, ret;
@@ -4056,10 +4053,23 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 	}
 	ioaddr = res->start;
 
-	smmu->base = devm_ioremap_resource(dev, res);
+	/*
+	 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
+	 * the PMCG registers which are reserved by the PMU driver.
+	 */
+	smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
 	if (IS_ERR(smmu->base))
 		return PTR_ERR(smmu->base);
 
+	if (arm_smmu_resource_size(smmu) > SZ_64K) {
+		smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
+					       ARM_SMMU_REG_SZ);
+		if (IS_ERR(smmu->page1))
+			return PTR_ERR(smmu->page1);
+	} else {
+		smmu->page1 = smmu->base;
+	}
+
 	/* Interrupt lines */
 
 	irq = platform_get_irq_byname_optional(pdev, "combined");
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index a6a5796e9c41..243bc4cb2705 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -220,7 +220,7 @@ static int arm_smmu_register_legacy_master(struct device *dev,
  * With the legacy DT binding in play, we have no guarantees about
  * probe order, but then we're also not doing default domains, so we can
  * delay setting bus ops until we're sure every possible SMMU is ready,
- * and that way ensure that no add_device() calls get missed.
+ * and that way ensure that no probe_device() calls get missed.
  */
 static int arm_smmu_legacy_bus_init(void)
 {
@@ -1062,7 +1062,6 @@ static int arm_smmu_master_alloc_smes(struct device *dev)
 	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
 	struct arm_smmu_device *smmu = cfg->smmu;
 	struct arm_smmu_smr *smrs = smmu->smrs;
-	struct iommu_group *group;
 	int i, idx, ret;
 
 	mutex_lock(&smmu->stream_map_mutex);
@@ -1090,18 +1089,9 @@ static int arm_smmu_master_alloc_smes(struct device *dev)
 		cfg->smendx[i] = (s16)idx;
 	}
 
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group)) {
-		ret = PTR_ERR(group);
-		goto out_err;
-	}
-	iommu_group_put(group);
-
 	/* It worked! Now, poke the actual hardware */
-	for_each_cfg_sme(cfg, fwspec, i, idx) {
+	for_each_cfg_sme(cfg, fwspec, i, idx)
 		arm_smmu_write_sme(smmu, idx);
-		smmu->s2crs[idx].group = group;
-	}
 
 	mutex_unlock(&smmu->stream_map_mutex);
 	return 0;
@@ -1172,7 +1162,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 
 	/*
 	 * FIXME: The arch/arm DMA API code tries to attach devices to its own
-	 * domains between of_xlate() and add_device() - we have no way to cope
+	 * domains between of_xlate() and probe_device() - we have no way to cope
 	 * with that, so until ARM gets converted to rely on groups and default
 	 * domains, just say no (but more politely than by dereferencing NULL).
 	 * This should be at least a WARN_ON once that's sorted.
@@ -1382,7 +1372,7 @@ struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
 	return dev ? dev_get_drvdata(dev) : NULL;
 }
 
-static int arm_smmu_add_device(struct device *dev)
+static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 {
 	struct arm_smmu_device *smmu = NULL;
 	struct arm_smmu_master_cfg *cfg;
@@ -1403,7 +1393,7 @@ static int arm_smmu_add_device(struct device *dev)
 	} else if (fwspec && fwspec->ops == &arm_smmu_ops) {
 		smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
 	} else {
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 	}
 
 	ret = -EINVAL;
@@ -1444,21 +1434,19 @@ static int arm_smmu_add_device(struct device *dev)
 	if (ret)
 		goto out_cfg_free;
 
-	iommu_device_link(&smmu->iommu, dev);
-
 	device_link_add(dev, smmu->dev,
 			DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
 
-	return 0;
+	return &smmu->iommu;
 
 out_cfg_free:
 	kfree(cfg);
 out_free:
 	iommu_fwspec_free(dev);
-	return ret;
+	return ERR_PTR(ret);
 }
 
-static void arm_smmu_remove_device(struct device *dev)
+static void arm_smmu_release_device(struct device *dev)
 {
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 	struct arm_smmu_master_cfg *cfg;
@@ -1475,13 +1463,11 @@ static void arm_smmu_remove_device(struct device *dev)
 	if (ret < 0)
 		return;
 
-	iommu_device_unlink(&smmu->iommu, dev);
 	arm_smmu_master_free_smes(cfg, fwspec);
 
 	arm_smmu_rpm_put(smmu);
 
 	dev_iommu_priv_set(dev, NULL);
-	iommu_group_remove_device(dev);
 	kfree(cfg);
 	iommu_fwspec_free(dev);
 }
@@ -1512,6 +1498,11 @@ static struct iommu_group *arm_smmu_device_group(struct device *dev)
 	else
 		group = generic_device_group(dev);
 
+	/* Remember group for faster lookups */
+	if (!IS_ERR(group))
+		for_each_cfg_sme(cfg, fwspec, i, idx)
+			smmu->s2crs[idx].group = group;
+
 	return group;
 }
 
@@ -1618,6 +1609,17 @@ static void arm_smmu_get_resv_regions(struct device *dev,
 	iommu_dma_get_resv_regions(dev, head);
 }
 
+static int arm_smmu_def_domain_type(struct device *dev)
+{
+	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
+	const struct arm_smmu_impl *impl = cfg->smmu->impl;
+
+	if (impl && impl->def_domain_type)
+		return impl->def_domain_type(dev);
+
+	return 0;
+}
+
 static struct iommu_ops arm_smmu_ops = {
 	.capable		= arm_smmu_capable,
 	.domain_alloc		= arm_smmu_domain_alloc,
@@ -1628,14 +1630,15 @@ static struct iommu_ops arm_smmu_ops = {
 	.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
 	.iotlb_sync		= arm_smmu_iotlb_sync,
 	.iova_to_phys		= arm_smmu_iova_to_phys,
-	.add_device		= arm_smmu_add_device,
-	.remove_device		= arm_smmu_remove_device,
+	.probe_device		= arm_smmu_probe_device,
+	.release_device		= arm_smmu_release_device,
 	.device_group		= arm_smmu_device_group,
 	.domain_get_attr	= arm_smmu_domain_get_attr,
 	.domain_set_attr	= arm_smmu_domain_set_attr,
 	.of_xlate		= arm_smmu_of_xlate,
 	.get_resv_regions	= arm_smmu_get_resv_regions,
 	.put_resv_regions	= generic_iommu_put_resv_regions,
+	.def_domain_type	= arm_smmu_def_domain_type,
 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
 };
 
@@ -2253,7 +2256,7 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
 		return -ENODEV;
 
 	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
-		dev_err(&pdev->dev, "removing device with active domains!\n");
+		dev_notice(&pdev->dev, "disabling translation\n");
 
 	arm_smmu_bus_init(NULL);
 	iommu_device_unregister(&smmu->iommu);
diff --git a/drivers/iommu/arm-smmu.h b/drivers/iommu/arm-smmu.h
index 8d1cd54d82a6..d172c024be61 100644
--- a/drivers/iommu/arm-smmu.h
+++ b/drivers/iommu/arm-smmu.h
@@ -386,6 +386,7 @@ struct arm_smmu_impl {
 	int (*init_context)(struct arm_smmu_domain *smmu_domain);
 	void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
 			 int status);
+	int (*def_domain_type)(struct device *dev);
 };
 
 static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index ba128d1cdaee..4959f5df21bd 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -952,7 +952,7 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
 
 	/* Non-coherent atomic allocation? Easy */
 	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
-	    dma_free_from_pool(cpu_addr, alloc_size))
+	    dma_free_from_pool(dev, cpu_addr, alloc_size))
 		return;
 
 	if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
@@ -1035,7 +1035,8 @@ static void *iommu_dma_alloc(struct device *dev, size_t size,
 
 	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
 	    !gfpflags_allow_blocking(gfp) && !coherent)
-		cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);
+		cpu_addr = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page,
+					       gfp);
 	else
 		cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
 	if (!cpu_addr)
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 186ff5cc975c..60c8a56e4a3f 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1235,19 +1235,13 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain,
 	return phys;
 }
 
-static int exynos_iommu_add_device(struct device *dev)
+static struct iommu_device *exynos_iommu_probe_device(struct device *dev)
 {
 	struct exynos_iommu_owner *owner = dev->archdata.iommu;
 	struct sysmmu_drvdata *data;
-	struct iommu_group *group;
 
 	if (!has_sysmmu(dev))
-		return -ENODEV;
-
-	group = iommu_group_get_for_dev(dev);
-
-	if (IS_ERR(group))
-		return PTR_ERR(group);
+		return ERR_PTR(-ENODEV);
 
 	list_for_each_entry(data, &owner->controllers, owner_node) {
 		/*
@@ -1259,12 +1253,15 @@ static int exynos_iommu_add_device(struct device *dev)
 					     DL_FLAG_STATELESS |
 					     DL_FLAG_PM_RUNTIME);
 	}
-	iommu_group_put(group);
 
-	return 0;
+	/* There is always at least one entry, see exynos_iommu_of_xlate() */
+	data = list_first_entry(&owner->controllers,
+				struct sysmmu_drvdata, owner_node);
+
+	return &data->iommu;
 }
 
-static void exynos_iommu_remove_device(struct device *dev)
+static void exynos_iommu_release_device(struct device *dev)
 {
 	struct exynos_iommu_owner *owner = dev->archdata.iommu;
 	struct sysmmu_drvdata *data;
@@ -1282,7 +1279,6 @@ static void exynos_iommu_remove_device(struct device *dev)
 			iommu_group_put(group);
 		}
 	}
-	iommu_group_remove_device(dev);
 
 	list_for_each_entry(data, &owner->controllers, owner_node)
 		device_link_del(data->link);
@@ -1331,8 +1327,8 @@ static const struct iommu_ops exynos_iommu_ops = {
 	.unmap = exynos_iommu_unmap,
 	.iova_to_phys = exynos_iommu_iova_to_phys,
 	.device_group = generic_device_group,
-	.add_device = exynos_iommu_add_device,
-	.remove_device = exynos_iommu_remove_device,
+	.probe_device = exynos_iommu_probe_device,
+	.release_device = exynos_iommu_release_device,
 	.pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
 	.of_xlate = exynos_iommu_of_xlate,
 };
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index 06828e2698d5..928d37771ece 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -1016,25 +1016,13 @@ static struct iommu_group *fsl_pamu_device_group(struct device *dev)
 	return group;
 }
 
-static int fsl_pamu_add_device(struct device *dev)
+static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
 {
-	struct iommu_group *group;
-
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	iommu_group_put(group);
-
-	iommu_device_link(&pamu_iommu, dev);
-
-	return 0;
+	return &pamu_iommu;
 }
 
-static void fsl_pamu_remove_device(struct device *dev)
+static void fsl_pamu_release_device(struct device *dev)
 {
-	iommu_device_unlink(&pamu_iommu, dev);
-	iommu_group_remove_device(dev);
 }
 
 static const struct iommu_ops fsl_pamu_ops = {
@@ -1048,8 +1036,8 @@ static const struct iommu_ops fsl_pamu_ops = {
 	.iova_to_phys	= fsl_pamu_iova_to_phys,
 	.domain_set_attr = fsl_pamu_set_domain_attr,
 	.domain_get_attr = fsl_pamu_get_domain_attr,
-	.add_device	= fsl_pamu_add_device,
-	.remove_device	= fsl_pamu_remove_device,
+	.probe_device	= fsl_pamu_probe_device,
+	.release_device	= fsl_pamu_release_device,
 	.device_group   = fsl_pamu_device_group,
 };
 
diff --git a/drivers/iommu/hyperv-iommu.c b/drivers/iommu/hyperv-iommu.c
index a386b83e0e34..3c0c67a99c7b 100644
--- a/drivers/iommu/hyperv-iommu.c
+++ b/drivers/iommu/hyperv-iommu.c
@@ -131,7 +131,7 @@ static int hyperv_irq_remapping_activate(struct irq_domain *domain,
 	return 0;
 }
 
-static struct irq_domain_ops hyperv_ir_domain_ops = {
+static const struct irq_domain_ops hyperv_ir_domain_ops = {
 	.alloc = hyperv_irq_remapping_alloc,
 	.free = hyperv_irq_remapping_free,
 	.activate = hyperv_irq_remapping_activate,
diff --git a/drivers/iommu/intel-iommu-debugfs.c b/drivers/iommu/intel/debugfs.c
index 3eb1fe240fb0..cf1ebb98e418 100644
--- a/drivers/iommu/intel-iommu-debugfs.c
+++ b/drivers/iommu/intel/debugfs.c
@@ -372,6 +372,66 @@ static int domain_translation_struct_show(struct seq_file *m, void *unused)
 }
 DEFINE_SHOW_ATTRIBUTE(domain_translation_struct);
 
+static void invalidation_queue_entry_show(struct seq_file *m,
+					  struct intel_iommu *iommu)
+{
+	int index, shift = qi_shift(iommu);
+	struct qi_desc *desc;
+	int offset;
+
+	if (ecap_smts(iommu->ecap))
+		seq_puts(m, "Index\t\tqw0\t\t\tqw1\t\t\tqw2\t\t\tqw3\t\t\tstatus\n");
+	else
+		seq_puts(m, "Index\t\tqw0\t\t\tqw1\t\t\tstatus\n");
+
+	for (index = 0; index < QI_LENGTH; index++) {
+		offset = index << shift;
+		desc = iommu->qi->desc + offset;
+		if (ecap_smts(iommu->ecap))
+			seq_printf(m, "%5d\t%016llx\t%016llx\t%016llx\t%016llx\t%016x\n",
+				   index, desc->qw0, desc->qw1,
+				   desc->qw2, desc->qw3,
+				   iommu->qi->desc_status[index]);
+		else
+			seq_printf(m, "%5d\t%016llx\t%016llx\t%016x\n",
+				   index, desc->qw0, desc->qw1,
+				   iommu->qi->desc_status[index]);
+	}
+}
+
+static int invalidation_queue_show(struct seq_file *m, void *unused)
+{
+	struct dmar_drhd_unit *drhd;
+	struct intel_iommu *iommu;
+	unsigned long flags;
+	struct q_inval *qi;
+	int shift;
+
+	rcu_read_lock();
+	for_each_active_iommu(iommu, drhd) {
+		qi = iommu->qi;
+		shift = qi_shift(iommu);
+
+		if (!qi || !ecap_qis(iommu->ecap))
+			continue;
+
+		seq_printf(m, "Invalidation queue on IOMMU: %s\n", iommu->name);
+
+		raw_spin_lock_irqsave(&qi->q_lock, flags);
+		seq_printf(m, " Base: 0x%llx\tHead: %lld\tTail: %lld\n",
+			   (u64)virt_to_phys(qi->desc),
+			   dmar_readq(iommu->reg + DMAR_IQH_REG) >> shift,
+			   dmar_readq(iommu->reg + DMAR_IQT_REG) >> shift);
+		invalidation_queue_entry_show(m, iommu);
+		raw_spin_unlock_irqrestore(&qi->q_lock, flags);
+		seq_putc(m, '\n');
+	}
+	rcu_read_unlock();
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(invalidation_queue);
+
 #ifdef CONFIG_IRQ_REMAP
 static void ir_tbl_remap_entry_show(struct seq_file *m,
 				    struct intel_iommu *iommu)
@@ -490,6 +550,8 @@ void __init intel_iommu_debugfs_init(void)
 	debugfs_create_file("domain_translation_struct", 0444,
 			    intel_iommu_debug, NULL,
 			    &domain_translation_struct_fops);
+	debugfs_create_file("invalidation_queue", 0444, intel_iommu_debug,
+			    NULL, &invalidation_queue_fops);
 #ifdef CONFIG_IRQ_REMAP
 	debugfs_create_file("ir_translation_struct", 0444, intel_iommu_debug,
 			    NULL, &ir_translation_struct_fops);
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/intel/dmar.c
index f77dae7ba7d4..cc46dff98fa0 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -32,7 +32,7 @@
 #include <asm/irq_remapping.h>
 #include <asm/iommu_table.h>
 
-#include "irq_remapping.h"
+#include "../irq_remapping.h"
 
 typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
 struct dmar_res_callback {
@@ -963,6 +963,7 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 		warn_invalid_dmar(phys_addr, " returns all ones");
 		goto unmap;
 	}
+	iommu->vccap = dmar_readq(iommu->reg + DMAR_VCCAP_REG);
 
 	/* the registers might be more than one page */
 	map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
@@ -1156,12 +1157,11 @@ static inline void reclaim_free_desc(struct q_inval *qi)
 	}
 }
 
-static int qi_check_fault(struct intel_iommu *iommu, int index)
+static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
 {
 	u32 fault;
 	int head, tail;
 	struct q_inval *qi = iommu->qi;
-	int wait_index = (index + 1) % QI_LENGTH;
 	int shift = qi_shift(iommu);
 
 	if (qi->desc_status[wait_index] == QI_ABORT)
@@ -1224,17 +1224,21 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
 }
 
 /*
- * Submit the queued invalidation descriptor to the remapping
- * hardware unit and wait for its completion.
+ * Function to submit invalidation descriptors of all types to the queued
+ * invalidation interface(QI). Multiple descriptors can be submitted at a
+ * time, a wait descriptor will be appended to each submission to ensure
+ * hardware has completed the invalidation before return. Wait descriptors
+ * can be part of the submission but it will not be polled for completion.
  */
-int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
+int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
+		   unsigned int count, unsigned long options)
 {
-	int rc;
 	struct q_inval *qi = iommu->qi;
-	int offset, shift, length;
 	struct qi_desc wait_desc;
 	int wait_index, index;
 	unsigned long flags;
+	int offset, shift;
+	int rc, i;
 
 	if (!qi)
 		return 0;
@@ -1243,32 +1247,41 @@ restart:
 	rc = 0;
 
 	raw_spin_lock_irqsave(&qi->q_lock, flags);
-	while (qi->free_cnt < 3) {
+	/*
+	 * Check if we have enough empty slots in the queue to submit,
+	 * the calculation is based on:
+	 * # of desc + 1 wait desc + 1 space between head and tail
+	 */
+	while (qi->free_cnt < count + 2) {
 		raw_spin_unlock_irqrestore(&qi->q_lock, flags);
 		cpu_relax();
 		raw_spin_lock_irqsave(&qi->q_lock, flags);
 	}
 
 	index = qi->free_head;
-	wait_index = (index + 1) % QI_LENGTH;
+	wait_index = (index + count) % QI_LENGTH;
 	shift = qi_shift(iommu);
-	length = 1 << shift;
 
-	qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
+	for (i = 0; i < count; i++) {
+		offset = ((index + i) % QI_LENGTH) << shift;
+		memcpy(qi->desc + offset, &desc[i], 1 << shift);
+		qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
+	}
+	qi->desc_status[wait_index] = QI_IN_USE;
 
-	offset = index << shift;
-	memcpy(qi->desc + offset, desc, length);
 	wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
 			QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
+	if (options & QI_OPT_WAIT_DRAIN)
+		wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
 	wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
 	wait_desc.qw2 = 0;
 	wait_desc.qw3 = 0;
 
 	offset = wait_index << shift;
-	memcpy(qi->desc + offset, &wait_desc, length);
+	memcpy(qi->desc + offset, &wait_desc, 1 << shift);
 
-	qi->free_head = (qi->free_head + 2) % QI_LENGTH;
-	qi->free_cnt -= 2;
+	qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
+	qi->free_cnt -= count + 1;
 
 	/*
 	 * update the HW tail register indicating the presence of
@@ -1284,7 +1297,7 @@ restart:
 		 * a deadlock where the interrupt context can wait indefinitely
 		 * for free slots in the queue.
 		 */
-		rc = qi_check_fault(iommu, index);
+		rc = qi_check_fault(iommu, index, wait_index);
 		if (rc)
 			break;
 
@@ -1293,7 +1306,8 @@ restart:
 		raw_spin_lock(&qi->q_lock);
 	}
 
-	qi->desc_status[index] = QI_DONE;
+	for (i = 0; i < count; i++)
+		qi->desc_status[(index + i) % QI_LENGTH] = QI_DONE;
 
 	reclaim_free_desc(qi);
 	raw_spin_unlock_irqrestore(&qi->q_lock, flags);
@@ -1317,7 +1331,7 @@ void qi_global_iec(struct intel_iommu *iommu)
 	desc.qw3 = 0;
 
 	/* should never fail */
-	qi_submit_sync(&desc, iommu);
+	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
@@ -1331,7 +1345,7 @@ void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 	desc.qw2 = 0;
 	desc.qw3 = 0;
 
-	qi_submit_sync(&desc, iommu);
+	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
@@ -1355,7 +1369,7 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 	desc.qw2 = 0;
 	desc.qw3 = 0;
 
-	qi_submit_sync(&desc, iommu);
+	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
@@ -1377,7 +1391,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
 	desc.qw2 = 0;
 	desc.qw3 = 0;
 
-	qi_submit_sync(&desc, iommu);
+	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 /* PASID-based IOTLB invalidation */
@@ -1418,7 +1432,46 @@ void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
 				QI_EIOTLB_AM(mask);
 	}
 
-	qi_submit_sync(&desc, iommu);
+	qi_submit_sync(iommu, &desc, 1, 0);
+}
+
+/* PASID-based device IOTLB Invalidate */
+void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
+			      u32 pasid,  u16 qdep, u64 addr,
+			      unsigned int size_order, u64 granu)
+{
+	unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
+	struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
+
+	desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
+		QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
+		QI_DEV_IOTLB_PFSID(pfsid);
+	desc.qw1 = QI_DEV_EIOTLB_GLOB(granu);
+
+	/*
+	 * If S bit is 0, we only flush a single page. If S bit is set,
+	 * The least significant zero bit indicates the invalidation address
+	 * range. VT-d spec 6.5.2.6.
+	 * e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
+	 * size order = 0 is PAGE_SIZE 4KB
+	 * Max Invs Pending (MIP) is set to 0 for now until we have DIT in
+	 * ECAP.
+	 */
+	desc.qw1 |= addr & ~mask;
+	if (size_order)
+		desc.qw1 |= QI_DEV_EIOTLB_SIZE;
+
+	qi_submit_sync(iommu, &desc, 1, 0);
+}
+
+void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
+			  u64 granu, int pasid)
+{
+	struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
+
+	desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
+			QI_PC_GRAN(granu) | QI_PC_TYPE;
+	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 /*
diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel/intel-pasid.h
index 92de6df24ccb..c5318d40e0fa 100644
--- a/drivers/iommu/intel-pasid.h
+++ b/drivers/iommu/intel/intel-pasid.h
@@ -15,6 +15,7 @@
 #define PASID_MAX			0x100000
 #define PASID_PTE_MASK			0x3F
 #define PASID_PTE_PRESENT		1
+#define PASID_PTE_FPD			2
 #define PDE_PFN_MASK			PAGE_MASK
 #define PASID_PDE_SHIFT			6
 #define MAX_NR_PASID_BITS		20
@@ -23,6 +24,16 @@
 #define is_pasid_enabled(entry)		(((entry)->lo >> 3) & 0x1)
 #define get_pasid_dir_size(entry)	(1 << ((((entry)->lo >> 9) & 0x7) + 7))
 
+/* Virtual command interface for enlightened pasid management. */
+#define VCMD_CMD_ALLOC			0x1
+#define VCMD_CMD_FREE			0x2
+#define VCMD_VRSP_IP			0x1
+#define VCMD_VRSP_SC(e)			(((e) >> 1) & 0x3)
+#define VCMD_VRSP_SC_SUCCESS		0
+#define VCMD_VRSP_SC_NO_PASID_AVAIL	1
+#define VCMD_VRSP_SC_INVALID_PASID	1
+#define VCMD_VRSP_RESULT_PASID(e)	(((e) >> 8) & 0xfffff)
+#define VCMD_CMD_OPERAND(e)		((e) << 8)
 /*
  * Domain ID reserved for pasid entries programmed for first-level
  * only and pass-through transfer modes.
@@ -36,6 +47,7 @@
  * to vmalloc or even module mappings.
  */
 #define PASID_FLAG_SUPERVISOR_MODE	BIT(0)
+#define PASID_FLAG_NESTED		BIT(1)
 
 /*
  * The PASID_FLAG_FL5LP flag Indicates using 5-level paging for first-
@@ -51,6 +63,11 @@ struct pasid_entry {
 	u64 val[8];
 };
 
+#define PASID_ENTRY_PGTT_FL_ONLY	(1)
+#define PASID_ENTRY_PGTT_SL_ONLY	(2)
+#define PASID_ENTRY_PGTT_NESTED		(3)
+#define PASID_ENTRY_PGTT_PT		(4)
+
 /* The representative of a PASID table */
 struct pasid_table {
 	void			*table;		/* pasid table pointer */
@@ -99,7 +116,13 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu,
 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
 				   struct dmar_domain *domain,
 				   struct device *dev, int pasid);
+int intel_pasid_setup_nested(struct intel_iommu *iommu,
+			     struct device *dev, pgd_t *pgd, int pasid,
+			     struct iommu_gpasid_bind_data_vtd *pasid_data,
+			     struct dmar_domain *domain, int addr_width);
 void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
-				 struct device *dev, int pasid);
-
+				 struct device *dev, int pasid,
+				 bool fault_ignore);
+int vcmd_alloc_pasid(struct intel_iommu *iommu, unsigned int *pasid);
+void vcmd_free_pasid(struct intel_iommu *iommu, unsigned int pasid);
 #endif /* __INTEL_PASID_H */
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel/iommu.c
index 0182cff2c7ac..9129663a7406 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -47,7 +47,7 @@
 #include <asm/iommu.h>
 #include <trace/events/intel_iommu.h>
 
-#include "irq_remapping.h"
+#include "../irq_remapping.h"
 #include "intel-pasid.h"
 
 #define ROOT_SIZE		VTD_PAGE_SIZE
@@ -296,31 +296,6 @@ static inline void context_clear_entry(struct context_entry *context)
 static struct dmar_domain *si_domain;
 static int hw_pass_through = 1;
 
-/* si_domain contains mulitple devices */
-#define DOMAIN_FLAG_STATIC_IDENTITY		BIT(0)
-
-/*
- * This is a DMA domain allocated through the iommu domain allocation
- * interface. But one or more devices belonging to this domain have
- * been chosen to use a private domain. We should avoid to use the
- * map/unmap/iova_to_phys APIs on it.
- */
-#define DOMAIN_FLAG_LOSE_CHILDREN		BIT(1)
-
-/*
- * When VT-d works in the scalable mode, it allows DMA translation to
- * happen through either first level or second level page table. This
- * bit marks that the DMA translation for the domain goes through the
- * first level page table, otherwise, it goes through the second level.
- */
-#define DOMAIN_FLAG_USE_FIRST_LEVEL		BIT(2)
-
-/*
- * Domain represents a virtual machine which demands iommu nested
- * translation mode support.
- */
-#define DOMAIN_FLAG_NESTING_MODE		BIT(3)
-
 #define for_each_domain_iommu(idx, domain)			\
 	for (idx = 0; idx < g_num_of_iommus; idx++)		\
 		if (domain->iommu_refcnt[idx])
@@ -355,11 +330,6 @@ static void domain_exit(struct dmar_domain *domain);
 static void domain_remove_dev_info(struct dmar_domain *domain);
 static void dmar_remove_one_dev_info(struct device *dev);
 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
-static void domain_context_clear(struct intel_iommu *iommu,
-				 struct device *dev);
-static int domain_detach_iommu(struct dmar_domain *domain,
-			       struct intel_iommu *iommu);
-static bool device_is_rmrr_locked(struct device *dev);
 static int intel_iommu_attach_device(struct iommu_domain *domain,
 				     struct device *dev);
 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
@@ -395,6 +365,21 @@ EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 
 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 #define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
+struct device_domain_info *get_domain_info(struct device *dev)
+{
+	struct device_domain_info *info;
+
+	if (!dev)
+		return NULL;
+
+	info = dev->archdata.iommu;
+	if (unlikely(info == DUMMY_DEVICE_DOMAIN_INFO ||
+		     info == DEFER_DEVICE_DOMAIN_INFO))
+		return NULL;
+
+	return info;
+}
+
 DEFINE_SPINLOCK(device_domain_lock);
 static LIST_HEAD(device_domain_list);
 
@@ -446,12 +431,6 @@ static void init_translation_status(struct intel_iommu *iommu)
 		iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 }
 
-/* Convert generic 'struct iommu_domain to private struct dmar_domain */
-static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
-{
-	return container_of(dom, struct dmar_domain, domain);
-}
-
 static int __init intel_iommu_setup(char *str)
 {
 	if (!str)
@@ -480,8 +459,7 @@ static int __init intel_iommu_setup(char *str)
 			pr_info("Intel-IOMMU: scalable mode supported\n");
 			intel_iommu_sm = 1;
 		} else if (!strncmp(str, "tboot_noforce", 13)) {
-			printk(KERN_INFO
-				"Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
+			pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 			intel_iommu_tboot_noforce = 1;
 		} else if (!strncmp(str, "nobounce", 8)) {
 			pr_info("Intel-IOMMU: No bounce buffer. This could expose security risks of DMA attacks\n");
@@ -1454,8 +1432,7 @@ static void iommu_enable_dev_iotlb(struct device_domain_info *info)
 	    !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
 		info->pri_enabled = 1;
 #endif
-	if (!pdev->untrusted && info->ats_supported &&
-	    pci_ats_page_aligned(pdev) &&
+	if (info->ats_supported && pci_ats_page_aligned(pdev) &&
 	    !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
 		info->ats_enabled = 1;
 		domain_update_iotlb(info->domain);
@@ -1763,6 +1740,9 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
 		if (ecap_prs(iommu->ecap))
 			intel_svm_finish_prq(iommu);
 	}
+	if (ecap_vcs(iommu->ecap) && vccap_pasid(iommu->vccap))
+		ioasid_unregister_allocator(&iommu->pasid_allocator);
+
 #endif
 }
 
@@ -1911,11 +1891,6 @@ static int dmar_init_reserved_ranges(void)
 	return 0;
 }
 
-static void domain_reserve_special_ranges(struct dmar_domain *domain)
-{
-	copy_reserved_iova(&reserved_iova_list, &domain->iovad);
-}
-
 static inline int guestwidth_to_adjustwidth(int gaw)
 {
 	int agaw;
@@ -1930,65 +1905,6 @@ static inline int guestwidth_to_adjustwidth(int gaw)
 	return agaw;
 }
 
-static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
-		       int guest_width)
-{
-	int adjust_width, agaw;
-	unsigned long sagaw;
-	int ret;
-
-	init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
-
-	if (!intel_iommu_strict) {
-		ret = init_iova_flush_queue(&domain->iovad,
-					    iommu_flush_iova, iova_entry_free);
-		if (ret)
-			pr_info("iova flush queue initialization failed\n");
-	}
-
-	domain_reserve_special_ranges(domain);
-
-	/* calculate AGAW */
-	if (guest_width > cap_mgaw(iommu->cap))
-		guest_width = cap_mgaw(iommu->cap);
-	domain->gaw = guest_width;
-	adjust_width = guestwidth_to_adjustwidth(guest_width);
-	agaw = width_to_agaw(adjust_width);
-	sagaw = cap_sagaw(iommu->cap);
-	if (!test_bit(agaw, &sagaw)) {
-		/* hardware doesn't support it, choose a bigger one */
-		pr_debug("Hardware doesn't support agaw %d\n", agaw);
-		agaw = find_next_bit(&sagaw, 5, agaw);
-		if (agaw >= 5)
-			return -ENODEV;
-	}
-	domain->agaw = agaw;
-
-	if (ecap_coherent(iommu->ecap))
-		domain->iommu_coherency = 1;
-	else
-		domain->iommu_coherency = 0;
-
-	if (ecap_sc_support(iommu->ecap))
-		domain->iommu_snooping = 1;
-	else
-		domain->iommu_snooping = 0;
-
-	if (intel_iommu_superpage)
-		domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
-	else
-		domain->iommu_superpage = 0;
-
-	domain->nid = iommu->node;
-
-	/* always allocate the top pgd */
-	domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
-	if (!domain->pgd)
-		return -ENOMEM;
-	__iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
-	return 0;
-}
-
 static void domain_exit(struct dmar_domain *domain)
 {
 
@@ -1996,7 +1912,8 @@ static void domain_exit(struct dmar_domain *domain)
 	domain_remove_dev_info(domain);
 
 	/* destroy iovas */
-	put_iova_domain(&domain->iovad);
+	if (domain->domain.type == IOMMU_DOMAIN_DMA)
+		put_iova_domain(&domain->iovad);
 
 	if (domain->pgd) {
 		struct page *freelist;
@@ -2518,11 +2435,8 @@ struct dmar_domain *find_domain(struct device *dev)
 	if (unlikely(attach_deferred(dev) || iommu_dummy(dev)))
 		return NULL;
 
-	if (dev_is_pci(dev))
-		dev = &pci_real_dma_dev(to_pci_dev(dev))->dev;
-
 	/* No lock here, assumes no domain exit in normal case */
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (likely(info))
 		return info->domain;
 
@@ -2545,7 +2459,7 @@ dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
 	struct device_domain_info *info;
 
 	list_for_each_entry(info, &device_domain_list, global)
-		if (info->iommu->segment == segment && info->bus == bus &&
+		if (info->segment == segment && info->bus == bus &&
 		    info->devfn == devfn)
 			return info;
 
@@ -2582,6 +2496,12 @@ static int domain_setup_first_level(struct intel_iommu *iommu,
 					     flags);
 }
 
+static bool dev_is_real_dma_subdevice(struct device *dev)
+{
+	return dev && dev_is_pci(dev) &&
+	       pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev);
+}
+
 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
 						    int bus, int devfn,
 						    struct device *dev,
@@ -2596,8 +2516,18 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
 	if (!info)
 		return NULL;
 
-	info->bus = bus;
-	info->devfn = devfn;
+	if (!dev_is_real_dma_subdevice(dev)) {
+		info->bus = bus;
+		info->devfn = devfn;
+		info->segment = iommu->segment;
+	} else {
+		struct pci_dev *pdev = to_pci_dev(dev);
+
+		info->bus = pdev->bus->number;
+		info->devfn = pdev->devfn;
+		info->segment = pci_domain_nr(pdev->bus);
+	}
+
 	info->ats_supported = info->pasid_supported = info->pri_supported = 0;
 	info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
 	info->ats_qdep = 0;
@@ -2611,10 +2541,8 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
 	if (dev && dev_is_pci(dev)) {
 		struct pci_dev *pdev = to_pci_dev(info->dev);
 
-		if (!pdev->untrusted &&
-		    !pci_ats_disabled() &&
-		    ecap_dev_iotlb_support(iommu->ecap) &&
-		    pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
+		if (ecap_dev_iotlb_support(iommu->ecap) &&
+		    pci_ats_supported(pdev) &&
 		    dmar_find_matched_atsr_unit(pdev))
 			info->ats_supported = 1;
 
@@ -2637,7 +2565,8 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
 
 	if (!found) {
 		struct device_domain_info *info2;
-		info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
+		info2 = dmar_search_domain_by_dev_info(info->segment, info->bus,
+						       info->devfn);
 		if (info2) {
 			found      = info2->domain;
 			info2->dev = dev;
@@ -2704,108 +2633,10 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
 	return domain;
 }
 
-static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
-{
-	*(u16 *)opaque = alias;
-	return 0;
-}
-
-static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
-{
-	struct device_domain_info *info;
-	struct dmar_domain *domain = NULL;
-	struct intel_iommu *iommu;
-	u16 dma_alias;
-	unsigned long flags;
-	u8 bus, devfn;
-
-	iommu = device_to_iommu(dev, &bus, &devfn);
-	if (!iommu)
-		return NULL;
-
-	if (dev_is_pci(dev)) {
-		struct pci_dev *pdev = to_pci_dev(dev);
-
-		pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
-
-		spin_lock_irqsave(&device_domain_lock, flags);
-		info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
-						      PCI_BUS_NUM(dma_alias),
-						      dma_alias & 0xff);
-		if (info) {
-			iommu = info->iommu;
-			domain = info->domain;
-		}
-		spin_unlock_irqrestore(&device_domain_lock, flags);
-
-		/* DMA alias already has a domain, use it */
-		if (info)
-			goto out;
-	}
-
-	/* Allocate and initialize new domain for the device */
-	domain = alloc_domain(0);
-	if (!domain)
-		return NULL;
-	if (domain_init(domain, iommu, gaw)) {
-		domain_exit(domain);
-		return NULL;
-	}
-
-out:
-	return domain;
-}
-
-static struct dmar_domain *set_domain_for_dev(struct device *dev,
-					      struct dmar_domain *domain)
-{
-	struct intel_iommu *iommu;
-	struct dmar_domain *tmp;
-	u16 req_id, dma_alias;
-	u8 bus, devfn;
-
-	iommu = device_to_iommu(dev, &bus, &devfn);
-	if (!iommu)
-		return NULL;
-
-	req_id = ((u16)bus << 8) | devfn;
-
-	if (dev_is_pci(dev)) {
-		struct pci_dev *pdev = to_pci_dev(dev);
-
-		pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
-
-		/* register PCI DMA alias device */
-		if (req_id != dma_alias) {
-			tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
-					dma_alias & 0xff, NULL, domain);
-
-			if (!tmp || tmp != domain)
-				return tmp;
-		}
-	}
-
-	tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
-	if (!tmp || tmp != domain)
-		return tmp;
-
-	return domain;
-}
-
 static int iommu_domain_identity_map(struct dmar_domain *domain,
-				     unsigned long long start,
-				     unsigned long long end)
+				     unsigned long first_vpfn,
+				     unsigned long last_vpfn)
 {
-	unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
-	unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
-
-	if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
-			  dma_to_mm_pfn(last_vpfn))) {
-		pr_err("Reserving iova failed\n");
-		return -ENOMEM;
-	}
-
-	pr_debug("Mapping reserved region %llx-%llx\n", start, end);
 	/*
 	 * RMRR range might have overlap with physical memory range,
 	 * clear it first
@@ -2817,45 +2648,6 @@ static int iommu_domain_identity_map(struct dmar_domain *domain,
 				DMA_PTE_READ|DMA_PTE_WRITE);
 }
 
-static int domain_prepare_identity_map(struct device *dev,
-				       struct dmar_domain *domain,
-				       unsigned long long start,
-				       unsigned long long end)
-{
-	/* For _hardware_ passthrough, don't bother. But for software
-	   passthrough, we do it anyway -- it may indicate a memory
-	   range which is reserved in E820, so which didn't get set
-	   up to start with in si_domain */
-	if (domain == si_domain && hw_pass_through) {
-		dev_warn(dev, "Ignoring identity map for HW passthrough [0x%Lx - 0x%Lx]\n",
-			 start, end);
-		return 0;
-	}
-
-	dev_info(dev, "Setting identity map [0x%Lx - 0x%Lx]\n", start, end);
-
-	if (end < start) {
-		WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
-			"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
-			dmi_get_system_info(DMI_BIOS_VENDOR),
-			dmi_get_system_info(DMI_BIOS_VERSION),
-		     dmi_get_system_info(DMI_PRODUCT_VERSION));
-		return -EIO;
-	}
-
-	if (end >> agaw_to_width(domain->agaw)) {
-		WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
-		     "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
-		     agaw_to_width(domain->agaw),
-		     dmi_get_system_info(DMI_BIOS_VENDOR),
-		     dmi_get_system_info(DMI_BIOS_VERSION),
-		     dmi_get_system_info(DMI_PRODUCT_VERSION));
-		return -EIO;
-	}
-
-	return iommu_domain_identity_map(domain, start, end);
-}
-
 static int md_domain_init(struct dmar_domain *domain, int guest_width);
 
 static int __init si_domain_init(int hw)
@@ -2882,7 +2674,8 @@ static int __init si_domain_init(int hw)
 
 		for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
 			ret = iommu_domain_identity_map(si_domain,
-					PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
+					mm_to_dma_pfn(start_pfn),
+					mm_to_dma_pfn(end_pfn));
 			if (ret)
 				return ret;
 		}
@@ -2911,17 +2704,6 @@ static int __init si_domain_init(int hw)
 	return 0;
 }
 
-static int identity_mapping(struct device *dev)
-{
-	struct device_domain_info *info;
-
-	info = dev->archdata.iommu;
-	if (info)
-		return (info->domain == si_domain);
-
-	return 0;
-}
-
 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
 {
 	struct dmar_domain *ndomain;
@@ -3048,31 +2830,6 @@ static int device_def_domain_type(struct device *dev)
 
 		if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
 			return IOMMU_DOMAIN_IDENTITY;
-
-		/*
-		 * We want to start off with all devices in the 1:1 domain, and
-		 * take them out later if we find they can't access all of memory.
-		 *
-		 * However, we can't do this for PCI devices behind bridges,
-		 * because all PCI devices behind the same bridge will end up
-		 * with the same source-id on their transactions.
-		 *
-		 * Practically speaking, we can't change things around for these
-		 * devices at run-time, because we can't be sure there'll be no
-		 * DMA transactions in flight for any of their siblings.
-		 *
-		 * So PCI devices (unless they're on the root bus) as well as
-		 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
-		 * the 1:1 domain, just in _case_ one of their siblings turns out
-		 * not to be able to map all of memory.
-		 */
-		if (!pci_is_pcie(pdev)) {
-			if (!pci_is_root_bus(pdev->bus))
-				return IOMMU_DOMAIN_DMA;
-			if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
-				return IOMMU_DOMAIN_DMA;
-		} else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
-			return IOMMU_DOMAIN_DMA;
 	}
 
 	return 0;
@@ -3297,6 +3054,85 @@ out_unmap:
 	return ret;
 }
 
+#ifdef CONFIG_INTEL_IOMMU_SVM
+static ioasid_t intel_vcmd_ioasid_alloc(ioasid_t min, ioasid_t max, void *data)
+{
+	struct intel_iommu *iommu = data;
+	ioasid_t ioasid;
+
+	if (!iommu)
+		return INVALID_IOASID;
+	/*
+	 * VT-d virtual command interface always uses the full 20 bit
+	 * PASID range. Host can partition guest PASID range based on
+	 * policies but it is out of guest's control.
+	 */
+	if (min < PASID_MIN || max > intel_pasid_max_id)
+		return INVALID_IOASID;
+
+	if (vcmd_alloc_pasid(iommu, &ioasid))
+		return INVALID_IOASID;
+
+	return ioasid;
+}
+
+static void intel_vcmd_ioasid_free(ioasid_t ioasid, void *data)
+{
+	struct intel_iommu *iommu = data;
+
+	if (!iommu)
+		return;
+	/*
+	 * Sanity check the ioasid owner is done at upper layer, e.g. VFIO
+	 * We can only free the PASID when all the devices are unbound.
+	 */
+	if (ioasid_find(NULL, ioasid, NULL)) {
+		pr_alert("Cannot free active IOASID %d\n", ioasid);
+		return;
+	}
+	vcmd_free_pasid(iommu, ioasid);
+}
+
+static void register_pasid_allocator(struct intel_iommu *iommu)
+{
+	/*
+	 * If we are running in the host, no need for custom allocator
+	 * in that PASIDs are allocated from the host system-wide.
+	 */
+	if (!cap_caching_mode(iommu->cap))
+		return;
+
+	if (!sm_supported(iommu)) {
+		pr_warn("VT-d Scalable Mode not enabled, no PASID allocation\n");
+		return;
+	}
+
+	/*
+	 * Register a custom PASID allocator if we are running in a guest,
+	 * guest PASID must be obtained via virtual command interface.
+	 * There can be multiple vIOMMUs in each guest but only one allocator
+	 * is active. All vIOMMU allocators will eventually be calling the same
+	 * host allocator.
+	 */
+	if (!ecap_vcs(iommu->ecap) || !vccap_pasid(iommu->vccap))
+		return;
+
+	pr_info("Register custom PASID allocator\n");
+	iommu->pasid_allocator.alloc = intel_vcmd_ioasid_alloc;
+	iommu->pasid_allocator.free = intel_vcmd_ioasid_free;
+	iommu->pasid_allocator.pdata = (void *)iommu;
+	if (ioasid_register_allocator(&iommu->pasid_allocator)) {
+		pr_warn("Custom PASID allocator failed, scalable mode disabled\n");
+		/*
+		 * Disable scalable mode on this IOMMU if there
+		 * is no custom allocator. Mixing SM capable vIOMMU
+		 * and non-SM vIOMMU are not supported.
+		 */
+		intel_iommu_sm = 0;
+	}
+}
+#endif
+
 static int __init init_dmars(void)
 {
 	struct dmar_drhd_unit *drhd;
@@ -3414,6 +3250,9 @@ static int __init init_dmars(void)
 	 */
 	for_each_active_iommu(iommu, drhd) {
 		iommu_flush_write_buffer(iommu);
+#ifdef CONFIG_INTEL_IOMMU_SVM
+		register_pasid_allocator(iommu);
+#endif
 		iommu_set_root_entry(iommu);
 		iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
 		iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
@@ -3531,100 +3370,6 @@ static unsigned long intel_alloc_iova(struct device *dev,
 	return iova_pfn;
 }
 
-static struct dmar_domain *get_private_domain_for_dev(struct device *dev)
-{
-	struct dmar_domain *domain, *tmp;
-	struct dmar_rmrr_unit *rmrr;
-	struct device *i_dev;
-	int i, ret;
-
-	/* Device shouldn't be attached by any domains. */
-	domain = find_domain(dev);
-	if (domain)
-		return NULL;
-
-	domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
-	if (!domain)
-		goto out;
-
-	/* We have a new domain - setup possible RMRRs for the device */
-	rcu_read_lock();
-	for_each_rmrr_units(rmrr) {
-		for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
-					  i, i_dev) {
-			if (i_dev != dev)
-				continue;
-
-			ret = domain_prepare_identity_map(dev, domain,
-							  rmrr->base_address,
-							  rmrr->end_address);
-			if (ret)
-				dev_err(dev, "Mapping reserved region failed\n");
-		}
-	}
-	rcu_read_unlock();
-
-	tmp = set_domain_for_dev(dev, domain);
-	if (!tmp || domain != tmp) {
-		domain_exit(domain);
-		domain = tmp;
-	}
-
-out:
-	if (!domain)
-		dev_err(dev, "Allocating domain failed\n");
-	else
-		domain->domain.type = IOMMU_DOMAIN_DMA;
-
-	return domain;
-}
-
-/* Check if the dev needs to go through non-identity map and unmap process.*/
-static bool iommu_need_mapping(struct device *dev)
-{
-	int ret;
-
-	if (iommu_dummy(dev))
-		return false;
-
-	if (unlikely(attach_deferred(dev)))
-		do_deferred_attach(dev);
-
-	ret = identity_mapping(dev);
-	if (ret) {
-		u64 dma_mask = *dev->dma_mask;
-
-		if (dev->coherent_dma_mask && dev->coherent_dma_mask < dma_mask)
-			dma_mask = dev->coherent_dma_mask;
-
-		if (dma_mask >= dma_direct_get_required_mask(dev))
-			return false;
-
-		/*
-		 * 32 bit DMA is removed from si_domain and fall back to
-		 * non-identity mapping.
-		 */
-		dmar_remove_one_dev_info(dev);
-		ret = iommu_request_dma_domain_for_dev(dev);
-		if (ret) {
-			struct iommu_domain *domain;
-			struct dmar_domain *dmar_domain;
-
-			domain = iommu_get_domain_for_dev(dev);
-			if (domain) {
-				dmar_domain = to_dmar_domain(domain);
-				dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;
-			}
-			dmar_remove_one_dev_info(dev);
-			get_private_domain_for_dev(dev);
-		}
-
-		dev_info(dev, "32bit DMA uses non-identity mapping\n");
-	}
-
-	return true;
-}
-
 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
 				     size_t size, int dir, u64 dma_mask)
 {
@@ -3638,6 +3383,9 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
 
 	BUG_ON(dir == DMA_NONE);
 
+	if (unlikely(attach_deferred(dev)))
+		do_deferred_attach(dev);
+
 	domain = find_domain(dev);
 	if (!domain)
 		return DMA_MAPPING_ERROR;
@@ -3689,20 +3437,15 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
 				 enum dma_data_direction dir,
 				 unsigned long attrs)
 {
-	if (iommu_need_mapping(dev))
-		return __intel_map_single(dev, page_to_phys(page) + offset,
-				size, dir, *dev->dma_mask);
-	return dma_direct_map_page(dev, page, offset, size, dir, attrs);
+	return __intel_map_single(dev, page_to_phys(page) + offset,
+				  size, dir, *dev->dma_mask);
 }
 
 static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
 				     size_t size, enum dma_data_direction dir,
 				     unsigned long attrs)
 {
-	if (iommu_need_mapping(dev))
-		return __intel_map_single(dev, phys_addr, size, dir,
-				*dev->dma_mask);
-	return dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
+	return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask);
 }
 
 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
@@ -3753,17 +3496,13 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
 			     size_t size, enum dma_data_direction dir,
 			     unsigned long attrs)
 {
-	if (iommu_need_mapping(dev))
-		intel_unmap(dev, dev_addr, size);
-	else
-		dma_direct_unmap_page(dev, dev_addr, size, dir, attrs);
+	intel_unmap(dev, dev_addr, size);
 }
 
 static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
 		size_t size, enum dma_data_direction dir, unsigned long attrs)
 {
-	if (iommu_need_mapping(dev))
-		intel_unmap(dev, dev_addr, size);
+	intel_unmap(dev, dev_addr, size);
 }
 
 static void *intel_alloc_coherent(struct device *dev, size_t size,
@@ -3773,8 +3512,8 @@ static void *intel_alloc_coherent(struct device *dev, size_t size,
 	struct page *page = NULL;
 	int order;
 
-	if (!iommu_need_mapping(dev))
-		return dma_direct_alloc(dev, size, dma_handle, flags, attrs);
+	if (unlikely(attach_deferred(dev)))
+		do_deferred_attach(dev);
 
 	size = PAGE_ALIGN(size);
 	order = get_order(size);
@@ -3809,9 +3548,6 @@ static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
 	int order;
 	struct page *page = virt_to_page(vaddr);
 
-	if (!iommu_need_mapping(dev))
-		return dma_direct_free(dev, size, vaddr, dma_handle, attrs);
-
 	size = PAGE_ALIGN(size);
 	order = get_order(size);
 
@@ -3829,9 +3565,6 @@ static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
 	struct scatterlist *sg;
 	int i;
 
-	if (!iommu_need_mapping(dev))
-		return dma_direct_unmap_sg(dev, sglist, nelems, dir, attrs);
-
 	for_each_sg(sglist, sg, nelems, i) {
 		nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
 	}
@@ -3855,8 +3588,9 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
 	struct intel_iommu *iommu;
 
 	BUG_ON(dir == DMA_NONE);
-	if (!iommu_need_mapping(dev))
-		return dma_direct_map_sg(dev, sglist, nelems, dir, attrs);
+
+	if (unlikely(attach_deferred(dev)))
+		do_deferred_attach(dev);
 
 	domain = find_domain(dev);
 	if (!domain)
@@ -3903,8 +3637,6 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
 
 static u64 intel_get_required_mask(struct device *dev)
 {
-	if (!iommu_need_mapping(dev))
-		return dma_direct_get_required_mask(dev);
 	return DMA_BIT_MASK(32);
 }
 
@@ -4813,58 +4545,37 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
 				       unsigned long val, void *v)
 {
 	struct memory_notify *mhp = v;
-	unsigned long long start, end;
-	unsigned long start_vpfn, last_vpfn;
+	unsigned long start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
+	unsigned long last_vpfn = mm_to_dma_pfn(mhp->start_pfn +
+			mhp->nr_pages - 1);
 
 	switch (val) {
 	case MEM_GOING_ONLINE:
-		start = mhp->start_pfn << PAGE_SHIFT;
-		end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
-		if (iommu_domain_identity_map(si_domain, start, end)) {
-			pr_warn("Failed to build identity map for [%llx-%llx]\n",
-				start, end);
+		if (iommu_domain_identity_map(si_domain,
+					      start_vpfn, last_vpfn)) {
+			pr_warn("Failed to build identity map for [%lx-%lx]\n",
+				start_vpfn, last_vpfn);
 			return NOTIFY_BAD;
 		}
 		break;
 
 	case MEM_OFFLINE:
 	case MEM_CANCEL_ONLINE:
-		start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
-		last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
-		while (start_vpfn <= last_vpfn) {
-			struct iova *iova;
+		{
 			struct dmar_drhd_unit *drhd;
 			struct intel_iommu *iommu;
 			struct page *freelist;
 
-			iova = find_iova(&si_domain->iovad, start_vpfn);
-			if (iova == NULL) {
-				pr_debug("Failed get IOVA for PFN %lx\n",
-					 start_vpfn);
-				break;
-			}
-
-			iova = split_and_remove_iova(&si_domain->iovad, iova,
-						     start_vpfn, last_vpfn);
-			if (iova == NULL) {
-				pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
-					start_vpfn, last_vpfn);
-				return NOTIFY_BAD;
-			}
-
-			freelist = domain_unmap(si_domain, iova->pfn_lo,
-					       iova->pfn_hi);
+			freelist = domain_unmap(si_domain,
+						start_vpfn, last_vpfn);
 
 			rcu_read_lock();
 			for_each_active_iommu(iommu, drhd)
 				iommu_flush_iotlb_psi(iommu, si_domain,
-					iova->pfn_lo, iova_size(iova),
+					start_vpfn, mhp->nr_pages,
 					!freelist, 0);
 			rcu_read_unlock();
 			dma_free_pagelist(freelist);
-
-			start_vpfn = iova->pfn_hi + 1;
-			free_iova_mem(iova);
 		}
 		break;
 	}
@@ -4892,8 +4603,9 @@ static void free_all_cpu_cached_iovas(unsigned int cpu)
 		for (did = 0; did < cap_ndoms(iommu->cap); did++) {
 			domain = get_iommu_domain(iommu, (u16)did);
 
-			if (!domain)
+			if (!domain || domain->domain.type != IOMMU_DOMAIN_DMA)
 				continue;
+
 			free_cpu_cached_iovas(cpu, &domain->iovad);
 		}
 	}
@@ -5186,18 +4898,6 @@ int __init intel_iommu_init(void)
 	}
 	up_write(&dmar_global_lock);
 
-#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
-	/*
-	 * If the system has no untrusted device or the user has decided
-	 * to disable the bounce page mechanisms, we don't need swiotlb.
-	 * Mark this and the pre-allocated bounce pages will be released
-	 * later.
-	 */
-	if (!has_untrusted_dev() || intel_no_bounce)
-		swiotlb = 0;
-#endif
-	dma_ops = &intel_dma_ops;
-
 	init_iommu_pm_ops();
 
 	down_read(&dmar_global_lock);
@@ -5283,10 +4983,11 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info)
 	if (info->dev) {
 		if (dev_is_pci(info->dev) && sm_supported(iommu))
 			intel_pasid_tear_down_entry(iommu, info->dev,
-					PASID_RID2PASID);
+					PASID_RID2PASID, false);
 
 		iommu_disable_dev_iotlb(info);
-		domain_context_clear(iommu, info->dev);
+		if (!dev_is_real_dma_subdevice(info->dev))
+			domain_context_clear(iommu, info->dev);
 		intel_pasid_free_table(info->dev);
 	}
 
@@ -5296,12 +4997,6 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info)
 	domain_detach_iommu(domain, iommu);
 	spin_unlock_irqrestore(&iommu->lock, flags);
 
-	/* free the private domain */
-	if (domain->flags & DOMAIN_FLAG_LOSE_CHILDREN &&
-	    !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
-	    list_empty(&domain->devices))
-		domain_exit(info->domain);
-
 	free_devinfo_mem(info);
 }
 
@@ -5311,9 +5006,8 @@ static void dmar_remove_one_dev_info(struct device *dev)
 	unsigned long flags;
 
 	spin_lock_irqsave(&device_domain_lock, flags);
-	info = dev->archdata.iommu;
-	if (info && info != DEFER_DEVICE_DOMAIN_INFO
-	    && info != DUMMY_DEVICE_DOMAIN_INFO)
+	info = get_domain_info(dev);
+	if (info)
 		__dmar_remove_one_dev_info(info);
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 }
@@ -5322,9 +5016,6 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
 {
 	int adjust_width;
 
-	init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
-	domain_reserve_special_ranges(domain);
-
 	/* calculate AGAW */
 	domain->gaw = guest_width;
 	adjust_width = guestwidth_to_adjustwidth(guest_width);
@@ -5343,11 +5034,21 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
 	return 0;
 }
 
+static void intel_init_iova_domain(struct dmar_domain *dmar_domain)
+{
+	init_iova_domain(&dmar_domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
+	copy_reserved_iova(&reserved_iova_list, &dmar_domain->iovad);
+
+	if (!intel_iommu_strict &&
+	    init_iova_flush_queue(&dmar_domain->iovad,
+				  iommu_flush_iova, iova_entry_free))
+		pr_info("iova flush queue initialization failed\n");
+}
+
 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
 {
 	struct dmar_domain *dmar_domain;
 	struct iommu_domain *domain;
-	int ret;
 
 	switch (type) {
 	case IOMMU_DOMAIN_DMA:
@@ -5364,13 +5065,8 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
 			return NULL;
 		}
 
-		if (!intel_iommu_strict && type == IOMMU_DOMAIN_DMA) {
-			ret = init_iova_flush_queue(&dmar_domain->iovad,
-						    iommu_flush_iova,
-						    iova_entry_free);
-			if (ret)
-				pr_info("iova flush queue initialization failed\n");
-		}
+		if (type == IOMMU_DOMAIN_DMA)
+			intel_init_iova_domain(dmar_domain);
 
 		domain_update_iommu_cap(dmar_domain);
 
@@ -5403,7 +5099,7 @@ static void intel_iommu_domain_free(struct iommu_domain *domain)
 static inline bool
 is_aux_domain(struct device *dev, struct iommu_domain *domain)
 {
-	struct device_domain_info *info = dev->archdata.iommu;
+	struct device_domain_info *info = get_domain_info(dev);
 
 	return info && info->auxd_enabled &&
 			domain->type == IOMMU_DOMAIN_UNMANAGED;
@@ -5412,7 +5108,7 @@ is_aux_domain(struct device *dev, struct iommu_domain *domain)
 static void auxiliary_link_device(struct dmar_domain *domain,
 				  struct device *dev)
 {
-	struct device_domain_info *info = dev->archdata.iommu;
+	struct device_domain_info *info = get_domain_info(dev);
 
 	assert_spin_locked(&device_domain_lock);
 	if (WARN_ON(!info))
@@ -5425,7 +5121,7 @@ static void auxiliary_link_device(struct dmar_domain *domain,
 static void auxiliary_unlink_device(struct dmar_domain *domain,
 				    struct device *dev)
 {
-	struct device_domain_info *info = dev->archdata.iommu;
+	struct device_domain_info *info = get_domain_info(dev);
 
 	assert_spin_locked(&device_domain_lock);
 	if (WARN_ON(!info))
@@ -5513,13 +5209,13 @@ static void aux_domain_remove_dev(struct dmar_domain *domain,
 		return;
 
 	spin_lock_irqsave(&device_domain_lock, flags);
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	iommu = info->iommu;
 
 	auxiliary_unlink_device(domain, dev);
 
 	spin_lock(&iommu->lock);
-	intel_pasid_tear_down_entry(iommu, dev, domain->default_pasid);
+	intel_pasid_tear_down_entry(iommu, dev, domain->default_pasid, false);
 	domain_detach_iommu(domain, iommu);
 	spin_unlock(&iommu->lock);
 
@@ -5626,6 +5322,176 @@ static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
 	aux_domain_remove_dev(to_dmar_domain(domain), dev);
 }
 
+/*
+ * 2D array for converting and sanitizing IOMMU generic TLB granularity to
+ * VT-d granularity. Invalidation is typically included in the unmap operation
+ * as a result of DMA or VFIO unmap. However, for assigned devices guest
+ * owns the first level page tables. Invalidations of translation caches in the
+ * guest are trapped and passed down to the host.
+ *
+ * vIOMMU in the guest will only expose first level page tables, therefore
+ * we do not support IOTLB granularity for request without PASID (second level).
+ *
+ * For example, to find the VT-d granularity encoding for IOTLB
+ * type and page selective granularity within PASID:
+ * X: indexed by iommu cache type
+ * Y: indexed by enum iommu_inv_granularity
+ * [IOMMU_CACHE_INV_TYPE_IOTLB][IOMMU_INV_GRANU_ADDR]
+ */
+
+static const int
+inv_type_granu_table[IOMMU_CACHE_INV_TYPE_NR][IOMMU_INV_GRANU_NR] = {
+	/*
+	 * PASID based IOTLB invalidation: PASID selective (per PASID),
+	 * page selective (address granularity)
+	 */
+	{-EINVAL, QI_GRAN_NONG_PASID, QI_GRAN_PSI_PASID},
+	/* PASID based dev TLBs */
+	{-EINVAL, -EINVAL, QI_DEV_IOTLB_GRAN_PASID_SEL},
+	/* PASID cache */
+	{-EINVAL, -EINVAL, -EINVAL}
+};
+
+static inline int to_vtd_granularity(int type, int granu)
+{
+	return inv_type_granu_table[type][granu];
+}
+
+static inline u64 to_vtd_size(u64 granu_size, u64 nr_granules)
+{
+	u64 nr_pages = (granu_size * nr_granules) >> VTD_PAGE_SHIFT;
+
+	/* VT-d size is encoded as 2^size of 4K pages, 0 for 4k, 9 for 2MB, etc.
+	 * IOMMU cache invalidate API passes granu_size in bytes, and number of
+	 * granu size in contiguous memory.
+	 */
+	return order_base_2(nr_pages);
+}
+
+#ifdef CONFIG_INTEL_IOMMU_SVM
+static int
+intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev,
+			   struct iommu_cache_invalidate_info *inv_info)
+{
+	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
+	struct device_domain_info *info;
+	struct intel_iommu *iommu;
+	unsigned long flags;
+	int cache_type;
+	u8 bus, devfn;
+	u16 did, sid;
+	int ret = 0;
+	u64 size = 0;
+
+	if (!inv_info || !dmar_domain ||
+	    inv_info->version != IOMMU_CACHE_INVALIDATE_INFO_VERSION_1)
+		return -EINVAL;
+
+	if (!dev || !dev_is_pci(dev))
+		return -ENODEV;
+
+	iommu = device_to_iommu(dev, &bus, &devfn);
+	if (!iommu)
+		return -ENODEV;
+
+	if (!(dmar_domain->flags & DOMAIN_FLAG_NESTING_MODE))
+		return -EINVAL;
+
+	spin_lock_irqsave(&device_domain_lock, flags);
+	spin_lock(&iommu->lock);
+	info = get_domain_info(dev);
+	if (!info) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+	did = dmar_domain->iommu_did[iommu->seq_id];
+	sid = PCI_DEVID(bus, devfn);
+
+	/* Size is only valid in address selective invalidation */
+	if (inv_info->granularity != IOMMU_INV_GRANU_PASID)
+		size = to_vtd_size(inv_info->addr_info.granule_size,
+				   inv_info->addr_info.nb_granules);
+
+	for_each_set_bit(cache_type,
+			 (unsigned long *)&inv_info->cache,
+			 IOMMU_CACHE_INV_TYPE_NR) {
+		int granu = 0;
+		u64 pasid = 0;
+
+		granu = to_vtd_granularity(cache_type, inv_info->granularity);
+		if (granu == -EINVAL) {
+			pr_err_ratelimited("Invalid cache type and granu combination %d/%d\n",
+					   cache_type, inv_info->granularity);
+			break;
+		}
+
+		/*
+		 * PASID is stored in different locations based on the
+		 * granularity.
+		 */
+		if (inv_info->granularity == IOMMU_INV_GRANU_PASID &&
+		    (inv_info->pasid_info.flags & IOMMU_INV_PASID_FLAGS_PASID))
+			pasid = inv_info->pasid_info.pasid;
+		else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
+			 (inv_info->addr_info.flags & IOMMU_INV_ADDR_FLAGS_PASID))
+			pasid = inv_info->addr_info.pasid;
+
+		switch (BIT(cache_type)) {
+		case IOMMU_CACHE_INV_TYPE_IOTLB:
+			if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
+			    size &&
+			    (inv_info->addr_info.addr & ((BIT(VTD_PAGE_SHIFT + size)) - 1))) {
+				pr_err_ratelimited("Address out of range, 0x%llx, size order %llu\n",
+						   inv_info->addr_info.addr, size);
+				ret = -ERANGE;
+				goto out_unlock;
+			}
+
+			/*
+			 * If granu is PASID-selective, address is ignored.
+			 * We use npages = -1 to indicate that.
+			 */
+			qi_flush_piotlb(iommu, did, pasid,
+					mm_to_dma_pfn(inv_info->addr_info.addr),
+					(granu == QI_GRAN_NONG_PASID) ? -1 : 1 << size,
+					inv_info->addr_info.flags & IOMMU_INV_ADDR_FLAGS_LEAF);
+
+			/*
+			 * Always flush device IOTLB if ATS is enabled. vIOMMU
+			 * in the guest may assume IOTLB flush is inclusive,
+			 * which is more efficient.
+			 */
+			if (info->ats_enabled)
+				qi_flush_dev_iotlb_pasid(iommu, sid,
+						info->pfsid, pasid,
+						info->ats_qdep,
+						inv_info->addr_info.addr,
+						size, granu);
+			break;
+		case IOMMU_CACHE_INV_TYPE_DEV_IOTLB:
+			if (info->ats_enabled)
+				qi_flush_dev_iotlb_pasid(iommu, sid,
+						info->pfsid, pasid,
+						info->ats_qdep,
+						inv_info->addr_info.addr,
+						size, granu);
+			else
+				pr_warn_ratelimited("Passdown device IOTLB flush w/o ATS!\n");
+			break;
+		default:
+			dev_err_ratelimited(dev, "Unsupported IOMMU invalidation type %d\n",
+					    cache_type);
+			ret = -EINVAL;
+		}
+	}
+out_unlock:
+	spin_unlock(&iommu->lock);
+	spin_unlock_irqrestore(&device_domain_lock, flags);
+
+	return ret;
+}
+#endif
+
 static int intel_iommu_map(struct iommu_domain *domain,
 			   unsigned long iova, phys_addr_t hpa,
 			   size_t size, int iommu_prot, gfp_t gfp)
@@ -5781,78 +5647,22 @@ static bool intel_iommu_capable(enum iommu_cap cap)
 	return false;
 }
 
-static int intel_iommu_add_device(struct device *dev)
+static struct iommu_device *intel_iommu_probe_device(struct device *dev)
 {
-	struct dmar_domain *dmar_domain;
-	struct iommu_domain *domain;
 	struct intel_iommu *iommu;
-	struct iommu_group *group;
 	u8 bus, devfn;
-	int ret;
 
 	iommu = device_to_iommu(dev, &bus, &devfn);
 	if (!iommu)
-		return -ENODEV;
-
-	iommu_device_link(&iommu->iommu, dev);
+		return ERR_PTR(-ENODEV);
 
 	if (translation_pre_enabled(iommu))
 		dev->archdata.iommu = DEFER_DEVICE_DOMAIN_INFO;
 
-	group = iommu_group_get_for_dev(dev);
-
-	if (IS_ERR(group)) {
-		ret = PTR_ERR(group);
-		goto unlink;
-	}
-
-	iommu_group_put(group);
-
-	domain = iommu_get_domain_for_dev(dev);
-	dmar_domain = to_dmar_domain(domain);
-	if (domain->type == IOMMU_DOMAIN_DMA) {
-		if (device_def_domain_type(dev) == IOMMU_DOMAIN_IDENTITY) {
-			ret = iommu_request_dm_for_dev(dev);
-			if (ret) {
-				dmar_remove_one_dev_info(dev);
-				dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;
-				domain_add_dev_info(si_domain, dev);
-				dev_info(dev,
-					 "Device uses a private identity domain.\n");
-			}
-		}
-	} else {
-		if (device_def_domain_type(dev) == IOMMU_DOMAIN_DMA) {
-			ret = iommu_request_dma_domain_for_dev(dev);
-			if (ret) {
-				dmar_remove_one_dev_info(dev);
-				dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;
-				if (!get_private_domain_for_dev(dev)) {
-					dev_warn(dev,
-						 "Failed to get a private domain.\n");
-					ret = -ENOMEM;
-					goto unlink;
-				}
-
-				dev_info(dev,
-					 "Device uses a private dma domain.\n");
-			}
-		}
-	}
-
-	if (device_needs_bounce(dev)) {
-		dev_info(dev, "Use Intel IOMMU bounce page dma_ops\n");
-		set_dma_ops(dev, &bounce_dma_ops);
-	}
-
-	return 0;
-
-unlink:
-	iommu_device_unlink(&iommu->iommu, dev);
-	return ret;
+	return &iommu->iommu;
 }
 
-static void intel_iommu_remove_device(struct device *dev)
+static void intel_iommu_release_device(struct device *dev)
 {
 	struct intel_iommu *iommu;
 	u8 bus, devfn;
@@ -5863,11 +5673,19 @@ static void intel_iommu_remove_device(struct device *dev)
 
 	dmar_remove_one_dev_info(dev);
 
-	iommu_group_remove_device(dev);
+	set_dma_ops(dev, NULL);
+}
 
-	iommu_device_unlink(&iommu->iommu, dev);
+static void intel_iommu_probe_finalize(struct device *dev)
+{
+	struct iommu_domain *domain;
 
+	domain = iommu_get_domain_for_dev(dev);
 	if (device_needs_bounce(dev))
+		set_dma_ops(dev, &bounce_dma_ops);
+	else if (domain && domain->type == IOMMU_DOMAIN_DMA)
+		set_dma_ops(dev, &intel_dma_ops);
+	else
 		set_dma_ops(dev, NULL);
 }
 
@@ -5945,7 +5763,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
 	spin_lock(&iommu->lock);
 
 	ret = -EINVAL;
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (!info || !info->pasid_supported)
 		goto out;
 
@@ -6041,7 +5859,7 @@ static int intel_iommu_enable_auxd(struct device *dev)
 		return -ENODEV;
 
 	spin_lock_irqsave(&device_domain_lock, flags);
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	info->auxd_enabled = 1;
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 
@@ -6054,7 +5872,7 @@ static int intel_iommu_disable_auxd(struct device *dev)
 	unsigned long flags;
 
 	spin_lock_irqsave(&device_domain_lock, flags);
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (!WARN_ON(!info))
 		info->auxd_enabled = 0;
 	spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -6107,6 +5925,14 @@ intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat)
 		return !!siov_find_pci_dvsec(to_pci_dev(dev));
 	}
 
+	if (feat == IOMMU_DEV_FEAT_SVA) {
+		struct device_domain_info *info = get_domain_info(dev);
+
+		return info && (info->iommu->flags & VTD_FLAG_SVM_CAPABLE) &&
+			info->pasid_supported && info->pri_supported &&
+			info->ats_supported;
+	}
+
 	return false;
 }
 
@@ -6116,6 +5942,16 @@ intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
 	if (feat == IOMMU_DEV_FEAT_AUX)
 		return intel_iommu_enable_auxd(dev);
 
+	if (feat == IOMMU_DEV_FEAT_SVA) {
+		struct device_domain_info *info = get_domain_info(dev);
+
+		if (!info)
+			return -EINVAL;
+
+		if (info->iommu->flags & VTD_FLAG_SVM_CAPABLE)
+			return 0;
+	}
+
 	return -ENODEV;
 }
 
@@ -6131,7 +5967,7 @@ intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
 static bool
 intel_iommu_dev_feat_enabled(struct device *dev, enum iommu_dev_features feat)
 {
-	struct device_domain_info *info = dev->archdata.iommu;
+	struct device_domain_info *info = get_domain_info(dev);
 
 	if (feat == IOMMU_DEV_FEAT_AUX)
 		return scalable_mode_support() && info && info->auxd_enabled;
@@ -6198,8 +6034,9 @@ const struct iommu_ops intel_iommu_ops = {
 	.map			= intel_iommu_map,
 	.unmap			= intel_iommu_unmap,
 	.iova_to_phys		= intel_iommu_iova_to_phys,
-	.add_device		= intel_iommu_add_device,
-	.remove_device		= intel_iommu_remove_device,
+	.probe_device		= intel_iommu_probe_device,
+	.probe_finalize		= intel_iommu_probe_finalize,
+	.release_device		= intel_iommu_release_device,
 	.get_resv_regions	= intel_iommu_get_resv_regions,
 	.put_resv_regions	= generic_iommu_put_resv_regions,
 	.apply_resv_region	= intel_iommu_apply_resv_region,
@@ -6209,7 +6046,16 @@ const struct iommu_ops intel_iommu_ops = {
 	.dev_enable_feat	= intel_iommu_dev_enable_feat,
 	.dev_disable_feat	= intel_iommu_dev_disable_feat,
 	.is_attach_deferred	= intel_iommu_is_attach_deferred,
+	.def_domain_type	= device_def_domain_type,
 	.pgsize_bitmap		= INTEL_IOMMU_PGSIZES,
+#ifdef CONFIG_INTEL_IOMMU_SVM
+	.cache_invalidate	= intel_iommu_sva_invalidate,
+	.sva_bind_gpasid	= intel_svm_bind_gpasid,
+	.sva_unbind_gpasid	= intel_svm_unbind_gpasid,
+	.sva_bind		= intel_svm_bind,
+	.sva_unbind		= intel_svm_unbind,
+	.sva_get_pasid		= intel_svm_get_pasid,
+#endif
 };
 
 static void quirk_iommu_igfx(struct pci_dev *dev)
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel/irq_remapping.c
index 81e43c1df7ec..7f8769800815 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel/irq_remapping.c
@@ -21,7 +21,7 @@
 #include <asm/pci-direct.h>
 #include <asm/msidef.h>
 
-#include "irq_remapping.h"
+#include "../irq_remapping.h"
 
 enum irq_mode {
 	IRQ_REMAPPING,
@@ -151,7 +151,7 @@ static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
 	desc.qw2 = 0;
 	desc.qw3 = 0;
 
-	return qi_submit_sync(&desc, iommu);
+	return qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 static int modify_irte(struct irq_2_iommu *irq_iommu,
diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel/pasid.c
index 22b30f10b396..c81f0f17c6ba 100644
--- a/drivers/iommu/intel-pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -27,6 +27,63 @@
 static DEFINE_SPINLOCK(pasid_lock);
 u32 intel_pasid_max_id = PASID_MAX;
 
+int vcmd_alloc_pasid(struct intel_iommu *iommu, unsigned int *pasid)
+{
+	unsigned long flags;
+	u8 status_code;
+	int ret = 0;
+	u64 res;
+
+	raw_spin_lock_irqsave(&iommu->register_lock, flags);
+	dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC);
+	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,
+		      !(res & VCMD_VRSP_IP), res);
+	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
+
+	status_code = VCMD_VRSP_SC(res);
+	switch (status_code) {
+	case VCMD_VRSP_SC_SUCCESS:
+		*pasid = VCMD_VRSP_RESULT_PASID(res);
+		break;
+	case VCMD_VRSP_SC_NO_PASID_AVAIL:
+		pr_info("IOMMU: %s: No PASID available\n", iommu->name);
+		ret = -ENOSPC;
+		break;
+	default:
+		ret = -ENODEV;
+		pr_warn("IOMMU: %s: Unexpected error code %d\n",
+			iommu->name, status_code);
+	}
+
+	return ret;
+}
+
+void vcmd_free_pasid(struct intel_iommu *iommu, unsigned int pasid)
+{
+	unsigned long flags;
+	u8 status_code;
+	u64 res;
+
+	raw_spin_lock_irqsave(&iommu->register_lock, flags);
+	dmar_writeq(iommu->reg + DMAR_VCMD_REG,
+		    VCMD_CMD_OPERAND(pasid) | VCMD_CMD_FREE);
+	IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,
+		      !(res & VCMD_VRSP_IP), res);
+	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
+
+	status_code = VCMD_VRSP_SC(res);
+	switch (status_code) {
+	case VCMD_VRSP_SC_SUCCESS:
+		break;
+	case VCMD_VRSP_SC_INVALID_PASID:
+		pr_info("IOMMU: %s: Invalid PASID\n", iommu->name);
+		break;
+	default:
+		pr_warn("IOMMU: %s: Unexpected error code %d\n",
+			iommu->name, status_code);
+	}
+}
+
 /*
  * Per device pasid table management:
  */
@@ -94,7 +151,7 @@ int intel_pasid_alloc_table(struct device *dev)
 	int size;
 
 	might_sleep();
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
 		return -EINVAL;
 
@@ -141,7 +198,7 @@ void intel_pasid_free_table(struct device *dev)
 	struct pasid_entry *table;
 	int i, max_pde;
 
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (!info || !dev_is_pci(dev) || !info->pasid_table)
 		return;
 
@@ -167,7 +224,7 @@ struct pasid_table *intel_pasid_get_table(struct device *dev)
 {
 	struct device_domain_info *info;
 
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (!info)
 		return NULL;
 
@@ -178,7 +235,7 @@ int intel_pasid_get_dev_max_id(struct device *dev)
 {
 	struct device_domain_info *info;
 
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (!info || !info->pasid_table)
 		return 0;
 
@@ -199,7 +256,7 @@ struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid)
 		return NULL;
 
 	dir = pasid_table->table;
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	dir_index = pasid >> PASID_PDE_SHIFT;
 	index = pasid & PASID_PTE_MASK;
 
@@ -235,7 +292,20 @@ static inline void pasid_clear_entry(struct pasid_entry *pe)
 	WRITE_ONCE(pe->val[7], 0);
 }
 
-static void intel_pasid_clear_entry(struct device *dev, int pasid)
+static inline void pasid_clear_entry_with_fpd(struct pasid_entry *pe)
+{
+	WRITE_ONCE(pe->val[0], PASID_PTE_FPD);
+	WRITE_ONCE(pe->val[1], 0);
+	WRITE_ONCE(pe->val[2], 0);
+	WRITE_ONCE(pe->val[3], 0);
+	WRITE_ONCE(pe->val[4], 0);
+	WRITE_ONCE(pe->val[5], 0);
+	WRITE_ONCE(pe->val[6], 0);
+	WRITE_ONCE(pe->val[7], 0);
+}
+
+static void
+intel_pasid_clear_entry(struct device *dev, int pasid, bool fault_ignore)
 {
 	struct pasid_entry *pe;
 
@@ -243,7 +313,10 @@ static void intel_pasid_clear_entry(struct device *dev, int pasid)
 	if (WARN_ON(!pe))
 		return;
 
-	pasid_clear_entry(pe);
+	if (fault_ignore && pasid_pte_is_present(pe))
+		pasid_clear_entry_with_fpd(pe);
+	else
+		pasid_clear_entry(pe);
 }
 
 static inline void pasid_set_bits(u64 *ptr, u64 mask, u64 bits)
@@ -359,18 +432,29 @@ pasid_set_flpm(struct pasid_entry *pe, u64 value)
 	pasid_set_bits(&pe->val[2], GENMASK_ULL(3, 2), value << 2);
 }
 
+/*
+ * Setup the Extended Access Flag Enable (EAFE) field (Bit 135)
+ * of a scalable mode PASID entry.
+ */
+static inline void
+pasid_set_eafe(struct pasid_entry *pe)
+{
+	pasid_set_bits(&pe->val[2], 1 << 7, 1 << 7);
+}
+
 static void
 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,
 				    u16 did, int pasid)
 {
 	struct qi_desc desc;
 
-	desc.qw0 = QI_PC_DID(did) | QI_PC_PASID_SEL | QI_PC_PASID(pasid);
+	desc.qw0 = QI_PC_DID(did) | QI_PC_GRAN(QI_PC_PASID_SEL) |
+		QI_PC_PASID(pasid) | QI_PC_TYPE;
 	desc.qw1 = 0;
 	desc.qw2 = 0;
 	desc.qw3 = 0;
 
-	qi_submit_sync(&desc, iommu);
+	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 static void
@@ -384,7 +468,7 @@ iotlb_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, u32 pasid)
 	desc.qw2 = 0;
 	desc.qw3 = 0;
 
-	qi_submit_sync(&desc, iommu);
+	qi_submit_sync(iommu, &desc, 1, 0);
 }
 
 static void
@@ -394,7 +478,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu,
 	struct device_domain_info *info;
 	u16 sid, qdep, pfsid;
 
-	info = dev->archdata.iommu;
+	info = get_domain_info(dev);
 	if (!info || !info->ats_enabled)
 		return;
 
@@ -405,8 +489,8 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu,
 	qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);
 }
 
-void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
-				 struct device *dev, int pasid)
+void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
+				 int pasid, bool fault_ignore)
 {
 	struct pasid_entry *pte;
 	u16 did;
@@ -416,7 +500,7 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
 		return;
 
 	did = pasid_get_domain_id(pte);
-	intel_pasid_clear_entry(dev, pasid);
+	intel_pasid_clear_entry(dev, pasid, fault_ignore);
 
 	if (!ecap_coherent(iommu->ecap))
 		clflush_cache_range(pte, sizeof(*pte));
@@ -492,7 +576,7 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu,
 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
 
 	/* Setup Present and PASID Granular Transfer Type: */
-	pasid_set_translation_type(pte, 1);
+	pasid_set_translation_type(pte, PASID_ENTRY_PGTT_FL_ONLY);
 	pasid_set_present(pte);
 	pasid_flush_caches(iommu, pte, pasid, did);
 
@@ -500,6 +584,25 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu,
 }
 
 /*
+ * Skip top levels of page tables for iommu which has less agaw
+ * than default. Unnecessary for PT mode.
+ */
+static inline int iommu_skip_agaw(struct dmar_domain *domain,
+				  struct intel_iommu *iommu,
+				  struct dma_pte **pgd)
+{
+	int agaw;
+
+	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
+		*pgd = phys_to_virt(dma_pte_addr(*pgd));
+		if (!dma_pte_present(*pgd))
+			return -EINVAL;
+	}
+
+	return agaw;
+}
+
+/*
  * Set up the scalable mode pasid entry for second only translation type.
  */
 int intel_pasid_setup_second_level(struct intel_iommu *iommu,
@@ -522,17 +625,11 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu,
 		return -EINVAL;
 	}
 
-	/*
-	 * Skip top levels of page tables for iommu which has less agaw
-	 * than default. Unnecessary for PT mode.
-	 */
 	pgd = domain->pgd;
-	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
-		pgd = phys_to_virt(dma_pte_addr(pgd));
-		if (!dma_pte_present(pgd)) {
-			dev_err(dev, "Invalid domain page table\n");
-			return -EINVAL;
-		}
+	agaw = iommu_skip_agaw(domain, iommu, &pgd);
+	if (agaw < 0) {
+		dev_err(dev, "Invalid domain page table\n");
+		return -EINVAL;
 	}
 
 	pgd_val = virt_to_phys(pgd);
@@ -548,7 +645,7 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu,
 	pasid_set_domain_id(pte, did);
 	pasid_set_slptr(pte, pgd_val);
 	pasid_set_address_width(pte, agaw);
-	pasid_set_translation_type(pte, 2);
+	pasid_set_translation_type(pte, PASID_ENTRY_PGTT_SL_ONLY);
 	pasid_set_fault_enable(pte);
 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
 
@@ -582,7 +679,7 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
 	pasid_clear_entry(pte);
 	pasid_set_domain_id(pte, did);
 	pasid_set_address_width(pte, iommu->agaw);
-	pasid_set_translation_type(pte, 4);
+	pasid_set_translation_type(pte, PASID_ENTRY_PGTT_PT);
 	pasid_set_fault_enable(pte);
 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
 
@@ -596,3 +693,161 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
 
 	return 0;
 }
+
+static int
+intel_pasid_setup_bind_data(struct intel_iommu *iommu, struct pasid_entry *pte,
+			    struct iommu_gpasid_bind_data_vtd *pasid_data)
+{
+	/*
+	 * Not all guest PASID table entry fields are passed down during bind,
+	 * here we only set up the ones that are dependent on guest settings.
+	 * Execution related bits such as NXE, SMEP are not supported.
+	 * Other fields, such as snoop related, are set based on host needs
+	 * regardless of guest settings.
+	 */
+	if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_SRE) {
+		if (!ecap_srs(iommu->ecap)) {
+			pr_err_ratelimited("No supervisor request support on %s\n",
+					   iommu->name);
+			return -EINVAL;
+		}
+		pasid_set_sre(pte);
+	}
+
+	if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_EAFE) {
+		if (!ecap_eafs(iommu->ecap)) {
+			pr_err_ratelimited("No extended access flag support on %s\n",
+					   iommu->name);
+			return -EINVAL;
+		}
+		pasid_set_eafe(pte);
+	}
+
+	/*
+	 * Memory type is only applicable to devices inside processor coherent
+	 * domain. Will add MTS support once coherent devices are available.
+	 */
+	if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_MTS_MASK) {
+		pr_warn_ratelimited("No memory type support %s\n",
+				    iommu->name);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * intel_pasid_setup_nested() - Set up PASID entry for nested translation.
+ * This could be used for guest shared virtual address. In this case, the
+ * first level page tables are used for GVA-GPA translation in the guest,
+ * second level page tables are used for GPA-HPA translation.
+ *
+ * @iommu:      IOMMU which the device belong to
+ * @dev:        Device to be set up for translation
+ * @gpgd:       FLPTPTR: First Level Page translation pointer in GPA
+ * @pasid:      PASID to be programmed in the device PASID table
+ * @pasid_data: Additional PASID info from the guest bind request
+ * @domain:     Domain info for setting up second level page tables
+ * @addr_width: Address width of the first level (guest)
+ */
+int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
+			     pgd_t *gpgd, int pasid,
+			     struct iommu_gpasid_bind_data_vtd *pasid_data,
+			     struct dmar_domain *domain, int addr_width)
+{
+	struct pasid_entry *pte;
+	struct dma_pte *pgd;
+	int ret = 0;
+	u64 pgd_val;
+	int agaw;
+	u16 did;
+
+	if (!ecap_nest(iommu->ecap)) {
+		pr_err_ratelimited("IOMMU: %s: No nested translation support\n",
+				   iommu->name);
+		return -EINVAL;
+	}
+
+	if (!(domain->flags & DOMAIN_FLAG_NESTING_MODE)) {
+		pr_err_ratelimited("Domain is not in nesting mode, %x\n",
+				   domain->flags);
+		return -EINVAL;
+	}
+
+	pte = intel_pasid_get_entry(dev, pasid);
+	if (WARN_ON(!pte))
+		return -EINVAL;
+
+	/*
+	 * Caller must ensure PASID entry is not in use, i.e. not bind the
+	 * same PASID to the same device twice.
+	 */
+	if (pasid_pte_is_present(pte))
+		return -EBUSY;
+
+	pasid_clear_entry(pte);
+
+	/* Sanity checking performed by caller to make sure address
+	 * width matching in two dimensions:
+	 * 1. CPU vs. IOMMU
+	 * 2. Guest vs. Host.
+	 */
+	switch (addr_width) {
+#ifdef CONFIG_X86
+	case ADDR_WIDTH_5LEVEL:
+		if (!cpu_feature_enabled(X86_FEATURE_LA57) ||
+		    !cap_5lp_support(iommu->cap)) {
+			dev_err_ratelimited(dev,
+					    "5-level paging not supported\n");
+			return -EINVAL;
+		}
+
+		pasid_set_flpm(pte, 1);
+		break;
+#endif
+	case ADDR_WIDTH_4LEVEL:
+		pasid_set_flpm(pte, 0);
+		break;
+	default:
+		dev_err_ratelimited(dev, "Invalid guest address width %d\n",
+				    addr_width);
+		return -EINVAL;
+	}
+
+	/* First level PGD is in GPA, must be supported by the second level */
+	if ((uintptr_t)gpgd > domain->max_addr) {
+		dev_err_ratelimited(dev,
+				    "Guest PGD %lx not supported, max %llx\n",
+				    (uintptr_t)gpgd, domain->max_addr);
+		return -EINVAL;
+	}
+	pasid_set_flptr(pte, (uintptr_t)gpgd);
+
+	ret = intel_pasid_setup_bind_data(iommu, pte, pasid_data);
+	if (ret)
+		return ret;
+
+	/* Setup the second level based on the given domain */
+	pgd = domain->pgd;
+
+	agaw = iommu_skip_agaw(domain, iommu, &pgd);
+	if (agaw < 0) {
+		dev_err_ratelimited(dev, "Invalid domain page table\n");
+		return -EINVAL;
+	}
+	pgd_val = virt_to_phys(pgd);
+	pasid_set_slptr(pte, pgd_val);
+	pasid_set_fault_enable(pte);
+
+	did = domain->iommu_did[iommu->seq_id];
+	pasid_set_domain_id(pte, did);
+
+	pasid_set_address_width(pte, agaw);
+	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
+
+	pasid_set_translation_type(pte, PASID_ENTRY_PGTT_NESTED);
+	pasid_set_present(pte);
+	pasid_flush_caches(iommu, pte, pasid, did);
+
+	return ret;
+}
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel/svm.c
index 2998418f0a38..6c87c807a0ab 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -23,6 +23,7 @@
 #include "intel-pasid.h"
 
 static irqreturn_t prq_event_thread(int irq, void *d);
+static void intel_svm_drain_prq(struct device *dev, int pasid);
 
 #define PRQ_ORDER 0
 
@@ -66,6 +67,8 @@ int intel_svm_enable_prq(struct intel_iommu *iommu)
 	dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
 	dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER);
 
+	init_completion(&iommu->prq_complete);
+
 	return 0;
 }
 
@@ -138,7 +141,7 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d
 	}
 	desc.qw2 = 0;
 	desc.qw3 = 0;
-	qi_submit_sync(&desc, svm->iommu);
+	qi_submit_sync(svm->iommu, &desc, 1, 0);
 
 	if (sdev->dev_iotlb) {
 		desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |
@@ -162,7 +165,7 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d
 		}
 		desc.qw2 = 0;
 		desc.qw3 = 0;
-		qi_submit_sync(&desc, svm->iommu);
+		qi_submit_sync(svm->iommu, &desc, 1, 0);
 	}
 }
 
@@ -206,10 +209,9 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	 * *has* to handle gracefully without affecting other processes.
 	 */
 	rcu_read_lock();
-	list_for_each_entry_rcu(sdev, &svm->devs, list) {
-		intel_pasid_tear_down_entry(svm->iommu, sdev->dev, svm->pasid);
-		intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);
-	}
+	list_for_each_entry_rcu(sdev, &svm->devs, list)
+		intel_pasid_tear_down_entry(svm->iommu, sdev->dev,
+					    svm->pasid, true);
 	rcu_read_unlock();
 
 }
@@ -226,13 +228,212 @@ static LIST_HEAD(global_svm_list);
 	list_for_each_entry((sdev), &(svm)->devs, list)	\
 		if ((d) != (sdev)->dev) {} else
 
-int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops)
+int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
+			  struct iommu_gpasid_bind_data *data)
+{
+	struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
+	struct dmar_domain *dmar_domain;
+	struct intel_svm_dev *sdev;
+	struct intel_svm *svm;
+	int ret = 0;
+
+	if (WARN_ON(!iommu) || !data)
+		return -EINVAL;
+
+	if (data->version != IOMMU_GPASID_BIND_VERSION_1 ||
+	    data->format != IOMMU_PASID_FORMAT_INTEL_VTD)
+		return -EINVAL;
+
+	if (!dev_is_pci(dev))
+		return -ENOTSUPP;
+
+	/* VT-d supports devices with full 20 bit PASIDs only */
+	if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX)
+		return -EINVAL;
+
+	/*
+	 * We only check host PASID range, we have no knowledge to check
+	 * guest PASID range.
+	 */
+	if (data->hpasid <= 0 || data->hpasid >= PASID_MAX)
+		return -EINVAL;
+
+	dmar_domain = to_dmar_domain(domain);
+
+	mutex_lock(&pasid_mutex);
+	svm = ioasid_find(NULL, data->hpasid, NULL);
+	if (IS_ERR(svm)) {
+		ret = PTR_ERR(svm);
+		goto out;
+	}
+
+	if (svm) {
+		/*
+		 * If we found svm for the PASID, there must be at
+		 * least one device bond, otherwise svm should be freed.
+		 */
+		if (WARN_ON(list_empty(&svm->devs))) {
+			ret = -EINVAL;
+			goto out;
+		}
+
+		for_each_svm_dev(sdev, svm, dev) {
+			/*
+			 * For devices with aux domains, we should allow
+			 * multiple bind calls with the same PASID and pdev.
+			 */
+			if (iommu_dev_feature_enabled(dev,
+						      IOMMU_DEV_FEAT_AUX)) {
+				sdev->users++;
+			} else {
+				dev_warn_ratelimited(dev,
+						     "Already bound with PASID %u\n",
+						     svm->pasid);
+				ret = -EBUSY;
+			}
+			goto out;
+		}
+	} else {
+		/* We come here when PASID has never been bond to a device. */
+		svm = kzalloc(sizeof(*svm), GFP_KERNEL);
+		if (!svm) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		/* REVISIT: upper layer/VFIO can track host process that bind
+		 * the PASID. ioasid_set = mm might be sufficient for vfio to
+		 * check pasid VMM ownership. We can drop the following line
+		 * once VFIO and IOASID set check is in place.
+		 */
+		svm->mm = get_task_mm(current);
+		svm->pasid = data->hpasid;
+		if (data->flags & IOMMU_SVA_GPASID_VAL) {
+			svm->gpasid = data->gpasid;
+			svm->flags |= SVM_FLAG_GUEST_PASID;
+		}
+		ioasid_set_data(data->hpasid, svm);
+		INIT_LIST_HEAD_RCU(&svm->devs);
+		mmput(svm->mm);
+	}
+	sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
+	if (!sdev) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	sdev->dev = dev;
+
+	/* Only count users if device has aux domains */
+	if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
+		sdev->users = 1;
+
+	/* Set up device context entry for PASID if not enabled already */
+	ret = intel_iommu_enable_pasid(iommu, sdev->dev);
+	if (ret) {
+		dev_err_ratelimited(dev, "Failed to enable PASID capability\n");
+		kfree(sdev);
+		goto out;
+	}
+
+	/*
+	 * PASID table is per device for better security. Therefore, for
+	 * each bind of a new device even with an existing PASID, we need to
+	 * call the nested mode setup function here.
+	 */
+	spin_lock(&iommu->lock);
+	ret = intel_pasid_setup_nested(iommu, dev,
+				       (pgd_t *)(uintptr_t)data->gpgd,
+				       data->hpasid, &data->vtd, dmar_domain,
+				       data->addr_width);
+	spin_unlock(&iommu->lock);
+	if (ret) {
+		dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n",
+				    data->hpasid, ret);
+		/*
+		 * PASID entry should be in cleared state if nested mode
+		 * set up failed. So we only need to clear IOASID tracking
+		 * data such that free call will succeed.
+		 */
+		kfree(sdev);
+		goto out;
+	}
+
+	svm->flags |= SVM_FLAG_GUEST_MODE;
+
+	init_rcu_head(&sdev->rcu);
+	list_add_rcu(&sdev->list, &svm->devs);
+ out:
+	if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) {
+		ioasid_set_data(data->hpasid, NULL);
+		kfree(svm);
+	}
+
+	mutex_unlock(&pasid_mutex);
+	return ret;
+}
+
+int intel_svm_unbind_gpasid(struct device *dev, int pasid)
+{
+	struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
+	struct intel_svm_dev *sdev;
+	struct intel_svm *svm;
+	int ret = -EINVAL;
+
+	if (WARN_ON(!iommu))
+		return -EINVAL;
+
+	mutex_lock(&pasid_mutex);
+	svm = ioasid_find(NULL, pasid, NULL);
+	if (!svm) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (IS_ERR(svm)) {
+		ret = PTR_ERR(svm);
+		goto out;
+	}
+
+	for_each_svm_dev(sdev, svm, dev) {
+		ret = 0;
+		if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
+			sdev->users--;
+		if (!sdev->users) {
+			list_del_rcu(&sdev->list);
+			intel_pasid_tear_down_entry(iommu, dev,
+						    svm->pasid, false);
+			intel_svm_drain_prq(dev, svm->pasid);
+			kfree_rcu(sdev, rcu);
+
+			if (list_empty(&svm->devs)) {
+				/*
+				 * We do not free the IOASID here in that
+				 * IOMMU driver did not allocate it.
+				 * Unlike native SVM, IOASID for guest use was
+				 * allocated prior to the bind call.
+				 * In any case, if the free call comes before
+				 * the unbind, IOMMU driver will get notified
+				 * and perform cleanup.
+				 */
+				ioasid_set_data(pasid, NULL);
+				kfree(svm);
+			}
+		}
+		break;
+	}
+out:
+	mutex_unlock(&pasid_mutex);
+	return ret;
+}
+
+/* Caller must hold pasid_mutex, mm reference */
+static int
+intel_svm_bind_mm(struct device *dev, int flags, struct svm_dev_ops *ops,
+		  struct mm_struct *mm, struct intel_svm_dev **sd)
 {
 	struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
 	struct device_domain_info *info;
 	struct intel_svm_dev *sdev;
 	struct intel_svm *svm = NULL;
-	struct mm_struct *mm = NULL;
 	int pasid_max;
 	int ret;
 
@@ -249,16 +450,15 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 	} else
 		pasid_max = 1 << 20;
 
+	/* Bind supervisor PASID shuld have mm = NULL */
 	if (flags & SVM_FLAG_SUPERVISOR_MODE) {
-		if (!ecap_srs(iommu->ecap))
+		if (!ecap_srs(iommu->ecap) || mm) {
+			pr_err("Supervisor PASID with user provided mm.\n");
 			return -EINVAL;
-	} else if (pasid) {
-		mm = get_task_mm(current);
-		BUG_ON(!mm);
+		}
 	}
 
-	mutex_lock(&pasid_mutex);
-	if (pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) {
+	if (!(flags & SVM_FLAG_PRIVATE_PASID)) {
 		struct intel_svm *t;
 
 		list_for_each_entry(t, &global_svm_list, list) {
@@ -296,19 +496,12 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 	sdev->dev = dev;
 
 	ret = intel_iommu_enable_pasid(iommu, dev);
-	if (ret || !pasid) {
-		/* If they don't actually want to assign a PASID, this is
-		 * just an enabling check/preparation. */
-		kfree(sdev);
-		goto out;
-	}
-
-	info = dev->archdata.iommu;
-	if (!info || !info->pasid_supported) {
+	if (ret) {
 		kfree(sdev);
 		goto out;
 	}
 
+	info = get_domain_info(dev);
 	sdev->did = FLPT_DEFAULT_DID;
 	sdev->sid = PCI_DEVID(info->bus, info->devfn);
 	if (info->ats_enabled) {
@@ -397,26 +590,24 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 		}
 	}
 	list_add_rcu(&sdev->list, &svm->devs);
-
- success:
-	*pasid = svm->pasid;
+success:
+	sdev->pasid = svm->pasid;
+	sdev->sva.dev = dev;
+	if (sd)
+		*sd = sdev;
 	ret = 0;
  out:
-	mutex_unlock(&pasid_mutex);
-	if (mm)
-		mmput(mm);
 	return ret;
 }
-EXPORT_SYMBOL_GPL(intel_svm_bind_mm);
 
-int intel_svm_unbind_mm(struct device *dev, int pasid)
+/* Caller must hold pasid_mutex */
+static int intel_svm_unbind_mm(struct device *dev, int pasid)
 {
 	struct intel_svm_dev *sdev;
 	struct intel_iommu *iommu;
 	struct intel_svm *svm;
 	int ret = -EINVAL;
 
-	mutex_lock(&pasid_mutex);
 	iommu = intel_svm_device_to_iommu(dev);
 	if (!iommu)
 		goto out;
@@ -442,8 +633,9 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
 			 * to use. We have a *shared* PASID table, because it's
 			 * large and has to be physically contiguous. So it's
 			 * hard to be as defensive as we might like. */
-			intel_pasid_tear_down_entry(iommu, dev, svm->pasid);
-			intel_flush_svm_range_dev(svm, sdev, 0, -1, 0);
+			intel_pasid_tear_down_entry(iommu, dev,
+						    svm->pasid, false);
+			intel_svm_drain_prq(dev, svm->pasid);
 			kfree_rcu(sdev, rcu);
 
 			if (list_empty(&svm->devs)) {
@@ -462,45 +654,9 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
 		break;
 	}
  out:
-	mutex_unlock(&pasid_mutex);
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(intel_svm_unbind_mm);
-
-int intel_svm_is_pasid_valid(struct device *dev, int pasid)
-{
-	struct intel_iommu *iommu;
-	struct intel_svm *svm;
-	int ret = -EINVAL;
-
-	mutex_lock(&pasid_mutex);
-	iommu = intel_svm_device_to_iommu(dev);
-	if (!iommu)
-		goto out;
-
-	svm = ioasid_find(NULL, pasid, NULL);
-	if (!svm)
-		goto out;
-
-	if (IS_ERR(svm)) {
-		ret = PTR_ERR(svm);
-		goto out;
-	}
-	/* init_mm is used in this case */
-	if (!svm->mm)
-		ret = 1;
-	else if (atomic_read(&svm->mm->mm_users) > 0)
-		ret = 1;
-	else
-		ret = 0;
-
- out:
-	mutex_unlock(&pasid_mutex);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(intel_svm_is_pasid_valid);
 
 /* Page request queue descriptor */
 struct page_req_dsc {
@@ -557,6 +713,93 @@ static bool is_canonical_address(u64 addr)
 	return (((saddr << shift) >> shift) == saddr);
 }
 
+/**
+ * intel_svm_drain_prq - Drain page requests and responses for a pasid
+ * @dev: target device
+ * @pasid: pasid for draining
+ *
+ * Drain all pending page requests and responses related to @pasid in both
+ * software and hardware. This is supposed to be called after the device
+ * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB
+ * and DevTLB have been invalidated.
+ *
+ * It waits until all pending page requests for @pasid in the page fault
+ * queue are completed by the prq handling thread. Then follow the steps
+ * described in VT-d spec CH7.10 to drain all page requests and page
+ * responses pending in the hardware.
+ */
+static void intel_svm_drain_prq(struct device *dev, int pasid)
+{
+	struct device_domain_info *info;
+	struct dmar_domain *domain;
+	struct intel_iommu *iommu;
+	struct qi_desc desc[3];
+	struct pci_dev *pdev;
+	int head, tail;
+	u16 sid, did;
+	int qdep;
+
+	info = get_domain_info(dev);
+	if (WARN_ON(!info || !dev_is_pci(dev)))
+		return;
+
+	if (!info->pri_enabled)
+		return;
+
+	iommu = info->iommu;
+	domain = info->domain;
+	pdev = to_pci_dev(dev);
+	sid = PCI_DEVID(info->bus, info->devfn);
+	did = domain->iommu_did[iommu->seq_id];
+	qdep = pci_ats_queue_depth(pdev);
+
+	/*
+	 * Check and wait until all pending page requests in the queue are
+	 * handled by the prq handling thread.
+	 */
+prq_retry:
+	reinit_completion(&iommu->prq_complete);
+	tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
+	head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
+	while (head != tail) {
+		struct page_req_dsc *req;
+
+		req = &iommu->prq[head / sizeof(*req)];
+		if (!req->pasid_present || req->pasid != pasid) {
+			head = (head + sizeof(*req)) & PRQ_RING_MASK;
+			continue;
+		}
+
+		wait_for_completion(&iommu->prq_complete);
+		goto prq_retry;
+	}
+
+	/*
+	 * Perform steps described in VT-d spec CH7.10 to drain page
+	 * requests and responses in hardware.
+	 */
+	memset(desc, 0, sizeof(desc));
+	desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
+			QI_IWD_FENCE |
+			QI_IWD_TYPE;
+	desc[1].qw0 = QI_EIOTLB_PASID(pasid) |
+			QI_EIOTLB_DID(did) |
+			QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
+			QI_EIOTLB_TYPE;
+	desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) |
+			QI_DEV_EIOTLB_SID(sid) |
+			QI_DEV_EIOTLB_QDEP(qdep) |
+			QI_DEIOTLB_TYPE |
+			QI_DEV_IOTLB_PFSID(info->pfsid);
+qi_retry:
+	reinit_completion(&iommu->prq_complete);
+	qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);
+	if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {
+		wait_for_completion(&iommu->prq_complete);
+		goto qi_retry;
+	}
+}
+
 static irqreturn_t prq_event_thread(int irq, void *d)
 {
 	struct intel_iommu *iommu = d;
@@ -620,7 +863,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
 		if (!mmget_not_zero(svm->mm))
 			goto bad_req;
 
-		down_read(&svm->mm->mmap_sem);
+		mmap_read_lock(svm->mm);
 		vma = find_extend_vma(svm->mm, address);
 		if (!vma || address < vma->vm_start)
 			goto invalid;
@@ -635,7 +878,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
 
 		result = QI_RESP_SUCCESS;
 	invalid:
-		up_read(&svm->mm->mmap_sem);
+		mmap_read_unlock(svm->mm);
 		mmput(svm->mm);
 	bad_req:
 		/* Accounting for major/minor faults? */
@@ -685,12 +928,75 @@ static irqreturn_t prq_event_thread(int irq, void *d)
 				       sizeof(req->priv_data));
 			resp.qw2 = 0;
 			resp.qw3 = 0;
-			qi_submit_sync(&resp, iommu);
+			qi_submit_sync(iommu, &resp, 1, 0);
 		}
 		head = (head + sizeof(*req)) & PRQ_RING_MASK;
 	}
 
 	dmar_writeq(iommu->reg + DMAR_PQH_REG, tail);
 
+	/*
+	 * Clear the page request overflow bit and wake up all threads that
+	 * are waiting for the completion of this handling.
+	 */
+	if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO)
+		writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG);
+
+	if (!completion_done(&iommu->prq_complete))
+		complete(&iommu->prq_complete);
+
 	return IRQ_RETVAL(handled);
 }
+
+#define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva)
+struct iommu_sva *
+intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
+{
+	struct iommu_sva *sva = ERR_PTR(-EINVAL);
+	struct intel_svm_dev *sdev = NULL;
+	int flags = 0;
+	int ret;
+
+	/*
+	 * TODO: Consolidate with generic iommu-sva bind after it is merged.
+	 * It will require shared SVM data structures, i.e. combine io_mm
+	 * and intel_svm etc.
+	 */
+	if (drvdata)
+		flags = *(int *)drvdata;
+	mutex_lock(&pasid_mutex);
+	ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev);
+	if (ret)
+		sva = ERR_PTR(ret);
+	else if (sdev)
+		sva = &sdev->sva;
+	else
+		WARN(!sdev, "SVM bind succeeded with no sdev!\n");
+
+	mutex_unlock(&pasid_mutex);
+
+	return sva;
+}
+
+void intel_svm_unbind(struct iommu_sva *sva)
+{
+	struct intel_svm_dev *sdev;
+
+	mutex_lock(&pasid_mutex);
+	sdev = to_intel_svm_dev(sva);
+	intel_svm_unbind_mm(sdev->dev, sdev->pasid);
+	mutex_unlock(&pasid_mutex);
+}
+
+int intel_svm_get_pasid(struct iommu_sva *sva)
+{
+	struct intel_svm_dev *sdev;
+	int pasid;
+
+	mutex_lock(&pasid_mutex);
+	sdev = to_intel_svm_dev(sva);
+	pasid = sdev->pasid;
+	mutex_unlock(&pasid_mutex);
+
+	return pasid;
+}
diff --git a/drivers/iommu/intel-trace.c b/drivers/iommu/intel/trace.c
index bfb6a6e37a88..bfb6a6e37a88 100644
--- a/drivers/iommu/intel-trace.c
+++ b/drivers/iommu/intel/trace.c
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 03d6a26687bc..d43120eb1dc5 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -44,6 +44,7 @@ struct iommu_group {
 	int id;
 	struct iommu_domain *default_domain;
 	struct iommu_domain *domain;
+	struct list_head entry;
 };
 
 struct group_device {
@@ -79,6 +80,20 @@ static bool iommu_cmd_line_dma_api(void)
 	return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API);
 }
 
+static int iommu_alloc_default_domain(struct iommu_group *group,
+				      struct device *dev);
+static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
+						 unsigned type);
+static int __iommu_attach_device(struct iommu_domain *domain,
+				 struct device *dev);
+static int __iommu_attach_group(struct iommu_domain *domain,
+				struct iommu_group *group);
+static void __iommu_detach_group(struct iommu_domain *domain,
+				 struct iommu_group *group);
+static int iommu_create_device_direct_mappings(struct iommu_group *group,
+					       struct device *dev);
+static struct iommu_group *iommu_group_get_for_dev(struct device *dev);
+
 #define IOMMU_GROUP_ATTR(_name, _mode, _show, _store)		\
 struct iommu_group_attribute iommu_group_attr_##_name =		\
 	__ATTR(_name, _mode, _show, _store)
@@ -175,57 +190,118 @@ static void dev_iommu_free(struct device *dev)
 	dev->iommu = NULL;
 }
 
-int iommu_probe_device(struct device *dev)
+static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
 {
 	const struct iommu_ops *ops = dev->bus->iommu_ops;
+	struct iommu_device *iommu_dev;
+	struct iommu_group *group;
 	int ret;
 
-	WARN_ON(dev->iommu_group);
 	if (!ops)
-		return -EINVAL;
+		return -ENODEV;
 
 	if (!dev_iommu_get(dev))
 		return -ENOMEM;
 
 	if (!try_module_get(ops->owner)) {
 		ret = -EINVAL;
-		goto err_free_dev_param;
+		goto err_free;
 	}
 
-	ret = ops->add_device(dev);
-	if (ret)
-		goto err_module_put;
+	iommu_dev = ops->probe_device(dev);
+	if (IS_ERR(iommu_dev)) {
+		ret = PTR_ERR(iommu_dev);
+		goto out_module_put;
+	}
+
+	dev->iommu->iommu_dev = iommu_dev;
+
+	group = iommu_group_get_for_dev(dev);
+	if (IS_ERR(group)) {
+		ret = PTR_ERR(group);
+		goto out_release;
+	}
+	iommu_group_put(group);
+
+	if (group_list && !group->default_domain && list_empty(&group->entry))
+		list_add_tail(&group->entry, group_list);
+
+	iommu_device_link(iommu_dev, dev);
 
 	return 0;
 
-err_module_put:
+out_release:
+	ops->release_device(dev);
+
+out_module_put:
 	module_put(ops->owner);
-err_free_dev_param:
+
+err_free:
 	dev_iommu_free(dev);
+
 	return ret;
 }
 
-void iommu_release_device(struct device *dev)
+int iommu_probe_device(struct device *dev)
 {
 	const struct iommu_ops *ops = dev->bus->iommu_ops;
+	struct iommu_group *group;
+	int ret;
 
-	if (dev->iommu_group)
-		ops->remove_device(dev);
+	ret = __iommu_probe_device(dev, NULL);
+	if (ret)
+		goto err_out;
+
+	group = iommu_group_get(dev);
+	if (!group)
+		goto err_release;
+
+	/*
+	 * Try to allocate a default domain - needs support from the
+	 * IOMMU driver. There are still some drivers which don't
+	 * support default domains, so the return value is not yet
+	 * checked.
+	 */
+	iommu_alloc_default_domain(group, dev);
+
+	if (group->default_domain)
+		ret = __iommu_attach_device(group->default_domain, dev);
+
+	iommu_create_device_direct_mappings(group, dev);
+
+	iommu_group_put(group);
+
+	if (ret)
+		goto err_release;
+
+	if (ops->probe_finalize)
+		ops->probe_finalize(dev);
+
+	return 0;
+
+err_release:
+	iommu_release_device(dev);
+
+err_out:
+	return ret;
 
-	if (dev->iommu) {
-		module_put(ops->owner);
-		dev_iommu_free(dev);
-	}
 }
 
-static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus,
-						 unsigned type);
-static int __iommu_attach_device(struct iommu_domain *domain,
-				 struct device *dev);
-static int __iommu_attach_group(struct iommu_domain *domain,
-				struct iommu_group *group);
-static void __iommu_detach_group(struct iommu_domain *domain,
-				 struct iommu_group *group);
+void iommu_release_device(struct device *dev)
+{
+	const struct iommu_ops *ops = dev->bus->iommu_ops;
+
+	if (!dev->iommu)
+		return;
+
+	iommu_device_unlink(dev->iommu->iommu_dev, dev);
+	iommu_group_remove_device(dev);
+
+	ops->release_device(dev);
+
+	module_put(ops->owner);
+	dev_iommu_free(dev);
+}
 
 static int __init iommu_set_def_domain_type(char *str)
 {
@@ -497,6 +573,7 @@ struct iommu_group *iommu_group_alloc(void)
 	group->kobj.kset = iommu_group_kset;
 	mutex_init(&group->mutex);
 	INIT_LIST_HEAD(&group->devices);
+	INIT_LIST_HEAD(&group->entry);
 	BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
 
 	ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL);
@@ -638,8 +715,8 @@ int iommu_group_set_name(struct iommu_group *group, const char *name)
 }
 EXPORT_SYMBOL_GPL(iommu_group_set_name);
 
-static int iommu_group_create_direct_mappings(struct iommu_group *group,
-					      struct device *dev)
+static int iommu_create_device_direct_mappings(struct iommu_group *group,
+					       struct device *dev)
 {
 	struct iommu_domain *domain = group->default_domain;
 	struct iommu_resv_region *entry;
@@ -752,8 +829,6 @@ rename:
 
 	dev->iommu_group = group;
 
-	iommu_group_create_direct_mappings(group, dev);
-
 	mutex_lock(&group->mutex);
 	list_add_tail(&device->list, &group->devices);
 	if (group->domain  && !iommu_is_attach_deferred(group->domain, dev))
@@ -1371,6 +1446,61 @@ struct iommu_group *fsl_mc_device_group(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(fsl_mc_device_group);
 
+static int iommu_get_def_domain_type(struct device *dev)
+{
+	const struct iommu_ops *ops = dev->bus->iommu_ops;
+	unsigned int type = 0;
+
+	if (ops->def_domain_type)
+		type = ops->def_domain_type(dev);
+
+	return (type == 0) ? iommu_def_domain_type : type;
+}
+
+static int iommu_group_alloc_default_domain(struct bus_type *bus,
+					    struct iommu_group *group,
+					    unsigned int type)
+{
+	struct iommu_domain *dom;
+
+	dom = __iommu_domain_alloc(bus, type);
+	if (!dom && type != IOMMU_DOMAIN_DMA) {
+		dom = __iommu_domain_alloc(bus, IOMMU_DOMAIN_DMA);
+		if (dom)
+			pr_warn("Failed to allocate default IOMMU domain of type %u for group %s - Falling back to IOMMU_DOMAIN_DMA",
+				type, group->name);
+	}
+
+	if (!dom)
+		return -ENOMEM;
+
+	group->default_domain = dom;
+	if (!group->domain)
+		group->domain = dom;
+
+	if (!iommu_dma_strict) {
+		int attr = 1;
+		iommu_domain_set_attr(dom,
+				      DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
+				      &attr);
+	}
+
+	return 0;
+}
+
+static int iommu_alloc_default_domain(struct iommu_group *group,
+				      struct device *dev)
+{
+	unsigned int type;
+
+	if (group->default_domain)
+		return 0;
+
+	type = iommu_get_def_domain_type(dev);
+
+	return iommu_group_alloc_default_domain(dev->bus, group, type);
+}
+
 /**
  * iommu_group_get_for_dev - Find or create the IOMMU group for a device
  * @dev: target device
@@ -1381,7 +1511,7 @@ EXPORT_SYMBOL_GPL(fsl_mc_device_group);
  * to the returned IOMMU group, which will already include the provided
  * device.  The reference should be released with iommu_group_put().
  */
-struct iommu_group *iommu_group_get_for_dev(struct device *dev)
+static struct iommu_group *iommu_group_get_for_dev(struct device *dev)
 {
 	const struct iommu_ops *ops = dev->bus->iommu_ops;
 	struct iommu_group *group;
@@ -1401,59 +1531,37 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
 	if (IS_ERR(group))
 		return group;
 
-	/*
-	 * Try to allocate a default domain - needs support from the
-	 * IOMMU driver.
-	 */
-	if (!group->default_domain) {
-		struct iommu_domain *dom;
-
-		dom = __iommu_domain_alloc(dev->bus, iommu_def_domain_type);
-		if (!dom && iommu_def_domain_type != IOMMU_DOMAIN_DMA) {
-			dom = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_DMA);
-			if (dom) {
-				dev_warn(dev,
-					 "failed to allocate default IOMMU domain of type %u; falling back to IOMMU_DOMAIN_DMA",
-					 iommu_def_domain_type);
-			}
-		}
-
-		group->default_domain = dom;
-		if (!group->domain)
-			group->domain = dom;
-
-		if (dom && !iommu_dma_strict) {
-			int attr = 1;
-			iommu_domain_set_attr(dom,
-					      DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE,
-					      &attr);
-		}
-	}
-
 	ret = iommu_group_add_device(group, dev);
-	if (ret) {
-		iommu_group_put(group);
-		return ERR_PTR(ret);
-	}
+	if (ret)
+		goto out_put_group;
 
 	return group;
+
+out_put_group:
+	iommu_group_put(group);
+
+	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_GPL(iommu_group_get_for_dev);
 
 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)
 {
 	return group->default_domain;
 }
 
-static int add_iommu_group(struct device *dev, void *data)
+static int probe_iommu_group(struct device *dev, void *data)
 {
-	int ret = iommu_probe_device(dev);
+	struct list_head *group_list = data;
+	struct iommu_group *group;
+	int ret;
 
-	/*
-	 * We ignore -ENODEV errors for now, as they just mean that the
-	 * device is not translated by an IOMMU. We still care about
-	 * other errors and fail to initialize when they happen.
-	 */
+	/* Device is probed already if in a group */
+	group = iommu_group_get(dev);
+	if (group) {
+		iommu_group_put(group);
+		return 0;
+	}
+
+	ret = __iommu_probe_device(dev, group_list);
 	if (ret == -ENODEV)
 		ret = 0;
 
@@ -1519,10 +1627,152 @@ static int iommu_bus_notifier(struct notifier_block *nb,
 	return 0;
 }
 
+struct __group_domain_type {
+	struct device *dev;
+	unsigned int type;
+};
+
+static int probe_get_default_domain_type(struct device *dev, void *data)
+{
+	const struct iommu_ops *ops = dev->bus->iommu_ops;
+	struct __group_domain_type *gtype = data;
+	unsigned int type = 0;
+
+	if (ops->def_domain_type)
+		type = ops->def_domain_type(dev);
+
+	if (type) {
+		if (gtype->type && gtype->type != type) {
+			dev_warn(dev, "Device needs domain type %s, but device %s in the same iommu group requires type %s - using default\n",
+				 iommu_domain_type_str(type),
+				 dev_name(gtype->dev),
+				 iommu_domain_type_str(gtype->type));
+			gtype->type = 0;
+		}
+
+		if (!gtype->dev) {
+			gtype->dev  = dev;
+			gtype->type = type;
+		}
+	}
+
+	return 0;
+}
+
+static void probe_alloc_default_domain(struct bus_type *bus,
+				       struct iommu_group *group)
+{
+	struct __group_domain_type gtype;
+
+	memset(&gtype, 0, sizeof(gtype));
+
+	/* Ask for default domain requirements of all devices in the group */
+	__iommu_group_for_each_dev(group, &gtype,
+				   probe_get_default_domain_type);
+
+	if (!gtype.type)
+		gtype.type = iommu_def_domain_type;
+
+	iommu_group_alloc_default_domain(bus, group, gtype.type);
+
+}
+
+static int iommu_group_do_dma_attach(struct device *dev, void *data)
+{
+	struct iommu_domain *domain = data;
+	int ret = 0;
+
+	if (!iommu_is_attach_deferred(domain, dev))
+		ret = __iommu_attach_device(domain, dev);
+
+	return ret;
+}
+
+static int __iommu_group_dma_attach(struct iommu_group *group)
+{
+	return __iommu_group_for_each_dev(group, group->default_domain,
+					  iommu_group_do_dma_attach);
+}
+
+static int iommu_group_do_probe_finalize(struct device *dev, void *data)
+{
+	struct iommu_domain *domain = data;
+
+	if (domain->ops->probe_finalize)
+		domain->ops->probe_finalize(dev);
+
+	return 0;
+}
+
+static void __iommu_group_dma_finalize(struct iommu_group *group)
+{
+	__iommu_group_for_each_dev(group, group->default_domain,
+				   iommu_group_do_probe_finalize);
+}
+
+static int iommu_do_create_direct_mappings(struct device *dev, void *data)
+{
+	struct iommu_group *group = data;
+
+	iommu_create_device_direct_mappings(group, dev);
+
+	return 0;
+}
+
+static int iommu_group_create_direct_mappings(struct iommu_group *group)
+{
+	return __iommu_group_for_each_dev(group, group,
+					  iommu_do_create_direct_mappings);
+}
+
+int bus_iommu_probe(struct bus_type *bus)
+{
+	struct iommu_group *group, *next;
+	LIST_HEAD(group_list);
+	int ret;
+
+	/*
+	 * This code-path does not allocate the default domain when
+	 * creating the iommu group, so do it after the groups are
+	 * created.
+	 */
+	ret = bus_for_each_dev(bus, NULL, &group_list, probe_iommu_group);
+	if (ret)
+		return ret;
+
+	list_for_each_entry_safe(group, next, &group_list, entry) {
+		/* Remove item from the list */
+		list_del_init(&group->entry);
+
+		mutex_lock(&group->mutex);
+
+		/* Try to allocate default domain */
+		probe_alloc_default_domain(bus, group);
+
+		if (!group->default_domain) {
+			mutex_unlock(&group->mutex);
+			continue;
+		}
+
+		iommu_group_create_direct_mappings(group);
+
+		ret = __iommu_group_dma_attach(group);
+
+		mutex_unlock(&group->mutex);
+
+		if (ret)
+			break;
+
+		__iommu_group_dma_finalize(group);
+	}
+
+	return ret;
+}
+
 static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
 {
-	int err;
 	struct notifier_block *nb;
+	int err;
 
 	nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
 	if (!nb)
@@ -1534,7 +1784,7 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops)
 	if (err)
 		goto out_free;
 
-	err = bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
+	err = bus_iommu_probe(bus);
 	if (err)
 		goto out_err;
 
@@ -2301,71 +2551,6 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
 }
 EXPORT_SYMBOL_GPL(iommu_alloc_resv_region);
 
-static int
-request_default_domain_for_dev(struct device *dev, unsigned long type)
-{
-	struct iommu_domain *domain;
-	struct iommu_group *group;
-	int ret;
-
-	/* Device must already be in a group before calling this function */
-	group = iommu_group_get(dev);
-	if (!group)
-		return -EINVAL;
-
-	mutex_lock(&group->mutex);
-
-	ret = 0;
-	if (group->default_domain && group->default_domain->type == type)
-		goto out;
-
-	/* Don't change mappings of existing devices */
-	ret = -EBUSY;
-	if (iommu_group_device_count(group) != 1)
-		goto out;
-
-	ret = -ENOMEM;
-	domain = __iommu_domain_alloc(dev->bus, type);
-	if (!domain)
-		goto out;
-
-	/* Attach the device to the domain */
-	ret = __iommu_attach_group(domain, group);
-	if (ret) {
-		iommu_domain_free(domain);
-		goto out;
-	}
-
-	/* Make the domain the default for this group */
-	if (group->default_domain)
-		iommu_domain_free(group->default_domain);
-	group->default_domain = domain;
-
-	iommu_group_create_direct_mappings(group, dev);
-
-	dev_info(dev, "Using iommu %s mapping\n",
-		 type == IOMMU_DOMAIN_DMA ? "dma" : "direct");
-
-	ret = 0;
-out:
-	mutex_unlock(&group->mutex);
-	iommu_group_put(group);
-
-	return ret;
-}
-
-/* Request that a device is direct mapped by the IOMMU */
-int iommu_request_dm_for_dev(struct device *dev)
-{
-	return request_default_domain_for_dev(dev, IOMMU_DOMAIN_IDENTITY);
-}
-
-/* Request that a device can't be direct mapped by the IOMMU */
-int iommu_request_dma_domain_for_dev(struct device *dev)
-{
-	return request_default_domain_for_dev(dev, IOMMU_DOMAIN_DMA);
-}
-
 void iommu_set_default_passthrough(bool cmd_line)
 {
 	if (cmd_line)
@@ -2643,17 +2828,6 @@ void iommu_sva_unbind_device(struct iommu_sva *handle)
 }
 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
 
-int iommu_sva_set_ops(struct iommu_sva *handle,
-		      const struct iommu_sva_ops *sva_ops)
-{
-	if (handle->ops && handle->ops != sva_ops)
-		return -EEXIST;
-
-	handle->ops = sva_ops;
-	return 0;
-}
-EXPORT_SYMBOL_GPL(iommu_sva_set_ops);
-
 int iommu_sva_get_pasid(struct iommu_sva *handle)
 {
 	const struct iommu_ops *ops = handle->dev->bus->iommu_ops;
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 0e6a9536eca6..49fc01f2a28d 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -253,7 +253,7 @@ int iova_cache_get(void)
 			SLAB_HWCACHE_ALIGN, NULL);
 		if (!iova_cache) {
 			mutex_unlock(&iova_cache_mutex);
-			printk(KERN_ERR "Couldn't create iova cache\n");
+			pr_err("Couldn't create iova cache\n");
 			return -ENOMEM;
 		}
 	}
@@ -718,8 +718,8 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
 
 		new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi);
 		if (!new_iova)
-			printk(KERN_ERR "Reserve iova range %lx@%lx failed\n",
-				iova->pfn_lo, iova->pfn_lo);
+			pr_err("Reserve iova range %lx@%lx failed\n",
+			       iova->pfn_lo, iova->pfn_lo);
 	}
 	spin_unlock_irqrestore(&from->iova_rbtree_lock, flags);
 }
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 310cf09feea3..4c2972f3153b 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -805,24 +805,8 @@ static int ipmmu_of_xlate(struct device *dev,
 static int ipmmu_init_arm_mapping(struct device *dev)
 {
 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
-	struct iommu_group *group;
 	int ret;
 
-	/* Create a device group and add the device to it. */
-	group = iommu_group_alloc();
-	if (IS_ERR(group)) {
-		dev_err(dev, "Failed to allocate IOMMU group\n");
-		return PTR_ERR(group);
-	}
-
-	ret = iommu_group_add_device(group, dev);
-	iommu_group_put(group);
-
-	if (ret < 0) {
-		dev_err(dev, "Failed to add device to IPMMU group\n");
-		return ret;
-	}
-
 	/*
 	 * Create the ARM mapping, used by the ARM DMA mapping core to allocate
 	 * VAs. This will allocate a corresponding IOMMU domain.
@@ -856,48 +840,39 @@ static int ipmmu_init_arm_mapping(struct device *dev)
 	return 0;
 
 error:
-	iommu_group_remove_device(dev);
 	if (mmu->mapping)
 		arm_iommu_release_mapping(mmu->mapping);
 
 	return ret;
 }
 
-static int ipmmu_add_device(struct device *dev)
+static struct iommu_device *ipmmu_probe_device(struct device *dev)
 {
 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
-	struct iommu_group *group;
-	int ret;
 
 	/*
 	 * Only let through devices that have been verified in xlate()
 	 */
 	if (!mmu)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
-	if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) {
-		ret = ipmmu_init_arm_mapping(dev);
-		if (ret)
-			return ret;
-	} else {
-		group = iommu_group_get_for_dev(dev);
-		if (IS_ERR(group))
-			return PTR_ERR(group);
+	return &mmu->iommu;
+}
 
-		iommu_group_put(group);
-	}
+static void ipmmu_probe_finalize(struct device *dev)
+{
+	int ret = 0;
 
-	iommu_device_link(&mmu->iommu, dev);
-	return 0;
+	if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA))
+		ret = ipmmu_init_arm_mapping(dev);
+
+	if (ret)
+		dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n");
 }
 
-static void ipmmu_remove_device(struct device *dev)
+static void ipmmu_release_device(struct device *dev)
 {
-	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
-
-	iommu_device_unlink(&mmu->iommu, dev);
 	arm_iommu_detach_device(dev);
-	iommu_group_remove_device(dev);
 }
 
 static struct iommu_group *ipmmu_find_group(struct device *dev)
@@ -925,9 +900,11 @@ static const struct iommu_ops ipmmu_ops = {
 	.flush_iotlb_all = ipmmu_flush_iotlb_all,
 	.iotlb_sync = ipmmu_iotlb_sync,
 	.iova_to_phys = ipmmu_iova_to_phys,
-	.add_device = ipmmu_add_device,
-	.remove_device = ipmmu_remove_device,
-	.device_group = ipmmu_find_group,
+	.probe_device = ipmmu_probe_device,
+	.release_device = ipmmu_release_device,
+	.probe_finalize = ipmmu_probe_finalize,
+	.device_group = IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)
+			? generic_device_group : ipmmu_find_group,
 	.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
 	.of_xlate = ipmmu_of_xlate,
 };
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 94a6df1bddd6..3d8a63555c25 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -34,7 +34,7 @@ __asm__ __volatile__ (							\
 /* bitmap of the page sizes currently supported */
 #define MSM_IOMMU_PGSIZES	(SZ_4K | SZ_64K | SZ_1M | SZ_16M)
 
-DEFINE_SPINLOCK(msm_iommu_lock);
+static DEFINE_SPINLOCK(msm_iommu_lock);
 static LIST_HEAD(qcom_iommu_devices);
 static struct iommu_ops msm_iommu_ops;
 
@@ -388,43 +388,23 @@ static struct msm_iommu_dev *find_iommu_for_dev(struct device *dev)
 	return ret;
 }
 
-static int msm_iommu_add_device(struct device *dev)
+static struct iommu_device *msm_iommu_probe_device(struct device *dev)
 {
 	struct msm_iommu_dev *iommu;
-	struct iommu_group *group;
 	unsigned long flags;
 
 	spin_lock_irqsave(&msm_iommu_lock, flags);
 	iommu = find_iommu_for_dev(dev);
 	spin_unlock_irqrestore(&msm_iommu_lock, flags);
 
-	if (iommu)
-		iommu_device_link(&iommu->iommu, dev);
-	else
-		return -ENODEV;
-
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	iommu_group_put(group);
+	if (!iommu)
+		return ERR_PTR(-ENODEV);
 
-	return 0;
+	return &iommu->iommu;
 }
 
-static void msm_iommu_remove_device(struct device *dev)
+static void msm_iommu_release_device(struct device *dev)
 {
-	struct msm_iommu_dev *iommu;
-	unsigned long flags;
-
-	spin_lock_irqsave(&msm_iommu_lock, flags);
-	iommu = find_iommu_for_dev(dev);
-	spin_unlock_irqrestore(&msm_iommu_lock, flags);
-
-	if (iommu)
-		iommu_device_unlink(&iommu->iommu, dev);
-
-	iommu_group_remove_device(dev);
 }
 
 static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
@@ -708,8 +688,8 @@ static struct iommu_ops msm_iommu_ops = {
 	 */
 	.iotlb_sync = NULL,
 	.iova_to_phys = msm_iommu_iova_to_phys,
-	.add_device = msm_iommu_add_device,
-	.remove_device = msm_iommu_remove_device,
+	.probe_device = msm_iommu_probe_device,
+	.release_device = msm_iommu_release_device,
 	.device_group = generic_device_group,
 	.pgsize_bitmap = MSM_IOMMU_PGSIZES,
 	.of_xlate = qcom_iommu_of_xlate,
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index 5f4d6df59cf6..2be96f1cdbd2 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -441,38 +441,26 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
 	return pa;
 }
 
-static int mtk_iommu_add_device(struct device *dev)
+static struct iommu_device *mtk_iommu_probe_device(struct device *dev)
 {
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 	struct mtk_iommu_data *data;
-	struct iommu_group *group;
 
 	if (!fwspec || fwspec->ops != &mtk_iommu_ops)
-		return -ENODEV; /* Not a iommu client device */
+		return ERR_PTR(-ENODEV); /* Not a iommu client device */
 
 	data = dev_iommu_priv_get(dev);
-	iommu_device_link(&data->iommu, dev);
 
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	iommu_group_put(group);
-	return 0;
+	return &data->iommu;
 }
 
-static void mtk_iommu_remove_device(struct device *dev)
+static void mtk_iommu_release_device(struct device *dev)
 {
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
-	struct mtk_iommu_data *data;
 
 	if (!fwspec || fwspec->ops != &mtk_iommu_ops)
 		return;
 
-	data = dev_iommu_priv_get(dev);
-	iommu_device_unlink(&data->iommu, dev);
-
-	iommu_group_remove_device(dev);
 	iommu_fwspec_free(dev);
 }
 
@@ -526,8 +514,8 @@ static const struct iommu_ops mtk_iommu_ops = {
 	.flush_iotlb_all = mtk_iommu_flush_iotlb_all,
 	.iotlb_sync	= mtk_iommu_iotlb_sync,
 	.iova_to_phys	= mtk_iommu_iova_to_phys,
-	.add_device	= mtk_iommu_add_device,
-	.remove_device	= mtk_iommu_remove_device,
+	.probe_device	= mtk_iommu_probe_device,
+	.release_device	= mtk_iommu_release_device,
 	.device_group	= mtk_iommu_device_group,
 	.of_xlate	= mtk_iommu_of_xlate,
 	.pgsize_bitmap	= SZ_4K | SZ_64K | SZ_1M | SZ_16M,
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index a31be05601c9..c9d79cff4d17 100644
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -265,10 +265,13 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
 {
 	struct mtk_iommu_data *data = dev_iommu_priv_get(dev);
 	struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+	struct dma_iommu_mapping *mtk_mapping;
 	int ret;
 
-	if (!data)
-		return -ENODEV;
+	/* Only allow the domain created internally. */
+	mtk_mapping = data->dev->archdata.iommu;
+	if (mtk_mapping->domain != domain)
+		return 0;
 
 	if (!data->m4u_dom) {
 		data->m4u_dom = dom;
@@ -288,9 +291,6 @@ static void mtk_iommu_detach_device(struct iommu_domain *domain,
 {
 	struct mtk_iommu_data *data = dev_iommu_priv_get(dev);
 
-	if (!data)
-		return;
-
 	mtk_iommu_config(data, dev, false);
 }
 
@@ -416,14 +416,17 @@ static int mtk_iommu_create_mapping(struct device *dev,
 	return 0;
 }
 
-static int mtk_iommu_add_device(struct device *dev)
+static int mtk_iommu_def_domain_type(struct device *dev)
+{
+	return IOMMU_DOMAIN_UNMANAGED;
+}
+
+static struct iommu_device *mtk_iommu_probe_device(struct device *dev)
 {
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
-	struct dma_iommu_mapping *mtk_mapping;
 	struct of_phandle_args iommu_spec;
 	struct of_phandle_iterator it;
 	struct mtk_iommu_data *data;
-	struct iommu_group *group;
 	int err;
 
 	of_for_each_phandle(&it, err, dev->of_node, "iommus",
@@ -442,46 +445,34 @@ static int mtk_iommu_add_device(struct device *dev)
 	}
 
 	if (!fwspec || fwspec->ops != &mtk_iommu_ops)
-		return -ENODEV; /* Not a iommu client device */
+		return ERR_PTR(-ENODEV); /* Not a iommu client device */
 
-	/*
-	 * This is a short-term bodge because the ARM DMA code doesn't
-	 * understand multi-device groups, but we have to call into it
-	 * successfully (and not just rely on a normal IOMMU API attach
-	 * here) in order to set the correct DMA API ops on @dev.
-	 */
-	group = iommu_group_alloc();
-	if (IS_ERR(group))
-		return PTR_ERR(group);
+	data = dev_iommu_priv_get(dev);
 
-	err = iommu_group_add_device(group, dev);
-	iommu_group_put(group);
-	if (err)
-		return err;
+	return &data->iommu;
+}
 
-	data = dev_iommu_priv_get(dev);
+static void mtk_iommu_probe_finalize(struct device *dev)
+{
+	struct dma_iommu_mapping *mtk_mapping;
+	struct mtk_iommu_data *data;
+	int err;
+
+	data        = dev_iommu_priv_get(dev);
 	mtk_mapping = data->dev->archdata.iommu;
-	err = arm_iommu_attach_device(dev, mtk_mapping);
-	if (err) {
-		iommu_group_remove_device(dev);
-		return err;
-	}
 
-	return iommu_device_link(&data->iommu, dev);
+	err = arm_iommu_attach_device(dev, mtk_mapping);
+	if (err)
+		dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n");
 }
 
-static void mtk_iommu_remove_device(struct device *dev)
+static void mtk_iommu_release_device(struct device *dev)
 {
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
-	struct mtk_iommu_data *data;
 
 	if (!fwspec || fwspec->ops != &mtk_iommu_ops)
 		return;
 
-	data = dev_iommu_priv_get(dev);
-	iommu_device_unlink(&data->iommu, dev);
-
-	iommu_group_remove_device(dev);
 	iommu_fwspec_free(dev);
 }
 
@@ -534,8 +525,11 @@ static const struct iommu_ops mtk_iommu_ops = {
 	.map		= mtk_iommu_map,
 	.unmap		= mtk_iommu_unmap,
 	.iova_to_phys	= mtk_iommu_iova_to_phys,
-	.add_device	= mtk_iommu_add_device,
-	.remove_device	= mtk_iommu_remove_device,
+	.probe_device	= mtk_iommu_probe_device,
+	.probe_finalize = mtk_iommu_probe_finalize,
+	.release_device	= mtk_iommu_release_device,
+	.def_domain_type = mtk_iommu_def_domain_type,
+	.device_group	= generic_device_group,
 	.pgsize_bitmap	= ~0UL << MT2701_IOMMU_PAGE_SHIFT,
 };
 
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 887fefcb03b4..c8282cc212cb 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -35,15 +35,6 @@
 
 static const struct iommu_ops omap_iommu_ops;
 
-struct orphan_dev {
-	struct device *dev;
-	struct list_head node;
-};
-
-static LIST_HEAD(orphan_dev_list);
-
-static DEFINE_SPINLOCK(orphan_lock);
-
 #define to_iommu(dev)	((struct omap_iommu *)dev_get_drvdata(dev))
 
 /* bitmap of the page sizes currently supported */
@@ -62,8 +53,6 @@ static DEFINE_SPINLOCK(orphan_lock);
 static struct platform_driver omap_iommu_driver;
 static struct kmem_cache *iopte_cachep;
 
-static int _omap_iommu_add_device(struct device *dev);
-
 /**
  * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
  * @dom:	generic iommu domain handle
@@ -1177,7 +1166,6 @@ static int omap_iommu_probe(struct platform_device *pdev)
 	struct omap_iommu *obj;
 	struct resource *res;
 	struct device_node *of = pdev->dev.of_node;
-	struct orphan_dev *orphan_dev, *tmp;
 
 	if (!of) {
 		pr_err("%s: only DT-based devices are supported\n", __func__);
@@ -1248,6 +1236,7 @@ static int omap_iommu_probe(struct platform_device *pdev)
 			goto out_group;
 
 		iommu_device_set_ops(&obj->iommu, &omap_iommu_ops);
+		iommu_device_set_fwnode(&obj->iommu, &of->fwnode);
 
 		err = iommu_device_register(&obj->iommu);
 		if (err)
@@ -1260,13 +1249,8 @@ static int omap_iommu_probe(struct platform_device *pdev)
 
 	dev_info(&pdev->dev, "%s registered\n", obj->name);
 
-	list_for_each_entry_safe(orphan_dev, tmp, &orphan_dev_list, node) {
-		err = _omap_iommu_add_device(orphan_dev->dev);
-		if (!err) {
-			list_del(&orphan_dev->node);
-			kfree(orphan_dev);
-		}
-	}
+	/* Re-probe bus to probe device attached to this IOMMU */
+	bus_iommu_probe(&platform_bus_type);
 
 	return 0;
 
@@ -1657,17 +1641,13 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
 	return ret;
 }
 
-static int _omap_iommu_add_device(struct device *dev)
+static struct iommu_device *omap_iommu_probe_device(struct device *dev)
 {
 	struct omap_iommu_arch_data *arch_data, *tmp;
+	struct platform_device *pdev;
 	struct omap_iommu *oiommu;
-	struct iommu_group *group;
 	struct device_node *np;
-	struct platform_device *pdev;
 	int num_iommus, i;
-	int ret;
-	struct orphan_dev *orphan_dev;
-	unsigned long flags;
 
 	/*
 	 * Allocate the archdata iommu structure for DT-based devices.
@@ -1676,7 +1656,7 @@ static int _omap_iommu_add_device(struct device *dev)
 	 * IOMMU users.
 	 */
 	if (!dev->of_node)
-		return 0;
+		return ERR_PTR(-ENODEV);
 
 	/*
 	 * retrieve the count of IOMMU nodes using phandle size as element size
@@ -1689,43 +1669,27 @@ static int _omap_iommu_add_device(struct device *dev)
 
 	arch_data = kcalloc(num_iommus + 1, sizeof(*arch_data), GFP_KERNEL);
 	if (!arch_data)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	for (i = 0, tmp = arch_data; i < num_iommus; i++, tmp++) {
 		np = of_parse_phandle(dev->of_node, "iommus", i);
 		if (!np) {
 			kfree(arch_data);
-			return -EINVAL;
+			return ERR_PTR(-EINVAL);
 		}
 
 		pdev = of_find_device_by_node(np);
 		if (!pdev) {
 			of_node_put(np);
 			kfree(arch_data);
-			spin_lock_irqsave(&orphan_lock, flags);
-			list_for_each_entry(orphan_dev, &orphan_dev_list,
-					    node) {
-				if (orphan_dev->dev == dev)
-					break;
-			}
-			spin_unlock_irqrestore(&orphan_lock, flags);
-
-			if (orphan_dev && orphan_dev->dev == dev)
-				return -EPROBE_DEFER;
-
-			orphan_dev = kzalloc(sizeof(*orphan_dev), GFP_KERNEL);
-			orphan_dev->dev = dev;
-			spin_lock_irqsave(&orphan_lock, flags);
-			list_add(&orphan_dev->node, &orphan_dev_list);
-			spin_unlock_irqrestore(&orphan_lock, flags);
-			return -EPROBE_DEFER;
+			return ERR_PTR(-ENODEV);
 		}
 
 		oiommu = platform_get_drvdata(pdev);
 		if (!oiommu) {
 			of_node_put(np);
 			kfree(arch_data);
-			return -EINVAL;
+			return ERR_PTR(-EINVAL);
 		}
 
 		tmp->iommu_dev = oiommu;
@@ -1734,57 +1698,25 @@ static int _omap_iommu_add_device(struct device *dev)
 		of_node_put(np);
 	}
 
+	dev->archdata.iommu = arch_data;
+
 	/*
 	 * use the first IOMMU alone for the sysfs device linking.
 	 * TODO: Evaluate if a single iommu_group needs to be
 	 * maintained for both IOMMUs
 	 */
 	oiommu = arch_data->iommu_dev;
-	ret = iommu_device_link(&oiommu->iommu, dev);
-	if (ret) {
-		kfree(arch_data);
-		return ret;
-	}
-
-	dev->archdata.iommu = arch_data;
-
-	/*
-	 * IOMMU group initialization calls into omap_iommu_device_group, which
-	 * needs a valid dev->archdata.iommu pointer
-	 */
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group)) {
-		iommu_device_unlink(&oiommu->iommu, dev);
-		dev->archdata.iommu = NULL;
-		kfree(arch_data);
-		return PTR_ERR(group);
-	}
-	iommu_group_put(group);
 
-	return 0;
+	return &oiommu->iommu;
 }
 
-static int omap_iommu_add_device(struct device *dev)
-{
-	int ret;
-
-	ret = _omap_iommu_add_device(dev);
-	if (ret == -EPROBE_DEFER)
-		return 0;
-
-	return ret;
-}
-
-static void omap_iommu_remove_device(struct device *dev)
+static void omap_iommu_release_device(struct device *dev)
 {
 	struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
 
 	if (!dev->of_node || !arch_data)
 		return;
 
-	iommu_device_unlink(&arch_data->iommu_dev->iommu, dev);
-	iommu_group_remove_device(dev);
-
 	dev->archdata.iommu = NULL;
 	kfree(arch_data);
 
@@ -1795,6 +1727,9 @@ static struct iommu_group *omap_iommu_device_group(struct device *dev)
 	struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
 	struct iommu_group *group = ERR_PTR(-EINVAL);
 
+	if (!arch_data)
+		return ERR_PTR(-ENODEV);
+
 	if (arch_data->iommu_dev)
 		group = iommu_group_ref_get(arch_data->iommu_dev->group);
 
@@ -1809,8 +1744,8 @@ static const struct iommu_ops omap_iommu_ops = {
 	.map		= omap_iommu_map,
 	.unmap		= omap_iommu_unmap,
 	.iova_to_phys	= omap_iommu_iova_to_phys,
-	.add_device	= omap_iommu_add_device,
-	.remove_device	= omap_iommu_remove_device,
+	.probe_device	= omap_iommu_probe_device,
+	.release_device	= omap_iommu_release_device,
 	.device_group	= omap_iommu_device_group,
 	.pgsize_bitmap	= OMAP_IOMMU_PGSIZES,
 };
diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
index 5b3b270972f8..c3e1fbd1988c 100644
--- a/drivers/iommu/qcom_iommu.c
+++ b/drivers/iommu/qcom_iommu.c
@@ -524,14 +524,13 @@ static bool qcom_iommu_capable(enum iommu_cap cap)
 	}
 }
 
-static int qcom_iommu_add_device(struct device *dev)
+static struct iommu_device *qcom_iommu_probe_device(struct device *dev)
 {
 	struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
-	struct iommu_group *group;
 	struct device_link *link;
 
 	if (!qcom_iommu)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	/*
 	 * Establish the link between iommu and master, so that the
@@ -542,28 +541,19 @@ static int qcom_iommu_add_device(struct device *dev)
 	if (!link) {
 		dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
 			dev_name(qcom_iommu->dev), dev_name(dev));
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 	}
 
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	iommu_group_put(group);
-	iommu_device_link(&qcom_iommu->iommu, dev);
-
-	return 0;
+	return &qcom_iommu->iommu;
 }
 
-static void qcom_iommu_remove_device(struct device *dev)
+static void qcom_iommu_release_device(struct device *dev)
 {
 	struct qcom_iommu_dev *qcom_iommu = to_iommu(dev);
 
 	if (!qcom_iommu)
 		return;
 
-	iommu_device_unlink(&qcom_iommu->iommu, dev);
-	iommu_group_remove_device(dev);
 	iommu_fwspec_free(dev);
 }
 
@@ -619,8 +609,8 @@ static const struct iommu_ops qcom_iommu_ops = {
 	.flush_iotlb_all = qcom_iommu_flush_iotlb_all,
 	.iotlb_sync	= qcom_iommu_iotlb_sync,
 	.iova_to_phys	= qcom_iommu_iova_to_phys,
-	.add_device	= qcom_iommu_add_device,
-	.remove_device	= qcom_iommu_remove_device,
+	.probe_device	= qcom_iommu_probe_device,
+	.release_device	= qcom_iommu_release_device,
 	.device_group	= generic_device_group,
 	.of_xlate	= qcom_iommu_of_xlate,
 	.pgsize_bitmap	= SZ_4K | SZ_64K | SZ_1M | SZ_16M,
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index b33cdd5aad81..d25c2486ca07 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -1054,40 +1054,28 @@ static void rk_iommu_domain_free(struct iommu_domain *domain)
 	kfree(rk_domain);
 }
 
-static int rk_iommu_add_device(struct device *dev)
+static struct iommu_device *rk_iommu_probe_device(struct device *dev)
 {
-	struct iommu_group *group;
-	struct rk_iommu *iommu;
 	struct rk_iommudata *data;
+	struct rk_iommu *iommu;
 
 	data = dev->archdata.iommu;
 	if (!data)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	iommu = rk_iommu_from_dev(dev);
 
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-	iommu_group_put(group);
-
-	iommu_device_link(&iommu->iommu, dev);
 	data->link = device_link_add(dev, iommu->dev,
 				     DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
 
-	return 0;
+	return &iommu->iommu;
 }
 
-static void rk_iommu_remove_device(struct device *dev)
+static void rk_iommu_release_device(struct device *dev)
 {
-	struct rk_iommu *iommu;
 	struct rk_iommudata *data = dev->archdata.iommu;
 
-	iommu = rk_iommu_from_dev(dev);
-
 	device_link_del(data->link);
-	iommu_device_unlink(&iommu->iommu, dev);
-	iommu_group_remove_device(dev);
 }
 
 static struct iommu_group *rk_iommu_device_group(struct device *dev)
@@ -1126,8 +1114,8 @@ static const struct iommu_ops rk_iommu_ops = {
 	.detach_dev = rk_iommu_detach_device,
 	.map = rk_iommu_map,
 	.unmap = rk_iommu_unmap,
-	.add_device = rk_iommu_add_device,
-	.remove_device = rk_iommu_remove_device,
+	.probe_device = rk_iommu_probe_device,
+	.release_device = rk_iommu_release_device,
 	.iova_to_phys = rk_iommu_iova_to_phys,
 	.device_group = rk_iommu_device_group,
 	.pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 1137f3ddcb85..8895dbb705eb 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -87,7 +87,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
 				    struct device *dev)
 {
 	struct s390_domain *s390_domain = to_s390_domain(domain);
-	struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
+	struct zpci_dev *zdev = to_zpci_dev(dev);
 	struct s390_domain_device *domain_device;
 	unsigned long flags;
 	int rc;
@@ -139,7 +139,7 @@ static void s390_iommu_detach_device(struct iommu_domain *domain,
 				     struct device *dev)
 {
 	struct s390_domain *s390_domain = to_s390_domain(domain);
-	struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
+	struct zpci_dev *zdev = to_zpci_dev(dev);
 	struct s390_domain_device *domain_device, *tmp;
 	unsigned long flags;
 	int found = 0;
@@ -166,23 +166,16 @@ static void s390_iommu_detach_device(struct iommu_domain *domain,
 	}
 }
 
-static int s390_iommu_add_device(struct device *dev)
+static struct iommu_device *s390_iommu_probe_device(struct device *dev)
 {
-	struct iommu_group *group = iommu_group_get_for_dev(dev);
-	struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
+	struct zpci_dev *zdev = to_zpci_dev(dev);
 
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	iommu_group_put(group);
-	iommu_device_link(&zdev->iommu_dev, dev);
-
-	return 0;
+	return &zdev->iommu_dev;
 }
 
-static void s390_iommu_remove_device(struct device *dev)
+static void s390_iommu_release_device(struct device *dev)
 {
-	struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
+	struct zpci_dev *zdev = to_zpci_dev(dev);
 	struct iommu_domain *domain;
 
 	/*
@@ -191,7 +184,7 @@ static void s390_iommu_remove_device(struct device *dev)
 	 * to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
 	 * the attach_dev), removing the device via
 	 * "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
-	 * only remove_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
+	 * only release_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
 	 * notifier.
 	 *
 	 * So let's call detach_dev from here if it hasn't been called before.
@@ -201,9 +194,6 @@ static void s390_iommu_remove_device(struct device *dev)
 		if (domain)
 			s390_iommu_detach_device(domain, dev);
 	}
-
-	iommu_device_unlink(&zdev->iommu_dev, dev);
-	iommu_group_remove_device(dev);
 }
 
 static int s390_iommu_update_trans(struct s390_domain *s390_domain,
@@ -373,8 +363,8 @@ static const struct iommu_ops s390_iommu_ops = {
 	.map = s390_iommu_map,
 	.unmap = s390_iommu_unmap,
 	.iova_to_phys = s390_iommu_iova_to_phys,
-	.add_device = s390_iommu_add_device,
-	.remove_device = s390_iommu_remove_device,
+	.probe_device = s390_iommu_probe_device,
+	.release_device = s390_iommu_release_device,
 	.device_group = generic_device_group,
 	.pgsize_bitmap = S390_IOMMU_PGSIZES,
 };
diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c
new file mode 100644
index 000000000000..fce605e96aa2
--- /dev/null
+++ b/drivers/iommu/sun50i-iommu.c
@@ -0,0 +1,1023 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+// Copyright (C) 2016-2018, Allwinner Technology CO., LTD.
+// Copyright (C) 2019-2020, Cerno
+
+#include <linux/bitfield.h>
+#include <linux/bug.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-iommu.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/iommu.h>
+#include <linux/iopoll.h>
+#include <linux/ioport.h>
+#include <linux/log2.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+#define IOMMU_RESET_REG			0x010
+#define IOMMU_ENABLE_REG		0x020
+#define IOMMU_ENABLE_ENABLE			BIT(0)
+
+#define IOMMU_BYPASS_REG		0x030
+#define IOMMU_AUTO_GATING_REG		0x040
+#define IOMMU_AUTO_GATING_ENABLE		BIT(0)
+
+#define IOMMU_WBUF_CTRL_REG		0x044
+#define IOMMU_OOO_CTRL_REG		0x048
+#define IOMMU_4KB_BDY_PRT_CTRL_REG	0x04c
+#define IOMMU_TTB_REG			0x050
+#define IOMMU_TLB_ENABLE_REG		0x060
+#define IOMMU_TLB_PREFETCH_REG		0x070
+#define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m)	BIT(m)
+
+#define IOMMU_TLB_FLUSH_REG		0x080
+#define IOMMU_TLB_FLUSH_PTW_CACHE		BIT(17)
+#define IOMMU_TLB_FLUSH_MACRO_TLB		BIT(16)
+#define IOMMU_TLB_FLUSH_MICRO_TLB(i)		(BIT(i) & GENMASK(5, 0))
+
+#define IOMMU_TLB_IVLD_ADDR_REG		0x090
+#define IOMMU_TLB_IVLD_ADDR_MASK_REG	0x094
+#define IOMMU_TLB_IVLD_ENABLE_REG	0x098
+#define IOMMU_TLB_IVLD_ENABLE_ENABLE		BIT(0)
+
+#define IOMMU_PC_IVLD_ADDR_REG		0x0a0
+#define IOMMU_PC_IVLD_ENABLE_REG	0x0a8
+#define IOMMU_PC_IVLD_ENABLE_ENABLE		BIT(0)
+
+#define IOMMU_DM_AUT_CTRL_REG(d)	(0x0b0 + ((d) / 2) * 4)
+#define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m)	(1 << (((d & 1) * 16) + ((m) * 2)))
+#define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m)	(1 << (((d & 1) * 16) + ((m) * 2) + 1))
+
+#define IOMMU_DM_AUT_OVWT_REG		0x0d0
+#define IOMMU_INT_ENABLE_REG		0x100
+#define IOMMU_INT_CLR_REG		0x104
+#define IOMMU_INT_STA_REG		0x108
+#define IOMMU_INT_ERR_ADDR_REG(i)	(0x110 + (i) * 4)
+#define IOMMU_INT_ERR_ADDR_L1_REG	0x130
+#define IOMMU_INT_ERR_ADDR_L2_REG	0x134
+#define IOMMU_INT_ERR_DATA_REG(i)	(0x150 + (i) * 4)
+#define IOMMU_L1PG_INT_REG		0x0180
+#define IOMMU_L2PG_INT_REG		0x0184
+
+#define IOMMU_INT_INVALID_L2PG			BIT(17)
+#define IOMMU_INT_INVALID_L1PG			BIT(16)
+#define IOMMU_INT_MASTER_PERMISSION(m)		BIT(m)
+#define IOMMU_INT_MASTER_MASK			(IOMMU_INT_MASTER_PERMISSION(0) | \
+						 IOMMU_INT_MASTER_PERMISSION(1) | \
+						 IOMMU_INT_MASTER_PERMISSION(2) | \
+						 IOMMU_INT_MASTER_PERMISSION(3) | \
+						 IOMMU_INT_MASTER_PERMISSION(4) | \
+						 IOMMU_INT_MASTER_PERMISSION(5))
+#define IOMMU_INT_MASK				(IOMMU_INT_INVALID_L1PG | \
+						 IOMMU_INT_INVALID_L2PG | \
+						 IOMMU_INT_MASTER_MASK)
+
+#define PT_ENTRY_SIZE			sizeof(u32)
+
+#define NUM_DT_ENTRIES			4096
+#define DT_SIZE				(NUM_DT_ENTRIES * PT_ENTRY_SIZE)
+
+#define NUM_PT_ENTRIES			256
+#define PT_SIZE				(NUM_PT_ENTRIES * PT_ENTRY_SIZE)
+
+struct sun50i_iommu {
+	struct iommu_device iommu;
+
+	/* Lock to modify the IOMMU registers */
+	spinlock_t iommu_lock;
+
+	struct device *dev;
+	void __iomem *base;
+	struct reset_control *reset;
+	struct clk *clk;
+
+	struct iommu_domain *domain;
+	struct iommu_group *group;
+	struct kmem_cache *pt_pool;
+};
+
+struct sun50i_iommu_domain {
+	struct iommu_domain domain;
+
+	/* Number of devices attached to the domain */
+	refcount_t refcnt;
+
+	/* L1 Page Table */
+	u32 *dt;
+	dma_addr_t dt_dma;
+
+	struct sun50i_iommu *iommu;
+};
+
+static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
+{
+	return container_of(domain, struct sun50i_iommu_domain, domain);
+}
+
+static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
+{
+	return dev_iommu_priv_get(dev);
+}
+
+static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
+{
+	return readl(iommu->base + offset);
+}
+
+static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
+{
+	writel(value, iommu->base + offset);
+}
+
+/*
+ * The Allwinner H6 IOMMU uses a 2-level page table.
+ *
+ * The first level is the usual Directory Table (DT), that consists of
+ * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page
+ * Table (PT).
+ *
+ * Each PT consits of 256 4-bytes Page Table Entries (PTE), each
+ * pointing to a 4kB page of physical memory.
+ *
+ * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG
+ * register that contains its physical address.
+ */
+
+#define SUN50I_IOVA_DTE_MASK	GENMASK(31, 20)
+#define SUN50I_IOVA_PTE_MASK	GENMASK(19, 12)
+#define SUN50I_IOVA_PAGE_MASK	GENMASK(11, 0)
+
+static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
+{
+	return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
+}
+
+static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
+{
+	return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
+}
+
+static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
+{
+	return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
+}
+
+/*
+ * Each Directory Table Entry has a Page Table address and a valid
+ * bit:
+
+ * +---------------------+-----------+-+
+ * | PT address          | Reserved  |V|
+ * +---------------------+-----------+-+
+ *  31:10 - Page Table address
+ *   9:2  - Reserved
+ *   1:0  - 1 if the entry is valid
+ */
+
+#define SUN50I_DTE_PT_ADDRESS_MASK	GENMASK(31, 10)
+#define SUN50I_DTE_PT_ATTRS		GENMASK(1, 0)
+#define SUN50I_DTE_PT_VALID		1
+
+static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
+{
+	return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
+}
+
+static bool sun50i_dte_is_pt_valid(u32 dte)
+{
+	return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
+}
+
+static u32 sun50i_mk_dte(dma_addr_t pt_dma)
+{
+	return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
+}
+
+/*
+ * Each PTE has a Page address, an authority index and a valid bit:
+ *
+ * +----------------+-----+-----+-----+---+-----+
+ * | Page address   | Rsv | ACI | Rsv | V | Rsv |
+ * +----------------+-----+-----+-----+---+-----+
+ *  31:12 - Page address
+ *  11:8  - Reserved
+ *   7:4  - Authority Control Index
+ *   3:2  - Reserved
+ *     1  - 1 if the entry is valid
+ *     0  - Reserved
+ *
+ * The way permissions work is that the IOMMU has 16 "domains" that
+ * can be configured to give each masters either read or write
+ * permissions through the IOMMU_DM_AUT_CTRL_REG registers. The domain
+ * 0 seems like the default domain, and its permissions in the
+ * IOMMU_DM_AUT_CTRL_REG are only read-only, so it's not really
+ * useful to enforce any particular permission.
+ *
+ * Each page entry will then have a reference to the domain they are
+ * affected to, so that we can actually enforce them on a per-page
+ * basis.
+ *
+ * In order to make it work with the IOMMU framework, we will be using
+ * 4 different domains, starting at 1: RD_WR, RD, WR and NONE
+ * depending on the permission we want to enforce. Each domain will
+ * have each master setup in the same way, since the IOMMU framework
+ * doesn't seem to restrict page access on a per-device basis. And
+ * then we will use the relevant domain index when generating the page
+ * table entry depending on the permissions we want to be enforced.
+ */
+
+enum sun50i_iommu_aci {
+	SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
+	SUN50I_IOMMU_ACI_NONE,
+	SUN50I_IOMMU_ACI_RD,
+	SUN50I_IOMMU_ACI_WR,
+	SUN50I_IOMMU_ACI_RD_WR,
+};
+
+#define SUN50I_PTE_PAGE_ADDRESS_MASK	GENMASK(31, 12)
+#define SUN50I_PTE_ACI_MASK		GENMASK(7, 4)
+#define SUN50I_PTE_PAGE_VALID		BIT(1)
+
+static phys_addr_t sun50i_pte_get_page_address(u32 pte)
+{
+	return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
+}
+
+static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
+{
+	return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
+}
+
+static bool sun50i_pte_is_page_valid(u32 pte)
+{
+	return pte & SUN50I_PTE_PAGE_VALID;
+}
+
+static u32 sun50i_mk_pte(phys_addr_t page, int prot)
+{
+	enum sun50i_iommu_aci aci;
+	u32 flags = 0;
+
+	if (prot & (IOMMU_READ | IOMMU_WRITE))
+		aci = SUN50I_IOMMU_ACI_RD_WR;
+	else if (prot & IOMMU_READ)
+		aci = SUN50I_IOMMU_ACI_RD;
+	else if (prot & IOMMU_WRITE)
+		aci = SUN50I_IOMMU_ACI_WR;
+	else
+		aci = SUN50I_IOMMU_ACI_NONE;
+
+	flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
+	page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
+	return page | flags | SUN50I_PTE_PAGE_VALID;
+}
+
+static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
+			       void *vaddr, unsigned int count)
+{
+	struct sun50i_iommu *iommu = sun50i_domain->iommu;
+	dma_addr_t dma = virt_to_phys(vaddr);
+	size_t size = count * PT_ENTRY_SIZE;
+
+	dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
+}
+
+static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
+{
+	u32 reg;
+	int ret;
+
+	assert_spin_locked(&iommu->iommu_lock);
+
+	iommu_write(iommu,
+		    IOMMU_TLB_FLUSH_REG,
+		    IOMMU_TLB_FLUSH_PTW_CACHE |
+		    IOMMU_TLB_FLUSH_MACRO_TLB |
+		    IOMMU_TLB_FLUSH_MICRO_TLB(5) |
+		    IOMMU_TLB_FLUSH_MICRO_TLB(4) |
+		    IOMMU_TLB_FLUSH_MICRO_TLB(3) |
+		    IOMMU_TLB_FLUSH_MICRO_TLB(2) |
+		    IOMMU_TLB_FLUSH_MICRO_TLB(1) |
+		    IOMMU_TLB_FLUSH_MICRO_TLB(0));
+
+	ret = readl_poll_timeout(iommu->base + IOMMU_TLB_FLUSH_REG,
+				 reg, !reg,
+				 1, 2000);
+	if (ret)
+		dev_warn(iommu->dev, "TLB Flush timed out!\n");
+
+	return ret;
+}
+
+static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
+{
+	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
+	struct sun50i_iommu *iommu = sun50i_domain->iommu;
+	unsigned long flags;
+
+	/*
+	 * At boot, we'll have a first call into .flush_iotlb_all right after
+	 * .probe_device, and since we link our (single) domain to our iommu in
+	 * the .attach_device callback, we don't have that pointer set.
+	 *
+	 * It shouldn't really be any trouble to ignore it though since we flush
+	 * all caches as part of the device powerup.
+	 */
+	if (!iommu)
+		return;
+
+	spin_lock_irqsave(&iommu->iommu_lock, flags);
+	sun50i_iommu_flush_all_tlb(iommu);
+	spin_unlock_irqrestore(&iommu->iommu_lock, flags);
+}
+
+static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
+				    struct iommu_iotlb_gather *gather)
+{
+	sun50i_iommu_flush_iotlb_all(domain);
+}
+
+static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
+{
+	struct sun50i_iommu_domain *sun50i_domain;
+	unsigned long flags;
+	int ret;
+
+	if (!iommu->domain)
+		return 0;
+
+	sun50i_domain = to_sun50i_domain(iommu->domain);
+
+	ret = reset_control_deassert(iommu->reset);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(iommu->clk);
+	if (ret)
+		goto err_reset_assert;
+
+	spin_lock_irqsave(&iommu->iommu_lock, flags);
+
+	iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
+	iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
+		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
+		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
+		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
+		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
+		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
+		    IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
+	iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
+	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));
+
+	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
+		    IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));
+
+	iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
+		    IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));
+
+	ret = sun50i_iommu_flush_all_tlb(iommu);
+	if (ret) {
+		spin_unlock_irqrestore(&iommu->iommu_lock, flags);
+		goto err_clk_disable;
+	}
+
+	iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
+	iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);
+
+	spin_unlock_irqrestore(&iommu->iommu_lock, flags);
+
+	return 0;
+
+err_clk_disable:
+	clk_disable_unprepare(iommu->clk);
+
+err_reset_assert:
+	reset_control_assert(iommu->reset);
+
+	return ret;
+}
+
+static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&iommu->iommu_lock, flags);
+
+	iommu_write(iommu, IOMMU_ENABLE_REG, 0);
+	iommu_write(iommu, IOMMU_TTB_REG, 0);
+
+	spin_unlock_irqrestore(&iommu->iommu_lock, flags);
+
+	clk_disable_unprepare(iommu->clk);
+	reset_control_assert(iommu->reset);
+}
+
+static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
+					   gfp_t gfp)
+{
+	dma_addr_t pt_dma;
+	u32 *page_table;
+
+	page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
+	if (!page_table)
+		return ERR_PTR(-ENOMEM);
+
+	pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
+	if (dma_mapping_error(iommu->dev, pt_dma)) {
+		dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
+		kmem_cache_free(iommu->pt_pool, page_table);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	/* We rely on the physical address and DMA address being the same */
+	WARN_ON(pt_dma != virt_to_phys(page_table));
+
+	return page_table;
+}
+
+static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
+					 u32 *page_table)
+{
+	phys_addr_t pt_phys = virt_to_phys(page_table);
+
+	dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
+	kmem_cache_free(iommu->pt_pool, page_table);
+}
+
+static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
+				      dma_addr_t iova, gfp_t gfp)
+{
+	struct sun50i_iommu *iommu = sun50i_domain->iommu;
+	u32 *page_table;
+	u32 *dte_addr;
+	u32 old_dte;
+	u32 dte;
+
+	dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
+	dte = *dte_addr;
+	if (sun50i_dte_is_pt_valid(dte)) {
+		phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
+		return (u32 *)phys_to_virt(pt_phys);
+	}
+
+	page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
+	if (IS_ERR(page_table))
+		return page_table;
+
+	dte = sun50i_mk_dte(virt_to_phys(page_table));
+	old_dte = cmpxchg(dte_addr, 0, dte);
+	if (old_dte) {
+		phys_addr_t installed_pt_phys =
+			sun50i_dte_get_pt_address(old_dte);
+		u32 *installed_pt = phys_to_virt(installed_pt_phys);
+		u32 *drop_pt = page_table;
+
+		page_table = installed_pt;
+		dte = old_dte;
+		sun50i_iommu_free_page_table(iommu, drop_pt);
+	}
+
+	sun50i_table_flush(sun50i_domain, page_table, PT_SIZE);
+	sun50i_table_flush(sun50i_domain, dte_addr, 1);
+
+	return page_table;
+}
+
+static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
+			    phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
+{
+	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
+	struct sun50i_iommu *iommu = sun50i_domain->iommu;
+	u32 pte_index;
+	u32 *page_table, *pte_addr;
+	int ret = 0;
+
+	page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
+	if (IS_ERR(page_table)) {
+		ret = PTR_ERR(page_table);
+		goto out;
+	}
+
+	pte_index = sun50i_iova_get_pte_index(iova);
+	pte_addr = &page_table[pte_index];
+	if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
+		phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
+		dev_err(iommu->dev,
+			"iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
+			&iova, &page_phys, &paddr, prot);
+		ret = -EBUSY;
+		goto out;
+	}
+
+	*pte_addr = sun50i_mk_pte(paddr, prot);
+	sun50i_table_flush(sun50i_domain, pte_addr, 1);
+
+out:
+	return ret;
+}
+
+static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+				 size_t size, struct iommu_iotlb_gather *gather)
+{
+	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
+	phys_addr_t pt_phys;
+	dma_addr_t pte_dma;
+	u32 *pte_addr;
+	u32 dte;
+
+	dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
+	if (!sun50i_dte_is_pt_valid(dte))
+		return 0;
+
+	pt_phys = sun50i_dte_get_pt_address(dte);
+	pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);
+	pte_dma = pt_phys + sun50i_iova_get_pte_index(iova) * PT_ENTRY_SIZE;
+
+	if (!sun50i_pte_is_page_valid(*pte_addr))
+		return 0;
+
+	memset(pte_addr, 0, sizeof(*pte_addr));
+	sun50i_table_flush(sun50i_domain, pte_addr, 1);
+
+	return SZ_4K;
+}
+
+static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
+					     dma_addr_t iova)
+{
+	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
+	phys_addr_t pt_phys;
+	u32 *page_table;
+	u32 dte, pte;
+
+	dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
+	if (!sun50i_dte_is_pt_valid(dte))
+		return 0;
+
+	pt_phys = sun50i_dte_get_pt_address(dte);
+	page_table = (u32 *)phys_to_virt(pt_phys);
+	pte = page_table[sun50i_iova_get_pte_index(iova)];
+	if (!sun50i_pte_is_page_valid(pte))
+		return 0;
+
+	return sun50i_pte_get_page_address(pte) +
+		sun50i_iova_get_page_offset(iova);
+}
+
+static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
+{
+	struct sun50i_iommu_domain *sun50i_domain;
+
+	if (type != IOMMU_DOMAIN_DMA &&
+	    type != IOMMU_DOMAIN_IDENTITY &&
+	    type != IOMMU_DOMAIN_UNMANAGED)
+		return NULL;
+
+	sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
+	if (!sun50i_domain)
+		return NULL;
+
+	if (type == IOMMU_DOMAIN_DMA &&
+	    iommu_get_dma_cookie(&sun50i_domain->domain))
+		goto err_free_domain;
+
+	sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+						    get_order(DT_SIZE));
+	if (!sun50i_domain->dt)
+		goto err_put_cookie;
+
+	refcount_set(&sun50i_domain->refcnt, 1);
+
+	sun50i_domain->domain.geometry.aperture_start = 0;
+	sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
+	sun50i_domain->domain.geometry.force_aperture = true;
+
+	return &sun50i_domain->domain;
+
+err_put_cookie:
+	if (type == IOMMU_DOMAIN_DMA)
+		iommu_put_dma_cookie(&sun50i_domain->domain);
+
+err_free_domain:
+	kfree(sun50i_domain);
+
+	return NULL;
+}
+
+static void sun50i_iommu_domain_free(struct iommu_domain *domain)
+{
+	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
+
+	free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE));
+	sun50i_domain->dt = NULL;
+
+	iommu_put_dma_cookie(domain);
+
+	kfree(sun50i_domain);
+}
+
+static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
+				      struct sun50i_iommu_domain *sun50i_domain)
+{
+	iommu->domain = &sun50i_domain->domain;
+	sun50i_domain->iommu = iommu;
+
+	sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
+					       DT_SIZE, DMA_TO_DEVICE);
+	if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
+		dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
+		return -ENOMEM;
+	}
+
+	return sun50i_iommu_enable(iommu);
+}
+
+static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
+				       struct sun50i_iommu_domain *sun50i_domain)
+{
+	unsigned int i;
+
+	for (i = 0; i < NUM_DT_ENTRIES; i++) {
+		phys_addr_t pt_phys;
+		u32 *page_table;
+		u32 *dte_addr;
+		u32 dte;
+
+		dte_addr = &sun50i_domain->dt[i];
+		dte = *dte_addr;
+		if (!sun50i_dte_is_pt_valid(dte))
+			continue;
+
+		memset(dte_addr, 0, sizeof(*dte_addr));
+		sun50i_table_flush(sun50i_domain, dte_addr, 1);
+
+		pt_phys = sun50i_dte_get_pt_address(dte);
+		page_table = phys_to_virt(pt_phys);
+		sun50i_iommu_free_page_table(iommu, page_table);
+	}
+
+
+	sun50i_iommu_disable(iommu);
+
+	dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
+			 DT_SIZE, DMA_TO_DEVICE);
+
+	iommu->domain = NULL;
+}
+
+static void sun50i_iommu_detach_device(struct iommu_domain *domain,
+				       struct device *dev)
+{
+	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
+	struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
+
+	dev_dbg(dev, "Detaching from IOMMU domain\n");
+
+	if (iommu->domain != domain)
+		return;
+
+	if (refcount_dec_and_test(&sun50i_domain->refcnt))
+		sun50i_iommu_detach_domain(iommu, sun50i_domain);
+}
+
+static int sun50i_iommu_attach_device(struct iommu_domain *domain,
+				      struct device *dev)
+{
+	struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
+	struct sun50i_iommu *iommu;
+
+	iommu = sun50i_iommu_from_dev(dev);
+	if (!iommu)
+		return -ENODEV;
+
+	dev_dbg(dev, "Attaching to IOMMU domain\n");
+
+	refcount_inc(&sun50i_domain->refcnt);
+
+	if (iommu->domain == domain)
+		return 0;
+
+	if (iommu->domain)
+		sun50i_iommu_detach_device(iommu->domain, dev);
+
+	sun50i_iommu_attach_domain(iommu, sun50i_domain);
+
+	return 0;
+}
+
+static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
+{
+	struct sun50i_iommu *iommu;
+
+	iommu = sun50i_iommu_from_dev(dev);
+	if (!iommu)
+		return ERR_PTR(-ENODEV);
+
+	return &iommu->iommu;
+}
+
+static void sun50i_iommu_release_device(struct device *dev) {}
+
+static struct iommu_group *sun50i_iommu_device_group(struct device *dev)
+{
+	struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev);
+
+	return iommu_group_ref_get(iommu->group);
+}
+
+static int sun50i_iommu_of_xlate(struct device *dev,
+				 struct of_phandle_args *args)
+{
+	struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
+	unsigned id = args->args[0];
+
+	dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));
+
+	return iommu_fwspec_add_ids(dev, &id, 1);
+}
+
+static const struct iommu_ops sun50i_iommu_ops = {
+	.pgsize_bitmap	= SZ_4K,
+	.attach_dev	= sun50i_iommu_attach_device,
+	.detach_dev	= sun50i_iommu_detach_device,
+	.device_group	= sun50i_iommu_device_group,
+	.domain_alloc	= sun50i_iommu_domain_alloc,
+	.domain_free	= sun50i_iommu_domain_free,
+	.flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
+	.iotlb_sync	= sun50i_iommu_iotlb_sync,
+	.iova_to_phys	= sun50i_iommu_iova_to_phys,
+	.map		= sun50i_iommu_map,
+	.of_xlate	= sun50i_iommu_of_xlate,
+	.probe_device	= sun50i_iommu_probe_device,
+	.release_device	= sun50i_iommu_release_device,
+	.unmap		= sun50i_iommu_unmap,
+};
+
+static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
+				      unsigned master, phys_addr_t iova,
+				      unsigned prot)
+{
+	dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
+		&iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");
+
+	if (iommu->domain)
+		report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
+	else
+		dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
+}
+
+static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
+					      unsigned addr_reg,
+					      unsigned blame_reg)
+{
+	phys_addr_t iova;
+	unsigned master;
+	u32 blame;
+
+	assert_spin_locked(&iommu->iommu_lock);
+
+	iova = iommu_read(iommu, addr_reg);
+	blame = iommu_read(iommu, blame_reg);
+	master = ilog2(blame & IOMMU_INT_MASTER_MASK);
+
+	/*
+	 * If the address is not in the page table, we can't get what
+	 * operation triggered the fault. Assume it's a read
+	 * operation.
+	 */
+	sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);
+
+	return iova;
+}
+
+static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
+{
+	enum sun50i_iommu_aci aci;
+	phys_addr_t iova;
+	unsigned master;
+	unsigned dir;
+	u32 blame;
+
+	assert_spin_locked(&iommu->iommu_lock);
+
+	blame = iommu_read(iommu, IOMMU_INT_STA_REG);
+	master = ilog2(blame & IOMMU_INT_MASTER_MASK);
+	iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
+	aci = sun50i_get_pte_aci(iommu_read(iommu,
+					    IOMMU_INT_ERR_DATA_REG(master)));
+
+	switch (aci) {
+		/*
+		 * If we are in the read-only domain, then it means we
+		 * tried to write.
+		 */
+	case SUN50I_IOMMU_ACI_RD:
+		dir = IOMMU_FAULT_WRITE;
+		break;
+
+		/*
+		 * If we are in the write-only domain, then it means
+		 * we tried to read.
+		 */
+	case SUN50I_IOMMU_ACI_WR:
+
+		/*
+		 * If we are in the domain without any permission, we
+		 * can't really tell. Let's default to a read
+		 * operation.
+		 */
+	case SUN50I_IOMMU_ACI_NONE:
+
+		/* WTF? */
+	case SUN50I_IOMMU_ACI_RD_WR:
+	default:
+		dir = IOMMU_FAULT_READ;
+		break;
+	}
+
+	/*
+	 * If the address is not in the page table, we can't get what
+	 * operation triggered the fault. Assume it's a read
+	 * operation.
+	 */
+	sun50i_iommu_report_fault(iommu, master, iova, dir);
+
+	return iova;
+}
+
+static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
+{
+	struct sun50i_iommu *iommu = dev_id;
+	phys_addr_t iova;
+	u32 status;
+
+	spin_lock(&iommu->iommu_lock);
+
+	status = iommu_read(iommu, IOMMU_INT_STA_REG);
+	if (!(status & IOMMU_INT_MASK)) {
+		spin_unlock(&iommu->iommu_lock);
+		return IRQ_NONE;
+	}
+
+	if (status & IOMMU_INT_INVALID_L2PG)
+		iova = sun50i_iommu_handle_pt_irq(iommu,
+						  IOMMU_INT_ERR_ADDR_L2_REG,
+						  IOMMU_L2PG_INT_REG);
+	else if (status & IOMMU_INT_INVALID_L1PG)
+		iova = sun50i_iommu_handle_pt_irq(iommu,
+						  IOMMU_INT_ERR_ADDR_L1_REG,
+						  IOMMU_L1PG_INT_REG);
+	else
+		iova = sun50i_iommu_handle_perm_irq(iommu);
+
+	iommu_write(iommu, IOMMU_INT_CLR_REG, status);
+
+	iommu_write(iommu, IOMMU_RESET_REG, ~status);
+	iommu_write(iommu, IOMMU_RESET_REG, status);
+
+	spin_unlock(&iommu->iommu_lock);
+
+	return IRQ_HANDLED;
+}
+
+static int sun50i_iommu_probe(struct platform_device *pdev)
+{
+	struct sun50i_iommu *iommu;
+	int ret, irq;
+
+	iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
+	if (!iommu)
+		return -ENOMEM;
+	spin_lock_init(&iommu->iommu_lock);
+	platform_set_drvdata(pdev, iommu);
+	iommu->dev = &pdev->dev;
+
+	iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
+					   PT_SIZE, PT_SIZE,
+					   SLAB_HWCACHE_ALIGN,
+					   NULL);
+	if (!iommu->pt_pool)
+		return -ENOMEM;
+
+	iommu->group = iommu_group_alloc();
+	if (IS_ERR(iommu->group)) {
+		ret = PTR_ERR(iommu->group);
+		goto err_free_cache;
+	}
+
+	iommu->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(iommu->base)) {
+		ret = PTR_ERR(iommu->base);
+		goto err_free_group;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
+		goto err_free_group;
+	}
+
+	iommu->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(iommu->clk)) {
+		dev_err(&pdev->dev, "Couldn't get our clock.\n");
+		ret = PTR_ERR(iommu->clk);
+		goto err_free_group;
+	}
+
+	iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
+	if (IS_ERR(iommu->reset)) {
+		dev_err(&pdev->dev, "Couldn't get our reset line.\n");
+		ret = PTR_ERR(iommu->reset);
+		goto err_free_group;
+	}
+
+	ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
+				     NULL, dev_name(&pdev->dev));
+	if (ret)
+		goto err_free_group;
+
+	iommu_device_set_ops(&iommu->iommu, &sun50i_iommu_ops);
+	iommu_device_set_fwnode(&iommu->iommu, &pdev->dev.of_node->fwnode);
+
+	ret = iommu_device_register(&iommu->iommu);
+	if (ret)
+		goto err_remove_sysfs;
+
+	ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
+			       dev_name(&pdev->dev), iommu);
+	if (ret < 0)
+		goto err_unregister;
+
+	bus_set_iommu(&platform_bus_type, &sun50i_iommu_ops);
+
+	return 0;
+
+err_unregister:
+	iommu_device_unregister(&iommu->iommu);
+
+err_remove_sysfs:
+	iommu_device_sysfs_remove(&iommu->iommu);
+
+err_free_group:
+	iommu_group_put(iommu->group);
+
+err_free_cache:
+	kmem_cache_destroy(iommu->pt_pool);
+
+	return ret;
+}
+
+static const struct of_device_id sun50i_iommu_dt[] = {
+	{ .compatible = "allwinner,sun50i-h6-iommu", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);
+
+static struct platform_driver sun50i_iommu_driver = {
+	.driver		= {
+		.name			= "sun50i-iommu",
+		.of_match_table 	= sun50i_iommu_dt,
+		.suppress_bind_attrs	= true,
+	}
+};
+builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
+
+MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
+MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
+MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index db6559e8336f..5fbdff6ff41a 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -243,28 +243,16 @@ static bool gart_iommu_capable(enum iommu_cap cap)
 	return false;
 }
 
-static int gart_iommu_add_device(struct device *dev)
+static struct iommu_device *gart_iommu_probe_device(struct device *dev)
 {
-	struct iommu_group *group;
-
 	if (!dev_iommu_fwspec_get(dev))
-		return -ENODEV;
-
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	iommu_group_put(group);
+		return ERR_PTR(-ENODEV);
 
-	iommu_device_link(&gart_handle->iommu, dev);
-
-	return 0;
+	return &gart_handle->iommu;
 }
 
-static void gart_iommu_remove_device(struct device *dev)
+static void gart_iommu_release_device(struct device *dev)
 {
-	iommu_group_remove_device(dev);
-	iommu_device_unlink(&gart_handle->iommu, dev);
 }
 
 static int gart_iommu_of_xlate(struct device *dev,
@@ -290,8 +278,8 @@ static const struct iommu_ops gart_iommu_ops = {
 	.domain_free	= gart_iommu_domain_free,
 	.attach_dev	= gart_iommu_attach_dev,
 	.detach_dev	= gart_iommu_detach_dev,
-	.add_device	= gart_iommu_add_device,
-	.remove_device	= gart_iommu_remove_device,
+	.probe_device	= gart_iommu_probe_device,
+	.release_device	= gart_iommu_release_device,
 	.device_group	= generic_device_group,
 	.map		= gart_iommu_map,
 	.unmap		= gart_iommu_unmap,
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 63a147b623e6..7426b7666e2b 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -757,11 +757,10 @@ static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,
 	return 0;
 }
 
-static int tegra_smmu_add_device(struct device *dev)
+static struct iommu_device *tegra_smmu_probe_device(struct device *dev)
 {
 	struct device_node *np = dev->of_node;
 	struct tegra_smmu *smmu = NULL;
-	struct iommu_group *group;
 	struct of_phandle_args args;
 	unsigned int index = 0;
 	int err;
@@ -774,7 +773,7 @@ static int tegra_smmu_add_device(struct device *dev)
 			of_node_put(args.np);
 
 			if (err < 0)
-				return err;
+				return ERR_PTR(err);
 
 			/*
 			 * Only a single IOMMU master interface is currently
@@ -783,8 +782,6 @@ static int tegra_smmu_add_device(struct device *dev)
 			 */
 			dev->archdata.iommu = smmu;
 
-			iommu_device_link(&smmu->iommu, dev);
-
 			break;
 		}
 
@@ -793,26 +790,14 @@ static int tegra_smmu_add_device(struct device *dev)
 	}
 
 	if (!smmu)
-		return -ENODEV;
-
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group))
-		return PTR_ERR(group);
-
-	iommu_group_put(group);
+		return ERR_PTR(-ENODEV);
 
-	return 0;
+	return &smmu->iommu;
 }
 
-static void tegra_smmu_remove_device(struct device *dev)
+static void tegra_smmu_release_device(struct device *dev)
 {
-	struct tegra_smmu *smmu = dev->archdata.iommu;
-
-	if (smmu)
-		iommu_device_unlink(&smmu->iommu, dev);
-
 	dev->archdata.iommu = NULL;
-	iommu_group_remove_device(dev);
 }
 
 static const struct tegra_smmu_group_soc *
@@ -895,8 +880,8 @@ static const struct iommu_ops tegra_smmu_ops = {
 	.domain_free = tegra_smmu_domain_free,
 	.attach_dev = tegra_smmu_attach_dev,
 	.detach_dev = tegra_smmu_detach_dev,
-	.add_device = tegra_smmu_add_device,
-	.remove_device = tegra_smmu_remove_device,
+	.probe_device = tegra_smmu_probe_device,
+	.release_device = tegra_smmu_release_device,
 	.device_group = tegra_smmu_device_group,
 	.map = tegra_smmu_map,
 	.unmap = tegra_smmu_unmap,
@@ -1015,7 +1000,7 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
 	 * value. However the IOMMU registration process will attempt to add
 	 * all devices to the IOMMU when bus_set_iommu() is called. In order
 	 * not to rely on global variables to track the IOMMU instance, we
-	 * set it here so that it can be looked up from the .add_device()
+	 * set it here so that it can be looked up from the .probe_device()
 	 * callback via the IOMMU device's .drvdata field.
 	 */
 	mc->smmu = smmu;
diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
index 4e1d11af23c8..f6f07489a9aa 100644
--- a/drivers/iommu/virtio-iommu.c
+++ b/drivers/iommu/virtio-iommu.c
@@ -865,24 +865,23 @@ static struct viommu_dev *viommu_get_by_fwnode(struct fwnode_handle *fwnode)
 	return dev ? dev_to_virtio(dev)->priv : NULL;
 }
 
-static int viommu_add_device(struct device *dev)
+static struct iommu_device *viommu_probe_device(struct device *dev)
 {
 	int ret;
-	struct iommu_group *group;
 	struct viommu_endpoint *vdev;
 	struct viommu_dev *viommu = NULL;
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 
 	if (!fwspec || fwspec->ops != &viommu_ops)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	viommu = viommu_get_by_fwnode(fwspec->iommu_fwnode);
 	if (!viommu)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
 	if (!vdev)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	vdev->dev = dev;
 	vdev->viommu = viommu;
@@ -896,45 +895,25 @@ static int viommu_add_device(struct device *dev)
 			goto err_free_dev;
 	}
 
-	ret = iommu_device_link(&viommu->iommu, dev);
-	if (ret)
-		goto err_free_dev;
+	return &viommu->iommu;
 
-	/*
-	 * Last step creates a default domain and attaches to it. Everything
-	 * must be ready.
-	 */
-	group = iommu_group_get_for_dev(dev);
-	if (IS_ERR(group)) {
-		ret = PTR_ERR(group);
-		goto err_unlink_dev;
-	}
-
-	iommu_group_put(group);
-
-	return PTR_ERR_OR_ZERO(group);
-
-err_unlink_dev:
-	iommu_device_unlink(&viommu->iommu, dev);
 err_free_dev:
 	generic_iommu_put_resv_regions(dev, &vdev->resv_regions);
 	kfree(vdev);
 
-	return ret;
+	return ERR_PTR(ret);
 }
 
-static void viommu_remove_device(struct device *dev)
+static void viommu_release_device(struct device *dev)
 {
-	struct viommu_endpoint *vdev;
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct viommu_endpoint *vdev;
 
 	if (!fwspec || fwspec->ops != &viommu_ops)
 		return;
 
 	vdev = dev_iommu_priv_get(dev);
 
-	iommu_group_remove_device(dev);
-	iommu_device_unlink(&vdev->viommu->iommu, dev);
 	generic_iommu_put_resv_regions(dev, &vdev->resv_regions);
 	kfree(vdev);
 }
@@ -960,8 +939,8 @@ static struct iommu_ops viommu_ops = {
 	.unmap			= viommu_unmap,
 	.iova_to_phys		= viommu_iova_to_phys,
 	.iotlb_sync		= viommu_iotlb_sync,
-	.add_device		= viommu_add_device,
-	.remove_device		= viommu_remove_device,
+	.probe_device		= viommu_probe_device,
+	.release_device		= viommu_release_device,
 	.device_group		= viommu_device_group,
 	.get_resv_regions	= viommu_get_resv_regions,
 	.put_resv_regions	= generic_iommu_put_resv_regions,
diff --git a/drivers/ipack/Kconfig b/drivers/ipack/Kconfig
index 68a422f7b271..fb30091dece3 100644
--- a/drivers/ipack/Kconfig
+++ b/drivers/ipack/Kconfig
@@ -6,7 +6,7 @@
 menuconfig IPACK_BUS
 	tristate "IndustryPack bus support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  This option provides support for the IndustryPack framework.  There
 	  are IndustryPack carrier boards, which interface another bus (such as
 	  PCI) to an IndustryPack bus, and IndustryPack modules, that are
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 66b9a68f5e9f..29fead208cad 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -493,6 +493,19 @@ config TI_SCI_INTA_IRQCHIP
 	  If you wish to use interrupt aggregator irq resources managed by the
 	  TI System Controller, say Y here. Otherwise, say N.
 
+config RISCV_INTC
+	bool "RISC-V Local Interrupt Controller"
+	depends on RISCV
+	default y
+	help
+	   This enables support for the per-HART local interrupt controller
+	   found in standard RISC-V systems.  The per-HART local interrupt
+	   controller handles timer interrupts, software interrupts, and
+	   hardware interrupts. Without a per-HART local interrupt controller,
+	   a RISC-V system will be unable to handle any interrupts.
+
+	   If you don't know what to do here, say Y.
+
 config SIFIVE_PLIC
 	bool "SiFive Platform-Level Interrupt Controller"
 	depends on RISCV
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 3a4ce283189a..133f9c45744a 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -98,6 +98,7 @@ obj-$(CONFIG_NDS32)			+= irq-ativic32.o
 obj-$(CONFIG_QCOM_PDC)			+= qcom-pdc.o
 obj-$(CONFIG_CSKY_MPINTC)		+= irq-csky-mpintc.o
 obj-$(CONFIG_CSKY_APB_INTC)		+= irq-csky-apb-intc.o
+obj-$(CONFIG_RISCV_INTC)		+= irq-riscv-intc.o
 obj-$(CONFIG_SIFIVE_PLIC)		+= irq-sifive-plic.o
 obj-$(CONFIG_IMX_IRQSTEER)		+= irq-imx-irqsteer.o
 obj-$(CONFIG_IMX_INTMUX)		+= irq-imx-intmux.o
diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c
new file mode 100644
index 000000000000..a6f97fa6ff69
--- /dev/null
+++ b/drivers/irqchip/irq-riscv-intc.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2012 Regents of the University of California
+ * Copyright (C) 2017-2018 SiFive
+ * Copyright (C) 2020 Western Digital Corporation or its affiliates.
+ */
+
+#define pr_fmt(fmt) "riscv-intc: " fmt
+#include <linux/atomic.h>
+#include <linux/bits.h>
+#include <linux/cpu.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqdomain.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/smp.h>
+
+static struct irq_domain *intc_domain;
+
+static asmlinkage void riscv_intc_irq(struct pt_regs *regs)
+{
+	unsigned long cause = regs->cause & ~CAUSE_IRQ_FLAG;
+
+	if (unlikely(cause >= BITS_PER_LONG))
+		panic("unexpected interrupt cause");
+
+	switch (cause) {
+#ifdef CONFIG_SMP
+	case RV_IRQ_SOFT:
+		/*
+		 * We only use software interrupts to pass IPIs, so if a
+		 * non-SMP system gets one, then we don't know what to do.
+		 */
+		handle_IPI(regs);
+		break;
+#endif
+	default:
+		handle_domain_irq(intc_domain, cause, regs);
+		break;
+	}
+}
+
+/*
+ * On RISC-V systems local interrupts are masked or unmasked by writing
+ * the SIE (Supervisor Interrupt Enable) CSR.  As CSRs can only be written
+ * on the local hart, these functions can only be called on the hart that
+ * corresponds to the IRQ chip.
+ */
+
+static void riscv_intc_irq_mask(struct irq_data *d)
+{
+	csr_clear(CSR_IE, BIT(d->hwirq));
+}
+
+static void riscv_intc_irq_unmask(struct irq_data *d)
+{
+	csr_set(CSR_IE, BIT(d->hwirq));
+}
+
+static int riscv_intc_cpu_starting(unsigned int cpu)
+{
+	csr_set(CSR_IE, BIT(RV_IRQ_SOFT));
+	return 0;
+}
+
+static int riscv_intc_cpu_dying(unsigned int cpu)
+{
+	csr_clear(CSR_IE, BIT(RV_IRQ_SOFT));
+	return 0;
+}
+
+static struct irq_chip riscv_intc_chip = {
+	.name = "RISC-V INTC",
+	.irq_mask = riscv_intc_irq_mask,
+	.irq_unmask = riscv_intc_irq_unmask,
+};
+
+static int riscv_intc_domain_map(struct irq_domain *d, unsigned int irq,
+				 irq_hw_number_t hwirq)
+{
+	irq_set_percpu_devid(irq);
+	irq_domain_set_info(d, irq, hwirq, &riscv_intc_chip, d->host_data,
+			    handle_percpu_devid_irq, NULL, NULL);
+
+	return 0;
+}
+
+static const struct irq_domain_ops riscv_intc_domain_ops = {
+	.map	= riscv_intc_domain_map,
+	.xlate	= irq_domain_xlate_onecell,
+};
+
+static int __init riscv_intc_init(struct device_node *node,
+				  struct device_node *parent)
+{
+	int rc, hartid;
+
+	hartid = riscv_of_parent_hartid(node);
+	if (hartid < 0) {
+		pr_warn("unable to fine hart id for %pOF\n", node);
+		return 0;
+	}
+
+	/*
+	 * The DT will have one INTC DT node under each CPU (or HART)
+	 * DT node so riscv_intc_init() function will be called once
+	 * for each INTC DT node. We only need to do INTC initialization
+	 * for the INTC DT node belonging to boot CPU (or boot HART).
+	 */
+	if (riscv_hartid_to_cpuid(hartid) != smp_processor_id())
+		return 0;
+
+	intc_domain = irq_domain_add_linear(node, BITS_PER_LONG,
+					    &riscv_intc_domain_ops, NULL);
+	if (!intc_domain) {
+		pr_err("unable to add IRQ domain\n");
+		return -ENXIO;
+	}
+
+	rc = set_handle_irq(&riscv_intc_irq);
+	if (rc) {
+		pr_err("failed to set irq handler\n");
+		return rc;
+	}
+
+	cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_STARTING,
+			  "irqchip/riscv/intc:starting",
+			  riscv_intc_cpu_starting,
+			  riscv_intc_cpu_dying);
+
+	pr_info("%d local interrupts mapped\n", BITS_PER_LONG);
+
+	return 0;
+}
+
+IRQCHIP_DECLARE(riscv, "riscv,cpu-intc", riscv_intc_init);
diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c
index d9c53f85a68e..eaa3e9fe54e9 100644
--- a/drivers/irqchip/irq-sifive-plic.c
+++ b/drivers/irqchip/irq-sifive-plic.c
@@ -9,6 +9,7 @@
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -76,6 +77,7 @@ struct plic_handler {
 	void __iomem		*enable_base;
 	struct plic_priv	*priv;
 };
+static int plic_parent_irq;
 static bool plic_cpuhp_setup_done;
 static DEFINE_PER_CPU(struct plic_handler, plic_handlers);
 
@@ -219,15 +221,17 @@ static const struct irq_domain_ops plic_irqdomain_ops = {
  * that source ID back to the same claim register.  This automatically enables
  * and disables the interrupt, so there's nothing else to do.
  */
-static void plic_handle_irq(struct pt_regs *regs)
+static void plic_handle_irq(struct irq_desc *desc)
 {
 	struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
 	void __iomem *claim = handler->hart_base + CONTEXT_CLAIM;
 	irq_hw_number_t hwirq;
 
 	WARN_ON_ONCE(!handler->present);
 
-	csr_clear(CSR_IE, IE_EIE);
+	chained_irq_enter(chip, desc);
+
 	while ((hwirq = readl(claim))) {
 		int irq = irq_find_mapping(handler->priv->irqdomain, hwirq);
 
@@ -237,21 +241,8 @@ static void plic_handle_irq(struct pt_regs *regs)
 		else
 			generic_handle_irq(irq);
 	}
-	csr_set(CSR_IE, IE_EIE);
-}
-
-/*
- * Walk up the DT tree until we find an active RISC-V core (HART) node and
- * extract the cpuid from it.
- */
-static int plic_find_hart_id(struct device_node *node)
-{
-	for (; node; node = node->parent) {
-		if (of_device_is_compatible(node, "riscv"))
-			return riscv_of_processor_hartid(node);
-	}
 
-	return -1;
+	chained_irq_exit(chip, desc);
 }
 
 static void plic_set_threshold(struct plic_handler *handler, u32 threshold)
@@ -262,10 +253,8 @@ static void plic_set_threshold(struct plic_handler *handler, u32 threshold)
 
 static int plic_dying_cpu(unsigned int cpu)
 {
-	struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
-
-	csr_clear(CSR_IE, IE_EIE);
-	plic_set_threshold(handler, PLIC_DISABLE_THRESHOLD);
+	if (plic_parent_irq)
+		disable_percpu_irq(plic_parent_irq);
 
 	return 0;
 }
@@ -274,7 +263,11 @@ static int plic_starting_cpu(unsigned int cpu)
 {
 	struct plic_handler *handler = this_cpu_ptr(&plic_handlers);
 
-	csr_set(CSR_IE, IE_EIE);
+	if (plic_parent_irq)
+		enable_percpu_irq(plic_parent_irq,
+				  irq_get_trigger_type(plic_parent_irq));
+	else
+		pr_warn("cpu%d: parent irq not available\n", cpu);
 	plic_set_threshold(handler, PLIC_ENABLE_THRESHOLD);
 
 	return 0;
@@ -330,7 +323,7 @@ static int __init plic_init(struct device_node *node,
 		if (parent.args[0] != RV_IRQ_EXT)
 			continue;
 
-		hartid = plic_find_hart_id(parent.np);
+		hartid = riscv_of_parent_hartid(parent.np);
 		if (hartid < 0) {
 			pr_warn("failed to parse hart ID for context %d.\n", i);
 			continue;
@@ -342,6 +335,14 @@ static int __init plic_init(struct device_node *node,
 			continue;
 		}
 
+		/* Find parent domain and register chained handler */
+		if (!plic_parent_irq && irq_find_host(parent.np)) {
+			plic_parent_irq = irq_of_parse_and_map(node, i);
+			if (plic_parent_irq)
+				irq_set_chained_handler(plic_parent_irq,
+							plic_handle_irq);
+		}
+
 		/*
 		 * When running in M-mode we need to ignore the S-mode handler.
 		 * Here we assume it always comes later, but that might be a
@@ -382,7 +383,6 @@ done:
 
 	pr_info("%pOFP: mapped %d interrupts with %d handlers for"
 		" %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts);
-	set_handle_irq(plic_handle_irq);
 	return 0;
 
 out_iounmap:
diff --git a/drivers/isdn/Kconfig b/drivers/isdn/Kconfig
index be8387c0eeef..2690e2c5a158 100644
--- a/drivers/isdn/Kconfig
+++ b/drivers/isdn/Kconfig
@@ -7,7 +7,7 @@ menuconfig ISDN
 	bool "ISDN support"
 	depends on NET && NETDEVICES
 	depends on !S390 && !UML
-	---help---
+	help
 	  ISDN ("Integrated Services Digital Network", called RNIS in France)
 	  is a fully digital telephone service that can be used for voice and
 	  data connections.  If your computer is equipped with an ISDN
diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig
index c664d84e1667..ed943140e1fd 100644
--- a/drivers/leds/Kconfig
+++ b/drivers/leds/Kconfig
@@ -83,6 +83,17 @@ config LEDS_APU
 	  To compile this driver as a module, choose M here: the
 	  module will be called leds-apu.
 
+config LEDS_ARIEL
+	tristate "Dell Wyse 3020 status LED support"
+	depends on LEDS_CLASS
+	depends on (MACH_MMP3_DT && MFD_ENE_KB3930) || COMPILE_TEST
+	help
+	  This driver adds support for controlling the front panel status
+	  LEDs on Dell Wyse 3020 (Ariel) board via the KB3930 Embedded
+	  Controller.
+
+	  Say Y to if your machine is a Dell Wyse 3020 thin client.
+
 config LEDS_AS3645A
 	tristate "AS3645A and LM3555 LED flash controllers support"
 	depends on I2C && LEDS_CLASS_FLASH
@@ -92,6 +103,16 @@ config LEDS_AS3645A
 	  controller. V4L2 flash API is provided as well if
 	  CONFIG_V4L2_FLASH_API is enabled.
 
+config LEDS_AW2013
+	tristate "LED support for Awinic AW2013"
+	depends on LEDS_CLASS && I2C && OF
+	help
+	  This option enables support for the AW2013 3-channel
+	  LED driver.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called leds-aw2013.
+
 config LEDS_BCM6328
 	tristate "LED Support for Broadcom BCM6328"
 	depends on LEDS_CLASS
@@ -857,6 +878,14 @@ config LEDS_IP30
 	  To compile this driver as a module, choose M here: the module
 	  will be called leds-ip30.
 
+config LEDS_SGM3140
+	tristate "LED support for the SGM3140"
+	depends on LEDS_CLASS_FLASH
+	depends on V4L2_FLASH_LED_CLASS || !V4L2_FLASH_LED_CLASS
+	help
+	  This option enables support for the SGM3140 500mA Buck/Boost Charge
+	  Pump LED Driver.
+
 comment "LED Triggers"
 source "drivers/leds/trigger/Kconfig"
 
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index 45235d5fb218..d6b8a792c936 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -12,8 +12,10 @@ obj-$(CONFIG_LEDS_AAT1290)		+= leds-aat1290.o
 obj-$(CONFIG_LEDS_ADP5520)		+= leds-adp5520.o
 obj-$(CONFIG_LEDS_AN30259A)		+= leds-an30259a.o
 obj-$(CONFIG_LEDS_APU)			+= leds-apu.o
+obj-$(CONFIG_LEDS_ARIEL)		+= leds-ariel.o
 obj-$(CONFIG_LEDS_AS3645A)		+= leds-as3645a.o
 obj-$(CONFIG_LEDS_ASIC3)		+= leds-asic3.o
+obj-$(CONFIG_LEDS_AW2013)		+= leds-aw2013.o
 obj-$(CONFIG_LEDS_BCM6328)		+= leds-bcm6328.o
 obj-$(CONFIG_LEDS_BCM6358)		+= leds-bcm6358.o
 obj-$(CONFIG_LEDS_BD2802)		+= leds-bd2802.o
@@ -77,6 +79,7 @@ obj-$(CONFIG_LEDS_PWM)			+= leds-pwm.o
 obj-$(CONFIG_LEDS_REGULATOR)		+= leds-regulator.o
 obj-$(CONFIG_LEDS_S3C24XX)		+= leds-s3c24xx.o
 obj-$(CONFIG_LEDS_SC27XX_BLTC)		+= leds-sc27xx-bltc.o
+obj-$(CONFIG_LEDS_SGM3140)		+= leds-sgm3140.o
 obj-$(CONFIG_LEDS_SUNFIRE)		+= leds-sunfire.o
 obj-$(CONFIG_LEDS_SYSCON)		+= leds-syscon.o
 obj-$(CONFIG_LEDS_TCA6507)		+= leds-tca6507.o
diff --git a/drivers/leds/leds-ariel.c b/drivers/leds/leds-ariel.c
new file mode 100644
index 000000000000..bb68ba23a7d4
--- /dev/null
+++ b/drivers/leds/leds-ariel.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-or-later
+/*
+ * Dell Wyse 3020 a.k.a. "Ariel" Embedded Controller LED Driver
+ *
+ * Copyright (C) 2020 Lubomir Rintel
+ */
+
+#include <linux/module.h>
+#include <linux/leds.h>
+#include <linux/regmap.h>
+#include <linux/of_platform.h>
+
+enum ec_index {
+	EC_BLUE_LED	= 0x01,
+	EC_AMBER_LED	= 0x02,
+	EC_GREEN_LED	= 0x03,
+};
+
+enum {
+	EC_LED_OFF	= 0x00,
+	EC_LED_STILL	= 0x01,
+	EC_LED_FADE	= 0x02,
+	EC_LED_BLINK	= 0x03,
+};
+
+struct ariel_led {
+	struct regmap *ec_ram;
+	enum ec_index ec_index;
+	struct led_classdev led_cdev;
+};
+
+#define led_cdev_to_ariel_led(c) container_of(c, struct ariel_led, led_cdev)
+
+static enum led_brightness ariel_led_get(struct led_classdev *led_cdev)
+{
+	struct ariel_led *led = led_cdev_to_ariel_led(led_cdev);
+	unsigned int led_status = 0;
+
+	if (regmap_read(led->ec_ram, led->ec_index, &led_status))
+		return LED_OFF;
+
+	if (led_status == EC_LED_STILL)
+		return LED_FULL;
+	else
+		return LED_OFF;
+}
+
+static void ariel_led_set(struct led_classdev *led_cdev,
+			  enum led_brightness brightness)
+{
+	struct ariel_led *led = led_cdev_to_ariel_led(led_cdev);
+
+	if (brightness == LED_OFF)
+		regmap_write(led->ec_ram, led->ec_index, EC_LED_OFF);
+	else
+		regmap_write(led->ec_ram, led->ec_index, EC_LED_STILL);
+}
+
+static int ariel_blink_set(struct led_classdev *led_cdev,
+			   unsigned long *delay_on, unsigned long *delay_off)
+{
+	struct ariel_led *led = led_cdev_to_ariel_led(led_cdev);
+
+	if (*delay_on == 0 && *delay_off == 0)
+		return -EINVAL;
+
+	if (*delay_on == 0) {
+		regmap_write(led->ec_ram, led->ec_index, EC_LED_OFF);
+	} else if (*delay_off == 0) {
+		regmap_write(led->ec_ram, led->ec_index, EC_LED_STILL);
+	} else {
+		*delay_on = 500;
+		*delay_off = 500;
+		regmap_write(led->ec_ram, led->ec_index, EC_LED_BLINK);
+	}
+
+	return 0;
+}
+
+#define NLEDS 3
+
+static int ariel_led_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ariel_led *leds;
+	struct regmap *ec_ram;
+	int ret;
+	int i;
+
+	ec_ram = dev_get_regmap(dev->parent, "ec_ram");
+	if (!ec_ram)
+		return -ENODEV;
+
+	leds = devm_kcalloc(dev, NLEDS, sizeof(*leds), GFP_KERNEL);
+	if (!leds)
+		return -ENOMEM;
+
+	leds[0].ec_index = EC_BLUE_LED;
+	leds[0].led_cdev.name = "blue:power",
+	leds[0].led_cdev.default_trigger = "default-on";
+
+	leds[1].ec_index = EC_AMBER_LED;
+	leds[1].led_cdev.name = "amber:status",
+
+	leds[2].ec_index = EC_GREEN_LED;
+	leds[2].led_cdev.name = "green:status",
+	leds[2].led_cdev.default_trigger = "default-on";
+
+	for (i = 0; i < NLEDS; i++) {
+		leds[i].ec_ram = ec_ram;
+		leds[i].led_cdev.brightness_get = ariel_led_get;
+		leds[i].led_cdev.brightness_set = ariel_led_set;
+		leds[i].led_cdev.blink_set = ariel_blink_set;
+
+		ret = devm_led_classdev_register(dev, &leds[i].led_cdev);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static struct platform_driver ariel_led_driver = {
+	.probe = ariel_led_probe,
+	.driver = {
+		.name = "dell-wyse-ariel-led",
+	},
+};
+module_platform_driver(ariel_led_driver);
+
+MODULE_AUTHOR("Lubomir Rintel <lkundrak@v3.sk>");
+MODULE_DESCRIPTION("Dell Wyse 3020 Status LEDs Driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/leds/leds-aw2013.c b/drivers/leds/leds-aw2013.c
new file mode 100644
index 000000000000..d709cc1f949e
--- /dev/null
+++ b/drivers/leds/leds-aw2013.c
@@ -0,0 +1,436 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Driver for Awinic AW2013 3-channel LED driver
+
+#include <linux/i2c.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+
+#define AW2013_MAX_LEDS 3
+
+/* Reset and ID register */
+#define AW2013_RSTR 0x00
+#define AW2013_RSTR_RESET 0x55
+#define AW2013_RSTR_CHIP_ID 0x33
+
+/* Global control register */
+#define AW2013_GCR 0x01
+#define AW2013_GCR_ENABLE BIT(0)
+
+/* LED channel enable register */
+#define AW2013_LCTR 0x30
+#define AW2013_LCTR_LE(x) BIT((x))
+
+/* LED channel control registers */
+#define AW2013_LCFG(x) (0x31 + (x))
+#define AW2013_LCFG_IMAX_MASK (BIT(0) | BIT(1)) // Should be 0-3
+#define AW2013_LCFG_MD BIT(4)
+#define AW2013_LCFG_FI BIT(5)
+#define AW2013_LCFG_FO BIT(6)
+
+/* LED channel PWM registers */
+#define AW2013_REG_PWM(x) (0x34 + (x))
+
+/* LED channel timing registers */
+#define AW2013_LEDT0(x) (0x37 + (x) * 3)
+#define AW2013_LEDT0_T1(x) ((x) << 4) // Should be 0-7
+#define AW2013_LEDT0_T2(x) (x) // Should be 0-5
+
+#define AW2013_LEDT1(x) (0x38 + (x) * 3)
+#define AW2013_LEDT1_T3(x) ((x) << 4) // Should be 0-7
+#define AW2013_LEDT1_T4(x) (x) // Should be 0-7
+
+#define AW2013_LEDT2(x) (0x39 + (x) * 3)
+#define AW2013_LEDT2_T0(x) ((x) << 4) // Should be 0-8
+#define AW2013_LEDT2_REPEAT(x) (x) // Should be 0-15
+
+#define AW2013_REG_MAX 0x77
+
+#define AW2013_TIME_STEP 130 /* ms */
+
+struct aw2013;
+
+struct aw2013_led {
+	struct aw2013 *chip;
+	struct led_classdev cdev;
+	u32 num;
+	unsigned int imax;
+};
+
+struct aw2013 {
+	struct mutex mutex; /* held when writing to registers */
+	struct regulator *vcc_regulator;
+	struct i2c_client *client;
+	struct aw2013_led leds[AW2013_MAX_LEDS];
+	struct regmap *regmap;
+	int num_leds;
+	bool enabled;
+};
+
+static int aw2013_chip_init(struct aw2013 *chip)
+{
+	int i, ret;
+
+	ret = regmap_write(chip->regmap, AW2013_GCR, AW2013_GCR_ENABLE);
+	if (ret) {
+		dev_err(&chip->client->dev, "Failed to enable the chip: %d\n",
+			ret);
+		return ret;
+	}
+
+	for (i = 0; i < chip->num_leds; i++) {
+		ret = regmap_update_bits(chip->regmap,
+					 AW2013_LCFG(chip->leds[i].num),
+					 AW2013_LCFG_IMAX_MASK,
+					 chip->leds[i].imax);
+		if (ret) {
+			dev_err(&chip->client->dev,
+				"Failed to set maximum current for led %d: %d\n",
+				chip->leds[i].num, ret);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+static void aw2013_chip_disable(struct aw2013 *chip)
+{
+	int ret;
+
+	if (!chip->enabled)
+		return;
+
+	regmap_write(chip->regmap, AW2013_GCR, 0);
+
+	ret = regulator_disable(chip->vcc_regulator);
+	if (ret) {
+		dev_err(&chip->client->dev,
+			"Failed to disable regulator: %d\n", ret);
+		return;
+	}
+
+	chip->enabled = false;
+}
+
+static int aw2013_chip_enable(struct aw2013 *chip)
+{
+	int ret;
+
+	if (chip->enabled)
+		return 0;
+
+	ret = regulator_enable(chip->vcc_regulator);
+	if (ret) {
+		dev_err(&chip->client->dev,
+			"Failed to enable regulator: %d\n", ret);
+		return ret;
+	}
+	chip->enabled = true;
+
+	ret = aw2013_chip_init(chip);
+	if (ret)
+		aw2013_chip_disable(chip);
+
+	return ret;
+}
+
+static bool aw2013_chip_in_use(struct aw2013 *chip)
+{
+	int i;
+
+	for (i = 0; i < chip->num_leds; i++)
+		if (chip->leds[i].cdev.brightness)
+			return true;
+
+	return false;
+}
+
+static int aw2013_brightness_set(struct led_classdev *cdev,
+				 enum led_brightness brightness)
+{
+	struct aw2013_led *led = container_of(cdev, struct aw2013_led, cdev);
+	int ret, num;
+
+	mutex_lock(&led->chip->mutex);
+
+	if (aw2013_chip_in_use(led->chip)) {
+		ret = aw2013_chip_enable(led->chip);
+		if (ret)
+			goto error;
+	}
+
+	num = led->num;
+
+	ret = regmap_write(led->chip->regmap, AW2013_REG_PWM(num), brightness);
+	if (ret)
+		goto error;
+
+	if (brightness) {
+		ret = regmap_update_bits(led->chip->regmap, AW2013_LCTR,
+					 AW2013_LCTR_LE(num), 0xFF);
+	} else {
+		ret = regmap_update_bits(led->chip->regmap, AW2013_LCTR,
+					 AW2013_LCTR_LE(num), 0);
+		if (ret)
+			goto error;
+		ret = regmap_update_bits(led->chip->regmap, AW2013_LCFG(num),
+					 AW2013_LCFG_MD, 0);
+	}
+	if (ret)
+		goto error;
+
+	if (!aw2013_chip_in_use(led->chip))
+		aw2013_chip_disable(led->chip);
+
+error:
+	mutex_unlock(&led->chip->mutex);
+
+	return ret;
+}
+
+static int aw2013_blink_set(struct led_classdev *cdev,
+			    unsigned long *delay_on, unsigned long *delay_off)
+{
+	struct aw2013_led *led = container_of(cdev, struct aw2013_led, cdev);
+	int ret, num = led->num;
+	unsigned long off = 0, on = 0;
+
+	/* If no blink specified, default to 1 Hz. */
+	if (!*delay_off && !*delay_on) {
+		*delay_off = 500;
+		*delay_on = 500;
+	}
+
+	if (!led->cdev.brightness) {
+		led->cdev.brightness = LED_FULL;
+		ret = aw2013_brightness_set(&led->cdev, led->cdev.brightness);
+		if (ret)
+			return ret;
+	}
+
+	/* Never on - just set to off */
+	if (!*delay_on) {
+		led->cdev.brightness = LED_OFF;
+		return aw2013_brightness_set(&led->cdev, LED_OFF);
+	}
+
+	mutex_lock(&led->chip->mutex);
+
+	/* Never off - brightness is already set, disable blinking */
+	if (!*delay_off) {
+		ret = regmap_update_bits(led->chip->regmap, AW2013_LCFG(num),
+					 AW2013_LCFG_MD, 0);
+		goto out;
+	}
+
+	/* Convert into values the HW will understand. */
+	off = min(5, ilog2((*delay_off - 1) / AW2013_TIME_STEP) + 1);
+	on = min(7, ilog2((*delay_on - 1) / AW2013_TIME_STEP) + 1);
+
+	*delay_off = BIT(off) * AW2013_TIME_STEP;
+	*delay_on = BIT(on) * AW2013_TIME_STEP;
+
+	/* Set timings */
+	ret = regmap_write(led->chip->regmap,
+			   AW2013_LEDT0(num), AW2013_LEDT0_T2(on));
+	if (ret)
+		goto out;
+	ret = regmap_write(led->chip->regmap,
+			   AW2013_LEDT1(num), AW2013_LEDT1_T4(off));
+	if (ret)
+		goto out;
+
+	/* Finally, enable the LED */
+	ret = regmap_update_bits(led->chip->regmap, AW2013_LCFG(num),
+				 AW2013_LCFG_MD, 0xFF);
+	if (ret)
+		goto out;
+
+	ret = regmap_update_bits(led->chip->regmap, AW2013_LCTR,
+				 AW2013_LCTR_LE(num), 0xFF);
+
+out:
+	mutex_unlock(&led->chip->mutex);
+
+	return ret;
+}
+
+static int aw2013_probe_dt(struct aw2013 *chip)
+{
+	struct device_node *np = chip->client->dev.of_node, *child;
+	int count, ret = 0, i = 0;
+	struct aw2013_led *led;
+
+	count = of_get_child_count(np);
+	if (!count || count > AW2013_MAX_LEDS)
+		return -EINVAL;
+
+	regmap_write(chip->regmap, AW2013_RSTR, AW2013_RSTR_RESET);
+
+	for_each_available_child_of_node(np, child) {
+		struct led_init_data init_data = {};
+		u32 source;
+		u32 imax;
+
+		ret = of_property_read_u32(child, "reg", &source);
+		if (ret != 0 || source >= AW2013_MAX_LEDS) {
+			dev_err(&chip->client->dev,
+				"Couldn't read LED address: %d\n", ret);
+			count--;
+			continue;
+		}
+
+		led = &chip->leds[i];
+		led->num = source;
+		led->chip = chip;
+		init_data.fwnode = of_fwnode_handle(child);
+
+		if (!of_property_read_u32(child, "led-max-microamp", &imax)) {
+			led->imax = min_t(u32, imax / 5000, 3);
+		} else {
+			led->imax = 1; // 5mA
+			dev_info(&chip->client->dev,
+				 "DT property led-max-microamp is missing\n");
+		}
+
+		of_property_read_string(child, "linux,default-trigger",
+					&led->cdev.default_trigger);
+
+		led->cdev.brightness_set_blocking = aw2013_brightness_set;
+		led->cdev.blink_set = aw2013_blink_set;
+
+		ret = devm_led_classdev_register_ext(&chip->client->dev,
+						     &led->cdev, &init_data);
+		if (ret < 0)
+			return ret;
+
+		i++;
+	}
+
+	if (!count)
+		return -EINVAL;
+
+	chip->num_leds = i;
+
+	return 0;
+}
+
+static const struct regmap_config aw2013_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = AW2013_REG_MAX,
+};
+
+static int aw2013_probe(struct i2c_client *client)
+{
+	struct aw2013 *chip;
+	int ret;
+	unsigned int chipid;
+
+	chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL);
+	if (!chip)
+		return -ENOMEM;
+
+	mutex_init(&chip->mutex);
+	mutex_lock(&chip->mutex);
+
+	chip->client = client;
+	i2c_set_clientdata(client, chip);
+
+	chip->regmap = devm_regmap_init_i2c(client, &aw2013_regmap_config);
+	if (IS_ERR(chip->regmap)) {
+		ret = PTR_ERR(chip->regmap);
+		dev_err(&client->dev, "Failed to allocate register map: %d\n",
+			ret);
+		goto error;
+	}
+
+	chip->vcc_regulator = devm_regulator_get(&client->dev, "vcc");
+	ret = PTR_ERR_OR_ZERO(chip->vcc_regulator);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&client->dev,
+				"Failed to request regulator: %d\n", ret);
+		goto error;
+	}
+
+	ret = regulator_enable(chip->vcc_regulator);
+	if (ret) {
+		dev_err(&client->dev,
+			"Failed to enable regulator: %d\n", ret);
+		goto error;
+	}
+
+	ret = regmap_read(chip->regmap, AW2013_RSTR, &chipid);
+	if (ret) {
+		dev_err(&client->dev, "Failed to read chip ID: %d\n",
+			ret);
+		goto error_reg;
+	}
+
+	if (chipid != AW2013_RSTR_CHIP_ID) {
+		dev_err(&client->dev, "Chip reported wrong ID: %x\n",
+			chipid);
+		ret = -ENODEV;
+		goto error_reg;
+	}
+
+	ret = aw2013_probe_dt(chip);
+	if (ret < 0)
+		goto error_reg;
+
+	ret = regulator_disable(chip->vcc_regulator);
+	if (ret) {
+		dev_err(&client->dev,
+			"Failed to disable regulator: %d\n", ret);
+		goto error;
+	}
+
+	mutex_unlock(&chip->mutex);
+
+	return 0;
+
+error_reg:
+	regulator_disable(chip->vcc_regulator);
+
+error:
+	mutex_destroy(&chip->mutex);
+	return ret;
+}
+
+static int aw2013_remove(struct i2c_client *client)
+{
+	struct aw2013 *chip = i2c_get_clientdata(client);
+
+	aw2013_chip_disable(chip);
+
+	mutex_destroy(&chip->mutex);
+
+	return 0;
+}
+
+static const struct of_device_id aw2013_match_table[] = {
+	{ .compatible = "awinic,aw2013", },
+	{ /* sentinel */ },
+};
+
+MODULE_DEVICE_TABLE(of, aw2013_match_table);
+
+static struct i2c_driver aw2013_driver = {
+	.driver = {
+		.name = "leds-aw2013",
+		.of_match_table = of_match_ptr(aw2013_match_table),
+	},
+	.probe_new = aw2013_probe,
+	.remove = aw2013_remove,
+};
+
+module_i2c_driver(aw2013_driver);
+
+MODULE_AUTHOR("Nikita Travkin <nikitos.tr@gmail.com>");
+MODULE_DESCRIPTION("AW2013 LED driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-lm355x.c b/drivers/leds/leds-lm355x.c
index a5abb499574b..11ce05249751 100644
--- a/drivers/leds/leds-lm355x.c
+++ b/drivers/leds/leds-lm355x.c
@@ -7,7 +7,6 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/i2c.h>
-#include <linux/gpio.h>
 #include <linux/leds.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
diff --git a/drivers/leds/leds-lp3952.c b/drivers/leds/leds-lp3952.c
index 4e4e542774cb..6ee9131fbf25 100644
--- a/drivers/leds/leds-lp3952.c
+++ b/drivers/leds/leds-lp3952.c
@@ -7,7 +7,7 @@
  */
 
 #include <linux/delay.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
diff --git a/drivers/leds/leds-lt3593.c b/drivers/leds/leds-lt3593.c
index c94995f0daa2..9079850e6ea4 100644
--- a/drivers/leds/leds-lt3593.c
+++ b/drivers/leds/leds-lt3593.c
@@ -5,7 +5,6 @@
 #include <linux/platform_device.h>
 #include <linux/leds.h>
 #include <linux/delay.h>
-#include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/module.h>
diff --git a/drivers/leds/leds-netxbig.c b/drivers/leds/leds-netxbig.c
index 14ef4ccdda3a..ceceeb6a0e96 100644
--- a/drivers/leds/leds-netxbig.c
+++ b/drivers/leds/leds-netxbig.c
@@ -12,16 +12,17 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/platform_device.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/leds.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
 
 struct netxbig_gpio_ext {
-	unsigned int	*addr;
+	struct gpio_desc **addr;
 	int		num_addr;
-	unsigned int	*data;
+	struct gpio_desc **data;
 	int		num_data;
-	unsigned int	enable;
+	struct gpio_desc *enable;
 };
 
 enum netxbig_led_mode {
@@ -69,7 +70,7 @@ static void gpio_ext_set_addr(struct netxbig_gpio_ext *gpio_ext, int addr)
 	int pin;
 
 	for (pin = 0; pin < gpio_ext->num_addr; pin++)
-		gpio_set_value(gpio_ext->addr[pin], (addr >> pin) & 1);
+		gpiod_set_value(gpio_ext->addr[pin], (addr >> pin) & 1);
 }
 
 static void gpio_ext_set_data(struct netxbig_gpio_ext *gpio_ext, int data)
@@ -77,14 +78,14 @@ static void gpio_ext_set_data(struct netxbig_gpio_ext *gpio_ext, int data)
 	int pin;
 
 	for (pin = 0; pin < gpio_ext->num_data; pin++)
-		gpio_set_value(gpio_ext->data[pin], (data >> pin) & 1);
+		gpiod_set_value(gpio_ext->data[pin], (data >> pin) & 1);
 }
 
 static void gpio_ext_enable_select(struct netxbig_gpio_ext *gpio_ext)
 {
 	/* Enable select is done on the raising edge. */
-	gpio_set_value(gpio_ext->enable, 0);
-	gpio_set_value(gpio_ext->enable, 1);
+	gpiod_set_value(gpio_ext->enable, 0);
+	gpiod_set_value(gpio_ext->enable, 1);
 }
 
 static void gpio_ext_set_value(struct netxbig_gpio_ext *gpio_ext,
@@ -99,41 +100,6 @@ static void gpio_ext_set_value(struct netxbig_gpio_ext *gpio_ext,
 	spin_unlock_irqrestore(&gpio_ext_lock, flags);
 }
 
-static int gpio_ext_init(struct platform_device *pdev,
-			 struct netxbig_gpio_ext *gpio_ext)
-{
-	int err;
-	int i;
-
-	if (unlikely(!gpio_ext))
-		return -EINVAL;
-
-	/* Configure address GPIOs. */
-	for (i = 0; i < gpio_ext->num_addr; i++) {
-		err = devm_gpio_request_one(&pdev->dev, gpio_ext->addr[i],
-					    GPIOF_OUT_INIT_LOW,
-					    "GPIO extension addr");
-		if (err)
-			return err;
-	}
-	/* Configure data GPIOs. */
-	for (i = 0; i < gpio_ext->num_data; i++) {
-		err = devm_gpio_request_one(&pdev->dev, gpio_ext->data[i],
-					    GPIOF_OUT_INIT_LOW,
-					    "GPIO extension data");
-		if (err)
-			return err;
-	}
-	/* Configure "enable select" GPIO. */
-	err = devm_gpio_request_one(&pdev->dev, gpio_ext->enable,
-				    GPIOF_OUT_INIT_LOW,
-				    "GPIO extension enable");
-	if (err)
-		return err;
-
-	return 0;
-}
-
 /*
  * Class LED driver.
  */
@@ -347,15 +313,47 @@ static int create_netxbig_led(struct platform_device *pdev,
 	return devm_led_classdev_register(&pdev->dev, &led_dat->cdev);
 }
 
-static int gpio_ext_get_of_pdata(struct device *dev, struct device_node *np,
-				 struct netxbig_gpio_ext *gpio_ext)
+/**
+ * netxbig_gpio_ext_remove() - Clean up GPIO extension data
+ * @data: managed resource data to clean up
+ *
+ * Since we pick GPIO descriptors from another device than the device our
+ * driver is probing to, we need to register a specific callback to free
+ * these up using managed resources.
+ */
+static void netxbig_gpio_ext_remove(void *data)
+{
+	struct netxbig_gpio_ext *gpio_ext = data;
+	int i;
+
+	for (i = 0; i < gpio_ext->num_addr; i++)
+		gpiod_put(gpio_ext->addr[i]);
+	for (i = 0; i < gpio_ext->num_data; i++)
+		gpiod_put(gpio_ext->data[i]);
+	gpiod_put(gpio_ext->enable);
+}
+
+/**
+ * netxbig_gpio_ext_get() - Obtain GPIO extension device data
+ * @dev: main LED device
+ * @gpio_ext_dev: the GPIO extension device
+ * @gpio_ext: the data structure holding the GPIO extension data
+ *
+ * This function walks the subdevice that only contain GPIO line
+ * handles in the device tree and obtains the GPIO descriptors from that
+ * device.
+ */
+static int netxbig_gpio_ext_get(struct device *dev,
+				struct device *gpio_ext_dev,
+				struct netxbig_gpio_ext *gpio_ext)
 {
-	int *addr, *data;
+	struct gpio_desc **addr, **data;
 	int num_addr, num_data;
+	struct gpio_desc *gpiod;
 	int ret;
 	int i;
 
-	ret = of_gpio_named_count(np, "addr-gpios");
+	ret = gpiod_count(gpio_ext_dev, "addr");
 	if (ret < 0) {
 		dev_err(dev,
 			"Failed to count GPIOs in DT property addr-gpios\n");
@@ -366,16 +364,25 @@ static int gpio_ext_get_of_pdata(struct device *dev, struct device_node *np,
 	if (!addr)
 		return -ENOMEM;
 
+	/*
+	 * We cannot use devm_ managed resources with these GPIO descriptors
+	 * since they are associated with the "GPIO extension device" which
+	 * does not probe any driver. The device tree parser will however
+	 * populate a platform device for it so we can anyway obtain the
+	 * GPIO descriptors from the device.
+	 */
 	for (i = 0; i < num_addr; i++) {
-		ret = of_get_named_gpio(np, "addr-gpios", i);
-		if (ret < 0)
-			return ret;
-		addr[i] = ret;
+		gpiod = gpiod_get_index(gpio_ext_dev, "addr", i,
+					GPIOD_OUT_LOW);
+		if (IS_ERR(gpiod))
+			return PTR_ERR(gpiod);
+		gpiod_set_consumer_name(gpiod, "GPIO extension addr");
+		addr[i] = gpiod;
 	}
 	gpio_ext->addr = addr;
 	gpio_ext->num_addr = num_addr;
 
-	ret = of_gpio_named_count(np, "data-gpios");
+	ret = gpiod_count(gpio_ext_dev, "data");
 	if (ret < 0) {
 		dev_err(dev,
 			"Failed to count GPIOs in DT property data-gpios\n");
@@ -387,23 +394,26 @@ static int gpio_ext_get_of_pdata(struct device *dev, struct device_node *np,
 		return -ENOMEM;
 
 	for (i = 0; i < num_data; i++) {
-		ret = of_get_named_gpio(np, "data-gpios", i);
-		if (ret < 0)
-			return ret;
-		data[i] = ret;
+		gpiod = gpiod_get_index(gpio_ext_dev, "data", i,
+					GPIOD_OUT_LOW);
+		if (IS_ERR(gpiod))
+			return PTR_ERR(gpiod);
+		gpiod_set_consumer_name(gpiod, "GPIO extension data");
+		data[i] = gpiod;
 	}
 	gpio_ext->data = data;
 	gpio_ext->num_data = num_data;
 
-	ret = of_get_named_gpio(np, "enable-gpio", 0);
-	if (ret < 0) {
+	gpiod = gpiod_get(gpio_ext_dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(gpiod)) {
 		dev_err(dev,
 			"Failed to get GPIO from DT property enable-gpio\n");
-		return ret;
+		return PTR_ERR(gpiod);
 	}
-	gpio_ext->enable = ret;
+	gpiod_set_consumer_name(gpiod, "GPIO extension enable");
+	gpio_ext->enable = gpiod;
 
-	return 0;
+	return devm_add_action_or_reset(dev, netxbig_gpio_ext_remove, gpio_ext);
 }
 
 static int netxbig_leds_get_of_pdata(struct device *dev,
@@ -411,6 +421,8 @@ static int netxbig_leds_get_of_pdata(struct device *dev,
 {
 	struct device_node *np = dev->of_node;
 	struct device_node *gpio_ext_np;
+	struct platform_device *gpio_ext_pdev;
+	struct device *gpio_ext_dev;
 	struct device_node *child;
 	struct netxbig_gpio_ext *gpio_ext;
 	struct netxbig_led_timer *timers;
@@ -426,13 +438,19 @@ static int netxbig_leds_get_of_pdata(struct device *dev,
 		dev_err(dev, "Failed to get DT handle gpio-ext\n");
 		return -EINVAL;
 	}
+	gpio_ext_pdev = of_find_device_by_node(gpio_ext_np);
+	if (!gpio_ext_pdev) {
+		dev_err(dev, "Failed to find platform device for gpio-ext\n");
+		return -ENODEV;
+	}
+	gpio_ext_dev = &gpio_ext_pdev->dev;
 
 	gpio_ext = devm_kzalloc(dev, sizeof(*gpio_ext), GFP_KERNEL);
 	if (!gpio_ext) {
 		of_node_put(gpio_ext_np);
 		return -ENOMEM;
 	}
-	ret = gpio_ext_get_of_pdata(dev, gpio_ext_np, gpio_ext);
+	ret = netxbig_gpio_ext_get(dev, gpio_ext_dev, gpio_ext);
 	of_node_put(gpio_ext_np);
 	if (ret)
 		return ret;
@@ -585,10 +603,6 @@ static int netxbig_led_probe(struct platform_device *pdev)
 	if (!leds_data)
 		return -ENOMEM;
 
-	ret = gpio_ext_init(pdev, pdata->gpio_ext);
-	if (ret < 0)
-		return ret;
-
 	for (i = 0; i < pdata->num_leds; i++) {
 		ret = create_netxbig_led(pdev, pdata,
 					 &leds_data[i], &pdata->leds[i]);
diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c
index 6c8a724aac51..ef7b91bd2064 100644
--- a/drivers/leds/leds-pwm.c
+++ b/drivers/leds/leds-pwm.c
@@ -91,15 +91,21 @@ static int led_pwm_add(struct device *dev, struct led_pwm_priv *priv,
 	pwm_init_state(led_data->pwm, &led_data->pwmstate);
 
 	ret = devm_led_classdev_register(dev, &led_data->cdev);
-	if (ret == 0) {
-		priv->num_leds++;
-		led_pwm_set(&led_data->cdev, led_data->cdev.brightness);
-	} else {
+	if (ret) {
 		dev_err(dev, "failed to register PWM led for %s: %d\n",
 			led->name, ret);
+		return ret;
 	}
 
-	return ret;
+	ret = led_pwm_set(&led_data->cdev, led_data->cdev.brightness);
+	if (ret) {
+		dev_err(dev, "failed to set led PWM value for %s: %d",
+			led->name, ret);
+		return ret;
+	}
+
+	priv->num_leds++;
+	return 0;
 }
 
 static int led_pwm_create_fwnode(struct device *dev, struct led_pwm_priv *priv)
diff --git a/drivers/leds/leds-sgm3140.c b/drivers/leds/leds-sgm3140.c
new file mode 100644
index 000000000000..c494b934ae09
--- /dev/null
+++ b/drivers/leds/leds-sgm3140.c
@@ -0,0 +1,320 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 Luca Weiss <luca@z3ntu.xyz>
+
+#include <linux/gpio/consumer.h>
+#include <linux/led-class-flash.h>
+#include <linux/module.h>
+#include <linux/regulator/consumer.h>
+#include <linux/platform_device.h>
+
+#include <media/v4l2-flash-led-class.h>
+
+#define FLASH_TIMEOUT_DEFAULT		250000U /* 250ms */
+#define FLASH_MAX_TIMEOUT_DEFAULT	300000U /* 300ms */
+
+struct sgm3140 {
+	struct led_classdev_flash fled_cdev;
+	struct v4l2_flash *v4l2_flash;
+
+	struct timer_list powerdown_timer;
+
+	struct gpio_desc *flash_gpio;
+	struct gpio_desc *enable_gpio;
+	struct regulator *vin_regulator;
+
+	bool enabled;
+
+	/* current timeout in us */
+	u32 timeout;
+	/* maximum timeout in us */
+	u32 max_timeout;
+};
+
+static struct sgm3140 *flcdev_to_sgm3140(struct led_classdev_flash *flcdev)
+{
+	return container_of(flcdev, struct sgm3140, fled_cdev);
+}
+
+static int sgm3140_strobe_set(struct led_classdev_flash *fled_cdev, bool state)
+{
+	struct sgm3140 *priv = flcdev_to_sgm3140(fled_cdev);
+	int ret;
+
+	if (priv->enabled == state)
+		return 0;
+
+	if (state) {
+		ret = regulator_enable(priv->vin_regulator);
+		if (ret) {
+			dev_err(fled_cdev->led_cdev.dev,
+				"failed to enable regulator: %d\n", ret);
+			return ret;
+		}
+		gpiod_set_value_cansleep(priv->flash_gpio, 1);
+		gpiod_set_value_cansleep(priv->enable_gpio, 1);
+		mod_timer(&priv->powerdown_timer,
+			  jiffies + usecs_to_jiffies(priv->timeout));
+	} else {
+		del_timer_sync(&priv->powerdown_timer);
+		gpiod_set_value_cansleep(priv->enable_gpio, 0);
+		gpiod_set_value_cansleep(priv->flash_gpio, 0);
+		ret = regulator_disable(priv->vin_regulator);
+		if (ret) {
+			dev_err(fled_cdev->led_cdev.dev,
+				"failed to disable regulator: %d\n", ret);
+			return ret;
+		}
+	}
+
+	priv->enabled = state;
+
+	return 0;
+}
+
+static int sgm3140_strobe_get(struct led_classdev_flash *fled_cdev, bool *state)
+{
+	struct sgm3140 *priv = flcdev_to_sgm3140(fled_cdev);
+
+	*state = timer_pending(&priv->powerdown_timer);
+
+	return 0;
+}
+
+static int sgm3140_timeout_set(struct led_classdev_flash *fled_cdev,
+			       u32 timeout)
+{
+	struct sgm3140 *priv = flcdev_to_sgm3140(fled_cdev);
+
+	priv->timeout = timeout;
+
+	return 0;
+}
+
+static const struct led_flash_ops sgm3140_flash_ops = {
+	.strobe_set = sgm3140_strobe_set,
+	.strobe_get = sgm3140_strobe_get,
+	.timeout_set = sgm3140_timeout_set,
+};
+
+static int sgm3140_brightness_set(struct led_classdev *led_cdev,
+				  enum led_brightness brightness)
+{
+	struct led_classdev_flash *fled_cdev = lcdev_to_flcdev(led_cdev);
+	struct sgm3140 *priv = flcdev_to_sgm3140(fled_cdev);
+	bool enable = brightness == LED_ON;
+	int ret;
+
+	if (priv->enabled == enable)
+		return 0;
+
+	if (enable) {
+		ret = regulator_enable(priv->vin_regulator);
+		if (ret) {
+			dev_err(led_cdev->dev,
+				"failed to enable regulator: %d\n", ret);
+			return ret;
+		}
+		gpiod_set_value_cansleep(priv->enable_gpio, 1);
+	} else {
+		gpiod_set_value_cansleep(priv->enable_gpio, 0);
+		ret = regulator_disable(priv->vin_regulator);
+		if (ret) {
+			dev_err(led_cdev->dev,
+				"failed to disable regulator: %d\n", ret);
+			return ret;
+		}
+	}
+
+	priv->enabled = enable;
+
+	return 0;
+}
+
+static void sgm3140_powerdown_timer(struct timer_list *t)
+{
+	struct sgm3140 *priv = from_timer(priv, t, powerdown_timer);
+
+	gpiod_set_value(priv->enable_gpio, 0);
+	gpiod_set_value(priv->flash_gpio, 0);
+	regulator_disable(priv->vin_regulator);
+
+	priv->enabled = false;
+}
+
+static void sgm3140_init_flash_timeout(struct sgm3140 *priv)
+{
+	struct led_classdev_flash *fled_cdev = &priv->fled_cdev;
+	struct led_flash_setting *s;
+
+	/* Init flash timeout setting */
+	s = &fled_cdev->timeout;
+	s->min = 1;
+	s->max = priv->max_timeout;
+	s->step = 1;
+	s->val = FLASH_TIMEOUT_DEFAULT;
+}
+
+#if IS_ENABLED(CONFIG_V4L2_FLASH_LED_CLASS)
+static void sgm3140_init_v4l2_flash_config(struct sgm3140 *priv,
+					struct v4l2_flash_config *v4l2_sd_cfg)
+{
+	struct led_classdev *led_cdev = &priv->fled_cdev.led_cdev;
+	struct led_flash_setting *s;
+
+	strscpy(v4l2_sd_cfg->dev_name, led_cdev->dev->kobj.name,
+		sizeof(v4l2_sd_cfg->dev_name));
+
+	/* Init flash intensity setting */
+	s = &v4l2_sd_cfg->intensity;
+	s->min = 0;
+	s->max = 1;
+	s->step = 1;
+	s->val = 1;
+}
+
+#else
+static void sgm3140_init_v4l2_flash_config(struct sgm3140 *priv,
+					struct v4l2_flash_config *v4l2_sd_cfg)
+{
+}
+#endif
+
+static int sgm3140_probe(struct platform_device *pdev)
+{
+	struct sgm3140 *priv;
+	struct led_classdev *led_cdev;
+	struct led_classdev_flash *fled_cdev;
+	struct led_init_data init_data = {};
+	struct fwnode_handle *child_node;
+	struct v4l2_flash_config v4l2_sd_cfg = {};
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->flash_gpio = devm_gpiod_get(&pdev->dev, "flash", GPIOD_OUT_LOW);
+	ret = PTR_ERR_OR_ZERO(priv->flash_gpio);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev,
+				"Failed to request flash gpio: %d\n", ret);
+		return ret;
+	}
+
+	priv->enable_gpio = devm_gpiod_get(&pdev->dev, "enable", GPIOD_OUT_LOW);
+	ret = PTR_ERR_OR_ZERO(priv->enable_gpio);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev,
+				"Failed to request enable gpio: %d\n", ret);
+		return ret;
+	}
+
+	priv->vin_regulator = devm_regulator_get(&pdev->dev, "vin");
+	ret = PTR_ERR_OR_ZERO(priv->vin_regulator);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev,
+				"Failed to request regulator: %d\n", ret);
+		return ret;
+	}
+
+	child_node = fwnode_get_next_available_child_node(pdev->dev.fwnode,
+							  NULL);
+	if (!child_node) {
+		dev_err(&pdev->dev,
+			"No fwnode child node found for connected LED.\n");
+		return -EINVAL;
+	}
+
+	ret = fwnode_property_read_u32(child_node, "flash-max-timeout-us",
+				       &priv->max_timeout);
+	if (ret) {
+		priv->max_timeout = FLASH_MAX_TIMEOUT_DEFAULT;
+		dev_warn(&pdev->dev,
+			 "flash-max-timeout-us property missing\n");
+	}
+
+	/*
+	 * Set default timeout to FLASH_DEFAULT_TIMEOUT except if max_timeout
+	 * from DT is lower.
+	 */
+	priv->timeout = min(priv->max_timeout, FLASH_TIMEOUT_DEFAULT);
+
+	timer_setup(&priv->powerdown_timer, sgm3140_powerdown_timer, 0);
+
+	fled_cdev = &priv->fled_cdev;
+	led_cdev = &fled_cdev->led_cdev;
+
+	fled_cdev->ops = &sgm3140_flash_ops;
+
+	led_cdev->brightness_set_blocking = sgm3140_brightness_set;
+	led_cdev->max_brightness = LED_ON;
+	led_cdev->flags |= LED_DEV_CAP_FLASH;
+
+	sgm3140_init_flash_timeout(priv);
+
+	init_data.fwnode = child_node;
+
+	platform_set_drvdata(pdev, priv);
+
+	/* Register in the LED subsystem */
+	ret = devm_led_classdev_flash_register_ext(&pdev->dev,
+						   fled_cdev, &init_data);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to register flash device: %d\n",
+			ret);
+		goto err;
+	}
+
+	sgm3140_init_v4l2_flash_config(priv, &v4l2_sd_cfg);
+
+	/* Create V4L2 Flash subdev */
+	priv->v4l2_flash = v4l2_flash_init(&pdev->dev,
+					   child_node,
+					   fled_cdev, NULL,
+					   &v4l2_sd_cfg);
+	if (IS_ERR(priv->v4l2_flash)) {
+		ret = PTR_ERR(priv->v4l2_flash);
+		goto err;
+	}
+
+	return ret;
+
+err:
+	fwnode_handle_put(child_node);
+	return ret;
+}
+
+static int sgm3140_remove(struct platform_device *pdev)
+{
+	struct sgm3140 *priv = platform_get_drvdata(pdev);
+
+	del_timer_sync(&priv->powerdown_timer);
+
+	v4l2_flash_release(priv->v4l2_flash);
+
+	return 0;
+}
+
+static const struct of_device_id sgm3140_dt_match[] = {
+	{ .compatible = "sgmicro,sgm3140" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sgm3140_dt_match);
+
+static struct platform_driver sgm3140_driver = {
+	.probe	= sgm3140_probe,
+	.remove	= sgm3140_remove,
+	.driver	= {
+		.name	= "sgm3140",
+		.of_match_table = sgm3140_dt_match,
+	},
+};
+
+module_platform_driver(sgm3140_driver);
+
+MODULE_AUTHOR("Luca Weiss <luca@z3ntu.xyz>");
+MODULE_DESCRIPTION("SG Micro SGM3140 charge pump led driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-tca6507.c b/drivers/leds/leds-tca6507.c
index 58be20cae183..1128ac75443c 100644
--- a/drivers/leds/leds-tca6507.c
+++ b/drivers/leds/leds-tca6507.c
@@ -93,7 +93,7 @@
 #include <linux/leds.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/workqueue.h>
 #include <linux/leds-tca6507.h>
 #include <linux/of.h>
diff --git a/drivers/leds/leds-tlc591xx.c b/drivers/leds/leds-tlc591xx.c
index a8911ebd30e5..0929f1275814 100644
--- a/drivers/leds/leds-tlc591xx.c
+++ b/drivers/leds/leds-tlc591xx.c
@@ -214,8 +214,9 @@ tlc591xx_probe(struct i2c_client *client,
 		err = devm_led_classdev_register_ext(dev, &led->ldev,
 						     &init_data);
 		if (err < 0) {
-			dev_err(dev, "couldn't register LED %s\n",
-				led->ldev.name);
+			if (err != -EPROBE_DEFER)
+				dev_err(dev, "couldn't register LED %s\n",
+					led->ldev.name);
 			return err;
 		}
 	}
diff --git a/drivers/leds/trigger/ledtrig-timer.c b/drivers/leds/trigger/ledtrig-timer.c
index 34a68604c46c..b4688d1d9d2b 100644
--- a/drivers/leds/trigger/ledtrig-timer.c
+++ b/drivers/leds/trigger/ledtrig-timer.c
@@ -28,7 +28,7 @@ static ssize_t led_delay_on_store(struct device *dev,
 {
 	struct led_classdev *led_cdev = led_trigger_get_led(dev);
 	unsigned long state;
-	ssize_t ret = -EINVAL;
+	ssize_t ret;
 
 	ret = kstrtoul(buf, 10, &state);
 	if (ret)
@@ -53,7 +53,7 @@ static ssize_t led_delay_off_store(struct device *dev,
 {
 	struct led_classdev *led_cdev = led_trigger_get_led(dev);
 	unsigned long state;
-	ssize_t ret = -EINVAL;
+	ssize_t ret;
 
 	ret = kstrtoul(buf, 10, &state);
 	if (ret)
diff --git a/drivers/macintosh/Kconfig b/drivers/macintosh/Kconfig
index cbd46c1c5bf7..5cdc361da37c 100644
--- a/drivers/macintosh/Kconfig
+++ b/drivers/macintosh/Kconfig
@@ -4,7 +4,7 @@ menuconfig MACINTOSH_DRIVERS
 	bool "Macintosh device drivers"
 	depends on PPC || MAC || X86
 	default y if (PPC_PMAC || MAC)
-	---help---
+	help
 	  Say Y here to get to see options for devices used with Macintosh
 	  computers. This option alone does not add any kernel code.
 
@@ -247,7 +247,6 @@ config PMAC_RACKMETER
 config SENSORS_AMS
 	tristate "Apple Motion Sensor driver"
 	depends on PPC_PMAC && !PPC64 && INPUT && ((ADB_PMU && I2C = y) || (ADB_PMU && !I2C) || I2C)
-	select INPUT_POLLDEV
 	help
 	  Support for the motion sensor included in PowerBooks. Includes
 	  implementations for PMU and I2C.
diff --git a/drivers/macintosh/ams/ams-input.c b/drivers/macintosh/ams/ams-input.c
index 06a96b3f11de..0da493d449b2 100644
--- a/drivers/macintosh/ams/ams-input.c
+++ b/drivers/macintosh/ams/ams-input.c
@@ -25,9 +25,8 @@ MODULE_PARM_DESC(invert, "Invert input data on X and Y axis");
 
 static DEFINE_MUTEX(ams_input_mutex);
 
-static void ams_idev_poll(struct input_polled_dev *dev)
+static void ams_idev_poll(struct input_dev *idev)
 {
-	struct input_dev *idev = dev->input;
 	s8 x, y, z;
 
 	mutex_lock(&ams_info.lock);
@@ -59,14 +58,10 @@ static int ams_input_enable(void)
 	ams_info.ycalib = y;
 	ams_info.zcalib = z;
 
-	ams_info.idev = input_allocate_polled_device();
-	if (!ams_info.idev)
+	input = input_allocate_device();
+	if (!input)
 		return -ENOMEM;
 
-	ams_info.idev->poll = ams_idev_poll;
-	ams_info.idev->poll_interval = 25;
-
-	input = ams_info.idev->input;
 	input->name = "Apple Motion Sensor";
 	input->id.bustype = ams_info.bustype;
 	input->id.vendor = 0;
@@ -75,28 +70,32 @@ static int ams_input_enable(void)
 	input_set_abs_params(input, ABS_X, -50, 50, 3, 0);
 	input_set_abs_params(input, ABS_Y, -50, 50, 3, 0);
 	input_set_abs_params(input, ABS_Z, -50, 50, 3, 0);
+	input_set_capability(input, EV_KEY, BTN_TOUCH);
 
-	set_bit(EV_ABS, input->evbit);
-	set_bit(EV_KEY, input->evbit);
-	set_bit(BTN_TOUCH, input->keybit);
+	error = input_setup_polling(input, ams_idev_poll);
+	if (error)
+		goto err_free_input;
 
-	error = input_register_polled_device(ams_info.idev);
-	if (error) {
-		input_free_polled_device(ams_info.idev);
-		ams_info.idev = NULL;
-		return error;
-	}
+	input_set_poll_interval(input, 25);
 
+	error = input_register_device(input);
+	if (error)
+		goto err_free_input;
+
+	ams_info.idev = input;
 	joystick = true;
 
 	return 0;
+
+err_free_input:
+	input_free_device(input);
+	return error;
 }
 
 static void ams_input_disable(void)
 {
 	if (ams_info.idev) {
-		input_unregister_polled_device(ams_info.idev);
-		input_free_polled_device(ams_info.idev);
+		input_unregister_device(ams_info.idev);
 		ams_info.idev = NULL;
 	}
 
diff --git a/drivers/macintosh/ams/ams.h b/drivers/macintosh/ams/ams.h
index fe8d596f9845..935bdd9cd9a6 100644
--- a/drivers/macintosh/ams/ams.h
+++ b/drivers/macintosh/ams/ams.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <linux/i2c.h>
-#include <linux/input-polldev.h>
+#include <linux/input.h>
 #include <linux/kthread.h>
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
@@ -51,7 +51,7 @@ struct ams {
 #endif
 
 	/* Joystick emulation */
-	struct input_polled_dev *idev;
+	struct input_dev *idev;
 	__u16 bustype;
 
 	/* calibrated null values */
diff --git a/drivers/macintosh/macio-adb.c b/drivers/macintosh/macio-adb.c
index eb3adfb7f88d..d4759db002c6 100644
--- a/drivers/macintosh/macio-adb.c
+++ b/drivers/macintosh/macio-adb.c
@@ -9,10 +9,10 @@
 #include <linux/delay.h>
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
+#include <linux/pgtable.h>
 #include <asm/prom.h>
 #include <linux/adb.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/hydra.h>
 #include <asm/irq.h>
 #include <linux/init.h>
diff --git a/drivers/macintosh/mediabay.c b/drivers/macintosh/mediabay.c
index 74bf2938276b..eab7e83c11c4 100644
--- a/drivers/macintosh/mediabay.c
+++ b/drivers/macintosh/mediabay.c
@@ -16,8 +16,8 @@
 #include <linux/init.h>
 #include <linux/kthread.h>
 #include <linux/mutex.h>
+#include <linux/pgtable.h>
 #include <asm/prom.h>
-#include <asm/pgtable.h>
 #include <asm/io.h>
 #include <asm/machdep.h>
 #include <asm/pmac_feature.h>
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index 8450d7c008d0..73e6ae88fafd 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -50,9 +50,9 @@
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/uaccess.h>
+#include <linux/pgtable.h>
 #include <asm/machdep.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/sections.h>
 #include <asm/irq.h>
 #ifdef CONFIG_PPC_PMAC
diff --git a/drivers/macintosh/windfarm_pm112.c b/drivers/macintosh/windfarm_pm112.c
index 4150301a89a5..e8377ce0a95a 100644
--- a/drivers/macintosh/windfarm_pm112.c
+++ b/drivers/macintosh/windfarm_pm112.c
@@ -132,14 +132,6 @@ static int create_cpu_loop(int cpu)
 	s32 tmax;
 	int fmin;
 
-	/* Get PID params from the appropriate SAT */
-	hdr = smu_sat_get_sdb_partition(chip, 0xC8 + core, NULL);
-	if (hdr == NULL) {
-		printk(KERN_WARNING"windfarm: can't get CPU PID fan config\n");
-		return -EINVAL;
-	}
-	piddata = (struct smu_sdbp_cpupiddata *)&hdr[1];
-
 	/* Get FVT params to get Tmax; if not found, assume default */
 	hdr = smu_sat_get_sdb_partition(chip, 0xC4 + core, NULL);
 	if (hdr) {
@@ -152,6 +144,16 @@ static int create_cpu_loop(int cpu)
 	if (tmax < cpu_all_tmax)
 		cpu_all_tmax = tmax;
 
+	kfree(hdr);
+
+	/* Get PID params from the appropriate SAT */
+	hdr = smu_sat_get_sdb_partition(chip, 0xC8 + core, NULL);
+	if (hdr == NULL) {
+		printk(KERN_WARNING"windfarm: can't get CPU PID fan config\n");
+		return -EINVAL;
+	}
+	piddata = (struct smu_sdbp_cpupiddata *)&hdr[1];
+
 	/*
 	 * Darwin has a minimum fan speed of 1000 rpm for the 4-way and
 	 * 515 for the 2-way.  That appears to be overkill, so for now,
@@ -174,6 +176,9 @@ static int create_cpu_loop(int cpu)
 		pid.min = fmin;
 
 	wf_cpu_pid_init(&cpu_pid[cpu], &pid);
+
+	kfree(hdr);
+
 	return 0;
 }
 
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index 5a577a6734cf..05b1009e2820 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -236,4 +236,22 @@ config SUN6I_MSGBOX
 	  various Allwinner SoCs. This mailbox is used for communication
 	  between the application CPUs and the power management coprocessor.
 
+config SPRD_MBOX
+	tristate "Spreadtrum Mailbox"
+	depends on ARCH_SPRD || COMPILE_TEST
+	help
+	  Mailbox driver implementation for the Spreadtrum platform. It is used
+	  to send message between application processors and MCU. Say Y here if
+	  you want to build the Spreatrum mailbox controller driver.
+
+config QCOM_IPCC
+	bool "Qualcomm Technologies, Inc. IPCC driver"
+	depends on ARCH_QCOM || COMPILE_TEST
+	help
+	  Qualcomm Technologies, Inc. Inter-Processor Communication Controller
+	  (IPCC) driver for MSM devices. The driver provides mailbox support for
+	  sending interrupts to the clients. On the other hand, the driver also
+	  acts as an interrupt controller for receiving interrupts from clients.
+	  Say Y here if you want to build this driver.
+
 endif
diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile
index 2e4364ef5c47..60d224b723a1 100644
--- a/drivers/mailbox/Makefile
+++ b/drivers/mailbox/Makefile
@@ -50,3 +50,7 @@ obj-$(CONFIG_MTK_CMDQ_MBOX)	+= mtk-cmdq-mailbox.o
 obj-$(CONFIG_ZYNQMP_IPI_MBOX)	+= zynqmp-ipi-mailbox.o
 
 obj-$(CONFIG_SUN6I_MSGBOX)	+= sun6i-msgbox.o
+
+obj-$(CONFIG_SPRD_MBOX)		+= sprd-mailbox.o
+
+obj-$(CONFIG_QCOM_IPCC)		+= qcom-ipcc.o
diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c
index 7906624a731c..7205b825c8b5 100644
--- a/drivers/mailbox/imx-mailbox.c
+++ b/drivers/mailbox/imx-mailbox.c
@@ -12,6 +12,7 @@
 #include <linux/mailbox_controller.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/slab.h>
 
 #define IMX_MU_xSR_GIPn(x)	BIT(28 + (3 - (x)))
@@ -66,6 +67,8 @@ struct imx_mu_priv {
 	struct clk		*clk;
 	int			irq;
 
+	u32 xcr;
+
 	bool			side_b;
 };
 
@@ -154,12 +157,17 @@ static int imx_mu_scu_tx(struct imx_mu_priv *priv,
 
 	switch (cp->type) {
 	case IMX_MU_TYPE_TX:
-		if (msg->hdr.size > sizeof(*msg)) {
+		/*
+		 * msg->hdr.size specifies the number of u32 words while
+		 * sizeof yields bytes.
+		 */
+
+		if (msg->hdr.size > sizeof(*msg) / 4) {
 			/*
 			 * The real message size can be different to
 			 * struct imx_sc_rpc_msg_max size
 			 */
-			dev_err(priv->dev, "Exceed max msg size (%zu) on TX, got: %i\n", sizeof(*msg), msg->hdr.size);
+			dev_err(priv->dev, "Maximal message size (%zu bytes) exceeded on TX; got: %i bytes\n", sizeof(*msg), msg->hdr.size << 2);
 			return -EINVAL;
 		}
 
@@ -198,9 +206,8 @@ static int imx_mu_scu_rx(struct imx_mu_priv *priv,
 	imx_mu_xcr_rmw(priv, 0, IMX_MU_xCR_RIEn(0));
 	*data++ = imx_mu_read(priv, priv->dcfg->xRR[0]);
 
-	if (msg.hdr.size > sizeof(msg)) {
-		dev_err(priv->dev, "Exceed max msg size (%zu) on RX, got: %i\n",
-			sizeof(msg), msg.hdr.size);
+	if (msg.hdr.size > sizeof(msg) / 4) {
+		dev_err(priv->dev, "Maximal message size (%zu bytes) exceeded on RX; got: %i bytes\n", sizeof(msg), msg.hdr.size << 2);
 		return -EINVAL;
 	}
 
@@ -285,8 +292,10 @@ static int imx_mu_startup(struct mbox_chan *chan)
 {
 	struct imx_mu_priv *priv = to_imx_mu_priv(chan->mbox);
 	struct imx_mu_con_priv *cp = chan->con_priv;
+	unsigned long irq_flag = IRQF_SHARED;
 	int ret;
 
+	pm_runtime_get_sync(priv->dev);
 	if (cp->type == IMX_MU_TYPE_TXDB) {
 		/* Tx doorbell don't have ACK support */
 		tasklet_init(&cp->txdb_tasklet, imx_mu_txdb_tasklet,
@@ -294,8 +303,12 @@ static int imx_mu_startup(struct mbox_chan *chan)
 		return 0;
 	}
 
-	ret = request_irq(priv->irq, imx_mu_isr, IRQF_SHARED |
-			  IRQF_NO_SUSPEND, cp->irq_desc, chan);
+	/* IPC MU should be with IRQF_NO_SUSPEND set */
+	if (!priv->dev->pm_domain)
+		irq_flag |= IRQF_NO_SUSPEND;
+
+	ret = request_irq(priv->irq, imx_mu_isr, irq_flag,
+			  cp->irq_desc, chan);
 	if (ret) {
 		dev_err(priv->dev,
 			"Unable to acquire IRQ %d\n", priv->irq);
@@ -323,6 +336,7 @@ static void imx_mu_shutdown(struct mbox_chan *chan)
 
 	if (cp->type == IMX_MU_TYPE_TXDB) {
 		tasklet_kill(&cp->txdb_tasklet);
+		pm_runtime_put_sync(priv->dev);
 		return;
 	}
 
@@ -341,6 +355,7 @@ static void imx_mu_shutdown(struct mbox_chan *chan)
 	}
 
 	free_irq(priv->irq, chan);
+	pm_runtime_put_sync(priv->dev);
 }
 
 static const struct mbox_chan_ops imx_mu_ops = {
@@ -374,7 +389,7 @@ static struct mbox_chan *imx_mu_scu_xlate(struct mbox_controller *mbox,
 		break;
 	default:
 		dev_err(mbox->dev, "Invalid chan type: %d\n", type);
-		return NULL;
+		return ERR_PTR(-EINVAL);
 	}
 
 	if (chan >= mbox->num_chans) {
@@ -508,14 +523,39 @@ static int imx_mu_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, priv);
 
-	return devm_mbox_controller_register(dev, &priv->mbox);
+	ret = devm_mbox_controller_register(dev, &priv->mbox);
+	if (ret) {
+		clk_disable_unprepare(priv->clk);
+		return ret;
+	}
+
+	pm_runtime_enable(dev);
+
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		goto disable_runtime_pm;
+	}
+
+	ret = pm_runtime_put_sync(dev);
+	if (ret < 0)
+		goto disable_runtime_pm;
+
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+
+disable_runtime_pm:
+	pm_runtime_disable(dev);
+	clk_disable_unprepare(priv->clk);
+	return ret;
 }
 
 static int imx_mu_remove(struct platform_device *pdev)
 {
 	struct imx_mu_priv *priv = platform_get_drvdata(pdev);
 
-	clk_disable_unprepare(priv->clk);
+	pm_runtime_disable(priv->dev);
 
 	return 0;
 }
@@ -558,12 +598,69 @@ static const struct of_device_id imx_mu_dt_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, imx_mu_dt_ids);
 
+static int imx_mu_suspend_noirq(struct device *dev)
+{
+	struct imx_mu_priv *priv = dev_get_drvdata(dev);
+
+	if (!priv->clk)
+		priv->xcr = imx_mu_read(priv, priv->dcfg->xCR);
+
+	return 0;
+}
+
+static int imx_mu_resume_noirq(struct device *dev)
+{
+	struct imx_mu_priv *priv = dev_get_drvdata(dev);
+
+	/*
+	 * ONLY restore MU when context lost, the TIE could
+	 * be set during noirq resume as there is MU data
+	 * communication going on, and restore the saved
+	 * value will overwrite the TIE and cause MU data
+	 * send failed, may lead to system freeze. This issue
+	 * is observed by testing freeze mode suspend.
+	 */
+	if (!imx_mu_read(priv, priv->dcfg->xCR) && !priv->clk)
+		imx_mu_write(priv, priv->xcr, priv->dcfg->xCR);
+
+	return 0;
+}
+
+static int imx_mu_runtime_suspend(struct device *dev)
+{
+	struct imx_mu_priv *priv = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+}
+
+static int imx_mu_runtime_resume(struct device *dev)
+{
+	struct imx_mu_priv *priv = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		dev_err(dev, "failed to enable clock\n");
+
+	return ret;
+}
+
+static const struct dev_pm_ops imx_mu_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(imx_mu_suspend_noirq,
+				      imx_mu_resume_noirq)
+	SET_RUNTIME_PM_OPS(imx_mu_runtime_suspend,
+			   imx_mu_runtime_resume, NULL)
+};
+
 static struct platform_driver imx_mu_driver = {
 	.probe		= imx_mu_probe,
 	.remove		= imx_mu_remove,
 	.driver = {
 		.name	= "imx_mu",
 		.of_match_table = imx_mu_dt_ids,
+		.pm = &imx_mu_pm_ops,
 	},
 };
 module_platform_driver(imx_mu_driver);
diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c
index 34844b7a3675..8c7fac38bb1c 100644
--- a/drivers/mailbox/pcc.c
+++ b/drivers/mailbox/pcc.c
@@ -568,7 +568,7 @@ static int pcc_mbox_probe(struct platform_device *pdev)
 	return ret;
 }
 
-struct platform_driver pcc_mbox_driver = {
+static struct platform_driver pcc_mbox_driver = {
 	.probe = pcc_mbox_probe,
 	.driver = {
 		.name = "PCCT",
diff --git a/drivers/mailbox/qcom-apcs-ipc-mailbox.c b/drivers/mailbox/qcom-apcs-ipc-mailbox.c
index eeebafd546e5..cec34f0af6ce 100644
--- a/drivers/mailbox/qcom-apcs-ipc-mailbox.c
+++ b/drivers/mailbox/qcom-apcs-ipc-mailbox.c
@@ -24,6 +24,35 @@ struct qcom_apcs_ipc {
 	struct platform_device *clk;
 };
 
+struct qcom_apcs_ipc_data {
+	int offset;
+	char *clk_name;
+};
+
+static const struct qcom_apcs_ipc_data ipq6018_apcs_data = {
+	.offset = 8, .clk_name = "qcom,apss-ipq6018-clk"
+};
+
+static const struct qcom_apcs_ipc_data ipq8074_apcs_data = {
+	.offset = 8, .clk_name = NULL
+};
+
+static const struct qcom_apcs_ipc_data msm8916_apcs_data = {
+	.offset = 8, .clk_name = "qcom-apcs-msm8916-clk"
+};
+
+static const struct qcom_apcs_ipc_data msm8996_apcs_data = {
+	.offset = 16, .clk_name = NULL
+};
+
+static const struct qcom_apcs_ipc_data msm8998_apcs_data = {
+	.offset = 8, .clk_name = NULL
+};
+
+static const struct qcom_apcs_ipc_data apps_shared_apcs_data = {
+	.offset = 12, .clk_name = NULL
+};
+
 static const struct regmap_config apcs_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
@@ -48,17 +77,12 @@ static const struct mbox_chan_ops qcom_apcs_ipc_ops = {
 static int qcom_apcs_ipc_probe(struct platform_device *pdev)
 {
 	struct qcom_apcs_ipc *apcs;
+	const struct qcom_apcs_ipc_data *apcs_data;
 	struct regmap *regmap;
 	struct resource *res;
-	unsigned long offset;
 	void __iomem *base;
 	unsigned long i;
 	int ret;
-	const struct of_device_id apcs_clk_match_table[] = {
-		{ .compatible = "qcom,msm8916-apcs-kpss-global", },
-		{ .compatible = "qcom,qcs404-apcs-apps-global", },
-		{}
-	};
 
 	apcs = devm_kzalloc(&pdev->dev, sizeof(*apcs), GFP_KERNEL);
 	if (!apcs)
@@ -73,10 +97,10 @@ static int qcom_apcs_ipc_probe(struct platform_device *pdev)
 	if (IS_ERR(regmap))
 		return PTR_ERR(regmap);
 
-	offset = (unsigned long)of_device_get_match_data(&pdev->dev);
+	apcs_data = of_device_get_match_data(&pdev->dev);
 
 	apcs->regmap = regmap;
-	apcs->offset = offset;
+	apcs->offset = apcs_data->offset;
 
 	/* Initialize channel identifiers */
 	for (i = 0; i < ARRAY_SIZE(apcs->mbox_chans); i++)
@@ -93,9 +117,9 @@ static int qcom_apcs_ipc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	if (of_match_device(apcs_clk_match_table, &pdev->dev)) {
+	if (apcs_data->clk_name) {
 		apcs->clk = platform_device_register_data(&pdev->dev,
-							  "qcom-apcs-msm8916-clk",
+							  apcs_data->clk_name,
 							  PLATFORM_DEVID_NONE,
 							  NULL, 0);
 		if (IS_ERR(apcs->clk))
@@ -119,14 +143,15 @@ static int qcom_apcs_ipc_remove(struct platform_device *pdev)
 
 /* .data is the offset of the ipc register within the global block */
 static const struct of_device_id qcom_apcs_ipc_of_match[] = {
-	{ .compatible = "qcom,msm8916-apcs-kpss-global", .data = (void *)8 },
-	{ .compatible = "qcom,msm8996-apcs-hmss-global", .data = (void *)16 },
-	{ .compatible = "qcom,msm8998-apcs-hmss-global", .data = (void *)8 },
-	{ .compatible = "qcom,qcs404-apcs-apps-global", .data = (void *)8 },
-	{ .compatible = "qcom,sc7180-apss-shared", .data = (void *)12 },
-	{ .compatible = "qcom,sdm845-apss-shared", .data = (void *)12 },
-	{ .compatible = "qcom,sm8150-apss-shared", .data = (void *)12 },
-	{ .compatible = "qcom,ipq8074-apcs-apps-global", .data = (void *)8 },
+	{ .compatible = "qcom,ipq6018-apcs-apps-global", .data = &ipq6018_apcs_data },
+	{ .compatible = "qcom,ipq8074-apcs-apps-global", .data = &ipq8074_apcs_data },
+	{ .compatible = "qcom,msm8916-apcs-kpss-global", .data = &msm8916_apcs_data },
+	{ .compatible = "qcom,msm8996-apcs-hmss-global", .data = &msm8996_apcs_data },
+	{ .compatible = "qcom,msm8998-apcs-hmss-global", .data = &msm8998_apcs_data },
+	{ .compatible = "qcom,qcs404-apcs-apps-global", .data = &msm8916_apcs_data },
+	{ .compatible = "qcom,sc7180-apss-shared", .data = &apps_shared_apcs_data },
+	{ .compatible = "qcom,sdm845-apss-shared", .data = &apps_shared_apcs_data },
+	{ .compatible = "qcom,sm8150-apss-shared", .data = &apps_shared_apcs_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, qcom_apcs_ipc_of_match);
diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c
new file mode 100644
index 000000000000..2d13c72944c6
--- /dev/null
+++ b/drivers/mailbox/qcom-ipcc.c
@@ -0,0 +1,286 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/mailbox_controller.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include <dt-bindings/mailbox/qcom-ipcc.h>
+
+#define IPCC_MBOX_MAX_CHAN		48
+
+/* IPCC Register offsets */
+#define IPCC_REG_SEND_ID		0x0c
+#define IPCC_REG_RECV_ID		0x10
+#define IPCC_REG_RECV_SIGNAL_ENABLE	0x14
+#define IPCC_REG_RECV_SIGNAL_DISABLE	0x18
+#define IPCC_REG_RECV_SIGNAL_CLEAR	0x1c
+#define IPCC_REG_CLIENT_CLEAR		0x38
+
+#define IPCC_SIGNAL_ID_MASK		GENMASK(15, 0)
+#define IPCC_CLIENT_ID_MASK		GENMASK(31, 16)
+
+#define IPCC_NO_PENDING_IRQ		GENMASK(31, 0)
+
+/**
+ * struct qcom_ipcc_chan_info - Per-mailbox-channel info
+ * @client_id:	The client-id to which the interrupt has to be triggered
+ * @signal_id:	The signal-id to which the interrupt has to be triggered
+ */
+struct qcom_ipcc_chan_info {
+	u16 client_id;
+	u16 signal_id;
+};
+
+/**
+ * struct qcom_ipcc - Holder for the mailbox driver
+ * @dev:		Device associated with this instance
+ * @base:		Base address of the IPCC frame associated to APSS
+ * @irq_domain:		The irq_domain associated with this instance
+ * @chan:		The mailbox channels array
+ * @mchan:		The per-mailbox channel info array
+ * @mbox:		The mailbox controller
+ * @irq:		Summary irq
+ */
+struct qcom_ipcc {
+	struct device *dev;
+	void __iomem *base;
+	struct irq_domain *irq_domain;
+	struct mbox_chan chan[IPCC_MBOX_MAX_CHAN];
+	struct qcom_ipcc_chan_info mchan[IPCC_MBOX_MAX_CHAN];
+	struct mbox_controller mbox;
+	int irq;
+};
+
+static inline struct qcom_ipcc *to_qcom_ipcc(struct mbox_controller *mbox)
+{
+	return container_of(mbox, struct qcom_ipcc, mbox);
+}
+
+static inline u32 qcom_ipcc_get_hwirq(u16 client_id, u16 signal_id)
+{
+	return FIELD_PREP(IPCC_CLIENT_ID_MASK, client_id) |
+	       FIELD_PREP(IPCC_SIGNAL_ID_MASK, signal_id);
+}
+
+static irqreturn_t qcom_ipcc_irq_fn(int irq, void *data)
+{
+	struct qcom_ipcc *ipcc = data;
+	u32 hwirq;
+	int virq;
+
+	for (;;) {
+		hwirq = readl(ipcc->base + IPCC_REG_RECV_ID);
+		if (hwirq == IPCC_NO_PENDING_IRQ)
+			break;
+
+		virq = irq_find_mapping(ipcc->irq_domain, hwirq);
+		writel(hwirq, ipcc->base + IPCC_REG_RECV_SIGNAL_CLEAR);
+		generic_handle_irq(virq);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void qcom_ipcc_mask_irq(struct irq_data *irqd)
+{
+	struct qcom_ipcc *ipcc = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t hwirq = irqd_to_hwirq(irqd);
+
+	writel(hwirq, ipcc->base + IPCC_REG_RECV_SIGNAL_DISABLE);
+}
+
+static void qcom_ipcc_unmask_irq(struct irq_data *irqd)
+{
+	struct qcom_ipcc *ipcc = irq_data_get_irq_chip_data(irqd);
+	irq_hw_number_t hwirq = irqd_to_hwirq(irqd);
+
+	writel(hwirq, ipcc->base + IPCC_REG_RECV_SIGNAL_ENABLE);
+}
+
+static struct irq_chip qcom_ipcc_irq_chip = {
+	.name = "ipcc",
+	.irq_mask = qcom_ipcc_mask_irq,
+	.irq_unmask = qcom_ipcc_unmask_irq,
+	.flags = IRQCHIP_SKIP_SET_WAKE,
+};
+
+static int qcom_ipcc_domain_map(struct irq_domain *d, unsigned int irq,
+				irq_hw_number_t hw)
+{
+	struct qcom_ipcc *ipcc = d->host_data;
+
+	irq_set_chip_and_handler(irq, &qcom_ipcc_irq_chip, handle_level_irq);
+	irq_set_chip_data(irq, ipcc);
+	irq_set_noprobe(irq);
+
+	return 0;
+}
+
+static int qcom_ipcc_domain_xlate(struct irq_domain *d,
+				  struct device_node *node, const u32 *intspec,
+				  unsigned int intsize,
+				  unsigned long *out_hwirq,
+				  unsigned int *out_type)
+{
+	if (intsize != 3)
+		return -EINVAL;
+
+	*out_hwirq = qcom_ipcc_get_hwirq(intspec[0], intspec[1]);
+	*out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
+
+	return 0;
+}
+
+static const struct irq_domain_ops qcom_ipcc_irq_ops = {
+	.map = qcom_ipcc_domain_map,
+	.xlate = qcom_ipcc_domain_xlate,
+};
+
+static int qcom_ipcc_mbox_send_data(struct mbox_chan *chan, void *data)
+{
+	struct qcom_ipcc *ipcc = to_qcom_ipcc(chan->mbox);
+	struct qcom_ipcc_chan_info *mchan = chan->con_priv;
+	u32 hwirq;
+
+	hwirq = qcom_ipcc_get_hwirq(mchan->client_id, mchan->signal_id);
+	writel(hwirq, ipcc->base + IPCC_REG_SEND_ID);
+
+	return 0;
+}
+
+static struct mbox_chan *qcom_ipcc_mbox_xlate(struct mbox_controller *mbox,
+					const struct of_phandle_args *ph)
+{
+	struct qcom_ipcc *ipcc = to_qcom_ipcc(mbox);
+	struct qcom_ipcc_chan_info *mchan;
+	struct mbox_chan *chan;
+	unsigned int i;
+
+	if (ph->args_count != 2)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < IPCC_MBOX_MAX_CHAN; i++) {
+		chan = &ipcc->chan[i];
+		if (!chan->con_priv) {
+			mchan = &ipcc->mchan[i];
+			mchan->client_id = ph->args[0];
+			mchan->signal_id = ph->args[1];
+			chan->con_priv = mchan;
+			break;
+		}
+
+		chan = NULL;
+	}
+
+	return chan ?: ERR_PTR(-EBUSY);
+}
+
+static const struct mbox_chan_ops ipcc_mbox_chan_ops = {
+	.send_data = qcom_ipcc_mbox_send_data,
+};
+
+static int qcom_ipcc_setup_mbox(struct qcom_ipcc *ipcc)
+{
+	struct mbox_controller *mbox;
+	struct device *dev = ipcc->dev;
+
+	mbox = &ipcc->mbox;
+	mbox->dev = dev;
+	mbox->num_chans = IPCC_MBOX_MAX_CHAN;
+	mbox->chans = ipcc->chan;
+	mbox->ops = &ipcc_mbox_chan_ops;
+	mbox->of_xlate = qcom_ipcc_mbox_xlate;
+	mbox->txdone_irq = false;
+	mbox->txdone_poll = false;
+
+	return devm_mbox_controller_register(dev, mbox);
+}
+
+static int qcom_ipcc_probe(struct platform_device *pdev)
+{
+	struct qcom_ipcc *ipcc;
+	int ret;
+
+	ipcc = devm_kzalloc(&pdev->dev, sizeof(*ipcc), GFP_KERNEL);
+	if (!ipcc)
+		return -ENOMEM;
+
+	ipcc->dev = &pdev->dev;
+
+	ipcc->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ipcc->base))
+		return PTR_ERR(ipcc->base);
+
+	ipcc->irq = platform_get_irq(pdev, 0);
+	if (ipcc->irq < 0)
+		return ipcc->irq;
+
+	ipcc->irq_domain = irq_domain_add_tree(pdev->dev.of_node,
+					       &qcom_ipcc_irq_ops, ipcc);
+	if (!ipcc->irq_domain)
+		return -ENOMEM;
+
+	ret = qcom_ipcc_setup_mbox(ipcc);
+	if (ret)
+		goto err_mbox;
+
+	ret = devm_request_irq(&pdev->dev, ipcc->irq, qcom_ipcc_irq_fn,
+			       IRQF_TRIGGER_HIGH, "ipcc", ipcc);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to register the irq: %d\n", ret);
+		goto err_mbox;
+	}
+
+	enable_irq_wake(ipcc->irq);
+	platform_set_drvdata(pdev, ipcc);
+
+	return 0;
+
+err_mbox:
+	irq_domain_remove(ipcc->irq_domain);
+
+	return ret;
+}
+
+static int qcom_ipcc_remove(struct platform_device *pdev)
+{
+	struct qcom_ipcc *ipcc = platform_get_drvdata(pdev);
+
+	disable_irq_wake(ipcc->irq);
+	irq_domain_remove(ipcc->irq_domain);
+
+	return 0;
+}
+
+static const struct of_device_id qcom_ipcc_of_match[] = {
+	{ .compatible = "qcom,ipcc"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, qcom_ipcc_of_match);
+
+static struct platform_driver qcom_ipcc_driver = {
+	.probe = qcom_ipcc_probe,
+	.remove = qcom_ipcc_remove,
+	.driver = {
+		.name = "qcom-ipcc",
+		.of_match_table = qcom_ipcc_of_match,
+	},
+};
+
+static int __init qcom_ipcc_init(void)
+{
+	return platform_driver_register(&qcom_ipcc_driver);
+}
+arch_initcall(qcom_ipcc_init);
+
+MODULE_AUTHOR("Venkata Narendra Kumar Gutta <vnkgutta@codeaurora.org>");
+MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
+MODULE_DESCRIPTION("Qualcomm Technologies, Inc. IPCC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mailbox/sprd-mailbox.c b/drivers/mailbox/sprd-mailbox.c
new file mode 100644
index 000000000000..f6fab24ae8a9
--- /dev/null
+++ b/drivers/mailbox/sprd-mailbox.c
@@ -0,0 +1,361 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Spreadtrum mailbox driver
+ *
+ * Copyright (c) 2020 Spreadtrum Communications Inc.
+ */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mailbox_controller.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+
+#define SPRD_MBOX_ID		0x0
+#define SPRD_MBOX_MSG_LOW	0x4
+#define SPRD_MBOX_MSG_HIGH	0x8
+#define SPRD_MBOX_TRIGGER	0xc
+#define SPRD_MBOX_FIFO_RST	0x10
+#define SPRD_MBOX_FIFO_STS	0x14
+#define SPRD_MBOX_IRQ_STS	0x18
+#define SPRD_MBOX_IRQ_MSK	0x1c
+#define SPRD_MBOX_LOCK		0x20
+#define SPRD_MBOX_FIFO_DEPTH	0x24
+
+/* Bit and mask definiation for inbox's SPRD_MBOX_FIFO_STS register */
+#define SPRD_INBOX_FIFO_DELIVER_MASK		GENMASK(23, 16)
+#define SPRD_INBOX_FIFO_OVERLOW_MASK		GENMASK(15, 8)
+#define SPRD_INBOX_FIFO_DELIVER_SHIFT		16
+#define SPRD_INBOX_FIFO_BUSY_MASK		GENMASK(7, 0)
+
+/* Bit and mask definiation for SPRD_MBOX_IRQ_STS register */
+#define SPRD_MBOX_IRQ_CLR			BIT(0)
+
+/* Bit and mask definiation for outbox's SPRD_MBOX_FIFO_STS register */
+#define SPRD_OUTBOX_FIFO_FULL			BIT(0)
+#define SPRD_OUTBOX_FIFO_WR_SHIFT		16
+#define SPRD_OUTBOX_FIFO_RD_SHIFT		24
+#define SPRD_OUTBOX_FIFO_POS_MASK		GENMASK(7, 0)
+
+/* Bit and mask definiation for inbox's SPRD_MBOX_IRQ_MSK register */
+#define SPRD_INBOX_FIFO_BLOCK_IRQ		BIT(0)
+#define SPRD_INBOX_FIFO_OVERFLOW_IRQ		BIT(1)
+#define SPRD_INBOX_FIFO_DELIVER_IRQ		BIT(2)
+#define SPRD_INBOX_FIFO_IRQ_MASK		GENMASK(2, 0)
+
+/* Bit and mask definiation for outbox's SPRD_MBOX_IRQ_MSK register */
+#define SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ		BIT(0)
+#define SPRD_OUTBOX_FIFO_IRQ_MASK		GENMASK(4, 0)
+
+#define SPRD_MBOX_CHAN_MAX			8
+
+struct sprd_mbox_priv {
+	struct mbox_controller	mbox;
+	struct device		*dev;
+	void __iomem		*inbox_base;
+	void __iomem		*outbox_base;
+	struct clk		*clk;
+	u32			outbox_fifo_depth;
+
+	struct mbox_chan	chan[SPRD_MBOX_CHAN_MAX];
+};
+
+static struct sprd_mbox_priv *to_sprd_mbox_priv(struct mbox_controller *mbox)
+{
+	return container_of(mbox, struct sprd_mbox_priv, mbox);
+}
+
+static u32 sprd_mbox_get_fifo_len(struct sprd_mbox_priv *priv, u32 fifo_sts)
+{
+	u32 wr_pos = (fifo_sts >> SPRD_OUTBOX_FIFO_WR_SHIFT) &
+		SPRD_OUTBOX_FIFO_POS_MASK;
+	u32 rd_pos = (fifo_sts >> SPRD_OUTBOX_FIFO_RD_SHIFT) &
+		SPRD_OUTBOX_FIFO_POS_MASK;
+	u32 fifo_len;
+
+	/*
+	 * If the read pointer is equal with write pointer, which means the fifo
+	 * is full or empty.
+	 */
+	if (wr_pos == rd_pos) {
+		if (fifo_sts & SPRD_OUTBOX_FIFO_FULL)
+			fifo_len = priv->outbox_fifo_depth;
+		else
+			fifo_len = 0;
+	} else if (wr_pos > rd_pos) {
+		fifo_len = wr_pos - rd_pos;
+	} else {
+		fifo_len = priv->outbox_fifo_depth - rd_pos + wr_pos;
+	}
+
+	return fifo_len;
+}
+
+static irqreturn_t sprd_mbox_outbox_isr(int irq, void *data)
+{
+	struct sprd_mbox_priv *priv = data;
+	struct mbox_chan *chan;
+	u32 fifo_sts, fifo_len, msg[2];
+	int i, id;
+
+	fifo_sts = readl(priv->outbox_base + SPRD_MBOX_FIFO_STS);
+
+	fifo_len = sprd_mbox_get_fifo_len(priv, fifo_sts);
+	if (!fifo_len) {
+		dev_warn_ratelimited(priv->dev, "spurious outbox interrupt\n");
+		return IRQ_NONE;
+	}
+
+	for (i = 0; i < fifo_len; i++) {
+		msg[0] = readl(priv->outbox_base + SPRD_MBOX_MSG_LOW);
+		msg[1] = readl(priv->outbox_base + SPRD_MBOX_MSG_HIGH);
+		id = readl(priv->outbox_base + SPRD_MBOX_ID);
+
+		chan = &priv->chan[id];
+		mbox_chan_received_data(chan, (void *)msg);
+
+		/* Trigger to update outbox FIFO pointer */
+		writel(0x1, priv->outbox_base + SPRD_MBOX_TRIGGER);
+	}
+
+	/* Clear irq status after reading all message. */
+	writel(SPRD_MBOX_IRQ_CLR, priv->outbox_base + SPRD_MBOX_IRQ_STS);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data)
+{
+	struct sprd_mbox_priv *priv = data;
+	struct mbox_chan *chan;
+	u32 fifo_sts, send_sts, busy, id;
+
+	fifo_sts = readl(priv->inbox_base + SPRD_MBOX_FIFO_STS);
+
+	/* Get the inbox data delivery status */
+	send_sts = (fifo_sts & SPRD_INBOX_FIFO_DELIVER_MASK) >>
+		SPRD_INBOX_FIFO_DELIVER_SHIFT;
+	if (!send_sts) {
+		dev_warn_ratelimited(priv->dev, "spurious inbox interrupt\n");
+		return IRQ_NONE;
+	}
+
+	while (send_sts) {
+		id = __ffs(send_sts);
+		send_sts &= (send_sts - 1);
+
+		chan = &priv->chan[id];
+
+		/*
+		 * Check if the message was fetched by remote traget, if yes,
+		 * that means the transmission has been completed.
+		 */
+		busy = fifo_sts & SPRD_INBOX_FIFO_BUSY_MASK;
+		if (!(busy & BIT(id)))
+			mbox_chan_txdone(chan, 0);
+	}
+
+	/* Clear FIFO delivery and overflow status */
+	writel(fifo_sts &
+	       (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK),
+	       priv->inbox_base + SPRD_MBOX_FIFO_RST);
+
+	/* Clear irq status */
+	writel(SPRD_MBOX_IRQ_CLR, priv->inbox_base + SPRD_MBOX_IRQ_STS);
+
+	return IRQ_HANDLED;
+}
+
+static int sprd_mbox_send_data(struct mbox_chan *chan, void *msg)
+{
+	struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox);
+	unsigned long id = (unsigned long)chan->con_priv;
+	u32 *data = msg;
+
+	/* Write data into inbox FIFO, and only support 8 bytes every time */
+	writel(data[0], priv->inbox_base + SPRD_MBOX_MSG_LOW);
+	writel(data[1], priv->inbox_base + SPRD_MBOX_MSG_HIGH);
+
+	/* Set target core id */
+	writel(id, priv->inbox_base + SPRD_MBOX_ID);
+
+	/* Trigger remote request */
+	writel(0x1, priv->inbox_base + SPRD_MBOX_TRIGGER);
+
+	return 0;
+}
+
+static int sprd_mbox_flush(struct mbox_chan *chan, unsigned long timeout)
+{
+	struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox);
+	unsigned long id = (unsigned long)chan->con_priv;
+	u32 busy;
+
+	timeout = jiffies + msecs_to_jiffies(timeout);
+
+	while (time_before(jiffies, timeout)) {
+		busy = readl(priv->inbox_base + SPRD_MBOX_FIFO_STS) &
+			SPRD_INBOX_FIFO_BUSY_MASK;
+		if (!(busy & BIT(id))) {
+			mbox_chan_txdone(chan, 0);
+			return 0;
+		}
+
+		udelay(1);
+	}
+
+	return -ETIME;
+}
+
+static int sprd_mbox_startup(struct mbox_chan *chan)
+{
+	struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox);
+	u32 val;
+
+	/* Select outbox FIFO mode and reset the outbox FIFO status */
+	writel(0x0, priv->outbox_base + SPRD_MBOX_FIFO_RST);
+
+	/* Enable inbox FIFO overflow and delivery interrupt */
+	val = readl(priv->inbox_base + SPRD_MBOX_IRQ_MSK);
+	val &= ~(SPRD_INBOX_FIFO_OVERFLOW_IRQ | SPRD_INBOX_FIFO_DELIVER_IRQ);
+	writel(val, priv->inbox_base + SPRD_MBOX_IRQ_MSK);
+
+	/* Enable outbox FIFO not empty interrupt */
+	val = readl(priv->outbox_base + SPRD_MBOX_IRQ_MSK);
+	val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ;
+	writel(val, priv->outbox_base + SPRD_MBOX_IRQ_MSK);
+
+	return 0;
+}
+
+static void sprd_mbox_shutdown(struct mbox_chan *chan)
+{
+	struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox);
+
+	/* Disable inbox & outbox interrupt */
+	writel(SPRD_INBOX_FIFO_IRQ_MASK, priv->inbox_base + SPRD_MBOX_IRQ_MSK);
+	writel(SPRD_OUTBOX_FIFO_IRQ_MASK, priv->outbox_base + SPRD_MBOX_IRQ_MSK);
+}
+
+static const struct mbox_chan_ops sprd_mbox_ops = {
+	.send_data	= sprd_mbox_send_data,
+	.flush		= sprd_mbox_flush,
+	.startup	= sprd_mbox_startup,
+	.shutdown	= sprd_mbox_shutdown,
+};
+
+static void sprd_mbox_disable(void *data)
+{
+	struct sprd_mbox_priv *priv = data;
+
+	clk_disable_unprepare(priv->clk);
+}
+
+static int sprd_mbox_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sprd_mbox_priv *priv;
+	int ret, inbox_irq, outbox_irq;
+	unsigned long id;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+
+	/*
+	 * The Spreadtrum mailbox uses an inbox to send messages to the target
+	 * core, and uses an outbox to receive messages from other cores.
+	 *
+	 * Thus the mailbox controller supplies 2 different register addresses
+	 * and IRQ numbers for inbox and outbox.
+	 */
+	priv->inbox_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(priv->inbox_base))
+		return PTR_ERR(priv->inbox_base);
+
+	priv->outbox_base = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(priv->outbox_base))
+		return PTR_ERR(priv->outbox_base);
+
+	priv->clk = devm_clk_get(dev, "enable");
+	if (IS_ERR(priv->clk)) {
+		dev_err(dev, "failed to get mailbox clock\n");
+		return PTR_ERR(priv->clk);
+	}
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, sprd_mbox_disable, priv);
+	if (ret) {
+		dev_err(dev, "failed to add mailbox disable action\n");
+		return ret;
+	}
+
+	inbox_irq = platform_get_irq(pdev, 0);
+	if (inbox_irq < 0)
+		return inbox_irq;
+
+	ret = devm_request_irq(dev, inbox_irq, sprd_mbox_inbox_isr,
+			       IRQF_NO_SUSPEND, dev_name(dev), priv);
+	if (ret) {
+		dev_err(dev, "failed to request inbox IRQ: %d\n", ret);
+		return ret;
+	}
+
+	outbox_irq = platform_get_irq(pdev, 1);
+	if (outbox_irq < 0)
+		return outbox_irq;
+
+	ret = devm_request_irq(dev, outbox_irq, sprd_mbox_outbox_isr,
+			       IRQF_NO_SUSPEND, dev_name(dev), priv);
+	if (ret) {
+		dev_err(dev, "failed to request outbox IRQ: %d\n", ret);
+		return ret;
+	}
+
+	/* Get the default outbox FIFO depth */
+	priv->outbox_fifo_depth =
+		readl(priv->outbox_base + SPRD_MBOX_FIFO_DEPTH) + 1;
+	priv->mbox.dev = dev;
+	priv->mbox.chans = &priv->chan[0];
+	priv->mbox.num_chans = SPRD_MBOX_CHAN_MAX;
+	priv->mbox.ops = &sprd_mbox_ops;
+	priv->mbox.txdone_irq = true;
+
+	for (id = 0; id < SPRD_MBOX_CHAN_MAX; id++)
+		priv->chan[id].con_priv = (void *)id;
+
+	ret = devm_mbox_controller_register(dev, &priv->mbox);
+	if (ret) {
+		dev_err(dev, "failed to register mailbox: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id sprd_mbox_of_match[] = {
+	{ .compatible = "sprd,sc9860-mailbox", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sprd_mbox_of_match);
+
+static struct platform_driver sprd_mbox_driver = {
+	.driver = {
+		.name = "sprd-mailbox",
+		.of_match_table = sprd_mbox_of_match,
+	},
+	.probe	= sprd_mbox_probe,
+};
+module_platform_driver(sprd_mbox_driver);
+
+MODULE_AUTHOR("Baolin Wang <baolin.wang@unisoc.com>");
+MODULE_DESCRIPTION("Spreadtrum mailbox driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mailbox/zynqmp-ipi-mailbox.c b/drivers/mailbox/zynqmp-ipi-mailbox.c
index 86887c9a349a..f44079d62b1a 100644
--- a/drivers/mailbox/zynqmp-ipi-mailbox.c
+++ b/drivers/mailbox/zynqmp-ipi-mailbox.c
@@ -504,10 +504,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox,
 		mchan->req_buf_size = resource_size(&res);
 		mchan->req_buf = devm_ioremap(mdev, res.start,
 					      mchan->req_buf_size);
-		if (IS_ERR(mchan->req_buf)) {
+		if (!mchan->req_buf) {
 			dev_err(mdev, "Unable to map IPI buffer I/O memory\n");
-			ret = PTR_ERR(mchan->req_buf);
-			return ret;
+			return -ENOMEM;
 		}
 	} else if (ret != -ENODEV) {
 		dev_err(mdev, "Unmatched resource %s, %d.\n", name, ret);
@@ -520,10 +519,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox,
 		mchan->resp_buf_size = resource_size(&res);
 		mchan->resp_buf = devm_ioremap(mdev, res.start,
 					       mchan->resp_buf_size);
-		if (IS_ERR(mchan->resp_buf)) {
+		if (!mchan->resp_buf) {
 			dev_err(mdev, "Unable to map IPI buffer I/O memory\n");
-			ret = PTR_ERR(mchan->resp_buf);
-			return ret;
+			return -ENOMEM;
 		}
 	} else if (ret != -ENODEV) {
 		dev_err(mdev, "Unmatched resource %s.\n", name);
@@ -543,10 +541,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox,
 		mchan->req_buf_size = resource_size(&res);
 		mchan->req_buf = devm_ioremap(mdev, res.start,
 					      mchan->req_buf_size);
-		if (IS_ERR(mchan->req_buf)) {
+		if (!mchan->req_buf) {
 			dev_err(mdev, "Unable to map IPI buffer I/O memory\n");
-			ret = PTR_ERR(mchan->req_buf);
-			return ret;
+			return -ENOMEM;
 		}
 	} else if (ret != -ENODEV) {
 		dev_err(mdev, "Unmatched resource %s.\n", name);
@@ -559,10 +556,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox,
 		mchan->resp_buf_size = resource_size(&res);
 		mchan->resp_buf = devm_ioremap(mdev, res.start,
 					       mchan->resp_buf_size);
-		if (IS_ERR(mchan->resp_buf)) {
+		if (!mchan->resp_buf) {
 			dev_err(mdev, "Unable to map IPI buffer I/O memory\n");
-			ret = PTR_ERR(mchan->resp_buf);
-			return ret;
+			return -ENOMEM;
 		}
 	} else if (ret != -ENODEV) {
 		dev_err(mdev, "Unmatched resource %s.\n", name);
@@ -668,10 +664,9 @@ static int zynqmp_ipi_probe(struct platform_device *pdev)
 
 	/* IPI IRQ */
 	ret = platform_get_irq(pdev, 0);
-	if (ret < 0) {
-		dev_err(dev, "unable to find IPI IRQ.\n");
+	if (ret < 0)
 		goto free_mbox_dev;
-	}
+
 	pdata->irq = ret;
 	ret = devm_request_irq(dev, pdata->irq, zynqmp_ipi_interrupt,
 			       IRQF_SHARED, dev_name(dev), pdata);
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index d6d5ab23c088..921888df6764 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -15,7 +15,7 @@ if MD
 
 config BLK_DEV_MD
 	tristate "RAID support"
-	---help---
+	help
 	  This driver lets you combine several hard disk partitions into one
 	  logical block device. This can be used to simply append one
 	  partition to another one or to combine several redundant hard disks
@@ -36,7 +36,7 @@ config MD_AUTODETECT
 	bool "Autodetect RAID arrays during kernel boot"
 	depends on BLK_DEV_MD=y
 	default y
-	---help---
+	help
 	  If you say Y here, then the kernel will try to autodetect raid
 	  arrays as part of its boot process.
 
@@ -49,7 +49,7 @@ config MD_AUTODETECT
 config MD_LINEAR
 	tristate "Linear (append) mode"
 	depends on BLK_DEV_MD
-	---help---
+	help
 	  If you say Y here, then your multiple devices driver will be able to
 	  use the so-called linear mode, i.e. it will combine the hard disk
 	  partitions by simply appending one to the other.
@@ -62,7 +62,7 @@ config MD_LINEAR
 config MD_RAID0
 	tristate "RAID-0 (striping) mode"
 	depends on BLK_DEV_MD
-	---help---
+	help
 	  If you say Y here, then your multiple devices driver will be able to
 	  use the so-called raid0 mode, i.e. it will combine the hard disk
 	  partitions into one logical device in such a fashion as to fill them
@@ -82,7 +82,7 @@ config MD_RAID0
 config MD_RAID1
 	tristate "RAID-1 (mirroring) mode"
 	depends on BLK_DEV_MD
-	---help---
+	help
 	  A RAID-1 set consists of several disk drives which are exact copies
 	  of each other.  In the event of a mirror failure, the RAID driver
 	  will continue to use the operational mirrors in the set, providing
@@ -104,7 +104,7 @@ config MD_RAID1
 config MD_RAID10
 	tristate "RAID-10 (mirrored striping) mode"
 	depends on BLK_DEV_MD
-	---help---
+	help
 	  RAID-10 provides a combination of striping (RAID-0) and
 	  mirroring (RAID-1) with easier configuration and more flexible
 	  layout.
@@ -129,7 +129,7 @@ config MD_RAID456
 	select ASYNC_XOR
 	select ASYNC_PQ
 	select ASYNC_RAID6_RECOV
-	---help---
+	help
 	  A RAID-5 set of N drives with a capacity of C MB per drive provides
 	  the capacity of C * (N - 1) MB, and protects against a failure
 	  of a single drive. For a given sector (row) number, (N - 1) drives
@@ -183,7 +183,7 @@ config MD_CLUSTER
 	depends on BLK_DEV_MD
 	depends on DLM
 	default n
-	---help---
+	help
 	Clustering support for MD devices. This enables locking and
 	synchronization across multiple systems on the cluster, so all
 	nodes in the cluster can access the MD devices simultaneously.
@@ -203,7 +203,7 @@ config BLK_DEV_DM
 	tristate "Device mapper support"
 	select BLK_DEV_DM_BUILTIN
 	depends on DAX || DAX=n
-	---help---
+	help
 	  Device-mapper is a low level volume manager.  It works by allowing
 	  people to specify mappings for ranges of logical sectors.  Various
 	  mapping types are available, in addition people may write their own
@@ -219,7 +219,7 @@ config BLK_DEV_DM
 config DM_DEBUG
 	bool "Device mapper debugging support"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	  Enable this for messages that may help debug device-mapper problems.
 
 	  If unsure, say N.
@@ -227,7 +227,7 @@ config DM_DEBUG
 config DM_BUFIO
        tristate
        depends on BLK_DEV_DM
-       ---help---
+	help
 	 This interface allows you to do buffered I/O on a device and acts
 	 as a cache, holding recently-read blocks in memory and performing
 	 delayed writes.
@@ -235,7 +235,7 @@ config DM_BUFIO
 config DM_DEBUG_BLOCK_MANAGER_LOCKING
        bool "Block manager locking"
        depends on DM_BUFIO
-       ---help---
+	help
 	 Block manager locking can catch various metadata corruption issues.
 
 	 If unsure, say N.
@@ -244,7 +244,7 @@ config DM_DEBUG_BLOCK_STACK_TRACING
        bool "Keep stack trace of persistent data block lock holders"
        depends on STACKTRACE_SUPPORT && DM_DEBUG_BLOCK_MANAGER_LOCKING
        select STACKTRACE
-       ---help---
+	help
 	 Enable this for messages that may help debug problems with the
 	 block manager locking used by thin provisioning and caching.
 
@@ -253,7 +253,7 @@ config DM_DEBUG_BLOCK_STACK_TRACING
 config DM_BIO_PRISON
        tristate
        depends on BLK_DEV_DM
-       ---help---
+	help
 	 Some bio locking schemes used by other device-mapper targets
 	 including thin provisioning.
 
@@ -262,17 +262,18 @@ source "drivers/md/persistent-data/Kconfig"
 config DM_UNSTRIPED
        tristate "Unstriped target"
        depends on BLK_DEV_DM
-       ---help---
+	help
 	  Unstripes I/O so it is issued solely on a single drive in a HW
 	  RAID0 or dm-striped target.
 
 config DM_CRYPT
 	tristate "Crypt target support"
 	depends on BLK_DEV_DM
+	depends on (ENCRYPTED_KEYS || ENCRYPTED_KEYS=n)
 	select CRYPTO
 	select CRYPTO_CBC
 	select CRYPTO_ESSIV
-	---help---
+	help
 	  This device-mapper target allows you to create a device that
 	  transparently encrypts the data on it. You'll need to activate
 	  the ciphers you're going to use in the cryptoapi configuration.
@@ -289,7 +290,7 @@ config DM_SNAPSHOT
        tristate "Snapshot target"
        depends on BLK_DEV_DM
        select DM_BUFIO
-       ---help---
+	help
 	 Allow volume managers to take writable snapshots of a device.
 
 config DM_THIN_PROVISIONING
@@ -297,7 +298,7 @@ config DM_THIN_PROVISIONING
        depends on BLK_DEV_DM
        select DM_PERSISTENT_DATA
        select DM_BIO_PRISON
-       ---help---
+	help
 	 Provides thin provisioning and snapshots that share a data store.
 
 config DM_CACHE
@@ -306,7 +307,7 @@ config DM_CACHE
        default n
        select DM_PERSISTENT_DATA
        select DM_BIO_PRISON
-       ---help---
+	help
 	 dm-cache attempts to improve performance of a block device by
 	 moving frequently used data to a smaller, higher performance
 	 device.  Different 'policy' plugins can be used to change the
@@ -317,7 +318,7 @@ config DM_CACHE_SMQ
        tristate "Stochastic MQ Cache Policy (EXPERIMENTAL)"
        depends on DM_CACHE
        default y
-       ---help---
+	help
 	 A cache policy that uses a multiqueue ordered by recent hits
 	 to select which blocks should be promoted and demoted.
 	 This is meant to be a general purpose policy.  It prioritises
@@ -328,7 +329,7 @@ config DM_CACHE_SMQ
 config DM_WRITECACHE
 	tristate "Writecache target"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	   The writecache target caches writes on persistent memory or SSD.
 	   It is intended for databases or other programs that need extremely
 	   low commit latency.
@@ -336,13 +337,21 @@ config DM_WRITECACHE
 	   The writecache target doesn't cache reads because reads are supposed
 	   to be cached in standard RAM.
 
+config DM_EBS
+	tristate "Emulated block size target (EXPERIMENTAL)"
+	depends on BLK_DEV_DM
+	select DM_BUFIO
+	help
+	  dm-ebs emulates smaller logical block size on backing devices
+	  with larger ones (e.g. 512 byte sectors on 4K native disks).
+
 config DM_ERA
        tristate "Era target (EXPERIMENTAL)"
        depends on BLK_DEV_DM
        default n
        select DM_PERSISTENT_DATA
        select DM_BIO_PRISON
-       ---help---
+	help
 	 dm-era tracks which parts of a block device are written to
 	 over time.  Useful for maintaining cache coherency when using
 	 vendor snapshots.
@@ -352,7 +361,7 @@ config DM_CLONE
        depends on BLK_DEV_DM
        default n
        select DM_PERSISTENT_DATA
-       ---help---
+	help
 	 dm-clone produces a one-to-one copy of an existing, read-only source
 	 device into a writable destination device. The cloned device is
 	 visible/mountable immediately and the copy of the source device to the
@@ -364,7 +373,7 @@ config DM_CLONE
 config DM_MIRROR
        tristate "Mirror target"
        depends on BLK_DEV_DM
-       ---help---
+	help
 	 Allow volume managers to mirror logical volumes, also
 	 needed for live data migration tools such as 'pvmove'.
 
@@ -372,7 +381,7 @@ config DM_LOG_USERSPACE
 	tristate "Mirror userspace logging"
 	depends on DM_MIRROR && NET
 	select CONNECTOR
-	---help---
+	help
 	  The userspace logging module provides a mechanism for
 	  relaying the dm-dirty-log API to userspace.  Log designs
 	  which are more suited to userspace implementation (e.g.
@@ -387,7 +396,7 @@ config DM_RAID
        select MD_RAID10
        select MD_RAID456
        select BLK_DEV_MD
-       ---help---
+	help
 	 A dm target that supports RAID1, RAID10, RAID4, RAID5 and RAID6 mappings
 
 	 A RAID-5 set of N drives with a capacity of C MB per drive provides
@@ -409,7 +418,7 @@ config DM_RAID
 config DM_ZERO
 	tristate "Zero target"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	  A target that discards writes, and returns all zeroes for
 	  reads.  Useful in some recovery situations.
 
@@ -421,13 +430,13 @@ config DM_MULTIPATH
 	# it is, DM_MULTIPATH must depend on it.  We get a build
 	# error if SCSI_DH=m and DM_MULTIPATH=y
 	depends on !SCSI_DH || SCSI
-	---help---
+	help
 	  Allow volume managers to support multipath hardware.
 
 config DM_MULTIPATH_QL
 	tristate "I/O Path Selector based on the number of in-flight I/Os"
 	depends on DM_MULTIPATH
-	---help---
+	help
 	  This path selector is a dynamic load balancer which selects
 	  the path with the least number of in-flight I/Os.
 
@@ -436,17 +445,28 @@ config DM_MULTIPATH_QL
 config DM_MULTIPATH_ST
 	tristate "I/O Path Selector based on the service time"
 	depends on DM_MULTIPATH
-	---help---
+	help
 	  This path selector is a dynamic load balancer which selects
 	  the path expected to complete the incoming I/O in the shortest
 	  time.
 
 	  If unsure, say N.
 
+config DM_MULTIPATH_HST
+	tristate "I/O Path Selector based on historical service time"
+	depends on DM_MULTIPATH
+	help
+	  This path selector is a dynamic load balancer which selects
+	  the path expected to complete the incoming I/O in the shortest
+	  time by comparing estimated service time (based on historical
+	  service time).
+
+	  If unsure, say N.
+
 config DM_DELAY
 	tristate "I/O delaying target"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	A target that delays reads and/or writes and can send
 	them to different devices.  Useful for testing.
 
@@ -455,7 +475,7 @@ config DM_DELAY
 config DM_DUST
 	tristate "Bad sector simulation target"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	A target that simulates bad sector behavior.
 	Useful for testing.
 
@@ -464,7 +484,7 @@ config DM_DUST
 config DM_INIT
 	bool "DM \"dm-mod.create=\" parameter support"
 	depends on BLK_DEV_DM=y
-	---help---
+	help
 	Enable "dm-mod.create=" parameter to create mapped devices at init time.
 	This option is useful to allow mounting rootfs without requiring an
 	initramfs.
@@ -476,13 +496,13 @@ config DM_INIT
 config DM_UEVENT
 	bool "DM uevents"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	Generate udev events for DM events.
 
 config DM_FLAKEY
        tristate "Flakey target"
        depends on BLK_DEV_DM
-       ---help---
+	help
 	 A target that intermittently fails I/O for debugging purposes.
 
 config DM_VERITY
@@ -491,7 +511,7 @@ config DM_VERITY
 	select CRYPTO
 	select CRYPTO_HASH
 	select DM_BUFIO
-	---help---
+	help
 	  This device-mapper target creates a read-only device that
 	  transparently validates the data on one underlying device against
 	  a pre-generated tree of cryptographic checksums stored on a second
@@ -522,7 +542,7 @@ config DM_VERITY_FEC
 	depends on DM_VERITY
 	select REED_SOLOMON
 	select REED_SOLOMON_DEC8
-	---help---
+	help
 	  Add forward error correction support to dm-verity. This option
 	  makes it possible to use pre-generated error correction data to
 	  recover from corrupted blocks.
@@ -532,7 +552,7 @@ config DM_VERITY_FEC
 config DM_SWITCH
 	tristate "Switch target support (EXPERIMENTAL)"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	  This device-mapper target creates a device that supports an arbitrary
 	  mapping of fixed-size regions of I/O across a fixed set of paths.
 	  The path used for any specific region can be switched dynamically
@@ -546,7 +566,7 @@ config DM_SWITCH
 config DM_LOG_WRITES
 	tristate "Log writes target support"
 	depends on BLK_DEV_DM
-	---help---
+	help
 	  This device-mapper target takes two devices, one device to use
 	  normally, one to log all write operations done to the first device.
 	  This is for use by file system developers wishing to verify that
@@ -566,7 +586,7 @@ config DM_INTEGRITY
 	select DM_BUFIO
 	select CRYPTO
 	select ASYNC_XOR
-	---help---
+	help
 	  This device-mapper target emulates a block device that has
 	  additional per-sector tags that can be used for storing
 	  integrity information.
@@ -582,7 +602,7 @@ config DM_ZONED
 	tristate "Drive-managed zoned block device target support"
 	depends on BLK_DEV_DM
 	depends on BLK_DEV_ZONED
-	---help---
+	help
 	  This device-mapper target takes a host-managed or host-aware zoned
 	  block device and exposes most of its capacity as a regular block
 	  device (drive-managed zoned block device) without any write
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index d91a7edcd2ab..31840f95cd40 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -17,6 +17,7 @@ dm-thin-pool-y	+= dm-thin.o dm-thin-metadata.o
 dm-cache-y	+= dm-cache-target.o dm-cache-metadata.o dm-cache-policy.o \
 		    dm-cache-background-tracker.o
 dm-cache-smq-y   += dm-cache-policy-smq.o
+dm-ebs-y	+= dm-ebs-target.o
 dm-era-y	+= dm-era-target.o
 dm-clone-y	+= dm-clone-target.o dm-clone-metadata.o
 dm-verity-y	+= dm-verity-target.o
@@ -54,6 +55,7 @@ obj-$(CONFIG_DM_FLAKEY)		+= dm-flakey.o
 obj-$(CONFIG_DM_MULTIPATH)	+= dm-multipath.o dm-round-robin.o
 obj-$(CONFIG_DM_MULTIPATH_QL)	+= dm-queue-length.o
 obj-$(CONFIG_DM_MULTIPATH_ST)	+= dm-service-time.o
+obj-$(CONFIG_DM_MULTIPATH_HST)	+= dm-historical-service-time.o
 obj-$(CONFIG_DM_SWITCH)		+= dm-switch.o
 obj-$(CONFIG_DM_SNAPSHOT)	+= dm-snapshot.o
 obj-$(CONFIG_DM_PERSISTENT_DATA)	+= persistent-data/
@@ -65,6 +67,7 @@ obj-$(CONFIG_DM_THIN_PROVISIONING)	+= dm-thin-pool.o
 obj-$(CONFIG_DM_VERITY)		+= dm-verity.o
 obj-$(CONFIG_DM_CACHE)		+= dm-cache.o
 obj-$(CONFIG_DM_CACHE_SMQ)	+= dm-cache-smq.o
+obj-$(CONFIG_DM_EBS)		+= dm-ebs.o
 obj-$(CONFIG_DM_ERA)		+= dm-era.o
 obj-$(CONFIG_DM_CLONE)		+= dm-clone.o
 obj-$(CONFIG_DM_LOG_WRITES)	+= dm-log-writes.o
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index d1786cfd7f22..6d1565021d74 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -256,12 +256,35 @@ static struct dm_buffer *__find(struct dm_bufio_client *c, sector_t block)
 		if (b->block == block)
 			return b;
 
-		n = (b->block < block) ? n->rb_left : n->rb_right;
+		n = block < b->block ? n->rb_left : n->rb_right;
 	}
 
 	return NULL;
 }
 
+static struct dm_buffer *__find_next(struct dm_bufio_client *c, sector_t block)
+{
+	struct rb_node *n = c->buffer_tree.rb_node;
+	struct dm_buffer *b;
+	struct dm_buffer *best = NULL;
+
+	while (n) {
+		b = container_of(n, struct dm_buffer, node);
+
+		if (b->block == block)
+			return b;
+
+		if (block <= b->block) {
+			n = n->rb_left;
+			best = b;
+		} else {
+			n = n->rb_right;
+		}
+	}
+
+	return best;
+}
+
 static void __insert(struct dm_bufio_client *c, struct dm_buffer *b)
 {
 	struct rb_node **new = &c->buffer_tree.rb_node, *parent = NULL;
@@ -276,8 +299,8 @@ static void __insert(struct dm_bufio_client *c, struct dm_buffer *b)
 		}
 
 		parent = *new;
-		new = (found->block < b->block) ?
-			&((*new)->rb_left) : &((*new)->rb_right);
+		new = b->block < found->block ?
+			&found->node.rb_left : &found->node.rb_right;
 	}
 
 	rb_link_node(&b->node, parent, new);
@@ -631,6 +654,19 @@ dmio:
 	submit_bio(bio);
 }
 
+static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block)
+{
+	sector_t sector;
+
+	if (likely(c->sectors_per_block_bits >= 0))
+		sector = block << c->sectors_per_block_bits;
+	else
+		sector = block * (c->block_size >> SECTOR_SHIFT);
+	sector += c->start;
+
+	return sector;
+}
+
 static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buffer *, blk_status_t))
 {
 	unsigned n_sectors;
@@ -639,11 +675,7 @@ static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buff
 
 	b->end_io = end_io;
 
-	if (likely(b->c->sectors_per_block_bits >= 0))
-		sector = b->block << b->c->sectors_per_block_bits;
-	else
-		sector = b->block * (b->c->block_size >> SECTOR_SHIFT);
-	sector += b->c->start;
+	sector = block_to_sector(b->c, b->block);
 
 	if (rw != REQ_OP_WRITE) {
 		n_sectors = b->c->block_size >> SECTOR_SHIFT;
@@ -1326,6 +1358,30 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c)
 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
 
 /*
+ * Use dm-io to send a discard request to flush the device.
+ */
+int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count)
+{
+	struct dm_io_request io_req = {
+		.bi_op = REQ_OP_DISCARD,
+		.bi_op_flags = REQ_SYNC,
+		.mem.type = DM_IO_KMEM,
+		.mem.ptr.addr = NULL,
+		.client = c->dm_io,
+	};
+	struct dm_io_region io_reg = {
+		.bdev = c->bdev,
+		.sector = block_to_sector(c, block),
+		.count = block_to_sector(c, count),
+	};
+
+	BUG_ON(dm_bufio_in_request());
+
+	return dm_io(&io_req, 1, &io_reg, NULL);
+}
+EXPORT_SYMBOL_GPL(dm_bufio_issue_discard);
+
+/*
  * We first delete any other buffer that may be at that new location.
  *
  * Then, we write the buffer to the original location if it was dirty.
@@ -1401,6 +1457,14 @@ retry:
 }
 EXPORT_SYMBOL_GPL(dm_bufio_release_move);
 
+static void forget_buffer_locked(struct dm_buffer *b)
+{
+	if (likely(!b->hold_count) && likely(!b->state)) {
+		__unlink_buffer(b);
+		__free_buffer_wake(b);
+	}
+}
+
 /*
  * Free the given buffer.
  *
@@ -1414,15 +1478,36 @@ void dm_bufio_forget(struct dm_bufio_client *c, sector_t block)
 	dm_bufio_lock(c);
 
 	b = __find(c, block);
-	if (b && likely(!b->hold_count) && likely(!b->state)) {
-		__unlink_buffer(b);
-		__free_buffer_wake(b);
-	}
+	if (b)
+		forget_buffer_locked(b);
 
 	dm_bufio_unlock(c);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_forget);
 
+void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks)
+{
+	struct dm_buffer *b;
+	sector_t end_block = block + n_blocks;
+
+	while (block < end_block) {
+		dm_bufio_lock(c);
+
+		b = __find_next(c, block);
+		if (b) {
+			block = b->block + 1;
+			forget_buffer_locked(b);
+		}
+
+		dm_bufio_unlock(c);
+
+		if (!b)
+			break;
+	}
+
+}
+EXPORT_SYMBOL_GPL(dm_bufio_forget_buffers);
+
 void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned n)
 {
 	c->minimum_buffers = n;
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 3df90daba89e..000ddfab5ba0 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -34,7 +34,9 @@
 #include <crypto/aead.h>
 #include <crypto/authenc.h>
 #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
+#include <linux/key-type.h>
 #include <keys/user-type.h>
+#include <keys/encrypted-type.h>
 
 #include <linux/device-mapper.h>
 
@@ -212,7 +214,7 @@ struct crypt_config {
 	struct mutex bio_alloc_lock;
 
 	u8 *authenc_key; /* space for keys in authenc() format (if used) */
-	u8 key[0];
+	u8 key[];
 };
 
 #define MIN_IOS		64
@@ -2215,12 +2217,47 @@ static bool contains_whitespace(const char *str)
 	return false;
 }
 
+static int set_key_user(struct crypt_config *cc, struct key *key)
+{
+	const struct user_key_payload *ukp;
+
+	ukp = user_key_payload_locked(key);
+	if (!ukp)
+		return -EKEYREVOKED;
+
+	if (cc->key_size != ukp->datalen)
+		return -EINVAL;
+
+	memcpy(cc->key, ukp->data, cc->key_size);
+
+	return 0;
+}
+
+#if defined(CONFIG_ENCRYPTED_KEYS) || defined(CONFIG_ENCRYPTED_KEYS_MODULE)
+static int set_key_encrypted(struct crypt_config *cc, struct key *key)
+{
+	const struct encrypted_key_payload *ekp;
+
+	ekp = key->payload.data[0];
+	if (!ekp)
+		return -EKEYREVOKED;
+
+	if (cc->key_size != ekp->decrypted_datalen)
+		return -EINVAL;
+
+	memcpy(cc->key, ekp->decrypted_data, cc->key_size);
+
+	return 0;
+}
+#endif /* CONFIG_ENCRYPTED_KEYS */
+
 static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
 {
 	char *new_key_string, *key_desc;
 	int ret;
+	struct key_type *type;
 	struct key *key;
-	const struct user_key_payload *ukp;
+	int (*set_key)(struct crypt_config *cc, struct key *key);
 
 	/*
 	 * Reject key_string with whitespace. dm core currently lacks code for
@@ -2236,16 +2273,26 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string
 	if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
 		return -EINVAL;
 
-	if (strncmp(key_string, "logon:", key_desc - key_string + 1) &&
-	    strncmp(key_string, "user:", key_desc - key_string + 1))
+	if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
+		type = &key_type_logon;
+		set_key = set_key_user;
+	} else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) {
+		type = &key_type_user;
+		set_key = set_key_user;
+#if defined(CONFIG_ENCRYPTED_KEYS) || defined(CONFIG_ENCRYPTED_KEYS_MODULE)
+	} else if (!strncmp(key_string, "encrypted:", key_desc - key_string + 1)) {
+		type = &key_type_encrypted;
+		set_key = set_key_encrypted;
+#endif
+	} else {
 		return -EINVAL;
+	}
 
 	new_key_string = kstrdup(key_string, GFP_KERNEL);
 	if (!new_key_string)
 		return -ENOMEM;
 
-	key = request_key(key_string[0] == 'l' ? &key_type_logon : &key_type_user,
-			  key_desc + 1, NULL);
+	key = request_key(type, key_desc + 1, NULL);
 	if (IS_ERR(key)) {
 		kzfree(new_key_string);
 		return PTR_ERR(key);
@@ -2253,23 +2300,14 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string
 
 	down_read(&key->sem);
 
-	ukp = user_key_payload_locked(key);
-	if (!ukp) {
-		up_read(&key->sem);
-		key_put(key);
-		kzfree(new_key_string);
-		return -EKEYREVOKED;
-	}
-
-	if (cc->key_size != ukp->datalen) {
+	ret = set_key(cc, key);
+	if (ret < 0) {
 		up_read(&key->sem);
 		key_put(key);
 		kzfree(new_key_string);
-		return -EINVAL;
+		return ret;
 	}
 
-	memcpy(cc->key, ukp->data, cc->key_size);
-
 	up_read(&key->sem);
 	key_put(key);
 
@@ -2323,7 +2361,7 @@ static int get_key_size(char **key_string)
 	return (*key_string[0] == ':') ? -EINVAL : strlen(*key_string) >> 1;
 }
 
-#endif
+#endif /* CONFIG_KEYS */
 
 static int crypt_set_key(struct crypt_config *cc, char *key)
 {
@@ -3274,7 +3312,7 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
 	limits->max_segment_size = PAGE_SIZE;
 
 	limits->logical_block_size =
-		max_t(unsigned short, limits->logical_block_size, cc->sector_size);
+		max_t(unsigned, limits->logical_block_size, cc->sector_size);
 	limits->physical_block_size =
 		max_t(unsigned, limits->physical_block_size, cc->sector_size);
 	limits->io_min = max_t(unsigned, limits->io_min, cc->sector_size);
@@ -3282,7 +3320,7 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
 
 static struct target_type crypt_target = {
 	.name   = "crypt",
-	.version = {1, 20, 0},
+	.version = {1, 21, 0},
 	.module = THIS_MODULE,
 	.ctr    = crypt_ctr,
 	.dtr    = crypt_dtr,
diff --git a/drivers/md/dm-ebs-target.c b/drivers/md/dm-ebs-target.c
new file mode 100644
index 000000000000..44451276f128
--- /dev/null
+++ b/drivers/md/dm-ebs-target.c
@@ -0,0 +1,471 @@
+/*
+ * Copyright (C) 2020 Red Hat GmbH
+ *
+ * This file is released under the GPL.
+ *
+ * Device-mapper target to emulate smaller logical block
+ * size on backing devices exposing (natively) larger ones.
+ *
+ * E.g. 512 byte sector emulation on 4K native disks.
+ */
+
+#include "dm.h"
+#include <linux/module.h>
+#include <linux/workqueue.h>
+#include <linux/dm-bufio.h>
+
+#define DM_MSG_PREFIX "ebs"
+
+static void ebs_dtr(struct dm_target *ti);
+
+/* Emulated block size context. */
+struct ebs_c {
+	struct dm_dev *dev;		/* Underlying device to emulate block size on. */
+	struct dm_bufio_client *bufio;	/* Use dm-bufio for read and read-modify-write processing. */
+	struct workqueue_struct *wq;	/* Workqueue for ^ processing of bios. */
+	struct work_struct ws;		/* Work item used for ^. */
+	struct bio_list bios_in;	/* Worker bios input list. */
+	spinlock_t lock;		/* Guard bios input list above. */
+	sector_t start;			/* <start> table line argument, see ebs_ctr below. */
+	unsigned int e_bs;		/* Emulated block size in sectors exposed to upper layer. */
+	unsigned int u_bs;		/* Underlying block size in sectors retrievd from/set on lower layer device. */
+	unsigned char block_shift;	/* bitshift sectors -> blocks used in dm-bufio API. */
+	bool u_bs_set:1;		/* Flag to indicate underlying block size is set on table line. */
+};
+
+static inline sector_t __sector_to_block(struct ebs_c *ec, sector_t sector)
+{
+	return sector >> ec->block_shift;
+}
+
+static inline sector_t __block_mod(sector_t sector, unsigned int bs)
+{
+	return sector & (bs - 1);
+}
+
+/* Return number of blocks for a bio, accounting for misalignement of start and end sectors. */
+static inline unsigned int __nr_blocks(struct ebs_c *ec, struct bio *bio)
+{
+	sector_t end_sector = __block_mod(bio->bi_iter.bi_sector, ec->u_bs) + bio_sectors(bio);
+
+	return __sector_to_block(ec, end_sector) + (__block_mod(end_sector, ec->u_bs) ? 1 : 0);
+}
+
+static inline bool __ebs_check_bs(unsigned int bs)
+{
+	return bs && is_power_of_2(bs);
+}
+
+/*
+ * READ/WRITE:
+ *
+ * copy blocks between bufio blocks and bio vector's (partial/overlapping) pages.
+ */
+static int __ebs_rw_bvec(struct ebs_c *ec, int rw, struct bio_vec *bv, struct bvec_iter *iter)
+{
+	int r = 0;
+	unsigned char *ba, *pa;
+	unsigned int cur_len;
+	unsigned int bv_len = bv->bv_len;
+	unsigned int buf_off = to_bytes(__block_mod(iter->bi_sector, ec->u_bs));
+	sector_t block = __sector_to_block(ec, iter->bi_sector);
+	struct dm_buffer *b;
+
+	if (unlikely(!bv->bv_page || !bv_len))
+		return -EIO;
+
+	pa = page_address(bv->bv_page) + bv->bv_offset;
+
+	/* Handle overlapping page <-> blocks */
+	while (bv_len) {
+		cur_len = min(dm_bufio_get_block_size(ec->bufio) - buf_off, bv_len);
+
+		/* Avoid reading for writes in case bio vector's page overwrites block completely. */
+		if (rw == READ || buf_off || bv_len < dm_bufio_get_block_size(ec->bufio))
+			ba = dm_bufio_read(ec->bufio, block, &b);
+		else
+			ba = dm_bufio_new(ec->bufio, block, &b);
+
+		if (unlikely(IS_ERR(ba))) {
+			/*
+			 * Carry on with next buffer, if any, to issue all possible
+			 * data but return error.
+			 */
+			r = PTR_ERR(ba);
+		} else {
+			/* Copy data to/from bio to buffer if read/new was successful above. */
+			ba += buf_off;
+			if (rw == READ) {
+				memcpy(pa, ba, cur_len);
+				flush_dcache_page(bv->bv_page);
+			} else {
+				flush_dcache_page(bv->bv_page);
+				memcpy(ba, pa, cur_len);
+				dm_bufio_mark_partial_buffer_dirty(b, buf_off, buf_off + cur_len);
+			}
+
+			dm_bufio_release(b);
+		}
+
+		pa += cur_len;
+		bv_len -= cur_len;
+		buf_off = 0;
+		block++;
+	}
+
+	return r;
+}
+
+/* READ/WRITE: iterate bio vector's copying between (partial) pages and bufio blocks. */
+static int __ebs_rw_bio(struct ebs_c *ec, int rw, struct bio *bio)
+{
+	int r = 0, rr;
+	struct bio_vec bv;
+	struct bvec_iter iter;
+
+	bio_for_each_bvec(bv, bio, iter) {
+		rr = __ebs_rw_bvec(ec, rw, &bv, &iter);
+		if (rr)
+			r = rr;
+	}
+
+	return r;
+}
+
+/*
+ * Discard bio's blocks, i.e. pass discards down.
+ *
+ * Avoid discarding partial blocks at beginning and end;
+ * return 0 in case no blocks can be discarded as a result.
+ */
+static int __ebs_discard_bio(struct ebs_c *ec, struct bio *bio)
+{
+	sector_t block, blocks, sector = bio->bi_iter.bi_sector;
+
+	block = __sector_to_block(ec, sector);
+	blocks = __nr_blocks(ec, bio);
+
+	/*
+	 * Partial first underlying block (__nr_blocks() may have
+	 * resulted in one block).
+	 */
+	if (__block_mod(sector, ec->u_bs)) {
+		block++;
+		blocks--;
+	}
+
+	/* Partial last underlying block if any. */
+	if (blocks && __block_mod(bio_end_sector(bio), ec->u_bs))
+		blocks--;
+
+	return blocks ? dm_bufio_issue_discard(ec->bufio, block, blocks) : 0;
+}
+
+/* Release blocks them from the bufio cache. */
+static void __ebs_forget_bio(struct ebs_c *ec, struct bio *bio)
+{
+	sector_t blocks, sector = bio->bi_iter.bi_sector;
+
+	blocks = __nr_blocks(ec, bio);
+
+	dm_bufio_forget_buffers(ec->bufio, __sector_to_block(ec, sector), blocks);
+}
+
+/* Worker funtion to process incoming bios. */
+static void __ebs_process_bios(struct work_struct *ws)
+{
+	int r;
+	bool write = false;
+	sector_t block1, block2;
+	struct ebs_c *ec = container_of(ws, struct ebs_c, ws);
+	struct bio *bio;
+	struct bio_list bios;
+
+	bio_list_init(&bios);
+
+	spin_lock_irq(&ec->lock);
+	bios = ec->bios_in;
+	bio_list_init(&ec->bios_in);
+	spin_unlock_irq(&ec->lock);
+
+	/* Prefetch all read and any mis-aligned write buffers */
+	bio_list_for_each(bio, &bios) {
+		block1 = __sector_to_block(ec, bio->bi_iter.bi_sector);
+		if (bio_op(bio) == REQ_OP_READ)
+			dm_bufio_prefetch(ec->bufio, block1, __nr_blocks(ec, bio));
+		else if (bio_op(bio) == REQ_OP_WRITE && !(bio->bi_opf & REQ_PREFLUSH)) {
+			block2 = __sector_to_block(ec, bio_end_sector(bio));
+			if (__block_mod(bio->bi_iter.bi_sector, ec->u_bs))
+				dm_bufio_prefetch(ec->bufio, block1, 1);
+			if (__block_mod(bio_end_sector(bio), ec->u_bs) && block2 != block1)
+				dm_bufio_prefetch(ec->bufio, block2, 1);
+		}
+	}
+
+	bio_list_for_each(bio, &bios) {
+		r = -EIO;
+		if (bio_op(bio) == REQ_OP_READ)
+			r = __ebs_rw_bio(ec, READ, bio);
+		else if (bio_op(bio) == REQ_OP_WRITE) {
+			write = true;
+			r = __ebs_rw_bio(ec, WRITE, bio);
+		} else if (bio_op(bio) == REQ_OP_DISCARD) {
+			__ebs_forget_bio(ec, bio);
+			r = __ebs_discard_bio(ec, bio);
+		}
+
+		if (r < 0)
+			bio->bi_status = errno_to_blk_status(r);
+	}
+
+	/*
+	 * We write dirty buffers after processing I/O on them
+	 * but before we endio thus addressing REQ_FUA/REQ_SYNC.
+	 */
+	r = write ? dm_bufio_write_dirty_buffers(ec->bufio) : 0;
+
+	while ((bio = bio_list_pop(&bios))) {
+		/* Any other request is endioed. */
+		if (unlikely(r && bio_op(bio) == REQ_OP_WRITE))
+			bio_io_error(bio);
+		else
+			bio_endio(bio);
+	}
+}
+
+/*
+ * Construct an emulated block size mapping: <dev_path> <offset> <ebs> [<ubs>]
+ *
+ * <dev_path>: path of the underlying device
+ * <offset>: offset in 512 bytes sectors into <dev_path>
+ * <ebs>: emulated block size in units of 512 bytes exposed to the upper layer
+ * [<ubs>]: underlying block size in units of 512 bytes imposed on the lower layer;
+ * 	    optional, if not supplied, retrieve logical block size from underlying device
+ */
+static int ebs_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+	int r;
+	unsigned short tmp1;
+	unsigned long long tmp;
+	char dummy;
+	struct ebs_c *ec;
+
+	if (argc < 3 || argc > 4) {
+		ti->error = "Invalid argument count";
+		return -EINVAL;
+	}
+
+	ec = ti->private = kzalloc(sizeof(*ec), GFP_KERNEL);
+	if (!ec) {
+		ti->error = "Cannot allocate ebs context";
+		return -ENOMEM;
+	}
+
+	r = -EINVAL;
+	if (sscanf(argv[1], "%llu%c", &tmp, &dummy) != 1 ||
+	    tmp != (sector_t)tmp ||
+	    (sector_t)tmp >= ti->len) {
+		ti->error = "Invalid device offset sector";
+		goto bad;
+	}
+	ec->start = tmp;
+
+	if (sscanf(argv[2], "%hu%c", &tmp1, &dummy) != 1 ||
+	    !__ebs_check_bs(tmp1) ||
+	    to_bytes(tmp1) > PAGE_SIZE) {
+		ti->error = "Invalid emulated block size";
+		goto bad;
+	}
+	ec->e_bs = tmp1;
+
+	if (argc > 3) {
+		if (sscanf(argv[3], "%hu%c", &tmp1, &dummy) != 1 || !__ebs_check_bs(tmp1)) {
+			ti->error = "Invalid underlying block size";
+			goto bad;
+		}
+		ec->u_bs = tmp1;
+		ec->u_bs_set = true;
+	} else
+		ec->u_bs_set = false;
+
+	r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ec->dev);
+	if (r) {
+		ti->error = "Device lookup failed";
+		ec->dev = NULL;
+		goto bad;
+	}
+
+	r = -EINVAL;
+	if (!ec->u_bs_set) {
+		ec->u_bs = to_sector(bdev_logical_block_size(ec->dev->bdev));
+		if (!__ebs_check_bs(ec->u_bs)) {
+			ti->error = "Invalid retrieved underlying block size";
+			goto bad;
+		}
+	}
+
+	if (!ec->u_bs_set && ec->e_bs == ec->u_bs)
+		DMINFO("Emulation superfluous: emulated equal to underlying block size");
+
+	if (__block_mod(ec->start, ec->u_bs)) {
+		ti->error = "Device offset must be multiple of underlying block size";
+		goto bad;
+	}
+
+	ec->bufio = dm_bufio_client_create(ec->dev->bdev, to_bytes(ec->u_bs), 1, 0, NULL, NULL);
+	if (IS_ERR(ec->bufio)) {
+		ti->error = "Cannot create dm bufio client";
+		r = PTR_ERR(ec->bufio);
+		ec->bufio = NULL;
+		goto bad;
+	}
+
+	ec->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
+	if (!ec->wq) {
+		ti->error = "Cannot create dm-" DM_MSG_PREFIX " workqueue";
+		r = -ENOMEM;
+		goto bad;
+	}
+
+	ec->block_shift = __ffs(ec->u_bs);
+	INIT_WORK(&ec->ws, &__ebs_process_bios);
+	bio_list_init(&ec->bios_in);
+	spin_lock_init(&ec->lock);
+
+	ti->num_flush_bios = 1;
+	ti->num_discard_bios = 1;
+	ti->num_secure_erase_bios = 0;
+	ti->num_write_same_bios = 0;
+	ti->num_write_zeroes_bios = 0;
+	return 0;
+bad:
+	ebs_dtr(ti);
+	return r;
+}
+
+static void ebs_dtr(struct dm_target *ti)
+{
+	struct ebs_c *ec = ti->private;
+
+	if (ec->wq)
+		destroy_workqueue(ec->wq);
+	if (ec->bufio)
+		dm_bufio_client_destroy(ec->bufio);
+	if (ec->dev)
+		dm_put_device(ti, ec->dev);
+	kfree(ec);
+}
+
+static int ebs_map(struct dm_target *ti, struct bio *bio)
+{
+	struct ebs_c *ec = ti->private;
+
+	bio_set_dev(bio, ec->dev->bdev);
+	bio->bi_iter.bi_sector = ec->start + dm_target_offset(ti, bio->bi_iter.bi_sector);
+
+	if (unlikely(bio->bi_opf & REQ_OP_FLUSH))
+		return DM_MAPIO_REMAPPED;
+	/*
+	 * Only queue for bufio processing in case of partial or overlapping buffers
+	 * -or-
+	 * emulation with ebs == ubs aiming for tests of dm-bufio overhead.
+	 */
+	if (likely(__block_mod(bio->bi_iter.bi_sector, ec->u_bs) ||
+		   __block_mod(bio_end_sector(bio), ec->u_bs) ||
+		   ec->e_bs == ec->u_bs)) {
+		spin_lock_irq(&ec->lock);
+		bio_list_add(&ec->bios_in, bio);
+		spin_unlock_irq(&ec->lock);
+
+		queue_work(ec->wq, &ec->ws);
+
+		return DM_MAPIO_SUBMITTED;
+	}
+
+	/* Forget any buffer content relative to this direct backing device I/O. */
+	__ebs_forget_bio(ec, bio);
+
+	return DM_MAPIO_REMAPPED;
+}
+
+static void ebs_status(struct dm_target *ti, status_type_t type,
+		       unsigned status_flags, char *result, unsigned maxlen)
+{
+	struct ebs_c *ec = ti->private;
+
+	switch (type) {
+	case STATUSTYPE_INFO:
+		*result = '\0';
+		break;
+	case STATUSTYPE_TABLE:
+		snprintf(result, maxlen, ec->u_bs_set ? "%s %llu %u %u" : "%s %llu %u",
+			 ec->dev->name, (unsigned long long) ec->start, ec->e_bs, ec->u_bs);
+		break;
+	}
+}
+
+static int ebs_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
+{
+	struct ebs_c *ec = ti->private;
+	struct dm_dev *dev = ec->dev;
+
+	/*
+	 * Only pass ioctls through if the device sizes match exactly.
+	 */
+	*bdev = dev->bdev;
+	return !!(ec->start || ti->len != i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT);
+}
+
+static void ebs_io_hints(struct dm_target *ti, struct queue_limits *limits)
+{
+	struct ebs_c *ec = ti->private;
+
+	limits->logical_block_size = to_bytes(ec->e_bs);
+	limits->physical_block_size = to_bytes(ec->u_bs);
+	limits->alignment_offset = limits->physical_block_size;
+	blk_limits_io_min(limits, limits->logical_block_size);
+}
+
+static int ebs_iterate_devices(struct dm_target *ti,
+				  iterate_devices_callout_fn fn, void *data)
+{
+	struct ebs_c *ec = ti->private;
+
+	return fn(ti, ec->dev, ec->start, ti->len, data);
+}
+
+static struct target_type ebs_target = {
+	.name		 = "ebs",
+	.version	 = {1, 0, 1},
+	.features	 = DM_TARGET_PASSES_INTEGRITY,
+	.module		 = THIS_MODULE,
+	.ctr		 = ebs_ctr,
+	.dtr		 = ebs_dtr,
+	.map		 = ebs_map,
+	.status		 = ebs_status,
+	.io_hints	 = ebs_io_hints,
+	.prepare_ioctl	 = ebs_prepare_ioctl,
+	.iterate_devices = ebs_iterate_devices,
+};
+
+static int __init dm_ebs_init(void)
+{
+	int r = dm_register_target(&ebs_target);
+
+	if (r < 0)
+		DMERR("register failed %d", r);
+
+	return r;
+}
+
+static void dm_ebs_exit(void)
+{
+	dm_unregister_target(&ebs_target);
+}
+
+module_init(dm_ebs_init);
+module_exit(dm_ebs_exit);
+
+MODULE_AUTHOR("Heinz Mauelshagen <dm-devel@redhat.com>");
+MODULE_DESCRIPTION(DM_NAME " emulated block size target");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-historical-service-time.c b/drivers/md/dm-historical-service-time.c
new file mode 100644
index 000000000000..186f91e2752c
--- /dev/null
+++ b/drivers/md/dm-historical-service-time.c
@@ -0,0 +1,561 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Historical Service Time
+ *
+ *  Keeps a time-weighted exponential moving average of the historical
+ *  service time. Estimates future service time based on the historical
+ *  service time and the number of outstanding requests.
+ *
+ *  Marks paths stale if they have not finished within hst *
+ *  num_paths. If a path is stale and unused, we will send a single
+ *  request to probe in case the path has improved. This situation
+ *  generally arises if the path is so much worse than others that it
+ *  will never have the best estimated service time, or if the entire
+ *  multipath device is unused. If a path is stale and in use, limit the
+ *  number of requests it can receive with the assumption that the path
+ *  has become degraded.
+ *
+ *  To avoid repeatedly calculating exponents for time weighting, times
+ *  are split into HST_WEIGHT_COUNT buckets each (1 >> HST_BUCKET_SHIFT)
+ *  ns, and the weighting is pre-calculated.
+ *
+ */
+
+#include "dm.h"
+#include "dm-path-selector.h"
+
+#include <linux/blkdev.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+
+
+#define DM_MSG_PREFIX	"multipath historical-service-time"
+#define HST_MIN_IO 1
+#define HST_VERSION "0.1.1"
+
+#define HST_FIXED_SHIFT 10  /* 10 bits of decimal precision */
+#define HST_FIXED_MAX (ULLONG_MAX >> HST_FIXED_SHIFT)
+#define HST_FIXED_1 (1 << HST_FIXED_SHIFT)
+#define HST_FIXED_95 972
+#define HST_MAX_INFLIGHT HST_FIXED_1
+#define HST_BUCKET_SHIFT 24 /* Buckets are ~ 16ms */
+#define HST_WEIGHT_COUNT 64ULL
+
+struct selector {
+	struct list_head valid_paths;
+	struct list_head failed_paths;
+	int valid_count;
+	spinlock_t lock;
+
+	unsigned int weights[HST_WEIGHT_COUNT];
+	unsigned int threshold_multiplier;
+};
+
+struct path_info {
+	struct list_head list;
+	struct dm_path *path;
+	unsigned int repeat_count;
+
+	spinlock_t lock;
+
+	u64 historical_service_time; /* Fixed point */
+
+	u64 stale_after;
+	u64 last_finish;
+
+	u64 outstanding;
+};
+
+/**
+ * fixed_power - compute: x^n, in O(log n) time
+ *
+ * @x:         base of the power
+ * @frac_bits: fractional bits of @x
+ * @n:         power to raise @x to.
+ *
+ * By exploiting the relation between the definition of the natural power
+ * function: x^n := x*x*...*x (x multiplied by itself for n times), and
+ * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i,
+ * (where: n_i \elem {0, 1}, the binary vector representing n),
+ * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is
+ * of course trivially computable in O(log_2 n), the length of our binary
+ * vector.
+ *
+ * (see: kernel/sched/loadavg.c)
+ */
+static u64 fixed_power(u64 x, unsigned int frac_bits, unsigned int n)
+{
+	unsigned long result = 1UL << frac_bits;
+
+	if (n) {
+		for (;;) {
+			if (n & 1) {
+				result *= x;
+				result += 1UL << (frac_bits - 1);
+				result >>= frac_bits;
+			}
+			n >>= 1;
+			if (!n)
+				break;
+			x *= x;
+			x += 1UL << (frac_bits - 1);
+			x >>= frac_bits;
+		}
+	}
+
+	return result;
+}
+
+/*
+ * Calculate the next value of an exponential moving average
+ * a_1 = a_0 * e + a * (1 - e)
+ *
+ * @last: [0, ULLONG_MAX >> HST_FIXED_SHIFT]
+ * @next: [0, ULLONG_MAX >> HST_FIXED_SHIFT]
+ * @weight: [0, HST_FIXED_1]
+ *
+ * Note:
+ *   To account for multiple periods in the same calculation,
+ *   a_n = a_0 * e^n + a * (1 - e^n),
+ *   so call fixed_ema(last, next, pow(weight, N))
+ */
+static u64 fixed_ema(u64 last, u64 next, u64 weight)
+{
+	last *= weight;
+	last += next * (HST_FIXED_1 - weight);
+	last += 1ULL << (HST_FIXED_SHIFT - 1);
+	return last >> HST_FIXED_SHIFT;
+}
+
+static struct selector *alloc_selector(void)
+{
+	struct selector *s = kmalloc(sizeof(*s), GFP_KERNEL);
+
+	if (s) {
+		INIT_LIST_HEAD(&s->valid_paths);
+		INIT_LIST_HEAD(&s->failed_paths);
+		spin_lock_init(&s->lock);
+		s->valid_count = 0;
+	}
+
+	return s;
+}
+
+/*
+ * Get the weight for a given time span.
+ */
+static u64 hst_weight(struct path_selector *ps, u64 delta)
+{
+	struct selector *s = ps->context;
+	int bucket = clamp(delta >> HST_BUCKET_SHIFT, 0ULL,
+			   HST_WEIGHT_COUNT - 1);
+
+	return s->weights[bucket];
+}
+
+/*
+ * Set up the weights array.
+ *
+ * weights[len-1] = 0
+ * weights[n] = base ^ (n + 1)
+ */
+static void hst_set_weights(struct path_selector *ps, unsigned int base)
+{
+	struct selector *s = ps->context;
+	int i;
+
+	if (base >= HST_FIXED_1)
+		return;
+
+	for (i = 0; i < HST_WEIGHT_COUNT - 1; i++)
+		s->weights[i] = fixed_power(base, HST_FIXED_SHIFT, i + 1);
+	s->weights[HST_WEIGHT_COUNT - 1] = 0;
+}
+
+static int hst_create(struct path_selector *ps, unsigned int argc, char **argv)
+{
+	struct selector *s;
+	unsigned int base_weight = HST_FIXED_95;
+	unsigned int threshold_multiplier = 0;
+	char dummy;
+
+	/*
+	 * Arguments: [<base_weight> [<threshold_multiplier>]]
+	 *   <base_weight>: Base weight for ema [0, 1024) 10-bit fixed point. A
+	 *                  value of 0 will completely ignore any history.
+	 *                  If not given, default (HST_FIXED_95) is used.
+	 *   <threshold_multiplier>: Minimum threshold multiplier for paths to
+	 *                  be considered different. That is, a path is
+	 *                  considered different iff (p1 > N * p2) where p1
+	 *                  is the path with higher service time. A threshold
+	 *                  of 1 or 0 has no effect. Defaults to 0.
+	 */
+	if (argc > 2)
+		return -EINVAL;
+
+	if (argc && (sscanf(argv[0], "%u%c", &base_weight, &dummy) != 1 ||
+	     base_weight >= HST_FIXED_1)) {
+		return -EINVAL;
+	}
+
+	if (argc > 1 && (sscanf(argv[1], "%u%c",
+				&threshold_multiplier, &dummy) != 1)) {
+		return -EINVAL;
+	}
+
+	s = alloc_selector();
+	if (!s)
+		return -ENOMEM;
+
+	ps->context = s;
+
+	hst_set_weights(ps, base_weight);
+	s->threshold_multiplier = threshold_multiplier;
+	return 0;
+}
+
+static void free_paths(struct list_head *paths)
+{
+	struct path_info *pi, *next;
+
+	list_for_each_entry_safe(pi, next, paths, list) {
+		list_del(&pi->list);
+		kfree(pi);
+	}
+}
+
+static void hst_destroy(struct path_selector *ps)
+{
+	struct selector *s = ps->context;
+
+	free_paths(&s->valid_paths);
+	free_paths(&s->failed_paths);
+	kfree(s);
+	ps->context = NULL;
+}
+
+static int hst_status(struct path_selector *ps, struct dm_path *path,
+		     status_type_t type, char *result, unsigned int maxlen)
+{
+	unsigned int sz = 0;
+	struct path_info *pi;
+
+	if (!path) {
+		struct selector *s = ps->context;
+
+		DMEMIT("2 %u %u ", s->weights[0], s->threshold_multiplier);
+	} else {
+		pi = path->pscontext;
+
+		switch (type) {
+		case STATUSTYPE_INFO:
+			DMEMIT("%llu %llu %llu ", pi->historical_service_time,
+			       pi->outstanding, pi->stale_after);
+			break;
+		case STATUSTYPE_TABLE:
+			DMEMIT("0 ");
+			break;
+		}
+	}
+
+	return sz;
+}
+
+static int hst_add_path(struct path_selector *ps, struct dm_path *path,
+		       int argc, char **argv, char **error)
+{
+	struct selector *s = ps->context;
+	struct path_info *pi;
+	unsigned int repeat_count = HST_MIN_IO;
+	char dummy;
+	unsigned long flags;
+
+	/*
+	 * Arguments: [<repeat_count>]
+	 *   <repeat_count>: The number of I/Os before switching path.
+	 *                   If not given, default (HST_MIN_IO) is used.
+	 */
+	if (argc > 1) {
+		*error = "historical-service-time ps: incorrect number of arguments";
+		return -EINVAL;
+	}
+
+	if (argc && (sscanf(argv[0], "%u%c", &repeat_count, &dummy) != 1)) {
+		*error = "historical-service-time ps: invalid repeat count";
+		return -EINVAL;
+	}
+
+	/* allocate the path */
+	pi = kmalloc(sizeof(*pi), GFP_KERNEL);
+	if (!pi) {
+		*error = "historical-service-time ps: Error allocating path context";
+		return -ENOMEM;
+	}
+
+	pi->path = path;
+	pi->repeat_count = repeat_count;
+
+	pi->historical_service_time = HST_FIXED_1;
+
+	spin_lock_init(&pi->lock);
+	pi->outstanding = 0;
+
+	pi->stale_after = 0;
+	pi->last_finish = 0;
+
+	path->pscontext = pi;
+
+	spin_lock_irqsave(&s->lock, flags);
+	list_add_tail(&pi->list, &s->valid_paths);
+	s->valid_count++;
+	spin_unlock_irqrestore(&s->lock, flags);
+
+	return 0;
+}
+
+static void hst_fail_path(struct path_selector *ps, struct dm_path *path)
+{
+	struct selector *s = ps->context;
+	struct path_info *pi = path->pscontext;
+	unsigned long flags;
+
+	spin_lock_irqsave(&s->lock, flags);
+	list_move(&pi->list, &s->failed_paths);
+	s->valid_count--;
+	spin_unlock_irqrestore(&s->lock, flags);
+}
+
+static int hst_reinstate_path(struct path_selector *ps, struct dm_path *path)
+{
+	struct selector *s = ps->context;
+	struct path_info *pi = path->pscontext;
+	unsigned long flags;
+
+	spin_lock_irqsave(&s->lock, flags);
+	list_move_tail(&pi->list, &s->valid_paths);
+	s->valid_count++;
+	spin_unlock_irqrestore(&s->lock, flags);
+
+	return 0;
+}
+
+static void hst_fill_compare(struct path_info *pi, u64 *hst,
+			     u64 *out, u64 *stale)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&pi->lock, flags);
+	*hst = pi->historical_service_time;
+	*out = pi->outstanding;
+	*stale = pi->stale_after;
+	spin_unlock_irqrestore(&pi->lock, flags);
+}
+
+/*
+ * Compare the estimated service time of 2 paths, pi1 and pi2,
+ * for the incoming I/O.
+ *
+ * Returns:
+ * < 0 : pi1 is better
+ * 0   : no difference between pi1 and pi2
+ * > 0 : pi2 is better
+ *
+ */
+static long long hst_compare(struct path_info *pi1, struct path_info *pi2,
+			     u64 time_now, struct path_selector *ps)
+{
+	struct selector *s = ps->context;
+	u64 hst1, hst2;
+	long long out1, out2, stale1, stale2;
+	int pi2_better, over_threshold;
+
+	hst_fill_compare(pi1, &hst1, &out1, &stale1);
+	hst_fill_compare(pi2, &hst2, &out2, &stale2);
+
+	/* Check here if estimated latency for two paths are too similar.
+	 * If this is the case, we skip extra calculation and just compare
+	 * outstanding requests. In this case, any unloaded paths will
+	 * be preferred.
+	 */
+	if (hst1 > hst2)
+		over_threshold = hst1 > (s->threshold_multiplier * hst2);
+	else
+		over_threshold = hst2 > (s->threshold_multiplier * hst1);
+
+	if (!over_threshold)
+		return out1 - out2;
+
+	/*
+	 * If an unloaded path is stale, choose it. If both paths are unloaded,
+	 * choose path that is the most stale.
+	 * (If one path is loaded, choose the other)
+	 */
+	if ((!out1 && stale1 < time_now) || (!out2 && stale2 < time_now) ||
+	    (!out1 && !out2))
+		return (!out2 * stale1) - (!out1 * stale2);
+
+	/* Compare estimated service time. If outstanding is the same, we
+	 * don't need to multiply
+	 */
+	if (out1 == out2) {
+		pi2_better = hst1 > hst2;
+	} else {
+		/* Potential overflow with out >= 1024 */
+		if (unlikely(out1 >= HST_MAX_INFLIGHT ||
+			     out2 >= HST_MAX_INFLIGHT)) {
+			/* If over 1023 in-flights, we may overflow if hst
+			 * is at max. (With this shift we still overflow at
+			 * 1048576 in-flights, which is high enough).
+			 */
+			hst1 >>= HST_FIXED_SHIFT;
+			hst2 >>= HST_FIXED_SHIFT;
+		}
+		pi2_better = (1 + out1) * hst1 > (1 + out2) * hst2;
+	}
+
+	/* In the case that the 'winner' is stale, limit to equal usage. */
+	if (pi2_better) {
+		if (stale2 < time_now)
+			return out1 - out2;
+		return 1;
+	}
+	if (stale1 < time_now)
+		return out1 - out2;
+	return -1;
+}
+
+static struct dm_path *hst_select_path(struct path_selector *ps,
+				       size_t nr_bytes)
+{
+	struct selector *s = ps->context;
+	struct path_info *pi = NULL, *best = NULL;
+	u64 time_now = sched_clock();
+	struct dm_path *ret = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&s->lock, flags);
+	if (list_empty(&s->valid_paths))
+		goto out;
+
+	list_for_each_entry(pi, &s->valid_paths, list) {
+		if (!best || (hst_compare(pi, best, time_now, ps) < 0))
+			best = pi;
+	}
+
+	if (!best)
+		goto out;
+
+	/* Move last used path to end (least preferred in case of ties) */
+	list_move_tail(&best->list, &s->valid_paths);
+
+	ret = best->path;
+
+out:
+	spin_unlock_irqrestore(&s->lock, flags);
+	return ret;
+}
+
+static int hst_start_io(struct path_selector *ps, struct dm_path *path,
+			size_t nr_bytes)
+{
+	struct path_info *pi = path->pscontext;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pi->lock, flags);
+	pi->outstanding++;
+	spin_unlock_irqrestore(&pi->lock, flags);
+
+	return 0;
+}
+
+static u64 path_service_time(struct path_info *pi, u64 start_time)
+{
+	u64 sched_now = ktime_get_ns();
+
+	/* if a previous disk request has finished after this IO was
+	 * sent to the hardware, pretend the submission happened
+	 * serially.
+	 */
+	if (time_after64(pi->last_finish, start_time))
+		start_time = pi->last_finish;
+
+	pi->last_finish = sched_now;
+	if (time_before64(sched_now, start_time))
+		return 0;
+
+	return sched_now - start_time;
+}
+
+static int hst_end_io(struct path_selector *ps, struct dm_path *path,
+		      size_t nr_bytes, u64 start_time)
+{
+	struct path_info *pi = path->pscontext;
+	struct selector *s = ps->context;
+	unsigned long flags;
+	u64 st;
+
+	spin_lock_irqsave(&pi->lock, flags);
+
+	st = path_service_time(pi, start_time);
+	pi->outstanding--;
+	pi->historical_service_time =
+		fixed_ema(pi->historical_service_time,
+			  min(st * HST_FIXED_1, HST_FIXED_MAX),
+			  hst_weight(ps, st));
+
+	/*
+	 * On request end, mark path as fresh. If a path hasn't
+	 * finished any requests within the fresh period, the estimated
+	 * service time is considered too optimistic and we limit the
+	 * maximum requests on that path.
+	 */
+	pi->stale_after = pi->last_finish +
+		(s->valid_count * (pi->historical_service_time >> HST_FIXED_SHIFT));
+
+	spin_unlock_irqrestore(&pi->lock, flags);
+
+	return 0;
+}
+
+static struct path_selector_type hst_ps = {
+	.name		= "historical-service-time",
+	.module		= THIS_MODULE,
+	.table_args	= 1,
+	.info_args	= 3,
+	.create		= hst_create,
+	.destroy	= hst_destroy,
+	.status		= hst_status,
+	.add_path	= hst_add_path,
+	.fail_path	= hst_fail_path,
+	.reinstate_path	= hst_reinstate_path,
+	.select_path	= hst_select_path,
+	.start_io	= hst_start_io,
+	.end_io		= hst_end_io,
+};
+
+static int __init dm_hst_init(void)
+{
+	int r = dm_register_path_selector(&hst_ps);
+
+	if (r < 0)
+		DMERR("register failed %d", r);
+
+	DMINFO("version " HST_VERSION " loaded");
+
+	return r;
+}
+
+static void __exit dm_hst_exit(void)
+{
+	int r = dm_unregister_path_selector(&hst_ps);
+
+	if (r < 0)
+		DMERR("unregister failed %d", r);
+}
+
+module_init(dm_hst_init);
+module_exit(dm_hst_exit);
+
+MODULE_DESCRIPTION(DM_NAME " measured service time oriented path selector");
+MODULE_AUTHOR("Khazhismel Kumykov <khazhy@google.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 84cb04904fab..81dc5ff08909 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -92,7 +92,7 @@ struct journal_entry {
 		} s;
 		__u64 sector;
 	} u;
-	commit_id_t last_bytes[0];
+	commit_id_t last_bytes[];
 	/* __u8 tag[0]; */
 };
 
@@ -1553,8 +1553,6 @@ static void integrity_metadata(struct work_struct *w)
 		char checksums_onstack[max((size_t)HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
 		sector_t sector;
 		unsigned sectors_to_process;
-		sector_t save_metadata_block;
-		unsigned save_metadata_offset;
 
 		if (unlikely(ic->mode == 'R'))
 			goto skip_io;
@@ -1605,8 +1603,6 @@ static void integrity_metadata(struct work_struct *w)
 			goto skip_io;
 		}
 
-		save_metadata_block = dio->metadata_block;
-		save_metadata_offset = dio->metadata_offset;
 		sector = dio->range.logical_sector;
 		sectors_to_process = dio->range.n_sectors;
 
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c
index 8ea20b56b4d6..e3d35c6c9f71 100644
--- a/drivers/md/dm-log-writes.c
+++ b/drivers/md/dm-log-writes.c
@@ -127,7 +127,7 @@ struct pending_block {
 	char *data;
 	u32 datalen;
 	struct list_head list;
-	struct bio_vec vecs[0];
+	struct bio_vec vecs[];
 };
 
 struct per_bio_data {
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 3e500098132f..78cff42d987e 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -439,7 +439,7 @@ failed:
 }
 
 /*
- * dm_report_EIO() is a macro instead of a function to make pr_debug()
+ * dm_report_EIO() is a macro instead of a function to make pr_debug_ratelimited()
  * report the function name and line number of the function from which
  * it has been invoked.
  */
@@ -447,43 +447,25 @@ failed:
 do {									\
 	struct mapped_device *md = dm_table_get_md((m)->ti->table);	\
 									\
-	pr_debug("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d\n", \
-		 dm_device_name(md),					\
-		 test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags),	\
-		 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags),	\
-		 dm_noflush_suspending((m)->ti));			\
+	DMDEBUG_LIMIT("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d", \
+		      dm_device_name(md),				\
+		      test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags),	\
+		      test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags), \
+		      dm_noflush_suspending((m)->ti));			\
 } while (0)
 
 /*
  * Check whether bios must be queued in the device-mapper core rather
  * than here in the target.
- *
- * If MPATHF_QUEUE_IF_NO_PATH and MPATHF_SAVED_QUEUE_IF_NO_PATH hold
- * the same value then we are not between multipath_presuspend()
- * and multipath_resume() calls and we have no need to check
- * for the DMF_NOFLUSH_SUSPENDING flag.
  */
-static bool __must_push_back(struct multipath *m, unsigned long flags)
+static bool __must_push_back(struct multipath *m)
 {
-	return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) !=
-		 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &flags)) &&
-		dm_noflush_suspending(m->ti));
+	return dm_noflush_suspending(m->ti);
 }
 
-/*
- * Following functions use READ_ONCE to get atomic access to
- * all m->flags to avoid taking spinlock
- */
 static bool must_push_back_rq(struct multipath *m)
 {
-	unsigned long flags = READ_ONCE(m->flags);
-	return test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) || __must_push_back(m, flags);
-}
-
-static bool must_push_back_bio(struct multipath *m)
-{
-	unsigned long flags = READ_ONCE(m->flags);
-	return __must_push_back(m, flags);
+	return test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) || __must_push_back(m);
 }
 
 /*
@@ -567,7 +549,8 @@ static void multipath_release_clone(struct request *clone,
 		if (pgpath && pgpath->pg->ps.type->end_io)
 			pgpath->pg->ps.type->end_io(&pgpath->pg->ps,
 						    &pgpath->path,
-						    mpio->nr_bytes);
+						    mpio->nr_bytes,
+						    clone->io_start_time_ns);
 	}
 
 	blk_put_request(clone);
@@ -619,7 +602,7 @@ static int __multipath_map_bio(struct multipath *m, struct bio *bio,
 		return DM_MAPIO_SUBMITTED;
 
 	if (!pgpath) {
-		if (must_push_back_bio(m))
+		if (__must_push_back(m))
 			return DM_MAPIO_REQUEUE;
 		dm_report_EIO(m);
 		return DM_MAPIO_KILL;
@@ -709,15 +692,38 @@ static void process_queued_bios(struct work_struct *work)
  * If we run out of usable paths, should we queue I/O or error it?
  */
 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
-			    bool save_old_value)
+			    bool save_old_value, const char *caller)
 {
 	unsigned long flags;
+	bool queue_if_no_path_bit, saved_queue_if_no_path_bit;
+	const char *dm_dev_name = dm_device_name(dm_table_get_md(m->ti->table));
+
+	DMDEBUG("%s: %s caller=%s queue_if_no_path=%d save_old_value=%d",
+		dm_dev_name, __func__, caller, queue_if_no_path, save_old_value);
 
 	spin_lock_irqsave(&m->lock, flags);
-	assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags,
-		   (save_old_value && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) ||
-		   (!save_old_value && queue_if_no_path));
+
+	queue_if_no_path_bit = test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
+	saved_queue_if_no_path_bit = test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
+
+	if (save_old_value) {
+		if (unlikely(!queue_if_no_path_bit && saved_queue_if_no_path_bit)) {
+			DMERR("%s: QIFNP disabled but saved as enabled, saving again loses state, not saving!",
+			      dm_dev_name);
+		} else
+			assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path_bit);
+	} else if (!queue_if_no_path && saved_queue_if_no_path_bit) {
+		/* due to "fail_if_no_path" message, need to honor it. */
+		clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
+	}
 	assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path);
+
+	DMDEBUG("%s: after %s changes; QIFNP = %d; SQIFNP = %d; DNFS = %d",
+		dm_dev_name, __func__,
+		test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags),
+		test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags),
+		dm_noflush_suspending(m->ti));
+
 	spin_unlock_irqrestore(&m->lock, flags);
 
 	if (!queue_if_no_path) {
@@ -738,7 +744,7 @@ static void queue_if_no_path_timeout_work(struct timer_list *t)
 	struct mapped_device *md = dm_table_get_md(m->ti->table);
 
 	DMWARN("queue_if_no_path timeout on %s, failing queued IO", dm_device_name(md));
-	queue_if_no_path(m, false, false);
+	queue_if_no_path(m, false, false, __func__);
 }
 
 /*
@@ -1078,7 +1084,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m)
 		argc--;
 
 		if (!strcasecmp(arg_name, "queue_if_no_path")) {
-			r = queue_if_no_path(m, true, false);
+			r = queue_if_no_path(m, true, false, __func__);
 			continue;
 		}
 
@@ -1279,7 +1285,9 @@ static int fail_path(struct pgpath *pgpath)
 	if (!pgpath->is_active)
 		goto out;
 
-	DMWARN("Failing path %s.", pgpath->path.dev->name);
+	DMWARN("%s: Failing path %s.",
+	       dm_device_name(dm_table_get_md(m->ti->table)),
+	       pgpath->path.dev->name);
 
 	pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
 	pgpath->is_active = false;
@@ -1318,7 +1326,9 @@ static int reinstate_path(struct pgpath *pgpath)
 	if (pgpath->is_active)
 		goto out;
 
-	DMWARN("Reinstating path %s.", pgpath->path.dev->name);
+	DMWARN("%s: Reinstating path %s.",
+	       dm_device_name(dm_table_get_md(m->ti->table)),
+	       pgpath->path.dev->name);
 
 	r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
 	if (r)
@@ -1617,7 +1627,8 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone,
 		struct path_selector *ps = &pgpath->pg->ps;
 
 		if (ps->type->end_io)
-			ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
+			ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes,
+					 clone->io_start_time_ns);
 	}
 
 	return r;
@@ -1640,7 +1651,7 @@ static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
 
 	if (atomic_read(&m->nr_valid_paths) == 0 &&
 	    !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
-		if (must_push_back_bio(m)) {
+		if (__must_push_back(m)) {
 			r = DM_ENDIO_REQUEUE;
 		} else {
 			dm_report_EIO(m);
@@ -1661,23 +1672,27 @@ done:
 		struct path_selector *ps = &pgpath->pg->ps;
 
 		if (ps->type->end_io)
-			ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
+			ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes,
+					 dm_start_time_ns_from_clone(clone));
 	}
 
 	return r;
 }
 
 /*
- * Suspend can't complete until all the I/O is processed so if
- * the last path fails we must error any remaining I/O.
- * Note that if the freeze_bdev fails while suspending, the
- * queue_if_no_path state is lost - userspace should reset it.
+ * Suspend with flush can't complete until all the I/O is processed
+ * so if the last path fails we must error any remaining I/O.
+ * - Note that if the freeze_bdev fails while suspending, the
+ *   queue_if_no_path state is lost - userspace should reset it.
+ * Otherwise, during noflush suspend, queue_if_no_path will not change.
  */
 static void multipath_presuspend(struct dm_target *ti)
 {
 	struct multipath *m = ti->private;
 
-	queue_if_no_path(m, false, true);
+	/* FIXME: bio-based shouldn't need to always disable queue_if_no_path */
+	if (m->queue_mode == DM_TYPE_BIO_BASED || !dm_noflush_suspending(m->ti))
+		queue_if_no_path(m, false, true, __func__);
 }
 
 static void multipath_postsuspend(struct dm_target *ti)
@@ -1698,8 +1713,16 @@ static void multipath_resume(struct dm_target *ti)
 	unsigned long flags;
 
 	spin_lock_irqsave(&m->lock, flags);
-	assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags,
-		   test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
+	if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) {
+		set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
+		clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
+	}
+
+	DMDEBUG("%s: %s finished; QIFNP = %d; SQIFNP = %d",
+		dm_device_name(dm_table_get_md(m->ti->table)), __func__,
+		test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags),
+		test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
+
 	spin_unlock_irqrestore(&m->lock, flags);
 }
 
@@ -1859,13 +1882,13 @@ static int multipath_message(struct dm_target *ti, unsigned argc, char **argv,
 
 	if (argc == 1) {
 		if (!strcasecmp(argv[0], "queue_if_no_path")) {
-			r = queue_if_no_path(m, true, false);
+			r = queue_if_no_path(m, true, false, __func__);
 			spin_lock_irqsave(&m->lock, flags);
 			enable_nopath_timeout(m);
 			spin_unlock_irqrestore(&m->lock, flags);
 			goto out;
 		} else if (!strcasecmp(argv[0], "fail_if_no_path")) {
-			r = queue_if_no_path(m, false, false);
+			r = queue_if_no_path(m, false, false, __func__);
 			disable_nopath_timeout(m);
 			goto out;
 		}
@@ -1918,7 +1941,7 @@ static int multipath_prepare_ioctl(struct dm_target *ti,
 	int r;
 
 	current_pgpath = READ_ONCE(m->current_pgpath);
-	if (!current_pgpath)
+	if (!current_pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 		current_pgpath = choose_pgpath(m, 0);
 
 	if (current_pgpath) {
diff --git a/drivers/md/dm-path-selector.h b/drivers/md/dm-path-selector.h
index b6eb5365b1a4..c47bc0e20275 100644
--- a/drivers/md/dm-path-selector.h
+++ b/drivers/md/dm-path-selector.h
@@ -74,7 +74,7 @@ struct path_selector_type {
 	int (*start_io) (struct path_selector *ps, struct dm_path *path,
 			 size_t nr_bytes);
 	int (*end_io) (struct path_selector *ps, struct dm_path *path,
-		       size_t nr_bytes);
+		       size_t nr_bytes, u64 start_time);
 };
 
 /* Register a path selector */
diff --git a/drivers/md/dm-queue-length.c b/drivers/md/dm-queue-length.c
index 969c4f1a3633..5fd018d18418 100644
--- a/drivers/md/dm-queue-length.c
+++ b/drivers/md/dm-queue-length.c
@@ -227,7 +227,7 @@ static int ql_start_io(struct path_selector *ps, struct dm_path *path,
 }
 
 static int ql_end_io(struct path_selector *ps, struct dm_path *path,
-		     size_t nr_bytes)
+		     size_t nr_bytes, u64 start_time)
 {
 	struct path_info *pi = path->pscontext;
 
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 9a18bef0a5ff..10e8b2fe787b 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -254,7 +254,7 @@ struct raid_set {
 		int mode;
 	} journal_dev;
 
-	struct raid_dev dev[0];
+	struct raid_dev dev[];
 };
 
 static void rs_config_backup(struct raid_set *rs, struct rs_layout *l)
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index 089aed57e083..2f655d9f4200 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -83,7 +83,7 @@ struct mirror_set {
 	struct work_struct trigger_event;
 
 	unsigned nr_mirrors;
-	struct mirror mirror[0];
+	struct mirror mirror[];
 };
 
 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(raid1_resync_throttle,
diff --git a/drivers/md/dm-service-time.c b/drivers/md/dm-service-time.c
index f006a9005593..9cfda665e9eb 100644
--- a/drivers/md/dm-service-time.c
+++ b/drivers/md/dm-service-time.c
@@ -309,7 +309,7 @@ static int st_start_io(struct path_selector *ps, struct dm_path *path,
 }
 
 static int st_end_io(struct path_selector *ps, struct dm_path *path,
-		     size_t nr_bytes)
+		     size_t nr_bytes, u64 start_time)
 {
 	struct path_info *pi = path->pscontext;
 
diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c
index 71417048256a..35d368c418d0 100644
--- a/drivers/md/dm-stats.c
+++ b/drivers/md/dm-stats.c
@@ -56,7 +56,7 @@ struct dm_stat {
 	size_t percpu_alloc_size;
 	size_t histogram_alloc_size;
 	struct dm_stat_percpu *stat_percpu[NR_CPUS];
-	struct dm_stat_shared stat_shared[0];
+	struct dm_stat_shared stat_shared[];
 };
 
 #define STAT_PRECISE_TIMESTAMPS		1
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index fa813c0f993d..151d022b032d 100644
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -41,7 +41,7 @@ struct stripe_c {
 	/* Work struct used for triggering events*/
 	struct work_struct trigger_event;
 
-	struct stripe stripe[0];
+	struct stripe stripe[];
 };
 
 /*
diff --git a/drivers/md/dm-switch.c b/drivers/md/dm-switch.c
index 8a0f057b8122..bff4c7fa1cd2 100644
--- a/drivers/md/dm-switch.c
+++ b/drivers/md/dm-switch.c
@@ -53,7 +53,7 @@ struct switch_ctx {
 	/*
 	 * Array of dm devices to switch between.
 	 */
-	struct switch_path path_list[0];
+	struct switch_path path_list[];
 };
 
 static struct switch_ctx *alloc_switch_ctx(struct dm_target *ti, unsigned nr_paths,
diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index 613c171b1b6d..74f3c506f084 100644
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -234,10 +234,6 @@ static int persistent_memory_claim(struct dm_writecache *wc)
 
 	wc->memory_vmapped = false;
 
-	if (!wc->ssd_dev->dax_dev) {
-		r = -EOPNOTSUPP;
-		goto err1;
-	}
 	s = wc->memory_map_size;
 	p = s >> PAGE_SHIFT;
 	if (!p) {
@@ -1143,6 +1139,42 @@ static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
 	return r;
 }
 
+static void memcpy_flushcache_optimized(void *dest, void *source, size_t size)
+{
+	/*
+	 * clflushopt performs better with block size 1024, 2048, 4096
+	 * non-temporal stores perform better with block size 512
+	 *
+	 * block size   512             1024            2048            4096
+	 * movnti       496 MB/s        642 MB/s        725 MB/s        744 MB/s
+	 * clflushopt   373 MB/s        688 MB/s        1.1 GB/s        1.2 GB/s
+	 *
+	 * We see that movnti performs better for 512-byte blocks, and
+	 * clflushopt performs better for 1024-byte and larger blocks. So, we
+	 * prefer clflushopt for sizes >= 768.
+	 *
+	 * NOTE: this happens to be the case now (with dm-writecache's single
+	 * threaded model) but re-evaluate this once memcpy_flushcache() is
+	 * enabled to use movdir64b which might invalidate this performance
+	 * advantage seen with cache-allocating-writes plus flushing.
+	 */
+#ifdef CONFIG_X86
+	if (static_cpu_has(X86_FEATURE_CLFLUSHOPT) &&
+	    likely(boot_cpu_data.x86_clflush_size == 64) &&
+	    likely(size >= 768)) {
+		do {
+			memcpy((void *)dest, (void *)source, 64);
+			clflushopt((void *)dest);
+			dest += 64;
+			source += 64;
+			size -= 64;
+		} while (size >= 64);
+		return;
+	}
+#endif
+	memcpy_flushcache(dest, source, size);
+}
+
 static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
 {
 	void *buf;
@@ -1168,7 +1200,7 @@ static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data
 			}
 		} else {
 			flush_dcache_page(bio_page(bio));
-			memcpy_flushcache(data, buf, size);
+			memcpy_flushcache_optimized(data, buf, size);
 		}
 
 		bvec_kunmap_irq(buf, &flags);
diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c
index bf2245370305..130b5a6d9f12 100644
--- a/drivers/md/dm-zoned-metadata.c
+++ b/drivers/md/dm-zoned-metadata.c
@@ -16,7 +16,7 @@
 /*
  * Metadata version.
  */
-#define DMZ_META_VER	1
+#define DMZ_META_VER	2
 
 /*
  * On-disk super block magic.
@@ -69,8 +69,17 @@ struct dmz_super {
 	/* Checksum */
 	__le32		crc;			/*  48 */
 
+	/* DM-Zoned label */
+	u8		dmz_label[32];		/*  80 */
+
+	/* DM-Zoned UUID */
+	u8		dmz_uuid[16];		/*  96 */
+
+	/* Device UUID */
+	u8		dev_uuid[16];		/* 112 */
+
 	/* Padding to full 512B sector */
-	u8		reserved[464];		/* 512 */
+	u8		reserved[400];		/* 512 */
 };
 
 /*
@@ -122,8 +131,10 @@ enum {
  */
 struct dmz_sb {
 	sector_t		block;
+	struct dmz_dev		*dev;
 	struct dmz_mblock	*mblk;
 	struct dmz_super	*sb;
+	struct dm_zone		*zone;
 };
 
 /*
@@ -131,28 +142,41 @@ struct dmz_sb {
  */
 struct dmz_metadata {
 	struct dmz_dev		*dev;
+	unsigned int		nr_devs;
+
+	char			devname[BDEVNAME_SIZE];
+	char			label[BDEVNAME_SIZE];
+	uuid_t			uuid;
 
 	sector_t		zone_bitmap_size;
 	unsigned int		zone_nr_bitmap_blocks;
 	unsigned int		zone_bits_per_mblk;
 
+	sector_t		zone_nr_blocks;
+	sector_t		zone_nr_blocks_shift;
+
+	sector_t		zone_nr_sectors;
+	sector_t		zone_nr_sectors_shift;
+
 	unsigned int		nr_bitmap_blocks;
 	unsigned int		nr_map_blocks;
 
+	unsigned int		nr_zones;
 	unsigned int		nr_useable_zones;
 	unsigned int		nr_meta_blocks;
 	unsigned int		nr_meta_zones;
 	unsigned int		nr_data_zones;
+	unsigned int		nr_cache_zones;
 	unsigned int		nr_rnd_zones;
 	unsigned int		nr_reserved_seq;
 	unsigned int		nr_chunks;
 
 	/* Zone information array */
-	struct dm_zone		*zones;
+	struct xarray		zones;
 
-	struct dm_zone		*sb_zone;
 	struct dmz_sb		sb[2];
 	unsigned int		mblk_primary;
+	unsigned int		sb_version;
 	u64			sb_gen;
 	unsigned int		min_nr_mblks;
 	unsigned int		max_nr_mblks;
@@ -168,15 +192,11 @@ struct dmz_metadata {
 	/* Zone allocation management */
 	struct mutex		map_lock;
 	struct dmz_mblock	**map_mblk;
-	unsigned int		nr_rnd;
-	atomic_t		unmap_nr_rnd;
-	struct list_head	unmap_rnd_list;
-	struct list_head	map_rnd_list;
 
-	unsigned int		nr_seq;
-	atomic_t		unmap_nr_seq;
-	struct list_head	unmap_seq_list;
-	struct list_head	map_seq_list;
+	unsigned int		nr_cache;
+	atomic_t		unmap_nr_cache;
+	struct list_head	unmap_cache_list;
+	struct list_head	map_cache_list;
 
 	atomic_t		nr_reserved_seq_zones;
 	struct list_head	reserved_seq_zones_list;
@@ -184,22 +204,65 @@ struct dmz_metadata {
 	wait_queue_head_t	free_wq;
 };
 
+#define dmz_zmd_info(zmd, format, args...)	\
+	DMINFO("(%s): " format, (zmd)->label, ## args)
+
+#define dmz_zmd_err(zmd, format, args...)	\
+	DMERR("(%s): " format, (zmd)->label, ## args)
+
+#define dmz_zmd_warn(zmd, format, args...)	\
+	DMWARN("(%s): " format, (zmd)->label, ## args)
+
+#define dmz_zmd_debug(zmd, format, args...)	\
+	DMDEBUG("(%s): " format, (zmd)->label, ## args)
 /*
  * Various accessors
  */
-unsigned int dmz_id(struct dmz_metadata *zmd, struct dm_zone *zone)
+static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)
 {
-	return ((unsigned int)(zone - zmd->zones));
+	if (WARN_ON(!zone))
+		return 0;
+
+	return zone->id - zone->dev->zone_offset;
 }
 
 sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)
 {
-	return (sector_t)dmz_id(zmd, zone) << zmd->dev->zone_nr_sectors_shift;
+	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
+
+	return (sector_t)zone_id << zmd->zone_nr_sectors_shift;
 }
 
 sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)
 {
-	return (sector_t)dmz_id(zmd, zone) << zmd->dev->zone_nr_blocks_shift;
+	unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
+
+	return (sector_t)zone_id << zmd->zone_nr_blocks_shift;
+}
+
+unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)
+{
+	return zmd->zone_nr_blocks;
+}
+
+unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd)
+{
+	return zmd->zone_nr_blocks_shift;
+}
+
+unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)
+{
+	return zmd->zone_nr_sectors;
+}
+
+unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)
+{
+	return zmd->zone_nr_sectors_shift;
+}
+
+unsigned int dmz_nr_zones(struct dmz_metadata *zmd)
+{
+	return zmd->nr_zones;
 }
 
 unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)
@@ -207,14 +270,88 @@ unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)
 	return zmd->nr_chunks;
 }
 
-unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd)
+unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)
 {
-	return zmd->nr_rnd;
+	return zmd->dev[idx].nr_rnd;
 }
 
-unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd)
+unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)
 {
-	return atomic_read(&zmd->unmap_nr_rnd);
+	return atomic_read(&zmd->dev[idx].unmap_nr_rnd);
+}
+
+unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)
+{
+	return zmd->nr_cache;
+}
+
+unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)
+{
+	return atomic_read(&zmd->unmap_nr_cache);
+}
+
+unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)
+{
+	return zmd->dev[idx].nr_seq;
+}
+
+unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)
+{
+	return atomic_read(&zmd->dev[idx].unmap_nr_seq);
+}
+
+static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)
+{
+	return xa_load(&zmd->zones, zone_id);
+}
+
+static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,
+				  unsigned int zone_id, struct dmz_dev *dev)
+{
+	struct dm_zone *zone = kzalloc(sizeof(struct dm_zone), GFP_KERNEL);
+
+	if (!zone)
+		return ERR_PTR(-ENOMEM);
+
+	if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {
+		kfree(zone);
+		return ERR_PTR(-EBUSY);
+	}
+
+	INIT_LIST_HEAD(&zone->link);
+	atomic_set(&zone->refcount, 0);
+	zone->id = zone_id;
+	zone->chunk = DMZ_MAP_UNMAPPED;
+	zone->dev = dev;
+
+	return zone;
+}
+
+const char *dmz_metadata_label(struct dmz_metadata *zmd)
+{
+	return (const char *)zmd->label;
+}
+
+bool dmz_check_dev(struct dmz_metadata *zmd)
+{
+	unsigned int i;
+
+	for (i = 0; i < zmd->nr_devs; i++) {
+		if (!dmz_check_bdev(&zmd->dev[i]))
+			return false;
+	}
+	return true;
+}
+
+bool dmz_dev_is_dying(struct dmz_metadata *zmd)
+{
+	unsigned int i;
+
+	for (i = 0; i < zmd->nr_devs; i++) {
+		if (dmz_bdev_is_dying(&zmd->dev[i]))
+			return true;
+	}
+	return false;
 }
 
 /*
@@ -402,9 +539,10 @@ static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,
 {
 	struct dmz_mblock *mblk, *m;
 	sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;
+	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
 	struct bio *bio;
 
-	if (dmz_bdev_is_dying(zmd->dev))
+	if (dmz_bdev_is_dying(dev))
 		return ERR_PTR(-EIO);
 
 	/* Get a new block and a BIO to read it */
@@ -440,7 +578,7 @@ static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,
 
 	/* Submit read BIO */
 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
-	bio_set_dev(bio, zmd->dev->bdev);
+	bio_set_dev(bio, dev->bdev);
 	bio->bi_private = mblk;
 	bio->bi_end_io = dmz_mblock_bio_end_io;
 	bio_set_op_attrs(bio, REQ_OP_READ, REQ_META | REQ_PRIO);
@@ -537,6 +675,7 @@ static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,
 					 sector_t mblk_no)
 {
 	struct dmz_mblock *mblk;
+	struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
 
 	/* Check rbtree */
 	spin_lock(&zmd->mblk_lock);
@@ -555,7 +694,7 @@ static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,
 		       TASK_UNINTERRUPTIBLE);
 	if (test_bit(DMZ_META_ERROR, &mblk->state)) {
 		dmz_release_mblock(zmd, mblk);
-		dmz_check_bdev(zmd->dev);
+		dmz_check_bdev(dev);
 		return ERR_PTR(-EIO);
 	}
 
@@ -579,10 +718,11 @@ static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
 static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
 			    unsigned int set)
 {
+	struct dmz_dev *dev = zmd->sb[set].dev;
 	sector_t block = zmd->sb[set].block + mblk->no;
 	struct bio *bio;
 
-	if (dmz_bdev_is_dying(zmd->dev))
+	if (dmz_bdev_is_dying(dev))
 		return -EIO;
 
 	bio = bio_alloc(GFP_NOIO, 1);
@@ -594,7 +734,7 @@ static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
 	set_bit(DMZ_META_WRITING, &mblk->state);
 
 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
-	bio_set_dev(bio, zmd->dev->bdev);
+	bio_set_dev(bio, dev->bdev);
 	bio->bi_private = mblk;
 	bio->bi_end_io = dmz_mblock_bio_end_io;
 	bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_META | REQ_PRIO);
@@ -607,13 +747,16 @@ static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
 /*
  * Read/write a metadata block.
  */
-static int dmz_rdwr_block(struct dmz_metadata *zmd, int op, sector_t block,
-			  struct page *page)
+static int dmz_rdwr_block(struct dmz_dev *dev, int op,
+			  sector_t block, struct page *page)
 {
 	struct bio *bio;
 	int ret;
 
-	if (dmz_bdev_is_dying(zmd->dev))
+	if (WARN_ON(!dev))
+		return -EIO;
+
+	if (dmz_bdev_is_dying(dev))
 		return -EIO;
 
 	bio = bio_alloc(GFP_NOIO, 1);
@@ -621,14 +764,14 @@ static int dmz_rdwr_block(struct dmz_metadata *zmd, int op, sector_t block,
 		return -ENOMEM;
 
 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
-	bio_set_dev(bio, zmd->dev->bdev);
+	bio_set_dev(bio, dev->bdev);
 	bio_set_op_attrs(bio, op, REQ_SYNC | REQ_META | REQ_PRIO);
 	bio_add_page(bio, page, DMZ_BLOCK_SIZE, 0);
 	ret = submit_bio_wait(bio);
 	bio_put(bio);
 
 	if (ret)
-		dmz_check_bdev(zmd->dev);
+		dmz_check_bdev(dev);
 	return ret;
 }
 
@@ -637,18 +780,32 @@ static int dmz_rdwr_block(struct dmz_metadata *zmd, int op, sector_t block,
  */
 static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)
 {
-	sector_t block = zmd->sb[set].block;
 	struct dmz_mblock *mblk = zmd->sb[set].mblk;
 	struct dmz_super *sb = zmd->sb[set].sb;
+	struct dmz_dev *dev = zmd->sb[set].dev;
+	sector_t sb_block;
 	u64 sb_gen = zmd->sb_gen + 1;
 	int ret;
 
 	sb->magic = cpu_to_le32(DMZ_MAGIC);
-	sb->version = cpu_to_le32(DMZ_META_VER);
+
+	sb->version = cpu_to_le32(zmd->sb_version);
+	if (zmd->sb_version > 1) {
+		BUILD_BUG_ON(UUID_SIZE != 16);
+		export_uuid(sb->dmz_uuid, &zmd->uuid);
+		memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);
+		export_uuid(sb->dev_uuid, &dev->uuid);
+	}
 
 	sb->gen = cpu_to_le64(sb_gen);
 
-	sb->sb_block = cpu_to_le64(block);
+	/*
+	 * The metadata always references the absolute block address,
+	 * ie relative to the entire block range, not the per-device
+	 * block address.
+	 */
+	sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;
+	sb->sb_block = cpu_to_le64(sb_block);
 	sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);
 	sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);
 	sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);
@@ -659,9 +816,10 @@ static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)
 	sb->crc = 0;
 	sb->crc = cpu_to_le32(crc32_le(sb_gen, (unsigned char *)sb, DMZ_BLOCK_SIZE));
 
-	ret = dmz_rdwr_block(zmd, REQ_OP_WRITE, block, mblk->page);
+	ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,
+			     mblk->page);
 	if (ret == 0)
-		ret = blkdev_issue_flush(zmd->dev->bdev, GFP_NOIO);
+		ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
 
 	return ret;
 }
@@ -674,6 +832,7 @@ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
 				   unsigned int set)
 {
 	struct dmz_mblock *mblk;
+	struct dmz_dev *dev = zmd->sb[set].dev;
 	struct blk_plug plug;
 	int ret = 0, nr_mblks_submitted = 0;
 
@@ -695,7 +854,7 @@ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
 			       TASK_UNINTERRUPTIBLE);
 		if (test_bit(DMZ_META_ERROR, &mblk->state)) {
 			clear_bit(DMZ_META_ERROR, &mblk->state);
-			dmz_check_bdev(zmd->dev);
+			dmz_check_bdev(dev);
 			ret = -EIO;
 		}
 		nr_mblks_submitted--;
@@ -703,7 +862,7 @@ static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
 
 	/* Flush drive cache (this will also sync data) */
 	if (ret == 0)
-		ret = blkdev_issue_flush(zmd->dev->bdev, GFP_NOIO);
+		ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
 
 	return ret;
 }
@@ -740,6 +899,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd)
 {
 	struct dmz_mblock *mblk;
 	struct list_head write_list;
+	struct dmz_dev *dev;
 	int ret;
 
 	if (WARN_ON(!zmd))
@@ -753,6 +913,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd)
 	 * from modifying metadata.
 	 */
 	down_write(&zmd->mblk_sem);
+	dev = zmd->sb[zmd->mblk_primary].dev;
 
 	/*
 	 * This is called from the target flush work and reclaim work.
@@ -760,7 +921,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd)
 	 */
 	dmz_lock_flush(zmd);
 
-	if (dmz_bdev_is_dying(zmd->dev)) {
+	if (dmz_bdev_is_dying(dev)) {
 		ret = -EIO;
 		goto out;
 	}
@@ -772,7 +933,7 @@ int dmz_flush_metadata(struct dmz_metadata *zmd)
 
 	/* If there are no dirty metadata blocks, just flush the device cache */
 	if (list_empty(&write_list)) {
-		ret = blkdev_issue_flush(zmd->dev->bdev, GFP_NOIO);
+		ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
 		goto err;
 	}
 
@@ -821,7 +982,7 @@ err:
 		list_splice(&write_list, &zmd->mblk_dirty_list);
 		spin_unlock(&zmd->mblk_lock);
 	}
-	if (!dmz_check_bdev(zmd->dev))
+	if (!dmz_check_bdev(dev))
 		ret = -EIO;
 	goto out;
 }
@@ -829,12 +990,31 @@ err:
 /*
  * Check super block.
  */
-static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_super *sb)
+static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,
+			bool tertiary)
 {
+	struct dmz_super *sb = dsb->sb;
+	struct dmz_dev *dev = dsb->dev;
 	unsigned int nr_meta_zones, nr_data_zones;
-	struct dmz_dev *dev = zmd->dev;
 	u32 crc, stored_crc;
-	u64 gen;
+	u64 gen, sb_block;
+
+	if (le32_to_cpu(sb->magic) != DMZ_MAGIC) {
+		dmz_dev_err(dev, "Invalid meta magic (needed 0x%08x, got 0x%08x)",
+			    DMZ_MAGIC, le32_to_cpu(sb->magic));
+		return -ENXIO;
+	}
+
+	zmd->sb_version = le32_to_cpu(sb->version);
+	if (zmd->sb_version > DMZ_META_VER) {
+		dmz_dev_err(dev, "Invalid meta version (needed %d, got %d)",
+			    DMZ_META_VER, zmd->sb_version);
+		return -EINVAL;
+	}
+	if (zmd->sb_version < 2 && tertiary) {
+		dmz_dev_err(dev, "Tertiary superblocks are not supported");
+		return -EINVAL;
+	}
 
 	gen = le64_to_cpu(sb->gen);
 	stored_crc = le32_to_cpu(sb->crc);
@@ -846,20 +1026,57 @@ static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_super *sb)
 		return -ENXIO;
 	}
 
-	if (le32_to_cpu(sb->magic) != DMZ_MAGIC) {
-		dmz_dev_err(dev, "Invalid meta magic (needed 0x%08x, got 0x%08x)",
-			    DMZ_MAGIC, le32_to_cpu(sb->magic));
-		return -ENXIO;
-	}
+	sb_block = le64_to_cpu(sb->sb_block);
+	if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift ) {
+		dmz_dev_err(dev, "Invalid superblock position "
+			    "(is %llu expected %llu)",
+			    sb_block,
+			    (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);
+		return -EINVAL;
+	}
+	if (zmd->sb_version > 1) {
+		uuid_t sb_uuid;
+
+		import_uuid(&sb_uuid, sb->dmz_uuid);
+		if (uuid_is_null(&sb_uuid)) {
+			dmz_dev_err(dev, "NULL DM-Zoned uuid");
+			return -ENXIO;
+		} else if (uuid_is_null(&zmd->uuid)) {
+			uuid_copy(&zmd->uuid, &sb_uuid);
+		} else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {
+			dmz_dev_err(dev, "mismatching DM-Zoned uuid, "
+				    "is %pUl expected %pUl",
+				    &sb_uuid, &zmd->uuid);
+			return -ENXIO;
+		}
+		if (!strlen(zmd->label))
+			memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);
+		else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {
+			dmz_dev_err(dev, "mismatching DM-Zoned label, "
+				    "is %s expected %s",
+				    sb->dmz_label, zmd->label);
+			return -ENXIO;
+		}
+		import_uuid(&dev->uuid, sb->dev_uuid);
+		if (uuid_is_null(&dev->uuid)) {
+			dmz_dev_err(dev, "NULL device uuid");
+			return -ENXIO;
+		}
 
-	if (le32_to_cpu(sb->version) != DMZ_META_VER) {
-		dmz_dev_err(dev, "Invalid meta version (needed %d, got %d)",
-			    DMZ_META_VER, le32_to_cpu(sb->version));
-		return -ENXIO;
+		if (tertiary) {
+			/*
+			 * Generation number should be 0, but it doesn't
+			 * really matter if it isn't.
+			 */
+			if (gen != 0)
+				dmz_dev_warn(dev, "Invalid generation %llu",
+					    gen);
+			return 0;
+		}
 	}
 
-	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + dev->zone_nr_blocks - 1)
-		>> dev->zone_nr_blocks_shift;
+	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)
+		>> zmd->zone_nr_blocks_shift;
 	if (!nr_meta_zones ||
 	    nr_meta_zones >= zmd->nr_rnd_zones) {
 		dmz_dev_err(dev, "Invalid number of metadata blocks");
@@ -895,10 +1112,13 @@ static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_super *sb)
 /*
  * Read the first or second super block from disk.
  */
-static int dmz_read_sb(struct dmz_metadata *zmd, unsigned int set)
+static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
 {
-	return dmz_rdwr_block(zmd, REQ_OP_READ, zmd->sb[set].block,
-			      zmd->sb[set].mblk->page);
+	dmz_zmd_debug(zmd, "read superblock set %d dev %s block %llu",
+		      set, sb->dev->name, sb->block);
+
+	return dmz_rdwr_block(sb->dev, REQ_OP_READ,
+			      sb->block, sb->mblk->page);
 }
 
 /*
@@ -908,8 +1128,9 @@ static int dmz_read_sb(struct dmz_metadata *zmd, unsigned int set)
  */
 static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)
 {
-	unsigned int zone_nr_blocks = zmd->dev->zone_nr_blocks;
+	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
 	struct dmz_mblock *mblk;
+	unsigned int zone_id = zmd->sb[0].zone->id;
 	int i;
 
 	/* Allocate a block */
@@ -922,24 +1143,29 @@ static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)
 
 	/* Bad first super block: search for the second one */
 	zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;
-	for (i = 0; i < zmd->nr_rnd_zones - 1; i++) {
-		if (dmz_read_sb(zmd, 1) != 0)
+	zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);
+	zmd->sb[1].dev = zmd->sb[0].dev;
+	for (i = 1; i < zmd->nr_rnd_zones; i++) {
+		if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)
 			break;
 		if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)
 			return 0;
 		zmd->sb[1].block += zone_nr_blocks;
+		zmd->sb[1].zone = dmz_get(zmd, zone_id + i);
 	}
 
 	dmz_free_mblock(zmd, mblk);
 	zmd->sb[1].mblk = NULL;
+	zmd->sb[1].zone = NULL;
+	zmd->sb[1].dev = NULL;
 
 	return -EIO;
 }
 
 /*
- * Read the first or second super block from disk.
+ * Read a super block from disk.
  */
-static int dmz_get_sb(struct dmz_metadata *zmd, unsigned int set)
+static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
 {
 	struct dmz_mblock *mblk;
 	int ret;
@@ -949,14 +1175,14 @@ static int dmz_get_sb(struct dmz_metadata *zmd, unsigned int set)
 	if (!mblk)
 		return -ENOMEM;
 
-	zmd->sb[set].mblk = mblk;
-	zmd->sb[set].sb = mblk->data;
+	sb->mblk = mblk;
+	sb->sb = mblk->data;
 
 	/* Read super block */
-	ret = dmz_read_sb(zmd, set);
+	ret = dmz_read_sb(zmd, sb, set);
 	if (ret) {
 		dmz_free_mblock(zmd, mblk);
-		zmd->sb[set].mblk = NULL;
+		sb->mblk = NULL;
 		return ret;
 	}
 
@@ -972,14 +1198,13 @@ static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)
 	struct page *page;
 	int i, ret;
 
-	dmz_dev_warn(zmd->dev, "Metadata set %u invalid: recovering", dst_set);
+	dmz_dev_warn(zmd->sb[dst_set].dev,
+		     "Metadata set %u invalid: recovering", dst_set);
 
 	if (dst_set == 0)
-		zmd->sb[0].block = dmz_start_block(zmd, zmd->sb_zone);
-	else {
-		zmd->sb[1].block = zmd->sb[0].block +
-			(zmd->nr_meta_zones << zmd->dev->zone_nr_blocks_shift);
-	}
+		zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
+	else
+		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
 
 	page = alloc_page(GFP_NOIO);
 	if (!page)
@@ -987,11 +1212,11 @@ static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)
 
 	/* Copy metadata blocks */
 	for (i = 1; i < zmd->nr_meta_blocks; i++) {
-		ret = dmz_rdwr_block(zmd, REQ_OP_READ,
+		ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,
 				     zmd->sb[src_set].block + i, page);
 		if (ret)
 			goto out;
-		ret = dmz_rdwr_block(zmd, REQ_OP_WRITE,
+		ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,
 				     zmd->sb[dst_set].block + i, page);
 		if (ret)
 			goto out;
@@ -1023,53 +1248,73 @@ static int dmz_load_sb(struct dmz_metadata *zmd)
 	u64 sb_gen[2] = {0, 0};
 	int ret;
 
+	if (!zmd->sb[0].zone) {
+		dmz_zmd_err(zmd, "Primary super block zone not set");
+		return -ENXIO;
+	}
+
 	/* Read and check the primary super block */
-	zmd->sb[0].block = dmz_start_block(zmd, zmd->sb_zone);
-	ret = dmz_get_sb(zmd, 0);
+	zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
+	zmd->sb[0].dev = zmd->sb[0].zone->dev;
+	ret = dmz_get_sb(zmd, &zmd->sb[0], 0);
 	if (ret) {
-		dmz_dev_err(zmd->dev, "Read primary super block failed");
+		dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");
 		return ret;
 	}
 
-	ret = dmz_check_sb(zmd, zmd->sb[0].sb);
+	ret = dmz_check_sb(zmd, &zmd->sb[0], false);
 
 	/* Read and check secondary super block */
 	if (ret == 0) {
 		sb_good[0] = true;
-		zmd->sb[1].block = zmd->sb[0].block +
-			(zmd->nr_meta_zones << zmd->dev->zone_nr_blocks_shift);
-		ret = dmz_get_sb(zmd, 1);
+		if (!zmd->sb[1].zone) {
+			unsigned int zone_id =
+				zmd->sb[0].zone->id + zmd->nr_meta_zones;
+
+			zmd->sb[1].zone = dmz_get(zmd, zone_id);
+		}
+		zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
+		zmd->sb[1].dev = zmd->sb[0].dev;
+		ret = dmz_get_sb(zmd, &zmd->sb[1], 1);
 	} else
 		ret = dmz_lookup_secondary_sb(zmd);
 
 	if (ret) {
-		dmz_dev_err(zmd->dev, "Read secondary super block failed");
+		dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");
 		return ret;
 	}
 
-	ret = dmz_check_sb(zmd, zmd->sb[1].sb);
+	ret = dmz_check_sb(zmd, &zmd->sb[1], false);
 	if (ret == 0)
 		sb_good[1] = true;
 
 	/* Use highest generation sb first */
 	if (!sb_good[0] && !sb_good[1]) {
-		dmz_dev_err(zmd->dev, "No valid super block found");
+		dmz_zmd_err(zmd, "No valid super block found");
 		return -EIO;
 	}
 
 	if (sb_good[0])
 		sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);
-	else
+	else {
 		ret = dmz_recover_mblocks(zmd, 0);
+		if (ret) {
+			dmz_dev_err(zmd->sb[0].dev,
+				    "Recovery of superblock 0 failed");
+			return -EIO;
+		}
+	}
 
 	if (sb_good[1])
 		sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);
-	else
+	else {
 		ret = dmz_recover_mblocks(zmd, 1);
 
-	if (ret) {
-		dmz_dev_err(zmd->dev, "Recovery failed");
-		return -EIO;
+		if (ret) {
+			dmz_dev_err(zmd->sb[1].dev,
+				    "Recovery of superblock 1 failed");
+			return -EIO;
+		}
 	}
 
 	if (sb_gen[0] >= sb_gen[1]) {
@@ -1080,32 +1325,70 @@ static int dmz_load_sb(struct dmz_metadata *zmd)
 		zmd->mblk_primary = 1;
 	}
 
-	dmz_dev_debug(zmd->dev, "Using super block %u (gen %llu)",
+	dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,
+		      "Using super block %u (gen %llu)",
 		      zmd->mblk_primary, zmd->sb_gen);
 
-	return 0;
+	if (zmd->sb_version > 1) {
+		int i;
+		struct dmz_sb *sb;
+
+		sb = kzalloc(sizeof(struct dmz_sb), GFP_KERNEL);
+		if (!sb)
+			return -ENOMEM;
+		for (i = 1; i < zmd->nr_devs; i++) {
+			sb->block = 0;
+			sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);
+			sb->dev = &zmd->dev[i];
+			if (!dmz_is_meta(sb->zone)) {
+				dmz_dev_err(sb->dev,
+					    "Tertiary super block zone %u not marked as metadata zone",
+					    sb->zone->id);
+				ret = -EINVAL;
+				goto out_kfree;
+			}
+			ret = dmz_get_sb(zmd, sb, i + 1);
+			if (ret) {
+				dmz_dev_err(sb->dev,
+					    "Read tertiary super block failed");
+				dmz_free_mblock(zmd, sb->mblk);
+				goto out_kfree;
+			}
+			ret = dmz_check_sb(zmd, sb, true);
+			dmz_free_mblock(zmd, sb->mblk);
+			if (ret == -EINVAL)
+				goto out_kfree;
+		}
+	out_kfree:
+		kfree(sb);
+	}
+	return ret;
 }
 
 /*
  * Initialize a zone descriptor.
  */
-static int dmz_init_zone(struct blk_zone *blkz, unsigned int idx, void *data)
+static int dmz_init_zone(struct blk_zone *blkz, unsigned int num, void *data)
 {
-	struct dmz_metadata *zmd = data;
-	struct dm_zone *zone = &zmd->zones[idx];
-	struct dmz_dev *dev = zmd->dev;
+	struct dmz_dev *dev = data;
+	struct dmz_metadata *zmd = dev->metadata;
+	int idx = num + dev->zone_offset;
+	struct dm_zone *zone;
+
+	zone = dmz_insert(zmd, idx, dev);
+	if (IS_ERR(zone))
+		return PTR_ERR(zone);
 
-	/* Ignore the eventual last runt (smaller) zone */
-	if (blkz->len != dev->zone_nr_sectors) {
-		if (blkz->start + blkz->len == dev->capacity)
+	if (blkz->len != zmd->zone_nr_sectors) {
+		if (zmd->sb_version > 1) {
+			/* Ignore the eventual runt (smaller) zone */
+			set_bit(DMZ_OFFLINE, &zone->flags);
+			return 0;
+		} else if (blkz->start + blkz->len == dev->capacity)
 			return 0;
 		return -ENXIO;
 	}
 
-	INIT_LIST_HEAD(&zone->link);
-	atomic_set(&zone->refcount, 0);
-	zone->chunk = DMZ_MAP_UNMAPPED;
-
 	switch (blkz->type) {
 	case BLK_ZONE_TYPE_CONVENTIONAL:
 		set_bit(DMZ_RND, &zone->flags);
@@ -1131,13 +1414,45 @@ static int dmz_init_zone(struct blk_zone *blkz, unsigned int idx, void *data)
 		zmd->nr_useable_zones++;
 		if (dmz_is_rnd(zone)) {
 			zmd->nr_rnd_zones++;
-			if (!zmd->sb_zone) {
-				/* Super block zone */
-				zmd->sb_zone = zone;
+			if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {
+				/* Primary super block zone */
+				zmd->sb[0].zone = zone;
 			}
 		}
+		if (zmd->nr_devs > 1 && num == 0) {
+			/*
+			 * Tertiary superblock zones are always at the
+			 * start of the zoned devices, so mark them
+			 * as metadata zone.
+			 */
+			set_bit(DMZ_META, &zone->flags);
+		}
 	}
+	return 0;
+}
+
+static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)
+{
+	int idx;
+	sector_t zone_offset = 0;
 
+	for(idx = 0; idx < dev->nr_zones; idx++) {
+		struct dm_zone *zone;
+
+		zone = dmz_insert(zmd, idx, dev);
+		if (IS_ERR(zone))
+			return PTR_ERR(zone);
+		set_bit(DMZ_CACHE, &zone->flags);
+		zone->wp_block = 0;
+		zmd->nr_cache_zones++;
+		zmd->nr_useable_zones++;
+		if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {
+			/* Disable runt zone */
+			set_bit(DMZ_OFFLINE, &zone->flags);
+			break;
+		}
+		zone_offset += zmd->zone_nr_sectors;
+	}
 	return 0;
 }
 
@@ -1146,8 +1461,15 @@ static int dmz_init_zone(struct blk_zone *blkz, unsigned int idx, void *data)
  */
 static void dmz_drop_zones(struct dmz_metadata *zmd)
 {
-	kfree(zmd->zones);
-	zmd->zones = NULL;
+	int idx;
+
+	for(idx = 0; idx < zmd->nr_zones; idx++) {
+		struct dm_zone *zone = xa_load(&zmd->zones, idx);
+
+		kfree(zone);
+		xa_erase(&zmd->zones, idx);
+	}
+	xa_destroy(&zmd->zones);
 }
 
 /*
@@ -1156,32 +1478,87 @@ static void dmz_drop_zones(struct dmz_metadata *zmd)
  */
 static int dmz_init_zones(struct dmz_metadata *zmd)
 {
-	struct dmz_dev *dev = zmd->dev;
-	int ret;
+	int i, ret;
+	struct dmz_dev *zoned_dev = &zmd->dev[0];
 
 	/* Init */
-	zmd->zone_bitmap_size = dev->zone_nr_blocks >> 3;
+	zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;
+	zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);
+	zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);
+	zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);
+	zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;
 	zmd->zone_nr_bitmap_blocks =
 		max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);
-	zmd->zone_bits_per_mblk = min_t(sector_t, dev->zone_nr_blocks,
+	zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,
 					DMZ_BLOCK_SIZE_BITS);
 
 	/* Allocate zone array */
-	zmd->zones = kcalloc(dev->nr_zones, sizeof(struct dm_zone), GFP_KERNEL);
-	if (!zmd->zones)
-		return -ENOMEM;
+	zmd->nr_zones = 0;
+	for (i = 0; i < zmd->nr_devs; i++) {
+		struct dmz_dev *dev = &zmd->dev[i];
+
+		dev->metadata = zmd;
+		zmd->nr_zones += dev->nr_zones;
+
+		atomic_set(&dev->unmap_nr_rnd, 0);
+		INIT_LIST_HEAD(&dev->unmap_rnd_list);
+		INIT_LIST_HEAD(&dev->map_rnd_list);
+
+		atomic_set(&dev->unmap_nr_seq, 0);
+		INIT_LIST_HEAD(&dev->unmap_seq_list);
+		INIT_LIST_HEAD(&dev->map_seq_list);
+	}
+
+	if (!zmd->nr_zones) {
+		DMERR("(%s): No zones found", zmd->devname);
+		return -ENXIO;
+	}
+	xa_init(&zmd->zones);
+
+	DMDEBUG("(%s): Using %zu B for zone information",
+		zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);
 
-	dmz_dev_info(dev, "Using %zu B for zone information",
-		     sizeof(struct dm_zone) * dev->nr_zones);
+	if (zmd->nr_devs > 1) {
+		ret = dmz_emulate_zones(zmd, &zmd->dev[0]);
+		if (ret < 0) {
+			DMDEBUG("(%s): Failed to emulate zones, error %d",
+				zmd->devname, ret);
+			dmz_drop_zones(zmd);
+			return ret;
+		}
+
+		/*
+		 * Primary superblock zone is always at zone 0 when multiple
+		 * drives are present.
+		 */
+		zmd->sb[0].zone = dmz_get(zmd, 0);
+
+		for (i = 1; i < zmd->nr_devs; i++) {
+			zoned_dev = &zmd->dev[i];
+
+			ret = blkdev_report_zones(zoned_dev->bdev, 0,
+						  BLK_ALL_ZONES,
+						  dmz_init_zone, zoned_dev);
+			if (ret < 0) {
+				DMDEBUG("(%s): Failed to report zones, error %d",
+					zmd->devname, ret);
+				dmz_drop_zones(zmd);
+				return ret;
+			}
+		}
+		return 0;
+	}
 
 	/*
 	 * Get zone information and initialize zone descriptors.  At the same
 	 * time, determine where the super block should be: first block of the
 	 * first randomly writable zone.
 	 */
-	ret = blkdev_report_zones(dev->bdev, 0, BLK_ALL_ZONES, dmz_init_zone,
-				  zmd);
+	ret = blkdev_report_zones(zoned_dev->bdev, 0, BLK_ALL_ZONES,
+				  dmz_init_zone, zoned_dev);
 	if (ret < 0) {
+		DMDEBUG("(%s): Failed to report zones, error %d",
+			zmd->devname, ret);
 		dmz_drop_zones(zmd);
 		return ret;
 	}
@@ -1213,9 +1590,13 @@ static int dmz_update_zone_cb(struct blk_zone *blkz, unsigned int idx,
  */
 static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
 {
+	struct dmz_dev *dev = zone->dev;
 	unsigned int noio_flag;
 	int ret;
 
+	if (dev->flags & DMZ_BDEV_REGULAR)
+		return 0;
+
 	/*
 	 * Get zone information from disk. Since blkdev_report_zones() uses
 	 * GFP_KERNEL by default for memory allocations, set the per-task
@@ -1223,16 +1604,16 @@ static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
 	 * GFP_NOIO was specified.
 	 */
 	noio_flag = memalloc_noio_save();
-	ret = blkdev_report_zones(zmd->dev->bdev, dmz_start_sect(zmd, zone), 1,
+	ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,
 				  dmz_update_zone_cb, zone);
 	memalloc_noio_restore(noio_flag);
 
 	if (ret == 0)
 		ret = -EIO;
 	if (ret < 0) {
-		dmz_dev_err(zmd->dev, "Get zone %u report failed",
-			    dmz_id(zmd, zone));
-		dmz_check_bdev(zmd->dev);
+		dmz_dev_err(dev, "Get zone %u report failed",
+			    zone->id);
+		dmz_check_bdev(dev);
 		return ret;
 	}
 
@@ -1246,6 +1627,7 @@ static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
 static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
 				    struct dm_zone *zone)
 {
+	struct dmz_dev *dev = zone->dev;
 	unsigned int wp = 0;
 	int ret;
 
@@ -1254,8 +1636,8 @@ static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
 	if (ret)
 		return ret;
 
-	dmz_dev_warn(zmd->dev, "Processing zone %u write error (zone wp %u/%u)",
-		     dmz_id(zmd, zone), zone->wp_block, wp);
+	dmz_dev_warn(dev, "Processing zone %u write error (zone wp %u/%u)",
+		     zone->id, zone->wp_block, wp);
 
 	if (zone->wp_block < wp) {
 		dmz_invalidate_blocks(zmd, zone, zone->wp_block,
@@ -1265,11 +1647,6 @@ static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
 	return 0;
 }
 
-static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)
-{
-	return &zmd->zones[zone_id];
-}
-
 /*
  * Reset a zone write pointer.
  */
@@ -1287,14 +1664,14 @@ static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
 		return 0;
 
 	if (!dmz_is_empty(zone) || dmz_seq_write_err(zone)) {
-		struct dmz_dev *dev = zmd->dev;
+		struct dmz_dev *dev = zone->dev;
 
 		ret = blkdev_zone_mgmt(dev->bdev, REQ_OP_ZONE_RESET,
 				       dmz_start_sect(zmd, zone),
-				       dev->zone_nr_sectors, GFP_NOIO);
+				       zmd->zone_nr_sectors, GFP_NOIO);
 		if (ret) {
 			dmz_dev_err(dev, "Reset zone %u failed %d",
-				    dmz_id(zmd, zone), ret);
+				    zone->id, ret);
 			return ret;
 		}
 	}
@@ -1313,7 +1690,6 @@ static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
  */
 static int dmz_load_mapping(struct dmz_metadata *zmd)
 {
-	struct dmz_dev *dev = zmd->dev;
 	struct dm_zone *dzone, *bzone;
 	struct dmz_mblock *dmap_mblk = NULL;
 	struct dmz_map *dmap;
@@ -1345,36 +1721,48 @@ static int dmz_load_mapping(struct dmz_metadata *zmd)
 		if (dzone_id == DMZ_MAP_UNMAPPED)
 			goto next;
 
-		if (dzone_id >= dev->nr_zones) {
-			dmz_dev_err(dev, "Chunk %u mapping: invalid data zone ID %u",
+		if (dzone_id >= zmd->nr_zones) {
+			dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",
 				    chunk, dzone_id);
 			return -EIO;
 		}
 
 		dzone = dmz_get(zmd, dzone_id);
+		if (!dzone) {
+			dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",
+				    chunk, dzone_id);
+			return -EIO;
+		}
 		set_bit(DMZ_DATA, &dzone->flags);
 		dzone->chunk = chunk;
 		dmz_get_zone_weight(zmd, dzone);
 
-		if (dmz_is_rnd(dzone))
-			list_add_tail(&dzone->link, &zmd->map_rnd_list);
+		if (dmz_is_cache(dzone))
+			list_add_tail(&dzone->link, &zmd->map_cache_list);
+		else if (dmz_is_rnd(dzone))
+			list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
 		else
-			list_add_tail(&dzone->link, &zmd->map_seq_list);
+			list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
 
 		/* Check buffer zone */
 		bzone_id = le32_to_cpu(dmap[e].bzone_id);
 		if (bzone_id == DMZ_MAP_UNMAPPED)
 			goto next;
 
-		if (bzone_id >= dev->nr_zones) {
-			dmz_dev_err(dev, "Chunk %u mapping: invalid buffer zone ID %u",
+		if (bzone_id >= zmd->nr_zones) {
+			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",
 				    chunk, bzone_id);
 			return -EIO;
 		}
 
 		bzone = dmz_get(zmd, bzone_id);
-		if (!dmz_is_rnd(bzone)) {
-			dmz_dev_err(dev, "Chunk %u mapping: invalid buffer zone %u",
+		if (!bzone) {
+			dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",
+				    chunk, bzone_id);
+			return -EIO;
+		}
+		if (!dmz_is_rnd(bzone) && !dmz_is_cache(bzone)) {
+			dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",
 				    chunk, bzone_id);
 			return -EIO;
 		}
@@ -1385,7 +1773,10 @@ static int dmz_load_mapping(struct dmz_metadata *zmd)
 		bzone->bzone = dzone;
 		dzone->bzone = bzone;
 		dmz_get_zone_weight(zmd, bzone);
-		list_add_tail(&bzone->link, &zmd->map_rnd_list);
+		if (dmz_is_cache(bzone))
+			list_add_tail(&bzone->link, &zmd->map_cache_list);
+		else
+			list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
 next:
 		chunk++;
 		e++;
@@ -1398,15 +1789,21 @@ next:
 	 * fully initialized. All remaining zones are unmapped data
 	 * zones. Finish initializing those here.
 	 */
-	for (i = 0; i < dev->nr_zones; i++) {
+	for (i = 0; i < zmd->nr_zones; i++) {
 		dzone = dmz_get(zmd, i);
+		if (!dzone)
+			continue;
 		if (dmz_is_meta(dzone))
 			continue;
+		if (dmz_is_offline(dzone))
+			continue;
 
-		if (dmz_is_rnd(dzone))
-			zmd->nr_rnd++;
+		if (dmz_is_cache(dzone))
+			zmd->nr_cache++;
+		else if (dmz_is_rnd(dzone))
+			dzone->dev->nr_rnd++;
 		else
-			zmd->nr_seq++;
+			dzone->dev->nr_seq++;
 
 		if (dmz_is_data(dzone)) {
 			/* Already initialized */
@@ -1416,16 +1813,22 @@ next:
 		/* Unmapped data zone */
 		set_bit(DMZ_DATA, &dzone->flags);
 		dzone->chunk = DMZ_MAP_UNMAPPED;
-		if (dmz_is_rnd(dzone)) {
-			list_add_tail(&dzone->link, &zmd->unmap_rnd_list);
-			atomic_inc(&zmd->unmap_nr_rnd);
+		if (dmz_is_cache(dzone)) {
+			list_add_tail(&dzone->link, &zmd->unmap_cache_list);
+			atomic_inc(&zmd->unmap_nr_cache);
+		} else if (dmz_is_rnd(dzone)) {
+			list_add_tail(&dzone->link,
+				      &dzone->dev->unmap_rnd_list);
+			atomic_inc(&dzone->dev->unmap_nr_rnd);
 		} else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {
 			list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);
+			set_bit(DMZ_RESERVED, &dzone->flags);
 			atomic_inc(&zmd->nr_reserved_seq_zones);
-			zmd->nr_seq--;
+			dzone->dev->nr_seq--;
 		} else {
-			list_add_tail(&dzone->link, &zmd->unmap_seq_list);
-			atomic_inc(&zmd->unmap_nr_seq);
+			list_add_tail(&dzone->link,
+				      &dzone->dev->unmap_seq_list);
+			atomic_inc(&dzone->dev->unmap_nr_seq);
 		}
 	}
 
@@ -1459,10 +1862,13 @@ static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
 	list_del_init(&zone->link);
 	if (dmz_is_seq(zone)) {
 		/* LRU rotate sequential zone */
-		list_add_tail(&zone->link, &zmd->map_seq_list);
+		list_add_tail(&zone->link, &zone->dev->map_seq_list);
+	} else if (dmz_is_cache(zone)) {
+		/* LRU rotate cache zone */
+		list_add_tail(&zone->link, &zmd->map_cache_list);
 	} else {
 		/* LRU rotate random zone */
-		list_add_tail(&zone->link, &zmd->map_rnd_list);
+		list_add_tail(&zone->link, &zone->dev->map_rnd_list);
 	}
 }
 
@@ -1529,58 +1935,76 @@ static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)
 {
 	dmz_unlock_map(zmd);
 	dmz_unlock_metadata(zmd);
+	set_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
 	wait_on_bit_timeout(&zone->flags, DMZ_RECLAIM, TASK_UNINTERRUPTIBLE, HZ);
+	clear_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
 	dmz_lock_metadata(zmd);
 	dmz_lock_map(zmd);
 }
 
 /*
- * Select a random write zone for reclaim.
+ * Select a cache or random write zone for reclaim.
  */
-static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd)
+static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,
+						    unsigned int idx, bool idle)
 {
 	struct dm_zone *dzone = NULL;
-	struct dm_zone *zone;
-
-	if (list_empty(&zmd->map_rnd_list))
-		return ERR_PTR(-EBUSY);
+	struct dm_zone *zone, *last = NULL;
+	struct list_head *zone_list;
+
+	/* If we have cache zones select from the cache zone list */
+	if (zmd->nr_cache) {
+		zone_list = &zmd->map_cache_list;
+		/* Try to relaim random zones, too, when idle */
+		if (idle && list_empty(zone_list))
+			zone_list = &zmd->dev[idx].map_rnd_list;
+	} else
+		zone_list = &zmd->dev[idx].map_rnd_list;
 
-	list_for_each_entry(zone, &zmd->map_rnd_list, link) {
-		if (dmz_is_buf(zone))
+	list_for_each_entry(zone, zone_list, link) {
+		if (dmz_is_buf(zone)) {
 			dzone = zone->bzone;
-		else
+			if (dzone->dev->dev_idx != idx)
+				continue;
+			if (!last) {
+				last = dzone;
+				continue;
+			}
+			if (last->weight < dzone->weight)
+				continue;
+			dzone = last;
+		} else
 			dzone = zone;
 		if (dmz_lock_zone_reclaim(dzone))
 			return dzone;
 	}
 
-	return ERR_PTR(-EBUSY);
+	return NULL;
 }
 
 /*
  * Select a buffered sequential zone for reclaim.
  */
-static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd)
+static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,
+						    unsigned int idx)
 {
 	struct dm_zone *zone;
 
-	if (list_empty(&zmd->map_seq_list))
-		return ERR_PTR(-EBUSY);
-
-	list_for_each_entry(zone, &zmd->map_seq_list, link) {
+	list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {
 		if (!zone->bzone)
 			continue;
 		if (dmz_lock_zone_reclaim(zone))
 			return zone;
 	}
 
-	return ERR_PTR(-EBUSY);
+	return NULL;
 }
 
 /*
  * Select a zone for reclaim.
  */
-struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd)
+struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,
+					 unsigned int dev_idx, bool idle)
 {
 	struct dm_zone *zone;
 
@@ -1594,9 +2018,9 @@ struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd)
 	 */
 	dmz_lock_map(zmd);
 	if (list_empty(&zmd->reserved_seq_zones_list))
-		zone = dmz_get_seq_zone_for_reclaim(zmd);
+		zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);
 	else
-		zone = dmz_get_rnd_zone_for_reclaim(zmd);
+		zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);
 	dmz_unlock_map(zmd);
 
 	return zone;
@@ -1616,6 +2040,7 @@ struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chu
 	unsigned int dzone_id;
 	struct dm_zone *dzone = NULL;
 	int ret = 0;
+	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
 
 	dmz_lock_map(zmd);
 again:
@@ -1630,9 +2055,9 @@ again:
 			goto out;
 
 		/* Allocate a random zone */
-		dzone = dmz_alloc_zone(zmd, DMZ_ALLOC_RND);
+		dzone = dmz_alloc_zone(zmd, 0, alloc_flags);
 		if (!dzone) {
-			if (dmz_bdev_is_dying(zmd->dev)) {
+			if (dmz_dev_is_dying(zmd)) {
 				dzone = ERR_PTR(-EIO);
 				goto out;
 			}
@@ -1645,6 +2070,10 @@ again:
 	} else {
 		/* The chunk is already mapped: get the mapping zone */
 		dzone = dmz_get(zmd, dzone_id);
+		if (!dzone) {
+			dzone = ERR_PTR(-EIO);
+			goto out;
+		}
 		if (dzone->chunk != chunk) {
 			dzone = ERR_PTR(-EIO);
 			goto out;
@@ -1723,6 +2152,7 @@ struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,
 				     struct dm_zone *dzone)
 {
 	struct dm_zone *bzone;
+	int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
 
 	dmz_lock_map(zmd);
 again:
@@ -1731,9 +2161,9 @@ again:
 		goto out;
 
 	/* Allocate a random zone */
-	bzone = dmz_alloc_zone(zmd, DMZ_ALLOC_RND);
+	bzone = dmz_alloc_zone(zmd, 0, alloc_flags);
 	if (!bzone) {
-		if (dmz_bdev_is_dying(zmd->dev)) {
+		if (dmz_dev_is_dying(zmd)) {
 			bzone = ERR_PTR(-EIO);
 			goto out;
 		}
@@ -1742,14 +2172,16 @@ again:
 	}
 
 	/* Update the chunk mapping */
-	dmz_set_chunk_mapping(zmd, dzone->chunk, dmz_id(zmd, dzone),
-			      dmz_id(zmd, bzone));
+	dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);
 
 	set_bit(DMZ_BUF, &bzone->flags);
 	bzone->chunk = dzone->chunk;
 	bzone->bzone = dzone;
 	dzone->bzone = bzone;
-	list_add_tail(&bzone->link, &zmd->map_rnd_list);
+	if (dmz_is_cache(bzone))
+		list_add_tail(&bzone->link, &zmd->map_cache_list);
+	else
+		list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
 out:
 	dmz_unlock_map(zmd);
 
@@ -1760,46 +2192,68 @@ out:
  * Get an unmapped (free) zone.
  * This must be called with the mapping lock held.
  */
-struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned long flags)
+struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,
+			       unsigned long flags)
 {
 	struct list_head *list;
 	struct dm_zone *zone;
+	int i = 0;
 
-	if (flags & DMZ_ALLOC_RND)
-		list = &zmd->unmap_rnd_list;
-	else
-		list = &zmd->unmap_seq_list;
 again:
+	if (flags & DMZ_ALLOC_CACHE)
+		list = &zmd->unmap_cache_list;
+	else if (flags & DMZ_ALLOC_RND)
+		list = &zmd->dev[dev_idx].unmap_rnd_list;
+	else
+		list = &zmd->dev[dev_idx].unmap_seq_list;
+
 	if (list_empty(list)) {
 		/*
-		 * No free zone: if this is for reclaim, allow using the
-		 * reserved sequential zones.
+		 * No free zone: return NULL if this is for not reclaim.
 		 */
-		if (!(flags & DMZ_ALLOC_RECLAIM) ||
-		    list_empty(&zmd->reserved_seq_zones_list))
+		if (!(flags & DMZ_ALLOC_RECLAIM))
 			return NULL;
+		/*
+		 * Try to allocate from other devices
+		 */
+		if (i < zmd->nr_devs) {
+			dev_idx = (dev_idx + 1) % zmd->nr_devs;
+			i++;
+			goto again;
+		}
 
-		zone = list_first_entry(&zmd->reserved_seq_zones_list,
-					struct dm_zone, link);
-		list_del_init(&zone->link);
-		atomic_dec(&zmd->nr_reserved_seq_zones);
+		/*
+		 * Fallback to the reserved sequential zones
+		 */
+		zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,
+						struct dm_zone, link);
+		if (zone) {
+			list_del_init(&zone->link);
+			atomic_dec(&zmd->nr_reserved_seq_zones);
+		}
 		return zone;
 	}
 
 	zone = list_first_entry(list, struct dm_zone, link);
 	list_del_init(&zone->link);
 
-	if (dmz_is_rnd(zone))
-		atomic_dec(&zmd->unmap_nr_rnd);
+	if (dmz_is_cache(zone))
+		atomic_dec(&zmd->unmap_nr_cache);
+	else if (dmz_is_rnd(zone))
+		atomic_dec(&zone->dev->unmap_nr_rnd);
 	else
-		atomic_dec(&zmd->unmap_nr_seq);
+		atomic_dec(&zone->dev->unmap_nr_seq);
 
 	if (dmz_is_offline(zone)) {
-		dmz_dev_warn(zmd->dev, "Zone %u is offline", dmz_id(zmd, zone));
+		dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);
+		zone = NULL;
+		goto again;
+	}
+	if (dmz_is_meta(zone)) {
+		dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);
 		zone = NULL;
 		goto again;
 	}
-
 	return zone;
 }
 
@@ -1814,16 +2268,18 @@ void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
 		dmz_reset_zone(zmd, zone);
 
 	/* Return the zone to its type unmap list */
-	if (dmz_is_rnd(zone)) {
-		list_add_tail(&zone->link, &zmd->unmap_rnd_list);
-		atomic_inc(&zmd->unmap_nr_rnd);
-	} else if (atomic_read(&zmd->nr_reserved_seq_zones) <
-		   zmd->nr_reserved_seq) {
+	if (dmz_is_cache(zone)) {
+		list_add_tail(&zone->link, &zmd->unmap_cache_list);
+		atomic_inc(&zmd->unmap_nr_cache);
+	} else if (dmz_is_rnd(zone)) {
+		list_add_tail(&zone->link, &zone->dev->unmap_rnd_list);
+		atomic_inc(&zone->dev->unmap_nr_rnd);
+	} else if (dmz_is_reserved(zone)) {
 		list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);
 		atomic_inc(&zmd->nr_reserved_seq_zones);
 	} else {
-		list_add_tail(&zone->link, &zmd->unmap_seq_list);
-		atomic_inc(&zmd->unmap_nr_seq);
+		list_add_tail(&zone->link, &zone->dev->unmap_seq_list);
+		atomic_inc(&zone->dev->unmap_nr_seq);
 	}
 
 	wake_up_all(&zmd->free_wq);
@@ -1837,13 +2293,15 @@ void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,
 		  unsigned int chunk)
 {
 	/* Set the chunk mapping */
-	dmz_set_chunk_mapping(zmd, chunk, dmz_id(zmd, dzone),
+	dmz_set_chunk_mapping(zmd, chunk, dzone->id,
 			      DMZ_MAP_UNMAPPED);
 	dzone->chunk = chunk;
-	if (dmz_is_rnd(dzone))
-		list_add_tail(&dzone->link, &zmd->map_rnd_list);
+	if (dmz_is_cache(dzone))
+		list_add_tail(&dzone->link, &zmd->map_cache_list);
+	else if (dmz_is_rnd(dzone))
+		list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
 	else
-		list_add_tail(&dzone->link, &zmd->map_seq_list);
+		list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
 }
 
 /*
@@ -1865,7 +2323,7 @@ void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
 		 * Unmapping the chunk buffer zone: clear only
 		 * the chunk buffer mapping
 		 */
-		dzone_id = dmz_id(zmd, zone->bzone);
+		dzone_id = zone->bzone->id;
 		zone->bzone->bzone = NULL;
 		zone->bzone = NULL;
 
@@ -1927,7 +2385,7 @@ static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,
 					 sector_t chunk_block)
 {
 	sector_t bitmap_block = 1 + zmd->nr_map_blocks +
-		(sector_t)(dmz_id(zmd, zone) * zmd->zone_nr_bitmap_blocks) +
+		(sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +
 		(chunk_block >> DMZ_BLOCK_SHIFT_BITS);
 
 	return dmz_get_mblock(zmd, bitmap_block);
@@ -1943,7 +2401,7 @@ int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
 	sector_t chunk_block = 0;
 
 	/* Get the zones bitmap blocks */
-	while (chunk_block < zmd->dev->zone_nr_blocks) {
+	while (chunk_block < zmd->zone_nr_blocks) {
 		from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);
 		if (IS_ERR(from_mblk))
 			return PTR_ERR(from_mblk);
@@ -1978,7 +2436,7 @@ int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
 	int ret;
 
 	/* Get the zones bitmap blocks */
-	while (chunk_block < zmd->dev->zone_nr_blocks) {
+	while (chunk_block < zmd->zone_nr_blocks) {
 		/* Get a valid region from the source zone */
 		ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);
 		if (ret <= 0)
@@ -2002,12 +2460,12 @@ int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
 			sector_t chunk_block, unsigned int nr_blocks)
 {
 	unsigned int count, bit, nr_bits;
-	unsigned int zone_nr_blocks = zmd->dev->zone_nr_blocks;
+	unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
 	struct dmz_mblock *mblk;
 	unsigned int n = 0;
 
-	dmz_dev_debug(zmd->dev, "=> VALIDATE zone %u, block %llu, %u blocks",
-		      dmz_id(zmd, zone), (unsigned long long)chunk_block,
+	dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",
+		      zone->id, (unsigned long long)chunk_block,
 		      nr_blocks);
 
 	WARN_ON(chunk_block + nr_blocks > zone_nr_blocks);
@@ -2036,8 +2494,8 @@ int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
 	if (likely(zone->weight + n <= zone_nr_blocks))
 		zone->weight += n;
 	else {
-		dmz_dev_warn(zmd->dev, "Zone %u: weight %u should be <= %u",
-			     dmz_id(zmd, zone), zone->weight,
+		dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",
+			     zone->id, zone->weight,
 			     zone_nr_blocks - n);
 		zone->weight = zone_nr_blocks;
 	}
@@ -2086,10 +2544,10 @@ int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
 	struct dmz_mblock *mblk;
 	unsigned int n = 0;
 
-	dmz_dev_debug(zmd->dev, "=> INVALIDATE zone %u, block %llu, %u blocks",
-		      dmz_id(zmd, zone), (u64)chunk_block, nr_blocks);
+	dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",
+		      zone->id, (u64)chunk_block, nr_blocks);
 
-	WARN_ON(chunk_block + nr_blocks > zmd->dev->zone_nr_blocks);
+	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
 
 	while (nr_blocks) {
 		/* Get bitmap block */
@@ -2116,8 +2574,8 @@ int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
 	if (zone->weight >= n)
 		zone->weight -= n;
 	else {
-		dmz_dev_warn(zmd->dev, "Zone %u: weight %u should be >= %u",
-			     dmz_id(zmd, zone), zone->weight, n);
+		dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",
+			     zone->id, zone->weight, n);
 		zone->weight = 0;
 	}
 
@@ -2133,7 +2591,7 @@ static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,
 	struct dmz_mblock *mblk;
 	int ret;
 
-	WARN_ON(chunk_block >= zmd->dev->zone_nr_blocks);
+	WARN_ON(chunk_block >= zmd->zone_nr_blocks);
 
 	/* Get bitmap block */
 	mblk = dmz_get_bitmap(zmd, zone, chunk_block);
@@ -2163,7 +2621,7 @@ static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,
 	unsigned long *bitmap;
 	int n = 0;
 
-	WARN_ON(chunk_block + nr_blocks > zmd->dev->zone_nr_blocks);
+	WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
 
 	while (nr_blocks) {
 		/* Get bitmap block */
@@ -2207,7 +2665,7 @@ int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,
 
 	/* The block is valid: get the number of valid blocks from block */
 	return dmz_to_next_set_block(zmd, zone, chunk_block,
-				     zmd->dev->zone_nr_blocks - chunk_block, 0);
+				     zmd->zone_nr_blocks - chunk_block, 0);
 }
 
 /*
@@ -2223,7 +2681,7 @@ int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,
 	int ret;
 
 	ret = dmz_to_next_set_block(zmd, zone, start_block,
-				    zmd->dev->zone_nr_blocks - start_block, 1);
+				    zmd->zone_nr_blocks - start_block, 1);
 	if (ret < 0)
 		return ret;
 
@@ -2231,7 +2689,7 @@ int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,
 	*chunk_block = start_block;
 
 	return dmz_to_next_set_block(zmd, zone, start_block,
-				     zmd->dev->zone_nr_blocks - start_block, 0);
+				     zmd->zone_nr_blocks - start_block, 0);
 }
 
 /*
@@ -2270,7 +2728,7 @@ static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)
 	struct dmz_mblock *mblk;
 	sector_t chunk_block = 0;
 	unsigned int bit, nr_bits;
-	unsigned int nr_blocks = zmd->dev->zone_nr_blocks;
+	unsigned int nr_blocks = zmd->zone_nr_blocks;
 	void *bitmap;
 	int n = 0;
 
@@ -2326,7 +2784,7 @@ static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
 	while (!list_empty(&zmd->mblk_dirty_list)) {
 		mblk = list_first_entry(&zmd->mblk_dirty_list,
 					struct dmz_mblock, link);
-		dmz_dev_warn(zmd->dev, "mblock %llu still in dirty list (ref %u)",
+		dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",
 			     (u64)mblk->no, mblk->ref);
 		list_del_init(&mblk->link);
 		rb_erase(&mblk->node, &zmd->mblk_rbtree);
@@ -2344,7 +2802,7 @@ static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
 	/* Sanity checks: the mblock rbtree should now be empty */
 	root = &zmd->mblk_rbtree;
 	rbtree_postorder_for_each_entry_safe(mblk, next, root, node) {
-		dmz_dev_warn(zmd->dev, "mblock %llu ref %u still in rbtree",
+		dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",
 			     (u64)mblk->no, mblk->ref);
 		mblk->ref = 0;
 		dmz_free_mblock(zmd, mblk);
@@ -2357,13 +2815,42 @@ static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
 	mutex_destroy(&zmd->map_lock);
 }
 
+static void dmz_print_dev(struct dmz_metadata *zmd, int num)
+{
+	struct dmz_dev *dev = &zmd->dev[num];
+
+	if (bdev_zoned_model(dev->bdev) == BLK_ZONED_NONE)
+		dmz_dev_info(dev, "Regular block device");
+	else
+		dmz_dev_info(dev, "Host-%s zoned block device",
+			     bdev_zoned_model(dev->bdev) == BLK_ZONED_HA ?
+			     "aware" : "managed");
+	if (zmd->sb_version > 1) {
+		sector_t sector_offset =
+			dev->zone_offset << zmd->zone_nr_sectors_shift;
+
+		dmz_dev_info(dev, "  %llu 512-byte logical sectors (offset %llu)",
+			     (u64)dev->capacity, (u64)sector_offset);
+		dmz_dev_info(dev, "  %u zones of %llu 512-byte logical sectors (offset %llu)",
+			     dev->nr_zones, (u64)zmd->zone_nr_sectors,
+			     (u64)dev->zone_offset);
+	} else {
+		dmz_dev_info(dev, "  %llu 512-byte logical sectors",
+			     (u64)dev->capacity);
+		dmz_dev_info(dev, "  %u zones of %llu 512-byte logical sectors",
+			     dev->nr_zones, (u64)zmd->zone_nr_sectors);
+	}
+}
+
 /*
  * Initialize the zoned metadata.
  */
-int dmz_ctr_metadata(struct dmz_dev *dev, struct dmz_metadata **metadata)
+int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev,
+		     struct dmz_metadata **metadata,
+		     const char *devname)
 {
 	struct dmz_metadata *zmd;
-	unsigned int i, zid;
+	unsigned int i;
 	struct dm_zone *zone;
 	int ret;
 
@@ -2371,7 +2858,9 @@ int dmz_ctr_metadata(struct dmz_dev *dev, struct dmz_metadata **metadata)
 	if (!zmd)
 		return -ENOMEM;
 
+	strcpy(zmd->devname, devname);
 	zmd->dev = dev;
+	zmd->nr_devs = num_dev;
 	zmd->mblk_rbtree = RB_ROOT;
 	init_rwsem(&zmd->mblk_sem);
 	mutex_init(&zmd->mblk_flush_lock);
@@ -2380,13 +2869,10 @@ int dmz_ctr_metadata(struct dmz_dev *dev, struct dmz_metadata **metadata)
 	INIT_LIST_HEAD(&zmd->mblk_dirty_list);
 
 	mutex_init(&zmd->map_lock);
-	atomic_set(&zmd->unmap_nr_rnd, 0);
-	INIT_LIST_HEAD(&zmd->unmap_rnd_list);
-	INIT_LIST_HEAD(&zmd->map_rnd_list);
 
-	atomic_set(&zmd->unmap_nr_seq, 0);
-	INIT_LIST_HEAD(&zmd->unmap_seq_list);
-	INIT_LIST_HEAD(&zmd->map_seq_list);
+	atomic_set(&zmd->unmap_nr_cache, 0);
+	INIT_LIST_HEAD(&zmd->unmap_cache_list);
+	INIT_LIST_HEAD(&zmd->map_cache_list);
 
 	atomic_set(&zmd->nr_reserved_seq_zones, 0);
 	INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);
@@ -2404,14 +2890,22 @@ int dmz_ctr_metadata(struct dmz_dev *dev, struct dmz_metadata **metadata)
 		goto err;
 
 	/* Set metadata zones starting from sb_zone */
-	zid = dmz_id(zmd, zmd->sb_zone);
 	for (i = 0; i < zmd->nr_meta_zones << 1; i++) {
-		zone = dmz_get(zmd, zid + i);
-		if (!dmz_is_rnd(zone))
+		zone = dmz_get(zmd, zmd->sb[0].zone->id + i);
+		if (!zone) {
+			dmz_zmd_err(zmd,
+				    "metadata zone %u not present", i);
+			ret = -ENXIO;
+			goto err;
+		}
+		if (!dmz_is_rnd(zone) && !dmz_is_cache(zone)) {
+			dmz_zmd_err(zmd,
+				    "metadata zone %d is not random", i);
+			ret = -ENXIO;
 			goto err;
+		}
 		set_bit(DMZ_META, &zone->flags);
 	}
-
 	/* Load mapping table */
 	ret = dmz_load_mapping(zmd);
 	if (ret)
@@ -2432,34 +2926,38 @@ int dmz_ctr_metadata(struct dmz_dev *dev, struct dmz_metadata **metadata)
 	/* Metadata cache shrinker */
 	ret = register_shrinker(&zmd->mblk_shrinker);
 	if (ret) {
-		dmz_dev_err(dev, "Register metadata cache shrinker failed");
+		dmz_zmd_err(zmd, "Register metadata cache shrinker failed");
 		goto err;
 	}
 
-	dmz_dev_info(dev, "Host-%s zoned block device",
-		     bdev_zoned_model(dev->bdev) == BLK_ZONED_HA ?
-		     "aware" : "managed");
-	dmz_dev_info(dev, "  %llu 512-byte logical sectors",
-		     (u64)dev->capacity);
-	dmz_dev_info(dev, "  %u zones of %llu 512-byte logical sectors",
-		     dev->nr_zones, (u64)dev->zone_nr_sectors);
-	dmz_dev_info(dev, "  %u metadata zones",
-		     zmd->nr_meta_zones * 2);
-	dmz_dev_info(dev, "  %u data zones for %u chunks",
-		     zmd->nr_data_zones, zmd->nr_chunks);
-	dmz_dev_info(dev, "    %u random zones (%u unmapped)",
-		     zmd->nr_rnd, atomic_read(&zmd->unmap_nr_rnd));
-	dmz_dev_info(dev, "    %u sequential zones (%u unmapped)",
-		     zmd->nr_seq, atomic_read(&zmd->unmap_nr_seq));
-	dmz_dev_info(dev, "  %u reserved sequential data zones",
-		     zmd->nr_reserved_seq);
-
-	dmz_dev_debug(dev, "Format:");
-	dmz_dev_debug(dev, "%u metadata blocks per set (%u max cache)",
+	dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);
+	for (i = 0; i < zmd->nr_devs; i++)
+		dmz_print_dev(zmd, i);
+
+	dmz_zmd_info(zmd, "  %u zones of %llu 512-byte logical sectors",
+		     zmd->nr_zones, (u64)zmd->zone_nr_sectors);
+	dmz_zmd_debug(zmd, "  %u metadata zones",
+		      zmd->nr_meta_zones * 2);
+	dmz_zmd_debug(zmd, "  %u data zones for %u chunks",
+		      zmd->nr_data_zones, zmd->nr_chunks);
+	dmz_zmd_debug(zmd, "    %u cache zones (%u unmapped)",
+		      zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));
+	for (i = 0; i < zmd->nr_devs; i++) {
+		dmz_zmd_debug(zmd, "    %u random zones (%u unmapped)",
+			      dmz_nr_rnd_zones(zmd, i),
+			      dmz_nr_unmap_rnd_zones(zmd, i));
+		dmz_zmd_debug(zmd, "    %u sequential zones (%u unmapped)",
+			      dmz_nr_seq_zones(zmd, i),
+			      dmz_nr_unmap_seq_zones(zmd, i));
+	}
+	dmz_zmd_debug(zmd, "  %u reserved sequential data zones",
+		      zmd->nr_reserved_seq);
+	dmz_zmd_debug(zmd, "Format:");
+	dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",
 		      zmd->nr_meta_blocks, zmd->max_nr_mblks);
-	dmz_dev_debug(dev, "  %u data zone mapping blocks",
+	dmz_zmd_debug(zmd, "  %u data zone mapping blocks",
 		      zmd->nr_map_blocks);
-	dmz_dev_debug(dev, "  %u bitmap blocks",
+	dmz_zmd_debug(zmd, "  %u bitmap blocks",
 		      zmd->nr_bitmap_blocks);
 
 	*metadata = zmd;
@@ -2488,30 +2986,28 @@ void dmz_dtr_metadata(struct dmz_metadata *zmd)
  */
 int dmz_resume_metadata(struct dmz_metadata *zmd)
 {
-	struct dmz_dev *dev = zmd->dev;
 	struct dm_zone *zone;
 	sector_t wp_block;
 	unsigned int i;
 	int ret;
 
 	/* Check zones */
-	for (i = 0; i < dev->nr_zones; i++) {
+	for (i = 0; i < zmd->nr_zones; i++) {
 		zone = dmz_get(zmd, i);
 		if (!zone) {
-			dmz_dev_err(dev, "Unable to get zone %u", i);
+			dmz_zmd_err(zmd, "Unable to get zone %u", i);
 			return -EIO;
 		}
-
 		wp_block = zone->wp_block;
 
 		ret = dmz_update_zone(zmd, zone);
 		if (ret) {
-			dmz_dev_err(dev, "Broken zone %u", i);
+			dmz_zmd_err(zmd, "Broken zone %u", i);
 			return ret;
 		}
 
 		if (dmz_is_offline(zone)) {
-			dmz_dev_warn(dev, "Zone %u is offline", i);
+			dmz_zmd_warn(zmd, "Zone %u is offline", i);
 			continue;
 		}
 
@@ -2519,11 +3015,11 @@ int dmz_resume_metadata(struct dmz_metadata *zmd)
 		if (!dmz_is_seq(zone))
 			zone->wp_block = 0;
 		else if (zone->wp_block != wp_block) {
-			dmz_dev_err(dev, "Zone %u: Invalid wp (%llu / %llu)",
+			dmz_zmd_err(zmd, "Zone %u: Invalid wp (%llu / %llu)",
 				    i, (u64)zone->wp_block, (u64)wp_block);
 			zone->wp_block = wp_block;
 			dmz_invalidate_blocks(zmd, zone, zone->wp_block,
-					      dev->zone_nr_blocks - zone->wp_block);
+					      zmd->zone_nr_blocks - zone->wp_block);
 		}
 	}
 
diff --git a/drivers/md/dm-zoned-reclaim.c b/drivers/md/dm-zoned-reclaim.c
index e7ace908a9b7..2261b4dd60b7 100644
--- a/drivers/md/dm-zoned-reclaim.c
+++ b/drivers/md/dm-zoned-reclaim.c
@@ -13,7 +13,6 @@
 
 struct dmz_reclaim {
 	struct dmz_metadata     *metadata;
-	struct dmz_dev		*dev;
 
 	struct delayed_work	work;
 	struct workqueue_struct *wq;
@@ -22,6 +21,8 @@ struct dmz_reclaim {
 	struct dm_kcopyd_throttle kc_throttle;
 	int			kc_err;
 
+	int			dev_idx;
+
 	unsigned long		flags;
 
 	/* Last target access time */
@@ -44,13 +45,13 @@ enum {
  * Percentage of unmapped (free) random zones below which reclaim starts
  * even if the target is busy.
  */
-#define DMZ_RECLAIM_LOW_UNMAP_RND	30
+#define DMZ_RECLAIM_LOW_UNMAP_ZONES	30
 
 /*
  * Percentage of unmapped (free) random zones above which reclaim will
  * stop if the target is busy.
  */
-#define DMZ_RECLAIM_HIGH_UNMAP_RND	50
+#define DMZ_RECLAIM_HIGH_UNMAP_ZONES	50
 
 /*
  * Align a sequential zone write pointer to chunk_block.
@@ -59,6 +60,7 @@ static int dmz_reclaim_align_wp(struct dmz_reclaim *zrc, struct dm_zone *zone,
 				sector_t block)
 {
 	struct dmz_metadata *zmd = zrc->metadata;
+	struct dmz_dev *dev = zone->dev;
 	sector_t wp_block = zone->wp_block;
 	unsigned int nr_blocks;
 	int ret;
@@ -74,15 +76,15 @@ static int dmz_reclaim_align_wp(struct dmz_reclaim *zrc, struct dm_zone *zone,
 	 * pointer and the requested position.
 	 */
 	nr_blocks = block - wp_block;
-	ret = blkdev_issue_zeroout(zrc->dev->bdev,
+	ret = blkdev_issue_zeroout(dev->bdev,
 				   dmz_start_sect(zmd, zone) + dmz_blk2sect(wp_block),
 				   dmz_blk2sect(nr_blocks), GFP_NOIO, 0);
 	if (ret) {
-		dmz_dev_err(zrc->dev,
+		dmz_dev_err(dev,
 			    "Align zone %u wp %llu to %llu (wp+%u) blocks failed %d",
-			    dmz_id(zmd, zone), (unsigned long long)wp_block,
+			    zone->id, (unsigned long long)wp_block,
 			    (unsigned long long)block, nr_blocks, ret);
-		dmz_check_bdev(zrc->dev);
+		dmz_check_bdev(dev);
 		return ret;
 	}
 
@@ -116,7 +118,6 @@ static int dmz_reclaim_copy(struct dmz_reclaim *zrc,
 			    struct dm_zone *src_zone, struct dm_zone *dst_zone)
 {
 	struct dmz_metadata *zmd = zrc->metadata;
-	struct dmz_dev *dev = zrc->dev;
 	struct dm_io_region src, dst;
 	sector_t block = 0, end_block;
 	sector_t nr_blocks;
@@ -128,7 +129,7 @@ static int dmz_reclaim_copy(struct dmz_reclaim *zrc,
 	if (dmz_is_seq(src_zone))
 		end_block = src_zone->wp_block;
 	else
-		end_block = dev->zone_nr_blocks;
+		end_block = dmz_zone_nr_blocks(zmd);
 	src_zone_block = dmz_start_block(zmd, src_zone);
 	dst_zone_block = dmz_start_block(zmd, dst_zone);
 
@@ -136,9 +137,14 @@ static int dmz_reclaim_copy(struct dmz_reclaim *zrc,
 		set_bit(DM_KCOPYD_WRITE_SEQ, &flags);
 
 	while (block < end_block) {
-		if (dev->flags & DMZ_BDEV_DYING)
+		if (src_zone->dev->flags & DMZ_BDEV_DYING)
+			return -EIO;
+		if (dst_zone->dev->flags & DMZ_BDEV_DYING)
 			return -EIO;
 
+		if (dmz_reclaim_should_terminate(src_zone))
+			return -EINTR;
+
 		/* Get a valid region from the source zone */
 		ret = dmz_first_valid_block(zmd, src_zone, &block);
 		if (ret <= 0)
@@ -156,11 +162,11 @@ static int dmz_reclaim_copy(struct dmz_reclaim *zrc,
 				return ret;
 		}
 
-		src.bdev = dev->bdev;
+		src.bdev = src_zone->dev->bdev;
 		src.sector = dmz_blk2sect(src_zone_block + block);
 		src.count = dmz_blk2sect(nr_blocks);
 
-		dst.bdev = dev->bdev;
+		dst.bdev = dst_zone->dev->bdev;
 		dst.sector = dmz_blk2sect(dst_zone_block + block);
 		dst.count = src.count;
 
@@ -194,10 +200,10 @@ static int dmz_reclaim_buf(struct dmz_reclaim *zrc, struct dm_zone *dzone)
 	struct dmz_metadata *zmd = zrc->metadata;
 	int ret;
 
-	dmz_dev_debug(zrc->dev,
-		      "Chunk %u, move buf zone %u (weight %u) to data zone %u (weight %u)",
-		      dzone->chunk, dmz_id(zmd, bzone), dmz_weight(bzone),
-		      dmz_id(zmd, dzone), dmz_weight(dzone));
+	DMDEBUG("(%s/%u): Chunk %u, move buf zone %u (weight %u) to data zone %u (weight %u)",
+		dmz_metadata_label(zmd), zrc->dev_idx,
+		dzone->chunk, bzone->id, dmz_weight(bzone),
+		dzone->id, dmz_weight(dzone));
 
 	/* Flush data zone into the buffer zone */
 	ret = dmz_reclaim_copy(zrc, bzone, dzone);
@@ -210,7 +216,7 @@ static int dmz_reclaim_buf(struct dmz_reclaim *zrc, struct dm_zone *dzone)
 	ret = dmz_merge_valid_blocks(zmd, bzone, dzone, chunk_block);
 	if (ret == 0) {
 		/* Free the buffer zone */
-		dmz_invalidate_blocks(zmd, bzone, 0, zrc->dev->zone_nr_blocks);
+		dmz_invalidate_blocks(zmd, bzone, 0, dmz_zone_nr_blocks(zmd));
 		dmz_lock_map(zmd);
 		dmz_unmap_zone(zmd, bzone);
 		dmz_unlock_zone_reclaim(dzone);
@@ -233,10 +239,10 @@ static int dmz_reclaim_seq_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)
 	struct dmz_metadata *zmd = zrc->metadata;
 	int ret = 0;
 
-	dmz_dev_debug(zrc->dev,
-		      "Chunk %u, move data zone %u (weight %u) to buf zone %u (weight %u)",
-		      chunk, dmz_id(zmd, dzone), dmz_weight(dzone),
-		      dmz_id(zmd, bzone), dmz_weight(bzone));
+	DMDEBUG("(%s/%u): Chunk %u, move data zone %u (weight %u) to buf zone %u (weight %u)",
+		dmz_metadata_label(zmd), zrc->dev_idx,
+		chunk, dzone->id, dmz_weight(dzone),
+		bzone->id, dmz_weight(bzone));
 
 	/* Flush data zone into the buffer zone */
 	ret = dmz_reclaim_copy(zrc, dzone, bzone);
@@ -252,7 +258,7 @@ static int dmz_reclaim_seq_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)
 		 * Free the data zone and remap the chunk to
 		 * the buffer zone.
 		 */
-		dmz_invalidate_blocks(zmd, dzone, 0, zrc->dev->zone_nr_blocks);
+		dmz_invalidate_blocks(zmd, dzone, 0, dmz_zone_nr_blocks(zmd));
 		dmz_lock_map(zmd);
 		dmz_unmap_zone(zmd, bzone);
 		dmz_unmap_zone(zmd, dzone);
@@ -277,18 +283,26 @@ static int dmz_reclaim_rnd_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)
 	struct dm_zone *szone = NULL;
 	struct dmz_metadata *zmd = zrc->metadata;
 	int ret;
+	int alloc_flags = DMZ_ALLOC_SEQ;
 
-	/* Get a free sequential zone */
+	/* Get a free random or sequential zone */
 	dmz_lock_map(zmd);
-	szone = dmz_alloc_zone(zmd, DMZ_ALLOC_RECLAIM);
+again:
+	szone = dmz_alloc_zone(zmd, zrc->dev_idx,
+			       alloc_flags | DMZ_ALLOC_RECLAIM);
+	if (!szone && alloc_flags == DMZ_ALLOC_SEQ && dmz_nr_cache_zones(zmd)) {
+		alloc_flags = DMZ_ALLOC_RND;
+		goto again;
+	}
 	dmz_unlock_map(zmd);
 	if (!szone)
 		return -ENOSPC;
 
-	dmz_dev_debug(zrc->dev,
-		      "Chunk %u, move rnd zone %u (weight %u) to seq zone %u",
-		      chunk, dmz_id(zmd, dzone), dmz_weight(dzone),
-		      dmz_id(zmd, szone));
+	DMDEBUG("(%s/%u): Chunk %u, move %s zone %u (weight %u) to %s zone %u",
+		dmz_metadata_label(zmd), zrc->dev_idx, chunk,
+		dmz_is_cache(dzone) ? "cache" : "rnd",
+		dzone->id, dmz_weight(dzone),
+		dmz_is_rnd(szone) ? "rnd" : "seq", szone->id);
 
 	/* Flush the random data zone into the sequential zone */
 	ret = dmz_reclaim_copy(zrc, dzone, szone);
@@ -306,7 +320,7 @@ static int dmz_reclaim_rnd_data(struct dmz_reclaim *zrc, struct dm_zone *dzone)
 		dmz_unlock_map(zmd);
 	} else {
 		/* Free the data zone and remap the chunk */
-		dmz_invalidate_blocks(zmd, dzone, 0, zrc->dev->zone_nr_blocks);
+		dmz_invalidate_blocks(zmd, dzone, 0, dmz_zone_nr_blocks(zmd));
 		dmz_lock_map(zmd);
 		dmz_unmap_zone(zmd, dzone);
 		dmz_unlock_zone_reclaim(dzone);
@@ -337,6 +351,14 @@ static void dmz_reclaim_empty(struct dmz_reclaim *zrc, struct dm_zone *dzone)
 }
 
 /*
+ * Test if the target device is idle.
+ */
+static inline int dmz_target_idle(struct dmz_reclaim *zrc)
+{
+	return time_is_before_jiffies(zrc->atime + DMZ_IDLE_PERIOD);
+}
+
+/*
  * Find a candidate zone for reclaim and process it.
  */
 static int dmz_do_reclaim(struct dmz_reclaim *zrc)
@@ -348,13 +370,16 @@ static int dmz_do_reclaim(struct dmz_reclaim *zrc)
 	int ret;
 
 	/* Get a data zone */
-	dzone = dmz_get_zone_for_reclaim(zmd);
-	if (IS_ERR(dzone))
-		return PTR_ERR(dzone);
+	dzone = dmz_get_zone_for_reclaim(zmd, zrc->dev_idx,
+					 dmz_target_idle(zrc));
+	if (!dzone) {
+		DMDEBUG("(%s/%u): No zone found to reclaim",
+			dmz_metadata_label(zmd), zrc->dev_idx);
+		return -EBUSY;
+	}
 
 	start = jiffies;
-
-	if (dmz_is_rnd(dzone)) {
+	if (dmz_is_cache(dzone) || dmz_is_rnd(dzone)) {
 		if (!dmz_weight(dzone)) {
 			/* Empty zone */
 			dmz_reclaim_empty(zrc, dzone);
@@ -395,54 +420,80 @@ static int dmz_do_reclaim(struct dmz_reclaim *zrc)
 	}
 out:
 	if (ret) {
+		if (ret == -EINTR)
+			DMDEBUG("(%s/%u): reclaim zone %u interrupted",
+				dmz_metadata_label(zmd), zrc->dev_idx,
+				rzone->id);
+		else
+			DMDEBUG("(%s/%u): Failed to reclaim zone %u, err %d",
+				dmz_metadata_label(zmd), zrc->dev_idx,
+				rzone->id, ret);
 		dmz_unlock_zone_reclaim(dzone);
 		return ret;
 	}
 
 	ret = dmz_flush_metadata(zrc->metadata);
 	if (ret) {
-		dmz_dev_debug(zrc->dev,
-			      "Metadata flush for zone %u failed, err %d\n",
-			      dmz_id(zmd, rzone), ret);
+		DMDEBUG("(%s/%u): Metadata flush for zone %u failed, err %d",
+			dmz_metadata_label(zmd), zrc->dev_idx, rzone->id, ret);
 		return ret;
 	}
 
-	dmz_dev_debug(zrc->dev, "Reclaimed zone %u in %u ms",
-		      dmz_id(zmd, rzone), jiffies_to_msecs(jiffies - start));
+	DMDEBUG("(%s/%u): Reclaimed zone %u in %u ms",
+		dmz_metadata_label(zmd), zrc->dev_idx,
+		rzone->id, jiffies_to_msecs(jiffies - start));
 	return 0;
 }
 
-/*
- * Test if the target device is idle.
- */
-static inline int dmz_target_idle(struct dmz_reclaim *zrc)
+static unsigned int dmz_reclaim_percentage(struct dmz_reclaim *zrc)
 {
-	return time_is_before_jiffies(zrc->atime + DMZ_IDLE_PERIOD);
+	struct dmz_metadata *zmd = zrc->metadata;
+	unsigned int nr_cache = dmz_nr_cache_zones(zmd);
+	unsigned int nr_unmap, nr_zones;
+
+	if (nr_cache) {
+		nr_zones = nr_cache;
+		nr_unmap = dmz_nr_unmap_cache_zones(zmd);
+	} else {
+		nr_zones = dmz_nr_rnd_zones(zmd, zrc->dev_idx);
+		nr_unmap = dmz_nr_unmap_rnd_zones(zmd, zrc->dev_idx);
+	}
+	return nr_unmap * 100 / nr_zones;
 }
 
 /*
  * Test if reclaim is necessary.
  */
-static bool dmz_should_reclaim(struct dmz_reclaim *zrc)
+static bool dmz_should_reclaim(struct dmz_reclaim *zrc, unsigned int p_unmap)
 {
-	struct dmz_metadata *zmd = zrc->metadata;
-	unsigned int nr_rnd = dmz_nr_rnd_zones(zmd);
-	unsigned int nr_unmap_rnd = dmz_nr_unmap_rnd_zones(zmd);
-	unsigned int p_unmap_rnd = nr_unmap_rnd * 100 / nr_rnd;
+	unsigned int nr_reclaim;
+
+	nr_reclaim = dmz_nr_rnd_zones(zrc->metadata, zrc->dev_idx);
+
+	if (dmz_nr_cache_zones(zrc->metadata)) {
+		/*
+		 * The first device in a multi-device
+		 * setup only contains cache zones, so
+		 * never start reclaim there.
+		 */
+		if (zrc->dev_idx == 0)
+			return false;
+		nr_reclaim += dmz_nr_cache_zones(zrc->metadata);
+	}
 
 	/* Reclaim when idle */
-	if (dmz_target_idle(zrc) && nr_unmap_rnd < nr_rnd)
+	if (dmz_target_idle(zrc) && nr_reclaim)
 		return true;
 
-	/* If there are still plenty of random zones, do not reclaim */
-	if (p_unmap_rnd >= DMZ_RECLAIM_HIGH_UNMAP_RND)
+	/* If there are still plenty of cache zones, do not reclaim */
+	if (p_unmap >= DMZ_RECLAIM_HIGH_UNMAP_ZONES)
 		return false;
 
 	/*
-	 * If the percentage of unmapped random zones is low,
+	 * If the percentage of unmapped cache zones is low,
 	 * reclaim even if the target is busy.
 	 */
-	return p_unmap_rnd <= DMZ_RECLAIM_LOW_UNMAP_RND;
+	return p_unmap <= DMZ_RECLAIM_LOW_UNMAP_ZONES;
 }
 
 /*
@@ -452,14 +503,14 @@ static void dmz_reclaim_work(struct work_struct *work)
 {
 	struct dmz_reclaim *zrc = container_of(work, struct dmz_reclaim, work.work);
 	struct dmz_metadata *zmd = zrc->metadata;
-	unsigned int nr_rnd, nr_unmap_rnd;
-	unsigned int p_unmap_rnd;
+	unsigned int p_unmap, nr_unmap_rnd = 0, nr_rnd = 0;
 	int ret;
 
-	if (dmz_bdev_is_dying(zrc->dev))
+	if (dmz_dev_is_dying(zmd))
 		return;
 
-	if (!dmz_should_reclaim(zrc)) {
+	p_unmap = dmz_reclaim_percentage(zrc);
+	if (!dmz_should_reclaim(zrc, p_unmap)) {
 		mod_delayed_work(zrc->wq, &zrc->work, DMZ_IDLE_PERIOD);
 		return;
 	}
@@ -470,27 +521,29 @@ static void dmz_reclaim_work(struct work_struct *work)
 	 * and slower if there are still some free random zones to avoid
 	 * as much as possible to negatively impact the user workload.
 	 */
-	nr_rnd = dmz_nr_rnd_zones(zmd);
-	nr_unmap_rnd = dmz_nr_unmap_rnd_zones(zmd);
-	p_unmap_rnd = nr_unmap_rnd * 100 / nr_rnd;
-	if (dmz_target_idle(zrc) || p_unmap_rnd < DMZ_RECLAIM_LOW_UNMAP_RND / 2) {
+	if (dmz_target_idle(zrc) || p_unmap < DMZ_RECLAIM_LOW_UNMAP_ZONES / 2) {
 		/* Idle or very low percentage: go fast */
 		zrc->kc_throttle.throttle = 100;
 	} else {
 		/* Busy but we still have some random zone: throttle */
-		zrc->kc_throttle.throttle = min(75U, 100U - p_unmap_rnd / 2);
+		zrc->kc_throttle.throttle = min(75U, 100U - p_unmap / 2);
 	}
 
-	dmz_dev_debug(zrc->dev,
-		      "Reclaim (%u): %s, %u%% free rnd zones (%u/%u)",
-		      zrc->kc_throttle.throttle,
-		      (dmz_target_idle(zrc) ? "Idle" : "Busy"),
-		      p_unmap_rnd, nr_unmap_rnd, nr_rnd);
+	nr_unmap_rnd = dmz_nr_unmap_rnd_zones(zmd, zrc->dev_idx);
+	nr_rnd = dmz_nr_rnd_zones(zmd, zrc->dev_idx);
+
+	DMDEBUG("(%s/%u): Reclaim (%u): %s, %u%% free zones (%u/%u cache %u/%u random)",
+		dmz_metadata_label(zmd), zrc->dev_idx,
+		zrc->kc_throttle.throttle,
+		(dmz_target_idle(zrc) ? "Idle" : "Busy"),
+		p_unmap, dmz_nr_unmap_cache_zones(zmd),
+		dmz_nr_cache_zones(zmd),
+		dmz_nr_unmap_rnd_zones(zmd, zrc->dev_idx),
+		dmz_nr_rnd_zones(zmd, zrc->dev_idx));
 
 	ret = dmz_do_reclaim(zrc);
-	if (ret) {
-		dmz_dev_debug(zrc->dev, "Reclaim error %d\n", ret);
-		if (!dmz_check_bdev(zrc->dev))
+	if (ret && ret != -EINTR) {
+		if (!dmz_check_dev(zmd))
 			return;
 	}
 
@@ -500,8 +553,8 @@ static void dmz_reclaim_work(struct work_struct *work)
 /*
  * Initialize reclaim.
  */
-int dmz_ctr_reclaim(struct dmz_dev *dev, struct dmz_metadata *zmd,
-		    struct dmz_reclaim **reclaim)
+int dmz_ctr_reclaim(struct dmz_metadata *zmd,
+		    struct dmz_reclaim **reclaim, int idx)
 {
 	struct dmz_reclaim *zrc;
 	int ret;
@@ -510,9 +563,9 @@ int dmz_ctr_reclaim(struct dmz_dev *dev, struct dmz_metadata *zmd,
 	if (!zrc)
 		return -ENOMEM;
 
-	zrc->dev = dev;
 	zrc->metadata = zmd;
 	zrc->atime = jiffies;
+	zrc->dev_idx = idx;
 
 	/* Reclaim kcopyd client */
 	zrc->kc = dm_kcopyd_client_create(&zrc->kc_throttle);
@@ -524,8 +577,8 @@ int dmz_ctr_reclaim(struct dmz_dev *dev, struct dmz_metadata *zmd,
 
 	/* Reclaim work */
 	INIT_DELAYED_WORK(&zrc->work, dmz_reclaim_work);
-	zrc->wq = alloc_ordered_workqueue("dmz_rwq_%s", WQ_MEM_RECLAIM,
-					  dev->name);
+	zrc->wq = alloc_ordered_workqueue("dmz_rwq_%s_%d", WQ_MEM_RECLAIM,
+					  dmz_metadata_label(zmd), idx);
 	if (!zrc->wq) {
 		ret = -ENOMEM;
 		goto err;
@@ -583,7 +636,8 @@ void dmz_reclaim_bio_acc(struct dmz_reclaim *zrc)
  */
 void dmz_schedule_reclaim(struct dmz_reclaim *zrc)
 {
-	if (dmz_should_reclaim(zrc))
+	unsigned int p_unmap = dmz_reclaim_percentage(zrc);
+
+	if (dmz_should_reclaim(zrc, p_unmap))
 		mod_delayed_work(zrc->wq, &zrc->work, 0);
 }
-
diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c
index f4f83d39b3dc..a907a9446c0b 100644
--- a/drivers/md/dm-zoned-target.c
+++ b/drivers/md/dm-zoned-target.c
@@ -17,7 +17,7 @@
  * Zone BIO context.
  */
 struct dmz_bioctx {
-	struct dmz_target	*target;
+	struct dmz_dev		*dev;
 	struct dm_zone		*zone;
 	struct bio		*bio;
 	refcount_t		ref;
@@ -38,9 +38,10 @@ struct dm_chunk_work {
  * Target descriptor.
  */
 struct dmz_target {
-	struct dm_dev		*ddev;
+	struct dm_dev		**ddev;
+	unsigned int		nr_ddevs;
 
-	unsigned long		flags;
+	unsigned int		flags;
 
 	/* Zoned block device information */
 	struct dmz_dev		*dev;
@@ -48,9 +49,6 @@ struct dmz_target {
 	/* For metadata handling */
 	struct dmz_metadata     *metadata;
 
-	/* For reclaim */
-	struct dmz_reclaim	*reclaim;
-
 	/* For chunk work */
 	struct radix_tree_root	chunk_rxtree;
 	struct workqueue_struct *chunk_wq;
@@ -76,12 +74,13 @@ struct dmz_target {
  */
 static inline void dmz_bio_endio(struct bio *bio, blk_status_t status)
 {
-	struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
+	struct dmz_bioctx *bioctx =
+		dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
 
 	if (status != BLK_STS_OK && bio->bi_status == BLK_STS_OK)
 		bio->bi_status = status;
-	if (bio->bi_status != BLK_STS_OK)
-		bioctx->target->dev->flags |= DMZ_CHECK_BDEV;
+	if (bioctx->dev && bio->bi_status != BLK_STS_OK)
+		bioctx->dev->flags |= DMZ_CHECK_BDEV;
 
 	if (refcount_dec_and_test(&bioctx->ref)) {
 		struct dm_zone *zone = bioctx->zone;
@@ -118,14 +117,20 @@ static int dmz_submit_bio(struct dmz_target *dmz, struct dm_zone *zone,
 			  struct bio *bio, sector_t chunk_block,
 			  unsigned int nr_blocks)
 {
-	struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
+	struct dmz_bioctx *bioctx =
+		dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
+	struct dmz_dev *dev = zone->dev;
 	struct bio *clone;
 
+	if (dev->flags & DMZ_BDEV_DYING)
+		return -EIO;
+
 	clone = bio_clone_fast(bio, GFP_NOIO, &dmz->bio_set);
 	if (!clone)
 		return -ENOMEM;
 
-	bio_set_dev(clone, dmz->dev->bdev);
+	bio_set_dev(clone, dev->bdev);
+	bioctx->dev = dev;
 	clone->bi_iter.bi_sector =
 		dmz_start_sect(dmz->metadata, zone) + dmz_blk2sect(chunk_block);
 	clone->bi_iter.bi_size = dmz_blk2sect(nr_blocks) << SECTOR_SHIFT;
@@ -165,7 +170,8 @@ static void dmz_handle_read_zero(struct dmz_target *dmz, struct bio *bio,
 static int dmz_handle_read(struct dmz_target *dmz, struct dm_zone *zone,
 			   struct bio *bio)
 {
-	sector_t chunk_block = dmz_chunk_block(dmz->dev, dmz_bio_block(bio));
+	struct dmz_metadata *zmd = dmz->metadata;
+	sector_t chunk_block = dmz_chunk_block(zmd, dmz_bio_block(bio));
 	unsigned int nr_blocks = dmz_bio_blocks(bio);
 	sector_t end_block = chunk_block + nr_blocks;
 	struct dm_zone *rzone, *bzone;
@@ -177,19 +183,22 @@ static int dmz_handle_read(struct dmz_target *dmz, struct dm_zone *zone,
 		return 0;
 	}
 
-	dmz_dev_debug(dmz->dev, "READ chunk %llu -> %s zone %u, block %llu, %u blocks",
-		      (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
-		      (dmz_is_rnd(zone) ? "RND" : "SEQ"),
-		      dmz_id(dmz->metadata, zone),
-		      (unsigned long long)chunk_block, nr_blocks);
+	DMDEBUG("(%s): READ chunk %llu -> %s zone %u, block %llu, %u blocks",
+		dmz_metadata_label(zmd),
+		(unsigned long long)dmz_bio_chunk(zmd, bio),
+		(dmz_is_rnd(zone) ? "RND" :
+		 (dmz_is_cache(zone) ? "CACHE" : "SEQ")),
+		zone->id,
+		(unsigned long long)chunk_block, nr_blocks);
 
 	/* Check block validity to determine the read location */
 	bzone = zone->bzone;
 	while (chunk_block < end_block) {
 		nr_blocks = 0;
-		if (dmz_is_rnd(zone) || chunk_block < zone->wp_block) {
+		if (dmz_is_rnd(zone) || dmz_is_cache(zone) ||
+		    chunk_block < zone->wp_block) {
 			/* Test block validity in the data zone */
-			ret = dmz_block_valid(dmz->metadata, zone, chunk_block);
+			ret = dmz_block_valid(zmd, zone, chunk_block);
 			if (ret < 0)
 				return ret;
 			if (ret > 0) {
@@ -204,7 +213,7 @@ static int dmz_handle_read(struct dmz_target *dmz, struct dm_zone *zone,
 		 * Check the buffer zone, if there is one.
 		 */
 		if (!nr_blocks && bzone) {
-			ret = dmz_block_valid(dmz->metadata, bzone, chunk_block);
+			ret = dmz_block_valid(zmd, bzone, chunk_block);
 			if (ret < 0)
 				return ret;
 			if (ret > 0) {
@@ -216,8 +225,10 @@ static int dmz_handle_read(struct dmz_target *dmz, struct dm_zone *zone,
 
 		if (nr_blocks) {
 			/* Valid blocks found: read them */
-			nr_blocks = min_t(unsigned int, nr_blocks, end_block - chunk_block);
-			ret = dmz_submit_bio(dmz, rzone, bio, chunk_block, nr_blocks);
+			nr_blocks = min_t(unsigned int, nr_blocks,
+					  end_block - chunk_block);
+			ret = dmz_submit_bio(dmz, rzone, bio,
+					     chunk_block, nr_blocks);
 			if (ret)
 				return ret;
 			chunk_block += nr_blocks;
@@ -308,25 +319,30 @@ static int dmz_handle_buffered_write(struct dmz_target *dmz,
 static int dmz_handle_write(struct dmz_target *dmz, struct dm_zone *zone,
 			    struct bio *bio)
 {
-	sector_t chunk_block = dmz_chunk_block(dmz->dev, dmz_bio_block(bio));
+	struct dmz_metadata *zmd = dmz->metadata;
+	sector_t chunk_block = dmz_chunk_block(zmd, dmz_bio_block(bio));
 	unsigned int nr_blocks = dmz_bio_blocks(bio);
 
 	if (!zone)
 		return -ENOSPC;
 
-	dmz_dev_debug(dmz->dev, "WRITE chunk %llu -> %s zone %u, block %llu, %u blocks",
-		      (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
-		      (dmz_is_rnd(zone) ? "RND" : "SEQ"),
-		      dmz_id(dmz->metadata, zone),
-		      (unsigned long long)chunk_block, nr_blocks);
+	DMDEBUG("(%s): WRITE chunk %llu -> %s zone %u, block %llu, %u blocks",
+		dmz_metadata_label(zmd),
+		(unsigned long long)dmz_bio_chunk(zmd, bio),
+		(dmz_is_rnd(zone) ? "RND" :
+		 (dmz_is_cache(zone) ? "CACHE" : "SEQ")),
+		zone->id,
+		(unsigned long long)chunk_block, nr_blocks);
 
-	if (dmz_is_rnd(zone) || chunk_block == zone->wp_block) {
+	if (dmz_is_rnd(zone) || dmz_is_cache(zone) ||
+	    chunk_block == zone->wp_block) {
 		/*
 		 * zone is a random zone or it is a sequential zone
 		 * and the BIO is aligned to the zone write pointer:
 		 * direct write the zone.
 		 */
-		return dmz_handle_direct_write(dmz, zone, bio, chunk_block, nr_blocks);
+		return dmz_handle_direct_write(dmz, zone, bio,
+					       chunk_block, nr_blocks);
 	}
 
 	/*
@@ -345,7 +361,7 @@ static int dmz_handle_discard(struct dmz_target *dmz, struct dm_zone *zone,
 	struct dmz_metadata *zmd = dmz->metadata;
 	sector_t block = dmz_bio_block(bio);
 	unsigned int nr_blocks = dmz_bio_blocks(bio);
-	sector_t chunk_block = dmz_chunk_block(dmz->dev, block);
+	sector_t chunk_block = dmz_chunk_block(zmd, block);
 	int ret = 0;
 
 	/* For unmapped chunks, there is nothing to do */
@@ -355,16 +371,18 @@ static int dmz_handle_discard(struct dmz_target *dmz, struct dm_zone *zone,
 	if (dmz_is_readonly(zone))
 		return -EROFS;
 
-	dmz_dev_debug(dmz->dev, "DISCARD chunk %llu -> zone %u, block %llu, %u blocks",
-		      (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
-		      dmz_id(zmd, zone),
-		      (unsigned long long)chunk_block, nr_blocks);
+	DMDEBUG("(%s): DISCARD chunk %llu -> zone %u, block %llu, %u blocks",
+		dmz_metadata_label(dmz->metadata),
+		(unsigned long long)dmz_bio_chunk(zmd, bio),
+		zone->id,
+		(unsigned long long)chunk_block, nr_blocks);
 
 	/*
 	 * Invalidate blocks in the data zone and its
 	 * buffer zone if one is mapped.
 	 */
-	if (dmz_is_rnd(zone) || chunk_block < zone->wp_block)
+	if (dmz_is_rnd(zone) || dmz_is_cache(zone) ||
+	    chunk_block < zone->wp_block)
 		ret = dmz_invalidate_blocks(zmd, zone, chunk_block, nr_blocks);
 	if (ret == 0 && zone->bzone)
 		ret = dmz_invalidate_blocks(zmd, zone->bzone,
@@ -378,31 +396,28 @@ static int dmz_handle_discard(struct dmz_target *dmz, struct dm_zone *zone,
 static void dmz_handle_bio(struct dmz_target *dmz, struct dm_chunk_work *cw,
 			   struct bio *bio)
 {
-	struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
+	struct dmz_bioctx *bioctx =
+		dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
 	struct dmz_metadata *zmd = dmz->metadata;
 	struct dm_zone *zone;
-	int ret;
+	int i, ret;
 
 	/*
 	 * Write may trigger a zone allocation. So make sure the
 	 * allocation can succeed.
 	 */
 	if (bio_op(bio) == REQ_OP_WRITE)
-		dmz_schedule_reclaim(dmz->reclaim);
+		for (i = 0; i < dmz->nr_ddevs; i++)
+			dmz_schedule_reclaim(dmz->dev[i].reclaim);
 
 	dmz_lock_metadata(zmd);
 
-	if (dmz->dev->flags & DMZ_BDEV_DYING) {
-		ret = -EIO;
-		goto out;
-	}
-
 	/*
 	 * Get the data zone mapping the chunk. There may be no
 	 * mapping for read and discard. If a mapping is obtained,
 	 + the zone returned will be set to active state.
 	 */
-	zone = dmz_get_chunk_mapping(zmd, dmz_bio_chunk(dmz->dev, bio),
+	zone = dmz_get_chunk_mapping(zmd, dmz_bio_chunk(zmd, bio),
 				     bio_op(bio));
 	if (IS_ERR(zone)) {
 		ret = PTR_ERR(zone);
@@ -413,6 +428,7 @@ static void dmz_handle_bio(struct dmz_target *dmz, struct dm_chunk_work *cw,
 	if (zone) {
 		dmz_activate_zone(zone);
 		bioctx->zone = zone;
+		dmz_reclaim_bio_acc(zone->dev->reclaim);
 	}
 
 	switch (bio_op(bio)) {
@@ -427,8 +443,8 @@ static void dmz_handle_bio(struct dmz_target *dmz, struct dm_chunk_work *cw,
 		ret = dmz_handle_discard(dmz, zone, bio);
 		break;
 	default:
-		dmz_dev_err(dmz->dev, "Unsupported BIO operation 0x%x",
-			    bio_op(bio));
+		DMERR("(%s): Unsupported BIO operation 0x%x",
+		      dmz_metadata_label(dmz->metadata), bio_op(bio));
 		ret = -EIO;
 	}
 
@@ -502,7 +518,8 @@ static void dmz_flush_work(struct work_struct *work)
 	/* Flush dirty metadata blocks */
 	ret = dmz_flush_metadata(dmz->metadata);
 	if (ret)
-		dmz_dev_debug(dmz->dev, "Metadata flush failed, rc=%d\n", ret);
+		DMDEBUG("(%s): Metadata flush failed, rc=%d",
+			dmz_metadata_label(dmz->metadata), ret);
 
 	/* Process queued flush requests */
 	while (1) {
@@ -525,7 +542,7 @@ static void dmz_flush_work(struct work_struct *work)
  */
 static int dmz_queue_chunk_work(struct dmz_target *dmz, struct bio *bio)
 {
-	unsigned int chunk = dmz_bio_chunk(dmz->dev, bio);
+	unsigned int chunk = dmz_bio_chunk(dmz->metadata, bio);
 	struct dm_chunk_work *cw;
 	int ret = 0;
 
@@ -558,7 +575,6 @@ static int dmz_queue_chunk_work(struct dmz_target *dmz, struct bio *bio)
 
 	bio_list_add(&cw->bio_list, bio);
 
-	dmz_reclaim_bio_acc(dmz->reclaim);
 	if (queue_work(dmz->chunk_wq, &cw->work))
 		dmz_get_chunk_work(cw);
 out:
@@ -618,23 +634,22 @@ bool dmz_check_bdev(struct dmz_dev *dmz_dev)
 static int dmz_map(struct dm_target *ti, struct bio *bio)
 {
 	struct dmz_target *dmz = ti->private;
-	struct dmz_dev *dev = dmz->dev;
+	struct dmz_metadata *zmd = dmz->metadata;
 	struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
 	sector_t sector = bio->bi_iter.bi_sector;
 	unsigned int nr_sectors = bio_sectors(bio);
 	sector_t chunk_sector;
 	int ret;
 
-	if (dmz_bdev_is_dying(dmz->dev))
+	if (dmz_dev_is_dying(zmd))
 		return DM_MAPIO_KILL;
 
-	dmz_dev_debug(dev, "BIO op %d sector %llu + %u => chunk %llu, block %llu, %u blocks",
-		      bio_op(bio), (unsigned long long)sector, nr_sectors,
-		      (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
-		      (unsigned long long)dmz_chunk_block(dmz->dev, dmz_bio_block(bio)),
-		      (unsigned int)dmz_bio_blocks(bio));
-
-	bio_set_dev(bio, dev->bdev);
+	DMDEBUG("(%s): BIO op %d sector %llu + %u => chunk %llu, block %llu, %u blocks",
+		dmz_metadata_label(zmd),
+		bio_op(bio), (unsigned long long)sector, nr_sectors,
+		(unsigned long long)dmz_bio_chunk(zmd, bio),
+		(unsigned long long)dmz_chunk_block(zmd, dmz_bio_block(bio)),
+		(unsigned int)dmz_bio_blocks(bio));
 
 	if (!nr_sectors && bio_op(bio) != REQ_OP_WRITE)
 		return DM_MAPIO_REMAPPED;
@@ -644,7 +659,7 @@ static int dmz_map(struct dm_target *ti, struct bio *bio)
 		return DM_MAPIO_KILL;
 
 	/* Initialize the BIO context */
-	bioctx->target = dmz;
+	bioctx->dev = NULL;
 	bioctx->zone = NULL;
 	bioctx->bio = bio;
 	refcount_set(&bioctx->ref, 1);
@@ -659,17 +674,17 @@ static int dmz_map(struct dm_target *ti, struct bio *bio)
 	}
 
 	/* Split zone BIOs to fit entirely into a zone */
-	chunk_sector = sector & (dev->zone_nr_sectors - 1);
-	if (chunk_sector + nr_sectors > dev->zone_nr_sectors)
-		dm_accept_partial_bio(bio, dev->zone_nr_sectors - chunk_sector);
+	chunk_sector = sector & (dmz_zone_nr_sectors(zmd) - 1);
+	if (chunk_sector + nr_sectors > dmz_zone_nr_sectors(zmd))
+		dm_accept_partial_bio(bio, dmz_zone_nr_sectors(zmd) - chunk_sector);
 
 	/* Now ready to handle this BIO */
 	ret = dmz_queue_chunk_work(dmz, bio);
 	if (ret) {
-		dmz_dev_debug(dmz->dev,
-			      "BIO op %d, can't process chunk %llu, err %i\n",
-			      bio_op(bio), (u64)dmz_bio_chunk(dmz->dev, bio),
-			      ret);
+		DMDEBUG("(%s): BIO op %d, can't process chunk %llu, err %i",
+			dmz_metadata_label(zmd),
+			bio_op(bio), (u64)dmz_bio_chunk(zmd, bio),
+			ret);
 		return DM_MAPIO_REQUEUE;
 	}
 
@@ -679,64 +694,65 @@ static int dmz_map(struct dm_target *ti, struct bio *bio)
 /*
  * Get zoned device information.
  */
-static int dmz_get_zoned_device(struct dm_target *ti, char *path)
+static int dmz_get_zoned_device(struct dm_target *ti, char *path,
+				int idx, int nr_devs)
 {
 	struct dmz_target *dmz = ti->private;
-	struct request_queue *q;
+	struct dm_dev *ddev;
 	struct dmz_dev *dev;
-	sector_t aligned_capacity;
 	int ret;
+	struct block_device *bdev;
 
 	/* Get the target device */
-	ret = dm_get_device(ti, path, dm_table_get_mode(ti->table), &dmz->ddev);
+	ret = dm_get_device(ti, path, dm_table_get_mode(ti->table), &ddev);
 	if (ret) {
 		ti->error = "Get target device failed";
-		dmz->ddev = NULL;
 		return ret;
 	}
 
-	dev = kzalloc(sizeof(struct dmz_dev), GFP_KERNEL);
-	if (!dev) {
-		ret = -ENOMEM;
-		goto err;
+	bdev = ddev->bdev;
+	if (bdev_zoned_model(bdev) == BLK_ZONED_NONE) {
+		if (nr_devs == 1) {
+			ti->error = "Invalid regular device";
+			goto err;
+		}
+		if (idx != 0) {
+			ti->error = "First device must be a regular device";
+			goto err;
+		}
+		if (dmz->ddev[0]) {
+			ti->error = "Too many regular devices";
+			goto err;
+		}
+		dev = &dmz->dev[idx];
+		dev->flags = DMZ_BDEV_REGULAR;
+	} else {
+		if (dmz->ddev[idx]) {
+			ti->error = "Too many zoned devices";
+			goto err;
+		}
+		if (nr_devs > 1 && idx == 0) {
+			ti->error = "First device must be a regular device";
+			goto err;
+		}
+		dev = &dmz->dev[idx];
 	}
-
-	dev->bdev = dmz->ddev->bdev;
+	dev->bdev = bdev;
+	dev->dev_idx = idx;
 	(void)bdevname(dev->bdev, dev->name);
 
-	if (bdev_zoned_model(dev->bdev) == BLK_ZONED_NONE) {
-		ti->error = "Not a zoned block device";
-		ret = -EINVAL;
+	dev->capacity = i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
+	if (ti->begin) {
+		ti->error = "Partial mapping is not supported";
 		goto err;
 	}
 
-	q = bdev_get_queue(dev->bdev);
-	dev->capacity = i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
-	aligned_capacity = dev->capacity &
-				~((sector_t)blk_queue_zone_sectors(q) - 1);
-	if (ti->begin ||
-	    ((ti->len != dev->capacity) && (ti->len != aligned_capacity))) {
-		ti->error = "Partial mapping not supported";
-		ret = -EINVAL;
-		goto err;
-	}
-
-	dev->zone_nr_sectors = blk_queue_zone_sectors(q);
-	dev->zone_nr_sectors_shift = ilog2(dev->zone_nr_sectors);
-
-	dev->zone_nr_blocks = dmz_sect2blk(dev->zone_nr_sectors);
-	dev->zone_nr_blocks_shift = ilog2(dev->zone_nr_blocks);
-
-	dev->nr_zones = blkdev_nr_zones(dev->bdev->bd_disk);
-
-	dmz->dev = dev;
+	dmz->ddev[idx] = ddev;
 
 	return 0;
 err:
-	dm_put_device(ti, dmz->ddev);
-	kfree(dev);
-
-	return ret;
+	dm_put_device(ti, ddev);
+	return -EINVAL;
 }
 
 /*
@@ -745,10 +761,78 @@ err:
 static void dmz_put_zoned_device(struct dm_target *ti)
 {
 	struct dmz_target *dmz = ti->private;
+	int i;
 
-	dm_put_device(ti, dmz->ddev);
-	kfree(dmz->dev);
-	dmz->dev = NULL;
+	for (i = 0; i < dmz->nr_ddevs; i++) {
+		if (dmz->ddev[i]) {
+			dm_put_device(ti, dmz->ddev[i]);
+			dmz->ddev[i] = NULL;
+		}
+	}
+}
+
+static int dmz_fixup_devices(struct dm_target *ti)
+{
+	struct dmz_target *dmz = ti->private;
+	struct dmz_dev *reg_dev, *zoned_dev;
+	struct request_queue *q;
+	sector_t zone_nr_sectors = 0;
+	int i;
+
+	/*
+	 * When we have more than on devices, the first one must be a
+	 * regular block device and the others zoned block devices.
+	 */
+	if (dmz->nr_ddevs > 1) {
+		reg_dev = &dmz->dev[0];
+		if (!(reg_dev->flags & DMZ_BDEV_REGULAR)) {
+			ti->error = "Primary disk is not a regular device";
+			return -EINVAL;
+		}
+		for (i = 1; i < dmz->nr_ddevs; i++) {
+			zoned_dev = &dmz->dev[i];
+			if (zoned_dev->flags & DMZ_BDEV_REGULAR) {
+				ti->error = "Secondary disk is not a zoned device";
+				return -EINVAL;
+			}
+			q = bdev_get_queue(zoned_dev->bdev);
+			if (zone_nr_sectors &&
+			    zone_nr_sectors != blk_queue_zone_sectors(q)) {
+				ti->error = "Zone nr sectors mismatch";
+				return -EINVAL;
+			}
+			zone_nr_sectors = blk_queue_zone_sectors(q);
+			zoned_dev->zone_nr_sectors = zone_nr_sectors;
+			zoned_dev->nr_zones =
+				blkdev_nr_zones(zoned_dev->bdev->bd_disk);
+		}
+	} else {
+		reg_dev = NULL;
+		zoned_dev = &dmz->dev[0];
+		if (zoned_dev->flags & DMZ_BDEV_REGULAR) {
+			ti->error = "Disk is not a zoned device";
+			return -EINVAL;
+		}
+		q = bdev_get_queue(zoned_dev->bdev);
+		zoned_dev->zone_nr_sectors = blk_queue_zone_sectors(q);
+		zoned_dev->nr_zones = blkdev_nr_zones(zoned_dev->bdev->bd_disk);
+	}
+
+	if (reg_dev) {
+		sector_t zone_offset;
+
+		reg_dev->zone_nr_sectors = zone_nr_sectors;
+		reg_dev->nr_zones =
+			DIV_ROUND_UP_SECTOR_T(reg_dev->capacity,
+					      reg_dev->zone_nr_sectors);
+		reg_dev->zone_offset = 0;
+		zone_offset = reg_dev->nr_zones;
+		for (i = 1; i < dmz->nr_ddevs; i++) {
+			dmz->dev[i].zone_offset = zone_offset;
+			zone_offset += dmz->dev[i].nr_zones;
+		}
+	}
+	return 0;
 }
 
 /*
@@ -757,11 +841,10 @@ static void dmz_put_zoned_device(struct dm_target *ti)
 static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 {
 	struct dmz_target *dmz;
-	struct dmz_dev *dev;
-	int ret;
+	int ret, i;
 
 	/* Check arguments */
-	if (argc != 1) {
+	if (argc < 1) {
 		ti->error = "Invalid argument count";
 		return -EINVAL;
 	}
@@ -772,25 +855,42 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		ti->error = "Unable to allocate the zoned target descriptor";
 		return -ENOMEM;
 	}
+	dmz->dev = kcalloc(argc, sizeof(struct dmz_dev), GFP_KERNEL);
+	if (!dmz->dev) {
+		ti->error = "Unable to allocate the zoned device descriptors";
+		kfree(dmz);
+		return -ENOMEM;
+	}
+	dmz->ddev = kcalloc(argc, sizeof(struct dm_dev *), GFP_KERNEL);
+	if (!dmz->ddev) {
+		ti->error = "Unable to allocate the dm device descriptors";
+		ret = -ENOMEM;
+		goto err;
+	}
+	dmz->nr_ddevs = argc;
+
 	ti->private = dmz;
 
 	/* Get the target zoned block device */
-	ret = dmz_get_zoned_device(ti, argv[0]);
-	if (ret) {
-		dmz->ddev = NULL;
-		goto err;
+	for (i = 0; i < argc; i++) {
+		ret = dmz_get_zoned_device(ti, argv[i], i, argc);
+		if (ret)
+			goto err_dev;
 	}
+	ret = dmz_fixup_devices(ti);
+	if (ret)
+		goto err_dev;
 
 	/* Initialize metadata */
-	dev = dmz->dev;
-	ret = dmz_ctr_metadata(dev, &dmz->metadata);
+	ret = dmz_ctr_metadata(dmz->dev, argc, &dmz->metadata,
+			       dm_table_device_name(ti->table));
 	if (ret) {
 		ti->error = "Metadata initialization failed";
 		goto err_dev;
 	}
 
 	/* Set target (no write same support) */
-	ti->max_io_len = dev->zone_nr_sectors << 9;
+	ti->max_io_len = dmz_zone_nr_sectors(dmz->metadata) << 9;
 	ti->num_flush_bios = 1;
 	ti->num_discard_bios = 1;
 	ti->num_write_zeroes_bios = 1;
@@ -799,7 +899,8 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	ti->discards_supported = true;
 
 	/* The exposed capacity is the number of chunks that can be mapped */
-	ti->len = (sector_t)dmz_nr_chunks(dmz->metadata) << dev->zone_nr_sectors_shift;
+	ti->len = (sector_t)dmz_nr_chunks(dmz->metadata) <<
+		dmz_zone_nr_sectors_shift(dmz->metadata);
 
 	/* Zone BIO */
 	ret = bioset_init(&dmz->bio_set, DMZ_MIN_BIOS, 0, 0);
@@ -811,8 +912,9 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	/* Chunk BIO work */
 	mutex_init(&dmz->chunk_lock);
 	INIT_RADIX_TREE(&dmz->chunk_rxtree, GFP_NOIO);
-	dmz->chunk_wq = alloc_workqueue("dmz_cwq_%s", WQ_MEM_RECLAIM | WQ_UNBOUND,
-					0, dev->name);
+	dmz->chunk_wq = alloc_workqueue("dmz_cwq_%s",
+					WQ_MEM_RECLAIM | WQ_UNBOUND, 0,
+					dmz_metadata_label(dmz->metadata));
 	if (!dmz->chunk_wq) {
 		ti->error = "Create chunk workqueue failed";
 		ret = -ENOMEM;
@@ -824,7 +926,7 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	bio_list_init(&dmz->flush_list);
 	INIT_DELAYED_WORK(&dmz->flush_work, dmz_flush_work);
 	dmz->flush_wq = alloc_ordered_workqueue("dmz_fwq_%s", WQ_MEM_RECLAIM,
-						dev->name);
+						dmz_metadata_label(dmz->metadata));
 	if (!dmz->flush_wq) {
 		ti->error = "Create flush workqueue failed";
 		ret = -ENOMEM;
@@ -833,15 +935,18 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	mod_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
 
 	/* Initialize reclaim */
-	ret = dmz_ctr_reclaim(dev, dmz->metadata, &dmz->reclaim);
-	if (ret) {
-		ti->error = "Zone reclaim initialization failed";
-		goto err_fwq;
+	for (i = 0; i < dmz->nr_ddevs; i++) {
+		ret = dmz_ctr_reclaim(dmz->metadata, &dmz->dev[i].reclaim, i);
+		if (ret) {
+			ti->error = "Zone reclaim initialization failed";
+			goto err_fwq;
+		}
 	}
 
-	dmz_dev_info(dev, "Target device: %llu 512-byte logical sectors (%llu blocks)",
-		     (unsigned long long)ti->len,
-		     (unsigned long long)dmz_sect2blk(ti->len));
+	DMINFO("(%s): Target device: %llu 512-byte logical sectors (%llu blocks)",
+	       dmz_metadata_label(dmz->metadata),
+	       (unsigned long long)ti->len,
+	       (unsigned long long)dmz_sect2blk(ti->len));
 
 	return 0;
 err_fwq:
@@ -856,6 +961,7 @@ err_meta:
 err_dev:
 	dmz_put_zoned_device(ti);
 err:
+	kfree(dmz->dev);
 	kfree(dmz);
 
 	return ret;
@@ -867,11 +973,13 @@ err:
 static void dmz_dtr(struct dm_target *ti)
 {
 	struct dmz_target *dmz = ti->private;
+	int i;
 
 	flush_workqueue(dmz->chunk_wq);
 	destroy_workqueue(dmz->chunk_wq);
 
-	dmz_dtr_reclaim(dmz->reclaim);
+	for (i = 0; i < dmz->nr_ddevs; i++)
+		dmz_dtr_reclaim(dmz->dev[i].reclaim);
 
 	cancel_delayed_work_sync(&dmz->flush_work);
 	destroy_workqueue(dmz->flush_wq);
@@ -886,6 +994,7 @@ static void dmz_dtr(struct dm_target *ti)
 
 	mutex_destroy(&dmz->chunk_lock);
 
+	kfree(dmz->dev);
 	kfree(dmz);
 }
 
@@ -895,7 +1004,7 @@ static void dmz_dtr(struct dm_target *ti)
 static void dmz_io_hints(struct dm_target *ti, struct queue_limits *limits)
 {
 	struct dmz_target *dmz = ti->private;
-	unsigned int chunk_sectors = dmz->dev->zone_nr_sectors;
+	unsigned int chunk_sectors = dmz_zone_nr_sectors(dmz->metadata);
 
 	limits->logical_block_size = DMZ_BLOCK_SIZE;
 	limits->physical_block_size = DMZ_BLOCK_SIZE;
@@ -923,11 +1032,12 @@ static void dmz_io_hints(struct dm_target *ti, struct queue_limits *limits)
 static int dmz_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
 {
 	struct dmz_target *dmz = ti->private;
+	struct dmz_dev *dev = &dmz->dev[0];
 
-	if (!dmz_check_bdev(dmz->dev))
+	if (!dmz_check_bdev(dev))
 		return -EIO;
 
-	*bdev = dmz->dev->bdev;
+	*bdev = dev->bdev;
 
 	return 0;
 }
@@ -938,9 +1048,11 @@ static int dmz_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
 static void dmz_suspend(struct dm_target *ti)
 {
 	struct dmz_target *dmz = ti->private;
+	int i;
 
 	flush_workqueue(dmz->chunk_wq);
-	dmz_suspend_reclaim(dmz->reclaim);
+	for (i = 0; i < dmz->nr_ddevs; i++)
+		dmz_suspend_reclaim(dmz->dev[i].reclaim);
 	cancel_delayed_work_sync(&dmz->flush_work);
 }
 
@@ -950,24 +1062,95 @@ static void dmz_suspend(struct dm_target *ti)
 static void dmz_resume(struct dm_target *ti)
 {
 	struct dmz_target *dmz = ti->private;
+	int i;
 
 	queue_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
-	dmz_resume_reclaim(dmz->reclaim);
+	for (i = 0; i < dmz->nr_ddevs; i++)
+		dmz_resume_reclaim(dmz->dev[i].reclaim);
 }
 
 static int dmz_iterate_devices(struct dm_target *ti,
 			       iterate_devices_callout_fn fn, void *data)
 {
 	struct dmz_target *dmz = ti->private;
-	struct dmz_dev *dev = dmz->dev;
-	sector_t capacity = dev->capacity & ~(dev->zone_nr_sectors - 1);
+	unsigned int zone_nr_sectors = dmz_zone_nr_sectors(dmz->metadata);
+	sector_t capacity;
+	int i, r;
+
+	for (i = 0; i < dmz->nr_ddevs; i++) {
+		capacity = dmz->dev[i].capacity & ~(zone_nr_sectors - 1);
+		r = fn(ti, dmz->ddev[i], 0, capacity, data);
+		if (r)
+			break;
+	}
+	return r;
+}
+
+static void dmz_status(struct dm_target *ti, status_type_t type,
+		       unsigned int status_flags, char *result,
+		       unsigned int maxlen)
+{
+	struct dmz_target *dmz = ti->private;
+	ssize_t sz = 0;
+	char buf[BDEVNAME_SIZE];
+	struct dmz_dev *dev;
+	int i;
+
+	switch (type) {
+	case STATUSTYPE_INFO:
+		DMEMIT("%u zones %u/%u cache",
+		       dmz_nr_zones(dmz->metadata),
+		       dmz_nr_unmap_cache_zones(dmz->metadata),
+		       dmz_nr_cache_zones(dmz->metadata));
+		for (i = 0; i < dmz->nr_ddevs; i++) {
+			/*
+			 * For a multi-device setup the first device
+			 * contains only cache zones.
+			 */
+			if ((i == 0) &&
+			    (dmz_nr_cache_zones(dmz->metadata) > 0))
+				continue;
+			DMEMIT(" %u/%u random %u/%u sequential",
+			       dmz_nr_unmap_rnd_zones(dmz->metadata, i),
+			       dmz_nr_rnd_zones(dmz->metadata, i),
+			       dmz_nr_unmap_seq_zones(dmz->metadata, i),
+			       dmz_nr_seq_zones(dmz->metadata, i));
+		}
+		break;
+	case STATUSTYPE_TABLE:
+		dev = &dmz->dev[0];
+		format_dev_t(buf, dev->bdev->bd_dev);
+		DMEMIT("%s", buf);
+		for (i = 1; i < dmz->nr_ddevs; i++) {
+			dev = &dmz->dev[i];
+			format_dev_t(buf, dev->bdev->bd_dev);
+			DMEMIT(" %s", buf);
+		}
+		break;
+	}
+	return;
+}
+
+static int dmz_message(struct dm_target *ti, unsigned int argc, char **argv,
+		       char *result, unsigned int maxlen)
+{
+	struct dmz_target *dmz = ti->private;
+	int r = -EINVAL;
+
+	if (!strcasecmp(argv[0], "reclaim")) {
+		int i;
 
-	return fn(ti, dmz->ddev, 0, capacity, data);
+		for (i = 0; i < dmz->nr_ddevs; i++)
+			dmz_schedule_reclaim(dmz->dev[i].reclaim);
+		r = 0;
+	} else
+		DMERR("unrecognized message %s", argv[0]);
+	return r;
 }
 
 static struct target_type dmz_type = {
 	.name		 = "zoned",
-	.version	 = {1, 1, 0},
+	.version	 = {2, 0, 0},
 	.features	 = DM_TARGET_SINGLETON | DM_TARGET_ZONED_HM,
 	.module		 = THIS_MODULE,
 	.ctr		 = dmz_ctr,
@@ -978,6 +1161,8 @@ static struct target_type dmz_type = {
 	.postsuspend	 = dmz_suspend,
 	.resume		 = dmz_resume,
 	.iterate_devices = dmz_iterate_devices,
+	.status		 = dmz_status,
+	.message	 = dmz_message,
 };
 
 static int __init dmz_init(void)
diff --git a/drivers/md/dm-zoned.h b/drivers/md/dm-zoned.h
index 5b5e493d479c..22f11440b423 100644
--- a/drivers/md/dm-zoned.h
+++ b/drivers/md/dm-zoned.h
@@ -45,34 +45,50 @@
 #define dmz_bio_block(bio)	dmz_sect2blk((bio)->bi_iter.bi_sector)
 #define dmz_bio_blocks(bio)	dmz_sect2blk(bio_sectors(bio))
 
+struct dmz_metadata;
+struct dmz_reclaim;
+
 /*
  * Zoned block device information.
  */
 struct dmz_dev {
 	struct block_device	*bdev;
+	struct dmz_metadata	*metadata;
+	struct dmz_reclaim	*reclaim;
 
 	char			name[BDEVNAME_SIZE];
+	uuid_t			uuid;
 
 	sector_t		capacity;
 
+	unsigned int		dev_idx;
+
 	unsigned int		nr_zones;
+	unsigned int		zone_offset;
 
 	unsigned int		flags;
 
 	sector_t		zone_nr_sectors;
-	unsigned int		zone_nr_sectors_shift;
 
-	sector_t		zone_nr_blocks;
-	sector_t		zone_nr_blocks_shift;
+	unsigned int		nr_rnd;
+	atomic_t		unmap_nr_rnd;
+	struct list_head	unmap_rnd_list;
+	struct list_head	map_rnd_list;
+
+	unsigned int		nr_seq;
+	atomic_t		unmap_nr_seq;
+	struct list_head	unmap_seq_list;
+	struct list_head	map_seq_list;
 };
 
-#define dmz_bio_chunk(dev, bio)	((bio)->bi_iter.bi_sector >> \
-				 (dev)->zone_nr_sectors_shift)
-#define dmz_chunk_block(dev, b)	((b) & ((dev)->zone_nr_blocks - 1))
+#define dmz_bio_chunk(zmd, bio)	((bio)->bi_iter.bi_sector >> \
+				 dmz_zone_nr_sectors_shift(zmd))
+#define dmz_chunk_block(zmd, b)	((b) & (dmz_zone_nr_blocks(zmd) - 1))
 
 /* Device flags. */
 #define DMZ_BDEV_DYING		(1 << 0)
 #define DMZ_CHECK_BDEV		(2 << 0)
+#define DMZ_BDEV_REGULAR	(4 << 0)
 
 /*
  * Zone descriptor.
@@ -81,12 +97,18 @@ struct dm_zone {
 	/* For listing the zone depending on its state */
 	struct list_head	link;
 
+	/* Device containing this zone */
+	struct dmz_dev		*dev;
+
 	/* Zone type and state */
 	unsigned long		flags;
 
 	/* Zone activation reference count */
 	atomic_t		refcount;
 
+	/* Zone id */
+	unsigned int		id;
+
 	/* Zone write pointer block (relative to the zone start block) */
 	unsigned int		wp_block;
 
@@ -109,6 +131,7 @@ struct dm_zone {
  */
 enum {
 	/* Zone write type */
+	DMZ_CACHE,
 	DMZ_RND,
 	DMZ_SEQ,
 
@@ -120,22 +143,28 @@ enum {
 	DMZ_META,
 	DMZ_DATA,
 	DMZ_BUF,
+	DMZ_RESERVED,
 
 	/* Zone internal state */
 	DMZ_RECLAIM,
 	DMZ_SEQ_WRITE_ERR,
+	DMZ_RECLAIM_TERMINATE,
 };
 
 /*
  * Zone data accessors.
  */
+#define dmz_is_cache(z)		test_bit(DMZ_CACHE, &(z)->flags)
 #define dmz_is_rnd(z)		test_bit(DMZ_RND, &(z)->flags)
 #define dmz_is_seq(z)		test_bit(DMZ_SEQ, &(z)->flags)
 #define dmz_is_empty(z)		((z)->wp_block == 0)
 #define dmz_is_offline(z)	test_bit(DMZ_OFFLINE, &(z)->flags)
 #define dmz_is_readonly(z)	test_bit(DMZ_READ_ONLY, &(z)->flags)
 #define dmz_in_reclaim(z)	test_bit(DMZ_RECLAIM, &(z)->flags)
+#define dmz_is_reserved(z)	test_bit(DMZ_RESERVED, &(z)->flags)
 #define dmz_seq_write_err(z)	test_bit(DMZ_SEQ_WRITE_ERR, &(z)->flags)
+#define dmz_reclaim_should_terminate(z) \
+				test_bit(DMZ_RECLAIM_TERMINATE, &(z)->flags)
 
 #define dmz_is_meta(z)		test_bit(DMZ_META, &(z)->flags)
 #define dmz_is_buf(z)		test_bit(DMZ_BUF, &(z)->flags)
@@ -158,13 +187,11 @@ enum {
 #define dmz_dev_debug(dev, format, args...)	\
 	DMDEBUG("(%s): " format, (dev)->name, ## args)
 
-struct dmz_metadata;
-struct dmz_reclaim;
-
 /*
  * Functions defined in dm-zoned-metadata.c
  */
-int dmz_ctr_metadata(struct dmz_dev *dev, struct dmz_metadata **zmd);
+int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev,
+		     struct dmz_metadata **zmd, const char *devname);
 void dmz_dtr_metadata(struct dmz_metadata *zmd);
 int dmz_resume_metadata(struct dmz_metadata *zmd);
 
@@ -175,23 +202,38 @@ void dmz_unlock_metadata(struct dmz_metadata *zmd);
 void dmz_lock_flush(struct dmz_metadata *zmd);
 void dmz_unlock_flush(struct dmz_metadata *zmd);
 int dmz_flush_metadata(struct dmz_metadata *zmd);
+const char *dmz_metadata_label(struct dmz_metadata *zmd);
 
-unsigned int dmz_id(struct dmz_metadata *zmd, struct dm_zone *zone);
 sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone);
 sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone);
 unsigned int dmz_nr_chunks(struct dmz_metadata *zmd);
 
+bool dmz_check_dev(struct dmz_metadata *zmd);
+bool dmz_dev_is_dying(struct dmz_metadata *zmd);
+
 #define DMZ_ALLOC_RND		0x01
-#define DMZ_ALLOC_RECLAIM	0x02
+#define DMZ_ALLOC_CACHE		0x02
+#define DMZ_ALLOC_SEQ		0x04
+#define DMZ_ALLOC_RECLAIM	0x10
 
-struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned long flags);
+struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd,
+			       unsigned int dev_idx, unsigned long flags);
 void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone);
 
 void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *zone,
 		  unsigned int chunk);
 void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone);
-unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd);
-unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd);
+unsigned int dmz_nr_zones(struct dmz_metadata *zmd);
+unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd);
+unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd);
+unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx);
+unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx);
+unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx);
+unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx);
+unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd);
+unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd);
+unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd);
+unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd);
 
 /*
  * Activate a zone (increment its reference count).
@@ -201,26 +243,10 @@ static inline void dmz_activate_zone(struct dm_zone *zone)
 	atomic_inc(&zone->refcount);
 }
 
-/*
- * Deactivate a zone. This decrement the zone reference counter
- * indicating that all BIOs to the zone have completed when the count is 0.
- */
-static inline void dmz_deactivate_zone(struct dm_zone *zone)
-{
-	atomic_dec(&zone->refcount);
-}
-
-/*
- * Test if a zone is active, that is, has a refcount > 0.
- */
-static inline bool dmz_is_active(struct dm_zone *zone)
-{
-	return atomic_read(&zone->refcount);
-}
-
 int dmz_lock_zone_reclaim(struct dm_zone *zone);
 void dmz_unlock_zone_reclaim(struct dm_zone *zone);
-struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd);
+struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,
+					 unsigned int dev_idx, bool idle);
 
 struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,
 				      unsigned int chunk, int op);
@@ -244,8 +270,7 @@ int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
 /*
  * Functions defined in dm-zoned-reclaim.c
  */
-int dmz_ctr_reclaim(struct dmz_dev *dev, struct dmz_metadata *zmd,
-		    struct dmz_reclaim **zrc);
+int dmz_ctr_reclaim(struct dmz_metadata *zmd, struct dmz_reclaim **zrc, int idx);
 void dmz_dtr_reclaim(struct dmz_reclaim *zrc);
 void dmz_suspend_reclaim(struct dmz_reclaim *zrc);
 void dmz_resume_reclaim(struct dmz_reclaim *zrc);
@@ -258,4 +283,22 @@ void dmz_schedule_reclaim(struct dmz_reclaim *zrc);
 bool dmz_bdev_is_dying(struct dmz_dev *dmz_dev);
 bool dmz_check_bdev(struct dmz_dev *dmz_dev);
 
+/*
+ * Deactivate a zone. This decrement the zone reference counter
+ * indicating that all BIOs to the zone have completed when the count is 0.
+ */
+static inline void dmz_deactivate_zone(struct dm_zone *zone)
+{
+	dmz_reclaim_bio_acc(zone->dev->reclaim);
+	atomic_dec(&zone->refcount);
+}
+
+/*
+ * Test if a zone is active, that is, has a refcount > 0.
+ */
+static inline bool dmz_is_active(struct dm_zone *zone)
+{
+	return atomic_read(&zone->refcount);
+}
+
 #endif /* DM_ZONED_H */
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 3f39fa1ac756..109e81f33edb 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -676,6 +676,15 @@ static bool md_in_flight(struct mapped_device *md)
 		return md_in_flight_bios(md);
 }
 
+u64 dm_start_time_ns_from_clone(struct bio *bio)
+{
+	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
+	struct dm_io *io = tio->io;
+
+	return jiffies_to_nsecs(io->start_time);
+}
+EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
+
 static void start_io_acct(struct dm_io *io)
 {
 	struct mapped_device *md = io->md;
@@ -2610,7 +2619,7 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
 	if (noflush)
 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
 	else
-		pr_debug("%s: suspending with flush\n", dm_device_name(md));
+		DMDEBUG("%s: suspending with flush", dm_device_name(md));
 
 	/*
 	 * This gets reverted if there's an error later and the targets
diff --git a/drivers/md/persistent-data/Kconfig b/drivers/md/persistent-data/Kconfig
index baaec1ae29c1..f4f948b0e173 100644
--- a/drivers/md/persistent-data/Kconfig
+++ b/drivers/md/persistent-data/Kconfig
@@ -4,7 +4,7 @@ config DM_PERSISTENT_DATA
        depends on BLK_DEV_DM
        select LIBCRC32C
        select DM_BUFIO
-       ---help---
+	help
 	 Library providing immutable on-disk data structure support for
 	 device-mapper targets such as the thin provisioning target.
 
diff --git a/drivers/md/persistent-data/dm-btree-internal.h b/drivers/md/persistent-data/dm-btree-internal.h
index a240990a7f33..564896659dd4 100644
--- a/drivers/md/persistent-data/dm-btree-internal.h
+++ b/drivers/md/persistent-data/dm-btree-internal.h
@@ -38,7 +38,7 @@ struct node_header {
 
 struct btree_node {
 	struct node_header header;
-	__le64 keys[0];
+	__le64 keys[];
 } __packed;
 
 
@@ -68,7 +68,7 @@ struct ro_spine {
 };
 
 void init_ro_spine(struct ro_spine *s, struct dm_btree_info *info);
-int exit_ro_spine(struct ro_spine *s);
+void exit_ro_spine(struct ro_spine *s);
 int ro_step(struct ro_spine *s, dm_block_t new_child);
 void ro_pop(struct ro_spine *s);
 struct btree_node *ro_node(struct ro_spine *s);
diff --git a/drivers/md/persistent-data/dm-btree-spine.c b/drivers/md/persistent-data/dm-btree-spine.c
index b27b8091a1ca..e03cb9e48773 100644
--- a/drivers/md/persistent-data/dm-btree-spine.c
+++ b/drivers/md/persistent-data/dm-btree-spine.c
@@ -132,15 +132,13 @@ void init_ro_spine(struct ro_spine *s, struct dm_btree_info *info)
 	s->nodes[1] = NULL;
 }
 
-int exit_ro_spine(struct ro_spine *s)
+void exit_ro_spine(struct ro_spine *s)
 {
-	int r = 0, i;
+	int i;
 
 	for (i = 0; i < s->count; i++) {
 		unlock_block(s->info, s->nodes[i]);
 	}
-
-	return r;
 }
 
 int ro_step(struct ro_spine *s, dm_block_t new_child)
diff --git a/drivers/media/cec/Kconfig b/drivers/media/cec/Kconfig
index eea74b7cfa8c..7e830444bdbb 100644
--- a/drivers/media/cec/Kconfig
+++ b/drivers/media/cec/Kconfig
@@ -24,7 +24,7 @@ config CEC_PIN_ERROR_INJ
 menuconfig MEDIA_CEC_SUPPORT
 	bool
 	prompt "HDMI CEC drivers"
-	default y if !MEDIA_SUPPORT_FILTER
+	default y if MEDIA_SUPPORT && !MEDIA_SUPPORT_FILTER
 	help
 	  Enable support for HDMI CEC (Consumer Electronics Control),
 	  which is an optional HDMI feature.
diff --git a/drivers/media/cec/platform/Kconfig b/drivers/media/cec/platform/Kconfig
index 350533cd8261..b672d3142eb7 100644
--- a/drivers/media/cec/platform/Kconfig
+++ b/drivers/media/cec/platform/Kconfig
@@ -32,7 +32,7 @@ config CEC_MESON_G12A_AO
 	select REGMAP_MMIO
 	select CEC_CORE
 	select CEC_NOTIFIER
-	---help---
+	help
 	  This is a driver for Amlogic Meson G12A SoCs AO CEC interface.
 	  This driver if for the new AO-CEC module found in G12A SoCs,
 	  usually named AO_CEC_B in documentation.
diff --git a/drivers/media/common/videobuf2/videobuf2-dma-contig.c b/drivers/media/common/videobuf2/videobuf2-dma-contig.c
index d3a3ee5b597b..f4b4a7c135eb 100644
--- a/drivers/media/common/videobuf2/videobuf2-dma-contig.c
+++ b/drivers/media/common/videobuf2/videobuf2-dma-contig.c
@@ -726,9 +726,8 @@ EXPORT_SYMBOL_GPL(vb2_dma_contig_memops);
 int vb2_dma_contig_set_max_seg_size(struct device *dev, unsigned int size)
 {
 	if (!dev->dma_parms) {
-		dev->dma_parms = kzalloc(sizeof(*dev->dma_parms), GFP_KERNEL);
-		if (!dev->dma_parms)
-			return -ENOMEM;
+		dev_err(dev, "Failed to set max_seg_size: dma_parms is NULL\n");
+		return -ENODEV;
 	}
 	if (dma_get_max_seg_size(dev) < size)
 		return dma_set_max_seg_size(dev, size);
@@ -737,21 +736,6 @@ int vb2_dma_contig_set_max_seg_size(struct device *dev, unsigned int size)
 }
 EXPORT_SYMBOL_GPL(vb2_dma_contig_set_max_seg_size);
 
-/*
- * vb2_dma_contig_clear_max_seg_size() - release resources for DMA parameters
- * @dev:	device for configuring DMA parameters
- *
- * This function releases resources allocated to configure DMA parameters
- * (see vb2_dma_contig_set_max_seg_size() function). It should be called from
- * device drivers on driver remove.
- */
-void vb2_dma_contig_clear_max_seg_size(struct device *dev)
-{
-	kfree(dev->dma_parms);
-	dev->dma_parms = NULL;
-}
-EXPORT_SYMBOL_GPL(vb2_dma_contig_clear_max_seg_size);
-
 MODULE_DESCRIPTION("DMA-contig memory handling routines for videobuf2");
 MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/pci/bt8xx/bt878.c b/drivers/media/pci/bt8xx/bt878.c
index 53af26ad1dfb..79ba15a9385a 100644
--- a/drivers/media/pci/bt8xx/bt878.c
+++ b/drivers/media/pci/bt8xx/bt878.c
@@ -13,9 +13,9 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
+#include <linux/pgtable.h>
 #include <asm/io.h>
 #include <linux/ioport.h>
-#include <asm/pgtable.h>
 #include <asm/page.h>
 #include <linux/types.h>
 #include <linux/interrupt.h>
diff --git a/drivers/media/pci/bt8xx/btcx-risc.c b/drivers/media/pci/bt8xx/btcx-risc.c
index 1139a5ad2418..51257980f539 100644
--- a/drivers/media/pci/bt8xx/btcx-risc.c
+++ b/drivers/media/pci/bt8xx/btcx-risc.c
@@ -17,8 +17,8 @@
 #include <linux/pci.h>
 #include <linux/interrupt.h>
 #include <linux/videodev2.h>
+#include <linux/pgtable.h>
 #include <asm/page.h>
-#include <asm/pgtable.h>
 
 #include "btcx-risc.h"
 
diff --git a/drivers/media/pci/bt8xx/bttv-risc.c b/drivers/media/pci/bt8xx/bttv-risc.c
index fc8708047be8..4af72826b006 100644
--- a/drivers/media/pci/bt8xx/bttv-risc.c
+++ b/drivers/media/pci/bt8xx/bttv-risc.c
@@ -20,8 +20,8 @@
 #include <linux/pci.h>
 #include <linux/vmalloc.h>
 #include <linux/interrupt.h>
+#include <linux/pgtable.h>
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <media/v4l2-ioctl.h>
 
 #include "bttvp.h"
diff --git a/drivers/media/platform/davinci/vpbe_display.c b/drivers/media/platform/davinci/vpbe_display.c
index 38d3088d4d38..7ab13eb7527d 100644
--- a/drivers/media/platform/davinci/vpbe_display.c
+++ b/drivers/media/platform/davinci/vpbe_display.c
@@ -17,7 +17,6 @@
 #include <linux/videodev2.h>
 #include <linux/slab.h>
 
-#include <asm/pgtable.h>
 
 #include <media/v4l2-dev.h>
 #include <media/v4l2-common.h>
diff --git a/drivers/media/platform/omap3isp/ispvideo.c b/drivers/media/platform/omap3isp/ispvideo.c
index 6f769c527fae..10c214bd0903 100644
--- a/drivers/media/platform/omap3isp/ispvideo.c
+++ b/drivers/media/platform/omap3isp/ispvideo.c
@@ -10,7 +10,6 @@
  *	     Sakari Ailus <sakari.ailus@iki.fi>
  */
 
-#include <asm/cacheflush.h>
 #include <linux/clk.h>
 #include <linux/mm.h>
 #include <linux/module.h>
@@ -19,6 +18,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
+#include <asm/cacheflush.h>
 
 #include <media/v4l2-dev.h>
 #include <media/v4l2-ioctl.h>
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc.c b/drivers/media/platform/s5p-mfc/s5p_mfc.c
index 5c2a23b953a4..eba2b9f040df 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc.c
@@ -1089,6 +1089,10 @@ static struct device *s5p_mfc_alloc_memdev(struct device *dev,
 	child->coherent_dma_mask = dev->coherent_dma_mask;
 	child->dma_mask = dev->dma_mask;
 	child->release = s5p_mfc_memdev_release;
+	child->dma_parms = devm_kzalloc(dev, sizeof(*child->dma_parms),
+					GFP_KERNEL);
+	if (!child->dma_parms)
+		goto err;
 
 	/*
 	 * The memdevs are not proper OF platform devices, so in order for them
@@ -1104,7 +1108,7 @@ static struct device *s5p_mfc_alloc_memdev(struct device *dev,
 			return child;
 		device_del(child);
 	}
-
+err:
 	put_device(child);
 	return NULL;
 }
diff --git a/drivers/media/usb/pwc/pwc.h b/drivers/media/usb/pwc/pwc.h
index 3362962d0d00..b02a3c7b7742 100644
--- a/drivers/media/usb/pwc/pwc.h
+++ b/drivers/media/usb/pwc/pwc.h
@@ -193,7 +193,7 @@ struct pwc_raw_frame {
 				   decompressor) */
 	__u8   cmd[4];		/* the four byte of the command (in case of
 				   nala, only the first 3 bytes is filled) */
-	__u8   rawframe[0];	/* frame_size = H / 4 * vbandlength */
+	__u8   rawframe[];	/* frame_size = H / 4 * vbandlength */
 } __packed;
 
 /* intermediate buffers with raw data from the USB cam */
diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c
index 9e8eb45a5b03..3dc17ebe14fa 100644
--- a/drivers/media/v4l2-core/v4l2-common.c
+++ b/drivers/media/v4l2-core/v4l2-common.c
@@ -41,7 +41,6 @@
 #include <linux/string.h>
 #include <linux/errno.h>
 #include <linux/uaccess.h>
-#include <asm/pgtable.h>
 #include <asm/io.h>
 #include <asm/div64.h>
 #include <media/v4l2-common.h>
diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c
index b188577db40f..3f3fbcd60cc6 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls.c
@@ -1844,7 +1844,7 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 			       sizeof(p_hevc_pps->row_height_minus1));
 
 			p_hevc_pps->flags &=
-				~V4L2_HEVC_PPS_FLAG_PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED;
+				~V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED;
 		}
 
 		if (p_hevc_pps->flags &
diff --git a/drivers/media/v4l2-core/videobuf-core.c b/drivers/media/v4l2-core/videobuf-core.c
index 2686f03b322e..5c91fc3e65b5 100644
--- a/drivers/media/v4l2-core/videobuf-core.c
+++ b/drivers/media/v4l2-core/videobuf-core.c
@@ -535,7 +535,7 @@ int videobuf_qbuf(struct videobuf_queue *q, struct v4l2_buffer *b)
 	MAGIC_CHECK(q->int_ops->magic, MAGIC_QTYPE_OPS);
 
 	if (b->memory == V4L2_MEMORY_MMAP)
-		down_read(&current->mm->mmap_sem);
+		mmap_read_lock(current->mm);
 
 	videobuf_queue_lock(q);
 	retval = -EBUSY;
@@ -622,7 +622,7 @@ done:
 	videobuf_queue_unlock(q);
 
 	if (b->memory == V4L2_MEMORY_MMAP)
-		up_read(&current->mm->mmap_sem);
+		mmap_read_unlock(current->mm);
 
 	return retval;
 }
diff --git a/drivers/media/v4l2-core/videobuf-dma-contig.c b/drivers/media/v4l2-core/videobuf-dma-contig.c
index aeb2f497c683..52312ce2ba05 100644
--- a/drivers/media/v4l2-core/videobuf-dma-contig.c
+++ b/drivers/media/v4l2-core/videobuf-dma-contig.c
@@ -169,7 +169,7 @@ static int videobuf_dma_contig_user_get(struct videobuf_dma_contig_memory *mem,
 	mem->size = PAGE_ALIGN(vb->size + offset);
 	ret = -EINVAL;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 
 	vma = find_vma(mm, untagged_baddr);
 	if (!vma)
@@ -201,7 +201,7 @@ static int videobuf_dma_contig_user_get(struct videobuf_dma_contig_memory *mem,
 	}
 
 out_up:
-	up_read(&current->mm->mmap_sem);
+	mmap_read_unlock(current->mm);
 
 	return ret;
 }
diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c
index 13b65ed9e74c..46ff19df9f53 100644
--- a/drivers/media/v4l2-core/videobuf-dma-sg.c
+++ b/drivers/media/v4l2-core/videobuf-dma-sg.c
@@ -21,13 +21,13 @@
 #include <linux/sched/mm.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
+#include <linux/pgtable.h>
 
 #include <linux/dma-mapping.h>
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <linux/scatterlist.h>
 #include <asm/page.h>
-#include <asm/pgtable.h>
 
 #include <media/videobuf-dma-sg.h>
 
@@ -200,9 +200,9 @@ static int videobuf_dma_init_user(struct videobuf_dmabuf *dma, int direction,
 {
 	int ret;
 
-	down_read(&current->mm->mmap_sem);
+	mmap_read_lock(current->mm);
 	ret = videobuf_dma_init_user_locked(dma, direction, data, size);
-	up_read(&current->mm->mmap_sem);
+	mmap_read_unlock(current->mm);
 
 	return ret;
 }
@@ -533,7 +533,7 @@ static int __videobuf_iolock(struct videobuf_queue *q,
 		} else {
 			/* NOTE: HACK: videobuf_iolock on V4L2_MEMORY_MMAP
 			buffers can only be called from videobuf_qbuf
-			we take current->mm->mmap_sem there, to prevent
+			we take current->mm->mmap_lock there, to prevent
 			locking inversion, so don't take it here */
 
 			err = videobuf_dma_init_user_locked(&mem->dma,
diff --git a/drivers/media/v4l2-core/videobuf-vmalloc.c b/drivers/media/v4l2-core/videobuf-vmalloc.c
index f8bd5a369560..9b2443720ab0 100644
--- a/drivers/media/v4l2-core/videobuf-vmalloc.c
+++ b/drivers/media/v4l2-core/videobuf-vmalloc.c
@@ -15,12 +15,12 @@
 #include <linux/moduleparam.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
+#include <linux/pgtable.h>
 
 #include <linux/pci.h>
 #include <linux/vmalloc.h>
 #include <linux/pagemap.h>
 #include <asm/page.h>
-#include <asm/pgtable.h>
 
 #include <media/videobuf-vmalloc.h>
 
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index 9bddca292330..04368ee2a809 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -46,6 +46,17 @@ config ATMEL_EBI
 	  tree is used. This bus supports NANDs, external ethernet controller,
 	  SRAMs, ATA devices, etc.
 
+config BT1_L2_CTL
+	bool "Baikal-T1 CM2 L2-RAM Cache Control Block"
+	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
+	select MFD_SYSCON
+	help
+	  Baikal-T1 CPU is based on the MIPS P5600 Warrior IP-core. The CPU
+	  resides Coherency Manager v2 with embedded 1MB L2-cache. It's
+	  possible to tune the L2 cache performance up by setting the data,
+	  tags and way-select latencies of RAM access. This driver provides a
+	  dt properties-based and sysfs interface for it.
+
 config TI_AEMIF
 	tristate "Texas Instruments AEMIF driver"
 	depends on (ARCH_DAVINCI || ARCH_KEYSTONE) && OF
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index 27b493435e61..6d7e3e64ba62 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_ARM_PL172_MPMC)	+= pl172.o
 obj-$(CONFIG_ATMEL_SDRAMC)	+= atmel-sdramc.o
 obj-$(CONFIG_ATMEL_EBI)		+= atmel-ebi.o
 obj-$(CONFIG_ARCH_BRCMSTB)	+= brcmstb_dpfe.o
+obj-$(CONFIG_BT1_L2_CTL)	+= bt1-l2-ctl.o
 obj-$(CONFIG_TI_AEMIF)		+= ti-aemif.o
 obj-$(CONFIG_TI_EMIF)		+= emif.o
 obj-$(CONFIG_OMAP_GPMC)		+= omap-gpmc.o
diff --git a/drivers/memory/bt1-l2-ctl.c b/drivers/memory/bt1-l2-ctl.c
new file mode 100644
index 000000000000..633fea6a4edf
--- /dev/null
+++ b/drivers/memory/bt1-l2-ctl.c
@@ -0,0 +1,322 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+ *
+ * Authors:
+ *   Serge Semin <Sergey.Semin@baikalelectronics.ru>
+ *
+ * Baikal-T1 CM2 L2-cache Control Block driver.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/bitfield.h>
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+#include <linux/sysfs.h>
+#include <linux/of.h>
+
+#define L2_CTL_REG			0x028
+#define L2_CTL_DATA_STALL_FLD		0
+#define L2_CTL_DATA_STALL_MASK		GENMASK(1, L2_CTL_DATA_STALL_FLD)
+#define L2_CTL_TAG_STALL_FLD		2
+#define L2_CTL_TAG_STALL_MASK		GENMASK(3, L2_CTL_TAG_STALL_FLD)
+#define L2_CTL_WS_STALL_FLD		4
+#define L2_CTL_WS_STALL_MASK		GENMASK(5, L2_CTL_WS_STALL_FLD)
+#define L2_CTL_SET_CLKRATIO		BIT(13)
+#define L2_CTL_CLKRATIO_LOCK		BIT(31)
+
+#define L2_CTL_STALL_MIN		0
+#define L2_CTL_STALL_MAX		3
+#define L2_CTL_STALL_SET_DELAY_US	1
+#define L2_CTL_STALL_SET_TOUT_US	1000
+
+/*
+ * struct l2_ctl - Baikal-T1 L2 Control block private data.
+ * @dev: Pointer to the device structure.
+ * @sys_regs: Baikal-T1 System Controller registers map.
+ */
+struct l2_ctl {
+	struct device *dev;
+
+	struct regmap *sys_regs;
+};
+
+/*
+ * enum l2_ctl_stall - Baikal-T1 L2-cache-RAM stall identifier.
+ * @L2_WSSTALL: Way-select latency.
+ * @L2_TAGSTALL: Tag latency.
+ * @L2_DATASTALL: Data latency.
+ */
+enum l2_ctl_stall {
+	L2_WS_STALL,
+	L2_TAG_STALL,
+	L2_DATA_STALL
+};
+
+/*
+ * struct l2_ctl_device_attribute - Baikal-T1 L2-cache device attribute.
+ * @dev_attr: Actual sysfs device attribute.
+ * @id: L2-cache stall field identifier.
+ */
+struct l2_ctl_device_attribute {
+	struct device_attribute dev_attr;
+	enum l2_ctl_stall id;
+};
+#define to_l2_ctl_dev_attr(_dev_attr) \
+	container_of(_dev_attr, struct l2_ctl_device_attribute, dev_attr)
+
+#define L2_CTL_ATTR_RW(_name, _prefix, _id) \
+	struct l2_ctl_device_attribute l2_ctl_attr_##_name = \
+		{ __ATTR(_name, 0644, _prefix##_show, _prefix##_store),	_id }
+
+static int l2_ctl_get_latency(struct l2_ctl *l2, enum l2_ctl_stall id, u32 *val)
+{
+	u32 data = 0;
+	int ret;
+
+	ret = regmap_read(l2->sys_regs, L2_CTL_REG, &data);
+	if (ret)
+		return ret;
+
+	switch (id) {
+	case L2_WS_STALL:
+		*val = FIELD_GET(L2_CTL_WS_STALL_MASK, data);
+		break;
+	case L2_TAG_STALL:
+		*val = FIELD_GET(L2_CTL_TAG_STALL_MASK, data);
+		break;
+	case L2_DATA_STALL:
+		*val = FIELD_GET(L2_CTL_DATA_STALL_MASK, data);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int l2_ctl_set_latency(struct l2_ctl *l2, enum l2_ctl_stall id, u32 val)
+{
+	u32 mask = 0, data = 0;
+	int ret;
+
+	val = clamp_val(val, L2_CTL_STALL_MIN, L2_CTL_STALL_MAX);
+
+	switch (id) {
+	case L2_WS_STALL:
+		data = FIELD_PREP(L2_CTL_WS_STALL_MASK, val);
+		mask = L2_CTL_WS_STALL_MASK;
+		break;
+	case L2_TAG_STALL:
+		data = FIELD_PREP(L2_CTL_TAG_STALL_MASK, val);
+		mask = L2_CTL_TAG_STALL_MASK;
+		break;
+	case L2_DATA_STALL:
+		data = FIELD_PREP(L2_CTL_DATA_STALL_MASK, val);
+		mask = L2_CTL_DATA_STALL_MASK;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	data |= L2_CTL_SET_CLKRATIO;
+	mask |= L2_CTL_SET_CLKRATIO;
+
+	ret = regmap_update_bits(l2->sys_regs, L2_CTL_REG, mask, data);
+	if (ret)
+		return ret;
+
+	return regmap_read_poll_timeout(l2->sys_regs, L2_CTL_REG, data,
+					data & L2_CTL_CLKRATIO_LOCK,
+					L2_CTL_STALL_SET_DELAY_US,
+					L2_CTL_STALL_SET_TOUT_US);
+}
+
+static void l2_ctl_clear_data(void *data)
+{
+	struct l2_ctl *l2 = data;
+	struct platform_device *pdev = to_platform_device(l2->dev);
+
+	platform_set_drvdata(pdev, NULL);
+}
+
+static struct l2_ctl *l2_ctl_create_data(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct l2_ctl *l2;
+	int ret;
+
+	l2 = devm_kzalloc(dev, sizeof(*l2), GFP_KERNEL);
+	if (!l2)
+		return ERR_PTR(-ENOMEM);
+
+	ret = devm_add_action(dev, l2_ctl_clear_data, l2);
+	if (ret) {
+		dev_err(dev, "Can't add L2 CTL data clear action\n");
+		return ERR_PTR(ret);
+	}
+
+	l2->dev = dev;
+	platform_set_drvdata(pdev, l2);
+
+	return l2;
+}
+
+static int l2_ctl_find_sys_regs(struct l2_ctl *l2)
+{
+	l2->sys_regs = syscon_node_to_regmap(l2->dev->of_node->parent);
+	if (IS_ERR(l2->sys_regs)) {
+		dev_err(l2->dev, "Couldn't get L2 CTL register map\n");
+		return PTR_ERR(l2->sys_regs);
+	}
+
+	return 0;
+}
+
+static int l2_ctl_of_parse_property(struct l2_ctl *l2, enum l2_ctl_stall id,
+				    const char *propname)
+{
+	int ret = 0;
+	u32 data;
+
+	if (!of_property_read_u32(l2->dev->of_node, propname, &data)) {
+		ret = l2_ctl_set_latency(l2, id, data);
+		if (ret)
+			dev_err(l2->dev, "Invalid value of '%s'\n", propname);
+	}
+
+	return ret;
+}
+
+static int l2_ctl_of_parse(struct l2_ctl *l2)
+{
+	int ret;
+
+	ret = l2_ctl_of_parse_property(l2, L2_WS_STALL, "baikal,l2-ws-latency");
+	if (ret)
+		return ret;
+
+	ret = l2_ctl_of_parse_property(l2, L2_TAG_STALL, "baikal,l2-tag-latency");
+	if (ret)
+		return ret;
+
+	return l2_ctl_of_parse_property(l2, L2_DATA_STALL,
+					"baikal,l2-data-latency");
+}
+
+static ssize_t l2_ctl_latency_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct l2_ctl_device_attribute *devattr = to_l2_ctl_dev_attr(attr);
+	struct l2_ctl *l2 = dev_get_drvdata(dev);
+	u32 data;
+	int ret;
+
+	ret = l2_ctl_get_latency(l2, devattr->id, &data);
+	if (ret)
+		return ret;
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", data);
+}
+
+static ssize_t l2_ctl_latency_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	struct l2_ctl_device_attribute *devattr = to_l2_ctl_dev_attr(attr);
+	struct l2_ctl *l2 = dev_get_drvdata(dev);
+	u32 data;
+	int ret;
+
+	if (kstrtouint(buf, 0, &data) < 0)
+		return -EINVAL;
+
+	ret = l2_ctl_set_latency(l2, devattr->id, data);
+	if (ret)
+		return ret;
+
+	return count;
+}
+static L2_CTL_ATTR_RW(l2_ws_latency, l2_ctl_latency, L2_WS_STALL);
+static L2_CTL_ATTR_RW(l2_tag_latency, l2_ctl_latency, L2_TAG_STALL);
+static L2_CTL_ATTR_RW(l2_data_latency, l2_ctl_latency, L2_DATA_STALL);
+
+static struct attribute *l2_ctl_sysfs_attrs[] = {
+	&l2_ctl_attr_l2_ws_latency.dev_attr.attr,
+	&l2_ctl_attr_l2_tag_latency.dev_attr.attr,
+	&l2_ctl_attr_l2_data_latency.dev_attr.attr,
+	NULL
+};
+ATTRIBUTE_GROUPS(l2_ctl_sysfs);
+
+static void l2_ctl_remove_sysfs(void *data)
+{
+	struct l2_ctl *l2 = data;
+
+	device_remove_groups(l2->dev, l2_ctl_sysfs_groups);
+}
+
+static int l2_ctl_init_sysfs(struct l2_ctl *l2)
+{
+	int ret;
+
+	ret = device_add_groups(l2->dev, l2_ctl_sysfs_groups);
+	if (ret) {
+		dev_err(l2->dev, "Failed to create L2 CTL sysfs nodes\n");
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(l2->dev, l2_ctl_remove_sysfs, l2);
+	if (ret)
+		dev_err(l2->dev, "Can't add L2 CTL sysfs remove action\n");
+
+	return ret;
+}
+
+static int l2_ctl_probe(struct platform_device *pdev)
+{
+	struct l2_ctl *l2;
+	int ret;
+
+	l2 = l2_ctl_create_data(pdev);
+	if (IS_ERR(l2))
+		return PTR_ERR(l2);
+
+	ret = l2_ctl_find_sys_regs(l2);
+	if (ret)
+		return ret;
+
+	ret = l2_ctl_of_parse(l2);
+	if (ret)
+		return ret;
+
+	ret = l2_ctl_init_sysfs(l2);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id l2_ctl_of_match[] = {
+	{ .compatible = "baikal,bt1-l2-ctl" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, l2_ctl_of_match);
+
+static struct platform_driver l2_ctl_driver = {
+	.probe = l2_ctl_probe,
+	.driver = {
+		.name = "bt1-l2-ctl",
+		.of_match_table = l2_ctl_of_match
+	}
+};
+module_platform_driver(l2_ctl_driver);
+
+MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
+MODULE_DESCRIPTION("Baikal-T1 L2-cache driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/samsung/exynos5422-dmc.c b/drivers/memory/samsung/exynos5422-dmc.c
index 81a1b1d01683..25196d6268e2 100644
--- a/drivers/memory/samsung/exynos5422-dmc.c
+++ b/drivers/memory/samsung/exynos5422-dmc.c
@@ -1091,7 +1091,7 @@ static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
 	/* power related timings */
 	val = dmc->timings->tFAW / clk_period_ps;
 	val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
-	val = max(val, dmc->min_tck->tXP);
+	val = max(val, dmc->min_tck->tFAW);
 	reg = &timing_power[0];
 	*reg_timing_power |= TIMING_VAL2REG(reg, val);
 
@@ -1346,15 +1346,13 @@ static irqreturn_t dmc_irq_thread(int irq, void *priv)
 	struct exynos5_dmc *dmc = priv;
 
 	mutex_lock(&dmc->df->lock);
-
 	exynos5_dmc_perf_events_check(dmc);
-
 	res = update_devfreq(dmc->df);
+	mutex_unlock(&dmc->df->lock);
+
 	if (res)
 		dev_warn(dmc->dev, "devfreq failed with %d\n", res);
 
-	mutex_unlock(&dmc->df->lock);
-
 	return IRQ_HANDLED;
 }
 
diff --git a/drivers/message/fusion/Kconfig b/drivers/message/fusion/Kconfig
index ba770c5ea8ef..a3d0288fd0e2 100644
--- a/drivers/message/fusion/Kconfig
+++ b/drivers/message/fusion/Kconfig
@@ -3,7 +3,7 @@
 menuconfig FUSION
 	bool "Fusion MPT device support"
 	depends on PCI
-	---help---
+	help
 	Say Y here to get to see options for Fusion Message
 	Passing Technology (MPT) drivers.
 	This option alone does not add any kernel code.
@@ -16,7 +16,7 @@ config FUSION_SPI
 	tristate "Fusion MPT ScsiHost drivers for SPI"
 	depends on PCI && SCSI
 	select SCSI_SPI_ATTRS
-	---help---
+	help
 	  SCSI HOST support for a parallel SCSI host adapters.
 
 	  List of supported controllers:
@@ -31,7 +31,7 @@ config FUSION_FC
 	tristate "Fusion MPT ScsiHost drivers for FC"
 	depends on PCI && SCSI
 	depends on SCSI_FC_ATTRS
-	---help---
+	help
 	  SCSI HOST support for a Fiber Channel host adapters.
 
 	  List of supported controllers:
@@ -50,7 +50,7 @@ config FUSION_SAS
 	tristate "Fusion MPT ScsiHost drivers for SAS"
 	depends on PCI && SCSI
 	select SCSI_SAS_ATTRS
-	---help---
+	help
 	  SCSI HOST support for a SAS host adapters.
 
 	  List of supported controllers:
@@ -75,7 +75,7 @@ config FUSION_MAX_SGE
 config FUSION_CTL
 	tristate "Fusion MPT misc device (ioctl) driver"
 	depends on FUSION_SPI || FUSION_FC || FUSION_SAS
-	---help---
+	help
 	  The Fusion MPT misc device driver provides specialized control
 	  of MPT adapters via system ioctl calls.  Use of ioctl calls to
 	  the MPT driver requires that you create and use a misc device
@@ -94,7 +94,7 @@ config FUSION_CTL
 config FUSION_LAN
 	tristate "Fusion MPT LAN driver"
 	depends on FUSION_FC && NET_FC
-	---help---
+	help
 	  This module supports LAN IP traffic over Fibre Channel port(s)
 	  on Fusion MPT compatible hardware (LSIFC9xx chips).
 	  The physical interface used is defined in RFC 2625.
@@ -110,7 +110,7 @@ config FUSION_LAN
 
 config FUSION_LOGGING
 	bool "Fusion MPT logging facility"
-	---help---
+	help
 	  This turns on a logging facility that can be used to debug a number
 	  of Fusion MPT related problems.
 
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index c2dd322691d1..68aea22f2b89 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -5052,9 +5052,11 @@ GetLanConfigPages(MPT_ADAPTER *ioc)
  *	@ioc: Pointer to MPT_ADAPTER structure
  *	@persist_opcode: see below
  *
- *	MPI_SAS_OP_CLEAR_NOT_PRESENT - Free all persist TargetID mappings for
- *		devices not currently present.
- *	MPI_SAS_OP_CLEAR_ALL_PERSISTENT - Clear al persist TargetID mappings
+ *	===============================  ======================================
+ *	MPI_SAS_OP_CLEAR_NOT_PRESENT     Free all persist TargetID mappings for
+ *					 devices not currently present.
+ *	MPI_SAS_OP_CLEAR_ALL_PERSISTENT  Clear al persist TargetID mappings
+ *	===============================  ======================================
  *
  *	NOTE: Don't use not this function during interrupt time.
  *
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index 687e9c848053..a37d7d171382 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -15,7 +15,7 @@ config MFD_CS5535
 	tristate "AMD CS5535 and CS5536 southbridge core functions"
 	select MFD_CORE
 	depends on PCI && (X86_32 || (X86 && COMPILE_TEST))
-	---help---
+	help
 	  This is the core driver for CS5535/CS5536 MFD functions.  This is
 	  necessary for using the board's GPIO and MFGPT functionality.
 
@@ -292,7 +292,7 @@ config MFD_ASIC3
 	bool "Compaq ASIC3"
 	depends on GPIOLIB && ARM
 	select MFD_CORE
-	 ---help---
+	help
 	  This driver supports the ASIC3 multifunction chip found on many
 	  PDAs (mainly iPAQ and HTC based ones)
 
@@ -449,6 +449,15 @@ config MFD_MC13XXX_I2C
 	help
 	  Select this if your MC13xxx is connected via an I2C bus.
 
+config MFD_MP2629
+	tristate "Monolithic Power Systems MP2629 ADC and Battery charger"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Select this option to enable support for Monolithic Power Systems
+	  battery charger. This provides ADC, thermal and battery charger power
+	  management functions.
+
 config MFD_MXS_LRADC
 	tristate "Freescale i.MX23/i.MX28 LRADC"
 	depends on ARCH_MXS || COMPILE_TEST
@@ -899,6 +908,18 @@ config MFD_MAX8998
 	  additional drivers must be enabled in order to use the functionality
 	  of the device.
 
+config MFD_MT6360
+	tristate "Mediatek MT6360 SubPMIC"
+	select MFD_CORE
+	select REGMAP_I2C
+	select REGMAP_IRQ
+	depends on I2C
+	help
+	  Say Y here to enable MT6360 PMU/PMIC/LDO functional support.
+	  PMU part includes Charger, Flashlight, RGB LED
+	  PMIC part includes 2-channel BUCKs and 2-channel LDOs
+	  LDO part includes 4-channel LDOs
+
 config MFD_MT6397
 	tristate "MediaTek MT6397 PMIC Support"
 	select MFD_CORE
@@ -1144,7 +1165,7 @@ config MFD_SI476X_CORE
 config MFD_SM501
 	tristate "Silicon Motion SM501"
 	depends on HAS_DMA
-	 ---help---
+	help
 	  This is the core driver for the Silicon Motion SM501 multimedia
 	  companion chip. This device is a multifunction device which may
 	  provide numerous interfaces including USB host controller, USB gadget,
@@ -1155,7 +1176,7 @@ config MFD_SM501
 config MFD_SM501_GPIO
 	bool "Export GPIO via GPIO layer"
 	depends on MFD_SM501 && GPIOLIB
-	 ---help---
+	help
 	  This option uses the gpio library layer to export the 64 GPIO
 	  lines on the SM501. The platform data is used to supply the
 	  base number for the first GPIO line to register.
@@ -1700,7 +1721,7 @@ config MFD_TIMBERDALE
 	tristate "Timberdale FPGA"
 	select MFD_CORE
 	depends on PCI && GPIOLIB && (X86_32 || COMPILE_TEST)
-	---help---
+	help
 	This is the core driver for the timberdale FPGA. This device is a
 	multifunction device which exposes numerous platform devices.
 
@@ -2057,10 +2078,9 @@ config MCP_UCB1200_TS
 endmenu
 
 config MFD_VEXPRESS_SYSREG
-	bool "Versatile Express System Registers"
-	depends on VEXPRESS_CONFIG && GPIOLIB && !ARCH_USES_GETTIMEOFFSET
+	tristate "Versatile Express System Registers"
+	depends on VEXPRESS_CONFIG && GPIOLIB
 	default y
-	select CLKSRC_MMIO
 	select GPIO_GENERIC_PLATFORM
 	select MFD_CORE
 	select MFD_SYSCON
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index bea2be419822..9367a92f795a 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -171,6 +171,8 @@ obj-$(CONFIG_MFD_MAX8925)	+= max8925.o
 obj-$(CONFIG_MFD_MAX8997)	+= max8997.o max8997-irq.o
 obj-$(CONFIG_MFD_MAX8998)	+= max8998.o max8998-irq.o
 
+obj-$(CONFIG_MFD_MP2629)	+= mp2629.o
+
 pcf50633-objs			:= pcf50633-core.o pcf50633-irq.o
 obj-$(CONFIG_MFD_PCF50633)	+= pcf50633.o
 obj-$(CONFIG_PCF50633_ADC)	+= pcf50633-adc.o
@@ -241,7 +243,8 @@ obj-$(CONFIG_INTEL_SOC_PMIC)	+= intel-soc-pmic.o
 obj-$(CONFIG_INTEL_SOC_PMIC_BXTWC)	+= intel_soc_pmic_bxtwc.o
 obj-$(CONFIG_INTEL_SOC_PMIC_CHTWC)	+= intel_soc_pmic_chtwc.o
 obj-$(CONFIG_INTEL_SOC_PMIC_CHTDC_TI)	+= intel_soc_pmic_chtdc_ti.o
-mt6397-objs	:= mt6397-core.o mt6397-irq.o
+obj-$(CONFIG_MFD_MT6360)	+= mt6360-core.o
+mt6397-objs			:= mt6397-core.o mt6397-irq.o mt6358-irq.o
 obj-$(CONFIG_MFD_MT6397)	+= mt6397.o
 obj-$(CONFIG_INTEL_SOC_PMIC_MRFLD)	+= intel_soc_pmic_mrfld.o
 
diff --git a/drivers/mfd/htc-i2cpld.c b/drivers/mfd/htc-i2cpld.c
index 8ad6768bd7a2..247f9849e54a 100644
--- a/drivers/mfd/htc-i2cpld.c
+++ b/drivers/mfd/htc-i2cpld.c
@@ -355,12 +355,12 @@ static int htcpld_register_chip_i2c(
 	info.platform_data = chip;
 
 	/* Add the I2C device.  This calls the probe() function. */
-	client = i2c_new_device(adapter, &info);
-	if (!client) {
+	client = i2c_new_client_device(adapter, &info);
+	if (IS_ERR(client)) {
 		/* I2C device registration failed, contineu with the next */
 		dev_warn(dev, "Unable to add I2C device for 0x%x\n",
 			 plat_chip_data->addr);
-		return -ENODEV;
+		return PTR_ERR(client);
 	}
 
 	i2c_set_clientdata(client, chip);
diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c
index 7fc0c5d4edff..046222684b8b 100644
--- a/drivers/mfd/intel-lpss-pci.c
+++ b/drivers/mfd/intel-lpss-pci.c
@@ -250,9 +250,9 @@ static const struct pci_device_id intel_lpss_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, 0x4da9), (kernel_ulong_t)&spt_uart_info },
 	{ PCI_VDEVICE(INTEL, 0x4daa), (kernel_ulong_t)&spt_info },
 	{ PCI_VDEVICE(INTEL, 0x4dab), (kernel_ulong_t)&spt_info },
-	{ PCI_VDEVICE(INTEL, 0x4daf), (kernel_ulong_t)&spt_uart_info },
 	{ PCI_VDEVICE(INTEL, 0x4dc5), (kernel_ulong_t)&bxt_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0x4dc6), (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4dc7), (kernel_ulong_t)&spt_uart_info },
 	{ PCI_VDEVICE(INTEL, 0x4de8), (kernel_ulong_t)&bxt_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0x4de9), (kernel_ulong_t)&bxt_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0x4dea), (kernel_ulong_t)&bxt_i2c_info },
diff --git a/drivers/mfd/intel_quark_i2c_gpio.c b/drivers/mfd/intel_quark_i2c_gpio.c
index 41326b48da55..84ca7902e1df 100644
--- a/drivers/mfd/intel_quark_i2c_gpio.c
+++ b/drivers/mfd/intel_quark_i2c_gpio.c
@@ -216,7 +216,6 @@ static int intel_quark_gpio_setup(struct pci_dev *pdev, struct mfd_cell *cell)
 	pdata->properties->ngpio	= INTEL_QUARK_MFD_NGPIO;
 	pdata->properties->gpio_base	= INTEL_QUARK_MFD_GPIO_BASE;
 	pdata->properties->irq[0]	= pdev->irq;
-	pdata->properties->has_irq	= true;
 	pdata->properties->irq_shared	= true;
 
 	cell->platform_data = pdata;
diff --git a/drivers/mfd/max77620.c b/drivers/mfd/max77620.c
index c7ed5c353553..fec2096474ad 100644
--- a/drivers/mfd/max77620.c
+++ b/drivers/mfd/max77620.c
@@ -177,6 +177,7 @@ static const struct regmap_config max77620_regmap_config = {
 	.rd_table = &max77620_readable_table,
 	.wr_table = &max77620_writable_table,
 	.volatile_table = &max77620_volatile_table,
+	.use_single_write = true,
 };
 
 static const struct regmap_config max20024_regmap_config = {
diff --git a/drivers/mfd/mp2629.c b/drivers/mfd/mp2629.c
new file mode 100644
index 000000000000..16840ec5fd1c
--- /dev/null
+++ b/drivers/mfd/mp2629.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * MP2629 parent driver for ADC and battery charger
+ *
+ * Copyright 2020 Monolithic Power Systems, Inc
+ *
+ * Author: Saravanan Sekar <sravanhome@gmail.com>
+ */
+
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/mfd/core.h>
+#include <linux/mfd/mp2629.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+static const struct mfd_cell mp2629_cell[] = {
+	{
+		.name = "mp2629_adc",
+		.of_compatible = "mps,mp2629_adc",
+	},
+	{
+		.name = "mp2629_charger",
+		.of_compatible = "mps,mp2629_charger",
+	}
+};
+
+static const struct regmap_config mp2629_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = 0x17,
+};
+
+static int mp2629_probe(struct i2c_client *client)
+{
+	struct mp2629_data *ddata;
+	int ret;
+
+	ddata = devm_kzalloc(&client->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	ddata->dev = &client->dev;
+	i2c_set_clientdata(client, ddata);
+
+	ddata->regmap = devm_regmap_init_i2c(client, &mp2629_regmap_config);
+	if (IS_ERR(ddata->regmap)) {
+		dev_err(ddata->dev, "Failed to allocate regmap\n");
+		return PTR_ERR(ddata->regmap);
+	}
+
+	ret = devm_mfd_add_devices(ddata->dev, PLATFORM_DEVID_AUTO, mp2629_cell,
+				   ARRAY_SIZE(mp2629_cell), NULL, 0, NULL);
+	if (ret)
+		dev_err(ddata->dev, "Failed to register sub-devices %d\n", ret);
+
+	return ret;
+}
+
+static const struct of_device_id mp2629_of_match[] = {
+	{ .compatible = "mps,mp2629"},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mp2629_of_match);
+
+static struct i2c_driver mp2629_driver = {
+	.driver = {
+		.name = "mp2629",
+		.of_match_table = mp2629_of_match,
+	},
+	.probe_new	= mp2629_probe,
+};
+module_i2c_driver(mp2629_driver);
+
+MODULE_AUTHOR("Saravanan Sekar <sravanhome@gmail.com>");
+MODULE_DESCRIPTION("MP2629 Battery charger parent driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mfd/mt6358-irq.c b/drivers/mfd/mt6358-irq.c
new file mode 100644
index 000000000000..db734f2831ff
--- /dev/null
+++ b/drivers/mfd/mt6358-irq.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2020 MediaTek Inc.
+
+#include <linux/interrupt.h>
+#include <linux/mfd/mt6358/core.h>
+#include <linux/mfd/mt6358/registers.h>
+#include <linux/mfd/mt6397/core.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+static struct irq_top_t mt6358_ints[] = {
+	MT6358_TOP_GEN(BUCK),
+	MT6358_TOP_GEN(LDO),
+	MT6358_TOP_GEN(PSC),
+	MT6358_TOP_GEN(SCK),
+	MT6358_TOP_GEN(BM),
+	MT6358_TOP_GEN(HK),
+	MT6358_TOP_GEN(AUD),
+	MT6358_TOP_GEN(MISC),
+};
+
+static void pmic_irq_enable(struct irq_data *data)
+{
+	unsigned int hwirq = irqd_to_hwirq(data);
+	struct mt6397_chip *chip = irq_data_get_irq_chip_data(data);
+	struct pmic_irq_data *irqd = chip->irq_data;
+
+	irqd->enable_hwirq[hwirq] = true;
+}
+
+static void pmic_irq_disable(struct irq_data *data)
+{
+	unsigned int hwirq = irqd_to_hwirq(data);
+	struct mt6397_chip *chip = irq_data_get_irq_chip_data(data);
+	struct pmic_irq_data *irqd = chip->irq_data;
+
+	irqd->enable_hwirq[hwirq] = false;
+}
+
+static void pmic_irq_lock(struct irq_data *data)
+{
+	struct mt6397_chip *chip = irq_data_get_irq_chip_data(data);
+
+	mutex_lock(&chip->irqlock);
+}
+
+static void pmic_irq_sync_unlock(struct irq_data *data)
+{
+	unsigned int i, top_gp, gp_offset, en_reg, int_regs, shift;
+	struct mt6397_chip *chip = irq_data_get_irq_chip_data(data);
+	struct pmic_irq_data *irqd = chip->irq_data;
+
+	for (i = 0; i < irqd->num_pmic_irqs; i++) {
+		if (irqd->enable_hwirq[i] == irqd->cache_hwirq[i])
+			continue;
+
+		/* Find out the IRQ group */
+		top_gp = 0;
+		while ((top_gp + 1) < irqd->num_top &&
+		       i >= mt6358_ints[top_gp + 1].hwirq_base)
+			top_gp++;
+
+		/* Find the IRQ registers */
+		gp_offset = i - mt6358_ints[top_gp].hwirq_base;
+		int_regs = gp_offset / MT6358_REG_WIDTH;
+		shift = gp_offset % MT6358_REG_WIDTH;
+		en_reg = mt6358_ints[top_gp].en_reg +
+			 (mt6358_ints[top_gp].en_reg_shift * int_regs);
+
+		regmap_update_bits(chip->regmap, en_reg, BIT(shift),
+				   irqd->enable_hwirq[i] << shift);
+
+		irqd->cache_hwirq[i] = irqd->enable_hwirq[i];
+	}
+	mutex_unlock(&chip->irqlock);
+}
+
+static struct irq_chip mt6358_irq_chip = {
+	.name = "mt6358-irq",
+	.flags = IRQCHIP_SKIP_SET_WAKE,
+	.irq_enable = pmic_irq_enable,
+	.irq_disable = pmic_irq_disable,
+	.irq_bus_lock = pmic_irq_lock,
+	.irq_bus_sync_unlock = pmic_irq_sync_unlock,
+};
+
+static void mt6358_irq_sp_handler(struct mt6397_chip *chip,
+				  unsigned int top_gp)
+{
+	unsigned int irq_status, sta_reg, status;
+	unsigned int hwirq, virq;
+	int i, j, ret;
+
+	for (i = 0; i < mt6358_ints[top_gp].num_int_regs; i++) {
+		sta_reg = mt6358_ints[top_gp].sta_reg +
+			mt6358_ints[top_gp].sta_reg_shift * i;
+
+		ret = regmap_read(chip->regmap, sta_reg, &irq_status);
+		if (ret) {
+			dev_err(chip->dev,
+				"Failed to read IRQ status, ret=%d\n", ret);
+			return;
+		}
+
+		if (!irq_status)
+			continue;
+
+		status = irq_status;
+		do {
+			j = __ffs(status);
+
+			hwirq = mt6358_ints[top_gp].hwirq_base +
+				MT6358_REG_WIDTH * i + j;
+
+			virq = irq_find_mapping(chip->irq_domain, hwirq);
+			if (virq)
+				handle_nested_irq(virq);
+
+			status &= ~BIT(j);
+		} while (status);
+
+		regmap_write(chip->regmap, sta_reg, irq_status);
+	}
+}
+
+static irqreturn_t mt6358_irq_handler(int irq, void *data)
+{
+	struct mt6397_chip *chip = data;
+	struct pmic_irq_data *mt6358_irq_data = chip->irq_data;
+	unsigned int bit, i, top_irq_status = 0;
+	int ret;
+
+	ret = regmap_read(chip->regmap,
+			  mt6358_irq_data->top_int_status_reg,
+			  &top_irq_status);
+	if (ret) {
+		dev_err(chip->dev,
+			"Failed to read status from the device, ret=%d\n", ret);
+		return IRQ_NONE;
+	}
+
+	for (i = 0; i < mt6358_irq_data->num_top; i++) {
+		bit = BIT(mt6358_ints[i].top_offset);
+		if (top_irq_status & bit) {
+			mt6358_irq_sp_handler(chip, i);
+			top_irq_status &= ~bit;
+			if (!top_irq_status)
+				break;
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int pmic_irq_domain_map(struct irq_domain *d, unsigned int irq,
+			       irq_hw_number_t hw)
+{
+	struct mt6397_chip *mt6397 = d->host_data;
+
+	irq_set_chip_data(irq, mt6397);
+	irq_set_chip_and_handler(irq, &mt6358_irq_chip, handle_level_irq);
+	irq_set_nested_thread(irq, 1);
+	irq_set_noprobe(irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops mt6358_irq_domain_ops = {
+	.map = pmic_irq_domain_map,
+	.xlate = irq_domain_xlate_twocell,
+};
+
+int mt6358_irq_init(struct mt6397_chip *chip)
+{
+	int i, j, ret;
+	struct pmic_irq_data *irqd;
+
+	irqd = devm_kzalloc(chip->dev, sizeof(*irqd), GFP_KERNEL);
+	if (!irqd)
+		return -ENOMEM;
+
+	chip->irq_data = irqd;
+
+	mutex_init(&chip->irqlock);
+	irqd->top_int_status_reg = MT6358_TOP_INT_STATUS0;
+	irqd->num_pmic_irqs = MT6358_IRQ_NR;
+	irqd->num_top = ARRAY_SIZE(mt6358_ints);
+
+	irqd->enable_hwirq = devm_kcalloc(chip->dev,
+					  irqd->num_pmic_irqs,
+					  sizeof(*irqd->enable_hwirq),
+					  GFP_KERNEL);
+	if (!irqd->enable_hwirq)
+		return -ENOMEM;
+
+	irqd->cache_hwirq = devm_kcalloc(chip->dev,
+					 irqd->num_pmic_irqs,
+					 sizeof(*irqd->cache_hwirq),
+					 GFP_KERNEL);
+	if (!irqd->cache_hwirq)
+		return -ENOMEM;
+
+	/* Disable all interrupts for initializing */
+	for (i = 0; i < irqd->num_top; i++) {
+		for (j = 0; j < mt6358_ints[i].num_int_regs; j++)
+			regmap_write(chip->regmap,
+				     mt6358_ints[i].en_reg +
+				     mt6358_ints[i].en_reg_shift * j, 0);
+	}
+
+	chip->irq_domain = irq_domain_add_linear(chip->dev->of_node,
+						 irqd->num_pmic_irqs,
+						 &mt6358_irq_domain_ops, chip);
+	if (!chip->irq_domain) {
+		dev_err(chip->dev, "Could not create IRQ domain\n");
+		return -ENODEV;
+	}
+
+	ret = devm_request_threaded_irq(chip->dev, chip->irq, NULL,
+					mt6358_irq_handler, IRQF_ONESHOT,
+					mt6358_irq_chip.name, chip);
+	if (ret) {
+		dev_err(chip->dev, "Failed to register IRQ=%d, ret=%d\n",
+			chip->irq, ret);
+		return ret;
+	}
+
+	enable_irq_wake(chip->irq);
+	return ret;
+}
diff --git a/drivers/mfd/mt6360-core.c b/drivers/mfd/mt6360-core.c
new file mode 100644
index 000000000000..e9cacc27d980
--- /dev/null
+++ b/drivers/mfd/mt6360-core.c
@@ -0,0 +1,425 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020 MediaTek Inc.
+ *
+ * Author: Gene Chen <gene_chen@richtek.com>
+ */
+
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/core.h>
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/version.h>
+
+#include <linux/mfd/mt6360.h>
+
+/* reg 0 -> 0 ~ 7 */
+#define MT6360_CHG_TREG_EVT		(4)
+#define MT6360_CHG_AICR_EVT		(5)
+#define MT6360_CHG_MIVR_EVT		(6)
+#define MT6360_PWR_RDY_EVT		(7)
+/* REG 1 -> 8 ~ 15 */
+#define MT6360_CHG_BATSYSUV_EVT		(9)
+#define MT6360_FLED_CHG_VINOVP_EVT	(11)
+#define MT6360_CHG_VSYSUV_EVT		(12)
+#define MT6360_CHG_VSYSOV_EVT		(13)
+#define MT6360_CHG_VBATOV_EVT		(14)
+#define MT6360_CHG_VBUSOV_EVT		(15)
+/* REG 2 -> 16 ~ 23 */
+/* REG 3 -> 24 ~ 31 */
+#define MT6360_WD_PMU_DET		(25)
+#define MT6360_WD_PMU_DONE		(26)
+#define MT6360_CHG_TMRI			(27)
+#define MT6360_CHG_ADPBADI		(29)
+#define MT6360_CHG_RVPI			(30)
+#define MT6360_OTPI			(31)
+/* REG 4 -> 32 ~ 39 */
+#define MT6360_CHG_AICCMEASL		(32)
+#define MT6360_CHGDET_DONEI		(34)
+#define MT6360_WDTMRI			(35)
+#define MT6360_SSFINISHI		(36)
+#define MT6360_CHG_RECHGI		(37)
+#define MT6360_CHG_TERMI		(38)
+#define MT6360_CHG_IEOCI		(39)
+/* REG 5 -> 40 ~ 47 */
+#define MT6360_PUMPX_DONEI		(40)
+#define MT6360_BAT_OVP_ADC_EVT		(41)
+#define MT6360_TYPEC_OTP_EVT		(42)
+#define MT6360_ADC_WAKEUP_EVT		(43)
+#define MT6360_ADC_DONEI		(44)
+#define MT6360_BST_BATUVI		(45)
+#define MT6360_BST_VBUSOVI		(46)
+#define MT6360_BST_OLPI			(47)
+/* REG 6 -> 48 ~ 55 */
+#define MT6360_ATTACH_I			(48)
+#define MT6360_DETACH_I			(49)
+#define MT6360_QC30_STPDONE		(51)
+#define MT6360_QC_VBUSDET_DONE		(52)
+#define MT6360_HVDCP_DET		(53)
+#define MT6360_CHGDETI			(54)
+#define MT6360_DCDTI			(55)
+/* REG 7 -> 56 ~ 63 */
+#define MT6360_FOD_DONE_EVT		(56)
+#define MT6360_FOD_OV_EVT		(57)
+#define MT6360_CHRDET_UVP_EVT		(58)
+#define MT6360_CHRDET_OVP_EVT		(59)
+#define MT6360_CHRDET_EXT_EVT		(60)
+#define MT6360_FOD_LR_EVT		(61)
+#define MT6360_FOD_HR_EVT		(62)
+#define MT6360_FOD_DISCHG_FAIL_EVT	(63)
+/* REG 8 -> 64 ~ 71 */
+#define MT6360_USBID_EVT		(64)
+#define MT6360_APWDTRST_EVT		(65)
+#define MT6360_EN_EVT			(66)
+#define MT6360_QONB_RST_EVT		(67)
+#define MT6360_MRSTB_EVT		(68)
+#define MT6360_OTP_EVT			(69)
+#define MT6360_VDDAOV_EVT		(70)
+#define MT6360_SYSUV_EVT		(71)
+/* REG 9 -> 72 ~ 79 */
+#define MT6360_FLED_STRBPIN_EVT		(72)
+#define MT6360_FLED_TORPIN_EVT		(73)
+#define MT6360_FLED_TX_EVT		(74)
+#define MT6360_FLED_LVF_EVT		(75)
+#define MT6360_FLED2_SHORT_EVT		(78)
+#define MT6360_FLED1_SHORT_EVT		(79)
+/* REG 10 -> 80 ~ 87 */
+#define MT6360_FLED2_STRB_EVT		(80)
+#define MT6360_FLED1_STRB_EVT		(81)
+#define MT6360_FLED2_STRB_TO_EVT	(82)
+#define MT6360_FLED1_STRB_TO_EVT	(83)
+#define MT6360_FLED2_TOR_EVT		(84)
+#define MT6360_FLED1_TOR_EVT		(85)
+/* REG 11 -> 88 ~ 95 */
+/* REG 12 -> 96 ~ 103 */
+#define MT6360_BUCK1_PGB_EVT		(96)
+#define MT6360_BUCK1_OC_EVT		(100)
+#define MT6360_BUCK1_OV_EVT		(101)
+#define MT6360_BUCK1_UV_EVT		(102)
+/* REG 13 -> 104 ~ 111 */
+#define MT6360_BUCK2_PGB_EVT		(104)
+#define MT6360_BUCK2_OC_EVT		(108)
+#define MT6360_BUCK2_OV_EVT		(109)
+#define MT6360_BUCK2_UV_EVT		(110)
+/* REG 14 -> 112 ~ 119 */
+#define MT6360_LDO1_OC_EVT		(113)
+#define MT6360_LDO2_OC_EVT		(114)
+#define MT6360_LDO3_OC_EVT		(115)
+#define MT6360_LDO5_OC_EVT		(117)
+#define MT6360_LDO6_OC_EVT		(118)
+#define MT6360_LDO7_OC_EVT		(119)
+/* REG 15 -> 120 ~ 127 */
+#define MT6360_LDO1_PGB_EVT		(121)
+#define MT6360_LDO2_PGB_EVT		(122)
+#define MT6360_LDO3_PGB_EVT		(123)
+#define MT6360_LDO5_PGB_EVT		(125)
+#define MT6360_LDO6_PGB_EVT		(126)
+#define MT6360_LDO7_PGB_EVT		(127)
+
+static const struct regmap_irq mt6360_pmu_irqs[] =  {
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_TREG_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_AICR_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_MIVR_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_PWR_RDY_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_BATSYSUV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED_CHG_VINOVP_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_VSYSUV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_VSYSOV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_VBATOV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_VBUSOV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_WD_PMU_DET, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_WD_PMU_DONE, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_TMRI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_ADPBADI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_RVPI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_OTPI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_AICCMEASL, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHGDET_DONEI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_WDTMRI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_SSFINISHI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_RECHGI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_TERMI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHG_IEOCI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_PUMPX_DONEI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BAT_OVP_ADC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_TYPEC_OTP_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_ADC_WAKEUP_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_ADC_DONEI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BST_BATUVI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BST_VBUSOVI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BST_OLPI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_ATTACH_I, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_DETACH_I, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_QC30_STPDONE, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_QC_VBUSDET_DONE, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_HVDCP_DET, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHGDETI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_DCDTI, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FOD_DONE_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FOD_OV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHRDET_UVP_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHRDET_OVP_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_CHRDET_EXT_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FOD_LR_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FOD_HR_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FOD_DISCHG_FAIL_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_USBID_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_APWDTRST_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_EN_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_QONB_RST_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_MRSTB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_OTP_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_VDDAOV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_SYSUV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED_STRBPIN_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED_TORPIN_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED_TX_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED_LVF_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED2_SHORT_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED1_SHORT_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED2_STRB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED1_STRB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED2_STRB_TO_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED1_STRB_TO_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED2_TOR_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_FLED1_TOR_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK1_PGB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK1_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK1_OV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK1_UV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK2_PGB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK2_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK2_OV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_BUCK2_UV_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO1_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO2_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO3_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO5_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO6_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO7_OC_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO1_PGB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO2_PGB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO3_PGB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO5_PGB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO6_PGB_EVT, 8),
+	REGMAP_IRQ_REG_LINE(MT6360_LDO7_PGB_EVT, 8),
+};
+
+static int mt6360_pmu_handle_post_irq(void *irq_drv_data)
+{
+	struct mt6360_pmu_data *mpd = irq_drv_data;
+
+	return regmap_update_bits(mpd->regmap,
+		MT6360_PMU_IRQ_SET, MT6360_IRQ_RETRIG, MT6360_IRQ_RETRIG);
+}
+
+static struct regmap_irq_chip mt6360_pmu_irq_chip = {
+	.irqs = mt6360_pmu_irqs,
+	.num_irqs = ARRAY_SIZE(mt6360_pmu_irqs),
+	.num_regs = MT6360_PMU_IRQ_REGNUM,
+	.mask_base = MT6360_PMU_CHG_MASK1,
+	.status_base = MT6360_PMU_CHG_IRQ1,
+	.ack_base = MT6360_PMU_CHG_IRQ1,
+	.init_ack_masked = true,
+	.use_ack = true,
+	.handle_post_irq = mt6360_pmu_handle_post_irq,
+};
+
+static const struct regmap_config mt6360_pmu_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = MT6360_PMU_MAXREG,
+};
+
+static const struct resource mt6360_adc_resources[] = {
+	DEFINE_RES_IRQ_NAMED(MT6360_ADC_DONEI, "adc_donei"),
+};
+
+static const struct resource mt6360_chg_resources[] = {
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_TREG_EVT, "chg_treg_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_PWR_RDY_EVT, "pwr_rdy_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_BATSYSUV_EVT, "chg_batsysuv_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_VSYSUV_EVT, "chg_vsysuv_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_VSYSOV_EVT, "chg_vsysov_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_VBATOV_EVT, "chg_vbatov_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_VBUSOV_EVT, "chg_vbusov_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_AICCMEASL, "chg_aiccmeasl"),
+	DEFINE_RES_IRQ_NAMED(MT6360_WDTMRI, "wdtmri"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_RECHGI, "chg_rechgi"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_TERMI, "chg_termi"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHG_IEOCI, "chg_ieoci"),
+	DEFINE_RES_IRQ_NAMED(MT6360_PUMPX_DONEI, "pumpx_donei"),
+	DEFINE_RES_IRQ_NAMED(MT6360_ATTACH_I, "attach_i"),
+	DEFINE_RES_IRQ_NAMED(MT6360_CHRDET_EXT_EVT, "chrdet_ext_evt"),
+};
+
+static const struct resource mt6360_led_resources[] = {
+	DEFINE_RES_IRQ_NAMED(MT6360_FLED_CHG_VINOVP_EVT, "fled_chg_vinovp_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_FLED_LVF_EVT, "fled_lvf_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_FLED2_SHORT_EVT, "fled2_short_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_FLED1_SHORT_EVT, "fled1_short_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_FLED2_STRB_TO_EVT, "fled2_strb_to_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_FLED1_STRB_TO_EVT, "fled1_strb_to_evt"),
+};
+
+static const struct resource mt6360_pmic_resources[] = {
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK1_PGB_EVT, "buck1_pgb_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK1_OC_EVT, "buck1_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK1_OV_EVT, "buck1_ov_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK1_UV_EVT, "buck1_uv_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK2_PGB_EVT, "buck2_pgb_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK2_OC_EVT, "buck2_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK2_OV_EVT, "buck2_ov_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_BUCK2_UV_EVT, "buck2_uv_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO6_OC_EVT, "ldo6_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO7_OC_EVT, "ldo7_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO6_PGB_EVT, "ldo6_pgb_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO7_PGB_EVT, "ldo7_pgb_evt"),
+};
+
+static const struct resource mt6360_ldo_resources[] = {
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO1_OC_EVT, "ldo1_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO2_OC_EVT, "ldo2_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO3_OC_EVT, "ldo3_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO5_OC_EVT, "ldo5_oc_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO1_PGB_EVT, "ldo1_pgb_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO2_PGB_EVT, "ldo2_pgb_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO3_PGB_EVT, "ldo3_pgb_evt"),
+	DEFINE_RES_IRQ_NAMED(MT6360_LDO5_PGB_EVT, "ldo5_pgb_evt"),
+};
+
+static const struct mfd_cell mt6360_devs[] = {
+	OF_MFD_CELL("mt6360_adc", mt6360_adc_resources,
+		    NULL, 0, 0, "mediatek,mt6360_adc"),
+	OF_MFD_CELL("mt6360_chg", mt6360_chg_resources,
+		    NULL, 0, 0, "mediatek,mt6360_chg"),
+	OF_MFD_CELL("mt6360_led", mt6360_led_resources,
+		    NULL, 0, 0, "mediatek,mt6360_led"),
+	OF_MFD_CELL("mt6360_pmic", mt6360_pmic_resources,
+		    NULL, 0, 0, "mediatek,mt6360_pmic"),
+	OF_MFD_CELL("mt6360_ldo", mt6360_ldo_resources,
+		    NULL, 0, 0, "mediatek,mt6360_ldo"),
+	OF_MFD_CELL("mt6360_tcpc", NULL,
+		    NULL, 0, 0, "mediatek,mt6360_tcpc"),
+};
+
+static const unsigned short mt6360_slave_addr[MT6360_SLAVE_MAX] = {
+	MT6360_PMU_SLAVEID,
+	MT6360_PMIC_SLAVEID,
+	MT6360_LDO_SLAVEID,
+	MT6360_TCPC_SLAVEID,
+};
+
+static int mt6360_pmu_probe(struct i2c_client *client)
+{
+	struct mt6360_pmu_data *mpd;
+	unsigned int reg_data;
+	int i, ret;
+
+	mpd = devm_kzalloc(&client->dev, sizeof(*mpd), GFP_KERNEL);
+	if (!mpd)
+		return -ENOMEM;
+
+	mpd->dev = &client->dev;
+	i2c_set_clientdata(client, mpd);
+
+	mpd->regmap = devm_regmap_init_i2c(client, &mt6360_pmu_regmap_config);
+	if (IS_ERR(mpd->regmap)) {
+		dev_err(&client->dev, "Failed to register regmap\n");
+		return PTR_ERR(mpd->regmap);
+	}
+
+	ret = regmap_read(mpd->regmap, MT6360_PMU_DEV_INFO, &reg_data);
+	if (ret) {
+		dev_err(&client->dev, "Device not found\n");
+		return ret;
+	}
+
+	mpd->chip_rev = reg_data & CHIP_REV_MASK;
+	if (mpd->chip_rev != CHIP_VEN_MT6360) {
+		dev_err(&client->dev, "Device not supported\n");
+		return -ENODEV;
+	}
+
+	mt6360_pmu_irq_chip.irq_drv_data = mpd;
+	ret = devm_regmap_add_irq_chip(&client->dev, mpd->regmap, client->irq,
+				       IRQF_TRIGGER_FALLING, 0,
+				       &mt6360_pmu_irq_chip, &mpd->irq_data);
+	if (ret) {
+		dev_err(&client->dev, "Failed to add Regmap IRQ Chip\n");
+		return ret;
+	}
+
+	mpd->i2c[0] = client;
+	for (i = 1; i < MT6360_SLAVE_MAX; i++) {
+		mpd->i2c[i] = devm_i2c_new_dummy_device(&client->dev,
+							client->adapter,
+							mt6360_slave_addr[i]);
+		if (IS_ERR(mpd->i2c[i])) {
+			dev_err(&client->dev,
+				"Failed to get new dummy I2C device for address 0x%x",
+				mt6360_slave_addr[i]);
+			return PTR_ERR(mpd->i2c[i]);
+		}
+		i2c_set_clientdata(mpd->i2c[i], mpd);
+	}
+
+	ret = devm_mfd_add_devices(&client->dev, PLATFORM_DEVID_AUTO,
+				   mt6360_devs, ARRAY_SIZE(mt6360_devs), NULL,
+				   0, regmap_irq_get_domain(mpd->irq_data));
+	if (ret) {
+		dev_err(&client->dev,
+			"Failed to register subordinate devices\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int __maybe_unused mt6360_pmu_suspend(struct device *dev)
+{
+	struct i2c_client *i2c = to_i2c_client(dev);
+
+	if (device_may_wakeup(dev))
+		enable_irq_wake(i2c->irq);
+
+	return 0;
+}
+
+static int __maybe_unused mt6360_pmu_resume(struct device *dev)
+{
+
+	struct i2c_client *i2c = to_i2c_client(dev);
+
+	if (device_may_wakeup(dev))
+		disable_irq_wake(i2c->irq);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(mt6360_pmu_pm_ops,
+			 mt6360_pmu_suspend, mt6360_pmu_resume);
+
+static const struct of_device_id __maybe_unused mt6360_pmu_of_id[] = {
+	{ .compatible = "mediatek,mt6360_pmu", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, mt6360_pmu_of_id);
+
+static struct i2c_driver mt6360_pmu_driver = {
+	.driver = {
+		.name = "mt6360_pmu",
+		.pm = &mt6360_pmu_pm_ops,
+		.of_match_table = of_match_ptr(mt6360_pmu_of_id),
+	},
+	.probe_new = mt6360_pmu_probe,
+};
+module_i2c_driver(mt6360_pmu_driver);
+
+MODULE_AUTHOR("Gene Chen <gene_chen@richtek.com>");
+MODULE_DESCRIPTION("MT6360 PMU I2C Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/mt6397-core.c b/drivers/mfd/mt6397-core.c
index 0437c858d115..f6cd8a660602 100644
--- a/drivers/mfd/mt6397-core.c
+++ b/drivers/mfd/mt6397-core.c
@@ -12,13 +12,18 @@
 #include <linux/regmap.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/mt6323/core.h>
+#include <linux/mfd/mt6358/core.h>
 #include <linux/mfd/mt6397/core.h>
 #include <linux/mfd/mt6323/registers.h>
+#include <linux/mfd/mt6358/registers.h>
 #include <linux/mfd/mt6397/registers.h>
 
 #define MT6323_RTC_BASE		0x8000
 #define MT6323_RTC_SIZE		0x40
 
+#define MT6358_RTC_BASE		0x0588
+#define MT6358_RTC_SIZE		0x3c
+
 #define MT6397_RTC_BASE		0xe000
 #define MT6397_RTC_SIZE		0x3e
 
@@ -30,6 +35,11 @@ static const struct resource mt6323_rtc_resources[] = {
 	DEFINE_RES_IRQ(MT6323_IRQ_STATUS_RTC),
 };
 
+static const struct resource mt6358_rtc_resources[] = {
+	DEFINE_RES_MEM(MT6358_RTC_BASE, MT6358_RTC_SIZE),
+	DEFINE_RES_IRQ(MT6358_IRQ_RTC),
+};
+
 static const struct resource mt6397_rtc_resources[] = {
 	DEFINE_RES_MEM(MT6397_RTC_BASE, MT6397_RTC_SIZE),
 	DEFINE_RES_IRQ(MT6397_IRQ_RTC),
@@ -74,6 +84,21 @@ static const struct mfd_cell mt6323_devs[] = {
 	},
 };
 
+static const struct mfd_cell mt6358_devs[] = {
+	{
+		.name = "mt6358-regulator",
+		.of_compatible = "mediatek,mt6358-regulator"
+	}, {
+		.name = "mt6358-rtc",
+		.num_resources = ARRAY_SIZE(mt6358_rtc_resources),
+		.resources = mt6358_rtc_resources,
+		.of_compatible = "mediatek,mt6358-rtc",
+	}, {
+		.name = "mt6358-sound",
+		.of_compatible = "mediatek,mt6358-sound"
+	},
+};
+
 static const struct mfd_cell mt6397_devs[] = {
 	{
 		.name = "mt6397-rtc",
@@ -100,54 +125,42 @@ static const struct mfd_cell mt6397_devs[] = {
 	}
 };
 
-#ifdef CONFIG_PM_SLEEP
-static int mt6397_irq_suspend(struct device *dev)
-{
-	struct mt6397_chip *chip = dev_get_drvdata(dev);
-
-	regmap_write(chip->regmap, chip->int_con[0], chip->wake_mask[0]);
-	regmap_write(chip->regmap, chip->int_con[1], chip->wake_mask[1]);
-
-	enable_irq_wake(chip->irq);
-
-	return 0;
-}
-
-static int mt6397_irq_resume(struct device *dev)
-{
-	struct mt6397_chip *chip = dev_get_drvdata(dev);
-
-	regmap_write(chip->regmap, chip->int_con[0], chip->irq_masks_cur[0]);
-	regmap_write(chip->regmap, chip->int_con[1], chip->irq_masks_cur[1]);
-
-	disable_irq_wake(chip->irq);
-
-	return 0;
-}
-#endif
-
-static SIMPLE_DEV_PM_OPS(mt6397_pm_ops, mt6397_irq_suspend,
-			mt6397_irq_resume);
-
 struct chip_data {
 	u32 cid_addr;
 	u32 cid_shift;
+	const struct mfd_cell *cells;
+	int cell_size;
+	int (*irq_init)(struct mt6397_chip *chip);
 };
 
 static const struct chip_data mt6323_core = {
 	.cid_addr = MT6323_CID,
 	.cid_shift = 0,
+	.cells = mt6323_devs,
+	.cell_size = ARRAY_SIZE(mt6323_devs),
+	.irq_init = mt6397_irq_init,
+};
+
+static const struct chip_data mt6358_core = {
+	.cid_addr = MT6358_SWCID,
+	.cid_shift = 8,
+	.cells = mt6358_devs,
+	.cell_size = ARRAY_SIZE(mt6358_devs),
+	.irq_init = mt6358_irq_init,
 };
 
 static const struct chip_data mt6397_core = {
 	.cid_addr = MT6397_CID,
 	.cid_shift = 0,
+	.cells = mt6397_devs,
+	.cell_size = ARRAY_SIZE(mt6397_devs),
+	.irq_init = mt6397_irq_init,
 };
 
 static int mt6397_probe(struct platform_device *pdev)
 {
 	int ret;
-	unsigned int id;
+	unsigned int id = 0;
 	struct mt6397_chip *pmic;
 	const struct chip_data *pmic_core;
 
@@ -183,29 +196,13 @@ static int mt6397_probe(struct platform_device *pdev)
 	if (pmic->irq <= 0)
 		return pmic->irq;
 
-	ret = mt6397_irq_init(pmic);
+	ret = pmic_core->irq_init(pmic);
 	if (ret)
 		return ret;
 
-	switch (pmic->chip_id) {
-	case MT6323_CHIP_ID:
-		ret = devm_mfd_add_devices(&pdev->dev, PLATFORM_DEVID_NONE,
-					   mt6323_devs, ARRAY_SIZE(mt6323_devs),
-					   NULL, 0, pmic->irq_domain);
-		break;
-
-	case MT6391_CHIP_ID:
-	case MT6397_CHIP_ID:
-		ret = devm_mfd_add_devices(&pdev->dev, PLATFORM_DEVID_NONE,
-					   mt6397_devs, ARRAY_SIZE(mt6397_devs),
-					   NULL, 0, pmic->irq_domain);
-		break;
-
-	default:
-		dev_err(&pdev->dev, "unsupported chip: %d\n", pmic->chip_id);
-		return -ENODEV;
-	}
-
+	ret = devm_mfd_add_devices(&pdev->dev, PLATFORM_DEVID_NONE,
+				   pmic_core->cells, pmic_core->cell_size,
+				   NULL, 0, pmic->irq_domain);
 	if (ret) {
 		irq_domain_remove(pmic->irq_domain);
 		dev_err(&pdev->dev, "failed to add child devices: %d\n", ret);
@@ -219,6 +216,9 @@ static const struct of_device_id mt6397_of_match[] = {
 		.compatible = "mediatek,mt6323",
 		.data = &mt6323_core,
 	}, {
+		.compatible = "mediatek,mt6358",
+		.data = &mt6358_core,
+	}, {
 		.compatible = "mediatek,mt6397",
 		.data = &mt6397_core,
 	}, {
@@ -238,7 +238,6 @@ static struct platform_driver mt6397_driver = {
 	.driver = {
 		.name = "mt6397",
 		.of_match_table = of_match_ptr(mt6397_of_match),
-		.pm = &mt6397_pm_ops,
 	},
 	.id_table = mt6397_id,
 };
diff --git a/drivers/mfd/mt6397-irq.c b/drivers/mfd/mt6397-irq.c
index b2d3ce1f3115..2924919da991 100644
--- a/drivers/mfd/mt6397-irq.c
+++ b/drivers/mfd/mt6397-irq.c
@@ -9,6 +9,7 @@
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/suspend.h>
 #include <linux/mfd/mt6323/core.h>
 #include <linux/mfd/mt6323/registers.h>
 #include <linux/mfd/mt6397/core.h>
@@ -81,7 +82,7 @@ static struct irq_chip mt6397_irq_chip = {
 static void mt6397_irq_handle_reg(struct mt6397_chip *mt6397, int reg,
 				  int irqbase)
 {
-	unsigned int status;
+	unsigned int status = 0;
 	int i, irq, ret;
 
 	ret = regmap_read(mt6397->regmap, reg, &status);
@@ -128,6 +129,36 @@ static const struct irq_domain_ops mt6397_irq_domain_ops = {
 	.map = mt6397_irq_domain_map,
 };
 
+static int mt6397_irq_pm_notifier(struct notifier_block *notifier,
+				  unsigned long pm_event, void *unused)
+{
+	struct mt6397_chip *chip =
+		container_of(notifier, struct mt6397_chip, pm_nb);
+
+	switch (pm_event) {
+	case PM_SUSPEND_PREPARE:
+		regmap_write(chip->regmap,
+			     chip->int_con[0], chip->wake_mask[0]);
+		regmap_write(chip->regmap,
+			     chip->int_con[1], chip->wake_mask[1]);
+		enable_irq_wake(chip->irq);
+		break;
+
+	case PM_POST_SUSPEND:
+		regmap_write(chip->regmap,
+			     chip->int_con[0], chip->irq_masks_cur[0]);
+		regmap_write(chip->regmap,
+			     chip->int_con[1], chip->irq_masks_cur[1]);
+		disable_irq_wake(chip->irq);
+		break;
+
+	default:
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
 int mt6397_irq_init(struct mt6397_chip *chip)
 {
 	int ret;
@@ -159,6 +190,7 @@ int mt6397_irq_init(struct mt6397_chip *chip)
 	regmap_write(chip->regmap, chip->int_con[0], 0x0);
 	regmap_write(chip->regmap, chip->int_con[1], 0x0);
 
+	chip->pm_nb.notifier_call = mt6397_irq_pm_notifier;
 	chip->irq_domain = irq_domain_add_linear(chip->dev->of_node,
 						 MT6397_IRQ_NR,
 						 &mt6397_irq_domain_ops,
@@ -177,5 +209,6 @@ int mt6397_irq_init(struct mt6397_chip *chip)
 		return ret;
 	}
 
+	register_pm_notifier(&chip->pm_nb);
 	return 0;
 }
diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c
index e49787e6bb93..ccd62b963952 100644
--- a/drivers/mfd/sm501.c
+++ b/drivers/mfd/sm501.c
@@ -1145,22 +1145,14 @@ static int sm501_register_gpio_i2c_instance(struct sm501_devdata *sm,
 		return -ENOMEM;
 
 	lookup->dev_id = "i2c-gpio";
-	if (iic->pin_sda < 32)
-		lookup->table[0].chip_label = "SM501-LOW";
-	else
-		lookup->table[0].chip_label = "SM501-HIGH";
-	lookup->table[0].chip_hwnum = iic->pin_sda % 32;
-	lookup->table[0].con_id = NULL;
-	lookup->table[0].idx = 0;
-	lookup->table[0].flags = GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN;
-	if (iic->pin_scl < 32)
-		lookup->table[1].chip_label = "SM501-LOW";
-	else
-		lookup->table[1].chip_label = "SM501-HIGH";
-	lookup->table[1].chip_hwnum = iic->pin_scl % 32;
-	lookup->table[1].con_id = NULL;
-	lookup->table[1].idx = 1;
-	lookup->table[1].flags = GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN;
+	lookup->table[0] = (struct gpiod_lookup)
+		GPIO_LOOKUP_IDX(iic->pin_sda < 32 ? "SM501-LOW" : "SM501-HIGH",
+				iic->pin_sda % 32, NULL, 0,
+				GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN);
+	lookup->table[1] = (struct gpiod_lookup)
+		GPIO_LOOKUP_IDX(iic->pin_scl < 32 ? "SM501-LOW" : "SM501-HIGH",
+				iic->pin_scl % 32, NULL, 1,
+				GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN);
 	gpiod_add_lookup_table(lookup);
 
 	icd = dev_get_platdata(&pdev->dev);
diff --git a/drivers/mfd/sprd-sc27xx-spi.c b/drivers/mfd/sprd-sc27xx-spi.c
index ebdf2f11ae28..33336cde4724 100644
--- a/drivers/mfd/sprd-sc27xx-spi.c
+++ b/drivers/mfd/sprd-sc27xx-spi.c
@@ -284,7 +284,6 @@ MODULE_DEVICE_TABLE(of, sprd_pmic_match);
 static struct spi_driver sprd_pmic_driver = {
 	.driver = {
 		.name = "sc27xx-pmic",
-		.bus = &spi_bus_type,
 		.of_match_table = sprd_pmic_match,
 	},
 	.probe = sprd_pmic_probe,
diff --git a/drivers/mfd/stm32-timers.c b/drivers/mfd/stm32-timers.c
index efcd4b980c94..add603359124 100644
--- a/drivers/mfd/stm32-timers.c
+++ b/drivers/mfd/stm32-timers.c
@@ -167,10 +167,11 @@ static void stm32_timers_get_arr_size(struct stm32_timers *ddata)
 	regmap_write(ddata->regmap, TIM_ARR, 0x0);
 }
 
-static void stm32_timers_dma_probe(struct device *dev,
+static int stm32_timers_dma_probe(struct device *dev,
 				   struct stm32_timers *ddata)
 {
 	int i;
+	int ret = 0;
 	char name[4];
 
 	init_completion(&ddata->dma.completion);
@@ -179,14 +180,23 @@ static void stm32_timers_dma_probe(struct device *dev,
 	/* Optional DMA support: get valid DMA channel(s) or NULL */
 	for (i = STM32_TIMERS_DMA_CH1; i <= STM32_TIMERS_DMA_CH4; i++) {
 		snprintf(name, ARRAY_SIZE(name), "ch%1d", i + 1);
-		ddata->dma.chans[i] = dma_request_slave_channel(dev, name);
+		ddata->dma.chans[i] = dma_request_chan(dev, name);
 	}
-	ddata->dma.chans[STM32_TIMERS_DMA_UP] =
-		dma_request_slave_channel(dev, "up");
-	ddata->dma.chans[STM32_TIMERS_DMA_TRIG] =
-		dma_request_slave_channel(dev, "trig");
-	ddata->dma.chans[STM32_TIMERS_DMA_COM] =
-		dma_request_slave_channel(dev, "com");
+	ddata->dma.chans[STM32_TIMERS_DMA_UP] = dma_request_chan(dev, "up");
+	ddata->dma.chans[STM32_TIMERS_DMA_TRIG] = dma_request_chan(dev, "trig");
+	ddata->dma.chans[STM32_TIMERS_DMA_COM] = dma_request_chan(dev, "com");
+
+	for (i = STM32_TIMERS_DMA_CH1; i < STM32_TIMERS_MAX_DMAS; i++) {
+		if (IS_ERR(ddata->dma.chans[i])) {
+			/* Save the first error code to return */
+			if (PTR_ERR(ddata->dma.chans[i]) != -ENODEV && !ret)
+				ret = PTR_ERR(ddata->dma.chans[i]);
+
+			ddata->dma.chans[i] = NULL;
+		}
+	}
+
+	return ret;
 }
 
 static void stm32_timers_dma_remove(struct device *dev,
@@ -230,7 +240,11 @@ static int stm32_timers_probe(struct platform_device *pdev)
 
 	stm32_timers_get_arr_size(ddata);
 
-	stm32_timers_dma_probe(dev, ddata);
+	ret = stm32_timers_dma_probe(dev, ddata);
+	if (ret) {
+		stm32_timers_dma_remove(dev, ddata);
+		return ret;
+	}
 
 	platform_set_drvdata(pdev, ddata);
 
diff --git a/drivers/mfd/stmfx.c b/drivers/mfd/stmfx.c
index 857991cb3cbb..711979afd90a 100644
--- a/drivers/mfd/stmfx.c
+++ b/drivers/mfd/stmfx.c
@@ -287,14 +287,21 @@ static int stmfx_irq_init(struct i2c_client *client)
 
 	ret = regmap_write(stmfx->map, STMFX_REG_IRQ_OUT_PIN, irqoutpin);
 	if (ret)
-		return ret;
+		goto irq_exit;
 
 	ret = devm_request_threaded_irq(stmfx->dev, client->irq,
 					NULL, stmfx_irq_handler,
 					irqtrigger | IRQF_ONESHOT,
 					"stmfx", stmfx);
 	if (ret)
-		stmfx_irq_exit(client);
+		goto irq_exit;
+
+	stmfx->irq = client->irq;
+
+	return 0;
+
+irq_exit:
+	stmfx_irq_exit(client);
 
 	return ret;
 }
@@ -481,6 +488,8 @@ static int stmfx_suspend(struct device *dev)
 	if (ret)
 		return ret;
 
+	disable_irq(stmfx->irq);
+
 	if (stmfx->vdd)
 		return regulator_disable(stmfx->vdd);
 
@@ -501,6 +510,13 @@ static int stmfx_resume(struct device *dev)
 		}
 	}
 
+	/* Reset STMFX - supply has been stopped during suspend */
+	ret = stmfx_chip_reset(stmfx);
+	if (ret) {
+		dev_err(stmfx->dev, "Failed to reset chip: %d\n", ret);
+		return ret;
+	}
+
 	ret = regmap_raw_write(stmfx->map, STMFX_REG_SYS_CTRL,
 			       &stmfx->bkp_sysctrl, sizeof(stmfx->bkp_sysctrl));
 	if (ret)
@@ -517,6 +533,8 @@ static int stmfx_resume(struct device *dev)
 	if (ret)
 		return ret;
 
+	enable_irq(stmfx->irq);
+
 	return 0;
 }
 #endif
diff --git a/drivers/mfd/stpmic1.c b/drivers/mfd/stpmic1.c
index 7dfbe8906cb8..eb3da558c3fb 100644
--- a/drivers/mfd/stpmic1.c
+++ b/drivers/mfd/stpmic1.c
@@ -59,7 +59,7 @@ static const struct regmap_access_table stpmic1_volatile_table = {
 	.n_yes_ranges = ARRAY_SIZE(stpmic1_volatile_ranges),
 };
 
-const struct regmap_config stpmic1_regmap_config = {
+static const struct regmap_config stpmic1_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
 	.cache_type = REGCACHE_RBTREE,
diff --git a/drivers/mfd/tqmx86.c b/drivers/mfd/tqmx86.c
index b9f48e588d95..ddddf08b6a4c 100644
--- a/drivers/mfd/tqmx86.c
+++ b/drivers/mfd/tqmx86.c
@@ -274,7 +274,7 @@ static int __init tqmx86_init(void)
 
 module_init(tqmx86_init);
 
-MODULE_DESCRIPTION("TQx86 PLD Core Driver");
+MODULE_DESCRIPTION("TQMx86 PLD Core Driver");
 MODULE_AUTHOR("Andrew Lunn <andrew@lunn.ch>");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:tqmx86");
diff --git a/drivers/mfd/vexpress-sysreg.c b/drivers/mfd/vexpress-sysreg.c
index c68ff56dbdb1..aaf24af287dd 100644
--- a/drivers/mfd/vexpress-sysreg.c
+++ b/drivers/mfd/vexpress-sysreg.c
@@ -8,13 +8,12 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/mfd/core.h>
-#include <linux/of_address.h>
+#include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/platform_data/syscon.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/stat.h>
-#include <linux/vexpress.h>
 
 #define SYS_ID			0x000
 #define SYS_SW			0x004
@@ -37,35 +36,8 @@
 #define SYS_CFGCTRL		0x0a4
 #define SYS_CFGSTAT		0x0a8
 
-#define SYS_HBI_MASK		0xfff
-#define SYS_PROCIDx_HBI_SHIFT	0
-
-#define SYS_MISC_MASTERSITE	(1 << 14)
-
-void vexpress_flags_set(u32 data)
-{
-	static void __iomem *base;
-
-	if (!base) {
-		struct device_node *node = of_find_compatible_node(NULL, NULL,
-				"arm,vexpress-sysreg");
-
-		base = of_iomap(node, 0);
-	}
-
-	if (WARN_ON(!base))
-		return;
-
-	writel(~0, base + SYS_FLAGSCLR);
-	writel(data, base + SYS_FLAGSSET);
-}
-
 /* The sysreg block is just a random collection of various functions... */
 
-static struct syscon_platform_data vexpress_sysreg_sys_id_pdata = {
-	.label = "sys_id",
-};
-
 static struct bgpio_pdata vexpress_sysreg_sys_led_pdata = {
 	.label = "sys_led",
 	.base = -1,
@@ -84,24 +56,8 @@ static struct bgpio_pdata vexpress_sysreg_sys_flash_pdata = {
 	.ngpio = 1,
 };
 
-static struct syscon_platform_data vexpress_sysreg_sys_misc_pdata = {
-	.label = "sys_misc",
-};
-
-static struct syscon_platform_data vexpress_sysreg_sys_procid_pdata = {
-	.label = "sys_procid",
-};
-
 static struct mfd_cell vexpress_sysreg_cells[] = {
 	{
-		.name = "syscon",
-		.num_resources = 1,
-		.resources = (struct resource []) {
-			DEFINE_RES_MEM(SYS_ID, 0x4),
-		},
-		.platform_data = &vexpress_sysreg_sys_id_pdata,
-		.pdata_size = sizeof(vexpress_sysreg_sys_id_pdata),
-	}, {
 		.name = "basic-mmio-gpio",
 		.of_compatible = "arm,vexpress-sysreg,sys_led",
 		.num_resources = 1,
@@ -129,26 +85,10 @@ static struct mfd_cell vexpress_sysreg_cells[] = {
 		.platform_data = &vexpress_sysreg_sys_flash_pdata,
 		.pdata_size = sizeof(vexpress_sysreg_sys_flash_pdata),
 	}, {
-		.name = "syscon",
-		.num_resources = 1,
-		.resources = (struct resource []) {
-			DEFINE_RES_MEM(SYS_MISC, 0x4),
-		},
-		.platform_data = &vexpress_sysreg_sys_misc_pdata,
-		.pdata_size = sizeof(vexpress_sysreg_sys_misc_pdata),
-	}, {
-		.name = "syscon",
-		.num_resources = 1,
-		.resources = (struct resource []) {
-			DEFINE_RES_MEM(SYS_PROCID0, 0x8),
-		},
-		.platform_data = &vexpress_sysreg_sys_procid_pdata,
-		.pdata_size = sizeof(vexpress_sysreg_sys_procid_pdata),
-	}, {
 		.name = "vexpress-syscfg",
 		.num_resources = 1,
 		.resources = (struct resource []) {
-			DEFINE_RES_MEM(SYS_CFGDATA, 0xc),
+			DEFINE_RES_MEM(SYS_MISC, 0x4c),
 		},
 	}
 };
@@ -158,8 +98,6 @@ static int vexpress_sysreg_probe(struct platform_device *pdev)
 	struct resource *mem;
 	void __iomem *base;
 	struct gpio_chip *mmc_gpio_chip;
-	int master;
-	u32 dt_hbi;
 
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!mem)
@@ -169,21 +107,6 @@ static int vexpress_sysreg_probe(struct platform_device *pdev)
 	if (!base)
 		return -ENOMEM;
 
-	master = readl(base + SYS_MISC) & SYS_MISC_MASTERSITE ?
-			VEXPRESS_SITE_DB2 : VEXPRESS_SITE_DB1;
-	vexpress_config_set_master(master);
-
-	/* Confirm board type against DT property, if available */
-	if (of_property_read_u32(of_root, "arm,hbi", &dt_hbi) == 0) {
-		u32 id = readl(base + (master == VEXPRESS_SITE_DB1 ?
-				 SYS_PROCID0 : SYS_PROCID1));
-		u32 hbi = (id >> SYS_PROCIDx_HBI_SHIFT) & SYS_HBI_MASK;
-
-		if (WARN_ON(dt_hbi != hbi))
-			dev_warn(&pdev->dev, "DT HBI (%x) is not matching hardware (%x)!\n",
-					dt_hbi, hbi);
-	}
-
 	/*
 	 * Duplicated SYS_MCI pseudo-GPIO controller for compatibility with
 	 * older trees using sysreg node for MMC control lines.
@@ -195,9 +118,9 @@ static int vexpress_sysreg_probe(struct platform_device *pdev)
 	bgpio_init(mmc_gpio_chip, &pdev->dev, 0x4, base + SYS_MCI,
 			NULL, NULL, NULL, NULL, 0);
 	mmc_gpio_chip->ngpio = 2;
-	gpiochip_add_data(mmc_gpio_chip, NULL);
+	devm_gpiochip_add_data(&pdev->dev, mmc_gpio_chip, NULL);
 
-	return mfd_add_devices(&pdev->dev, PLATFORM_DEVID_AUTO,
+	return devm_mfd_add_devices(&pdev->dev, PLATFORM_DEVID_AUTO,
 			vexpress_sysreg_cells,
 			ARRAY_SIZE(vexpress_sysreg_cells), mem, 0, NULL);
 }
@@ -206,6 +129,7 @@ static const struct of_device_id vexpress_sysreg_match[] = {
 	{ .compatible = "arm,vexpress-sysreg", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, vexpress_sysreg_match);
 
 static struct platform_driver vexpress_sysreg_driver = {
 	.driver = {
@@ -215,14 +139,5 @@ static struct platform_driver vexpress_sysreg_driver = {
 	.probe = vexpress_sysreg_probe,
 };
 
-static int __init vexpress_sysreg_init(void)
-{
-	struct device_node *node;
-
-	/* Need the sysreg early, before any other device... */
-	for_each_matching_node(node, vexpress_sysreg_match)
-		of_platform_device_create(node, NULL, NULL);
-
-	return platform_driver_register(&vexpress_sysreg_driver);
-}
-core_initcall(vexpress_sysreg_init);
+module_platform_driver(vexpress_sysreg_driver);
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/wcd934x.c b/drivers/mfd/wcd934x.c
index 90341f3c6810..da910302d51a 100644
--- a/drivers/mfd/wcd934x.c
+++ b/drivers/mfd/wcd934x.c
@@ -280,7 +280,6 @@ static void wcd934x_slim_remove(struct slim_device *sdev)
 
 	regulator_bulk_disable(WCD934X_MAX_SUPPLY, ddata->supplies);
 	mfd_remove_devices(&sdev->dev);
-	kfree(ddata);
 }
 
 static const struct slim_device_id wcd934x_slim_id[] = {
diff --git a/drivers/mfd/wm8994-core.c b/drivers/mfd/wm8994-core.c
index 1e9fe7d92597..3b2b93c5bbcb 100644
--- a/drivers/mfd/wm8994-core.c
+++ b/drivers/mfd/wm8994-core.c
@@ -393,7 +393,9 @@ static int wm8994_device_init(struct wm8994 *wm8994, int irq)
 	ret = regulator_bulk_get(wm8994->dev, wm8994->num_supplies,
 				 wm8994->supplies);
 	if (ret != 0) {
-		dev_err(wm8994->dev, "Failed to get supplies: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(wm8994->dev, "Failed to get supplies: %d\n",
+				ret);
 		goto err;
 	}
 
@@ -584,6 +586,7 @@ static int wm8994_device_init(struct wm8994 *wm8994, int irq)
 		goto err_irq;
 	}
 
+	pm_runtime_set_active(wm8994->dev);
 	pm_runtime_enable(wm8994->dev);
 	pm_runtime_idle(wm8994->dev);
 
@@ -603,7 +606,9 @@ err:
 
 static void wm8994_device_exit(struct wm8994 *wm8994)
 {
+	pm_runtime_get_sync(wm8994->dev);
 	pm_runtime_disable(wm8994->dev);
+	pm_runtime_put_noidle(wm8994->dev);
 	wm8994_irq_exit(wm8994);
 	regulator_bulk_disable(wm8994->num_supplies, wm8994->supplies);
 	regulator_bulk_free(wm8994->num_supplies, wm8994->supplies);
@@ -690,3 +695,4 @@ module_i2c_driver(wm8994_i2c_driver);
 MODULE_DESCRIPTION("Core support for the WM8994 audio CODEC");
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Mark Brown <broonie@opensource.wolfsonmicro.com>");
+MODULE_SOFTDEP("pre: wm8994_regulator");
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 99e151475d8f..e1b1ba5e2b92 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -60,7 +60,7 @@ config ATMEL_TCLIB
 
 config DUMMY_IRQ
 	tristate "Dummy IRQ handler"
-	---help---
+	help
 	  This module accepts a single 'irq' parameter, which it should register for.
 	  The sole purpose of this module is to help with debugging of systems on
 	  which spurious IRQs would happen on disabled IRQ vector.
@@ -69,7 +69,7 @@ config IBM_ASM
 	tristate "Device driver for IBM RSA service processor"
 	depends on X86 && PCI && INPUT
 	depends on SERIAL_8250 || SERIAL_8250=n
-	---help---
+	help
 	  This option enables device driver support for in-band access to the
 	  IBM RSA (Condor) service processor in eServer xSeries systems.
 	  The ibmasm device driver allows user space application to access
@@ -167,7 +167,7 @@ config ICS932S401
 config ATMEL_SSC
 	tristate "Device driver for Atmel SSC peripheral"
 	depends on HAS_IOMEM && (ARCH_AT91 || COMPILE_TEST)
-	---help---
+	help
 	  This option enables device driver support for Atmel Synchronized
 	  Serial Communication peripheral (SSC).
 
@@ -190,7 +190,7 @@ config SGI_XP
 	depends on (IA64_SGI_UV || X86_UV) && SMP
 	depends on X86_64 || BROKEN
 	select SGI_GRU if X86_64 && SMP
-	---help---
+	help
 	  An SGI machine can be divided into multiple Single System
 	  Images which act independently of each other and have
 	  hardware based memory protection from the others.  Enabling
@@ -267,7 +267,7 @@ config SGI_GRU
 	tristate "SGI GRU driver"
 	depends on X86_UV && SMP
 	select MMU_NOTIFIER
-	---help---
+	help
 	The GRU is a hardware resource located in the system chipset. The GRU
 	contains memory that can be mmapped into the user address space. This memory is
 	used to communicate with the GRU to perform functions such as load/store,
@@ -280,7 +280,7 @@ config SGI_GRU
 config SGI_GRU_DEBUG
 	bool  "SGI GRU driver debug"
 	depends on SGI_GRU
-	---help---
+	help
 	This option enables additional debugging code for the SGI GRU driver.
 	If you are unsure, say N.
 
@@ -327,7 +327,7 @@ config SENSORS_TSL2550
 config SENSORS_BH1770
 	 tristate "BH1770GLC / SFH7770 combined ALS - Proximity sensor"
 	 depends on I2C
-	 ---help---
+	help
 	   Say Y here if you want to build a driver for BH1770GLC (ROHM) or
 	   SFH7770 (Osram) combined ambient light and proximity sensor chip.
 
@@ -337,7 +337,7 @@ config SENSORS_BH1770
 config SENSORS_APDS990X
 	 tristate "APDS990X combined als and proximity sensors"
 	 depends on I2C
-	 ---help---
+	help
 	   Say Y here if you want to build a driver for Avago APDS990x
 	   combined ambient light and proximity sensor chip.
 
@@ -423,20 +423,11 @@ config SRAM
 config SRAM_EXEC
 	bool
 
-config VEXPRESS_SYSCFG
-	bool "Versatile Express System Configuration driver"
-	depends on VEXPRESS_CONFIG
-	default y
-	help
-	  ARM Ltd. Versatile Express uses specialised platform configuration
-	  bus. System Configuration interface is one of the possible means
-	  of generating transactions on this bus.
-
 config PCI_ENDPOINT_TEST
 	depends on PCI
 	select CRC32
 	tristate "PCI Endpoint Test driver"
-	---help---
+	help
 	   Enable this configuration option to enable the host side test driver
 	   for PCI Endpoint.
 
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index 9abf2923d831..c7bd01ac6291 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -49,7 +49,6 @@ obj-$(CONFIG_SRAM_EXEC)		+= sram-exec.o
 obj-y				+= mic/
 obj-$(CONFIG_GENWQE)		+= genwqe/
 obj-$(CONFIG_ECHO)		+= echo/
-obj-$(CONFIG_VEXPRESS_SYSCFG)	+= vexpress-syscfg.o
 obj-$(CONFIG_CXL_BASE)		+= cxl/
 obj-$(CONFIG_PCI_ENDPOINT_TEST)	+= pci_endpoint_test.o
 obj-$(CONFIG_OCXL)		+= ocxl/
diff --git a/drivers/misc/cardreader/rts5249.c b/drivers/misc/cardreader/rts5249.c
index 1a81cda948c1..6c6c9e95a29f 100644
--- a/drivers/misc/cardreader/rts5249.c
+++ b/drivers/misc/cardreader/rts5249.c
@@ -347,31 +347,6 @@ static int rtsx_base_switch_output_voltage(struct rtsx_pcr *pcr, u8 voltage)
 	return rtsx_pci_send_cmd(pcr, 100);
 }
 
-static void rts5249_set_aspm(struct rtsx_pcr *pcr, bool enable)
-{
-	struct rtsx_cr_option *option = &pcr->option;
-	u8 val = 0;
-
-	if (pcr->aspm_enabled == enable)
-		return;
-
-	if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
-		if (enable)
-			val = pcr->aspm_en;
-		rtsx_pci_update_cfg_byte(pcr,
-			pcr->pcie_cap + PCI_EXP_LNKCTL,
-			ASPM_MASK_NEG, val);
-	} else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
-		u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
-		if (!enable)
-			val = FORCE_ASPM_CTL0;
-		rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
-	}
-
-	pcr->aspm_enabled = enable;
-}
-
 static const struct pcr_ops rts5249_pcr_ops = {
 	.fetch_vendor_settings = rtsx_base_fetch_vendor_settings,
 	.extra_init_hw = rts5249_extra_init_hw,
@@ -384,7 +359,6 @@ static const struct pcr_ops rts5249_pcr_ops = {
 	.card_power_off = rtsx_base_card_power_off,
 	.switch_output_voltage = rtsx_base_switch_output_voltage,
 	.force_power_down = rtsx_base_force_power_down,
-	.set_aspm = rts5249_set_aspm,
 };
 
 /* SD Pull Control Enable:
@@ -471,7 +445,6 @@ void rts5249_init_params(struct rtsx_pcr *pcr)
 	option->ltr_active_latency = LTR_ACTIVE_LATENCY_DEF;
 	option->ltr_idle_latency = LTR_IDLE_LATENCY_DEF;
 	option->ltr_l1off_latency = LTR_L1OFF_LATENCY_DEF;
-	option->dev_aspm_mode = DEV_ASPM_DYNAMIC;
 	option->l1_snooze_delay = L1_SNOOZE_DELAY_DEF;
 	option->ltr_l1off_sspwrgate = LTR_L1OFF_SSPWRGATE_5249_DEF;
 	option->ltr_l1off_snooze_sspwrgate =
@@ -612,7 +585,6 @@ static const struct pcr_ops rts524a_pcr_ops = {
 	.switch_output_voltage = rtsx_base_switch_output_voltage,
 	.force_power_down = rtsx_base_force_power_down,
 	.set_l1off_cfg_sub_d0 = rts5250_set_l1off_cfg_sub_d0,
-	.set_aspm = rts5249_set_aspm,
 };
 
 void rts524a_init_params(struct rtsx_pcr *pcr)
@@ -728,7 +700,6 @@ static const struct pcr_ops rts525a_pcr_ops = {
 	.switch_output_voltage = rts525a_switch_output_voltage,
 	.force_power_down = rtsx_base_force_power_down,
 	.set_l1off_cfg_sub_d0 = rts5250_set_l1off_cfg_sub_d0,
-	.set_aspm = rts5249_set_aspm,
 };
 
 void rts525a_init_params(struct rtsx_pcr *pcr)
diff --git a/drivers/misc/cardreader/rts5260.c b/drivers/misc/cardreader/rts5260.c
index 711054ebad74..7a9dbb778e84 100644
--- a/drivers/misc/cardreader/rts5260.c
+++ b/drivers/misc/cardreader/rts5260.c
@@ -570,30 +570,6 @@ static int rts5260_extra_init_hw(struct rtsx_pcr *pcr)
 	return 0;
 }
 
-static void rts5260_set_aspm(struct rtsx_pcr *pcr, bool enable)
-{
-	struct rtsx_cr_option *option = &pcr->option;
-	u8 val = 0;
-
-	if (pcr->aspm_enabled == enable)
-		return;
-
-	if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
-		if (enable)
-			val = pcr->aspm_en;
-		rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
-					 ASPM_MASK_NEG, val);
-	} else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
-		u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
-		if (!enable)
-			val = FORCE_ASPM_CTL0;
-		rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
-	}
-
-	pcr->aspm_enabled = enable;
-}
-
 static void rts5260_set_l1off_cfg_sub_d0(struct rtsx_pcr *pcr, int active)
 {
 	struct rtsx_cr_option *option = &pcr->option;
@@ -639,7 +615,6 @@ static const struct pcr_ops rts5260_pcr_ops = {
 	.switch_output_voltage = rts5260_switch_output_voltage,
 	.force_power_down = rtsx_base_force_power_down,
 	.stop_cmd = rts5260_stop_cmd,
-	.set_aspm = rts5260_set_aspm,
 	.set_l1off_cfg_sub_d0 = rts5260_set_l1off_cfg_sub_d0,
 	.enable_ocp = rts5260_enable_ocp,
 	.disable_ocp = rts5260_disable_ocp,
@@ -683,7 +658,6 @@ void rts5260_init_params(struct rtsx_pcr *pcr)
 	option->ltr_active_latency = LTR_ACTIVE_LATENCY_DEF;
 	option->ltr_idle_latency = LTR_IDLE_LATENCY_DEF;
 	option->ltr_l1off_latency = LTR_L1OFF_LATENCY_DEF;
-	option->dev_aspm_mode = DEV_ASPM_DYNAMIC;
 	option->l1_snooze_delay = L1_SNOOZE_DELAY_DEF;
 	option->ltr_l1off_sspwrgate = LTR_L1OFF_SSPWRGATE_5250_DEF;
 	option->ltr_l1off_snooze_sspwrgate =
diff --git a/drivers/misc/cardreader/rts5261.c b/drivers/misc/cardreader/rts5261.c
index 78c3b1d424c3..195822ec858e 100644
--- a/drivers/misc/cardreader/rts5261.c
+++ b/drivers/misc/cardreader/rts5261.c
@@ -518,51 +518,22 @@ static int rts5261_extra_init_hw(struct rtsx_pcr *pcr)
 
 static void rts5261_enable_aspm(struct rtsx_pcr *pcr, bool enable)
 {
-	struct rtsx_cr_option *option = &pcr->option;
-	u8 val = 0;
-
 	if (pcr->aspm_enabled == enable)
 		return;
 
-	if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
-		val = pcr->aspm_en;
-		rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
-					 ASPM_MASK_NEG, val);
-	} else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
-		u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
-		val = FORCE_ASPM_CTL0;
-		val |= (pcr->aspm_en & 0x02);
-		rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
-		val = pcr->aspm_en;
-		rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
-					 ASPM_MASK_NEG, val);
-	}
+	pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+					   PCI_EXP_LNKCTL_ASPMC, pcr->aspm_en);
 	pcr->aspm_enabled = enable;
 
 }
 
 static void rts5261_disable_aspm(struct rtsx_pcr *pcr, bool enable)
 {
-	struct rtsx_cr_option *option = &pcr->option;
-	u8 val = 0;
-
 	if (pcr->aspm_enabled == enable)
 		return;
 
-	if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
-		val = 0;
-		rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
-					 ASPM_MASK_NEG, val);
-	} else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
-		u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
-
-		val = 0;
-		rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
-					 ASPM_MASK_NEG, val);
-		val = FORCE_ASPM_CTL0;
-		rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
-	}
+	pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+					   PCI_EXP_LNKCTL_ASPMC, 0);
 	rtsx_pci_write_register(pcr, SD_CFG1, SD_ASYNC_FIFO_NOT_RST, 0);
 	udelay(10);
 	pcr->aspm_enabled = enable;
@@ -639,8 +610,13 @@ int rts5261_pci_switch_clock(struct rtsx_pcr *pcr, unsigned int card_clock,
 
 	if (initial_mode) {
 		/* We use 250k(around) here, in initial stage */
-		clk_divider = SD_CLK_DIVIDE_128;
-		card_clock = 30000000;
+		if (is_version(pcr, PID_5261, IC_VER_D)) {
+			clk_divider = SD_CLK_DIVIDE_256;
+			card_clock = 60000000;
+		} else {
+			clk_divider = SD_CLK_DIVIDE_128;
+			card_clock = 30000000;
+		}
 	} else {
 		clk_divider = SD_CLK_DIVIDE_0;
 	}
@@ -784,7 +760,6 @@ void rts5261_init_params(struct rtsx_pcr *pcr)
 	option->l1_snooze_delay = L1_SNOOZE_DELAY_DEF;
 	option->ltr_l1off_sspwrgate = 0x7F;
 	option->ltr_l1off_snooze_sspwrgate = 0x78;
-	option->dev_aspm_mode = DEV_ASPM_DYNAMIC;
 
 	option->ocp_en = 1;
 	hw_param->interrupt_en |= SD_OC_INT_EN;
diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c
index 55da6428ceb0..0d5928bc1b6d 100644
--- a/drivers/misc/cardreader/rtsx_pcr.c
+++ b/drivers/misc/cardreader/rtsx_pcr.c
@@ -55,16 +55,10 @@ static const struct pci_device_id rtsx_pci_ids[] = {
 
 MODULE_DEVICE_TABLE(pci, rtsx_pci_ids);
 
-static inline void rtsx_pci_enable_aspm(struct rtsx_pcr *pcr)
-{
-	rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
-		0xFC, pcr->aspm_en);
-}
-
 static inline void rtsx_pci_disable_aspm(struct rtsx_pcr *pcr)
 {
-	rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
-		0xFC, 0);
+	pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+					   PCI_EXP_LNKCTL_ASPMC, 0);
 }
 
 static int rtsx_comm_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency)
@@ -85,32 +79,17 @@ static int rtsx_comm_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency)
 
 int rtsx_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency)
 {
-	if (pcr->ops->set_ltr_latency)
-		return pcr->ops->set_ltr_latency(pcr, latency);
-	else
-		return rtsx_comm_set_ltr_latency(pcr, latency);
+	return rtsx_comm_set_ltr_latency(pcr, latency);
 }
 
 static void rtsx_comm_set_aspm(struct rtsx_pcr *pcr, bool enable)
 {
-	struct rtsx_cr_option *option = &pcr->option;
-
 	if (pcr->aspm_enabled == enable)
 		return;
 
-	if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
-		if (enable)
-			rtsx_pci_enable_aspm(pcr);
-		else
-			rtsx_pci_disable_aspm(pcr);
-	} else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
-		u8 mask = FORCE_ASPM_VAL_MASK;
-		u8 val = 0;
-
-		if (enable)
-			val = pcr->aspm_en;
-		rtsx_pci_write_register(pcr, ASPM_FORCE_CTL,  mask, val);
-	}
+	pcie_capability_clear_and_set_word(pcr->pci, PCI_EXP_LNKCTL,
+					   PCI_EXP_LNKCTL_ASPMC,
+					   enable ? pcr->aspm_en : 0);
 
 	pcr->aspm_enabled = enable;
 }
@@ -154,10 +133,7 @@ static void rtsx_comm_pm_full_on(struct rtsx_pcr *pcr)
 
 static void rtsx_pm_full_on(struct rtsx_pcr *pcr)
 {
-	if (pcr->ops->full_on)
-		pcr->ops->full_on(pcr);
-	else
-		rtsx_comm_pm_full_on(pcr);
+	rtsx_comm_pm_full_on(pcr);
 }
 
 void rtsx_pci_start_run(struct rtsx_pcr *pcr)
@@ -1111,10 +1087,7 @@ static void rtsx_comm_pm_power_saving(struct rtsx_pcr *pcr)
 
 static void rtsx_pm_power_saving(struct rtsx_pcr *pcr)
 {
-	if (pcr->ops->power_saving)
-		pcr->ops->power_saving(pcr);
-	else
-		rtsx_comm_pm_power_saving(pcr);
+	rtsx_comm_pm_power_saving(pcr);
 }
 
 static void rtsx_pci_idle_work(struct work_struct *work)
diff --git a/drivers/misc/cardreader/rtsx_pcr.h b/drivers/misc/cardreader/rtsx_pcr.h
index ed391df52f4f..024cbd998b2a 100644
--- a/drivers/misc/cardreader/rtsx_pcr.h
+++ b/drivers/misc/cardreader/rtsx_pcr.h
@@ -29,7 +29,6 @@
 #define LTR_L1OFF_SNOOZE_SSPWRGATE_5249_DEF	0xAC
 #define LTR_L1OFF_SNOOZE_SSPWRGATE_5250_DEF	0xF8
 #define CMD_TIMEOUT_DEF		100
-#define ASPM_MASK_NEG		0xFC
 #define MASK_8_BIT_DEF		0xFF
 
 #define SSC_CLOCK_STABLE_WAIT	130
diff --git a/drivers/misc/cxl/Kconfig b/drivers/misc/cxl/Kconfig
index 39eec9031487..51aecafdcbdf 100644
--- a/drivers/misc/cxl/Kconfig
+++ b/drivers/misc/cxl/Kconfig
@@ -7,18 +7,10 @@ config CXL_BASE
 	bool
 	select PPC_COPRO_BASE
 
-config CXL_AFU_DRIVER_OPS
-	bool
-
-config CXL_LIB
-	bool
-
 config CXL
 	tristate "Support for IBM Coherent Accelerators (CXL)"
 	depends on PPC_POWERNV && PCI_MSI && EEH
 	select CXL_BASE
-	select CXL_AFU_DRIVER_OPS
-	select CXL_LIB
 	default m
 	help
 	  Select this option to enable driver support for IBM Coherent
diff --git a/drivers/misc/cxl/cxllib.c b/drivers/misc/cxl/cxllib.c
index 258c43a95ac3..2a1783f32254 100644
--- a/drivers/misc/cxl/cxllib.c
+++ b/drivers/misc/cxl/cxllib.c
@@ -207,7 +207,7 @@ static int get_vma_info(struct mm_struct *mm, u64 addr,
 	struct vm_area_struct *vma = NULL;
 	int rc = 0;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 
 	vma = find_vma(mm, addr);
 	if (!vma) {
@@ -218,7 +218,7 @@ static int get_vma_info(struct mm_struct *mm, u64 addr,
 	*vma_start = vma->vm_start;
 	*vma_end = vma->vm_end;
 out:
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	return rc;
 }
 
@@ -245,9 +245,8 @@ int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags)
 	     dar += page_size) {
 		if (dar < vma_start || dar >= vma_end) {
 			/*
-			 * We don't hold the mm->mmap_sem semaphore
-			 * while iterating, since the semaphore is
-			 * required by one of the lower-level page
+			 * We don't hold mm->mmap_lock while iterating, since
+			 * the lock is required by one of the lower-level page
 			 * fault processing functions and it could
 			 * create a deadlock.
 			 *
diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c
index 2297e6fc1544..01153b74334a 100644
--- a/drivers/misc/cxl/fault.c
+++ b/drivers/misc/cxl/fault.c
@@ -321,7 +321,7 @@ static void cxl_prefault_vma(struct cxl_context *ctx)
 		return;
 	}
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 		for (ea = vma->vm_start; ea < vma->vm_end;
 				ea = next_segment(ea, slb.vsid)) {
@@ -336,7 +336,7 @@ static void cxl_prefault_vma(struct cxl_context *ctx)
 			last_esid = slb.esid;
 		}
 	}
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 
 	mmput(mm);
 }
diff --git a/drivers/misc/echo/Kconfig b/drivers/misc/echo/Kconfig
index be70b263e271..ce0a37a47fc1 100644
--- a/drivers/misc/echo/Kconfig
+++ b/drivers/misc/echo/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config ECHO
 	tristate "Line Echo Canceller support"
-	---help---
+	help
 	  This driver provides line echo cancelling support for mISDN and
 	  Zaptel drivers.
 
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
index e3e085e33d46..7939c55daceb 100644
--- a/drivers/misc/fastrpc.c
+++ b/drivers/misc/fastrpc.c
@@ -904,6 +904,7 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
 	struct fastrpc_channel_ctx *cctx;
 	struct fastrpc_user *fl = ctx->fl;
 	struct fastrpc_msg *msg = &ctx->msg;
+	int ret;
 
 	cctx = fl->cctx;
 	msg->pid = fl->tgid;
@@ -919,7 +920,13 @@ static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
 	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
 	fastrpc_context_get(ctx);
 
-	return rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
+	ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));
+
+	if (ret)
+		fastrpc_context_put(ctx);
+
+	return ret;
+
 }
 
 static int fastrpc_internal_invoke(struct fastrpc_user *fl,  u32 kernel,
@@ -1613,8 +1620,10 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
 					    domains[domain_id]);
 	data->miscdev.fops = &fastrpc_fops;
 	err = misc_register(&data->miscdev);
-	if (err)
+	if (err) {
+		kfree(data);
 		return err;
+	}
 
 	kref_init(&data->refcount);
 
diff --git a/drivers/misc/genwqe/card_utils.c b/drivers/misc/genwqe/card_utils.c
index 2e1c4d2905e8..77c21caf2acd 100644
--- a/drivers/misc/genwqe/card_utils.c
+++ b/drivers/misc/genwqe/card_utils.c
@@ -27,7 +27,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/delay.h>
-#include <asm/pgtable.h>
+#include <linux/pgtable.h>
 
 #include "genwqe_driver.h"
 #include "card_base.h"
@@ -515,30 +515,6 @@ int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
 }
 
 /**
- * genwqe_free_user_pages() - Give pinned pages back
- *
- * Documentation of get_user_pages is in mm/gup.c:
- *
- * If the page is written to, set_page_dirty (or set_page_dirty_lock,
- * as appropriate) must be called after the page is finished with, and
- * before put_page is called.
- */
-static int genwqe_free_user_pages(struct page **page_list,
-			unsigned int nr_pages, int dirty)
-{
-	unsigned int i;
-
-	for (i = 0; i < nr_pages; i++) {
-		if (page_list[i] != NULL) {
-			if (dirty)
-				set_page_dirty_lock(page_list[i]);
-			put_page(page_list[i]);
-		}
-	}
-	return 0;
-}
-
-/**
  * genwqe_user_vmap() - Map user-space memory to virtual kernel memory
  * @cd:         pointer to genwqe device
  * @m:          mapping params
@@ -597,18 +573,18 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
 	m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages);
 
 	/* pin user pages in memory */
-	rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
+	rc = pin_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
 				 m->nr_pages,
 				 m->write ? FOLL_WRITE : 0,	/* readable/writable */
 				 m->page_list);	/* ptrs to pages */
 	if (rc < 0)
-		goto fail_get_user_pages;
+		goto fail_pin_user_pages;
 
-	/* assumption: get_user_pages can be killed by signals. */
+	/* assumption: pin_user_pages can be killed by signals. */
 	if (rc < m->nr_pages) {
-		genwqe_free_user_pages(m->page_list, rc, m->write);
+		unpin_user_pages_dirty_lock(m->page_list, rc, m->write);
 		rc = -EFAULT;
-		goto fail_get_user_pages;
+		goto fail_pin_user_pages;
 	}
 
 	rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list);
@@ -618,9 +594,9 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
 	return 0;
 
  fail_free_user_pages:
-	genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
+	unpin_user_pages_dirty_lock(m->page_list, m->nr_pages, m->write);
 
- fail_get_user_pages:
+ fail_pin_user_pages:
 	kfree(m->page_list);
 	m->page_list = NULL;
 	m->dma_list = NULL;
@@ -650,8 +626,8 @@ int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m)
 		genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
 
 	if (m->page_list) {
-		genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
-
+		unpin_user_pages_dirty_lock(m->page_list, m->nr_pages,
+					    m->write);
 		kfree(m->page_list);
 		m->page_list = NULL;
 		m->dma_list = NULL;
diff --git a/drivers/misc/habanalabs/Makefile b/drivers/misc/habanalabs/Makefile
index 482f6227dbba..421ebd903069 100644
--- a/drivers/misc/habanalabs/Makefile
+++ b/drivers/misc/habanalabs/Makefile
@@ -13,3 +13,6 @@ habanalabs-$(CONFIG_DEBUG_FS) += debugfs.o
 
 include $(src)/goya/Makefile
 habanalabs-y += $(HL_GOYA_FILES)
+
+include $(src)/gaudi/Makefile
+habanalabs-y += $(HL_GAUDI_FILES)
diff --git a/drivers/misc/habanalabs/command_buffer.c b/drivers/misc/habanalabs/command_buffer.c
index 53fddbd8e693..02d13f71b1df 100644
--- a/drivers/misc/habanalabs/command_buffer.c
+++ b/drivers/misc/habanalabs/command_buffer.c
@@ -105,10 +105,9 @@ int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 		goto out_err;
 	}
 
-	if (cb_size > HL_MAX_CB_SIZE) {
-		dev_err(hdev->dev,
-			"CB size %d must be less then %d\n",
-			cb_size, HL_MAX_CB_SIZE);
+	if (cb_size > SZ_2M) {
+		dev_err(hdev->dev, "CB size %d must be less than %d\n",
+			cb_size, SZ_2M);
 		rc = -EINVAL;
 		goto out_err;
 	}
@@ -211,7 +210,7 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
 {
 	union hl_cb_args *args = data;
 	struct hl_device *hdev = hpriv->hdev;
-	u64 handle;
+	u64 handle = 0;
 	int rc;
 
 	if (hl_device_disabled_or_in_reset(hdev)) {
@@ -223,15 +222,26 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
 
 	switch (args->in.op) {
 	case HL_CB_OP_CREATE:
-		rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size,
-					&handle, hpriv->ctx->asid);
+		if (args->in.cb_size > HL_MAX_CB_SIZE) {
+			dev_err(hdev->dev,
+				"User requested CB size %d must be less than %d\n",
+				args->in.cb_size, HL_MAX_CB_SIZE);
+			rc = -EINVAL;
+		} else {
+			rc = hl_cb_create(hdev, &hpriv->cb_mgr,
+						args->in.cb_size, &handle,
+						hpriv->ctx->asid);
+		}
+
 		memset(args, 0, sizeof(*args));
 		args->out.cb_handle = handle;
 		break;
+
 	case HL_CB_OP_DESTROY:
 		rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
 					args->in.cb_handle);
 		break;
+
 	default:
 		rc = -ENOTTY;
 		break;
@@ -278,7 +288,7 @@ int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
 	cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
 	if (!cb) {
 		dev_err(hdev->dev,
-			"CB mmap failed, no match to handle %d\n", handle);
+			"CB mmap failed, no match to handle 0x%x\n", handle);
 		return -EINVAL;
 	}
 
@@ -347,7 +357,7 @@ struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 	if (!cb) {
 		spin_unlock(&mgr->cb_lock);
 		dev_warn(hdev->dev,
-			"CB get failed, no match to handle %d\n", handle);
+			"CB get failed, no match to handle 0x%x\n", handle);
 		return NULL;
 	}
 
diff --git a/drivers/misc/habanalabs/command_submission.c b/drivers/misc/habanalabs/command_submission.c
index 409276b6374d..f82974a916c3 100644
--- a/drivers/misc/habanalabs/command_submission.c
+++ b/drivers/misc/habanalabs/command_submission.c
@@ -11,11 +11,33 @@
 #include <linux/uaccess.h>
 #include <linux/slab.h>
 
+#define HL_CS_FLAGS_SIG_WAIT	(HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT)
+
 static void job_wq_completion(struct work_struct *work);
 static long _hl_cs_wait_ioctl(struct hl_device *hdev,
 		struct hl_ctx *ctx, u64 timeout_us, u64 seq);
 static void cs_do_release(struct kref *ref);
 
+static void hl_sob_reset(struct kref *ref)
+{
+	struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
+							kref);
+	struct hl_device *hdev = hw_sob->hdev;
+
+	hdev->asic_funcs->reset_sob(hdev, hw_sob);
+}
+
+void hl_sob_reset_error(struct kref *ref)
+{
+	struct hl_hw_sob *hw_sob = container_of(ref, struct hl_hw_sob,
+							kref);
+	struct hl_device *hdev = hw_sob->hdev;
+
+	dev_crit(hdev->dev,
+			"SOB release shouldn't be called here, q_idx: %d, sob_id: %d\n",
+			hw_sob->q_idx, hw_sob->sob_id);
+}
+
 static const char *hl_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "HabanaLabs";
@@ -23,10 +45,10 @@ static const char *hl_fence_get_driver_name(struct dma_fence *fence)
 
 static const char *hl_fence_get_timeline_name(struct dma_fence *fence)
 {
-	struct hl_dma_fence *hl_fence =
-		container_of(fence, struct hl_dma_fence, base_fence);
+	struct hl_cs_compl *hl_cs_compl =
+		container_of(fence, struct hl_cs_compl, base_fence);
 
-	return dev_name(hl_fence->hdev->dev);
+	return dev_name(hl_cs_compl->hdev->dev);
 }
 
 static bool hl_fence_enable_signaling(struct dma_fence *fence)
@@ -36,17 +58,47 @@ static bool hl_fence_enable_signaling(struct dma_fence *fence)
 
 static void hl_fence_release(struct dma_fence *fence)
 {
-	struct hl_dma_fence *hl_fence =
-		container_of(fence, struct hl_dma_fence, base_fence);
+	struct hl_cs_compl *hl_cs_cmpl =
+		container_of(fence, struct hl_cs_compl, base_fence);
+	struct hl_device *hdev = hl_cs_cmpl->hdev;
+
+	if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) ||
+			(hl_cs_cmpl->type == CS_TYPE_WAIT)) {
 
-	kfree_rcu(hl_fence, base_fence.rcu);
+		dev_dbg(hdev->dev,
+			"CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n",
+			hl_cs_cmpl->cs_seq,
+			hl_cs_cmpl->type,
+			hl_cs_cmpl->hw_sob->sob_id,
+			hl_cs_cmpl->sob_val);
+
+		/*
+		 * A signal CS can get completion while the corresponding wait
+		 * for signal CS is on its way to the PQ. The wait for signal CS
+		 * will get stuck if the signal CS incremented the SOB to its
+		 * max value and there are no pending (submitted) waits on this
+		 * SOB.
+		 * We do the following to void this situation:
+		 * 1. The wait for signal CS must get a ref for the signal CS as
+		 *    soon as possible in cs_ioctl_signal_wait() and put it
+		 *    before being submitted to the PQ but after it incremented
+		 *    the SOB refcnt in init_signal_wait_cs().
+		 * 2. Signal/Wait for signal CS will decrement the SOB refcnt
+		 *    here.
+		 * These two measures guarantee that the wait for signal CS will
+		 * reset the SOB upon completion rather than the signal CS and
+		 * hence the above scenario is avoided.
+		 */
+		kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset);
+	}
+
+	kfree_rcu(hl_cs_cmpl, base_fence.rcu);
 }
 
 static const struct dma_fence_ops hl_fence_ops = {
 	.get_driver_name = hl_fence_get_driver_name,
 	.get_timeline_name = hl_fence_get_timeline_name,
 	.enable_signaling = hl_fence_enable_signaling,
-	.wait = dma_fence_default_wait,
 	.release = hl_fence_release
 };
 
@@ -113,6 +165,7 @@ static int cs_parser(struct hl_fpriv *hpriv, struct hl_cs_job *job)
 		if (!rc) {
 			job->patched_cb = parser.patched_cb;
 			job->job_cb_size = parser.patched_cb_size;
+			job->contains_dma_pkt = parser.contains_dma_pkt;
 
 			spin_lock(&job->patched_cb->lock);
 			job->patched_cb->cs_cnt++;
@@ -259,6 +312,12 @@ static void cs_do_release(struct kref *ref)
 
 			spin_unlock(&hdev->hw_queues_mirror_lock);
 		}
+	} else if (cs->type == CS_TYPE_WAIT) {
+		/*
+		 * In case the wait for signal CS was submitted, the put occurs
+		 * in init_signal_wait_cs() right before hanging on the PQ.
+		 */
+		dma_fence_put(cs->signal_fence);
 	}
 
 	/*
@@ -312,9 +371,9 @@ static void cs_timedout(struct work_struct *work)
 }
 
 static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
-			struct hl_cs **cs_new)
+			enum hl_cs_type cs_type, struct hl_cs **cs_new)
 {
-	struct hl_dma_fence *fence;
+	struct hl_cs_compl *cs_cmpl;
 	struct dma_fence *other = NULL;
 	struct hl_cs *cs;
 	int rc;
@@ -326,25 +385,27 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
 	cs->ctx = ctx;
 	cs->submitted = false;
 	cs->completed = false;
+	cs->type = cs_type;
 	INIT_LIST_HEAD(&cs->job_list);
 	INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);
 	kref_init(&cs->refcount);
 	spin_lock_init(&cs->job_lock);
 
-	fence = kmalloc(sizeof(*fence), GFP_ATOMIC);
-	if (!fence) {
+	cs_cmpl = kmalloc(sizeof(*cs_cmpl), GFP_ATOMIC);
+	if (!cs_cmpl) {
 		rc = -ENOMEM;
 		goto free_cs;
 	}
 
-	fence->hdev = hdev;
-	spin_lock_init(&fence->lock);
-	cs->fence = &fence->base_fence;
+	cs_cmpl->hdev = hdev;
+	cs_cmpl->type = cs->type;
+	spin_lock_init(&cs_cmpl->lock);
+	cs->fence = &cs_cmpl->base_fence;
 
 	spin_lock(&ctx->cs_lock);
 
-	fence->cs_seq = ctx->cs_sequence;
-	other = ctx->cs_pending[fence->cs_seq & (HL_MAX_PENDING_CS - 1)];
+	cs_cmpl->cs_seq = ctx->cs_sequence;
+	other = ctx->cs_pending[cs_cmpl->cs_seq & (HL_MAX_PENDING_CS - 1)];
 	if ((other) && (!dma_fence_is_signaled(other))) {
 		spin_unlock(&ctx->cs_lock);
 		dev_dbg(hdev->dev,
@@ -353,16 +414,16 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
 		goto free_fence;
 	}
 
-	dma_fence_init(&fence->base_fence, &hl_fence_ops, &fence->lock,
+	dma_fence_init(&cs_cmpl->base_fence, &hl_fence_ops, &cs_cmpl->lock,
 			ctx->asid, ctx->cs_sequence);
 
-	cs->sequence = fence->cs_seq;
+	cs->sequence = cs_cmpl->cs_seq;
 
-	ctx->cs_pending[fence->cs_seq & (HL_MAX_PENDING_CS - 1)] =
-							&fence->base_fence;
+	ctx->cs_pending[cs_cmpl->cs_seq & (HL_MAX_PENDING_CS - 1)] =
+							&cs_cmpl->base_fence;
 	ctx->cs_sequence++;
 
-	dma_fence_get(&fence->base_fence);
+	dma_fence_get(&cs_cmpl->base_fence);
 
 	dma_fence_put(other);
 
@@ -373,7 +434,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
 	return 0;
 
 free_fence:
-	kfree(fence);
+	kfree(cs_cmpl);
 free_cs:
 	kfree(cs);
 	return rc;
@@ -499,8 +560,8 @@ struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
 	return job;
 }
 
-static int _hl_cs_ioctl(struct hl_fpriv *hpriv, void __user *chunks,
-			u32 num_chunks, u64 *cs_seq)
+static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
+				u32 num_chunks, u64 *cs_seq)
 {
 	struct hl_device *hdev = hpriv->hdev;
 	struct hl_cs_chunk *cs_chunk_array;
@@ -538,7 +599,7 @@ static int _hl_cs_ioctl(struct hl_fpriv *hpriv, void __user *chunks,
 	/* increment refcnt for context */
 	hl_ctx_get(hdev, hpriv->ctx);
 
-	rc = allocate_cs(hdev, hpriv->ctx, &cs);
+	rc = allocate_cs(hdev, hpriv->ctx, CS_TYPE_DEFAULT, &cs);
 	if (rc) {
 		hl_ctx_put(hpriv->ctx);
 		goto free_cs_chunk_array;
@@ -652,13 +713,230 @@ out:
 	return rc;
 }
 
+static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
+				void __user *chunks, u32 num_chunks,
+				u64 *cs_seq)
+{
+	struct hl_device *hdev = hpriv->hdev;
+	struct hl_ctx *ctx = hpriv->ctx;
+	struct hl_cs_chunk *cs_chunk_array, *chunk;
+	struct hw_queue_properties *hw_queue_prop;
+	struct dma_fence *sig_fence = NULL;
+	struct hl_cs_job *job;
+	struct hl_cs *cs;
+	struct hl_cb *cb;
+	u64 *signal_seq_arr = NULL, signal_seq;
+	u32 size_to_copy, q_idx, signal_seq_arr_len, cb_size;
+	int rc;
+
+	*cs_seq = ULLONG_MAX;
+
+	if (num_chunks > HL_MAX_JOBS_PER_CS) {
+		dev_err(hdev->dev,
+			"Number of chunks can NOT be larger than %d\n",
+			HL_MAX_JOBS_PER_CS);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	cs_chunk_array = kmalloc_array(num_chunks, sizeof(*cs_chunk_array),
+					GFP_ATOMIC);
+	if (!cs_chunk_array) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	size_to_copy = num_chunks * sizeof(struct hl_cs_chunk);
+	if (copy_from_user(cs_chunk_array, chunks, size_to_copy)) {
+		dev_err(hdev->dev, "Failed to copy cs chunk array from user\n");
+		rc = -EFAULT;
+		goto free_cs_chunk_array;
+	}
+
+	/* currently it is guaranteed to have only one chunk */
+	chunk = &cs_chunk_array[0];
+	q_idx = chunk->queue_index;
+	hw_queue_prop = &hdev->asic_prop.hw_queues_props[q_idx];
+
+	if ((q_idx >= HL_MAX_QUEUES) ||
+			(hw_queue_prop->type != QUEUE_TYPE_EXT)) {
+		dev_err(hdev->dev, "Queue index %d is invalid\n", q_idx);
+		rc = -EINVAL;
+		goto free_cs_chunk_array;
+	}
+
+	if (cs_type == CS_TYPE_WAIT) {
+		struct hl_cs_compl *sig_waitcs_cmpl;
+
+		signal_seq_arr_len = chunk->num_signal_seq_arr;
+
+		/* currently only one signal seq is supported */
+		if (signal_seq_arr_len != 1) {
+			dev_err(hdev->dev,
+				"Wait for signal CS supports only one signal CS seq\n");
+			rc = -EINVAL;
+			goto free_cs_chunk_array;
+		}
+
+		signal_seq_arr = kmalloc_array(signal_seq_arr_len,
+						sizeof(*signal_seq_arr),
+						GFP_ATOMIC);
+		if (!signal_seq_arr) {
+			rc = -ENOMEM;
+			goto free_cs_chunk_array;
+		}
+
+		size_to_copy = chunk->num_signal_seq_arr *
+				sizeof(*signal_seq_arr);
+		if (copy_from_user(signal_seq_arr,
+					u64_to_user_ptr(chunk->signal_seq_arr),
+					size_to_copy)) {
+			dev_err(hdev->dev,
+				"Failed to copy signal seq array from user\n");
+			rc = -EFAULT;
+			goto free_signal_seq_array;
+		}
+
+		/* currently it is guaranteed to have only one signal seq */
+		signal_seq = signal_seq_arr[0];
+		sig_fence = hl_ctx_get_fence(ctx, signal_seq);
+		if (IS_ERR(sig_fence)) {
+			dev_err(hdev->dev,
+				"Failed to get signal CS with seq 0x%llx\n",
+				signal_seq);
+			rc = PTR_ERR(sig_fence);
+			goto free_signal_seq_array;
+		}
+
+		if (!sig_fence) {
+			/* signal CS already finished */
+			rc = 0;
+			goto free_signal_seq_array;
+		}
+
+		sig_waitcs_cmpl =
+			container_of(sig_fence, struct hl_cs_compl, base_fence);
+
+		if (sig_waitcs_cmpl->type != CS_TYPE_SIGNAL) {
+			dev_err(hdev->dev,
+				"CS seq 0x%llx is not of a signal CS\n",
+				signal_seq);
+			dma_fence_put(sig_fence);
+			rc = -EINVAL;
+			goto free_signal_seq_array;
+		}
+
+		if (dma_fence_is_signaled(sig_fence)) {
+			/* signal CS already finished */
+			dma_fence_put(sig_fence);
+			rc = 0;
+			goto free_signal_seq_array;
+		}
+	}
+
+	/* increment refcnt for context */
+	hl_ctx_get(hdev, ctx);
+
+	rc = allocate_cs(hdev, ctx, cs_type, &cs);
+	if (rc) {
+		if (cs_type == CS_TYPE_WAIT)
+			dma_fence_put(sig_fence);
+		hl_ctx_put(ctx);
+		goto free_signal_seq_array;
+	}
+
+	/*
+	 * Save the signal CS fence for later initialization right before
+	 * hanging the wait CS on the queue.
+	 */
+	if (cs->type == CS_TYPE_WAIT)
+		cs->signal_fence = sig_fence;
+
+	hl_debugfs_add_cs(cs);
+
+	*cs_seq = cs->sequence;
+
+	job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
+	if (!job) {
+		dev_err(hdev->dev, "Failed to allocate a new job\n");
+		rc = -ENOMEM;
+		goto put_cs;
+	}
+
+	cb = hl_cb_kernel_create(hdev, PAGE_SIZE);
+	if (!cb) {
+		kfree(job);
+		rc = -EFAULT;
+		goto put_cs;
+	}
+
+	if (cs->type == CS_TYPE_WAIT)
+		cb_size = hdev->asic_funcs->get_wait_cb_size(hdev);
+	else
+		cb_size = hdev->asic_funcs->get_signal_cb_size(hdev);
+
+	job->id = 0;
+	job->cs = cs;
+	job->user_cb = cb;
+	job->user_cb->cs_cnt++;
+	job->user_cb_size = cb_size;
+	job->hw_queue_id = q_idx;
+
+	/*
+	 * No need in parsing, user CB is the patched CB.
+	 * We call hl_cb_destroy() out of two reasons - we don't need the CB in
+	 * the CB idr anymore and to decrement its refcount as it was
+	 * incremented inside hl_cb_kernel_create().
+	 */
+	job->patched_cb = job->user_cb;
+	job->job_cb_size = job->user_cb_size;
+	hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
+
+	cs->jobs_in_queue_cnt[job->hw_queue_id]++;
+
+	list_add_tail(&job->cs_node, &cs->job_list);
+
+	/* increment refcount as for external queues we get completion */
+	cs_get(cs);
+
+	hl_debugfs_add_job(hdev, job);
+
+	rc = hl_hw_queue_schedule_cs(cs);
+	if (rc) {
+		if (rc != -EAGAIN)
+			dev_err(hdev->dev,
+				"Failed to submit CS %d.%llu to H/W queues, error %d\n",
+				ctx->asid, cs->sequence, rc);
+		goto free_cs_object;
+	}
+
+	rc = HL_CS_STATUS_SUCCESS;
+	goto put_cs;
+
+free_cs_object:
+	cs_rollback(hdev, cs);
+	*cs_seq = ULLONG_MAX;
+	/* The path below is both for good and erroneous exits */
+put_cs:
+	/* We finished with the CS in this function, so put the ref */
+	cs_put(cs);
+free_signal_seq_array:
+	if (cs_type == CS_TYPE_WAIT)
+		kfree(signal_seq_arr);
+free_cs_chunk_array:
+	kfree(cs_chunk_array);
+out:
+	return rc;
+}
+
 int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
 {
 	struct hl_device *hdev = hpriv->hdev;
 	union hl_cs_args *args = data;
 	struct hl_ctx *ctx = hpriv->ctx;
 	void __user *chunks_execute, *chunks_restore;
-	u32 num_chunks_execute, num_chunks_restore;
+	enum hl_cs_type cs_type;
+	u32 num_chunks_execute, num_chunks_restore, sig_wait_flags;
 	u64 cs_seq = ULONG_MAX;
 	int rc, do_ctx_switch;
 	bool need_soft_reset = false;
@@ -671,12 +949,44 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
 		goto out;
 	}
 
+	sig_wait_flags = args->in.cs_flags & HL_CS_FLAGS_SIG_WAIT;
+
+	if (unlikely(sig_wait_flags == HL_CS_FLAGS_SIG_WAIT)) {
+		dev_err(hdev->dev,
+			"Signal and wait CS flags are mutually exclusive, context %d\n",
+		ctx->asid);
+		rc = -EINVAL;
+		goto out;
+	}
+
+	if (unlikely((sig_wait_flags & HL_CS_FLAGS_SIG_WAIT) &&
+			(!hdev->supports_sync_stream))) {
+		dev_err(hdev->dev, "Sync stream CS is not supported\n");
+		rc = -EINVAL;
+		goto out;
+	}
+
+	if (args->in.cs_flags & HL_CS_FLAGS_SIGNAL)
+		cs_type = CS_TYPE_SIGNAL;
+	else if (args->in.cs_flags & HL_CS_FLAGS_WAIT)
+		cs_type = CS_TYPE_WAIT;
+	else
+		cs_type = CS_TYPE_DEFAULT;
+
 	chunks_execute = (void __user *) (uintptr_t) args->in.chunks_execute;
 	num_chunks_execute = args->in.num_chunks_execute;
 
-	if (!num_chunks_execute) {
+	if (cs_type == CS_TYPE_DEFAULT) {
+		if (!num_chunks_execute) {
+			dev_err(hdev->dev,
+				"Got execute CS with 0 chunks, context %d\n",
+				ctx->asid);
+			rc = -EINVAL;
+			goto out;
+		}
+	} else if (num_chunks_execute != 1) {
 		dev_err(hdev->dev,
-			"Got execute CS with 0 chunks, context %d\n",
+			"Sync stream CS mandates one chunk only, context %d\n",
 			ctx->asid);
 		rc = -EINVAL;
 		goto out;
@@ -722,7 +1032,7 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
 			"Need to run restore phase but restore CS is empty\n");
 			rc = 0;
 		} else {
-			rc = _hl_cs_ioctl(hpriv, chunks_restore,
+			rc = cs_ioctl_default(hpriv, chunks_restore,
 						num_chunks_restore, &cs_seq);
 		}
 
@@ -764,7 +1074,12 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data)
 		}
 	}
 
-	rc = _hl_cs_ioctl(hpriv, chunks_execute, num_chunks_execute, &cs_seq);
+	if (cs_type == CS_TYPE_DEFAULT)
+		rc = cs_ioctl_default(hpriv, chunks_execute, num_chunks_execute,
+					&cs_seq);
+	else
+		rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks_execute,
+						num_chunks_execute, &cs_seq);
 
 out:
 	if (rc != -EAGAIN) {
@@ -796,6 +1111,10 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev,
 	fence = hl_ctx_get_fence(ctx, seq);
 	if (IS_ERR(fence)) {
 		rc = PTR_ERR(fence);
+		if (rc == -EINVAL)
+			dev_notice_ratelimited(hdev->dev,
+				"Can't wait on seq %llu because current CS is at seq %llu\n",
+				seq, ctx->cs_sequence);
 	} else if (fence) {
 		rc = dma_fence_wait_timeout(fence, true, timeout);
 		if (fence->error == -ETIMEDOUT)
@@ -803,8 +1122,12 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev,
 		else if (fence->error == -EIO)
 			rc = -EIO;
 		dma_fence_put(fence);
-	} else
+	} else {
+		dev_dbg(hdev->dev,
+			"Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
+			seq, ctx->cs_sequence);
 		rc = 1;
+	}
 
 	hl_ctx_put(ctx);
 
diff --git a/drivers/misc/habanalabs/context.c b/drivers/misc/habanalabs/context.c
index 2df6fb87e7ff..ec92b3506b1f 100644
--- a/drivers/misc/habanalabs/context.c
+++ b/drivers/misc/habanalabs/context.c
@@ -170,24 +170,16 @@ int hl_ctx_put(struct hl_ctx *ctx)
 
 struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
 {
-	struct hl_device *hdev = ctx->hdev;
 	struct dma_fence *fence;
 
 	spin_lock(&ctx->cs_lock);
 
 	if (seq >= ctx->cs_sequence) {
-		dev_notice_ratelimited(hdev->dev,
-			"Can't wait on seq %llu because current CS is at seq %llu\n",
-			seq, ctx->cs_sequence);
 		spin_unlock(&ctx->cs_lock);
 		return ERR_PTR(-EINVAL);
 	}
 
-
 	if (seq + HL_MAX_PENDING_CS < ctx->cs_sequence) {
-		dev_dbg(hdev->dev,
-			"Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n",
-			seq, ctx->cs_sequence);
 		spin_unlock(&ctx->cs_lock);
 		return NULL;
 	}
diff --git a/drivers/misc/habanalabs/debugfs.c b/drivers/misc/habanalabs/debugfs.c
index 756d36ed5d95..3c8dcdfba20c 100644
--- a/drivers/misc/habanalabs/debugfs.c
+++ b/drivers/misc/habanalabs/debugfs.c
@@ -970,6 +970,98 @@ static ssize_t hl_device_write(struct file *f, const char __user *buf,
 	return count;
 }
 
+static ssize_t hl_clk_gate_read(struct file *f, char __user *buf,
+					size_t count, loff_t *ppos)
+{
+	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+	struct hl_device *hdev = entry->hdev;
+	char tmp_buf[200];
+	ssize_t rc;
+
+	if (*ppos)
+		return 0;
+
+	sprintf(tmp_buf, "%d\n", hdev->clock_gating);
+	rc = simple_read_from_buffer(buf, strlen(tmp_buf) + 1, ppos, tmp_buf,
+			strlen(tmp_buf) + 1);
+
+	return rc;
+}
+
+static ssize_t hl_clk_gate_write(struct file *f, const char __user *buf,
+				     size_t count, loff_t *ppos)
+{
+	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+	struct hl_device *hdev = entry->hdev;
+	u32 value;
+	ssize_t rc;
+
+	if (atomic_read(&hdev->in_reset)) {
+		dev_warn_ratelimited(hdev->dev,
+				"Can't change clock gating during reset\n");
+		return 0;
+	}
+
+	rc = kstrtouint_from_user(buf, count, 10, &value);
+	if (rc)
+		return rc;
+
+	if (value) {
+		hdev->clock_gating = 1;
+		if (hdev->asic_funcs->enable_clock_gating)
+			hdev->asic_funcs->enable_clock_gating(hdev);
+	} else {
+		if (hdev->asic_funcs->disable_clock_gating)
+			hdev->asic_funcs->disable_clock_gating(hdev);
+		hdev->clock_gating = 0;
+	}
+
+	return count;
+}
+
+static ssize_t hl_stop_on_err_read(struct file *f, char __user *buf,
+					size_t count, loff_t *ppos)
+{
+	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+	struct hl_device *hdev = entry->hdev;
+	char tmp_buf[200];
+	ssize_t rc;
+
+	if (*ppos)
+		return 0;
+
+	sprintf(tmp_buf, "%d\n", hdev->stop_on_err);
+	rc = simple_read_from_buffer(buf, strlen(tmp_buf) + 1, ppos, tmp_buf,
+			strlen(tmp_buf) + 1);
+
+	return rc;
+}
+
+static ssize_t hl_stop_on_err_write(struct file *f, const char __user *buf,
+				     size_t count, loff_t *ppos)
+{
+	struct hl_dbg_device_entry *entry = file_inode(f)->i_private;
+	struct hl_device *hdev = entry->hdev;
+	u32 value;
+	ssize_t rc;
+
+	if (atomic_read(&hdev->in_reset)) {
+		dev_warn_ratelimited(hdev->dev,
+				"Can't change stop on error during reset\n");
+		return 0;
+	}
+
+	rc = kstrtouint_from_user(buf, count, 10, &value);
+	if (rc)
+		return rc;
+
+	hdev->stop_on_err = value ? 1 : 0;
+
+	hl_device_reset(hdev, false, false);
+
+	return count;
+}
+
 static const struct file_operations hl_data32b_fops = {
 	.owner = THIS_MODULE,
 	.read = hl_data_read32,
@@ -1015,6 +1107,18 @@ static const struct file_operations hl_device_fops = {
 	.write = hl_device_write
 };
 
+static const struct file_operations hl_clk_gate_fops = {
+	.owner = THIS_MODULE,
+	.read = hl_clk_gate_read,
+	.write = hl_clk_gate_write
+};
+
+static const struct file_operations hl_stop_on_err_fops = {
+	.owner = THIS_MODULE,
+	.read = hl_stop_on_err_read,
+	.write = hl_stop_on_err_write
+};
+
 static const struct hl_info_list hl_debugfs_list[] = {
 	{"command_buffers", command_buffers_show, NULL},
 	{"command_submission", command_submission_show, NULL},
@@ -1152,6 +1256,18 @@ void hl_debugfs_add_device(struct hl_device *hdev)
 				dev_entry,
 				&hl_device_fops);
 
+	debugfs_create_file("clk_gate",
+				0200,
+				dev_entry->root,
+				dev_entry,
+				&hl_clk_gate_fops);
+
+	debugfs_create_file("stop_on_err",
+				0644,
+				dev_entry->root,
+				dev_entry,
+				&hl_stop_on_err_fops);
+
 	for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {
 
 		ent = debugfs_create_file(hl_debugfs_list[i].name,
diff --git a/drivers/misc/habanalabs/device.c b/drivers/misc/habanalabs/device.c
index aef4de36b7aa..2b38a119704c 100644
--- a/drivers/misc/habanalabs/device.c
+++ b/drivers/misc/habanalabs/device.c
@@ -256,6 +256,10 @@ static int device_early_init(struct hl_device *hdev)
 		goya_set_asic_funcs(hdev);
 		strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
 		break;
+	case ASIC_GAUDI:
+		gaudi_set_asic_funcs(hdev);
+		sprintf(hdev->asic_name, "GAUDI");
+		break;
 	default:
 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 			hdev->asic_type);
@@ -603,6 +607,9 @@ int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
 
 		hdev->in_debug = 0;
 
+		if (!hdev->hard_reset_pending)
+			hdev->asic_funcs->enable_clock_gating(hdev);
+
 		goto out;
 	}
 
@@ -613,6 +620,7 @@ int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
 		goto out;
 	}
 
+	hdev->asic_funcs->disable_clock_gating(hdev);
 	hdev->in_debug = 1;
 
 out:
@@ -718,7 +726,7 @@ disable_device:
 	return rc;
 }
 
-static void device_kill_open_processes(struct hl_device *hdev)
+static int device_kill_open_processes(struct hl_device *hdev)
 {
 	u16 pending_total, pending_cnt;
 	struct hl_fpriv	*hpriv;
@@ -771,9 +779,7 @@ static void device_kill_open_processes(struct hl_device *hdev)
 		ssleep(1);
 	}
 
-	if (!list_empty(&hdev->fpriv_list))
-		dev_crit(hdev->dev,
-			"Going to hard reset with open user contexts\n");
+	return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;
 }
 
 static void device_hard_reset_pending(struct work_struct *work)
@@ -793,6 +799,7 @@ static void device_hard_reset_pending(struct work_struct *work)
  * @hdev: pointer to habanalabs device structure
  * @hard_reset: should we do hard reset to all engines or just reset the
  *              compute/dma engines
+ * @from_hard_reset_thread: is the caller the hard-reset thread
  *
  * Block future CS and wait for pending CS to be enqueued
  * Call ASIC H/W fini
@@ -815,6 +822,11 @@ int hl_device_reset(struct hl_device *hdev, bool hard_reset,
 		return 0;
 	}
 
+	if ((!hard_reset) && (!hdev->supports_soft_reset)) {
+		dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
+		hard_reset = true;
+	}
+
 	/*
 	 * Prevent concurrency in this function - only one reset should be
 	 * done at any given time. Only need to perform this if we didn't
@@ -894,7 +906,12 @@ again:
 		 * process can't really exit until all its CSs are done, which
 		 * is what we do in cs rollback
 		 */
-		device_kill_open_processes(hdev);
+		rc = device_kill_open_processes(hdev);
+		if (rc) {
+			dev_crit(hdev->dev,
+				"Failed to kill all open processes, stopping hard reset\n");
+			goto out_err;
+		}
 
 		/* Flush the Event queue workers to make sure no other thread is
 		 * reading or writing to registers during the reset
@@ -1062,7 +1079,7 @@ out_err:
  */
 int hl_device_init(struct hl_device *hdev, struct class *hclass)
 {
-	int i, rc, cq_ready_cnt;
+	int i, rc, cq_cnt, cq_ready_cnt;
 	char *name;
 	bool add_cdev_sysfs_on_err = false;
 
@@ -1120,14 +1137,16 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
 		goto sw_fini;
 	}
 
+	cq_cnt = hdev->asic_prop.completion_queues_count;
+
 	/*
 	 * Initialize the completion queues. Must be done before hw_init,
 	 * because there the addresses of the completion queues are being
 	 * passed as arguments to request_irq
 	 */
-	hdev->completion_queue =
-			kcalloc(hdev->asic_prop.completion_queues_count,
-				sizeof(*hdev->completion_queue), GFP_KERNEL);
+	hdev->completion_queue = kcalloc(cq_cnt,
+						sizeof(*hdev->completion_queue),
+						GFP_KERNEL);
 
 	if (!hdev->completion_queue) {
 		dev_err(hdev->dev, "failed to allocate completion queues\n");
@@ -1135,10 +1154,9 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
 		goto hw_queues_destroy;
 	}
 
-	for (i = 0, cq_ready_cnt = 0;
-			i < hdev->asic_prop.completion_queues_count;
-			i++, cq_ready_cnt++) {
-		rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
+	for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
+		rc = hl_cq_init(hdev, &hdev->completion_queue[i],
+				hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
 		if (rc) {
 			dev_err(hdev->dev,
 				"failed to initialize completion queue\n");
@@ -1325,11 +1343,12 @@ void hl_device_fini(struct hl_device *hdev)
 	 * This function is competing with the reset function, so try to
 	 * take the reset atomic and if we are already in middle of reset,
 	 * wait until reset function is finished. Reset function is designed
-	 * to always finish (could take up to a few seconds in worst case).
+	 * to always finish. However, in Gaudi, because of all the network
+	 * ports, the hard reset could take between 10-30 seconds
 	 */
 
 	timeout = ktime_add_us(ktime_get(),
-				HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4);
+				HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000);
 	rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
 	while (rc) {
 		usleep_range(50, 200);
@@ -1375,7 +1394,9 @@ void hl_device_fini(struct hl_device *hdev)
 	 * can't really exit until all its CSs are done, which is what we
 	 * do in cs rollback
 	 */
-	device_kill_open_processes(hdev);
+	rc = device_kill_open_processes(hdev);
+	if (rc)
+		dev_crit(hdev->dev, "Failed to kill all open processes\n");
 
 	hl_cb_pool_fini(hdev);
 
diff --git a/drivers/misc/habanalabs/firmware_if.c b/drivers/misc/habanalabs/firmware_if.c
index f5bd03171dac..baf790cf4b78 100644
--- a/drivers/misc/habanalabs/firmware_if.c
+++ b/drivers/misc/habanalabs/firmware_if.c
@@ -6,20 +6,22 @@
  */
 
 #include "habanalabs.h"
+#include "include/hl_boot_if.h"
 
 #include <linux/firmware.h>
 #include <linux/genalloc.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/slab.h>
 
 /**
- * hl_fw_push_fw_to_device() - Push FW code to device.
+ * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
  * @hdev: pointer to hl_device structure.
  *
  * Copy fw code from firmware file to device memory.
  *
  * Return: 0 on success, non-zero for failure.
  */
-int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
+int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
 				void __iomem *dst)
 {
 	const struct firmware *fw;
@@ -129,6 +131,68 @@ out:
 	return rc;
 }
 
+int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)
+{
+	struct armcp_packet pkt;
+	long result;
+	int rc;
+
+	memset(&pkt, 0, sizeof(pkt));
+
+	pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ <<
+				ARMCP_PKT_CTL_OPCODE_SHIFT);
+	pkt.value = cpu_to_le64(event_type);
+
+	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+			HL_DEVICE_TIMEOUT_USEC, &result);
+
+	if (rc)
+		dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
+
+	return rc;
+}
+
+int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
+		size_t irq_arr_size)
+{
+	struct armcp_unmask_irq_arr_packet *pkt;
+	size_t total_pkt_size;
+	long result;
+	int rc;
+
+	total_pkt_size = sizeof(struct armcp_unmask_irq_arr_packet) +
+			irq_arr_size;
+
+	/* data should be aligned to 8 bytes in order to ArmCP to copy it */
+	total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
+
+	/* total_pkt_size is casted to u16 later on */
+	if (total_pkt_size > USHRT_MAX) {
+		dev_err(hdev->dev, "too many elements in IRQ array\n");
+		return -EINVAL;
+	}
+
+	pkt = kzalloc(total_pkt_size, GFP_KERNEL);
+	if (!pkt)
+		return -ENOMEM;
+
+	pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));
+	memcpy(&pkt->irqs, irq_arr, irq_arr_size);
+
+	pkt->armcp_pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
+						ARMCP_PKT_CTL_OPCODE_SHIFT);
+
+	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
+			total_pkt_size, HL_DEVICE_TIMEOUT_USEC, &result);
+
+	if (rc)
+		dev_err(hdev->dev, "failed to unmask IRQ array\n");
+
+	kfree(pkt);
+
+	return rc;
+}
+
 int hl_fw_test_cpu_queue(struct hl_device *hdev)
 {
 	struct armcp_packet test_pkt = {};
@@ -286,3 +350,232 @@ out:
 
 	return rc;
 }
+
+static void fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg)
+{
+	u32 err_val;
+
+	/* Some of the firmware status codes are deprecated in newer f/w
+	 * versions. In those versions, the errors are reported
+	 * in different registers. Therefore, we need to check those
+	 * registers and print the exact errors. Moreover, there
+	 * may be multiple errors, so we need to report on each error
+	 * separately. Some of the error codes might indicate a state
+	 * that is not an error per-se, but it is an error in production
+	 * environment
+	 */
+	err_val = RREG32(boot_err0_reg);
+	if (!(err_val & CPU_BOOT_ERR0_ENABLED))
+		return;
+
+	if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL)
+		dev_err(hdev->dev,
+			"Device boot error - DRAM initialization failed\n");
+	if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED)
+		dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
+	if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL)
+		dev_err(hdev->dev,
+			"Device boot error - Thermal Sensor initialization failed\n");
+	if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED)
+		dev_warn(hdev->dev,
+			"Device boot warning - Skipped DRAM initialization\n");
+	if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED)
+		dev_warn(hdev->dev,
+			"Device boot error - Skipped waiting for BMC\n");
+	if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY)
+		dev_err(hdev->dev,
+			"Device boot error - Serdes data from BMC not available\n");
+	if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL)
+		dev_err(hdev->dev,
+			"Device boot error - NIC F/W initialization failed\n");
+}
+
+int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
+			u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
+			u32 boot_err0_reg, bool skip_bmc,
+			u32 cpu_timeout, u32 boot_fit_timeout)
+{
+	u32 status;
+	int rc;
+
+	dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
+		cpu_timeout / USEC_PER_SEC);
+
+	/* Wait for boot FIT request */
+	rc = hl_poll_timeout(
+		hdev,
+		cpu_boot_status_reg,
+		status,
+		status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
+		10000,
+		boot_fit_timeout);
+
+	if (rc) {
+		dev_dbg(hdev->dev,
+			"No boot fit request received, resuming boot\n");
+	} else {
+		rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);
+		if (rc)
+			goto out;
+
+		/* Clear device CPU message status */
+		WREG32(cpu_msg_status_reg, CPU_MSG_CLR);
+
+		/* Signal device CPU that boot loader is ready */
+		WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
+
+		/* Poll for CPU device ack */
+		rc = hl_poll_timeout(
+			hdev,
+			cpu_msg_status_reg,
+			status,
+			status == CPU_MSG_OK,
+			10000,
+			boot_fit_timeout);
+
+		if (rc) {
+			dev_err(hdev->dev,
+				"Timeout waiting for boot fit load ack\n");
+			goto out;
+		}
+
+		/* Clear message */
+		WREG32(msg_to_cpu_reg, KMD_MSG_NA);
+	}
+
+	/* Make sure CPU boot-loader is running */
+	rc = hl_poll_timeout(
+		hdev,
+		cpu_boot_status_reg,
+		status,
+		(status == CPU_BOOT_STATUS_DRAM_RDY) ||
+		(status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
+		(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
+		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
+		10000,
+		cpu_timeout);
+
+	/* Read U-Boot, preboot versions now in case we will later fail */
+	hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT);
+	hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT);
+
+	/* Some of the status codes below are deprecated in newer f/w
+	 * versions but we keep them here for backward compatibility
+	 */
+	if (rc) {
+		switch (status) {
+		case CPU_BOOT_STATUS_NA:
+			dev_err(hdev->dev,
+				"Device boot error - BTL did NOT run\n");
+			break;
+		case CPU_BOOT_STATUS_IN_WFE:
+			dev_err(hdev->dev,
+				"Device boot error - Stuck inside WFE loop\n");
+			break;
+		case CPU_BOOT_STATUS_IN_BTL:
+			dev_err(hdev->dev,
+				"Device boot error - Stuck in BTL\n");
+			break;
+		case CPU_BOOT_STATUS_IN_PREBOOT:
+			dev_err(hdev->dev,
+				"Device boot error - Stuck in Preboot\n");
+			break;
+		case CPU_BOOT_STATUS_IN_SPL:
+			dev_err(hdev->dev,
+				"Device boot error - Stuck in SPL\n");
+			break;
+		case CPU_BOOT_STATUS_IN_UBOOT:
+			dev_err(hdev->dev,
+				"Device boot error - Stuck in u-boot\n");
+			break;
+		case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
+			dev_err(hdev->dev,
+				"Device boot error - DRAM initialization failed\n");
+			break;
+		case CPU_BOOT_STATUS_UBOOT_NOT_READY:
+			dev_err(hdev->dev,
+				"Device boot error - u-boot stopped by user\n");
+			break;
+		case CPU_BOOT_STATUS_TS_INIT_FAIL:
+			dev_err(hdev->dev,
+				"Device boot error - Thermal Sensor initialization failed\n");
+			break;
+		default:
+			dev_err(hdev->dev,
+				"Device boot error - Invalid status code %d\n",
+				status);
+			break;
+		}
+
+		rc = -EIO;
+		goto out;
+	}
+
+	if (!hdev->fw_loading) {
+		dev_info(hdev->dev, "Skip loading FW\n");
+		goto out;
+	}
+
+	if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
+		goto out;
+
+	dev_info(hdev->dev,
+		"Loading firmware to device, may take some time...\n");
+
+	rc = hdev->asic_funcs->load_firmware_to_device(hdev);
+	if (rc)
+		goto out;
+
+	if (skip_bmc) {
+		WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
+
+		rc = hl_poll_timeout(
+			hdev,
+			cpu_boot_status_reg,
+			status,
+			(status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
+			10000,
+			cpu_timeout);
+
+		if (rc) {
+			dev_err(hdev->dev,
+				"Failed to get ACK on skipping BMC, %d\n",
+				status);
+			WREG32(msg_to_cpu_reg, KMD_MSG_NA);
+			rc = -EIO;
+			goto out;
+		}
+	}
+
+	WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);
+
+	rc = hl_poll_timeout(
+		hdev,
+		cpu_boot_status_reg,
+		status,
+		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
+		10000,
+		cpu_timeout);
+
+	/* Clear message */
+	WREG32(msg_to_cpu_reg, KMD_MSG_NA);
+
+	if (rc) {
+		if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
+			dev_err(hdev->dev,
+				"Device reports FIT image is corrupted\n");
+		else
+			dev_err(hdev->dev,
+				"Device failed to load, %d\n", status);
+
+		rc = -EIO;
+		goto out;
+	}
+
+	dev_info(hdev->dev, "Successfully loaded firmware to device\n");
+
+out:
+	fw_read_errors(hdev, boot_err0_reg);
+
+	return rc;
+}
diff --git a/drivers/misc/habanalabs/gaudi/Makefile b/drivers/misc/habanalabs/gaudi/Makefile
new file mode 100644
index 000000000000..f802cdc980ca
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+subdir-ccflags-y += -I$(src)
+
+HL_GAUDI_FILES := gaudi/gaudi.o gaudi/gaudi_hwmgr.o gaudi/gaudi_security.o \
+	gaudi/gaudi_coresight.o
diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
new file mode 100644
index 000000000000..61f88e9884ce
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -0,0 +1,6748 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "gaudiP.h"
+#include "include/hw_ip/mmu/mmu_general.h"
+#include "include/hw_ip/mmu/mmu_v1_1.h"
+#include "include/gaudi/gaudi_masks.h"
+#include "include/gaudi/gaudi_fw_if.h"
+#include "include/gaudi/gaudi_reg_map.h"
+#include "include/gaudi/gaudi_async_ids_map_extended.h"
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/firmware.h>
+#include <linux/hwmon.h>
+#include <linux/genalloc.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/iommu.h>
+#include <linux/seq_file.h>
+
+/*
+ * Gaudi security scheme:
+ *
+ * 1. Host is protected by:
+ *        - Range registers
+ *        - MMU
+ *
+ * 2. DDR is protected by:
+ *        - Range registers (protect the first 512MB)
+ *
+ * 3. Configuration is protected by:
+ *        - Range registers
+ *        - Protection bits
+ *
+ * MMU is always enabled.
+ *
+ * QMAN DMA channels 0,1,5 (PCI DMAN):
+ *     - DMA is not secured.
+ *     - PQ and CQ are secured.
+ *     - CP is secured: The driver needs to parse CB but WREG should be allowed
+ *                      because of TDMA (tensor DMA). Hence, WREG is always not
+ *                      secured.
+ *
+ * When the driver needs to use DMA it will check that Gaudi is idle, set DMA
+ * channel 0 to be secured, execute the DMA and change it back to not secured.
+ * Currently, the driver doesn't use the DMA while there are compute jobs
+ * running.
+ *
+ * The current use cases for the driver to use the DMA are:
+ *     - Clear SRAM on context switch (happens on context switch when device is
+ *       idle)
+ *     - MMU page tables area clear (happens on init)
+ *
+ * QMAN DMA 2-4,6,7, TPC, MME, NIC:
+ * PQ is secured and is located on the Host (HBM CON TPC3 bug)
+ * CQ, CP and the engine are not secured
+ *
+ */
+
+#define GAUDI_BOOT_FIT_FILE	"habanalabs/gaudi/gaudi-boot-fit.itb"
+#define GAUDI_LINUX_FW_FILE	"habanalabs/gaudi/gaudi-fit.itb"
+#define GAUDI_TPC_FW_FILE	"habanalabs/gaudi/gaudi_tpc.bin"
+
+#define GAUDI_DMA_POOL_BLK_SIZE		0x100 /* 256 bytes */
+
+#define GAUDI_RESET_TIMEOUT_MSEC	1000		/* 1000ms */
+#define GAUDI_RESET_WAIT_MSEC		1		/* 1ms */
+#define GAUDI_CPU_RESET_WAIT_MSEC	200		/* 200ms */
+#define GAUDI_TEST_QUEUE_WAIT_USEC	100000		/* 100ms */
+
+#define GAUDI_PLDM_RESET_WAIT_MSEC	1000		/* 1s */
+#define GAUDI_PLDM_HRESET_TIMEOUT_MSEC	20000		/* 20s */
+#define GAUDI_PLDM_SRESET_TIMEOUT_MSEC	14000		/* 14s */
+#define GAUDI_PLDM_TEST_QUEUE_WAIT_USEC	1000000		/* 1s */
+#define GAUDI_PLDM_MMU_TIMEOUT_USEC	(MMU_CONFIG_TIMEOUT_USEC * 100)
+#define GAUDI_PLDM_QMAN0_TIMEOUT_USEC	(HL_DEVICE_TIMEOUT_USEC * 30)
+#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC	(HL_DEVICE_TIMEOUT_USEC * 30)
+#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC	1000000		/* 1s */
+
+#define GAUDI_QMAN0_FENCE_VAL		0x72E91AB9
+
+#define GAUDI_MAX_STRING_LEN		20
+
+#define GAUDI_CB_POOL_CB_CNT		512
+#define GAUDI_CB_POOL_CB_SIZE		0x20000 /* 128KB */
+
+#define GAUDI_ALLOC_CPU_MEM_RETRY_CNT	3
+
+#define GAUDI_NUM_OF_TPC_INTR_CAUSE	20
+
+#define GAUDI_NUM_OF_QM_ERR_CAUSE	16
+
+#define GAUDI_NUM_OF_QM_ARB_ERR_CAUSE	3
+
+#define GAUDI_ARB_WDT_TIMEOUT		0x400000
+
+static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = {
+		"gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3",
+		"gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3",
+		"gaudi cq 5_0", "gaudi cq 5_1", "gaudi cq 5_2", "gaudi cq 5_3",
+		"gaudi cpu eq"
+};
+
+static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX] = {
+	[GAUDI_PCI_DMA_1] = 0,
+	[GAUDI_PCI_DMA_2] = 1,
+	[GAUDI_PCI_DMA_3] = 5,
+	[GAUDI_HBM_DMA_1] = 2,
+	[GAUDI_HBM_DMA_2] = 3,
+	[GAUDI_HBM_DMA_3] = 4,
+	[GAUDI_HBM_DMA_4] = 6,
+	[GAUDI_HBM_DMA_5] = 7
+};
+
+static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = {
+	[0] = GAUDI_QUEUE_ID_DMA_0_0,
+	[1] = GAUDI_QUEUE_ID_DMA_0_1,
+	[2] = GAUDI_QUEUE_ID_DMA_0_2,
+	[3] = GAUDI_QUEUE_ID_DMA_0_3,
+	[4] = GAUDI_QUEUE_ID_DMA_1_0,
+	[5] = GAUDI_QUEUE_ID_DMA_1_1,
+	[6] = GAUDI_QUEUE_ID_DMA_1_2,
+	[7] = GAUDI_QUEUE_ID_DMA_1_3,
+	[8] = GAUDI_QUEUE_ID_DMA_5_0,
+	[9] = GAUDI_QUEUE_ID_DMA_5_1,
+	[10] = GAUDI_QUEUE_ID_DMA_5_2,
+	[11] = GAUDI_QUEUE_ID_DMA_5_3
+};
+
+static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = {
+	[PACKET_WREG_32]	= sizeof(struct packet_wreg32),
+	[PACKET_WREG_BULK]	= sizeof(struct packet_wreg_bulk),
+	[PACKET_MSG_LONG]	= sizeof(struct packet_msg_long),
+	[PACKET_MSG_SHORT]	= sizeof(struct packet_msg_short),
+	[PACKET_CP_DMA]		= sizeof(struct packet_cp_dma),
+	[PACKET_REPEAT]		= sizeof(struct packet_repeat),
+	[PACKET_MSG_PROT]	= sizeof(struct packet_msg_prot),
+	[PACKET_FENCE]		= sizeof(struct packet_fence),
+	[PACKET_LIN_DMA]	= sizeof(struct packet_lin_dma),
+	[PACKET_NOP]		= sizeof(struct packet_nop),
+	[PACKET_STOP]		= sizeof(struct packet_stop),
+	[PACKET_ARB_POINT]	= sizeof(struct packet_arb_point),
+	[PACKET_WAIT]		= sizeof(struct packet_wait),
+	[PACKET_LOAD_AND_EXE]	= sizeof(struct packet_load_and_exe)
+};
+
+static const char * const
+gaudi_tpc_interrupts_cause[GAUDI_NUM_OF_TPC_INTR_CAUSE] = {
+	"tpc_address_exceed_slm",
+	"tpc_div_by_0",
+	"tpc_spu_mac_overflow",
+	"tpc_spu_addsub_overflow",
+	"tpc_spu_abs_overflow",
+	"tpc_spu_fp_dst_nan_inf",
+	"tpc_spu_fp_dst_denorm",
+	"tpc_vpu_mac_overflow",
+	"tpc_vpu_addsub_overflow",
+	"tpc_vpu_abs_overflow",
+	"tpc_vpu_fp_dst_nan_inf",
+	"tpc_vpu_fp_dst_denorm",
+	"tpc_assertions",
+	"tpc_illegal_instruction",
+	"tpc_pc_wrap_around",
+	"tpc_qm_sw_err",
+	"tpc_hbw_rresp_err",
+	"tpc_hbw_bresp_err",
+	"tpc_lbw_rresp_err",
+	"tpc_lbw_bresp_err"
+};
+
+static const char * const
+gaudi_qman_error_cause[GAUDI_NUM_OF_QM_ERR_CAUSE] = {
+	"PQ AXI HBW error",
+	"CQ AXI HBW error",
+	"CP AXI HBW error",
+	"CP error due to undefined OPCODE",
+	"CP encountered STOP OPCODE",
+	"CP AXI LBW error",
+	"CP WRREG32 or WRBULK returned error",
+	"N/A",
+	"FENCE 0 inc over max value and clipped",
+	"FENCE 1 inc over max value and clipped",
+	"FENCE 2 inc over max value and clipped",
+	"FENCE 3 inc over max value and clipped",
+	"FENCE 0 dec under min value and clipped",
+	"FENCE 1 dec under min value and clipped",
+	"FENCE 2 dec under min value and clipped",
+	"FENCE 3 dec under min value and clipped"
+};
+
+static const char * const
+gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = {
+	"Choice push while full error",
+	"Choice Q watchdog error",
+	"MSG AXI LBW returned with error"
+};
+
+static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = {
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_0 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_1 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_2 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_3 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_0 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_1 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_2 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_3 */
+	QUEUE_TYPE_CPU, /* GAUDI_QUEUE_ID_CPU_PQ */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_3 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_0 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_1 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_2 */
+	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_3 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_0 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_1 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_2 */
+	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_0_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_0_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_0_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_0_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_1_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_1_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_1_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_1_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_2_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_2_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_2_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_2_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_3_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_3_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_3_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_3_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_4_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_4_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_4_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_4_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_5_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_5_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_5_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_5_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_6_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_6_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_6_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_6_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_7_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_7_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_7_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_7_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_8_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_8_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_8_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_8_3 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_9_0 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_9_1 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_9_2 */
+	QUEUE_TYPE_NA,  /* GAUDI_QUEUE_ID_NIC_9_3 */
+};
+
+static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
+								u64 phys_addr);
+static int gaudi_send_job_on_qman0(struct hl_device *hdev,
+					struct hl_cs_job *job);
+static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
+					u32 size, u64 val);
+static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
+				u32 tpc_id);
+static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);
+static int gaudi_armcp_info_get(struct hl_device *hdev);
+static void gaudi_disable_clock_gating(struct hl_device *hdev);
+static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid);
+
+static int gaudi_get_fixed_properties(struct hl_device *hdev)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	int i;
+
+	if (GAUDI_QUEUE_ID_SIZE >= HL_MAX_QUEUES) {
+		dev_err(hdev->dev,
+			"Number of H/W queues must be smaller than %d\n",
+			HL_MAX_QUEUES);
+		return -EFAULT;
+	}
+
+	for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
+		if (gaudi_queue_type[i] == QUEUE_TYPE_EXT) {
+			prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
+			prop->hw_queues_props[i].driver_only = 0;
+			prop->hw_queues_props[i].requires_kernel_cb = 1;
+		} else if (gaudi_queue_type[i] == QUEUE_TYPE_CPU) {
+			prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
+			prop->hw_queues_props[i].driver_only = 1;
+			prop->hw_queues_props[i].requires_kernel_cb = 0;
+		} else if (gaudi_queue_type[i] == QUEUE_TYPE_INT) {
+			prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
+			prop->hw_queues_props[i].driver_only = 0;
+			prop->hw_queues_props[i].requires_kernel_cb = 0;
+		} else if (gaudi_queue_type[i] == QUEUE_TYPE_NA) {
+			prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
+			prop->hw_queues_props[i].driver_only = 0;
+			prop->hw_queues_props[i].requires_kernel_cb = 0;
+		}
+	}
+
+	for (; i < HL_MAX_QUEUES; i++)
+		prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
+
+	prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
+
+	prop->dram_base_address = DRAM_PHYS_BASE;
+	prop->dram_size = GAUDI_HBM_SIZE_32GB;
+	prop->dram_end_address = prop->dram_base_address +
+					prop->dram_size;
+	prop->dram_user_base_address = DRAM_BASE_ADDR_USER;
+
+	prop->sram_base_address = SRAM_BASE_ADDR;
+	prop->sram_size = SRAM_SIZE;
+	prop->sram_end_address = prop->sram_base_address +
+					prop->sram_size;
+	prop->sram_user_base_address = prop->sram_base_address +
+					SRAM_USER_BASE_OFFSET;
+
+	prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;
+	if (hdev->pldm)
+		prop->mmu_pgt_size = 0x800000; /* 8MB */
+	else
+		prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;
+	prop->mmu_pte_size = HL_PTE_SIZE;
+	prop->mmu_hop_table_size = HOP_TABLE_SIZE;
+	prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
+	prop->dram_page_size = PAGE_SIZE_2MB;
+
+	prop->pmmu.hop0_shift = HOP0_SHIFT;
+	prop->pmmu.hop1_shift = HOP1_SHIFT;
+	prop->pmmu.hop2_shift = HOP2_SHIFT;
+	prop->pmmu.hop3_shift = HOP3_SHIFT;
+	prop->pmmu.hop4_shift = HOP4_SHIFT;
+	prop->pmmu.hop0_mask = HOP0_MASK;
+	prop->pmmu.hop1_mask = HOP1_MASK;
+	prop->pmmu.hop2_mask = HOP2_MASK;
+	prop->pmmu.hop3_mask = HOP3_MASK;
+	prop->pmmu.hop4_mask = HOP4_MASK;
+	prop->pmmu.start_addr = VA_HOST_SPACE_START;
+	prop->pmmu.end_addr =
+			(VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2) - 1;
+	prop->pmmu.page_size = PAGE_SIZE_4KB;
+
+	/* PMMU and HPMMU are the same except of page size */
+	memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
+	prop->pmmu_huge.page_size = PAGE_SIZE_2MB;
+
+	/* shifts and masks are the same in PMMU and DMMU */
+	memcpy(&prop->dmmu, &prop->pmmu, sizeof(prop->pmmu));
+	prop->dmmu.start_addr = (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2);
+	prop->dmmu.end_addr = VA_HOST_SPACE_END;
+	prop->dmmu.page_size = PAGE_SIZE_2MB;
+
+	prop->cfg_size = CFG_SIZE;
+	prop->max_asid = MAX_ASID;
+	prop->num_of_events = GAUDI_EVENT_SIZE;
+	prop->tpc_enabled_mask = TPC_ENABLED_MASK;
+
+	prop->max_power_default = MAX_POWER_DEFAULT;
+
+	prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT;
+	prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE;
+
+	prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
+	prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
+
+	strncpy(prop->armcp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
+					CARD_NAME_MAX_LEN);
+
+	return 0;
+}
+
+static int gaudi_pci_bars_map(struct hl_device *hdev)
+{
+	static const char * const name[] = {"SRAM", "CFG", "HBM"};
+	bool is_wc[3] = {false, false, true};
+	int rc;
+
+	rc = hl_pci_bars_map(hdev, name, is_wc);
+	if (rc)
+		return rc;
+
+	hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] +
+			(CFG_BASE - SPI_FLASH_BASE_ADDR);
+
+	return 0;
+}
+
+static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 old_addr = addr;
+	int rc;
+
+	if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr))
+		return old_addr;
+
+	/* Inbound Region 2 - Bar 4 - Point to HBM */
+	rc = hl_pci_set_dram_bar_base(hdev, 2, 4, addr);
+	if (rc)
+		return U64_MAX;
+
+	if (gaudi) {
+		old_addr = gaudi->hbm_bar_cur_addr;
+		gaudi->hbm_bar_cur_addr = addr;
+	}
+
+	return old_addr;
+}
+
+static int gaudi_init_iatu(struct hl_device *hdev)
+{
+	int rc = 0;
+
+	/* Inbound Region 1 - Bar 2 - Point to SPI FLASH */
+	rc  = hl_pci_iatu_write(hdev, 0x314,
+				lower_32_bits(SPI_FLASH_BASE_ADDR));
+	rc |= hl_pci_iatu_write(hdev, 0x318,
+				upper_32_bits(SPI_FLASH_BASE_ADDR));
+	rc |= hl_pci_iatu_write(hdev, 0x300, 0);
+	/* Enable + Bar match + match enable */
+	rc |= hl_pci_iatu_write(hdev, 0x304, 0xC0080200);
+
+	if (rc)
+		return -EIO;
+
+	return hl_pci_init_iatu(hdev, SRAM_BASE_ADDR, DRAM_PHYS_BASE,
+				HOST_PHYS_BASE, HOST_PHYS_SIZE);
+}
+
+static int gaudi_early_init(struct hl_device *hdev)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct pci_dev *pdev = hdev->pdev;
+	int rc;
+
+	rc = gaudi_get_fixed_properties(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to get fixed properties\n");
+		return rc;
+	}
+
+	/* Check BAR sizes */
+	if (pci_resource_len(pdev, SRAM_BAR_ID) != SRAM_BAR_SIZE) {
+		dev_err(hdev->dev,
+			"Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
+			SRAM_BAR_ID,
+			(unsigned long long) pci_resource_len(pdev,
+							SRAM_BAR_ID),
+			SRAM_BAR_SIZE);
+		return -ENODEV;
+	}
+
+	if (pci_resource_len(pdev, CFG_BAR_ID) != CFG_BAR_SIZE) {
+		dev_err(hdev->dev,
+			"Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
+			CFG_BAR_ID,
+			(unsigned long long) pci_resource_len(pdev,
+								CFG_BAR_ID),
+			CFG_BAR_SIZE);
+		return -ENODEV;
+	}
+
+	prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID);
+
+	rc = hl_pci_init(hdev);
+	if (rc)
+		return rc;
+
+	return 0;
+}
+
+static int gaudi_early_fini(struct hl_device *hdev)
+{
+	hl_pci_fini(hdev);
+
+	return 0;
+}
+
+/**
+ * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values
+ *
+ * @hdev: pointer to hl_device structure
+ *
+ */
+static void gaudi_fetch_psoc_frequency(struct hl_device *hdev)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+
+	prop->psoc_pci_pll_nr = RREG32(mmPSOC_PCI_PLL_NR);
+	prop->psoc_pci_pll_nf = RREG32(mmPSOC_PCI_PLL_NF);
+	prop->psoc_pci_pll_od = RREG32(mmPSOC_PCI_PLL_OD);
+	prop->psoc_pci_pll_div_factor = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
+}
+
+static int _gaudi_init_tpc_mem(struct hl_device *hdev,
+		dma_addr_t tpc_kernel_src_addr, u32 tpc_kernel_size)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct packet_lin_dma *init_tpc_mem_pkt;
+	struct hl_cs_job *job;
+	struct hl_cb *cb;
+	u64 dst_addr;
+	u32 cb_size, ctl;
+	u8 tpc_id;
+	int rc;
+
+	cb = hl_cb_kernel_create(hdev, PAGE_SIZE);
+	if (!cb)
+		return -EFAULT;
+
+	init_tpc_mem_pkt = (struct packet_lin_dma *) (uintptr_t)
+					cb->kernel_address;
+	cb_size = sizeof(*init_tpc_mem_pkt);
+	memset(init_tpc_mem_pkt, 0, cb_size);
+
+	init_tpc_mem_pkt->tsize = cpu_to_le32(tpc_kernel_size);
+
+	ctl = ((PACKET_LIN_DMA << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+			(1 << GAUDI_PKT_LIN_DMA_CTL_LIN_SHIFT) |
+			(1 << GAUDI_PKT_CTL_RB_SHIFT) |
+			(1 << GAUDI_PKT_CTL_MB_SHIFT));
+
+	init_tpc_mem_pkt->ctl = cpu_to_le32(ctl);
+
+	init_tpc_mem_pkt->src_addr = cpu_to_le64(tpc_kernel_src_addr);
+	dst_addr = (prop->sram_user_base_address &
+			GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>
+			GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;
+	init_tpc_mem_pkt->dst_addr |= cpu_to_le64(dst_addr);
+
+	job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
+	if (!job) {
+		dev_err(hdev->dev, "Failed to allocate a new job\n");
+		rc = -ENOMEM;
+		goto release_cb;
+	}
+
+	job->id = 0;
+	job->user_cb = cb;
+	job->user_cb->cs_cnt++;
+	job->user_cb_size = cb_size;
+	job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;
+	job->patched_cb = job->user_cb;
+	job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot);
+
+	hl_debugfs_add_job(hdev, job);
+
+	rc = gaudi_send_job_on_qman0(hdev, job);
+
+	if (rc)
+		goto free_job;
+
+	for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {
+		rc = gaudi_run_tpc_kernel(hdev, dst_addr, tpc_id);
+		if (rc)
+			break;
+	}
+
+free_job:
+	hl_userptr_delete_list(hdev, &job->userptr_list);
+	hl_debugfs_remove_job(hdev, job);
+	kfree(job);
+	cb->cs_cnt--;
+
+release_cb:
+	hl_cb_put(cb);
+	hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
+
+	return rc;
+}
+
+/*
+ * gaudi_init_tpc_mem() - Initialize TPC memories.
+ * @hdev: Pointer to hl_device structure.
+ *
+ * Copy TPC kernel fw from firmware file and run it to initialize TPC memories.
+ *
+ * Return: 0 for success, negative value for error.
+ */
+static int gaudi_init_tpc_mem(struct hl_device *hdev)
+{
+	const struct firmware *fw;
+	size_t fw_size;
+	void *cpu_addr;
+	dma_addr_t dma_handle;
+	int rc;
+
+	rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev);
+	if (rc) {
+		dev_err(hdev->dev, "Firmware file %s is not found!\n",
+				GAUDI_TPC_FW_FILE);
+		goto out;
+	}
+
+	fw_size = fw->size;
+	cpu_addr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, fw_size,
+			&dma_handle, GFP_KERNEL | __GFP_ZERO);
+	if (!cpu_addr) {
+		dev_err(hdev->dev,
+			"Failed to allocate %zu of dma memory for TPC kernel\n",
+			fw_size);
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	memcpy(cpu_addr, fw->data, fw_size);
+
+	rc = _gaudi_init_tpc_mem(hdev, dma_handle, fw_size);
+
+	hdev->asic_funcs->asic_dma_free_coherent(hdev, fw->size, cpu_addr,
+			dma_handle);
+
+out:
+	release_firmware(fw);
+	return rc;
+}
+
+static int gaudi_late_init(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	int rc;
+
+	rc = gaudi->armcp_info_get(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to get armcp info\n");
+		return rc;
+	}
+
+	rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to enable PCI access from CPU\n");
+		return rc;
+	}
+
+	WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER);
+
+	gaudi_fetch_psoc_frequency(hdev);
+
+	rc = gaudi_mmu_clear_pgt_range(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to clear MMU page tables range\n");
+		goto disable_pci_access;
+	}
+
+	rc = gaudi_init_tpc_mem(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to initialize TPC memories\n");
+		goto disable_pci_access;
+	}
+
+	return 0;
+
+disable_pci_access:
+	hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
+
+	return rc;
+}
+
+static void gaudi_late_fini(struct hl_device *hdev)
+{
+	const struct hwmon_channel_info **channel_info_arr;
+	int i = 0;
+
+	if (!hdev->hl_chip_info->info)
+		return;
+
+	channel_info_arr = hdev->hl_chip_info->info;
+
+	while (channel_info_arr[i]) {
+		kfree(channel_info_arr[i]->config);
+		kfree(channel_info_arr[i]);
+		i++;
+	}
+
+	kfree(channel_info_arr);
+
+	hdev->hl_chip_info->info = NULL;
+}
+
+static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
+{
+	dma_addr_t dma_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr;
+	void *virt_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {};
+	int i, j, rc = 0;
+
+	/*
+	 * The device CPU works with 40-bits addresses, while bit 39 must be set
+	 * to '1' when accessing the host.
+	 * Bits 49:39 of the full host address are saved for a later
+	 * configuration of the HW to perform extension to 50 bits.
+	 * Because there is a single HW register that holds the extension bits,
+	 * these bits must be identical in all allocated range.
+	 */
+
+	for (i = 0 ; i < GAUDI_ALLOC_CPU_MEM_RETRY_CNT ; i++) {
+		virt_addr_arr[i] =
+			hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
+						HL_CPU_ACCESSIBLE_MEM_SIZE,
+						&dma_addr_arr[i],
+						GFP_KERNEL | __GFP_ZERO);
+		if (!virt_addr_arr[i]) {
+			rc = -ENOMEM;
+			goto free_dma_mem_arr;
+		}
+
+		end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;
+		if (GAUDI_CPU_PCI_MSB_ADDR(dma_addr_arr[i]) ==
+				GAUDI_CPU_PCI_MSB_ADDR(end_addr))
+			break;
+	}
+
+	if (i == GAUDI_ALLOC_CPU_MEM_RETRY_CNT) {
+		dev_err(hdev->dev,
+			"MSB of CPU accessible DMA memory are not identical in all range\n");
+		rc = -EFAULT;
+		goto free_dma_mem_arr;
+	}
+
+	hdev->cpu_accessible_dma_mem = virt_addr_arr[i];
+	hdev->cpu_accessible_dma_address = dma_addr_arr[i];
+	hdev->cpu_pci_msb_addr =
+		GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address);
+
+	GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address);
+
+free_dma_mem_arr:
+	for (j = 0 ; j < i ; j++)
+		hdev->asic_funcs->asic_dma_free_coherent(hdev,
+						HL_CPU_ACCESSIBLE_MEM_SIZE,
+						virt_addr_arr[j],
+						dma_addr_arr[j]);
+
+	return rc;
+}
+
+static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct gaudi_internal_qman_info *q;
+	u32 i;
+
+	for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
+		q = &gaudi->internal_qmans[i];
+		if (!q->pq_kernel_addr)
+			continue;
+		hdev->asic_funcs->asic_dma_free_coherent(hdev, q->pq_size,
+							q->pq_kernel_addr,
+							q->pq_dma_addr);
+	}
+}
+
+static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct gaudi_internal_qman_info *q;
+	int rc, i;
+
+	for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {
+		if (gaudi_queue_type[i] != QUEUE_TYPE_INT)
+			continue;
+
+		q = &gaudi->internal_qmans[i];
+
+		switch (i) {
+		case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_4_3:
+		case GAUDI_QUEUE_ID_DMA_6_0 ... GAUDI_QUEUE_ID_DMA_7_3:
+			q->pq_size = HBM_DMA_QMAN_SIZE_IN_BYTES;
+			break;
+		case GAUDI_QUEUE_ID_MME_0_0 ... GAUDI_QUEUE_ID_MME_1_3:
+			q->pq_size = MME_QMAN_SIZE_IN_BYTES;
+			break;
+		case GAUDI_QUEUE_ID_TPC_0_0 ... GAUDI_QUEUE_ID_TPC_7_3:
+			q->pq_size = TPC_QMAN_SIZE_IN_BYTES;
+			break;
+		default:
+			dev_err(hdev->dev, "Bad internal queue index %d", i);
+			rc = -EINVAL;
+			goto free_internal_qmans_pq_mem;
+		}
+
+		q->pq_kernel_addr = hdev->asic_funcs->asic_dma_alloc_coherent(
+						hdev, q->pq_size,
+						&q->pq_dma_addr,
+						GFP_KERNEL | __GFP_ZERO);
+		if (!q->pq_kernel_addr) {
+			rc = -ENOMEM;
+			goto free_internal_qmans_pq_mem;
+		}
+	}
+
+	return 0;
+
+free_internal_qmans_pq_mem:
+	gaudi_free_internal_qmans_pq_mem(hdev);
+	return rc;
+}
+
+static int gaudi_sw_init(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi;
+	u32 i, event_id = 0;
+	int rc;
+
+	/* Allocate device structure */
+	gaudi = kzalloc(sizeof(*gaudi), GFP_KERNEL);
+	if (!gaudi)
+		return -ENOMEM;
+
+	for (i = 0 ; i < ARRAY_SIZE(gaudi_irq_map_table) ; i++) {
+		if (gaudi_irq_map_table[i].valid) {
+			if (event_id == GAUDI_EVENT_SIZE) {
+				dev_err(hdev->dev,
+					"Event array exceeds the limit of %u events\n",
+					GAUDI_EVENT_SIZE);
+				rc = -EINVAL;
+				goto free_gaudi_device;
+			}
+
+			gaudi->events[event_id++] =
+					gaudi_irq_map_table[i].fc_id;
+		}
+	}
+
+	gaudi->armcp_info_get = gaudi_armcp_info_get;
+
+	gaudi->max_freq_value = GAUDI_MAX_CLK_FREQ;
+
+	hdev->asic_specific = gaudi;
+
+	/* Create DMA pool for small allocations */
+	hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
+			&hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);
+	if (!hdev->dma_pool) {
+		dev_err(hdev->dev, "failed to create DMA pool\n");
+		rc = -ENOMEM;
+		goto free_gaudi_device;
+	}
+
+	rc = gaudi_alloc_cpu_accessible_dma_mem(hdev);
+	if (rc)
+		goto free_dma_pool;
+
+	hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
+	if (!hdev->cpu_accessible_dma_pool) {
+		dev_err(hdev->dev,
+			"Failed to create CPU accessible DMA pool\n");
+		rc = -ENOMEM;
+		goto free_cpu_dma_mem;
+	}
+
+	rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
+				(uintptr_t) hdev->cpu_accessible_dma_mem,
+				HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to add memory to CPU accessible DMA pool\n");
+		rc = -EFAULT;
+		goto free_cpu_accessible_dma_pool;
+	}
+
+	rc = gaudi_alloc_internal_qmans_pq_mem(hdev);
+	if (rc)
+		goto free_cpu_accessible_dma_pool;
+
+	spin_lock_init(&gaudi->hw_queues_lock);
+	mutex_init(&gaudi->clk_gate_mutex);
+
+	hdev->supports_sync_stream = true;
+	hdev->supports_coresight = true;
+
+	return 0;
+
+free_cpu_accessible_dma_pool:
+	gen_pool_destroy(hdev->cpu_accessible_dma_pool);
+free_cpu_dma_mem:
+	GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
+				hdev->cpu_pci_msb_addr);
+	hdev->asic_funcs->asic_dma_free_coherent(hdev,
+			HL_CPU_ACCESSIBLE_MEM_SIZE,
+			hdev->cpu_accessible_dma_mem,
+			hdev->cpu_accessible_dma_address);
+free_dma_pool:
+	dma_pool_destroy(hdev->dma_pool);
+free_gaudi_device:
+	kfree(gaudi);
+	return rc;
+}
+
+static int gaudi_sw_fini(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	gaudi_free_internal_qmans_pq_mem(hdev);
+
+	gen_pool_destroy(hdev->cpu_accessible_dma_pool);
+
+	GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
+					hdev->cpu_pci_msb_addr);
+	hdev->asic_funcs->asic_dma_free_coherent(hdev,
+			HL_CPU_ACCESSIBLE_MEM_SIZE,
+			hdev->cpu_accessible_dma_mem,
+			hdev->cpu_accessible_dma_address);
+
+	dma_pool_destroy(hdev->dma_pool);
+
+	mutex_destroy(&gaudi->clk_gate_mutex);
+
+	kfree(gaudi);
+
+	return 0;
+}
+
+static irqreturn_t gaudi_irq_handler_single(int irq, void *arg)
+{
+	struct hl_device *hdev = arg;
+	int i;
+
+	if (hdev->disabled)
+		return IRQ_HANDLED;
+
+	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
+		hl_irq_handler_cq(irq, &hdev->completion_queue[i]);
+
+	hl_irq_handler_eq(irq, &hdev->event_queue);
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * For backward compatibility, new MSI interrupts should be set after the
+ * existing CPU and NIC interrupts.
+ */
+static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr,
+				bool cpu_eq)
+{
+	int msi_vec;
+
+	if ((nr != GAUDI_EVENT_QUEUE_MSI_IDX) && (cpu_eq))
+		dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n",
+				GAUDI_EVENT_QUEUE_MSI_IDX);
+
+	msi_vec = ((nr < GAUDI_EVENT_QUEUE_MSI_IDX) || (cpu_eq)) ? nr :
+			(nr + NIC_NUMBER_OF_ENGINES + 1);
+
+	return pci_irq_vector(hdev->pdev, msi_vec);
+}
+
+static int gaudi_enable_msi_single(struct hl_device *hdev)
+{
+	int rc, irq;
+
+	dev_info(hdev->dev, "Working in single MSI IRQ mode\n");
+
+	irq = gaudi_pci_irq_vector(hdev, 0, false);
+	rc = request_irq(irq, gaudi_irq_handler_single, 0,
+			"gaudi single msi", hdev);
+	if (rc)
+		dev_err(hdev->dev,
+			"Failed to request single MSI IRQ\n");
+
+	return rc;
+}
+
+static int gaudi_enable_msi_multi(struct hl_device *hdev)
+{
+	int cq_cnt = hdev->asic_prop.completion_queues_count;
+	int rc, i, irq_cnt_init, irq;
+
+	for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
+		irq = gaudi_pci_irq_vector(hdev, i, false);
+		rc = request_irq(irq, hl_irq_handler_cq, 0, gaudi_irq_name[i],
+				&hdev->completion_queue[i]);
+		if (rc) {
+			dev_err(hdev->dev, "Failed to request IRQ %d", irq);
+			goto free_irqs;
+		}
+	}
+
+	irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX, true);
+	rc = request_irq(irq, hl_irq_handler_eq, 0, gaudi_irq_name[cq_cnt],
+				&hdev->event_queue);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to request IRQ %d", irq);
+		goto free_irqs;
+	}
+
+	return 0;
+
+free_irqs:
+	for (i = 0 ; i < irq_cnt_init ; i++)
+		free_irq(gaudi_pci_irq_vector(hdev, i, false),
+				&hdev->completion_queue[i]);
+	return rc;
+}
+
+static int gaudi_enable_msi(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	int rc;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_MSI)
+		return 0;
+
+	rc = pci_alloc_irq_vectors(hdev->pdev, 1, GAUDI_MSI_ENTRIES,
+					PCI_IRQ_MSI);
+	if (rc < 0) {
+		dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);
+		return rc;
+	}
+
+	if (rc < NUMBER_OF_INTERRUPTS) {
+		gaudi->multi_msi_mode = false;
+		rc = gaudi_enable_msi_single(hdev);
+	} else {
+		gaudi->multi_msi_mode = true;
+		rc = gaudi_enable_msi_multi(hdev);
+	}
+
+	if (rc)
+		goto free_pci_irq_vectors;
+
+	gaudi->hw_cap_initialized |= HW_CAP_MSI;
+
+	return 0;
+
+free_pci_irq_vectors:
+	pci_free_irq_vectors(hdev->pdev);
+	return rc;
+}
+
+static void gaudi_sync_irqs(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	int i, cq_cnt = hdev->asic_prop.completion_queues_count;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
+		return;
+
+	/* Wait for all pending IRQs to be finished */
+	if (gaudi->multi_msi_mode) {
+		for (i = 0 ; i < cq_cnt ; i++)
+			synchronize_irq(gaudi_pci_irq_vector(hdev, i, false));
+
+		synchronize_irq(gaudi_pci_irq_vector(hdev,
+						GAUDI_EVENT_QUEUE_MSI_IDX,
+						true));
+	} else {
+		synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));
+	}
+}
+
+static void gaudi_disable_msi(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	int i, irq, cq_cnt = hdev->asic_prop.completion_queues_count;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))
+		return;
+
+	gaudi_sync_irqs(hdev);
+
+	if (gaudi->multi_msi_mode) {
+		irq = gaudi_pci_irq_vector(hdev, GAUDI_EVENT_QUEUE_MSI_IDX,
+						true);
+		free_irq(irq, &hdev->event_queue);
+
+		for (i = 0 ; i < cq_cnt ; i++) {
+			irq = gaudi_pci_irq_vector(hdev, i, false);
+			free_irq(irq, &hdev->completion_queue[i]);
+		}
+	} else {
+		free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);
+	}
+
+	pci_free_irq_vectors(hdev->pdev);
+
+	gaudi->hw_cap_initialized &= ~HW_CAP_MSI;
+}
+
+static void gaudi_init_scrambler_sram(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER)
+		return;
+
+	if (!hdev->sram_scrambler_enable)
+		return;
+
+	WREG32(mmNIF_RTR_CTRL_0_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_1_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_2_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_3_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_4_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_5_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_6_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_7_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_0_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_1_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_2_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_3_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_4_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_5_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_6_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_7_SCRAM_SRAM_EN,
+			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_SRAM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);
+
+	gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER;
+}
+
+static void gaudi_init_scrambler_hbm(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER)
+		return;
+
+	if (!hdev->dram_scrambler_enable)
+		return;
+
+	WREG32(mmNIF_RTR_CTRL_0_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_1_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_2_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_3_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_4_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_5_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_6_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_7_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_0_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_1_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_2_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_3_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_4_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_5_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_6_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_7_SCRAM_HBM_EN,
+			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);
+
+	gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER;
+}
+
+static void gaudi_init_e2e(struct hl_device *hdev)
+{
+	WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3);
+	WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 785 >> 3);
+	WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 49);
+	WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 101);
+
+	WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);
+	WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);
+	WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);
+
+	WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);
+	WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);
+	WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);
+	WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);
+	WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);
+	WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);
+	WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);
+	WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);
+	WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);
+
+	WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 297 >> 3);
+	WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 908 >> 3);
+	WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 19);
+	WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 19);
+
+	WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 318 >> 3);
+	WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 956 >> 3);
+	WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 79);
+	WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 163);
+
+	WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);
+	WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);
+	WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);
+
+	WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);
+	WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);
+	WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);
+	WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);
+	WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);
+	WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);
+	WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);
+
+	WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);
+	WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);
+	WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);
+	WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);
+
+	WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 318 >> 3);
+	WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 956 >> 3);
+	WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 79);
+	WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 79);
+
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);
+
+	if (!hdev->dram_scrambler_enable) {
+		WREG32(mmSIF_RTR_CTRL_0_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_0_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_0_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmSIF_RTR_CTRL_1_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_1_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_1_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmSIF_RTR_CTRL_2_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_2_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_2_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmSIF_RTR_CTRL_3_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_3_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_3_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmSIF_RTR_CTRL_4_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_4_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_4_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmSIF_RTR_CTRL_5_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_5_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_5_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmSIF_RTR_CTRL_6_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_6_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_6_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmSIF_RTR_CTRL_7_NL_HBM_SEL_0, 0x21);
+		WREG32(mmSIF_RTR_CTRL_7_NL_HBM_SEL_1, 0x22);
+		WREG32(mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmSIF_RTR_CTRL_7_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_0_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_0_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_0_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_1_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_1_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_1_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_2_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_2_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_2_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_3_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_3_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_3_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_4_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_4_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_4_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_5_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_5_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_5_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_6_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_6_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_6_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmNIF_RTR_CTRL_7_NL_HBM_SEL_0, 0x21);
+		WREG32(mmNIF_RTR_CTRL_7_NL_HBM_SEL_1, 0x22);
+		WREG32(mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmNIF_RTR_CTRL_7_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_E_N_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_E_N_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_E_S_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_E_S_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_W_N_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_W_N_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_W_S_DOWN_CH0_NL_HBM_PC_SEL_3, 0x20);
+
+		WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_0, 0x21);
+		WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_1, 0x22);
+		WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_18, 0x1F);
+		WREG32(mmDMA_IF_W_S_DOWN_CH1_NL_HBM_PC_SEL_3, 0x20);
+	}
+
+	WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_EN,
+			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_EN,
+			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_EN,
+			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);
+}
+
+static void gaudi_init_hbm_cred(struct hl_device *hdev)
+{
+	uint32_t hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd;
+
+	hbm0_wr = 0x33333333;
+	hbm1_wr = 0x33333333;
+	hbm0_rd = 0x77777777;
+	hbm1_rd = 0xDDDDDDDD;
+
+	WREG32(mmDMA_IF_E_N_HBM0_WR_CRED_CNT, hbm0_wr);
+	WREG32(mmDMA_IF_E_N_HBM1_WR_CRED_CNT, hbm1_wr);
+	WREG32(mmDMA_IF_E_N_HBM0_RD_CRED_CNT, hbm0_rd);
+	WREG32(mmDMA_IF_E_N_HBM1_RD_CRED_CNT, hbm1_rd);
+
+	WREG32(mmDMA_IF_E_S_HBM0_WR_CRED_CNT, hbm0_wr);
+	WREG32(mmDMA_IF_E_S_HBM1_WR_CRED_CNT, hbm1_wr);
+	WREG32(mmDMA_IF_E_S_HBM0_RD_CRED_CNT, hbm0_rd);
+	WREG32(mmDMA_IF_E_S_HBM1_RD_CRED_CNT, hbm1_rd);
+
+	WREG32(mmDMA_IF_W_N_HBM0_WR_CRED_CNT, hbm0_wr);
+	WREG32(mmDMA_IF_W_N_HBM1_WR_CRED_CNT, hbm1_wr);
+	WREG32(mmDMA_IF_W_N_HBM0_RD_CRED_CNT, hbm0_rd);
+	WREG32(mmDMA_IF_W_N_HBM1_RD_CRED_CNT, hbm1_rd);
+
+	WREG32(mmDMA_IF_W_S_HBM0_WR_CRED_CNT, hbm0_wr);
+	WREG32(mmDMA_IF_W_S_HBM1_WR_CRED_CNT, hbm1_wr);
+	WREG32(mmDMA_IF_W_S_HBM0_RD_CRED_CNT, hbm0_rd);
+	WREG32(mmDMA_IF_W_S_HBM1_RD_CRED_CNT, hbm1_rd);
+
+	WREG32(mmDMA_IF_E_N_HBM_CRED_EN_0,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+	WREG32(mmDMA_IF_E_S_HBM_CRED_EN_0,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+	WREG32(mmDMA_IF_W_N_HBM_CRED_EN_0,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+	WREG32(mmDMA_IF_W_S_HBM_CRED_EN_0,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+
+	WREG32(mmDMA_IF_E_N_HBM_CRED_EN_1,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+	WREG32(mmDMA_IF_E_S_HBM_CRED_EN_1,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+	WREG32(mmDMA_IF_W_N_HBM_CRED_EN_1,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+	WREG32(mmDMA_IF_W_S_HBM_CRED_EN_1,
+			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |
+			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));
+}
+
+static void gaudi_init_rate_limiter(struct hl_device *hdev)
+{
+	u32 nr, nf, od, sat, rst, timeout;
+	u64 freq;
+
+	nr = RREG32(mmPSOC_HBM_PLL_NR);
+	nf = RREG32(mmPSOC_HBM_PLL_NF);
+	od = RREG32(mmPSOC_HBM_PLL_OD);
+	freq = (50 * (nf + 1)) / ((nr + 1) * (od + 1));
+
+	dev_dbg(hdev->dev, "HBM frequency is %lluMHz\n", freq);
+
+	/* Configuration is for five (5) DDMA channels */
+	if (freq == 800) {
+		sat = 4;
+		rst = 11;
+		timeout = 15;
+	} else if (freq == 900) {
+		sat = 4;
+		rst = 15;
+		timeout = 16;
+	} else if (freq == 950) {
+		sat = 4;
+		rst = 15;
+		timeout = 15;
+	} else {
+		dev_warn(hdev->dev,
+			"unsupported HBM frequency %lluMHz, no rate-limiters\n",
+			freq);
+		return;
+	}
+
+	WREG32(mmDMA_IF_W_S_DOWN_RSP_MID_WGHT_0, 0x111);
+	WREG32(mmDMA_IF_W_S_DOWN_RSP_MID_WGHT_1, 0x111);
+	WREG32(mmDMA_IF_E_S_DOWN_RSP_MID_WGHT_0, 0x111);
+	WREG32(mmDMA_IF_E_S_DOWN_RSP_MID_WGHT_1, 0x111);
+	WREG32(mmDMA_IF_W_N_DOWN_RSP_MID_WGHT_0, 0x111);
+	WREG32(mmDMA_IF_W_N_DOWN_RSP_MID_WGHT_1, 0x111);
+	WREG32(mmDMA_IF_E_N_DOWN_RSP_MID_WGHT_0, 0x111);
+	WREG32(mmDMA_IF_E_N_DOWN_RSP_MID_WGHT_1, 0x111);
+
+	if (!hdev->rl_enable) {
+		dev_info(hdev->dev, "Rate limiters disabled\n");
+		return;
+	}
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_SAT, sat);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_SAT, sat);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_SAT, sat);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_SAT, sat);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_SAT, sat);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_SAT, sat);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_SAT, sat);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_SAT, sat);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_RST, rst);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_RST, rst);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_RST, rst);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_RST, rst);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_RST, rst);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_RST, rst);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_RST, rst);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_RST, rst);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_TIMEOUT, timeout);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_HBM_EN, 1);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_HBM_EN, 1);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_HBM_EN, 1);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_HBM_EN, 1);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_HBM_EN, 1);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_HBM_EN, 1);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_HBM_EN, 1);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_HBM_EN, 1);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_SAT, sat);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_SAT, sat);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_SAT, sat);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_SAT, sat);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_SAT, sat);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_SAT, sat);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_SAT, sat);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_SAT, sat);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_RST, rst);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_RST, rst);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_RST, rst);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_RST, rst);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_RST, rst);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_RST, rst);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_RST, rst);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_RST, rst);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_TIMEOUT, timeout);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_TIMEOUT, timeout);
+
+	WREG32(mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_EN, 1);
+	WREG32(mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_EN, 1);
+	WREG32(mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_EN, 1);
+	WREG32(mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_EN, 1);
+	WREG32(mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_EN, 1);
+	WREG32(mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_EN, 1);
+	WREG32(mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_EN, 1);
+	WREG32(mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_EN, 1);
+}
+
+static void gaudi_init_golden_registers(struct hl_device *hdev)
+{
+	u32 tpc_offset;
+	int tpc_id, i;
+
+	gaudi_init_e2e(hdev);
+
+	gaudi_init_hbm_cred(hdev);
+
+	gaudi_init_rate_limiter(hdev);
+
+	gaudi_disable_clock_gating(hdev);
+
+	for (tpc_id = 0, tpc_offset = 0;
+				tpc_id < TPC_NUMBER_OF_ENGINES;
+				tpc_id++, tpc_offset += TPC_CFG_OFFSET) {
+		/* Mask all arithmetic interrupts from TPC */
+		WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFF);
+		/* Set 16 cache lines */
+		WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset,
+				ICACHE_FETCH_LINE_NUM, 2);
+	}
+
+	/* Make sure 1st 128 bytes in SRAM are 0 for Tensor DMA */
+	for (i = 0 ; i < 128 ; i += 8)
+		writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i);
+
+	WREG32(mmMME0_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+	WREG32(mmMME1_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+	WREG32(mmMME2_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+	WREG32(mmMME3_CTRL_EUS_ROLLUP_CNT_ADD, 3);
+
+	/* WA for H3-2081 */
+	WREG32(mmPCIE_WRAP_MAX_OUTSTAND, 0x10ff);
+}
+
+static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
+					int qman_id, dma_addr_t qman_pq_addr)
+{
+	u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
+	u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
+	u32 q_off, dma_qm_offset;
+	u32 dma_qm_err_cfg;
+
+	dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+
+	mtr_base_en_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	mtr_base_en_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	so_base_en_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+	so_base_en_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+	mtr_base_ws_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	mtr_base_ws_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	so_base_ws_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);
+	so_base_ws_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+	q_off = dma_qm_offset + qman_id * 4;
+
+	WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_pq_addr));
+	WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off, upper_32_bits(qman_pq_addr));
+
+	WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HL_QUEUE_LENGTH));
+	WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);
+	WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);
+
+	WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+	WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+	WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);
+	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);
+	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);
+	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);
+	WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo);
+	WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi);
+	WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo);
+	WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi);
+
+	/* The following configuration is needed only once per QMAN */
+	if (qman_id == 0) {
+		/* Configure RAZWI IRQ */
+		dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+		if (hdev->stop_on_err) {
+			dma_qm_err_cfg |=
+				PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+		}
+
+		WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);
+		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
+			lower_32_bits(CFG_BASE +
+					mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
+			upper_32_bits(CFG_BASE +
+					mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
+			gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
+									dma_id);
+
+		WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,
+				QM_ARB_ERR_MSG_EN_MASK);
+
+		/* Increase ARB WDT to support streams architecture */
+		WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset,
+				GAUDI_ARB_WDT_TIMEOUT);
+
+		WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,
+				QMAN_EXTERNAL_MAKE_TRUSTED);
+
+		WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);
+	}
+}
+
+static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)
+{
+	u32 dma_offset = dma_id * DMA_CORE_OFFSET;
+	u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT;
+
+	/* Set to maximum possible according to physical size */
+	WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0);
+	WREG32(mmDMA0_CORE_RD_MAX_SIZE + dma_offset, 0);
+
+	/* STOP_ON bit implies no completion to operation in case of RAZWI */
+	if (hdev->stop_on_err)
+		dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT;
+
+	WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg);
+	WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset,
+		lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+	WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset,
+		upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+	WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset,
+		gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id);
+	WREG32(mmDMA0_CORE_PROT + dma_offset,
+			1 << DMA0_CORE_PROT_ERR_VAL_SHIFT);
+	/* If the channel is secured, it should be in MMU bypass mode */
+	WREG32(mmDMA0_CORE_SECURE_PROPS + dma_offset,
+			1 << DMA0_CORE_SECURE_PROPS_MMBP_SHIFT);
+	WREG32(mmDMA0_CORE_CFG_0 + dma_offset, 1 << DMA0_CORE_CFG_0_EN_SHIFT);
+}
+
+static void gaudi_enable_qman(struct hl_device *hdev, int dma_id,
+				u32 enable_mask)
+{
+	u32 dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+
+	WREG32(mmDMA0_QM_GLBL_CFG0 + dma_qm_offset, enable_mask);
+}
+
+static void gaudi_init_pci_dma_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct hl_hw_queue *q;
+	int i, j, dma_id, cpu_skip, nic_skip, cq_id = 0, q_idx, msi_vec = 0;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)
+		return;
+
+	for (i = 0 ; i < PCI_DMA_NUMBER_OF_CHNLS ; i++) {
+		dma_id = gaudi_dma_assignment[i];
+		/*
+		 * For queues after the CPU Q need to add 1 to get the correct
+		 * queue. In addition, need to add the CPU EQ and NIC IRQs in
+		 * order to get the correct MSI register.
+		 */
+		if (dma_id > 1) {
+			cpu_skip = 1;
+			nic_skip = NIC_NUMBER_OF_ENGINES;
+		} else {
+			cpu_skip = 0;
+			nic_skip = 0;
+		}
+
+		for (j = 0 ; j < QMAN_STREAMS ; j++) {
+			q_idx = 4 * dma_id + j + cpu_skip;
+			q = &hdev->kernel_queues[q_idx];
+			q->cq_id = cq_id++;
+			q->msi_vec = nic_skip + cpu_skip + msi_vec++;
+			gaudi_init_pci_dma_qman(hdev, dma_id, j,
+						q->bus_address);
+		}
+
+		gaudi_init_dma_core(hdev, dma_id);
+
+		gaudi_enable_qman(hdev, dma_id, PCI_DMA_QMAN_ENABLE);
+	}
+
+	gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA;
+}
+
+static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
+					int qman_id, u64 qman_base_addr)
+{
+	u32 mtr_base_lo, mtr_base_hi;
+	u32 so_base_lo, so_base_hi;
+	u32 q_off, dma_qm_offset;
+	u32 dma_qm_err_cfg;
+
+	dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+
+	mtr_base_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	mtr_base_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	so_base_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+	so_base_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+	q_off = dma_qm_offset + qman_id * 4;
+
+	if (qman_id < 4) {
+		WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off,
+					lower_32_bits(qman_base_addr));
+		WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off,
+					upper_32_bits(qman_base_addr));
+
+		WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HBM_DMA_QMAN_LENGTH));
+		WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);
+		WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);
+
+		WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x81BC);
+		WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x81B4);
+		WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+	} else {
+		WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+		WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+		WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+		/* Configure RAZWI IRQ */
+		dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+		if (hdev->stop_on_err) {
+			dma_qm_err_cfg |=
+				HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+		}
+		WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);
+
+		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
+			lower_32_bits(CFG_BASE +
+					mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
+			upper_32_bits(CFG_BASE +
+					mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
+			gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
+									dma_id);
+
+		WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,
+				QM_ARB_ERR_MSG_EN_MASK);
+
+		/* Increase ARB WDT to support streams architecture */
+		WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset,
+				GAUDI_ARB_WDT_TIMEOUT);
+
+		WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);
+		WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,
+				QMAN_INTERNAL_MAKE_TRUSTED);
+	}
+
+	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);
+	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);
+	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);
+	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);
+}
+
+static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct gaudi_internal_qman_info *q;
+	u64 qman_base_addr;
+	int i, j, dma_id, internal_q_index;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)
+		return;
+
+	for (i = 0 ; i < HBM_DMA_NUMBER_OF_CHNLS ; i++) {
+		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1 + i];
+
+		for (j = 0 ; j < QMAN_STREAMS ; j++) {
+			 /*
+			  * Add the CPU queue in order to get the correct queue
+			  * number as all internal queue are placed after it
+			  */
+			internal_q_index = dma_id * QMAN_STREAMS + j + 1;
+
+			q = &gaudi->internal_qmans[internal_q_index];
+			qman_base_addr = (u64) q->pq_dma_addr;
+			gaudi_init_hbm_dma_qman(hdev, dma_id, j,
+						qman_base_addr);
+		}
+
+		/* Initializing lower CP for HBM DMA QMAN */
+		gaudi_init_hbm_dma_qman(hdev, dma_id, 4, 0);
+
+		gaudi_init_dma_core(hdev, dma_id);
+
+		gaudi_enable_qman(hdev, dma_id, HBM_DMA_QMAN_ENABLE);
+	}
+
+	gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA;
+}
+
+static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
+					int qman_id, u64 qman_base_addr)
+{
+	u32 mtr_base_lo, mtr_base_hi;
+	u32 so_base_lo, so_base_hi;
+	u32 q_off, mme_id;
+	u32 mme_qm_err_cfg;
+
+	mtr_base_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	mtr_base_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	so_base_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+	so_base_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+	q_off = mme_offset + qman_id * 4;
+
+	if (qman_id < 4) {
+		WREG32(mmMME0_QM_PQ_BASE_LO_0 + q_off,
+					lower_32_bits(qman_base_addr));
+		WREG32(mmMME0_QM_PQ_BASE_HI_0 + q_off,
+					upper_32_bits(qman_base_addr));
+
+		WREG32(mmMME0_QM_PQ_SIZE_0 + q_off, ilog2(MME_QMAN_LENGTH));
+		WREG32(mmMME0_QM_PQ_PI_0 + q_off, 0);
+		WREG32(mmMME0_QM_PQ_CI_0 + q_off, 0);
+
+		WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x81BC);
+		WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x81B4);
+		WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+	} else {
+		WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+		WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+		WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+		/* Configure RAZWI IRQ */
+		mme_id = mme_offset /
+				(mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0);
+
+		mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+		if (hdev->stop_on_err) {
+			mme_qm_err_cfg |=
+				MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+		}
+		WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg);
+		WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset,
+			lower_32_bits(CFG_BASE +
+					mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset,
+			upper_32_bits(CFG_BASE +
+					mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset,
+			gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id +
+									mme_id);
+
+		WREG32(mmMME0_QM_ARB_ERR_MSG_EN + mme_offset,
+				QM_ARB_ERR_MSG_EN_MASK);
+
+		/* Increase ARB WDT to support streams architecture */
+		WREG32(mmMME0_QM_ARB_SLV_CHOISE_WDT + mme_offset,
+				GAUDI_ARB_WDT_TIMEOUT);
+
+		WREG32(mmMME0_QM_GLBL_CFG1 + mme_offset, 0);
+		WREG32(mmMME0_QM_GLBL_PROT + mme_offset,
+				QMAN_INTERNAL_MAKE_TRUSTED);
+	}
+
+	WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);
+	WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);
+	WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);
+	WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);
+}
+
+static void gaudi_init_mme_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct gaudi_internal_qman_info *q;
+	u64 qman_base_addr;
+	u32 mme_offset;
+	int i, internal_q_index;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_MME)
+		return;
+
+	/*
+	 * map GAUDI_QUEUE_ID_MME_0_X to the N_W_MME (mmMME2_QM_BASE)
+	 * and GAUDI_QUEUE_ID_MME_1_X to the S_W_MME (mmMME0_QM_BASE)
+	 */
+
+	mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;
+
+	for (i = 0 ; i < MME_NUMBER_OF_QMANS ; i++) {
+		internal_q_index = GAUDI_QUEUE_ID_MME_0_0 + i;
+		q = &gaudi->internal_qmans[internal_q_index];
+		qman_base_addr = (u64) q->pq_dma_addr;
+		gaudi_init_mme_qman(hdev, mme_offset, (i & 0x3),
+					qman_base_addr);
+		if (i == 3)
+			mme_offset = 0;
+	}
+
+	/* Initializing lower CP for MME QMANs */
+	mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;
+	gaudi_init_mme_qman(hdev, mme_offset, 4, 0);
+	gaudi_init_mme_qman(hdev, 0, 4, 0);
+
+	WREG32(mmMME2_QM_GLBL_CFG0, QMAN_MME_ENABLE);
+	WREG32(mmMME0_QM_GLBL_CFG0, QMAN_MME_ENABLE);
+
+	gaudi->hw_cap_initialized |= HW_CAP_MME;
+}
+
+static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
+				int qman_id, u64 qman_base_addr)
+{
+	u32 mtr_base_lo, mtr_base_hi;
+	u32 so_base_lo, so_base_hi;
+	u32 q_off, tpc_id;
+	u32 tpc_qm_err_cfg;
+
+	mtr_base_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	mtr_base_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
+	so_base_lo = lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+	so_base_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+	q_off = tpc_offset + qman_id * 4;
+
+	if (qman_id < 4) {
+		WREG32(mmTPC0_QM_PQ_BASE_LO_0 + q_off,
+					lower_32_bits(qman_base_addr));
+		WREG32(mmTPC0_QM_PQ_BASE_HI_0 + q_off,
+					upper_32_bits(qman_base_addr));
+
+		WREG32(mmTPC0_QM_PQ_SIZE_0 + q_off, ilog2(TPC_QMAN_LENGTH));
+		WREG32(mmTPC0_QM_PQ_PI_0 + q_off, 0);
+		WREG32(mmTPC0_QM_PQ_CI_0 + q_off, 0);
+
+		WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x81BC);
+		WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x81B4);
+		WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+	} else {
+		WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74);
+		WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14);
+		WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C);
+
+		/* Configure RAZWI IRQ */
+		tpc_id = tpc_offset /
+				(mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0);
+
+		tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
+		if (hdev->stop_on_err) {
+			tpc_qm_err_cfg |=
+				TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
+		}
+
+		WREG32(mmTPC0_QM_GLBL_ERR_CFG + tpc_offset, tpc_qm_err_cfg);
+		WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + tpc_offset,
+			lower_32_bits(CFG_BASE +
+				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + tpc_offset,
+			upper_32_bits(CFG_BASE +
+				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+		WREG32(mmTPC0_QM_GLBL_ERR_WDATA + tpc_offset,
+			gaudi_irq_map_table[GAUDI_EVENT_TPC0_QM].cpu_id +
+									tpc_id);
+
+		WREG32(mmTPC0_QM_ARB_ERR_MSG_EN + tpc_offset,
+				QM_ARB_ERR_MSG_EN_MASK);
+
+		/* Increase ARB WDT to support streams architecture */
+		WREG32(mmTPC0_QM_ARB_SLV_CHOISE_WDT + tpc_offset,
+				GAUDI_ARB_WDT_TIMEOUT);
+
+		WREG32(mmTPC0_QM_GLBL_CFG1 + tpc_offset, 0);
+		WREG32(mmTPC0_QM_GLBL_PROT + tpc_offset,
+				QMAN_INTERNAL_MAKE_TRUSTED);
+	}
+
+	WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);
+	WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);
+	WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);
+	WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);
+}
+
+static void gaudi_init_tpc_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct gaudi_internal_qman_info *q;
+	u64 qman_base_addr;
+	u32 so_base_hi, tpc_offset = 0;
+	u32 tpc_delta = mmTPC1_CFG_SM_BASE_ADDRESS_HIGH -
+			mmTPC0_CFG_SM_BASE_ADDRESS_HIGH;
+	int i, tpc_id, internal_q_index;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)
+		return;
+
+	so_base_hi = upper_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);
+
+	for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {
+		for (i = 0 ; i < QMAN_STREAMS ; i++) {
+			internal_q_index = GAUDI_QUEUE_ID_TPC_0_0 +
+						tpc_id * QMAN_STREAMS + i;
+			q = &gaudi->internal_qmans[internal_q_index];
+			qman_base_addr = (u64) q->pq_dma_addr;
+			gaudi_init_tpc_qman(hdev, tpc_offset, i,
+						qman_base_addr);
+
+			if (i == 3) {
+				/* Initializing lower CP for TPC QMAN */
+				gaudi_init_tpc_qman(hdev, tpc_offset, 4, 0);
+
+				/* Enable the QMAN and TPC channel */
+				WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset,
+						QMAN_TPC_ENABLE);
+			}
+		}
+
+		WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + tpc_id * tpc_delta,
+				so_base_hi);
+
+		tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0;
+
+		gaudi->hw_cap_initialized |= 1 << (HW_CAP_TPC_SHIFT + tpc_id);
+	}
+}
+
+static void gaudi_disable_pci_dma_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))
+		return;
+
+	WREG32(mmDMA0_QM_GLBL_CFG0, 0);
+	WREG32(mmDMA1_QM_GLBL_CFG0, 0);
+	WREG32(mmDMA5_QM_GLBL_CFG0, 0);
+}
+
+static void gaudi_disable_hbm_dma_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))
+		return;
+
+	WREG32(mmDMA2_QM_GLBL_CFG0, 0);
+	WREG32(mmDMA3_QM_GLBL_CFG0, 0);
+	WREG32(mmDMA4_QM_GLBL_CFG0, 0);
+	WREG32(mmDMA6_QM_GLBL_CFG0, 0);
+	WREG32(mmDMA7_QM_GLBL_CFG0, 0);
+}
+
+static void gaudi_disable_mme_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))
+		return;
+
+	WREG32(mmMME2_QM_GLBL_CFG0, 0);
+	WREG32(mmMME0_QM_GLBL_CFG0, 0);
+}
+
+static void gaudi_disable_tpc_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 tpc_offset = 0;
+	int tpc_id;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))
+		return;
+
+	for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {
+		WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset, 0);
+		tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0;
+	}
+}
+
+static void gaudi_stop_pci_dma_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))
+		return;
+
+	/* Stop upper CPs of QMANs 0.0 to 1.3 and 5.0 to 5.3 */
+	WREG32(mmDMA0_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmDMA1_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmDMA5_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_stop_hbm_dma_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))
+		return;
+
+	/* Stop CPs of HBM DMA QMANs */
+
+	WREG32(mmDMA2_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmDMA3_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmDMA4_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmDMA6_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmDMA7_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_stop_mme_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))
+		return;
+
+	/* Stop CPs of MME QMANs */
+	WREG32(mmMME2_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmMME0_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_stop_tpc_qmans(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))
+		return;
+
+	WREG32(mmTPC0_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmTPC1_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmTPC2_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmTPC3_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmTPC4_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmTPC5_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmTPC6_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+	WREG32(mmTPC7_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
+}
+
+static void gaudi_pci_dma_stall(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))
+		return;
+
+	WREG32(mmDMA0_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+	WREG32(mmDMA1_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+	WREG32(mmDMA5_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+}
+
+static void gaudi_hbm_dma_stall(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))
+		return;
+
+	WREG32(mmDMA2_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+	WREG32(mmDMA3_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+	WREG32(mmDMA4_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+	WREG32(mmDMA6_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+	WREG32(mmDMA7_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);
+}
+
+static void gaudi_mme_stall(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))
+		return;
+
+	/* WA for H3-1800 bug: do ACC and SBAB writes twice */
+	WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+	WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+	WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+	WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+	WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+	WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+	WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);
+	WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+	WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);
+}
+
+static void gaudi_tpc_stall(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))
+		return;
+
+	WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+	WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+	WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+	WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+	WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+	WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+	WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+	WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
+}
+
+static void gaudi_enable_clock_gating(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 qman_offset;
+	int i;
+
+	if (!hdev->clock_gating)
+		return;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE)
+		return;
+
+	/* In case we are during debug session, don't enable the clock gate
+	 * as it may interfere
+	 */
+	if (hdev->in_debug)
+		return;
+
+	for (i = 0, qman_offset = 0 ; i < PCI_DMA_NUMBER_OF_CHNLS ; i++) {
+		qman_offset = gaudi_dma_assignment[i] * DMA_QMAN_OFFSET;
+		WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, QMAN_CGM1_PWR_GATE_EN);
+		WREG32(mmDMA0_QM_CGM_CFG + qman_offset,
+				QMAN_UPPER_CP_CGM_PWR_GATE_EN);
+	}
+
+	for (; i < HBM_DMA_NUMBER_OF_CHNLS ; i++) {
+		qman_offset = gaudi_dma_assignment[i] * DMA_QMAN_OFFSET;
+		WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, QMAN_CGM1_PWR_GATE_EN);
+		WREG32(mmDMA0_QM_CGM_CFG + qman_offset,
+				QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+	}
+
+	WREG32(mmMME0_QM_CGM_CFG1, QMAN_CGM1_PWR_GATE_EN);
+	WREG32(mmMME0_QM_CGM_CFG,
+			QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+	WREG32(mmMME2_QM_CGM_CFG1, QMAN_CGM1_PWR_GATE_EN);
+	WREG32(mmMME2_QM_CGM_CFG,
+			QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+
+	for (i = 0, qman_offset = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+		WREG32(mmTPC0_QM_CGM_CFG1 + qman_offset,
+				QMAN_CGM1_PWR_GATE_EN);
+		WREG32(mmTPC0_QM_CGM_CFG + qman_offset,
+				QMAN_COMMON_CP_CGM_PWR_GATE_EN);
+
+		qman_offset += TPC_QMAN_OFFSET;
+	}
+
+	gaudi->hw_cap_initialized |= HW_CAP_CLK_GATE;
+}
+
+static void gaudi_disable_clock_gating(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 qman_offset;
+	int i;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_CLK_GATE))
+		return;
+
+	for (i = 0, qman_offset = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {
+		WREG32(mmDMA0_QM_CGM_CFG + qman_offset, 0);
+		WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, 0);
+
+		qman_offset += (mmDMA1_QM_CGM_CFG - mmDMA0_QM_CGM_CFG);
+	}
+
+	WREG32(mmMME0_QM_CGM_CFG, 0);
+	WREG32(mmMME0_QM_CGM_CFG1, 0);
+	WREG32(mmMME2_QM_CGM_CFG, 0);
+	WREG32(mmMME2_QM_CGM_CFG1, 0);
+
+	for (i = 0, qman_offset = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+		WREG32(mmTPC0_QM_CGM_CFG + qman_offset, 0);
+		WREG32(mmTPC0_QM_CGM_CFG1 + qman_offset, 0);
+
+		qman_offset += (mmTPC1_QM_CGM_CFG - mmTPC0_QM_CGM_CFG);
+	}
+
+	gaudi->hw_cap_initialized &= ~(HW_CAP_CLK_GATE);
+}
+
+static void gaudi_enable_timestamp(struct hl_device *hdev)
+{
+	/* Disable the timestamp counter */
+	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
+
+	/* Zero the lower/upper parts of the 64-bit counter */
+	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0);
+	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0);
+
+	/* Enable the counter */
+	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1);
+}
+
+static void gaudi_disable_timestamp(struct hl_device *hdev)
+{
+	/* Disable the timestamp counter */
+	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
+}
+
+static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset)
+{
+	u32 wait_timeout_ms, cpu_timeout_ms;
+
+	dev_info(hdev->dev,
+		"Halting compute engines and disabling interrupts\n");
+
+	if (hdev->pldm) {
+		wait_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC;
+		cpu_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC;
+	} else {
+		wait_timeout_ms = GAUDI_RESET_WAIT_MSEC;
+		cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC;
+	}
+
+	if (hard_reset) {
+		/*
+		 * I don't know what is the state of the CPU so make sure it is
+		 * stopped in any means necessary
+		 */
+		WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_GOTO_WFE);
+		WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
+				GAUDI_EVENT_HALT_MACHINE);
+		msleep(cpu_timeout_ms);
+	}
+
+	gaudi_stop_mme_qmans(hdev);
+	gaudi_stop_tpc_qmans(hdev);
+	gaudi_stop_hbm_dma_qmans(hdev);
+	gaudi_stop_pci_dma_qmans(hdev);
+
+	gaudi_disable_clock_gating(hdev);
+
+	msleep(wait_timeout_ms);
+
+	gaudi_pci_dma_stall(hdev);
+	gaudi_hbm_dma_stall(hdev);
+	gaudi_tpc_stall(hdev);
+	gaudi_mme_stall(hdev);
+
+	msleep(wait_timeout_ms);
+
+	gaudi_disable_mme_qmans(hdev);
+	gaudi_disable_tpc_qmans(hdev);
+	gaudi_disable_hbm_dma_qmans(hdev);
+	gaudi_disable_pci_dma_qmans(hdev);
+
+	gaudi_disable_timestamp(hdev);
+
+	if (hard_reset)
+		gaudi_disable_msi(hdev);
+	else
+		gaudi_sync_irqs(hdev);
+}
+
+static int gaudi_mmu_init(struct hl_device *hdev)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 hop0_addr;
+	int rc, i;
+
+	if (!hdev->mmu_enable)
+		return 0;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_MMU)
+		return 0;
+
+	hdev->dram_supports_virtual_memory = false;
+
+	for (i = 0 ; i < prop->max_asid ; i++) {
+		hop0_addr = prop->mmu_pgt_addr +
+				(i * prop->mmu_hop_table_size);
+
+		rc = gaudi_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
+		if (rc) {
+			dev_err(hdev->dev,
+				"failed to set hop0 addr for asid %d\n", i);
+			goto err;
+		}
+	}
+
+	/* init MMU cache manage page */
+	WREG32(mmSTLB_CACHE_INV_BASE_39_8, MMU_CACHE_MNG_ADDR >> 8);
+	WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
+
+	hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
+					VM_TYPE_USERPTR | VM_TYPE_PHYS_PACK);
+
+	WREG32(mmMMU_UP_MMU_ENABLE, 1);
+	WREG32(mmMMU_UP_SPI_MASK, 0xF);
+
+	WREG32(mmSTLB_HOP_CONFIGURATION,
+			hdev->mmu_huge_page_opt ? 0x30440 : 0x40440);
+
+	gaudi->hw_cap_initialized |= HW_CAP_MMU;
+
+	return 0;
+
+err:
+	return rc;
+}
+
+static int gaudi_load_firmware_to_device(struct hl_device *hdev)
+{
+	void __iomem *dst;
+
+	/* HBM scrambler must be initialized before pushing F/W to HBM */
+	gaudi_init_scrambler_hbm(hdev);
+
+	dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET;
+
+	return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst);
+}
+
+static int gaudi_load_boot_fit_to_device(struct hl_device *hdev)
+{
+	void __iomem *dst;
+
+	dst = hdev->pcie_bar[SRAM_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
+
+	return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst);
+}
+
+static void gaudi_read_device_fw_version(struct hl_device *hdev,
+					enum hl_fw_component fwc)
+{
+	const char *name;
+	u32 ver_off;
+	char *dest;
+
+	switch (fwc) {
+	case FW_COMP_UBOOT:
+		ver_off = RREG32(mmUBOOT_VER_OFFSET);
+		dest = hdev->asic_prop.uboot_ver;
+		name = "U-Boot";
+		break;
+	case FW_COMP_PREBOOT:
+		ver_off = RREG32(mmPREBOOT_VER_OFFSET);
+		dest = hdev->asic_prop.preboot_ver;
+		name = "Preboot";
+		break;
+	default:
+		dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
+		return;
+	}
+
+	ver_off &= ~((u32)SRAM_BASE_ADDR);
+
+	if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
+		memcpy_fromio(dest, hdev->pcie_bar[SRAM_BAR_ID] + ver_off,
+							VERSION_MAX_LEN);
+	} else {
+		dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
+								name, ver_off);
+		strcpy(dest, "unavailable");
+	}
+}
+
+static int gaudi_init_cpu(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	int rc;
+
+	if (!hdev->cpu_enable)
+		return 0;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_CPU)
+		return 0;
+
+	/*
+	 * The device CPU works with 40 bits addresses.
+	 * This register sets the extension to 50 bits.
+	 */
+	WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr);
+
+	rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
+			mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU,
+			mmCPU_CMD_STATUS_TO_HOST,
+			mmCPU_BOOT_ERR0,
+			!hdev->bmc_enable, GAUDI_CPU_TIMEOUT_USEC,
+			GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
+
+	if (rc)
+		return rc;
+
+	gaudi->hw_cap_initialized |= HW_CAP_CPU;
+
+	return 0;
+}
+
+static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct hl_eq *eq;
+	u32 status;
+	struct hl_hw_queue *cpu_pq =
+			&hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];
+	int err;
+
+	if (!hdev->cpu_queues_enable)
+		return 0;
+
+	if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)
+		return 0;
+
+	eq = &hdev->event_queue;
+
+	WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
+	WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
+
+	WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
+	WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
+
+	WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW,
+			lower_32_bits(hdev->cpu_accessible_dma_address));
+	WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH,
+			upper_32_bits(hdev->cpu_accessible_dma_address));
+
+	WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
+	WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
+	WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
+
+	/* Used for EQ CI */
+	WREG32(mmCPU_IF_EQ_RD_OFFS, 0);
+
+	WREG32(mmCPU_IF_PF_PQ_PI, 0);
+
+	if (gaudi->multi_msi_mode)
+		WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP);
+	else
+		WREG32(mmCPU_IF_QUEUE_INIT,
+			PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI);
+
+	WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_PI_UPDATE);
+
+	err = hl_poll_timeout(
+		hdev,
+		mmCPU_IF_QUEUE_INIT,
+		status,
+		(status == PQ_INIT_STATUS_READY_FOR_HOST),
+		1000,
+		cpu_timeout);
+
+	if (err) {
+		dev_err(hdev->dev,
+			"Failed to communicate with ARM CPU (ArmCP timeout)\n");
+		return -EIO;
+	}
+
+	gaudi->hw_cap_initialized |= HW_CAP_CPU_Q;
+	return 0;
+}
+
+static void gaudi_pre_hw_init(struct hl_device *hdev)
+{
+	/* Perform read from the device to make sure device is up */
+	RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
+
+	/*
+	 * Let's mark in the H/W that we have reached this point. We check
+	 * this value in the reset_before_init function to understand whether
+	 * we need to reset the chip before doing H/W init. This register is
+	 * cleared by the H/W upon H/W reset
+	 */
+	WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
+
+	/* Set the access through PCI bars (Linux driver only) as secured */
+	WREG32(mmPCIE_WRAP_LBW_PROT_OVR, (PCIE_WRAP_LBW_PROT_OVR_RD_EN_MASK |
+					PCIE_WRAP_LBW_PROT_OVR_WR_EN_MASK));
+
+	/* Perform read to flush the waiting writes to ensure configuration
+	 * was set in the device
+	 */
+	RREG32(mmPCIE_WRAP_LBW_PROT_OVR);
+
+	if (hdev->axi_drain) {
+		WREG32(mmPCIE_WRAP_LBW_DRAIN_CFG,
+			1 << PCIE_WRAP_LBW_DRAIN_CFG_EN_SHIFT);
+		WREG32(mmPCIE_WRAP_HBW_DRAIN_CFG,
+			1 << PCIE_WRAP_HBW_DRAIN_CFG_EN_SHIFT);
+
+		/* Perform read to flush the DRAIN cfg */
+		RREG32(mmPCIE_WRAP_HBW_DRAIN_CFG);
+	} else {
+		WREG32(mmPCIE_WRAP_LBW_DRAIN_CFG, 0);
+		WREG32(mmPCIE_WRAP_HBW_DRAIN_CFG, 0);
+
+		/* Perform read to flush the DRAIN cfg */
+		RREG32(mmPCIE_WRAP_HBW_DRAIN_CFG);
+	}
+
+	/* Configure the reset registers. Must be done as early as possible
+	 * in case we fail during H/W initialization
+	 */
+	WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_H,
+					(CFG_RST_H_DMA_MASK |
+					CFG_RST_H_MME_MASK |
+					CFG_RST_H_SM_MASK |
+					CFG_RST_H_TPC_MASK));
+
+	WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_L, CFG_RST_L_TPC_MASK);
+
+	WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H,
+					(CFG_RST_H_HBM_MASK |
+					CFG_RST_H_TPC_MASK |
+					CFG_RST_H_NIC_MASK |
+					CFG_RST_H_SM_MASK |
+					CFG_RST_H_DMA_MASK |
+					CFG_RST_H_MME_MASK |
+					CFG_RST_H_CPU_MASK |
+					CFG_RST_H_MMU_MASK));
+
+	WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L,
+					(CFG_RST_L_IF_MASK |
+					CFG_RST_L_PSOC_MASK |
+					CFG_RST_L_TPC_MASK));
+}
+
+static int gaudi_hw_init(struct hl_device *hdev)
+{
+	int rc;
+
+	dev_info(hdev->dev, "Starting initialization of H/W\n");
+
+	gaudi_pre_hw_init(hdev);
+
+	gaudi_init_pci_dma_qmans(hdev);
+
+	gaudi_init_hbm_dma_qmans(hdev);
+
+	/*
+	 * Before pushing u-boot/linux to device, need to set the hbm bar to
+	 * base address of dram
+	 */
+	if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
+		dev_err(hdev->dev,
+			"failed to map HBM bar to DRAM base address\n");
+		return -EIO;
+	}
+
+	rc = gaudi_init_cpu(hdev);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize CPU\n");
+		return rc;
+	}
+
+	/* SRAM scrambler must be initialized after CPU is running from HBM */
+	gaudi_init_scrambler_sram(hdev);
+
+	/* This is here just in case we are working without CPU */
+	gaudi_init_scrambler_hbm(hdev);
+
+	gaudi_init_golden_registers(hdev);
+
+	rc = gaudi_mmu_init(hdev);
+	if (rc)
+		return rc;
+
+	gaudi_init_security(hdev);
+
+	gaudi_init_mme_qmans(hdev);
+
+	gaudi_init_tpc_qmans(hdev);
+
+	gaudi_enable_clock_gating(hdev);
+
+	gaudi_enable_timestamp(hdev);
+
+	/* MSI must be enabled before CPU queues are initialized */
+	rc = gaudi_enable_msi(hdev);
+	if (rc)
+		goto disable_queues;
+
+	/* must be called after MSI was enabled */
+	rc = gaudi_init_cpu_queues(hdev, GAUDI_CPU_TIMEOUT_USEC);
+	if (rc) {
+		dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n",
+			rc);
+		goto disable_msi;
+	}
+
+	/* Perform read from the device to flush all configuration */
+	RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
+
+	return 0;
+
+disable_msi:
+	gaudi_disable_msi(hdev);
+disable_queues:
+	gaudi_disable_mme_qmans(hdev);
+	gaudi_disable_pci_dma_qmans(hdev);
+
+	return rc;
+}
+
+static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 status, reset_timeout_ms, boot_strap = 0;
+
+	if (hdev->pldm) {
+		if (hard_reset)
+			reset_timeout_ms = GAUDI_PLDM_HRESET_TIMEOUT_MSEC;
+		else
+			reset_timeout_ms = GAUDI_PLDM_SRESET_TIMEOUT_MSEC;
+	} else {
+		reset_timeout_ms = GAUDI_RESET_TIMEOUT_MSEC;
+	}
+
+	if (hard_reset) {
+		/* Tell ASIC not to re-initialize PCIe */
+		WREG32(mmPREBOOT_PCIE_EN, LKD_HARD_RESET_MAGIC);
+
+		boot_strap = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS);
+		/* H/W bug WA:
+		 * rdata[31:0] = strap_read_val;
+		 * wdata[31:0] = rdata[30:21],1'b0,rdata[20:0]
+		 */
+		boot_strap = (((boot_strap & 0x7FE00000) << 1) |
+				(boot_strap & 0x001FFFFF));
+		WREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS, boot_strap & ~0x2);
+
+		/* Restart BTL/BLR upon hard-reset */
+		WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1);
+
+		WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST,
+				1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT);
+		dev_info(hdev->dev,
+			"Issued HARD reset command, going to wait %dms\n",
+			reset_timeout_ms);
+	} else {
+		/* Don't restart BTL/BLR upon soft-reset */
+		WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 0);
+
+		WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST,
+				1 << PSOC_GLOBAL_CONF_SOFT_RST_IND_SHIFT);
+		dev_info(hdev->dev,
+			"Issued SOFT reset command, going to wait %dms\n",
+			reset_timeout_ms);
+	}
+
+	/*
+	 * After hard reset, we can't poll the BTM_FSM register because the PSOC
+	 * itself is in reset. Need to wait until the reset is deasserted
+	 */
+	msleep(reset_timeout_ms);
+
+	status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
+	if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
+		dev_err(hdev->dev,
+			"Timeout while waiting for device to reset 0x%x\n",
+			status);
+
+	if (!hard_reset) {
+		gaudi->hw_cap_initialized &= ~(HW_CAP_PCI_DMA | HW_CAP_MME |
+						HW_CAP_TPC_MASK |
+						HW_CAP_HBM_DMA);
+
+		WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
+				GAUDI_EVENT_SOFT_RESET);
+		return;
+	}
+
+	/* We continue here only for hard-reset */
+
+	WREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS, boot_strap);
+
+	gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |
+					HW_CAP_HBM | HW_CAP_PCI_DMA |
+					HW_CAP_MME | HW_CAP_TPC_MASK |
+					HW_CAP_HBM_DMA | HW_CAP_PLL |
+					HW_CAP_MMU |
+					HW_CAP_SRAM_SCRAMBLER |
+					HW_CAP_HBM_SCRAMBLER);
+	memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat));
+}
+
+static int gaudi_suspend(struct hl_device *hdev)
+{
+	int rc;
+
+	rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
+	if (rc)
+		dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
+
+	return rc;
+}
+
+static int gaudi_resume(struct hl_device *hdev)
+{
+	return gaudi_init_iatu(hdev);
+}
+
+static int gaudi_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
+		u64 kaddress, phys_addr_t paddress, u32 size)
+{
+	int rc;
+
+	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
+			VM_DONTCOPY | VM_NORESERVE;
+
+	rc = remap_pfn_range(vma, vma->vm_start, paddress >> PAGE_SHIFT,
+				size, vma->vm_page_prot);
+	if (rc)
+		dev_err(hdev->dev, "remap_pfn_range error %d", rc);
+
+	return rc;
+}
+
+static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 db_reg_offset, db_value, dma_qm_offset, q_off;
+	int dma_id;
+	bool invalid_queue = false;
+
+	switch (hw_queue_id) {
+	case GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3:
+		dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_1];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_DMA_1_0...GAUDI_QUEUE_ID_DMA_1_3:
+		dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_2];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_DMA_2_0...GAUDI_QUEUE_ID_DMA_2_3:
+		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_DMA_3_0...GAUDI_QUEUE_ID_DMA_3_3:
+		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_2];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_DMA_4_0...GAUDI_QUEUE_ID_DMA_4_3:
+		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_3];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3:
+		dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_3];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3:
+		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3:
+		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5];
+		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
+		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;
+		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;
+		break;
+
+	case GAUDI_QUEUE_ID_CPU_PQ:
+		if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)
+			db_reg_offset = mmCPU_IF_PF_PQ_PI;
+		else
+			invalid_queue = true;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_0_0:
+		db_reg_offset = mmMME2_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_0_1:
+		db_reg_offset = mmMME2_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_0_2:
+		db_reg_offset = mmMME2_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_0_3:
+		db_reg_offset = mmMME2_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_1_0:
+		db_reg_offset = mmMME0_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_1_1:
+		db_reg_offset = mmMME0_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_1_2:
+		db_reg_offset = mmMME0_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_MME_1_3:
+		db_reg_offset = mmMME0_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_0_0:
+		db_reg_offset = mmTPC0_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_0_1:
+		db_reg_offset = mmTPC0_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_0_2:
+		db_reg_offset = mmTPC0_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_0_3:
+		db_reg_offset = mmTPC0_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_1_0:
+		db_reg_offset = mmTPC1_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_1_1:
+		db_reg_offset = mmTPC1_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_1_2:
+		db_reg_offset = mmTPC1_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_1_3:
+		db_reg_offset = mmTPC1_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_2_0:
+		db_reg_offset = mmTPC2_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_2_1:
+		db_reg_offset = mmTPC2_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_2_2:
+		db_reg_offset = mmTPC2_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_2_3:
+		db_reg_offset = mmTPC2_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_3_0:
+		db_reg_offset = mmTPC3_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_3_1:
+		db_reg_offset = mmTPC3_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_3_2:
+		db_reg_offset = mmTPC3_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_3_3:
+		db_reg_offset = mmTPC3_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_4_0:
+		db_reg_offset = mmTPC4_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_4_1:
+		db_reg_offset = mmTPC4_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_4_2:
+		db_reg_offset = mmTPC4_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_4_3:
+		db_reg_offset = mmTPC4_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_5_0:
+		db_reg_offset = mmTPC5_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_5_1:
+		db_reg_offset = mmTPC5_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_5_2:
+		db_reg_offset = mmTPC5_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_5_3:
+		db_reg_offset = mmTPC5_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_6_0:
+		db_reg_offset = mmTPC6_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_6_1:
+		db_reg_offset = mmTPC6_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_6_2:
+		db_reg_offset = mmTPC6_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_6_3:
+		db_reg_offset = mmTPC6_QM_PQ_PI_3;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_7_0:
+		db_reg_offset = mmTPC7_QM_PQ_PI_0;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_7_1:
+		db_reg_offset = mmTPC7_QM_PQ_PI_1;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_7_2:
+		db_reg_offset = mmTPC7_QM_PQ_PI_2;
+		break;
+
+	case GAUDI_QUEUE_ID_TPC_7_3:
+		db_reg_offset = mmTPC7_QM_PQ_PI_3;
+		break;
+
+	default:
+		invalid_queue = true;
+	}
+
+	if (invalid_queue) {
+		/* Should never get here */
+		dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n",
+			hw_queue_id);
+		return;
+	}
+
+	db_value = pi;
+
+	/* ring the doorbell */
+	WREG32(db_reg_offset, db_value);
+
+	if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ)
+		WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
+				GAUDI_EVENT_PI_UPDATE);
+}
+
+static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe,
+				struct hl_bd *bd)
+{
+	__le64 *pbd = (__le64 *) bd;
+
+	/* The QMANs are on the host memory so a simple copy suffice */
+	pqe[0] = pbd[0];
+	pqe[1] = pbd[1];
+}
+
+static void *gaudi_dma_alloc_coherent(struct hl_device *hdev, size_t size,
+					dma_addr_t *dma_handle, gfp_t flags)
+{
+	void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
+						dma_handle, flags);
+
+	/* Shift to the device's base physical address of host memory */
+	if (kernel_addr)
+		*dma_handle += HOST_PHYS_BASE;
+
+	return kernel_addr;
+}
+
+static void gaudi_dma_free_coherent(struct hl_device *hdev, size_t size,
+		void *cpu_addr, dma_addr_t dma_handle)
+{
+	/* Cancel the device's base physical address of host memory */
+	dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;
+
+	dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
+}
+
+static void *gaudi_get_int_queue_base(struct hl_device *hdev,
+				u32 queue_id, dma_addr_t *dma_handle,
+				u16 *queue_len)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct gaudi_internal_qman_info *q;
+
+	if (queue_id >= GAUDI_QUEUE_ID_SIZE ||
+			gaudi_queue_type[queue_id] != QUEUE_TYPE_INT) {
+		dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
+		return NULL;
+	}
+
+	q = &gaudi->internal_qmans[queue_id];
+	*dma_handle = q->pq_dma_addr;
+	*queue_len = q->pq_size / QMAN_PQ_ENTRY_SIZE;
+
+	return q->pq_kernel_addr;
+}
+
+static int gaudi_send_cpu_message(struct hl_device *hdev, u32 *msg,
+				u16 len, u32 timeout, long *result)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) {
+		if (result)
+			*result = 0;
+		return 0;
+	}
+
+	return hl_fw_send_cpu_message(hdev, GAUDI_QUEUE_ID_CPU_PQ, msg, len,
+						timeout, result);
+}
+
+static int gaudi_test_queue(struct hl_device *hdev, u32 hw_queue_id)
+{
+	struct packet_msg_prot *fence_pkt;
+	dma_addr_t pkt_dma_addr;
+	u32 fence_val, tmp, timeout_usec;
+	dma_addr_t fence_dma_addr;
+	u32 *fence_ptr;
+	int rc;
+
+	if (hdev->pldm)
+		timeout_usec = GAUDI_PLDM_TEST_QUEUE_WAIT_USEC;
+	else
+		timeout_usec = GAUDI_TEST_QUEUE_WAIT_USEC;
+
+	fence_val = GAUDI_QMAN0_FENCE_VAL;
+
+	fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
+							&fence_dma_addr);
+	if (!fence_ptr) {
+		dev_err(hdev->dev,
+			"Failed to allocate memory for queue testing\n");
+		return -ENOMEM;
+	}
+
+	*fence_ptr = 0;
+
+	fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
+					sizeof(struct packet_msg_prot),
+					GFP_KERNEL, &pkt_dma_addr);
+	if (!fence_pkt) {
+		dev_err(hdev->dev,
+			"Failed to allocate packet for queue testing\n");
+		rc = -ENOMEM;
+		goto free_fence_ptr;
+	}
+
+	tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+			(1 << GAUDI_PKT_CTL_EB_SHIFT) |
+			(1 << GAUDI_PKT_CTL_MB_SHIFT);
+	fence_pkt->ctl = cpu_to_le32(tmp);
+	fence_pkt->value = cpu_to_le32(fence_val);
+	fence_pkt->addr = cpu_to_le64(fence_dma_addr);
+
+	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
+					sizeof(struct packet_msg_prot),
+					pkt_dma_addr);
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to send fence packet\n");
+		goto free_pkt;
+	}
+
+	rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
+					1000, timeout_usec, true);
+
+	hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
+
+	if (rc == -ETIMEDOUT) {
+		dev_err(hdev->dev,
+			"H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
+			hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
+		rc = -EIO;
+	}
+
+free_pkt:
+	hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
+					pkt_dma_addr);
+free_fence_ptr:
+	hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
+					fence_dma_addr);
+	return rc;
+}
+
+static int gaudi_test_cpu_queue(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	/*
+	 * check capability here as send_cpu_message() won't update the result
+	 * value if no capability
+	 */
+	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+		return 0;
+
+	return hl_fw_test_cpu_queue(hdev);
+}
+
+static int gaudi_test_queues(struct hl_device *hdev)
+{
+	int i, rc, ret_val = 0;
+
+	for (i = 0 ; i < HL_MAX_QUEUES ; i++) {
+		if (hdev->asic_prop.hw_queues_props[i].type == QUEUE_TYPE_EXT) {
+			rc = gaudi_test_queue(hdev, i);
+			if (rc)
+				ret_val = -EINVAL;
+		}
+	}
+
+	rc = gaudi_test_cpu_queue(hdev);
+	if (rc)
+		ret_val = -EINVAL;
+
+	return ret_val;
+}
+
+static void *gaudi_dma_pool_zalloc(struct hl_device *hdev, size_t size,
+		gfp_t mem_flags, dma_addr_t *dma_handle)
+{
+	void *kernel_addr;
+
+	if (size > GAUDI_DMA_POOL_BLK_SIZE)
+		return NULL;
+
+	kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
+
+	/* Shift to the device's base physical address of host memory */
+	if (kernel_addr)
+		*dma_handle += HOST_PHYS_BASE;
+
+	return kernel_addr;
+}
+
+static void gaudi_dma_pool_free(struct hl_device *hdev, void *vaddr,
+			dma_addr_t dma_addr)
+{
+	/* Cancel the device's base physical address of host memory */
+	dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;
+
+	dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
+}
+
+static void *gaudi_cpu_accessible_dma_pool_alloc(struct hl_device *hdev,
+					size_t size, dma_addr_t *dma_handle)
+{
+	return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
+}
+
+static void gaudi_cpu_accessible_dma_pool_free(struct hl_device *hdev,
+						size_t size, void *vaddr)
+{
+	hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
+}
+
+static int gaudi_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
+			int nents, enum dma_data_direction dir)
+{
+	struct scatterlist *sg;
+	int i;
+
+	if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
+		return -ENOMEM;
+
+	/* Shift to the device's base physical address of host memory */
+	for_each_sg(sgl, sg, nents, i)
+		sg->dma_address += HOST_PHYS_BASE;
+
+	return 0;
+}
+
+static void gaudi_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
+			int nents, enum dma_data_direction dir)
+{
+	struct scatterlist *sg;
+	int i;
+
+	/* Cancel the device's base physical address of host memory */
+	for_each_sg(sgl, sg, nents, i)
+		sg->dma_address -= HOST_PHYS_BASE;
+
+	dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
+}
+
+static u32 gaudi_get_dma_desc_list_size(struct hl_device *hdev,
+					struct sg_table *sgt)
+{
+	struct scatterlist *sg, *sg_next_iter;
+	u32 count, dma_desc_cnt;
+	u64 len, len_next;
+	dma_addr_t addr, addr_next;
+
+	dma_desc_cnt = 0;
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, count) {
+
+		len = sg_dma_len(sg);
+		addr = sg_dma_address(sg);
+
+		if (len == 0)
+			break;
+
+		while ((count + 1) < sgt->nents) {
+			sg_next_iter = sg_next(sg);
+			len_next = sg_dma_len(sg_next_iter);
+			addr_next = sg_dma_address(sg_next_iter);
+
+			if (len_next == 0)
+				break;
+
+			if ((addr + len == addr_next) &&
+				(len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
+				len += len_next;
+				count++;
+				sg = sg_next_iter;
+			} else {
+				break;
+			}
+		}
+
+		dma_desc_cnt++;
+	}
+
+	return dma_desc_cnt * sizeof(struct packet_lin_dma);
+}
+
+static int gaudi_pin_memory_before_cs(struct hl_device *hdev,
+				struct hl_cs_parser *parser,
+				struct packet_lin_dma *user_dma_pkt,
+				u64 addr, enum dma_data_direction dir)
+{
+	struct hl_userptr *userptr;
+	int rc;
+
+	if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
+			parser->job_userptr_list, &userptr))
+		goto already_pinned;
+
+	userptr = kzalloc(sizeof(*userptr), GFP_ATOMIC);
+	if (!userptr)
+		return -ENOMEM;
+
+	rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
+				userptr);
+	if (rc)
+		goto free_userptr;
+
+	list_add_tail(&userptr->job_node, parser->job_userptr_list);
+
+	rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
+					userptr->sgt->nents, dir);
+	if (rc) {
+		dev_err(hdev->dev, "failed to map sgt with DMA region\n");
+		goto unpin_memory;
+	}
+
+	userptr->dma_mapped = true;
+	userptr->dir = dir;
+
+already_pinned:
+	parser->patched_cb_size +=
+			gaudi_get_dma_desc_list_size(hdev, userptr->sgt);
+
+	return 0;
+
+unpin_memory:
+	hl_unpin_host_memory(hdev, userptr);
+free_userptr:
+	kfree(userptr);
+	return rc;
+}
+
+static int gaudi_validate_dma_pkt_host(struct hl_device *hdev,
+				struct hl_cs_parser *parser,
+				struct packet_lin_dma *user_dma_pkt,
+				bool src_in_host)
+{
+	enum dma_data_direction dir;
+	bool skip_host_mem_pin = false, user_memset;
+	u64 addr;
+	int rc = 0;
+
+	user_memset = (le32_to_cpu(user_dma_pkt->ctl) &
+			GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
+			GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
+
+	if (src_in_host) {
+		if (user_memset)
+			skip_host_mem_pin = true;
+
+		dev_dbg(hdev->dev, "DMA direction is HOST --> DEVICE\n");
+		dir = DMA_TO_DEVICE;
+		addr = le64_to_cpu(user_dma_pkt->src_addr);
+	} else {
+		dev_dbg(hdev->dev, "DMA direction is DEVICE --> HOST\n");
+		dir = DMA_FROM_DEVICE;
+		addr = (le64_to_cpu(user_dma_pkt->dst_addr) &
+				GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>
+				GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;
+	}
+
+	if (skip_host_mem_pin)
+		parser->patched_cb_size += sizeof(*user_dma_pkt);
+	else
+		rc = gaudi_pin_memory_before_cs(hdev, parser, user_dma_pkt,
+						addr, dir);
+
+	return rc;
+}
+
+static int gaudi_validate_dma_pkt_no_mmu(struct hl_device *hdev,
+				struct hl_cs_parser *parser,
+				struct packet_lin_dma *user_dma_pkt)
+{
+	bool src_in_host = false;
+	u64 dst_addr = (le64_to_cpu(user_dma_pkt->dst_addr) &
+			GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>
+			GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;
+
+	dev_dbg(hdev->dev, "DMA packet details:\n");
+	dev_dbg(hdev->dev, "source == 0x%llx\n",
+				le64_to_cpu(user_dma_pkt->src_addr));
+	dev_dbg(hdev->dev, "destination == 0x%llx\n", dst_addr);
+	dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
+
+	/*
+	 * Special handling for DMA with size 0. Bypass all validations
+	 * because no transactions will be done except for WR_COMP, which
+	 * is not a security issue
+	 */
+	if (!le32_to_cpu(user_dma_pkt->tsize)) {
+		parser->patched_cb_size += sizeof(*user_dma_pkt);
+		return 0;
+	}
+
+	if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3)
+		src_in_host = true;
+
+	return gaudi_validate_dma_pkt_host(hdev, parser, user_dma_pkt,
+						src_in_host);
+}
+
+static int gaudi_validate_cb(struct hl_device *hdev,
+			struct hl_cs_parser *parser, bool is_mmu)
+{
+	u32 cb_parsed_length = 0;
+	int rc = 0;
+
+	parser->patched_cb_size = 0;
+
+	/* cb_user_size is more than 0 so loop will always be executed */
+	while (cb_parsed_length < parser->user_cb_size) {
+		enum packet_id pkt_id;
+		u16 pkt_size;
+		struct gaudi_packet *user_pkt;
+
+		user_pkt = (struct gaudi_packet *) (uintptr_t)
+			(parser->user_cb->kernel_address + cb_parsed_length);
+
+		pkt_id = (enum packet_id) (
+				(le64_to_cpu(user_pkt->header) &
+				PACKET_HEADER_PACKET_ID_MASK) >>
+					PACKET_HEADER_PACKET_ID_SHIFT);
+
+		pkt_size = gaudi_packet_sizes[pkt_id];
+		cb_parsed_length += pkt_size;
+		if (cb_parsed_length > parser->user_cb_size) {
+			dev_err(hdev->dev,
+				"packet 0x%x is out of CB boundary\n", pkt_id);
+			rc = -EINVAL;
+			break;
+		}
+
+		switch (pkt_id) {
+		case PACKET_MSG_PROT:
+			dev_err(hdev->dev,
+				"User not allowed to use MSG_PROT\n");
+			rc = -EPERM;
+			break;
+
+		case PACKET_CP_DMA:
+			dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
+			rc = -EPERM;
+			break;
+
+		case PACKET_STOP:
+			dev_err(hdev->dev, "User not allowed to use STOP\n");
+			rc = -EPERM;
+			break;
+
+		case PACKET_LIN_DMA:
+			parser->contains_dma_pkt = true;
+			if (is_mmu)
+				parser->patched_cb_size += pkt_size;
+			else
+				rc = gaudi_validate_dma_pkt_no_mmu(hdev, parser,
+					(struct packet_lin_dma *) user_pkt);
+			break;
+
+		case PACKET_WREG_32:
+		case PACKET_WREG_BULK:
+		case PACKET_MSG_LONG:
+		case PACKET_MSG_SHORT:
+		case PACKET_REPEAT:
+		case PACKET_FENCE:
+		case PACKET_NOP:
+		case PACKET_ARB_POINT:
+		case PACKET_LOAD_AND_EXE:
+			parser->patched_cb_size += pkt_size;
+			break;
+
+		default:
+			dev_err(hdev->dev, "Invalid packet header 0x%x\n",
+				pkt_id);
+			rc = -EINVAL;
+			break;
+		}
+
+		if (rc)
+			break;
+	}
+
+	/*
+	 * The new CB should have space at the end for two MSG_PROT packets:
+	 * 1. A packet that will act as a completion packet
+	 * 2. A packet that will generate MSI-X interrupt
+	 */
+	parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2;
+
+	return rc;
+}
+
+static int gaudi_patch_dma_packet(struct hl_device *hdev,
+				struct hl_cs_parser *parser,
+				struct packet_lin_dma *user_dma_pkt,
+				struct packet_lin_dma *new_dma_pkt,
+				u32 *new_dma_pkt_size)
+{
+	struct hl_userptr *userptr;
+	struct scatterlist *sg, *sg_next_iter;
+	u32 count, dma_desc_cnt, user_wrcomp_en_mask, ctl;
+	u64 len, len_next;
+	dma_addr_t dma_addr, dma_addr_next;
+	u64 device_memory_addr, addr;
+	enum dma_data_direction dir;
+	struct sg_table *sgt;
+	bool src_in_host = false;
+	bool skip_host_mem_pin = false;
+	bool user_memset;
+
+	ctl = le32_to_cpu(user_dma_pkt->ctl);
+
+	if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3)
+		src_in_host = true;
+
+	user_memset = (ctl & GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
+			GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
+
+	if (src_in_host) {
+		addr = le64_to_cpu(user_dma_pkt->src_addr);
+		device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
+		dir = DMA_TO_DEVICE;
+		if (user_memset)
+			skip_host_mem_pin = true;
+	} else {
+		addr = le64_to_cpu(user_dma_pkt->dst_addr);
+		device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
+		dir = DMA_FROM_DEVICE;
+	}
+
+	if ((!skip_host_mem_pin) &&
+		(!hl_userptr_is_pinned(hdev, addr,
+					le32_to_cpu(user_dma_pkt->tsize),
+					parser->job_userptr_list, &userptr))) {
+		dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
+				addr, user_dma_pkt->tsize);
+		return -EFAULT;
+	}
+
+	if ((user_memset) && (dir == DMA_TO_DEVICE)) {
+		memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt));
+		*new_dma_pkt_size = sizeof(*user_dma_pkt);
+		return 0;
+	}
+
+	user_wrcomp_en_mask = ctl & GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK;
+
+	sgt = userptr->sgt;
+	dma_desc_cnt = 0;
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, count) {
+		len = sg_dma_len(sg);
+		dma_addr = sg_dma_address(sg);
+
+		if (len == 0)
+			break;
+
+		while ((count + 1) < sgt->nents) {
+			sg_next_iter = sg_next(sg);
+			len_next = sg_dma_len(sg_next_iter);
+			dma_addr_next = sg_dma_address(sg_next_iter);
+
+			if (len_next == 0)
+				break;
+
+			if ((dma_addr + len == dma_addr_next) &&
+				(len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
+				len += len_next;
+				count++;
+				sg = sg_next_iter;
+			} else {
+				break;
+			}
+		}
+
+		new_dma_pkt->ctl = user_dma_pkt->ctl;
+
+		ctl = le32_to_cpu(user_dma_pkt->ctl);
+		if (likely(dma_desc_cnt))
+			ctl &= ~GAUDI_PKT_CTL_EB_MASK;
+		ctl &= ~GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK;
+		new_dma_pkt->ctl = cpu_to_le32(ctl);
+		new_dma_pkt->tsize = cpu_to_le32(len);
+
+		if (dir == DMA_TO_DEVICE) {
+			new_dma_pkt->src_addr = cpu_to_le64(dma_addr);
+			new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);
+		} else {
+			new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr);
+			new_dma_pkt->dst_addr = cpu_to_le64(dma_addr);
+		}
+
+		if (!user_memset)
+			device_memory_addr += len;
+		dma_desc_cnt++;
+		new_dma_pkt++;
+	}
+
+	if (!dma_desc_cnt) {
+		dev_err(hdev->dev,
+			"Error of 0 SG entries when patching DMA packet\n");
+		return -EFAULT;
+	}
+
+	/* Fix the last dma packet - wrcomp must be as user set it */
+	new_dma_pkt--;
+	new_dma_pkt->ctl |= cpu_to_le32(user_wrcomp_en_mask);
+
+	*new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma);
+
+	return 0;
+}
+
+static int gaudi_patch_cb(struct hl_device *hdev,
+				struct hl_cs_parser *parser)
+{
+	u32 cb_parsed_length = 0;
+	u32 cb_patched_cur_length = 0;
+	int rc = 0;
+
+	/* cb_user_size is more than 0 so loop will always be executed */
+	while (cb_parsed_length < parser->user_cb_size) {
+		enum packet_id pkt_id;
+		u16 pkt_size;
+		u32 new_pkt_size = 0;
+		struct gaudi_packet *user_pkt, *kernel_pkt;
+
+		user_pkt = (struct gaudi_packet *) (uintptr_t)
+			(parser->user_cb->kernel_address + cb_parsed_length);
+		kernel_pkt = (struct gaudi_packet *) (uintptr_t)
+			(parser->patched_cb->kernel_address +
+					cb_patched_cur_length);
+
+		pkt_id = (enum packet_id) (
+				(le64_to_cpu(user_pkt->header) &
+				PACKET_HEADER_PACKET_ID_MASK) >>
+					PACKET_HEADER_PACKET_ID_SHIFT);
+
+		pkt_size = gaudi_packet_sizes[pkt_id];
+		cb_parsed_length += pkt_size;
+		if (cb_parsed_length > parser->user_cb_size) {
+			dev_err(hdev->dev,
+				"packet 0x%x is out of CB boundary\n", pkt_id);
+			rc = -EINVAL;
+			break;
+		}
+
+		switch (pkt_id) {
+		case PACKET_LIN_DMA:
+			rc = gaudi_patch_dma_packet(hdev, parser,
+					(struct packet_lin_dma *) user_pkt,
+					(struct packet_lin_dma *) kernel_pkt,
+					&new_pkt_size);
+			cb_patched_cur_length += new_pkt_size;
+			break;
+
+		case PACKET_MSG_PROT:
+			dev_err(hdev->dev,
+				"User not allowed to use MSG_PROT\n");
+			rc = -EPERM;
+			break;
+
+		case PACKET_CP_DMA:
+			dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
+			rc = -EPERM;
+			break;
+
+		case PACKET_STOP:
+			dev_err(hdev->dev, "User not allowed to use STOP\n");
+			rc = -EPERM;
+			break;
+
+		case PACKET_WREG_32:
+		case PACKET_WREG_BULK:
+		case PACKET_MSG_LONG:
+		case PACKET_MSG_SHORT:
+		case PACKET_REPEAT:
+		case PACKET_FENCE:
+		case PACKET_NOP:
+		case PACKET_ARB_POINT:
+		case PACKET_LOAD_AND_EXE:
+			memcpy(kernel_pkt, user_pkt, pkt_size);
+			cb_patched_cur_length += pkt_size;
+			break;
+
+		default:
+			dev_err(hdev->dev, "Invalid packet header 0x%x\n",
+				pkt_id);
+			rc = -EINVAL;
+			break;
+		}
+
+		if (rc)
+			break;
+	}
+
+	return rc;
+}
+
+static int gaudi_parse_cb_mmu(struct hl_device *hdev,
+		struct hl_cs_parser *parser)
+{
+	u64 patched_cb_handle;
+	u32 patched_cb_size;
+	struct hl_cb *user_cb;
+	int rc;
+
+	/*
+	 * The new CB should have space at the end for two MSG_PROT pkt:
+	 * 1. A packet that will act as a completion packet
+	 * 2. A packet that will generate MSI interrupt
+	 */
+	parser->patched_cb_size = parser->user_cb_size +
+			sizeof(struct packet_msg_prot) * 2;
+
+	rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
+				parser->patched_cb_size,
+				&patched_cb_handle, HL_KERNEL_ASID_ID);
+
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to allocate patched CB for DMA CS %d\n",
+			rc);
+		return rc;
+	}
+
+	patched_cb_handle >>= PAGE_SHIFT;
+	parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
+				(u32) patched_cb_handle);
+	/* hl_cb_get should never fail here so use kernel WARN */
+	WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
+			(u32) patched_cb_handle);
+	if (!parser->patched_cb) {
+		rc = -EFAULT;
+		goto out;
+	}
+
+	/*
+	 * The check that parser->user_cb_size <= parser->user_cb->size was done
+	 * in validate_queue_index().
+	 */
+	memcpy((void *) (uintptr_t) parser->patched_cb->kernel_address,
+		(void *) (uintptr_t) parser->user_cb->kernel_address,
+		parser->user_cb_size);
+
+	patched_cb_size = parser->patched_cb_size;
+
+	/* Validate patched CB instead of user CB */
+	user_cb = parser->user_cb;
+	parser->user_cb = parser->patched_cb;
+	rc = gaudi_validate_cb(hdev, parser, true);
+	parser->user_cb = user_cb;
+
+	if (rc) {
+		hl_cb_put(parser->patched_cb);
+		goto out;
+	}
+
+	if (patched_cb_size != parser->patched_cb_size) {
+		dev_err(hdev->dev, "user CB size mismatch\n");
+		hl_cb_put(parser->patched_cb);
+		rc = -EINVAL;
+		goto out;
+	}
+
+out:
+	/*
+	 * Always call cb destroy here because we still have 1 reference
+	 * to it by calling cb_get earlier. After the job will be completed,
+	 * cb_put will release it, but here we want to remove it from the
+	 * idr
+	 */
+	hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
+					patched_cb_handle << PAGE_SHIFT);
+
+	return rc;
+}
+
+static int gaudi_parse_cb_no_mmu(struct hl_device *hdev,
+		struct hl_cs_parser *parser)
+{
+	u64 patched_cb_handle;
+	int rc;
+
+	rc = gaudi_validate_cb(hdev, parser, false);
+
+	if (rc)
+		goto free_userptr;
+
+	rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
+				parser->patched_cb_size,
+				&patched_cb_handle, HL_KERNEL_ASID_ID);
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to allocate patched CB for DMA CS %d\n", rc);
+		goto free_userptr;
+	}
+
+	patched_cb_handle >>= PAGE_SHIFT;
+	parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
+				(u32) patched_cb_handle);
+	/* hl_cb_get should never fail here so use kernel WARN */
+	WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
+			(u32) patched_cb_handle);
+	if (!parser->patched_cb) {
+		rc = -EFAULT;
+		goto out;
+	}
+
+	rc = gaudi_patch_cb(hdev, parser);
+
+	if (rc)
+		hl_cb_put(parser->patched_cb);
+
+out:
+	/*
+	 * Always call cb destroy here because we still have 1 reference
+	 * to it by calling cb_get earlier. After the job will be completed,
+	 * cb_put will release it, but here we want to remove it from the
+	 * idr
+	 */
+	hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
+				patched_cb_handle << PAGE_SHIFT);
+
+free_userptr:
+	if (rc)
+		hl_userptr_delete_list(hdev, parser->job_userptr_list);
+	return rc;
+}
+
+static int gaudi_parse_cb_no_ext_queue(struct hl_device *hdev,
+					struct hl_cs_parser *parser)
+{
+	struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
+
+	/* For internal queue jobs just check if CB address is valid */
+	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
+					parser->user_cb_size,
+					asic_prop->sram_user_base_address,
+					asic_prop->sram_end_address))
+		return 0;
+
+	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
+					parser->user_cb_size,
+					asic_prop->dram_user_base_address,
+					asic_prop->dram_end_address))
+		return 0;
+
+	/* PMMU and HPMMU addresses are equal, check only one of them */
+	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,
+					parser->user_cb_size,
+					asic_prop->pmmu.start_addr,
+					asic_prop->pmmu.end_addr))
+		return 0;
+
+	dev_err(hdev->dev,
+		"CB address 0x%px + 0x%x for internal QMAN is not valid\n",
+		parser->user_cb, parser->user_cb_size);
+
+	return -EFAULT;
+}
+
+static int gaudi_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (parser->queue_type == QUEUE_TYPE_INT)
+		return gaudi_parse_cb_no_ext_queue(hdev, parser);
+
+	if (gaudi->hw_cap_initialized & HW_CAP_MMU)
+		return gaudi_parse_cb_mmu(hdev, parser);
+	else
+		return gaudi_parse_cb_no_mmu(hdev, parser);
+}
+
+static void gaudi_add_end_of_cb_packets(struct hl_device *hdev,
+					u64 kernel_address, u32 len,
+					u64 cq_addr, u32 cq_val, u32 msi_vec,
+					bool eb)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct packet_msg_prot *cq_pkt;
+	u32 tmp;
+
+	cq_pkt = (struct packet_msg_prot *) (uintptr_t)
+		(kernel_address + len - (sizeof(struct packet_msg_prot) * 2));
+
+	tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+			(1 << GAUDI_PKT_CTL_MB_SHIFT);
+
+	if (eb)
+		tmp |= (1 << GAUDI_PKT_CTL_EB_SHIFT);
+
+	cq_pkt->ctl = cpu_to_le32(tmp);
+	cq_pkt->value = cpu_to_le32(cq_val);
+	cq_pkt->addr = cpu_to_le64(cq_addr);
+
+	cq_pkt++;
+
+	tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+			(1 << GAUDI_PKT_CTL_MB_SHIFT);
+	cq_pkt->ctl = cpu_to_le32(tmp);
+	cq_pkt->value = cpu_to_le32(1);
+
+	if (!gaudi->multi_msi_mode)
+		msi_vec = 0;
+
+	cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_MSI_INTR_0 + msi_vec * 4);
+}
+
+static void gaudi_update_eq_ci(struct hl_device *hdev, u32 val)
+{
+	WREG32(mmCPU_IF_EQ_RD_OFFS, val);
+}
+
+static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,
+					u32 size, u64 val)
+{
+	struct packet_lin_dma *lin_dma_pkt;
+	struct hl_cs_job *job;
+	u32 cb_size, ctl;
+	struct hl_cb *cb;
+	int rc;
+
+	cb = hl_cb_kernel_create(hdev, PAGE_SIZE);
+	if (!cb)
+		return -EFAULT;
+
+	lin_dma_pkt = (struct packet_lin_dma *) (uintptr_t) cb->kernel_address;
+	memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt));
+	cb_size = sizeof(*lin_dma_pkt);
+
+	ctl = ((PACKET_LIN_DMA << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+			(1 << GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT) |
+			(1 << GAUDI_PKT_LIN_DMA_CTL_LIN_SHIFT) |
+			(1 << GAUDI_PKT_CTL_RB_SHIFT) |
+			(1 << GAUDI_PKT_CTL_MB_SHIFT));
+	lin_dma_pkt->ctl = cpu_to_le32(ctl);
+	lin_dma_pkt->src_addr = cpu_to_le64(val);
+	lin_dma_pkt->dst_addr |= cpu_to_le64(addr);
+	lin_dma_pkt->tsize = cpu_to_le32(size);
+
+	job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
+	if (!job) {
+		dev_err(hdev->dev, "Failed to allocate a new job\n");
+		rc = -ENOMEM;
+		goto release_cb;
+	}
+
+	job->id = 0;
+	job->user_cb = cb;
+	job->user_cb->cs_cnt++;
+	job->user_cb_size = cb_size;
+	job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;
+	job->patched_cb = job->user_cb;
+	job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot);
+
+	hl_debugfs_add_job(hdev, job);
+
+	rc = gaudi_send_job_on_qman0(hdev, job);
+
+	hl_debugfs_remove_job(hdev, job);
+	kfree(job);
+	cb->cs_cnt--;
+
+release_cb:
+	hl_cb_put(cb);
+	hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
+
+	return rc;
+}
+
+static void gaudi_restore_sm_registers(struct hl_device *hdev)
+{
+	int i;
+
+	for (i = 0 ; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4) {
+		WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+		WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+		WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+	}
+
+	for (i = 0 ; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4) {
+		WREG32(mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+		WREG32(mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+		WREG32(mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+	}
+
+	i = GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4;
+
+	for (; i < NUM_OF_SOB_IN_BLOCK << 2 ; i += 4)
+		WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + i, 0);
+
+	i = GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4;
+
+	for (; i < NUM_OF_MONITORS_IN_BLOCK << 2 ; i += 4)
+		WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 + i, 0);
+}
+
+static void gaudi_restore_dma_registers(struct hl_device *hdev)
+{
+	u32 sob_delta = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 -
+			mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0;
+	int i;
+
+	for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {
+		u64 sob_addr = CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 +
+				(i * sob_delta);
+		u32 dma_offset = i * DMA_CORE_OFFSET;
+
+		WREG32(mmDMA0_CORE_WR_COMP_ADDR_LO + dma_offset,
+				lower_32_bits(sob_addr));
+		WREG32(mmDMA0_CORE_WR_COMP_ADDR_HI + dma_offset,
+				upper_32_bits(sob_addr));
+		WREG32(mmDMA0_CORE_WR_COMP_WDATA + dma_offset, 0x80000001);
+
+		/* For DMAs 2-7, need to restore WR_AWUSER_31_11 as it can be
+		 * modified by the user for SRAM reduction
+		 */
+		if (i > 1)
+			WREG32(mmDMA0_CORE_WR_AWUSER_31_11 + dma_offset,
+								0x00000001);
+	}
+}
+
+static void gaudi_restore_qm_registers(struct hl_device *hdev)
+{
+	u32 qman_offset;
+	int i;
+
+	for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {
+		qman_offset = i * DMA_QMAN_OFFSET;
+		WREG32(mmDMA0_QM_ARB_CFG_0 + qman_offset, 0);
+	}
+
+	for (i = 0 ; i < MME_NUMBER_OF_MASTER_ENGINES ; i++) {
+		qman_offset = i * (mmMME2_QM_BASE - mmMME0_QM_BASE);
+		WREG32(mmMME0_QM_ARB_CFG_0 + qman_offset, 0);
+	}
+
+	for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+		qman_offset = i * TPC_QMAN_OFFSET;
+		WREG32(mmTPC0_QM_ARB_CFG_0 + qman_offset, 0);
+	}
+}
+
+static void gaudi_restore_user_registers(struct hl_device *hdev)
+{
+	gaudi_restore_sm_registers(hdev);
+	gaudi_restore_dma_registers(hdev);
+	gaudi_restore_qm_registers(hdev);
+}
+
+static int gaudi_context_switch(struct hl_device *hdev, u32 asid)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	u64 addr = prop->sram_user_base_address;
+	u32 size = hdev->pldm ? 0x10000 :
+			(prop->sram_size - SRAM_USER_BASE_OFFSET);
+	u64 val = 0x7777777777777777ull;
+	int rc;
+
+	rc = gaudi_memset_device_memory(hdev, addr, size, val);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to clear SRAM in context switch\n");
+		return rc;
+	}
+
+	gaudi_mmu_prepare(hdev, asid);
+
+	gaudi_restore_user_registers(hdev);
+
+	return 0;
+}
+
+static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 addr = prop->mmu_pgt_addr;
+	u32 size = prop->mmu_pgt_size + MMU_CACHE_MNG_SIZE;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
+		return 0;
+
+	return gaudi_memset_device_memory(hdev, addr, size, 0);
+}
+
+static void gaudi_restore_phase_topology(struct hl_device *hdev)
+{
+
+}
+
+static int gaudi_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 hbm_bar_addr;
+	int rc = 0;
+
+	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
+		if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+			dev_err_ratelimited(hdev->dev,
+				"Can't read register - clock gating is enabled!\n");
+			rc = -EFAULT;
+		} else {
+			*val = RREG32(addr - CFG_BASE);
+		}
+	} else if ((addr >= SRAM_BASE_ADDR) &&
+			(addr < SRAM_BASE_ADDR + SRAM_BAR_SIZE)) {
+		*val = readl(hdev->pcie_bar[SRAM_BAR_ID] +
+				(addr - SRAM_BASE_ADDR));
+	} else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
+		u64 bar_base_addr = DRAM_PHYS_BASE +
+				(addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+		hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+		if (hbm_bar_addr != U64_MAX) {
+			*val = readl(hdev->pcie_bar[HBM_BAR_ID] +
+						(addr - bar_base_addr));
+
+			hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+						hbm_bar_addr);
+		}
+		if (hbm_bar_addr == U64_MAX)
+			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+		*val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE);
+	} else {
+		rc = -EFAULT;
+	}
+
+	return rc;
+}
+
+static int gaudi_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 hbm_bar_addr;
+	int rc = 0;
+
+	if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
+		if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+			dev_err_ratelimited(hdev->dev,
+				"Can't write register - clock gating is enabled!\n");
+			rc = -EFAULT;
+		} else {
+			WREG32(addr - CFG_BASE, val);
+		}
+	} else if ((addr >= SRAM_BASE_ADDR) &&
+			(addr < SRAM_BASE_ADDR + SRAM_BAR_SIZE)) {
+		writel(val, hdev->pcie_bar[SRAM_BAR_ID] +
+					(addr - SRAM_BASE_ADDR));
+	} else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
+		u64 bar_base_addr = DRAM_PHYS_BASE +
+				(addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+		hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+		if (hbm_bar_addr != U64_MAX) {
+			writel(val, hdev->pcie_bar[HBM_BAR_ID] +
+						(addr - bar_base_addr));
+
+			hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+						hbm_bar_addr);
+		}
+		if (hbm_bar_addr == U64_MAX)
+			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+		*(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
+	} else {
+		rc = -EFAULT;
+	}
+
+	return rc;
+}
+
+static int gaudi_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 hbm_bar_addr;
+	int rc = 0;
+
+	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
+		if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+			dev_err_ratelimited(hdev->dev,
+				"Can't read register - clock gating is enabled!\n");
+			rc = -EFAULT;
+		} else {
+			u32 val_l = RREG32(addr - CFG_BASE);
+			u32 val_h = RREG32(addr + sizeof(u32) - CFG_BASE);
+
+			*val = (((u64) val_h) << 32) | val_l;
+		}
+	} else if ((addr >= SRAM_BASE_ADDR) &&
+		   (addr <= SRAM_BASE_ADDR + SRAM_BAR_SIZE - sizeof(u64))) {
+		*val = readq(hdev->pcie_bar[SRAM_BAR_ID] +
+				(addr - SRAM_BASE_ADDR));
+	} else if (addr <=
+		    DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
+		u64 bar_base_addr = DRAM_PHYS_BASE +
+				(addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+		hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+		if (hbm_bar_addr != U64_MAX) {
+			*val = readq(hdev->pcie_bar[HBM_BAR_ID] +
+						(addr - bar_base_addr));
+
+			hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+						hbm_bar_addr);
+		}
+		if (hbm_bar_addr == U64_MAX)
+			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+		*val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE);
+	} else {
+		rc = -EFAULT;
+	}
+
+	return rc;
+}
+
+static int gaudi_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 hbm_bar_addr;
+	int rc = 0;
+
+	if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
+		if (gaudi->hw_cap_initialized & HW_CAP_CLK_GATE) {
+			dev_err_ratelimited(hdev->dev,
+				"Can't write register - clock gating is enabled!\n");
+			rc = -EFAULT;
+		} else {
+			WREG32(addr - CFG_BASE, lower_32_bits(val));
+			WREG32(addr + sizeof(u32) - CFG_BASE,
+				upper_32_bits(val));
+		}
+	} else if ((addr >= SRAM_BASE_ADDR) &&
+		   (addr <= SRAM_BASE_ADDR + SRAM_BAR_SIZE - sizeof(u64))) {
+		writeq(val, hdev->pcie_bar[SRAM_BAR_ID] +
+					(addr - SRAM_BASE_ADDR));
+	} else if (addr <=
+		    DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
+		u64 bar_base_addr = DRAM_PHYS_BASE +
+				(addr & ~(prop->dram_pci_bar_size - 0x1ull));
+
+		hbm_bar_addr = gaudi_set_hbm_bar_base(hdev, bar_base_addr);
+		if (hbm_bar_addr != U64_MAX) {
+			writeq(val, hdev->pcie_bar[HBM_BAR_ID] +
+						(addr - bar_base_addr));
+
+			hbm_bar_addr = gaudi_set_hbm_bar_base(hdev,
+						hbm_bar_addr);
+		}
+		if (hbm_bar_addr == U64_MAX)
+			rc = -EIO;
+	} else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
+		*(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
+	} else {
+		rc = -EFAULT;
+	}
+
+	return rc;
+}
+
+static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (hdev->hard_reset_pending)
+		return U64_MAX;
+
+	return readq(hdev->pcie_bar[HBM_BAR_ID] +
+			(addr - gaudi->hbm_bar_cur_addr));
+}
+
+static void gaudi_write_pte(struct hl_device *hdev, u64 addr, u64 val)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (hdev->hard_reset_pending)
+		return;
+
+	writeq(val, hdev->pcie_bar[HBM_BAR_ID] +
+			(addr - gaudi->hbm_bar_cur_addr));
+}
+
+static void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
+{
+	/* mask to zero the MMBP and ASID bits */
+	WREG32_AND(reg, ~0x7FF);
+	WREG32_OR(reg, asid);
+}
+
+static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
+		return;
+
+	if (asid & ~DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK) {
+		WARN(1, "asid %u is too big\n", asid);
+		return;
+	}
+
+	mutex_lock(&gaudi->clk_gate_mutex);
+
+	hdev->asic_funcs->disable_clock_gating(hdev);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmDMA0_CORE_NON_SECURE_PROPS, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA1_CORE_NON_SECURE_PROPS, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA2_CORE_NON_SECURE_PROPS, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA3_CORE_NON_SECURE_PROPS, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA4_CORE_NON_SECURE_PROPS, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA5_CORE_NON_SECURE_PROPS, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA6_CORE_NON_SECURE_PROPS, asid);
+	gaudi_mmu_prepare_reg(hdev, mmDMA7_CORE_NON_SECURE_PROPS, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_ARUSER_LO, asid);
+	gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_AWUSER_LO, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_4, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_2, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_3, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_4, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER0, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER1, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME0_ACC_WBC, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME1_ACC_WBC, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid);
+	gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid);
+
+	gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER, asid);
+	gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER, asid);
+
+	hdev->asic_funcs->enable_clock_gating(hdev);
+
+	mutex_unlock(&gaudi->clk_gate_mutex);
+}
+
+static int gaudi_send_job_on_qman0(struct hl_device *hdev,
+		struct hl_cs_job *job)
+{
+	struct packet_msg_prot *fence_pkt;
+	u32 *fence_ptr;
+	dma_addr_t fence_dma_addr;
+	struct hl_cb *cb;
+	u32 tmp, timeout, dma_offset;
+	int rc;
+
+	if (hdev->pldm)
+		timeout = GAUDI_PLDM_QMAN0_TIMEOUT_USEC;
+	else
+		timeout = HL_DEVICE_TIMEOUT_USEC;
+
+	if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) {
+		dev_err_ratelimited(hdev->dev,
+			"Can't send driver job on QMAN0 because the device is not idle\n");
+		return -EBUSY;
+	}
+
+	fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
+							&fence_dma_addr);
+	if (!fence_ptr) {
+		dev_err(hdev->dev,
+			"Failed to allocate fence memory for QMAN0\n");
+		return -ENOMEM;
+	}
+
+	cb = job->patched_cb;
+
+	fence_pkt = (struct packet_msg_prot *) (uintptr_t) (cb->kernel_address +
+			job->job_cb_size - sizeof(struct packet_msg_prot));
+
+	tmp = (PACKET_MSG_PROT << GAUDI_PKT_CTL_OPCODE_SHIFT) |
+			(1 << GAUDI_PKT_CTL_EB_SHIFT) |
+			(1 << GAUDI_PKT_CTL_MB_SHIFT);
+	fence_pkt->ctl = cpu_to_le32(tmp);
+	fence_pkt->value = cpu_to_le32(GAUDI_QMAN0_FENCE_VAL);
+	fence_pkt->addr = cpu_to_le64(fence_dma_addr);
+
+	dma_offset = gaudi_dma_assignment[GAUDI_PCI_DMA_1] * DMA_CORE_OFFSET;
+
+	WREG32_OR(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_VAL_SHIFT));
+
+	rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,
+					job->job_cb_size, cb->bus_address);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
+		goto free_fence_ptr;
+	}
+
+	rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
+				(tmp == GAUDI_QMAN0_FENCE_VAL), 1000,
+				timeout, true);
+
+	hl_hw_queue_inc_ci_kernel(hdev, GAUDI_QUEUE_ID_DMA_0_0);
+
+	if (rc == -ETIMEDOUT) {
+		dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
+		goto free_fence_ptr;
+	}
+
+free_fence_ptr:
+	WREG32_AND(mmDMA0_CORE_PROT + dma_offset,
+			~BIT(DMA0_CORE_PROT_VAL_SHIFT));
+
+	hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
+					fence_dma_addr);
+	return rc;
+}
+
+static void gaudi_get_event_desc(u16 event_type, char *desc, size_t size)
+{
+	if (event_type >= GAUDI_EVENT_SIZE)
+		goto event_not_supported;
+
+	if (!gaudi_irq_map_table[event_type].valid)
+		goto event_not_supported;
+
+	snprintf(desc, size, gaudi_irq_map_table[event_type].name);
+
+	return;
+
+event_not_supported:
+	snprintf(desc, size, "N/A");
+}
+
+static const char *gaudi_get_razwi_initiator_dma_name(struct hl_device *hdev,
+							u32 x_y, bool is_write)
+{
+	u32 dma_id[2], dma_offset, err_cause[2], mask, i;
+
+	mask = is_write ? DMA0_CORE_ERR_CAUSE_HBW_WR_ERR_MASK :
+				DMA0_CORE_ERR_CAUSE_HBW_RD_ERR_MASK;
+
+	switch (x_y) {
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:
+		dma_id[0] = 0;
+		dma_id[1] = 2;
+		break;
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:
+		dma_id[0] = 1;
+		dma_id[1] = 3;
+		break;
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:
+		dma_id[0] = 4;
+		dma_id[1] = 6;
+		break;
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:
+		dma_id[0] = 5;
+		dma_id[1] = 7;
+		break;
+	default:
+		goto unknown_initiator;
+	}
+
+	for (i = 0 ; i < 2 ; i++) {
+		dma_offset = dma_id[i] * DMA_CORE_OFFSET;
+		err_cause[i] = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset);
+	}
+
+	switch (x_y) {
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:
+		if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+			return "DMA0";
+		else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+			return "DMA2";
+		else
+			return "DMA0 or DMA2";
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:
+		if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+			return "DMA1";
+		else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+			return "DMA3";
+		else
+			return "DMA1 or DMA3";
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:
+		if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+			return "DMA4";
+		else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+			return "DMA6";
+		else
+			return "DMA4 or DMA6";
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:
+		if ((err_cause[0] & mask) && !(err_cause[1] & mask))
+			return "DMA5";
+		else if (!(err_cause[0] & mask) && (err_cause[1] & mask))
+			return "DMA7";
+		else
+			return "DMA5 or DMA7";
+	}
+
+unknown_initiator:
+	return "unknown initiator";
+}
+
+static const char *gaudi_get_razwi_initiator_name(struct hl_device *hdev,
+							bool is_write)
+{
+	u32 val, x_y, axi_id;
+
+	val = is_write ? RREG32(mmMMU_UP_RAZWI_WRITE_ID) :
+				RREG32(mmMMU_UP_RAZWI_READ_ID);
+	x_y = val & ((RAZWI_INITIATOR_Y_MASK << RAZWI_INITIATOR_Y_SHIFT) |
+			(RAZWI_INITIATOR_X_MASK << RAZWI_INITIATOR_X_SHIFT));
+	axi_id = val & (RAZWI_INITIATOR_AXI_ID_MASK <<
+			RAZWI_INITIATOR_AXI_ID_SHIFT);
+
+	switch (x_y) {
+	case RAZWI_INITIATOR_ID_X_Y_TPC0_NIC0:
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+			return "TPC0";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC))
+			return "NIC0";
+		break;
+	case RAZWI_INITIATOR_ID_X_Y_TPC1:
+		return "TPC1";
+	case RAZWI_INITIATOR_ID_X_Y_MME0_0:
+	case RAZWI_INITIATOR_ID_X_Y_MME0_1:
+		return "MME0";
+	case RAZWI_INITIATOR_ID_X_Y_MME1_0:
+	case RAZWI_INITIATOR_ID_X_Y_MME1_1:
+		return "MME1";
+	case RAZWI_INITIATOR_ID_X_Y_TPC2:
+		return "TPC2";
+	case RAZWI_INITIATOR_ID_X_Y_TPC3_PCI_CPU_PSOC:
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+			return "TPC3";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PCI))
+			return "PCI";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_CPU))
+			return "CPU";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PSOC))
+			return "PSOC";
+		break;
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:
+	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:
+		return gaudi_get_razwi_initiator_dma_name(hdev, x_y, is_write);
+	case RAZWI_INITIATOR_ID_X_Y_TPC4_NIC1_NIC2:
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+			return "TPC4";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC))
+			return "NIC1";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT))
+			return "NIC2";
+		break;
+	case RAZWI_INITIATOR_ID_X_Y_TPC5:
+		return "TPC5";
+	case RAZWI_INITIATOR_ID_X_Y_MME2_0:
+	case RAZWI_INITIATOR_ID_X_Y_MME2_1:
+		return "MME2";
+	case RAZWI_INITIATOR_ID_X_Y_MME3_0:
+	case RAZWI_INITIATOR_ID_X_Y_MME3_1:
+		return "MME3";
+	case RAZWI_INITIATOR_ID_X_Y_TPC6:
+		return "TPC6";
+	case RAZWI_INITIATOR_ID_X_Y_TPC7_NIC4_NIC5:
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC))
+			return "TPC7";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC))
+			return "NIC4";
+		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT))
+			return "NIC5";
+		break;
+	default:
+		break;
+	}
+
+	dev_err(hdev->dev,
+		"Unknown RAZWI initiator ID 0x%x [Y=%d, X=%d, AXI_ID=%d]\n",
+		val,
+		(val >> RAZWI_INITIATOR_Y_SHIFT) & RAZWI_INITIATOR_Y_MASK,
+		(val >> RAZWI_INITIATOR_X_SHIFT) & RAZWI_INITIATOR_X_MASK,
+		(val >> RAZWI_INITIATOR_AXI_ID_SHIFT) &
+			RAZWI_INITIATOR_AXI_ID_MASK);
+
+	return "unknown initiator";
+}
+
+static void gaudi_print_razwi_info(struct hl_device *hdev)
+{
+	if (RREG32(mmMMU_UP_RAZWI_WRITE_VLD)) {
+		dev_err_ratelimited(hdev->dev,
+			"RAZWI event caused by illegal write of %s\n",
+			gaudi_get_razwi_initiator_name(hdev, true));
+		WREG32(mmMMU_UP_RAZWI_WRITE_VLD, 0);
+	}
+
+	if (RREG32(mmMMU_UP_RAZWI_READ_VLD)) {
+		dev_err_ratelimited(hdev->dev,
+			"RAZWI event caused by illegal read of %s\n",
+			gaudi_get_razwi_initiator_name(hdev, false));
+		WREG32(mmMMU_UP_RAZWI_READ_VLD, 0);
+	}
+}
+
+static void gaudi_print_mmu_error_info(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 addr;
+	u32 val;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
+		return;
+
+	val = RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE);
+	if (val & MMU_UP_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) {
+		addr = val & MMU_UP_PAGE_ERROR_CAPTURE_VA_49_32_MASK;
+		addr <<= 32;
+		addr |= RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE_VA);
+
+		dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n",
+					addr);
+
+		WREG32(mmMMU_UP_PAGE_ERROR_CAPTURE, 0);
+	}
+
+	val = RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE);
+	if (val & MMU_UP_ACCESS_ERROR_CAPTURE_ENTRY_VALID_MASK) {
+		addr = val & MMU_UP_ACCESS_ERROR_CAPTURE_VA_49_32_MASK;
+		addr <<= 32;
+		addr |= RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE_VA);
+
+		dev_err_ratelimited(hdev->dev,
+				"MMU access error on va 0x%llx\n", addr);
+
+		WREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE, 0);
+	}
+}
+
+/*
+ *  +-------------------+------------------------------------------------------+
+ *  | Configuration Reg |                     Description                      |
+ *  |      Address      |                                                      |
+ *  +-------------------+------------------------------------------------------+
+ *  |  0xF30 - 0xF3F    |ECC single error indication (1 bit per memory wrapper)|
+ *  |                   |0xF30 memory wrappers 31:0 (MSB to LSB)               |
+ *  |                   |0xF34 memory wrappers 63:32                           |
+ *  |                   |0xF38 memory wrappers 95:64                           |
+ *  |                   |0xF3C memory wrappers 127:96                          |
+ *  +-------------------+------------------------------------------------------+
+ *  |  0xF40 - 0xF4F    |ECC double error indication (1 bit per memory wrapper)|
+ *  |                   |0xF40 memory wrappers 31:0 (MSB to LSB)               |
+ *  |                   |0xF44 memory wrappers 63:32                           |
+ *  |                   |0xF48 memory wrappers 95:64                           |
+ *  |                   |0xF4C memory wrappers 127:96                          |
+ *  +-------------------+------------------------------------------------------+
+ */
+static void gaudi_print_ecc_info_generic(struct hl_device *hdev,
+					const char *block_name,
+					u64 block_address, int num_memories,
+					bool derr, bool disable_clock_gating)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	int num_mem_regs = num_memories / 32 + ((num_memories % 32) ? 1 : 0);
+
+	if (block_address >= CFG_BASE)
+		block_address -= CFG_BASE;
+
+	if (derr)
+		block_address += GAUDI_ECC_DERR0_OFFSET;
+	else
+		block_address += GAUDI_ECC_SERR0_OFFSET;
+
+	if (disable_clock_gating) {
+		mutex_lock(&gaudi->clk_gate_mutex);
+		hdev->asic_funcs->disable_clock_gating(hdev);
+	}
+
+	switch (num_mem_regs) {
+	case 1:
+		dev_err(hdev->dev,
+			"%s ECC indication: 0x%08x\n",
+			block_name, RREG32(block_address));
+		break;
+	case 2:
+		dev_err(hdev->dev,
+			"%s ECC indication: 0x%08x 0x%08x\n",
+			block_name,
+			RREG32(block_address), RREG32(block_address + 4));
+		break;
+	case 3:
+		dev_err(hdev->dev,
+			"%s ECC indication: 0x%08x 0x%08x 0x%08x\n",
+			block_name,
+			RREG32(block_address), RREG32(block_address + 4),
+			RREG32(block_address + 8));
+		break;
+	case 4:
+		dev_err(hdev->dev,
+			"%s ECC indication: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+			block_name,
+			RREG32(block_address), RREG32(block_address + 4),
+			RREG32(block_address + 8), RREG32(block_address + 0xc));
+		break;
+	default:
+		break;
+
+	}
+
+	if (disable_clock_gating) {
+		hdev->asic_funcs->enable_clock_gating(hdev);
+		mutex_unlock(&gaudi->clk_gate_mutex);
+	}
+}
+
+static void gaudi_handle_qman_err_generic(struct hl_device *hdev,
+					  const char *qm_name,
+					  u64 glbl_sts_addr,
+					  u64 arb_err_addr)
+{
+	u32 i, j, glbl_sts_val, arb_err_val, glbl_sts_clr_val;
+	char reg_desc[32];
+
+	/* Iterate through all stream GLBL_STS1 registers + Lower CP */
+	for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {
+		glbl_sts_clr_val = 0;
+		glbl_sts_val = RREG32(glbl_sts_addr + 4 * i);
+
+		if (!glbl_sts_val)
+			continue;
+
+		if (i == QMAN_STREAMS)
+			snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerCP");
+		else
+			snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i);
+
+		for (j = 0 ; j < GAUDI_NUM_OF_QM_ERR_CAUSE ; j++) {
+			if (glbl_sts_val & BIT(j)) {
+				dev_err_ratelimited(hdev->dev,
+						"%s %s. err cause: %s\n",
+						qm_name, reg_desc,
+						gaudi_qman_error_cause[j]);
+				glbl_sts_clr_val |= BIT(j);
+			}
+		}
+
+		/* Write 1 clear errors */
+		WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val);
+	}
+
+	arb_err_val = RREG32(arb_err_addr);
+
+	if (!arb_err_val)
+		return;
+
+	for (j = 0 ; j < GAUDI_NUM_OF_QM_ARB_ERR_CAUSE ; j++) {
+		if (arb_err_val & BIT(j)) {
+			dev_err_ratelimited(hdev->dev,
+					"%s ARB_ERR. err cause: %s\n",
+					qm_name,
+					gaudi_qman_arb_error_cause[j]);
+		}
+	}
+}
+
+static void gaudi_print_ecc_info(struct hl_device *hdev, u16 event_type)
+{
+	u64 block_address;
+	u8 index;
+	int num_memories;
+	char desc[32];
+	bool derr;
+	bool disable_clock_gating;
+
+	switch (event_type) {
+	case GAUDI_EVENT_PCIE_CORE_SERR:
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_CORE");
+		block_address = mmPCIE_CORE_BASE;
+		num_memories = 51;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PCIE_CORE_DERR:
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_CORE");
+		block_address = mmPCIE_CORE_BASE;
+		num_memories = 51;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PCIE_IF_SERR:
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_WRAP");
+		block_address = mmPCIE_WRAP_BASE;
+		num_memories = 11;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PCIE_IF_DERR:
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_WRAP");
+		block_address = mmPCIE_WRAP_BASE;
+		num_memories = 11;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PCIE_PHY_SERR:
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_PHY");
+		block_address = mmPCIE_PHY_BASE;
+		num_memories = 4;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PCIE_PHY_DERR:
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "PCIE_PHY");
+		block_address = mmPCIE_PHY_BASE;
+		num_memories = 4;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR:
+		index = event_type - GAUDI_EVENT_TPC0_SERR;
+		block_address = mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC", index);
+		num_memories = 90;
+		derr = false;
+		disable_clock_gating = true;
+		break;
+	case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR:
+		index = event_type - GAUDI_EVENT_TPC0_DERR;
+		block_address =
+			mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC", index);
+		num_memories = 90;
+		derr = true;
+		disable_clock_gating = true;
+		break;
+	case GAUDI_EVENT_MME0_ACC_SERR:
+	case GAUDI_EVENT_MME1_ACC_SERR:
+	case GAUDI_EVENT_MME2_ACC_SERR:
+	case GAUDI_EVENT_MME3_ACC_SERR:
+		index = (event_type - GAUDI_EVENT_MME0_ACC_SERR) / 4;
+		block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "MME%d_ACC", index);
+		num_memories = 128;
+		derr = false;
+		disable_clock_gating = true;
+		break;
+	case GAUDI_EVENT_MME0_ACC_DERR:
+	case GAUDI_EVENT_MME1_ACC_DERR:
+	case GAUDI_EVENT_MME2_ACC_DERR:
+	case GAUDI_EVENT_MME3_ACC_DERR:
+		index = (event_type - GAUDI_EVENT_MME0_ACC_DERR) / 4;
+		block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "MME%d_ACC", index);
+		num_memories = 128;
+		derr = true;
+		disable_clock_gating = true;
+		break;
+	case GAUDI_EVENT_MME0_SBAB_SERR:
+	case GAUDI_EVENT_MME1_SBAB_SERR:
+	case GAUDI_EVENT_MME2_SBAB_SERR:
+	case GAUDI_EVENT_MME3_SBAB_SERR:
+		index = (event_type - GAUDI_EVENT_MME0_SBAB_SERR) / 4;
+		block_address = mmMME0_SBAB_BASE + index * MME_ACC_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "MME%d_SBAB", index);
+		num_memories = 33;
+		derr = false;
+		disable_clock_gating = true;
+		break;
+	case GAUDI_EVENT_MME0_SBAB_DERR:
+	case GAUDI_EVENT_MME1_SBAB_DERR:
+	case GAUDI_EVENT_MME2_SBAB_DERR:
+	case GAUDI_EVENT_MME3_SBAB_DERR:
+		index = (event_type - GAUDI_EVENT_MME0_SBAB_DERR) / 4;
+		block_address = mmMME0_SBAB_BASE + index * MME_ACC_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "MME%d_SBAB", index);
+		num_memories = 33;
+		derr = true;
+		disable_clock_gating = true;
+		break;
+	case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_DMA7_SERR_ECC:
+		index = event_type - GAUDI_EVENT_DMA0_SERR_ECC;
+		block_address = mmDMA0_CORE_BASE + index * DMA_CORE_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "DMA%d_CORE", index);
+		num_memories = 16;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_DMA0_DERR_ECC ... GAUDI_EVENT_DMA7_DERR_ECC:
+		index = event_type - GAUDI_EVENT_DMA0_DERR_ECC;
+		block_address = mmDMA0_CORE_BASE + index * DMA_CORE_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "DMA%d_CORE", index);
+		num_memories = 16;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_CPU_IF_ECC_SERR:
+		block_address = mmCPU_IF_BASE;
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+		num_memories = 4;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_CPU_IF_ECC_DERR:
+		block_address = mmCPU_IF_BASE;
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+		num_memories = 4;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PSOC_MEM_SERR:
+		block_address = mmPSOC_GLOBAL_CONF_BASE;
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+		num_memories = 4;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PSOC_MEM_DERR:
+		block_address = mmPSOC_GLOBAL_CONF_BASE;
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+		num_memories = 4;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PSOC_CORESIGHT_SERR:
+		block_address = mmPSOC_CS_TRACE_BASE;
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+		num_memories = 2;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_PSOC_CORESIGHT_DERR:
+		block_address = mmPSOC_CS_TRACE_BASE;
+		snprintf(desc, ARRAY_SIZE(desc), "%s", "CPU");
+		num_memories = 2;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_SRAM0_SERR ... GAUDI_EVENT_SRAM28_SERR:
+		index = event_type - GAUDI_EVENT_SRAM0_SERR;
+		block_address =
+			mmSRAM_Y0_X0_BANK_BASE + index * SRAM_BANK_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "SRAM%d", index);
+		num_memories = 2;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_SRAM0_DERR ... GAUDI_EVENT_SRAM28_DERR:
+		index = event_type - GAUDI_EVENT_SRAM0_DERR;
+		block_address =
+			mmSRAM_Y0_X0_BANK_BASE + index * SRAM_BANK_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "SRAM%d", index);
+		num_memories = 2;
+		derr = true;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_DMA_IF0_SERR ... GAUDI_EVENT_DMA_IF3_SERR:
+		index = event_type - GAUDI_EVENT_DMA_IF0_SERR;
+		block_address = mmDMA_IF_W_S_BASE +
+				index * (mmDMA_IF_E_S_BASE - mmDMA_IF_W_S_BASE);
+		snprintf(desc, ARRAY_SIZE(desc), "DMA_IF%d", index);
+		num_memories = 60;
+		derr = false;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR:
+		index = event_type - GAUDI_EVENT_DMA_IF0_DERR;
+		block_address = mmDMA_IF_W_S_BASE +
+				index * (mmDMA_IF_E_S_BASE - mmDMA_IF_W_S_BASE);
+		snprintf(desc, ARRAY_SIZE(desc), "DMA_IF%d", index);
+		derr = true;
+		num_memories = 60;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_HBM_0_SERR ... GAUDI_EVENT_HBM_3_SERR:
+		index = event_type - GAUDI_EVENT_HBM_0_SERR;
+		/* HBM Registers are at different offsets */
+		block_address = mmHBM0_BASE + 0x8000 +
+				index * (mmHBM1_BASE - mmHBM0_BASE);
+		snprintf(desc, ARRAY_SIZE(desc), "HBM%d", index);
+		derr = false;
+		num_memories = 64;
+		disable_clock_gating = false;
+		break;
+	case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR:
+		index = event_type - GAUDI_EVENT_HBM_0_SERR;
+		/* HBM Registers are at different offsets */
+		block_address = mmHBM0_BASE + 0x8000 +
+				index * (mmHBM1_BASE - mmHBM0_BASE);
+		snprintf(desc, ARRAY_SIZE(desc), "HBM%d", index);
+		derr = true;
+		num_memories = 64;
+		disable_clock_gating = false;
+		break;
+	default:
+		return;
+	}
+
+	gaudi_print_ecc_info_generic(hdev, desc, block_address, num_memories,
+					derr, disable_clock_gating);
+}
+
+static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type)
+{
+	u64 glbl_sts_addr, arb_err_addr;
+	u8 index;
+	char desc[32];
+
+	switch (event_type) {
+	case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM:
+		index = event_type - GAUDI_EVENT_TPC0_QM;
+		glbl_sts_addr =
+			mmTPC0_QM_GLBL_STS1_0 + index * TPC_QMAN_OFFSET;
+		arb_err_addr =
+			mmTPC0_QM_ARB_ERR_CAUSE + index * TPC_QMAN_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC_QM", index);
+		break;
+	case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM:
+		index = event_type - GAUDI_EVENT_MME0_QM;
+		glbl_sts_addr =
+			mmMME0_QM_GLBL_STS1_0 + index * MME_QMAN_OFFSET;
+		arb_err_addr =
+			mmMME0_QM_ARB_ERR_CAUSE + index * MME_QMAN_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "MME_QM", index);
+		break;
+	case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM:
+		index = event_type - GAUDI_EVENT_DMA0_QM;
+		glbl_sts_addr =
+			mmDMA0_QM_GLBL_STS1_0 + index * DMA_QMAN_OFFSET;
+		arb_err_addr =
+			mmDMA0_QM_ARB_ERR_CAUSE + index * DMA_QMAN_OFFSET;
+		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "DMA_QM", index);
+		break;
+	default:
+		return;
+	}
+
+	gaudi_handle_qman_err_generic(hdev, desc, glbl_sts_addr, arb_err_addr);
+}
+
+static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
+					bool razwi)
+{
+	char desc[64] = "";
+
+	gaudi_get_event_desc(event_type, desc, sizeof(desc));
+	dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
+		event_type, desc);
+
+	gaudi_print_ecc_info(hdev, event_type);
+
+	if (razwi) {
+		gaudi_print_razwi_info(hdev);
+		gaudi_print_mmu_error_info(hdev);
+	}
+}
+
+static int gaudi_soft_reset_late_init(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	/* Unmask all IRQs since some could have been received
+	 * during the soft reset
+	 */
+	return hl_fw_unmask_irq_arr(hdev, gaudi->events, sizeof(gaudi->events));
+}
+
+static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device)
+{
+	int ch, err = 0;
+	u32 base, val, val2;
+
+	base = GAUDI_HBM_CFG_BASE + device * GAUDI_HBM_CFG_OFFSET;
+	for (ch = 0 ; ch < GAUDI_HBM_CHANNELS ; ch++) {
+		val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF);
+		val = (val & 0xFF) | ((val >> 8) & 0xFF);
+		if (val) {
+			err = 1;
+			dev_err(hdev->dev,
+				"HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",
+				device, ch * 2, val & 0x1, (val >> 1) & 0x1,
+				(val >> 2) & 0x1, (val >> 3) & 0x1,
+				(val >> 4) & 0x1);
+
+			val2 = RREG32(base + ch * 0x1000 + 0x060);
+			dev_err(hdev->dev,
+				"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DED_CNT=%d\n",
+				device, ch * 2,
+				RREG32(base + ch * 0x1000 + 0x064),
+				(val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10,
+				(val2 & 0xFF0000) >> 16,
+				(val2 & 0xFF000000) >> 24);
+		}
+
+		val = RREG32_MASK(base + ch * 0x1000 + 0x07C, 0x0000FFFF);
+		val = (val & 0xFF) | ((val >> 8) & 0xFF);
+		if (val) {
+			err = 1;
+			dev_err(hdev->dev,
+				"HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",
+				device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1,
+				(val >> 2) & 0x1, (val >> 3) & 0x1,
+				(val >> 4) & 0x1);
+
+			val2 = RREG32(base + ch * 0x1000 + 0x070);
+			dev_err(hdev->dev,
+				"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DED_CNT=%d\n",
+				device, ch * 2 + 1,
+				RREG32(base + ch * 0x1000 + 0x074),
+				(val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10,
+				(val2 & 0xFF0000) >> 16,
+				(val2 & 0xFF000000) >> 24);
+		}
+
+		/* Clear interrupts */
+		RMWREG32(base + (ch * 0x1000) + 0x060, 0x1C8, 0x1FF);
+		RMWREG32(base + (ch * 0x1000) + 0x070, 0x1C8, 0x1FF);
+		WREG32(base + (ch * 0x1000) + 0x06C, 0x1F1F);
+		WREG32(base + (ch * 0x1000) + 0x07C, 0x1F1F);
+		RMWREG32(base + (ch * 0x1000) + 0x060, 0x0, 0xF);
+		RMWREG32(base + (ch * 0x1000) + 0x070, 0x0, 0xF);
+	}
+
+	val  = RREG32(base + 0x8F30);
+	val2 = RREG32(base + 0x8F34);
+	if (val | val2) {
+		err = 1;
+		dev_err(hdev->dev,
+			"HBM %d MC SRAM SERR info: Reg 0x8F30=0x%x, Reg 0x8F34=0x%x\n",
+			device, val, val2);
+	}
+	val  = RREG32(base + 0x8F40);
+	val2 = RREG32(base + 0x8F44);
+	if (val | val2) {
+		err = 1;
+		dev_err(hdev->dev,
+			"HBM %d MC SRAM DERR info: Reg 0x8F40=0x%x, Reg 0x8F44=0x%x\n",
+			device, val, val2);
+	}
+
+	return err;
+}
+
+static int gaudi_hbm_event_to_dev(u16 hbm_event_type)
+{
+	switch (hbm_event_type) {
+	case GAUDI_EVENT_HBM0_SPI_0:
+	case GAUDI_EVENT_HBM0_SPI_1:
+		return 0;
+	case GAUDI_EVENT_HBM1_SPI_0:
+	case GAUDI_EVENT_HBM1_SPI_1:
+		return 1;
+	case GAUDI_EVENT_HBM2_SPI_0:
+	case GAUDI_EVENT_HBM2_SPI_1:
+		return 2;
+	case GAUDI_EVENT_HBM3_SPI_0:
+	case GAUDI_EVENT_HBM3_SPI_1:
+		return 3;
+	default:
+		break;
+	}
+
+	/* Should never happen */
+	return 0;
+}
+
+static bool gaudi_tpc_read_interrupts(struct hl_device *hdev, u8 tpc_id,
+					char *interrupt_name)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 tpc_offset = tpc_id * TPC_CFG_OFFSET, tpc_interrupts_cause, i;
+	bool soft_reset_required = false;
+
+	/* Accessing the TPC_INTR_CAUSE registers requires disabling the clock
+	 * gating, and thus cannot be done in ArmCP and should be done instead
+	 * by the driver.
+	 */
+
+	mutex_lock(&gaudi->clk_gate_mutex);
+
+	hdev->asic_funcs->disable_clock_gating(hdev);
+
+	tpc_interrupts_cause = RREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset) &
+				TPC0_CFG_TPC_INTR_CAUSE_CAUSE_MASK;
+
+	for (i = 0 ; i < GAUDI_NUM_OF_TPC_INTR_CAUSE ; i++)
+		if (tpc_interrupts_cause & BIT(i)) {
+			dev_err_ratelimited(hdev->dev,
+					"TPC%d_%s interrupt cause: %s\n",
+					tpc_id, interrupt_name,
+					gaudi_tpc_interrupts_cause[i]);
+			/* If this is QM error, we need to soft-reset */
+			if (i == 15)
+				soft_reset_required = true;
+		}
+
+	/* Clear interrupts */
+	WREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset, 0);
+
+	hdev->asic_funcs->enable_clock_gating(hdev);
+
+	mutex_unlock(&gaudi->clk_gate_mutex);
+
+	return soft_reset_required;
+}
+
+static int tpc_dec_event_to_tpc_id(u16 tpc_dec_event_type)
+{
+	return (tpc_dec_event_type - GAUDI_EVENT_TPC0_DEC) >> 1;
+}
+
+static int tpc_krn_event_to_tpc_id(u16 tpc_dec_event_type)
+{
+	return (tpc_dec_event_type - GAUDI_EVENT_TPC0_KRN_ERR) / 6;
+}
+
+static void gaudi_print_clk_change_info(struct hl_device *hdev,
+					u16 event_type)
+{
+	switch (event_type) {
+	case GAUDI_EVENT_FIX_POWER_ENV_S:
+		dev_info_ratelimited(hdev->dev,
+			"Clock throttling due to power consumption\n");
+		break;
+
+	case GAUDI_EVENT_FIX_POWER_ENV_E:
+		dev_info_ratelimited(hdev->dev,
+			"Power envelop is safe, back to optimal clock\n");
+		break;
+
+	case GAUDI_EVENT_FIX_THERMAL_ENV_S:
+		dev_info_ratelimited(hdev->dev,
+			"Clock throttling due to overheating\n");
+		break;
+
+	case GAUDI_EVENT_FIX_THERMAL_ENV_E:
+		dev_info_ratelimited(hdev->dev,
+			"Thermal envelop is safe, back to optimal clock\n");
+		break;
+
+	default:
+		dev_err(hdev->dev, "Received invalid clock change event %d\n",
+			event_type);
+		break;
+	}
+}
+
+static void gaudi_handle_eqe(struct hl_device *hdev,
+				struct hl_eq_entry *eq_entry)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
+	u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)
+			>> EQ_CTL_EVENT_TYPE_SHIFT);
+	u8 cause;
+	bool reset_required;
+
+	gaudi->events_stat[event_type]++;
+	gaudi->events_stat_aggregate[event_type]++;
+
+	switch (event_type) {
+	case GAUDI_EVENT_PCIE_CORE_DERR:
+	case GAUDI_EVENT_PCIE_IF_DERR:
+	case GAUDI_EVENT_PCIE_PHY_DERR:
+	case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR:
+	case GAUDI_EVENT_MME0_ACC_DERR:
+	case GAUDI_EVENT_MME0_SBAB_DERR:
+	case GAUDI_EVENT_MME1_ACC_DERR:
+	case GAUDI_EVENT_MME1_SBAB_DERR:
+	case GAUDI_EVENT_MME2_ACC_DERR:
+	case GAUDI_EVENT_MME2_SBAB_DERR:
+	case GAUDI_EVENT_MME3_ACC_DERR:
+	case GAUDI_EVENT_MME3_SBAB_DERR:
+	case GAUDI_EVENT_DMA0_DERR_ECC ... GAUDI_EVENT_DMA7_DERR_ECC:
+		fallthrough;
+	case GAUDI_EVENT_CPU_IF_ECC_DERR:
+	case GAUDI_EVENT_PSOC_MEM_DERR:
+	case GAUDI_EVENT_PSOC_CORESIGHT_DERR:
+	case GAUDI_EVENT_SRAM0_DERR ... GAUDI_EVENT_SRAM28_DERR:
+	case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR:
+		fallthrough;
+	case GAUDI_EVENT_GIC500:
+	case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR:
+	case GAUDI_EVENT_MMU_DERR:
+	case GAUDI_EVENT_AXI_ECC:
+	case GAUDI_EVENT_L2_RAM_ECC:
+	case GAUDI_EVENT_PLL0 ... GAUDI_EVENT_PLL17:
+		gaudi_print_irq_info(hdev, event_type, false);
+		if (hdev->hard_reset_on_fw_events)
+			hl_device_reset(hdev, true, false);
+		break;
+
+	case GAUDI_EVENT_HBM0_SPI_0:
+	case GAUDI_EVENT_HBM1_SPI_0:
+	case GAUDI_EVENT_HBM2_SPI_0:
+	case GAUDI_EVENT_HBM3_SPI_0:
+		gaudi_print_irq_info(hdev, event_type, false);
+		gaudi_hbm_read_interrupts(hdev,
+					  gaudi_hbm_event_to_dev(event_type));
+		if (hdev->hard_reset_on_fw_events)
+			hl_device_reset(hdev, true, false);
+		break;
+
+	case GAUDI_EVENT_HBM0_SPI_1:
+	case GAUDI_EVENT_HBM1_SPI_1:
+	case GAUDI_EVENT_HBM2_SPI_1:
+	case GAUDI_EVENT_HBM3_SPI_1:
+		gaudi_print_irq_info(hdev, event_type, false);
+		gaudi_hbm_read_interrupts(hdev,
+					  gaudi_hbm_event_to_dev(event_type));
+		break;
+
+	case GAUDI_EVENT_TPC0_DEC:
+	case GAUDI_EVENT_TPC1_DEC:
+	case GAUDI_EVENT_TPC2_DEC:
+	case GAUDI_EVENT_TPC3_DEC:
+	case GAUDI_EVENT_TPC4_DEC:
+	case GAUDI_EVENT_TPC5_DEC:
+	case GAUDI_EVENT_TPC6_DEC:
+	case GAUDI_EVENT_TPC7_DEC:
+		gaudi_print_irq_info(hdev, event_type, true);
+		reset_required = gaudi_tpc_read_interrupts(hdev,
+					tpc_dec_event_to_tpc_id(event_type),
+					"AXI_SLV_DEC_Error");
+		if (reset_required) {
+			dev_err(hdev->dev, "hard reset required due to %s\n",
+				gaudi_irq_map_table[event_type].name);
+
+			if (hdev->hard_reset_on_fw_events)
+				hl_device_reset(hdev, true, false);
+		} else {
+			hl_fw_unmask_irq(hdev, event_type);
+		}
+		break;
+
+	case GAUDI_EVENT_TPC0_KRN_ERR:
+	case GAUDI_EVENT_TPC1_KRN_ERR:
+	case GAUDI_EVENT_TPC2_KRN_ERR:
+	case GAUDI_EVENT_TPC3_KRN_ERR:
+	case GAUDI_EVENT_TPC4_KRN_ERR:
+	case GAUDI_EVENT_TPC5_KRN_ERR:
+	case GAUDI_EVENT_TPC6_KRN_ERR:
+	case GAUDI_EVENT_TPC7_KRN_ERR:
+		gaudi_print_irq_info(hdev, event_type, true);
+		reset_required = gaudi_tpc_read_interrupts(hdev,
+					tpc_krn_event_to_tpc_id(event_type),
+					"KRN_ERR");
+		if (reset_required) {
+			dev_err(hdev->dev, "hard reset required due to %s\n",
+				gaudi_irq_map_table[event_type].name);
+
+			if (hdev->hard_reset_on_fw_events)
+				hl_device_reset(hdev, true, false);
+		} else {
+			hl_fw_unmask_irq(hdev, event_type);
+		}
+		break;
+
+	case GAUDI_EVENT_PCIE_CORE_SERR:
+	case GAUDI_EVENT_PCIE_IF_SERR:
+	case GAUDI_EVENT_PCIE_PHY_SERR:
+	case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR:
+	case GAUDI_EVENT_MME0_ACC_SERR:
+	case GAUDI_EVENT_MME0_SBAB_SERR:
+	case GAUDI_EVENT_MME1_ACC_SERR:
+	case GAUDI_EVENT_MME1_SBAB_SERR:
+	case GAUDI_EVENT_MME2_ACC_SERR:
+	case GAUDI_EVENT_MME2_SBAB_SERR:
+	case GAUDI_EVENT_MME3_ACC_SERR:
+	case GAUDI_EVENT_MME3_SBAB_SERR:
+	case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_DMA7_SERR_ECC:
+	case GAUDI_EVENT_CPU_IF_ECC_SERR:
+	case GAUDI_EVENT_PSOC_MEM_SERR:
+	case GAUDI_EVENT_PSOC_CORESIGHT_SERR:
+	case GAUDI_EVENT_SRAM0_SERR ... GAUDI_EVENT_SRAM28_SERR:
+	case GAUDI_EVENT_DMA_IF0_SERR ... GAUDI_EVENT_DMA_IF3_SERR:
+	case GAUDI_EVENT_HBM_0_SERR ... GAUDI_EVENT_HBM_3_SERR:
+		fallthrough;
+	case GAUDI_EVENT_MMU_SERR:
+	case GAUDI_EVENT_PCIE_DEC:
+	case GAUDI_EVENT_MME0_WBC_RSP:
+	case GAUDI_EVENT_MME0_SBAB0_RSP:
+	case GAUDI_EVENT_MME1_WBC_RSP:
+	case GAUDI_EVENT_MME1_SBAB0_RSP:
+	case GAUDI_EVENT_MME2_WBC_RSP:
+	case GAUDI_EVENT_MME2_SBAB0_RSP:
+	case GAUDI_EVENT_MME3_WBC_RSP:
+	case GAUDI_EVENT_MME3_SBAB0_RSP:
+	case GAUDI_EVENT_CPU_AXI_SPLITTER:
+	case GAUDI_EVENT_PSOC_AXI_DEC:
+	case GAUDI_EVENT_PSOC_PRSTN_FALL:
+	case GAUDI_EVENT_MMU_PAGE_FAULT:
+	case GAUDI_EVENT_MMU_WR_PERM:
+	case GAUDI_EVENT_RAZWI_OR_ADC:
+	case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM:
+	case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM:
+	case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM:
+		fallthrough;
+	case GAUDI_EVENT_DMA0_CORE ... GAUDI_EVENT_DMA7_CORE:
+		gaudi_print_irq_info(hdev, event_type, true);
+		gaudi_handle_qman_err(hdev, event_type);
+		hl_fw_unmask_irq(hdev, event_type);
+		break;
+
+	case GAUDI_EVENT_RAZWI_OR_ADC_SW:
+		gaudi_print_irq_info(hdev, event_type, true);
+		if (hdev->hard_reset_on_fw_events)
+			hl_device_reset(hdev, true, false);
+		break;
+
+	case GAUDI_EVENT_TPC0_BMON_SPMU:
+	case GAUDI_EVENT_TPC1_BMON_SPMU:
+	case GAUDI_EVENT_TPC2_BMON_SPMU:
+	case GAUDI_EVENT_TPC3_BMON_SPMU:
+	case GAUDI_EVENT_TPC4_BMON_SPMU:
+	case GAUDI_EVENT_TPC5_BMON_SPMU:
+	case GAUDI_EVENT_TPC6_BMON_SPMU:
+	case GAUDI_EVENT_TPC7_BMON_SPMU:
+	case GAUDI_EVENT_DMA_BM_CH0 ... GAUDI_EVENT_DMA_BM_CH7:
+		gaudi_print_irq_info(hdev, event_type, false);
+		hl_fw_unmask_irq(hdev, event_type);
+		break;
+
+	case GAUDI_EVENT_FIX_POWER_ENV_S ... GAUDI_EVENT_FIX_THERMAL_ENV_E:
+		gaudi_print_clk_change_info(hdev, event_type);
+		hl_fw_unmask_irq(hdev, event_type);
+		break;
+
+	case GAUDI_EVENT_PSOC_GPIO_U16_0:
+		cause = le64_to_cpu(eq_entry->data[0]) & 0xFF;
+		dev_err(hdev->dev,
+			"Received high temp H/W interrupt %d (cause %d)\n",
+			event_type, cause);
+		break;
+
+	default:
+		dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
+				event_type);
+		break;
+	}
+}
+
+static void *gaudi_get_events_stat(struct hl_device *hdev, bool aggregate,
+					u32 *size)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (aggregate) {
+		*size = (u32) sizeof(gaudi->events_stat_aggregate);
+		return gaudi->events_stat_aggregate;
+	}
+
+	*size = (u32) sizeof(gaudi->events_stat);
+	return gaudi->events_stat;
+}
+
+static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
+					u32 flags)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 status, timeout_usec;
+	int rc;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||
+		hdev->hard_reset_pending)
+		return 0;
+
+	if (hdev->pldm)
+		timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;
+	else
+		timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
+
+	mutex_lock(&hdev->mmu_cache_lock);
+
+	/* L0 & L1 invalidation */
+	WREG32(mmSTLB_INV_PS, 2);
+
+	rc = hl_poll_timeout(
+		hdev,
+		mmSTLB_INV_PS,
+		status,
+		!status,
+		1000,
+		timeout_usec);
+
+	WREG32(mmSTLB_INV_SET, 0);
+
+	mutex_unlock(&hdev->mmu_cache_lock);
+
+	if (rc) {
+		dev_err_ratelimited(hdev->dev,
+					"MMU cache invalidation timeout\n");
+		hl_device_reset(hdev, true, false);
+	}
+
+	return rc;
+}
+
+static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev,
+				bool is_hard, u32 asid, u64 va, u64 size)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u32 status, timeout_usec;
+	u32 inv_data;
+	u32 pi;
+	int rc;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||
+		hdev->hard_reset_pending)
+		return 0;
+
+	mutex_lock(&hdev->mmu_cache_lock);
+
+	if (hdev->pldm)
+		timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;
+	else
+		timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
+
+	/*
+	 * TODO: currently invalidate entire L0 & L1 as in regular hard
+	 * invalidation. Need to apply invalidation of specific cache
+	 * lines with mask of ASID & VA & size.
+	 * Note that L1 with be flushed entirely in any case.
+	 */
+
+	/* L0 & L1 invalidation */
+	inv_data = RREG32(mmSTLB_CACHE_INV);
+	/* PI is 8 bit */
+	pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF;
+	WREG32(mmSTLB_CACHE_INV,
+		(inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi);
+
+	rc = hl_poll_timeout(
+		hdev,
+		mmSTLB_INV_CONSUMER_INDEX,
+		status,
+		status == pi,
+		1000,
+		timeout_usec);
+
+	mutex_unlock(&hdev->mmu_cache_lock);
+
+	if (rc) {
+		dev_err_ratelimited(hdev->dev,
+					"MMU cache invalidation timeout\n");
+		hl_device_reset(hdev, true, false);
+	}
+
+	return rc;
+}
+
+static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev,
+					u32 asid, u64 phys_addr)
+{
+	u32 status, timeout_usec;
+	int rc;
+
+	if (hdev->pldm)
+		timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;
+	else
+		timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
+
+	WREG32(MMU_ASID, asid);
+	WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
+	WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
+	WREG32(MMU_BUSY, 0x80000000);
+
+	rc = hl_poll_timeout(
+		hdev,
+		MMU_BUSY,
+		status,
+		!(status & 0x80000000),
+		1000,
+		timeout_usec);
+
+	if (rc) {
+		dev_err(hdev->dev,
+			"Timeout during MMU hop0 config of asid %d\n", asid);
+		return rc;
+	}
+
+	return 0;
+}
+
+static int gaudi_send_heartbeat(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+		return 0;
+
+	return hl_fw_send_heartbeat(hdev);
+}
+
+static int gaudi_armcp_info_get(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	int rc;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+		return 0;
+
+	rc = hl_fw_armcp_info_get(hdev);
+	if (rc)
+		return rc;
+
+	if (!strlen(prop->armcp_info.card_name))
+		strncpy(prop->armcp_info.card_name, GAUDI_DEFAULT_CARD_NAME,
+				CARD_NAME_MAX_LEN);
+
+	return 0;
+}
+
+static bool gaudi_is_device_idle(struct hl_device *hdev, u32 *mask,
+					struct seq_file *s)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n";
+	const char *mme_slave_fmt = "%-5d%-9s%-14s%-12s%#x\n";
+	u32 qm_glbl_sts0, qm_cgm_sts, dma_core_sts0, tpc_cfg_sts, mme_arch_sts;
+	bool is_idle = true, is_eng_idle, is_slave;
+	u64 offset;
+	int i, dma_id;
+
+	mutex_lock(&gaudi->clk_gate_mutex);
+
+	hdev->asic_funcs->disable_clock_gating(hdev);
+
+	if (s)
+		seq_puts(s,
+			"\nDMA  is_idle  QM_GLBL_STS0  QM_CGM_STS  DMA_CORE_STS0\n"
+			"---  -------  ------------  ----------  -------------\n");
+
+	for (i = 0 ; i < DMA_NUMBER_OF_CHNLS ; i++) {
+		dma_id = gaudi_dma_assignment[i];
+		offset = dma_id * DMA_QMAN_OFFSET;
+
+		qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + offset);
+		qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + offset);
+		dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + offset);
+		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&
+				IS_DMA_IDLE(dma_core_sts0);
+		is_idle &= is_eng_idle;
+
+		if (mask)
+			*mask |= !is_eng_idle <<
+					(GAUDI_ENGINE_ID_DMA_0 + dma_id);
+		if (s)
+			seq_printf(s, fmt, dma_id,
+				is_eng_idle ? "Y" : "N", qm_glbl_sts0,
+				qm_cgm_sts, dma_core_sts0);
+	}
+
+	if (s)
+		seq_puts(s,
+			"\nTPC  is_idle  QM_GLBL_STS0  QM_CGM_STS  CFG_STATUS\n"
+			"---  -------  ------------  ----------  ----------\n");
+
+	for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {
+		offset = i * TPC_QMAN_OFFSET;
+		qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + offset);
+		qm_cgm_sts = RREG32(mmTPC0_QM_CGM_STS + offset);
+		tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + offset);
+		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&
+				IS_TPC_IDLE(tpc_cfg_sts);
+		is_idle &= is_eng_idle;
+
+		if (mask)
+			*mask |= !is_eng_idle << (GAUDI_ENGINE_ID_TPC_0 + i);
+		if (s)
+			seq_printf(s, fmt, i,
+				is_eng_idle ? "Y" : "N",
+				qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);
+	}
+
+	if (s)
+		seq_puts(s,
+			"\nMME  is_idle  QM_GLBL_STS0  QM_CGM_STS  ARCH_STATUS\n"
+			"---  -------  ------------  ----------  -----------\n");
+
+	for (i = 0 ; i < MME_NUMBER_OF_ENGINES ; i++) {
+		offset = i * MME_QMAN_OFFSET;
+		mme_arch_sts = RREG32(mmMME0_CTRL_ARCH_STATUS + offset);
+		is_eng_idle = IS_MME_IDLE(mme_arch_sts);
+
+		/* MME 1 & 3 are slaves, no need to check their QMANs */
+		is_slave = i % 2;
+		if (!is_slave) {
+			qm_glbl_sts0 = RREG32(mmMME0_QM_GLBL_STS0 + offset);
+			qm_cgm_sts = RREG32(mmMME0_QM_CGM_STS + offset);
+			is_eng_idle &= IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts);
+		}
+
+		is_idle &= is_eng_idle;
+
+		if (mask)
+			*mask |= !is_eng_idle << (GAUDI_ENGINE_ID_MME_0 + i);
+		if (s) {
+			if (!is_slave)
+				seq_printf(s, fmt, i,
+					is_eng_idle ? "Y" : "N",
+					qm_glbl_sts0, qm_cgm_sts, mme_arch_sts);
+			else
+				seq_printf(s, mme_slave_fmt, i,
+					is_eng_idle ? "Y" : "N", "-",
+					"-", mme_arch_sts);
+		}
+	}
+
+	if (s)
+		seq_puts(s, "\n");
+
+	hdev->asic_funcs->enable_clock_gating(hdev);
+
+	mutex_unlock(&gaudi->clk_gate_mutex);
+
+	return is_idle;
+}
+
+static void gaudi_hw_queues_lock(struct hl_device *hdev)
+	__acquires(&gaudi->hw_queues_lock)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	spin_lock(&gaudi->hw_queues_lock);
+}
+
+static void gaudi_hw_queues_unlock(struct hl_device *hdev)
+	__releases(&gaudi->hw_queues_lock)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	spin_unlock(&gaudi->hw_queues_lock);
+}
+
+static u32 gaudi_get_pci_id(struct hl_device *hdev)
+{
+	return hdev->pdev->device;
+}
+
+static int gaudi_get_eeprom_data(struct hl_device *hdev, void *data,
+				size_t max_size)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
+		return 0;
+
+	return hl_fw_get_eeprom_data(hdev, data, max_size);
+}
+
+/*
+ * this function should be used only during initialization and/or after reset,
+ * when there are no active users.
+ */
+static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,
+				u32 tpc_id)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	u64 kernel_timeout;
+	u32 status, offset;
+	int rc;
+
+	offset = tpc_id * (mmTPC1_CFG_STATUS - mmTPC0_CFG_STATUS);
+
+	if (hdev->pldm)
+		kernel_timeout = GAUDI_PLDM_TPC_KERNEL_WAIT_USEC;
+	else
+		kernel_timeout = HL_DEVICE_TIMEOUT_USEC;
+
+	mutex_lock(&gaudi->clk_gate_mutex);
+
+	hdev->asic_funcs->disable_clock_gating(hdev);
+
+	WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW + offset,
+			lower_32_bits(tpc_kernel));
+	WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH + offset,
+			upper_32_bits(tpc_kernel));
+
+	WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_LOW + offset,
+			lower_32_bits(tpc_kernel));
+	WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_HIGH + offset,
+			upper_32_bits(tpc_kernel));
+	/* set a valid LUT pointer, content is of no significance */
+	WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_LO + offset,
+			lower_32_bits(tpc_kernel));
+	WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_HI + offset,
+			upper_32_bits(tpc_kernel));
+
+	WREG32(mmTPC0_CFG_QM_SYNC_OBJECT_ADDR + offset,
+			lower_32_bits(CFG_BASE +
+				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0));
+
+	WREG32(mmTPC0_CFG_TPC_CMD + offset,
+			(1 << TPC0_CFG_TPC_CMD_ICACHE_INVALIDATE_SHIFT |
+			1 << TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_64KB_SHIFT));
+	/* wait a bit for the engine to start executing */
+	usleep_range(1000, 1500);
+
+	/* wait until engine has finished executing */
+	rc = hl_poll_timeout(
+		hdev,
+		mmTPC0_CFG_STATUS + offset,
+		status,
+		(status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) ==
+				TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK,
+		1000,
+		kernel_timeout);
+
+	if (rc) {
+		dev_err(hdev->dev,
+			"Timeout while waiting for TPC%d icache prefetch\n",
+			tpc_id);
+		hdev->asic_funcs->enable_clock_gating(hdev);
+		mutex_unlock(&gaudi->clk_gate_mutex);
+		return -EIO;
+	}
+
+	WREG32(mmTPC0_CFG_TPC_EXECUTE + offset,
+			1 << TPC0_CFG_TPC_EXECUTE_V_SHIFT);
+
+	/* wait a bit for the engine to start executing */
+	usleep_range(1000, 1500);
+
+	/* wait until engine has finished executing */
+	rc = hl_poll_timeout(
+		hdev,
+		mmTPC0_CFG_STATUS + offset,
+		status,
+		(status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) ==
+				TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK,
+		1000,
+		kernel_timeout);
+
+	rc = hl_poll_timeout(
+		hdev,
+		mmTPC0_CFG_WQ_INFLIGHT_CNTR + offset,
+		status,
+		(status == 0),
+		1000,
+		kernel_timeout);
+
+	hdev->asic_funcs->enable_clock_gating(hdev);
+	mutex_unlock(&gaudi->clk_gate_mutex);
+
+	if (rc) {
+		dev_err(hdev->dev,
+			"Timeout while waiting for TPC%d kernel to execute\n",
+			tpc_id);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev)
+{
+	return RREG32(mmHW_STATE);
+}
+
+static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
+{
+	return gaudi_cq_assignment[cq_idx];
+}
+
+static void gaudi_ext_queue_init(struct hl_device *hdev, u32 q_idx)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	struct hl_hw_queue *hw_queue = &hdev->kernel_queues[q_idx];
+	struct hl_hw_sob *hw_sob;
+	int sob, ext_idx = gaudi->ext_queue_idx++;
+
+	/*
+	 * The external queues might not sit sequentially, hence use the
+	 * real external queue index for the SOB/MON base id.
+	 */
+	hw_queue->base_sob_id = ext_idx * HL_RSVD_SOBS;
+	hw_queue->base_mon_id = ext_idx * HL_RSVD_MONS;
+	hw_queue->next_sob_val = 1;
+	hw_queue->curr_sob_offset = 0;
+
+	for (sob = 0 ; sob < HL_RSVD_SOBS ; sob++) {
+		hw_sob = &hw_queue->hw_sob[sob];
+		hw_sob->hdev = hdev;
+		hw_sob->sob_id = hw_queue->base_sob_id + sob;
+		hw_sob->q_idx = q_idx;
+		kref_init(&hw_sob->kref);
+	}
+}
+
+static void gaudi_ext_queue_reset(struct hl_device *hdev, u32 q_idx)
+{
+	struct hl_hw_queue *hw_queue = &hdev->kernel_queues[q_idx];
+
+	/*
+	 * In case we got here due to a stuck CS, the refcnt might be bigger
+	 * than 1 and therefore we reset it.
+	 */
+	kref_init(&hw_queue->hw_sob[hw_queue->curr_sob_offset].kref);
+	hw_queue->curr_sob_offset = 0;
+	hw_queue->next_sob_val = 1;
+}
+
+static u32 gaudi_get_signal_cb_size(struct hl_device *hdev)
+{
+	return sizeof(struct packet_msg_short) +
+			sizeof(struct packet_msg_prot) * 2;
+}
+
+static u32 gaudi_get_wait_cb_size(struct hl_device *hdev)
+{
+	return sizeof(struct packet_msg_short) * 4 +
+			sizeof(struct packet_fence) +
+			sizeof(struct packet_msg_prot) * 2;
+}
+
+static void gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id)
+{
+	struct hl_cb *cb = (struct hl_cb *) data;
+	struct packet_msg_short *pkt;
+	u32 value, ctl;
+
+	pkt = (struct packet_msg_short *) (uintptr_t) cb->kernel_address;
+	memset(pkt, 0, sizeof(*pkt));
+
+	value = 1 << GAUDI_PKT_SHORT_VAL_SOB_SYNC_VAL_SHIFT; /* inc by 1 */
+	value |= 1 << GAUDI_PKT_SHORT_VAL_SOB_MOD_SHIFT; /* add mode */
+
+	ctl = (sob_id * 4) << GAUDI_PKT_SHORT_CTL_ADDR_SHIFT; /* SOB id */
+	ctl |= 0 << GAUDI_PKT_SHORT_CTL_OP_SHIFT; /* write the value */
+	ctl |= 3 << GAUDI_PKT_SHORT_CTL_BASE_SHIFT; /* W_S SOB base */
+	ctl |= PACKET_MSG_SHORT << GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT;
+	ctl |= 1 << GAUDI_PKT_SHORT_CTL_EB_SHIFT;
+	ctl |= 1 << GAUDI_PKT_SHORT_CTL_RB_SHIFT;
+	ctl |= 1 << GAUDI_PKT_SHORT_CTL_MB_SHIFT;
+
+	pkt->value = cpu_to_le32(value);
+	pkt->ctl = cpu_to_le32(ctl);
+}
+
+static u32 gaudi_add_mon_msg_short(struct packet_msg_short *pkt, u32 value,
+					u16 addr)
+{
+	u32 ctl, pkt_size = sizeof(*pkt);
+
+	memset(pkt, 0, pkt_size);
+
+	ctl = addr << GAUDI_PKT_SHORT_CTL_ADDR_SHIFT;
+	ctl |= 2 << GAUDI_PKT_SHORT_CTL_BASE_SHIFT; /* W_S MON base */
+	ctl |= PACKET_MSG_SHORT << GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT;
+	ctl |= 0 << GAUDI_PKT_SHORT_CTL_EB_SHIFT;
+	ctl |= 1 << GAUDI_PKT_SHORT_CTL_RB_SHIFT;
+	ctl |= 0 << GAUDI_PKT_SHORT_CTL_MB_SHIFT; /* only last pkt needs MB */
+
+	pkt->value = cpu_to_le32(value);
+	pkt->ctl = cpu_to_le32(ctl);
+
+	return pkt_size;
+}
+
+static u32 gaudi_add_arm_monitor_pkt(struct packet_msg_short *pkt, u16 sob_id,
+					u16 sob_val, u16 addr)
+{
+	u32 ctl, value, pkt_size = sizeof(*pkt);
+	u8 mask = ~(1 << (sob_id & 0x7));
+
+	memset(pkt, 0, pkt_size);
+
+	value = (sob_id / 8) << GAUDI_PKT_SHORT_VAL_MON_SYNC_GID_SHIFT;
+	value |= sob_val << GAUDI_PKT_SHORT_VAL_MON_SYNC_VAL_SHIFT;
+	value |= 0 << GAUDI_PKT_SHORT_VAL_MON_MODE_SHIFT; /* GREATER_OR_EQUAL */
+	value |= mask << GAUDI_PKT_SHORT_VAL_MON_MASK_SHIFT;
+
+	ctl = addr << GAUDI_PKT_SHORT_CTL_ADDR_SHIFT;
+	ctl |= 0 << GAUDI_PKT_SHORT_CTL_OP_SHIFT; /* write the value */
+	ctl |= 2 << GAUDI_PKT_SHORT_CTL_BASE_SHIFT; /* W_S MON base */
+	ctl |= PACKET_MSG_SHORT << GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT;
+	ctl |= 0 << GAUDI_PKT_SHORT_CTL_EB_SHIFT;
+	ctl |= 1 << GAUDI_PKT_SHORT_CTL_RB_SHIFT;
+	ctl |= 1 << GAUDI_PKT_SHORT_CTL_MB_SHIFT;
+
+	pkt->value = cpu_to_le32(value);
+	pkt->ctl = cpu_to_le32(ctl);
+
+	return pkt_size;
+}
+
+static u32 gaudi_add_fence_pkt(struct packet_fence *pkt)
+{
+	u32 ctl, cfg, pkt_size = sizeof(*pkt);
+
+	memset(pkt, 0, pkt_size);
+
+	cfg = 1 << GAUDI_PKT_FENCE_CFG_DEC_VAL_SHIFT;
+	cfg |= 1 << GAUDI_PKT_FENCE_CFG_TARGET_VAL_SHIFT;
+	cfg |= 2 << GAUDI_PKT_FENCE_CFG_ID_SHIFT;
+
+	ctl = 0 << GAUDI_PKT_FENCE_CTL_PRED_SHIFT;
+	ctl |= PACKET_FENCE << GAUDI_PKT_FENCE_CTL_OPCODE_SHIFT;
+	ctl |= 0 << GAUDI_PKT_FENCE_CTL_EB_SHIFT;
+	ctl |= 1 << GAUDI_PKT_FENCE_CTL_RB_SHIFT;
+	ctl |= 1 << GAUDI_PKT_FENCE_CTL_MB_SHIFT;
+
+	pkt->cfg = cpu_to_le32(cfg);
+	pkt->ctl = cpu_to_le32(ctl);
+
+	return pkt_size;
+}
+
+static void gaudi_gen_wait_cb(struct hl_device *hdev, void *data, u16 sob_id,
+			u16 sob_val, u16 mon_id, u32 q_idx)
+{
+	struct hl_cb *cb = (struct hl_cb *) data;
+	void *buf = (void *) (uintptr_t) cb->kernel_address;
+	u64 monitor_base, fence_addr = 0;
+	u32 size = 0;
+	u16 msg_addr_offset;
+
+	switch (q_idx) {
+	case GAUDI_QUEUE_ID_DMA_0_0:
+		fence_addr = mmDMA0_QM_CP_FENCE2_RDATA_0;
+		break;
+	case GAUDI_QUEUE_ID_DMA_0_1:
+		fence_addr = mmDMA0_QM_CP_FENCE2_RDATA_1;
+		break;
+	case GAUDI_QUEUE_ID_DMA_0_2:
+		fence_addr = mmDMA0_QM_CP_FENCE2_RDATA_2;
+		break;
+	case GAUDI_QUEUE_ID_DMA_0_3:
+		fence_addr = mmDMA0_QM_CP_FENCE2_RDATA_3;
+		break;
+	case GAUDI_QUEUE_ID_DMA_1_0:
+		fence_addr = mmDMA1_QM_CP_FENCE2_RDATA_0;
+		break;
+	case GAUDI_QUEUE_ID_DMA_1_1:
+		fence_addr = mmDMA1_QM_CP_FENCE2_RDATA_1;
+		break;
+	case GAUDI_QUEUE_ID_DMA_1_2:
+		fence_addr = mmDMA1_QM_CP_FENCE2_RDATA_2;
+		break;
+	case GAUDI_QUEUE_ID_DMA_1_3:
+		fence_addr = mmDMA1_QM_CP_FENCE2_RDATA_3;
+		break;
+	case GAUDI_QUEUE_ID_DMA_5_0:
+		fence_addr = mmDMA5_QM_CP_FENCE2_RDATA_0;
+		break;
+	case GAUDI_QUEUE_ID_DMA_5_1:
+		fence_addr = mmDMA5_QM_CP_FENCE2_RDATA_1;
+		break;
+	case GAUDI_QUEUE_ID_DMA_5_2:
+		fence_addr = mmDMA5_QM_CP_FENCE2_RDATA_2;
+		break;
+	case GAUDI_QUEUE_ID_DMA_5_3:
+		fence_addr = mmDMA5_QM_CP_FENCE2_RDATA_3;
+		break;
+	default:
+		/* queue index should be valid here */
+		dev_crit(hdev->dev, "wrong queue id %d for wait packet\n",
+				q_idx);
+		return;
+	}
+
+	fence_addr += CFG_BASE;
+
+	/*
+	 * monitor_base should be the content of the base0 address registers,
+	 * so it will be added to the msg short offsets
+	 */
+	monitor_base = mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0;
+
+	/* First monitor config packet: low address of the sync */
+	msg_addr_offset =
+		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_id * 4) -
+				monitor_base;
+
+	size += gaudi_add_mon_msg_short(buf + size, (u32) fence_addr,
+					msg_addr_offset);
+
+	/* Second monitor config packet: high address of the sync */
+	msg_addr_offset =
+		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_id * 4) -
+				monitor_base;
+
+	size += gaudi_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32),
+					msg_addr_offset);
+
+	/*
+	 * Third monitor config packet: the payload, i.e. what to write when the
+	 * sync triggers
+	 */
+	msg_addr_offset =
+		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_id * 4) -
+				monitor_base;
+
+	size += gaudi_add_mon_msg_short(buf + size, 1, msg_addr_offset);
+
+	/* Fourth monitor config packet: bind the monitor to a sync object */
+	msg_addr_offset =
+		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0 + mon_id * 4) -
+				monitor_base;
+	size += gaudi_add_arm_monitor_pkt(buf + size, sob_id, sob_val,
+						msg_addr_offset);
+
+	/* Fence packet */
+	size += gaudi_add_fence_pkt(buf + size);
+}
+
+static void gaudi_reset_sob(struct hl_device *hdev, void *data)
+{
+	struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data;
+
+	dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx,
+		hw_sob->sob_id);
+
+	WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4,
+		0);
+
+	kref_init(&hw_sob->kref);
+}
+
+static void gaudi_set_dma_mask_from_fw(struct hl_device *hdev)
+{
+	if (RREG32(mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_0) ==
+							HL_POWER9_HOST_MAGIC) {
+		hdev->power9_64bit_dma_enable = 1;
+		hdev->dma_mask = 64;
+	} else {
+		hdev->power9_64bit_dma_enable = 0;
+		hdev->dma_mask = 48;
+	}
+}
+
+static u64 gaudi_get_device_time(struct hl_device *hdev)
+{
+	u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32;
+
+	return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);
+}
+
+static const struct hl_asic_funcs gaudi_funcs = {
+	.early_init = gaudi_early_init,
+	.early_fini = gaudi_early_fini,
+	.late_init = gaudi_late_init,
+	.late_fini = gaudi_late_fini,
+	.sw_init = gaudi_sw_init,
+	.sw_fini = gaudi_sw_fini,
+	.hw_init = gaudi_hw_init,
+	.hw_fini = gaudi_hw_fini,
+	.halt_engines = gaudi_halt_engines,
+	.suspend = gaudi_suspend,
+	.resume = gaudi_resume,
+	.cb_mmap = gaudi_cb_mmap,
+	.ring_doorbell = gaudi_ring_doorbell,
+	.pqe_write = gaudi_pqe_write,
+	.asic_dma_alloc_coherent = gaudi_dma_alloc_coherent,
+	.asic_dma_free_coherent = gaudi_dma_free_coherent,
+	.get_int_queue_base = gaudi_get_int_queue_base,
+	.test_queues = gaudi_test_queues,
+	.asic_dma_pool_zalloc = gaudi_dma_pool_zalloc,
+	.asic_dma_pool_free = gaudi_dma_pool_free,
+	.cpu_accessible_dma_pool_alloc = gaudi_cpu_accessible_dma_pool_alloc,
+	.cpu_accessible_dma_pool_free = gaudi_cpu_accessible_dma_pool_free,
+	.hl_dma_unmap_sg = gaudi_dma_unmap_sg,
+	.cs_parser = gaudi_cs_parser,
+	.asic_dma_map_sg = gaudi_dma_map_sg,
+	.get_dma_desc_list_size = gaudi_get_dma_desc_list_size,
+	.add_end_of_cb_packets = gaudi_add_end_of_cb_packets,
+	.update_eq_ci = gaudi_update_eq_ci,
+	.context_switch = gaudi_context_switch,
+	.restore_phase_topology = gaudi_restore_phase_topology,
+	.debugfs_read32 = gaudi_debugfs_read32,
+	.debugfs_write32 = gaudi_debugfs_write32,
+	.debugfs_read64 = gaudi_debugfs_read64,
+	.debugfs_write64 = gaudi_debugfs_write64,
+	.add_device_attr = gaudi_add_device_attr,
+	.handle_eqe = gaudi_handle_eqe,
+	.set_pll_profile = gaudi_set_pll_profile,
+	.get_events_stat = gaudi_get_events_stat,
+	.read_pte = gaudi_read_pte,
+	.write_pte = gaudi_write_pte,
+	.mmu_invalidate_cache = gaudi_mmu_invalidate_cache,
+	.mmu_invalidate_cache_range = gaudi_mmu_invalidate_cache_range,
+	.send_heartbeat = gaudi_send_heartbeat,
+	.enable_clock_gating = gaudi_enable_clock_gating,
+	.disable_clock_gating = gaudi_disable_clock_gating,
+	.debug_coresight = gaudi_debug_coresight,
+	.is_device_idle = gaudi_is_device_idle,
+	.soft_reset_late_init = gaudi_soft_reset_late_init,
+	.hw_queues_lock = gaudi_hw_queues_lock,
+	.hw_queues_unlock = gaudi_hw_queues_unlock,
+	.get_pci_id = gaudi_get_pci_id,
+	.get_eeprom_data = gaudi_get_eeprom_data,
+	.send_cpu_message = gaudi_send_cpu_message,
+	.get_hw_state = gaudi_get_hw_state,
+	.pci_bars_map = gaudi_pci_bars_map,
+	.set_dram_bar_base = gaudi_set_hbm_bar_base,
+	.init_iatu = gaudi_init_iatu,
+	.rreg = hl_rreg,
+	.wreg = hl_wreg,
+	.halt_coresight = gaudi_halt_coresight,
+	.get_clk_rate = gaudi_get_clk_rate,
+	.get_queue_id_for_cq = gaudi_get_queue_id_for_cq,
+	.read_device_fw_version = gaudi_read_device_fw_version,
+	.load_firmware_to_device = gaudi_load_firmware_to_device,
+	.load_boot_fit_to_device = gaudi_load_boot_fit_to_device,
+	.ext_queue_init = gaudi_ext_queue_init,
+	.ext_queue_reset = gaudi_ext_queue_reset,
+	.get_signal_cb_size = gaudi_get_signal_cb_size,
+	.get_wait_cb_size = gaudi_get_wait_cb_size,
+	.gen_signal_cb = gaudi_gen_signal_cb,
+	.gen_wait_cb = gaudi_gen_wait_cb,
+	.reset_sob = gaudi_reset_sob,
+	.set_dma_mask_from_fw = gaudi_set_dma_mask_from_fw,
+	.get_device_time = gaudi_get_device_time
+};
+
+/**
+ * gaudi_set_asic_funcs - set GAUDI function pointers
+ *
+ * @*hdev: pointer to hl_device structure
+ *
+ */
+void gaudi_set_asic_funcs(struct hl_device *hdev)
+{
+	hdev->asic_funcs = &gaudi_funcs;
+}
diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h
new file mode 100644
index 000000000000..a46530d375fa
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/gaudiP.h
@@ -0,0 +1,261 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2019-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GAUDIP_H_
+#define GAUDIP_H_
+
+#include <uapi/misc/habanalabs.h>
+#include "habanalabs.h"
+#include "include/hl_boot_if.h"
+#include "include/gaudi/gaudi_packets.h"
+#include "include/gaudi/gaudi.h"
+#include "include/gaudi/gaudi_async_events.h"
+
+#define NUMBER_OF_EXT_HW_QUEUES		12
+#define NUMBER_OF_CMPLT_QUEUES		NUMBER_OF_EXT_HW_QUEUES
+#define NUMBER_OF_CPU_HW_QUEUES		1
+#define NUMBER_OF_INT_HW_QUEUES		100
+#define NUMBER_OF_HW_QUEUES		(NUMBER_OF_EXT_HW_QUEUES + \
+					NUMBER_OF_CPU_HW_QUEUES + \
+					NUMBER_OF_INT_HW_QUEUES)
+
+/*
+ * Number of MSI interrupts IDS:
+ * Each completion queue has 1 ID
+ * The event queue has 1 ID
+ */
+#define NUMBER_OF_INTERRUPTS		(NUMBER_OF_CMPLT_QUEUES + \
+						NUMBER_OF_CPU_HW_QUEUES)
+
+#if (NUMBER_OF_INTERRUPTS > GAUDI_MSI_ENTRIES)
+#error "Number of MSI interrupts must be smaller or equal to GAUDI_MSI_ENTRIES"
+#endif
+
+#define QMAN_FENCE_TIMEOUT_USEC		10000		/* 10 ms */
+
+#define CORESIGHT_TIMEOUT_USEC		100000		/* 100 ms */
+
+#define GAUDI_MAX_CLK_FREQ		2200000000ull	/* 2200 MHz */
+
+#define MAX_POWER_DEFAULT		200000		/* 200W */
+
+#define GAUDI_CPU_TIMEOUT_USEC		15000000	/* 15s */
+
+#define TPC_ENABLED_MASK		0xFF
+
+#define GAUDI_HBM_SIZE_32GB		0x800000000ull
+#define GAUDI_HBM_DEVICES		4
+#define GAUDI_HBM_CHANNELS		8
+#define GAUDI_HBM_CFG_BASE		(mmHBM0_BASE - CFG_BASE)
+#define GAUDI_HBM_CFG_OFFSET		(mmHBM1_BASE - mmHBM0_BASE)
+
+#define DMA_MAX_TRANSFER_SIZE		U32_MAX
+
+#define GAUDI_DEFAULT_CARD_NAME		"HL2000"
+
+#define PCI_DMA_NUMBER_OF_CHNLS		3
+#define HBM_DMA_NUMBER_OF_CHNLS		5
+#define DMA_NUMBER_OF_CHNLS		(PCI_DMA_NUMBER_OF_CHNLS + \
+						HBM_DMA_NUMBER_OF_CHNLS)
+
+#define MME_NUMBER_OF_SLAVE_ENGINES	2
+#define MME_NUMBER_OF_ENGINES		(MME_NUMBER_OF_MASTER_ENGINES + \
+					MME_NUMBER_OF_SLAVE_ENGINES)
+#define MME_NUMBER_OF_QMANS		(MME_NUMBER_OF_MASTER_ENGINES * \
+					QMAN_STREAMS)
+
+#define QMAN_STREAMS		4
+
+#define DMA_QMAN_OFFSET		(mmDMA1_QM_BASE - mmDMA0_QM_BASE)
+#define TPC_QMAN_OFFSET		(mmTPC1_QM_BASE - mmTPC0_QM_BASE)
+#define MME_QMAN_OFFSET		(mmMME1_QM_BASE - mmMME0_QM_BASE)
+#define NIC_MACRO_QMAN_OFFSET	(mmNIC1_QM0_BASE - mmNIC0_QM0_BASE)
+
+#define TPC_CFG_OFFSET		(mmTPC1_CFG_BASE - mmTPC0_CFG_BASE)
+
+#define DMA_CORE_OFFSET		(mmDMA1_CORE_BASE - mmDMA0_CORE_BASE)
+
+#define SIF_RTR_CTRL_OFFSET	(mmSIF_RTR_CTRL_1_BASE - mmSIF_RTR_CTRL_0_BASE)
+
+#define NIF_RTR_CTRL_OFFSET	(mmNIF_RTR_CTRL_1_BASE - mmNIF_RTR_CTRL_0_BASE)
+
+#define MME_ACC_OFFSET		(mmMME1_ACC_BASE - mmMME0_ACC_BASE)
+#define SRAM_BANK_OFFSET	(mmSRAM_Y0_X1_RTR_BASE - mmSRAM_Y0_X0_RTR_BASE)
+
+#define NUM_OF_SOB_IN_BLOCK		\
+	(((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 - \
+	mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0) + 4) >> 2)
+
+#define NUM_OF_MONITORS_IN_BLOCK	\
+	(((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \
+	mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2)
+
+
+/* DRAM Memory Map */
+
+#define CPU_FW_IMAGE_SIZE	0x10000000	/* 256MB */
+#define MMU_PAGE_TABLES_SIZE	0x0BF00000	/* 191MB */
+#define MMU_CACHE_MNG_SIZE	0x00100000	/* 1MB */
+#define RESERVED		0x04000000	/* 64MB */
+
+#define CPU_FW_IMAGE_ADDR	DRAM_PHYS_BASE
+#define MMU_PAGE_TABLES_ADDR	(CPU_FW_IMAGE_ADDR + CPU_FW_IMAGE_SIZE)
+#define MMU_CACHE_MNG_ADDR	(MMU_PAGE_TABLES_ADDR + MMU_PAGE_TABLES_SIZE)
+
+#define DRAM_DRIVER_END_ADDR	(MMU_CACHE_MNG_ADDR + MMU_CACHE_MNG_SIZE +\
+								RESERVED)
+
+#define DRAM_BASE_ADDR_USER	0x20000000
+
+#if (DRAM_DRIVER_END_ADDR > DRAM_BASE_ADDR_USER)
+#error "Driver must reserve no more than 512MB"
+#endif
+
+/* Internal QMANs PQ sizes */
+
+#define MME_QMAN_LENGTH			64
+#define MME_QMAN_SIZE_IN_BYTES		(MME_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
+
+#define HBM_DMA_QMAN_LENGTH		64
+#define HBM_DMA_QMAN_SIZE_IN_BYTES	\
+				(HBM_DMA_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
+
+#define TPC_QMAN_LENGTH			64
+#define TPC_QMAN_SIZE_IN_BYTES		(TPC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)
+
+#define SRAM_USER_BASE_OFFSET  GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START
+
+/* Virtual address space */
+#define VA_HOST_SPACE_START	0x1000000000000ull	/* 256TB */
+#define VA_HOST_SPACE_END	0x3FF8000000000ull	/* 1PB - 1TB */
+#define VA_HOST_SPACE_SIZE	(VA_HOST_SPACE_END - \
+					VA_HOST_SPACE_START) /* 767TB */
+
+#define HW_CAP_PLL		0x00000001
+#define HW_CAP_HBM		0x00000002
+#define HW_CAP_MMU		0x00000004
+#define HW_CAP_MME		0x00000008
+#define HW_CAP_CPU		0x00000010
+#define HW_CAP_PCI_DMA		0x00000020
+#define HW_CAP_MSI		0x00000040
+#define HW_CAP_CPU_Q		0x00000080
+#define HW_CAP_HBM_DMA		0x00000100
+#define HW_CAP_CLK_GATE		0x00000200
+#define HW_CAP_SRAM_SCRAMBLER	0x00000400
+#define HW_CAP_HBM_SCRAMBLER	0x00000800
+
+#define HW_CAP_TPC0		0x01000000
+#define HW_CAP_TPC1		0x02000000
+#define HW_CAP_TPC2		0x04000000
+#define HW_CAP_TPC3		0x08000000
+#define HW_CAP_TPC4		0x10000000
+#define HW_CAP_TPC5		0x20000000
+#define HW_CAP_TPC6		0x40000000
+#define HW_CAP_TPC7		0x80000000
+#define HW_CAP_TPC_MASK		0xFF000000
+#define HW_CAP_TPC_SHIFT	24
+
+#define GAUDI_CPU_PCI_MSB_ADDR(addr)	(((addr) & GENMASK_ULL(49, 39)) >> 39)
+#define GAUDI_PCI_TO_CPU_ADDR(addr)			\
+	do {						\
+		(addr) &= ~GENMASK_ULL(49, 39);		\
+		(addr) |= BIT_ULL(39);			\
+	} while (0)
+#define GAUDI_CPU_TO_PCI_ADDR(addr, extension)		\
+	do {						\
+		(addr) &= ~GENMASK_ULL(49, 39);		\
+		(addr) |= (u64) (extension) << 39;	\
+	} while (0)
+
+enum gaudi_dma_channels {
+	GAUDI_PCI_DMA_1,
+	GAUDI_PCI_DMA_2,
+	GAUDI_PCI_DMA_3,
+	GAUDI_HBM_DMA_1,
+	GAUDI_HBM_DMA_2,
+	GAUDI_HBM_DMA_3,
+	GAUDI_HBM_DMA_4,
+	GAUDI_HBM_DMA_5,
+	GAUDI_DMA_MAX
+};
+
+enum gaudi_tpc_mask {
+	GAUDI_TPC_MASK_TPC0 = 0x01,
+	GAUDI_TPC_MASK_TPC1 = 0x02,
+	GAUDI_TPC_MASK_TPC2 = 0x04,
+	GAUDI_TPC_MASK_TPC3 = 0x08,
+	GAUDI_TPC_MASK_TPC4 = 0x10,
+	GAUDI_TPC_MASK_TPC5 = 0x20,
+	GAUDI_TPC_MASK_TPC6 = 0x40,
+	GAUDI_TPC_MASK_TPC7 = 0x80,
+	GAUDI_TPC_MASK_ALL = 0xFF
+};
+
+/**
+ * struct gaudi_internal_qman_info - Internal QMAN information.
+ * @pq_kernel_addr: Kernel address of the PQ memory area in the host.
+ * @pq_dma_addr: DMA address of the PQ memory area in the host.
+ * @pq_size: Size of allocated host memory for PQ.
+ */
+struct gaudi_internal_qman_info {
+	void		*pq_kernel_addr;
+	dma_addr_t	pq_dma_addr;
+	size_t		pq_size;
+};
+
+/**
+ * struct gaudi_device - ASIC specific manage structure.
+ * @armcp_info_get: get information on device from ArmCP
+ * @hw_queues_lock: protects the H/W queues from concurrent access.
+ * @clk_gate_mutex: protects code areas that require clock gating to be disabled
+ *                  temporarily
+ * @internal_qmans: Internal QMANs information. The array size is larger than
+ *                  the actual number of internal queues because they are not in
+ *                  consecutive order.
+ * @hbm_bar_cur_addr: current address of HBM PCI bar.
+ * @max_freq_value: current max clk frequency.
+ * @events: array that holds all event id's
+ * @events_stat: array that holds histogram of all received events.
+ * @events_stat_aggregate: same as events_stat but doesn't get cleared on reset
+ * @hw_cap_initialized: This field contains a bit per H/W engine. When that
+ *                      engine is initialized, that bit is set by the driver to
+ *                      signal we can use this engine in later code paths.
+ *                      Each bit is cleared upon reset of its corresponding H/W
+ *                      engine.
+ * @multi_msi_mode: whether we are working in multi MSI single MSI mode.
+ *                  Multi MSI is possible only with IOMMU enabled.
+ * @ext_queue_idx: helper index for external queues initialization.
+ */
+struct gaudi_device {
+	int (*armcp_info_get)(struct hl_device *hdev);
+
+	/* TODO: remove hw_queues_lock after moving to scheduler code */
+	spinlock_t			hw_queues_lock;
+	struct mutex			clk_gate_mutex;
+
+	struct gaudi_internal_qman_info	internal_qmans[GAUDI_QUEUE_ID_SIZE];
+
+	u64				hbm_bar_cur_addr;
+	u64				max_freq_value;
+
+	u32				events[GAUDI_EVENT_SIZE];
+	u32				events_stat[GAUDI_EVENT_SIZE];
+	u32				events_stat_aggregate[GAUDI_EVENT_SIZE];
+	u32				hw_cap_initialized;
+	u8				multi_msi_mode;
+	u8				ext_queue_idx;
+};
+
+void gaudi_init_security(struct hl_device *hdev);
+void gaudi_add_device_attr(struct hl_device *hdev,
+			struct attribute_group *dev_attr_grp);
+void gaudi_set_pll_profile(struct hl_device *hdev, enum hl_pll_frequency freq);
+int gaudi_debug_coresight(struct hl_device *hdev, void *data);
+void gaudi_halt_coresight(struct hl_device *hdev);
+int gaudi_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
+
+#endif /* GAUDIP_H_ */
diff --git a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c
new file mode 100644
index 000000000000..bf0e062d7b87
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c
@@ -0,0 +1,884 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "gaudiP.h"
+#include "include/gaudi/gaudi_coresight.h"
+#include "include/gaudi/asic_reg/gaudi_regs.h"
+#include "include/gaudi/gaudi_masks.h"
+
+#include <uapi/misc/habanalabs.h>
+#include <linux/coresight.h>
+
+#define SPMU_SECTION_SIZE		MME0_ACC_SPMU_MAX_OFFSET
+#define SPMU_EVENT_TYPES_OFFSET		0x400
+#define SPMU_MAX_COUNTERS		6
+
+static u64 debug_stm_regs[GAUDI_STM_LAST + 1] = {
+	[GAUDI_STM_MME0_ACC]	= mmMME0_ACC_STM_BASE,
+	[GAUDI_STM_MME0_SBAB]	= mmMME0_SBAB_STM_BASE,
+	[GAUDI_STM_MME0_CTRL]	= mmMME0_CTRL_STM_BASE,
+	[GAUDI_STM_MME1_ACC]	= mmMME1_ACC_STM_BASE,
+	[GAUDI_STM_MME1_SBAB]	= mmMME1_SBAB_STM_BASE,
+	[GAUDI_STM_MME1_CTRL]	= mmMME1_CTRL_STM_BASE,
+	[GAUDI_STM_MME2_ACC]	= mmMME2_ACC_STM_BASE,
+	[GAUDI_STM_MME2_SBAB]	= mmMME2_SBAB_STM_BASE,
+	[GAUDI_STM_MME2_CTRL]	= mmMME2_CTRL_STM_BASE,
+	[GAUDI_STM_MME3_ACC]	= mmMME3_ACC_STM_BASE,
+	[GAUDI_STM_MME3_SBAB]	= mmMME3_SBAB_STM_BASE,
+	[GAUDI_STM_MME3_CTRL]	= mmMME3_CTRL_STM_BASE,
+	[GAUDI_STM_DMA_IF_W_S]	= mmDMA_IF_W_S_STM_BASE,
+	[GAUDI_STM_DMA_IF_E_S]	= mmDMA_IF_E_S_STM_BASE,
+	[GAUDI_STM_DMA_IF_W_N]	= mmDMA_IF_W_N_STM_BASE,
+	[GAUDI_STM_DMA_IF_E_N]	= mmDMA_IF_E_N_STM_BASE,
+	[GAUDI_STM_CPU]		= mmCPU_STM_BASE,
+	[GAUDI_STM_DMA_CH_0_CS]	= mmDMA_CH_0_CS_STM_BASE,
+	[GAUDI_STM_DMA_CH_1_CS]	= mmDMA_CH_1_CS_STM_BASE,
+	[GAUDI_STM_DMA_CH_2_CS]	= mmDMA_CH_2_CS_STM_BASE,
+	[GAUDI_STM_DMA_CH_3_CS]	= mmDMA_CH_3_CS_STM_BASE,
+	[GAUDI_STM_DMA_CH_4_CS]	= mmDMA_CH_4_CS_STM_BASE,
+	[GAUDI_STM_DMA_CH_5_CS]	= mmDMA_CH_5_CS_STM_BASE,
+	[GAUDI_STM_DMA_CH_6_CS]	= mmDMA_CH_6_CS_STM_BASE,
+	[GAUDI_STM_DMA_CH_7_CS]	= mmDMA_CH_7_CS_STM_BASE,
+	[GAUDI_STM_PCIE]	= mmPCIE_STM_BASE,
+	[GAUDI_STM_MMU_CS]	= mmMMU_CS_STM_BASE,
+	[GAUDI_STM_PSOC]	= mmPSOC_STM_BASE,
+	[GAUDI_STM_NIC0_0]	= mmSTM_0_NIC0_DBG_BASE,
+	[GAUDI_STM_NIC0_1]	= mmSTM_1_NIC0_DBG_BASE,
+	[GAUDI_STM_NIC1_0]	= mmSTM_0_NIC1_DBG_BASE,
+	[GAUDI_STM_NIC1_1]	= mmSTM_1_NIC1_DBG_BASE,
+	[GAUDI_STM_NIC2_0]	= mmSTM_0_NIC2_DBG_BASE,
+	[GAUDI_STM_NIC2_1]	= mmSTM_1_NIC2_DBG_BASE,
+	[GAUDI_STM_NIC3_0]	= mmSTM_0_NIC3_DBG_BASE,
+	[GAUDI_STM_NIC3_1]	= mmSTM_1_NIC3_DBG_BASE,
+	[GAUDI_STM_NIC4_0]	= mmSTM_0_NIC4_DBG_BASE,
+	[GAUDI_STM_NIC4_1]	= mmSTM_1_NIC4_DBG_BASE,
+	[GAUDI_STM_TPC0_EML]	= mmTPC0_EML_STM_BASE,
+	[GAUDI_STM_TPC1_EML]	= mmTPC1_EML_STM_BASE,
+	[GAUDI_STM_TPC2_EML]	= mmTPC2_EML_STM_BASE,
+	[GAUDI_STM_TPC3_EML]	= mmTPC3_EML_STM_BASE,
+	[GAUDI_STM_TPC4_EML]	= mmTPC4_EML_STM_BASE,
+	[GAUDI_STM_TPC5_EML]	= mmTPC5_EML_STM_BASE,
+	[GAUDI_STM_TPC6_EML]	= mmTPC6_EML_STM_BASE,
+	[GAUDI_STM_TPC7_EML]	= mmTPC7_EML_STM_BASE
+};
+
+static u64 debug_etf_regs[GAUDI_ETF_LAST + 1] = {
+	[GAUDI_ETF_MME0_ACC]		= mmMME0_ACC_ETF_BASE,
+	[GAUDI_ETF_MME0_SBAB]		= mmMME0_SBAB_ETF_BASE,
+	[GAUDI_ETF_MME0_CTRL]		= mmMME0_CTRL_ETF_BASE,
+	[GAUDI_ETF_MME1_ACC]		= mmMME1_ACC_ETF_BASE,
+	[GAUDI_ETF_MME1_SBAB]		= mmMME1_SBAB_ETF_BASE,
+	[GAUDI_ETF_MME1_CTRL]		= mmMME1_CTRL_ETF_BASE,
+	[GAUDI_ETF_MME2_ACC]		= mmMME2_MME2_ACC_ETF_BASE,
+	[GAUDI_ETF_MME2_SBAB]		= mmMME2_SBAB_ETF_BASE,
+	[GAUDI_ETF_MME2_CTRL]		= mmMME2_CTRL_ETF_BASE,
+	[GAUDI_ETF_MME3_ACC]		= mmMME3_ACC_ETF_BASE,
+	[GAUDI_ETF_MME3_SBAB]		= mmMME3_SBAB_ETF_BASE,
+	[GAUDI_ETF_MME3_CTRL]		= mmMME3_CTRL_ETF_BASE,
+	[GAUDI_ETF_DMA_IF_W_S]		= mmDMA_IF_W_S_ETF_BASE,
+	[GAUDI_ETF_DMA_IF_E_S]		= mmDMA_IF_E_S_ETF_BASE,
+	[GAUDI_ETF_DMA_IF_W_N]		= mmDMA_IF_W_N_ETF_BASE,
+	[GAUDI_ETF_DMA_IF_E_N]		= mmDMA_IF_E_N_ETF_BASE,
+	[GAUDI_ETF_CPU_0]		= mmCPU_ETF_0_BASE,
+	[GAUDI_ETF_CPU_1]		= mmCPU_ETF_1_BASE,
+	[GAUDI_ETF_CPU_TRACE]		= mmCPU_ETF_TRACE_BASE,
+	[GAUDI_ETF_DMA_CH_0_CS]		= mmDMA_CH_0_CS_ETF_BASE,
+	[GAUDI_ETF_DMA_CH_1_CS]		= mmDMA_CH_1_CS_ETF_BASE,
+	[GAUDI_ETF_DMA_CH_2_CS]		= mmDMA_CH_2_CS_ETF_BASE,
+	[GAUDI_ETF_DMA_CH_3_CS]		= mmDMA_CH_3_CS_ETF_BASE,
+	[GAUDI_ETF_DMA_CH_4_CS]		= mmDMA_CH_4_CS_ETF_BASE,
+	[GAUDI_ETF_DMA_CH_5_CS]		= mmDMA_CH_5_CS_ETF_BASE,
+	[GAUDI_ETF_DMA_CH_6_CS]		= mmDMA_CH_6_CS_ETF_BASE,
+	[GAUDI_ETF_DMA_CH_7_CS]		= mmDMA_CH_7_CS_ETF_BASE,
+	[GAUDI_ETF_PCIE]		= mmPCIE_ETF_BASE,
+	[GAUDI_ETF_MMU_CS]		= mmMMU_CS_ETF_BASE,
+	[GAUDI_ETF_PSOC]		= mmPSOC_ETF_BASE,
+	[GAUDI_ETF_NIC0_0]		= mmETF_0_NIC0_DBG_BASE,
+	[GAUDI_ETF_NIC0_1]		= mmETF_1_NIC0_DBG_BASE,
+	[GAUDI_ETF_NIC1_0]		= mmETF_0_NIC1_DBG_BASE,
+	[GAUDI_ETF_NIC1_1]		= mmETF_1_NIC1_DBG_BASE,
+	[GAUDI_ETF_NIC2_0]		= mmETF_0_NIC2_DBG_BASE,
+	[GAUDI_ETF_NIC2_1]		= mmETF_1_NIC2_DBG_BASE,
+	[GAUDI_ETF_NIC3_0]		= mmETF_0_NIC3_DBG_BASE,
+	[GAUDI_ETF_NIC3_1]		= mmETF_1_NIC3_DBG_BASE,
+	[GAUDI_ETF_NIC4_0]		= mmETF_0_NIC4_DBG_BASE,
+	[GAUDI_ETF_NIC4_1]		= mmETF_1_NIC4_DBG_BASE,
+	[GAUDI_ETF_TPC0_EML]		= mmTPC0_EML_ETF_BASE,
+	[GAUDI_ETF_TPC1_EML]		= mmTPC1_EML_ETF_BASE,
+	[GAUDI_ETF_TPC2_EML]		= mmTPC2_EML_ETF_BASE,
+	[GAUDI_ETF_TPC3_EML]		= mmTPC3_EML_ETF_BASE,
+	[GAUDI_ETF_TPC4_EML]		= mmTPC4_EML_ETF_BASE,
+	[GAUDI_ETF_TPC5_EML]		= mmTPC5_EML_ETF_BASE,
+	[GAUDI_ETF_TPC6_EML]		= mmTPC6_EML_ETF_BASE,
+	[GAUDI_ETF_TPC7_EML]		= mmTPC7_EML_ETF_BASE
+};
+
+static u64 debug_funnel_regs[GAUDI_FUNNEL_LAST + 1] = {
+	[GAUDI_FUNNEL_MME0_ACC]		= mmMME0_ACC_FUNNEL_BASE,
+	[GAUDI_FUNNEL_MME1_ACC]		= mmMME1_ACC_FUNNEL_BASE,
+	[GAUDI_FUNNEL_MME2_ACC]		= mmMME2_ACC_FUNNEL_BASE,
+	[GAUDI_FUNNEL_MME3_ACC]		= mmMME3_ACC_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X0]	= mmSRAM_Y0_X0_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X1]	= mmSRAM_Y0_X1_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X2]	= mmSRAM_Y0_X2_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X3]	= mmSRAM_Y0_X3_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X4]	= mmSRAM_Y0_X4_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X5]	= mmSRAM_Y0_X5_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X6]	= mmSRAM_Y0_X6_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y0_X7]	= mmSRAM_Y0_X7_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X0]	= mmSRAM_Y1_X0_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X1]	= mmSRAM_Y1_X1_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X2]	= mmSRAM_Y1_X2_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X3]	= mmSRAM_Y1_X3_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X4]	= mmSRAM_Y1_X4_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X5]	= mmSRAM_Y1_X5_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X6]	= mmSRAM_Y1_X6_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y1_X7]	= mmSRAM_Y1_X7_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X0]	= mmSRAM_Y2_X0_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X1]	= mmSRAM_Y2_X1_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X2]	= mmSRAM_Y2_X2_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X3]	= mmSRAM_Y2_X3_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X4]	= mmSRAM_Y2_X4_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X5]	= mmSRAM_Y2_X5_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X6]	= mmSRAM_Y2_X6_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y2_X7]	= mmSRAM_Y2_X7_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X0]	= mmSRAM_Y3_X0_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X1]	= mmSRAM_Y3_X1_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X2]	= mmSRAM_Y3_X2_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X4]	= mmSRAM_Y3_X4_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X3]	= mmSRAM_Y3_X3_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X5]	= mmSRAM_Y3_X5_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X6]	= mmSRAM_Y3_X6_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SRAM_Y3_X7]	= mmSRAM_Y3_X7_FUNNEL_BASE,
+	[GAUDI_FUNNEL_SIF_0]		= mmSIF_FUNNEL_0_BASE,
+	[GAUDI_FUNNEL_SIF_1]		= mmSIF_FUNNEL_1_BASE,
+	[GAUDI_FUNNEL_SIF_2]		= mmSIF_FUNNEL_2_BASE,
+	[GAUDI_FUNNEL_SIF_3]		= mmSIF_FUNNEL_3_BASE,
+	[GAUDI_FUNNEL_SIF_4]		= mmSIF_FUNNEL_4_BASE,
+	[GAUDI_FUNNEL_SIF_5]		= mmSIF_FUNNEL_5_BASE,
+	[GAUDI_FUNNEL_SIF_6]		= mmSIF_FUNNEL_6_BASE,
+	[GAUDI_FUNNEL_SIF_7]		= mmSIF_FUNNEL_7_BASE,
+	[GAUDI_FUNNEL_NIF_0]		= mmNIF_FUNNEL_0_BASE,
+	[GAUDI_FUNNEL_NIF_1]		= mmNIF_FUNNEL_1_BASE,
+	[GAUDI_FUNNEL_NIF_2]		= mmNIF_FUNNEL_2_BASE,
+	[GAUDI_FUNNEL_NIF_3]		= mmNIF_FUNNEL_3_BASE,
+	[GAUDI_FUNNEL_NIF_4]		= mmNIF_FUNNEL_4_BASE,
+	[GAUDI_FUNNEL_NIF_5]		= mmNIF_FUNNEL_5_BASE,
+	[GAUDI_FUNNEL_NIF_6]		= mmNIF_FUNNEL_6_BASE,
+	[GAUDI_FUNNEL_NIF_7]		= mmNIF_FUNNEL_7_BASE,
+	[GAUDI_FUNNEL_DMA_IF_W_S]	= mmDMA_IF_W_S_FUNNEL_BASE,
+	[GAUDI_FUNNEL_DMA_IF_E_S]	= mmDMA_IF_E_S_FUNNEL_BASE,
+	[GAUDI_FUNNEL_DMA_IF_W_N]	= mmDMA_IF_W_N_FUNNEL_BASE,
+	[GAUDI_FUNNEL_DMA_IF_E_N]	= mmDMA_IF_E_N_FUNNEL_BASE,
+	[GAUDI_FUNNEL_CPU]		= mmCPU_FUNNEL_BASE,
+	[GAUDI_FUNNEL_NIC_TPC_W_S]	= mmNIC_TPC_FUNNEL_W_S_BASE,
+	[GAUDI_FUNNEL_NIC_TPC_E_S]	= mmNIC_TPC_FUNNEL_E_S_BASE,
+	[GAUDI_FUNNEL_NIC_TPC_W_N]	= mmNIC_TPC_FUNNEL_W_N_BASE,
+	[GAUDI_FUNNEL_NIC_TPC_E_N]	= mmNIC_TPC_FUNNEL_E_N_BASE,
+	[GAUDI_FUNNEL_PCIE]		= mmPCIE_FUNNEL_BASE,
+	[GAUDI_FUNNEL_PSOC]		= mmPSOC_FUNNEL_BASE,
+	[GAUDI_FUNNEL_NIC0]		= mmFUNNEL_NIC0_DBG_BASE,
+	[GAUDI_FUNNEL_NIC1]		= mmFUNNEL_NIC1_DBG_BASE,
+	[GAUDI_FUNNEL_NIC2]		= mmFUNNEL_NIC2_DBG_BASE,
+	[GAUDI_FUNNEL_NIC3]		= mmFUNNEL_NIC3_DBG_BASE,
+	[GAUDI_FUNNEL_NIC4]		= mmFUNNEL_NIC4_DBG_BASE,
+	[GAUDI_FUNNEL_TPC0_EML]		= mmTPC0_EML_FUNNEL_BASE,
+	[GAUDI_FUNNEL_TPC1_EML]		= mmTPC1_EML_FUNNEL_BASE,
+	[GAUDI_FUNNEL_TPC2_EML]		= mmTPC2_EML_FUNNEL_BASE,
+	[GAUDI_FUNNEL_TPC3_EML]		= mmTPC3_EML_FUNNEL_BASE,
+	[GAUDI_FUNNEL_TPC4_EML]		= mmTPC4_EML_FUNNEL_BASE,
+	[GAUDI_FUNNEL_TPC5_EML]		= mmTPC5_EML_FUNNEL_BASE,
+	[GAUDI_FUNNEL_TPC6_EML]		= mmTPC6_EML_FUNNEL_BASE,
+	[GAUDI_FUNNEL_TPC7_EML]		= mmTPC7_EML_FUNNEL_BASE
+};
+
+static u64 debug_bmon_regs[GAUDI_BMON_LAST + 1] = {
+	[GAUDI_BMON_MME0_ACC_0]		= mmMME0_ACC_BMON0_BASE,
+	[GAUDI_BMON_MME0_SBAB_0]	= mmMME0_SBAB_BMON0_BASE,
+	[GAUDI_BMON_MME0_SBAB_1]	= mmMME0_SBAB_BMON1_BASE,
+	[GAUDI_BMON_MME0_CTRL_0]	= mmMME0_CTRL_BMON0_BASE,
+	[GAUDI_BMON_MME0_CTRL_1]	= mmMME0_CTRL_BMON1_BASE,
+	[GAUDI_BMON_MME1_ACC_0]		= mmMME1_ACC_BMON0_BASE,
+	[GAUDI_BMON_MME1_SBAB_0]	= mmMME1_SBAB_BMON0_BASE,
+	[GAUDI_BMON_MME1_SBAB_1]	= mmMME1_SBAB_BMON1_BASE,
+	[GAUDI_BMON_MME1_CTRL_0]	= mmMME1_CTRL_BMON0_BASE,
+	[GAUDI_BMON_MME1_CTRL_1]	= mmMME1_CTRL_BMON1_BASE,
+	[GAUDI_BMON_MME2_ACC_0]		= mmMME2_ACC_BMON0_BASE,
+	[GAUDI_BMON_MME2_SBAB_0]	= mmMME2_SBAB_BMON0_BASE,
+	[GAUDI_BMON_MME2_SBAB_1]	= mmMME2_SBAB_BMON1_BASE,
+	[GAUDI_BMON_MME2_CTRL_0]	= mmMME2_CTRL_BMON0_BASE,
+	[GAUDI_BMON_MME2_CTRL_1]	= mmMME2_CTRL_BMON1_BASE,
+	[GAUDI_BMON_MME3_ACC_0]		= mmMME3_ACC_BMON0_BASE,
+	[GAUDI_BMON_MME3_SBAB_0]	= mmMME3_SBAB_BMON0_BASE,
+	[GAUDI_BMON_MME3_SBAB_1]	= mmMME3_SBAB_BMON1_BASE,
+	[GAUDI_BMON_MME3_CTRL_0]	= mmMME3_CTRL_BMON0_BASE,
+	[GAUDI_BMON_MME3_CTRL_1]	= mmMME3_CTRL_BMON1_BASE,
+	[GAUDI_BMON_DMA_IF_W_S_SOB_WR]	= mmDMA_IF_W_S_SOB_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_S_0_WR]	= mmDMA_IF_W_S_HBM0_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_S_0_RD]	= mmDMA_IF_W_S_HBM0_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_S_1_WR]	= mmDMA_IF_W_S_HBM1_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_S_1_RD]	= mmDMA_IF_W_S_HBM1_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_S_SOB_WR]	= mmDMA_IF_E_S_SOB_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_S_0_WR]	= mmDMA_IF_E_S_HBM0_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_S_0_RD]	= mmDMA_IF_E_S_HBM0_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_S_1_WR]	= mmDMA_IF_E_S_HBM1_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_S_1_RD]	= mmDMA_IF_E_S_HBM1_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_N_SOB_WR]	= mmDMA_IF_W_N_SOB_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_N_HBM0_WR]	= mmDMA_IF_W_N_HBM0_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_N_HBM0_RD]	= mmDMA_IF_W_N_HBM0_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_N_HBM1_WR]	= mmDMA_IF_W_N_HBM1_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_W_N_HBM1_RD]	= mmDMA_IF_W_N_HBM1_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_N_SOB_WR]	= mmDMA_IF_E_N_SOB_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_N_HBM0_WR]	= mmDMA_IF_E_N_HBM0_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_N_HBM0_RD]	= mmDMA_IF_E_N_HBM0_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_N_HBM1_WR]	= mmDMA_IF_E_N_HBM1_WR_BMON_BASE,
+	[GAUDI_BMON_DMA_IF_E_N_HBM1_RD]	= mmDMA_IF_E_N_HBM1_RD_BMON_BASE,
+	[GAUDI_BMON_CPU_WR]		= mmCPU_WR_BMON_BASE,
+	[GAUDI_BMON_CPU_RD]		= mmCPU_RD_BMON_BASE,
+	[GAUDI_BMON_DMA_CH_0_0]		= mmDMA_CH_0_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_0_1]		= mmDMA_CH_0_BMON_1_BASE,
+	[GAUDI_BMON_DMA_CH_1_0]		= mmDMA_CH_1_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_1_1]		= mmDMA_CH_1_BMON_1_BASE,
+	[GAUDI_BMON_DMA_CH_2_0]		= mmDMA_CH_2_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_2_1]		= mmDMA_CH_2_BMON_1_BASE,
+	[GAUDI_BMON_DMA_CH_3_0]		= mmDMA_CH_3_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_3_1]		= mmDMA_CH_3_BMON_1_BASE,
+	[GAUDI_BMON_DMA_CH_4_0]		= mmDMA_CH_4_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_4_1]		= mmDMA_CH_4_BMON_1_BASE,
+	[GAUDI_BMON_DMA_CH_5_0]		= mmDMA_CH_5_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_5_1]		= mmDMA_CH_5_BMON_1_BASE,
+	[GAUDI_BMON_DMA_CH_6_0]		= mmDMA_CH_6_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_6_1]		= mmDMA_CH_6_BMON_1_BASE,
+	[GAUDI_BMON_DMA_CH_7_0]		= mmDMA_CH_7_BMON_0_BASE,
+	[GAUDI_BMON_DMA_CH_7_1]		= mmDMA_CH_7_BMON_1_BASE,
+	[GAUDI_BMON_PCIE_MSTR_WR]	= mmPCIE_BMON_MSTR_WR_BASE,
+	[GAUDI_BMON_PCIE_MSTR_RD]	= mmPCIE_BMON_MSTR_RD_BASE,
+	[GAUDI_BMON_PCIE_SLV_WR]	= mmPCIE_BMON_SLV_WR_BASE,
+	[GAUDI_BMON_PCIE_SLV_RD]	= mmPCIE_BMON_SLV_RD_BASE,
+	[GAUDI_BMON_MMU_0]		= mmMMU_BMON_0_BASE,
+	[GAUDI_BMON_MMU_1]		= mmMMU_BMON_1_BASE,
+	[GAUDI_BMON_NIC0_0]		= mmBMON0_NIC0_DBG_BASE,
+	[GAUDI_BMON_NIC0_1]		= mmBMON1_NIC0_DBG_BASE,
+	[GAUDI_BMON_NIC0_2]		= mmBMON2_NIC0_DBG_BASE,
+	[GAUDI_BMON_NIC0_3]		= mmBMON3_NIC0_DBG_BASE,
+	[GAUDI_BMON_NIC0_4]		= mmBMON4_NIC0_DBG_BASE,
+	[GAUDI_BMON_NIC1_0]		= mmBMON0_NIC1_DBG_BASE,
+	[GAUDI_BMON_NIC1_1]		= mmBMON1_NIC1_DBG_BASE,
+	[GAUDI_BMON_NIC1_2]		= mmBMON2_NIC1_DBG_BASE,
+	[GAUDI_BMON_NIC1_3]		= mmBMON3_NIC1_DBG_BASE,
+	[GAUDI_BMON_NIC1_4]		= mmBMON4_NIC1_DBG_BASE,
+	[GAUDI_BMON_NIC2_0]		= mmBMON0_NIC2_DBG_BASE,
+	[GAUDI_BMON_NIC2_1]		= mmBMON1_NIC2_DBG_BASE,
+	[GAUDI_BMON_NIC2_2]		= mmBMON2_NIC2_DBG_BASE,
+	[GAUDI_BMON_NIC2_3]		= mmBMON3_NIC2_DBG_BASE,
+	[GAUDI_BMON_NIC2_4]		= mmBMON4_NIC2_DBG_BASE,
+	[GAUDI_BMON_NIC3_0]		= mmBMON0_NIC3_DBG_BASE,
+	[GAUDI_BMON_NIC3_1]		= mmBMON1_NIC3_DBG_BASE,
+	[GAUDI_BMON_NIC3_2]		= mmBMON2_NIC3_DBG_BASE,
+	[GAUDI_BMON_NIC3_3]		= mmBMON3_NIC3_DBG_BASE,
+	[GAUDI_BMON_NIC3_4]		= mmBMON4_NIC3_DBG_BASE,
+	[GAUDI_BMON_NIC4_0]		= mmBMON0_NIC4_DBG_BASE,
+	[GAUDI_BMON_NIC4_1]		= mmBMON1_NIC4_DBG_BASE,
+	[GAUDI_BMON_NIC4_2]		= mmBMON2_NIC4_DBG_BASE,
+	[GAUDI_BMON_NIC4_3]		= mmBMON3_NIC4_DBG_BASE,
+	[GAUDI_BMON_NIC4_4]		= mmBMON4_NIC4_DBG_BASE,
+	[GAUDI_BMON_TPC0_EML_0]		= mmTPC0_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC0_EML_1]		= mmTPC0_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC0_EML_2]		= mmTPC0_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC0_EML_3]		= mmTPC0_EML_BUSMON_3_BASE,
+	[GAUDI_BMON_TPC1_EML_0]		= mmTPC1_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC1_EML_1]		= mmTPC1_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC1_EML_2]		= mmTPC1_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC1_EML_3]		= mmTPC1_EML_BUSMON_3_BASE,
+	[GAUDI_BMON_TPC2_EML_0]		= mmTPC2_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC2_EML_1]		= mmTPC2_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC2_EML_2]		= mmTPC2_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC2_EML_3]		= mmTPC2_EML_BUSMON_3_BASE,
+	[GAUDI_BMON_TPC3_EML_0]		= mmTPC3_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC3_EML_1]		= mmTPC3_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC3_EML_2]		= mmTPC3_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC3_EML_3]		= mmTPC3_EML_BUSMON_3_BASE,
+	[GAUDI_BMON_TPC4_EML_0]		= mmTPC4_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC4_EML_1]		= mmTPC4_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC4_EML_2]		= mmTPC4_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC4_EML_3]		= mmTPC4_EML_BUSMON_3_BASE,
+	[GAUDI_BMON_TPC5_EML_0]		= mmTPC5_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC5_EML_1]		= mmTPC5_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC5_EML_2]		= mmTPC5_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC5_EML_3]		= mmTPC5_EML_BUSMON_3_BASE,
+	[GAUDI_BMON_TPC6_EML_0]		= mmTPC6_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC6_EML_1]		= mmTPC6_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC6_EML_2]		= mmTPC6_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC6_EML_3]		= mmTPC6_EML_BUSMON_3_BASE,
+	[GAUDI_BMON_TPC7_EML_0]		= mmTPC7_EML_BUSMON_0_BASE,
+	[GAUDI_BMON_TPC7_EML_1]		= mmTPC7_EML_BUSMON_1_BASE,
+	[GAUDI_BMON_TPC7_EML_2]		= mmTPC7_EML_BUSMON_2_BASE,
+	[GAUDI_BMON_TPC7_EML_3]		= mmTPC7_EML_BUSMON_3_BASE
+};
+
+static u64 debug_spmu_regs[GAUDI_SPMU_LAST + 1] = {
+	[GAUDI_SPMU_MME0_ACC]		= mmMME0_ACC_SPMU_BASE,
+	[GAUDI_SPMU_MME0_SBAB]		= mmMME0_SBAB_SPMU_BASE,
+	[GAUDI_SPMU_MME0_CTRL]		= mmMME0_CTRL_SPMU_BASE,
+	[GAUDI_SPMU_MME1_ACC]		= mmMME1_ACC_SPMU_BASE,
+	[GAUDI_SPMU_MME1_SBAB]		= mmMME1_SBAB_SPMU_BASE,
+	[GAUDI_SPMU_MME1_CTRL]		= mmMME1_CTRL_SPMU_BASE,
+	[GAUDI_SPMU_MME2_MME2_ACC]	= mmMME2_ACC_SPMU_BASE,
+	[GAUDI_SPMU_MME2_SBAB]		= mmMME2_SBAB_SPMU_BASE,
+	[GAUDI_SPMU_MME2_CTRL]		= mmMME2_CTRL_SPMU_BASE,
+	[GAUDI_SPMU_MME3_ACC]		= mmMME3_ACC_SPMU_BASE,
+	[GAUDI_SPMU_MME3_SBAB]		= mmMME3_SBAB_SPMU_BASE,
+	[GAUDI_SPMU_MME3_CTRL]		= mmMME3_CTRL_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_0_CS]	= mmDMA_CH_0_CS_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_1_CS]	= mmDMA_CH_1_CS_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_2_CS]	= mmDMA_CH_2_CS_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_3_CS]	= mmDMA_CH_3_CS_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_4_CS]	= mmDMA_CH_4_CS_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_5_CS]	= mmDMA_CH_5_CS_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_6_CS]	= mmDMA_CH_6_CS_SPMU_BASE,
+	[GAUDI_SPMU_DMA_CH_7_CS]	= mmDMA_CH_7_CS_SPMU_BASE,
+	[GAUDI_SPMU_PCIE]		= mmPCIE_SPMU_BASE,
+	[GAUDI_SPMU_MMU_CS]		= mmMMU_CS_SPMU_BASE,
+	[GAUDI_SPMU_NIC0_0]		= mmSPMU_0_NIC0_DBG_BASE,
+	[GAUDI_SPMU_NIC0_1]		= mmSPMU_1_NIC0_DBG_BASE,
+	[GAUDI_SPMU_NIC1_0]		= mmSPMU_0_NIC1_DBG_BASE,
+	[GAUDI_SPMU_NIC1_1]		= mmSPMU_1_NIC1_DBG_BASE,
+	[GAUDI_SPMU_NIC2_0]		= mmSPMU_0_NIC2_DBG_BASE,
+	[GAUDI_SPMU_NIC2_1]		= mmSPMU_1_NIC2_DBG_BASE,
+	[GAUDI_SPMU_NIC3_0]		= mmSPMU_0_NIC3_DBG_BASE,
+	[GAUDI_SPMU_NIC3_1]		= mmSPMU_1_NIC3_DBG_BASE,
+	[GAUDI_SPMU_NIC4_0]		= mmSPMU_0_NIC4_DBG_BASE,
+	[GAUDI_SPMU_NIC4_1]		= mmSPMU_1_NIC4_DBG_BASE,
+	[GAUDI_SPMU_TPC0_EML]		= mmTPC0_EML_SPMU_BASE,
+	[GAUDI_SPMU_TPC1_EML]		= mmTPC1_EML_SPMU_BASE,
+	[GAUDI_SPMU_TPC2_EML]		= mmTPC2_EML_SPMU_BASE,
+	[GAUDI_SPMU_TPC3_EML]		= mmTPC3_EML_SPMU_BASE,
+	[GAUDI_SPMU_TPC4_EML]		= mmTPC4_EML_SPMU_BASE,
+	[GAUDI_SPMU_TPC5_EML]		= mmTPC5_EML_SPMU_BASE,
+	[GAUDI_SPMU_TPC6_EML]		= mmTPC6_EML_SPMU_BASE,
+	[GAUDI_SPMU_TPC7_EML]		= mmTPC7_EML_SPMU_BASE
+};
+
+static int gaudi_coresight_timeout(struct hl_device *hdev, u64 addr,
+		int position, bool up)
+{
+	int rc;
+	u32 val;
+
+	rc = hl_poll_timeout(
+		hdev,
+		addr,
+		val,
+		up ? val & BIT(position) : !(val & BIT(position)),
+		1000,
+		CORESIGHT_TIMEOUT_USEC);
+
+	if (rc) {
+		dev_err(hdev->dev,
+			"Timeout while waiting for coresight, addr: 0x%llx, position: %d, up: %d\n",
+				addr, position, up);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static int gaudi_config_stm(struct hl_device *hdev,
+		struct hl_debug_params *params)
+{
+	struct hl_debug_params_stm *input;
+	u64 base_reg;
+	int rc;
+
+	if (params->reg_idx >= ARRAY_SIZE(debug_stm_regs)) {
+		dev_err(hdev->dev, "Invalid register index in STM\n");
+		return -EINVAL;
+	}
+
+	base_reg = debug_stm_regs[params->reg_idx] - CFG_BASE;
+
+	WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+
+	if (params->enable) {
+		input = params->input;
+
+		if (!input)
+			return -EINVAL;
+
+		WREG32(base_reg + 0xE80, 0x80004);
+		WREG32(base_reg + 0xD64, 7);
+		WREG32(base_reg + 0xD60, 0);
+		WREG32(base_reg + 0xD00, lower_32_bits(input->he_mask));
+		WREG32(base_reg + 0xD60, 1);
+		WREG32(base_reg + 0xD00, upper_32_bits(input->he_mask));
+		WREG32(base_reg + 0xE70, 0x10);
+		WREG32(base_reg + 0xE60, 0);
+		WREG32(base_reg + 0xE00, lower_32_bits(input->sp_mask));
+		WREG32(base_reg + 0xEF4, input->id);
+		WREG32(base_reg + 0xDF4, 0x80);
+		WREG32(base_reg + 0xE8C, input->frequency);
+		WREG32(base_reg + 0xE90, 0x7FF);
+
+		/* SW-2176 - SW WA for HW bug */
+		if ((CFG_BASE + base_reg) >= mmDMA_CH_0_CS_STM_BASE &&
+			(CFG_BASE + base_reg) <= mmDMA_CH_7_CS_STM_BASE) {
+
+			WREG32(base_reg + 0xE68, 0xffff8005);
+			WREG32(base_reg + 0xE6C, 0x0);
+		}
+
+		WREG32(base_reg + 0xE80, 0x27 | (input->id << 16));
+	} else {
+		WREG32(base_reg + 0xE80, 4);
+		WREG32(base_reg + 0xD64, 0);
+		WREG32(base_reg + 0xD60, 1);
+		WREG32(base_reg + 0xD00, 0);
+		WREG32(base_reg + 0xD20, 0);
+		WREG32(base_reg + 0xD60, 0);
+		WREG32(base_reg + 0xE20, 0);
+		WREG32(base_reg + 0xE00, 0);
+		WREG32(base_reg + 0xDF4, 0x80);
+		WREG32(base_reg + 0xE70, 0);
+		WREG32(base_reg + 0xE60, 0);
+		WREG32(base_reg + 0xE64, 0);
+		WREG32(base_reg + 0xE8C, 0);
+
+		rc = gaudi_coresight_timeout(hdev, base_reg + 0xE80, 23, false);
+		if (rc) {
+			dev_err(hdev->dev,
+				"Failed to disable STM on timeout, error %d\n",
+				rc);
+			return rc;
+		}
+
+		WREG32(base_reg + 0xE80, 4);
+	}
+
+	return 0;
+}
+
+static int gaudi_config_etf(struct hl_device *hdev,
+		struct hl_debug_params *params)
+{
+	struct hl_debug_params_etf *input;
+	u64 base_reg;
+	u32 val;
+	int rc;
+
+	if (params->reg_idx >= ARRAY_SIZE(debug_etf_regs)) {
+		dev_err(hdev->dev, "Invalid register index in ETF\n");
+		return -EINVAL;
+	}
+
+	base_reg = debug_etf_regs[params->reg_idx] - CFG_BASE;
+
+	WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+
+	val = RREG32(base_reg + 0x304);
+	val |= 0x1000;
+	WREG32(base_reg + 0x304, val);
+	val |= 0x40;
+	WREG32(base_reg + 0x304, val);
+
+	rc = gaudi_coresight_timeout(hdev, base_reg + 0x304, 6, false);
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to %s ETF on timeout, error %d\n",
+				params->enable ? "enable" : "disable", rc);
+		return rc;
+	}
+
+	rc = gaudi_coresight_timeout(hdev, base_reg + 0xC, 2, true);
+	if (rc) {
+		dev_err(hdev->dev,
+			"Failed to %s ETF on timeout, error %d\n",
+				params->enable ? "enable" : "disable", rc);
+		return rc;
+	}
+
+	WREG32(base_reg + 0x20, 0);
+
+	if (params->enable) {
+		input = params->input;
+
+		if (!input)
+			return -EINVAL;
+
+		WREG32(base_reg + 0x34, 0x3FFC);
+		WREG32(base_reg + 0x28, input->sink_mode);
+		WREG32(base_reg + 0x304, 0x4001);
+		WREG32(base_reg + 0x308, 0xA);
+		WREG32(base_reg + 0x20, 1);
+	} else {
+		WREG32(base_reg + 0x34, 0);
+		WREG32(base_reg + 0x28, 0);
+		WREG32(base_reg + 0x304, 0);
+	}
+
+	return 0;
+}
+
+static bool gaudi_etr_validate_address(struct hl_device *hdev, u64 addr,
+					u32 size, bool *is_host)
+{
+	struct asic_fixed_properties *prop = &hdev->asic_prop;
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	/* maximum address length is 50 bits */
+	if (addr >> 50) {
+		dev_err(hdev->dev,
+			"ETR buffer address shouldn't exceed 50 bits\n");
+		return false;
+	}
+
+	/* PMMU and HPMMU addresses are equal, check only one of them */
+	if ((gaudi->hw_cap_initialized & HW_CAP_MMU) &&
+		hl_mem_area_inside_range(addr, size,
+				prop->pmmu.start_addr,
+				prop->pmmu.end_addr)) {
+		*is_host = true;
+		return true;
+	}
+
+	if (hl_mem_area_inside_range(addr, size,
+			prop->dram_user_base_address,
+			prop->dram_end_address))
+		return true;
+
+	if (hl_mem_area_inside_range(addr, size,
+			prop->sram_user_base_address,
+			prop->sram_end_address))
+		return true;
+
+	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
+		dev_err(hdev->dev, "ETR buffer should be in SRAM/DRAM\n");
+
+	return false;
+}
+
+static int gaudi_config_etr(struct hl_device *hdev,
+		struct hl_debug_params *params)
+{
+	struct hl_debug_params_etr *input;
+	u64 msb;
+	u32 val;
+	int rc;
+
+	WREG32(mmPSOC_ETR_LAR, CORESIGHT_UNLOCK);
+
+	val = RREG32(mmPSOC_ETR_FFCR);
+	val |= 0x1000;
+	WREG32(mmPSOC_ETR_FFCR, val);
+	val |= 0x40;
+	WREG32(mmPSOC_ETR_FFCR, val);
+
+	rc = gaudi_coresight_timeout(hdev, mmPSOC_ETR_FFCR, 6, false);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to %s ETR on timeout, error %d\n",
+				params->enable ? "enable" : "disable", rc);
+		return rc;
+	}
+
+	rc = gaudi_coresight_timeout(hdev, mmPSOC_ETR_STS, 2, true);
+	if (rc) {
+		dev_err(hdev->dev, "Failed to %s ETR on timeout, error %d\n",
+				params->enable ? "enable" : "disable", rc);
+		return rc;
+	}
+
+	WREG32(mmPSOC_ETR_CTL, 0);
+
+	if (params->enable) {
+		bool is_host = false;
+
+		input = params->input;
+
+		if (!input)
+			return -EINVAL;
+
+		if (input->buffer_size == 0) {
+			dev_err(hdev->dev,
+				"ETR buffer size should be bigger than 0\n");
+			return -EINVAL;
+		}
+
+		if (!gaudi_etr_validate_address(hdev,
+				input->buffer_address, input->buffer_size,
+				&is_host)) {
+			dev_err(hdev->dev, "ETR buffer address is invalid\n");
+			return -EINVAL;
+		}
+
+		msb = upper_32_bits(input->buffer_address) >> 8;
+		msb &= PSOC_GLOBAL_CONF_TRACE_ADDR_MSB_MASK;
+		WREG32(mmPSOC_GLOBAL_CONF_TRACE_ADDR, msb);
+
+		WREG32(mmPSOC_ETR_BUFWM, 0x3FFC);
+		WREG32(mmPSOC_ETR_RSZ, input->buffer_size);
+		WREG32(mmPSOC_ETR_MODE, input->sink_mode);
+		/* Workaround for H3 #HW-2075 bug: use small data chunks */
+		WREG32(mmPSOC_ETR_AXICTL, (is_host ? 0 : 0x700) |
+					PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT);
+		WREG32(mmPSOC_ETR_DBALO,
+				lower_32_bits(input->buffer_address));
+		WREG32(mmPSOC_ETR_DBAHI,
+				upper_32_bits(input->buffer_address));
+		WREG32(mmPSOC_ETR_FFCR, 3);
+		WREG32(mmPSOC_ETR_PSCR, 0xA);
+		WREG32(mmPSOC_ETR_CTL, 1);
+	} else {
+		WREG32(mmPSOC_ETR_BUFWM, 0);
+		WREG32(mmPSOC_ETR_RSZ, 0x400);
+		WREG32(mmPSOC_ETR_DBALO, 0);
+		WREG32(mmPSOC_ETR_DBAHI, 0);
+		WREG32(mmPSOC_ETR_PSCR, 0);
+		WREG32(mmPSOC_ETR_MODE, 0);
+		WREG32(mmPSOC_ETR_FFCR, 0);
+
+		if (params->output_size >= sizeof(u64)) {
+			u32 rwp, rwphi;
+
+			/*
+			 * The trace buffer address is 50 bits wide. The end of
+			 * the buffer is set in the RWP register (lower 32
+			 * bits), and in the RWPHI register (upper 8 bits).
+			 * The 10 msb of the 50-bit address are stored in a
+			 * global configuration register.
+			 */
+			rwp = RREG32(mmPSOC_ETR_RWP);
+			rwphi = RREG32(mmPSOC_ETR_RWPHI) & 0xff;
+			msb = RREG32(mmPSOC_GLOBAL_CONF_TRACE_ADDR) &
+					PSOC_GLOBAL_CONF_TRACE_ADDR_MSB_MASK;
+			*(u64 *) params->output = ((u64) msb << 40) |
+						((u64) rwphi << 32) | rwp;
+		}
+	}
+
+	return 0;
+}
+
+static int gaudi_config_funnel(struct hl_device *hdev,
+		struct hl_debug_params *params)
+{
+	u64 base_reg;
+
+	if (params->reg_idx >= ARRAY_SIZE(debug_funnel_regs)) {
+		dev_err(hdev->dev, "Invalid register index in FUNNEL\n");
+		return -EINVAL;
+	}
+
+	base_reg = debug_funnel_regs[params->reg_idx] - CFG_BASE;
+
+	WREG32(base_reg + 0xFB0, CORESIGHT_UNLOCK);
+
+	WREG32(base_reg, params->enable ? 0x33F : 0);
+
+	return 0;
+}
+
+static int gaudi_config_bmon(struct hl_device *hdev,
+		struct hl_debug_params *params)
+{
+	struct hl_debug_params_bmon *input;
+	u64 base_reg;
+
+	if (params->reg_idx >= ARRAY_SIZE(debug_bmon_regs)) {
+		dev_err(hdev->dev, "Invalid register index in BMON\n");
+		return -EINVAL;
+	}
+
+	base_reg = debug_bmon_regs[params->reg_idx] - CFG_BASE;
+
+	WREG32(base_reg + 0x104, 1);
+
+	if (params->enable) {
+		input = params->input;
+
+		if (!input)
+			return -EINVAL;
+
+		WREG32(base_reg + 0x200, lower_32_bits(input->start_addr0));
+		WREG32(base_reg + 0x204, upper_32_bits(input->start_addr0));
+		WREG32(base_reg + 0x208, lower_32_bits(input->addr_mask0));
+		WREG32(base_reg + 0x20C, upper_32_bits(input->addr_mask0));
+		WREG32(base_reg + 0x240, lower_32_bits(input->start_addr1));
+		WREG32(base_reg + 0x244, upper_32_bits(input->start_addr1));
+		WREG32(base_reg + 0x248, lower_32_bits(input->addr_mask1));
+		WREG32(base_reg + 0x24C, upper_32_bits(input->addr_mask1));
+		WREG32(base_reg + 0x224, 0);
+		WREG32(base_reg + 0x234, 0);
+		WREG32(base_reg + 0x30C, input->bw_win);
+		WREG32(base_reg + 0x308, input->win_capture);
+		WREG32(base_reg + 0x700, 0xA000B00 | (input->id << 12));
+		WREG32(base_reg + 0x708, 0xA000A00 | (input->id << 12));
+		WREG32(base_reg + 0x70C, 0xA000C00 | (input->id << 12));
+		WREG32(base_reg + 0x100, 0x11);
+		WREG32(base_reg + 0x304, 0x1);
+	} else {
+		WREG32(base_reg + 0x200, 0);
+		WREG32(base_reg + 0x204, 0);
+		WREG32(base_reg + 0x208, 0xFFFFFFFF);
+		WREG32(base_reg + 0x20C, 0xFFFFFFFF);
+		WREG32(base_reg + 0x240, 0);
+		WREG32(base_reg + 0x244, 0);
+		WREG32(base_reg + 0x248, 0xFFFFFFFF);
+		WREG32(base_reg + 0x24C, 0xFFFFFFFF);
+		WREG32(base_reg + 0x224, 0xFFFFFFFF);
+		WREG32(base_reg + 0x234, 0x1070F);
+		WREG32(base_reg + 0x30C, 0);
+		WREG32(base_reg + 0x308, 0xFFFF);
+		WREG32(base_reg + 0x700, 0xA000B00);
+		WREG32(base_reg + 0x708, 0xA000A00);
+		WREG32(base_reg + 0x70C, 0xA000C00);
+		WREG32(base_reg + 0x100, 1);
+		WREG32(base_reg + 0x304, 0);
+		WREG32(base_reg + 0x104, 0);
+	}
+
+	return 0;
+}
+
+static int gaudi_config_spmu(struct hl_device *hdev,
+		struct hl_debug_params *params)
+{
+	u64 base_reg;
+	struct hl_debug_params_spmu *input = params->input;
+	u64 *output;
+	u32 output_arr_len;
+	u32 events_num;
+	u32 overflow_idx;
+	u32 cycle_cnt_idx;
+	int i;
+
+	if (params->reg_idx >= ARRAY_SIZE(debug_spmu_regs)) {
+		dev_err(hdev->dev, "Invalid register index in SPMU\n");
+		return -EINVAL;
+	}
+
+	base_reg = debug_spmu_regs[params->reg_idx] - CFG_BASE;
+
+	if (params->enable) {
+		input = params->input;
+
+		if (!input)
+			return -EINVAL;
+
+		if (input->event_types_num < 3) {
+			dev_err(hdev->dev,
+				"not enough event types values for SPMU enable\n");
+			return -EINVAL;
+		}
+
+		if (input->event_types_num > SPMU_MAX_COUNTERS) {
+			dev_err(hdev->dev,
+				"too many event types values for SPMU enable\n");
+			return -EINVAL;
+		}
+
+		WREG32(base_reg + 0xE04, 0x41013046);
+		WREG32(base_reg + 0xE04, 0x41013040);
+
+		for (i = 0 ; i < input->event_types_num ; i++)
+			WREG32(base_reg + SPMU_EVENT_TYPES_OFFSET + i * 4,
+				input->event_types[i]);
+
+		WREG32(base_reg + 0xE04, 0x41013041);
+		WREG32(base_reg + 0xC00, 0x8000003F);
+	} else {
+		output = params->output;
+		output_arr_len = params->output_size / 8;
+		events_num = output_arr_len - 2;
+		overflow_idx = output_arr_len - 2;
+		cycle_cnt_idx = output_arr_len - 1;
+
+		if (!output)
+			return -EINVAL;
+
+		if (output_arr_len < 3) {
+			dev_err(hdev->dev,
+				"not enough values for SPMU disable\n");
+			return -EINVAL;
+		}
+
+		if (events_num > SPMU_MAX_COUNTERS) {
+			dev_err(hdev->dev,
+				"too many events values for SPMU disable\n");
+			return -EINVAL;
+		}
+
+		WREG32(base_reg + 0xE04, 0x41013040);
+
+		for (i = 0 ; i < events_num ; i++)
+			output[i] = RREG32(base_reg + i * 8);
+
+		output[overflow_idx] = RREG32(base_reg + 0xCC0);
+
+		output[cycle_cnt_idx] = RREG32(base_reg + 0xFC);
+		output[cycle_cnt_idx] <<= 32;
+		output[cycle_cnt_idx] |= RREG32(base_reg + 0xF8);
+
+		WREG32(base_reg + 0xCC0, 0);
+	}
+
+	return 0;
+}
+
+int gaudi_debug_coresight(struct hl_device *hdev, void *data)
+{
+	struct hl_debug_params *params = data;
+	int rc = 0;
+
+	switch (params->op) {
+	case HL_DEBUG_OP_STM:
+		rc = gaudi_config_stm(hdev, params);
+		break;
+	case HL_DEBUG_OP_ETF:
+		rc = gaudi_config_etf(hdev, params);
+		break;
+	case HL_DEBUG_OP_ETR:
+		rc = gaudi_config_etr(hdev, params);
+		break;
+	case HL_DEBUG_OP_FUNNEL:
+		rc = gaudi_config_funnel(hdev, params);
+		break;
+	case HL_DEBUG_OP_BMON:
+		rc = gaudi_config_bmon(hdev, params);
+		break;
+	case HL_DEBUG_OP_SPMU:
+		rc = gaudi_config_spmu(hdev, params);
+		break;
+	case HL_DEBUG_OP_TIMESTAMP:
+		/* Do nothing as this opcode is deprecated */
+		break;
+
+	default:
+		dev_err(hdev->dev, "Unknown coresight id %d\n", params->op);
+		return -EINVAL;
+	}
+
+	/* Perform read from the device to flush all configuration */
+	RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
+
+	return rc;
+}
+
+void gaudi_halt_coresight(struct hl_device *hdev)
+{
+	struct hl_debug_params params = {};
+	int i, rc;
+
+	for (i = GAUDI_ETF_FIRST ; i <= GAUDI_ETF_LAST ; i++) {
+		params.reg_idx = i;
+		rc = gaudi_config_etf(hdev, &params);
+		if (rc)
+			dev_err(hdev->dev, "halt ETF failed, %d/%d\n", rc, i);
+	}
+
+	rc = gaudi_config_etr(hdev, &params);
+	if (rc)
+		dev_err(hdev->dev, "halt ETR failed, %d\n", rc);
+}
diff --git a/drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c b/drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c
new file mode 100644
index 000000000000..6dd2c2a1cd70
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "gaudiP.h"
+#include "include/gaudi/gaudi_fw_if.h"
+
+void gaudi_set_pll_profile(struct hl_device *hdev, enum hl_pll_frequency freq)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	if (freq == PLL_LAST)
+		hl_set_frequency(hdev, MME_PLL, gaudi->max_freq_value);
+}
+
+int gaudi_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)
+{
+	long value;
+
+	if (hl_device_disabled_or_in_reset(hdev))
+		return -ENODEV;
+
+	value = hl_get_frequency(hdev, MME_PLL, false);
+
+	if (value < 0) {
+		dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n",
+			value);
+		return value;
+	}
+
+	*max_clk = (value / 1000 / 1000);
+
+	value = hl_get_frequency(hdev, MME_PLL, true);
+
+	if (value < 0) {
+		dev_err(hdev->dev,
+			"Failed to retrieve device current clock %ld\n",
+			value);
+		return value;
+	}
+
+	*cur_clk = (value / 1000 / 1000);
+
+	return 0;
+}
+
+static ssize_t clk_max_freq_mhz_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct hl_device *hdev = dev_get_drvdata(dev);
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	long value;
+
+	if (hl_device_disabled_or_in_reset(hdev))
+		return -ENODEV;
+
+	value = hl_get_frequency(hdev, MME_PLL, false);
+
+	gaudi->max_freq_value = value;
+
+	return sprintf(buf, "%lu\n", (value / 1000 / 1000));
+}
+
+static ssize_t clk_max_freq_mhz_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct hl_device *hdev = dev_get_drvdata(dev);
+	struct gaudi_device *gaudi = hdev->asic_specific;
+	int rc;
+	u64 value;
+
+	if (hl_device_disabled_or_in_reset(hdev)) {
+		count = -ENODEV;
+		goto fail;
+	}
+
+	rc = kstrtoull(buf, 0, &value);
+	if (rc) {
+		count = -EINVAL;
+		goto fail;
+	}
+
+	gaudi->max_freq_value = value * 1000 * 1000;
+
+	hl_set_frequency(hdev, MME_PLL, gaudi->max_freq_value);
+
+fail:
+	return count;
+}
+
+static ssize_t clk_cur_freq_mhz_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct hl_device *hdev = dev_get_drvdata(dev);
+	long value;
+
+	if (hl_device_disabled_or_in_reset(hdev))
+		return -ENODEV;
+
+	value = hl_get_frequency(hdev, MME_PLL, true);
+
+	return sprintf(buf, "%lu\n", (value / 1000 / 1000));
+}
+
+static DEVICE_ATTR_RW(clk_max_freq_mhz);
+static DEVICE_ATTR_RO(clk_cur_freq_mhz);
+
+static struct attribute *gaudi_dev_attrs[] = {
+	&dev_attr_clk_max_freq_mhz.attr,
+	&dev_attr_clk_cur_freq_mhz.attr,
+	NULL,
+};
+
+void gaudi_add_device_attr(struct hl_device *hdev,
+			struct attribute_group *dev_attr_grp)
+{
+	dev_attr_grp->attrs = gaudi_dev_attrs;
+}
diff --git a/drivers/misc/habanalabs/gaudi/gaudi_security.c b/drivers/misc/habanalabs/gaudi/gaudi_security.c
new file mode 100644
index 000000000000..6a351e31fa6a
--- /dev/null
+++ b/drivers/misc/habanalabs/gaudi/gaudi_security.c
@@ -0,0 +1,9090 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ */
+
+#include "gaudiP.h"
+#include "include/gaudi/asic_reg/gaudi_regs.h"
+
+#define GAUDI_NUMBER_OF_RR_REGS		24
+#define GAUDI_NUMBER_OF_LBW_RANGES	12
+
+static u64 gaudi_rr_lbw_hit_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DMA0_HIT_WPROT,
+	mmDMA_IF_W_S_DMA1_HIT_WPROT,
+	mmDMA_IF_E_S_DMA0_HIT_WPROT,
+	mmDMA_IF_E_S_DMA1_HIT_WPROT,
+	mmDMA_IF_W_N_DMA0_HIT_WPROT,
+	mmDMA_IF_W_N_DMA1_HIT_WPROT,
+	mmDMA_IF_E_N_DMA0_HIT_WPROT,
+	mmDMA_IF_E_N_DMA1_HIT_WPROT,
+	mmSIF_RTR_0_LBW_RANGE_PROT_HIT_AW,
+	mmSIF_RTR_1_LBW_RANGE_PROT_HIT_AW,
+	mmSIF_RTR_2_LBW_RANGE_PROT_HIT_AW,
+	mmSIF_RTR_3_LBW_RANGE_PROT_HIT_AW,
+	mmSIF_RTR_4_LBW_RANGE_PROT_HIT_AW,
+	mmSIF_RTR_5_LBW_RANGE_PROT_HIT_AW,
+	mmSIF_RTR_6_LBW_RANGE_PROT_HIT_AW,
+	mmSIF_RTR_7_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_0_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_1_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_2_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_3_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_4_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_5_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_6_LBW_RANGE_PROT_HIT_AW,
+	mmNIF_RTR_7_LBW_RANGE_PROT_HIT_AW,
+};
+
+static u64 gaudi_rr_lbw_hit_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DMA0_HIT_RPROT,
+	mmDMA_IF_W_S_DMA1_HIT_RPROT,
+	mmDMA_IF_E_S_DMA0_HIT_RPROT,
+	mmDMA_IF_E_S_DMA1_HIT_RPROT,
+	mmDMA_IF_W_N_DMA0_HIT_RPROT,
+	mmDMA_IF_W_N_DMA1_HIT_RPROT,
+	mmDMA_IF_E_N_DMA0_HIT_RPROT,
+	mmDMA_IF_E_N_DMA1_HIT_RPROT,
+	mmSIF_RTR_0_LBW_RANGE_PROT_HIT_AR,
+	mmSIF_RTR_1_LBW_RANGE_PROT_HIT_AR,
+	mmSIF_RTR_2_LBW_RANGE_PROT_HIT_AR,
+	mmSIF_RTR_3_LBW_RANGE_PROT_HIT_AR,
+	mmSIF_RTR_4_LBW_RANGE_PROT_HIT_AR,
+	mmSIF_RTR_5_LBW_RANGE_PROT_HIT_AR,
+	mmSIF_RTR_6_LBW_RANGE_PROT_HIT_AR,
+	mmSIF_RTR_7_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_0_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_1_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_2_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_3_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_4_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_5_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_6_LBW_RANGE_PROT_HIT_AR,
+	mmNIF_RTR_7_LBW_RANGE_PROT_HIT_AR,
+};
+
+static u64 gaudi_rr_lbw_min_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DMA0_MIN_WPROT_0,
+	mmDMA_IF_W_S_DMA1_MIN_WPROT_0,
+	mmDMA_IF_E_S_DMA0_MIN_WPROT_0,
+	mmDMA_IF_E_S_DMA1_MIN_WPROT_0,
+	mmDMA_IF_W_N_DMA0_MIN_WPROT_0,
+	mmDMA_IF_W_N_DMA1_MIN_WPROT_0,
+	mmDMA_IF_E_N_DMA0_MIN_WPROT_0,
+	mmDMA_IF_E_N_DMA1_MIN_WPROT_0,
+	mmSIF_RTR_0_LBW_RANGE_PROT_MIN_AW_0,
+	mmSIF_RTR_1_LBW_RANGE_PROT_MIN_AW_0,
+	mmSIF_RTR_2_LBW_RANGE_PROT_MIN_AW_0,
+	mmSIF_RTR_3_LBW_RANGE_PROT_MIN_AW_0,
+	mmSIF_RTR_4_LBW_RANGE_PROT_MIN_AW_0,
+	mmSIF_RTR_5_LBW_RANGE_PROT_MIN_AW_0,
+	mmSIF_RTR_6_LBW_RANGE_PROT_MIN_AW_0,
+	mmSIF_RTR_7_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_0_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_1_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_2_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_3_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_4_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_5_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_6_LBW_RANGE_PROT_MIN_AW_0,
+	mmNIF_RTR_7_LBW_RANGE_PROT_MIN_AW_0,
+};
+
+static u64 gaudi_rr_lbw_max_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DMA0_MAX_WPROT_0,
+	mmDMA_IF_W_S_DMA1_MAX_WPROT_0,
+	mmDMA_IF_E_S_DMA0_MAX_WPROT_0,
+	mmDMA_IF_E_S_DMA1_MAX_WPROT_0,
+	mmDMA_IF_W_N_DMA0_MAX_WPROT_0,
+	mmDMA_IF_W_N_DMA1_MAX_WPROT_0,
+	mmDMA_IF_E_N_DMA0_MAX_WPROT_0,
+	mmDMA_IF_E_N_DMA1_MAX_WPROT_0,
+	mmSIF_RTR_0_LBW_RANGE_PROT_MAX_AW_0,
+	mmSIF_RTR_1_LBW_RANGE_PROT_MAX_AW_0,
+	mmSIF_RTR_2_LBW_RANGE_PROT_MAX_AW_0,
+	mmSIF_RTR_3_LBW_RANGE_PROT_MAX_AW_0,
+	mmSIF_RTR_4_LBW_RANGE_PROT_MAX_AW_0,
+	mmSIF_RTR_5_LBW_RANGE_PROT_MAX_AW_0,
+	mmSIF_RTR_6_LBW_RANGE_PROT_MAX_AW_0,
+	mmSIF_RTR_7_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_0_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_1_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_2_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_3_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_4_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_5_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_6_LBW_RANGE_PROT_MAX_AW_0,
+	mmNIF_RTR_7_LBW_RANGE_PROT_MAX_AW_0,
+};
+
+static u64 gaudi_rr_lbw_min_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DMA0_MIN_RPROT_0,
+	mmDMA_IF_W_S_DMA1_MIN_RPROT_0,
+	mmDMA_IF_E_S_DMA0_MIN_RPROT_0,
+	mmDMA_IF_E_S_DMA1_MIN_RPROT_0,
+	mmDMA_IF_W_N_DMA0_MIN_RPROT_0,
+	mmDMA_IF_W_N_DMA1_MIN_RPROT_0,
+	mmDMA_IF_E_N_DMA0_MIN_RPROT_0,
+	mmDMA_IF_E_N_DMA1_MIN_RPROT_0,
+	mmSIF_RTR_0_LBW_RANGE_PROT_MIN_AR_0,
+	mmSIF_RTR_1_LBW_RANGE_PROT_MIN_AR_0,
+	mmSIF_RTR_2_LBW_RANGE_PROT_MIN_AR_0,
+	mmSIF_RTR_3_LBW_RANGE_PROT_MIN_AR_0,
+	mmSIF_RTR_4_LBW_RANGE_PROT_MIN_AR_0,
+	mmSIF_RTR_5_LBW_RANGE_PROT_MIN_AR_0,
+	mmSIF_RTR_6_LBW_RANGE_PROT_MIN_AR_0,
+	mmSIF_RTR_7_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_0_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_1_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_2_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_3_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_4_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_5_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_6_LBW_RANGE_PROT_MIN_AR_0,
+	mmNIF_RTR_7_LBW_RANGE_PROT_MIN_AR_0,
+};
+
+static u64 gaudi_rr_lbw_max_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DMA0_MAX_RPROT_0,
+	mmDMA_IF_W_S_DMA1_MAX_RPROT_0,
+	mmDMA_IF_E_S_DMA0_MAX_RPROT_0,
+	mmDMA_IF_E_S_DMA1_MAX_RPROT_0,
+	mmDMA_IF_W_N_DMA0_MAX_RPROT_0,
+	mmDMA_IF_W_N_DMA1_MAX_RPROT_0,
+	mmDMA_IF_E_N_DMA0_MAX_RPROT_0,
+	mmDMA_IF_E_N_DMA1_MAX_RPROT_0,
+	mmSIF_RTR_0_LBW_RANGE_PROT_MAX_AR_0,
+	mmSIF_RTR_1_LBW_RANGE_PROT_MAX_AR_0,
+	mmSIF_RTR_2_LBW_RANGE_PROT_MAX_AR_0,
+	mmSIF_RTR_3_LBW_RANGE_PROT_MAX_AR_0,
+	mmSIF_RTR_4_LBW_RANGE_PROT_MAX_AR_0,
+	mmSIF_RTR_5_LBW_RANGE_PROT_MAX_AR_0,
+	mmSIF_RTR_6_LBW_RANGE_PROT_MAX_AR_0,
+	mmSIF_RTR_7_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_0_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_1_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_2_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_3_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_4_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_5_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_6_LBW_RANGE_PROT_MAX_AR_0,
+	mmNIF_RTR_7_LBW_RANGE_PROT_MAX_AR_0,
+};
+
+static u64 gaudi_rr_hbw_hit_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_HIT_AW,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_HIT_AW,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_HIT_AW,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_HIT_AW,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_HIT_AW,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_HIT_AW,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_HIT_AW,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_HIT_AW,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_HIT_AW,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_HIT_AW
+};
+
+static u64 gaudi_rr_hbw_hit_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_HIT_AR,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_HIT_AR,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_HIT_AR,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_HIT_AR,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_HIT_AR,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_HIT_AR,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_HIT_AR,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_HIT_AR,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_HIT_AR,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_HIT_AR
+};
+
+static u64 gaudi_rr_hbw_base_low_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_0
+};
+
+static u64 gaudi_rr_hbw_base_high_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_0
+};
+
+static u64 gaudi_rr_hbw_mask_low_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_0
+};
+
+static u64 gaudi_rr_hbw_mask_high_aw_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_0
+};
+
+static u64 gaudi_rr_hbw_base_low_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_0
+};
+
+static u64 gaudi_rr_hbw_base_high_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_0
+};
+
+static u64 gaudi_rr_hbw_mask_low_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_0
+};
+
+static u64 gaudi_rr_hbw_mask_high_ar_regs[GAUDI_NUMBER_OF_RR_REGS] = {
+	mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0,
+	mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0,
+	mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0,
+	mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0,
+	mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0,
+	mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0,
+	mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0,
+	mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_0,
+	mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_0,
+	mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_0
+};
+
+/**
+ * gaudi_set_block_as_protected - set the given block as protected
+ *
+ * @hdev: pointer to hl_device structure
+ * @block: block base address
+ *
+ */
+static void gaudi_pb_set_block(struct hl_device *hdev, u64 base)
+{
+	u32 pb_addr = base - CFG_BASE + PROT_BITS_OFFS;
+
+	while (pb_addr & 0xFFF) {
+		WREG32(pb_addr, 0);
+		pb_addr += 4;
+	}
+}
+
+static void gaudi_init_mme_protection_bits(struct hl_device *hdev)
+{
+	u32 pb_addr, mask;
+	u8 word_offset;
+
+	gaudi_pb_set_block(hdev, mmMME0_ACC_BASE);
+	gaudi_pb_set_block(hdev, mmMME0_SBAB_BASE);
+	gaudi_pb_set_block(hdev, mmMME0_PRTN_BASE);
+	gaudi_pb_set_block(hdev, mmMME1_ACC_BASE);
+	gaudi_pb_set_block(hdev, mmMME1_SBAB_BASE);
+	gaudi_pb_set_block(hdev, mmMME1_PRTN_BASE);
+	gaudi_pb_set_block(hdev, mmMME2_ACC_BASE);
+	gaudi_pb_set_block(hdev, mmMME2_SBAB_BASE);
+	gaudi_pb_set_block(hdev, mmMME2_PRTN_BASE);
+	gaudi_pb_set_block(hdev, mmMME3_ACC_BASE);
+	gaudi_pb_set_block(hdev, mmMME3_SBAB_BASE);
+	gaudi_pb_set_block(hdev, mmMME3_PRTN_BASE);
+
+	WREG32(mmMME0_CTRL_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmMME1_CTRL_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmMME2_CTRL_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmMME3_CTRL_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	WREG32(mmMME0_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmMME2_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmMME0_CTRL_RESET & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_CTRL_RESET & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_CTRL_RESET & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_QM_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_SYNC_OBJECT_FIFO_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_EUS_ROLLUP_CNT_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_LOG_SHADOW & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_RL_DESC0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_RL_TOKEN_UPDATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_RL_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_RL_MIN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_RL_CTRL_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_RL_HISTORY_LOG_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_DUMMY_A_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_DUMMY_B_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_DUMMY_A_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_DUMMY_A_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_DUMMY_B_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_DUMMY_B_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_EU_POWER_SAVE_DISABLE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_CS_DBG_BLOCK_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_CS_DBG_STATUS_DROP_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_TE_CLOSE_CGATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_AGU_SM_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_AGU_SM_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_EZSYNC_OUT_CREDIT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_PCU_RL_SAT_SEC & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_AGU_SYNC_MSG_AXI_USER & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_CTRL_QM_SLV_LBW_CLK_EN & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_CTRL_SHADOW_0_STATUS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_CTRL_SHADOW_0_STATUS & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME0_CTRL_SHADOW_0_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 &
+			PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_23 &
+			PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME0_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME0_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmMME0_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME0_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME0_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME0_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME1_CTRL_RESET & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME1_CTRL_RESET & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME1_CTRL_RESET & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_QM_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_SYNC_OBJECT_FIFO_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_EUS_ROLLUP_CNT_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_LOG_SHADOW & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_RL_DESC0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_RL_TOKEN_UPDATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_RL_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_RL_MIN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_RL_CTRL_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_RL_HISTORY_LOG_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_DUMMY_A_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_DUMMY_B_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_DUMMY_A_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_DUMMY_A_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_DUMMY_B_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_DUMMY_B_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_EU_POWER_SAVE_DISABLE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_CS_DBG_BLOCK_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_CS_DBG_STATUS_DROP_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_TE_CLOSE_CGATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_AGU_SM_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_AGU_SM_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_EZSYNC_OUT_CREDIT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_PCU_RL_SAT_SEC & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_AGU_SYNC_MSG_AXI_USER & 0x7F) >> 2);
+	mask |= 1 << ((mmMME1_CTRL_QM_SLV_LBW_CLK_EN & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME1_CTRL_SHADOW_0_STATUS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME1_CTRL_SHADOW_0_STATUS & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME1_CTRL_SHADOW_0_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	/* MME 1 is slave, hence its whole QM block is protected (with RR) */
+
+	pb_addr = (mmMME2_CTRL_RESET & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_CTRL_RESET & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_CTRL_RESET & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_QM_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_SYNC_OBJECT_FIFO_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_EUS_ROLLUP_CNT_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_LOG_SHADOW & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_RL_DESC0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_RL_TOKEN_UPDATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_RL_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_RL_MIN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_RL_CTRL_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_RL_HISTORY_LOG_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_DUMMY_A_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_DUMMY_B_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_DUMMY_A_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_DUMMY_A_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_DUMMY_B_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_DUMMY_B_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_EU_POWER_SAVE_DISABLE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_CS_DBG_BLOCK_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_CS_DBG_STATUS_DROP_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_TE_CLOSE_CGATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_AGU_SM_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_AGU_SM_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_EZSYNC_OUT_CREDIT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_PCU_RL_SAT_SEC & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_AGU_SYNC_MSG_AXI_USER & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_CTRL_QM_SLV_LBW_CLK_EN & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_CTRL_SHADOW_0_STATUS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_CTRL_SHADOW_0_STATUS & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME2_CTRL_SHADOW_0_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+			>> 7) << 2;
+	mask = 1 << ((mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_23 &
+			PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME2_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME2_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmMME2_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME2_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME2_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME2_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME3_CTRL_RESET & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME3_CTRL_RESET & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmMME3_CTRL_RESET & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_QM_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_SYNC_OBJECT_FIFO_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_EUS_ROLLUP_CNT_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_LOG_SHADOW & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_RL_DESC0 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_RL_TOKEN_UPDATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_RL_TH & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_RL_MIN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_RL_CTRL_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_RL_HISTORY_LOG_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_DUMMY_A_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_DUMMY_B_BF16 & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_DUMMY_A_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_DUMMY_A_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_DUMMY_B_FP32_ODD & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_DUMMY_B_FP32_EVEN & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_EU_POWER_SAVE_DISABLE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_CS_DBG_BLOCK_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_CS_DBG_STATUS_DROP_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_TE_CLOSE_CGATE & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_AGU_SM_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_AGU_SM_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_EZSYNC_OUT_CREDIT & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_PCU_RL_SAT_SEC & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_AGU_SYNC_MSG_AXI_USER & 0x7F) >> 2);
+	mask |= 1 << ((mmMME3_CTRL_QM_SLV_LBW_CLK_EN & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmMME3_CTRL_SHADOW_0_STATUS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmMME3_CTRL_SHADOW_0_STATUS & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmMME3_CTRL_SHADOW_0_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	/* MME 3 is slave, hence its whole QM block is protected (with RR) */
+}
+
+static void gaudi_init_dma_protection_bits(struct hl_device *hdev)
+{
+	u32 pb_addr, mask;
+	u8 word_offset;
+
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_S_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH0_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH1_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_E_PLL_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_BASE);
+
+	gaudi_pb_set_block(hdev, mmDMA_IF_W_N_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_W_N_DOWN_CH0_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_W_N_DOWN_CH1_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_W_N_DOWN_BASE);
+
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_N_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_N_DOWN_CH0_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_N_DOWN_CH1_BASE);
+	gaudi_pb_set_block(hdev, mmDMA_IF_E_N_DOWN_BASE);
+
+	WREG32(mmDMA0_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA1_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA2_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA3_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA4_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA5_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA6_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA7_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	WREG32(mmDMA0_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA1_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA2_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA3_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA4_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA5_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA6_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmDMA7_CORE_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmDMA0_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA0_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA0_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA1_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA1_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA2_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA2_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA3_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA3_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA4_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA4_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA5_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA5_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA6_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA6_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset =
+		((mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS) >> 7)
+		<< 2;
+	mask = 1 << ((mmDMA7_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA7_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA0_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_WR_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA0_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA0_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA0_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA0_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA1_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_WR_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA1_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA1_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA1_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA1_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA2_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA2_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA2_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA2_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA2_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA3_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA3_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA3_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA3_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA3_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA4_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA4_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA4_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA4_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA4_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA5_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA5_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA5_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA5_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA5_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA6_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA6_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA6_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA6_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA6_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_CORE_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_CORE_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_CORE_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_LBW_MAX_OUTSTAND & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_CORE_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_CORE_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_CORE_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_SECURE_PROPS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_NON_SECURE_PROPS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_CORE_RD_MAX_OUTSTAND & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_CORE_RD_MAX_OUTSTAND & PROT_BITS_OFFS) >> 7)
+			<< 2;
+	mask = 1 << ((mmDMA7_CORE_RD_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_ARCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_ARUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_WR_MAX_OUTSTAND & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_WR_MAX_AWID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_WR_AWCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_WR_INFLIGHTS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_ERRMSG_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_ERRMSG_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_ERRMSG_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_CORE_STS0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_CORE_STS0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_CORE_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_STS1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmDMA7_CORE_RD_DBGMEM_ADD & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmDMA7_CORE_RD_DBGMEM_ADD & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmDMA7_CORE_RD_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_RD_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_DBG_HBW_AXI_AR_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_DBG_HBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_DBG_LBW_AXI_AW_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_DBG_DESC_CNT & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_DBG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_DBG_RD_DESC_ID & 0x7F) >> 2);
+	mask |= 1 << ((mmDMA7_CORE_DBG_WR_DESC_ID & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+}
+
+static void gaudi_init_tpc_protection_bits(struct hl_device *hdev)
+{
+	u32 pb_addr, mask;
+	u8 word_offset;
+
+	gaudi_pb_set_block(hdev, mmTPC0_E2E_CRED_BASE);
+	gaudi_pb_set_block(hdev, mmTPC1_E2E_CRED_BASE);
+	gaudi_pb_set_block(hdev, mmTPC2_E2E_CRED_BASE);
+	gaudi_pb_set_block(hdev, mmTPC3_E2E_CRED_BASE);
+	gaudi_pb_set_block(hdev, mmTPC4_E2E_CRED_BASE);
+	gaudi_pb_set_block(hdev, mmTPC5_E2E_CRED_BASE);
+	gaudi_pb_set_block(hdev, mmTPC6_E2E_CRED_BASE);
+	gaudi_pb_set_block(hdev, mmTPC7_E2E_CRED_BASE);
+
+	WREG32(mmTPC0_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC0_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC0_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+
+	word_offset = ((mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+
+	mask = 1 << ((mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+								PROT_BITS_OFFS;
+
+	word_offset = ((mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC0_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC0_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC0_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC0_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC0_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC0_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	WREG32(mmTPC1_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC1_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC1_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+								PROT_BITS_OFFS;
+
+	word_offset = ((mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC1_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC1_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC1_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC1_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	WREG32(mmTPC2_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC2_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC2_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC2_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC2_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC2_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+								<< 2;
+	mask = 1 << ((mmTPC2_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	WREG32(mmTPC3_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC3_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC3_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC3_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC3_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC3_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC3_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	WREG32(mmTPC4_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC4_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC4_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC4_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC4_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC4_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC4_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	WREG32(mmTPC5_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC5_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC5_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC5_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC5_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC5_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC5_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	WREG32(mmTPC6_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC6_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC6_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+								PROT_BITS_OFFS;
+
+	word_offset = ((mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+
+	mask = 1 << ((mmTPC6_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC6_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC6_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC6_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	WREG32(mmTPC7_QM_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+	WREG32(mmTPC7_CFG_BASE - CFG_BASE + PROT_BITS_OFFS + 0x7C, 0);
+
+	pb_addr = (mmTPC7_QM_GLBL_CFG0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_GLBL_CFG0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_GLBL_CFG0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_CFG1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_ERR_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_NON_SECURE_PROPS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_NON_SECURE_PROPS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_NON_SECURE_PROPS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_NON_SECURE_PROPS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_NON_SECURE_PROPS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_STS0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_STS1_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_MSG_EN_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_MSG_EN_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_MSG_EN_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_MSG_EN_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_MSG_EN_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_BASE_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_BASE_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_BASE_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_BASE_LO_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_PQ_BASE_HI_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_PQ_BASE_HI_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_PQ_BASE_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_BASE_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_BASE_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_BASE_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_SIZE_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_SIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_SIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_SIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_PI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_PI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_PI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_PI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_CFG1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_STS0_3 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_PQ_STS1_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_PQ_STS1_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_PQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_PQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS0_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS1_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS1_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS1_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_STS1_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_0 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_CQ_CTL_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_CQ_CTL_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_CQ_CTL_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_CTL_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_CTL_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_CTL_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_LO_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_PTR_HI_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_TSIZE_STS_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_CQ_CTL_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_CQ_CTL_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_CQ_CTL_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_CTL_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_CTL_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_CTL_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_CTL_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_IFIFO_CNT_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_IFIFO_CNT_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_IFIFO_CNT_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_IFIFO_CNT_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CQ_IFIFO_CNT_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_2 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & ~0xFFF) +
+								PROT_BITS_OFFS;
+
+	word_offset = ((mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & PROT_BITS_OFFS)
+								>> 7) << 2;
+
+	mask = 1 << ((mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_CP_STS_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_CP_STS_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_CP_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_LO_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_LO_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_LO_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_LO_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_LO_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_HI_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_HI_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_HI_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_HI_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_CURRENT_INST_HI_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_BARRIER_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_BARRIER_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_BARRIER_CFG_2 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_CP_BARRIER_CFG_3 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_CP_BARRIER_CFG_3 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_CP_BARRIER_CFG_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_BARRIER_CFG_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_DBG_0_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_DBG_0_1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_CP_DBG_0_2 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_CP_DBG_0_2 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_CP_DBG_0_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_DBG_0_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_DBG_0_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_ARUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_ARUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_ARUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_ARUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_ARUSER_31_11_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_AWUSER_31_11_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_AWUSER_31_11_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_AWUSER_31_11_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_AWUSER_31_11_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CP_AWUSER_31_11_4 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_ARB_CFG_0 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_ARB_CFG_0 & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_ARB_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_19 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_23 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_ARB_MST_AVAIL_CRED_24 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_ARB_MST_AVAIL_CRED_24 & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_AVAIL_CRED_31 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_23 & ~0xFFF) +
+			PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_23 & PROT_BITS_OFFS)
+								>> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_ARB_MST_QUIET_PER & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_SLV_CHOISE_WDT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MSG_MAX_INFLIGHT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MSG_AWUSER_31_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MSG_AWUSER_SEC_PROP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MSG_AWUSER_NON_SEC_PROP & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_ARB_STATE_STS & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_ARB_STATE_STS & PROT_BITS_OFFS) >> 7)	<< 2;
+	mask = 1 << ((mmTPC7_QM_ARB_STATE_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_CHOISE_FULLNESS_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MSG_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_SLV_CHOISE_Q_HEAD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_ERR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_ERR_MSG_EN & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_ERR_STS_DRP & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_9 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_10 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_11 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_12 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_13 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_14 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_15 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_16 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_17 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_18 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_19 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_ARB_MST_CRED_STS_20 & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_ARB_MST_CRED_STS_20 & PROT_BITS_OFFS)	>> 7)
+									<< 2;
+	mask = 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_20 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_21 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_22 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_23 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_24 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_25 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_26 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_27 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_28 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_29 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_30 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_ARB_MST_CRED_STS_31 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CGM_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CGM_STS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CGM_CFG1 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_LOCAL_RANGE_BASE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_LOCAL_RANGE_BASE & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_QM_LOCAL_RANGE_BASE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_LOCAL_RANGE_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_CSMR_STRICT_PRIO_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_HBW_RD_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_LBW_WR_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_LBW_WR_RATE_LIM_CFG_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_HBW_RD_RATE_LIM_CFG_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_AXCACHE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_IND_GW_APB_CFG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_IND_GW_APB_WDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_IND_GW_APB_RDATA & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_IND_GW_APB_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_ERR_ADDR_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_ERR_ADDR_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_QM_GLBL_ERR_WDATA & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_QM_GLBL_MEM_INIT_BUSY & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_QM_GLBL_MEM_INIT_BUSY & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC7_QM_GLBL_MEM_INIT_BUSY & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_CFG_ROUND_CSR & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_CFG_ROUND_CSR & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_CFG_ROUND_CSR & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_CFG_PROT & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_CFG_PROT & PROT_BITS_OFFS) >> 7) << 2;
+	mask = 1 << ((mmTPC7_CFG_PROT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_STATUS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_RD_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_WR_RATE_LIMIT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_TPC_INTR_MASK & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_WQ_CREDITS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_ARUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_ARUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_AWUSER_LO & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_AWUSER_HI & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_OPCODE_EXEC & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
+	pb_addr = (mmTPC7_CFG_TSB_CFG_MAX_SIZE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_CFG_TSB_CFG_MAX_SIZE & PROT_BITS_OFFS) >> 7)
+									<< 2;
+	mask = 1 << ((mmTPC7_CFG_TSB_CFG_MAX_SIZE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_DBGMEM_ADD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_DBGMEM_DATA_WR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_DBGMEM_DATA_RD & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_DBGMEM_CTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_DBGMEM_RC & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_TSB_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_WQ_INFLIGHT_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_WQ_LBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_WQ_HBW_TOTAL_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_IRQ_OCCOUPY_CNTR & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_CNTRL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_PAT & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_0 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_1 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_2 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_3 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_4 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_5 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_6 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_7 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_8 & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_FUNC_MBIST_MEM_9 & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+}
+
+/**
+ * gaudi_init_protection_bits - Initialize protection bits of specific registers
+ *
+ * @hdev: pointer to hl_device structure
+ *
+ * All protection bits are 1 by default, means not protected. Need to set to 0
+ * each bit that belongs to a protected register.
+ *
+ */
+static void gaudi_init_protection_bits(struct hl_device *hdev)
+{
+	/*
+	 * In each 4K block of registers, the last 128 bytes are protection
+	 * bits - total of 1024 bits, one for each register. Each bit is related
+	 * to a specific register, by the order of the registers.
+	 * So in order to calculate the bit that is related to a given register,
+	 * we need to calculate its word offset and then the exact bit inside
+	 * the word (which is 4 bytes).
+	 *
+	 * Register address:
+	 *
+	 * 31                 12 11           7   6             2  1      0
+	 * -----------------------------------------------------------------
+	 * |      Don't         |    word       |  bit location  |    0    |
+	 * |      care          |   offset      |  inside word   |         |
+	 * -----------------------------------------------------------------
+	 *
+	 * Bits 7-11 represents the word offset inside the 128 bytes.
+	 * Bits 2-6 represents the bit location inside the word.
+	 *
+	 * When a bit is cleared, it means the register it represents can only
+	 * be accessed by a secured entity. When the bit is set, any entity can
+	 * access the register.
+	 *
+	 * The last 4 bytes in the block of the PBs control the security of
+	 * the PBs themselves, so they always need to be configured to be
+	 * secured
+	 */
+
+	gaudi_pb_set_block(hdev, mmIF_E_PLL_BASE);
+	gaudi_pb_set_block(hdev, mmMESH_W_PLL_BASE);
+	gaudi_pb_set_block(hdev, mmSRAM_W_PLL_BASE);
+	gaudi_pb_set_block(hdev, mmMESH_E_PLL_BASE);
+	gaudi_pb_set_block(hdev, mmSRAM_E_PLL_BASE);
+
+	gaudi_init_dma_protection_bits(hdev);
+
+	gaudi_init_mme_protection_bits(hdev);
+
+	gaudi_init_tpc_protection_bits(hdev);
+}
+
+static void gaudi_init_range_registers_lbw(struct hl_device *hdev)
+{
+	u32 lbw_rng_start[GAUDI_NUMBER_OF_LBW_RANGES];
+	u32 lbw_rng_end[GAUDI_NUMBER_OF_LBW_RANGES];
+	int i, j;
+
+	lbw_rng_start[0]  = (0xFBFE0000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[0]    = (0xFBFFF000 & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[1]  = (0xFC0E8000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[1]    = (0xFC120000 & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[2]  = (0xFC1E8000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[2]    = (0xFC48FFFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[3]  = (0xFC600000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[3]    = (0xFCC48FFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[4]  = (0xFCC4A000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[4]    = (0xFCCDFFFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[5]  = (0xFCCE4000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[5]    = (0xFCD1FFFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[6]  = (0xFCD24000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[6]    = (0xFCD5FFFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[7]  = (0xFCD64000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[7]    = (0xFCD9FFFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[8]  = (0xFCDA4000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[8]    = (0xFCDDFFFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[9]  = (0xFCDE4000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[9]    = (0xFCE05FFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[10]  = (0xFEC43000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[10]    = (0xFEC43FFF & 0x3FFFFFF) + 1;
+
+	lbw_rng_start[11] = (0xFE484000 & 0x3FFFFFF) - 1;
+	lbw_rng_end[11]   = (0xFE484FFF & 0x3FFFFFF) + 1;
+
+	for (i = 0 ; i < GAUDI_NUMBER_OF_RR_REGS ; i++) {
+		WREG32(gaudi_rr_lbw_hit_aw_regs[i],
+				(1 << GAUDI_NUMBER_OF_LBW_RANGES) - 1);
+		WREG32(gaudi_rr_lbw_hit_ar_regs[i],
+				(1 << GAUDI_NUMBER_OF_LBW_RANGES) - 1);
+	}
+
+	for (i = 0 ; i < GAUDI_NUMBER_OF_RR_REGS ; i++)
+		for (j = 0 ; j < GAUDI_NUMBER_OF_LBW_RANGES ; j++) {
+			WREG32(gaudi_rr_lbw_min_aw_regs[i] + (j << 2),
+							lbw_rng_start[j]);
+
+			WREG32(gaudi_rr_lbw_min_ar_regs[i] + (j << 2),
+							lbw_rng_start[j]);
+
+			WREG32(gaudi_rr_lbw_max_aw_regs[i] + (j << 2),
+							lbw_rng_end[j]);
+
+			WREG32(gaudi_rr_lbw_max_ar_regs[i] + (j << 2),
+							lbw_rng_end[j]);
+		}
+}
+
+static void gaudi_init_range_registers_hbw(struct hl_device *hdev)
+{
+	struct gaudi_device *gaudi = hdev->asic_specific;
+
+	u32 dram_addr_lo = lower_32_bits(DRAM_PHYS_BASE);
+	u32 dram_addr_hi = upper_32_bits(DRAM_PHYS_BASE);
+
+	u32 sram_addr_lo = lower_32_bits(SRAM_BASE_ADDR);
+	u32 sram_addr_hi = upper_32_bits(SRAM_BASE_ADDR);
+
+	u32 scratch_addr_lo = lower_32_bits(PSOC_SCRATCHPAD_ADDR);
+	u32 scratch_addr_hi = upper_32_bits(PSOC_SCRATCHPAD_ADDR);
+
+	u32 pcie_fw_addr_lo = lower_32_bits(PCIE_FW_SRAM_ADDR);
+	u32 pcie_fw_addr_hi = upper_32_bits(PCIE_FW_SRAM_ADDR);
+
+	u32 spi_addr_lo = lower_32_bits(SPI_FLASH_BASE_ADDR);
+	u32 spi_addr_hi = upper_32_bits(SPI_FLASH_BASE_ADDR);
+
+	int i;
+
+	/* Configure HBW RR:
+	 * 1st range is the DRAM (first 512MB)
+	 * 2nd range is the 1st 128 bytes in SRAM (for tensor DMA). This area
+	 * is defined as read-only for user
+	 * 3rd range is the PSOC scratch-pad
+	 * 4th range is the PCIe F/W SRAM area
+	 * 5th range is the SPI FLASH area
+	 * 6th range is the host
+	 */
+
+	for (i = 0 ; i < GAUDI_NUMBER_OF_RR_REGS ; i++) {
+		WREG32(gaudi_rr_hbw_hit_aw_regs[i], 0x1F);
+		WREG32(gaudi_rr_hbw_hit_ar_regs[i], 0x1D);
+	}
+
+	for (i = 0 ; i < GAUDI_NUMBER_OF_RR_REGS ; i++) {
+		WREG32(gaudi_rr_hbw_base_low_aw_regs[i], dram_addr_lo);
+		WREG32(gaudi_rr_hbw_base_low_ar_regs[i], dram_addr_lo);
+
+		WREG32(gaudi_rr_hbw_base_high_aw_regs[i], dram_addr_hi);
+		WREG32(gaudi_rr_hbw_base_high_ar_regs[i], dram_addr_hi);
+
+		WREG32(gaudi_rr_hbw_mask_low_aw_regs[i], 0xE0000000);
+		WREG32(gaudi_rr_hbw_mask_low_ar_regs[i], 0xE0000000);
+
+		WREG32(gaudi_rr_hbw_mask_high_aw_regs[i], 0x3FFFF);
+		WREG32(gaudi_rr_hbw_mask_high_ar_regs[i], 0x3FFFF);
+
+		WREG32(gaudi_rr_hbw_base_low_aw_regs[i] + 4, sram_addr_lo);
+		WREG32(gaudi_rr_hbw_base_high_aw_regs[i] + 4, sram_addr_hi);
+		WREG32(gaudi_rr_hbw_mask_low_aw_regs[i] + 4, 0xFFFFFF80);
+		WREG32(gaudi_rr_hbw_mask_high_aw_regs[i] + 4, 0x3FFFF);
+
+		WREG32(gaudi_rr_hbw_base_low_aw_regs[i] + 8, scratch_addr_lo);
+		WREG32(gaudi_rr_hbw_base_low_ar_regs[i] + 8, scratch_addr_lo);
+
+		WREG32(gaudi_rr_hbw_base_high_aw_regs[i] + 8, scratch_addr_hi);
+		WREG32(gaudi_rr_hbw_base_high_ar_regs[i] + 8, scratch_addr_hi);
+
+		WREG32(gaudi_rr_hbw_mask_low_aw_regs[i] + 8, 0xFFFF0000);
+		WREG32(gaudi_rr_hbw_mask_low_ar_regs[i] + 8, 0xFFFF0000);
+
+		WREG32(gaudi_rr_hbw_mask_high_aw_regs[i] + 8, 0x3FFFF);
+		WREG32(gaudi_rr_hbw_mask_high_ar_regs[i] + 8, 0x3FFFF);
+
+		WREG32(gaudi_rr_hbw_base_low_aw_regs[i] + 12, pcie_fw_addr_lo);
+		WREG32(gaudi_rr_hbw_base_low_ar_regs[i] + 12, pcie_fw_addr_lo);
+
+		WREG32(gaudi_rr_hbw_base_high_aw_regs[i] + 12, pcie_fw_addr_hi);
+		WREG32(gaudi_rr_hbw_base_high_ar_regs[i] + 12, pcie_fw_addr_hi);
+
+		WREG32(gaudi_rr_hbw_mask_low_aw_regs[i] + 12, 0xFFFF8000);
+		WREG32(gaudi_rr_hbw_mask_low_ar_regs[i] + 12, 0xFFFF8000);
+
+		WREG32(gaudi_rr_hbw_mask_high_aw_regs[i] + 12, 0x3FFFF);
+		WREG32(gaudi_rr_hbw_mask_high_ar_regs[i] + 12, 0x3FFFF);
+
+		WREG32(gaudi_rr_hbw_base_low_aw_regs[i] + 16, spi_addr_lo);
+		WREG32(gaudi_rr_hbw_base_low_ar_regs[i] + 16, spi_addr_lo);
+
+		WREG32(gaudi_rr_hbw_base_high_aw_regs[i] + 16, spi_addr_hi);
+		WREG32(gaudi_rr_hbw_base_high_ar_regs[i] + 16, spi_addr_hi);
+
+		WREG32(gaudi_rr_hbw_mask_low_aw_regs[i] + 16, 0xFE000000);
+		WREG32(gaudi_rr_hbw_mask_low_ar_regs[i] + 16, 0xFE000000);
+
+		WREG32(gaudi_rr_hbw_mask_high_aw_regs[i] + 16, 0x3FFFF);
+		WREG32(gaudi_rr_hbw_mask_high_ar_regs[i] + 16, 0x3FFFF);
+
+		if (gaudi->hw_cap_initialized & HW_CAP_MMU)
+			continue;
+
+		/* Protect HOST */
+		WREG32(gaudi_rr_hbw_base_low_aw_regs[i] + 20, 0);
+		WREG32(gaudi_rr_hbw_base_low_ar_regs[i] + 20, 0);
+
+		WREG32(gaudi_rr_hbw_base_high_aw_regs[i] + 20, 0);
+		WREG32(gaudi_rr_hbw_base_high_ar_regs[i] + 20, 0);
+
+		WREG32(gaudi_rr_hbw_mask_low_aw_regs[i] + 20, 0);
+		WREG32(gaudi_rr_hbw_mask_low_ar_regs[i] + 20, 0);
+
+		WREG32(gaudi_rr_hbw_mask_high_aw_regs[i] + 20, 0xFFF80);
+		WREG32(gaudi_rr_hbw_mask_high_ar_regs[i] + 20, 0xFFF80);
+	}
+}
+
+/**
+ * gaudi_init_security - Initialize security model
+ *
+ * @hdev: pointer to hl_device structure
+ *
+ * Initialize the security model of the device
+ * That includes range registers and protection bit per register
+ *
+ */
+void gaudi_init_security(struct hl_device *hdev)
+{
+	/* Due to H/W errata GAUDI0500, need to override default security
+	 * property configuration of MME SBAB and ACC to be non-privileged and
+	 * non-secured
+	 */
+	WREG32(mmMME0_SBAB_PROT, 0x2);
+	WREG32(mmMME0_ACC_PROT, 0x2);
+	WREG32(mmMME1_SBAB_PROT, 0x2);
+	WREG32(mmMME1_ACC_PROT, 0x2);
+	WREG32(mmMME2_SBAB_PROT, 0x2);
+	WREG32(mmMME2_ACC_PROT, 0x2);
+	WREG32(mmMME3_SBAB_PROT, 0x2);
+	WREG32(mmMME3_ACC_PROT, 0x2);
+
+	/* On RAZWI, 0 will be returned from RR and 0xBABA0BAD from PB */
+	WREG32(0xC01B28, 0x1);
+
+	gaudi_init_range_registers_lbw(hdev);
+
+	gaudi_init_range_registers_hbw(hdev);
+
+	gaudi_init_protection_bits(hdev);
+}
diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c
index 68f065607544..0d2952bb58df 100644
--- a/drivers/misc/habanalabs/goya/goya.c
+++ b/drivers/misc/habanalabs/goya/goya.c
@@ -72,7 +72,7 @@
  *
  */
 
-#define GOYA_UBOOT_FW_FILE	"habanalabs/goya/goya-u-boot.bin"
+#define GOYA_BOOT_FIT_FILE	"habanalabs/goya/goya-boot-fit.itb"
 #define GOYA_LINUX_FW_FILE	"habanalabs/goya/goya-fit.itb"
 
 #define GOYA_MMU_REGS_NUM		63
@@ -87,6 +87,7 @@
 #define GOYA_TEST_QUEUE_WAIT_USEC	100000		/* 100ms */
 #define GOYA_PLDM_MMU_TIMEOUT_USEC	(MMU_CONFIG_TIMEOUT_USEC * 100)
 #define GOYA_PLDM_QMAN0_TIMEOUT_USEC	(HL_DEVICE_TIMEOUT_USEC * 30)
+#define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC	1000000		/* 1s */
 
 #define GOYA_QMAN0_FENCE_VAL		0xD169B243
 
@@ -531,7 +532,7 @@ static int goya_early_init(struct hl_device *hdev)
 
 	prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
 
-	rc = hl_pci_init(hdev, 48);
+	rc = hl_pci_init(hdev);
 	if (rc)
 		return rc;
 
@@ -750,6 +751,8 @@ static int goya_sw_init(struct hl_device *hdev)
 	}
 
 	spin_lock_init(&goya->hw_queues_lock);
+	hdev->supports_coresight = true;
+	hdev->supports_soft_reset = true;
 
 	return 0;
 
@@ -800,6 +803,7 @@ static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
 	u32 so_base_lo, so_base_hi;
 	u32 gic_base_lo, gic_base_hi;
 	u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI);
+	u32 dma_err_cfg = QMAN_DMA_ERR_MSG_EN;
 
 	mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
 	mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
@@ -836,7 +840,10 @@ static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
 	else
 		WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED);
 
-	WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, QMAN_DMA_ERR_MSG_EN);
+	if (hdev->stop_on_err)
+		dma_err_cfg |= 1 << DMA_QM_0_GLBL_ERR_CFG_DMA_STOP_ON_ERR_SHIFT;
+
+	WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, dma_err_cfg);
 	WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE);
 }
 
@@ -886,6 +893,7 @@ void goya_init_dma_qmans(struct hl_device *hdev)
 	q = &hdev->kernel_queues[0];
 
 	for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
+		q->cq_id = q->msi_vec = i;
 		goya_init_dma_qman(hdev, i, q->bus_address);
 		goya_init_dma_ch(hdev, i);
 	}
@@ -2205,80 +2213,37 @@ static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
 }
 
 /*
- * goya_push_uboot_to_device() - Push u-boot FW code to device.
+ * goya_load_firmware_to_device() - Load LINUX FW code to device.
  * @hdev: Pointer to hl_device structure.
  *
- * Copy u-boot fw code from firmware file to SRAM BAR.
+ * Copy LINUX fw code from firmware file to HBM BAR.
  *
  * Return: 0 on success, non-zero for failure.
  */
-static int goya_push_uboot_to_device(struct hl_device *hdev)
+static int goya_load_firmware_to_device(struct hl_device *hdev)
 {
 	void __iomem *dst;
 
-	dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
+	dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
 
-	return hl_fw_push_fw_to_device(hdev, GOYA_UBOOT_FW_FILE, dst);
+	return hl_fw_load_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst);
 }
 
 /*
- * goya_push_linux_to_device() - Push LINUX FW code to device.
+ * goya_load_boot_fit_to_device() - Load boot fit to device.
  * @hdev: Pointer to hl_device structure.
  *
- * Copy LINUX fw code from firmware file to HBM BAR.
+ * Copy boot fit file to SRAM BAR.
  *
  * Return: 0 on success, non-zero for failure.
  */
-static int goya_push_linux_to_device(struct hl_device *hdev)
+static int goya_load_boot_fit_to_device(struct hl_device *hdev)
 {
 	void __iomem *dst;
 
-	dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
+	dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
 
-	return hl_fw_push_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst);
-}
-
-static int goya_pldm_init_cpu(struct hl_device *hdev)
-{
-	u32 unit_rst_val;
-	int rc;
-
-	/* Must initialize SRAM scrambler before pushing u-boot to SRAM */
-	goya_init_golden_registers(hdev);
-
-	/* Put ARM cores into reset */
-	WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL, CPU_RESET_ASSERT);
-	RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
-
-	/* Reset the CA53 MACRO */
-	unit_rst_val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
-	WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, CA53_RESET);
-	RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
-	WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
-	RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
-
-	rc = goya_push_uboot_to_device(hdev);
-	if (rc)
-		return rc;
-
-	rc = goya_push_linux_to_device(hdev);
-	if (rc)
-		return rc;
-
-	WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
-	WREG32(mmPSOC_GLOBAL_CONF_WARM_REBOOT, CPU_BOOT_STATUS_NA);
-
-	WREG32(mmCPU_CA53_CFG_RST_ADDR_LSB_0,
-		lower_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
-	WREG32(mmCPU_CA53_CFG_RST_ADDR_MSB_0,
-		upper_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
-
-	/* Release ARM core 0 from reset */
-	WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL,
-					CPU_RESET_CORE0_DEASSERT);
-	RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
-
-	return 0;
+	return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst);
 }
 
 /*
@@ -2286,7 +2251,7 @@ static int goya_pldm_init_cpu(struct hl_device *hdev)
  * The version string should be located by that offset.
  */
 static void goya_read_device_fw_version(struct hl_device *hdev,
-					enum goya_fw_component fwc)
+					enum hl_fw_component fwc)
 {
 	const char *name;
 	u32 ver_off;
@@ -2320,10 +2285,9 @@ static void goya_read_device_fw_version(struct hl_device *hdev,
 	}
 }
 
-static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
+static int goya_init_cpu(struct hl_device *hdev)
 {
 	struct goya_device *goya = hdev->asic_specific;
-	u32 status;
 	int rc;
 
 	if (!hdev->cpu_enable)
@@ -2342,115 +2306,15 @@ static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
 		return -EIO;
 	}
 
-	if (hdev->pldm) {
-		rc = goya_pldm_init_cpu(hdev);
-		if (rc)
-			return rc;
-
-		goto out;
-	}
-
-	/* Make sure CPU boot-loader is running */
-	rc = hl_poll_timeout(
-		hdev,
-		mmPSOC_GLOBAL_CONF_WARM_REBOOT,
-		status,
-		(status == CPU_BOOT_STATUS_DRAM_RDY) ||
-		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
-		10000,
-		cpu_timeout);
+	rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
+			mmPSOC_GLOBAL_CONF_UBOOT_MAGIC,
+			mmCPU_CMD_STATUS_TO_HOST, mmCPU_BOOT_ERR0,
+			false, GOYA_CPU_TIMEOUT_USEC,
+			GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
 
-	/* Read U-Boot version now in case we will later fail */
-	goya_read_device_fw_version(hdev, FW_COMP_UBOOT);
-	goya_read_device_fw_version(hdev, FW_COMP_PREBOOT);
-
-	if (rc) {
-		dev_err(hdev->dev, "Error in ARM u-boot!");
-		switch (status) {
-		case CPU_BOOT_STATUS_NA:
-			dev_err(hdev->dev,
-				"ARM status %d - BTL did NOT run\n", status);
-			break;
-		case CPU_BOOT_STATUS_IN_WFE:
-			dev_err(hdev->dev,
-				"ARM status %d - Inside WFE loop\n", status);
-			break;
-		case CPU_BOOT_STATUS_IN_BTL:
-			dev_err(hdev->dev,
-				"ARM status %d - Stuck in BTL\n", status);
-			break;
-		case CPU_BOOT_STATUS_IN_PREBOOT:
-			dev_err(hdev->dev,
-				"ARM status %d - Stuck in Preboot\n", status);
-			break;
-		case CPU_BOOT_STATUS_IN_SPL:
-			dev_err(hdev->dev,
-				"ARM status %d - Stuck in SPL\n", status);
-			break;
-		case CPU_BOOT_STATUS_IN_UBOOT:
-			dev_err(hdev->dev,
-				"ARM status %d - Stuck in u-boot\n", status);
-			break;
-		case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
-			dev_err(hdev->dev,
-				"ARM status %d - DDR initialization failed\n",
-				status);
-			break;
-		case CPU_BOOT_STATUS_UBOOT_NOT_READY:
-			dev_err(hdev->dev,
-				"ARM status %d - u-boot stopped by user\n",
-				status);
-			break;
-		case CPU_BOOT_STATUS_TS_INIT_FAIL:
-			dev_err(hdev->dev,
-				"ARM status %d - Thermal Sensor initialization failed\n",
-				status);
-			break;
-		default:
-			dev_err(hdev->dev,
-				"ARM status %d - Invalid status code\n",
-				status);
-			break;
-		}
-		return -EIO;
-	}
-
-	if (!hdev->fw_loading) {
-		dev_info(hdev->dev, "Skip loading FW\n");
-		goto out;
-	}
-
-	if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
-		goto out;
-
-	rc = goya_push_linux_to_device(hdev);
 	if (rc)
 		return rc;
 
-	WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
-
-	rc = hl_poll_timeout(
-		hdev,
-		mmPSOC_GLOBAL_CONF_WARM_REBOOT,
-		status,
-		(status == CPU_BOOT_STATUS_SRAM_AVAIL),
-		10000,
-		cpu_timeout);
-
-	if (rc) {
-		if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
-			dev_err(hdev->dev,
-				"ARM u-boot reports FIT image is corrupted\n");
-		else
-			dev_err(hdev->dev,
-				"ARM Linux failed to load, %d\n", status);
-		WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_NA);
-		return -EIO;
-	}
-
-	dev_info(hdev->dev, "Successfully loaded firmware to device\n");
-
-out:
 	goya->hw_cap_initialized |= HW_CAP_CPU;
 
 	return 0;
@@ -2565,7 +2429,7 @@ static int goya_hw_init(struct hl_device *hdev)
 	 */
 	WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
 
-	rc = goya_init_cpu(hdev, GOYA_CPU_TIMEOUT_USEC);
+	rc = goya_init_cpu(hdev);
 	if (rc) {
 		dev_err(hdev->dev, "failed to initialize CPU\n");
 		return rc;
@@ -2684,30 +2548,6 @@ static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
 					HW_CAP_MMU | HW_CAP_TPC_MBIST |
 					HW_CAP_GOLDEN | HW_CAP_TPC);
 	memset(goya->events_stat, 0, sizeof(goya->events_stat));
-
-	if (!hdev->pldm) {
-		int rc;
-		/* In case we are running inside VM and the VM is
-		 * shutting down, we need to make sure CPU boot-loader
-		 * is running before we can continue the VM shutdown.
-		 * That is because the VM will send an FLR signal that
-		 * we must answer
-		 */
-		dev_info(hdev->dev,
-			"Going to wait up to %ds for CPU boot loader\n",
-			GOYA_CPU_TIMEOUT_USEC / 1000 / 1000);
-
-		rc = hl_poll_timeout(
-			hdev,
-			mmPSOC_GLOBAL_CONF_WARM_REBOOT,
-			status,
-			(status == CPU_BOOT_STATUS_DRAM_RDY),
-			10000,
-			GOYA_CPU_TIMEOUT_USEC);
-		if (rc)
-			dev_err(hdev->dev,
-				"failed to wait for CPU boot loader\n");
-	}
 }
 
 int goya_suspend(struct hl_device *hdev)
@@ -3555,6 +3395,7 @@ static int goya_validate_cb(struct hl_device *hdev,
 			 */
 			rc = goya_validate_wreg32(hdev,
 				parser, (struct packet_wreg32 *) user_pkt);
+			parser->patched_cb_size += pkt_size;
 			break;
 
 		case PACKET_WREG_BULK:
@@ -4016,7 +3857,8 @@ int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
 }
 
 void goya_add_end_of_cb_packets(struct hl_device *hdev, u64 kernel_address,
-				u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec)
+				u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec,
+				bool eb)
 {
 	struct packet_msg_prot *cq_pkt;
 	u32 tmp;
@@ -5042,7 +4884,7 @@ static void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
 		goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid);
 }
 
-static void goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
+static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
 					u32 flags)
 {
 	struct goya_device *goya = hdev->asic_specific;
@@ -5051,11 +4893,11 @@ static void goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
 
 	if (!(goya->hw_cap_initialized & HW_CAP_MMU) ||
 		hdev->hard_reset_pending)
-		return;
+		return 0;
 
 	/* no need in L1 only invalidation in Goya */
 	if (!is_hard)
-		return;
+		return 0;
 
 	if (hdev->pldm)
 		timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
@@ -5077,13 +4919,17 @@ static void goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
 
 	mutex_unlock(&hdev->mmu_cache_lock);
 
-	if (rc)
-		dev_notice_ratelimited(hdev->dev,
-			"Timeout when waiting for MMU cache invalidation\n");
+	if (rc) {
+		dev_err_ratelimited(hdev->dev,
+					"MMU cache invalidation timeout\n");
+		hl_device_reset(hdev, true, false);
+	}
+
+	return rc;
 }
 
-static void goya_mmu_invalidate_cache_range(struct hl_device *hdev,
-		bool is_hard, u32 asid, u64 va, u64 size)
+static int goya_mmu_invalidate_cache_range(struct hl_device *hdev,
+				bool is_hard, u32 asid, u64 va, u64 size)
 {
 	struct goya_device *goya = hdev->asic_specific;
 	u32 status, timeout_usec, inv_data, pi;
@@ -5091,11 +4937,11 @@ static void goya_mmu_invalidate_cache_range(struct hl_device *hdev,
 
 	if (!(goya->hw_cap_initialized & HW_CAP_MMU) ||
 		hdev->hard_reset_pending)
-		return;
+		return 0;
 
 	/* no need in L1 only invalidation in Goya */
 	if (!is_hard)
-		return;
+		return 0;
 
 	if (hdev->pldm)
 		timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
@@ -5128,9 +4974,13 @@ static void goya_mmu_invalidate_cache_range(struct hl_device *hdev,
 
 	mutex_unlock(&hdev->mmu_cache_lock);
 
-	if (rc)
-		dev_notice_ratelimited(hdev->dev,
-			"Timeout when waiting for MMU cache invalidation\n");
+	if (rc) {
+		dev_err_ratelimited(hdev->dev,
+					"MMU cache invalidation timeout\n");
+		hl_device_reset(hdev, true, false);
+	}
+
+	return rc;
 }
 
 int goya_send_heartbeat(struct hl_device *hdev)
@@ -5178,6 +5028,16 @@ int goya_armcp_info_get(struct hl_device *hdev)
 	return 0;
 }
 
+static void goya_enable_clock_gating(struct hl_device *hdev)
+{
+
+}
+
+static void goya_disable_clock_gating(struct hl_device *hdev)
+{
+
+}
+
 static bool goya_is_device_idle(struct hl_device *hdev, u32 *mask,
 				struct seq_file *s)
 {
@@ -5293,6 +5153,68 @@ static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev)
 	return RREG32(mmHW_STATE);
 }
 
+u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
+{
+	return cq_idx;
+}
+
+static void goya_ext_queue_init(struct hl_device *hdev, u32 q_idx)
+{
+
+}
+
+static void goya_ext_queue_reset(struct hl_device *hdev, u32 q_idx)
+{
+
+}
+
+static u32 goya_get_signal_cb_size(struct hl_device *hdev)
+{
+	return 0;
+}
+
+static u32 goya_get_wait_cb_size(struct hl_device *hdev)
+{
+	return 0;
+}
+
+static void goya_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id)
+{
+
+}
+
+static void goya_gen_wait_cb(struct hl_device *hdev, void *data, u16 sob_id,
+			u16 sob_val, u16 mon_id, u32 q_idx)
+{
+
+}
+
+static void goya_reset_sob(struct hl_device *hdev, void *data)
+{
+
+}
+
+static void goya_set_dma_mask_from_fw(struct hl_device *hdev)
+{
+	if (RREG32(mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_0) ==
+							HL_POWER9_HOST_MAGIC) {
+		dev_dbg(hdev->dev, "Working in 64-bit DMA mode\n");
+		hdev->power9_64bit_dma_enable = 1;
+		hdev->dma_mask = 64;
+	} else {
+		dev_dbg(hdev->dev, "Working in 48-bit DMA mode\n");
+		hdev->power9_64bit_dma_enable = 0;
+		hdev->dma_mask = 48;
+	}
+}
+
+u64 goya_get_device_time(struct hl_device *hdev)
+{
+	u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32;
+
+	return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);
+}
+
 static const struct hl_asic_funcs goya_funcs = {
 	.early_init = goya_early_init,
 	.early_fini = goya_early_fini,
@@ -5337,6 +5259,8 @@ static const struct hl_asic_funcs goya_funcs = {
 	.mmu_invalidate_cache = goya_mmu_invalidate_cache,
 	.mmu_invalidate_cache_range = goya_mmu_invalidate_cache_range,
 	.send_heartbeat = goya_send_heartbeat,
+	.enable_clock_gating = goya_enable_clock_gating,
+	.disable_clock_gating = goya_disable_clock_gating,
 	.debug_coresight = goya_debug_coresight,
 	.is_device_idle = goya_is_device_idle,
 	.soft_reset_late_init = goya_soft_reset_late_init,
@@ -5352,7 +5276,20 @@ static const struct hl_asic_funcs goya_funcs = {
 	.rreg = hl_rreg,
 	.wreg = hl_wreg,
 	.halt_coresight = goya_halt_coresight,
-	.get_clk_rate = goya_get_clk_rate
+	.get_clk_rate = goya_get_clk_rate,
+	.get_queue_id_for_cq = goya_get_queue_id_for_cq,
+	.read_device_fw_version = goya_read_device_fw_version,
+	.load_firmware_to_device = goya_load_firmware_to_device,
+	.load_boot_fit_to_device = goya_load_boot_fit_to_device,
+	.ext_queue_init = goya_ext_queue_init,
+	.ext_queue_reset = goya_ext_queue_reset,
+	.get_signal_cb_size = goya_get_signal_cb_size,
+	.get_wait_cb_size = goya_get_wait_cb_size,
+	.gen_signal_cb = goya_gen_signal_cb,
+	.gen_wait_cb = goya_gen_wait_cb,
+	.reset_sob = goya_reset_sob,
+	.set_dma_mask_from_fw = goya_set_dma_mask_from_fw,
+	.get_device_time = goya_get_device_time
 };
 
 /*
diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h
index c3230cb6e25c..d36f8d90c9c9 100644
--- a/drivers/misc/habanalabs/goya/goyaP.h
+++ b/drivers/misc/habanalabs/goya/goyaP.h
@@ -45,7 +45,7 @@
 
 #define CORESIGHT_TIMEOUT_USEC		100000		/* 100 ms */
 
-#define GOYA_CPU_TIMEOUT_USEC		10000000	/* 10s */
+#define GOYA_CPU_TIMEOUT_USEC		15000000	/* 15s */
 
 #define TPC_ENABLED_MASK		0xFF
 
@@ -149,11 +149,6 @@
 #define HW_CAP_GOLDEN		0x00000400
 #define HW_CAP_TPC		0x00000800
 
-enum goya_fw_component {
-	FW_COMP_UBOOT,
-	FW_COMP_PREBOOT
-};
-
 struct goya_device {
 	/* TODO: remove hw_queues_lock after moving to scheduler code */
 	spinlock_t	hw_queues_lock;
@@ -221,7 +216,8 @@ void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry);
 void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size);
 
 void goya_add_end_of_cb_packets(struct hl_device *hdev, u64 kernel_address,
-				u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec);
+				u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec,
+				bool eb);
 int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser);
 void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
 				dma_addr_t *dma_handle,	u16 *queue_len);
@@ -234,5 +230,7 @@ void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
 void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev);
 
 int goya_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
+u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx);
+u64 goya_get_device_time(struct hl_device *hdev);
 
 #endif /* GOYAP_H_ */
diff --git a/drivers/misc/habanalabs/goya/goya_coresight.c b/drivers/misc/habanalabs/goya/goya_coresight.c
index a1bc930d904f..1258724ea510 100644
--- a/drivers/misc/habanalabs/goya/goya_coresight.c
+++ b/drivers/misc/habanalabs/goya/goya_coresight.c
@@ -266,7 +266,7 @@ static int goya_config_stm(struct hl_device *hdev,
 		WREG32(base_reg + 0xDF4, 0x80);
 		WREG32(base_reg + 0xE8C, input->frequency);
 		WREG32(base_reg + 0xE90, 0x7FF);
-		WREG32(base_reg + 0xE80, 0x7 | (input->id << 16));
+		WREG32(base_reg + 0xE80, 0x27 | (input->id << 16));
 	} else {
 		WREG32(base_reg + 0xE80, 4);
 		WREG32(base_reg + 0xD64, 0);
diff --git a/drivers/misc/habanalabs/goya/goya_security.c b/drivers/misc/habanalabs/goya/goya_security.c
index d6ec12b3e692..de8297001fea 100644
--- a/drivers/misc/habanalabs/goya/goya_security.c
+++ b/drivers/misc/habanalabs/goya/goya_security.c
@@ -683,7 +683,6 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask = 1 << ((mmTPC0_CFG_SEMAPHORE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_VFLAGS & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_SFLAGS & 0x7F) >> 2);
-	mask |= 1 << ((mmTPC0_CFG_LFSR_POLYNOM & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_STATUS & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
@@ -695,7 +694,6 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
-	mask |= 1 << ((mmTPC0_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_TPC_STALL & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_MSS_CONFIG & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC0_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
@@ -875,6 +873,16 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	goya_pb_set_block(hdev, mmTPC1_RD_REGULATOR_BASE);
 	goya_pb_set_block(hdev, mmTPC1_WR_REGULATOR_BASE);
 
+	pb_addr = (mmTPC1_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC1_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2;
+
+	mask = 1 << ((mmTPC1_CFG_SEMAPHORE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
 	pb_addr = (mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) + PROT_BITS_OFFS;
 	word_offset = ((mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH &
 			PROT_BITS_OFFS) >> 7) << 2;
@@ -882,6 +890,10 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC1_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC1_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC1_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC1_CFG_TPC_INTR_MASK & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
 
@@ -1057,6 +1069,16 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	goya_pb_set_block(hdev, mmTPC2_RD_REGULATOR_BASE);
 	goya_pb_set_block(hdev, mmTPC2_WR_REGULATOR_BASE);
 
+	pb_addr = (mmTPC2_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC2_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2;
+
+	mask = 1 << ((mmTPC2_CFG_SEMAPHORE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
 	pb_addr = (mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) + PROT_BITS_OFFS;
 	word_offset = ((mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH &
 			PROT_BITS_OFFS) >> 7) << 2;
@@ -1064,6 +1086,10 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC2_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC2_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC2_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC2_CFG_TPC_INTR_MASK & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
 
@@ -1239,6 +1265,16 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	goya_pb_set_block(hdev, mmTPC3_RD_REGULATOR_BASE);
 	goya_pb_set_block(hdev, mmTPC3_WR_REGULATOR_BASE);
 
+	pb_addr = (mmTPC3_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC3_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2;
+
+	mask = 1 << ((mmTPC3_CFG_SEMAPHORE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
 	pb_addr = (mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) + PROT_BITS_OFFS;
 	word_offset = ((mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH
 			& PROT_BITS_OFFS) >> 7) << 2;
@@ -1246,6 +1282,10 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC3_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC3_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC3_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC3_CFG_TPC_INTR_MASK & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
 
@@ -1421,6 +1461,16 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	goya_pb_set_block(hdev, mmTPC4_RD_REGULATOR_BASE);
 	goya_pb_set_block(hdev, mmTPC4_WR_REGULATOR_BASE);
 
+	pb_addr = (mmTPC4_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC4_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2;
+
+	mask = 1 << ((mmTPC4_CFG_SEMAPHORE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
 	pb_addr = (mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) + PROT_BITS_OFFS;
 	word_offset = ((mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH &
 			PROT_BITS_OFFS) >> 7) << 2;
@@ -1428,6 +1478,10 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC4_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC4_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC4_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC4_CFG_TPC_INTR_MASK & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
 
@@ -1603,6 +1657,16 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	goya_pb_set_block(hdev, mmTPC5_RD_REGULATOR_BASE);
 	goya_pb_set_block(hdev, mmTPC5_WR_REGULATOR_BASE);
 
+	pb_addr = (mmTPC5_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC5_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2;
+
+	mask = 1 << ((mmTPC5_CFG_SEMAPHORE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
 	pb_addr = (mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) + PROT_BITS_OFFS;
 	word_offset = ((mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH &
 			PROT_BITS_OFFS) >> 7) << 2;
@@ -1610,6 +1674,10 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC5_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC5_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC5_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC5_CFG_TPC_INTR_MASK & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
 
@@ -1785,6 +1853,16 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	goya_pb_set_block(hdev, mmTPC6_RD_REGULATOR_BASE);
 	goya_pb_set_block(hdev, mmTPC6_WR_REGULATOR_BASE);
 
+	pb_addr = (mmTPC6_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC6_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2;
+
+	mask = 1 << ((mmTPC6_CFG_SEMAPHORE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
 	pb_addr = (mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) + PROT_BITS_OFFS;
 	word_offset = ((mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH &
 			PROT_BITS_OFFS) >> 7) << 2;
@@ -1792,6 +1870,10 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC6_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC6_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC6_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC6_CFG_TPC_INTR_MASK & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
 
@@ -1967,6 +2049,16 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	goya_pb_set_block(hdev, mmTPC7_RD_REGULATOR_BASE);
 	goya_pb_set_block(hdev, mmTPC7_WR_REGULATOR_BASE);
 
+	pb_addr = (mmTPC7_CFG_SEMAPHORE & ~0xFFF) + PROT_BITS_OFFS;
+	word_offset = ((mmTPC7_CFG_SEMAPHORE & PROT_BITS_OFFS) >> 7) << 2;
+
+	mask = 1 << ((mmTPC7_CFG_SEMAPHORE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_VFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_SFLAGS & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_STATUS & 0x7F) >> 2);
+
+	WREG32(pb_addr + word_offset, ~mask);
+
 	pb_addr = (mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH & ~0xFFF) +	PROT_BITS_OFFS;
 	word_offset = ((mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH &
 			PROT_BITS_OFFS) >> 7) << 2;
@@ -1974,6 +2066,10 @@ static void goya_init_tpc_protection_bits(struct hl_device *hdev)
 	mask |= 1 << ((mmTPC7_CFG_CFG_SUBTRACT_VALUE & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC7_CFG_SM_BASE_ADDRESS_LOW & 0x7F) >> 2);
 	mask |= 1 << ((mmTPC7_CFG_SM_BASE_ADDRESS_HIGH & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_TPC_STALL & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_MSS_CONFIG & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_TPC_INTR_CAUSE & 0x7F) >> 2);
+	mask |= 1 << ((mmTPC7_CFG_TPC_INTR_MASK & 0x7F) >> 2);
 
 	WREG32(pb_addr + word_offset, ~mask);
 
diff --git a/drivers/misc/habanalabs/habanalabs.h b/drivers/misc/habanalabs/habanalabs.h
index 31ebcf9458fe..1ecdcf8b763a 100644
--- a/drivers/misc/habanalabs/habanalabs.h
+++ b/drivers/misc/habanalabs/habanalabs.h
@@ -23,7 +23,9 @@
 
 #define HL_MMAP_CB_MASK			(0x8000000000000000ull >> PAGE_SHIFT)
 
-#define HL_PENDING_RESET_PER_SEC	5
+#define HL_PENDING_RESET_PER_SEC	30
+
+#define HL_HARD_RESET_MAX_TIMEOUT	120
 
 #define HL_DEVICE_TIMEOUT_USEC		1000000 /* 1 s */
 
@@ -51,6 +53,14 @@
 /* MMU */
 #define MMU_HASH_TABLE_BITS		7 /* 1 << 7 buckets */
 
+#define HL_RSVD_SOBS			4
+#define HL_RSVD_MONS			2
+
+#define HL_RSVD_SOBS_IN_USE		2
+#define HL_RSVD_MONS_IN_USE		1
+
+#define HL_MAX_SOB_VAL			(1 << 15)
+
 /**
  * struct pgt_info - MMU hop page info.
  * @node: hash linked-list node for the pgts shadow hash of pgts.
@@ -76,6 +86,16 @@ struct hl_device;
 struct hl_fpriv;
 
 /**
+ * enum hl_fw_component - F/W components to read version through registers.
+ * @FW_COMP_UBOOT: u-boot.
+ * @FW_COMP_PREBOOT: preboot.
+ */
+enum hl_fw_component {
+	FW_COMP_UBOOT,
+	FW_COMP_PREBOOT
+};
+
+/**
  * enum hl_queue_type - Supported QUEUE types.
  * @QUEUE_TYPE_NA: queue is not available.
  * @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the
@@ -94,6 +114,26 @@ enum hl_queue_type {
 	QUEUE_TYPE_HW
 };
 
+enum hl_cs_type {
+	CS_TYPE_DEFAULT,
+	CS_TYPE_SIGNAL,
+	CS_TYPE_WAIT
+};
+
+/*
+ * struct hl_hw_sob - H/W SOB info.
+ * @hdev: habanalabs device structure.
+ * @kref: refcount of this SOB. The SOB will reset once the refcount is zero.
+ * @sob_id: id of this SOB.
+ * @q_idx: the H/W queue that uses this SOB.
+ */
+struct hl_hw_sob {
+	struct hl_device	*hdev;
+	struct kref		kref;
+	u32			sob_id;
+	u32			q_idx;
+};
+
 /**
  * struct hw_queue_properties - queue information.
  * @type: queue type.
@@ -250,17 +290,23 @@ struct asic_fixed_properties {
 };
 
 /**
- * struct hl_dma_fence - wrapper for fence object used by command submissions.
+ * struct hl_cs_compl - command submission completion object.
  * @base_fence: kernel fence object.
  * @lock: spinlock to protect fence.
  * @hdev: habanalabs device structure.
+ * @hw_sob: the H/W SOB used in this signal/wait CS.
  * @cs_seq: command submission sequence number.
+ * @type: type of the CS - signal/wait.
+ * @sob_val: the SOB value that is used in this signal/wait CS.
  */
-struct hl_dma_fence {
+struct hl_cs_compl {
 	struct dma_fence	base_fence;
 	spinlock_t		lock;
 	struct hl_device	*hdev;
+	struct hl_hw_sob	*hw_sob;
 	u64			cs_seq;
+	enum hl_cs_type		type;
+	u16			sob_val;
 };
 
 /*
@@ -358,6 +404,7 @@ struct hl_cs_job;
 
 /**
  * struct hl_hw_queue - describes a H/W transport queue.
+ * @hw_sob: array of the used H/W SOBs by this H/W queue.
  * @shadow_queue: pointer to a shadow queue that holds pointers to jobs.
  * @queue_type: type of queue.
  * @kernel_address: holds the queue's kernel virtual address.
@@ -365,11 +412,19 @@ struct hl_cs_job;
  * @pi: holds the queue's pi value.
  * @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci).
  * @hw_queue_id: the id of the H/W queue.
+ * @cq_id: the id for the corresponding CQ for this H/W queue.
+ * @msi_vec: the IRQ number of the H/W queue.
  * @int_queue_len: length of internal queue (number of entries).
+ * @next_sob_val: the next value to use for the currently used SOB.
+ * @base_sob_id: the base SOB id of the SOBs used by this queue.
+ * @base_mon_id: the base MON id of the MONs used by this queue.
  * @valid: is the queue valid (we have array of 32 queues, not all of them
- *		exists).
+ *         exist).
+ * @curr_sob_offset: the id offset to the currently used SOB from the
+ *                   HL_RSVD_SOBS that are being used by this queue.
  */
 struct hl_hw_queue {
+	struct hl_hw_sob	hw_sob[HL_RSVD_SOBS];
 	struct hl_cs_job	**shadow_queue;
 	enum hl_queue_type	queue_type;
 	u64			kernel_address;
@@ -377,8 +432,14 @@ struct hl_hw_queue {
 	u32			pi;
 	u32			ci;
 	u32			hw_queue_id;
+	u32			cq_id;
+	u32			msi_vec;
 	u16			int_queue_len;
+	u16			next_sob_val;
+	u16			base_sob_id;
+	u16			base_mon_id;
 	u8			valid;
+	u8			curr_sob_offset;
 };
 
 /**
@@ -517,6 +578,8 @@ enum hl_pll_frequency {
  * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with
  *                              ASID-VA-size mask.
  * @send_heartbeat: send is-alive packet to ArmCP and verify response.
+ * @enable_clock_gating: enable clock gating for reducing power consumption.
+ * @disable_clock_gating: disable clock for accessing registers on HBW.
  * @debug_coresight: perform certain actions on Coresight for debugging.
  * @is_device_idle: return true if device is idle, false otherwise.
  * @soft_reset_late_init: perform certain actions needed after soft reset.
@@ -534,6 +597,21 @@ enum hl_pll_frequency {
  * @wreg: Write a register. Needed for simulator support.
  * @halt_coresight: stop the ETF and ETR traces.
  * @get_clk_rate: Retrieve the ASIC current and maximum clock rate in MHz
+ * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index.
+ * @read_device_fw_version: read the device's firmware versions that are
+ *                          contained in registers
+ * @load_firmware_to_device: load the firmware to the device's memory
+ * @load_boot_fit_to_device: load boot fit to device's memory
+ * @ext_queue_init: Initialize the given external queue.
+ * @ext_queue_reset: Reset the given external queue.
+ * @get_signal_cb_size: Get signal CB size.
+ * @get_wait_cb_size: Get wait CB size.
+ * @gen_signal_cb: Generate a signal CB.
+ * @gen_wait_cb: Generate a wait CB.
+ * @reset_sob: Reset a SOB.
+ * @set_dma_mask_from_fw: set the DMA mask in the driver according to the
+ *                        firmware configuration
+ * @get_device_time: Get the device time.
  */
 struct hl_asic_funcs {
 	int (*early_init)(struct hl_device *hdev);
@@ -578,7 +656,8 @@ struct hl_asic_funcs {
 					struct sg_table *sgt);
 	void (*add_end_of_cb_packets)(struct hl_device *hdev,
 					u64 kernel_address, u32 len,
-					u64 cq_addr, u32 cq_val, u32 msix_num);
+					u64 cq_addr, u32 cq_val, u32 msix_num,
+					bool eb);
 	void (*update_eq_ci)(struct hl_device *hdev, u32 val);
 	int (*context_switch)(struct hl_device *hdev, u32 asid);
 	void (*restore_phase_topology)(struct hl_device *hdev);
@@ -596,11 +675,13 @@ struct hl_asic_funcs {
 				u32 *size);
 	u64 (*read_pte)(struct hl_device *hdev, u64 addr);
 	void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val);
-	void (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard,
+	int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard,
 					u32 flags);
-	void (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
+	int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
 			u32 asid, u64 va, u64 size);
 	int (*send_heartbeat)(struct hl_device *hdev);
+	void (*enable_clock_gating)(struct hl_device *hdev);
+	void (*disable_clock_gating)(struct hl_device *hdev);
 	int (*debug_coresight)(struct hl_device *hdev, void *data);
 	bool (*is_device_idle)(struct hl_device *hdev, u32 *mask,
 				struct seq_file *s);
@@ -620,6 +701,21 @@ struct hl_asic_funcs {
 	void (*wreg)(struct hl_device *hdev, u32 reg, u32 val);
 	void (*halt_coresight)(struct hl_device *hdev);
 	int (*get_clk_rate)(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
+	u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx);
+	void (*read_device_fw_version)(struct hl_device *hdev,
+					enum hl_fw_component fwc);
+	int (*load_firmware_to_device)(struct hl_device *hdev);
+	int (*load_boot_fit_to_device)(struct hl_device *hdev);
+	void (*ext_queue_init)(struct hl_device *hdev, u32 hw_queue_id);
+	void (*ext_queue_reset)(struct hl_device *hdev, u32 hw_queue_id);
+	u32 (*get_signal_cb_size)(struct hl_device *hdev);
+	u32 (*get_wait_cb_size)(struct hl_device *hdev);
+	void (*gen_signal_cb)(struct hl_device *hdev, void *data, u16 sob_id);
+	void (*gen_wait_cb)(struct hl_device *hdev, void *data, u16 sob_id,
+				u16 sob_val, u16 mon_id, u32 q_idx);
+	void (*reset_sob)(struct hl_device *hdev, void *data);
+	void (*set_dma_mask_from_fw)(struct hl_device *hdev);
+	u64 (*get_device_time)(struct hl_device *hdev);
 };
 
 
@@ -659,8 +755,8 @@ struct hl_va_range {
  *                      with huge pages.
  * @dram_va_range: holds available virtual addresses for DRAM mappings.
  * @mem_hash_lock: protects the mem_hash.
- * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifing the
- *            MMU hash or walking the PGT requires talking this lock
+ * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the
+ *            MMU hash or walking the PGT requires talking this lock.
  * @debugfs_list: node in debugfs list of contexts.
  * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed
  *			to user so user could inquire about CS. It is used as
@@ -751,10 +847,14 @@ struct hl_userptr {
  * @job_lock: spinlock for the CS's jobs list. Needed for free_job.
  * @refcount: reference counter for usage of the CS.
  * @fence: pointer to the fence object of this CS.
+ * @signal_fence: pointer to the fence object of the signal CS (used by wait
+ *                CS only).
+ * @finish_work: workqueue object to run when CS is completed by H/W.
  * @work_tdr: delayed work node for TDR.
  * @mirror_node : node in device mirror list of command submissions.
  * @debugfs_list: node in debugfs list of command submissions.
  * @sequence: the sequence number of this CS.
+ * @type: CS_TYPE_*.
  * @submitted: true if CS was submitted to H/W.
  * @completed: true if CS was completed by device.
  * @timedout : true if CS was timedout.
@@ -769,10 +869,13 @@ struct hl_cs {
 	spinlock_t		job_lock;
 	struct kref		refcount;
 	struct dma_fence	*fence;
+	struct dma_fence	*signal_fence;
+	struct work_struct	finish_work;
 	struct delayed_work	work_tdr;
 	struct list_head	mirror_node;
 	struct list_head	debugfs_list;
 	u64			sequence;
+	enum hl_cs_type		type;
 	u8			submitted;
 	u8			completed;
 	u8			timedout;
@@ -799,6 +902,12 @@ struct hl_cs {
  * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
  *                          handle to a kernel-allocated CB object, false
  *                          otherwise (SRAM/DRAM/host address).
+ * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This
+ *                    info is needed later, when adding the 2xMSG_PROT at the
+ *                    end of the JOB, to know which barriers to put in the
+ *                    MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't
+ *                    have streams so the engine can't be busy by another
+ *                    stream.
  */
 struct hl_cs_job {
 	struct list_head	cs_node;
@@ -814,6 +923,7 @@ struct hl_cs_job {
 	u32			user_cb_size;
 	u32			job_cb_size;
 	u8			is_kernel_allocated_cb;
+	u8			contains_dma_pkt;
 };
 
 /**
@@ -833,6 +943,12 @@ struct hl_cs_job {
  * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a
  *                          handle to a kernel-allocated CB object, false
  *                          otherwise (SRAM/DRAM/host address).
+ * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This
+ *                    info is needed later, when adding the 2xMSG_PROT at the
+ *                    end of the JOB, to know which barriers to put in the
+ *                    MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't
+ *                    have streams so the engine can't be busy by another
+ *                    stream.
  */
 struct hl_cs_parser {
 	struct hl_cb		*user_cb;
@@ -846,6 +962,7 @@ struct hl_cs_parser {
 	u32			patched_cb_size;
 	u8			job_id;
 	u8			is_kernel_allocated_cb;
+	u8			contains_dma_pkt;
 };
 
 
@@ -1093,6 +1210,16 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
 #define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
 #define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
 
+#define RMWREG32(reg, val, mask)				\
+	do {							\
+		u32 tmp_ = RREG32(reg);				\
+		tmp_ &= ~(mask);				\
+		tmp_ |= ((val) << __ffs(mask));			\
+		WREG32(reg, tmp_);				\
+	} while (0)
+
+#define RREG32_MASK(reg, mask) ((RREG32(reg) & mask) >> __ffs(mask))
+
 #define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT
 #define REG_FIELD_MASK(reg, field) reg##_##field##_MASK
 #define WREG32_FIELD(reg, offset, field, val)	\
@@ -1282,6 +1409,8 @@ struct hl_device_idle_busy_ts {
  * @idle_busy_ts_idx: index of current entry in idle_busy_ts_arr
  * @id: device minor.
  * @id_control: minor of the control device
+ * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit
+ *                    addresses.
  * @disabled: is device disabled.
  * @late_init_done: is late init stage was done during initialization.
  * @hwmon_initialized: is H/W monitor sensors was initialized.
@@ -1295,11 +1424,19 @@ struct hl_device_idle_busy_ts {
  *                   huge pages.
  * @init_done: is the initialization of the device done.
  * @mmu_enable: is MMU enabled.
+ * @mmu_huge_page_opt: is MMU huge pages optimization enabled.
+ * @clock_gating: is clock gating enabled.
  * @device_cpu_disabled: is the device CPU disabled (due to timeouts)
  * @dma_mask: the dma mask that was set for this device
  * @in_debug: is device under debug. This, together with fpriv_list, enforces
  *            that only a single user is configuring the debug infrastructure.
+ * @power9_64bit_dma_enable: true to enable 64-bit DMA mask support. Relevant
+ *                           only to POWER9 machines.
  * @cdev_sysfs_created: were char devices and sysfs nodes created.
+ * @stop_on_err: true if engines should stop on error.
+ * @supports_sync_stream: is sync stream supported.
+ * @supports_coresight: is CoreSight supported.
+ * @supports_soft_reset: is soft reset supported.
  */
 struct hl_device {
 	struct pci_dev			*pdev;
@@ -1366,6 +1503,7 @@ struct hl_device {
 	u32				idle_busy_ts_idx;
 	u16				id;
 	u16				id_control;
+	u16				cpu_pci_msb_addr;
 	u8				disabled;
 	u8				late_init_done;
 	u8				hwmon_initialized;
@@ -1376,18 +1514,31 @@ struct hl_device {
 	u8				dram_default_page_mapping;
 	u8				pmmu_huge_range;
 	u8				init_done;
+	u8				clock_gating;
 	u8				device_cpu_disabled;
 	u8				dma_mask;
 	u8				in_debug;
+	u8				power9_64bit_dma_enable;
 	u8				cdev_sysfs_created;
+	u8				stop_on_err;
+	u8				supports_sync_stream;
+	u8				supports_coresight;
+	u8				supports_soft_reset;
 
 	/* Parameters for bring-up */
 	u8				mmu_enable;
+	u8				mmu_huge_page_opt;
 	u8				cpu_enable;
 	u8				reset_pcilink;
 	u8				cpu_queues_enable;
 	u8				fw_loading;
 	u8				pldm;
+	u8				axi_drain;
+	u8				sram_scrambler_enable;
+	u8				dram_scrambler_enable;
+	u8				hard_reset_on_fw_events;
+	u8				bmc_enable;
+	u8				rl_enable;
 };
 
 
@@ -1554,8 +1705,10 @@ int hl_cb_pool_fini(struct hl_device *hdev);
 void hl_cs_rollback_all(struct hl_device *hdev);
 struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev,
 		enum hl_queue_type queue_type, bool is_kernel_allocated_cb);
+void hl_sob_reset_error(struct kref *ref);
 
 void goya_set_asic_funcs(struct hl_device *hdev);
+void gaudi_set_asic_funcs(struct hl_device *hdev);
 
 int hl_vm_ctx_init(struct hl_ctx *ctx);
 void hl_vm_ctx_fini(struct hl_ctx *ctx);
@@ -1583,11 +1736,14 @@ int hl_mmu_unmap(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
 void hl_mmu_swap_out(struct hl_ctx *ctx);
 void hl_mmu_swap_in(struct hl_ctx *ctx);
 
-int hl_fw_push_fw_to_device(struct hl_device *hdev, const char *fw_name,
+int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
 				void __iomem *dst);
 int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode);
 int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
 				u16 len, u32 timeout, long *result);
+int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type);
+int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,
+		size_t irq_arr_size);
 int hl_fw_test_cpu_queue(struct hl_device *hdev);
 void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
 						dma_addr_t *dma_handle);
@@ -1596,6 +1752,10 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
 int hl_fw_send_heartbeat(struct hl_device *hdev);
 int hl_fw_armcp_info_get(struct hl_device *hdev);
 int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
+int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
+			u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
+			u32 boot_err0_reg, bool skip_bmc,
+			u32 cpu_timeout, u32 boot_fit_timeout);
 
 int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
 			bool is_wc[3]);
@@ -1605,9 +1765,8 @@ int hl_pci_set_dram_bar_base(struct hl_device *hdev, u8 inbound_region, u8 bar,
 int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
 			u64 dram_base_address, u64 host_phys_base_address,
 			u64 host_phys_size);
-int hl_pci_init(struct hl_device *hdev, u8 dma_mask);
+int hl_pci_init(struct hl_device *hdev);
 void hl_pci_fini(struct hl_device *hdev);
-int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask);
 
 long hl_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr);
 void hl_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq);
@@ -1627,6 +1786,10 @@ void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
 			long value);
 u64 hl_get_max_power(struct hl_device *hdev);
 void hl_set_max_power(struct hl_device *hdev, u64 value);
+int hl_set_voltage(struct hl_device *hdev,
+			int sensor_index, u32 attr, long value);
+int hl_set_current(struct hl_device *hdev,
+			int sensor_index, u32 attr, long value);
 
 #ifdef CONFIG_DEBUG_FS
 
diff --git a/drivers/misc/habanalabs/habanalabs_drv.c b/drivers/misc/habanalabs/habanalabs_drv.c
index b670859c677a..8652c7e5d7f1 100644
--- a/drivers/misc/habanalabs/habanalabs_drv.c
+++ b/drivers/misc/habanalabs/habanalabs_drv.c
@@ -47,6 +47,7 @@ static const struct pci_device_id ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI), },
 	{ 0, }
 };
+MODULE_DEVICE_TABLE(pci, ids);
 
 /*
  * get_asic_type - translate device id to asic type
@@ -171,6 +172,7 @@ out_err:
 	put_pid(hpriv->taskpid);
 
 	kfree(hpriv);
+
 	return rc;
 }
 
@@ -230,8 +232,15 @@ static void set_driver_behavior_per_device(struct hl_device *hdev)
 	hdev->fw_loading = 1;
 	hdev->cpu_queues_enable = 1;
 	hdev->heartbeat = 1;
+	hdev->clock_gating = 1;
 
 	hdev->reset_pcilink = 0;
+	hdev->axi_drain = 0;
+	hdev->sram_scrambler_enable = 1;
+	hdev->dram_scrambler_enable = 1;
+	hdev->rl_enable = 1;
+	hdev->bmc_enable = 1;
+	hdev->hard_reset_on_fw_events = 1;
 }
 
 /*
@@ -267,11 +276,6 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
 			dev_err(&pdev->dev, "Unsupported ASIC\n");
 			rc = -ENODEV;
 			goto free_hdev;
-		} else if (hdev->asic_type == ASIC_GAUDI) {
-			dev_err(&pdev->dev,
-				"GAUDI is not supported by the current kernel\n");
-			rc = -ENODEV;
-			goto free_hdev;
 		}
 	} else {
 		hdev->asic_type = asic_type;
diff --git a/drivers/misc/habanalabs/habanalabs_ioctl.c b/drivers/misc/habanalabs/habanalabs_ioctl.c
index 6474b868ef27..52eedd3a6c3a 100644
--- a/drivers/misc/habanalabs/habanalabs_ioctl.c
+++ b/drivers/misc/habanalabs/habanalabs_ioctl.c
@@ -71,6 +71,8 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
 		min(CARD_NAME_MAX_LEN, HL_INFO_CARD_NAME_MAX_LEN));
 
 	hw_ip.armcp_cpld_version = le32_to_cpu(prop->armcp_info.cpld_version);
+	hw_ip.module_id = le32_to_cpu(prop->armcp_info.card_location);
+
 	hw_ip.psoc_pci_pll_nr = prop->psoc_pci_pll_nr;
 	hw_ip.psoc_pci_pll_nf = prop->psoc_pci_pll_nf;
 	hw_ip.psoc_pci_pll_od = prop->psoc_pci_pll_od;
@@ -258,6 +260,22 @@ static int get_reset_count(struct hl_device *hdev, struct hl_info_args *args)
 		min((size_t) max_size, sizeof(reset_count))) ? -EFAULT : 0;
 }
 
+static int time_sync_info(struct hl_device *hdev, struct hl_info_args *args)
+{
+	struct hl_info_time_sync time_sync = {0};
+	u32 max_size = args->return_size;
+	void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+
+	if ((!max_size) || (!out))
+		return -EINVAL;
+
+	time_sync.device_time = hdev->asic_funcs->get_device_time(hdev);
+	time_sync.host_time = ktime_get_raw_ns();
+
+	return copy_to_user(out, &time_sync,
+		min((size_t) max_size, sizeof(time_sync))) ? -EFAULT : 0;
+}
+
 static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 				struct device *dev)
 {
@@ -315,6 +333,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
 		rc = get_clk_rate(hdev, args);
 		break;
 
+	case HL_INFO_TIME_SYNC:
+		return time_sync_info(hdev, args);
+
 	default:
 		dev_err(dev, "Invalid request %d\n", args->op);
 		rc = -ENOTTY;
diff --git a/drivers/misc/habanalabs/hw_queue.c b/drivers/misc/habanalabs/hw_queue.c
index 91579dde9262..f4434b39ef1b 100644
--- a/drivers/misc/habanalabs/hw_queue.c
+++ b/drivers/misc/habanalabs/hw_queue.c
@@ -111,7 +111,7 @@ static int ext_queue_sanity_checks(struct hl_device *hdev,
 				bool reserve_cq_entry)
 {
 	atomic_t *free_slots =
-			&hdev->completion_queue[q->hw_queue_id].free_slots_cnt;
+			&hdev->completion_queue[q->cq_id].free_slots_cnt;
 	int free_slots_cnt;
 
 	/* Check we have enough space in the queue */
@@ -194,7 +194,7 @@ static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,
 					int num_of_entries)
 {
 	atomic_t *free_slots =
-			&hdev->completion_queue[q->hw_queue_id].free_slots_cnt;
+			&hdev->completion_queue[q->cq_id].free_slots_cnt;
 
 	/*
 	 * Check we have enough space in the completion queue.
@@ -308,13 +308,14 @@ static void ext_queue_schedule_job(struct hl_cs_job *job)
 	 * No need to check if CQ is full because it was already
 	 * checked in ext_queue_sanity_checks
 	 */
-	cq = &hdev->completion_queue[q->hw_queue_id];
+	cq = &hdev->completion_queue[q->cq_id];
 	cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry);
 
 	hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,
 						cq_addr,
 						le32_to_cpu(cq_pkt.data),
-						q->hw_queue_id);
+						q->msi_vec,
+						job->contains_dma_pkt);
 
 	q->shadow_queue[hl_pi_2_offset(q->pi)] = job;
 
@@ -401,21 +402,111 @@ static void hw_queue_schedule_job(struct hl_cs_job *job)
 	 * No need to check if CQ is full because it was already
 	 * checked in hw_queue_sanity_checks
 	 */
-	cq = &hdev->completion_queue[q->hw_queue_id];
+	cq = &hdev->completion_queue[q->cq_id];
+
 	cq->pi = hl_cq_inc_ptr(cq->pi);
 
 	ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
 }
 
 /*
- * hl_hw_queue_schedule_cs - schedule a command submission
- *
- * @job        : pointer to the CS
+ * init_signal_wait_cs - initialize a signal/wait CS
+ * @cs: pointer to the signal/wait CS
  *
+ * H/W queues spinlock should be taken before calling this function
+ */
+static void init_signal_wait_cs(struct hl_cs *cs)
+{
+	struct hl_ctx *ctx = cs->ctx;
+	struct hl_device *hdev = ctx->hdev;
+	struct hl_hw_queue *hw_queue;
+	struct hl_cs_compl *cs_cmpl =
+			container_of(cs->fence, struct hl_cs_compl, base_fence);
+
+	struct hl_hw_sob *hw_sob;
+	struct hl_cs_job *job;
+	u32 q_idx;
+
+	/* There is only one job in a signal/wait CS */
+	job = list_first_entry(&cs->job_list, struct hl_cs_job,
+				cs_node);
+	q_idx = job->hw_queue_id;
+	hw_queue = &hdev->kernel_queues[q_idx];
+
+	if (cs->type & CS_TYPE_SIGNAL) {
+		hw_sob = &hw_queue->hw_sob[hw_queue->curr_sob_offset];
+
+		cs_cmpl->hw_sob = hw_sob;
+		cs_cmpl->sob_val = hw_queue->next_sob_val++;
+
+		dev_dbg(hdev->dev,
+			"generate signal CB, sob_id: %d, sob val: 0x%x, q_idx: %d\n",
+			cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx);
+
+		hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,
+					cs_cmpl->hw_sob->sob_id);
+
+		kref_get(&hw_sob->kref);
+
+		/* check for wraparound */
+		if (hw_queue->next_sob_val == HL_MAX_SOB_VAL) {
+			/*
+			 * Decrement as we reached the max value.
+			 * The release function won't be called here as we've
+			 * just incremented the refcount.
+			 */
+			kref_put(&hw_sob->kref, hl_sob_reset_error);
+			hw_queue->next_sob_val = 1;
+			/* only two SOBs are currently in use */
+			hw_queue->curr_sob_offset =
+					(hw_queue->curr_sob_offset + 1) %
+						HL_RSVD_SOBS_IN_USE;
+
+			dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
+					hw_queue->curr_sob_offset, q_idx);
+		}
+	} else if (cs->type & CS_TYPE_WAIT) {
+		struct hl_cs_compl *signal_cs_cmpl;
+
+		signal_cs_cmpl = container_of(cs->signal_fence,
+						struct hl_cs_compl,
+						base_fence);
+
+		/* copy the the SOB id and value of the signal CS */
+		cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
+		cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
+
+		dev_dbg(hdev->dev,
+			"generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d\n",
+			cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
+			hw_queue->base_mon_id, q_idx);
+
+		hdev->asic_funcs->gen_wait_cb(hdev, job->patched_cb,
+						cs_cmpl->hw_sob->sob_id,
+						cs_cmpl->sob_val,
+						hw_queue->base_mon_id,
+						q_idx);
+
+		kref_get(&cs_cmpl->hw_sob->kref);
+		/*
+		 * Must put the signal fence after the SOB refcnt increment so
+		 * the SOB refcnt won't turn 0 and reset the SOB before the
+		 * wait CS was submitted.
+		 */
+		mb();
+		dma_fence_put(cs->signal_fence);
+		cs->signal_fence = NULL;
+	}
+}
+
+/*
+ * hl_hw_queue_schedule_cs - schedule a command submission
+ * @cs: pointer to the CS
  */
 int hl_hw_queue_schedule_cs(struct hl_cs *cs)
 {
-	struct hl_device *hdev = cs->ctx->hdev;
+	struct hl_ctx *ctx = cs->ctx;
+	struct hl_device *hdev = ctx->hdev;
 	struct hl_cs_job *job, *tmp;
 	struct hl_hw_queue *q;
 	int rc = 0, i, cq_cnt;
@@ -461,6 +552,9 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs)
 		}
 	}
 
+	if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT))
+		init_signal_wait_cs(cs);
+
 	spin_lock(&hdev->hw_queues_mirror_lock);
 	list_add_tail(&cs->mirror_node, &hdev->hw_queues_mirror_list);
 
@@ -569,6 +663,9 @@ static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
 	q->ci = 0;
 	q->pi = 0;
 
+	if (!is_cpu_queue)
+		hdev->asic_funcs->ext_queue_init(hdev, q->hw_queue_id);
+
 	return 0;
 
 free_queue:
@@ -791,5 +888,8 @@ void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)
 			((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))
 			continue;
 		q->pi = q->ci = 0;
+
+		if (q->queue_type == QUEUE_TYPE_EXT)
+			hdev->asic_funcs->ext_queue_reset(hdev, q->hw_queue_id);
 	}
 }
diff --git a/drivers/misc/habanalabs/hwmon.c b/drivers/misc/habanalabs/hwmon.c
index a21a26e07c3b..8c6cd77e6af6 100644
--- a/drivers/misc/habanalabs/hwmon.c
+++ b/drivers/misc/habanalabs/hwmon.c
@@ -200,6 +200,7 @@ static int hl_write(struct device *dev, enum hwmon_sensor_types type,
 	case hwmon_temp:
 		switch (attr) {
 		case hwmon_temp_offset:
+		case hwmon_temp_reset_history:
 			break;
 		default:
 			return -EINVAL;
@@ -216,6 +217,24 @@ static int hl_write(struct device *dev, enum hwmon_sensor_types type,
 		}
 		hl_set_pwm_info(hdev, channel, attr, val);
 		break;
+	case hwmon_in:
+		switch (attr) {
+		case hwmon_in_reset_history:
+			break;
+		default:
+			return -EINVAL;
+		}
+		hl_set_voltage(hdev, channel, attr, val);
+		break;
+	case hwmon_curr:
+		switch (attr) {
+		case hwmon_curr_reset_history:
+			break;
+		default:
+			return -EINVAL;
+		}
+		hl_set_current(hdev, channel, attr, val);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -237,6 +256,8 @@ static umode_t hl_is_visible(const void *data, enum hwmon_sensor_types type,
 			return 0444;
 		case hwmon_temp_offset:
 			return 0644;
+		case hwmon_temp_reset_history:
+			return 0200;
 		}
 		break;
 	case hwmon_in:
@@ -246,6 +267,8 @@ static umode_t hl_is_visible(const void *data, enum hwmon_sensor_types type,
 		case hwmon_in_max:
 		case hwmon_in_highest:
 			return 0444;
+		case hwmon_in_reset_history:
+			return 0200;
 		}
 		break;
 	case hwmon_curr:
@@ -255,6 +278,8 @@ static umode_t hl_is_visible(const void *data, enum hwmon_sensor_types type,
 		case hwmon_curr_max:
 		case hwmon_curr_highest:
 			return 0444;
+		case hwmon_curr_reset_history:
+			return 0200;
 		}
 		break;
 	case hwmon_fan:
@@ -462,6 +487,56 @@ void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr,
 			sensor_index, rc);
 }
 
+int hl_set_voltage(struct hl_device *hdev,
+			int sensor_index, u32 attr, long value)
+{
+	struct armcp_packet pkt;
+	int rc;
+
+	memset(&pkt, 0, sizeof(pkt));
+
+	pkt.ctl = cpu_to_le32(ARMCP_PACKET_VOLTAGE_SET <<
+				ARMCP_PKT_CTL_OPCODE_SHIFT);
+	pkt.sensor_index = __cpu_to_le16(sensor_index);
+	pkt.type = __cpu_to_le16(attr);
+	pkt.value = __cpu_to_le64(value);
+
+	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+						SENSORS_PKT_TIMEOUT, NULL);
+
+	if (rc)
+		dev_err(hdev->dev,
+			"Failed to set voltage of sensor %d, error %d\n",
+			sensor_index, rc);
+
+	return rc;
+}
+
+int hl_set_current(struct hl_device *hdev,
+			int sensor_index, u32 attr, long value)
+{
+	struct armcp_packet pkt;
+	int rc;
+
+	memset(&pkt, 0, sizeof(pkt));
+
+	pkt.ctl = cpu_to_le32(ARMCP_PACKET_CURRENT_SET <<
+				ARMCP_PKT_CTL_OPCODE_SHIFT);
+	pkt.sensor_index = __cpu_to_le16(sensor_index);
+	pkt.type = __cpu_to_le16(attr);
+	pkt.value = __cpu_to_le64(value);
+
+	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
+						SENSORS_PKT_TIMEOUT, NULL);
+
+	if (rc)
+		dev_err(hdev->dev,
+			"Failed to set current of sensor %d, error %d\n",
+			sensor_index, rc);
+
+	return rc;
+}
+
 int hl_hwmon_init(struct hl_device *hdev)
 {
 	struct device *dev = hdev->pdev ? &hdev->pdev->dev : hdev->dev;
diff --git a/drivers/misc/habanalabs/include/armcp_if.h b/drivers/misc/habanalabs/include/armcp_if.h
index bdd0a4c3a9cf..a34fc39ad87e 100644
--- a/drivers/misc/habanalabs/include/armcp_if.h
+++ b/drivers/misc/habanalabs/include/armcp_if.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0
  *
- * Copyright 2016-2019 HabanaLabs, Ltd.
+ * Copyright 2016-2020 HabanaLabs, Ltd.
  * All Rights Reserved.
  *
  */
@@ -35,7 +35,8 @@ struct hl_eq_entry {
 enum pq_init_status {
 	PQ_INIT_STATUS_NA = 0,
 	PQ_INIT_STATUS_READY_FOR_CP,
-	PQ_INIT_STATUS_READY_FOR_HOST
+	PQ_INIT_STATUS_READY_FOR_HOST,
+	PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI
 };
 
 /*
@@ -193,6 +194,16 @@ enum pq_init_status {
  *       Set the value of the offset property of a specified thermal sensor.
  *       The packet's arguments specify the desired sensor and the field to
  *       set.
+ *
+ * ARMCP_PACKET_VOLTAGE_SET -
+ *       Trigger the reset_history property of a specified voltage sensor.
+ *       The packet's arguments specify the desired sensor and the field to
+ *       set.
+ *
+ * ARMCP_PACKET_CURRENT_SET -
+ *       Trigger the reset_history property of a specified current sensor.
+ *       The packet's arguments specify the desired sensor and the field to
+ *       set.
  */
 
 enum armcp_packet_id {
@@ -220,6 +231,8 @@ enum armcp_packet_id {
 	ARMCP_PACKET_EEPROM_DATA_GET,		/* sysfs */
 	ARMCP_RESERVED,
 	ARMCP_PACKET_TEMPERATURE_SET,		/* sysfs */
+	ARMCP_PACKET_VOLTAGE_SET,		/* sysfs */
+	ARMCP_PACKET_CURRENT_SET,		/* sysfs */
 };
 
 #define ARMCP_PACKET_FENCE_VAL	0xFE8CE7A5
@@ -288,21 +301,24 @@ enum armcp_temp_type {
 	armcp_temp_crit,
 	armcp_temp_crit_hyst,
 	armcp_temp_offset = 19,
-	armcp_temp_highest = 22
+	armcp_temp_highest = 22,
+	armcp_temp_reset_history = 23
 };
 
 enum armcp_in_attributes {
 	armcp_in_input,
 	armcp_in_min,
 	armcp_in_max,
-	armcp_in_highest = 7
+	armcp_in_highest = 7,
+	armcp_in_reset_history
 };
 
 enum armcp_curr_attributes {
 	armcp_curr_input,
 	armcp_curr_min,
 	armcp_curr_max,
-	armcp_curr_highest = 7
+	armcp_curr_highest = 7,
+	armcp_curr_reset_history
 };
 
 enum armcp_fan_attributes {
@@ -336,10 +352,23 @@ struct armcp_sensor {
 };
 
 /**
+ * struct armcp_card_types - ASIC card type.
+ * @armcp_card_type_pci: PCI card.
+ * @armcp_card_type_pmc: PCI Mezzanine Card.
+ */
+enum armcp_card_types {
+	armcp_card_type_pci,
+	armcp_card_type_pmc
+};
+
+/**
  * struct armcp_info - Info from ArmCP that is necessary to the host's driver
  * @sensors: available sensors description.
  * @kernel_version: ArmCP linux kernel version.
  * @reserved: reserved field.
+ * @card_type: card configuration type.
+ * @card_location: in a server, each card has different connections topology
+ *                 depending on its location (relevant for PMC card type)
  * @cpld_version: CPLD programmed F/W version.
  * @infineon_version: Infineon main DC-DC version.
  * @fuse_version: silicon production FUSE information.
@@ -351,7 +380,9 @@ struct armcp_sensor {
 struct armcp_info {
 	struct armcp_sensor sensors[ARMCP_MAX_SENSORS];
 	__u8 kernel_version[VERSION_MAX_LEN];
-	__le32 reserved[3];
+	__le32 reserved;
+	__le32 card_type;
+	__le32 card_location;
 	__le32 cpld_version;
 	__le32 infineon_version;
 	__u8 fuse_version[VERSION_MAX_LEN];
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/cpu_if_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/cpu_if_regs.h
new file mode 100644
index 000000000000..cf80e31317ad
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/cpu_if_regs.h
@@ -0,0 +1,174 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_CPU_IF_REGS_H_
+#define ASIC_REG_CPU_IF_REGS_H_
+
+/*
+ *****************************************
+ *   CPU_IF (Prototype: CPU_IF)
+ *****************************************
+ */
+
+#define mmCPU_IF_ARUSER_OVR                                          0x442104
+
+#define mmCPU_IF_ARUSER_OVR_EN                                       0x442108
+
+#define mmCPU_IF_AWUSER_OVR                                          0x44210C
+
+#define mmCPU_IF_AWUSER_OVR_EN                                       0x442110
+
+#define mmCPU_IF_AXCACHE_OVR                                         0x442114
+
+#define mmCPU_IF_LOCK_OVR                                            0x442118
+
+#define mmCPU_IF_PROT_OVR                                            0x44211C
+
+#define mmCPU_IF_MAX_OUTSTANDING                                     0x442120
+
+#define mmCPU_IF_EARLY_BRESP_EN                                      0x442124
+
+#define mmCPU_IF_FORCE_RSP_OK                                        0x442128
+
+#define mmCPU_IF_CPU_MSB_ADDR                                        0x44212C
+
+#define mmCPU_IF_AXI_SPLIT_INTR                                      0x442130
+
+#define mmCPU_IF_TOTAL_WR_CNT                                        0x442140
+
+#define mmCPU_IF_INFLIGHT_WR_CNT                                     0x442144
+
+#define mmCPU_IF_TOTAL_RD_CNT                                        0x442150
+
+#define mmCPU_IF_INFLIGHT_RD_CNT                                     0x442154
+
+#define mmCPU_IF_PF_PQ_PI                                            0x442200
+
+#define mmCPU_IF_PQ_BASE_ADDR_LOW                                    0x442204
+
+#define mmCPU_IF_PQ_BASE_ADDR_HIGH                                   0x442208
+
+#define mmCPU_IF_PQ_LENGTH                                           0x44220C
+
+#define mmCPU_IF_CQ_BASE_ADDR_LOW                                    0x442210
+
+#define mmCPU_IF_CQ_BASE_ADDR_HIGH                                   0x442214
+
+#define mmCPU_IF_CQ_LENGTH                                           0x442218
+
+#define mmCPU_IF_EQ_BASE_ADDR_LOW                                    0x442220
+
+#define mmCPU_IF_EQ_BASE_ADDR_HIGH                                   0x442224
+
+#define mmCPU_IF_EQ_LENGTH                                           0x442228
+
+#define mmCPU_IF_EQ_RD_OFFS                                          0x44222C
+
+#define mmCPU_IF_QUEUE_INIT                                          0x442230
+
+#define mmCPU_IF_TPC_SERR_INTR_STS                                   0x442300
+
+#define mmCPU_IF_TPC_SERR_INTR_CLR                                   0x442304
+
+#define mmCPU_IF_TPC_SERR_INTR_MASK                                  0x442308
+
+#define mmCPU_IF_TPC_DERR_INTR_STS                                   0x442310
+
+#define mmCPU_IF_TPC_DERR_INTR_CLR                                   0x442314
+
+#define mmCPU_IF_TPC_DERR_INTR_MASK                                  0x442318
+
+#define mmCPU_IF_DMA_SERR_INTR_STS                                   0x442320
+
+#define mmCPU_IF_DMA_SERR_INTR_CLR                                   0x442324
+
+#define mmCPU_IF_DMA_SERR_INTR_MASK                                  0x442328
+
+#define mmCPU_IF_DMA_DERR_INTR_STS                                   0x442330
+
+#define mmCPU_IF_DMA_DERR_INTR_CLR                                   0x442334
+
+#define mmCPU_IF_DMA_DERR_INTR_MASK                                  0x442338
+
+#define mmCPU_IF_SRAM_SERR_INTR_STS                                  0x442340
+
+#define mmCPU_IF_SRAM_SERR_INTR_CLR                                  0x442344
+
+#define mmCPU_IF_SRAM_SERR_INTR_MASK                                 0x442348
+
+#define mmCPU_IF_SRAM_DERR_INTR_STS                                  0x442350
+
+#define mmCPU_IF_SRAM_DERR_INTR_CLR                                  0x442354
+
+#define mmCPU_IF_SRAM_DERR_INTR_MASK                                 0x442358
+
+#define mmCPU_IF_NIC_SERR_INTR_STS                                   0x442360
+
+#define mmCPU_IF_NIC_SERR_INTR_CLR                                   0x442364
+
+#define mmCPU_IF_NIC_SERR_INTR_MASK                                  0x442368
+
+#define mmCPU_IF_NIC_DERR_INTR_STS                                   0x442370
+
+#define mmCPU_IF_NIC_DERR_INTR_CLR                                   0x442374
+
+#define mmCPU_IF_NIC_DERR_INTR_MASK                                  0x442378
+
+#define mmCPU_IF_DMA_IF_SERR_INTR_STS                                0x442380
+
+#define mmCPU_IF_DMA_IF_SERR_INTR_CLR                                0x442384
+
+#define mmCPU_IF_DMA_IF_SERR_INTR_MASK                               0x442388
+
+#define mmCPU_IF_DMA_IF_DERR_INTR_STS                                0x442390
+
+#define mmCPU_IF_DMA_IF_DERR_INTR_CLR                                0x442394
+
+#define mmCPU_IF_DMA_IF_DERR_INTR_MASK                               0x442398
+
+#define mmCPU_IF_HBM_SERR_INTR_STS                                   0x4423A0
+
+#define mmCPU_IF_HBM_SERR_INTR_CLR                                   0x4423A4
+
+#define mmCPU_IF_HBM_SERR_INTR_MASK                                  0x4423A8
+
+#define mmCPU_IF_HBM_DERR_INTR_STS                                   0x4423B0
+
+#define mmCPU_IF_HBM_DERR_INTR_CLR                                   0x4423B4
+
+#define mmCPU_IF_HBM_DERR_INTR_MASK                                  0x4423B8
+
+#define mmCPU_IF_PLL_SEI_INTR_STS                                    0x442400
+
+#define mmCPU_IF_PLL_SEI_INTR_CLR                                    0x442404
+
+#define mmCPU_IF_PLL_SEI_INTR_MASK                                   0x442408
+
+#define mmCPU_IF_NIC_SEI_INTR_STS                                    0x442410
+
+#define mmCPU_IF_NIC_SEI_INTR_CLR                                    0x442414
+
+#define mmCPU_IF_NIC_SEI_INTR_MASK                                   0x442418
+
+#define mmCPU_IF_DMA_SEI_INTR_STS                                    0x442420
+
+#define mmCPU_IF_DMA_SEI_INTR_CLR                                    0x442424
+
+#define mmCPU_IF_DMA_SEI_INTR_MASK                                   0x442428
+
+#define mmCPU_IF_DMA_IF_SEI_INTR_STS                                 0x442430
+
+#define mmCPU_IF_DMA_IF_SEI_INTR_CLR                                 0x442434
+
+#define mmCPU_IF_DMA_IF_SEI_INTR_MASK                                0x442438
+
+#endif /* ASIC_REG_CPU_IF_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_masks.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_masks.h
new file mode 100644
index 000000000000..d079a37acab8
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_masks.h
@@ -0,0 +1,348 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA0_CORE_MASKS_H_
+#define ASIC_REG_DMA0_CORE_MASKS_H_
+
+/*
+ *****************************************
+ *   DMA0_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+/* DMA0_CORE_CFG_0 */
+#define DMA0_CORE_CFG_0_EN_SHIFT                                     0
+#define DMA0_CORE_CFG_0_EN_MASK                                      0x1
+
+/* DMA0_CORE_CFG_1 */
+#define DMA0_CORE_CFG_1_HALT_SHIFT                                   0
+#define DMA0_CORE_CFG_1_HALT_MASK                                    0x1
+#define DMA0_CORE_CFG_1_FLUSH_SHIFT                                  1
+#define DMA0_CORE_CFG_1_FLUSH_MASK                                   0x2
+#define DMA0_CORE_CFG_1_SB_FORCE_MISS_SHIFT                          2
+#define DMA0_CORE_CFG_1_SB_FORCE_MISS_MASK                           0x4
+
+/* DMA0_CORE_LBW_MAX_OUTSTAND */
+#define DMA0_CORE_LBW_MAX_OUTSTAND_VAL_SHIFT                         0
+#define DMA0_CORE_LBW_MAX_OUTSTAND_VAL_MASK                          0x1F
+
+/* DMA0_CORE_SRC_BASE_LO */
+#define DMA0_CORE_SRC_BASE_LO_VAL_SHIFT                              0
+#define DMA0_CORE_SRC_BASE_LO_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_SRC_BASE_HI */
+#define DMA0_CORE_SRC_BASE_HI_VAL_SHIFT                              0
+#define DMA0_CORE_SRC_BASE_HI_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_DST_BASE_LO */
+#define DMA0_CORE_DST_BASE_LO_VAL_SHIFT                              0
+#define DMA0_CORE_DST_BASE_LO_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_DST_BASE_HI */
+#define DMA0_CORE_DST_BASE_HI_VAL_SHIFT                              0
+#define DMA0_CORE_DST_BASE_HI_VAL_MASK                               0xFFFFFF
+#define DMA0_CORE_DST_BASE_HI_CTX_ID_HI_SHIFT                        24
+#define DMA0_CORE_DST_BASE_HI_CTX_ID_HI_MASK                         0xFF000000
+
+/* DMA0_CORE_SRC_TSIZE_1 */
+#define DMA0_CORE_SRC_TSIZE_1_VAL_SHIFT                              0
+#define DMA0_CORE_SRC_TSIZE_1_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_SRC_STRIDE_1 */
+#define DMA0_CORE_SRC_STRIDE_1_VAL_SHIFT                             0
+#define DMA0_CORE_SRC_STRIDE_1_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_SRC_TSIZE_2 */
+#define DMA0_CORE_SRC_TSIZE_2_VAL_SHIFT                              0
+#define DMA0_CORE_SRC_TSIZE_2_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_SRC_STRIDE_2 */
+#define DMA0_CORE_SRC_STRIDE_2_VAL_SHIFT                             0
+#define DMA0_CORE_SRC_STRIDE_2_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_SRC_TSIZE_3 */
+#define DMA0_CORE_SRC_TSIZE_3_VAL_SHIFT                              0
+#define DMA0_CORE_SRC_TSIZE_3_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_SRC_STRIDE_3 */
+#define DMA0_CORE_SRC_STRIDE_3_VAL_SHIFT                             0
+#define DMA0_CORE_SRC_STRIDE_3_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_SRC_TSIZE_4 */
+#define DMA0_CORE_SRC_TSIZE_4_VAL_SHIFT                              0
+#define DMA0_CORE_SRC_TSIZE_4_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_SRC_STRIDE_4 */
+#define DMA0_CORE_SRC_STRIDE_4_VAL_SHIFT                             0
+#define DMA0_CORE_SRC_STRIDE_4_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_SRC_TSIZE_0 */
+#define DMA0_CORE_SRC_TSIZE_0_VAL_SHIFT                              0
+#define DMA0_CORE_SRC_TSIZE_0_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_DST_TSIZE_1 */
+#define DMA0_CORE_DST_TSIZE_1_VAL_SHIFT                              0
+#define DMA0_CORE_DST_TSIZE_1_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_DST_STRIDE_1 */
+#define DMA0_CORE_DST_STRIDE_1_VAL_SHIFT                             0
+#define DMA0_CORE_DST_STRIDE_1_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_DST_TSIZE_2 */
+#define DMA0_CORE_DST_TSIZE_2_VAL_SHIFT                              0
+#define DMA0_CORE_DST_TSIZE_2_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_DST_STRIDE_2 */
+#define DMA0_CORE_DST_STRIDE_2_VAL_SHIFT                             0
+#define DMA0_CORE_DST_STRIDE_2_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_DST_TSIZE_3 */
+#define DMA0_CORE_DST_TSIZE_3_VAL_SHIFT                              0
+#define DMA0_CORE_DST_TSIZE_3_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_DST_STRIDE_3 */
+#define DMA0_CORE_DST_STRIDE_3_VAL_SHIFT                             0
+#define DMA0_CORE_DST_STRIDE_3_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_DST_TSIZE_4 */
+#define DMA0_CORE_DST_TSIZE_4_VAL_SHIFT                              0
+#define DMA0_CORE_DST_TSIZE_4_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_DST_STRIDE_4 */
+#define DMA0_CORE_DST_STRIDE_4_VAL_SHIFT                             0
+#define DMA0_CORE_DST_STRIDE_4_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_DST_TSIZE_0 */
+#define DMA0_CORE_DST_TSIZE_0_VAL_SHIFT                              0
+#define DMA0_CORE_DST_TSIZE_0_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_CORE_COMMIT */
+#define DMA0_CORE_COMMIT_WR_COMP_EN_SHIFT                            0
+#define DMA0_CORE_COMMIT_WR_COMP_EN_MASK                             0x1
+#define DMA0_CORE_COMMIT_TRANSPOSE_SHIFT                             1
+#define DMA0_CORE_COMMIT_TRANSPOSE_MASK                              0x2
+#define DMA0_CORE_COMMIT_DTYPE_SHIFT                                 2
+#define DMA0_CORE_COMMIT_DTYPE_MASK                                  0x4
+#define DMA0_CORE_COMMIT_LIN_SHIFT                                   3
+#define DMA0_CORE_COMMIT_LIN_MASK                                    0x8
+#define DMA0_CORE_COMMIT_MEM_SET_SHIFT                               4
+#define DMA0_CORE_COMMIT_MEM_SET_MASK                                0x10
+#define DMA0_CORE_COMMIT_COMPRESS_SHIFT                              5
+#define DMA0_CORE_COMMIT_COMPRESS_MASK                               0x20
+#define DMA0_CORE_COMMIT_DECOMPRESS_SHIFT                            6
+#define DMA0_CORE_COMMIT_DECOMPRESS_MASK                             0x40
+#define DMA0_CORE_COMMIT_CTX_ID_SHIFT                                16
+#define DMA0_CORE_COMMIT_CTX_ID_MASK                                 0xFF0000
+
+/* DMA0_CORE_WR_COMP_WDATA */
+#define DMA0_CORE_WR_COMP_WDATA_VAL_SHIFT                            0
+#define DMA0_CORE_WR_COMP_WDATA_VAL_MASK                             0xFFFFFFFF
+
+/* DMA0_CORE_WR_COMP_ADDR_LO */
+#define DMA0_CORE_WR_COMP_ADDR_LO_VAL_SHIFT                          0
+#define DMA0_CORE_WR_COMP_ADDR_LO_VAL_MASK                           0xFFFFFFFF
+
+/* DMA0_CORE_WR_COMP_ADDR_HI */
+#define DMA0_CORE_WR_COMP_ADDR_HI_VAL_SHIFT                          0
+#define DMA0_CORE_WR_COMP_ADDR_HI_VAL_MASK                           0xFFFFFFFF
+
+/* DMA0_CORE_WR_COMP_AWUSER_31_11 */
+#define DMA0_CORE_WR_COMP_AWUSER_31_11_VAL_SHIFT                     0
+#define DMA0_CORE_WR_COMP_AWUSER_31_11_VAL_MASK                      0x1FFFFF
+
+/* DMA0_CORE_TE_NUMROWS */
+#define DMA0_CORE_TE_NUMROWS_VAL_SHIFT                               0
+#define DMA0_CORE_TE_NUMROWS_VAL_MASK                                0xFFFFFFFF
+
+/* DMA0_CORE_PROT */
+#define DMA0_CORE_PROT_VAL_SHIFT                                     0
+#define DMA0_CORE_PROT_VAL_MASK                                      0x1
+#define DMA0_CORE_PROT_ERR_VAL_SHIFT                                 1
+#define DMA0_CORE_PROT_ERR_VAL_MASK                                  0x2
+
+/* DMA0_CORE_SECURE_PROPS */
+#define DMA0_CORE_SECURE_PROPS_ASID_SHIFT                            0
+#define DMA0_CORE_SECURE_PROPS_ASID_MASK                             0x3FF
+#define DMA0_CORE_SECURE_PROPS_MMBP_SHIFT                            10
+#define DMA0_CORE_SECURE_PROPS_MMBP_MASK                             0x400
+
+/* DMA0_CORE_NON_SECURE_PROPS */
+#define DMA0_CORE_NON_SECURE_PROPS_ASID_SHIFT                        0
+#define DMA0_CORE_NON_SECURE_PROPS_ASID_MASK                         0x3FF
+#define DMA0_CORE_NON_SECURE_PROPS_MMBP_SHIFT                        10
+#define DMA0_CORE_NON_SECURE_PROPS_MMBP_MASK                         0x400
+
+/* DMA0_CORE_RD_MAX_OUTSTAND */
+#define DMA0_CORE_RD_MAX_OUTSTAND_VAL_SHIFT                          0
+#define DMA0_CORE_RD_MAX_OUTSTAND_VAL_MASK                           0xFFF
+
+/* DMA0_CORE_RD_MAX_SIZE */
+#define DMA0_CORE_RD_MAX_SIZE_DATA_SHIFT                             0
+#define DMA0_CORE_RD_MAX_SIZE_DATA_MASK                              0x7FF
+#define DMA0_CORE_RD_MAX_SIZE_MD_SHIFT                               16
+#define DMA0_CORE_RD_MAX_SIZE_MD_MASK                                0x7FF0000
+
+/* DMA0_CORE_RD_ARCACHE */
+#define DMA0_CORE_RD_ARCACHE_VAL_SHIFT                               0
+#define DMA0_CORE_RD_ARCACHE_VAL_MASK                                0xF
+
+/* DMA0_CORE_RD_ARUSER_31_11 */
+#define DMA0_CORE_RD_ARUSER_31_11_VAL_SHIFT                          0
+#define DMA0_CORE_RD_ARUSER_31_11_VAL_MASK                           0x1FFFFF
+
+/* DMA0_CORE_RD_INFLIGHTS */
+#define DMA0_CORE_RD_INFLIGHTS_VAL_SHIFT                             0
+#define DMA0_CORE_RD_INFLIGHTS_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_WR_MAX_OUTSTAND */
+#define DMA0_CORE_WR_MAX_OUTSTAND_VAL_SHIFT                          0
+#define DMA0_CORE_WR_MAX_OUTSTAND_VAL_MASK                           0xFFF
+
+/* DMA0_CORE_WR_MAX_AWID */
+#define DMA0_CORE_WR_MAX_AWID_VAL_SHIFT                              0
+#define DMA0_CORE_WR_MAX_AWID_VAL_MASK                               0xFFFF
+
+/* DMA0_CORE_WR_AWCACHE */
+#define DMA0_CORE_WR_AWCACHE_VAL_SHIFT                               0
+#define DMA0_CORE_WR_AWCACHE_VAL_MASK                                0xF
+
+/* DMA0_CORE_WR_AWUSER_31_11 */
+#define DMA0_CORE_WR_AWUSER_31_11_VAL_SHIFT                          0
+#define DMA0_CORE_WR_AWUSER_31_11_VAL_MASK                           0x1FFFFF
+
+/* DMA0_CORE_WR_INFLIGHTS */
+#define DMA0_CORE_WR_INFLIGHTS_VAL_SHIFT                             0
+#define DMA0_CORE_WR_INFLIGHTS_VAL_MASK                              0xFFFF
+
+/* DMA0_CORE_RD_RATE_LIM_CFG_0 */
+#define DMA0_CORE_RD_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                  0
+#define DMA0_CORE_RD_RATE_LIM_CFG_0_RST_TOKEN_MASK                   0xFF
+#define DMA0_CORE_RD_RATE_LIM_CFG_0_SAT_SHIFT                        16
+#define DMA0_CORE_RD_RATE_LIM_CFG_0_SAT_MASK                         0xFF0000
+
+/* DMA0_CORE_RD_RATE_LIM_CFG_1 */
+#define DMA0_CORE_RD_RATE_LIM_CFG_1_TOUT_SHIFT                       0
+#define DMA0_CORE_RD_RATE_LIM_CFG_1_TOUT_MASK                        0xFF
+#define DMA0_CORE_RD_RATE_LIM_CFG_1_EN_SHIFT                         31
+#define DMA0_CORE_RD_RATE_LIM_CFG_1_EN_MASK                          0x80000000
+
+/* DMA0_CORE_WR_RATE_LIM_CFG_0 */
+#define DMA0_CORE_WR_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                  0
+#define DMA0_CORE_WR_RATE_LIM_CFG_0_RST_TOKEN_MASK                   0xFF
+#define DMA0_CORE_WR_RATE_LIM_CFG_0_SAT_SHIFT                        16
+#define DMA0_CORE_WR_RATE_LIM_CFG_0_SAT_MASK                         0xFF0000
+
+/* DMA0_CORE_WR_RATE_LIM_CFG_1 */
+#define DMA0_CORE_WR_RATE_LIM_CFG_1_TOUT_SHIFT                       0
+#define DMA0_CORE_WR_RATE_LIM_CFG_1_TOUT_MASK                        0xFF
+#define DMA0_CORE_WR_RATE_LIM_CFG_1_EN_SHIFT                         31
+#define DMA0_CORE_WR_RATE_LIM_CFG_1_EN_MASK                          0x80000000
+
+/* DMA0_CORE_ERR_CFG */
+#define DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT                           0
+#define DMA0_CORE_ERR_CFG_ERR_MSG_EN_MASK                            0x1
+#define DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT                          1
+#define DMA0_CORE_ERR_CFG_STOP_ON_ERR_MASK                           0x2
+
+/* DMA0_CORE_ERR_CAUSE */
+#define DMA0_CORE_ERR_CAUSE_HBW_RD_ERR_SHIFT                         0
+#define DMA0_CORE_ERR_CAUSE_HBW_RD_ERR_MASK                          0x1
+#define DMA0_CORE_ERR_CAUSE_HBW_WR_ERR_SHIFT                         1
+#define DMA0_CORE_ERR_CAUSE_HBW_WR_ERR_MASK                          0x2
+#define DMA0_CORE_ERR_CAUSE_LBW_WR_ERR_SHIFT                         2
+#define DMA0_CORE_ERR_CAUSE_LBW_WR_ERR_MASK                          0x4
+#define DMA0_CORE_ERR_CAUSE_DESC_OVF_SHIFT                           3
+#define DMA0_CORE_ERR_CAUSE_DESC_OVF_MASK                            0x8
+
+/* DMA0_CORE_ERRMSG_ADDR_LO */
+#define DMA0_CORE_ERRMSG_ADDR_LO_VAL_SHIFT                           0
+#define DMA0_CORE_ERRMSG_ADDR_LO_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_CORE_ERRMSG_ADDR_HI */
+#define DMA0_CORE_ERRMSG_ADDR_HI_VAL_SHIFT                           0
+#define DMA0_CORE_ERRMSG_ADDR_HI_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_CORE_ERRMSG_WDATA */
+#define DMA0_CORE_ERRMSG_WDATA_VAL_SHIFT                             0
+#define DMA0_CORE_ERRMSG_WDATA_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_CORE_STS0 */
+#define DMA0_CORE_STS0_RD_REQ_CNT_SHIFT                              0
+#define DMA0_CORE_STS0_RD_REQ_CNT_MASK                               0x7FFF
+#define DMA0_CORE_STS0_WR_REQ_CNT_SHIFT                              16
+#define DMA0_CORE_STS0_WR_REQ_CNT_MASK                               0x7FFF0000
+#define DMA0_CORE_STS0_BUSY_SHIFT                                    31
+#define DMA0_CORE_STS0_BUSY_MASK                                     0x80000000
+
+/* DMA0_CORE_STS1 */
+#define DMA0_CORE_STS1_IS_HALT_SHIFT                                 0
+#define DMA0_CORE_STS1_IS_HALT_MASK                                  0x1
+
+/* DMA0_CORE_RD_DBGMEM_ADD */
+#define DMA0_CORE_RD_DBGMEM_ADD_VAL_SHIFT                            0
+#define DMA0_CORE_RD_DBGMEM_ADD_VAL_MASK                             0xFFFFFFFF
+
+/* DMA0_CORE_RD_DBGMEM_DATA_WR */
+#define DMA0_CORE_RD_DBGMEM_DATA_WR_VAL_SHIFT                        0
+#define DMA0_CORE_RD_DBGMEM_DATA_WR_VAL_MASK                         0xFFFFFFFF
+
+/* DMA0_CORE_RD_DBGMEM_DATA_RD */
+#define DMA0_CORE_RD_DBGMEM_DATA_RD_VAL_SHIFT                        0
+#define DMA0_CORE_RD_DBGMEM_DATA_RD_VAL_MASK                         0xFFFFFFFF
+
+/* DMA0_CORE_RD_DBGMEM_CTRL */
+#define DMA0_CORE_RD_DBGMEM_CTRL_WR_NRD_SHIFT                        0
+#define DMA0_CORE_RD_DBGMEM_CTRL_WR_NRD_MASK                         0x1
+
+/* DMA0_CORE_RD_DBGMEM_RC */
+#define DMA0_CORE_RD_DBGMEM_RC_VALID_SHIFT                           0
+#define DMA0_CORE_RD_DBGMEM_RC_VALID_MASK                            0x1
+
+/* DMA0_CORE_DBG_HBW_AXI_AR_CNT */
+
+/* DMA0_CORE_DBG_HBW_AXI_AW_CNT */
+
+/* DMA0_CORE_DBG_LBW_AXI_AW_CNT */
+
+/* DMA0_CORE_DBG_DESC_CNT */
+#define DMA0_CORE_DBG_DESC_CNT_RD_STS_CTX_CNT_SHIFT                  0
+#define DMA0_CORE_DBG_DESC_CNT_RD_STS_CTX_CNT_MASK                   0xFFFFFFFF
+
+/* DMA0_CORE_DBG_STS */
+#define DMA0_CORE_DBG_STS_RD_CTX_FULL_SHIFT                          0
+#define DMA0_CORE_DBG_STS_RD_CTX_FULL_MASK                           0x1
+#define DMA0_CORE_DBG_STS_WR_CTX_FULL_SHIFT                          1
+#define DMA0_CORE_DBG_STS_WR_CTX_FULL_MASK                           0x2
+#define DMA0_CORE_DBG_STS_WR_COMP_FULL_SHIFT                         2
+#define DMA0_CORE_DBG_STS_WR_COMP_FULL_MASK                          0x4
+#define DMA0_CORE_DBG_STS_RD_CTX_EMPTY_SHIFT                         3
+#define DMA0_CORE_DBG_STS_RD_CTX_EMPTY_MASK                          0x8
+#define DMA0_CORE_DBG_STS_WR_CTX_EMPTY_SHIFT                         4
+#define DMA0_CORE_DBG_STS_WR_CTX_EMPTY_MASK                          0x10
+#define DMA0_CORE_DBG_STS_WR_COMP_EMPTY_SHIFT                        5
+#define DMA0_CORE_DBG_STS_WR_COMP_EMPTY_MASK                         0x20
+#define DMA0_CORE_DBG_STS_TE_EMPTY_SHIFT                             6
+#define DMA0_CORE_DBG_STS_TE_EMPTY_MASK                              0x40
+#define DMA0_CORE_DBG_STS_TE_BUSY_SHIFT                              7
+#define DMA0_CORE_DBG_STS_TE_BUSY_MASK                               0x80
+#define DMA0_CORE_DBG_STS_GSKT_EMPTY_SHIFT                           8
+#define DMA0_CORE_DBG_STS_GSKT_EMPTY_MASK                            0x100
+#define DMA0_CORE_DBG_STS_GSKT_FULL_SHIFT                            9
+#define DMA0_CORE_DBG_STS_GSKT_FULL_MASK                             0x200
+#define DMA0_CORE_DBG_STS_RDBUF_FULLNESS_SHIFT                       20
+#define DMA0_CORE_DBG_STS_RDBUF_FULLNESS_MASK                        0x7FF00000
+
+/* DMA0_CORE_DBG_RD_DESC_ID */
+
+/* DMA0_CORE_DBG_WR_DESC_ID */
+
+#endif /* ASIC_REG_DMA0_CORE_MASKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_regs.h
new file mode 100644
index 000000000000..1fdd5d5fc6d2
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA0_CORE_REGS_H_
+#define ASIC_REG_DMA0_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA0_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA0_CORE_CFG_0                                            0x500000
+
+#define mmDMA0_CORE_CFG_1                                            0x500004
+
+#define mmDMA0_CORE_LBW_MAX_OUTSTAND                                 0x500008
+
+#define mmDMA0_CORE_SRC_BASE_LO                                      0x500014
+
+#define mmDMA0_CORE_SRC_BASE_HI                                      0x500018
+
+#define mmDMA0_CORE_DST_BASE_LO                                      0x50001C
+
+#define mmDMA0_CORE_DST_BASE_HI                                      0x500020
+
+#define mmDMA0_CORE_SRC_TSIZE_1                                      0x50002C
+
+#define mmDMA0_CORE_SRC_STRIDE_1                                     0x500030
+
+#define mmDMA0_CORE_SRC_TSIZE_2                                      0x500034
+
+#define mmDMA0_CORE_SRC_STRIDE_2                                     0x500038
+
+#define mmDMA0_CORE_SRC_TSIZE_3                                      0x50003C
+
+#define mmDMA0_CORE_SRC_STRIDE_3                                     0x500040
+
+#define mmDMA0_CORE_SRC_TSIZE_4                                      0x500044
+
+#define mmDMA0_CORE_SRC_STRIDE_4                                     0x500048
+
+#define mmDMA0_CORE_SRC_TSIZE_0                                      0x50004C
+
+#define mmDMA0_CORE_DST_TSIZE_1                                      0x500054
+
+#define mmDMA0_CORE_DST_STRIDE_1                                     0x500058
+
+#define mmDMA0_CORE_DST_TSIZE_2                                      0x50005C
+
+#define mmDMA0_CORE_DST_STRIDE_2                                     0x500060
+
+#define mmDMA0_CORE_DST_TSIZE_3                                      0x500064
+
+#define mmDMA0_CORE_DST_STRIDE_3                                     0x500068
+
+#define mmDMA0_CORE_DST_TSIZE_4                                      0x50006C
+
+#define mmDMA0_CORE_DST_STRIDE_4                                     0x500070
+
+#define mmDMA0_CORE_DST_TSIZE_0                                      0x500074
+
+#define mmDMA0_CORE_COMMIT                                           0x500078
+
+#define mmDMA0_CORE_WR_COMP_WDATA                                    0x50007C
+
+#define mmDMA0_CORE_WR_COMP_ADDR_LO                                  0x500080
+
+#define mmDMA0_CORE_WR_COMP_ADDR_HI                                  0x500084
+
+#define mmDMA0_CORE_WR_COMP_AWUSER_31_11                             0x500088
+
+#define mmDMA0_CORE_TE_NUMROWS                                       0x500094
+
+#define mmDMA0_CORE_PROT                                             0x5000B8
+
+#define mmDMA0_CORE_SECURE_PROPS                                     0x5000F0
+
+#define mmDMA0_CORE_NON_SECURE_PROPS                                 0x5000F4
+
+#define mmDMA0_CORE_RD_MAX_OUTSTAND                                  0x500100
+
+#define mmDMA0_CORE_RD_MAX_SIZE                                      0x500104
+
+#define mmDMA0_CORE_RD_ARCACHE                                       0x500108
+
+#define mmDMA0_CORE_RD_ARUSER_31_11                                  0x500110
+
+#define mmDMA0_CORE_RD_INFLIGHTS                                     0x500114
+
+#define mmDMA0_CORE_WR_MAX_OUTSTAND                                  0x500120
+
+#define mmDMA0_CORE_WR_MAX_AWID                                      0x500124
+
+#define mmDMA0_CORE_WR_AWCACHE                                       0x500128
+
+#define mmDMA0_CORE_WR_AWUSER_31_11                                  0x500130
+
+#define mmDMA0_CORE_WR_INFLIGHTS                                     0x500134
+
+#define mmDMA0_CORE_RD_RATE_LIM_CFG_0                                0x500150
+
+#define mmDMA0_CORE_RD_RATE_LIM_CFG_1                                0x500154
+
+#define mmDMA0_CORE_WR_RATE_LIM_CFG_0                                0x500158
+
+#define mmDMA0_CORE_WR_RATE_LIM_CFG_1                                0x50015C
+
+#define mmDMA0_CORE_ERR_CFG                                          0x500160
+
+#define mmDMA0_CORE_ERR_CAUSE                                        0x500164
+
+#define mmDMA0_CORE_ERRMSG_ADDR_LO                                   0x500170
+
+#define mmDMA0_CORE_ERRMSG_ADDR_HI                                   0x500174
+
+#define mmDMA0_CORE_ERRMSG_WDATA                                     0x500178
+
+#define mmDMA0_CORE_STS0                                             0x500190
+
+#define mmDMA0_CORE_STS1                                             0x500194
+
+#define mmDMA0_CORE_RD_DBGMEM_ADD                                    0x500200
+
+#define mmDMA0_CORE_RD_DBGMEM_DATA_WR                                0x500204
+
+#define mmDMA0_CORE_RD_DBGMEM_DATA_RD                                0x500208
+
+#define mmDMA0_CORE_RD_DBGMEM_CTRL                                   0x50020C
+
+#define mmDMA0_CORE_RD_DBGMEM_RC                                     0x500210
+
+#define mmDMA0_CORE_DBG_HBW_AXI_AR_CNT                               0x500220
+
+#define mmDMA0_CORE_DBG_HBW_AXI_AW_CNT                               0x500224
+
+#define mmDMA0_CORE_DBG_LBW_AXI_AW_CNT                               0x500228
+
+#define mmDMA0_CORE_DBG_DESC_CNT                                     0x50022C
+
+#define mmDMA0_CORE_DBG_STS                                          0x500230
+
+#define mmDMA0_CORE_DBG_RD_DESC_ID                                   0x500234
+
+#define mmDMA0_CORE_DBG_WR_DESC_ID                                   0x500238
+
+#endif /* ASIC_REG_DMA0_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_masks.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_masks.h
new file mode 100644
index 000000000000..48376aabc3ba
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_masks.h
@@ -0,0 +1,800 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA0_QM_MASKS_H_
+#define ASIC_REG_DMA0_QM_MASKS_H_
+
+/*
+ *****************************************
+ *   DMA0_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+/* DMA0_QM_GLBL_CFG0 */
+#define DMA0_QM_GLBL_CFG0_PQF_EN_SHIFT                               0
+#define DMA0_QM_GLBL_CFG0_PQF_EN_MASK                                0xF
+#define DMA0_QM_GLBL_CFG0_CQF_EN_SHIFT                               4
+#define DMA0_QM_GLBL_CFG0_CQF_EN_MASK                                0x1F0
+#define DMA0_QM_GLBL_CFG0_CP_EN_SHIFT                                9
+#define DMA0_QM_GLBL_CFG0_CP_EN_MASK                                 0x3E00
+
+/* DMA0_QM_GLBL_CFG1 */
+#define DMA0_QM_GLBL_CFG1_PQF_STOP_SHIFT                             0
+#define DMA0_QM_GLBL_CFG1_PQF_STOP_MASK                              0xF
+#define DMA0_QM_GLBL_CFG1_CQF_STOP_SHIFT                             4
+#define DMA0_QM_GLBL_CFG1_CQF_STOP_MASK                              0x1F0
+#define DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT                              9
+#define DMA0_QM_GLBL_CFG1_CP_STOP_MASK                               0x3E00
+#define DMA0_QM_GLBL_CFG1_PQF_FLUSH_SHIFT                            16
+#define DMA0_QM_GLBL_CFG1_PQF_FLUSH_MASK                             0xF0000
+#define DMA0_QM_GLBL_CFG1_CQF_FLUSH_SHIFT                            20
+#define DMA0_QM_GLBL_CFG1_CQF_FLUSH_MASK                             0x1F00000
+#define DMA0_QM_GLBL_CFG1_CP_FLUSH_SHIFT                             25
+#define DMA0_QM_GLBL_CFG1_CP_FLUSH_MASK                              0x3E000000
+
+/* DMA0_QM_GLBL_PROT */
+#define DMA0_QM_GLBL_PROT_PQF_SHIFT                                  0
+#define DMA0_QM_GLBL_PROT_PQF_MASK                                   0xF
+#define DMA0_QM_GLBL_PROT_CQF_SHIFT                                  4
+#define DMA0_QM_GLBL_PROT_CQF_MASK                                   0x1F0
+#define DMA0_QM_GLBL_PROT_CP_SHIFT                                   9
+#define DMA0_QM_GLBL_PROT_CP_MASK                                    0x3E00
+#define DMA0_QM_GLBL_PROT_ERR_SHIFT                                  14
+#define DMA0_QM_GLBL_PROT_ERR_MASK                                   0x4000
+#define DMA0_QM_GLBL_PROT_ARB_SHIFT                                  15
+#define DMA0_QM_GLBL_PROT_ARB_MASK                                   0x8000
+
+/* DMA0_QM_GLBL_ERR_CFG */
+#define DMA0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_SHIFT                    0
+#define DMA0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK                     0xF
+#define DMA0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_SHIFT                    4
+#define DMA0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_MASK                     0x1F0
+#define DMA0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_SHIFT                     9
+#define DMA0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_MASK                      0x3E00
+#define DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT                   16
+#define DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK                    0xF0000
+#define DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT                   20
+#define DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK                    0x1F00000
+#define DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT                    25
+#define DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK                     0x3E000000
+#define DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_SHIFT                   31
+#define DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK                    0x80000000
+
+/* DMA0_QM_GLBL_SECURE_PROPS */
+#define DMA0_QM_GLBL_SECURE_PROPS_0_ASID_SHIFT                       0
+#define DMA0_QM_GLBL_SECURE_PROPS_0_ASID_MASK                        0x3FF
+#define DMA0_QM_GLBL_SECURE_PROPS_1_ASID_SHIFT                       0
+#define DMA0_QM_GLBL_SECURE_PROPS_1_ASID_MASK                        0x3FF
+#define DMA0_QM_GLBL_SECURE_PROPS_2_ASID_SHIFT                       0
+#define DMA0_QM_GLBL_SECURE_PROPS_2_ASID_MASK                        0x3FF
+#define DMA0_QM_GLBL_SECURE_PROPS_3_ASID_SHIFT                       0
+#define DMA0_QM_GLBL_SECURE_PROPS_3_ASID_MASK                        0x3FF
+#define DMA0_QM_GLBL_SECURE_PROPS_4_ASID_SHIFT                       0
+#define DMA0_QM_GLBL_SECURE_PROPS_4_ASID_MASK                        0x3FF
+#define DMA0_QM_GLBL_SECURE_PROPS_0_MMBP_SHIFT                       10
+#define DMA0_QM_GLBL_SECURE_PROPS_0_MMBP_MASK                        0x400
+#define DMA0_QM_GLBL_SECURE_PROPS_1_MMBP_SHIFT                       10
+#define DMA0_QM_GLBL_SECURE_PROPS_1_MMBP_MASK                        0x400
+#define DMA0_QM_GLBL_SECURE_PROPS_2_MMBP_SHIFT                       10
+#define DMA0_QM_GLBL_SECURE_PROPS_2_MMBP_MASK                        0x400
+#define DMA0_QM_GLBL_SECURE_PROPS_3_MMBP_SHIFT                       10
+#define DMA0_QM_GLBL_SECURE_PROPS_3_MMBP_MASK                        0x400
+#define DMA0_QM_GLBL_SECURE_PROPS_4_MMBP_SHIFT                       10
+#define DMA0_QM_GLBL_SECURE_PROPS_4_MMBP_MASK                        0x400
+
+/* DMA0_QM_GLBL_NON_SECURE_PROPS */
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_SHIFT                   0
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK                    0x3FF
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_1_ASID_SHIFT                   0
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_1_ASID_MASK                    0x3FF
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_2_ASID_SHIFT                   0
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_2_ASID_MASK                    0x3FF
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_3_ASID_SHIFT                   0
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_3_ASID_MASK                    0x3FF
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_4_ASID_SHIFT                   0
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_4_ASID_MASK                    0x3FF
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_0_MMBP_SHIFT                   10
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_0_MMBP_MASK                    0x400
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_1_MMBP_SHIFT                   10
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_1_MMBP_MASK                    0x400
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_2_MMBP_SHIFT                   10
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_2_MMBP_MASK                    0x400
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_3_MMBP_SHIFT                   10
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_3_MMBP_MASK                    0x400
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_4_MMBP_SHIFT                   10
+#define DMA0_QM_GLBL_NON_SECURE_PROPS_4_MMBP_MASK                    0x400
+
+/* DMA0_QM_GLBL_STS0 */
+#define DMA0_QM_GLBL_STS0_PQF_IDLE_SHIFT                             0
+#define DMA0_QM_GLBL_STS0_PQF_IDLE_MASK                              0xF
+#define DMA0_QM_GLBL_STS0_CQF_IDLE_SHIFT                             4
+#define DMA0_QM_GLBL_STS0_CQF_IDLE_MASK                              0x1F0
+#define DMA0_QM_GLBL_STS0_CP_IDLE_SHIFT                              9
+#define DMA0_QM_GLBL_STS0_CP_IDLE_MASK                               0x3E00
+#define DMA0_QM_GLBL_STS0_PQF_IS_STOP_SHIFT                          16
+#define DMA0_QM_GLBL_STS0_PQF_IS_STOP_MASK                           0xF0000
+#define DMA0_QM_GLBL_STS0_CQF_IS_STOP_SHIFT                          20
+#define DMA0_QM_GLBL_STS0_CQF_IS_STOP_MASK                           0x1F00000
+#define DMA0_QM_GLBL_STS0_CP_IS_STOP_SHIFT                           25
+#define DMA0_QM_GLBL_STS0_CP_IS_STOP_MASK                            0x3E000000
+#define DMA0_QM_GLBL_STS0_ARB_IS_STOP_SHIFT                          31
+#define DMA0_QM_GLBL_STS0_ARB_IS_STOP_MASK                           0x80000000
+
+/* DMA0_QM_GLBL_STS1 */
+#define DMA0_QM_GLBL_STS1_PQF_RD_ERR_SHIFT                           0
+#define DMA0_QM_GLBL_STS1_PQF_RD_ERR_MASK                            0x1
+#define DMA0_QM_GLBL_STS1_CQF_RD_ERR_SHIFT                           1
+#define DMA0_QM_GLBL_STS1_CQF_RD_ERR_MASK                            0x2
+#define DMA0_QM_GLBL_STS1_CP_RD_ERR_SHIFT                            2
+#define DMA0_QM_GLBL_STS1_CP_RD_ERR_MASK                             0x4
+#define DMA0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_SHIFT                     3
+#define DMA0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_MASK                      0x8
+#define DMA0_QM_GLBL_STS1_CP_STOP_OP_SHIFT                           4
+#define DMA0_QM_GLBL_STS1_CP_STOP_OP_MASK                            0x10
+#define DMA0_QM_GLBL_STS1_CP_MSG_WR_ERR_SHIFT                        5
+#define DMA0_QM_GLBL_STS1_CP_MSG_WR_ERR_MASK                         0x20
+#define DMA0_QM_GLBL_STS1_CP_WREG_ERR_SHIFT                          6
+#define DMA0_QM_GLBL_STS1_CP_WREG_ERR_MASK                           0x40
+#define DMA0_QM_GLBL_STS1_CP_FENCE0_OVF_ERR_SHIFT                    8
+#define DMA0_QM_GLBL_STS1_CP_FENCE0_OVF_ERR_MASK                     0x100
+#define DMA0_QM_GLBL_STS1_CP_FENCE1_OVF_ERR_SHIFT                    9
+#define DMA0_QM_GLBL_STS1_CP_FENCE1_OVF_ERR_MASK                     0x200
+#define DMA0_QM_GLBL_STS1_CP_FENCE2_OVF_ERR_SHIFT                    10
+#define DMA0_QM_GLBL_STS1_CP_FENCE2_OVF_ERR_MASK                     0x400
+#define DMA0_QM_GLBL_STS1_CP_FENCE3_OVF_ERR_SHIFT                    11
+#define DMA0_QM_GLBL_STS1_CP_FENCE3_OVF_ERR_MASK                     0x800
+#define DMA0_QM_GLBL_STS1_CP_FENCE0_UDF_ERR_SHIFT                    12
+#define DMA0_QM_GLBL_STS1_CP_FENCE0_UDF_ERR_MASK                     0x1000
+#define DMA0_QM_GLBL_STS1_CP_FENCE1_UDF_ERR_SHIFT                    13
+#define DMA0_QM_GLBL_STS1_CP_FENCE1_UDF_ERR_MASK                     0x2000
+#define DMA0_QM_GLBL_STS1_CP_FENCE2_UDF_ERR_SHIFT                    14
+#define DMA0_QM_GLBL_STS1_CP_FENCE2_UDF_ERR_MASK                     0x4000
+#define DMA0_QM_GLBL_STS1_CP_FENCE3_UDF_ERR_SHIFT                    15
+#define DMA0_QM_GLBL_STS1_CP_FENCE3_UDF_ERR_MASK                     0x8000
+
+/* DMA0_QM_GLBL_STS1_4 */
+#define DMA0_QM_GLBL_STS1_4_CQF_RD_ERR_SHIFT                         1
+#define DMA0_QM_GLBL_STS1_4_CQF_RD_ERR_MASK                          0x2
+#define DMA0_QM_GLBL_STS1_4_CP_RD_ERR_SHIFT                          2
+#define DMA0_QM_GLBL_STS1_4_CP_RD_ERR_MASK                           0x4
+#define DMA0_QM_GLBL_STS1_4_CP_UNDEF_CMD_ERR_SHIFT                   3
+#define DMA0_QM_GLBL_STS1_4_CP_UNDEF_CMD_ERR_MASK                    0x8
+#define DMA0_QM_GLBL_STS1_4_CP_STOP_OP_SHIFT                         4
+#define DMA0_QM_GLBL_STS1_4_CP_STOP_OP_MASK                          0x10
+#define DMA0_QM_GLBL_STS1_4_CP_MSG_WR_ERR_SHIFT                      5
+#define DMA0_QM_GLBL_STS1_4_CP_MSG_WR_ERR_MASK                       0x20
+#define DMA0_QM_GLBL_STS1_4_CP_WREG_ERR_SHIFT                        6
+#define DMA0_QM_GLBL_STS1_4_CP_WREG_ERR_MASK                         0x40
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE0_OVF_ERR_SHIFT                  8
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE0_OVF_ERR_MASK                   0x100
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE1_OVF_ERR_SHIFT                  9
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE1_OVF_ERR_MASK                   0x200
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE2_OVF_ERR_SHIFT                  10
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE2_OVF_ERR_MASK                   0x400
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE3_OVF_ERR_SHIFT                  11
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE3_OVF_ERR_MASK                   0x800
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE0_UDF_ERR_SHIFT                  12
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE0_UDF_ERR_MASK                   0x1000
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE1_UDF_ERR_SHIFT                  13
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE1_UDF_ERR_MASK                   0x2000
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE2_UDF_ERR_SHIFT                  14
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE2_UDF_ERR_MASK                   0x4000
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE3_UDF_ERR_SHIFT                  15
+#define DMA0_QM_GLBL_STS1_4_CP_FENCE3_UDF_ERR_MASK                   0x8000
+
+/* DMA0_QM_GLBL_MSG_EN */
+#define DMA0_QM_GLBL_MSG_EN_PQF_RD_ERR_SHIFT                         0
+#define DMA0_QM_GLBL_MSG_EN_PQF_RD_ERR_MASK                          0x1
+#define DMA0_QM_GLBL_MSG_EN_CQF_RD_ERR_SHIFT                         1
+#define DMA0_QM_GLBL_MSG_EN_CQF_RD_ERR_MASK                          0x2
+#define DMA0_QM_GLBL_MSG_EN_CP_RD_ERR_SHIFT                          2
+#define DMA0_QM_GLBL_MSG_EN_CP_RD_ERR_MASK                           0x4
+#define DMA0_QM_GLBL_MSG_EN_CP_UNDEF_CMD_ERR_SHIFT                   3
+#define DMA0_QM_GLBL_MSG_EN_CP_UNDEF_CMD_ERR_MASK                    0x8
+#define DMA0_QM_GLBL_MSG_EN_CP_STOP_OP_SHIFT                         4
+#define DMA0_QM_GLBL_MSG_EN_CP_STOP_OP_MASK                          0x10
+#define DMA0_QM_GLBL_MSG_EN_CP_MSG_WR_ERR_SHIFT                      5
+#define DMA0_QM_GLBL_MSG_EN_CP_MSG_WR_ERR_MASK                       0x20
+#define DMA0_QM_GLBL_MSG_EN_CP_WREG_ERR_SHIFT                        6
+#define DMA0_QM_GLBL_MSG_EN_CP_WREG_ERR_MASK                         0x40
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE0_OVF_ERR_SHIFT                  8
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE0_OVF_ERR_MASK                   0x100
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE1_OVF_ERR_SHIFT                  9
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE1_OVF_ERR_MASK                   0x200
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE2_OVF_ERR_SHIFT                  10
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE2_OVF_ERR_MASK                   0x400
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE3_OVF_ERR_SHIFT                  11
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE3_OVF_ERR_MASK                   0x800
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE0_UDF_ERR_SHIFT                  12
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE0_UDF_ERR_MASK                   0x1000
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE1_UDF_ERR_SHIFT                  13
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE1_UDF_ERR_MASK                   0x2000
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE2_UDF_ERR_SHIFT                  14
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE2_UDF_ERR_MASK                   0x4000
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE3_UDF_ERR_SHIFT                  15
+#define DMA0_QM_GLBL_MSG_EN_CP_FENCE3_UDF_ERR_MASK                   0x8000
+
+/* DMA0_QM_GLBL_MSG_EN_4 */
+#define DMA0_QM_GLBL_MSG_EN_4_CQF_RD_ERR_SHIFT                       1
+#define DMA0_QM_GLBL_MSG_EN_4_CQF_RD_ERR_MASK                        0x2
+#define DMA0_QM_GLBL_MSG_EN_4_CP_RD_ERR_SHIFT                        2
+#define DMA0_QM_GLBL_MSG_EN_4_CP_RD_ERR_MASK                         0x4
+#define DMA0_QM_GLBL_MSG_EN_4_CP_UNDEF_CMD_ERR_SHIFT                 3
+#define DMA0_QM_GLBL_MSG_EN_4_CP_UNDEF_CMD_ERR_MASK                  0x8
+#define DMA0_QM_GLBL_MSG_EN_4_CP_STOP_OP_SHIFT                       4
+#define DMA0_QM_GLBL_MSG_EN_4_CP_STOP_OP_MASK                        0x10
+#define DMA0_QM_GLBL_MSG_EN_4_CP_MSG_WR_ERR_SHIFT                    5
+#define DMA0_QM_GLBL_MSG_EN_4_CP_MSG_WR_ERR_MASK                     0x20
+#define DMA0_QM_GLBL_MSG_EN_4_CP_WREG_ERR_SHIFT                      6
+#define DMA0_QM_GLBL_MSG_EN_4_CP_WREG_ERR_MASK                       0x40
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE0_OVF_ERR_SHIFT                8
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE0_OVF_ERR_MASK                 0x100
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE1_OVF_ERR_SHIFT                9
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE1_OVF_ERR_MASK                 0x200
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE2_OVF_ERR_SHIFT                10
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE2_OVF_ERR_MASK                 0x400
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE3_OVF_ERR_SHIFT                11
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE3_OVF_ERR_MASK                 0x800
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE0_UDF_ERR_SHIFT                12
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE0_UDF_ERR_MASK                 0x1000
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE1_UDF_ERR_SHIFT                13
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE1_UDF_ERR_MASK                 0x2000
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE2_UDF_ERR_SHIFT                14
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE2_UDF_ERR_MASK                 0x4000
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE3_UDF_ERR_SHIFT                15
+#define DMA0_QM_GLBL_MSG_EN_4_CP_FENCE3_UDF_ERR_MASK                 0x8000
+
+/* DMA0_QM_PQ_BASE_LO */
+#define DMA0_QM_PQ_BASE_LO_VAL_SHIFT                                 0
+#define DMA0_QM_PQ_BASE_LO_VAL_MASK                                  0xFFFFFFFF
+
+/* DMA0_QM_PQ_BASE_HI */
+#define DMA0_QM_PQ_BASE_HI_VAL_SHIFT                                 0
+#define DMA0_QM_PQ_BASE_HI_VAL_MASK                                  0xFFFFFFFF
+
+/* DMA0_QM_PQ_SIZE */
+#define DMA0_QM_PQ_SIZE_VAL_SHIFT                                    0
+#define DMA0_QM_PQ_SIZE_VAL_MASK                                     0xFFFFFFFF
+
+/* DMA0_QM_PQ_PI */
+#define DMA0_QM_PQ_PI_VAL_SHIFT                                      0
+#define DMA0_QM_PQ_PI_VAL_MASK                                       0xFFFFFFFF
+
+/* DMA0_QM_PQ_CI */
+#define DMA0_QM_PQ_CI_VAL_SHIFT                                      0
+#define DMA0_QM_PQ_CI_VAL_MASK                                       0xFFFFFFFF
+
+/* DMA0_QM_PQ_CFG0 */
+#define DMA0_QM_PQ_CFG0_RESERVED_SHIFT                               0
+#define DMA0_QM_PQ_CFG0_RESERVED_MASK                                0x1
+
+/* DMA0_QM_PQ_CFG1 */
+#define DMA0_QM_PQ_CFG1_CREDIT_LIM_SHIFT                             0
+#define DMA0_QM_PQ_CFG1_CREDIT_LIM_MASK                              0xFFFF
+#define DMA0_QM_PQ_CFG1_MAX_INFLIGHT_SHIFT                           16
+#define DMA0_QM_PQ_CFG1_MAX_INFLIGHT_MASK                            0xFFFF0000
+
+/* DMA0_QM_PQ_ARUSER_31_11 */
+#define DMA0_QM_PQ_ARUSER_31_11_VAL_SHIFT                            0
+#define DMA0_QM_PQ_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* DMA0_QM_PQ_STS0 */
+#define DMA0_QM_PQ_STS0_PQ_CREDIT_CNT_SHIFT                          0
+#define DMA0_QM_PQ_STS0_PQ_CREDIT_CNT_MASK                           0xFFFF
+#define DMA0_QM_PQ_STS0_PQ_FREE_CNT_SHIFT                            16
+#define DMA0_QM_PQ_STS0_PQ_FREE_CNT_MASK                             0xFFFF0000
+
+/* DMA0_QM_PQ_STS1 */
+#define DMA0_QM_PQ_STS1_PQ_INFLIGHT_CNT_SHIFT                        0
+#define DMA0_QM_PQ_STS1_PQ_INFLIGHT_CNT_MASK                         0xFFFF
+#define DMA0_QM_PQ_STS1_PQ_BUF_EMPTY_SHIFT                           30
+#define DMA0_QM_PQ_STS1_PQ_BUF_EMPTY_MASK                            0x40000000
+#define DMA0_QM_PQ_STS1_PQ_BUSY_SHIFT                                31
+#define DMA0_QM_PQ_STS1_PQ_BUSY_MASK                                 0x80000000
+
+/* DMA0_QM_CQ_CFG0 */
+#define DMA0_QM_CQ_CFG0_RESERVED_SHIFT                               0
+#define DMA0_QM_CQ_CFG0_RESERVED_MASK                                0x1
+
+/* DMA0_QM_CQ_CFG1 */
+#define DMA0_QM_CQ_CFG1_CREDIT_LIM_SHIFT                             0
+#define DMA0_QM_CQ_CFG1_CREDIT_LIM_MASK                              0xFFFF
+#define DMA0_QM_CQ_CFG1_MAX_INFLIGHT_SHIFT                           16
+#define DMA0_QM_CQ_CFG1_MAX_INFLIGHT_MASK                            0xFFFF0000
+
+/* DMA0_QM_CQ_ARUSER_31_11 */
+#define DMA0_QM_CQ_ARUSER_31_11_VAL_SHIFT                            0
+#define DMA0_QM_CQ_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* DMA0_QM_CQ_STS0 */
+#define DMA0_QM_CQ_STS0_CQ_CREDIT_CNT_SHIFT                          0
+#define DMA0_QM_CQ_STS0_CQ_CREDIT_CNT_MASK                           0xFFFF
+#define DMA0_QM_CQ_STS0_CQ_FREE_CNT_SHIFT                            16
+#define DMA0_QM_CQ_STS0_CQ_FREE_CNT_MASK                             0xFFFF0000
+
+/* DMA0_QM_CQ_STS1 */
+#define DMA0_QM_CQ_STS1_CQ_INFLIGHT_CNT_SHIFT                        0
+#define DMA0_QM_CQ_STS1_CQ_INFLIGHT_CNT_MASK                         0xFFFF
+#define DMA0_QM_CQ_STS1_CQ_BUF_EMPTY_SHIFT                           30
+#define DMA0_QM_CQ_STS1_CQ_BUF_EMPTY_MASK                            0x40000000
+#define DMA0_QM_CQ_STS1_CQ_BUSY_SHIFT                                31
+#define DMA0_QM_CQ_STS1_CQ_BUSY_MASK                                 0x80000000
+
+/* DMA0_QM_CQ_PTR_LO_0 */
+#define DMA0_QM_CQ_PTR_LO_0_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_LO_0_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_PTR_HI_0 */
+#define DMA0_QM_CQ_PTR_HI_0_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_HI_0_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_TSIZE_0 */
+#define DMA0_QM_CQ_TSIZE_0_VAL_SHIFT                                 0
+#define DMA0_QM_CQ_TSIZE_0_VAL_MASK                                  0xFFFFFFFF
+
+/* DMA0_QM_CQ_CTL_0 */
+#define DMA0_QM_CQ_CTL_0_RPT_SHIFT                                   0
+#define DMA0_QM_CQ_CTL_0_RPT_MASK                                    0xFFFF
+#define DMA0_QM_CQ_CTL_0_CTL_SHIFT                                   16
+#define DMA0_QM_CQ_CTL_0_CTL_MASK                                    0xFFFF0000
+
+/* DMA0_QM_CQ_PTR_LO_1 */
+#define DMA0_QM_CQ_PTR_LO_1_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_LO_1_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_PTR_HI_1 */
+#define DMA0_QM_CQ_PTR_HI_1_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_HI_1_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_TSIZE_1 */
+#define DMA0_QM_CQ_TSIZE_1_VAL_SHIFT                                 0
+#define DMA0_QM_CQ_TSIZE_1_VAL_MASK                                  0xFFFFFFFF
+
+/* DMA0_QM_CQ_CTL_1 */
+#define DMA0_QM_CQ_CTL_1_RPT_SHIFT                                   0
+#define DMA0_QM_CQ_CTL_1_RPT_MASK                                    0xFFFF
+#define DMA0_QM_CQ_CTL_1_CTL_SHIFT                                   16
+#define DMA0_QM_CQ_CTL_1_CTL_MASK                                    0xFFFF0000
+
+/* DMA0_QM_CQ_PTR_LO_2 */
+#define DMA0_QM_CQ_PTR_LO_2_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_LO_2_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_PTR_HI_2 */
+#define DMA0_QM_CQ_PTR_HI_2_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_HI_2_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_TSIZE_2 */
+#define DMA0_QM_CQ_TSIZE_2_VAL_SHIFT                                 0
+#define DMA0_QM_CQ_TSIZE_2_VAL_MASK                                  0xFFFFFFFF
+
+/* DMA0_QM_CQ_CTL_2 */
+#define DMA0_QM_CQ_CTL_2_RPT_SHIFT                                   0
+#define DMA0_QM_CQ_CTL_2_RPT_MASK                                    0xFFFF
+#define DMA0_QM_CQ_CTL_2_CTL_SHIFT                                   16
+#define DMA0_QM_CQ_CTL_2_CTL_MASK                                    0xFFFF0000
+
+/* DMA0_QM_CQ_PTR_LO_3 */
+#define DMA0_QM_CQ_PTR_LO_3_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_LO_3_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_PTR_HI_3 */
+#define DMA0_QM_CQ_PTR_HI_3_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_HI_3_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_TSIZE_3 */
+#define DMA0_QM_CQ_TSIZE_3_VAL_SHIFT                                 0
+#define DMA0_QM_CQ_TSIZE_3_VAL_MASK                                  0xFFFFFFFF
+
+/* DMA0_QM_CQ_CTL_3 */
+#define DMA0_QM_CQ_CTL_3_RPT_SHIFT                                   0
+#define DMA0_QM_CQ_CTL_3_RPT_MASK                                    0xFFFF
+#define DMA0_QM_CQ_CTL_3_CTL_SHIFT                                   16
+#define DMA0_QM_CQ_CTL_3_CTL_MASK                                    0xFFFF0000
+
+/* DMA0_QM_CQ_PTR_LO_4 */
+#define DMA0_QM_CQ_PTR_LO_4_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_LO_4_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_PTR_HI_4 */
+#define DMA0_QM_CQ_PTR_HI_4_VAL_SHIFT                                0
+#define DMA0_QM_CQ_PTR_HI_4_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_CQ_TSIZE_4 */
+#define DMA0_QM_CQ_TSIZE_4_VAL_SHIFT                                 0
+#define DMA0_QM_CQ_TSIZE_4_VAL_MASK                                  0xFFFFFFFF
+
+/* DMA0_QM_CQ_CTL_4 */
+#define DMA0_QM_CQ_CTL_4_RPT_SHIFT                                   0
+#define DMA0_QM_CQ_CTL_4_RPT_MASK                                    0xFFFF
+#define DMA0_QM_CQ_CTL_4_CTL_SHIFT                                   16
+#define DMA0_QM_CQ_CTL_4_CTL_MASK                                    0xFFFF0000
+
+/* DMA0_QM_CQ_PTR_LO_STS */
+#define DMA0_QM_CQ_PTR_LO_STS_VAL_SHIFT                              0
+#define DMA0_QM_CQ_PTR_LO_STS_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_QM_CQ_PTR_HI_STS */
+#define DMA0_QM_CQ_PTR_HI_STS_VAL_SHIFT                              0
+#define DMA0_QM_CQ_PTR_HI_STS_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_QM_CQ_TSIZE_STS */
+#define DMA0_QM_CQ_TSIZE_STS_VAL_SHIFT                               0
+#define DMA0_QM_CQ_TSIZE_STS_VAL_MASK                                0xFFFFFFFF
+
+/* DMA0_QM_CQ_CTL_STS */
+#define DMA0_QM_CQ_CTL_STS_RPT_SHIFT                                 0
+#define DMA0_QM_CQ_CTL_STS_RPT_MASK                                  0xFFFF
+#define DMA0_QM_CQ_CTL_STS_CTL_SHIFT                                 16
+#define DMA0_QM_CQ_CTL_STS_CTL_MASK                                  0xFFFF0000
+
+/* DMA0_QM_CQ_IFIFO_CNT */
+#define DMA0_QM_CQ_IFIFO_CNT_VAL_SHIFT                               0
+#define DMA0_QM_CQ_IFIFO_CNT_VAL_MASK                                0x3
+
+/* DMA0_QM_CP_MSG_BASE0_ADDR_LO */
+#define DMA0_QM_CP_MSG_BASE0_ADDR_LO_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE0_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_MSG_BASE0_ADDR_HI */
+#define DMA0_QM_CP_MSG_BASE0_ADDR_HI_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE0_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_MSG_BASE1_ADDR_LO */
+#define DMA0_QM_CP_MSG_BASE1_ADDR_LO_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE1_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_MSG_BASE1_ADDR_HI */
+#define DMA0_QM_CP_MSG_BASE1_ADDR_HI_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE1_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_MSG_BASE2_ADDR_LO */
+#define DMA0_QM_CP_MSG_BASE2_ADDR_LO_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE2_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_MSG_BASE2_ADDR_HI */
+#define DMA0_QM_CP_MSG_BASE2_ADDR_HI_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE2_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_MSG_BASE3_ADDR_LO */
+#define DMA0_QM_CP_MSG_BASE3_ADDR_LO_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE3_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_MSG_BASE3_ADDR_HI */
+#define DMA0_QM_CP_MSG_BASE3_ADDR_HI_VAL_SHIFT                       0
+#define DMA0_QM_CP_MSG_BASE3_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_LDMA_TSIZE_OFFSET */
+#define DMA0_QM_CP_LDMA_TSIZE_OFFSET_VAL_SHIFT                       0
+#define DMA0_QM_CP_LDMA_TSIZE_OFFSET_VAL_MASK                        0xFFFFFFFF
+
+/* DMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET */
+#define DMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_VAL_SHIFT                 0
+#define DMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_VAL_MASK                  0xFFFFFFFF
+
+/* DMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET */
+#define DMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_VAL_SHIFT                 0
+#define DMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_VAL_MASK                  0xFFFFFFFF
+
+/* DMA0_QM_CP_FENCE0_RDATA */
+#define DMA0_QM_CP_FENCE0_RDATA_INC_VAL_SHIFT                        0
+#define DMA0_QM_CP_FENCE0_RDATA_INC_VAL_MASK                         0xF
+
+/* DMA0_QM_CP_FENCE1_RDATA */
+#define DMA0_QM_CP_FENCE1_RDATA_INC_VAL_SHIFT                        0
+#define DMA0_QM_CP_FENCE1_RDATA_INC_VAL_MASK                         0xF
+
+/* DMA0_QM_CP_FENCE2_RDATA */
+#define DMA0_QM_CP_FENCE2_RDATA_INC_VAL_SHIFT                        0
+#define DMA0_QM_CP_FENCE2_RDATA_INC_VAL_MASK                         0xF
+
+/* DMA0_QM_CP_FENCE3_RDATA */
+#define DMA0_QM_CP_FENCE3_RDATA_INC_VAL_SHIFT                        0
+#define DMA0_QM_CP_FENCE3_RDATA_INC_VAL_MASK                         0xF
+
+/* DMA0_QM_CP_FENCE0_CNT */
+#define DMA0_QM_CP_FENCE0_CNT_VAL_SHIFT                              0
+#define DMA0_QM_CP_FENCE0_CNT_VAL_MASK                               0x3FFF
+
+/* DMA0_QM_CP_FENCE1_CNT */
+#define DMA0_QM_CP_FENCE1_CNT_VAL_SHIFT                              0
+#define DMA0_QM_CP_FENCE1_CNT_VAL_MASK                               0x3FFF
+
+/* DMA0_QM_CP_FENCE2_CNT */
+#define DMA0_QM_CP_FENCE2_CNT_VAL_SHIFT                              0
+#define DMA0_QM_CP_FENCE2_CNT_VAL_MASK                               0x3FFF
+
+/* DMA0_QM_CP_FENCE3_CNT */
+#define DMA0_QM_CP_FENCE3_CNT_VAL_SHIFT                              0
+#define DMA0_QM_CP_FENCE3_CNT_VAL_MASK                               0x3FFF
+
+/* DMA0_QM_CP_STS */
+#define DMA0_QM_CP_STS_MSG_INFLIGHT_CNT_SHIFT                        0
+#define DMA0_QM_CP_STS_MSG_INFLIGHT_CNT_MASK                         0xFFFF
+#define DMA0_QM_CP_STS_ERDY_SHIFT                                    16
+#define DMA0_QM_CP_STS_ERDY_MASK                                     0x10000
+#define DMA0_QM_CP_STS_RRDY_SHIFT                                    17
+#define DMA0_QM_CP_STS_RRDY_MASK                                     0x20000
+#define DMA0_QM_CP_STS_MRDY_SHIFT                                    18
+#define DMA0_QM_CP_STS_MRDY_MASK                                     0x40000
+#define DMA0_QM_CP_STS_SW_STOP_SHIFT                                 19
+#define DMA0_QM_CP_STS_SW_STOP_MASK                                  0x80000
+#define DMA0_QM_CP_STS_FENCE_ID_SHIFT                                20
+#define DMA0_QM_CP_STS_FENCE_ID_MASK                                 0x300000
+#define DMA0_QM_CP_STS_FENCE_IN_PROGRESS_SHIFT                       22
+#define DMA0_QM_CP_STS_FENCE_IN_PROGRESS_MASK                        0x400000
+
+/* DMA0_QM_CP_CURRENT_INST_LO */
+#define DMA0_QM_CP_CURRENT_INST_LO_VAL_SHIFT                         0
+#define DMA0_QM_CP_CURRENT_INST_LO_VAL_MASK                          0xFFFFFFFF
+
+/* DMA0_QM_CP_CURRENT_INST_HI */
+#define DMA0_QM_CP_CURRENT_INST_HI_VAL_SHIFT                         0
+#define DMA0_QM_CP_CURRENT_INST_HI_VAL_MASK                          0xFFFFFFFF
+
+/* DMA0_QM_CP_BARRIER_CFG */
+#define DMA0_QM_CP_BARRIER_CFG_EBGUARD_SHIFT                         0
+#define DMA0_QM_CP_BARRIER_CFG_EBGUARD_MASK                          0xFFF
+#define DMA0_QM_CP_BARRIER_CFG_RBGUARD_SHIFT                         16
+#define DMA0_QM_CP_BARRIER_CFG_RBGUARD_MASK                          0xF0000
+
+/* DMA0_QM_CP_DBG_0 */
+#define DMA0_QM_CP_DBG_0_CS_SHIFT                                    0
+#define DMA0_QM_CP_DBG_0_CS_MASK                                     0xF
+#define DMA0_QM_CP_DBG_0_EB_CNT_NOT_ZERO_SHIFT                       4
+#define DMA0_QM_CP_DBG_0_EB_CNT_NOT_ZERO_MASK                        0x10
+#define DMA0_QM_CP_DBG_0_BULK_CNT_NOT_ZERO_SHIFT                     5
+#define DMA0_QM_CP_DBG_0_BULK_CNT_NOT_ZERO_MASK                      0x20
+#define DMA0_QM_CP_DBG_0_MREB_STALL_SHIFT                            6
+#define DMA0_QM_CP_DBG_0_MREB_STALL_MASK                             0x40
+#define DMA0_QM_CP_DBG_0_STALL_SHIFT                                 7
+#define DMA0_QM_CP_DBG_0_STALL_MASK                                  0x80
+
+/* DMA0_QM_CP_ARUSER_31_11 */
+#define DMA0_QM_CP_ARUSER_31_11_VAL_SHIFT                            0
+#define DMA0_QM_CP_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* DMA0_QM_CP_AWUSER_31_11 */
+#define DMA0_QM_CP_AWUSER_31_11_VAL_SHIFT                            0
+#define DMA0_QM_CP_AWUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* DMA0_QM_ARB_CFG_0 */
+#define DMA0_QM_ARB_CFG_0_TYPE_SHIFT                                 0
+#define DMA0_QM_ARB_CFG_0_TYPE_MASK                                  0x1
+#define DMA0_QM_ARB_CFG_0_IS_MASTER_SHIFT                            4
+#define DMA0_QM_ARB_CFG_0_IS_MASTER_MASK                             0x10
+#define DMA0_QM_ARB_CFG_0_EN_SHIFT                                   8
+#define DMA0_QM_ARB_CFG_0_EN_MASK                                    0x100
+#define DMA0_QM_ARB_CFG_0_MASK_SHIFT                                 12
+#define DMA0_QM_ARB_CFG_0_MASK_MASK                                  0xF000
+#define DMA0_QM_ARB_CFG_0_MST_MSG_NOSTALL_SHIFT                      16
+#define DMA0_QM_ARB_CFG_0_MST_MSG_NOSTALL_MASK                       0x10000
+
+/* DMA0_QM_ARB_CHOISE_Q_PUSH */
+#define DMA0_QM_ARB_CHOISE_Q_PUSH_VAL_SHIFT                          0
+#define DMA0_QM_ARB_CHOISE_Q_PUSH_VAL_MASK                           0x3
+
+/* DMA0_QM_ARB_WRR_WEIGHT */
+#define DMA0_QM_ARB_WRR_WEIGHT_VAL_SHIFT                             0
+#define DMA0_QM_ARB_WRR_WEIGHT_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_QM_ARB_CFG_1 */
+#define DMA0_QM_ARB_CFG_1_CLR_SHIFT                                  0
+#define DMA0_QM_ARB_CFG_1_CLR_MASK                                   0x1
+
+/* DMA0_QM_ARB_MST_AVAIL_CRED */
+#define DMA0_QM_ARB_MST_AVAIL_CRED_VAL_SHIFT                         0
+#define DMA0_QM_ARB_MST_AVAIL_CRED_VAL_MASK                          0x7F
+
+/* DMA0_QM_ARB_MST_CRED_INC */
+#define DMA0_QM_ARB_MST_CRED_INC_VAL_SHIFT                           0
+#define DMA0_QM_ARB_MST_CRED_INC_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_QM_ARB_MST_CHOISE_PUSH_OFST */
+#define DMA0_QM_ARB_MST_CHOISE_PUSH_OFST_VAL_SHIFT                   0
+#define DMA0_QM_ARB_MST_CHOISE_PUSH_OFST_VAL_MASK                    0xFFFFFFFF
+
+/* DMA0_QM_ARB_SLV_MASTER_INC_CRED_OFST */
+#define DMA0_QM_ARB_SLV_MASTER_INC_CRED_OFST_VAL_SHIFT               0
+#define DMA0_QM_ARB_SLV_MASTER_INC_CRED_OFST_VAL_MASK                0xFFFFFFFF
+
+/* DMA0_QM_ARB_MST_SLAVE_EN */
+#define DMA0_QM_ARB_MST_SLAVE_EN_VAL_SHIFT                           0
+#define DMA0_QM_ARB_MST_SLAVE_EN_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_QM_ARB_MST_QUIET_PER */
+#define DMA0_QM_ARB_MST_QUIET_PER_VAL_SHIFT                          0
+#define DMA0_QM_ARB_MST_QUIET_PER_VAL_MASK                           0xFFFFFFFF
+
+/* DMA0_QM_ARB_SLV_CHOISE_WDT */
+#define DMA0_QM_ARB_SLV_CHOISE_WDT_VAL_SHIFT                         0
+#define DMA0_QM_ARB_SLV_CHOISE_WDT_VAL_MASK                          0xFFFFFFFF
+
+/* DMA0_QM_ARB_SLV_ID */
+#define DMA0_QM_ARB_SLV_ID_VAL_SHIFT                                 0
+#define DMA0_QM_ARB_SLV_ID_VAL_MASK                                  0x1F
+
+/* DMA0_QM_ARB_MSG_MAX_INFLIGHT */
+#define DMA0_QM_ARB_MSG_MAX_INFLIGHT_VAL_SHIFT                       0
+#define DMA0_QM_ARB_MSG_MAX_INFLIGHT_VAL_MASK                        0x3F
+
+/* DMA0_QM_ARB_MSG_AWUSER_31_11 */
+#define DMA0_QM_ARB_MSG_AWUSER_31_11_VAL_SHIFT                       0
+#define DMA0_QM_ARB_MSG_AWUSER_31_11_VAL_MASK                        0x1FFFFF
+
+/* DMA0_QM_ARB_MSG_AWUSER_SEC_PROP */
+#define DMA0_QM_ARB_MSG_AWUSER_SEC_PROP_ASID_SHIFT                   0
+#define DMA0_QM_ARB_MSG_AWUSER_SEC_PROP_ASID_MASK                    0x3FF
+#define DMA0_QM_ARB_MSG_AWUSER_SEC_PROP_MMBP_SHIFT                   10
+#define DMA0_QM_ARB_MSG_AWUSER_SEC_PROP_MMBP_MASK                    0x400
+
+/* DMA0_QM_ARB_MSG_AWUSER_NON_SEC_PROP */
+#define DMA0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_ASID_SHIFT               0
+#define DMA0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_ASID_MASK                0x3FF
+#define DMA0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_MMBP_SHIFT               10
+#define DMA0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_MMBP_MASK                0x400
+
+/* DMA0_QM_ARB_BASE_LO */
+#define DMA0_QM_ARB_BASE_LO_VAL_SHIFT                                0
+#define DMA0_QM_ARB_BASE_LO_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_ARB_BASE_HI */
+#define DMA0_QM_ARB_BASE_HI_VAL_SHIFT                                0
+#define DMA0_QM_ARB_BASE_HI_VAL_MASK                                 0xFFFFFFFF
+
+/* DMA0_QM_ARB_STATE_STS */
+#define DMA0_QM_ARB_STATE_STS_VAL_SHIFT                              0
+#define DMA0_QM_ARB_STATE_STS_VAL_MASK                               0xFFFFFFFF
+
+/* DMA0_QM_ARB_CHOISE_FULLNESS_STS */
+#define DMA0_QM_ARB_CHOISE_FULLNESS_STS_VAL_SHIFT                    0
+#define DMA0_QM_ARB_CHOISE_FULLNESS_STS_VAL_MASK                     0x7F
+
+/* DMA0_QM_ARB_MSG_STS */
+#define DMA0_QM_ARB_MSG_STS_FULL_SHIFT                               0
+#define DMA0_QM_ARB_MSG_STS_FULL_MASK                                0x1
+#define DMA0_QM_ARB_MSG_STS_NO_INFLIGHT_SHIFT                        1
+#define DMA0_QM_ARB_MSG_STS_NO_INFLIGHT_MASK                         0x2
+
+/* DMA0_QM_ARB_SLV_CHOISE_Q_HEAD */
+#define DMA0_QM_ARB_SLV_CHOISE_Q_HEAD_VAL_SHIFT                      0
+#define DMA0_QM_ARB_SLV_CHOISE_Q_HEAD_VAL_MASK                       0x3
+
+/* DMA0_QM_ARB_ERR_CAUSE */
+#define DMA0_QM_ARB_ERR_CAUSE_CHOISE_OVF_SHIFT                       0
+#define DMA0_QM_ARB_ERR_CAUSE_CHOISE_OVF_MASK                        0x1
+#define DMA0_QM_ARB_ERR_CAUSE_CHOISE_WDT_SHIFT                       1
+#define DMA0_QM_ARB_ERR_CAUSE_CHOISE_WDT_MASK                        0x2
+#define DMA0_QM_ARB_ERR_CAUSE_AXI_LBW_ERR_SHIFT                      2
+#define DMA0_QM_ARB_ERR_CAUSE_AXI_LBW_ERR_MASK                       0x4
+
+/* DMA0_QM_ARB_ERR_MSG_EN */
+#define DMA0_QM_ARB_ERR_MSG_EN_CHOISE_OVF_SHIFT                      0
+#define DMA0_QM_ARB_ERR_MSG_EN_CHOISE_OVF_MASK                       0x1
+#define DMA0_QM_ARB_ERR_MSG_EN_CHOISE_WDT_SHIFT                      1
+#define DMA0_QM_ARB_ERR_MSG_EN_CHOISE_WDT_MASK                       0x2
+#define DMA0_QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_SHIFT                     2
+#define DMA0_QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_MASK                      0x4
+
+/* DMA0_QM_ARB_ERR_STS_DRP */
+#define DMA0_QM_ARB_ERR_STS_DRP_VAL_SHIFT                            0
+#define DMA0_QM_ARB_ERR_STS_DRP_VAL_MASK                             0x3
+
+/* DMA0_QM_ARB_MST_CRED_STS */
+#define DMA0_QM_ARB_MST_CRED_STS_VAL_SHIFT                           0
+#define DMA0_QM_ARB_MST_CRED_STS_VAL_MASK                            0x7F
+
+/* DMA0_QM_CGM_CFG */
+#define DMA0_QM_CGM_CFG_IDLE_TH_SHIFT                                0
+#define DMA0_QM_CGM_CFG_IDLE_TH_MASK                                 0xFFF
+#define DMA0_QM_CGM_CFG_G2F_TH_SHIFT                                 16
+#define DMA0_QM_CGM_CFG_G2F_TH_MASK                                  0xFF0000
+#define DMA0_QM_CGM_CFG_CP_IDLE_MASK_SHIFT                           24
+#define DMA0_QM_CGM_CFG_CP_IDLE_MASK_MASK                            0x1F000000
+#define DMA0_QM_CGM_CFG_EN_SHIFT                                     31
+#define DMA0_QM_CGM_CFG_EN_MASK                                      0x80000000
+
+/* DMA0_QM_CGM_STS */
+#define DMA0_QM_CGM_STS_ST_SHIFT                                     0
+#define DMA0_QM_CGM_STS_ST_MASK                                      0x3
+#define DMA0_QM_CGM_STS_CG_SHIFT                                     4
+#define DMA0_QM_CGM_STS_CG_MASK                                      0x10
+#define DMA0_QM_CGM_STS_AGENT_IDLE_SHIFT                             8
+#define DMA0_QM_CGM_STS_AGENT_IDLE_MASK                              0x100
+#define DMA0_QM_CGM_STS_AXI_IDLE_SHIFT                               9
+#define DMA0_QM_CGM_STS_AXI_IDLE_MASK                                0x200
+#define DMA0_QM_CGM_STS_CP_IDLE_SHIFT                                10
+#define DMA0_QM_CGM_STS_CP_IDLE_MASK                                 0x400
+
+/* DMA0_QM_CGM_CFG1 */
+#define DMA0_QM_CGM_CFG1_MASK_TH_SHIFT                               0
+#define DMA0_QM_CGM_CFG1_MASK_TH_MASK                                0xFF
+
+/* DMA0_QM_LOCAL_RANGE_BASE */
+#define DMA0_QM_LOCAL_RANGE_BASE_VAL_SHIFT                           0
+#define DMA0_QM_LOCAL_RANGE_BASE_VAL_MASK                            0xFFFF
+
+/* DMA0_QM_LOCAL_RANGE_SIZE */
+#define DMA0_QM_LOCAL_RANGE_SIZE_VAL_SHIFT                           0
+#define DMA0_QM_LOCAL_RANGE_SIZE_VAL_MASK                            0xFFFF
+
+/* DMA0_QM_CSMR_STRICT_PRIO_CFG */
+#define DMA0_QM_CSMR_STRICT_PRIO_CFG_TYPE_SHIFT                      0
+#define DMA0_QM_CSMR_STRICT_PRIO_CFG_TYPE_MASK                       0x1
+
+/* DMA0_QM_HBW_RD_RATE_LIM_CFG_1 */
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_1_TOUT_SHIFT                     0
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_1_TOUT_MASK                      0xFF
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_1_EN_SHIFT                       31
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_1_EN_MASK                        0x80000000
+
+/* DMA0_QM_LBW_WR_RATE_LIM_CFG_0 */
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                0
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_0_RST_TOKEN_MASK                 0xFF
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_0_SAT_SHIFT                      16
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_0_SAT_MASK                       0xFF0000
+
+/* DMA0_QM_LBW_WR_RATE_LIM_CFG_1 */
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_1_TOUT_SHIFT                     0
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_1_TOUT_MASK                      0xFF
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_1_EN_SHIFT                       31
+#define DMA0_QM_LBW_WR_RATE_LIM_CFG_1_EN_MASK                        0x80000000
+
+/* DMA0_QM_HBW_RD_RATE_LIM_CFG_0 */
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                0
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_0_RST_TOKEN_MASK                 0xFF
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_0_SAT_SHIFT                      16
+#define DMA0_QM_HBW_RD_RATE_LIM_CFG_0_SAT_MASK                       0xFF0000
+
+/* DMA0_QM_GLBL_AXCACHE */
+#define DMA0_QM_GLBL_AXCACHE_AR_SHIFT                                0
+#define DMA0_QM_GLBL_AXCACHE_AR_MASK                                 0xF
+#define DMA0_QM_GLBL_AXCACHE_AW_SHIFT                                16
+#define DMA0_QM_GLBL_AXCACHE_AW_MASK                                 0xF0000
+
+/* DMA0_QM_IND_GW_APB_CFG */
+#define DMA0_QM_IND_GW_APB_CFG_ADDR_SHIFT                            0
+#define DMA0_QM_IND_GW_APB_CFG_ADDR_MASK                             0x7FFFFFFF
+#define DMA0_QM_IND_GW_APB_CFG_CMD_SHIFT                             31
+#define DMA0_QM_IND_GW_APB_CFG_CMD_MASK                              0x80000000
+
+/* DMA0_QM_IND_GW_APB_WDATA */
+#define DMA0_QM_IND_GW_APB_WDATA_VAL_SHIFT                           0
+#define DMA0_QM_IND_GW_APB_WDATA_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_QM_IND_GW_APB_RDATA */
+#define DMA0_QM_IND_GW_APB_RDATA_VAL_SHIFT                           0
+#define DMA0_QM_IND_GW_APB_RDATA_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_QM_IND_GW_APB_STATUS */
+#define DMA0_QM_IND_GW_APB_STATUS_RDY_SHIFT                          0
+#define DMA0_QM_IND_GW_APB_STATUS_RDY_MASK                           0x1
+#define DMA0_QM_IND_GW_APB_STATUS_ERR_SHIFT                          1
+#define DMA0_QM_IND_GW_APB_STATUS_ERR_MASK                           0x2
+
+/* DMA0_QM_GLBL_ERR_ADDR_LO */
+#define DMA0_QM_GLBL_ERR_ADDR_LO_VAL_SHIFT                           0
+#define DMA0_QM_GLBL_ERR_ADDR_LO_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_QM_GLBL_ERR_ADDR_HI */
+#define DMA0_QM_GLBL_ERR_ADDR_HI_VAL_SHIFT                           0
+#define DMA0_QM_GLBL_ERR_ADDR_HI_VAL_MASK                            0xFFFFFFFF
+
+/* DMA0_QM_GLBL_ERR_WDATA */
+#define DMA0_QM_GLBL_ERR_WDATA_VAL_SHIFT                             0
+#define DMA0_QM_GLBL_ERR_WDATA_VAL_MASK                              0xFFFFFFFF
+
+/* DMA0_QM_GLBL_MEM_INIT_BUSY */
+#define DMA0_QM_GLBL_MEM_INIT_BUSY_RBUF_SHIFT                        0
+#define DMA0_QM_GLBL_MEM_INIT_BUSY_RBUF_MASK                         0xF
+
+#endif /* ASIC_REG_DMA0_QM_MASKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_regs.h
new file mode 100644
index 000000000000..8e56a93d88a1
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma0_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA0_QM_REGS_H_
+#define ASIC_REG_DMA0_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA0_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA0_QM_GLBL_CFG0                                          0x508000
+
+#define mmDMA0_QM_GLBL_CFG1                                          0x508004
+
+#define mmDMA0_QM_GLBL_PROT                                          0x508008
+
+#define mmDMA0_QM_GLBL_ERR_CFG                                       0x50800C
+
+#define mmDMA0_QM_GLBL_SECURE_PROPS_0                                0x508010
+
+#define mmDMA0_QM_GLBL_SECURE_PROPS_1                                0x508014
+
+#define mmDMA0_QM_GLBL_SECURE_PROPS_2                                0x508018
+
+#define mmDMA0_QM_GLBL_SECURE_PROPS_3                                0x50801C
+
+#define mmDMA0_QM_GLBL_SECURE_PROPS_4                                0x508020
+
+#define mmDMA0_QM_GLBL_NON_SECURE_PROPS_0                            0x508024
+
+#define mmDMA0_QM_GLBL_NON_SECURE_PROPS_1                            0x508028
+
+#define mmDMA0_QM_GLBL_NON_SECURE_PROPS_2                            0x50802C
+
+#define mmDMA0_QM_GLBL_NON_SECURE_PROPS_3                            0x508030
+
+#define mmDMA0_QM_GLBL_NON_SECURE_PROPS_4                            0x508034
+
+#define mmDMA0_QM_GLBL_STS0                                          0x508038
+
+#define mmDMA0_QM_GLBL_STS1_0                                        0x508040
+
+#define mmDMA0_QM_GLBL_STS1_1                                        0x508044
+
+#define mmDMA0_QM_GLBL_STS1_2                                        0x508048
+
+#define mmDMA0_QM_GLBL_STS1_3                                        0x50804C
+
+#define mmDMA0_QM_GLBL_STS1_4                                        0x508050
+
+#define mmDMA0_QM_GLBL_MSG_EN_0                                      0x508054
+
+#define mmDMA0_QM_GLBL_MSG_EN_1                                      0x508058
+
+#define mmDMA0_QM_GLBL_MSG_EN_2                                      0x50805C
+
+#define mmDMA0_QM_GLBL_MSG_EN_3                                      0x508060
+
+#define mmDMA0_QM_GLBL_MSG_EN_4                                      0x508068
+
+#define mmDMA0_QM_PQ_BASE_LO_0                                       0x508070
+
+#define mmDMA0_QM_PQ_BASE_LO_1                                       0x508074
+
+#define mmDMA0_QM_PQ_BASE_LO_2                                       0x508078
+
+#define mmDMA0_QM_PQ_BASE_LO_3                                       0x50807C
+
+#define mmDMA0_QM_PQ_BASE_HI_0                                       0x508080
+
+#define mmDMA0_QM_PQ_BASE_HI_1                                       0x508084
+
+#define mmDMA0_QM_PQ_BASE_HI_2                                       0x508088
+
+#define mmDMA0_QM_PQ_BASE_HI_3                                       0x50808C
+
+#define mmDMA0_QM_PQ_SIZE_0                                          0x508090
+
+#define mmDMA0_QM_PQ_SIZE_1                                          0x508094
+
+#define mmDMA0_QM_PQ_SIZE_2                                          0x508098
+
+#define mmDMA0_QM_PQ_SIZE_3                                          0x50809C
+
+#define mmDMA0_QM_PQ_PI_0                                            0x5080A0
+
+#define mmDMA0_QM_PQ_PI_1                                            0x5080A4
+
+#define mmDMA0_QM_PQ_PI_2                                            0x5080A8
+
+#define mmDMA0_QM_PQ_PI_3                                            0x5080AC
+
+#define mmDMA0_QM_PQ_CI_0                                            0x5080B0
+
+#define mmDMA0_QM_PQ_CI_1                                            0x5080B4
+
+#define mmDMA0_QM_PQ_CI_2                                            0x5080B8
+
+#define mmDMA0_QM_PQ_CI_3                                            0x5080BC
+
+#define mmDMA0_QM_PQ_CFG0_0                                          0x5080C0
+
+#define mmDMA0_QM_PQ_CFG0_1                                          0x5080C4
+
+#define mmDMA0_QM_PQ_CFG0_2                                          0x5080C8
+
+#define mmDMA0_QM_PQ_CFG0_3                                          0x5080CC
+
+#define mmDMA0_QM_PQ_CFG1_0                                          0x5080D0
+
+#define mmDMA0_QM_PQ_CFG1_1                                          0x5080D4
+
+#define mmDMA0_QM_PQ_CFG1_2                                          0x5080D8
+
+#define mmDMA0_QM_PQ_CFG1_3                                          0x5080DC
+
+#define mmDMA0_QM_PQ_ARUSER_31_11_0                                  0x5080E0
+
+#define mmDMA0_QM_PQ_ARUSER_31_11_1                                  0x5080E4
+
+#define mmDMA0_QM_PQ_ARUSER_31_11_2                                  0x5080E8
+
+#define mmDMA0_QM_PQ_ARUSER_31_11_3                                  0x5080EC
+
+#define mmDMA0_QM_PQ_STS0_0                                          0x5080F0
+
+#define mmDMA0_QM_PQ_STS0_1                                          0x5080F4
+
+#define mmDMA0_QM_PQ_STS0_2                                          0x5080F8
+
+#define mmDMA0_QM_PQ_STS0_3                                          0x5080FC
+
+#define mmDMA0_QM_PQ_STS1_0                                          0x508100
+
+#define mmDMA0_QM_PQ_STS1_1                                          0x508104
+
+#define mmDMA0_QM_PQ_STS1_2                                          0x508108
+
+#define mmDMA0_QM_PQ_STS1_3                                          0x50810C
+
+#define mmDMA0_QM_CQ_CFG0_0                                          0x508110
+
+#define mmDMA0_QM_CQ_CFG0_1                                          0x508114
+
+#define mmDMA0_QM_CQ_CFG0_2                                          0x508118
+
+#define mmDMA0_QM_CQ_CFG0_3                                          0x50811C
+
+#define mmDMA0_QM_CQ_CFG0_4                                          0x508120
+
+#define mmDMA0_QM_CQ_CFG1_0                                          0x508124
+
+#define mmDMA0_QM_CQ_CFG1_1                                          0x508128
+
+#define mmDMA0_QM_CQ_CFG1_2                                          0x50812C
+
+#define mmDMA0_QM_CQ_CFG1_3                                          0x508130
+
+#define mmDMA0_QM_CQ_CFG1_4                                          0x508134
+
+#define mmDMA0_QM_CQ_ARUSER_31_11_0                                  0x508138
+
+#define mmDMA0_QM_CQ_ARUSER_31_11_1                                  0x50813C
+
+#define mmDMA0_QM_CQ_ARUSER_31_11_2                                  0x508140
+
+#define mmDMA0_QM_CQ_ARUSER_31_11_3                                  0x508144
+
+#define mmDMA0_QM_CQ_ARUSER_31_11_4                                  0x508148
+
+#define mmDMA0_QM_CQ_STS0_0                                          0x50814C
+
+#define mmDMA0_QM_CQ_STS0_1                                          0x508150
+
+#define mmDMA0_QM_CQ_STS0_2                                          0x508154
+
+#define mmDMA0_QM_CQ_STS0_3                                          0x508158
+
+#define mmDMA0_QM_CQ_STS0_4                                          0x50815C
+
+#define mmDMA0_QM_CQ_STS1_0                                          0x508160
+
+#define mmDMA0_QM_CQ_STS1_1                                          0x508164
+
+#define mmDMA0_QM_CQ_STS1_2                                          0x508168
+
+#define mmDMA0_QM_CQ_STS1_3                                          0x50816C
+
+#define mmDMA0_QM_CQ_STS1_4                                          0x508170
+
+#define mmDMA0_QM_CQ_PTR_LO_0                                        0x508174
+
+#define mmDMA0_QM_CQ_PTR_HI_0                                        0x508178
+
+#define mmDMA0_QM_CQ_TSIZE_0                                         0x50817C
+
+#define mmDMA0_QM_CQ_CTL_0                                           0x508180
+
+#define mmDMA0_QM_CQ_PTR_LO_1                                        0x508184
+
+#define mmDMA0_QM_CQ_PTR_HI_1                                        0x508188
+
+#define mmDMA0_QM_CQ_TSIZE_1                                         0x50818C
+
+#define mmDMA0_QM_CQ_CTL_1                                           0x508190
+
+#define mmDMA0_QM_CQ_PTR_LO_2                                        0x508194
+
+#define mmDMA0_QM_CQ_PTR_HI_2                                        0x508198
+
+#define mmDMA0_QM_CQ_TSIZE_2                                         0x50819C
+
+#define mmDMA0_QM_CQ_CTL_2                                           0x5081A0
+
+#define mmDMA0_QM_CQ_PTR_LO_3                                        0x5081A4
+
+#define mmDMA0_QM_CQ_PTR_HI_3                                        0x5081A8
+
+#define mmDMA0_QM_CQ_TSIZE_3                                         0x5081AC
+
+#define mmDMA0_QM_CQ_CTL_3                                           0x5081B0
+
+#define mmDMA0_QM_CQ_PTR_LO_4                                        0x5081B4
+
+#define mmDMA0_QM_CQ_PTR_HI_4                                        0x5081B8
+
+#define mmDMA0_QM_CQ_TSIZE_4                                         0x5081BC
+
+#define mmDMA0_QM_CQ_CTL_4                                           0x5081C0
+
+#define mmDMA0_QM_CQ_PTR_LO_STS_0                                    0x5081C4
+
+#define mmDMA0_QM_CQ_PTR_LO_STS_1                                    0x5081C8
+
+#define mmDMA0_QM_CQ_PTR_LO_STS_2                                    0x5081CC
+
+#define mmDMA0_QM_CQ_PTR_LO_STS_3                                    0x5081D0
+
+#define mmDMA0_QM_CQ_PTR_LO_STS_4                                    0x5081D4
+
+#define mmDMA0_QM_CQ_PTR_HI_STS_0                                    0x5081D8
+
+#define mmDMA0_QM_CQ_PTR_HI_STS_1                                    0x5081DC
+
+#define mmDMA0_QM_CQ_PTR_HI_STS_2                                    0x5081E0
+
+#define mmDMA0_QM_CQ_PTR_HI_STS_3                                    0x5081E4
+
+#define mmDMA0_QM_CQ_PTR_HI_STS_4                                    0x5081E8
+
+#define mmDMA0_QM_CQ_TSIZE_STS_0                                     0x5081EC
+
+#define mmDMA0_QM_CQ_TSIZE_STS_1                                     0x5081F0
+
+#define mmDMA0_QM_CQ_TSIZE_STS_2                                     0x5081F4
+
+#define mmDMA0_QM_CQ_TSIZE_STS_3                                     0x5081F8
+
+#define mmDMA0_QM_CQ_TSIZE_STS_4                                     0x5081FC
+
+#define mmDMA0_QM_CQ_CTL_STS_0                                       0x508200
+
+#define mmDMA0_QM_CQ_CTL_STS_1                                       0x508204
+
+#define mmDMA0_QM_CQ_CTL_STS_2                                       0x508208
+
+#define mmDMA0_QM_CQ_CTL_STS_3                                       0x50820C
+
+#define mmDMA0_QM_CQ_CTL_STS_4                                       0x508210
+
+#define mmDMA0_QM_CQ_IFIFO_CNT_0                                     0x508214
+
+#define mmDMA0_QM_CQ_IFIFO_CNT_1                                     0x508218
+
+#define mmDMA0_QM_CQ_IFIFO_CNT_2                                     0x50821C
+
+#define mmDMA0_QM_CQ_IFIFO_CNT_3                                     0x508220
+
+#define mmDMA0_QM_CQ_IFIFO_CNT_4                                     0x508224
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0                             0x508228
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_1                             0x50822C
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_2                             0x508230
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_3                             0x508234
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_4                             0x508238
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0                             0x50823C
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_1                             0x508240
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_2                             0x508244
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_3                             0x508248
+
+#define mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_4                             0x50824C
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0                             0x508250
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_1                             0x508254
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_2                             0x508258
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_3                             0x50825C
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_4                             0x508260
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0                             0x508264
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_1                             0x508268
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_2                             0x50826C
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_3                             0x508270
+
+#define mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_4                             0x508274
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0                             0x508278
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_1                             0x50827C
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_2                             0x508280
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_3                             0x508284
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_4                             0x508288
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0                             0x50828C
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_1                             0x508290
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_2                             0x508294
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_3                             0x508298
+
+#define mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_4                             0x50829C
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5082A0
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5082A4
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5082A8
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5082AC
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5082B0
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5082B4
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5082B8
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5082BC
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5082C0
+
+#define mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5082C4
+
+#define mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5082C8
+
+#define mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5082CC
+
+#define mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5082D0
+
+#define mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5082D4
+
+#define mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5082D8
+
+#define mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5082E0
+
+#define mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5082E4
+
+#define mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5082E8
+
+#define mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5082EC
+
+#define mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5082F0
+
+#define mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5082F4
+
+#define mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5082F8
+
+#define mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5082FC
+
+#define mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x508300
+
+#define mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x508304
+
+#define mmDMA0_QM_CP_FENCE0_RDATA_0                                  0x508308
+
+#define mmDMA0_QM_CP_FENCE0_RDATA_1                                  0x50830C
+
+#define mmDMA0_QM_CP_FENCE0_RDATA_2                                  0x508310
+
+#define mmDMA0_QM_CP_FENCE0_RDATA_3                                  0x508314
+
+#define mmDMA0_QM_CP_FENCE0_RDATA_4                                  0x508318
+
+#define mmDMA0_QM_CP_FENCE1_RDATA_0                                  0x50831C
+
+#define mmDMA0_QM_CP_FENCE1_RDATA_1                                  0x508320
+
+#define mmDMA0_QM_CP_FENCE1_RDATA_2                                  0x508324
+
+#define mmDMA0_QM_CP_FENCE1_RDATA_3                                  0x508328
+
+#define mmDMA0_QM_CP_FENCE1_RDATA_4                                  0x50832C
+
+#define mmDMA0_QM_CP_FENCE2_RDATA_0                                  0x508330
+
+#define mmDMA0_QM_CP_FENCE2_RDATA_1                                  0x508334
+
+#define mmDMA0_QM_CP_FENCE2_RDATA_2                                  0x508338
+
+#define mmDMA0_QM_CP_FENCE2_RDATA_3                                  0x50833C
+
+#define mmDMA0_QM_CP_FENCE2_RDATA_4                                  0x508340
+
+#define mmDMA0_QM_CP_FENCE3_RDATA_0                                  0x508344
+
+#define mmDMA0_QM_CP_FENCE3_RDATA_1                                  0x508348
+
+#define mmDMA0_QM_CP_FENCE3_RDATA_2                                  0x50834C
+
+#define mmDMA0_QM_CP_FENCE3_RDATA_3                                  0x508350
+
+#define mmDMA0_QM_CP_FENCE3_RDATA_4                                  0x508354
+
+#define mmDMA0_QM_CP_FENCE0_CNT_0                                    0x508358
+
+#define mmDMA0_QM_CP_FENCE0_CNT_1                                    0x50835C
+
+#define mmDMA0_QM_CP_FENCE0_CNT_2                                    0x508360
+
+#define mmDMA0_QM_CP_FENCE0_CNT_3                                    0x508364
+
+#define mmDMA0_QM_CP_FENCE0_CNT_4                                    0x508368
+
+#define mmDMA0_QM_CP_FENCE1_CNT_0                                    0x50836C
+
+#define mmDMA0_QM_CP_FENCE1_CNT_1                                    0x508370
+
+#define mmDMA0_QM_CP_FENCE1_CNT_2                                    0x508374
+
+#define mmDMA0_QM_CP_FENCE1_CNT_3                                    0x508378
+
+#define mmDMA0_QM_CP_FENCE1_CNT_4                                    0x50837C
+
+#define mmDMA0_QM_CP_FENCE2_CNT_0                                    0x508380
+
+#define mmDMA0_QM_CP_FENCE2_CNT_1                                    0x508384
+
+#define mmDMA0_QM_CP_FENCE2_CNT_2                                    0x508388
+
+#define mmDMA0_QM_CP_FENCE2_CNT_3                                    0x50838C
+
+#define mmDMA0_QM_CP_FENCE2_CNT_4                                    0x508390
+
+#define mmDMA0_QM_CP_FENCE3_CNT_0                                    0x508394
+
+#define mmDMA0_QM_CP_FENCE3_CNT_1                                    0x508398
+
+#define mmDMA0_QM_CP_FENCE3_CNT_2                                    0x50839C
+
+#define mmDMA0_QM_CP_FENCE3_CNT_3                                    0x5083A0
+
+#define mmDMA0_QM_CP_FENCE3_CNT_4                                    0x5083A4
+
+#define mmDMA0_QM_CP_STS_0                                           0x5083A8
+
+#define mmDMA0_QM_CP_STS_1                                           0x5083AC
+
+#define mmDMA0_QM_CP_STS_2                                           0x5083B0
+
+#define mmDMA0_QM_CP_STS_3                                           0x5083B4
+
+#define mmDMA0_QM_CP_STS_4                                           0x5083B8
+
+#define mmDMA0_QM_CP_CURRENT_INST_LO_0                               0x5083BC
+
+#define mmDMA0_QM_CP_CURRENT_INST_LO_1                               0x5083C0
+
+#define mmDMA0_QM_CP_CURRENT_INST_LO_2                               0x5083C4
+
+#define mmDMA0_QM_CP_CURRENT_INST_LO_3                               0x5083C8
+
+#define mmDMA0_QM_CP_CURRENT_INST_LO_4                               0x5083CC
+
+#define mmDMA0_QM_CP_CURRENT_INST_HI_0                               0x5083D0
+
+#define mmDMA0_QM_CP_CURRENT_INST_HI_1                               0x5083D4
+
+#define mmDMA0_QM_CP_CURRENT_INST_HI_2                               0x5083D8
+
+#define mmDMA0_QM_CP_CURRENT_INST_HI_3                               0x5083DC
+
+#define mmDMA0_QM_CP_CURRENT_INST_HI_4                               0x5083E0
+
+#define mmDMA0_QM_CP_BARRIER_CFG_0                                   0x5083F4
+
+#define mmDMA0_QM_CP_BARRIER_CFG_1                                   0x5083F8
+
+#define mmDMA0_QM_CP_BARRIER_CFG_2                                   0x5083FC
+
+#define mmDMA0_QM_CP_BARRIER_CFG_3                                   0x508400
+
+#define mmDMA0_QM_CP_BARRIER_CFG_4                                   0x508404
+
+#define mmDMA0_QM_CP_DBG_0_0                                         0x508408
+
+#define mmDMA0_QM_CP_DBG_0_1                                         0x50840C
+
+#define mmDMA0_QM_CP_DBG_0_2                                         0x508410
+
+#define mmDMA0_QM_CP_DBG_0_3                                         0x508414
+
+#define mmDMA0_QM_CP_DBG_0_4                                         0x508418
+
+#define mmDMA0_QM_CP_ARUSER_31_11_0                                  0x50841C
+
+#define mmDMA0_QM_CP_ARUSER_31_11_1                                  0x508420
+
+#define mmDMA0_QM_CP_ARUSER_31_11_2                                  0x508424
+
+#define mmDMA0_QM_CP_ARUSER_31_11_3                                  0x508428
+
+#define mmDMA0_QM_CP_ARUSER_31_11_4                                  0x50842C
+
+#define mmDMA0_QM_CP_AWUSER_31_11_0                                  0x508430
+
+#define mmDMA0_QM_CP_AWUSER_31_11_1                                  0x508434
+
+#define mmDMA0_QM_CP_AWUSER_31_11_2                                  0x508438
+
+#define mmDMA0_QM_CP_AWUSER_31_11_3                                  0x50843C
+
+#define mmDMA0_QM_CP_AWUSER_31_11_4                                  0x508440
+
+#define mmDMA0_QM_ARB_CFG_0                                          0x508A00
+
+#define mmDMA0_QM_ARB_CHOISE_Q_PUSH                                  0x508A04
+
+#define mmDMA0_QM_ARB_WRR_WEIGHT_0                                   0x508A08
+
+#define mmDMA0_QM_ARB_WRR_WEIGHT_1                                   0x508A0C
+
+#define mmDMA0_QM_ARB_WRR_WEIGHT_2                                   0x508A10
+
+#define mmDMA0_QM_ARB_WRR_WEIGHT_3                                   0x508A14
+
+#define mmDMA0_QM_ARB_CFG_1                                          0x508A18
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_0                               0x508A20
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_1                               0x508A24
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_2                               0x508A28
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_3                               0x508A2C
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_4                               0x508A30
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_5                               0x508A34
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_6                               0x508A38
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_7                               0x508A3C
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_8                               0x508A40
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_9                               0x508A44
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_10                              0x508A48
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_11                              0x508A4C
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_12                              0x508A50
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_13                              0x508A54
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_14                              0x508A58
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_15                              0x508A5C
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_16                              0x508A60
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_17                              0x508A64
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_18                              0x508A68
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_19                              0x508A6C
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_20                              0x508A70
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_21                              0x508A74
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_22                              0x508A78
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_23                              0x508A7C
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_24                              0x508A80
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_25                              0x508A84
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_26                              0x508A88
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_27                              0x508A8C
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_28                              0x508A90
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_29                              0x508A94
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_30                              0x508A98
+
+#define mmDMA0_QM_ARB_MST_AVAIL_CRED_31                              0x508A9C
+
+#define mmDMA0_QM_ARB_MST_CRED_INC                                   0x508AA0
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x508AA4
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x508AA8
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x508AAC
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x508AB0
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x508AB4
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x508AB8
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x508ABC
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x508AC0
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x508AC4
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x508AC8
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x508ACC
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x508AD0
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x508AD4
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x508AD8
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x508ADC
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x508AE0
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x508AE4
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x508AE8
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x508AEC
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x508AF0
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x508AF4
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x508AF8
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x508AFC
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x508B00
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x508B04
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x508B08
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x508B0C
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x508B10
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x508B14
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x508B18
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x508B1C
+
+#define mmDMA0_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x508B20
+
+#define mmDMA0_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x508B28
+
+#define mmDMA0_QM_ARB_MST_SLAVE_EN                                   0x508B2C
+
+#define mmDMA0_QM_ARB_MST_QUIET_PER                                  0x508B34
+
+#define mmDMA0_QM_ARB_SLV_CHOISE_WDT                                 0x508B38
+
+#define mmDMA0_QM_ARB_SLV_ID                                         0x508B3C
+
+#define mmDMA0_QM_ARB_MSG_MAX_INFLIGHT                               0x508B44
+
+#define mmDMA0_QM_ARB_MSG_AWUSER_31_11                               0x508B48
+
+#define mmDMA0_QM_ARB_MSG_AWUSER_SEC_PROP                            0x508B4C
+
+#define mmDMA0_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x508B50
+
+#define mmDMA0_QM_ARB_BASE_LO                                        0x508B54
+
+#define mmDMA0_QM_ARB_BASE_HI                                        0x508B58
+
+#define mmDMA0_QM_ARB_STATE_STS                                      0x508B80
+
+#define mmDMA0_QM_ARB_CHOISE_FULLNESS_STS                            0x508B84
+
+#define mmDMA0_QM_ARB_MSG_STS                                        0x508B88
+
+#define mmDMA0_QM_ARB_SLV_CHOISE_Q_HEAD                              0x508B8C
+
+#define mmDMA0_QM_ARB_ERR_CAUSE                                      0x508B9C
+
+#define mmDMA0_QM_ARB_ERR_MSG_EN                                     0x508BA0
+
+#define mmDMA0_QM_ARB_ERR_STS_DRP                                    0x508BA8
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_0                                 0x508BB0
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_1                                 0x508BB4
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_2                                 0x508BB8
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_3                                 0x508BBC
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_4                                 0x508BC0
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_5                                 0x508BC4
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_6                                 0x508BC8
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_7                                 0x508BCC
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_8                                 0x508BD0
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_9                                 0x508BD4
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_10                                0x508BD8
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_11                                0x508BDC
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_12                                0x508BE0
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_13                                0x508BE4
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_14                                0x508BE8
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_15                                0x508BEC
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_16                                0x508BF0
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_17                                0x508BF4
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_18                                0x508BF8
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_19                                0x508BFC
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_20                                0x508C00
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_21                                0x508C04
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_22                                0x508C08
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_23                                0x508C0C
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_24                                0x508C10
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_25                                0x508C14
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_26                                0x508C18
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_27                                0x508C1C
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_28                                0x508C20
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_29                                0x508C24
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_30                                0x508C28
+
+#define mmDMA0_QM_ARB_MST_CRED_STS_31                                0x508C2C
+
+#define mmDMA0_QM_CGM_CFG                                            0x508C70
+
+#define mmDMA0_QM_CGM_STS                                            0x508C74
+
+#define mmDMA0_QM_CGM_CFG1                                           0x508C78
+
+#define mmDMA0_QM_LOCAL_RANGE_BASE                                   0x508C80
+
+#define mmDMA0_QM_LOCAL_RANGE_SIZE                                   0x508C84
+
+#define mmDMA0_QM_CSMR_STRICT_PRIO_CFG                               0x508C90
+
+#define mmDMA0_QM_HBW_RD_RATE_LIM_CFG_1                              0x508C94
+
+#define mmDMA0_QM_LBW_WR_RATE_LIM_CFG_0                              0x508C98
+
+#define mmDMA0_QM_LBW_WR_RATE_LIM_CFG_1                              0x508C9C
+
+#define mmDMA0_QM_HBW_RD_RATE_LIM_CFG_0                              0x508CA0
+
+#define mmDMA0_QM_GLBL_AXCACHE                                       0x508CA4
+
+#define mmDMA0_QM_IND_GW_APB_CFG                                     0x508CB0
+
+#define mmDMA0_QM_IND_GW_APB_WDATA                                   0x508CB4
+
+#define mmDMA0_QM_IND_GW_APB_RDATA                                   0x508CB8
+
+#define mmDMA0_QM_IND_GW_APB_STATUS                                  0x508CBC
+
+#define mmDMA0_QM_GLBL_ERR_ADDR_LO                                   0x508CD0
+
+#define mmDMA0_QM_GLBL_ERR_ADDR_HI                                   0x508CD4
+
+#define mmDMA0_QM_GLBL_ERR_WDATA                                     0x508CD8
+
+#define mmDMA0_QM_GLBL_MEM_INIT_BUSY                                 0x508D00
+
+#endif /* ASIC_REG_DMA0_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_core_regs.h
new file mode 100644
index 000000000000..4d8d8f26c5d4
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA1_CORE_REGS_H_
+#define ASIC_REG_DMA1_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA1_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA1_CORE_CFG_0                                            0x520000
+
+#define mmDMA1_CORE_CFG_1                                            0x520004
+
+#define mmDMA1_CORE_LBW_MAX_OUTSTAND                                 0x520008
+
+#define mmDMA1_CORE_SRC_BASE_LO                                      0x520014
+
+#define mmDMA1_CORE_SRC_BASE_HI                                      0x520018
+
+#define mmDMA1_CORE_DST_BASE_LO                                      0x52001C
+
+#define mmDMA1_CORE_DST_BASE_HI                                      0x520020
+
+#define mmDMA1_CORE_SRC_TSIZE_1                                      0x52002C
+
+#define mmDMA1_CORE_SRC_STRIDE_1                                     0x520030
+
+#define mmDMA1_CORE_SRC_TSIZE_2                                      0x520034
+
+#define mmDMA1_CORE_SRC_STRIDE_2                                     0x520038
+
+#define mmDMA1_CORE_SRC_TSIZE_3                                      0x52003C
+
+#define mmDMA1_CORE_SRC_STRIDE_3                                     0x520040
+
+#define mmDMA1_CORE_SRC_TSIZE_4                                      0x520044
+
+#define mmDMA1_CORE_SRC_STRIDE_4                                     0x520048
+
+#define mmDMA1_CORE_SRC_TSIZE_0                                      0x52004C
+
+#define mmDMA1_CORE_DST_TSIZE_1                                      0x520054
+
+#define mmDMA1_CORE_DST_STRIDE_1                                     0x520058
+
+#define mmDMA1_CORE_DST_TSIZE_2                                      0x52005C
+
+#define mmDMA1_CORE_DST_STRIDE_2                                     0x520060
+
+#define mmDMA1_CORE_DST_TSIZE_3                                      0x520064
+
+#define mmDMA1_CORE_DST_STRIDE_3                                     0x520068
+
+#define mmDMA1_CORE_DST_TSIZE_4                                      0x52006C
+
+#define mmDMA1_CORE_DST_STRIDE_4                                     0x520070
+
+#define mmDMA1_CORE_DST_TSIZE_0                                      0x520074
+
+#define mmDMA1_CORE_COMMIT                                           0x520078
+
+#define mmDMA1_CORE_WR_COMP_WDATA                                    0x52007C
+
+#define mmDMA1_CORE_WR_COMP_ADDR_LO                                  0x520080
+
+#define mmDMA1_CORE_WR_COMP_ADDR_HI                                  0x520084
+
+#define mmDMA1_CORE_WR_COMP_AWUSER_31_11                             0x520088
+
+#define mmDMA1_CORE_TE_NUMROWS                                       0x520094
+
+#define mmDMA1_CORE_PROT                                             0x5200B8
+
+#define mmDMA1_CORE_SECURE_PROPS                                     0x5200F0
+
+#define mmDMA1_CORE_NON_SECURE_PROPS                                 0x5200F4
+
+#define mmDMA1_CORE_RD_MAX_OUTSTAND                                  0x520100
+
+#define mmDMA1_CORE_RD_MAX_SIZE                                      0x520104
+
+#define mmDMA1_CORE_RD_ARCACHE                                       0x520108
+
+#define mmDMA1_CORE_RD_ARUSER_31_11                                  0x520110
+
+#define mmDMA1_CORE_RD_INFLIGHTS                                     0x520114
+
+#define mmDMA1_CORE_WR_MAX_OUTSTAND                                  0x520120
+
+#define mmDMA1_CORE_WR_MAX_AWID                                      0x520124
+
+#define mmDMA1_CORE_WR_AWCACHE                                       0x520128
+
+#define mmDMA1_CORE_WR_AWUSER_31_11                                  0x520130
+
+#define mmDMA1_CORE_WR_INFLIGHTS                                     0x520134
+
+#define mmDMA1_CORE_RD_RATE_LIM_CFG_0                                0x520150
+
+#define mmDMA1_CORE_RD_RATE_LIM_CFG_1                                0x520154
+
+#define mmDMA1_CORE_WR_RATE_LIM_CFG_0                                0x520158
+
+#define mmDMA1_CORE_WR_RATE_LIM_CFG_1                                0x52015C
+
+#define mmDMA1_CORE_ERR_CFG                                          0x520160
+
+#define mmDMA1_CORE_ERR_CAUSE                                        0x520164
+
+#define mmDMA1_CORE_ERRMSG_ADDR_LO                                   0x520170
+
+#define mmDMA1_CORE_ERRMSG_ADDR_HI                                   0x520174
+
+#define mmDMA1_CORE_ERRMSG_WDATA                                     0x520178
+
+#define mmDMA1_CORE_STS0                                             0x520190
+
+#define mmDMA1_CORE_STS1                                             0x520194
+
+#define mmDMA1_CORE_RD_DBGMEM_ADD                                    0x520200
+
+#define mmDMA1_CORE_RD_DBGMEM_DATA_WR                                0x520204
+
+#define mmDMA1_CORE_RD_DBGMEM_DATA_RD                                0x520208
+
+#define mmDMA1_CORE_RD_DBGMEM_CTRL                                   0x52020C
+
+#define mmDMA1_CORE_RD_DBGMEM_RC                                     0x520210
+
+#define mmDMA1_CORE_DBG_HBW_AXI_AR_CNT                               0x520220
+
+#define mmDMA1_CORE_DBG_HBW_AXI_AW_CNT                               0x520224
+
+#define mmDMA1_CORE_DBG_LBW_AXI_AW_CNT                               0x520228
+
+#define mmDMA1_CORE_DBG_DESC_CNT                                     0x52022C
+
+#define mmDMA1_CORE_DBG_STS                                          0x520230
+
+#define mmDMA1_CORE_DBG_RD_DESC_ID                                   0x520234
+
+#define mmDMA1_CORE_DBG_WR_DESC_ID                                   0x520238
+
+#endif /* ASIC_REG_DMA1_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_qm_regs.h
new file mode 100644
index 000000000000..c3ef300849be
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma1_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA1_QM_REGS_H_
+#define ASIC_REG_DMA1_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA1_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA1_QM_GLBL_CFG0                                          0x528000
+
+#define mmDMA1_QM_GLBL_CFG1                                          0x528004
+
+#define mmDMA1_QM_GLBL_PROT                                          0x528008
+
+#define mmDMA1_QM_GLBL_ERR_CFG                                       0x52800C
+
+#define mmDMA1_QM_GLBL_SECURE_PROPS_0                                0x528010
+
+#define mmDMA1_QM_GLBL_SECURE_PROPS_1                                0x528014
+
+#define mmDMA1_QM_GLBL_SECURE_PROPS_2                                0x528018
+
+#define mmDMA1_QM_GLBL_SECURE_PROPS_3                                0x52801C
+
+#define mmDMA1_QM_GLBL_SECURE_PROPS_4                                0x528020
+
+#define mmDMA1_QM_GLBL_NON_SECURE_PROPS_0                            0x528024
+
+#define mmDMA1_QM_GLBL_NON_SECURE_PROPS_1                            0x528028
+
+#define mmDMA1_QM_GLBL_NON_SECURE_PROPS_2                            0x52802C
+
+#define mmDMA1_QM_GLBL_NON_SECURE_PROPS_3                            0x528030
+
+#define mmDMA1_QM_GLBL_NON_SECURE_PROPS_4                            0x528034
+
+#define mmDMA1_QM_GLBL_STS0                                          0x528038
+
+#define mmDMA1_QM_GLBL_STS1_0                                        0x528040
+
+#define mmDMA1_QM_GLBL_STS1_1                                        0x528044
+
+#define mmDMA1_QM_GLBL_STS1_2                                        0x528048
+
+#define mmDMA1_QM_GLBL_STS1_3                                        0x52804C
+
+#define mmDMA1_QM_GLBL_STS1_4                                        0x528050
+
+#define mmDMA1_QM_GLBL_MSG_EN_0                                      0x528054
+
+#define mmDMA1_QM_GLBL_MSG_EN_1                                      0x528058
+
+#define mmDMA1_QM_GLBL_MSG_EN_2                                      0x52805C
+
+#define mmDMA1_QM_GLBL_MSG_EN_3                                      0x528060
+
+#define mmDMA1_QM_GLBL_MSG_EN_4                                      0x528068
+
+#define mmDMA1_QM_PQ_BASE_LO_0                                       0x528070
+
+#define mmDMA1_QM_PQ_BASE_LO_1                                       0x528074
+
+#define mmDMA1_QM_PQ_BASE_LO_2                                       0x528078
+
+#define mmDMA1_QM_PQ_BASE_LO_3                                       0x52807C
+
+#define mmDMA1_QM_PQ_BASE_HI_0                                       0x528080
+
+#define mmDMA1_QM_PQ_BASE_HI_1                                       0x528084
+
+#define mmDMA1_QM_PQ_BASE_HI_2                                       0x528088
+
+#define mmDMA1_QM_PQ_BASE_HI_3                                       0x52808C
+
+#define mmDMA1_QM_PQ_SIZE_0                                          0x528090
+
+#define mmDMA1_QM_PQ_SIZE_1                                          0x528094
+
+#define mmDMA1_QM_PQ_SIZE_2                                          0x528098
+
+#define mmDMA1_QM_PQ_SIZE_3                                          0x52809C
+
+#define mmDMA1_QM_PQ_PI_0                                            0x5280A0
+
+#define mmDMA1_QM_PQ_PI_1                                            0x5280A4
+
+#define mmDMA1_QM_PQ_PI_2                                            0x5280A8
+
+#define mmDMA1_QM_PQ_PI_3                                            0x5280AC
+
+#define mmDMA1_QM_PQ_CI_0                                            0x5280B0
+
+#define mmDMA1_QM_PQ_CI_1                                            0x5280B4
+
+#define mmDMA1_QM_PQ_CI_2                                            0x5280B8
+
+#define mmDMA1_QM_PQ_CI_3                                            0x5280BC
+
+#define mmDMA1_QM_PQ_CFG0_0                                          0x5280C0
+
+#define mmDMA1_QM_PQ_CFG0_1                                          0x5280C4
+
+#define mmDMA1_QM_PQ_CFG0_2                                          0x5280C8
+
+#define mmDMA1_QM_PQ_CFG0_3                                          0x5280CC
+
+#define mmDMA1_QM_PQ_CFG1_0                                          0x5280D0
+
+#define mmDMA1_QM_PQ_CFG1_1                                          0x5280D4
+
+#define mmDMA1_QM_PQ_CFG1_2                                          0x5280D8
+
+#define mmDMA1_QM_PQ_CFG1_3                                          0x5280DC
+
+#define mmDMA1_QM_PQ_ARUSER_31_11_0                                  0x5280E0
+
+#define mmDMA1_QM_PQ_ARUSER_31_11_1                                  0x5280E4
+
+#define mmDMA1_QM_PQ_ARUSER_31_11_2                                  0x5280E8
+
+#define mmDMA1_QM_PQ_ARUSER_31_11_3                                  0x5280EC
+
+#define mmDMA1_QM_PQ_STS0_0                                          0x5280F0
+
+#define mmDMA1_QM_PQ_STS0_1                                          0x5280F4
+
+#define mmDMA1_QM_PQ_STS0_2                                          0x5280F8
+
+#define mmDMA1_QM_PQ_STS0_3                                          0x5280FC
+
+#define mmDMA1_QM_PQ_STS1_0                                          0x528100
+
+#define mmDMA1_QM_PQ_STS1_1                                          0x528104
+
+#define mmDMA1_QM_PQ_STS1_2                                          0x528108
+
+#define mmDMA1_QM_PQ_STS1_3                                          0x52810C
+
+#define mmDMA1_QM_CQ_CFG0_0                                          0x528110
+
+#define mmDMA1_QM_CQ_CFG0_1                                          0x528114
+
+#define mmDMA1_QM_CQ_CFG0_2                                          0x528118
+
+#define mmDMA1_QM_CQ_CFG0_3                                          0x52811C
+
+#define mmDMA1_QM_CQ_CFG0_4                                          0x528120
+
+#define mmDMA1_QM_CQ_CFG1_0                                          0x528124
+
+#define mmDMA1_QM_CQ_CFG1_1                                          0x528128
+
+#define mmDMA1_QM_CQ_CFG1_2                                          0x52812C
+
+#define mmDMA1_QM_CQ_CFG1_3                                          0x528130
+
+#define mmDMA1_QM_CQ_CFG1_4                                          0x528134
+
+#define mmDMA1_QM_CQ_ARUSER_31_11_0                                  0x528138
+
+#define mmDMA1_QM_CQ_ARUSER_31_11_1                                  0x52813C
+
+#define mmDMA1_QM_CQ_ARUSER_31_11_2                                  0x528140
+
+#define mmDMA1_QM_CQ_ARUSER_31_11_3                                  0x528144
+
+#define mmDMA1_QM_CQ_ARUSER_31_11_4                                  0x528148
+
+#define mmDMA1_QM_CQ_STS0_0                                          0x52814C
+
+#define mmDMA1_QM_CQ_STS0_1                                          0x528150
+
+#define mmDMA1_QM_CQ_STS0_2                                          0x528154
+
+#define mmDMA1_QM_CQ_STS0_3                                          0x528158
+
+#define mmDMA1_QM_CQ_STS0_4                                          0x52815C
+
+#define mmDMA1_QM_CQ_STS1_0                                          0x528160
+
+#define mmDMA1_QM_CQ_STS1_1                                          0x528164
+
+#define mmDMA1_QM_CQ_STS1_2                                          0x528168
+
+#define mmDMA1_QM_CQ_STS1_3                                          0x52816C
+
+#define mmDMA1_QM_CQ_STS1_4                                          0x528170
+
+#define mmDMA1_QM_CQ_PTR_LO_0                                        0x528174
+
+#define mmDMA1_QM_CQ_PTR_HI_0                                        0x528178
+
+#define mmDMA1_QM_CQ_TSIZE_0                                         0x52817C
+
+#define mmDMA1_QM_CQ_CTL_0                                           0x528180
+
+#define mmDMA1_QM_CQ_PTR_LO_1                                        0x528184
+
+#define mmDMA1_QM_CQ_PTR_HI_1                                        0x528188
+
+#define mmDMA1_QM_CQ_TSIZE_1                                         0x52818C
+
+#define mmDMA1_QM_CQ_CTL_1                                           0x528190
+
+#define mmDMA1_QM_CQ_PTR_LO_2                                        0x528194
+
+#define mmDMA1_QM_CQ_PTR_HI_2                                        0x528198
+
+#define mmDMA1_QM_CQ_TSIZE_2                                         0x52819C
+
+#define mmDMA1_QM_CQ_CTL_2                                           0x5281A0
+
+#define mmDMA1_QM_CQ_PTR_LO_3                                        0x5281A4
+
+#define mmDMA1_QM_CQ_PTR_HI_3                                        0x5281A8
+
+#define mmDMA1_QM_CQ_TSIZE_3                                         0x5281AC
+
+#define mmDMA1_QM_CQ_CTL_3                                           0x5281B0
+
+#define mmDMA1_QM_CQ_PTR_LO_4                                        0x5281B4
+
+#define mmDMA1_QM_CQ_PTR_HI_4                                        0x5281B8
+
+#define mmDMA1_QM_CQ_TSIZE_4                                         0x5281BC
+
+#define mmDMA1_QM_CQ_CTL_4                                           0x5281C0
+
+#define mmDMA1_QM_CQ_PTR_LO_STS_0                                    0x5281C4
+
+#define mmDMA1_QM_CQ_PTR_LO_STS_1                                    0x5281C8
+
+#define mmDMA1_QM_CQ_PTR_LO_STS_2                                    0x5281CC
+
+#define mmDMA1_QM_CQ_PTR_LO_STS_3                                    0x5281D0
+
+#define mmDMA1_QM_CQ_PTR_LO_STS_4                                    0x5281D4
+
+#define mmDMA1_QM_CQ_PTR_HI_STS_0                                    0x5281D8
+
+#define mmDMA1_QM_CQ_PTR_HI_STS_1                                    0x5281DC
+
+#define mmDMA1_QM_CQ_PTR_HI_STS_2                                    0x5281E0
+
+#define mmDMA1_QM_CQ_PTR_HI_STS_3                                    0x5281E4
+
+#define mmDMA1_QM_CQ_PTR_HI_STS_4                                    0x5281E8
+
+#define mmDMA1_QM_CQ_TSIZE_STS_0                                     0x5281EC
+
+#define mmDMA1_QM_CQ_TSIZE_STS_1                                     0x5281F0
+
+#define mmDMA1_QM_CQ_TSIZE_STS_2                                     0x5281F4
+
+#define mmDMA1_QM_CQ_TSIZE_STS_3                                     0x5281F8
+
+#define mmDMA1_QM_CQ_TSIZE_STS_4                                     0x5281FC
+
+#define mmDMA1_QM_CQ_CTL_STS_0                                       0x528200
+
+#define mmDMA1_QM_CQ_CTL_STS_1                                       0x528204
+
+#define mmDMA1_QM_CQ_CTL_STS_2                                       0x528208
+
+#define mmDMA1_QM_CQ_CTL_STS_3                                       0x52820C
+
+#define mmDMA1_QM_CQ_CTL_STS_4                                       0x528210
+
+#define mmDMA1_QM_CQ_IFIFO_CNT_0                                     0x528214
+
+#define mmDMA1_QM_CQ_IFIFO_CNT_1                                     0x528218
+
+#define mmDMA1_QM_CQ_IFIFO_CNT_2                                     0x52821C
+
+#define mmDMA1_QM_CQ_IFIFO_CNT_3                                     0x528220
+
+#define mmDMA1_QM_CQ_IFIFO_CNT_4                                     0x528224
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_0                             0x528228
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_1                             0x52822C
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_2                             0x528230
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_3                             0x528234
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_LO_4                             0x528238
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_0                             0x52823C
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_1                             0x528240
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_2                             0x528244
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_3                             0x528248
+
+#define mmDMA1_QM_CP_MSG_BASE0_ADDR_HI_4                             0x52824C
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_0                             0x528250
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_1                             0x528254
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_2                             0x528258
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_3                             0x52825C
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_LO_4                             0x528260
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_0                             0x528264
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_1                             0x528268
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_2                             0x52826C
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_3                             0x528270
+
+#define mmDMA1_QM_CP_MSG_BASE1_ADDR_HI_4                             0x528274
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_0                             0x528278
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_1                             0x52827C
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_2                             0x528280
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_3                             0x528284
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_LO_4                             0x528288
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_0                             0x52828C
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_1                             0x528290
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_2                             0x528294
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_3                             0x528298
+
+#define mmDMA1_QM_CP_MSG_BASE2_ADDR_HI_4                             0x52829C
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5282A0
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5282A4
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5282A8
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5282AC
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5282B0
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5282B4
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5282B8
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5282BC
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5282C0
+
+#define mmDMA1_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5282C4
+
+#define mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5282C8
+
+#define mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5282CC
+
+#define mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5282D0
+
+#define mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5282D4
+
+#define mmDMA1_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5282D8
+
+#define mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5282E0
+
+#define mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5282E4
+
+#define mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5282E8
+
+#define mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5282EC
+
+#define mmDMA1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5282F0
+
+#define mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5282F4
+
+#define mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5282F8
+
+#define mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5282FC
+
+#define mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x528300
+
+#define mmDMA1_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x528304
+
+#define mmDMA1_QM_CP_FENCE0_RDATA_0                                  0x528308
+
+#define mmDMA1_QM_CP_FENCE0_RDATA_1                                  0x52830C
+
+#define mmDMA1_QM_CP_FENCE0_RDATA_2                                  0x528310
+
+#define mmDMA1_QM_CP_FENCE0_RDATA_3                                  0x528314
+
+#define mmDMA1_QM_CP_FENCE0_RDATA_4                                  0x528318
+
+#define mmDMA1_QM_CP_FENCE1_RDATA_0                                  0x52831C
+
+#define mmDMA1_QM_CP_FENCE1_RDATA_1                                  0x528320
+
+#define mmDMA1_QM_CP_FENCE1_RDATA_2                                  0x528324
+
+#define mmDMA1_QM_CP_FENCE1_RDATA_3                                  0x528328
+
+#define mmDMA1_QM_CP_FENCE1_RDATA_4                                  0x52832C
+
+#define mmDMA1_QM_CP_FENCE2_RDATA_0                                  0x528330
+
+#define mmDMA1_QM_CP_FENCE2_RDATA_1                                  0x528334
+
+#define mmDMA1_QM_CP_FENCE2_RDATA_2                                  0x528338
+
+#define mmDMA1_QM_CP_FENCE2_RDATA_3                                  0x52833C
+
+#define mmDMA1_QM_CP_FENCE2_RDATA_4                                  0x528340
+
+#define mmDMA1_QM_CP_FENCE3_RDATA_0                                  0x528344
+
+#define mmDMA1_QM_CP_FENCE3_RDATA_1                                  0x528348
+
+#define mmDMA1_QM_CP_FENCE3_RDATA_2                                  0x52834C
+
+#define mmDMA1_QM_CP_FENCE3_RDATA_3                                  0x528350
+
+#define mmDMA1_QM_CP_FENCE3_RDATA_4                                  0x528354
+
+#define mmDMA1_QM_CP_FENCE0_CNT_0                                    0x528358
+
+#define mmDMA1_QM_CP_FENCE0_CNT_1                                    0x52835C
+
+#define mmDMA1_QM_CP_FENCE0_CNT_2                                    0x528360
+
+#define mmDMA1_QM_CP_FENCE0_CNT_3                                    0x528364
+
+#define mmDMA1_QM_CP_FENCE0_CNT_4                                    0x528368
+
+#define mmDMA1_QM_CP_FENCE1_CNT_0                                    0x52836C
+
+#define mmDMA1_QM_CP_FENCE1_CNT_1                                    0x528370
+
+#define mmDMA1_QM_CP_FENCE1_CNT_2                                    0x528374
+
+#define mmDMA1_QM_CP_FENCE1_CNT_3                                    0x528378
+
+#define mmDMA1_QM_CP_FENCE1_CNT_4                                    0x52837C
+
+#define mmDMA1_QM_CP_FENCE2_CNT_0                                    0x528380
+
+#define mmDMA1_QM_CP_FENCE2_CNT_1                                    0x528384
+
+#define mmDMA1_QM_CP_FENCE2_CNT_2                                    0x528388
+
+#define mmDMA1_QM_CP_FENCE2_CNT_3                                    0x52838C
+
+#define mmDMA1_QM_CP_FENCE2_CNT_4                                    0x528390
+
+#define mmDMA1_QM_CP_FENCE3_CNT_0                                    0x528394
+
+#define mmDMA1_QM_CP_FENCE3_CNT_1                                    0x528398
+
+#define mmDMA1_QM_CP_FENCE3_CNT_2                                    0x52839C
+
+#define mmDMA1_QM_CP_FENCE3_CNT_3                                    0x5283A0
+
+#define mmDMA1_QM_CP_FENCE3_CNT_4                                    0x5283A4
+
+#define mmDMA1_QM_CP_STS_0                                           0x5283A8
+
+#define mmDMA1_QM_CP_STS_1                                           0x5283AC
+
+#define mmDMA1_QM_CP_STS_2                                           0x5283B0
+
+#define mmDMA1_QM_CP_STS_3                                           0x5283B4
+
+#define mmDMA1_QM_CP_STS_4                                           0x5283B8
+
+#define mmDMA1_QM_CP_CURRENT_INST_LO_0                               0x5283BC
+
+#define mmDMA1_QM_CP_CURRENT_INST_LO_1                               0x5283C0
+
+#define mmDMA1_QM_CP_CURRENT_INST_LO_2                               0x5283C4
+
+#define mmDMA1_QM_CP_CURRENT_INST_LO_3                               0x5283C8
+
+#define mmDMA1_QM_CP_CURRENT_INST_LO_4                               0x5283CC
+
+#define mmDMA1_QM_CP_CURRENT_INST_HI_0                               0x5283D0
+
+#define mmDMA1_QM_CP_CURRENT_INST_HI_1                               0x5283D4
+
+#define mmDMA1_QM_CP_CURRENT_INST_HI_2                               0x5283D8
+
+#define mmDMA1_QM_CP_CURRENT_INST_HI_3                               0x5283DC
+
+#define mmDMA1_QM_CP_CURRENT_INST_HI_4                               0x5283E0
+
+#define mmDMA1_QM_CP_BARRIER_CFG_0                                   0x5283F4
+
+#define mmDMA1_QM_CP_BARRIER_CFG_1                                   0x5283F8
+
+#define mmDMA1_QM_CP_BARRIER_CFG_2                                   0x5283FC
+
+#define mmDMA1_QM_CP_BARRIER_CFG_3                                   0x528400
+
+#define mmDMA1_QM_CP_BARRIER_CFG_4                                   0x528404
+
+#define mmDMA1_QM_CP_DBG_0_0                                         0x528408
+
+#define mmDMA1_QM_CP_DBG_0_1                                         0x52840C
+
+#define mmDMA1_QM_CP_DBG_0_2                                         0x528410
+
+#define mmDMA1_QM_CP_DBG_0_3                                         0x528414
+
+#define mmDMA1_QM_CP_DBG_0_4                                         0x528418
+
+#define mmDMA1_QM_CP_ARUSER_31_11_0                                  0x52841C
+
+#define mmDMA1_QM_CP_ARUSER_31_11_1                                  0x528420
+
+#define mmDMA1_QM_CP_ARUSER_31_11_2                                  0x528424
+
+#define mmDMA1_QM_CP_ARUSER_31_11_3                                  0x528428
+
+#define mmDMA1_QM_CP_ARUSER_31_11_4                                  0x52842C
+
+#define mmDMA1_QM_CP_AWUSER_31_11_0                                  0x528430
+
+#define mmDMA1_QM_CP_AWUSER_31_11_1                                  0x528434
+
+#define mmDMA1_QM_CP_AWUSER_31_11_2                                  0x528438
+
+#define mmDMA1_QM_CP_AWUSER_31_11_3                                  0x52843C
+
+#define mmDMA1_QM_CP_AWUSER_31_11_4                                  0x528440
+
+#define mmDMA1_QM_ARB_CFG_0                                          0x528A00
+
+#define mmDMA1_QM_ARB_CHOISE_Q_PUSH                                  0x528A04
+
+#define mmDMA1_QM_ARB_WRR_WEIGHT_0                                   0x528A08
+
+#define mmDMA1_QM_ARB_WRR_WEIGHT_1                                   0x528A0C
+
+#define mmDMA1_QM_ARB_WRR_WEIGHT_2                                   0x528A10
+
+#define mmDMA1_QM_ARB_WRR_WEIGHT_3                                   0x528A14
+
+#define mmDMA1_QM_ARB_CFG_1                                          0x528A18
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_0                               0x528A20
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_1                               0x528A24
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_2                               0x528A28
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_3                               0x528A2C
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_4                               0x528A30
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_5                               0x528A34
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_6                               0x528A38
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_7                               0x528A3C
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_8                               0x528A40
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_9                               0x528A44
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_10                              0x528A48
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_11                              0x528A4C
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_12                              0x528A50
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_13                              0x528A54
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_14                              0x528A58
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_15                              0x528A5C
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_16                              0x528A60
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_17                              0x528A64
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_18                              0x528A68
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_19                              0x528A6C
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_20                              0x528A70
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_21                              0x528A74
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_22                              0x528A78
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_23                              0x528A7C
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_24                              0x528A80
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_25                              0x528A84
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_26                              0x528A88
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_27                              0x528A8C
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_28                              0x528A90
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_29                              0x528A94
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_30                              0x528A98
+
+#define mmDMA1_QM_ARB_MST_AVAIL_CRED_31                              0x528A9C
+
+#define mmDMA1_QM_ARB_MST_CRED_INC                                   0x528AA0
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x528AA4
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x528AA8
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x528AAC
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x528AB0
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x528AB4
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x528AB8
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x528ABC
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x528AC0
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x528AC4
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x528AC8
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x528ACC
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x528AD0
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x528AD4
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x528AD8
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x528ADC
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x528AE0
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x528AE4
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x528AE8
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x528AEC
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x528AF0
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x528AF4
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x528AF8
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x528AFC
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x528B00
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x528B04
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x528B08
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x528B0C
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x528B10
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x528B14
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x528B18
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x528B1C
+
+#define mmDMA1_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x528B20
+
+#define mmDMA1_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x528B28
+
+#define mmDMA1_QM_ARB_MST_SLAVE_EN                                   0x528B2C
+
+#define mmDMA1_QM_ARB_MST_QUIET_PER                                  0x528B34
+
+#define mmDMA1_QM_ARB_SLV_CHOISE_WDT                                 0x528B38
+
+#define mmDMA1_QM_ARB_SLV_ID                                         0x528B3C
+
+#define mmDMA1_QM_ARB_MSG_MAX_INFLIGHT                               0x528B44
+
+#define mmDMA1_QM_ARB_MSG_AWUSER_31_11                               0x528B48
+
+#define mmDMA1_QM_ARB_MSG_AWUSER_SEC_PROP                            0x528B4C
+
+#define mmDMA1_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x528B50
+
+#define mmDMA1_QM_ARB_BASE_LO                                        0x528B54
+
+#define mmDMA1_QM_ARB_BASE_HI                                        0x528B58
+
+#define mmDMA1_QM_ARB_STATE_STS                                      0x528B80
+
+#define mmDMA1_QM_ARB_CHOISE_FULLNESS_STS                            0x528B84
+
+#define mmDMA1_QM_ARB_MSG_STS                                        0x528B88
+
+#define mmDMA1_QM_ARB_SLV_CHOISE_Q_HEAD                              0x528B8C
+
+#define mmDMA1_QM_ARB_ERR_CAUSE                                      0x528B9C
+
+#define mmDMA1_QM_ARB_ERR_MSG_EN                                     0x528BA0
+
+#define mmDMA1_QM_ARB_ERR_STS_DRP                                    0x528BA8
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_0                                 0x528BB0
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_1                                 0x528BB4
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_2                                 0x528BB8
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_3                                 0x528BBC
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_4                                 0x528BC0
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_5                                 0x528BC4
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_6                                 0x528BC8
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_7                                 0x528BCC
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_8                                 0x528BD0
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_9                                 0x528BD4
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_10                                0x528BD8
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_11                                0x528BDC
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_12                                0x528BE0
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_13                                0x528BE4
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_14                                0x528BE8
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_15                                0x528BEC
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_16                                0x528BF0
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_17                                0x528BF4
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_18                                0x528BF8
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_19                                0x528BFC
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_20                                0x528C00
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_21                                0x528C04
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_22                                0x528C08
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_23                                0x528C0C
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_24                                0x528C10
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_25                                0x528C14
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_26                                0x528C18
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_27                                0x528C1C
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_28                                0x528C20
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_29                                0x528C24
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_30                                0x528C28
+
+#define mmDMA1_QM_ARB_MST_CRED_STS_31                                0x528C2C
+
+#define mmDMA1_QM_CGM_CFG                                            0x528C70
+
+#define mmDMA1_QM_CGM_STS                                            0x528C74
+
+#define mmDMA1_QM_CGM_CFG1                                           0x528C78
+
+#define mmDMA1_QM_LOCAL_RANGE_BASE                                   0x528C80
+
+#define mmDMA1_QM_LOCAL_RANGE_SIZE                                   0x528C84
+
+#define mmDMA1_QM_CSMR_STRICT_PRIO_CFG                               0x528C90
+
+#define mmDMA1_QM_HBW_RD_RATE_LIM_CFG_1                              0x528C94
+
+#define mmDMA1_QM_LBW_WR_RATE_LIM_CFG_0                              0x528C98
+
+#define mmDMA1_QM_LBW_WR_RATE_LIM_CFG_1                              0x528C9C
+
+#define mmDMA1_QM_HBW_RD_RATE_LIM_CFG_0                              0x528CA0
+
+#define mmDMA1_QM_GLBL_AXCACHE                                       0x528CA4
+
+#define mmDMA1_QM_IND_GW_APB_CFG                                     0x528CB0
+
+#define mmDMA1_QM_IND_GW_APB_WDATA                                   0x528CB4
+
+#define mmDMA1_QM_IND_GW_APB_RDATA                                   0x528CB8
+
+#define mmDMA1_QM_IND_GW_APB_STATUS                                  0x528CBC
+
+#define mmDMA1_QM_GLBL_ERR_ADDR_LO                                   0x528CD0
+
+#define mmDMA1_QM_GLBL_ERR_ADDR_HI                                   0x528CD4
+
+#define mmDMA1_QM_GLBL_ERR_WDATA                                     0x528CD8
+
+#define mmDMA1_QM_GLBL_MEM_INIT_BUSY                                 0x528D00
+
+#endif /* ASIC_REG_DMA1_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_core_regs.h
new file mode 100644
index 000000000000..a42862cd5ae0
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA2_CORE_REGS_H_
+#define ASIC_REG_DMA2_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA2_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA2_CORE_CFG_0                                            0x540000
+
+#define mmDMA2_CORE_CFG_1                                            0x540004
+
+#define mmDMA2_CORE_LBW_MAX_OUTSTAND                                 0x540008
+
+#define mmDMA2_CORE_SRC_BASE_LO                                      0x540014
+
+#define mmDMA2_CORE_SRC_BASE_HI                                      0x540018
+
+#define mmDMA2_CORE_DST_BASE_LO                                      0x54001C
+
+#define mmDMA2_CORE_DST_BASE_HI                                      0x540020
+
+#define mmDMA2_CORE_SRC_TSIZE_1                                      0x54002C
+
+#define mmDMA2_CORE_SRC_STRIDE_1                                     0x540030
+
+#define mmDMA2_CORE_SRC_TSIZE_2                                      0x540034
+
+#define mmDMA2_CORE_SRC_STRIDE_2                                     0x540038
+
+#define mmDMA2_CORE_SRC_TSIZE_3                                      0x54003C
+
+#define mmDMA2_CORE_SRC_STRIDE_3                                     0x540040
+
+#define mmDMA2_CORE_SRC_TSIZE_4                                      0x540044
+
+#define mmDMA2_CORE_SRC_STRIDE_4                                     0x540048
+
+#define mmDMA2_CORE_SRC_TSIZE_0                                      0x54004C
+
+#define mmDMA2_CORE_DST_TSIZE_1                                      0x540054
+
+#define mmDMA2_CORE_DST_STRIDE_1                                     0x540058
+
+#define mmDMA2_CORE_DST_TSIZE_2                                      0x54005C
+
+#define mmDMA2_CORE_DST_STRIDE_2                                     0x540060
+
+#define mmDMA2_CORE_DST_TSIZE_3                                      0x540064
+
+#define mmDMA2_CORE_DST_STRIDE_3                                     0x540068
+
+#define mmDMA2_CORE_DST_TSIZE_4                                      0x54006C
+
+#define mmDMA2_CORE_DST_STRIDE_4                                     0x540070
+
+#define mmDMA2_CORE_DST_TSIZE_0                                      0x540074
+
+#define mmDMA2_CORE_COMMIT                                           0x540078
+
+#define mmDMA2_CORE_WR_COMP_WDATA                                    0x54007C
+
+#define mmDMA2_CORE_WR_COMP_ADDR_LO                                  0x540080
+
+#define mmDMA2_CORE_WR_COMP_ADDR_HI                                  0x540084
+
+#define mmDMA2_CORE_WR_COMP_AWUSER_31_11                             0x540088
+
+#define mmDMA2_CORE_TE_NUMROWS                                       0x540094
+
+#define mmDMA2_CORE_PROT                                             0x5400B8
+
+#define mmDMA2_CORE_SECURE_PROPS                                     0x5400F0
+
+#define mmDMA2_CORE_NON_SECURE_PROPS                                 0x5400F4
+
+#define mmDMA2_CORE_RD_MAX_OUTSTAND                                  0x540100
+
+#define mmDMA2_CORE_RD_MAX_SIZE                                      0x540104
+
+#define mmDMA2_CORE_RD_ARCACHE                                       0x540108
+
+#define mmDMA2_CORE_RD_ARUSER_31_11                                  0x540110
+
+#define mmDMA2_CORE_RD_INFLIGHTS                                     0x540114
+
+#define mmDMA2_CORE_WR_MAX_OUTSTAND                                  0x540120
+
+#define mmDMA2_CORE_WR_MAX_AWID                                      0x540124
+
+#define mmDMA2_CORE_WR_AWCACHE                                       0x540128
+
+#define mmDMA2_CORE_WR_AWUSER_31_11                                  0x540130
+
+#define mmDMA2_CORE_WR_INFLIGHTS                                     0x540134
+
+#define mmDMA2_CORE_RD_RATE_LIM_CFG_0                                0x540150
+
+#define mmDMA2_CORE_RD_RATE_LIM_CFG_1                                0x540154
+
+#define mmDMA2_CORE_WR_RATE_LIM_CFG_0                                0x540158
+
+#define mmDMA2_CORE_WR_RATE_LIM_CFG_1                                0x54015C
+
+#define mmDMA2_CORE_ERR_CFG                                          0x540160
+
+#define mmDMA2_CORE_ERR_CAUSE                                        0x540164
+
+#define mmDMA2_CORE_ERRMSG_ADDR_LO                                   0x540170
+
+#define mmDMA2_CORE_ERRMSG_ADDR_HI                                   0x540174
+
+#define mmDMA2_CORE_ERRMSG_WDATA                                     0x540178
+
+#define mmDMA2_CORE_STS0                                             0x540190
+
+#define mmDMA2_CORE_STS1                                             0x540194
+
+#define mmDMA2_CORE_RD_DBGMEM_ADD                                    0x540200
+
+#define mmDMA2_CORE_RD_DBGMEM_DATA_WR                                0x540204
+
+#define mmDMA2_CORE_RD_DBGMEM_DATA_RD                                0x540208
+
+#define mmDMA2_CORE_RD_DBGMEM_CTRL                                   0x54020C
+
+#define mmDMA2_CORE_RD_DBGMEM_RC                                     0x540210
+
+#define mmDMA2_CORE_DBG_HBW_AXI_AR_CNT                               0x540220
+
+#define mmDMA2_CORE_DBG_HBW_AXI_AW_CNT                               0x540224
+
+#define mmDMA2_CORE_DBG_LBW_AXI_AW_CNT                               0x540228
+
+#define mmDMA2_CORE_DBG_DESC_CNT                                     0x54022C
+
+#define mmDMA2_CORE_DBG_STS                                          0x540230
+
+#define mmDMA2_CORE_DBG_RD_DESC_ID                                   0x540234
+
+#define mmDMA2_CORE_DBG_WR_DESC_ID                                   0x540238
+
+#endif /* ASIC_REG_DMA2_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_qm_regs.h
new file mode 100644
index 000000000000..8c4d4e016852
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma2_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA2_QM_REGS_H_
+#define ASIC_REG_DMA2_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA2_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA2_QM_GLBL_CFG0                                          0x548000
+
+#define mmDMA2_QM_GLBL_CFG1                                          0x548004
+
+#define mmDMA2_QM_GLBL_PROT                                          0x548008
+
+#define mmDMA2_QM_GLBL_ERR_CFG                                       0x54800C
+
+#define mmDMA2_QM_GLBL_SECURE_PROPS_0                                0x548010
+
+#define mmDMA2_QM_GLBL_SECURE_PROPS_1                                0x548014
+
+#define mmDMA2_QM_GLBL_SECURE_PROPS_2                                0x548018
+
+#define mmDMA2_QM_GLBL_SECURE_PROPS_3                                0x54801C
+
+#define mmDMA2_QM_GLBL_SECURE_PROPS_4                                0x548020
+
+#define mmDMA2_QM_GLBL_NON_SECURE_PROPS_0                            0x548024
+
+#define mmDMA2_QM_GLBL_NON_SECURE_PROPS_1                            0x548028
+
+#define mmDMA2_QM_GLBL_NON_SECURE_PROPS_2                            0x54802C
+
+#define mmDMA2_QM_GLBL_NON_SECURE_PROPS_3                            0x548030
+
+#define mmDMA2_QM_GLBL_NON_SECURE_PROPS_4                            0x548034
+
+#define mmDMA2_QM_GLBL_STS0                                          0x548038
+
+#define mmDMA2_QM_GLBL_STS1_0                                        0x548040
+
+#define mmDMA2_QM_GLBL_STS1_1                                        0x548044
+
+#define mmDMA2_QM_GLBL_STS1_2                                        0x548048
+
+#define mmDMA2_QM_GLBL_STS1_3                                        0x54804C
+
+#define mmDMA2_QM_GLBL_STS1_4                                        0x548050
+
+#define mmDMA2_QM_GLBL_MSG_EN_0                                      0x548054
+
+#define mmDMA2_QM_GLBL_MSG_EN_1                                      0x548058
+
+#define mmDMA2_QM_GLBL_MSG_EN_2                                      0x54805C
+
+#define mmDMA2_QM_GLBL_MSG_EN_3                                      0x548060
+
+#define mmDMA2_QM_GLBL_MSG_EN_4                                      0x548068
+
+#define mmDMA2_QM_PQ_BASE_LO_0                                       0x548070
+
+#define mmDMA2_QM_PQ_BASE_LO_1                                       0x548074
+
+#define mmDMA2_QM_PQ_BASE_LO_2                                       0x548078
+
+#define mmDMA2_QM_PQ_BASE_LO_3                                       0x54807C
+
+#define mmDMA2_QM_PQ_BASE_HI_0                                       0x548080
+
+#define mmDMA2_QM_PQ_BASE_HI_1                                       0x548084
+
+#define mmDMA2_QM_PQ_BASE_HI_2                                       0x548088
+
+#define mmDMA2_QM_PQ_BASE_HI_3                                       0x54808C
+
+#define mmDMA2_QM_PQ_SIZE_0                                          0x548090
+
+#define mmDMA2_QM_PQ_SIZE_1                                          0x548094
+
+#define mmDMA2_QM_PQ_SIZE_2                                          0x548098
+
+#define mmDMA2_QM_PQ_SIZE_3                                          0x54809C
+
+#define mmDMA2_QM_PQ_PI_0                                            0x5480A0
+
+#define mmDMA2_QM_PQ_PI_1                                            0x5480A4
+
+#define mmDMA2_QM_PQ_PI_2                                            0x5480A8
+
+#define mmDMA2_QM_PQ_PI_3                                            0x5480AC
+
+#define mmDMA2_QM_PQ_CI_0                                            0x5480B0
+
+#define mmDMA2_QM_PQ_CI_1                                            0x5480B4
+
+#define mmDMA2_QM_PQ_CI_2                                            0x5480B8
+
+#define mmDMA2_QM_PQ_CI_3                                            0x5480BC
+
+#define mmDMA2_QM_PQ_CFG0_0                                          0x5480C0
+
+#define mmDMA2_QM_PQ_CFG0_1                                          0x5480C4
+
+#define mmDMA2_QM_PQ_CFG0_2                                          0x5480C8
+
+#define mmDMA2_QM_PQ_CFG0_3                                          0x5480CC
+
+#define mmDMA2_QM_PQ_CFG1_0                                          0x5480D0
+
+#define mmDMA2_QM_PQ_CFG1_1                                          0x5480D4
+
+#define mmDMA2_QM_PQ_CFG1_2                                          0x5480D8
+
+#define mmDMA2_QM_PQ_CFG1_3                                          0x5480DC
+
+#define mmDMA2_QM_PQ_ARUSER_31_11_0                                  0x5480E0
+
+#define mmDMA2_QM_PQ_ARUSER_31_11_1                                  0x5480E4
+
+#define mmDMA2_QM_PQ_ARUSER_31_11_2                                  0x5480E8
+
+#define mmDMA2_QM_PQ_ARUSER_31_11_3                                  0x5480EC
+
+#define mmDMA2_QM_PQ_STS0_0                                          0x5480F0
+
+#define mmDMA2_QM_PQ_STS0_1                                          0x5480F4
+
+#define mmDMA2_QM_PQ_STS0_2                                          0x5480F8
+
+#define mmDMA2_QM_PQ_STS0_3                                          0x5480FC
+
+#define mmDMA2_QM_PQ_STS1_0                                          0x548100
+
+#define mmDMA2_QM_PQ_STS1_1                                          0x548104
+
+#define mmDMA2_QM_PQ_STS1_2                                          0x548108
+
+#define mmDMA2_QM_PQ_STS1_3                                          0x54810C
+
+#define mmDMA2_QM_CQ_CFG0_0                                          0x548110
+
+#define mmDMA2_QM_CQ_CFG0_1                                          0x548114
+
+#define mmDMA2_QM_CQ_CFG0_2                                          0x548118
+
+#define mmDMA2_QM_CQ_CFG0_3                                          0x54811C
+
+#define mmDMA2_QM_CQ_CFG0_4                                          0x548120
+
+#define mmDMA2_QM_CQ_CFG1_0                                          0x548124
+
+#define mmDMA2_QM_CQ_CFG1_1                                          0x548128
+
+#define mmDMA2_QM_CQ_CFG1_2                                          0x54812C
+
+#define mmDMA2_QM_CQ_CFG1_3                                          0x548130
+
+#define mmDMA2_QM_CQ_CFG1_4                                          0x548134
+
+#define mmDMA2_QM_CQ_ARUSER_31_11_0                                  0x548138
+
+#define mmDMA2_QM_CQ_ARUSER_31_11_1                                  0x54813C
+
+#define mmDMA2_QM_CQ_ARUSER_31_11_2                                  0x548140
+
+#define mmDMA2_QM_CQ_ARUSER_31_11_3                                  0x548144
+
+#define mmDMA2_QM_CQ_ARUSER_31_11_4                                  0x548148
+
+#define mmDMA2_QM_CQ_STS0_0                                          0x54814C
+
+#define mmDMA2_QM_CQ_STS0_1                                          0x548150
+
+#define mmDMA2_QM_CQ_STS0_2                                          0x548154
+
+#define mmDMA2_QM_CQ_STS0_3                                          0x548158
+
+#define mmDMA2_QM_CQ_STS0_4                                          0x54815C
+
+#define mmDMA2_QM_CQ_STS1_0                                          0x548160
+
+#define mmDMA2_QM_CQ_STS1_1                                          0x548164
+
+#define mmDMA2_QM_CQ_STS1_2                                          0x548168
+
+#define mmDMA2_QM_CQ_STS1_3                                          0x54816C
+
+#define mmDMA2_QM_CQ_STS1_4                                          0x548170
+
+#define mmDMA2_QM_CQ_PTR_LO_0                                        0x548174
+
+#define mmDMA2_QM_CQ_PTR_HI_0                                        0x548178
+
+#define mmDMA2_QM_CQ_TSIZE_0                                         0x54817C
+
+#define mmDMA2_QM_CQ_CTL_0                                           0x548180
+
+#define mmDMA2_QM_CQ_PTR_LO_1                                        0x548184
+
+#define mmDMA2_QM_CQ_PTR_HI_1                                        0x548188
+
+#define mmDMA2_QM_CQ_TSIZE_1                                         0x54818C
+
+#define mmDMA2_QM_CQ_CTL_1                                           0x548190
+
+#define mmDMA2_QM_CQ_PTR_LO_2                                        0x548194
+
+#define mmDMA2_QM_CQ_PTR_HI_2                                        0x548198
+
+#define mmDMA2_QM_CQ_TSIZE_2                                         0x54819C
+
+#define mmDMA2_QM_CQ_CTL_2                                           0x5481A0
+
+#define mmDMA2_QM_CQ_PTR_LO_3                                        0x5481A4
+
+#define mmDMA2_QM_CQ_PTR_HI_3                                        0x5481A8
+
+#define mmDMA2_QM_CQ_TSIZE_3                                         0x5481AC
+
+#define mmDMA2_QM_CQ_CTL_3                                           0x5481B0
+
+#define mmDMA2_QM_CQ_PTR_LO_4                                        0x5481B4
+
+#define mmDMA2_QM_CQ_PTR_HI_4                                        0x5481B8
+
+#define mmDMA2_QM_CQ_TSIZE_4                                         0x5481BC
+
+#define mmDMA2_QM_CQ_CTL_4                                           0x5481C0
+
+#define mmDMA2_QM_CQ_PTR_LO_STS_0                                    0x5481C4
+
+#define mmDMA2_QM_CQ_PTR_LO_STS_1                                    0x5481C8
+
+#define mmDMA2_QM_CQ_PTR_LO_STS_2                                    0x5481CC
+
+#define mmDMA2_QM_CQ_PTR_LO_STS_3                                    0x5481D0
+
+#define mmDMA2_QM_CQ_PTR_LO_STS_4                                    0x5481D4
+
+#define mmDMA2_QM_CQ_PTR_HI_STS_0                                    0x5481D8
+
+#define mmDMA2_QM_CQ_PTR_HI_STS_1                                    0x5481DC
+
+#define mmDMA2_QM_CQ_PTR_HI_STS_2                                    0x5481E0
+
+#define mmDMA2_QM_CQ_PTR_HI_STS_3                                    0x5481E4
+
+#define mmDMA2_QM_CQ_PTR_HI_STS_4                                    0x5481E8
+
+#define mmDMA2_QM_CQ_TSIZE_STS_0                                     0x5481EC
+
+#define mmDMA2_QM_CQ_TSIZE_STS_1                                     0x5481F0
+
+#define mmDMA2_QM_CQ_TSIZE_STS_2                                     0x5481F4
+
+#define mmDMA2_QM_CQ_TSIZE_STS_3                                     0x5481F8
+
+#define mmDMA2_QM_CQ_TSIZE_STS_4                                     0x5481FC
+
+#define mmDMA2_QM_CQ_CTL_STS_0                                       0x548200
+
+#define mmDMA2_QM_CQ_CTL_STS_1                                       0x548204
+
+#define mmDMA2_QM_CQ_CTL_STS_2                                       0x548208
+
+#define mmDMA2_QM_CQ_CTL_STS_3                                       0x54820C
+
+#define mmDMA2_QM_CQ_CTL_STS_4                                       0x548210
+
+#define mmDMA2_QM_CQ_IFIFO_CNT_0                                     0x548214
+
+#define mmDMA2_QM_CQ_IFIFO_CNT_1                                     0x548218
+
+#define mmDMA2_QM_CQ_IFIFO_CNT_2                                     0x54821C
+
+#define mmDMA2_QM_CQ_IFIFO_CNT_3                                     0x548220
+
+#define mmDMA2_QM_CQ_IFIFO_CNT_4                                     0x548224
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_0                             0x548228
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_1                             0x54822C
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_2                             0x548230
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_3                             0x548234
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_LO_4                             0x548238
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_0                             0x54823C
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_1                             0x548240
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_2                             0x548244
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_3                             0x548248
+
+#define mmDMA2_QM_CP_MSG_BASE0_ADDR_HI_4                             0x54824C
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_0                             0x548250
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_1                             0x548254
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_2                             0x548258
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_3                             0x54825C
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_LO_4                             0x548260
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_0                             0x548264
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_1                             0x548268
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_2                             0x54826C
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_3                             0x548270
+
+#define mmDMA2_QM_CP_MSG_BASE1_ADDR_HI_4                             0x548274
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_0                             0x548278
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_1                             0x54827C
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_2                             0x548280
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_3                             0x548284
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_LO_4                             0x548288
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_0                             0x54828C
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_1                             0x548290
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_2                             0x548294
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_3                             0x548298
+
+#define mmDMA2_QM_CP_MSG_BASE2_ADDR_HI_4                             0x54829C
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5482A0
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5482A4
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5482A8
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5482AC
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5482B0
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5482B4
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5482B8
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5482BC
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5482C0
+
+#define mmDMA2_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5482C4
+
+#define mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5482C8
+
+#define mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5482CC
+
+#define mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5482D0
+
+#define mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5482D4
+
+#define mmDMA2_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5482D8
+
+#define mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5482E0
+
+#define mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5482E4
+
+#define mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5482E8
+
+#define mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5482EC
+
+#define mmDMA2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5482F0
+
+#define mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5482F4
+
+#define mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5482F8
+
+#define mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5482FC
+
+#define mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x548300
+
+#define mmDMA2_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x548304
+
+#define mmDMA2_QM_CP_FENCE0_RDATA_0                                  0x548308
+
+#define mmDMA2_QM_CP_FENCE0_RDATA_1                                  0x54830C
+
+#define mmDMA2_QM_CP_FENCE0_RDATA_2                                  0x548310
+
+#define mmDMA2_QM_CP_FENCE0_RDATA_3                                  0x548314
+
+#define mmDMA2_QM_CP_FENCE0_RDATA_4                                  0x548318
+
+#define mmDMA2_QM_CP_FENCE1_RDATA_0                                  0x54831C
+
+#define mmDMA2_QM_CP_FENCE1_RDATA_1                                  0x548320
+
+#define mmDMA2_QM_CP_FENCE1_RDATA_2                                  0x548324
+
+#define mmDMA2_QM_CP_FENCE1_RDATA_3                                  0x548328
+
+#define mmDMA2_QM_CP_FENCE1_RDATA_4                                  0x54832C
+
+#define mmDMA2_QM_CP_FENCE2_RDATA_0                                  0x548330
+
+#define mmDMA2_QM_CP_FENCE2_RDATA_1                                  0x548334
+
+#define mmDMA2_QM_CP_FENCE2_RDATA_2                                  0x548338
+
+#define mmDMA2_QM_CP_FENCE2_RDATA_3                                  0x54833C
+
+#define mmDMA2_QM_CP_FENCE2_RDATA_4                                  0x548340
+
+#define mmDMA2_QM_CP_FENCE3_RDATA_0                                  0x548344
+
+#define mmDMA2_QM_CP_FENCE3_RDATA_1                                  0x548348
+
+#define mmDMA2_QM_CP_FENCE3_RDATA_2                                  0x54834C
+
+#define mmDMA2_QM_CP_FENCE3_RDATA_3                                  0x548350
+
+#define mmDMA2_QM_CP_FENCE3_RDATA_4                                  0x548354
+
+#define mmDMA2_QM_CP_FENCE0_CNT_0                                    0x548358
+
+#define mmDMA2_QM_CP_FENCE0_CNT_1                                    0x54835C
+
+#define mmDMA2_QM_CP_FENCE0_CNT_2                                    0x548360
+
+#define mmDMA2_QM_CP_FENCE0_CNT_3                                    0x548364
+
+#define mmDMA2_QM_CP_FENCE0_CNT_4                                    0x548368
+
+#define mmDMA2_QM_CP_FENCE1_CNT_0                                    0x54836C
+
+#define mmDMA2_QM_CP_FENCE1_CNT_1                                    0x548370
+
+#define mmDMA2_QM_CP_FENCE1_CNT_2                                    0x548374
+
+#define mmDMA2_QM_CP_FENCE1_CNT_3                                    0x548378
+
+#define mmDMA2_QM_CP_FENCE1_CNT_4                                    0x54837C
+
+#define mmDMA2_QM_CP_FENCE2_CNT_0                                    0x548380
+
+#define mmDMA2_QM_CP_FENCE2_CNT_1                                    0x548384
+
+#define mmDMA2_QM_CP_FENCE2_CNT_2                                    0x548388
+
+#define mmDMA2_QM_CP_FENCE2_CNT_3                                    0x54838C
+
+#define mmDMA2_QM_CP_FENCE2_CNT_4                                    0x548390
+
+#define mmDMA2_QM_CP_FENCE3_CNT_0                                    0x548394
+
+#define mmDMA2_QM_CP_FENCE3_CNT_1                                    0x548398
+
+#define mmDMA2_QM_CP_FENCE3_CNT_2                                    0x54839C
+
+#define mmDMA2_QM_CP_FENCE3_CNT_3                                    0x5483A0
+
+#define mmDMA2_QM_CP_FENCE3_CNT_4                                    0x5483A4
+
+#define mmDMA2_QM_CP_STS_0                                           0x5483A8
+
+#define mmDMA2_QM_CP_STS_1                                           0x5483AC
+
+#define mmDMA2_QM_CP_STS_2                                           0x5483B0
+
+#define mmDMA2_QM_CP_STS_3                                           0x5483B4
+
+#define mmDMA2_QM_CP_STS_4                                           0x5483B8
+
+#define mmDMA2_QM_CP_CURRENT_INST_LO_0                               0x5483BC
+
+#define mmDMA2_QM_CP_CURRENT_INST_LO_1                               0x5483C0
+
+#define mmDMA2_QM_CP_CURRENT_INST_LO_2                               0x5483C4
+
+#define mmDMA2_QM_CP_CURRENT_INST_LO_3                               0x5483C8
+
+#define mmDMA2_QM_CP_CURRENT_INST_LO_4                               0x5483CC
+
+#define mmDMA2_QM_CP_CURRENT_INST_HI_0                               0x5483D0
+
+#define mmDMA2_QM_CP_CURRENT_INST_HI_1                               0x5483D4
+
+#define mmDMA2_QM_CP_CURRENT_INST_HI_2                               0x5483D8
+
+#define mmDMA2_QM_CP_CURRENT_INST_HI_3                               0x5483DC
+
+#define mmDMA2_QM_CP_CURRENT_INST_HI_4                               0x5483E0
+
+#define mmDMA2_QM_CP_BARRIER_CFG_0                                   0x5483F4
+
+#define mmDMA2_QM_CP_BARRIER_CFG_1                                   0x5483F8
+
+#define mmDMA2_QM_CP_BARRIER_CFG_2                                   0x5483FC
+
+#define mmDMA2_QM_CP_BARRIER_CFG_3                                   0x548400
+
+#define mmDMA2_QM_CP_BARRIER_CFG_4                                   0x548404
+
+#define mmDMA2_QM_CP_DBG_0_0                                         0x548408
+
+#define mmDMA2_QM_CP_DBG_0_1                                         0x54840C
+
+#define mmDMA2_QM_CP_DBG_0_2                                         0x548410
+
+#define mmDMA2_QM_CP_DBG_0_3                                         0x548414
+
+#define mmDMA2_QM_CP_DBG_0_4                                         0x548418
+
+#define mmDMA2_QM_CP_ARUSER_31_11_0                                  0x54841C
+
+#define mmDMA2_QM_CP_ARUSER_31_11_1                                  0x548420
+
+#define mmDMA2_QM_CP_ARUSER_31_11_2                                  0x548424
+
+#define mmDMA2_QM_CP_ARUSER_31_11_3                                  0x548428
+
+#define mmDMA2_QM_CP_ARUSER_31_11_4                                  0x54842C
+
+#define mmDMA2_QM_CP_AWUSER_31_11_0                                  0x548430
+
+#define mmDMA2_QM_CP_AWUSER_31_11_1                                  0x548434
+
+#define mmDMA2_QM_CP_AWUSER_31_11_2                                  0x548438
+
+#define mmDMA2_QM_CP_AWUSER_31_11_3                                  0x54843C
+
+#define mmDMA2_QM_CP_AWUSER_31_11_4                                  0x548440
+
+#define mmDMA2_QM_ARB_CFG_0                                          0x548A00
+
+#define mmDMA2_QM_ARB_CHOISE_Q_PUSH                                  0x548A04
+
+#define mmDMA2_QM_ARB_WRR_WEIGHT_0                                   0x548A08
+
+#define mmDMA2_QM_ARB_WRR_WEIGHT_1                                   0x548A0C
+
+#define mmDMA2_QM_ARB_WRR_WEIGHT_2                                   0x548A10
+
+#define mmDMA2_QM_ARB_WRR_WEIGHT_3                                   0x548A14
+
+#define mmDMA2_QM_ARB_CFG_1                                          0x548A18
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_0                               0x548A20
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_1                               0x548A24
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_2                               0x548A28
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_3                               0x548A2C
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_4                               0x548A30
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_5                               0x548A34
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_6                               0x548A38
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_7                               0x548A3C
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_8                               0x548A40
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_9                               0x548A44
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_10                              0x548A48
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_11                              0x548A4C
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_12                              0x548A50
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_13                              0x548A54
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_14                              0x548A58
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_15                              0x548A5C
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_16                              0x548A60
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_17                              0x548A64
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_18                              0x548A68
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_19                              0x548A6C
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_20                              0x548A70
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_21                              0x548A74
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_22                              0x548A78
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_23                              0x548A7C
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_24                              0x548A80
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_25                              0x548A84
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_26                              0x548A88
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_27                              0x548A8C
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_28                              0x548A90
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_29                              0x548A94
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_30                              0x548A98
+
+#define mmDMA2_QM_ARB_MST_AVAIL_CRED_31                              0x548A9C
+
+#define mmDMA2_QM_ARB_MST_CRED_INC                                   0x548AA0
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x548AA4
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x548AA8
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x548AAC
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x548AB0
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x548AB4
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x548AB8
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x548ABC
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x548AC0
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x548AC4
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x548AC8
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x548ACC
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x548AD0
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x548AD4
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x548AD8
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x548ADC
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x548AE0
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x548AE4
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x548AE8
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x548AEC
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x548AF0
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x548AF4
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x548AF8
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x548AFC
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x548B00
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x548B04
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x548B08
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x548B0C
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x548B10
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x548B14
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x548B18
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x548B1C
+
+#define mmDMA2_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x548B20
+
+#define mmDMA2_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x548B28
+
+#define mmDMA2_QM_ARB_MST_SLAVE_EN                                   0x548B2C
+
+#define mmDMA2_QM_ARB_MST_QUIET_PER                                  0x548B34
+
+#define mmDMA2_QM_ARB_SLV_CHOISE_WDT                                 0x548B38
+
+#define mmDMA2_QM_ARB_SLV_ID                                         0x548B3C
+
+#define mmDMA2_QM_ARB_MSG_MAX_INFLIGHT                               0x548B44
+
+#define mmDMA2_QM_ARB_MSG_AWUSER_31_11                               0x548B48
+
+#define mmDMA2_QM_ARB_MSG_AWUSER_SEC_PROP                            0x548B4C
+
+#define mmDMA2_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x548B50
+
+#define mmDMA2_QM_ARB_BASE_LO                                        0x548B54
+
+#define mmDMA2_QM_ARB_BASE_HI                                        0x548B58
+
+#define mmDMA2_QM_ARB_STATE_STS                                      0x548B80
+
+#define mmDMA2_QM_ARB_CHOISE_FULLNESS_STS                            0x548B84
+
+#define mmDMA2_QM_ARB_MSG_STS                                        0x548B88
+
+#define mmDMA2_QM_ARB_SLV_CHOISE_Q_HEAD                              0x548B8C
+
+#define mmDMA2_QM_ARB_ERR_CAUSE                                      0x548B9C
+
+#define mmDMA2_QM_ARB_ERR_MSG_EN                                     0x548BA0
+
+#define mmDMA2_QM_ARB_ERR_STS_DRP                                    0x548BA8
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_0                                 0x548BB0
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_1                                 0x548BB4
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_2                                 0x548BB8
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_3                                 0x548BBC
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_4                                 0x548BC0
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_5                                 0x548BC4
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_6                                 0x548BC8
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_7                                 0x548BCC
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_8                                 0x548BD0
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_9                                 0x548BD4
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_10                                0x548BD8
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_11                                0x548BDC
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_12                                0x548BE0
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_13                                0x548BE4
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_14                                0x548BE8
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_15                                0x548BEC
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_16                                0x548BF0
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_17                                0x548BF4
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_18                                0x548BF8
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_19                                0x548BFC
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_20                                0x548C00
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_21                                0x548C04
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_22                                0x548C08
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_23                                0x548C0C
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_24                                0x548C10
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_25                                0x548C14
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_26                                0x548C18
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_27                                0x548C1C
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_28                                0x548C20
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_29                                0x548C24
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_30                                0x548C28
+
+#define mmDMA2_QM_ARB_MST_CRED_STS_31                                0x548C2C
+
+#define mmDMA2_QM_CGM_CFG                                            0x548C70
+
+#define mmDMA2_QM_CGM_STS                                            0x548C74
+
+#define mmDMA2_QM_CGM_CFG1                                           0x548C78
+
+#define mmDMA2_QM_LOCAL_RANGE_BASE                                   0x548C80
+
+#define mmDMA2_QM_LOCAL_RANGE_SIZE                                   0x548C84
+
+#define mmDMA2_QM_CSMR_STRICT_PRIO_CFG                               0x548C90
+
+#define mmDMA2_QM_HBW_RD_RATE_LIM_CFG_1                              0x548C94
+
+#define mmDMA2_QM_LBW_WR_RATE_LIM_CFG_0                              0x548C98
+
+#define mmDMA2_QM_LBW_WR_RATE_LIM_CFG_1                              0x548C9C
+
+#define mmDMA2_QM_HBW_RD_RATE_LIM_CFG_0                              0x548CA0
+
+#define mmDMA2_QM_GLBL_AXCACHE                                       0x548CA4
+
+#define mmDMA2_QM_IND_GW_APB_CFG                                     0x548CB0
+
+#define mmDMA2_QM_IND_GW_APB_WDATA                                   0x548CB4
+
+#define mmDMA2_QM_IND_GW_APB_RDATA                                   0x548CB8
+
+#define mmDMA2_QM_IND_GW_APB_STATUS                                  0x548CBC
+
+#define mmDMA2_QM_GLBL_ERR_ADDR_LO                                   0x548CD0
+
+#define mmDMA2_QM_GLBL_ERR_ADDR_HI                                   0x548CD4
+
+#define mmDMA2_QM_GLBL_ERR_WDATA                                     0x548CD8
+
+#define mmDMA2_QM_GLBL_MEM_INIT_BUSY                                 0x548D00
+
+#endif /* ASIC_REG_DMA2_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_core_regs.h
new file mode 100644
index 000000000000..fb145f416fe6
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA3_CORE_REGS_H_
+#define ASIC_REG_DMA3_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA3_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA3_CORE_CFG_0                                            0x560000
+
+#define mmDMA3_CORE_CFG_1                                            0x560004
+
+#define mmDMA3_CORE_LBW_MAX_OUTSTAND                                 0x560008
+
+#define mmDMA3_CORE_SRC_BASE_LO                                      0x560014
+
+#define mmDMA3_CORE_SRC_BASE_HI                                      0x560018
+
+#define mmDMA3_CORE_DST_BASE_LO                                      0x56001C
+
+#define mmDMA3_CORE_DST_BASE_HI                                      0x560020
+
+#define mmDMA3_CORE_SRC_TSIZE_1                                      0x56002C
+
+#define mmDMA3_CORE_SRC_STRIDE_1                                     0x560030
+
+#define mmDMA3_CORE_SRC_TSIZE_2                                      0x560034
+
+#define mmDMA3_CORE_SRC_STRIDE_2                                     0x560038
+
+#define mmDMA3_CORE_SRC_TSIZE_3                                      0x56003C
+
+#define mmDMA3_CORE_SRC_STRIDE_3                                     0x560040
+
+#define mmDMA3_CORE_SRC_TSIZE_4                                      0x560044
+
+#define mmDMA3_CORE_SRC_STRIDE_4                                     0x560048
+
+#define mmDMA3_CORE_SRC_TSIZE_0                                      0x56004C
+
+#define mmDMA3_CORE_DST_TSIZE_1                                      0x560054
+
+#define mmDMA3_CORE_DST_STRIDE_1                                     0x560058
+
+#define mmDMA3_CORE_DST_TSIZE_2                                      0x56005C
+
+#define mmDMA3_CORE_DST_STRIDE_2                                     0x560060
+
+#define mmDMA3_CORE_DST_TSIZE_3                                      0x560064
+
+#define mmDMA3_CORE_DST_STRIDE_3                                     0x560068
+
+#define mmDMA3_CORE_DST_TSIZE_4                                      0x56006C
+
+#define mmDMA3_CORE_DST_STRIDE_4                                     0x560070
+
+#define mmDMA3_CORE_DST_TSIZE_0                                      0x560074
+
+#define mmDMA3_CORE_COMMIT                                           0x560078
+
+#define mmDMA3_CORE_WR_COMP_WDATA                                    0x56007C
+
+#define mmDMA3_CORE_WR_COMP_ADDR_LO                                  0x560080
+
+#define mmDMA3_CORE_WR_COMP_ADDR_HI                                  0x560084
+
+#define mmDMA3_CORE_WR_COMP_AWUSER_31_11                             0x560088
+
+#define mmDMA3_CORE_TE_NUMROWS                                       0x560094
+
+#define mmDMA3_CORE_PROT                                             0x5600B8
+
+#define mmDMA3_CORE_SECURE_PROPS                                     0x5600F0
+
+#define mmDMA3_CORE_NON_SECURE_PROPS                                 0x5600F4
+
+#define mmDMA3_CORE_RD_MAX_OUTSTAND                                  0x560100
+
+#define mmDMA3_CORE_RD_MAX_SIZE                                      0x560104
+
+#define mmDMA3_CORE_RD_ARCACHE                                       0x560108
+
+#define mmDMA3_CORE_RD_ARUSER_31_11                                  0x560110
+
+#define mmDMA3_CORE_RD_INFLIGHTS                                     0x560114
+
+#define mmDMA3_CORE_WR_MAX_OUTSTAND                                  0x560120
+
+#define mmDMA3_CORE_WR_MAX_AWID                                      0x560124
+
+#define mmDMA3_CORE_WR_AWCACHE                                       0x560128
+
+#define mmDMA3_CORE_WR_AWUSER_31_11                                  0x560130
+
+#define mmDMA3_CORE_WR_INFLIGHTS                                     0x560134
+
+#define mmDMA3_CORE_RD_RATE_LIM_CFG_0                                0x560150
+
+#define mmDMA3_CORE_RD_RATE_LIM_CFG_1                                0x560154
+
+#define mmDMA3_CORE_WR_RATE_LIM_CFG_0                                0x560158
+
+#define mmDMA3_CORE_WR_RATE_LIM_CFG_1                                0x56015C
+
+#define mmDMA3_CORE_ERR_CFG                                          0x560160
+
+#define mmDMA3_CORE_ERR_CAUSE                                        0x560164
+
+#define mmDMA3_CORE_ERRMSG_ADDR_LO                                   0x560170
+
+#define mmDMA3_CORE_ERRMSG_ADDR_HI                                   0x560174
+
+#define mmDMA3_CORE_ERRMSG_WDATA                                     0x560178
+
+#define mmDMA3_CORE_STS0                                             0x560190
+
+#define mmDMA3_CORE_STS1                                             0x560194
+
+#define mmDMA3_CORE_RD_DBGMEM_ADD                                    0x560200
+
+#define mmDMA3_CORE_RD_DBGMEM_DATA_WR                                0x560204
+
+#define mmDMA3_CORE_RD_DBGMEM_DATA_RD                                0x560208
+
+#define mmDMA3_CORE_RD_DBGMEM_CTRL                                   0x56020C
+
+#define mmDMA3_CORE_RD_DBGMEM_RC                                     0x560210
+
+#define mmDMA3_CORE_DBG_HBW_AXI_AR_CNT                               0x560220
+
+#define mmDMA3_CORE_DBG_HBW_AXI_AW_CNT                               0x560224
+
+#define mmDMA3_CORE_DBG_LBW_AXI_AW_CNT                               0x560228
+
+#define mmDMA3_CORE_DBG_DESC_CNT                                     0x56022C
+
+#define mmDMA3_CORE_DBG_STS                                          0x560230
+
+#define mmDMA3_CORE_DBG_RD_DESC_ID                                   0x560234
+
+#define mmDMA3_CORE_DBG_WR_DESC_ID                                   0x560238
+
+#endif /* ASIC_REG_DMA3_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_qm_regs.h
new file mode 100644
index 000000000000..a4b461ca3d94
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma3_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA3_QM_REGS_H_
+#define ASIC_REG_DMA3_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA3_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA3_QM_GLBL_CFG0                                          0x568000
+
+#define mmDMA3_QM_GLBL_CFG1                                          0x568004
+
+#define mmDMA3_QM_GLBL_PROT                                          0x568008
+
+#define mmDMA3_QM_GLBL_ERR_CFG                                       0x56800C
+
+#define mmDMA3_QM_GLBL_SECURE_PROPS_0                                0x568010
+
+#define mmDMA3_QM_GLBL_SECURE_PROPS_1                                0x568014
+
+#define mmDMA3_QM_GLBL_SECURE_PROPS_2                                0x568018
+
+#define mmDMA3_QM_GLBL_SECURE_PROPS_3                                0x56801C
+
+#define mmDMA3_QM_GLBL_SECURE_PROPS_4                                0x568020
+
+#define mmDMA3_QM_GLBL_NON_SECURE_PROPS_0                            0x568024
+
+#define mmDMA3_QM_GLBL_NON_SECURE_PROPS_1                            0x568028
+
+#define mmDMA3_QM_GLBL_NON_SECURE_PROPS_2                            0x56802C
+
+#define mmDMA3_QM_GLBL_NON_SECURE_PROPS_3                            0x568030
+
+#define mmDMA3_QM_GLBL_NON_SECURE_PROPS_4                            0x568034
+
+#define mmDMA3_QM_GLBL_STS0                                          0x568038
+
+#define mmDMA3_QM_GLBL_STS1_0                                        0x568040
+
+#define mmDMA3_QM_GLBL_STS1_1                                        0x568044
+
+#define mmDMA3_QM_GLBL_STS1_2                                        0x568048
+
+#define mmDMA3_QM_GLBL_STS1_3                                        0x56804C
+
+#define mmDMA3_QM_GLBL_STS1_4                                        0x568050
+
+#define mmDMA3_QM_GLBL_MSG_EN_0                                      0x568054
+
+#define mmDMA3_QM_GLBL_MSG_EN_1                                      0x568058
+
+#define mmDMA3_QM_GLBL_MSG_EN_2                                      0x56805C
+
+#define mmDMA3_QM_GLBL_MSG_EN_3                                      0x568060
+
+#define mmDMA3_QM_GLBL_MSG_EN_4                                      0x568068
+
+#define mmDMA3_QM_PQ_BASE_LO_0                                       0x568070
+
+#define mmDMA3_QM_PQ_BASE_LO_1                                       0x568074
+
+#define mmDMA3_QM_PQ_BASE_LO_2                                       0x568078
+
+#define mmDMA3_QM_PQ_BASE_LO_3                                       0x56807C
+
+#define mmDMA3_QM_PQ_BASE_HI_0                                       0x568080
+
+#define mmDMA3_QM_PQ_BASE_HI_1                                       0x568084
+
+#define mmDMA3_QM_PQ_BASE_HI_2                                       0x568088
+
+#define mmDMA3_QM_PQ_BASE_HI_3                                       0x56808C
+
+#define mmDMA3_QM_PQ_SIZE_0                                          0x568090
+
+#define mmDMA3_QM_PQ_SIZE_1                                          0x568094
+
+#define mmDMA3_QM_PQ_SIZE_2                                          0x568098
+
+#define mmDMA3_QM_PQ_SIZE_3                                          0x56809C
+
+#define mmDMA3_QM_PQ_PI_0                                            0x5680A0
+
+#define mmDMA3_QM_PQ_PI_1                                            0x5680A4
+
+#define mmDMA3_QM_PQ_PI_2                                            0x5680A8
+
+#define mmDMA3_QM_PQ_PI_3                                            0x5680AC
+
+#define mmDMA3_QM_PQ_CI_0                                            0x5680B0
+
+#define mmDMA3_QM_PQ_CI_1                                            0x5680B4
+
+#define mmDMA3_QM_PQ_CI_2                                            0x5680B8
+
+#define mmDMA3_QM_PQ_CI_3                                            0x5680BC
+
+#define mmDMA3_QM_PQ_CFG0_0                                          0x5680C0
+
+#define mmDMA3_QM_PQ_CFG0_1                                          0x5680C4
+
+#define mmDMA3_QM_PQ_CFG0_2                                          0x5680C8
+
+#define mmDMA3_QM_PQ_CFG0_3                                          0x5680CC
+
+#define mmDMA3_QM_PQ_CFG1_0                                          0x5680D0
+
+#define mmDMA3_QM_PQ_CFG1_1                                          0x5680D4
+
+#define mmDMA3_QM_PQ_CFG1_2                                          0x5680D8
+
+#define mmDMA3_QM_PQ_CFG1_3                                          0x5680DC
+
+#define mmDMA3_QM_PQ_ARUSER_31_11_0                                  0x5680E0
+
+#define mmDMA3_QM_PQ_ARUSER_31_11_1                                  0x5680E4
+
+#define mmDMA3_QM_PQ_ARUSER_31_11_2                                  0x5680E8
+
+#define mmDMA3_QM_PQ_ARUSER_31_11_3                                  0x5680EC
+
+#define mmDMA3_QM_PQ_STS0_0                                          0x5680F0
+
+#define mmDMA3_QM_PQ_STS0_1                                          0x5680F4
+
+#define mmDMA3_QM_PQ_STS0_2                                          0x5680F8
+
+#define mmDMA3_QM_PQ_STS0_3                                          0x5680FC
+
+#define mmDMA3_QM_PQ_STS1_0                                          0x568100
+
+#define mmDMA3_QM_PQ_STS1_1                                          0x568104
+
+#define mmDMA3_QM_PQ_STS1_2                                          0x568108
+
+#define mmDMA3_QM_PQ_STS1_3                                          0x56810C
+
+#define mmDMA3_QM_CQ_CFG0_0                                          0x568110
+
+#define mmDMA3_QM_CQ_CFG0_1                                          0x568114
+
+#define mmDMA3_QM_CQ_CFG0_2                                          0x568118
+
+#define mmDMA3_QM_CQ_CFG0_3                                          0x56811C
+
+#define mmDMA3_QM_CQ_CFG0_4                                          0x568120
+
+#define mmDMA3_QM_CQ_CFG1_0                                          0x568124
+
+#define mmDMA3_QM_CQ_CFG1_1                                          0x568128
+
+#define mmDMA3_QM_CQ_CFG1_2                                          0x56812C
+
+#define mmDMA3_QM_CQ_CFG1_3                                          0x568130
+
+#define mmDMA3_QM_CQ_CFG1_4                                          0x568134
+
+#define mmDMA3_QM_CQ_ARUSER_31_11_0                                  0x568138
+
+#define mmDMA3_QM_CQ_ARUSER_31_11_1                                  0x56813C
+
+#define mmDMA3_QM_CQ_ARUSER_31_11_2                                  0x568140
+
+#define mmDMA3_QM_CQ_ARUSER_31_11_3                                  0x568144
+
+#define mmDMA3_QM_CQ_ARUSER_31_11_4                                  0x568148
+
+#define mmDMA3_QM_CQ_STS0_0                                          0x56814C
+
+#define mmDMA3_QM_CQ_STS0_1                                          0x568150
+
+#define mmDMA3_QM_CQ_STS0_2                                          0x568154
+
+#define mmDMA3_QM_CQ_STS0_3                                          0x568158
+
+#define mmDMA3_QM_CQ_STS0_4                                          0x56815C
+
+#define mmDMA3_QM_CQ_STS1_0                                          0x568160
+
+#define mmDMA3_QM_CQ_STS1_1                                          0x568164
+
+#define mmDMA3_QM_CQ_STS1_2                                          0x568168
+
+#define mmDMA3_QM_CQ_STS1_3                                          0x56816C
+
+#define mmDMA3_QM_CQ_STS1_4                                          0x568170
+
+#define mmDMA3_QM_CQ_PTR_LO_0                                        0x568174
+
+#define mmDMA3_QM_CQ_PTR_HI_0                                        0x568178
+
+#define mmDMA3_QM_CQ_TSIZE_0                                         0x56817C
+
+#define mmDMA3_QM_CQ_CTL_0                                           0x568180
+
+#define mmDMA3_QM_CQ_PTR_LO_1                                        0x568184
+
+#define mmDMA3_QM_CQ_PTR_HI_1                                        0x568188
+
+#define mmDMA3_QM_CQ_TSIZE_1                                         0x56818C
+
+#define mmDMA3_QM_CQ_CTL_1                                           0x568190
+
+#define mmDMA3_QM_CQ_PTR_LO_2                                        0x568194
+
+#define mmDMA3_QM_CQ_PTR_HI_2                                        0x568198
+
+#define mmDMA3_QM_CQ_TSIZE_2                                         0x56819C
+
+#define mmDMA3_QM_CQ_CTL_2                                           0x5681A0
+
+#define mmDMA3_QM_CQ_PTR_LO_3                                        0x5681A4
+
+#define mmDMA3_QM_CQ_PTR_HI_3                                        0x5681A8
+
+#define mmDMA3_QM_CQ_TSIZE_3                                         0x5681AC
+
+#define mmDMA3_QM_CQ_CTL_3                                           0x5681B0
+
+#define mmDMA3_QM_CQ_PTR_LO_4                                        0x5681B4
+
+#define mmDMA3_QM_CQ_PTR_HI_4                                        0x5681B8
+
+#define mmDMA3_QM_CQ_TSIZE_4                                         0x5681BC
+
+#define mmDMA3_QM_CQ_CTL_4                                           0x5681C0
+
+#define mmDMA3_QM_CQ_PTR_LO_STS_0                                    0x5681C4
+
+#define mmDMA3_QM_CQ_PTR_LO_STS_1                                    0x5681C8
+
+#define mmDMA3_QM_CQ_PTR_LO_STS_2                                    0x5681CC
+
+#define mmDMA3_QM_CQ_PTR_LO_STS_3                                    0x5681D0
+
+#define mmDMA3_QM_CQ_PTR_LO_STS_4                                    0x5681D4
+
+#define mmDMA3_QM_CQ_PTR_HI_STS_0                                    0x5681D8
+
+#define mmDMA3_QM_CQ_PTR_HI_STS_1                                    0x5681DC
+
+#define mmDMA3_QM_CQ_PTR_HI_STS_2                                    0x5681E0
+
+#define mmDMA3_QM_CQ_PTR_HI_STS_3                                    0x5681E4
+
+#define mmDMA3_QM_CQ_PTR_HI_STS_4                                    0x5681E8
+
+#define mmDMA3_QM_CQ_TSIZE_STS_0                                     0x5681EC
+
+#define mmDMA3_QM_CQ_TSIZE_STS_1                                     0x5681F0
+
+#define mmDMA3_QM_CQ_TSIZE_STS_2                                     0x5681F4
+
+#define mmDMA3_QM_CQ_TSIZE_STS_3                                     0x5681F8
+
+#define mmDMA3_QM_CQ_TSIZE_STS_4                                     0x5681FC
+
+#define mmDMA3_QM_CQ_CTL_STS_0                                       0x568200
+
+#define mmDMA3_QM_CQ_CTL_STS_1                                       0x568204
+
+#define mmDMA3_QM_CQ_CTL_STS_2                                       0x568208
+
+#define mmDMA3_QM_CQ_CTL_STS_3                                       0x56820C
+
+#define mmDMA3_QM_CQ_CTL_STS_4                                       0x568210
+
+#define mmDMA3_QM_CQ_IFIFO_CNT_0                                     0x568214
+
+#define mmDMA3_QM_CQ_IFIFO_CNT_1                                     0x568218
+
+#define mmDMA3_QM_CQ_IFIFO_CNT_2                                     0x56821C
+
+#define mmDMA3_QM_CQ_IFIFO_CNT_3                                     0x568220
+
+#define mmDMA3_QM_CQ_IFIFO_CNT_4                                     0x568224
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_0                             0x568228
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_1                             0x56822C
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_2                             0x568230
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_3                             0x568234
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_LO_4                             0x568238
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_0                             0x56823C
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_1                             0x568240
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_2                             0x568244
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_3                             0x568248
+
+#define mmDMA3_QM_CP_MSG_BASE0_ADDR_HI_4                             0x56824C
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_0                             0x568250
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_1                             0x568254
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_2                             0x568258
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_3                             0x56825C
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_LO_4                             0x568260
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_0                             0x568264
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_1                             0x568268
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_2                             0x56826C
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_3                             0x568270
+
+#define mmDMA3_QM_CP_MSG_BASE1_ADDR_HI_4                             0x568274
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_0                             0x568278
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_1                             0x56827C
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_2                             0x568280
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_3                             0x568284
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_LO_4                             0x568288
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_0                             0x56828C
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_1                             0x568290
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_2                             0x568294
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_3                             0x568298
+
+#define mmDMA3_QM_CP_MSG_BASE2_ADDR_HI_4                             0x56829C
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5682A0
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5682A4
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5682A8
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5682AC
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5682B0
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5682B4
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5682B8
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5682BC
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5682C0
+
+#define mmDMA3_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5682C4
+
+#define mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5682C8
+
+#define mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5682CC
+
+#define mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5682D0
+
+#define mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5682D4
+
+#define mmDMA3_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5682D8
+
+#define mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5682E0
+
+#define mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5682E4
+
+#define mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5682E8
+
+#define mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5682EC
+
+#define mmDMA3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5682F0
+
+#define mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5682F4
+
+#define mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5682F8
+
+#define mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5682FC
+
+#define mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x568300
+
+#define mmDMA3_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x568304
+
+#define mmDMA3_QM_CP_FENCE0_RDATA_0                                  0x568308
+
+#define mmDMA3_QM_CP_FENCE0_RDATA_1                                  0x56830C
+
+#define mmDMA3_QM_CP_FENCE0_RDATA_2                                  0x568310
+
+#define mmDMA3_QM_CP_FENCE0_RDATA_3                                  0x568314
+
+#define mmDMA3_QM_CP_FENCE0_RDATA_4                                  0x568318
+
+#define mmDMA3_QM_CP_FENCE1_RDATA_0                                  0x56831C
+
+#define mmDMA3_QM_CP_FENCE1_RDATA_1                                  0x568320
+
+#define mmDMA3_QM_CP_FENCE1_RDATA_2                                  0x568324
+
+#define mmDMA3_QM_CP_FENCE1_RDATA_3                                  0x568328
+
+#define mmDMA3_QM_CP_FENCE1_RDATA_4                                  0x56832C
+
+#define mmDMA3_QM_CP_FENCE2_RDATA_0                                  0x568330
+
+#define mmDMA3_QM_CP_FENCE2_RDATA_1                                  0x568334
+
+#define mmDMA3_QM_CP_FENCE2_RDATA_2                                  0x568338
+
+#define mmDMA3_QM_CP_FENCE2_RDATA_3                                  0x56833C
+
+#define mmDMA3_QM_CP_FENCE2_RDATA_4                                  0x568340
+
+#define mmDMA3_QM_CP_FENCE3_RDATA_0                                  0x568344
+
+#define mmDMA3_QM_CP_FENCE3_RDATA_1                                  0x568348
+
+#define mmDMA3_QM_CP_FENCE3_RDATA_2                                  0x56834C
+
+#define mmDMA3_QM_CP_FENCE3_RDATA_3                                  0x568350
+
+#define mmDMA3_QM_CP_FENCE3_RDATA_4                                  0x568354
+
+#define mmDMA3_QM_CP_FENCE0_CNT_0                                    0x568358
+
+#define mmDMA3_QM_CP_FENCE0_CNT_1                                    0x56835C
+
+#define mmDMA3_QM_CP_FENCE0_CNT_2                                    0x568360
+
+#define mmDMA3_QM_CP_FENCE0_CNT_3                                    0x568364
+
+#define mmDMA3_QM_CP_FENCE0_CNT_4                                    0x568368
+
+#define mmDMA3_QM_CP_FENCE1_CNT_0                                    0x56836C
+
+#define mmDMA3_QM_CP_FENCE1_CNT_1                                    0x568370
+
+#define mmDMA3_QM_CP_FENCE1_CNT_2                                    0x568374
+
+#define mmDMA3_QM_CP_FENCE1_CNT_3                                    0x568378
+
+#define mmDMA3_QM_CP_FENCE1_CNT_4                                    0x56837C
+
+#define mmDMA3_QM_CP_FENCE2_CNT_0                                    0x568380
+
+#define mmDMA3_QM_CP_FENCE2_CNT_1                                    0x568384
+
+#define mmDMA3_QM_CP_FENCE2_CNT_2                                    0x568388
+
+#define mmDMA3_QM_CP_FENCE2_CNT_3                                    0x56838C
+
+#define mmDMA3_QM_CP_FENCE2_CNT_4                                    0x568390
+
+#define mmDMA3_QM_CP_FENCE3_CNT_0                                    0x568394
+
+#define mmDMA3_QM_CP_FENCE3_CNT_1                                    0x568398
+
+#define mmDMA3_QM_CP_FENCE3_CNT_2                                    0x56839C
+
+#define mmDMA3_QM_CP_FENCE3_CNT_3                                    0x5683A0
+
+#define mmDMA3_QM_CP_FENCE3_CNT_4                                    0x5683A4
+
+#define mmDMA3_QM_CP_STS_0                                           0x5683A8
+
+#define mmDMA3_QM_CP_STS_1                                           0x5683AC
+
+#define mmDMA3_QM_CP_STS_2                                           0x5683B0
+
+#define mmDMA3_QM_CP_STS_3                                           0x5683B4
+
+#define mmDMA3_QM_CP_STS_4                                           0x5683B8
+
+#define mmDMA3_QM_CP_CURRENT_INST_LO_0                               0x5683BC
+
+#define mmDMA3_QM_CP_CURRENT_INST_LO_1                               0x5683C0
+
+#define mmDMA3_QM_CP_CURRENT_INST_LO_2                               0x5683C4
+
+#define mmDMA3_QM_CP_CURRENT_INST_LO_3                               0x5683C8
+
+#define mmDMA3_QM_CP_CURRENT_INST_LO_4                               0x5683CC
+
+#define mmDMA3_QM_CP_CURRENT_INST_HI_0                               0x5683D0
+
+#define mmDMA3_QM_CP_CURRENT_INST_HI_1                               0x5683D4
+
+#define mmDMA3_QM_CP_CURRENT_INST_HI_2                               0x5683D8
+
+#define mmDMA3_QM_CP_CURRENT_INST_HI_3                               0x5683DC
+
+#define mmDMA3_QM_CP_CURRENT_INST_HI_4                               0x5683E0
+
+#define mmDMA3_QM_CP_BARRIER_CFG_0                                   0x5683F4
+
+#define mmDMA3_QM_CP_BARRIER_CFG_1                                   0x5683F8
+
+#define mmDMA3_QM_CP_BARRIER_CFG_2                                   0x5683FC
+
+#define mmDMA3_QM_CP_BARRIER_CFG_3                                   0x568400
+
+#define mmDMA3_QM_CP_BARRIER_CFG_4                                   0x568404
+
+#define mmDMA3_QM_CP_DBG_0_0                                         0x568408
+
+#define mmDMA3_QM_CP_DBG_0_1                                         0x56840C
+
+#define mmDMA3_QM_CP_DBG_0_2                                         0x568410
+
+#define mmDMA3_QM_CP_DBG_0_3                                         0x568414
+
+#define mmDMA3_QM_CP_DBG_0_4                                         0x568418
+
+#define mmDMA3_QM_CP_ARUSER_31_11_0                                  0x56841C
+
+#define mmDMA3_QM_CP_ARUSER_31_11_1                                  0x568420
+
+#define mmDMA3_QM_CP_ARUSER_31_11_2                                  0x568424
+
+#define mmDMA3_QM_CP_ARUSER_31_11_3                                  0x568428
+
+#define mmDMA3_QM_CP_ARUSER_31_11_4                                  0x56842C
+
+#define mmDMA3_QM_CP_AWUSER_31_11_0                                  0x568430
+
+#define mmDMA3_QM_CP_AWUSER_31_11_1                                  0x568434
+
+#define mmDMA3_QM_CP_AWUSER_31_11_2                                  0x568438
+
+#define mmDMA3_QM_CP_AWUSER_31_11_3                                  0x56843C
+
+#define mmDMA3_QM_CP_AWUSER_31_11_4                                  0x568440
+
+#define mmDMA3_QM_ARB_CFG_0                                          0x568A00
+
+#define mmDMA3_QM_ARB_CHOISE_Q_PUSH                                  0x568A04
+
+#define mmDMA3_QM_ARB_WRR_WEIGHT_0                                   0x568A08
+
+#define mmDMA3_QM_ARB_WRR_WEIGHT_1                                   0x568A0C
+
+#define mmDMA3_QM_ARB_WRR_WEIGHT_2                                   0x568A10
+
+#define mmDMA3_QM_ARB_WRR_WEIGHT_3                                   0x568A14
+
+#define mmDMA3_QM_ARB_CFG_1                                          0x568A18
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_0                               0x568A20
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_1                               0x568A24
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_2                               0x568A28
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_3                               0x568A2C
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_4                               0x568A30
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_5                               0x568A34
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_6                               0x568A38
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_7                               0x568A3C
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_8                               0x568A40
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_9                               0x568A44
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_10                              0x568A48
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_11                              0x568A4C
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_12                              0x568A50
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_13                              0x568A54
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_14                              0x568A58
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_15                              0x568A5C
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_16                              0x568A60
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_17                              0x568A64
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_18                              0x568A68
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_19                              0x568A6C
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_20                              0x568A70
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_21                              0x568A74
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_22                              0x568A78
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_23                              0x568A7C
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_24                              0x568A80
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_25                              0x568A84
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_26                              0x568A88
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_27                              0x568A8C
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_28                              0x568A90
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_29                              0x568A94
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_30                              0x568A98
+
+#define mmDMA3_QM_ARB_MST_AVAIL_CRED_31                              0x568A9C
+
+#define mmDMA3_QM_ARB_MST_CRED_INC                                   0x568AA0
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x568AA4
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x568AA8
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x568AAC
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x568AB0
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x568AB4
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x568AB8
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x568ABC
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x568AC0
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x568AC4
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x568AC8
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x568ACC
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x568AD0
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x568AD4
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x568AD8
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x568ADC
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x568AE0
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x568AE4
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x568AE8
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x568AEC
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x568AF0
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x568AF4
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x568AF8
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x568AFC
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x568B00
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x568B04
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x568B08
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x568B0C
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x568B10
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x568B14
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x568B18
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x568B1C
+
+#define mmDMA3_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x568B20
+
+#define mmDMA3_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x568B28
+
+#define mmDMA3_QM_ARB_MST_SLAVE_EN                                   0x568B2C
+
+#define mmDMA3_QM_ARB_MST_QUIET_PER                                  0x568B34
+
+#define mmDMA3_QM_ARB_SLV_CHOISE_WDT                                 0x568B38
+
+#define mmDMA3_QM_ARB_SLV_ID                                         0x568B3C
+
+#define mmDMA3_QM_ARB_MSG_MAX_INFLIGHT                               0x568B44
+
+#define mmDMA3_QM_ARB_MSG_AWUSER_31_11                               0x568B48
+
+#define mmDMA3_QM_ARB_MSG_AWUSER_SEC_PROP                            0x568B4C
+
+#define mmDMA3_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x568B50
+
+#define mmDMA3_QM_ARB_BASE_LO                                        0x568B54
+
+#define mmDMA3_QM_ARB_BASE_HI                                        0x568B58
+
+#define mmDMA3_QM_ARB_STATE_STS                                      0x568B80
+
+#define mmDMA3_QM_ARB_CHOISE_FULLNESS_STS                            0x568B84
+
+#define mmDMA3_QM_ARB_MSG_STS                                        0x568B88
+
+#define mmDMA3_QM_ARB_SLV_CHOISE_Q_HEAD                              0x568B8C
+
+#define mmDMA3_QM_ARB_ERR_CAUSE                                      0x568B9C
+
+#define mmDMA3_QM_ARB_ERR_MSG_EN                                     0x568BA0
+
+#define mmDMA3_QM_ARB_ERR_STS_DRP                                    0x568BA8
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_0                                 0x568BB0
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_1                                 0x568BB4
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_2                                 0x568BB8
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_3                                 0x568BBC
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_4                                 0x568BC0
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_5                                 0x568BC4
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_6                                 0x568BC8
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_7                                 0x568BCC
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_8                                 0x568BD0
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_9                                 0x568BD4
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_10                                0x568BD8
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_11                                0x568BDC
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_12                                0x568BE0
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_13                                0x568BE4
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_14                                0x568BE8
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_15                                0x568BEC
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_16                                0x568BF0
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_17                                0x568BF4
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_18                                0x568BF8
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_19                                0x568BFC
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_20                                0x568C00
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_21                                0x568C04
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_22                                0x568C08
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_23                                0x568C0C
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_24                                0x568C10
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_25                                0x568C14
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_26                                0x568C18
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_27                                0x568C1C
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_28                                0x568C20
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_29                                0x568C24
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_30                                0x568C28
+
+#define mmDMA3_QM_ARB_MST_CRED_STS_31                                0x568C2C
+
+#define mmDMA3_QM_CGM_CFG                                            0x568C70
+
+#define mmDMA3_QM_CGM_STS                                            0x568C74
+
+#define mmDMA3_QM_CGM_CFG1                                           0x568C78
+
+#define mmDMA3_QM_LOCAL_RANGE_BASE                                   0x568C80
+
+#define mmDMA3_QM_LOCAL_RANGE_SIZE                                   0x568C84
+
+#define mmDMA3_QM_CSMR_STRICT_PRIO_CFG                               0x568C90
+
+#define mmDMA3_QM_HBW_RD_RATE_LIM_CFG_1                              0x568C94
+
+#define mmDMA3_QM_LBW_WR_RATE_LIM_CFG_0                              0x568C98
+
+#define mmDMA3_QM_LBW_WR_RATE_LIM_CFG_1                              0x568C9C
+
+#define mmDMA3_QM_HBW_RD_RATE_LIM_CFG_0                              0x568CA0
+
+#define mmDMA3_QM_GLBL_AXCACHE                                       0x568CA4
+
+#define mmDMA3_QM_IND_GW_APB_CFG                                     0x568CB0
+
+#define mmDMA3_QM_IND_GW_APB_WDATA                                   0x568CB4
+
+#define mmDMA3_QM_IND_GW_APB_RDATA                                   0x568CB8
+
+#define mmDMA3_QM_IND_GW_APB_STATUS                                  0x568CBC
+
+#define mmDMA3_QM_GLBL_ERR_ADDR_LO                                   0x568CD0
+
+#define mmDMA3_QM_GLBL_ERR_ADDR_HI                                   0x568CD4
+
+#define mmDMA3_QM_GLBL_ERR_WDATA                                     0x568CD8
+
+#define mmDMA3_QM_GLBL_MEM_INIT_BUSY                                 0x568D00
+
+#endif /* ASIC_REG_DMA3_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_core_regs.h
new file mode 100644
index 000000000000..192d11404b1c
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA4_CORE_REGS_H_
+#define ASIC_REG_DMA4_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA4_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA4_CORE_CFG_0                                            0x580000
+
+#define mmDMA4_CORE_CFG_1                                            0x580004
+
+#define mmDMA4_CORE_LBW_MAX_OUTSTAND                                 0x580008
+
+#define mmDMA4_CORE_SRC_BASE_LO                                      0x580014
+
+#define mmDMA4_CORE_SRC_BASE_HI                                      0x580018
+
+#define mmDMA4_CORE_DST_BASE_LO                                      0x58001C
+
+#define mmDMA4_CORE_DST_BASE_HI                                      0x580020
+
+#define mmDMA4_CORE_SRC_TSIZE_1                                      0x58002C
+
+#define mmDMA4_CORE_SRC_STRIDE_1                                     0x580030
+
+#define mmDMA4_CORE_SRC_TSIZE_2                                      0x580034
+
+#define mmDMA4_CORE_SRC_STRIDE_2                                     0x580038
+
+#define mmDMA4_CORE_SRC_TSIZE_3                                      0x58003C
+
+#define mmDMA4_CORE_SRC_STRIDE_3                                     0x580040
+
+#define mmDMA4_CORE_SRC_TSIZE_4                                      0x580044
+
+#define mmDMA4_CORE_SRC_STRIDE_4                                     0x580048
+
+#define mmDMA4_CORE_SRC_TSIZE_0                                      0x58004C
+
+#define mmDMA4_CORE_DST_TSIZE_1                                      0x580054
+
+#define mmDMA4_CORE_DST_STRIDE_1                                     0x580058
+
+#define mmDMA4_CORE_DST_TSIZE_2                                      0x58005C
+
+#define mmDMA4_CORE_DST_STRIDE_2                                     0x580060
+
+#define mmDMA4_CORE_DST_TSIZE_3                                      0x580064
+
+#define mmDMA4_CORE_DST_STRIDE_3                                     0x580068
+
+#define mmDMA4_CORE_DST_TSIZE_4                                      0x58006C
+
+#define mmDMA4_CORE_DST_STRIDE_4                                     0x580070
+
+#define mmDMA4_CORE_DST_TSIZE_0                                      0x580074
+
+#define mmDMA4_CORE_COMMIT                                           0x580078
+
+#define mmDMA4_CORE_WR_COMP_WDATA                                    0x58007C
+
+#define mmDMA4_CORE_WR_COMP_ADDR_LO                                  0x580080
+
+#define mmDMA4_CORE_WR_COMP_ADDR_HI                                  0x580084
+
+#define mmDMA4_CORE_WR_COMP_AWUSER_31_11                             0x580088
+
+#define mmDMA4_CORE_TE_NUMROWS                                       0x580094
+
+#define mmDMA4_CORE_PROT                                             0x5800B8
+
+#define mmDMA4_CORE_SECURE_PROPS                                     0x5800F0
+
+#define mmDMA4_CORE_NON_SECURE_PROPS                                 0x5800F4
+
+#define mmDMA4_CORE_RD_MAX_OUTSTAND                                  0x580100
+
+#define mmDMA4_CORE_RD_MAX_SIZE                                      0x580104
+
+#define mmDMA4_CORE_RD_ARCACHE                                       0x580108
+
+#define mmDMA4_CORE_RD_ARUSER_31_11                                  0x580110
+
+#define mmDMA4_CORE_RD_INFLIGHTS                                     0x580114
+
+#define mmDMA4_CORE_WR_MAX_OUTSTAND                                  0x580120
+
+#define mmDMA4_CORE_WR_MAX_AWID                                      0x580124
+
+#define mmDMA4_CORE_WR_AWCACHE                                       0x580128
+
+#define mmDMA4_CORE_WR_AWUSER_31_11                                  0x580130
+
+#define mmDMA4_CORE_WR_INFLIGHTS                                     0x580134
+
+#define mmDMA4_CORE_RD_RATE_LIM_CFG_0                                0x580150
+
+#define mmDMA4_CORE_RD_RATE_LIM_CFG_1                                0x580154
+
+#define mmDMA4_CORE_WR_RATE_LIM_CFG_0                                0x580158
+
+#define mmDMA4_CORE_WR_RATE_LIM_CFG_1                                0x58015C
+
+#define mmDMA4_CORE_ERR_CFG                                          0x580160
+
+#define mmDMA4_CORE_ERR_CAUSE                                        0x580164
+
+#define mmDMA4_CORE_ERRMSG_ADDR_LO                                   0x580170
+
+#define mmDMA4_CORE_ERRMSG_ADDR_HI                                   0x580174
+
+#define mmDMA4_CORE_ERRMSG_WDATA                                     0x580178
+
+#define mmDMA4_CORE_STS0                                             0x580190
+
+#define mmDMA4_CORE_STS1                                             0x580194
+
+#define mmDMA4_CORE_RD_DBGMEM_ADD                                    0x580200
+
+#define mmDMA4_CORE_RD_DBGMEM_DATA_WR                                0x580204
+
+#define mmDMA4_CORE_RD_DBGMEM_DATA_RD                                0x580208
+
+#define mmDMA4_CORE_RD_DBGMEM_CTRL                                   0x58020C
+
+#define mmDMA4_CORE_RD_DBGMEM_RC                                     0x580210
+
+#define mmDMA4_CORE_DBG_HBW_AXI_AR_CNT                               0x580220
+
+#define mmDMA4_CORE_DBG_HBW_AXI_AW_CNT                               0x580224
+
+#define mmDMA4_CORE_DBG_LBW_AXI_AW_CNT                               0x580228
+
+#define mmDMA4_CORE_DBG_DESC_CNT                                     0x58022C
+
+#define mmDMA4_CORE_DBG_STS                                          0x580230
+
+#define mmDMA4_CORE_DBG_RD_DESC_ID                                   0x580234
+
+#define mmDMA4_CORE_DBG_WR_DESC_ID                                   0x580238
+
+#endif /* ASIC_REG_DMA4_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_qm_regs.h
new file mode 100644
index 000000000000..f0cbda0d1e4d
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma4_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA4_QM_REGS_H_
+#define ASIC_REG_DMA4_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA4_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA4_QM_GLBL_CFG0                                          0x588000
+
+#define mmDMA4_QM_GLBL_CFG1                                          0x588004
+
+#define mmDMA4_QM_GLBL_PROT                                          0x588008
+
+#define mmDMA4_QM_GLBL_ERR_CFG                                       0x58800C
+
+#define mmDMA4_QM_GLBL_SECURE_PROPS_0                                0x588010
+
+#define mmDMA4_QM_GLBL_SECURE_PROPS_1                                0x588014
+
+#define mmDMA4_QM_GLBL_SECURE_PROPS_2                                0x588018
+
+#define mmDMA4_QM_GLBL_SECURE_PROPS_3                                0x58801C
+
+#define mmDMA4_QM_GLBL_SECURE_PROPS_4                                0x588020
+
+#define mmDMA4_QM_GLBL_NON_SECURE_PROPS_0                            0x588024
+
+#define mmDMA4_QM_GLBL_NON_SECURE_PROPS_1                            0x588028
+
+#define mmDMA4_QM_GLBL_NON_SECURE_PROPS_2                            0x58802C
+
+#define mmDMA4_QM_GLBL_NON_SECURE_PROPS_3                            0x588030
+
+#define mmDMA4_QM_GLBL_NON_SECURE_PROPS_4                            0x588034
+
+#define mmDMA4_QM_GLBL_STS0                                          0x588038
+
+#define mmDMA4_QM_GLBL_STS1_0                                        0x588040
+
+#define mmDMA4_QM_GLBL_STS1_1                                        0x588044
+
+#define mmDMA4_QM_GLBL_STS1_2                                        0x588048
+
+#define mmDMA4_QM_GLBL_STS1_3                                        0x58804C
+
+#define mmDMA4_QM_GLBL_STS1_4                                        0x588050
+
+#define mmDMA4_QM_GLBL_MSG_EN_0                                      0x588054
+
+#define mmDMA4_QM_GLBL_MSG_EN_1                                      0x588058
+
+#define mmDMA4_QM_GLBL_MSG_EN_2                                      0x58805C
+
+#define mmDMA4_QM_GLBL_MSG_EN_3                                      0x588060
+
+#define mmDMA4_QM_GLBL_MSG_EN_4                                      0x588068
+
+#define mmDMA4_QM_PQ_BASE_LO_0                                       0x588070
+
+#define mmDMA4_QM_PQ_BASE_LO_1                                       0x588074
+
+#define mmDMA4_QM_PQ_BASE_LO_2                                       0x588078
+
+#define mmDMA4_QM_PQ_BASE_LO_3                                       0x58807C
+
+#define mmDMA4_QM_PQ_BASE_HI_0                                       0x588080
+
+#define mmDMA4_QM_PQ_BASE_HI_1                                       0x588084
+
+#define mmDMA4_QM_PQ_BASE_HI_2                                       0x588088
+
+#define mmDMA4_QM_PQ_BASE_HI_3                                       0x58808C
+
+#define mmDMA4_QM_PQ_SIZE_0                                          0x588090
+
+#define mmDMA4_QM_PQ_SIZE_1                                          0x588094
+
+#define mmDMA4_QM_PQ_SIZE_2                                          0x588098
+
+#define mmDMA4_QM_PQ_SIZE_3                                          0x58809C
+
+#define mmDMA4_QM_PQ_PI_0                                            0x5880A0
+
+#define mmDMA4_QM_PQ_PI_1                                            0x5880A4
+
+#define mmDMA4_QM_PQ_PI_2                                            0x5880A8
+
+#define mmDMA4_QM_PQ_PI_3                                            0x5880AC
+
+#define mmDMA4_QM_PQ_CI_0                                            0x5880B0
+
+#define mmDMA4_QM_PQ_CI_1                                            0x5880B4
+
+#define mmDMA4_QM_PQ_CI_2                                            0x5880B8
+
+#define mmDMA4_QM_PQ_CI_3                                            0x5880BC
+
+#define mmDMA4_QM_PQ_CFG0_0                                          0x5880C0
+
+#define mmDMA4_QM_PQ_CFG0_1                                          0x5880C4
+
+#define mmDMA4_QM_PQ_CFG0_2                                          0x5880C8
+
+#define mmDMA4_QM_PQ_CFG0_3                                          0x5880CC
+
+#define mmDMA4_QM_PQ_CFG1_0                                          0x5880D0
+
+#define mmDMA4_QM_PQ_CFG1_1                                          0x5880D4
+
+#define mmDMA4_QM_PQ_CFG1_2                                          0x5880D8
+
+#define mmDMA4_QM_PQ_CFG1_3                                          0x5880DC
+
+#define mmDMA4_QM_PQ_ARUSER_31_11_0                                  0x5880E0
+
+#define mmDMA4_QM_PQ_ARUSER_31_11_1                                  0x5880E4
+
+#define mmDMA4_QM_PQ_ARUSER_31_11_2                                  0x5880E8
+
+#define mmDMA4_QM_PQ_ARUSER_31_11_3                                  0x5880EC
+
+#define mmDMA4_QM_PQ_STS0_0                                          0x5880F0
+
+#define mmDMA4_QM_PQ_STS0_1                                          0x5880F4
+
+#define mmDMA4_QM_PQ_STS0_2                                          0x5880F8
+
+#define mmDMA4_QM_PQ_STS0_3                                          0x5880FC
+
+#define mmDMA4_QM_PQ_STS1_0                                          0x588100
+
+#define mmDMA4_QM_PQ_STS1_1                                          0x588104
+
+#define mmDMA4_QM_PQ_STS1_2                                          0x588108
+
+#define mmDMA4_QM_PQ_STS1_3                                          0x58810C
+
+#define mmDMA4_QM_CQ_CFG0_0                                          0x588110
+
+#define mmDMA4_QM_CQ_CFG0_1                                          0x588114
+
+#define mmDMA4_QM_CQ_CFG0_2                                          0x588118
+
+#define mmDMA4_QM_CQ_CFG0_3                                          0x58811C
+
+#define mmDMA4_QM_CQ_CFG0_4                                          0x588120
+
+#define mmDMA4_QM_CQ_CFG1_0                                          0x588124
+
+#define mmDMA4_QM_CQ_CFG1_1                                          0x588128
+
+#define mmDMA4_QM_CQ_CFG1_2                                          0x58812C
+
+#define mmDMA4_QM_CQ_CFG1_3                                          0x588130
+
+#define mmDMA4_QM_CQ_CFG1_4                                          0x588134
+
+#define mmDMA4_QM_CQ_ARUSER_31_11_0                                  0x588138
+
+#define mmDMA4_QM_CQ_ARUSER_31_11_1                                  0x58813C
+
+#define mmDMA4_QM_CQ_ARUSER_31_11_2                                  0x588140
+
+#define mmDMA4_QM_CQ_ARUSER_31_11_3                                  0x588144
+
+#define mmDMA4_QM_CQ_ARUSER_31_11_4                                  0x588148
+
+#define mmDMA4_QM_CQ_STS0_0                                          0x58814C
+
+#define mmDMA4_QM_CQ_STS0_1                                          0x588150
+
+#define mmDMA4_QM_CQ_STS0_2                                          0x588154
+
+#define mmDMA4_QM_CQ_STS0_3                                          0x588158
+
+#define mmDMA4_QM_CQ_STS0_4                                          0x58815C
+
+#define mmDMA4_QM_CQ_STS1_0                                          0x588160
+
+#define mmDMA4_QM_CQ_STS1_1                                          0x588164
+
+#define mmDMA4_QM_CQ_STS1_2                                          0x588168
+
+#define mmDMA4_QM_CQ_STS1_3                                          0x58816C
+
+#define mmDMA4_QM_CQ_STS1_4                                          0x588170
+
+#define mmDMA4_QM_CQ_PTR_LO_0                                        0x588174
+
+#define mmDMA4_QM_CQ_PTR_HI_0                                        0x588178
+
+#define mmDMA4_QM_CQ_TSIZE_0                                         0x58817C
+
+#define mmDMA4_QM_CQ_CTL_0                                           0x588180
+
+#define mmDMA4_QM_CQ_PTR_LO_1                                        0x588184
+
+#define mmDMA4_QM_CQ_PTR_HI_1                                        0x588188
+
+#define mmDMA4_QM_CQ_TSIZE_1                                         0x58818C
+
+#define mmDMA4_QM_CQ_CTL_1                                           0x588190
+
+#define mmDMA4_QM_CQ_PTR_LO_2                                        0x588194
+
+#define mmDMA4_QM_CQ_PTR_HI_2                                        0x588198
+
+#define mmDMA4_QM_CQ_TSIZE_2                                         0x58819C
+
+#define mmDMA4_QM_CQ_CTL_2                                           0x5881A0
+
+#define mmDMA4_QM_CQ_PTR_LO_3                                        0x5881A4
+
+#define mmDMA4_QM_CQ_PTR_HI_3                                        0x5881A8
+
+#define mmDMA4_QM_CQ_TSIZE_3                                         0x5881AC
+
+#define mmDMA4_QM_CQ_CTL_3                                           0x5881B0
+
+#define mmDMA4_QM_CQ_PTR_LO_4                                        0x5881B4
+
+#define mmDMA4_QM_CQ_PTR_HI_4                                        0x5881B8
+
+#define mmDMA4_QM_CQ_TSIZE_4                                         0x5881BC
+
+#define mmDMA4_QM_CQ_CTL_4                                           0x5881C0
+
+#define mmDMA4_QM_CQ_PTR_LO_STS_0                                    0x5881C4
+
+#define mmDMA4_QM_CQ_PTR_LO_STS_1                                    0x5881C8
+
+#define mmDMA4_QM_CQ_PTR_LO_STS_2                                    0x5881CC
+
+#define mmDMA4_QM_CQ_PTR_LO_STS_3                                    0x5881D0
+
+#define mmDMA4_QM_CQ_PTR_LO_STS_4                                    0x5881D4
+
+#define mmDMA4_QM_CQ_PTR_HI_STS_0                                    0x5881D8
+
+#define mmDMA4_QM_CQ_PTR_HI_STS_1                                    0x5881DC
+
+#define mmDMA4_QM_CQ_PTR_HI_STS_2                                    0x5881E0
+
+#define mmDMA4_QM_CQ_PTR_HI_STS_3                                    0x5881E4
+
+#define mmDMA4_QM_CQ_PTR_HI_STS_4                                    0x5881E8
+
+#define mmDMA4_QM_CQ_TSIZE_STS_0                                     0x5881EC
+
+#define mmDMA4_QM_CQ_TSIZE_STS_1                                     0x5881F0
+
+#define mmDMA4_QM_CQ_TSIZE_STS_2                                     0x5881F4
+
+#define mmDMA4_QM_CQ_TSIZE_STS_3                                     0x5881F8
+
+#define mmDMA4_QM_CQ_TSIZE_STS_4                                     0x5881FC
+
+#define mmDMA4_QM_CQ_CTL_STS_0                                       0x588200
+
+#define mmDMA4_QM_CQ_CTL_STS_1                                       0x588204
+
+#define mmDMA4_QM_CQ_CTL_STS_2                                       0x588208
+
+#define mmDMA4_QM_CQ_CTL_STS_3                                       0x58820C
+
+#define mmDMA4_QM_CQ_CTL_STS_4                                       0x588210
+
+#define mmDMA4_QM_CQ_IFIFO_CNT_0                                     0x588214
+
+#define mmDMA4_QM_CQ_IFIFO_CNT_1                                     0x588218
+
+#define mmDMA4_QM_CQ_IFIFO_CNT_2                                     0x58821C
+
+#define mmDMA4_QM_CQ_IFIFO_CNT_3                                     0x588220
+
+#define mmDMA4_QM_CQ_IFIFO_CNT_4                                     0x588224
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_0                             0x588228
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_1                             0x58822C
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_2                             0x588230
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_3                             0x588234
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_LO_4                             0x588238
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_0                             0x58823C
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_1                             0x588240
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_2                             0x588244
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_3                             0x588248
+
+#define mmDMA4_QM_CP_MSG_BASE0_ADDR_HI_4                             0x58824C
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_0                             0x588250
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_1                             0x588254
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_2                             0x588258
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_3                             0x58825C
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_LO_4                             0x588260
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_0                             0x588264
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_1                             0x588268
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_2                             0x58826C
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_3                             0x588270
+
+#define mmDMA4_QM_CP_MSG_BASE1_ADDR_HI_4                             0x588274
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_0                             0x588278
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_1                             0x58827C
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_2                             0x588280
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_3                             0x588284
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_LO_4                             0x588288
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_0                             0x58828C
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_1                             0x588290
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_2                             0x588294
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_3                             0x588298
+
+#define mmDMA4_QM_CP_MSG_BASE2_ADDR_HI_4                             0x58829C
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5882A0
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5882A4
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5882A8
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5882AC
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5882B0
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5882B4
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5882B8
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5882BC
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5882C0
+
+#define mmDMA4_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5882C4
+
+#define mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5882C8
+
+#define mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5882CC
+
+#define mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5882D0
+
+#define mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5882D4
+
+#define mmDMA4_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5882D8
+
+#define mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5882E0
+
+#define mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5882E4
+
+#define mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5882E8
+
+#define mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5882EC
+
+#define mmDMA4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5882F0
+
+#define mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5882F4
+
+#define mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5882F8
+
+#define mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5882FC
+
+#define mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x588300
+
+#define mmDMA4_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x588304
+
+#define mmDMA4_QM_CP_FENCE0_RDATA_0                                  0x588308
+
+#define mmDMA4_QM_CP_FENCE0_RDATA_1                                  0x58830C
+
+#define mmDMA4_QM_CP_FENCE0_RDATA_2                                  0x588310
+
+#define mmDMA4_QM_CP_FENCE0_RDATA_3                                  0x588314
+
+#define mmDMA4_QM_CP_FENCE0_RDATA_4                                  0x588318
+
+#define mmDMA4_QM_CP_FENCE1_RDATA_0                                  0x58831C
+
+#define mmDMA4_QM_CP_FENCE1_RDATA_1                                  0x588320
+
+#define mmDMA4_QM_CP_FENCE1_RDATA_2                                  0x588324
+
+#define mmDMA4_QM_CP_FENCE1_RDATA_3                                  0x588328
+
+#define mmDMA4_QM_CP_FENCE1_RDATA_4                                  0x58832C
+
+#define mmDMA4_QM_CP_FENCE2_RDATA_0                                  0x588330
+
+#define mmDMA4_QM_CP_FENCE2_RDATA_1                                  0x588334
+
+#define mmDMA4_QM_CP_FENCE2_RDATA_2                                  0x588338
+
+#define mmDMA4_QM_CP_FENCE2_RDATA_3                                  0x58833C
+
+#define mmDMA4_QM_CP_FENCE2_RDATA_4                                  0x588340
+
+#define mmDMA4_QM_CP_FENCE3_RDATA_0                                  0x588344
+
+#define mmDMA4_QM_CP_FENCE3_RDATA_1                                  0x588348
+
+#define mmDMA4_QM_CP_FENCE3_RDATA_2                                  0x58834C
+
+#define mmDMA4_QM_CP_FENCE3_RDATA_3                                  0x588350
+
+#define mmDMA4_QM_CP_FENCE3_RDATA_4                                  0x588354
+
+#define mmDMA4_QM_CP_FENCE0_CNT_0                                    0x588358
+
+#define mmDMA4_QM_CP_FENCE0_CNT_1                                    0x58835C
+
+#define mmDMA4_QM_CP_FENCE0_CNT_2                                    0x588360
+
+#define mmDMA4_QM_CP_FENCE0_CNT_3                                    0x588364
+
+#define mmDMA4_QM_CP_FENCE0_CNT_4                                    0x588368
+
+#define mmDMA4_QM_CP_FENCE1_CNT_0                                    0x58836C
+
+#define mmDMA4_QM_CP_FENCE1_CNT_1                                    0x588370
+
+#define mmDMA4_QM_CP_FENCE1_CNT_2                                    0x588374
+
+#define mmDMA4_QM_CP_FENCE1_CNT_3                                    0x588378
+
+#define mmDMA4_QM_CP_FENCE1_CNT_4                                    0x58837C
+
+#define mmDMA4_QM_CP_FENCE2_CNT_0                                    0x588380
+
+#define mmDMA4_QM_CP_FENCE2_CNT_1                                    0x588384
+
+#define mmDMA4_QM_CP_FENCE2_CNT_2                                    0x588388
+
+#define mmDMA4_QM_CP_FENCE2_CNT_3                                    0x58838C
+
+#define mmDMA4_QM_CP_FENCE2_CNT_4                                    0x588390
+
+#define mmDMA4_QM_CP_FENCE3_CNT_0                                    0x588394
+
+#define mmDMA4_QM_CP_FENCE3_CNT_1                                    0x588398
+
+#define mmDMA4_QM_CP_FENCE3_CNT_2                                    0x58839C
+
+#define mmDMA4_QM_CP_FENCE3_CNT_3                                    0x5883A0
+
+#define mmDMA4_QM_CP_FENCE3_CNT_4                                    0x5883A4
+
+#define mmDMA4_QM_CP_STS_0                                           0x5883A8
+
+#define mmDMA4_QM_CP_STS_1                                           0x5883AC
+
+#define mmDMA4_QM_CP_STS_2                                           0x5883B0
+
+#define mmDMA4_QM_CP_STS_3                                           0x5883B4
+
+#define mmDMA4_QM_CP_STS_4                                           0x5883B8
+
+#define mmDMA4_QM_CP_CURRENT_INST_LO_0                               0x5883BC
+
+#define mmDMA4_QM_CP_CURRENT_INST_LO_1                               0x5883C0
+
+#define mmDMA4_QM_CP_CURRENT_INST_LO_2                               0x5883C4
+
+#define mmDMA4_QM_CP_CURRENT_INST_LO_3                               0x5883C8
+
+#define mmDMA4_QM_CP_CURRENT_INST_LO_4                               0x5883CC
+
+#define mmDMA4_QM_CP_CURRENT_INST_HI_0                               0x5883D0
+
+#define mmDMA4_QM_CP_CURRENT_INST_HI_1                               0x5883D4
+
+#define mmDMA4_QM_CP_CURRENT_INST_HI_2                               0x5883D8
+
+#define mmDMA4_QM_CP_CURRENT_INST_HI_3                               0x5883DC
+
+#define mmDMA4_QM_CP_CURRENT_INST_HI_4                               0x5883E0
+
+#define mmDMA4_QM_CP_BARRIER_CFG_0                                   0x5883F4
+
+#define mmDMA4_QM_CP_BARRIER_CFG_1                                   0x5883F8
+
+#define mmDMA4_QM_CP_BARRIER_CFG_2                                   0x5883FC
+
+#define mmDMA4_QM_CP_BARRIER_CFG_3                                   0x588400
+
+#define mmDMA4_QM_CP_BARRIER_CFG_4                                   0x588404
+
+#define mmDMA4_QM_CP_DBG_0_0                                         0x588408
+
+#define mmDMA4_QM_CP_DBG_0_1                                         0x58840C
+
+#define mmDMA4_QM_CP_DBG_0_2                                         0x588410
+
+#define mmDMA4_QM_CP_DBG_0_3                                         0x588414
+
+#define mmDMA4_QM_CP_DBG_0_4                                         0x588418
+
+#define mmDMA4_QM_CP_ARUSER_31_11_0                                  0x58841C
+
+#define mmDMA4_QM_CP_ARUSER_31_11_1                                  0x588420
+
+#define mmDMA4_QM_CP_ARUSER_31_11_2                                  0x588424
+
+#define mmDMA4_QM_CP_ARUSER_31_11_3                                  0x588428
+
+#define mmDMA4_QM_CP_ARUSER_31_11_4                                  0x58842C
+
+#define mmDMA4_QM_CP_AWUSER_31_11_0                                  0x588430
+
+#define mmDMA4_QM_CP_AWUSER_31_11_1                                  0x588434
+
+#define mmDMA4_QM_CP_AWUSER_31_11_2                                  0x588438
+
+#define mmDMA4_QM_CP_AWUSER_31_11_3                                  0x58843C
+
+#define mmDMA4_QM_CP_AWUSER_31_11_4                                  0x588440
+
+#define mmDMA4_QM_ARB_CFG_0                                          0x588A00
+
+#define mmDMA4_QM_ARB_CHOISE_Q_PUSH                                  0x588A04
+
+#define mmDMA4_QM_ARB_WRR_WEIGHT_0                                   0x588A08
+
+#define mmDMA4_QM_ARB_WRR_WEIGHT_1                                   0x588A0C
+
+#define mmDMA4_QM_ARB_WRR_WEIGHT_2                                   0x588A10
+
+#define mmDMA4_QM_ARB_WRR_WEIGHT_3                                   0x588A14
+
+#define mmDMA4_QM_ARB_CFG_1                                          0x588A18
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_0                               0x588A20
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_1                               0x588A24
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_2                               0x588A28
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_3                               0x588A2C
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_4                               0x588A30
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_5                               0x588A34
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_6                               0x588A38
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_7                               0x588A3C
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_8                               0x588A40
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_9                               0x588A44
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_10                              0x588A48
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_11                              0x588A4C
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_12                              0x588A50
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_13                              0x588A54
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_14                              0x588A58
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_15                              0x588A5C
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_16                              0x588A60
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_17                              0x588A64
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_18                              0x588A68
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_19                              0x588A6C
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_20                              0x588A70
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_21                              0x588A74
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_22                              0x588A78
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_23                              0x588A7C
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_24                              0x588A80
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_25                              0x588A84
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_26                              0x588A88
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_27                              0x588A8C
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_28                              0x588A90
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_29                              0x588A94
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_30                              0x588A98
+
+#define mmDMA4_QM_ARB_MST_AVAIL_CRED_31                              0x588A9C
+
+#define mmDMA4_QM_ARB_MST_CRED_INC                                   0x588AA0
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x588AA4
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x588AA8
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x588AAC
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x588AB0
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x588AB4
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x588AB8
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x588ABC
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x588AC0
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x588AC4
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x588AC8
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x588ACC
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x588AD0
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x588AD4
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x588AD8
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x588ADC
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x588AE0
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x588AE4
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x588AE8
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x588AEC
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x588AF0
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x588AF4
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x588AF8
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x588AFC
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x588B00
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x588B04
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x588B08
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x588B0C
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x588B10
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x588B14
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x588B18
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x588B1C
+
+#define mmDMA4_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x588B20
+
+#define mmDMA4_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x588B28
+
+#define mmDMA4_QM_ARB_MST_SLAVE_EN                                   0x588B2C
+
+#define mmDMA4_QM_ARB_MST_QUIET_PER                                  0x588B34
+
+#define mmDMA4_QM_ARB_SLV_CHOISE_WDT                                 0x588B38
+
+#define mmDMA4_QM_ARB_SLV_ID                                         0x588B3C
+
+#define mmDMA4_QM_ARB_MSG_MAX_INFLIGHT                               0x588B44
+
+#define mmDMA4_QM_ARB_MSG_AWUSER_31_11                               0x588B48
+
+#define mmDMA4_QM_ARB_MSG_AWUSER_SEC_PROP                            0x588B4C
+
+#define mmDMA4_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x588B50
+
+#define mmDMA4_QM_ARB_BASE_LO                                        0x588B54
+
+#define mmDMA4_QM_ARB_BASE_HI                                        0x588B58
+
+#define mmDMA4_QM_ARB_STATE_STS                                      0x588B80
+
+#define mmDMA4_QM_ARB_CHOISE_FULLNESS_STS                            0x588B84
+
+#define mmDMA4_QM_ARB_MSG_STS                                        0x588B88
+
+#define mmDMA4_QM_ARB_SLV_CHOISE_Q_HEAD                              0x588B8C
+
+#define mmDMA4_QM_ARB_ERR_CAUSE                                      0x588B9C
+
+#define mmDMA4_QM_ARB_ERR_MSG_EN                                     0x588BA0
+
+#define mmDMA4_QM_ARB_ERR_STS_DRP                                    0x588BA8
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_0                                 0x588BB0
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_1                                 0x588BB4
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_2                                 0x588BB8
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_3                                 0x588BBC
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_4                                 0x588BC0
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_5                                 0x588BC4
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_6                                 0x588BC8
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_7                                 0x588BCC
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_8                                 0x588BD0
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_9                                 0x588BD4
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_10                                0x588BD8
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_11                                0x588BDC
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_12                                0x588BE0
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_13                                0x588BE4
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_14                                0x588BE8
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_15                                0x588BEC
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_16                                0x588BF0
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_17                                0x588BF4
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_18                                0x588BF8
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_19                                0x588BFC
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_20                                0x588C00
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_21                                0x588C04
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_22                                0x588C08
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_23                                0x588C0C
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_24                                0x588C10
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_25                                0x588C14
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_26                                0x588C18
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_27                                0x588C1C
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_28                                0x588C20
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_29                                0x588C24
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_30                                0x588C28
+
+#define mmDMA4_QM_ARB_MST_CRED_STS_31                                0x588C2C
+
+#define mmDMA4_QM_CGM_CFG                                            0x588C70
+
+#define mmDMA4_QM_CGM_STS                                            0x588C74
+
+#define mmDMA4_QM_CGM_CFG1                                           0x588C78
+
+#define mmDMA4_QM_LOCAL_RANGE_BASE                                   0x588C80
+
+#define mmDMA4_QM_LOCAL_RANGE_SIZE                                   0x588C84
+
+#define mmDMA4_QM_CSMR_STRICT_PRIO_CFG                               0x588C90
+
+#define mmDMA4_QM_HBW_RD_RATE_LIM_CFG_1                              0x588C94
+
+#define mmDMA4_QM_LBW_WR_RATE_LIM_CFG_0                              0x588C98
+
+#define mmDMA4_QM_LBW_WR_RATE_LIM_CFG_1                              0x588C9C
+
+#define mmDMA4_QM_HBW_RD_RATE_LIM_CFG_0                              0x588CA0
+
+#define mmDMA4_QM_GLBL_AXCACHE                                       0x588CA4
+
+#define mmDMA4_QM_IND_GW_APB_CFG                                     0x588CB0
+
+#define mmDMA4_QM_IND_GW_APB_WDATA                                   0x588CB4
+
+#define mmDMA4_QM_IND_GW_APB_RDATA                                   0x588CB8
+
+#define mmDMA4_QM_IND_GW_APB_STATUS                                  0x588CBC
+
+#define mmDMA4_QM_GLBL_ERR_ADDR_LO                                   0x588CD0
+
+#define mmDMA4_QM_GLBL_ERR_ADDR_HI                                   0x588CD4
+
+#define mmDMA4_QM_GLBL_ERR_WDATA                                     0x588CD8
+
+#define mmDMA4_QM_GLBL_MEM_INIT_BUSY                                 0x588D00
+
+#endif /* ASIC_REG_DMA4_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_core_regs.h
new file mode 100644
index 000000000000..6e07c6fb6fc9
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA5_CORE_REGS_H_
+#define ASIC_REG_DMA5_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA5_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA5_CORE_CFG_0                                            0x5A0000
+
+#define mmDMA5_CORE_CFG_1                                            0x5A0004
+
+#define mmDMA5_CORE_LBW_MAX_OUTSTAND                                 0x5A0008
+
+#define mmDMA5_CORE_SRC_BASE_LO                                      0x5A0014
+
+#define mmDMA5_CORE_SRC_BASE_HI                                      0x5A0018
+
+#define mmDMA5_CORE_DST_BASE_LO                                      0x5A001C
+
+#define mmDMA5_CORE_DST_BASE_HI                                      0x5A0020
+
+#define mmDMA5_CORE_SRC_TSIZE_1                                      0x5A002C
+
+#define mmDMA5_CORE_SRC_STRIDE_1                                     0x5A0030
+
+#define mmDMA5_CORE_SRC_TSIZE_2                                      0x5A0034
+
+#define mmDMA5_CORE_SRC_STRIDE_2                                     0x5A0038
+
+#define mmDMA5_CORE_SRC_TSIZE_3                                      0x5A003C
+
+#define mmDMA5_CORE_SRC_STRIDE_3                                     0x5A0040
+
+#define mmDMA5_CORE_SRC_TSIZE_4                                      0x5A0044
+
+#define mmDMA5_CORE_SRC_STRIDE_4                                     0x5A0048
+
+#define mmDMA5_CORE_SRC_TSIZE_0                                      0x5A004C
+
+#define mmDMA5_CORE_DST_TSIZE_1                                      0x5A0054
+
+#define mmDMA5_CORE_DST_STRIDE_1                                     0x5A0058
+
+#define mmDMA5_CORE_DST_TSIZE_2                                      0x5A005C
+
+#define mmDMA5_CORE_DST_STRIDE_2                                     0x5A0060
+
+#define mmDMA5_CORE_DST_TSIZE_3                                      0x5A0064
+
+#define mmDMA5_CORE_DST_STRIDE_3                                     0x5A0068
+
+#define mmDMA5_CORE_DST_TSIZE_4                                      0x5A006C
+
+#define mmDMA5_CORE_DST_STRIDE_4                                     0x5A0070
+
+#define mmDMA5_CORE_DST_TSIZE_0                                      0x5A0074
+
+#define mmDMA5_CORE_COMMIT                                           0x5A0078
+
+#define mmDMA5_CORE_WR_COMP_WDATA                                    0x5A007C
+
+#define mmDMA5_CORE_WR_COMP_ADDR_LO                                  0x5A0080
+
+#define mmDMA5_CORE_WR_COMP_ADDR_HI                                  0x5A0084
+
+#define mmDMA5_CORE_WR_COMP_AWUSER_31_11                             0x5A0088
+
+#define mmDMA5_CORE_TE_NUMROWS                                       0x5A0094
+
+#define mmDMA5_CORE_PROT                                             0x5A00B8
+
+#define mmDMA5_CORE_SECURE_PROPS                                     0x5A00F0
+
+#define mmDMA5_CORE_NON_SECURE_PROPS                                 0x5A00F4
+
+#define mmDMA5_CORE_RD_MAX_OUTSTAND                                  0x5A0100
+
+#define mmDMA5_CORE_RD_MAX_SIZE                                      0x5A0104
+
+#define mmDMA5_CORE_RD_ARCACHE                                       0x5A0108
+
+#define mmDMA5_CORE_RD_ARUSER_31_11                                  0x5A0110
+
+#define mmDMA5_CORE_RD_INFLIGHTS                                     0x5A0114
+
+#define mmDMA5_CORE_WR_MAX_OUTSTAND                                  0x5A0120
+
+#define mmDMA5_CORE_WR_MAX_AWID                                      0x5A0124
+
+#define mmDMA5_CORE_WR_AWCACHE                                       0x5A0128
+
+#define mmDMA5_CORE_WR_AWUSER_31_11                                  0x5A0130
+
+#define mmDMA5_CORE_WR_INFLIGHTS                                     0x5A0134
+
+#define mmDMA5_CORE_RD_RATE_LIM_CFG_0                                0x5A0150
+
+#define mmDMA5_CORE_RD_RATE_LIM_CFG_1                                0x5A0154
+
+#define mmDMA5_CORE_WR_RATE_LIM_CFG_0                                0x5A0158
+
+#define mmDMA5_CORE_WR_RATE_LIM_CFG_1                                0x5A015C
+
+#define mmDMA5_CORE_ERR_CFG                                          0x5A0160
+
+#define mmDMA5_CORE_ERR_CAUSE                                        0x5A0164
+
+#define mmDMA5_CORE_ERRMSG_ADDR_LO                                   0x5A0170
+
+#define mmDMA5_CORE_ERRMSG_ADDR_HI                                   0x5A0174
+
+#define mmDMA5_CORE_ERRMSG_WDATA                                     0x5A0178
+
+#define mmDMA5_CORE_STS0                                             0x5A0190
+
+#define mmDMA5_CORE_STS1                                             0x5A0194
+
+#define mmDMA5_CORE_RD_DBGMEM_ADD                                    0x5A0200
+
+#define mmDMA5_CORE_RD_DBGMEM_DATA_WR                                0x5A0204
+
+#define mmDMA5_CORE_RD_DBGMEM_DATA_RD                                0x5A0208
+
+#define mmDMA5_CORE_RD_DBGMEM_CTRL                                   0x5A020C
+
+#define mmDMA5_CORE_RD_DBGMEM_RC                                     0x5A0210
+
+#define mmDMA5_CORE_DBG_HBW_AXI_AR_CNT                               0x5A0220
+
+#define mmDMA5_CORE_DBG_HBW_AXI_AW_CNT                               0x5A0224
+
+#define mmDMA5_CORE_DBG_LBW_AXI_AW_CNT                               0x5A0228
+
+#define mmDMA5_CORE_DBG_DESC_CNT                                     0x5A022C
+
+#define mmDMA5_CORE_DBG_STS                                          0x5A0230
+
+#define mmDMA5_CORE_DBG_RD_DESC_ID                                   0x5A0234
+
+#define mmDMA5_CORE_DBG_WR_DESC_ID                                   0x5A0238
+
+#endif /* ASIC_REG_DMA5_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_qm_regs.h
new file mode 100644
index 000000000000..0faea21756c5
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma5_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA5_QM_REGS_H_
+#define ASIC_REG_DMA5_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA5_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA5_QM_GLBL_CFG0                                          0x5A8000
+
+#define mmDMA5_QM_GLBL_CFG1                                          0x5A8004
+
+#define mmDMA5_QM_GLBL_PROT                                          0x5A8008
+
+#define mmDMA5_QM_GLBL_ERR_CFG                                       0x5A800C
+
+#define mmDMA5_QM_GLBL_SECURE_PROPS_0                                0x5A8010
+
+#define mmDMA5_QM_GLBL_SECURE_PROPS_1                                0x5A8014
+
+#define mmDMA5_QM_GLBL_SECURE_PROPS_2                                0x5A8018
+
+#define mmDMA5_QM_GLBL_SECURE_PROPS_3                                0x5A801C
+
+#define mmDMA5_QM_GLBL_SECURE_PROPS_4                                0x5A8020
+
+#define mmDMA5_QM_GLBL_NON_SECURE_PROPS_0                            0x5A8024
+
+#define mmDMA5_QM_GLBL_NON_SECURE_PROPS_1                            0x5A8028
+
+#define mmDMA5_QM_GLBL_NON_SECURE_PROPS_2                            0x5A802C
+
+#define mmDMA5_QM_GLBL_NON_SECURE_PROPS_3                            0x5A8030
+
+#define mmDMA5_QM_GLBL_NON_SECURE_PROPS_4                            0x5A8034
+
+#define mmDMA5_QM_GLBL_STS0                                          0x5A8038
+
+#define mmDMA5_QM_GLBL_STS1_0                                        0x5A8040
+
+#define mmDMA5_QM_GLBL_STS1_1                                        0x5A8044
+
+#define mmDMA5_QM_GLBL_STS1_2                                        0x5A8048
+
+#define mmDMA5_QM_GLBL_STS1_3                                        0x5A804C
+
+#define mmDMA5_QM_GLBL_STS1_4                                        0x5A8050
+
+#define mmDMA5_QM_GLBL_MSG_EN_0                                      0x5A8054
+
+#define mmDMA5_QM_GLBL_MSG_EN_1                                      0x5A8058
+
+#define mmDMA5_QM_GLBL_MSG_EN_2                                      0x5A805C
+
+#define mmDMA5_QM_GLBL_MSG_EN_3                                      0x5A8060
+
+#define mmDMA5_QM_GLBL_MSG_EN_4                                      0x5A8068
+
+#define mmDMA5_QM_PQ_BASE_LO_0                                       0x5A8070
+
+#define mmDMA5_QM_PQ_BASE_LO_1                                       0x5A8074
+
+#define mmDMA5_QM_PQ_BASE_LO_2                                       0x5A8078
+
+#define mmDMA5_QM_PQ_BASE_LO_3                                       0x5A807C
+
+#define mmDMA5_QM_PQ_BASE_HI_0                                       0x5A8080
+
+#define mmDMA5_QM_PQ_BASE_HI_1                                       0x5A8084
+
+#define mmDMA5_QM_PQ_BASE_HI_2                                       0x5A8088
+
+#define mmDMA5_QM_PQ_BASE_HI_3                                       0x5A808C
+
+#define mmDMA5_QM_PQ_SIZE_0                                          0x5A8090
+
+#define mmDMA5_QM_PQ_SIZE_1                                          0x5A8094
+
+#define mmDMA5_QM_PQ_SIZE_2                                          0x5A8098
+
+#define mmDMA5_QM_PQ_SIZE_3                                          0x5A809C
+
+#define mmDMA5_QM_PQ_PI_0                                            0x5A80A0
+
+#define mmDMA5_QM_PQ_PI_1                                            0x5A80A4
+
+#define mmDMA5_QM_PQ_PI_2                                            0x5A80A8
+
+#define mmDMA5_QM_PQ_PI_3                                            0x5A80AC
+
+#define mmDMA5_QM_PQ_CI_0                                            0x5A80B0
+
+#define mmDMA5_QM_PQ_CI_1                                            0x5A80B4
+
+#define mmDMA5_QM_PQ_CI_2                                            0x5A80B8
+
+#define mmDMA5_QM_PQ_CI_3                                            0x5A80BC
+
+#define mmDMA5_QM_PQ_CFG0_0                                          0x5A80C0
+
+#define mmDMA5_QM_PQ_CFG0_1                                          0x5A80C4
+
+#define mmDMA5_QM_PQ_CFG0_2                                          0x5A80C8
+
+#define mmDMA5_QM_PQ_CFG0_3                                          0x5A80CC
+
+#define mmDMA5_QM_PQ_CFG1_0                                          0x5A80D0
+
+#define mmDMA5_QM_PQ_CFG1_1                                          0x5A80D4
+
+#define mmDMA5_QM_PQ_CFG1_2                                          0x5A80D8
+
+#define mmDMA5_QM_PQ_CFG1_3                                          0x5A80DC
+
+#define mmDMA5_QM_PQ_ARUSER_31_11_0                                  0x5A80E0
+
+#define mmDMA5_QM_PQ_ARUSER_31_11_1                                  0x5A80E4
+
+#define mmDMA5_QM_PQ_ARUSER_31_11_2                                  0x5A80E8
+
+#define mmDMA5_QM_PQ_ARUSER_31_11_3                                  0x5A80EC
+
+#define mmDMA5_QM_PQ_STS0_0                                          0x5A80F0
+
+#define mmDMA5_QM_PQ_STS0_1                                          0x5A80F4
+
+#define mmDMA5_QM_PQ_STS0_2                                          0x5A80F8
+
+#define mmDMA5_QM_PQ_STS0_3                                          0x5A80FC
+
+#define mmDMA5_QM_PQ_STS1_0                                          0x5A8100
+
+#define mmDMA5_QM_PQ_STS1_1                                          0x5A8104
+
+#define mmDMA5_QM_PQ_STS1_2                                          0x5A8108
+
+#define mmDMA5_QM_PQ_STS1_3                                          0x5A810C
+
+#define mmDMA5_QM_CQ_CFG0_0                                          0x5A8110
+
+#define mmDMA5_QM_CQ_CFG0_1                                          0x5A8114
+
+#define mmDMA5_QM_CQ_CFG0_2                                          0x5A8118
+
+#define mmDMA5_QM_CQ_CFG0_3                                          0x5A811C
+
+#define mmDMA5_QM_CQ_CFG0_4                                          0x5A8120
+
+#define mmDMA5_QM_CQ_CFG1_0                                          0x5A8124
+
+#define mmDMA5_QM_CQ_CFG1_1                                          0x5A8128
+
+#define mmDMA5_QM_CQ_CFG1_2                                          0x5A812C
+
+#define mmDMA5_QM_CQ_CFG1_3                                          0x5A8130
+
+#define mmDMA5_QM_CQ_CFG1_4                                          0x5A8134
+
+#define mmDMA5_QM_CQ_ARUSER_31_11_0                                  0x5A8138
+
+#define mmDMA5_QM_CQ_ARUSER_31_11_1                                  0x5A813C
+
+#define mmDMA5_QM_CQ_ARUSER_31_11_2                                  0x5A8140
+
+#define mmDMA5_QM_CQ_ARUSER_31_11_3                                  0x5A8144
+
+#define mmDMA5_QM_CQ_ARUSER_31_11_4                                  0x5A8148
+
+#define mmDMA5_QM_CQ_STS0_0                                          0x5A814C
+
+#define mmDMA5_QM_CQ_STS0_1                                          0x5A8150
+
+#define mmDMA5_QM_CQ_STS0_2                                          0x5A8154
+
+#define mmDMA5_QM_CQ_STS0_3                                          0x5A8158
+
+#define mmDMA5_QM_CQ_STS0_4                                          0x5A815C
+
+#define mmDMA5_QM_CQ_STS1_0                                          0x5A8160
+
+#define mmDMA5_QM_CQ_STS1_1                                          0x5A8164
+
+#define mmDMA5_QM_CQ_STS1_2                                          0x5A8168
+
+#define mmDMA5_QM_CQ_STS1_3                                          0x5A816C
+
+#define mmDMA5_QM_CQ_STS1_4                                          0x5A8170
+
+#define mmDMA5_QM_CQ_PTR_LO_0                                        0x5A8174
+
+#define mmDMA5_QM_CQ_PTR_HI_0                                        0x5A8178
+
+#define mmDMA5_QM_CQ_TSIZE_0                                         0x5A817C
+
+#define mmDMA5_QM_CQ_CTL_0                                           0x5A8180
+
+#define mmDMA5_QM_CQ_PTR_LO_1                                        0x5A8184
+
+#define mmDMA5_QM_CQ_PTR_HI_1                                        0x5A8188
+
+#define mmDMA5_QM_CQ_TSIZE_1                                         0x5A818C
+
+#define mmDMA5_QM_CQ_CTL_1                                           0x5A8190
+
+#define mmDMA5_QM_CQ_PTR_LO_2                                        0x5A8194
+
+#define mmDMA5_QM_CQ_PTR_HI_2                                        0x5A8198
+
+#define mmDMA5_QM_CQ_TSIZE_2                                         0x5A819C
+
+#define mmDMA5_QM_CQ_CTL_2                                           0x5A81A0
+
+#define mmDMA5_QM_CQ_PTR_LO_3                                        0x5A81A4
+
+#define mmDMA5_QM_CQ_PTR_HI_3                                        0x5A81A8
+
+#define mmDMA5_QM_CQ_TSIZE_3                                         0x5A81AC
+
+#define mmDMA5_QM_CQ_CTL_3                                           0x5A81B0
+
+#define mmDMA5_QM_CQ_PTR_LO_4                                        0x5A81B4
+
+#define mmDMA5_QM_CQ_PTR_HI_4                                        0x5A81B8
+
+#define mmDMA5_QM_CQ_TSIZE_4                                         0x5A81BC
+
+#define mmDMA5_QM_CQ_CTL_4                                           0x5A81C0
+
+#define mmDMA5_QM_CQ_PTR_LO_STS_0                                    0x5A81C4
+
+#define mmDMA5_QM_CQ_PTR_LO_STS_1                                    0x5A81C8
+
+#define mmDMA5_QM_CQ_PTR_LO_STS_2                                    0x5A81CC
+
+#define mmDMA5_QM_CQ_PTR_LO_STS_3                                    0x5A81D0
+
+#define mmDMA5_QM_CQ_PTR_LO_STS_4                                    0x5A81D4
+
+#define mmDMA5_QM_CQ_PTR_HI_STS_0                                    0x5A81D8
+
+#define mmDMA5_QM_CQ_PTR_HI_STS_1                                    0x5A81DC
+
+#define mmDMA5_QM_CQ_PTR_HI_STS_2                                    0x5A81E0
+
+#define mmDMA5_QM_CQ_PTR_HI_STS_3                                    0x5A81E4
+
+#define mmDMA5_QM_CQ_PTR_HI_STS_4                                    0x5A81E8
+
+#define mmDMA5_QM_CQ_TSIZE_STS_0                                     0x5A81EC
+
+#define mmDMA5_QM_CQ_TSIZE_STS_1                                     0x5A81F0
+
+#define mmDMA5_QM_CQ_TSIZE_STS_2                                     0x5A81F4
+
+#define mmDMA5_QM_CQ_TSIZE_STS_3                                     0x5A81F8
+
+#define mmDMA5_QM_CQ_TSIZE_STS_4                                     0x5A81FC
+
+#define mmDMA5_QM_CQ_CTL_STS_0                                       0x5A8200
+
+#define mmDMA5_QM_CQ_CTL_STS_1                                       0x5A8204
+
+#define mmDMA5_QM_CQ_CTL_STS_2                                       0x5A8208
+
+#define mmDMA5_QM_CQ_CTL_STS_3                                       0x5A820C
+
+#define mmDMA5_QM_CQ_CTL_STS_4                                       0x5A8210
+
+#define mmDMA5_QM_CQ_IFIFO_CNT_0                                     0x5A8214
+
+#define mmDMA5_QM_CQ_IFIFO_CNT_1                                     0x5A8218
+
+#define mmDMA5_QM_CQ_IFIFO_CNT_2                                     0x5A821C
+
+#define mmDMA5_QM_CQ_IFIFO_CNT_3                                     0x5A8220
+
+#define mmDMA5_QM_CQ_IFIFO_CNT_4                                     0x5A8224
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_0                             0x5A8228
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_1                             0x5A822C
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_2                             0x5A8230
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_3                             0x5A8234
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_LO_4                             0x5A8238
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_0                             0x5A823C
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_1                             0x5A8240
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_2                             0x5A8244
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_3                             0x5A8248
+
+#define mmDMA5_QM_CP_MSG_BASE0_ADDR_HI_4                             0x5A824C
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_0                             0x5A8250
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_1                             0x5A8254
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_2                             0x5A8258
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_3                             0x5A825C
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_LO_4                             0x5A8260
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_0                             0x5A8264
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_1                             0x5A8268
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_2                             0x5A826C
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_3                             0x5A8270
+
+#define mmDMA5_QM_CP_MSG_BASE1_ADDR_HI_4                             0x5A8274
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_0                             0x5A8278
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_1                             0x5A827C
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_2                             0x5A8280
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_3                             0x5A8284
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_LO_4                             0x5A8288
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_0                             0x5A828C
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_1                             0x5A8290
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_2                             0x5A8294
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_3                             0x5A8298
+
+#define mmDMA5_QM_CP_MSG_BASE2_ADDR_HI_4                             0x5A829C
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5A82A0
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5A82A4
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5A82A8
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5A82AC
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5A82B0
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5A82B4
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5A82B8
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5A82BC
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5A82C0
+
+#define mmDMA5_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5A82C4
+
+#define mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5A82C8
+
+#define mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5A82CC
+
+#define mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5A82D0
+
+#define mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5A82D4
+
+#define mmDMA5_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5A82D8
+
+#define mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5A82E0
+
+#define mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5A82E4
+
+#define mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5A82E8
+
+#define mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5A82EC
+
+#define mmDMA5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5A82F0
+
+#define mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5A82F4
+
+#define mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5A82F8
+
+#define mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5A82FC
+
+#define mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x5A8300
+
+#define mmDMA5_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x5A8304
+
+#define mmDMA5_QM_CP_FENCE0_RDATA_0                                  0x5A8308
+
+#define mmDMA5_QM_CP_FENCE0_RDATA_1                                  0x5A830C
+
+#define mmDMA5_QM_CP_FENCE0_RDATA_2                                  0x5A8310
+
+#define mmDMA5_QM_CP_FENCE0_RDATA_3                                  0x5A8314
+
+#define mmDMA5_QM_CP_FENCE0_RDATA_4                                  0x5A8318
+
+#define mmDMA5_QM_CP_FENCE1_RDATA_0                                  0x5A831C
+
+#define mmDMA5_QM_CP_FENCE1_RDATA_1                                  0x5A8320
+
+#define mmDMA5_QM_CP_FENCE1_RDATA_2                                  0x5A8324
+
+#define mmDMA5_QM_CP_FENCE1_RDATA_3                                  0x5A8328
+
+#define mmDMA5_QM_CP_FENCE1_RDATA_4                                  0x5A832C
+
+#define mmDMA5_QM_CP_FENCE2_RDATA_0                                  0x5A8330
+
+#define mmDMA5_QM_CP_FENCE2_RDATA_1                                  0x5A8334
+
+#define mmDMA5_QM_CP_FENCE2_RDATA_2                                  0x5A8338
+
+#define mmDMA5_QM_CP_FENCE2_RDATA_3                                  0x5A833C
+
+#define mmDMA5_QM_CP_FENCE2_RDATA_4                                  0x5A8340
+
+#define mmDMA5_QM_CP_FENCE3_RDATA_0                                  0x5A8344
+
+#define mmDMA5_QM_CP_FENCE3_RDATA_1                                  0x5A8348
+
+#define mmDMA5_QM_CP_FENCE3_RDATA_2                                  0x5A834C
+
+#define mmDMA5_QM_CP_FENCE3_RDATA_3                                  0x5A8350
+
+#define mmDMA5_QM_CP_FENCE3_RDATA_4                                  0x5A8354
+
+#define mmDMA5_QM_CP_FENCE0_CNT_0                                    0x5A8358
+
+#define mmDMA5_QM_CP_FENCE0_CNT_1                                    0x5A835C
+
+#define mmDMA5_QM_CP_FENCE0_CNT_2                                    0x5A8360
+
+#define mmDMA5_QM_CP_FENCE0_CNT_3                                    0x5A8364
+
+#define mmDMA5_QM_CP_FENCE0_CNT_4                                    0x5A8368
+
+#define mmDMA5_QM_CP_FENCE1_CNT_0                                    0x5A836C
+
+#define mmDMA5_QM_CP_FENCE1_CNT_1                                    0x5A8370
+
+#define mmDMA5_QM_CP_FENCE1_CNT_2                                    0x5A8374
+
+#define mmDMA5_QM_CP_FENCE1_CNT_3                                    0x5A8378
+
+#define mmDMA5_QM_CP_FENCE1_CNT_4                                    0x5A837C
+
+#define mmDMA5_QM_CP_FENCE2_CNT_0                                    0x5A8380
+
+#define mmDMA5_QM_CP_FENCE2_CNT_1                                    0x5A8384
+
+#define mmDMA5_QM_CP_FENCE2_CNT_2                                    0x5A8388
+
+#define mmDMA5_QM_CP_FENCE2_CNT_3                                    0x5A838C
+
+#define mmDMA5_QM_CP_FENCE2_CNT_4                                    0x5A8390
+
+#define mmDMA5_QM_CP_FENCE3_CNT_0                                    0x5A8394
+
+#define mmDMA5_QM_CP_FENCE3_CNT_1                                    0x5A8398
+
+#define mmDMA5_QM_CP_FENCE3_CNT_2                                    0x5A839C
+
+#define mmDMA5_QM_CP_FENCE3_CNT_3                                    0x5A83A0
+
+#define mmDMA5_QM_CP_FENCE3_CNT_4                                    0x5A83A4
+
+#define mmDMA5_QM_CP_STS_0                                           0x5A83A8
+
+#define mmDMA5_QM_CP_STS_1                                           0x5A83AC
+
+#define mmDMA5_QM_CP_STS_2                                           0x5A83B0
+
+#define mmDMA5_QM_CP_STS_3                                           0x5A83B4
+
+#define mmDMA5_QM_CP_STS_4                                           0x5A83B8
+
+#define mmDMA5_QM_CP_CURRENT_INST_LO_0                               0x5A83BC
+
+#define mmDMA5_QM_CP_CURRENT_INST_LO_1                               0x5A83C0
+
+#define mmDMA5_QM_CP_CURRENT_INST_LO_2                               0x5A83C4
+
+#define mmDMA5_QM_CP_CURRENT_INST_LO_3                               0x5A83C8
+
+#define mmDMA5_QM_CP_CURRENT_INST_LO_4                               0x5A83CC
+
+#define mmDMA5_QM_CP_CURRENT_INST_HI_0                               0x5A83D0
+
+#define mmDMA5_QM_CP_CURRENT_INST_HI_1                               0x5A83D4
+
+#define mmDMA5_QM_CP_CURRENT_INST_HI_2                               0x5A83D8
+
+#define mmDMA5_QM_CP_CURRENT_INST_HI_3                               0x5A83DC
+
+#define mmDMA5_QM_CP_CURRENT_INST_HI_4                               0x5A83E0
+
+#define mmDMA5_QM_CP_BARRIER_CFG_0                                   0x5A83F4
+
+#define mmDMA5_QM_CP_BARRIER_CFG_1                                   0x5A83F8
+
+#define mmDMA5_QM_CP_BARRIER_CFG_2                                   0x5A83FC
+
+#define mmDMA5_QM_CP_BARRIER_CFG_3                                   0x5A8400
+
+#define mmDMA5_QM_CP_BARRIER_CFG_4                                   0x5A8404
+
+#define mmDMA5_QM_CP_DBG_0_0                                         0x5A8408
+
+#define mmDMA5_QM_CP_DBG_0_1                                         0x5A840C
+
+#define mmDMA5_QM_CP_DBG_0_2                                         0x5A8410
+
+#define mmDMA5_QM_CP_DBG_0_3                                         0x5A8414
+
+#define mmDMA5_QM_CP_DBG_0_4                                         0x5A8418
+
+#define mmDMA5_QM_CP_ARUSER_31_11_0                                  0x5A841C
+
+#define mmDMA5_QM_CP_ARUSER_31_11_1                                  0x5A8420
+
+#define mmDMA5_QM_CP_ARUSER_31_11_2                                  0x5A8424
+
+#define mmDMA5_QM_CP_ARUSER_31_11_3                                  0x5A8428
+
+#define mmDMA5_QM_CP_ARUSER_31_11_4                                  0x5A842C
+
+#define mmDMA5_QM_CP_AWUSER_31_11_0                                  0x5A8430
+
+#define mmDMA5_QM_CP_AWUSER_31_11_1                                  0x5A8434
+
+#define mmDMA5_QM_CP_AWUSER_31_11_2                                  0x5A8438
+
+#define mmDMA5_QM_CP_AWUSER_31_11_3                                  0x5A843C
+
+#define mmDMA5_QM_CP_AWUSER_31_11_4                                  0x5A8440
+
+#define mmDMA5_QM_ARB_CFG_0                                          0x5A8A00
+
+#define mmDMA5_QM_ARB_CHOISE_Q_PUSH                                  0x5A8A04
+
+#define mmDMA5_QM_ARB_WRR_WEIGHT_0                                   0x5A8A08
+
+#define mmDMA5_QM_ARB_WRR_WEIGHT_1                                   0x5A8A0C
+
+#define mmDMA5_QM_ARB_WRR_WEIGHT_2                                   0x5A8A10
+
+#define mmDMA5_QM_ARB_WRR_WEIGHT_3                                   0x5A8A14
+
+#define mmDMA5_QM_ARB_CFG_1                                          0x5A8A18
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_0                               0x5A8A20
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_1                               0x5A8A24
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_2                               0x5A8A28
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_3                               0x5A8A2C
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_4                               0x5A8A30
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_5                               0x5A8A34
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_6                               0x5A8A38
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_7                               0x5A8A3C
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_8                               0x5A8A40
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_9                               0x5A8A44
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_10                              0x5A8A48
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_11                              0x5A8A4C
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_12                              0x5A8A50
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_13                              0x5A8A54
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_14                              0x5A8A58
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_15                              0x5A8A5C
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_16                              0x5A8A60
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_17                              0x5A8A64
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_18                              0x5A8A68
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_19                              0x5A8A6C
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_20                              0x5A8A70
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_21                              0x5A8A74
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_22                              0x5A8A78
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_23                              0x5A8A7C
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_24                              0x5A8A80
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_25                              0x5A8A84
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_26                              0x5A8A88
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_27                              0x5A8A8C
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_28                              0x5A8A90
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_29                              0x5A8A94
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_30                              0x5A8A98
+
+#define mmDMA5_QM_ARB_MST_AVAIL_CRED_31                              0x5A8A9C
+
+#define mmDMA5_QM_ARB_MST_CRED_INC                                   0x5A8AA0
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x5A8AA4
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x5A8AA8
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x5A8AAC
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x5A8AB0
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x5A8AB4
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x5A8AB8
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x5A8ABC
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x5A8AC0
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x5A8AC4
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x5A8AC8
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x5A8ACC
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x5A8AD0
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x5A8AD4
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x5A8AD8
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x5A8ADC
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x5A8AE0
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x5A8AE4
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x5A8AE8
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x5A8AEC
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x5A8AF0
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x5A8AF4
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x5A8AF8
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x5A8AFC
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x5A8B00
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x5A8B04
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x5A8B08
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x5A8B0C
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x5A8B10
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x5A8B14
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x5A8B18
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x5A8B1C
+
+#define mmDMA5_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x5A8B20
+
+#define mmDMA5_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x5A8B28
+
+#define mmDMA5_QM_ARB_MST_SLAVE_EN                                   0x5A8B2C
+
+#define mmDMA5_QM_ARB_MST_QUIET_PER                                  0x5A8B34
+
+#define mmDMA5_QM_ARB_SLV_CHOISE_WDT                                 0x5A8B38
+
+#define mmDMA5_QM_ARB_SLV_ID                                         0x5A8B3C
+
+#define mmDMA5_QM_ARB_MSG_MAX_INFLIGHT                               0x5A8B44
+
+#define mmDMA5_QM_ARB_MSG_AWUSER_31_11                               0x5A8B48
+
+#define mmDMA5_QM_ARB_MSG_AWUSER_SEC_PROP                            0x5A8B4C
+
+#define mmDMA5_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x5A8B50
+
+#define mmDMA5_QM_ARB_BASE_LO                                        0x5A8B54
+
+#define mmDMA5_QM_ARB_BASE_HI                                        0x5A8B58
+
+#define mmDMA5_QM_ARB_STATE_STS                                      0x5A8B80
+
+#define mmDMA5_QM_ARB_CHOISE_FULLNESS_STS                            0x5A8B84
+
+#define mmDMA5_QM_ARB_MSG_STS                                        0x5A8B88
+
+#define mmDMA5_QM_ARB_SLV_CHOISE_Q_HEAD                              0x5A8B8C
+
+#define mmDMA5_QM_ARB_ERR_CAUSE                                      0x5A8B9C
+
+#define mmDMA5_QM_ARB_ERR_MSG_EN                                     0x5A8BA0
+
+#define mmDMA5_QM_ARB_ERR_STS_DRP                                    0x5A8BA8
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_0                                 0x5A8BB0
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_1                                 0x5A8BB4
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_2                                 0x5A8BB8
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_3                                 0x5A8BBC
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_4                                 0x5A8BC0
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_5                                 0x5A8BC4
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_6                                 0x5A8BC8
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_7                                 0x5A8BCC
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_8                                 0x5A8BD0
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_9                                 0x5A8BD4
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_10                                0x5A8BD8
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_11                                0x5A8BDC
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_12                                0x5A8BE0
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_13                                0x5A8BE4
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_14                                0x5A8BE8
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_15                                0x5A8BEC
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_16                                0x5A8BF0
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_17                                0x5A8BF4
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_18                                0x5A8BF8
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_19                                0x5A8BFC
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_20                                0x5A8C00
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_21                                0x5A8C04
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_22                                0x5A8C08
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_23                                0x5A8C0C
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_24                                0x5A8C10
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_25                                0x5A8C14
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_26                                0x5A8C18
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_27                                0x5A8C1C
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_28                                0x5A8C20
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_29                                0x5A8C24
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_30                                0x5A8C28
+
+#define mmDMA5_QM_ARB_MST_CRED_STS_31                                0x5A8C2C
+
+#define mmDMA5_QM_CGM_CFG                                            0x5A8C70
+
+#define mmDMA5_QM_CGM_STS                                            0x5A8C74
+
+#define mmDMA5_QM_CGM_CFG1                                           0x5A8C78
+
+#define mmDMA5_QM_LOCAL_RANGE_BASE                                   0x5A8C80
+
+#define mmDMA5_QM_LOCAL_RANGE_SIZE                                   0x5A8C84
+
+#define mmDMA5_QM_CSMR_STRICT_PRIO_CFG                               0x5A8C90
+
+#define mmDMA5_QM_HBW_RD_RATE_LIM_CFG_1                              0x5A8C94
+
+#define mmDMA5_QM_LBW_WR_RATE_LIM_CFG_0                              0x5A8C98
+
+#define mmDMA5_QM_LBW_WR_RATE_LIM_CFG_1                              0x5A8C9C
+
+#define mmDMA5_QM_HBW_RD_RATE_LIM_CFG_0                              0x5A8CA0
+
+#define mmDMA5_QM_GLBL_AXCACHE                                       0x5A8CA4
+
+#define mmDMA5_QM_IND_GW_APB_CFG                                     0x5A8CB0
+
+#define mmDMA5_QM_IND_GW_APB_WDATA                                   0x5A8CB4
+
+#define mmDMA5_QM_IND_GW_APB_RDATA                                   0x5A8CB8
+
+#define mmDMA5_QM_IND_GW_APB_STATUS                                  0x5A8CBC
+
+#define mmDMA5_QM_GLBL_ERR_ADDR_LO                                   0x5A8CD0
+
+#define mmDMA5_QM_GLBL_ERR_ADDR_HI                                   0x5A8CD4
+
+#define mmDMA5_QM_GLBL_ERR_WDATA                                     0x5A8CD8
+
+#define mmDMA5_QM_GLBL_MEM_INIT_BUSY                                 0x5A8D00
+
+#endif /* ASIC_REG_DMA5_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_core_regs.h
new file mode 100644
index 000000000000..4962c13e2e2e
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA6_CORE_REGS_H_
+#define ASIC_REG_DMA6_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA6_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA6_CORE_CFG_0                                            0x5C0000
+
+#define mmDMA6_CORE_CFG_1                                            0x5C0004
+
+#define mmDMA6_CORE_LBW_MAX_OUTSTAND                                 0x5C0008
+
+#define mmDMA6_CORE_SRC_BASE_LO                                      0x5C0014
+
+#define mmDMA6_CORE_SRC_BASE_HI                                      0x5C0018
+
+#define mmDMA6_CORE_DST_BASE_LO                                      0x5C001C
+
+#define mmDMA6_CORE_DST_BASE_HI                                      0x5C0020
+
+#define mmDMA6_CORE_SRC_TSIZE_1                                      0x5C002C
+
+#define mmDMA6_CORE_SRC_STRIDE_1                                     0x5C0030
+
+#define mmDMA6_CORE_SRC_TSIZE_2                                      0x5C0034
+
+#define mmDMA6_CORE_SRC_STRIDE_2                                     0x5C0038
+
+#define mmDMA6_CORE_SRC_TSIZE_3                                      0x5C003C
+
+#define mmDMA6_CORE_SRC_STRIDE_3                                     0x5C0040
+
+#define mmDMA6_CORE_SRC_TSIZE_4                                      0x5C0044
+
+#define mmDMA6_CORE_SRC_STRIDE_4                                     0x5C0048
+
+#define mmDMA6_CORE_SRC_TSIZE_0                                      0x5C004C
+
+#define mmDMA6_CORE_DST_TSIZE_1                                      0x5C0054
+
+#define mmDMA6_CORE_DST_STRIDE_1                                     0x5C0058
+
+#define mmDMA6_CORE_DST_TSIZE_2                                      0x5C005C
+
+#define mmDMA6_CORE_DST_STRIDE_2                                     0x5C0060
+
+#define mmDMA6_CORE_DST_TSIZE_3                                      0x5C0064
+
+#define mmDMA6_CORE_DST_STRIDE_3                                     0x5C0068
+
+#define mmDMA6_CORE_DST_TSIZE_4                                      0x5C006C
+
+#define mmDMA6_CORE_DST_STRIDE_4                                     0x5C0070
+
+#define mmDMA6_CORE_DST_TSIZE_0                                      0x5C0074
+
+#define mmDMA6_CORE_COMMIT                                           0x5C0078
+
+#define mmDMA6_CORE_WR_COMP_WDATA                                    0x5C007C
+
+#define mmDMA6_CORE_WR_COMP_ADDR_LO                                  0x5C0080
+
+#define mmDMA6_CORE_WR_COMP_ADDR_HI                                  0x5C0084
+
+#define mmDMA6_CORE_WR_COMP_AWUSER_31_11                             0x5C0088
+
+#define mmDMA6_CORE_TE_NUMROWS                                       0x5C0094
+
+#define mmDMA6_CORE_PROT                                             0x5C00B8
+
+#define mmDMA6_CORE_SECURE_PROPS                                     0x5C00F0
+
+#define mmDMA6_CORE_NON_SECURE_PROPS                                 0x5C00F4
+
+#define mmDMA6_CORE_RD_MAX_OUTSTAND                                  0x5C0100
+
+#define mmDMA6_CORE_RD_MAX_SIZE                                      0x5C0104
+
+#define mmDMA6_CORE_RD_ARCACHE                                       0x5C0108
+
+#define mmDMA6_CORE_RD_ARUSER_31_11                                  0x5C0110
+
+#define mmDMA6_CORE_RD_INFLIGHTS                                     0x5C0114
+
+#define mmDMA6_CORE_WR_MAX_OUTSTAND                                  0x5C0120
+
+#define mmDMA6_CORE_WR_MAX_AWID                                      0x5C0124
+
+#define mmDMA6_CORE_WR_AWCACHE                                       0x5C0128
+
+#define mmDMA6_CORE_WR_AWUSER_31_11                                  0x5C0130
+
+#define mmDMA6_CORE_WR_INFLIGHTS                                     0x5C0134
+
+#define mmDMA6_CORE_RD_RATE_LIM_CFG_0                                0x5C0150
+
+#define mmDMA6_CORE_RD_RATE_LIM_CFG_1                                0x5C0154
+
+#define mmDMA6_CORE_WR_RATE_LIM_CFG_0                                0x5C0158
+
+#define mmDMA6_CORE_WR_RATE_LIM_CFG_1                                0x5C015C
+
+#define mmDMA6_CORE_ERR_CFG                                          0x5C0160
+
+#define mmDMA6_CORE_ERR_CAUSE                                        0x5C0164
+
+#define mmDMA6_CORE_ERRMSG_ADDR_LO                                   0x5C0170
+
+#define mmDMA6_CORE_ERRMSG_ADDR_HI                                   0x5C0174
+
+#define mmDMA6_CORE_ERRMSG_WDATA                                     0x5C0178
+
+#define mmDMA6_CORE_STS0                                             0x5C0190
+
+#define mmDMA6_CORE_STS1                                             0x5C0194
+
+#define mmDMA6_CORE_RD_DBGMEM_ADD                                    0x5C0200
+
+#define mmDMA6_CORE_RD_DBGMEM_DATA_WR                                0x5C0204
+
+#define mmDMA6_CORE_RD_DBGMEM_DATA_RD                                0x5C0208
+
+#define mmDMA6_CORE_RD_DBGMEM_CTRL                                   0x5C020C
+
+#define mmDMA6_CORE_RD_DBGMEM_RC                                     0x5C0210
+
+#define mmDMA6_CORE_DBG_HBW_AXI_AR_CNT                               0x5C0220
+
+#define mmDMA6_CORE_DBG_HBW_AXI_AW_CNT                               0x5C0224
+
+#define mmDMA6_CORE_DBG_LBW_AXI_AW_CNT                               0x5C0228
+
+#define mmDMA6_CORE_DBG_DESC_CNT                                     0x5C022C
+
+#define mmDMA6_CORE_DBG_STS                                          0x5C0230
+
+#define mmDMA6_CORE_DBG_RD_DESC_ID                                   0x5C0234
+
+#define mmDMA6_CORE_DBG_WR_DESC_ID                                   0x5C0238
+
+#endif /* ASIC_REG_DMA6_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_qm_regs.h
new file mode 100644
index 000000000000..af87adb94c94
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma6_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA6_QM_REGS_H_
+#define ASIC_REG_DMA6_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA6_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA6_QM_GLBL_CFG0                                          0x5C8000
+
+#define mmDMA6_QM_GLBL_CFG1                                          0x5C8004
+
+#define mmDMA6_QM_GLBL_PROT                                          0x5C8008
+
+#define mmDMA6_QM_GLBL_ERR_CFG                                       0x5C800C
+
+#define mmDMA6_QM_GLBL_SECURE_PROPS_0                                0x5C8010
+
+#define mmDMA6_QM_GLBL_SECURE_PROPS_1                                0x5C8014
+
+#define mmDMA6_QM_GLBL_SECURE_PROPS_2                                0x5C8018
+
+#define mmDMA6_QM_GLBL_SECURE_PROPS_3                                0x5C801C
+
+#define mmDMA6_QM_GLBL_SECURE_PROPS_4                                0x5C8020
+
+#define mmDMA6_QM_GLBL_NON_SECURE_PROPS_0                            0x5C8024
+
+#define mmDMA6_QM_GLBL_NON_SECURE_PROPS_1                            0x5C8028
+
+#define mmDMA6_QM_GLBL_NON_SECURE_PROPS_2                            0x5C802C
+
+#define mmDMA6_QM_GLBL_NON_SECURE_PROPS_3                            0x5C8030
+
+#define mmDMA6_QM_GLBL_NON_SECURE_PROPS_4                            0x5C8034
+
+#define mmDMA6_QM_GLBL_STS0                                          0x5C8038
+
+#define mmDMA6_QM_GLBL_STS1_0                                        0x5C8040
+
+#define mmDMA6_QM_GLBL_STS1_1                                        0x5C8044
+
+#define mmDMA6_QM_GLBL_STS1_2                                        0x5C8048
+
+#define mmDMA6_QM_GLBL_STS1_3                                        0x5C804C
+
+#define mmDMA6_QM_GLBL_STS1_4                                        0x5C8050
+
+#define mmDMA6_QM_GLBL_MSG_EN_0                                      0x5C8054
+
+#define mmDMA6_QM_GLBL_MSG_EN_1                                      0x5C8058
+
+#define mmDMA6_QM_GLBL_MSG_EN_2                                      0x5C805C
+
+#define mmDMA6_QM_GLBL_MSG_EN_3                                      0x5C8060
+
+#define mmDMA6_QM_GLBL_MSG_EN_4                                      0x5C8068
+
+#define mmDMA6_QM_PQ_BASE_LO_0                                       0x5C8070
+
+#define mmDMA6_QM_PQ_BASE_LO_1                                       0x5C8074
+
+#define mmDMA6_QM_PQ_BASE_LO_2                                       0x5C8078
+
+#define mmDMA6_QM_PQ_BASE_LO_3                                       0x5C807C
+
+#define mmDMA6_QM_PQ_BASE_HI_0                                       0x5C8080
+
+#define mmDMA6_QM_PQ_BASE_HI_1                                       0x5C8084
+
+#define mmDMA6_QM_PQ_BASE_HI_2                                       0x5C8088
+
+#define mmDMA6_QM_PQ_BASE_HI_3                                       0x5C808C
+
+#define mmDMA6_QM_PQ_SIZE_0                                          0x5C8090
+
+#define mmDMA6_QM_PQ_SIZE_1                                          0x5C8094
+
+#define mmDMA6_QM_PQ_SIZE_2                                          0x5C8098
+
+#define mmDMA6_QM_PQ_SIZE_3                                          0x5C809C
+
+#define mmDMA6_QM_PQ_PI_0                                            0x5C80A0
+
+#define mmDMA6_QM_PQ_PI_1                                            0x5C80A4
+
+#define mmDMA6_QM_PQ_PI_2                                            0x5C80A8
+
+#define mmDMA6_QM_PQ_PI_3                                            0x5C80AC
+
+#define mmDMA6_QM_PQ_CI_0                                            0x5C80B0
+
+#define mmDMA6_QM_PQ_CI_1                                            0x5C80B4
+
+#define mmDMA6_QM_PQ_CI_2                                            0x5C80B8
+
+#define mmDMA6_QM_PQ_CI_3                                            0x5C80BC
+
+#define mmDMA6_QM_PQ_CFG0_0                                          0x5C80C0
+
+#define mmDMA6_QM_PQ_CFG0_1                                          0x5C80C4
+
+#define mmDMA6_QM_PQ_CFG0_2                                          0x5C80C8
+
+#define mmDMA6_QM_PQ_CFG0_3                                          0x5C80CC
+
+#define mmDMA6_QM_PQ_CFG1_0                                          0x5C80D0
+
+#define mmDMA6_QM_PQ_CFG1_1                                          0x5C80D4
+
+#define mmDMA6_QM_PQ_CFG1_2                                          0x5C80D8
+
+#define mmDMA6_QM_PQ_CFG1_3                                          0x5C80DC
+
+#define mmDMA6_QM_PQ_ARUSER_31_11_0                                  0x5C80E0
+
+#define mmDMA6_QM_PQ_ARUSER_31_11_1                                  0x5C80E4
+
+#define mmDMA6_QM_PQ_ARUSER_31_11_2                                  0x5C80E8
+
+#define mmDMA6_QM_PQ_ARUSER_31_11_3                                  0x5C80EC
+
+#define mmDMA6_QM_PQ_STS0_0                                          0x5C80F0
+
+#define mmDMA6_QM_PQ_STS0_1                                          0x5C80F4
+
+#define mmDMA6_QM_PQ_STS0_2                                          0x5C80F8
+
+#define mmDMA6_QM_PQ_STS0_3                                          0x5C80FC
+
+#define mmDMA6_QM_PQ_STS1_0                                          0x5C8100
+
+#define mmDMA6_QM_PQ_STS1_1                                          0x5C8104
+
+#define mmDMA6_QM_PQ_STS1_2                                          0x5C8108
+
+#define mmDMA6_QM_PQ_STS1_3                                          0x5C810C
+
+#define mmDMA6_QM_CQ_CFG0_0                                          0x5C8110
+
+#define mmDMA6_QM_CQ_CFG0_1                                          0x5C8114
+
+#define mmDMA6_QM_CQ_CFG0_2                                          0x5C8118
+
+#define mmDMA6_QM_CQ_CFG0_3                                          0x5C811C
+
+#define mmDMA6_QM_CQ_CFG0_4                                          0x5C8120
+
+#define mmDMA6_QM_CQ_CFG1_0                                          0x5C8124
+
+#define mmDMA6_QM_CQ_CFG1_1                                          0x5C8128
+
+#define mmDMA6_QM_CQ_CFG1_2                                          0x5C812C
+
+#define mmDMA6_QM_CQ_CFG1_3                                          0x5C8130
+
+#define mmDMA6_QM_CQ_CFG1_4                                          0x5C8134
+
+#define mmDMA6_QM_CQ_ARUSER_31_11_0                                  0x5C8138
+
+#define mmDMA6_QM_CQ_ARUSER_31_11_1                                  0x5C813C
+
+#define mmDMA6_QM_CQ_ARUSER_31_11_2                                  0x5C8140
+
+#define mmDMA6_QM_CQ_ARUSER_31_11_3                                  0x5C8144
+
+#define mmDMA6_QM_CQ_ARUSER_31_11_4                                  0x5C8148
+
+#define mmDMA6_QM_CQ_STS0_0                                          0x5C814C
+
+#define mmDMA6_QM_CQ_STS0_1                                          0x5C8150
+
+#define mmDMA6_QM_CQ_STS0_2                                          0x5C8154
+
+#define mmDMA6_QM_CQ_STS0_3                                          0x5C8158
+
+#define mmDMA6_QM_CQ_STS0_4                                          0x5C815C
+
+#define mmDMA6_QM_CQ_STS1_0                                          0x5C8160
+
+#define mmDMA6_QM_CQ_STS1_1                                          0x5C8164
+
+#define mmDMA6_QM_CQ_STS1_2                                          0x5C8168
+
+#define mmDMA6_QM_CQ_STS1_3                                          0x5C816C
+
+#define mmDMA6_QM_CQ_STS1_4                                          0x5C8170
+
+#define mmDMA6_QM_CQ_PTR_LO_0                                        0x5C8174
+
+#define mmDMA6_QM_CQ_PTR_HI_0                                        0x5C8178
+
+#define mmDMA6_QM_CQ_TSIZE_0                                         0x5C817C
+
+#define mmDMA6_QM_CQ_CTL_0                                           0x5C8180
+
+#define mmDMA6_QM_CQ_PTR_LO_1                                        0x5C8184
+
+#define mmDMA6_QM_CQ_PTR_HI_1                                        0x5C8188
+
+#define mmDMA6_QM_CQ_TSIZE_1                                         0x5C818C
+
+#define mmDMA6_QM_CQ_CTL_1                                           0x5C8190
+
+#define mmDMA6_QM_CQ_PTR_LO_2                                        0x5C8194
+
+#define mmDMA6_QM_CQ_PTR_HI_2                                        0x5C8198
+
+#define mmDMA6_QM_CQ_TSIZE_2                                         0x5C819C
+
+#define mmDMA6_QM_CQ_CTL_2                                           0x5C81A0
+
+#define mmDMA6_QM_CQ_PTR_LO_3                                        0x5C81A4
+
+#define mmDMA6_QM_CQ_PTR_HI_3                                        0x5C81A8
+
+#define mmDMA6_QM_CQ_TSIZE_3                                         0x5C81AC
+
+#define mmDMA6_QM_CQ_CTL_3                                           0x5C81B0
+
+#define mmDMA6_QM_CQ_PTR_LO_4                                        0x5C81B4
+
+#define mmDMA6_QM_CQ_PTR_HI_4                                        0x5C81B8
+
+#define mmDMA6_QM_CQ_TSIZE_4                                         0x5C81BC
+
+#define mmDMA6_QM_CQ_CTL_4                                           0x5C81C0
+
+#define mmDMA6_QM_CQ_PTR_LO_STS_0                                    0x5C81C4
+
+#define mmDMA6_QM_CQ_PTR_LO_STS_1                                    0x5C81C8
+
+#define mmDMA6_QM_CQ_PTR_LO_STS_2                                    0x5C81CC
+
+#define mmDMA6_QM_CQ_PTR_LO_STS_3                                    0x5C81D0
+
+#define mmDMA6_QM_CQ_PTR_LO_STS_4                                    0x5C81D4
+
+#define mmDMA6_QM_CQ_PTR_HI_STS_0                                    0x5C81D8
+
+#define mmDMA6_QM_CQ_PTR_HI_STS_1                                    0x5C81DC
+
+#define mmDMA6_QM_CQ_PTR_HI_STS_2                                    0x5C81E0
+
+#define mmDMA6_QM_CQ_PTR_HI_STS_3                                    0x5C81E4
+
+#define mmDMA6_QM_CQ_PTR_HI_STS_4                                    0x5C81E8
+
+#define mmDMA6_QM_CQ_TSIZE_STS_0                                     0x5C81EC
+
+#define mmDMA6_QM_CQ_TSIZE_STS_1                                     0x5C81F0
+
+#define mmDMA6_QM_CQ_TSIZE_STS_2                                     0x5C81F4
+
+#define mmDMA6_QM_CQ_TSIZE_STS_3                                     0x5C81F8
+
+#define mmDMA6_QM_CQ_TSIZE_STS_4                                     0x5C81FC
+
+#define mmDMA6_QM_CQ_CTL_STS_0                                       0x5C8200
+
+#define mmDMA6_QM_CQ_CTL_STS_1                                       0x5C8204
+
+#define mmDMA6_QM_CQ_CTL_STS_2                                       0x5C8208
+
+#define mmDMA6_QM_CQ_CTL_STS_3                                       0x5C820C
+
+#define mmDMA6_QM_CQ_CTL_STS_4                                       0x5C8210
+
+#define mmDMA6_QM_CQ_IFIFO_CNT_0                                     0x5C8214
+
+#define mmDMA6_QM_CQ_IFIFO_CNT_1                                     0x5C8218
+
+#define mmDMA6_QM_CQ_IFIFO_CNT_2                                     0x5C821C
+
+#define mmDMA6_QM_CQ_IFIFO_CNT_3                                     0x5C8220
+
+#define mmDMA6_QM_CQ_IFIFO_CNT_4                                     0x5C8224
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_0                             0x5C8228
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_1                             0x5C822C
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_2                             0x5C8230
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_3                             0x5C8234
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_LO_4                             0x5C8238
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_0                             0x5C823C
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_1                             0x5C8240
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_2                             0x5C8244
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_3                             0x5C8248
+
+#define mmDMA6_QM_CP_MSG_BASE0_ADDR_HI_4                             0x5C824C
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_0                             0x5C8250
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_1                             0x5C8254
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_2                             0x5C8258
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_3                             0x5C825C
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_LO_4                             0x5C8260
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_0                             0x5C8264
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_1                             0x5C8268
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_2                             0x5C826C
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_3                             0x5C8270
+
+#define mmDMA6_QM_CP_MSG_BASE1_ADDR_HI_4                             0x5C8274
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_0                             0x5C8278
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_1                             0x5C827C
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_2                             0x5C8280
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_3                             0x5C8284
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_LO_4                             0x5C8288
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_0                             0x5C828C
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_1                             0x5C8290
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_2                             0x5C8294
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_3                             0x5C8298
+
+#define mmDMA6_QM_CP_MSG_BASE2_ADDR_HI_4                             0x5C829C
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5C82A0
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5C82A4
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5C82A8
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5C82AC
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5C82B0
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5C82B4
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5C82B8
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5C82BC
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5C82C0
+
+#define mmDMA6_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5C82C4
+
+#define mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5C82C8
+
+#define mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5C82CC
+
+#define mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5C82D0
+
+#define mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5C82D4
+
+#define mmDMA6_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5C82D8
+
+#define mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5C82E0
+
+#define mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5C82E4
+
+#define mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5C82E8
+
+#define mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5C82EC
+
+#define mmDMA6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5C82F0
+
+#define mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5C82F4
+
+#define mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5C82F8
+
+#define mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5C82FC
+
+#define mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x5C8300
+
+#define mmDMA6_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x5C8304
+
+#define mmDMA6_QM_CP_FENCE0_RDATA_0                                  0x5C8308
+
+#define mmDMA6_QM_CP_FENCE0_RDATA_1                                  0x5C830C
+
+#define mmDMA6_QM_CP_FENCE0_RDATA_2                                  0x5C8310
+
+#define mmDMA6_QM_CP_FENCE0_RDATA_3                                  0x5C8314
+
+#define mmDMA6_QM_CP_FENCE0_RDATA_4                                  0x5C8318
+
+#define mmDMA6_QM_CP_FENCE1_RDATA_0                                  0x5C831C
+
+#define mmDMA6_QM_CP_FENCE1_RDATA_1                                  0x5C8320
+
+#define mmDMA6_QM_CP_FENCE1_RDATA_2                                  0x5C8324
+
+#define mmDMA6_QM_CP_FENCE1_RDATA_3                                  0x5C8328
+
+#define mmDMA6_QM_CP_FENCE1_RDATA_4                                  0x5C832C
+
+#define mmDMA6_QM_CP_FENCE2_RDATA_0                                  0x5C8330
+
+#define mmDMA6_QM_CP_FENCE2_RDATA_1                                  0x5C8334
+
+#define mmDMA6_QM_CP_FENCE2_RDATA_2                                  0x5C8338
+
+#define mmDMA6_QM_CP_FENCE2_RDATA_3                                  0x5C833C
+
+#define mmDMA6_QM_CP_FENCE2_RDATA_4                                  0x5C8340
+
+#define mmDMA6_QM_CP_FENCE3_RDATA_0                                  0x5C8344
+
+#define mmDMA6_QM_CP_FENCE3_RDATA_1                                  0x5C8348
+
+#define mmDMA6_QM_CP_FENCE3_RDATA_2                                  0x5C834C
+
+#define mmDMA6_QM_CP_FENCE3_RDATA_3                                  0x5C8350
+
+#define mmDMA6_QM_CP_FENCE3_RDATA_4                                  0x5C8354
+
+#define mmDMA6_QM_CP_FENCE0_CNT_0                                    0x5C8358
+
+#define mmDMA6_QM_CP_FENCE0_CNT_1                                    0x5C835C
+
+#define mmDMA6_QM_CP_FENCE0_CNT_2                                    0x5C8360
+
+#define mmDMA6_QM_CP_FENCE0_CNT_3                                    0x5C8364
+
+#define mmDMA6_QM_CP_FENCE0_CNT_4                                    0x5C8368
+
+#define mmDMA6_QM_CP_FENCE1_CNT_0                                    0x5C836C
+
+#define mmDMA6_QM_CP_FENCE1_CNT_1                                    0x5C8370
+
+#define mmDMA6_QM_CP_FENCE1_CNT_2                                    0x5C8374
+
+#define mmDMA6_QM_CP_FENCE1_CNT_3                                    0x5C8378
+
+#define mmDMA6_QM_CP_FENCE1_CNT_4                                    0x5C837C
+
+#define mmDMA6_QM_CP_FENCE2_CNT_0                                    0x5C8380
+
+#define mmDMA6_QM_CP_FENCE2_CNT_1                                    0x5C8384
+
+#define mmDMA6_QM_CP_FENCE2_CNT_2                                    0x5C8388
+
+#define mmDMA6_QM_CP_FENCE2_CNT_3                                    0x5C838C
+
+#define mmDMA6_QM_CP_FENCE2_CNT_4                                    0x5C8390
+
+#define mmDMA6_QM_CP_FENCE3_CNT_0                                    0x5C8394
+
+#define mmDMA6_QM_CP_FENCE3_CNT_1                                    0x5C8398
+
+#define mmDMA6_QM_CP_FENCE3_CNT_2                                    0x5C839C
+
+#define mmDMA6_QM_CP_FENCE3_CNT_3                                    0x5C83A0
+
+#define mmDMA6_QM_CP_FENCE3_CNT_4                                    0x5C83A4
+
+#define mmDMA6_QM_CP_STS_0                                           0x5C83A8
+
+#define mmDMA6_QM_CP_STS_1                                           0x5C83AC
+
+#define mmDMA6_QM_CP_STS_2                                           0x5C83B0
+
+#define mmDMA6_QM_CP_STS_3                                           0x5C83B4
+
+#define mmDMA6_QM_CP_STS_4                                           0x5C83B8
+
+#define mmDMA6_QM_CP_CURRENT_INST_LO_0                               0x5C83BC
+
+#define mmDMA6_QM_CP_CURRENT_INST_LO_1                               0x5C83C0
+
+#define mmDMA6_QM_CP_CURRENT_INST_LO_2                               0x5C83C4
+
+#define mmDMA6_QM_CP_CURRENT_INST_LO_3                               0x5C83C8
+
+#define mmDMA6_QM_CP_CURRENT_INST_LO_4                               0x5C83CC
+
+#define mmDMA6_QM_CP_CURRENT_INST_HI_0                               0x5C83D0
+
+#define mmDMA6_QM_CP_CURRENT_INST_HI_1                               0x5C83D4
+
+#define mmDMA6_QM_CP_CURRENT_INST_HI_2                               0x5C83D8
+
+#define mmDMA6_QM_CP_CURRENT_INST_HI_3                               0x5C83DC
+
+#define mmDMA6_QM_CP_CURRENT_INST_HI_4                               0x5C83E0
+
+#define mmDMA6_QM_CP_BARRIER_CFG_0                                   0x5C83F4
+
+#define mmDMA6_QM_CP_BARRIER_CFG_1                                   0x5C83F8
+
+#define mmDMA6_QM_CP_BARRIER_CFG_2                                   0x5C83FC
+
+#define mmDMA6_QM_CP_BARRIER_CFG_3                                   0x5C8400
+
+#define mmDMA6_QM_CP_BARRIER_CFG_4                                   0x5C8404
+
+#define mmDMA6_QM_CP_DBG_0_0                                         0x5C8408
+
+#define mmDMA6_QM_CP_DBG_0_1                                         0x5C840C
+
+#define mmDMA6_QM_CP_DBG_0_2                                         0x5C8410
+
+#define mmDMA6_QM_CP_DBG_0_3                                         0x5C8414
+
+#define mmDMA6_QM_CP_DBG_0_4                                         0x5C8418
+
+#define mmDMA6_QM_CP_ARUSER_31_11_0                                  0x5C841C
+
+#define mmDMA6_QM_CP_ARUSER_31_11_1                                  0x5C8420
+
+#define mmDMA6_QM_CP_ARUSER_31_11_2                                  0x5C8424
+
+#define mmDMA6_QM_CP_ARUSER_31_11_3                                  0x5C8428
+
+#define mmDMA6_QM_CP_ARUSER_31_11_4                                  0x5C842C
+
+#define mmDMA6_QM_CP_AWUSER_31_11_0                                  0x5C8430
+
+#define mmDMA6_QM_CP_AWUSER_31_11_1                                  0x5C8434
+
+#define mmDMA6_QM_CP_AWUSER_31_11_2                                  0x5C8438
+
+#define mmDMA6_QM_CP_AWUSER_31_11_3                                  0x5C843C
+
+#define mmDMA6_QM_CP_AWUSER_31_11_4                                  0x5C8440
+
+#define mmDMA6_QM_ARB_CFG_0                                          0x5C8A00
+
+#define mmDMA6_QM_ARB_CHOISE_Q_PUSH                                  0x5C8A04
+
+#define mmDMA6_QM_ARB_WRR_WEIGHT_0                                   0x5C8A08
+
+#define mmDMA6_QM_ARB_WRR_WEIGHT_1                                   0x5C8A0C
+
+#define mmDMA6_QM_ARB_WRR_WEIGHT_2                                   0x5C8A10
+
+#define mmDMA6_QM_ARB_WRR_WEIGHT_3                                   0x5C8A14
+
+#define mmDMA6_QM_ARB_CFG_1                                          0x5C8A18
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_0                               0x5C8A20
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_1                               0x5C8A24
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_2                               0x5C8A28
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_3                               0x5C8A2C
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_4                               0x5C8A30
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_5                               0x5C8A34
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_6                               0x5C8A38
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_7                               0x5C8A3C
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_8                               0x5C8A40
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_9                               0x5C8A44
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_10                              0x5C8A48
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_11                              0x5C8A4C
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_12                              0x5C8A50
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_13                              0x5C8A54
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_14                              0x5C8A58
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_15                              0x5C8A5C
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_16                              0x5C8A60
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_17                              0x5C8A64
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_18                              0x5C8A68
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_19                              0x5C8A6C
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_20                              0x5C8A70
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_21                              0x5C8A74
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_22                              0x5C8A78
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_23                              0x5C8A7C
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_24                              0x5C8A80
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_25                              0x5C8A84
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_26                              0x5C8A88
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_27                              0x5C8A8C
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_28                              0x5C8A90
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_29                              0x5C8A94
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_30                              0x5C8A98
+
+#define mmDMA6_QM_ARB_MST_AVAIL_CRED_31                              0x5C8A9C
+
+#define mmDMA6_QM_ARB_MST_CRED_INC                                   0x5C8AA0
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x5C8AA4
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x5C8AA8
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x5C8AAC
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x5C8AB0
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x5C8AB4
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x5C8AB8
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x5C8ABC
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x5C8AC0
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x5C8AC4
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x5C8AC8
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x5C8ACC
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x5C8AD0
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x5C8AD4
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x5C8AD8
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x5C8ADC
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x5C8AE0
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x5C8AE4
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x5C8AE8
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x5C8AEC
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x5C8AF0
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x5C8AF4
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x5C8AF8
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x5C8AFC
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x5C8B00
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x5C8B04
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x5C8B08
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x5C8B0C
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x5C8B10
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x5C8B14
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x5C8B18
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x5C8B1C
+
+#define mmDMA6_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x5C8B20
+
+#define mmDMA6_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x5C8B28
+
+#define mmDMA6_QM_ARB_MST_SLAVE_EN                                   0x5C8B2C
+
+#define mmDMA6_QM_ARB_MST_QUIET_PER                                  0x5C8B34
+
+#define mmDMA6_QM_ARB_SLV_CHOISE_WDT                                 0x5C8B38
+
+#define mmDMA6_QM_ARB_SLV_ID                                         0x5C8B3C
+
+#define mmDMA6_QM_ARB_MSG_MAX_INFLIGHT                               0x5C8B44
+
+#define mmDMA6_QM_ARB_MSG_AWUSER_31_11                               0x5C8B48
+
+#define mmDMA6_QM_ARB_MSG_AWUSER_SEC_PROP                            0x5C8B4C
+
+#define mmDMA6_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x5C8B50
+
+#define mmDMA6_QM_ARB_BASE_LO                                        0x5C8B54
+
+#define mmDMA6_QM_ARB_BASE_HI                                        0x5C8B58
+
+#define mmDMA6_QM_ARB_STATE_STS                                      0x5C8B80
+
+#define mmDMA6_QM_ARB_CHOISE_FULLNESS_STS                            0x5C8B84
+
+#define mmDMA6_QM_ARB_MSG_STS                                        0x5C8B88
+
+#define mmDMA6_QM_ARB_SLV_CHOISE_Q_HEAD                              0x5C8B8C
+
+#define mmDMA6_QM_ARB_ERR_CAUSE                                      0x5C8B9C
+
+#define mmDMA6_QM_ARB_ERR_MSG_EN                                     0x5C8BA0
+
+#define mmDMA6_QM_ARB_ERR_STS_DRP                                    0x5C8BA8
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_0                                 0x5C8BB0
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_1                                 0x5C8BB4
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_2                                 0x5C8BB8
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_3                                 0x5C8BBC
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_4                                 0x5C8BC0
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_5                                 0x5C8BC4
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_6                                 0x5C8BC8
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_7                                 0x5C8BCC
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_8                                 0x5C8BD0
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_9                                 0x5C8BD4
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_10                                0x5C8BD8
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_11                                0x5C8BDC
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_12                                0x5C8BE0
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_13                                0x5C8BE4
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_14                                0x5C8BE8
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_15                                0x5C8BEC
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_16                                0x5C8BF0
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_17                                0x5C8BF4
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_18                                0x5C8BF8
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_19                                0x5C8BFC
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_20                                0x5C8C00
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_21                                0x5C8C04
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_22                                0x5C8C08
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_23                                0x5C8C0C
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_24                                0x5C8C10
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_25                                0x5C8C14
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_26                                0x5C8C18
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_27                                0x5C8C1C
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_28                                0x5C8C20
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_29                                0x5C8C24
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_30                                0x5C8C28
+
+#define mmDMA6_QM_ARB_MST_CRED_STS_31                                0x5C8C2C
+
+#define mmDMA6_QM_CGM_CFG                                            0x5C8C70
+
+#define mmDMA6_QM_CGM_STS                                            0x5C8C74
+
+#define mmDMA6_QM_CGM_CFG1                                           0x5C8C78
+
+#define mmDMA6_QM_LOCAL_RANGE_BASE                                   0x5C8C80
+
+#define mmDMA6_QM_LOCAL_RANGE_SIZE                                   0x5C8C84
+
+#define mmDMA6_QM_CSMR_STRICT_PRIO_CFG                               0x5C8C90
+
+#define mmDMA6_QM_HBW_RD_RATE_LIM_CFG_1                              0x5C8C94
+
+#define mmDMA6_QM_LBW_WR_RATE_LIM_CFG_0                              0x5C8C98
+
+#define mmDMA6_QM_LBW_WR_RATE_LIM_CFG_1                              0x5C8C9C
+
+#define mmDMA6_QM_HBW_RD_RATE_LIM_CFG_0                              0x5C8CA0
+
+#define mmDMA6_QM_GLBL_AXCACHE                                       0x5C8CA4
+
+#define mmDMA6_QM_IND_GW_APB_CFG                                     0x5C8CB0
+
+#define mmDMA6_QM_IND_GW_APB_WDATA                                   0x5C8CB4
+
+#define mmDMA6_QM_IND_GW_APB_RDATA                                   0x5C8CB8
+
+#define mmDMA6_QM_IND_GW_APB_STATUS                                  0x5C8CBC
+
+#define mmDMA6_QM_GLBL_ERR_ADDR_LO                                   0x5C8CD0
+
+#define mmDMA6_QM_GLBL_ERR_ADDR_HI                                   0x5C8CD4
+
+#define mmDMA6_QM_GLBL_ERR_WDATA                                     0x5C8CD8
+
+#define mmDMA6_QM_GLBL_MEM_INIT_BUSY                                 0x5C8D00
+
+#endif /* ASIC_REG_DMA6_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_core_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_core_regs.h
new file mode 100644
index 000000000000..8dd705d20195
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_core_regs.h
@@ -0,0 +1,156 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA7_CORE_REGS_H_
+#define ASIC_REG_DMA7_CORE_REGS_H_
+
+/*
+ *****************************************
+ *   DMA7_CORE (Prototype: DMA_CORE)
+ *****************************************
+ */
+
+#define mmDMA7_CORE_CFG_0                                            0x5E0000
+
+#define mmDMA7_CORE_CFG_1                                            0x5E0004
+
+#define mmDMA7_CORE_LBW_MAX_OUTSTAND                                 0x5E0008
+
+#define mmDMA7_CORE_SRC_BASE_LO                                      0x5E0014
+
+#define mmDMA7_CORE_SRC_BASE_HI                                      0x5E0018
+
+#define mmDMA7_CORE_DST_BASE_LO                                      0x5E001C
+
+#define mmDMA7_CORE_DST_BASE_HI                                      0x5E0020
+
+#define mmDMA7_CORE_SRC_TSIZE_1                                      0x5E002C
+
+#define mmDMA7_CORE_SRC_STRIDE_1                                     0x5E0030
+
+#define mmDMA7_CORE_SRC_TSIZE_2                                      0x5E0034
+
+#define mmDMA7_CORE_SRC_STRIDE_2                                     0x5E0038
+
+#define mmDMA7_CORE_SRC_TSIZE_3                                      0x5E003C
+
+#define mmDMA7_CORE_SRC_STRIDE_3                                     0x5E0040
+
+#define mmDMA7_CORE_SRC_TSIZE_4                                      0x5E0044
+
+#define mmDMA7_CORE_SRC_STRIDE_4                                     0x5E0048
+
+#define mmDMA7_CORE_SRC_TSIZE_0                                      0x5E004C
+
+#define mmDMA7_CORE_DST_TSIZE_1                                      0x5E0054
+
+#define mmDMA7_CORE_DST_STRIDE_1                                     0x5E0058
+
+#define mmDMA7_CORE_DST_TSIZE_2                                      0x5E005C
+
+#define mmDMA7_CORE_DST_STRIDE_2                                     0x5E0060
+
+#define mmDMA7_CORE_DST_TSIZE_3                                      0x5E0064
+
+#define mmDMA7_CORE_DST_STRIDE_3                                     0x5E0068
+
+#define mmDMA7_CORE_DST_TSIZE_4                                      0x5E006C
+
+#define mmDMA7_CORE_DST_STRIDE_4                                     0x5E0070
+
+#define mmDMA7_CORE_DST_TSIZE_0                                      0x5E0074
+
+#define mmDMA7_CORE_COMMIT                                           0x5E0078
+
+#define mmDMA7_CORE_WR_COMP_WDATA                                    0x5E007C
+
+#define mmDMA7_CORE_WR_COMP_ADDR_LO                                  0x5E0080
+
+#define mmDMA7_CORE_WR_COMP_ADDR_HI                                  0x5E0084
+
+#define mmDMA7_CORE_WR_COMP_AWUSER_31_11                             0x5E0088
+
+#define mmDMA7_CORE_TE_NUMROWS                                       0x5E0094
+
+#define mmDMA7_CORE_PROT                                             0x5E00B8
+
+#define mmDMA7_CORE_SECURE_PROPS                                     0x5E00F0
+
+#define mmDMA7_CORE_NON_SECURE_PROPS                                 0x5E00F4
+
+#define mmDMA7_CORE_RD_MAX_OUTSTAND                                  0x5E0100
+
+#define mmDMA7_CORE_RD_MAX_SIZE                                      0x5E0104
+
+#define mmDMA7_CORE_RD_ARCACHE                                       0x5E0108
+
+#define mmDMA7_CORE_RD_ARUSER_31_11                                  0x5E0110
+
+#define mmDMA7_CORE_RD_INFLIGHTS                                     0x5E0114
+
+#define mmDMA7_CORE_WR_MAX_OUTSTAND                                  0x5E0120
+
+#define mmDMA7_CORE_WR_MAX_AWID                                      0x5E0124
+
+#define mmDMA7_CORE_WR_AWCACHE                                       0x5E0128
+
+#define mmDMA7_CORE_WR_AWUSER_31_11                                  0x5E0130
+
+#define mmDMA7_CORE_WR_INFLIGHTS                                     0x5E0134
+
+#define mmDMA7_CORE_RD_RATE_LIM_CFG_0                                0x5E0150
+
+#define mmDMA7_CORE_RD_RATE_LIM_CFG_1                                0x5E0154
+
+#define mmDMA7_CORE_WR_RATE_LIM_CFG_0                                0x5E0158
+
+#define mmDMA7_CORE_WR_RATE_LIM_CFG_1                                0x5E015C
+
+#define mmDMA7_CORE_ERR_CFG                                          0x5E0160
+
+#define mmDMA7_CORE_ERR_CAUSE                                        0x5E0164
+
+#define mmDMA7_CORE_ERRMSG_ADDR_LO                                   0x5E0170
+
+#define mmDMA7_CORE_ERRMSG_ADDR_HI                                   0x5E0174
+
+#define mmDMA7_CORE_ERRMSG_WDATA                                     0x5E0178
+
+#define mmDMA7_CORE_STS0                                             0x5E0190
+
+#define mmDMA7_CORE_STS1                                             0x5E0194
+
+#define mmDMA7_CORE_RD_DBGMEM_ADD                                    0x5E0200
+
+#define mmDMA7_CORE_RD_DBGMEM_DATA_WR                                0x5E0204
+
+#define mmDMA7_CORE_RD_DBGMEM_DATA_RD                                0x5E0208
+
+#define mmDMA7_CORE_RD_DBGMEM_CTRL                                   0x5E020C
+
+#define mmDMA7_CORE_RD_DBGMEM_RC                                     0x5E0210
+
+#define mmDMA7_CORE_DBG_HBW_AXI_AR_CNT                               0x5E0220
+
+#define mmDMA7_CORE_DBG_HBW_AXI_AW_CNT                               0x5E0224
+
+#define mmDMA7_CORE_DBG_LBW_AXI_AW_CNT                               0x5E0228
+
+#define mmDMA7_CORE_DBG_DESC_CNT                                     0x5E022C
+
+#define mmDMA7_CORE_DBG_STS                                          0x5E0230
+
+#define mmDMA7_CORE_DBG_RD_DESC_ID                                   0x5E0234
+
+#define mmDMA7_CORE_DBG_WR_DESC_ID                                   0x5E0238
+
+#endif /* ASIC_REG_DMA7_CORE_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_qm_regs.h
new file mode 100644
index 000000000000..d6c631f63e3e
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma7_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA7_QM_REGS_H_
+#define ASIC_REG_DMA7_QM_REGS_H_
+
+/*
+ *****************************************
+ *   DMA7_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmDMA7_QM_GLBL_CFG0                                          0x5E8000
+
+#define mmDMA7_QM_GLBL_CFG1                                          0x5E8004
+
+#define mmDMA7_QM_GLBL_PROT                                          0x5E8008
+
+#define mmDMA7_QM_GLBL_ERR_CFG                                       0x5E800C
+
+#define mmDMA7_QM_GLBL_SECURE_PROPS_0                                0x5E8010
+
+#define mmDMA7_QM_GLBL_SECURE_PROPS_1                                0x5E8014
+
+#define mmDMA7_QM_GLBL_SECURE_PROPS_2                                0x5E8018
+
+#define mmDMA7_QM_GLBL_SECURE_PROPS_3                                0x5E801C
+
+#define mmDMA7_QM_GLBL_SECURE_PROPS_4                                0x5E8020
+
+#define mmDMA7_QM_GLBL_NON_SECURE_PROPS_0                            0x5E8024
+
+#define mmDMA7_QM_GLBL_NON_SECURE_PROPS_1                            0x5E8028
+
+#define mmDMA7_QM_GLBL_NON_SECURE_PROPS_2                            0x5E802C
+
+#define mmDMA7_QM_GLBL_NON_SECURE_PROPS_3                            0x5E8030
+
+#define mmDMA7_QM_GLBL_NON_SECURE_PROPS_4                            0x5E8034
+
+#define mmDMA7_QM_GLBL_STS0                                          0x5E8038
+
+#define mmDMA7_QM_GLBL_STS1_0                                        0x5E8040
+
+#define mmDMA7_QM_GLBL_STS1_1                                        0x5E8044
+
+#define mmDMA7_QM_GLBL_STS1_2                                        0x5E8048
+
+#define mmDMA7_QM_GLBL_STS1_3                                        0x5E804C
+
+#define mmDMA7_QM_GLBL_STS1_4                                        0x5E8050
+
+#define mmDMA7_QM_GLBL_MSG_EN_0                                      0x5E8054
+
+#define mmDMA7_QM_GLBL_MSG_EN_1                                      0x5E8058
+
+#define mmDMA7_QM_GLBL_MSG_EN_2                                      0x5E805C
+
+#define mmDMA7_QM_GLBL_MSG_EN_3                                      0x5E8060
+
+#define mmDMA7_QM_GLBL_MSG_EN_4                                      0x5E8068
+
+#define mmDMA7_QM_PQ_BASE_LO_0                                       0x5E8070
+
+#define mmDMA7_QM_PQ_BASE_LO_1                                       0x5E8074
+
+#define mmDMA7_QM_PQ_BASE_LO_2                                       0x5E8078
+
+#define mmDMA7_QM_PQ_BASE_LO_3                                       0x5E807C
+
+#define mmDMA7_QM_PQ_BASE_HI_0                                       0x5E8080
+
+#define mmDMA7_QM_PQ_BASE_HI_1                                       0x5E8084
+
+#define mmDMA7_QM_PQ_BASE_HI_2                                       0x5E8088
+
+#define mmDMA7_QM_PQ_BASE_HI_3                                       0x5E808C
+
+#define mmDMA7_QM_PQ_SIZE_0                                          0x5E8090
+
+#define mmDMA7_QM_PQ_SIZE_1                                          0x5E8094
+
+#define mmDMA7_QM_PQ_SIZE_2                                          0x5E8098
+
+#define mmDMA7_QM_PQ_SIZE_3                                          0x5E809C
+
+#define mmDMA7_QM_PQ_PI_0                                            0x5E80A0
+
+#define mmDMA7_QM_PQ_PI_1                                            0x5E80A4
+
+#define mmDMA7_QM_PQ_PI_2                                            0x5E80A8
+
+#define mmDMA7_QM_PQ_PI_3                                            0x5E80AC
+
+#define mmDMA7_QM_PQ_CI_0                                            0x5E80B0
+
+#define mmDMA7_QM_PQ_CI_1                                            0x5E80B4
+
+#define mmDMA7_QM_PQ_CI_2                                            0x5E80B8
+
+#define mmDMA7_QM_PQ_CI_3                                            0x5E80BC
+
+#define mmDMA7_QM_PQ_CFG0_0                                          0x5E80C0
+
+#define mmDMA7_QM_PQ_CFG0_1                                          0x5E80C4
+
+#define mmDMA7_QM_PQ_CFG0_2                                          0x5E80C8
+
+#define mmDMA7_QM_PQ_CFG0_3                                          0x5E80CC
+
+#define mmDMA7_QM_PQ_CFG1_0                                          0x5E80D0
+
+#define mmDMA7_QM_PQ_CFG1_1                                          0x5E80D4
+
+#define mmDMA7_QM_PQ_CFG1_2                                          0x5E80D8
+
+#define mmDMA7_QM_PQ_CFG1_3                                          0x5E80DC
+
+#define mmDMA7_QM_PQ_ARUSER_31_11_0                                  0x5E80E0
+
+#define mmDMA7_QM_PQ_ARUSER_31_11_1                                  0x5E80E4
+
+#define mmDMA7_QM_PQ_ARUSER_31_11_2                                  0x5E80E8
+
+#define mmDMA7_QM_PQ_ARUSER_31_11_3                                  0x5E80EC
+
+#define mmDMA7_QM_PQ_STS0_0                                          0x5E80F0
+
+#define mmDMA7_QM_PQ_STS0_1                                          0x5E80F4
+
+#define mmDMA7_QM_PQ_STS0_2                                          0x5E80F8
+
+#define mmDMA7_QM_PQ_STS0_3                                          0x5E80FC
+
+#define mmDMA7_QM_PQ_STS1_0                                          0x5E8100
+
+#define mmDMA7_QM_PQ_STS1_1                                          0x5E8104
+
+#define mmDMA7_QM_PQ_STS1_2                                          0x5E8108
+
+#define mmDMA7_QM_PQ_STS1_3                                          0x5E810C
+
+#define mmDMA7_QM_CQ_CFG0_0                                          0x5E8110
+
+#define mmDMA7_QM_CQ_CFG0_1                                          0x5E8114
+
+#define mmDMA7_QM_CQ_CFG0_2                                          0x5E8118
+
+#define mmDMA7_QM_CQ_CFG0_3                                          0x5E811C
+
+#define mmDMA7_QM_CQ_CFG0_4                                          0x5E8120
+
+#define mmDMA7_QM_CQ_CFG1_0                                          0x5E8124
+
+#define mmDMA7_QM_CQ_CFG1_1                                          0x5E8128
+
+#define mmDMA7_QM_CQ_CFG1_2                                          0x5E812C
+
+#define mmDMA7_QM_CQ_CFG1_3                                          0x5E8130
+
+#define mmDMA7_QM_CQ_CFG1_4                                          0x5E8134
+
+#define mmDMA7_QM_CQ_ARUSER_31_11_0                                  0x5E8138
+
+#define mmDMA7_QM_CQ_ARUSER_31_11_1                                  0x5E813C
+
+#define mmDMA7_QM_CQ_ARUSER_31_11_2                                  0x5E8140
+
+#define mmDMA7_QM_CQ_ARUSER_31_11_3                                  0x5E8144
+
+#define mmDMA7_QM_CQ_ARUSER_31_11_4                                  0x5E8148
+
+#define mmDMA7_QM_CQ_STS0_0                                          0x5E814C
+
+#define mmDMA7_QM_CQ_STS0_1                                          0x5E8150
+
+#define mmDMA7_QM_CQ_STS0_2                                          0x5E8154
+
+#define mmDMA7_QM_CQ_STS0_3                                          0x5E8158
+
+#define mmDMA7_QM_CQ_STS0_4                                          0x5E815C
+
+#define mmDMA7_QM_CQ_STS1_0                                          0x5E8160
+
+#define mmDMA7_QM_CQ_STS1_1                                          0x5E8164
+
+#define mmDMA7_QM_CQ_STS1_2                                          0x5E8168
+
+#define mmDMA7_QM_CQ_STS1_3                                          0x5E816C
+
+#define mmDMA7_QM_CQ_STS1_4                                          0x5E8170
+
+#define mmDMA7_QM_CQ_PTR_LO_0                                        0x5E8174
+
+#define mmDMA7_QM_CQ_PTR_HI_0                                        0x5E8178
+
+#define mmDMA7_QM_CQ_TSIZE_0                                         0x5E817C
+
+#define mmDMA7_QM_CQ_CTL_0                                           0x5E8180
+
+#define mmDMA7_QM_CQ_PTR_LO_1                                        0x5E8184
+
+#define mmDMA7_QM_CQ_PTR_HI_1                                        0x5E8188
+
+#define mmDMA7_QM_CQ_TSIZE_1                                         0x5E818C
+
+#define mmDMA7_QM_CQ_CTL_1                                           0x5E8190
+
+#define mmDMA7_QM_CQ_PTR_LO_2                                        0x5E8194
+
+#define mmDMA7_QM_CQ_PTR_HI_2                                        0x5E8198
+
+#define mmDMA7_QM_CQ_TSIZE_2                                         0x5E819C
+
+#define mmDMA7_QM_CQ_CTL_2                                           0x5E81A0
+
+#define mmDMA7_QM_CQ_PTR_LO_3                                        0x5E81A4
+
+#define mmDMA7_QM_CQ_PTR_HI_3                                        0x5E81A8
+
+#define mmDMA7_QM_CQ_TSIZE_3                                         0x5E81AC
+
+#define mmDMA7_QM_CQ_CTL_3                                           0x5E81B0
+
+#define mmDMA7_QM_CQ_PTR_LO_4                                        0x5E81B4
+
+#define mmDMA7_QM_CQ_PTR_HI_4                                        0x5E81B8
+
+#define mmDMA7_QM_CQ_TSIZE_4                                         0x5E81BC
+
+#define mmDMA7_QM_CQ_CTL_4                                           0x5E81C0
+
+#define mmDMA7_QM_CQ_PTR_LO_STS_0                                    0x5E81C4
+
+#define mmDMA7_QM_CQ_PTR_LO_STS_1                                    0x5E81C8
+
+#define mmDMA7_QM_CQ_PTR_LO_STS_2                                    0x5E81CC
+
+#define mmDMA7_QM_CQ_PTR_LO_STS_3                                    0x5E81D0
+
+#define mmDMA7_QM_CQ_PTR_LO_STS_4                                    0x5E81D4
+
+#define mmDMA7_QM_CQ_PTR_HI_STS_0                                    0x5E81D8
+
+#define mmDMA7_QM_CQ_PTR_HI_STS_1                                    0x5E81DC
+
+#define mmDMA7_QM_CQ_PTR_HI_STS_2                                    0x5E81E0
+
+#define mmDMA7_QM_CQ_PTR_HI_STS_3                                    0x5E81E4
+
+#define mmDMA7_QM_CQ_PTR_HI_STS_4                                    0x5E81E8
+
+#define mmDMA7_QM_CQ_TSIZE_STS_0                                     0x5E81EC
+
+#define mmDMA7_QM_CQ_TSIZE_STS_1                                     0x5E81F0
+
+#define mmDMA7_QM_CQ_TSIZE_STS_2                                     0x5E81F4
+
+#define mmDMA7_QM_CQ_TSIZE_STS_3                                     0x5E81F8
+
+#define mmDMA7_QM_CQ_TSIZE_STS_4                                     0x5E81FC
+
+#define mmDMA7_QM_CQ_CTL_STS_0                                       0x5E8200
+
+#define mmDMA7_QM_CQ_CTL_STS_1                                       0x5E8204
+
+#define mmDMA7_QM_CQ_CTL_STS_2                                       0x5E8208
+
+#define mmDMA7_QM_CQ_CTL_STS_3                                       0x5E820C
+
+#define mmDMA7_QM_CQ_CTL_STS_4                                       0x5E8210
+
+#define mmDMA7_QM_CQ_IFIFO_CNT_0                                     0x5E8214
+
+#define mmDMA7_QM_CQ_IFIFO_CNT_1                                     0x5E8218
+
+#define mmDMA7_QM_CQ_IFIFO_CNT_2                                     0x5E821C
+
+#define mmDMA7_QM_CQ_IFIFO_CNT_3                                     0x5E8220
+
+#define mmDMA7_QM_CQ_IFIFO_CNT_4                                     0x5E8224
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_0                             0x5E8228
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_1                             0x5E822C
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_2                             0x5E8230
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_3                             0x5E8234
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_LO_4                             0x5E8238
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_0                             0x5E823C
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_1                             0x5E8240
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_2                             0x5E8244
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_3                             0x5E8248
+
+#define mmDMA7_QM_CP_MSG_BASE0_ADDR_HI_4                             0x5E824C
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_0                             0x5E8250
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_1                             0x5E8254
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_2                             0x5E8258
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_3                             0x5E825C
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_LO_4                             0x5E8260
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_0                             0x5E8264
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_1                             0x5E8268
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_2                             0x5E826C
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_3                             0x5E8270
+
+#define mmDMA7_QM_CP_MSG_BASE1_ADDR_HI_4                             0x5E8274
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_0                             0x5E8278
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_1                             0x5E827C
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_2                             0x5E8280
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_3                             0x5E8284
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_LO_4                             0x5E8288
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_0                             0x5E828C
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_1                             0x5E8290
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_2                             0x5E8294
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_3                             0x5E8298
+
+#define mmDMA7_QM_CP_MSG_BASE2_ADDR_HI_4                             0x5E829C
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_0                             0x5E82A0
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_1                             0x5E82A4
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_2                             0x5E82A8
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_3                             0x5E82AC
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_LO_4                             0x5E82B0
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_0                             0x5E82B4
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_1                             0x5E82B8
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_2                             0x5E82BC
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_3                             0x5E82C0
+
+#define mmDMA7_QM_CP_MSG_BASE3_ADDR_HI_4                             0x5E82C4
+
+#define mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_0                             0x5E82C8
+
+#define mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_1                             0x5E82CC
+
+#define mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_2                             0x5E82D0
+
+#define mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_3                             0x5E82D4
+
+#define mmDMA7_QM_CP_LDMA_TSIZE_OFFSET_4                             0x5E82D8
+
+#define mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x5E82E0
+
+#define mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x5E82E4
+
+#define mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x5E82E8
+
+#define mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x5E82EC
+
+#define mmDMA7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x5E82F0
+
+#define mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x5E82F4
+
+#define mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x5E82F8
+
+#define mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x5E82FC
+
+#define mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x5E8300
+
+#define mmDMA7_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x5E8304
+
+#define mmDMA7_QM_CP_FENCE0_RDATA_0                                  0x5E8308
+
+#define mmDMA7_QM_CP_FENCE0_RDATA_1                                  0x5E830C
+
+#define mmDMA7_QM_CP_FENCE0_RDATA_2                                  0x5E8310
+
+#define mmDMA7_QM_CP_FENCE0_RDATA_3                                  0x5E8314
+
+#define mmDMA7_QM_CP_FENCE0_RDATA_4                                  0x5E8318
+
+#define mmDMA7_QM_CP_FENCE1_RDATA_0                                  0x5E831C
+
+#define mmDMA7_QM_CP_FENCE1_RDATA_1                                  0x5E8320
+
+#define mmDMA7_QM_CP_FENCE1_RDATA_2                                  0x5E8324
+
+#define mmDMA7_QM_CP_FENCE1_RDATA_3                                  0x5E8328
+
+#define mmDMA7_QM_CP_FENCE1_RDATA_4                                  0x5E832C
+
+#define mmDMA7_QM_CP_FENCE2_RDATA_0                                  0x5E8330
+
+#define mmDMA7_QM_CP_FENCE2_RDATA_1                                  0x5E8334
+
+#define mmDMA7_QM_CP_FENCE2_RDATA_2                                  0x5E8338
+
+#define mmDMA7_QM_CP_FENCE2_RDATA_3                                  0x5E833C
+
+#define mmDMA7_QM_CP_FENCE2_RDATA_4                                  0x5E8340
+
+#define mmDMA7_QM_CP_FENCE3_RDATA_0                                  0x5E8344
+
+#define mmDMA7_QM_CP_FENCE3_RDATA_1                                  0x5E8348
+
+#define mmDMA7_QM_CP_FENCE3_RDATA_2                                  0x5E834C
+
+#define mmDMA7_QM_CP_FENCE3_RDATA_3                                  0x5E8350
+
+#define mmDMA7_QM_CP_FENCE3_RDATA_4                                  0x5E8354
+
+#define mmDMA7_QM_CP_FENCE0_CNT_0                                    0x5E8358
+
+#define mmDMA7_QM_CP_FENCE0_CNT_1                                    0x5E835C
+
+#define mmDMA7_QM_CP_FENCE0_CNT_2                                    0x5E8360
+
+#define mmDMA7_QM_CP_FENCE0_CNT_3                                    0x5E8364
+
+#define mmDMA7_QM_CP_FENCE0_CNT_4                                    0x5E8368
+
+#define mmDMA7_QM_CP_FENCE1_CNT_0                                    0x5E836C
+
+#define mmDMA7_QM_CP_FENCE1_CNT_1                                    0x5E8370
+
+#define mmDMA7_QM_CP_FENCE1_CNT_2                                    0x5E8374
+
+#define mmDMA7_QM_CP_FENCE1_CNT_3                                    0x5E8378
+
+#define mmDMA7_QM_CP_FENCE1_CNT_4                                    0x5E837C
+
+#define mmDMA7_QM_CP_FENCE2_CNT_0                                    0x5E8380
+
+#define mmDMA7_QM_CP_FENCE2_CNT_1                                    0x5E8384
+
+#define mmDMA7_QM_CP_FENCE2_CNT_2                                    0x5E8388
+
+#define mmDMA7_QM_CP_FENCE2_CNT_3                                    0x5E838C
+
+#define mmDMA7_QM_CP_FENCE2_CNT_4                                    0x5E8390
+
+#define mmDMA7_QM_CP_FENCE3_CNT_0                                    0x5E8394
+
+#define mmDMA7_QM_CP_FENCE3_CNT_1                                    0x5E8398
+
+#define mmDMA7_QM_CP_FENCE3_CNT_2                                    0x5E839C
+
+#define mmDMA7_QM_CP_FENCE3_CNT_3                                    0x5E83A0
+
+#define mmDMA7_QM_CP_FENCE3_CNT_4                                    0x5E83A4
+
+#define mmDMA7_QM_CP_STS_0                                           0x5E83A8
+
+#define mmDMA7_QM_CP_STS_1                                           0x5E83AC
+
+#define mmDMA7_QM_CP_STS_2                                           0x5E83B0
+
+#define mmDMA7_QM_CP_STS_3                                           0x5E83B4
+
+#define mmDMA7_QM_CP_STS_4                                           0x5E83B8
+
+#define mmDMA7_QM_CP_CURRENT_INST_LO_0                               0x5E83BC
+
+#define mmDMA7_QM_CP_CURRENT_INST_LO_1                               0x5E83C0
+
+#define mmDMA7_QM_CP_CURRENT_INST_LO_2                               0x5E83C4
+
+#define mmDMA7_QM_CP_CURRENT_INST_LO_3                               0x5E83C8
+
+#define mmDMA7_QM_CP_CURRENT_INST_LO_4                               0x5E83CC
+
+#define mmDMA7_QM_CP_CURRENT_INST_HI_0                               0x5E83D0
+
+#define mmDMA7_QM_CP_CURRENT_INST_HI_1                               0x5E83D4
+
+#define mmDMA7_QM_CP_CURRENT_INST_HI_2                               0x5E83D8
+
+#define mmDMA7_QM_CP_CURRENT_INST_HI_3                               0x5E83DC
+
+#define mmDMA7_QM_CP_CURRENT_INST_HI_4                               0x5E83E0
+
+#define mmDMA7_QM_CP_BARRIER_CFG_0                                   0x5E83F4
+
+#define mmDMA7_QM_CP_BARRIER_CFG_1                                   0x5E83F8
+
+#define mmDMA7_QM_CP_BARRIER_CFG_2                                   0x5E83FC
+
+#define mmDMA7_QM_CP_BARRIER_CFG_3                                   0x5E8400
+
+#define mmDMA7_QM_CP_BARRIER_CFG_4                                   0x5E8404
+
+#define mmDMA7_QM_CP_DBG_0_0                                         0x5E8408
+
+#define mmDMA7_QM_CP_DBG_0_1                                         0x5E840C
+
+#define mmDMA7_QM_CP_DBG_0_2                                         0x5E8410
+
+#define mmDMA7_QM_CP_DBG_0_3                                         0x5E8414
+
+#define mmDMA7_QM_CP_DBG_0_4                                         0x5E8418
+
+#define mmDMA7_QM_CP_ARUSER_31_11_0                                  0x5E841C
+
+#define mmDMA7_QM_CP_ARUSER_31_11_1                                  0x5E8420
+
+#define mmDMA7_QM_CP_ARUSER_31_11_2                                  0x5E8424
+
+#define mmDMA7_QM_CP_ARUSER_31_11_3                                  0x5E8428
+
+#define mmDMA7_QM_CP_ARUSER_31_11_4                                  0x5E842C
+
+#define mmDMA7_QM_CP_AWUSER_31_11_0                                  0x5E8430
+
+#define mmDMA7_QM_CP_AWUSER_31_11_1                                  0x5E8434
+
+#define mmDMA7_QM_CP_AWUSER_31_11_2                                  0x5E8438
+
+#define mmDMA7_QM_CP_AWUSER_31_11_3                                  0x5E843C
+
+#define mmDMA7_QM_CP_AWUSER_31_11_4                                  0x5E8440
+
+#define mmDMA7_QM_ARB_CFG_0                                          0x5E8A00
+
+#define mmDMA7_QM_ARB_CHOISE_Q_PUSH                                  0x5E8A04
+
+#define mmDMA7_QM_ARB_WRR_WEIGHT_0                                   0x5E8A08
+
+#define mmDMA7_QM_ARB_WRR_WEIGHT_1                                   0x5E8A0C
+
+#define mmDMA7_QM_ARB_WRR_WEIGHT_2                                   0x5E8A10
+
+#define mmDMA7_QM_ARB_WRR_WEIGHT_3                                   0x5E8A14
+
+#define mmDMA7_QM_ARB_CFG_1                                          0x5E8A18
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_0                               0x5E8A20
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_1                               0x5E8A24
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_2                               0x5E8A28
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_3                               0x5E8A2C
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_4                               0x5E8A30
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_5                               0x5E8A34
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_6                               0x5E8A38
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_7                               0x5E8A3C
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_8                               0x5E8A40
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_9                               0x5E8A44
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_10                              0x5E8A48
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_11                              0x5E8A4C
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_12                              0x5E8A50
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_13                              0x5E8A54
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_14                              0x5E8A58
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_15                              0x5E8A5C
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_16                              0x5E8A60
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_17                              0x5E8A64
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_18                              0x5E8A68
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_19                              0x5E8A6C
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_20                              0x5E8A70
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_21                              0x5E8A74
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_22                              0x5E8A78
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_23                              0x5E8A7C
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_24                              0x5E8A80
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_25                              0x5E8A84
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_26                              0x5E8A88
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_27                              0x5E8A8C
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_28                              0x5E8A90
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_29                              0x5E8A94
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_30                              0x5E8A98
+
+#define mmDMA7_QM_ARB_MST_AVAIL_CRED_31                              0x5E8A9C
+
+#define mmDMA7_QM_ARB_MST_CRED_INC                                   0x5E8AA0
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x5E8AA4
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x5E8AA8
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x5E8AAC
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x5E8AB0
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x5E8AB4
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x5E8AB8
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x5E8ABC
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x5E8AC0
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x5E8AC4
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x5E8AC8
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x5E8ACC
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x5E8AD0
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x5E8AD4
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x5E8AD8
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x5E8ADC
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x5E8AE0
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x5E8AE4
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x5E8AE8
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x5E8AEC
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x5E8AF0
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x5E8AF4
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x5E8AF8
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x5E8AFC
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x5E8B00
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x5E8B04
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x5E8B08
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x5E8B0C
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x5E8B10
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x5E8B14
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x5E8B18
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x5E8B1C
+
+#define mmDMA7_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x5E8B20
+
+#define mmDMA7_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x5E8B28
+
+#define mmDMA7_QM_ARB_MST_SLAVE_EN                                   0x5E8B2C
+
+#define mmDMA7_QM_ARB_MST_QUIET_PER                                  0x5E8B34
+
+#define mmDMA7_QM_ARB_SLV_CHOISE_WDT                                 0x5E8B38
+
+#define mmDMA7_QM_ARB_SLV_ID                                         0x5E8B3C
+
+#define mmDMA7_QM_ARB_MSG_MAX_INFLIGHT                               0x5E8B44
+
+#define mmDMA7_QM_ARB_MSG_AWUSER_31_11                               0x5E8B48
+
+#define mmDMA7_QM_ARB_MSG_AWUSER_SEC_PROP                            0x5E8B4C
+
+#define mmDMA7_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x5E8B50
+
+#define mmDMA7_QM_ARB_BASE_LO                                        0x5E8B54
+
+#define mmDMA7_QM_ARB_BASE_HI                                        0x5E8B58
+
+#define mmDMA7_QM_ARB_STATE_STS                                      0x5E8B80
+
+#define mmDMA7_QM_ARB_CHOISE_FULLNESS_STS                            0x5E8B84
+
+#define mmDMA7_QM_ARB_MSG_STS                                        0x5E8B88
+
+#define mmDMA7_QM_ARB_SLV_CHOISE_Q_HEAD                              0x5E8B8C
+
+#define mmDMA7_QM_ARB_ERR_CAUSE                                      0x5E8B9C
+
+#define mmDMA7_QM_ARB_ERR_MSG_EN                                     0x5E8BA0
+
+#define mmDMA7_QM_ARB_ERR_STS_DRP                                    0x5E8BA8
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_0                                 0x5E8BB0
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_1                                 0x5E8BB4
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_2                                 0x5E8BB8
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_3                                 0x5E8BBC
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_4                                 0x5E8BC0
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_5                                 0x5E8BC4
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_6                                 0x5E8BC8
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_7                                 0x5E8BCC
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_8                                 0x5E8BD0
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_9                                 0x5E8BD4
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_10                                0x5E8BD8
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_11                                0x5E8BDC
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_12                                0x5E8BE0
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_13                                0x5E8BE4
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_14                                0x5E8BE8
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_15                                0x5E8BEC
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_16                                0x5E8BF0
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_17                                0x5E8BF4
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_18                                0x5E8BF8
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_19                                0x5E8BFC
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_20                                0x5E8C00
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_21                                0x5E8C04
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_22                                0x5E8C08
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_23                                0x5E8C0C
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_24                                0x5E8C10
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_25                                0x5E8C14
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_26                                0x5E8C18
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_27                                0x5E8C1C
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_28                                0x5E8C20
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_29                                0x5E8C24
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_30                                0x5E8C28
+
+#define mmDMA7_QM_ARB_MST_CRED_STS_31                                0x5E8C2C
+
+#define mmDMA7_QM_CGM_CFG                                            0x5E8C70
+
+#define mmDMA7_QM_CGM_STS                                            0x5E8C74
+
+#define mmDMA7_QM_CGM_CFG1                                           0x5E8C78
+
+#define mmDMA7_QM_LOCAL_RANGE_BASE                                   0x5E8C80
+
+#define mmDMA7_QM_LOCAL_RANGE_SIZE                                   0x5E8C84
+
+#define mmDMA7_QM_CSMR_STRICT_PRIO_CFG                               0x5E8C90
+
+#define mmDMA7_QM_HBW_RD_RATE_LIM_CFG_1                              0x5E8C94
+
+#define mmDMA7_QM_LBW_WR_RATE_LIM_CFG_0                              0x5E8C98
+
+#define mmDMA7_QM_LBW_WR_RATE_LIM_CFG_1                              0x5E8C9C
+
+#define mmDMA7_QM_HBW_RD_RATE_LIM_CFG_0                              0x5E8CA0
+
+#define mmDMA7_QM_GLBL_AXCACHE                                       0x5E8CA4
+
+#define mmDMA7_QM_IND_GW_APB_CFG                                     0x5E8CB0
+
+#define mmDMA7_QM_IND_GW_APB_WDATA                                   0x5E8CB4
+
+#define mmDMA7_QM_IND_GW_APB_RDATA                                   0x5E8CB8
+
+#define mmDMA7_QM_IND_GW_APB_STATUS                                  0x5E8CBC
+
+#define mmDMA7_QM_GLBL_ERR_ADDR_LO                                   0x5E8CD0
+
+#define mmDMA7_QM_GLBL_ERR_ADDR_HI                                   0x5E8CD4
+
+#define mmDMA7_QM_GLBL_ERR_WDATA                                     0x5E8CD8
+
+#define mmDMA7_QM_GLBL_MEM_INIT_BUSY                                 0x5E8D00
+
+#endif /* ASIC_REG_DMA7_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch0_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch0_regs.h
new file mode 100644
index 000000000000..8c1c72df4469
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch0_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_E_N_DOWN_CH0_REGS_H_
+#define ASIC_REG_DMA_IF_E_N_DOWN_CH0_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_E_N_DOWN_CH0 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_E_N_DOWN_CH0_PERM_SEL                               0x4E1108
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_0                          0x4E1114
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_1                          0x4E1118
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_2                          0x4E111C
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_3                          0x4E1120
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_4                          0x4E1124
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_5                          0x4E1128
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_6                          0x4E112C
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_7                          0x4E1130
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_8                          0x4E1134
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_9                          0x4E1138
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_10                         0x4E113C
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_11                         0x4E1140
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_12                         0x4E1144
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_13                         0x4E1148
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_14                         0x4E114C
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_15                         0x4E1150
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_16                         0x4E1154
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_17                         0x4E1158
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_18                         0x4E115C
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_19                         0x4E1160
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_20                         0x4E1164
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_21                         0x4E1168
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_22                         0x4E116C
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_23                         0x4E1170
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_24                         0x4E1174
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_25                         0x4E1178
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_26                         0x4E117C
+
+#define mmDMA_IF_E_N_DOWN_CH0_HBM_POLY_H3_27                         0x4E1180
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_0                         0x4E1184
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_1                         0x4E1188
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_2                         0x4E118C
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_3                         0x4E1190
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_4                         0x4E1194
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_5                         0x4E1198
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_6                         0x4E119C
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_7                         0x4E11A0
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_8                         0x4E11A4
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_9                         0x4E11A8
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_10                        0x4E11AC
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_11                        0x4E11B0
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_12                        0x4E11B4
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_13                        0x4E11B8
+
+#define mmDMA_IF_E_N_DOWN_CH0_SRAM_POLY_H3_14                        0x4E11BC
+
+#define mmDMA_IF_E_N_DOWN_CH0_SCRAM_SRAM_EN                          0x4E126C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_HBM_EN                              0x4E1274
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_HBM_SAT                             0x4E1278
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_HBM_RST                             0x4E127C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_HBM_TIMEOUT                         0x4E1280
+
+#define mmDMA_IF_E_N_DOWN_CH0_SCRAM_HBM_EN                           0x4E1284
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_PCI_EN                              0x4E1288
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_PCI_SAT                             0x4E128C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_PCI_RST                             0x4E1290
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_PCI_TIMEOUT                         0x4E1294
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_EN                             0x4E129C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_SAT                            0x4E12A0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_RST                            0x4E12A4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_TIMEOUT                        0x4E12AC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RL_SRAM_RED                            0x4E12B4
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_EN                             0x4E12EC
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_EN                             0x4E12F0
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_WR_SIZE                        0x4E12F4
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_WR_SIZE                        0x4E12F8
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_PCI_CTR_SET_EN                  0x4E1404
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_PCI_CTR_SET                     0x4E1408
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_PCI_CTR_WRAP                    0x4E140C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_PCI_CTR_CNT                     0x4E1410
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM_CTR_SET_EN                  0x4E1414
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM_CTR_SET                     0x4E1418
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_RD_SIZE                        0x4E141C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_RD_SIZE                        0x4E1420
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_PCI_CTR_SET_EN                  0x4E1424
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_PCI_CTR_SET                     0x4E1428
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_PCI_CTR_WRAP                    0x4E142C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_PCI_CTR_CNT                     0x4E1430
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM_CTR_SET_EN                  0x4E1434
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM_CTR_SET                     0x4E1438
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_0                           0x4E1450
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_SEL_1                           0x4E1454
+
+#define mmDMA_IF_E_N_DOWN_CH0_NON_LIN_EN                             0x4E1480
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_BANK_0                         0x4E1500
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_BANK_1                         0x4E1504
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_BANK_2                         0x4E1508
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_BANK_3                         0x4E150C
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_BANK_4                         0x4E1510
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_0                       0x4E1514
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_1                       0x4E1520
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_2                       0x4E1524
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_3                       0x4E1528
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_4                       0x4E152C
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_5                       0x4E1530
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_6                       0x4E1534
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_7                       0x4E1538
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_8                       0x4E153C
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_SRAM_OFFSET_9                       0x4E1540
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_0                        0x4E1550
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_1                        0x4E1554
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_2                        0x4E1558
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_3                        0x4E155C
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_4                        0x4E1560
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_5                        0x4E1564
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_6                        0x4E1568
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_7                        0x4E156C
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_8                        0x4E1570
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_9                        0x4E1574
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_10                       0x4E1578
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_11                       0x4E157C
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_12                       0x4E1580
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_13                       0x4E1584
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_14                       0x4E1588
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_15                       0x4E158C
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_16                       0x4E1590
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_17                       0x4E1594
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_OFFSET_18                       0x4E1598
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0                0x4E15E4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_1                0x4E15E8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_2                0x4E15EC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_3                0x4E15F0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_4                0x4E15F4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_5                0x4E15F8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_6                0x4E15FC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_7                0x4E1600
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_8                0x4E1604
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_9                0x4E1608
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_10               0x4E160C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_11               0x4E1610
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_12               0x4E1614
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_13               0x4E1618
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_14               0x4E161C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_15               0x4E1620
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0               0x4E1624
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_1               0x4E1628
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_2               0x4E162C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_3               0x4E1630
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_4               0x4E1634
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_5               0x4E1638
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_6               0x4E163C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_7               0x4E1640
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_8               0x4E1644
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_9               0x4E1648
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_10              0x4E164C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_11              0x4E1650
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_12              0x4E1654
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_13              0x4E1658
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_14              0x4E165C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_15              0x4E1660
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0                0x4E1664
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_1                0x4E1668
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_2                0x4E166C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_3                0x4E1670
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_4                0x4E1674
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_5                0x4E1678
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_6                0x4E167C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_7                0x4E1680
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_8                0x4E1684
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_9                0x4E1688
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_10               0x4E168C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_11               0x4E1690
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_12               0x4E1694
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_13               0x4E1698
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_14               0x4E169C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_15               0x4E16A0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0               0x4E16A4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_1               0x4E16A8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_2               0x4E16AC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_3               0x4E16B0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_4               0x4E16B4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_5               0x4E16B8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_6               0x4E16BC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_7               0x4E16C0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_8               0x4E16C4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_9               0x4E16C8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_10              0x4E16CC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_11              0x4E16D0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_12              0x4E16D4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_13              0x4E16D8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_14              0x4E16DC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_15              0x4E16E0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_0               0x4E16E4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_1               0x4E16E8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_2               0x4E16EC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_3               0x4E16F0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_4               0x4E16F4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_5               0x4E16F8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_6               0x4E16FC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_7               0x4E1700
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_8               0x4E1704
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_9               0x4E1708
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_10              0x4E170C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_11              0x4E1710
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_12              0x4E1714
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_13              0x4E1718
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_14              0x4E171C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_15              0x4E1720
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_0              0x4E1724
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_1              0x4E1728
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_2              0x4E172C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_3              0x4E1730
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_4              0x4E1734
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_5              0x4E1738
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_6              0x4E173C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_7              0x4E1740
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_8              0x4E1744
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_9              0x4E1748
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_10             0x4E174C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_11             0x4E1750
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_12             0x4E1754
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_13             0x4E1758
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_14             0x4E175C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_15             0x4E1760
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_0               0x4E1764
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_1               0x4E1768
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_2               0x4E176C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_3               0x4E1770
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_4               0x4E1774
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_5               0x4E1778
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_6               0x4E177C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_7               0x4E1780
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_8               0x4E1784
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_9               0x4E1788
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_10              0x4E178C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_11              0x4E1790
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_12              0x4E1794
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_13              0x4E1798
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_14              0x4E179C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_15              0x4E17A0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_0              0x4E17A4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_1              0x4E17A8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_2              0x4E17AC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_3              0x4E17B0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_4              0x4E17B4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_5              0x4E17B8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_6              0x4E17BC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_7              0x4E17C0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_8              0x4E17C4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_9              0x4E17C8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_10             0x4E17CC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_11             0x4E17D0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_12             0x4E17D4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_13             0x4E17D8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_14             0x4E17DC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_15             0x4E17E0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0                0x4E1824
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_1                0x4E1828
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_2                0x4E182C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_3                0x4E1830
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_4                0x4E1834
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_5                0x4E1838
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_6                0x4E183C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_7                0x4E1840
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_8                0x4E1844
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_9                0x4E1848
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_10               0x4E184C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_11               0x4E1850
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_12               0x4E1854
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_13               0x4E1858
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_14               0x4E185C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_15               0x4E1860
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0               0x4E1864
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_1               0x4E1868
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_2               0x4E186C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_3               0x4E1870
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_4               0x4E1874
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_5               0x4E1878
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_6               0x4E187C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_7               0x4E1880
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_8               0x4E1884
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_9               0x4E1888
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_10              0x4E188C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_11              0x4E1890
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_12              0x4E1894
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_13              0x4E1898
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_14              0x4E189C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_15              0x4E18A0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0                0x4E18A4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_1                0x4E18A8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_2                0x4E18AC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_3                0x4E18B0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_4                0x4E18B4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_5                0x4E18B8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_6                0x4E18BC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_7                0x4E18C0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_8                0x4E18C4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_9                0x4E18C8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_10               0x4E18CC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_11               0x4E18D0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_12               0x4E18D4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_13               0x4E18D8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_14               0x4E18DC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_15               0x4E18E0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0               0x4E18E4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_1               0x4E18E8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_2               0x4E18EC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_3               0x4E18F0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_4               0x4E18F4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_5               0x4E18F8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_6               0x4E18FC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_7               0x4E1900
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_8               0x4E1904
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_9               0x4E1908
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_10              0x4E190C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_11              0x4E1910
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_12              0x4E1914
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_13              0x4E1918
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_14              0x4E191C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_15              0x4E1920
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_0               0x4E1924
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_1               0x4E1928
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_2               0x4E192C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_3               0x4E1930
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_4               0x4E1934
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_5               0x4E1938
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_6               0x4E193C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_7               0x4E1940
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_8               0x4E1944
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_9               0x4E1948
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_10              0x4E194C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_11              0x4E1950
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_12              0x4E1954
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_13              0x4E1958
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_14              0x4E195C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_15              0x4E1960
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_0              0x4E1964
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_1              0x4E1968
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_2              0x4E196C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_3              0x4E1970
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_4              0x4E1974
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_5              0x4E1978
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_6              0x4E197C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_7              0x4E1980
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_8              0x4E1984
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_9              0x4E1988
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_10             0x4E198C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_11             0x4E1990
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_12             0x4E1994
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_13             0x4E1998
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_14             0x4E199C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_15             0x4E19A0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_0               0x4E19A4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_1               0x4E19A8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_2               0x4E19AC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_3               0x4E19B0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_4               0x4E19B4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_5               0x4E19B8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_6               0x4E19BC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_7               0x4E19C0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_8               0x4E19C4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_9               0x4E19C8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_10              0x4E19CC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_11              0x4E19D0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_12              0x4E19D4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_13              0x4E19D8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_14              0x4E19DC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_15              0x4E19E0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_0              0x4E19E4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_1              0x4E19E8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_2              0x4E19EC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_3              0x4E19F0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_4              0x4E19F4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_5              0x4E19F8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_6              0x4E19FC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_7              0x4E1A00
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_8              0x4E1A04
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_9              0x4E1A08
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_10             0x4E1A0C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_11             0x4E1A10
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_12             0x4E1A14
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_13             0x4E1A18
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_14             0x4E1A1C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_15             0x4E1A20
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_HIT_AW                       0x4E1A64
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_SEC_HIT_AR                       0x4E1A68
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_HIT_AW                      0x4E1A6C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RANGE_PRIV_HIT_AR                      0x4E1A70
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_CFG                                0x4E1B64
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_SHIFT                              0x4E1B68
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_0                     0x4E1B6C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_1                     0x4E1B70
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_2                     0x4E1B74
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_3                     0x4E1B78
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_4                     0x4E1B7C
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_5                     0x4E1B80
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_6                     0x4E1B84
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_EXPECTED_LAT_7                     0x4E1B88
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_0                            0x4E1BAC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_1                            0x4E1BB0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_2                            0x4E1BB4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_3                            0x4E1BB8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_4                            0x4E1BBC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_5                            0x4E1BC0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_6                            0x4E1BC4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_TOKEN_7                            0x4E1BC8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_0                          0x4E1BEC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_1                          0x4E1BF0
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_2                          0x4E1BF4
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_3                          0x4E1BF8
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_4                          0x4E1BFC
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_5                          0x4E1C00
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_6                          0x4E1C04
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_BANK_ID_7                          0x4E1C08
+
+#define mmDMA_IF_E_N_DOWN_CH0_RGL_WDT                                0x4E1C2C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_WRAP               0x4E1C30
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_WRAP               0x4E1C34
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_WRAP               0x4E1C38
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_WRAP               0x4E1C3C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_WRAP               0x4E1C40
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_WRAP               0x4E1C44
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_WRAP               0x4E1C48
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_WRAP               0x4E1C4C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_CNT                0x4E1C50
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_CNT                0x4E1C54
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_CNT                0x4E1C58
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_CNT                0x4E1C5C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_CNT                0x4E1C60
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_CNT                0x4E1C64
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_CNT                0x4E1C68
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_CNT                0x4E1C6C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_WRAP               0x4E1C70
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_WRAP               0x4E1C74
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_WRAP               0x4E1C78
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_WRAP               0x4E1C7C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_WRAP               0x4E1C80
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_WRAP               0x4E1C84
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_WRAP               0x4E1C88
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_WRAP               0x4E1C8C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_CNT                0x4E1C90
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_CNT                0x4E1C94
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_CNT                0x4E1C98
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_CNT                0x4E1C9C
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_CNT                0x4E1CA0
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_CNT                0x4E1CA4
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_CNT                0x4E1CA8
+
+#define mmDMA_IF_E_N_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_CNT                0x4E1CAC
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_PC_SEL_0                        0x4E1CB0
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_PC_SEL_1                        0x4E1CB4
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_PC_SEL_2                        0x4E1CB8
+
+#define mmDMA_IF_E_N_DOWN_CH0_NL_HBM_PC_SEL_3                        0x4E1CBC
+
+#endif /* ASIC_REG_DMA_IF_E_N_DOWN_CH0_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch1_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch1_regs.h
new file mode 100644
index 000000000000..b2b593fcec30
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_down_ch1_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_E_N_DOWN_CH1_REGS_H_
+#define ASIC_REG_DMA_IF_E_N_DOWN_CH1_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_E_N_DOWN_CH1 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_E_N_DOWN_CH1_PERM_SEL                               0x4E2108
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_0                          0x4E2114
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_1                          0x4E2118
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_2                          0x4E211C
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_3                          0x4E2120
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_4                          0x4E2124
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_5                          0x4E2128
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_6                          0x4E212C
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_7                          0x4E2130
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_8                          0x4E2134
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_9                          0x4E2138
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_10                         0x4E213C
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_11                         0x4E2140
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_12                         0x4E2144
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_13                         0x4E2148
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_14                         0x4E214C
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_15                         0x4E2150
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_16                         0x4E2154
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_17                         0x4E2158
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_18                         0x4E215C
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_19                         0x4E2160
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_20                         0x4E2164
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_21                         0x4E2168
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_22                         0x4E216C
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_23                         0x4E2170
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_24                         0x4E2174
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_25                         0x4E2178
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_26                         0x4E217C
+
+#define mmDMA_IF_E_N_DOWN_CH1_HBM_POLY_H3_27                         0x4E2180
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_0                         0x4E2184
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_1                         0x4E2188
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_2                         0x4E218C
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_3                         0x4E2190
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_4                         0x4E2194
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_5                         0x4E2198
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_6                         0x4E219C
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_7                         0x4E21A0
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_8                         0x4E21A4
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_9                         0x4E21A8
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_10                        0x4E21AC
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_11                        0x4E21B0
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_12                        0x4E21B4
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_13                        0x4E21B8
+
+#define mmDMA_IF_E_N_DOWN_CH1_SRAM_POLY_H3_14                        0x4E21BC
+
+#define mmDMA_IF_E_N_DOWN_CH1_SCRAM_SRAM_EN                          0x4E226C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_HBM_EN                              0x4E2274
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_HBM_SAT                             0x4E2278
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_HBM_RST                             0x4E227C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_HBM_TIMEOUT                         0x4E2280
+
+#define mmDMA_IF_E_N_DOWN_CH1_SCRAM_HBM_EN                           0x4E2284
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_PCI_EN                              0x4E2288
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_PCI_SAT                             0x4E228C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_PCI_RST                             0x4E2290
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_PCI_TIMEOUT                         0x4E2294
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_EN                             0x4E229C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_SAT                            0x4E22A0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_RST                            0x4E22A4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_TIMEOUT                        0x4E22AC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RL_SRAM_RED                            0x4E22B4
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_EN                             0x4E22EC
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_EN                             0x4E22F0
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_WR_SIZE                        0x4E22F4
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_WR_SIZE                        0x4E22F8
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_PCI_CTR_SET_EN                  0x4E2404
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_PCI_CTR_SET                     0x4E2408
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_PCI_CTR_WRAP                    0x4E240C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_PCI_CTR_CNT                     0x4E2410
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM_CTR_SET_EN                  0x4E2414
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM_CTR_SET                     0x4E2418
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_RD_SIZE                        0x4E241C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_RD_SIZE                        0x4E2420
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_PCI_CTR_SET_EN                  0x4E2424
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_PCI_CTR_SET                     0x4E2428
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_PCI_CTR_WRAP                    0x4E242C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_PCI_CTR_CNT                     0x4E2430
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM_CTR_SET_EN                  0x4E2434
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM_CTR_SET                     0x4E2438
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_0                           0x4E2450
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_SEL_1                           0x4E2454
+
+#define mmDMA_IF_E_N_DOWN_CH1_NON_LIN_EN                             0x4E2480
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_BANK_0                         0x4E2500
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_BANK_1                         0x4E2504
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_BANK_2                         0x4E2508
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_BANK_3                         0x4E250C
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_BANK_4                         0x4E2510
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_0                       0x4E2514
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_1                       0x4E2520
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_2                       0x4E2524
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_3                       0x4E2528
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_4                       0x4E252C
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_5                       0x4E2530
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_6                       0x4E2534
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_7                       0x4E2538
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_8                       0x4E253C
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_SRAM_OFFSET_9                       0x4E2540
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_0                        0x4E2550
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_1                        0x4E2554
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_2                        0x4E2558
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_3                        0x4E255C
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_4                        0x4E2560
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_5                        0x4E2564
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_6                        0x4E2568
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_7                        0x4E256C
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_8                        0x4E2570
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_9                        0x4E2574
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_10                       0x4E2578
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_11                       0x4E257C
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_12                       0x4E2580
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_13                       0x4E2584
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_14                       0x4E2588
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_15                       0x4E258C
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_16                       0x4E2590
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_17                       0x4E2594
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_OFFSET_18                       0x4E2598
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0                0x4E25E4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_1                0x4E25E8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_2                0x4E25EC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_3                0x4E25F0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_4                0x4E25F4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_5                0x4E25F8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_6                0x4E25FC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_7                0x4E2600
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_8                0x4E2604
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_9                0x4E2608
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_10               0x4E260C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_11               0x4E2610
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_12               0x4E2614
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_13               0x4E2618
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_14               0x4E261C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_15               0x4E2620
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0               0x4E2624
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_1               0x4E2628
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_2               0x4E262C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_3               0x4E2630
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_4               0x4E2634
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_5               0x4E2638
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_6               0x4E263C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_7               0x4E2640
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_8               0x4E2644
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_9               0x4E2648
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_10              0x4E264C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_11              0x4E2650
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_12              0x4E2654
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_13              0x4E2658
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_14              0x4E265C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_15              0x4E2660
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0                0x4E2664
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_1                0x4E2668
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_2                0x4E266C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_3                0x4E2670
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_4                0x4E2674
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_5                0x4E2678
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_6                0x4E267C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_7                0x4E2680
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_8                0x4E2684
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_9                0x4E2688
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_10               0x4E268C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_11               0x4E2690
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_12               0x4E2694
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_13               0x4E2698
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_14               0x4E269C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_15               0x4E26A0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0               0x4E26A4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_1               0x4E26A8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_2               0x4E26AC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_3               0x4E26B0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_4               0x4E26B4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_5               0x4E26B8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_6               0x4E26BC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_7               0x4E26C0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_8               0x4E26C4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_9               0x4E26C8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_10              0x4E26CC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_11              0x4E26D0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_12              0x4E26D4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_13              0x4E26D8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_14              0x4E26DC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_15              0x4E26E0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_0               0x4E26E4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_1               0x4E26E8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_2               0x4E26EC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_3               0x4E26F0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_4               0x4E26F4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_5               0x4E26F8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_6               0x4E26FC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_7               0x4E2700
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_8               0x4E2704
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_9               0x4E2708
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_10              0x4E270C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_11              0x4E2710
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_12              0x4E2714
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_13              0x4E2718
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_14              0x4E271C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_15              0x4E2720
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_0              0x4E2724
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_1              0x4E2728
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_2              0x4E272C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_3              0x4E2730
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_4              0x4E2734
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_5              0x4E2738
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_6              0x4E273C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_7              0x4E2740
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_8              0x4E2744
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_9              0x4E2748
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_10             0x4E274C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_11             0x4E2750
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_12             0x4E2754
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_13             0x4E2758
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_14             0x4E275C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_15             0x4E2760
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_0               0x4E2764
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_1               0x4E2768
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_2               0x4E276C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_3               0x4E2770
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_4               0x4E2774
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_5               0x4E2778
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_6               0x4E277C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_7               0x4E2780
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_8               0x4E2784
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_9               0x4E2788
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_10              0x4E278C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_11              0x4E2790
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_12              0x4E2794
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_13              0x4E2798
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_14              0x4E279C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_15              0x4E27A0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_0              0x4E27A4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_1              0x4E27A8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_2              0x4E27AC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_3              0x4E27B0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_4              0x4E27B4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_5              0x4E27B8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_6              0x4E27BC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_7              0x4E27C0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_8              0x4E27C4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_9              0x4E27C8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_10             0x4E27CC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_11             0x4E27D0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_12             0x4E27D4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_13             0x4E27D8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_14             0x4E27DC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_15             0x4E27E0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0                0x4E2824
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_1                0x4E2828
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_2                0x4E282C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_3                0x4E2830
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_4                0x4E2834
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_5                0x4E2838
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_6                0x4E283C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_7                0x4E2840
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_8                0x4E2844
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_9                0x4E2848
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_10               0x4E284C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_11               0x4E2850
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_12               0x4E2854
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_13               0x4E2858
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_14               0x4E285C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_15               0x4E2860
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0               0x4E2864
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_1               0x4E2868
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_2               0x4E286C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_3               0x4E2870
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_4               0x4E2874
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_5               0x4E2878
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_6               0x4E287C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_7               0x4E2880
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_8               0x4E2884
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_9               0x4E2888
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_10              0x4E288C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_11              0x4E2890
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_12              0x4E2894
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_13              0x4E2898
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_14              0x4E289C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_15              0x4E28A0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0                0x4E28A4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_1                0x4E28A8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_2                0x4E28AC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_3                0x4E28B0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_4                0x4E28B4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_5                0x4E28B8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_6                0x4E28BC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_7                0x4E28C0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_8                0x4E28C4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_9                0x4E28C8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_10               0x4E28CC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_11               0x4E28D0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_12               0x4E28D4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_13               0x4E28D8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_14               0x4E28DC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_15               0x4E28E0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0               0x4E28E4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_1               0x4E28E8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_2               0x4E28EC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_3               0x4E28F0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_4               0x4E28F4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_5               0x4E28F8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_6               0x4E28FC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_7               0x4E2900
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_8               0x4E2904
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_9               0x4E2908
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_10              0x4E290C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_11              0x4E2910
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_12              0x4E2914
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_13              0x4E2918
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_14              0x4E291C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_15              0x4E2920
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_0               0x4E2924
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_1               0x4E2928
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_2               0x4E292C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_3               0x4E2930
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_4               0x4E2934
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_5               0x4E2938
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_6               0x4E293C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_7               0x4E2940
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_8               0x4E2944
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_9               0x4E2948
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_10              0x4E294C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_11              0x4E2950
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_12              0x4E2954
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_13              0x4E2958
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_14              0x4E295C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_15              0x4E2960
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_0              0x4E2964
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_1              0x4E2968
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_2              0x4E296C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_3              0x4E2970
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_4              0x4E2974
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_5              0x4E2978
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_6              0x4E297C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_7              0x4E2980
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_8              0x4E2984
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_9              0x4E2988
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_10             0x4E298C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_11             0x4E2990
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_12             0x4E2994
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_13             0x4E2998
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_14             0x4E299C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_15             0x4E29A0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_0               0x4E29A4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_1               0x4E29A8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_2               0x4E29AC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_3               0x4E29B0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_4               0x4E29B4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_5               0x4E29B8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_6               0x4E29BC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_7               0x4E29C0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_8               0x4E29C4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_9               0x4E29C8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_10              0x4E29CC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_11              0x4E29D0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_12              0x4E29D4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_13              0x4E29D8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_14              0x4E29DC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_15              0x4E29E0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_0              0x4E29E4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_1              0x4E29E8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_2              0x4E29EC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_3              0x4E29F0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_4              0x4E29F4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_5              0x4E29F8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_6              0x4E29FC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_7              0x4E2A00
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_8              0x4E2A04
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_9              0x4E2A08
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_10             0x4E2A0C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_11             0x4E2A10
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_12             0x4E2A14
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_13             0x4E2A18
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_14             0x4E2A1C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_15             0x4E2A20
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_HIT_AW                       0x4E2A64
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_SEC_HIT_AR                       0x4E2A68
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_HIT_AW                      0x4E2A6C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RANGE_PRIV_HIT_AR                      0x4E2A70
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_CFG                                0x4E2B64
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_SHIFT                              0x4E2B68
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_0                     0x4E2B6C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_1                     0x4E2B70
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_2                     0x4E2B74
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_3                     0x4E2B78
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_4                     0x4E2B7C
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_5                     0x4E2B80
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_6                     0x4E2B84
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_EXPECTED_LAT_7                     0x4E2B88
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_0                            0x4E2BAC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_1                            0x4E2BB0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_2                            0x4E2BB4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_3                            0x4E2BB8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_4                            0x4E2BBC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_5                            0x4E2BC0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_6                            0x4E2BC4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_TOKEN_7                            0x4E2BC8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_0                          0x4E2BEC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_1                          0x4E2BF0
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_2                          0x4E2BF4
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_3                          0x4E2BF8
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_4                          0x4E2BFC
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_5                          0x4E2C00
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_6                          0x4E2C04
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_BANK_ID_7                          0x4E2C08
+
+#define mmDMA_IF_E_N_DOWN_CH1_RGL_WDT                                0x4E2C2C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_WRAP               0x4E2C30
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_WRAP               0x4E2C34
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_WRAP               0x4E2C38
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_WRAP               0x4E2C3C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_WRAP               0x4E2C40
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_WRAP               0x4E2C44
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_WRAP               0x4E2C48
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_WRAP               0x4E2C4C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_CNT                0x4E2C50
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_CNT                0x4E2C54
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_CNT                0x4E2C58
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_CNT                0x4E2C5C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_CNT                0x4E2C60
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_CNT                0x4E2C64
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_CNT                0x4E2C68
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_CNT                0x4E2C6C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_WRAP               0x4E2C70
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_WRAP               0x4E2C74
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_WRAP               0x4E2C78
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_WRAP               0x4E2C7C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_WRAP               0x4E2C80
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_WRAP               0x4E2C84
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_WRAP               0x4E2C88
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_WRAP               0x4E2C8C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_CNT                0x4E2C90
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_CNT                0x4E2C94
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_CNT                0x4E2C98
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_CNT                0x4E2C9C
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_CNT                0x4E2CA0
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_CNT                0x4E2CA4
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_CNT                0x4E2CA8
+
+#define mmDMA_IF_E_N_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_CNT                0x4E2CAC
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_PC_SEL_0                        0x4E2CB0
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_PC_SEL_1                        0x4E2CB4
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_PC_SEL_2                        0x4E2CB8
+
+#define mmDMA_IF_E_N_DOWN_CH1_NL_HBM_PC_SEL_3                        0x4E2CBC
+
+#endif /* ASIC_REG_DMA_IF_E_N_DOWN_CH1_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_regs.h
new file mode 100644
index 000000000000..8a10c6a76156
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_n_regs.h
@@ -0,0 +1,860 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_E_N_REGS_H_
+#define ASIC_REG_DMA_IF_E_N_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_E_N (Prototype: DMA_IF)
+ *****************************************
+ */
+
+#define mmDMA_IF_E_N_HBM0_WR_CRED_CNT                                0x4E0000
+
+#define mmDMA_IF_E_N_HBM1_WR_CRED_CNT                                0x4E0004
+
+#define mmDMA_IF_E_N_HBM0_RD_CRED_CNT                                0x4E0008
+
+#define mmDMA_IF_E_N_HBM1_RD_CRED_CNT                                0x4E000C
+
+#define mmDMA_IF_E_N_HBM_LIMITER_0                                   0x4E0030
+
+#define mmDMA_IF_E_N_HBM_LIMITER_1                                   0x4E0034
+
+#define mmDMA_IF_E_N_HBM_LIMITER_2                                   0x4E0038
+
+#define mmDMA_IF_E_N_HBM_LIMITER_3                                   0x4E003C
+
+#define mmDMA_IF_E_N_HBM_ALMOST_EN_0                                 0x4E0040
+
+#define mmDMA_IF_E_N_HBM_ALMOST_EN_1                                 0x4E0044
+
+#define mmDMA_IF_E_N_HBM_CRED_EN_0                                   0x4E0050
+
+#define mmDMA_IF_E_N_HBM_CRED_EN_1                                   0x4E0054
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_0                                 0x4E0100
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_1                                 0x4E0104
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_2                                 0x4E0108
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_3                                 0x4E010C
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_4                                 0x4E0110
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_5                                 0x4E0114
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_6                                 0x4E0118
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_7                                 0x4E011C
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_8                                 0x4E0120
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_9                                 0x4E0124
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_10                                0x4E0128
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_11                                0x4E012C
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_12                                0x4E0130
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_13                                0x4E0134
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_14                                0x4E0138
+
+#define mmDMA_IF_E_N_SOB_MIN_RPROT_15                                0x4E013C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_0                                 0x4E0140
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_1                                 0x4E0144
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_2                                 0x4E0148
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_3                                 0x4E014C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_4                                 0x4E0150
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_5                                 0x4E0154
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_6                                 0x4E0158
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_7                                 0x4E015C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_8                                 0x4E0160
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_9                                 0x4E0164
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_10                                0x4E0168
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_11                                0x4E016C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_12                                0x4E0170
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_13                                0x4E0174
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_14                                0x4E0178
+
+#define mmDMA_IF_E_N_SOB_MAX_RPROT_15                                0x4E017C
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_0                                 0x4E0180
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_1                                 0x4E0184
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_2                                 0x4E0188
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_3                                 0x4E018C
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_4                                 0x4E0190
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_5                                 0x4E0194
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_6                                 0x4E0198
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_7                                 0x4E019C
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_8                                 0x4E01A0
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_9                                 0x4E01A4
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_10                                0x4E01A8
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_11                                0x4E01AC
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_12                                0x4E01B0
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_13                                0x4E01B4
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_14                                0x4E01B8
+
+#define mmDMA_IF_E_N_SOB_MIN_WPROT_15                                0x4E01BC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_0                                 0x4E01C0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_1                                 0x4E01C4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_2                                 0x4E01C8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_3                                 0x4E01CC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_4                                 0x4E01D0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_5                                 0x4E01D4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_6                                 0x4E01D8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_7                                 0x4E01DC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_8                                 0x4E01E0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_9                                 0x4E01E4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_10                                0x4E01E8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_11                                0x4E01EC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_12                                0x4E01F0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_13                                0x4E01F4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_14                                0x4E01F8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPROT_15                                0x4E01FC
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_0                                 0x4E0200
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_1                                 0x4E0204
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_2                                 0x4E0208
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_3                                 0x4E020C
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_4                                 0x4E0210
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_5                                 0x4E0214
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_6                                 0x4E0218
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_7                                 0x4E021C
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_8                                 0x4E0220
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_9                                 0x4E0224
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_10                                0x4E0228
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_11                                0x4E022C
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_12                                0x4E0230
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_13                                0x4E0234
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_14                                0x4E0238
+
+#define mmDMA_IF_E_N_SOB_MIN_RPRIV_15                                0x4E023C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_0                                 0x4E0240
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_1                                 0x4E0244
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_2                                 0x4E0248
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_3                                 0x4E024C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_4                                 0x4E0250
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_5                                 0x4E0254
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_6                                 0x4E0258
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_7                                 0x4E025C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_8                                 0x4E0260
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_9                                 0x4E0264
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_10                                0x4E0268
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_11                                0x4E026C
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_12                                0x4E0270
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_13                                0x4E0274
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_14                                0x4E0278
+
+#define mmDMA_IF_E_N_SOB_MAX_RPRIV_15                                0x4E027C
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_0                                 0x4E0280
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_1                                 0x4E0284
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_2                                 0x4E0288
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_3                                 0x4E028C
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_4                                 0x4E0290
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_5                                 0x4E0294
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_6                                 0x4E0298
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_7                                 0x4E029C
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_8                                 0x4E02A0
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_9                                 0x4E02A4
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_10                                0x4E02A8
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_11                                0x4E02AC
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_12                                0x4E02B0
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_13                                0x4E02B4
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_14                                0x4E02B8
+
+#define mmDMA_IF_E_N_SOB_MIN_WPRIV_15                                0x4E02BC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_0                                 0x4E02C0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_1                                 0x4E02C4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_2                                 0x4E02C8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_3                                 0x4E02CC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_4                                 0x4E02D0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_5                                 0x4E02D4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_6                                 0x4E02D8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_7                                 0x4E02DC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_8                                 0x4E02E0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_9                                 0x4E02E4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_10                                0x4E02E8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_11                                0x4E02EC
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_12                                0x4E02F0
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_13                                0x4E02F4
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_14                                0x4E02F8
+
+#define mmDMA_IF_E_N_SOB_MAX_WPRIV_15                                0x4E02FC
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_0                                0x4E0300
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_1                                0x4E0304
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_2                                0x4E0308
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_3                                0x4E030C
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_4                                0x4E0310
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_5                                0x4E0314
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_6                                0x4E0318
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_7                                0x4E031C
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_8                                0x4E0320
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_9                                0x4E0324
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_10                               0x4E0328
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_11                               0x4E032C
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_12                               0x4E0330
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_13                               0x4E0334
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_14                               0x4E0338
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPROT_15                               0x4E033C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_0                                0x4E0340
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_1                                0x4E0344
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_2                                0x4E0348
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_3                                0x4E034C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_4                                0x4E0350
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_5                                0x4E0354
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_6                                0x4E0358
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_7                                0x4E035C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_8                                0x4E0360
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_9                                0x4E0364
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_10                               0x4E0368
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_11                               0x4E036C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_12                               0x4E0370
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_13                               0x4E0374
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_14                               0x4E0378
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPROT_15                               0x4E037C
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_0                                0x4E0380
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_1                                0x4E0384
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_2                                0x4E0388
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_3                                0x4E038C
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_4                                0x4E0390
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_5                                0x4E0394
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_6                                0x4E0398
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_7                                0x4E039C
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_8                                0x4E03A0
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_9                                0x4E03A4
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_10                               0x4E03A8
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_11                               0x4E03AC
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_12                               0x4E03B0
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_13                               0x4E03B4
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_14                               0x4E03B8
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPROT_15                               0x4E03BC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_0                                0x4E03C0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_1                                0x4E03C4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_2                                0x4E03C8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_3                                0x4E03CC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_4                                0x4E03D0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_5                                0x4E03D4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_6                                0x4E03D8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_7                                0x4E03DC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_8                                0x4E03E0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_9                                0x4E03E4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_10                               0x4E03E8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_11                               0x4E03EC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_12                               0x4E03F0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_13                               0x4E03F4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_14                               0x4E03F8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPROT_15                               0x4E03FC
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_0                                0x4E0400
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_1                                0x4E0404
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_2                                0x4E0408
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_3                                0x4E040C
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_4                                0x4E0410
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_5                                0x4E0414
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_6                                0x4E0418
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_7                                0x4E041C
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_8                                0x4E0420
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_9                                0x4E0424
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_10                               0x4E0428
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_11                               0x4E042C
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_12                               0x4E0430
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_13                               0x4E0434
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_14                               0x4E0438
+
+#define mmDMA_IF_E_N_DMA0_MIN_RPRIV_15                               0x4E043C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_0                                0x4E0440
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_1                                0x4E0444
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_2                                0x4E0448
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_3                                0x4E044C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_4                                0x4E0450
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_5                                0x4E0454
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_6                                0x4E0458
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_7                                0x4E045C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_8                                0x4E0460
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_9                                0x4E0464
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_10                               0x4E0468
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_11                               0x4E046C
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_12                               0x4E0470
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_13                               0x4E0474
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_14                               0x4E0478
+
+#define mmDMA_IF_E_N_DMA0_MAX_RPRIV_15                               0x4E047C
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_0                                0x4E0480
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_1                                0x4E0484
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_2                                0x4E0488
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_3                                0x4E048C
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_4                                0x4E0490
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_5                                0x4E0494
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_6                                0x4E0498
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_7                                0x4E049C
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_8                                0x4E04A0
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_9                                0x4E04A4
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_10                               0x4E04A8
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_11                               0x4E04AC
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_12                               0x4E04B0
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_13                               0x4E04B4
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_14                               0x4E04B8
+
+#define mmDMA_IF_E_N_DMA0_MIN_WPRIV_15                               0x4E04BC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_0                                0x4E04C0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_1                                0x4E04C4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_2                                0x4E04C8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_3                                0x4E04CC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_4                                0x4E04D0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_5                                0x4E04D4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_6                                0x4E04D8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_7                                0x4E04DC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_8                                0x4E04E0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_9                                0x4E04E4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_10                               0x4E04E8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_11                               0x4E04EC
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_12                               0x4E04F0
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_13                               0x4E04F4
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_14                               0x4E04F8
+
+#define mmDMA_IF_E_N_DMA0_MAX_WPRIV_15                               0x4E04FC
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_0                                0x4E0500
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_1                                0x4E0504
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_2                                0x4E0508
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_3                                0x4E050C
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_4                                0x4E0510
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_5                                0x4E0514
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_6                                0x4E0518
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_7                                0x4E051C
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_8                                0x4E0520
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_9                                0x4E0524
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_10                               0x4E0528
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_11                               0x4E052C
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_12                               0x4E0530
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_13                               0x4E0534
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_14                               0x4E0538
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPROT_15                               0x4E053C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_0                                0x4E0540
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_1                                0x4E0544
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_2                                0x4E0548
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_3                                0x4E054C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_4                                0x4E0550
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_5                                0x4E0554
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_6                                0x4E0558
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_7                                0x4E055C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_8                                0x4E0560
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_9                                0x4E0564
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_10                               0x4E0568
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_11                               0x4E056C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_12                               0x4E0570
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_13                               0x4E0574
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_14                               0x4E0578
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPROT_15                               0x4E057C
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_0                                0x4E0580
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_1                                0x4E0584
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_2                                0x4E0588
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_3                                0x4E058C
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_4                                0x4E0590
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_5                                0x4E0594
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_6                                0x4E0598
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_7                                0x4E059C
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_8                                0x4E05A0
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_9                                0x4E05A4
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_10                               0x4E05A8
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_11                               0x4E05AC
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_12                               0x4E05B0
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_13                               0x4E05B4
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_14                               0x4E05B8
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPROT_15                               0x4E05BC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_0                                0x4E05C0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_1                                0x4E05C4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_2                                0x4E05C8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_3                                0x4E05CC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_4                                0x4E05D0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_5                                0x4E05D4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_6                                0x4E05D8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_7                                0x4E05DC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_8                                0x4E05E0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_9                                0x4E05E4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_10                               0x4E05E8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_11                               0x4E05EC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_12                               0x4E05F0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_13                               0x4E05F4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_14                               0x4E05F8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPROT_15                               0x4E05FC
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_0                                0x4E0600
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_1                                0x4E0604
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_2                                0x4E0608
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_3                                0x4E060C
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_4                                0x4E0610
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_5                                0x4E0614
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_6                                0x4E0618
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_7                                0x4E061C
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_8                                0x4E0620
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_9                                0x4E0624
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_10                               0x4E0628
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_11                               0x4E062C
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_12                               0x4E0630
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_13                               0x4E0634
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_14                               0x4E0638
+
+#define mmDMA_IF_E_N_DMA1_MIN_RPRIV_15                               0x4E063C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_0                                0x4E0640
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_1                                0x4E0644
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_2                                0x4E0648
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_3                                0x4E064C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_4                                0x4E0650
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_5                                0x4E0654
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_6                                0x4E0658
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_7                                0x4E065C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_8                                0x4E0660
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_9                                0x4E0664
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_10                               0x4E0668
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_11                               0x4E066C
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_12                               0x4E0670
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_13                               0x4E0674
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_14                               0x4E0678
+
+#define mmDMA_IF_E_N_DMA1_MAX_RPRIV_15                               0x4E067C
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_0                                0x4E0680
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_1                                0x4E0684
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_2                                0x4E0688
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_3                                0x4E068C
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_4                                0x4E0690
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_5                                0x4E0694
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_6                                0x4E0698
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_7                                0x4E069C
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_8                                0x4E06A0
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_9                                0x4E06A4
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_10                               0x4E06A8
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_11                               0x4E06AC
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_12                               0x4E06B0
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_13                               0x4E06B4
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_14                               0x4E06B8
+
+#define mmDMA_IF_E_N_DMA1_MIN_WPRIV_15                               0x4E06BC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_0                                0x4E06C0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_1                                0x4E06C4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_2                                0x4E06C8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_3                                0x4E06CC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_4                                0x4E06D0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_5                                0x4E06D4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_6                                0x4E06D8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_7                                0x4E06DC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_8                                0x4E06E0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_9                                0x4E06E4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_10                               0x4E06E8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_11                               0x4E06EC
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_12                               0x4E06F0
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_13                               0x4E06F4
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_14                               0x4E06F8
+
+#define mmDMA_IF_E_N_DMA1_MAX_WPRIV_15                               0x4E06FC
+
+#define mmDMA_IF_E_N_SOB_HIT_RPROT                                   0x4E0700
+
+#define mmDMA_IF_E_N_SOB_HIT_WPROT                                   0x4E0704
+
+#define mmDMA_IF_E_N_SOB_HIT_RPRIV                                   0x4E070C
+
+#define mmDMA_IF_E_N_SOB_HIT_WPRIV                                   0x4E0710
+
+#define mmDMA_IF_E_N_DMA0_HIT_RPROT                                  0x4E071C
+
+#define mmDMA_IF_E_N_DMA0_HIT_WPROT                                  0x4E0720
+
+#define mmDMA_IF_E_N_DMA0_HIT_RPRIV                                  0x4E0724
+
+#define mmDMA_IF_E_N_DMA0_HIT_WPRIV                                  0x4E0728
+
+#define mmDMA_IF_E_N_DMA1_HIT_RPROT                                  0x4E0730
+
+#define mmDMA_IF_E_N_DMA1_HIT_WPROT                                  0x4E0734
+
+#define mmDMA_IF_E_N_DMA1_HIT_RPRIV                                  0x4E0738
+
+#define mmDMA_IF_E_N_DMA1_HIT_WPRIV                                  0x4E073C
+
+#define mmDMA_IF_E_N_HBM_BIN                                         0x4E0800
+
+#define mmDMA_IF_E_N_MME_BIN                                         0x4E0804
+
+#define mmDMA_IF_E_N_TPC_BIN                                         0x4E0808
+
+#define mmDMA_IF_E_N_DMA_BIN                                         0x4E080C
+
+#define mmDMA_IF_E_N_SOB_CG_EN                                       0x4E0810
+
+#define mmDMA_IF_E_N_HBM_I2C_ADDR_0                                  0x4E0820
+
+#define mmDMA_IF_E_N_HBM_I2C_ADDR_1                                  0x4E0824
+
+#define mmDMA_IF_E_N_HBM_I2C_ADDR_2                                  0x4E0828
+
+#define mmDMA_IF_E_N_HBM_I2C_ADDR_3                                  0x4E082C
+
+#define mmDMA_IF_E_N_HBM_I2C_ADDR_4                                  0x4E0830
+
+#define mmDMA_IF_E_N_HBM_MISC                                        0x4E0834
+
+#endif /* ASIC_REG_DMA_IF_E_N_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch0_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch0_regs.h
new file mode 100644
index 000000000000..cd61289a1e8a
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch0_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_E_S_DOWN_CH0_REGS_H_
+#define ASIC_REG_DMA_IF_E_S_DOWN_CH0_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_E_S_DOWN_CH0 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_E_S_DOWN_CH0_PERM_SEL                               0x4A1108
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_0                          0x4A1114
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_1                          0x4A1118
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_2                          0x4A111C
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_3                          0x4A1120
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_4                          0x4A1124
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_5                          0x4A1128
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_6                          0x4A112C
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_7                          0x4A1130
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_8                          0x4A1134
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_9                          0x4A1138
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_10                         0x4A113C
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_11                         0x4A1140
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_12                         0x4A1144
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_13                         0x4A1148
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_14                         0x4A114C
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_15                         0x4A1150
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_16                         0x4A1154
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_17                         0x4A1158
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_18                         0x4A115C
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_19                         0x4A1160
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_20                         0x4A1164
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_21                         0x4A1168
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_22                         0x4A116C
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_23                         0x4A1170
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_24                         0x4A1174
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_25                         0x4A1178
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_26                         0x4A117C
+
+#define mmDMA_IF_E_S_DOWN_CH0_HBM_POLY_H3_27                         0x4A1180
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_0                         0x4A1184
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_1                         0x4A1188
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_2                         0x4A118C
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_3                         0x4A1190
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_4                         0x4A1194
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_5                         0x4A1198
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_6                         0x4A119C
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_7                         0x4A11A0
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_8                         0x4A11A4
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_9                         0x4A11A8
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_10                        0x4A11AC
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_11                        0x4A11B0
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_12                        0x4A11B4
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_13                        0x4A11B8
+
+#define mmDMA_IF_E_S_DOWN_CH0_SRAM_POLY_H3_14                        0x4A11BC
+
+#define mmDMA_IF_E_S_DOWN_CH0_SCRAM_SRAM_EN                          0x4A126C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_HBM_EN                              0x4A1274
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_HBM_SAT                             0x4A1278
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_HBM_RST                             0x4A127C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_HBM_TIMEOUT                         0x4A1280
+
+#define mmDMA_IF_E_S_DOWN_CH0_SCRAM_HBM_EN                           0x4A1284
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_PCI_EN                              0x4A1288
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_PCI_SAT                             0x4A128C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_PCI_RST                             0x4A1290
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_PCI_TIMEOUT                         0x4A1294
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_EN                             0x4A129C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_SAT                            0x4A12A0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_RST                            0x4A12A4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_TIMEOUT                        0x4A12AC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RL_SRAM_RED                            0x4A12B4
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_EN                             0x4A12EC
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_EN                             0x4A12F0
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_WR_SIZE                        0x4A12F4
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_WR_SIZE                        0x4A12F8
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_PCI_CTR_SET_EN                  0x4A1404
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_PCI_CTR_SET                     0x4A1408
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_PCI_CTR_WRAP                    0x4A140C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_PCI_CTR_CNT                     0x4A1410
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM_CTR_SET_EN                  0x4A1414
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM_CTR_SET                     0x4A1418
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_RD_SIZE                        0x4A141C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_RD_SIZE                        0x4A1420
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_PCI_CTR_SET_EN                  0x4A1424
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_PCI_CTR_SET                     0x4A1428
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_PCI_CTR_WRAP                    0x4A142C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_PCI_CTR_CNT                     0x4A1430
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM_CTR_SET_EN                  0x4A1434
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM_CTR_SET                     0x4A1438
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_0                           0x4A1450
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_SEL_1                           0x4A1454
+
+#define mmDMA_IF_E_S_DOWN_CH0_NON_LIN_EN                             0x4A1480
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_BANK_0                         0x4A1500
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_BANK_1                         0x4A1504
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_BANK_2                         0x4A1508
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_BANK_3                         0x4A150C
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_BANK_4                         0x4A1510
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_0                       0x4A1514
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_1                       0x4A1520
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_2                       0x4A1524
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_3                       0x4A1528
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_4                       0x4A152C
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_5                       0x4A1530
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_6                       0x4A1534
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_7                       0x4A1538
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_8                       0x4A153C
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_SRAM_OFFSET_9                       0x4A1540
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_0                        0x4A1550
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_1                        0x4A1554
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_2                        0x4A1558
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_3                        0x4A155C
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_4                        0x4A1560
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_5                        0x4A1564
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_6                        0x4A1568
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_7                        0x4A156C
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_8                        0x4A1570
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_9                        0x4A1574
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_10                       0x4A1578
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_11                       0x4A157C
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_12                       0x4A1580
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_13                       0x4A1584
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_14                       0x4A1588
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_15                       0x4A158C
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_16                       0x4A1590
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_17                       0x4A1594
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_OFFSET_18                       0x4A1598
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0                0x4A15E4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_1                0x4A15E8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_2                0x4A15EC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_3                0x4A15F0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_4                0x4A15F4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_5                0x4A15F8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_6                0x4A15FC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_7                0x4A1600
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_8                0x4A1604
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_9                0x4A1608
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_10               0x4A160C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_11               0x4A1610
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_12               0x4A1614
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_13               0x4A1618
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_14               0x4A161C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_15               0x4A1620
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0               0x4A1624
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_1               0x4A1628
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_2               0x4A162C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_3               0x4A1630
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_4               0x4A1634
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_5               0x4A1638
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_6               0x4A163C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_7               0x4A1640
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_8               0x4A1644
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_9               0x4A1648
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_10              0x4A164C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_11              0x4A1650
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_12              0x4A1654
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_13              0x4A1658
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_14              0x4A165C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_15              0x4A1660
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0                0x4A1664
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_1                0x4A1668
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_2                0x4A166C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_3                0x4A1670
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_4                0x4A1674
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_5                0x4A1678
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_6                0x4A167C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_7                0x4A1680
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_8                0x4A1684
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_9                0x4A1688
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_10               0x4A168C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_11               0x4A1690
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_12               0x4A1694
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_13               0x4A1698
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_14               0x4A169C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_15               0x4A16A0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0               0x4A16A4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_1               0x4A16A8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_2               0x4A16AC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_3               0x4A16B0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_4               0x4A16B4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_5               0x4A16B8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_6               0x4A16BC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_7               0x4A16C0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_8               0x4A16C4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_9               0x4A16C8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_10              0x4A16CC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_11              0x4A16D0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_12              0x4A16D4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_13              0x4A16D8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_14              0x4A16DC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_15              0x4A16E0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_0               0x4A16E4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_1               0x4A16E8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_2               0x4A16EC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_3               0x4A16F0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_4               0x4A16F4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_5               0x4A16F8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_6               0x4A16FC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_7               0x4A1700
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_8               0x4A1704
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_9               0x4A1708
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_10              0x4A170C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_11              0x4A1710
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_12              0x4A1714
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_13              0x4A1718
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_14              0x4A171C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_15              0x4A1720
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_0              0x4A1724
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_1              0x4A1728
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_2              0x4A172C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_3              0x4A1730
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_4              0x4A1734
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_5              0x4A1738
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_6              0x4A173C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_7              0x4A1740
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_8              0x4A1744
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_9              0x4A1748
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_10             0x4A174C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_11             0x4A1750
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_12             0x4A1754
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_13             0x4A1758
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_14             0x4A175C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_15             0x4A1760
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_0               0x4A1764
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_1               0x4A1768
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_2               0x4A176C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_3               0x4A1770
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_4               0x4A1774
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_5               0x4A1778
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_6               0x4A177C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_7               0x4A1780
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_8               0x4A1784
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_9               0x4A1788
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_10              0x4A178C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_11              0x4A1790
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_12              0x4A1794
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_13              0x4A1798
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_14              0x4A179C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_15              0x4A17A0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_0              0x4A17A4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_1              0x4A17A8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_2              0x4A17AC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_3              0x4A17B0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_4              0x4A17B4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_5              0x4A17B8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_6              0x4A17BC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_7              0x4A17C0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_8              0x4A17C4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_9              0x4A17C8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_10             0x4A17CC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_11             0x4A17D0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_12             0x4A17D4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_13             0x4A17D8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_14             0x4A17DC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_15             0x4A17E0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0                0x4A1824
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_1                0x4A1828
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_2                0x4A182C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_3                0x4A1830
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_4                0x4A1834
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_5                0x4A1838
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_6                0x4A183C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_7                0x4A1840
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_8                0x4A1844
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_9                0x4A1848
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_10               0x4A184C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_11               0x4A1850
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_12               0x4A1854
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_13               0x4A1858
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_14               0x4A185C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_15               0x4A1860
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0               0x4A1864
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_1               0x4A1868
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_2               0x4A186C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_3               0x4A1870
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_4               0x4A1874
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_5               0x4A1878
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_6               0x4A187C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_7               0x4A1880
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_8               0x4A1884
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_9               0x4A1888
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_10              0x4A188C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_11              0x4A1890
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_12              0x4A1894
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_13              0x4A1898
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_14              0x4A189C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_15              0x4A18A0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0                0x4A18A4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_1                0x4A18A8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_2                0x4A18AC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_3                0x4A18B0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_4                0x4A18B4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_5                0x4A18B8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_6                0x4A18BC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_7                0x4A18C0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_8                0x4A18C4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_9                0x4A18C8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_10               0x4A18CC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_11               0x4A18D0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_12               0x4A18D4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_13               0x4A18D8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_14               0x4A18DC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_15               0x4A18E0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0               0x4A18E4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_1               0x4A18E8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_2               0x4A18EC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_3               0x4A18F0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_4               0x4A18F4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_5               0x4A18F8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_6               0x4A18FC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_7               0x4A1900
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_8               0x4A1904
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_9               0x4A1908
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_10              0x4A190C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_11              0x4A1910
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_12              0x4A1914
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_13              0x4A1918
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_14              0x4A191C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_15              0x4A1920
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_0               0x4A1924
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_1               0x4A1928
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_2               0x4A192C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_3               0x4A1930
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_4               0x4A1934
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_5               0x4A1938
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_6               0x4A193C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_7               0x4A1940
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_8               0x4A1944
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_9               0x4A1948
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_10              0x4A194C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_11              0x4A1950
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_12              0x4A1954
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_13              0x4A1958
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_14              0x4A195C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_15              0x4A1960
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_0              0x4A1964
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_1              0x4A1968
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_2              0x4A196C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_3              0x4A1970
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_4              0x4A1974
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_5              0x4A1978
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_6              0x4A197C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_7              0x4A1980
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_8              0x4A1984
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_9              0x4A1988
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_10             0x4A198C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_11             0x4A1990
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_12             0x4A1994
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_13             0x4A1998
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_14             0x4A199C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_15             0x4A19A0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_0               0x4A19A4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_1               0x4A19A8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_2               0x4A19AC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_3               0x4A19B0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_4               0x4A19B4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_5               0x4A19B8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_6               0x4A19BC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_7               0x4A19C0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_8               0x4A19C4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_9               0x4A19C8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_10              0x4A19CC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_11              0x4A19D0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_12              0x4A19D4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_13              0x4A19D8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_14              0x4A19DC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_15              0x4A19E0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_0              0x4A19E4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_1              0x4A19E8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_2              0x4A19EC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_3              0x4A19F0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_4              0x4A19F4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_5              0x4A19F8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_6              0x4A19FC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_7              0x4A1A00
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_8              0x4A1A04
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_9              0x4A1A08
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_10             0x4A1A0C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_11             0x4A1A10
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_12             0x4A1A14
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_13             0x4A1A18
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_14             0x4A1A1C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_15             0x4A1A20
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_HIT_AW                       0x4A1A64
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_SEC_HIT_AR                       0x4A1A68
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_HIT_AW                      0x4A1A6C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RANGE_PRIV_HIT_AR                      0x4A1A70
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_CFG                                0x4A1B64
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_SHIFT                              0x4A1B68
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_0                     0x4A1B6C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_1                     0x4A1B70
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_2                     0x4A1B74
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_3                     0x4A1B78
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_4                     0x4A1B7C
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_5                     0x4A1B80
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_6                     0x4A1B84
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_EXPECTED_LAT_7                     0x4A1B88
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_0                            0x4A1BAC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_1                            0x4A1BB0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_2                            0x4A1BB4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_3                            0x4A1BB8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_4                            0x4A1BBC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_5                            0x4A1BC0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_6                            0x4A1BC4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_TOKEN_7                            0x4A1BC8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_0                          0x4A1BEC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_1                          0x4A1BF0
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_2                          0x4A1BF4
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_3                          0x4A1BF8
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_4                          0x4A1BFC
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_5                          0x4A1C00
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_6                          0x4A1C04
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_BANK_ID_7                          0x4A1C08
+
+#define mmDMA_IF_E_S_DOWN_CH0_RGL_WDT                                0x4A1C2C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_WRAP               0x4A1C30
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_WRAP               0x4A1C34
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_WRAP               0x4A1C38
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_WRAP               0x4A1C3C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_WRAP               0x4A1C40
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_WRAP               0x4A1C44
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_WRAP               0x4A1C48
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_WRAP               0x4A1C4C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_CNT                0x4A1C50
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_CNT                0x4A1C54
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_CNT                0x4A1C58
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_CNT                0x4A1C5C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_CNT                0x4A1C60
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_CNT                0x4A1C64
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_CNT                0x4A1C68
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_CNT                0x4A1C6C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_WRAP               0x4A1C70
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_WRAP               0x4A1C74
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_WRAP               0x4A1C78
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_WRAP               0x4A1C7C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_WRAP               0x4A1C80
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_WRAP               0x4A1C84
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_WRAP               0x4A1C88
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_WRAP               0x4A1C8C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_CNT                0x4A1C90
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_CNT                0x4A1C94
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_CNT                0x4A1C98
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_CNT                0x4A1C9C
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_CNT                0x4A1CA0
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_CNT                0x4A1CA4
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_CNT                0x4A1CA8
+
+#define mmDMA_IF_E_S_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_CNT                0x4A1CAC
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_PC_SEL_0                        0x4A1CB0
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_PC_SEL_1                        0x4A1CB4
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_PC_SEL_2                        0x4A1CB8
+
+#define mmDMA_IF_E_S_DOWN_CH0_NL_HBM_PC_SEL_3                        0x4A1CBC
+
+#endif /* ASIC_REG_DMA_IF_E_S_DOWN_CH0_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch1_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch1_regs.h
new file mode 100644
index 000000000000..3f32370a14c7
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_down_ch1_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_E_S_DOWN_CH1_REGS_H_
+#define ASIC_REG_DMA_IF_E_S_DOWN_CH1_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_E_S_DOWN_CH1 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_E_S_DOWN_CH1_PERM_SEL                               0x4A2108
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_0                          0x4A2114
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_1                          0x4A2118
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_2                          0x4A211C
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_3                          0x4A2120
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_4                          0x4A2124
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_5                          0x4A2128
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_6                          0x4A212C
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_7                          0x4A2130
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_8                          0x4A2134
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_9                          0x4A2138
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_10                         0x4A213C
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_11                         0x4A2140
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_12                         0x4A2144
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_13                         0x4A2148
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_14                         0x4A214C
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_15                         0x4A2150
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_16                         0x4A2154
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_17                         0x4A2158
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_18                         0x4A215C
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_19                         0x4A2160
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_20                         0x4A2164
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_21                         0x4A2168
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_22                         0x4A216C
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_23                         0x4A2170
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_24                         0x4A2174
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_25                         0x4A2178
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_26                         0x4A217C
+
+#define mmDMA_IF_E_S_DOWN_CH1_HBM_POLY_H3_27                         0x4A2180
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_0                         0x4A2184
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_1                         0x4A2188
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_2                         0x4A218C
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_3                         0x4A2190
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_4                         0x4A2194
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_5                         0x4A2198
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_6                         0x4A219C
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_7                         0x4A21A0
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_8                         0x4A21A4
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_9                         0x4A21A8
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_10                        0x4A21AC
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_11                        0x4A21B0
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_12                        0x4A21B4
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_13                        0x4A21B8
+
+#define mmDMA_IF_E_S_DOWN_CH1_SRAM_POLY_H3_14                        0x4A21BC
+
+#define mmDMA_IF_E_S_DOWN_CH1_SCRAM_SRAM_EN                          0x4A226C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_HBM_EN                              0x4A2274
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_HBM_SAT                             0x4A2278
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_HBM_RST                             0x4A227C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_HBM_TIMEOUT                         0x4A2280
+
+#define mmDMA_IF_E_S_DOWN_CH1_SCRAM_HBM_EN                           0x4A2284
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_PCI_EN                              0x4A2288
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_PCI_SAT                             0x4A228C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_PCI_RST                             0x4A2290
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_PCI_TIMEOUT                         0x4A2294
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_EN                             0x4A229C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_SAT                            0x4A22A0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_RST                            0x4A22A4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_TIMEOUT                        0x4A22AC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RL_SRAM_RED                            0x4A22B4
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_EN                             0x4A22EC
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_EN                             0x4A22F0
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_WR_SIZE                        0x4A22F4
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_WR_SIZE                        0x4A22F8
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_PCI_CTR_SET_EN                  0x4A2404
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_PCI_CTR_SET                     0x4A2408
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_PCI_CTR_WRAP                    0x4A240C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_PCI_CTR_CNT                     0x4A2410
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM_CTR_SET_EN                  0x4A2414
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM_CTR_SET                     0x4A2418
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_RD_SIZE                        0x4A241C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_RD_SIZE                        0x4A2420
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_PCI_CTR_SET_EN                  0x4A2424
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_PCI_CTR_SET                     0x4A2428
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_PCI_CTR_WRAP                    0x4A242C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_PCI_CTR_CNT                     0x4A2430
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM_CTR_SET_EN                  0x4A2434
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM_CTR_SET                     0x4A2438
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_0                           0x4A2450
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_SEL_1                           0x4A2454
+
+#define mmDMA_IF_E_S_DOWN_CH1_NON_LIN_EN                             0x4A2480
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_BANK_0                         0x4A2500
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_BANK_1                         0x4A2504
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_BANK_2                         0x4A2508
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_BANK_3                         0x4A250C
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_BANK_4                         0x4A2510
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_0                       0x4A2514
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_1                       0x4A2520
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_2                       0x4A2524
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_3                       0x4A2528
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_4                       0x4A252C
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_5                       0x4A2530
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_6                       0x4A2534
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_7                       0x4A2538
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_8                       0x4A253C
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_SRAM_OFFSET_9                       0x4A2540
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_0                        0x4A2550
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_1                        0x4A2554
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_2                        0x4A2558
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_3                        0x4A255C
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_4                        0x4A2560
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_5                        0x4A2564
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_6                        0x4A2568
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_7                        0x4A256C
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_8                        0x4A2570
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_9                        0x4A2574
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_10                       0x4A2578
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_11                       0x4A257C
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_12                       0x4A2580
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_13                       0x4A2584
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_14                       0x4A2588
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_15                       0x4A258C
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_16                       0x4A2590
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_17                       0x4A2594
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_OFFSET_18                       0x4A2598
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0                0x4A25E4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_1                0x4A25E8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_2                0x4A25EC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_3                0x4A25F0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_4                0x4A25F4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_5                0x4A25F8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_6                0x4A25FC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_7                0x4A2600
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_8                0x4A2604
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_9                0x4A2608
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_10               0x4A260C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_11               0x4A2610
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_12               0x4A2614
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_13               0x4A2618
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_14               0x4A261C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_15               0x4A2620
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0               0x4A2624
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_1               0x4A2628
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_2               0x4A262C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_3               0x4A2630
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_4               0x4A2634
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_5               0x4A2638
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_6               0x4A263C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_7               0x4A2640
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_8               0x4A2644
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_9               0x4A2648
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_10              0x4A264C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_11              0x4A2650
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_12              0x4A2654
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_13              0x4A2658
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_14              0x4A265C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_15              0x4A2660
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0                0x4A2664
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_1                0x4A2668
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_2                0x4A266C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_3                0x4A2670
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_4                0x4A2674
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_5                0x4A2678
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_6                0x4A267C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_7                0x4A2680
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_8                0x4A2684
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_9                0x4A2688
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_10               0x4A268C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_11               0x4A2690
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_12               0x4A2694
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_13               0x4A2698
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_14               0x4A269C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_15               0x4A26A0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0               0x4A26A4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_1               0x4A26A8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_2               0x4A26AC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_3               0x4A26B0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_4               0x4A26B4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_5               0x4A26B8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_6               0x4A26BC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_7               0x4A26C0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_8               0x4A26C4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_9               0x4A26C8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_10              0x4A26CC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_11              0x4A26D0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_12              0x4A26D4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_13              0x4A26D8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_14              0x4A26DC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_15              0x4A26E0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_0               0x4A26E4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_1               0x4A26E8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_2               0x4A26EC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_3               0x4A26F0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_4               0x4A26F4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_5               0x4A26F8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_6               0x4A26FC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_7               0x4A2700
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_8               0x4A2704
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_9               0x4A2708
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_10              0x4A270C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_11              0x4A2710
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_12              0x4A2714
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_13              0x4A2718
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_14              0x4A271C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_15              0x4A2720
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_0              0x4A2724
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_1              0x4A2728
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_2              0x4A272C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_3              0x4A2730
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_4              0x4A2734
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_5              0x4A2738
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_6              0x4A273C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_7              0x4A2740
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_8              0x4A2744
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_9              0x4A2748
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_10             0x4A274C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_11             0x4A2750
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_12             0x4A2754
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_13             0x4A2758
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_14             0x4A275C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_15             0x4A2760
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_0               0x4A2764
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_1               0x4A2768
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_2               0x4A276C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_3               0x4A2770
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_4               0x4A2774
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_5               0x4A2778
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_6               0x4A277C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_7               0x4A2780
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_8               0x4A2784
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_9               0x4A2788
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_10              0x4A278C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_11              0x4A2790
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_12              0x4A2794
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_13              0x4A2798
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_14              0x4A279C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_15              0x4A27A0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_0              0x4A27A4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_1              0x4A27A8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_2              0x4A27AC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_3              0x4A27B0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_4              0x4A27B4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_5              0x4A27B8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_6              0x4A27BC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_7              0x4A27C0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_8              0x4A27C4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_9              0x4A27C8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_10             0x4A27CC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_11             0x4A27D0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_12             0x4A27D4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_13             0x4A27D8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_14             0x4A27DC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_15             0x4A27E0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0                0x4A2824
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_1                0x4A2828
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_2                0x4A282C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_3                0x4A2830
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_4                0x4A2834
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_5                0x4A2838
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_6                0x4A283C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_7                0x4A2840
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_8                0x4A2844
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_9                0x4A2848
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_10               0x4A284C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_11               0x4A2850
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_12               0x4A2854
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_13               0x4A2858
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_14               0x4A285C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_15               0x4A2860
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0               0x4A2864
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_1               0x4A2868
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_2               0x4A286C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_3               0x4A2870
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_4               0x4A2874
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_5               0x4A2878
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_6               0x4A287C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_7               0x4A2880
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_8               0x4A2884
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_9               0x4A2888
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_10              0x4A288C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_11              0x4A2890
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_12              0x4A2894
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_13              0x4A2898
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_14              0x4A289C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_15              0x4A28A0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0                0x4A28A4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_1                0x4A28A8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_2                0x4A28AC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_3                0x4A28B0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_4                0x4A28B4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_5                0x4A28B8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_6                0x4A28BC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_7                0x4A28C0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_8                0x4A28C4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_9                0x4A28C8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_10               0x4A28CC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_11               0x4A28D0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_12               0x4A28D4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_13               0x4A28D8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_14               0x4A28DC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_15               0x4A28E0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0               0x4A28E4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_1               0x4A28E8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_2               0x4A28EC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_3               0x4A28F0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_4               0x4A28F4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_5               0x4A28F8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_6               0x4A28FC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_7               0x4A2900
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_8               0x4A2904
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_9               0x4A2908
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_10              0x4A290C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_11              0x4A2910
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_12              0x4A2914
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_13              0x4A2918
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_14              0x4A291C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_15              0x4A2920
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_0               0x4A2924
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_1               0x4A2928
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_2               0x4A292C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_3               0x4A2930
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_4               0x4A2934
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_5               0x4A2938
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_6               0x4A293C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_7               0x4A2940
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_8               0x4A2944
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_9               0x4A2948
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_10              0x4A294C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_11              0x4A2950
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_12              0x4A2954
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_13              0x4A2958
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_14              0x4A295C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_15              0x4A2960
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_0              0x4A2964
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_1              0x4A2968
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_2              0x4A296C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_3              0x4A2970
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_4              0x4A2974
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_5              0x4A2978
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_6              0x4A297C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_7              0x4A2980
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_8              0x4A2984
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_9              0x4A2988
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_10             0x4A298C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_11             0x4A2990
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_12             0x4A2994
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_13             0x4A2998
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_14             0x4A299C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_15             0x4A29A0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_0               0x4A29A4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_1               0x4A29A8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_2               0x4A29AC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_3               0x4A29B0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_4               0x4A29B4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_5               0x4A29B8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_6               0x4A29BC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_7               0x4A29C0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_8               0x4A29C4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_9               0x4A29C8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_10              0x4A29CC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_11              0x4A29D0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_12              0x4A29D4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_13              0x4A29D8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_14              0x4A29DC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_15              0x4A29E0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_0              0x4A29E4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_1              0x4A29E8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_2              0x4A29EC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_3              0x4A29F0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_4              0x4A29F4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_5              0x4A29F8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_6              0x4A29FC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_7              0x4A2A00
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_8              0x4A2A04
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_9              0x4A2A08
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_10             0x4A2A0C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_11             0x4A2A10
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_12             0x4A2A14
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_13             0x4A2A18
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_14             0x4A2A1C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_15             0x4A2A20
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_HIT_AW                       0x4A2A64
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_SEC_HIT_AR                       0x4A2A68
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_HIT_AW                      0x4A2A6C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RANGE_PRIV_HIT_AR                      0x4A2A70
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_CFG                                0x4A2B64
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_SHIFT                              0x4A2B68
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_0                     0x4A2B6C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_1                     0x4A2B70
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_2                     0x4A2B74
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_3                     0x4A2B78
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_4                     0x4A2B7C
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_5                     0x4A2B80
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_6                     0x4A2B84
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_EXPECTED_LAT_7                     0x4A2B88
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_0                            0x4A2BAC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_1                            0x4A2BB0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_2                            0x4A2BB4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_3                            0x4A2BB8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_4                            0x4A2BBC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_5                            0x4A2BC0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_6                            0x4A2BC4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_TOKEN_7                            0x4A2BC8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_0                          0x4A2BEC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_1                          0x4A2BF0
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_2                          0x4A2BF4
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_3                          0x4A2BF8
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_4                          0x4A2BFC
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_5                          0x4A2C00
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_6                          0x4A2C04
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_BANK_ID_7                          0x4A2C08
+
+#define mmDMA_IF_E_S_DOWN_CH1_RGL_WDT                                0x4A2C2C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_WRAP               0x4A2C30
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_WRAP               0x4A2C34
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_WRAP               0x4A2C38
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_WRAP               0x4A2C3C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_WRAP               0x4A2C40
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_WRAP               0x4A2C44
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_WRAP               0x4A2C48
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_WRAP               0x4A2C4C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_CNT                0x4A2C50
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_CNT                0x4A2C54
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_CNT                0x4A2C58
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_CNT                0x4A2C5C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_CNT                0x4A2C60
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_CNT                0x4A2C64
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_CNT                0x4A2C68
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_CNT                0x4A2C6C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_WRAP               0x4A2C70
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_WRAP               0x4A2C74
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_WRAP               0x4A2C78
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_WRAP               0x4A2C7C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_WRAP               0x4A2C80
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_WRAP               0x4A2C84
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_WRAP               0x4A2C88
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_WRAP               0x4A2C8C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_CNT                0x4A2C90
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_CNT                0x4A2C94
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_CNT                0x4A2C98
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_CNT                0x4A2C9C
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_CNT                0x4A2CA0
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_CNT                0x4A2CA4
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_CNT                0x4A2CA8
+
+#define mmDMA_IF_E_S_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_CNT                0x4A2CAC
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_PC_SEL_0                        0x4A2CB0
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_PC_SEL_1                        0x4A2CB4
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_PC_SEL_2                        0x4A2CB8
+
+#define mmDMA_IF_E_S_DOWN_CH1_NL_HBM_PC_SEL_3                        0x4A2CBC
+
+#endif /* ASIC_REG_DMA_IF_E_S_DOWN_CH1_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_regs.h
new file mode 100644
index 000000000000..78c18da7154b
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_e_s_regs.h
@@ -0,0 +1,860 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_E_S_REGS_H_
+#define ASIC_REG_DMA_IF_E_S_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_E_S (Prototype: DMA_IF)
+ *****************************************
+ */
+
+#define mmDMA_IF_E_S_HBM0_WR_CRED_CNT                                0x4A0000
+
+#define mmDMA_IF_E_S_HBM1_WR_CRED_CNT                                0x4A0004
+
+#define mmDMA_IF_E_S_HBM0_RD_CRED_CNT                                0x4A0008
+
+#define mmDMA_IF_E_S_HBM1_RD_CRED_CNT                                0x4A000C
+
+#define mmDMA_IF_E_S_HBM_LIMITER_0                                   0x4A0030
+
+#define mmDMA_IF_E_S_HBM_LIMITER_1                                   0x4A0034
+
+#define mmDMA_IF_E_S_HBM_LIMITER_2                                   0x4A0038
+
+#define mmDMA_IF_E_S_HBM_LIMITER_3                                   0x4A003C
+
+#define mmDMA_IF_E_S_HBM_ALMOST_EN_0                                 0x4A0040
+
+#define mmDMA_IF_E_S_HBM_ALMOST_EN_1                                 0x4A0044
+
+#define mmDMA_IF_E_S_HBM_CRED_EN_0                                   0x4A0050
+
+#define mmDMA_IF_E_S_HBM_CRED_EN_1                                   0x4A0054
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_0                                 0x4A0100
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_1                                 0x4A0104
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_2                                 0x4A0108
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_3                                 0x4A010C
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_4                                 0x4A0110
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_5                                 0x4A0114
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_6                                 0x4A0118
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_7                                 0x4A011C
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_8                                 0x4A0120
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_9                                 0x4A0124
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_10                                0x4A0128
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_11                                0x4A012C
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_12                                0x4A0130
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_13                                0x4A0134
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_14                                0x4A0138
+
+#define mmDMA_IF_E_S_SOB_MIN_RPROT_15                                0x4A013C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_0                                 0x4A0140
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_1                                 0x4A0144
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_2                                 0x4A0148
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_3                                 0x4A014C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_4                                 0x4A0150
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_5                                 0x4A0154
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_6                                 0x4A0158
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_7                                 0x4A015C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_8                                 0x4A0160
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_9                                 0x4A0164
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_10                                0x4A0168
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_11                                0x4A016C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_12                                0x4A0170
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_13                                0x4A0174
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_14                                0x4A0178
+
+#define mmDMA_IF_E_S_SOB_MAX_RPROT_15                                0x4A017C
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_0                                 0x4A0180
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_1                                 0x4A0184
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_2                                 0x4A0188
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_3                                 0x4A018C
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_4                                 0x4A0190
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_5                                 0x4A0194
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_6                                 0x4A0198
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_7                                 0x4A019C
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_8                                 0x4A01A0
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_9                                 0x4A01A4
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_10                                0x4A01A8
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_11                                0x4A01AC
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_12                                0x4A01B0
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_13                                0x4A01B4
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_14                                0x4A01B8
+
+#define mmDMA_IF_E_S_SOB_MIN_WPROT_15                                0x4A01BC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_0                                 0x4A01C0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_1                                 0x4A01C4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_2                                 0x4A01C8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_3                                 0x4A01CC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_4                                 0x4A01D0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_5                                 0x4A01D4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_6                                 0x4A01D8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_7                                 0x4A01DC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_8                                 0x4A01E0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_9                                 0x4A01E4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_10                                0x4A01E8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_11                                0x4A01EC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_12                                0x4A01F0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_13                                0x4A01F4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_14                                0x4A01F8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPROT_15                                0x4A01FC
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_0                                 0x4A0200
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_1                                 0x4A0204
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_2                                 0x4A0208
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_3                                 0x4A020C
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_4                                 0x4A0210
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_5                                 0x4A0214
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_6                                 0x4A0218
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_7                                 0x4A021C
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_8                                 0x4A0220
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_9                                 0x4A0224
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_10                                0x4A0228
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_11                                0x4A022C
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_12                                0x4A0230
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_13                                0x4A0234
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_14                                0x4A0238
+
+#define mmDMA_IF_E_S_SOB_MIN_RPRIV_15                                0x4A023C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_0                                 0x4A0240
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_1                                 0x4A0244
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_2                                 0x4A0248
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_3                                 0x4A024C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_4                                 0x4A0250
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_5                                 0x4A0254
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_6                                 0x4A0258
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_7                                 0x4A025C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_8                                 0x4A0260
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_9                                 0x4A0264
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_10                                0x4A0268
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_11                                0x4A026C
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_12                                0x4A0270
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_13                                0x4A0274
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_14                                0x4A0278
+
+#define mmDMA_IF_E_S_SOB_MAX_RPRIV_15                                0x4A027C
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_0                                 0x4A0280
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_1                                 0x4A0284
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_2                                 0x4A0288
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_3                                 0x4A028C
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_4                                 0x4A0290
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_5                                 0x4A0294
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_6                                 0x4A0298
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_7                                 0x4A029C
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_8                                 0x4A02A0
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_9                                 0x4A02A4
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_10                                0x4A02A8
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_11                                0x4A02AC
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_12                                0x4A02B0
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_13                                0x4A02B4
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_14                                0x4A02B8
+
+#define mmDMA_IF_E_S_SOB_MIN_WPRIV_15                                0x4A02BC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_0                                 0x4A02C0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_1                                 0x4A02C4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_2                                 0x4A02C8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_3                                 0x4A02CC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_4                                 0x4A02D0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_5                                 0x4A02D4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_6                                 0x4A02D8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_7                                 0x4A02DC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_8                                 0x4A02E0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_9                                 0x4A02E4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_10                                0x4A02E8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_11                                0x4A02EC
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_12                                0x4A02F0
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_13                                0x4A02F4
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_14                                0x4A02F8
+
+#define mmDMA_IF_E_S_SOB_MAX_WPRIV_15                                0x4A02FC
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_0                                0x4A0300
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_1                                0x4A0304
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_2                                0x4A0308
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_3                                0x4A030C
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_4                                0x4A0310
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_5                                0x4A0314
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_6                                0x4A0318
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_7                                0x4A031C
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_8                                0x4A0320
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_9                                0x4A0324
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_10                               0x4A0328
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_11                               0x4A032C
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_12                               0x4A0330
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_13                               0x4A0334
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_14                               0x4A0338
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPROT_15                               0x4A033C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_0                                0x4A0340
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_1                                0x4A0344
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_2                                0x4A0348
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_3                                0x4A034C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_4                                0x4A0350
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_5                                0x4A0354
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_6                                0x4A0358
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_7                                0x4A035C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_8                                0x4A0360
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_9                                0x4A0364
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_10                               0x4A0368
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_11                               0x4A036C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_12                               0x4A0370
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_13                               0x4A0374
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_14                               0x4A0378
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPROT_15                               0x4A037C
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_0                                0x4A0380
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_1                                0x4A0384
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_2                                0x4A0388
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_3                                0x4A038C
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_4                                0x4A0390
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_5                                0x4A0394
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_6                                0x4A0398
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_7                                0x4A039C
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_8                                0x4A03A0
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_9                                0x4A03A4
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_10                               0x4A03A8
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_11                               0x4A03AC
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_12                               0x4A03B0
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_13                               0x4A03B4
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_14                               0x4A03B8
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPROT_15                               0x4A03BC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_0                                0x4A03C0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_1                                0x4A03C4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_2                                0x4A03C8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_3                                0x4A03CC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_4                                0x4A03D0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_5                                0x4A03D4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_6                                0x4A03D8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_7                                0x4A03DC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_8                                0x4A03E0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_9                                0x4A03E4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_10                               0x4A03E8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_11                               0x4A03EC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_12                               0x4A03F0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_13                               0x4A03F4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_14                               0x4A03F8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPROT_15                               0x4A03FC
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_0                                0x4A0400
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_1                                0x4A0404
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_2                                0x4A0408
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_3                                0x4A040C
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_4                                0x4A0410
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_5                                0x4A0414
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_6                                0x4A0418
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_7                                0x4A041C
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_8                                0x4A0420
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_9                                0x4A0424
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_10                               0x4A0428
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_11                               0x4A042C
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_12                               0x4A0430
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_13                               0x4A0434
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_14                               0x4A0438
+
+#define mmDMA_IF_E_S_DMA0_MIN_RPRIV_15                               0x4A043C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_0                                0x4A0440
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_1                                0x4A0444
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_2                                0x4A0448
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_3                                0x4A044C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_4                                0x4A0450
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_5                                0x4A0454
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_6                                0x4A0458
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_7                                0x4A045C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_8                                0x4A0460
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_9                                0x4A0464
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_10                               0x4A0468
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_11                               0x4A046C
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_12                               0x4A0470
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_13                               0x4A0474
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_14                               0x4A0478
+
+#define mmDMA_IF_E_S_DMA0_MAX_RPRIV_15                               0x4A047C
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_0                                0x4A0480
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_1                                0x4A0484
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_2                                0x4A0488
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_3                                0x4A048C
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_4                                0x4A0490
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_5                                0x4A0494
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_6                                0x4A0498
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_7                                0x4A049C
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_8                                0x4A04A0
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_9                                0x4A04A4
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_10                               0x4A04A8
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_11                               0x4A04AC
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_12                               0x4A04B0
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_13                               0x4A04B4
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_14                               0x4A04B8
+
+#define mmDMA_IF_E_S_DMA0_MIN_WPRIV_15                               0x4A04BC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_0                                0x4A04C0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_1                                0x4A04C4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_2                                0x4A04C8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_3                                0x4A04CC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_4                                0x4A04D0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_5                                0x4A04D4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_6                                0x4A04D8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_7                                0x4A04DC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_8                                0x4A04E0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_9                                0x4A04E4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_10                               0x4A04E8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_11                               0x4A04EC
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_12                               0x4A04F0
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_13                               0x4A04F4
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_14                               0x4A04F8
+
+#define mmDMA_IF_E_S_DMA0_MAX_WPRIV_15                               0x4A04FC
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_0                                0x4A0500
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_1                                0x4A0504
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_2                                0x4A0508
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_3                                0x4A050C
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_4                                0x4A0510
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_5                                0x4A0514
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_6                                0x4A0518
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_7                                0x4A051C
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_8                                0x4A0520
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_9                                0x4A0524
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_10                               0x4A0528
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_11                               0x4A052C
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_12                               0x4A0530
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_13                               0x4A0534
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_14                               0x4A0538
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPROT_15                               0x4A053C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_0                                0x4A0540
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_1                                0x4A0544
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_2                                0x4A0548
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_3                                0x4A054C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_4                                0x4A0550
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_5                                0x4A0554
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_6                                0x4A0558
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_7                                0x4A055C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_8                                0x4A0560
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_9                                0x4A0564
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_10                               0x4A0568
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_11                               0x4A056C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_12                               0x4A0570
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_13                               0x4A0574
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_14                               0x4A0578
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPROT_15                               0x4A057C
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_0                                0x4A0580
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_1                                0x4A0584
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_2                                0x4A0588
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_3                                0x4A058C
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_4                                0x4A0590
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_5                                0x4A0594
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_6                                0x4A0598
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_7                                0x4A059C
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_8                                0x4A05A0
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_9                                0x4A05A4
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_10                               0x4A05A8
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_11                               0x4A05AC
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_12                               0x4A05B0
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_13                               0x4A05B4
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_14                               0x4A05B8
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPROT_15                               0x4A05BC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_0                                0x4A05C0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_1                                0x4A05C4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_2                                0x4A05C8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_3                                0x4A05CC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_4                                0x4A05D0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_5                                0x4A05D4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_6                                0x4A05D8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_7                                0x4A05DC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_8                                0x4A05E0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_9                                0x4A05E4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_10                               0x4A05E8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_11                               0x4A05EC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_12                               0x4A05F0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_13                               0x4A05F4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_14                               0x4A05F8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPROT_15                               0x4A05FC
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_0                                0x4A0600
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_1                                0x4A0604
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_2                                0x4A0608
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_3                                0x4A060C
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_4                                0x4A0610
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_5                                0x4A0614
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_6                                0x4A0618
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_7                                0x4A061C
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_8                                0x4A0620
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_9                                0x4A0624
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_10                               0x4A0628
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_11                               0x4A062C
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_12                               0x4A0630
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_13                               0x4A0634
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_14                               0x4A0638
+
+#define mmDMA_IF_E_S_DMA1_MIN_RPRIV_15                               0x4A063C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_0                                0x4A0640
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_1                                0x4A0644
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_2                                0x4A0648
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_3                                0x4A064C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_4                                0x4A0650
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_5                                0x4A0654
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_6                                0x4A0658
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_7                                0x4A065C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_8                                0x4A0660
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_9                                0x4A0664
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_10                               0x4A0668
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_11                               0x4A066C
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_12                               0x4A0670
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_13                               0x4A0674
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_14                               0x4A0678
+
+#define mmDMA_IF_E_S_DMA1_MAX_RPRIV_15                               0x4A067C
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_0                                0x4A0680
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_1                                0x4A0684
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_2                                0x4A0688
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_3                                0x4A068C
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_4                                0x4A0690
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_5                                0x4A0694
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_6                                0x4A0698
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_7                                0x4A069C
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_8                                0x4A06A0
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_9                                0x4A06A4
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_10                               0x4A06A8
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_11                               0x4A06AC
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_12                               0x4A06B0
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_13                               0x4A06B4
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_14                               0x4A06B8
+
+#define mmDMA_IF_E_S_DMA1_MIN_WPRIV_15                               0x4A06BC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_0                                0x4A06C0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_1                                0x4A06C4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_2                                0x4A06C8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_3                                0x4A06CC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_4                                0x4A06D0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_5                                0x4A06D4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_6                                0x4A06D8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_7                                0x4A06DC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_8                                0x4A06E0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_9                                0x4A06E4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_10                               0x4A06E8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_11                               0x4A06EC
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_12                               0x4A06F0
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_13                               0x4A06F4
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_14                               0x4A06F8
+
+#define mmDMA_IF_E_S_DMA1_MAX_WPRIV_15                               0x4A06FC
+
+#define mmDMA_IF_E_S_SOB_HIT_RPROT                                   0x4A0700
+
+#define mmDMA_IF_E_S_SOB_HIT_WPROT                                   0x4A0704
+
+#define mmDMA_IF_E_S_SOB_HIT_RPRIV                                   0x4A070C
+
+#define mmDMA_IF_E_S_SOB_HIT_WPRIV                                   0x4A0710
+
+#define mmDMA_IF_E_S_DMA0_HIT_RPROT                                  0x4A071C
+
+#define mmDMA_IF_E_S_DMA0_HIT_WPROT                                  0x4A0720
+
+#define mmDMA_IF_E_S_DMA0_HIT_RPRIV                                  0x4A0724
+
+#define mmDMA_IF_E_S_DMA0_HIT_WPRIV                                  0x4A0728
+
+#define mmDMA_IF_E_S_DMA1_HIT_RPROT                                  0x4A0730
+
+#define mmDMA_IF_E_S_DMA1_HIT_WPROT                                  0x4A0734
+
+#define mmDMA_IF_E_S_DMA1_HIT_RPRIV                                  0x4A0738
+
+#define mmDMA_IF_E_S_DMA1_HIT_WPRIV                                  0x4A073C
+
+#define mmDMA_IF_E_S_HBM_BIN                                         0x4A0800
+
+#define mmDMA_IF_E_S_MME_BIN                                         0x4A0804
+
+#define mmDMA_IF_E_S_TPC_BIN                                         0x4A0808
+
+#define mmDMA_IF_E_S_DMA_BIN                                         0x4A080C
+
+#define mmDMA_IF_E_S_SOB_CG_EN                                       0x4A0810
+
+#define mmDMA_IF_E_S_HBM_I2C_ADDR_0                                  0x4A0820
+
+#define mmDMA_IF_E_S_HBM_I2C_ADDR_1                                  0x4A0824
+
+#define mmDMA_IF_E_S_HBM_I2C_ADDR_2                                  0x4A0828
+
+#define mmDMA_IF_E_S_HBM_I2C_ADDR_3                                  0x4A082C
+
+#define mmDMA_IF_E_S_HBM_I2C_ADDR_4                                  0x4A0830
+
+#define mmDMA_IF_E_S_HBM_MISC                                        0x4A0834
+
+#endif /* ASIC_REG_DMA_IF_E_S_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch0_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch0_regs.h
new file mode 100644
index 000000000000..4ccaf8712948
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch0_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_W_N_DOWN_CH0_REGS_H_
+#define ASIC_REG_DMA_IF_W_N_DOWN_CH0_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_W_N_DOWN_CH0 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_W_N_DOWN_CH0_PERM_SEL                               0x4C1108
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_0                          0x4C1114
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_1                          0x4C1118
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_2                          0x4C111C
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_3                          0x4C1120
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_4                          0x4C1124
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_5                          0x4C1128
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_6                          0x4C112C
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_7                          0x4C1130
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_8                          0x4C1134
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_9                          0x4C1138
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_10                         0x4C113C
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_11                         0x4C1140
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_12                         0x4C1144
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_13                         0x4C1148
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_14                         0x4C114C
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_15                         0x4C1150
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_16                         0x4C1154
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_17                         0x4C1158
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_18                         0x4C115C
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_19                         0x4C1160
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_20                         0x4C1164
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_21                         0x4C1168
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_22                         0x4C116C
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_23                         0x4C1170
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_24                         0x4C1174
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_25                         0x4C1178
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_26                         0x4C117C
+
+#define mmDMA_IF_W_N_DOWN_CH0_HBM_POLY_H3_27                         0x4C1180
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_0                         0x4C1184
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_1                         0x4C1188
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_2                         0x4C118C
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_3                         0x4C1190
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_4                         0x4C1194
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_5                         0x4C1198
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_6                         0x4C119C
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_7                         0x4C11A0
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_8                         0x4C11A4
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_9                         0x4C11A8
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_10                        0x4C11AC
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_11                        0x4C11B0
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_12                        0x4C11B4
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_13                        0x4C11B8
+
+#define mmDMA_IF_W_N_DOWN_CH0_SRAM_POLY_H3_14                        0x4C11BC
+
+#define mmDMA_IF_W_N_DOWN_CH0_SCRAM_SRAM_EN                          0x4C126C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_HBM_EN                              0x4C1274
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_HBM_SAT                             0x4C1278
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_HBM_RST                             0x4C127C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_HBM_TIMEOUT                         0x4C1280
+
+#define mmDMA_IF_W_N_DOWN_CH0_SCRAM_HBM_EN                           0x4C1284
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_PCI_EN                              0x4C1288
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_PCI_SAT                             0x4C128C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_PCI_RST                             0x4C1290
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_PCI_TIMEOUT                         0x4C1294
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_EN                             0x4C129C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_SAT                            0x4C12A0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_RST                            0x4C12A4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_TIMEOUT                        0x4C12AC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RL_SRAM_RED                            0x4C12B4
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_EN                             0x4C12EC
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_EN                             0x4C12F0
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_WR_SIZE                        0x4C12F4
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_WR_SIZE                        0x4C12F8
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_PCI_CTR_SET_EN                  0x4C1404
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_PCI_CTR_SET                     0x4C1408
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_PCI_CTR_WRAP                    0x4C140C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_PCI_CTR_CNT                     0x4C1410
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM_CTR_SET_EN                  0x4C1414
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM_CTR_SET                     0x4C1418
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_RD_SIZE                        0x4C141C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_RD_SIZE                        0x4C1420
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_PCI_CTR_SET_EN                  0x4C1424
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_PCI_CTR_SET                     0x4C1428
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_PCI_CTR_WRAP                    0x4C142C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_PCI_CTR_CNT                     0x4C1430
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM_CTR_SET_EN                  0x4C1434
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM_CTR_SET                     0x4C1438
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_0                           0x4C1450
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_SEL_1                           0x4C1454
+
+#define mmDMA_IF_W_N_DOWN_CH0_NON_LIN_EN                             0x4C1480
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_BANK_0                         0x4C1500
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_BANK_1                         0x4C1504
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_BANK_2                         0x4C1508
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_BANK_3                         0x4C150C
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_BANK_4                         0x4C1510
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_0                       0x4C1514
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_1                       0x4C1520
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_2                       0x4C1524
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_3                       0x4C1528
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_4                       0x4C152C
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_5                       0x4C1530
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_6                       0x4C1534
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_7                       0x4C1538
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_8                       0x4C153C
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_SRAM_OFFSET_9                       0x4C1540
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_0                        0x4C1550
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_1                        0x4C1554
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_2                        0x4C1558
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_3                        0x4C155C
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_4                        0x4C1560
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_5                        0x4C1564
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_6                        0x4C1568
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_7                        0x4C156C
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_8                        0x4C1570
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_9                        0x4C1574
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_10                       0x4C1578
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_11                       0x4C157C
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_12                       0x4C1580
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_13                       0x4C1584
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_14                       0x4C1588
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_15                       0x4C158C
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_16                       0x4C1590
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_17                       0x4C1594
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_OFFSET_18                       0x4C1598
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0                0x4C15E4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_1                0x4C15E8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_2                0x4C15EC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_3                0x4C15F0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_4                0x4C15F4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_5                0x4C15F8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_6                0x4C15FC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_7                0x4C1600
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_8                0x4C1604
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_9                0x4C1608
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_10               0x4C160C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_11               0x4C1610
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_12               0x4C1614
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_13               0x4C1618
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_14               0x4C161C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_15               0x4C1620
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0               0x4C1624
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_1               0x4C1628
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_2               0x4C162C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_3               0x4C1630
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_4               0x4C1634
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_5               0x4C1638
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_6               0x4C163C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_7               0x4C1640
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_8               0x4C1644
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_9               0x4C1648
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_10              0x4C164C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_11              0x4C1650
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_12              0x4C1654
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_13              0x4C1658
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_14              0x4C165C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_15              0x4C1660
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0                0x4C1664
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_1                0x4C1668
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_2                0x4C166C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_3                0x4C1670
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_4                0x4C1674
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_5                0x4C1678
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_6                0x4C167C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_7                0x4C1680
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_8                0x4C1684
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_9                0x4C1688
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_10               0x4C168C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_11               0x4C1690
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_12               0x4C1694
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_13               0x4C1698
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_14               0x4C169C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_15               0x4C16A0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0               0x4C16A4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_1               0x4C16A8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_2               0x4C16AC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_3               0x4C16B0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_4               0x4C16B4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_5               0x4C16B8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_6               0x4C16BC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_7               0x4C16C0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_8               0x4C16C4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_9               0x4C16C8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_10              0x4C16CC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_11              0x4C16D0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_12              0x4C16D4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_13              0x4C16D8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_14              0x4C16DC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_15              0x4C16E0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_0               0x4C16E4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_1               0x4C16E8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_2               0x4C16EC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_3               0x4C16F0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_4               0x4C16F4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_5               0x4C16F8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_6               0x4C16FC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_7               0x4C1700
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_8               0x4C1704
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_9               0x4C1708
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_10              0x4C170C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_11              0x4C1710
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_12              0x4C1714
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_13              0x4C1718
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_14              0x4C171C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_15              0x4C1720
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_0              0x4C1724
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_1              0x4C1728
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_2              0x4C172C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_3              0x4C1730
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_4              0x4C1734
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_5              0x4C1738
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_6              0x4C173C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_7              0x4C1740
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_8              0x4C1744
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_9              0x4C1748
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_10             0x4C174C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_11             0x4C1750
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_12             0x4C1754
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_13             0x4C1758
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_14             0x4C175C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_15             0x4C1760
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_0               0x4C1764
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_1               0x4C1768
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_2               0x4C176C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_3               0x4C1770
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_4               0x4C1774
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_5               0x4C1778
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_6               0x4C177C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_7               0x4C1780
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_8               0x4C1784
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_9               0x4C1788
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_10              0x4C178C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_11              0x4C1790
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_12              0x4C1794
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_13              0x4C1798
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_14              0x4C179C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_15              0x4C17A0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_0              0x4C17A4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_1              0x4C17A8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_2              0x4C17AC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_3              0x4C17B0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_4              0x4C17B4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_5              0x4C17B8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_6              0x4C17BC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_7              0x4C17C0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_8              0x4C17C4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_9              0x4C17C8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_10             0x4C17CC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_11             0x4C17D0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_12             0x4C17D4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_13             0x4C17D8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_14             0x4C17DC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_15             0x4C17E0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0                0x4C1824
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_1                0x4C1828
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_2                0x4C182C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_3                0x4C1830
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_4                0x4C1834
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_5                0x4C1838
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_6                0x4C183C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_7                0x4C1840
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_8                0x4C1844
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_9                0x4C1848
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_10               0x4C184C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_11               0x4C1850
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_12               0x4C1854
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_13               0x4C1858
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_14               0x4C185C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_15               0x4C1860
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0               0x4C1864
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_1               0x4C1868
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_2               0x4C186C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_3               0x4C1870
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_4               0x4C1874
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_5               0x4C1878
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_6               0x4C187C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_7               0x4C1880
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_8               0x4C1884
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_9               0x4C1888
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_10              0x4C188C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_11              0x4C1890
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_12              0x4C1894
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_13              0x4C1898
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_14              0x4C189C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_15              0x4C18A0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0                0x4C18A4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_1                0x4C18A8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_2                0x4C18AC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_3                0x4C18B0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_4                0x4C18B4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_5                0x4C18B8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_6                0x4C18BC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_7                0x4C18C0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_8                0x4C18C4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_9                0x4C18C8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_10               0x4C18CC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_11               0x4C18D0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_12               0x4C18D4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_13               0x4C18D8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_14               0x4C18DC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_15               0x4C18E0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0               0x4C18E4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_1               0x4C18E8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_2               0x4C18EC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_3               0x4C18F0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_4               0x4C18F4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_5               0x4C18F8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_6               0x4C18FC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_7               0x4C1900
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_8               0x4C1904
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_9               0x4C1908
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_10              0x4C190C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_11              0x4C1910
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_12              0x4C1914
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_13              0x4C1918
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_14              0x4C191C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_15              0x4C1920
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_0               0x4C1924
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_1               0x4C1928
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_2               0x4C192C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_3               0x4C1930
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_4               0x4C1934
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_5               0x4C1938
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_6               0x4C193C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_7               0x4C1940
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_8               0x4C1944
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_9               0x4C1948
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_10              0x4C194C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_11              0x4C1950
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_12              0x4C1954
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_13              0x4C1958
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_14              0x4C195C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_15              0x4C1960
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_0              0x4C1964
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_1              0x4C1968
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_2              0x4C196C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_3              0x4C1970
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_4              0x4C1974
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_5              0x4C1978
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_6              0x4C197C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_7              0x4C1980
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_8              0x4C1984
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_9              0x4C1988
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_10             0x4C198C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_11             0x4C1990
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_12             0x4C1994
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_13             0x4C1998
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_14             0x4C199C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_15             0x4C19A0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_0               0x4C19A4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_1               0x4C19A8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_2               0x4C19AC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_3               0x4C19B0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_4               0x4C19B4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_5               0x4C19B8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_6               0x4C19BC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_7               0x4C19C0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_8               0x4C19C4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_9               0x4C19C8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_10              0x4C19CC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_11              0x4C19D0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_12              0x4C19D4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_13              0x4C19D8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_14              0x4C19DC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_15              0x4C19E0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_0              0x4C19E4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_1              0x4C19E8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_2              0x4C19EC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_3              0x4C19F0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_4              0x4C19F4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_5              0x4C19F8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_6              0x4C19FC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_7              0x4C1A00
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_8              0x4C1A04
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_9              0x4C1A08
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_10             0x4C1A0C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_11             0x4C1A10
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_12             0x4C1A14
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_13             0x4C1A18
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_14             0x4C1A1C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_15             0x4C1A20
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_HIT_AW                       0x4C1A64
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_SEC_HIT_AR                       0x4C1A68
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_HIT_AW                      0x4C1A6C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RANGE_PRIV_HIT_AR                      0x4C1A70
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_CFG                                0x4C1B64
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_SHIFT                              0x4C1B68
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_0                     0x4C1B6C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_1                     0x4C1B70
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_2                     0x4C1B74
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_3                     0x4C1B78
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_4                     0x4C1B7C
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_5                     0x4C1B80
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_6                     0x4C1B84
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_EXPECTED_LAT_7                     0x4C1B88
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_0                            0x4C1BAC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_1                            0x4C1BB0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_2                            0x4C1BB4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_3                            0x4C1BB8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_4                            0x4C1BBC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_5                            0x4C1BC0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_6                            0x4C1BC4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_TOKEN_7                            0x4C1BC8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_0                          0x4C1BEC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_1                          0x4C1BF0
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_2                          0x4C1BF4
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_3                          0x4C1BF8
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_4                          0x4C1BFC
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_5                          0x4C1C00
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_6                          0x4C1C04
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_BANK_ID_7                          0x4C1C08
+
+#define mmDMA_IF_W_N_DOWN_CH0_RGL_WDT                                0x4C1C2C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_WRAP               0x4C1C30
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_WRAP               0x4C1C34
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_WRAP               0x4C1C38
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_WRAP               0x4C1C3C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_WRAP               0x4C1C40
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_WRAP               0x4C1C44
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_WRAP               0x4C1C48
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_WRAP               0x4C1C4C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_CNT                0x4C1C50
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_CNT                0x4C1C54
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_CNT                0x4C1C58
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_CNT                0x4C1C5C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_CNT                0x4C1C60
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_CNT                0x4C1C64
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_CNT                0x4C1C68
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_CNT                0x4C1C6C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_WRAP               0x4C1C70
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_WRAP               0x4C1C74
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_WRAP               0x4C1C78
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_WRAP               0x4C1C7C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_WRAP               0x4C1C80
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_WRAP               0x4C1C84
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_WRAP               0x4C1C88
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_WRAP               0x4C1C8C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_CNT                0x4C1C90
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_CNT                0x4C1C94
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_CNT                0x4C1C98
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_CNT                0x4C1C9C
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_CNT                0x4C1CA0
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_CNT                0x4C1CA4
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_CNT                0x4C1CA8
+
+#define mmDMA_IF_W_N_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_CNT                0x4C1CAC
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_PC_SEL_0                        0x4C1CB0
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_PC_SEL_1                        0x4C1CB4
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_PC_SEL_2                        0x4C1CB8
+
+#define mmDMA_IF_W_N_DOWN_CH0_NL_HBM_PC_SEL_3                        0x4C1CBC
+
+#endif /* ASIC_REG_DMA_IF_W_N_DOWN_CH0_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch1_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch1_regs.h
new file mode 100644
index 000000000000..9236f4183084
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_down_ch1_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_W_N_DOWN_CH1_REGS_H_
+#define ASIC_REG_DMA_IF_W_N_DOWN_CH1_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_W_N_DOWN_CH1 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_W_N_DOWN_CH1_PERM_SEL                               0x4C2108
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_0                          0x4C2114
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_1                          0x4C2118
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_2                          0x4C211C
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_3                          0x4C2120
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_4                          0x4C2124
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_5                          0x4C2128
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_6                          0x4C212C
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_7                          0x4C2130
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_8                          0x4C2134
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_9                          0x4C2138
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_10                         0x4C213C
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_11                         0x4C2140
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_12                         0x4C2144
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_13                         0x4C2148
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_14                         0x4C214C
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_15                         0x4C2150
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_16                         0x4C2154
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_17                         0x4C2158
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_18                         0x4C215C
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_19                         0x4C2160
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_20                         0x4C2164
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_21                         0x4C2168
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_22                         0x4C216C
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_23                         0x4C2170
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_24                         0x4C2174
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_25                         0x4C2178
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_26                         0x4C217C
+
+#define mmDMA_IF_W_N_DOWN_CH1_HBM_POLY_H3_27                         0x4C2180
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_0                         0x4C2184
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_1                         0x4C2188
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_2                         0x4C218C
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_3                         0x4C2190
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_4                         0x4C2194
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_5                         0x4C2198
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_6                         0x4C219C
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_7                         0x4C21A0
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_8                         0x4C21A4
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_9                         0x4C21A8
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_10                        0x4C21AC
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_11                        0x4C21B0
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_12                        0x4C21B4
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_13                        0x4C21B8
+
+#define mmDMA_IF_W_N_DOWN_CH1_SRAM_POLY_H3_14                        0x4C21BC
+
+#define mmDMA_IF_W_N_DOWN_CH1_SCRAM_SRAM_EN                          0x4C226C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_HBM_EN                              0x4C2274
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_HBM_SAT                             0x4C2278
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_HBM_RST                             0x4C227C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_HBM_TIMEOUT                         0x4C2280
+
+#define mmDMA_IF_W_N_DOWN_CH1_SCRAM_HBM_EN                           0x4C2284
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_PCI_EN                              0x4C2288
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_PCI_SAT                             0x4C228C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_PCI_RST                             0x4C2290
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_PCI_TIMEOUT                         0x4C2294
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_EN                             0x4C229C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_SAT                            0x4C22A0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_RST                            0x4C22A4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_TIMEOUT                        0x4C22AC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RL_SRAM_RED                            0x4C22B4
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_EN                             0x4C22EC
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_EN                             0x4C22F0
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_WR_SIZE                        0x4C22F4
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_WR_SIZE                        0x4C22F8
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_PCI_CTR_SET_EN                  0x4C2404
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_PCI_CTR_SET                     0x4C2408
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_PCI_CTR_WRAP                    0x4C240C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_PCI_CTR_CNT                     0x4C2410
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM_CTR_SET_EN                  0x4C2414
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM_CTR_SET                     0x4C2418
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_RD_SIZE                        0x4C241C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_RD_SIZE                        0x4C2420
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_PCI_CTR_SET_EN                  0x4C2424
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_PCI_CTR_SET                     0x4C2428
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_PCI_CTR_WRAP                    0x4C242C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_PCI_CTR_CNT                     0x4C2430
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM_CTR_SET_EN                  0x4C2434
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM_CTR_SET                     0x4C2438
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_0                           0x4C2450
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_SEL_1                           0x4C2454
+
+#define mmDMA_IF_W_N_DOWN_CH1_NON_LIN_EN                             0x4C2480
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_BANK_0                         0x4C2500
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_BANK_1                         0x4C2504
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_BANK_2                         0x4C2508
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_BANK_3                         0x4C250C
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_BANK_4                         0x4C2510
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_0                       0x4C2514
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_1                       0x4C2520
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_2                       0x4C2524
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_3                       0x4C2528
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_4                       0x4C252C
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_5                       0x4C2530
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_6                       0x4C2534
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_7                       0x4C2538
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_8                       0x4C253C
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_SRAM_OFFSET_9                       0x4C2540
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_0                        0x4C2550
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_1                        0x4C2554
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_2                        0x4C2558
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_3                        0x4C255C
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_4                        0x4C2560
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_5                        0x4C2564
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_6                        0x4C2568
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_7                        0x4C256C
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_8                        0x4C2570
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_9                        0x4C2574
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_10                       0x4C2578
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_11                       0x4C257C
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_12                       0x4C2580
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_13                       0x4C2584
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_14                       0x4C2588
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_15                       0x4C258C
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_16                       0x4C2590
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_17                       0x4C2594
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_OFFSET_18                       0x4C2598
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0                0x4C25E4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_1                0x4C25E8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_2                0x4C25EC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_3                0x4C25F0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_4                0x4C25F4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_5                0x4C25F8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_6                0x4C25FC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_7                0x4C2600
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_8                0x4C2604
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_9                0x4C2608
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_10               0x4C260C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_11               0x4C2610
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_12               0x4C2614
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_13               0x4C2618
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_14               0x4C261C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_15               0x4C2620
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0               0x4C2624
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_1               0x4C2628
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_2               0x4C262C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_3               0x4C2630
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_4               0x4C2634
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_5               0x4C2638
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_6               0x4C263C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_7               0x4C2640
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_8               0x4C2644
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_9               0x4C2648
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_10              0x4C264C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_11              0x4C2650
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_12              0x4C2654
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_13              0x4C2658
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_14              0x4C265C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_15              0x4C2660
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0                0x4C2664
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_1                0x4C2668
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_2                0x4C266C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_3                0x4C2670
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_4                0x4C2674
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_5                0x4C2678
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_6                0x4C267C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_7                0x4C2680
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_8                0x4C2684
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_9                0x4C2688
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_10               0x4C268C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_11               0x4C2690
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_12               0x4C2694
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_13               0x4C2698
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_14               0x4C269C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_15               0x4C26A0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0               0x4C26A4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_1               0x4C26A8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_2               0x4C26AC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_3               0x4C26B0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_4               0x4C26B4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_5               0x4C26B8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_6               0x4C26BC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_7               0x4C26C0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_8               0x4C26C4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_9               0x4C26C8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_10              0x4C26CC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_11              0x4C26D0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_12              0x4C26D4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_13              0x4C26D8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_14              0x4C26DC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_15              0x4C26E0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_0               0x4C26E4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_1               0x4C26E8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_2               0x4C26EC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_3               0x4C26F0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_4               0x4C26F4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_5               0x4C26F8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_6               0x4C26FC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_7               0x4C2700
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_8               0x4C2704
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_9               0x4C2708
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_10              0x4C270C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_11              0x4C2710
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_12              0x4C2714
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_13              0x4C2718
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_14              0x4C271C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_15              0x4C2720
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_0              0x4C2724
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_1              0x4C2728
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_2              0x4C272C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_3              0x4C2730
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_4              0x4C2734
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_5              0x4C2738
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_6              0x4C273C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_7              0x4C2740
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_8              0x4C2744
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_9              0x4C2748
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_10             0x4C274C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_11             0x4C2750
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_12             0x4C2754
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_13             0x4C2758
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_14             0x4C275C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_15             0x4C2760
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_0               0x4C2764
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_1               0x4C2768
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_2               0x4C276C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_3               0x4C2770
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_4               0x4C2774
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_5               0x4C2778
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_6               0x4C277C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_7               0x4C2780
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_8               0x4C2784
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_9               0x4C2788
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_10              0x4C278C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_11              0x4C2790
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_12              0x4C2794
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_13              0x4C2798
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_14              0x4C279C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_15              0x4C27A0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_0              0x4C27A4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_1              0x4C27A8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_2              0x4C27AC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_3              0x4C27B0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_4              0x4C27B4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_5              0x4C27B8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_6              0x4C27BC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_7              0x4C27C0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_8              0x4C27C4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_9              0x4C27C8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_10             0x4C27CC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_11             0x4C27D0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_12             0x4C27D4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_13             0x4C27D8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_14             0x4C27DC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_15             0x4C27E0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0                0x4C2824
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_1                0x4C2828
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_2                0x4C282C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_3                0x4C2830
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_4                0x4C2834
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_5                0x4C2838
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_6                0x4C283C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_7                0x4C2840
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_8                0x4C2844
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_9                0x4C2848
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_10               0x4C284C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_11               0x4C2850
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_12               0x4C2854
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_13               0x4C2858
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_14               0x4C285C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_15               0x4C2860
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0               0x4C2864
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_1               0x4C2868
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_2               0x4C286C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_3               0x4C2870
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_4               0x4C2874
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_5               0x4C2878
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_6               0x4C287C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_7               0x4C2880
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_8               0x4C2884
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_9               0x4C2888
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_10              0x4C288C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_11              0x4C2890
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_12              0x4C2894
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_13              0x4C2898
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_14              0x4C289C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_15              0x4C28A0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0                0x4C28A4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_1                0x4C28A8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_2                0x4C28AC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_3                0x4C28B0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_4                0x4C28B4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_5                0x4C28B8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_6                0x4C28BC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_7                0x4C28C0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_8                0x4C28C4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_9                0x4C28C8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_10               0x4C28CC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_11               0x4C28D0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_12               0x4C28D4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_13               0x4C28D8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_14               0x4C28DC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_15               0x4C28E0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0               0x4C28E4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_1               0x4C28E8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_2               0x4C28EC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_3               0x4C28F0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_4               0x4C28F4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_5               0x4C28F8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_6               0x4C28FC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_7               0x4C2900
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_8               0x4C2904
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_9               0x4C2908
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_10              0x4C290C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_11              0x4C2910
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_12              0x4C2914
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_13              0x4C2918
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_14              0x4C291C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_15              0x4C2920
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_0               0x4C2924
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_1               0x4C2928
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_2               0x4C292C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_3               0x4C2930
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_4               0x4C2934
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_5               0x4C2938
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_6               0x4C293C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_7               0x4C2940
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_8               0x4C2944
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_9               0x4C2948
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_10              0x4C294C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_11              0x4C2950
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_12              0x4C2954
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_13              0x4C2958
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_14              0x4C295C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_15              0x4C2960
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_0              0x4C2964
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_1              0x4C2968
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_2              0x4C296C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_3              0x4C2970
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_4              0x4C2974
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_5              0x4C2978
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_6              0x4C297C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_7              0x4C2980
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_8              0x4C2984
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_9              0x4C2988
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_10             0x4C298C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_11             0x4C2990
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_12             0x4C2994
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_13             0x4C2998
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_14             0x4C299C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_15             0x4C29A0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_0               0x4C29A4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_1               0x4C29A8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_2               0x4C29AC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_3               0x4C29B0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_4               0x4C29B4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_5               0x4C29B8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_6               0x4C29BC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_7               0x4C29C0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_8               0x4C29C4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_9               0x4C29C8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_10              0x4C29CC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_11              0x4C29D0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_12              0x4C29D4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_13              0x4C29D8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_14              0x4C29DC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_15              0x4C29E0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_0              0x4C29E4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_1              0x4C29E8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_2              0x4C29EC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_3              0x4C29F0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_4              0x4C29F4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_5              0x4C29F8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_6              0x4C29FC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_7              0x4C2A00
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_8              0x4C2A04
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_9              0x4C2A08
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_10             0x4C2A0C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_11             0x4C2A10
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_12             0x4C2A14
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_13             0x4C2A18
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_14             0x4C2A1C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_15             0x4C2A20
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_HIT_AW                       0x4C2A64
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_SEC_HIT_AR                       0x4C2A68
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_HIT_AW                      0x4C2A6C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RANGE_PRIV_HIT_AR                      0x4C2A70
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_CFG                                0x4C2B64
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_SHIFT                              0x4C2B68
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_0                     0x4C2B6C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_1                     0x4C2B70
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_2                     0x4C2B74
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_3                     0x4C2B78
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_4                     0x4C2B7C
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_5                     0x4C2B80
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_6                     0x4C2B84
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_EXPECTED_LAT_7                     0x4C2B88
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_0                            0x4C2BAC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_1                            0x4C2BB0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_2                            0x4C2BB4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_3                            0x4C2BB8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_4                            0x4C2BBC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_5                            0x4C2BC0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_6                            0x4C2BC4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_TOKEN_7                            0x4C2BC8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_0                          0x4C2BEC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_1                          0x4C2BF0
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_2                          0x4C2BF4
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_3                          0x4C2BF8
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_4                          0x4C2BFC
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_5                          0x4C2C00
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_6                          0x4C2C04
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_BANK_ID_7                          0x4C2C08
+
+#define mmDMA_IF_W_N_DOWN_CH1_RGL_WDT                                0x4C2C2C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_WRAP               0x4C2C30
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_WRAP               0x4C2C34
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_WRAP               0x4C2C38
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_WRAP               0x4C2C3C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_WRAP               0x4C2C40
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_WRAP               0x4C2C44
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_WRAP               0x4C2C48
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_WRAP               0x4C2C4C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_CNT                0x4C2C50
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_CNT                0x4C2C54
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_CNT                0x4C2C58
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_CNT                0x4C2C5C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_CNT                0x4C2C60
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_CNT                0x4C2C64
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_CNT                0x4C2C68
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_CNT                0x4C2C6C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_WRAP               0x4C2C70
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_WRAP               0x4C2C74
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_WRAP               0x4C2C78
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_WRAP               0x4C2C7C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_WRAP               0x4C2C80
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_WRAP               0x4C2C84
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_WRAP               0x4C2C88
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_WRAP               0x4C2C8C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_CNT                0x4C2C90
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_CNT                0x4C2C94
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_CNT                0x4C2C98
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_CNT                0x4C2C9C
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_CNT                0x4C2CA0
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_CNT                0x4C2CA4
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_CNT                0x4C2CA8
+
+#define mmDMA_IF_W_N_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_CNT                0x4C2CAC
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_PC_SEL_0                        0x4C2CB0
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_PC_SEL_1                        0x4C2CB4
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_PC_SEL_2                        0x4C2CB8
+
+#define mmDMA_IF_W_N_DOWN_CH1_NL_HBM_PC_SEL_3                        0x4C2CBC
+
+#endif /* ASIC_REG_DMA_IF_W_N_DOWN_CH1_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_regs.h
new file mode 100644
index 000000000000..da60893a5fab
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_n_regs.h
@@ -0,0 +1,860 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_W_N_REGS_H_
+#define ASIC_REG_DMA_IF_W_N_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_W_N (Prototype: DMA_IF)
+ *****************************************
+ */
+
+#define mmDMA_IF_W_N_HBM0_WR_CRED_CNT                                0x4C0000
+
+#define mmDMA_IF_W_N_HBM1_WR_CRED_CNT                                0x4C0004
+
+#define mmDMA_IF_W_N_HBM0_RD_CRED_CNT                                0x4C0008
+
+#define mmDMA_IF_W_N_HBM1_RD_CRED_CNT                                0x4C000C
+
+#define mmDMA_IF_W_N_HBM_LIMITER_0                                   0x4C0030
+
+#define mmDMA_IF_W_N_HBM_LIMITER_1                                   0x4C0034
+
+#define mmDMA_IF_W_N_HBM_LIMITER_2                                   0x4C0038
+
+#define mmDMA_IF_W_N_HBM_LIMITER_3                                   0x4C003C
+
+#define mmDMA_IF_W_N_HBM_ALMOST_EN_0                                 0x4C0040
+
+#define mmDMA_IF_W_N_HBM_ALMOST_EN_1                                 0x4C0044
+
+#define mmDMA_IF_W_N_HBM_CRED_EN_0                                   0x4C0050
+
+#define mmDMA_IF_W_N_HBM_CRED_EN_1                                   0x4C0054
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_0                                 0x4C0100
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_1                                 0x4C0104
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_2                                 0x4C0108
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_3                                 0x4C010C
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_4                                 0x4C0110
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_5                                 0x4C0114
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_6                                 0x4C0118
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_7                                 0x4C011C
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_8                                 0x4C0120
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_9                                 0x4C0124
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_10                                0x4C0128
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_11                                0x4C012C
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_12                                0x4C0130
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_13                                0x4C0134
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_14                                0x4C0138
+
+#define mmDMA_IF_W_N_SOB_MIN_RPROT_15                                0x4C013C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_0                                 0x4C0140
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_1                                 0x4C0144
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_2                                 0x4C0148
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_3                                 0x4C014C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_4                                 0x4C0150
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_5                                 0x4C0154
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_6                                 0x4C0158
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_7                                 0x4C015C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_8                                 0x4C0160
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_9                                 0x4C0164
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_10                                0x4C0168
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_11                                0x4C016C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_12                                0x4C0170
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_13                                0x4C0174
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_14                                0x4C0178
+
+#define mmDMA_IF_W_N_SOB_MAX_RPROT_15                                0x4C017C
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_0                                 0x4C0180
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_1                                 0x4C0184
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_2                                 0x4C0188
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_3                                 0x4C018C
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_4                                 0x4C0190
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_5                                 0x4C0194
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_6                                 0x4C0198
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_7                                 0x4C019C
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_8                                 0x4C01A0
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_9                                 0x4C01A4
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_10                                0x4C01A8
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_11                                0x4C01AC
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_12                                0x4C01B0
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_13                                0x4C01B4
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_14                                0x4C01B8
+
+#define mmDMA_IF_W_N_SOB_MIN_WPROT_15                                0x4C01BC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_0                                 0x4C01C0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_1                                 0x4C01C4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_2                                 0x4C01C8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_3                                 0x4C01CC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_4                                 0x4C01D0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_5                                 0x4C01D4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_6                                 0x4C01D8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_7                                 0x4C01DC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_8                                 0x4C01E0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_9                                 0x4C01E4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_10                                0x4C01E8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_11                                0x4C01EC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_12                                0x4C01F0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_13                                0x4C01F4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_14                                0x4C01F8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPROT_15                                0x4C01FC
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_0                                 0x4C0200
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_1                                 0x4C0204
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_2                                 0x4C0208
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_3                                 0x4C020C
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_4                                 0x4C0210
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_5                                 0x4C0214
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_6                                 0x4C0218
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_7                                 0x4C021C
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_8                                 0x4C0220
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_9                                 0x4C0224
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_10                                0x4C0228
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_11                                0x4C022C
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_12                                0x4C0230
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_13                                0x4C0234
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_14                                0x4C0238
+
+#define mmDMA_IF_W_N_SOB_MIN_RPRIV_15                                0x4C023C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_0                                 0x4C0240
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_1                                 0x4C0244
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_2                                 0x4C0248
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_3                                 0x4C024C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_4                                 0x4C0250
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_5                                 0x4C0254
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_6                                 0x4C0258
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_7                                 0x4C025C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_8                                 0x4C0260
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_9                                 0x4C0264
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_10                                0x4C0268
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_11                                0x4C026C
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_12                                0x4C0270
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_13                                0x4C0274
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_14                                0x4C0278
+
+#define mmDMA_IF_W_N_SOB_MAX_RPRIV_15                                0x4C027C
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_0                                 0x4C0280
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_1                                 0x4C0284
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_2                                 0x4C0288
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_3                                 0x4C028C
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_4                                 0x4C0290
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_5                                 0x4C0294
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_6                                 0x4C0298
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_7                                 0x4C029C
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_8                                 0x4C02A0
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_9                                 0x4C02A4
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_10                                0x4C02A8
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_11                                0x4C02AC
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_12                                0x4C02B0
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_13                                0x4C02B4
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_14                                0x4C02B8
+
+#define mmDMA_IF_W_N_SOB_MIN_WPRIV_15                                0x4C02BC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_0                                 0x4C02C0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_1                                 0x4C02C4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_2                                 0x4C02C8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_3                                 0x4C02CC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_4                                 0x4C02D0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_5                                 0x4C02D4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_6                                 0x4C02D8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_7                                 0x4C02DC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_8                                 0x4C02E0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_9                                 0x4C02E4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_10                                0x4C02E8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_11                                0x4C02EC
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_12                                0x4C02F0
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_13                                0x4C02F4
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_14                                0x4C02F8
+
+#define mmDMA_IF_W_N_SOB_MAX_WPRIV_15                                0x4C02FC
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_0                                0x4C0300
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_1                                0x4C0304
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_2                                0x4C0308
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_3                                0x4C030C
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_4                                0x4C0310
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_5                                0x4C0314
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_6                                0x4C0318
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_7                                0x4C031C
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_8                                0x4C0320
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_9                                0x4C0324
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_10                               0x4C0328
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_11                               0x4C032C
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_12                               0x4C0330
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_13                               0x4C0334
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_14                               0x4C0338
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPROT_15                               0x4C033C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_0                                0x4C0340
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_1                                0x4C0344
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_2                                0x4C0348
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_3                                0x4C034C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_4                                0x4C0350
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_5                                0x4C0354
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_6                                0x4C0358
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_7                                0x4C035C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_8                                0x4C0360
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_9                                0x4C0364
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_10                               0x4C0368
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_11                               0x4C036C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_12                               0x4C0370
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_13                               0x4C0374
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_14                               0x4C0378
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPROT_15                               0x4C037C
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_0                                0x4C0380
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_1                                0x4C0384
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_2                                0x4C0388
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_3                                0x4C038C
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_4                                0x4C0390
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_5                                0x4C0394
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_6                                0x4C0398
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_7                                0x4C039C
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_8                                0x4C03A0
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_9                                0x4C03A4
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_10                               0x4C03A8
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_11                               0x4C03AC
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_12                               0x4C03B0
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_13                               0x4C03B4
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_14                               0x4C03B8
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPROT_15                               0x4C03BC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_0                                0x4C03C0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_1                                0x4C03C4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_2                                0x4C03C8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_3                                0x4C03CC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_4                                0x4C03D0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_5                                0x4C03D4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_6                                0x4C03D8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_7                                0x4C03DC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_8                                0x4C03E0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_9                                0x4C03E4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_10                               0x4C03E8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_11                               0x4C03EC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_12                               0x4C03F0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_13                               0x4C03F4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_14                               0x4C03F8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPROT_15                               0x4C03FC
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_0                                0x4C0400
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_1                                0x4C0404
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_2                                0x4C0408
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_3                                0x4C040C
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_4                                0x4C0410
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_5                                0x4C0414
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_6                                0x4C0418
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_7                                0x4C041C
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_8                                0x4C0420
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_9                                0x4C0424
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_10                               0x4C0428
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_11                               0x4C042C
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_12                               0x4C0430
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_13                               0x4C0434
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_14                               0x4C0438
+
+#define mmDMA_IF_W_N_DMA0_MIN_RPRIV_15                               0x4C043C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_0                                0x4C0440
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_1                                0x4C0444
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_2                                0x4C0448
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_3                                0x4C044C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_4                                0x4C0450
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_5                                0x4C0454
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_6                                0x4C0458
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_7                                0x4C045C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_8                                0x4C0460
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_9                                0x4C0464
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_10                               0x4C0468
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_11                               0x4C046C
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_12                               0x4C0470
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_13                               0x4C0474
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_14                               0x4C0478
+
+#define mmDMA_IF_W_N_DMA0_MAX_RPRIV_15                               0x4C047C
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_0                                0x4C0480
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_1                                0x4C0484
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_2                                0x4C0488
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_3                                0x4C048C
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_4                                0x4C0490
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_5                                0x4C0494
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_6                                0x4C0498
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_7                                0x4C049C
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_8                                0x4C04A0
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_9                                0x4C04A4
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_10                               0x4C04A8
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_11                               0x4C04AC
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_12                               0x4C04B0
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_13                               0x4C04B4
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_14                               0x4C04B8
+
+#define mmDMA_IF_W_N_DMA0_MIN_WPRIV_15                               0x4C04BC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_0                                0x4C04C0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_1                                0x4C04C4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_2                                0x4C04C8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_3                                0x4C04CC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_4                                0x4C04D0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_5                                0x4C04D4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_6                                0x4C04D8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_7                                0x4C04DC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_8                                0x4C04E0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_9                                0x4C04E4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_10                               0x4C04E8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_11                               0x4C04EC
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_12                               0x4C04F0
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_13                               0x4C04F4
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_14                               0x4C04F8
+
+#define mmDMA_IF_W_N_DMA0_MAX_WPRIV_15                               0x4C04FC
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_0                                0x4C0500
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_1                                0x4C0504
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_2                                0x4C0508
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_3                                0x4C050C
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_4                                0x4C0510
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_5                                0x4C0514
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_6                                0x4C0518
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_7                                0x4C051C
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_8                                0x4C0520
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_9                                0x4C0524
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_10                               0x4C0528
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_11                               0x4C052C
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_12                               0x4C0530
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_13                               0x4C0534
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_14                               0x4C0538
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPROT_15                               0x4C053C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_0                                0x4C0540
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_1                                0x4C0544
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_2                                0x4C0548
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_3                                0x4C054C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_4                                0x4C0550
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_5                                0x4C0554
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_6                                0x4C0558
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_7                                0x4C055C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_8                                0x4C0560
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_9                                0x4C0564
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_10                               0x4C0568
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_11                               0x4C056C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_12                               0x4C0570
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_13                               0x4C0574
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_14                               0x4C0578
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPROT_15                               0x4C057C
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_0                                0x4C0580
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_1                                0x4C0584
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_2                                0x4C0588
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_3                                0x4C058C
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_4                                0x4C0590
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_5                                0x4C0594
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_6                                0x4C0598
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_7                                0x4C059C
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_8                                0x4C05A0
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_9                                0x4C05A4
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_10                               0x4C05A8
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_11                               0x4C05AC
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_12                               0x4C05B0
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_13                               0x4C05B4
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_14                               0x4C05B8
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPROT_15                               0x4C05BC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_0                                0x4C05C0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_1                                0x4C05C4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_2                                0x4C05C8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_3                                0x4C05CC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_4                                0x4C05D0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_5                                0x4C05D4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_6                                0x4C05D8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_7                                0x4C05DC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_8                                0x4C05E0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_9                                0x4C05E4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_10                               0x4C05E8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_11                               0x4C05EC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_12                               0x4C05F0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_13                               0x4C05F4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_14                               0x4C05F8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPROT_15                               0x4C05FC
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_0                                0x4C0600
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_1                                0x4C0604
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_2                                0x4C0608
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_3                                0x4C060C
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_4                                0x4C0610
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_5                                0x4C0614
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_6                                0x4C0618
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_7                                0x4C061C
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_8                                0x4C0620
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_9                                0x4C0624
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_10                               0x4C0628
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_11                               0x4C062C
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_12                               0x4C0630
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_13                               0x4C0634
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_14                               0x4C0638
+
+#define mmDMA_IF_W_N_DMA1_MIN_RPRIV_15                               0x4C063C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_0                                0x4C0640
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_1                                0x4C0644
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_2                                0x4C0648
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_3                                0x4C064C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_4                                0x4C0650
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_5                                0x4C0654
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_6                                0x4C0658
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_7                                0x4C065C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_8                                0x4C0660
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_9                                0x4C0664
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_10                               0x4C0668
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_11                               0x4C066C
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_12                               0x4C0670
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_13                               0x4C0674
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_14                               0x4C0678
+
+#define mmDMA_IF_W_N_DMA1_MAX_RPRIV_15                               0x4C067C
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_0                                0x4C0680
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_1                                0x4C0684
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_2                                0x4C0688
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_3                                0x4C068C
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_4                                0x4C0690
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_5                                0x4C0694
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_6                                0x4C0698
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_7                                0x4C069C
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_8                                0x4C06A0
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_9                                0x4C06A4
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_10                               0x4C06A8
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_11                               0x4C06AC
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_12                               0x4C06B0
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_13                               0x4C06B4
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_14                               0x4C06B8
+
+#define mmDMA_IF_W_N_DMA1_MIN_WPRIV_15                               0x4C06BC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_0                                0x4C06C0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_1                                0x4C06C4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_2                                0x4C06C8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_3                                0x4C06CC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_4                                0x4C06D0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_5                                0x4C06D4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_6                                0x4C06D8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_7                                0x4C06DC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_8                                0x4C06E0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_9                                0x4C06E4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_10                               0x4C06E8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_11                               0x4C06EC
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_12                               0x4C06F0
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_13                               0x4C06F4
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_14                               0x4C06F8
+
+#define mmDMA_IF_W_N_DMA1_MAX_WPRIV_15                               0x4C06FC
+
+#define mmDMA_IF_W_N_SOB_HIT_RPROT                                   0x4C0700
+
+#define mmDMA_IF_W_N_SOB_HIT_WPROT                                   0x4C0704
+
+#define mmDMA_IF_W_N_SOB_HIT_RPRIV                                   0x4C070C
+
+#define mmDMA_IF_W_N_SOB_HIT_WPRIV                                   0x4C0710
+
+#define mmDMA_IF_W_N_DMA0_HIT_RPROT                                  0x4C071C
+
+#define mmDMA_IF_W_N_DMA0_HIT_WPROT                                  0x4C0720
+
+#define mmDMA_IF_W_N_DMA0_HIT_RPRIV                                  0x4C0724
+
+#define mmDMA_IF_W_N_DMA0_HIT_WPRIV                                  0x4C0728
+
+#define mmDMA_IF_W_N_DMA1_HIT_RPROT                                  0x4C0730
+
+#define mmDMA_IF_W_N_DMA1_HIT_WPROT                                  0x4C0734
+
+#define mmDMA_IF_W_N_DMA1_HIT_RPRIV                                  0x4C0738
+
+#define mmDMA_IF_W_N_DMA1_HIT_WPRIV                                  0x4C073C
+
+#define mmDMA_IF_W_N_HBM_BIN                                         0x4C0800
+
+#define mmDMA_IF_W_N_MME_BIN                                         0x4C0804
+
+#define mmDMA_IF_W_N_TPC_BIN                                         0x4C0808
+
+#define mmDMA_IF_W_N_DMA_BIN                                         0x4C080C
+
+#define mmDMA_IF_W_N_SOB_CG_EN                                       0x4C0810
+
+#define mmDMA_IF_W_N_HBM_I2C_ADDR_0                                  0x4C0820
+
+#define mmDMA_IF_W_N_HBM_I2C_ADDR_1                                  0x4C0824
+
+#define mmDMA_IF_W_N_HBM_I2C_ADDR_2                                  0x4C0828
+
+#define mmDMA_IF_W_N_HBM_I2C_ADDR_3                                  0x4C082C
+
+#define mmDMA_IF_W_N_HBM_I2C_ADDR_4                                  0x4C0830
+
+#define mmDMA_IF_W_N_HBM_MISC                                        0x4C0834
+
+#endif /* ASIC_REG_DMA_IF_W_N_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch0_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch0_regs.h
new file mode 100644
index 000000000000..56ffc920d58a
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch0_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_W_S_DOWN_CH0_REGS_H_
+#define ASIC_REG_DMA_IF_W_S_DOWN_CH0_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_W_S_DOWN_CH0 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_W_S_DOWN_CH0_PERM_SEL                               0x481108
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_0                          0x481114
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_1                          0x481118
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_2                          0x48111C
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_3                          0x481120
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_4                          0x481124
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_5                          0x481128
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_6                          0x48112C
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_7                          0x481130
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_8                          0x481134
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_9                          0x481138
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_10                         0x48113C
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_11                         0x481140
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_12                         0x481144
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_13                         0x481148
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_14                         0x48114C
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_15                         0x481150
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_16                         0x481154
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_17                         0x481158
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_18                         0x48115C
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_19                         0x481160
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_20                         0x481164
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_21                         0x481168
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_22                         0x48116C
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_23                         0x481170
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_24                         0x481174
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_25                         0x481178
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_26                         0x48117C
+
+#define mmDMA_IF_W_S_DOWN_CH0_HBM_POLY_H3_27                         0x481180
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_0                         0x481184
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_1                         0x481188
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_2                         0x48118C
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_3                         0x481190
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_4                         0x481194
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_5                         0x481198
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_6                         0x48119C
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_7                         0x4811A0
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_8                         0x4811A4
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_9                         0x4811A8
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_10                        0x4811AC
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_11                        0x4811B0
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_12                        0x4811B4
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_13                        0x4811B8
+
+#define mmDMA_IF_W_S_DOWN_CH0_SRAM_POLY_H3_14                        0x4811BC
+
+#define mmDMA_IF_W_S_DOWN_CH0_SCRAM_SRAM_EN                          0x48126C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_HBM_EN                              0x481274
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_HBM_SAT                             0x481278
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_HBM_RST                             0x48127C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_HBM_TIMEOUT                         0x481280
+
+#define mmDMA_IF_W_S_DOWN_CH0_SCRAM_HBM_EN                           0x481284
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_PCI_EN                              0x481288
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_PCI_SAT                             0x48128C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_PCI_RST                             0x481290
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_PCI_TIMEOUT                         0x481294
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_EN                             0x48129C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_SAT                            0x4812A0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_RST                            0x4812A4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_TIMEOUT                        0x4812AC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RL_SRAM_RED                            0x4812B4
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_EN                             0x4812EC
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_EN                             0x4812F0
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_WR_SIZE                        0x4812F4
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_WR_SIZE                        0x4812F8
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_PCI_CTR_SET_EN                  0x481404
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_PCI_CTR_SET                     0x481408
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_PCI_CTR_WRAP                    0x48140C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_PCI_CTR_CNT                     0x481410
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM_CTR_SET_EN                  0x481414
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM_CTR_SET                     0x481418
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_RD_SIZE                        0x48141C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_RD_SIZE                        0x481420
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_PCI_CTR_SET_EN                  0x481424
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_PCI_CTR_SET                     0x481428
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_PCI_CTR_WRAP                    0x48142C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_PCI_CTR_CNT                     0x481430
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM_CTR_SET_EN                  0x481434
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM_CTR_SET                     0x481438
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_0                           0x481450
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_SEL_1                           0x481454
+
+#define mmDMA_IF_W_S_DOWN_CH0_NON_LIN_EN                             0x481480
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_BANK_0                         0x481500
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_BANK_1                         0x481504
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_BANK_2                         0x481508
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_BANK_3                         0x48150C
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_BANK_4                         0x481510
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_0                       0x481514
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_1                       0x481520
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_2                       0x481524
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_3                       0x481528
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_4                       0x48152C
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_5                       0x481530
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_6                       0x481534
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_7                       0x481538
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_8                       0x48153C
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_SRAM_OFFSET_9                       0x481540
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_0                        0x481550
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_1                        0x481554
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_2                        0x481558
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_3                        0x48155C
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_4                        0x481560
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_5                        0x481564
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_6                        0x481568
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_7                        0x48156C
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_8                        0x481570
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_9                        0x481574
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_10                       0x481578
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_11                       0x48157C
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_12                       0x481580
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_13                       0x481584
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_14                       0x481588
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_15                       0x48158C
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_16                       0x481590
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_17                       0x481594
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_OFFSET_18                       0x481598
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_0                0x4815E4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_1                0x4815E8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_2                0x4815EC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_3                0x4815F0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_4                0x4815F4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_5                0x4815F8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_6                0x4815FC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_7                0x481600
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_8                0x481604
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_9                0x481608
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_10               0x48160C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_11               0x481610
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_12               0x481614
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_13               0x481618
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_14               0x48161C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AW_15               0x481620
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_0               0x481624
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_1               0x481628
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_2               0x48162C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_3               0x481630
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_4               0x481634
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_5               0x481638
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_6               0x48163C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_7               0x481640
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_8               0x481644
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_9               0x481648
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_10              0x48164C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_11              0x481650
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_12              0x481654
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_13              0x481658
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_14              0x48165C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AW_15              0x481660
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_0                0x481664
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_1                0x481668
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_2                0x48166C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_3                0x481670
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_4                0x481674
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_5                0x481678
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_6                0x48167C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_7                0x481680
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_8                0x481684
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_9                0x481688
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_10               0x48168C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_11               0x481690
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_12               0x481694
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_13               0x481698
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_14               0x48169C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AW_15               0x4816A0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_0               0x4816A4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_1               0x4816A8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_2               0x4816AC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_3               0x4816B0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_4               0x4816B4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_5               0x4816B8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_6               0x4816BC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_7               0x4816C0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_8               0x4816C4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_9               0x4816C8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_10              0x4816CC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_11              0x4816D0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_12              0x4816D4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_13              0x4816D8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_14              0x4816DC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AW_15              0x4816E0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_0               0x4816E4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_1               0x4816E8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_2               0x4816EC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_3               0x4816F0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_4               0x4816F4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_5               0x4816F8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_6               0x4816FC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_7               0x481700
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_8               0x481704
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_9               0x481708
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_10              0x48170C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_11              0x481710
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_12              0x481714
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_13              0x481718
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_14              0x48171C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AW_15              0x481720
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_0              0x481724
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_1              0x481728
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_2              0x48172C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_3              0x481730
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_4              0x481734
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_5              0x481738
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_6              0x48173C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_7              0x481740
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_8              0x481744
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_9              0x481748
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_10             0x48174C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_11             0x481750
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_12             0x481754
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_13             0x481758
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_14             0x48175C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AW_15             0x481760
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_0               0x481764
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_1               0x481768
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_2               0x48176C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_3               0x481770
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_4               0x481774
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_5               0x481778
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_6               0x48177C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_7               0x481780
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_8               0x481784
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_9               0x481788
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_10              0x48178C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_11              0x481790
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_12              0x481794
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_13              0x481798
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_14              0x48179C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AW_15              0x4817A0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_0              0x4817A4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_1              0x4817A8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_2              0x4817AC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_3              0x4817B0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_4              0x4817B4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_5              0x4817B8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_6              0x4817BC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_7              0x4817C0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_8              0x4817C4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_9              0x4817C8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_10             0x4817CC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_11             0x4817D0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_12             0x4817D4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_13             0x4817D8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_14             0x4817DC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AW_15             0x4817E0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_0                0x481824
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_1                0x481828
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_2                0x48182C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_3                0x481830
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_4                0x481834
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_5                0x481838
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_6                0x48183C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_7                0x481840
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_8                0x481844
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_9                0x481848
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_10               0x48184C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_11               0x481850
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_12               0x481854
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_13               0x481858
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_14               0x48185C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_LOW_AR_15               0x481860
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_0               0x481864
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_1               0x481868
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_2               0x48186C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_3               0x481870
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_4               0x481874
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_5               0x481878
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_6               0x48187C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_7               0x481880
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_8               0x481884
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_9               0x481888
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_10              0x48188C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_11              0x481890
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_12              0x481894
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_13              0x481898
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_14              0x48189C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_BASE_HIGH_AR_15              0x4818A0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_0                0x4818A4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_1                0x4818A8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_2                0x4818AC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_3                0x4818B0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_4                0x4818B4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_5                0x4818B8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_6                0x4818BC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_7                0x4818C0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_8                0x4818C4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_9                0x4818C8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_10               0x4818CC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_11               0x4818D0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_12               0x4818D4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_13               0x4818D8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_14               0x4818DC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_LOW_AR_15               0x4818E0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_0               0x4818E4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_1               0x4818E8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_2               0x4818EC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_3               0x4818F0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_4               0x4818F4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_5               0x4818F8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_6               0x4818FC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_7               0x481900
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_8               0x481904
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_9               0x481908
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_10              0x48190C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_11              0x481910
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_12              0x481914
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_13              0x481918
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_14              0x48191C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_MASK_HIGH_AR_15              0x481920
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_0               0x481924
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_1               0x481928
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_2               0x48192C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_3               0x481930
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_4               0x481934
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_5               0x481938
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_6               0x48193C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_7               0x481940
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_8               0x481944
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_9               0x481948
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_10              0x48194C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_11              0x481950
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_12              0x481954
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_13              0x481958
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_14              0x48195C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_LOW_AR_15              0x481960
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_0              0x481964
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_1              0x481968
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_2              0x48196C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_3              0x481970
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_4              0x481974
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_5              0x481978
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_6              0x48197C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_7              0x481980
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_8              0x481984
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_9              0x481988
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_10             0x48198C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_11             0x481990
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_12             0x481994
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_13             0x481998
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_14             0x48199C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_BASE_HIGH_AR_15             0x4819A0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_0               0x4819A4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_1               0x4819A8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_2               0x4819AC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_3               0x4819B0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_4               0x4819B4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_5               0x4819B8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_6               0x4819BC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_7               0x4819C0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_8               0x4819C4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_9               0x4819C8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_10              0x4819CC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_11              0x4819D0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_12              0x4819D4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_13              0x4819D8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_14              0x4819DC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_LOW_AR_15              0x4819E0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_0              0x4819E4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_1              0x4819E8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_2              0x4819EC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_3              0x4819F0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_4              0x4819F4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_5              0x4819F8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_6              0x4819FC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_7              0x481A00
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_8              0x481A04
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_9              0x481A08
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_10             0x481A0C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_11             0x481A10
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_12             0x481A14
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_13             0x481A18
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_14             0x481A1C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_MASK_HIGH_AR_15             0x481A20
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_HIT_AW                       0x481A64
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_SEC_HIT_AR                       0x481A68
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_HIT_AW                      0x481A6C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RANGE_PRIV_HIT_AR                      0x481A70
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_CFG                                0x481B64
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_SHIFT                              0x481B68
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_0                     0x481B6C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_1                     0x481B70
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_2                     0x481B74
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_3                     0x481B78
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_4                     0x481B7C
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_5                     0x481B80
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_6                     0x481B84
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_EXPECTED_LAT_7                     0x481B88
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_0                            0x481BAC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_1                            0x481BB0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_2                            0x481BB4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_3                            0x481BB8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_4                            0x481BBC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_5                            0x481BC0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_6                            0x481BC4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_TOKEN_7                            0x481BC8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_0                          0x481BEC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_1                          0x481BF0
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_2                          0x481BF4
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_3                          0x481BF8
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_4                          0x481BFC
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_5                          0x481C00
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_6                          0x481C04
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_BANK_ID_7                          0x481C08
+
+#define mmDMA_IF_W_S_DOWN_CH0_RGL_WDT                                0x481C2C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_WRAP               0x481C30
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_WRAP               0x481C34
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_WRAP               0x481C38
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_WRAP               0x481C3C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_WRAP               0x481C40
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_WRAP               0x481C44
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_WRAP               0x481C48
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_WRAP               0x481C4C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM0_CH0_CTR_CNT                0x481C50
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM0_CH1_CTR_CNT                0x481C54
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM1_CH0_CTR_CNT                0x481C58
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM1_CH1_CTR_CNT                0x481C5C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM2_CH0_CTR_CNT                0x481C60
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM2_CH1_CTR_CNT                0x481C64
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM3_CH0_CTR_CNT                0x481C68
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AR_HBM3_CH1_CTR_CNT                0x481C6C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_WRAP               0x481C70
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_WRAP               0x481C74
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_WRAP               0x481C78
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_WRAP               0x481C7C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_WRAP               0x481C80
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_WRAP               0x481C84
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_WRAP               0x481C88
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_WRAP               0x481C8C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM0_CH0_CTR_CNT                0x481C90
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM0_CH1_CTR_CNT                0x481C94
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM1_CH0_CTR_CNT                0x481C98
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM1_CH1_CTR_CNT                0x481C9C
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM2_CH0_CTR_CNT                0x481CA0
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM2_CH1_CTR_CNT                0x481CA4
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM3_CH0_CTR_CNT                0x481CA8
+
+#define mmDMA_IF_W_S_DOWN_CH0_E2E_AW_HBM3_CH1_CTR_CNT                0x481CAC
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_PC_SEL_0                        0x481CB0
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_PC_SEL_1                        0x481CB4
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_PC_SEL_2                        0x481CB8
+
+#define mmDMA_IF_W_S_DOWN_CH0_NL_HBM_PC_SEL_3                        0x481CBC
+
+#endif /* ASIC_REG_DMA_IF_W_S_DOWN_CH0_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch1_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch1_regs.h
new file mode 100644
index 000000000000..cbc642918deb
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_down_ch1_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_W_S_DOWN_CH1_REGS_H_
+#define ASIC_REG_DMA_IF_W_S_DOWN_CH1_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_W_S_DOWN_CH1 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmDMA_IF_W_S_DOWN_CH1_PERM_SEL                               0x482108
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_0                          0x482114
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_1                          0x482118
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_2                          0x48211C
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_3                          0x482120
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_4                          0x482124
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_5                          0x482128
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_6                          0x48212C
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_7                          0x482130
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_8                          0x482134
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_9                          0x482138
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_10                         0x48213C
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_11                         0x482140
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_12                         0x482144
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_13                         0x482148
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_14                         0x48214C
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_15                         0x482150
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_16                         0x482154
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_17                         0x482158
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_18                         0x48215C
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_19                         0x482160
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_20                         0x482164
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_21                         0x482168
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_22                         0x48216C
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_23                         0x482170
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_24                         0x482174
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_25                         0x482178
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_26                         0x48217C
+
+#define mmDMA_IF_W_S_DOWN_CH1_HBM_POLY_H3_27                         0x482180
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_0                         0x482184
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_1                         0x482188
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_2                         0x48218C
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_3                         0x482190
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_4                         0x482194
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_5                         0x482198
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_6                         0x48219C
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_7                         0x4821A0
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_8                         0x4821A4
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_9                         0x4821A8
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_10                        0x4821AC
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_11                        0x4821B0
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_12                        0x4821B4
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_13                        0x4821B8
+
+#define mmDMA_IF_W_S_DOWN_CH1_SRAM_POLY_H3_14                        0x4821BC
+
+#define mmDMA_IF_W_S_DOWN_CH1_SCRAM_SRAM_EN                          0x48226C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_HBM_EN                              0x482274
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_HBM_SAT                             0x482278
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_HBM_RST                             0x48227C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_HBM_TIMEOUT                         0x482280
+
+#define mmDMA_IF_W_S_DOWN_CH1_SCRAM_HBM_EN                           0x482284
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_PCI_EN                              0x482288
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_PCI_SAT                             0x48228C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_PCI_RST                             0x482290
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_PCI_TIMEOUT                         0x482294
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_EN                             0x48229C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_SAT                            0x4822A0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_RST                            0x4822A4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_TIMEOUT                        0x4822AC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RL_SRAM_RED                            0x4822B4
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_EN                             0x4822EC
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_EN                             0x4822F0
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_WR_SIZE                        0x4822F4
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_WR_SIZE                        0x4822F8
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_PCI_CTR_SET_EN                  0x482404
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_PCI_CTR_SET                     0x482408
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_PCI_CTR_WRAP                    0x48240C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_PCI_CTR_CNT                     0x482410
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM_CTR_SET_EN                  0x482414
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM_CTR_SET                     0x482418
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_RD_SIZE                        0x48241C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_RD_SIZE                        0x482420
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_PCI_CTR_SET_EN                  0x482424
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_PCI_CTR_SET                     0x482428
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_PCI_CTR_WRAP                    0x48242C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_PCI_CTR_CNT                     0x482430
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM_CTR_SET_EN                  0x482434
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM_CTR_SET                     0x482438
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_0                           0x482450
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_SEL_1                           0x482454
+
+#define mmDMA_IF_W_S_DOWN_CH1_NON_LIN_EN                             0x482480
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_BANK_0                         0x482500
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_BANK_1                         0x482504
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_BANK_2                         0x482508
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_BANK_3                         0x48250C
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_BANK_4                         0x482510
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_0                       0x482514
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_1                       0x482520
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_2                       0x482524
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_3                       0x482528
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_4                       0x48252C
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_5                       0x482530
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_6                       0x482534
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_7                       0x482538
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_8                       0x48253C
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_SRAM_OFFSET_9                       0x482540
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_0                        0x482550
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_1                        0x482554
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_2                        0x482558
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_3                        0x48255C
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_4                        0x482560
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_5                        0x482564
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_6                        0x482568
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_7                        0x48256C
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_8                        0x482570
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_9                        0x482574
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_10                       0x482578
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_11                       0x48257C
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_12                       0x482580
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_13                       0x482584
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_14                       0x482588
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_15                       0x48258C
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_16                       0x482590
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_17                       0x482594
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_OFFSET_18                       0x482598
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_0                0x4825E4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_1                0x4825E8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_2                0x4825EC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_3                0x4825F0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_4                0x4825F4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_5                0x4825F8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_6                0x4825FC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_7                0x482600
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_8                0x482604
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_9                0x482608
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_10               0x48260C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_11               0x482610
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_12               0x482614
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_13               0x482618
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_14               0x48261C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AW_15               0x482620
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_0               0x482624
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_1               0x482628
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_2               0x48262C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_3               0x482630
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_4               0x482634
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_5               0x482638
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_6               0x48263C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_7               0x482640
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_8               0x482644
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_9               0x482648
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_10              0x48264C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_11              0x482650
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_12              0x482654
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_13              0x482658
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_14              0x48265C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AW_15              0x482660
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_0                0x482664
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_1                0x482668
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_2                0x48266C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_3                0x482670
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_4                0x482674
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_5                0x482678
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_6                0x48267C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_7                0x482680
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_8                0x482684
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_9                0x482688
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_10               0x48268C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_11               0x482690
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_12               0x482694
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_13               0x482698
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_14               0x48269C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AW_15               0x4826A0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_0               0x4826A4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_1               0x4826A8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_2               0x4826AC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_3               0x4826B0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_4               0x4826B4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_5               0x4826B8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_6               0x4826BC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_7               0x4826C0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_8               0x4826C4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_9               0x4826C8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_10              0x4826CC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_11              0x4826D0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_12              0x4826D4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_13              0x4826D8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_14              0x4826DC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AW_15              0x4826E0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_0               0x4826E4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_1               0x4826E8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_2               0x4826EC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_3               0x4826F0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_4               0x4826F4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_5               0x4826F8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_6               0x4826FC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_7               0x482700
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_8               0x482704
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_9               0x482708
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_10              0x48270C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_11              0x482710
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_12              0x482714
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_13              0x482718
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_14              0x48271C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AW_15              0x482720
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_0              0x482724
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_1              0x482728
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_2              0x48272C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_3              0x482730
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_4              0x482734
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_5              0x482738
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_6              0x48273C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_7              0x482740
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_8              0x482744
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_9              0x482748
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_10             0x48274C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_11             0x482750
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_12             0x482754
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_13             0x482758
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_14             0x48275C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AW_15             0x482760
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_0               0x482764
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_1               0x482768
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_2               0x48276C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_3               0x482770
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_4               0x482774
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_5               0x482778
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_6               0x48277C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_7               0x482780
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_8               0x482784
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_9               0x482788
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_10              0x48278C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_11              0x482790
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_12              0x482794
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_13              0x482798
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_14              0x48279C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AW_15              0x4827A0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_0              0x4827A4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_1              0x4827A8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_2              0x4827AC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_3              0x4827B0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_4              0x4827B4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_5              0x4827B8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_6              0x4827BC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_7              0x4827C0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_8              0x4827C4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_9              0x4827C8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_10             0x4827CC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_11             0x4827D0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_12             0x4827D4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_13             0x4827D8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_14             0x4827DC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AW_15             0x4827E0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_0                0x482824
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_1                0x482828
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_2                0x48282C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_3                0x482830
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_4                0x482834
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_5                0x482838
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_6                0x48283C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_7                0x482840
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_8                0x482844
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_9                0x482848
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_10               0x48284C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_11               0x482850
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_12               0x482854
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_13               0x482858
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_14               0x48285C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_LOW_AR_15               0x482860
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_0               0x482864
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_1               0x482868
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_2               0x48286C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_3               0x482870
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_4               0x482874
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_5               0x482878
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_6               0x48287C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_7               0x482880
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_8               0x482884
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_9               0x482888
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_10              0x48288C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_11              0x482890
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_12              0x482894
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_13              0x482898
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_14              0x48289C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_BASE_HIGH_AR_15              0x4828A0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_0                0x4828A4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_1                0x4828A8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_2                0x4828AC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_3                0x4828B0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_4                0x4828B4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_5                0x4828B8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_6                0x4828BC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_7                0x4828C0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_8                0x4828C4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_9                0x4828C8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_10               0x4828CC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_11               0x4828D0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_12               0x4828D4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_13               0x4828D8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_14               0x4828DC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_LOW_AR_15               0x4828E0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_0               0x4828E4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_1               0x4828E8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_2               0x4828EC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_3               0x4828F0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_4               0x4828F4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_5               0x4828F8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_6               0x4828FC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_7               0x482900
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_8               0x482904
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_9               0x482908
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_10              0x48290C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_11              0x482910
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_12              0x482914
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_13              0x482918
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_14              0x48291C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_MASK_HIGH_AR_15              0x482920
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_0               0x482924
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_1               0x482928
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_2               0x48292C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_3               0x482930
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_4               0x482934
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_5               0x482938
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_6               0x48293C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_7               0x482940
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_8               0x482944
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_9               0x482948
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_10              0x48294C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_11              0x482950
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_12              0x482954
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_13              0x482958
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_14              0x48295C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_LOW_AR_15              0x482960
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_0              0x482964
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_1              0x482968
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_2              0x48296C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_3              0x482970
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_4              0x482974
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_5              0x482978
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_6              0x48297C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_7              0x482980
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_8              0x482984
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_9              0x482988
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_10             0x48298C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_11             0x482990
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_12             0x482994
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_13             0x482998
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_14             0x48299C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_BASE_HIGH_AR_15             0x4829A0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_0               0x4829A4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_1               0x4829A8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_2               0x4829AC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_3               0x4829B0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_4               0x4829B4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_5               0x4829B8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_6               0x4829BC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_7               0x4829C0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_8               0x4829C4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_9               0x4829C8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_10              0x4829CC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_11              0x4829D0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_12              0x4829D4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_13              0x4829D8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_14              0x4829DC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_LOW_AR_15              0x4829E0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_0              0x4829E4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_1              0x4829E8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_2              0x4829EC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_3              0x4829F0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_4              0x4829F4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_5              0x4829F8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_6              0x4829FC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_7              0x482A00
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_8              0x482A04
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_9              0x482A08
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_10             0x482A0C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_11             0x482A10
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_12             0x482A14
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_13             0x482A18
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_14             0x482A1C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_MASK_HIGH_AR_15             0x482A20
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_HIT_AW                       0x482A64
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_SEC_HIT_AR                       0x482A68
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_HIT_AW                      0x482A6C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RANGE_PRIV_HIT_AR                      0x482A70
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_CFG                                0x482B64
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_SHIFT                              0x482B68
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_0                     0x482B6C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_1                     0x482B70
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_2                     0x482B74
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_3                     0x482B78
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_4                     0x482B7C
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_5                     0x482B80
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_6                     0x482B84
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_EXPECTED_LAT_7                     0x482B88
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_0                            0x482BAC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_1                            0x482BB0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_2                            0x482BB4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_3                            0x482BB8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_4                            0x482BBC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_5                            0x482BC0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_6                            0x482BC4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_TOKEN_7                            0x482BC8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_0                          0x482BEC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_1                          0x482BF0
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_2                          0x482BF4
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_3                          0x482BF8
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_4                          0x482BFC
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_5                          0x482C00
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_6                          0x482C04
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_BANK_ID_7                          0x482C08
+
+#define mmDMA_IF_W_S_DOWN_CH1_RGL_WDT                                0x482C2C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_WRAP               0x482C30
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_WRAP               0x482C34
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_WRAP               0x482C38
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_WRAP               0x482C3C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_WRAP               0x482C40
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_WRAP               0x482C44
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_WRAP               0x482C48
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_WRAP               0x482C4C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM0_CH0_CTR_CNT                0x482C50
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM0_CH1_CTR_CNT                0x482C54
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM1_CH0_CTR_CNT                0x482C58
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM1_CH1_CTR_CNT                0x482C5C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM2_CH0_CTR_CNT                0x482C60
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM2_CH1_CTR_CNT                0x482C64
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM3_CH0_CTR_CNT                0x482C68
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AR_HBM3_CH1_CTR_CNT                0x482C6C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_WRAP               0x482C70
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_WRAP               0x482C74
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_WRAP               0x482C78
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_WRAP               0x482C7C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_WRAP               0x482C80
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_WRAP               0x482C84
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_WRAP               0x482C88
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_WRAP               0x482C8C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM0_CH0_CTR_CNT                0x482C90
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM0_CH1_CTR_CNT                0x482C94
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM1_CH0_CTR_CNT                0x482C98
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM1_CH1_CTR_CNT                0x482C9C
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM2_CH0_CTR_CNT                0x482CA0
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM2_CH1_CTR_CNT                0x482CA4
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM3_CH0_CTR_CNT                0x482CA8
+
+#define mmDMA_IF_W_S_DOWN_CH1_E2E_AW_HBM3_CH1_CTR_CNT                0x482CAC
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_PC_SEL_0                        0x482CB0
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_PC_SEL_1                        0x482CB4
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_PC_SEL_2                        0x482CB8
+
+#define mmDMA_IF_W_S_DOWN_CH1_NL_HBM_PC_SEL_3                        0x482CBC
+
+#endif /* ASIC_REG_DMA_IF_W_S_DOWN_CH1_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_regs.h
new file mode 100644
index 000000000000..2382bc41bea6
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/dma_if_w_s_regs.h
@@ -0,0 +1,860 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_DMA_IF_W_S_REGS_H_
+#define ASIC_REG_DMA_IF_W_S_REGS_H_
+
+/*
+ *****************************************
+ *   DMA_IF_W_S (Prototype: DMA_IF)
+ *****************************************
+ */
+
+#define mmDMA_IF_W_S_HBM0_WR_CRED_CNT                                0x480000
+
+#define mmDMA_IF_W_S_HBM1_WR_CRED_CNT                                0x480004
+
+#define mmDMA_IF_W_S_HBM0_RD_CRED_CNT                                0x480008
+
+#define mmDMA_IF_W_S_HBM1_RD_CRED_CNT                                0x48000C
+
+#define mmDMA_IF_W_S_HBM_LIMITER_0                                   0x480030
+
+#define mmDMA_IF_W_S_HBM_LIMITER_1                                   0x480034
+
+#define mmDMA_IF_W_S_HBM_LIMITER_2                                   0x480038
+
+#define mmDMA_IF_W_S_HBM_LIMITER_3                                   0x48003C
+
+#define mmDMA_IF_W_S_HBM_ALMOST_EN_0                                 0x480040
+
+#define mmDMA_IF_W_S_HBM_ALMOST_EN_1                                 0x480044
+
+#define mmDMA_IF_W_S_HBM_CRED_EN_0                                   0x480050
+
+#define mmDMA_IF_W_S_HBM_CRED_EN_1                                   0x480054
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_0                                 0x480100
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_1                                 0x480104
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_2                                 0x480108
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_3                                 0x48010C
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_4                                 0x480110
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_5                                 0x480114
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_6                                 0x480118
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_7                                 0x48011C
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_8                                 0x480120
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_9                                 0x480124
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_10                                0x480128
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_11                                0x48012C
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_12                                0x480130
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_13                                0x480134
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_14                                0x480138
+
+#define mmDMA_IF_W_S_SOB_MIN_RPROT_15                                0x48013C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_0                                 0x480140
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_1                                 0x480144
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_2                                 0x480148
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_3                                 0x48014C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_4                                 0x480150
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_5                                 0x480154
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_6                                 0x480158
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_7                                 0x48015C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_8                                 0x480160
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_9                                 0x480164
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_10                                0x480168
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_11                                0x48016C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_12                                0x480170
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_13                                0x480174
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_14                                0x480178
+
+#define mmDMA_IF_W_S_SOB_MAX_RPROT_15                                0x48017C
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_0                                 0x480180
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_1                                 0x480184
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_2                                 0x480188
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_3                                 0x48018C
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_4                                 0x480190
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_5                                 0x480194
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_6                                 0x480198
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_7                                 0x48019C
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_8                                 0x4801A0
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_9                                 0x4801A4
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_10                                0x4801A8
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_11                                0x4801AC
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_12                                0x4801B0
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_13                                0x4801B4
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_14                                0x4801B8
+
+#define mmDMA_IF_W_S_SOB_MIN_WPROT_15                                0x4801BC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_0                                 0x4801C0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_1                                 0x4801C4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_2                                 0x4801C8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_3                                 0x4801CC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_4                                 0x4801D0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_5                                 0x4801D4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_6                                 0x4801D8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_7                                 0x4801DC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_8                                 0x4801E0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_9                                 0x4801E4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_10                                0x4801E8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_11                                0x4801EC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_12                                0x4801F0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_13                                0x4801F4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_14                                0x4801F8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPROT_15                                0x4801FC
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_0                                 0x480200
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_1                                 0x480204
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_2                                 0x480208
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_3                                 0x48020C
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_4                                 0x480210
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_5                                 0x480214
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_6                                 0x480218
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_7                                 0x48021C
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_8                                 0x480220
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_9                                 0x480224
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_10                                0x480228
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_11                                0x48022C
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_12                                0x480230
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_13                                0x480234
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_14                                0x480238
+
+#define mmDMA_IF_W_S_SOB_MIN_RPRIV_15                                0x48023C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_0                                 0x480240
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_1                                 0x480244
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_2                                 0x480248
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_3                                 0x48024C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_4                                 0x480250
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_5                                 0x480254
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_6                                 0x480258
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_7                                 0x48025C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_8                                 0x480260
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_9                                 0x480264
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_10                                0x480268
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_11                                0x48026C
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_12                                0x480270
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_13                                0x480274
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_14                                0x480278
+
+#define mmDMA_IF_W_S_SOB_MAX_RPRIV_15                                0x48027C
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_0                                 0x480280
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_1                                 0x480284
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_2                                 0x480288
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_3                                 0x48028C
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_4                                 0x480290
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_5                                 0x480294
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_6                                 0x480298
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_7                                 0x48029C
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_8                                 0x4802A0
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_9                                 0x4802A4
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_10                                0x4802A8
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_11                                0x4802AC
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_12                                0x4802B0
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_13                                0x4802B4
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_14                                0x4802B8
+
+#define mmDMA_IF_W_S_SOB_MIN_WPRIV_15                                0x4802BC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_0                                 0x4802C0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_1                                 0x4802C4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_2                                 0x4802C8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_3                                 0x4802CC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_4                                 0x4802D0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_5                                 0x4802D4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_6                                 0x4802D8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_7                                 0x4802DC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_8                                 0x4802E0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_9                                 0x4802E4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_10                                0x4802E8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_11                                0x4802EC
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_12                                0x4802F0
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_13                                0x4802F4
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_14                                0x4802F8
+
+#define mmDMA_IF_W_S_SOB_MAX_WPRIV_15                                0x4802FC
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_0                                0x480300
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_1                                0x480304
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_2                                0x480308
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_3                                0x48030C
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_4                                0x480310
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_5                                0x480314
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_6                                0x480318
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_7                                0x48031C
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_8                                0x480320
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_9                                0x480324
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_10                               0x480328
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_11                               0x48032C
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_12                               0x480330
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_13                               0x480334
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_14                               0x480338
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPROT_15                               0x48033C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_0                                0x480340
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_1                                0x480344
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_2                                0x480348
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_3                                0x48034C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_4                                0x480350
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_5                                0x480354
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_6                                0x480358
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_7                                0x48035C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_8                                0x480360
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_9                                0x480364
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_10                               0x480368
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_11                               0x48036C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_12                               0x480370
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_13                               0x480374
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_14                               0x480378
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPROT_15                               0x48037C
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_0                                0x480380
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_1                                0x480384
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_2                                0x480388
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_3                                0x48038C
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_4                                0x480390
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_5                                0x480394
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_6                                0x480398
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_7                                0x48039C
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_8                                0x4803A0
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_9                                0x4803A4
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_10                               0x4803A8
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_11                               0x4803AC
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_12                               0x4803B0
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_13                               0x4803B4
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_14                               0x4803B8
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPROT_15                               0x4803BC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_0                                0x4803C0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_1                                0x4803C4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_2                                0x4803C8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_3                                0x4803CC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_4                                0x4803D0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_5                                0x4803D4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_6                                0x4803D8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_7                                0x4803DC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_8                                0x4803E0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_9                                0x4803E4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_10                               0x4803E8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_11                               0x4803EC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_12                               0x4803F0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_13                               0x4803F4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_14                               0x4803F8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPROT_15                               0x4803FC
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_0                                0x480400
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_1                                0x480404
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_2                                0x480408
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_3                                0x48040C
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_4                                0x480410
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_5                                0x480414
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_6                                0x480418
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_7                                0x48041C
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_8                                0x480420
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_9                                0x480424
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_10                               0x480428
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_11                               0x48042C
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_12                               0x480430
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_13                               0x480434
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_14                               0x480438
+
+#define mmDMA_IF_W_S_DMA0_MIN_RPRIV_15                               0x48043C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_0                                0x480440
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_1                                0x480444
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_2                                0x480448
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_3                                0x48044C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_4                                0x480450
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_5                                0x480454
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_6                                0x480458
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_7                                0x48045C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_8                                0x480460
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_9                                0x480464
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_10                               0x480468
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_11                               0x48046C
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_12                               0x480470
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_13                               0x480474
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_14                               0x480478
+
+#define mmDMA_IF_W_S_DMA0_MAX_RPRIV_15                               0x48047C
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_0                                0x480480
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_1                                0x480484
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_2                                0x480488
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_3                                0x48048C
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_4                                0x480490
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_5                                0x480494
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_6                                0x480498
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_7                                0x48049C
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_8                                0x4804A0
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_9                                0x4804A4
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_10                               0x4804A8
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_11                               0x4804AC
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_12                               0x4804B0
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_13                               0x4804B4
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_14                               0x4804B8
+
+#define mmDMA_IF_W_S_DMA0_MIN_WPRIV_15                               0x4804BC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_0                                0x4804C0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_1                                0x4804C4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_2                                0x4804C8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_3                                0x4804CC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_4                                0x4804D0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_5                                0x4804D4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_6                                0x4804D8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_7                                0x4804DC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_8                                0x4804E0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_9                                0x4804E4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_10                               0x4804E8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_11                               0x4804EC
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_12                               0x4804F0
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_13                               0x4804F4
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_14                               0x4804F8
+
+#define mmDMA_IF_W_S_DMA0_MAX_WPRIV_15                               0x4804FC
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_0                                0x480500
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_1                                0x480504
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_2                                0x480508
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_3                                0x48050C
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_4                                0x480510
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_5                                0x480514
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_6                                0x480518
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_7                                0x48051C
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_8                                0x480520
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_9                                0x480524
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_10                               0x480528
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_11                               0x48052C
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_12                               0x480530
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_13                               0x480534
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_14                               0x480538
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPROT_15                               0x48053C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_0                                0x480540
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_1                                0x480544
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_2                                0x480548
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_3                                0x48054C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_4                                0x480550
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_5                                0x480554
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_6                                0x480558
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_7                                0x48055C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_8                                0x480560
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_9                                0x480564
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_10                               0x480568
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_11                               0x48056C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_12                               0x480570
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_13                               0x480574
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_14                               0x480578
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPROT_15                               0x48057C
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_0                                0x480580
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_1                                0x480584
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_2                                0x480588
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_3                                0x48058C
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_4                                0x480590
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_5                                0x480594
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_6                                0x480598
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_7                                0x48059C
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_8                                0x4805A0
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_9                                0x4805A4
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_10                               0x4805A8
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_11                               0x4805AC
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_12                               0x4805B0
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_13                               0x4805B4
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_14                               0x4805B8
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPROT_15                               0x4805BC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_0                                0x4805C0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_1                                0x4805C4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_2                                0x4805C8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_3                                0x4805CC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_4                                0x4805D0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_5                                0x4805D4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_6                                0x4805D8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_7                                0x4805DC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_8                                0x4805E0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_9                                0x4805E4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_10                               0x4805E8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_11                               0x4805EC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_12                               0x4805F0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_13                               0x4805F4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_14                               0x4805F8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPROT_15                               0x4805FC
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_0                                0x480600
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_1                                0x480604
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_2                                0x480608
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_3                                0x48060C
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_4                                0x480610
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_5                                0x480614
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_6                                0x480618
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_7                                0x48061C
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_8                                0x480620
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_9                                0x480624
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_10                               0x480628
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_11                               0x48062C
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_12                               0x480630
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_13                               0x480634
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_14                               0x480638
+
+#define mmDMA_IF_W_S_DMA1_MIN_RPRIV_15                               0x48063C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_0                                0x480640
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_1                                0x480644
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_2                                0x480648
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_3                                0x48064C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_4                                0x480650
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_5                                0x480654
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_6                                0x480658
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_7                                0x48065C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_8                                0x480660
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_9                                0x480664
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_10                               0x480668
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_11                               0x48066C
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_12                               0x480670
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_13                               0x480674
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_14                               0x480678
+
+#define mmDMA_IF_W_S_DMA1_MAX_RPRIV_15                               0x48067C
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_0                                0x480680
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_1                                0x480684
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_2                                0x480688
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_3                                0x48068C
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_4                                0x480690
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_5                                0x480694
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_6                                0x480698
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_7                                0x48069C
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_8                                0x4806A0
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_9                                0x4806A4
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_10                               0x4806A8
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_11                               0x4806AC
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_12                               0x4806B0
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_13                               0x4806B4
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_14                               0x4806B8
+
+#define mmDMA_IF_W_S_DMA1_MIN_WPRIV_15                               0x4806BC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_0                                0x4806C0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_1                                0x4806C4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_2                                0x4806C8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_3                                0x4806CC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_4                                0x4806D0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_5                                0x4806D4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_6                                0x4806D8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_7                                0x4806DC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_8                                0x4806E0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_9                                0x4806E4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_10                               0x4806E8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_11                               0x4806EC
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_12                               0x4806F0
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_13                               0x4806F4
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_14                               0x4806F8
+
+#define mmDMA_IF_W_S_DMA1_MAX_WPRIV_15                               0x4806FC
+
+#define mmDMA_IF_W_S_SOB_HIT_RPROT                                   0x480700
+
+#define mmDMA_IF_W_S_SOB_HIT_WPROT                                   0x480704
+
+#define mmDMA_IF_W_S_SOB_HIT_RPRIV                                   0x48070C
+
+#define mmDMA_IF_W_S_SOB_HIT_WPRIV                                   0x480710
+
+#define mmDMA_IF_W_S_DMA0_HIT_RPROT                                  0x48071C
+
+#define mmDMA_IF_W_S_DMA0_HIT_WPROT                                  0x480720
+
+#define mmDMA_IF_W_S_DMA0_HIT_RPRIV                                  0x480724
+
+#define mmDMA_IF_W_S_DMA0_HIT_WPRIV                                  0x480728
+
+#define mmDMA_IF_W_S_DMA1_HIT_RPROT                                  0x480730
+
+#define mmDMA_IF_W_S_DMA1_HIT_WPROT                                  0x480734
+
+#define mmDMA_IF_W_S_DMA1_HIT_RPRIV                                  0x480738
+
+#define mmDMA_IF_W_S_DMA1_HIT_WPRIV                                  0x48073C
+
+#define mmDMA_IF_W_S_HBM_BIN                                         0x480800
+
+#define mmDMA_IF_W_S_MME_BIN                                         0x480804
+
+#define mmDMA_IF_W_S_TPC_BIN                                         0x480808
+
+#define mmDMA_IF_W_S_DMA_BIN                                         0x48080C
+
+#define mmDMA_IF_W_S_SOB_CG_EN                                       0x480810
+
+#define mmDMA_IF_W_S_HBM_I2C_ADDR_0                                  0x480820
+
+#define mmDMA_IF_W_S_HBM_I2C_ADDR_1                                  0x480824
+
+#define mmDMA_IF_W_S_HBM_I2C_ADDR_2                                  0x480828
+
+#define mmDMA_IF_W_S_HBM_I2C_ADDR_3                                  0x48082C
+
+#define mmDMA_IF_W_S_HBM_I2C_ADDR_4                                  0x480830
+
+#define mmDMA_IF_W_S_HBM_MISC                                        0x480834
+
+#endif /* ASIC_REG_DMA_IF_W_S_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_blocks.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_blocks.h
new file mode 100644
index 000000000000..c7596aac7a5c
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_blocks.h
@@ -0,0 +1,4974 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef GAUDI_BLOCKS_H_
+#define GAUDI_BLOCKS_H_
+
+#define mmNIC0_PHY0_BASE                           0x0ull
+#define NIC0_PHY0_MAX_OFFSET                       0x9F13
+#define mmMME0_ACC_BASE                            0x7FFC020000ull
+#define MME0_ACC_MAX_OFFSET                        0x5C00
+#define MME0_ACC_SECTION                           0x20000
+#define mmMME0_SBAB_BASE                           0x7FFC040000ull
+#define MME0_SBAB_MAX_OFFSET                       0x5800
+#define MME0_SBAB_SECTION                          0x1000
+#define mmMME0_PRTN_BASE                           0x7FFC041000ull
+#define MME0_PRTN_MAX_OFFSET                       0x5000
+#define MME0_PRTN_SECTION                          0x1F000
+#define mmMME0_CTRL_BASE                           0x7FFC060000ull
+#define MME0_CTRL_MAX_OFFSET                       0xDA80
+#define MME0_CTRL_SECTION                          0x8000
+#define mmARCH_MME0_CTRL_BASE                      0x7FFC060008ull
+#define ARCH_MME0_CTRL_MAX_OFFSET                  0x3400
+#define ARCH_MME0_CTRL_SECTION                     0x3400
+#define mmARCH_TENSOR_S_MME0_CTRL_BASE             0x7FFC06003Cull
+#define ARCH_TENSOR_S_MME0_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_S_MME0_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_S_MME0_CTRL_BASE                0x7FFC060088ull
+#define ARCH_AGU_S_MME0_CTRL_MAX_OFFSET            0x2400
+#define ARCH_AGU_S_MME0_CTRL_SECTION               0x2400
+#define mmARCH_TENSOR_L_MME0_CTRL_BASE             0x7FFC0600ACull
+#define ARCH_TENSOR_L_MME0_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_L_MME0_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_L_LOCAL_MME0_CTRL_BASE          0x7FFC0600F8ull
+#define ARCH_AGU_L_LOCAL_MME0_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_L_LOCAL_MME0_CTRL_SECTION         0x2400
+#define mmARCH_AGU_L_REMOTE_MME0_CTRL_BASE         0x7FFC06011Cull
+#define ARCH_AGU_L_REMOTE_MME0_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_L_REMOTE_MME0_CTRL_SECTION        0x2400
+#define mmARCH_TENSOR_O_MME0_CTRL_BASE             0x7FFC060140ull
+#define ARCH_TENSOR_O_MME0_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_O_MME0_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_O_LOCAL_MME0_CTRL_BASE          0x7FFC06018Cull
+#define ARCH_AGU_O_LOCAL_MME0_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_O_LOCAL_MME0_CTRL_SECTION         0x2400
+#define mmARCH_AGU_O_REMOTE_MME0_CTRL_BASE         0x7FFC0601B0ull
+#define ARCH_AGU_O_REMOTE_MME0_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_O_REMOTE_MME0_CTRL_SECTION        0x2400
+#define mmARCH_DESC_MME0_CTRL_BASE                 0x7FFC0601D4ull
+#define ARCH_DESC_MME0_CTRL_MAX_OFFSET             0x5400
+#define ARCH_DESC_MME0_CTRL_SECTION                0x2340
+#define mmSHADOW_0_MME0_CTRL_BASE                  0x7FFC060408ull
+#define SHADOW_0_MME0_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_0_MME0_CTRL_SECTION                 0x3400
+#define mmSHADOW_0_TENSOR_S_MME0_CTRL_BASE         0x7FFC06043Cull
+#define SHADOW_0_TENSOR_S_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_S_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_S_MME0_CTRL_BASE            0x7FFC060488ull
+#define SHADOW_0_AGU_S_MME0_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_0_AGU_S_MME0_CTRL_SECTION           0x2400
+#define mmSHADOW_0_TENSOR_L_MME0_CTRL_BASE         0x7FFC0604ACull
+#define SHADOW_0_TENSOR_L_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_L_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_L_LOCAL_MME0_CTRL_BASE      0x7FFC0604F8ull
+#define SHADOW_0_AGU_L_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_L_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_L_REMOTE_MME0_CTRL_BASE     0x7FFC06051Cull
+#define SHADOW_0_AGU_L_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_L_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_0_TENSOR_O_MME0_CTRL_BASE         0x7FFC060540ull
+#define SHADOW_0_TENSOR_O_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_O_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_O_LOCAL_MME0_CTRL_BASE      0x7FFC06058Cull
+#define SHADOW_0_AGU_O_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_O_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_O_REMOTE_MME0_CTRL_BASE     0x7FFC0605B0ull
+#define SHADOW_0_AGU_O_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_O_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_0_DESC_MME0_CTRL_BASE             0x7FFC0605D4ull
+#define SHADOW_0_DESC_MME0_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_0_DESC_MME0_CTRL_SECTION            0xB400
+#define mmSHADOW_1_MME0_CTRL_BASE                  0x7FFC060688ull
+#define SHADOW_1_MME0_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_1_MME0_CTRL_SECTION                 0x3400
+#define mmSHADOW_1_TENSOR_S_MME0_CTRL_BASE         0x7FFC0606BCull
+#define SHADOW_1_TENSOR_S_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_S_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_S_MME0_CTRL_BASE            0x7FFC060708ull
+#define SHADOW_1_AGU_S_MME0_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_1_AGU_S_MME0_CTRL_SECTION           0x2400
+#define mmSHADOW_1_TENSOR_L_MME0_CTRL_BASE         0x7FFC06072Cull
+#define SHADOW_1_TENSOR_L_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_L_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_L_LOCAL_MME0_CTRL_BASE      0x7FFC060778ull
+#define SHADOW_1_AGU_L_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_L_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_L_REMOTE_MME0_CTRL_BASE     0x7FFC06079Cull
+#define SHADOW_1_AGU_L_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_L_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_1_TENSOR_O_MME0_CTRL_BASE         0x7FFC0607C0ull
+#define SHADOW_1_TENSOR_O_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_O_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_O_LOCAL_MME0_CTRL_BASE      0x7FFC06080Cull
+#define SHADOW_1_AGU_O_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_O_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_O_REMOTE_MME0_CTRL_BASE     0x7FFC060830ull
+#define SHADOW_1_AGU_O_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_O_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_1_DESC_MME0_CTRL_BASE             0x7FFC060854ull
+#define SHADOW_1_DESC_MME0_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_1_DESC_MME0_CTRL_SECTION            0xB400
+#define mmSHADOW_2_MME0_CTRL_BASE                  0x7FFC060908ull
+#define SHADOW_2_MME0_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_2_MME0_CTRL_SECTION                 0x3400
+#define mmSHADOW_2_TENSOR_S_MME0_CTRL_BASE         0x7FFC06093Cull
+#define SHADOW_2_TENSOR_S_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_S_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_S_MME0_CTRL_BASE            0x7FFC060988ull
+#define SHADOW_2_AGU_S_MME0_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_2_AGU_S_MME0_CTRL_SECTION           0x2400
+#define mmSHADOW_2_TENSOR_L_MME0_CTRL_BASE         0x7FFC0609ACull
+#define SHADOW_2_TENSOR_L_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_L_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_L_LOCAL_MME0_CTRL_BASE      0x7FFC0609F8ull
+#define SHADOW_2_AGU_L_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_L_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_L_REMOTE_MME0_CTRL_BASE     0x7FFC060A1Cull
+#define SHADOW_2_AGU_L_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_L_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_2_TENSOR_O_MME0_CTRL_BASE         0x7FFC060A40ull
+#define SHADOW_2_TENSOR_O_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_O_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_O_LOCAL_MME0_CTRL_BASE      0x7FFC060A8Cull
+#define SHADOW_2_AGU_O_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_O_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_O_REMOTE_MME0_CTRL_BASE     0x7FFC060AB0ull
+#define SHADOW_2_AGU_O_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_O_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_2_DESC_MME0_CTRL_BASE             0x7FFC060AD4ull
+#define SHADOW_2_DESC_MME0_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_2_DESC_MME0_CTRL_SECTION            0xB400
+#define mmSHADOW_3_MME0_CTRL_BASE                  0x7FFC060B88ull
+#define SHADOW_3_MME0_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_3_MME0_CTRL_SECTION                 0x3400
+#define mmSHADOW_3_TENSOR_S_MME0_CTRL_BASE         0x7FFC060BBCull
+#define SHADOW_3_TENSOR_S_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_S_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_S_MME0_CTRL_BASE            0x7FFC060C08ull
+#define SHADOW_3_AGU_S_MME0_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_3_AGU_S_MME0_CTRL_SECTION           0x2400
+#define mmSHADOW_3_TENSOR_L_MME0_CTRL_BASE         0x7FFC060C2Cull
+#define SHADOW_3_TENSOR_L_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_L_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_L_LOCAL_MME0_CTRL_BASE      0x7FFC060C78ull
+#define SHADOW_3_AGU_L_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_L_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_L_REMOTE_MME0_CTRL_BASE     0x7FFC060C9Cull
+#define SHADOW_3_AGU_L_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_L_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_3_TENSOR_O_MME0_CTRL_BASE         0x7FFC060CC0ull
+#define SHADOW_3_TENSOR_O_MME0_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_O_MME0_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_O_LOCAL_MME0_CTRL_BASE      0x7FFC060D0Cull
+#define SHADOW_3_AGU_O_LOCAL_MME0_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_O_LOCAL_MME0_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_O_REMOTE_MME0_CTRL_BASE     0x7FFC060D30ull
+#define SHADOW_3_AGU_O_REMOTE_MME0_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_O_REMOTE_MME0_CTRL_SECTION    0x2400
+#define mmSHADOW_3_DESC_MME0_CTRL_BASE             0x7FFC060D54ull
+#define SHADOW_3_DESC_MME0_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_3_DESC_MME0_CTRL_SECTION            0x72AC
+#define mmMME0_QM_BASE                             0x7FFC068000ull
+#define MME0_QM_MAX_OFFSET                         0xD040
+#define MME0_QM_SECTION                            0x38000
+#define mmMME1_ACC_BASE                            0x7FFC0A0000ull
+#define MME1_ACC_MAX_OFFSET                        0x5C00
+#define MME1_ACC_SECTION                           0x20000
+#define mmMME1_SBAB_BASE                           0x7FFC0C0000ull
+#define MME1_SBAB_MAX_OFFSET                       0x5800
+#define MME1_SBAB_SECTION                          0x1000
+#define mmMME1_PRTN_BASE                           0x7FFC0C1000ull
+#define MME1_PRTN_MAX_OFFSET                       0x5000
+#define MME1_PRTN_SECTION                          0x1F000
+#define mmMME1_CTRL_BASE                           0x7FFC0E0000ull
+#define MME1_CTRL_MAX_OFFSET                       0xDA80
+#define MME1_CTRL_SECTION                          0x8000
+#define mmARCH_MME1_CTRL_BASE                      0x7FFC0E0008ull
+#define ARCH_MME1_CTRL_MAX_OFFSET                  0x3400
+#define ARCH_MME1_CTRL_SECTION                     0x3400
+#define mmARCH_TENSOR_S_MME1_CTRL_BASE             0x7FFC0E003Cull
+#define ARCH_TENSOR_S_MME1_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_S_MME1_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_S_MME1_CTRL_BASE                0x7FFC0E0088ull
+#define ARCH_AGU_S_MME1_CTRL_MAX_OFFSET            0x2400
+#define ARCH_AGU_S_MME1_CTRL_SECTION               0x2400
+#define mmARCH_TENSOR_L_MME1_CTRL_BASE             0x7FFC0E00ACull
+#define ARCH_TENSOR_L_MME1_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_L_MME1_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_L_LOCAL_MME1_CTRL_BASE          0x7FFC0E00F8ull
+#define ARCH_AGU_L_LOCAL_MME1_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_L_LOCAL_MME1_CTRL_SECTION         0x2400
+#define mmARCH_AGU_L_REMOTE_MME1_CTRL_BASE         0x7FFC0E011Cull
+#define ARCH_AGU_L_REMOTE_MME1_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_L_REMOTE_MME1_CTRL_SECTION        0x2400
+#define mmARCH_TENSOR_O_MME1_CTRL_BASE             0x7FFC0E0140ull
+#define ARCH_TENSOR_O_MME1_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_O_MME1_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_O_LOCAL_MME1_CTRL_BASE          0x7FFC0E018Cull
+#define ARCH_AGU_O_LOCAL_MME1_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_O_LOCAL_MME1_CTRL_SECTION         0x2400
+#define mmARCH_AGU_O_REMOTE_MME1_CTRL_BASE         0x7FFC0E01B0ull
+#define ARCH_AGU_O_REMOTE_MME1_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_O_REMOTE_MME1_CTRL_SECTION        0x2400
+#define mmARCH_DESC_MME1_CTRL_BASE                 0x7FFC0E01D4ull
+#define ARCH_DESC_MME1_CTRL_MAX_OFFSET             0x5400
+#define ARCH_DESC_MME1_CTRL_SECTION                0x2340
+#define mmSHADOW_0_MME1_CTRL_BASE                  0x7FFC0E0408ull
+#define SHADOW_0_MME1_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_0_MME1_CTRL_SECTION                 0x3400
+#define mmSHADOW_0_TENSOR_S_MME1_CTRL_BASE         0x7FFC0E043Cull
+#define SHADOW_0_TENSOR_S_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_S_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_S_MME1_CTRL_BASE            0x7FFC0E0488ull
+#define SHADOW_0_AGU_S_MME1_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_0_AGU_S_MME1_CTRL_SECTION           0x2400
+#define mmSHADOW_0_TENSOR_L_MME1_CTRL_BASE         0x7FFC0E04ACull
+#define SHADOW_0_TENSOR_L_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_L_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_L_LOCAL_MME1_CTRL_BASE      0x7FFC0E04F8ull
+#define SHADOW_0_AGU_L_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_L_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_L_REMOTE_MME1_CTRL_BASE     0x7FFC0E051Cull
+#define SHADOW_0_AGU_L_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_L_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_0_TENSOR_O_MME1_CTRL_BASE         0x7FFC0E0540ull
+#define SHADOW_0_TENSOR_O_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_O_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_O_LOCAL_MME1_CTRL_BASE      0x7FFC0E058Cull
+#define SHADOW_0_AGU_O_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_O_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_O_REMOTE_MME1_CTRL_BASE     0x7FFC0E05B0ull
+#define SHADOW_0_AGU_O_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_O_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_0_DESC_MME1_CTRL_BASE             0x7FFC0E05D4ull
+#define SHADOW_0_DESC_MME1_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_0_DESC_MME1_CTRL_SECTION            0xB400
+#define mmSHADOW_1_MME1_CTRL_BASE                  0x7FFC0E0688ull
+#define SHADOW_1_MME1_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_1_MME1_CTRL_SECTION                 0x3400
+#define mmSHADOW_1_TENSOR_S_MME1_CTRL_BASE         0x7FFC0E06BCull
+#define SHADOW_1_TENSOR_S_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_S_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_S_MME1_CTRL_BASE            0x7FFC0E0708ull
+#define SHADOW_1_AGU_S_MME1_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_1_AGU_S_MME1_CTRL_SECTION           0x2400
+#define mmSHADOW_1_TENSOR_L_MME1_CTRL_BASE         0x7FFC0E072Cull
+#define SHADOW_1_TENSOR_L_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_L_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_L_LOCAL_MME1_CTRL_BASE      0x7FFC0E0778ull
+#define SHADOW_1_AGU_L_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_L_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_L_REMOTE_MME1_CTRL_BASE     0x7FFC0E079Cull
+#define SHADOW_1_AGU_L_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_L_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_1_TENSOR_O_MME1_CTRL_BASE         0x7FFC0E07C0ull
+#define SHADOW_1_TENSOR_O_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_O_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_O_LOCAL_MME1_CTRL_BASE      0x7FFC0E080Cull
+#define SHADOW_1_AGU_O_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_O_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_O_REMOTE_MME1_CTRL_BASE     0x7FFC0E0830ull
+#define SHADOW_1_AGU_O_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_O_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_1_DESC_MME1_CTRL_BASE             0x7FFC0E0854ull
+#define SHADOW_1_DESC_MME1_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_1_DESC_MME1_CTRL_SECTION            0xB400
+#define mmSHADOW_2_MME1_CTRL_BASE                  0x7FFC0E0908ull
+#define SHADOW_2_MME1_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_2_MME1_CTRL_SECTION                 0x3400
+#define mmSHADOW_2_TENSOR_S_MME1_CTRL_BASE         0x7FFC0E093Cull
+#define SHADOW_2_TENSOR_S_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_S_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_S_MME1_CTRL_BASE            0x7FFC0E0988ull
+#define SHADOW_2_AGU_S_MME1_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_2_AGU_S_MME1_CTRL_SECTION           0x2400
+#define mmSHADOW_2_TENSOR_L_MME1_CTRL_BASE         0x7FFC0E09ACull
+#define SHADOW_2_TENSOR_L_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_L_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_L_LOCAL_MME1_CTRL_BASE      0x7FFC0E09F8ull
+#define SHADOW_2_AGU_L_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_L_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_L_REMOTE_MME1_CTRL_BASE     0x7FFC0E0A1Cull
+#define SHADOW_2_AGU_L_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_L_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_2_TENSOR_O_MME1_CTRL_BASE         0x7FFC0E0A40ull
+#define SHADOW_2_TENSOR_O_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_O_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_O_LOCAL_MME1_CTRL_BASE      0x7FFC0E0A8Cull
+#define SHADOW_2_AGU_O_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_O_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_O_REMOTE_MME1_CTRL_BASE     0x7FFC0E0AB0ull
+#define SHADOW_2_AGU_O_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_O_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_2_DESC_MME1_CTRL_BASE             0x7FFC0E0AD4ull
+#define SHADOW_2_DESC_MME1_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_2_DESC_MME1_CTRL_SECTION            0xB400
+#define mmSHADOW_3_MME1_CTRL_BASE                  0x7FFC0E0B88ull
+#define SHADOW_3_MME1_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_3_MME1_CTRL_SECTION                 0x3400
+#define mmSHADOW_3_TENSOR_S_MME1_CTRL_BASE         0x7FFC0E0BBCull
+#define SHADOW_3_TENSOR_S_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_S_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_S_MME1_CTRL_BASE            0x7FFC0E0C08ull
+#define SHADOW_3_AGU_S_MME1_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_3_AGU_S_MME1_CTRL_SECTION           0x2400
+#define mmSHADOW_3_TENSOR_L_MME1_CTRL_BASE         0x7FFC0E0C2Cull
+#define SHADOW_3_TENSOR_L_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_L_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_L_LOCAL_MME1_CTRL_BASE      0x7FFC0E0C78ull
+#define SHADOW_3_AGU_L_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_L_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_L_REMOTE_MME1_CTRL_BASE     0x7FFC0E0C9Cull
+#define SHADOW_3_AGU_L_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_L_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_3_TENSOR_O_MME1_CTRL_BASE         0x7FFC0E0CC0ull
+#define SHADOW_3_TENSOR_O_MME1_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_O_MME1_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_O_LOCAL_MME1_CTRL_BASE      0x7FFC0E0D0Cull
+#define SHADOW_3_AGU_O_LOCAL_MME1_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_O_LOCAL_MME1_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_O_REMOTE_MME1_CTRL_BASE     0x7FFC0E0D30ull
+#define SHADOW_3_AGU_O_REMOTE_MME1_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_O_REMOTE_MME1_CTRL_SECTION    0x2400
+#define mmSHADOW_3_DESC_MME1_CTRL_BASE             0x7FFC0E0D54ull
+#define SHADOW_3_DESC_MME1_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_3_DESC_MME1_CTRL_SECTION            0x72AC
+#define mmMME1_QM_BASE                             0x7FFC0E8000ull
+#define MME1_QM_MAX_OFFSET                         0xD040
+#define MME1_QM_SECTION                            0x38000
+#define mmMME2_ACC_BASE                            0x7FFC120000ull
+#define MME2_ACC_MAX_OFFSET                        0x5C00
+#define MME2_ACC_SECTION                           0x20000
+#define mmMME2_SBAB_BASE                           0x7FFC140000ull
+#define MME2_SBAB_MAX_OFFSET                       0x5800
+#define MME2_SBAB_SECTION                          0x1000
+#define mmMME2_PRTN_BASE                           0x7FFC141000ull
+#define MME2_PRTN_MAX_OFFSET                       0x5000
+#define MME2_PRTN_SECTION                          0x1F000
+#define mmMME2_CTRL_BASE                           0x7FFC160000ull
+#define MME2_CTRL_MAX_OFFSET                       0xDA80
+#define MME2_CTRL_SECTION                          0x8000
+#define mmARCH_MME2_CTRL_BASE                      0x7FFC160008ull
+#define ARCH_MME2_CTRL_MAX_OFFSET                  0x3400
+#define ARCH_MME2_CTRL_SECTION                     0x3400
+#define mmARCH_TENSOR_S_MME2_CTRL_BASE             0x7FFC16003Cull
+#define ARCH_TENSOR_S_MME2_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_S_MME2_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_S_MME2_CTRL_BASE                0x7FFC160088ull
+#define ARCH_AGU_S_MME2_CTRL_MAX_OFFSET            0x2400
+#define ARCH_AGU_S_MME2_CTRL_SECTION               0x2400
+#define mmARCH_TENSOR_L_MME2_CTRL_BASE             0x7FFC1600ACull
+#define ARCH_TENSOR_L_MME2_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_L_MME2_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_L_LOCAL_MME2_CTRL_BASE          0x7FFC1600F8ull
+#define ARCH_AGU_L_LOCAL_MME2_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_L_LOCAL_MME2_CTRL_SECTION         0x2400
+#define mmARCH_AGU_L_REMOTE_MME2_CTRL_BASE         0x7FFC16011Cull
+#define ARCH_AGU_L_REMOTE_MME2_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_L_REMOTE_MME2_CTRL_SECTION        0x2400
+#define mmARCH_TENSOR_O_MME2_CTRL_BASE             0x7FFC160140ull
+#define ARCH_TENSOR_O_MME2_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_O_MME2_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_O_LOCAL_MME2_CTRL_BASE          0x7FFC16018Cull
+#define ARCH_AGU_O_LOCAL_MME2_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_O_LOCAL_MME2_CTRL_SECTION         0x2400
+#define mmARCH_AGU_O_REMOTE_MME2_CTRL_BASE         0x7FFC1601B0ull
+#define ARCH_AGU_O_REMOTE_MME2_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_O_REMOTE_MME2_CTRL_SECTION        0x2400
+#define mmARCH_DESC_MME2_CTRL_BASE                 0x7FFC1601D4ull
+#define ARCH_DESC_MME2_CTRL_MAX_OFFSET             0x5400
+#define ARCH_DESC_MME2_CTRL_SECTION                0x2340
+#define mmSHADOW_0_MME2_CTRL_BASE                  0x7FFC160408ull
+#define SHADOW_0_MME2_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_0_MME2_CTRL_SECTION                 0x3400
+#define mmSHADOW_0_TENSOR_S_MME2_CTRL_BASE         0x7FFC16043Cull
+#define SHADOW_0_TENSOR_S_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_S_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_S_MME2_CTRL_BASE            0x7FFC160488ull
+#define SHADOW_0_AGU_S_MME2_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_0_AGU_S_MME2_CTRL_SECTION           0x2400
+#define mmSHADOW_0_TENSOR_L_MME2_CTRL_BASE         0x7FFC1604ACull
+#define SHADOW_0_TENSOR_L_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_L_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_L_LOCAL_MME2_CTRL_BASE      0x7FFC1604F8ull
+#define SHADOW_0_AGU_L_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_L_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_L_REMOTE_MME2_CTRL_BASE     0x7FFC16051Cull
+#define SHADOW_0_AGU_L_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_L_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_0_TENSOR_O_MME2_CTRL_BASE         0x7FFC160540ull
+#define SHADOW_0_TENSOR_O_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_O_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_O_LOCAL_MME2_CTRL_BASE      0x7FFC16058Cull
+#define SHADOW_0_AGU_O_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_O_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_O_REMOTE_MME2_CTRL_BASE     0x7FFC1605B0ull
+#define SHADOW_0_AGU_O_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_O_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_0_DESC_MME2_CTRL_BASE             0x7FFC1605D4ull
+#define SHADOW_0_DESC_MME2_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_0_DESC_MME2_CTRL_SECTION            0xB400
+#define mmSHADOW_1_MME2_CTRL_BASE                  0x7FFC160688ull
+#define SHADOW_1_MME2_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_1_MME2_CTRL_SECTION                 0x3400
+#define mmSHADOW_1_TENSOR_S_MME2_CTRL_BASE         0x7FFC1606BCull
+#define SHADOW_1_TENSOR_S_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_S_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_S_MME2_CTRL_BASE            0x7FFC160708ull
+#define SHADOW_1_AGU_S_MME2_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_1_AGU_S_MME2_CTRL_SECTION           0x2400
+#define mmSHADOW_1_TENSOR_L_MME2_CTRL_BASE         0x7FFC16072Cull
+#define SHADOW_1_TENSOR_L_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_L_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_L_LOCAL_MME2_CTRL_BASE      0x7FFC160778ull
+#define SHADOW_1_AGU_L_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_L_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_L_REMOTE_MME2_CTRL_BASE     0x7FFC16079Cull
+#define SHADOW_1_AGU_L_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_L_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_1_TENSOR_O_MME2_CTRL_BASE         0x7FFC1607C0ull
+#define SHADOW_1_TENSOR_O_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_O_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_O_LOCAL_MME2_CTRL_BASE      0x7FFC16080Cull
+#define SHADOW_1_AGU_O_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_O_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_O_REMOTE_MME2_CTRL_BASE     0x7FFC160830ull
+#define SHADOW_1_AGU_O_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_O_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_1_DESC_MME2_CTRL_BASE             0x7FFC160854ull
+#define SHADOW_1_DESC_MME2_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_1_DESC_MME2_CTRL_SECTION            0xB400
+#define mmSHADOW_2_MME2_CTRL_BASE                  0x7FFC160908ull
+#define SHADOW_2_MME2_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_2_MME2_CTRL_SECTION                 0x3400
+#define mmSHADOW_2_TENSOR_S_MME2_CTRL_BASE         0x7FFC16093Cull
+#define SHADOW_2_TENSOR_S_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_S_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_S_MME2_CTRL_BASE            0x7FFC160988ull
+#define SHADOW_2_AGU_S_MME2_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_2_AGU_S_MME2_CTRL_SECTION           0x2400
+#define mmSHADOW_2_TENSOR_L_MME2_CTRL_BASE         0x7FFC1609ACull
+#define SHADOW_2_TENSOR_L_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_L_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_L_LOCAL_MME2_CTRL_BASE      0x7FFC1609F8ull
+#define SHADOW_2_AGU_L_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_L_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_L_REMOTE_MME2_CTRL_BASE     0x7FFC160A1Cull
+#define SHADOW_2_AGU_L_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_L_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_2_TENSOR_O_MME2_CTRL_BASE         0x7FFC160A40ull
+#define SHADOW_2_TENSOR_O_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_O_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_O_LOCAL_MME2_CTRL_BASE      0x7FFC160A8Cull
+#define SHADOW_2_AGU_O_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_O_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_O_REMOTE_MME2_CTRL_BASE     0x7FFC160AB0ull
+#define SHADOW_2_AGU_O_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_O_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_2_DESC_MME2_CTRL_BASE             0x7FFC160AD4ull
+#define SHADOW_2_DESC_MME2_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_2_DESC_MME2_CTRL_SECTION            0xB400
+#define mmSHADOW_3_MME2_CTRL_BASE                  0x7FFC160B88ull
+#define SHADOW_3_MME2_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_3_MME2_CTRL_SECTION                 0x3400
+#define mmSHADOW_3_TENSOR_S_MME2_CTRL_BASE         0x7FFC160BBCull
+#define SHADOW_3_TENSOR_S_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_S_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_S_MME2_CTRL_BASE            0x7FFC160C08ull
+#define SHADOW_3_AGU_S_MME2_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_3_AGU_S_MME2_CTRL_SECTION           0x2400
+#define mmSHADOW_3_TENSOR_L_MME2_CTRL_BASE         0x7FFC160C2Cull
+#define SHADOW_3_TENSOR_L_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_L_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_L_LOCAL_MME2_CTRL_BASE      0x7FFC160C78ull
+#define SHADOW_3_AGU_L_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_L_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_L_REMOTE_MME2_CTRL_BASE     0x7FFC160C9Cull
+#define SHADOW_3_AGU_L_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_L_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_3_TENSOR_O_MME2_CTRL_BASE         0x7FFC160CC0ull
+#define SHADOW_3_TENSOR_O_MME2_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_O_MME2_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_O_LOCAL_MME2_CTRL_BASE      0x7FFC160D0Cull
+#define SHADOW_3_AGU_O_LOCAL_MME2_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_O_LOCAL_MME2_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_O_REMOTE_MME2_CTRL_BASE     0x7FFC160D30ull
+#define SHADOW_3_AGU_O_REMOTE_MME2_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_O_REMOTE_MME2_CTRL_SECTION    0x2400
+#define mmSHADOW_3_DESC_MME2_CTRL_BASE             0x7FFC160D54ull
+#define SHADOW_3_DESC_MME2_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_3_DESC_MME2_CTRL_SECTION            0x72AC
+#define mmMME2_QM_BASE                             0x7FFC168000ull
+#define MME2_QM_MAX_OFFSET                         0xD040
+#define MME2_QM_SECTION                            0x38000
+#define mmMME3_ACC_BASE                            0x7FFC1A0000ull
+#define MME3_ACC_MAX_OFFSET                        0x5C00
+#define MME3_ACC_SECTION                           0x20000
+#define mmMME3_SBAB_BASE                           0x7FFC1C0000ull
+#define MME3_SBAB_MAX_OFFSET                       0x5800
+#define MME3_SBAB_SECTION                          0x1000
+#define mmMME3_PRTN_BASE                           0x7FFC1C1000ull
+#define MME3_PRTN_MAX_OFFSET                       0x5000
+#define MME3_PRTN_SECTION                          0x1F000
+#define mmMME3_CTRL_BASE                           0x7FFC1E0000ull
+#define MME3_CTRL_MAX_OFFSET                       0xDA80
+#define MME3_CTRL_SECTION                          0x8000
+#define mmARCH_MME3_CTRL_BASE                      0x7FFC1E0008ull
+#define ARCH_MME3_CTRL_MAX_OFFSET                  0x3400
+#define ARCH_MME3_CTRL_SECTION                     0x3400
+#define mmARCH_TENSOR_S_MME3_CTRL_BASE             0x7FFC1E003Cull
+#define ARCH_TENSOR_S_MME3_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_S_MME3_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_S_MME3_CTRL_BASE                0x7FFC1E0088ull
+#define ARCH_AGU_S_MME3_CTRL_MAX_OFFSET            0x2400
+#define ARCH_AGU_S_MME3_CTRL_SECTION               0x2400
+#define mmARCH_TENSOR_L_MME3_CTRL_BASE             0x7FFC1E00ACull
+#define ARCH_TENSOR_L_MME3_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_L_MME3_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_L_LOCAL_MME3_CTRL_BASE          0x7FFC1E00F8ull
+#define ARCH_AGU_L_LOCAL_MME3_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_L_LOCAL_MME3_CTRL_SECTION         0x2400
+#define mmARCH_AGU_L_REMOTE_MME3_CTRL_BASE         0x7FFC1E011Cull
+#define ARCH_AGU_L_REMOTE_MME3_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_L_REMOTE_MME3_CTRL_SECTION        0x2400
+#define mmARCH_TENSOR_O_MME3_CTRL_BASE             0x7FFC1E0140ull
+#define ARCH_TENSOR_O_MME3_CTRL_MAX_OFFSET         0x4C00
+#define ARCH_TENSOR_O_MME3_CTRL_SECTION            0x4C00
+#define mmARCH_AGU_O_LOCAL_MME3_CTRL_BASE          0x7FFC1E018Cull
+#define ARCH_AGU_O_LOCAL_MME3_CTRL_MAX_OFFSET      0x2400
+#define ARCH_AGU_O_LOCAL_MME3_CTRL_SECTION         0x2400
+#define mmARCH_AGU_O_REMOTE_MME3_CTRL_BASE         0x7FFC1E01B0ull
+#define ARCH_AGU_O_REMOTE_MME3_CTRL_MAX_OFFSET     0x2400
+#define ARCH_AGU_O_REMOTE_MME3_CTRL_SECTION        0x2400
+#define mmARCH_DESC_MME3_CTRL_BASE                 0x7FFC1E01D4ull
+#define ARCH_DESC_MME3_CTRL_MAX_OFFSET             0x5400
+#define ARCH_DESC_MME3_CTRL_SECTION                0x2340
+#define mmSHADOW_0_MME3_CTRL_BASE                  0x7FFC1E0408ull
+#define SHADOW_0_MME3_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_0_MME3_CTRL_SECTION                 0x3400
+#define mmSHADOW_0_TENSOR_S_MME3_CTRL_BASE         0x7FFC1E043Cull
+#define SHADOW_0_TENSOR_S_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_S_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_S_MME3_CTRL_BASE            0x7FFC1E0488ull
+#define SHADOW_0_AGU_S_MME3_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_0_AGU_S_MME3_CTRL_SECTION           0x2400
+#define mmSHADOW_0_TENSOR_L_MME3_CTRL_BASE         0x7FFC1E04ACull
+#define SHADOW_0_TENSOR_L_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_L_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_L_LOCAL_MME3_CTRL_BASE      0x7FFC1E04F8ull
+#define SHADOW_0_AGU_L_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_L_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_L_REMOTE_MME3_CTRL_BASE     0x7FFC1E051Cull
+#define SHADOW_0_AGU_L_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_L_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_0_TENSOR_O_MME3_CTRL_BASE         0x7FFC1E0540ull
+#define SHADOW_0_TENSOR_O_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_0_TENSOR_O_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_0_AGU_O_LOCAL_MME3_CTRL_BASE      0x7FFC1E058Cull
+#define SHADOW_0_AGU_O_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_0_AGU_O_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_0_AGU_O_REMOTE_MME3_CTRL_BASE     0x7FFC1E05B0ull
+#define SHADOW_0_AGU_O_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_0_AGU_O_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_0_DESC_MME3_CTRL_BASE             0x7FFC1E05D4ull
+#define SHADOW_0_DESC_MME3_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_0_DESC_MME3_CTRL_SECTION            0xB400
+#define mmSHADOW_1_MME3_CTRL_BASE                  0x7FFC1E0688ull
+#define SHADOW_1_MME3_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_1_MME3_CTRL_SECTION                 0x3400
+#define mmSHADOW_1_TENSOR_S_MME3_CTRL_BASE         0x7FFC1E06BCull
+#define SHADOW_1_TENSOR_S_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_S_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_S_MME3_CTRL_BASE            0x7FFC1E0708ull
+#define SHADOW_1_AGU_S_MME3_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_1_AGU_S_MME3_CTRL_SECTION           0x2400
+#define mmSHADOW_1_TENSOR_L_MME3_CTRL_BASE         0x7FFC1E072Cull
+#define SHADOW_1_TENSOR_L_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_L_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_L_LOCAL_MME3_CTRL_BASE      0x7FFC1E0778ull
+#define SHADOW_1_AGU_L_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_L_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_L_REMOTE_MME3_CTRL_BASE     0x7FFC1E079Cull
+#define SHADOW_1_AGU_L_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_L_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_1_TENSOR_O_MME3_CTRL_BASE         0x7FFC1E07C0ull
+#define SHADOW_1_TENSOR_O_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_1_TENSOR_O_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_1_AGU_O_LOCAL_MME3_CTRL_BASE      0x7FFC1E080Cull
+#define SHADOW_1_AGU_O_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_1_AGU_O_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_1_AGU_O_REMOTE_MME3_CTRL_BASE     0x7FFC1E0830ull
+#define SHADOW_1_AGU_O_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_1_AGU_O_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_1_DESC_MME3_CTRL_BASE             0x7FFC1E0854ull
+#define SHADOW_1_DESC_MME3_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_1_DESC_MME3_CTRL_SECTION            0xB400
+#define mmSHADOW_2_MME3_CTRL_BASE                  0x7FFC1E0908ull
+#define SHADOW_2_MME3_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_2_MME3_CTRL_SECTION                 0x3400
+#define mmSHADOW_2_TENSOR_S_MME3_CTRL_BASE         0x7FFC1E093Cull
+#define SHADOW_2_TENSOR_S_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_S_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_S_MME3_CTRL_BASE            0x7FFC1E0988ull
+#define SHADOW_2_AGU_S_MME3_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_2_AGU_S_MME3_CTRL_SECTION           0x2400
+#define mmSHADOW_2_TENSOR_L_MME3_CTRL_BASE         0x7FFC1E09ACull
+#define SHADOW_2_TENSOR_L_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_L_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_L_LOCAL_MME3_CTRL_BASE      0x7FFC1E09F8ull
+#define SHADOW_2_AGU_L_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_L_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_L_REMOTE_MME3_CTRL_BASE     0x7FFC1E0A1Cull
+#define SHADOW_2_AGU_L_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_L_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_2_TENSOR_O_MME3_CTRL_BASE         0x7FFC1E0A40ull
+#define SHADOW_2_TENSOR_O_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_2_TENSOR_O_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_2_AGU_O_LOCAL_MME3_CTRL_BASE      0x7FFC1E0A8Cull
+#define SHADOW_2_AGU_O_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_2_AGU_O_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_2_AGU_O_REMOTE_MME3_CTRL_BASE     0x7FFC1E0AB0ull
+#define SHADOW_2_AGU_O_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_2_AGU_O_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_2_DESC_MME3_CTRL_BASE             0x7FFC1E0AD4ull
+#define SHADOW_2_DESC_MME3_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_2_DESC_MME3_CTRL_SECTION            0xB400
+#define mmSHADOW_3_MME3_CTRL_BASE                  0x7FFC1E0B88ull
+#define SHADOW_3_MME3_CTRL_MAX_OFFSET              0x3400
+#define SHADOW_3_MME3_CTRL_SECTION                 0x3400
+#define mmSHADOW_3_TENSOR_S_MME3_CTRL_BASE         0x7FFC1E0BBCull
+#define SHADOW_3_TENSOR_S_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_S_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_S_MME3_CTRL_BASE            0x7FFC1E0C08ull
+#define SHADOW_3_AGU_S_MME3_CTRL_MAX_OFFSET        0x2400
+#define SHADOW_3_AGU_S_MME3_CTRL_SECTION           0x2400
+#define mmSHADOW_3_TENSOR_L_MME3_CTRL_BASE         0x7FFC1E0C2Cull
+#define SHADOW_3_TENSOR_L_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_L_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_L_LOCAL_MME3_CTRL_BASE      0x7FFC1E0C78ull
+#define SHADOW_3_AGU_L_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_L_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_L_REMOTE_MME3_CTRL_BASE     0x7FFC1E0C9Cull
+#define SHADOW_3_AGU_L_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_L_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_3_TENSOR_O_MME3_CTRL_BASE         0x7FFC1E0CC0ull
+#define SHADOW_3_TENSOR_O_MME3_CTRL_MAX_OFFSET     0x4C00
+#define SHADOW_3_TENSOR_O_MME3_CTRL_SECTION        0x4C00
+#define mmSHADOW_3_AGU_O_LOCAL_MME3_CTRL_BASE      0x7FFC1E0D0Cull
+#define SHADOW_3_AGU_O_LOCAL_MME3_CTRL_MAX_OFFSET  0x2400
+#define SHADOW_3_AGU_O_LOCAL_MME3_CTRL_SECTION     0x2400
+#define mmSHADOW_3_AGU_O_REMOTE_MME3_CTRL_BASE     0x7FFC1E0D30ull
+#define SHADOW_3_AGU_O_REMOTE_MME3_CTRL_MAX_OFFSET 0x2400
+#define SHADOW_3_AGU_O_REMOTE_MME3_CTRL_SECTION    0x2400
+#define mmSHADOW_3_DESC_MME3_CTRL_BASE             0x7FFC1E0D54ull
+#define SHADOW_3_DESC_MME3_CTRL_MAX_OFFSET         0x5400
+#define SHADOW_3_DESC_MME3_CTRL_SECTION            0x72AC
+#define mmMME3_QM_BASE                             0x7FFC1E8000ull
+#define MME3_QM_MAX_OFFSET                         0xD040
+#define MME3_QM_SECTION                            0x18000
+#define mmSRAM_Y0_X0_BANK_BASE                     0x7FFC200000ull
+#define SRAM_Y0_X0_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X0_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X0_RTR_BASE                      0x7FFC201000ull
+#define SRAM_Y0_X0_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X0_RTR_SECTION                     0x7000
+#define mmSRAM_Y0_X1_BANK_BASE                     0x7FFC208000ull
+#define SRAM_Y0_X1_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X1_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X1_RTR_BASE                      0x7FFC209000ull
+#define SRAM_Y0_X1_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X1_RTR_SECTION                     0x7000
+#define mmSRAM_Y0_X2_BANK_BASE                     0x7FFC210000ull
+#define SRAM_Y0_X2_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X2_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X2_RTR_BASE                      0x7FFC211000ull
+#define SRAM_Y0_X2_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X2_RTR_SECTION                     0x7000
+#define mmSRAM_Y0_X3_BANK_BASE                     0x7FFC218000ull
+#define SRAM_Y0_X3_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X3_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X3_RTR_BASE                      0x7FFC219000ull
+#define SRAM_Y0_X3_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X3_RTR_SECTION                     0x7000
+#define mmSRAM_Y0_X4_BANK_BASE                     0x7FFC220000ull
+#define SRAM_Y0_X4_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X4_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X4_RTR_BASE                      0x7FFC221000ull
+#define SRAM_Y0_X4_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X4_RTR_SECTION                     0x7000
+#define mmSRAM_Y0_X5_BANK_BASE                     0x7FFC228000ull
+#define SRAM_Y0_X5_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X5_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X5_RTR_BASE                      0x7FFC229000ull
+#define SRAM_Y0_X5_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X5_RTR_SECTION                     0x7000
+#define mmSRAM_Y0_X6_BANK_BASE                     0x7FFC230000ull
+#define SRAM_Y0_X6_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X6_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X6_RTR_BASE                      0x7FFC231000ull
+#define SRAM_Y0_X6_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X6_RTR_SECTION                     0x7000
+#define mmSRAM_Y0_X7_BANK_BASE                     0x7FFC238000ull
+#define SRAM_Y0_X7_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y0_X7_BANK_SECTION                    0x1000
+#define mmSRAM_Y0_X7_RTR_BASE                      0x7FFC239000ull
+#define SRAM_Y0_X7_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y0_X7_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X0_BANK_BASE                     0x7FFC240000ull
+#define SRAM_Y1_X0_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X0_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X0_RTR_BASE                      0x7FFC241000ull
+#define SRAM_Y1_X0_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X0_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X1_BANK_BASE                     0x7FFC248000ull
+#define SRAM_Y1_X1_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X1_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X1_RTR_BASE                      0x7FFC249000ull
+#define SRAM_Y1_X1_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X1_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X2_BANK_BASE                     0x7FFC250000ull
+#define SRAM_Y1_X2_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X2_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X2_RTR_BASE                      0x7FFC251000ull
+#define SRAM_Y1_X2_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X2_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X3_BANK_BASE                     0x7FFC258000ull
+#define SRAM_Y1_X3_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X3_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X3_RTR_BASE                      0x7FFC259000ull
+#define SRAM_Y1_X3_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X3_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X4_BANK_BASE                     0x7FFC260000ull
+#define SRAM_Y1_X4_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X4_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X4_RTR_BASE                      0x7FFC261000ull
+#define SRAM_Y1_X4_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X4_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X5_BANK_BASE                     0x7FFC268000ull
+#define SRAM_Y1_X5_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X5_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X5_RTR_BASE                      0x7FFC269000ull
+#define SRAM_Y1_X5_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X5_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X6_BANK_BASE                     0x7FFC270000ull
+#define SRAM_Y1_X6_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X6_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X6_RTR_BASE                      0x7FFC271000ull
+#define SRAM_Y1_X6_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X6_RTR_SECTION                     0x7000
+#define mmSRAM_Y1_X7_BANK_BASE                     0x7FFC278000ull
+#define SRAM_Y1_X7_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y1_X7_BANK_SECTION                    0x1000
+#define mmSRAM_Y1_X7_RTR_BASE                      0x7FFC279000ull
+#define SRAM_Y1_X7_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y1_X7_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X0_BANK_BASE                     0x7FFC280000ull
+#define SRAM_Y2_X0_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X0_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X0_RTR_BASE                      0x7FFC281000ull
+#define SRAM_Y2_X0_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X0_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X1_BANK_BASE                     0x7FFC288000ull
+#define SRAM_Y2_X1_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X1_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X1_RTR_BASE                      0x7FFC289000ull
+#define SRAM_Y2_X1_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X1_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X2_BANK_BASE                     0x7FFC290000ull
+#define SRAM_Y2_X2_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X2_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X2_RTR_BASE                      0x7FFC291000ull
+#define SRAM_Y2_X2_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X2_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X3_BANK_BASE                     0x7FFC298000ull
+#define SRAM_Y2_X3_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X3_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X3_RTR_BASE                      0x7FFC299000ull
+#define SRAM_Y2_X3_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X3_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X4_BANK_BASE                     0x7FFC2A0000ull
+#define SRAM_Y2_X4_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X4_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X4_RTR_BASE                      0x7FFC2A1000ull
+#define SRAM_Y2_X4_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X4_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X5_BANK_BASE                     0x7FFC2A8000ull
+#define SRAM_Y2_X5_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X5_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X5_RTR_BASE                      0x7FFC2A9000ull
+#define SRAM_Y2_X5_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X5_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X6_BANK_BASE                     0x7FFC2B0000ull
+#define SRAM_Y2_X6_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X6_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X6_RTR_BASE                      0x7FFC2B1000ull
+#define SRAM_Y2_X6_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X6_RTR_SECTION                     0x7000
+#define mmSRAM_Y2_X7_BANK_BASE                     0x7FFC2B8000ull
+#define SRAM_Y2_X7_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y2_X7_BANK_SECTION                    0x1000
+#define mmSRAM_Y2_X7_RTR_BASE                      0x7FFC2B9000ull
+#define SRAM_Y2_X7_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y2_X7_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X0_BANK_BASE                     0x7FFC2C0000ull
+#define SRAM_Y3_X0_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X0_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X0_RTR_BASE                      0x7FFC2C1000ull
+#define SRAM_Y3_X0_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X0_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X1_BANK_BASE                     0x7FFC2C8000ull
+#define SRAM_Y3_X1_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X1_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X1_RTR_BASE                      0x7FFC2C9000ull
+#define SRAM_Y3_X1_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X1_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X2_BANK_BASE                     0x7FFC2D0000ull
+#define SRAM_Y3_X2_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X2_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X2_RTR_BASE                      0x7FFC2D1000ull
+#define SRAM_Y3_X2_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X2_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X3_BANK_BASE                     0x7FFC2D8000ull
+#define SRAM_Y3_X3_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X3_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X3_RTR_BASE                      0x7FFC2D9000ull
+#define SRAM_Y3_X3_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X3_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X4_BANK_BASE                     0x7FFC2E0000ull
+#define SRAM_Y3_X4_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X4_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X4_RTR_BASE                      0x7FFC2E1000ull
+#define SRAM_Y3_X4_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X4_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X5_BANK_BASE                     0x7FFC2E8000ull
+#define SRAM_Y3_X5_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X5_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X5_RTR_BASE                      0x7FFC2E9000ull
+#define SRAM_Y3_X5_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X5_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X6_BANK_BASE                     0x7FFC2F0000ull
+#define SRAM_Y3_X6_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X6_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X6_RTR_BASE                      0x7FFC2F1000ull
+#define SRAM_Y3_X6_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X6_RTR_SECTION                     0x7000
+#define mmSRAM_Y3_X7_BANK_BASE                     0x7FFC2F8000ull
+#define SRAM_Y3_X7_BANK_MAX_OFFSET                 0x4000
+#define SRAM_Y3_X7_BANK_SECTION                    0x1000
+#define mmSRAM_Y3_X7_RTR_BASE                      0x7FFC2F9000ull
+#define SRAM_Y3_X7_RTR_MAX_OFFSET                  0x3340
+#define SRAM_Y3_X7_RTR_SECTION                     0x7000
+#define mmSIF_RTR_0_BASE                           0x7FFC300000ull
+#define SIF_RTR_0_MAX_OFFSET                       0x6500
+#define SIF_RTR_0_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_0_BASE                      0x7FFC306000ull
+#define SIF_RTR_CTRL_0_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_0_SECTION                     0xA000
+#define mmSIF_RTR_1_BASE                           0x7FFC310000ull
+#define SIF_RTR_1_MAX_OFFSET                       0x6500
+#define SIF_RTR_1_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_1_BASE                      0x7FFC316000ull
+#define SIF_RTR_CTRL_1_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_1_SECTION                     0xA000
+#define mmSIF_RTR_2_BASE                           0x7FFC320000ull
+#define SIF_RTR_2_MAX_OFFSET                       0x6500
+#define SIF_RTR_2_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_2_BASE                      0x7FFC326000ull
+#define SIF_RTR_CTRL_2_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_2_SECTION                     0xA000
+#define mmSIF_RTR_3_BASE                           0x7FFC330000ull
+#define SIF_RTR_3_MAX_OFFSET                       0x6500
+#define SIF_RTR_3_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_3_BASE                      0x7FFC336000ull
+#define SIF_RTR_CTRL_3_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_3_SECTION                     0xA000
+#define mmSIF_RTR_4_BASE                           0x7FFC340000ull
+#define SIF_RTR_4_MAX_OFFSET                       0x6500
+#define SIF_RTR_4_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_4_BASE                      0x7FFC346000ull
+#define SIF_RTR_CTRL_4_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_4_SECTION                     0xA000
+#define mmSIF_RTR_5_BASE                           0x7FFC350000ull
+#define SIF_RTR_5_MAX_OFFSET                       0x6500
+#define SIF_RTR_5_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_5_BASE                      0x7FFC356000ull
+#define SIF_RTR_CTRL_5_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_5_SECTION                     0xA000
+#define mmSIF_RTR_6_BASE                           0x7FFC360000ull
+#define SIF_RTR_6_MAX_OFFSET                       0x6500
+#define SIF_RTR_6_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_6_BASE                      0x7FFC366000ull
+#define SIF_RTR_CTRL_6_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_6_SECTION                     0xA000
+#define mmSIF_RTR_7_BASE                           0x7FFC370000ull
+#define SIF_RTR_7_MAX_OFFSET                       0x6500
+#define SIF_RTR_7_SECTION                          0x6000
+#define mmSIF_RTR_CTRL_7_BASE                      0x7FFC376000ull
+#define SIF_RTR_CTRL_7_MAX_OFFSET                  0xCC00
+#define SIF_RTR_CTRL_7_SECTION                     0xA000
+#define mmNIF_RTR_0_BASE                           0x7FFC380000ull
+#define NIF_RTR_0_MAX_OFFSET                       0x6500
+#define NIF_RTR_0_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_0_BASE                      0x7FFC386000ull
+#define NIF_RTR_CTRL_0_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_0_SECTION                     0xA000
+#define mmNIF_RTR_1_BASE                           0x7FFC390000ull
+#define NIF_RTR_1_MAX_OFFSET                       0x6500
+#define NIF_RTR_1_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_1_BASE                      0x7FFC396000ull
+#define NIF_RTR_CTRL_1_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_1_SECTION                     0xA000
+#define mmNIF_RTR_2_BASE                           0x7FFC3A0000ull
+#define NIF_RTR_2_MAX_OFFSET                       0x6500
+#define NIF_RTR_2_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_2_BASE                      0x7FFC3A6000ull
+#define NIF_RTR_CTRL_2_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_2_SECTION                     0xA000
+#define mmNIF_RTR_3_BASE                           0x7FFC3B0000ull
+#define NIF_RTR_3_MAX_OFFSET                       0x6500
+#define NIF_RTR_3_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_3_BASE                      0x7FFC3B6000ull
+#define NIF_RTR_CTRL_3_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_3_SECTION                     0xA000
+#define mmNIF_RTR_4_BASE                           0x7FFC3C0000ull
+#define NIF_RTR_4_MAX_OFFSET                       0x6500
+#define NIF_RTR_4_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_4_BASE                      0x7FFC3C6000ull
+#define NIF_RTR_CTRL_4_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_4_SECTION                     0xA000
+#define mmNIF_RTR_5_BASE                           0x7FFC3D0000ull
+#define NIF_RTR_5_MAX_OFFSET                       0x6500
+#define NIF_RTR_5_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_5_BASE                      0x7FFC3D6000ull
+#define NIF_RTR_CTRL_5_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_5_SECTION                     0xA000
+#define mmNIF_RTR_6_BASE                           0x7FFC3E0000ull
+#define NIF_RTR_6_MAX_OFFSET                       0x6500
+#define NIF_RTR_6_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_6_BASE                      0x7FFC3E6000ull
+#define NIF_RTR_CTRL_6_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_6_SECTION                     0xA000
+#define mmNIF_RTR_7_BASE                           0x7FFC3F0000ull
+#define NIF_RTR_7_MAX_OFFSET                       0x6500
+#define NIF_RTR_7_SECTION                          0x6000
+#define mmNIF_RTR_CTRL_7_BASE                      0x7FFC3F6000ull
+#define NIF_RTR_CTRL_7_MAX_OFFSET                  0xCC00
+#define NIF_RTR_CTRL_7_SECTION                     0x4B000
+#define mmCPU_CA53_CFG_BASE                        0x7FFC441000ull
+#define CPU_CA53_CFG_MAX_OFFSET                    0x2180
+#define CPU_CA53_CFG_SECTION                       0x1000
+#define mmCPU_IF_BASE                              0x7FFC442000ull
+#define CPU_IF_MAX_OFFSET                          0x43C0
+#define CPU_IF_SECTION                             0x2000
+#define mmCPU_TIMESTAMP_BASE                       0x7FFC444000ull
+#define CPU_TIMESTAMP_MAX_OFFSET                   0x1000
+#define CPU_TIMESTAMP_SECTION                      0x3C000
+#define mmDMA_IF_W_S_BASE                          0x7FFC480000ull
+#define DMA_IF_W_S_MAX_OFFSET                      0x8380
+#define DMA_IF_W_S_SECTION                         0x1000
+#define mmDMA_IF_W_S_DOWN_CH0_BASE                 0x7FFC481000ull
+#define DMA_IF_W_S_DOWN_CH0_MAX_OFFSET             0xCC00
+#define DMA_IF_W_S_DOWN_CH0_SECTION                0x1000
+#define mmDMA_IF_W_S_DOWN_CH1_BASE                 0x7FFC482000ull
+#define DMA_IF_W_S_DOWN_CH1_MAX_OFFSET             0xCC00
+#define DMA_IF_W_S_DOWN_CH1_SECTION                0x5000
+#define mmDMA_W_PLL_BASE                           0x7FFC487000ull
+#define DMA_W_PLL_MAX_OFFSET                       0x5200
+#define DMA_W_PLL_SECTION                          0x1000
+#define mmIF_W_PLL_BASE                            0x7FFC488000ull
+#define IF_W_PLL_MAX_OFFSET                        0x5200
+#define IF_W_PLL_SECTION                           0x1000
+#define mmDMA_IF_W_S_DOWN_BASE                     0x7FFC489000ull
+#define DMA_IF_W_S_DOWN_MAX_OFFSET                 0x1500
+#define DMA_IF_W_S_DOWN_SECTION                    0x7000
+#define mmSYNC_MNGR_GLBL_W_S_BASE                  0x7FFC490000ull
+#define SYNC_MNGR_GLBL_W_S_MAX_OFFSET              0x6C00
+#define SYNC_MNGR_GLBL_W_S_SECTION                 0x1000
+#define mmSYNC_MNGR_OBJS_W_S_BASE                  0x7FFC491000ull
+#define SYNC_MNGR_OBJS_W_S_MAX_OFFSET              0x5C00
+#define SYNC_MNGR_OBJS_W_S_SECTION                 0xF000
+#define mmDMA_IF_E_S_BASE                          0x7FFC4A0000ull
+#define DMA_IF_E_S_MAX_OFFSET                      0x8380
+#define DMA_IF_E_S_SECTION                         0x1000
+#define mmDMA_IF_E_S_DOWN_CH0_BASE                 0x7FFC4A1000ull
+#define DMA_IF_E_S_DOWN_CH0_MAX_OFFSET             0xCC00
+#define DMA_IF_E_S_DOWN_CH0_SECTION                0x1000
+#define mmDMA_IF_E_S_DOWN_CH1_BASE                 0x7FFC4A2000ull
+#define DMA_IF_E_S_DOWN_CH1_MAX_OFFSET             0xCC00
+#define DMA_IF_E_S_DOWN_CH1_SECTION                0x5000
+#define mmIF_E_PLL_BASE                            0x7FFC4A7000ull
+#define IF_E_PLL_MAX_OFFSET                        0x5200
+#define IF_E_PLL_SECTION                           0x1000
+#define mmDMA_E_PLL_BASE                           0x7FFC4A8000ull
+#define DMA_E_PLL_MAX_OFFSET                       0x5200
+#define DMA_E_PLL_SECTION                          0x1000
+#define mmDMA_IF_E_S_DOWN_BASE                     0x7FFC4A9000ull
+#define DMA_IF_E_S_DOWN_MAX_OFFSET                 0x1500
+#define DMA_IF_E_S_DOWN_SECTION                    0x7000
+#define mmSYNC_MNGR_GLBL_E_S_BASE                  0x7FFC4B0000ull
+#define SYNC_MNGR_GLBL_E_S_MAX_OFFSET              0x6C00
+#define SYNC_MNGR_GLBL_E_S_SECTION                 0x1000
+#define mmSYNC_MNGR_OBJS_E_S_BASE                  0x7FFC4B1000ull
+#define SYNC_MNGR_OBJS_E_S_MAX_OFFSET              0x5C00
+#define SYNC_MNGR_OBJS_E_S_SECTION                 0xF000
+#define mmDMA_IF_W_N_BASE                          0x7FFC4C0000ull
+#define DMA_IF_W_N_MAX_OFFSET                      0x8380
+#define DMA_IF_W_N_SECTION                         0x1000
+#define mmDMA_IF_W_N_DOWN_CH0_BASE                 0x7FFC4C1000ull
+#define DMA_IF_W_N_DOWN_CH0_MAX_OFFSET             0xCC00
+#define DMA_IF_W_N_DOWN_CH0_SECTION                0x1000
+#define mmDMA_IF_W_N_DOWN_CH1_BASE                 0x7FFC4C2000ull
+#define DMA_IF_W_N_DOWN_CH1_MAX_OFFSET             0xCC00
+#define DMA_IF_W_N_DOWN_CH1_SECTION                0x5000
+#define mmMESH_W_PLL_BASE                          0x7FFC4C7000ull
+#define MESH_W_PLL_MAX_OFFSET                      0x5200
+#define MESH_W_PLL_SECTION                         0x1000
+#define mmSRAM_W_PLL_BASE                          0x7FFC4C8000ull
+#define SRAM_W_PLL_MAX_OFFSET                      0x5200
+#define SRAM_W_PLL_SECTION                         0x1000
+#define mmDMA_IF_W_N_DOWN_BASE                     0x7FFC4C9000ull
+#define DMA_IF_W_N_DOWN_MAX_OFFSET                 0x1500
+#define DMA_IF_W_N_DOWN_SECTION                    0x7000
+#define mmSYNC_MNGR_GLBL_W_N_BASE                  0x7FFC4D0000ull
+#define SYNC_MNGR_GLBL_W_N_MAX_OFFSET              0x6C00
+#define SYNC_MNGR_GLBL_W_N_SECTION                 0x1000
+#define mmSYNC_MNGR_OBJS_W_N_BASE                  0x7FFC4D1000ull
+#define SYNC_MNGR_OBJS_W_N_MAX_OFFSET              0x5C00
+#define SYNC_MNGR_OBJS_W_N_SECTION                 0xF000
+#define mmDMA_IF_E_N_BASE                          0x7FFC4E0000ull
+#define DMA_IF_E_N_MAX_OFFSET                      0x8380
+#define DMA_IF_E_N_SECTION                         0x1000
+#define mmDMA_IF_E_N_DOWN_CH0_BASE                 0x7FFC4E1000ull
+#define DMA_IF_E_N_DOWN_CH0_MAX_OFFSET             0xCC00
+#define DMA_IF_E_N_DOWN_CH0_SECTION                0x1000
+#define mmDMA_IF_E_N_DOWN_CH1_BASE                 0x7FFC4E2000ull
+#define DMA_IF_E_N_DOWN_CH1_MAX_OFFSET             0xCC00
+#define DMA_IF_E_N_DOWN_CH1_SECTION                0x5000
+#define mmMESH_E_PLL_BASE                          0x7FFC4E7000ull
+#define MESH_E_PLL_MAX_OFFSET                      0x5200
+#define MESH_E_PLL_SECTION                         0x1000
+#define mmSRAM_E_PLL_BASE                          0x7FFC4E8000ull
+#define SRAM_E_PLL_MAX_OFFSET                      0x5200
+#define SRAM_E_PLL_SECTION                         0x1000
+#define mmDMA_IF_E_N_DOWN_BASE                     0x7FFC4E9000ull
+#define DMA_IF_E_N_DOWN_MAX_OFFSET                 0x1500
+#define DMA_IF_E_N_DOWN_SECTION                    0x7000
+#define mmSYNC_MNGR_GLBL_E_N_BASE                  0x7FFC4F0000ull
+#define SYNC_MNGR_GLBL_E_N_MAX_OFFSET              0x6C00
+#define SYNC_MNGR_GLBL_E_N_SECTION                 0x1000
+#define mmSYNC_MNGR_OBJS_E_N_BASE                  0x7FFC4F1000ull
+#define SYNC_MNGR_OBJS_E_N_MAX_OFFSET              0x5C00
+#define SYNC_MNGR_OBJS_E_N_SECTION                 0xF000
+#define mmDMA0_CORE_BASE                           0x7FFC500000ull
+#define DMA0_CORE_MAX_OFFSET                       0x23C0
+#define DMA0_CORE_SECTION                          0x8000
+#define mmDMA0_QM_BASE                             0x7FFC508000ull
+#define DMA0_QM_MAX_OFFSET                         0xD040
+#define DMA0_QM_SECTION                            0x18000
+#define mmDMA1_CORE_BASE                           0x7FFC520000ull
+#define DMA1_CORE_MAX_OFFSET                       0x23C0
+#define DMA1_CORE_SECTION                          0x8000
+#define mmDMA1_QM_BASE                             0x7FFC528000ull
+#define DMA1_QM_MAX_OFFSET                         0xD040
+#define DMA1_QM_SECTION                            0x18000
+#define mmDMA2_CORE_BASE                           0x7FFC540000ull
+#define DMA2_CORE_MAX_OFFSET                       0x23C0
+#define DMA2_CORE_SECTION                          0x8000
+#define mmDMA2_QM_BASE                             0x7FFC548000ull
+#define DMA2_QM_MAX_OFFSET                         0xD040
+#define DMA2_QM_SECTION                            0x18000
+#define mmDMA3_CORE_BASE                           0x7FFC560000ull
+#define DMA3_CORE_MAX_OFFSET                       0x23C0
+#define DMA3_CORE_SECTION                          0x8000
+#define mmDMA3_QM_BASE                             0x7FFC568000ull
+#define DMA3_QM_MAX_OFFSET                         0xD040
+#define DMA3_QM_SECTION                            0x18000
+#define mmDMA4_CORE_BASE                           0x7FFC580000ull
+#define DMA4_CORE_MAX_OFFSET                       0x23C0
+#define DMA4_CORE_SECTION                          0x8000
+#define mmDMA4_QM_BASE                             0x7FFC588000ull
+#define DMA4_QM_MAX_OFFSET                         0xD040
+#define DMA4_QM_SECTION                            0x18000
+#define mmDMA5_CORE_BASE                           0x7FFC5A0000ull
+#define DMA5_CORE_MAX_OFFSET                       0x23C0
+#define DMA5_CORE_SECTION                          0x8000
+#define mmDMA5_QM_BASE                             0x7FFC5A8000ull
+#define DMA5_QM_MAX_OFFSET                         0xD040
+#define DMA5_QM_SECTION                            0x18000
+#define mmDMA6_CORE_BASE                           0x7FFC5C0000ull
+#define DMA6_CORE_MAX_OFFSET                       0x23C0
+#define DMA6_CORE_SECTION                          0x8000
+#define mmDMA6_QM_BASE                             0x7FFC5C8000ull
+#define DMA6_QM_MAX_OFFSET                         0xD040
+#define DMA6_QM_SECTION                            0x18000
+#define mmDMA7_CORE_BASE                           0x7FFC5E0000ull
+#define DMA7_CORE_MAX_OFFSET                       0x23C0
+#define DMA7_CORE_SECTION                          0x8000
+#define mmDMA7_QM_BASE                             0x7FFC5E8000ull
+#define DMA7_QM_MAX_OFFSET                         0xD040
+#define DMA7_QM_SECTION                            0x18000
+#define mmHBM0_BASE                                0x7FFC600000ull
+#define HBM0_MAX_OFFSET                            0x8F58
+#define HBM0_SECTION                               0x80000
+#define mmHBM1_BASE                                0x7FFC680000ull
+#define HBM1_MAX_OFFSET                            0x8F58
+#define HBM1_SECTION                               0x80000
+#define mmHBM2_BASE                                0x7FFC700000ull
+#define HBM2_MAX_OFFSET                            0x8F58
+#define HBM2_SECTION                               0x80000
+#define mmHBM3_BASE                                0x7FFC780000ull
+#define HBM3_MAX_OFFSET                            0x8F58
+#define HBM3_SECTION                               0x80000
+#define mmGIC_BASE                                 0x7FFC800000ull
+#define GIC_MAX_OFFSET                             0x10000
+#define GIC_SECTION                                0x401000
+#define mmPCIE_WRAP_BASE                           0x7FFCC01000ull
+#define PCIE_WRAP_MAX_OFFSET                       0xDF00
+#define PCIE_WRAP_SECTION                          0x1000
+#define mmPCIE_DBI_BASE                            0x7FFCC02000ull
+#define PCIE_DBI_MAX_OFFSET                        0xC040
+#define PCIE_DBI_SECTION                           0x2000
+#define mmPCIE_CORE_BASE                           0x7FFCC04000ull
+#define PCIE_CORE_MAX_OFFSET                       0x9BC0
+#define PCIE_CORE_SECTION                          0x3000
+#define mmPCIE_AUX_BASE                            0x7FFCC07000ull
+#define PCIE_AUX_MAX_OFFSET                        0x9C40
+#define PCIE_AUX_SECTION                           0x9000
+#define mmPCIE_PHY_BASE                            0x7FFCC10000ull
+#define PCIE_PHY_MAX_OFFSET                        0x9640
+#define PCIE_PHY_SECTION                           0x1000
+#define mmMMU_UP_BASE                              0x7FFCC11000ull
+#define MMU_UP_MAX_OFFSET                          0x7000
+#define MMU_UP_SECTION                             0x1000
+#define mmSTLB_BASE                                0x7FFCC12000ull
+#define STLB_MAX_OFFSET                            0x8800
+#define STLB_SECTION                               0x1000
+#define mmPCIE_MSI_BASE                            0x7FFCC13000ull
+#define PCIE_MSI_MAX_OFFSET                        0x8000
+#define PCIE_MSI_SECTION                           0x2D000
+#define mmPSOC_I2C_M0_BASE                         0x7FFCC40000ull
+#define PSOC_I2C_M0_MAX_OFFSET                     0x1000
+#define PSOC_I2C_M0_SECTION                        0x1000
+#define mmPSOC_I2C_M1_BASE                         0x7FFCC41000ull
+#define PSOC_I2C_M1_MAX_OFFSET                     0x1000
+#define PSOC_I2C_M1_SECTION                        0x1000
+#define mmPSOC_I2C_S_BASE                          0x7FFCC42000ull
+#define PSOC_I2C_S_MAX_OFFSET                      0x1000
+#define PSOC_I2C_S_SECTION                         0x1000
+#define mmPSOC_SPI_BASE                            0x7FFCC43000ull
+#define PSOC_SPI_MAX_OFFSET                        0x1000
+#define PSOC_SPI_SECTION                           0x2000
+#define mmPSOC_UART_0_BASE                         0x7FFCC45000ull
+#define PSOC_UART_0_MAX_OFFSET                     0x1000
+#define PSOC_UART_0_SECTION                        0x1000
+#define mmPSOC_UART_1_BASE                         0x7FFCC46000ull
+#define PSOC_UART_1_MAX_OFFSET                     0x1000
+#define PSOC_UART_1_SECTION                        0x1000
+#define mmPSOC_TIMER_BASE                          0x7FFCC47000ull
+#define PSOC_TIMER_MAX_OFFSET                      0x1000
+#define PSOC_TIMER_SECTION                         0x1000
+#define mmPSOC_WDOG_BASE                           0x7FFCC48000ull
+#define PSOC_WDOG_MAX_OFFSET                       0x1000
+#define PSOC_WDOG_SECTION                          0x1000
+#define mmPSOC_TIMESTAMP_BASE                      0x7FFCC49000ull
+#define PSOC_TIMESTAMP_MAX_OFFSET                  0x1000
+#define PSOC_TIMESTAMP_SECTION                     0x1000
+#define mmPSOC_EFUSE_BASE                          0x7FFCC4A000ull
+#define PSOC_EFUSE_MAX_OFFSET                      0x3040
+#define PSOC_EFUSE_SECTION                         0x1000
+#define mmPSOC_GLOBAL_CONF_BASE                    0x7FFCC4B000ull
+#define PSOC_GLOBAL_CONF_MAX_OFFSET                0xCD80
+#define PSOC_GLOBAL_CONF_SECTION                   0x1000
+#define mmPSOC_GPIO0_BASE                          0x7FFCC4C000ull
+#define PSOC_GPIO0_MAX_OFFSET                      0x1000
+#define PSOC_GPIO0_SECTION                         0x1000
+#define mmPSOC_GPIO1_BASE                          0x7FFCC4D000ull
+#define PSOC_GPIO1_MAX_OFFSET                      0x1000
+#define PSOC_GPIO1_SECTION                         0x1000
+#define mmPSOC_BTL_BASE                            0x7FFCC4E000ull
+#define PSOC_BTL_MAX_OFFSET                        0x1480
+#define PSOC_BTL_SECTION                           0x1000
+#define mmPSOC_CS_TRACE_BASE                       0x7FFCC4F000ull
+#define PSOC_CS_TRACE_MAX_OFFSET                   0x1680
+#define PSOC_CS_TRACE_SECTION                      0x1000
+#define mmPSOC_GPIO2_BASE                          0x7FFCC50000ull
+#define PSOC_GPIO2_MAX_OFFSET                      0x1000
+#define PSOC_GPIO2_SECTION                         0x1000
+#define mmPSOC_GPIO3_BASE                          0x7FFCC51000ull
+#define PSOC_GPIO3_MAX_OFFSET                      0x1000
+#define PSOC_GPIO3_SECTION                         0x1000
+#define mmPSOC_GPIO4_BASE                          0x7FFCC52000ull
+#define PSOC_GPIO4_MAX_OFFSET                      0x1000
+#define PSOC_GPIO4_SECTION                         0x1000
+#define mmPSOC_DFT_EFUSE_BASE                      0x7FFCC53000ull
+#define PSOC_DFT_EFUSE_MAX_OFFSET                  0x3040
+#define PSOC_DFT_EFUSE_SECTION                     0x1000
+#define mmPSOC_RPM_0_BASE                          0x7FFCC54000ull
+#define PSOC_RPM_0_MAX_OFFSET                      0x8800
+#define PSOC_RPM_0_SECTION                         0x1000
+#define mmPSOC_RPM_1_BASE                          0x7FFCC55000ull
+#define PSOC_RPM_1_MAX_OFFSET                      0x8800
+#define PSOC_RPM_1_SECTION                         0x1000
+#define mmPSOC_RPM_2_BASE                          0x7FFCC56000ull
+#define PSOC_RPM_2_MAX_OFFSET                      0x8800
+#define PSOC_RPM_2_SECTION                         0x1000
+#define mmPSOC_RPM_3_BASE                          0x7FFCC57000ull
+#define PSOC_RPM_3_MAX_OFFSET                      0x8800
+#define PSOC_RPM_3_SECTION                         0x19000
+#define mmPSOC_CPU_PLL_BASE                        0x7FFCC70000ull
+#define PSOC_CPU_PLL_MAX_OFFSET                    0x5200
+#define PSOC_CPU_PLL_SECTION                       0x1000
+#define mmPSOC_MME_PLL_BASE                        0x7FFCC71000ull
+#define PSOC_MME_PLL_MAX_OFFSET                    0x5200
+#define PSOC_MME_PLL_SECTION                       0x1000
+#define mmPSOC_PCI_PLL_BASE                        0x7FFCC72000ull
+#define PSOC_PCI_PLL_MAX_OFFSET                    0x5200
+#define PSOC_PCI_PLL_SECTION                       0x1000
+#define mmPSOC_TPC_PLL_BASE                        0x7FFCC73000ull
+#define PSOC_TPC_PLL_MAX_OFFSET                    0x5200
+#define PSOC_TPC_PLL_SECTION                       0x1000
+#define mmPSOC_HBM_PLL_BASE                        0x7FFCC74000ull
+#define PSOC_HBM_PLL_MAX_OFFSET                    0x5200
+#define PSOC_HBM_PLL_SECTION                       0x1000
+#define mmPSOC_PM_BASE                             0x7FFCC75000ull
+#define PSOC_PM_MAX_OFFSET                         0x1F00
+#define PSOC_PM_SECTION                            0x1000
+#define mmPSOC_TS_BASE                             0x7FFCC76000ull
+#define PSOC_TS_MAX_OFFSET                         0xE640
+#define PSOC_TS_SECTION                            0x2000
+#define mmPSOC_PWM0_BASE                           0x7FFCC78000ull
+#define PSOC_PWM0_MAX_OFFSET                       0x5800
+#define PSOC_PWM0_SECTION                          0x1000
+#define mmPSOC_PWM1_BASE                           0x7FFCC79000ull
+#define PSOC_PWM1_MAX_OFFSET                       0x5800
+#define PSOC_PWM1_SECTION                          0x1000
+#define mmPSOC_PWM2_BASE                           0x7FFCC7A000ull
+#define PSOC_PWM2_MAX_OFFSET                       0x5800
+#define PSOC_PWM2_SECTION                          0x1000
+#define mmPSOC_PWM3_BASE                           0x7FFCC7B000ull
+#define PSOC_PWM3_MAX_OFFSET                       0x5800
+#define PSOC_PWM3_SECTION                          0x1000
+#define mmPSOC_GPIO5_BASE                          0x7FFCC7C000ull
+#define PSOC_GPIO5_MAX_OFFSET                      0x1000
+#define PSOC_GPIO5_SECTION                         0x1000
+#define mmPSOC_GPIO6_BASE                          0x7FFCC7D000ull
+#define PSOC_GPIO6_MAX_OFFSET                      0x1000
+#define PSOC_GPIO6_SECTION                         0x3000
+#define mmPCIE_PMA_0_BASE                          0x7FFCC80000ull
+#define PCIE_PMA_0_MAX_OFFSET                      0x10003
+#define PCIE_PMA_0_SECTION                         0x10000
+#define mmPCIE_PMA_1_BASE                          0x7FFCC90000ull
+#define PCIE_PMA_1_MAX_OFFSET                      0x10003
+#define PCIE_PMA_1_SECTION                         0x10000
+#define mmPCIE_PMA_2_BASE                          0x7FFCCA0000ull
+#define PCIE_PMA_2_MAX_OFFSET                      0x10003
+#define PCIE_PMA_2_SECTION                         0x10000
+#define mmPCIE_PMA_3_BASE                          0x7FFCCB0000ull
+#define PCIE_PMA_3_MAX_OFFSET                      0x10003
+#define PCIE_PMA_3_SECTION                         0x10000
+#define mmNIC0_MAC_CH0_BASE                        0x7FFCCC0000ull
+#define NIC0_MAC_CH0_MAX_OFFSET                    0x8400
+#define NIC0_MAC_CH0_SECTION                       0x1000
+#define mmNIC0_MAC_CH1_BASE                        0x7FFCCC1000ull
+#define NIC0_MAC_CH1_MAX_OFFSET                    0x8400
+#define NIC0_MAC_CH1_SECTION                       0x1000
+#define mmNIC0_MAC_CH2_BASE                        0x7FFCCC2000ull
+#define NIC0_MAC_CH2_MAX_OFFSET                    0x8400
+#define NIC0_MAC_CH2_SECTION                       0x1000
+#define mmNIC0_MAC_CH3_BASE                        0x7FFCCC3000ull
+#define NIC0_MAC_CH3_MAX_OFFSET                    0x8400
+#define NIC0_MAC_CH3_SECTION                       0x1000
+#define mmNIC0_STAT_BASE                           0x7FFCCC4000ull
+#define NIC0_STAT_MAX_OFFSET                       0x4D00
+#define NIC0_STAT_SECTION                          0x1000
+#define mmNIC0_MAC_XPCS91_BASE                     0x7FFCCC5000ull
+#define NIC0_MAC_XPCS91_MAX_OFFSET                 0x2380
+#define NIC0_MAC_XPCS91_SECTION                    0x3000
+#define mmNIC0_MAC_CORE_BASE                       0x7FFCCC8000ull
+#define NIC0_MAC_CORE_MAX_OFFSET                   0x5400
+#define NIC0_MAC_CORE_SECTION                      0x1000
+#define mmNIC0_MAC_AUX_BASE                        0x7FFCCC9000ull
+#define NIC0_MAC_AUX_MAX_OFFSET                    0x3000
+#define NIC0_MAC_AUX_SECTION                       0xF000
+#define mmNIC0_PHY_BASE                            0x7FFCCD8000ull
+#define NIC0_PHY_MAX_OFFSET                        0x3400
+#define NIC0_PHY_SECTION                           0x8000
+#define mmNIC0_QM0_BASE                            0x7FFCCE0000ull
+#define NIC0_QM0_MAX_OFFSET                        0xD040
+#define NIC0_QM0_SECTION                           0x2000
+#define mmNIC0_QM1_BASE                            0x7FFCCE2000ull
+#define NIC0_QM1_MAX_OFFSET                        0xD040
+#define NIC0_QM1_SECTION                           0x2000
+#define mmNIC0_QPC0_BASE                           0x7FFCCE4000ull
+#define NIC0_QPC0_MAX_OFFSET                       0x7140
+#define NIC0_QPC0_SECTION                          0x1000
+#define mmNIC0_QPC1_BASE                           0x7FFCCE5000ull
+#define NIC0_QPC1_MAX_OFFSET                       0x7140
+#define NIC0_QPC1_SECTION                          0x3000
+#define mmNIC0_RXB_BASE                            0x7FFCCE8000ull
+#define NIC0_RXB_MAX_OFFSET                        0x6040
+#define NIC0_RXB_SECTION                           0x1000
+#define mmNIC0_RXE0_BASE                           0x7FFCCE9000ull
+#define NIC0_RXE0_MAX_OFFSET                       0x2FC0
+#define NIC0_RXE0_SECTION                          0x1000
+#define mmNIC0_RXE1_BASE                           0x7FFCCEA000ull
+#define NIC0_RXE1_MAX_OFFSET                       0x2FC0
+#define NIC0_RXE1_SECTION                          0x1000
+#define mmNIC0_RX_GW_BASE                          0x7FFCCEB000ull
+#define NIC0_RX_GW_MAX_OFFSET                      0x4540
+#define NIC0_RX_GW_SECTION                         0x5000
+#define mmNIC0_TXS0_BASE                           0x7FFCCF0000ull
+#define NIC0_TXS0_MAX_OFFSET                       0x19C0
+#define NIC0_TXS0_SECTION                          0x1000
+#define mmNIC0_TXS1_BASE                           0x7FFCCF1000ull
+#define NIC0_TXS1_MAX_OFFSET                       0x19C0
+#define NIC0_TXS1_SECTION                          0x1000
+#define mmNIC0_TXE0_BASE                           0x7FFCCF2000ull
+#define NIC0_TXE0_MAX_OFFSET                       0x2040
+#define NIC0_TXE0_SECTION                          0x1000
+#define mmNIC0_TXE1_BASE                           0x7FFCCF3000ull
+#define NIC0_TXE1_MAX_OFFSET                       0x2040
+#define NIC0_TXE1_SECTION                          0x1000
+#define mmNIC0_TXB_BASE                            0x7FFCCF4000ull
+#define NIC0_TXB_MAX_OFFSET                        0xD400
+#define NIC0_TXB_SECTION                           0x1000
+#define mmNIC0_TMR_BASE                            0x7FFCCF5000ull
+#define NIC0_TMR_MAX_OFFSET                        0x1600
+#define NIC0_TMR_SECTION                           0x1000
+#define mmNIC0_TX_GW_BASE                          0x7FFCCF6000ull
+#define NIC0_TX_GW_MAX_OFFSET                      0x1400
+#define NIC0_TX_GW_SECTION                         0x2000
+#define mmNIC0_TS_BASE                             0x7FFCCF8000ull
+#define NIC0_TS_MAX_OFFSET                         0xE640
+#define NIC0_TS_SECTION                            0x1000
+#define mmNIC0_PLL_BASE                            0x7FFCCF9000ull
+#define NIC0_PLL_MAX_OFFSET                        0x5200
+#define NIC0_PLL_SECTION                           0x1000
+#define mmNIC0_PM_BASE                             0x7FFCCFA000ull
+#define NIC0_PM_MAX_OFFSET                         0x1F00
+#define NIC0_PM_SECTION                            0x6000
+#define mmNIC1_MAC_CH0_BASE                        0x7FFCD00000ull
+#define NIC1_MAC_CH0_MAX_OFFSET                    0x8400
+#define NIC1_MAC_CH0_SECTION                       0x1000
+#define mmNIC1_MAC_CH1_BASE                        0x7FFCD01000ull
+#define NIC1_MAC_CH1_MAX_OFFSET                    0x8400
+#define NIC1_MAC_CH1_SECTION                       0x1000
+#define mmNIC1_MAC_CH2_BASE                        0x7FFCD02000ull
+#define NIC1_MAC_CH2_MAX_OFFSET                    0x8400
+#define NIC1_MAC_CH2_SECTION                       0x1000
+#define mmNIC1_MAC_CH3_BASE                        0x7FFCD03000ull
+#define NIC1_MAC_CH3_MAX_OFFSET                    0x8400
+#define NIC1_MAC_CH3_SECTION                       0x1000
+#define mmNIC1_STAT_BASE                           0x7FFCD04000ull
+#define NIC1_STAT_MAX_OFFSET                       0x4D00
+#define NIC1_STAT_SECTION                          0x1000
+#define mmNIC1_MAC_XPCS91_BASE                     0x7FFCD05000ull
+#define NIC1_MAC_XPCS91_MAX_OFFSET                 0x2380
+#define NIC1_MAC_XPCS91_SECTION                    0x3000
+#define mmNIC1_MAC_CORE_BASE                       0x7FFCD08000ull
+#define NIC1_MAC_CORE_MAX_OFFSET                   0x5400
+#define NIC1_MAC_CORE_SECTION                      0x1000
+#define mmNIC1_MAC_AUX_BASE                        0x7FFCD09000ull
+#define NIC1_MAC_AUX_MAX_OFFSET                    0x3000
+#define NIC1_MAC_AUX_SECTION                       0xF000
+#define mmNIC1_PHY_BASE                            0x7FFCD18000ull
+#define NIC1_PHY_MAX_OFFSET                        0x3400
+#define NIC1_PHY_SECTION                           0x8000
+#define mmNIC1_QM0_BASE                            0x7FFCD20000ull
+#define NIC1_QM0_MAX_OFFSET                        0xD040
+#define NIC1_QM0_SECTION                           0x2000
+#define mmNIC1_QM1_BASE                            0x7FFCD22000ull
+#define NIC1_QM1_MAX_OFFSET                        0xD040
+#define NIC1_QM1_SECTION                           0x2000
+#define mmNIC1_QPC0_BASE                           0x7FFCD24000ull
+#define NIC1_QPC0_MAX_OFFSET                       0x7140
+#define NIC1_QPC0_SECTION                          0x1000
+#define mmNIC1_QPC1_BASE                           0x7FFCD25000ull
+#define NIC1_QPC1_MAX_OFFSET                       0x7140
+#define NIC1_QPC1_SECTION                          0x3000
+#define mmNIC1_RXB_BASE                            0x7FFCD28000ull
+#define NIC1_RXB_MAX_OFFSET                        0x6040
+#define NIC1_RXB_SECTION                           0x1000
+#define mmNIC1_RXE0_BASE                           0x7FFCD29000ull
+#define NIC1_RXE0_MAX_OFFSET                       0x2FC0
+#define NIC1_RXE0_SECTION                          0x1000
+#define mmNIC1_RXE1_BASE                           0x7FFCD2A000ull
+#define NIC1_RXE1_MAX_OFFSET                       0x2FC0
+#define NIC1_RXE1_SECTION                          0x1000
+#define mmNIC1_RX_GW_BASE                          0x7FFCD2B000ull
+#define NIC1_RX_GW_MAX_OFFSET                      0x4540
+#define NIC1_RX_GW_SECTION                         0x5000
+#define mmNIC1_TXS0_BASE                           0x7FFCD30000ull
+#define NIC1_TXS0_MAX_OFFSET                       0x19C0
+#define NIC1_TXS0_SECTION                          0x1000
+#define mmNIC1_TXS1_BASE                           0x7FFCD31000ull
+#define NIC1_TXS1_MAX_OFFSET                       0x19C0
+#define NIC1_TXS1_SECTION                          0x1000
+#define mmNIC1_TXE0_BASE                           0x7FFCD32000ull
+#define NIC1_TXE0_MAX_OFFSET                       0x2040
+#define NIC1_TXE0_SECTION                          0x1000
+#define mmNIC1_TXE1_BASE                           0x7FFCD33000ull
+#define NIC1_TXE1_MAX_OFFSET                       0x2040
+#define NIC1_TXE1_SECTION                          0x1000
+#define mmNIC1_TXB_BASE                            0x7FFCD34000ull
+#define NIC1_TXB_MAX_OFFSET                        0xD400
+#define NIC1_TXB_SECTION                           0x1000
+#define mmNIC1_TMR_BASE                            0x7FFCD35000ull
+#define NIC1_TMR_MAX_OFFSET                        0x1600
+#define NIC1_TMR_SECTION                           0x1000
+#define mmNIC1_TX_GW_BASE                          0x7FFCD36000ull
+#define NIC1_TX_GW_MAX_OFFSET                      0x1400
+#define NIC1_TX_GW_SECTION                         0x2000
+#define mmNIC1_TS_BASE                             0x7FFCD38000ull
+#define NIC1_TS_MAX_OFFSET                         0xE640
+#define NIC1_TS_SECTION                            0x1000
+#define mmNIC1_PLL_BASE                            0x7FFCD39000ull
+#define NIC1_PLL_MAX_OFFSET                        0x5200
+#define NIC1_PLL_SECTION                           0x1000
+#define mmNIC1_PM_BASE                             0x7FFCD3A000ull
+#define NIC1_PM_MAX_OFFSET                         0x1F00
+#define NIC1_PM_SECTION                            0x6000
+#define mmNIC2_MAC_CH0_BASE                        0x7FFCD40000ull
+#define NIC2_MAC_CH0_MAX_OFFSET                    0x8400
+#define NIC2_MAC_CH0_SECTION                       0x1000
+#define mmNIC2_MAC_CH1_BASE                        0x7FFCD41000ull
+#define NIC2_MAC_CH1_MAX_OFFSET                    0x8400
+#define NIC2_MAC_CH1_SECTION                       0x1000
+#define mmNIC2_MAC_CH2_BASE                        0x7FFCD42000ull
+#define NIC2_MAC_CH2_MAX_OFFSET                    0x8400
+#define NIC2_MAC_CH2_SECTION                       0x1000
+#define mmNIC2_MAC_CH3_BASE                        0x7FFCD43000ull
+#define NIC2_MAC_CH3_MAX_OFFSET                    0x8400
+#define NIC2_MAC_CH3_SECTION                       0x1000
+#define mmNIC2_STAT_BASE                           0x7FFCD44000ull
+#define NIC2_STAT_MAX_OFFSET                       0x4D00
+#define NIC2_STAT_SECTION                          0x1000
+#define mmNIC2_MAC_XPCS91_BASE                     0x7FFCD45000ull
+#define NIC2_MAC_XPCS91_MAX_OFFSET                 0x2380
+#define NIC2_MAC_XPCS91_SECTION                    0x3000
+#define mmNIC2_MAC_CORE_BASE                       0x7FFCD48000ull
+#define NIC2_MAC_CORE_MAX_OFFSET                   0x5400
+#define NIC2_MAC_CORE_SECTION                      0x1000
+#define mmNIC2_MAC_AUX_BASE                        0x7FFCD49000ull
+#define NIC2_MAC_AUX_MAX_OFFSET                    0x3000
+#define NIC2_MAC_AUX_SECTION                       0xF000
+#define mmNIC2_PHY_BASE                            0x7FFCD58000ull
+#define NIC2_PHY_MAX_OFFSET                        0x3400
+#define NIC2_PHY_SECTION                           0x8000
+#define mmNIC2_QM0_BASE                            0x7FFCD60000ull
+#define NIC2_QM0_MAX_OFFSET                        0xD040
+#define NIC2_QM0_SECTION                           0x2000
+#define mmNIC2_QM1_BASE                            0x7FFCD62000ull
+#define NIC2_QM1_MAX_OFFSET                        0xD040
+#define NIC2_QM1_SECTION                           0x2000
+#define mmNIC2_QPC0_BASE                           0x7FFCD64000ull
+#define NIC2_QPC0_MAX_OFFSET                       0x7140
+#define NIC2_QPC0_SECTION                          0x1000
+#define mmNIC2_QPC1_BASE                           0x7FFCD65000ull
+#define NIC2_QPC1_MAX_OFFSET                       0x7140
+#define NIC2_QPC1_SECTION                          0x3000
+#define mmNIC2_RXB_BASE                            0x7FFCD68000ull
+#define NIC2_RXB_MAX_OFFSET                        0x6040
+#define NIC2_RXB_SECTION                           0x1000
+#define mmNIC2_RXE0_BASE                           0x7FFCD69000ull
+#define NIC2_RXE0_MAX_OFFSET                       0x2FC0
+#define NIC2_RXE0_SECTION                          0x1000
+#define mmNIC2_RXE1_BASE                           0x7FFCD6A000ull
+#define NIC2_RXE1_MAX_OFFSET                       0x2FC0
+#define NIC2_RXE1_SECTION                          0x1000
+#define mmNIC2_RX_GW_BASE                          0x7FFCD6B000ull
+#define NIC2_RX_GW_MAX_OFFSET                      0x4540
+#define NIC2_RX_GW_SECTION                         0x5000
+#define mmNIC2_TXS0_BASE                           0x7FFCD70000ull
+#define NIC2_TXS0_MAX_OFFSET                       0x19C0
+#define NIC2_TXS0_SECTION                          0x1000
+#define mmNIC2_TXS1_BASE                           0x7FFCD71000ull
+#define NIC2_TXS1_MAX_OFFSET                       0x19C0
+#define NIC2_TXS1_SECTION                          0x1000
+#define mmNIC2_TXE0_BASE                           0x7FFCD72000ull
+#define NIC2_TXE0_MAX_OFFSET                       0x2040
+#define NIC2_TXE0_SECTION                          0x1000
+#define mmNIC2_TXE1_BASE                           0x7FFCD73000ull
+#define NIC2_TXE1_MAX_OFFSET                       0x2040
+#define NIC2_TXE1_SECTION                          0x1000
+#define mmNIC2_TXB_BASE                            0x7FFCD74000ull
+#define NIC2_TXB_MAX_OFFSET                        0xD400
+#define NIC2_TXB_SECTION                           0x1000
+#define mmNIC2_TMR_BASE                            0x7FFCD75000ull
+#define NIC2_TMR_MAX_OFFSET                        0x1600
+#define NIC2_TMR_SECTION                           0x1000
+#define mmNIC2_TX_GW_BASE                          0x7FFCD76000ull
+#define NIC2_TX_GW_MAX_OFFSET                      0x1400
+#define NIC2_TX_GW_SECTION                         0x2000
+#define mmNIC2_HBM_PLL_BASE                        0x7FFCD78000ull
+#define NIC2_HBM_PLL_MAX_OFFSET                    0x5200
+#define NIC2_HBM_PLL_SECTION                       0x1000
+#define mmNIC2_MME_PLL_BASE                        0x7FFCD79000ull
+#define NIC2_MME_PLL_MAX_OFFSET                    0x5200
+#define NIC2_MME_PLL_SECTION                       0x1000
+#define mmNIC2_TPC_PLL_BASE                        0x7FFCD7A000ull
+#define NIC2_TPC_PLL_MAX_OFFSET                    0x5200
+#define NIC2_TPC_PLL_SECTION                       0x6000
+#define mmNIC3_MAC_CH0_BASE                        0x7FFCD80000ull
+#define NIC3_MAC_CH0_MAX_OFFSET                    0x8400
+#define NIC3_MAC_CH0_SECTION                       0x1000
+#define mmNIC3_MAC_CH1_BASE                        0x7FFCD81000ull
+#define NIC3_MAC_CH1_MAX_OFFSET                    0x8400
+#define NIC3_MAC_CH1_SECTION                       0x1000
+#define mmNIC3_MAC_CH2_BASE                        0x7FFCD82000ull
+#define NIC3_MAC_CH2_MAX_OFFSET                    0x8400
+#define NIC3_MAC_CH2_SECTION                       0x1000
+#define mmNIC3_MAC_CH3_BASE                        0x7FFCD83000ull
+#define NIC3_MAC_CH3_MAX_OFFSET                    0x8400
+#define NIC3_MAC_CH3_SECTION                       0x1000
+#define mmNIC3_STAT_BASE                           0x7FFCD84000ull
+#define NIC3_STAT_MAX_OFFSET                       0x4D00
+#define NIC3_STAT_SECTION                          0x1000
+#define mmNIC3_MAC_XPCS91_BASE                     0x7FFCD85000ull
+#define NIC3_MAC_XPCS91_MAX_OFFSET                 0x2380
+#define NIC3_MAC_XPCS91_SECTION                    0x3000
+#define mmNIC3_MAC_CORE_BASE                       0x7FFCD88000ull
+#define NIC3_MAC_CORE_MAX_OFFSET                   0x5400
+#define NIC3_MAC_CORE_SECTION                      0x1000
+#define mmNIC3_MAC_AUX_BASE                        0x7FFCD89000ull
+#define NIC3_MAC_AUX_MAX_OFFSET                    0x3000
+#define NIC3_MAC_AUX_SECTION                       0xF000
+#define mmNIC3_PHY_BASE                            0x7FFCD98000ull
+#define NIC3_PHY_MAX_OFFSET                        0x3400
+#define NIC3_PHY_SECTION                           0x8000
+#define mmNIC3_QM0_BASE                            0x7FFCDA0000ull
+#define NIC3_QM0_MAX_OFFSET                        0xD040
+#define NIC3_QM0_SECTION                           0x2000
+#define mmNIC3_QM1_BASE                            0x7FFCDA2000ull
+#define NIC3_QM1_MAX_OFFSET                        0xD040
+#define NIC3_QM1_SECTION                           0x2000
+#define mmNIC3_QPC0_BASE                           0x7FFCDA4000ull
+#define NIC3_QPC0_MAX_OFFSET                       0x7140
+#define NIC3_QPC0_SECTION                          0x1000
+#define mmNIC3_QPC1_BASE                           0x7FFCDA5000ull
+#define NIC3_QPC1_MAX_OFFSET                       0x7140
+#define NIC3_QPC1_SECTION                          0x3000
+#define mmNIC3_RXB_BASE                            0x7FFCDA8000ull
+#define NIC3_RXB_MAX_OFFSET                        0x6040
+#define NIC3_RXB_SECTION                           0x1000
+#define mmNIC3_RXE0_BASE                           0x7FFCDA9000ull
+#define NIC3_RXE0_MAX_OFFSET                       0x2FC0
+#define NIC3_RXE0_SECTION                          0x1000
+#define mmNIC3_RXE1_BASE                           0x7FFCDAA000ull
+#define NIC3_RXE1_MAX_OFFSET                       0x2FC0
+#define NIC3_RXE1_SECTION                          0x1000
+#define mmNIC3_RX_GW_BASE                          0x7FFCDAB000ull
+#define NIC3_RX_GW_MAX_OFFSET                      0x4540
+#define NIC3_RX_GW_SECTION                         0x5000
+#define mmNIC3_TXS0_BASE                           0x7FFCDB0000ull
+#define NIC3_TXS0_MAX_OFFSET                       0x19C0
+#define NIC3_TXS0_SECTION                          0x1000
+#define mmNIC3_TXS1_BASE                           0x7FFCDB1000ull
+#define NIC3_TXS1_MAX_OFFSET                       0x19C0
+#define NIC3_TXS1_SECTION                          0x1000
+#define mmNIC3_TXE0_BASE                           0x7FFCDB2000ull
+#define NIC3_TXE0_MAX_OFFSET                       0x2040
+#define NIC3_TXE0_SECTION                          0x1000
+#define mmNIC3_TXE1_BASE                           0x7FFCDB3000ull
+#define NIC3_TXE1_MAX_OFFSET                       0x2040
+#define NIC3_TXE1_SECTION                          0x1000
+#define mmNIC3_TXB_BASE                            0x7FFCDB4000ull
+#define NIC3_TXB_MAX_OFFSET                        0xD400
+#define NIC3_TXB_SECTION                           0x1000
+#define mmNIC3_TMR_BASE                            0x7FFCDB5000ull
+#define NIC3_TMR_MAX_OFFSET                        0x1600
+#define NIC3_TMR_SECTION                           0x1000
+#define mmNIC3_TX_GW_BASE                          0x7FFCDB6000ull
+#define NIC3_TX_GW_MAX_OFFSET                      0x1400
+#define NIC3_TX_GW_SECTION                         0x2000
+#define mmNIC3_TS_BASE                             0x7FFCDB8000ull
+#define NIC3_TS_MAX_OFFSET                         0xE640
+#define NIC3_TS_SECTION                            0x2000
+#define mmNIC3_PM_BASE                             0x7FFCDBA000ull
+#define NIC3_PM_MAX_OFFSET                         0x1F00
+#define NIC3_PM_SECTION                            0x6000
+#define mmNIC4_MAC_CH0_BASE                        0x7FFCDC0000ull
+#define NIC4_MAC_CH0_MAX_OFFSET                    0x8400
+#define NIC4_MAC_CH0_SECTION                       0x1000
+#define mmNIC4_MAC_CH1_BASE                        0x7FFCDC1000ull
+#define NIC4_MAC_CH1_MAX_OFFSET                    0x8400
+#define NIC4_MAC_CH1_SECTION                       0x1000
+#define mmNIC4_MAC_CH2_BASE                        0x7FFCDC2000ull
+#define NIC4_MAC_CH2_MAX_OFFSET                    0x8400
+#define NIC4_MAC_CH2_SECTION                       0x1000
+#define mmNIC4_MAC_CH3_BASE                        0x7FFCDC3000ull
+#define NIC4_MAC_CH3_MAX_OFFSET                    0x8400
+#define NIC4_MAC_CH3_SECTION                       0x1000
+#define mmNIC4_STAT_BASE                           0x7FFCDC4000ull
+#define NIC4_STAT_MAX_OFFSET                       0x4D00
+#define NIC4_STAT_SECTION                          0x1000
+#define mmNIC4_MAC_XPCS91_BASE                     0x7FFCDC5000ull
+#define NIC4_MAC_XPCS91_MAX_OFFSET                 0x2380
+#define NIC4_MAC_XPCS91_SECTION                    0x3000
+#define mmNIC4_MAC_CORE_BASE                       0x7FFCDC8000ull
+#define NIC4_MAC_CORE_MAX_OFFSET                   0x5400
+#define NIC4_MAC_CORE_SECTION                      0x1000
+#define mmNIC4_MAC_AUX_BASE                        0x7FFCDC9000ull
+#define NIC4_MAC_AUX_MAX_OFFSET                    0x3000
+#define NIC4_MAC_AUX_SECTION                       0xF000
+#define mmNIC4_PHY_BASE                            0x7FFCDD8000ull
+#define NIC4_PHY_MAX_OFFSET                        0x3400
+#define NIC4_PHY_SECTION                           0x8000
+#define mmNIC4_QM0_BASE                            0x7FFCDE0000ull
+#define NIC4_QM0_MAX_OFFSET                        0xD040
+#define NIC4_QM0_SECTION                           0x2000
+#define mmNIC4_QM1_BASE                            0x7FFCDE2000ull
+#define NIC4_QM1_MAX_OFFSET                        0xD040
+#define NIC4_QM1_SECTION                           0x2000
+#define mmNIC4_QPC0_BASE                           0x7FFCDE4000ull
+#define NIC4_QPC0_MAX_OFFSET                       0x7140
+#define NIC4_QPC0_SECTION                          0x1000
+#define mmNIC4_QPC1_BASE                           0x7FFCDE5000ull
+#define NIC4_QPC1_MAX_OFFSET                       0x7140
+#define NIC4_QPC1_SECTION                          0x3000
+#define mmNIC4_RXB_BASE                            0x7FFCDE8000ull
+#define NIC4_RXB_MAX_OFFSET                        0x6040
+#define NIC4_RXB_SECTION                           0x1000
+#define mmNIC4_RXE0_BASE                           0x7FFCDE9000ull
+#define NIC4_RXE0_MAX_OFFSET                       0x2FC0
+#define NIC4_RXE0_SECTION                          0x1000
+#define mmNIC4_RXE1_BASE                           0x7FFCDEA000ull
+#define NIC4_RXE1_MAX_OFFSET                       0x2FC0
+#define NIC4_RXE1_SECTION                          0x1000
+#define mmNIC4_RX_GW_BASE                          0x7FFCDEB000ull
+#define NIC4_RX_GW_MAX_OFFSET                      0x4540
+#define NIC4_RX_GW_SECTION                         0x5000
+#define mmNIC4_TXS0_BASE                           0x7FFCDF0000ull
+#define NIC4_TXS0_MAX_OFFSET                       0x19C0
+#define NIC4_TXS0_SECTION                          0x1000
+#define mmNIC4_TXS1_BASE                           0x7FFCDF1000ull
+#define NIC4_TXS1_MAX_OFFSET                       0x19C0
+#define NIC4_TXS1_SECTION                          0x1000
+#define mmNIC4_TXE0_BASE                           0x7FFCDF2000ull
+#define NIC4_TXE0_MAX_OFFSET                       0x2040
+#define NIC4_TXE0_SECTION                          0x1000
+#define mmNIC4_TXE1_BASE                           0x7FFCDF3000ull
+#define NIC4_TXE1_MAX_OFFSET                       0x2040
+#define NIC4_TXE1_SECTION                          0x1000
+#define mmNIC4_TXB_BASE                            0x7FFCDF4000ull
+#define NIC4_TXB_MAX_OFFSET                        0xD400
+#define NIC4_TXB_SECTION                           0x1000
+#define mmNIC4_TMR_BASE                            0x7FFCDF5000ull
+#define NIC4_TMR_MAX_OFFSET                        0x1600
+#define NIC4_TMR_SECTION                           0x1000
+#define mmNIC4_TX_GW_BASE                          0x7FFCDF6000ull
+#define NIC4_TX_GW_MAX_OFFSET                      0x1400
+#define NIC4_TX_GW_SECTION                         0x10000
+#define mmTPC0_CFG_BASE                            0x7FFCE06000ull
+#define TPC0_CFG_MAX_OFFSET                        0xE400
+#define TPC0_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC0_CFG_BASE            0x7FFCE06400ull
+#define KERNEL_TENSOR_0_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC0_CFG_BASE            0x7FFCE06438ull
+#define KERNEL_TENSOR_1_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC0_CFG_BASE            0x7FFCE06470ull
+#define KERNEL_TENSOR_2_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC0_CFG_BASE            0x7FFCE064A8ull
+#define KERNEL_TENSOR_3_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC0_CFG_BASE            0x7FFCE064E0ull
+#define KERNEL_TENSOR_4_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC0_CFG_BASE            0x7FFCE06518ull
+#define KERNEL_TENSOR_5_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC0_CFG_BASE            0x7FFCE06550ull
+#define KERNEL_TENSOR_6_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC0_CFG_BASE            0x7FFCE06588ull
+#define KERNEL_TENSOR_7_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC0_CFG_BASE            0x7FFCE065C0ull
+#define KERNEL_TENSOR_8_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC0_CFG_BASE            0x7FFCE065F8ull
+#define KERNEL_TENSOR_9_TPC0_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC0_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC0_CFG_BASE           0x7FFCE06630ull
+#define KERNEL_TENSOR_10_TPC0_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC0_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC0_CFG_BASE           0x7FFCE06668ull
+#define KERNEL_TENSOR_11_TPC0_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC0_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC0_CFG_BASE           0x7FFCE066A0ull
+#define KERNEL_TENSOR_12_TPC0_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC0_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC0_CFG_BASE           0x7FFCE066D8ull
+#define KERNEL_TENSOR_13_TPC0_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC0_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC0_CFG_BASE           0x7FFCE06710ull
+#define KERNEL_TENSOR_14_TPC0_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC0_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC0_CFG_BASE           0x7FFCE06748ull
+#define KERNEL_TENSOR_15_TPC0_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC0_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC0_CFG_BASE         0x7FFCE06780ull
+#define KERNEL_SYNC_OBJECT_TPC0_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC0_CFG_SECTION        0x8000
+#define mmKERNEL_TPC0_CFG_BASE                     0x7FFCE06788ull
+#define KERNEL_TPC0_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC0_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC0_CFG_BASE                0x7FFCE06A00ull
+#define QM_TENSOR_0_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC0_CFG_BASE                0x7FFCE06A38ull
+#define QM_TENSOR_1_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC0_CFG_BASE                0x7FFCE06A70ull
+#define QM_TENSOR_2_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC0_CFG_BASE                0x7FFCE06AA8ull
+#define QM_TENSOR_3_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC0_CFG_BASE                0x7FFCE06AE0ull
+#define QM_TENSOR_4_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC0_CFG_BASE                0x7FFCE06B18ull
+#define QM_TENSOR_5_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC0_CFG_BASE                0x7FFCE06B50ull
+#define QM_TENSOR_6_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC0_CFG_BASE                0x7FFCE06B88ull
+#define QM_TENSOR_7_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC0_CFG_BASE                0x7FFCE06BC0ull
+#define QM_TENSOR_8_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC0_CFG_BASE                0x7FFCE06BF8ull
+#define QM_TENSOR_9_TPC0_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC0_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC0_CFG_BASE               0x7FFCE06C30ull
+#define QM_TENSOR_10_TPC0_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC0_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC0_CFG_BASE               0x7FFCE06C68ull
+#define QM_TENSOR_11_TPC0_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC0_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC0_CFG_BASE               0x7FFCE06CA0ull
+#define QM_TENSOR_12_TPC0_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC0_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC0_CFG_BASE               0x7FFCE06CD8ull
+#define QM_TENSOR_13_TPC0_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC0_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC0_CFG_BASE               0x7FFCE06D10ull
+#define QM_TENSOR_14_TPC0_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC0_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC0_CFG_BASE               0x7FFCE06D48ull
+#define QM_TENSOR_15_TPC0_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC0_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC0_CFG_BASE             0x7FFCE06D80ull
+#define QM_SYNC_OBJECT_TPC0_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC0_CFG_SECTION            0x8000
+#define mmQM_TPC0_CFG_BASE                         0x7FFCE06D88ull
+#define QM_TPC0_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC0_CFG_SECTION                        0x2780
+#define mmTPC0_E2E_CRED_BASE                       0x7FFCE07000ull
+#define TPC0_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC0_E2E_CRED_SECTION                      0x1000
+#define mmTPC0_QM_BASE                             0x7FFCE08000ull
+#define TPC0_QM_MAX_OFFSET                         0xD040
+#define TPC0_QM_SECTION                            0x3E000
+#define mmTPC1_CFG_BASE                            0x7FFCE46000ull
+#define TPC1_CFG_MAX_OFFSET                        0xE400
+#define TPC1_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC1_CFG_BASE            0x7FFCE46400ull
+#define KERNEL_TENSOR_0_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC1_CFG_BASE            0x7FFCE46438ull
+#define KERNEL_TENSOR_1_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC1_CFG_BASE            0x7FFCE46470ull
+#define KERNEL_TENSOR_2_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC1_CFG_BASE            0x7FFCE464A8ull
+#define KERNEL_TENSOR_3_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC1_CFG_BASE            0x7FFCE464E0ull
+#define KERNEL_TENSOR_4_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC1_CFG_BASE            0x7FFCE46518ull
+#define KERNEL_TENSOR_5_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC1_CFG_BASE            0x7FFCE46550ull
+#define KERNEL_TENSOR_6_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC1_CFG_BASE            0x7FFCE46588ull
+#define KERNEL_TENSOR_7_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC1_CFG_BASE            0x7FFCE465C0ull
+#define KERNEL_TENSOR_8_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC1_CFG_BASE            0x7FFCE465F8ull
+#define KERNEL_TENSOR_9_TPC1_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC1_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC1_CFG_BASE           0x7FFCE46630ull
+#define KERNEL_TENSOR_10_TPC1_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC1_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC1_CFG_BASE           0x7FFCE46668ull
+#define KERNEL_TENSOR_11_TPC1_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC1_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC1_CFG_BASE           0x7FFCE466A0ull
+#define KERNEL_TENSOR_12_TPC1_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC1_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC1_CFG_BASE           0x7FFCE466D8ull
+#define KERNEL_TENSOR_13_TPC1_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC1_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC1_CFG_BASE           0x7FFCE46710ull
+#define KERNEL_TENSOR_14_TPC1_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC1_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC1_CFG_BASE           0x7FFCE46748ull
+#define KERNEL_TENSOR_15_TPC1_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC1_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC1_CFG_BASE         0x7FFCE46780ull
+#define KERNEL_SYNC_OBJECT_TPC1_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC1_CFG_SECTION        0x8000
+#define mmKERNEL_TPC1_CFG_BASE                     0x7FFCE46788ull
+#define KERNEL_TPC1_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC1_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC1_CFG_BASE                0x7FFCE46A00ull
+#define QM_TENSOR_0_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC1_CFG_BASE                0x7FFCE46A38ull
+#define QM_TENSOR_1_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC1_CFG_BASE                0x7FFCE46A70ull
+#define QM_TENSOR_2_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC1_CFG_BASE                0x7FFCE46AA8ull
+#define QM_TENSOR_3_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC1_CFG_BASE                0x7FFCE46AE0ull
+#define QM_TENSOR_4_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC1_CFG_BASE                0x7FFCE46B18ull
+#define QM_TENSOR_5_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC1_CFG_BASE                0x7FFCE46B50ull
+#define QM_TENSOR_6_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC1_CFG_BASE                0x7FFCE46B88ull
+#define QM_TENSOR_7_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC1_CFG_BASE                0x7FFCE46BC0ull
+#define QM_TENSOR_8_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC1_CFG_BASE                0x7FFCE46BF8ull
+#define QM_TENSOR_9_TPC1_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC1_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC1_CFG_BASE               0x7FFCE46C30ull
+#define QM_TENSOR_10_TPC1_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC1_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC1_CFG_BASE               0x7FFCE46C68ull
+#define QM_TENSOR_11_TPC1_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC1_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC1_CFG_BASE               0x7FFCE46CA0ull
+#define QM_TENSOR_12_TPC1_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC1_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC1_CFG_BASE               0x7FFCE46CD8ull
+#define QM_TENSOR_13_TPC1_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC1_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC1_CFG_BASE               0x7FFCE46D10ull
+#define QM_TENSOR_14_TPC1_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC1_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC1_CFG_BASE               0x7FFCE46D48ull
+#define QM_TENSOR_15_TPC1_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC1_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC1_CFG_BASE             0x7FFCE46D80ull
+#define QM_SYNC_OBJECT_TPC1_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC1_CFG_SECTION            0x8000
+#define mmQM_TPC1_CFG_BASE                         0x7FFCE46D88ull
+#define QM_TPC1_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC1_CFG_SECTION                        0x2780
+#define mmTPC1_E2E_CRED_BASE                       0x7FFCE47000ull
+#define TPC1_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC1_E2E_CRED_SECTION                      0x1000
+#define mmTPC1_QM_BASE                             0x7FFCE48000ull
+#define TPC1_QM_MAX_OFFSET                         0xD040
+#define TPC1_QM_SECTION                            0x3E000
+#define mmTPC2_CFG_BASE                            0x7FFCE86000ull
+#define TPC2_CFG_MAX_OFFSET                        0xE400
+#define TPC2_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC2_CFG_BASE            0x7FFCE86400ull
+#define KERNEL_TENSOR_0_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC2_CFG_BASE            0x7FFCE86438ull
+#define KERNEL_TENSOR_1_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC2_CFG_BASE            0x7FFCE86470ull
+#define KERNEL_TENSOR_2_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC2_CFG_BASE            0x7FFCE864A8ull
+#define KERNEL_TENSOR_3_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC2_CFG_BASE            0x7FFCE864E0ull
+#define KERNEL_TENSOR_4_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC2_CFG_BASE            0x7FFCE86518ull
+#define KERNEL_TENSOR_5_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC2_CFG_BASE            0x7FFCE86550ull
+#define KERNEL_TENSOR_6_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC2_CFG_BASE            0x7FFCE86588ull
+#define KERNEL_TENSOR_7_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC2_CFG_BASE            0x7FFCE865C0ull
+#define KERNEL_TENSOR_8_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC2_CFG_BASE            0x7FFCE865F8ull
+#define KERNEL_TENSOR_9_TPC2_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC2_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC2_CFG_BASE           0x7FFCE86630ull
+#define KERNEL_TENSOR_10_TPC2_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC2_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC2_CFG_BASE           0x7FFCE86668ull
+#define KERNEL_TENSOR_11_TPC2_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC2_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC2_CFG_BASE           0x7FFCE866A0ull
+#define KERNEL_TENSOR_12_TPC2_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC2_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC2_CFG_BASE           0x7FFCE866D8ull
+#define KERNEL_TENSOR_13_TPC2_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC2_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC2_CFG_BASE           0x7FFCE86710ull
+#define KERNEL_TENSOR_14_TPC2_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC2_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC2_CFG_BASE           0x7FFCE86748ull
+#define KERNEL_TENSOR_15_TPC2_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC2_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC2_CFG_BASE         0x7FFCE86780ull
+#define KERNEL_SYNC_OBJECT_TPC2_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC2_CFG_SECTION        0x8000
+#define mmKERNEL_TPC2_CFG_BASE                     0x7FFCE86788ull
+#define KERNEL_TPC2_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC2_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC2_CFG_BASE                0x7FFCE86A00ull
+#define QM_TENSOR_0_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC2_CFG_BASE                0x7FFCE86A38ull
+#define QM_TENSOR_1_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC2_CFG_BASE                0x7FFCE86A70ull
+#define QM_TENSOR_2_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC2_CFG_BASE                0x7FFCE86AA8ull
+#define QM_TENSOR_3_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC2_CFG_BASE                0x7FFCE86AE0ull
+#define QM_TENSOR_4_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC2_CFG_BASE                0x7FFCE86B18ull
+#define QM_TENSOR_5_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC2_CFG_BASE                0x7FFCE86B50ull
+#define QM_TENSOR_6_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC2_CFG_BASE                0x7FFCE86B88ull
+#define QM_TENSOR_7_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC2_CFG_BASE                0x7FFCE86BC0ull
+#define QM_TENSOR_8_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC2_CFG_BASE                0x7FFCE86BF8ull
+#define QM_TENSOR_9_TPC2_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC2_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC2_CFG_BASE               0x7FFCE86C30ull
+#define QM_TENSOR_10_TPC2_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC2_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC2_CFG_BASE               0x7FFCE86C68ull
+#define QM_TENSOR_11_TPC2_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC2_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC2_CFG_BASE               0x7FFCE86CA0ull
+#define QM_TENSOR_12_TPC2_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC2_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC2_CFG_BASE               0x7FFCE86CD8ull
+#define QM_TENSOR_13_TPC2_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC2_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC2_CFG_BASE               0x7FFCE86D10ull
+#define QM_TENSOR_14_TPC2_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC2_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC2_CFG_BASE               0x7FFCE86D48ull
+#define QM_TENSOR_15_TPC2_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC2_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC2_CFG_BASE             0x7FFCE86D80ull
+#define QM_SYNC_OBJECT_TPC2_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC2_CFG_SECTION            0x8000
+#define mmQM_TPC2_CFG_BASE                         0x7FFCE86D88ull
+#define QM_TPC2_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC2_CFG_SECTION                        0x2780
+#define mmTPC2_E2E_CRED_BASE                       0x7FFCE87000ull
+#define TPC2_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC2_E2E_CRED_SECTION                      0x1000
+#define mmTPC2_QM_BASE                             0x7FFCE88000ull
+#define TPC2_QM_MAX_OFFSET                         0xD040
+#define TPC2_QM_SECTION                            0x3E000
+#define mmTPC3_CFG_BASE                            0x7FFCEC6000ull
+#define TPC3_CFG_MAX_OFFSET                        0xE400
+#define TPC3_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC3_CFG_BASE            0x7FFCEC6400ull
+#define KERNEL_TENSOR_0_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC3_CFG_BASE            0x7FFCEC6438ull
+#define KERNEL_TENSOR_1_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC3_CFG_BASE            0x7FFCEC6470ull
+#define KERNEL_TENSOR_2_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC3_CFG_BASE            0x7FFCEC64A8ull
+#define KERNEL_TENSOR_3_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC3_CFG_BASE            0x7FFCEC64E0ull
+#define KERNEL_TENSOR_4_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC3_CFG_BASE            0x7FFCEC6518ull
+#define KERNEL_TENSOR_5_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC3_CFG_BASE            0x7FFCEC6550ull
+#define KERNEL_TENSOR_6_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC3_CFG_BASE            0x7FFCEC6588ull
+#define KERNEL_TENSOR_7_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC3_CFG_BASE            0x7FFCEC65C0ull
+#define KERNEL_TENSOR_8_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC3_CFG_BASE            0x7FFCEC65F8ull
+#define KERNEL_TENSOR_9_TPC3_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC3_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC3_CFG_BASE           0x7FFCEC6630ull
+#define KERNEL_TENSOR_10_TPC3_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC3_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC3_CFG_BASE           0x7FFCEC6668ull
+#define KERNEL_TENSOR_11_TPC3_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC3_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC3_CFG_BASE           0x7FFCEC66A0ull
+#define KERNEL_TENSOR_12_TPC3_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC3_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC3_CFG_BASE           0x7FFCEC66D8ull
+#define KERNEL_TENSOR_13_TPC3_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC3_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC3_CFG_BASE           0x7FFCEC6710ull
+#define KERNEL_TENSOR_14_TPC3_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC3_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC3_CFG_BASE           0x7FFCEC6748ull
+#define KERNEL_TENSOR_15_TPC3_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC3_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC3_CFG_BASE         0x7FFCEC6780ull
+#define KERNEL_SYNC_OBJECT_TPC3_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC3_CFG_SECTION        0x8000
+#define mmKERNEL_TPC3_CFG_BASE                     0x7FFCEC6788ull
+#define KERNEL_TPC3_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC3_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC3_CFG_BASE                0x7FFCEC6A00ull
+#define QM_TENSOR_0_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC3_CFG_BASE                0x7FFCEC6A38ull
+#define QM_TENSOR_1_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC3_CFG_BASE                0x7FFCEC6A70ull
+#define QM_TENSOR_2_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC3_CFG_BASE                0x7FFCEC6AA8ull
+#define QM_TENSOR_3_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC3_CFG_BASE                0x7FFCEC6AE0ull
+#define QM_TENSOR_4_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC3_CFG_BASE                0x7FFCEC6B18ull
+#define QM_TENSOR_5_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC3_CFG_BASE                0x7FFCEC6B50ull
+#define QM_TENSOR_6_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC3_CFG_BASE                0x7FFCEC6B88ull
+#define QM_TENSOR_7_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC3_CFG_BASE                0x7FFCEC6BC0ull
+#define QM_TENSOR_8_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC3_CFG_BASE                0x7FFCEC6BF8ull
+#define QM_TENSOR_9_TPC3_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC3_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC3_CFG_BASE               0x7FFCEC6C30ull
+#define QM_TENSOR_10_TPC3_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC3_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC3_CFG_BASE               0x7FFCEC6C68ull
+#define QM_TENSOR_11_TPC3_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC3_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC3_CFG_BASE               0x7FFCEC6CA0ull
+#define QM_TENSOR_12_TPC3_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC3_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC3_CFG_BASE               0x7FFCEC6CD8ull
+#define QM_TENSOR_13_TPC3_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC3_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC3_CFG_BASE               0x7FFCEC6D10ull
+#define QM_TENSOR_14_TPC3_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC3_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC3_CFG_BASE               0x7FFCEC6D48ull
+#define QM_TENSOR_15_TPC3_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC3_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC3_CFG_BASE             0x7FFCEC6D80ull
+#define QM_SYNC_OBJECT_TPC3_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC3_CFG_SECTION            0x8000
+#define mmQM_TPC3_CFG_BASE                         0x7FFCEC6D88ull
+#define QM_TPC3_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC3_CFG_SECTION                        0x2780
+#define mmTPC3_E2E_CRED_BASE                       0x7FFCEC7000ull
+#define TPC3_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC3_E2E_CRED_SECTION                      0x1000
+#define mmTPC3_QM_BASE                             0x7FFCEC8000ull
+#define TPC3_QM_MAX_OFFSET                         0xD040
+#define TPC3_QM_SECTION                            0x3E000
+#define mmTPC4_CFG_BASE                            0x7FFCF06000ull
+#define TPC4_CFG_MAX_OFFSET                        0xE400
+#define TPC4_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC4_CFG_BASE            0x7FFCF06400ull
+#define KERNEL_TENSOR_0_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC4_CFG_BASE            0x7FFCF06438ull
+#define KERNEL_TENSOR_1_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC4_CFG_BASE            0x7FFCF06470ull
+#define KERNEL_TENSOR_2_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC4_CFG_BASE            0x7FFCF064A8ull
+#define KERNEL_TENSOR_3_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC4_CFG_BASE            0x7FFCF064E0ull
+#define KERNEL_TENSOR_4_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC4_CFG_BASE            0x7FFCF06518ull
+#define KERNEL_TENSOR_5_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC4_CFG_BASE            0x7FFCF06550ull
+#define KERNEL_TENSOR_6_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC4_CFG_BASE            0x7FFCF06588ull
+#define KERNEL_TENSOR_7_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC4_CFG_BASE            0x7FFCF065C0ull
+#define KERNEL_TENSOR_8_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC4_CFG_BASE            0x7FFCF065F8ull
+#define KERNEL_TENSOR_9_TPC4_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC4_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC4_CFG_BASE           0x7FFCF06630ull
+#define KERNEL_TENSOR_10_TPC4_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC4_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC4_CFG_BASE           0x7FFCF06668ull
+#define KERNEL_TENSOR_11_TPC4_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC4_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC4_CFG_BASE           0x7FFCF066A0ull
+#define KERNEL_TENSOR_12_TPC4_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC4_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC4_CFG_BASE           0x7FFCF066D8ull
+#define KERNEL_TENSOR_13_TPC4_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC4_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC4_CFG_BASE           0x7FFCF06710ull
+#define KERNEL_TENSOR_14_TPC4_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC4_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC4_CFG_BASE           0x7FFCF06748ull
+#define KERNEL_TENSOR_15_TPC4_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC4_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC4_CFG_BASE         0x7FFCF06780ull
+#define KERNEL_SYNC_OBJECT_TPC4_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC4_CFG_SECTION        0x8000
+#define mmKERNEL_TPC4_CFG_BASE                     0x7FFCF06788ull
+#define KERNEL_TPC4_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC4_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC4_CFG_BASE                0x7FFCF06A00ull
+#define QM_TENSOR_0_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC4_CFG_BASE                0x7FFCF06A38ull
+#define QM_TENSOR_1_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC4_CFG_BASE                0x7FFCF06A70ull
+#define QM_TENSOR_2_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC4_CFG_BASE                0x7FFCF06AA8ull
+#define QM_TENSOR_3_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC4_CFG_BASE                0x7FFCF06AE0ull
+#define QM_TENSOR_4_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC4_CFG_BASE                0x7FFCF06B18ull
+#define QM_TENSOR_5_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC4_CFG_BASE                0x7FFCF06B50ull
+#define QM_TENSOR_6_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC4_CFG_BASE                0x7FFCF06B88ull
+#define QM_TENSOR_7_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC4_CFG_BASE                0x7FFCF06BC0ull
+#define QM_TENSOR_8_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC4_CFG_BASE                0x7FFCF06BF8ull
+#define QM_TENSOR_9_TPC4_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC4_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC4_CFG_BASE               0x7FFCF06C30ull
+#define QM_TENSOR_10_TPC4_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC4_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC4_CFG_BASE               0x7FFCF06C68ull
+#define QM_TENSOR_11_TPC4_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC4_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC4_CFG_BASE               0x7FFCF06CA0ull
+#define QM_TENSOR_12_TPC4_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC4_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC4_CFG_BASE               0x7FFCF06CD8ull
+#define QM_TENSOR_13_TPC4_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC4_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC4_CFG_BASE               0x7FFCF06D10ull
+#define QM_TENSOR_14_TPC4_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC4_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC4_CFG_BASE               0x7FFCF06D48ull
+#define QM_TENSOR_15_TPC4_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC4_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC4_CFG_BASE             0x7FFCF06D80ull
+#define QM_SYNC_OBJECT_TPC4_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC4_CFG_SECTION            0x8000
+#define mmQM_TPC4_CFG_BASE                         0x7FFCF06D88ull
+#define QM_TPC4_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC4_CFG_SECTION                        0x2780
+#define mmTPC4_E2E_CRED_BASE                       0x7FFCF07000ull
+#define TPC4_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC4_E2E_CRED_SECTION                      0x1000
+#define mmTPC4_QM_BASE                             0x7FFCF08000ull
+#define TPC4_QM_MAX_OFFSET                         0xD040
+#define TPC4_QM_SECTION                            0x3E000
+#define mmTPC5_CFG_BASE                            0x7FFCF46000ull
+#define TPC5_CFG_MAX_OFFSET                        0xE400
+#define TPC5_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC5_CFG_BASE            0x7FFCF46400ull
+#define KERNEL_TENSOR_0_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC5_CFG_BASE            0x7FFCF46438ull
+#define KERNEL_TENSOR_1_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC5_CFG_BASE            0x7FFCF46470ull
+#define KERNEL_TENSOR_2_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC5_CFG_BASE            0x7FFCF464A8ull
+#define KERNEL_TENSOR_3_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC5_CFG_BASE            0x7FFCF464E0ull
+#define KERNEL_TENSOR_4_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC5_CFG_BASE            0x7FFCF46518ull
+#define KERNEL_TENSOR_5_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC5_CFG_BASE            0x7FFCF46550ull
+#define KERNEL_TENSOR_6_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC5_CFG_BASE            0x7FFCF46588ull
+#define KERNEL_TENSOR_7_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC5_CFG_BASE            0x7FFCF465C0ull
+#define KERNEL_TENSOR_8_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC5_CFG_BASE            0x7FFCF465F8ull
+#define KERNEL_TENSOR_9_TPC5_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC5_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC5_CFG_BASE           0x7FFCF46630ull
+#define KERNEL_TENSOR_10_TPC5_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC5_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC5_CFG_BASE           0x7FFCF46668ull
+#define KERNEL_TENSOR_11_TPC5_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC5_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC5_CFG_BASE           0x7FFCF466A0ull
+#define KERNEL_TENSOR_12_TPC5_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC5_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC5_CFG_BASE           0x7FFCF466D8ull
+#define KERNEL_TENSOR_13_TPC5_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC5_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC5_CFG_BASE           0x7FFCF46710ull
+#define KERNEL_TENSOR_14_TPC5_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC5_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC5_CFG_BASE           0x7FFCF46748ull
+#define KERNEL_TENSOR_15_TPC5_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC5_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC5_CFG_BASE         0x7FFCF46780ull
+#define KERNEL_SYNC_OBJECT_TPC5_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC5_CFG_SECTION        0x8000
+#define mmKERNEL_TPC5_CFG_BASE                     0x7FFCF46788ull
+#define KERNEL_TPC5_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC5_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC5_CFG_BASE                0x7FFCF46A00ull
+#define QM_TENSOR_0_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC5_CFG_BASE                0x7FFCF46A38ull
+#define QM_TENSOR_1_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC5_CFG_BASE                0x7FFCF46A70ull
+#define QM_TENSOR_2_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC5_CFG_BASE                0x7FFCF46AA8ull
+#define QM_TENSOR_3_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC5_CFG_BASE                0x7FFCF46AE0ull
+#define QM_TENSOR_4_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC5_CFG_BASE                0x7FFCF46B18ull
+#define QM_TENSOR_5_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC5_CFG_BASE                0x7FFCF46B50ull
+#define QM_TENSOR_6_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC5_CFG_BASE                0x7FFCF46B88ull
+#define QM_TENSOR_7_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC5_CFG_BASE                0x7FFCF46BC0ull
+#define QM_TENSOR_8_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC5_CFG_BASE                0x7FFCF46BF8ull
+#define QM_TENSOR_9_TPC5_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC5_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC5_CFG_BASE               0x7FFCF46C30ull
+#define QM_TENSOR_10_TPC5_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC5_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC5_CFG_BASE               0x7FFCF46C68ull
+#define QM_TENSOR_11_TPC5_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC5_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC5_CFG_BASE               0x7FFCF46CA0ull
+#define QM_TENSOR_12_TPC5_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC5_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC5_CFG_BASE               0x7FFCF46CD8ull
+#define QM_TENSOR_13_TPC5_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC5_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC5_CFG_BASE               0x7FFCF46D10ull
+#define QM_TENSOR_14_TPC5_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC5_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC5_CFG_BASE               0x7FFCF46D48ull
+#define QM_TENSOR_15_TPC5_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC5_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC5_CFG_BASE             0x7FFCF46D80ull
+#define QM_SYNC_OBJECT_TPC5_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC5_CFG_SECTION            0x8000
+#define mmQM_TPC5_CFG_BASE                         0x7FFCF46D88ull
+#define QM_TPC5_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC5_CFG_SECTION                        0x2780
+#define mmTPC5_E2E_CRED_BASE                       0x7FFCF47000ull
+#define TPC5_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC5_E2E_CRED_SECTION                      0x1000
+#define mmTPC5_QM_BASE                             0x7FFCF48000ull
+#define TPC5_QM_MAX_OFFSET                         0xD040
+#define TPC5_QM_SECTION                            0x3E000
+#define mmTPC6_CFG_BASE                            0x7FFCF86000ull
+#define TPC6_CFG_MAX_OFFSET                        0xE400
+#define TPC6_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC6_CFG_BASE            0x7FFCF86400ull
+#define KERNEL_TENSOR_0_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC6_CFG_BASE            0x7FFCF86438ull
+#define KERNEL_TENSOR_1_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC6_CFG_BASE            0x7FFCF86470ull
+#define KERNEL_TENSOR_2_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC6_CFG_BASE            0x7FFCF864A8ull
+#define KERNEL_TENSOR_3_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC6_CFG_BASE            0x7FFCF864E0ull
+#define KERNEL_TENSOR_4_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC6_CFG_BASE            0x7FFCF86518ull
+#define KERNEL_TENSOR_5_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC6_CFG_BASE            0x7FFCF86550ull
+#define KERNEL_TENSOR_6_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC6_CFG_BASE            0x7FFCF86588ull
+#define KERNEL_TENSOR_7_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC6_CFG_BASE            0x7FFCF865C0ull
+#define KERNEL_TENSOR_8_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC6_CFG_BASE            0x7FFCF865F8ull
+#define KERNEL_TENSOR_9_TPC6_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC6_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC6_CFG_BASE           0x7FFCF86630ull
+#define KERNEL_TENSOR_10_TPC6_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC6_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC6_CFG_BASE           0x7FFCF86668ull
+#define KERNEL_TENSOR_11_TPC6_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC6_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC6_CFG_BASE           0x7FFCF866A0ull
+#define KERNEL_TENSOR_12_TPC6_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC6_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC6_CFG_BASE           0x7FFCF866D8ull
+#define KERNEL_TENSOR_13_TPC6_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC6_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC6_CFG_BASE           0x7FFCF86710ull
+#define KERNEL_TENSOR_14_TPC6_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC6_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC6_CFG_BASE           0x7FFCF86748ull
+#define KERNEL_TENSOR_15_TPC6_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC6_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC6_CFG_BASE         0x7FFCF86780ull
+#define KERNEL_SYNC_OBJECT_TPC6_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC6_CFG_SECTION        0x8000
+#define mmKERNEL_TPC6_CFG_BASE                     0x7FFCF86788ull
+#define KERNEL_TPC6_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC6_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC6_CFG_BASE                0x7FFCF86A00ull
+#define QM_TENSOR_0_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC6_CFG_BASE                0x7FFCF86A38ull
+#define QM_TENSOR_1_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC6_CFG_BASE                0x7FFCF86A70ull
+#define QM_TENSOR_2_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC6_CFG_BASE                0x7FFCF86AA8ull
+#define QM_TENSOR_3_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC6_CFG_BASE                0x7FFCF86AE0ull
+#define QM_TENSOR_4_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC6_CFG_BASE                0x7FFCF86B18ull
+#define QM_TENSOR_5_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC6_CFG_BASE                0x7FFCF86B50ull
+#define QM_TENSOR_6_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC6_CFG_BASE                0x7FFCF86B88ull
+#define QM_TENSOR_7_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC6_CFG_BASE                0x7FFCF86BC0ull
+#define QM_TENSOR_8_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC6_CFG_BASE                0x7FFCF86BF8ull
+#define QM_TENSOR_9_TPC6_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC6_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC6_CFG_BASE               0x7FFCF86C30ull
+#define QM_TENSOR_10_TPC6_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC6_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC6_CFG_BASE               0x7FFCF86C68ull
+#define QM_TENSOR_11_TPC6_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC6_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC6_CFG_BASE               0x7FFCF86CA0ull
+#define QM_TENSOR_12_TPC6_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC6_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC6_CFG_BASE               0x7FFCF86CD8ull
+#define QM_TENSOR_13_TPC6_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC6_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC6_CFG_BASE               0x7FFCF86D10ull
+#define QM_TENSOR_14_TPC6_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC6_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC6_CFG_BASE               0x7FFCF86D48ull
+#define QM_TENSOR_15_TPC6_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC6_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC6_CFG_BASE             0x7FFCF86D80ull
+#define QM_SYNC_OBJECT_TPC6_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC6_CFG_SECTION            0x8000
+#define mmQM_TPC6_CFG_BASE                         0x7FFCF86D88ull
+#define QM_TPC6_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC6_CFG_SECTION                        0x2780
+#define mmTPC6_E2E_CRED_BASE                       0x7FFCF87000ull
+#define TPC6_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC6_E2E_CRED_SECTION                      0x1000
+#define mmTPC6_QM_BASE                             0x7FFCF88000ull
+#define TPC6_QM_MAX_OFFSET                         0xD040
+#define TPC6_QM_SECTION                            0x3E000
+#define mmTPC7_CFG_BASE                            0x7FFCFC6000ull
+#define TPC7_CFG_MAX_OFFSET                        0xE400
+#define TPC7_CFG_SECTION                           0x4000
+#define mmKERNEL_TENSOR_0_TPC7_CFG_BASE            0x7FFCFC6400ull
+#define KERNEL_TENSOR_0_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_0_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_1_TPC7_CFG_BASE            0x7FFCFC6438ull
+#define KERNEL_TENSOR_1_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_1_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_2_TPC7_CFG_BASE            0x7FFCFC6470ull
+#define KERNEL_TENSOR_2_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_2_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_3_TPC7_CFG_BASE            0x7FFCFC64A8ull
+#define KERNEL_TENSOR_3_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_3_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_4_TPC7_CFG_BASE            0x7FFCFC64E0ull
+#define KERNEL_TENSOR_4_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_4_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_5_TPC7_CFG_BASE            0x7FFCFC6518ull
+#define KERNEL_TENSOR_5_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_5_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_6_TPC7_CFG_BASE            0x7FFCFC6550ull
+#define KERNEL_TENSOR_6_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_6_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_7_TPC7_CFG_BASE            0x7FFCFC6588ull
+#define KERNEL_TENSOR_7_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_7_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_8_TPC7_CFG_BASE            0x7FFCFC65C0ull
+#define KERNEL_TENSOR_8_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_8_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_9_TPC7_CFG_BASE            0x7FFCFC65F8ull
+#define KERNEL_TENSOR_9_TPC7_CFG_MAX_OFFSET        0x3800
+#define KERNEL_TENSOR_9_TPC7_CFG_SECTION           0x3800
+#define mmKERNEL_TENSOR_10_TPC7_CFG_BASE           0x7FFCFC6630ull
+#define KERNEL_TENSOR_10_TPC7_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_10_TPC7_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_11_TPC7_CFG_BASE           0x7FFCFC6668ull
+#define KERNEL_TENSOR_11_TPC7_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_11_TPC7_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_12_TPC7_CFG_BASE           0x7FFCFC66A0ull
+#define KERNEL_TENSOR_12_TPC7_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_12_TPC7_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_13_TPC7_CFG_BASE           0x7FFCFC66D8ull
+#define KERNEL_TENSOR_13_TPC7_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_13_TPC7_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_14_TPC7_CFG_BASE           0x7FFCFC6710ull
+#define KERNEL_TENSOR_14_TPC7_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_14_TPC7_CFG_SECTION          0x3800
+#define mmKERNEL_TENSOR_15_TPC7_CFG_BASE           0x7FFCFC6748ull
+#define KERNEL_TENSOR_15_TPC7_CFG_MAX_OFFSET       0x3800
+#define KERNEL_TENSOR_15_TPC7_CFG_SECTION          0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC7_CFG_BASE         0x7FFCFC6780ull
+#define KERNEL_SYNC_OBJECT_TPC7_CFG_MAX_OFFSET     0x8000
+#define KERNEL_SYNC_OBJECT_TPC7_CFG_SECTION        0x8000
+#define mmKERNEL_TPC7_CFG_BASE                     0x7FFCFC6788ull
+#define KERNEL_TPC7_CFG_MAX_OFFSET                 0xB800
+#define KERNEL_TPC7_CFG_SECTION                    0x2780
+#define mmQM_TENSOR_0_TPC7_CFG_BASE                0x7FFCFC6A00ull
+#define QM_TENSOR_0_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_0_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_1_TPC7_CFG_BASE                0x7FFCFC6A38ull
+#define QM_TENSOR_1_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_1_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_2_TPC7_CFG_BASE                0x7FFCFC6A70ull
+#define QM_TENSOR_2_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_2_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_3_TPC7_CFG_BASE                0x7FFCFC6AA8ull
+#define QM_TENSOR_3_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_3_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_4_TPC7_CFG_BASE                0x7FFCFC6AE0ull
+#define QM_TENSOR_4_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_4_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_5_TPC7_CFG_BASE                0x7FFCFC6B18ull
+#define QM_TENSOR_5_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_5_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_6_TPC7_CFG_BASE                0x7FFCFC6B50ull
+#define QM_TENSOR_6_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_6_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_7_TPC7_CFG_BASE                0x7FFCFC6B88ull
+#define QM_TENSOR_7_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_7_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_8_TPC7_CFG_BASE                0x7FFCFC6BC0ull
+#define QM_TENSOR_8_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_8_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_9_TPC7_CFG_BASE                0x7FFCFC6BF8ull
+#define QM_TENSOR_9_TPC7_CFG_MAX_OFFSET            0x3800
+#define QM_TENSOR_9_TPC7_CFG_SECTION               0x3800
+#define mmQM_TENSOR_10_TPC7_CFG_BASE               0x7FFCFC6C30ull
+#define QM_TENSOR_10_TPC7_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_10_TPC7_CFG_SECTION              0x3800
+#define mmQM_TENSOR_11_TPC7_CFG_BASE               0x7FFCFC6C68ull
+#define QM_TENSOR_11_TPC7_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_11_TPC7_CFG_SECTION              0x3800
+#define mmQM_TENSOR_12_TPC7_CFG_BASE               0x7FFCFC6CA0ull
+#define QM_TENSOR_12_TPC7_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_12_TPC7_CFG_SECTION              0x3800
+#define mmQM_TENSOR_13_TPC7_CFG_BASE               0x7FFCFC6CD8ull
+#define QM_TENSOR_13_TPC7_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_13_TPC7_CFG_SECTION              0x3800
+#define mmQM_TENSOR_14_TPC7_CFG_BASE               0x7FFCFC6D10ull
+#define QM_TENSOR_14_TPC7_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_14_TPC7_CFG_SECTION              0x3800
+#define mmQM_TENSOR_15_TPC7_CFG_BASE               0x7FFCFC6D48ull
+#define QM_TENSOR_15_TPC7_CFG_MAX_OFFSET           0x3800
+#define QM_TENSOR_15_TPC7_CFG_SECTION              0x3800
+#define mmQM_SYNC_OBJECT_TPC7_CFG_BASE             0x7FFCFC6D80ull
+#define QM_SYNC_OBJECT_TPC7_CFG_MAX_OFFSET         0x8000
+#define QM_SYNC_OBJECT_TPC7_CFG_SECTION            0x8000
+#define mmQM_TPC7_CFG_BASE                         0x7FFCFC6D88ull
+#define QM_TPC7_CFG_MAX_OFFSET                     0xB800
+#define QM_TPC7_CFG_SECTION                        0x2780
+#define mmTPC7_E2E_CRED_BASE                       0x7FFCFC7000ull
+#define TPC7_E2E_CRED_MAX_OFFSET                   0x1680
+#define TPC7_E2E_CRED_SECTION                      0x1000
+#define mmTPC7_QM_BASE                             0x7FFCFC8000ull
+#define TPC7_QM_MAX_OFFSET                         0xD040
+#define TPC7_QM_SECTION                            0x1038000
+#define mmMME_S_ROM_TABLE_BASE                     0x7FFE000000ull
+#define MME_S_ROM_TABLE_MAX_OFFSET                 0x1000
+#define MME_S_ROM_TABLE_SECTION                    0x21000
+#define mmMME0_ACC_STM_BASE                        0x7FFE021000ull
+#define MME0_ACC_STM_MAX_OFFSET                    0x1000
+#define MME0_ACC_STM_SECTION                       0x1000
+#define mmMME0_ACC_CTI_BASE                        0x7FFE022000ull
+#define MME0_ACC_CTI_MAX_OFFSET                    0x1000
+#define MME0_ACC_CTI_SECTION                       0x1000
+#define mmMME0_ACC_ETF_BASE                        0x7FFE023000ull
+#define MME0_ACC_ETF_MAX_OFFSET                    0x1000
+#define MME0_ACC_ETF_SECTION                       0x1000
+#define mmMME0_ACC_SPMU_BASE                       0x7FFE024000ull
+#define MME0_ACC_SPMU_MAX_OFFSET                   0x1000
+#define MME0_ACC_SPMU_SECTION                      0x1000
+#define mmMME0_ACC_CTI0_BASE                       0x7FFE025000ull
+#define MME0_ACC_CTI0_MAX_OFFSET                   0x1000
+#define MME0_ACC_CTI0_SECTION                      0x1000
+#define mmMME0_ACC_CTI1_BASE                       0x7FFE026000ull
+#define MME0_ACC_CTI1_MAX_OFFSET                   0x1000
+#define MME0_ACC_CTI1_SECTION                      0x1000
+#define mmMME0_ACC_BMON0_BASE                      0x7FFE027000ull
+#define MME0_ACC_BMON0_MAX_OFFSET                  0x1000
+#define MME0_ACC_BMON0_SECTION                     0x9000
+#define mmMME0_ACC_FUNNEL_BASE                     0x7FFE030000ull
+#define MME0_ACC_FUNNEL_MAX_OFFSET                 0x1000
+#define MME0_ACC_FUNNEL_SECTION                    0x11000
+#define mmMME0_SBAB_STM_BASE                       0x7FFE041000ull
+#define MME0_SBAB_STM_MAX_OFFSET                   0x1000
+#define MME0_SBAB_STM_SECTION                      0x1000
+#define mmMME0_SBAB_CTI_BASE                       0x7FFE042000ull
+#define MME0_SBAB_CTI_MAX_OFFSET                   0x1000
+#define MME0_SBAB_CTI_SECTION                      0x1000
+#define mmMME0_SBAB_ETF_BASE                       0x7FFE043000ull
+#define MME0_SBAB_ETF_MAX_OFFSET                   0x1000
+#define MME0_SBAB_ETF_SECTION                      0x1000
+#define mmMME0_SBAB_SPMU_BASE                      0x7FFE044000ull
+#define MME0_SBAB_SPMU_MAX_OFFSET                  0x1000
+#define MME0_SBAB_SPMU_SECTION                     0x1000
+#define mmMME0_SBAB_CTI0_BASE                      0x7FFE045000ull
+#define MME0_SBAB_CTI0_MAX_OFFSET                  0x1000
+#define MME0_SBAB_CTI0_SECTION                     0x1000
+#define mmMME0_SBAB_CTI1_BASE                      0x7FFE046000ull
+#define MME0_SBAB_CTI1_MAX_OFFSET                  0x1000
+#define MME0_SBAB_CTI1_SECTION                     0x1000
+#define mmMME0_SBAB_BMON0_BASE                     0x7FFE047000ull
+#define MME0_SBAB_BMON0_MAX_OFFSET                 0x1000
+#define MME0_SBAB_BMON0_SECTION                    0x1000
+#define mmMME0_SBAB_BMON1_BASE                     0x7FFE048000ull
+#define MME0_SBAB_BMON1_MAX_OFFSET                 0x1000
+#define MME0_SBAB_BMON1_SECTION                    0x19000
+#define mmMME0_CTRL_STM_BASE                       0x7FFE061000ull
+#define MME0_CTRL_STM_MAX_OFFSET                   0x1000
+#define MME0_CTRL_STM_SECTION                      0x1000
+#define mmMME0_CTRL_CTI_BASE                       0x7FFE062000ull
+#define MME0_CTRL_CTI_MAX_OFFSET                   0x1000
+#define MME0_CTRL_CTI_SECTION                      0x1000
+#define mmMME0_CTRL_ETF_BASE                       0x7FFE063000ull
+#define MME0_CTRL_ETF_MAX_OFFSET                   0x1000
+#define MME0_CTRL_ETF_SECTION                      0x1000
+#define mmMME0_CTRL_SPMU_BASE                      0x7FFE064000ull
+#define MME0_CTRL_SPMU_MAX_OFFSET                  0x1000
+#define MME0_CTRL_SPMU_SECTION                     0x1000
+#define mmMME0_CTRL_CTI0_BASE                      0x7FFE065000ull
+#define MME0_CTRL_CTI0_MAX_OFFSET                  0x1000
+#define MME0_CTRL_CTI0_SECTION                     0x1000
+#define mmMME0_CTRL_CTI1_BASE                      0x7FFE066000ull
+#define MME0_CTRL_CTI1_MAX_OFFSET                  0x1000
+#define MME0_CTRL_CTI1_SECTION                     0x1000
+#define mmMME0_CTRL_BMON0_BASE                     0x7FFE067000ull
+#define MME0_CTRL_BMON0_MAX_OFFSET                 0x1000
+#define MME0_CTRL_BMON0_SECTION                    0x1000
+#define mmMME0_CTRL_BMON1_BASE                     0x7FFE068000ull
+#define MME0_CTRL_BMON1_MAX_OFFSET                 0x1000
+#define MME0_CTRL_BMON1_SECTION                    0x39000
+#define mmMME1_ACC_STM_BASE                        0x7FFE0A1000ull
+#define MME1_ACC_STM_MAX_OFFSET                    0x1000
+#define MME1_ACC_STM_SECTION                       0x1000
+#define mmMME1_ACC_CTI_BASE                        0x7FFE0A2000ull
+#define MME1_ACC_CTI_MAX_OFFSET                    0x1000
+#define MME1_ACC_CTI_SECTION                       0x1000
+#define mmMME1_ACC_ETF_BASE                        0x7FFE0A3000ull
+#define MME1_ACC_ETF_MAX_OFFSET                    0x1000
+#define MME1_ACC_ETF_SECTION                       0x1000
+#define mmMME1_ACC_SPMU_BASE                       0x7FFE0A4000ull
+#define MME1_ACC_SPMU_MAX_OFFSET                   0x1000
+#define MME1_ACC_SPMU_SECTION                      0x1000
+#define mmMME1_ACC_CTI0_BASE                       0x7FFE0A5000ull
+#define MME1_ACC_CTI0_MAX_OFFSET                   0x1000
+#define MME1_ACC_CTI0_SECTION                      0x1000
+#define mmMME1_ACC_CTI1_BASE                       0x7FFE0A6000ull
+#define MME1_ACC_CTI1_MAX_OFFSET                   0x1000
+#define MME1_ACC_CTI1_SECTION                      0x1000
+#define mmMME1_ACC_BMON0_BASE                      0x7FFE0A7000ull
+#define MME1_ACC_BMON0_MAX_OFFSET                  0x1000
+#define MME1_ACC_BMON0_SECTION                     0x9000
+#define mmMME1_ACC_FUNNEL_BASE                     0x7FFE0B0000ull
+#define MME1_ACC_FUNNEL_MAX_OFFSET                 0x1000
+#define MME1_ACC_FUNNEL_SECTION                    0x11000
+#define mmMME1_SBAB_STM_BASE                       0x7FFE0C1000ull
+#define MME1_SBAB_STM_MAX_OFFSET                   0x1000
+#define MME1_SBAB_STM_SECTION                      0x1000
+#define mmMME1_SBAB_CTI_BASE                       0x7FFE0C2000ull
+#define MME1_SBAB_CTI_MAX_OFFSET                   0x1000
+#define MME1_SBAB_CTI_SECTION                      0x1000
+#define mmMME1_SBAB_ETF_BASE                       0x7FFE0C3000ull
+#define MME1_SBAB_ETF_MAX_OFFSET                   0x1000
+#define MME1_SBAB_ETF_SECTION                      0x1000
+#define mmMME1_SBAB_SPMU_BASE                      0x7FFE0C4000ull
+#define MME1_SBAB_SPMU_MAX_OFFSET                  0x1000
+#define MME1_SBAB_SPMU_SECTION                     0x1000
+#define mmMME1_SBAB_CTI0_BASE                      0x7FFE0C5000ull
+#define MME1_SBAB_CTI0_MAX_OFFSET                  0x1000
+#define MME1_SBAB_CTI0_SECTION                     0x1000
+#define mmMME1_SBAB_CTI1_BASE                      0x7FFE0C6000ull
+#define MME1_SBAB_CTI1_MAX_OFFSET                  0x1000
+#define MME1_SBAB_CTI1_SECTION                     0x1000
+#define mmMME1_SBAB_BMON0_BASE                     0x7FFE0C7000ull
+#define MME1_SBAB_BMON0_MAX_OFFSET                 0x1000
+#define MME1_SBAB_BMON0_SECTION                    0x1000
+#define mmMME1_SBAB_BMON1_BASE                     0x7FFE0C8000ull
+#define MME1_SBAB_BMON1_MAX_OFFSET                 0x1000
+#define MME1_SBAB_BMON1_SECTION                    0x19000
+#define mmMME1_CTRL_STM_BASE                       0x7FFE0E1000ull
+#define MME1_CTRL_STM_MAX_OFFSET                   0x1000
+#define MME1_CTRL_STM_SECTION                      0x1000
+#define mmMME1_CTRL_CTI_BASE                       0x7FFE0E2000ull
+#define MME1_CTRL_CTI_MAX_OFFSET                   0x1000
+#define MME1_CTRL_CTI_SECTION                      0x1000
+#define mmMME1_CTRL_ETF_BASE                       0x7FFE0E3000ull
+#define MME1_CTRL_ETF_MAX_OFFSET                   0x1000
+#define MME1_CTRL_ETF_SECTION                      0x1000
+#define mmMME1_CTRL_SPMU_BASE                      0x7FFE0E4000ull
+#define MME1_CTRL_SPMU_MAX_OFFSET                  0x1000
+#define MME1_CTRL_SPMU_SECTION                     0x1000
+#define mmMME1_CTRL_CTI0_BASE                      0x7FFE0E5000ull
+#define MME1_CTRL_CTI0_MAX_OFFSET                  0x1000
+#define MME1_CTRL_CTI0_SECTION                     0x1000
+#define mmMME1_CTRL_CTI1_BASE                      0x7FFE0E6000ull
+#define MME1_CTRL_CTI1_MAX_OFFSET                  0x1000
+#define MME1_CTRL_CTI1_SECTION                     0x1000
+#define mmMME1_CTRL_BMON0_BASE                     0x7FFE0E7000ull
+#define MME1_CTRL_BMON0_MAX_OFFSET                 0x1000
+#define MME1_CTRL_BMON0_SECTION                    0x1000
+#define mmMME1_CTRL_BMON1_BASE                     0x7FFE0E8000ull
+#define MME1_CTRL_BMON1_MAX_OFFSET                 0x1000
+#define MME1_CTRL_BMON1_SECTION                    0x18000
+#define mmMME_N_ROM_TABLE_BASE                     0x7FFE100000ull
+#define MME_N_ROM_TABLE_MAX_OFFSET                 0x1000
+#define MME_N_ROM_TABLE_SECTION                    0x21000
+#define mmMME2_ACC_STM_BASE                        0x7FFE121000ull
+#define MME2_ACC_STM_MAX_OFFSET                    0x1000
+#define MME2_ACC_STM_SECTION                       0x1000
+#define mmMME2_ACC_CTI_BASE                        0x7FFE122000ull
+#define MME2_ACC_CTI_MAX_OFFSET                    0x1000
+#define MME2_ACC_CTI_SECTION                       0x1000
+#define mmMME2_MME2_ACC_ETF_BASE                   0x7FFE123000ull
+#define MME2_MME2_ACC_ETF_MAX_OFFSET               0x1000
+#define MME2_MME2_ACC_ETF_SECTION                  0x1000
+#define mmMME2_ACC_SPMU_BASE                       0x7FFE124000ull
+#define MME2_ACC_SPMU_MAX_OFFSET                   0x1000
+#define MME2_ACC_SPMU_SECTION                      0x1000
+#define mmMME2_ACC_CTI0_BASE                       0x7FFE125000ull
+#define MME2_ACC_CTI0_MAX_OFFSET                   0x1000
+#define MME2_ACC_CTI0_SECTION                      0x1000
+#define mmMME2_ACC_CTI1_BASE                       0x7FFE126000ull
+#define MME2_ACC_CTI1_MAX_OFFSET                   0x1000
+#define MME2_ACC_CTI1_SECTION                      0x1000
+#define mmMME2_ACC_BMON0_BASE                      0x7FFE127000ull
+#define MME2_ACC_BMON0_MAX_OFFSET                  0x1000
+#define MME2_ACC_BMON0_SECTION                     0x9000
+#define mmMME2_ACC_FUNNEL_BASE                     0x7FFE130000ull
+#define MME2_ACC_FUNNEL_MAX_OFFSET                 0x1000
+#define MME2_ACC_FUNNEL_SECTION                    0x11000
+#define mmMME2_SBAB_STM_BASE                       0x7FFE141000ull
+#define MME2_SBAB_STM_MAX_OFFSET                   0x1000
+#define MME2_SBAB_STM_SECTION                      0x1000
+#define mmMME2_SBAB_CTI_BASE                       0x7FFE142000ull
+#define MME2_SBAB_CTI_MAX_OFFSET                   0x1000
+#define MME2_SBAB_CTI_SECTION                      0x1000
+#define mmMME2_SBAB_ETF_BASE                       0x7FFE143000ull
+#define MME2_SBAB_ETF_MAX_OFFSET                   0x1000
+#define MME2_SBAB_ETF_SECTION                      0x1000
+#define mmMME2_SBAB_SPMU_BASE                      0x7FFE144000ull
+#define MME2_SBAB_SPMU_MAX_OFFSET                  0x1000
+#define MME2_SBAB_SPMU_SECTION                     0x1000
+#define mmMME2_SBAB_CTI0_BASE                      0x7FFE145000ull
+#define MME2_SBAB_CTI0_MAX_OFFSET                  0x1000
+#define MME2_SBAB_CTI0_SECTION                     0x1000
+#define mmMME2_SBAB_CTI1_BASE                      0x7FFE146000ull
+#define MME2_SBAB_CTI1_MAX_OFFSET                  0x1000
+#define MME2_SBAB_CTI1_SECTION                     0x1000
+#define mmMME2_SBAB_BMON0_BASE                     0x7FFE147000ull
+#define MME2_SBAB_BMON0_MAX_OFFSET                 0x1000
+#define MME2_SBAB_BMON0_SECTION                    0x1000
+#define mmMME2_SBAB_BMON1_BASE                     0x7FFE148000ull
+#define MME2_SBAB_BMON1_MAX_OFFSET                 0x1000
+#define MME2_SBAB_BMON1_SECTION                    0x19000
+#define mmMME2_CTRL_STM_BASE                       0x7FFE161000ull
+#define MME2_CTRL_STM_MAX_OFFSET                   0x1000
+#define MME2_CTRL_STM_SECTION                      0x1000
+#define mmMME2_CTRL_CTI_BASE                       0x7FFE162000ull
+#define MME2_CTRL_CTI_MAX_OFFSET                   0x1000
+#define MME2_CTRL_CTI_SECTION                      0x1000
+#define mmMME2_CTRL_ETF_BASE                       0x7FFE163000ull
+#define MME2_CTRL_ETF_MAX_OFFSET                   0x1000
+#define MME2_CTRL_ETF_SECTION                      0x1000
+#define mmMME2_CTRL_SPMU_BASE                      0x7FFE164000ull
+#define MME2_CTRL_SPMU_MAX_OFFSET                  0x1000
+#define MME2_CTRL_SPMU_SECTION                     0x1000
+#define mmMME2_CTRL_CTI0_BASE                      0x7FFE165000ull
+#define MME2_CTRL_CTI0_MAX_OFFSET                  0x1000
+#define MME2_CTRL_CTI0_SECTION                     0x1000
+#define mmMME2_CTRL_CTI1_BASE                      0x7FFE166000ull
+#define MME2_CTRL_CTI1_MAX_OFFSET                  0x1000
+#define MME2_CTRL_CTI1_SECTION                     0x1000
+#define mmMME2_CTRL_BMON0_BASE                     0x7FFE167000ull
+#define MME2_CTRL_BMON0_MAX_OFFSET                 0x1000
+#define MME2_CTRL_BMON0_SECTION                    0x1000
+#define mmMME2_CTRL_BMON1_BASE                     0x7FFE168000ull
+#define MME2_CTRL_BMON1_MAX_OFFSET                 0x1000
+#define MME2_CTRL_BMON1_SECTION                    0x39000
+#define mmMME3_ACC_STM_BASE                        0x7FFE1A1000ull
+#define MME3_ACC_STM_MAX_OFFSET                    0x1000
+#define MME3_ACC_STM_SECTION                       0x1000
+#define mmMME3_ACC_CTI_BASE                        0x7FFE1A2000ull
+#define MME3_ACC_CTI_MAX_OFFSET                    0x1000
+#define MME3_ACC_CTI_SECTION                       0x1000
+#define mmMME3_ACC_ETF_BASE                        0x7FFE1A3000ull
+#define MME3_ACC_ETF_MAX_OFFSET                    0x1000
+#define MME3_ACC_ETF_SECTION                       0x1000
+#define mmMME3_ACC_SPMU_BASE                       0x7FFE1A4000ull
+#define MME3_ACC_SPMU_MAX_OFFSET                   0x1000
+#define MME3_ACC_SPMU_SECTION                      0x1000
+#define mmMME3_ACC_CTI0_BASE                       0x7FFE1A5000ull
+#define MME3_ACC_CTI0_MAX_OFFSET                   0x1000
+#define MME3_ACC_CTI0_SECTION                      0x1000
+#define mmMME3_ACC_CTI1_BASE                       0x7FFE1A6000ull
+#define MME3_ACC_CTI1_MAX_OFFSET                   0x1000
+#define MME3_ACC_CTI1_SECTION                      0x1000
+#define mmMME3_ACC_BMON0_BASE                      0x7FFE1A7000ull
+#define MME3_ACC_BMON0_MAX_OFFSET                  0x1000
+#define MME3_ACC_BMON0_SECTION                     0x9000
+#define mmMME3_ACC_FUNNEL_BASE                     0x7FFE1B0000ull
+#define MME3_ACC_FUNNEL_MAX_OFFSET                 0x1000
+#define MME3_ACC_FUNNEL_SECTION                    0x11000
+#define mmMME3_SBAB_STM_BASE                       0x7FFE1C1000ull
+#define MME3_SBAB_STM_MAX_OFFSET                   0x1000
+#define MME3_SBAB_STM_SECTION                      0x1000
+#define mmMME3_SBAB_CTI_BASE                       0x7FFE1C2000ull
+#define MME3_SBAB_CTI_MAX_OFFSET                   0x1000
+#define MME3_SBAB_CTI_SECTION                      0x1000
+#define mmMME3_SBAB_ETF_BASE                       0x7FFE1C3000ull
+#define MME3_SBAB_ETF_MAX_OFFSET                   0x1000
+#define MME3_SBAB_ETF_SECTION                      0x1000
+#define mmMME3_SBAB_SPMU_BASE                      0x7FFE1C4000ull
+#define MME3_SBAB_SPMU_MAX_OFFSET                  0x1000
+#define MME3_SBAB_SPMU_SECTION                     0x1000
+#define mmMME3_SBAB_CTI0_BASE                      0x7FFE1C5000ull
+#define MME3_SBAB_CTI0_MAX_OFFSET                  0x1000
+#define MME3_SBAB_CTI0_SECTION                     0x1000
+#define mmMME3_SBAB_CTI1_BASE                      0x7FFE1C6000ull
+#define MME3_SBAB_CTI1_MAX_OFFSET                  0x1000
+#define MME3_SBAB_CTI1_SECTION                     0x1000
+#define mmMME3_SBAB_BMON0_BASE                     0x7FFE1C7000ull
+#define MME3_SBAB_BMON0_MAX_OFFSET                 0x1000
+#define MME3_SBAB_BMON0_SECTION                    0x1000
+#define mmMME3_SBAB_BMON1_BASE                     0x7FFE1C8000ull
+#define MME3_SBAB_BMON1_MAX_OFFSET                 0x1000
+#define MME3_SBAB_BMON1_SECTION                    0x19000
+#define mmMME3_CTRL_STM_BASE                       0x7FFE1E1000ull
+#define MME3_CTRL_STM_MAX_OFFSET                   0x1000
+#define MME3_CTRL_STM_SECTION                      0x1000
+#define mmMME3_CTRL_CTI_BASE                       0x7FFE1E2000ull
+#define MME3_CTRL_CTI_MAX_OFFSET                   0x1000
+#define MME3_CTRL_CTI_SECTION                      0x1000
+#define mmMME3_CTRL_ETF_BASE                       0x7FFE1E3000ull
+#define MME3_CTRL_ETF_MAX_OFFSET                   0x1000
+#define MME3_CTRL_ETF_SECTION                      0x1000
+#define mmMME3_CTRL_SPMU_BASE                      0x7FFE1E4000ull
+#define MME3_CTRL_SPMU_MAX_OFFSET                  0x1000
+#define MME3_CTRL_SPMU_SECTION                     0x1000
+#define mmMME3_CTRL_CTI0_BASE                      0x7FFE1E5000ull
+#define MME3_CTRL_CTI0_MAX_OFFSET                  0x1000
+#define MME3_CTRL_CTI0_SECTION                     0x1000
+#define mmMME3_CTRL_CTI1_BASE                      0x7FFE1E6000ull
+#define MME3_CTRL_CTI1_MAX_OFFSET                  0x1000
+#define MME3_CTRL_CTI1_SECTION                     0x1000
+#define mmMME3_CTRL_BMON0_BASE                     0x7FFE1E7000ull
+#define MME3_CTRL_BMON0_MAX_OFFSET                 0x1000
+#define MME3_CTRL_BMON0_SECTION                    0x1000
+#define mmMME3_CTRL_BMON1_BASE                     0x7FFE1E8000ull
+#define MME3_CTRL_BMON1_MAX_OFFSET                 0x1000
+#define MME3_CTRL_BMON1_SECTION                    0x18000
+#define mmIC_ROM_TABLE_BASE                        0x7FFE200000ull
+#define IC_ROM_TABLE_MAX_OFFSET                    0x1000
+#define IC_ROM_TABLE_SECTION                       0x1000
+#define mmSRAM_Y0_X0_FUNNEL_BASE                   0x7FFE201000ull
+#define SRAM_Y0_X0_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X0_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y0_X1_FUNNEL_BASE                   0x7FFE209000ull
+#define SRAM_Y0_X1_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X1_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y0_X2_FUNNEL_BASE                   0x7FFE211000ull
+#define SRAM_Y0_X2_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X2_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y0_X3_FUNNEL_BASE                   0x7FFE219000ull
+#define SRAM_Y0_X3_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X3_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y0_X4_FUNNEL_BASE                   0x7FFE221000ull
+#define SRAM_Y0_X4_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X4_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y0_X5_FUNNEL_BASE                   0x7FFE229000ull
+#define SRAM_Y0_X5_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X5_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y0_X6_FUNNEL_BASE                   0x7FFE231000ull
+#define SRAM_Y0_X6_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X6_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y0_X7_FUNNEL_BASE                   0x7FFE239000ull
+#define SRAM_Y0_X7_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y0_X7_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X0_FUNNEL_BASE                   0x7FFE241000ull
+#define SRAM_Y1_X0_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X0_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X1_FUNNEL_BASE                   0x7FFE249000ull
+#define SRAM_Y1_X1_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X1_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X2_FUNNEL_BASE                   0x7FFE251000ull
+#define SRAM_Y1_X2_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X2_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X3_FUNNEL_BASE                   0x7FFE259000ull
+#define SRAM_Y1_X3_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X3_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X4_FUNNEL_BASE                   0x7FFE261000ull
+#define SRAM_Y1_X4_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X4_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X5_FUNNEL_BASE                   0x7FFE269000ull
+#define SRAM_Y1_X5_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X5_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X6_FUNNEL_BASE                   0x7FFE271000ull
+#define SRAM_Y1_X6_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X6_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y1_X7_FUNNEL_BASE                   0x7FFE279000ull
+#define SRAM_Y1_X7_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y1_X7_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X0_FUNNEL_BASE                   0x7FFE281000ull
+#define SRAM_Y2_X0_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X0_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X1_FUNNEL_BASE                   0x7FFE289000ull
+#define SRAM_Y2_X1_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X1_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X2_FUNNEL_BASE                   0x7FFE291000ull
+#define SRAM_Y2_X2_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X2_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X3_FUNNEL_BASE                   0x7FFE299000ull
+#define SRAM_Y2_X3_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X3_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X4_FUNNEL_BASE                   0x7FFE2A1000ull
+#define SRAM_Y2_X4_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X4_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X5_FUNNEL_BASE                   0x7FFE2A9000ull
+#define SRAM_Y2_X5_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X5_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X6_FUNNEL_BASE                   0x7FFE2B1000ull
+#define SRAM_Y2_X6_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X6_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y2_X7_FUNNEL_BASE                   0x7FFE2B9000ull
+#define SRAM_Y2_X7_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y2_X7_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X0_FUNNEL_BASE                   0x7FFE2C1000ull
+#define SRAM_Y3_X0_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X0_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X1_FUNNEL_BASE                   0x7FFE2C9000ull
+#define SRAM_Y3_X1_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X1_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X2_FUNNEL_BASE                   0x7FFE2D1000ull
+#define SRAM_Y3_X2_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X2_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X4_FUNNEL_BASE                   0x7FFE2D9000ull
+#define SRAM_Y3_X4_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X4_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X3_FUNNEL_BASE                   0x7FFE2E1000ull
+#define SRAM_Y3_X3_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X3_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X5_FUNNEL_BASE                   0x7FFE2E9000ull
+#define SRAM_Y3_X5_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X5_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X6_FUNNEL_BASE                   0x7FFE2F1000ull
+#define SRAM_Y3_X6_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X6_FUNNEL_SECTION                  0x8000
+#define mmSRAM_Y3_X7_FUNNEL_BASE                   0x7FFE2F9000ull
+#define SRAM_Y3_X7_FUNNEL_MAX_OFFSET               0x1000
+#define SRAM_Y3_X7_FUNNEL_SECTION                  0x7000
+#define mmIF_ROM_TABLE_BASE                        0x7FFE300000ull
+#define IF_ROM_TABLE_MAX_OFFSET                    0x1000
+#define IF_ROM_TABLE_SECTION                       0x1000
+#define mmSIF_FUNNEL_0_BASE                        0x7FFE301000ull
+#define SIF_FUNNEL_0_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_0_SECTION                       0x10000
+#define mmSIF_FUNNEL_1_BASE                        0x7FFE311000ull
+#define SIF_FUNNEL_1_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_1_SECTION                       0x10000
+#define mmSIF_FUNNEL_2_BASE                        0x7FFE321000ull
+#define SIF_FUNNEL_2_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_2_SECTION                       0x10000
+#define mmSIF_FUNNEL_3_BASE                        0x7FFE331000ull
+#define SIF_FUNNEL_3_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_3_SECTION                       0x10000
+#define mmSIF_FUNNEL_4_BASE                        0x7FFE341000ull
+#define SIF_FUNNEL_4_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_4_SECTION                       0x10000
+#define mmSIF_FUNNEL_5_BASE                        0x7FFE351000ull
+#define SIF_FUNNEL_5_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_5_SECTION                       0x10000
+#define mmSIF_FUNNEL_6_BASE                        0x7FFE361000ull
+#define SIF_FUNNEL_6_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_6_SECTION                       0x10000
+#define mmSIF_FUNNEL_7_BASE                        0x7FFE371000ull
+#define SIF_FUNNEL_7_MAX_OFFSET                    0x1000
+#define SIF_FUNNEL_7_SECTION                       0x10000
+#define mmNIF_FUNNEL_0_BASE                        0x7FFE381000ull
+#define NIF_FUNNEL_0_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_0_SECTION                       0x10000
+#define mmNIF_FUNNEL_1_BASE                        0x7FFE391000ull
+#define NIF_FUNNEL_1_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_1_SECTION                       0x10000
+#define mmNIF_FUNNEL_2_BASE                        0x7FFE3A1000ull
+#define NIF_FUNNEL_2_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_2_SECTION                       0x10000
+#define mmNIF_FUNNEL_3_BASE                        0x7FFE3B1000ull
+#define NIF_FUNNEL_3_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_3_SECTION                       0x10000
+#define mmNIF_FUNNEL_4_BASE                        0x7FFE3C1000ull
+#define NIF_FUNNEL_4_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_4_SECTION                       0x10000
+#define mmNIF_FUNNEL_5_BASE                        0x7FFE3D1000ull
+#define NIF_FUNNEL_5_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_5_SECTION                       0x10000
+#define mmNIF_FUNNEL_6_BASE                        0x7FFE3E1000ull
+#define NIF_FUNNEL_6_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_6_SECTION                       0x10000
+#define mmNIF_FUNNEL_7_BASE                        0x7FFE3F1000ull
+#define NIF_FUNNEL_7_MAX_OFFSET                    0x1000
+#define NIF_FUNNEL_7_SECTION                       0xF000
+#define mmDMA_IF_ROM_TABLE_BASE                    0x7FFE400000ull
+#define DMA_IF_ROM_TABLE_MAX_OFFSET                0x1000
+#define DMA_IF_ROM_TABLE_SECTION                   0x1000
+#define mmDMA_IF_W_S_STM_BASE                      0x7FFE401000ull
+#define DMA_IF_W_S_STM_MAX_OFFSET                  0x1000
+#define DMA_IF_W_S_STM_SECTION                     0x1000
+#define mmDMA_IF_W_S_CTI_BASE                      0x7FFE402000ull
+#define DMA_IF_W_S_CTI_MAX_OFFSET                  0x1000
+#define DMA_IF_W_S_CTI_SECTION                     0x1000
+#define mmDMA_IF_W_S_ETF_BASE                      0x7FFE403000ull
+#define DMA_IF_W_S_ETF_MAX_OFFSET                  0x1000
+#define DMA_IF_W_S_ETF_SECTION                     0x2000
+#define mmDMA_IF_W_S_BMON0_CTI_BASE                0x7FFE405000ull
+#define DMA_IF_W_S_BMON0_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_W_S_BMON0_CTI_SECTION               0x1000
+#define mmDMA_IF_W_S_BMON1_CTI_BASE                0x7FFE406000ull
+#define DMA_IF_W_S_BMON1_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_W_S_BMON1_CTI_SECTION               0x1000
+#define mmDMA_IF_W_S_HBM0_WR_BMON_BASE             0x7FFE407000ull
+#define DMA_IF_W_S_HBM0_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_S_HBM0_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_W_S_HBM0_RD_BMON_BASE             0x7FFE408000ull
+#define DMA_IF_W_S_HBM0_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_S_HBM0_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_W_S_HBM1_WR_BMON_BASE             0x7FFE409000ull
+#define DMA_IF_W_S_HBM1_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_S_HBM1_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_W_S_HBM1_RD_BMON_BASE             0x7FFE40A000ull
+#define DMA_IF_W_S_HBM1_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_S_HBM1_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_W_S_SOB_WR_BMON_BASE              0x7FFE40B000ull
+#define DMA_IF_W_S_SOB_WR_BMON_MAX_OFFSET          0x1000
+#define DMA_IF_W_S_SOB_WR_BMON_SECTION             0x4000
+#define mmDMA_IF_W_S_FUNNEL_BASE                   0x7FFE40F000ull
+#define DMA_IF_W_S_FUNNEL_MAX_OFFSET               0x1000
+#define DMA_IF_W_S_FUNNEL_SECTION                  0x12000
+#define mmDMA_IF_E_S_STM_BASE                      0x7FFE421000ull
+#define DMA_IF_E_S_STM_MAX_OFFSET                  0x1000
+#define DMA_IF_E_S_STM_SECTION                     0x1000
+#define mmDMA_IF_E_S_CTI_BASE                      0x7FFE422000ull
+#define DMA_IF_E_S_CTI_MAX_OFFSET                  0x1000
+#define DMA_IF_E_S_CTI_SECTION                     0x1000
+#define mmDMA_IF_E_S_ETF_BASE                      0x7FFE423000ull
+#define DMA_IF_E_S_ETF_MAX_OFFSET                  0x1000
+#define DMA_IF_E_S_ETF_SECTION                     0x2000
+#define mmDMA_IF_E_S_BMON0_CTI_BASE                0x7FFE425000ull
+#define DMA_IF_E_S_BMON0_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_E_S_BMON0_CTI_SECTION               0x1000
+#define mmDMA_IF_E_S_BMON1_CTI_BASE                0x7FFE426000ull
+#define DMA_IF_E_S_BMON1_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_E_S_BMON1_CTI_SECTION               0x1000
+#define mmDMA_IF_E_S_HBM0_WR_BMON_BASE             0x7FFE427000ull
+#define DMA_IF_E_S_HBM0_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_S_HBM0_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_E_S_HBM0_RD_BMON_BASE             0x7FFE428000ull
+#define DMA_IF_E_S_HBM0_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_S_HBM0_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_E_S_HBM1_WR_BMON_BASE             0x7FFE429000ull
+#define DMA_IF_E_S_HBM1_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_S_HBM1_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_E_S_HBM1_RD_BMON_BASE             0x7FFE42A000ull
+#define DMA_IF_E_S_HBM1_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_S_HBM1_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_E_S_SOB_WR_BMON_BASE              0x7FFE42B000ull
+#define DMA_IF_E_S_SOB_WR_BMON_MAX_OFFSET          0x1000
+#define DMA_IF_E_S_SOB_WR_BMON_SECTION             0x4000
+#define mmDMA_IF_E_S_FUNNEL_BASE                   0x7FFE42F000ull
+#define DMA_IF_E_S_FUNNEL_MAX_OFFSET               0x1000
+#define DMA_IF_E_S_FUNNEL_SECTION                  0x12000
+#define mmDMA_IF_W_N_STM_BASE                      0x7FFE441000ull
+#define DMA_IF_W_N_STM_MAX_OFFSET                  0x1000
+#define DMA_IF_W_N_STM_SECTION                     0x1000
+#define mmDMA_IF_W_N_CTI_BASE                      0x7FFE442000ull
+#define DMA_IF_W_N_CTI_MAX_OFFSET                  0x1000
+#define DMA_IF_W_N_CTI_SECTION                     0x1000
+#define mmDMA_IF_W_N_ETF_BASE                      0x7FFE443000ull
+#define DMA_IF_W_N_ETF_MAX_OFFSET                  0x1000
+#define DMA_IF_W_N_ETF_SECTION                     0x2000
+#define mmDMA_IF_W_N_BMON0_CTI_BASE                0x7FFE445000ull
+#define DMA_IF_W_N_BMON0_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_W_N_BMON0_CTI_SECTION               0x1000
+#define mmDMA_IF_W_N_BMON1_CTI_BASE                0x7FFE446000ull
+#define DMA_IF_W_N_BMON1_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_W_N_BMON1_CTI_SECTION               0x1000
+#define mmDMA_IF_W_N_HBM0_WR_BMON_BASE             0x7FFE447000ull
+#define DMA_IF_W_N_HBM0_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_N_HBM0_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_W_N_HBM0_RD_BMON_BASE             0x7FFE448000ull
+#define DMA_IF_W_N_HBM0_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_N_HBM0_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_W_N_HBM1_WR_BMON_BASE             0x7FFE449000ull
+#define DMA_IF_W_N_HBM1_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_N_HBM1_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_W_N_HBM1_RD_BMON_BASE             0x7FFE44A000ull
+#define DMA_IF_W_N_HBM1_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_W_N_HBM1_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_W_N_SOB_WR_BMON_BASE              0x7FFE44B000ull
+#define DMA_IF_W_N_SOB_WR_BMON_MAX_OFFSET          0x1000
+#define DMA_IF_W_N_SOB_WR_BMON_SECTION             0x4000
+#define mmDMA_IF_W_N_FUNNEL_BASE                   0x7FFE44F000ull
+#define DMA_IF_W_N_FUNNEL_MAX_OFFSET               0x1000
+#define DMA_IF_W_N_FUNNEL_SECTION                  0x12000
+#define mmDMA_IF_E_N_STM_BASE                      0x7FFE461000ull
+#define DMA_IF_E_N_STM_MAX_OFFSET                  0x1000
+#define DMA_IF_E_N_STM_SECTION                     0x1000
+#define mmDMA_IF_E_N_CTI_BASE                      0x7FFE462000ull
+#define DMA_IF_E_N_CTI_MAX_OFFSET                  0x1000
+#define DMA_IF_E_N_CTI_SECTION                     0x1000
+#define mmDMA_IF_E_N_ETF_BASE                      0x7FFE463000ull
+#define DMA_IF_E_N_ETF_MAX_OFFSET                  0x1000
+#define DMA_IF_E_N_ETF_SECTION                     0x2000
+#define mmDMA_IF_E_N_BMON0_CTI_BASE                0x7FFE465000ull
+#define DMA_IF_E_N_BMON0_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_E_N_BMON0_CTI_SECTION               0x1000
+#define mmDMA_IF_E_N_BMON1_CTI_BASE                0x7FFE466000ull
+#define DMA_IF_E_N_BMON1_CTI_MAX_OFFSET            0x1000
+#define DMA_IF_E_N_BMON1_CTI_SECTION               0x1000
+#define mmDMA_IF_E_N_HBM0_WR_BMON_BASE             0x7FFE467000ull
+#define DMA_IF_E_N_HBM0_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_N_HBM0_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_E_N_HBM0_RD_BMON_BASE             0x7FFE468000ull
+#define DMA_IF_E_N_HBM0_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_N_HBM0_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_E_N_HBM1_WR_BMON_BASE             0x7FFE469000ull
+#define DMA_IF_E_N_HBM1_WR_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_N_HBM1_WR_BMON_SECTION            0x1000
+#define mmDMA_IF_E_N_HBM1_RD_BMON_BASE             0x7FFE46A000ull
+#define DMA_IF_E_N_HBM1_RD_BMON_MAX_OFFSET         0x1000
+#define DMA_IF_E_N_HBM1_RD_BMON_SECTION            0x1000
+#define mmDMA_IF_E_N_SOB_WR_BMON_BASE              0x7FFE46B000ull
+#define DMA_IF_E_N_SOB_WR_BMON_MAX_OFFSET          0x1000
+#define DMA_IF_E_N_SOB_WR_BMON_SECTION             0x4000
+#define mmDMA_IF_E_N_FUNNEL_BASE                   0x7FFE46F000ull
+#define DMA_IF_E_N_FUNNEL_MAX_OFFSET               0x1000
+#define DMA_IF_E_N_FUNNEL_SECTION                  0x11000
+#define mmCPU_ROM_TABLE_BASE                       0x7FFE480000ull
+#define CPU_ROM_TABLE_MAX_OFFSET                   0x1000
+#define CPU_ROM_TABLE_SECTION                      0x1000
+#define mmCPU_ETF_0_BASE                           0x7FFE481000ull
+#define CPU_ETF_0_MAX_OFFSET                       0x1000
+#define CPU_ETF_0_SECTION                          0x1000
+#define mmCPU_ETF_1_BASE                           0x7FFE482000ull
+#define CPU_ETF_1_MAX_OFFSET                       0x1000
+#define CPU_ETF_1_SECTION                          0x2000
+#define mmCPU_CTI_BASE                             0x7FFE484000ull
+#define CPU_CTI_MAX_OFFSET                         0x1000
+#define CPU_CTI_SECTION                            0x1000
+#define mmCPU_FUNNEL_BASE                          0x7FFE485000ull
+#define CPU_FUNNEL_MAX_OFFSET                      0x1000
+#define CPU_FUNNEL_SECTION                         0x1000
+#define mmCPU_STM_BASE                             0x7FFE486000ull
+#define CPU_STM_MAX_OFFSET                         0x1000
+#define CPU_STM_SECTION                            0x1000
+#define mmCPU_CTI_TRACE_BASE                       0x7FFE487000ull
+#define CPU_CTI_TRACE_MAX_OFFSET                   0x1000
+#define CPU_CTI_TRACE_SECTION                      0x1000
+#define mmCPU_ETF_TRACE_BASE                       0x7FFE488000ull
+#define CPU_ETF_TRACE_MAX_OFFSET                   0x1000
+#define CPU_ETF_TRACE_SECTION                      0x1000
+#define mmCPU_WR_BMON_BASE                         0x7FFE489000ull
+#define CPU_WR_BMON_MAX_OFFSET                     0x1000
+#define CPU_WR_BMON_SECTION                        0x1000
+#define mmCPU_RD_BMON_BASE                         0x7FFE48A000ull
+#define CPU_RD_BMON_MAX_OFFSET                     0x1000
+#define CPU_RD_BMON_SECTION                        0x76000
+#define mmDMA_ROM_TABLE_BASE                       0x7FFE500000ull
+#define DMA_ROM_TABLE_MAX_OFFSET                   0x1000
+#define DMA_ROM_TABLE_SECTION                      0x1000
+#define mmDMA_CH_0_CS_STM_BASE                     0x7FFE501000ull
+#define DMA_CH_0_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_0_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_0_CS_CTI_BASE                     0x7FFE502000ull
+#define DMA_CH_0_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_0_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_0_CS_ETF_BASE                     0x7FFE503000ull
+#define DMA_CH_0_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_0_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_0_CS_SPMU_BASE                    0x7FFE504000ull
+#define DMA_CH_0_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_0_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_0_BMON_CTI_BASE                   0x7FFE505000ull
+#define DMA_CH_0_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_0_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_0_USER_CTI_BASE                   0x7FFE506000ull
+#define DMA_CH_0_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_0_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_0_BMON_0_BASE                     0x7FFE507000ull
+#define DMA_CH_0_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_0_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_0_BMON_1_BASE                     0x7FFE508000ull
+#define DMA_CH_0_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_0_BMON_1_SECTION                    0x19000
+#define mmDMA_CH_1_CS_STM_BASE                     0x7FFE521000ull
+#define DMA_CH_1_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_1_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_1_CS_CTI_BASE                     0x7FFE522000ull
+#define DMA_CH_1_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_1_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_1_CS_ETF_BASE                     0x7FFE523000ull
+#define DMA_CH_1_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_1_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_1_CS_SPMU_BASE                    0x7FFE524000ull
+#define DMA_CH_1_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_1_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_1_BMON_CTI_BASE                   0x7FFE525000ull
+#define DMA_CH_1_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_1_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_1_USER_CTI_BASE                   0x7FFE526000ull
+#define DMA_CH_1_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_1_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_1_BMON_0_BASE                     0x7FFE527000ull
+#define DMA_CH_1_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_1_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_1_BMON_1_BASE                     0x7FFE528000ull
+#define DMA_CH_1_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_1_BMON_1_SECTION                    0x19000
+#define mmDMA_CH_2_CS_STM_BASE                     0x7FFE541000ull
+#define DMA_CH_2_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_2_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_2_CS_CTI_BASE                     0x7FFE542000ull
+#define DMA_CH_2_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_2_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_2_CS_ETF_BASE                     0x7FFE543000ull
+#define DMA_CH_2_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_2_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_2_CS_SPMU_BASE                    0x7FFE544000ull
+#define DMA_CH_2_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_2_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_2_BMON_CTI_BASE                   0x7FFE545000ull
+#define DMA_CH_2_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_2_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_2_USER_CTI_BASE                   0x7FFE546000ull
+#define DMA_CH_2_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_2_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_2_BMON_0_BASE                     0x7FFE547000ull
+#define DMA_CH_2_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_2_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_2_BMON_1_BASE                     0x7FFE548000ull
+#define DMA_CH_2_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_2_BMON_1_SECTION                    0x19000
+#define mmDMA_CH_3_CS_STM_BASE                     0x7FFE561000ull
+#define DMA_CH_3_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_3_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_3_CS_CTI_BASE                     0x7FFE562000ull
+#define DMA_CH_3_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_3_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_3_CS_ETF_BASE                     0x7FFE563000ull
+#define DMA_CH_3_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_3_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_3_CS_SPMU_BASE                    0x7FFE564000ull
+#define DMA_CH_3_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_3_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_3_BMON_CTI_BASE                   0x7FFE565000ull
+#define DMA_CH_3_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_3_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_3_USER_CTI_BASE                   0x7FFE566000ull
+#define DMA_CH_3_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_3_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_3_BMON_0_BASE                     0x7FFE567000ull
+#define DMA_CH_3_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_3_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_3_BMON_1_BASE                     0x7FFE568000ull
+#define DMA_CH_3_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_3_BMON_1_SECTION                    0x19000
+#define mmDMA_CH_4_CS_STM_BASE                     0x7FFE581000ull
+#define DMA_CH_4_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_4_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_4_CS_CTI_BASE                     0x7FFE582000ull
+#define DMA_CH_4_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_4_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_4_CS_ETF_BASE                     0x7FFE583000ull
+#define DMA_CH_4_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_4_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_4_CS_SPMU_BASE                    0x7FFE584000ull
+#define DMA_CH_4_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_4_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_4_BMON_CTI_BASE                   0x7FFE585000ull
+#define DMA_CH_4_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_4_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_4_USER_CTI_BASE                   0x7FFE586000ull
+#define DMA_CH_4_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_4_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_4_BMON_0_BASE                     0x7FFE587000ull
+#define DMA_CH_4_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_4_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_4_BMON_1_BASE                     0x7FFE588000ull
+#define DMA_CH_4_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_4_BMON_1_SECTION                    0x19000
+#define mmDMA_CH_5_CS_STM_BASE                     0x7FFE5A1000ull
+#define DMA_CH_5_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_5_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_5_CS_CTI_BASE                     0x7FFE5A2000ull
+#define DMA_CH_5_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_5_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_5_CS_ETF_BASE                     0x7FFE5A3000ull
+#define DMA_CH_5_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_5_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_5_CS_SPMU_BASE                    0x7FFE5A4000ull
+#define DMA_CH_5_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_5_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_5_BMON_CTI_BASE                   0x7FFE5A5000ull
+#define DMA_CH_5_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_5_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_5_USER_CTI_BASE                   0x7FFE5A6000ull
+#define DMA_CH_5_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_5_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_5_BMON_0_BASE                     0x7FFE5A7000ull
+#define DMA_CH_5_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_5_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_5_BMON_1_BASE                     0x7FFE5A8000ull
+#define DMA_CH_5_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_5_BMON_1_SECTION                    0x19000
+#define mmDMA_CH_6_CS_STM_BASE                     0x7FFE5C1000ull
+#define DMA_CH_6_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_6_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_6_CS_CTI_BASE                     0x7FFE5C2000ull
+#define DMA_CH_6_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_6_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_6_CS_ETF_BASE                     0x7FFE5C3000ull
+#define DMA_CH_6_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_6_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_6_CS_SPMU_BASE                    0x7FFE5C4000ull
+#define DMA_CH_6_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_6_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_6_BMON_CTI_BASE                   0x7FFE5C5000ull
+#define DMA_CH_6_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_6_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_6_USER_CTI_BASE                   0x7FFE5C6000ull
+#define DMA_CH_6_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_6_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_6_BMON_0_BASE                     0x7FFE5C7000ull
+#define DMA_CH_6_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_6_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_6_BMON_1_BASE                     0x7FFE5C8000ull
+#define DMA_CH_6_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_6_BMON_1_SECTION                    0x19000
+#define mmDMA_CH_7_CS_STM_BASE                     0x7FFE5E1000ull
+#define DMA_CH_7_CS_STM_MAX_OFFSET                 0x1000
+#define DMA_CH_7_CS_STM_SECTION                    0x1000
+#define mmDMA_CH_7_CS_CTI_BASE                     0x7FFE5E2000ull
+#define DMA_CH_7_CS_CTI_MAX_OFFSET                 0x1000
+#define DMA_CH_7_CS_CTI_SECTION                    0x1000
+#define mmDMA_CH_7_CS_ETF_BASE                     0x7FFE5E3000ull
+#define DMA_CH_7_CS_ETF_MAX_OFFSET                 0x1000
+#define DMA_CH_7_CS_ETF_SECTION                    0x1000
+#define mmDMA_CH_7_CS_SPMU_BASE                    0x7FFE5E4000ull
+#define DMA_CH_7_CS_SPMU_MAX_OFFSET                0x1000
+#define DMA_CH_7_CS_SPMU_SECTION                   0x1000
+#define mmDMA_CH_7_BMON_CTI_BASE                   0x7FFE5E5000ull
+#define DMA_CH_7_BMON_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_7_BMON_CTI_SECTION                  0x1000
+#define mmDMA_CH_7_USER_CTI_BASE                   0x7FFE5E6000ull
+#define DMA_CH_7_USER_CTI_MAX_OFFSET               0x1000
+#define DMA_CH_7_USER_CTI_SECTION                  0x1000
+#define mmDMA_CH_7_BMON_0_BASE                     0x7FFE5E7000ull
+#define DMA_CH_7_BMON_0_MAX_OFFSET                 0x1000
+#define DMA_CH_7_BMON_0_SECTION                    0x1000
+#define mmDMA_CH_7_BMON_1_BASE                     0x7FFE5E8000ull
+#define DMA_CH_7_BMON_1_MAX_OFFSET                 0x1000
+#define DMA_CH_7_BMON_1_SECTION                    0x18000
+#define mmNIC_TPC_FUNNEL_W_S_BASE                  0x7FFE600000ull
+#define NIC_TPC_FUNNEL_W_S_MAX_OFFSET              0x1000
+#define NIC_TPC_FUNNEL_W_S_SECTION                 0x80000
+#define mmNIC_TPC_FUNNEL_E_S_BASE                  0x7FFE680000ull
+#define NIC_TPC_FUNNEL_E_S_MAX_OFFSET              0x1000
+#define NIC_TPC_FUNNEL_E_S_SECTION                 0x80000
+#define mmNIC_TPC_FUNNEL_W_N_BASE                  0x7FFE700000ull
+#define NIC_TPC_FUNNEL_W_N_MAX_OFFSET              0x1000
+#define NIC_TPC_FUNNEL_W_N_SECTION                 0x80000
+#define mmNIC_TPC_FUNNEL_E_N_BASE                  0x7FFE780000ull
+#define NIC_TPC_FUNNEL_E_N_MAX_OFFSET              0x1000
+#define NIC_TPC_FUNNEL_E_N_SECTION                 0x80000
+#define mmCA53_BASE                                0x7FFE800000ull
+#define CA53_MAX_OFFSET                            0x141000
+#define CA53_SECTION                               0x400000
+#define mmPCI_ROM_TABLE_BASE                       0x7FFEC00000ull
+#define PCI_ROM_TABLE_MAX_OFFSET                   0x1000
+#define PCI_ROM_TABLE_SECTION                      0x1000
+#define mmPCIE_STM_BASE                            0x7FFEC01000ull
+#define PCIE_STM_MAX_OFFSET                        0x1000
+#define PCIE_STM_SECTION                           0x1000
+#define mmPCIE_ETF_BASE                            0x7FFEC02000ull
+#define PCIE_ETF_MAX_OFFSET                        0x1000
+#define PCIE_ETF_SECTION                           0x1000
+#define mmPCIE_CTI_0_BASE                          0x7FFEC03000ull
+#define PCIE_CTI_0_MAX_OFFSET                      0x1000
+#define PCIE_CTI_0_SECTION                         0x1000
+#define mmPCIE_SPMU_BASE                           0x7FFEC04000ull
+#define PCIE_SPMU_MAX_OFFSET                       0x1000
+#define PCIE_SPMU_SECTION                          0x1000
+#define mmPCIE_CTI_1_BASE                          0x7FFEC05000ull
+#define PCIE_CTI_1_MAX_OFFSET                      0x1000
+#define PCIE_CTI_1_SECTION                         0x1000
+#define mmPCIE_FUNNEL_BASE                         0x7FFEC06000ull
+#define PCIE_FUNNEL_MAX_OFFSET                     0x1000
+#define PCIE_FUNNEL_SECTION                        0x1000
+#define mmPCIE_BMON_MSTR_WR_BASE                   0x7FFEC07000ull
+#define PCIE_BMON_MSTR_WR_MAX_OFFSET               0x1000
+#define PCIE_BMON_MSTR_WR_SECTION                  0x1000
+#define mmPCIE_BMON_MSTR_RD_BASE                   0x7FFEC08000ull
+#define PCIE_BMON_MSTR_RD_MAX_OFFSET               0x1000
+#define PCIE_BMON_MSTR_RD_SECTION                  0x1000
+#define mmPCIE_BMON_SLV_WR_BASE                    0x7FFEC09000ull
+#define PCIE_BMON_SLV_WR_MAX_OFFSET                0x1000
+#define PCIE_BMON_SLV_WR_SECTION                   0x1000
+#define mmPCIE_BMON_SLV_RD_BASE                    0x7FFEC0A000ull
+#define PCIE_BMON_SLV_RD_MAX_OFFSET                0x1000
+#define PCIE_BMON_SLV_RD_SECTION                   0x7000
+#define mmMMU_CS_STM_BASE                          0x7FFEC11000ull
+#define MMU_CS_STM_MAX_OFFSET                      0x1000
+#define MMU_CS_STM_SECTION                         0x1000
+#define mmMMU_CS_CTI_BASE                          0x7FFEC12000ull
+#define MMU_CS_CTI_MAX_OFFSET                      0x1000
+#define MMU_CS_CTI_SECTION                         0x1000
+#define mmMMU_CS_ETF_BASE                          0x7FFEC13000ull
+#define MMU_CS_ETF_MAX_OFFSET                      0x1000
+#define MMU_CS_ETF_SECTION                         0x1000
+#define mmMMU_CS_SPMU_BASE                         0x7FFEC14000ull
+#define MMU_CS_SPMU_MAX_OFFSET                     0x1000
+#define MMU_CS_SPMU_SECTION                        0x1000
+#define mmMMU_BMON_CTI_BASE                        0x7FFEC15000ull
+#define MMU_BMON_CTI_MAX_OFFSET                    0x1000
+#define MMU_BMON_CTI_SECTION                       0x1000
+#define mmMMU_USER_CTI_BASE                        0x7FFEC16000ull
+#define MMU_USER_CTI_MAX_OFFSET                    0x1000
+#define MMU_USER_CTI_SECTION                       0x1000
+#define mmMMU_BMON_0_BASE                          0x7FFEC17000ull
+#define MMU_BMON_0_MAX_OFFSET                      0x1000
+#define MMU_BMON_0_SECTION                         0x1000
+#define mmMMU_BMON_1_BASE                          0x7FFEC18000ull
+#define MMU_BMON_1_MAX_OFFSET                      0x1000
+#define MMU_BMON_1_SECTION                         0x28000
+#define mmPSOC_CTI_BASE                            0x7FFEC40000ull
+#define PSOC_CTI_MAX_OFFSET                        0x1000
+#define PSOC_CTI_SECTION                           0x1000
+#define mmPSOC_STM_BASE                            0x7FFEC41000ull
+#define PSOC_STM_MAX_OFFSET                        0x1000
+#define PSOC_STM_SECTION                           0x1000
+#define mmPSOC_FUNNEL_BASE                         0x7FFEC42000ull
+#define PSOC_FUNNEL_MAX_OFFSET                     0x1000
+#define PSOC_FUNNEL_SECTION                        0x1000
+#define mmPSOC_ETR_BASE                            0x7FFEC43000ull
+#define PSOC_ETR_MAX_OFFSET                        0x1000
+#define PSOC_ETR_SECTION                           0x1000
+#define mmPSOC_ETF_BASE                            0x7FFEC44000ull
+#define PSOC_ETF_MAX_OFFSET                        0x1000
+#define PSOC_ETF_SECTION                           0x1000
+#define mmPSOC_TS_CTI_BASE                         0x7FFEC45000ull
+#define PSOC_TS_CTI_MAX_OFFSET                     0x1000
+#define PSOC_TS_CTI_SECTION                        0xB000
+#define mmTOP_ROM_TABLE_BASE                       0x7FFEC50000ull
+#define TOP_ROM_TABLE_MAX_OFFSET                   0x1000
+#define TOP_ROM_TABLE_SECTION                      0x70000
+#define mmNIC0_ROM_TABLE_BASE                      0x7FFECC0000ull
+#define NIC0_ROM_TABLE_MAX_OFFSET                  0x1000
+#define NIC0_ROM_TABLE_SECTION                     0x1000
+#define mmSTM_0_NIC0_DBG_BASE                      0x7FFECC1000ull
+#define STM_0_NIC0_DBG_MAX_OFFSET                  0x21000
+#define STM_0_NIC0_DBG_SECTION                     0x1000
+#define mmCTI_0_NIC0_DBG_BASE                      0x7FFECC2000ull
+#define CTI_0_NIC0_DBG_MAX_OFFSET                  0x1000
+#define CTI_0_NIC0_DBG_SECTION                     0x1000
+#define mmETF_0_NIC0_DBG_BASE                      0x7FFECC3000ull
+#define ETF_0_NIC0_DBG_MAX_OFFSET                  0x1000
+#define ETF_0_NIC0_DBG_SECTION                     0x1000
+#define mmSPMU_0_NIC0_DBG_BASE                     0x7FFECC4000ull
+#define SPMU_0_NIC0_DBG_MAX_OFFSET                 0x1000
+#define SPMU_0_NIC0_DBG_SECTION                    0x2000
+#define mmUSER_CTI_0_NIC0_DBG_BASE                 0x7FFECC6000ull
+#define USER_CTI_0_NIC0_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_0_NIC0_DBG_SECTION                0xB000
+#define mmSTM_1_NIC0_DBG_BASE                      0x7FFECD1000ull
+#define STM_1_NIC0_DBG_MAX_OFFSET                  0x1000
+#define STM_1_NIC0_DBG_SECTION                     0x1000
+#define mmCTI_1_NIC0_DBG_BASE                      0x7FFECD2000ull
+#define CTI_1_NIC0_DBG_MAX_OFFSET                  0x1000
+#define CTI_1_NIC0_DBG_SECTION                     0x1000
+#define mmETF_1_NIC0_DBG_BASE                      0x7FFECD3000ull
+#define ETF_1_NIC0_DBG_MAX_OFFSET                  0x1000
+#define ETF_1_NIC0_DBG_SECTION                     0x1000
+#define mmSPMU_1_NIC0_DBG_BASE                     0x7FFECD4000ull
+#define SPMU_1_NIC0_DBG_MAX_OFFSET                 0x1000
+#define SPMU_1_NIC0_DBG_SECTION                    0x1000
+#define mmBMON_CTI_NIC0_DBG_BASE                   0x7FFECD5000ull
+#define BMON_CTI_NIC0_DBG_MAX_OFFSET               0x1000
+#define BMON_CTI_NIC0_DBG_SECTION                  0x1000
+#define mmUSER_CTI_1_NIC0_DBG_BASE                 0x7FFECD6000ull
+#define USER_CTI_1_NIC0_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_1_NIC0_DBG_SECTION                0x1000
+#define mmBMON0_NIC0_DBG_BASE                      0x7FFECD7000ull
+#define BMON0_NIC0_DBG_MAX_OFFSET                  0x1000
+#define BMON0_NIC0_DBG_SECTION                     0x1000
+#define mmBMON1_NIC0_DBG_BASE                      0x7FFECD8000ull
+#define BMON1_NIC0_DBG_MAX_OFFSET                  0x1000
+#define BMON1_NIC0_DBG_SECTION                     0x1000
+#define mmBMON2_NIC0_DBG_BASE                      0x7FFECD9000ull
+#define BMON2_NIC0_DBG_MAX_OFFSET                  0x1000
+#define BMON2_NIC0_DBG_SECTION                     0x1000
+#define mmBMON3_NIC0_DBG_BASE                      0x7FFECDA000ull
+#define BMON3_NIC0_DBG_MAX_OFFSET                  0x1000
+#define BMON3_NIC0_DBG_SECTION                     0x1000
+#define mmBMON4_NIC0_DBG_BASE                      0x7FFECDB000ull
+#define BMON4_NIC0_DBG_MAX_OFFSET                  0x1000
+#define BMON4_NIC0_DBG_SECTION                     0x6000
+#define mmFUNNEL_NIC0_DBG_BASE                     0x7FFECE1000ull
+#define FUNNEL_NIC0_DBG_MAX_OFFSET                 0x1000
+#define FUNNEL_NIC0_DBG_SECTION                    0x1F000
+#define mmNIC1_ROM_TABLE_BASE                      0x7FFED00000ull
+#define NIC1_ROM_TABLE_MAX_OFFSET                  0x1000
+#define NIC1_ROM_TABLE_SECTION                     0x1000
+#define mmSTM_0_NIC1_DBG_BASE                      0x7FFED01000ull
+#define STM_0_NIC1_DBG_MAX_OFFSET                  0x21000
+#define STM_0_NIC1_DBG_SECTION                     0x1000
+#define mmCTI_0_NIC1_DBG_BASE                      0x7FFED02000ull
+#define CTI_0_NIC1_DBG_MAX_OFFSET                  0x1000
+#define CTI_0_NIC1_DBG_SECTION                     0x1000
+#define mmETF_0_NIC1_DBG_BASE                      0x7FFED03000ull
+#define ETF_0_NIC1_DBG_MAX_OFFSET                  0x1000
+#define ETF_0_NIC1_DBG_SECTION                     0x1000
+#define mmSPMU_0_NIC1_DBG_BASE                     0x7FFED04000ull
+#define SPMU_0_NIC1_DBG_MAX_OFFSET                 0x1000
+#define SPMU_0_NIC1_DBG_SECTION                    0x2000
+#define mmUSER_CTI_0_NIC1_DBG_BASE                 0x7FFED06000ull
+#define USER_CTI_0_NIC1_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_0_NIC1_DBG_SECTION                0xB000
+#define mmSTM_1_NIC1_DBG_BASE                      0x7FFED11000ull
+#define STM_1_NIC1_DBG_MAX_OFFSET                  0x1000
+#define STM_1_NIC1_DBG_SECTION                     0x1000
+#define mmCTI_1_NIC1_DBG_BASE                      0x7FFED12000ull
+#define CTI_1_NIC1_DBG_MAX_OFFSET                  0x1000
+#define CTI_1_NIC1_DBG_SECTION                     0x1000
+#define mmETF_1_NIC1_DBG_BASE                      0x7FFED13000ull
+#define ETF_1_NIC1_DBG_MAX_OFFSET                  0x1000
+#define ETF_1_NIC1_DBG_SECTION                     0x1000
+#define mmSPMU_1_NIC1_DBG_BASE                     0x7FFED14000ull
+#define SPMU_1_NIC1_DBG_MAX_OFFSET                 0x1000
+#define SPMU_1_NIC1_DBG_SECTION                    0x1000
+#define mmBMON_CTI_NIC1_DBG_BASE                   0x7FFED15000ull
+#define BMON_CTI_NIC1_DBG_MAX_OFFSET               0x1000
+#define BMON_CTI_NIC1_DBG_SECTION                  0x1000
+#define mmUSER_CTI_1_NIC1_DBG_BASE                 0x7FFED16000ull
+#define USER_CTI_1_NIC1_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_1_NIC1_DBG_SECTION                0x1000
+#define mmBMON0_NIC1_DBG_BASE                      0x7FFED17000ull
+#define BMON0_NIC1_DBG_MAX_OFFSET                  0x1000
+#define BMON0_NIC1_DBG_SECTION                     0x1000
+#define mmBMON1_NIC1_DBG_BASE                      0x7FFED18000ull
+#define BMON1_NIC1_DBG_MAX_OFFSET                  0x1000
+#define BMON1_NIC1_DBG_SECTION                     0x1000
+#define mmBMON2_NIC1_DBG_BASE                      0x7FFED19000ull
+#define BMON2_NIC1_DBG_MAX_OFFSET                  0x1000
+#define BMON2_NIC1_DBG_SECTION                     0x1000
+#define mmBMON3_NIC1_DBG_BASE                      0x7FFED1A000ull
+#define BMON3_NIC1_DBG_MAX_OFFSET                  0x1000
+#define BMON3_NIC1_DBG_SECTION                     0x1000
+#define mmBMON4_NIC1_DBG_BASE                      0x7FFED1B000ull
+#define BMON4_NIC1_DBG_MAX_OFFSET                  0x1000
+#define BMON4_NIC1_DBG_SECTION                     0x6000
+#define mmFUNNEL_NIC1_DBG_BASE                     0x7FFED21000ull
+#define FUNNEL_NIC1_DBG_MAX_OFFSET                 0x1000
+#define FUNNEL_NIC1_DBG_SECTION                    0x1F000
+#define mmNIC2_ROM_TABLE_BASE                      0x7FFED40000ull
+#define NIC2_ROM_TABLE_MAX_OFFSET                  0x1000
+#define NIC2_ROM_TABLE_SECTION                     0x1000
+#define mmSTM_0_NIC2_DBG_BASE                      0x7FFED41000ull
+#define STM_0_NIC2_DBG_MAX_OFFSET                  0x21000
+#define STM_0_NIC2_DBG_SECTION                     0x1000
+#define mmCTI_0_NIC2_DBG_BASE                      0x7FFED42000ull
+#define CTI_0_NIC2_DBG_MAX_OFFSET                  0x1000
+#define CTI_0_NIC2_DBG_SECTION                     0x1000
+#define mmETF_0_NIC2_DBG_BASE                      0x7FFED43000ull
+#define ETF_0_NIC2_DBG_MAX_OFFSET                  0x1000
+#define ETF_0_NIC2_DBG_SECTION                     0x1000
+#define mmSPMU_0_NIC2_DBG_BASE                     0x7FFED44000ull
+#define SPMU_0_NIC2_DBG_MAX_OFFSET                 0x1000
+#define SPMU_0_NIC2_DBG_SECTION                    0x2000
+#define mmUSER_CTI_0_NIC2_DBG_BASE                 0x7FFED46000ull
+#define USER_CTI_0_NIC2_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_0_NIC2_DBG_SECTION                0xB000
+#define mmSTM_1_NIC2_DBG_BASE                      0x7FFED51000ull
+#define STM_1_NIC2_DBG_MAX_OFFSET                  0x1000
+#define STM_1_NIC2_DBG_SECTION                     0x1000
+#define mmCTI_1_NIC2_DBG_BASE                      0x7FFED52000ull
+#define CTI_1_NIC2_DBG_MAX_OFFSET                  0x1000
+#define CTI_1_NIC2_DBG_SECTION                     0x1000
+#define mmETF_1_NIC2_DBG_BASE                      0x7FFED53000ull
+#define ETF_1_NIC2_DBG_MAX_OFFSET                  0x1000
+#define ETF_1_NIC2_DBG_SECTION                     0x1000
+#define mmSPMU_1_NIC2_DBG_BASE                     0x7FFED54000ull
+#define SPMU_1_NIC2_DBG_MAX_OFFSET                 0x1000
+#define SPMU_1_NIC2_DBG_SECTION                    0x1000
+#define mmBMON_CTI_NIC2_DBG_BASE                   0x7FFED55000ull
+#define BMON_CTI_NIC2_DBG_MAX_OFFSET               0x1000
+#define BMON_CTI_NIC2_DBG_SECTION                  0x1000
+#define mmUSER_CTI_1_NIC2_DBG_BASE                 0x7FFED56000ull
+#define USER_CTI_1_NIC2_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_1_NIC2_DBG_SECTION                0x1000
+#define mmBMON0_NIC2_DBG_BASE                      0x7FFED57000ull
+#define BMON0_NIC2_DBG_MAX_OFFSET                  0x1000
+#define BMON0_NIC2_DBG_SECTION                     0x1000
+#define mmBMON1_NIC2_DBG_BASE                      0x7FFED58000ull
+#define BMON1_NIC2_DBG_MAX_OFFSET                  0x1000
+#define BMON1_NIC2_DBG_SECTION                     0x1000
+#define mmBMON2_NIC2_DBG_BASE                      0x7FFED59000ull
+#define BMON2_NIC2_DBG_MAX_OFFSET                  0x1000
+#define BMON2_NIC2_DBG_SECTION                     0x1000
+#define mmBMON3_NIC2_DBG_BASE                      0x7FFED5A000ull
+#define BMON3_NIC2_DBG_MAX_OFFSET                  0x1000
+#define BMON3_NIC2_DBG_SECTION                     0x1000
+#define mmBMON4_NIC2_DBG_BASE                      0x7FFED5B000ull
+#define BMON4_NIC2_DBG_MAX_OFFSET                  0x1000
+#define BMON4_NIC2_DBG_SECTION                     0x6000
+#define mmFUNNEL_NIC2_DBG_BASE                     0x7FFED61000ull
+#define FUNNEL_NIC2_DBG_MAX_OFFSET                 0x1000
+#define FUNNEL_NIC2_DBG_SECTION                    0x1F000
+#define mmNIC3_ROM_TABLE_BASE                      0x7FFED80000ull
+#define NIC3_ROM_TABLE_MAX_OFFSET                  0x1000
+#define NIC3_ROM_TABLE_SECTION                     0x1000
+#define mmSTM_0_NIC3_DBG_BASE                      0x7FFED81000ull
+#define STM_0_NIC3_DBG_MAX_OFFSET                  0x21000
+#define STM_0_NIC3_DBG_SECTION                     0x1000
+#define mmCTI_0_NIC3_DBG_BASE                      0x7FFED82000ull
+#define CTI_0_NIC3_DBG_MAX_OFFSET                  0x1000
+#define CTI_0_NIC3_DBG_SECTION                     0x1000
+#define mmETF_0_NIC3_DBG_BASE                      0x7FFED83000ull
+#define ETF_0_NIC3_DBG_MAX_OFFSET                  0x1000
+#define ETF_0_NIC3_DBG_SECTION                     0x1000
+#define mmSPMU_0_NIC3_DBG_BASE                     0x7FFED84000ull
+#define SPMU_0_NIC3_DBG_MAX_OFFSET                 0x1000
+#define SPMU_0_NIC3_DBG_SECTION                    0x2000
+#define mmUSER_CTI_0_NIC3_DBG_BASE                 0x7FFED86000ull
+#define USER_CTI_0_NIC3_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_0_NIC3_DBG_SECTION                0xB000
+#define mmSTM_1_NIC3_DBG_BASE                      0x7FFED91000ull
+#define STM_1_NIC3_DBG_MAX_OFFSET                  0x1000
+#define STM_1_NIC3_DBG_SECTION                     0x1000
+#define mmCTI_1_NIC3_DBG_BASE                      0x7FFED92000ull
+#define CTI_1_NIC3_DBG_MAX_OFFSET                  0x1000
+#define CTI_1_NIC3_DBG_SECTION                     0x1000
+#define mmETF_1_NIC3_DBG_BASE                      0x7FFED93000ull
+#define ETF_1_NIC3_DBG_MAX_OFFSET                  0x1000
+#define ETF_1_NIC3_DBG_SECTION                     0x1000
+#define mmSPMU_1_NIC3_DBG_BASE                     0x7FFED94000ull
+#define SPMU_1_NIC3_DBG_MAX_OFFSET                 0x1000
+#define SPMU_1_NIC3_DBG_SECTION                    0x1000
+#define mmBMON_CTI_NIC3_DBG_BASE                   0x7FFED95000ull
+#define BMON_CTI_NIC3_DBG_MAX_OFFSET               0x1000
+#define BMON_CTI_NIC3_DBG_SECTION                  0x1000
+#define mmUSER_CTI_1_NIC3_DBG_BASE                 0x7FFED96000ull
+#define USER_CTI_1_NIC3_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_1_NIC3_DBG_SECTION                0x1000
+#define mmBMON0_NIC3_DBG_BASE                      0x7FFED97000ull
+#define BMON0_NIC3_DBG_MAX_OFFSET                  0x1000
+#define BMON0_NIC3_DBG_SECTION                     0x1000
+#define mmBMON1_NIC3_DBG_BASE                      0x7FFED98000ull
+#define BMON1_NIC3_DBG_MAX_OFFSET                  0x1000
+#define BMON1_NIC3_DBG_SECTION                     0x1000
+#define mmBMON2_NIC3_DBG_BASE                      0x7FFED99000ull
+#define BMON2_NIC3_DBG_MAX_OFFSET                  0x1000
+#define BMON2_NIC3_DBG_SECTION                     0x1000
+#define mmBMON3_NIC3_DBG_BASE                      0x7FFED9A000ull
+#define BMON3_NIC3_DBG_MAX_OFFSET                  0x1000
+#define BMON3_NIC3_DBG_SECTION                     0x1000
+#define mmBMON4_NIC3_DBG_BASE                      0x7FFED9B000ull
+#define BMON4_NIC3_DBG_MAX_OFFSET                  0x1000
+#define BMON4_NIC3_DBG_SECTION                     0x6000
+#define mmFUNNEL_NIC3_DBG_BASE                     0x7FFEDA1000ull
+#define FUNNEL_NIC3_DBG_MAX_OFFSET                 0x1000
+#define FUNNEL_NIC3_DBG_SECTION                    0x1F000
+#define mmNIC4_ROM_TABLE_BASE                      0x7FFEDC0000ull
+#define NIC4_ROM_TABLE_MAX_OFFSET                  0x1000
+#define NIC4_ROM_TABLE_SECTION                     0x1000
+#define mmSTM_0_NIC4_DBG_BASE                      0x7FFEDC1000ull
+#define STM_0_NIC4_DBG_MAX_OFFSET                  0x21000
+#define STM_0_NIC4_DBG_SECTION                     0x1000
+#define mmCTI_0_NIC4_DBG_BASE                      0x7FFEDC2000ull
+#define CTI_0_NIC4_DBG_MAX_OFFSET                  0x1000
+#define CTI_0_NIC4_DBG_SECTION                     0x1000
+#define mmETF_0_NIC4_DBG_BASE                      0x7FFEDC3000ull
+#define ETF_0_NIC4_DBG_MAX_OFFSET                  0x1000
+#define ETF_0_NIC4_DBG_SECTION                     0x1000
+#define mmSPMU_0_NIC4_DBG_BASE                     0x7FFEDC4000ull
+#define SPMU_0_NIC4_DBG_MAX_OFFSET                 0x1000
+#define SPMU_0_NIC4_DBG_SECTION                    0x2000
+#define mmUSER_CTI_0_NIC4_DBG_BASE                 0x7FFEDC6000ull
+#define USER_CTI_0_NIC4_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_0_NIC4_DBG_SECTION                0xB000
+#define mmSTM_1_NIC4_DBG_BASE                      0x7FFEDD1000ull
+#define STM_1_NIC4_DBG_MAX_OFFSET                  0x1000
+#define STM_1_NIC4_DBG_SECTION                     0x1000
+#define mmCTI_1_NIC4_DBG_BASE                      0x7FFEDD2000ull
+#define CTI_1_NIC4_DBG_MAX_OFFSET                  0x1000
+#define CTI_1_NIC4_DBG_SECTION                     0x1000
+#define mmETF_1_NIC4_DBG_BASE                      0x7FFEDD3000ull
+#define ETF_1_NIC4_DBG_MAX_OFFSET                  0x1000
+#define ETF_1_NIC4_DBG_SECTION                     0x1000
+#define mmSPMU_1_NIC4_DBG_BASE                     0x7FFEDD4000ull
+#define SPMU_1_NIC4_DBG_MAX_OFFSET                 0x1000
+#define SPMU_1_NIC4_DBG_SECTION                    0x1000
+#define mmBMON_CTI_NIC4_DBG_BASE                   0x7FFEDD5000ull
+#define BMON_CTI_NIC4_DBG_MAX_OFFSET               0x1000
+#define BMON_CTI_NIC4_DBG_SECTION                  0x1000
+#define mmUSER_CTI_1_NIC4_DBG_BASE                 0x7FFEDD6000ull
+#define USER_CTI_1_NIC4_DBG_MAX_OFFSET             0x1000
+#define USER_CTI_1_NIC4_DBG_SECTION                0x1000
+#define mmBMON0_NIC4_DBG_BASE                      0x7FFEDD7000ull
+#define BMON0_NIC4_DBG_MAX_OFFSET                  0x1000
+#define BMON0_NIC4_DBG_SECTION                     0x1000
+#define mmBMON1_NIC4_DBG_BASE                      0x7FFEDD8000ull
+#define BMON1_NIC4_DBG_MAX_OFFSET                  0x1000
+#define BMON1_NIC4_DBG_SECTION                     0x1000
+#define mmBMON2_NIC4_DBG_BASE                      0x7FFEDD9000ull
+#define BMON2_NIC4_DBG_MAX_OFFSET                  0x1000
+#define BMON2_NIC4_DBG_SECTION                     0x1000
+#define mmBMON3_NIC4_DBG_BASE                      0x7FFEDDA000ull
+#define BMON3_NIC4_DBG_MAX_OFFSET                  0x1000
+#define BMON3_NIC4_DBG_SECTION                     0x1000
+#define mmBMON4_NIC4_DBG_BASE                      0x7FFEDDB000ull
+#define BMON4_NIC4_DBG_MAX_OFFSET                  0x1000
+#define BMON4_NIC4_DBG_SECTION                     0x6000
+#define mmFUNNEL_NIC4_DBG_BASE                     0x7FFEDE1000ull
+#define FUNNEL_NIC4_DBG_MAX_OFFSET                 0x1000
+#define FUNNEL_NIC4_DBG_SECTION                    0x21F000
+#define mmTPC0_ROM_TABLE_BASE                      0x7FFF000000ull
+#define TPC0_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC0_ROM_TABLE_SECTION                     0x1000
+#define mmTPC0_EML_SPMU_BASE                       0x7FFF001000ull
+#define TPC0_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC0_EML_SPMU_SECTION                      0x1000
+#define mmTPC0_EML_ETF_BASE                        0x7FFF002000ull
+#define TPC0_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC0_EML_ETF_SECTION                       0x1000
+#define mmTPC0_EML_STM_BASE                        0x7FFF003000ull
+#define TPC0_EML_STM_MAX_OFFSET                    0x1000
+#define TPC0_EML_STM_SECTION                       0x2000
+#define mmTPC0_EML_CTI_BASE                        0x7FFF005000ull
+#define TPC0_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC0_EML_CTI_SECTION                       0x1000
+#define mmTPC0_EML_FUNNEL_BASE                     0x7FFF006000ull
+#define TPC0_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC0_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC0_EML_BUSMON_0_BASE                   0x7FFF007000ull
+#define TPC0_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC0_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC0_EML_BUSMON_1_BASE                   0x7FFF008000ull
+#define TPC0_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC0_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC0_EML_BUSMON_2_BASE                   0x7FFF009000ull
+#define TPC0_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC0_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC0_EML_BUSMON_3_BASE                   0x7FFF00A000ull
+#define TPC0_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC0_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC0_EML_CFG_BASE                        0x7FFF040000ull
+#define TPC0_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC0_EML_CFG_SECTION                       0x1000
+#define mmTPC0_EML_TPC_CFG_BASE                    0x7FFF041000ull
+#define TPC0_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC0_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC0_EML_TPC_CFG_BASE    0x7FFF041400ull
+#define KERNEL_TENSOR_0_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC0_EML_TPC_CFG_BASE    0x7FFF041438ull
+#define KERNEL_TENSOR_1_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC0_EML_TPC_CFG_BASE    0x7FFF041470ull
+#define KERNEL_TENSOR_2_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC0_EML_TPC_CFG_BASE    0x7FFF0414A8ull
+#define KERNEL_TENSOR_3_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC0_EML_TPC_CFG_BASE    0x7FFF0414E0ull
+#define KERNEL_TENSOR_4_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC0_EML_TPC_CFG_BASE    0x7FFF041518ull
+#define KERNEL_TENSOR_5_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC0_EML_TPC_CFG_BASE    0x7FFF041550ull
+#define KERNEL_TENSOR_6_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC0_EML_TPC_CFG_BASE    0x7FFF041588ull
+#define KERNEL_TENSOR_7_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC0_EML_TPC_CFG_BASE    0x7FFF0415C0ull
+#define KERNEL_TENSOR_8_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC0_EML_TPC_CFG_BASE    0x7FFF0415F8ull
+#define KERNEL_TENSOR_9_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC0_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC0_EML_TPC_CFG_BASE   0x7FFF041630ull
+#define KERNEL_TENSOR_10_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC0_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC0_EML_TPC_CFG_BASE   0x7FFF041668ull
+#define KERNEL_TENSOR_11_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC0_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC0_EML_TPC_CFG_BASE   0x7FFF0416A0ull
+#define KERNEL_TENSOR_12_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC0_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC0_EML_TPC_CFG_BASE   0x7FFF0416D8ull
+#define KERNEL_TENSOR_13_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC0_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC0_EML_TPC_CFG_BASE   0x7FFF041710ull
+#define KERNEL_TENSOR_14_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC0_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC0_EML_TPC_CFG_BASE   0x7FFF041748ull
+#define KERNEL_TENSOR_15_TPC0_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC0_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC0_EML_TPC_CFG_BASE 0x7FFF041780ull
+#define KERNEL_SYNC_OBJECT_TPC0_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC0_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC0_EML_TPC_CFG_BASE             0x7FFF041788ull
+#define KERNEL_TPC0_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC0_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC0_EML_TPC_CFG_BASE        0x7FFF041A00ull
+#define QM_TENSOR_0_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC0_EML_TPC_CFG_BASE        0x7FFF041A38ull
+#define QM_TENSOR_1_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC0_EML_TPC_CFG_BASE        0x7FFF041A70ull
+#define QM_TENSOR_2_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC0_EML_TPC_CFG_BASE        0x7FFF041AA8ull
+#define QM_TENSOR_3_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC0_EML_TPC_CFG_BASE        0x7FFF041AE0ull
+#define QM_TENSOR_4_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC0_EML_TPC_CFG_BASE        0x7FFF041B18ull
+#define QM_TENSOR_5_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC0_EML_TPC_CFG_BASE        0x7FFF041B50ull
+#define QM_TENSOR_6_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC0_EML_TPC_CFG_BASE        0x7FFF041B88ull
+#define QM_TENSOR_7_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC0_EML_TPC_CFG_BASE        0x7FFF041BC0ull
+#define QM_TENSOR_8_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC0_EML_TPC_CFG_BASE        0x7FFF041BF8ull
+#define QM_TENSOR_9_TPC0_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC0_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC0_EML_TPC_CFG_BASE       0x7FFF041C30ull
+#define QM_TENSOR_10_TPC0_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC0_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC0_EML_TPC_CFG_BASE       0x7FFF041C68ull
+#define QM_TENSOR_11_TPC0_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC0_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC0_EML_TPC_CFG_BASE       0x7FFF041CA0ull
+#define QM_TENSOR_12_TPC0_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC0_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC0_EML_TPC_CFG_BASE       0x7FFF041CD8ull
+#define QM_TENSOR_13_TPC0_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC0_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC0_EML_TPC_CFG_BASE       0x7FFF041D10ull
+#define QM_TENSOR_14_TPC0_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC0_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC0_EML_TPC_CFG_BASE       0x7FFF041D48ull
+#define QM_TENSOR_15_TPC0_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC0_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC0_EML_TPC_CFG_BASE     0x7FFF041D80ull
+#define QM_SYNC_OBJECT_TPC0_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC0_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC0_EML_TPC_CFG_BASE                 0x7FFF041D88ull
+#define QM_TPC0_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC0_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC0_EML_TPC_QM_BASE                     0x7FFF042000ull
+#define TPC0_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC0_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC0_EML_CS_BASE                         0x7FFF1FF000ull
+#define TPC0_EML_CS_MAX_OFFSET                     0x1000
+#define TPC0_EML_CS_SECTION                        0x1000
+#define mmTPC1_ROM_TABLE_BASE                      0x7FFF200000ull
+#define TPC1_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC1_ROM_TABLE_SECTION                     0x1000
+#define mmTPC1_EML_SPMU_BASE                       0x7FFF201000ull
+#define TPC1_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC1_EML_SPMU_SECTION                      0x1000
+#define mmTPC1_EML_ETF_BASE                        0x7FFF202000ull
+#define TPC1_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC1_EML_ETF_SECTION                       0x1000
+#define mmTPC1_EML_STM_BASE                        0x7FFF203000ull
+#define TPC1_EML_STM_MAX_OFFSET                    0x1000
+#define TPC1_EML_STM_SECTION                       0x2000
+#define mmTPC1_EML_CTI_BASE                        0x7FFF205000ull
+#define TPC1_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC1_EML_CTI_SECTION                       0x1000
+#define mmTPC1_EML_FUNNEL_BASE                     0x7FFF206000ull
+#define TPC1_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC1_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC1_EML_BUSMON_0_BASE                   0x7FFF207000ull
+#define TPC1_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC1_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC1_EML_BUSMON_1_BASE                   0x7FFF208000ull
+#define TPC1_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC1_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC1_EML_BUSMON_2_BASE                   0x7FFF209000ull
+#define TPC1_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC1_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC1_EML_BUSMON_3_BASE                   0x7FFF20A000ull
+#define TPC1_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC1_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC1_EML_CFG_BASE                        0x7FFF240000ull
+#define TPC1_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC1_EML_CFG_SECTION                       0x1000
+#define mmTPC1_EML_TPC_CFG_BASE                    0x7FFF241000ull
+#define TPC1_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC1_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC1_EML_TPC_CFG_BASE    0x7FFF241400ull
+#define KERNEL_TENSOR_0_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC1_EML_TPC_CFG_BASE    0x7FFF241438ull
+#define KERNEL_TENSOR_1_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC1_EML_TPC_CFG_BASE    0x7FFF241470ull
+#define KERNEL_TENSOR_2_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC1_EML_TPC_CFG_BASE    0x7FFF2414A8ull
+#define KERNEL_TENSOR_3_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC1_EML_TPC_CFG_BASE    0x7FFF2414E0ull
+#define KERNEL_TENSOR_4_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC1_EML_TPC_CFG_BASE    0x7FFF241518ull
+#define KERNEL_TENSOR_5_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC1_EML_TPC_CFG_BASE    0x7FFF241550ull
+#define KERNEL_TENSOR_6_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC1_EML_TPC_CFG_BASE    0x7FFF241588ull
+#define KERNEL_TENSOR_7_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC1_EML_TPC_CFG_BASE    0x7FFF2415C0ull
+#define KERNEL_TENSOR_8_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC1_EML_TPC_CFG_BASE    0x7FFF2415F8ull
+#define KERNEL_TENSOR_9_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC1_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC1_EML_TPC_CFG_BASE   0x7FFF241630ull
+#define KERNEL_TENSOR_10_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC1_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC1_EML_TPC_CFG_BASE   0x7FFF241668ull
+#define KERNEL_TENSOR_11_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC1_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC1_EML_TPC_CFG_BASE   0x7FFF2416A0ull
+#define KERNEL_TENSOR_12_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC1_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC1_EML_TPC_CFG_BASE   0x7FFF2416D8ull
+#define KERNEL_TENSOR_13_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC1_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC1_EML_TPC_CFG_BASE   0x7FFF241710ull
+#define KERNEL_TENSOR_14_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC1_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC1_EML_TPC_CFG_BASE   0x7FFF241748ull
+#define KERNEL_TENSOR_15_TPC1_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC1_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC1_EML_TPC_CFG_BASE 0x7FFF241780ull
+#define KERNEL_SYNC_OBJECT_TPC1_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC1_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC1_EML_TPC_CFG_BASE             0x7FFF241788ull
+#define KERNEL_TPC1_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC1_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC1_EML_TPC_CFG_BASE        0x7FFF241A00ull
+#define QM_TENSOR_0_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC1_EML_TPC_CFG_BASE        0x7FFF241A38ull
+#define QM_TENSOR_1_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC1_EML_TPC_CFG_BASE        0x7FFF241A70ull
+#define QM_TENSOR_2_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC1_EML_TPC_CFG_BASE        0x7FFF241AA8ull
+#define QM_TENSOR_3_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC1_EML_TPC_CFG_BASE        0x7FFF241AE0ull
+#define QM_TENSOR_4_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC1_EML_TPC_CFG_BASE        0x7FFF241B18ull
+#define QM_TENSOR_5_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC1_EML_TPC_CFG_BASE        0x7FFF241B50ull
+#define QM_TENSOR_6_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC1_EML_TPC_CFG_BASE        0x7FFF241B88ull
+#define QM_TENSOR_7_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC1_EML_TPC_CFG_BASE        0x7FFF241BC0ull
+#define QM_TENSOR_8_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC1_EML_TPC_CFG_BASE        0x7FFF241BF8ull
+#define QM_TENSOR_9_TPC1_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC1_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC1_EML_TPC_CFG_BASE       0x7FFF241C30ull
+#define QM_TENSOR_10_TPC1_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC1_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC1_EML_TPC_CFG_BASE       0x7FFF241C68ull
+#define QM_TENSOR_11_TPC1_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC1_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC1_EML_TPC_CFG_BASE       0x7FFF241CA0ull
+#define QM_TENSOR_12_TPC1_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC1_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC1_EML_TPC_CFG_BASE       0x7FFF241CD8ull
+#define QM_TENSOR_13_TPC1_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC1_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC1_EML_TPC_CFG_BASE       0x7FFF241D10ull
+#define QM_TENSOR_14_TPC1_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC1_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC1_EML_TPC_CFG_BASE       0x7FFF241D48ull
+#define QM_TENSOR_15_TPC1_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC1_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC1_EML_TPC_CFG_BASE     0x7FFF241D80ull
+#define QM_SYNC_OBJECT_TPC1_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC1_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC1_EML_TPC_CFG_BASE                 0x7FFF241D88ull
+#define QM_TPC1_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC1_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC1_EML_TPC_QM_BASE                     0x7FFF242000ull
+#define TPC1_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC1_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC1_EML_CS_BASE                         0x7FFF3FF000ull
+#define TPC1_EML_CS_MAX_OFFSET                     0x1000
+#define TPC1_EML_CS_SECTION                        0x1000
+#define mmTPC2_ROM_TABLE_BASE                      0x7FFF400000ull
+#define TPC2_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC2_ROM_TABLE_SECTION                     0x1000
+#define mmTPC2_EML_SPMU_BASE                       0x7FFF401000ull
+#define TPC2_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC2_EML_SPMU_SECTION                      0x1000
+#define mmTPC2_EML_ETF_BASE                        0x7FFF402000ull
+#define TPC2_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC2_EML_ETF_SECTION                       0x1000
+#define mmTPC2_EML_STM_BASE                        0x7FFF403000ull
+#define TPC2_EML_STM_MAX_OFFSET                    0x1000
+#define TPC2_EML_STM_SECTION                       0x2000
+#define mmTPC2_EML_CTI_BASE                        0x7FFF405000ull
+#define TPC2_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC2_EML_CTI_SECTION                       0x1000
+#define mmTPC2_EML_FUNNEL_BASE                     0x7FFF406000ull
+#define TPC2_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC2_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC2_EML_BUSMON_0_BASE                   0x7FFF407000ull
+#define TPC2_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC2_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC2_EML_BUSMON_1_BASE                   0x7FFF408000ull
+#define TPC2_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC2_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC2_EML_BUSMON_2_BASE                   0x7FFF409000ull
+#define TPC2_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC2_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC2_EML_BUSMON_3_BASE                   0x7FFF40A000ull
+#define TPC2_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC2_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC2_EML_CFG_BASE                        0x7FFF440000ull
+#define TPC2_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC2_EML_CFG_SECTION                       0x1000
+#define mmTPC2_EML_TPC_CFG_BASE                    0x7FFF441000ull
+#define TPC2_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC2_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC2_EML_TPC_CFG_BASE    0x7FFF441400ull
+#define KERNEL_TENSOR_0_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC2_EML_TPC_CFG_BASE    0x7FFF441438ull
+#define KERNEL_TENSOR_1_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC2_EML_TPC_CFG_BASE    0x7FFF441470ull
+#define KERNEL_TENSOR_2_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC2_EML_TPC_CFG_BASE    0x7FFF4414A8ull
+#define KERNEL_TENSOR_3_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC2_EML_TPC_CFG_BASE    0x7FFF4414E0ull
+#define KERNEL_TENSOR_4_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC2_EML_TPC_CFG_BASE    0x7FFF441518ull
+#define KERNEL_TENSOR_5_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC2_EML_TPC_CFG_BASE    0x7FFF441550ull
+#define KERNEL_TENSOR_6_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC2_EML_TPC_CFG_BASE    0x7FFF441588ull
+#define KERNEL_TENSOR_7_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC2_EML_TPC_CFG_BASE    0x7FFF4415C0ull
+#define KERNEL_TENSOR_8_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC2_EML_TPC_CFG_BASE    0x7FFF4415F8ull
+#define KERNEL_TENSOR_9_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC2_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC2_EML_TPC_CFG_BASE   0x7FFF441630ull
+#define KERNEL_TENSOR_10_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC2_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC2_EML_TPC_CFG_BASE   0x7FFF441668ull
+#define KERNEL_TENSOR_11_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC2_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC2_EML_TPC_CFG_BASE   0x7FFF4416A0ull
+#define KERNEL_TENSOR_12_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC2_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC2_EML_TPC_CFG_BASE   0x7FFF4416D8ull
+#define KERNEL_TENSOR_13_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC2_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC2_EML_TPC_CFG_BASE   0x7FFF441710ull
+#define KERNEL_TENSOR_14_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC2_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC2_EML_TPC_CFG_BASE   0x7FFF441748ull
+#define KERNEL_TENSOR_15_TPC2_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC2_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC2_EML_TPC_CFG_BASE 0x7FFF441780ull
+#define KERNEL_SYNC_OBJECT_TPC2_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC2_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC2_EML_TPC_CFG_BASE             0x7FFF441788ull
+#define KERNEL_TPC2_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC2_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC2_EML_TPC_CFG_BASE        0x7FFF441A00ull
+#define QM_TENSOR_0_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC2_EML_TPC_CFG_BASE        0x7FFF441A38ull
+#define QM_TENSOR_1_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC2_EML_TPC_CFG_BASE        0x7FFF441A70ull
+#define QM_TENSOR_2_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC2_EML_TPC_CFG_BASE        0x7FFF441AA8ull
+#define QM_TENSOR_3_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC2_EML_TPC_CFG_BASE        0x7FFF441AE0ull
+#define QM_TENSOR_4_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC2_EML_TPC_CFG_BASE        0x7FFF441B18ull
+#define QM_TENSOR_5_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC2_EML_TPC_CFG_BASE        0x7FFF441B50ull
+#define QM_TENSOR_6_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC2_EML_TPC_CFG_BASE        0x7FFF441B88ull
+#define QM_TENSOR_7_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC2_EML_TPC_CFG_BASE        0x7FFF441BC0ull
+#define QM_TENSOR_8_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC2_EML_TPC_CFG_BASE        0x7FFF441BF8ull
+#define QM_TENSOR_9_TPC2_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC2_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC2_EML_TPC_CFG_BASE       0x7FFF441C30ull
+#define QM_TENSOR_10_TPC2_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC2_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC2_EML_TPC_CFG_BASE       0x7FFF441C68ull
+#define QM_TENSOR_11_TPC2_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC2_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC2_EML_TPC_CFG_BASE       0x7FFF441CA0ull
+#define QM_TENSOR_12_TPC2_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC2_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC2_EML_TPC_CFG_BASE       0x7FFF441CD8ull
+#define QM_TENSOR_13_TPC2_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC2_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC2_EML_TPC_CFG_BASE       0x7FFF441D10ull
+#define QM_TENSOR_14_TPC2_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC2_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC2_EML_TPC_CFG_BASE       0x7FFF441D48ull
+#define QM_TENSOR_15_TPC2_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC2_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC2_EML_TPC_CFG_BASE     0x7FFF441D80ull
+#define QM_SYNC_OBJECT_TPC2_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC2_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC2_EML_TPC_CFG_BASE                 0x7FFF441D88ull
+#define QM_TPC2_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC2_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC2_EML_TPC_QM_BASE                     0x7FFF442000ull
+#define TPC2_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC2_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC2_EML_CS_BASE                         0x7FFF5FF000ull
+#define TPC2_EML_CS_MAX_OFFSET                     0x1000
+#define TPC2_EML_CS_SECTION                        0x1000
+#define mmTPC3_ROM_TABLE_BASE                      0x7FFF600000ull
+#define TPC3_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC3_ROM_TABLE_SECTION                     0x1000
+#define mmTPC3_EML_SPMU_BASE                       0x7FFF601000ull
+#define TPC3_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC3_EML_SPMU_SECTION                      0x1000
+#define mmTPC3_EML_ETF_BASE                        0x7FFF602000ull
+#define TPC3_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC3_EML_ETF_SECTION                       0x1000
+#define mmTPC3_EML_STM_BASE                        0x7FFF603000ull
+#define TPC3_EML_STM_MAX_OFFSET                    0x1000
+#define TPC3_EML_STM_SECTION                       0x2000
+#define mmTPC3_EML_CTI_BASE                        0x7FFF605000ull
+#define TPC3_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC3_EML_CTI_SECTION                       0x1000
+#define mmTPC3_EML_FUNNEL_BASE                     0x7FFF606000ull
+#define TPC3_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC3_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC3_EML_BUSMON_0_BASE                   0x7FFF607000ull
+#define TPC3_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC3_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC3_EML_BUSMON_1_BASE                   0x7FFF608000ull
+#define TPC3_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC3_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC3_EML_BUSMON_2_BASE                   0x7FFF609000ull
+#define TPC3_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC3_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC3_EML_BUSMON_3_BASE                   0x7FFF60A000ull
+#define TPC3_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC3_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC3_EML_CFG_BASE                        0x7FFF640000ull
+#define TPC3_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC3_EML_CFG_SECTION                       0x1000
+#define mmTPC3_EML_TPC_CFG_BASE                    0x7FFF641000ull
+#define TPC3_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC3_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC3_EML_TPC_CFG_BASE    0x7FFF641400ull
+#define KERNEL_TENSOR_0_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC3_EML_TPC_CFG_BASE    0x7FFF641438ull
+#define KERNEL_TENSOR_1_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC3_EML_TPC_CFG_BASE    0x7FFF641470ull
+#define KERNEL_TENSOR_2_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC3_EML_TPC_CFG_BASE    0x7FFF6414A8ull
+#define KERNEL_TENSOR_3_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC3_EML_TPC_CFG_BASE    0x7FFF6414E0ull
+#define KERNEL_TENSOR_4_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC3_EML_TPC_CFG_BASE    0x7FFF641518ull
+#define KERNEL_TENSOR_5_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC3_EML_TPC_CFG_BASE    0x7FFF641550ull
+#define KERNEL_TENSOR_6_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC3_EML_TPC_CFG_BASE    0x7FFF641588ull
+#define KERNEL_TENSOR_7_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC3_EML_TPC_CFG_BASE    0x7FFF6415C0ull
+#define KERNEL_TENSOR_8_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC3_EML_TPC_CFG_BASE    0x7FFF6415F8ull
+#define KERNEL_TENSOR_9_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC3_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC3_EML_TPC_CFG_BASE   0x7FFF641630ull
+#define KERNEL_TENSOR_10_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC3_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC3_EML_TPC_CFG_BASE   0x7FFF641668ull
+#define KERNEL_TENSOR_11_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC3_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC3_EML_TPC_CFG_BASE   0x7FFF6416A0ull
+#define KERNEL_TENSOR_12_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC3_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC3_EML_TPC_CFG_BASE   0x7FFF6416D8ull
+#define KERNEL_TENSOR_13_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC3_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC3_EML_TPC_CFG_BASE   0x7FFF641710ull
+#define KERNEL_TENSOR_14_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC3_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC3_EML_TPC_CFG_BASE   0x7FFF641748ull
+#define KERNEL_TENSOR_15_TPC3_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC3_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC3_EML_TPC_CFG_BASE 0x7FFF641780ull
+#define KERNEL_SYNC_OBJECT_TPC3_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC3_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC3_EML_TPC_CFG_BASE             0x7FFF641788ull
+#define KERNEL_TPC3_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC3_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC3_EML_TPC_CFG_BASE        0x7FFF641A00ull
+#define QM_TENSOR_0_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC3_EML_TPC_CFG_BASE        0x7FFF641A38ull
+#define QM_TENSOR_1_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC3_EML_TPC_CFG_BASE        0x7FFF641A70ull
+#define QM_TENSOR_2_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC3_EML_TPC_CFG_BASE        0x7FFF641AA8ull
+#define QM_TENSOR_3_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC3_EML_TPC_CFG_BASE        0x7FFF641AE0ull
+#define QM_TENSOR_4_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC3_EML_TPC_CFG_BASE        0x7FFF641B18ull
+#define QM_TENSOR_5_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC3_EML_TPC_CFG_BASE        0x7FFF641B50ull
+#define QM_TENSOR_6_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC3_EML_TPC_CFG_BASE        0x7FFF641B88ull
+#define QM_TENSOR_7_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC3_EML_TPC_CFG_BASE        0x7FFF641BC0ull
+#define QM_TENSOR_8_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC3_EML_TPC_CFG_BASE        0x7FFF641BF8ull
+#define QM_TENSOR_9_TPC3_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC3_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC3_EML_TPC_CFG_BASE       0x7FFF641C30ull
+#define QM_TENSOR_10_TPC3_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC3_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC3_EML_TPC_CFG_BASE       0x7FFF641C68ull
+#define QM_TENSOR_11_TPC3_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC3_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC3_EML_TPC_CFG_BASE       0x7FFF641CA0ull
+#define QM_TENSOR_12_TPC3_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC3_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC3_EML_TPC_CFG_BASE       0x7FFF641CD8ull
+#define QM_TENSOR_13_TPC3_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC3_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC3_EML_TPC_CFG_BASE       0x7FFF641D10ull
+#define QM_TENSOR_14_TPC3_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC3_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC3_EML_TPC_CFG_BASE       0x7FFF641D48ull
+#define QM_TENSOR_15_TPC3_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC3_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC3_EML_TPC_CFG_BASE     0x7FFF641D80ull
+#define QM_SYNC_OBJECT_TPC3_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC3_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC3_EML_TPC_CFG_BASE                 0x7FFF641D88ull
+#define QM_TPC3_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC3_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC3_EML_TPC_QM_BASE                     0x7FFF642000ull
+#define TPC3_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC3_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC3_EML_CS_BASE                         0x7FFF7FF000ull
+#define TPC3_EML_CS_MAX_OFFSET                     0x1000
+#define TPC3_EML_CS_SECTION                        0x1000
+#define mmTPC4_ROM_TABLE_BASE                      0x7FFF800000ull
+#define TPC4_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC4_ROM_TABLE_SECTION                     0x1000
+#define mmTPC4_EML_SPMU_BASE                       0x7FFF801000ull
+#define TPC4_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC4_EML_SPMU_SECTION                      0x1000
+#define mmTPC4_EML_ETF_BASE                        0x7FFF802000ull
+#define TPC4_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC4_EML_ETF_SECTION                       0x1000
+#define mmTPC4_EML_STM_BASE                        0x7FFF803000ull
+#define TPC4_EML_STM_MAX_OFFSET                    0x1000
+#define TPC4_EML_STM_SECTION                       0x2000
+#define mmTPC4_EML_CTI_BASE                        0x7FFF805000ull
+#define TPC4_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC4_EML_CTI_SECTION                       0x1000
+#define mmTPC4_EML_FUNNEL_BASE                     0x7FFF806000ull
+#define TPC4_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC4_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC4_EML_BUSMON_0_BASE                   0x7FFF807000ull
+#define TPC4_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC4_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC4_EML_BUSMON_1_BASE                   0x7FFF808000ull
+#define TPC4_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC4_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC4_EML_BUSMON_2_BASE                   0x7FFF809000ull
+#define TPC4_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC4_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC4_EML_BUSMON_3_BASE                   0x7FFF80A000ull
+#define TPC4_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC4_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC4_EML_CFG_BASE                        0x7FFF840000ull
+#define TPC4_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC4_EML_CFG_SECTION                       0x1000
+#define mmTPC4_EML_TPC_CFG_BASE                    0x7FFF841000ull
+#define TPC4_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC4_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC4_EML_TPC_CFG_BASE    0x7FFF841400ull
+#define KERNEL_TENSOR_0_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC4_EML_TPC_CFG_BASE    0x7FFF841438ull
+#define KERNEL_TENSOR_1_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC4_EML_TPC_CFG_BASE    0x7FFF841470ull
+#define KERNEL_TENSOR_2_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC4_EML_TPC_CFG_BASE    0x7FFF8414A8ull
+#define KERNEL_TENSOR_3_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC4_EML_TPC_CFG_BASE    0x7FFF8414E0ull
+#define KERNEL_TENSOR_4_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC4_EML_TPC_CFG_BASE    0x7FFF841518ull
+#define KERNEL_TENSOR_5_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC4_EML_TPC_CFG_BASE    0x7FFF841550ull
+#define KERNEL_TENSOR_6_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC4_EML_TPC_CFG_BASE    0x7FFF841588ull
+#define KERNEL_TENSOR_7_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC4_EML_TPC_CFG_BASE    0x7FFF8415C0ull
+#define KERNEL_TENSOR_8_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC4_EML_TPC_CFG_BASE    0x7FFF8415F8ull
+#define KERNEL_TENSOR_9_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC4_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC4_EML_TPC_CFG_BASE   0x7FFF841630ull
+#define KERNEL_TENSOR_10_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC4_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC4_EML_TPC_CFG_BASE   0x7FFF841668ull
+#define KERNEL_TENSOR_11_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC4_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC4_EML_TPC_CFG_BASE   0x7FFF8416A0ull
+#define KERNEL_TENSOR_12_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC4_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC4_EML_TPC_CFG_BASE   0x7FFF8416D8ull
+#define KERNEL_TENSOR_13_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC4_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC4_EML_TPC_CFG_BASE   0x7FFF841710ull
+#define KERNEL_TENSOR_14_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC4_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC4_EML_TPC_CFG_BASE   0x7FFF841748ull
+#define KERNEL_TENSOR_15_TPC4_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC4_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC4_EML_TPC_CFG_BASE 0x7FFF841780ull
+#define KERNEL_SYNC_OBJECT_TPC4_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC4_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC4_EML_TPC_CFG_BASE             0x7FFF841788ull
+#define KERNEL_TPC4_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC4_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC4_EML_TPC_CFG_BASE        0x7FFF841A00ull
+#define QM_TENSOR_0_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC4_EML_TPC_CFG_BASE        0x7FFF841A38ull
+#define QM_TENSOR_1_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC4_EML_TPC_CFG_BASE        0x7FFF841A70ull
+#define QM_TENSOR_2_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC4_EML_TPC_CFG_BASE        0x7FFF841AA8ull
+#define QM_TENSOR_3_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC4_EML_TPC_CFG_BASE        0x7FFF841AE0ull
+#define QM_TENSOR_4_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC4_EML_TPC_CFG_BASE        0x7FFF841B18ull
+#define QM_TENSOR_5_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC4_EML_TPC_CFG_BASE        0x7FFF841B50ull
+#define QM_TENSOR_6_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC4_EML_TPC_CFG_BASE        0x7FFF841B88ull
+#define QM_TENSOR_7_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC4_EML_TPC_CFG_BASE        0x7FFF841BC0ull
+#define QM_TENSOR_8_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC4_EML_TPC_CFG_BASE        0x7FFF841BF8ull
+#define QM_TENSOR_9_TPC4_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC4_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC4_EML_TPC_CFG_BASE       0x7FFF841C30ull
+#define QM_TENSOR_10_TPC4_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC4_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC4_EML_TPC_CFG_BASE       0x7FFF841C68ull
+#define QM_TENSOR_11_TPC4_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC4_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC4_EML_TPC_CFG_BASE       0x7FFF841CA0ull
+#define QM_TENSOR_12_TPC4_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC4_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC4_EML_TPC_CFG_BASE       0x7FFF841CD8ull
+#define QM_TENSOR_13_TPC4_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC4_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC4_EML_TPC_CFG_BASE       0x7FFF841D10ull
+#define QM_TENSOR_14_TPC4_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC4_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC4_EML_TPC_CFG_BASE       0x7FFF841D48ull
+#define QM_TENSOR_15_TPC4_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC4_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC4_EML_TPC_CFG_BASE     0x7FFF841D80ull
+#define QM_SYNC_OBJECT_TPC4_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC4_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC4_EML_TPC_CFG_BASE                 0x7FFF841D88ull
+#define QM_TPC4_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC4_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC4_EML_TPC_QM_BASE                     0x7FFF842000ull
+#define TPC4_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC4_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC4_EML_CS_BASE                         0x7FFF9FF000ull
+#define TPC4_EML_CS_MAX_OFFSET                     0x1000
+#define TPC4_EML_CS_SECTION                        0x1000
+#define mmTPC5_ROM_TABLE_BASE                      0x7FFFA00000ull
+#define TPC5_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC5_ROM_TABLE_SECTION                     0x1000
+#define mmTPC5_EML_SPMU_BASE                       0x7FFFA01000ull
+#define TPC5_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC5_EML_SPMU_SECTION                      0x1000
+#define mmTPC5_EML_ETF_BASE                        0x7FFFA02000ull
+#define TPC5_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC5_EML_ETF_SECTION                       0x1000
+#define mmTPC5_EML_STM_BASE                        0x7FFFA03000ull
+#define TPC5_EML_STM_MAX_OFFSET                    0x1000
+#define TPC5_EML_STM_SECTION                       0x2000
+#define mmTPC5_EML_CTI_BASE                        0x7FFFA05000ull
+#define TPC5_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC5_EML_CTI_SECTION                       0x1000
+#define mmTPC5_EML_FUNNEL_BASE                     0x7FFFA06000ull
+#define TPC5_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC5_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC5_EML_BUSMON_0_BASE                   0x7FFFA07000ull
+#define TPC5_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC5_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC5_EML_BUSMON_1_BASE                   0x7FFFA08000ull
+#define TPC5_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC5_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC5_EML_BUSMON_2_BASE                   0x7FFFA09000ull
+#define TPC5_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC5_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC5_EML_BUSMON_3_BASE                   0x7FFFA0A000ull
+#define TPC5_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC5_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC5_EML_CFG_BASE                        0x7FFFA40000ull
+#define TPC5_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC5_EML_CFG_SECTION                       0x1000
+#define mmTPC5_EML_TPC_CFG_BASE                    0x7FFFA41000ull
+#define TPC5_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC5_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC5_EML_TPC_CFG_BASE    0x7FFFA41400ull
+#define KERNEL_TENSOR_0_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC5_EML_TPC_CFG_BASE    0x7FFFA41438ull
+#define KERNEL_TENSOR_1_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC5_EML_TPC_CFG_BASE    0x7FFFA41470ull
+#define KERNEL_TENSOR_2_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC5_EML_TPC_CFG_BASE    0x7FFFA414A8ull
+#define KERNEL_TENSOR_3_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC5_EML_TPC_CFG_BASE    0x7FFFA414E0ull
+#define KERNEL_TENSOR_4_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC5_EML_TPC_CFG_BASE    0x7FFFA41518ull
+#define KERNEL_TENSOR_5_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC5_EML_TPC_CFG_BASE    0x7FFFA41550ull
+#define KERNEL_TENSOR_6_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC5_EML_TPC_CFG_BASE    0x7FFFA41588ull
+#define KERNEL_TENSOR_7_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC5_EML_TPC_CFG_BASE    0x7FFFA415C0ull
+#define KERNEL_TENSOR_8_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC5_EML_TPC_CFG_BASE    0x7FFFA415F8ull
+#define KERNEL_TENSOR_9_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC5_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC5_EML_TPC_CFG_BASE   0x7FFFA41630ull
+#define KERNEL_TENSOR_10_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC5_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC5_EML_TPC_CFG_BASE   0x7FFFA41668ull
+#define KERNEL_TENSOR_11_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC5_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC5_EML_TPC_CFG_BASE   0x7FFFA416A0ull
+#define KERNEL_TENSOR_12_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC5_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC5_EML_TPC_CFG_BASE   0x7FFFA416D8ull
+#define KERNEL_TENSOR_13_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC5_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC5_EML_TPC_CFG_BASE   0x7FFFA41710ull
+#define KERNEL_TENSOR_14_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC5_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC5_EML_TPC_CFG_BASE   0x7FFFA41748ull
+#define KERNEL_TENSOR_15_TPC5_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC5_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC5_EML_TPC_CFG_BASE 0x7FFFA41780ull
+#define KERNEL_SYNC_OBJECT_TPC5_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC5_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC5_EML_TPC_CFG_BASE             0x7FFFA41788ull
+#define KERNEL_TPC5_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC5_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC5_EML_TPC_CFG_BASE        0x7FFFA41A00ull
+#define QM_TENSOR_0_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC5_EML_TPC_CFG_BASE        0x7FFFA41A38ull
+#define QM_TENSOR_1_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC5_EML_TPC_CFG_BASE        0x7FFFA41A70ull
+#define QM_TENSOR_2_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC5_EML_TPC_CFG_BASE        0x7FFFA41AA8ull
+#define QM_TENSOR_3_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC5_EML_TPC_CFG_BASE        0x7FFFA41AE0ull
+#define QM_TENSOR_4_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC5_EML_TPC_CFG_BASE        0x7FFFA41B18ull
+#define QM_TENSOR_5_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC5_EML_TPC_CFG_BASE        0x7FFFA41B50ull
+#define QM_TENSOR_6_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC5_EML_TPC_CFG_BASE        0x7FFFA41B88ull
+#define QM_TENSOR_7_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC5_EML_TPC_CFG_BASE        0x7FFFA41BC0ull
+#define QM_TENSOR_8_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC5_EML_TPC_CFG_BASE        0x7FFFA41BF8ull
+#define QM_TENSOR_9_TPC5_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC5_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC5_EML_TPC_CFG_BASE       0x7FFFA41C30ull
+#define QM_TENSOR_10_TPC5_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC5_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC5_EML_TPC_CFG_BASE       0x7FFFA41C68ull
+#define QM_TENSOR_11_TPC5_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC5_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC5_EML_TPC_CFG_BASE       0x7FFFA41CA0ull
+#define QM_TENSOR_12_TPC5_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC5_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC5_EML_TPC_CFG_BASE       0x7FFFA41CD8ull
+#define QM_TENSOR_13_TPC5_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC5_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC5_EML_TPC_CFG_BASE       0x7FFFA41D10ull
+#define QM_TENSOR_14_TPC5_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC5_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC5_EML_TPC_CFG_BASE       0x7FFFA41D48ull
+#define QM_TENSOR_15_TPC5_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC5_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC5_EML_TPC_CFG_BASE     0x7FFFA41D80ull
+#define QM_SYNC_OBJECT_TPC5_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC5_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC5_EML_TPC_CFG_BASE                 0x7FFFA41D88ull
+#define QM_TPC5_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC5_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC5_EML_TPC_QM_BASE                     0x7FFFA42000ull
+#define TPC5_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC5_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC5_EML_CS_BASE                         0x7FFFBFF000ull
+#define TPC5_EML_CS_MAX_OFFSET                     0x1000
+#define TPC5_EML_CS_SECTION                        0x1000
+#define mmTPC6_ROM_TABLE_BASE                      0x7FFFC00000ull
+#define TPC6_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC6_ROM_TABLE_SECTION                     0x1000
+#define mmTPC6_EML_SPMU_BASE                       0x7FFFC01000ull
+#define TPC6_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC6_EML_SPMU_SECTION                      0x1000
+#define mmTPC6_EML_ETF_BASE                        0x7FFFC02000ull
+#define TPC6_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC6_EML_ETF_SECTION                       0x1000
+#define mmTPC6_EML_STM_BASE                        0x7FFFC03000ull
+#define TPC6_EML_STM_MAX_OFFSET                    0x1000
+#define TPC6_EML_STM_SECTION                       0x2000
+#define mmTPC6_EML_CTI_BASE                        0x7FFFC05000ull
+#define TPC6_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC6_EML_CTI_SECTION                       0x1000
+#define mmTPC6_EML_FUNNEL_BASE                     0x7FFFC06000ull
+#define TPC6_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC6_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC6_EML_BUSMON_0_BASE                   0x7FFFC07000ull
+#define TPC6_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC6_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC6_EML_BUSMON_1_BASE                   0x7FFFC08000ull
+#define TPC6_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC6_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC6_EML_BUSMON_2_BASE                   0x7FFFC09000ull
+#define TPC6_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC6_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC6_EML_BUSMON_3_BASE                   0x7FFFC0A000ull
+#define TPC6_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC6_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC6_EML_CFG_BASE                        0x7FFFC40000ull
+#define TPC6_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC6_EML_CFG_SECTION                       0x1000
+#define mmTPC6_EML_TPC_CFG_BASE                    0x7FFFC41000ull
+#define TPC6_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC6_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC6_EML_TPC_CFG_BASE    0x7FFFC41400ull
+#define KERNEL_TENSOR_0_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC6_EML_TPC_CFG_BASE    0x7FFFC41438ull
+#define KERNEL_TENSOR_1_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC6_EML_TPC_CFG_BASE    0x7FFFC41470ull
+#define KERNEL_TENSOR_2_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC6_EML_TPC_CFG_BASE    0x7FFFC414A8ull
+#define KERNEL_TENSOR_3_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC6_EML_TPC_CFG_BASE    0x7FFFC414E0ull
+#define KERNEL_TENSOR_4_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC6_EML_TPC_CFG_BASE    0x7FFFC41518ull
+#define KERNEL_TENSOR_5_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC6_EML_TPC_CFG_BASE    0x7FFFC41550ull
+#define KERNEL_TENSOR_6_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC6_EML_TPC_CFG_BASE    0x7FFFC41588ull
+#define KERNEL_TENSOR_7_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC6_EML_TPC_CFG_BASE    0x7FFFC415C0ull
+#define KERNEL_TENSOR_8_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC6_EML_TPC_CFG_BASE    0x7FFFC415F8ull
+#define KERNEL_TENSOR_9_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC6_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC6_EML_TPC_CFG_BASE   0x7FFFC41630ull
+#define KERNEL_TENSOR_10_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC6_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC6_EML_TPC_CFG_BASE   0x7FFFC41668ull
+#define KERNEL_TENSOR_11_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC6_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC6_EML_TPC_CFG_BASE   0x7FFFC416A0ull
+#define KERNEL_TENSOR_12_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC6_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC6_EML_TPC_CFG_BASE   0x7FFFC416D8ull
+#define KERNEL_TENSOR_13_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC6_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC6_EML_TPC_CFG_BASE   0x7FFFC41710ull
+#define KERNEL_TENSOR_14_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC6_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC6_EML_TPC_CFG_BASE   0x7FFFC41748ull
+#define KERNEL_TENSOR_15_TPC6_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC6_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC6_EML_TPC_CFG_BASE 0x7FFFC41780ull
+#define KERNEL_SYNC_OBJECT_TPC6_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC6_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC6_EML_TPC_CFG_BASE             0x7FFFC41788ull
+#define KERNEL_TPC6_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC6_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC6_EML_TPC_CFG_BASE        0x7FFFC41A00ull
+#define QM_TENSOR_0_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC6_EML_TPC_CFG_BASE        0x7FFFC41A38ull
+#define QM_TENSOR_1_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC6_EML_TPC_CFG_BASE        0x7FFFC41A70ull
+#define QM_TENSOR_2_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC6_EML_TPC_CFG_BASE        0x7FFFC41AA8ull
+#define QM_TENSOR_3_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC6_EML_TPC_CFG_BASE        0x7FFFC41AE0ull
+#define QM_TENSOR_4_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC6_EML_TPC_CFG_BASE        0x7FFFC41B18ull
+#define QM_TENSOR_5_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC6_EML_TPC_CFG_BASE        0x7FFFC41B50ull
+#define QM_TENSOR_6_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC6_EML_TPC_CFG_BASE        0x7FFFC41B88ull
+#define QM_TENSOR_7_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC6_EML_TPC_CFG_BASE        0x7FFFC41BC0ull
+#define QM_TENSOR_8_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC6_EML_TPC_CFG_BASE        0x7FFFC41BF8ull
+#define QM_TENSOR_9_TPC6_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC6_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC6_EML_TPC_CFG_BASE       0x7FFFC41C30ull
+#define QM_TENSOR_10_TPC6_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC6_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC6_EML_TPC_CFG_BASE       0x7FFFC41C68ull
+#define QM_TENSOR_11_TPC6_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC6_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC6_EML_TPC_CFG_BASE       0x7FFFC41CA0ull
+#define QM_TENSOR_12_TPC6_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC6_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC6_EML_TPC_CFG_BASE       0x7FFFC41CD8ull
+#define QM_TENSOR_13_TPC6_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC6_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC6_EML_TPC_CFG_BASE       0x7FFFC41D10ull
+#define QM_TENSOR_14_TPC6_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC6_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC6_EML_TPC_CFG_BASE       0x7FFFC41D48ull
+#define QM_TENSOR_15_TPC6_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC6_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC6_EML_TPC_CFG_BASE     0x7FFFC41D80ull
+#define QM_SYNC_OBJECT_TPC6_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC6_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC6_EML_TPC_CFG_BASE                 0x7FFFC41D88ull
+#define QM_TPC6_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC6_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC6_EML_TPC_QM_BASE                     0x7FFFC42000ull
+#define TPC6_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC6_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC6_EML_CS_BASE                         0x7FFFDFF000ull
+#define TPC6_EML_CS_MAX_OFFSET                     0x1000
+#define TPC6_EML_CS_SECTION                        0x1000
+#define mmTPC7_ROM_TABLE_BASE                      0x7FFFE00000ull
+#define TPC7_ROM_TABLE_MAX_OFFSET                  0x1000
+#define TPC7_ROM_TABLE_SECTION                     0x1000
+#define mmTPC7_EML_SPMU_BASE                       0x7FFFE01000ull
+#define TPC7_EML_SPMU_MAX_OFFSET                   0x1000
+#define TPC7_EML_SPMU_SECTION                      0x1000
+#define mmTPC7_EML_ETF_BASE                        0x7FFFE02000ull
+#define TPC7_EML_ETF_MAX_OFFSET                    0x1000
+#define TPC7_EML_ETF_SECTION                       0x1000
+#define mmTPC7_EML_STM_BASE                        0x7FFFE03000ull
+#define TPC7_EML_STM_MAX_OFFSET                    0x1000
+#define TPC7_EML_STM_SECTION                       0x2000
+#define mmTPC7_EML_CTI_BASE                        0x7FFFE05000ull
+#define TPC7_EML_CTI_MAX_OFFSET                    0x1000
+#define TPC7_EML_CTI_SECTION                       0x1000
+#define mmTPC7_EML_FUNNEL_BASE                     0x7FFFE06000ull
+#define TPC7_EML_FUNNEL_MAX_OFFSET                 0x1000
+#define TPC7_EML_FUNNEL_SECTION                    0x1000
+#define mmTPC7_EML_BUSMON_0_BASE                   0x7FFFE07000ull
+#define TPC7_EML_BUSMON_0_MAX_OFFSET               0x1000
+#define TPC7_EML_BUSMON_0_SECTION                  0x1000
+#define mmTPC7_EML_BUSMON_1_BASE                   0x7FFFE08000ull
+#define TPC7_EML_BUSMON_1_MAX_OFFSET               0x1000
+#define TPC7_EML_BUSMON_1_SECTION                  0x1000
+#define mmTPC7_EML_BUSMON_2_BASE                   0x7FFFE09000ull
+#define TPC7_EML_BUSMON_2_MAX_OFFSET               0x1000
+#define TPC7_EML_BUSMON_2_SECTION                  0x1000
+#define mmTPC7_EML_BUSMON_3_BASE                   0x7FFFE0A000ull
+#define TPC7_EML_BUSMON_3_MAX_OFFSET               0x1000
+#define TPC7_EML_BUSMON_3_SECTION                  0x36000
+#define mmTPC7_EML_CFG_BASE                        0x7FFFE40000ull
+#define TPC7_EML_CFG_MAX_OFFSET                    0x3380
+#define TPC7_EML_CFG_SECTION                       0x1000
+#define mmTPC7_EML_TPC_CFG_BASE                    0x7FFFE41000ull
+#define TPC7_EML_TPC_CFG_MAX_OFFSET                0xE400
+#define TPC7_EML_TPC_CFG_SECTION                   0x4000
+#define mmKERNEL_TENSOR_0_TPC7_EML_TPC_CFG_BASE    0x7FFFE41400ull
+#define KERNEL_TENSOR_0_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_0_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_1_TPC7_EML_TPC_CFG_BASE    0x7FFFE41438ull
+#define KERNEL_TENSOR_1_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_1_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_2_TPC7_EML_TPC_CFG_BASE    0x7FFFE41470ull
+#define KERNEL_TENSOR_2_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_2_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_3_TPC7_EML_TPC_CFG_BASE    0x7FFFE414A8ull
+#define KERNEL_TENSOR_3_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_3_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_4_TPC7_EML_TPC_CFG_BASE    0x7FFFE414E0ull
+#define KERNEL_TENSOR_4_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_4_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_5_TPC7_EML_TPC_CFG_BASE    0x7FFFE41518ull
+#define KERNEL_TENSOR_5_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_5_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_6_TPC7_EML_TPC_CFG_BASE    0x7FFFE41550ull
+#define KERNEL_TENSOR_6_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_6_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_7_TPC7_EML_TPC_CFG_BASE    0x7FFFE41588ull
+#define KERNEL_TENSOR_7_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_7_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_8_TPC7_EML_TPC_CFG_BASE    0x7FFFE415C0ull
+#define KERNEL_TENSOR_8_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_8_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_9_TPC7_EML_TPC_CFG_BASE    0x7FFFE415F8ull
+#define KERNEL_TENSOR_9_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_9_TPC7_EML_TPC_CFG_SECTION   0x3800
+#define mmKERNEL_TENSOR_10_TPC7_EML_TPC_CFG_BASE   0x7FFFE41630ull
+#define KERNEL_TENSOR_10_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_10_TPC7_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_11_TPC7_EML_TPC_CFG_BASE   0x7FFFE41668ull
+#define KERNEL_TENSOR_11_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_11_TPC7_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_12_TPC7_EML_TPC_CFG_BASE   0x7FFFE416A0ull
+#define KERNEL_TENSOR_12_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_12_TPC7_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_13_TPC7_EML_TPC_CFG_BASE   0x7FFFE416D8ull
+#define KERNEL_TENSOR_13_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_13_TPC7_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_14_TPC7_EML_TPC_CFG_BASE   0x7FFFE41710ull
+#define KERNEL_TENSOR_14_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_14_TPC7_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_TENSOR_15_TPC7_EML_TPC_CFG_BASE   0x7FFFE41748ull
+#define KERNEL_TENSOR_15_TPC7_EML_TPC_CFG_MAX_OFFSET 0x3800
+#define KERNEL_TENSOR_15_TPC7_EML_TPC_CFG_SECTION  0x3800
+#define mmKERNEL_SYNC_OBJECT_TPC7_EML_TPC_CFG_BASE 0x7FFFE41780ull
+#define KERNEL_SYNC_OBJECT_TPC7_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define KERNEL_SYNC_OBJECT_TPC7_EML_TPC_CFG_SECTION 0x8000
+#define mmKERNEL_TPC7_EML_TPC_CFG_BASE             0x7FFFE41788ull
+#define KERNEL_TPC7_EML_TPC_CFG_MAX_OFFSET         0xB800
+#define KERNEL_TPC7_EML_TPC_CFG_SECTION            0x2780
+#define mmQM_TENSOR_0_TPC7_EML_TPC_CFG_BASE        0x7FFFE41A00ull
+#define QM_TENSOR_0_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_0_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_1_TPC7_EML_TPC_CFG_BASE        0x7FFFE41A38ull
+#define QM_TENSOR_1_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_1_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_2_TPC7_EML_TPC_CFG_BASE        0x7FFFE41A70ull
+#define QM_TENSOR_2_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_2_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_3_TPC7_EML_TPC_CFG_BASE        0x7FFFE41AA8ull
+#define QM_TENSOR_3_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_3_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_4_TPC7_EML_TPC_CFG_BASE        0x7FFFE41AE0ull
+#define QM_TENSOR_4_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_4_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_5_TPC7_EML_TPC_CFG_BASE        0x7FFFE41B18ull
+#define QM_TENSOR_5_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_5_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_6_TPC7_EML_TPC_CFG_BASE        0x7FFFE41B50ull
+#define QM_TENSOR_6_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_6_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_7_TPC7_EML_TPC_CFG_BASE        0x7FFFE41B88ull
+#define QM_TENSOR_7_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_7_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_8_TPC7_EML_TPC_CFG_BASE        0x7FFFE41BC0ull
+#define QM_TENSOR_8_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_8_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_9_TPC7_EML_TPC_CFG_BASE        0x7FFFE41BF8ull
+#define QM_TENSOR_9_TPC7_EML_TPC_CFG_MAX_OFFSET    0x3800
+#define QM_TENSOR_9_TPC7_EML_TPC_CFG_SECTION       0x3800
+#define mmQM_TENSOR_10_TPC7_EML_TPC_CFG_BASE       0x7FFFE41C30ull
+#define QM_TENSOR_10_TPC7_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_10_TPC7_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_11_TPC7_EML_TPC_CFG_BASE       0x7FFFE41C68ull
+#define QM_TENSOR_11_TPC7_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_11_TPC7_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_12_TPC7_EML_TPC_CFG_BASE       0x7FFFE41CA0ull
+#define QM_TENSOR_12_TPC7_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_12_TPC7_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_13_TPC7_EML_TPC_CFG_BASE       0x7FFFE41CD8ull
+#define QM_TENSOR_13_TPC7_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_13_TPC7_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_14_TPC7_EML_TPC_CFG_BASE       0x7FFFE41D10ull
+#define QM_TENSOR_14_TPC7_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_14_TPC7_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_TENSOR_15_TPC7_EML_TPC_CFG_BASE       0x7FFFE41D48ull
+#define QM_TENSOR_15_TPC7_EML_TPC_CFG_MAX_OFFSET   0x3800
+#define QM_TENSOR_15_TPC7_EML_TPC_CFG_SECTION      0x3800
+#define mmQM_SYNC_OBJECT_TPC7_EML_TPC_CFG_BASE     0x7FFFE41D80ull
+#define QM_SYNC_OBJECT_TPC7_EML_TPC_CFG_MAX_OFFSET 0x8000
+#define QM_SYNC_OBJECT_TPC7_EML_TPC_CFG_SECTION    0x8000
+#define mmQM_TPC7_EML_TPC_CFG_BASE                 0x7FFFE41D88ull
+#define QM_TPC7_EML_TPC_CFG_MAX_OFFSET             0xB800
+#define QM_TPC7_EML_TPC_CFG_SECTION                0x2780
+#define mmTPC7_EML_TPC_QM_BASE                     0x7FFFE42000ull
+#define TPC7_EML_TPC_QM_MAX_OFFSET                 0xD040
+#define TPC7_EML_TPC_QM_SECTION                    0x1BD000
+#define mmTPC7_EML_CS_BASE                         0x7FFFFFF000ull
+#define TPC7_EML_CS_MAX_OFFSET                     0x1000
+
+#endif /* GAUDI_BLOCKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h
new file mode 100644
index 000000000000..85e3b5148595
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h
@@ -0,0 +1,299 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef ASIC_REG_GAUDI_REGS_H_
+#define ASIC_REG_GAUDI_REGS_H_
+
+#include "gaudi_blocks.h"
+#include "psoc_global_conf_regs.h"
+#include "psoc_timestamp_regs.h"
+#include "cpu_if_regs.h"
+#include "mmu_up_regs.h"
+#include "stlb_regs.h"
+#include "dma0_qm_regs.h"
+#include "dma1_qm_regs.h"
+#include "dma2_qm_regs.h"
+#include "dma3_qm_regs.h"
+#include "dma4_qm_regs.h"
+#include "dma5_qm_regs.h"
+#include "dma6_qm_regs.h"
+#include "dma7_qm_regs.h"
+#include "dma0_core_regs.h"
+#include "dma1_core_regs.h"
+#include "dma2_core_regs.h"
+#include "dma3_core_regs.h"
+#include "dma4_core_regs.h"
+#include "dma5_core_regs.h"
+#include "dma6_core_regs.h"
+#include "dma7_core_regs.h"
+#include "mme0_ctrl_regs.h"
+#include "mme1_ctrl_regs.h"
+#include "mme2_ctrl_regs.h"
+#include "mme3_ctrl_regs.h"
+#include "mme0_qm_regs.h"
+#include "mme2_qm_regs.h"
+#include "tpc0_cfg_regs.h"
+#include "tpc1_cfg_regs.h"
+#include "tpc2_cfg_regs.h"
+#include "tpc3_cfg_regs.h"
+#include "tpc4_cfg_regs.h"
+#include "tpc5_cfg_regs.h"
+#include "tpc6_cfg_regs.h"
+#include "tpc7_cfg_regs.h"
+#include "tpc0_qm_regs.h"
+#include "tpc1_qm_regs.h"
+#include "tpc2_qm_regs.h"
+#include "tpc3_qm_regs.h"
+#include "tpc4_qm_regs.h"
+#include "tpc5_qm_regs.h"
+#include "tpc6_qm_regs.h"
+#include "tpc7_qm_regs.h"
+#include "dma_if_e_n_down_ch0_regs.h"
+#include "dma_if_e_n_down_ch1_regs.h"
+#include "dma_if_e_s_down_ch0_regs.h"
+#include "dma_if_e_s_down_ch1_regs.h"
+#include "dma_if_w_n_down_ch0_regs.h"
+#include "dma_if_w_n_down_ch1_regs.h"
+#include "dma_if_w_s_down_ch0_regs.h"
+#include "dma_if_w_s_down_ch1_regs.h"
+#include "dma_if_e_n_regs.h"
+#include "dma_if_e_s_regs.h"
+#include "dma_if_w_n_regs.h"
+#include "dma_if_w_s_regs.h"
+#include "nif_rtr_ctrl_0_regs.h"
+#include "nif_rtr_ctrl_1_regs.h"
+#include "nif_rtr_ctrl_2_regs.h"
+#include "nif_rtr_ctrl_3_regs.h"
+#include "nif_rtr_ctrl_4_regs.h"
+#include "nif_rtr_ctrl_5_regs.h"
+#include "nif_rtr_ctrl_6_regs.h"
+#include "nif_rtr_ctrl_7_regs.h"
+#include "sif_rtr_ctrl_0_regs.h"
+#include "sif_rtr_ctrl_1_regs.h"
+#include "sif_rtr_ctrl_2_regs.h"
+#include "sif_rtr_ctrl_3_regs.h"
+#include "sif_rtr_ctrl_4_regs.h"
+#include "sif_rtr_ctrl_5_regs.h"
+#include "sif_rtr_ctrl_6_regs.h"
+#include "sif_rtr_ctrl_7_regs.h"
+#include "psoc_etr_regs.h"
+
+#include "dma0_qm_masks.h"
+#include "mme0_qm_masks.h"
+#include "tpc0_qm_masks.h"
+#include "dma0_core_masks.h"
+#include "tpc0_cfg_masks.h"
+#include "psoc_global_conf_masks.h"
+
+#include "psoc_pci_pll_regs.h"
+#include "psoc_hbm_pll_regs.h"
+
+#define GAUDI_ECC_MEM_SEL_OFFSET	0xF18
+#define GAUDI_ECC_ADDRESS_OFFSET	0xF1C
+#define GAUDI_ECC_SYNDROME_OFFSET	0xF20
+#define GAUDI_ECC_SERR0_OFFSET		0xF30
+#define GAUDI_ECC_SERR1_OFFSET		0xF34
+#define GAUDI_ECC_SERR2_OFFSET		0xF38
+#define GAUDI_ECC_SERR3_OFFSET		0xF3C
+#define GAUDI_ECC_DERR0_OFFSET		0xF40
+#define GAUDI_ECC_DERR1_OFFSET		0xF44
+#define GAUDI_ECC_DERR2_OFFSET		0xF48
+#define GAUDI_ECC_DERR3_OFFSET		0xF4C
+
+#define mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0                     0x492000
+#define mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0               0x494000
+#define mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0               0x494800
+#define mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_DATA_0                0x495000
+#define mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0                     0x495800
+#define mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0                  0x496000
+#define mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0                     0x4B2000
+#define mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0                  0x4B6000
+#define mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0                     0x4D2000
+#define mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0                  0x4D6000
+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0                     0x4F2000
+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1                     0x4F2004
+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047                  0x4F3FFC
+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0               0x4F4000
+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0                  0x4F6000
+#define mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511                0x4F67FC
+
+#define mmSIF_RTR_0_LBW_RANGE_PROT_HIT_AW                            0x300400
+#define mmSIF_RTR_1_LBW_RANGE_PROT_HIT_AW                            0x310400
+#define mmSIF_RTR_2_LBW_RANGE_PROT_HIT_AW                            0x320400
+#define mmSIF_RTR_3_LBW_RANGE_PROT_HIT_AW                            0x330400
+#define mmSIF_RTR_4_LBW_RANGE_PROT_HIT_AW                            0x340400
+#define mmSIF_RTR_5_LBW_RANGE_PROT_HIT_AW                            0x350400
+#define mmSIF_RTR_6_LBW_RANGE_PROT_HIT_AW                            0x360400
+#define mmSIF_RTR_7_LBW_RANGE_PROT_HIT_AW                            0x370400
+
+#define mmSIF_RTR_0_LBW_RANGE_PROT_HIT_AR                            0x300490
+#define mmSIF_RTR_1_LBW_RANGE_PROT_HIT_AR                            0x310490
+#define mmSIF_RTR_2_LBW_RANGE_PROT_HIT_AR                            0x320490
+#define mmSIF_RTR_3_LBW_RANGE_PROT_HIT_AR                            0x330490
+#define mmSIF_RTR_4_LBW_RANGE_PROT_HIT_AR                            0x340490
+#define mmSIF_RTR_5_LBW_RANGE_PROT_HIT_AR                            0x350490
+#define mmSIF_RTR_6_LBW_RANGE_PROT_HIT_AR                            0x360490
+#define mmSIF_RTR_7_LBW_RANGE_PROT_HIT_AR                            0x370490
+
+#define mmSIF_RTR_0_LBW_RANGE_PROT_MIN_AW_0                          0x300410
+#define mmSIF_RTR_1_LBW_RANGE_PROT_MIN_AW_0                          0x310410
+#define mmSIF_RTR_2_LBW_RANGE_PROT_MIN_AW_0                          0x320410
+#define mmSIF_RTR_3_LBW_RANGE_PROT_MIN_AW_0                          0x330410
+#define mmSIF_RTR_4_LBW_RANGE_PROT_MIN_AW_0                          0x340410
+#define mmSIF_RTR_5_LBW_RANGE_PROT_MIN_AW_0                          0x350410
+#define mmSIF_RTR_6_LBW_RANGE_PROT_MIN_AW_0                          0x360410
+#define mmSIF_RTR_7_LBW_RANGE_PROT_MIN_AW_0                          0x370410
+
+#define mmSIF_RTR_0_LBW_RANGE_PROT_MAX_AW_0                          0x300450
+#define mmSIF_RTR_1_LBW_RANGE_PROT_MAX_AW_0                          0x310450
+#define mmSIF_RTR_2_LBW_RANGE_PROT_MAX_AW_0                          0x320450
+#define mmSIF_RTR_3_LBW_RANGE_PROT_MAX_AW_0                          0x330450
+#define mmSIF_RTR_4_LBW_RANGE_PROT_MAX_AW_0                          0x340450
+#define mmSIF_RTR_5_LBW_RANGE_PROT_MAX_AW_0                          0x350450
+#define mmSIF_RTR_6_LBW_RANGE_PROT_MAX_AW_0                          0x360450
+#define mmSIF_RTR_7_LBW_RANGE_PROT_MAX_AW_0                          0x370450
+
+#define mmSIF_RTR_0_LBW_RANGE_PROT_MIN_AR_0                          0x3004A0
+#define mmSIF_RTR_1_LBW_RANGE_PROT_MIN_AR_0                          0x3104A0
+#define mmSIF_RTR_2_LBW_RANGE_PROT_MIN_AR_0                          0x3204A0
+#define mmSIF_RTR_3_LBW_RANGE_PROT_MIN_AR_0                          0x3304A0
+#define mmSIF_RTR_4_LBW_RANGE_PROT_MIN_AR_0                          0x3404A0
+#define mmSIF_RTR_5_LBW_RANGE_PROT_MIN_AR_0                          0x3504A0
+#define mmSIF_RTR_6_LBW_RANGE_PROT_MIN_AR_0                          0x3604A0
+#define mmSIF_RTR_7_LBW_RANGE_PROT_MIN_AR_0                          0x3704A0
+
+#define mmSIF_RTR_0_LBW_RANGE_PROT_MAX_AR_0                          0x3004E0
+#define mmSIF_RTR_1_LBW_RANGE_PROT_MAX_AR_0                          0x3104E0
+#define mmSIF_RTR_2_LBW_RANGE_PROT_MAX_AR_0                          0x3204E0
+#define mmSIF_RTR_3_LBW_RANGE_PROT_MAX_AR_0                          0x3304E0
+#define mmSIF_RTR_4_LBW_RANGE_PROT_MAX_AR_0                          0x3404E0
+#define mmSIF_RTR_5_LBW_RANGE_PROT_MAX_AR_0                          0x3504E0
+#define mmSIF_RTR_6_LBW_RANGE_PROT_MAX_AR_0                          0x3604E0
+#define mmSIF_RTR_7_LBW_RANGE_PROT_MAX_AR_0                          0x3704E0
+
+#define mmNIF_RTR_0_LBW_RANGE_PROT_HIT_AW                            0x380400
+#define mmNIF_RTR_1_LBW_RANGE_PROT_HIT_AW                            0x390400
+#define mmNIF_RTR_2_LBW_RANGE_PROT_HIT_AW                            0x3A0400
+#define mmNIF_RTR_3_LBW_RANGE_PROT_HIT_AW                            0x3B0400
+#define mmNIF_RTR_4_LBW_RANGE_PROT_HIT_AW                            0x3C0400
+#define mmNIF_RTR_5_LBW_RANGE_PROT_HIT_AW                            0x3D0400
+#define mmNIF_RTR_6_LBW_RANGE_PROT_HIT_AW                            0x3E0400
+#define mmNIF_RTR_7_LBW_RANGE_PROT_HIT_AW                            0x3F0400
+
+#define mmNIF_RTR_0_LBW_RANGE_PROT_HIT_AR                            0x380490
+#define mmNIF_RTR_1_LBW_RANGE_PROT_HIT_AR                            0x390490
+#define mmNIF_RTR_2_LBW_RANGE_PROT_HIT_AR                            0x3A0490
+#define mmNIF_RTR_3_LBW_RANGE_PROT_HIT_AR                            0x3B0490
+#define mmNIF_RTR_4_LBW_RANGE_PROT_HIT_AR                            0x3C0490
+#define mmNIF_RTR_5_LBW_RANGE_PROT_HIT_AR                            0x3D0490
+#define mmNIF_RTR_6_LBW_RANGE_PROT_HIT_AR                            0x3E0490
+#define mmNIF_RTR_7_LBW_RANGE_PROT_HIT_AR                            0x3F0490
+
+#define mmNIF_RTR_0_LBW_RANGE_PROT_MIN_AW_0                          0x380410
+#define mmNIF_RTR_1_LBW_RANGE_PROT_MIN_AW_0                          0x390410
+#define mmNIF_RTR_2_LBW_RANGE_PROT_MIN_AW_0                          0x3A0410
+#define mmNIF_RTR_3_LBW_RANGE_PROT_MIN_AW_0                          0x3B0410
+#define mmNIF_RTR_4_LBW_RANGE_PROT_MIN_AW_0                          0x3C0410
+#define mmNIF_RTR_5_LBW_RANGE_PROT_MIN_AW_0                          0x3D0410
+#define mmNIF_RTR_6_LBW_RANGE_PROT_MIN_AW_0                          0x3E0410
+#define mmNIF_RTR_7_LBW_RANGE_PROT_MIN_AW_0                          0x3F0410
+
+#define mmNIF_RTR_0_LBW_RANGE_PROT_MAX_AW_0                          0x380450
+#define mmNIF_RTR_1_LBW_RANGE_PROT_MAX_AW_0                          0x390450
+#define mmNIF_RTR_2_LBW_RANGE_PROT_MAX_AW_0                          0x3A0450
+#define mmNIF_RTR_3_LBW_RANGE_PROT_MAX_AW_0                          0x3B0450
+#define mmNIF_RTR_4_LBW_RANGE_PROT_MAX_AW_0                          0x3C0450
+#define mmNIF_RTR_5_LBW_RANGE_PROT_MAX_AW_0                          0x3D0450
+#define mmNIF_RTR_6_LBW_RANGE_PROT_MAX_AW_0                          0x3E0450
+#define mmNIF_RTR_7_LBW_RANGE_PROT_MAX_AW_0                          0x3F0450
+
+#define mmNIF_RTR_0_LBW_RANGE_PROT_MIN_AR_0                          0x3804A0
+#define mmNIF_RTR_1_LBW_RANGE_PROT_MIN_AR_0                          0x3904A0
+#define mmNIF_RTR_2_LBW_RANGE_PROT_MIN_AR_0                          0x3A04A0
+#define mmNIF_RTR_3_LBW_RANGE_PROT_MIN_AR_0                          0x3B04A0
+#define mmNIF_RTR_4_LBW_RANGE_PROT_MIN_AR_0                          0x3C04A0
+#define mmNIF_RTR_5_LBW_RANGE_PROT_MIN_AR_0                          0x3D04A0
+#define mmNIF_RTR_6_LBW_RANGE_PROT_MIN_AR_0                          0x3E04A0
+#define mmNIF_RTR_7_LBW_RANGE_PROT_MIN_AR_0                          0x3F04A0
+
+#define mmNIF_RTR_0_LBW_RANGE_PROT_MAX_AR_0                          0x3804E0
+#define mmNIF_RTR_1_LBW_RANGE_PROT_MAX_AR_0                          0x3904E0
+#define mmNIF_RTR_2_LBW_RANGE_PROT_MAX_AR_0                          0x3A04E0
+#define mmNIF_RTR_3_LBW_RANGE_PROT_MAX_AR_0                          0x3B04E0
+#define mmNIF_RTR_4_LBW_RANGE_PROT_MAX_AR_0                          0x3C04E0
+#define mmNIF_RTR_5_LBW_RANGE_PROT_MAX_AR_0                          0x3D04E0
+#define mmNIF_RTR_6_LBW_RANGE_PROT_MAX_AR_0                          0x3E04E0
+#define mmNIF_RTR_7_LBW_RANGE_PROT_MAX_AR_0                          0x3F04E0
+
+#define mmDMA_IF_W_S_DOWN_RSP_MID_WGHT_0                             0x489030
+#define mmDMA_IF_W_S_DOWN_RSP_MID_WGHT_1                             0x489034
+
+#define mmDMA_IF_E_S_DOWN_RSP_MID_WGHT_0                             0x4A9030
+#define mmDMA_IF_E_S_DOWN_RSP_MID_WGHT_1                             0x4A9034
+
+#define mmDMA_IF_W_N_DOWN_RSP_MID_WGHT_0                             0x4C9030
+#define mmDMA_IF_W_N_DOWN_RSP_MID_WGHT_1                             0x4C9034
+
+#define mmDMA_IF_E_N_DOWN_RSP_MID_WGHT_0                             0x4E9030
+#define mmDMA_IF_E_N_DOWN_RSP_MID_WGHT_1                             0x4E9034
+
+#define mmMME1_QM_GLBL_CFG0                                          0xE8000
+#define mmMME1_QM_GLBL_STS0                                          0xE8038
+
+#define mmMME0_SBAB_SB_STALL                                         0x4002C
+#define mmMME0_SBAB_ARUSER0                                          0x40034
+#define mmMME0_SBAB_ARUSER1                                          0x40038
+#define mmMME0_SBAB_PROT                                             0x40050
+
+#define mmMME1_SBAB_SB_STALL                                         0xC002C
+#define mmMME1_SBAB_ARUSER0                                          0xC0034
+#define mmMME1_SBAB_ARUSER1                                          0xC0038
+#define mmMME1_SBAB_PROT                                             0xC0050
+
+#define mmMME2_SBAB_SB_STALL                                         0x14002C
+#define mmMME2_SBAB_ARUSER0                                          0x140034
+#define mmMME2_SBAB_ARUSER1                                          0x140038
+#define mmMME2_SBAB_PROT                                             0x140050
+
+#define mmMME3_SBAB_SB_STALL                                         0x1C002C
+#define mmMME3_SBAB_ARUSER0                                          0x1C0034
+#define mmMME3_SBAB_ARUSER1                                          0x1C0038
+#define mmMME3_SBAB_PROT                                             0x1C0050
+
+#define mmMME0_ACC_ACC_STALL                                         0x20028
+#define mmMME0_ACC_WBC                                               0x20038
+#define mmMME0_ACC_PROT                                              0x20050
+
+#define mmMME1_ACC_ACC_STALL                                         0xA0028
+#define mmMME1_ACC_WBC                                               0xA0038
+#define mmMME1_ACC_PROT                                              0xA0050
+
+#define mmMME2_ACC_ACC_STALL                                         0x120028
+#define mmMME2_ACC_WBC                                               0x120038
+#define mmMME2_ACC_PROT                                              0x120050
+
+#define mmMME3_ACC_ACC_STALL                                         0x1A0028
+#define mmMME3_ACC_WBC                                               0x1A0038
+#define mmMME3_ACC_PROT                                              0x1A0050
+
+#define mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR                         0x800040
+
+#define mmPSOC_EFUSE_READ                                            0xC4A000
+#define mmPSOC_EFUSE_DATA_0                                          0xC4A080
+
+#define mmPCIE_WRAP_MAX_OUTSTAND                                     0xC01B20
+#define mmPCIE_WRAP_LBW_PROT_OVR                                     0xC01B48
+#define mmPCIE_WRAP_HBW_DRAIN_CFG                                    0xC01D54
+#define mmPCIE_WRAP_LBW_DRAIN_CFG                                    0xC01D5C
+
+#define mmPCIE_MSI_INTR_0                                            0xC13000
+
+#define mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG                           0xC02000
+
+#define mmPCIE_AUX_DBI                                               0xC07490
+
+#endif /* ASIC_REG_GAUDI_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_ctrl_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_ctrl_regs.h
new file mode 100644
index 000000000000..083d073a0128
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_ctrl_regs.h
@@ -0,0 +1,1456 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MME0_CTRL_REGS_H_
+#define ASIC_REG_MME0_CTRL_REGS_H_
+
+/*
+ *****************************************
+ *   MME0_CTRL (Prototype: MME)
+ *****************************************
+ */
+
+#define mmMME0_CTRL_ARCH_STATUS                                      0x60000
+
+#define mmMME0_CTRL_ARCH_BASE_ADDR_HIGH_S                            0x60008
+
+#define mmMME0_CTRL_ARCH_BASE_ADDR_HIGH_L                            0x6000C
+
+#define mmMME0_CTRL_ARCH_BASE_ADDR_HIGH_O                            0x60010
+
+#define mmMME0_CTRL_ARCH_BASE_ADDR_LOW_S                             0x60014
+
+#define mmMME0_CTRL_ARCH_BASE_ADDR_LOW_L                             0x60018
+
+#define mmMME0_CTRL_ARCH_BASE_ADDR_LOW_O                             0x6001C
+
+#define mmMME0_CTRL_ARCH_HEADER_LOW                                  0x60020
+
+#define mmMME0_CTRL_ARCH_HEADER_HIGH                                 0x60024
+
+#define mmMME0_CTRL_ARCH_CONV_KERNEL_SIZE_MINUS_1                    0x60028
+
+#define mmMME0_CTRL_ARCH_CONV_ASSOCIATED_DIMS_LOW                    0x6002C
+
+#define mmMME0_CTRL_ARCH_CONV_ASSOCIATED_DIMS_HIGH                   0x60030
+
+#define mmMME0_CTRL_ARCH_NUM_ITERATIONS_MINUS_1                      0x60034
+
+#define mmMME0_CTRL_ARCH_OUTER_LOOP                                  0x60038
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_0                   0x6003C
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_1                   0x60040
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_2                   0x60044
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_3                   0x60048
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_4                   0x6004C
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_0                      0x60050
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_1                      0x60054
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_2                      0x60058
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_3                      0x6005C
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_4                      0x60060
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_ROI_SIZE_0                         0x60064
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_ROI_SIZE_1                         0x60068
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_ROI_SIZE_2                         0x6006C
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_ROI_SIZE_3                         0x60070
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_0                  0x60074
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_1                  0x60078
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_2                  0x6007C
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_3                  0x60080
+
+#define mmMME0_CTRL_ARCH_TENSOR_S_SPATIAL_SIZE_MINUS_1               0x60084
+
+#define mmMME0_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_0                     0x60088
+
+#define mmMME0_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_1                     0x6008C
+
+#define mmMME0_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_2                     0x60090
+
+#define mmMME0_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_3                     0x60094
+
+#define mmMME0_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_4                     0x60098
+
+#define mmMME0_CTRL_ARCH_AGU_S_START_OFFSET_0                        0x6009C
+
+#define mmMME0_CTRL_ARCH_AGU_S_START_OFFSET_1                        0x600A0
+
+#define mmMME0_CTRL_ARCH_AGU_S_START_OFFSET_2                        0x600A4
+
+#define mmMME0_CTRL_ARCH_AGU_S_START_OFFSET_3                        0x600A8
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_0                   0x600AC
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_1                   0x600B0
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_2                   0x600B4
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_3                   0x600B8
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_4                   0x600BC
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_0                      0x600C0
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_1                      0x600C4
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_2                      0x600C8
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_3                      0x600CC
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_4                      0x600D0
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_ROI_SIZE_0                         0x600D4
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_ROI_SIZE_1                         0x600D8
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_ROI_SIZE_2                         0x600DC
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_ROI_SIZE_3                         0x600E0
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_0                  0x600E4
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_1                  0x600E8
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_2                  0x600EC
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_3                  0x600F0
+
+#define mmMME0_CTRL_ARCH_TENSOR_L_SPATIAL_SIZE_MINUS_1               0x600F4
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_0               0x600F8
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_1               0x600FC
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_2               0x60100
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_3               0x60104
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_4               0x60108
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_0                  0x6010C
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_1                  0x60110
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_2                  0x60114
+
+#define mmMME0_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_3                  0x60118
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_0              0x6011C
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_1              0x60120
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_2              0x60124
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_3              0x60128
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_4              0x6012C
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_0                 0x60130
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_1                 0x60134
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_2                 0x60138
+
+#define mmMME0_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_3                 0x6013C
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_0                   0x60140
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_1                   0x60144
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_2                   0x60148
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_3                   0x6014C
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_4                   0x60150
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_0                      0x60154
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_1                      0x60158
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_2                      0x6015C
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_3                      0x60160
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_4                      0x60164
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_ROI_SIZE_0                         0x60168
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_ROI_SIZE_1                         0x6016C
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_ROI_SIZE_2                         0x60170
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_ROI_SIZE_3                         0x60174
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_0                  0x60178
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_1                  0x6017C
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_2                  0x60180
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_3                  0x60184
+
+#define mmMME0_CTRL_ARCH_TENSOR_O_SPATIAL_SIZE_MINUS_1               0x60188
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_0               0x6018C
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_1               0x60190
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_2               0x60194
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_3               0x60198
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_4               0x6019C
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_0                  0x601A0
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_1                  0x601A4
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_2                  0x601A8
+
+#define mmMME0_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_3                  0x601AC
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_0              0x601B0
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_1              0x601B4
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_2              0x601B8
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_3              0x601BC
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_4              0x601C0
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_0                 0x601C4
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_1                 0x601C8
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_2                 0x601CC
+
+#define mmMME0_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_3                 0x601D0
+
+#define mmMME0_CTRL_ARCH_DESC_SB_REPEAT                              0x601D4
+
+#define mmMME0_CTRL_ARCH_DESC_RATE_LIMITER                           0x601D8
+
+#define mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL             0x601DC
+
+#define mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE            0x601E0
+
+#define mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_HIGH                  0x601E4
+
+#define mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_DATA                       0x601E8
+
+#define mmMME0_CTRL_ARCH_DESC_AXI_USER_DATA                          0x601EC
+
+#define mmMME0_CTRL_ARCH_DESC_PERF_EVT_S                             0x601F0
+
+#define mmMME0_CTRL_ARCH_DESC_PERF_EVT_L_LOCAL                       0x601F4
+
+#define mmMME0_CTRL_ARCH_DESC_PERF_EVT_L_REMOTE                      0x601F8
+
+#define mmMME0_CTRL_ARCH_DESC_PERF_EVT_O_LOCAL                       0x601FC
+
+#define mmMME0_CTRL_ARCH_DESC_PERF_EVT_O_REMOTE                      0x60200
+
+#define mmMME0_CTRL_ARCH_DESC_PADDING_VALUE_S                        0x60204
+
+#define mmMME0_CTRL_ARCH_DESC_PADDING_VALUE_L                        0x60208
+
+#define mmMME0_CTRL_ARCH_DESC_META_DATA_AGU_S                        0x6020C
+
+#define mmMME0_CTRL_ARCH_DESC_META_DATA_AGU_L_LOCAL                  0x60210
+
+#define mmMME0_CTRL_ARCH_DESC_META_DATA_AGU_L_REMOTE                 0x60214
+
+#define mmMME0_CTRL_ARCH_DESC_META_DATA_AGU_O_LOCAL                  0x60218
+
+#define mmMME0_CTRL_ARCH_DESC_META_DATA_AGU_O_REMOTE                 0x6021C
+
+#define mmMME0_CTRL_ARCH_DESC_PCU_RL_SATURATION                      0x60220
+
+#define mmMME0_CTRL_ARCH_DESC_DUMMY                                  0x60224
+
+#define mmMME0_CTRL_CMD                                              0x60280
+
+#define mmMME0_CTRL_STATUS1                                          0x60284
+
+#define mmMME0_CTRL_RESET                                            0x60288
+
+#define mmMME0_CTRL_QM_STALL                                         0x6028C
+
+#define mmMME0_CTRL_SYNC_OBJECT_FIFO_TH                              0x60290
+
+#define mmMME0_CTRL_EUS_ROLLUP_CNT_ADD                               0x60294
+
+#define mmMME0_CTRL_INTR_CAUSE                                       0x60298
+
+#define mmMME0_CTRL_INTR_MASK                                        0x6029C
+
+#define mmMME0_CTRL_LOG_SHADOW                                       0x602A0
+
+#define mmMME0_CTRL_PCU_RL_DESC0                                     0x602A4
+
+#define mmMME0_CTRL_PCU_RL_TOKEN_UPDATE                              0x602A8
+
+#define mmMME0_CTRL_PCU_RL_TH                                        0x602AC
+
+#define mmMME0_CTRL_PCU_RL_MIN                                       0x602B0
+
+#define mmMME0_CTRL_PCU_RL_CTRL_EN                                   0x602B4
+
+#define mmMME0_CTRL_PCU_RL_HISTORY_LOG_SIZE                          0x602B8
+
+#define mmMME0_CTRL_PCU_DUMMY_A_BF16                                 0x602BC
+
+#define mmMME0_CTRL_PCU_DUMMY_B_BF16                                 0x602C0
+
+#define mmMME0_CTRL_PCU_DUMMY_A_FP32_ODD                             0x602C4
+
+#define mmMME0_CTRL_PCU_DUMMY_A_FP32_EVEN                            0x602C8
+
+#define mmMME0_CTRL_PCU_DUMMY_B_FP32_ODD                             0x602CC
+
+#define mmMME0_CTRL_PCU_DUMMY_B_FP32_EVEN                            0x602D0
+
+#define mmMME0_CTRL_PROT                                             0x602D4
+
+#define mmMME0_CTRL_EU_POWER_SAVE_DISABLE                            0x602D8
+
+#define mmMME0_CTRL_CS_DBG_BLOCK_ID                                  0x602DC
+
+#define mmMME0_CTRL_CS_DBG_STATUS_DROP_CNT                           0x602E0
+
+#define mmMME0_CTRL_TE_CLOSE_CGATE                                   0x602E4
+
+#define mmMME0_CTRL_AGU_SM_INFLIGHT_CNTR                             0x602E8
+
+#define mmMME0_CTRL_AGU_SM_TOTAL_CNTR                                0x602EC
+
+#define mmMME0_CTRL_EZSYNC_OUT_CREDIT                                0x602F0
+
+#define mmMME0_CTRL_PCU_RL_SAT_SEC                                   0x602F4
+
+#define mmMME0_CTRL_AGU_SYNC_MSG_AXI_USER                            0x602F8
+
+#define mmMME0_CTRL_QM_SLV_LBW_CLK_EN                                0x602FC
+
+#define mmMME0_CTRL_SHADOW_0_STATUS                                  0x60400
+
+#define mmMME0_CTRL_SHADOW_0_BASE_ADDR_HIGH_S                        0x60408
+
+#define mmMME0_CTRL_SHADOW_0_BASE_ADDR_HIGH_L                        0x6040C
+
+#define mmMME0_CTRL_SHADOW_0_BASE_ADDR_HIGH_O                        0x60410
+
+#define mmMME0_CTRL_SHADOW_0_BASE_ADDR_LOW_S                         0x60414
+
+#define mmMME0_CTRL_SHADOW_0_BASE_ADDR_LOW_L                         0x60418
+
+#define mmMME0_CTRL_SHADOW_0_BASE_ADDR_LOW_O                         0x6041C
+
+#define mmMME0_CTRL_SHADOW_0_HEADER_LOW                              0x60420
+
+#define mmMME0_CTRL_SHADOW_0_HEADER_HIGH                             0x60424
+
+#define mmMME0_CTRL_SHADOW_0_CONV_KERNEL_SIZE_MINUS_1                0x60428
+
+#define mmMME0_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_LOW                0x6042C
+
+#define mmMME0_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_HIGH               0x60430
+
+#define mmMME0_CTRL_SHADOW_0_NUM_ITERATIONS_MINUS_1                  0x60434
+
+#define mmMME0_CTRL_SHADOW_0_OUTER_LOOP                              0x60438
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_0               0x6043C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_1               0x60440
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_2               0x60444
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_3               0x60448
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_4               0x6044C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_0                  0x60450
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_1                  0x60454
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_2                  0x60458
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_3                  0x6045C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_4                  0x60460
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_0                     0x60464
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_1                     0x60468
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_2                     0x6046C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_3                     0x60470
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_0              0x60474
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_1              0x60478
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_2              0x6047C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_3              0x60480
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x60484
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_0                 0x60488
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_1                 0x6048C
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_2                 0x60490
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_3                 0x60494
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_4                 0x60498
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_START_OFFSET_0                    0x6049C
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_START_OFFSET_1                    0x604A0
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_START_OFFSET_2                    0x604A4
+
+#define mmMME0_CTRL_SHADOW_0_AGU_S_START_OFFSET_3                    0x604A8
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_0               0x604AC
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_1               0x604B0
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_2               0x604B4
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_3               0x604B8
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_4               0x604BC
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_0                  0x604C0
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_1                  0x604C4
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_2                  0x604C8
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_3                  0x604CC
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_4                  0x604D0
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_0                     0x604D4
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_1                     0x604D8
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_2                     0x604DC
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_3                     0x604E0
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_0              0x604E4
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_1              0x604E8
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_2              0x604EC
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_3              0x604F0
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x604F4
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x604F8
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x604FC
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x60500
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x60504
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x60508
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_0              0x6050C
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_1              0x60510
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_2              0x60514
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_3              0x60518
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x6051C
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x60520
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x60524
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x60528
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x6052C
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_0             0x60530
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_1             0x60534
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_2             0x60538
+
+#define mmMME0_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_3             0x6053C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_0               0x60540
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_1               0x60544
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_2               0x60548
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_3               0x6054C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_4               0x60550
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_0                  0x60554
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_1                  0x60558
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_2                  0x6055C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_3                  0x60560
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_4                  0x60564
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_0                     0x60568
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_1                     0x6056C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_2                     0x60570
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_3                     0x60574
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_0              0x60578
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_1              0x6057C
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_2              0x60580
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_3              0x60584
+
+#define mmMME0_CTRL_SHADOW_0_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x60588
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x6058C
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x60590
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x60594
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x60598
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x6059C
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_0              0x605A0
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_1              0x605A4
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_2              0x605A8
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_3              0x605AC
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x605B0
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x605B4
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x605B8
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x605BC
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x605C0
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_0             0x605C4
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_1             0x605C8
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_2             0x605CC
+
+#define mmMME0_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_3             0x605D0
+
+#define mmMME0_CTRL_SHADOW_0_DESC_SB_REPEAT                          0x605D4
+
+#define mmMME0_CTRL_SHADOW_0_DESC_RATE_LIMITER                       0x605D8
+
+#define mmMME0_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x605DC
+
+#define mmMME0_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x605E0
+
+#define mmMME0_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_HIGH              0x605E4
+
+#define mmMME0_CTRL_SHADOW_0_DESC_SYNC_OBJECT_DATA                   0x605E8
+
+#define mmMME0_CTRL_SHADOW_0_DESC_AXI_USER_DATA                      0x605EC
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PERF_EVT_S                         0x605F0
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PERF_EVT_L_LOCAL                   0x605F4
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PERF_EVT_L_REMOTE                  0x605F8
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PERF_EVT_O_LOCAL                   0x605FC
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PERF_EVT_O_REMOTE                  0x60600
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PADDING_VALUE_S                    0x60604
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PADDING_VALUE_L                    0x60608
+
+#define mmMME0_CTRL_SHADOW_0_DESC_META_DATA_AGU_S                    0x6060C
+
+#define mmMME0_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_LOCAL              0x60610
+
+#define mmMME0_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_REMOTE             0x60614
+
+#define mmMME0_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_LOCAL              0x60618
+
+#define mmMME0_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_REMOTE             0x6061C
+
+#define mmMME0_CTRL_SHADOW_0_DESC_PCU_RL_SATURATION                  0x60620
+
+#define mmMME0_CTRL_SHADOW_0_DESC_DUMMY                              0x60624
+
+#define mmMME0_CTRL_SHADOW_1_STATUS                                  0x60680
+
+#define mmMME0_CTRL_SHADOW_1_BASE_ADDR_HIGH_S                        0x60688
+
+#define mmMME0_CTRL_SHADOW_1_BASE_ADDR_HIGH_L                        0x6068C
+
+#define mmMME0_CTRL_SHADOW_1_BASE_ADDR_HIGH_O                        0x60690
+
+#define mmMME0_CTRL_SHADOW_1_BASE_ADDR_LOW_S                         0x60694
+
+#define mmMME0_CTRL_SHADOW_1_BASE_ADDR_LOW_L                         0x60698
+
+#define mmMME0_CTRL_SHADOW_1_BASE_ADDR_LOW_O                         0x6069C
+
+#define mmMME0_CTRL_SHADOW_1_HEADER_LOW                              0x606A0
+
+#define mmMME0_CTRL_SHADOW_1_HEADER_HIGH                             0x606A4
+
+#define mmMME0_CTRL_SHADOW_1_CONV_KERNEL_SIZE_MINUS_1                0x606A8
+
+#define mmMME0_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_LOW                0x606AC
+
+#define mmMME0_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_HIGH               0x606B0
+
+#define mmMME0_CTRL_SHADOW_1_NUM_ITERATIONS_MINUS_1                  0x606B4
+
+#define mmMME0_CTRL_SHADOW_1_OUTER_LOOP                              0x606B8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_0               0x606BC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_1               0x606C0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_2               0x606C4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_3               0x606C8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_4               0x606CC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_0                  0x606D0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_1                  0x606D4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_2                  0x606D8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_3                  0x606DC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_4                  0x606E0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_0                     0x606E4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_1                     0x606E8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_2                     0x606EC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_3                     0x606F0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_0              0x606F4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_1              0x606F8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_2              0x606FC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_3              0x60700
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x60704
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_0                 0x60708
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_1                 0x6070C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_2                 0x60710
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_3                 0x60714
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_4                 0x60718
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_START_OFFSET_0                    0x6071C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_START_OFFSET_1                    0x60720
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_START_OFFSET_2                    0x60724
+
+#define mmMME0_CTRL_SHADOW_1_AGU_S_START_OFFSET_3                    0x60728
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_0               0x6072C
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_1               0x60730
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_2               0x60734
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_3               0x60738
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_4               0x6073C
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_0                  0x60740
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_1                  0x60744
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_2                  0x60748
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_3                  0x6074C
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_4                  0x60750
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_0                     0x60754
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_1                     0x60758
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_2                     0x6075C
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_3                     0x60760
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_0              0x60764
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_1              0x60768
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_2              0x6076C
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_3              0x60770
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x60774
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x60778
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x6077C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x60780
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x60784
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x60788
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_0              0x6078C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_1              0x60790
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_2              0x60794
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_3              0x60798
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x6079C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x607A0
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x607A4
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x607A8
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x607AC
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_0             0x607B0
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_1             0x607B4
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_2             0x607B8
+
+#define mmMME0_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_3             0x607BC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_0               0x607C0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_1               0x607C4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_2               0x607C8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_3               0x607CC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_4               0x607D0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_0                  0x607D4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_1                  0x607D8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_2                  0x607DC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_3                  0x607E0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_4                  0x607E4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_0                     0x607E8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_1                     0x607EC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_2                     0x607F0
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_3                     0x607F4
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_0              0x607F8
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_1              0x607FC
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_2              0x60800
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_3              0x60804
+
+#define mmMME0_CTRL_SHADOW_1_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x60808
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x6080C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x60810
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x60814
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x60818
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x6081C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_0              0x60820
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_1              0x60824
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_2              0x60828
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_3              0x6082C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x60830
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x60834
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x60838
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x6083C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x60840
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_0             0x60844
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_1             0x60848
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_2             0x6084C
+
+#define mmMME0_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_3             0x60850
+
+#define mmMME0_CTRL_SHADOW_1_DESC_SB_REPEAT                          0x60854
+
+#define mmMME0_CTRL_SHADOW_1_DESC_RATE_LIMITER                       0x60858
+
+#define mmMME0_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x6085C
+
+#define mmMME0_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x60860
+
+#define mmMME0_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_HIGH              0x60864
+
+#define mmMME0_CTRL_SHADOW_1_DESC_SYNC_OBJECT_DATA                   0x60868
+
+#define mmMME0_CTRL_SHADOW_1_DESC_AXI_USER_DATA                      0x6086C
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PERF_EVT_S                         0x60870
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PERF_EVT_L_LOCAL                   0x60874
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PERF_EVT_L_REMOTE                  0x60878
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PERF_EVT_O_LOCAL                   0x6087C
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PERF_EVT_O_REMOTE                  0x60880
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PADDING_VALUE_S                    0x60884
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PADDING_VALUE_L                    0x60888
+
+#define mmMME0_CTRL_SHADOW_1_DESC_META_DATA_AGU_S                    0x6088C
+
+#define mmMME0_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_LOCAL              0x60890
+
+#define mmMME0_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_REMOTE             0x60894
+
+#define mmMME0_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_LOCAL              0x60898
+
+#define mmMME0_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_REMOTE             0x6089C
+
+#define mmMME0_CTRL_SHADOW_1_DESC_PCU_RL_SATURATION                  0x608A0
+
+#define mmMME0_CTRL_SHADOW_1_DESC_DUMMY                              0x608A4
+
+#define mmMME0_CTRL_SHADOW_2_STATUS                                  0x60900
+
+#define mmMME0_CTRL_SHADOW_2_BASE_ADDR_HIGH_S                        0x60908
+
+#define mmMME0_CTRL_SHADOW_2_BASE_ADDR_HIGH_L                        0x6090C
+
+#define mmMME0_CTRL_SHADOW_2_BASE_ADDR_HIGH_O                        0x60910
+
+#define mmMME0_CTRL_SHADOW_2_BASE_ADDR_LOW_S                         0x60914
+
+#define mmMME0_CTRL_SHADOW_2_BASE_ADDR_LOW_L                         0x60918
+
+#define mmMME0_CTRL_SHADOW_2_BASE_ADDR_LOW_O                         0x6091C
+
+#define mmMME0_CTRL_SHADOW_2_HEADER_LOW                              0x60920
+
+#define mmMME0_CTRL_SHADOW_2_HEADER_HIGH                             0x60924
+
+#define mmMME0_CTRL_SHADOW_2_CONV_KERNEL_SIZE_MINUS_1                0x60928
+
+#define mmMME0_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_LOW                0x6092C
+
+#define mmMME0_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_HIGH               0x60930
+
+#define mmMME0_CTRL_SHADOW_2_NUM_ITERATIONS_MINUS_1                  0x60934
+
+#define mmMME0_CTRL_SHADOW_2_OUTER_LOOP                              0x60938
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_0               0x6093C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_1               0x60940
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_2               0x60944
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_3               0x60948
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_4               0x6094C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_0                  0x60950
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_1                  0x60954
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_2                  0x60958
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_3                  0x6095C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_4                  0x60960
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_0                     0x60964
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_1                     0x60968
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_2                     0x6096C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_3                     0x60970
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_0              0x60974
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_1              0x60978
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_2              0x6097C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_3              0x60980
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x60984
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_0                 0x60988
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_1                 0x6098C
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_2                 0x60990
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_3                 0x60994
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_4                 0x60998
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_START_OFFSET_0                    0x6099C
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_START_OFFSET_1                    0x609A0
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_START_OFFSET_2                    0x609A4
+
+#define mmMME0_CTRL_SHADOW_2_AGU_S_START_OFFSET_3                    0x609A8
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_0               0x609AC
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_1               0x609B0
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_2               0x609B4
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_3               0x609B8
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_4               0x609BC
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_0                  0x609C0
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_1                  0x609C4
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_2                  0x609C8
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_3                  0x609CC
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_4                  0x609D0
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_0                     0x609D4
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_1                     0x609D8
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_2                     0x609DC
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_3                     0x609E0
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_0              0x609E4
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_1              0x609E8
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_2              0x609EC
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_3              0x609F0
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x609F4
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x609F8
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x609FC
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x60A00
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x60A04
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x60A08
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_0              0x60A0C
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_1              0x60A10
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_2              0x60A14
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_3              0x60A18
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x60A1C
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x60A20
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x60A24
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x60A28
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x60A2C
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_0             0x60A30
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_1             0x60A34
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_2             0x60A38
+
+#define mmMME0_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_3             0x60A3C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_0               0x60A40
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_1               0x60A44
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_2               0x60A48
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_3               0x60A4C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_4               0x60A50
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_0                  0x60A54
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_1                  0x60A58
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_2                  0x60A5C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_3                  0x60A60
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_4                  0x60A64
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_0                     0x60A68
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_1                     0x60A6C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_2                     0x60A70
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_3                     0x60A74
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_0              0x60A78
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_1              0x60A7C
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_2              0x60A80
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_3              0x60A84
+
+#define mmMME0_CTRL_SHADOW_2_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x60A88
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x60A8C
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x60A90
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x60A94
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x60A98
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x60A9C
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_0              0x60AA0
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_1              0x60AA4
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_2              0x60AA8
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_3              0x60AAC
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x60AB0
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x60AB4
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x60AB8
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x60ABC
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x60AC0
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_0             0x60AC4
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_1             0x60AC8
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_2             0x60ACC
+
+#define mmMME0_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_3             0x60AD0
+
+#define mmMME0_CTRL_SHADOW_2_DESC_SB_REPEAT                          0x60AD4
+
+#define mmMME0_CTRL_SHADOW_2_DESC_RATE_LIMITER                       0x60AD8
+
+#define mmMME0_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x60ADC
+
+#define mmMME0_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x60AE0
+
+#define mmMME0_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_HIGH              0x60AE4
+
+#define mmMME0_CTRL_SHADOW_2_DESC_SYNC_OBJECT_DATA                   0x60AE8
+
+#define mmMME0_CTRL_SHADOW_2_DESC_AXI_USER_DATA                      0x60AEC
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PERF_EVT_S                         0x60AF0
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PERF_EVT_L_LOCAL                   0x60AF4
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PERF_EVT_L_REMOTE                  0x60AF8
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PERF_EVT_O_LOCAL                   0x60AFC
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PERF_EVT_O_REMOTE                  0x60B00
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PADDING_VALUE_S                    0x60B04
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PADDING_VALUE_L                    0x60B08
+
+#define mmMME0_CTRL_SHADOW_2_DESC_META_DATA_AGU_S                    0x60B0C
+
+#define mmMME0_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_LOCAL              0x60B10
+
+#define mmMME0_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_REMOTE             0x60B14
+
+#define mmMME0_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_LOCAL              0x60B18
+
+#define mmMME0_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_REMOTE             0x60B1C
+
+#define mmMME0_CTRL_SHADOW_2_DESC_PCU_RL_SATURATION                  0x60B20
+
+#define mmMME0_CTRL_SHADOW_2_DESC_DUMMY                              0x60B24
+
+#define mmMME0_CTRL_SHADOW_3_STATUS                                  0x60B80
+
+#define mmMME0_CTRL_SHADOW_3_BASE_ADDR_HIGH_S                        0x60B88
+
+#define mmMME0_CTRL_SHADOW_3_BASE_ADDR_HIGH_L                        0x60B8C
+
+#define mmMME0_CTRL_SHADOW_3_BASE_ADDR_HIGH_O                        0x60B90
+
+#define mmMME0_CTRL_SHADOW_3_BASE_ADDR_LOW_S                         0x60B94
+
+#define mmMME0_CTRL_SHADOW_3_BASE_ADDR_LOW_L                         0x60B98
+
+#define mmMME0_CTRL_SHADOW_3_BASE_ADDR_LOW_O                         0x60B9C
+
+#define mmMME0_CTRL_SHADOW_3_HEADER_LOW                              0x60BA0
+
+#define mmMME0_CTRL_SHADOW_3_HEADER_HIGH                             0x60BA4
+
+#define mmMME0_CTRL_SHADOW_3_CONV_KERNEL_SIZE_MINUS_1                0x60BA8
+
+#define mmMME0_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_LOW                0x60BAC
+
+#define mmMME0_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_HIGH               0x60BB0
+
+#define mmMME0_CTRL_SHADOW_3_NUM_ITERATIONS_MINUS_1                  0x60BB4
+
+#define mmMME0_CTRL_SHADOW_3_OUTER_LOOP                              0x60BB8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_0               0x60BBC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_1               0x60BC0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_2               0x60BC4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_3               0x60BC8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_4               0x60BCC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_0                  0x60BD0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_1                  0x60BD4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_2                  0x60BD8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_3                  0x60BDC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_4                  0x60BE0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_0                     0x60BE4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_1                     0x60BE8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_2                     0x60BEC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_3                     0x60BF0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_0              0x60BF4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_1              0x60BF8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_2              0x60BFC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_3              0x60C00
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x60C04
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_0                 0x60C08
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_1                 0x60C0C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_2                 0x60C10
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_3                 0x60C14
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_4                 0x60C18
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_START_OFFSET_0                    0x60C1C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_START_OFFSET_1                    0x60C20
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_START_OFFSET_2                    0x60C24
+
+#define mmMME0_CTRL_SHADOW_3_AGU_S_START_OFFSET_3                    0x60C28
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_0               0x60C2C
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_1               0x60C30
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_2               0x60C34
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_3               0x60C38
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_4               0x60C3C
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_0                  0x60C40
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_1                  0x60C44
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_2                  0x60C48
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_3                  0x60C4C
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_4                  0x60C50
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_0                     0x60C54
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_1                     0x60C58
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_2                     0x60C5C
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_3                     0x60C60
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_0              0x60C64
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_1              0x60C68
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_2              0x60C6C
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_3              0x60C70
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x60C74
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x60C78
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x60C7C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x60C80
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x60C84
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x60C88
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_0              0x60C8C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_1              0x60C90
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_2              0x60C94
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_3              0x60C98
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x60C9C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x60CA0
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x60CA4
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x60CA8
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x60CAC
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_0             0x60CB0
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_1             0x60CB4
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_2             0x60CB8
+
+#define mmMME0_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_3             0x60CBC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_0               0x60CC0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_1               0x60CC4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_2               0x60CC8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_3               0x60CCC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_4               0x60CD0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_0                  0x60CD4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_1                  0x60CD8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_2                  0x60CDC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_3                  0x60CE0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_4                  0x60CE4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_0                     0x60CE8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_1                     0x60CEC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_2                     0x60CF0
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_3                     0x60CF4
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_0              0x60CF8
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_1              0x60CFC
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_2              0x60D00
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_3              0x60D04
+
+#define mmMME0_CTRL_SHADOW_3_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x60D08
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x60D0C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x60D10
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x60D14
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x60D18
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x60D1C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_0              0x60D20
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_1              0x60D24
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_2              0x60D28
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_3              0x60D2C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x60D30
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x60D34
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x60D38
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x60D3C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x60D40
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_0             0x60D44
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_1             0x60D48
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_2             0x60D4C
+
+#define mmMME0_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_3             0x60D50
+
+#define mmMME0_CTRL_SHADOW_3_DESC_SB_REPEAT                          0x60D54
+
+#define mmMME0_CTRL_SHADOW_3_DESC_RATE_LIMITER                       0x60D58
+
+#define mmMME0_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x60D5C
+
+#define mmMME0_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x60D60
+
+#define mmMME0_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_HIGH              0x60D64
+
+#define mmMME0_CTRL_SHADOW_3_DESC_SYNC_OBJECT_DATA                   0x60D68
+
+#define mmMME0_CTRL_SHADOW_3_DESC_AXI_USER_DATA                      0x60D6C
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PERF_EVT_S                         0x60D70
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PERF_EVT_L_LOCAL                   0x60D74
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PERF_EVT_L_REMOTE                  0x60D78
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PERF_EVT_O_LOCAL                   0x60D7C
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PERF_EVT_O_REMOTE                  0x60D80
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PADDING_VALUE_S                    0x60D84
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PADDING_VALUE_L                    0x60D88
+
+#define mmMME0_CTRL_SHADOW_3_DESC_META_DATA_AGU_S                    0x60D8C
+
+#define mmMME0_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_LOCAL              0x60D90
+
+#define mmMME0_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_REMOTE             0x60D94
+
+#define mmMME0_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_LOCAL              0x60D98
+
+#define mmMME0_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_REMOTE             0x60D9C
+
+#define mmMME0_CTRL_SHADOW_3_DESC_PCU_RL_SATURATION                  0x60DA0
+
+#define mmMME0_CTRL_SHADOW_3_DESC_DUMMY                              0x60DA4
+
+#endif /* ASIC_REG_MME0_CTRL_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_masks.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_masks.h
new file mode 100644
index 000000000000..e6dd30ce0ca7
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_masks.h
@@ -0,0 +1,800 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MME0_QM_MASKS_H_
+#define ASIC_REG_MME0_QM_MASKS_H_
+
+/*
+ *****************************************
+ *   MME0_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+/* MME0_QM_GLBL_CFG0 */
+#define MME0_QM_GLBL_CFG0_PQF_EN_SHIFT                               0
+#define MME0_QM_GLBL_CFG0_PQF_EN_MASK                                0xF
+#define MME0_QM_GLBL_CFG0_CQF_EN_SHIFT                               4
+#define MME0_QM_GLBL_CFG0_CQF_EN_MASK                                0x1F0
+#define MME0_QM_GLBL_CFG0_CP_EN_SHIFT                                9
+#define MME0_QM_GLBL_CFG0_CP_EN_MASK                                 0x3E00
+
+/* MME0_QM_GLBL_CFG1 */
+#define MME0_QM_GLBL_CFG1_PQF_STOP_SHIFT                             0
+#define MME0_QM_GLBL_CFG1_PQF_STOP_MASK                              0xF
+#define MME0_QM_GLBL_CFG1_CQF_STOP_SHIFT                             4
+#define MME0_QM_GLBL_CFG1_CQF_STOP_MASK                              0x1F0
+#define MME0_QM_GLBL_CFG1_CP_STOP_SHIFT                              9
+#define MME0_QM_GLBL_CFG1_CP_STOP_MASK                               0x3E00
+#define MME0_QM_GLBL_CFG1_PQF_FLUSH_SHIFT                            16
+#define MME0_QM_GLBL_CFG1_PQF_FLUSH_MASK                             0xF0000
+#define MME0_QM_GLBL_CFG1_CQF_FLUSH_SHIFT                            20
+#define MME0_QM_GLBL_CFG1_CQF_FLUSH_MASK                             0x1F00000
+#define MME0_QM_GLBL_CFG1_CP_FLUSH_SHIFT                             25
+#define MME0_QM_GLBL_CFG1_CP_FLUSH_MASK                              0x3E000000
+
+/* MME0_QM_GLBL_PROT */
+#define MME0_QM_GLBL_PROT_PQF_SHIFT                                  0
+#define MME0_QM_GLBL_PROT_PQF_MASK                                   0xF
+#define MME0_QM_GLBL_PROT_CQF_SHIFT                                  4
+#define MME0_QM_GLBL_PROT_CQF_MASK                                   0x1F0
+#define MME0_QM_GLBL_PROT_CP_SHIFT                                   9
+#define MME0_QM_GLBL_PROT_CP_MASK                                    0x3E00
+#define MME0_QM_GLBL_PROT_ERR_SHIFT                                  14
+#define MME0_QM_GLBL_PROT_ERR_MASK                                   0x4000
+#define MME0_QM_GLBL_PROT_ARB_SHIFT                                  15
+#define MME0_QM_GLBL_PROT_ARB_MASK                                   0x8000
+
+/* MME0_QM_GLBL_ERR_CFG */
+#define MME0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_SHIFT                    0
+#define MME0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK                     0xF
+#define MME0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_SHIFT                    4
+#define MME0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_MASK                     0x1F0
+#define MME0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_SHIFT                     9
+#define MME0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_MASK                      0x3E00
+#define MME0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT                   16
+#define MME0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK                    0xF0000
+#define MME0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT                   20
+#define MME0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK                    0x1F00000
+#define MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT                    25
+#define MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK                     0x3E000000
+#define MME0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_SHIFT                   31
+#define MME0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK                    0x80000000
+
+/* MME0_QM_GLBL_SECURE_PROPS */
+#define MME0_QM_GLBL_SECURE_PROPS_0_ASID_SHIFT                       0
+#define MME0_QM_GLBL_SECURE_PROPS_0_ASID_MASK                        0x3FF
+#define MME0_QM_GLBL_SECURE_PROPS_1_ASID_SHIFT                       0
+#define MME0_QM_GLBL_SECURE_PROPS_1_ASID_MASK                        0x3FF
+#define MME0_QM_GLBL_SECURE_PROPS_2_ASID_SHIFT                       0
+#define MME0_QM_GLBL_SECURE_PROPS_2_ASID_MASK                        0x3FF
+#define MME0_QM_GLBL_SECURE_PROPS_3_ASID_SHIFT                       0
+#define MME0_QM_GLBL_SECURE_PROPS_3_ASID_MASK                        0x3FF
+#define MME0_QM_GLBL_SECURE_PROPS_4_ASID_SHIFT                       0
+#define MME0_QM_GLBL_SECURE_PROPS_4_ASID_MASK                        0x3FF
+#define MME0_QM_GLBL_SECURE_PROPS_0_MMBP_SHIFT                       10
+#define MME0_QM_GLBL_SECURE_PROPS_0_MMBP_MASK                        0x400
+#define MME0_QM_GLBL_SECURE_PROPS_1_MMBP_SHIFT                       10
+#define MME0_QM_GLBL_SECURE_PROPS_1_MMBP_MASK                        0x400
+#define MME0_QM_GLBL_SECURE_PROPS_2_MMBP_SHIFT                       10
+#define MME0_QM_GLBL_SECURE_PROPS_2_MMBP_MASK                        0x400
+#define MME0_QM_GLBL_SECURE_PROPS_3_MMBP_SHIFT                       10
+#define MME0_QM_GLBL_SECURE_PROPS_3_MMBP_MASK                        0x400
+#define MME0_QM_GLBL_SECURE_PROPS_4_MMBP_SHIFT                       10
+#define MME0_QM_GLBL_SECURE_PROPS_4_MMBP_MASK                        0x400
+
+/* MME0_QM_GLBL_NON_SECURE_PROPS */
+#define MME0_QM_GLBL_NON_SECURE_PROPS_0_ASID_SHIFT                   0
+#define MME0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK                    0x3FF
+#define MME0_QM_GLBL_NON_SECURE_PROPS_1_ASID_SHIFT                   0
+#define MME0_QM_GLBL_NON_SECURE_PROPS_1_ASID_MASK                    0x3FF
+#define MME0_QM_GLBL_NON_SECURE_PROPS_2_ASID_SHIFT                   0
+#define MME0_QM_GLBL_NON_SECURE_PROPS_2_ASID_MASK                    0x3FF
+#define MME0_QM_GLBL_NON_SECURE_PROPS_3_ASID_SHIFT                   0
+#define MME0_QM_GLBL_NON_SECURE_PROPS_3_ASID_MASK                    0x3FF
+#define MME0_QM_GLBL_NON_SECURE_PROPS_4_ASID_SHIFT                   0
+#define MME0_QM_GLBL_NON_SECURE_PROPS_4_ASID_MASK                    0x3FF
+#define MME0_QM_GLBL_NON_SECURE_PROPS_0_MMBP_SHIFT                   10
+#define MME0_QM_GLBL_NON_SECURE_PROPS_0_MMBP_MASK                    0x400
+#define MME0_QM_GLBL_NON_SECURE_PROPS_1_MMBP_SHIFT                   10
+#define MME0_QM_GLBL_NON_SECURE_PROPS_1_MMBP_MASK                    0x400
+#define MME0_QM_GLBL_NON_SECURE_PROPS_2_MMBP_SHIFT                   10
+#define MME0_QM_GLBL_NON_SECURE_PROPS_2_MMBP_MASK                    0x400
+#define MME0_QM_GLBL_NON_SECURE_PROPS_3_MMBP_SHIFT                   10
+#define MME0_QM_GLBL_NON_SECURE_PROPS_3_MMBP_MASK                    0x400
+#define MME0_QM_GLBL_NON_SECURE_PROPS_4_MMBP_SHIFT                   10
+#define MME0_QM_GLBL_NON_SECURE_PROPS_4_MMBP_MASK                    0x400
+
+/* MME0_QM_GLBL_STS0 */
+#define MME0_QM_GLBL_STS0_PQF_IDLE_SHIFT                             0
+#define MME0_QM_GLBL_STS0_PQF_IDLE_MASK                              0xF
+#define MME0_QM_GLBL_STS0_CQF_IDLE_SHIFT                             4
+#define MME0_QM_GLBL_STS0_CQF_IDLE_MASK                              0x1F0
+#define MME0_QM_GLBL_STS0_CP_IDLE_SHIFT                              9
+#define MME0_QM_GLBL_STS0_CP_IDLE_MASK                               0x3E00
+#define MME0_QM_GLBL_STS0_PQF_IS_STOP_SHIFT                          16
+#define MME0_QM_GLBL_STS0_PQF_IS_STOP_MASK                           0xF0000
+#define MME0_QM_GLBL_STS0_CQF_IS_STOP_SHIFT                          20
+#define MME0_QM_GLBL_STS0_CQF_IS_STOP_MASK                           0x1F00000
+#define MME0_QM_GLBL_STS0_CP_IS_STOP_SHIFT                           25
+#define MME0_QM_GLBL_STS0_CP_IS_STOP_MASK                            0x3E000000
+#define MME0_QM_GLBL_STS0_ARB_IS_STOP_SHIFT                          31
+#define MME0_QM_GLBL_STS0_ARB_IS_STOP_MASK                           0x80000000
+
+/* MME0_QM_GLBL_STS1 */
+#define MME0_QM_GLBL_STS1_PQF_RD_ERR_SHIFT                           0
+#define MME0_QM_GLBL_STS1_PQF_RD_ERR_MASK                            0x1
+#define MME0_QM_GLBL_STS1_CQF_RD_ERR_SHIFT                           1
+#define MME0_QM_GLBL_STS1_CQF_RD_ERR_MASK                            0x2
+#define MME0_QM_GLBL_STS1_CP_RD_ERR_SHIFT                            2
+#define MME0_QM_GLBL_STS1_CP_RD_ERR_MASK                             0x4
+#define MME0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_SHIFT                     3
+#define MME0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_MASK                      0x8
+#define MME0_QM_GLBL_STS1_CP_STOP_OP_SHIFT                           4
+#define MME0_QM_GLBL_STS1_CP_STOP_OP_MASK                            0x10
+#define MME0_QM_GLBL_STS1_CP_MSG_WR_ERR_SHIFT                        5
+#define MME0_QM_GLBL_STS1_CP_MSG_WR_ERR_MASK                         0x20
+#define MME0_QM_GLBL_STS1_CP_WREG_ERR_SHIFT                          6
+#define MME0_QM_GLBL_STS1_CP_WREG_ERR_MASK                           0x40
+#define MME0_QM_GLBL_STS1_CP_FENCE0_OVF_ERR_SHIFT                    8
+#define MME0_QM_GLBL_STS1_CP_FENCE0_OVF_ERR_MASK                     0x100
+#define MME0_QM_GLBL_STS1_CP_FENCE1_OVF_ERR_SHIFT                    9
+#define MME0_QM_GLBL_STS1_CP_FENCE1_OVF_ERR_MASK                     0x200
+#define MME0_QM_GLBL_STS1_CP_FENCE2_OVF_ERR_SHIFT                    10
+#define MME0_QM_GLBL_STS1_CP_FENCE2_OVF_ERR_MASK                     0x400
+#define MME0_QM_GLBL_STS1_CP_FENCE3_OVF_ERR_SHIFT                    11
+#define MME0_QM_GLBL_STS1_CP_FENCE3_OVF_ERR_MASK                     0x800
+#define MME0_QM_GLBL_STS1_CP_FENCE0_UDF_ERR_SHIFT                    12
+#define MME0_QM_GLBL_STS1_CP_FENCE0_UDF_ERR_MASK                     0x1000
+#define MME0_QM_GLBL_STS1_CP_FENCE1_UDF_ERR_SHIFT                    13
+#define MME0_QM_GLBL_STS1_CP_FENCE1_UDF_ERR_MASK                     0x2000
+#define MME0_QM_GLBL_STS1_CP_FENCE2_UDF_ERR_SHIFT                    14
+#define MME0_QM_GLBL_STS1_CP_FENCE2_UDF_ERR_MASK                     0x4000
+#define MME0_QM_GLBL_STS1_CP_FENCE3_UDF_ERR_SHIFT                    15
+#define MME0_QM_GLBL_STS1_CP_FENCE3_UDF_ERR_MASK                     0x8000
+
+/* MME0_QM_GLBL_STS1_4 */
+#define MME0_QM_GLBL_STS1_4_CQF_RD_ERR_SHIFT                         1
+#define MME0_QM_GLBL_STS1_4_CQF_RD_ERR_MASK                          0x2
+#define MME0_QM_GLBL_STS1_4_CP_RD_ERR_SHIFT                          2
+#define MME0_QM_GLBL_STS1_4_CP_RD_ERR_MASK                           0x4
+#define MME0_QM_GLBL_STS1_4_CP_UNDEF_CMD_ERR_SHIFT                   3
+#define MME0_QM_GLBL_STS1_4_CP_UNDEF_CMD_ERR_MASK                    0x8
+#define MME0_QM_GLBL_STS1_4_CP_STOP_OP_SHIFT                         4
+#define MME0_QM_GLBL_STS1_4_CP_STOP_OP_MASK                          0x10
+#define MME0_QM_GLBL_STS1_4_CP_MSG_WR_ERR_SHIFT                      5
+#define MME0_QM_GLBL_STS1_4_CP_MSG_WR_ERR_MASK                       0x20
+#define MME0_QM_GLBL_STS1_4_CP_WREG_ERR_SHIFT                        6
+#define MME0_QM_GLBL_STS1_4_CP_WREG_ERR_MASK                         0x40
+#define MME0_QM_GLBL_STS1_4_CP_FENCE0_OVF_ERR_SHIFT                  8
+#define MME0_QM_GLBL_STS1_4_CP_FENCE0_OVF_ERR_MASK                   0x100
+#define MME0_QM_GLBL_STS1_4_CP_FENCE1_OVF_ERR_SHIFT                  9
+#define MME0_QM_GLBL_STS1_4_CP_FENCE1_OVF_ERR_MASK                   0x200
+#define MME0_QM_GLBL_STS1_4_CP_FENCE2_OVF_ERR_SHIFT                  10
+#define MME0_QM_GLBL_STS1_4_CP_FENCE2_OVF_ERR_MASK                   0x400
+#define MME0_QM_GLBL_STS1_4_CP_FENCE3_OVF_ERR_SHIFT                  11
+#define MME0_QM_GLBL_STS1_4_CP_FENCE3_OVF_ERR_MASK                   0x800
+#define MME0_QM_GLBL_STS1_4_CP_FENCE0_UDF_ERR_SHIFT                  12
+#define MME0_QM_GLBL_STS1_4_CP_FENCE0_UDF_ERR_MASK                   0x1000
+#define MME0_QM_GLBL_STS1_4_CP_FENCE1_UDF_ERR_SHIFT                  13
+#define MME0_QM_GLBL_STS1_4_CP_FENCE1_UDF_ERR_MASK                   0x2000
+#define MME0_QM_GLBL_STS1_4_CP_FENCE2_UDF_ERR_SHIFT                  14
+#define MME0_QM_GLBL_STS1_4_CP_FENCE2_UDF_ERR_MASK                   0x4000
+#define MME0_QM_GLBL_STS1_4_CP_FENCE3_UDF_ERR_SHIFT                  15
+#define MME0_QM_GLBL_STS1_4_CP_FENCE3_UDF_ERR_MASK                   0x8000
+
+/* MME0_QM_GLBL_MSG_EN */
+#define MME0_QM_GLBL_MSG_EN_PQF_RD_ERR_SHIFT                         0
+#define MME0_QM_GLBL_MSG_EN_PQF_RD_ERR_MASK                          0x1
+#define MME0_QM_GLBL_MSG_EN_CQF_RD_ERR_SHIFT                         1
+#define MME0_QM_GLBL_MSG_EN_CQF_RD_ERR_MASK                          0x2
+#define MME0_QM_GLBL_MSG_EN_CP_RD_ERR_SHIFT                          2
+#define MME0_QM_GLBL_MSG_EN_CP_RD_ERR_MASK                           0x4
+#define MME0_QM_GLBL_MSG_EN_CP_UNDEF_CMD_ERR_SHIFT                   3
+#define MME0_QM_GLBL_MSG_EN_CP_UNDEF_CMD_ERR_MASK                    0x8
+#define MME0_QM_GLBL_MSG_EN_CP_STOP_OP_SHIFT                         4
+#define MME0_QM_GLBL_MSG_EN_CP_STOP_OP_MASK                          0x10
+#define MME0_QM_GLBL_MSG_EN_CP_MSG_WR_ERR_SHIFT                      5
+#define MME0_QM_GLBL_MSG_EN_CP_MSG_WR_ERR_MASK                       0x20
+#define MME0_QM_GLBL_MSG_EN_CP_WREG_ERR_SHIFT                        6
+#define MME0_QM_GLBL_MSG_EN_CP_WREG_ERR_MASK                         0x40
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE0_OVF_ERR_SHIFT                  8
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE0_OVF_ERR_MASK                   0x100
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE1_OVF_ERR_SHIFT                  9
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE1_OVF_ERR_MASK                   0x200
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE2_OVF_ERR_SHIFT                  10
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE2_OVF_ERR_MASK                   0x400
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE3_OVF_ERR_SHIFT                  11
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE3_OVF_ERR_MASK                   0x800
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE0_UDF_ERR_SHIFT                  12
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE0_UDF_ERR_MASK                   0x1000
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE1_UDF_ERR_SHIFT                  13
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE1_UDF_ERR_MASK                   0x2000
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE2_UDF_ERR_SHIFT                  14
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE2_UDF_ERR_MASK                   0x4000
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE3_UDF_ERR_SHIFT                  15
+#define MME0_QM_GLBL_MSG_EN_CP_FENCE3_UDF_ERR_MASK                   0x8000
+
+/* MME0_QM_GLBL_MSG_EN_4 */
+#define MME0_QM_GLBL_MSG_EN_4_CQF_RD_ERR_SHIFT                       1
+#define MME0_QM_GLBL_MSG_EN_4_CQF_RD_ERR_MASK                        0x2
+#define MME0_QM_GLBL_MSG_EN_4_CP_RD_ERR_SHIFT                        2
+#define MME0_QM_GLBL_MSG_EN_4_CP_RD_ERR_MASK                         0x4
+#define MME0_QM_GLBL_MSG_EN_4_CP_UNDEF_CMD_ERR_SHIFT                 3
+#define MME0_QM_GLBL_MSG_EN_4_CP_UNDEF_CMD_ERR_MASK                  0x8
+#define MME0_QM_GLBL_MSG_EN_4_CP_STOP_OP_SHIFT                       4
+#define MME0_QM_GLBL_MSG_EN_4_CP_STOP_OP_MASK                        0x10
+#define MME0_QM_GLBL_MSG_EN_4_CP_MSG_WR_ERR_SHIFT                    5
+#define MME0_QM_GLBL_MSG_EN_4_CP_MSG_WR_ERR_MASK                     0x20
+#define MME0_QM_GLBL_MSG_EN_4_CP_WREG_ERR_SHIFT                      6
+#define MME0_QM_GLBL_MSG_EN_4_CP_WREG_ERR_MASK                       0x40
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE0_OVF_ERR_SHIFT                8
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE0_OVF_ERR_MASK                 0x100
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE1_OVF_ERR_SHIFT                9
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE1_OVF_ERR_MASK                 0x200
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE2_OVF_ERR_SHIFT                10
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE2_OVF_ERR_MASK                 0x400
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE3_OVF_ERR_SHIFT                11
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE3_OVF_ERR_MASK                 0x800
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE0_UDF_ERR_SHIFT                12
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE0_UDF_ERR_MASK                 0x1000
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE1_UDF_ERR_SHIFT                13
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE1_UDF_ERR_MASK                 0x2000
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE2_UDF_ERR_SHIFT                14
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE2_UDF_ERR_MASK                 0x4000
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE3_UDF_ERR_SHIFT                15
+#define MME0_QM_GLBL_MSG_EN_4_CP_FENCE3_UDF_ERR_MASK                 0x8000
+
+/* MME0_QM_PQ_BASE_LO */
+#define MME0_QM_PQ_BASE_LO_VAL_SHIFT                                 0
+#define MME0_QM_PQ_BASE_LO_VAL_MASK                                  0xFFFFFFFF
+
+/* MME0_QM_PQ_BASE_HI */
+#define MME0_QM_PQ_BASE_HI_VAL_SHIFT                                 0
+#define MME0_QM_PQ_BASE_HI_VAL_MASK                                  0xFFFFFFFF
+
+/* MME0_QM_PQ_SIZE */
+#define MME0_QM_PQ_SIZE_VAL_SHIFT                                    0
+#define MME0_QM_PQ_SIZE_VAL_MASK                                     0xFFFFFFFF
+
+/* MME0_QM_PQ_PI */
+#define MME0_QM_PQ_PI_VAL_SHIFT                                      0
+#define MME0_QM_PQ_PI_VAL_MASK                                       0xFFFFFFFF
+
+/* MME0_QM_PQ_CI */
+#define MME0_QM_PQ_CI_VAL_SHIFT                                      0
+#define MME0_QM_PQ_CI_VAL_MASK                                       0xFFFFFFFF
+
+/* MME0_QM_PQ_CFG0 */
+#define MME0_QM_PQ_CFG0_RESERVED_SHIFT                               0
+#define MME0_QM_PQ_CFG0_RESERVED_MASK                                0x1
+
+/* MME0_QM_PQ_CFG1 */
+#define MME0_QM_PQ_CFG1_CREDIT_LIM_SHIFT                             0
+#define MME0_QM_PQ_CFG1_CREDIT_LIM_MASK                              0xFFFF
+#define MME0_QM_PQ_CFG1_MAX_INFLIGHT_SHIFT                           16
+#define MME0_QM_PQ_CFG1_MAX_INFLIGHT_MASK                            0xFFFF0000
+
+/* MME0_QM_PQ_ARUSER_31_11 */
+#define MME0_QM_PQ_ARUSER_31_11_VAL_SHIFT                            0
+#define MME0_QM_PQ_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* MME0_QM_PQ_STS0 */
+#define MME0_QM_PQ_STS0_PQ_CREDIT_CNT_SHIFT                          0
+#define MME0_QM_PQ_STS0_PQ_CREDIT_CNT_MASK                           0xFFFF
+#define MME0_QM_PQ_STS0_PQ_FREE_CNT_SHIFT                            16
+#define MME0_QM_PQ_STS0_PQ_FREE_CNT_MASK                             0xFFFF0000
+
+/* MME0_QM_PQ_STS1 */
+#define MME0_QM_PQ_STS1_PQ_INFLIGHT_CNT_SHIFT                        0
+#define MME0_QM_PQ_STS1_PQ_INFLIGHT_CNT_MASK                         0xFFFF
+#define MME0_QM_PQ_STS1_PQ_BUF_EMPTY_SHIFT                           30
+#define MME0_QM_PQ_STS1_PQ_BUF_EMPTY_MASK                            0x40000000
+#define MME0_QM_PQ_STS1_PQ_BUSY_SHIFT                                31
+#define MME0_QM_PQ_STS1_PQ_BUSY_MASK                                 0x80000000
+
+/* MME0_QM_CQ_CFG0 */
+#define MME0_QM_CQ_CFG0_RESERVED_SHIFT                               0
+#define MME0_QM_CQ_CFG0_RESERVED_MASK                                0x1
+
+/* MME0_QM_CQ_CFG1 */
+#define MME0_QM_CQ_CFG1_CREDIT_LIM_SHIFT                             0
+#define MME0_QM_CQ_CFG1_CREDIT_LIM_MASK                              0xFFFF
+#define MME0_QM_CQ_CFG1_MAX_INFLIGHT_SHIFT                           16
+#define MME0_QM_CQ_CFG1_MAX_INFLIGHT_MASK                            0xFFFF0000
+
+/* MME0_QM_CQ_ARUSER_31_11 */
+#define MME0_QM_CQ_ARUSER_31_11_VAL_SHIFT                            0
+#define MME0_QM_CQ_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* MME0_QM_CQ_STS0 */
+#define MME0_QM_CQ_STS0_CQ_CREDIT_CNT_SHIFT                          0
+#define MME0_QM_CQ_STS0_CQ_CREDIT_CNT_MASK                           0xFFFF
+#define MME0_QM_CQ_STS0_CQ_FREE_CNT_SHIFT                            16
+#define MME0_QM_CQ_STS0_CQ_FREE_CNT_MASK                             0xFFFF0000
+
+/* MME0_QM_CQ_STS1 */
+#define MME0_QM_CQ_STS1_CQ_INFLIGHT_CNT_SHIFT                        0
+#define MME0_QM_CQ_STS1_CQ_INFLIGHT_CNT_MASK                         0xFFFF
+#define MME0_QM_CQ_STS1_CQ_BUF_EMPTY_SHIFT                           30
+#define MME0_QM_CQ_STS1_CQ_BUF_EMPTY_MASK                            0x40000000
+#define MME0_QM_CQ_STS1_CQ_BUSY_SHIFT                                31
+#define MME0_QM_CQ_STS1_CQ_BUSY_MASK                                 0x80000000
+
+/* MME0_QM_CQ_PTR_LO_0 */
+#define MME0_QM_CQ_PTR_LO_0_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_LO_0_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_PTR_HI_0 */
+#define MME0_QM_CQ_PTR_HI_0_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_HI_0_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_TSIZE_0 */
+#define MME0_QM_CQ_TSIZE_0_VAL_SHIFT                                 0
+#define MME0_QM_CQ_TSIZE_0_VAL_MASK                                  0xFFFFFFFF
+
+/* MME0_QM_CQ_CTL_0 */
+#define MME0_QM_CQ_CTL_0_RPT_SHIFT                                   0
+#define MME0_QM_CQ_CTL_0_RPT_MASK                                    0xFFFF
+#define MME0_QM_CQ_CTL_0_CTL_SHIFT                                   16
+#define MME0_QM_CQ_CTL_0_CTL_MASK                                    0xFFFF0000
+
+/* MME0_QM_CQ_PTR_LO_1 */
+#define MME0_QM_CQ_PTR_LO_1_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_LO_1_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_PTR_HI_1 */
+#define MME0_QM_CQ_PTR_HI_1_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_HI_1_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_TSIZE_1 */
+#define MME0_QM_CQ_TSIZE_1_VAL_SHIFT                                 0
+#define MME0_QM_CQ_TSIZE_1_VAL_MASK                                  0xFFFFFFFF
+
+/* MME0_QM_CQ_CTL_1 */
+#define MME0_QM_CQ_CTL_1_RPT_SHIFT                                   0
+#define MME0_QM_CQ_CTL_1_RPT_MASK                                    0xFFFF
+#define MME0_QM_CQ_CTL_1_CTL_SHIFT                                   16
+#define MME0_QM_CQ_CTL_1_CTL_MASK                                    0xFFFF0000
+
+/* MME0_QM_CQ_PTR_LO_2 */
+#define MME0_QM_CQ_PTR_LO_2_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_LO_2_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_PTR_HI_2 */
+#define MME0_QM_CQ_PTR_HI_2_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_HI_2_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_TSIZE_2 */
+#define MME0_QM_CQ_TSIZE_2_VAL_SHIFT                                 0
+#define MME0_QM_CQ_TSIZE_2_VAL_MASK                                  0xFFFFFFFF
+
+/* MME0_QM_CQ_CTL_2 */
+#define MME0_QM_CQ_CTL_2_RPT_SHIFT                                   0
+#define MME0_QM_CQ_CTL_2_RPT_MASK                                    0xFFFF
+#define MME0_QM_CQ_CTL_2_CTL_SHIFT                                   16
+#define MME0_QM_CQ_CTL_2_CTL_MASK                                    0xFFFF0000
+
+/* MME0_QM_CQ_PTR_LO_3 */
+#define MME0_QM_CQ_PTR_LO_3_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_LO_3_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_PTR_HI_3 */
+#define MME0_QM_CQ_PTR_HI_3_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_HI_3_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_TSIZE_3 */
+#define MME0_QM_CQ_TSIZE_3_VAL_SHIFT                                 0
+#define MME0_QM_CQ_TSIZE_3_VAL_MASK                                  0xFFFFFFFF
+
+/* MME0_QM_CQ_CTL_3 */
+#define MME0_QM_CQ_CTL_3_RPT_SHIFT                                   0
+#define MME0_QM_CQ_CTL_3_RPT_MASK                                    0xFFFF
+#define MME0_QM_CQ_CTL_3_CTL_SHIFT                                   16
+#define MME0_QM_CQ_CTL_3_CTL_MASK                                    0xFFFF0000
+
+/* MME0_QM_CQ_PTR_LO_4 */
+#define MME0_QM_CQ_PTR_LO_4_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_LO_4_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_PTR_HI_4 */
+#define MME0_QM_CQ_PTR_HI_4_VAL_SHIFT                                0
+#define MME0_QM_CQ_PTR_HI_4_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_CQ_TSIZE_4 */
+#define MME0_QM_CQ_TSIZE_4_VAL_SHIFT                                 0
+#define MME0_QM_CQ_TSIZE_4_VAL_MASK                                  0xFFFFFFFF
+
+/* MME0_QM_CQ_CTL_4 */
+#define MME0_QM_CQ_CTL_4_RPT_SHIFT                                   0
+#define MME0_QM_CQ_CTL_4_RPT_MASK                                    0xFFFF
+#define MME0_QM_CQ_CTL_4_CTL_SHIFT                                   16
+#define MME0_QM_CQ_CTL_4_CTL_MASK                                    0xFFFF0000
+
+/* MME0_QM_CQ_PTR_LO_STS */
+#define MME0_QM_CQ_PTR_LO_STS_VAL_SHIFT                              0
+#define MME0_QM_CQ_PTR_LO_STS_VAL_MASK                               0xFFFFFFFF
+
+/* MME0_QM_CQ_PTR_HI_STS */
+#define MME0_QM_CQ_PTR_HI_STS_VAL_SHIFT                              0
+#define MME0_QM_CQ_PTR_HI_STS_VAL_MASK                               0xFFFFFFFF
+
+/* MME0_QM_CQ_TSIZE_STS */
+#define MME0_QM_CQ_TSIZE_STS_VAL_SHIFT                               0
+#define MME0_QM_CQ_TSIZE_STS_VAL_MASK                                0xFFFFFFFF
+
+/* MME0_QM_CQ_CTL_STS */
+#define MME0_QM_CQ_CTL_STS_RPT_SHIFT                                 0
+#define MME0_QM_CQ_CTL_STS_RPT_MASK                                  0xFFFF
+#define MME0_QM_CQ_CTL_STS_CTL_SHIFT                                 16
+#define MME0_QM_CQ_CTL_STS_CTL_MASK                                  0xFFFF0000
+
+/* MME0_QM_CQ_IFIFO_CNT */
+#define MME0_QM_CQ_IFIFO_CNT_VAL_SHIFT                               0
+#define MME0_QM_CQ_IFIFO_CNT_VAL_MASK                                0x3
+
+/* MME0_QM_CP_MSG_BASE0_ADDR_LO */
+#define MME0_QM_CP_MSG_BASE0_ADDR_LO_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE0_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_MSG_BASE0_ADDR_HI */
+#define MME0_QM_CP_MSG_BASE0_ADDR_HI_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE0_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_MSG_BASE1_ADDR_LO */
+#define MME0_QM_CP_MSG_BASE1_ADDR_LO_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE1_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_MSG_BASE1_ADDR_HI */
+#define MME0_QM_CP_MSG_BASE1_ADDR_HI_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE1_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_MSG_BASE2_ADDR_LO */
+#define MME0_QM_CP_MSG_BASE2_ADDR_LO_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE2_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_MSG_BASE2_ADDR_HI */
+#define MME0_QM_CP_MSG_BASE2_ADDR_HI_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE2_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_MSG_BASE3_ADDR_LO */
+#define MME0_QM_CP_MSG_BASE3_ADDR_LO_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE3_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_MSG_BASE3_ADDR_HI */
+#define MME0_QM_CP_MSG_BASE3_ADDR_HI_VAL_SHIFT                       0
+#define MME0_QM_CP_MSG_BASE3_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_LDMA_TSIZE_OFFSET */
+#define MME0_QM_CP_LDMA_TSIZE_OFFSET_VAL_SHIFT                       0
+#define MME0_QM_CP_LDMA_TSIZE_OFFSET_VAL_MASK                        0xFFFFFFFF
+
+/* MME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET */
+#define MME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_VAL_SHIFT                 0
+#define MME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_VAL_MASK                  0xFFFFFFFF
+
+/* MME0_QM_CP_LDMA_DST_BASE_LO_OFFSET */
+#define MME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_VAL_SHIFT                 0
+#define MME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_VAL_MASK                  0xFFFFFFFF
+
+/* MME0_QM_CP_FENCE0_RDATA */
+#define MME0_QM_CP_FENCE0_RDATA_INC_VAL_SHIFT                        0
+#define MME0_QM_CP_FENCE0_RDATA_INC_VAL_MASK                         0xF
+
+/* MME0_QM_CP_FENCE1_RDATA */
+#define MME0_QM_CP_FENCE1_RDATA_INC_VAL_SHIFT                        0
+#define MME0_QM_CP_FENCE1_RDATA_INC_VAL_MASK                         0xF
+
+/* MME0_QM_CP_FENCE2_RDATA */
+#define MME0_QM_CP_FENCE2_RDATA_INC_VAL_SHIFT                        0
+#define MME0_QM_CP_FENCE2_RDATA_INC_VAL_MASK                         0xF
+
+/* MME0_QM_CP_FENCE3_RDATA */
+#define MME0_QM_CP_FENCE3_RDATA_INC_VAL_SHIFT                        0
+#define MME0_QM_CP_FENCE3_RDATA_INC_VAL_MASK                         0xF
+
+/* MME0_QM_CP_FENCE0_CNT */
+#define MME0_QM_CP_FENCE0_CNT_VAL_SHIFT                              0
+#define MME0_QM_CP_FENCE0_CNT_VAL_MASK                               0x3FFF
+
+/* MME0_QM_CP_FENCE1_CNT */
+#define MME0_QM_CP_FENCE1_CNT_VAL_SHIFT                              0
+#define MME0_QM_CP_FENCE1_CNT_VAL_MASK                               0x3FFF
+
+/* MME0_QM_CP_FENCE2_CNT */
+#define MME0_QM_CP_FENCE2_CNT_VAL_SHIFT                              0
+#define MME0_QM_CP_FENCE2_CNT_VAL_MASK                               0x3FFF
+
+/* MME0_QM_CP_FENCE3_CNT */
+#define MME0_QM_CP_FENCE3_CNT_VAL_SHIFT                              0
+#define MME0_QM_CP_FENCE3_CNT_VAL_MASK                               0x3FFF
+
+/* MME0_QM_CP_STS */
+#define MME0_QM_CP_STS_MSG_INFLIGHT_CNT_SHIFT                        0
+#define MME0_QM_CP_STS_MSG_INFLIGHT_CNT_MASK                         0xFFFF
+#define MME0_QM_CP_STS_ERDY_SHIFT                                    16
+#define MME0_QM_CP_STS_ERDY_MASK                                     0x10000
+#define MME0_QM_CP_STS_RRDY_SHIFT                                    17
+#define MME0_QM_CP_STS_RRDY_MASK                                     0x20000
+#define MME0_QM_CP_STS_MRDY_SHIFT                                    18
+#define MME0_QM_CP_STS_MRDY_MASK                                     0x40000
+#define MME0_QM_CP_STS_SW_STOP_SHIFT                                 19
+#define MME0_QM_CP_STS_SW_STOP_MASK                                  0x80000
+#define MME0_QM_CP_STS_FENCE_ID_SHIFT                                20
+#define MME0_QM_CP_STS_FENCE_ID_MASK                                 0x300000
+#define MME0_QM_CP_STS_FENCE_IN_PROGRESS_SHIFT                       22
+#define MME0_QM_CP_STS_FENCE_IN_PROGRESS_MASK                        0x400000
+
+/* MME0_QM_CP_CURRENT_INST_LO */
+#define MME0_QM_CP_CURRENT_INST_LO_VAL_SHIFT                         0
+#define MME0_QM_CP_CURRENT_INST_LO_VAL_MASK                          0xFFFFFFFF
+
+/* MME0_QM_CP_CURRENT_INST_HI */
+#define MME0_QM_CP_CURRENT_INST_HI_VAL_SHIFT                         0
+#define MME0_QM_CP_CURRENT_INST_HI_VAL_MASK                          0xFFFFFFFF
+
+/* MME0_QM_CP_BARRIER_CFG */
+#define MME0_QM_CP_BARRIER_CFG_EBGUARD_SHIFT                         0
+#define MME0_QM_CP_BARRIER_CFG_EBGUARD_MASK                          0xFFF
+#define MME0_QM_CP_BARRIER_CFG_RBGUARD_SHIFT                         16
+#define MME0_QM_CP_BARRIER_CFG_RBGUARD_MASK                          0xF0000
+
+/* MME0_QM_CP_DBG_0 */
+#define MME0_QM_CP_DBG_0_CS_SHIFT                                    0
+#define MME0_QM_CP_DBG_0_CS_MASK                                     0xF
+#define MME0_QM_CP_DBG_0_EB_CNT_NOT_ZERO_SHIFT                       4
+#define MME0_QM_CP_DBG_0_EB_CNT_NOT_ZERO_MASK                        0x10
+#define MME0_QM_CP_DBG_0_BULK_CNT_NOT_ZERO_SHIFT                     5
+#define MME0_QM_CP_DBG_0_BULK_CNT_NOT_ZERO_MASK                      0x20
+#define MME0_QM_CP_DBG_0_MREB_STALL_SHIFT                            6
+#define MME0_QM_CP_DBG_0_MREB_STALL_MASK                             0x40
+#define MME0_QM_CP_DBG_0_STALL_SHIFT                                 7
+#define MME0_QM_CP_DBG_0_STALL_MASK                                  0x80
+
+/* MME0_QM_CP_ARUSER_31_11 */
+#define MME0_QM_CP_ARUSER_31_11_VAL_SHIFT                            0
+#define MME0_QM_CP_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* MME0_QM_CP_AWUSER_31_11 */
+#define MME0_QM_CP_AWUSER_31_11_VAL_SHIFT                            0
+#define MME0_QM_CP_AWUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* MME0_QM_ARB_CFG_0 */
+#define MME0_QM_ARB_CFG_0_TYPE_SHIFT                                 0
+#define MME0_QM_ARB_CFG_0_TYPE_MASK                                  0x1
+#define MME0_QM_ARB_CFG_0_IS_MASTER_SHIFT                            4
+#define MME0_QM_ARB_CFG_0_IS_MASTER_MASK                             0x10
+#define MME0_QM_ARB_CFG_0_EN_SHIFT                                   8
+#define MME0_QM_ARB_CFG_0_EN_MASK                                    0x100
+#define MME0_QM_ARB_CFG_0_MASK_SHIFT                                 12
+#define MME0_QM_ARB_CFG_0_MASK_MASK                                  0xF000
+#define MME0_QM_ARB_CFG_0_MST_MSG_NOSTALL_SHIFT                      16
+#define MME0_QM_ARB_CFG_0_MST_MSG_NOSTALL_MASK                       0x10000
+
+/* MME0_QM_ARB_CHOISE_Q_PUSH */
+#define MME0_QM_ARB_CHOISE_Q_PUSH_VAL_SHIFT                          0
+#define MME0_QM_ARB_CHOISE_Q_PUSH_VAL_MASK                           0x3
+
+/* MME0_QM_ARB_WRR_WEIGHT */
+#define MME0_QM_ARB_WRR_WEIGHT_VAL_SHIFT                             0
+#define MME0_QM_ARB_WRR_WEIGHT_VAL_MASK                              0xFFFFFFFF
+
+/* MME0_QM_ARB_CFG_1 */
+#define MME0_QM_ARB_CFG_1_CLR_SHIFT                                  0
+#define MME0_QM_ARB_CFG_1_CLR_MASK                                   0x1
+
+/* MME0_QM_ARB_MST_AVAIL_CRED */
+#define MME0_QM_ARB_MST_AVAIL_CRED_VAL_SHIFT                         0
+#define MME0_QM_ARB_MST_AVAIL_CRED_VAL_MASK                          0x7F
+
+/* MME0_QM_ARB_MST_CRED_INC */
+#define MME0_QM_ARB_MST_CRED_INC_VAL_SHIFT                           0
+#define MME0_QM_ARB_MST_CRED_INC_VAL_MASK                            0xFFFFFFFF
+
+/* MME0_QM_ARB_MST_CHOISE_PUSH_OFST */
+#define MME0_QM_ARB_MST_CHOISE_PUSH_OFST_VAL_SHIFT                   0
+#define MME0_QM_ARB_MST_CHOISE_PUSH_OFST_VAL_MASK                    0xFFFFFFFF
+
+/* MME0_QM_ARB_SLV_MASTER_INC_CRED_OFST */
+#define MME0_QM_ARB_SLV_MASTER_INC_CRED_OFST_VAL_SHIFT               0
+#define MME0_QM_ARB_SLV_MASTER_INC_CRED_OFST_VAL_MASK                0xFFFFFFFF
+
+/* MME0_QM_ARB_MST_SLAVE_EN */
+#define MME0_QM_ARB_MST_SLAVE_EN_VAL_SHIFT                           0
+#define MME0_QM_ARB_MST_SLAVE_EN_VAL_MASK                            0xFFFFFFFF
+
+/* MME0_QM_ARB_MST_QUIET_PER */
+#define MME0_QM_ARB_MST_QUIET_PER_VAL_SHIFT                          0
+#define MME0_QM_ARB_MST_QUIET_PER_VAL_MASK                           0xFFFFFFFF
+
+/* MME0_QM_ARB_SLV_CHOISE_WDT */
+#define MME0_QM_ARB_SLV_CHOISE_WDT_VAL_SHIFT                         0
+#define MME0_QM_ARB_SLV_CHOISE_WDT_VAL_MASK                          0xFFFFFFFF
+
+/* MME0_QM_ARB_SLV_ID */
+#define MME0_QM_ARB_SLV_ID_VAL_SHIFT                                 0
+#define MME0_QM_ARB_SLV_ID_VAL_MASK                                  0x1F
+
+/* MME0_QM_ARB_MSG_MAX_INFLIGHT */
+#define MME0_QM_ARB_MSG_MAX_INFLIGHT_VAL_SHIFT                       0
+#define MME0_QM_ARB_MSG_MAX_INFLIGHT_VAL_MASK                        0x3F
+
+/* MME0_QM_ARB_MSG_AWUSER_31_11 */
+#define MME0_QM_ARB_MSG_AWUSER_31_11_VAL_SHIFT                       0
+#define MME0_QM_ARB_MSG_AWUSER_31_11_VAL_MASK                        0x1FFFFF
+
+/* MME0_QM_ARB_MSG_AWUSER_SEC_PROP */
+#define MME0_QM_ARB_MSG_AWUSER_SEC_PROP_ASID_SHIFT                   0
+#define MME0_QM_ARB_MSG_AWUSER_SEC_PROP_ASID_MASK                    0x3FF
+#define MME0_QM_ARB_MSG_AWUSER_SEC_PROP_MMBP_SHIFT                   10
+#define MME0_QM_ARB_MSG_AWUSER_SEC_PROP_MMBP_MASK                    0x400
+
+/* MME0_QM_ARB_MSG_AWUSER_NON_SEC_PROP */
+#define MME0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_ASID_SHIFT               0
+#define MME0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_ASID_MASK                0x3FF
+#define MME0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_MMBP_SHIFT               10
+#define MME0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_MMBP_MASK                0x400
+
+/* MME0_QM_ARB_BASE_LO */
+#define MME0_QM_ARB_BASE_LO_VAL_SHIFT                                0
+#define MME0_QM_ARB_BASE_LO_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_ARB_BASE_HI */
+#define MME0_QM_ARB_BASE_HI_VAL_SHIFT                                0
+#define MME0_QM_ARB_BASE_HI_VAL_MASK                                 0xFFFFFFFF
+
+/* MME0_QM_ARB_STATE_STS */
+#define MME0_QM_ARB_STATE_STS_VAL_SHIFT                              0
+#define MME0_QM_ARB_STATE_STS_VAL_MASK                               0xFFFFFFFF
+
+/* MME0_QM_ARB_CHOISE_FULLNESS_STS */
+#define MME0_QM_ARB_CHOISE_FULLNESS_STS_VAL_SHIFT                    0
+#define MME0_QM_ARB_CHOISE_FULLNESS_STS_VAL_MASK                     0x7F
+
+/* MME0_QM_ARB_MSG_STS */
+#define MME0_QM_ARB_MSG_STS_FULL_SHIFT                               0
+#define MME0_QM_ARB_MSG_STS_FULL_MASK                                0x1
+#define MME0_QM_ARB_MSG_STS_NO_INFLIGHT_SHIFT                        1
+#define MME0_QM_ARB_MSG_STS_NO_INFLIGHT_MASK                         0x2
+
+/* MME0_QM_ARB_SLV_CHOISE_Q_HEAD */
+#define MME0_QM_ARB_SLV_CHOISE_Q_HEAD_VAL_SHIFT                      0
+#define MME0_QM_ARB_SLV_CHOISE_Q_HEAD_VAL_MASK                       0x3
+
+/* MME0_QM_ARB_ERR_CAUSE */
+#define MME0_QM_ARB_ERR_CAUSE_CHOISE_OVF_SHIFT                       0
+#define MME0_QM_ARB_ERR_CAUSE_CHOISE_OVF_MASK                        0x1
+#define MME0_QM_ARB_ERR_CAUSE_CHOISE_WDT_SHIFT                       1
+#define MME0_QM_ARB_ERR_CAUSE_CHOISE_WDT_MASK                        0x2
+#define MME0_QM_ARB_ERR_CAUSE_AXI_LBW_ERR_SHIFT                      2
+#define MME0_QM_ARB_ERR_CAUSE_AXI_LBW_ERR_MASK                       0x4
+
+/* MME0_QM_ARB_ERR_MSG_EN */
+#define MME0_QM_ARB_ERR_MSG_EN_CHOISE_OVF_SHIFT                      0
+#define MME0_QM_ARB_ERR_MSG_EN_CHOISE_OVF_MASK                       0x1
+#define MME0_QM_ARB_ERR_MSG_EN_CHOISE_WDT_SHIFT                      1
+#define MME0_QM_ARB_ERR_MSG_EN_CHOISE_WDT_MASK                       0x2
+#define MME0_QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_SHIFT                     2
+#define MME0_QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_MASK                      0x4
+
+/* MME0_QM_ARB_ERR_STS_DRP */
+#define MME0_QM_ARB_ERR_STS_DRP_VAL_SHIFT                            0
+#define MME0_QM_ARB_ERR_STS_DRP_VAL_MASK                             0x3
+
+/* MME0_QM_ARB_MST_CRED_STS */
+#define MME0_QM_ARB_MST_CRED_STS_VAL_SHIFT                           0
+#define MME0_QM_ARB_MST_CRED_STS_VAL_MASK                            0x7F
+
+/* MME0_QM_CGM_CFG */
+#define MME0_QM_CGM_CFG_IDLE_TH_SHIFT                                0
+#define MME0_QM_CGM_CFG_IDLE_TH_MASK                                 0xFFF
+#define MME0_QM_CGM_CFG_G2F_TH_SHIFT                                 16
+#define MME0_QM_CGM_CFG_G2F_TH_MASK                                  0xFF0000
+#define MME0_QM_CGM_CFG_CP_IDLE_MASK_SHIFT                           24
+#define MME0_QM_CGM_CFG_CP_IDLE_MASK_MASK                            0x1F000000
+#define MME0_QM_CGM_CFG_EN_SHIFT                                     31
+#define MME0_QM_CGM_CFG_EN_MASK                                      0x80000000
+
+/* MME0_QM_CGM_STS */
+#define MME0_QM_CGM_STS_ST_SHIFT                                     0
+#define MME0_QM_CGM_STS_ST_MASK                                      0x3
+#define MME0_QM_CGM_STS_CG_SHIFT                                     4
+#define MME0_QM_CGM_STS_CG_MASK                                      0x10
+#define MME0_QM_CGM_STS_AGENT_IDLE_SHIFT                             8
+#define MME0_QM_CGM_STS_AGENT_IDLE_MASK                              0x100
+#define MME0_QM_CGM_STS_AXI_IDLE_SHIFT                               9
+#define MME0_QM_CGM_STS_AXI_IDLE_MASK                                0x200
+#define MME0_QM_CGM_STS_CP_IDLE_SHIFT                                10
+#define MME0_QM_CGM_STS_CP_IDLE_MASK                                 0x400
+
+/* MME0_QM_CGM_CFG1 */
+#define MME0_QM_CGM_CFG1_MASK_TH_SHIFT                               0
+#define MME0_QM_CGM_CFG1_MASK_TH_MASK                                0xFF
+
+/* MME0_QM_LOCAL_RANGE_BASE */
+#define MME0_QM_LOCAL_RANGE_BASE_VAL_SHIFT                           0
+#define MME0_QM_LOCAL_RANGE_BASE_VAL_MASK                            0xFFFF
+
+/* MME0_QM_LOCAL_RANGE_SIZE */
+#define MME0_QM_LOCAL_RANGE_SIZE_VAL_SHIFT                           0
+#define MME0_QM_LOCAL_RANGE_SIZE_VAL_MASK                            0xFFFF
+
+/* MME0_QM_CSMR_STRICT_PRIO_CFG */
+#define MME0_QM_CSMR_STRICT_PRIO_CFG_TYPE_SHIFT                      0
+#define MME0_QM_CSMR_STRICT_PRIO_CFG_TYPE_MASK                       0x1
+
+/* MME0_QM_HBW_RD_RATE_LIM_CFG_1 */
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_1_TOUT_SHIFT                     0
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_1_TOUT_MASK                      0xFF
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_1_EN_SHIFT                       31
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_1_EN_MASK                        0x80000000
+
+/* MME0_QM_LBW_WR_RATE_LIM_CFG_0 */
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                0
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_0_RST_TOKEN_MASK                 0xFF
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_0_SAT_SHIFT                      16
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_0_SAT_MASK                       0xFF0000
+
+/* MME0_QM_LBW_WR_RATE_LIM_CFG_1 */
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_1_TOUT_SHIFT                     0
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_1_TOUT_MASK                      0xFF
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_1_EN_SHIFT                       31
+#define MME0_QM_LBW_WR_RATE_LIM_CFG_1_EN_MASK                        0x80000000
+
+/* MME0_QM_HBW_RD_RATE_LIM_CFG_0 */
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                0
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_0_RST_TOKEN_MASK                 0xFF
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_0_SAT_SHIFT                      16
+#define MME0_QM_HBW_RD_RATE_LIM_CFG_0_SAT_MASK                       0xFF0000
+
+/* MME0_QM_GLBL_AXCACHE */
+#define MME0_QM_GLBL_AXCACHE_AR_SHIFT                                0
+#define MME0_QM_GLBL_AXCACHE_AR_MASK                                 0xF
+#define MME0_QM_GLBL_AXCACHE_AW_SHIFT                                16
+#define MME0_QM_GLBL_AXCACHE_AW_MASK                                 0xF0000
+
+/* MME0_QM_IND_GW_APB_CFG */
+#define MME0_QM_IND_GW_APB_CFG_ADDR_SHIFT                            0
+#define MME0_QM_IND_GW_APB_CFG_ADDR_MASK                             0x7FFFFFFF
+#define MME0_QM_IND_GW_APB_CFG_CMD_SHIFT                             31
+#define MME0_QM_IND_GW_APB_CFG_CMD_MASK                              0x80000000
+
+/* MME0_QM_IND_GW_APB_WDATA */
+#define MME0_QM_IND_GW_APB_WDATA_VAL_SHIFT                           0
+#define MME0_QM_IND_GW_APB_WDATA_VAL_MASK                            0xFFFFFFFF
+
+/* MME0_QM_IND_GW_APB_RDATA */
+#define MME0_QM_IND_GW_APB_RDATA_VAL_SHIFT                           0
+#define MME0_QM_IND_GW_APB_RDATA_VAL_MASK                            0xFFFFFFFF
+
+/* MME0_QM_IND_GW_APB_STATUS */
+#define MME0_QM_IND_GW_APB_STATUS_RDY_SHIFT                          0
+#define MME0_QM_IND_GW_APB_STATUS_RDY_MASK                           0x1
+#define MME0_QM_IND_GW_APB_STATUS_ERR_SHIFT                          1
+#define MME0_QM_IND_GW_APB_STATUS_ERR_MASK                           0x2
+
+/* MME0_QM_GLBL_ERR_ADDR_LO */
+#define MME0_QM_GLBL_ERR_ADDR_LO_VAL_SHIFT                           0
+#define MME0_QM_GLBL_ERR_ADDR_LO_VAL_MASK                            0xFFFFFFFF
+
+/* MME0_QM_GLBL_ERR_ADDR_HI */
+#define MME0_QM_GLBL_ERR_ADDR_HI_VAL_SHIFT                           0
+#define MME0_QM_GLBL_ERR_ADDR_HI_VAL_MASK                            0xFFFFFFFF
+
+/* MME0_QM_GLBL_ERR_WDATA */
+#define MME0_QM_GLBL_ERR_WDATA_VAL_SHIFT                             0
+#define MME0_QM_GLBL_ERR_WDATA_VAL_MASK                              0xFFFFFFFF
+
+/* MME0_QM_GLBL_MEM_INIT_BUSY */
+#define MME0_QM_GLBL_MEM_INIT_BUSY_RBUF_SHIFT                        0
+#define MME0_QM_GLBL_MEM_INIT_BUSY_RBUF_MASK                         0xF
+
+#endif /* ASIC_REG_MME0_QM_MASKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_regs.h
new file mode 100644
index 000000000000..4f078b328b00
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme0_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MME0_QM_REGS_H_
+#define ASIC_REG_MME0_QM_REGS_H_
+
+/*
+ *****************************************
+ *   MME0_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmMME0_QM_GLBL_CFG0                                          0x68000
+
+#define mmMME0_QM_GLBL_CFG1                                          0x68004
+
+#define mmMME0_QM_GLBL_PROT                                          0x68008
+
+#define mmMME0_QM_GLBL_ERR_CFG                                       0x6800C
+
+#define mmMME0_QM_GLBL_SECURE_PROPS_0                                0x68010
+
+#define mmMME0_QM_GLBL_SECURE_PROPS_1                                0x68014
+
+#define mmMME0_QM_GLBL_SECURE_PROPS_2                                0x68018
+
+#define mmMME0_QM_GLBL_SECURE_PROPS_3                                0x6801C
+
+#define mmMME0_QM_GLBL_SECURE_PROPS_4                                0x68020
+
+#define mmMME0_QM_GLBL_NON_SECURE_PROPS_0                            0x68024
+
+#define mmMME0_QM_GLBL_NON_SECURE_PROPS_1                            0x68028
+
+#define mmMME0_QM_GLBL_NON_SECURE_PROPS_2                            0x6802C
+
+#define mmMME0_QM_GLBL_NON_SECURE_PROPS_3                            0x68030
+
+#define mmMME0_QM_GLBL_NON_SECURE_PROPS_4                            0x68034
+
+#define mmMME0_QM_GLBL_STS0                                          0x68038
+
+#define mmMME0_QM_GLBL_STS1_0                                        0x68040
+
+#define mmMME0_QM_GLBL_STS1_1                                        0x68044
+
+#define mmMME0_QM_GLBL_STS1_2                                        0x68048
+
+#define mmMME0_QM_GLBL_STS1_3                                        0x6804C
+
+#define mmMME0_QM_GLBL_STS1_4                                        0x68050
+
+#define mmMME0_QM_GLBL_MSG_EN_0                                      0x68054
+
+#define mmMME0_QM_GLBL_MSG_EN_1                                      0x68058
+
+#define mmMME0_QM_GLBL_MSG_EN_2                                      0x6805C
+
+#define mmMME0_QM_GLBL_MSG_EN_3                                      0x68060
+
+#define mmMME0_QM_GLBL_MSG_EN_4                                      0x68068
+
+#define mmMME0_QM_PQ_BASE_LO_0                                       0x68070
+
+#define mmMME0_QM_PQ_BASE_LO_1                                       0x68074
+
+#define mmMME0_QM_PQ_BASE_LO_2                                       0x68078
+
+#define mmMME0_QM_PQ_BASE_LO_3                                       0x6807C
+
+#define mmMME0_QM_PQ_BASE_HI_0                                       0x68080
+
+#define mmMME0_QM_PQ_BASE_HI_1                                       0x68084
+
+#define mmMME0_QM_PQ_BASE_HI_2                                       0x68088
+
+#define mmMME0_QM_PQ_BASE_HI_3                                       0x6808C
+
+#define mmMME0_QM_PQ_SIZE_0                                          0x68090
+
+#define mmMME0_QM_PQ_SIZE_1                                          0x68094
+
+#define mmMME0_QM_PQ_SIZE_2                                          0x68098
+
+#define mmMME0_QM_PQ_SIZE_3                                          0x6809C
+
+#define mmMME0_QM_PQ_PI_0                                            0x680A0
+
+#define mmMME0_QM_PQ_PI_1                                            0x680A4
+
+#define mmMME0_QM_PQ_PI_2                                            0x680A8
+
+#define mmMME0_QM_PQ_PI_3                                            0x680AC
+
+#define mmMME0_QM_PQ_CI_0                                            0x680B0
+
+#define mmMME0_QM_PQ_CI_1                                            0x680B4
+
+#define mmMME0_QM_PQ_CI_2                                            0x680B8
+
+#define mmMME0_QM_PQ_CI_3                                            0x680BC
+
+#define mmMME0_QM_PQ_CFG0_0                                          0x680C0
+
+#define mmMME0_QM_PQ_CFG0_1                                          0x680C4
+
+#define mmMME0_QM_PQ_CFG0_2                                          0x680C8
+
+#define mmMME0_QM_PQ_CFG0_3                                          0x680CC
+
+#define mmMME0_QM_PQ_CFG1_0                                          0x680D0
+
+#define mmMME0_QM_PQ_CFG1_1                                          0x680D4
+
+#define mmMME0_QM_PQ_CFG1_2                                          0x680D8
+
+#define mmMME0_QM_PQ_CFG1_3                                          0x680DC
+
+#define mmMME0_QM_PQ_ARUSER_31_11_0                                  0x680E0
+
+#define mmMME0_QM_PQ_ARUSER_31_11_1                                  0x680E4
+
+#define mmMME0_QM_PQ_ARUSER_31_11_2                                  0x680E8
+
+#define mmMME0_QM_PQ_ARUSER_31_11_3                                  0x680EC
+
+#define mmMME0_QM_PQ_STS0_0                                          0x680F0
+
+#define mmMME0_QM_PQ_STS0_1                                          0x680F4
+
+#define mmMME0_QM_PQ_STS0_2                                          0x680F8
+
+#define mmMME0_QM_PQ_STS0_3                                          0x680FC
+
+#define mmMME0_QM_PQ_STS1_0                                          0x68100
+
+#define mmMME0_QM_PQ_STS1_1                                          0x68104
+
+#define mmMME0_QM_PQ_STS1_2                                          0x68108
+
+#define mmMME0_QM_PQ_STS1_3                                          0x6810C
+
+#define mmMME0_QM_CQ_CFG0_0                                          0x68110
+
+#define mmMME0_QM_CQ_CFG0_1                                          0x68114
+
+#define mmMME0_QM_CQ_CFG0_2                                          0x68118
+
+#define mmMME0_QM_CQ_CFG0_3                                          0x6811C
+
+#define mmMME0_QM_CQ_CFG0_4                                          0x68120
+
+#define mmMME0_QM_CQ_CFG1_0                                          0x68124
+
+#define mmMME0_QM_CQ_CFG1_1                                          0x68128
+
+#define mmMME0_QM_CQ_CFG1_2                                          0x6812C
+
+#define mmMME0_QM_CQ_CFG1_3                                          0x68130
+
+#define mmMME0_QM_CQ_CFG1_4                                          0x68134
+
+#define mmMME0_QM_CQ_ARUSER_31_11_0                                  0x68138
+
+#define mmMME0_QM_CQ_ARUSER_31_11_1                                  0x6813C
+
+#define mmMME0_QM_CQ_ARUSER_31_11_2                                  0x68140
+
+#define mmMME0_QM_CQ_ARUSER_31_11_3                                  0x68144
+
+#define mmMME0_QM_CQ_ARUSER_31_11_4                                  0x68148
+
+#define mmMME0_QM_CQ_STS0_0                                          0x6814C
+
+#define mmMME0_QM_CQ_STS0_1                                          0x68150
+
+#define mmMME0_QM_CQ_STS0_2                                          0x68154
+
+#define mmMME0_QM_CQ_STS0_3                                          0x68158
+
+#define mmMME0_QM_CQ_STS0_4                                          0x6815C
+
+#define mmMME0_QM_CQ_STS1_0                                          0x68160
+
+#define mmMME0_QM_CQ_STS1_1                                          0x68164
+
+#define mmMME0_QM_CQ_STS1_2                                          0x68168
+
+#define mmMME0_QM_CQ_STS1_3                                          0x6816C
+
+#define mmMME0_QM_CQ_STS1_4                                          0x68170
+
+#define mmMME0_QM_CQ_PTR_LO_0                                        0x68174
+
+#define mmMME0_QM_CQ_PTR_HI_0                                        0x68178
+
+#define mmMME0_QM_CQ_TSIZE_0                                         0x6817C
+
+#define mmMME0_QM_CQ_CTL_0                                           0x68180
+
+#define mmMME0_QM_CQ_PTR_LO_1                                        0x68184
+
+#define mmMME0_QM_CQ_PTR_HI_1                                        0x68188
+
+#define mmMME0_QM_CQ_TSIZE_1                                         0x6818C
+
+#define mmMME0_QM_CQ_CTL_1                                           0x68190
+
+#define mmMME0_QM_CQ_PTR_LO_2                                        0x68194
+
+#define mmMME0_QM_CQ_PTR_HI_2                                        0x68198
+
+#define mmMME0_QM_CQ_TSIZE_2                                         0x6819C
+
+#define mmMME0_QM_CQ_CTL_2                                           0x681A0
+
+#define mmMME0_QM_CQ_PTR_LO_3                                        0x681A4
+
+#define mmMME0_QM_CQ_PTR_HI_3                                        0x681A8
+
+#define mmMME0_QM_CQ_TSIZE_3                                         0x681AC
+
+#define mmMME0_QM_CQ_CTL_3                                           0x681B0
+
+#define mmMME0_QM_CQ_PTR_LO_4                                        0x681B4
+
+#define mmMME0_QM_CQ_PTR_HI_4                                        0x681B8
+
+#define mmMME0_QM_CQ_TSIZE_4                                         0x681BC
+
+#define mmMME0_QM_CQ_CTL_4                                           0x681C0
+
+#define mmMME0_QM_CQ_PTR_LO_STS_0                                    0x681C4
+
+#define mmMME0_QM_CQ_PTR_LO_STS_1                                    0x681C8
+
+#define mmMME0_QM_CQ_PTR_LO_STS_2                                    0x681CC
+
+#define mmMME0_QM_CQ_PTR_LO_STS_3                                    0x681D0
+
+#define mmMME0_QM_CQ_PTR_LO_STS_4                                    0x681D4
+
+#define mmMME0_QM_CQ_PTR_HI_STS_0                                    0x681D8
+
+#define mmMME0_QM_CQ_PTR_HI_STS_1                                    0x681DC
+
+#define mmMME0_QM_CQ_PTR_HI_STS_2                                    0x681E0
+
+#define mmMME0_QM_CQ_PTR_HI_STS_3                                    0x681E4
+
+#define mmMME0_QM_CQ_PTR_HI_STS_4                                    0x681E8
+
+#define mmMME0_QM_CQ_TSIZE_STS_0                                     0x681EC
+
+#define mmMME0_QM_CQ_TSIZE_STS_1                                     0x681F0
+
+#define mmMME0_QM_CQ_TSIZE_STS_2                                     0x681F4
+
+#define mmMME0_QM_CQ_TSIZE_STS_3                                     0x681F8
+
+#define mmMME0_QM_CQ_TSIZE_STS_4                                     0x681FC
+
+#define mmMME0_QM_CQ_CTL_STS_0                                       0x68200
+
+#define mmMME0_QM_CQ_CTL_STS_1                                       0x68204
+
+#define mmMME0_QM_CQ_CTL_STS_2                                       0x68208
+
+#define mmMME0_QM_CQ_CTL_STS_3                                       0x6820C
+
+#define mmMME0_QM_CQ_CTL_STS_4                                       0x68210
+
+#define mmMME0_QM_CQ_IFIFO_CNT_0                                     0x68214
+
+#define mmMME0_QM_CQ_IFIFO_CNT_1                                     0x68218
+
+#define mmMME0_QM_CQ_IFIFO_CNT_2                                     0x6821C
+
+#define mmMME0_QM_CQ_IFIFO_CNT_3                                     0x68220
+
+#define mmMME0_QM_CQ_IFIFO_CNT_4                                     0x68224
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_LO_0                             0x68228
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_LO_1                             0x6822C
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_LO_2                             0x68230
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_LO_3                             0x68234
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_LO_4                             0x68238
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_HI_0                             0x6823C
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_HI_1                             0x68240
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_HI_2                             0x68244
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_HI_3                             0x68248
+
+#define mmMME0_QM_CP_MSG_BASE0_ADDR_HI_4                             0x6824C
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_LO_0                             0x68250
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_LO_1                             0x68254
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_LO_2                             0x68258
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_LO_3                             0x6825C
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_LO_4                             0x68260
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_HI_0                             0x68264
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_HI_1                             0x68268
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_HI_2                             0x6826C
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_HI_3                             0x68270
+
+#define mmMME0_QM_CP_MSG_BASE1_ADDR_HI_4                             0x68274
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_LO_0                             0x68278
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_LO_1                             0x6827C
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_LO_2                             0x68280
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_LO_3                             0x68284
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_LO_4                             0x68288
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_HI_0                             0x6828C
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_HI_1                             0x68290
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_HI_2                             0x68294
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_HI_3                             0x68298
+
+#define mmMME0_QM_CP_MSG_BASE2_ADDR_HI_4                             0x6829C
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_LO_0                             0x682A0
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_LO_1                             0x682A4
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_LO_2                             0x682A8
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_LO_3                             0x682AC
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_LO_4                             0x682B0
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_HI_0                             0x682B4
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_HI_1                             0x682B8
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_HI_2                             0x682BC
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_HI_3                             0x682C0
+
+#define mmMME0_QM_CP_MSG_BASE3_ADDR_HI_4                             0x682C4
+
+#define mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0                             0x682C8
+
+#define mmMME0_QM_CP_LDMA_TSIZE_OFFSET_1                             0x682CC
+
+#define mmMME0_QM_CP_LDMA_TSIZE_OFFSET_2                             0x682D0
+
+#define mmMME0_QM_CP_LDMA_TSIZE_OFFSET_3                             0x682D4
+
+#define mmMME0_QM_CP_LDMA_TSIZE_OFFSET_4                             0x682D8
+
+#define mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x682E0
+
+#define mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x682E4
+
+#define mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x682E8
+
+#define mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x682EC
+
+#define mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x682F0
+
+#define mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x682F4
+
+#define mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x682F8
+
+#define mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x682FC
+
+#define mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x68300
+
+#define mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x68304
+
+#define mmMME0_QM_CP_FENCE0_RDATA_0                                  0x68308
+
+#define mmMME0_QM_CP_FENCE0_RDATA_1                                  0x6830C
+
+#define mmMME0_QM_CP_FENCE0_RDATA_2                                  0x68310
+
+#define mmMME0_QM_CP_FENCE0_RDATA_3                                  0x68314
+
+#define mmMME0_QM_CP_FENCE0_RDATA_4                                  0x68318
+
+#define mmMME0_QM_CP_FENCE1_RDATA_0                                  0x6831C
+
+#define mmMME0_QM_CP_FENCE1_RDATA_1                                  0x68320
+
+#define mmMME0_QM_CP_FENCE1_RDATA_2                                  0x68324
+
+#define mmMME0_QM_CP_FENCE1_RDATA_3                                  0x68328
+
+#define mmMME0_QM_CP_FENCE1_RDATA_4                                  0x6832C
+
+#define mmMME0_QM_CP_FENCE2_RDATA_0                                  0x68330
+
+#define mmMME0_QM_CP_FENCE2_RDATA_1                                  0x68334
+
+#define mmMME0_QM_CP_FENCE2_RDATA_2                                  0x68338
+
+#define mmMME0_QM_CP_FENCE2_RDATA_3                                  0x6833C
+
+#define mmMME0_QM_CP_FENCE2_RDATA_4                                  0x68340
+
+#define mmMME0_QM_CP_FENCE3_RDATA_0                                  0x68344
+
+#define mmMME0_QM_CP_FENCE3_RDATA_1                                  0x68348
+
+#define mmMME0_QM_CP_FENCE3_RDATA_2                                  0x6834C
+
+#define mmMME0_QM_CP_FENCE3_RDATA_3                                  0x68350
+
+#define mmMME0_QM_CP_FENCE3_RDATA_4                                  0x68354
+
+#define mmMME0_QM_CP_FENCE0_CNT_0                                    0x68358
+
+#define mmMME0_QM_CP_FENCE0_CNT_1                                    0x6835C
+
+#define mmMME0_QM_CP_FENCE0_CNT_2                                    0x68360
+
+#define mmMME0_QM_CP_FENCE0_CNT_3                                    0x68364
+
+#define mmMME0_QM_CP_FENCE0_CNT_4                                    0x68368
+
+#define mmMME0_QM_CP_FENCE1_CNT_0                                    0x6836C
+
+#define mmMME0_QM_CP_FENCE1_CNT_1                                    0x68370
+
+#define mmMME0_QM_CP_FENCE1_CNT_2                                    0x68374
+
+#define mmMME0_QM_CP_FENCE1_CNT_3                                    0x68378
+
+#define mmMME0_QM_CP_FENCE1_CNT_4                                    0x6837C
+
+#define mmMME0_QM_CP_FENCE2_CNT_0                                    0x68380
+
+#define mmMME0_QM_CP_FENCE2_CNT_1                                    0x68384
+
+#define mmMME0_QM_CP_FENCE2_CNT_2                                    0x68388
+
+#define mmMME0_QM_CP_FENCE2_CNT_3                                    0x6838C
+
+#define mmMME0_QM_CP_FENCE2_CNT_4                                    0x68390
+
+#define mmMME0_QM_CP_FENCE3_CNT_0                                    0x68394
+
+#define mmMME0_QM_CP_FENCE3_CNT_1                                    0x68398
+
+#define mmMME0_QM_CP_FENCE3_CNT_2                                    0x6839C
+
+#define mmMME0_QM_CP_FENCE3_CNT_3                                    0x683A0
+
+#define mmMME0_QM_CP_FENCE3_CNT_4                                    0x683A4
+
+#define mmMME0_QM_CP_STS_0                                           0x683A8
+
+#define mmMME0_QM_CP_STS_1                                           0x683AC
+
+#define mmMME0_QM_CP_STS_2                                           0x683B0
+
+#define mmMME0_QM_CP_STS_3                                           0x683B4
+
+#define mmMME0_QM_CP_STS_4                                           0x683B8
+
+#define mmMME0_QM_CP_CURRENT_INST_LO_0                               0x683BC
+
+#define mmMME0_QM_CP_CURRENT_INST_LO_1                               0x683C0
+
+#define mmMME0_QM_CP_CURRENT_INST_LO_2                               0x683C4
+
+#define mmMME0_QM_CP_CURRENT_INST_LO_3                               0x683C8
+
+#define mmMME0_QM_CP_CURRENT_INST_LO_4                               0x683CC
+
+#define mmMME0_QM_CP_CURRENT_INST_HI_0                               0x683D0
+
+#define mmMME0_QM_CP_CURRENT_INST_HI_1                               0x683D4
+
+#define mmMME0_QM_CP_CURRENT_INST_HI_2                               0x683D8
+
+#define mmMME0_QM_CP_CURRENT_INST_HI_3                               0x683DC
+
+#define mmMME0_QM_CP_CURRENT_INST_HI_4                               0x683E0
+
+#define mmMME0_QM_CP_BARRIER_CFG_0                                   0x683F4
+
+#define mmMME0_QM_CP_BARRIER_CFG_1                                   0x683F8
+
+#define mmMME0_QM_CP_BARRIER_CFG_2                                   0x683FC
+
+#define mmMME0_QM_CP_BARRIER_CFG_3                                   0x68400
+
+#define mmMME0_QM_CP_BARRIER_CFG_4                                   0x68404
+
+#define mmMME0_QM_CP_DBG_0_0                                         0x68408
+
+#define mmMME0_QM_CP_DBG_0_1                                         0x6840C
+
+#define mmMME0_QM_CP_DBG_0_2                                         0x68410
+
+#define mmMME0_QM_CP_DBG_0_3                                         0x68414
+
+#define mmMME0_QM_CP_DBG_0_4                                         0x68418
+
+#define mmMME0_QM_CP_ARUSER_31_11_0                                  0x6841C
+
+#define mmMME0_QM_CP_ARUSER_31_11_1                                  0x68420
+
+#define mmMME0_QM_CP_ARUSER_31_11_2                                  0x68424
+
+#define mmMME0_QM_CP_ARUSER_31_11_3                                  0x68428
+
+#define mmMME0_QM_CP_ARUSER_31_11_4                                  0x6842C
+
+#define mmMME0_QM_CP_AWUSER_31_11_0                                  0x68430
+
+#define mmMME0_QM_CP_AWUSER_31_11_1                                  0x68434
+
+#define mmMME0_QM_CP_AWUSER_31_11_2                                  0x68438
+
+#define mmMME0_QM_CP_AWUSER_31_11_3                                  0x6843C
+
+#define mmMME0_QM_CP_AWUSER_31_11_4                                  0x68440
+
+#define mmMME0_QM_ARB_CFG_0                                          0x68A00
+
+#define mmMME0_QM_ARB_CHOISE_Q_PUSH                                  0x68A04
+
+#define mmMME0_QM_ARB_WRR_WEIGHT_0                                   0x68A08
+
+#define mmMME0_QM_ARB_WRR_WEIGHT_1                                   0x68A0C
+
+#define mmMME0_QM_ARB_WRR_WEIGHT_2                                   0x68A10
+
+#define mmMME0_QM_ARB_WRR_WEIGHT_3                                   0x68A14
+
+#define mmMME0_QM_ARB_CFG_1                                          0x68A18
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_0                               0x68A20
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_1                               0x68A24
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_2                               0x68A28
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_3                               0x68A2C
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_4                               0x68A30
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_5                               0x68A34
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_6                               0x68A38
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_7                               0x68A3C
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_8                               0x68A40
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_9                               0x68A44
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_10                              0x68A48
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_11                              0x68A4C
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_12                              0x68A50
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_13                              0x68A54
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_14                              0x68A58
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_15                              0x68A5C
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_16                              0x68A60
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_17                              0x68A64
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_18                              0x68A68
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_19                              0x68A6C
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_20                              0x68A70
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_21                              0x68A74
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_22                              0x68A78
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_23                              0x68A7C
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_24                              0x68A80
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_25                              0x68A84
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_26                              0x68A88
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_27                              0x68A8C
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_28                              0x68A90
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_29                              0x68A94
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_30                              0x68A98
+
+#define mmMME0_QM_ARB_MST_AVAIL_CRED_31                              0x68A9C
+
+#define mmMME0_QM_ARB_MST_CRED_INC                                   0x68AA0
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x68AA4
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x68AA8
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x68AAC
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x68AB0
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x68AB4
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x68AB8
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x68ABC
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x68AC0
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x68AC4
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x68AC8
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x68ACC
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x68AD0
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x68AD4
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x68AD8
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x68ADC
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x68AE0
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x68AE4
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x68AE8
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x68AEC
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x68AF0
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x68AF4
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x68AF8
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x68AFC
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x68B00
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x68B04
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x68B08
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x68B0C
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x68B10
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x68B14
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x68B18
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x68B1C
+
+#define mmMME0_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x68B20
+
+#define mmMME0_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x68B28
+
+#define mmMME0_QM_ARB_MST_SLAVE_EN                                   0x68B2C
+
+#define mmMME0_QM_ARB_MST_QUIET_PER                                  0x68B34
+
+#define mmMME0_QM_ARB_SLV_CHOISE_WDT                                 0x68B38
+
+#define mmMME0_QM_ARB_SLV_ID                                         0x68B3C
+
+#define mmMME0_QM_ARB_MSG_MAX_INFLIGHT                               0x68B44
+
+#define mmMME0_QM_ARB_MSG_AWUSER_31_11                               0x68B48
+
+#define mmMME0_QM_ARB_MSG_AWUSER_SEC_PROP                            0x68B4C
+
+#define mmMME0_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x68B50
+
+#define mmMME0_QM_ARB_BASE_LO                                        0x68B54
+
+#define mmMME0_QM_ARB_BASE_HI                                        0x68B58
+
+#define mmMME0_QM_ARB_STATE_STS                                      0x68B80
+
+#define mmMME0_QM_ARB_CHOISE_FULLNESS_STS                            0x68B84
+
+#define mmMME0_QM_ARB_MSG_STS                                        0x68B88
+
+#define mmMME0_QM_ARB_SLV_CHOISE_Q_HEAD                              0x68B8C
+
+#define mmMME0_QM_ARB_ERR_CAUSE                                      0x68B9C
+
+#define mmMME0_QM_ARB_ERR_MSG_EN                                     0x68BA0
+
+#define mmMME0_QM_ARB_ERR_STS_DRP                                    0x68BA8
+
+#define mmMME0_QM_ARB_MST_CRED_STS_0                                 0x68BB0
+
+#define mmMME0_QM_ARB_MST_CRED_STS_1                                 0x68BB4
+
+#define mmMME0_QM_ARB_MST_CRED_STS_2                                 0x68BB8
+
+#define mmMME0_QM_ARB_MST_CRED_STS_3                                 0x68BBC
+
+#define mmMME0_QM_ARB_MST_CRED_STS_4                                 0x68BC0
+
+#define mmMME0_QM_ARB_MST_CRED_STS_5                                 0x68BC4
+
+#define mmMME0_QM_ARB_MST_CRED_STS_6                                 0x68BC8
+
+#define mmMME0_QM_ARB_MST_CRED_STS_7                                 0x68BCC
+
+#define mmMME0_QM_ARB_MST_CRED_STS_8                                 0x68BD0
+
+#define mmMME0_QM_ARB_MST_CRED_STS_9                                 0x68BD4
+
+#define mmMME0_QM_ARB_MST_CRED_STS_10                                0x68BD8
+
+#define mmMME0_QM_ARB_MST_CRED_STS_11                                0x68BDC
+
+#define mmMME0_QM_ARB_MST_CRED_STS_12                                0x68BE0
+
+#define mmMME0_QM_ARB_MST_CRED_STS_13                                0x68BE4
+
+#define mmMME0_QM_ARB_MST_CRED_STS_14                                0x68BE8
+
+#define mmMME0_QM_ARB_MST_CRED_STS_15                                0x68BEC
+
+#define mmMME0_QM_ARB_MST_CRED_STS_16                                0x68BF0
+
+#define mmMME0_QM_ARB_MST_CRED_STS_17                                0x68BF4
+
+#define mmMME0_QM_ARB_MST_CRED_STS_18                                0x68BF8
+
+#define mmMME0_QM_ARB_MST_CRED_STS_19                                0x68BFC
+
+#define mmMME0_QM_ARB_MST_CRED_STS_20                                0x68C00
+
+#define mmMME0_QM_ARB_MST_CRED_STS_21                                0x68C04
+
+#define mmMME0_QM_ARB_MST_CRED_STS_22                                0x68C08
+
+#define mmMME0_QM_ARB_MST_CRED_STS_23                                0x68C0C
+
+#define mmMME0_QM_ARB_MST_CRED_STS_24                                0x68C10
+
+#define mmMME0_QM_ARB_MST_CRED_STS_25                                0x68C14
+
+#define mmMME0_QM_ARB_MST_CRED_STS_26                                0x68C18
+
+#define mmMME0_QM_ARB_MST_CRED_STS_27                                0x68C1C
+
+#define mmMME0_QM_ARB_MST_CRED_STS_28                                0x68C20
+
+#define mmMME0_QM_ARB_MST_CRED_STS_29                                0x68C24
+
+#define mmMME0_QM_ARB_MST_CRED_STS_30                                0x68C28
+
+#define mmMME0_QM_ARB_MST_CRED_STS_31                                0x68C2C
+
+#define mmMME0_QM_CGM_CFG                                            0x68C70
+
+#define mmMME0_QM_CGM_STS                                            0x68C74
+
+#define mmMME0_QM_CGM_CFG1                                           0x68C78
+
+#define mmMME0_QM_LOCAL_RANGE_BASE                                   0x68C80
+
+#define mmMME0_QM_LOCAL_RANGE_SIZE                                   0x68C84
+
+#define mmMME0_QM_CSMR_STRICT_PRIO_CFG                               0x68C90
+
+#define mmMME0_QM_HBW_RD_RATE_LIM_CFG_1                              0x68C94
+
+#define mmMME0_QM_LBW_WR_RATE_LIM_CFG_0                              0x68C98
+
+#define mmMME0_QM_LBW_WR_RATE_LIM_CFG_1                              0x68C9C
+
+#define mmMME0_QM_HBW_RD_RATE_LIM_CFG_0                              0x68CA0
+
+#define mmMME0_QM_GLBL_AXCACHE                                       0x68CA4
+
+#define mmMME0_QM_IND_GW_APB_CFG                                     0x68CB0
+
+#define mmMME0_QM_IND_GW_APB_WDATA                                   0x68CB4
+
+#define mmMME0_QM_IND_GW_APB_RDATA                                   0x68CB8
+
+#define mmMME0_QM_IND_GW_APB_STATUS                                  0x68CBC
+
+#define mmMME0_QM_GLBL_ERR_ADDR_LO                                   0x68CD0
+
+#define mmMME0_QM_GLBL_ERR_ADDR_HI                                   0x68CD4
+
+#define mmMME0_QM_GLBL_ERR_WDATA                                     0x68CD8
+
+#define mmMME0_QM_GLBL_MEM_INIT_BUSY                                 0x68D00
+
+#endif /* ASIC_REG_MME0_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mme1_ctrl_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme1_ctrl_regs.h
new file mode 100644
index 000000000000..6c07f7d45490
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme1_ctrl_regs.h
@@ -0,0 +1,1456 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MME1_CTRL_REGS_H_
+#define ASIC_REG_MME1_CTRL_REGS_H_
+
+/*
+ *****************************************
+ *   MME1_CTRL (Prototype: MME)
+ *****************************************
+ */
+
+#define mmMME1_CTRL_ARCH_STATUS                                      0xE0000
+
+#define mmMME1_CTRL_ARCH_BASE_ADDR_HIGH_S                            0xE0008
+
+#define mmMME1_CTRL_ARCH_BASE_ADDR_HIGH_L                            0xE000C
+
+#define mmMME1_CTRL_ARCH_BASE_ADDR_HIGH_O                            0xE0010
+
+#define mmMME1_CTRL_ARCH_BASE_ADDR_LOW_S                             0xE0014
+
+#define mmMME1_CTRL_ARCH_BASE_ADDR_LOW_L                             0xE0018
+
+#define mmMME1_CTRL_ARCH_BASE_ADDR_LOW_O                             0xE001C
+
+#define mmMME1_CTRL_ARCH_HEADER_LOW                                  0xE0020
+
+#define mmMME1_CTRL_ARCH_HEADER_HIGH                                 0xE0024
+
+#define mmMME1_CTRL_ARCH_CONV_KERNEL_SIZE_MINUS_1                    0xE0028
+
+#define mmMME1_CTRL_ARCH_CONV_ASSOCIATED_DIMS_LOW                    0xE002C
+
+#define mmMME1_CTRL_ARCH_CONV_ASSOCIATED_DIMS_HIGH                   0xE0030
+
+#define mmMME1_CTRL_ARCH_NUM_ITERATIONS_MINUS_1                      0xE0034
+
+#define mmMME1_CTRL_ARCH_OUTER_LOOP                                  0xE0038
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_0                   0xE003C
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_1                   0xE0040
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_2                   0xE0044
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_3                   0xE0048
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_4                   0xE004C
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_0                      0xE0050
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_1                      0xE0054
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_2                      0xE0058
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_3                      0xE005C
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_4                      0xE0060
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_ROI_SIZE_0                         0xE0064
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_ROI_SIZE_1                         0xE0068
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_ROI_SIZE_2                         0xE006C
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_ROI_SIZE_3                         0xE0070
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_0                  0xE0074
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_1                  0xE0078
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_2                  0xE007C
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_3                  0xE0080
+
+#define mmMME1_CTRL_ARCH_TENSOR_S_SPATIAL_SIZE_MINUS_1               0xE0084
+
+#define mmMME1_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_0                     0xE0088
+
+#define mmMME1_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_1                     0xE008C
+
+#define mmMME1_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_2                     0xE0090
+
+#define mmMME1_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_3                     0xE0094
+
+#define mmMME1_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_4                     0xE0098
+
+#define mmMME1_CTRL_ARCH_AGU_S_START_OFFSET_0                        0xE009C
+
+#define mmMME1_CTRL_ARCH_AGU_S_START_OFFSET_1                        0xE00A0
+
+#define mmMME1_CTRL_ARCH_AGU_S_START_OFFSET_2                        0xE00A4
+
+#define mmMME1_CTRL_ARCH_AGU_S_START_OFFSET_3                        0xE00A8
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_0                   0xE00AC
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_1                   0xE00B0
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_2                   0xE00B4
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_3                   0xE00B8
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_4                   0xE00BC
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_0                      0xE00C0
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_1                      0xE00C4
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_2                      0xE00C8
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_3                      0xE00CC
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_4                      0xE00D0
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_ROI_SIZE_0                         0xE00D4
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_ROI_SIZE_1                         0xE00D8
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_ROI_SIZE_2                         0xE00DC
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_ROI_SIZE_3                         0xE00E0
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_0                  0xE00E4
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_1                  0xE00E8
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_2                  0xE00EC
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_3                  0xE00F0
+
+#define mmMME1_CTRL_ARCH_TENSOR_L_SPATIAL_SIZE_MINUS_1               0xE00F4
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_0               0xE00F8
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_1               0xE00FC
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_2               0xE0100
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_3               0xE0104
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_4               0xE0108
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_0                  0xE010C
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_1                  0xE0110
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_2                  0xE0114
+
+#define mmMME1_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_3                  0xE0118
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_0              0xE011C
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_1              0xE0120
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_2              0xE0124
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_3              0xE0128
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_4              0xE012C
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_0                 0xE0130
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_1                 0xE0134
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_2                 0xE0138
+
+#define mmMME1_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_3                 0xE013C
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_0                   0xE0140
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_1                   0xE0144
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_2                   0xE0148
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_3                   0xE014C
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_4                   0xE0150
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_0                      0xE0154
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_1                      0xE0158
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_2                      0xE015C
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_3                      0xE0160
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_4                      0xE0164
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_ROI_SIZE_0                         0xE0168
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_ROI_SIZE_1                         0xE016C
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_ROI_SIZE_2                         0xE0170
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_ROI_SIZE_3                         0xE0174
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_0                  0xE0178
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_1                  0xE017C
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_2                  0xE0180
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_3                  0xE0184
+
+#define mmMME1_CTRL_ARCH_TENSOR_O_SPATIAL_SIZE_MINUS_1               0xE0188
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_0               0xE018C
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_1               0xE0190
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_2               0xE0194
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_3               0xE0198
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_4               0xE019C
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_0                  0xE01A0
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_1                  0xE01A4
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_2                  0xE01A8
+
+#define mmMME1_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_3                  0xE01AC
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_0              0xE01B0
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_1              0xE01B4
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_2              0xE01B8
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_3              0xE01BC
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_4              0xE01C0
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_0                 0xE01C4
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_1                 0xE01C8
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_2                 0xE01CC
+
+#define mmMME1_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_3                 0xE01D0
+
+#define mmMME1_CTRL_ARCH_DESC_SB_REPEAT                              0xE01D4
+
+#define mmMME1_CTRL_ARCH_DESC_RATE_LIMITER                           0xE01D8
+
+#define mmMME1_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL             0xE01DC
+
+#define mmMME1_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE            0xE01E0
+
+#define mmMME1_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_HIGH                  0xE01E4
+
+#define mmMME1_CTRL_ARCH_DESC_SYNC_OBJECT_DATA                       0xE01E8
+
+#define mmMME1_CTRL_ARCH_DESC_AXI_USER_DATA                          0xE01EC
+
+#define mmMME1_CTRL_ARCH_DESC_PERF_EVT_S                             0xE01F0
+
+#define mmMME1_CTRL_ARCH_DESC_PERF_EVT_L_LOCAL                       0xE01F4
+
+#define mmMME1_CTRL_ARCH_DESC_PERF_EVT_L_REMOTE                      0xE01F8
+
+#define mmMME1_CTRL_ARCH_DESC_PERF_EVT_O_LOCAL                       0xE01FC
+
+#define mmMME1_CTRL_ARCH_DESC_PERF_EVT_O_REMOTE                      0xE0200
+
+#define mmMME1_CTRL_ARCH_DESC_PADDING_VALUE_S                        0xE0204
+
+#define mmMME1_CTRL_ARCH_DESC_PADDING_VALUE_L                        0xE0208
+
+#define mmMME1_CTRL_ARCH_DESC_META_DATA_AGU_S                        0xE020C
+
+#define mmMME1_CTRL_ARCH_DESC_META_DATA_AGU_L_LOCAL                  0xE0210
+
+#define mmMME1_CTRL_ARCH_DESC_META_DATA_AGU_L_REMOTE                 0xE0214
+
+#define mmMME1_CTRL_ARCH_DESC_META_DATA_AGU_O_LOCAL                  0xE0218
+
+#define mmMME1_CTRL_ARCH_DESC_META_DATA_AGU_O_REMOTE                 0xE021C
+
+#define mmMME1_CTRL_ARCH_DESC_PCU_RL_SATURATION                      0xE0220
+
+#define mmMME1_CTRL_ARCH_DESC_DUMMY                                  0xE0224
+
+#define mmMME1_CTRL_CMD                                              0xE0280
+
+#define mmMME1_CTRL_STATUS1                                          0xE0284
+
+#define mmMME1_CTRL_RESET                                            0xE0288
+
+#define mmMME1_CTRL_QM_STALL                                         0xE028C
+
+#define mmMME1_CTRL_SYNC_OBJECT_FIFO_TH                              0xE0290
+
+#define mmMME1_CTRL_EUS_ROLLUP_CNT_ADD                               0xE0294
+
+#define mmMME1_CTRL_INTR_CAUSE                                       0xE0298
+
+#define mmMME1_CTRL_INTR_MASK                                        0xE029C
+
+#define mmMME1_CTRL_LOG_SHADOW                                       0xE02A0
+
+#define mmMME1_CTRL_PCU_RL_DESC0                                     0xE02A4
+
+#define mmMME1_CTRL_PCU_RL_TOKEN_UPDATE                              0xE02A8
+
+#define mmMME1_CTRL_PCU_RL_TH                                        0xE02AC
+
+#define mmMME1_CTRL_PCU_RL_MIN                                       0xE02B0
+
+#define mmMME1_CTRL_PCU_RL_CTRL_EN                                   0xE02B4
+
+#define mmMME1_CTRL_PCU_RL_HISTORY_LOG_SIZE                          0xE02B8
+
+#define mmMME1_CTRL_PCU_DUMMY_A_BF16                                 0xE02BC
+
+#define mmMME1_CTRL_PCU_DUMMY_B_BF16                                 0xE02C0
+
+#define mmMME1_CTRL_PCU_DUMMY_A_FP32_ODD                             0xE02C4
+
+#define mmMME1_CTRL_PCU_DUMMY_A_FP32_EVEN                            0xE02C8
+
+#define mmMME1_CTRL_PCU_DUMMY_B_FP32_ODD                             0xE02CC
+
+#define mmMME1_CTRL_PCU_DUMMY_B_FP32_EVEN                            0xE02D0
+
+#define mmMME1_CTRL_PROT                                             0xE02D4
+
+#define mmMME1_CTRL_EU_POWER_SAVE_DISABLE                            0xE02D8
+
+#define mmMME1_CTRL_CS_DBG_BLOCK_ID                                  0xE02DC
+
+#define mmMME1_CTRL_CS_DBG_STATUS_DROP_CNT                           0xE02E0
+
+#define mmMME1_CTRL_TE_CLOSE_CGATE                                   0xE02E4
+
+#define mmMME1_CTRL_AGU_SM_INFLIGHT_CNTR                             0xE02E8
+
+#define mmMME1_CTRL_AGU_SM_TOTAL_CNTR                                0xE02EC
+
+#define mmMME1_CTRL_EZSYNC_OUT_CREDIT                                0xE02F0
+
+#define mmMME1_CTRL_PCU_RL_SAT_SEC                                   0xE02F4
+
+#define mmMME1_CTRL_AGU_SYNC_MSG_AXI_USER                            0xE02F8
+
+#define mmMME1_CTRL_QM_SLV_LBW_CLK_EN                                0xE02FC
+
+#define mmMME1_CTRL_SHADOW_0_STATUS                                  0xE0400
+
+#define mmMME1_CTRL_SHADOW_0_BASE_ADDR_HIGH_S                        0xE0408
+
+#define mmMME1_CTRL_SHADOW_0_BASE_ADDR_HIGH_L                        0xE040C
+
+#define mmMME1_CTRL_SHADOW_0_BASE_ADDR_HIGH_O                        0xE0410
+
+#define mmMME1_CTRL_SHADOW_0_BASE_ADDR_LOW_S                         0xE0414
+
+#define mmMME1_CTRL_SHADOW_0_BASE_ADDR_LOW_L                         0xE0418
+
+#define mmMME1_CTRL_SHADOW_0_BASE_ADDR_LOW_O                         0xE041C
+
+#define mmMME1_CTRL_SHADOW_0_HEADER_LOW                              0xE0420
+
+#define mmMME1_CTRL_SHADOW_0_HEADER_HIGH                             0xE0424
+
+#define mmMME1_CTRL_SHADOW_0_CONV_KERNEL_SIZE_MINUS_1                0xE0428
+
+#define mmMME1_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_LOW                0xE042C
+
+#define mmMME1_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_HIGH               0xE0430
+
+#define mmMME1_CTRL_SHADOW_0_NUM_ITERATIONS_MINUS_1                  0xE0434
+
+#define mmMME1_CTRL_SHADOW_0_OUTER_LOOP                              0xE0438
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_0               0xE043C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_1               0xE0440
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_2               0xE0444
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_3               0xE0448
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_4               0xE044C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_0                  0xE0450
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_1                  0xE0454
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_2                  0xE0458
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_3                  0xE045C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_4                  0xE0460
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_0                     0xE0464
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_1                     0xE0468
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_2                     0xE046C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_3                     0xE0470
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_0              0xE0474
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_1              0xE0478
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_2              0xE047C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_3              0xE0480
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_S_SPATIAL_SIZE_MINUS_1           0xE0484
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_0                 0xE0488
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_1                 0xE048C
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_2                 0xE0490
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_3                 0xE0494
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_4                 0xE0498
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_START_OFFSET_0                    0xE049C
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_START_OFFSET_1                    0xE04A0
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_START_OFFSET_2                    0xE04A4
+
+#define mmMME1_CTRL_SHADOW_0_AGU_S_START_OFFSET_3                    0xE04A8
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_0               0xE04AC
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_1               0xE04B0
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_2               0xE04B4
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_3               0xE04B8
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_4               0xE04BC
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_0                  0xE04C0
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_1                  0xE04C4
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_2                  0xE04C8
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_3                  0xE04CC
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_4                  0xE04D0
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_0                     0xE04D4
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_1                     0xE04D8
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_2                     0xE04DC
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_3                     0xE04E0
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_0              0xE04E4
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_1              0xE04E8
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_2              0xE04EC
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_3              0xE04F0
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_L_SPATIAL_SIZE_MINUS_1           0xE04F4
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0xE04F8
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0xE04FC
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0xE0500
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0xE0504
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0xE0508
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_0              0xE050C
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_1              0xE0510
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_2              0xE0514
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_3              0xE0518
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0xE051C
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0xE0520
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0xE0524
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0xE0528
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0xE052C
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_0             0xE0530
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_1             0xE0534
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_2             0xE0538
+
+#define mmMME1_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_3             0xE053C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_0               0xE0540
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_1               0xE0544
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_2               0xE0548
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_3               0xE054C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_4               0xE0550
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_0                  0xE0554
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_1                  0xE0558
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_2                  0xE055C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_3                  0xE0560
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_4                  0xE0564
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_0                     0xE0568
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_1                     0xE056C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_2                     0xE0570
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_3                     0xE0574
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_0              0xE0578
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_1              0xE057C
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_2              0xE0580
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_3              0xE0584
+
+#define mmMME1_CTRL_SHADOW_0_TENSOR_O_SPATIAL_SIZE_MINUS_1           0xE0588
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0xE058C
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0xE0590
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0xE0594
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0xE0598
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0xE059C
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_0              0xE05A0
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_1              0xE05A4
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_2              0xE05A8
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_3              0xE05AC
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0xE05B0
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0xE05B4
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0xE05B8
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0xE05BC
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0xE05C0
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_0             0xE05C4
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_1             0xE05C8
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_2             0xE05CC
+
+#define mmMME1_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_3             0xE05D0
+
+#define mmMME1_CTRL_SHADOW_0_DESC_SB_REPEAT                          0xE05D4
+
+#define mmMME1_CTRL_SHADOW_0_DESC_RATE_LIMITER                       0xE05D8
+
+#define mmMME1_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0xE05DC
+
+#define mmMME1_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0xE05E0
+
+#define mmMME1_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_HIGH              0xE05E4
+
+#define mmMME1_CTRL_SHADOW_0_DESC_SYNC_OBJECT_DATA                   0xE05E8
+
+#define mmMME1_CTRL_SHADOW_0_DESC_AXI_USER_DATA                      0xE05EC
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PERF_EVT_S                         0xE05F0
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PERF_EVT_L_LOCAL                   0xE05F4
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PERF_EVT_L_REMOTE                  0xE05F8
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PERF_EVT_O_LOCAL                   0xE05FC
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PERF_EVT_O_REMOTE                  0xE0600
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PADDING_VALUE_S                    0xE0604
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PADDING_VALUE_L                    0xE0608
+
+#define mmMME1_CTRL_SHADOW_0_DESC_META_DATA_AGU_S                    0xE060C
+
+#define mmMME1_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_LOCAL              0xE0610
+
+#define mmMME1_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_REMOTE             0xE0614
+
+#define mmMME1_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_LOCAL              0xE0618
+
+#define mmMME1_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_REMOTE             0xE061C
+
+#define mmMME1_CTRL_SHADOW_0_DESC_PCU_RL_SATURATION                  0xE0620
+
+#define mmMME1_CTRL_SHADOW_0_DESC_DUMMY                              0xE0624
+
+#define mmMME1_CTRL_SHADOW_1_STATUS                                  0xE0680
+
+#define mmMME1_CTRL_SHADOW_1_BASE_ADDR_HIGH_S                        0xE0688
+
+#define mmMME1_CTRL_SHADOW_1_BASE_ADDR_HIGH_L                        0xE068C
+
+#define mmMME1_CTRL_SHADOW_1_BASE_ADDR_HIGH_O                        0xE0690
+
+#define mmMME1_CTRL_SHADOW_1_BASE_ADDR_LOW_S                         0xE0694
+
+#define mmMME1_CTRL_SHADOW_1_BASE_ADDR_LOW_L                         0xE0698
+
+#define mmMME1_CTRL_SHADOW_1_BASE_ADDR_LOW_O                         0xE069C
+
+#define mmMME1_CTRL_SHADOW_1_HEADER_LOW                              0xE06A0
+
+#define mmMME1_CTRL_SHADOW_1_HEADER_HIGH                             0xE06A4
+
+#define mmMME1_CTRL_SHADOW_1_CONV_KERNEL_SIZE_MINUS_1                0xE06A8
+
+#define mmMME1_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_LOW                0xE06AC
+
+#define mmMME1_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_HIGH               0xE06B0
+
+#define mmMME1_CTRL_SHADOW_1_NUM_ITERATIONS_MINUS_1                  0xE06B4
+
+#define mmMME1_CTRL_SHADOW_1_OUTER_LOOP                              0xE06B8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_0               0xE06BC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_1               0xE06C0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_2               0xE06C4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_3               0xE06C8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_4               0xE06CC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_0                  0xE06D0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_1                  0xE06D4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_2                  0xE06D8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_3                  0xE06DC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_4                  0xE06E0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_0                     0xE06E4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_1                     0xE06E8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_2                     0xE06EC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_3                     0xE06F0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_0              0xE06F4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_1              0xE06F8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_2              0xE06FC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_3              0xE0700
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_S_SPATIAL_SIZE_MINUS_1           0xE0704
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_0                 0xE0708
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_1                 0xE070C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_2                 0xE0710
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_3                 0xE0714
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_4                 0xE0718
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_START_OFFSET_0                    0xE071C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_START_OFFSET_1                    0xE0720
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_START_OFFSET_2                    0xE0724
+
+#define mmMME1_CTRL_SHADOW_1_AGU_S_START_OFFSET_3                    0xE0728
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_0               0xE072C
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_1               0xE0730
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_2               0xE0734
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_3               0xE0738
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_4               0xE073C
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_0                  0xE0740
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_1                  0xE0744
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_2                  0xE0748
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_3                  0xE074C
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_4                  0xE0750
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_0                     0xE0754
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_1                     0xE0758
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_2                     0xE075C
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_3                     0xE0760
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_0              0xE0764
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_1              0xE0768
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_2              0xE076C
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_3              0xE0770
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_L_SPATIAL_SIZE_MINUS_1           0xE0774
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0xE0778
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0xE077C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0xE0780
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0xE0784
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0xE0788
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_0              0xE078C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_1              0xE0790
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_2              0xE0794
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_3              0xE0798
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0xE079C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0xE07A0
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0xE07A4
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0xE07A8
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0xE07AC
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_0             0xE07B0
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_1             0xE07B4
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_2             0xE07B8
+
+#define mmMME1_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_3             0xE07BC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_0               0xE07C0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_1               0xE07C4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_2               0xE07C8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_3               0xE07CC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_4               0xE07D0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_0                  0xE07D4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_1                  0xE07D8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_2                  0xE07DC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_3                  0xE07E0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_4                  0xE07E4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_0                     0xE07E8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_1                     0xE07EC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_2                     0xE07F0
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_3                     0xE07F4
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_0              0xE07F8
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_1              0xE07FC
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_2              0xE0800
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_3              0xE0804
+
+#define mmMME1_CTRL_SHADOW_1_TENSOR_O_SPATIAL_SIZE_MINUS_1           0xE0808
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0xE080C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0xE0810
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0xE0814
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0xE0818
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0xE081C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_0              0xE0820
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_1              0xE0824
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_2              0xE0828
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_3              0xE082C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0xE0830
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0xE0834
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0xE0838
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0xE083C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0xE0840
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_0             0xE0844
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_1             0xE0848
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_2             0xE084C
+
+#define mmMME1_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_3             0xE0850
+
+#define mmMME1_CTRL_SHADOW_1_DESC_SB_REPEAT                          0xE0854
+
+#define mmMME1_CTRL_SHADOW_1_DESC_RATE_LIMITER                       0xE0858
+
+#define mmMME1_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0xE085C
+
+#define mmMME1_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0xE0860
+
+#define mmMME1_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_HIGH              0xE0864
+
+#define mmMME1_CTRL_SHADOW_1_DESC_SYNC_OBJECT_DATA                   0xE0868
+
+#define mmMME1_CTRL_SHADOW_1_DESC_AXI_USER_DATA                      0xE086C
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PERF_EVT_S                         0xE0870
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PERF_EVT_L_LOCAL                   0xE0874
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PERF_EVT_L_REMOTE                  0xE0878
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PERF_EVT_O_LOCAL                   0xE087C
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PERF_EVT_O_REMOTE                  0xE0880
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PADDING_VALUE_S                    0xE0884
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PADDING_VALUE_L                    0xE0888
+
+#define mmMME1_CTRL_SHADOW_1_DESC_META_DATA_AGU_S                    0xE088C
+
+#define mmMME1_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_LOCAL              0xE0890
+
+#define mmMME1_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_REMOTE             0xE0894
+
+#define mmMME1_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_LOCAL              0xE0898
+
+#define mmMME1_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_REMOTE             0xE089C
+
+#define mmMME1_CTRL_SHADOW_1_DESC_PCU_RL_SATURATION                  0xE08A0
+
+#define mmMME1_CTRL_SHADOW_1_DESC_DUMMY                              0xE08A4
+
+#define mmMME1_CTRL_SHADOW_2_STATUS                                  0xE0900
+
+#define mmMME1_CTRL_SHADOW_2_BASE_ADDR_HIGH_S                        0xE0908
+
+#define mmMME1_CTRL_SHADOW_2_BASE_ADDR_HIGH_L                        0xE090C
+
+#define mmMME1_CTRL_SHADOW_2_BASE_ADDR_HIGH_O                        0xE0910
+
+#define mmMME1_CTRL_SHADOW_2_BASE_ADDR_LOW_S                         0xE0914
+
+#define mmMME1_CTRL_SHADOW_2_BASE_ADDR_LOW_L                         0xE0918
+
+#define mmMME1_CTRL_SHADOW_2_BASE_ADDR_LOW_O                         0xE091C
+
+#define mmMME1_CTRL_SHADOW_2_HEADER_LOW                              0xE0920
+
+#define mmMME1_CTRL_SHADOW_2_HEADER_HIGH                             0xE0924
+
+#define mmMME1_CTRL_SHADOW_2_CONV_KERNEL_SIZE_MINUS_1                0xE0928
+
+#define mmMME1_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_LOW                0xE092C
+
+#define mmMME1_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_HIGH               0xE0930
+
+#define mmMME1_CTRL_SHADOW_2_NUM_ITERATIONS_MINUS_1                  0xE0934
+
+#define mmMME1_CTRL_SHADOW_2_OUTER_LOOP                              0xE0938
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_0               0xE093C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_1               0xE0940
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_2               0xE0944
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_3               0xE0948
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_4               0xE094C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_0                  0xE0950
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_1                  0xE0954
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_2                  0xE0958
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_3                  0xE095C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_4                  0xE0960
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_0                     0xE0964
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_1                     0xE0968
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_2                     0xE096C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_3                     0xE0970
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_0              0xE0974
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_1              0xE0978
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_2              0xE097C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_3              0xE0980
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_S_SPATIAL_SIZE_MINUS_1           0xE0984
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_0                 0xE0988
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_1                 0xE098C
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_2                 0xE0990
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_3                 0xE0994
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_4                 0xE0998
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_START_OFFSET_0                    0xE099C
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_START_OFFSET_1                    0xE09A0
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_START_OFFSET_2                    0xE09A4
+
+#define mmMME1_CTRL_SHADOW_2_AGU_S_START_OFFSET_3                    0xE09A8
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_0               0xE09AC
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_1               0xE09B0
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_2               0xE09B4
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_3               0xE09B8
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_4               0xE09BC
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_0                  0xE09C0
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_1                  0xE09C4
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_2                  0xE09C8
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_3                  0xE09CC
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_4                  0xE09D0
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_0                     0xE09D4
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_1                     0xE09D8
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_2                     0xE09DC
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_3                     0xE09E0
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_0              0xE09E4
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_1              0xE09E8
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_2              0xE09EC
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_3              0xE09F0
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_L_SPATIAL_SIZE_MINUS_1           0xE09F4
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0xE09F8
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0xE09FC
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0xE0A00
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0xE0A04
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0xE0A08
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_0              0xE0A0C
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_1              0xE0A10
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_2              0xE0A14
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_3              0xE0A18
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0xE0A1C
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0xE0A20
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0xE0A24
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0xE0A28
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0xE0A2C
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_0             0xE0A30
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_1             0xE0A34
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_2             0xE0A38
+
+#define mmMME1_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_3             0xE0A3C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_0               0xE0A40
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_1               0xE0A44
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_2               0xE0A48
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_3               0xE0A4C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_4               0xE0A50
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_0                  0xE0A54
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_1                  0xE0A58
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_2                  0xE0A5C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_3                  0xE0A60
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_4                  0xE0A64
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_0                     0xE0A68
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_1                     0xE0A6C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_2                     0xE0A70
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_3                     0xE0A74
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_0              0xE0A78
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_1              0xE0A7C
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_2              0xE0A80
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_3              0xE0A84
+
+#define mmMME1_CTRL_SHADOW_2_TENSOR_O_SPATIAL_SIZE_MINUS_1           0xE0A88
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0xE0A8C
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0xE0A90
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0xE0A94
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0xE0A98
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0xE0A9C
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_0              0xE0AA0
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_1              0xE0AA4
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_2              0xE0AA8
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_3              0xE0AAC
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0xE0AB0
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0xE0AB4
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0xE0AB8
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0xE0ABC
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0xE0AC0
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_0             0xE0AC4
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_1             0xE0AC8
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_2             0xE0ACC
+
+#define mmMME1_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_3             0xE0AD0
+
+#define mmMME1_CTRL_SHADOW_2_DESC_SB_REPEAT                          0xE0AD4
+
+#define mmMME1_CTRL_SHADOW_2_DESC_RATE_LIMITER                       0xE0AD8
+
+#define mmMME1_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0xE0ADC
+
+#define mmMME1_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0xE0AE0
+
+#define mmMME1_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_HIGH              0xE0AE4
+
+#define mmMME1_CTRL_SHADOW_2_DESC_SYNC_OBJECT_DATA                   0xE0AE8
+
+#define mmMME1_CTRL_SHADOW_2_DESC_AXI_USER_DATA                      0xE0AEC
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PERF_EVT_S                         0xE0AF0
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PERF_EVT_L_LOCAL                   0xE0AF4
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PERF_EVT_L_REMOTE                  0xE0AF8
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PERF_EVT_O_LOCAL                   0xE0AFC
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PERF_EVT_O_REMOTE                  0xE0B00
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PADDING_VALUE_S                    0xE0B04
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PADDING_VALUE_L                    0xE0B08
+
+#define mmMME1_CTRL_SHADOW_2_DESC_META_DATA_AGU_S                    0xE0B0C
+
+#define mmMME1_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_LOCAL              0xE0B10
+
+#define mmMME1_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_REMOTE             0xE0B14
+
+#define mmMME1_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_LOCAL              0xE0B18
+
+#define mmMME1_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_REMOTE             0xE0B1C
+
+#define mmMME1_CTRL_SHADOW_2_DESC_PCU_RL_SATURATION                  0xE0B20
+
+#define mmMME1_CTRL_SHADOW_2_DESC_DUMMY                              0xE0B24
+
+#define mmMME1_CTRL_SHADOW_3_STATUS                                  0xE0B80
+
+#define mmMME1_CTRL_SHADOW_3_BASE_ADDR_HIGH_S                        0xE0B88
+
+#define mmMME1_CTRL_SHADOW_3_BASE_ADDR_HIGH_L                        0xE0B8C
+
+#define mmMME1_CTRL_SHADOW_3_BASE_ADDR_HIGH_O                        0xE0B90
+
+#define mmMME1_CTRL_SHADOW_3_BASE_ADDR_LOW_S                         0xE0B94
+
+#define mmMME1_CTRL_SHADOW_3_BASE_ADDR_LOW_L                         0xE0B98
+
+#define mmMME1_CTRL_SHADOW_3_BASE_ADDR_LOW_O                         0xE0B9C
+
+#define mmMME1_CTRL_SHADOW_3_HEADER_LOW                              0xE0BA0
+
+#define mmMME1_CTRL_SHADOW_3_HEADER_HIGH                             0xE0BA4
+
+#define mmMME1_CTRL_SHADOW_3_CONV_KERNEL_SIZE_MINUS_1                0xE0BA8
+
+#define mmMME1_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_LOW                0xE0BAC
+
+#define mmMME1_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_HIGH               0xE0BB0
+
+#define mmMME1_CTRL_SHADOW_3_NUM_ITERATIONS_MINUS_1                  0xE0BB4
+
+#define mmMME1_CTRL_SHADOW_3_OUTER_LOOP                              0xE0BB8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_0               0xE0BBC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_1               0xE0BC0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_2               0xE0BC4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_3               0xE0BC8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_4               0xE0BCC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_0                  0xE0BD0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_1                  0xE0BD4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_2                  0xE0BD8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_3                  0xE0BDC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_4                  0xE0BE0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_0                     0xE0BE4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_1                     0xE0BE8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_2                     0xE0BEC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_3                     0xE0BF0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_0              0xE0BF4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_1              0xE0BF8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_2              0xE0BFC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_3              0xE0C00
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_S_SPATIAL_SIZE_MINUS_1           0xE0C04
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_0                 0xE0C08
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_1                 0xE0C0C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_2                 0xE0C10
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_3                 0xE0C14
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_4                 0xE0C18
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_START_OFFSET_0                    0xE0C1C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_START_OFFSET_1                    0xE0C20
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_START_OFFSET_2                    0xE0C24
+
+#define mmMME1_CTRL_SHADOW_3_AGU_S_START_OFFSET_3                    0xE0C28
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_0               0xE0C2C
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_1               0xE0C30
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_2               0xE0C34
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_3               0xE0C38
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_4               0xE0C3C
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_0                  0xE0C40
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_1                  0xE0C44
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_2                  0xE0C48
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_3                  0xE0C4C
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_4                  0xE0C50
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_0                     0xE0C54
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_1                     0xE0C58
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_2                     0xE0C5C
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_3                     0xE0C60
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_0              0xE0C64
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_1              0xE0C68
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_2              0xE0C6C
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_3              0xE0C70
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_L_SPATIAL_SIZE_MINUS_1           0xE0C74
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0xE0C78
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0xE0C7C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0xE0C80
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0xE0C84
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0xE0C88
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_0              0xE0C8C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_1              0xE0C90
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_2              0xE0C94
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_3              0xE0C98
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0xE0C9C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0xE0CA0
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0xE0CA4
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0xE0CA8
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0xE0CAC
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_0             0xE0CB0
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_1             0xE0CB4
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_2             0xE0CB8
+
+#define mmMME1_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_3             0xE0CBC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_0               0xE0CC0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_1               0xE0CC4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_2               0xE0CC8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_3               0xE0CCC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_4               0xE0CD0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_0                  0xE0CD4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_1                  0xE0CD8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_2                  0xE0CDC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_3                  0xE0CE0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_4                  0xE0CE4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_0                     0xE0CE8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_1                     0xE0CEC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_2                     0xE0CF0
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_3                     0xE0CF4
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_0              0xE0CF8
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_1              0xE0CFC
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_2              0xE0D00
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_3              0xE0D04
+
+#define mmMME1_CTRL_SHADOW_3_TENSOR_O_SPATIAL_SIZE_MINUS_1           0xE0D08
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0xE0D0C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0xE0D10
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0xE0D14
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0xE0D18
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0xE0D1C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_0              0xE0D20
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_1              0xE0D24
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_2              0xE0D28
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_3              0xE0D2C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0xE0D30
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0xE0D34
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0xE0D38
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0xE0D3C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0xE0D40
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_0             0xE0D44
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_1             0xE0D48
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_2             0xE0D4C
+
+#define mmMME1_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_3             0xE0D50
+
+#define mmMME1_CTRL_SHADOW_3_DESC_SB_REPEAT                          0xE0D54
+
+#define mmMME1_CTRL_SHADOW_3_DESC_RATE_LIMITER                       0xE0D58
+
+#define mmMME1_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0xE0D5C
+
+#define mmMME1_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0xE0D60
+
+#define mmMME1_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_HIGH              0xE0D64
+
+#define mmMME1_CTRL_SHADOW_3_DESC_SYNC_OBJECT_DATA                   0xE0D68
+
+#define mmMME1_CTRL_SHADOW_3_DESC_AXI_USER_DATA                      0xE0D6C
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PERF_EVT_S                         0xE0D70
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PERF_EVT_L_LOCAL                   0xE0D74
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PERF_EVT_L_REMOTE                  0xE0D78
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PERF_EVT_O_LOCAL                   0xE0D7C
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PERF_EVT_O_REMOTE                  0xE0D80
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PADDING_VALUE_S                    0xE0D84
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PADDING_VALUE_L                    0xE0D88
+
+#define mmMME1_CTRL_SHADOW_3_DESC_META_DATA_AGU_S                    0xE0D8C
+
+#define mmMME1_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_LOCAL              0xE0D90
+
+#define mmMME1_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_REMOTE             0xE0D94
+
+#define mmMME1_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_LOCAL              0xE0D98
+
+#define mmMME1_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_REMOTE             0xE0D9C
+
+#define mmMME1_CTRL_SHADOW_3_DESC_PCU_RL_SATURATION                  0xE0DA0
+
+#define mmMME1_CTRL_SHADOW_3_DESC_DUMMY                              0xE0DA4
+
+#endif /* ASIC_REG_MME1_CTRL_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_ctrl_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_ctrl_regs.h
new file mode 100644
index 000000000000..a1f2eb8b91bd
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_ctrl_regs.h
@@ -0,0 +1,1456 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MME2_CTRL_REGS_H_
+#define ASIC_REG_MME2_CTRL_REGS_H_
+
+/*
+ *****************************************
+ *   MME2_CTRL (Prototype: MME)
+ *****************************************
+ */
+
+#define mmMME2_CTRL_ARCH_STATUS                                      0x160000
+
+#define mmMME2_CTRL_ARCH_BASE_ADDR_HIGH_S                            0x160008
+
+#define mmMME2_CTRL_ARCH_BASE_ADDR_HIGH_L                            0x16000C
+
+#define mmMME2_CTRL_ARCH_BASE_ADDR_HIGH_O                            0x160010
+
+#define mmMME2_CTRL_ARCH_BASE_ADDR_LOW_S                             0x160014
+
+#define mmMME2_CTRL_ARCH_BASE_ADDR_LOW_L                             0x160018
+
+#define mmMME2_CTRL_ARCH_BASE_ADDR_LOW_O                             0x16001C
+
+#define mmMME2_CTRL_ARCH_HEADER_LOW                                  0x160020
+
+#define mmMME2_CTRL_ARCH_HEADER_HIGH                                 0x160024
+
+#define mmMME2_CTRL_ARCH_CONV_KERNEL_SIZE_MINUS_1                    0x160028
+
+#define mmMME2_CTRL_ARCH_CONV_ASSOCIATED_DIMS_LOW                    0x16002C
+
+#define mmMME2_CTRL_ARCH_CONV_ASSOCIATED_DIMS_HIGH                   0x160030
+
+#define mmMME2_CTRL_ARCH_NUM_ITERATIONS_MINUS_1                      0x160034
+
+#define mmMME2_CTRL_ARCH_OUTER_LOOP                                  0x160038
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_0                   0x16003C
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_1                   0x160040
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_2                   0x160044
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_3                   0x160048
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_4                   0x16004C
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_0                      0x160050
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_1                      0x160054
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_2                      0x160058
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_3                      0x16005C
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_4                      0x160060
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_ROI_SIZE_0                         0x160064
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_ROI_SIZE_1                         0x160068
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_ROI_SIZE_2                         0x16006C
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_ROI_SIZE_3                         0x160070
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_0                  0x160074
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_1                  0x160078
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_2                  0x16007C
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_3                  0x160080
+
+#define mmMME2_CTRL_ARCH_TENSOR_S_SPATIAL_SIZE_MINUS_1               0x160084
+
+#define mmMME2_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_0                     0x160088
+
+#define mmMME2_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_1                     0x16008C
+
+#define mmMME2_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_2                     0x160090
+
+#define mmMME2_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_3                     0x160094
+
+#define mmMME2_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_4                     0x160098
+
+#define mmMME2_CTRL_ARCH_AGU_S_START_OFFSET_0                        0x16009C
+
+#define mmMME2_CTRL_ARCH_AGU_S_START_OFFSET_1                        0x1600A0
+
+#define mmMME2_CTRL_ARCH_AGU_S_START_OFFSET_2                        0x1600A4
+
+#define mmMME2_CTRL_ARCH_AGU_S_START_OFFSET_3                        0x1600A8
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_0                   0x1600AC
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_1                   0x1600B0
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_2                   0x1600B4
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_3                   0x1600B8
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_4                   0x1600BC
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_0                      0x1600C0
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_1                      0x1600C4
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_2                      0x1600C8
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_3                      0x1600CC
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_4                      0x1600D0
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_ROI_SIZE_0                         0x1600D4
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_ROI_SIZE_1                         0x1600D8
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_ROI_SIZE_2                         0x1600DC
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_ROI_SIZE_3                         0x1600E0
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_0                  0x1600E4
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_1                  0x1600E8
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_2                  0x1600EC
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_3                  0x1600F0
+
+#define mmMME2_CTRL_ARCH_TENSOR_L_SPATIAL_SIZE_MINUS_1               0x1600F4
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_0               0x1600F8
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_1               0x1600FC
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_2               0x160100
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_3               0x160104
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_4               0x160108
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_0                  0x16010C
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_1                  0x160110
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_2                  0x160114
+
+#define mmMME2_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_3                  0x160118
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_0              0x16011C
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_1              0x160120
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_2              0x160124
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_3              0x160128
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_4              0x16012C
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_0                 0x160130
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_1                 0x160134
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_2                 0x160138
+
+#define mmMME2_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_3                 0x16013C
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_0                   0x160140
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_1                   0x160144
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_2                   0x160148
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_3                   0x16014C
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_4                   0x160150
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_0                      0x160154
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_1                      0x160158
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_2                      0x16015C
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_3                      0x160160
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_4                      0x160164
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_ROI_SIZE_0                         0x160168
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_ROI_SIZE_1                         0x16016C
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_ROI_SIZE_2                         0x160170
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_ROI_SIZE_3                         0x160174
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_0                  0x160178
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_1                  0x16017C
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_2                  0x160180
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_3                  0x160184
+
+#define mmMME2_CTRL_ARCH_TENSOR_O_SPATIAL_SIZE_MINUS_1               0x160188
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_0               0x16018C
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_1               0x160190
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_2               0x160194
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_3               0x160198
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_4               0x16019C
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_0                  0x1601A0
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_1                  0x1601A4
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_2                  0x1601A8
+
+#define mmMME2_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_3                  0x1601AC
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_0              0x1601B0
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_1              0x1601B4
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_2              0x1601B8
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_3              0x1601BC
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_4              0x1601C0
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_0                 0x1601C4
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_1                 0x1601C8
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_2                 0x1601CC
+
+#define mmMME2_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_3                 0x1601D0
+
+#define mmMME2_CTRL_ARCH_DESC_SB_REPEAT                              0x1601D4
+
+#define mmMME2_CTRL_ARCH_DESC_RATE_LIMITER                           0x1601D8
+
+#define mmMME2_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL             0x1601DC
+
+#define mmMME2_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE            0x1601E0
+
+#define mmMME2_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_HIGH                  0x1601E4
+
+#define mmMME2_CTRL_ARCH_DESC_SYNC_OBJECT_DATA                       0x1601E8
+
+#define mmMME2_CTRL_ARCH_DESC_AXI_USER_DATA                          0x1601EC
+
+#define mmMME2_CTRL_ARCH_DESC_PERF_EVT_S                             0x1601F0
+
+#define mmMME2_CTRL_ARCH_DESC_PERF_EVT_L_LOCAL                       0x1601F4
+
+#define mmMME2_CTRL_ARCH_DESC_PERF_EVT_L_REMOTE                      0x1601F8
+
+#define mmMME2_CTRL_ARCH_DESC_PERF_EVT_O_LOCAL                       0x1601FC
+
+#define mmMME2_CTRL_ARCH_DESC_PERF_EVT_O_REMOTE                      0x160200
+
+#define mmMME2_CTRL_ARCH_DESC_PADDING_VALUE_S                        0x160204
+
+#define mmMME2_CTRL_ARCH_DESC_PADDING_VALUE_L                        0x160208
+
+#define mmMME2_CTRL_ARCH_DESC_META_DATA_AGU_S                        0x16020C
+
+#define mmMME2_CTRL_ARCH_DESC_META_DATA_AGU_L_LOCAL                  0x160210
+
+#define mmMME2_CTRL_ARCH_DESC_META_DATA_AGU_L_REMOTE                 0x160214
+
+#define mmMME2_CTRL_ARCH_DESC_META_DATA_AGU_O_LOCAL                  0x160218
+
+#define mmMME2_CTRL_ARCH_DESC_META_DATA_AGU_O_REMOTE                 0x16021C
+
+#define mmMME2_CTRL_ARCH_DESC_PCU_RL_SATURATION                      0x160220
+
+#define mmMME2_CTRL_ARCH_DESC_DUMMY                                  0x160224
+
+#define mmMME2_CTRL_CMD                                              0x160280
+
+#define mmMME2_CTRL_STATUS1                                          0x160284
+
+#define mmMME2_CTRL_RESET                                            0x160288
+
+#define mmMME2_CTRL_QM_STALL                                         0x16028C
+
+#define mmMME2_CTRL_SYNC_OBJECT_FIFO_TH                              0x160290
+
+#define mmMME2_CTRL_EUS_ROLLUP_CNT_ADD                               0x160294
+
+#define mmMME2_CTRL_INTR_CAUSE                                       0x160298
+
+#define mmMME2_CTRL_INTR_MASK                                        0x16029C
+
+#define mmMME2_CTRL_LOG_SHADOW                                       0x1602A0
+
+#define mmMME2_CTRL_PCU_RL_DESC0                                     0x1602A4
+
+#define mmMME2_CTRL_PCU_RL_TOKEN_UPDATE                              0x1602A8
+
+#define mmMME2_CTRL_PCU_RL_TH                                        0x1602AC
+
+#define mmMME2_CTRL_PCU_RL_MIN                                       0x1602B0
+
+#define mmMME2_CTRL_PCU_RL_CTRL_EN                                   0x1602B4
+
+#define mmMME2_CTRL_PCU_RL_HISTORY_LOG_SIZE                          0x1602B8
+
+#define mmMME2_CTRL_PCU_DUMMY_A_BF16                                 0x1602BC
+
+#define mmMME2_CTRL_PCU_DUMMY_B_BF16                                 0x1602C0
+
+#define mmMME2_CTRL_PCU_DUMMY_A_FP32_ODD                             0x1602C4
+
+#define mmMME2_CTRL_PCU_DUMMY_A_FP32_EVEN                            0x1602C8
+
+#define mmMME2_CTRL_PCU_DUMMY_B_FP32_ODD                             0x1602CC
+
+#define mmMME2_CTRL_PCU_DUMMY_B_FP32_EVEN                            0x1602D0
+
+#define mmMME2_CTRL_PROT                                             0x1602D4
+
+#define mmMME2_CTRL_EU_POWER_SAVE_DISABLE                            0x1602D8
+
+#define mmMME2_CTRL_CS_DBG_BLOCK_ID                                  0x1602DC
+
+#define mmMME2_CTRL_CS_DBG_STATUS_DROP_CNT                           0x1602E0
+
+#define mmMME2_CTRL_TE_CLOSE_CGATE                                   0x1602E4
+
+#define mmMME2_CTRL_AGU_SM_INFLIGHT_CNTR                             0x1602E8
+
+#define mmMME2_CTRL_AGU_SM_TOTAL_CNTR                                0x1602EC
+
+#define mmMME2_CTRL_EZSYNC_OUT_CREDIT                                0x1602F0
+
+#define mmMME2_CTRL_PCU_RL_SAT_SEC                                   0x1602F4
+
+#define mmMME2_CTRL_AGU_SYNC_MSG_AXI_USER                            0x1602F8
+
+#define mmMME2_CTRL_QM_SLV_LBW_CLK_EN                                0x1602FC
+
+#define mmMME2_CTRL_SHADOW_0_STATUS                                  0x160400
+
+#define mmMME2_CTRL_SHADOW_0_BASE_ADDR_HIGH_S                        0x160408
+
+#define mmMME2_CTRL_SHADOW_0_BASE_ADDR_HIGH_L                        0x16040C
+
+#define mmMME2_CTRL_SHADOW_0_BASE_ADDR_HIGH_O                        0x160410
+
+#define mmMME2_CTRL_SHADOW_0_BASE_ADDR_LOW_S                         0x160414
+
+#define mmMME2_CTRL_SHADOW_0_BASE_ADDR_LOW_L                         0x160418
+
+#define mmMME2_CTRL_SHADOW_0_BASE_ADDR_LOW_O                         0x16041C
+
+#define mmMME2_CTRL_SHADOW_0_HEADER_LOW                              0x160420
+
+#define mmMME2_CTRL_SHADOW_0_HEADER_HIGH                             0x160424
+
+#define mmMME2_CTRL_SHADOW_0_CONV_KERNEL_SIZE_MINUS_1                0x160428
+
+#define mmMME2_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_LOW                0x16042C
+
+#define mmMME2_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_HIGH               0x160430
+
+#define mmMME2_CTRL_SHADOW_0_NUM_ITERATIONS_MINUS_1                  0x160434
+
+#define mmMME2_CTRL_SHADOW_0_OUTER_LOOP                              0x160438
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_0               0x16043C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_1               0x160440
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_2               0x160444
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_3               0x160448
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_4               0x16044C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_0                  0x160450
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_1                  0x160454
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_2                  0x160458
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_3                  0x16045C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_4                  0x160460
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_0                     0x160464
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_1                     0x160468
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_2                     0x16046C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_3                     0x160470
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_0              0x160474
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_1              0x160478
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_2              0x16047C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_3              0x160480
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x160484
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_0                 0x160488
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_1                 0x16048C
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_2                 0x160490
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_3                 0x160494
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_4                 0x160498
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_START_OFFSET_0                    0x16049C
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_START_OFFSET_1                    0x1604A0
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_START_OFFSET_2                    0x1604A4
+
+#define mmMME2_CTRL_SHADOW_0_AGU_S_START_OFFSET_3                    0x1604A8
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_0               0x1604AC
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_1               0x1604B0
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_2               0x1604B4
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_3               0x1604B8
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_4               0x1604BC
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_0                  0x1604C0
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_1                  0x1604C4
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_2                  0x1604C8
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_3                  0x1604CC
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_4                  0x1604D0
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_0                     0x1604D4
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_1                     0x1604D8
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_2                     0x1604DC
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_3                     0x1604E0
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_0              0x1604E4
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_1              0x1604E8
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_2              0x1604EC
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_3              0x1604F0
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x1604F4
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x1604F8
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x1604FC
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x160500
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x160504
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x160508
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_0              0x16050C
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_1              0x160510
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_2              0x160514
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_3              0x160518
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x16051C
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x160520
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x160524
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x160528
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x16052C
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_0             0x160530
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_1             0x160534
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_2             0x160538
+
+#define mmMME2_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_3             0x16053C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_0               0x160540
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_1               0x160544
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_2               0x160548
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_3               0x16054C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_4               0x160550
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_0                  0x160554
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_1                  0x160558
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_2                  0x16055C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_3                  0x160560
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_4                  0x160564
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_0                     0x160568
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_1                     0x16056C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_2                     0x160570
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_3                     0x160574
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_0              0x160578
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_1              0x16057C
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_2              0x160580
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_3              0x160584
+
+#define mmMME2_CTRL_SHADOW_0_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x160588
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x16058C
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x160590
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x160594
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x160598
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x16059C
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_0              0x1605A0
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_1              0x1605A4
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_2              0x1605A8
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_3              0x1605AC
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x1605B0
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x1605B4
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x1605B8
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x1605BC
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x1605C0
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_0             0x1605C4
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_1             0x1605C8
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_2             0x1605CC
+
+#define mmMME2_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_3             0x1605D0
+
+#define mmMME2_CTRL_SHADOW_0_DESC_SB_REPEAT                          0x1605D4
+
+#define mmMME2_CTRL_SHADOW_0_DESC_RATE_LIMITER                       0x1605D8
+
+#define mmMME2_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x1605DC
+
+#define mmMME2_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x1605E0
+
+#define mmMME2_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_HIGH              0x1605E4
+
+#define mmMME2_CTRL_SHADOW_0_DESC_SYNC_OBJECT_DATA                   0x1605E8
+
+#define mmMME2_CTRL_SHADOW_0_DESC_AXI_USER_DATA                      0x1605EC
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PERF_EVT_S                         0x1605F0
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PERF_EVT_L_LOCAL                   0x1605F4
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PERF_EVT_L_REMOTE                  0x1605F8
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PERF_EVT_O_LOCAL                   0x1605FC
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PERF_EVT_O_REMOTE                  0x160600
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PADDING_VALUE_S                    0x160604
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PADDING_VALUE_L                    0x160608
+
+#define mmMME2_CTRL_SHADOW_0_DESC_META_DATA_AGU_S                    0x16060C
+
+#define mmMME2_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_LOCAL              0x160610
+
+#define mmMME2_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_REMOTE             0x160614
+
+#define mmMME2_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_LOCAL              0x160618
+
+#define mmMME2_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_REMOTE             0x16061C
+
+#define mmMME2_CTRL_SHADOW_0_DESC_PCU_RL_SATURATION                  0x160620
+
+#define mmMME2_CTRL_SHADOW_0_DESC_DUMMY                              0x160624
+
+#define mmMME2_CTRL_SHADOW_1_STATUS                                  0x160680
+
+#define mmMME2_CTRL_SHADOW_1_BASE_ADDR_HIGH_S                        0x160688
+
+#define mmMME2_CTRL_SHADOW_1_BASE_ADDR_HIGH_L                        0x16068C
+
+#define mmMME2_CTRL_SHADOW_1_BASE_ADDR_HIGH_O                        0x160690
+
+#define mmMME2_CTRL_SHADOW_1_BASE_ADDR_LOW_S                         0x160694
+
+#define mmMME2_CTRL_SHADOW_1_BASE_ADDR_LOW_L                         0x160698
+
+#define mmMME2_CTRL_SHADOW_1_BASE_ADDR_LOW_O                         0x16069C
+
+#define mmMME2_CTRL_SHADOW_1_HEADER_LOW                              0x1606A0
+
+#define mmMME2_CTRL_SHADOW_1_HEADER_HIGH                             0x1606A4
+
+#define mmMME2_CTRL_SHADOW_1_CONV_KERNEL_SIZE_MINUS_1                0x1606A8
+
+#define mmMME2_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_LOW                0x1606AC
+
+#define mmMME2_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_HIGH               0x1606B0
+
+#define mmMME2_CTRL_SHADOW_1_NUM_ITERATIONS_MINUS_1                  0x1606B4
+
+#define mmMME2_CTRL_SHADOW_1_OUTER_LOOP                              0x1606B8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_0               0x1606BC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_1               0x1606C0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_2               0x1606C4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_3               0x1606C8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_4               0x1606CC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_0                  0x1606D0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_1                  0x1606D4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_2                  0x1606D8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_3                  0x1606DC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_4                  0x1606E0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_0                     0x1606E4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_1                     0x1606E8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_2                     0x1606EC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_3                     0x1606F0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_0              0x1606F4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_1              0x1606F8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_2              0x1606FC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_3              0x160700
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x160704
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_0                 0x160708
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_1                 0x16070C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_2                 0x160710
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_3                 0x160714
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_4                 0x160718
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_START_OFFSET_0                    0x16071C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_START_OFFSET_1                    0x160720
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_START_OFFSET_2                    0x160724
+
+#define mmMME2_CTRL_SHADOW_1_AGU_S_START_OFFSET_3                    0x160728
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_0               0x16072C
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_1               0x160730
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_2               0x160734
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_3               0x160738
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_4               0x16073C
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_0                  0x160740
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_1                  0x160744
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_2                  0x160748
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_3                  0x16074C
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_4                  0x160750
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_0                     0x160754
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_1                     0x160758
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_2                     0x16075C
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_3                     0x160760
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_0              0x160764
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_1              0x160768
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_2              0x16076C
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_3              0x160770
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x160774
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x160778
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x16077C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x160780
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x160784
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x160788
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_0              0x16078C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_1              0x160790
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_2              0x160794
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_3              0x160798
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x16079C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x1607A0
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x1607A4
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x1607A8
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x1607AC
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_0             0x1607B0
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_1             0x1607B4
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_2             0x1607B8
+
+#define mmMME2_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_3             0x1607BC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_0               0x1607C0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_1               0x1607C4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_2               0x1607C8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_3               0x1607CC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_4               0x1607D0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_0                  0x1607D4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_1                  0x1607D8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_2                  0x1607DC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_3                  0x1607E0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_4                  0x1607E4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_0                     0x1607E8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_1                     0x1607EC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_2                     0x1607F0
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_3                     0x1607F4
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_0              0x1607F8
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_1              0x1607FC
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_2              0x160800
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_3              0x160804
+
+#define mmMME2_CTRL_SHADOW_1_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x160808
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x16080C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x160810
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x160814
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x160818
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x16081C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_0              0x160820
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_1              0x160824
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_2              0x160828
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_3              0x16082C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x160830
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x160834
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x160838
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x16083C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x160840
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_0             0x160844
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_1             0x160848
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_2             0x16084C
+
+#define mmMME2_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_3             0x160850
+
+#define mmMME2_CTRL_SHADOW_1_DESC_SB_REPEAT                          0x160854
+
+#define mmMME2_CTRL_SHADOW_1_DESC_RATE_LIMITER                       0x160858
+
+#define mmMME2_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x16085C
+
+#define mmMME2_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x160860
+
+#define mmMME2_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_HIGH              0x160864
+
+#define mmMME2_CTRL_SHADOW_1_DESC_SYNC_OBJECT_DATA                   0x160868
+
+#define mmMME2_CTRL_SHADOW_1_DESC_AXI_USER_DATA                      0x16086C
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PERF_EVT_S                         0x160870
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PERF_EVT_L_LOCAL                   0x160874
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PERF_EVT_L_REMOTE                  0x160878
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PERF_EVT_O_LOCAL                   0x16087C
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PERF_EVT_O_REMOTE                  0x160880
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PADDING_VALUE_S                    0x160884
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PADDING_VALUE_L                    0x160888
+
+#define mmMME2_CTRL_SHADOW_1_DESC_META_DATA_AGU_S                    0x16088C
+
+#define mmMME2_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_LOCAL              0x160890
+
+#define mmMME2_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_REMOTE             0x160894
+
+#define mmMME2_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_LOCAL              0x160898
+
+#define mmMME2_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_REMOTE             0x16089C
+
+#define mmMME2_CTRL_SHADOW_1_DESC_PCU_RL_SATURATION                  0x1608A0
+
+#define mmMME2_CTRL_SHADOW_1_DESC_DUMMY                              0x1608A4
+
+#define mmMME2_CTRL_SHADOW_2_STATUS                                  0x160900
+
+#define mmMME2_CTRL_SHADOW_2_BASE_ADDR_HIGH_S                        0x160908
+
+#define mmMME2_CTRL_SHADOW_2_BASE_ADDR_HIGH_L                        0x16090C
+
+#define mmMME2_CTRL_SHADOW_2_BASE_ADDR_HIGH_O                        0x160910
+
+#define mmMME2_CTRL_SHADOW_2_BASE_ADDR_LOW_S                         0x160914
+
+#define mmMME2_CTRL_SHADOW_2_BASE_ADDR_LOW_L                         0x160918
+
+#define mmMME2_CTRL_SHADOW_2_BASE_ADDR_LOW_O                         0x16091C
+
+#define mmMME2_CTRL_SHADOW_2_HEADER_LOW                              0x160920
+
+#define mmMME2_CTRL_SHADOW_2_HEADER_HIGH                             0x160924
+
+#define mmMME2_CTRL_SHADOW_2_CONV_KERNEL_SIZE_MINUS_1                0x160928
+
+#define mmMME2_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_LOW                0x16092C
+
+#define mmMME2_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_HIGH               0x160930
+
+#define mmMME2_CTRL_SHADOW_2_NUM_ITERATIONS_MINUS_1                  0x160934
+
+#define mmMME2_CTRL_SHADOW_2_OUTER_LOOP                              0x160938
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_0               0x16093C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_1               0x160940
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_2               0x160944
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_3               0x160948
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_4               0x16094C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_0                  0x160950
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_1                  0x160954
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_2                  0x160958
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_3                  0x16095C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_4                  0x160960
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_0                     0x160964
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_1                     0x160968
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_2                     0x16096C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_3                     0x160970
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_0              0x160974
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_1              0x160978
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_2              0x16097C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_3              0x160980
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x160984
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_0                 0x160988
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_1                 0x16098C
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_2                 0x160990
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_3                 0x160994
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_4                 0x160998
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_START_OFFSET_0                    0x16099C
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_START_OFFSET_1                    0x1609A0
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_START_OFFSET_2                    0x1609A4
+
+#define mmMME2_CTRL_SHADOW_2_AGU_S_START_OFFSET_3                    0x1609A8
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_0               0x1609AC
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_1               0x1609B0
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_2               0x1609B4
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_3               0x1609B8
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_4               0x1609BC
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_0                  0x1609C0
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_1                  0x1609C4
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_2                  0x1609C8
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_3                  0x1609CC
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_4                  0x1609D0
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_0                     0x1609D4
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_1                     0x1609D8
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_2                     0x1609DC
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_3                     0x1609E0
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_0              0x1609E4
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_1              0x1609E8
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_2              0x1609EC
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_3              0x1609F0
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x1609F4
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x1609F8
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x1609FC
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x160A00
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x160A04
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x160A08
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_0              0x160A0C
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_1              0x160A10
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_2              0x160A14
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_3              0x160A18
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x160A1C
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x160A20
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x160A24
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x160A28
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x160A2C
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_0             0x160A30
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_1             0x160A34
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_2             0x160A38
+
+#define mmMME2_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_3             0x160A3C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_0               0x160A40
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_1               0x160A44
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_2               0x160A48
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_3               0x160A4C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_4               0x160A50
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_0                  0x160A54
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_1                  0x160A58
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_2                  0x160A5C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_3                  0x160A60
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_4                  0x160A64
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_0                     0x160A68
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_1                     0x160A6C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_2                     0x160A70
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_3                     0x160A74
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_0              0x160A78
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_1              0x160A7C
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_2              0x160A80
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_3              0x160A84
+
+#define mmMME2_CTRL_SHADOW_2_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x160A88
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x160A8C
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x160A90
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x160A94
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x160A98
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x160A9C
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_0              0x160AA0
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_1              0x160AA4
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_2              0x160AA8
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_3              0x160AAC
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x160AB0
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x160AB4
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x160AB8
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x160ABC
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x160AC0
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_0             0x160AC4
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_1             0x160AC8
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_2             0x160ACC
+
+#define mmMME2_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_3             0x160AD0
+
+#define mmMME2_CTRL_SHADOW_2_DESC_SB_REPEAT                          0x160AD4
+
+#define mmMME2_CTRL_SHADOW_2_DESC_RATE_LIMITER                       0x160AD8
+
+#define mmMME2_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x160ADC
+
+#define mmMME2_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x160AE0
+
+#define mmMME2_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_HIGH              0x160AE4
+
+#define mmMME2_CTRL_SHADOW_2_DESC_SYNC_OBJECT_DATA                   0x160AE8
+
+#define mmMME2_CTRL_SHADOW_2_DESC_AXI_USER_DATA                      0x160AEC
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PERF_EVT_S                         0x160AF0
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PERF_EVT_L_LOCAL                   0x160AF4
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PERF_EVT_L_REMOTE                  0x160AF8
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PERF_EVT_O_LOCAL                   0x160AFC
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PERF_EVT_O_REMOTE                  0x160B00
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PADDING_VALUE_S                    0x160B04
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PADDING_VALUE_L                    0x160B08
+
+#define mmMME2_CTRL_SHADOW_2_DESC_META_DATA_AGU_S                    0x160B0C
+
+#define mmMME2_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_LOCAL              0x160B10
+
+#define mmMME2_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_REMOTE             0x160B14
+
+#define mmMME2_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_LOCAL              0x160B18
+
+#define mmMME2_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_REMOTE             0x160B1C
+
+#define mmMME2_CTRL_SHADOW_2_DESC_PCU_RL_SATURATION                  0x160B20
+
+#define mmMME2_CTRL_SHADOW_2_DESC_DUMMY                              0x160B24
+
+#define mmMME2_CTRL_SHADOW_3_STATUS                                  0x160B80
+
+#define mmMME2_CTRL_SHADOW_3_BASE_ADDR_HIGH_S                        0x160B88
+
+#define mmMME2_CTRL_SHADOW_3_BASE_ADDR_HIGH_L                        0x160B8C
+
+#define mmMME2_CTRL_SHADOW_3_BASE_ADDR_HIGH_O                        0x160B90
+
+#define mmMME2_CTRL_SHADOW_3_BASE_ADDR_LOW_S                         0x160B94
+
+#define mmMME2_CTRL_SHADOW_3_BASE_ADDR_LOW_L                         0x160B98
+
+#define mmMME2_CTRL_SHADOW_3_BASE_ADDR_LOW_O                         0x160B9C
+
+#define mmMME2_CTRL_SHADOW_3_HEADER_LOW                              0x160BA0
+
+#define mmMME2_CTRL_SHADOW_3_HEADER_HIGH                             0x160BA4
+
+#define mmMME2_CTRL_SHADOW_3_CONV_KERNEL_SIZE_MINUS_1                0x160BA8
+
+#define mmMME2_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_LOW                0x160BAC
+
+#define mmMME2_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_HIGH               0x160BB0
+
+#define mmMME2_CTRL_SHADOW_3_NUM_ITERATIONS_MINUS_1                  0x160BB4
+
+#define mmMME2_CTRL_SHADOW_3_OUTER_LOOP                              0x160BB8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_0               0x160BBC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_1               0x160BC0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_2               0x160BC4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_3               0x160BC8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_4               0x160BCC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_0                  0x160BD0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_1                  0x160BD4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_2                  0x160BD8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_3                  0x160BDC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_4                  0x160BE0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_0                     0x160BE4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_1                     0x160BE8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_2                     0x160BEC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_3                     0x160BF0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_0              0x160BF4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_1              0x160BF8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_2              0x160BFC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_3              0x160C00
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x160C04
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_0                 0x160C08
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_1                 0x160C0C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_2                 0x160C10
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_3                 0x160C14
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_4                 0x160C18
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_START_OFFSET_0                    0x160C1C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_START_OFFSET_1                    0x160C20
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_START_OFFSET_2                    0x160C24
+
+#define mmMME2_CTRL_SHADOW_3_AGU_S_START_OFFSET_3                    0x160C28
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_0               0x160C2C
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_1               0x160C30
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_2               0x160C34
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_3               0x160C38
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_4               0x160C3C
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_0                  0x160C40
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_1                  0x160C44
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_2                  0x160C48
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_3                  0x160C4C
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_4                  0x160C50
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_0                     0x160C54
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_1                     0x160C58
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_2                     0x160C5C
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_3                     0x160C60
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_0              0x160C64
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_1              0x160C68
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_2              0x160C6C
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_3              0x160C70
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x160C74
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x160C78
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x160C7C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x160C80
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x160C84
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x160C88
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_0              0x160C8C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_1              0x160C90
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_2              0x160C94
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_3              0x160C98
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x160C9C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x160CA0
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x160CA4
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x160CA8
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x160CAC
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_0             0x160CB0
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_1             0x160CB4
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_2             0x160CB8
+
+#define mmMME2_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_3             0x160CBC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_0               0x160CC0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_1               0x160CC4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_2               0x160CC8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_3               0x160CCC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_4               0x160CD0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_0                  0x160CD4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_1                  0x160CD8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_2                  0x160CDC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_3                  0x160CE0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_4                  0x160CE4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_0                     0x160CE8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_1                     0x160CEC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_2                     0x160CF0
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_3                     0x160CF4
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_0              0x160CF8
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_1              0x160CFC
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_2              0x160D00
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_3              0x160D04
+
+#define mmMME2_CTRL_SHADOW_3_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x160D08
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x160D0C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x160D10
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x160D14
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x160D18
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x160D1C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_0              0x160D20
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_1              0x160D24
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_2              0x160D28
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_3              0x160D2C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x160D30
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x160D34
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x160D38
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x160D3C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x160D40
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_0             0x160D44
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_1             0x160D48
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_2             0x160D4C
+
+#define mmMME2_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_3             0x160D50
+
+#define mmMME2_CTRL_SHADOW_3_DESC_SB_REPEAT                          0x160D54
+
+#define mmMME2_CTRL_SHADOW_3_DESC_RATE_LIMITER                       0x160D58
+
+#define mmMME2_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x160D5C
+
+#define mmMME2_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x160D60
+
+#define mmMME2_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_HIGH              0x160D64
+
+#define mmMME2_CTRL_SHADOW_3_DESC_SYNC_OBJECT_DATA                   0x160D68
+
+#define mmMME2_CTRL_SHADOW_3_DESC_AXI_USER_DATA                      0x160D6C
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PERF_EVT_S                         0x160D70
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PERF_EVT_L_LOCAL                   0x160D74
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PERF_EVT_L_REMOTE                  0x160D78
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PERF_EVT_O_LOCAL                   0x160D7C
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PERF_EVT_O_REMOTE                  0x160D80
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PADDING_VALUE_S                    0x160D84
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PADDING_VALUE_L                    0x160D88
+
+#define mmMME2_CTRL_SHADOW_3_DESC_META_DATA_AGU_S                    0x160D8C
+
+#define mmMME2_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_LOCAL              0x160D90
+
+#define mmMME2_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_REMOTE             0x160D94
+
+#define mmMME2_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_LOCAL              0x160D98
+
+#define mmMME2_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_REMOTE             0x160D9C
+
+#define mmMME2_CTRL_SHADOW_3_DESC_PCU_RL_SATURATION                  0x160DA0
+
+#define mmMME2_CTRL_SHADOW_3_DESC_DUMMY                              0x160DA4
+
+#endif /* ASIC_REG_MME2_CTRL_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_qm_regs.h
new file mode 100644
index 000000000000..c1ea6a422010
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme2_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MME2_QM_REGS_H_
+#define ASIC_REG_MME2_QM_REGS_H_
+
+/*
+ *****************************************
+ *   MME2_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmMME2_QM_GLBL_CFG0                                          0x168000
+
+#define mmMME2_QM_GLBL_CFG1                                          0x168004
+
+#define mmMME2_QM_GLBL_PROT                                          0x168008
+
+#define mmMME2_QM_GLBL_ERR_CFG                                       0x16800C
+
+#define mmMME2_QM_GLBL_SECURE_PROPS_0                                0x168010
+
+#define mmMME2_QM_GLBL_SECURE_PROPS_1                                0x168014
+
+#define mmMME2_QM_GLBL_SECURE_PROPS_2                                0x168018
+
+#define mmMME2_QM_GLBL_SECURE_PROPS_3                                0x16801C
+
+#define mmMME2_QM_GLBL_SECURE_PROPS_4                                0x168020
+
+#define mmMME2_QM_GLBL_NON_SECURE_PROPS_0                            0x168024
+
+#define mmMME2_QM_GLBL_NON_SECURE_PROPS_1                            0x168028
+
+#define mmMME2_QM_GLBL_NON_SECURE_PROPS_2                            0x16802C
+
+#define mmMME2_QM_GLBL_NON_SECURE_PROPS_3                            0x168030
+
+#define mmMME2_QM_GLBL_NON_SECURE_PROPS_4                            0x168034
+
+#define mmMME2_QM_GLBL_STS0                                          0x168038
+
+#define mmMME2_QM_GLBL_STS1_0                                        0x168040
+
+#define mmMME2_QM_GLBL_STS1_1                                        0x168044
+
+#define mmMME2_QM_GLBL_STS1_2                                        0x168048
+
+#define mmMME2_QM_GLBL_STS1_3                                        0x16804C
+
+#define mmMME2_QM_GLBL_STS1_4                                        0x168050
+
+#define mmMME2_QM_GLBL_MSG_EN_0                                      0x168054
+
+#define mmMME2_QM_GLBL_MSG_EN_1                                      0x168058
+
+#define mmMME2_QM_GLBL_MSG_EN_2                                      0x16805C
+
+#define mmMME2_QM_GLBL_MSG_EN_3                                      0x168060
+
+#define mmMME2_QM_GLBL_MSG_EN_4                                      0x168068
+
+#define mmMME2_QM_PQ_BASE_LO_0                                       0x168070
+
+#define mmMME2_QM_PQ_BASE_LO_1                                       0x168074
+
+#define mmMME2_QM_PQ_BASE_LO_2                                       0x168078
+
+#define mmMME2_QM_PQ_BASE_LO_3                                       0x16807C
+
+#define mmMME2_QM_PQ_BASE_HI_0                                       0x168080
+
+#define mmMME2_QM_PQ_BASE_HI_1                                       0x168084
+
+#define mmMME2_QM_PQ_BASE_HI_2                                       0x168088
+
+#define mmMME2_QM_PQ_BASE_HI_3                                       0x16808C
+
+#define mmMME2_QM_PQ_SIZE_0                                          0x168090
+
+#define mmMME2_QM_PQ_SIZE_1                                          0x168094
+
+#define mmMME2_QM_PQ_SIZE_2                                          0x168098
+
+#define mmMME2_QM_PQ_SIZE_3                                          0x16809C
+
+#define mmMME2_QM_PQ_PI_0                                            0x1680A0
+
+#define mmMME2_QM_PQ_PI_1                                            0x1680A4
+
+#define mmMME2_QM_PQ_PI_2                                            0x1680A8
+
+#define mmMME2_QM_PQ_PI_3                                            0x1680AC
+
+#define mmMME2_QM_PQ_CI_0                                            0x1680B0
+
+#define mmMME2_QM_PQ_CI_1                                            0x1680B4
+
+#define mmMME2_QM_PQ_CI_2                                            0x1680B8
+
+#define mmMME2_QM_PQ_CI_3                                            0x1680BC
+
+#define mmMME2_QM_PQ_CFG0_0                                          0x1680C0
+
+#define mmMME2_QM_PQ_CFG0_1                                          0x1680C4
+
+#define mmMME2_QM_PQ_CFG0_2                                          0x1680C8
+
+#define mmMME2_QM_PQ_CFG0_3                                          0x1680CC
+
+#define mmMME2_QM_PQ_CFG1_0                                          0x1680D0
+
+#define mmMME2_QM_PQ_CFG1_1                                          0x1680D4
+
+#define mmMME2_QM_PQ_CFG1_2                                          0x1680D8
+
+#define mmMME2_QM_PQ_CFG1_3                                          0x1680DC
+
+#define mmMME2_QM_PQ_ARUSER_31_11_0                                  0x1680E0
+
+#define mmMME2_QM_PQ_ARUSER_31_11_1                                  0x1680E4
+
+#define mmMME2_QM_PQ_ARUSER_31_11_2                                  0x1680E8
+
+#define mmMME2_QM_PQ_ARUSER_31_11_3                                  0x1680EC
+
+#define mmMME2_QM_PQ_STS0_0                                          0x1680F0
+
+#define mmMME2_QM_PQ_STS0_1                                          0x1680F4
+
+#define mmMME2_QM_PQ_STS0_2                                          0x1680F8
+
+#define mmMME2_QM_PQ_STS0_3                                          0x1680FC
+
+#define mmMME2_QM_PQ_STS1_0                                          0x168100
+
+#define mmMME2_QM_PQ_STS1_1                                          0x168104
+
+#define mmMME2_QM_PQ_STS1_2                                          0x168108
+
+#define mmMME2_QM_PQ_STS1_3                                          0x16810C
+
+#define mmMME2_QM_CQ_CFG0_0                                          0x168110
+
+#define mmMME2_QM_CQ_CFG0_1                                          0x168114
+
+#define mmMME2_QM_CQ_CFG0_2                                          0x168118
+
+#define mmMME2_QM_CQ_CFG0_3                                          0x16811C
+
+#define mmMME2_QM_CQ_CFG0_4                                          0x168120
+
+#define mmMME2_QM_CQ_CFG1_0                                          0x168124
+
+#define mmMME2_QM_CQ_CFG1_1                                          0x168128
+
+#define mmMME2_QM_CQ_CFG1_2                                          0x16812C
+
+#define mmMME2_QM_CQ_CFG1_3                                          0x168130
+
+#define mmMME2_QM_CQ_CFG1_4                                          0x168134
+
+#define mmMME2_QM_CQ_ARUSER_31_11_0                                  0x168138
+
+#define mmMME2_QM_CQ_ARUSER_31_11_1                                  0x16813C
+
+#define mmMME2_QM_CQ_ARUSER_31_11_2                                  0x168140
+
+#define mmMME2_QM_CQ_ARUSER_31_11_3                                  0x168144
+
+#define mmMME2_QM_CQ_ARUSER_31_11_4                                  0x168148
+
+#define mmMME2_QM_CQ_STS0_0                                          0x16814C
+
+#define mmMME2_QM_CQ_STS0_1                                          0x168150
+
+#define mmMME2_QM_CQ_STS0_2                                          0x168154
+
+#define mmMME2_QM_CQ_STS0_3                                          0x168158
+
+#define mmMME2_QM_CQ_STS0_4                                          0x16815C
+
+#define mmMME2_QM_CQ_STS1_0                                          0x168160
+
+#define mmMME2_QM_CQ_STS1_1                                          0x168164
+
+#define mmMME2_QM_CQ_STS1_2                                          0x168168
+
+#define mmMME2_QM_CQ_STS1_3                                          0x16816C
+
+#define mmMME2_QM_CQ_STS1_4                                          0x168170
+
+#define mmMME2_QM_CQ_PTR_LO_0                                        0x168174
+
+#define mmMME2_QM_CQ_PTR_HI_0                                        0x168178
+
+#define mmMME2_QM_CQ_TSIZE_0                                         0x16817C
+
+#define mmMME2_QM_CQ_CTL_0                                           0x168180
+
+#define mmMME2_QM_CQ_PTR_LO_1                                        0x168184
+
+#define mmMME2_QM_CQ_PTR_HI_1                                        0x168188
+
+#define mmMME2_QM_CQ_TSIZE_1                                         0x16818C
+
+#define mmMME2_QM_CQ_CTL_1                                           0x168190
+
+#define mmMME2_QM_CQ_PTR_LO_2                                        0x168194
+
+#define mmMME2_QM_CQ_PTR_HI_2                                        0x168198
+
+#define mmMME2_QM_CQ_TSIZE_2                                         0x16819C
+
+#define mmMME2_QM_CQ_CTL_2                                           0x1681A0
+
+#define mmMME2_QM_CQ_PTR_LO_3                                        0x1681A4
+
+#define mmMME2_QM_CQ_PTR_HI_3                                        0x1681A8
+
+#define mmMME2_QM_CQ_TSIZE_3                                         0x1681AC
+
+#define mmMME2_QM_CQ_CTL_3                                           0x1681B0
+
+#define mmMME2_QM_CQ_PTR_LO_4                                        0x1681B4
+
+#define mmMME2_QM_CQ_PTR_HI_4                                        0x1681B8
+
+#define mmMME2_QM_CQ_TSIZE_4                                         0x1681BC
+
+#define mmMME2_QM_CQ_CTL_4                                           0x1681C0
+
+#define mmMME2_QM_CQ_PTR_LO_STS_0                                    0x1681C4
+
+#define mmMME2_QM_CQ_PTR_LO_STS_1                                    0x1681C8
+
+#define mmMME2_QM_CQ_PTR_LO_STS_2                                    0x1681CC
+
+#define mmMME2_QM_CQ_PTR_LO_STS_3                                    0x1681D0
+
+#define mmMME2_QM_CQ_PTR_LO_STS_4                                    0x1681D4
+
+#define mmMME2_QM_CQ_PTR_HI_STS_0                                    0x1681D8
+
+#define mmMME2_QM_CQ_PTR_HI_STS_1                                    0x1681DC
+
+#define mmMME2_QM_CQ_PTR_HI_STS_2                                    0x1681E0
+
+#define mmMME2_QM_CQ_PTR_HI_STS_3                                    0x1681E4
+
+#define mmMME2_QM_CQ_PTR_HI_STS_4                                    0x1681E8
+
+#define mmMME2_QM_CQ_TSIZE_STS_0                                     0x1681EC
+
+#define mmMME2_QM_CQ_TSIZE_STS_1                                     0x1681F0
+
+#define mmMME2_QM_CQ_TSIZE_STS_2                                     0x1681F4
+
+#define mmMME2_QM_CQ_TSIZE_STS_3                                     0x1681F8
+
+#define mmMME2_QM_CQ_TSIZE_STS_4                                     0x1681FC
+
+#define mmMME2_QM_CQ_CTL_STS_0                                       0x168200
+
+#define mmMME2_QM_CQ_CTL_STS_1                                       0x168204
+
+#define mmMME2_QM_CQ_CTL_STS_2                                       0x168208
+
+#define mmMME2_QM_CQ_CTL_STS_3                                       0x16820C
+
+#define mmMME2_QM_CQ_CTL_STS_4                                       0x168210
+
+#define mmMME2_QM_CQ_IFIFO_CNT_0                                     0x168214
+
+#define mmMME2_QM_CQ_IFIFO_CNT_1                                     0x168218
+
+#define mmMME2_QM_CQ_IFIFO_CNT_2                                     0x16821C
+
+#define mmMME2_QM_CQ_IFIFO_CNT_3                                     0x168220
+
+#define mmMME2_QM_CQ_IFIFO_CNT_4                                     0x168224
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_LO_0                             0x168228
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_LO_1                             0x16822C
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_LO_2                             0x168230
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_LO_3                             0x168234
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_LO_4                             0x168238
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_HI_0                             0x16823C
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_HI_1                             0x168240
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_HI_2                             0x168244
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_HI_3                             0x168248
+
+#define mmMME2_QM_CP_MSG_BASE0_ADDR_HI_4                             0x16824C
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_LO_0                             0x168250
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_LO_1                             0x168254
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_LO_2                             0x168258
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_LO_3                             0x16825C
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_LO_4                             0x168260
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_HI_0                             0x168264
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_HI_1                             0x168268
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_HI_2                             0x16826C
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_HI_3                             0x168270
+
+#define mmMME2_QM_CP_MSG_BASE1_ADDR_HI_4                             0x168274
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_LO_0                             0x168278
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_LO_1                             0x16827C
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_LO_2                             0x168280
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_LO_3                             0x168284
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_LO_4                             0x168288
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_HI_0                             0x16828C
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_HI_1                             0x168290
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_HI_2                             0x168294
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_HI_3                             0x168298
+
+#define mmMME2_QM_CP_MSG_BASE2_ADDR_HI_4                             0x16829C
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_LO_0                             0x1682A0
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_LO_1                             0x1682A4
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_LO_2                             0x1682A8
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_LO_3                             0x1682AC
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_LO_4                             0x1682B0
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_HI_0                             0x1682B4
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_HI_1                             0x1682B8
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_HI_2                             0x1682BC
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_HI_3                             0x1682C0
+
+#define mmMME2_QM_CP_MSG_BASE3_ADDR_HI_4                             0x1682C4
+
+#define mmMME2_QM_CP_LDMA_TSIZE_OFFSET_0                             0x1682C8
+
+#define mmMME2_QM_CP_LDMA_TSIZE_OFFSET_1                             0x1682CC
+
+#define mmMME2_QM_CP_LDMA_TSIZE_OFFSET_2                             0x1682D0
+
+#define mmMME2_QM_CP_LDMA_TSIZE_OFFSET_3                             0x1682D4
+
+#define mmMME2_QM_CP_LDMA_TSIZE_OFFSET_4                             0x1682D8
+
+#define mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0x1682E0
+
+#define mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0x1682E4
+
+#define mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0x1682E8
+
+#define mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0x1682EC
+
+#define mmMME2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0x1682F0
+
+#define mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0x1682F4
+
+#define mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0x1682F8
+
+#define mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0x1682FC
+
+#define mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0x168300
+
+#define mmMME2_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0x168304
+
+#define mmMME2_QM_CP_FENCE0_RDATA_0                                  0x168308
+
+#define mmMME2_QM_CP_FENCE0_RDATA_1                                  0x16830C
+
+#define mmMME2_QM_CP_FENCE0_RDATA_2                                  0x168310
+
+#define mmMME2_QM_CP_FENCE0_RDATA_3                                  0x168314
+
+#define mmMME2_QM_CP_FENCE0_RDATA_4                                  0x168318
+
+#define mmMME2_QM_CP_FENCE1_RDATA_0                                  0x16831C
+
+#define mmMME2_QM_CP_FENCE1_RDATA_1                                  0x168320
+
+#define mmMME2_QM_CP_FENCE1_RDATA_2                                  0x168324
+
+#define mmMME2_QM_CP_FENCE1_RDATA_3                                  0x168328
+
+#define mmMME2_QM_CP_FENCE1_RDATA_4                                  0x16832C
+
+#define mmMME2_QM_CP_FENCE2_RDATA_0                                  0x168330
+
+#define mmMME2_QM_CP_FENCE2_RDATA_1                                  0x168334
+
+#define mmMME2_QM_CP_FENCE2_RDATA_2                                  0x168338
+
+#define mmMME2_QM_CP_FENCE2_RDATA_3                                  0x16833C
+
+#define mmMME2_QM_CP_FENCE2_RDATA_4                                  0x168340
+
+#define mmMME2_QM_CP_FENCE3_RDATA_0                                  0x168344
+
+#define mmMME2_QM_CP_FENCE3_RDATA_1                                  0x168348
+
+#define mmMME2_QM_CP_FENCE3_RDATA_2                                  0x16834C
+
+#define mmMME2_QM_CP_FENCE3_RDATA_3                                  0x168350
+
+#define mmMME2_QM_CP_FENCE3_RDATA_4                                  0x168354
+
+#define mmMME2_QM_CP_FENCE0_CNT_0                                    0x168358
+
+#define mmMME2_QM_CP_FENCE0_CNT_1                                    0x16835C
+
+#define mmMME2_QM_CP_FENCE0_CNT_2                                    0x168360
+
+#define mmMME2_QM_CP_FENCE0_CNT_3                                    0x168364
+
+#define mmMME2_QM_CP_FENCE0_CNT_4                                    0x168368
+
+#define mmMME2_QM_CP_FENCE1_CNT_0                                    0x16836C
+
+#define mmMME2_QM_CP_FENCE1_CNT_1                                    0x168370
+
+#define mmMME2_QM_CP_FENCE1_CNT_2                                    0x168374
+
+#define mmMME2_QM_CP_FENCE1_CNT_3                                    0x168378
+
+#define mmMME2_QM_CP_FENCE1_CNT_4                                    0x16837C
+
+#define mmMME2_QM_CP_FENCE2_CNT_0                                    0x168380
+
+#define mmMME2_QM_CP_FENCE2_CNT_1                                    0x168384
+
+#define mmMME2_QM_CP_FENCE2_CNT_2                                    0x168388
+
+#define mmMME2_QM_CP_FENCE2_CNT_3                                    0x16838C
+
+#define mmMME2_QM_CP_FENCE2_CNT_4                                    0x168390
+
+#define mmMME2_QM_CP_FENCE3_CNT_0                                    0x168394
+
+#define mmMME2_QM_CP_FENCE3_CNT_1                                    0x168398
+
+#define mmMME2_QM_CP_FENCE3_CNT_2                                    0x16839C
+
+#define mmMME2_QM_CP_FENCE3_CNT_3                                    0x1683A0
+
+#define mmMME2_QM_CP_FENCE3_CNT_4                                    0x1683A4
+
+#define mmMME2_QM_CP_STS_0                                           0x1683A8
+
+#define mmMME2_QM_CP_STS_1                                           0x1683AC
+
+#define mmMME2_QM_CP_STS_2                                           0x1683B0
+
+#define mmMME2_QM_CP_STS_3                                           0x1683B4
+
+#define mmMME2_QM_CP_STS_4                                           0x1683B8
+
+#define mmMME2_QM_CP_CURRENT_INST_LO_0                               0x1683BC
+
+#define mmMME2_QM_CP_CURRENT_INST_LO_1                               0x1683C0
+
+#define mmMME2_QM_CP_CURRENT_INST_LO_2                               0x1683C4
+
+#define mmMME2_QM_CP_CURRENT_INST_LO_3                               0x1683C8
+
+#define mmMME2_QM_CP_CURRENT_INST_LO_4                               0x1683CC
+
+#define mmMME2_QM_CP_CURRENT_INST_HI_0                               0x1683D0
+
+#define mmMME2_QM_CP_CURRENT_INST_HI_1                               0x1683D4
+
+#define mmMME2_QM_CP_CURRENT_INST_HI_2                               0x1683D8
+
+#define mmMME2_QM_CP_CURRENT_INST_HI_3                               0x1683DC
+
+#define mmMME2_QM_CP_CURRENT_INST_HI_4                               0x1683E0
+
+#define mmMME2_QM_CP_BARRIER_CFG_0                                   0x1683F4
+
+#define mmMME2_QM_CP_BARRIER_CFG_1                                   0x1683F8
+
+#define mmMME2_QM_CP_BARRIER_CFG_2                                   0x1683FC
+
+#define mmMME2_QM_CP_BARRIER_CFG_3                                   0x168400
+
+#define mmMME2_QM_CP_BARRIER_CFG_4                                   0x168404
+
+#define mmMME2_QM_CP_DBG_0_0                                         0x168408
+
+#define mmMME2_QM_CP_DBG_0_1                                         0x16840C
+
+#define mmMME2_QM_CP_DBG_0_2                                         0x168410
+
+#define mmMME2_QM_CP_DBG_0_3                                         0x168414
+
+#define mmMME2_QM_CP_DBG_0_4                                         0x168418
+
+#define mmMME2_QM_CP_ARUSER_31_11_0                                  0x16841C
+
+#define mmMME2_QM_CP_ARUSER_31_11_1                                  0x168420
+
+#define mmMME2_QM_CP_ARUSER_31_11_2                                  0x168424
+
+#define mmMME2_QM_CP_ARUSER_31_11_3                                  0x168428
+
+#define mmMME2_QM_CP_ARUSER_31_11_4                                  0x16842C
+
+#define mmMME2_QM_CP_AWUSER_31_11_0                                  0x168430
+
+#define mmMME2_QM_CP_AWUSER_31_11_1                                  0x168434
+
+#define mmMME2_QM_CP_AWUSER_31_11_2                                  0x168438
+
+#define mmMME2_QM_CP_AWUSER_31_11_3                                  0x16843C
+
+#define mmMME2_QM_CP_AWUSER_31_11_4                                  0x168440
+
+#define mmMME2_QM_ARB_CFG_0                                          0x168A00
+
+#define mmMME2_QM_ARB_CHOISE_Q_PUSH                                  0x168A04
+
+#define mmMME2_QM_ARB_WRR_WEIGHT_0                                   0x168A08
+
+#define mmMME2_QM_ARB_WRR_WEIGHT_1                                   0x168A0C
+
+#define mmMME2_QM_ARB_WRR_WEIGHT_2                                   0x168A10
+
+#define mmMME2_QM_ARB_WRR_WEIGHT_3                                   0x168A14
+
+#define mmMME2_QM_ARB_CFG_1                                          0x168A18
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_0                               0x168A20
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_1                               0x168A24
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_2                               0x168A28
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_3                               0x168A2C
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_4                               0x168A30
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_5                               0x168A34
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_6                               0x168A38
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_7                               0x168A3C
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_8                               0x168A40
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_9                               0x168A44
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_10                              0x168A48
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_11                              0x168A4C
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_12                              0x168A50
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_13                              0x168A54
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_14                              0x168A58
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_15                              0x168A5C
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_16                              0x168A60
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_17                              0x168A64
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_18                              0x168A68
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_19                              0x168A6C
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_20                              0x168A70
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_21                              0x168A74
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_22                              0x168A78
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_23                              0x168A7C
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_24                              0x168A80
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_25                              0x168A84
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_26                              0x168A88
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_27                              0x168A8C
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_28                              0x168A90
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_29                              0x168A94
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_30                              0x168A98
+
+#define mmMME2_QM_ARB_MST_AVAIL_CRED_31                              0x168A9C
+
+#define mmMME2_QM_ARB_MST_CRED_INC                                   0x168AA0
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0x168AA4
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0x168AA8
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0x168AAC
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0x168AB0
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0x168AB4
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0x168AB8
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0x168ABC
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0x168AC0
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0x168AC4
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0x168AC8
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0x168ACC
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0x168AD0
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0x168AD4
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0x168AD8
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0x168ADC
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0x168AE0
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0x168AE4
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0x168AE8
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0x168AEC
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0x168AF0
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0x168AF4
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0x168AF8
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0x168AFC
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0x168B00
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0x168B04
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0x168B08
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0x168B0C
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0x168B10
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0x168B14
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0x168B18
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0x168B1C
+
+#define mmMME2_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0x168B20
+
+#define mmMME2_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0x168B28
+
+#define mmMME2_QM_ARB_MST_SLAVE_EN                                   0x168B2C
+
+#define mmMME2_QM_ARB_MST_QUIET_PER                                  0x168B34
+
+#define mmMME2_QM_ARB_SLV_CHOISE_WDT                                 0x168B38
+
+#define mmMME2_QM_ARB_SLV_ID                                         0x168B3C
+
+#define mmMME2_QM_ARB_MSG_MAX_INFLIGHT                               0x168B44
+
+#define mmMME2_QM_ARB_MSG_AWUSER_31_11                               0x168B48
+
+#define mmMME2_QM_ARB_MSG_AWUSER_SEC_PROP                            0x168B4C
+
+#define mmMME2_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0x168B50
+
+#define mmMME2_QM_ARB_BASE_LO                                        0x168B54
+
+#define mmMME2_QM_ARB_BASE_HI                                        0x168B58
+
+#define mmMME2_QM_ARB_STATE_STS                                      0x168B80
+
+#define mmMME2_QM_ARB_CHOISE_FULLNESS_STS                            0x168B84
+
+#define mmMME2_QM_ARB_MSG_STS                                        0x168B88
+
+#define mmMME2_QM_ARB_SLV_CHOISE_Q_HEAD                              0x168B8C
+
+#define mmMME2_QM_ARB_ERR_CAUSE                                      0x168B9C
+
+#define mmMME2_QM_ARB_ERR_MSG_EN                                     0x168BA0
+
+#define mmMME2_QM_ARB_ERR_STS_DRP                                    0x168BA8
+
+#define mmMME2_QM_ARB_MST_CRED_STS_0                                 0x168BB0
+
+#define mmMME2_QM_ARB_MST_CRED_STS_1                                 0x168BB4
+
+#define mmMME2_QM_ARB_MST_CRED_STS_2                                 0x168BB8
+
+#define mmMME2_QM_ARB_MST_CRED_STS_3                                 0x168BBC
+
+#define mmMME2_QM_ARB_MST_CRED_STS_4                                 0x168BC0
+
+#define mmMME2_QM_ARB_MST_CRED_STS_5                                 0x168BC4
+
+#define mmMME2_QM_ARB_MST_CRED_STS_6                                 0x168BC8
+
+#define mmMME2_QM_ARB_MST_CRED_STS_7                                 0x168BCC
+
+#define mmMME2_QM_ARB_MST_CRED_STS_8                                 0x168BD0
+
+#define mmMME2_QM_ARB_MST_CRED_STS_9                                 0x168BD4
+
+#define mmMME2_QM_ARB_MST_CRED_STS_10                                0x168BD8
+
+#define mmMME2_QM_ARB_MST_CRED_STS_11                                0x168BDC
+
+#define mmMME2_QM_ARB_MST_CRED_STS_12                                0x168BE0
+
+#define mmMME2_QM_ARB_MST_CRED_STS_13                                0x168BE4
+
+#define mmMME2_QM_ARB_MST_CRED_STS_14                                0x168BE8
+
+#define mmMME2_QM_ARB_MST_CRED_STS_15                                0x168BEC
+
+#define mmMME2_QM_ARB_MST_CRED_STS_16                                0x168BF0
+
+#define mmMME2_QM_ARB_MST_CRED_STS_17                                0x168BF4
+
+#define mmMME2_QM_ARB_MST_CRED_STS_18                                0x168BF8
+
+#define mmMME2_QM_ARB_MST_CRED_STS_19                                0x168BFC
+
+#define mmMME2_QM_ARB_MST_CRED_STS_20                                0x168C00
+
+#define mmMME2_QM_ARB_MST_CRED_STS_21                                0x168C04
+
+#define mmMME2_QM_ARB_MST_CRED_STS_22                                0x168C08
+
+#define mmMME2_QM_ARB_MST_CRED_STS_23                                0x168C0C
+
+#define mmMME2_QM_ARB_MST_CRED_STS_24                                0x168C10
+
+#define mmMME2_QM_ARB_MST_CRED_STS_25                                0x168C14
+
+#define mmMME2_QM_ARB_MST_CRED_STS_26                                0x168C18
+
+#define mmMME2_QM_ARB_MST_CRED_STS_27                                0x168C1C
+
+#define mmMME2_QM_ARB_MST_CRED_STS_28                                0x168C20
+
+#define mmMME2_QM_ARB_MST_CRED_STS_29                                0x168C24
+
+#define mmMME2_QM_ARB_MST_CRED_STS_30                                0x168C28
+
+#define mmMME2_QM_ARB_MST_CRED_STS_31                                0x168C2C
+
+#define mmMME2_QM_CGM_CFG                                            0x168C70
+
+#define mmMME2_QM_CGM_STS                                            0x168C74
+
+#define mmMME2_QM_CGM_CFG1                                           0x168C78
+
+#define mmMME2_QM_LOCAL_RANGE_BASE                                   0x168C80
+
+#define mmMME2_QM_LOCAL_RANGE_SIZE                                   0x168C84
+
+#define mmMME2_QM_CSMR_STRICT_PRIO_CFG                               0x168C90
+
+#define mmMME2_QM_HBW_RD_RATE_LIM_CFG_1                              0x168C94
+
+#define mmMME2_QM_LBW_WR_RATE_LIM_CFG_0                              0x168C98
+
+#define mmMME2_QM_LBW_WR_RATE_LIM_CFG_1                              0x168C9C
+
+#define mmMME2_QM_HBW_RD_RATE_LIM_CFG_0                              0x168CA0
+
+#define mmMME2_QM_GLBL_AXCACHE                                       0x168CA4
+
+#define mmMME2_QM_IND_GW_APB_CFG                                     0x168CB0
+
+#define mmMME2_QM_IND_GW_APB_WDATA                                   0x168CB4
+
+#define mmMME2_QM_IND_GW_APB_RDATA                                   0x168CB8
+
+#define mmMME2_QM_IND_GW_APB_STATUS                                  0x168CBC
+
+#define mmMME2_QM_GLBL_ERR_ADDR_LO                                   0x168CD0
+
+#define mmMME2_QM_GLBL_ERR_ADDR_HI                                   0x168CD4
+
+#define mmMME2_QM_GLBL_ERR_WDATA                                     0x168CD8
+
+#define mmMME2_QM_GLBL_MEM_INIT_BUSY                                 0x168D00
+
+#endif /* ASIC_REG_MME2_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mme3_ctrl_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme3_ctrl_regs.h
new file mode 100644
index 000000000000..36f6edc72e3d
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mme3_ctrl_regs.h
@@ -0,0 +1,1456 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MME3_CTRL_REGS_H_
+#define ASIC_REG_MME3_CTRL_REGS_H_
+
+/*
+ *****************************************
+ *   MME3_CTRL (Prototype: MME)
+ *****************************************
+ */
+
+#define mmMME3_CTRL_ARCH_STATUS                                      0x1E0000
+
+#define mmMME3_CTRL_ARCH_BASE_ADDR_HIGH_S                            0x1E0008
+
+#define mmMME3_CTRL_ARCH_BASE_ADDR_HIGH_L                            0x1E000C
+
+#define mmMME3_CTRL_ARCH_BASE_ADDR_HIGH_O                            0x1E0010
+
+#define mmMME3_CTRL_ARCH_BASE_ADDR_LOW_S                             0x1E0014
+
+#define mmMME3_CTRL_ARCH_BASE_ADDR_LOW_L                             0x1E0018
+
+#define mmMME3_CTRL_ARCH_BASE_ADDR_LOW_O                             0x1E001C
+
+#define mmMME3_CTRL_ARCH_HEADER_LOW                                  0x1E0020
+
+#define mmMME3_CTRL_ARCH_HEADER_HIGH                                 0x1E0024
+
+#define mmMME3_CTRL_ARCH_CONV_KERNEL_SIZE_MINUS_1                    0x1E0028
+
+#define mmMME3_CTRL_ARCH_CONV_ASSOCIATED_DIMS_LOW                    0x1E002C
+
+#define mmMME3_CTRL_ARCH_CONV_ASSOCIATED_DIMS_HIGH                   0x1E0030
+
+#define mmMME3_CTRL_ARCH_NUM_ITERATIONS_MINUS_1                      0x1E0034
+
+#define mmMME3_CTRL_ARCH_OUTER_LOOP                                  0x1E0038
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_0                   0x1E003C
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_1                   0x1E0040
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_2                   0x1E0044
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_3                   0x1E0048
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_VALID_ELEMENTS_4                   0x1E004C
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_0                      0x1E0050
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_1                      0x1E0054
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_2                      0x1E0058
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_3                      0x1E005C
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_LOOP_STRIDE_4                      0x1E0060
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_ROI_SIZE_0                         0x1E0064
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_ROI_SIZE_1                         0x1E0068
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_ROI_SIZE_2                         0x1E006C
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_ROI_SIZE_3                         0x1E0070
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_0                  0x1E0074
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_1                  0x1E0078
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_2                  0x1E007C
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_SPATIAL_STRIDES_3                  0x1E0080
+
+#define mmMME3_CTRL_ARCH_TENSOR_S_SPATIAL_SIZE_MINUS_1               0x1E0084
+
+#define mmMME3_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_0                     0x1E0088
+
+#define mmMME3_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_1                     0x1E008C
+
+#define mmMME3_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_2                     0x1E0090
+
+#define mmMME3_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_3                     0x1E0094
+
+#define mmMME3_CTRL_ARCH_AGU_S_ROI_BASE_OFFSET_4                     0x1E0098
+
+#define mmMME3_CTRL_ARCH_AGU_S_START_OFFSET_0                        0x1E009C
+
+#define mmMME3_CTRL_ARCH_AGU_S_START_OFFSET_1                        0x1E00A0
+
+#define mmMME3_CTRL_ARCH_AGU_S_START_OFFSET_2                        0x1E00A4
+
+#define mmMME3_CTRL_ARCH_AGU_S_START_OFFSET_3                        0x1E00A8
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_0                   0x1E00AC
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_1                   0x1E00B0
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_2                   0x1E00B4
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_3                   0x1E00B8
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_VALID_ELEMENTS_4                   0x1E00BC
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_0                      0x1E00C0
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_1                      0x1E00C4
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_2                      0x1E00C8
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_3                      0x1E00CC
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_LOOP_STRIDE_4                      0x1E00D0
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_ROI_SIZE_0                         0x1E00D4
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_ROI_SIZE_1                         0x1E00D8
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_ROI_SIZE_2                         0x1E00DC
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_ROI_SIZE_3                         0x1E00E0
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_0                  0x1E00E4
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_1                  0x1E00E8
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_2                  0x1E00EC
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_SPATIAL_STRIDES_3                  0x1E00F0
+
+#define mmMME3_CTRL_ARCH_TENSOR_L_SPATIAL_SIZE_MINUS_1               0x1E00F4
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_0               0x1E00F8
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_1               0x1E00FC
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_2               0x1E0100
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_3               0x1E0104
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_ROI_BASE_OFFSET_4               0x1E0108
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_0                  0x1E010C
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_1                  0x1E0110
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_2                  0x1E0114
+
+#define mmMME3_CTRL_ARCH_AGU_L_LOCAL_START_OFFSET_3                  0x1E0118
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_0              0x1E011C
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_1              0x1E0120
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_2              0x1E0124
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_3              0x1E0128
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_ROI_BASE_OFFSET_4              0x1E012C
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_0                 0x1E0130
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_1                 0x1E0134
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_2                 0x1E0138
+
+#define mmMME3_CTRL_ARCH_AGU_L_REMOTE_START_OFFSET_3                 0x1E013C
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_0                   0x1E0140
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_1                   0x1E0144
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_2                   0x1E0148
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_3                   0x1E014C
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_VALID_ELEMENTS_4                   0x1E0150
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_0                      0x1E0154
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_1                      0x1E0158
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_2                      0x1E015C
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_3                      0x1E0160
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_LOOP_STRIDE_4                      0x1E0164
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_ROI_SIZE_0                         0x1E0168
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_ROI_SIZE_1                         0x1E016C
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_ROI_SIZE_2                         0x1E0170
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_ROI_SIZE_3                         0x1E0174
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_0                  0x1E0178
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_1                  0x1E017C
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_2                  0x1E0180
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_SPATIAL_STRIDES_3                  0x1E0184
+
+#define mmMME3_CTRL_ARCH_TENSOR_O_SPATIAL_SIZE_MINUS_1               0x1E0188
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_0               0x1E018C
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_1               0x1E0190
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_2               0x1E0194
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_3               0x1E0198
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_ROI_BASE_OFFSET_4               0x1E019C
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_0                  0x1E01A0
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_1                  0x1E01A4
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_2                  0x1E01A8
+
+#define mmMME3_CTRL_ARCH_AGU_O_LOCAL_START_OFFSET_3                  0x1E01AC
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_0              0x1E01B0
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_1              0x1E01B4
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_2              0x1E01B8
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_3              0x1E01BC
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_ROI_BASE_OFFSET_4              0x1E01C0
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_0                 0x1E01C4
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_1                 0x1E01C8
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_2                 0x1E01CC
+
+#define mmMME3_CTRL_ARCH_AGU_O_REMOTE_START_OFFSET_3                 0x1E01D0
+
+#define mmMME3_CTRL_ARCH_DESC_SB_REPEAT                              0x1E01D4
+
+#define mmMME3_CTRL_ARCH_DESC_RATE_LIMITER                           0x1E01D8
+
+#define mmMME3_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL             0x1E01DC
+
+#define mmMME3_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE            0x1E01E0
+
+#define mmMME3_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_HIGH                  0x1E01E4
+
+#define mmMME3_CTRL_ARCH_DESC_SYNC_OBJECT_DATA                       0x1E01E8
+
+#define mmMME3_CTRL_ARCH_DESC_AXI_USER_DATA                          0x1E01EC
+
+#define mmMME3_CTRL_ARCH_DESC_PERF_EVT_S                             0x1E01F0
+
+#define mmMME3_CTRL_ARCH_DESC_PERF_EVT_L_LOCAL                       0x1E01F4
+
+#define mmMME3_CTRL_ARCH_DESC_PERF_EVT_L_REMOTE                      0x1E01F8
+
+#define mmMME3_CTRL_ARCH_DESC_PERF_EVT_O_LOCAL                       0x1E01FC
+
+#define mmMME3_CTRL_ARCH_DESC_PERF_EVT_O_REMOTE                      0x1E0200
+
+#define mmMME3_CTRL_ARCH_DESC_PADDING_VALUE_S                        0x1E0204
+
+#define mmMME3_CTRL_ARCH_DESC_PADDING_VALUE_L                        0x1E0208
+
+#define mmMME3_CTRL_ARCH_DESC_META_DATA_AGU_S                        0x1E020C
+
+#define mmMME3_CTRL_ARCH_DESC_META_DATA_AGU_L_LOCAL                  0x1E0210
+
+#define mmMME3_CTRL_ARCH_DESC_META_DATA_AGU_L_REMOTE                 0x1E0214
+
+#define mmMME3_CTRL_ARCH_DESC_META_DATA_AGU_O_LOCAL                  0x1E0218
+
+#define mmMME3_CTRL_ARCH_DESC_META_DATA_AGU_O_REMOTE                 0x1E021C
+
+#define mmMME3_CTRL_ARCH_DESC_PCU_RL_SATURATION                      0x1E0220
+
+#define mmMME3_CTRL_ARCH_DESC_DUMMY                                  0x1E0224
+
+#define mmMME3_CTRL_CMD                                              0x1E0280
+
+#define mmMME3_CTRL_STATUS1                                          0x1E0284
+
+#define mmMME3_CTRL_RESET                                            0x1E0288
+
+#define mmMME3_CTRL_QM_STALL                                         0x1E028C
+
+#define mmMME3_CTRL_SYNC_OBJECT_FIFO_TH                              0x1E0290
+
+#define mmMME3_CTRL_EUS_ROLLUP_CNT_ADD                               0x1E0294
+
+#define mmMME3_CTRL_INTR_CAUSE                                       0x1E0298
+
+#define mmMME3_CTRL_INTR_MASK                                        0x1E029C
+
+#define mmMME3_CTRL_LOG_SHADOW                                       0x1E02A0
+
+#define mmMME3_CTRL_PCU_RL_DESC0                                     0x1E02A4
+
+#define mmMME3_CTRL_PCU_RL_TOKEN_UPDATE                              0x1E02A8
+
+#define mmMME3_CTRL_PCU_RL_TH                                        0x1E02AC
+
+#define mmMME3_CTRL_PCU_RL_MIN                                       0x1E02B0
+
+#define mmMME3_CTRL_PCU_RL_CTRL_EN                                   0x1E02B4
+
+#define mmMME3_CTRL_PCU_RL_HISTORY_LOG_SIZE                          0x1E02B8
+
+#define mmMME3_CTRL_PCU_DUMMY_A_BF16                                 0x1E02BC
+
+#define mmMME3_CTRL_PCU_DUMMY_B_BF16                                 0x1E02C0
+
+#define mmMME3_CTRL_PCU_DUMMY_A_FP32_ODD                             0x1E02C4
+
+#define mmMME3_CTRL_PCU_DUMMY_A_FP32_EVEN                            0x1E02C8
+
+#define mmMME3_CTRL_PCU_DUMMY_B_FP32_ODD                             0x1E02CC
+
+#define mmMME3_CTRL_PCU_DUMMY_B_FP32_EVEN                            0x1E02D0
+
+#define mmMME3_CTRL_PROT                                             0x1E02D4
+
+#define mmMME3_CTRL_EU_POWER_SAVE_DISABLE                            0x1E02D8
+
+#define mmMME3_CTRL_CS_DBG_BLOCK_ID                                  0x1E02DC
+
+#define mmMME3_CTRL_CS_DBG_STATUS_DROP_CNT                           0x1E02E0
+
+#define mmMME3_CTRL_TE_CLOSE_CGATE                                   0x1E02E4
+
+#define mmMME3_CTRL_AGU_SM_INFLIGHT_CNTR                             0x1E02E8
+
+#define mmMME3_CTRL_AGU_SM_TOTAL_CNTR                                0x1E02EC
+
+#define mmMME3_CTRL_EZSYNC_OUT_CREDIT                                0x1E02F0
+
+#define mmMME3_CTRL_PCU_RL_SAT_SEC                                   0x1E02F4
+
+#define mmMME3_CTRL_AGU_SYNC_MSG_AXI_USER                            0x1E02F8
+
+#define mmMME3_CTRL_QM_SLV_LBW_CLK_EN                                0x1E02FC
+
+#define mmMME3_CTRL_SHADOW_0_STATUS                                  0x1E0400
+
+#define mmMME3_CTRL_SHADOW_0_BASE_ADDR_HIGH_S                        0x1E0408
+
+#define mmMME3_CTRL_SHADOW_0_BASE_ADDR_HIGH_L                        0x1E040C
+
+#define mmMME3_CTRL_SHADOW_0_BASE_ADDR_HIGH_O                        0x1E0410
+
+#define mmMME3_CTRL_SHADOW_0_BASE_ADDR_LOW_S                         0x1E0414
+
+#define mmMME3_CTRL_SHADOW_0_BASE_ADDR_LOW_L                         0x1E0418
+
+#define mmMME3_CTRL_SHADOW_0_BASE_ADDR_LOW_O                         0x1E041C
+
+#define mmMME3_CTRL_SHADOW_0_HEADER_LOW                              0x1E0420
+
+#define mmMME3_CTRL_SHADOW_0_HEADER_HIGH                             0x1E0424
+
+#define mmMME3_CTRL_SHADOW_0_CONV_KERNEL_SIZE_MINUS_1                0x1E0428
+
+#define mmMME3_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_LOW                0x1E042C
+
+#define mmMME3_CTRL_SHADOW_0_CONV_ASSOCIATED_DIMS_HIGH               0x1E0430
+
+#define mmMME3_CTRL_SHADOW_0_NUM_ITERATIONS_MINUS_1                  0x1E0434
+
+#define mmMME3_CTRL_SHADOW_0_OUTER_LOOP                              0x1E0438
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_0               0x1E043C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_1               0x1E0440
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_2               0x1E0444
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_3               0x1E0448
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_VALID_ELEMENTS_4               0x1E044C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_0                  0x1E0450
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_1                  0x1E0454
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_2                  0x1E0458
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_3                  0x1E045C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_LOOP_STRIDE_4                  0x1E0460
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_0                     0x1E0464
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_1                     0x1E0468
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_2                     0x1E046C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_ROI_SIZE_3                     0x1E0470
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_0              0x1E0474
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_1              0x1E0478
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_2              0x1E047C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_SPATIAL_STRIDES_3              0x1E0480
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x1E0484
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_0                 0x1E0488
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_1                 0x1E048C
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_2                 0x1E0490
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_3                 0x1E0494
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_ROI_BASE_OFFSET_4                 0x1E0498
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_START_OFFSET_0                    0x1E049C
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_START_OFFSET_1                    0x1E04A0
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_START_OFFSET_2                    0x1E04A4
+
+#define mmMME3_CTRL_SHADOW_0_AGU_S_START_OFFSET_3                    0x1E04A8
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_0               0x1E04AC
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_1               0x1E04B0
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_2               0x1E04B4
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_3               0x1E04B8
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_VALID_ELEMENTS_4               0x1E04BC
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_0                  0x1E04C0
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_1                  0x1E04C4
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_2                  0x1E04C8
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_3                  0x1E04CC
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_LOOP_STRIDE_4                  0x1E04D0
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_0                     0x1E04D4
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_1                     0x1E04D8
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_2                     0x1E04DC
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_ROI_SIZE_3                     0x1E04E0
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_0              0x1E04E4
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_1              0x1E04E8
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_2              0x1E04EC
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_SPATIAL_STRIDES_3              0x1E04F0
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x1E04F4
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x1E04F8
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x1E04FC
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x1E0500
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x1E0504
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x1E0508
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_0              0x1E050C
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_1              0x1E0510
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_2              0x1E0514
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_LOCAL_START_OFFSET_3              0x1E0518
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x1E051C
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x1E0520
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x1E0524
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x1E0528
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x1E052C
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_0             0x1E0530
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_1             0x1E0534
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_2             0x1E0538
+
+#define mmMME3_CTRL_SHADOW_0_AGU_L_REMOTE_START_OFFSET_3             0x1E053C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_0               0x1E0540
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_1               0x1E0544
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_2               0x1E0548
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_3               0x1E054C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_VALID_ELEMENTS_4               0x1E0550
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_0                  0x1E0554
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_1                  0x1E0558
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_2                  0x1E055C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_3                  0x1E0560
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_LOOP_STRIDE_4                  0x1E0564
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_0                     0x1E0568
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_1                     0x1E056C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_2                     0x1E0570
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_ROI_SIZE_3                     0x1E0574
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_0              0x1E0578
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_1              0x1E057C
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_2              0x1E0580
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_SPATIAL_STRIDES_3              0x1E0584
+
+#define mmMME3_CTRL_SHADOW_0_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x1E0588
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x1E058C
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x1E0590
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x1E0594
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x1E0598
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x1E059C
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_0              0x1E05A0
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_1              0x1E05A4
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_2              0x1E05A8
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_LOCAL_START_OFFSET_3              0x1E05AC
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x1E05B0
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x1E05B4
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x1E05B8
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x1E05BC
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x1E05C0
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_0             0x1E05C4
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_1             0x1E05C8
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_2             0x1E05CC
+
+#define mmMME3_CTRL_SHADOW_0_AGU_O_REMOTE_START_OFFSET_3             0x1E05D0
+
+#define mmMME3_CTRL_SHADOW_0_DESC_SB_REPEAT                          0x1E05D4
+
+#define mmMME3_CTRL_SHADOW_0_DESC_RATE_LIMITER                       0x1E05D8
+
+#define mmMME3_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x1E05DC
+
+#define mmMME3_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x1E05E0
+
+#define mmMME3_CTRL_SHADOW_0_DESC_SYNC_OBJECT_ADDR_HIGH              0x1E05E4
+
+#define mmMME3_CTRL_SHADOW_0_DESC_SYNC_OBJECT_DATA                   0x1E05E8
+
+#define mmMME3_CTRL_SHADOW_0_DESC_AXI_USER_DATA                      0x1E05EC
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PERF_EVT_S                         0x1E05F0
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PERF_EVT_L_LOCAL                   0x1E05F4
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PERF_EVT_L_REMOTE                  0x1E05F8
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PERF_EVT_O_LOCAL                   0x1E05FC
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PERF_EVT_O_REMOTE                  0x1E0600
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PADDING_VALUE_S                    0x1E0604
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PADDING_VALUE_L                    0x1E0608
+
+#define mmMME3_CTRL_SHADOW_0_DESC_META_DATA_AGU_S                    0x1E060C
+
+#define mmMME3_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_LOCAL              0x1E0610
+
+#define mmMME3_CTRL_SHADOW_0_DESC_META_DATA_AGU_L_REMOTE             0x1E0614
+
+#define mmMME3_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_LOCAL              0x1E0618
+
+#define mmMME3_CTRL_SHADOW_0_DESC_META_DATA_AGU_O_REMOTE             0x1E061C
+
+#define mmMME3_CTRL_SHADOW_0_DESC_PCU_RL_SATURATION                  0x1E0620
+
+#define mmMME3_CTRL_SHADOW_0_DESC_DUMMY                              0x1E0624
+
+#define mmMME3_CTRL_SHADOW_1_STATUS                                  0x1E0680
+
+#define mmMME3_CTRL_SHADOW_1_BASE_ADDR_HIGH_S                        0x1E0688
+
+#define mmMME3_CTRL_SHADOW_1_BASE_ADDR_HIGH_L                        0x1E068C
+
+#define mmMME3_CTRL_SHADOW_1_BASE_ADDR_HIGH_O                        0x1E0690
+
+#define mmMME3_CTRL_SHADOW_1_BASE_ADDR_LOW_S                         0x1E0694
+
+#define mmMME3_CTRL_SHADOW_1_BASE_ADDR_LOW_L                         0x1E0698
+
+#define mmMME3_CTRL_SHADOW_1_BASE_ADDR_LOW_O                         0x1E069C
+
+#define mmMME3_CTRL_SHADOW_1_HEADER_LOW                              0x1E06A0
+
+#define mmMME3_CTRL_SHADOW_1_HEADER_HIGH                             0x1E06A4
+
+#define mmMME3_CTRL_SHADOW_1_CONV_KERNEL_SIZE_MINUS_1                0x1E06A8
+
+#define mmMME3_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_LOW                0x1E06AC
+
+#define mmMME3_CTRL_SHADOW_1_CONV_ASSOCIATED_DIMS_HIGH               0x1E06B0
+
+#define mmMME3_CTRL_SHADOW_1_NUM_ITERATIONS_MINUS_1                  0x1E06B4
+
+#define mmMME3_CTRL_SHADOW_1_OUTER_LOOP                              0x1E06B8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_0               0x1E06BC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_1               0x1E06C0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_2               0x1E06C4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_3               0x1E06C8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_VALID_ELEMENTS_4               0x1E06CC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_0                  0x1E06D0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_1                  0x1E06D4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_2                  0x1E06D8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_3                  0x1E06DC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_LOOP_STRIDE_4                  0x1E06E0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_0                     0x1E06E4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_1                     0x1E06E8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_2                     0x1E06EC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_ROI_SIZE_3                     0x1E06F0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_0              0x1E06F4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_1              0x1E06F8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_2              0x1E06FC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_SPATIAL_STRIDES_3              0x1E0700
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x1E0704
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_0                 0x1E0708
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_1                 0x1E070C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_2                 0x1E0710
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_3                 0x1E0714
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_ROI_BASE_OFFSET_4                 0x1E0718
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_START_OFFSET_0                    0x1E071C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_START_OFFSET_1                    0x1E0720
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_START_OFFSET_2                    0x1E0724
+
+#define mmMME3_CTRL_SHADOW_1_AGU_S_START_OFFSET_3                    0x1E0728
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_0               0x1E072C
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_1               0x1E0730
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_2               0x1E0734
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_3               0x1E0738
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_VALID_ELEMENTS_4               0x1E073C
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_0                  0x1E0740
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_1                  0x1E0744
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_2                  0x1E0748
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_3                  0x1E074C
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_LOOP_STRIDE_4                  0x1E0750
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_0                     0x1E0754
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_1                     0x1E0758
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_2                     0x1E075C
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_ROI_SIZE_3                     0x1E0760
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_0              0x1E0764
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_1              0x1E0768
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_2              0x1E076C
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_SPATIAL_STRIDES_3              0x1E0770
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x1E0774
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x1E0778
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x1E077C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x1E0780
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x1E0784
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x1E0788
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_0              0x1E078C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_1              0x1E0790
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_2              0x1E0794
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_LOCAL_START_OFFSET_3              0x1E0798
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x1E079C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x1E07A0
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x1E07A4
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x1E07A8
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x1E07AC
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_0             0x1E07B0
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_1             0x1E07B4
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_2             0x1E07B8
+
+#define mmMME3_CTRL_SHADOW_1_AGU_L_REMOTE_START_OFFSET_3             0x1E07BC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_0               0x1E07C0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_1               0x1E07C4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_2               0x1E07C8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_3               0x1E07CC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_VALID_ELEMENTS_4               0x1E07D0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_0                  0x1E07D4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_1                  0x1E07D8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_2                  0x1E07DC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_3                  0x1E07E0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_LOOP_STRIDE_4                  0x1E07E4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_0                     0x1E07E8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_1                     0x1E07EC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_2                     0x1E07F0
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_ROI_SIZE_3                     0x1E07F4
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_0              0x1E07F8
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_1              0x1E07FC
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_2              0x1E0800
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_SPATIAL_STRIDES_3              0x1E0804
+
+#define mmMME3_CTRL_SHADOW_1_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x1E0808
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x1E080C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x1E0810
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x1E0814
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x1E0818
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x1E081C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_0              0x1E0820
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_1              0x1E0824
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_2              0x1E0828
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_LOCAL_START_OFFSET_3              0x1E082C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x1E0830
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x1E0834
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x1E0838
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x1E083C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x1E0840
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_0             0x1E0844
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_1             0x1E0848
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_2             0x1E084C
+
+#define mmMME3_CTRL_SHADOW_1_AGU_O_REMOTE_START_OFFSET_3             0x1E0850
+
+#define mmMME3_CTRL_SHADOW_1_DESC_SB_REPEAT                          0x1E0854
+
+#define mmMME3_CTRL_SHADOW_1_DESC_RATE_LIMITER                       0x1E0858
+
+#define mmMME3_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x1E085C
+
+#define mmMME3_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x1E0860
+
+#define mmMME3_CTRL_SHADOW_1_DESC_SYNC_OBJECT_ADDR_HIGH              0x1E0864
+
+#define mmMME3_CTRL_SHADOW_1_DESC_SYNC_OBJECT_DATA                   0x1E0868
+
+#define mmMME3_CTRL_SHADOW_1_DESC_AXI_USER_DATA                      0x1E086C
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PERF_EVT_S                         0x1E0870
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PERF_EVT_L_LOCAL                   0x1E0874
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PERF_EVT_L_REMOTE                  0x1E0878
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PERF_EVT_O_LOCAL                   0x1E087C
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PERF_EVT_O_REMOTE                  0x1E0880
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PADDING_VALUE_S                    0x1E0884
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PADDING_VALUE_L                    0x1E0888
+
+#define mmMME3_CTRL_SHADOW_1_DESC_META_DATA_AGU_S                    0x1E088C
+
+#define mmMME3_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_LOCAL              0x1E0890
+
+#define mmMME3_CTRL_SHADOW_1_DESC_META_DATA_AGU_L_REMOTE             0x1E0894
+
+#define mmMME3_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_LOCAL              0x1E0898
+
+#define mmMME3_CTRL_SHADOW_1_DESC_META_DATA_AGU_O_REMOTE             0x1E089C
+
+#define mmMME3_CTRL_SHADOW_1_DESC_PCU_RL_SATURATION                  0x1E08A0
+
+#define mmMME3_CTRL_SHADOW_1_DESC_DUMMY                              0x1E08A4
+
+#define mmMME3_CTRL_SHADOW_2_STATUS                                  0x1E0900
+
+#define mmMME3_CTRL_SHADOW_2_BASE_ADDR_HIGH_S                        0x1E0908
+
+#define mmMME3_CTRL_SHADOW_2_BASE_ADDR_HIGH_L                        0x1E090C
+
+#define mmMME3_CTRL_SHADOW_2_BASE_ADDR_HIGH_O                        0x1E0910
+
+#define mmMME3_CTRL_SHADOW_2_BASE_ADDR_LOW_S                         0x1E0914
+
+#define mmMME3_CTRL_SHADOW_2_BASE_ADDR_LOW_L                         0x1E0918
+
+#define mmMME3_CTRL_SHADOW_2_BASE_ADDR_LOW_O                         0x1E091C
+
+#define mmMME3_CTRL_SHADOW_2_HEADER_LOW                              0x1E0920
+
+#define mmMME3_CTRL_SHADOW_2_HEADER_HIGH                             0x1E0924
+
+#define mmMME3_CTRL_SHADOW_2_CONV_KERNEL_SIZE_MINUS_1                0x1E0928
+
+#define mmMME3_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_LOW                0x1E092C
+
+#define mmMME3_CTRL_SHADOW_2_CONV_ASSOCIATED_DIMS_HIGH               0x1E0930
+
+#define mmMME3_CTRL_SHADOW_2_NUM_ITERATIONS_MINUS_1                  0x1E0934
+
+#define mmMME3_CTRL_SHADOW_2_OUTER_LOOP                              0x1E0938
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_0               0x1E093C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_1               0x1E0940
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_2               0x1E0944
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_3               0x1E0948
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_VALID_ELEMENTS_4               0x1E094C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_0                  0x1E0950
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_1                  0x1E0954
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_2                  0x1E0958
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_3                  0x1E095C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_LOOP_STRIDE_4                  0x1E0960
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_0                     0x1E0964
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_1                     0x1E0968
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_2                     0x1E096C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_ROI_SIZE_3                     0x1E0970
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_0              0x1E0974
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_1              0x1E0978
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_2              0x1E097C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_SPATIAL_STRIDES_3              0x1E0980
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x1E0984
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_0                 0x1E0988
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_1                 0x1E098C
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_2                 0x1E0990
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_3                 0x1E0994
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_ROI_BASE_OFFSET_4                 0x1E0998
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_START_OFFSET_0                    0x1E099C
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_START_OFFSET_1                    0x1E09A0
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_START_OFFSET_2                    0x1E09A4
+
+#define mmMME3_CTRL_SHADOW_2_AGU_S_START_OFFSET_3                    0x1E09A8
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_0               0x1E09AC
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_1               0x1E09B0
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_2               0x1E09B4
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_3               0x1E09B8
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_VALID_ELEMENTS_4               0x1E09BC
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_0                  0x1E09C0
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_1                  0x1E09C4
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_2                  0x1E09C8
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_3                  0x1E09CC
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_LOOP_STRIDE_4                  0x1E09D0
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_0                     0x1E09D4
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_1                     0x1E09D8
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_2                     0x1E09DC
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_ROI_SIZE_3                     0x1E09E0
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_0              0x1E09E4
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_1              0x1E09E8
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_2              0x1E09EC
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_SPATIAL_STRIDES_3              0x1E09F0
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x1E09F4
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x1E09F8
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x1E09FC
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x1E0A00
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x1E0A04
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x1E0A08
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_0              0x1E0A0C
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_1              0x1E0A10
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_2              0x1E0A14
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_LOCAL_START_OFFSET_3              0x1E0A18
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x1E0A1C
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x1E0A20
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x1E0A24
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x1E0A28
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x1E0A2C
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_0             0x1E0A30
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_1             0x1E0A34
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_2             0x1E0A38
+
+#define mmMME3_CTRL_SHADOW_2_AGU_L_REMOTE_START_OFFSET_3             0x1E0A3C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_0               0x1E0A40
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_1               0x1E0A44
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_2               0x1E0A48
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_3               0x1E0A4C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_VALID_ELEMENTS_4               0x1E0A50
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_0                  0x1E0A54
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_1                  0x1E0A58
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_2                  0x1E0A5C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_3                  0x1E0A60
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_LOOP_STRIDE_4                  0x1E0A64
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_0                     0x1E0A68
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_1                     0x1E0A6C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_2                     0x1E0A70
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_ROI_SIZE_3                     0x1E0A74
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_0              0x1E0A78
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_1              0x1E0A7C
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_2              0x1E0A80
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_SPATIAL_STRIDES_3              0x1E0A84
+
+#define mmMME3_CTRL_SHADOW_2_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x1E0A88
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x1E0A8C
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x1E0A90
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x1E0A94
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x1E0A98
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x1E0A9C
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_0              0x1E0AA0
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_1              0x1E0AA4
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_2              0x1E0AA8
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_LOCAL_START_OFFSET_3              0x1E0AAC
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x1E0AB0
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x1E0AB4
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x1E0AB8
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x1E0ABC
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x1E0AC0
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_0             0x1E0AC4
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_1             0x1E0AC8
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_2             0x1E0ACC
+
+#define mmMME3_CTRL_SHADOW_2_AGU_O_REMOTE_START_OFFSET_3             0x1E0AD0
+
+#define mmMME3_CTRL_SHADOW_2_DESC_SB_REPEAT                          0x1E0AD4
+
+#define mmMME3_CTRL_SHADOW_2_DESC_RATE_LIMITER                       0x1E0AD8
+
+#define mmMME3_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x1E0ADC
+
+#define mmMME3_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x1E0AE0
+
+#define mmMME3_CTRL_SHADOW_2_DESC_SYNC_OBJECT_ADDR_HIGH              0x1E0AE4
+
+#define mmMME3_CTRL_SHADOW_2_DESC_SYNC_OBJECT_DATA                   0x1E0AE8
+
+#define mmMME3_CTRL_SHADOW_2_DESC_AXI_USER_DATA                      0x1E0AEC
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PERF_EVT_S                         0x1E0AF0
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PERF_EVT_L_LOCAL                   0x1E0AF4
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PERF_EVT_L_REMOTE                  0x1E0AF8
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PERF_EVT_O_LOCAL                   0x1E0AFC
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PERF_EVT_O_REMOTE                  0x1E0B00
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PADDING_VALUE_S                    0x1E0B04
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PADDING_VALUE_L                    0x1E0B08
+
+#define mmMME3_CTRL_SHADOW_2_DESC_META_DATA_AGU_S                    0x1E0B0C
+
+#define mmMME3_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_LOCAL              0x1E0B10
+
+#define mmMME3_CTRL_SHADOW_2_DESC_META_DATA_AGU_L_REMOTE             0x1E0B14
+
+#define mmMME3_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_LOCAL              0x1E0B18
+
+#define mmMME3_CTRL_SHADOW_2_DESC_META_DATA_AGU_O_REMOTE             0x1E0B1C
+
+#define mmMME3_CTRL_SHADOW_2_DESC_PCU_RL_SATURATION                  0x1E0B20
+
+#define mmMME3_CTRL_SHADOW_2_DESC_DUMMY                              0x1E0B24
+
+#define mmMME3_CTRL_SHADOW_3_STATUS                                  0x1E0B80
+
+#define mmMME3_CTRL_SHADOW_3_BASE_ADDR_HIGH_S                        0x1E0B88
+
+#define mmMME3_CTRL_SHADOW_3_BASE_ADDR_HIGH_L                        0x1E0B8C
+
+#define mmMME3_CTRL_SHADOW_3_BASE_ADDR_HIGH_O                        0x1E0B90
+
+#define mmMME3_CTRL_SHADOW_3_BASE_ADDR_LOW_S                         0x1E0B94
+
+#define mmMME3_CTRL_SHADOW_3_BASE_ADDR_LOW_L                         0x1E0B98
+
+#define mmMME3_CTRL_SHADOW_3_BASE_ADDR_LOW_O                         0x1E0B9C
+
+#define mmMME3_CTRL_SHADOW_3_HEADER_LOW                              0x1E0BA0
+
+#define mmMME3_CTRL_SHADOW_3_HEADER_HIGH                             0x1E0BA4
+
+#define mmMME3_CTRL_SHADOW_3_CONV_KERNEL_SIZE_MINUS_1                0x1E0BA8
+
+#define mmMME3_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_LOW                0x1E0BAC
+
+#define mmMME3_CTRL_SHADOW_3_CONV_ASSOCIATED_DIMS_HIGH               0x1E0BB0
+
+#define mmMME3_CTRL_SHADOW_3_NUM_ITERATIONS_MINUS_1                  0x1E0BB4
+
+#define mmMME3_CTRL_SHADOW_3_OUTER_LOOP                              0x1E0BB8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_0               0x1E0BBC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_1               0x1E0BC0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_2               0x1E0BC4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_3               0x1E0BC8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_VALID_ELEMENTS_4               0x1E0BCC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_0                  0x1E0BD0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_1                  0x1E0BD4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_2                  0x1E0BD8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_3                  0x1E0BDC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_LOOP_STRIDE_4                  0x1E0BE0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_0                     0x1E0BE4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_1                     0x1E0BE8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_2                     0x1E0BEC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_ROI_SIZE_3                     0x1E0BF0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_0              0x1E0BF4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_1              0x1E0BF8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_2              0x1E0BFC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_SPATIAL_STRIDES_3              0x1E0C00
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_S_SPATIAL_SIZE_MINUS_1           0x1E0C04
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_0                 0x1E0C08
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_1                 0x1E0C0C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_2                 0x1E0C10
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_3                 0x1E0C14
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_ROI_BASE_OFFSET_4                 0x1E0C18
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_START_OFFSET_0                    0x1E0C1C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_START_OFFSET_1                    0x1E0C20
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_START_OFFSET_2                    0x1E0C24
+
+#define mmMME3_CTRL_SHADOW_3_AGU_S_START_OFFSET_3                    0x1E0C28
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_0               0x1E0C2C
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_1               0x1E0C30
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_2               0x1E0C34
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_3               0x1E0C38
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_VALID_ELEMENTS_4               0x1E0C3C
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_0                  0x1E0C40
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_1                  0x1E0C44
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_2                  0x1E0C48
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_3                  0x1E0C4C
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_LOOP_STRIDE_4                  0x1E0C50
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_0                     0x1E0C54
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_1                     0x1E0C58
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_2                     0x1E0C5C
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_ROI_SIZE_3                     0x1E0C60
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_0              0x1E0C64
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_1              0x1E0C68
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_2              0x1E0C6C
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_SPATIAL_STRIDES_3              0x1E0C70
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_L_SPATIAL_SIZE_MINUS_1           0x1E0C74
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_0           0x1E0C78
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_1           0x1E0C7C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_2           0x1E0C80
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_3           0x1E0C84
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_ROI_BASE_OFFSET_4           0x1E0C88
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_0              0x1E0C8C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_1              0x1E0C90
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_2              0x1E0C94
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_LOCAL_START_OFFSET_3              0x1E0C98
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_0          0x1E0C9C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_1          0x1E0CA0
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_2          0x1E0CA4
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_3          0x1E0CA8
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_ROI_BASE_OFFSET_4          0x1E0CAC
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_0             0x1E0CB0
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_1             0x1E0CB4
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_2             0x1E0CB8
+
+#define mmMME3_CTRL_SHADOW_3_AGU_L_REMOTE_START_OFFSET_3             0x1E0CBC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_0               0x1E0CC0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_1               0x1E0CC4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_2               0x1E0CC8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_3               0x1E0CCC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_VALID_ELEMENTS_4               0x1E0CD0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_0                  0x1E0CD4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_1                  0x1E0CD8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_2                  0x1E0CDC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_3                  0x1E0CE0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_LOOP_STRIDE_4                  0x1E0CE4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_0                     0x1E0CE8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_1                     0x1E0CEC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_2                     0x1E0CF0
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_ROI_SIZE_3                     0x1E0CF4
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_0              0x1E0CF8
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_1              0x1E0CFC
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_2              0x1E0D00
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_SPATIAL_STRIDES_3              0x1E0D04
+
+#define mmMME3_CTRL_SHADOW_3_TENSOR_O_SPATIAL_SIZE_MINUS_1           0x1E0D08
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_0           0x1E0D0C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_1           0x1E0D10
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_2           0x1E0D14
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_3           0x1E0D18
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_ROI_BASE_OFFSET_4           0x1E0D1C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_0              0x1E0D20
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_1              0x1E0D24
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_2              0x1E0D28
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_LOCAL_START_OFFSET_3              0x1E0D2C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_0          0x1E0D30
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_1          0x1E0D34
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_2          0x1E0D38
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_3          0x1E0D3C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_ROI_BASE_OFFSET_4          0x1E0D40
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_0             0x1E0D44
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_1             0x1E0D48
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_2             0x1E0D4C
+
+#define mmMME3_CTRL_SHADOW_3_AGU_O_REMOTE_START_OFFSET_3             0x1E0D50
+
+#define mmMME3_CTRL_SHADOW_3_DESC_SB_REPEAT                          0x1E0D54
+
+#define mmMME3_CTRL_SHADOW_3_DESC_RATE_LIMITER                       0x1E0D58
+
+#define mmMME3_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL         0x1E0D5C
+
+#define mmMME3_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_LOW_REMOTE        0x1E0D60
+
+#define mmMME3_CTRL_SHADOW_3_DESC_SYNC_OBJECT_ADDR_HIGH              0x1E0D64
+
+#define mmMME3_CTRL_SHADOW_3_DESC_SYNC_OBJECT_DATA                   0x1E0D68
+
+#define mmMME3_CTRL_SHADOW_3_DESC_AXI_USER_DATA                      0x1E0D6C
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PERF_EVT_S                         0x1E0D70
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PERF_EVT_L_LOCAL                   0x1E0D74
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PERF_EVT_L_REMOTE                  0x1E0D78
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PERF_EVT_O_LOCAL                   0x1E0D7C
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PERF_EVT_O_REMOTE                  0x1E0D80
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PADDING_VALUE_S                    0x1E0D84
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PADDING_VALUE_L                    0x1E0D88
+
+#define mmMME3_CTRL_SHADOW_3_DESC_META_DATA_AGU_S                    0x1E0D8C
+
+#define mmMME3_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_LOCAL              0x1E0D90
+
+#define mmMME3_CTRL_SHADOW_3_DESC_META_DATA_AGU_L_REMOTE             0x1E0D94
+
+#define mmMME3_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_LOCAL              0x1E0D98
+
+#define mmMME3_CTRL_SHADOW_3_DESC_META_DATA_AGU_O_REMOTE             0x1E0D9C
+
+#define mmMME3_CTRL_SHADOW_3_DESC_PCU_RL_SATURATION                  0x1E0DA0
+
+#define mmMME3_CTRL_SHADOW_3_DESC_DUMMY                              0x1E0DA4
+
+#endif /* ASIC_REG_MME3_CTRL_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/mmu_up_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/mmu_up_regs.h
new file mode 100644
index 000000000000..61465b599850
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/mmu_up_regs.h
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_MMU_UP_REGS_H_
+#define ASIC_REG_MMU_UP_REGS_H_
+
+/*
+ *****************************************
+ *   MMU_UP (Prototype: MMU)
+ *****************************************
+ */
+
+#define mmMMU_UP_MMU_ENABLE                                          0xC1100C
+
+#define mmMMU_UP_FORCE_ORDERING                                      0xC11010
+
+#define mmMMU_UP_FEATURE_ENABLE                                      0xC11014
+
+#define mmMMU_UP_VA_ORDERING_MASK_31_7                               0xC11018
+
+#define mmMMU_UP_VA_ORDERING_MASK_49_32                              0xC1101C
+
+#define mmMMU_UP_LOG2_DDR_SIZE                                       0xC11020
+
+#define mmMMU_UP_SCRAMBLER                                           0xC11024
+
+#define mmMMU_UP_MEM_INIT_BUSY                                       0xC11028
+
+#define mmMMU_UP_SPI_MASK                                            0xC1102C
+
+#define mmMMU_UP_SPI_CAUSE                                           0xC11030
+
+#define mmMMU_UP_PAGE_ERROR_CAPTURE                                  0xC11034
+
+#define mmMMU_UP_PAGE_ERROR_CAPTURE_VA                               0xC11038
+
+#define mmMMU_UP_ACCESS_ERROR_CAPTURE                                0xC1103C
+
+#define mmMMU_UP_ACCESS_ERROR_CAPTURE_VA                             0xC11040
+
+#define mmMMU_UP_SPI_INTERRUPT_CLR                                   0xC11044
+
+#define mmMMU_UP_SPI_INTERRUPT_MASK                                  0xC11048
+
+#define mmMMU_UP_DBG_MEM_WRAP_RM                                     0xC1104C
+
+#define mmMMU_UP_SPI_CAUSE_CLR                                       0xC11050
+
+#define mmMMU_UP_SLICE_CREDIT                                        0xC11054
+
+#define mmMMU_UP_PIPE_CREDIT                                         0xC11058
+
+#define mmMMU_UP_RAZWI_WRITE_VLD                                     0xC1105C
+
+#define mmMMU_UP_RAZWI_WRITE_ID                                      0xC11060
+
+#define mmMMU_UP_RAZWI_READ_VLD                                      0xC11064
+
+#define mmMMU_UP_RAZWI_READ_ID                                       0xC11068
+
+#define mmMMU_UP_MMU_BYPASS                                          0xC1106C
+
+#endif /* ASIC_REG_MMU_UP_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_0_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_0_regs.h
new file mode 100644
index 000000000000..2efa2a54deb4
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_0_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_0_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_0_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_0 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_0_PERM_SEL                                    0x386108
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_0                               0x386114
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_1                               0x386118
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_2                               0x38611C
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_3                               0x386120
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_4                               0x386124
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_5                               0x386128
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_6                               0x38612C
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_7                               0x386130
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_8                               0x386134
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_9                               0x386138
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_10                              0x38613C
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_11                              0x386140
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_12                              0x386144
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_13                              0x386148
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_14                              0x38614C
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_15                              0x386150
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_16                              0x386154
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_17                              0x386158
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_18                              0x38615C
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_19                              0x386160
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_20                              0x386164
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_21                              0x386168
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_22                              0x38616C
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_23                              0x386170
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_24                              0x386174
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_25                              0x386178
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_26                              0x38617C
+
+#define mmNIF_RTR_CTRL_0_HBM_POLY_H3_27                              0x386180
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_0                              0x386184
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_1                              0x386188
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_2                              0x38618C
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_3                              0x386190
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_4                              0x386194
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_5                              0x386198
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_6                              0x38619C
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_7                              0x3861A0
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_8                              0x3861A4
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_9                              0x3861A8
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_10                             0x3861AC
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_11                             0x3861B0
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_12                             0x3861B4
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_13                             0x3861B8
+
+#define mmNIF_RTR_CTRL_0_SRAM_POLY_H3_14                             0x3861BC
+
+#define mmNIF_RTR_CTRL_0_SCRAM_SRAM_EN                               0x38626C
+
+#define mmNIF_RTR_CTRL_0_RL_HBM_EN                                   0x386274
+
+#define mmNIF_RTR_CTRL_0_RL_HBM_SAT                                  0x386278
+
+#define mmNIF_RTR_CTRL_0_RL_HBM_RST                                  0x38627C
+
+#define mmNIF_RTR_CTRL_0_RL_HBM_TIMEOUT                              0x386280
+
+#define mmNIF_RTR_CTRL_0_SCRAM_HBM_EN                                0x386284
+
+#define mmNIF_RTR_CTRL_0_RL_PCI_EN                                   0x386288
+
+#define mmNIF_RTR_CTRL_0_RL_PCI_SAT                                  0x38628C
+
+#define mmNIF_RTR_CTRL_0_RL_PCI_RST                                  0x386290
+
+#define mmNIF_RTR_CTRL_0_RL_PCI_TIMEOUT                              0x386294
+
+#define mmNIF_RTR_CTRL_0_RL_SRAM_EN                                  0x38629C
+
+#define mmNIF_RTR_CTRL_0_RL_SRAM_SAT                                 0x3862A0
+
+#define mmNIF_RTR_CTRL_0_RL_SRAM_RST                                 0x3862A4
+
+#define mmNIF_RTR_CTRL_0_RL_SRAM_TIMEOUT                             0x3862AC
+
+#define mmNIF_RTR_CTRL_0_RL_SRAM_RED                                 0x3862B4
+
+#define mmNIF_RTR_CTRL_0_E2E_HBM_EN                                  0x3862EC
+
+#define mmNIF_RTR_CTRL_0_E2E_PCI_EN                                  0x3862F0
+
+#define mmNIF_RTR_CTRL_0_E2E_HBM_WR_SIZE                             0x3862F4
+
+#define mmNIF_RTR_CTRL_0_E2E_PCI_WR_SIZE                             0x3862F8
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_PCI_CTR_SET_EN                       0x386404
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_PCI_CTR_SET                          0x386408
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_PCI_CTR_WRAP                         0x38640C
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_PCI_CTR_CNT                          0x386410
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM_CTR_SET_EN                       0x386414
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM_CTR_SET                          0x386418
+
+#define mmNIF_RTR_CTRL_0_E2E_HBM_RD_SIZE                             0x38641C
+
+#define mmNIF_RTR_CTRL_0_E2E_PCI_RD_SIZE                             0x386420
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_PCI_CTR_SET_EN                       0x386424
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_PCI_CTR_SET                          0x386428
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_PCI_CTR_WRAP                         0x38642C
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_PCI_CTR_CNT                          0x386430
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM_CTR_SET_EN                       0x386434
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM_CTR_SET                          0x386438
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_SEL_0                                0x386450
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_SEL_1                                0x386454
+
+#define mmNIF_RTR_CTRL_0_NON_LIN_EN                                  0x386480
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_BANK_0                              0x386500
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_BANK_1                              0x386504
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_BANK_2                              0x386508
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_BANK_3                              0x38650C
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_BANK_4                              0x386510
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_0                            0x386514
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_1                            0x386520
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_2                            0x386524
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_3                            0x386528
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_4                            0x38652C
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_5                            0x386530
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_6                            0x386534
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_7                            0x386538
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_8                            0x38653C
+
+#define mmNIF_RTR_CTRL_0_NL_SRAM_OFFSET_9                            0x386540
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_0                             0x386550
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_1                             0x386554
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_2                             0x386558
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_3                             0x38655C
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_4                             0x386560
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_5                             0x386564
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_6                             0x386568
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_7                             0x38656C
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_8                             0x386570
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_9                             0x386574
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_10                            0x386578
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_11                            0x38657C
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_12                            0x386580
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_13                            0x386584
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_14                            0x386588
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_15                            0x38658C
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_16                            0x386590
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_17                            0x386594
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_OFFSET_18                            0x386598
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_0                     0x3865E4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_1                     0x3865E8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_2                     0x3865EC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_3                     0x3865F0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_4                     0x3865F4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_5                     0x3865F8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_6                     0x3865FC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_7                     0x386600
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_8                     0x386604
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_9                     0x386608
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_10                    0x38660C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_11                    0x386610
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_12                    0x386614
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_13                    0x386618
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_14                    0x38661C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_15                    0x386620
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_0                    0x386624
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_1                    0x386628
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_2                    0x38662C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_3                    0x386630
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_4                    0x386634
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_5                    0x386638
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_6                    0x38663C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_7                    0x386640
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_8                    0x386644
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_9                    0x386648
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_10                   0x38664C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_11                   0x386650
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_12                   0x386654
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_13                   0x386658
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_14                   0x38665C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_15                   0x386660
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_0                     0x386664
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_1                     0x386668
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_2                     0x38666C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_3                     0x386670
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_4                     0x386674
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_5                     0x386678
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_6                     0x38667C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_7                     0x386680
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_8                     0x386684
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_9                     0x386688
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_10                    0x38668C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_11                    0x386690
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_12                    0x386694
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_13                    0x386698
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_14                    0x38669C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_15                    0x3866A0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_0                    0x3866A4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_1                    0x3866A8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_2                    0x3866AC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_3                    0x3866B0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_4                    0x3866B4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_5                    0x3866B8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_6                    0x3866BC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_7                    0x3866C0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_8                    0x3866C4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_9                    0x3866C8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_10                   0x3866CC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_11                   0x3866D0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_12                   0x3866D4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_13                   0x3866D8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_14                   0x3866DC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_15                   0x3866E0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_0                    0x3866E4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_1                    0x3866E8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_2                    0x3866EC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_3                    0x3866F0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_4                    0x3866F4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_5                    0x3866F8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_6                    0x3866FC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_7                    0x386700
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_8                    0x386704
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_9                    0x386708
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_10                   0x38670C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_11                   0x386710
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_12                   0x386714
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_13                   0x386718
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_14                   0x38671C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_15                   0x386720
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_0                   0x386724
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_1                   0x386728
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_2                   0x38672C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_3                   0x386730
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_4                   0x386734
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_5                   0x386738
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_6                   0x38673C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_7                   0x386740
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_8                   0x386744
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_9                   0x386748
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_10                  0x38674C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_11                  0x386750
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_12                  0x386754
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_13                  0x386758
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_14                  0x38675C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_15                  0x386760
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_0                    0x386764
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_1                    0x386768
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_2                    0x38676C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_3                    0x386770
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_4                    0x386774
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_5                    0x386778
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_6                    0x38677C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_7                    0x386780
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_8                    0x386784
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_9                    0x386788
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_10                   0x38678C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_11                   0x386790
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_12                   0x386794
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_13                   0x386798
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_14                   0x38679C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_15                   0x3867A0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_0                   0x3867A4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_1                   0x3867A8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_2                   0x3867AC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_3                   0x3867B0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_4                   0x3867B4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_5                   0x3867B8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_6                   0x3867BC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_7                   0x3867C0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_8                   0x3867C4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_9                   0x3867C8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_10                  0x3867CC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_11                  0x3867D0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_12                  0x3867D4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_13                  0x3867D8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_14                  0x3867DC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_15                  0x3867E0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_0                     0x386824
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_1                     0x386828
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_2                     0x38682C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_3                     0x386830
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_4                     0x386834
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_5                     0x386838
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_6                     0x38683C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_7                     0x386840
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_8                     0x386844
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_9                     0x386848
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_10                    0x38684C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_11                    0x386850
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_12                    0x386854
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_13                    0x386858
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_14                    0x38685C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_15                    0x386860
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_0                    0x386864
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_1                    0x386868
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_2                    0x38686C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_3                    0x386870
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_4                    0x386874
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_5                    0x386878
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_6                    0x38687C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_7                    0x386880
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_8                    0x386884
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_9                    0x386888
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_10                   0x38688C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_11                   0x386890
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_12                   0x386894
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_13                   0x386898
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_14                   0x38689C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_15                   0x3868A0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_0                     0x3868A4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_1                     0x3868A8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_2                     0x3868AC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_3                     0x3868B0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_4                     0x3868B4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_5                     0x3868B8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_6                     0x3868BC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_7                     0x3868C0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_8                     0x3868C4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_9                     0x3868C8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_10                    0x3868CC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_11                    0x3868D0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_12                    0x3868D4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_13                    0x3868D8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_14                    0x3868DC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_15                    0x3868E0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_0                    0x3868E4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_1                    0x3868E8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_2                    0x3868EC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_3                    0x3868F0
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_4                    0x3868F4
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_5                    0x3868F8
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_6                    0x3868FC
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_7                    0x386900
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_8                    0x386904
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_9                    0x386908
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_10                   0x38690C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_11                   0x386910
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_12                   0x386914
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_13                   0x386918
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_14                   0x38691C
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_15                   0x386920
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_0                    0x386924
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_1                    0x386928
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_2                    0x38692C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_3                    0x386930
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_4                    0x386934
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_5                    0x386938
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_6                    0x38693C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_7                    0x386940
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_8                    0x386944
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_9                    0x386948
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_10                   0x38694C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_11                   0x386950
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_12                   0x386954
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_13                   0x386958
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_14                   0x38695C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_15                   0x386960
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_0                   0x386964
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_1                   0x386968
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_2                   0x38696C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_3                   0x386970
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_4                   0x386974
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_5                   0x386978
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_6                   0x38697C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_7                   0x386980
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_8                   0x386984
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_9                   0x386988
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_10                  0x38698C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_11                  0x386990
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_12                  0x386994
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_13                  0x386998
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_14                  0x38699C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_15                  0x3869A0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_0                    0x3869A4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_1                    0x3869A8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_2                    0x3869AC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_3                    0x3869B0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_4                    0x3869B4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_5                    0x3869B8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_6                    0x3869BC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_7                    0x3869C0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_8                    0x3869C4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_9                    0x3869C8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_10                   0x3869CC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_11                   0x3869D0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_12                   0x3869D4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_13                   0x3869D8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_14                   0x3869DC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_15                   0x3869E0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_0                   0x3869E4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_1                   0x3869E8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_2                   0x3869EC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_3                   0x3869F0
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_4                   0x3869F4
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_5                   0x3869F8
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_6                   0x3869FC
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_7                   0x386A00
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_8                   0x386A04
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_9                   0x386A08
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_10                  0x386A0C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_11                  0x386A10
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_12                  0x386A14
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_13                  0x386A18
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_14                  0x386A1C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_15                  0x386A20
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_HIT_AW                            0x386A64
+
+#define mmNIF_RTR_CTRL_0_RANGE_SEC_HIT_AR                            0x386A68
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_HIT_AW                           0x386A6C
+
+#define mmNIF_RTR_CTRL_0_RANGE_PRIV_HIT_AR                           0x386A70
+
+#define mmNIF_RTR_CTRL_0_RGL_CFG                                     0x386B64
+
+#define mmNIF_RTR_CTRL_0_RGL_SHIFT                                   0x386B68
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_0                          0x386B6C
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_1                          0x386B70
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_2                          0x386B74
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_3                          0x386B78
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_4                          0x386B7C
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_5                          0x386B80
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_6                          0x386B84
+
+#define mmNIF_RTR_CTRL_0_RGL_EXPECTED_LAT_7                          0x386B88
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_0                                 0x386BAC
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_1                                 0x386BB0
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_2                                 0x386BB4
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_3                                 0x386BB8
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_4                                 0x386BBC
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_5                                 0x386BC0
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_6                                 0x386BC4
+
+#define mmNIF_RTR_CTRL_0_RGL_TOKEN_7                                 0x386BC8
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_0                               0x386BEC
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_1                               0x386BF0
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_2                               0x386BF4
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_3                               0x386BF8
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_4                               0x386BFC
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_5                               0x386C00
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_6                               0x386C04
+
+#define mmNIF_RTR_CTRL_0_RGL_BANK_ID_7                               0x386C08
+
+#define mmNIF_RTR_CTRL_0_RGL_WDT                                     0x386C2C
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM0_CH0_CTR_WRAP                    0x386C30
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM0_CH1_CTR_WRAP                    0x386C34
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM1_CH0_CTR_WRAP                    0x386C38
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM1_CH1_CTR_WRAP                    0x386C3C
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM2_CH0_CTR_WRAP                    0x386C40
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM2_CH1_CTR_WRAP                    0x386C44
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM3_CH0_CTR_WRAP                    0x386C48
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM3_CH1_CTR_WRAP                    0x386C4C
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM0_CH0_CTR_CNT                     0x386C50
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM0_CH1_CTR_CNT                     0x386C54
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM1_CH0_CTR_CNT                     0x386C58
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM1_CH1_CTR_CNT                     0x386C5C
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM2_CH0_CTR_CNT                     0x386C60
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM2_CH1_CTR_CNT                     0x386C64
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM3_CH0_CTR_CNT                     0x386C68
+
+#define mmNIF_RTR_CTRL_0_E2E_AR_HBM3_CH1_CTR_CNT                     0x386C6C
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM0_CH0_CTR_WRAP                    0x386C70
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM0_CH1_CTR_WRAP                    0x386C74
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM1_CH0_CTR_WRAP                    0x386C78
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM1_CH1_CTR_WRAP                    0x386C7C
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM2_CH0_CTR_WRAP                    0x386C80
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM2_CH1_CTR_WRAP                    0x386C84
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM3_CH0_CTR_WRAP                    0x386C88
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM3_CH1_CTR_WRAP                    0x386C8C
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM0_CH0_CTR_CNT                     0x386C90
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM0_CH1_CTR_CNT                     0x386C94
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM1_CH0_CTR_CNT                     0x386C98
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM1_CH1_CTR_CNT                     0x386C9C
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM2_CH0_CTR_CNT                     0x386CA0
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM2_CH1_CTR_CNT                     0x386CA4
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM3_CH0_CTR_CNT                     0x386CA8
+
+#define mmNIF_RTR_CTRL_0_E2E_AW_HBM3_CH1_CTR_CNT                     0x386CAC
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_PC_SEL_0                             0x386CB0
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_PC_SEL_1                             0x386CB4
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_PC_SEL_2                             0x386CB8
+
+#define mmNIF_RTR_CTRL_0_NL_HBM_PC_SEL_3                             0x386CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_0_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_1_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_1_regs.h
new file mode 100644
index 000000000000..a6047d4e2560
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_1_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_1_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_1_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_1 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_1_PERM_SEL                                    0x396108
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_0                               0x396114
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_1                               0x396118
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_2                               0x39611C
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_3                               0x396120
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_4                               0x396124
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_5                               0x396128
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_6                               0x39612C
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_7                               0x396130
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_8                               0x396134
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_9                               0x396138
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_10                              0x39613C
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_11                              0x396140
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_12                              0x396144
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_13                              0x396148
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_14                              0x39614C
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_15                              0x396150
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_16                              0x396154
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_17                              0x396158
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_18                              0x39615C
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_19                              0x396160
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_20                              0x396164
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_21                              0x396168
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_22                              0x39616C
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_23                              0x396170
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_24                              0x396174
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_25                              0x396178
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_26                              0x39617C
+
+#define mmNIF_RTR_CTRL_1_HBM_POLY_H3_27                              0x396180
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_0                              0x396184
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_1                              0x396188
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_2                              0x39618C
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_3                              0x396190
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_4                              0x396194
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_5                              0x396198
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_6                              0x39619C
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_7                              0x3961A0
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_8                              0x3961A4
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_9                              0x3961A8
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_10                             0x3961AC
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_11                             0x3961B0
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_12                             0x3961B4
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_13                             0x3961B8
+
+#define mmNIF_RTR_CTRL_1_SRAM_POLY_H3_14                             0x3961BC
+
+#define mmNIF_RTR_CTRL_1_SCRAM_SRAM_EN                               0x39626C
+
+#define mmNIF_RTR_CTRL_1_RL_HBM_EN                                   0x396274
+
+#define mmNIF_RTR_CTRL_1_RL_HBM_SAT                                  0x396278
+
+#define mmNIF_RTR_CTRL_1_RL_HBM_RST                                  0x39627C
+
+#define mmNIF_RTR_CTRL_1_RL_HBM_TIMEOUT                              0x396280
+
+#define mmNIF_RTR_CTRL_1_SCRAM_HBM_EN                                0x396284
+
+#define mmNIF_RTR_CTRL_1_RL_PCI_EN                                   0x396288
+
+#define mmNIF_RTR_CTRL_1_RL_PCI_SAT                                  0x39628C
+
+#define mmNIF_RTR_CTRL_1_RL_PCI_RST                                  0x396290
+
+#define mmNIF_RTR_CTRL_1_RL_PCI_TIMEOUT                              0x396294
+
+#define mmNIF_RTR_CTRL_1_RL_SRAM_EN                                  0x39629C
+
+#define mmNIF_RTR_CTRL_1_RL_SRAM_SAT                                 0x3962A0
+
+#define mmNIF_RTR_CTRL_1_RL_SRAM_RST                                 0x3962A4
+
+#define mmNIF_RTR_CTRL_1_RL_SRAM_TIMEOUT                             0x3962AC
+
+#define mmNIF_RTR_CTRL_1_RL_SRAM_RED                                 0x3962B4
+
+#define mmNIF_RTR_CTRL_1_E2E_HBM_EN                                  0x3962EC
+
+#define mmNIF_RTR_CTRL_1_E2E_PCI_EN                                  0x3962F0
+
+#define mmNIF_RTR_CTRL_1_E2E_HBM_WR_SIZE                             0x3962F4
+
+#define mmNIF_RTR_CTRL_1_E2E_PCI_WR_SIZE                             0x3962F8
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_PCI_CTR_SET_EN                       0x396404
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_PCI_CTR_SET                          0x396408
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_PCI_CTR_WRAP                         0x39640C
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_PCI_CTR_CNT                          0x396410
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM_CTR_SET_EN                       0x396414
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM_CTR_SET                          0x396418
+
+#define mmNIF_RTR_CTRL_1_E2E_HBM_RD_SIZE                             0x39641C
+
+#define mmNIF_RTR_CTRL_1_E2E_PCI_RD_SIZE                             0x396420
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_PCI_CTR_SET_EN                       0x396424
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_PCI_CTR_SET                          0x396428
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_PCI_CTR_WRAP                         0x39642C
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_PCI_CTR_CNT                          0x396430
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM_CTR_SET_EN                       0x396434
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM_CTR_SET                          0x396438
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_SEL_0                                0x396450
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_SEL_1                                0x396454
+
+#define mmNIF_RTR_CTRL_1_NON_LIN_EN                                  0x396480
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_BANK_0                              0x396500
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_BANK_1                              0x396504
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_BANK_2                              0x396508
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_BANK_3                              0x39650C
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_BANK_4                              0x396510
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_0                            0x396514
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_1                            0x396520
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_2                            0x396524
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_3                            0x396528
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_4                            0x39652C
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_5                            0x396530
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_6                            0x396534
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_7                            0x396538
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_8                            0x39653C
+
+#define mmNIF_RTR_CTRL_1_NL_SRAM_OFFSET_9                            0x396540
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_0                             0x396550
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_1                             0x396554
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_2                             0x396558
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_3                             0x39655C
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_4                             0x396560
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_5                             0x396564
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_6                             0x396568
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_7                             0x39656C
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_8                             0x396570
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_9                             0x396574
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_10                            0x396578
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_11                            0x39657C
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_12                            0x396580
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_13                            0x396584
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_14                            0x396588
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_15                            0x39658C
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_16                            0x396590
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_17                            0x396594
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_OFFSET_18                            0x396598
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_0                     0x3965E4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_1                     0x3965E8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_2                     0x3965EC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_3                     0x3965F0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_4                     0x3965F4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_5                     0x3965F8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_6                     0x3965FC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_7                     0x396600
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_8                     0x396604
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_9                     0x396608
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_10                    0x39660C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_11                    0x396610
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_12                    0x396614
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_13                    0x396618
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_14                    0x39661C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_15                    0x396620
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_0                    0x396624
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_1                    0x396628
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_2                    0x39662C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_3                    0x396630
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_4                    0x396634
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_5                    0x396638
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_6                    0x39663C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_7                    0x396640
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_8                    0x396644
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_9                    0x396648
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_10                   0x39664C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_11                   0x396650
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_12                   0x396654
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_13                   0x396658
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_14                   0x39665C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_15                   0x396660
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_0                     0x396664
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_1                     0x396668
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_2                     0x39666C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_3                     0x396670
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_4                     0x396674
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_5                     0x396678
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_6                     0x39667C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_7                     0x396680
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_8                     0x396684
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_9                     0x396688
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_10                    0x39668C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_11                    0x396690
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_12                    0x396694
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_13                    0x396698
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_14                    0x39669C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_15                    0x3966A0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_0                    0x3966A4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_1                    0x3966A8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_2                    0x3966AC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_3                    0x3966B0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_4                    0x3966B4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_5                    0x3966B8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_6                    0x3966BC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_7                    0x3966C0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_8                    0x3966C4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_9                    0x3966C8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_10                   0x3966CC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_11                   0x3966D0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_12                   0x3966D4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_13                   0x3966D8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_14                   0x3966DC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_15                   0x3966E0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_0                    0x3966E4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_1                    0x3966E8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_2                    0x3966EC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_3                    0x3966F0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_4                    0x3966F4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_5                    0x3966F8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_6                    0x3966FC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_7                    0x396700
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_8                    0x396704
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_9                    0x396708
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_10                   0x39670C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_11                   0x396710
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_12                   0x396714
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_13                   0x396718
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_14                   0x39671C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_15                   0x396720
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_0                   0x396724
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_1                   0x396728
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_2                   0x39672C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_3                   0x396730
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_4                   0x396734
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_5                   0x396738
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_6                   0x39673C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_7                   0x396740
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_8                   0x396744
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_9                   0x396748
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_10                  0x39674C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_11                  0x396750
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_12                  0x396754
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_13                  0x396758
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_14                  0x39675C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_15                  0x396760
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_0                    0x396764
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_1                    0x396768
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_2                    0x39676C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_3                    0x396770
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_4                    0x396774
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_5                    0x396778
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_6                    0x39677C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_7                    0x396780
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_8                    0x396784
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_9                    0x396788
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_10                   0x39678C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_11                   0x396790
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_12                   0x396794
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_13                   0x396798
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_14                   0x39679C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_15                   0x3967A0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_0                   0x3967A4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_1                   0x3967A8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_2                   0x3967AC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_3                   0x3967B0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_4                   0x3967B4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_5                   0x3967B8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_6                   0x3967BC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_7                   0x3967C0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_8                   0x3967C4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_9                   0x3967C8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_10                  0x3967CC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_11                  0x3967D0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_12                  0x3967D4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_13                  0x3967D8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_14                  0x3967DC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_15                  0x3967E0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_0                     0x396824
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_1                     0x396828
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_2                     0x39682C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_3                     0x396830
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_4                     0x396834
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_5                     0x396838
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_6                     0x39683C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_7                     0x396840
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_8                     0x396844
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_9                     0x396848
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_10                    0x39684C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_11                    0x396850
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_12                    0x396854
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_13                    0x396858
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_14                    0x39685C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_15                    0x396860
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_0                    0x396864
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_1                    0x396868
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_2                    0x39686C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_3                    0x396870
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_4                    0x396874
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_5                    0x396878
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_6                    0x39687C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_7                    0x396880
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_8                    0x396884
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_9                    0x396888
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_10                   0x39688C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_11                   0x396890
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_12                   0x396894
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_13                   0x396898
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_14                   0x39689C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_15                   0x3968A0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_0                     0x3968A4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_1                     0x3968A8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_2                     0x3968AC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_3                     0x3968B0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_4                     0x3968B4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_5                     0x3968B8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_6                     0x3968BC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_7                     0x3968C0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_8                     0x3968C4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_9                     0x3968C8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_10                    0x3968CC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_11                    0x3968D0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_12                    0x3968D4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_13                    0x3968D8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_14                    0x3968DC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_15                    0x3968E0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_0                    0x3968E4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_1                    0x3968E8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_2                    0x3968EC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_3                    0x3968F0
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_4                    0x3968F4
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_5                    0x3968F8
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_6                    0x3968FC
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_7                    0x396900
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_8                    0x396904
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_9                    0x396908
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_10                   0x39690C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_11                   0x396910
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_12                   0x396914
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_13                   0x396918
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_14                   0x39691C
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_15                   0x396920
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_0                    0x396924
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_1                    0x396928
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_2                    0x39692C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_3                    0x396930
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_4                    0x396934
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_5                    0x396938
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_6                    0x39693C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_7                    0x396940
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_8                    0x396944
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_9                    0x396948
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_10                   0x39694C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_11                   0x396950
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_12                   0x396954
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_13                   0x396958
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_14                   0x39695C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_15                   0x396960
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_0                   0x396964
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_1                   0x396968
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_2                   0x39696C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_3                   0x396970
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_4                   0x396974
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_5                   0x396978
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_6                   0x39697C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_7                   0x396980
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_8                   0x396984
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_9                   0x396988
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_10                  0x39698C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_11                  0x396990
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_12                  0x396994
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_13                  0x396998
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_14                  0x39699C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_15                  0x3969A0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_0                    0x3969A4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_1                    0x3969A8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_2                    0x3969AC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_3                    0x3969B0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_4                    0x3969B4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_5                    0x3969B8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_6                    0x3969BC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_7                    0x3969C0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_8                    0x3969C4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_9                    0x3969C8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_10                   0x3969CC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_11                   0x3969D0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_12                   0x3969D4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_13                   0x3969D8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_14                   0x3969DC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_15                   0x3969E0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_0                   0x3969E4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_1                   0x3969E8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_2                   0x3969EC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_3                   0x3969F0
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_4                   0x3969F4
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_5                   0x3969F8
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_6                   0x3969FC
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_7                   0x396A00
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_8                   0x396A04
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_9                   0x396A08
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_10                  0x396A0C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_11                  0x396A10
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_12                  0x396A14
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_13                  0x396A18
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_14                  0x396A1C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_15                  0x396A20
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_HIT_AW                            0x396A64
+
+#define mmNIF_RTR_CTRL_1_RANGE_SEC_HIT_AR                            0x396A68
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_HIT_AW                           0x396A6C
+
+#define mmNIF_RTR_CTRL_1_RANGE_PRIV_HIT_AR                           0x396A70
+
+#define mmNIF_RTR_CTRL_1_RGL_CFG                                     0x396B64
+
+#define mmNIF_RTR_CTRL_1_RGL_SHIFT                                   0x396B68
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_0                          0x396B6C
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_1                          0x396B70
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_2                          0x396B74
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_3                          0x396B78
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_4                          0x396B7C
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_5                          0x396B80
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_6                          0x396B84
+
+#define mmNIF_RTR_CTRL_1_RGL_EXPECTED_LAT_7                          0x396B88
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_0                                 0x396BAC
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_1                                 0x396BB0
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_2                                 0x396BB4
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_3                                 0x396BB8
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_4                                 0x396BBC
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_5                                 0x396BC0
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_6                                 0x396BC4
+
+#define mmNIF_RTR_CTRL_1_RGL_TOKEN_7                                 0x396BC8
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_0                               0x396BEC
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_1                               0x396BF0
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_2                               0x396BF4
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_3                               0x396BF8
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_4                               0x396BFC
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_5                               0x396C00
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_6                               0x396C04
+
+#define mmNIF_RTR_CTRL_1_RGL_BANK_ID_7                               0x396C08
+
+#define mmNIF_RTR_CTRL_1_RGL_WDT                                     0x396C2C
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM0_CH0_CTR_WRAP                    0x396C30
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM0_CH1_CTR_WRAP                    0x396C34
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM1_CH0_CTR_WRAP                    0x396C38
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM1_CH1_CTR_WRAP                    0x396C3C
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM2_CH0_CTR_WRAP                    0x396C40
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM2_CH1_CTR_WRAP                    0x396C44
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM3_CH0_CTR_WRAP                    0x396C48
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM3_CH1_CTR_WRAP                    0x396C4C
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM0_CH0_CTR_CNT                     0x396C50
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM0_CH1_CTR_CNT                     0x396C54
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM1_CH0_CTR_CNT                     0x396C58
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM1_CH1_CTR_CNT                     0x396C5C
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM2_CH0_CTR_CNT                     0x396C60
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM2_CH1_CTR_CNT                     0x396C64
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM3_CH0_CTR_CNT                     0x396C68
+
+#define mmNIF_RTR_CTRL_1_E2E_AR_HBM3_CH1_CTR_CNT                     0x396C6C
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM0_CH0_CTR_WRAP                    0x396C70
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM0_CH1_CTR_WRAP                    0x396C74
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM1_CH0_CTR_WRAP                    0x396C78
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM1_CH1_CTR_WRAP                    0x396C7C
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM2_CH0_CTR_WRAP                    0x396C80
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM2_CH1_CTR_WRAP                    0x396C84
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM3_CH0_CTR_WRAP                    0x396C88
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM3_CH1_CTR_WRAP                    0x396C8C
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM0_CH0_CTR_CNT                     0x396C90
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM0_CH1_CTR_CNT                     0x396C94
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM1_CH0_CTR_CNT                     0x396C98
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM1_CH1_CTR_CNT                     0x396C9C
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM2_CH0_CTR_CNT                     0x396CA0
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM2_CH1_CTR_CNT                     0x396CA4
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM3_CH0_CTR_CNT                     0x396CA8
+
+#define mmNIF_RTR_CTRL_1_E2E_AW_HBM3_CH1_CTR_CNT                     0x396CAC
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_PC_SEL_0                             0x396CB0
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_PC_SEL_1                             0x396CB4
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_PC_SEL_2                             0x396CB8
+
+#define mmNIF_RTR_CTRL_1_NL_HBM_PC_SEL_3                             0x396CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_1_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_2_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_2_regs.h
new file mode 100644
index 000000000000..9de8442f9bc2
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_2_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_2_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_2_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_2 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_2_PERM_SEL                                    0x3A6108
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_0                               0x3A6114
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_1                               0x3A6118
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_2                               0x3A611C
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_3                               0x3A6120
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_4                               0x3A6124
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_5                               0x3A6128
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_6                               0x3A612C
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_7                               0x3A6130
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_8                               0x3A6134
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_9                               0x3A6138
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_10                              0x3A613C
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_11                              0x3A6140
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_12                              0x3A6144
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_13                              0x3A6148
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_14                              0x3A614C
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_15                              0x3A6150
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_16                              0x3A6154
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_17                              0x3A6158
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_18                              0x3A615C
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_19                              0x3A6160
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_20                              0x3A6164
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_21                              0x3A6168
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_22                              0x3A616C
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_23                              0x3A6170
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_24                              0x3A6174
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_25                              0x3A6178
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_26                              0x3A617C
+
+#define mmNIF_RTR_CTRL_2_HBM_POLY_H3_27                              0x3A6180
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_0                              0x3A6184
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_1                              0x3A6188
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_2                              0x3A618C
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_3                              0x3A6190
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_4                              0x3A6194
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_5                              0x3A6198
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_6                              0x3A619C
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_7                              0x3A61A0
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_8                              0x3A61A4
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_9                              0x3A61A8
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_10                             0x3A61AC
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_11                             0x3A61B0
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_12                             0x3A61B4
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_13                             0x3A61B8
+
+#define mmNIF_RTR_CTRL_2_SRAM_POLY_H3_14                             0x3A61BC
+
+#define mmNIF_RTR_CTRL_2_SCRAM_SRAM_EN                               0x3A626C
+
+#define mmNIF_RTR_CTRL_2_RL_HBM_EN                                   0x3A6274
+
+#define mmNIF_RTR_CTRL_2_RL_HBM_SAT                                  0x3A6278
+
+#define mmNIF_RTR_CTRL_2_RL_HBM_RST                                  0x3A627C
+
+#define mmNIF_RTR_CTRL_2_RL_HBM_TIMEOUT                              0x3A6280
+
+#define mmNIF_RTR_CTRL_2_SCRAM_HBM_EN                                0x3A6284
+
+#define mmNIF_RTR_CTRL_2_RL_PCI_EN                                   0x3A6288
+
+#define mmNIF_RTR_CTRL_2_RL_PCI_SAT                                  0x3A628C
+
+#define mmNIF_RTR_CTRL_2_RL_PCI_RST                                  0x3A6290
+
+#define mmNIF_RTR_CTRL_2_RL_PCI_TIMEOUT                              0x3A6294
+
+#define mmNIF_RTR_CTRL_2_RL_SRAM_EN                                  0x3A629C
+
+#define mmNIF_RTR_CTRL_2_RL_SRAM_SAT                                 0x3A62A0
+
+#define mmNIF_RTR_CTRL_2_RL_SRAM_RST                                 0x3A62A4
+
+#define mmNIF_RTR_CTRL_2_RL_SRAM_TIMEOUT                             0x3A62AC
+
+#define mmNIF_RTR_CTRL_2_RL_SRAM_RED                                 0x3A62B4
+
+#define mmNIF_RTR_CTRL_2_E2E_HBM_EN                                  0x3A62EC
+
+#define mmNIF_RTR_CTRL_2_E2E_PCI_EN                                  0x3A62F0
+
+#define mmNIF_RTR_CTRL_2_E2E_HBM_WR_SIZE                             0x3A62F4
+
+#define mmNIF_RTR_CTRL_2_E2E_PCI_WR_SIZE                             0x3A62F8
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_PCI_CTR_SET_EN                       0x3A6404
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_PCI_CTR_SET                          0x3A6408
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_PCI_CTR_WRAP                         0x3A640C
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_PCI_CTR_CNT                          0x3A6410
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM_CTR_SET_EN                       0x3A6414
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM_CTR_SET                          0x3A6418
+
+#define mmNIF_RTR_CTRL_2_E2E_HBM_RD_SIZE                             0x3A641C
+
+#define mmNIF_RTR_CTRL_2_E2E_PCI_RD_SIZE                             0x3A6420
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_PCI_CTR_SET_EN                       0x3A6424
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_PCI_CTR_SET                          0x3A6428
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_PCI_CTR_WRAP                         0x3A642C
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_PCI_CTR_CNT                          0x3A6430
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM_CTR_SET_EN                       0x3A6434
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM_CTR_SET                          0x3A6438
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_SEL_0                                0x3A6450
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_SEL_1                                0x3A6454
+
+#define mmNIF_RTR_CTRL_2_NON_LIN_EN                                  0x3A6480
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_BANK_0                              0x3A6500
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_BANK_1                              0x3A6504
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_BANK_2                              0x3A6508
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_BANK_3                              0x3A650C
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_BANK_4                              0x3A6510
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_0                            0x3A6514
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_1                            0x3A6520
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_2                            0x3A6524
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_3                            0x3A6528
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_4                            0x3A652C
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_5                            0x3A6530
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_6                            0x3A6534
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_7                            0x3A6538
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_8                            0x3A653C
+
+#define mmNIF_RTR_CTRL_2_NL_SRAM_OFFSET_9                            0x3A6540
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_0                             0x3A6550
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_1                             0x3A6554
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_2                             0x3A6558
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_3                             0x3A655C
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_4                             0x3A6560
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_5                             0x3A6564
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_6                             0x3A6568
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_7                             0x3A656C
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_8                             0x3A6570
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_9                             0x3A6574
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_10                            0x3A6578
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_11                            0x3A657C
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_12                            0x3A6580
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_13                            0x3A6584
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_14                            0x3A6588
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_15                            0x3A658C
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_16                            0x3A6590
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_17                            0x3A6594
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_OFFSET_18                            0x3A6598
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_0                     0x3A65E4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_1                     0x3A65E8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_2                     0x3A65EC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_3                     0x3A65F0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_4                     0x3A65F4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_5                     0x3A65F8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_6                     0x3A65FC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_7                     0x3A6600
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_8                     0x3A6604
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_9                     0x3A6608
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_10                    0x3A660C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_11                    0x3A6610
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_12                    0x3A6614
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_13                    0x3A6618
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_14                    0x3A661C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_15                    0x3A6620
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_0                    0x3A6624
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_1                    0x3A6628
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_2                    0x3A662C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_3                    0x3A6630
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_4                    0x3A6634
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_5                    0x3A6638
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_6                    0x3A663C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_7                    0x3A6640
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_8                    0x3A6644
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_9                    0x3A6648
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_10                   0x3A664C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_11                   0x3A6650
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_12                   0x3A6654
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_13                   0x3A6658
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_14                   0x3A665C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_15                   0x3A6660
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_0                     0x3A6664
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_1                     0x3A6668
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_2                     0x3A666C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_3                     0x3A6670
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_4                     0x3A6674
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_5                     0x3A6678
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_6                     0x3A667C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_7                     0x3A6680
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_8                     0x3A6684
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_9                     0x3A6688
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_10                    0x3A668C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_11                    0x3A6690
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_12                    0x3A6694
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_13                    0x3A6698
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_14                    0x3A669C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_15                    0x3A66A0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_0                    0x3A66A4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_1                    0x3A66A8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_2                    0x3A66AC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_3                    0x3A66B0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_4                    0x3A66B4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_5                    0x3A66B8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_6                    0x3A66BC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_7                    0x3A66C0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_8                    0x3A66C4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_9                    0x3A66C8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_10                   0x3A66CC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_11                   0x3A66D0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_12                   0x3A66D4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_13                   0x3A66D8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_14                   0x3A66DC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_15                   0x3A66E0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_0                    0x3A66E4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_1                    0x3A66E8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_2                    0x3A66EC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_3                    0x3A66F0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_4                    0x3A66F4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_5                    0x3A66F8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_6                    0x3A66FC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_7                    0x3A6700
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_8                    0x3A6704
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_9                    0x3A6708
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_10                   0x3A670C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_11                   0x3A6710
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_12                   0x3A6714
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_13                   0x3A6718
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_14                   0x3A671C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_15                   0x3A6720
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_0                   0x3A6724
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_1                   0x3A6728
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_2                   0x3A672C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_3                   0x3A6730
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_4                   0x3A6734
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_5                   0x3A6738
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_6                   0x3A673C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_7                   0x3A6740
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_8                   0x3A6744
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_9                   0x3A6748
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_10                  0x3A674C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_11                  0x3A6750
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_12                  0x3A6754
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_13                  0x3A6758
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_14                  0x3A675C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_15                  0x3A6760
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_0                    0x3A6764
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_1                    0x3A6768
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_2                    0x3A676C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_3                    0x3A6770
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_4                    0x3A6774
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_5                    0x3A6778
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_6                    0x3A677C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_7                    0x3A6780
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_8                    0x3A6784
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_9                    0x3A6788
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_10                   0x3A678C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_11                   0x3A6790
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_12                   0x3A6794
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_13                   0x3A6798
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_14                   0x3A679C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_15                   0x3A67A0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_0                   0x3A67A4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_1                   0x3A67A8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_2                   0x3A67AC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_3                   0x3A67B0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_4                   0x3A67B4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_5                   0x3A67B8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_6                   0x3A67BC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_7                   0x3A67C0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_8                   0x3A67C4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_9                   0x3A67C8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_10                  0x3A67CC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_11                  0x3A67D0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_12                  0x3A67D4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_13                  0x3A67D8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_14                  0x3A67DC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_15                  0x3A67E0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_0                     0x3A6824
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_1                     0x3A6828
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_2                     0x3A682C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_3                     0x3A6830
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_4                     0x3A6834
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_5                     0x3A6838
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_6                     0x3A683C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_7                     0x3A6840
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_8                     0x3A6844
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_9                     0x3A6848
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_10                    0x3A684C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_11                    0x3A6850
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_12                    0x3A6854
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_13                    0x3A6858
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_14                    0x3A685C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_15                    0x3A6860
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_0                    0x3A6864
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_1                    0x3A6868
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_2                    0x3A686C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_3                    0x3A6870
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_4                    0x3A6874
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_5                    0x3A6878
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_6                    0x3A687C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_7                    0x3A6880
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_8                    0x3A6884
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_9                    0x3A6888
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_10                   0x3A688C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_11                   0x3A6890
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_12                   0x3A6894
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_13                   0x3A6898
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_14                   0x3A689C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_15                   0x3A68A0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_0                     0x3A68A4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_1                     0x3A68A8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_2                     0x3A68AC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_3                     0x3A68B0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_4                     0x3A68B4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_5                     0x3A68B8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_6                     0x3A68BC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_7                     0x3A68C0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_8                     0x3A68C4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_9                     0x3A68C8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_10                    0x3A68CC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_11                    0x3A68D0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_12                    0x3A68D4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_13                    0x3A68D8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_14                    0x3A68DC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_15                    0x3A68E0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_0                    0x3A68E4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_1                    0x3A68E8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_2                    0x3A68EC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_3                    0x3A68F0
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_4                    0x3A68F4
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_5                    0x3A68F8
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_6                    0x3A68FC
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_7                    0x3A6900
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_8                    0x3A6904
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_9                    0x3A6908
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_10                   0x3A690C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_11                   0x3A6910
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_12                   0x3A6914
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_13                   0x3A6918
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_14                   0x3A691C
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_15                   0x3A6920
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_0                    0x3A6924
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_1                    0x3A6928
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_2                    0x3A692C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_3                    0x3A6930
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_4                    0x3A6934
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_5                    0x3A6938
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_6                    0x3A693C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_7                    0x3A6940
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_8                    0x3A6944
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_9                    0x3A6948
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_10                   0x3A694C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_11                   0x3A6950
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_12                   0x3A6954
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_13                   0x3A6958
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_14                   0x3A695C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_15                   0x3A6960
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_0                   0x3A6964
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_1                   0x3A6968
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_2                   0x3A696C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_3                   0x3A6970
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_4                   0x3A6974
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_5                   0x3A6978
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_6                   0x3A697C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_7                   0x3A6980
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_8                   0x3A6984
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_9                   0x3A6988
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_10                  0x3A698C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_11                  0x3A6990
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_12                  0x3A6994
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_13                  0x3A6998
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_14                  0x3A699C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_15                  0x3A69A0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_0                    0x3A69A4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_1                    0x3A69A8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_2                    0x3A69AC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_3                    0x3A69B0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_4                    0x3A69B4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_5                    0x3A69B8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_6                    0x3A69BC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_7                    0x3A69C0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_8                    0x3A69C4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_9                    0x3A69C8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_10                   0x3A69CC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_11                   0x3A69D0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_12                   0x3A69D4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_13                   0x3A69D8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_14                   0x3A69DC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_15                   0x3A69E0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_0                   0x3A69E4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_1                   0x3A69E8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_2                   0x3A69EC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_3                   0x3A69F0
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_4                   0x3A69F4
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_5                   0x3A69F8
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_6                   0x3A69FC
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_7                   0x3A6A00
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_8                   0x3A6A04
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_9                   0x3A6A08
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_10                  0x3A6A0C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_11                  0x3A6A10
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_12                  0x3A6A14
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_13                  0x3A6A18
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_14                  0x3A6A1C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_15                  0x3A6A20
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_HIT_AW                            0x3A6A64
+
+#define mmNIF_RTR_CTRL_2_RANGE_SEC_HIT_AR                            0x3A6A68
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_HIT_AW                           0x3A6A6C
+
+#define mmNIF_RTR_CTRL_2_RANGE_PRIV_HIT_AR                           0x3A6A70
+
+#define mmNIF_RTR_CTRL_2_RGL_CFG                                     0x3A6B64
+
+#define mmNIF_RTR_CTRL_2_RGL_SHIFT                                   0x3A6B68
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_0                          0x3A6B6C
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_1                          0x3A6B70
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_2                          0x3A6B74
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_3                          0x3A6B78
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_4                          0x3A6B7C
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_5                          0x3A6B80
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_6                          0x3A6B84
+
+#define mmNIF_RTR_CTRL_2_RGL_EXPECTED_LAT_7                          0x3A6B88
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_0                                 0x3A6BAC
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_1                                 0x3A6BB0
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_2                                 0x3A6BB4
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_3                                 0x3A6BB8
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_4                                 0x3A6BBC
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_5                                 0x3A6BC0
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_6                                 0x3A6BC4
+
+#define mmNIF_RTR_CTRL_2_RGL_TOKEN_7                                 0x3A6BC8
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_0                               0x3A6BEC
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_1                               0x3A6BF0
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_2                               0x3A6BF4
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_3                               0x3A6BF8
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_4                               0x3A6BFC
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_5                               0x3A6C00
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_6                               0x3A6C04
+
+#define mmNIF_RTR_CTRL_2_RGL_BANK_ID_7                               0x3A6C08
+
+#define mmNIF_RTR_CTRL_2_RGL_WDT                                     0x3A6C2C
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM0_CH0_CTR_WRAP                    0x3A6C30
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM0_CH1_CTR_WRAP                    0x3A6C34
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM1_CH0_CTR_WRAP                    0x3A6C38
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM1_CH1_CTR_WRAP                    0x3A6C3C
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM2_CH0_CTR_WRAP                    0x3A6C40
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM2_CH1_CTR_WRAP                    0x3A6C44
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM3_CH0_CTR_WRAP                    0x3A6C48
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM3_CH1_CTR_WRAP                    0x3A6C4C
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM0_CH0_CTR_CNT                     0x3A6C50
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM0_CH1_CTR_CNT                     0x3A6C54
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM1_CH0_CTR_CNT                     0x3A6C58
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM1_CH1_CTR_CNT                     0x3A6C5C
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM2_CH0_CTR_CNT                     0x3A6C60
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM2_CH1_CTR_CNT                     0x3A6C64
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM3_CH0_CTR_CNT                     0x3A6C68
+
+#define mmNIF_RTR_CTRL_2_E2E_AR_HBM3_CH1_CTR_CNT                     0x3A6C6C
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM0_CH0_CTR_WRAP                    0x3A6C70
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM0_CH1_CTR_WRAP                    0x3A6C74
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM1_CH0_CTR_WRAP                    0x3A6C78
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM1_CH1_CTR_WRAP                    0x3A6C7C
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM2_CH0_CTR_WRAP                    0x3A6C80
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM2_CH1_CTR_WRAP                    0x3A6C84
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM3_CH0_CTR_WRAP                    0x3A6C88
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM3_CH1_CTR_WRAP                    0x3A6C8C
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM0_CH0_CTR_CNT                     0x3A6C90
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM0_CH1_CTR_CNT                     0x3A6C94
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM1_CH0_CTR_CNT                     0x3A6C98
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM1_CH1_CTR_CNT                     0x3A6C9C
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM2_CH0_CTR_CNT                     0x3A6CA0
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM2_CH1_CTR_CNT                     0x3A6CA4
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM3_CH0_CTR_CNT                     0x3A6CA8
+
+#define mmNIF_RTR_CTRL_2_E2E_AW_HBM3_CH1_CTR_CNT                     0x3A6CAC
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_PC_SEL_0                             0x3A6CB0
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_PC_SEL_1                             0x3A6CB4
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_PC_SEL_2                             0x3A6CB8
+
+#define mmNIF_RTR_CTRL_2_NL_HBM_PC_SEL_3                             0x3A6CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_2_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_3_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_3_regs.h
new file mode 100644
index 000000000000..34fd47685edd
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_3_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_3_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_3_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_3 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_3_PERM_SEL                                    0x3B6108
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_0                               0x3B6114
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_1                               0x3B6118
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_2                               0x3B611C
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_3                               0x3B6120
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_4                               0x3B6124
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_5                               0x3B6128
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_6                               0x3B612C
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_7                               0x3B6130
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_8                               0x3B6134
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_9                               0x3B6138
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_10                              0x3B613C
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_11                              0x3B6140
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_12                              0x3B6144
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_13                              0x3B6148
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_14                              0x3B614C
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_15                              0x3B6150
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_16                              0x3B6154
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_17                              0x3B6158
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_18                              0x3B615C
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_19                              0x3B6160
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_20                              0x3B6164
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_21                              0x3B6168
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_22                              0x3B616C
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_23                              0x3B6170
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_24                              0x3B6174
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_25                              0x3B6178
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_26                              0x3B617C
+
+#define mmNIF_RTR_CTRL_3_HBM_POLY_H3_27                              0x3B6180
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_0                              0x3B6184
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_1                              0x3B6188
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_2                              0x3B618C
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_3                              0x3B6190
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_4                              0x3B6194
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_5                              0x3B6198
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_6                              0x3B619C
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_7                              0x3B61A0
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_8                              0x3B61A4
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_9                              0x3B61A8
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_10                             0x3B61AC
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_11                             0x3B61B0
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_12                             0x3B61B4
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_13                             0x3B61B8
+
+#define mmNIF_RTR_CTRL_3_SRAM_POLY_H3_14                             0x3B61BC
+
+#define mmNIF_RTR_CTRL_3_SCRAM_SRAM_EN                               0x3B626C
+
+#define mmNIF_RTR_CTRL_3_RL_HBM_EN                                   0x3B6274
+
+#define mmNIF_RTR_CTRL_3_RL_HBM_SAT                                  0x3B6278
+
+#define mmNIF_RTR_CTRL_3_RL_HBM_RST                                  0x3B627C
+
+#define mmNIF_RTR_CTRL_3_RL_HBM_TIMEOUT                              0x3B6280
+
+#define mmNIF_RTR_CTRL_3_SCRAM_HBM_EN                                0x3B6284
+
+#define mmNIF_RTR_CTRL_3_RL_PCI_EN                                   0x3B6288
+
+#define mmNIF_RTR_CTRL_3_RL_PCI_SAT                                  0x3B628C
+
+#define mmNIF_RTR_CTRL_3_RL_PCI_RST                                  0x3B6290
+
+#define mmNIF_RTR_CTRL_3_RL_PCI_TIMEOUT                              0x3B6294
+
+#define mmNIF_RTR_CTRL_3_RL_SRAM_EN                                  0x3B629C
+
+#define mmNIF_RTR_CTRL_3_RL_SRAM_SAT                                 0x3B62A0
+
+#define mmNIF_RTR_CTRL_3_RL_SRAM_RST                                 0x3B62A4
+
+#define mmNIF_RTR_CTRL_3_RL_SRAM_TIMEOUT                             0x3B62AC
+
+#define mmNIF_RTR_CTRL_3_RL_SRAM_RED                                 0x3B62B4
+
+#define mmNIF_RTR_CTRL_3_E2E_HBM_EN                                  0x3B62EC
+
+#define mmNIF_RTR_CTRL_3_E2E_PCI_EN                                  0x3B62F0
+
+#define mmNIF_RTR_CTRL_3_E2E_HBM_WR_SIZE                             0x3B62F4
+
+#define mmNIF_RTR_CTRL_3_E2E_PCI_WR_SIZE                             0x3B62F8
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_PCI_CTR_SET_EN                       0x3B6404
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_PCI_CTR_SET                          0x3B6408
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_PCI_CTR_WRAP                         0x3B640C
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_PCI_CTR_CNT                          0x3B6410
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM_CTR_SET_EN                       0x3B6414
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM_CTR_SET                          0x3B6418
+
+#define mmNIF_RTR_CTRL_3_E2E_HBM_RD_SIZE                             0x3B641C
+
+#define mmNIF_RTR_CTRL_3_E2E_PCI_RD_SIZE                             0x3B6420
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_PCI_CTR_SET_EN                       0x3B6424
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_PCI_CTR_SET                          0x3B6428
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_PCI_CTR_WRAP                         0x3B642C
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_PCI_CTR_CNT                          0x3B6430
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM_CTR_SET_EN                       0x3B6434
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM_CTR_SET                          0x3B6438
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_SEL_0                                0x3B6450
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_SEL_1                                0x3B6454
+
+#define mmNIF_RTR_CTRL_3_NON_LIN_EN                                  0x3B6480
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_BANK_0                              0x3B6500
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_BANK_1                              0x3B6504
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_BANK_2                              0x3B6508
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_BANK_3                              0x3B650C
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_BANK_4                              0x3B6510
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_0                            0x3B6514
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_1                            0x3B6520
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_2                            0x3B6524
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_3                            0x3B6528
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_4                            0x3B652C
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_5                            0x3B6530
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_6                            0x3B6534
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_7                            0x3B6538
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_8                            0x3B653C
+
+#define mmNIF_RTR_CTRL_3_NL_SRAM_OFFSET_9                            0x3B6540
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_0                             0x3B6550
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_1                             0x3B6554
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_2                             0x3B6558
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_3                             0x3B655C
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_4                             0x3B6560
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_5                             0x3B6564
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_6                             0x3B6568
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_7                             0x3B656C
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_8                             0x3B6570
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_9                             0x3B6574
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_10                            0x3B6578
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_11                            0x3B657C
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_12                            0x3B6580
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_13                            0x3B6584
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_14                            0x3B6588
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_15                            0x3B658C
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_16                            0x3B6590
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_17                            0x3B6594
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_OFFSET_18                            0x3B6598
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_0                     0x3B65E4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_1                     0x3B65E8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_2                     0x3B65EC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_3                     0x3B65F0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_4                     0x3B65F4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_5                     0x3B65F8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_6                     0x3B65FC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_7                     0x3B6600
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_8                     0x3B6604
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_9                     0x3B6608
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_10                    0x3B660C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_11                    0x3B6610
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_12                    0x3B6614
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_13                    0x3B6618
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_14                    0x3B661C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_15                    0x3B6620
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_0                    0x3B6624
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_1                    0x3B6628
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_2                    0x3B662C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_3                    0x3B6630
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_4                    0x3B6634
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_5                    0x3B6638
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_6                    0x3B663C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_7                    0x3B6640
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_8                    0x3B6644
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_9                    0x3B6648
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_10                   0x3B664C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_11                   0x3B6650
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_12                   0x3B6654
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_13                   0x3B6658
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_14                   0x3B665C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_15                   0x3B6660
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_0                     0x3B6664
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_1                     0x3B6668
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_2                     0x3B666C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_3                     0x3B6670
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_4                     0x3B6674
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_5                     0x3B6678
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_6                     0x3B667C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_7                     0x3B6680
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_8                     0x3B6684
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_9                     0x3B6688
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_10                    0x3B668C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_11                    0x3B6690
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_12                    0x3B6694
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_13                    0x3B6698
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_14                    0x3B669C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_15                    0x3B66A0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_0                    0x3B66A4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_1                    0x3B66A8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_2                    0x3B66AC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_3                    0x3B66B0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_4                    0x3B66B4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_5                    0x3B66B8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_6                    0x3B66BC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_7                    0x3B66C0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_8                    0x3B66C4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_9                    0x3B66C8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_10                   0x3B66CC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_11                   0x3B66D0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_12                   0x3B66D4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_13                   0x3B66D8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_14                   0x3B66DC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_15                   0x3B66E0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_0                    0x3B66E4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_1                    0x3B66E8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_2                    0x3B66EC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_3                    0x3B66F0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_4                    0x3B66F4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_5                    0x3B66F8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_6                    0x3B66FC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_7                    0x3B6700
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_8                    0x3B6704
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_9                    0x3B6708
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_10                   0x3B670C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_11                   0x3B6710
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_12                   0x3B6714
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_13                   0x3B6718
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_14                   0x3B671C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_15                   0x3B6720
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_0                   0x3B6724
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_1                   0x3B6728
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_2                   0x3B672C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_3                   0x3B6730
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_4                   0x3B6734
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_5                   0x3B6738
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_6                   0x3B673C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_7                   0x3B6740
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_8                   0x3B6744
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_9                   0x3B6748
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_10                  0x3B674C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_11                  0x3B6750
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_12                  0x3B6754
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_13                  0x3B6758
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_14                  0x3B675C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_15                  0x3B6760
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_0                    0x3B6764
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_1                    0x3B6768
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_2                    0x3B676C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_3                    0x3B6770
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_4                    0x3B6774
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_5                    0x3B6778
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_6                    0x3B677C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_7                    0x3B6780
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_8                    0x3B6784
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_9                    0x3B6788
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_10                   0x3B678C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_11                   0x3B6790
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_12                   0x3B6794
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_13                   0x3B6798
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_14                   0x3B679C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_15                   0x3B67A0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_0                   0x3B67A4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_1                   0x3B67A8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_2                   0x3B67AC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_3                   0x3B67B0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_4                   0x3B67B4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_5                   0x3B67B8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_6                   0x3B67BC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_7                   0x3B67C0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_8                   0x3B67C4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_9                   0x3B67C8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_10                  0x3B67CC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_11                  0x3B67D0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_12                  0x3B67D4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_13                  0x3B67D8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_14                  0x3B67DC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_15                  0x3B67E0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_0                     0x3B6824
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_1                     0x3B6828
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_2                     0x3B682C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_3                     0x3B6830
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_4                     0x3B6834
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_5                     0x3B6838
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_6                     0x3B683C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_7                     0x3B6840
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_8                     0x3B6844
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_9                     0x3B6848
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_10                    0x3B684C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_11                    0x3B6850
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_12                    0x3B6854
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_13                    0x3B6858
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_14                    0x3B685C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_15                    0x3B6860
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_0                    0x3B6864
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_1                    0x3B6868
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_2                    0x3B686C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_3                    0x3B6870
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_4                    0x3B6874
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_5                    0x3B6878
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_6                    0x3B687C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_7                    0x3B6880
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_8                    0x3B6884
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_9                    0x3B6888
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_10                   0x3B688C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_11                   0x3B6890
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_12                   0x3B6894
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_13                   0x3B6898
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_14                   0x3B689C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_15                   0x3B68A0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_0                     0x3B68A4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_1                     0x3B68A8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_2                     0x3B68AC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_3                     0x3B68B0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_4                     0x3B68B4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_5                     0x3B68B8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_6                     0x3B68BC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_7                     0x3B68C0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_8                     0x3B68C4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_9                     0x3B68C8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_10                    0x3B68CC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_11                    0x3B68D0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_12                    0x3B68D4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_13                    0x3B68D8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_14                    0x3B68DC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_15                    0x3B68E0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_0                    0x3B68E4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_1                    0x3B68E8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_2                    0x3B68EC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_3                    0x3B68F0
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_4                    0x3B68F4
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_5                    0x3B68F8
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_6                    0x3B68FC
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_7                    0x3B6900
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_8                    0x3B6904
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_9                    0x3B6908
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_10                   0x3B690C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_11                   0x3B6910
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_12                   0x3B6914
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_13                   0x3B6918
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_14                   0x3B691C
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_15                   0x3B6920
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_0                    0x3B6924
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_1                    0x3B6928
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_2                    0x3B692C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_3                    0x3B6930
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_4                    0x3B6934
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_5                    0x3B6938
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_6                    0x3B693C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_7                    0x3B6940
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_8                    0x3B6944
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_9                    0x3B6948
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_10                   0x3B694C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_11                   0x3B6950
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_12                   0x3B6954
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_13                   0x3B6958
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_14                   0x3B695C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_15                   0x3B6960
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_0                   0x3B6964
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_1                   0x3B6968
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_2                   0x3B696C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_3                   0x3B6970
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_4                   0x3B6974
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_5                   0x3B6978
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_6                   0x3B697C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_7                   0x3B6980
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_8                   0x3B6984
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_9                   0x3B6988
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_10                  0x3B698C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_11                  0x3B6990
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_12                  0x3B6994
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_13                  0x3B6998
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_14                  0x3B699C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_15                  0x3B69A0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_0                    0x3B69A4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_1                    0x3B69A8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_2                    0x3B69AC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_3                    0x3B69B0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_4                    0x3B69B4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_5                    0x3B69B8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_6                    0x3B69BC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_7                    0x3B69C0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_8                    0x3B69C4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_9                    0x3B69C8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_10                   0x3B69CC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_11                   0x3B69D0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_12                   0x3B69D4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_13                   0x3B69D8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_14                   0x3B69DC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_15                   0x3B69E0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_0                   0x3B69E4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_1                   0x3B69E8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_2                   0x3B69EC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_3                   0x3B69F0
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_4                   0x3B69F4
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_5                   0x3B69F8
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_6                   0x3B69FC
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_7                   0x3B6A00
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_8                   0x3B6A04
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_9                   0x3B6A08
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_10                  0x3B6A0C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_11                  0x3B6A10
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_12                  0x3B6A14
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_13                  0x3B6A18
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_14                  0x3B6A1C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_15                  0x3B6A20
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_HIT_AW                            0x3B6A64
+
+#define mmNIF_RTR_CTRL_3_RANGE_SEC_HIT_AR                            0x3B6A68
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_HIT_AW                           0x3B6A6C
+
+#define mmNIF_RTR_CTRL_3_RANGE_PRIV_HIT_AR                           0x3B6A70
+
+#define mmNIF_RTR_CTRL_3_RGL_CFG                                     0x3B6B64
+
+#define mmNIF_RTR_CTRL_3_RGL_SHIFT                                   0x3B6B68
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_0                          0x3B6B6C
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_1                          0x3B6B70
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_2                          0x3B6B74
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_3                          0x3B6B78
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_4                          0x3B6B7C
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_5                          0x3B6B80
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_6                          0x3B6B84
+
+#define mmNIF_RTR_CTRL_3_RGL_EXPECTED_LAT_7                          0x3B6B88
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_0                                 0x3B6BAC
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_1                                 0x3B6BB0
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_2                                 0x3B6BB4
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_3                                 0x3B6BB8
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_4                                 0x3B6BBC
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_5                                 0x3B6BC0
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_6                                 0x3B6BC4
+
+#define mmNIF_RTR_CTRL_3_RGL_TOKEN_7                                 0x3B6BC8
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_0                               0x3B6BEC
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_1                               0x3B6BF0
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_2                               0x3B6BF4
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_3                               0x3B6BF8
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_4                               0x3B6BFC
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_5                               0x3B6C00
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_6                               0x3B6C04
+
+#define mmNIF_RTR_CTRL_3_RGL_BANK_ID_7                               0x3B6C08
+
+#define mmNIF_RTR_CTRL_3_RGL_WDT                                     0x3B6C2C
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM0_CH0_CTR_WRAP                    0x3B6C30
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM0_CH1_CTR_WRAP                    0x3B6C34
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM1_CH0_CTR_WRAP                    0x3B6C38
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM1_CH1_CTR_WRAP                    0x3B6C3C
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM2_CH0_CTR_WRAP                    0x3B6C40
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM2_CH1_CTR_WRAP                    0x3B6C44
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM3_CH0_CTR_WRAP                    0x3B6C48
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM3_CH1_CTR_WRAP                    0x3B6C4C
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM0_CH0_CTR_CNT                     0x3B6C50
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM0_CH1_CTR_CNT                     0x3B6C54
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM1_CH0_CTR_CNT                     0x3B6C58
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM1_CH1_CTR_CNT                     0x3B6C5C
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM2_CH0_CTR_CNT                     0x3B6C60
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM2_CH1_CTR_CNT                     0x3B6C64
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM3_CH0_CTR_CNT                     0x3B6C68
+
+#define mmNIF_RTR_CTRL_3_E2E_AR_HBM3_CH1_CTR_CNT                     0x3B6C6C
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM0_CH0_CTR_WRAP                    0x3B6C70
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM0_CH1_CTR_WRAP                    0x3B6C74
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM1_CH0_CTR_WRAP                    0x3B6C78
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM1_CH1_CTR_WRAP                    0x3B6C7C
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM2_CH0_CTR_WRAP                    0x3B6C80
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM2_CH1_CTR_WRAP                    0x3B6C84
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM3_CH0_CTR_WRAP                    0x3B6C88
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM3_CH1_CTR_WRAP                    0x3B6C8C
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM0_CH0_CTR_CNT                     0x3B6C90
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM0_CH1_CTR_CNT                     0x3B6C94
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM1_CH0_CTR_CNT                     0x3B6C98
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM1_CH1_CTR_CNT                     0x3B6C9C
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM2_CH0_CTR_CNT                     0x3B6CA0
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM2_CH1_CTR_CNT                     0x3B6CA4
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM3_CH0_CTR_CNT                     0x3B6CA8
+
+#define mmNIF_RTR_CTRL_3_E2E_AW_HBM3_CH1_CTR_CNT                     0x3B6CAC
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_PC_SEL_0                             0x3B6CB0
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_PC_SEL_1                             0x3B6CB4
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_PC_SEL_2                             0x3B6CB8
+
+#define mmNIF_RTR_CTRL_3_NL_HBM_PC_SEL_3                             0x3B6CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_3_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_4_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_4_regs.h
new file mode 100644
index 000000000000..543a98f81767
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_4_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_4_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_4_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_4 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_4_PERM_SEL                                    0x3C6108
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_0                               0x3C6114
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_1                               0x3C6118
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_2                               0x3C611C
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_3                               0x3C6120
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_4                               0x3C6124
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_5                               0x3C6128
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_6                               0x3C612C
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_7                               0x3C6130
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_8                               0x3C6134
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_9                               0x3C6138
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_10                              0x3C613C
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_11                              0x3C6140
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_12                              0x3C6144
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_13                              0x3C6148
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_14                              0x3C614C
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_15                              0x3C6150
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_16                              0x3C6154
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_17                              0x3C6158
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_18                              0x3C615C
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_19                              0x3C6160
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_20                              0x3C6164
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_21                              0x3C6168
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_22                              0x3C616C
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_23                              0x3C6170
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_24                              0x3C6174
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_25                              0x3C6178
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_26                              0x3C617C
+
+#define mmNIF_RTR_CTRL_4_HBM_POLY_H3_27                              0x3C6180
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_0                              0x3C6184
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_1                              0x3C6188
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_2                              0x3C618C
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_3                              0x3C6190
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_4                              0x3C6194
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_5                              0x3C6198
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_6                              0x3C619C
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_7                              0x3C61A0
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_8                              0x3C61A4
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_9                              0x3C61A8
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_10                             0x3C61AC
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_11                             0x3C61B0
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_12                             0x3C61B4
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_13                             0x3C61B8
+
+#define mmNIF_RTR_CTRL_4_SRAM_POLY_H3_14                             0x3C61BC
+
+#define mmNIF_RTR_CTRL_4_SCRAM_SRAM_EN                               0x3C626C
+
+#define mmNIF_RTR_CTRL_4_RL_HBM_EN                                   0x3C6274
+
+#define mmNIF_RTR_CTRL_4_RL_HBM_SAT                                  0x3C6278
+
+#define mmNIF_RTR_CTRL_4_RL_HBM_RST                                  0x3C627C
+
+#define mmNIF_RTR_CTRL_4_RL_HBM_TIMEOUT                              0x3C6280
+
+#define mmNIF_RTR_CTRL_4_SCRAM_HBM_EN                                0x3C6284
+
+#define mmNIF_RTR_CTRL_4_RL_PCI_EN                                   0x3C6288
+
+#define mmNIF_RTR_CTRL_4_RL_PCI_SAT                                  0x3C628C
+
+#define mmNIF_RTR_CTRL_4_RL_PCI_RST                                  0x3C6290
+
+#define mmNIF_RTR_CTRL_4_RL_PCI_TIMEOUT                              0x3C6294
+
+#define mmNIF_RTR_CTRL_4_RL_SRAM_EN                                  0x3C629C
+
+#define mmNIF_RTR_CTRL_4_RL_SRAM_SAT                                 0x3C62A0
+
+#define mmNIF_RTR_CTRL_4_RL_SRAM_RST                                 0x3C62A4
+
+#define mmNIF_RTR_CTRL_4_RL_SRAM_TIMEOUT                             0x3C62AC
+
+#define mmNIF_RTR_CTRL_4_RL_SRAM_RED                                 0x3C62B4
+
+#define mmNIF_RTR_CTRL_4_E2E_HBM_EN                                  0x3C62EC
+
+#define mmNIF_RTR_CTRL_4_E2E_PCI_EN                                  0x3C62F0
+
+#define mmNIF_RTR_CTRL_4_E2E_HBM_WR_SIZE                             0x3C62F4
+
+#define mmNIF_RTR_CTRL_4_E2E_PCI_WR_SIZE                             0x3C62F8
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_PCI_CTR_SET_EN                       0x3C6404
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_PCI_CTR_SET                          0x3C6408
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_PCI_CTR_WRAP                         0x3C640C
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_PCI_CTR_CNT                          0x3C6410
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM_CTR_SET_EN                       0x3C6414
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM_CTR_SET                          0x3C6418
+
+#define mmNIF_RTR_CTRL_4_E2E_HBM_RD_SIZE                             0x3C641C
+
+#define mmNIF_RTR_CTRL_4_E2E_PCI_RD_SIZE                             0x3C6420
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_PCI_CTR_SET_EN                       0x3C6424
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_PCI_CTR_SET                          0x3C6428
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_PCI_CTR_WRAP                         0x3C642C
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_PCI_CTR_CNT                          0x3C6430
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM_CTR_SET_EN                       0x3C6434
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM_CTR_SET                          0x3C6438
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_SEL_0                                0x3C6450
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_SEL_1                                0x3C6454
+
+#define mmNIF_RTR_CTRL_4_NON_LIN_EN                                  0x3C6480
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_BANK_0                              0x3C6500
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_BANK_1                              0x3C6504
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_BANK_2                              0x3C6508
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_BANK_3                              0x3C650C
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_BANK_4                              0x3C6510
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_0                            0x3C6514
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_1                            0x3C6520
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_2                            0x3C6524
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_3                            0x3C6528
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_4                            0x3C652C
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_5                            0x3C6530
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_6                            0x3C6534
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_7                            0x3C6538
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_8                            0x3C653C
+
+#define mmNIF_RTR_CTRL_4_NL_SRAM_OFFSET_9                            0x3C6540
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_0                             0x3C6550
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_1                             0x3C6554
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_2                             0x3C6558
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_3                             0x3C655C
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_4                             0x3C6560
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_5                             0x3C6564
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_6                             0x3C6568
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_7                             0x3C656C
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_8                             0x3C6570
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_9                             0x3C6574
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_10                            0x3C6578
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_11                            0x3C657C
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_12                            0x3C6580
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_13                            0x3C6584
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_14                            0x3C6588
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_15                            0x3C658C
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_16                            0x3C6590
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_17                            0x3C6594
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_OFFSET_18                            0x3C6598
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_0                     0x3C65E4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_1                     0x3C65E8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_2                     0x3C65EC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_3                     0x3C65F0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_4                     0x3C65F4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_5                     0x3C65F8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_6                     0x3C65FC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_7                     0x3C6600
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_8                     0x3C6604
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_9                     0x3C6608
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_10                    0x3C660C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_11                    0x3C6610
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_12                    0x3C6614
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_13                    0x3C6618
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_14                    0x3C661C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_15                    0x3C6620
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_0                    0x3C6624
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_1                    0x3C6628
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_2                    0x3C662C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_3                    0x3C6630
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_4                    0x3C6634
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_5                    0x3C6638
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_6                    0x3C663C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_7                    0x3C6640
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_8                    0x3C6644
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_9                    0x3C6648
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_10                   0x3C664C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_11                   0x3C6650
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_12                   0x3C6654
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_13                   0x3C6658
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_14                   0x3C665C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_15                   0x3C6660
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_0                     0x3C6664
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_1                     0x3C6668
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_2                     0x3C666C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_3                     0x3C6670
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_4                     0x3C6674
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_5                     0x3C6678
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_6                     0x3C667C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_7                     0x3C6680
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_8                     0x3C6684
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_9                     0x3C6688
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_10                    0x3C668C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_11                    0x3C6690
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_12                    0x3C6694
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_13                    0x3C6698
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_14                    0x3C669C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_15                    0x3C66A0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_0                    0x3C66A4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_1                    0x3C66A8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_2                    0x3C66AC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_3                    0x3C66B0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_4                    0x3C66B4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_5                    0x3C66B8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_6                    0x3C66BC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_7                    0x3C66C0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_8                    0x3C66C4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_9                    0x3C66C8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_10                   0x3C66CC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_11                   0x3C66D0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_12                   0x3C66D4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_13                   0x3C66D8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_14                   0x3C66DC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_15                   0x3C66E0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_0                    0x3C66E4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_1                    0x3C66E8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_2                    0x3C66EC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_3                    0x3C66F0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_4                    0x3C66F4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_5                    0x3C66F8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_6                    0x3C66FC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_7                    0x3C6700
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_8                    0x3C6704
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_9                    0x3C6708
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_10                   0x3C670C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_11                   0x3C6710
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_12                   0x3C6714
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_13                   0x3C6718
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_14                   0x3C671C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_15                   0x3C6720
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_0                   0x3C6724
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_1                   0x3C6728
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_2                   0x3C672C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_3                   0x3C6730
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_4                   0x3C6734
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_5                   0x3C6738
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_6                   0x3C673C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_7                   0x3C6740
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_8                   0x3C6744
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_9                   0x3C6748
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_10                  0x3C674C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_11                  0x3C6750
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_12                  0x3C6754
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_13                  0x3C6758
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_14                  0x3C675C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_15                  0x3C6760
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_0                    0x3C6764
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_1                    0x3C6768
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_2                    0x3C676C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_3                    0x3C6770
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_4                    0x3C6774
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_5                    0x3C6778
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_6                    0x3C677C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_7                    0x3C6780
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_8                    0x3C6784
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_9                    0x3C6788
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_10                   0x3C678C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_11                   0x3C6790
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_12                   0x3C6794
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_13                   0x3C6798
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_14                   0x3C679C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_15                   0x3C67A0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_0                   0x3C67A4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_1                   0x3C67A8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_2                   0x3C67AC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_3                   0x3C67B0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_4                   0x3C67B4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_5                   0x3C67B8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_6                   0x3C67BC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_7                   0x3C67C0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_8                   0x3C67C4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_9                   0x3C67C8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_10                  0x3C67CC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_11                  0x3C67D0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_12                  0x3C67D4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_13                  0x3C67D8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_14                  0x3C67DC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_15                  0x3C67E0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_0                     0x3C6824
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_1                     0x3C6828
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_2                     0x3C682C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_3                     0x3C6830
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_4                     0x3C6834
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_5                     0x3C6838
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_6                     0x3C683C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_7                     0x3C6840
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_8                     0x3C6844
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_9                     0x3C6848
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_10                    0x3C684C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_11                    0x3C6850
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_12                    0x3C6854
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_13                    0x3C6858
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_14                    0x3C685C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_15                    0x3C6860
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_0                    0x3C6864
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_1                    0x3C6868
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_2                    0x3C686C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_3                    0x3C6870
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_4                    0x3C6874
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_5                    0x3C6878
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_6                    0x3C687C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_7                    0x3C6880
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_8                    0x3C6884
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_9                    0x3C6888
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_10                   0x3C688C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_11                   0x3C6890
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_12                   0x3C6894
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_13                   0x3C6898
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_14                   0x3C689C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_15                   0x3C68A0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_0                     0x3C68A4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_1                     0x3C68A8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_2                     0x3C68AC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_3                     0x3C68B0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_4                     0x3C68B4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_5                     0x3C68B8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_6                     0x3C68BC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_7                     0x3C68C0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_8                     0x3C68C4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_9                     0x3C68C8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_10                    0x3C68CC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_11                    0x3C68D0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_12                    0x3C68D4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_13                    0x3C68D8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_14                    0x3C68DC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_15                    0x3C68E0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_0                    0x3C68E4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_1                    0x3C68E8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_2                    0x3C68EC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_3                    0x3C68F0
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_4                    0x3C68F4
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_5                    0x3C68F8
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_6                    0x3C68FC
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_7                    0x3C6900
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_8                    0x3C6904
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_9                    0x3C6908
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_10                   0x3C690C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_11                   0x3C6910
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_12                   0x3C6914
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_13                   0x3C6918
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_14                   0x3C691C
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_15                   0x3C6920
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_0                    0x3C6924
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_1                    0x3C6928
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_2                    0x3C692C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_3                    0x3C6930
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_4                    0x3C6934
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_5                    0x3C6938
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_6                    0x3C693C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_7                    0x3C6940
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_8                    0x3C6944
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_9                    0x3C6948
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_10                   0x3C694C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_11                   0x3C6950
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_12                   0x3C6954
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_13                   0x3C6958
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_14                   0x3C695C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_15                   0x3C6960
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_0                   0x3C6964
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_1                   0x3C6968
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_2                   0x3C696C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_3                   0x3C6970
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_4                   0x3C6974
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_5                   0x3C6978
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_6                   0x3C697C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_7                   0x3C6980
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_8                   0x3C6984
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_9                   0x3C6988
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_10                  0x3C698C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_11                  0x3C6990
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_12                  0x3C6994
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_13                  0x3C6998
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_14                  0x3C699C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_15                  0x3C69A0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_0                    0x3C69A4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_1                    0x3C69A8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_2                    0x3C69AC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_3                    0x3C69B0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_4                    0x3C69B4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_5                    0x3C69B8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_6                    0x3C69BC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_7                    0x3C69C0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_8                    0x3C69C4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_9                    0x3C69C8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_10                   0x3C69CC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_11                   0x3C69D0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_12                   0x3C69D4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_13                   0x3C69D8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_14                   0x3C69DC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_15                   0x3C69E0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_0                   0x3C69E4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_1                   0x3C69E8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_2                   0x3C69EC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_3                   0x3C69F0
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_4                   0x3C69F4
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_5                   0x3C69F8
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_6                   0x3C69FC
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_7                   0x3C6A00
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_8                   0x3C6A04
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_9                   0x3C6A08
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_10                  0x3C6A0C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_11                  0x3C6A10
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_12                  0x3C6A14
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_13                  0x3C6A18
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_14                  0x3C6A1C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_15                  0x3C6A20
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_HIT_AW                            0x3C6A64
+
+#define mmNIF_RTR_CTRL_4_RANGE_SEC_HIT_AR                            0x3C6A68
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_HIT_AW                           0x3C6A6C
+
+#define mmNIF_RTR_CTRL_4_RANGE_PRIV_HIT_AR                           0x3C6A70
+
+#define mmNIF_RTR_CTRL_4_RGL_CFG                                     0x3C6B64
+
+#define mmNIF_RTR_CTRL_4_RGL_SHIFT                                   0x3C6B68
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_0                          0x3C6B6C
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_1                          0x3C6B70
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_2                          0x3C6B74
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_3                          0x3C6B78
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_4                          0x3C6B7C
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_5                          0x3C6B80
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_6                          0x3C6B84
+
+#define mmNIF_RTR_CTRL_4_RGL_EXPECTED_LAT_7                          0x3C6B88
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_0                                 0x3C6BAC
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_1                                 0x3C6BB0
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_2                                 0x3C6BB4
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_3                                 0x3C6BB8
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_4                                 0x3C6BBC
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_5                                 0x3C6BC0
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_6                                 0x3C6BC4
+
+#define mmNIF_RTR_CTRL_4_RGL_TOKEN_7                                 0x3C6BC8
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_0                               0x3C6BEC
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_1                               0x3C6BF0
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_2                               0x3C6BF4
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_3                               0x3C6BF8
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_4                               0x3C6BFC
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_5                               0x3C6C00
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_6                               0x3C6C04
+
+#define mmNIF_RTR_CTRL_4_RGL_BANK_ID_7                               0x3C6C08
+
+#define mmNIF_RTR_CTRL_4_RGL_WDT                                     0x3C6C2C
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM0_CH0_CTR_WRAP                    0x3C6C30
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM0_CH1_CTR_WRAP                    0x3C6C34
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM1_CH0_CTR_WRAP                    0x3C6C38
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM1_CH1_CTR_WRAP                    0x3C6C3C
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM2_CH0_CTR_WRAP                    0x3C6C40
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM2_CH1_CTR_WRAP                    0x3C6C44
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM3_CH0_CTR_WRAP                    0x3C6C48
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM3_CH1_CTR_WRAP                    0x3C6C4C
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM0_CH0_CTR_CNT                     0x3C6C50
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM0_CH1_CTR_CNT                     0x3C6C54
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM1_CH0_CTR_CNT                     0x3C6C58
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM1_CH1_CTR_CNT                     0x3C6C5C
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM2_CH0_CTR_CNT                     0x3C6C60
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM2_CH1_CTR_CNT                     0x3C6C64
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM3_CH0_CTR_CNT                     0x3C6C68
+
+#define mmNIF_RTR_CTRL_4_E2E_AR_HBM3_CH1_CTR_CNT                     0x3C6C6C
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM0_CH0_CTR_WRAP                    0x3C6C70
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM0_CH1_CTR_WRAP                    0x3C6C74
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM1_CH0_CTR_WRAP                    0x3C6C78
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM1_CH1_CTR_WRAP                    0x3C6C7C
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM2_CH0_CTR_WRAP                    0x3C6C80
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM2_CH1_CTR_WRAP                    0x3C6C84
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM3_CH0_CTR_WRAP                    0x3C6C88
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM3_CH1_CTR_WRAP                    0x3C6C8C
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM0_CH0_CTR_CNT                     0x3C6C90
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM0_CH1_CTR_CNT                     0x3C6C94
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM1_CH0_CTR_CNT                     0x3C6C98
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM1_CH1_CTR_CNT                     0x3C6C9C
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM2_CH0_CTR_CNT                     0x3C6CA0
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM2_CH1_CTR_CNT                     0x3C6CA4
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM3_CH0_CTR_CNT                     0x3C6CA8
+
+#define mmNIF_RTR_CTRL_4_E2E_AW_HBM3_CH1_CTR_CNT                     0x3C6CAC
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_PC_SEL_0                             0x3C6CB0
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_PC_SEL_1                             0x3C6CB4
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_PC_SEL_2                             0x3C6CB8
+
+#define mmNIF_RTR_CTRL_4_NL_HBM_PC_SEL_3                             0x3C6CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_4_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_5_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_5_regs.h
new file mode 100644
index 000000000000..95486b7ddf1d
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_5_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_5_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_5_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_5 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_5_PERM_SEL                                    0x3D6108
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_0                               0x3D6114
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_1                               0x3D6118
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_2                               0x3D611C
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_3                               0x3D6120
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_4                               0x3D6124
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_5                               0x3D6128
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_6                               0x3D612C
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_7                               0x3D6130
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_8                               0x3D6134
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_9                               0x3D6138
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_10                              0x3D613C
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_11                              0x3D6140
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_12                              0x3D6144
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_13                              0x3D6148
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_14                              0x3D614C
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_15                              0x3D6150
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_16                              0x3D6154
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_17                              0x3D6158
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_18                              0x3D615C
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_19                              0x3D6160
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_20                              0x3D6164
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_21                              0x3D6168
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_22                              0x3D616C
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_23                              0x3D6170
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_24                              0x3D6174
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_25                              0x3D6178
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_26                              0x3D617C
+
+#define mmNIF_RTR_CTRL_5_HBM_POLY_H3_27                              0x3D6180
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_0                              0x3D6184
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_1                              0x3D6188
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_2                              0x3D618C
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_3                              0x3D6190
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_4                              0x3D6194
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_5                              0x3D6198
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_6                              0x3D619C
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_7                              0x3D61A0
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_8                              0x3D61A4
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_9                              0x3D61A8
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_10                             0x3D61AC
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_11                             0x3D61B0
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_12                             0x3D61B4
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_13                             0x3D61B8
+
+#define mmNIF_RTR_CTRL_5_SRAM_POLY_H3_14                             0x3D61BC
+
+#define mmNIF_RTR_CTRL_5_SCRAM_SRAM_EN                               0x3D626C
+
+#define mmNIF_RTR_CTRL_5_RL_HBM_EN                                   0x3D6274
+
+#define mmNIF_RTR_CTRL_5_RL_HBM_SAT                                  0x3D6278
+
+#define mmNIF_RTR_CTRL_5_RL_HBM_RST                                  0x3D627C
+
+#define mmNIF_RTR_CTRL_5_RL_HBM_TIMEOUT                              0x3D6280
+
+#define mmNIF_RTR_CTRL_5_SCRAM_HBM_EN                                0x3D6284
+
+#define mmNIF_RTR_CTRL_5_RL_PCI_EN                                   0x3D6288
+
+#define mmNIF_RTR_CTRL_5_RL_PCI_SAT                                  0x3D628C
+
+#define mmNIF_RTR_CTRL_5_RL_PCI_RST                                  0x3D6290
+
+#define mmNIF_RTR_CTRL_5_RL_PCI_TIMEOUT                              0x3D6294
+
+#define mmNIF_RTR_CTRL_5_RL_SRAM_EN                                  0x3D629C
+
+#define mmNIF_RTR_CTRL_5_RL_SRAM_SAT                                 0x3D62A0
+
+#define mmNIF_RTR_CTRL_5_RL_SRAM_RST                                 0x3D62A4
+
+#define mmNIF_RTR_CTRL_5_RL_SRAM_TIMEOUT                             0x3D62AC
+
+#define mmNIF_RTR_CTRL_5_RL_SRAM_RED                                 0x3D62B4
+
+#define mmNIF_RTR_CTRL_5_E2E_HBM_EN                                  0x3D62EC
+
+#define mmNIF_RTR_CTRL_5_E2E_PCI_EN                                  0x3D62F0
+
+#define mmNIF_RTR_CTRL_5_E2E_HBM_WR_SIZE                             0x3D62F4
+
+#define mmNIF_RTR_CTRL_5_E2E_PCI_WR_SIZE                             0x3D62F8
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_PCI_CTR_SET_EN                       0x3D6404
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_PCI_CTR_SET                          0x3D6408
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_PCI_CTR_WRAP                         0x3D640C
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_PCI_CTR_CNT                          0x3D6410
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM_CTR_SET_EN                       0x3D6414
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM_CTR_SET                          0x3D6418
+
+#define mmNIF_RTR_CTRL_5_E2E_HBM_RD_SIZE                             0x3D641C
+
+#define mmNIF_RTR_CTRL_5_E2E_PCI_RD_SIZE                             0x3D6420
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_PCI_CTR_SET_EN                       0x3D6424
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_PCI_CTR_SET                          0x3D6428
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_PCI_CTR_WRAP                         0x3D642C
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_PCI_CTR_CNT                          0x3D6430
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM_CTR_SET_EN                       0x3D6434
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM_CTR_SET                          0x3D6438
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_SEL_0                                0x3D6450
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_SEL_1                                0x3D6454
+
+#define mmNIF_RTR_CTRL_5_NON_LIN_EN                                  0x3D6480
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_BANK_0                              0x3D6500
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_BANK_1                              0x3D6504
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_BANK_2                              0x3D6508
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_BANK_3                              0x3D650C
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_BANK_4                              0x3D6510
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_0                            0x3D6514
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_1                            0x3D6520
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_2                            0x3D6524
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_3                            0x3D6528
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_4                            0x3D652C
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_5                            0x3D6530
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_6                            0x3D6534
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_7                            0x3D6538
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_8                            0x3D653C
+
+#define mmNIF_RTR_CTRL_5_NL_SRAM_OFFSET_9                            0x3D6540
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_0                             0x3D6550
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_1                             0x3D6554
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_2                             0x3D6558
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_3                             0x3D655C
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_4                             0x3D6560
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_5                             0x3D6564
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_6                             0x3D6568
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_7                             0x3D656C
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_8                             0x3D6570
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_9                             0x3D6574
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_10                            0x3D6578
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_11                            0x3D657C
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_12                            0x3D6580
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_13                            0x3D6584
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_14                            0x3D6588
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_15                            0x3D658C
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_16                            0x3D6590
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_17                            0x3D6594
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_OFFSET_18                            0x3D6598
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_0                     0x3D65E4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_1                     0x3D65E8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_2                     0x3D65EC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_3                     0x3D65F0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_4                     0x3D65F4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_5                     0x3D65F8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_6                     0x3D65FC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_7                     0x3D6600
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_8                     0x3D6604
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_9                     0x3D6608
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_10                    0x3D660C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_11                    0x3D6610
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_12                    0x3D6614
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_13                    0x3D6618
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_14                    0x3D661C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_15                    0x3D6620
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_0                    0x3D6624
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_1                    0x3D6628
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_2                    0x3D662C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_3                    0x3D6630
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_4                    0x3D6634
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_5                    0x3D6638
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_6                    0x3D663C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_7                    0x3D6640
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_8                    0x3D6644
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_9                    0x3D6648
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_10                   0x3D664C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_11                   0x3D6650
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_12                   0x3D6654
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_13                   0x3D6658
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_14                   0x3D665C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_15                   0x3D6660
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_0                     0x3D6664
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_1                     0x3D6668
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_2                     0x3D666C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_3                     0x3D6670
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_4                     0x3D6674
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_5                     0x3D6678
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_6                     0x3D667C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_7                     0x3D6680
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_8                     0x3D6684
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_9                     0x3D6688
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_10                    0x3D668C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_11                    0x3D6690
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_12                    0x3D6694
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_13                    0x3D6698
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_14                    0x3D669C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_15                    0x3D66A0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_0                    0x3D66A4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_1                    0x3D66A8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_2                    0x3D66AC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_3                    0x3D66B0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_4                    0x3D66B4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_5                    0x3D66B8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_6                    0x3D66BC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_7                    0x3D66C0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_8                    0x3D66C4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_9                    0x3D66C8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_10                   0x3D66CC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_11                   0x3D66D0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_12                   0x3D66D4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_13                   0x3D66D8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_14                   0x3D66DC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_15                   0x3D66E0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_0                    0x3D66E4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_1                    0x3D66E8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_2                    0x3D66EC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_3                    0x3D66F0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_4                    0x3D66F4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_5                    0x3D66F8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_6                    0x3D66FC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_7                    0x3D6700
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_8                    0x3D6704
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_9                    0x3D6708
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_10                   0x3D670C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_11                   0x3D6710
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_12                   0x3D6714
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_13                   0x3D6718
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_14                   0x3D671C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_15                   0x3D6720
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_0                   0x3D6724
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_1                   0x3D6728
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_2                   0x3D672C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_3                   0x3D6730
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_4                   0x3D6734
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_5                   0x3D6738
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_6                   0x3D673C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_7                   0x3D6740
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_8                   0x3D6744
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_9                   0x3D6748
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_10                  0x3D674C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_11                  0x3D6750
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_12                  0x3D6754
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_13                  0x3D6758
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_14                  0x3D675C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_15                  0x3D6760
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_0                    0x3D6764
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_1                    0x3D6768
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_2                    0x3D676C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_3                    0x3D6770
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_4                    0x3D6774
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_5                    0x3D6778
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_6                    0x3D677C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_7                    0x3D6780
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_8                    0x3D6784
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_9                    0x3D6788
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_10                   0x3D678C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_11                   0x3D6790
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_12                   0x3D6794
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_13                   0x3D6798
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_14                   0x3D679C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_15                   0x3D67A0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_0                   0x3D67A4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_1                   0x3D67A8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_2                   0x3D67AC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_3                   0x3D67B0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_4                   0x3D67B4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_5                   0x3D67B8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_6                   0x3D67BC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_7                   0x3D67C0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_8                   0x3D67C4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_9                   0x3D67C8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_10                  0x3D67CC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_11                  0x3D67D0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_12                  0x3D67D4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_13                  0x3D67D8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_14                  0x3D67DC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_15                  0x3D67E0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_0                     0x3D6824
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_1                     0x3D6828
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_2                     0x3D682C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_3                     0x3D6830
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_4                     0x3D6834
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_5                     0x3D6838
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_6                     0x3D683C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_7                     0x3D6840
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_8                     0x3D6844
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_9                     0x3D6848
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_10                    0x3D684C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_11                    0x3D6850
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_12                    0x3D6854
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_13                    0x3D6858
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_14                    0x3D685C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_15                    0x3D6860
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_0                    0x3D6864
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_1                    0x3D6868
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_2                    0x3D686C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_3                    0x3D6870
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_4                    0x3D6874
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_5                    0x3D6878
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_6                    0x3D687C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_7                    0x3D6880
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_8                    0x3D6884
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_9                    0x3D6888
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_10                   0x3D688C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_11                   0x3D6890
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_12                   0x3D6894
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_13                   0x3D6898
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_14                   0x3D689C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_15                   0x3D68A0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_0                     0x3D68A4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_1                     0x3D68A8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_2                     0x3D68AC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_3                     0x3D68B0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_4                     0x3D68B4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_5                     0x3D68B8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_6                     0x3D68BC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_7                     0x3D68C0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_8                     0x3D68C4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_9                     0x3D68C8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_10                    0x3D68CC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_11                    0x3D68D0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_12                    0x3D68D4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_13                    0x3D68D8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_14                    0x3D68DC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_15                    0x3D68E0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_0                    0x3D68E4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_1                    0x3D68E8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_2                    0x3D68EC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_3                    0x3D68F0
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_4                    0x3D68F4
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_5                    0x3D68F8
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_6                    0x3D68FC
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_7                    0x3D6900
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_8                    0x3D6904
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_9                    0x3D6908
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_10                   0x3D690C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_11                   0x3D6910
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_12                   0x3D6914
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_13                   0x3D6918
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_14                   0x3D691C
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_15                   0x3D6920
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_0                    0x3D6924
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_1                    0x3D6928
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_2                    0x3D692C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_3                    0x3D6930
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_4                    0x3D6934
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_5                    0x3D6938
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_6                    0x3D693C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_7                    0x3D6940
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_8                    0x3D6944
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_9                    0x3D6948
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_10                   0x3D694C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_11                   0x3D6950
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_12                   0x3D6954
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_13                   0x3D6958
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_14                   0x3D695C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_15                   0x3D6960
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_0                   0x3D6964
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_1                   0x3D6968
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_2                   0x3D696C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_3                   0x3D6970
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_4                   0x3D6974
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_5                   0x3D6978
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_6                   0x3D697C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_7                   0x3D6980
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_8                   0x3D6984
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_9                   0x3D6988
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_10                  0x3D698C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_11                  0x3D6990
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_12                  0x3D6994
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_13                  0x3D6998
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_14                  0x3D699C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_15                  0x3D69A0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_0                    0x3D69A4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_1                    0x3D69A8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_2                    0x3D69AC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_3                    0x3D69B0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_4                    0x3D69B4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_5                    0x3D69B8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_6                    0x3D69BC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_7                    0x3D69C0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_8                    0x3D69C4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_9                    0x3D69C8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_10                   0x3D69CC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_11                   0x3D69D0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_12                   0x3D69D4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_13                   0x3D69D8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_14                   0x3D69DC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_15                   0x3D69E0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_0                   0x3D69E4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_1                   0x3D69E8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_2                   0x3D69EC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_3                   0x3D69F0
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_4                   0x3D69F4
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_5                   0x3D69F8
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_6                   0x3D69FC
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_7                   0x3D6A00
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_8                   0x3D6A04
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_9                   0x3D6A08
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_10                  0x3D6A0C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_11                  0x3D6A10
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_12                  0x3D6A14
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_13                  0x3D6A18
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_14                  0x3D6A1C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_15                  0x3D6A20
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_HIT_AW                            0x3D6A64
+
+#define mmNIF_RTR_CTRL_5_RANGE_SEC_HIT_AR                            0x3D6A68
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_HIT_AW                           0x3D6A6C
+
+#define mmNIF_RTR_CTRL_5_RANGE_PRIV_HIT_AR                           0x3D6A70
+
+#define mmNIF_RTR_CTRL_5_RGL_CFG                                     0x3D6B64
+
+#define mmNIF_RTR_CTRL_5_RGL_SHIFT                                   0x3D6B68
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_0                          0x3D6B6C
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_1                          0x3D6B70
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_2                          0x3D6B74
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_3                          0x3D6B78
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_4                          0x3D6B7C
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_5                          0x3D6B80
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_6                          0x3D6B84
+
+#define mmNIF_RTR_CTRL_5_RGL_EXPECTED_LAT_7                          0x3D6B88
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_0                                 0x3D6BAC
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_1                                 0x3D6BB0
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_2                                 0x3D6BB4
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_3                                 0x3D6BB8
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_4                                 0x3D6BBC
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_5                                 0x3D6BC0
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_6                                 0x3D6BC4
+
+#define mmNIF_RTR_CTRL_5_RGL_TOKEN_7                                 0x3D6BC8
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_0                               0x3D6BEC
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_1                               0x3D6BF0
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_2                               0x3D6BF4
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_3                               0x3D6BF8
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_4                               0x3D6BFC
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_5                               0x3D6C00
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_6                               0x3D6C04
+
+#define mmNIF_RTR_CTRL_5_RGL_BANK_ID_7                               0x3D6C08
+
+#define mmNIF_RTR_CTRL_5_RGL_WDT                                     0x3D6C2C
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM0_CH0_CTR_WRAP                    0x3D6C30
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM0_CH1_CTR_WRAP                    0x3D6C34
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM1_CH0_CTR_WRAP                    0x3D6C38
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM1_CH1_CTR_WRAP                    0x3D6C3C
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM2_CH0_CTR_WRAP                    0x3D6C40
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM2_CH1_CTR_WRAP                    0x3D6C44
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM3_CH0_CTR_WRAP                    0x3D6C48
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM3_CH1_CTR_WRAP                    0x3D6C4C
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM0_CH0_CTR_CNT                     0x3D6C50
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM0_CH1_CTR_CNT                     0x3D6C54
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM1_CH0_CTR_CNT                     0x3D6C58
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM1_CH1_CTR_CNT                     0x3D6C5C
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM2_CH0_CTR_CNT                     0x3D6C60
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM2_CH1_CTR_CNT                     0x3D6C64
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM3_CH0_CTR_CNT                     0x3D6C68
+
+#define mmNIF_RTR_CTRL_5_E2E_AR_HBM3_CH1_CTR_CNT                     0x3D6C6C
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM0_CH0_CTR_WRAP                    0x3D6C70
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM0_CH1_CTR_WRAP                    0x3D6C74
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM1_CH0_CTR_WRAP                    0x3D6C78
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM1_CH1_CTR_WRAP                    0x3D6C7C
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM2_CH0_CTR_WRAP                    0x3D6C80
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM2_CH1_CTR_WRAP                    0x3D6C84
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM3_CH0_CTR_WRAP                    0x3D6C88
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM3_CH1_CTR_WRAP                    0x3D6C8C
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM0_CH0_CTR_CNT                     0x3D6C90
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM0_CH1_CTR_CNT                     0x3D6C94
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM1_CH0_CTR_CNT                     0x3D6C98
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM1_CH1_CTR_CNT                     0x3D6C9C
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM2_CH0_CTR_CNT                     0x3D6CA0
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM2_CH1_CTR_CNT                     0x3D6CA4
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM3_CH0_CTR_CNT                     0x3D6CA8
+
+#define mmNIF_RTR_CTRL_5_E2E_AW_HBM3_CH1_CTR_CNT                     0x3D6CAC
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_PC_SEL_0                             0x3D6CB0
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_PC_SEL_1                             0x3D6CB4
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_PC_SEL_2                             0x3D6CB8
+
+#define mmNIF_RTR_CTRL_5_NL_HBM_PC_SEL_3                             0x3D6CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_5_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_6_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_6_regs.h
new file mode 100644
index 000000000000..b79c59887b21
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_6_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_6_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_6_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_6 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_6_PERM_SEL                                    0x3E6108
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_0                               0x3E6114
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_1                               0x3E6118
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_2                               0x3E611C
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_3                               0x3E6120
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_4                               0x3E6124
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_5                               0x3E6128
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_6                               0x3E612C
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_7                               0x3E6130
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_8                               0x3E6134
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_9                               0x3E6138
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_10                              0x3E613C
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_11                              0x3E6140
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_12                              0x3E6144
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_13                              0x3E6148
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_14                              0x3E614C
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_15                              0x3E6150
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_16                              0x3E6154
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_17                              0x3E6158
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_18                              0x3E615C
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_19                              0x3E6160
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_20                              0x3E6164
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_21                              0x3E6168
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_22                              0x3E616C
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_23                              0x3E6170
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_24                              0x3E6174
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_25                              0x3E6178
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_26                              0x3E617C
+
+#define mmNIF_RTR_CTRL_6_HBM_POLY_H3_27                              0x3E6180
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_0                              0x3E6184
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_1                              0x3E6188
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_2                              0x3E618C
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_3                              0x3E6190
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_4                              0x3E6194
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_5                              0x3E6198
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_6                              0x3E619C
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_7                              0x3E61A0
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_8                              0x3E61A4
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_9                              0x3E61A8
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_10                             0x3E61AC
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_11                             0x3E61B0
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_12                             0x3E61B4
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_13                             0x3E61B8
+
+#define mmNIF_RTR_CTRL_6_SRAM_POLY_H3_14                             0x3E61BC
+
+#define mmNIF_RTR_CTRL_6_SCRAM_SRAM_EN                               0x3E626C
+
+#define mmNIF_RTR_CTRL_6_RL_HBM_EN                                   0x3E6274
+
+#define mmNIF_RTR_CTRL_6_RL_HBM_SAT                                  0x3E6278
+
+#define mmNIF_RTR_CTRL_6_RL_HBM_RST                                  0x3E627C
+
+#define mmNIF_RTR_CTRL_6_RL_HBM_TIMEOUT                              0x3E6280
+
+#define mmNIF_RTR_CTRL_6_SCRAM_HBM_EN                                0x3E6284
+
+#define mmNIF_RTR_CTRL_6_RL_PCI_EN                                   0x3E6288
+
+#define mmNIF_RTR_CTRL_6_RL_PCI_SAT                                  0x3E628C
+
+#define mmNIF_RTR_CTRL_6_RL_PCI_RST                                  0x3E6290
+
+#define mmNIF_RTR_CTRL_6_RL_PCI_TIMEOUT                              0x3E6294
+
+#define mmNIF_RTR_CTRL_6_RL_SRAM_EN                                  0x3E629C
+
+#define mmNIF_RTR_CTRL_6_RL_SRAM_SAT                                 0x3E62A0
+
+#define mmNIF_RTR_CTRL_6_RL_SRAM_RST                                 0x3E62A4
+
+#define mmNIF_RTR_CTRL_6_RL_SRAM_TIMEOUT                             0x3E62AC
+
+#define mmNIF_RTR_CTRL_6_RL_SRAM_RED                                 0x3E62B4
+
+#define mmNIF_RTR_CTRL_6_E2E_HBM_EN                                  0x3E62EC
+
+#define mmNIF_RTR_CTRL_6_E2E_PCI_EN                                  0x3E62F0
+
+#define mmNIF_RTR_CTRL_6_E2E_HBM_WR_SIZE                             0x3E62F4
+
+#define mmNIF_RTR_CTRL_6_E2E_PCI_WR_SIZE                             0x3E62F8
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_PCI_CTR_SET_EN                       0x3E6404
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_PCI_CTR_SET                          0x3E6408
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_PCI_CTR_WRAP                         0x3E640C
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_PCI_CTR_CNT                          0x3E6410
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM_CTR_SET_EN                       0x3E6414
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM_CTR_SET                          0x3E6418
+
+#define mmNIF_RTR_CTRL_6_E2E_HBM_RD_SIZE                             0x3E641C
+
+#define mmNIF_RTR_CTRL_6_E2E_PCI_RD_SIZE                             0x3E6420
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_PCI_CTR_SET_EN                       0x3E6424
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_PCI_CTR_SET                          0x3E6428
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_PCI_CTR_WRAP                         0x3E642C
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_PCI_CTR_CNT                          0x3E6430
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM_CTR_SET_EN                       0x3E6434
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM_CTR_SET                          0x3E6438
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_SEL_0                                0x3E6450
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_SEL_1                                0x3E6454
+
+#define mmNIF_RTR_CTRL_6_NON_LIN_EN                                  0x3E6480
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_BANK_0                              0x3E6500
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_BANK_1                              0x3E6504
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_BANK_2                              0x3E6508
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_BANK_3                              0x3E650C
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_BANK_4                              0x3E6510
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_0                            0x3E6514
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_1                            0x3E6520
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_2                            0x3E6524
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_3                            0x3E6528
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_4                            0x3E652C
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_5                            0x3E6530
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_6                            0x3E6534
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_7                            0x3E6538
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_8                            0x3E653C
+
+#define mmNIF_RTR_CTRL_6_NL_SRAM_OFFSET_9                            0x3E6540
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_0                             0x3E6550
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_1                             0x3E6554
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_2                             0x3E6558
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_3                             0x3E655C
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_4                             0x3E6560
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_5                             0x3E6564
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_6                             0x3E6568
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_7                             0x3E656C
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_8                             0x3E6570
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_9                             0x3E6574
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_10                            0x3E6578
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_11                            0x3E657C
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_12                            0x3E6580
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_13                            0x3E6584
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_14                            0x3E6588
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_15                            0x3E658C
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_16                            0x3E6590
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_17                            0x3E6594
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_OFFSET_18                            0x3E6598
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_0                     0x3E65E4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_1                     0x3E65E8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_2                     0x3E65EC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_3                     0x3E65F0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_4                     0x3E65F4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_5                     0x3E65F8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_6                     0x3E65FC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_7                     0x3E6600
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_8                     0x3E6604
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_9                     0x3E6608
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_10                    0x3E660C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_11                    0x3E6610
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_12                    0x3E6614
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_13                    0x3E6618
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_14                    0x3E661C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_15                    0x3E6620
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_0                    0x3E6624
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_1                    0x3E6628
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_2                    0x3E662C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_3                    0x3E6630
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_4                    0x3E6634
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_5                    0x3E6638
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_6                    0x3E663C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_7                    0x3E6640
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_8                    0x3E6644
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_9                    0x3E6648
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_10                   0x3E664C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_11                   0x3E6650
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_12                   0x3E6654
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_13                   0x3E6658
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_14                   0x3E665C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_15                   0x3E6660
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_0                     0x3E6664
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_1                     0x3E6668
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_2                     0x3E666C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_3                     0x3E6670
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_4                     0x3E6674
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_5                     0x3E6678
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_6                     0x3E667C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_7                     0x3E6680
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_8                     0x3E6684
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_9                     0x3E6688
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_10                    0x3E668C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_11                    0x3E6690
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_12                    0x3E6694
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_13                    0x3E6698
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_14                    0x3E669C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_15                    0x3E66A0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_0                    0x3E66A4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_1                    0x3E66A8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_2                    0x3E66AC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_3                    0x3E66B0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_4                    0x3E66B4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_5                    0x3E66B8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_6                    0x3E66BC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_7                    0x3E66C0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_8                    0x3E66C4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_9                    0x3E66C8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_10                   0x3E66CC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_11                   0x3E66D0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_12                   0x3E66D4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_13                   0x3E66D8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_14                   0x3E66DC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_15                   0x3E66E0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_0                    0x3E66E4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_1                    0x3E66E8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_2                    0x3E66EC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_3                    0x3E66F0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_4                    0x3E66F4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_5                    0x3E66F8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_6                    0x3E66FC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_7                    0x3E6700
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_8                    0x3E6704
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_9                    0x3E6708
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_10                   0x3E670C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_11                   0x3E6710
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_12                   0x3E6714
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_13                   0x3E6718
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_14                   0x3E671C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_15                   0x3E6720
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_0                   0x3E6724
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_1                   0x3E6728
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_2                   0x3E672C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_3                   0x3E6730
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_4                   0x3E6734
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_5                   0x3E6738
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_6                   0x3E673C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_7                   0x3E6740
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_8                   0x3E6744
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_9                   0x3E6748
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_10                  0x3E674C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_11                  0x3E6750
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_12                  0x3E6754
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_13                  0x3E6758
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_14                  0x3E675C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_15                  0x3E6760
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_0                    0x3E6764
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_1                    0x3E6768
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_2                    0x3E676C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_3                    0x3E6770
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_4                    0x3E6774
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_5                    0x3E6778
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_6                    0x3E677C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_7                    0x3E6780
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_8                    0x3E6784
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_9                    0x3E6788
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_10                   0x3E678C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_11                   0x3E6790
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_12                   0x3E6794
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_13                   0x3E6798
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_14                   0x3E679C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_15                   0x3E67A0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_0                   0x3E67A4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_1                   0x3E67A8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_2                   0x3E67AC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_3                   0x3E67B0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_4                   0x3E67B4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_5                   0x3E67B8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_6                   0x3E67BC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_7                   0x3E67C0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_8                   0x3E67C4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_9                   0x3E67C8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_10                  0x3E67CC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_11                  0x3E67D0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_12                  0x3E67D4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_13                  0x3E67D8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_14                  0x3E67DC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_15                  0x3E67E0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_0                     0x3E6824
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_1                     0x3E6828
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_2                     0x3E682C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_3                     0x3E6830
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_4                     0x3E6834
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_5                     0x3E6838
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_6                     0x3E683C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_7                     0x3E6840
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_8                     0x3E6844
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_9                     0x3E6848
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_10                    0x3E684C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_11                    0x3E6850
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_12                    0x3E6854
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_13                    0x3E6858
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_14                    0x3E685C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_15                    0x3E6860
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_0                    0x3E6864
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_1                    0x3E6868
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_2                    0x3E686C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_3                    0x3E6870
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_4                    0x3E6874
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_5                    0x3E6878
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_6                    0x3E687C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_7                    0x3E6880
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_8                    0x3E6884
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_9                    0x3E6888
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_10                   0x3E688C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_11                   0x3E6890
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_12                   0x3E6894
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_13                   0x3E6898
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_14                   0x3E689C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_15                   0x3E68A0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_0                     0x3E68A4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_1                     0x3E68A8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_2                     0x3E68AC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_3                     0x3E68B0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_4                     0x3E68B4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_5                     0x3E68B8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_6                     0x3E68BC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_7                     0x3E68C0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_8                     0x3E68C4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_9                     0x3E68C8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_10                    0x3E68CC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_11                    0x3E68D0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_12                    0x3E68D4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_13                    0x3E68D8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_14                    0x3E68DC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_15                    0x3E68E0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_0                    0x3E68E4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_1                    0x3E68E8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_2                    0x3E68EC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_3                    0x3E68F0
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_4                    0x3E68F4
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_5                    0x3E68F8
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_6                    0x3E68FC
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_7                    0x3E6900
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_8                    0x3E6904
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_9                    0x3E6908
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_10                   0x3E690C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_11                   0x3E6910
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_12                   0x3E6914
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_13                   0x3E6918
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_14                   0x3E691C
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_15                   0x3E6920
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_0                    0x3E6924
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_1                    0x3E6928
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_2                    0x3E692C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_3                    0x3E6930
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_4                    0x3E6934
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_5                    0x3E6938
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_6                    0x3E693C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_7                    0x3E6940
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_8                    0x3E6944
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_9                    0x3E6948
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_10                   0x3E694C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_11                   0x3E6950
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_12                   0x3E6954
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_13                   0x3E6958
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_14                   0x3E695C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_15                   0x3E6960
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_0                   0x3E6964
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_1                   0x3E6968
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_2                   0x3E696C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_3                   0x3E6970
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_4                   0x3E6974
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_5                   0x3E6978
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_6                   0x3E697C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_7                   0x3E6980
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_8                   0x3E6984
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_9                   0x3E6988
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_10                  0x3E698C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_11                  0x3E6990
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_12                  0x3E6994
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_13                  0x3E6998
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_14                  0x3E699C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_15                  0x3E69A0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_0                    0x3E69A4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_1                    0x3E69A8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_2                    0x3E69AC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_3                    0x3E69B0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_4                    0x3E69B4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_5                    0x3E69B8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_6                    0x3E69BC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_7                    0x3E69C0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_8                    0x3E69C4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_9                    0x3E69C8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_10                   0x3E69CC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_11                   0x3E69D0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_12                   0x3E69D4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_13                   0x3E69D8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_14                   0x3E69DC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_15                   0x3E69E0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_0                   0x3E69E4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_1                   0x3E69E8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_2                   0x3E69EC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_3                   0x3E69F0
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_4                   0x3E69F4
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_5                   0x3E69F8
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_6                   0x3E69FC
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_7                   0x3E6A00
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_8                   0x3E6A04
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_9                   0x3E6A08
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_10                  0x3E6A0C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_11                  0x3E6A10
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_12                  0x3E6A14
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_13                  0x3E6A18
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_14                  0x3E6A1C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_15                  0x3E6A20
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_HIT_AW                            0x3E6A64
+
+#define mmNIF_RTR_CTRL_6_RANGE_SEC_HIT_AR                            0x3E6A68
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_HIT_AW                           0x3E6A6C
+
+#define mmNIF_RTR_CTRL_6_RANGE_PRIV_HIT_AR                           0x3E6A70
+
+#define mmNIF_RTR_CTRL_6_RGL_CFG                                     0x3E6B64
+
+#define mmNIF_RTR_CTRL_6_RGL_SHIFT                                   0x3E6B68
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_0                          0x3E6B6C
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_1                          0x3E6B70
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_2                          0x3E6B74
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_3                          0x3E6B78
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_4                          0x3E6B7C
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_5                          0x3E6B80
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_6                          0x3E6B84
+
+#define mmNIF_RTR_CTRL_6_RGL_EXPECTED_LAT_7                          0x3E6B88
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_0                                 0x3E6BAC
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_1                                 0x3E6BB0
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_2                                 0x3E6BB4
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_3                                 0x3E6BB8
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_4                                 0x3E6BBC
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_5                                 0x3E6BC0
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_6                                 0x3E6BC4
+
+#define mmNIF_RTR_CTRL_6_RGL_TOKEN_7                                 0x3E6BC8
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_0                               0x3E6BEC
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_1                               0x3E6BF0
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_2                               0x3E6BF4
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_3                               0x3E6BF8
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_4                               0x3E6BFC
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_5                               0x3E6C00
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_6                               0x3E6C04
+
+#define mmNIF_RTR_CTRL_6_RGL_BANK_ID_7                               0x3E6C08
+
+#define mmNIF_RTR_CTRL_6_RGL_WDT                                     0x3E6C2C
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM0_CH0_CTR_WRAP                    0x3E6C30
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM0_CH1_CTR_WRAP                    0x3E6C34
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM1_CH0_CTR_WRAP                    0x3E6C38
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM1_CH1_CTR_WRAP                    0x3E6C3C
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM2_CH0_CTR_WRAP                    0x3E6C40
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM2_CH1_CTR_WRAP                    0x3E6C44
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM3_CH0_CTR_WRAP                    0x3E6C48
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM3_CH1_CTR_WRAP                    0x3E6C4C
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM0_CH0_CTR_CNT                     0x3E6C50
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM0_CH1_CTR_CNT                     0x3E6C54
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM1_CH0_CTR_CNT                     0x3E6C58
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM1_CH1_CTR_CNT                     0x3E6C5C
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM2_CH0_CTR_CNT                     0x3E6C60
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM2_CH1_CTR_CNT                     0x3E6C64
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM3_CH0_CTR_CNT                     0x3E6C68
+
+#define mmNIF_RTR_CTRL_6_E2E_AR_HBM3_CH1_CTR_CNT                     0x3E6C6C
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM0_CH0_CTR_WRAP                    0x3E6C70
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM0_CH1_CTR_WRAP                    0x3E6C74
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM1_CH0_CTR_WRAP                    0x3E6C78
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM1_CH1_CTR_WRAP                    0x3E6C7C
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM2_CH0_CTR_WRAP                    0x3E6C80
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM2_CH1_CTR_WRAP                    0x3E6C84
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM3_CH0_CTR_WRAP                    0x3E6C88
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM3_CH1_CTR_WRAP                    0x3E6C8C
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM0_CH0_CTR_CNT                     0x3E6C90
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM0_CH1_CTR_CNT                     0x3E6C94
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM1_CH0_CTR_CNT                     0x3E6C98
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM1_CH1_CTR_CNT                     0x3E6C9C
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM2_CH0_CTR_CNT                     0x3E6CA0
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM2_CH1_CTR_CNT                     0x3E6CA4
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM3_CH0_CTR_CNT                     0x3E6CA8
+
+#define mmNIF_RTR_CTRL_6_E2E_AW_HBM3_CH1_CTR_CNT                     0x3E6CAC
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_PC_SEL_0                             0x3E6CB0
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_PC_SEL_1                             0x3E6CB4
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_PC_SEL_2                             0x3E6CB8
+
+#define mmNIF_RTR_CTRL_6_NL_HBM_PC_SEL_3                             0x3E6CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_6_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_7_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_7_regs.h
new file mode 100644
index 000000000000..3a6a34ba2958
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/nif_rtr_ctrl_7_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_NIF_RTR_CTRL_7_REGS_H_
+#define ASIC_REG_NIF_RTR_CTRL_7_REGS_H_
+
+/*
+ *****************************************
+ *   NIF_RTR_CTRL_7 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmNIF_RTR_CTRL_7_PERM_SEL                                    0x3F6108
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_0                               0x3F6114
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_1                               0x3F6118
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_2                               0x3F611C
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_3                               0x3F6120
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_4                               0x3F6124
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_5                               0x3F6128
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_6                               0x3F612C
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_7                               0x3F6130
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_8                               0x3F6134
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_9                               0x3F6138
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_10                              0x3F613C
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_11                              0x3F6140
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_12                              0x3F6144
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_13                              0x3F6148
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_14                              0x3F614C
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_15                              0x3F6150
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_16                              0x3F6154
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_17                              0x3F6158
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_18                              0x3F615C
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_19                              0x3F6160
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_20                              0x3F6164
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_21                              0x3F6168
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_22                              0x3F616C
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_23                              0x3F6170
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_24                              0x3F6174
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_25                              0x3F6178
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_26                              0x3F617C
+
+#define mmNIF_RTR_CTRL_7_HBM_POLY_H3_27                              0x3F6180
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_0                              0x3F6184
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_1                              0x3F6188
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_2                              0x3F618C
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_3                              0x3F6190
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_4                              0x3F6194
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_5                              0x3F6198
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_6                              0x3F619C
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_7                              0x3F61A0
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_8                              0x3F61A4
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_9                              0x3F61A8
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_10                             0x3F61AC
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_11                             0x3F61B0
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_12                             0x3F61B4
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_13                             0x3F61B8
+
+#define mmNIF_RTR_CTRL_7_SRAM_POLY_H3_14                             0x3F61BC
+
+#define mmNIF_RTR_CTRL_7_SCRAM_SRAM_EN                               0x3F626C
+
+#define mmNIF_RTR_CTRL_7_RL_HBM_EN                                   0x3F6274
+
+#define mmNIF_RTR_CTRL_7_RL_HBM_SAT                                  0x3F6278
+
+#define mmNIF_RTR_CTRL_7_RL_HBM_RST                                  0x3F627C
+
+#define mmNIF_RTR_CTRL_7_RL_HBM_TIMEOUT                              0x3F6280
+
+#define mmNIF_RTR_CTRL_7_SCRAM_HBM_EN                                0x3F6284
+
+#define mmNIF_RTR_CTRL_7_RL_PCI_EN                                   0x3F6288
+
+#define mmNIF_RTR_CTRL_7_RL_PCI_SAT                                  0x3F628C
+
+#define mmNIF_RTR_CTRL_7_RL_PCI_RST                                  0x3F6290
+
+#define mmNIF_RTR_CTRL_7_RL_PCI_TIMEOUT                              0x3F6294
+
+#define mmNIF_RTR_CTRL_7_RL_SRAM_EN                                  0x3F629C
+
+#define mmNIF_RTR_CTRL_7_RL_SRAM_SAT                                 0x3F62A0
+
+#define mmNIF_RTR_CTRL_7_RL_SRAM_RST                                 0x3F62A4
+
+#define mmNIF_RTR_CTRL_7_RL_SRAM_TIMEOUT                             0x3F62AC
+
+#define mmNIF_RTR_CTRL_7_RL_SRAM_RED                                 0x3F62B4
+
+#define mmNIF_RTR_CTRL_7_E2E_HBM_EN                                  0x3F62EC
+
+#define mmNIF_RTR_CTRL_7_E2E_PCI_EN                                  0x3F62F0
+
+#define mmNIF_RTR_CTRL_7_E2E_HBM_WR_SIZE                             0x3F62F4
+
+#define mmNIF_RTR_CTRL_7_E2E_PCI_WR_SIZE                             0x3F62F8
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_PCI_CTR_SET_EN                       0x3F6404
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_PCI_CTR_SET                          0x3F6408
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_PCI_CTR_WRAP                         0x3F640C
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_PCI_CTR_CNT                          0x3F6410
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM_CTR_SET_EN                       0x3F6414
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM_CTR_SET                          0x3F6418
+
+#define mmNIF_RTR_CTRL_7_E2E_HBM_RD_SIZE                             0x3F641C
+
+#define mmNIF_RTR_CTRL_7_E2E_PCI_RD_SIZE                             0x3F6420
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_PCI_CTR_SET_EN                       0x3F6424
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_PCI_CTR_SET                          0x3F6428
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_PCI_CTR_WRAP                         0x3F642C
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_PCI_CTR_CNT                          0x3F6430
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM_CTR_SET_EN                       0x3F6434
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM_CTR_SET                          0x3F6438
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_SEL_0                                0x3F6450
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_SEL_1                                0x3F6454
+
+#define mmNIF_RTR_CTRL_7_NON_LIN_EN                                  0x3F6480
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_BANK_0                              0x3F6500
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_BANK_1                              0x3F6504
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_BANK_2                              0x3F6508
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_BANK_3                              0x3F650C
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_BANK_4                              0x3F6510
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_0                            0x3F6514
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_1                            0x3F6520
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_2                            0x3F6524
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_3                            0x3F6528
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_4                            0x3F652C
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_5                            0x3F6530
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_6                            0x3F6534
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_7                            0x3F6538
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_8                            0x3F653C
+
+#define mmNIF_RTR_CTRL_7_NL_SRAM_OFFSET_9                            0x3F6540
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_0                             0x3F6550
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_1                             0x3F6554
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_2                             0x3F6558
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_3                             0x3F655C
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_4                             0x3F6560
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_5                             0x3F6564
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_6                             0x3F6568
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_7                             0x3F656C
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_8                             0x3F6570
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_9                             0x3F6574
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_10                            0x3F6578
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_11                            0x3F657C
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_12                            0x3F6580
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_13                            0x3F6584
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_14                            0x3F6588
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_15                            0x3F658C
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_16                            0x3F6590
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_17                            0x3F6594
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_OFFSET_18                            0x3F6598
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_0                     0x3F65E4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_1                     0x3F65E8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_2                     0x3F65EC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_3                     0x3F65F0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_4                     0x3F65F4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_5                     0x3F65F8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_6                     0x3F65FC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_7                     0x3F6600
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_8                     0x3F6604
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_9                     0x3F6608
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_10                    0x3F660C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_11                    0x3F6610
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_12                    0x3F6614
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_13                    0x3F6618
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_14                    0x3F661C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_15                    0x3F6620
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_0                    0x3F6624
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_1                    0x3F6628
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_2                    0x3F662C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_3                    0x3F6630
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_4                    0x3F6634
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_5                    0x3F6638
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_6                    0x3F663C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_7                    0x3F6640
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_8                    0x3F6644
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_9                    0x3F6648
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_10                   0x3F664C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_11                   0x3F6650
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_12                   0x3F6654
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_13                   0x3F6658
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_14                   0x3F665C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_15                   0x3F6660
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_0                     0x3F6664
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_1                     0x3F6668
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_2                     0x3F666C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_3                     0x3F6670
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_4                     0x3F6674
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_5                     0x3F6678
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_6                     0x3F667C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_7                     0x3F6680
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_8                     0x3F6684
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_9                     0x3F6688
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_10                    0x3F668C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_11                    0x3F6690
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_12                    0x3F6694
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_13                    0x3F6698
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_14                    0x3F669C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_15                    0x3F66A0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_0                    0x3F66A4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_1                    0x3F66A8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_2                    0x3F66AC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_3                    0x3F66B0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_4                    0x3F66B4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_5                    0x3F66B8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_6                    0x3F66BC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_7                    0x3F66C0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_8                    0x3F66C4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_9                    0x3F66C8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_10                   0x3F66CC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_11                   0x3F66D0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_12                   0x3F66D4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_13                   0x3F66D8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_14                   0x3F66DC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_15                   0x3F66E0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_0                    0x3F66E4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_1                    0x3F66E8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_2                    0x3F66EC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_3                    0x3F66F0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_4                    0x3F66F4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_5                    0x3F66F8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_6                    0x3F66FC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_7                    0x3F6700
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_8                    0x3F6704
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_9                    0x3F6708
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_10                   0x3F670C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_11                   0x3F6710
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_12                   0x3F6714
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_13                   0x3F6718
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_14                   0x3F671C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_15                   0x3F6720
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_0                   0x3F6724
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_1                   0x3F6728
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_2                   0x3F672C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_3                   0x3F6730
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_4                   0x3F6734
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_5                   0x3F6738
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_6                   0x3F673C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_7                   0x3F6740
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_8                   0x3F6744
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_9                   0x3F6748
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_10                  0x3F674C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_11                  0x3F6750
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_12                  0x3F6754
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_13                  0x3F6758
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_14                  0x3F675C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_15                  0x3F6760
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_0                    0x3F6764
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_1                    0x3F6768
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_2                    0x3F676C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_3                    0x3F6770
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_4                    0x3F6774
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_5                    0x3F6778
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_6                    0x3F677C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_7                    0x3F6780
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_8                    0x3F6784
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_9                    0x3F6788
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_10                   0x3F678C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_11                   0x3F6790
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_12                   0x3F6794
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_13                   0x3F6798
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_14                   0x3F679C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_15                   0x3F67A0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_0                   0x3F67A4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_1                   0x3F67A8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_2                   0x3F67AC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_3                   0x3F67B0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_4                   0x3F67B4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_5                   0x3F67B8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_6                   0x3F67BC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_7                   0x3F67C0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_8                   0x3F67C4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_9                   0x3F67C8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_10                  0x3F67CC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_11                  0x3F67D0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_12                  0x3F67D4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_13                  0x3F67D8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_14                  0x3F67DC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_15                  0x3F67E0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_0                     0x3F6824
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_1                     0x3F6828
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_2                     0x3F682C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_3                     0x3F6830
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_4                     0x3F6834
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_5                     0x3F6838
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_6                     0x3F683C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_7                     0x3F6840
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_8                     0x3F6844
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_9                     0x3F6848
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_10                    0x3F684C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_11                    0x3F6850
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_12                    0x3F6854
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_13                    0x3F6858
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_14                    0x3F685C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_15                    0x3F6860
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_0                    0x3F6864
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_1                    0x3F6868
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_2                    0x3F686C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_3                    0x3F6870
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_4                    0x3F6874
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_5                    0x3F6878
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_6                    0x3F687C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_7                    0x3F6880
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_8                    0x3F6884
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_9                    0x3F6888
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_10                   0x3F688C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_11                   0x3F6890
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_12                   0x3F6894
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_13                   0x3F6898
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_14                   0x3F689C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_15                   0x3F68A0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_0                     0x3F68A4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_1                     0x3F68A8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_2                     0x3F68AC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_3                     0x3F68B0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_4                     0x3F68B4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_5                     0x3F68B8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_6                     0x3F68BC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_7                     0x3F68C0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_8                     0x3F68C4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_9                     0x3F68C8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_10                    0x3F68CC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_11                    0x3F68D0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_12                    0x3F68D4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_13                    0x3F68D8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_14                    0x3F68DC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_15                    0x3F68E0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_0                    0x3F68E4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_1                    0x3F68E8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_2                    0x3F68EC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_3                    0x3F68F0
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_4                    0x3F68F4
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_5                    0x3F68F8
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_6                    0x3F68FC
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_7                    0x3F6900
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_8                    0x3F6904
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_9                    0x3F6908
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_10                   0x3F690C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_11                   0x3F6910
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_12                   0x3F6914
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_13                   0x3F6918
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_14                   0x3F691C
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_15                   0x3F6920
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_0                    0x3F6924
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_1                    0x3F6928
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_2                    0x3F692C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_3                    0x3F6930
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_4                    0x3F6934
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_5                    0x3F6938
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_6                    0x3F693C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_7                    0x3F6940
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_8                    0x3F6944
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_9                    0x3F6948
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_10                   0x3F694C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_11                   0x3F6950
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_12                   0x3F6954
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_13                   0x3F6958
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_14                   0x3F695C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_15                   0x3F6960
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_0                   0x3F6964
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_1                   0x3F6968
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_2                   0x3F696C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_3                   0x3F6970
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_4                   0x3F6974
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_5                   0x3F6978
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_6                   0x3F697C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_7                   0x3F6980
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_8                   0x3F6984
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_9                   0x3F6988
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_10                  0x3F698C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_11                  0x3F6990
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_12                  0x3F6994
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_13                  0x3F6998
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_14                  0x3F699C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_15                  0x3F69A0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_0                    0x3F69A4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_1                    0x3F69A8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_2                    0x3F69AC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_3                    0x3F69B0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_4                    0x3F69B4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_5                    0x3F69B8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_6                    0x3F69BC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_7                    0x3F69C0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_8                    0x3F69C4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_9                    0x3F69C8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_10                   0x3F69CC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_11                   0x3F69D0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_12                   0x3F69D4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_13                   0x3F69D8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_14                   0x3F69DC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_15                   0x3F69E0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_0                   0x3F69E4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_1                   0x3F69E8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_2                   0x3F69EC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_3                   0x3F69F0
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_4                   0x3F69F4
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_5                   0x3F69F8
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_6                   0x3F69FC
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_7                   0x3F6A00
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_8                   0x3F6A04
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_9                   0x3F6A08
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_10                  0x3F6A0C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_11                  0x3F6A10
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_12                  0x3F6A14
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_13                  0x3F6A18
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_14                  0x3F6A1C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_15                  0x3F6A20
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_HIT_AW                            0x3F6A64
+
+#define mmNIF_RTR_CTRL_7_RANGE_SEC_HIT_AR                            0x3F6A68
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_HIT_AW                           0x3F6A6C
+
+#define mmNIF_RTR_CTRL_7_RANGE_PRIV_HIT_AR                           0x3F6A70
+
+#define mmNIF_RTR_CTRL_7_RGL_CFG                                     0x3F6B64
+
+#define mmNIF_RTR_CTRL_7_RGL_SHIFT                                   0x3F6B68
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_0                          0x3F6B6C
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_1                          0x3F6B70
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_2                          0x3F6B74
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_3                          0x3F6B78
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_4                          0x3F6B7C
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_5                          0x3F6B80
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_6                          0x3F6B84
+
+#define mmNIF_RTR_CTRL_7_RGL_EXPECTED_LAT_7                          0x3F6B88
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_0                                 0x3F6BAC
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_1                                 0x3F6BB0
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_2                                 0x3F6BB4
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_3                                 0x3F6BB8
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_4                                 0x3F6BBC
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_5                                 0x3F6BC0
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_6                                 0x3F6BC4
+
+#define mmNIF_RTR_CTRL_7_RGL_TOKEN_7                                 0x3F6BC8
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_0                               0x3F6BEC
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_1                               0x3F6BF0
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_2                               0x3F6BF4
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_3                               0x3F6BF8
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_4                               0x3F6BFC
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_5                               0x3F6C00
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_6                               0x3F6C04
+
+#define mmNIF_RTR_CTRL_7_RGL_BANK_ID_7                               0x3F6C08
+
+#define mmNIF_RTR_CTRL_7_RGL_WDT                                     0x3F6C2C
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM0_CH0_CTR_WRAP                    0x3F6C30
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM0_CH1_CTR_WRAP                    0x3F6C34
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM1_CH0_CTR_WRAP                    0x3F6C38
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM1_CH1_CTR_WRAP                    0x3F6C3C
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM2_CH0_CTR_WRAP                    0x3F6C40
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM2_CH1_CTR_WRAP                    0x3F6C44
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM3_CH0_CTR_WRAP                    0x3F6C48
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM3_CH1_CTR_WRAP                    0x3F6C4C
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM0_CH0_CTR_CNT                     0x3F6C50
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM0_CH1_CTR_CNT                     0x3F6C54
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM1_CH0_CTR_CNT                     0x3F6C58
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM1_CH1_CTR_CNT                     0x3F6C5C
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM2_CH0_CTR_CNT                     0x3F6C60
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM2_CH1_CTR_CNT                     0x3F6C64
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM3_CH0_CTR_CNT                     0x3F6C68
+
+#define mmNIF_RTR_CTRL_7_E2E_AR_HBM3_CH1_CTR_CNT                     0x3F6C6C
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM0_CH0_CTR_WRAP                    0x3F6C70
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM0_CH1_CTR_WRAP                    0x3F6C74
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM1_CH0_CTR_WRAP                    0x3F6C78
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM1_CH1_CTR_WRAP                    0x3F6C7C
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM2_CH0_CTR_WRAP                    0x3F6C80
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM2_CH1_CTR_WRAP                    0x3F6C84
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM3_CH0_CTR_WRAP                    0x3F6C88
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM3_CH1_CTR_WRAP                    0x3F6C8C
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM0_CH0_CTR_CNT                     0x3F6C90
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM0_CH1_CTR_CNT                     0x3F6C94
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM1_CH0_CTR_CNT                     0x3F6C98
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM1_CH1_CTR_CNT                     0x3F6C9C
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM2_CH0_CTR_CNT                     0x3F6CA0
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM2_CH1_CTR_CNT                     0x3F6CA4
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM3_CH0_CTR_CNT                     0x3F6CA8
+
+#define mmNIF_RTR_CTRL_7_E2E_AW_HBM3_CH1_CTR_CNT                     0x3F6CAC
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_PC_SEL_0                             0x3F6CB0
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_PC_SEL_1                             0x3F6CB4
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_PC_SEL_2                             0x3F6CB8
+
+#define mmNIF_RTR_CTRL_7_NL_HBM_PC_SEL_3                             0x3F6CBC
+
+#endif /* ASIC_REG_NIF_RTR_CTRL_7_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_etr_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_etr_regs.h
new file mode 100644
index 000000000000..b7c33e025db5
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_etr_regs.h
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_PSOC_ETR_REGS_H_
+#define ASIC_REG_PSOC_ETR_REGS_H_
+
+/*
+ *****************************************
+ *   PSOC_ETR (Prototype: ETR)
+ *****************************************
+ */
+
+#define mmPSOC_ETR_RSZ                                               0x2C43004
+
+#define mmPSOC_ETR_STS                                               0x2C4300C
+
+#define mmPSOC_ETR_RRD                                               0x2C43010
+
+#define mmPSOC_ETR_RRP                                               0x2C43014
+
+#define mmPSOC_ETR_RWP                                               0x2C43018
+
+#define mmPSOC_ETR_TRG                                               0x2C4301C
+
+#define mmPSOC_ETR_CTL                                               0x2C43020
+
+#define mmPSOC_ETR_RWD                                               0x2C43024
+
+#define mmPSOC_ETR_MODE                                              0x2C43028
+
+#define mmPSOC_ETR_LBUFLEVEL                                         0x2C4302C
+
+#define mmPSOC_ETR_CBUFLEVEL                                         0x2C43030
+
+#define mmPSOC_ETR_BUFWM                                             0x2C43034
+
+#define mmPSOC_ETR_RRPHI                                             0x2C43038
+
+#define mmPSOC_ETR_RWPHI                                             0x2C4303C
+
+#define mmPSOC_ETR_AXICTL                                            0x2C43110
+
+#define mmPSOC_ETR_DBALO                                             0x2C43118
+
+#define mmPSOC_ETR_DBAHI                                             0x2C4311C
+
+#define mmPSOC_ETR_FFSR                                              0x2C43300
+
+#define mmPSOC_ETR_FFCR                                              0x2C43304
+
+#define mmPSOC_ETR_PSCR                                              0x2C43308
+
+#define mmPSOC_ETR_ITMISCOP0                                         0x2C43EE0
+
+#define mmPSOC_ETR_ITTRFLIN                                          0x2C43EE8
+
+#define mmPSOC_ETR_ITATBDATA0                                        0x2C43EEC
+
+#define mmPSOC_ETR_ITATBCTR2                                         0x2C43EF0
+
+#define mmPSOC_ETR_ITATBCTR1                                         0x2C43EF4
+
+#define mmPSOC_ETR_ITATBCTR0                                         0x2C43EF8
+
+#define mmPSOC_ETR_ITCTRL                                            0x2C43F00
+
+#define mmPSOC_ETR_CLAIMSET                                          0x2C43FA0
+
+#define mmPSOC_ETR_CLAIMCLR                                          0x2C43FA4
+
+#define mmPSOC_ETR_LAR                                               0x2C43FB0
+
+#define mmPSOC_ETR_LSR                                               0x2C43FB4
+
+#define mmPSOC_ETR_AUTHSTATUS                                        0x2C43FB8
+
+#define mmPSOC_ETR_DEVID                                             0x2C43FC8
+
+#define mmPSOC_ETR_DEVTYPE                                           0x2C43FCC
+
+#define mmPSOC_ETR_PERIPHID4                                         0x2C43FD0
+
+#define mmPSOC_ETR_PERIPHID5                                         0x2C43FD4
+
+#define mmPSOC_ETR_PERIPHID6                                         0x2C43FD8
+
+#define mmPSOC_ETR_PERIPHID7                                         0x2C43FDC
+
+#define mmPSOC_ETR_PERIPHID0                                         0x2C43FE0
+
+#define mmPSOC_ETR_PERIPHID1                                         0x2C43FE4
+
+#define mmPSOC_ETR_PERIPHID2                                         0x2C43FE8
+
+#define mmPSOC_ETR_PERIPHID3                                         0x2C43FEC
+
+#define mmPSOC_ETR_COMPID0                                           0x2C43FF0
+
+#define mmPSOC_ETR_COMPID1                                           0x2C43FF4
+
+#define mmPSOC_ETR_COMPID2                                           0x2C43FF8
+
+#define mmPSOC_ETR_COMPID3                                           0x2C43FFC
+
+#endif /* ASIC_REG_PSOC_ETR_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_masks.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_masks.h
new file mode 100644
index 000000000000..6703e678ee9f
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_masks.h
@@ -0,0 +1,502 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_PSOC_GLOBAL_CONF_MASKS_H_
+#define ASIC_REG_PSOC_GLOBAL_CONF_MASKS_H_
+
+/*
+ *****************************************
+ *   PSOC_GLOBAL_CONF (Prototype: GLOBAL_CONF)
+ *****************************************
+ */
+
+/* PSOC_GLOBAL_CONF_NON_RST_FLOPS */
+#define PSOC_GLOBAL_CONF_NON_RST_FLOPS_VAL_SHIFT                     0
+#define PSOC_GLOBAL_CONF_NON_RST_FLOPS_VAL_MASK                      0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_PCI_FW_FSM */
+#define PSOC_GLOBAL_CONF_PCI_FW_FSM_EN_SHIFT                         0
+#define PSOC_GLOBAL_CONF_PCI_FW_FSM_EN_MASK                          0x1
+
+/* PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START */
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START_IND_SHIFT                 0
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START_IND_MASK                  0x1
+
+/* PSOC_GLOBAL_CONF_BTM_FSM */
+#define PSOC_GLOBAL_CONF_BTM_FSM_STATE_SHIFT                         0
+#define PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK                          0xF
+
+/* PSOC_GLOBAL_CONF_SW_BTM_FSM */
+#define PSOC_GLOBAL_CONF_SW_BTM_FSM_CTRL_SHIFT                       0
+#define PSOC_GLOBAL_CONF_SW_BTM_FSM_CTRL_MASK                        0xF
+
+/* PSOC_GLOBAL_CONF_SW_BOOT_SEQ_FSM */
+#define PSOC_GLOBAL_CONF_SW_BOOT_SEQ_FSM_CTRL_SHIFT                  0
+#define PSOC_GLOBAL_CONF_SW_BOOT_SEQ_FSM_CTRL_MASK                   0xF
+
+/* PSOC_GLOBAL_CONF_BOOT_SEQ_TIMEOUT */
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_TIMEOUT_VAL_SHIFT                  0
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_TIMEOUT_VAL_MASK                   0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_SPI_MEM_EN */
+#define PSOC_GLOBAL_CONF_SPI_MEM_EN_IND_SHIFT                        0
+#define PSOC_GLOBAL_CONF_SPI_MEM_EN_IND_MASK                         0x1
+
+/* PSOC_GLOBAL_CONF_PRSTN */
+#define PSOC_GLOBAL_CONF_PRSTN_VAL_SHIFT                             0
+#define PSOC_GLOBAL_CONF_PRSTN_VAL_MASK                              0x1
+
+/* PSOC_GLOBAL_CONF_PCIE_EN */
+#define PSOC_GLOBAL_CONF_PCIE_EN_MASK_SHIFT                          0
+#define PSOC_GLOBAL_CONF_PCIE_EN_MASK_MASK                           0x1
+
+/* PSOC_GLOBAL_CONF_PCIE_PRSTN_INTR */
+#define PSOC_GLOBAL_CONF_PCIE_PRSTN_INTR_IND_SHIFT                   0
+#define PSOC_GLOBAL_CONF_PCIE_PRSTN_INTR_IND_MASK                    0x1
+
+/* PSOC_GLOBAL_CONF_SPI_IMG_STS */
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_PRI_SHIFT                       0
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_PRI_MASK                        0x1
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_SEC_SHIFT                       1
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_SEC_MASK                        0x2
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_PRSTN_SHIFT                     2
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_PRSTN_MASK                      0x4
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_PCI_SHIFT                       3
+#define PSOC_GLOBAL_CONF_SPI_IMG_STS_PCI_MASK                        0x8
+
+/* PSOC_GLOBAL_CONF_BOOT_SEQ_FSM */
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_IDLE_SHIFT                     0
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_IDLE_MASK                      0x1
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_BOOT_INIT_SHIFT                1
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_BOOT_INIT_MASK                 0x2
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_PRI_SHIFT                  2
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_PRI_MASK                   0x4
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_SEC_SHIFT                  3
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_SEC_MASK                   0x8
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_PRSTN_SHIFT                4
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_PRSTN_MASK                 0x10
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_PCIE_SHIFT                 5
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_SPI_PCIE_MASK                  0x20
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_ROM_SHIFT                      6
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_ROM_MASK                       0x40
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_PCLK_READY_SHIFT               7
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_PCLK_READY_MASK                0x80
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_LTSSM_EN_SHIFT                 8
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_FSM_LTSSM_EN_MASK                  0x100
+
+/* PSOC_GLOBAL_CONF_BOOT_SEQ_EXT_LD */
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_EXT_LD_DONE_SHIFT                  0
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_EXT_LD_DONE_MASK                   0x1
+
+/* PSOC_GLOBAL_CONF_PHY_STABLE */
+#define PSOC_GLOBAL_CONF_PHY_STABLE_PRSTN_SHIFT                      0
+#define PSOC_GLOBAL_CONF_PHY_STABLE_PRSTN_MASK                       0x1
+
+/* PSOC_GLOBAL_CONF_PRSTN_OVR */
+#define PSOC_GLOBAL_CONF_PRSTN_OVR_EN_SHIFT                          0
+#define PSOC_GLOBAL_CONF_PRSTN_OVR_EN_MASK                           0x1
+#define PSOC_GLOBAL_CONF_PRSTN_OVR_VAL_SHIFT                         4
+#define PSOC_GLOBAL_CONF_PRSTN_OVR_VAL_MASK                          0x10
+
+/* PSOC_GLOBAL_CONF_ETR_FLUSH */
+#define PSOC_GLOBAL_CONF_ETR_FLUSH_MASK_SHIFT                        0
+#define PSOC_GLOBAL_CONF_ETR_FLUSH_MASK_MASK                         0x1
+
+/* PSOC_GLOBAL_CONF_COLD_RST_FLOPS */
+#define PSOC_GLOBAL_CONF_COLD_RST_FLOPS_VAL_SHIFT                    0
+#define PSOC_GLOBAL_CONF_COLD_RST_FLOPS_VAL_MASK                     0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_DIS_RAZWI_ERR */
+#define PSOC_GLOBAL_CONF_DIS_RAZWI_ERR_IND_SHIFT                     0
+#define PSOC_GLOBAL_CONF_DIS_RAZWI_ERR_IND_MASK                      0x1
+
+/* PSOC_GLOBAL_CONF_PCIE_PHY_RST_N */
+#define PSOC_GLOBAL_CONF_PCIE_PHY_RST_N_IND_SHIFT                    0
+#define PSOC_GLOBAL_CONF_PCIE_PHY_RST_N_IND_MASK                     0x1
+
+/* PSOC_GLOBAL_CONF_RAZWI */
+#define PSOC_GLOBAL_CONF_RAZWI_INTR_SHIFT                            0
+#define PSOC_GLOBAL_CONF_RAZWI_INTR_MASK                             0x1
+#define PSOC_GLOBAL_CONF_RAZWI_MASK_SHIFT                            4
+#define PSOC_GLOBAL_CONF_RAZWI_MASK_MASK                             0x10
+
+/* PSOC_GLOBAL_CONF_PROT */
+#define PSOC_GLOBAL_CONF_PROT_AR_SHIFT                               0
+#define PSOC_GLOBAL_CONF_PROT_AR_MASK                                0x7
+#define PSOC_GLOBAL_CONF_PROT_AW_SHIFT                               4
+#define PSOC_GLOBAL_CONF_PROT_AW_MASK                                0x70
+
+/* PSOC_GLOBAL_CONF_ADC */
+#define PSOC_GLOBAL_CONF_ADC_INTR_SHIFT                              0
+#define PSOC_GLOBAL_CONF_ADC_INTR_MASK                               0x1
+#define PSOC_GLOBAL_CONF_ADC_MASK_SHIFT                              4
+#define PSOC_GLOBAL_CONF_ADC_MASK_MASK                               0x10
+
+/* PSOC_GLOBAL_CONF_BOOT_SEQ_TO */
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_TO_MASK_SHIFT                      0
+#define PSOC_GLOBAL_CONF_BOOT_SEQ_TO_MASK_MASK                       0x1
+
+/* PSOC_GLOBAL_CONF_SCRATCHPAD */
+#define PSOC_GLOBAL_CONF_SCRATCHPAD_REG_SHIFT                        0
+#define PSOC_GLOBAL_CONF_SCRATCHPAD_REG_MASK                         0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_SEMAPHORE */
+#define PSOC_GLOBAL_CONF_SEMAPHORE_REG_SHIFT                         0
+#define PSOC_GLOBAL_CONF_SEMAPHORE_REG_MASK                          0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_CPU_BOOT_STATUS */
+#define PSOC_GLOBAL_CONF_CPU_BOOT_STATUS_CNTR_SHIFT                  0
+#define PSOC_GLOBAL_CONF_CPU_BOOT_STATUS_CNTR_MASK                   0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_KMD_MSG_TO_CPU */
+#define PSOC_GLOBAL_CONF_KMD_MSG_TO_CPU_VAL_SHIFT                    0
+#define PSOC_GLOBAL_CONF_KMD_MSG_TO_CPU_VAL_MASK                     0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_SPL_SOURCE */
+#define PSOC_GLOBAL_CONF_SPL_SOURCE_VAL_SHIFT                        0
+#define PSOC_GLOBAL_CONF_SPL_SOURCE_VAL_MASK                         0x7
+
+/* PSOC_GLOBAL_CONF_I2C_MSTR1_DBG */
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_S_GEN_SHIFT                   0
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_S_GEN_MASK                    0x1
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_P_GEN_SHIFT                   1
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_P_GEN_MASK                    0x2
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_DATA_SHIFT                    2
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_DATA_MASK                     0x4
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_ADDR_SHIFT                    3
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_ADDR_MASK                     0x8
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_RD_SHIFT                      4
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_RD_MASK                       0x10
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_WR_SHIFT                      5
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_WR_MASK                       0x20
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_HS_SHIFT                      6
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_HS_MASK                       0x40
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_MASTER_ACT_SHIFT              7
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_MASTER_ACT_MASK               0x80
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_SLAVE_ACT_SHIFT               8
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_SLAVE_ACT_MASK                0x100
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_ADDR_10BIT_SHIFT              9
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_ADDR_10BIT_MASK               0x200
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_MST_CSTATE_SHIFT              10
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_MST_CSTATE_MASK               0x7C00
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_SLV_CSTATE_SHIFT              15
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_SLV_CSTATE_MASK               0x78000
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_IC_EN_SHIFT                   19
+#define PSOC_GLOBAL_CONF_I2C_MSTR1_DBG_IC_EN_MASK                    0x80000
+
+/* PSOC_GLOBAL_CONF_I2C_SLV */
+#define PSOC_GLOBAL_CONF_I2C_SLV_CPU_CTRL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_I2C_SLV_CPU_CTRL_MASK                       0x1
+
+/* PSOC_GLOBAL_CONF_I2C_SLV_INTR_MASK */
+#define PSOC_GLOBAL_CONF_I2C_SLV_INTR_MASK_FLD_INT_SHIFT             0
+#define PSOC_GLOBAL_CONF_I2C_SLV_INTR_MASK_FLD_INT_MASK              0x1
+
+/* PSOC_GLOBAL_CONF_TRACE_ADDR */
+#define PSOC_GLOBAL_CONF_TRACE_ADDR_MSB_SHIFT                        0
+#define PSOC_GLOBAL_CONF_TRACE_ADDR_MSB_MASK                         0x3FF
+
+/* PSOC_GLOBAL_CONF_ARUSER */
+#define PSOC_GLOBAL_CONF_ARUSER_VAL_SHIFT                            0
+#define PSOC_GLOBAL_CONF_ARUSER_VAL_MASK                             0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_AWUSER */
+#define PSOC_GLOBAL_CONF_AWUSER_VAL_SHIFT                            0
+#define PSOC_GLOBAL_CONF_AWUSER_VAL_MASK                             0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_TRACE_AWUSER */
+#define PSOC_GLOBAL_CONF_TRACE_AWUSER_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_TRACE_AWUSER_VAL_MASK                       0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_TRACE_ARUSER */
+#define PSOC_GLOBAL_CONF_TRACE_ARUSER_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_TRACE_ARUSER_VAL_MASK                       0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_BTL_STS */
+#define PSOC_GLOBAL_CONF_BTL_STS_DONE_SHIFT                          0
+#define PSOC_GLOBAL_CONF_BTL_STS_DONE_MASK                           0x1
+#define PSOC_GLOBAL_CONF_BTL_STS_FAIL_SHIFT                          4
+#define PSOC_GLOBAL_CONF_BTL_STS_FAIL_MASK                           0x10
+#define PSOC_GLOBAL_CONF_BTL_STS_FAIL_CODE_SHIFT                     8
+#define PSOC_GLOBAL_CONF_BTL_STS_FAIL_CODE_MASK                      0xF00
+
+/* PSOC_GLOBAL_CONF_TIMEOUT_INTR */
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_0_SHIFT                   0
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_0_MASK                    0x1
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_1_SHIFT                   1
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_1_MASK                    0x2
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_2_SHIFT                   2
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_2_MASK                    0x4
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_3_SHIFT                   3
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_3_MASK                    0x8
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_4_SHIFT                   4
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_4_MASK                    0x10
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_TIMER_SHIFT                    5
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_TIMER_MASK                     0x20
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_UART_0_SHIFT                   6
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_UART_0_MASK                    0x40
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_UART_1_SHIFT                   7
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_UART_1_MASK                    0x80
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_5_SHIFT                   8
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_5_MASK                    0x100
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_6_SHIFT                   9
+#define PSOC_GLOBAL_CONF_TIMEOUT_INTR_GPIO_6_MASK                    0x200
+
+/* PSOC_GLOBAL_CONF_COMB_TIMEOUT_INTR */
+#define PSOC_GLOBAL_CONF_COMB_TIMEOUT_INTR_IND_SHIFT                 0
+#define PSOC_GLOBAL_CONF_COMB_TIMEOUT_INTR_IND_MASK                  0x1
+
+/* PSOC_GLOBAL_CONF_PERIPH_INTR */
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_TX_SHIFT                 0
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_TX_MASK                  0x1
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_RX_SHIFT                 1
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_RX_MASK                  0x2
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_TXOVR_SHIFT              2
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_TXOVR_MASK               0x4
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_RXOVR_SHIFT              3
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_0_RXOVR_MASK               0x8
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_TX_SHIFT                 4
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_TX_MASK                  0x10
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_RX_SHIFT                 5
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_RX_MASK                  0x20
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_TXOVR_SHIFT              6
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_TXOVR_MASK               0x40
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_RXOVR_SHIFT              7
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_UART_1_RXOVR_MASK               0x80
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_EMMC_SHIFT                      12
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_EMMC_MASK                       0x1000
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_EMMC_WAKEUP_SHIFT               13
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_EMMC_WAKEUP_MASK                0x2000
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_MII_SHIFT                       16
+#define PSOC_GLOBAL_CONF_PERIPH_INTR_MII_MASK                        0x10000
+
+/* PSOC_GLOBAL_CONF_COMB_PERIPH_INTR */
+#define PSOC_GLOBAL_CONF_COMB_PERIPH_INTR_IND_SHIFT                  0
+#define PSOC_GLOBAL_CONF_COMB_PERIPH_INTR_IND_MASK                   0x1
+
+/* PSOC_GLOBAL_CONF_AXI_ERR_INTR */
+#define PSOC_GLOBAL_CONF_AXI_ERR_INTR_IND_SHIFT                      0
+#define PSOC_GLOBAL_CONF_AXI_ERR_INTR_IND_MASK                       0x1
+
+/* PSOC_GLOBAL_CONF_TARGETID */
+#define PSOC_GLOBAL_CONF_TARGETID_TDESIGNER_SHIFT                    1
+#define PSOC_GLOBAL_CONF_TARGETID_TDESIGNER_MASK                     0xFFE
+#define PSOC_GLOBAL_CONF_TARGETID_TPARTNO_SHIFT                      16
+#define PSOC_GLOBAL_CONF_TARGETID_TPARTNO_MASK                       0xFFF0000
+#define PSOC_GLOBAL_CONF_TARGETID_TREVISION_SHIFT                    28
+#define PSOC_GLOBAL_CONF_TARGETID_TREVISION_MASK                     0xF0000000
+
+/* PSOC_GLOBAL_CONF_EMMC_INT_VOL_STABLE */
+#define PSOC_GLOBAL_CONF_EMMC_INT_VOL_STABLE_IND_SHIFT               0
+#define PSOC_GLOBAL_CONF_EMMC_INT_VOL_STABLE_IND_MASK                0x1
+
+/* PSOC_GLOBAL_CONF_BOOT_STRAP_PINS */
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_I2C_SLV_ADDR_SHIFT          0
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_I2C_SLV_ADDR_MASK           0x1
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_PCIE_EN_SHIFT               1
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_PCIE_EN_MASK                0x2
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_REPAIR_CFG_SHIFT            2
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_REPAIR_CFG_MASK             0xC
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_CPOL_SHIFT                  4
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_CPOL_MASK                   0x10
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_CPHA_SHIFT                  5
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_CPHA_MASK                   0x20
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_BTL_EN_SHIFT                6
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_BTL_EN_MASK                 0x40
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_BTL_ROM_EN_SHIFT            7
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_BTL_ROM_EN_MASK             0x80
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_DUMP_SEL_SHIFT              8
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_DUMP_SEL_MASK               0x1FFF00
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_GRAD_RST_SHIFT              22
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_GRAD_RST_MASK               0x400000
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_DUMP_DIS_SHIFT              23
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_DUMP_DIS_MASK               0x800000
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_I2C_SHIFT                   24
+#define PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_I2C_MASK                    0x1F000000
+
+/* PSOC_GLOBAL_CONF_MEM_REPAIR_DIV */
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_DIV_EN_SHIFT                     0
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_DIV_EN_MASK                      0x1
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_DIV_VAL_SHIFT                    8
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_DIV_VAL_MASK                     0xFF00
+
+/* PSOC_GLOBAL_CONF_MEM_REPAIR_CTRL */
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_CTRL_SET_SHIFT                   0
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_CTRL_SET_MASK                    0x1
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_CTRL_CLR_SHIFT                   1
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_CTRL_CLR_MASK                    0x2
+
+/* PSOC_GLOBAL_CONF_MEM_REPAIR_STS */
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_STS_IND_SHIFT                    0
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_STS_IND_MASK                     0x1
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_STS_FAIL_SHIFT                   4
+#define PSOC_GLOBAL_CONF_MEM_REPAIR_STS_FAIL_MASK                    0x10
+
+/* PSOC_GLOBAL_CONF_OUTSTANT_TRANS */
+#define PSOC_GLOBAL_CONF_OUTSTANT_TRANS_RD_SHIFT                     0
+#define PSOC_GLOBAL_CONF_OUTSTANT_TRANS_RD_MASK                      0x1
+#define PSOC_GLOBAL_CONF_OUTSTANT_TRANS_WR_SHIFT                     1
+#define PSOC_GLOBAL_CONF_OUTSTANT_TRANS_WR_MASK                      0x2
+
+/* PSOC_GLOBAL_CONF_MASK_REQ */
+#define PSOC_GLOBAL_CONF_MASK_REQ_IND_SHIFT                          0
+#define PSOC_GLOBAL_CONF_MASK_REQ_IND_MASK                           0x1
+
+/* PSOC_GLOBAL_CONF_WD_RST_CFG_L */
+#define PSOC_GLOBAL_CONF_WD_RST_CFG_L_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_WD_RST_CFG_L_VAL_MASK                       0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_WD_RST_CFG_H */
+#define PSOC_GLOBAL_CONF_WD_RST_CFG_H_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_WD_RST_CFG_H_VAL_MASK                       0x3FFFFF
+
+/* PSOC_GLOBAL_CONF_MNL_RST_CFG_L */
+#define PSOC_GLOBAL_CONF_MNL_RST_CFG_L_VAL_SHIFT                     0
+#define PSOC_GLOBAL_CONF_MNL_RST_CFG_L_VAL_MASK                      0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_MNL_RST_CFG_H */
+#define PSOC_GLOBAL_CONF_MNL_RST_CFG_H_VAL_SHIFT                     0
+#define PSOC_GLOBAL_CONF_MNL_RST_CFG_H_VAL_MASK                      0x3FFFFF
+
+/* PSOC_GLOBAL_CONF_PRSTN_RST_CFG_L */
+#define PSOC_GLOBAL_CONF_PRSTN_RST_CFG_L_VAL_SHIFT                   0
+#define PSOC_GLOBAL_CONF_PRSTN_RST_CFG_L_VAL_MASK                    0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_PRSTN_RST_CFG_H */
+#define PSOC_GLOBAL_CONF_PRSTN_RST_CFG_H_VAL_SHIFT                   0
+#define PSOC_GLOBAL_CONF_PRSTN_RST_CFG_H_VAL_MASK                    0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L */
+#define PSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L_VAL_SHIFT                  0
+#define PSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L_VAL_MASK                   0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H */
+#define PSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H_VAL_SHIFT                  0
+#define PSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H_VAL_MASK                   0x3FFFFF
+
+/* PSOC_GLOBAL_CONF_SW_ALL_RST */
+#define PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT                        0
+#define PSOC_GLOBAL_CONF_SW_ALL_RST_IND_MASK                         0x1
+
+/* PSOC_GLOBAL_CONF_SOFT_RST */
+#define PSOC_GLOBAL_CONF_SOFT_RST_IND_SHIFT                          0
+#define PSOC_GLOBAL_CONF_SOFT_RST_IND_MASK                           0x1
+
+/* PSOC_GLOBAL_CONF_SOFT_RST_CFG_L */
+#define PSOC_GLOBAL_CONF_SOFT_RST_CFG_L_VAL_SHIFT                    0
+#define PSOC_GLOBAL_CONF_SOFT_RST_CFG_L_VAL_MASK                     0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_SOFT_RST_CFG_H */
+#define PSOC_GLOBAL_CONF_SOFT_RST_CFG_H_VAL_SHIFT                    0
+#define PSOC_GLOBAL_CONF_SOFT_RST_CFG_H_VAL_MASK                     0x3FFFFF
+
+/* PSOC_GLOBAL_CONF_UNIT_RST_N */
+#define PSOC_GLOBAL_CONF_UNIT_RST_N_IND_SHIFT                        0
+#define PSOC_GLOBAL_CONF_UNIT_RST_N_IND_MASK                         0x1
+
+/* PSOC_GLOBAL_CONF_UNIT_RST_N_L */
+#define PSOC_GLOBAL_CONF_UNIT_RST_N_L_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_UNIT_RST_N_L_VAL_MASK                       0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_UNIT_RST_N_H */
+#define PSOC_GLOBAL_CONF_UNIT_RST_N_H_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_UNIT_RST_N_H_VAL_MASK                       0x3FFFFF
+
+/* PSOC_GLOBAL_CONF_BTL_IMG */
+#define PSOC_GLOBAL_CONF_BTL_IMG_SEL_SHIFT                           0
+#define PSOC_GLOBAL_CONF_BTL_IMG_SEL_MASK                            0x1
+
+/* PSOC_GLOBAL_CONF_PRSTN_MASK */
+#define PSOC_GLOBAL_CONF_PRSTN_MASK_IND_SHIFT                        0
+#define PSOC_GLOBAL_CONF_PRSTN_MASK_IND_MASK                         0x1
+
+/* PSOC_GLOBAL_CONF_WD_MASK */
+#define PSOC_GLOBAL_CONF_WD_MASK_IND_SHIFT                           0
+#define PSOC_GLOBAL_CONF_WD_MASK_IND_MASK                            0x1
+
+/* PSOC_GLOBAL_CONF_RST_SRC */
+#define PSOC_GLOBAL_CONF_RST_SRC_VAL_SHIFT                           0
+#define PSOC_GLOBAL_CONF_RST_SRC_VAL_MASK                            0xF
+
+/* PSOC_GLOBAL_CONF_BOOT_STATE */
+#define PSOC_GLOBAL_CONF_BOOT_STATE_VAL_SHIFT                        0
+#define PSOC_GLOBAL_CONF_BOOT_STATE_VAL_MASK                         0x1
+
+/* PSOC_GLOBAL_CONF_PAD_1V8_CFG */
+#define PSOC_GLOBAL_CONF_PAD_1V8_CFG_VAL_SHIFT                       0
+#define PSOC_GLOBAL_CONF_PAD_1V8_CFG_VAL_MASK                        0x7F
+
+/* PSOC_GLOBAL_CONF_PAD_3V3_CFG */
+#define PSOC_GLOBAL_CONF_PAD_3V3_CFG_VAL_SHIFT                       0
+#define PSOC_GLOBAL_CONF_PAD_3V3_CFG_VAL_MASK                        0x7F
+
+/* PSOC_GLOBAL_CONF_PAD_1V8_INPUT */
+#define PSOC_GLOBAL_CONF_PAD_1V8_INPUT_CFG_SHIFT                     0
+#define PSOC_GLOBAL_CONF_PAD_1V8_INPUT_CFG_MASK                      0x7
+
+/* PSOC_GLOBAL_CONF_BNK3V3_MS */
+#define PSOC_GLOBAL_CONF_BNK3V3_MS_VAL_SHIFT                         0
+#define PSOC_GLOBAL_CONF_BNK3V3_MS_VAL_MASK                          0x3
+
+/* PSOC_GLOBAL_CONF_ADC_CLK_FREQ */
+#define PSOC_GLOBAL_CONF_ADC_CLK_FREQ_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_ADC_CLK_FREQ_VAL_MASK                       0xFF
+
+/* PSOC_GLOBAL_CONF_ADC_DELAY_FROM_START */
+#define PSOC_GLOBAL_CONF_ADC_DELAY_FROM_START_VAL_SHIFT              0
+#define PSOC_GLOBAL_CONF_ADC_DELAY_FROM_START_VAL_MASK               0xFF
+
+/* PSOC_GLOBAL_CONF_ADC_DATA_SAMPLES */
+#define PSOC_GLOBAL_CONF_ADC_DATA_SAMPLES_VAL_SHIFT                  0
+#define PSOC_GLOBAL_CONF_ADC_DATA_SAMPLES_VAL_MASK                   0x1F
+
+/* PSOC_GLOBAL_CONF_ADC_TPH_CS */
+#define PSOC_GLOBAL_CONF_ADC_TPH_CS_VAL_SHIFT                        0
+#define PSOC_GLOBAL_CONF_ADC_TPH_CS_VAL_MASK                         0xFF
+
+/* PSOC_GLOBAL_CONF_ADC_LSB_NMSB */
+#define PSOC_GLOBAL_CONF_ADC_LSB_NMSB_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_ADC_LSB_NMSB_VAL_MASK                       0x1
+
+/* PSOC_GLOBAL_CONF_ADC_ONE_NCONTIUES */
+#define PSOC_GLOBAL_CONF_ADC_ONE_NCONTIUES_VAL_SHIFT                 0
+#define PSOC_GLOBAL_CONF_ADC_ONE_NCONTIUES_VAL_MASK                  0x1
+
+/* PSOC_GLOBAL_CONF_ADC_BLOCK_ENABLE */
+#define PSOC_GLOBAL_CONF_ADC_BLOCK_ENABLE_VAL_SHIFT                  0
+#define PSOC_GLOBAL_CONF_ADC_BLOCK_ENABLE_VAL_MASK                   0x1
+
+/* PSOC_GLOBAL_CONF_ADC_CFG_DATA */
+#define PSOC_GLOBAL_CONF_ADC_CFG_DATA_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_ADC_CFG_DATA_VAL_MASK                       0xFFFFFFFF
+
+/* PSOC_GLOBAL_CONF_ADC_TDV_CSDO */
+#define PSOC_GLOBAL_CONF_ADC_TDV_CSDO_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_ADC_TDV_CSDO_VAL_MASK                       0xFF
+
+/* PSOC_GLOBAL_CONF_ADC_TSU_CSCK */
+#define PSOC_GLOBAL_CONF_ADC_TSU_CSCK_VAL_SHIFT                      0
+#define PSOC_GLOBAL_CONF_ADC_TSU_CSCK_VAL_MASK                       0xFF
+
+/* PSOC_GLOBAL_CONF_PAD_DEFAULT */
+#define PSOC_GLOBAL_CONF_PAD_DEFAULT_VAL_SHIFT                       0
+#define PSOC_GLOBAL_CONF_PAD_DEFAULT_VAL_MASK                        0xF
+
+/* PSOC_GLOBAL_CONF_PAD_SEL */
+#define PSOC_GLOBAL_CONF_PAD_SEL_VAL_SHIFT                           0
+#define PSOC_GLOBAL_CONF_PAD_SEL_VAL_MASK                            0x3
+
+/* PSOC_GLOBAL_CONF_RST_CTRL */
+#define PSOC_GLOBAL_CONF_RST_CTRL_SEL_SHIFT                          0
+#define PSOC_GLOBAL_CONF_RST_CTRL_SEL_MASK                           0xFF
+
+#endif /* ASIC_REG_PSOC_GLOBAL_CONF_MASKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_regs.h
new file mode 100644
index 000000000000..1b5cfcc1d85f
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_global_conf_regs.h
@@ -0,0 +1,1062 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_PSOC_GLOBAL_CONF_REGS_H_
+#define ASIC_REG_PSOC_GLOBAL_CONF_REGS_H_
+
+/*
+ *****************************************
+ *   PSOC_GLOBAL_CONF (Prototype: GLOBAL_CONF)
+ *****************************************
+ */
+
+#define mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_0                           0xC4B000
+
+#define mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_1                           0xC4B004
+
+#define mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_2                           0xC4B008
+
+#define mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_3                           0xC4B00C
+
+#define mmPSOC_GLOBAL_CONF_PCI_FW_FSM                                0xC4B020
+
+#define mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START                         0xC4B024
+
+#define mmPSOC_GLOBAL_CONF_BTM_FSM                                   0xC4B028
+
+#define mmPSOC_GLOBAL_CONF_SW_BTM_FSM                                0xC4B030
+
+#define mmPSOC_GLOBAL_CONF_SW_BOOT_SEQ_FSM                           0xC4B034
+
+#define mmPSOC_GLOBAL_CONF_BOOT_SEQ_TIMEOUT                          0xC4B038
+
+#define mmPSOC_GLOBAL_CONF_SPI_MEM_EN                                0xC4B040
+
+#define mmPSOC_GLOBAL_CONF_PRSTN                                     0xC4B044
+
+#define mmPSOC_GLOBAL_CONF_PCIE_EN                                   0xC4B048
+
+#define mmPSOC_GLOBAL_CONF_PCIE_PRSTN_INTR                           0xC4B04C
+
+#define mmPSOC_GLOBAL_CONF_SPI_IMG_STS                               0xC4B050
+
+#define mmPSOC_GLOBAL_CONF_BOOT_SEQ_FSM                              0xC4B054
+
+#define mmPSOC_GLOBAL_CONF_BOOT_SEQ_EXT_LD                           0xC4B058
+
+#define mmPSOC_GLOBAL_CONF_PHY_STABLE                                0xC4B060
+
+#define mmPSOC_GLOBAL_CONF_PRSTN_OVR                                 0xC4B064
+
+#define mmPSOC_GLOBAL_CONF_ETR_FLUSH                                 0xC4B068
+
+#define mmPSOC_GLOBAL_CONF_COLD_RST_FLOPS_0                          0xC4B070
+
+#define mmPSOC_GLOBAL_CONF_COLD_RST_FLOPS_1                          0xC4B074
+
+#define mmPSOC_GLOBAL_CONF_COLD_RST_FLOPS_2                          0xC4B078
+
+#define mmPSOC_GLOBAL_CONF_COLD_RST_FLOPS_3                          0xC4B07C
+
+#define mmPSOC_GLOBAL_CONF_DIS_RAZWI_ERR                             0xC4B080
+
+#define mmPSOC_GLOBAL_CONF_PCIE_PHY_RST_N                            0xC4B084
+
+#define mmPSOC_GLOBAL_CONF_RAZWI                                     0xC4B088
+
+#define mmPSOC_GLOBAL_CONF_PROT                                      0xC4B090
+
+#define mmPSOC_GLOBAL_CONF_ADC                                       0xC4B094
+
+#define mmPSOC_GLOBAL_CONF_BOOT_SEQ_TO                               0xC4B098
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_0                              0xC4B100
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_1                              0xC4B104
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_2                              0xC4B108
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_3                              0xC4B10C
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_4                              0xC4B110
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_5                              0xC4B114
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_6                              0xC4B118
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_7                              0xC4B11C
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_8                              0xC4B120
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_9                              0xC4B124
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_10                             0xC4B128
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_11                             0xC4B12C
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_12                             0xC4B130
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_13                             0xC4B134
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_14                             0xC4B138
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_15                             0xC4B13C
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_16                             0xC4B140
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_17                             0xC4B144
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_18                             0xC4B148
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_19                             0xC4B14C
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_20                             0xC4B150
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_21                             0xC4B154
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_22                             0xC4B158
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_23                             0xC4B15C
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_24                             0xC4B160
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_25                             0xC4B164
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_26                             0xC4B168
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_27                             0xC4B16C
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_28                             0xC4B170
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_29                             0xC4B174
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_30                             0xC4B178
+
+#define mmPSOC_GLOBAL_CONF_SCRATCHPAD_31                             0xC4B17C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_0                               0xC4B200
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_1                               0xC4B204
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_2                               0xC4B208
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_3                               0xC4B20C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_4                               0xC4B210
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_5                               0xC4B214
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_6                               0xC4B218
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_7                               0xC4B21C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_8                               0xC4B220
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_9                               0xC4B224
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_10                              0xC4B228
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_11                              0xC4B22C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_12                              0xC4B230
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_13                              0xC4B234
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_14                              0xC4B238
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_15                              0xC4B23C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_16                              0xC4B240
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_17                              0xC4B244
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_18                              0xC4B248
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_19                              0xC4B24C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_20                              0xC4B250
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_21                              0xC4B254
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_22                              0xC4B258
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_23                              0xC4B25C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_24                              0xC4B260
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_25                              0xC4B264
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_26                              0xC4B268
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_27                              0xC4B26C
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_28                              0xC4B270
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_29                              0xC4B274
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_30                              0xC4B278
+
+#define mmPSOC_GLOBAL_CONF_SEMAPHORE_31                              0xC4B27C
+
+#define mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS                           0xC4B300
+
+#define mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU                            0xC4B304
+
+#define mmPSOC_GLOBAL_CONF_SPL_SOURCE                                0xC4B308
+
+#define mmPSOC_GLOBAL_CONF_I2C_MSTR1_DBG                             0xC4B30C
+
+#define mmPSOC_GLOBAL_CONF_I2C_SLV                                   0xC4B310
+
+#define mmPSOC_GLOBAL_CONF_I2C_SLV_INTR_MASK                         0xC4B314
+
+#define mmPSOC_GLOBAL_CONF_TRACE_ADDR                                0xC4B320
+
+#define mmPSOC_GLOBAL_CONF_ARUSER                                    0xC4B330
+
+#define mmPSOC_GLOBAL_CONF_AWUSER                                    0xC4B334
+
+#define mmPSOC_GLOBAL_CONF_TRACE_AWUSER                              0xC4B338
+
+#define mmPSOC_GLOBAL_CONF_TRACE_ARUSER                              0xC4B33C
+
+#define mmPSOC_GLOBAL_CONF_BTL_STS                                   0xC4B340
+
+#define mmPSOC_GLOBAL_CONF_TIMEOUT_INTR                              0xC4B350
+
+#define mmPSOC_GLOBAL_CONF_COMB_TIMEOUT_INTR                         0xC4B354
+
+#define mmPSOC_GLOBAL_CONF_PERIPH_INTR                               0xC4B358
+
+#define mmPSOC_GLOBAL_CONF_COMB_PERIPH_INTR                          0xC4B35C
+
+#define mmPSOC_GLOBAL_CONF_AXI_ERR_INTR                              0xC4B360
+
+#define mmPSOC_GLOBAL_CONF_TARGETID                                  0xC4B400
+
+#define mmPSOC_GLOBAL_CONF_EMMC_INT_VOL_STABLE                       0xC4B420
+
+#define mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS                           0xC4B430
+
+#define mmPSOC_GLOBAL_CONF_MEM_REPAIR_DIV                            0xC4B44C
+
+#define mmPSOC_GLOBAL_CONF_MEM_REPAIR_CTRL                           0xC4B450
+
+#define mmPSOC_GLOBAL_CONF_MEM_REPAIR_STS                            0xC4B454
+
+#define mmPSOC_GLOBAL_CONF_OUTSTANT_TRANS                            0xC4B458
+
+#define mmPSOC_GLOBAL_CONF_MASK_REQ                                  0xC4B45C
+
+#define mmPSOC_GLOBAL_CONF_WD_RST_CFG_L                              0xC4B460
+
+#define mmPSOC_GLOBAL_CONF_WD_RST_CFG_H                              0xC4B464
+
+#define mmPSOC_GLOBAL_CONF_MNL_RST_CFG_L                             0xC4B470
+
+#define mmPSOC_GLOBAL_CONF_MNL_RST_CFG_H                             0xC4B474
+
+#define mmPSOC_GLOBAL_CONF_PRSTN_RST_CFG_L                           0xC4B480
+
+#define mmPSOC_GLOBAL_CONF_PRSTN_RST_CFG_H                           0xC4B484
+
+#define mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L                          0xC4B490
+
+#define mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H                          0xC4B494
+
+#define mmPSOC_GLOBAL_CONF_SW_ALL_RST                                0xC4B498
+
+#define mmPSOC_GLOBAL_CONF_SOFT_RST                                  0xC4B4A0
+
+#define mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_L                            0xC4B4A4
+
+#define mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_H                            0xC4B4A8
+
+#define mmPSOC_GLOBAL_CONF_UNIT_RST_N                                0xC4B4B0
+
+#define mmPSOC_GLOBAL_CONF_UNIT_RST_N_L                              0xC4B4B4
+
+#define mmPSOC_GLOBAL_CONF_UNIT_RST_N_H                              0xC4B4B8
+
+#define mmPSOC_GLOBAL_CONF_BTL_IMG                                   0xC4B4E0
+
+#define mmPSOC_GLOBAL_CONF_PRSTN_MASK                                0xC4B4E4
+
+#define mmPSOC_GLOBAL_CONF_WD_MASK                                   0xC4B4E8
+
+#define mmPSOC_GLOBAL_CONF_RST_SRC                                   0xC4B4F0
+
+#define mmPSOC_GLOBAL_CONF_BOOT_STATE                                0xC4B4F4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_0                             0xC4B500
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_1                             0xC4B504
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_2                             0xC4B508
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_3                             0xC4B50C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_4                             0xC4B510
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_5                             0xC4B514
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_6                             0xC4B518
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_7                             0xC4B51C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_8                             0xC4B520
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_9                             0xC4B524
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_10                            0xC4B528
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_11                            0xC4B52C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_12                            0xC4B530
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_13                            0xC4B534
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_14                            0xC4B538
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_15                            0xC4B53C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_16                            0xC4B540
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_17                            0xC4B544
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_18                            0xC4B548
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_19                            0xC4B54C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_20                            0xC4B550
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_21                            0xC4B554
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_22                            0xC4B558
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_23                            0xC4B55C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_24                            0xC4B560
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_25                            0xC4B564
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_26                            0xC4B568
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_27                            0xC4B56C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_28                            0xC4B570
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_29                            0xC4B574
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_30                            0xC4B578
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_31                            0xC4B57C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_32                            0xC4B580
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_33                            0xC4B584
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_34                            0xC4B588
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_35                            0xC4B58C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_36                            0xC4B590
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_37                            0xC4B594
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_38                            0xC4B598
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_39                            0xC4B59C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_40                            0xC4B5A0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_41                            0xC4B5A4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_42                            0xC4B5A8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_43                            0xC4B5AC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_44                            0xC4B5B0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_45                            0xC4B5B4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_46                            0xC4B5B8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_47                            0xC4B5BC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_48                            0xC4B5C0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_49                            0xC4B5C4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_50                            0xC4B5C8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_51                            0xC4B5CC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_52                            0xC4B5D0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_53                            0xC4B5D4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_54                            0xC4B5D8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_55                            0xC4B5DC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_56                            0xC4B5E0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_57                            0xC4B5E4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_58                            0xC4B5E8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_59                            0xC4B5EC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_60                            0xC4B5F0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_61                            0xC4B5F4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_62                            0xC4B5F8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_63                            0xC4B5FC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_64                            0xC4B600
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_65                            0xC4B604
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_66                            0xC4B608
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_67                            0xC4B60C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_68                            0xC4B610
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_69                            0xC4B614
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_70                            0xC4B618
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_71                            0xC4B61C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_72                            0xC4B620
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_73                            0xC4B624
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_74                            0xC4B628
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_75                            0xC4B62C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_76                            0xC4B630
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_77                            0xC4B634
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_78                            0xC4B638
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_79                            0xC4B63C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_80                            0xC4B640
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_81                            0xC4B644
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_82                            0xC4B648
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_83                            0xC4B64C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_84                            0xC4B650
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_85                            0xC4B654
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_86                            0xC4B658
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_87                            0xC4B65C
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_88                            0xC4B660
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_CFG_89                            0xC4B664
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_0                             0xC4B690
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_1                             0xC4B694
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_2                             0xC4B698
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_3                             0xC4B69C
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_4                             0xC4B6A0
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_5                             0xC4B6A4
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_6                             0xC4B6A8
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_7                             0xC4B6AC
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_8                             0xC4B6B0
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_9                             0xC4B6B4
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_10                            0xC4B6B8
+
+#define mmPSOC_GLOBAL_CONF_PAD_3V3_CFG_11                            0xC4B6BC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_0                           0xC4B6C0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_1                           0xC4B6C4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_2                           0xC4B6C8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_3                           0xC4B6CC
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_4                           0xC4B6D0
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_5                           0xC4B6D4
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_6                           0xC4B6D8
+
+#define mmPSOC_GLOBAL_CONF_PAD_1V8_INPUT_7                           0xC4B6DC
+
+#define mmPSOC_GLOBAL_CONF_BNK3V3_MS                                 0xC4B710
+
+#define mmPSOC_GLOBAL_CONF_ADC_CLK_FREQ                              0xC4B720
+
+#define mmPSOC_GLOBAL_CONF_ADC_DELAY_FROM_START                      0xC4B724
+
+#define mmPSOC_GLOBAL_CONF_ADC_DATA_SAMPLES                          0xC4B728
+
+#define mmPSOC_GLOBAL_CONF_ADC_TPH_CS                                0xC4B72C
+
+#define mmPSOC_GLOBAL_CONF_ADC_LSB_NMSB                              0xC4B730
+
+#define mmPSOC_GLOBAL_CONF_ADC_ONE_NCONTIUES                         0xC4B734
+
+#define mmPSOC_GLOBAL_CONF_ADC_BLOCK_ENABLE                          0xC4B738
+
+#define mmPSOC_GLOBAL_CONF_ADC_CFG_DATA                              0xC4B73C
+
+#define mmPSOC_GLOBAL_CONF_ADC_TDV_CSDO                              0xC4B740
+
+#define mmPSOC_GLOBAL_CONF_ADC_TSU_CSCK                              0xC4B744
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_0                             0xC4B800
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_1                             0xC4B804
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_2                             0xC4B808
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_3                             0xC4B80C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_4                             0xC4B810
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_5                             0xC4B814
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_6                             0xC4B818
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_7                             0xC4B81C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_8                             0xC4B820
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_9                             0xC4B824
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_10                            0xC4B828
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_11                            0xC4B82C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_12                            0xC4B830
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_13                            0xC4B834
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_14                            0xC4B838
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_15                            0xC4B83C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_16                            0xC4B840
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_17                            0xC4B844
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_18                            0xC4B848
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_19                            0xC4B84C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_20                            0xC4B850
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_21                            0xC4B854
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_22                            0xC4B858
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_23                            0xC4B85C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_24                            0xC4B860
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_25                            0xC4B864
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_26                            0xC4B868
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_27                            0xC4B86C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_28                            0xC4B870
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_29                            0xC4B874
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_30                            0xC4B878
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_31                            0xC4B87C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_32                            0xC4B880
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_33                            0xC4B884
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_34                            0xC4B888
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_35                            0xC4B88C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_36                            0xC4B890
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_37                            0xC4B894
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_38                            0xC4B898
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_39                            0xC4B89C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_40                            0xC4B8A0
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_41                            0xC4B8A4
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_42                            0xC4B8A8
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_43                            0xC4B8AC
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_44                            0xC4B8B0
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_45                            0xC4B8B4
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_46                            0xC4B8B8
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_47                            0xC4B8BC
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_48                            0xC4B8C0
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_49                            0xC4B8C4
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_50                            0xC4B8C8
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_51                            0xC4B8CC
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_52                            0xC4B8D0
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_53                            0xC4B8D4
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_54                            0xC4B8D8
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_55                            0xC4B8DC
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_56                            0xC4B8E0
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_57                            0xC4B8E4
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_58                            0xC4B8E8
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_59                            0xC4B8EC
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_60                            0xC4B8F0
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_61                            0xC4B8F4
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_62                            0xC4B8F8
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_63                            0xC4B8FC
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_64                            0xC4B900
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_65                            0xC4B904
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_66                            0xC4B908
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_67                            0xC4B90C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_68                            0xC4B910
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_69                            0xC4B914
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_70                            0xC4B918
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_71                            0xC4B91C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_72                            0xC4B920
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_73                            0xC4B924
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_74                            0xC4B928
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_75                            0xC4B92C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_76                            0xC4B930
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_77                            0xC4B934
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_78                            0xC4B938
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_79                            0xC4B93C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_80                            0xC4B940
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_81                            0xC4B944
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_82                            0xC4B948
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_83                            0xC4B94C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_84                            0xC4B950
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_85                            0xC4B954
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_86                            0xC4B958
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_87                            0xC4B95C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_88                            0xC4B960
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_89                            0xC4B964
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_90                            0xC4B968
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_91                            0xC4B96C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_92                            0xC4B970
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_93                            0xC4B974
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_94                            0xC4B978
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_95                            0xC4B97C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_96                            0xC4B980
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_97                            0xC4B984
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_98                            0xC4B988
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_99                            0xC4B98C
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_100                           0xC4B990
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_101                           0xC4B994
+
+#define mmPSOC_GLOBAL_CONF_PAD_DEFAULT_102                           0xC4B998
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_0                                 0xC4BA00
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_1                                 0xC4BA04
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_2                                 0xC4BA08
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_3                                 0xC4BA0C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_4                                 0xC4BA10
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_5                                 0xC4BA14
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_6                                 0xC4BA18
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_7                                 0xC4BA1C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_8                                 0xC4BA20
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_9                                 0xC4BA24
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_10                                0xC4BA28
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_11                                0xC4BA2C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_12                                0xC4BA30
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_13                                0xC4BA34
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_14                                0xC4BA38
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_15                                0xC4BA3C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_16                                0xC4BA40
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_17                                0xC4BA44
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_18                                0xC4BA48
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_19                                0xC4BA4C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_20                                0xC4BA50
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_21                                0xC4BA54
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_22                                0xC4BA58
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_23                                0xC4BA5C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_24                                0xC4BA60
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_25                                0xC4BA64
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_26                                0xC4BA68
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_27                                0xC4BA6C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_28                                0xC4BA70
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_29                                0xC4BA74
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_30                                0xC4BA78
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_31                                0xC4BA7C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_32                                0xC4BA80
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_33                                0xC4BA84
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_34                                0xC4BA88
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_35                                0xC4BA8C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_36                                0xC4BA90
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_37                                0xC4BA94
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_38                                0xC4BA98
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_39                                0xC4BA9C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_40                                0xC4BAA0
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_41                                0xC4BAA4
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_42                                0xC4BAA8
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_43                                0xC4BAAC
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_44                                0xC4BAB0
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_45                                0xC4BAB4
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_46                                0xC4BAB8
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_47                                0xC4BABC
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_48                                0xC4BAC0
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_49                                0xC4BAC4
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_50                                0xC4BAC8
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_51                                0xC4BACC
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_52                                0xC4BAD0
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_53                                0xC4BAD4
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_54                                0xC4BAD8
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_55                                0xC4BADC
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_56                                0xC4BAE0
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_57                                0xC4BAE4
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_58                                0xC4BAE8
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_59                                0xC4BAEC
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_60                                0xC4BAF0
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_61                                0xC4BAF4
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_62                                0xC4BAF8
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_63                                0xC4BAFC
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_64                                0xC4BB00
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_65                                0xC4BB04
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_66                                0xC4BB08
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_67                                0xC4BB0C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_68                                0xC4BB10
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_69                                0xC4BB14
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_70                                0xC4BB18
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_71                                0xC4BB1C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_72                                0xC4BB20
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_73                                0xC4BB24
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_74                                0xC4BB28
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_75                                0xC4BB2C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_76                                0xC4BB30
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_77                                0xC4BB34
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_78                                0xC4BB38
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_79                                0xC4BB3C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_80                                0xC4BB40
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_81                                0xC4BB44
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_82                                0xC4BB48
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_83                                0xC4BB4C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_84                                0xC4BB50
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_85                                0xC4BB54
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_86                                0xC4BB58
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_87                                0xC4BB5C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_88                                0xC4BB60
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_89                                0xC4BB64
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_90                                0xC4BB68
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_91                                0xC4BB6C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_92                                0xC4BB70
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_93                                0xC4BB74
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_94                                0xC4BB78
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_95                                0xC4BB7C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_96                                0xC4BB80
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_97                                0xC4BB84
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_98                                0xC4BB88
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_99                                0xC4BB8C
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_100                               0xC4BB90
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_101                               0xC4BB94
+
+#define mmPSOC_GLOBAL_CONF_PAD_SEL_102                               0xC4BB98
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_0                                0xC4BC00
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_1                                0xC4BC04
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_2                                0xC4BC08
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_3                                0xC4BC0C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_4                                0xC4BC10
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_5                                0xC4BC14
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_6                                0xC4BC18
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_7                                0xC4BC1C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_8                                0xC4BC20
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_9                                0xC4BC24
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_10                               0xC4BC28
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_11                               0xC4BC2C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_12                               0xC4BC30
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_13                               0xC4BC34
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_14                               0xC4BC38
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_15                               0xC4BC3C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_16                               0xC4BC40
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_17                               0xC4BC44
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_18                               0xC4BC48
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_19                               0xC4BC4C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_20                               0xC4BC50
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_21                               0xC4BC54
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_22                               0xC4BC58
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_23                               0xC4BC5C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_24                               0xC4BC60
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_25                               0xC4BC64
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_26                               0xC4BC68
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_27                               0xC4BC6C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_28                               0xC4BC70
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_29                               0xC4BC74
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_30                               0xC4BC78
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_31                               0xC4BC7C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_32                               0xC4BC80
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_33                               0xC4BC84
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_34                               0xC4BC88
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_35                               0xC4BC8C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_36                               0xC4BC90
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_37                               0xC4BC94
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_38                               0xC4BC98
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_39                               0xC4BC9C
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_40                               0xC4BCA0
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_41                               0xC4BCA4
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_42                               0xC4BCA8
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_43                               0xC4BCAC
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_44                               0xC4BCB0
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_45                               0xC4BCB4
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_46                               0xC4BCB8
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_47                               0xC4BCBC
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_48                               0xC4BCC0
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_49                               0xC4BCC4
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_50                               0xC4BCC8
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_51                               0xC4BCCC
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_52                               0xC4BCD0
+
+#define mmPSOC_GLOBAL_CONF_RST_CTRL_53                               0xC4BCD4
+
+#endif /* ASIC_REG_PSOC_GLOBAL_CONF_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h
new file mode 100644
index 000000000000..687e2255cb19
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_PSOC_HBM_PLL_REGS_H_
+#define ASIC_REG_PSOC_HBM_PLL_REGS_H_
+
+/*
+ *****************************************
+ *   PSOC_HBM_PLL (Prototype: PLL)
+ *****************************************
+ */
+
+#define mmPSOC_HBM_PLL_NR                                            0xC74100
+
+#define mmPSOC_HBM_PLL_NF                                            0xC74104
+
+#define mmPSOC_HBM_PLL_OD                                            0xC74108
+
+#define mmPSOC_HBM_PLL_NB                                            0xC7410C
+
+#define mmPSOC_HBM_PLL_CFG                                           0xC74110
+
+#define mmPSOC_HBM_PLL_LOSE_MASK                                     0xC74120
+
+#define mmPSOC_HBM_PLL_LOCK_INTR                                     0xC74128
+
+#define mmPSOC_HBM_PLL_LOCK_BYPASS                                   0xC7412C
+
+#define mmPSOC_HBM_PLL_DATA_CHNG                                     0xC74130
+
+#define mmPSOC_HBM_PLL_RST                                           0xC74134
+
+#define mmPSOC_HBM_PLL_SLIP_WD_CNTR                                  0xC74150
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_0                                  0xC74200
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_1                                  0xC74204
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_2                                  0xC74208
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_3                                  0xC7420C
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_0                              0xC74220
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_1                              0xC74224
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_2                              0xC74228
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_3                              0xC7422C
+
+#define mmPSOC_HBM_PLL_DIV_SEL_0                                     0xC74280
+
+#define mmPSOC_HBM_PLL_DIV_SEL_1                                     0xC74284
+
+#define mmPSOC_HBM_PLL_DIV_SEL_2                                     0xC74288
+
+#define mmPSOC_HBM_PLL_DIV_SEL_3                                     0xC7428C
+
+#define mmPSOC_HBM_PLL_DIV_EN_0                                      0xC742A0
+
+#define mmPSOC_HBM_PLL_DIV_EN_1                                      0xC742A4
+
+#define mmPSOC_HBM_PLL_DIV_EN_2                                      0xC742A8
+
+#define mmPSOC_HBM_PLL_DIV_EN_3                                      0xC742AC
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_0                             0xC742C0
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_1                             0xC742C4
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_2                             0xC742C8
+
+#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_3                             0xC742CC
+
+#define mmPSOC_HBM_PLL_CLK_GATER                                     0xC74300
+
+#define mmPSOC_HBM_PLL_CLK_RLX_0                                     0xC74310
+
+#define mmPSOC_HBM_PLL_CLK_RLX_1                                     0xC74314
+
+#define mmPSOC_HBM_PLL_CLK_RLX_2                                     0xC74318
+
+#define mmPSOC_HBM_PLL_CLK_RLX_3                                     0xC7431C
+
+#define mmPSOC_HBM_PLL_REF_CNTR_PERIOD                               0xC74400
+
+#define mmPSOC_HBM_PLL_REF_LOW_THRESHOLD                             0xC74410
+
+#define mmPSOC_HBM_PLL_REF_HIGH_THRESHOLD                            0xC74420
+
+#define mmPSOC_HBM_PLL_PLL_NOT_STABLE                                0xC74430
+
+#define mmPSOC_HBM_PLL_FREQ_CALC_EN                                  0xC74440
+
+#define mmPSOC_HBM_PLL_RLX_BITMAP_CFG                                0xC74500
+
+#define mmPSOC_HBM_PLL_RLX_BITMAP_0                                  0xC74510
+
+#define mmPSOC_HBM_PLL_RLX_BITMAP_1                                  0xC74514
+
+#define mmPSOC_HBM_PLL_RLX_BITMAP_2                                  0xC74518
+
+#define mmPSOC_HBM_PLL_RLX_BITMAP_3                                  0xC7451C
+
+#endif /* ASIC_REG_PSOC_HBM_PLL_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h
new file mode 100644
index 000000000000..3dc9bb4542dd
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_PSOC_PCI_PLL_REGS_H_
+#define ASIC_REG_PSOC_PCI_PLL_REGS_H_
+
+/*
+ *****************************************
+ *   PSOC_PCI_PLL (Prototype: PLL)
+ *****************************************
+ */
+
+#define mmPSOC_PCI_PLL_NR                                            0xC72100
+
+#define mmPSOC_PCI_PLL_NF                                            0xC72104
+
+#define mmPSOC_PCI_PLL_OD                                            0xC72108
+
+#define mmPSOC_PCI_PLL_NB                                            0xC7210C
+
+#define mmPSOC_PCI_PLL_CFG                                           0xC72110
+
+#define mmPSOC_PCI_PLL_LOSE_MASK                                     0xC72120
+
+#define mmPSOC_PCI_PLL_LOCK_INTR                                     0xC72128
+
+#define mmPSOC_PCI_PLL_LOCK_BYPASS                                   0xC7212C
+
+#define mmPSOC_PCI_PLL_DATA_CHNG                                     0xC72130
+
+#define mmPSOC_PCI_PLL_RST                                           0xC72134
+
+#define mmPSOC_PCI_PLL_SLIP_WD_CNTR                                  0xC72150
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_0                                  0xC72200
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_1                                  0xC72204
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_2                                  0xC72208
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_3                                  0xC7220C
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_0                              0xC72220
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_1                              0xC72224
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_2                              0xC72228
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_3                              0xC7222C
+
+#define mmPSOC_PCI_PLL_DIV_SEL_0                                     0xC72280
+
+#define mmPSOC_PCI_PLL_DIV_SEL_1                                     0xC72284
+
+#define mmPSOC_PCI_PLL_DIV_SEL_2                                     0xC72288
+
+#define mmPSOC_PCI_PLL_DIV_SEL_3                                     0xC7228C
+
+#define mmPSOC_PCI_PLL_DIV_EN_0                                      0xC722A0
+
+#define mmPSOC_PCI_PLL_DIV_EN_1                                      0xC722A4
+
+#define mmPSOC_PCI_PLL_DIV_EN_2                                      0xC722A8
+
+#define mmPSOC_PCI_PLL_DIV_EN_3                                      0xC722AC
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_0                             0xC722C0
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_1                             0xC722C4
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_2                             0xC722C8
+
+#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_3                             0xC722CC
+
+#define mmPSOC_PCI_PLL_CLK_GATER                                     0xC72300
+
+#define mmPSOC_PCI_PLL_CLK_RLX_0                                     0xC72310
+
+#define mmPSOC_PCI_PLL_CLK_RLX_1                                     0xC72314
+
+#define mmPSOC_PCI_PLL_CLK_RLX_2                                     0xC72318
+
+#define mmPSOC_PCI_PLL_CLK_RLX_3                                     0xC7231C
+
+#define mmPSOC_PCI_PLL_REF_CNTR_PERIOD                               0xC72400
+
+#define mmPSOC_PCI_PLL_REF_LOW_THRESHOLD                             0xC72410
+
+#define mmPSOC_PCI_PLL_REF_HIGH_THRESHOLD                            0xC72420
+
+#define mmPSOC_PCI_PLL_PLL_NOT_STABLE                                0xC72430
+
+#define mmPSOC_PCI_PLL_FREQ_CALC_EN                                  0xC72440
+
+#define mmPSOC_PCI_PLL_RLX_BITMAP_CFG                                0xC72500
+
+#define mmPSOC_PCI_PLL_RLX_BITMAP_0                                  0xC72510
+
+#define mmPSOC_PCI_PLL_RLX_BITMAP_1                                  0xC72514
+
+#define mmPSOC_PCI_PLL_RLX_BITMAP_2                                  0xC72518
+
+#define mmPSOC_PCI_PLL_RLX_BITMAP_3                                  0xC7251C
+
+#endif /* ASIC_REG_PSOC_PCI_PLL_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_timestamp_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_timestamp_regs.h
new file mode 100644
index 000000000000..9ce24597d4b0
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_timestamp_regs.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_PSOC_TIMESTAMP_REGS_H_
+#define ASIC_REG_PSOC_TIMESTAMP_REGS_H_
+
+/*
+ *****************************************
+ *   PSOC_TIMESTAMP (Prototype: TIMESTAMP)
+ *****************************************
+ */
+
+#define mmPSOC_TIMESTAMP_CNTCR                                       0xC49000
+
+#define mmPSOC_TIMESTAMP_CNTSR                                       0xC49004
+
+#define mmPSOC_TIMESTAMP_CNTCVL                                      0xC49008
+
+#define mmPSOC_TIMESTAMP_CNTCVU                                      0xC4900C
+
+#define mmPSOC_TIMESTAMP_CNTFID0                                     0xC49020
+
+#define mmPSOC_TIMESTAMP_PIDR4                                       0xC49FD0
+
+#define mmPSOC_TIMESTAMP_PIDR5                                       0xC49FD4
+
+#define mmPSOC_TIMESTAMP_PIDR6                                       0xC49FD8
+
+#define mmPSOC_TIMESTAMP_PIDR7                                       0xC49FDC
+
+#define mmPSOC_TIMESTAMP_PIDR0                                       0xC49FE0
+
+#define mmPSOC_TIMESTAMP_PIDR1                                       0xC49FE4
+
+#define mmPSOC_TIMESTAMP_PIDR2                                       0xC49FE8
+
+#define mmPSOC_TIMESTAMP_PIDR3                                       0xC49FEC
+
+#define mmPSOC_TIMESTAMP_CIDR0                                       0xC49FF0
+
+#define mmPSOC_TIMESTAMP_CIDR1                                       0xC49FF4
+
+#define mmPSOC_TIMESTAMP_CIDR2                                       0xC49FF8
+
+#define mmPSOC_TIMESTAMP_CIDR3                                       0xC49FFC
+
+#endif /* ASIC_REG_PSOC_TIMESTAMP_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_0_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_0_regs.h
new file mode 100644
index 000000000000..ddf824392cf7
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_0_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_0_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_0_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_0 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_0_PERM_SEL                                    0x306108
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_0                               0x306114
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_1                               0x306118
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_2                               0x30611C
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_3                               0x306120
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_4                               0x306124
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_5                               0x306128
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_6                               0x30612C
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_7                               0x306130
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_8                               0x306134
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_9                               0x306138
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_10                              0x30613C
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_11                              0x306140
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_12                              0x306144
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_13                              0x306148
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_14                              0x30614C
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_15                              0x306150
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_16                              0x306154
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_17                              0x306158
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_18                              0x30615C
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_19                              0x306160
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_20                              0x306164
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_21                              0x306168
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_22                              0x30616C
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_23                              0x306170
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_24                              0x306174
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_25                              0x306178
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_26                              0x30617C
+
+#define mmSIF_RTR_CTRL_0_HBM_POLY_H3_27                              0x306180
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_0                              0x306184
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_1                              0x306188
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_2                              0x30618C
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_3                              0x306190
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_4                              0x306194
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_5                              0x306198
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_6                              0x30619C
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_7                              0x3061A0
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_8                              0x3061A4
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_9                              0x3061A8
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_10                             0x3061AC
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_11                             0x3061B0
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_12                             0x3061B4
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_13                             0x3061B8
+
+#define mmSIF_RTR_CTRL_0_SRAM_POLY_H3_14                             0x3061BC
+
+#define mmSIF_RTR_CTRL_0_SCRAM_SRAM_EN                               0x30626C
+
+#define mmSIF_RTR_CTRL_0_RL_HBM_EN                                   0x306274
+
+#define mmSIF_RTR_CTRL_0_RL_HBM_SAT                                  0x306278
+
+#define mmSIF_RTR_CTRL_0_RL_HBM_RST                                  0x30627C
+
+#define mmSIF_RTR_CTRL_0_RL_HBM_TIMEOUT                              0x306280
+
+#define mmSIF_RTR_CTRL_0_SCRAM_HBM_EN                                0x306284
+
+#define mmSIF_RTR_CTRL_0_RL_PCI_EN                                   0x306288
+
+#define mmSIF_RTR_CTRL_0_RL_PCI_SAT                                  0x30628C
+
+#define mmSIF_RTR_CTRL_0_RL_PCI_RST                                  0x306290
+
+#define mmSIF_RTR_CTRL_0_RL_PCI_TIMEOUT                              0x306294
+
+#define mmSIF_RTR_CTRL_0_RL_SRAM_EN                                  0x30629C
+
+#define mmSIF_RTR_CTRL_0_RL_SRAM_SAT                                 0x3062A0
+
+#define mmSIF_RTR_CTRL_0_RL_SRAM_RST                                 0x3062A4
+
+#define mmSIF_RTR_CTRL_0_RL_SRAM_TIMEOUT                             0x3062AC
+
+#define mmSIF_RTR_CTRL_0_RL_SRAM_RED                                 0x3062B4
+
+#define mmSIF_RTR_CTRL_0_E2E_HBM_EN                                  0x3062EC
+
+#define mmSIF_RTR_CTRL_0_E2E_PCI_EN                                  0x3062F0
+
+#define mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE                             0x3062F4
+
+#define mmSIF_RTR_CTRL_0_E2E_PCI_WR_SIZE                             0x3062F8
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_PCI_CTR_SET_EN                       0x306404
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_PCI_CTR_SET                          0x306408
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_PCI_CTR_WRAP                         0x30640C
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_PCI_CTR_CNT                          0x306410
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM_CTR_SET_EN                       0x306414
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM_CTR_SET                          0x306418
+
+#define mmSIF_RTR_CTRL_0_E2E_HBM_RD_SIZE                             0x30641C
+
+#define mmSIF_RTR_CTRL_0_E2E_PCI_RD_SIZE                             0x306420
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_PCI_CTR_SET_EN                       0x306424
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_PCI_CTR_SET                          0x306428
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_PCI_CTR_WRAP                         0x30642C
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_PCI_CTR_CNT                          0x306430
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM_CTR_SET_EN                       0x306434
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM_CTR_SET                          0x306438
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_SEL_0                                0x306450
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_SEL_1                                0x306454
+
+#define mmSIF_RTR_CTRL_0_NON_LIN_EN                                  0x306480
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_BANK_0                              0x306500
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_BANK_1                              0x306504
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_BANK_2                              0x306508
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_BANK_3                              0x30650C
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_BANK_4                              0x306510
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_0                            0x306514
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_1                            0x306520
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_2                            0x306524
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_3                            0x306528
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_4                            0x30652C
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_5                            0x306530
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_6                            0x306534
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_7                            0x306538
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_8                            0x30653C
+
+#define mmSIF_RTR_CTRL_0_NL_SRAM_OFFSET_9                            0x306540
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_0                             0x306550
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_1                             0x306554
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_2                             0x306558
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_3                             0x30655C
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_4                             0x306560
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_5                             0x306564
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_6                             0x306568
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_7                             0x30656C
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_8                             0x306570
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_9                             0x306574
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_10                            0x306578
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_11                            0x30657C
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_12                            0x306580
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_13                            0x306584
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_14                            0x306588
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_15                            0x30658C
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_16                            0x306590
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_17                            0x306594
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_OFFSET_18                            0x306598
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_0                     0x3065E4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_1                     0x3065E8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_2                     0x3065EC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_3                     0x3065F0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_4                     0x3065F4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_5                     0x3065F8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_6                     0x3065FC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_7                     0x306600
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_8                     0x306604
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_9                     0x306608
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_10                    0x30660C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_11                    0x306610
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_12                    0x306614
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_13                    0x306618
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_14                    0x30661C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AW_15                    0x306620
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_0                    0x306624
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_1                    0x306628
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_2                    0x30662C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_3                    0x306630
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_4                    0x306634
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_5                    0x306638
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_6                    0x30663C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_7                    0x306640
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_8                    0x306644
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_9                    0x306648
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_10                   0x30664C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_11                   0x306650
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_12                   0x306654
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_13                   0x306658
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_14                   0x30665C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AW_15                   0x306660
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_0                     0x306664
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_1                     0x306668
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_2                     0x30666C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_3                     0x306670
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_4                     0x306674
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_5                     0x306678
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_6                     0x30667C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_7                     0x306680
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_8                     0x306684
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_9                     0x306688
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_10                    0x30668C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_11                    0x306690
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_12                    0x306694
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_13                    0x306698
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_14                    0x30669C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AW_15                    0x3066A0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_0                    0x3066A4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_1                    0x3066A8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_2                    0x3066AC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_3                    0x3066B0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_4                    0x3066B4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_5                    0x3066B8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_6                    0x3066BC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_7                    0x3066C0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_8                    0x3066C4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_9                    0x3066C8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_10                   0x3066CC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_11                   0x3066D0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_12                   0x3066D4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_13                   0x3066D8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_14                   0x3066DC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AW_15                   0x3066E0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_0                    0x3066E4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_1                    0x3066E8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_2                    0x3066EC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_3                    0x3066F0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_4                    0x3066F4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_5                    0x3066F8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_6                    0x3066FC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_7                    0x306700
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_8                    0x306704
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_9                    0x306708
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_10                   0x30670C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_11                   0x306710
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_12                   0x306714
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_13                   0x306718
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_14                   0x30671C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AW_15                   0x306720
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_0                   0x306724
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_1                   0x306728
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_2                   0x30672C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_3                   0x306730
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_4                   0x306734
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_5                   0x306738
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_6                   0x30673C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_7                   0x306740
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_8                   0x306744
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_9                   0x306748
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_10                  0x30674C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_11                  0x306750
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_12                  0x306754
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_13                  0x306758
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_14                  0x30675C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AW_15                  0x306760
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_0                    0x306764
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_1                    0x306768
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_2                    0x30676C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_3                    0x306770
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_4                    0x306774
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_5                    0x306778
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_6                    0x30677C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_7                    0x306780
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_8                    0x306784
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_9                    0x306788
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_10                   0x30678C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_11                   0x306790
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_12                   0x306794
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_13                   0x306798
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_14                   0x30679C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AW_15                   0x3067A0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_0                   0x3067A4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_1                   0x3067A8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_2                   0x3067AC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_3                   0x3067B0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_4                   0x3067B4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_5                   0x3067B8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_6                   0x3067BC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_7                   0x3067C0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_8                   0x3067C4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_9                   0x3067C8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_10                  0x3067CC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_11                  0x3067D0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_12                  0x3067D4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_13                  0x3067D8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_14                  0x3067DC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AW_15                  0x3067E0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_0                     0x306824
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_1                     0x306828
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_2                     0x30682C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_3                     0x306830
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_4                     0x306834
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_5                     0x306838
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_6                     0x30683C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_7                     0x306840
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_8                     0x306844
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_9                     0x306848
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_10                    0x30684C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_11                    0x306850
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_12                    0x306854
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_13                    0x306858
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_14                    0x30685C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_LOW_AR_15                    0x306860
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_0                    0x306864
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_1                    0x306868
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_2                    0x30686C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_3                    0x306870
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_4                    0x306874
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_5                    0x306878
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_6                    0x30687C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_7                    0x306880
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_8                    0x306884
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_9                    0x306888
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_10                   0x30688C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_11                   0x306890
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_12                   0x306894
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_13                   0x306898
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_14                   0x30689C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_BASE_HIGH_AR_15                   0x3068A0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_0                     0x3068A4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_1                     0x3068A8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_2                     0x3068AC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_3                     0x3068B0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_4                     0x3068B4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_5                     0x3068B8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_6                     0x3068BC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_7                     0x3068C0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_8                     0x3068C4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_9                     0x3068C8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_10                    0x3068CC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_11                    0x3068D0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_12                    0x3068D4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_13                    0x3068D8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_14                    0x3068DC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_LOW_AR_15                    0x3068E0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_0                    0x3068E4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_1                    0x3068E8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_2                    0x3068EC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_3                    0x3068F0
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_4                    0x3068F4
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_5                    0x3068F8
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_6                    0x3068FC
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_7                    0x306900
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_8                    0x306904
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_9                    0x306908
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_10                   0x30690C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_11                   0x306910
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_12                   0x306914
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_13                   0x306918
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_14                   0x30691C
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_MASK_HIGH_AR_15                   0x306920
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_0                    0x306924
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_1                    0x306928
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_2                    0x30692C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_3                    0x306930
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_4                    0x306934
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_5                    0x306938
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_6                    0x30693C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_7                    0x306940
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_8                    0x306944
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_9                    0x306948
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_10                   0x30694C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_11                   0x306950
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_12                   0x306954
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_13                   0x306958
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_14                   0x30695C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_LOW_AR_15                   0x306960
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_0                   0x306964
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_1                   0x306968
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_2                   0x30696C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_3                   0x306970
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_4                   0x306974
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_5                   0x306978
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_6                   0x30697C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_7                   0x306980
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_8                   0x306984
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_9                   0x306988
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_10                  0x30698C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_11                  0x306990
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_12                  0x306994
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_13                  0x306998
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_14                  0x30699C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_BASE_HIGH_AR_15                  0x3069A0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_0                    0x3069A4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_1                    0x3069A8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_2                    0x3069AC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_3                    0x3069B0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_4                    0x3069B4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_5                    0x3069B8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_6                    0x3069BC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_7                    0x3069C0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_8                    0x3069C4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_9                    0x3069C8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_10                   0x3069CC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_11                   0x3069D0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_12                   0x3069D4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_13                   0x3069D8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_14                   0x3069DC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_LOW_AR_15                   0x3069E0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_0                   0x3069E4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_1                   0x3069E8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_2                   0x3069EC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_3                   0x3069F0
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_4                   0x3069F4
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_5                   0x3069F8
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_6                   0x3069FC
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_7                   0x306A00
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_8                   0x306A04
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_9                   0x306A08
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_10                  0x306A0C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_11                  0x306A10
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_12                  0x306A14
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_13                  0x306A18
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_14                  0x306A1C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_MASK_HIGH_AR_15                  0x306A20
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_HIT_AW                            0x306A64
+
+#define mmSIF_RTR_CTRL_0_RANGE_SEC_HIT_AR                            0x306A68
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_HIT_AW                           0x306A6C
+
+#define mmSIF_RTR_CTRL_0_RANGE_PRIV_HIT_AR                           0x306A70
+
+#define mmSIF_RTR_CTRL_0_RGL_CFG                                     0x306B64
+
+#define mmSIF_RTR_CTRL_0_RGL_SHIFT                                   0x306B68
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_0                          0x306B6C
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_1                          0x306B70
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_2                          0x306B74
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_3                          0x306B78
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_4                          0x306B7C
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_5                          0x306B80
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_6                          0x306B84
+
+#define mmSIF_RTR_CTRL_0_RGL_EXPECTED_LAT_7                          0x306B88
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_0                                 0x306BAC
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_1                                 0x306BB0
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_2                                 0x306BB4
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_3                                 0x306BB8
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_4                                 0x306BBC
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_5                                 0x306BC0
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_6                                 0x306BC4
+
+#define mmSIF_RTR_CTRL_0_RGL_TOKEN_7                                 0x306BC8
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_0                               0x306BEC
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_1                               0x306BF0
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_2                               0x306BF4
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_3                               0x306BF8
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_4                               0x306BFC
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_5                               0x306C00
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_6                               0x306C04
+
+#define mmSIF_RTR_CTRL_0_RGL_BANK_ID_7                               0x306C08
+
+#define mmSIF_RTR_CTRL_0_RGL_WDT                                     0x306C2C
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM0_CH0_CTR_WRAP                    0x306C30
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM0_CH1_CTR_WRAP                    0x306C34
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM1_CH0_CTR_WRAP                    0x306C38
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM1_CH1_CTR_WRAP                    0x306C3C
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM2_CH0_CTR_WRAP                    0x306C40
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM2_CH1_CTR_WRAP                    0x306C44
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM3_CH0_CTR_WRAP                    0x306C48
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM3_CH1_CTR_WRAP                    0x306C4C
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM0_CH0_CTR_CNT                     0x306C50
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM0_CH1_CTR_CNT                     0x306C54
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM1_CH0_CTR_CNT                     0x306C58
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM1_CH1_CTR_CNT                     0x306C5C
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM2_CH0_CTR_CNT                     0x306C60
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM2_CH1_CTR_CNT                     0x306C64
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM3_CH0_CTR_CNT                     0x306C68
+
+#define mmSIF_RTR_CTRL_0_E2E_AR_HBM3_CH1_CTR_CNT                     0x306C6C
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM0_CH0_CTR_WRAP                    0x306C70
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM0_CH1_CTR_WRAP                    0x306C74
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM1_CH0_CTR_WRAP                    0x306C78
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM1_CH1_CTR_WRAP                    0x306C7C
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM2_CH0_CTR_WRAP                    0x306C80
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM2_CH1_CTR_WRAP                    0x306C84
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM3_CH0_CTR_WRAP                    0x306C88
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM3_CH1_CTR_WRAP                    0x306C8C
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM0_CH0_CTR_CNT                     0x306C90
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM0_CH1_CTR_CNT                     0x306C94
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM1_CH0_CTR_CNT                     0x306C98
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM1_CH1_CTR_CNT                     0x306C9C
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM2_CH0_CTR_CNT                     0x306CA0
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM2_CH1_CTR_CNT                     0x306CA4
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM3_CH0_CTR_CNT                     0x306CA8
+
+#define mmSIF_RTR_CTRL_0_E2E_AW_HBM3_CH1_CTR_CNT                     0x306CAC
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_PC_SEL_0                             0x306CB0
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_PC_SEL_1                             0x306CB4
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_PC_SEL_2                             0x306CB8
+
+#define mmSIF_RTR_CTRL_0_NL_HBM_PC_SEL_3                             0x306CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_0_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_1_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_1_regs.h
new file mode 100644
index 000000000000..c6d517dbbd54
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_1_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_1_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_1_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_1 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_1_PERM_SEL                                    0x316108
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_0                               0x316114
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_1                               0x316118
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_2                               0x31611C
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_3                               0x316120
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_4                               0x316124
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_5                               0x316128
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_6                               0x31612C
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_7                               0x316130
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_8                               0x316134
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_9                               0x316138
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_10                              0x31613C
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_11                              0x316140
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_12                              0x316144
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_13                              0x316148
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_14                              0x31614C
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_15                              0x316150
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_16                              0x316154
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_17                              0x316158
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_18                              0x31615C
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_19                              0x316160
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_20                              0x316164
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_21                              0x316168
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_22                              0x31616C
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_23                              0x316170
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_24                              0x316174
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_25                              0x316178
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_26                              0x31617C
+
+#define mmSIF_RTR_CTRL_1_HBM_POLY_H3_27                              0x316180
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_0                              0x316184
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_1                              0x316188
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_2                              0x31618C
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_3                              0x316190
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_4                              0x316194
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_5                              0x316198
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_6                              0x31619C
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_7                              0x3161A0
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_8                              0x3161A4
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_9                              0x3161A8
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_10                             0x3161AC
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_11                             0x3161B0
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_12                             0x3161B4
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_13                             0x3161B8
+
+#define mmSIF_RTR_CTRL_1_SRAM_POLY_H3_14                             0x3161BC
+
+#define mmSIF_RTR_CTRL_1_SCRAM_SRAM_EN                               0x31626C
+
+#define mmSIF_RTR_CTRL_1_RL_HBM_EN                                   0x316274
+
+#define mmSIF_RTR_CTRL_1_RL_HBM_SAT                                  0x316278
+
+#define mmSIF_RTR_CTRL_1_RL_HBM_RST                                  0x31627C
+
+#define mmSIF_RTR_CTRL_1_RL_HBM_TIMEOUT                              0x316280
+
+#define mmSIF_RTR_CTRL_1_SCRAM_HBM_EN                                0x316284
+
+#define mmSIF_RTR_CTRL_1_RL_PCI_EN                                   0x316288
+
+#define mmSIF_RTR_CTRL_1_RL_PCI_SAT                                  0x31628C
+
+#define mmSIF_RTR_CTRL_1_RL_PCI_RST                                  0x316290
+
+#define mmSIF_RTR_CTRL_1_RL_PCI_TIMEOUT                              0x316294
+
+#define mmSIF_RTR_CTRL_1_RL_SRAM_EN                                  0x31629C
+
+#define mmSIF_RTR_CTRL_1_RL_SRAM_SAT                                 0x3162A0
+
+#define mmSIF_RTR_CTRL_1_RL_SRAM_RST                                 0x3162A4
+
+#define mmSIF_RTR_CTRL_1_RL_SRAM_TIMEOUT                             0x3162AC
+
+#define mmSIF_RTR_CTRL_1_RL_SRAM_RED                                 0x3162B4
+
+#define mmSIF_RTR_CTRL_1_E2E_HBM_EN                                  0x3162EC
+
+#define mmSIF_RTR_CTRL_1_E2E_PCI_EN                                  0x3162F0
+
+#define mmSIF_RTR_CTRL_1_E2E_HBM_WR_SIZE                             0x3162F4
+
+#define mmSIF_RTR_CTRL_1_E2E_PCI_WR_SIZE                             0x3162F8
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_PCI_CTR_SET_EN                       0x316404
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_PCI_CTR_SET                          0x316408
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_PCI_CTR_WRAP                         0x31640C
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_PCI_CTR_CNT                          0x316410
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM_CTR_SET_EN                       0x316414
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM_CTR_SET                          0x316418
+
+#define mmSIF_RTR_CTRL_1_E2E_HBM_RD_SIZE                             0x31641C
+
+#define mmSIF_RTR_CTRL_1_E2E_PCI_RD_SIZE                             0x316420
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_PCI_CTR_SET_EN                       0x316424
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_PCI_CTR_SET                          0x316428
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_PCI_CTR_WRAP                         0x31642C
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_PCI_CTR_CNT                          0x316430
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM_CTR_SET_EN                       0x316434
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM_CTR_SET                          0x316438
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_SEL_0                                0x316450
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_SEL_1                                0x316454
+
+#define mmSIF_RTR_CTRL_1_NON_LIN_EN                                  0x316480
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_BANK_0                              0x316500
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_BANK_1                              0x316504
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_BANK_2                              0x316508
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_BANK_3                              0x31650C
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_BANK_4                              0x316510
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_0                            0x316514
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_1                            0x316520
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_2                            0x316524
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_3                            0x316528
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_4                            0x31652C
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_5                            0x316530
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_6                            0x316534
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_7                            0x316538
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_8                            0x31653C
+
+#define mmSIF_RTR_CTRL_1_NL_SRAM_OFFSET_9                            0x316540
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_0                             0x316550
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_1                             0x316554
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_2                             0x316558
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_3                             0x31655C
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_4                             0x316560
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_5                             0x316564
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_6                             0x316568
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_7                             0x31656C
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_8                             0x316570
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_9                             0x316574
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_10                            0x316578
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_11                            0x31657C
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_12                            0x316580
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_13                            0x316584
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_14                            0x316588
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_15                            0x31658C
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_16                            0x316590
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_17                            0x316594
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_OFFSET_18                            0x316598
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_0                     0x3165E4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_1                     0x3165E8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_2                     0x3165EC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_3                     0x3165F0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_4                     0x3165F4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_5                     0x3165F8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_6                     0x3165FC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_7                     0x316600
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_8                     0x316604
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_9                     0x316608
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_10                    0x31660C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_11                    0x316610
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_12                    0x316614
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_13                    0x316618
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_14                    0x31661C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AW_15                    0x316620
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_0                    0x316624
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_1                    0x316628
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_2                    0x31662C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_3                    0x316630
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_4                    0x316634
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_5                    0x316638
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_6                    0x31663C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_7                    0x316640
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_8                    0x316644
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_9                    0x316648
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_10                   0x31664C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_11                   0x316650
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_12                   0x316654
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_13                   0x316658
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_14                   0x31665C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AW_15                   0x316660
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_0                     0x316664
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_1                     0x316668
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_2                     0x31666C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_3                     0x316670
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_4                     0x316674
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_5                     0x316678
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_6                     0x31667C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_7                     0x316680
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_8                     0x316684
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_9                     0x316688
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_10                    0x31668C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_11                    0x316690
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_12                    0x316694
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_13                    0x316698
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_14                    0x31669C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AW_15                    0x3166A0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_0                    0x3166A4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_1                    0x3166A8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_2                    0x3166AC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_3                    0x3166B0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_4                    0x3166B4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_5                    0x3166B8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_6                    0x3166BC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_7                    0x3166C0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_8                    0x3166C4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_9                    0x3166C8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_10                   0x3166CC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_11                   0x3166D0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_12                   0x3166D4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_13                   0x3166D8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_14                   0x3166DC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AW_15                   0x3166E0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_0                    0x3166E4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_1                    0x3166E8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_2                    0x3166EC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_3                    0x3166F0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_4                    0x3166F4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_5                    0x3166F8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_6                    0x3166FC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_7                    0x316700
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_8                    0x316704
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_9                    0x316708
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_10                   0x31670C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_11                   0x316710
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_12                   0x316714
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_13                   0x316718
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_14                   0x31671C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AW_15                   0x316720
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_0                   0x316724
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_1                   0x316728
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_2                   0x31672C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_3                   0x316730
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_4                   0x316734
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_5                   0x316738
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_6                   0x31673C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_7                   0x316740
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_8                   0x316744
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_9                   0x316748
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_10                  0x31674C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_11                  0x316750
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_12                  0x316754
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_13                  0x316758
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_14                  0x31675C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AW_15                  0x316760
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_0                    0x316764
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_1                    0x316768
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_2                    0x31676C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_3                    0x316770
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_4                    0x316774
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_5                    0x316778
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_6                    0x31677C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_7                    0x316780
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_8                    0x316784
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_9                    0x316788
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_10                   0x31678C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_11                   0x316790
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_12                   0x316794
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_13                   0x316798
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_14                   0x31679C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AW_15                   0x3167A0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_0                   0x3167A4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_1                   0x3167A8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_2                   0x3167AC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_3                   0x3167B0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_4                   0x3167B4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_5                   0x3167B8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_6                   0x3167BC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_7                   0x3167C0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_8                   0x3167C4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_9                   0x3167C8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_10                  0x3167CC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_11                  0x3167D0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_12                  0x3167D4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_13                  0x3167D8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_14                  0x3167DC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AW_15                  0x3167E0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_0                     0x316824
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_1                     0x316828
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_2                     0x31682C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_3                     0x316830
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_4                     0x316834
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_5                     0x316838
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_6                     0x31683C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_7                     0x316840
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_8                     0x316844
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_9                     0x316848
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_10                    0x31684C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_11                    0x316850
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_12                    0x316854
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_13                    0x316858
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_14                    0x31685C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_LOW_AR_15                    0x316860
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_0                    0x316864
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_1                    0x316868
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_2                    0x31686C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_3                    0x316870
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_4                    0x316874
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_5                    0x316878
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_6                    0x31687C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_7                    0x316880
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_8                    0x316884
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_9                    0x316888
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_10                   0x31688C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_11                   0x316890
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_12                   0x316894
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_13                   0x316898
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_14                   0x31689C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_BASE_HIGH_AR_15                   0x3168A0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_0                     0x3168A4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_1                     0x3168A8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_2                     0x3168AC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_3                     0x3168B0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_4                     0x3168B4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_5                     0x3168B8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_6                     0x3168BC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_7                     0x3168C0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_8                     0x3168C4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_9                     0x3168C8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_10                    0x3168CC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_11                    0x3168D0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_12                    0x3168D4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_13                    0x3168D8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_14                    0x3168DC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_LOW_AR_15                    0x3168E0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_0                    0x3168E4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_1                    0x3168E8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_2                    0x3168EC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_3                    0x3168F0
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_4                    0x3168F4
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_5                    0x3168F8
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_6                    0x3168FC
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_7                    0x316900
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_8                    0x316904
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_9                    0x316908
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_10                   0x31690C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_11                   0x316910
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_12                   0x316914
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_13                   0x316918
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_14                   0x31691C
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_MASK_HIGH_AR_15                   0x316920
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_0                    0x316924
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_1                    0x316928
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_2                    0x31692C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_3                    0x316930
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_4                    0x316934
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_5                    0x316938
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_6                    0x31693C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_7                    0x316940
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_8                    0x316944
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_9                    0x316948
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_10                   0x31694C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_11                   0x316950
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_12                   0x316954
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_13                   0x316958
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_14                   0x31695C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_LOW_AR_15                   0x316960
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_0                   0x316964
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_1                   0x316968
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_2                   0x31696C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_3                   0x316970
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_4                   0x316974
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_5                   0x316978
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_6                   0x31697C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_7                   0x316980
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_8                   0x316984
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_9                   0x316988
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_10                  0x31698C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_11                  0x316990
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_12                  0x316994
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_13                  0x316998
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_14                  0x31699C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_BASE_HIGH_AR_15                  0x3169A0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_0                    0x3169A4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_1                    0x3169A8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_2                    0x3169AC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_3                    0x3169B0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_4                    0x3169B4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_5                    0x3169B8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_6                    0x3169BC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_7                    0x3169C0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_8                    0x3169C4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_9                    0x3169C8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_10                   0x3169CC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_11                   0x3169D0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_12                   0x3169D4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_13                   0x3169D8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_14                   0x3169DC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_LOW_AR_15                   0x3169E0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_0                   0x3169E4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_1                   0x3169E8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_2                   0x3169EC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_3                   0x3169F0
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_4                   0x3169F4
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_5                   0x3169F8
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_6                   0x3169FC
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_7                   0x316A00
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_8                   0x316A04
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_9                   0x316A08
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_10                  0x316A0C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_11                  0x316A10
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_12                  0x316A14
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_13                  0x316A18
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_14                  0x316A1C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_MASK_HIGH_AR_15                  0x316A20
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_HIT_AW                            0x316A64
+
+#define mmSIF_RTR_CTRL_1_RANGE_SEC_HIT_AR                            0x316A68
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_HIT_AW                           0x316A6C
+
+#define mmSIF_RTR_CTRL_1_RANGE_PRIV_HIT_AR                           0x316A70
+
+#define mmSIF_RTR_CTRL_1_RGL_CFG                                     0x316B64
+
+#define mmSIF_RTR_CTRL_1_RGL_SHIFT                                   0x316B68
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_0                          0x316B6C
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_1                          0x316B70
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_2                          0x316B74
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_3                          0x316B78
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_4                          0x316B7C
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_5                          0x316B80
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_6                          0x316B84
+
+#define mmSIF_RTR_CTRL_1_RGL_EXPECTED_LAT_7                          0x316B88
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_0                                 0x316BAC
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_1                                 0x316BB0
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_2                                 0x316BB4
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_3                                 0x316BB8
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_4                                 0x316BBC
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_5                                 0x316BC0
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_6                                 0x316BC4
+
+#define mmSIF_RTR_CTRL_1_RGL_TOKEN_7                                 0x316BC8
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_0                               0x316BEC
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_1                               0x316BF0
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_2                               0x316BF4
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_3                               0x316BF8
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_4                               0x316BFC
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_5                               0x316C00
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_6                               0x316C04
+
+#define mmSIF_RTR_CTRL_1_RGL_BANK_ID_7                               0x316C08
+
+#define mmSIF_RTR_CTRL_1_RGL_WDT                                     0x316C2C
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM0_CH0_CTR_WRAP                    0x316C30
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM0_CH1_CTR_WRAP                    0x316C34
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM1_CH0_CTR_WRAP                    0x316C38
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM1_CH1_CTR_WRAP                    0x316C3C
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM2_CH0_CTR_WRAP                    0x316C40
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM2_CH1_CTR_WRAP                    0x316C44
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM3_CH0_CTR_WRAP                    0x316C48
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM3_CH1_CTR_WRAP                    0x316C4C
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM0_CH0_CTR_CNT                     0x316C50
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM0_CH1_CTR_CNT                     0x316C54
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM1_CH0_CTR_CNT                     0x316C58
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM1_CH1_CTR_CNT                     0x316C5C
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM2_CH0_CTR_CNT                     0x316C60
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM2_CH1_CTR_CNT                     0x316C64
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM3_CH0_CTR_CNT                     0x316C68
+
+#define mmSIF_RTR_CTRL_1_E2E_AR_HBM3_CH1_CTR_CNT                     0x316C6C
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM0_CH0_CTR_WRAP                    0x316C70
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM0_CH1_CTR_WRAP                    0x316C74
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM1_CH0_CTR_WRAP                    0x316C78
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM1_CH1_CTR_WRAP                    0x316C7C
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM2_CH0_CTR_WRAP                    0x316C80
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM2_CH1_CTR_WRAP                    0x316C84
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM3_CH0_CTR_WRAP                    0x316C88
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM3_CH1_CTR_WRAP                    0x316C8C
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM0_CH0_CTR_CNT                     0x316C90
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM0_CH1_CTR_CNT                     0x316C94
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM1_CH0_CTR_CNT                     0x316C98
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM1_CH1_CTR_CNT                     0x316C9C
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM2_CH0_CTR_CNT                     0x316CA0
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM2_CH1_CTR_CNT                     0x316CA4
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM3_CH0_CTR_CNT                     0x316CA8
+
+#define mmSIF_RTR_CTRL_1_E2E_AW_HBM3_CH1_CTR_CNT                     0x316CAC
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_PC_SEL_0                             0x316CB0
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_PC_SEL_1                             0x316CB4
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_PC_SEL_2                             0x316CB8
+
+#define mmSIF_RTR_CTRL_1_NL_HBM_PC_SEL_3                             0x316CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_1_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_2_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_2_regs.h
new file mode 100644
index 000000000000..330e5b42d679
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_2_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_2_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_2_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_2 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_2_PERM_SEL                                    0x326108
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_0                               0x326114
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_1                               0x326118
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_2                               0x32611C
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_3                               0x326120
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_4                               0x326124
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_5                               0x326128
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_6                               0x32612C
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_7                               0x326130
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_8                               0x326134
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_9                               0x326138
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_10                              0x32613C
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_11                              0x326140
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_12                              0x326144
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_13                              0x326148
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_14                              0x32614C
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_15                              0x326150
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_16                              0x326154
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_17                              0x326158
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_18                              0x32615C
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_19                              0x326160
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_20                              0x326164
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_21                              0x326168
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_22                              0x32616C
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_23                              0x326170
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_24                              0x326174
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_25                              0x326178
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_26                              0x32617C
+
+#define mmSIF_RTR_CTRL_2_HBM_POLY_H3_27                              0x326180
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_0                              0x326184
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_1                              0x326188
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_2                              0x32618C
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_3                              0x326190
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_4                              0x326194
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_5                              0x326198
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_6                              0x32619C
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_7                              0x3261A0
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_8                              0x3261A4
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_9                              0x3261A8
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_10                             0x3261AC
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_11                             0x3261B0
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_12                             0x3261B4
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_13                             0x3261B8
+
+#define mmSIF_RTR_CTRL_2_SRAM_POLY_H3_14                             0x3261BC
+
+#define mmSIF_RTR_CTRL_2_SCRAM_SRAM_EN                               0x32626C
+
+#define mmSIF_RTR_CTRL_2_RL_HBM_EN                                   0x326274
+
+#define mmSIF_RTR_CTRL_2_RL_HBM_SAT                                  0x326278
+
+#define mmSIF_RTR_CTRL_2_RL_HBM_RST                                  0x32627C
+
+#define mmSIF_RTR_CTRL_2_RL_HBM_TIMEOUT                              0x326280
+
+#define mmSIF_RTR_CTRL_2_SCRAM_HBM_EN                                0x326284
+
+#define mmSIF_RTR_CTRL_2_RL_PCI_EN                                   0x326288
+
+#define mmSIF_RTR_CTRL_2_RL_PCI_SAT                                  0x32628C
+
+#define mmSIF_RTR_CTRL_2_RL_PCI_RST                                  0x326290
+
+#define mmSIF_RTR_CTRL_2_RL_PCI_TIMEOUT                              0x326294
+
+#define mmSIF_RTR_CTRL_2_RL_SRAM_EN                                  0x32629C
+
+#define mmSIF_RTR_CTRL_2_RL_SRAM_SAT                                 0x3262A0
+
+#define mmSIF_RTR_CTRL_2_RL_SRAM_RST                                 0x3262A4
+
+#define mmSIF_RTR_CTRL_2_RL_SRAM_TIMEOUT                             0x3262AC
+
+#define mmSIF_RTR_CTRL_2_RL_SRAM_RED                                 0x3262B4
+
+#define mmSIF_RTR_CTRL_2_E2E_HBM_EN                                  0x3262EC
+
+#define mmSIF_RTR_CTRL_2_E2E_PCI_EN                                  0x3262F0
+
+#define mmSIF_RTR_CTRL_2_E2E_HBM_WR_SIZE                             0x3262F4
+
+#define mmSIF_RTR_CTRL_2_E2E_PCI_WR_SIZE                             0x3262F8
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_PCI_CTR_SET_EN                       0x326404
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_PCI_CTR_SET                          0x326408
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_PCI_CTR_WRAP                         0x32640C
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_PCI_CTR_CNT                          0x326410
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM_CTR_SET_EN                       0x326414
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM_CTR_SET                          0x326418
+
+#define mmSIF_RTR_CTRL_2_E2E_HBM_RD_SIZE                             0x32641C
+
+#define mmSIF_RTR_CTRL_2_E2E_PCI_RD_SIZE                             0x326420
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_PCI_CTR_SET_EN                       0x326424
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_PCI_CTR_SET                          0x326428
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_PCI_CTR_WRAP                         0x32642C
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_PCI_CTR_CNT                          0x326430
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM_CTR_SET_EN                       0x326434
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM_CTR_SET                          0x326438
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_SEL_0                                0x326450
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_SEL_1                                0x326454
+
+#define mmSIF_RTR_CTRL_2_NON_LIN_EN                                  0x326480
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_BANK_0                              0x326500
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_BANK_1                              0x326504
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_BANK_2                              0x326508
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_BANK_3                              0x32650C
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_BANK_4                              0x326510
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_0                            0x326514
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_1                            0x326520
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_2                            0x326524
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_3                            0x326528
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_4                            0x32652C
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_5                            0x326530
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_6                            0x326534
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_7                            0x326538
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_8                            0x32653C
+
+#define mmSIF_RTR_CTRL_2_NL_SRAM_OFFSET_9                            0x326540
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_0                             0x326550
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_1                             0x326554
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_2                             0x326558
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_3                             0x32655C
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_4                             0x326560
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_5                             0x326564
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_6                             0x326568
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_7                             0x32656C
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_8                             0x326570
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_9                             0x326574
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_10                            0x326578
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_11                            0x32657C
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_12                            0x326580
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_13                            0x326584
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_14                            0x326588
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_15                            0x32658C
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_16                            0x326590
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_17                            0x326594
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_OFFSET_18                            0x326598
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_0                     0x3265E4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_1                     0x3265E8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_2                     0x3265EC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_3                     0x3265F0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_4                     0x3265F4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_5                     0x3265F8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_6                     0x3265FC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_7                     0x326600
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_8                     0x326604
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_9                     0x326608
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_10                    0x32660C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_11                    0x326610
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_12                    0x326614
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_13                    0x326618
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_14                    0x32661C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AW_15                    0x326620
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_0                    0x326624
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_1                    0x326628
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_2                    0x32662C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_3                    0x326630
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_4                    0x326634
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_5                    0x326638
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_6                    0x32663C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_7                    0x326640
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_8                    0x326644
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_9                    0x326648
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_10                   0x32664C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_11                   0x326650
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_12                   0x326654
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_13                   0x326658
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_14                   0x32665C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AW_15                   0x326660
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_0                     0x326664
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_1                     0x326668
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_2                     0x32666C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_3                     0x326670
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_4                     0x326674
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_5                     0x326678
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_6                     0x32667C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_7                     0x326680
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_8                     0x326684
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_9                     0x326688
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_10                    0x32668C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_11                    0x326690
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_12                    0x326694
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_13                    0x326698
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_14                    0x32669C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AW_15                    0x3266A0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_0                    0x3266A4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_1                    0x3266A8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_2                    0x3266AC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_3                    0x3266B0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_4                    0x3266B4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_5                    0x3266B8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_6                    0x3266BC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_7                    0x3266C0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_8                    0x3266C4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_9                    0x3266C8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_10                   0x3266CC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_11                   0x3266D0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_12                   0x3266D4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_13                   0x3266D8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_14                   0x3266DC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AW_15                   0x3266E0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_0                    0x3266E4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_1                    0x3266E8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_2                    0x3266EC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_3                    0x3266F0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_4                    0x3266F4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_5                    0x3266F8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_6                    0x3266FC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_7                    0x326700
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_8                    0x326704
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_9                    0x326708
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_10                   0x32670C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_11                   0x326710
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_12                   0x326714
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_13                   0x326718
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_14                   0x32671C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AW_15                   0x326720
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_0                   0x326724
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_1                   0x326728
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_2                   0x32672C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_3                   0x326730
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_4                   0x326734
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_5                   0x326738
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_6                   0x32673C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_7                   0x326740
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_8                   0x326744
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_9                   0x326748
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_10                  0x32674C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_11                  0x326750
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_12                  0x326754
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_13                  0x326758
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_14                  0x32675C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AW_15                  0x326760
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_0                    0x326764
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_1                    0x326768
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_2                    0x32676C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_3                    0x326770
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_4                    0x326774
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_5                    0x326778
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_6                    0x32677C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_7                    0x326780
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_8                    0x326784
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_9                    0x326788
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_10                   0x32678C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_11                   0x326790
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_12                   0x326794
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_13                   0x326798
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_14                   0x32679C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AW_15                   0x3267A0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_0                   0x3267A4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_1                   0x3267A8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_2                   0x3267AC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_3                   0x3267B0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_4                   0x3267B4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_5                   0x3267B8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_6                   0x3267BC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_7                   0x3267C0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_8                   0x3267C4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_9                   0x3267C8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_10                  0x3267CC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_11                  0x3267D0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_12                  0x3267D4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_13                  0x3267D8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_14                  0x3267DC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AW_15                  0x3267E0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_0                     0x326824
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_1                     0x326828
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_2                     0x32682C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_3                     0x326830
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_4                     0x326834
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_5                     0x326838
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_6                     0x32683C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_7                     0x326840
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_8                     0x326844
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_9                     0x326848
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_10                    0x32684C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_11                    0x326850
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_12                    0x326854
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_13                    0x326858
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_14                    0x32685C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_LOW_AR_15                    0x326860
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_0                    0x326864
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_1                    0x326868
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_2                    0x32686C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_3                    0x326870
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_4                    0x326874
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_5                    0x326878
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_6                    0x32687C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_7                    0x326880
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_8                    0x326884
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_9                    0x326888
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_10                   0x32688C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_11                   0x326890
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_12                   0x326894
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_13                   0x326898
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_14                   0x32689C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_BASE_HIGH_AR_15                   0x3268A0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_0                     0x3268A4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_1                     0x3268A8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_2                     0x3268AC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_3                     0x3268B0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_4                     0x3268B4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_5                     0x3268B8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_6                     0x3268BC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_7                     0x3268C0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_8                     0x3268C4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_9                     0x3268C8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_10                    0x3268CC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_11                    0x3268D0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_12                    0x3268D4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_13                    0x3268D8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_14                    0x3268DC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_LOW_AR_15                    0x3268E0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_0                    0x3268E4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_1                    0x3268E8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_2                    0x3268EC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_3                    0x3268F0
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_4                    0x3268F4
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_5                    0x3268F8
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_6                    0x3268FC
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_7                    0x326900
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_8                    0x326904
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_9                    0x326908
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_10                   0x32690C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_11                   0x326910
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_12                   0x326914
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_13                   0x326918
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_14                   0x32691C
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_MASK_HIGH_AR_15                   0x326920
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_0                    0x326924
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_1                    0x326928
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_2                    0x32692C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_3                    0x326930
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_4                    0x326934
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_5                    0x326938
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_6                    0x32693C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_7                    0x326940
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_8                    0x326944
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_9                    0x326948
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_10                   0x32694C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_11                   0x326950
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_12                   0x326954
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_13                   0x326958
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_14                   0x32695C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_LOW_AR_15                   0x326960
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_0                   0x326964
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_1                   0x326968
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_2                   0x32696C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_3                   0x326970
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_4                   0x326974
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_5                   0x326978
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_6                   0x32697C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_7                   0x326980
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_8                   0x326984
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_9                   0x326988
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_10                  0x32698C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_11                  0x326990
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_12                  0x326994
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_13                  0x326998
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_14                  0x32699C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_BASE_HIGH_AR_15                  0x3269A0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_0                    0x3269A4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_1                    0x3269A8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_2                    0x3269AC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_3                    0x3269B0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_4                    0x3269B4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_5                    0x3269B8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_6                    0x3269BC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_7                    0x3269C0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_8                    0x3269C4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_9                    0x3269C8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_10                   0x3269CC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_11                   0x3269D0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_12                   0x3269D4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_13                   0x3269D8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_14                   0x3269DC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_LOW_AR_15                   0x3269E0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_0                   0x3269E4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_1                   0x3269E8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_2                   0x3269EC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_3                   0x3269F0
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_4                   0x3269F4
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_5                   0x3269F8
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_6                   0x3269FC
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_7                   0x326A00
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_8                   0x326A04
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_9                   0x326A08
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_10                  0x326A0C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_11                  0x326A10
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_12                  0x326A14
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_13                  0x326A18
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_14                  0x326A1C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_MASK_HIGH_AR_15                  0x326A20
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_HIT_AW                            0x326A64
+
+#define mmSIF_RTR_CTRL_2_RANGE_SEC_HIT_AR                            0x326A68
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_HIT_AW                           0x326A6C
+
+#define mmSIF_RTR_CTRL_2_RANGE_PRIV_HIT_AR                           0x326A70
+
+#define mmSIF_RTR_CTRL_2_RGL_CFG                                     0x326B64
+
+#define mmSIF_RTR_CTRL_2_RGL_SHIFT                                   0x326B68
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_0                          0x326B6C
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_1                          0x326B70
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_2                          0x326B74
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_3                          0x326B78
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_4                          0x326B7C
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_5                          0x326B80
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_6                          0x326B84
+
+#define mmSIF_RTR_CTRL_2_RGL_EXPECTED_LAT_7                          0x326B88
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_0                                 0x326BAC
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_1                                 0x326BB0
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_2                                 0x326BB4
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_3                                 0x326BB8
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_4                                 0x326BBC
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_5                                 0x326BC0
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_6                                 0x326BC4
+
+#define mmSIF_RTR_CTRL_2_RGL_TOKEN_7                                 0x326BC8
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_0                               0x326BEC
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_1                               0x326BF0
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_2                               0x326BF4
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_3                               0x326BF8
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_4                               0x326BFC
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_5                               0x326C00
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_6                               0x326C04
+
+#define mmSIF_RTR_CTRL_2_RGL_BANK_ID_7                               0x326C08
+
+#define mmSIF_RTR_CTRL_2_RGL_WDT                                     0x326C2C
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM0_CH0_CTR_WRAP                    0x326C30
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM0_CH1_CTR_WRAP                    0x326C34
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM1_CH0_CTR_WRAP                    0x326C38
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM1_CH1_CTR_WRAP                    0x326C3C
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM2_CH0_CTR_WRAP                    0x326C40
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM2_CH1_CTR_WRAP                    0x326C44
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM3_CH0_CTR_WRAP                    0x326C48
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM3_CH1_CTR_WRAP                    0x326C4C
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM0_CH0_CTR_CNT                     0x326C50
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM0_CH1_CTR_CNT                     0x326C54
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM1_CH0_CTR_CNT                     0x326C58
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM1_CH1_CTR_CNT                     0x326C5C
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM2_CH0_CTR_CNT                     0x326C60
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM2_CH1_CTR_CNT                     0x326C64
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM3_CH0_CTR_CNT                     0x326C68
+
+#define mmSIF_RTR_CTRL_2_E2E_AR_HBM3_CH1_CTR_CNT                     0x326C6C
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM0_CH0_CTR_WRAP                    0x326C70
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM0_CH1_CTR_WRAP                    0x326C74
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM1_CH0_CTR_WRAP                    0x326C78
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM1_CH1_CTR_WRAP                    0x326C7C
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM2_CH0_CTR_WRAP                    0x326C80
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM2_CH1_CTR_WRAP                    0x326C84
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM3_CH0_CTR_WRAP                    0x326C88
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM3_CH1_CTR_WRAP                    0x326C8C
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM0_CH0_CTR_CNT                     0x326C90
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM0_CH1_CTR_CNT                     0x326C94
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM1_CH0_CTR_CNT                     0x326C98
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM1_CH1_CTR_CNT                     0x326C9C
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM2_CH0_CTR_CNT                     0x326CA0
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM2_CH1_CTR_CNT                     0x326CA4
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM3_CH0_CTR_CNT                     0x326CA8
+
+#define mmSIF_RTR_CTRL_2_E2E_AW_HBM3_CH1_CTR_CNT                     0x326CAC
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_PC_SEL_0                             0x326CB0
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_PC_SEL_1                             0x326CB4
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_PC_SEL_2                             0x326CB8
+
+#define mmSIF_RTR_CTRL_2_NL_HBM_PC_SEL_3                             0x326CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_2_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_3_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_3_regs.h
new file mode 100644
index 000000000000..d749f1968e5e
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_3_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_3_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_3_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_3 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_3_PERM_SEL                                    0x336108
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_0                               0x336114
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_1                               0x336118
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_2                               0x33611C
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_3                               0x336120
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_4                               0x336124
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_5                               0x336128
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_6                               0x33612C
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_7                               0x336130
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_8                               0x336134
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_9                               0x336138
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_10                              0x33613C
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_11                              0x336140
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_12                              0x336144
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_13                              0x336148
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_14                              0x33614C
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_15                              0x336150
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_16                              0x336154
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_17                              0x336158
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_18                              0x33615C
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_19                              0x336160
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_20                              0x336164
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_21                              0x336168
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_22                              0x33616C
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_23                              0x336170
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_24                              0x336174
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_25                              0x336178
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_26                              0x33617C
+
+#define mmSIF_RTR_CTRL_3_HBM_POLY_H3_27                              0x336180
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_0                              0x336184
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_1                              0x336188
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_2                              0x33618C
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_3                              0x336190
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_4                              0x336194
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_5                              0x336198
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_6                              0x33619C
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_7                              0x3361A0
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_8                              0x3361A4
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_9                              0x3361A8
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_10                             0x3361AC
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_11                             0x3361B0
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_12                             0x3361B4
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_13                             0x3361B8
+
+#define mmSIF_RTR_CTRL_3_SRAM_POLY_H3_14                             0x3361BC
+
+#define mmSIF_RTR_CTRL_3_SCRAM_SRAM_EN                               0x33626C
+
+#define mmSIF_RTR_CTRL_3_RL_HBM_EN                                   0x336274
+
+#define mmSIF_RTR_CTRL_3_RL_HBM_SAT                                  0x336278
+
+#define mmSIF_RTR_CTRL_3_RL_HBM_RST                                  0x33627C
+
+#define mmSIF_RTR_CTRL_3_RL_HBM_TIMEOUT                              0x336280
+
+#define mmSIF_RTR_CTRL_3_SCRAM_HBM_EN                                0x336284
+
+#define mmSIF_RTR_CTRL_3_RL_PCI_EN                                   0x336288
+
+#define mmSIF_RTR_CTRL_3_RL_PCI_SAT                                  0x33628C
+
+#define mmSIF_RTR_CTRL_3_RL_PCI_RST                                  0x336290
+
+#define mmSIF_RTR_CTRL_3_RL_PCI_TIMEOUT                              0x336294
+
+#define mmSIF_RTR_CTRL_3_RL_SRAM_EN                                  0x33629C
+
+#define mmSIF_RTR_CTRL_3_RL_SRAM_SAT                                 0x3362A0
+
+#define mmSIF_RTR_CTRL_3_RL_SRAM_RST                                 0x3362A4
+
+#define mmSIF_RTR_CTRL_3_RL_SRAM_TIMEOUT                             0x3362AC
+
+#define mmSIF_RTR_CTRL_3_RL_SRAM_RED                                 0x3362B4
+
+#define mmSIF_RTR_CTRL_3_E2E_HBM_EN                                  0x3362EC
+
+#define mmSIF_RTR_CTRL_3_E2E_PCI_EN                                  0x3362F0
+
+#define mmSIF_RTR_CTRL_3_E2E_HBM_WR_SIZE                             0x3362F4
+
+#define mmSIF_RTR_CTRL_3_E2E_PCI_WR_SIZE                             0x3362F8
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_PCI_CTR_SET_EN                       0x336404
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_PCI_CTR_SET                          0x336408
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_PCI_CTR_WRAP                         0x33640C
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_PCI_CTR_CNT                          0x336410
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM_CTR_SET_EN                       0x336414
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM_CTR_SET                          0x336418
+
+#define mmSIF_RTR_CTRL_3_E2E_HBM_RD_SIZE                             0x33641C
+
+#define mmSIF_RTR_CTRL_3_E2E_PCI_RD_SIZE                             0x336420
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_PCI_CTR_SET_EN                       0x336424
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_PCI_CTR_SET                          0x336428
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_PCI_CTR_WRAP                         0x33642C
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_PCI_CTR_CNT                          0x336430
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM_CTR_SET_EN                       0x336434
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM_CTR_SET                          0x336438
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_SEL_0                                0x336450
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_SEL_1                                0x336454
+
+#define mmSIF_RTR_CTRL_3_NON_LIN_EN                                  0x336480
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_BANK_0                              0x336500
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_BANK_1                              0x336504
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_BANK_2                              0x336508
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_BANK_3                              0x33650C
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_BANK_4                              0x336510
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_0                            0x336514
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_1                            0x336520
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_2                            0x336524
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_3                            0x336528
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_4                            0x33652C
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_5                            0x336530
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_6                            0x336534
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_7                            0x336538
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_8                            0x33653C
+
+#define mmSIF_RTR_CTRL_3_NL_SRAM_OFFSET_9                            0x336540
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_0                             0x336550
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_1                             0x336554
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_2                             0x336558
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_3                             0x33655C
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_4                             0x336560
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_5                             0x336564
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_6                             0x336568
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_7                             0x33656C
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_8                             0x336570
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_9                             0x336574
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_10                            0x336578
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_11                            0x33657C
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_12                            0x336580
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_13                            0x336584
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_14                            0x336588
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_15                            0x33658C
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_16                            0x336590
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_17                            0x336594
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_OFFSET_18                            0x336598
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_0                     0x3365E4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_1                     0x3365E8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_2                     0x3365EC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_3                     0x3365F0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_4                     0x3365F4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_5                     0x3365F8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_6                     0x3365FC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_7                     0x336600
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_8                     0x336604
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_9                     0x336608
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_10                    0x33660C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_11                    0x336610
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_12                    0x336614
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_13                    0x336618
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_14                    0x33661C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AW_15                    0x336620
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_0                    0x336624
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_1                    0x336628
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_2                    0x33662C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_3                    0x336630
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_4                    0x336634
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_5                    0x336638
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_6                    0x33663C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_7                    0x336640
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_8                    0x336644
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_9                    0x336648
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_10                   0x33664C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_11                   0x336650
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_12                   0x336654
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_13                   0x336658
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_14                   0x33665C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AW_15                   0x336660
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_0                     0x336664
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_1                     0x336668
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_2                     0x33666C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_3                     0x336670
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_4                     0x336674
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_5                     0x336678
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_6                     0x33667C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_7                     0x336680
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_8                     0x336684
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_9                     0x336688
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_10                    0x33668C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_11                    0x336690
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_12                    0x336694
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_13                    0x336698
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_14                    0x33669C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AW_15                    0x3366A0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_0                    0x3366A4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_1                    0x3366A8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_2                    0x3366AC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_3                    0x3366B0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_4                    0x3366B4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_5                    0x3366B8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_6                    0x3366BC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_7                    0x3366C0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_8                    0x3366C4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_9                    0x3366C8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_10                   0x3366CC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_11                   0x3366D0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_12                   0x3366D4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_13                   0x3366D8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_14                   0x3366DC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AW_15                   0x3366E0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_0                    0x3366E4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_1                    0x3366E8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_2                    0x3366EC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_3                    0x3366F0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_4                    0x3366F4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_5                    0x3366F8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_6                    0x3366FC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_7                    0x336700
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_8                    0x336704
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_9                    0x336708
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_10                   0x33670C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_11                   0x336710
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_12                   0x336714
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_13                   0x336718
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_14                   0x33671C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AW_15                   0x336720
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_0                   0x336724
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_1                   0x336728
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_2                   0x33672C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_3                   0x336730
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_4                   0x336734
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_5                   0x336738
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_6                   0x33673C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_7                   0x336740
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_8                   0x336744
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_9                   0x336748
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_10                  0x33674C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_11                  0x336750
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_12                  0x336754
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_13                  0x336758
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_14                  0x33675C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AW_15                  0x336760
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_0                    0x336764
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_1                    0x336768
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_2                    0x33676C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_3                    0x336770
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_4                    0x336774
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_5                    0x336778
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_6                    0x33677C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_7                    0x336780
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_8                    0x336784
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_9                    0x336788
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_10                   0x33678C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_11                   0x336790
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_12                   0x336794
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_13                   0x336798
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_14                   0x33679C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AW_15                   0x3367A0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_0                   0x3367A4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_1                   0x3367A8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_2                   0x3367AC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_3                   0x3367B0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_4                   0x3367B4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_5                   0x3367B8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_6                   0x3367BC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_7                   0x3367C0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_8                   0x3367C4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_9                   0x3367C8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_10                  0x3367CC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_11                  0x3367D0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_12                  0x3367D4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_13                  0x3367D8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_14                  0x3367DC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AW_15                  0x3367E0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_0                     0x336824
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_1                     0x336828
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_2                     0x33682C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_3                     0x336830
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_4                     0x336834
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_5                     0x336838
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_6                     0x33683C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_7                     0x336840
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_8                     0x336844
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_9                     0x336848
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_10                    0x33684C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_11                    0x336850
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_12                    0x336854
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_13                    0x336858
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_14                    0x33685C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_LOW_AR_15                    0x336860
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_0                    0x336864
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_1                    0x336868
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_2                    0x33686C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_3                    0x336870
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_4                    0x336874
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_5                    0x336878
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_6                    0x33687C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_7                    0x336880
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_8                    0x336884
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_9                    0x336888
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_10                   0x33688C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_11                   0x336890
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_12                   0x336894
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_13                   0x336898
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_14                   0x33689C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_BASE_HIGH_AR_15                   0x3368A0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_0                     0x3368A4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_1                     0x3368A8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_2                     0x3368AC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_3                     0x3368B0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_4                     0x3368B4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_5                     0x3368B8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_6                     0x3368BC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_7                     0x3368C0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_8                     0x3368C4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_9                     0x3368C8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_10                    0x3368CC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_11                    0x3368D0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_12                    0x3368D4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_13                    0x3368D8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_14                    0x3368DC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_LOW_AR_15                    0x3368E0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_0                    0x3368E4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_1                    0x3368E8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_2                    0x3368EC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_3                    0x3368F0
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_4                    0x3368F4
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_5                    0x3368F8
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_6                    0x3368FC
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_7                    0x336900
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_8                    0x336904
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_9                    0x336908
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_10                   0x33690C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_11                   0x336910
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_12                   0x336914
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_13                   0x336918
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_14                   0x33691C
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_MASK_HIGH_AR_15                   0x336920
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_0                    0x336924
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_1                    0x336928
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_2                    0x33692C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_3                    0x336930
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_4                    0x336934
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_5                    0x336938
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_6                    0x33693C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_7                    0x336940
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_8                    0x336944
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_9                    0x336948
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_10                   0x33694C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_11                   0x336950
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_12                   0x336954
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_13                   0x336958
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_14                   0x33695C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_LOW_AR_15                   0x336960
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_0                   0x336964
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_1                   0x336968
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_2                   0x33696C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_3                   0x336970
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_4                   0x336974
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_5                   0x336978
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_6                   0x33697C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_7                   0x336980
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_8                   0x336984
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_9                   0x336988
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_10                  0x33698C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_11                  0x336990
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_12                  0x336994
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_13                  0x336998
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_14                  0x33699C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_BASE_HIGH_AR_15                  0x3369A0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_0                    0x3369A4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_1                    0x3369A8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_2                    0x3369AC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_3                    0x3369B0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_4                    0x3369B4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_5                    0x3369B8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_6                    0x3369BC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_7                    0x3369C0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_8                    0x3369C4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_9                    0x3369C8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_10                   0x3369CC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_11                   0x3369D0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_12                   0x3369D4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_13                   0x3369D8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_14                   0x3369DC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_LOW_AR_15                   0x3369E0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_0                   0x3369E4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_1                   0x3369E8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_2                   0x3369EC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_3                   0x3369F0
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_4                   0x3369F4
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_5                   0x3369F8
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_6                   0x3369FC
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_7                   0x336A00
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_8                   0x336A04
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_9                   0x336A08
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_10                  0x336A0C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_11                  0x336A10
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_12                  0x336A14
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_13                  0x336A18
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_14                  0x336A1C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_MASK_HIGH_AR_15                  0x336A20
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_HIT_AW                            0x336A64
+
+#define mmSIF_RTR_CTRL_3_RANGE_SEC_HIT_AR                            0x336A68
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_HIT_AW                           0x336A6C
+
+#define mmSIF_RTR_CTRL_3_RANGE_PRIV_HIT_AR                           0x336A70
+
+#define mmSIF_RTR_CTRL_3_RGL_CFG                                     0x336B64
+
+#define mmSIF_RTR_CTRL_3_RGL_SHIFT                                   0x336B68
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_0                          0x336B6C
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_1                          0x336B70
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_2                          0x336B74
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_3                          0x336B78
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_4                          0x336B7C
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_5                          0x336B80
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_6                          0x336B84
+
+#define mmSIF_RTR_CTRL_3_RGL_EXPECTED_LAT_7                          0x336B88
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_0                                 0x336BAC
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_1                                 0x336BB0
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_2                                 0x336BB4
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_3                                 0x336BB8
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_4                                 0x336BBC
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_5                                 0x336BC0
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_6                                 0x336BC4
+
+#define mmSIF_RTR_CTRL_3_RGL_TOKEN_7                                 0x336BC8
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_0                               0x336BEC
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_1                               0x336BF0
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_2                               0x336BF4
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_3                               0x336BF8
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_4                               0x336BFC
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_5                               0x336C00
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_6                               0x336C04
+
+#define mmSIF_RTR_CTRL_3_RGL_BANK_ID_7                               0x336C08
+
+#define mmSIF_RTR_CTRL_3_RGL_WDT                                     0x336C2C
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM0_CH0_CTR_WRAP                    0x336C30
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM0_CH1_CTR_WRAP                    0x336C34
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM1_CH0_CTR_WRAP                    0x336C38
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM1_CH1_CTR_WRAP                    0x336C3C
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM2_CH0_CTR_WRAP                    0x336C40
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM2_CH1_CTR_WRAP                    0x336C44
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM3_CH0_CTR_WRAP                    0x336C48
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM3_CH1_CTR_WRAP                    0x336C4C
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM0_CH0_CTR_CNT                     0x336C50
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM0_CH1_CTR_CNT                     0x336C54
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM1_CH0_CTR_CNT                     0x336C58
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM1_CH1_CTR_CNT                     0x336C5C
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM2_CH0_CTR_CNT                     0x336C60
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM2_CH1_CTR_CNT                     0x336C64
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM3_CH0_CTR_CNT                     0x336C68
+
+#define mmSIF_RTR_CTRL_3_E2E_AR_HBM3_CH1_CTR_CNT                     0x336C6C
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM0_CH0_CTR_WRAP                    0x336C70
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM0_CH1_CTR_WRAP                    0x336C74
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM1_CH0_CTR_WRAP                    0x336C78
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM1_CH1_CTR_WRAP                    0x336C7C
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM2_CH0_CTR_WRAP                    0x336C80
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM2_CH1_CTR_WRAP                    0x336C84
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM3_CH0_CTR_WRAP                    0x336C88
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM3_CH1_CTR_WRAP                    0x336C8C
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM0_CH0_CTR_CNT                     0x336C90
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM0_CH1_CTR_CNT                     0x336C94
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM1_CH0_CTR_CNT                     0x336C98
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM1_CH1_CTR_CNT                     0x336C9C
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM2_CH0_CTR_CNT                     0x336CA0
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM2_CH1_CTR_CNT                     0x336CA4
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM3_CH0_CTR_CNT                     0x336CA8
+
+#define mmSIF_RTR_CTRL_3_E2E_AW_HBM3_CH1_CTR_CNT                     0x336CAC
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_PC_SEL_0                             0x336CB0
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_PC_SEL_1                             0x336CB4
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_PC_SEL_2                             0x336CB8
+
+#define mmSIF_RTR_CTRL_3_NL_HBM_PC_SEL_3                             0x336CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_3_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_4_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_4_regs.h
new file mode 100644
index 000000000000..ad48773c4bbd
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_4_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_4_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_4_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_4 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_4_PERM_SEL                                    0x346108
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_0                               0x346114
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_1                               0x346118
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_2                               0x34611C
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_3                               0x346120
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_4                               0x346124
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_5                               0x346128
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_6                               0x34612C
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_7                               0x346130
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_8                               0x346134
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_9                               0x346138
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_10                              0x34613C
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_11                              0x346140
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_12                              0x346144
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_13                              0x346148
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_14                              0x34614C
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_15                              0x346150
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_16                              0x346154
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_17                              0x346158
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_18                              0x34615C
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_19                              0x346160
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_20                              0x346164
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_21                              0x346168
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_22                              0x34616C
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_23                              0x346170
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_24                              0x346174
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_25                              0x346178
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_26                              0x34617C
+
+#define mmSIF_RTR_CTRL_4_HBM_POLY_H3_27                              0x346180
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_0                              0x346184
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_1                              0x346188
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_2                              0x34618C
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_3                              0x346190
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_4                              0x346194
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_5                              0x346198
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_6                              0x34619C
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_7                              0x3461A0
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_8                              0x3461A4
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_9                              0x3461A8
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_10                             0x3461AC
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_11                             0x3461B0
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_12                             0x3461B4
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_13                             0x3461B8
+
+#define mmSIF_RTR_CTRL_4_SRAM_POLY_H3_14                             0x3461BC
+
+#define mmSIF_RTR_CTRL_4_SCRAM_SRAM_EN                               0x34626C
+
+#define mmSIF_RTR_CTRL_4_RL_HBM_EN                                   0x346274
+
+#define mmSIF_RTR_CTRL_4_RL_HBM_SAT                                  0x346278
+
+#define mmSIF_RTR_CTRL_4_RL_HBM_RST                                  0x34627C
+
+#define mmSIF_RTR_CTRL_4_RL_HBM_TIMEOUT                              0x346280
+
+#define mmSIF_RTR_CTRL_4_SCRAM_HBM_EN                                0x346284
+
+#define mmSIF_RTR_CTRL_4_RL_PCI_EN                                   0x346288
+
+#define mmSIF_RTR_CTRL_4_RL_PCI_SAT                                  0x34628C
+
+#define mmSIF_RTR_CTRL_4_RL_PCI_RST                                  0x346290
+
+#define mmSIF_RTR_CTRL_4_RL_PCI_TIMEOUT                              0x346294
+
+#define mmSIF_RTR_CTRL_4_RL_SRAM_EN                                  0x34629C
+
+#define mmSIF_RTR_CTRL_4_RL_SRAM_SAT                                 0x3462A0
+
+#define mmSIF_RTR_CTRL_4_RL_SRAM_RST                                 0x3462A4
+
+#define mmSIF_RTR_CTRL_4_RL_SRAM_TIMEOUT                             0x3462AC
+
+#define mmSIF_RTR_CTRL_4_RL_SRAM_RED                                 0x3462B4
+
+#define mmSIF_RTR_CTRL_4_E2E_HBM_EN                                  0x3462EC
+
+#define mmSIF_RTR_CTRL_4_E2E_PCI_EN                                  0x3462F0
+
+#define mmSIF_RTR_CTRL_4_E2E_HBM_WR_SIZE                             0x3462F4
+
+#define mmSIF_RTR_CTRL_4_E2E_PCI_WR_SIZE                             0x3462F8
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_PCI_CTR_SET_EN                       0x346404
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_PCI_CTR_SET                          0x346408
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_PCI_CTR_WRAP                         0x34640C
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_PCI_CTR_CNT                          0x346410
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM_CTR_SET_EN                       0x346414
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM_CTR_SET                          0x346418
+
+#define mmSIF_RTR_CTRL_4_E2E_HBM_RD_SIZE                             0x34641C
+
+#define mmSIF_RTR_CTRL_4_E2E_PCI_RD_SIZE                             0x346420
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_PCI_CTR_SET_EN                       0x346424
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_PCI_CTR_SET                          0x346428
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_PCI_CTR_WRAP                         0x34642C
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_PCI_CTR_CNT                          0x346430
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM_CTR_SET_EN                       0x346434
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM_CTR_SET                          0x346438
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_SEL_0                                0x346450
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_SEL_1                                0x346454
+
+#define mmSIF_RTR_CTRL_4_NON_LIN_EN                                  0x346480
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_BANK_0                              0x346500
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_BANK_1                              0x346504
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_BANK_2                              0x346508
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_BANK_3                              0x34650C
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_BANK_4                              0x346510
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_0                            0x346514
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_1                            0x346520
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_2                            0x346524
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_3                            0x346528
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_4                            0x34652C
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_5                            0x346530
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_6                            0x346534
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_7                            0x346538
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_8                            0x34653C
+
+#define mmSIF_RTR_CTRL_4_NL_SRAM_OFFSET_9                            0x346540
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_0                             0x346550
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_1                             0x346554
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_2                             0x346558
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_3                             0x34655C
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_4                             0x346560
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_5                             0x346564
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_6                             0x346568
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_7                             0x34656C
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_8                             0x346570
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_9                             0x346574
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_10                            0x346578
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_11                            0x34657C
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_12                            0x346580
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_13                            0x346584
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_14                            0x346588
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_15                            0x34658C
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_16                            0x346590
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_17                            0x346594
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_OFFSET_18                            0x346598
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_0                     0x3465E4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_1                     0x3465E8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_2                     0x3465EC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_3                     0x3465F0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_4                     0x3465F4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_5                     0x3465F8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_6                     0x3465FC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_7                     0x346600
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_8                     0x346604
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_9                     0x346608
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_10                    0x34660C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_11                    0x346610
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_12                    0x346614
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_13                    0x346618
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_14                    0x34661C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AW_15                    0x346620
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_0                    0x346624
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_1                    0x346628
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_2                    0x34662C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_3                    0x346630
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_4                    0x346634
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_5                    0x346638
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_6                    0x34663C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_7                    0x346640
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_8                    0x346644
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_9                    0x346648
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_10                   0x34664C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_11                   0x346650
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_12                   0x346654
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_13                   0x346658
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_14                   0x34665C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AW_15                   0x346660
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_0                     0x346664
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_1                     0x346668
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_2                     0x34666C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_3                     0x346670
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_4                     0x346674
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_5                     0x346678
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_6                     0x34667C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_7                     0x346680
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_8                     0x346684
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_9                     0x346688
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_10                    0x34668C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_11                    0x346690
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_12                    0x346694
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_13                    0x346698
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_14                    0x34669C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AW_15                    0x3466A0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_0                    0x3466A4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_1                    0x3466A8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_2                    0x3466AC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_3                    0x3466B0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_4                    0x3466B4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_5                    0x3466B8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_6                    0x3466BC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_7                    0x3466C0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_8                    0x3466C4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_9                    0x3466C8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_10                   0x3466CC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_11                   0x3466D0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_12                   0x3466D4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_13                   0x3466D8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_14                   0x3466DC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AW_15                   0x3466E0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_0                    0x3466E4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_1                    0x3466E8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_2                    0x3466EC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_3                    0x3466F0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_4                    0x3466F4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_5                    0x3466F8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_6                    0x3466FC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_7                    0x346700
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_8                    0x346704
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_9                    0x346708
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_10                   0x34670C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_11                   0x346710
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_12                   0x346714
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_13                   0x346718
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_14                   0x34671C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AW_15                   0x346720
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_0                   0x346724
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_1                   0x346728
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_2                   0x34672C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_3                   0x346730
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_4                   0x346734
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_5                   0x346738
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_6                   0x34673C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_7                   0x346740
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_8                   0x346744
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_9                   0x346748
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_10                  0x34674C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_11                  0x346750
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_12                  0x346754
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_13                  0x346758
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_14                  0x34675C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AW_15                  0x346760
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_0                    0x346764
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_1                    0x346768
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_2                    0x34676C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_3                    0x346770
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_4                    0x346774
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_5                    0x346778
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_6                    0x34677C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_7                    0x346780
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_8                    0x346784
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_9                    0x346788
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_10                   0x34678C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_11                   0x346790
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_12                   0x346794
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_13                   0x346798
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_14                   0x34679C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AW_15                   0x3467A0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_0                   0x3467A4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_1                   0x3467A8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_2                   0x3467AC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_3                   0x3467B0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_4                   0x3467B4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_5                   0x3467B8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_6                   0x3467BC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_7                   0x3467C0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_8                   0x3467C4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_9                   0x3467C8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_10                  0x3467CC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_11                  0x3467D0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_12                  0x3467D4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_13                  0x3467D8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_14                  0x3467DC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AW_15                  0x3467E0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_0                     0x346824
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_1                     0x346828
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_2                     0x34682C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_3                     0x346830
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_4                     0x346834
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_5                     0x346838
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_6                     0x34683C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_7                     0x346840
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_8                     0x346844
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_9                     0x346848
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_10                    0x34684C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_11                    0x346850
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_12                    0x346854
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_13                    0x346858
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_14                    0x34685C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_LOW_AR_15                    0x346860
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_0                    0x346864
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_1                    0x346868
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_2                    0x34686C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_3                    0x346870
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_4                    0x346874
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_5                    0x346878
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_6                    0x34687C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_7                    0x346880
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_8                    0x346884
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_9                    0x346888
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_10                   0x34688C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_11                   0x346890
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_12                   0x346894
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_13                   0x346898
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_14                   0x34689C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_BASE_HIGH_AR_15                   0x3468A0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_0                     0x3468A4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_1                     0x3468A8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_2                     0x3468AC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_3                     0x3468B0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_4                     0x3468B4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_5                     0x3468B8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_6                     0x3468BC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_7                     0x3468C0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_8                     0x3468C4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_9                     0x3468C8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_10                    0x3468CC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_11                    0x3468D0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_12                    0x3468D4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_13                    0x3468D8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_14                    0x3468DC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_LOW_AR_15                    0x3468E0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_0                    0x3468E4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_1                    0x3468E8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_2                    0x3468EC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_3                    0x3468F0
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_4                    0x3468F4
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_5                    0x3468F8
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_6                    0x3468FC
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_7                    0x346900
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_8                    0x346904
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_9                    0x346908
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_10                   0x34690C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_11                   0x346910
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_12                   0x346914
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_13                   0x346918
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_14                   0x34691C
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_MASK_HIGH_AR_15                   0x346920
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_0                    0x346924
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_1                    0x346928
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_2                    0x34692C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_3                    0x346930
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_4                    0x346934
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_5                    0x346938
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_6                    0x34693C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_7                    0x346940
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_8                    0x346944
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_9                    0x346948
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_10                   0x34694C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_11                   0x346950
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_12                   0x346954
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_13                   0x346958
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_14                   0x34695C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_LOW_AR_15                   0x346960
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_0                   0x346964
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_1                   0x346968
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_2                   0x34696C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_3                   0x346970
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_4                   0x346974
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_5                   0x346978
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_6                   0x34697C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_7                   0x346980
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_8                   0x346984
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_9                   0x346988
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_10                  0x34698C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_11                  0x346990
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_12                  0x346994
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_13                  0x346998
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_14                  0x34699C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_BASE_HIGH_AR_15                  0x3469A0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_0                    0x3469A4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_1                    0x3469A8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_2                    0x3469AC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_3                    0x3469B0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_4                    0x3469B4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_5                    0x3469B8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_6                    0x3469BC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_7                    0x3469C0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_8                    0x3469C4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_9                    0x3469C8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_10                   0x3469CC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_11                   0x3469D0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_12                   0x3469D4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_13                   0x3469D8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_14                   0x3469DC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_LOW_AR_15                   0x3469E0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_0                   0x3469E4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_1                   0x3469E8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_2                   0x3469EC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_3                   0x3469F0
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_4                   0x3469F4
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_5                   0x3469F8
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_6                   0x3469FC
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_7                   0x346A00
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_8                   0x346A04
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_9                   0x346A08
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_10                  0x346A0C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_11                  0x346A10
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_12                  0x346A14
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_13                  0x346A18
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_14                  0x346A1C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_MASK_HIGH_AR_15                  0x346A20
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_HIT_AW                            0x346A64
+
+#define mmSIF_RTR_CTRL_4_RANGE_SEC_HIT_AR                            0x346A68
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_HIT_AW                           0x346A6C
+
+#define mmSIF_RTR_CTRL_4_RANGE_PRIV_HIT_AR                           0x346A70
+
+#define mmSIF_RTR_CTRL_4_RGL_CFG                                     0x346B64
+
+#define mmSIF_RTR_CTRL_4_RGL_SHIFT                                   0x346B68
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_0                          0x346B6C
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_1                          0x346B70
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_2                          0x346B74
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_3                          0x346B78
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_4                          0x346B7C
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_5                          0x346B80
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_6                          0x346B84
+
+#define mmSIF_RTR_CTRL_4_RGL_EXPECTED_LAT_7                          0x346B88
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_0                                 0x346BAC
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_1                                 0x346BB0
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_2                                 0x346BB4
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_3                                 0x346BB8
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_4                                 0x346BBC
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_5                                 0x346BC0
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_6                                 0x346BC4
+
+#define mmSIF_RTR_CTRL_4_RGL_TOKEN_7                                 0x346BC8
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_0                               0x346BEC
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_1                               0x346BF0
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_2                               0x346BF4
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_3                               0x346BF8
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_4                               0x346BFC
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_5                               0x346C00
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_6                               0x346C04
+
+#define mmSIF_RTR_CTRL_4_RGL_BANK_ID_7                               0x346C08
+
+#define mmSIF_RTR_CTRL_4_RGL_WDT                                     0x346C2C
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM0_CH0_CTR_WRAP                    0x346C30
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM0_CH1_CTR_WRAP                    0x346C34
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM1_CH0_CTR_WRAP                    0x346C38
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM1_CH1_CTR_WRAP                    0x346C3C
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM2_CH0_CTR_WRAP                    0x346C40
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM2_CH1_CTR_WRAP                    0x346C44
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM3_CH0_CTR_WRAP                    0x346C48
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM3_CH1_CTR_WRAP                    0x346C4C
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM0_CH0_CTR_CNT                     0x346C50
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM0_CH1_CTR_CNT                     0x346C54
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM1_CH0_CTR_CNT                     0x346C58
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM1_CH1_CTR_CNT                     0x346C5C
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM2_CH0_CTR_CNT                     0x346C60
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM2_CH1_CTR_CNT                     0x346C64
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM3_CH0_CTR_CNT                     0x346C68
+
+#define mmSIF_RTR_CTRL_4_E2E_AR_HBM3_CH1_CTR_CNT                     0x346C6C
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM0_CH0_CTR_WRAP                    0x346C70
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM0_CH1_CTR_WRAP                    0x346C74
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM1_CH0_CTR_WRAP                    0x346C78
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM1_CH1_CTR_WRAP                    0x346C7C
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM2_CH0_CTR_WRAP                    0x346C80
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM2_CH1_CTR_WRAP                    0x346C84
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM3_CH0_CTR_WRAP                    0x346C88
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM3_CH1_CTR_WRAP                    0x346C8C
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM0_CH0_CTR_CNT                     0x346C90
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM0_CH1_CTR_CNT                     0x346C94
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM1_CH0_CTR_CNT                     0x346C98
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM1_CH1_CTR_CNT                     0x346C9C
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM2_CH0_CTR_CNT                     0x346CA0
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM2_CH1_CTR_CNT                     0x346CA4
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM3_CH0_CTR_CNT                     0x346CA8
+
+#define mmSIF_RTR_CTRL_4_E2E_AW_HBM3_CH1_CTR_CNT                     0x346CAC
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_PC_SEL_0                             0x346CB0
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_PC_SEL_1                             0x346CB4
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_PC_SEL_2                             0x346CB8
+
+#define mmSIF_RTR_CTRL_4_NL_HBM_PC_SEL_3                             0x346CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_4_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_5_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_5_regs.h
new file mode 100644
index 000000000000..6c27850ca3f5
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_5_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_5_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_5_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_5 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_5_PERM_SEL                                    0x356108
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_0                               0x356114
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_1                               0x356118
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_2                               0x35611C
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_3                               0x356120
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_4                               0x356124
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_5                               0x356128
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_6                               0x35612C
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_7                               0x356130
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_8                               0x356134
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_9                               0x356138
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_10                              0x35613C
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_11                              0x356140
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_12                              0x356144
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_13                              0x356148
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_14                              0x35614C
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_15                              0x356150
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_16                              0x356154
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_17                              0x356158
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_18                              0x35615C
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_19                              0x356160
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_20                              0x356164
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_21                              0x356168
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_22                              0x35616C
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_23                              0x356170
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_24                              0x356174
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_25                              0x356178
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_26                              0x35617C
+
+#define mmSIF_RTR_CTRL_5_HBM_POLY_H3_27                              0x356180
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_0                              0x356184
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_1                              0x356188
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_2                              0x35618C
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_3                              0x356190
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_4                              0x356194
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_5                              0x356198
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_6                              0x35619C
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_7                              0x3561A0
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_8                              0x3561A4
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_9                              0x3561A8
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_10                             0x3561AC
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_11                             0x3561B0
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_12                             0x3561B4
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_13                             0x3561B8
+
+#define mmSIF_RTR_CTRL_5_SRAM_POLY_H3_14                             0x3561BC
+
+#define mmSIF_RTR_CTRL_5_SCRAM_SRAM_EN                               0x35626C
+
+#define mmSIF_RTR_CTRL_5_RL_HBM_EN                                   0x356274
+
+#define mmSIF_RTR_CTRL_5_RL_HBM_SAT                                  0x356278
+
+#define mmSIF_RTR_CTRL_5_RL_HBM_RST                                  0x35627C
+
+#define mmSIF_RTR_CTRL_5_RL_HBM_TIMEOUT                              0x356280
+
+#define mmSIF_RTR_CTRL_5_SCRAM_HBM_EN                                0x356284
+
+#define mmSIF_RTR_CTRL_5_RL_PCI_EN                                   0x356288
+
+#define mmSIF_RTR_CTRL_5_RL_PCI_SAT                                  0x35628C
+
+#define mmSIF_RTR_CTRL_5_RL_PCI_RST                                  0x356290
+
+#define mmSIF_RTR_CTRL_5_RL_PCI_TIMEOUT                              0x356294
+
+#define mmSIF_RTR_CTRL_5_RL_SRAM_EN                                  0x35629C
+
+#define mmSIF_RTR_CTRL_5_RL_SRAM_SAT                                 0x3562A0
+
+#define mmSIF_RTR_CTRL_5_RL_SRAM_RST                                 0x3562A4
+
+#define mmSIF_RTR_CTRL_5_RL_SRAM_TIMEOUT                             0x3562AC
+
+#define mmSIF_RTR_CTRL_5_RL_SRAM_RED                                 0x3562B4
+
+#define mmSIF_RTR_CTRL_5_E2E_HBM_EN                                  0x3562EC
+
+#define mmSIF_RTR_CTRL_5_E2E_PCI_EN                                  0x3562F0
+
+#define mmSIF_RTR_CTRL_5_E2E_HBM_WR_SIZE                             0x3562F4
+
+#define mmSIF_RTR_CTRL_5_E2E_PCI_WR_SIZE                             0x3562F8
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_PCI_CTR_SET_EN                       0x356404
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_PCI_CTR_SET                          0x356408
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_PCI_CTR_WRAP                         0x35640C
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_PCI_CTR_CNT                          0x356410
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM_CTR_SET_EN                       0x356414
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM_CTR_SET                          0x356418
+
+#define mmSIF_RTR_CTRL_5_E2E_HBM_RD_SIZE                             0x35641C
+
+#define mmSIF_RTR_CTRL_5_E2E_PCI_RD_SIZE                             0x356420
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_PCI_CTR_SET_EN                       0x356424
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_PCI_CTR_SET                          0x356428
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_PCI_CTR_WRAP                         0x35642C
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_PCI_CTR_CNT                          0x356430
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM_CTR_SET_EN                       0x356434
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM_CTR_SET                          0x356438
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_SEL_0                                0x356450
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_SEL_1                                0x356454
+
+#define mmSIF_RTR_CTRL_5_NON_LIN_EN                                  0x356480
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_BANK_0                              0x356500
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_BANK_1                              0x356504
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_BANK_2                              0x356508
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_BANK_3                              0x35650C
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_BANK_4                              0x356510
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_0                            0x356514
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_1                            0x356520
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_2                            0x356524
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_3                            0x356528
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_4                            0x35652C
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_5                            0x356530
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_6                            0x356534
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_7                            0x356538
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_8                            0x35653C
+
+#define mmSIF_RTR_CTRL_5_NL_SRAM_OFFSET_9                            0x356540
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_0                             0x356550
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_1                             0x356554
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_2                             0x356558
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_3                             0x35655C
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_4                             0x356560
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_5                             0x356564
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_6                             0x356568
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_7                             0x35656C
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_8                             0x356570
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_9                             0x356574
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_10                            0x356578
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_11                            0x35657C
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_12                            0x356580
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_13                            0x356584
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_14                            0x356588
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_15                            0x35658C
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_16                            0x356590
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_17                            0x356594
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_OFFSET_18                            0x356598
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_0                     0x3565E4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_1                     0x3565E8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_2                     0x3565EC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_3                     0x3565F0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_4                     0x3565F4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_5                     0x3565F8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_6                     0x3565FC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_7                     0x356600
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_8                     0x356604
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_9                     0x356608
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_10                    0x35660C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_11                    0x356610
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_12                    0x356614
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_13                    0x356618
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_14                    0x35661C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AW_15                    0x356620
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_0                    0x356624
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_1                    0x356628
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_2                    0x35662C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_3                    0x356630
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_4                    0x356634
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_5                    0x356638
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_6                    0x35663C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_7                    0x356640
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_8                    0x356644
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_9                    0x356648
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_10                   0x35664C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_11                   0x356650
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_12                   0x356654
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_13                   0x356658
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_14                   0x35665C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AW_15                   0x356660
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_0                     0x356664
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_1                     0x356668
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_2                     0x35666C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_3                     0x356670
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_4                     0x356674
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_5                     0x356678
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_6                     0x35667C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_7                     0x356680
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_8                     0x356684
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_9                     0x356688
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_10                    0x35668C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_11                    0x356690
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_12                    0x356694
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_13                    0x356698
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_14                    0x35669C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AW_15                    0x3566A0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_0                    0x3566A4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_1                    0x3566A8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_2                    0x3566AC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_3                    0x3566B0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_4                    0x3566B4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_5                    0x3566B8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_6                    0x3566BC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_7                    0x3566C0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_8                    0x3566C4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_9                    0x3566C8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_10                   0x3566CC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_11                   0x3566D0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_12                   0x3566D4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_13                   0x3566D8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_14                   0x3566DC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AW_15                   0x3566E0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_0                    0x3566E4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_1                    0x3566E8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_2                    0x3566EC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_3                    0x3566F0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_4                    0x3566F4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_5                    0x3566F8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_6                    0x3566FC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_7                    0x356700
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_8                    0x356704
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_9                    0x356708
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_10                   0x35670C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_11                   0x356710
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_12                   0x356714
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_13                   0x356718
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_14                   0x35671C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AW_15                   0x356720
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_0                   0x356724
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_1                   0x356728
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_2                   0x35672C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_3                   0x356730
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_4                   0x356734
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_5                   0x356738
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_6                   0x35673C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_7                   0x356740
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_8                   0x356744
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_9                   0x356748
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_10                  0x35674C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_11                  0x356750
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_12                  0x356754
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_13                  0x356758
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_14                  0x35675C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AW_15                  0x356760
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_0                    0x356764
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_1                    0x356768
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_2                    0x35676C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_3                    0x356770
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_4                    0x356774
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_5                    0x356778
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_6                    0x35677C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_7                    0x356780
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_8                    0x356784
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_9                    0x356788
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_10                   0x35678C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_11                   0x356790
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_12                   0x356794
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_13                   0x356798
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_14                   0x35679C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AW_15                   0x3567A0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_0                   0x3567A4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_1                   0x3567A8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_2                   0x3567AC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_3                   0x3567B0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_4                   0x3567B4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_5                   0x3567B8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_6                   0x3567BC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_7                   0x3567C0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_8                   0x3567C4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_9                   0x3567C8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_10                  0x3567CC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_11                  0x3567D0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_12                  0x3567D4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_13                  0x3567D8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_14                  0x3567DC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AW_15                  0x3567E0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_0                     0x356824
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_1                     0x356828
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_2                     0x35682C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_3                     0x356830
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_4                     0x356834
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_5                     0x356838
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_6                     0x35683C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_7                     0x356840
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_8                     0x356844
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_9                     0x356848
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_10                    0x35684C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_11                    0x356850
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_12                    0x356854
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_13                    0x356858
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_14                    0x35685C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_LOW_AR_15                    0x356860
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_0                    0x356864
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_1                    0x356868
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_2                    0x35686C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_3                    0x356870
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_4                    0x356874
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_5                    0x356878
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_6                    0x35687C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_7                    0x356880
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_8                    0x356884
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_9                    0x356888
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_10                   0x35688C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_11                   0x356890
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_12                   0x356894
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_13                   0x356898
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_14                   0x35689C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_BASE_HIGH_AR_15                   0x3568A0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_0                     0x3568A4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_1                     0x3568A8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_2                     0x3568AC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_3                     0x3568B0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_4                     0x3568B4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_5                     0x3568B8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_6                     0x3568BC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_7                     0x3568C0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_8                     0x3568C4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_9                     0x3568C8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_10                    0x3568CC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_11                    0x3568D0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_12                    0x3568D4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_13                    0x3568D8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_14                    0x3568DC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_LOW_AR_15                    0x3568E0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_0                    0x3568E4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_1                    0x3568E8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_2                    0x3568EC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_3                    0x3568F0
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_4                    0x3568F4
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_5                    0x3568F8
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_6                    0x3568FC
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_7                    0x356900
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_8                    0x356904
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_9                    0x356908
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_10                   0x35690C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_11                   0x356910
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_12                   0x356914
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_13                   0x356918
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_14                   0x35691C
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_MASK_HIGH_AR_15                   0x356920
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_0                    0x356924
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_1                    0x356928
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_2                    0x35692C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_3                    0x356930
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_4                    0x356934
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_5                    0x356938
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_6                    0x35693C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_7                    0x356940
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_8                    0x356944
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_9                    0x356948
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_10                   0x35694C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_11                   0x356950
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_12                   0x356954
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_13                   0x356958
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_14                   0x35695C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_LOW_AR_15                   0x356960
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_0                   0x356964
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_1                   0x356968
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_2                   0x35696C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_3                   0x356970
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_4                   0x356974
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_5                   0x356978
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_6                   0x35697C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_7                   0x356980
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_8                   0x356984
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_9                   0x356988
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_10                  0x35698C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_11                  0x356990
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_12                  0x356994
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_13                  0x356998
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_14                  0x35699C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_BASE_HIGH_AR_15                  0x3569A0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_0                    0x3569A4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_1                    0x3569A8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_2                    0x3569AC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_3                    0x3569B0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_4                    0x3569B4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_5                    0x3569B8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_6                    0x3569BC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_7                    0x3569C0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_8                    0x3569C4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_9                    0x3569C8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_10                   0x3569CC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_11                   0x3569D0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_12                   0x3569D4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_13                   0x3569D8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_14                   0x3569DC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_LOW_AR_15                   0x3569E0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_0                   0x3569E4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_1                   0x3569E8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_2                   0x3569EC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_3                   0x3569F0
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_4                   0x3569F4
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_5                   0x3569F8
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_6                   0x3569FC
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_7                   0x356A00
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_8                   0x356A04
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_9                   0x356A08
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_10                  0x356A0C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_11                  0x356A10
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_12                  0x356A14
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_13                  0x356A18
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_14                  0x356A1C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_MASK_HIGH_AR_15                  0x356A20
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_HIT_AW                            0x356A64
+
+#define mmSIF_RTR_CTRL_5_RANGE_SEC_HIT_AR                            0x356A68
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_HIT_AW                           0x356A6C
+
+#define mmSIF_RTR_CTRL_5_RANGE_PRIV_HIT_AR                           0x356A70
+
+#define mmSIF_RTR_CTRL_5_RGL_CFG                                     0x356B64
+
+#define mmSIF_RTR_CTRL_5_RGL_SHIFT                                   0x356B68
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_0                          0x356B6C
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_1                          0x356B70
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_2                          0x356B74
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_3                          0x356B78
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_4                          0x356B7C
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_5                          0x356B80
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_6                          0x356B84
+
+#define mmSIF_RTR_CTRL_5_RGL_EXPECTED_LAT_7                          0x356B88
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_0                                 0x356BAC
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_1                                 0x356BB0
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_2                                 0x356BB4
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_3                                 0x356BB8
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_4                                 0x356BBC
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_5                                 0x356BC0
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_6                                 0x356BC4
+
+#define mmSIF_RTR_CTRL_5_RGL_TOKEN_7                                 0x356BC8
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_0                               0x356BEC
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_1                               0x356BF0
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_2                               0x356BF4
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_3                               0x356BF8
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_4                               0x356BFC
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_5                               0x356C00
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_6                               0x356C04
+
+#define mmSIF_RTR_CTRL_5_RGL_BANK_ID_7                               0x356C08
+
+#define mmSIF_RTR_CTRL_5_RGL_WDT                                     0x356C2C
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM0_CH0_CTR_WRAP                    0x356C30
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM0_CH1_CTR_WRAP                    0x356C34
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM1_CH0_CTR_WRAP                    0x356C38
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM1_CH1_CTR_WRAP                    0x356C3C
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM2_CH0_CTR_WRAP                    0x356C40
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM2_CH1_CTR_WRAP                    0x356C44
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM3_CH0_CTR_WRAP                    0x356C48
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM3_CH1_CTR_WRAP                    0x356C4C
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM0_CH0_CTR_CNT                     0x356C50
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM0_CH1_CTR_CNT                     0x356C54
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM1_CH0_CTR_CNT                     0x356C58
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM1_CH1_CTR_CNT                     0x356C5C
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM2_CH0_CTR_CNT                     0x356C60
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM2_CH1_CTR_CNT                     0x356C64
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM3_CH0_CTR_CNT                     0x356C68
+
+#define mmSIF_RTR_CTRL_5_E2E_AR_HBM3_CH1_CTR_CNT                     0x356C6C
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM0_CH0_CTR_WRAP                    0x356C70
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM0_CH1_CTR_WRAP                    0x356C74
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM1_CH0_CTR_WRAP                    0x356C78
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM1_CH1_CTR_WRAP                    0x356C7C
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM2_CH0_CTR_WRAP                    0x356C80
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM2_CH1_CTR_WRAP                    0x356C84
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM3_CH0_CTR_WRAP                    0x356C88
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM3_CH1_CTR_WRAP                    0x356C8C
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM0_CH0_CTR_CNT                     0x356C90
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM0_CH1_CTR_CNT                     0x356C94
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM1_CH0_CTR_CNT                     0x356C98
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM1_CH1_CTR_CNT                     0x356C9C
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM2_CH0_CTR_CNT                     0x356CA0
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM2_CH1_CTR_CNT                     0x356CA4
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM3_CH0_CTR_CNT                     0x356CA8
+
+#define mmSIF_RTR_CTRL_5_E2E_AW_HBM3_CH1_CTR_CNT                     0x356CAC
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_PC_SEL_0                             0x356CB0
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_PC_SEL_1                             0x356CB4
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_PC_SEL_2                             0x356CB8
+
+#define mmSIF_RTR_CTRL_5_NL_HBM_PC_SEL_3                             0x356CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_5_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_6_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_6_regs.h
new file mode 100644
index 000000000000..a9ea89aa6405
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_6_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_6_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_6_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_6 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_6_PERM_SEL                                    0x366108
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_0                               0x366114
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_1                               0x366118
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_2                               0x36611C
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_3                               0x366120
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_4                               0x366124
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_5                               0x366128
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_6                               0x36612C
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_7                               0x366130
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_8                               0x366134
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_9                               0x366138
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_10                              0x36613C
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_11                              0x366140
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_12                              0x366144
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_13                              0x366148
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_14                              0x36614C
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_15                              0x366150
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_16                              0x366154
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_17                              0x366158
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_18                              0x36615C
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_19                              0x366160
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_20                              0x366164
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_21                              0x366168
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_22                              0x36616C
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_23                              0x366170
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_24                              0x366174
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_25                              0x366178
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_26                              0x36617C
+
+#define mmSIF_RTR_CTRL_6_HBM_POLY_H3_27                              0x366180
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_0                              0x366184
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_1                              0x366188
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_2                              0x36618C
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_3                              0x366190
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_4                              0x366194
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_5                              0x366198
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_6                              0x36619C
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_7                              0x3661A0
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_8                              0x3661A4
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_9                              0x3661A8
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_10                             0x3661AC
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_11                             0x3661B0
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_12                             0x3661B4
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_13                             0x3661B8
+
+#define mmSIF_RTR_CTRL_6_SRAM_POLY_H3_14                             0x3661BC
+
+#define mmSIF_RTR_CTRL_6_SCRAM_SRAM_EN                               0x36626C
+
+#define mmSIF_RTR_CTRL_6_RL_HBM_EN                                   0x366274
+
+#define mmSIF_RTR_CTRL_6_RL_HBM_SAT                                  0x366278
+
+#define mmSIF_RTR_CTRL_6_RL_HBM_RST                                  0x36627C
+
+#define mmSIF_RTR_CTRL_6_RL_HBM_TIMEOUT                              0x366280
+
+#define mmSIF_RTR_CTRL_6_SCRAM_HBM_EN                                0x366284
+
+#define mmSIF_RTR_CTRL_6_RL_PCI_EN                                   0x366288
+
+#define mmSIF_RTR_CTRL_6_RL_PCI_SAT                                  0x36628C
+
+#define mmSIF_RTR_CTRL_6_RL_PCI_RST                                  0x366290
+
+#define mmSIF_RTR_CTRL_6_RL_PCI_TIMEOUT                              0x366294
+
+#define mmSIF_RTR_CTRL_6_RL_SRAM_EN                                  0x36629C
+
+#define mmSIF_RTR_CTRL_6_RL_SRAM_SAT                                 0x3662A0
+
+#define mmSIF_RTR_CTRL_6_RL_SRAM_RST                                 0x3662A4
+
+#define mmSIF_RTR_CTRL_6_RL_SRAM_TIMEOUT                             0x3662AC
+
+#define mmSIF_RTR_CTRL_6_RL_SRAM_RED                                 0x3662B4
+
+#define mmSIF_RTR_CTRL_6_E2E_HBM_EN                                  0x3662EC
+
+#define mmSIF_RTR_CTRL_6_E2E_PCI_EN                                  0x3662F0
+
+#define mmSIF_RTR_CTRL_6_E2E_HBM_WR_SIZE                             0x3662F4
+
+#define mmSIF_RTR_CTRL_6_E2E_PCI_WR_SIZE                             0x3662F8
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_PCI_CTR_SET_EN                       0x366404
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_PCI_CTR_SET                          0x366408
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_PCI_CTR_WRAP                         0x36640C
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_PCI_CTR_CNT                          0x366410
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM_CTR_SET_EN                       0x366414
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM_CTR_SET                          0x366418
+
+#define mmSIF_RTR_CTRL_6_E2E_HBM_RD_SIZE                             0x36641C
+
+#define mmSIF_RTR_CTRL_6_E2E_PCI_RD_SIZE                             0x366420
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_PCI_CTR_SET_EN                       0x366424
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_PCI_CTR_SET                          0x366428
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_PCI_CTR_WRAP                         0x36642C
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_PCI_CTR_CNT                          0x366430
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM_CTR_SET_EN                       0x366434
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM_CTR_SET                          0x366438
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_SEL_0                                0x366450
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_SEL_1                                0x366454
+
+#define mmSIF_RTR_CTRL_6_NON_LIN_EN                                  0x366480
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_BANK_0                              0x366500
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_BANK_1                              0x366504
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_BANK_2                              0x366508
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_BANK_3                              0x36650C
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_BANK_4                              0x366510
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_0                            0x366514
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_1                            0x366520
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_2                            0x366524
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_3                            0x366528
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_4                            0x36652C
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_5                            0x366530
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_6                            0x366534
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_7                            0x366538
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_8                            0x36653C
+
+#define mmSIF_RTR_CTRL_6_NL_SRAM_OFFSET_9                            0x366540
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_0                             0x366550
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_1                             0x366554
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_2                             0x366558
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_3                             0x36655C
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_4                             0x366560
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_5                             0x366564
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_6                             0x366568
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_7                             0x36656C
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_8                             0x366570
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_9                             0x366574
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_10                            0x366578
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_11                            0x36657C
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_12                            0x366580
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_13                            0x366584
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_14                            0x366588
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_15                            0x36658C
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_16                            0x366590
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_17                            0x366594
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_OFFSET_18                            0x366598
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_0                     0x3665E4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_1                     0x3665E8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_2                     0x3665EC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_3                     0x3665F0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_4                     0x3665F4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_5                     0x3665F8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_6                     0x3665FC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_7                     0x366600
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_8                     0x366604
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_9                     0x366608
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_10                    0x36660C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_11                    0x366610
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_12                    0x366614
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_13                    0x366618
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_14                    0x36661C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AW_15                    0x366620
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_0                    0x366624
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_1                    0x366628
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_2                    0x36662C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_3                    0x366630
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_4                    0x366634
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_5                    0x366638
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_6                    0x36663C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_7                    0x366640
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_8                    0x366644
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_9                    0x366648
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_10                   0x36664C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_11                   0x366650
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_12                   0x366654
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_13                   0x366658
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_14                   0x36665C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AW_15                   0x366660
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_0                     0x366664
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_1                     0x366668
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_2                     0x36666C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_3                     0x366670
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_4                     0x366674
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_5                     0x366678
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_6                     0x36667C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_7                     0x366680
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_8                     0x366684
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_9                     0x366688
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_10                    0x36668C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_11                    0x366690
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_12                    0x366694
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_13                    0x366698
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_14                    0x36669C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AW_15                    0x3666A0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_0                    0x3666A4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_1                    0x3666A8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_2                    0x3666AC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_3                    0x3666B0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_4                    0x3666B4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_5                    0x3666B8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_6                    0x3666BC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_7                    0x3666C0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_8                    0x3666C4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_9                    0x3666C8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_10                   0x3666CC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_11                   0x3666D0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_12                   0x3666D4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_13                   0x3666D8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_14                   0x3666DC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AW_15                   0x3666E0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_0                    0x3666E4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_1                    0x3666E8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_2                    0x3666EC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_3                    0x3666F0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_4                    0x3666F4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_5                    0x3666F8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_6                    0x3666FC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_7                    0x366700
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_8                    0x366704
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_9                    0x366708
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_10                   0x36670C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_11                   0x366710
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_12                   0x366714
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_13                   0x366718
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_14                   0x36671C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AW_15                   0x366720
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_0                   0x366724
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_1                   0x366728
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_2                   0x36672C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_3                   0x366730
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_4                   0x366734
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_5                   0x366738
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_6                   0x36673C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_7                   0x366740
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_8                   0x366744
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_9                   0x366748
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_10                  0x36674C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_11                  0x366750
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_12                  0x366754
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_13                  0x366758
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_14                  0x36675C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AW_15                  0x366760
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_0                    0x366764
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_1                    0x366768
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_2                    0x36676C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_3                    0x366770
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_4                    0x366774
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_5                    0x366778
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_6                    0x36677C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_7                    0x366780
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_8                    0x366784
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_9                    0x366788
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_10                   0x36678C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_11                   0x366790
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_12                   0x366794
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_13                   0x366798
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_14                   0x36679C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AW_15                   0x3667A0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_0                   0x3667A4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_1                   0x3667A8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_2                   0x3667AC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_3                   0x3667B0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_4                   0x3667B4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_5                   0x3667B8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_6                   0x3667BC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_7                   0x3667C0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_8                   0x3667C4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_9                   0x3667C8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_10                  0x3667CC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_11                  0x3667D0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_12                  0x3667D4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_13                  0x3667D8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_14                  0x3667DC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AW_15                  0x3667E0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_0                     0x366824
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_1                     0x366828
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_2                     0x36682C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_3                     0x366830
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_4                     0x366834
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_5                     0x366838
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_6                     0x36683C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_7                     0x366840
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_8                     0x366844
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_9                     0x366848
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_10                    0x36684C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_11                    0x366850
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_12                    0x366854
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_13                    0x366858
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_14                    0x36685C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_LOW_AR_15                    0x366860
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_0                    0x366864
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_1                    0x366868
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_2                    0x36686C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_3                    0x366870
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_4                    0x366874
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_5                    0x366878
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_6                    0x36687C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_7                    0x366880
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_8                    0x366884
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_9                    0x366888
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_10                   0x36688C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_11                   0x366890
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_12                   0x366894
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_13                   0x366898
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_14                   0x36689C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_BASE_HIGH_AR_15                   0x3668A0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_0                     0x3668A4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_1                     0x3668A8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_2                     0x3668AC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_3                     0x3668B0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_4                     0x3668B4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_5                     0x3668B8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_6                     0x3668BC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_7                     0x3668C0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_8                     0x3668C4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_9                     0x3668C8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_10                    0x3668CC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_11                    0x3668D0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_12                    0x3668D4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_13                    0x3668D8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_14                    0x3668DC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_LOW_AR_15                    0x3668E0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_0                    0x3668E4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_1                    0x3668E8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_2                    0x3668EC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_3                    0x3668F0
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_4                    0x3668F4
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_5                    0x3668F8
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_6                    0x3668FC
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_7                    0x366900
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_8                    0x366904
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_9                    0x366908
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_10                   0x36690C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_11                   0x366910
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_12                   0x366914
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_13                   0x366918
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_14                   0x36691C
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_MASK_HIGH_AR_15                   0x366920
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_0                    0x366924
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_1                    0x366928
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_2                    0x36692C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_3                    0x366930
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_4                    0x366934
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_5                    0x366938
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_6                    0x36693C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_7                    0x366940
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_8                    0x366944
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_9                    0x366948
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_10                   0x36694C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_11                   0x366950
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_12                   0x366954
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_13                   0x366958
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_14                   0x36695C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_LOW_AR_15                   0x366960
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_0                   0x366964
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_1                   0x366968
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_2                   0x36696C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_3                   0x366970
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_4                   0x366974
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_5                   0x366978
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_6                   0x36697C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_7                   0x366980
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_8                   0x366984
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_9                   0x366988
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_10                  0x36698C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_11                  0x366990
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_12                  0x366994
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_13                  0x366998
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_14                  0x36699C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_BASE_HIGH_AR_15                  0x3669A0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_0                    0x3669A4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_1                    0x3669A8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_2                    0x3669AC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_3                    0x3669B0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_4                    0x3669B4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_5                    0x3669B8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_6                    0x3669BC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_7                    0x3669C0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_8                    0x3669C4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_9                    0x3669C8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_10                   0x3669CC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_11                   0x3669D0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_12                   0x3669D4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_13                   0x3669D8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_14                   0x3669DC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_LOW_AR_15                   0x3669E0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_0                   0x3669E4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_1                   0x3669E8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_2                   0x3669EC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_3                   0x3669F0
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_4                   0x3669F4
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_5                   0x3669F8
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_6                   0x3669FC
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_7                   0x366A00
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_8                   0x366A04
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_9                   0x366A08
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_10                  0x366A0C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_11                  0x366A10
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_12                  0x366A14
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_13                  0x366A18
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_14                  0x366A1C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_MASK_HIGH_AR_15                  0x366A20
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_HIT_AW                            0x366A64
+
+#define mmSIF_RTR_CTRL_6_RANGE_SEC_HIT_AR                            0x366A68
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_HIT_AW                           0x366A6C
+
+#define mmSIF_RTR_CTRL_6_RANGE_PRIV_HIT_AR                           0x366A70
+
+#define mmSIF_RTR_CTRL_6_RGL_CFG                                     0x366B64
+
+#define mmSIF_RTR_CTRL_6_RGL_SHIFT                                   0x366B68
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_0                          0x366B6C
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_1                          0x366B70
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_2                          0x366B74
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_3                          0x366B78
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_4                          0x366B7C
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_5                          0x366B80
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_6                          0x366B84
+
+#define mmSIF_RTR_CTRL_6_RGL_EXPECTED_LAT_7                          0x366B88
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_0                                 0x366BAC
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_1                                 0x366BB0
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_2                                 0x366BB4
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_3                                 0x366BB8
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_4                                 0x366BBC
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_5                                 0x366BC0
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_6                                 0x366BC4
+
+#define mmSIF_RTR_CTRL_6_RGL_TOKEN_7                                 0x366BC8
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_0                               0x366BEC
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_1                               0x366BF0
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_2                               0x366BF4
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_3                               0x366BF8
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_4                               0x366BFC
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_5                               0x366C00
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_6                               0x366C04
+
+#define mmSIF_RTR_CTRL_6_RGL_BANK_ID_7                               0x366C08
+
+#define mmSIF_RTR_CTRL_6_RGL_WDT                                     0x366C2C
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM0_CH0_CTR_WRAP                    0x366C30
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM0_CH1_CTR_WRAP                    0x366C34
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM1_CH0_CTR_WRAP                    0x366C38
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM1_CH1_CTR_WRAP                    0x366C3C
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM2_CH0_CTR_WRAP                    0x366C40
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM2_CH1_CTR_WRAP                    0x366C44
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM3_CH0_CTR_WRAP                    0x366C48
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM3_CH1_CTR_WRAP                    0x366C4C
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM0_CH0_CTR_CNT                     0x366C50
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM0_CH1_CTR_CNT                     0x366C54
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM1_CH0_CTR_CNT                     0x366C58
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM1_CH1_CTR_CNT                     0x366C5C
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM2_CH0_CTR_CNT                     0x366C60
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM2_CH1_CTR_CNT                     0x366C64
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM3_CH0_CTR_CNT                     0x366C68
+
+#define mmSIF_RTR_CTRL_6_E2E_AR_HBM3_CH1_CTR_CNT                     0x366C6C
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM0_CH0_CTR_WRAP                    0x366C70
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM0_CH1_CTR_WRAP                    0x366C74
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM1_CH0_CTR_WRAP                    0x366C78
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM1_CH1_CTR_WRAP                    0x366C7C
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM2_CH0_CTR_WRAP                    0x366C80
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM2_CH1_CTR_WRAP                    0x366C84
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM3_CH0_CTR_WRAP                    0x366C88
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM3_CH1_CTR_WRAP                    0x366C8C
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM0_CH0_CTR_CNT                     0x366C90
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM0_CH1_CTR_CNT                     0x366C94
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM1_CH0_CTR_CNT                     0x366C98
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM1_CH1_CTR_CNT                     0x366C9C
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM2_CH0_CTR_CNT                     0x366CA0
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM2_CH1_CTR_CNT                     0x366CA4
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM3_CH0_CTR_CNT                     0x366CA8
+
+#define mmSIF_RTR_CTRL_6_E2E_AW_HBM3_CH1_CTR_CNT                     0x366CAC
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_PC_SEL_0                             0x366CB0
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_PC_SEL_1                             0x366CB4
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_PC_SEL_2                             0x366CB8
+
+#define mmSIF_RTR_CTRL_6_NL_HBM_PC_SEL_3                             0x366CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_6_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_7_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_7_regs.h
new file mode 100644
index 000000000000..a37772c531d9
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/sif_rtr_ctrl_7_regs.h
@@ -0,0 +1,896 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_SIF_RTR_CTRL_7_REGS_H_
+#define ASIC_REG_SIF_RTR_CTRL_7_REGS_H_
+
+/*
+ *****************************************
+ *   SIF_RTR_CTRL_7 (Prototype: RTR_CTRL)
+ *****************************************
+ */
+
+#define mmSIF_RTR_CTRL_7_PERM_SEL                                    0x376108
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_0                               0x376114
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_1                               0x376118
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_2                               0x37611C
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_3                               0x376120
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_4                               0x376124
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_5                               0x376128
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_6                               0x37612C
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_7                               0x376130
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_8                               0x376134
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_9                               0x376138
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_10                              0x37613C
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_11                              0x376140
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_12                              0x376144
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_13                              0x376148
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_14                              0x37614C
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_15                              0x376150
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_16                              0x376154
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_17                              0x376158
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_18                              0x37615C
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_19                              0x376160
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_20                              0x376164
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_21                              0x376168
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_22                              0x37616C
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_23                              0x376170
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_24                              0x376174
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_25                              0x376178
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_26                              0x37617C
+
+#define mmSIF_RTR_CTRL_7_HBM_POLY_H3_27                              0x376180
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_0                              0x376184
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_1                              0x376188
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_2                              0x37618C
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_3                              0x376190
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_4                              0x376194
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_5                              0x376198
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_6                              0x37619C
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_7                              0x3761A0
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_8                              0x3761A4
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_9                              0x3761A8
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_10                             0x3761AC
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_11                             0x3761B0
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_12                             0x3761B4
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_13                             0x3761B8
+
+#define mmSIF_RTR_CTRL_7_SRAM_POLY_H3_14                             0x3761BC
+
+#define mmSIF_RTR_CTRL_7_SCRAM_SRAM_EN                               0x37626C
+
+#define mmSIF_RTR_CTRL_7_RL_HBM_EN                                   0x376274
+
+#define mmSIF_RTR_CTRL_7_RL_HBM_SAT                                  0x376278
+
+#define mmSIF_RTR_CTRL_7_RL_HBM_RST                                  0x37627C
+
+#define mmSIF_RTR_CTRL_7_RL_HBM_TIMEOUT                              0x376280
+
+#define mmSIF_RTR_CTRL_7_SCRAM_HBM_EN                                0x376284
+
+#define mmSIF_RTR_CTRL_7_RL_PCI_EN                                   0x376288
+
+#define mmSIF_RTR_CTRL_7_RL_PCI_SAT                                  0x37628C
+
+#define mmSIF_RTR_CTRL_7_RL_PCI_RST                                  0x376290
+
+#define mmSIF_RTR_CTRL_7_RL_PCI_TIMEOUT                              0x376294
+
+#define mmSIF_RTR_CTRL_7_RL_SRAM_EN                                  0x37629C
+
+#define mmSIF_RTR_CTRL_7_RL_SRAM_SAT                                 0x3762A0
+
+#define mmSIF_RTR_CTRL_7_RL_SRAM_RST                                 0x3762A4
+
+#define mmSIF_RTR_CTRL_7_RL_SRAM_TIMEOUT                             0x3762AC
+
+#define mmSIF_RTR_CTRL_7_RL_SRAM_RED                                 0x3762B4
+
+#define mmSIF_RTR_CTRL_7_E2E_HBM_EN                                  0x3762EC
+
+#define mmSIF_RTR_CTRL_7_E2E_PCI_EN                                  0x3762F0
+
+#define mmSIF_RTR_CTRL_7_E2E_HBM_WR_SIZE                             0x3762F4
+
+#define mmSIF_RTR_CTRL_7_E2E_PCI_WR_SIZE                             0x3762F8
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_PCI_CTR_SET_EN                       0x376404
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_PCI_CTR_SET                          0x376408
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_PCI_CTR_WRAP                         0x37640C
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_PCI_CTR_CNT                          0x376410
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM_CTR_SET_EN                       0x376414
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM_CTR_SET                          0x376418
+
+#define mmSIF_RTR_CTRL_7_E2E_HBM_RD_SIZE                             0x37641C
+
+#define mmSIF_RTR_CTRL_7_E2E_PCI_RD_SIZE                             0x376420
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_PCI_CTR_SET_EN                       0x376424
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_PCI_CTR_SET                          0x376428
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_PCI_CTR_WRAP                         0x37642C
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_PCI_CTR_CNT                          0x376430
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM_CTR_SET_EN                       0x376434
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM_CTR_SET                          0x376438
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_SEL_0                                0x376450
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_SEL_1                                0x376454
+
+#define mmSIF_RTR_CTRL_7_NON_LIN_EN                                  0x376480
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_BANK_0                              0x376500
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_BANK_1                              0x376504
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_BANK_2                              0x376508
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_BANK_3                              0x37650C
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_BANK_4                              0x376510
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_0                            0x376514
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_1                            0x376520
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_2                            0x376524
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_3                            0x376528
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_4                            0x37652C
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_5                            0x376530
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_6                            0x376534
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_7                            0x376538
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_8                            0x37653C
+
+#define mmSIF_RTR_CTRL_7_NL_SRAM_OFFSET_9                            0x376540
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_0                             0x376550
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_1                             0x376554
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_2                             0x376558
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_3                             0x37655C
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_4                             0x376560
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_5                             0x376564
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_6                             0x376568
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_7                             0x37656C
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_8                             0x376570
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_9                             0x376574
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_10                            0x376578
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_11                            0x37657C
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_12                            0x376580
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_13                            0x376584
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_14                            0x376588
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_15                            0x37658C
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_16                            0x376590
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_17                            0x376594
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_OFFSET_18                            0x376598
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_0                     0x3765E4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_1                     0x3765E8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_2                     0x3765EC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_3                     0x3765F0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_4                     0x3765F4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_5                     0x3765F8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_6                     0x3765FC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_7                     0x376600
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_8                     0x376604
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_9                     0x376608
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_10                    0x37660C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_11                    0x376610
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_12                    0x376614
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_13                    0x376618
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_14                    0x37661C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AW_15                    0x376620
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_0                    0x376624
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_1                    0x376628
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_2                    0x37662C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_3                    0x376630
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_4                    0x376634
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_5                    0x376638
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_6                    0x37663C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_7                    0x376640
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_8                    0x376644
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_9                    0x376648
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_10                   0x37664C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_11                   0x376650
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_12                   0x376654
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_13                   0x376658
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_14                   0x37665C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AW_15                   0x376660
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_0                     0x376664
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_1                     0x376668
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_2                     0x37666C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_3                     0x376670
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_4                     0x376674
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_5                     0x376678
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_6                     0x37667C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_7                     0x376680
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_8                     0x376684
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_9                     0x376688
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_10                    0x37668C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_11                    0x376690
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_12                    0x376694
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_13                    0x376698
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_14                    0x37669C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AW_15                    0x3766A0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_0                    0x3766A4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_1                    0x3766A8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_2                    0x3766AC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_3                    0x3766B0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_4                    0x3766B4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_5                    0x3766B8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_6                    0x3766BC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_7                    0x3766C0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_8                    0x3766C4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_9                    0x3766C8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_10                   0x3766CC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_11                   0x3766D0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_12                   0x3766D4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_13                   0x3766D8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_14                   0x3766DC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AW_15                   0x3766E0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_0                    0x3766E4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_1                    0x3766E8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_2                    0x3766EC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_3                    0x3766F0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_4                    0x3766F4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_5                    0x3766F8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_6                    0x3766FC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_7                    0x376700
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_8                    0x376704
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_9                    0x376708
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_10                   0x37670C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_11                   0x376710
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_12                   0x376714
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_13                   0x376718
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_14                   0x37671C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AW_15                   0x376720
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_0                   0x376724
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_1                   0x376728
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_2                   0x37672C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_3                   0x376730
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_4                   0x376734
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_5                   0x376738
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_6                   0x37673C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_7                   0x376740
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_8                   0x376744
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_9                   0x376748
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_10                  0x37674C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_11                  0x376750
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_12                  0x376754
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_13                  0x376758
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_14                  0x37675C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AW_15                  0x376760
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_0                    0x376764
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_1                    0x376768
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_2                    0x37676C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_3                    0x376770
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_4                    0x376774
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_5                    0x376778
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_6                    0x37677C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_7                    0x376780
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_8                    0x376784
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_9                    0x376788
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_10                   0x37678C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_11                   0x376790
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_12                   0x376794
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_13                   0x376798
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_14                   0x37679C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AW_15                   0x3767A0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_0                   0x3767A4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_1                   0x3767A8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_2                   0x3767AC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_3                   0x3767B0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_4                   0x3767B4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_5                   0x3767B8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_6                   0x3767BC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_7                   0x3767C0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_8                   0x3767C4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_9                   0x3767C8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_10                  0x3767CC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_11                  0x3767D0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_12                  0x3767D4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_13                  0x3767D8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_14                  0x3767DC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AW_15                  0x3767E0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_0                     0x376824
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_1                     0x376828
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_2                     0x37682C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_3                     0x376830
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_4                     0x376834
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_5                     0x376838
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_6                     0x37683C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_7                     0x376840
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_8                     0x376844
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_9                     0x376848
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_10                    0x37684C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_11                    0x376850
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_12                    0x376854
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_13                    0x376858
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_14                    0x37685C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_LOW_AR_15                    0x376860
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_0                    0x376864
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_1                    0x376868
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_2                    0x37686C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_3                    0x376870
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_4                    0x376874
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_5                    0x376878
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_6                    0x37687C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_7                    0x376880
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_8                    0x376884
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_9                    0x376888
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_10                   0x37688C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_11                   0x376890
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_12                   0x376894
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_13                   0x376898
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_14                   0x37689C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_BASE_HIGH_AR_15                   0x3768A0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_0                     0x3768A4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_1                     0x3768A8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_2                     0x3768AC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_3                     0x3768B0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_4                     0x3768B4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_5                     0x3768B8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_6                     0x3768BC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_7                     0x3768C0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_8                     0x3768C4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_9                     0x3768C8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_10                    0x3768CC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_11                    0x3768D0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_12                    0x3768D4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_13                    0x3768D8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_14                    0x3768DC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_LOW_AR_15                    0x3768E0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_0                    0x3768E4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_1                    0x3768E8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_2                    0x3768EC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_3                    0x3768F0
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_4                    0x3768F4
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_5                    0x3768F8
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_6                    0x3768FC
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_7                    0x376900
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_8                    0x376904
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_9                    0x376908
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_10                   0x37690C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_11                   0x376910
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_12                   0x376914
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_13                   0x376918
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_14                   0x37691C
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_MASK_HIGH_AR_15                   0x376920
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_0                    0x376924
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_1                    0x376928
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_2                    0x37692C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_3                    0x376930
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_4                    0x376934
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_5                    0x376938
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_6                    0x37693C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_7                    0x376940
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_8                    0x376944
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_9                    0x376948
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_10                   0x37694C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_11                   0x376950
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_12                   0x376954
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_13                   0x376958
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_14                   0x37695C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_LOW_AR_15                   0x376960
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_0                   0x376964
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_1                   0x376968
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_2                   0x37696C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_3                   0x376970
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_4                   0x376974
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_5                   0x376978
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_6                   0x37697C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_7                   0x376980
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_8                   0x376984
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_9                   0x376988
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_10                  0x37698C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_11                  0x376990
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_12                  0x376994
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_13                  0x376998
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_14                  0x37699C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_BASE_HIGH_AR_15                  0x3769A0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_0                    0x3769A4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_1                    0x3769A8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_2                    0x3769AC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_3                    0x3769B0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_4                    0x3769B4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_5                    0x3769B8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_6                    0x3769BC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_7                    0x3769C0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_8                    0x3769C4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_9                    0x3769C8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_10                   0x3769CC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_11                   0x3769D0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_12                   0x3769D4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_13                   0x3769D8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_14                   0x3769DC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_LOW_AR_15                   0x3769E0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_0                   0x3769E4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_1                   0x3769E8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_2                   0x3769EC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_3                   0x3769F0
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_4                   0x3769F4
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_5                   0x3769F8
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_6                   0x3769FC
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_7                   0x376A00
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_8                   0x376A04
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_9                   0x376A08
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_10                  0x376A0C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_11                  0x376A10
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_12                  0x376A14
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_13                  0x376A18
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_14                  0x376A1C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_MASK_HIGH_AR_15                  0x376A20
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_HIT_AW                            0x376A64
+
+#define mmSIF_RTR_CTRL_7_RANGE_SEC_HIT_AR                            0x376A68
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_HIT_AW                           0x376A6C
+
+#define mmSIF_RTR_CTRL_7_RANGE_PRIV_HIT_AR                           0x376A70
+
+#define mmSIF_RTR_CTRL_7_RGL_CFG                                     0x376B64
+
+#define mmSIF_RTR_CTRL_7_RGL_SHIFT                                   0x376B68
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_0                          0x376B6C
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_1                          0x376B70
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_2                          0x376B74
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_3                          0x376B78
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_4                          0x376B7C
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_5                          0x376B80
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_6                          0x376B84
+
+#define mmSIF_RTR_CTRL_7_RGL_EXPECTED_LAT_7                          0x376B88
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_0                                 0x376BAC
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_1                                 0x376BB0
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_2                                 0x376BB4
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_3                                 0x376BB8
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_4                                 0x376BBC
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_5                                 0x376BC0
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_6                                 0x376BC4
+
+#define mmSIF_RTR_CTRL_7_RGL_TOKEN_7                                 0x376BC8
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_0                               0x376BEC
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_1                               0x376BF0
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_2                               0x376BF4
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_3                               0x376BF8
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_4                               0x376BFC
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_5                               0x376C00
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_6                               0x376C04
+
+#define mmSIF_RTR_CTRL_7_RGL_BANK_ID_7                               0x376C08
+
+#define mmSIF_RTR_CTRL_7_RGL_WDT                                     0x376C2C
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM0_CH0_CTR_WRAP                    0x376C30
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM0_CH1_CTR_WRAP                    0x376C34
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM1_CH0_CTR_WRAP                    0x376C38
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM1_CH1_CTR_WRAP                    0x376C3C
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM2_CH0_CTR_WRAP                    0x376C40
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM2_CH1_CTR_WRAP                    0x376C44
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM3_CH0_CTR_WRAP                    0x376C48
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM3_CH1_CTR_WRAP                    0x376C4C
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM0_CH0_CTR_CNT                     0x376C50
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM0_CH1_CTR_CNT                     0x376C54
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM1_CH0_CTR_CNT                     0x376C58
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM1_CH1_CTR_CNT                     0x376C5C
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM2_CH0_CTR_CNT                     0x376C60
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM2_CH1_CTR_CNT                     0x376C64
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM3_CH0_CTR_CNT                     0x376C68
+
+#define mmSIF_RTR_CTRL_7_E2E_AR_HBM3_CH1_CTR_CNT                     0x376C6C
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM0_CH0_CTR_WRAP                    0x376C70
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM0_CH1_CTR_WRAP                    0x376C74
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM1_CH0_CTR_WRAP                    0x376C78
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM1_CH1_CTR_WRAP                    0x376C7C
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM2_CH0_CTR_WRAP                    0x376C80
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM2_CH1_CTR_WRAP                    0x376C84
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM3_CH0_CTR_WRAP                    0x376C88
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM3_CH1_CTR_WRAP                    0x376C8C
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM0_CH0_CTR_CNT                     0x376C90
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM0_CH1_CTR_CNT                     0x376C94
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM1_CH0_CTR_CNT                     0x376C98
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM1_CH1_CTR_CNT                     0x376C9C
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM2_CH0_CTR_CNT                     0x376CA0
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM2_CH1_CTR_CNT                     0x376CA4
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM3_CH0_CTR_CNT                     0x376CA8
+
+#define mmSIF_RTR_CTRL_7_E2E_AW_HBM3_CH1_CTR_CNT                     0x376CAC
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_PC_SEL_0                             0x376CB0
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_PC_SEL_1                             0x376CB4
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_PC_SEL_2                             0x376CB8
+
+#define mmSIF_RTR_CTRL_7_NL_HBM_PC_SEL_3                             0x376CBC
+
+#endif /* ASIC_REG_SIF_RTR_CTRL_7_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/stlb_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/stlb_regs.h
new file mode 100644
index 000000000000..07d2a9000102
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/stlb_regs.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_STLB_REGS_H_
+#define ASIC_REG_STLB_REGS_H_
+
+/*
+ *****************************************
+ *   STLB (Prototype: STLB)
+ *****************************************
+ */
+
+#define mmSTLB_CACHE_INV                                             0xC12010
+
+#define mmSTLB_CACHE_INV_BASE_39_8                                   0xC12014
+
+#define mmSTLB_CACHE_INV_BASE_49_40                                  0xC12018
+
+#define mmSTLB_STLB_FEATURE_EN                                       0xC1201C
+
+#define mmSTLB_STLB_AXI_CACHE                                        0xC12020
+
+#define mmSTLB_HOP_CONFIGURATION                                     0xC12024
+
+#define mmSTLB_LINK_LIST_LOOKUP_MASK_49_32                           0xC12028
+
+#define mmSTLB_LINK_LIST_LOOKUP_MASK_31_0                            0xC1202C
+
+#define mmSTLB_LINK_LIST                                             0xC12030
+
+#define mmSTLB_INV_ALL_START                                         0xC12034
+
+#define mmSTLB_INV_ALL_SET                                           0xC12038
+
+#define mmSTLB_INV_PS                                                0xC1203C
+
+#define mmSTLB_INV_CONSUMER_INDEX                                    0xC12040
+
+#define mmSTLB_INV_HIT_COUNT                                         0xC12044
+
+#define mmSTLB_INV_SET                                               0xC12048
+
+#define mmSTLB_SRAM_INIT                                             0xC1204C
+
+#define mmSTLB_MEM_CACHE_INVALIDATION                                0xC12050
+
+#define mmSTLB_MEM_CACHE_INV_STATUS                                  0xC12054
+
+#define mmSTLB_MEM_CACHE_BASE_38_7                                   0xC12058
+
+#define mmSTLB_MEM_CACHE_BASE_49_39                                  0xC1205C
+
+#define mmSTLB_MEM_CACHE_CONFIG                                      0xC12060
+
+#define mmSTLB_SET_THRESHOLD_HOP4                                    0xC12064
+
+#define mmSTLB_SET_THRESHOLD_HOP3                                    0xC12068
+
+#define mmSTLB_SET_THRESHOLD_HOP2                                    0xC1206C
+
+#define mmSTLB_SET_THRESHOLD_HOP1                                    0xC12070
+
+#define mmSTLB_SET_THRESHOLD_HOP0                                    0xC12074
+
+#define mmSTLB_MULTI_HIT_INTERRUPT_CLR                               0xC12078
+
+#define mmSTLB_MULTI_HIT_INTERRUPT_MASK                              0xC1207C
+
+#define mmSTLB_MEM_L0_CACHE_CFG                                      0xC12080
+
+#define mmSTLB_MEM_READ_ARPROT                                       0xC12084
+
+#endif /* ASIC_REG_STLB_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_masks.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_masks.h
new file mode 100644
index 000000000000..8f67c11c8de9
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_masks.h
@@ -0,0 +1,2578 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC0_CFG_MASKS_H_
+#define ASIC_REG_TPC0_CFG_MASKS_H_
+
+/*
+ *****************************************
+ *   TPC0_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+/* TPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_0_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_0_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_1_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_1_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_2_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_2_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_3_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_3_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_4_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_4_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_5_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_5_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_6_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_6_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_7_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_7_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_8_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_8_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_9_PADDING_VALUE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_9_PADDING_VALUE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_DATA_TYPE_SHIFT       0
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_DATA_TYPE_MASK        0x7
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT  8
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_VALID_DIM_MASK_MASK   0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_LAST_DIM_SHIFT        16
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_LAST_DIM_MASK         0x70000
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_RMW_SET_SHIFT         19
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_RMW_SET_MASK          0x80000
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_RMW_RESERV_SHIFT      20
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_RMW_RESERV_MASK       0x100000
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_RMW_OP_SHIFT          21
+#define TPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG_RMW_OP_MASK           0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_0_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_0_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_1_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_1_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_2_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_2_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_3_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_3_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_4_SIZE_V_SHIFT                  0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_4_SIZE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE_V_SHIFT                0
+#define TPC0_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE_V_MASK                 0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH_V_SHIFT             0
+#define TPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH_V_MASK              0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_10_PADDING_VALUE_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_10_PADDING_VALUE_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_DATA_TYPE_SHIFT      0
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_DATA_TYPE_MASK       0x7
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT 8
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_VALID_DIM_MASK_MASK  0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_LAST_DIM_SHIFT       16
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_LAST_DIM_MASK        0x70000
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_RMW_SET_SHIFT        19
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_RMW_SET_MASK         0x80000
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_RMW_RESERV_SHIFT     20
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_RMW_RESERV_MASK      0x100000
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_RMW_OP_SHIFT         21
+#define TPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG_RMW_OP_MASK          0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_0_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_0_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_1_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_1_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_2_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_2_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_3_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_3_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_4_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_4_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH_V_SHIFT             0
+#define TPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH_V_MASK              0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_11_PADDING_VALUE_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_11_PADDING_VALUE_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_DATA_TYPE_SHIFT      0
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_DATA_TYPE_MASK       0x7
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT 8
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_VALID_DIM_MASK_MASK  0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_LAST_DIM_SHIFT       16
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_LAST_DIM_MASK        0x70000
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_RMW_SET_SHIFT        19
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_RMW_SET_MASK         0x80000
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_RMW_RESERV_SHIFT     20
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_RMW_RESERV_MASK      0x100000
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_RMW_OP_SHIFT         21
+#define TPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG_RMW_OP_MASK          0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_0_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_0_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_1_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_1_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_2_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_2_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_3_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_3_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_4_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_4_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH_V_SHIFT             0
+#define TPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH_V_MASK              0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_12_PADDING_VALUE_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_12_PADDING_VALUE_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_DATA_TYPE_SHIFT      0
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_DATA_TYPE_MASK       0x7
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT 8
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_VALID_DIM_MASK_MASK  0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_LAST_DIM_SHIFT       16
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_LAST_DIM_MASK        0x70000
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_RMW_SET_SHIFT        19
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_RMW_SET_MASK         0x80000
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_RMW_RESERV_SHIFT     20
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_RMW_RESERV_MASK      0x100000
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_RMW_OP_SHIFT         21
+#define TPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG_RMW_OP_MASK          0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_0_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_0_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_1_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_1_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_2_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_2_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_3_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_3_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_4_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_4_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH_V_SHIFT             0
+#define TPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH_V_MASK              0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_13_PADDING_VALUE_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_13_PADDING_VALUE_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_DATA_TYPE_SHIFT      0
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_DATA_TYPE_MASK       0x7
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT 8
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_VALID_DIM_MASK_MASK  0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_LAST_DIM_SHIFT       16
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_LAST_DIM_MASK        0x70000
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_RMW_SET_SHIFT        19
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_RMW_SET_MASK         0x80000
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_RMW_RESERV_SHIFT     20
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_RMW_RESERV_MASK      0x100000
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_RMW_OP_SHIFT         21
+#define TPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG_RMW_OP_MASK          0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_0_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_0_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_1_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_1_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_2_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_2_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_3_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_3_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_4_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_4_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH_V_SHIFT             0
+#define TPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH_V_MASK              0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_14_PADDING_VALUE_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_14_PADDING_VALUE_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_DATA_TYPE_SHIFT      0
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_DATA_TYPE_MASK       0x7
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT 8
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_VALID_DIM_MASK_MASK  0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_LAST_DIM_SHIFT       16
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_LAST_DIM_MASK        0x70000
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_RMW_SET_SHIFT        19
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_RMW_SET_MASK         0x80000
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_RMW_RESERV_SHIFT     20
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_RMW_RESERV_MASK      0x100000
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_RMW_OP_SHIFT         21
+#define TPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG_RMW_OP_MASK          0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_0_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_0_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_1_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_1_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_2_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_2_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_3_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_3_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_4_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_4_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW */
+#define TPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH */
+#define TPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH_V_SHIFT             0
+#define TPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH_V_MASK              0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_PADDING_VALUE */
+#define TPC0_CFG_KERNEL_TENSOR_15_PADDING_VALUE_V_SHIFT              0
+#define TPC0_CFG_KERNEL_TENSOR_15_PADDING_VALUE_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG */
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_DATA_TYPE_SHIFT      0
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_DATA_TYPE_MASK       0x7
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT 8
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_VALID_DIM_MASK_MASK  0x1F00
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_LAST_DIM_SHIFT       16
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_LAST_DIM_MASK        0x70000
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_RMW_SET_SHIFT        19
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_RMW_SET_MASK         0x80000
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_RMW_RESERV_SHIFT     20
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_RMW_RESERV_MASK      0x100000
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_RMW_OP_SHIFT         21
+#define TPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG_RMW_OP_MASK          0x600000
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_0_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_0_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_0_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_1_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_1_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_1_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_2_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_2_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_2_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_3_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_3_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_3_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_4_SIZE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_4_SIZE_V_SHIFT                 0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_4_SIZE_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE */
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE_V_SHIFT               0
+#define TPC0_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE_V_MASK                0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE */
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE_SO_WRITE_VALUE_SHIFT     0
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE_SO_WRITE_VALUE_MASK      0xFFFF
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE_RSV_SHIFT                16
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE_RSV_MASK                 0x1FFF0000
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE_SO_OPERATION_SHIFT       29
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE_SO_OPERATION_MASK        0xE0000000
+
+/* TPC0_CFG_KERNEL_SYNC_OBJECT_ADDR */
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_ADDR_V_SHIFT                     0
+#define TPC0_CFG_KERNEL_SYNC_OBJECT_ADDR_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW */
+#define TPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW_V_SHIFT              0
+#define TPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW_V_MASK               0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH */
+#define TPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH_V_SHIFT             0
+#define TPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH_V_MASK              0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_BASE_DIM_0 */
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_0_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_0_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_SIZE_DIM_0 */
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_0_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_0_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_BASE_DIM_1 */
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_1_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_1_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_SIZE_DIM_1 */
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_1_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_1_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_BASE_DIM_2 */
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_2_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_2_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_SIZE_DIM_2 */
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_2_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_2_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_BASE_DIM_3 */
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_3_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_3_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_SIZE_DIM_3 */
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_3_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_3_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_BASE_DIM_4 */
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_4_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_BASE_DIM_4_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_TID_SIZE_DIM_4 */
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_4_V_SHIFT                       0
+#define TPC0_CFG_KERNEL_TID_SIZE_DIM_4_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_KERNEL_KERNEL_CONFIG */
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_SMALL_VLM_SHIFT                0
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_SMALL_VLM_MASK                 0x1
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_ASO_EVICT_L0_SHIFT             1
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_ASO_EVICT_L0_MASK              0x2
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_NUM_VALID_SRFS_SHIFT           2
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_NUM_VALID_SRFS_MASK            0xFC
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_RD_RATE_LIMIT_RST_TOKEN_SHIFT  8
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_RD_RATE_LIMIT_RST_TOKEN_MASK   0xFF00
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_WR_RATE_LIMIT_RST_TOKEN_SHIFT  16
+#define TPC0_CFG_KERNEL_KERNEL_CONFIG_WR_RATE_LIMIT_RST_TOKEN_MASK   0xFF0000
+
+/* TPC0_CFG_KERNEL_KERNEL_ID */
+#define TPC0_CFG_KERNEL_KERNEL_ID_V_SHIFT                            0
+#define TPC0_CFG_KERNEL_KERNEL_ID_V_MASK                             0xFFFF
+
+/* TPC0_CFG_KERNEL_SRF */
+#define TPC0_CFG_KERNEL_SRF_V_SHIFT                                  0
+#define TPC0_CFG_KERNEL_SRF_V_MASK                                   0xFFFFFFFF
+
+/* TPC0_CFG_ROUND_CSR */
+#define TPC0_CFG_ROUND_CSR_MODE_SHIFT                                0
+#define TPC0_CFG_ROUND_CSR_MODE_MASK                                 0x7
+
+/* TPC0_CFG_PROT */
+#define TPC0_CFG_PROT_AWPROT_SHIFT                                   0
+#define TPC0_CFG_PROT_AWPROT_MASK                                    0x7
+#define TPC0_CFG_PROT_ARPROT_SHIFT                                   3
+#define TPC0_CFG_PROT_ARPROT_MASK                                    0x38
+
+/* TPC0_CFG_SEMAPHORE */
+#define TPC0_CFG_SEMAPHORE_V_SHIFT                                   0
+#define TPC0_CFG_SEMAPHORE_V_MASK                                    0xFFFFFFFF
+
+/* TPC0_CFG_VFLAGS */
+#define TPC0_CFG_VFLAGS_V_SHIFT                                      0
+#define TPC0_CFG_VFLAGS_V_MASK                                       0xF
+
+/* TPC0_CFG_SFLAGS */
+#define TPC0_CFG_SFLAGS_V_SHIFT                                      0
+#define TPC0_CFG_SFLAGS_V_MASK                                       0xF
+
+/* TPC0_CFG_LFSR_POLYNOM */
+#define TPC0_CFG_LFSR_POLYNOM_V_SHIFT                                0
+#define TPC0_CFG_LFSR_POLYNOM_V_MASK                                 0xFFFFFFFF
+
+/* TPC0_CFG_STATUS */
+#define TPC0_CFG_STATUS_SCALAR_PIPE_EMPTY_SHIFT                      1
+#define TPC0_CFG_STATUS_SCALAR_PIPE_EMPTY_MASK                       0x2
+#define TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_SHIFT                      2
+#define TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK                       0x4
+#define TPC0_CFG_STATUS_IQ_EMPTY_SHIFT                               3
+#define TPC0_CFG_STATUS_IQ_EMPTY_MASK                                0x8
+#define TPC0_CFG_STATUS_SB_EMPTY_SHIFT                               5
+#define TPC0_CFG_STATUS_SB_EMPTY_MASK                                0x20
+#define TPC0_CFG_STATUS_QM_IDLE_SHIFT                                6
+#define TPC0_CFG_STATUS_QM_IDLE_MASK                                 0x40
+#define TPC0_CFG_STATUS_QM_RDY_SHIFT                                 7
+#define TPC0_CFG_STATUS_QM_RDY_MASK                                  0x80
+
+/* TPC0_CFG_CFG_BASE_ADDRESS_HIGH */
+#define TPC0_CFG_CFG_BASE_ADDRESS_HIGH_V_SHIFT                       0
+#define TPC0_CFG_CFG_BASE_ADDRESS_HIGH_V_MASK                        0xFFFFFFFF
+
+/* TPC0_CFG_CFG_SUBTRACT_VALUE */
+#define TPC0_CFG_CFG_SUBTRACT_VALUE_V_SHIFT                          0
+#define TPC0_CFG_CFG_SUBTRACT_VALUE_V_MASK                           0xFFFFFFFF
+
+/* TPC0_CFG_SM_BASE_ADDRESS_HIGH */
+#define TPC0_CFG_SM_BASE_ADDRESS_HIGH_V_SHIFT                        0
+#define TPC0_CFG_SM_BASE_ADDRESS_HIGH_V_MASK                         0xFFFFFFFF
+
+/* TPC0_CFG_TPC_CMD */
+#define TPC0_CFG_TPC_CMD_ICACHE_INVALIDATE_SHIFT                     0
+#define TPC0_CFG_TPC_CMD_ICACHE_INVALIDATE_MASK                      0x1
+#define TPC0_CFG_TPC_CMD_DCACHE_INVALIDATE_SHIFT                     1
+#define TPC0_CFG_TPC_CMD_DCACHE_INVALIDATE_MASK                      0x2
+#define TPC0_CFG_TPC_CMD_LCACHE_INVALIDATE_SHIFT                     2
+#define TPC0_CFG_TPC_CMD_LCACHE_INVALIDATE_MASK                      0x4
+#define TPC0_CFG_TPC_CMD_TCACHE_INVALIDATE_SHIFT                     3
+#define TPC0_CFG_TPC_CMD_TCACHE_INVALIDATE_MASK                      0x8
+#define TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_64KB_SHIFT                  4
+#define TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_64KB_MASK                   0x10
+#define TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_32KB_SHIFT                  5
+#define TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_32KB_MASK                   0x20
+#define TPC0_CFG_TPC_CMD_QMAN_STOP_SHIFT                             6
+#define TPC0_CFG_TPC_CMD_QMAN_STOP_MASK                              0x40
+
+/* TPC0_CFG_TPC_EXECUTE */
+#define TPC0_CFG_TPC_EXECUTE_V_SHIFT                                 0
+#define TPC0_CFG_TPC_EXECUTE_V_MASK                                  0x1
+
+/* TPC0_CFG_TPC_STALL */
+#define TPC0_CFG_TPC_STALL_V_SHIFT                                   0
+#define TPC0_CFG_TPC_STALL_V_MASK                                    0x1
+
+/* TPC0_CFG_ICACHE_BASE_ADDERESS_LOW */
+#define TPC0_CFG_ICACHE_BASE_ADDERESS_LOW_V_SHIFT                    0
+#define TPC0_CFG_ICACHE_BASE_ADDERESS_LOW_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_ICACHE_BASE_ADDERESS_HIGH */
+#define TPC0_CFG_ICACHE_BASE_ADDERESS_HIGH_V_SHIFT                   0
+#define TPC0_CFG_ICACHE_BASE_ADDERESS_HIGH_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_RD_RATE_LIMIT */
+#define TPC0_CFG_RD_RATE_LIMIT_ENABLE_SHIFT                          0
+#define TPC0_CFG_RD_RATE_LIMIT_ENABLE_MASK                           0x1
+#define TPC0_CFG_RD_RATE_LIMIT_SATURATION_SHIFT                      1
+#define TPC0_CFG_RD_RATE_LIMIT_SATURATION_MASK                       0x1FE
+#define TPC0_CFG_RD_RATE_LIMIT_TIMEOUT_SHIFT                         9
+#define TPC0_CFG_RD_RATE_LIMIT_TIMEOUT_MASK                          0x1FE00
+
+/* TPC0_CFG_WR_RATE_LIMIT */
+#define TPC0_CFG_WR_RATE_LIMIT_ENABLE_SHIFT                          0
+#define TPC0_CFG_WR_RATE_LIMIT_ENABLE_MASK                           0x1
+#define TPC0_CFG_WR_RATE_LIMIT_SATURATION_SHIFT                      1
+#define TPC0_CFG_WR_RATE_LIMIT_SATURATION_MASK                       0x1FE
+#define TPC0_CFG_WR_RATE_LIMIT_TIMEOUT_SHIFT                         9
+#define TPC0_CFG_WR_RATE_LIMIT_TIMEOUT_MASK                          0x1FE00
+
+/* TPC0_CFG_MSS_CONFIG */
+#define TPC0_CFG_MSS_CONFIG_AWCACHE_SHIFT                            0
+#define TPC0_CFG_MSS_CONFIG_AWCACHE_MASK                             0xF
+#define TPC0_CFG_MSS_CONFIG_ARCACHE_SHIFT                            4
+#define TPC0_CFG_MSS_CONFIG_ARCACHE_MASK                             0xF0
+#define TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_SHIFT              8
+#define TPC0_CFG_MSS_CONFIG_ICACHE_FETCH_LINE_NUM_MASK               0x300
+#define TPC0_CFG_MSS_CONFIG_EXPOSED_PIPE_DIS_SHIFT                   10
+#define TPC0_CFG_MSS_CONFIG_EXPOSED_PIPE_DIS_MASK                    0x400
+#define TPC0_CFG_MSS_CONFIG_DCACHE_PREFETCH_DIS_SHIFT                11
+#define TPC0_CFG_MSS_CONFIG_DCACHE_PREFETCH_DIS_MASK                 0x800
+
+/* TPC0_CFG_TPC_INTR_CAUSE */
+#define TPC0_CFG_TPC_INTR_CAUSE_CAUSE_SHIFT                          0
+#define TPC0_CFG_TPC_INTR_CAUSE_CAUSE_MASK                           0xFFFFF
+
+/* TPC0_CFG_TPC_INTR_MASK */
+#define TPC0_CFG_TPC_INTR_MASK_MASK_SHIFT                            0
+#define TPC0_CFG_TPC_INTR_MASK_MASK_MASK                             0xFFFFF
+
+/* TPC0_CFG_WQ_CREDITS */
+#define TPC0_CFG_WQ_CREDITS_ST_G_SHIFT                               0
+#define TPC0_CFG_WQ_CREDITS_ST_G_MASK                                0xF
+#define TPC0_CFG_WQ_CREDITS_KERNEL_FIFO_SHIFT                        4
+#define TPC0_CFG_WQ_CREDITS_KERNEL_FIFO_MASK                         0x70
+
+/* TPC0_CFG_ARUSER_LO */
+#define TPC0_CFG_ARUSER_LO_V_SHIFT                                   0
+#define TPC0_CFG_ARUSER_LO_V_MASK                                    0x7FF
+
+/* TPC0_CFG_ARUSER_HI */
+#define TPC0_CFG_ARUSER_HI_V_SHIFT                                   11
+#define TPC0_CFG_ARUSER_HI_V_MASK                                    0x1800
+#define TPC0_CFG_ARUSER_HI_RSRV_SHIFT                                13
+#define TPC0_CFG_ARUSER_HI_RSRV_MASK                                 0xFFFFE000
+
+/* TPC0_CFG_AWUSER_LO */
+#define TPC0_CFG_AWUSER_LO_V_SHIFT                                   0
+#define TPC0_CFG_AWUSER_LO_V_MASK                                    0x7FF
+
+/* TPC0_CFG_AWUSER_HI */
+#define TPC0_CFG_AWUSER_HI_V_SHIFT                                   11
+#define TPC0_CFG_AWUSER_HI_V_MASK                                    0x1800
+#define TPC0_CFG_AWUSER_HI_RSRV_SHIFT                                13
+#define TPC0_CFG_AWUSER_HI_RSRV_MASK                                 0xFFFFE000
+
+/* TPC0_CFG_OPCODE_EXEC */
+#define TPC0_CFG_OPCODE_EXEC_SPU_OP_SHIFT                            0
+#define TPC0_CFG_OPCODE_EXEC_SPU_OP_MASK                             0x7F
+#define TPC0_CFG_OPCODE_EXEC_SPU_EN_SHIFT                            7
+#define TPC0_CFG_OPCODE_EXEC_SPU_EN_MASK                             0x80
+#define TPC0_CFG_OPCODE_EXEC_VPU_OP_SHIFT                            8
+#define TPC0_CFG_OPCODE_EXEC_VPU_OP_MASK                             0x7F00
+#define TPC0_CFG_OPCODE_EXEC_VPU_EN_SHIFT                            15
+#define TPC0_CFG_OPCODE_EXEC_VPU_EN_MASK                             0x8000
+#define TPC0_CFG_OPCODE_EXEC_LD_OP_SHIFT                             16
+#define TPC0_CFG_OPCODE_EXEC_LD_OP_MASK                              0x7F0000
+#define TPC0_CFG_OPCODE_EXEC_LD_EN_SHIFT                             23
+#define TPC0_CFG_OPCODE_EXEC_LD_EN_MASK                              0x800000
+#define TPC0_CFG_OPCODE_EXEC_ST_OP_SHIFT                             24
+#define TPC0_CFG_OPCODE_EXEC_ST_OP_MASK                              0x7F000000
+#define TPC0_CFG_OPCODE_EXEC_ST_EN_SHIFT                             31
+#define TPC0_CFG_OPCODE_EXEC_ST_EN_MASK                              0x80000000
+
+/* TPC0_CFG_LUT_FUNC32_BASE_ADDR_LO */
+#define TPC0_CFG_LUT_FUNC32_BASE_ADDR_LO_V_SHIFT                     0
+#define TPC0_CFG_LUT_FUNC32_BASE_ADDR_LO_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_LUT_FUNC32_BASE_ADDR_HI */
+#define TPC0_CFG_LUT_FUNC32_BASE_ADDR_HI_V_SHIFT                     0
+#define TPC0_CFG_LUT_FUNC32_BASE_ADDR_HI_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_LUT_FUNC64_BASE_ADDR_LO */
+#define TPC0_CFG_LUT_FUNC64_BASE_ADDR_LO_V_SHIFT                     0
+#define TPC0_CFG_LUT_FUNC64_BASE_ADDR_LO_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_LUT_FUNC64_BASE_ADDR_HI */
+#define TPC0_CFG_LUT_FUNC64_BASE_ADDR_HI_V_SHIFT                     0
+#define TPC0_CFG_LUT_FUNC64_BASE_ADDR_HI_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_LUT_FUNC128_BASE_ADDR_LO */
+#define TPC0_CFG_LUT_FUNC128_BASE_ADDR_LO_V_SHIFT                    0
+#define TPC0_CFG_LUT_FUNC128_BASE_ADDR_LO_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_LUT_FUNC128_BASE_ADDR_HI */
+#define TPC0_CFG_LUT_FUNC128_BASE_ADDR_HI_V_SHIFT                    0
+#define TPC0_CFG_LUT_FUNC128_BASE_ADDR_HI_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_LUT_FUNC256_BASE_ADDR_LO */
+#define TPC0_CFG_LUT_FUNC256_BASE_ADDR_LO_V_SHIFT                    0
+#define TPC0_CFG_LUT_FUNC256_BASE_ADDR_LO_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_LUT_FUNC256_BASE_ADDR_HI */
+#define TPC0_CFG_LUT_FUNC256_BASE_ADDR_HI_V_SHIFT                    0
+#define TPC0_CFG_LUT_FUNC256_BASE_ADDR_HI_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_TSB_CFG_MAX_SIZE */
+#define TPC0_CFG_TSB_CFG_MAX_SIZE_DATA_SHIFT                         0
+#define TPC0_CFG_TSB_CFG_MAX_SIZE_DATA_MASK                          0xFFFF
+#define TPC0_CFG_TSB_CFG_MAX_SIZE_MD_SHIFT                           16
+#define TPC0_CFG_TSB_CFG_MAX_SIZE_MD_MASK                            0xFFFF0000
+
+/* TPC0_CFG_TSB_CFG */
+#define TPC0_CFG_TSB_CFG_FORCE_MISS_SHIFT                            0
+#define TPC0_CFG_TSB_CFG_FORCE_MISS_MASK                             0x1
+#define TPC0_CFG_TSB_CFG_MAX_OS_SHIFT                                1
+#define TPC0_CFG_TSB_CFG_MAX_OS_MASK                                 0x1FFFE
+
+/* TPC0_CFG_DBGMEM_ADD */
+#define TPC0_CFG_DBGMEM_ADD_V_SHIFT                                  0
+#define TPC0_CFG_DBGMEM_ADD_V_MASK                                   0xFFFFFFFF
+
+/* TPC0_CFG_DBGMEM_DATA_WR */
+#define TPC0_CFG_DBGMEM_DATA_WR_V_SHIFT                              0
+#define TPC0_CFG_DBGMEM_DATA_WR_V_MASK                               0xFFFFFFFF
+
+/* TPC0_CFG_DBGMEM_DATA_RD */
+#define TPC0_CFG_DBGMEM_DATA_RD_V_SHIFT                              0
+#define TPC0_CFG_DBGMEM_DATA_RD_V_MASK                               0xFFFFFFFF
+
+/* TPC0_CFG_DBGMEM_CTRL */
+#define TPC0_CFG_DBGMEM_CTRL_WR_NRD_SHIFT                            0
+#define TPC0_CFG_DBGMEM_CTRL_WR_NRD_MASK                             0x1
+
+/* TPC0_CFG_DBGMEM_RC */
+#define TPC0_CFG_DBGMEM_RC_VALID_SHIFT                               0
+#define TPC0_CFG_DBGMEM_RC_VALID_MASK                                0x1
+
+/* TPC0_CFG_TSB_INFLIGHT_CNTR */
+#define TPC0_CFG_TSB_INFLIGHT_CNTR_V_SHIFT                           0
+#define TPC0_CFG_TSB_INFLIGHT_CNTR_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_WQ_INFLIGHT_CNTR */
+#define TPC0_CFG_WQ_INFLIGHT_CNTR_HBW_SHIFT                          0
+#define TPC0_CFG_WQ_INFLIGHT_CNTR_HBW_MASK                           0xFFFF
+#define TPC0_CFG_WQ_INFLIGHT_CNTR_LBW_SHIFT                          16
+#define TPC0_CFG_WQ_INFLIGHT_CNTR_LBW_MASK                           0xF0000
+
+/* TPC0_CFG_WQ_LBW_TOTAL_CNTR */
+#define TPC0_CFG_WQ_LBW_TOTAL_CNTR_V_SHIFT                           0
+#define TPC0_CFG_WQ_LBW_TOTAL_CNTR_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_WQ_HBW_TOTAL_CNTR */
+#define TPC0_CFG_WQ_HBW_TOTAL_CNTR_V_SHIFT                           0
+#define TPC0_CFG_WQ_HBW_TOTAL_CNTR_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_IRQ_OCCOUPY_CNTR */
+#define TPC0_CFG_IRQ_OCCOUPY_CNTR_V_SHIFT                            0
+#define TPC0_CFG_IRQ_OCCOUPY_CNTR_V_MASK                             0xFFFFFFFF
+
+/* TPC0_CFG_FUNC_MBIST_CNTRL */
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_START_SHIFT                  0
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_START_MASK                   0x1
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_DONE_SHIFT                   1
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_DONE_MASK                    0x2
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_ACTIVE_SHIFT                 2
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_ACTIVE_MASK                  0x4
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_FAILED_SHIFT                 16
+#define TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_FAILED_MASK                  0x3FF0000
+
+/* TPC0_CFG_FUNC_MBIST_PAT */
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN0_EVEN_SHIFT            0
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN0_EVEN_MASK             0x3
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN0_ODD_SHIFT             2
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN0_ODD_MASK              0xC
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN1_EVEN_SHIFT            4
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN1_EVEN_MASK             0x30
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN1_ODD_SHIFT             6
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN1_ODD_MASK              0xC0
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN2_EVEN_SHIFT            8
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN2_EVEN_MASK             0x300
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN2_ODD_SHIFT             10
+#define TPC0_CFG_FUNC_MBIST_PAT_MBIST_PATTERN2_ODD_MASK              0xC00
+
+/* TPC0_CFG_FUNC_MBIST_MEM */
+#define TPC0_CFG_FUNC_MBIST_MEM_MAX_ADDR_SHIFT                       0
+#define TPC0_CFG_FUNC_MBIST_MEM_MAX_ADDR_MASK                        0x7FF
+#define TPC0_CFG_FUNC_MBIST_MEM_PATTERN_EN_SHIFT                     12
+#define TPC0_CFG_FUNC_MBIST_MEM_PATTERN_EN_MASK                      0x7000
+#define TPC0_CFG_FUNC_MBIST_MEM_LAST_FAILED_ADDR_SHIFT               16
+#define TPC0_CFG_FUNC_MBIST_MEM_LAST_FAILED_ADDR_MASK                0x7FF0000
+#define TPC0_CFG_FUNC_MBIST_MEM_LAST_FAILED_PATTERN_SHIFT            28
+#define TPC0_CFG_FUNC_MBIST_MEM_LAST_FAILED_PATTERN_MASK             0x70000000
+
+/* TPC0_CFG_QM_TENSOR_0_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_0_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_0_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_0_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_0_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_0_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_0_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_0_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_0_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_0_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_0_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_0_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_0_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_0_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_0_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_0_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_0_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_0_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_0_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_1_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_1_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_1_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_1_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_1_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_1_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_1_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_1_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_1_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_1_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_1_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_1_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_1_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_1_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_1_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_1_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_1_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_1_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_2_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_2_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_2_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_2_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_2_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_2_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_2_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_2_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_2_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_2_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_2_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_2_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_2_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_2_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_2_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_2_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_2_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_2_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_3_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_3_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_3_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_3_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_3_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_3_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_3_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_3_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_3_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_3_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_3_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_3_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_3_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_3_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_3_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_3_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_3_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_3_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_4_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_4_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_4_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_4_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_4_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_4_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_4_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_4_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_4_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_4_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_4_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_4_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_4_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_4_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_4_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_4_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_4_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_4_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_5_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_5_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_5_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_5_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_5_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_5_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_5_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_5_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_5_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_5_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_5_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_5_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_5_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_5_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_5_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_5_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_5_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_5_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_6_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_6_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_6_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_6_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_6_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_6_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_6_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_6_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_6_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_6_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_6_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_6_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_6_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_6_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_6_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_6_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_6_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_6_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_7_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_7_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_7_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_7_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_7_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_7_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_7_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_7_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_7_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_7_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_7_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_7_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_7_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_7_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_7_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_7_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_7_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_7_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_8_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_8_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_8_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_8_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_8_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_8_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_8_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_8_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_8_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_8_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_8_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_8_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_8_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_8_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_8_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_8_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_8_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_8_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_9_BASE_ADDR_LOW_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_9_BASE_ADDR_LOW_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_9_BASE_ADDR_HIGH_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_9_BASE_ADDR_HIGH_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_9_PADDING_VALUE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_9_PADDING_VALUE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_DATA_TYPE_SHIFT           0
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_DATA_TYPE_MASK            0x7
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT      8
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_VALID_DIM_MASK_MASK       0x1F00
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_LAST_DIM_SHIFT            16
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_LAST_DIM_MASK             0x70000
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_RMW_SET_SHIFT             19
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_RMW_SET_MASK              0x80000
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_RMW_RESERV_SHIFT          20
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_RMW_RESERV_MASK           0x100000
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_RMW_OP_SHIFT              21
+#define TPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG_RMW_OP_MASK               0x600000
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_0_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_9_DIM_0_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_0_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_9_DIM_0_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_1_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_9_DIM_1_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_1_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_9_DIM_1_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_2_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_9_DIM_2_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_2_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_9_DIM_2_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_3_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_9_DIM_3_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_3_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_9_DIM_3_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_4_SIZE_V_SHIFT                      0
+#define TPC0_CFG_QM_TENSOR_9_DIM_4_SIZE_V_MASK                       0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_9_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_9_DIM_4_STRIDE_V_SHIFT                    0
+#define TPC0_CFG_QM_TENSOR_9_DIM_4_STRIDE_V_MASK                     0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_10_BASE_ADDR_LOW_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_10_BASE_ADDR_LOW_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_10_BASE_ADDR_HIGH_V_SHIFT                 0
+#define TPC0_CFG_QM_TENSOR_10_BASE_ADDR_HIGH_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_10_PADDING_VALUE_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_10_PADDING_VALUE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_DATA_TYPE_SHIFT          0
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_DATA_TYPE_MASK           0x7
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT     8
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_VALID_DIM_MASK_MASK      0x1F00
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_LAST_DIM_SHIFT           16
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_LAST_DIM_MASK            0x70000
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_RMW_SET_SHIFT            19
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_RMW_SET_MASK             0x80000
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_RMW_RESERV_SHIFT         20
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_RMW_RESERV_MASK          0x100000
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_RMW_OP_SHIFT             21
+#define TPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG_RMW_OP_MASK              0x600000
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_0_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_10_DIM_0_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_0_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_10_DIM_0_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_1_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_10_DIM_1_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_1_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_10_DIM_1_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_2_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_10_DIM_2_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_2_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_10_DIM_2_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_3_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_10_DIM_3_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_3_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_10_DIM_3_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_4_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_10_DIM_4_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_10_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_10_DIM_4_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_10_DIM_4_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_11_BASE_ADDR_LOW_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_11_BASE_ADDR_LOW_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_11_BASE_ADDR_HIGH_V_SHIFT                 0
+#define TPC0_CFG_QM_TENSOR_11_BASE_ADDR_HIGH_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_11_PADDING_VALUE_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_11_PADDING_VALUE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_DATA_TYPE_SHIFT          0
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_DATA_TYPE_MASK           0x7
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT     8
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_VALID_DIM_MASK_MASK      0x1F00
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_LAST_DIM_SHIFT           16
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_LAST_DIM_MASK            0x70000
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_RMW_SET_SHIFT            19
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_RMW_SET_MASK             0x80000
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_RMW_RESERV_SHIFT         20
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_RMW_RESERV_MASK          0x100000
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_RMW_OP_SHIFT             21
+#define TPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG_RMW_OP_MASK              0x600000
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_0_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_11_DIM_0_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_0_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_11_DIM_0_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_1_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_11_DIM_1_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_1_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_11_DIM_1_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_2_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_11_DIM_2_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_2_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_11_DIM_2_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_3_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_11_DIM_3_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_3_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_11_DIM_3_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_4_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_11_DIM_4_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_11_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_11_DIM_4_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_11_DIM_4_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_12_BASE_ADDR_LOW_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_12_BASE_ADDR_LOW_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_12_BASE_ADDR_HIGH_V_SHIFT                 0
+#define TPC0_CFG_QM_TENSOR_12_BASE_ADDR_HIGH_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_12_PADDING_VALUE_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_12_PADDING_VALUE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_DATA_TYPE_SHIFT          0
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_DATA_TYPE_MASK           0x7
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT     8
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_VALID_DIM_MASK_MASK      0x1F00
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_LAST_DIM_SHIFT           16
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_LAST_DIM_MASK            0x70000
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_RMW_SET_SHIFT            19
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_RMW_SET_MASK             0x80000
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_RMW_RESERV_SHIFT         20
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_RMW_RESERV_MASK          0x100000
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_RMW_OP_SHIFT             21
+#define TPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG_RMW_OP_MASK              0x600000
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_0_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_12_DIM_0_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_0_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_12_DIM_0_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_1_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_12_DIM_1_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_1_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_12_DIM_1_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_2_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_12_DIM_2_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_2_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_12_DIM_2_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_3_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_12_DIM_3_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_3_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_12_DIM_3_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_4_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_12_DIM_4_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_12_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_12_DIM_4_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_12_DIM_4_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_13_BASE_ADDR_LOW_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_13_BASE_ADDR_LOW_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_13_BASE_ADDR_HIGH_V_SHIFT                 0
+#define TPC0_CFG_QM_TENSOR_13_BASE_ADDR_HIGH_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_13_PADDING_VALUE_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_13_PADDING_VALUE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_DATA_TYPE_SHIFT          0
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_DATA_TYPE_MASK           0x7
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT     8
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_VALID_DIM_MASK_MASK      0x1F00
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_LAST_DIM_SHIFT           16
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_LAST_DIM_MASK            0x70000
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_RMW_SET_SHIFT            19
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_RMW_SET_MASK             0x80000
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_RMW_RESERV_SHIFT         20
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_RMW_RESERV_MASK          0x100000
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_RMW_OP_SHIFT             21
+#define TPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG_RMW_OP_MASK              0x600000
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_0_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_13_DIM_0_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_0_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_13_DIM_0_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_1_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_13_DIM_1_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_1_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_13_DIM_1_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_2_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_13_DIM_2_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_2_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_13_DIM_2_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_3_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_13_DIM_3_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_3_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_13_DIM_3_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_4_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_13_DIM_4_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_13_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_13_DIM_4_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_13_DIM_4_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_14_BASE_ADDR_LOW_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_14_BASE_ADDR_LOW_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_14_BASE_ADDR_HIGH_V_SHIFT                 0
+#define TPC0_CFG_QM_TENSOR_14_BASE_ADDR_HIGH_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_14_PADDING_VALUE_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_14_PADDING_VALUE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_DATA_TYPE_SHIFT          0
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_DATA_TYPE_MASK           0x7
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT     8
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_VALID_DIM_MASK_MASK      0x1F00
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_LAST_DIM_SHIFT           16
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_LAST_DIM_MASK            0x70000
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_RMW_SET_SHIFT            19
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_RMW_SET_MASK             0x80000
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_RMW_RESERV_SHIFT         20
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_RMW_RESERV_MASK          0x100000
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_RMW_OP_SHIFT             21
+#define TPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG_RMW_OP_MASK              0x600000
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_0_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_14_DIM_0_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_0_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_14_DIM_0_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_1_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_14_DIM_1_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_1_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_14_DIM_1_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_2_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_14_DIM_2_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_2_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_14_DIM_2_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_3_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_14_DIM_3_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_3_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_14_DIM_3_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_4_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_14_DIM_4_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_14_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_14_DIM_4_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_14_DIM_4_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_BASE_ADDR_LOW */
+#define TPC0_CFG_QM_TENSOR_15_BASE_ADDR_LOW_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_15_BASE_ADDR_LOW_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_BASE_ADDR_HIGH */
+#define TPC0_CFG_QM_TENSOR_15_BASE_ADDR_HIGH_V_SHIFT                 0
+#define TPC0_CFG_QM_TENSOR_15_BASE_ADDR_HIGH_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_PADDING_VALUE */
+#define TPC0_CFG_QM_TENSOR_15_PADDING_VALUE_V_SHIFT                  0
+#define TPC0_CFG_QM_TENSOR_15_PADDING_VALUE_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG */
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_DATA_TYPE_SHIFT          0
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_DATA_TYPE_MASK           0x7
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_VALID_DIM_MASK_SHIFT     8
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_VALID_DIM_MASK_MASK      0x1F00
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_LAST_DIM_SHIFT           16
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_LAST_DIM_MASK            0x70000
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_RMW_SET_SHIFT            19
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_RMW_SET_MASK             0x80000
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_RMW_RESERV_SHIFT         20
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_RMW_RESERV_MASK          0x100000
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_RMW_OP_SHIFT             21
+#define TPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG_RMW_OP_MASK              0x600000
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_0_SIZE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_0_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_15_DIM_0_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_0_STRIDE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_0_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_15_DIM_0_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_1_SIZE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_1_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_15_DIM_1_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_1_STRIDE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_1_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_15_DIM_1_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_2_SIZE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_2_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_15_DIM_2_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_2_STRIDE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_2_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_15_DIM_2_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_3_SIZE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_3_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_15_DIM_3_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_3_STRIDE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_3_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_15_DIM_3_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_4_SIZE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_4_SIZE_V_SHIFT                     0
+#define TPC0_CFG_QM_TENSOR_15_DIM_4_SIZE_V_MASK                      0xFFFFFFFF
+
+/* TPC0_CFG_QM_TENSOR_15_DIM_4_STRIDE */
+#define TPC0_CFG_QM_TENSOR_15_DIM_4_STRIDE_V_SHIFT                   0
+#define TPC0_CFG_QM_TENSOR_15_DIM_4_STRIDE_V_MASK                    0xFFFFFFFF
+
+/* TPC0_CFG_QM_SYNC_OBJECT_MESSAGE */
+#define TPC0_CFG_QM_SYNC_OBJECT_MESSAGE_SO_WRITE_VALUE_SHIFT         0
+#define TPC0_CFG_QM_SYNC_OBJECT_MESSAGE_SO_WRITE_VALUE_MASK          0xFFFF
+#define TPC0_CFG_QM_SYNC_OBJECT_MESSAGE_RSV_SHIFT                    16
+#define TPC0_CFG_QM_SYNC_OBJECT_MESSAGE_RSV_MASK                     0x1FFF0000
+#define TPC0_CFG_QM_SYNC_OBJECT_MESSAGE_SO_OPERATION_SHIFT           29
+#define TPC0_CFG_QM_SYNC_OBJECT_MESSAGE_SO_OPERATION_MASK            0xE0000000
+
+/* TPC0_CFG_QM_SYNC_OBJECT_ADDR */
+#define TPC0_CFG_QM_SYNC_OBJECT_ADDR_V_SHIFT                         0
+#define TPC0_CFG_QM_SYNC_OBJECT_ADDR_V_MASK                          0xFFFFFFFF
+
+/* TPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW */
+#define TPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW_V_SHIFT                  0
+#define TPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW_V_MASK                   0xFFFFFFFF
+
+/* TPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH */
+#define TPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH_V_SHIFT                 0
+#define TPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH_V_MASK                  0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_BASE_DIM_0 */
+#define TPC0_CFG_QM_TID_BASE_DIM_0_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_BASE_DIM_0_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_SIZE_DIM_0 */
+#define TPC0_CFG_QM_TID_SIZE_DIM_0_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_SIZE_DIM_0_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_BASE_DIM_1 */
+#define TPC0_CFG_QM_TID_BASE_DIM_1_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_BASE_DIM_1_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_SIZE_DIM_1 */
+#define TPC0_CFG_QM_TID_SIZE_DIM_1_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_SIZE_DIM_1_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_BASE_DIM_2 */
+#define TPC0_CFG_QM_TID_BASE_DIM_2_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_BASE_DIM_2_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_SIZE_DIM_2 */
+#define TPC0_CFG_QM_TID_SIZE_DIM_2_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_SIZE_DIM_2_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_BASE_DIM_3 */
+#define TPC0_CFG_QM_TID_BASE_DIM_3_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_BASE_DIM_3_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_SIZE_DIM_3 */
+#define TPC0_CFG_QM_TID_SIZE_DIM_3_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_SIZE_DIM_3_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_BASE_DIM_4 */
+#define TPC0_CFG_QM_TID_BASE_DIM_4_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_BASE_DIM_4_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_TID_SIZE_DIM_4 */
+#define TPC0_CFG_QM_TID_SIZE_DIM_4_V_SHIFT                           0
+#define TPC0_CFG_QM_TID_SIZE_DIM_4_V_MASK                            0xFFFFFFFF
+
+/* TPC0_CFG_QM_KERNEL_CONFIG */
+#define TPC0_CFG_QM_KERNEL_CONFIG_SMALL_VLM_SHIFT                    0
+#define TPC0_CFG_QM_KERNEL_CONFIG_SMALL_VLM_MASK                     0x1
+#define TPC0_CFG_QM_KERNEL_CONFIG_ASO_EVICT_L0_SHIFT                 1
+#define TPC0_CFG_QM_KERNEL_CONFIG_ASO_EVICT_L0_MASK                  0x2
+#define TPC0_CFG_QM_KERNEL_CONFIG_NUM_VALID_SRFS_SHIFT               2
+#define TPC0_CFG_QM_KERNEL_CONFIG_NUM_VALID_SRFS_MASK                0xFC
+#define TPC0_CFG_QM_KERNEL_CONFIG_RD_RATE_LIMIT_RST_TOKEN_SHIFT      8
+#define TPC0_CFG_QM_KERNEL_CONFIG_RD_RATE_LIMIT_RST_TOKEN_MASK       0xFF00
+#define TPC0_CFG_QM_KERNEL_CONFIG_WR_RATE_LIMIT_RST_TOKEN_SHIFT      16
+#define TPC0_CFG_QM_KERNEL_CONFIG_WR_RATE_LIMIT_RST_TOKEN_MASK       0xFF0000
+
+/* TPC0_CFG_QM_KERNEL_ID */
+#define TPC0_CFG_QM_KERNEL_ID_V_SHIFT                                0
+#define TPC0_CFG_QM_KERNEL_ID_V_MASK                                 0xFFFF
+
+/* TPC0_CFG_QM_SRF */
+#define TPC0_CFG_QM_SRF_V_SHIFT                                      0
+#define TPC0_CFG_QM_SRF_V_MASK                                       0xFFFFFFFF
+
+#endif /* ASIC_REG_TPC0_CFG_MASKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_regs.h
new file mode 100644
index 000000000000..b82a906265a8
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC0_CFG_REGS_H_
+#define ASIC_REG_TPC0_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC0_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xE06400
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xE06404
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xE06408
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xE0640C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xE06410
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xE06414
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xE06418
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xE0641C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xE06420
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xE06424
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xE06428
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xE0642C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xE06430
+
+#define mmTPC0_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xE06434
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xE06438
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xE0643C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xE06440
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xE06444
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xE06448
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xE0644C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xE06450
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xE06454
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xE06458
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xE0645C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xE06460
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xE06464
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xE06468
+
+#define mmTPC0_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xE0646C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xE06470
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xE06474
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xE06478
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xE0647C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xE06480
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xE06484
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xE06488
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xE0648C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xE06490
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xE06494
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xE06498
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xE0649C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xE064A0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xE064A4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xE064A8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xE064AC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xE064B0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xE064B4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xE064B8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xE064BC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xE064C0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xE064C4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xE064C8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xE064CC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xE064D0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xE064D4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xE064D8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xE064DC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xE064E0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xE064E4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xE064E8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xE064EC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xE064F0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xE064F4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xE064F8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xE064FC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xE06500
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xE06504
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xE06508
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xE0650C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xE06510
+
+#define mmTPC0_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xE06514
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xE06518
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xE0651C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xE06520
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xE06524
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xE06528
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xE0652C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xE06530
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xE06534
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xE06538
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xE0653C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xE06540
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xE06544
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xE06548
+
+#define mmTPC0_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xE0654C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xE06550
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xE06554
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xE06558
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xE0655C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xE06560
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xE06564
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xE06568
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xE0656C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xE06570
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xE06574
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xE06578
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xE0657C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xE06580
+
+#define mmTPC0_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xE06584
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xE06588
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xE0658C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xE06590
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xE06594
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xE06598
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xE0659C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xE065A0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xE065A4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xE065A8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xE065AC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xE065B0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xE065B4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xE065B8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xE065BC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xE065C0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xE065C4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xE065C8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xE065CC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xE065D0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xE065D4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xE065D8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xE065DC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xE065E0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xE065E4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xE065E8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xE065EC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xE065F0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xE065F4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xE065F8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xE065FC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xE06600
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xE06604
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xE06608
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xE0660C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xE06610
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xE06614
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xE06618
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xE0661C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xE06620
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xE06624
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xE06628
+
+#define mmTPC0_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xE0662C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xE06630
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xE06634
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xE06638
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xE0663C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xE06640
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xE06644
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xE06648
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xE0664C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xE06650
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xE06654
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xE06658
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xE0665C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xE06660
+
+#define mmTPC0_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xE06664
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xE06668
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xE0666C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xE06670
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xE06674
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xE06678
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xE0667C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xE06680
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xE06684
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xE06688
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xE0668C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xE06690
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xE06694
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xE06698
+
+#define mmTPC0_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xE0669C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xE066A0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xE066A4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xE066A8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xE066AC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xE066B0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xE066B4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xE066B8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xE066BC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xE066C0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xE066C4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xE066C8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xE066CC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xE066D0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xE066D4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xE066D8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xE066DC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xE066E0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xE066E4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xE066E8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xE066EC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xE066F0
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xE066F4
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xE066F8
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xE066FC
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xE06700
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xE06704
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xE06708
+
+#define mmTPC0_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xE0670C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xE06710
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xE06714
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xE06718
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xE0671C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xE06720
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xE06724
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xE06728
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xE0672C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xE06730
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xE06734
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xE06738
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xE0673C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xE06740
+
+#define mmTPC0_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xE06744
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xE06748
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xE0674C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xE06750
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xE06754
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xE06758
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xE0675C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xE06760
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xE06764
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xE06768
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xE0676C
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xE06770
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xE06774
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xE06778
+
+#define mmTPC0_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xE0677C
+
+#define mmTPC0_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xE06780
+
+#define mmTPC0_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xE06784
+
+#define mmTPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xE06788
+
+#define mmTPC0_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xE0678C
+
+#define mmTPC0_CFG_KERNEL_TID_BASE_DIM_0                             0xE06790
+
+#define mmTPC0_CFG_KERNEL_TID_SIZE_DIM_0                             0xE06794
+
+#define mmTPC0_CFG_KERNEL_TID_BASE_DIM_1                             0xE06798
+
+#define mmTPC0_CFG_KERNEL_TID_SIZE_DIM_1                             0xE0679C
+
+#define mmTPC0_CFG_KERNEL_TID_BASE_DIM_2                             0xE067A0
+
+#define mmTPC0_CFG_KERNEL_TID_SIZE_DIM_2                             0xE067A4
+
+#define mmTPC0_CFG_KERNEL_TID_BASE_DIM_3                             0xE067A8
+
+#define mmTPC0_CFG_KERNEL_TID_SIZE_DIM_3                             0xE067AC
+
+#define mmTPC0_CFG_KERNEL_TID_BASE_DIM_4                             0xE067B0
+
+#define mmTPC0_CFG_KERNEL_TID_SIZE_DIM_4                             0xE067B4
+
+#define mmTPC0_CFG_KERNEL_KERNEL_CONFIG                              0xE067B8
+
+#define mmTPC0_CFG_KERNEL_KERNEL_ID                                  0xE067BC
+
+#define mmTPC0_CFG_KERNEL_SRF_0                                      0xE067C0
+
+#define mmTPC0_CFG_KERNEL_SRF_1                                      0xE067C4
+
+#define mmTPC0_CFG_KERNEL_SRF_2                                      0xE067C8
+
+#define mmTPC0_CFG_KERNEL_SRF_3                                      0xE067CC
+
+#define mmTPC0_CFG_KERNEL_SRF_4                                      0xE067D0
+
+#define mmTPC0_CFG_KERNEL_SRF_5                                      0xE067D4
+
+#define mmTPC0_CFG_KERNEL_SRF_6                                      0xE067D8
+
+#define mmTPC0_CFG_KERNEL_SRF_7                                      0xE067DC
+
+#define mmTPC0_CFG_KERNEL_SRF_8                                      0xE067E0
+
+#define mmTPC0_CFG_KERNEL_SRF_9                                      0xE067E4
+
+#define mmTPC0_CFG_KERNEL_SRF_10                                     0xE067E8
+
+#define mmTPC0_CFG_KERNEL_SRF_11                                     0xE067EC
+
+#define mmTPC0_CFG_KERNEL_SRF_12                                     0xE067F0
+
+#define mmTPC0_CFG_KERNEL_SRF_13                                     0xE067F4
+
+#define mmTPC0_CFG_KERNEL_SRF_14                                     0xE067F8
+
+#define mmTPC0_CFG_KERNEL_SRF_15                                     0xE067FC
+
+#define mmTPC0_CFG_KERNEL_SRF_16                                     0xE06800
+
+#define mmTPC0_CFG_KERNEL_SRF_17                                     0xE06804
+
+#define mmTPC0_CFG_KERNEL_SRF_18                                     0xE06808
+
+#define mmTPC0_CFG_KERNEL_SRF_19                                     0xE0680C
+
+#define mmTPC0_CFG_KERNEL_SRF_20                                     0xE06810
+
+#define mmTPC0_CFG_KERNEL_SRF_21                                     0xE06814
+
+#define mmTPC0_CFG_KERNEL_SRF_22                                     0xE06818
+
+#define mmTPC0_CFG_KERNEL_SRF_23                                     0xE0681C
+
+#define mmTPC0_CFG_KERNEL_SRF_24                                     0xE06820
+
+#define mmTPC0_CFG_KERNEL_SRF_25                                     0xE06824
+
+#define mmTPC0_CFG_KERNEL_SRF_26                                     0xE06828
+
+#define mmTPC0_CFG_KERNEL_SRF_27                                     0xE0682C
+
+#define mmTPC0_CFG_KERNEL_SRF_28                                     0xE06830
+
+#define mmTPC0_CFG_KERNEL_SRF_29                                     0xE06834
+
+#define mmTPC0_CFG_KERNEL_SRF_30                                     0xE06838
+
+#define mmTPC0_CFG_KERNEL_SRF_31                                     0xE0683C
+
+#define mmTPC0_CFG_ROUND_CSR                                         0xE068FC
+
+#define mmTPC0_CFG_PROT                                              0xE06900
+
+#define mmTPC0_CFG_SEMAPHORE                                         0xE06908
+
+#define mmTPC0_CFG_VFLAGS                                            0xE0690C
+
+#define mmTPC0_CFG_SFLAGS                                            0xE06910
+
+#define mmTPC0_CFG_LFSR_POLYNOM                                      0xE06918
+
+#define mmTPC0_CFG_STATUS                                            0xE0691C
+
+#define mmTPC0_CFG_CFG_BASE_ADDRESS_HIGH                             0xE06920
+
+#define mmTPC0_CFG_CFG_SUBTRACT_VALUE                                0xE06924
+
+#define mmTPC0_CFG_SM_BASE_ADDRESS_HIGH                              0xE0692C
+
+#define mmTPC0_CFG_TPC_CMD                                           0xE06930
+
+#define mmTPC0_CFG_TPC_EXECUTE                                       0xE06938
+
+#define mmTPC0_CFG_TPC_STALL                                         0xE0693C
+
+#define mmTPC0_CFG_ICACHE_BASE_ADDERESS_LOW                          0xE06940
+
+#define mmTPC0_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xE06944
+
+#define mmTPC0_CFG_RD_RATE_LIMIT                                     0xE06948
+
+#define mmTPC0_CFG_WR_RATE_LIMIT                                     0xE06950
+
+#define mmTPC0_CFG_MSS_CONFIG                                        0xE06954
+
+#define mmTPC0_CFG_TPC_INTR_CAUSE                                    0xE06958
+
+#define mmTPC0_CFG_TPC_INTR_MASK                                     0xE0695C
+
+#define mmTPC0_CFG_WQ_CREDITS                                        0xE06960
+
+#define mmTPC0_CFG_ARUSER_LO                                         0xE06964
+
+#define mmTPC0_CFG_ARUSER_HI                                         0xE06968
+
+#define mmTPC0_CFG_AWUSER_LO                                         0xE0696C
+
+#define mmTPC0_CFG_AWUSER_HI                                         0xE06970
+
+#define mmTPC0_CFG_OPCODE_EXEC                                       0xE06974
+
+#define mmTPC0_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xE06978
+
+#define mmTPC0_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xE0697C
+
+#define mmTPC0_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xE06980
+
+#define mmTPC0_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xE06984
+
+#define mmTPC0_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xE06988
+
+#define mmTPC0_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xE0698C
+
+#define mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xE06990
+
+#define mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xE06994
+
+#define mmTPC0_CFG_TSB_CFG_MAX_SIZE                                  0xE06998
+
+#define mmTPC0_CFG_TSB_CFG                                           0xE0699C
+
+#define mmTPC0_CFG_DBGMEM_ADD                                        0xE069A0
+
+#define mmTPC0_CFG_DBGMEM_DATA_WR                                    0xE069A4
+
+#define mmTPC0_CFG_DBGMEM_DATA_RD                                    0xE069A8
+
+#define mmTPC0_CFG_DBGMEM_CTRL                                       0xE069AC
+
+#define mmTPC0_CFG_DBGMEM_RC                                         0xE069B0
+
+#define mmTPC0_CFG_TSB_INFLIGHT_CNTR                                 0xE069B4
+
+#define mmTPC0_CFG_WQ_INFLIGHT_CNTR                                  0xE069B8
+
+#define mmTPC0_CFG_WQ_LBW_TOTAL_CNTR                                 0xE069BC
+
+#define mmTPC0_CFG_WQ_HBW_TOTAL_CNTR                                 0xE069C0
+
+#define mmTPC0_CFG_IRQ_OCCOUPY_CNTR                                  0xE069C4
+
+#define mmTPC0_CFG_FUNC_MBIST_CNTRL                                  0xE069D0
+
+#define mmTPC0_CFG_FUNC_MBIST_PAT                                    0xE069D4
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_0                                  0xE069D8
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_1                                  0xE069DC
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_2                                  0xE069E0
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_3                                  0xE069E4
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_4                                  0xE069E8
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_5                                  0xE069EC
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_6                                  0xE069F0
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_7                                  0xE069F4
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_8                                  0xE069F8
+
+#define mmTPC0_CFG_FUNC_MBIST_MEM_9                                  0xE069FC
+
+#define mmTPC0_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xE06A00
+
+#define mmTPC0_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xE06A04
+
+#define mmTPC0_CFG_QM_TENSOR_0_PADDING_VALUE                         0xE06A08
+
+#define mmTPC0_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xE06A0C
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xE06A10
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xE06A14
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xE06A18
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xE06A1C
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xE06A20
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xE06A24
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xE06A28
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xE06A2C
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xE06A30
+
+#define mmTPC0_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xE06A34
+
+#define mmTPC0_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xE06A38
+
+#define mmTPC0_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xE06A3C
+
+#define mmTPC0_CFG_QM_TENSOR_1_PADDING_VALUE                         0xE06A40
+
+#define mmTPC0_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xE06A44
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xE06A48
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xE06A4C
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xE06A50
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xE06A54
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xE06A58
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xE06A5C
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xE06A60
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xE06A64
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xE06A68
+
+#define mmTPC0_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xE06A6C
+
+#define mmTPC0_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xE06A70
+
+#define mmTPC0_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xE06A74
+
+#define mmTPC0_CFG_QM_TENSOR_2_PADDING_VALUE                         0xE06A78
+
+#define mmTPC0_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xE06A7C
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xE06A80
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xE06A84
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xE06A88
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xE06A8C
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xE06A90
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xE06A94
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xE06A98
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xE06A9C
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xE06AA0
+
+#define mmTPC0_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xE06AA4
+
+#define mmTPC0_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xE06AA8
+
+#define mmTPC0_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xE06AAC
+
+#define mmTPC0_CFG_QM_TENSOR_3_PADDING_VALUE                         0xE06AB0
+
+#define mmTPC0_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xE06AB4
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xE06AB8
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xE06ABC
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xE06AC0
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xE06AC4
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xE06AC8
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xE06ACC
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xE06AD0
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xE06AD4
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xE06AD8
+
+#define mmTPC0_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xE06ADC
+
+#define mmTPC0_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xE06AE0
+
+#define mmTPC0_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xE06AE4
+
+#define mmTPC0_CFG_QM_TENSOR_4_PADDING_VALUE                         0xE06AE8
+
+#define mmTPC0_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xE06AEC
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xE06AF0
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xE06AF4
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xE06AF8
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xE06AFC
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xE06B00
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xE06B04
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xE06B08
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xE06B0C
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xE06B10
+
+#define mmTPC0_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xE06B14
+
+#define mmTPC0_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xE06B18
+
+#define mmTPC0_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xE06B1C
+
+#define mmTPC0_CFG_QM_TENSOR_5_PADDING_VALUE                         0xE06B20
+
+#define mmTPC0_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xE06B24
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xE06B28
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xE06B2C
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xE06B30
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xE06B34
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xE06B38
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xE06B3C
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xE06B40
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xE06B44
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xE06B48
+
+#define mmTPC0_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xE06B4C
+
+#define mmTPC0_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xE06B50
+
+#define mmTPC0_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xE06B54
+
+#define mmTPC0_CFG_QM_TENSOR_6_PADDING_VALUE                         0xE06B58
+
+#define mmTPC0_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xE06B5C
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xE06B60
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xE06B64
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xE06B68
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xE06B6C
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xE06B70
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xE06B74
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xE06B78
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xE06B7C
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xE06B80
+
+#define mmTPC0_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xE06B84
+
+#define mmTPC0_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xE06B88
+
+#define mmTPC0_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xE06B8C
+
+#define mmTPC0_CFG_QM_TENSOR_7_PADDING_VALUE                         0xE06B90
+
+#define mmTPC0_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xE06B94
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xE06B98
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xE06B9C
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xE06BA0
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xE06BA4
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xE06BA8
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xE06BAC
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xE06BB0
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xE06BB4
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xE06BB8
+
+#define mmTPC0_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xE06BBC
+
+#define mmTPC0_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xE06BC0
+
+#define mmTPC0_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xE06BC4
+
+#define mmTPC0_CFG_QM_TENSOR_8_PADDING_VALUE                         0xE06BC8
+
+#define mmTPC0_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xE06BCC
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xE06BD0
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xE06BD4
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xE06BD8
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xE06BDC
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xE06BE0
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xE06BE4
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xE06BE8
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xE06BEC
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xE06BF0
+
+#define mmTPC0_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xE06BF4
+
+#define mmTPC0_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xE06BF8
+
+#define mmTPC0_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xE06BFC
+
+#define mmTPC0_CFG_QM_TENSOR_9_PADDING_VALUE                         0xE06C00
+
+#define mmTPC0_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xE06C04
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xE06C08
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xE06C0C
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xE06C10
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xE06C14
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xE06C18
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xE06C1C
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xE06C20
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xE06C24
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xE06C28
+
+#define mmTPC0_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xE06C2C
+
+#define mmTPC0_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xE06C30
+
+#define mmTPC0_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xE06C34
+
+#define mmTPC0_CFG_QM_TENSOR_10_PADDING_VALUE                        0xE06C38
+
+#define mmTPC0_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xE06C3C
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xE06C40
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xE06C44
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xE06C48
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xE06C4C
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xE06C50
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xE06C54
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xE06C58
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xE06C5C
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xE06C60
+
+#define mmTPC0_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xE06C64
+
+#define mmTPC0_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xE06C68
+
+#define mmTPC0_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xE06C6C
+
+#define mmTPC0_CFG_QM_TENSOR_11_PADDING_VALUE                        0xE06C70
+
+#define mmTPC0_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xE06C74
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xE06C78
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xE06C7C
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xE06C80
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xE06C84
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xE06C88
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xE06C8C
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xE06C90
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xE06C94
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xE06C98
+
+#define mmTPC0_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xE06C9C
+
+#define mmTPC0_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xE06CA0
+
+#define mmTPC0_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xE06CA4
+
+#define mmTPC0_CFG_QM_TENSOR_12_PADDING_VALUE                        0xE06CA8
+
+#define mmTPC0_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xE06CAC
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xE06CB0
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xE06CB4
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xE06CB8
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xE06CBC
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xE06CC0
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xE06CC4
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xE06CC8
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xE06CCC
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xE06CD0
+
+#define mmTPC0_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xE06CD4
+
+#define mmTPC0_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xE06CD8
+
+#define mmTPC0_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xE06CDC
+
+#define mmTPC0_CFG_QM_TENSOR_13_PADDING_VALUE                        0xE06CE0
+
+#define mmTPC0_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xE06CE4
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xE06CE8
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xE06CEC
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xE06CF0
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xE06CF4
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xE06CF8
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xE06CFC
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xE06D00
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xE06D04
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xE06D08
+
+#define mmTPC0_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xE06D0C
+
+#define mmTPC0_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xE06D10
+
+#define mmTPC0_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xE06D14
+
+#define mmTPC0_CFG_QM_TENSOR_14_PADDING_VALUE                        0xE06D18
+
+#define mmTPC0_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xE06D1C
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xE06D20
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xE06D24
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xE06D28
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xE06D2C
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xE06D30
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xE06D34
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xE06D38
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xE06D3C
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xE06D40
+
+#define mmTPC0_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xE06D44
+
+#define mmTPC0_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xE06D48
+
+#define mmTPC0_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xE06D4C
+
+#define mmTPC0_CFG_QM_TENSOR_15_PADDING_VALUE                        0xE06D50
+
+#define mmTPC0_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xE06D54
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xE06D58
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xE06D5C
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xE06D60
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xE06D64
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xE06D68
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xE06D6C
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xE06D70
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xE06D74
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xE06D78
+
+#define mmTPC0_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xE06D7C
+
+#define mmTPC0_CFG_QM_SYNC_OBJECT_MESSAGE                            0xE06D80
+
+#define mmTPC0_CFG_QM_SYNC_OBJECT_ADDR                               0xE06D84
+
+#define mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xE06D88
+
+#define mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xE06D8C
+
+#define mmTPC0_CFG_QM_TID_BASE_DIM_0                                 0xE06D90
+
+#define mmTPC0_CFG_QM_TID_SIZE_DIM_0                                 0xE06D94
+
+#define mmTPC0_CFG_QM_TID_BASE_DIM_1                                 0xE06D98
+
+#define mmTPC0_CFG_QM_TID_SIZE_DIM_1                                 0xE06D9C
+
+#define mmTPC0_CFG_QM_TID_BASE_DIM_2                                 0xE06DA0
+
+#define mmTPC0_CFG_QM_TID_SIZE_DIM_2                                 0xE06DA4
+
+#define mmTPC0_CFG_QM_TID_BASE_DIM_3                                 0xE06DA8
+
+#define mmTPC0_CFG_QM_TID_SIZE_DIM_3                                 0xE06DAC
+
+#define mmTPC0_CFG_QM_TID_BASE_DIM_4                                 0xE06DB0
+
+#define mmTPC0_CFG_QM_TID_SIZE_DIM_4                                 0xE06DB4
+
+#define mmTPC0_CFG_QM_KERNEL_CONFIG                                  0xE06DB8
+
+#define mmTPC0_CFG_QM_KERNEL_ID                                      0xE06DBC
+
+#define mmTPC0_CFG_QM_SRF_0                                          0xE06DC0
+
+#define mmTPC0_CFG_QM_SRF_1                                          0xE06DC4
+
+#define mmTPC0_CFG_QM_SRF_2                                          0xE06DC8
+
+#define mmTPC0_CFG_QM_SRF_3                                          0xE06DCC
+
+#define mmTPC0_CFG_QM_SRF_4                                          0xE06DD0
+
+#define mmTPC0_CFG_QM_SRF_5                                          0xE06DD4
+
+#define mmTPC0_CFG_QM_SRF_6                                          0xE06DD8
+
+#define mmTPC0_CFG_QM_SRF_7                                          0xE06DDC
+
+#define mmTPC0_CFG_QM_SRF_8                                          0xE06DE0
+
+#define mmTPC0_CFG_QM_SRF_9                                          0xE06DE4
+
+#define mmTPC0_CFG_QM_SRF_10                                         0xE06DE8
+
+#define mmTPC0_CFG_QM_SRF_11                                         0xE06DEC
+
+#define mmTPC0_CFG_QM_SRF_12                                         0xE06DF0
+
+#define mmTPC0_CFG_QM_SRF_13                                         0xE06DF4
+
+#define mmTPC0_CFG_QM_SRF_14                                         0xE06DF8
+
+#define mmTPC0_CFG_QM_SRF_15                                         0xE06DFC
+
+#define mmTPC0_CFG_QM_SRF_16                                         0xE06E00
+
+#define mmTPC0_CFG_QM_SRF_17                                         0xE06E04
+
+#define mmTPC0_CFG_QM_SRF_18                                         0xE06E08
+
+#define mmTPC0_CFG_QM_SRF_19                                         0xE06E0C
+
+#define mmTPC0_CFG_QM_SRF_20                                         0xE06E10
+
+#define mmTPC0_CFG_QM_SRF_21                                         0xE06E14
+
+#define mmTPC0_CFG_QM_SRF_22                                         0xE06E18
+
+#define mmTPC0_CFG_QM_SRF_23                                         0xE06E1C
+
+#define mmTPC0_CFG_QM_SRF_24                                         0xE06E20
+
+#define mmTPC0_CFG_QM_SRF_25                                         0xE06E24
+
+#define mmTPC0_CFG_QM_SRF_26                                         0xE06E28
+
+#define mmTPC0_CFG_QM_SRF_27                                         0xE06E2C
+
+#define mmTPC0_CFG_QM_SRF_28                                         0xE06E30
+
+#define mmTPC0_CFG_QM_SRF_29                                         0xE06E34
+
+#define mmTPC0_CFG_QM_SRF_30                                         0xE06E38
+
+#define mmTPC0_CFG_QM_SRF_31                                         0xE06E3C
+
+#endif /* ASIC_REG_TPC0_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_masks.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_masks.h
new file mode 100644
index 000000000000..8e71532c6f36
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_masks.h
@@ -0,0 +1,800 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC0_QM_MASKS_H_
+#define ASIC_REG_TPC0_QM_MASKS_H_
+
+/*
+ *****************************************
+ *   TPC0_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+/* TPC0_QM_GLBL_CFG0 */
+#define TPC0_QM_GLBL_CFG0_PQF_EN_SHIFT                               0
+#define TPC0_QM_GLBL_CFG0_PQF_EN_MASK                                0xF
+#define TPC0_QM_GLBL_CFG0_CQF_EN_SHIFT                               4
+#define TPC0_QM_GLBL_CFG0_CQF_EN_MASK                                0x1F0
+#define TPC0_QM_GLBL_CFG0_CP_EN_SHIFT                                9
+#define TPC0_QM_GLBL_CFG0_CP_EN_MASK                                 0x3E00
+
+/* TPC0_QM_GLBL_CFG1 */
+#define TPC0_QM_GLBL_CFG1_PQF_STOP_SHIFT                             0
+#define TPC0_QM_GLBL_CFG1_PQF_STOP_MASK                              0xF
+#define TPC0_QM_GLBL_CFG1_CQF_STOP_SHIFT                             4
+#define TPC0_QM_GLBL_CFG1_CQF_STOP_MASK                              0x1F0
+#define TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT                              9
+#define TPC0_QM_GLBL_CFG1_CP_STOP_MASK                               0x3E00
+#define TPC0_QM_GLBL_CFG1_PQF_FLUSH_SHIFT                            16
+#define TPC0_QM_GLBL_CFG1_PQF_FLUSH_MASK                             0xF0000
+#define TPC0_QM_GLBL_CFG1_CQF_FLUSH_SHIFT                            20
+#define TPC0_QM_GLBL_CFG1_CQF_FLUSH_MASK                             0x1F00000
+#define TPC0_QM_GLBL_CFG1_CP_FLUSH_SHIFT                             25
+#define TPC0_QM_GLBL_CFG1_CP_FLUSH_MASK                              0x3E000000
+
+/* TPC0_QM_GLBL_PROT */
+#define TPC0_QM_GLBL_PROT_PQF_SHIFT                                  0
+#define TPC0_QM_GLBL_PROT_PQF_MASK                                   0xF
+#define TPC0_QM_GLBL_PROT_CQF_SHIFT                                  4
+#define TPC0_QM_GLBL_PROT_CQF_MASK                                   0x1F0
+#define TPC0_QM_GLBL_PROT_CP_SHIFT                                   9
+#define TPC0_QM_GLBL_PROT_CP_MASK                                    0x3E00
+#define TPC0_QM_GLBL_PROT_ERR_SHIFT                                  14
+#define TPC0_QM_GLBL_PROT_ERR_MASK                                   0x4000
+#define TPC0_QM_GLBL_PROT_ARB_SHIFT                                  15
+#define TPC0_QM_GLBL_PROT_ARB_MASK                                   0x8000
+
+/* TPC0_QM_GLBL_ERR_CFG */
+#define TPC0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_SHIFT                    0
+#define TPC0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK                     0xF
+#define TPC0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_SHIFT                    4
+#define TPC0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_MASK                     0x1F0
+#define TPC0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_SHIFT                     9
+#define TPC0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_MASK                      0x3E00
+#define TPC0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT                   16
+#define TPC0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK                    0xF0000
+#define TPC0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT                   20
+#define TPC0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK                    0x1F00000
+#define TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT                    25
+#define TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK                     0x3E000000
+#define TPC0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_SHIFT                   31
+#define TPC0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK                    0x80000000
+
+/* TPC0_QM_GLBL_SECURE_PROPS */
+#define TPC0_QM_GLBL_SECURE_PROPS_0_ASID_SHIFT                       0
+#define TPC0_QM_GLBL_SECURE_PROPS_0_ASID_MASK                        0x3FF
+#define TPC0_QM_GLBL_SECURE_PROPS_1_ASID_SHIFT                       0
+#define TPC0_QM_GLBL_SECURE_PROPS_1_ASID_MASK                        0x3FF
+#define TPC0_QM_GLBL_SECURE_PROPS_2_ASID_SHIFT                       0
+#define TPC0_QM_GLBL_SECURE_PROPS_2_ASID_MASK                        0x3FF
+#define TPC0_QM_GLBL_SECURE_PROPS_3_ASID_SHIFT                       0
+#define TPC0_QM_GLBL_SECURE_PROPS_3_ASID_MASK                        0x3FF
+#define TPC0_QM_GLBL_SECURE_PROPS_4_ASID_SHIFT                       0
+#define TPC0_QM_GLBL_SECURE_PROPS_4_ASID_MASK                        0x3FF
+#define TPC0_QM_GLBL_SECURE_PROPS_0_MMBP_SHIFT                       10
+#define TPC0_QM_GLBL_SECURE_PROPS_0_MMBP_MASK                        0x400
+#define TPC0_QM_GLBL_SECURE_PROPS_1_MMBP_SHIFT                       10
+#define TPC0_QM_GLBL_SECURE_PROPS_1_MMBP_MASK                        0x400
+#define TPC0_QM_GLBL_SECURE_PROPS_2_MMBP_SHIFT                       10
+#define TPC0_QM_GLBL_SECURE_PROPS_2_MMBP_MASK                        0x400
+#define TPC0_QM_GLBL_SECURE_PROPS_3_MMBP_SHIFT                       10
+#define TPC0_QM_GLBL_SECURE_PROPS_3_MMBP_MASK                        0x400
+#define TPC0_QM_GLBL_SECURE_PROPS_4_MMBP_SHIFT                       10
+#define TPC0_QM_GLBL_SECURE_PROPS_4_MMBP_MASK                        0x400
+
+/* TPC0_QM_GLBL_NON_SECURE_PROPS */
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_0_ASID_SHIFT                   0
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK                    0x3FF
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_1_ASID_SHIFT                   0
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_1_ASID_MASK                    0x3FF
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_2_ASID_SHIFT                   0
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_2_ASID_MASK                    0x3FF
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_3_ASID_SHIFT                   0
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_3_ASID_MASK                    0x3FF
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_4_ASID_SHIFT                   0
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_4_ASID_MASK                    0x3FF
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_0_MMBP_SHIFT                   10
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_0_MMBP_MASK                    0x400
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_1_MMBP_SHIFT                   10
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_1_MMBP_MASK                    0x400
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_2_MMBP_SHIFT                   10
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_2_MMBP_MASK                    0x400
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_3_MMBP_SHIFT                   10
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_3_MMBP_MASK                    0x400
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_4_MMBP_SHIFT                   10
+#define TPC0_QM_GLBL_NON_SECURE_PROPS_4_MMBP_MASK                    0x400
+
+/* TPC0_QM_GLBL_STS0 */
+#define TPC0_QM_GLBL_STS0_PQF_IDLE_SHIFT                             0
+#define TPC0_QM_GLBL_STS0_PQF_IDLE_MASK                              0xF
+#define TPC0_QM_GLBL_STS0_CQF_IDLE_SHIFT                             4
+#define TPC0_QM_GLBL_STS0_CQF_IDLE_MASK                              0x1F0
+#define TPC0_QM_GLBL_STS0_CP_IDLE_SHIFT                              9
+#define TPC0_QM_GLBL_STS0_CP_IDLE_MASK                               0x3E00
+#define TPC0_QM_GLBL_STS0_PQF_IS_STOP_SHIFT                          16
+#define TPC0_QM_GLBL_STS0_PQF_IS_STOP_MASK                           0xF0000
+#define TPC0_QM_GLBL_STS0_CQF_IS_STOP_SHIFT                          20
+#define TPC0_QM_GLBL_STS0_CQF_IS_STOP_MASK                           0x1F00000
+#define TPC0_QM_GLBL_STS0_CP_IS_STOP_SHIFT                           25
+#define TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK                            0x3E000000
+#define TPC0_QM_GLBL_STS0_ARB_IS_STOP_SHIFT                          31
+#define TPC0_QM_GLBL_STS0_ARB_IS_STOP_MASK                           0x80000000
+
+/* TPC0_QM_GLBL_STS1 */
+#define TPC0_QM_GLBL_STS1_PQF_RD_ERR_SHIFT                           0
+#define TPC0_QM_GLBL_STS1_PQF_RD_ERR_MASK                            0x1
+#define TPC0_QM_GLBL_STS1_CQF_RD_ERR_SHIFT                           1
+#define TPC0_QM_GLBL_STS1_CQF_RD_ERR_MASK                            0x2
+#define TPC0_QM_GLBL_STS1_CP_RD_ERR_SHIFT                            2
+#define TPC0_QM_GLBL_STS1_CP_RD_ERR_MASK                             0x4
+#define TPC0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_SHIFT                     3
+#define TPC0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_MASK                      0x8
+#define TPC0_QM_GLBL_STS1_CP_STOP_OP_SHIFT                           4
+#define TPC0_QM_GLBL_STS1_CP_STOP_OP_MASK                            0x10
+#define TPC0_QM_GLBL_STS1_CP_MSG_WR_ERR_SHIFT                        5
+#define TPC0_QM_GLBL_STS1_CP_MSG_WR_ERR_MASK                         0x20
+#define TPC0_QM_GLBL_STS1_CP_WREG_ERR_SHIFT                          6
+#define TPC0_QM_GLBL_STS1_CP_WREG_ERR_MASK                           0x40
+#define TPC0_QM_GLBL_STS1_CP_FENCE0_OVF_ERR_SHIFT                    8
+#define TPC0_QM_GLBL_STS1_CP_FENCE0_OVF_ERR_MASK                     0x100
+#define TPC0_QM_GLBL_STS1_CP_FENCE1_OVF_ERR_SHIFT                    9
+#define TPC0_QM_GLBL_STS1_CP_FENCE1_OVF_ERR_MASK                     0x200
+#define TPC0_QM_GLBL_STS1_CP_FENCE2_OVF_ERR_SHIFT                    10
+#define TPC0_QM_GLBL_STS1_CP_FENCE2_OVF_ERR_MASK                     0x400
+#define TPC0_QM_GLBL_STS1_CP_FENCE3_OVF_ERR_SHIFT                    11
+#define TPC0_QM_GLBL_STS1_CP_FENCE3_OVF_ERR_MASK                     0x800
+#define TPC0_QM_GLBL_STS1_CP_FENCE0_UDF_ERR_SHIFT                    12
+#define TPC0_QM_GLBL_STS1_CP_FENCE0_UDF_ERR_MASK                     0x1000
+#define TPC0_QM_GLBL_STS1_CP_FENCE1_UDF_ERR_SHIFT                    13
+#define TPC0_QM_GLBL_STS1_CP_FENCE1_UDF_ERR_MASK                     0x2000
+#define TPC0_QM_GLBL_STS1_CP_FENCE2_UDF_ERR_SHIFT                    14
+#define TPC0_QM_GLBL_STS1_CP_FENCE2_UDF_ERR_MASK                     0x4000
+#define TPC0_QM_GLBL_STS1_CP_FENCE3_UDF_ERR_SHIFT                    15
+#define TPC0_QM_GLBL_STS1_CP_FENCE3_UDF_ERR_MASK                     0x8000
+
+/* TPC0_QM_GLBL_STS1_4 */
+#define TPC0_QM_GLBL_STS1_4_CQF_RD_ERR_SHIFT                         1
+#define TPC0_QM_GLBL_STS1_4_CQF_RD_ERR_MASK                          0x2
+#define TPC0_QM_GLBL_STS1_4_CP_RD_ERR_SHIFT                          2
+#define TPC0_QM_GLBL_STS1_4_CP_RD_ERR_MASK                           0x4
+#define TPC0_QM_GLBL_STS1_4_CP_UNDEF_CMD_ERR_SHIFT                   3
+#define TPC0_QM_GLBL_STS1_4_CP_UNDEF_CMD_ERR_MASK                    0x8
+#define TPC0_QM_GLBL_STS1_4_CP_STOP_OP_SHIFT                         4
+#define TPC0_QM_GLBL_STS1_4_CP_STOP_OP_MASK                          0x10
+#define TPC0_QM_GLBL_STS1_4_CP_MSG_WR_ERR_SHIFT                      5
+#define TPC0_QM_GLBL_STS1_4_CP_MSG_WR_ERR_MASK                       0x20
+#define TPC0_QM_GLBL_STS1_4_CP_WREG_ERR_SHIFT                        6
+#define TPC0_QM_GLBL_STS1_4_CP_WREG_ERR_MASK                         0x40
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE0_OVF_ERR_SHIFT                  8
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE0_OVF_ERR_MASK                   0x100
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE1_OVF_ERR_SHIFT                  9
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE1_OVF_ERR_MASK                   0x200
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE2_OVF_ERR_SHIFT                  10
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE2_OVF_ERR_MASK                   0x400
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE3_OVF_ERR_SHIFT                  11
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE3_OVF_ERR_MASK                   0x800
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE0_UDF_ERR_SHIFT                  12
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE0_UDF_ERR_MASK                   0x1000
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE1_UDF_ERR_SHIFT                  13
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE1_UDF_ERR_MASK                   0x2000
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE2_UDF_ERR_SHIFT                  14
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE2_UDF_ERR_MASK                   0x4000
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE3_UDF_ERR_SHIFT                  15
+#define TPC0_QM_GLBL_STS1_4_CP_FENCE3_UDF_ERR_MASK                   0x8000
+
+/* TPC0_QM_GLBL_MSG_EN */
+#define TPC0_QM_GLBL_MSG_EN_PQF_RD_ERR_SHIFT                         0
+#define TPC0_QM_GLBL_MSG_EN_PQF_RD_ERR_MASK                          0x1
+#define TPC0_QM_GLBL_MSG_EN_CQF_RD_ERR_SHIFT                         1
+#define TPC0_QM_GLBL_MSG_EN_CQF_RD_ERR_MASK                          0x2
+#define TPC0_QM_GLBL_MSG_EN_CP_RD_ERR_SHIFT                          2
+#define TPC0_QM_GLBL_MSG_EN_CP_RD_ERR_MASK                           0x4
+#define TPC0_QM_GLBL_MSG_EN_CP_UNDEF_CMD_ERR_SHIFT                   3
+#define TPC0_QM_GLBL_MSG_EN_CP_UNDEF_CMD_ERR_MASK                    0x8
+#define TPC0_QM_GLBL_MSG_EN_CP_STOP_OP_SHIFT                         4
+#define TPC0_QM_GLBL_MSG_EN_CP_STOP_OP_MASK                          0x10
+#define TPC0_QM_GLBL_MSG_EN_CP_MSG_WR_ERR_SHIFT                      5
+#define TPC0_QM_GLBL_MSG_EN_CP_MSG_WR_ERR_MASK                       0x20
+#define TPC0_QM_GLBL_MSG_EN_CP_WREG_ERR_SHIFT                        6
+#define TPC0_QM_GLBL_MSG_EN_CP_WREG_ERR_MASK                         0x40
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE0_OVF_ERR_SHIFT                  8
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE0_OVF_ERR_MASK                   0x100
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE1_OVF_ERR_SHIFT                  9
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE1_OVF_ERR_MASK                   0x200
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE2_OVF_ERR_SHIFT                  10
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE2_OVF_ERR_MASK                   0x400
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE3_OVF_ERR_SHIFT                  11
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE3_OVF_ERR_MASK                   0x800
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE0_UDF_ERR_SHIFT                  12
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE0_UDF_ERR_MASK                   0x1000
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE1_UDF_ERR_SHIFT                  13
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE1_UDF_ERR_MASK                   0x2000
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE2_UDF_ERR_SHIFT                  14
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE2_UDF_ERR_MASK                   0x4000
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE3_UDF_ERR_SHIFT                  15
+#define TPC0_QM_GLBL_MSG_EN_CP_FENCE3_UDF_ERR_MASK                   0x8000
+
+/* TPC0_QM_GLBL_MSG_EN_4 */
+#define TPC0_QM_GLBL_MSG_EN_4_CQF_RD_ERR_SHIFT                       1
+#define TPC0_QM_GLBL_MSG_EN_4_CQF_RD_ERR_MASK                        0x2
+#define TPC0_QM_GLBL_MSG_EN_4_CP_RD_ERR_SHIFT                        2
+#define TPC0_QM_GLBL_MSG_EN_4_CP_RD_ERR_MASK                         0x4
+#define TPC0_QM_GLBL_MSG_EN_4_CP_UNDEF_CMD_ERR_SHIFT                 3
+#define TPC0_QM_GLBL_MSG_EN_4_CP_UNDEF_CMD_ERR_MASK                  0x8
+#define TPC0_QM_GLBL_MSG_EN_4_CP_STOP_OP_SHIFT                       4
+#define TPC0_QM_GLBL_MSG_EN_4_CP_STOP_OP_MASK                        0x10
+#define TPC0_QM_GLBL_MSG_EN_4_CP_MSG_WR_ERR_SHIFT                    5
+#define TPC0_QM_GLBL_MSG_EN_4_CP_MSG_WR_ERR_MASK                     0x20
+#define TPC0_QM_GLBL_MSG_EN_4_CP_WREG_ERR_SHIFT                      6
+#define TPC0_QM_GLBL_MSG_EN_4_CP_WREG_ERR_MASK                       0x40
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE0_OVF_ERR_SHIFT                8
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE0_OVF_ERR_MASK                 0x100
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE1_OVF_ERR_SHIFT                9
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE1_OVF_ERR_MASK                 0x200
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE2_OVF_ERR_SHIFT                10
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE2_OVF_ERR_MASK                 0x400
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE3_OVF_ERR_SHIFT                11
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE3_OVF_ERR_MASK                 0x800
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE0_UDF_ERR_SHIFT                12
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE0_UDF_ERR_MASK                 0x1000
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE1_UDF_ERR_SHIFT                13
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE1_UDF_ERR_MASK                 0x2000
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE2_UDF_ERR_SHIFT                14
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE2_UDF_ERR_MASK                 0x4000
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE3_UDF_ERR_SHIFT                15
+#define TPC0_QM_GLBL_MSG_EN_4_CP_FENCE3_UDF_ERR_MASK                 0x8000
+
+/* TPC0_QM_PQ_BASE_LO */
+#define TPC0_QM_PQ_BASE_LO_VAL_SHIFT                                 0
+#define TPC0_QM_PQ_BASE_LO_VAL_MASK                                  0xFFFFFFFF
+
+/* TPC0_QM_PQ_BASE_HI */
+#define TPC0_QM_PQ_BASE_HI_VAL_SHIFT                                 0
+#define TPC0_QM_PQ_BASE_HI_VAL_MASK                                  0xFFFFFFFF
+
+/* TPC0_QM_PQ_SIZE */
+#define TPC0_QM_PQ_SIZE_VAL_SHIFT                                    0
+#define TPC0_QM_PQ_SIZE_VAL_MASK                                     0xFFFFFFFF
+
+/* TPC0_QM_PQ_PI */
+#define TPC0_QM_PQ_PI_VAL_SHIFT                                      0
+#define TPC0_QM_PQ_PI_VAL_MASK                                       0xFFFFFFFF
+
+/* TPC0_QM_PQ_CI */
+#define TPC0_QM_PQ_CI_VAL_SHIFT                                      0
+#define TPC0_QM_PQ_CI_VAL_MASK                                       0xFFFFFFFF
+
+/* TPC0_QM_PQ_CFG0 */
+#define TPC0_QM_PQ_CFG0_RESERVED_SHIFT                               0
+#define TPC0_QM_PQ_CFG0_RESERVED_MASK                                0x1
+
+/* TPC0_QM_PQ_CFG1 */
+#define TPC0_QM_PQ_CFG1_CREDIT_LIM_SHIFT                             0
+#define TPC0_QM_PQ_CFG1_CREDIT_LIM_MASK                              0xFFFF
+#define TPC0_QM_PQ_CFG1_MAX_INFLIGHT_SHIFT                           16
+#define TPC0_QM_PQ_CFG1_MAX_INFLIGHT_MASK                            0xFFFF0000
+
+/* TPC0_QM_PQ_ARUSER_31_11 */
+#define TPC0_QM_PQ_ARUSER_31_11_VAL_SHIFT                            0
+#define TPC0_QM_PQ_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* TPC0_QM_PQ_STS0 */
+#define TPC0_QM_PQ_STS0_PQ_CREDIT_CNT_SHIFT                          0
+#define TPC0_QM_PQ_STS0_PQ_CREDIT_CNT_MASK                           0xFFFF
+#define TPC0_QM_PQ_STS0_PQ_FREE_CNT_SHIFT                            16
+#define TPC0_QM_PQ_STS0_PQ_FREE_CNT_MASK                             0xFFFF0000
+
+/* TPC0_QM_PQ_STS1 */
+#define TPC0_QM_PQ_STS1_PQ_INFLIGHT_CNT_SHIFT                        0
+#define TPC0_QM_PQ_STS1_PQ_INFLIGHT_CNT_MASK                         0xFFFF
+#define TPC0_QM_PQ_STS1_PQ_BUF_EMPTY_SHIFT                           30
+#define TPC0_QM_PQ_STS1_PQ_BUF_EMPTY_MASK                            0x40000000
+#define TPC0_QM_PQ_STS1_PQ_BUSY_SHIFT                                31
+#define TPC0_QM_PQ_STS1_PQ_BUSY_MASK                                 0x80000000
+
+/* TPC0_QM_CQ_CFG0 */
+#define TPC0_QM_CQ_CFG0_RESERVED_SHIFT                               0
+#define TPC0_QM_CQ_CFG0_RESERVED_MASK                                0x1
+
+/* TPC0_QM_CQ_CFG1 */
+#define TPC0_QM_CQ_CFG1_CREDIT_LIM_SHIFT                             0
+#define TPC0_QM_CQ_CFG1_CREDIT_LIM_MASK                              0xFFFF
+#define TPC0_QM_CQ_CFG1_MAX_INFLIGHT_SHIFT                           16
+#define TPC0_QM_CQ_CFG1_MAX_INFLIGHT_MASK                            0xFFFF0000
+
+/* TPC0_QM_CQ_ARUSER_31_11 */
+#define TPC0_QM_CQ_ARUSER_31_11_VAL_SHIFT                            0
+#define TPC0_QM_CQ_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* TPC0_QM_CQ_STS0 */
+#define TPC0_QM_CQ_STS0_CQ_CREDIT_CNT_SHIFT                          0
+#define TPC0_QM_CQ_STS0_CQ_CREDIT_CNT_MASK                           0xFFFF
+#define TPC0_QM_CQ_STS0_CQ_FREE_CNT_SHIFT                            16
+#define TPC0_QM_CQ_STS0_CQ_FREE_CNT_MASK                             0xFFFF0000
+
+/* TPC0_QM_CQ_STS1 */
+#define TPC0_QM_CQ_STS1_CQ_INFLIGHT_CNT_SHIFT                        0
+#define TPC0_QM_CQ_STS1_CQ_INFLIGHT_CNT_MASK                         0xFFFF
+#define TPC0_QM_CQ_STS1_CQ_BUF_EMPTY_SHIFT                           30
+#define TPC0_QM_CQ_STS1_CQ_BUF_EMPTY_MASK                            0x40000000
+#define TPC0_QM_CQ_STS1_CQ_BUSY_SHIFT                                31
+#define TPC0_QM_CQ_STS1_CQ_BUSY_MASK                                 0x80000000
+
+/* TPC0_QM_CQ_PTR_LO_0 */
+#define TPC0_QM_CQ_PTR_LO_0_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_LO_0_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_PTR_HI_0 */
+#define TPC0_QM_CQ_PTR_HI_0_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_HI_0_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_TSIZE_0 */
+#define TPC0_QM_CQ_TSIZE_0_VAL_SHIFT                                 0
+#define TPC0_QM_CQ_TSIZE_0_VAL_MASK                                  0xFFFFFFFF
+
+/* TPC0_QM_CQ_CTL_0 */
+#define TPC0_QM_CQ_CTL_0_RPT_SHIFT                                   0
+#define TPC0_QM_CQ_CTL_0_RPT_MASK                                    0xFFFF
+#define TPC0_QM_CQ_CTL_0_CTL_SHIFT                                   16
+#define TPC0_QM_CQ_CTL_0_CTL_MASK                                    0xFFFF0000
+
+/* TPC0_QM_CQ_PTR_LO_1 */
+#define TPC0_QM_CQ_PTR_LO_1_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_LO_1_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_PTR_HI_1 */
+#define TPC0_QM_CQ_PTR_HI_1_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_HI_1_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_TSIZE_1 */
+#define TPC0_QM_CQ_TSIZE_1_VAL_SHIFT                                 0
+#define TPC0_QM_CQ_TSIZE_1_VAL_MASK                                  0xFFFFFFFF
+
+/* TPC0_QM_CQ_CTL_1 */
+#define TPC0_QM_CQ_CTL_1_RPT_SHIFT                                   0
+#define TPC0_QM_CQ_CTL_1_RPT_MASK                                    0xFFFF
+#define TPC0_QM_CQ_CTL_1_CTL_SHIFT                                   16
+#define TPC0_QM_CQ_CTL_1_CTL_MASK                                    0xFFFF0000
+
+/* TPC0_QM_CQ_PTR_LO_2 */
+#define TPC0_QM_CQ_PTR_LO_2_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_LO_2_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_PTR_HI_2 */
+#define TPC0_QM_CQ_PTR_HI_2_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_HI_2_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_TSIZE_2 */
+#define TPC0_QM_CQ_TSIZE_2_VAL_SHIFT                                 0
+#define TPC0_QM_CQ_TSIZE_2_VAL_MASK                                  0xFFFFFFFF
+
+/* TPC0_QM_CQ_CTL_2 */
+#define TPC0_QM_CQ_CTL_2_RPT_SHIFT                                   0
+#define TPC0_QM_CQ_CTL_2_RPT_MASK                                    0xFFFF
+#define TPC0_QM_CQ_CTL_2_CTL_SHIFT                                   16
+#define TPC0_QM_CQ_CTL_2_CTL_MASK                                    0xFFFF0000
+
+/* TPC0_QM_CQ_PTR_LO_3 */
+#define TPC0_QM_CQ_PTR_LO_3_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_LO_3_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_PTR_HI_3 */
+#define TPC0_QM_CQ_PTR_HI_3_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_HI_3_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_TSIZE_3 */
+#define TPC0_QM_CQ_TSIZE_3_VAL_SHIFT                                 0
+#define TPC0_QM_CQ_TSIZE_3_VAL_MASK                                  0xFFFFFFFF
+
+/* TPC0_QM_CQ_CTL_3 */
+#define TPC0_QM_CQ_CTL_3_RPT_SHIFT                                   0
+#define TPC0_QM_CQ_CTL_3_RPT_MASK                                    0xFFFF
+#define TPC0_QM_CQ_CTL_3_CTL_SHIFT                                   16
+#define TPC0_QM_CQ_CTL_3_CTL_MASK                                    0xFFFF0000
+
+/* TPC0_QM_CQ_PTR_LO_4 */
+#define TPC0_QM_CQ_PTR_LO_4_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_LO_4_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_PTR_HI_4 */
+#define TPC0_QM_CQ_PTR_HI_4_VAL_SHIFT                                0
+#define TPC0_QM_CQ_PTR_HI_4_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_CQ_TSIZE_4 */
+#define TPC0_QM_CQ_TSIZE_4_VAL_SHIFT                                 0
+#define TPC0_QM_CQ_TSIZE_4_VAL_MASK                                  0xFFFFFFFF
+
+/* TPC0_QM_CQ_CTL_4 */
+#define TPC0_QM_CQ_CTL_4_RPT_SHIFT                                   0
+#define TPC0_QM_CQ_CTL_4_RPT_MASK                                    0xFFFF
+#define TPC0_QM_CQ_CTL_4_CTL_SHIFT                                   16
+#define TPC0_QM_CQ_CTL_4_CTL_MASK                                    0xFFFF0000
+
+/* TPC0_QM_CQ_PTR_LO_STS */
+#define TPC0_QM_CQ_PTR_LO_STS_VAL_SHIFT                              0
+#define TPC0_QM_CQ_PTR_LO_STS_VAL_MASK                               0xFFFFFFFF
+
+/* TPC0_QM_CQ_PTR_HI_STS */
+#define TPC0_QM_CQ_PTR_HI_STS_VAL_SHIFT                              0
+#define TPC0_QM_CQ_PTR_HI_STS_VAL_MASK                               0xFFFFFFFF
+
+/* TPC0_QM_CQ_TSIZE_STS */
+#define TPC0_QM_CQ_TSIZE_STS_VAL_SHIFT                               0
+#define TPC0_QM_CQ_TSIZE_STS_VAL_MASK                                0xFFFFFFFF
+
+/* TPC0_QM_CQ_CTL_STS */
+#define TPC0_QM_CQ_CTL_STS_RPT_SHIFT                                 0
+#define TPC0_QM_CQ_CTL_STS_RPT_MASK                                  0xFFFF
+#define TPC0_QM_CQ_CTL_STS_CTL_SHIFT                                 16
+#define TPC0_QM_CQ_CTL_STS_CTL_MASK                                  0xFFFF0000
+
+/* TPC0_QM_CQ_IFIFO_CNT */
+#define TPC0_QM_CQ_IFIFO_CNT_VAL_SHIFT                               0
+#define TPC0_QM_CQ_IFIFO_CNT_VAL_MASK                                0x3
+
+/* TPC0_QM_CP_MSG_BASE0_ADDR_LO */
+#define TPC0_QM_CP_MSG_BASE0_ADDR_LO_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE0_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_MSG_BASE0_ADDR_HI */
+#define TPC0_QM_CP_MSG_BASE0_ADDR_HI_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE0_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_MSG_BASE1_ADDR_LO */
+#define TPC0_QM_CP_MSG_BASE1_ADDR_LO_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE1_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_MSG_BASE1_ADDR_HI */
+#define TPC0_QM_CP_MSG_BASE1_ADDR_HI_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE1_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_MSG_BASE2_ADDR_LO */
+#define TPC0_QM_CP_MSG_BASE2_ADDR_LO_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE2_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_MSG_BASE2_ADDR_HI */
+#define TPC0_QM_CP_MSG_BASE2_ADDR_HI_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE2_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_MSG_BASE3_ADDR_LO */
+#define TPC0_QM_CP_MSG_BASE3_ADDR_LO_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE3_ADDR_LO_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_MSG_BASE3_ADDR_HI */
+#define TPC0_QM_CP_MSG_BASE3_ADDR_HI_VAL_SHIFT                       0
+#define TPC0_QM_CP_MSG_BASE3_ADDR_HI_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_LDMA_TSIZE_OFFSET */
+#define TPC0_QM_CP_LDMA_TSIZE_OFFSET_VAL_SHIFT                       0
+#define TPC0_QM_CP_LDMA_TSIZE_OFFSET_VAL_MASK                        0xFFFFFFFF
+
+/* TPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET */
+#define TPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_VAL_SHIFT                 0
+#define TPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_VAL_MASK                  0xFFFFFFFF
+
+/* TPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET */
+#define TPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_VAL_SHIFT                 0
+#define TPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_VAL_MASK                  0xFFFFFFFF
+
+/* TPC0_QM_CP_FENCE0_RDATA */
+#define TPC0_QM_CP_FENCE0_RDATA_INC_VAL_SHIFT                        0
+#define TPC0_QM_CP_FENCE0_RDATA_INC_VAL_MASK                         0xF
+
+/* TPC0_QM_CP_FENCE1_RDATA */
+#define TPC0_QM_CP_FENCE1_RDATA_INC_VAL_SHIFT                        0
+#define TPC0_QM_CP_FENCE1_RDATA_INC_VAL_MASK                         0xF
+
+/* TPC0_QM_CP_FENCE2_RDATA */
+#define TPC0_QM_CP_FENCE2_RDATA_INC_VAL_SHIFT                        0
+#define TPC0_QM_CP_FENCE2_RDATA_INC_VAL_MASK                         0xF
+
+/* TPC0_QM_CP_FENCE3_RDATA */
+#define TPC0_QM_CP_FENCE3_RDATA_INC_VAL_SHIFT                        0
+#define TPC0_QM_CP_FENCE3_RDATA_INC_VAL_MASK                         0xF
+
+/* TPC0_QM_CP_FENCE0_CNT */
+#define TPC0_QM_CP_FENCE0_CNT_VAL_SHIFT                              0
+#define TPC0_QM_CP_FENCE0_CNT_VAL_MASK                               0x3FFF
+
+/* TPC0_QM_CP_FENCE1_CNT */
+#define TPC0_QM_CP_FENCE1_CNT_VAL_SHIFT                              0
+#define TPC0_QM_CP_FENCE1_CNT_VAL_MASK                               0x3FFF
+
+/* TPC0_QM_CP_FENCE2_CNT */
+#define TPC0_QM_CP_FENCE2_CNT_VAL_SHIFT                              0
+#define TPC0_QM_CP_FENCE2_CNT_VAL_MASK                               0x3FFF
+
+/* TPC0_QM_CP_FENCE3_CNT */
+#define TPC0_QM_CP_FENCE3_CNT_VAL_SHIFT                              0
+#define TPC0_QM_CP_FENCE3_CNT_VAL_MASK                               0x3FFF
+
+/* TPC0_QM_CP_STS */
+#define TPC0_QM_CP_STS_MSG_INFLIGHT_CNT_SHIFT                        0
+#define TPC0_QM_CP_STS_MSG_INFLIGHT_CNT_MASK                         0xFFFF
+#define TPC0_QM_CP_STS_ERDY_SHIFT                                    16
+#define TPC0_QM_CP_STS_ERDY_MASK                                     0x10000
+#define TPC0_QM_CP_STS_RRDY_SHIFT                                    17
+#define TPC0_QM_CP_STS_RRDY_MASK                                     0x20000
+#define TPC0_QM_CP_STS_MRDY_SHIFT                                    18
+#define TPC0_QM_CP_STS_MRDY_MASK                                     0x40000
+#define TPC0_QM_CP_STS_SW_STOP_SHIFT                                 19
+#define TPC0_QM_CP_STS_SW_STOP_MASK                                  0x80000
+#define TPC0_QM_CP_STS_FENCE_ID_SHIFT                                20
+#define TPC0_QM_CP_STS_FENCE_ID_MASK                                 0x300000
+#define TPC0_QM_CP_STS_FENCE_IN_PROGRESS_SHIFT                       22
+#define TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK                        0x400000
+
+/* TPC0_QM_CP_CURRENT_INST_LO */
+#define TPC0_QM_CP_CURRENT_INST_LO_VAL_SHIFT                         0
+#define TPC0_QM_CP_CURRENT_INST_LO_VAL_MASK                          0xFFFFFFFF
+
+/* TPC0_QM_CP_CURRENT_INST_HI */
+#define TPC0_QM_CP_CURRENT_INST_HI_VAL_SHIFT                         0
+#define TPC0_QM_CP_CURRENT_INST_HI_VAL_MASK                          0xFFFFFFFF
+
+/* TPC0_QM_CP_BARRIER_CFG */
+#define TPC0_QM_CP_BARRIER_CFG_EBGUARD_SHIFT                         0
+#define TPC0_QM_CP_BARRIER_CFG_EBGUARD_MASK                          0xFFF
+#define TPC0_QM_CP_BARRIER_CFG_RBGUARD_SHIFT                         16
+#define TPC0_QM_CP_BARRIER_CFG_RBGUARD_MASK                          0xF0000
+
+/* TPC0_QM_CP_DBG_0 */
+#define TPC0_QM_CP_DBG_0_CS_SHIFT                                    0
+#define TPC0_QM_CP_DBG_0_CS_MASK                                     0xF
+#define TPC0_QM_CP_DBG_0_EB_CNT_NOT_ZERO_SHIFT                       4
+#define TPC0_QM_CP_DBG_0_EB_CNT_NOT_ZERO_MASK                        0x10
+#define TPC0_QM_CP_DBG_0_BULK_CNT_NOT_ZERO_SHIFT                     5
+#define TPC0_QM_CP_DBG_0_BULK_CNT_NOT_ZERO_MASK                      0x20
+#define TPC0_QM_CP_DBG_0_MREB_STALL_SHIFT                            6
+#define TPC0_QM_CP_DBG_0_MREB_STALL_MASK                             0x40
+#define TPC0_QM_CP_DBG_0_STALL_SHIFT                                 7
+#define TPC0_QM_CP_DBG_0_STALL_MASK                                  0x80
+
+/* TPC0_QM_CP_ARUSER_31_11 */
+#define TPC0_QM_CP_ARUSER_31_11_VAL_SHIFT                            0
+#define TPC0_QM_CP_ARUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* TPC0_QM_CP_AWUSER_31_11 */
+#define TPC0_QM_CP_AWUSER_31_11_VAL_SHIFT                            0
+#define TPC0_QM_CP_AWUSER_31_11_VAL_MASK                             0x1FFFFF
+
+/* TPC0_QM_ARB_CFG_0 */
+#define TPC0_QM_ARB_CFG_0_TYPE_SHIFT                                 0
+#define TPC0_QM_ARB_CFG_0_TYPE_MASK                                  0x1
+#define TPC0_QM_ARB_CFG_0_IS_MASTER_SHIFT                            4
+#define TPC0_QM_ARB_CFG_0_IS_MASTER_MASK                             0x10
+#define TPC0_QM_ARB_CFG_0_EN_SHIFT                                   8
+#define TPC0_QM_ARB_CFG_0_EN_MASK                                    0x100
+#define TPC0_QM_ARB_CFG_0_MASK_SHIFT                                 12
+#define TPC0_QM_ARB_CFG_0_MASK_MASK                                  0xF000
+#define TPC0_QM_ARB_CFG_0_MST_MSG_NOSTALL_SHIFT                      16
+#define TPC0_QM_ARB_CFG_0_MST_MSG_NOSTALL_MASK                       0x10000
+
+/* TPC0_QM_ARB_CHOISE_Q_PUSH */
+#define TPC0_QM_ARB_CHOISE_Q_PUSH_VAL_SHIFT                          0
+#define TPC0_QM_ARB_CHOISE_Q_PUSH_VAL_MASK                           0x3
+
+/* TPC0_QM_ARB_WRR_WEIGHT */
+#define TPC0_QM_ARB_WRR_WEIGHT_VAL_SHIFT                             0
+#define TPC0_QM_ARB_WRR_WEIGHT_VAL_MASK                              0xFFFFFFFF
+
+/* TPC0_QM_ARB_CFG_1 */
+#define TPC0_QM_ARB_CFG_1_CLR_SHIFT                                  0
+#define TPC0_QM_ARB_CFG_1_CLR_MASK                                   0x1
+
+/* TPC0_QM_ARB_MST_AVAIL_CRED */
+#define TPC0_QM_ARB_MST_AVAIL_CRED_VAL_SHIFT                         0
+#define TPC0_QM_ARB_MST_AVAIL_CRED_VAL_MASK                          0x7F
+
+/* TPC0_QM_ARB_MST_CRED_INC */
+#define TPC0_QM_ARB_MST_CRED_INC_VAL_SHIFT                           0
+#define TPC0_QM_ARB_MST_CRED_INC_VAL_MASK                            0xFFFFFFFF
+
+/* TPC0_QM_ARB_MST_CHOISE_PUSH_OFST */
+#define TPC0_QM_ARB_MST_CHOISE_PUSH_OFST_VAL_SHIFT                   0
+#define TPC0_QM_ARB_MST_CHOISE_PUSH_OFST_VAL_MASK                    0xFFFFFFFF
+
+/* TPC0_QM_ARB_SLV_MASTER_INC_CRED_OFST */
+#define TPC0_QM_ARB_SLV_MASTER_INC_CRED_OFST_VAL_SHIFT               0
+#define TPC0_QM_ARB_SLV_MASTER_INC_CRED_OFST_VAL_MASK                0xFFFFFFFF
+
+/* TPC0_QM_ARB_MST_SLAVE_EN */
+#define TPC0_QM_ARB_MST_SLAVE_EN_VAL_SHIFT                           0
+#define TPC0_QM_ARB_MST_SLAVE_EN_VAL_MASK                            0xFFFFFFFF
+
+/* TPC0_QM_ARB_MST_QUIET_PER */
+#define TPC0_QM_ARB_MST_QUIET_PER_VAL_SHIFT                          0
+#define TPC0_QM_ARB_MST_QUIET_PER_VAL_MASK                           0xFFFFFFFF
+
+/* TPC0_QM_ARB_SLV_CHOISE_WDT */
+#define TPC0_QM_ARB_SLV_CHOISE_WDT_VAL_SHIFT                         0
+#define TPC0_QM_ARB_SLV_CHOISE_WDT_VAL_MASK                          0xFFFFFFFF
+
+/* TPC0_QM_ARB_SLV_ID */
+#define TPC0_QM_ARB_SLV_ID_VAL_SHIFT                                 0
+#define TPC0_QM_ARB_SLV_ID_VAL_MASK                                  0x1F
+
+/* TPC0_QM_ARB_MSG_MAX_INFLIGHT */
+#define TPC0_QM_ARB_MSG_MAX_INFLIGHT_VAL_SHIFT                       0
+#define TPC0_QM_ARB_MSG_MAX_INFLIGHT_VAL_MASK                        0x3F
+
+/* TPC0_QM_ARB_MSG_AWUSER_31_11 */
+#define TPC0_QM_ARB_MSG_AWUSER_31_11_VAL_SHIFT                       0
+#define TPC0_QM_ARB_MSG_AWUSER_31_11_VAL_MASK                        0x1FFFFF
+
+/* TPC0_QM_ARB_MSG_AWUSER_SEC_PROP */
+#define TPC0_QM_ARB_MSG_AWUSER_SEC_PROP_ASID_SHIFT                   0
+#define TPC0_QM_ARB_MSG_AWUSER_SEC_PROP_ASID_MASK                    0x3FF
+#define TPC0_QM_ARB_MSG_AWUSER_SEC_PROP_MMBP_SHIFT                   10
+#define TPC0_QM_ARB_MSG_AWUSER_SEC_PROP_MMBP_MASK                    0x400
+
+/* TPC0_QM_ARB_MSG_AWUSER_NON_SEC_PROP */
+#define TPC0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_ASID_SHIFT               0
+#define TPC0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_ASID_MASK                0x3FF
+#define TPC0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_MMBP_SHIFT               10
+#define TPC0_QM_ARB_MSG_AWUSER_NON_SEC_PROP_MMBP_MASK                0x400
+
+/* TPC0_QM_ARB_BASE_LO */
+#define TPC0_QM_ARB_BASE_LO_VAL_SHIFT                                0
+#define TPC0_QM_ARB_BASE_LO_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_ARB_BASE_HI */
+#define TPC0_QM_ARB_BASE_HI_VAL_SHIFT                                0
+#define TPC0_QM_ARB_BASE_HI_VAL_MASK                                 0xFFFFFFFF
+
+/* TPC0_QM_ARB_STATE_STS */
+#define TPC0_QM_ARB_STATE_STS_VAL_SHIFT                              0
+#define TPC0_QM_ARB_STATE_STS_VAL_MASK                               0xFFFFFFFF
+
+/* TPC0_QM_ARB_CHOISE_FULLNESS_STS */
+#define TPC0_QM_ARB_CHOISE_FULLNESS_STS_VAL_SHIFT                    0
+#define TPC0_QM_ARB_CHOISE_FULLNESS_STS_VAL_MASK                     0x7F
+
+/* TPC0_QM_ARB_MSG_STS */
+#define TPC0_QM_ARB_MSG_STS_FULL_SHIFT                               0
+#define TPC0_QM_ARB_MSG_STS_FULL_MASK                                0x1
+#define TPC0_QM_ARB_MSG_STS_NO_INFLIGHT_SHIFT                        1
+#define TPC0_QM_ARB_MSG_STS_NO_INFLIGHT_MASK                         0x2
+
+/* TPC0_QM_ARB_SLV_CHOISE_Q_HEAD */
+#define TPC0_QM_ARB_SLV_CHOISE_Q_HEAD_VAL_SHIFT                      0
+#define TPC0_QM_ARB_SLV_CHOISE_Q_HEAD_VAL_MASK                       0x3
+
+/* TPC0_QM_ARB_ERR_CAUSE */
+#define TPC0_QM_ARB_ERR_CAUSE_CHOISE_OVF_SHIFT                       0
+#define TPC0_QM_ARB_ERR_CAUSE_CHOISE_OVF_MASK                        0x1
+#define TPC0_QM_ARB_ERR_CAUSE_CHOISE_WDT_SHIFT                       1
+#define TPC0_QM_ARB_ERR_CAUSE_CHOISE_WDT_MASK                        0x2
+#define TPC0_QM_ARB_ERR_CAUSE_AXI_LBW_ERR_SHIFT                      2
+#define TPC0_QM_ARB_ERR_CAUSE_AXI_LBW_ERR_MASK                       0x4
+
+/* TPC0_QM_ARB_ERR_MSG_EN */
+#define TPC0_QM_ARB_ERR_MSG_EN_CHOISE_OVF_SHIFT                      0
+#define TPC0_QM_ARB_ERR_MSG_EN_CHOISE_OVF_MASK                       0x1
+#define TPC0_QM_ARB_ERR_MSG_EN_CHOISE_WDT_SHIFT                      1
+#define TPC0_QM_ARB_ERR_MSG_EN_CHOISE_WDT_MASK                       0x2
+#define TPC0_QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_SHIFT                     2
+#define TPC0_QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_MASK                      0x4
+
+/* TPC0_QM_ARB_ERR_STS_DRP */
+#define TPC0_QM_ARB_ERR_STS_DRP_VAL_SHIFT                            0
+#define TPC0_QM_ARB_ERR_STS_DRP_VAL_MASK                             0x3
+
+/* TPC0_QM_ARB_MST_CRED_STS */
+#define TPC0_QM_ARB_MST_CRED_STS_VAL_SHIFT                           0
+#define TPC0_QM_ARB_MST_CRED_STS_VAL_MASK                            0x7F
+
+/* TPC0_QM_CGM_CFG */
+#define TPC0_QM_CGM_CFG_IDLE_TH_SHIFT                                0
+#define TPC0_QM_CGM_CFG_IDLE_TH_MASK                                 0xFFF
+#define TPC0_QM_CGM_CFG_G2F_TH_SHIFT                                 16
+#define TPC0_QM_CGM_CFG_G2F_TH_MASK                                  0xFF0000
+#define TPC0_QM_CGM_CFG_CP_IDLE_MASK_SHIFT                           24
+#define TPC0_QM_CGM_CFG_CP_IDLE_MASK_MASK                            0x1F000000
+#define TPC0_QM_CGM_CFG_EN_SHIFT                                     31
+#define TPC0_QM_CGM_CFG_EN_MASK                                      0x80000000
+
+/* TPC0_QM_CGM_STS */
+#define TPC0_QM_CGM_STS_ST_SHIFT                                     0
+#define TPC0_QM_CGM_STS_ST_MASK                                      0x3
+#define TPC0_QM_CGM_STS_CG_SHIFT                                     4
+#define TPC0_QM_CGM_STS_CG_MASK                                      0x10
+#define TPC0_QM_CGM_STS_AGENT_IDLE_SHIFT                             8
+#define TPC0_QM_CGM_STS_AGENT_IDLE_MASK                              0x100
+#define TPC0_QM_CGM_STS_AXI_IDLE_SHIFT                               9
+#define TPC0_QM_CGM_STS_AXI_IDLE_MASK                                0x200
+#define TPC0_QM_CGM_STS_CP_IDLE_SHIFT                                10
+#define TPC0_QM_CGM_STS_CP_IDLE_MASK                                 0x400
+
+/* TPC0_QM_CGM_CFG1 */
+#define TPC0_QM_CGM_CFG1_MASK_TH_SHIFT                               0
+#define TPC0_QM_CGM_CFG1_MASK_TH_MASK                                0xFF
+
+/* TPC0_QM_LOCAL_RANGE_BASE */
+#define TPC0_QM_LOCAL_RANGE_BASE_VAL_SHIFT                           0
+#define TPC0_QM_LOCAL_RANGE_BASE_VAL_MASK                            0xFFFF
+
+/* TPC0_QM_LOCAL_RANGE_SIZE */
+#define TPC0_QM_LOCAL_RANGE_SIZE_VAL_SHIFT                           0
+#define TPC0_QM_LOCAL_RANGE_SIZE_VAL_MASK                            0xFFFF
+
+/* TPC0_QM_CSMR_STRICT_PRIO_CFG */
+#define TPC0_QM_CSMR_STRICT_PRIO_CFG_TYPE_SHIFT                      0
+#define TPC0_QM_CSMR_STRICT_PRIO_CFG_TYPE_MASK                       0x1
+
+/* TPC0_QM_HBW_RD_RATE_LIM_CFG_1 */
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_1_TOUT_SHIFT                     0
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_1_TOUT_MASK                      0xFF
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_1_EN_SHIFT                       31
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_1_EN_MASK                        0x80000000
+
+/* TPC0_QM_LBW_WR_RATE_LIM_CFG_0 */
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                0
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_0_RST_TOKEN_MASK                 0xFF
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_0_SAT_SHIFT                      16
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_0_SAT_MASK                       0xFF0000
+
+/* TPC0_QM_LBW_WR_RATE_LIM_CFG_1 */
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_1_TOUT_SHIFT                     0
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_1_TOUT_MASK                      0xFF
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_1_EN_SHIFT                       31
+#define TPC0_QM_LBW_WR_RATE_LIM_CFG_1_EN_MASK                        0x80000000
+
+/* TPC0_QM_HBW_RD_RATE_LIM_CFG_0 */
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_0_RST_TOKEN_SHIFT                0
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_0_RST_TOKEN_MASK                 0xFF
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_0_SAT_SHIFT                      16
+#define TPC0_QM_HBW_RD_RATE_LIM_CFG_0_SAT_MASK                       0xFF0000
+
+/* TPC0_QM_GLBL_AXCACHE */
+#define TPC0_QM_GLBL_AXCACHE_AR_SHIFT                                0
+#define TPC0_QM_GLBL_AXCACHE_AR_MASK                                 0xF
+#define TPC0_QM_GLBL_AXCACHE_AW_SHIFT                                16
+#define TPC0_QM_GLBL_AXCACHE_AW_MASK                                 0xF0000
+
+/* TPC0_QM_IND_GW_APB_CFG */
+#define TPC0_QM_IND_GW_APB_CFG_ADDR_SHIFT                            0
+#define TPC0_QM_IND_GW_APB_CFG_ADDR_MASK                             0x7FFFFFFF
+#define TPC0_QM_IND_GW_APB_CFG_CMD_SHIFT                             31
+#define TPC0_QM_IND_GW_APB_CFG_CMD_MASK                              0x80000000
+
+/* TPC0_QM_IND_GW_APB_WDATA */
+#define TPC0_QM_IND_GW_APB_WDATA_VAL_SHIFT                           0
+#define TPC0_QM_IND_GW_APB_WDATA_VAL_MASK                            0xFFFFFFFF
+
+/* TPC0_QM_IND_GW_APB_RDATA */
+#define TPC0_QM_IND_GW_APB_RDATA_VAL_SHIFT                           0
+#define TPC0_QM_IND_GW_APB_RDATA_VAL_MASK                            0xFFFFFFFF
+
+/* TPC0_QM_IND_GW_APB_STATUS */
+#define TPC0_QM_IND_GW_APB_STATUS_RDY_SHIFT                          0
+#define TPC0_QM_IND_GW_APB_STATUS_RDY_MASK                           0x1
+#define TPC0_QM_IND_GW_APB_STATUS_ERR_SHIFT                          1
+#define TPC0_QM_IND_GW_APB_STATUS_ERR_MASK                           0x2
+
+/* TPC0_QM_GLBL_ERR_ADDR_LO */
+#define TPC0_QM_GLBL_ERR_ADDR_LO_VAL_SHIFT                           0
+#define TPC0_QM_GLBL_ERR_ADDR_LO_VAL_MASK                            0xFFFFFFFF
+
+/* TPC0_QM_GLBL_ERR_ADDR_HI */
+#define TPC0_QM_GLBL_ERR_ADDR_HI_VAL_SHIFT                           0
+#define TPC0_QM_GLBL_ERR_ADDR_HI_VAL_MASK                            0xFFFFFFFF
+
+/* TPC0_QM_GLBL_ERR_WDATA */
+#define TPC0_QM_GLBL_ERR_WDATA_VAL_SHIFT                             0
+#define TPC0_QM_GLBL_ERR_WDATA_VAL_MASK                              0xFFFFFFFF
+
+/* TPC0_QM_GLBL_MEM_INIT_BUSY */
+#define TPC0_QM_GLBL_MEM_INIT_BUSY_RBUF_SHIFT                        0
+#define TPC0_QM_GLBL_MEM_INIT_BUSY_RBUF_MASK                         0xF
+
+#endif /* ASIC_REG_TPC0_QM_MASKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_regs.h
new file mode 100644
index 000000000000..f9e310ab6df2
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc0_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC0_QM_REGS_H_
+#define ASIC_REG_TPC0_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC0_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC0_QM_GLBL_CFG0                                          0xE08000
+
+#define mmTPC0_QM_GLBL_CFG1                                          0xE08004
+
+#define mmTPC0_QM_GLBL_PROT                                          0xE08008
+
+#define mmTPC0_QM_GLBL_ERR_CFG                                       0xE0800C
+
+#define mmTPC0_QM_GLBL_SECURE_PROPS_0                                0xE08010
+
+#define mmTPC0_QM_GLBL_SECURE_PROPS_1                                0xE08014
+
+#define mmTPC0_QM_GLBL_SECURE_PROPS_2                                0xE08018
+
+#define mmTPC0_QM_GLBL_SECURE_PROPS_3                                0xE0801C
+
+#define mmTPC0_QM_GLBL_SECURE_PROPS_4                                0xE08020
+
+#define mmTPC0_QM_GLBL_NON_SECURE_PROPS_0                            0xE08024
+
+#define mmTPC0_QM_GLBL_NON_SECURE_PROPS_1                            0xE08028
+
+#define mmTPC0_QM_GLBL_NON_SECURE_PROPS_2                            0xE0802C
+
+#define mmTPC0_QM_GLBL_NON_SECURE_PROPS_3                            0xE08030
+
+#define mmTPC0_QM_GLBL_NON_SECURE_PROPS_4                            0xE08034
+
+#define mmTPC0_QM_GLBL_STS0                                          0xE08038
+
+#define mmTPC0_QM_GLBL_STS1_0                                        0xE08040
+
+#define mmTPC0_QM_GLBL_STS1_1                                        0xE08044
+
+#define mmTPC0_QM_GLBL_STS1_2                                        0xE08048
+
+#define mmTPC0_QM_GLBL_STS1_3                                        0xE0804C
+
+#define mmTPC0_QM_GLBL_STS1_4                                        0xE08050
+
+#define mmTPC0_QM_GLBL_MSG_EN_0                                      0xE08054
+
+#define mmTPC0_QM_GLBL_MSG_EN_1                                      0xE08058
+
+#define mmTPC0_QM_GLBL_MSG_EN_2                                      0xE0805C
+
+#define mmTPC0_QM_GLBL_MSG_EN_3                                      0xE08060
+
+#define mmTPC0_QM_GLBL_MSG_EN_4                                      0xE08068
+
+#define mmTPC0_QM_PQ_BASE_LO_0                                       0xE08070
+
+#define mmTPC0_QM_PQ_BASE_LO_1                                       0xE08074
+
+#define mmTPC0_QM_PQ_BASE_LO_2                                       0xE08078
+
+#define mmTPC0_QM_PQ_BASE_LO_3                                       0xE0807C
+
+#define mmTPC0_QM_PQ_BASE_HI_0                                       0xE08080
+
+#define mmTPC0_QM_PQ_BASE_HI_1                                       0xE08084
+
+#define mmTPC0_QM_PQ_BASE_HI_2                                       0xE08088
+
+#define mmTPC0_QM_PQ_BASE_HI_3                                       0xE0808C
+
+#define mmTPC0_QM_PQ_SIZE_0                                          0xE08090
+
+#define mmTPC0_QM_PQ_SIZE_1                                          0xE08094
+
+#define mmTPC0_QM_PQ_SIZE_2                                          0xE08098
+
+#define mmTPC0_QM_PQ_SIZE_3                                          0xE0809C
+
+#define mmTPC0_QM_PQ_PI_0                                            0xE080A0
+
+#define mmTPC0_QM_PQ_PI_1                                            0xE080A4
+
+#define mmTPC0_QM_PQ_PI_2                                            0xE080A8
+
+#define mmTPC0_QM_PQ_PI_3                                            0xE080AC
+
+#define mmTPC0_QM_PQ_CI_0                                            0xE080B0
+
+#define mmTPC0_QM_PQ_CI_1                                            0xE080B4
+
+#define mmTPC0_QM_PQ_CI_2                                            0xE080B8
+
+#define mmTPC0_QM_PQ_CI_3                                            0xE080BC
+
+#define mmTPC0_QM_PQ_CFG0_0                                          0xE080C0
+
+#define mmTPC0_QM_PQ_CFG0_1                                          0xE080C4
+
+#define mmTPC0_QM_PQ_CFG0_2                                          0xE080C8
+
+#define mmTPC0_QM_PQ_CFG0_3                                          0xE080CC
+
+#define mmTPC0_QM_PQ_CFG1_0                                          0xE080D0
+
+#define mmTPC0_QM_PQ_CFG1_1                                          0xE080D4
+
+#define mmTPC0_QM_PQ_CFG1_2                                          0xE080D8
+
+#define mmTPC0_QM_PQ_CFG1_3                                          0xE080DC
+
+#define mmTPC0_QM_PQ_ARUSER_31_11_0                                  0xE080E0
+
+#define mmTPC0_QM_PQ_ARUSER_31_11_1                                  0xE080E4
+
+#define mmTPC0_QM_PQ_ARUSER_31_11_2                                  0xE080E8
+
+#define mmTPC0_QM_PQ_ARUSER_31_11_3                                  0xE080EC
+
+#define mmTPC0_QM_PQ_STS0_0                                          0xE080F0
+
+#define mmTPC0_QM_PQ_STS0_1                                          0xE080F4
+
+#define mmTPC0_QM_PQ_STS0_2                                          0xE080F8
+
+#define mmTPC0_QM_PQ_STS0_3                                          0xE080FC
+
+#define mmTPC0_QM_PQ_STS1_0                                          0xE08100
+
+#define mmTPC0_QM_PQ_STS1_1                                          0xE08104
+
+#define mmTPC0_QM_PQ_STS1_2                                          0xE08108
+
+#define mmTPC0_QM_PQ_STS1_3                                          0xE0810C
+
+#define mmTPC0_QM_CQ_CFG0_0                                          0xE08110
+
+#define mmTPC0_QM_CQ_CFG0_1                                          0xE08114
+
+#define mmTPC0_QM_CQ_CFG0_2                                          0xE08118
+
+#define mmTPC0_QM_CQ_CFG0_3                                          0xE0811C
+
+#define mmTPC0_QM_CQ_CFG0_4                                          0xE08120
+
+#define mmTPC0_QM_CQ_CFG1_0                                          0xE08124
+
+#define mmTPC0_QM_CQ_CFG1_1                                          0xE08128
+
+#define mmTPC0_QM_CQ_CFG1_2                                          0xE0812C
+
+#define mmTPC0_QM_CQ_CFG1_3                                          0xE08130
+
+#define mmTPC0_QM_CQ_CFG1_4                                          0xE08134
+
+#define mmTPC0_QM_CQ_ARUSER_31_11_0                                  0xE08138
+
+#define mmTPC0_QM_CQ_ARUSER_31_11_1                                  0xE0813C
+
+#define mmTPC0_QM_CQ_ARUSER_31_11_2                                  0xE08140
+
+#define mmTPC0_QM_CQ_ARUSER_31_11_3                                  0xE08144
+
+#define mmTPC0_QM_CQ_ARUSER_31_11_4                                  0xE08148
+
+#define mmTPC0_QM_CQ_STS0_0                                          0xE0814C
+
+#define mmTPC0_QM_CQ_STS0_1                                          0xE08150
+
+#define mmTPC0_QM_CQ_STS0_2                                          0xE08154
+
+#define mmTPC0_QM_CQ_STS0_3                                          0xE08158
+
+#define mmTPC0_QM_CQ_STS0_4                                          0xE0815C
+
+#define mmTPC0_QM_CQ_STS1_0                                          0xE08160
+
+#define mmTPC0_QM_CQ_STS1_1                                          0xE08164
+
+#define mmTPC0_QM_CQ_STS1_2                                          0xE08168
+
+#define mmTPC0_QM_CQ_STS1_3                                          0xE0816C
+
+#define mmTPC0_QM_CQ_STS1_4                                          0xE08170
+
+#define mmTPC0_QM_CQ_PTR_LO_0                                        0xE08174
+
+#define mmTPC0_QM_CQ_PTR_HI_0                                        0xE08178
+
+#define mmTPC0_QM_CQ_TSIZE_0                                         0xE0817C
+
+#define mmTPC0_QM_CQ_CTL_0                                           0xE08180
+
+#define mmTPC0_QM_CQ_PTR_LO_1                                        0xE08184
+
+#define mmTPC0_QM_CQ_PTR_HI_1                                        0xE08188
+
+#define mmTPC0_QM_CQ_TSIZE_1                                         0xE0818C
+
+#define mmTPC0_QM_CQ_CTL_1                                           0xE08190
+
+#define mmTPC0_QM_CQ_PTR_LO_2                                        0xE08194
+
+#define mmTPC0_QM_CQ_PTR_HI_2                                        0xE08198
+
+#define mmTPC0_QM_CQ_TSIZE_2                                         0xE0819C
+
+#define mmTPC0_QM_CQ_CTL_2                                           0xE081A0
+
+#define mmTPC0_QM_CQ_PTR_LO_3                                        0xE081A4
+
+#define mmTPC0_QM_CQ_PTR_HI_3                                        0xE081A8
+
+#define mmTPC0_QM_CQ_TSIZE_3                                         0xE081AC
+
+#define mmTPC0_QM_CQ_CTL_3                                           0xE081B0
+
+#define mmTPC0_QM_CQ_PTR_LO_4                                        0xE081B4
+
+#define mmTPC0_QM_CQ_PTR_HI_4                                        0xE081B8
+
+#define mmTPC0_QM_CQ_TSIZE_4                                         0xE081BC
+
+#define mmTPC0_QM_CQ_CTL_4                                           0xE081C0
+
+#define mmTPC0_QM_CQ_PTR_LO_STS_0                                    0xE081C4
+
+#define mmTPC0_QM_CQ_PTR_LO_STS_1                                    0xE081C8
+
+#define mmTPC0_QM_CQ_PTR_LO_STS_2                                    0xE081CC
+
+#define mmTPC0_QM_CQ_PTR_LO_STS_3                                    0xE081D0
+
+#define mmTPC0_QM_CQ_PTR_LO_STS_4                                    0xE081D4
+
+#define mmTPC0_QM_CQ_PTR_HI_STS_0                                    0xE081D8
+
+#define mmTPC0_QM_CQ_PTR_HI_STS_1                                    0xE081DC
+
+#define mmTPC0_QM_CQ_PTR_HI_STS_2                                    0xE081E0
+
+#define mmTPC0_QM_CQ_PTR_HI_STS_3                                    0xE081E4
+
+#define mmTPC0_QM_CQ_PTR_HI_STS_4                                    0xE081E8
+
+#define mmTPC0_QM_CQ_TSIZE_STS_0                                     0xE081EC
+
+#define mmTPC0_QM_CQ_TSIZE_STS_1                                     0xE081F0
+
+#define mmTPC0_QM_CQ_TSIZE_STS_2                                     0xE081F4
+
+#define mmTPC0_QM_CQ_TSIZE_STS_3                                     0xE081F8
+
+#define mmTPC0_QM_CQ_TSIZE_STS_4                                     0xE081FC
+
+#define mmTPC0_QM_CQ_CTL_STS_0                                       0xE08200
+
+#define mmTPC0_QM_CQ_CTL_STS_1                                       0xE08204
+
+#define mmTPC0_QM_CQ_CTL_STS_2                                       0xE08208
+
+#define mmTPC0_QM_CQ_CTL_STS_3                                       0xE0820C
+
+#define mmTPC0_QM_CQ_CTL_STS_4                                       0xE08210
+
+#define mmTPC0_QM_CQ_IFIFO_CNT_0                                     0xE08214
+
+#define mmTPC0_QM_CQ_IFIFO_CNT_1                                     0xE08218
+
+#define mmTPC0_QM_CQ_IFIFO_CNT_2                                     0xE0821C
+
+#define mmTPC0_QM_CQ_IFIFO_CNT_3                                     0xE08220
+
+#define mmTPC0_QM_CQ_IFIFO_CNT_4                                     0xE08224
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0                             0xE08228
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_1                             0xE0822C
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_2                             0xE08230
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_3                             0xE08234
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_4                             0xE08238
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0                             0xE0823C
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_1                             0xE08240
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_2                             0xE08244
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_3                             0xE08248
+
+#define mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_4                             0xE0824C
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0                             0xE08250
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_1                             0xE08254
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_2                             0xE08258
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_3                             0xE0825C
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_4                             0xE08260
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0                             0xE08264
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_1                             0xE08268
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_2                             0xE0826C
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_3                             0xE08270
+
+#define mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_4                             0xE08274
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_0                             0xE08278
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_1                             0xE0827C
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_2                             0xE08280
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_3                             0xE08284
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_4                             0xE08288
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_0                             0xE0828C
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_1                             0xE08290
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_2                             0xE08294
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_3                             0xE08298
+
+#define mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_4                             0xE0829C
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_0                             0xE082A0
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_1                             0xE082A4
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_2                             0xE082A8
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_3                             0xE082AC
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_4                             0xE082B0
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_0                             0xE082B4
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_1                             0xE082B8
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_2                             0xE082BC
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_3                             0xE082C0
+
+#define mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_4                             0xE082C4
+
+#define mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0                             0xE082C8
+
+#define mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_1                             0xE082CC
+
+#define mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_2                             0xE082D0
+
+#define mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_3                             0xE082D4
+
+#define mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_4                             0xE082D8
+
+#define mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xE082E0
+
+#define mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xE082E4
+
+#define mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xE082E8
+
+#define mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xE082EC
+
+#define mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xE082F0
+
+#define mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xE082F4
+
+#define mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xE082F8
+
+#define mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xE082FC
+
+#define mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xE08300
+
+#define mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xE08304
+
+#define mmTPC0_QM_CP_FENCE0_RDATA_0                                  0xE08308
+
+#define mmTPC0_QM_CP_FENCE0_RDATA_1                                  0xE0830C
+
+#define mmTPC0_QM_CP_FENCE0_RDATA_2                                  0xE08310
+
+#define mmTPC0_QM_CP_FENCE0_RDATA_3                                  0xE08314
+
+#define mmTPC0_QM_CP_FENCE0_RDATA_4                                  0xE08318
+
+#define mmTPC0_QM_CP_FENCE1_RDATA_0                                  0xE0831C
+
+#define mmTPC0_QM_CP_FENCE1_RDATA_1                                  0xE08320
+
+#define mmTPC0_QM_CP_FENCE1_RDATA_2                                  0xE08324
+
+#define mmTPC0_QM_CP_FENCE1_RDATA_3                                  0xE08328
+
+#define mmTPC0_QM_CP_FENCE1_RDATA_4                                  0xE0832C
+
+#define mmTPC0_QM_CP_FENCE2_RDATA_0                                  0xE08330
+
+#define mmTPC0_QM_CP_FENCE2_RDATA_1                                  0xE08334
+
+#define mmTPC0_QM_CP_FENCE2_RDATA_2                                  0xE08338
+
+#define mmTPC0_QM_CP_FENCE2_RDATA_3                                  0xE0833C
+
+#define mmTPC0_QM_CP_FENCE2_RDATA_4                                  0xE08340
+
+#define mmTPC0_QM_CP_FENCE3_RDATA_0                                  0xE08344
+
+#define mmTPC0_QM_CP_FENCE3_RDATA_1                                  0xE08348
+
+#define mmTPC0_QM_CP_FENCE3_RDATA_2                                  0xE0834C
+
+#define mmTPC0_QM_CP_FENCE3_RDATA_3                                  0xE08350
+
+#define mmTPC0_QM_CP_FENCE3_RDATA_4                                  0xE08354
+
+#define mmTPC0_QM_CP_FENCE0_CNT_0                                    0xE08358
+
+#define mmTPC0_QM_CP_FENCE0_CNT_1                                    0xE0835C
+
+#define mmTPC0_QM_CP_FENCE0_CNT_2                                    0xE08360
+
+#define mmTPC0_QM_CP_FENCE0_CNT_3                                    0xE08364
+
+#define mmTPC0_QM_CP_FENCE0_CNT_4                                    0xE08368
+
+#define mmTPC0_QM_CP_FENCE1_CNT_0                                    0xE0836C
+
+#define mmTPC0_QM_CP_FENCE1_CNT_1                                    0xE08370
+
+#define mmTPC0_QM_CP_FENCE1_CNT_2                                    0xE08374
+
+#define mmTPC0_QM_CP_FENCE1_CNT_3                                    0xE08378
+
+#define mmTPC0_QM_CP_FENCE1_CNT_4                                    0xE0837C
+
+#define mmTPC0_QM_CP_FENCE2_CNT_0                                    0xE08380
+
+#define mmTPC0_QM_CP_FENCE2_CNT_1                                    0xE08384
+
+#define mmTPC0_QM_CP_FENCE2_CNT_2                                    0xE08388
+
+#define mmTPC0_QM_CP_FENCE2_CNT_3                                    0xE0838C
+
+#define mmTPC0_QM_CP_FENCE2_CNT_4                                    0xE08390
+
+#define mmTPC0_QM_CP_FENCE3_CNT_0                                    0xE08394
+
+#define mmTPC0_QM_CP_FENCE3_CNT_1                                    0xE08398
+
+#define mmTPC0_QM_CP_FENCE3_CNT_2                                    0xE0839C
+
+#define mmTPC0_QM_CP_FENCE3_CNT_3                                    0xE083A0
+
+#define mmTPC0_QM_CP_FENCE3_CNT_4                                    0xE083A4
+
+#define mmTPC0_QM_CP_STS_0                                           0xE083A8
+
+#define mmTPC0_QM_CP_STS_1                                           0xE083AC
+
+#define mmTPC0_QM_CP_STS_2                                           0xE083B0
+
+#define mmTPC0_QM_CP_STS_3                                           0xE083B4
+
+#define mmTPC0_QM_CP_STS_4                                           0xE083B8
+
+#define mmTPC0_QM_CP_CURRENT_INST_LO_0                               0xE083BC
+
+#define mmTPC0_QM_CP_CURRENT_INST_LO_1                               0xE083C0
+
+#define mmTPC0_QM_CP_CURRENT_INST_LO_2                               0xE083C4
+
+#define mmTPC0_QM_CP_CURRENT_INST_LO_3                               0xE083C8
+
+#define mmTPC0_QM_CP_CURRENT_INST_LO_4                               0xE083CC
+
+#define mmTPC0_QM_CP_CURRENT_INST_HI_0                               0xE083D0
+
+#define mmTPC0_QM_CP_CURRENT_INST_HI_1                               0xE083D4
+
+#define mmTPC0_QM_CP_CURRENT_INST_HI_2                               0xE083D8
+
+#define mmTPC0_QM_CP_CURRENT_INST_HI_3                               0xE083DC
+
+#define mmTPC0_QM_CP_CURRENT_INST_HI_4                               0xE083E0
+
+#define mmTPC0_QM_CP_BARRIER_CFG_0                                   0xE083F4
+
+#define mmTPC0_QM_CP_BARRIER_CFG_1                                   0xE083F8
+
+#define mmTPC0_QM_CP_BARRIER_CFG_2                                   0xE083FC
+
+#define mmTPC0_QM_CP_BARRIER_CFG_3                                   0xE08400
+
+#define mmTPC0_QM_CP_BARRIER_CFG_4                                   0xE08404
+
+#define mmTPC0_QM_CP_DBG_0_0                                         0xE08408
+
+#define mmTPC0_QM_CP_DBG_0_1                                         0xE0840C
+
+#define mmTPC0_QM_CP_DBG_0_2                                         0xE08410
+
+#define mmTPC0_QM_CP_DBG_0_3                                         0xE08414
+
+#define mmTPC0_QM_CP_DBG_0_4                                         0xE08418
+
+#define mmTPC0_QM_CP_ARUSER_31_11_0                                  0xE0841C
+
+#define mmTPC0_QM_CP_ARUSER_31_11_1                                  0xE08420
+
+#define mmTPC0_QM_CP_ARUSER_31_11_2                                  0xE08424
+
+#define mmTPC0_QM_CP_ARUSER_31_11_3                                  0xE08428
+
+#define mmTPC0_QM_CP_ARUSER_31_11_4                                  0xE0842C
+
+#define mmTPC0_QM_CP_AWUSER_31_11_0                                  0xE08430
+
+#define mmTPC0_QM_CP_AWUSER_31_11_1                                  0xE08434
+
+#define mmTPC0_QM_CP_AWUSER_31_11_2                                  0xE08438
+
+#define mmTPC0_QM_CP_AWUSER_31_11_3                                  0xE0843C
+
+#define mmTPC0_QM_CP_AWUSER_31_11_4                                  0xE08440
+
+#define mmTPC0_QM_ARB_CFG_0                                          0xE08A00
+
+#define mmTPC0_QM_ARB_CHOISE_Q_PUSH                                  0xE08A04
+
+#define mmTPC0_QM_ARB_WRR_WEIGHT_0                                   0xE08A08
+
+#define mmTPC0_QM_ARB_WRR_WEIGHT_1                                   0xE08A0C
+
+#define mmTPC0_QM_ARB_WRR_WEIGHT_2                                   0xE08A10
+
+#define mmTPC0_QM_ARB_WRR_WEIGHT_3                                   0xE08A14
+
+#define mmTPC0_QM_ARB_CFG_1                                          0xE08A18
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_0                               0xE08A20
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_1                               0xE08A24
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_2                               0xE08A28
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_3                               0xE08A2C
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_4                               0xE08A30
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_5                               0xE08A34
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_6                               0xE08A38
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_7                               0xE08A3C
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_8                               0xE08A40
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_9                               0xE08A44
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_10                              0xE08A48
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_11                              0xE08A4C
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_12                              0xE08A50
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_13                              0xE08A54
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_14                              0xE08A58
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_15                              0xE08A5C
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_16                              0xE08A60
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_17                              0xE08A64
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_18                              0xE08A68
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_19                              0xE08A6C
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_20                              0xE08A70
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_21                              0xE08A74
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_22                              0xE08A78
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_23                              0xE08A7C
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_24                              0xE08A80
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_25                              0xE08A84
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_26                              0xE08A88
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_27                              0xE08A8C
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_28                              0xE08A90
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_29                              0xE08A94
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_30                              0xE08A98
+
+#define mmTPC0_QM_ARB_MST_AVAIL_CRED_31                              0xE08A9C
+
+#define mmTPC0_QM_ARB_MST_CRED_INC                                   0xE08AA0
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xE08AA4
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xE08AA8
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xE08AAC
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xE08AB0
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xE08AB4
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xE08AB8
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xE08ABC
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xE08AC0
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xE08AC4
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xE08AC8
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xE08ACC
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xE08AD0
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xE08AD4
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xE08AD8
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xE08ADC
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xE08AE0
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xE08AE4
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xE08AE8
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xE08AEC
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xE08AF0
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xE08AF4
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xE08AF8
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xE08AFC
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xE08B00
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xE08B04
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xE08B08
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xE08B0C
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xE08B10
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xE08B14
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xE08B18
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xE08B1C
+
+#define mmTPC0_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xE08B20
+
+#define mmTPC0_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xE08B28
+
+#define mmTPC0_QM_ARB_MST_SLAVE_EN                                   0xE08B2C
+
+#define mmTPC0_QM_ARB_MST_QUIET_PER                                  0xE08B34
+
+#define mmTPC0_QM_ARB_SLV_CHOISE_WDT                                 0xE08B38
+
+#define mmTPC0_QM_ARB_SLV_ID                                         0xE08B3C
+
+#define mmTPC0_QM_ARB_MSG_MAX_INFLIGHT                               0xE08B44
+
+#define mmTPC0_QM_ARB_MSG_AWUSER_31_11                               0xE08B48
+
+#define mmTPC0_QM_ARB_MSG_AWUSER_SEC_PROP                            0xE08B4C
+
+#define mmTPC0_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xE08B50
+
+#define mmTPC0_QM_ARB_BASE_LO                                        0xE08B54
+
+#define mmTPC0_QM_ARB_BASE_HI                                        0xE08B58
+
+#define mmTPC0_QM_ARB_STATE_STS                                      0xE08B80
+
+#define mmTPC0_QM_ARB_CHOISE_FULLNESS_STS                            0xE08B84
+
+#define mmTPC0_QM_ARB_MSG_STS                                        0xE08B88
+
+#define mmTPC0_QM_ARB_SLV_CHOISE_Q_HEAD                              0xE08B8C
+
+#define mmTPC0_QM_ARB_ERR_CAUSE                                      0xE08B9C
+
+#define mmTPC0_QM_ARB_ERR_MSG_EN                                     0xE08BA0
+
+#define mmTPC0_QM_ARB_ERR_STS_DRP                                    0xE08BA8
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_0                                 0xE08BB0
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_1                                 0xE08BB4
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_2                                 0xE08BB8
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_3                                 0xE08BBC
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_4                                 0xE08BC0
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_5                                 0xE08BC4
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_6                                 0xE08BC8
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_7                                 0xE08BCC
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_8                                 0xE08BD0
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_9                                 0xE08BD4
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_10                                0xE08BD8
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_11                                0xE08BDC
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_12                                0xE08BE0
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_13                                0xE08BE4
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_14                                0xE08BE8
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_15                                0xE08BEC
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_16                                0xE08BF0
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_17                                0xE08BF4
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_18                                0xE08BF8
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_19                                0xE08BFC
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_20                                0xE08C00
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_21                                0xE08C04
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_22                                0xE08C08
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_23                                0xE08C0C
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_24                                0xE08C10
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_25                                0xE08C14
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_26                                0xE08C18
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_27                                0xE08C1C
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_28                                0xE08C20
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_29                                0xE08C24
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_30                                0xE08C28
+
+#define mmTPC0_QM_ARB_MST_CRED_STS_31                                0xE08C2C
+
+#define mmTPC0_QM_CGM_CFG                                            0xE08C70
+
+#define mmTPC0_QM_CGM_STS                                            0xE08C74
+
+#define mmTPC0_QM_CGM_CFG1                                           0xE08C78
+
+#define mmTPC0_QM_LOCAL_RANGE_BASE                                   0xE08C80
+
+#define mmTPC0_QM_LOCAL_RANGE_SIZE                                   0xE08C84
+
+#define mmTPC0_QM_CSMR_STRICT_PRIO_CFG                               0xE08C90
+
+#define mmTPC0_QM_HBW_RD_RATE_LIM_CFG_1                              0xE08C94
+
+#define mmTPC0_QM_LBW_WR_RATE_LIM_CFG_0                              0xE08C98
+
+#define mmTPC0_QM_LBW_WR_RATE_LIM_CFG_1                              0xE08C9C
+
+#define mmTPC0_QM_HBW_RD_RATE_LIM_CFG_0                              0xE08CA0
+
+#define mmTPC0_QM_GLBL_AXCACHE                                       0xE08CA4
+
+#define mmTPC0_QM_IND_GW_APB_CFG                                     0xE08CB0
+
+#define mmTPC0_QM_IND_GW_APB_WDATA                                   0xE08CB4
+
+#define mmTPC0_QM_IND_GW_APB_RDATA                                   0xE08CB8
+
+#define mmTPC0_QM_IND_GW_APB_STATUS                                  0xE08CBC
+
+#define mmTPC0_QM_GLBL_ERR_ADDR_LO                                   0xE08CD0
+
+#define mmTPC0_QM_GLBL_ERR_ADDR_HI                                   0xE08CD4
+
+#define mmTPC0_QM_GLBL_ERR_WDATA                                     0xE08CD8
+
+#define mmTPC0_QM_GLBL_MEM_INIT_BUSY                                 0xE08D00
+
+#endif /* ASIC_REG_TPC0_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_cfg_regs.h
new file mode 100644
index 000000000000..6736c476d979
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC1_CFG_REGS_H_
+#define ASIC_REG_TPC1_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC1_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xE46400
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xE46404
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xE46408
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xE4640C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xE46410
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xE46414
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xE46418
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xE4641C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xE46420
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xE46424
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xE46428
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xE4642C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xE46430
+
+#define mmTPC1_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xE46434
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xE46438
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xE4643C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xE46440
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xE46444
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xE46448
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xE4644C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xE46450
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xE46454
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xE46458
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xE4645C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xE46460
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xE46464
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xE46468
+
+#define mmTPC1_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xE4646C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xE46470
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xE46474
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xE46478
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xE4647C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xE46480
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xE46484
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xE46488
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xE4648C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xE46490
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xE46494
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xE46498
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xE4649C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xE464A0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xE464A4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xE464A8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xE464AC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xE464B0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xE464B4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xE464B8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xE464BC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xE464C0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xE464C4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xE464C8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xE464CC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xE464D0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xE464D4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xE464D8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xE464DC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xE464E0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xE464E4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xE464E8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xE464EC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xE464F0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xE464F4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xE464F8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xE464FC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xE46500
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xE46504
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xE46508
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xE4650C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xE46510
+
+#define mmTPC1_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xE46514
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xE46518
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xE4651C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xE46520
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xE46524
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xE46528
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xE4652C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xE46530
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xE46534
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xE46538
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xE4653C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xE46540
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xE46544
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xE46548
+
+#define mmTPC1_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xE4654C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xE46550
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xE46554
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xE46558
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xE4655C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xE46560
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xE46564
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xE46568
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xE4656C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xE46570
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xE46574
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xE46578
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xE4657C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xE46580
+
+#define mmTPC1_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xE46584
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xE46588
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xE4658C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xE46590
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xE46594
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xE46598
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xE4659C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xE465A0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xE465A4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xE465A8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xE465AC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xE465B0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xE465B4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xE465B8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xE465BC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xE465C0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xE465C4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xE465C8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xE465CC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xE465D0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xE465D4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xE465D8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xE465DC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xE465E0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xE465E4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xE465E8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xE465EC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xE465F0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xE465F4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xE465F8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xE465FC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xE46600
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xE46604
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xE46608
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xE4660C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xE46610
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xE46614
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xE46618
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xE4661C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xE46620
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xE46624
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xE46628
+
+#define mmTPC1_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xE4662C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xE46630
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xE46634
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xE46638
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xE4663C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xE46640
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xE46644
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xE46648
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xE4664C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xE46650
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xE46654
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xE46658
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xE4665C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xE46660
+
+#define mmTPC1_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xE46664
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xE46668
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xE4666C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xE46670
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xE46674
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xE46678
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xE4667C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xE46680
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xE46684
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xE46688
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xE4668C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xE46690
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xE46694
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xE46698
+
+#define mmTPC1_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xE4669C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xE466A0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xE466A4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xE466A8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xE466AC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xE466B0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xE466B4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xE466B8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xE466BC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xE466C0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xE466C4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xE466C8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xE466CC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xE466D0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xE466D4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xE466D8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xE466DC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xE466E0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xE466E4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xE466E8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xE466EC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xE466F0
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xE466F4
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xE466F8
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xE466FC
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xE46700
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xE46704
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xE46708
+
+#define mmTPC1_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xE4670C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xE46710
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xE46714
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xE46718
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xE4671C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xE46720
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xE46724
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xE46728
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xE4672C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xE46730
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xE46734
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xE46738
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xE4673C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xE46740
+
+#define mmTPC1_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xE46744
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xE46748
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xE4674C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xE46750
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xE46754
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xE46758
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xE4675C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xE46760
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xE46764
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xE46768
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xE4676C
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xE46770
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xE46774
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xE46778
+
+#define mmTPC1_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xE4677C
+
+#define mmTPC1_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xE46780
+
+#define mmTPC1_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xE46784
+
+#define mmTPC1_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xE46788
+
+#define mmTPC1_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xE4678C
+
+#define mmTPC1_CFG_KERNEL_TID_BASE_DIM_0                             0xE46790
+
+#define mmTPC1_CFG_KERNEL_TID_SIZE_DIM_0                             0xE46794
+
+#define mmTPC1_CFG_KERNEL_TID_BASE_DIM_1                             0xE46798
+
+#define mmTPC1_CFG_KERNEL_TID_SIZE_DIM_1                             0xE4679C
+
+#define mmTPC1_CFG_KERNEL_TID_BASE_DIM_2                             0xE467A0
+
+#define mmTPC1_CFG_KERNEL_TID_SIZE_DIM_2                             0xE467A4
+
+#define mmTPC1_CFG_KERNEL_TID_BASE_DIM_3                             0xE467A8
+
+#define mmTPC1_CFG_KERNEL_TID_SIZE_DIM_3                             0xE467AC
+
+#define mmTPC1_CFG_KERNEL_TID_BASE_DIM_4                             0xE467B0
+
+#define mmTPC1_CFG_KERNEL_TID_SIZE_DIM_4                             0xE467B4
+
+#define mmTPC1_CFG_KERNEL_KERNEL_CONFIG                              0xE467B8
+
+#define mmTPC1_CFG_KERNEL_KERNEL_ID                                  0xE467BC
+
+#define mmTPC1_CFG_KERNEL_SRF_0                                      0xE467C0
+
+#define mmTPC1_CFG_KERNEL_SRF_1                                      0xE467C4
+
+#define mmTPC1_CFG_KERNEL_SRF_2                                      0xE467C8
+
+#define mmTPC1_CFG_KERNEL_SRF_3                                      0xE467CC
+
+#define mmTPC1_CFG_KERNEL_SRF_4                                      0xE467D0
+
+#define mmTPC1_CFG_KERNEL_SRF_5                                      0xE467D4
+
+#define mmTPC1_CFG_KERNEL_SRF_6                                      0xE467D8
+
+#define mmTPC1_CFG_KERNEL_SRF_7                                      0xE467DC
+
+#define mmTPC1_CFG_KERNEL_SRF_8                                      0xE467E0
+
+#define mmTPC1_CFG_KERNEL_SRF_9                                      0xE467E4
+
+#define mmTPC1_CFG_KERNEL_SRF_10                                     0xE467E8
+
+#define mmTPC1_CFG_KERNEL_SRF_11                                     0xE467EC
+
+#define mmTPC1_CFG_KERNEL_SRF_12                                     0xE467F0
+
+#define mmTPC1_CFG_KERNEL_SRF_13                                     0xE467F4
+
+#define mmTPC1_CFG_KERNEL_SRF_14                                     0xE467F8
+
+#define mmTPC1_CFG_KERNEL_SRF_15                                     0xE467FC
+
+#define mmTPC1_CFG_KERNEL_SRF_16                                     0xE46800
+
+#define mmTPC1_CFG_KERNEL_SRF_17                                     0xE46804
+
+#define mmTPC1_CFG_KERNEL_SRF_18                                     0xE46808
+
+#define mmTPC1_CFG_KERNEL_SRF_19                                     0xE4680C
+
+#define mmTPC1_CFG_KERNEL_SRF_20                                     0xE46810
+
+#define mmTPC1_CFG_KERNEL_SRF_21                                     0xE46814
+
+#define mmTPC1_CFG_KERNEL_SRF_22                                     0xE46818
+
+#define mmTPC1_CFG_KERNEL_SRF_23                                     0xE4681C
+
+#define mmTPC1_CFG_KERNEL_SRF_24                                     0xE46820
+
+#define mmTPC1_CFG_KERNEL_SRF_25                                     0xE46824
+
+#define mmTPC1_CFG_KERNEL_SRF_26                                     0xE46828
+
+#define mmTPC1_CFG_KERNEL_SRF_27                                     0xE4682C
+
+#define mmTPC1_CFG_KERNEL_SRF_28                                     0xE46830
+
+#define mmTPC1_CFG_KERNEL_SRF_29                                     0xE46834
+
+#define mmTPC1_CFG_KERNEL_SRF_30                                     0xE46838
+
+#define mmTPC1_CFG_KERNEL_SRF_31                                     0xE4683C
+
+#define mmTPC1_CFG_ROUND_CSR                                         0xE468FC
+
+#define mmTPC1_CFG_PROT                                              0xE46900
+
+#define mmTPC1_CFG_SEMAPHORE                                         0xE46908
+
+#define mmTPC1_CFG_VFLAGS                                            0xE4690C
+
+#define mmTPC1_CFG_SFLAGS                                            0xE46910
+
+#define mmTPC1_CFG_LFSR_POLYNOM                                      0xE46918
+
+#define mmTPC1_CFG_STATUS                                            0xE4691C
+
+#define mmTPC1_CFG_CFG_BASE_ADDRESS_HIGH                             0xE46920
+
+#define mmTPC1_CFG_CFG_SUBTRACT_VALUE                                0xE46924
+
+#define mmTPC1_CFG_SM_BASE_ADDRESS_HIGH                              0xE4692C
+
+#define mmTPC1_CFG_TPC_CMD                                           0xE46930
+
+#define mmTPC1_CFG_TPC_EXECUTE                                       0xE46938
+
+#define mmTPC1_CFG_TPC_STALL                                         0xE4693C
+
+#define mmTPC1_CFG_ICACHE_BASE_ADDERESS_LOW                          0xE46940
+
+#define mmTPC1_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xE46944
+
+#define mmTPC1_CFG_RD_RATE_LIMIT                                     0xE46948
+
+#define mmTPC1_CFG_WR_RATE_LIMIT                                     0xE46950
+
+#define mmTPC1_CFG_MSS_CONFIG                                        0xE46954
+
+#define mmTPC1_CFG_TPC_INTR_CAUSE                                    0xE46958
+
+#define mmTPC1_CFG_TPC_INTR_MASK                                     0xE4695C
+
+#define mmTPC1_CFG_WQ_CREDITS                                        0xE46960
+
+#define mmTPC1_CFG_ARUSER_LO                                         0xE46964
+
+#define mmTPC1_CFG_ARUSER_HI                                         0xE46968
+
+#define mmTPC1_CFG_AWUSER_LO                                         0xE4696C
+
+#define mmTPC1_CFG_AWUSER_HI                                         0xE46970
+
+#define mmTPC1_CFG_OPCODE_EXEC                                       0xE46974
+
+#define mmTPC1_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xE46978
+
+#define mmTPC1_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xE4697C
+
+#define mmTPC1_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xE46980
+
+#define mmTPC1_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xE46984
+
+#define mmTPC1_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xE46988
+
+#define mmTPC1_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xE4698C
+
+#define mmTPC1_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xE46990
+
+#define mmTPC1_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xE46994
+
+#define mmTPC1_CFG_TSB_CFG_MAX_SIZE                                  0xE46998
+
+#define mmTPC1_CFG_TSB_CFG                                           0xE4699C
+
+#define mmTPC1_CFG_DBGMEM_ADD                                        0xE469A0
+
+#define mmTPC1_CFG_DBGMEM_DATA_WR                                    0xE469A4
+
+#define mmTPC1_CFG_DBGMEM_DATA_RD                                    0xE469A8
+
+#define mmTPC1_CFG_DBGMEM_CTRL                                       0xE469AC
+
+#define mmTPC1_CFG_DBGMEM_RC                                         0xE469B0
+
+#define mmTPC1_CFG_TSB_INFLIGHT_CNTR                                 0xE469B4
+
+#define mmTPC1_CFG_WQ_INFLIGHT_CNTR                                  0xE469B8
+
+#define mmTPC1_CFG_WQ_LBW_TOTAL_CNTR                                 0xE469BC
+
+#define mmTPC1_CFG_WQ_HBW_TOTAL_CNTR                                 0xE469C0
+
+#define mmTPC1_CFG_IRQ_OCCOUPY_CNTR                                  0xE469C4
+
+#define mmTPC1_CFG_FUNC_MBIST_CNTRL                                  0xE469D0
+
+#define mmTPC1_CFG_FUNC_MBIST_PAT                                    0xE469D4
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_0                                  0xE469D8
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_1                                  0xE469DC
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_2                                  0xE469E0
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_3                                  0xE469E4
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_4                                  0xE469E8
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_5                                  0xE469EC
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_6                                  0xE469F0
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_7                                  0xE469F4
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_8                                  0xE469F8
+
+#define mmTPC1_CFG_FUNC_MBIST_MEM_9                                  0xE469FC
+
+#define mmTPC1_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xE46A00
+
+#define mmTPC1_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xE46A04
+
+#define mmTPC1_CFG_QM_TENSOR_0_PADDING_VALUE                         0xE46A08
+
+#define mmTPC1_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xE46A0C
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xE46A10
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xE46A14
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xE46A18
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xE46A1C
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xE46A20
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xE46A24
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xE46A28
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xE46A2C
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xE46A30
+
+#define mmTPC1_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xE46A34
+
+#define mmTPC1_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xE46A38
+
+#define mmTPC1_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xE46A3C
+
+#define mmTPC1_CFG_QM_TENSOR_1_PADDING_VALUE                         0xE46A40
+
+#define mmTPC1_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xE46A44
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xE46A48
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xE46A4C
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xE46A50
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xE46A54
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xE46A58
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xE46A5C
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xE46A60
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xE46A64
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xE46A68
+
+#define mmTPC1_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xE46A6C
+
+#define mmTPC1_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xE46A70
+
+#define mmTPC1_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xE46A74
+
+#define mmTPC1_CFG_QM_TENSOR_2_PADDING_VALUE                         0xE46A78
+
+#define mmTPC1_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xE46A7C
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xE46A80
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xE46A84
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xE46A88
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xE46A8C
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xE46A90
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xE46A94
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xE46A98
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xE46A9C
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xE46AA0
+
+#define mmTPC1_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xE46AA4
+
+#define mmTPC1_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xE46AA8
+
+#define mmTPC1_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xE46AAC
+
+#define mmTPC1_CFG_QM_TENSOR_3_PADDING_VALUE                         0xE46AB0
+
+#define mmTPC1_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xE46AB4
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xE46AB8
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xE46ABC
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xE46AC0
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xE46AC4
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xE46AC8
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xE46ACC
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xE46AD0
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xE46AD4
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xE46AD8
+
+#define mmTPC1_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xE46ADC
+
+#define mmTPC1_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xE46AE0
+
+#define mmTPC1_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xE46AE4
+
+#define mmTPC1_CFG_QM_TENSOR_4_PADDING_VALUE                         0xE46AE8
+
+#define mmTPC1_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xE46AEC
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xE46AF0
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xE46AF4
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xE46AF8
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xE46AFC
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xE46B00
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xE46B04
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xE46B08
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xE46B0C
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xE46B10
+
+#define mmTPC1_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xE46B14
+
+#define mmTPC1_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xE46B18
+
+#define mmTPC1_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xE46B1C
+
+#define mmTPC1_CFG_QM_TENSOR_5_PADDING_VALUE                         0xE46B20
+
+#define mmTPC1_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xE46B24
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xE46B28
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xE46B2C
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xE46B30
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xE46B34
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xE46B38
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xE46B3C
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xE46B40
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xE46B44
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xE46B48
+
+#define mmTPC1_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xE46B4C
+
+#define mmTPC1_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xE46B50
+
+#define mmTPC1_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xE46B54
+
+#define mmTPC1_CFG_QM_TENSOR_6_PADDING_VALUE                         0xE46B58
+
+#define mmTPC1_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xE46B5C
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xE46B60
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xE46B64
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xE46B68
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xE46B6C
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xE46B70
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xE46B74
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xE46B78
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xE46B7C
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xE46B80
+
+#define mmTPC1_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xE46B84
+
+#define mmTPC1_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xE46B88
+
+#define mmTPC1_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xE46B8C
+
+#define mmTPC1_CFG_QM_TENSOR_7_PADDING_VALUE                         0xE46B90
+
+#define mmTPC1_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xE46B94
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xE46B98
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xE46B9C
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xE46BA0
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xE46BA4
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xE46BA8
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xE46BAC
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xE46BB0
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xE46BB4
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xE46BB8
+
+#define mmTPC1_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xE46BBC
+
+#define mmTPC1_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xE46BC0
+
+#define mmTPC1_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xE46BC4
+
+#define mmTPC1_CFG_QM_TENSOR_8_PADDING_VALUE                         0xE46BC8
+
+#define mmTPC1_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xE46BCC
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xE46BD0
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xE46BD4
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xE46BD8
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xE46BDC
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xE46BE0
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xE46BE4
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xE46BE8
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xE46BEC
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xE46BF0
+
+#define mmTPC1_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xE46BF4
+
+#define mmTPC1_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xE46BF8
+
+#define mmTPC1_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xE46BFC
+
+#define mmTPC1_CFG_QM_TENSOR_9_PADDING_VALUE                         0xE46C00
+
+#define mmTPC1_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xE46C04
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xE46C08
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xE46C0C
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xE46C10
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xE46C14
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xE46C18
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xE46C1C
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xE46C20
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xE46C24
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xE46C28
+
+#define mmTPC1_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xE46C2C
+
+#define mmTPC1_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xE46C30
+
+#define mmTPC1_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xE46C34
+
+#define mmTPC1_CFG_QM_TENSOR_10_PADDING_VALUE                        0xE46C38
+
+#define mmTPC1_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xE46C3C
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xE46C40
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xE46C44
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xE46C48
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xE46C4C
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xE46C50
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xE46C54
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xE46C58
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xE46C5C
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xE46C60
+
+#define mmTPC1_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xE46C64
+
+#define mmTPC1_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xE46C68
+
+#define mmTPC1_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xE46C6C
+
+#define mmTPC1_CFG_QM_TENSOR_11_PADDING_VALUE                        0xE46C70
+
+#define mmTPC1_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xE46C74
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xE46C78
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xE46C7C
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xE46C80
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xE46C84
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xE46C88
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xE46C8C
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xE46C90
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xE46C94
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xE46C98
+
+#define mmTPC1_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xE46C9C
+
+#define mmTPC1_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xE46CA0
+
+#define mmTPC1_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xE46CA4
+
+#define mmTPC1_CFG_QM_TENSOR_12_PADDING_VALUE                        0xE46CA8
+
+#define mmTPC1_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xE46CAC
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xE46CB0
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xE46CB4
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xE46CB8
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xE46CBC
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xE46CC0
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xE46CC4
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xE46CC8
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xE46CCC
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xE46CD0
+
+#define mmTPC1_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xE46CD4
+
+#define mmTPC1_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xE46CD8
+
+#define mmTPC1_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xE46CDC
+
+#define mmTPC1_CFG_QM_TENSOR_13_PADDING_VALUE                        0xE46CE0
+
+#define mmTPC1_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xE46CE4
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xE46CE8
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xE46CEC
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xE46CF0
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xE46CF4
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xE46CF8
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xE46CFC
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xE46D00
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xE46D04
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xE46D08
+
+#define mmTPC1_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xE46D0C
+
+#define mmTPC1_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xE46D10
+
+#define mmTPC1_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xE46D14
+
+#define mmTPC1_CFG_QM_TENSOR_14_PADDING_VALUE                        0xE46D18
+
+#define mmTPC1_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xE46D1C
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xE46D20
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xE46D24
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xE46D28
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xE46D2C
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xE46D30
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xE46D34
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xE46D38
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xE46D3C
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xE46D40
+
+#define mmTPC1_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xE46D44
+
+#define mmTPC1_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xE46D48
+
+#define mmTPC1_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xE46D4C
+
+#define mmTPC1_CFG_QM_TENSOR_15_PADDING_VALUE                        0xE46D50
+
+#define mmTPC1_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xE46D54
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xE46D58
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xE46D5C
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xE46D60
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xE46D64
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xE46D68
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xE46D6C
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xE46D70
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xE46D74
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xE46D78
+
+#define mmTPC1_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xE46D7C
+
+#define mmTPC1_CFG_QM_SYNC_OBJECT_MESSAGE                            0xE46D80
+
+#define mmTPC1_CFG_QM_SYNC_OBJECT_ADDR                               0xE46D84
+
+#define mmTPC1_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xE46D88
+
+#define mmTPC1_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xE46D8C
+
+#define mmTPC1_CFG_QM_TID_BASE_DIM_0                                 0xE46D90
+
+#define mmTPC1_CFG_QM_TID_SIZE_DIM_0                                 0xE46D94
+
+#define mmTPC1_CFG_QM_TID_BASE_DIM_1                                 0xE46D98
+
+#define mmTPC1_CFG_QM_TID_SIZE_DIM_1                                 0xE46D9C
+
+#define mmTPC1_CFG_QM_TID_BASE_DIM_2                                 0xE46DA0
+
+#define mmTPC1_CFG_QM_TID_SIZE_DIM_2                                 0xE46DA4
+
+#define mmTPC1_CFG_QM_TID_BASE_DIM_3                                 0xE46DA8
+
+#define mmTPC1_CFG_QM_TID_SIZE_DIM_3                                 0xE46DAC
+
+#define mmTPC1_CFG_QM_TID_BASE_DIM_4                                 0xE46DB0
+
+#define mmTPC1_CFG_QM_TID_SIZE_DIM_4                                 0xE46DB4
+
+#define mmTPC1_CFG_QM_KERNEL_CONFIG                                  0xE46DB8
+
+#define mmTPC1_CFG_QM_KERNEL_ID                                      0xE46DBC
+
+#define mmTPC1_CFG_QM_SRF_0                                          0xE46DC0
+
+#define mmTPC1_CFG_QM_SRF_1                                          0xE46DC4
+
+#define mmTPC1_CFG_QM_SRF_2                                          0xE46DC8
+
+#define mmTPC1_CFG_QM_SRF_3                                          0xE46DCC
+
+#define mmTPC1_CFG_QM_SRF_4                                          0xE46DD0
+
+#define mmTPC1_CFG_QM_SRF_5                                          0xE46DD4
+
+#define mmTPC1_CFG_QM_SRF_6                                          0xE46DD8
+
+#define mmTPC1_CFG_QM_SRF_7                                          0xE46DDC
+
+#define mmTPC1_CFG_QM_SRF_8                                          0xE46DE0
+
+#define mmTPC1_CFG_QM_SRF_9                                          0xE46DE4
+
+#define mmTPC1_CFG_QM_SRF_10                                         0xE46DE8
+
+#define mmTPC1_CFG_QM_SRF_11                                         0xE46DEC
+
+#define mmTPC1_CFG_QM_SRF_12                                         0xE46DF0
+
+#define mmTPC1_CFG_QM_SRF_13                                         0xE46DF4
+
+#define mmTPC1_CFG_QM_SRF_14                                         0xE46DF8
+
+#define mmTPC1_CFG_QM_SRF_15                                         0xE46DFC
+
+#define mmTPC1_CFG_QM_SRF_16                                         0xE46E00
+
+#define mmTPC1_CFG_QM_SRF_17                                         0xE46E04
+
+#define mmTPC1_CFG_QM_SRF_18                                         0xE46E08
+
+#define mmTPC1_CFG_QM_SRF_19                                         0xE46E0C
+
+#define mmTPC1_CFG_QM_SRF_20                                         0xE46E10
+
+#define mmTPC1_CFG_QM_SRF_21                                         0xE46E14
+
+#define mmTPC1_CFG_QM_SRF_22                                         0xE46E18
+
+#define mmTPC1_CFG_QM_SRF_23                                         0xE46E1C
+
+#define mmTPC1_CFG_QM_SRF_24                                         0xE46E20
+
+#define mmTPC1_CFG_QM_SRF_25                                         0xE46E24
+
+#define mmTPC1_CFG_QM_SRF_26                                         0xE46E28
+
+#define mmTPC1_CFG_QM_SRF_27                                         0xE46E2C
+
+#define mmTPC1_CFG_QM_SRF_28                                         0xE46E30
+
+#define mmTPC1_CFG_QM_SRF_29                                         0xE46E34
+
+#define mmTPC1_CFG_QM_SRF_30                                         0xE46E38
+
+#define mmTPC1_CFG_QM_SRF_31                                         0xE46E3C
+
+#endif /* ASIC_REG_TPC1_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_qm_regs.h
new file mode 100644
index 000000000000..af10ef7a87d9
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc1_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC1_QM_REGS_H_
+#define ASIC_REG_TPC1_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC1_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC1_QM_GLBL_CFG0                                          0xE48000
+
+#define mmTPC1_QM_GLBL_CFG1                                          0xE48004
+
+#define mmTPC1_QM_GLBL_PROT                                          0xE48008
+
+#define mmTPC1_QM_GLBL_ERR_CFG                                       0xE4800C
+
+#define mmTPC1_QM_GLBL_SECURE_PROPS_0                                0xE48010
+
+#define mmTPC1_QM_GLBL_SECURE_PROPS_1                                0xE48014
+
+#define mmTPC1_QM_GLBL_SECURE_PROPS_2                                0xE48018
+
+#define mmTPC1_QM_GLBL_SECURE_PROPS_3                                0xE4801C
+
+#define mmTPC1_QM_GLBL_SECURE_PROPS_4                                0xE48020
+
+#define mmTPC1_QM_GLBL_NON_SECURE_PROPS_0                            0xE48024
+
+#define mmTPC1_QM_GLBL_NON_SECURE_PROPS_1                            0xE48028
+
+#define mmTPC1_QM_GLBL_NON_SECURE_PROPS_2                            0xE4802C
+
+#define mmTPC1_QM_GLBL_NON_SECURE_PROPS_3                            0xE48030
+
+#define mmTPC1_QM_GLBL_NON_SECURE_PROPS_4                            0xE48034
+
+#define mmTPC1_QM_GLBL_STS0                                          0xE48038
+
+#define mmTPC1_QM_GLBL_STS1_0                                        0xE48040
+
+#define mmTPC1_QM_GLBL_STS1_1                                        0xE48044
+
+#define mmTPC1_QM_GLBL_STS1_2                                        0xE48048
+
+#define mmTPC1_QM_GLBL_STS1_3                                        0xE4804C
+
+#define mmTPC1_QM_GLBL_STS1_4                                        0xE48050
+
+#define mmTPC1_QM_GLBL_MSG_EN_0                                      0xE48054
+
+#define mmTPC1_QM_GLBL_MSG_EN_1                                      0xE48058
+
+#define mmTPC1_QM_GLBL_MSG_EN_2                                      0xE4805C
+
+#define mmTPC1_QM_GLBL_MSG_EN_3                                      0xE48060
+
+#define mmTPC1_QM_GLBL_MSG_EN_4                                      0xE48068
+
+#define mmTPC1_QM_PQ_BASE_LO_0                                       0xE48070
+
+#define mmTPC1_QM_PQ_BASE_LO_1                                       0xE48074
+
+#define mmTPC1_QM_PQ_BASE_LO_2                                       0xE48078
+
+#define mmTPC1_QM_PQ_BASE_LO_3                                       0xE4807C
+
+#define mmTPC1_QM_PQ_BASE_HI_0                                       0xE48080
+
+#define mmTPC1_QM_PQ_BASE_HI_1                                       0xE48084
+
+#define mmTPC1_QM_PQ_BASE_HI_2                                       0xE48088
+
+#define mmTPC1_QM_PQ_BASE_HI_3                                       0xE4808C
+
+#define mmTPC1_QM_PQ_SIZE_0                                          0xE48090
+
+#define mmTPC1_QM_PQ_SIZE_1                                          0xE48094
+
+#define mmTPC1_QM_PQ_SIZE_2                                          0xE48098
+
+#define mmTPC1_QM_PQ_SIZE_3                                          0xE4809C
+
+#define mmTPC1_QM_PQ_PI_0                                            0xE480A0
+
+#define mmTPC1_QM_PQ_PI_1                                            0xE480A4
+
+#define mmTPC1_QM_PQ_PI_2                                            0xE480A8
+
+#define mmTPC1_QM_PQ_PI_3                                            0xE480AC
+
+#define mmTPC1_QM_PQ_CI_0                                            0xE480B0
+
+#define mmTPC1_QM_PQ_CI_1                                            0xE480B4
+
+#define mmTPC1_QM_PQ_CI_2                                            0xE480B8
+
+#define mmTPC1_QM_PQ_CI_3                                            0xE480BC
+
+#define mmTPC1_QM_PQ_CFG0_0                                          0xE480C0
+
+#define mmTPC1_QM_PQ_CFG0_1                                          0xE480C4
+
+#define mmTPC1_QM_PQ_CFG0_2                                          0xE480C8
+
+#define mmTPC1_QM_PQ_CFG0_3                                          0xE480CC
+
+#define mmTPC1_QM_PQ_CFG1_0                                          0xE480D0
+
+#define mmTPC1_QM_PQ_CFG1_1                                          0xE480D4
+
+#define mmTPC1_QM_PQ_CFG1_2                                          0xE480D8
+
+#define mmTPC1_QM_PQ_CFG1_3                                          0xE480DC
+
+#define mmTPC1_QM_PQ_ARUSER_31_11_0                                  0xE480E0
+
+#define mmTPC1_QM_PQ_ARUSER_31_11_1                                  0xE480E4
+
+#define mmTPC1_QM_PQ_ARUSER_31_11_2                                  0xE480E8
+
+#define mmTPC1_QM_PQ_ARUSER_31_11_3                                  0xE480EC
+
+#define mmTPC1_QM_PQ_STS0_0                                          0xE480F0
+
+#define mmTPC1_QM_PQ_STS0_1                                          0xE480F4
+
+#define mmTPC1_QM_PQ_STS0_2                                          0xE480F8
+
+#define mmTPC1_QM_PQ_STS0_3                                          0xE480FC
+
+#define mmTPC1_QM_PQ_STS1_0                                          0xE48100
+
+#define mmTPC1_QM_PQ_STS1_1                                          0xE48104
+
+#define mmTPC1_QM_PQ_STS1_2                                          0xE48108
+
+#define mmTPC1_QM_PQ_STS1_3                                          0xE4810C
+
+#define mmTPC1_QM_CQ_CFG0_0                                          0xE48110
+
+#define mmTPC1_QM_CQ_CFG0_1                                          0xE48114
+
+#define mmTPC1_QM_CQ_CFG0_2                                          0xE48118
+
+#define mmTPC1_QM_CQ_CFG0_3                                          0xE4811C
+
+#define mmTPC1_QM_CQ_CFG0_4                                          0xE48120
+
+#define mmTPC1_QM_CQ_CFG1_0                                          0xE48124
+
+#define mmTPC1_QM_CQ_CFG1_1                                          0xE48128
+
+#define mmTPC1_QM_CQ_CFG1_2                                          0xE4812C
+
+#define mmTPC1_QM_CQ_CFG1_3                                          0xE48130
+
+#define mmTPC1_QM_CQ_CFG1_4                                          0xE48134
+
+#define mmTPC1_QM_CQ_ARUSER_31_11_0                                  0xE48138
+
+#define mmTPC1_QM_CQ_ARUSER_31_11_1                                  0xE4813C
+
+#define mmTPC1_QM_CQ_ARUSER_31_11_2                                  0xE48140
+
+#define mmTPC1_QM_CQ_ARUSER_31_11_3                                  0xE48144
+
+#define mmTPC1_QM_CQ_ARUSER_31_11_4                                  0xE48148
+
+#define mmTPC1_QM_CQ_STS0_0                                          0xE4814C
+
+#define mmTPC1_QM_CQ_STS0_1                                          0xE48150
+
+#define mmTPC1_QM_CQ_STS0_2                                          0xE48154
+
+#define mmTPC1_QM_CQ_STS0_3                                          0xE48158
+
+#define mmTPC1_QM_CQ_STS0_4                                          0xE4815C
+
+#define mmTPC1_QM_CQ_STS1_0                                          0xE48160
+
+#define mmTPC1_QM_CQ_STS1_1                                          0xE48164
+
+#define mmTPC1_QM_CQ_STS1_2                                          0xE48168
+
+#define mmTPC1_QM_CQ_STS1_3                                          0xE4816C
+
+#define mmTPC1_QM_CQ_STS1_4                                          0xE48170
+
+#define mmTPC1_QM_CQ_PTR_LO_0                                        0xE48174
+
+#define mmTPC1_QM_CQ_PTR_HI_0                                        0xE48178
+
+#define mmTPC1_QM_CQ_TSIZE_0                                         0xE4817C
+
+#define mmTPC1_QM_CQ_CTL_0                                           0xE48180
+
+#define mmTPC1_QM_CQ_PTR_LO_1                                        0xE48184
+
+#define mmTPC1_QM_CQ_PTR_HI_1                                        0xE48188
+
+#define mmTPC1_QM_CQ_TSIZE_1                                         0xE4818C
+
+#define mmTPC1_QM_CQ_CTL_1                                           0xE48190
+
+#define mmTPC1_QM_CQ_PTR_LO_2                                        0xE48194
+
+#define mmTPC1_QM_CQ_PTR_HI_2                                        0xE48198
+
+#define mmTPC1_QM_CQ_TSIZE_2                                         0xE4819C
+
+#define mmTPC1_QM_CQ_CTL_2                                           0xE481A0
+
+#define mmTPC1_QM_CQ_PTR_LO_3                                        0xE481A4
+
+#define mmTPC1_QM_CQ_PTR_HI_3                                        0xE481A8
+
+#define mmTPC1_QM_CQ_TSIZE_3                                         0xE481AC
+
+#define mmTPC1_QM_CQ_CTL_3                                           0xE481B0
+
+#define mmTPC1_QM_CQ_PTR_LO_4                                        0xE481B4
+
+#define mmTPC1_QM_CQ_PTR_HI_4                                        0xE481B8
+
+#define mmTPC1_QM_CQ_TSIZE_4                                         0xE481BC
+
+#define mmTPC1_QM_CQ_CTL_4                                           0xE481C0
+
+#define mmTPC1_QM_CQ_PTR_LO_STS_0                                    0xE481C4
+
+#define mmTPC1_QM_CQ_PTR_LO_STS_1                                    0xE481C8
+
+#define mmTPC1_QM_CQ_PTR_LO_STS_2                                    0xE481CC
+
+#define mmTPC1_QM_CQ_PTR_LO_STS_3                                    0xE481D0
+
+#define mmTPC1_QM_CQ_PTR_LO_STS_4                                    0xE481D4
+
+#define mmTPC1_QM_CQ_PTR_HI_STS_0                                    0xE481D8
+
+#define mmTPC1_QM_CQ_PTR_HI_STS_1                                    0xE481DC
+
+#define mmTPC1_QM_CQ_PTR_HI_STS_2                                    0xE481E0
+
+#define mmTPC1_QM_CQ_PTR_HI_STS_3                                    0xE481E4
+
+#define mmTPC1_QM_CQ_PTR_HI_STS_4                                    0xE481E8
+
+#define mmTPC1_QM_CQ_TSIZE_STS_0                                     0xE481EC
+
+#define mmTPC1_QM_CQ_TSIZE_STS_1                                     0xE481F0
+
+#define mmTPC1_QM_CQ_TSIZE_STS_2                                     0xE481F4
+
+#define mmTPC1_QM_CQ_TSIZE_STS_3                                     0xE481F8
+
+#define mmTPC1_QM_CQ_TSIZE_STS_4                                     0xE481FC
+
+#define mmTPC1_QM_CQ_CTL_STS_0                                       0xE48200
+
+#define mmTPC1_QM_CQ_CTL_STS_1                                       0xE48204
+
+#define mmTPC1_QM_CQ_CTL_STS_2                                       0xE48208
+
+#define mmTPC1_QM_CQ_CTL_STS_3                                       0xE4820C
+
+#define mmTPC1_QM_CQ_CTL_STS_4                                       0xE48210
+
+#define mmTPC1_QM_CQ_IFIFO_CNT_0                                     0xE48214
+
+#define mmTPC1_QM_CQ_IFIFO_CNT_1                                     0xE48218
+
+#define mmTPC1_QM_CQ_IFIFO_CNT_2                                     0xE4821C
+
+#define mmTPC1_QM_CQ_IFIFO_CNT_3                                     0xE48220
+
+#define mmTPC1_QM_CQ_IFIFO_CNT_4                                     0xE48224
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_0                             0xE48228
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_1                             0xE4822C
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_2                             0xE48230
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_3                             0xE48234
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_LO_4                             0xE48238
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_0                             0xE4823C
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_1                             0xE48240
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_2                             0xE48244
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_3                             0xE48248
+
+#define mmTPC1_QM_CP_MSG_BASE0_ADDR_HI_4                             0xE4824C
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_0                             0xE48250
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_1                             0xE48254
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_2                             0xE48258
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_3                             0xE4825C
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_LO_4                             0xE48260
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_0                             0xE48264
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_1                             0xE48268
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_2                             0xE4826C
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_3                             0xE48270
+
+#define mmTPC1_QM_CP_MSG_BASE1_ADDR_HI_4                             0xE48274
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_0                             0xE48278
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_1                             0xE4827C
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_2                             0xE48280
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_3                             0xE48284
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_LO_4                             0xE48288
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_0                             0xE4828C
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_1                             0xE48290
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_2                             0xE48294
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_3                             0xE48298
+
+#define mmTPC1_QM_CP_MSG_BASE2_ADDR_HI_4                             0xE4829C
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_0                             0xE482A0
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_1                             0xE482A4
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_2                             0xE482A8
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_3                             0xE482AC
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_LO_4                             0xE482B0
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_0                             0xE482B4
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_1                             0xE482B8
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_2                             0xE482BC
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_3                             0xE482C0
+
+#define mmTPC1_QM_CP_MSG_BASE3_ADDR_HI_4                             0xE482C4
+
+#define mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_0                             0xE482C8
+
+#define mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_1                             0xE482CC
+
+#define mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_2                             0xE482D0
+
+#define mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_3                             0xE482D4
+
+#define mmTPC1_QM_CP_LDMA_TSIZE_OFFSET_4                             0xE482D8
+
+#define mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xE482E0
+
+#define mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xE482E4
+
+#define mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xE482E8
+
+#define mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xE482EC
+
+#define mmTPC1_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xE482F0
+
+#define mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xE482F4
+
+#define mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xE482F8
+
+#define mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xE482FC
+
+#define mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xE48300
+
+#define mmTPC1_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xE48304
+
+#define mmTPC1_QM_CP_FENCE0_RDATA_0                                  0xE48308
+
+#define mmTPC1_QM_CP_FENCE0_RDATA_1                                  0xE4830C
+
+#define mmTPC1_QM_CP_FENCE0_RDATA_2                                  0xE48310
+
+#define mmTPC1_QM_CP_FENCE0_RDATA_3                                  0xE48314
+
+#define mmTPC1_QM_CP_FENCE0_RDATA_4                                  0xE48318
+
+#define mmTPC1_QM_CP_FENCE1_RDATA_0                                  0xE4831C
+
+#define mmTPC1_QM_CP_FENCE1_RDATA_1                                  0xE48320
+
+#define mmTPC1_QM_CP_FENCE1_RDATA_2                                  0xE48324
+
+#define mmTPC1_QM_CP_FENCE1_RDATA_3                                  0xE48328
+
+#define mmTPC1_QM_CP_FENCE1_RDATA_4                                  0xE4832C
+
+#define mmTPC1_QM_CP_FENCE2_RDATA_0                                  0xE48330
+
+#define mmTPC1_QM_CP_FENCE2_RDATA_1                                  0xE48334
+
+#define mmTPC1_QM_CP_FENCE2_RDATA_2                                  0xE48338
+
+#define mmTPC1_QM_CP_FENCE2_RDATA_3                                  0xE4833C
+
+#define mmTPC1_QM_CP_FENCE2_RDATA_4                                  0xE48340
+
+#define mmTPC1_QM_CP_FENCE3_RDATA_0                                  0xE48344
+
+#define mmTPC1_QM_CP_FENCE3_RDATA_1                                  0xE48348
+
+#define mmTPC1_QM_CP_FENCE3_RDATA_2                                  0xE4834C
+
+#define mmTPC1_QM_CP_FENCE3_RDATA_3                                  0xE48350
+
+#define mmTPC1_QM_CP_FENCE3_RDATA_4                                  0xE48354
+
+#define mmTPC1_QM_CP_FENCE0_CNT_0                                    0xE48358
+
+#define mmTPC1_QM_CP_FENCE0_CNT_1                                    0xE4835C
+
+#define mmTPC1_QM_CP_FENCE0_CNT_2                                    0xE48360
+
+#define mmTPC1_QM_CP_FENCE0_CNT_3                                    0xE48364
+
+#define mmTPC1_QM_CP_FENCE0_CNT_4                                    0xE48368
+
+#define mmTPC1_QM_CP_FENCE1_CNT_0                                    0xE4836C
+
+#define mmTPC1_QM_CP_FENCE1_CNT_1                                    0xE48370
+
+#define mmTPC1_QM_CP_FENCE1_CNT_2                                    0xE48374
+
+#define mmTPC1_QM_CP_FENCE1_CNT_3                                    0xE48378
+
+#define mmTPC1_QM_CP_FENCE1_CNT_4                                    0xE4837C
+
+#define mmTPC1_QM_CP_FENCE2_CNT_0                                    0xE48380
+
+#define mmTPC1_QM_CP_FENCE2_CNT_1                                    0xE48384
+
+#define mmTPC1_QM_CP_FENCE2_CNT_2                                    0xE48388
+
+#define mmTPC1_QM_CP_FENCE2_CNT_3                                    0xE4838C
+
+#define mmTPC1_QM_CP_FENCE2_CNT_4                                    0xE48390
+
+#define mmTPC1_QM_CP_FENCE3_CNT_0                                    0xE48394
+
+#define mmTPC1_QM_CP_FENCE3_CNT_1                                    0xE48398
+
+#define mmTPC1_QM_CP_FENCE3_CNT_2                                    0xE4839C
+
+#define mmTPC1_QM_CP_FENCE3_CNT_3                                    0xE483A0
+
+#define mmTPC1_QM_CP_FENCE3_CNT_4                                    0xE483A4
+
+#define mmTPC1_QM_CP_STS_0                                           0xE483A8
+
+#define mmTPC1_QM_CP_STS_1                                           0xE483AC
+
+#define mmTPC1_QM_CP_STS_2                                           0xE483B0
+
+#define mmTPC1_QM_CP_STS_3                                           0xE483B4
+
+#define mmTPC1_QM_CP_STS_4                                           0xE483B8
+
+#define mmTPC1_QM_CP_CURRENT_INST_LO_0                               0xE483BC
+
+#define mmTPC1_QM_CP_CURRENT_INST_LO_1                               0xE483C0
+
+#define mmTPC1_QM_CP_CURRENT_INST_LO_2                               0xE483C4
+
+#define mmTPC1_QM_CP_CURRENT_INST_LO_3                               0xE483C8
+
+#define mmTPC1_QM_CP_CURRENT_INST_LO_4                               0xE483CC
+
+#define mmTPC1_QM_CP_CURRENT_INST_HI_0                               0xE483D0
+
+#define mmTPC1_QM_CP_CURRENT_INST_HI_1                               0xE483D4
+
+#define mmTPC1_QM_CP_CURRENT_INST_HI_2                               0xE483D8
+
+#define mmTPC1_QM_CP_CURRENT_INST_HI_3                               0xE483DC
+
+#define mmTPC1_QM_CP_CURRENT_INST_HI_4                               0xE483E0
+
+#define mmTPC1_QM_CP_BARRIER_CFG_0                                   0xE483F4
+
+#define mmTPC1_QM_CP_BARRIER_CFG_1                                   0xE483F8
+
+#define mmTPC1_QM_CP_BARRIER_CFG_2                                   0xE483FC
+
+#define mmTPC1_QM_CP_BARRIER_CFG_3                                   0xE48400
+
+#define mmTPC1_QM_CP_BARRIER_CFG_4                                   0xE48404
+
+#define mmTPC1_QM_CP_DBG_0_0                                         0xE48408
+
+#define mmTPC1_QM_CP_DBG_0_1                                         0xE4840C
+
+#define mmTPC1_QM_CP_DBG_0_2                                         0xE48410
+
+#define mmTPC1_QM_CP_DBG_0_3                                         0xE48414
+
+#define mmTPC1_QM_CP_DBG_0_4                                         0xE48418
+
+#define mmTPC1_QM_CP_ARUSER_31_11_0                                  0xE4841C
+
+#define mmTPC1_QM_CP_ARUSER_31_11_1                                  0xE48420
+
+#define mmTPC1_QM_CP_ARUSER_31_11_2                                  0xE48424
+
+#define mmTPC1_QM_CP_ARUSER_31_11_3                                  0xE48428
+
+#define mmTPC1_QM_CP_ARUSER_31_11_4                                  0xE4842C
+
+#define mmTPC1_QM_CP_AWUSER_31_11_0                                  0xE48430
+
+#define mmTPC1_QM_CP_AWUSER_31_11_1                                  0xE48434
+
+#define mmTPC1_QM_CP_AWUSER_31_11_2                                  0xE48438
+
+#define mmTPC1_QM_CP_AWUSER_31_11_3                                  0xE4843C
+
+#define mmTPC1_QM_CP_AWUSER_31_11_4                                  0xE48440
+
+#define mmTPC1_QM_ARB_CFG_0                                          0xE48A00
+
+#define mmTPC1_QM_ARB_CHOISE_Q_PUSH                                  0xE48A04
+
+#define mmTPC1_QM_ARB_WRR_WEIGHT_0                                   0xE48A08
+
+#define mmTPC1_QM_ARB_WRR_WEIGHT_1                                   0xE48A0C
+
+#define mmTPC1_QM_ARB_WRR_WEIGHT_2                                   0xE48A10
+
+#define mmTPC1_QM_ARB_WRR_WEIGHT_3                                   0xE48A14
+
+#define mmTPC1_QM_ARB_CFG_1                                          0xE48A18
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_0                               0xE48A20
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_1                               0xE48A24
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_2                               0xE48A28
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_3                               0xE48A2C
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_4                               0xE48A30
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_5                               0xE48A34
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_6                               0xE48A38
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_7                               0xE48A3C
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_8                               0xE48A40
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_9                               0xE48A44
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_10                              0xE48A48
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_11                              0xE48A4C
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_12                              0xE48A50
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_13                              0xE48A54
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_14                              0xE48A58
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_15                              0xE48A5C
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_16                              0xE48A60
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_17                              0xE48A64
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_18                              0xE48A68
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_19                              0xE48A6C
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_20                              0xE48A70
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_21                              0xE48A74
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_22                              0xE48A78
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_23                              0xE48A7C
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_24                              0xE48A80
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_25                              0xE48A84
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_26                              0xE48A88
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_27                              0xE48A8C
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_28                              0xE48A90
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_29                              0xE48A94
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_30                              0xE48A98
+
+#define mmTPC1_QM_ARB_MST_AVAIL_CRED_31                              0xE48A9C
+
+#define mmTPC1_QM_ARB_MST_CRED_INC                                   0xE48AA0
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xE48AA4
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xE48AA8
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xE48AAC
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xE48AB0
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xE48AB4
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xE48AB8
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xE48ABC
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xE48AC0
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xE48AC4
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xE48AC8
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xE48ACC
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xE48AD0
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xE48AD4
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xE48AD8
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xE48ADC
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xE48AE0
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xE48AE4
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xE48AE8
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xE48AEC
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xE48AF0
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xE48AF4
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xE48AF8
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xE48AFC
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xE48B00
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xE48B04
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xE48B08
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xE48B0C
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xE48B10
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xE48B14
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xE48B18
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xE48B1C
+
+#define mmTPC1_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xE48B20
+
+#define mmTPC1_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xE48B28
+
+#define mmTPC1_QM_ARB_MST_SLAVE_EN                                   0xE48B2C
+
+#define mmTPC1_QM_ARB_MST_QUIET_PER                                  0xE48B34
+
+#define mmTPC1_QM_ARB_SLV_CHOISE_WDT                                 0xE48B38
+
+#define mmTPC1_QM_ARB_SLV_ID                                         0xE48B3C
+
+#define mmTPC1_QM_ARB_MSG_MAX_INFLIGHT                               0xE48B44
+
+#define mmTPC1_QM_ARB_MSG_AWUSER_31_11                               0xE48B48
+
+#define mmTPC1_QM_ARB_MSG_AWUSER_SEC_PROP                            0xE48B4C
+
+#define mmTPC1_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xE48B50
+
+#define mmTPC1_QM_ARB_BASE_LO                                        0xE48B54
+
+#define mmTPC1_QM_ARB_BASE_HI                                        0xE48B58
+
+#define mmTPC1_QM_ARB_STATE_STS                                      0xE48B80
+
+#define mmTPC1_QM_ARB_CHOISE_FULLNESS_STS                            0xE48B84
+
+#define mmTPC1_QM_ARB_MSG_STS                                        0xE48B88
+
+#define mmTPC1_QM_ARB_SLV_CHOISE_Q_HEAD                              0xE48B8C
+
+#define mmTPC1_QM_ARB_ERR_CAUSE                                      0xE48B9C
+
+#define mmTPC1_QM_ARB_ERR_MSG_EN                                     0xE48BA0
+
+#define mmTPC1_QM_ARB_ERR_STS_DRP                                    0xE48BA8
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_0                                 0xE48BB0
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_1                                 0xE48BB4
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_2                                 0xE48BB8
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_3                                 0xE48BBC
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_4                                 0xE48BC0
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_5                                 0xE48BC4
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_6                                 0xE48BC8
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_7                                 0xE48BCC
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_8                                 0xE48BD0
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_9                                 0xE48BD4
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_10                                0xE48BD8
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_11                                0xE48BDC
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_12                                0xE48BE0
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_13                                0xE48BE4
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_14                                0xE48BE8
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_15                                0xE48BEC
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_16                                0xE48BF0
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_17                                0xE48BF4
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_18                                0xE48BF8
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_19                                0xE48BFC
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_20                                0xE48C00
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_21                                0xE48C04
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_22                                0xE48C08
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_23                                0xE48C0C
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_24                                0xE48C10
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_25                                0xE48C14
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_26                                0xE48C18
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_27                                0xE48C1C
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_28                                0xE48C20
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_29                                0xE48C24
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_30                                0xE48C28
+
+#define mmTPC1_QM_ARB_MST_CRED_STS_31                                0xE48C2C
+
+#define mmTPC1_QM_CGM_CFG                                            0xE48C70
+
+#define mmTPC1_QM_CGM_STS                                            0xE48C74
+
+#define mmTPC1_QM_CGM_CFG1                                           0xE48C78
+
+#define mmTPC1_QM_LOCAL_RANGE_BASE                                   0xE48C80
+
+#define mmTPC1_QM_LOCAL_RANGE_SIZE                                   0xE48C84
+
+#define mmTPC1_QM_CSMR_STRICT_PRIO_CFG                               0xE48C90
+
+#define mmTPC1_QM_HBW_RD_RATE_LIM_CFG_1                              0xE48C94
+
+#define mmTPC1_QM_LBW_WR_RATE_LIM_CFG_0                              0xE48C98
+
+#define mmTPC1_QM_LBW_WR_RATE_LIM_CFG_1                              0xE48C9C
+
+#define mmTPC1_QM_HBW_RD_RATE_LIM_CFG_0                              0xE48CA0
+
+#define mmTPC1_QM_GLBL_AXCACHE                                       0xE48CA4
+
+#define mmTPC1_QM_IND_GW_APB_CFG                                     0xE48CB0
+
+#define mmTPC1_QM_IND_GW_APB_WDATA                                   0xE48CB4
+
+#define mmTPC1_QM_IND_GW_APB_RDATA                                   0xE48CB8
+
+#define mmTPC1_QM_IND_GW_APB_STATUS                                  0xE48CBC
+
+#define mmTPC1_QM_GLBL_ERR_ADDR_LO                                   0xE48CD0
+
+#define mmTPC1_QM_GLBL_ERR_ADDR_HI                                   0xE48CD4
+
+#define mmTPC1_QM_GLBL_ERR_WDATA                                     0xE48CD8
+
+#define mmTPC1_QM_GLBL_MEM_INIT_BUSY                                 0xE48D00
+
+#endif /* ASIC_REG_TPC1_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_cfg_regs.h
new file mode 100644
index 000000000000..3e77c37952bc
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC2_CFG_REGS_H_
+#define ASIC_REG_TPC2_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC2_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xE86400
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xE86404
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xE86408
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xE8640C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xE86410
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xE86414
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xE86418
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xE8641C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xE86420
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xE86424
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xE86428
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xE8642C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xE86430
+
+#define mmTPC2_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xE86434
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xE86438
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xE8643C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xE86440
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xE86444
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xE86448
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xE8644C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xE86450
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xE86454
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xE86458
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xE8645C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xE86460
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xE86464
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xE86468
+
+#define mmTPC2_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xE8646C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xE86470
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xE86474
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xE86478
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xE8647C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xE86480
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xE86484
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xE86488
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xE8648C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xE86490
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xE86494
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xE86498
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xE8649C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xE864A0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xE864A4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xE864A8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xE864AC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xE864B0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xE864B4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xE864B8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xE864BC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xE864C0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xE864C4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xE864C8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xE864CC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xE864D0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xE864D4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xE864D8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xE864DC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xE864E0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xE864E4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xE864E8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xE864EC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xE864F0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xE864F4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xE864F8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xE864FC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xE86500
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xE86504
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xE86508
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xE8650C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xE86510
+
+#define mmTPC2_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xE86514
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xE86518
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xE8651C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xE86520
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xE86524
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xE86528
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xE8652C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xE86530
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xE86534
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xE86538
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xE8653C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xE86540
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xE86544
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xE86548
+
+#define mmTPC2_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xE8654C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xE86550
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xE86554
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xE86558
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xE8655C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xE86560
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xE86564
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xE86568
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xE8656C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xE86570
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xE86574
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xE86578
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xE8657C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xE86580
+
+#define mmTPC2_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xE86584
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xE86588
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xE8658C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xE86590
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xE86594
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xE86598
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xE8659C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xE865A0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xE865A4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xE865A8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xE865AC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xE865B0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xE865B4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xE865B8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xE865BC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xE865C0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xE865C4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xE865C8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xE865CC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xE865D0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xE865D4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xE865D8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xE865DC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xE865E0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xE865E4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xE865E8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xE865EC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xE865F0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xE865F4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xE865F8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xE865FC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xE86600
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xE86604
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xE86608
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xE8660C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xE86610
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xE86614
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xE86618
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xE8661C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xE86620
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xE86624
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xE86628
+
+#define mmTPC2_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xE8662C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xE86630
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xE86634
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xE86638
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xE8663C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xE86640
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xE86644
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xE86648
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xE8664C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xE86650
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xE86654
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xE86658
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xE8665C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xE86660
+
+#define mmTPC2_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xE86664
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xE86668
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xE8666C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xE86670
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xE86674
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xE86678
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xE8667C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xE86680
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xE86684
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xE86688
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xE8668C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xE86690
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xE86694
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xE86698
+
+#define mmTPC2_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xE8669C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xE866A0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xE866A4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xE866A8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xE866AC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xE866B0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xE866B4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xE866B8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xE866BC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xE866C0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xE866C4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xE866C8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xE866CC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xE866D0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xE866D4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xE866D8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xE866DC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xE866E0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xE866E4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xE866E8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xE866EC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xE866F0
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xE866F4
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xE866F8
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xE866FC
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xE86700
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xE86704
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xE86708
+
+#define mmTPC2_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xE8670C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xE86710
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xE86714
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xE86718
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xE8671C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xE86720
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xE86724
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xE86728
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xE8672C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xE86730
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xE86734
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xE86738
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xE8673C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xE86740
+
+#define mmTPC2_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xE86744
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xE86748
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xE8674C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xE86750
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xE86754
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xE86758
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xE8675C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xE86760
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xE86764
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xE86768
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xE8676C
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xE86770
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xE86774
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xE86778
+
+#define mmTPC2_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xE8677C
+
+#define mmTPC2_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xE86780
+
+#define mmTPC2_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xE86784
+
+#define mmTPC2_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xE86788
+
+#define mmTPC2_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xE8678C
+
+#define mmTPC2_CFG_KERNEL_TID_BASE_DIM_0                             0xE86790
+
+#define mmTPC2_CFG_KERNEL_TID_SIZE_DIM_0                             0xE86794
+
+#define mmTPC2_CFG_KERNEL_TID_BASE_DIM_1                             0xE86798
+
+#define mmTPC2_CFG_KERNEL_TID_SIZE_DIM_1                             0xE8679C
+
+#define mmTPC2_CFG_KERNEL_TID_BASE_DIM_2                             0xE867A0
+
+#define mmTPC2_CFG_KERNEL_TID_SIZE_DIM_2                             0xE867A4
+
+#define mmTPC2_CFG_KERNEL_TID_BASE_DIM_3                             0xE867A8
+
+#define mmTPC2_CFG_KERNEL_TID_SIZE_DIM_3                             0xE867AC
+
+#define mmTPC2_CFG_KERNEL_TID_BASE_DIM_4                             0xE867B0
+
+#define mmTPC2_CFG_KERNEL_TID_SIZE_DIM_4                             0xE867B4
+
+#define mmTPC2_CFG_KERNEL_KERNEL_CONFIG                              0xE867B8
+
+#define mmTPC2_CFG_KERNEL_KERNEL_ID                                  0xE867BC
+
+#define mmTPC2_CFG_KERNEL_SRF_0                                      0xE867C0
+
+#define mmTPC2_CFG_KERNEL_SRF_1                                      0xE867C4
+
+#define mmTPC2_CFG_KERNEL_SRF_2                                      0xE867C8
+
+#define mmTPC2_CFG_KERNEL_SRF_3                                      0xE867CC
+
+#define mmTPC2_CFG_KERNEL_SRF_4                                      0xE867D0
+
+#define mmTPC2_CFG_KERNEL_SRF_5                                      0xE867D4
+
+#define mmTPC2_CFG_KERNEL_SRF_6                                      0xE867D8
+
+#define mmTPC2_CFG_KERNEL_SRF_7                                      0xE867DC
+
+#define mmTPC2_CFG_KERNEL_SRF_8                                      0xE867E0
+
+#define mmTPC2_CFG_KERNEL_SRF_9                                      0xE867E4
+
+#define mmTPC2_CFG_KERNEL_SRF_10                                     0xE867E8
+
+#define mmTPC2_CFG_KERNEL_SRF_11                                     0xE867EC
+
+#define mmTPC2_CFG_KERNEL_SRF_12                                     0xE867F0
+
+#define mmTPC2_CFG_KERNEL_SRF_13                                     0xE867F4
+
+#define mmTPC2_CFG_KERNEL_SRF_14                                     0xE867F8
+
+#define mmTPC2_CFG_KERNEL_SRF_15                                     0xE867FC
+
+#define mmTPC2_CFG_KERNEL_SRF_16                                     0xE86800
+
+#define mmTPC2_CFG_KERNEL_SRF_17                                     0xE86804
+
+#define mmTPC2_CFG_KERNEL_SRF_18                                     0xE86808
+
+#define mmTPC2_CFG_KERNEL_SRF_19                                     0xE8680C
+
+#define mmTPC2_CFG_KERNEL_SRF_20                                     0xE86810
+
+#define mmTPC2_CFG_KERNEL_SRF_21                                     0xE86814
+
+#define mmTPC2_CFG_KERNEL_SRF_22                                     0xE86818
+
+#define mmTPC2_CFG_KERNEL_SRF_23                                     0xE8681C
+
+#define mmTPC2_CFG_KERNEL_SRF_24                                     0xE86820
+
+#define mmTPC2_CFG_KERNEL_SRF_25                                     0xE86824
+
+#define mmTPC2_CFG_KERNEL_SRF_26                                     0xE86828
+
+#define mmTPC2_CFG_KERNEL_SRF_27                                     0xE8682C
+
+#define mmTPC2_CFG_KERNEL_SRF_28                                     0xE86830
+
+#define mmTPC2_CFG_KERNEL_SRF_29                                     0xE86834
+
+#define mmTPC2_CFG_KERNEL_SRF_30                                     0xE86838
+
+#define mmTPC2_CFG_KERNEL_SRF_31                                     0xE8683C
+
+#define mmTPC2_CFG_ROUND_CSR                                         0xE868FC
+
+#define mmTPC2_CFG_PROT                                              0xE86900
+
+#define mmTPC2_CFG_SEMAPHORE                                         0xE86908
+
+#define mmTPC2_CFG_VFLAGS                                            0xE8690C
+
+#define mmTPC2_CFG_SFLAGS                                            0xE86910
+
+#define mmTPC2_CFG_LFSR_POLYNOM                                      0xE86918
+
+#define mmTPC2_CFG_STATUS                                            0xE8691C
+
+#define mmTPC2_CFG_CFG_BASE_ADDRESS_HIGH                             0xE86920
+
+#define mmTPC2_CFG_CFG_SUBTRACT_VALUE                                0xE86924
+
+#define mmTPC2_CFG_SM_BASE_ADDRESS_HIGH                              0xE8692C
+
+#define mmTPC2_CFG_TPC_CMD                                           0xE86930
+
+#define mmTPC2_CFG_TPC_EXECUTE                                       0xE86938
+
+#define mmTPC2_CFG_TPC_STALL                                         0xE8693C
+
+#define mmTPC2_CFG_ICACHE_BASE_ADDERESS_LOW                          0xE86940
+
+#define mmTPC2_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xE86944
+
+#define mmTPC2_CFG_RD_RATE_LIMIT                                     0xE86948
+
+#define mmTPC2_CFG_WR_RATE_LIMIT                                     0xE86950
+
+#define mmTPC2_CFG_MSS_CONFIG                                        0xE86954
+
+#define mmTPC2_CFG_TPC_INTR_CAUSE                                    0xE86958
+
+#define mmTPC2_CFG_TPC_INTR_MASK                                     0xE8695C
+
+#define mmTPC2_CFG_WQ_CREDITS                                        0xE86960
+
+#define mmTPC2_CFG_ARUSER_LO                                         0xE86964
+
+#define mmTPC2_CFG_ARUSER_HI                                         0xE86968
+
+#define mmTPC2_CFG_AWUSER_LO                                         0xE8696C
+
+#define mmTPC2_CFG_AWUSER_HI                                         0xE86970
+
+#define mmTPC2_CFG_OPCODE_EXEC                                       0xE86974
+
+#define mmTPC2_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xE86978
+
+#define mmTPC2_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xE8697C
+
+#define mmTPC2_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xE86980
+
+#define mmTPC2_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xE86984
+
+#define mmTPC2_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xE86988
+
+#define mmTPC2_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xE8698C
+
+#define mmTPC2_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xE86990
+
+#define mmTPC2_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xE86994
+
+#define mmTPC2_CFG_TSB_CFG_MAX_SIZE                                  0xE86998
+
+#define mmTPC2_CFG_TSB_CFG                                           0xE8699C
+
+#define mmTPC2_CFG_DBGMEM_ADD                                        0xE869A0
+
+#define mmTPC2_CFG_DBGMEM_DATA_WR                                    0xE869A4
+
+#define mmTPC2_CFG_DBGMEM_DATA_RD                                    0xE869A8
+
+#define mmTPC2_CFG_DBGMEM_CTRL                                       0xE869AC
+
+#define mmTPC2_CFG_DBGMEM_RC                                         0xE869B0
+
+#define mmTPC2_CFG_TSB_INFLIGHT_CNTR                                 0xE869B4
+
+#define mmTPC2_CFG_WQ_INFLIGHT_CNTR                                  0xE869B8
+
+#define mmTPC2_CFG_WQ_LBW_TOTAL_CNTR                                 0xE869BC
+
+#define mmTPC2_CFG_WQ_HBW_TOTAL_CNTR                                 0xE869C0
+
+#define mmTPC2_CFG_IRQ_OCCOUPY_CNTR                                  0xE869C4
+
+#define mmTPC2_CFG_FUNC_MBIST_CNTRL                                  0xE869D0
+
+#define mmTPC2_CFG_FUNC_MBIST_PAT                                    0xE869D4
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_0                                  0xE869D8
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_1                                  0xE869DC
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_2                                  0xE869E0
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_3                                  0xE869E4
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_4                                  0xE869E8
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_5                                  0xE869EC
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_6                                  0xE869F0
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_7                                  0xE869F4
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_8                                  0xE869F8
+
+#define mmTPC2_CFG_FUNC_MBIST_MEM_9                                  0xE869FC
+
+#define mmTPC2_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xE86A00
+
+#define mmTPC2_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xE86A04
+
+#define mmTPC2_CFG_QM_TENSOR_0_PADDING_VALUE                         0xE86A08
+
+#define mmTPC2_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xE86A0C
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xE86A10
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xE86A14
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xE86A18
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xE86A1C
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xE86A20
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xE86A24
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xE86A28
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xE86A2C
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xE86A30
+
+#define mmTPC2_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xE86A34
+
+#define mmTPC2_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xE86A38
+
+#define mmTPC2_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xE86A3C
+
+#define mmTPC2_CFG_QM_TENSOR_1_PADDING_VALUE                         0xE86A40
+
+#define mmTPC2_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xE86A44
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xE86A48
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xE86A4C
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xE86A50
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xE86A54
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xE86A58
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xE86A5C
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xE86A60
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xE86A64
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xE86A68
+
+#define mmTPC2_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xE86A6C
+
+#define mmTPC2_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xE86A70
+
+#define mmTPC2_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xE86A74
+
+#define mmTPC2_CFG_QM_TENSOR_2_PADDING_VALUE                         0xE86A78
+
+#define mmTPC2_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xE86A7C
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xE86A80
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xE86A84
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xE86A88
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xE86A8C
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xE86A90
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xE86A94
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xE86A98
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xE86A9C
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xE86AA0
+
+#define mmTPC2_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xE86AA4
+
+#define mmTPC2_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xE86AA8
+
+#define mmTPC2_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xE86AAC
+
+#define mmTPC2_CFG_QM_TENSOR_3_PADDING_VALUE                         0xE86AB0
+
+#define mmTPC2_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xE86AB4
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xE86AB8
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xE86ABC
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xE86AC0
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xE86AC4
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xE86AC8
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xE86ACC
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xE86AD0
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xE86AD4
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xE86AD8
+
+#define mmTPC2_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xE86ADC
+
+#define mmTPC2_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xE86AE0
+
+#define mmTPC2_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xE86AE4
+
+#define mmTPC2_CFG_QM_TENSOR_4_PADDING_VALUE                         0xE86AE8
+
+#define mmTPC2_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xE86AEC
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xE86AF0
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xE86AF4
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xE86AF8
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xE86AFC
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xE86B00
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xE86B04
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xE86B08
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xE86B0C
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xE86B10
+
+#define mmTPC2_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xE86B14
+
+#define mmTPC2_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xE86B18
+
+#define mmTPC2_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xE86B1C
+
+#define mmTPC2_CFG_QM_TENSOR_5_PADDING_VALUE                         0xE86B20
+
+#define mmTPC2_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xE86B24
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xE86B28
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xE86B2C
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xE86B30
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xE86B34
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xE86B38
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xE86B3C
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xE86B40
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xE86B44
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xE86B48
+
+#define mmTPC2_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xE86B4C
+
+#define mmTPC2_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xE86B50
+
+#define mmTPC2_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xE86B54
+
+#define mmTPC2_CFG_QM_TENSOR_6_PADDING_VALUE                         0xE86B58
+
+#define mmTPC2_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xE86B5C
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xE86B60
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xE86B64
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xE86B68
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xE86B6C
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xE86B70
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xE86B74
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xE86B78
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xE86B7C
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xE86B80
+
+#define mmTPC2_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xE86B84
+
+#define mmTPC2_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xE86B88
+
+#define mmTPC2_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xE86B8C
+
+#define mmTPC2_CFG_QM_TENSOR_7_PADDING_VALUE                         0xE86B90
+
+#define mmTPC2_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xE86B94
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xE86B98
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xE86B9C
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xE86BA0
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xE86BA4
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xE86BA8
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xE86BAC
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xE86BB0
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xE86BB4
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xE86BB8
+
+#define mmTPC2_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xE86BBC
+
+#define mmTPC2_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xE86BC0
+
+#define mmTPC2_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xE86BC4
+
+#define mmTPC2_CFG_QM_TENSOR_8_PADDING_VALUE                         0xE86BC8
+
+#define mmTPC2_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xE86BCC
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xE86BD0
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xE86BD4
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xE86BD8
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xE86BDC
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xE86BE0
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xE86BE4
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xE86BE8
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xE86BEC
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xE86BF0
+
+#define mmTPC2_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xE86BF4
+
+#define mmTPC2_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xE86BF8
+
+#define mmTPC2_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xE86BFC
+
+#define mmTPC2_CFG_QM_TENSOR_9_PADDING_VALUE                         0xE86C00
+
+#define mmTPC2_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xE86C04
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xE86C08
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xE86C0C
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xE86C10
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xE86C14
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xE86C18
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xE86C1C
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xE86C20
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xE86C24
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xE86C28
+
+#define mmTPC2_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xE86C2C
+
+#define mmTPC2_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xE86C30
+
+#define mmTPC2_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xE86C34
+
+#define mmTPC2_CFG_QM_TENSOR_10_PADDING_VALUE                        0xE86C38
+
+#define mmTPC2_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xE86C3C
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xE86C40
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xE86C44
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xE86C48
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xE86C4C
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xE86C50
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xE86C54
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xE86C58
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xE86C5C
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xE86C60
+
+#define mmTPC2_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xE86C64
+
+#define mmTPC2_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xE86C68
+
+#define mmTPC2_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xE86C6C
+
+#define mmTPC2_CFG_QM_TENSOR_11_PADDING_VALUE                        0xE86C70
+
+#define mmTPC2_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xE86C74
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xE86C78
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xE86C7C
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xE86C80
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xE86C84
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xE86C88
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xE86C8C
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xE86C90
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xE86C94
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xE86C98
+
+#define mmTPC2_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xE86C9C
+
+#define mmTPC2_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xE86CA0
+
+#define mmTPC2_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xE86CA4
+
+#define mmTPC2_CFG_QM_TENSOR_12_PADDING_VALUE                        0xE86CA8
+
+#define mmTPC2_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xE86CAC
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xE86CB0
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xE86CB4
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xE86CB8
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xE86CBC
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xE86CC0
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xE86CC4
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xE86CC8
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xE86CCC
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xE86CD0
+
+#define mmTPC2_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xE86CD4
+
+#define mmTPC2_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xE86CD8
+
+#define mmTPC2_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xE86CDC
+
+#define mmTPC2_CFG_QM_TENSOR_13_PADDING_VALUE                        0xE86CE0
+
+#define mmTPC2_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xE86CE4
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xE86CE8
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xE86CEC
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xE86CF0
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xE86CF4
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xE86CF8
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xE86CFC
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xE86D00
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xE86D04
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xE86D08
+
+#define mmTPC2_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xE86D0C
+
+#define mmTPC2_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xE86D10
+
+#define mmTPC2_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xE86D14
+
+#define mmTPC2_CFG_QM_TENSOR_14_PADDING_VALUE                        0xE86D18
+
+#define mmTPC2_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xE86D1C
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xE86D20
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xE86D24
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xE86D28
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xE86D2C
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xE86D30
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xE86D34
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xE86D38
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xE86D3C
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xE86D40
+
+#define mmTPC2_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xE86D44
+
+#define mmTPC2_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xE86D48
+
+#define mmTPC2_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xE86D4C
+
+#define mmTPC2_CFG_QM_TENSOR_15_PADDING_VALUE                        0xE86D50
+
+#define mmTPC2_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xE86D54
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xE86D58
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xE86D5C
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xE86D60
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xE86D64
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xE86D68
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xE86D6C
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xE86D70
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xE86D74
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xE86D78
+
+#define mmTPC2_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xE86D7C
+
+#define mmTPC2_CFG_QM_SYNC_OBJECT_MESSAGE                            0xE86D80
+
+#define mmTPC2_CFG_QM_SYNC_OBJECT_ADDR                               0xE86D84
+
+#define mmTPC2_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xE86D88
+
+#define mmTPC2_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xE86D8C
+
+#define mmTPC2_CFG_QM_TID_BASE_DIM_0                                 0xE86D90
+
+#define mmTPC2_CFG_QM_TID_SIZE_DIM_0                                 0xE86D94
+
+#define mmTPC2_CFG_QM_TID_BASE_DIM_1                                 0xE86D98
+
+#define mmTPC2_CFG_QM_TID_SIZE_DIM_1                                 0xE86D9C
+
+#define mmTPC2_CFG_QM_TID_BASE_DIM_2                                 0xE86DA0
+
+#define mmTPC2_CFG_QM_TID_SIZE_DIM_2                                 0xE86DA4
+
+#define mmTPC2_CFG_QM_TID_BASE_DIM_3                                 0xE86DA8
+
+#define mmTPC2_CFG_QM_TID_SIZE_DIM_3                                 0xE86DAC
+
+#define mmTPC2_CFG_QM_TID_BASE_DIM_4                                 0xE86DB0
+
+#define mmTPC2_CFG_QM_TID_SIZE_DIM_4                                 0xE86DB4
+
+#define mmTPC2_CFG_QM_KERNEL_CONFIG                                  0xE86DB8
+
+#define mmTPC2_CFG_QM_KERNEL_ID                                      0xE86DBC
+
+#define mmTPC2_CFG_QM_SRF_0                                          0xE86DC0
+
+#define mmTPC2_CFG_QM_SRF_1                                          0xE86DC4
+
+#define mmTPC2_CFG_QM_SRF_2                                          0xE86DC8
+
+#define mmTPC2_CFG_QM_SRF_3                                          0xE86DCC
+
+#define mmTPC2_CFG_QM_SRF_4                                          0xE86DD0
+
+#define mmTPC2_CFG_QM_SRF_5                                          0xE86DD4
+
+#define mmTPC2_CFG_QM_SRF_6                                          0xE86DD8
+
+#define mmTPC2_CFG_QM_SRF_7                                          0xE86DDC
+
+#define mmTPC2_CFG_QM_SRF_8                                          0xE86DE0
+
+#define mmTPC2_CFG_QM_SRF_9                                          0xE86DE4
+
+#define mmTPC2_CFG_QM_SRF_10                                         0xE86DE8
+
+#define mmTPC2_CFG_QM_SRF_11                                         0xE86DEC
+
+#define mmTPC2_CFG_QM_SRF_12                                         0xE86DF0
+
+#define mmTPC2_CFG_QM_SRF_13                                         0xE86DF4
+
+#define mmTPC2_CFG_QM_SRF_14                                         0xE86DF8
+
+#define mmTPC2_CFG_QM_SRF_15                                         0xE86DFC
+
+#define mmTPC2_CFG_QM_SRF_16                                         0xE86E00
+
+#define mmTPC2_CFG_QM_SRF_17                                         0xE86E04
+
+#define mmTPC2_CFG_QM_SRF_18                                         0xE86E08
+
+#define mmTPC2_CFG_QM_SRF_19                                         0xE86E0C
+
+#define mmTPC2_CFG_QM_SRF_20                                         0xE86E10
+
+#define mmTPC2_CFG_QM_SRF_21                                         0xE86E14
+
+#define mmTPC2_CFG_QM_SRF_22                                         0xE86E18
+
+#define mmTPC2_CFG_QM_SRF_23                                         0xE86E1C
+
+#define mmTPC2_CFG_QM_SRF_24                                         0xE86E20
+
+#define mmTPC2_CFG_QM_SRF_25                                         0xE86E24
+
+#define mmTPC2_CFG_QM_SRF_26                                         0xE86E28
+
+#define mmTPC2_CFG_QM_SRF_27                                         0xE86E2C
+
+#define mmTPC2_CFG_QM_SRF_28                                         0xE86E30
+
+#define mmTPC2_CFG_QM_SRF_29                                         0xE86E34
+
+#define mmTPC2_CFG_QM_SRF_30                                         0xE86E38
+
+#define mmTPC2_CFG_QM_SRF_31                                         0xE86E3C
+
+#endif /* ASIC_REG_TPC2_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_qm_regs.h
new file mode 100644
index 000000000000..2919e2fa58f8
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc2_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC2_QM_REGS_H_
+#define ASIC_REG_TPC2_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC2_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC2_QM_GLBL_CFG0                                          0xE88000
+
+#define mmTPC2_QM_GLBL_CFG1                                          0xE88004
+
+#define mmTPC2_QM_GLBL_PROT                                          0xE88008
+
+#define mmTPC2_QM_GLBL_ERR_CFG                                       0xE8800C
+
+#define mmTPC2_QM_GLBL_SECURE_PROPS_0                                0xE88010
+
+#define mmTPC2_QM_GLBL_SECURE_PROPS_1                                0xE88014
+
+#define mmTPC2_QM_GLBL_SECURE_PROPS_2                                0xE88018
+
+#define mmTPC2_QM_GLBL_SECURE_PROPS_3                                0xE8801C
+
+#define mmTPC2_QM_GLBL_SECURE_PROPS_4                                0xE88020
+
+#define mmTPC2_QM_GLBL_NON_SECURE_PROPS_0                            0xE88024
+
+#define mmTPC2_QM_GLBL_NON_SECURE_PROPS_1                            0xE88028
+
+#define mmTPC2_QM_GLBL_NON_SECURE_PROPS_2                            0xE8802C
+
+#define mmTPC2_QM_GLBL_NON_SECURE_PROPS_3                            0xE88030
+
+#define mmTPC2_QM_GLBL_NON_SECURE_PROPS_4                            0xE88034
+
+#define mmTPC2_QM_GLBL_STS0                                          0xE88038
+
+#define mmTPC2_QM_GLBL_STS1_0                                        0xE88040
+
+#define mmTPC2_QM_GLBL_STS1_1                                        0xE88044
+
+#define mmTPC2_QM_GLBL_STS1_2                                        0xE88048
+
+#define mmTPC2_QM_GLBL_STS1_3                                        0xE8804C
+
+#define mmTPC2_QM_GLBL_STS1_4                                        0xE88050
+
+#define mmTPC2_QM_GLBL_MSG_EN_0                                      0xE88054
+
+#define mmTPC2_QM_GLBL_MSG_EN_1                                      0xE88058
+
+#define mmTPC2_QM_GLBL_MSG_EN_2                                      0xE8805C
+
+#define mmTPC2_QM_GLBL_MSG_EN_3                                      0xE88060
+
+#define mmTPC2_QM_GLBL_MSG_EN_4                                      0xE88068
+
+#define mmTPC2_QM_PQ_BASE_LO_0                                       0xE88070
+
+#define mmTPC2_QM_PQ_BASE_LO_1                                       0xE88074
+
+#define mmTPC2_QM_PQ_BASE_LO_2                                       0xE88078
+
+#define mmTPC2_QM_PQ_BASE_LO_3                                       0xE8807C
+
+#define mmTPC2_QM_PQ_BASE_HI_0                                       0xE88080
+
+#define mmTPC2_QM_PQ_BASE_HI_1                                       0xE88084
+
+#define mmTPC2_QM_PQ_BASE_HI_2                                       0xE88088
+
+#define mmTPC2_QM_PQ_BASE_HI_3                                       0xE8808C
+
+#define mmTPC2_QM_PQ_SIZE_0                                          0xE88090
+
+#define mmTPC2_QM_PQ_SIZE_1                                          0xE88094
+
+#define mmTPC2_QM_PQ_SIZE_2                                          0xE88098
+
+#define mmTPC2_QM_PQ_SIZE_3                                          0xE8809C
+
+#define mmTPC2_QM_PQ_PI_0                                            0xE880A0
+
+#define mmTPC2_QM_PQ_PI_1                                            0xE880A4
+
+#define mmTPC2_QM_PQ_PI_2                                            0xE880A8
+
+#define mmTPC2_QM_PQ_PI_3                                            0xE880AC
+
+#define mmTPC2_QM_PQ_CI_0                                            0xE880B0
+
+#define mmTPC2_QM_PQ_CI_1                                            0xE880B4
+
+#define mmTPC2_QM_PQ_CI_2                                            0xE880B8
+
+#define mmTPC2_QM_PQ_CI_3                                            0xE880BC
+
+#define mmTPC2_QM_PQ_CFG0_0                                          0xE880C0
+
+#define mmTPC2_QM_PQ_CFG0_1                                          0xE880C4
+
+#define mmTPC2_QM_PQ_CFG0_2                                          0xE880C8
+
+#define mmTPC2_QM_PQ_CFG0_3                                          0xE880CC
+
+#define mmTPC2_QM_PQ_CFG1_0                                          0xE880D0
+
+#define mmTPC2_QM_PQ_CFG1_1                                          0xE880D4
+
+#define mmTPC2_QM_PQ_CFG1_2                                          0xE880D8
+
+#define mmTPC2_QM_PQ_CFG1_3                                          0xE880DC
+
+#define mmTPC2_QM_PQ_ARUSER_31_11_0                                  0xE880E0
+
+#define mmTPC2_QM_PQ_ARUSER_31_11_1                                  0xE880E4
+
+#define mmTPC2_QM_PQ_ARUSER_31_11_2                                  0xE880E8
+
+#define mmTPC2_QM_PQ_ARUSER_31_11_3                                  0xE880EC
+
+#define mmTPC2_QM_PQ_STS0_0                                          0xE880F0
+
+#define mmTPC2_QM_PQ_STS0_1                                          0xE880F4
+
+#define mmTPC2_QM_PQ_STS0_2                                          0xE880F8
+
+#define mmTPC2_QM_PQ_STS0_3                                          0xE880FC
+
+#define mmTPC2_QM_PQ_STS1_0                                          0xE88100
+
+#define mmTPC2_QM_PQ_STS1_1                                          0xE88104
+
+#define mmTPC2_QM_PQ_STS1_2                                          0xE88108
+
+#define mmTPC2_QM_PQ_STS1_3                                          0xE8810C
+
+#define mmTPC2_QM_CQ_CFG0_0                                          0xE88110
+
+#define mmTPC2_QM_CQ_CFG0_1                                          0xE88114
+
+#define mmTPC2_QM_CQ_CFG0_2                                          0xE88118
+
+#define mmTPC2_QM_CQ_CFG0_3                                          0xE8811C
+
+#define mmTPC2_QM_CQ_CFG0_4                                          0xE88120
+
+#define mmTPC2_QM_CQ_CFG1_0                                          0xE88124
+
+#define mmTPC2_QM_CQ_CFG1_1                                          0xE88128
+
+#define mmTPC2_QM_CQ_CFG1_2                                          0xE8812C
+
+#define mmTPC2_QM_CQ_CFG1_3                                          0xE88130
+
+#define mmTPC2_QM_CQ_CFG1_4                                          0xE88134
+
+#define mmTPC2_QM_CQ_ARUSER_31_11_0                                  0xE88138
+
+#define mmTPC2_QM_CQ_ARUSER_31_11_1                                  0xE8813C
+
+#define mmTPC2_QM_CQ_ARUSER_31_11_2                                  0xE88140
+
+#define mmTPC2_QM_CQ_ARUSER_31_11_3                                  0xE88144
+
+#define mmTPC2_QM_CQ_ARUSER_31_11_4                                  0xE88148
+
+#define mmTPC2_QM_CQ_STS0_0                                          0xE8814C
+
+#define mmTPC2_QM_CQ_STS0_1                                          0xE88150
+
+#define mmTPC2_QM_CQ_STS0_2                                          0xE88154
+
+#define mmTPC2_QM_CQ_STS0_3                                          0xE88158
+
+#define mmTPC2_QM_CQ_STS0_4                                          0xE8815C
+
+#define mmTPC2_QM_CQ_STS1_0                                          0xE88160
+
+#define mmTPC2_QM_CQ_STS1_1                                          0xE88164
+
+#define mmTPC2_QM_CQ_STS1_2                                          0xE88168
+
+#define mmTPC2_QM_CQ_STS1_3                                          0xE8816C
+
+#define mmTPC2_QM_CQ_STS1_4                                          0xE88170
+
+#define mmTPC2_QM_CQ_PTR_LO_0                                        0xE88174
+
+#define mmTPC2_QM_CQ_PTR_HI_0                                        0xE88178
+
+#define mmTPC2_QM_CQ_TSIZE_0                                         0xE8817C
+
+#define mmTPC2_QM_CQ_CTL_0                                           0xE88180
+
+#define mmTPC2_QM_CQ_PTR_LO_1                                        0xE88184
+
+#define mmTPC2_QM_CQ_PTR_HI_1                                        0xE88188
+
+#define mmTPC2_QM_CQ_TSIZE_1                                         0xE8818C
+
+#define mmTPC2_QM_CQ_CTL_1                                           0xE88190
+
+#define mmTPC2_QM_CQ_PTR_LO_2                                        0xE88194
+
+#define mmTPC2_QM_CQ_PTR_HI_2                                        0xE88198
+
+#define mmTPC2_QM_CQ_TSIZE_2                                         0xE8819C
+
+#define mmTPC2_QM_CQ_CTL_2                                           0xE881A0
+
+#define mmTPC2_QM_CQ_PTR_LO_3                                        0xE881A4
+
+#define mmTPC2_QM_CQ_PTR_HI_3                                        0xE881A8
+
+#define mmTPC2_QM_CQ_TSIZE_3                                         0xE881AC
+
+#define mmTPC2_QM_CQ_CTL_3                                           0xE881B0
+
+#define mmTPC2_QM_CQ_PTR_LO_4                                        0xE881B4
+
+#define mmTPC2_QM_CQ_PTR_HI_4                                        0xE881B8
+
+#define mmTPC2_QM_CQ_TSIZE_4                                         0xE881BC
+
+#define mmTPC2_QM_CQ_CTL_4                                           0xE881C0
+
+#define mmTPC2_QM_CQ_PTR_LO_STS_0                                    0xE881C4
+
+#define mmTPC2_QM_CQ_PTR_LO_STS_1                                    0xE881C8
+
+#define mmTPC2_QM_CQ_PTR_LO_STS_2                                    0xE881CC
+
+#define mmTPC2_QM_CQ_PTR_LO_STS_3                                    0xE881D0
+
+#define mmTPC2_QM_CQ_PTR_LO_STS_4                                    0xE881D4
+
+#define mmTPC2_QM_CQ_PTR_HI_STS_0                                    0xE881D8
+
+#define mmTPC2_QM_CQ_PTR_HI_STS_1                                    0xE881DC
+
+#define mmTPC2_QM_CQ_PTR_HI_STS_2                                    0xE881E0
+
+#define mmTPC2_QM_CQ_PTR_HI_STS_3                                    0xE881E4
+
+#define mmTPC2_QM_CQ_PTR_HI_STS_4                                    0xE881E8
+
+#define mmTPC2_QM_CQ_TSIZE_STS_0                                     0xE881EC
+
+#define mmTPC2_QM_CQ_TSIZE_STS_1                                     0xE881F0
+
+#define mmTPC2_QM_CQ_TSIZE_STS_2                                     0xE881F4
+
+#define mmTPC2_QM_CQ_TSIZE_STS_3                                     0xE881F8
+
+#define mmTPC2_QM_CQ_TSIZE_STS_4                                     0xE881FC
+
+#define mmTPC2_QM_CQ_CTL_STS_0                                       0xE88200
+
+#define mmTPC2_QM_CQ_CTL_STS_1                                       0xE88204
+
+#define mmTPC2_QM_CQ_CTL_STS_2                                       0xE88208
+
+#define mmTPC2_QM_CQ_CTL_STS_3                                       0xE8820C
+
+#define mmTPC2_QM_CQ_CTL_STS_4                                       0xE88210
+
+#define mmTPC2_QM_CQ_IFIFO_CNT_0                                     0xE88214
+
+#define mmTPC2_QM_CQ_IFIFO_CNT_1                                     0xE88218
+
+#define mmTPC2_QM_CQ_IFIFO_CNT_2                                     0xE8821C
+
+#define mmTPC2_QM_CQ_IFIFO_CNT_3                                     0xE88220
+
+#define mmTPC2_QM_CQ_IFIFO_CNT_4                                     0xE88224
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_0                             0xE88228
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_1                             0xE8822C
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_2                             0xE88230
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_3                             0xE88234
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_LO_4                             0xE88238
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_0                             0xE8823C
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_1                             0xE88240
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_2                             0xE88244
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_3                             0xE88248
+
+#define mmTPC2_QM_CP_MSG_BASE0_ADDR_HI_4                             0xE8824C
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_0                             0xE88250
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_1                             0xE88254
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_2                             0xE88258
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_3                             0xE8825C
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_LO_4                             0xE88260
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_0                             0xE88264
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_1                             0xE88268
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_2                             0xE8826C
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_3                             0xE88270
+
+#define mmTPC2_QM_CP_MSG_BASE1_ADDR_HI_4                             0xE88274
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_0                             0xE88278
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_1                             0xE8827C
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_2                             0xE88280
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_3                             0xE88284
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_LO_4                             0xE88288
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_0                             0xE8828C
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_1                             0xE88290
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_2                             0xE88294
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_3                             0xE88298
+
+#define mmTPC2_QM_CP_MSG_BASE2_ADDR_HI_4                             0xE8829C
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_0                             0xE882A0
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_1                             0xE882A4
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_2                             0xE882A8
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_3                             0xE882AC
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_LO_4                             0xE882B0
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_0                             0xE882B4
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_1                             0xE882B8
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_2                             0xE882BC
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_3                             0xE882C0
+
+#define mmTPC2_QM_CP_MSG_BASE3_ADDR_HI_4                             0xE882C4
+
+#define mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_0                             0xE882C8
+
+#define mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_1                             0xE882CC
+
+#define mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_2                             0xE882D0
+
+#define mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_3                             0xE882D4
+
+#define mmTPC2_QM_CP_LDMA_TSIZE_OFFSET_4                             0xE882D8
+
+#define mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xE882E0
+
+#define mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xE882E4
+
+#define mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xE882E8
+
+#define mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xE882EC
+
+#define mmTPC2_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xE882F0
+
+#define mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xE882F4
+
+#define mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xE882F8
+
+#define mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xE882FC
+
+#define mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xE88300
+
+#define mmTPC2_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xE88304
+
+#define mmTPC2_QM_CP_FENCE0_RDATA_0                                  0xE88308
+
+#define mmTPC2_QM_CP_FENCE0_RDATA_1                                  0xE8830C
+
+#define mmTPC2_QM_CP_FENCE0_RDATA_2                                  0xE88310
+
+#define mmTPC2_QM_CP_FENCE0_RDATA_3                                  0xE88314
+
+#define mmTPC2_QM_CP_FENCE0_RDATA_4                                  0xE88318
+
+#define mmTPC2_QM_CP_FENCE1_RDATA_0                                  0xE8831C
+
+#define mmTPC2_QM_CP_FENCE1_RDATA_1                                  0xE88320
+
+#define mmTPC2_QM_CP_FENCE1_RDATA_2                                  0xE88324
+
+#define mmTPC2_QM_CP_FENCE1_RDATA_3                                  0xE88328
+
+#define mmTPC2_QM_CP_FENCE1_RDATA_4                                  0xE8832C
+
+#define mmTPC2_QM_CP_FENCE2_RDATA_0                                  0xE88330
+
+#define mmTPC2_QM_CP_FENCE2_RDATA_1                                  0xE88334
+
+#define mmTPC2_QM_CP_FENCE2_RDATA_2                                  0xE88338
+
+#define mmTPC2_QM_CP_FENCE2_RDATA_3                                  0xE8833C
+
+#define mmTPC2_QM_CP_FENCE2_RDATA_4                                  0xE88340
+
+#define mmTPC2_QM_CP_FENCE3_RDATA_0                                  0xE88344
+
+#define mmTPC2_QM_CP_FENCE3_RDATA_1                                  0xE88348
+
+#define mmTPC2_QM_CP_FENCE3_RDATA_2                                  0xE8834C
+
+#define mmTPC2_QM_CP_FENCE3_RDATA_3                                  0xE88350
+
+#define mmTPC2_QM_CP_FENCE3_RDATA_4                                  0xE88354
+
+#define mmTPC2_QM_CP_FENCE0_CNT_0                                    0xE88358
+
+#define mmTPC2_QM_CP_FENCE0_CNT_1                                    0xE8835C
+
+#define mmTPC2_QM_CP_FENCE0_CNT_2                                    0xE88360
+
+#define mmTPC2_QM_CP_FENCE0_CNT_3                                    0xE88364
+
+#define mmTPC2_QM_CP_FENCE0_CNT_4                                    0xE88368
+
+#define mmTPC2_QM_CP_FENCE1_CNT_0                                    0xE8836C
+
+#define mmTPC2_QM_CP_FENCE1_CNT_1                                    0xE88370
+
+#define mmTPC2_QM_CP_FENCE1_CNT_2                                    0xE88374
+
+#define mmTPC2_QM_CP_FENCE1_CNT_3                                    0xE88378
+
+#define mmTPC2_QM_CP_FENCE1_CNT_4                                    0xE8837C
+
+#define mmTPC2_QM_CP_FENCE2_CNT_0                                    0xE88380
+
+#define mmTPC2_QM_CP_FENCE2_CNT_1                                    0xE88384
+
+#define mmTPC2_QM_CP_FENCE2_CNT_2                                    0xE88388
+
+#define mmTPC2_QM_CP_FENCE2_CNT_3                                    0xE8838C
+
+#define mmTPC2_QM_CP_FENCE2_CNT_4                                    0xE88390
+
+#define mmTPC2_QM_CP_FENCE3_CNT_0                                    0xE88394
+
+#define mmTPC2_QM_CP_FENCE3_CNT_1                                    0xE88398
+
+#define mmTPC2_QM_CP_FENCE3_CNT_2                                    0xE8839C
+
+#define mmTPC2_QM_CP_FENCE3_CNT_3                                    0xE883A0
+
+#define mmTPC2_QM_CP_FENCE3_CNT_4                                    0xE883A4
+
+#define mmTPC2_QM_CP_STS_0                                           0xE883A8
+
+#define mmTPC2_QM_CP_STS_1                                           0xE883AC
+
+#define mmTPC2_QM_CP_STS_2                                           0xE883B0
+
+#define mmTPC2_QM_CP_STS_3                                           0xE883B4
+
+#define mmTPC2_QM_CP_STS_4                                           0xE883B8
+
+#define mmTPC2_QM_CP_CURRENT_INST_LO_0                               0xE883BC
+
+#define mmTPC2_QM_CP_CURRENT_INST_LO_1                               0xE883C0
+
+#define mmTPC2_QM_CP_CURRENT_INST_LO_2                               0xE883C4
+
+#define mmTPC2_QM_CP_CURRENT_INST_LO_3                               0xE883C8
+
+#define mmTPC2_QM_CP_CURRENT_INST_LO_4                               0xE883CC
+
+#define mmTPC2_QM_CP_CURRENT_INST_HI_0                               0xE883D0
+
+#define mmTPC2_QM_CP_CURRENT_INST_HI_1                               0xE883D4
+
+#define mmTPC2_QM_CP_CURRENT_INST_HI_2                               0xE883D8
+
+#define mmTPC2_QM_CP_CURRENT_INST_HI_3                               0xE883DC
+
+#define mmTPC2_QM_CP_CURRENT_INST_HI_4                               0xE883E0
+
+#define mmTPC2_QM_CP_BARRIER_CFG_0                                   0xE883F4
+
+#define mmTPC2_QM_CP_BARRIER_CFG_1                                   0xE883F8
+
+#define mmTPC2_QM_CP_BARRIER_CFG_2                                   0xE883FC
+
+#define mmTPC2_QM_CP_BARRIER_CFG_3                                   0xE88400
+
+#define mmTPC2_QM_CP_BARRIER_CFG_4                                   0xE88404
+
+#define mmTPC2_QM_CP_DBG_0_0                                         0xE88408
+
+#define mmTPC2_QM_CP_DBG_0_1                                         0xE8840C
+
+#define mmTPC2_QM_CP_DBG_0_2                                         0xE88410
+
+#define mmTPC2_QM_CP_DBG_0_3                                         0xE88414
+
+#define mmTPC2_QM_CP_DBG_0_4                                         0xE88418
+
+#define mmTPC2_QM_CP_ARUSER_31_11_0                                  0xE8841C
+
+#define mmTPC2_QM_CP_ARUSER_31_11_1                                  0xE88420
+
+#define mmTPC2_QM_CP_ARUSER_31_11_2                                  0xE88424
+
+#define mmTPC2_QM_CP_ARUSER_31_11_3                                  0xE88428
+
+#define mmTPC2_QM_CP_ARUSER_31_11_4                                  0xE8842C
+
+#define mmTPC2_QM_CP_AWUSER_31_11_0                                  0xE88430
+
+#define mmTPC2_QM_CP_AWUSER_31_11_1                                  0xE88434
+
+#define mmTPC2_QM_CP_AWUSER_31_11_2                                  0xE88438
+
+#define mmTPC2_QM_CP_AWUSER_31_11_3                                  0xE8843C
+
+#define mmTPC2_QM_CP_AWUSER_31_11_4                                  0xE88440
+
+#define mmTPC2_QM_ARB_CFG_0                                          0xE88A00
+
+#define mmTPC2_QM_ARB_CHOISE_Q_PUSH                                  0xE88A04
+
+#define mmTPC2_QM_ARB_WRR_WEIGHT_0                                   0xE88A08
+
+#define mmTPC2_QM_ARB_WRR_WEIGHT_1                                   0xE88A0C
+
+#define mmTPC2_QM_ARB_WRR_WEIGHT_2                                   0xE88A10
+
+#define mmTPC2_QM_ARB_WRR_WEIGHT_3                                   0xE88A14
+
+#define mmTPC2_QM_ARB_CFG_1                                          0xE88A18
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_0                               0xE88A20
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_1                               0xE88A24
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_2                               0xE88A28
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_3                               0xE88A2C
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_4                               0xE88A30
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_5                               0xE88A34
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_6                               0xE88A38
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_7                               0xE88A3C
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_8                               0xE88A40
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_9                               0xE88A44
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_10                              0xE88A48
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_11                              0xE88A4C
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_12                              0xE88A50
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_13                              0xE88A54
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_14                              0xE88A58
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_15                              0xE88A5C
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_16                              0xE88A60
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_17                              0xE88A64
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_18                              0xE88A68
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_19                              0xE88A6C
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_20                              0xE88A70
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_21                              0xE88A74
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_22                              0xE88A78
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_23                              0xE88A7C
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_24                              0xE88A80
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_25                              0xE88A84
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_26                              0xE88A88
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_27                              0xE88A8C
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_28                              0xE88A90
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_29                              0xE88A94
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_30                              0xE88A98
+
+#define mmTPC2_QM_ARB_MST_AVAIL_CRED_31                              0xE88A9C
+
+#define mmTPC2_QM_ARB_MST_CRED_INC                                   0xE88AA0
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xE88AA4
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xE88AA8
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xE88AAC
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xE88AB0
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xE88AB4
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xE88AB8
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xE88ABC
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xE88AC0
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xE88AC4
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xE88AC8
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xE88ACC
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xE88AD0
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xE88AD4
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xE88AD8
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xE88ADC
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xE88AE0
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xE88AE4
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xE88AE8
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xE88AEC
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xE88AF0
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xE88AF4
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xE88AF8
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xE88AFC
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xE88B00
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xE88B04
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xE88B08
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xE88B0C
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xE88B10
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xE88B14
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xE88B18
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xE88B1C
+
+#define mmTPC2_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xE88B20
+
+#define mmTPC2_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xE88B28
+
+#define mmTPC2_QM_ARB_MST_SLAVE_EN                                   0xE88B2C
+
+#define mmTPC2_QM_ARB_MST_QUIET_PER                                  0xE88B34
+
+#define mmTPC2_QM_ARB_SLV_CHOISE_WDT                                 0xE88B38
+
+#define mmTPC2_QM_ARB_SLV_ID                                         0xE88B3C
+
+#define mmTPC2_QM_ARB_MSG_MAX_INFLIGHT                               0xE88B44
+
+#define mmTPC2_QM_ARB_MSG_AWUSER_31_11                               0xE88B48
+
+#define mmTPC2_QM_ARB_MSG_AWUSER_SEC_PROP                            0xE88B4C
+
+#define mmTPC2_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xE88B50
+
+#define mmTPC2_QM_ARB_BASE_LO                                        0xE88B54
+
+#define mmTPC2_QM_ARB_BASE_HI                                        0xE88B58
+
+#define mmTPC2_QM_ARB_STATE_STS                                      0xE88B80
+
+#define mmTPC2_QM_ARB_CHOISE_FULLNESS_STS                            0xE88B84
+
+#define mmTPC2_QM_ARB_MSG_STS                                        0xE88B88
+
+#define mmTPC2_QM_ARB_SLV_CHOISE_Q_HEAD                              0xE88B8C
+
+#define mmTPC2_QM_ARB_ERR_CAUSE                                      0xE88B9C
+
+#define mmTPC2_QM_ARB_ERR_MSG_EN                                     0xE88BA0
+
+#define mmTPC2_QM_ARB_ERR_STS_DRP                                    0xE88BA8
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_0                                 0xE88BB0
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_1                                 0xE88BB4
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_2                                 0xE88BB8
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_3                                 0xE88BBC
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_4                                 0xE88BC0
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_5                                 0xE88BC4
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_6                                 0xE88BC8
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_7                                 0xE88BCC
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_8                                 0xE88BD0
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_9                                 0xE88BD4
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_10                                0xE88BD8
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_11                                0xE88BDC
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_12                                0xE88BE0
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_13                                0xE88BE4
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_14                                0xE88BE8
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_15                                0xE88BEC
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_16                                0xE88BF0
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_17                                0xE88BF4
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_18                                0xE88BF8
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_19                                0xE88BFC
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_20                                0xE88C00
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_21                                0xE88C04
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_22                                0xE88C08
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_23                                0xE88C0C
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_24                                0xE88C10
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_25                                0xE88C14
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_26                                0xE88C18
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_27                                0xE88C1C
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_28                                0xE88C20
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_29                                0xE88C24
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_30                                0xE88C28
+
+#define mmTPC2_QM_ARB_MST_CRED_STS_31                                0xE88C2C
+
+#define mmTPC2_QM_CGM_CFG                                            0xE88C70
+
+#define mmTPC2_QM_CGM_STS                                            0xE88C74
+
+#define mmTPC2_QM_CGM_CFG1                                           0xE88C78
+
+#define mmTPC2_QM_LOCAL_RANGE_BASE                                   0xE88C80
+
+#define mmTPC2_QM_LOCAL_RANGE_SIZE                                   0xE88C84
+
+#define mmTPC2_QM_CSMR_STRICT_PRIO_CFG                               0xE88C90
+
+#define mmTPC2_QM_HBW_RD_RATE_LIM_CFG_1                              0xE88C94
+
+#define mmTPC2_QM_LBW_WR_RATE_LIM_CFG_0                              0xE88C98
+
+#define mmTPC2_QM_LBW_WR_RATE_LIM_CFG_1                              0xE88C9C
+
+#define mmTPC2_QM_HBW_RD_RATE_LIM_CFG_0                              0xE88CA0
+
+#define mmTPC2_QM_GLBL_AXCACHE                                       0xE88CA4
+
+#define mmTPC2_QM_IND_GW_APB_CFG                                     0xE88CB0
+
+#define mmTPC2_QM_IND_GW_APB_WDATA                                   0xE88CB4
+
+#define mmTPC2_QM_IND_GW_APB_RDATA                                   0xE88CB8
+
+#define mmTPC2_QM_IND_GW_APB_STATUS                                  0xE88CBC
+
+#define mmTPC2_QM_GLBL_ERR_ADDR_LO                                   0xE88CD0
+
+#define mmTPC2_QM_GLBL_ERR_ADDR_HI                                   0xE88CD4
+
+#define mmTPC2_QM_GLBL_ERR_WDATA                                     0xE88CD8
+
+#define mmTPC2_QM_GLBL_MEM_INIT_BUSY                                 0xE88D00
+
+#endif /* ASIC_REG_TPC2_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_cfg_regs.h
new file mode 100644
index 000000000000..6d42469659f1
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC3_CFG_REGS_H_
+#define ASIC_REG_TPC3_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC3_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xEC6400
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xEC6404
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xEC6408
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xEC640C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xEC6410
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xEC6414
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xEC6418
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xEC641C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xEC6420
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xEC6424
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xEC6428
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xEC642C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xEC6430
+
+#define mmTPC3_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xEC6434
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xEC6438
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xEC643C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xEC6440
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xEC6444
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xEC6448
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xEC644C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xEC6450
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xEC6454
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xEC6458
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xEC645C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xEC6460
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xEC6464
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xEC6468
+
+#define mmTPC3_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xEC646C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xEC6470
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xEC6474
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xEC6478
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xEC647C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xEC6480
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xEC6484
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xEC6488
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xEC648C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xEC6490
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xEC6494
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xEC6498
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xEC649C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xEC64A0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xEC64A4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xEC64A8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xEC64AC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xEC64B0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xEC64B4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xEC64B8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xEC64BC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xEC64C0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xEC64C4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xEC64C8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xEC64CC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xEC64D0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xEC64D4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xEC64D8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xEC64DC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xEC64E0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xEC64E4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xEC64E8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xEC64EC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xEC64F0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xEC64F4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xEC64F8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xEC64FC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xEC6500
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xEC6504
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xEC6508
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xEC650C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xEC6510
+
+#define mmTPC3_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xEC6514
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xEC6518
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xEC651C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xEC6520
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xEC6524
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xEC6528
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xEC652C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xEC6530
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xEC6534
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xEC6538
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xEC653C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xEC6540
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xEC6544
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xEC6548
+
+#define mmTPC3_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xEC654C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xEC6550
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xEC6554
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xEC6558
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xEC655C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xEC6560
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xEC6564
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xEC6568
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xEC656C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xEC6570
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xEC6574
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xEC6578
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xEC657C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xEC6580
+
+#define mmTPC3_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xEC6584
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xEC6588
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xEC658C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xEC6590
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xEC6594
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xEC6598
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xEC659C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xEC65A0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xEC65A4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xEC65A8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xEC65AC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xEC65B0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xEC65B4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xEC65B8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xEC65BC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xEC65C0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xEC65C4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xEC65C8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xEC65CC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xEC65D0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xEC65D4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xEC65D8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xEC65DC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xEC65E0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xEC65E4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xEC65E8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xEC65EC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xEC65F0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xEC65F4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xEC65F8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xEC65FC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xEC6600
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xEC6604
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xEC6608
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xEC660C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xEC6610
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xEC6614
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xEC6618
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xEC661C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xEC6620
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xEC6624
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xEC6628
+
+#define mmTPC3_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xEC662C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xEC6630
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xEC6634
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xEC6638
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xEC663C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xEC6640
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xEC6644
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xEC6648
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xEC664C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xEC6650
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xEC6654
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xEC6658
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xEC665C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xEC6660
+
+#define mmTPC3_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xEC6664
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xEC6668
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xEC666C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xEC6670
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xEC6674
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xEC6678
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xEC667C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xEC6680
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xEC6684
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xEC6688
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xEC668C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xEC6690
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xEC6694
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xEC6698
+
+#define mmTPC3_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xEC669C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xEC66A0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xEC66A4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xEC66A8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xEC66AC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xEC66B0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xEC66B4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xEC66B8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xEC66BC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xEC66C0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xEC66C4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xEC66C8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xEC66CC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xEC66D0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xEC66D4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xEC66D8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xEC66DC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xEC66E0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xEC66E4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xEC66E8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xEC66EC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xEC66F0
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xEC66F4
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xEC66F8
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xEC66FC
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xEC6700
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xEC6704
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xEC6708
+
+#define mmTPC3_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xEC670C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xEC6710
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xEC6714
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xEC6718
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xEC671C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xEC6720
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xEC6724
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xEC6728
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xEC672C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xEC6730
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xEC6734
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xEC6738
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xEC673C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xEC6740
+
+#define mmTPC3_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xEC6744
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xEC6748
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xEC674C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xEC6750
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xEC6754
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xEC6758
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xEC675C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xEC6760
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xEC6764
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xEC6768
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xEC676C
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xEC6770
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xEC6774
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xEC6778
+
+#define mmTPC3_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xEC677C
+
+#define mmTPC3_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xEC6780
+
+#define mmTPC3_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xEC6784
+
+#define mmTPC3_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xEC6788
+
+#define mmTPC3_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xEC678C
+
+#define mmTPC3_CFG_KERNEL_TID_BASE_DIM_0                             0xEC6790
+
+#define mmTPC3_CFG_KERNEL_TID_SIZE_DIM_0                             0xEC6794
+
+#define mmTPC3_CFG_KERNEL_TID_BASE_DIM_1                             0xEC6798
+
+#define mmTPC3_CFG_KERNEL_TID_SIZE_DIM_1                             0xEC679C
+
+#define mmTPC3_CFG_KERNEL_TID_BASE_DIM_2                             0xEC67A0
+
+#define mmTPC3_CFG_KERNEL_TID_SIZE_DIM_2                             0xEC67A4
+
+#define mmTPC3_CFG_KERNEL_TID_BASE_DIM_3                             0xEC67A8
+
+#define mmTPC3_CFG_KERNEL_TID_SIZE_DIM_3                             0xEC67AC
+
+#define mmTPC3_CFG_KERNEL_TID_BASE_DIM_4                             0xEC67B0
+
+#define mmTPC3_CFG_KERNEL_TID_SIZE_DIM_4                             0xEC67B4
+
+#define mmTPC3_CFG_KERNEL_KERNEL_CONFIG                              0xEC67B8
+
+#define mmTPC3_CFG_KERNEL_KERNEL_ID                                  0xEC67BC
+
+#define mmTPC3_CFG_KERNEL_SRF_0                                      0xEC67C0
+
+#define mmTPC3_CFG_KERNEL_SRF_1                                      0xEC67C4
+
+#define mmTPC3_CFG_KERNEL_SRF_2                                      0xEC67C8
+
+#define mmTPC3_CFG_KERNEL_SRF_3                                      0xEC67CC
+
+#define mmTPC3_CFG_KERNEL_SRF_4                                      0xEC67D0
+
+#define mmTPC3_CFG_KERNEL_SRF_5                                      0xEC67D4
+
+#define mmTPC3_CFG_KERNEL_SRF_6                                      0xEC67D8
+
+#define mmTPC3_CFG_KERNEL_SRF_7                                      0xEC67DC
+
+#define mmTPC3_CFG_KERNEL_SRF_8                                      0xEC67E0
+
+#define mmTPC3_CFG_KERNEL_SRF_9                                      0xEC67E4
+
+#define mmTPC3_CFG_KERNEL_SRF_10                                     0xEC67E8
+
+#define mmTPC3_CFG_KERNEL_SRF_11                                     0xEC67EC
+
+#define mmTPC3_CFG_KERNEL_SRF_12                                     0xEC67F0
+
+#define mmTPC3_CFG_KERNEL_SRF_13                                     0xEC67F4
+
+#define mmTPC3_CFG_KERNEL_SRF_14                                     0xEC67F8
+
+#define mmTPC3_CFG_KERNEL_SRF_15                                     0xEC67FC
+
+#define mmTPC3_CFG_KERNEL_SRF_16                                     0xEC6800
+
+#define mmTPC3_CFG_KERNEL_SRF_17                                     0xEC6804
+
+#define mmTPC3_CFG_KERNEL_SRF_18                                     0xEC6808
+
+#define mmTPC3_CFG_KERNEL_SRF_19                                     0xEC680C
+
+#define mmTPC3_CFG_KERNEL_SRF_20                                     0xEC6810
+
+#define mmTPC3_CFG_KERNEL_SRF_21                                     0xEC6814
+
+#define mmTPC3_CFG_KERNEL_SRF_22                                     0xEC6818
+
+#define mmTPC3_CFG_KERNEL_SRF_23                                     0xEC681C
+
+#define mmTPC3_CFG_KERNEL_SRF_24                                     0xEC6820
+
+#define mmTPC3_CFG_KERNEL_SRF_25                                     0xEC6824
+
+#define mmTPC3_CFG_KERNEL_SRF_26                                     0xEC6828
+
+#define mmTPC3_CFG_KERNEL_SRF_27                                     0xEC682C
+
+#define mmTPC3_CFG_KERNEL_SRF_28                                     0xEC6830
+
+#define mmTPC3_CFG_KERNEL_SRF_29                                     0xEC6834
+
+#define mmTPC3_CFG_KERNEL_SRF_30                                     0xEC6838
+
+#define mmTPC3_CFG_KERNEL_SRF_31                                     0xEC683C
+
+#define mmTPC3_CFG_ROUND_CSR                                         0xEC68FC
+
+#define mmTPC3_CFG_PROT                                              0xEC6900
+
+#define mmTPC3_CFG_SEMAPHORE                                         0xEC6908
+
+#define mmTPC3_CFG_VFLAGS                                            0xEC690C
+
+#define mmTPC3_CFG_SFLAGS                                            0xEC6910
+
+#define mmTPC3_CFG_LFSR_POLYNOM                                      0xEC6918
+
+#define mmTPC3_CFG_STATUS                                            0xEC691C
+
+#define mmTPC3_CFG_CFG_BASE_ADDRESS_HIGH                             0xEC6920
+
+#define mmTPC3_CFG_CFG_SUBTRACT_VALUE                                0xEC6924
+
+#define mmTPC3_CFG_SM_BASE_ADDRESS_HIGH                              0xEC692C
+
+#define mmTPC3_CFG_TPC_CMD                                           0xEC6930
+
+#define mmTPC3_CFG_TPC_EXECUTE                                       0xEC6938
+
+#define mmTPC3_CFG_TPC_STALL                                         0xEC693C
+
+#define mmTPC3_CFG_ICACHE_BASE_ADDERESS_LOW                          0xEC6940
+
+#define mmTPC3_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xEC6944
+
+#define mmTPC3_CFG_RD_RATE_LIMIT                                     0xEC6948
+
+#define mmTPC3_CFG_WR_RATE_LIMIT                                     0xEC6950
+
+#define mmTPC3_CFG_MSS_CONFIG                                        0xEC6954
+
+#define mmTPC3_CFG_TPC_INTR_CAUSE                                    0xEC6958
+
+#define mmTPC3_CFG_TPC_INTR_MASK                                     0xEC695C
+
+#define mmTPC3_CFG_WQ_CREDITS                                        0xEC6960
+
+#define mmTPC3_CFG_ARUSER_LO                                         0xEC6964
+
+#define mmTPC3_CFG_ARUSER_HI                                         0xEC6968
+
+#define mmTPC3_CFG_AWUSER_LO                                         0xEC696C
+
+#define mmTPC3_CFG_AWUSER_HI                                         0xEC6970
+
+#define mmTPC3_CFG_OPCODE_EXEC                                       0xEC6974
+
+#define mmTPC3_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xEC6978
+
+#define mmTPC3_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xEC697C
+
+#define mmTPC3_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xEC6980
+
+#define mmTPC3_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xEC6984
+
+#define mmTPC3_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xEC6988
+
+#define mmTPC3_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xEC698C
+
+#define mmTPC3_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xEC6990
+
+#define mmTPC3_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xEC6994
+
+#define mmTPC3_CFG_TSB_CFG_MAX_SIZE                                  0xEC6998
+
+#define mmTPC3_CFG_TSB_CFG                                           0xEC699C
+
+#define mmTPC3_CFG_DBGMEM_ADD                                        0xEC69A0
+
+#define mmTPC3_CFG_DBGMEM_DATA_WR                                    0xEC69A4
+
+#define mmTPC3_CFG_DBGMEM_DATA_RD                                    0xEC69A8
+
+#define mmTPC3_CFG_DBGMEM_CTRL                                       0xEC69AC
+
+#define mmTPC3_CFG_DBGMEM_RC                                         0xEC69B0
+
+#define mmTPC3_CFG_TSB_INFLIGHT_CNTR                                 0xEC69B4
+
+#define mmTPC3_CFG_WQ_INFLIGHT_CNTR                                  0xEC69B8
+
+#define mmTPC3_CFG_WQ_LBW_TOTAL_CNTR                                 0xEC69BC
+
+#define mmTPC3_CFG_WQ_HBW_TOTAL_CNTR                                 0xEC69C0
+
+#define mmTPC3_CFG_IRQ_OCCOUPY_CNTR                                  0xEC69C4
+
+#define mmTPC3_CFG_FUNC_MBIST_CNTRL                                  0xEC69D0
+
+#define mmTPC3_CFG_FUNC_MBIST_PAT                                    0xEC69D4
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_0                                  0xEC69D8
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_1                                  0xEC69DC
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_2                                  0xEC69E0
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_3                                  0xEC69E4
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_4                                  0xEC69E8
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_5                                  0xEC69EC
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_6                                  0xEC69F0
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_7                                  0xEC69F4
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_8                                  0xEC69F8
+
+#define mmTPC3_CFG_FUNC_MBIST_MEM_9                                  0xEC69FC
+
+#define mmTPC3_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xEC6A00
+
+#define mmTPC3_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xEC6A04
+
+#define mmTPC3_CFG_QM_TENSOR_0_PADDING_VALUE                         0xEC6A08
+
+#define mmTPC3_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xEC6A0C
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xEC6A10
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xEC6A14
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xEC6A18
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xEC6A1C
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xEC6A20
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xEC6A24
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xEC6A28
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xEC6A2C
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xEC6A30
+
+#define mmTPC3_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xEC6A34
+
+#define mmTPC3_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xEC6A38
+
+#define mmTPC3_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xEC6A3C
+
+#define mmTPC3_CFG_QM_TENSOR_1_PADDING_VALUE                         0xEC6A40
+
+#define mmTPC3_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xEC6A44
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xEC6A48
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xEC6A4C
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xEC6A50
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xEC6A54
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xEC6A58
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xEC6A5C
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xEC6A60
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xEC6A64
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xEC6A68
+
+#define mmTPC3_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xEC6A6C
+
+#define mmTPC3_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xEC6A70
+
+#define mmTPC3_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xEC6A74
+
+#define mmTPC3_CFG_QM_TENSOR_2_PADDING_VALUE                         0xEC6A78
+
+#define mmTPC3_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xEC6A7C
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xEC6A80
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xEC6A84
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xEC6A88
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xEC6A8C
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xEC6A90
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xEC6A94
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xEC6A98
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xEC6A9C
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xEC6AA0
+
+#define mmTPC3_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xEC6AA4
+
+#define mmTPC3_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xEC6AA8
+
+#define mmTPC3_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xEC6AAC
+
+#define mmTPC3_CFG_QM_TENSOR_3_PADDING_VALUE                         0xEC6AB0
+
+#define mmTPC3_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xEC6AB4
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xEC6AB8
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xEC6ABC
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xEC6AC0
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xEC6AC4
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xEC6AC8
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xEC6ACC
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xEC6AD0
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xEC6AD4
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xEC6AD8
+
+#define mmTPC3_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xEC6ADC
+
+#define mmTPC3_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xEC6AE0
+
+#define mmTPC3_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xEC6AE4
+
+#define mmTPC3_CFG_QM_TENSOR_4_PADDING_VALUE                         0xEC6AE8
+
+#define mmTPC3_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xEC6AEC
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xEC6AF0
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xEC6AF4
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xEC6AF8
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xEC6AFC
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xEC6B00
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xEC6B04
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xEC6B08
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xEC6B0C
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xEC6B10
+
+#define mmTPC3_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xEC6B14
+
+#define mmTPC3_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xEC6B18
+
+#define mmTPC3_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xEC6B1C
+
+#define mmTPC3_CFG_QM_TENSOR_5_PADDING_VALUE                         0xEC6B20
+
+#define mmTPC3_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xEC6B24
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xEC6B28
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xEC6B2C
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xEC6B30
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xEC6B34
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xEC6B38
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xEC6B3C
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xEC6B40
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xEC6B44
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xEC6B48
+
+#define mmTPC3_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xEC6B4C
+
+#define mmTPC3_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xEC6B50
+
+#define mmTPC3_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xEC6B54
+
+#define mmTPC3_CFG_QM_TENSOR_6_PADDING_VALUE                         0xEC6B58
+
+#define mmTPC3_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xEC6B5C
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xEC6B60
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xEC6B64
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xEC6B68
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xEC6B6C
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xEC6B70
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xEC6B74
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xEC6B78
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xEC6B7C
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xEC6B80
+
+#define mmTPC3_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xEC6B84
+
+#define mmTPC3_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xEC6B88
+
+#define mmTPC3_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xEC6B8C
+
+#define mmTPC3_CFG_QM_TENSOR_7_PADDING_VALUE                         0xEC6B90
+
+#define mmTPC3_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xEC6B94
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xEC6B98
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xEC6B9C
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xEC6BA0
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xEC6BA4
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xEC6BA8
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xEC6BAC
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xEC6BB0
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xEC6BB4
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xEC6BB8
+
+#define mmTPC3_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xEC6BBC
+
+#define mmTPC3_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xEC6BC0
+
+#define mmTPC3_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xEC6BC4
+
+#define mmTPC3_CFG_QM_TENSOR_8_PADDING_VALUE                         0xEC6BC8
+
+#define mmTPC3_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xEC6BCC
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xEC6BD0
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xEC6BD4
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xEC6BD8
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xEC6BDC
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xEC6BE0
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xEC6BE4
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xEC6BE8
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xEC6BEC
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xEC6BF0
+
+#define mmTPC3_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xEC6BF4
+
+#define mmTPC3_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xEC6BF8
+
+#define mmTPC3_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xEC6BFC
+
+#define mmTPC3_CFG_QM_TENSOR_9_PADDING_VALUE                         0xEC6C00
+
+#define mmTPC3_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xEC6C04
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xEC6C08
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xEC6C0C
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xEC6C10
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xEC6C14
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xEC6C18
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xEC6C1C
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xEC6C20
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xEC6C24
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xEC6C28
+
+#define mmTPC3_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xEC6C2C
+
+#define mmTPC3_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xEC6C30
+
+#define mmTPC3_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xEC6C34
+
+#define mmTPC3_CFG_QM_TENSOR_10_PADDING_VALUE                        0xEC6C38
+
+#define mmTPC3_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xEC6C3C
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xEC6C40
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xEC6C44
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xEC6C48
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xEC6C4C
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xEC6C50
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xEC6C54
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xEC6C58
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xEC6C5C
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xEC6C60
+
+#define mmTPC3_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xEC6C64
+
+#define mmTPC3_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xEC6C68
+
+#define mmTPC3_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xEC6C6C
+
+#define mmTPC3_CFG_QM_TENSOR_11_PADDING_VALUE                        0xEC6C70
+
+#define mmTPC3_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xEC6C74
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xEC6C78
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xEC6C7C
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xEC6C80
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xEC6C84
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xEC6C88
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xEC6C8C
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xEC6C90
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xEC6C94
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xEC6C98
+
+#define mmTPC3_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xEC6C9C
+
+#define mmTPC3_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xEC6CA0
+
+#define mmTPC3_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xEC6CA4
+
+#define mmTPC3_CFG_QM_TENSOR_12_PADDING_VALUE                        0xEC6CA8
+
+#define mmTPC3_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xEC6CAC
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xEC6CB0
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xEC6CB4
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xEC6CB8
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xEC6CBC
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xEC6CC0
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xEC6CC4
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xEC6CC8
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xEC6CCC
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xEC6CD0
+
+#define mmTPC3_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xEC6CD4
+
+#define mmTPC3_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xEC6CD8
+
+#define mmTPC3_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xEC6CDC
+
+#define mmTPC3_CFG_QM_TENSOR_13_PADDING_VALUE                        0xEC6CE0
+
+#define mmTPC3_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xEC6CE4
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xEC6CE8
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xEC6CEC
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xEC6CF0
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xEC6CF4
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xEC6CF8
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xEC6CFC
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xEC6D00
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xEC6D04
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xEC6D08
+
+#define mmTPC3_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xEC6D0C
+
+#define mmTPC3_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xEC6D10
+
+#define mmTPC3_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xEC6D14
+
+#define mmTPC3_CFG_QM_TENSOR_14_PADDING_VALUE                        0xEC6D18
+
+#define mmTPC3_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xEC6D1C
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xEC6D20
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xEC6D24
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xEC6D28
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xEC6D2C
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xEC6D30
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xEC6D34
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xEC6D38
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xEC6D3C
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xEC6D40
+
+#define mmTPC3_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xEC6D44
+
+#define mmTPC3_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xEC6D48
+
+#define mmTPC3_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xEC6D4C
+
+#define mmTPC3_CFG_QM_TENSOR_15_PADDING_VALUE                        0xEC6D50
+
+#define mmTPC3_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xEC6D54
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xEC6D58
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xEC6D5C
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xEC6D60
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xEC6D64
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xEC6D68
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xEC6D6C
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xEC6D70
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xEC6D74
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xEC6D78
+
+#define mmTPC3_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xEC6D7C
+
+#define mmTPC3_CFG_QM_SYNC_OBJECT_MESSAGE                            0xEC6D80
+
+#define mmTPC3_CFG_QM_SYNC_OBJECT_ADDR                               0xEC6D84
+
+#define mmTPC3_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xEC6D88
+
+#define mmTPC3_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xEC6D8C
+
+#define mmTPC3_CFG_QM_TID_BASE_DIM_0                                 0xEC6D90
+
+#define mmTPC3_CFG_QM_TID_SIZE_DIM_0                                 0xEC6D94
+
+#define mmTPC3_CFG_QM_TID_BASE_DIM_1                                 0xEC6D98
+
+#define mmTPC3_CFG_QM_TID_SIZE_DIM_1                                 0xEC6D9C
+
+#define mmTPC3_CFG_QM_TID_BASE_DIM_2                                 0xEC6DA0
+
+#define mmTPC3_CFG_QM_TID_SIZE_DIM_2                                 0xEC6DA4
+
+#define mmTPC3_CFG_QM_TID_BASE_DIM_3                                 0xEC6DA8
+
+#define mmTPC3_CFG_QM_TID_SIZE_DIM_3                                 0xEC6DAC
+
+#define mmTPC3_CFG_QM_TID_BASE_DIM_4                                 0xEC6DB0
+
+#define mmTPC3_CFG_QM_TID_SIZE_DIM_4                                 0xEC6DB4
+
+#define mmTPC3_CFG_QM_KERNEL_CONFIG                                  0xEC6DB8
+
+#define mmTPC3_CFG_QM_KERNEL_ID                                      0xEC6DBC
+
+#define mmTPC3_CFG_QM_SRF_0                                          0xEC6DC0
+
+#define mmTPC3_CFG_QM_SRF_1                                          0xEC6DC4
+
+#define mmTPC3_CFG_QM_SRF_2                                          0xEC6DC8
+
+#define mmTPC3_CFG_QM_SRF_3                                          0xEC6DCC
+
+#define mmTPC3_CFG_QM_SRF_4                                          0xEC6DD0
+
+#define mmTPC3_CFG_QM_SRF_5                                          0xEC6DD4
+
+#define mmTPC3_CFG_QM_SRF_6                                          0xEC6DD8
+
+#define mmTPC3_CFG_QM_SRF_7                                          0xEC6DDC
+
+#define mmTPC3_CFG_QM_SRF_8                                          0xEC6DE0
+
+#define mmTPC3_CFG_QM_SRF_9                                          0xEC6DE4
+
+#define mmTPC3_CFG_QM_SRF_10                                         0xEC6DE8
+
+#define mmTPC3_CFG_QM_SRF_11                                         0xEC6DEC
+
+#define mmTPC3_CFG_QM_SRF_12                                         0xEC6DF0
+
+#define mmTPC3_CFG_QM_SRF_13                                         0xEC6DF4
+
+#define mmTPC3_CFG_QM_SRF_14                                         0xEC6DF8
+
+#define mmTPC3_CFG_QM_SRF_15                                         0xEC6DFC
+
+#define mmTPC3_CFG_QM_SRF_16                                         0xEC6E00
+
+#define mmTPC3_CFG_QM_SRF_17                                         0xEC6E04
+
+#define mmTPC3_CFG_QM_SRF_18                                         0xEC6E08
+
+#define mmTPC3_CFG_QM_SRF_19                                         0xEC6E0C
+
+#define mmTPC3_CFG_QM_SRF_20                                         0xEC6E10
+
+#define mmTPC3_CFG_QM_SRF_21                                         0xEC6E14
+
+#define mmTPC3_CFG_QM_SRF_22                                         0xEC6E18
+
+#define mmTPC3_CFG_QM_SRF_23                                         0xEC6E1C
+
+#define mmTPC3_CFG_QM_SRF_24                                         0xEC6E20
+
+#define mmTPC3_CFG_QM_SRF_25                                         0xEC6E24
+
+#define mmTPC3_CFG_QM_SRF_26                                         0xEC6E28
+
+#define mmTPC3_CFG_QM_SRF_27                                         0xEC6E2C
+
+#define mmTPC3_CFG_QM_SRF_28                                         0xEC6E30
+
+#define mmTPC3_CFG_QM_SRF_29                                         0xEC6E34
+
+#define mmTPC3_CFG_QM_SRF_30                                         0xEC6E38
+
+#define mmTPC3_CFG_QM_SRF_31                                         0xEC6E3C
+
+#endif /* ASIC_REG_TPC3_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_qm_regs.h
new file mode 100644
index 000000000000..5f2a0fd86c9e
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc3_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC3_QM_REGS_H_
+#define ASIC_REG_TPC3_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC3_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC3_QM_GLBL_CFG0                                          0xEC8000
+
+#define mmTPC3_QM_GLBL_CFG1                                          0xEC8004
+
+#define mmTPC3_QM_GLBL_PROT                                          0xEC8008
+
+#define mmTPC3_QM_GLBL_ERR_CFG                                       0xEC800C
+
+#define mmTPC3_QM_GLBL_SECURE_PROPS_0                                0xEC8010
+
+#define mmTPC3_QM_GLBL_SECURE_PROPS_1                                0xEC8014
+
+#define mmTPC3_QM_GLBL_SECURE_PROPS_2                                0xEC8018
+
+#define mmTPC3_QM_GLBL_SECURE_PROPS_3                                0xEC801C
+
+#define mmTPC3_QM_GLBL_SECURE_PROPS_4                                0xEC8020
+
+#define mmTPC3_QM_GLBL_NON_SECURE_PROPS_0                            0xEC8024
+
+#define mmTPC3_QM_GLBL_NON_SECURE_PROPS_1                            0xEC8028
+
+#define mmTPC3_QM_GLBL_NON_SECURE_PROPS_2                            0xEC802C
+
+#define mmTPC3_QM_GLBL_NON_SECURE_PROPS_3                            0xEC8030
+
+#define mmTPC3_QM_GLBL_NON_SECURE_PROPS_4                            0xEC8034
+
+#define mmTPC3_QM_GLBL_STS0                                          0xEC8038
+
+#define mmTPC3_QM_GLBL_STS1_0                                        0xEC8040
+
+#define mmTPC3_QM_GLBL_STS1_1                                        0xEC8044
+
+#define mmTPC3_QM_GLBL_STS1_2                                        0xEC8048
+
+#define mmTPC3_QM_GLBL_STS1_3                                        0xEC804C
+
+#define mmTPC3_QM_GLBL_STS1_4                                        0xEC8050
+
+#define mmTPC3_QM_GLBL_MSG_EN_0                                      0xEC8054
+
+#define mmTPC3_QM_GLBL_MSG_EN_1                                      0xEC8058
+
+#define mmTPC3_QM_GLBL_MSG_EN_2                                      0xEC805C
+
+#define mmTPC3_QM_GLBL_MSG_EN_3                                      0xEC8060
+
+#define mmTPC3_QM_GLBL_MSG_EN_4                                      0xEC8068
+
+#define mmTPC3_QM_PQ_BASE_LO_0                                       0xEC8070
+
+#define mmTPC3_QM_PQ_BASE_LO_1                                       0xEC8074
+
+#define mmTPC3_QM_PQ_BASE_LO_2                                       0xEC8078
+
+#define mmTPC3_QM_PQ_BASE_LO_3                                       0xEC807C
+
+#define mmTPC3_QM_PQ_BASE_HI_0                                       0xEC8080
+
+#define mmTPC3_QM_PQ_BASE_HI_1                                       0xEC8084
+
+#define mmTPC3_QM_PQ_BASE_HI_2                                       0xEC8088
+
+#define mmTPC3_QM_PQ_BASE_HI_3                                       0xEC808C
+
+#define mmTPC3_QM_PQ_SIZE_0                                          0xEC8090
+
+#define mmTPC3_QM_PQ_SIZE_1                                          0xEC8094
+
+#define mmTPC3_QM_PQ_SIZE_2                                          0xEC8098
+
+#define mmTPC3_QM_PQ_SIZE_3                                          0xEC809C
+
+#define mmTPC3_QM_PQ_PI_0                                            0xEC80A0
+
+#define mmTPC3_QM_PQ_PI_1                                            0xEC80A4
+
+#define mmTPC3_QM_PQ_PI_2                                            0xEC80A8
+
+#define mmTPC3_QM_PQ_PI_3                                            0xEC80AC
+
+#define mmTPC3_QM_PQ_CI_0                                            0xEC80B0
+
+#define mmTPC3_QM_PQ_CI_1                                            0xEC80B4
+
+#define mmTPC3_QM_PQ_CI_2                                            0xEC80B8
+
+#define mmTPC3_QM_PQ_CI_3                                            0xEC80BC
+
+#define mmTPC3_QM_PQ_CFG0_0                                          0xEC80C0
+
+#define mmTPC3_QM_PQ_CFG0_1                                          0xEC80C4
+
+#define mmTPC3_QM_PQ_CFG0_2                                          0xEC80C8
+
+#define mmTPC3_QM_PQ_CFG0_3                                          0xEC80CC
+
+#define mmTPC3_QM_PQ_CFG1_0                                          0xEC80D0
+
+#define mmTPC3_QM_PQ_CFG1_1                                          0xEC80D4
+
+#define mmTPC3_QM_PQ_CFG1_2                                          0xEC80D8
+
+#define mmTPC3_QM_PQ_CFG1_3                                          0xEC80DC
+
+#define mmTPC3_QM_PQ_ARUSER_31_11_0                                  0xEC80E0
+
+#define mmTPC3_QM_PQ_ARUSER_31_11_1                                  0xEC80E4
+
+#define mmTPC3_QM_PQ_ARUSER_31_11_2                                  0xEC80E8
+
+#define mmTPC3_QM_PQ_ARUSER_31_11_3                                  0xEC80EC
+
+#define mmTPC3_QM_PQ_STS0_0                                          0xEC80F0
+
+#define mmTPC3_QM_PQ_STS0_1                                          0xEC80F4
+
+#define mmTPC3_QM_PQ_STS0_2                                          0xEC80F8
+
+#define mmTPC3_QM_PQ_STS0_3                                          0xEC80FC
+
+#define mmTPC3_QM_PQ_STS1_0                                          0xEC8100
+
+#define mmTPC3_QM_PQ_STS1_1                                          0xEC8104
+
+#define mmTPC3_QM_PQ_STS1_2                                          0xEC8108
+
+#define mmTPC3_QM_PQ_STS1_3                                          0xEC810C
+
+#define mmTPC3_QM_CQ_CFG0_0                                          0xEC8110
+
+#define mmTPC3_QM_CQ_CFG0_1                                          0xEC8114
+
+#define mmTPC3_QM_CQ_CFG0_2                                          0xEC8118
+
+#define mmTPC3_QM_CQ_CFG0_3                                          0xEC811C
+
+#define mmTPC3_QM_CQ_CFG0_4                                          0xEC8120
+
+#define mmTPC3_QM_CQ_CFG1_0                                          0xEC8124
+
+#define mmTPC3_QM_CQ_CFG1_1                                          0xEC8128
+
+#define mmTPC3_QM_CQ_CFG1_2                                          0xEC812C
+
+#define mmTPC3_QM_CQ_CFG1_3                                          0xEC8130
+
+#define mmTPC3_QM_CQ_CFG1_4                                          0xEC8134
+
+#define mmTPC3_QM_CQ_ARUSER_31_11_0                                  0xEC8138
+
+#define mmTPC3_QM_CQ_ARUSER_31_11_1                                  0xEC813C
+
+#define mmTPC3_QM_CQ_ARUSER_31_11_2                                  0xEC8140
+
+#define mmTPC3_QM_CQ_ARUSER_31_11_3                                  0xEC8144
+
+#define mmTPC3_QM_CQ_ARUSER_31_11_4                                  0xEC8148
+
+#define mmTPC3_QM_CQ_STS0_0                                          0xEC814C
+
+#define mmTPC3_QM_CQ_STS0_1                                          0xEC8150
+
+#define mmTPC3_QM_CQ_STS0_2                                          0xEC8154
+
+#define mmTPC3_QM_CQ_STS0_3                                          0xEC8158
+
+#define mmTPC3_QM_CQ_STS0_4                                          0xEC815C
+
+#define mmTPC3_QM_CQ_STS1_0                                          0xEC8160
+
+#define mmTPC3_QM_CQ_STS1_1                                          0xEC8164
+
+#define mmTPC3_QM_CQ_STS1_2                                          0xEC8168
+
+#define mmTPC3_QM_CQ_STS1_3                                          0xEC816C
+
+#define mmTPC3_QM_CQ_STS1_4                                          0xEC8170
+
+#define mmTPC3_QM_CQ_PTR_LO_0                                        0xEC8174
+
+#define mmTPC3_QM_CQ_PTR_HI_0                                        0xEC8178
+
+#define mmTPC3_QM_CQ_TSIZE_0                                         0xEC817C
+
+#define mmTPC3_QM_CQ_CTL_0                                           0xEC8180
+
+#define mmTPC3_QM_CQ_PTR_LO_1                                        0xEC8184
+
+#define mmTPC3_QM_CQ_PTR_HI_1                                        0xEC8188
+
+#define mmTPC3_QM_CQ_TSIZE_1                                         0xEC818C
+
+#define mmTPC3_QM_CQ_CTL_1                                           0xEC8190
+
+#define mmTPC3_QM_CQ_PTR_LO_2                                        0xEC8194
+
+#define mmTPC3_QM_CQ_PTR_HI_2                                        0xEC8198
+
+#define mmTPC3_QM_CQ_TSIZE_2                                         0xEC819C
+
+#define mmTPC3_QM_CQ_CTL_2                                           0xEC81A0
+
+#define mmTPC3_QM_CQ_PTR_LO_3                                        0xEC81A4
+
+#define mmTPC3_QM_CQ_PTR_HI_3                                        0xEC81A8
+
+#define mmTPC3_QM_CQ_TSIZE_3                                         0xEC81AC
+
+#define mmTPC3_QM_CQ_CTL_3                                           0xEC81B0
+
+#define mmTPC3_QM_CQ_PTR_LO_4                                        0xEC81B4
+
+#define mmTPC3_QM_CQ_PTR_HI_4                                        0xEC81B8
+
+#define mmTPC3_QM_CQ_TSIZE_4                                         0xEC81BC
+
+#define mmTPC3_QM_CQ_CTL_4                                           0xEC81C0
+
+#define mmTPC3_QM_CQ_PTR_LO_STS_0                                    0xEC81C4
+
+#define mmTPC3_QM_CQ_PTR_LO_STS_1                                    0xEC81C8
+
+#define mmTPC3_QM_CQ_PTR_LO_STS_2                                    0xEC81CC
+
+#define mmTPC3_QM_CQ_PTR_LO_STS_3                                    0xEC81D0
+
+#define mmTPC3_QM_CQ_PTR_LO_STS_4                                    0xEC81D4
+
+#define mmTPC3_QM_CQ_PTR_HI_STS_0                                    0xEC81D8
+
+#define mmTPC3_QM_CQ_PTR_HI_STS_1                                    0xEC81DC
+
+#define mmTPC3_QM_CQ_PTR_HI_STS_2                                    0xEC81E0
+
+#define mmTPC3_QM_CQ_PTR_HI_STS_3                                    0xEC81E4
+
+#define mmTPC3_QM_CQ_PTR_HI_STS_4                                    0xEC81E8
+
+#define mmTPC3_QM_CQ_TSIZE_STS_0                                     0xEC81EC
+
+#define mmTPC3_QM_CQ_TSIZE_STS_1                                     0xEC81F0
+
+#define mmTPC3_QM_CQ_TSIZE_STS_2                                     0xEC81F4
+
+#define mmTPC3_QM_CQ_TSIZE_STS_3                                     0xEC81F8
+
+#define mmTPC3_QM_CQ_TSIZE_STS_4                                     0xEC81FC
+
+#define mmTPC3_QM_CQ_CTL_STS_0                                       0xEC8200
+
+#define mmTPC3_QM_CQ_CTL_STS_1                                       0xEC8204
+
+#define mmTPC3_QM_CQ_CTL_STS_2                                       0xEC8208
+
+#define mmTPC3_QM_CQ_CTL_STS_3                                       0xEC820C
+
+#define mmTPC3_QM_CQ_CTL_STS_4                                       0xEC8210
+
+#define mmTPC3_QM_CQ_IFIFO_CNT_0                                     0xEC8214
+
+#define mmTPC3_QM_CQ_IFIFO_CNT_1                                     0xEC8218
+
+#define mmTPC3_QM_CQ_IFIFO_CNT_2                                     0xEC821C
+
+#define mmTPC3_QM_CQ_IFIFO_CNT_3                                     0xEC8220
+
+#define mmTPC3_QM_CQ_IFIFO_CNT_4                                     0xEC8224
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_0                             0xEC8228
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_1                             0xEC822C
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_2                             0xEC8230
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_3                             0xEC8234
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_LO_4                             0xEC8238
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_0                             0xEC823C
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_1                             0xEC8240
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_2                             0xEC8244
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_3                             0xEC8248
+
+#define mmTPC3_QM_CP_MSG_BASE0_ADDR_HI_4                             0xEC824C
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_0                             0xEC8250
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_1                             0xEC8254
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_2                             0xEC8258
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_3                             0xEC825C
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_LO_4                             0xEC8260
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_0                             0xEC8264
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_1                             0xEC8268
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_2                             0xEC826C
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_3                             0xEC8270
+
+#define mmTPC3_QM_CP_MSG_BASE1_ADDR_HI_4                             0xEC8274
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_0                             0xEC8278
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_1                             0xEC827C
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_2                             0xEC8280
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_3                             0xEC8284
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_LO_4                             0xEC8288
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_0                             0xEC828C
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_1                             0xEC8290
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_2                             0xEC8294
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_3                             0xEC8298
+
+#define mmTPC3_QM_CP_MSG_BASE2_ADDR_HI_4                             0xEC829C
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_0                             0xEC82A0
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_1                             0xEC82A4
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_2                             0xEC82A8
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_3                             0xEC82AC
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_LO_4                             0xEC82B0
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_0                             0xEC82B4
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_1                             0xEC82B8
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_2                             0xEC82BC
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_3                             0xEC82C0
+
+#define mmTPC3_QM_CP_MSG_BASE3_ADDR_HI_4                             0xEC82C4
+
+#define mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_0                             0xEC82C8
+
+#define mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_1                             0xEC82CC
+
+#define mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_2                             0xEC82D0
+
+#define mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_3                             0xEC82D4
+
+#define mmTPC3_QM_CP_LDMA_TSIZE_OFFSET_4                             0xEC82D8
+
+#define mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xEC82E0
+
+#define mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xEC82E4
+
+#define mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xEC82E8
+
+#define mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xEC82EC
+
+#define mmTPC3_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xEC82F0
+
+#define mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xEC82F4
+
+#define mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xEC82F8
+
+#define mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xEC82FC
+
+#define mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xEC8300
+
+#define mmTPC3_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xEC8304
+
+#define mmTPC3_QM_CP_FENCE0_RDATA_0                                  0xEC8308
+
+#define mmTPC3_QM_CP_FENCE0_RDATA_1                                  0xEC830C
+
+#define mmTPC3_QM_CP_FENCE0_RDATA_2                                  0xEC8310
+
+#define mmTPC3_QM_CP_FENCE0_RDATA_3                                  0xEC8314
+
+#define mmTPC3_QM_CP_FENCE0_RDATA_4                                  0xEC8318
+
+#define mmTPC3_QM_CP_FENCE1_RDATA_0                                  0xEC831C
+
+#define mmTPC3_QM_CP_FENCE1_RDATA_1                                  0xEC8320
+
+#define mmTPC3_QM_CP_FENCE1_RDATA_2                                  0xEC8324
+
+#define mmTPC3_QM_CP_FENCE1_RDATA_3                                  0xEC8328
+
+#define mmTPC3_QM_CP_FENCE1_RDATA_4                                  0xEC832C
+
+#define mmTPC3_QM_CP_FENCE2_RDATA_0                                  0xEC8330
+
+#define mmTPC3_QM_CP_FENCE2_RDATA_1                                  0xEC8334
+
+#define mmTPC3_QM_CP_FENCE2_RDATA_2                                  0xEC8338
+
+#define mmTPC3_QM_CP_FENCE2_RDATA_3                                  0xEC833C
+
+#define mmTPC3_QM_CP_FENCE2_RDATA_4                                  0xEC8340
+
+#define mmTPC3_QM_CP_FENCE3_RDATA_0                                  0xEC8344
+
+#define mmTPC3_QM_CP_FENCE3_RDATA_1                                  0xEC8348
+
+#define mmTPC3_QM_CP_FENCE3_RDATA_2                                  0xEC834C
+
+#define mmTPC3_QM_CP_FENCE3_RDATA_3                                  0xEC8350
+
+#define mmTPC3_QM_CP_FENCE3_RDATA_4                                  0xEC8354
+
+#define mmTPC3_QM_CP_FENCE0_CNT_0                                    0xEC8358
+
+#define mmTPC3_QM_CP_FENCE0_CNT_1                                    0xEC835C
+
+#define mmTPC3_QM_CP_FENCE0_CNT_2                                    0xEC8360
+
+#define mmTPC3_QM_CP_FENCE0_CNT_3                                    0xEC8364
+
+#define mmTPC3_QM_CP_FENCE0_CNT_4                                    0xEC8368
+
+#define mmTPC3_QM_CP_FENCE1_CNT_0                                    0xEC836C
+
+#define mmTPC3_QM_CP_FENCE1_CNT_1                                    0xEC8370
+
+#define mmTPC3_QM_CP_FENCE1_CNT_2                                    0xEC8374
+
+#define mmTPC3_QM_CP_FENCE1_CNT_3                                    0xEC8378
+
+#define mmTPC3_QM_CP_FENCE1_CNT_4                                    0xEC837C
+
+#define mmTPC3_QM_CP_FENCE2_CNT_0                                    0xEC8380
+
+#define mmTPC3_QM_CP_FENCE2_CNT_1                                    0xEC8384
+
+#define mmTPC3_QM_CP_FENCE2_CNT_2                                    0xEC8388
+
+#define mmTPC3_QM_CP_FENCE2_CNT_3                                    0xEC838C
+
+#define mmTPC3_QM_CP_FENCE2_CNT_4                                    0xEC8390
+
+#define mmTPC3_QM_CP_FENCE3_CNT_0                                    0xEC8394
+
+#define mmTPC3_QM_CP_FENCE3_CNT_1                                    0xEC8398
+
+#define mmTPC3_QM_CP_FENCE3_CNT_2                                    0xEC839C
+
+#define mmTPC3_QM_CP_FENCE3_CNT_3                                    0xEC83A0
+
+#define mmTPC3_QM_CP_FENCE3_CNT_4                                    0xEC83A4
+
+#define mmTPC3_QM_CP_STS_0                                           0xEC83A8
+
+#define mmTPC3_QM_CP_STS_1                                           0xEC83AC
+
+#define mmTPC3_QM_CP_STS_2                                           0xEC83B0
+
+#define mmTPC3_QM_CP_STS_3                                           0xEC83B4
+
+#define mmTPC3_QM_CP_STS_4                                           0xEC83B8
+
+#define mmTPC3_QM_CP_CURRENT_INST_LO_0                               0xEC83BC
+
+#define mmTPC3_QM_CP_CURRENT_INST_LO_1                               0xEC83C0
+
+#define mmTPC3_QM_CP_CURRENT_INST_LO_2                               0xEC83C4
+
+#define mmTPC3_QM_CP_CURRENT_INST_LO_3                               0xEC83C8
+
+#define mmTPC3_QM_CP_CURRENT_INST_LO_4                               0xEC83CC
+
+#define mmTPC3_QM_CP_CURRENT_INST_HI_0                               0xEC83D0
+
+#define mmTPC3_QM_CP_CURRENT_INST_HI_1                               0xEC83D4
+
+#define mmTPC3_QM_CP_CURRENT_INST_HI_2                               0xEC83D8
+
+#define mmTPC3_QM_CP_CURRENT_INST_HI_3                               0xEC83DC
+
+#define mmTPC3_QM_CP_CURRENT_INST_HI_4                               0xEC83E0
+
+#define mmTPC3_QM_CP_BARRIER_CFG_0                                   0xEC83F4
+
+#define mmTPC3_QM_CP_BARRIER_CFG_1                                   0xEC83F8
+
+#define mmTPC3_QM_CP_BARRIER_CFG_2                                   0xEC83FC
+
+#define mmTPC3_QM_CP_BARRIER_CFG_3                                   0xEC8400
+
+#define mmTPC3_QM_CP_BARRIER_CFG_4                                   0xEC8404
+
+#define mmTPC3_QM_CP_DBG_0_0                                         0xEC8408
+
+#define mmTPC3_QM_CP_DBG_0_1                                         0xEC840C
+
+#define mmTPC3_QM_CP_DBG_0_2                                         0xEC8410
+
+#define mmTPC3_QM_CP_DBG_0_3                                         0xEC8414
+
+#define mmTPC3_QM_CP_DBG_0_4                                         0xEC8418
+
+#define mmTPC3_QM_CP_ARUSER_31_11_0                                  0xEC841C
+
+#define mmTPC3_QM_CP_ARUSER_31_11_1                                  0xEC8420
+
+#define mmTPC3_QM_CP_ARUSER_31_11_2                                  0xEC8424
+
+#define mmTPC3_QM_CP_ARUSER_31_11_3                                  0xEC8428
+
+#define mmTPC3_QM_CP_ARUSER_31_11_4                                  0xEC842C
+
+#define mmTPC3_QM_CP_AWUSER_31_11_0                                  0xEC8430
+
+#define mmTPC3_QM_CP_AWUSER_31_11_1                                  0xEC8434
+
+#define mmTPC3_QM_CP_AWUSER_31_11_2                                  0xEC8438
+
+#define mmTPC3_QM_CP_AWUSER_31_11_3                                  0xEC843C
+
+#define mmTPC3_QM_CP_AWUSER_31_11_4                                  0xEC8440
+
+#define mmTPC3_QM_ARB_CFG_0                                          0xEC8A00
+
+#define mmTPC3_QM_ARB_CHOISE_Q_PUSH                                  0xEC8A04
+
+#define mmTPC3_QM_ARB_WRR_WEIGHT_0                                   0xEC8A08
+
+#define mmTPC3_QM_ARB_WRR_WEIGHT_1                                   0xEC8A0C
+
+#define mmTPC3_QM_ARB_WRR_WEIGHT_2                                   0xEC8A10
+
+#define mmTPC3_QM_ARB_WRR_WEIGHT_3                                   0xEC8A14
+
+#define mmTPC3_QM_ARB_CFG_1                                          0xEC8A18
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_0                               0xEC8A20
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_1                               0xEC8A24
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_2                               0xEC8A28
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_3                               0xEC8A2C
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_4                               0xEC8A30
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_5                               0xEC8A34
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_6                               0xEC8A38
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_7                               0xEC8A3C
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_8                               0xEC8A40
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_9                               0xEC8A44
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_10                              0xEC8A48
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_11                              0xEC8A4C
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_12                              0xEC8A50
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_13                              0xEC8A54
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_14                              0xEC8A58
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_15                              0xEC8A5C
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_16                              0xEC8A60
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_17                              0xEC8A64
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_18                              0xEC8A68
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_19                              0xEC8A6C
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_20                              0xEC8A70
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_21                              0xEC8A74
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_22                              0xEC8A78
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_23                              0xEC8A7C
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_24                              0xEC8A80
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_25                              0xEC8A84
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_26                              0xEC8A88
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_27                              0xEC8A8C
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_28                              0xEC8A90
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_29                              0xEC8A94
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_30                              0xEC8A98
+
+#define mmTPC3_QM_ARB_MST_AVAIL_CRED_31                              0xEC8A9C
+
+#define mmTPC3_QM_ARB_MST_CRED_INC                                   0xEC8AA0
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xEC8AA4
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xEC8AA8
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xEC8AAC
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xEC8AB0
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xEC8AB4
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xEC8AB8
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xEC8ABC
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xEC8AC0
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xEC8AC4
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xEC8AC8
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xEC8ACC
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xEC8AD0
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xEC8AD4
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xEC8AD8
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xEC8ADC
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xEC8AE0
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xEC8AE4
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xEC8AE8
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xEC8AEC
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xEC8AF0
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xEC8AF4
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xEC8AF8
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xEC8AFC
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xEC8B00
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xEC8B04
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xEC8B08
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xEC8B0C
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xEC8B10
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xEC8B14
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xEC8B18
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xEC8B1C
+
+#define mmTPC3_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xEC8B20
+
+#define mmTPC3_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xEC8B28
+
+#define mmTPC3_QM_ARB_MST_SLAVE_EN                                   0xEC8B2C
+
+#define mmTPC3_QM_ARB_MST_QUIET_PER                                  0xEC8B34
+
+#define mmTPC3_QM_ARB_SLV_CHOISE_WDT                                 0xEC8B38
+
+#define mmTPC3_QM_ARB_SLV_ID                                         0xEC8B3C
+
+#define mmTPC3_QM_ARB_MSG_MAX_INFLIGHT                               0xEC8B44
+
+#define mmTPC3_QM_ARB_MSG_AWUSER_31_11                               0xEC8B48
+
+#define mmTPC3_QM_ARB_MSG_AWUSER_SEC_PROP                            0xEC8B4C
+
+#define mmTPC3_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xEC8B50
+
+#define mmTPC3_QM_ARB_BASE_LO                                        0xEC8B54
+
+#define mmTPC3_QM_ARB_BASE_HI                                        0xEC8B58
+
+#define mmTPC3_QM_ARB_STATE_STS                                      0xEC8B80
+
+#define mmTPC3_QM_ARB_CHOISE_FULLNESS_STS                            0xEC8B84
+
+#define mmTPC3_QM_ARB_MSG_STS                                        0xEC8B88
+
+#define mmTPC3_QM_ARB_SLV_CHOISE_Q_HEAD                              0xEC8B8C
+
+#define mmTPC3_QM_ARB_ERR_CAUSE                                      0xEC8B9C
+
+#define mmTPC3_QM_ARB_ERR_MSG_EN                                     0xEC8BA0
+
+#define mmTPC3_QM_ARB_ERR_STS_DRP                                    0xEC8BA8
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_0                                 0xEC8BB0
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_1                                 0xEC8BB4
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_2                                 0xEC8BB8
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_3                                 0xEC8BBC
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_4                                 0xEC8BC0
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_5                                 0xEC8BC4
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_6                                 0xEC8BC8
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_7                                 0xEC8BCC
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_8                                 0xEC8BD0
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_9                                 0xEC8BD4
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_10                                0xEC8BD8
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_11                                0xEC8BDC
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_12                                0xEC8BE0
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_13                                0xEC8BE4
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_14                                0xEC8BE8
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_15                                0xEC8BEC
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_16                                0xEC8BF0
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_17                                0xEC8BF4
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_18                                0xEC8BF8
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_19                                0xEC8BFC
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_20                                0xEC8C00
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_21                                0xEC8C04
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_22                                0xEC8C08
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_23                                0xEC8C0C
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_24                                0xEC8C10
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_25                                0xEC8C14
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_26                                0xEC8C18
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_27                                0xEC8C1C
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_28                                0xEC8C20
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_29                                0xEC8C24
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_30                                0xEC8C28
+
+#define mmTPC3_QM_ARB_MST_CRED_STS_31                                0xEC8C2C
+
+#define mmTPC3_QM_CGM_CFG                                            0xEC8C70
+
+#define mmTPC3_QM_CGM_STS                                            0xEC8C74
+
+#define mmTPC3_QM_CGM_CFG1                                           0xEC8C78
+
+#define mmTPC3_QM_LOCAL_RANGE_BASE                                   0xEC8C80
+
+#define mmTPC3_QM_LOCAL_RANGE_SIZE                                   0xEC8C84
+
+#define mmTPC3_QM_CSMR_STRICT_PRIO_CFG                               0xEC8C90
+
+#define mmTPC3_QM_HBW_RD_RATE_LIM_CFG_1                              0xEC8C94
+
+#define mmTPC3_QM_LBW_WR_RATE_LIM_CFG_0                              0xEC8C98
+
+#define mmTPC3_QM_LBW_WR_RATE_LIM_CFG_1                              0xEC8C9C
+
+#define mmTPC3_QM_HBW_RD_RATE_LIM_CFG_0                              0xEC8CA0
+
+#define mmTPC3_QM_GLBL_AXCACHE                                       0xEC8CA4
+
+#define mmTPC3_QM_IND_GW_APB_CFG                                     0xEC8CB0
+
+#define mmTPC3_QM_IND_GW_APB_WDATA                                   0xEC8CB4
+
+#define mmTPC3_QM_IND_GW_APB_RDATA                                   0xEC8CB8
+
+#define mmTPC3_QM_IND_GW_APB_STATUS                                  0xEC8CBC
+
+#define mmTPC3_QM_GLBL_ERR_ADDR_LO                                   0xEC8CD0
+
+#define mmTPC3_QM_GLBL_ERR_ADDR_HI                                   0xEC8CD4
+
+#define mmTPC3_QM_GLBL_ERR_WDATA                                     0xEC8CD8
+
+#define mmTPC3_QM_GLBL_MEM_INIT_BUSY                                 0xEC8D00
+
+#endif /* ASIC_REG_TPC3_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_cfg_regs.h
new file mode 100644
index 000000000000..7a9447f39a74
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC4_CFG_REGS_H_
+#define ASIC_REG_TPC4_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC4_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xF06400
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xF06404
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xF06408
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xF0640C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xF06410
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xF06414
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xF06418
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xF0641C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xF06420
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xF06424
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xF06428
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xF0642C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xF06430
+
+#define mmTPC4_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xF06434
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xF06438
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xF0643C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xF06440
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xF06444
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xF06448
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xF0644C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xF06450
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xF06454
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xF06458
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xF0645C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xF06460
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xF06464
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xF06468
+
+#define mmTPC4_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xF0646C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xF06470
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xF06474
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xF06478
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xF0647C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xF06480
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xF06484
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xF06488
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xF0648C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xF06490
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xF06494
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xF06498
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xF0649C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xF064A0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xF064A4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xF064A8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xF064AC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xF064B0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xF064B4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xF064B8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xF064BC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xF064C0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xF064C4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xF064C8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xF064CC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xF064D0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xF064D4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xF064D8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xF064DC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xF064E0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xF064E4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xF064E8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xF064EC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xF064F0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xF064F4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xF064F8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xF064FC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xF06500
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xF06504
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xF06508
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xF0650C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xF06510
+
+#define mmTPC4_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xF06514
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xF06518
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xF0651C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xF06520
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xF06524
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xF06528
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xF0652C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xF06530
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xF06534
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xF06538
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xF0653C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xF06540
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xF06544
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xF06548
+
+#define mmTPC4_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xF0654C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xF06550
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xF06554
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xF06558
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xF0655C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xF06560
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xF06564
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xF06568
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xF0656C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xF06570
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xF06574
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xF06578
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xF0657C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xF06580
+
+#define mmTPC4_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xF06584
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xF06588
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xF0658C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xF06590
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xF06594
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xF06598
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xF0659C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xF065A0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xF065A4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xF065A8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xF065AC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xF065B0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xF065B4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xF065B8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xF065BC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xF065C0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xF065C4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xF065C8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xF065CC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xF065D0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xF065D4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xF065D8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xF065DC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xF065E0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xF065E4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xF065E8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xF065EC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xF065F0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xF065F4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xF065F8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xF065FC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xF06600
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xF06604
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xF06608
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xF0660C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xF06610
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xF06614
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xF06618
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xF0661C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xF06620
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xF06624
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xF06628
+
+#define mmTPC4_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xF0662C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xF06630
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xF06634
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xF06638
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xF0663C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xF06640
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xF06644
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xF06648
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xF0664C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xF06650
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xF06654
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xF06658
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xF0665C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xF06660
+
+#define mmTPC4_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xF06664
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xF06668
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xF0666C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xF06670
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xF06674
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xF06678
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xF0667C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xF06680
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xF06684
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xF06688
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xF0668C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xF06690
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xF06694
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xF06698
+
+#define mmTPC4_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xF0669C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xF066A0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xF066A4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xF066A8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xF066AC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xF066B0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xF066B4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xF066B8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xF066BC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xF066C0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xF066C4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xF066C8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xF066CC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xF066D0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xF066D4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xF066D8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xF066DC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xF066E0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xF066E4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xF066E8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xF066EC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xF066F0
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xF066F4
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xF066F8
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xF066FC
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xF06700
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xF06704
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xF06708
+
+#define mmTPC4_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xF0670C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xF06710
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xF06714
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xF06718
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xF0671C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xF06720
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xF06724
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xF06728
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xF0672C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xF06730
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xF06734
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xF06738
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xF0673C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xF06740
+
+#define mmTPC4_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xF06744
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xF06748
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xF0674C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xF06750
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xF06754
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xF06758
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xF0675C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xF06760
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xF06764
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xF06768
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xF0676C
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xF06770
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xF06774
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xF06778
+
+#define mmTPC4_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xF0677C
+
+#define mmTPC4_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xF06780
+
+#define mmTPC4_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xF06784
+
+#define mmTPC4_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xF06788
+
+#define mmTPC4_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xF0678C
+
+#define mmTPC4_CFG_KERNEL_TID_BASE_DIM_0                             0xF06790
+
+#define mmTPC4_CFG_KERNEL_TID_SIZE_DIM_0                             0xF06794
+
+#define mmTPC4_CFG_KERNEL_TID_BASE_DIM_1                             0xF06798
+
+#define mmTPC4_CFG_KERNEL_TID_SIZE_DIM_1                             0xF0679C
+
+#define mmTPC4_CFG_KERNEL_TID_BASE_DIM_2                             0xF067A0
+
+#define mmTPC4_CFG_KERNEL_TID_SIZE_DIM_2                             0xF067A4
+
+#define mmTPC4_CFG_KERNEL_TID_BASE_DIM_3                             0xF067A8
+
+#define mmTPC4_CFG_KERNEL_TID_SIZE_DIM_3                             0xF067AC
+
+#define mmTPC4_CFG_KERNEL_TID_BASE_DIM_4                             0xF067B0
+
+#define mmTPC4_CFG_KERNEL_TID_SIZE_DIM_4                             0xF067B4
+
+#define mmTPC4_CFG_KERNEL_KERNEL_CONFIG                              0xF067B8
+
+#define mmTPC4_CFG_KERNEL_KERNEL_ID                                  0xF067BC
+
+#define mmTPC4_CFG_KERNEL_SRF_0                                      0xF067C0
+
+#define mmTPC4_CFG_KERNEL_SRF_1                                      0xF067C4
+
+#define mmTPC4_CFG_KERNEL_SRF_2                                      0xF067C8
+
+#define mmTPC4_CFG_KERNEL_SRF_3                                      0xF067CC
+
+#define mmTPC4_CFG_KERNEL_SRF_4                                      0xF067D0
+
+#define mmTPC4_CFG_KERNEL_SRF_5                                      0xF067D4
+
+#define mmTPC4_CFG_KERNEL_SRF_6                                      0xF067D8
+
+#define mmTPC4_CFG_KERNEL_SRF_7                                      0xF067DC
+
+#define mmTPC4_CFG_KERNEL_SRF_8                                      0xF067E0
+
+#define mmTPC4_CFG_KERNEL_SRF_9                                      0xF067E4
+
+#define mmTPC4_CFG_KERNEL_SRF_10                                     0xF067E8
+
+#define mmTPC4_CFG_KERNEL_SRF_11                                     0xF067EC
+
+#define mmTPC4_CFG_KERNEL_SRF_12                                     0xF067F0
+
+#define mmTPC4_CFG_KERNEL_SRF_13                                     0xF067F4
+
+#define mmTPC4_CFG_KERNEL_SRF_14                                     0xF067F8
+
+#define mmTPC4_CFG_KERNEL_SRF_15                                     0xF067FC
+
+#define mmTPC4_CFG_KERNEL_SRF_16                                     0xF06800
+
+#define mmTPC4_CFG_KERNEL_SRF_17                                     0xF06804
+
+#define mmTPC4_CFG_KERNEL_SRF_18                                     0xF06808
+
+#define mmTPC4_CFG_KERNEL_SRF_19                                     0xF0680C
+
+#define mmTPC4_CFG_KERNEL_SRF_20                                     0xF06810
+
+#define mmTPC4_CFG_KERNEL_SRF_21                                     0xF06814
+
+#define mmTPC4_CFG_KERNEL_SRF_22                                     0xF06818
+
+#define mmTPC4_CFG_KERNEL_SRF_23                                     0xF0681C
+
+#define mmTPC4_CFG_KERNEL_SRF_24                                     0xF06820
+
+#define mmTPC4_CFG_KERNEL_SRF_25                                     0xF06824
+
+#define mmTPC4_CFG_KERNEL_SRF_26                                     0xF06828
+
+#define mmTPC4_CFG_KERNEL_SRF_27                                     0xF0682C
+
+#define mmTPC4_CFG_KERNEL_SRF_28                                     0xF06830
+
+#define mmTPC4_CFG_KERNEL_SRF_29                                     0xF06834
+
+#define mmTPC4_CFG_KERNEL_SRF_30                                     0xF06838
+
+#define mmTPC4_CFG_KERNEL_SRF_31                                     0xF0683C
+
+#define mmTPC4_CFG_ROUND_CSR                                         0xF068FC
+
+#define mmTPC4_CFG_PROT                                              0xF06900
+
+#define mmTPC4_CFG_SEMAPHORE                                         0xF06908
+
+#define mmTPC4_CFG_VFLAGS                                            0xF0690C
+
+#define mmTPC4_CFG_SFLAGS                                            0xF06910
+
+#define mmTPC4_CFG_LFSR_POLYNOM                                      0xF06918
+
+#define mmTPC4_CFG_STATUS                                            0xF0691C
+
+#define mmTPC4_CFG_CFG_BASE_ADDRESS_HIGH                             0xF06920
+
+#define mmTPC4_CFG_CFG_SUBTRACT_VALUE                                0xF06924
+
+#define mmTPC4_CFG_SM_BASE_ADDRESS_HIGH                              0xF0692C
+
+#define mmTPC4_CFG_TPC_CMD                                           0xF06930
+
+#define mmTPC4_CFG_TPC_EXECUTE                                       0xF06938
+
+#define mmTPC4_CFG_TPC_STALL                                         0xF0693C
+
+#define mmTPC4_CFG_ICACHE_BASE_ADDERESS_LOW                          0xF06940
+
+#define mmTPC4_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xF06944
+
+#define mmTPC4_CFG_RD_RATE_LIMIT                                     0xF06948
+
+#define mmTPC4_CFG_WR_RATE_LIMIT                                     0xF06950
+
+#define mmTPC4_CFG_MSS_CONFIG                                        0xF06954
+
+#define mmTPC4_CFG_TPC_INTR_CAUSE                                    0xF06958
+
+#define mmTPC4_CFG_TPC_INTR_MASK                                     0xF0695C
+
+#define mmTPC4_CFG_WQ_CREDITS                                        0xF06960
+
+#define mmTPC4_CFG_ARUSER_LO                                         0xF06964
+
+#define mmTPC4_CFG_ARUSER_HI                                         0xF06968
+
+#define mmTPC4_CFG_AWUSER_LO                                         0xF0696C
+
+#define mmTPC4_CFG_AWUSER_HI                                         0xF06970
+
+#define mmTPC4_CFG_OPCODE_EXEC                                       0xF06974
+
+#define mmTPC4_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xF06978
+
+#define mmTPC4_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xF0697C
+
+#define mmTPC4_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xF06980
+
+#define mmTPC4_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xF06984
+
+#define mmTPC4_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xF06988
+
+#define mmTPC4_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xF0698C
+
+#define mmTPC4_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xF06990
+
+#define mmTPC4_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xF06994
+
+#define mmTPC4_CFG_TSB_CFG_MAX_SIZE                                  0xF06998
+
+#define mmTPC4_CFG_TSB_CFG                                           0xF0699C
+
+#define mmTPC4_CFG_DBGMEM_ADD                                        0xF069A0
+
+#define mmTPC4_CFG_DBGMEM_DATA_WR                                    0xF069A4
+
+#define mmTPC4_CFG_DBGMEM_DATA_RD                                    0xF069A8
+
+#define mmTPC4_CFG_DBGMEM_CTRL                                       0xF069AC
+
+#define mmTPC4_CFG_DBGMEM_RC                                         0xF069B0
+
+#define mmTPC4_CFG_TSB_INFLIGHT_CNTR                                 0xF069B4
+
+#define mmTPC4_CFG_WQ_INFLIGHT_CNTR                                  0xF069B8
+
+#define mmTPC4_CFG_WQ_LBW_TOTAL_CNTR                                 0xF069BC
+
+#define mmTPC4_CFG_WQ_HBW_TOTAL_CNTR                                 0xF069C0
+
+#define mmTPC4_CFG_IRQ_OCCOUPY_CNTR                                  0xF069C4
+
+#define mmTPC4_CFG_FUNC_MBIST_CNTRL                                  0xF069D0
+
+#define mmTPC4_CFG_FUNC_MBIST_PAT                                    0xF069D4
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_0                                  0xF069D8
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_1                                  0xF069DC
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_2                                  0xF069E0
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_3                                  0xF069E4
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_4                                  0xF069E8
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_5                                  0xF069EC
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_6                                  0xF069F0
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_7                                  0xF069F4
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_8                                  0xF069F8
+
+#define mmTPC4_CFG_FUNC_MBIST_MEM_9                                  0xF069FC
+
+#define mmTPC4_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xF06A00
+
+#define mmTPC4_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xF06A04
+
+#define mmTPC4_CFG_QM_TENSOR_0_PADDING_VALUE                         0xF06A08
+
+#define mmTPC4_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xF06A0C
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xF06A10
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xF06A14
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xF06A18
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xF06A1C
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xF06A20
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xF06A24
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xF06A28
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xF06A2C
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xF06A30
+
+#define mmTPC4_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xF06A34
+
+#define mmTPC4_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xF06A38
+
+#define mmTPC4_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xF06A3C
+
+#define mmTPC4_CFG_QM_TENSOR_1_PADDING_VALUE                         0xF06A40
+
+#define mmTPC4_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xF06A44
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xF06A48
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xF06A4C
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xF06A50
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xF06A54
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xF06A58
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xF06A5C
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xF06A60
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xF06A64
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xF06A68
+
+#define mmTPC4_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xF06A6C
+
+#define mmTPC4_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xF06A70
+
+#define mmTPC4_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xF06A74
+
+#define mmTPC4_CFG_QM_TENSOR_2_PADDING_VALUE                         0xF06A78
+
+#define mmTPC4_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xF06A7C
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xF06A80
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xF06A84
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xF06A88
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xF06A8C
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xF06A90
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xF06A94
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xF06A98
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xF06A9C
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xF06AA0
+
+#define mmTPC4_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xF06AA4
+
+#define mmTPC4_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xF06AA8
+
+#define mmTPC4_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xF06AAC
+
+#define mmTPC4_CFG_QM_TENSOR_3_PADDING_VALUE                         0xF06AB0
+
+#define mmTPC4_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xF06AB4
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xF06AB8
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xF06ABC
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xF06AC0
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xF06AC4
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xF06AC8
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xF06ACC
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xF06AD0
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xF06AD4
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xF06AD8
+
+#define mmTPC4_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xF06ADC
+
+#define mmTPC4_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xF06AE0
+
+#define mmTPC4_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xF06AE4
+
+#define mmTPC4_CFG_QM_TENSOR_4_PADDING_VALUE                         0xF06AE8
+
+#define mmTPC4_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xF06AEC
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xF06AF0
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xF06AF4
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xF06AF8
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xF06AFC
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xF06B00
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xF06B04
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xF06B08
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xF06B0C
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xF06B10
+
+#define mmTPC4_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xF06B14
+
+#define mmTPC4_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xF06B18
+
+#define mmTPC4_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xF06B1C
+
+#define mmTPC4_CFG_QM_TENSOR_5_PADDING_VALUE                         0xF06B20
+
+#define mmTPC4_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xF06B24
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xF06B28
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xF06B2C
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xF06B30
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xF06B34
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xF06B38
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xF06B3C
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xF06B40
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xF06B44
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xF06B48
+
+#define mmTPC4_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xF06B4C
+
+#define mmTPC4_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xF06B50
+
+#define mmTPC4_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xF06B54
+
+#define mmTPC4_CFG_QM_TENSOR_6_PADDING_VALUE                         0xF06B58
+
+#define mmTPC4_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xF06B5C
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xF06B60
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xF06B64
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xF06B68
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xF06B6C
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xF06B70
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xF06B74
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xF06B78
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xF06B7C
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xF06B80
+
+#define mmTPC4_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xF06B84
+
+#define mmTPC4_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xF06B88
+
+#define mmTPC4_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xF06B8C
+
+#define mmTPC4_CFG_QM_TENSOR_7_PADDING_VALUE                         0xF06B90
+
+#define mmTPC4_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xF06B94
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xF06B98
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xF06B9C
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xF06BA0
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xF06BA4
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xF06BA8
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xF06BAC
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xF06BB0
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xF06BB4
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xF06BB8
+
+#define mmTPC4_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xF06BBC
+
+#define mmTPC4_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xF06BC0
+
+#define mmTPC4_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xF06BC4
+
+#define mmTPC4_CFG_QM_TENSOR_8_PADDING_VALUE                         0xF06BC8
+
+#define mmTPC4_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xF06BCC
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xF06BD0
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xF06BD4
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xF06BD8
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xF06BDC
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xF06BE0
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xF06BE4
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xF06BE8
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xF06BEC
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xF06BF0
+
+#define mmTPC4_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xF06BF4
+
+#define mmTPC4_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xF06BF8
+
+#define mmTPC4_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xF06BFC
+
+#define mmTPC4_CFG_QM_TENSOR_9_PADDING_VALUE                         0xF06C00
+
+#define mmTPC4_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xF06C04
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xF06C08
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xF06C0C
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xF06C10
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xF06C14
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xF06C18
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xF06C1C
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xF06C20
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xF06C24
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xF06C28
+
+#define mmTPC4_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xF06C2C
+
+#define mmTPC4_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xF06C30
+
+#define mmTPC4_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xF06C34
+
+#define mmTPC4_CFG_QM_TENSOR_10_PADDING_VALUE                        0xF06C38
+
+#define mmTPC4_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xF06C3C
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xF06C40
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xF06C44
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xF06C48
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xF06C4C
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xF06C50
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xF06C54
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xF06C58
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xF06C5C
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xF06C60
+
+#define mmTPC4_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xF06C64
+
+#define mmTPC4_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xF06C68
+
+#define mmTPC4_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xF06C6C
+
+#define mmTPC4_CFG_QM_TENSOR_11_PADDING_VALUE                        0xF06C70
+
+#define mmTPC4_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xF06C74
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xF06C78
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xF06C7C
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xF06C80
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xF06C84
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xF06C88
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xF06C8C
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xF06C90
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xF06C94
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xF06C98
+
+#define mmTPC4_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xF06C9C
+
+#define mmTPC4_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xF06CA0
+
+#define mmTPC4_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xF06CA4
+
+#define mmTPC4_CFG_QM_TENSOR_12_PADDING_VALUE                        0xF06CA8
+
+#define mmTPC4_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xF06CAC
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xF06CB0
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xF06CB4
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xF06CB8
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xF06CBC
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xF06CC0
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xF06CC4
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xF06CC8
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xF06CCC
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xF06CD0
+
+#define mmTPC4_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xF06CD4
+
+#define mmTPC4_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xF06CD8
+
+#define mmTPC4_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xF06CDC
+
+#define mmTPC4_CFG_QM_TENSOR_13_PADDING_VALUE                        0xF06CE0
+
+#define mmTPC4_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xF06CE4
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xF06CE8
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xF06CEC
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xF06CF0
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xF06CF4
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xF06CF8
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xF06CFC
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xF06D00
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xF06D04
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xF06D08
+
+#define mmTPC4_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xF06D0C
+
+#define mmTPC4_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xF06D10
+
+#define mmTPC4_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xF06D14
+
+#define mmTPC4_CFG_QM_TENSOR_14_PADDING_VALUE                        0xF06D18
+
+#define mmTPC4_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xF06D1C
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xF06D20
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xF06D24
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xF06D28
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xF06D2C
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xF06D30
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xF06D34
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xF06D38
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xF06D3C
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xF06D40
+
+#define mmTPC4_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xF06D44
+
+#define mmTPC4_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xF06D48
+
+#define mmTPC4_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xF06D4C
+
+#define mmTPC4_CFG_QM_TENSOR_15_PADDING_VALUE                        0xF06D50
+
+#define mmTPC4_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xF06D54
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xF06D58
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xF06D5C
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xF06D60
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xF06D64
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xF06D68
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xF06D6C
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xF06D70
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xF06D74
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xF06D78
+
+#define mmTPC4_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xF06D7C
+
+#define mmTPC4_CFG_QM_SYNC_OBJECT_MESSAGE                            0xF06D80
+
+#define mmTPC4_CFG_QM_SYNC_OBJECT_ADDR                               0xF06D84
+
+#define mmTPC4_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xF06D88
+
+#define mmTPC4_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xF06D8C
+
+#define mmTPC4_CFG_QM_TID_BASE_DIM_0                                 0xF06D90
+
+#define mmTPC4_CFG_QM_TID_SIZE_DIM_0                                 0xF06D94
+
+#define mmTPC4_CFG_QM_TID_BASE_DIM_1                                 0xF06D98
+
+#define mmTPC4_CFG_QM_TID_SIZE_DIM_1                                 0xF06D9C
+
+#define mmTPC4_CFG_QM_TID_BASE_DIM_2                                 0xF06DA0
+
+#define mmTPC4_CFG_QM_TID_SIZE_DIM_2                                 0xF06DA4
+
+#define mmTPC4_CFG_QM_TID_BASE_DIM_3                                 0xF06DA8
+
+#define mmTPC4_CFG_QM_TID_SIZE_DIM_3                                 0xF06DAC
+
+#define mmTPC4_CFG_QM_TID_BASE_DIM_4                                 0xF06DB0
+
+#define mmTPC4_CFG_QM_TID_SIZE_DIM_4                                 0xF06DB4
+
+#define mmTPC4_CFG_QM_KERNEL_CONFIG                                  0xF06DB8
+
+#define mmTPC4_CFG_QM_KERNEL_ID                                      0xF06DBC
+
+#define mmTPC4_CFG_QM_SRF_0                                          0xF06DC0
+
+#define mmTPC4_CFG_QM_SRF_1                                          0xF06DC4
+
+#define mmTPC4_CFG_QM_SRF_2                                          0xF06DC8
+
+#define mmTPC4_CFG_QM_SRF_3                                          0xF06DCC
+
+#define mmTPC4_CFG_QM_SRF_4                                          0xF06DD0
+
+#define mmTPC4_CFG_QM_SRF_5                                          0xF06DD4
+
+#define mmTPC4_CFG_QM_SRF_6                                          0xF06DD8
+
+#define mmTPC4_CFG_QM_SRF_7                                          0xF06DDC
+
+#define mmTPC4_CFG_QM_SRF_8                                          0xF06DE0
+
+#define mmTPC4_CFG_QM_SRF_9                                          0xF06DE4
+
+#define mmTPC4_CFG_QM_SRF_10                                         0xF06DE8
+
+#define mmTPC4_CFG_QM_SRF_11                                         0xF06DEC
+
+#define mmTPC4_CFG_QM_SRF_12                                         0xF06DF0
+
+#define mmTPC4_CFG_QM_SRF_13                                         0xF06DF4
+
+#define mmTPC4_CFG_QM_SRF_14                                         0xF06DF8
+
+#define mmTPC4_CFG_QM_SRF_15                                         0xF06DFC
+
+#define mmTPC4_CFG_QM_SRF_16                                         0xF06E00
+
+#define mmTPC4_CFG_QM_SRF_17                                         0xF06E04
+
+#define mmTPC4_CFG_QM_SRF_18                                         0xF06E08
+
+#define mmTPC4_CFG_QM_SRF_19                                         0xF06E0C
+
+#define mmTPC4_CFG_QM_SRF_20                                         0xF06E10
+
+#define mmTPC4_CFG_QM_SRF_21                                         0xF06E14
+
+#define mmTPC4_CFG_QM_SRF_22                                         0xF06E18
+
+#define mmTPC4_CFG_QM_SRF_23                                         0xF06E1C
+
+#define mmTPC4_CFG_QM_SRF_24                                         0xF06E20
+
+#define mmTPC4_CFG_QM_SRF_25                                         0xF06E24
+
+#define mmTPC4_CFG_QM_SRF_26                                         0xF06E28
+
+#define mmTPC4_CFG_QM_SRF_27                                         0xF06E2C
+
+#define mmTPC4_CFG_QM_SRF_28                                         0xF06E30
+
+#define mmTPC4_CFG_QM_SRF_29                                         0xF06E34
+
+#define mmTPC4_CFG_QM_SRF_30                                         0xF06E38
+
+#define mmTPC4_CFG_QM_SRF_31                                         0xF06E3C
+
+#endif /* ASIC_REG_TPC4_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_qm_regs.h
new file mode 100644
index 000000000000..80e63402f6e0
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc4_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC4_QM_REGS_H_
+#define ASIC_REG_TPC4_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC4_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC4_QM_GLBL_CFG0                                          0xF08000
+
+#define mmTPC4_QM_GLBL_CFG1                                          0xF08004
+
+#define mmTPC4_QM_GLBL_PROT                                          0xF08008
+
+#define mmTPC4_QM_GLBL_ERR_CFG                                       0xF0800C
+
+#define mmTPC4_QM_GLBL_SECURE_PROPS_0                                0xF08010
+
+#define mmTPC4_QM_GLBL_SECURE_PROPS_1                                0xF08014
+
+#define mmTPC4_QM_GLBL_SECURE_PROPS_2                                0xF08018
+
+#define mmTPC4_QM_GLBL_SECURE_PROPS_3                                0xF0801C
+
+#define mmTPC4_QM_GLBL_SECURE_PROPS_4                                0xF08020
+
+#define mmTPC4_QM_GLBL_NON_SECURE_PROPS_0                            0xF08024
+
+#define mmTPC4_QM_GLBL_NON_SECURE_PROPS_1                            0xF08028
+
+#define mmTPC4_QM_GLBL_NON_SECURE_PROPS_2                            0xF0802C
+
+#define mmTPC4_QM_GLBL_NON_SECURE_PROPS_3                            0xF08030
+
+#define mmTPC4_QM_GLBL_NON_SECURE_PROPS_4                            0xF08034
+
+#define mmTPC4_QM_GLBL_STS0                                          0xF08038
+
+#define mmTPC4_QM_GLBL_STS1_0                                        0xF08040
+
+#define mmTPC4_QM_GLBL_STS1_1                                        0xF08044
+
+#define mmTPC4_QM_GLBL_STS1_2                                        0xF08048
+
+#define mmTPC4_QM_GLBL_STS1_3                                        0xF0804C
+
+#define mmTPC4_QM_GLBL_STS1_4                                        0xF08050
+
+#define mmTPC4_QM_GLBL_MSG_EN_0                                      0xF08054
+
+#define mmTPC4_QM_GLBL_MSG_EN_1                                      0xF08058
+
+#define mmTPC4_QM_GLBL_MSG_EN_2                                      0xF0805C
+
+#define mmTPC4_QM_GLBL_MSG_EN_3                                      0xF08060
+
+#define mmTPC4_QM_GLBL_MSG_EN_4                                      0xF08068
+
+#define mmTPC4_QM_PQ_BASE_LO_0                                       0xF08070
+
+#define mmTPC4_QM_PQ_BASE_LO_1                                       0xF08074
+
+#define mmTPC4_QM_PQ_BASE_LO_2                                       0xF08078
+
+#define mmTPC4_QM_PQ_BASE_LO_3                                       0xF0807C
+
+#define mmTPC4_QM_PQ_BASE_HI_0                                       0xF08080
+
+#define mmTPC4_QM_PQ_BASE_HI_1                                       0xF08084
+
+#define mmTPC4_QM_PQ_BASE_HI_2                                       0xF08088
+
+#define mmTPC4_QM_PQ_BASE_HI_3                                       0xF0808C
+
+#define mmTPC4_QM_PQ_SIZE_0                                          0xF08090
+
+#define mmTPC4_QM_PQ_SIZE_1                                          0xF08094
+
+#define mmTPC4_QM_PQ_SIZE_2                                          0xF08098
+
+#define mmTPC4_QM_PQ_SIZE_3                                          0xF0809C
+
+#define mmTPC4_QM_PQ_PI_0                                            0xF080A0
+
+#define mmTPC4_QM_PQ_PI_1                                            0xF080A4
+
+#define mmTPC4_QM_PQ_PI_2                                            0xF080A8
+
+#define mmTPC4_QM_PQ_PI_3                                            0xF080AC
+
+#define mmTPC4_QM_PQ_CI_0                                            0xF080B0
+
+#define mmTPC4_QM_PQ_CI_1                                            0xF080B4
+
+#define mmTPC4_QM_PQ_CI_2                                            0xF080B8
+
+#define mmTPC4_QM_PQ_CI_3                                            0xF080BC
+
+#define mmTPC4_QM_PQ_CFG0_0                                          0xF080C0
+
+#define mmTPC4_QM_PQ_CFG0_1                                          0xF080C4
+
+#define mmTPC4_QM_PQ_CFG0_2                                          0xF080C8
+
+#define mmTPC4_QM_PQ_CFG0_3                                          0xF080CC
+
+#define mmTPC4_QM_PQ_CFG1_0                                          0xF080D0
+
+#define mmTPC4_QM_PQ_CFG1_1                                          0xF080D4
+
+#define mmTPC4_QM_PQ_CFG1_2                                          0xF080D8
+
+#define mmTPC4_QM_PQ_CFG1_3                                          0xF080DC
+
+#define mmTPC4_QM_PQ_ARUSER_31_11_0                                  0xF080E0
+
+#define mmTPC4_QM_PQ_ARUSER_31_11_1                                  0xF080E4
+
+#define mmTPC4_QM_PQ_ARUSER_31_11_2                                  0xF080E8
+
+#define mmTPC4_QM_PQ_ARUSER_31_11_3                                  0xF080EC
+
+#define mmTPC4_QM_PQ_STS0_0                                          0xF080F0
+
+#define mmTPC4_QM_PQ_STS0_1                                          0xF080F4
+
+#define mmTPC4_QM_PQ_STS0_2                                          0xF080F8
+
+#define mmTPC4_QM_PQ_STS0_3                                          0xF080FC
+
+#define mmTPC4_QM_PQ_STS1_0                                          0xF08100
+
+#define mmTPC4_QM_PQ_STS1_1                                          0xF08104
+
+#define mmTPC4_QM_PQ_STS1_2                                          0xF08108
+
+#define mmTPC4_QM_PQ_STS1_3                                          0xF0810C
+
+#define mmTPC4_QM_CQ_CFG0_0                                          0xF08110
+
+#define mmTPC4_QM_CQ_CFG0_1                                          0xF08114
+
+#define mmTPC4_QM_CQ_CFG0_2                                          0xF08118
+
+#define mmTPC4_QM_CQ_CFG0_3                                          0xF0811C
+
+#define mmTPC4_QM_CQ_CFG0_4                                          0xF08120
+
+#define mmTPC4_QM_CQ_CFG1_0                                          0xF08124
+
+#define mmTPC4_QM_CQ_CFG1_1                                          0xF08128
+
+#define mmTPC4_QM_CQ_CFG1_2                                          0xF0812C
+
+#define mmTPC4_QM_CQ_CFG1_3                                          0xF08130
+
+#define mmTPC4_QM_CQ_CFG1_4                                          0xF08134
+
+#define mmTPC4_QM_CQ_ARUSER_31_11_0                                  0xF08138
+
+#define mmTPC4_QM_CQ_ARUSER_31_11_1                                  0xF0813C
+
+#define mmTPC4_QM_CQ_ARUSER_31_11_2                                  0xF08140
+
+#define mmTPC4_QM_CQ_ARUSER_31_11_3                                  0xF08144
+
+#define mmTPC4_QM_CQ_ARUSER_31_11_4                                  0xF08148
+
+#define mmTPC4_QM_CQ_STS0_0                                          0xF0814C
+
+#define mmTPC4_QM_CQ_STS0_1                                          0xF08150
+
+#define mmTPC4_QM_CQ_STS0_2                                          0xF08154
+
+#define mmTPC4_QM_CQ_STS0_3                                          0xF08158
+
+#define mmTPC4_QM_CQ_STS0_4                                          0xF0815C
+
+#define mmTPC4_QM_CQ_STS1_0                                          0xF08160
+
+#define mmTPC4_QM_CQ_STS1_1                                          0xF08164
+
+#define mmTPC4_QM_CQ_STS1_2                                          0xF08168
+
+#define mmTPC4_QM_CQ_STS1_3                                          0xF0816C
+
+#define mmTPC4_QM_CQ_STS1_4                                          0xF08170
+
+#define mmTPC4_QM_CQ_PTR_LO_0                                        0xF08174
+
+#define mmTPC4_QM_CQ_PTR_HI_0                                        0xF08178
+
+#define mmTPC4_QM_CQ_TSIZE_0                                         0xF0817C
+
+#define mmTPC4_QM_CQ_CTL_0                                           0xF08180
+
+#define mmTPC4_QM_CQ_PTR_LO_1                                        0xF08184
+
+#define mmTPC4_QM_CQ_PTR_HI_1                                        0xF08188
+
+#define mmTPC4_QM_CQ_TSIZE_1                                         0xF0818C
+
+#define mmTPC4_QM_CQ_CTL_1                                           0xF08190
+
+#define mmTPC4_QM_CQ_PTR_LO_2                                        0xF08194
+
+#define mmTPC4_QM_CQ_PTR_HI_2                                        0xF08198
+
+#define mmTPC4_QM_CQ_TSIZE_2                                         0xF0819C
+
+#define mmTPC4_QM_CQ_CTL_2                                           0xF081A0
+
+#define mmTPC4_QM_CQ_PTR_LO_3                                        0xF081A4
+
+#define mmTPC4_QM_CQ_PTR_HI_3                                        0xF081A8
+
+#define mmTPC4_QM_CQ_TSIZE_3                                         0xF081AC
+
+#define mmTPC4_QM_CQ_CTL_3                                           0xF081B0
+
+#define mmTPC4_QM_CQ_PTR_LO_4                                        0xF081B4
+
+#define mmTPC4_QM_CQ_PTR_HI_4                                        0xF081B8
+
+#define mmTPC4_QM_CQ_TSIZE_4                                         0xF081BC
+
+#define mmTPC4_QM_CQ_CTL_4                                           0xF081C0
+
+#define mmTPC4_QM_CQ_PTR_LO_STS_0                                    0xF081C4
+
+#define mmTPC4_QM_CQ_PTR_LO_STS_1                                    0xF081C8
+
+#define mmTPC4_QM_CQ_PTR_LO_STS_2                                    0xF081CC
+
+#define mmTPC4_QM_CQ_PTR_LO_STS_3                                    0xF081D0
+
+#define mmTPC4_QM_CQ_PTR_LO_STS_4                                    0xF081D4
+
+#define mmTPC4_QM_CQ_PTR_HI_STS_0                                    0xF081D8
+
+#define mmTPC4_QM_CQ_PTR_HI_STS_1                                    0xF081DC
+
+#define mmTPC4_QM_CQ_PTR_HI_STS_2                                    0xF081E0
+
+#define mmTPC4_QM_CQ_PTR_HI_STS_3                                    0xF081E4
+
+#define mmTPC4_QM_CQ_PTR_HI_STS_4                                    0xF081E8
+
+#define mmTPC4_QM_CQ_TSIZE_STS_0                                     0xF081EC
+
+#define mmTPC4_QM_CQ_TSIZE_STS_1                                     0xF081F0
+
+#define mmTPC4_QM_CQ_TSIZE_STS_2                                     0xF081F4
+
+#define mmTPC4_QM_CQ_TSIZE_STS_3                                     0xF081F8
+
+#define mmTPC4_QM_CQ_TSIZE_STS_4                                     0xF081FC
+
+#define mmTPC4_QM_CQ_CTL_STS_0                                       0xF08200
+
+#define mmTPC4_QM_CQ_CTL_STS_1                                       0xF08204
+
+#define mmTPC4_QM_CQ_CTL_STS_2                                       0xF08208
+
+#define mmTPC4_QM_CQ_CTL_STS_3                                       0xF0820C
+
+#define mmTPC4_QM_CQ_CTL_STS_4                                       0xF08210
+
+#define mmTPC4_QM_CQ_IFIFO_CNT_0                                     0xF08214
+
+#define mmTPC4_QM_CQ_IFIFO_CNT_1                                     0xF08218
+
+#define mmTPC4_QM_CQ_IFIFO_CNT_2                                     0xF0821C
+
+#define mmTPC4_QM_CQ_IFIFO_CNT_3                                     0xF08220
+
+#define mmTPC4_QM_CQ_IFIFO_CNT_4                                     0xF08224
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_0                             0xF08228
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_1                             0xF0822C
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_2                             0xF08230
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_3                             0xF08234
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_LO_4                             0xF08238
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_0                             0xF0823C
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_1                             0xF08240
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_2                             0xF08244
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_3                             0xF08248
+
+#define mmTPC4_QM_CP_MSG_BASE0_ADDR_HI_4                             0xF0824C
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_0                             0xF08250
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_1                             0xF08254
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_2                             0xF08258
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_3                             0xF0825C
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_LO_4                             0xF08260
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_0                             0xF08264
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_1                             0xF08268
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_2                             0xF0826C
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_3                             0xF08270
+
+#define mmTPC4_QM_CP_MSG_BASE1_ADDR_HI_4                             0xF08274
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_0                             0xF08278
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_1                             0xF0827C
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_2                             0xF08280
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_3                             0xF08284
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_LO_4                             0xF08288
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_0                             0xF0828C
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_1                             0xF08290
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_2                             0xF08294
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_3                             0xF08298
+
+#define mmTPC4_QM_CP_MSG_BASE2_ADDR_HI_4                             0xF0829C
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_0                             0xF082A0
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_1                             0xF082A4
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_2                             0xF082A8
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_3                             0xF082AC
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_LO_4                             0xF082B0
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_0                             0xF082B4
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_1                             0xF082B8
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_2                             0xF082BC
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_3                             0xF082C0
+
+#define mmTPC4_QM_CP_MSG_BASE3_ADDR_HI_4                             0xF082C4
+
+#define mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_0                             0xF082C8
+
+#define mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_1                             0xF082CC
+
+#define mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_2                             0xF082D0
+
+#define mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_3                             0xF082D4
+
+#define mmTPC4_QM_CP_LDMA_TSIZE_OFFSET_4                             0xF082D8
+
+#define mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xF082E0
+
+#define mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xF082E4
+
+#define mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xF082E8
+
+#define mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xF082EC
+
+#define mmTPC4_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xF082F0
+
+#define mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xF082F4
+
+#define mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xF082F8
+
+#define mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xF082FC
+
+#define mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xF08300
+
+#define mmTPC4_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xF08304
+
+#define mmTPC4_QM_CP_FENCE0_RDATA_0                                  0xF08308
+
+#define mmTPC4_QM_CP_FENCE0_RDATA_1                                  0xF0830C
+
+#define mmTPC4_QM_CP_FENCE0_RDATA_2                                  0xF08310
+
+#define mmTPC4_QM_CP_FENCE0_RDATA_3                                  0xF08314
+
+#define mmTPC4_QM_CP_FENCE0_RDATA_4                                  0xF08318
+
+#define mmTPC4_QM_CP_FENCE1_RDATA_0                                  0xF0831C
+
+#define mmTPC4_QM_CP_FENCE1_RDATA_1                                  0xF08320
+
+#define mmTPC4_QM_CP_FENCE1_RDATA_2                                  0xF08324
+
+#define mmTPC4_QM_CP_FENCE1_RDATA_3                                  0xF08328
+
+#define mmTPC4_QM_CP_FENCE1_RDATA_4                                  0xF0832C
+
+#define mmTPC4_QM_CP_FENCE2_RDATA_0                                  0xF08330
+
+#define mmTPC4_QM_CP_FENCE2_RDATA_1                                  0xF08334
+
+#define mmTPC4_QM_CP_FENCE2_RDATA_2                                  0xF08338
+
+#define mmTPC4_QM_CP_FENCE2_RDATA_3                                  0xF0833C
+
+#define mmTPC4_QM_CP_FENCE2_RDATA_4                                  0xF08340
+
+#define mmTPC4_QM_CP_FENCE3_RDATA_0                                  0xF08344
+
+#define mmTPC4_QM_CP_FENCE3_RDATA_1                                  0xF08348
+
+#define mmTPC4_QM_CP_FENCE3_RDATA_2                                  0xF0834C
+
+#define mmTPC4_QM_CP_FENCE3_RDATA_3                                  0xF08350
+
+#define mmTPC4_QM_CP_FENCE3_RDATA_4                                  0xF08354
+
+#define mmTPC4_QM_CP_FENCE0_CNT_0                                    0xF08358
+
+#define mmTPC4_QM_CP_FENCE0_CNT_1                                    0xF0835C
+
+#define mmTPC4_QM_CP_FENCE0_CNT_2                                    0xF08360
+
+#define mmTPC4_QM_CP_FENCE0_CNT_3                                    0xF08364
+
+#define mmTPC4_QM_CP_FENCE0_CNT_4                                    0xF08368
+
+#define mmTPC4_QM_CP_FENCE1_CNT_0                                    0xF0836C
+
+#define mmTPC4_QM_CP_FENCE1_CNT_1                                    0xF08370
+
+#define mmTPC4_QM_CP_FENCE1_CNT_2                                    0xF08374
+
+#define mmTPC4_QM_CP_FENCE1_CNT_3                                    0xF08378
+
+#define mmTPC4_QM_CP_FENCE1_CNT_4                                    0xF0837C
+
+#define mmTPC4_QM_CP_FENCE2_CNT_0                                    0xF08380
+
+#define mmTPC4_QM_CP_FENCE2_CNT_1                                    0xF08384
+
+#define mmTPC4_QM_CP_FENCE2_CNT_2                                    0xF08388
+
+#define mmTPC4_QM_CP_FENCE2_CNT_3                                    0xF0838C
+
+#define mmTPC4_QM_CP_FENCE2_CNT_4                                    0xF08390
+
+#define mmTPC4_QM_CP_FENCE3_CNT_0                                    0xF08394
+
+#define mmTPC4_QM_CP_FENCE3_CNT_1                                    0xF08398
+
+#define mmTPC4_QM_CP_FENCE3_CNT_2                                    0xF0839C
+
+#define mmTPC4_QM_CP_FENCE3_CNT_3                                    0xF083A0
+
+#define mmTPC4_QM_CP_FENCE3_CNT_4                                    0xF083A4
+
+#define mmTPC4_QM_CP_STS_0                                           0xF083A8
+
+#define mmTPC4_QM_CP_STS_1                                           0xF083AC
+
+#define mmTPC4_QM_CP_STS_2                                           0xF083B0
+
+#define mmTPC4_QM_CP_STS_3                                           0xF083B4
+
+#define mmTPC4_QM_CP_STS_4                                           0xF083B8
+
+#define mmTPC4_QM_CP_CURRENT_INST_LO_0                               0xF083BC
+
+#define mmTPC4_QM_CP_CURRENT_INST_LO_1                               0xF083C0
+
+#define mmTPC4_QM_CP_CURRENT_INST_LO_2                               0xF083C4
+
+#define mmTPC4_QM_CP_CURRENT_INST_LO_3                               0xF083C8
+
+#define mmTPC4_QM_CP_CURRENT_INST_LO_4                               0xF083CC
+
+#define mmTPC4_QM_CP_CURRENT_INST_HI_0                               0xF083D0
+
+#define mmTPC4_QM_CP_CURRENT_INST_HI_1                               0xF083D4
+
+#define mmTPC4_QM_CP_CURRENT_INST_HI_2                               0xF083D8
+
+#define mmTPC4_QM_CP_CURRENT_INST_HI_3                               0xF083DC
+
+#define mmTPC4_QM_CP_CURRENT_INST_HI_4                               0xF083E0
+
+#define mmTPC4_QM_CP_BARRIER_CFG_0                                   0xF083F4
+
+#define mmTPC4_QM_CP_BARRIER_CFG_1                                   0xF083F8
+
+#define mmTPC4_QM_CP_BARRIER_CFG_2                                   0xF083FC
+
+#define mmTPC4_QM_CP_BARRIER_CFG_3                                   0xF08400
+
+#define mmTPC4_QM_CP_BARRIER_CFG_4                                   0xF08404
+
+#define mmTPC4_QM_CP_DBG_0_0                                         0xF08408
+
+#define mmTPC4_QM_CP_DBG_0_1                                         0xF0840C
+
+#define mmTPC4_QM_CP_DBG_0_2                                         0xF08410
+
+#define mmTPC4_QM_CP_DBG_0_3                                         0xF08414
+
+#define mmTPC4_QM_CP_DBG_0_4                                         0xF08418
+
+#define mmTPC4_QM_CP_ARUSER_31_11_0                                  0xF0841C
+
+#define mmTPC4_QM_CP_ARUSER_31_11_1                                  0xF08420
+
+#define mmTPC4_QM_CP_ARUSER_31_11_2                                  0xF08424
+
+#define mmTPC4_QM_CP_ARUSER_31_11_3                                  0xF08428
+
+#define mmTPC4_QM_CP_ARUSER_31_11_4                                  0xF0842C
+
+#define mmTPC4_QM_CP_AWUSER_31_11_0                                  0xF08430
+
+#define mmTPC4_QM_CP_AWUSER_31_11_1                                  0xF08434
+
+#define mmTPC4_QM_CP_AWUSER_31_11_2                                  0xF08438
+
+#define mmTPC4_QM_CP_AWUSER_31_11_3                                  0xF0843C
+
+#define mmTPC4_QM_CP_AWUSER_31_11_4                                  0xF08440
+
+#define mmTPC4_QM_ARB_CFG_0                                          0xF08A00
+
+#define mmTPC4_QM_ARB_CHOISE_Q_PUSH                                  0xF08A04
+
+#define mmTPC4_QM_ARB_WRR_WEIGHT_0                                   0xF08A08
+
+#define mmTPC4_QM_ARB_WRR_WEIGHT_1                                   0xF08A0C
+
+#define mmTPC4_QM_ARB_WRR_WEIGHT_2                                   0xF08A10
+
+#define mmTPC4_QM_ARB_WRR_WEIGHT_3                                   0xF08A14
+
+#define mmTPC4_QM_ARB_CFG_1                                          0xF08A18
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_0                               0xF08A20
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_1                               0xF08A24
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_2                               0xF08A28
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_3                               0xF08A2C
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_4                               0xF08A30
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_5                               0xF08A34
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_6                               0xF08A38
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_7                               0xF08A3C
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_8                               0xF08A40
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_9                               0xF08A44
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_10                              0xF08A48
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_11                              0xF08A4C
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_12                              0xF08A50
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_13                              0xF08A54
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_14                              0xF08A58
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_15                              0xF08A5C
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_16                              0xF08A60
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_17                              0xF08A64
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_18                              0xF08A68
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_19                              0xF08A6C
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_20                              0xF08A70
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_21                              0xF08A74
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_22                              0xF08A78
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_23                              0xF08A7C
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_24                              0xF08A80
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_25                              0xF08A84
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_26                              0xF08A88
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_27                              0xF08A8C
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_28                              0xF08A90
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_29                              0xF08A94
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_30                              0xF08A98
+
+#define mmTPC4_QM_ARB_MST_AVAIL_CRED_31                              0xF08A9C
+
+#define mmTPC4_QM_ARB_MST_CRED_INC                                   0xF08AA0
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xF08AA4
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xF08AA8
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xF08AAC
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xF08AB0
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xF08AB4
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xF08AB8
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xF08ABC
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xF08AC0
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xF08AC4
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xF08AC8
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xF08ACC
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xF08AD0
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xF08AD4
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xF08AD8
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xF08ADC
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xF08AE0
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xF08AE4
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xF08AE8
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xF08AEC
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xF08AF0
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xF08AF4
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xF08AF8
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xF08AFC
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xF08B00
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xF08B04
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xF08B08
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xF08B0C
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xF08B10
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xF08B14
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xF08B18
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xF08B1C
+
+#define mmTPC4_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xF08B20
+
+#define mmTPC4_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xF08B28
+
+#define mmTPC4_QM_ARB_MST_SLAVE_EN                                   0xF08B2C
+
+#define mmTPC4_QM_ARB_MST_QUIET_PER                                  0xF08B34
+
+#define mmTPC4_QM_ARB_SLV_CHOISE_WDT                                 0xF08B38
+
+#define mmTPC4_QM_ARB_SLV_ID                                         0xF08B3C
+
+#define mmTPC4_QM_ARB_MSG_MAX_INFLIGHT                               0xF08B44
+
+#define mmTPC4_QM_ARB_MSG_AWUSER_31_11                               0xF08B48
+
+#define mmTPC4_QM_ARB_MSG_AWUSER_SEC_PROP                            0xF08B4C
+
+#define mmTPC4_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xF08B50
+
+#define mmTPC4_QM_ARB_BASE_LO                                        0xF08B54
+
+#define mmTPC4_QM_ARB_BASE_HI                                        0xF08B58
+
+#define mmTPC4_QM_ARB_STATE_STS                                      0xF08B80
+
+#define mmTPC4_QM_ARB_CHOISE_FULLNESS_STS                            0xF08B84
+
+#define mmTPC4_QM_ARB_MSG_STS                                        0xF08B88
+
+#define mmTPC4_QM_ARB_SLV_CHOISE_Q_HEAD                              0xF08B8C
+
+#define mmTPC4_QM_ARB_ERR_CAUSE                                      0xF08B9C
+
+#define mmTPC4_QM_ARB_ERR_MSG_EN                                     0xF08BA0
+
+#define mmTPC4_QM_ARB_ERR_STS_DRP                                    0xF08BA8
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_0                                 0xF08BB0
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_1                                 0xF08BB4
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_2                                 0xF08BB8
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_3                                 0xF08BBC
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_4                                 0xF08BC0
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_5                                 0xF08BC4
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_6                                 0xF08BC8
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_7                                 0xF08BCC
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_8                                 0xF08BD0
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_9                                 0xF08BD4
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_10                                0xF08BD8
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_11                                0xF08BDC
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_12                                0xF08BE0
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_13                                0xF08BE4
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_14                                0xF08BE8
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_15                                0xF08BEC
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_16                                0xF08BF0
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_17                                0xF08BF4
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_18                                0xF08BF8
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_19                                0xF08BFC
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_20                                0xF08C00
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_21                                0xF08C04
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_22                                0xF08C08
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_23                                0xF08C0C
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_24                                0xF08C10
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_25                                0xF08C14
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_26                                0xF08C18
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_27                                0xF08C1C
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_28                                0xF08C20
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_29                                0xF08C24
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_30                                0xF08C28
+
+#define mmTPC4_QM_ARB_MST_CRED_STS_31                                0xF08C2C
+
+#define mmTPC4_QM_CGM_CFG                                            0xF08C70
+
+#define mmTPC4_QM_CGM_STS                                            0xF08C74
+
+#define mmTPC4_QM_CGM_CFG1                                           0xF08C78
+
+#define mmTPC4_QM_LOCAL_RANGE_BASE                                   0xF08C80
+
+#define mmTPC4_QM_LOCAL_RANGE_SIZE                                   0xF08C84
+
+#define mmTPC4_QM_CSMR_STRICT_PRIO_CFG                               0xF08C90
+
+#define mmTPC4_QM_HBW_RD_RATE_LIM_CFG_1                              0xF08C94
+
+#define mmTPC4_QM_LBW_WR_RATE_LIM_CFG_0                              0xF08C98
+
+#define mmTPC4_QM_LBW_WR_RATE_LIM_CFG_1                              0xF08C9C
+
+#define mmTPC4_QM_HBW_RD_RATE_LIM_CFG_0                              0xF08CA0
+
+#define mmTPC4_QM_GLBL_AXCACHE                                       0xF08CA4
+
+#define mmTPC4_QM_IND_GW_APB_CFG                                     0xF08CB0
+
+#define mmTPC4_QM_IND_GW_APB_WDATA                                   0xF08CB4
+
+#define mmTPC4_QM_IND_GW_APB_RDATA                                   0xF08CB8
+
+#define mmTPC4_QM_IND_GW_APB_STATUS                                  0xF08CBC
+
+#define mmTPC4_QM_GLBL_ERR_ADDR_LO                                   0xF08CD0
+
+#define mmTPC4_QM_GLBL_ERR_ADDR_HI                                   0xF08CD4
+
+#define mmTPC4_QM_GLBL_ERR_WDATA                                     0xF08CD8
+
+#define mmTPC4_QM_GLBL_MEM_INIT_BUSY                                 0xF08D00
+
+#endif /* ASIC_REG_TPC4_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_cfg_regs.h
new file mode 100644
index 000000000000..f428f891935a
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC5_CFG_REGS_H_
+#define ASIC_REG_TPC5_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC5_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xF46400
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xF46404
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xF46408
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xF4640C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xF46410
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xF46414
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xF46418
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xF4641C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xF46420
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xF46424
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xF46428
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xF4642C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xF46430
+
+#define mmTPC5_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xF46434
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xF46438
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xF4643C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xF46440
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xF46444
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xF46448
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xF4644C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xF46450
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xF46454
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xF46458
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xF4645C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xF46460
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xF46464
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xF46468
+
+#define mmTPC5_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xF4646C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xF46470
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xF46474
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xF46478
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xF4647C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xF46480
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xF46484
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xF46488
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xF4648C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xF46490
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xF46494
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xF46498
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xF4649C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xF464A0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xF464A4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xF464A8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xF464AC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xF464B0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xF464B4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xF464B8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xF464BC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xF464C0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xF464C4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xF464C8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xF464CC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xF464D0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xF464D4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xF464D8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xF464DC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xF464E0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xF464E4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xF464E8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xF464EC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xF464F0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xF464F4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xF464F8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xF464FC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xF46500
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xF46504
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xF46508
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xF4650C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xF46510
+
+#define mmTPC5_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xF46514
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xF46518
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xF4651C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xF46520
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xF46524
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xF46528
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xF4652C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xF46530
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xF46534
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xF46538
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xF4653C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xF46540
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xF46544
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xF46548
+
+#define mmTPC5_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xF4654C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xF46550
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xF46554
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xF46558
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xF4655C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xF46560
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xF46564
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xF46568
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xF4656C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xF46570
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xF46574
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xF46578
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xF4657C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xF46580
+
+#define mmTPC5_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xF46584
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xF46588
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xF4658C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xF46590
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xF46594
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xF46598
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xF4659C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xF465A0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xF465A4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xF465A8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xF465AC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xF465B0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xF465B4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xF465B8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xF465BC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xF465C0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xF465C4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xF465C8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xF465CC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xF465D0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xF465D4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xF465D8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xF465DC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xF465E0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xF465E4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xF465E8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xF465EC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xF465F0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xF465F4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xF465F8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xF465FC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xF46600
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xF46604
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xF46608
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xF4660C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xF46610
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xF46614
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xF46618
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xF4661C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xF46620
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xF46624
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xF46628
+
+#define mmTPC5_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xF4662C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xF46630
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xF46634
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xF46638
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xF4663C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xF46640
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xF46644
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xF46648
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xF4664C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xF46650
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xF46654
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xF46658
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xF4665C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xF46660
+
+#define mmTPC5_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xF46664
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xF46668
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xF4666C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xF46670
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xF46674
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xF46678
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xF4667C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xF46680
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xF46684
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xF46688
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xF4668C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xF46690
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xF46694
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xF46698
+
+#define mmTPC5_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xF4669C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xF466A0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xF466A4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xF466A8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xF466AC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xF466B0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xF466B4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xF466B8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xF466BC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xF466C0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xF466C4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xF466C8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xF466CC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xF466D0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xF466D4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xF466D8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xF466DC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xF466E0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xF466E4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xF466E8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xF466EC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xF466F0
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xF466F4
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xF466F8
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xF466FC
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xF46700
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xF46704
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xF46708
+
+#define mmTPC5_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xF4670C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xF46710
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xF46714
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xF46718
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xF4671C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xF46720
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xF46724
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xF46728
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xF4672C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xF46730
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xF46734
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xF46738
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xF4673C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xF46740
+
+#define mmTPC5_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xF46744
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xF46748
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xF4674C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xF46750
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xF46754
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xF46758
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xF4675C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xF46760
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xF46764
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xF46768
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xF4676C
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xF46770
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xF46774
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xF46778
+
+#define mmTPC5_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xF4677C
+
+#define mmTPC5_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xF46780
+
+#define mmTPC5_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xF46784
+
+#define mmTPC5_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xF46788
+
+#define mmTPC5_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xF4678C
+
+#define mmTPC5_CFG_KERNEL_TID_BASE_DIM_0                             0xF46790
+
+#define mmTPC5_CFG_KERNEL_TID_SIZE_DIM_0                             0xF46794
+
+#define mmTPC5_CFG_KERNEL_TID_BASE_DIM_1                             0xF46798
+
+#define mmTPC5_CFG_KERNEL_TID_SIZE_DIM_1                             0xF4679C
+
+#define mmTPC5_CFG_KERNEL_TID_BASE_DIM_2                             0xF467A0
+
+#define mmTPC5_CFG_KERNEL_TID_SIZE_DIM_2                             0xF467A4
+
+#define mmTPC5_CFG_KERNEL_TID_BASE_DIM_3                             0xF467A8
+
+#define mmTPC5_CFG_KERNEL_TID_SIZE_DIM_3                             0xF467AC
+
+#define mmTPC5_CFG_KERNEL_TID_BASE_DIM_4                             0xF467B0
+
+#define mmTPC5_CFG_KERNEL_TID_SIZE_DIM_4                             0xF467B4
+
+#define mmTPC5_CFG_KERNEL_KERNEL_CONFIG                              0xF467B8
+
+#define mmTPC5_CFG_KERNEL_KERNEL_ID                                  0xF467BC
+
+#define mmTPC5_CFG_KERNEL_SRF_0                                      0xF467C0
+
+#define mmTPC5_CFG_KERNEL_SRF_1                                      0xF467C4
+
+#define mmTPC5_CFG_KERNEL_SRF_2                                      0xF467C8
+
+#define mmTPC5_CFG_KERNEL_SRF_3                                      0xF467CC
+
+#define mmTPC5_CFG_KERNEL_SRF_4                                      0xF467D0
+
+#define mmTPC5_CFG_KERNEL_SRF_5                                      0xF467D4
+
+#define mmTPC5_CFG_KERNEL_SRF_6                                      0xF467D8
+
+#define mmTPC5_CFG_KERNEL_SRF_7                                      0xF467DC
+
+#define mmTPC5_CFG_KERNEL_SRF_8                                      0xF467E0
+
+#define mmTPC5_CFG_KERNEL_SRF_9                                      0xF467E4
+
+#define mmTPC5_CFG_KERNEL_SRF_10                                     0xF467E8
+
+#define mmTPC5_CFG_KERNEL_SRF_11                                     0xF467EC
+
+#define mmTPC5_CFG_KERNEL_SRF_12                                     0xF467F0
+
+#define mmTPC5_CFG_KERNEL_SRF_13                                     0xF467F4
+
+#define mmTPC5_CFG_KERNEL_SRF_14                                     0xF467F8
+
+#define mmTPC5_CFG_KERNEL_SRF_15                                     0xF467FC
+
+#define mmTPC5_CFG_KERNEL_SRF_16                                     0xF46800
+
+#define mmTPC5_CFG_KERNEL_SRF_17                                     0xF46804
+
+#define mmTPC5_CFG_KERNEL_SRF_18                                     0xF46808
+
+#define mmTPC5_CFG_KERNEL_SRF_19                                     0xF4680C
+
+#define mmTPC5_CFG_KERNEL_SRF_20                                     0xF46810
+
+#define mmTPC5_CFG_KERNEL_SRF_21                                     0xF46814
+
+#define mmTPC5_CFG_KERNEL_SRF_22                                     0xF46818
+
+#define mmTPC5_CFG_KERNEL_SRF_23                                     0xF4681C
+
+#define mmTPC5_CFG_KERNEL_SRF_24                                     0xF46820
+
+#define mmTPC5_CFG_KERNEL_SRF_25                                     0xF46824
+
+#define mmTPC5_CFG_KERNEL_SRF_26                                     0xF46828
+
+#define mmTPC5_CFG_KERNEL_SRF_27                                     0xF4682C
+
+#define mmTPC5_CFG_KERNEL_SRF_28                                     0xF46830
+
+#define mmTPC5_CFG_KERNEL_SRF_29                                     0xF46834
+
+#define mmTPC5_CFG_KERNEL_SRF_30                                     0xF46838
+
+#define mmTPC5_CFG_KERNEL_SRF_31                                     0xF4683C
+
+#define mmTPC5_CFG_ROUND_CSR                                         0xF468FC
+
+#define mmTPC5_CFG_PROT                                              0xF46900
+
+#define mmTPC5_CFG_SEMAPHORE                                         0xF46908
+
+#define mmTPC5_CFG_VFLAGS                                            0xF4690C
+
+#define mmTPC5_CFG_SFLAGS                                            0xF46910
+
+#define mmTPC5_CFG_LFSR_POLYNOM                                      0xF46918
+
+#define mmTPC5_CFG_STATUS                                            0xF4691C
+
+#define mmTPC5_CFG_CFG_BASE_ADDRESS_HIGH                             0xF46920
+
+#define mmTPC5_CFG_CFG_SUBTRACT_VALUE                                0xF46924
+
+#define mmTPC5_CFG_SM_BASE_ADDRESS_HIGH                              0xF4692C
+
+#define mmTPC5_CFG_TPC_CMD                                           0xF46930
+
+#define mmTPC5_CFG_TPC_EXECUTE                                       0xF46938
+
+#define mmTPC5_CFG_TPC_STALL                                         0xF4693C
+
+#define mmTPC5_CFG_ICACHE_BASE_ADDERESS_LOW                          0xF46940
+
+#define mmTPC5_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xF46944
+
+#define mmTPC5_CFG_RD_RATE_LIMIT                                     0xF46948
+
+#define mmTPC5_CFG_WR_RATE_LIMIT                                     0xF46950
+
+#define mmTPC5_CFG_MSS_CONFIG                                        0xF46954
+
+#define mmTPC5_CFG_TPC_INTR_CAUSE                                    0xF46958
+
+#define mmTPC5_CFG_TPC_INTR_MASK                                     0xF4695C
+
+#define mmTPC5_CFG_WQ_CREDITS                                        0xF46960
+
+#define mmTPC5_CFG_ARUSER_LO                                         0xF46964
+
+#define mmTPC5_CFG_ARUSER_HI                                         0xF46968
+
+#define mmTPC5_CFG_AWUSER_LO                                         0xF4696C
+
+#define mmTPC5_CFG_AWUSER_HI                                         0xF46970
+
+#define mmTPC5_CFG_OPCODE_EXEC                                       0xF46974
+
+#define mmTPC5_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xF46978
+
+#define mmTPC5_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xF4697C
+
+#define mmTPC5_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xF46980
+
+#define mmTPC5_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xF46984
+
+#define mmTPC5_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xF46988
+
+#define mmTPC5_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xF4698C
+
+#define mmTPC5_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xF46990
+
+#define mmTPC5_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xF46994
+
+#define mmTPC5_CFG_TSB_CFG_MAX_SIZE                                  0xF46998
+
+#define mmTPC5_CFG_TSB_CFG                                           0xF4699C
+
+#define mmTPC5_CFG_DBGMEM_ADD                                        0xF469A0
+
+#define mmTPC5_CFG_DBGMEM_DATA_WR                                    0xF469A4
+
+#define mmTPC5_CFG_DBGMEM_DATA_RD                                    0xF469A8
+
+#define mmTPC5_CFG_DBGMEM_CTRL                                       0xF469AC
+
+#define mmTPC5_CFG_DBGMEM_RC                                         0xF469B0
+
+#define mmTPC5_CFG_TSB_INFLIGHT_CNTR                                 0xF469B4
+
+#define mmTPC5_CFG_WQ_INFLIGHT_CNTR                                  0xF469B8
+
+#define mmTPC5_CFG_WQ_LBW_TOTAL_CNTR                                 0xF469BC
+
+#define mmTPC5_CFG_WQ_HBW_TOTAL_CNTR                                 0xF469C0
+
+#define mmTPC5_CFG_IRQ_OCCOUPY_CNTR                                  0xF469C4
+
+#define mmTPC5_CFG_FUNC_MBIST_CNTRL                                  0xF469D0
+
+#define mmTPC5_CFG_FUNC_MBIST_PAT                                    0xF469D4
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_0                                  0xF469D8
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_1                                  0xF469DC
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_2                                  0xF469E0
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_3                                  0xF469E4
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_4                                  0xF469E8
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_5                                  0xF469EC
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_6                                  0xF469F0
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_7                                  0xF469F4
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_8                                  0xF469F8
+
+#define mmTPC5_CFG_FUNC_MBIST_MEM_9                                  0xF469FC
+
+#define mmTPC5_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xF46A00
+
+#define mmTPC5_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xF46A04
+
+#define mmTPC5_CFG_QM_TENSOR_0_PADDING_VALUE                         0xF46A08
+
+#define mmTPC5_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xF46A0C
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xF46A10
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xF46A14
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xF46A18
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xF46A1C
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xF46A20
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xF46A24
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xF46A28
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xF46A2C
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xF46A30
+
+#define mmTPC5_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xF46A34
+
+#define mmTPC5_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xF46A38
+
+#define mmTPC5_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xF46A3C
+
+#define mmTPC5_CFG_QM_TENSOR_1_PADDING_VALUE                         0xF46A40
+
+#define mmTPC5_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xF46A44
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xF46A48
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xF46A4C
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xF46A50
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xF46A54
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xF46A58
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xF46A5C
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xF46A60
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xF46A64
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xF46A68
+
+#define mmTPC5_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xF46A6C
+
+#define mmTPC5_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xF46A70
+
+#define mmTPC5_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xF46A74
+
+#define mmTPC5_CFG_QM_TENSOR_2_PADDING_VALUE                         0xF46A78
+
+#define mmTPC5_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xF46A7C
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xF46A80
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xF46A84
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xF46A88
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xF46A8C
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xF46A90
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xF46A94
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xF46A98
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xF46A9C
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xF46AA0
+
+#define mmTPC5_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xF46AA4
+
+#define mmTPC5_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xF46AA8
+
+#define mmTPC5_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xF46AAC
+
+#define mmTPC5_CFG_QM_TENSOR_3_PADDING_VALUE                         0xF46AB0
+
+#define mmTPC5_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xF46AB4
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xF46AB8
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xF46ABC
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xF46AC0
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xF46AC4
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xF46AC8
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xF46ACC
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xF46AD0
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xF46AD4
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xF46AD8
+
+#define mmTPC5_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xF46ADC
+
+#define mmTPC5_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xF46AE0
+
+#define mmTPC5_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xF46AE4
+
+#define mmTPC5_CFG_QM_TENSOR_4_PADDING_VALUE                         0xF46AE8
+
+#define mmTPC5_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xF46AEC
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xF46AF0
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xF46AF4
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xF46AF8
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xF46AFC
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xF46B00
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xF46B04
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xF46B08
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xF46B0C
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xF46B10
+
+#define mmTPC5_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xF46B14
+
+#define mmTPC5_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xF46B18
+
+#define mmTPC5_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xF46B1C
+
+#define mmTPC5_CFG_QM_TENSOR_5_PADDING_VALUE                         0xF46B20
+
+#define mmTPC5_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xF46B24
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xF46B28
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xF46B2C
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xF46B30
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xF46B34
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xF46B38
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xF46B3C
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xF46B40
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xF46B44
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xF46B48
+
+#define mmTPC5_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xF46B4C
+
+#define mmTPC5_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xF46B50
+
+#define mmTPC5_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xF46B54
+
+#define mmTPC5_CFG_QM_TENSOR_6_PADDING_VALUE                         0xF46B58
+
+#define mmTPC5_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xF46B5C
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xF46B60
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xF46B64
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xF46B68
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xF46B6C
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xF46B70
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xF46B74
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xF46B78
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xF46B7C
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xF46B80
+
+#define mmTPC5_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xF46B84
+
+#define mmTPC5_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xF46B88
+
+#define mmTPC5_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xF46B8C
+
+#define mmTPC5_CFG_QM_TENSOR_7_PADDING_VALUE                         0xF46B90
+
+#define mmTPC5_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xF46B94
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xF46B98
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xF46B9C
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xF46BA0
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xF46BA4
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xF46BA8
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xF46BAC
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xF46BB0
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xF46BB4
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xF46BB8
+
+#define mmTPC5_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xF46BBC
+
+#define mmTPC5_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xF46BC0
+
+#define mmTPC5_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xF46BC4
+
+#define mmTPC5_CFG_QM_TENSOR_8_PADDING_VALUE                         0xF46BC8
+
+#define mmTPC5_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xF46BCC
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xF46BD0
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xF46BD4
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xF46BD8
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xF46BDC
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xF46BE0
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xF46BE4
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xF46BE8
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xF46BEC
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xF46BF0
+
+#define mmTPC5_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xF46BF4
+
+#define mmTPC5_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xF46BF8
+
+#define mmTPC5_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xF46BFC
+
+#define mmTPC5_CFG_QM_TENSOR_9_PADDING_VALUE                         0xF46C00
+
+#define mmTPC5_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xF46C04
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xF46C08
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xF46C0C
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xF46C10
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xF46C14
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xF46C18
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xF46C1C
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xF46C20
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xF46C24
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xF46C28
+
+#define mmTPC5_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xF46C2C
+
+#define mmTPC5_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xF46C30
+
+#define mmTPC5_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xF46C34
+
+#define mmTPC5_CFG_QM_TENSOR_10_PADDING_VALUE                        0xF46C38
+
+#define mmTPC5_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xF46C3C
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xF46C40
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xF46C44
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xF46C48
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xF46C4C
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xF46C50
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xF46C54
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xF46C58
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xF46C5C
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xF46C60
+
+#define mmTPC5_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xF46C64
+
+#define mmTPC5_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xF46C68
+
+#define mmTPC5_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xF46C6C
+
+#define mmTPC5_CFG_QM_TENSOR_11_PADDING_VALUE                        0xF46C70
+
+#define mmTPC5_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xF46C74
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xF46C78
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xF46C7C
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xF46C80
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xF46C84
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xF46C88
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xF46C8C
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xF46C90
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xF46C94
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xF46C98
+
+#define mmTPC5_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xF46C9C
+
+#define mmTPC5_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xF46CA0
+
+#define mmTPC5_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xF46CA4
+
+#define mmTPC5_CFG_QM_TENSOR_12_PADDING_VALUE                        0xF46CA8
+
+#define mmTPC5_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xF46CAC
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xF46CB0
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xF46CB4
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xF46CB8
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xF46CBC
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xF46CC0
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xF46CC4
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xF46CC8
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xF46CCC
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xF46CD0
+
+#define mmTPC5_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xF46CD4
+
+#define mmTPC5_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xF46CD8
+
+#define mmTPC5_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xF46CDC
+
+#define mmTPC5_CFG_QM_TENSOR_13_PADDING_VALUE                        0xF46CE0
+
+#define mmTPC5_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xF46CE4
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xF46CE8
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xF46CEC
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xF46CF0
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xF46CF4
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xF46CF8
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xF46CFC
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xF46D00
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xF46D04
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xF46D08
+
+#define mmTPC5_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xF46D0C
+
+#define mmTPC5_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xF46D10
+
+#define mmTPC5_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xF46D14
+
+#define mmTPC5_CFG_QM_TENSOR_14_PADDING_VALUE                        0xF46D18
+
+#define mmTPC5_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xF46D1C
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xF46D20
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xF46D24
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xF46D28
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xF46D2C
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xF46D30
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xF46D34
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xF46D38
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xF46D3C
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xF46D40
+
+#define mmTPC5_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xF46D44
+
+#define mmTPC5_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xF46D48
+
+#define mmTPC5_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xF46D4C
+
+#define mmTPC5_CFG_QM_TENSOR_15_PADDING_VALUE                        0xF46D50
+
+#define mmTPC5_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xF46D54
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xF46D58
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xF46D5C
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xF46D60
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xF46D64
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xF46D68
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xF46D6C
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xF46D70
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xF46D74
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xF46D78
+
+#define mmTPC5_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xF46D7C
+
+#define mmTPC5_CFG_QM_SYNC_OBJECT_MESSAGE                            0xF46D80
+
+#define mmTPC5_CFG_QM_SYNC_OBJECT_ADDR                               0xF46D84
+
+#define mmTPC5_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xF46D88
+
+#define mmTPC5_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xF46D8C
+
+#define mmTPC5_CFG_QM_TID_BASE_DIM_0                                 0xF46D90
+
+#define mmTPC5_CFG_QM_TID_SIZE_DIM_0                                 0xF46D94
+
+#define mmTPC5_CFG_QM_TID_BASE_DIM_1                                 0xF46D98
+
+#define mmTPC5_CFG_QM_TID_SIZE_DIM_1                                 0xF46D9C
+
+#define mmTPC5_CFG_QM_TID_BASE_DIM_2                                 0xF46DA0
+
+#define mmTPC5_CFG_QM_TID_SIZE_DIM_2                                 0xF46DA4
+
+#define mmTPC5_CFG_QM_TID_BASE_DIM_3                                 0xF46DA8
+
+#define mmTPC5_CFG_QM_TID_SIZE_DIM_3                                 0xF46DAC
+
+#define mmTPC5_CFG_QM_TID_BASE_DIM_4                                 0xF46DB0
+
+#define mmTPC5_CFG_QM_TID_SIZE_DIM_4                                 0xF46DB4
+
+#define mmTPC5_CFG_QM_KERNEL_CONFIG                                  0xF46DB8
+
+#define mmTPC5_CFG_QM_KERNEL_ID                                      0xF46DBC
+
+#define mmTPC5_CFG_QM_SRF_0                                          0xF46DC0
+
+#define mmTPC5_CFG_QM_SRF_1                                          0xF46DC4
+
+#define mmTPC5_CFG_QM_SRF_2                                          0xF46DC8
+
+#define mmTPC5_CFG_QM_SRF_3                                          0xF46DCC
+
+#define mmTPC5_CFG_QM_SRF_4                                          0xF46DD0
+
+#define mmTPC5_CFG_QM_SRF_5                                          0xF46DD4
+
+#define mmTPC5_CFG_QM_SRF_6                                          0xF46DD8
+
+#define mmTPC5_CFG_QM_SRF_7                                          0xF46DDC
+
+#define mmTPC5_CFG_QM_SRF_8                                          0xF46DE0
+
+#define mmTPC5_CFG_QM_SRF_9                                          0xF46DE4
+
+#define mmTPC5_CFG_QM_SRF_10                                         0xF46DE8
+
+#define mmTPC5_CFG_QM_SRF_11                                         0xF46DEC
+
+#define mmTPC5_CFG_QM_SRF_12                                         0xF46DF0
+
+#define mmTPC5_CFG_QM_SRF_13                                         0xF46DF4
+
+#define mmTPC5_CFG_QM_SRF_14                                         0xF46DF8
+
+#define mmTPC5_CFG_QM_SRF_15                                         0xF46DFC
+
+#define mmTPC5_CFG_QM_SRF_16                                         0xF46E00
+
+#define mmTPC5_CFG_QM_SRF_17                                         0xF46E04
+
+#define mmTPC5_CFG_QM_SRF_18                                         0xF46E08
+
+#define mmTPC5_CFG_QM_SRF_19                                         0xF46E0C
+
+#define mmTPC5_CFG_QM_SRF_20                                         0xF46E10
+
+#define mmTPC5_CFG_QM_SRF_21                                         0xF46E14
+
+#define mmTPC5_CFG_QM_SRF_22                                         0xF46E18
+
+#define mmTPC5_CFG_QM_SRF_23                                         0xF46E1C
+
+#define mmTPC5_CFG_QM_SRF_24                                         0xF46E20
+
+#define mmTPC5_CFG_QM_SRF_25                                         0xF46E24
+
+#define mmTPC5_CFG_QM_SRF_26                                         0xF46E28
+
+#define mmTPC5_CFG_QM_SRF_27                                         0xF46E2C
+
+#define mmTPC5_CFG_QM_SRF_28                                         0xF46E30
+
+#define mmTPC5_CFG_QM_SRF_29                                         0xF46E34
+
+#define mmTPC5_CFG_QM_SRF_30                                         0xF46E38
+
+#define mmTPC5_CFG_QM_SRF_31                                         0xF46E3C
+
+#endif /* ASIC_REG_TPC5_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_qm_regs.h
new file mode 100644
index 000000000000..cd3a810ff4c4
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc5_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC5_QM_REGS_H_
+#define ASIC_REG_TPC5_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC5_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC5_QM_GLBL_CFG0                                          0xF48000
+
+#define mmTPC5_QM_GLBL_CFG1                                          0xF48004
+
+#define mmTPC5_QM_GLBL_PROT                                          0xF48008
+
+#define mmTPC5_QM_GLBL_ERR_CFG                                       0xF4800C
+
+#define mmTPC5_QM_GLBL_SECURE_PROPS_0                                0xF48010
+
+#define mmTPC5_QM_GLBL_SECURE_PROPS_1                                0xF48014
+
+#define mmTPC5_QM_GLBL_SECURE_PROPS_2                                0xF48018
+
+#define mmTPC5_QM_GLBL_SECURE_PROPS_3                                0xF4801C
+
+#define mmTPC5_QM_GLBL_SECURE_PROPS_4                                0xF48020
+
+#define mmTPC5_QM_GLBL_NON_SECURE_PROPS_0                            0xF48024
+
+#define mmTPC5_QM_GLBL_NON_SECURE_PROPS_1                            0xF48028
+
+#define mmTPC5_QM_GLBL_NON_SECURE_PROPS_2                            0xF4802C
+
+#define mmTPC5_QM_GLBL_NON_SECURE_PROPS_3                            0xF48030
+
+#define mmTPC5_QM_GLBL_NON_SECURE_PROPS_4                            0xF48034
+
+#define mmTPC5_QM_GLBL_STS0                                          0xF48038
+
+#define mmTPC5_QM_GLBL_STS1_0                                        0xF48040
+
+#define mmTPC5_QM_GLBL_STS1_1                                        0xF48044
+
+#define mmTPC5_QM_GLBL_STS1_2                                        0xF48048
+
+#define mmTPC5_QM_GLBL_STS1_3                                        0xF4804C
+
+#define mmTPC5_QM_GLBL_STS1_4                                        0xF48050
+
+#define mmTPC5_QM_GLBL_MSG_EN_0                                      0xF48054
+
+#define mmTPC5_QM_GLBL_MSG_EN_1                                      0xF48058
+
+#define mmTPC5_QM_GLBL_MSG_EN_2                                      0xF4805C
+
+#define mmTPC5_QM_GLBL_MSG_EN_3                                      0xF48060
+
+#define mmTPC5_QM_GLBL_MSG_EN_4                                      0xF48068
+
+#define mmTPC5_QM_PQ_BASE_LO_0                                       0xF48070
+
+#define mmTPC5_QM_PQ_BASE_LO_1                                       0xF48074
+
+#define mmTPC5_QM_PQ_BASE_LO_2                                       0xF48078
+
+#define mmTPC5_QM_PQ_BASE_LO_3                                       0xF4807C
+
+#define mmTPC5_QM_PQ_BASE_HI_0                                       0xF48080
+
+#define mmTPC5_QM_PQ_BASE_HI_1                                       0xF48084
+
+#define mmTPC5_QM_PQ_BASE_HI_2                                       0xF48088
+
+#define mmTPC5_QM_PQ_BASE_HI_3                                       0xF4808C
+
+#define mmTPC5_QM_PQ_SIZE_0                                          0xF48090
+
+#define mmTPC5_QM_PQ_SIZE_1                                          0xF48094
+
+#define mmTPC5_QM_PQ_SIZE_2                                          0xF48098
+
+#define mmTPC5_QM_PQ_SIZE_3                                          0xF4809C
+
+#define mmTPC5_QM_PQ_PI_0                                            0xF480A0
+
+#define mmTPC5_QM_PQ_PI_1                                            0xF480A4
+
+#define mmTPC5_QM_PQ_PI_2                                            0xF480A8
+
+#define mmTPC5_QM_PQ_PI_3                                            0xF480AC
+
+#define mmTPC5_QM_PQ_CI_0                                            0xF480B0
+
+#define mmTPC5_QM_PQ_CI_1                                            0xF480B4
+
+#define mmTPC5_QM_PQ_CI_2                                            0xF480B8
+
+#define mmTPC5_QM_PQ_CI_3                                            0xF480BC
+
+#define mmTPC5_QM_PQ_CFG0_0                                          0xF480C0
+
+#define mmTPC5_QM_PQ_CFG0_1                                          0xF480C4
+
+#define mmTPC5_QM_PQ_CFG0_2                                          0xF480C8
+
+#define mmTPC5_QM_PQ_CFG0_3                                          0xF480CC
+
+#define mmTPC5_QM_PQ_CFG1_0                                          0xF480D0
+
+#define mmTPC5_QM_PQ_CFG1_1                                          0xF480D4
+
+#define mmTPC5_QM_PQ_CFG1_2                                          0xF480D8
+
+#define mmTPC5_QM_PQ_CFG1_3                                          0xF480DC
+
+#define mmTPC5_QM_PQ_ARUSER_31_11_0                                  0xF480E0
+
+#define mmTPC5_QM_PQ_ARUSER_31_11_1                                  0xF480E4
+
+#define mmTPC5_QM_PQ_ARUSER_31_11_2                                  0xF480E8
+
+#define mmTPC5_QM_PQ_ARUSER_31_11_3                                  0xF480EC
+
+#define mmTPC5_QM_PQ_STS0_0                                          0xF480F0
+
+#define mmTPC5_QM_PQ_STS0_1                                          0xF480F4
+
+#define mmTPC5_QM_PQ_STS0_2                                          0xF480F8
+
+#define mmTPC5_QM_PQ_STS0_3                                          0xF480FC
+
+#define mmTPC5_QM_PQ_STS1_0                                          0xF48100
+
+#define mmTPC5_QM_PQ_STS1_1                                          0xF48104
+
+#define mmTPC5_QM_PQ_STS1_2                                          0xF48108
+
+#define mmTPC5_QM_PQ_STS1_3                                          0xF4810C
+
+#define mmTPC5_QM_CQ_CFG0_0                                          0xF48110
+
+#define mmTPC5_QM_CQ_CFG0_1                                          0xF48114
+
+#define mmTPC5_QM_CQ_CFG0_2                                          0xF48118
+
+#define mmTPC5_QM_CQ_CFG0_3                                          0xF4811C
+
+#define mmTPC5_QM_CQ_CFG0_4                                          0xF48120
+
+#define mmTPC5_QM_CQ_CFG1_0                                          0xF48124
+
+#define mmTPC5_QM_CQ_CFG1_1                                          0xF48128
+
+#define mmTPC5_QM_CQ_CFG1_2                                          0xF4812C
+
+#define mmTPC5_QM_CQ_CFG1_3                                          0xF48130
+
+#define mmTPC5_QM_CQ_CFG1_4                                          0xF48134
+
+#define mmTPC5_QM_CQ_ARUSER_31_11_0                                  0xF48138
+
+#define mmTPC5_QM_CQ_ARUSER_31_11_1                                  0xF4813C
+
+#define mmTPC5_QM_CQ_ARUSER_31_11_2                                  0xF48140
+
+#define mmTPC5_QM_CQ_ARUSER_31_11_3                                  0xF48144
+
+#define mmTPC5_QM_CQ_ARUSER_31_11_4                                  0xF48148
+
+#define mmTPC5_QM_CQ_STS0_0                                          0xF4814C
+
+#define mmTPC5_QM_CQ_STS0_1                                          0xF48150
+
+#define mmTPC5_QM_CQ_STS0_2                                          0xF48154
+
+#define mmTPC5_QM_CQ_STS0_3                                          0xF48158
+
+#define mmTPC5_QM_CQ_STS0_4                                          0xF4815C
+
+#define mmTPC5_QM_CQ_STS1_0                                          0xF48160
+
+#define mmTPC5_QM_CQ_STS1_1                                          0xF48164
+
+#define mmTPC5_QM_CQ_STS1_2                                          0xF48168
+
+#define mmTPC5_QM_CQ_STS1_3                                          0xF4816C
+
+#define mmTPC5_QM_CQ_STS1_4                                          0xF48170
+
+#define mmTPC5_QM_CQ_PTR_LO_0                                        0xF48174
+
+#define mmTPC5_QM_CQ_PTR_HI_0                                        0xF48178
+
+#define mmTPC5_QM_CQ_TSIZE_0                                         0xF4817C
+
+#define mmTPC5_QM_CQ_CTL_0                                           0xF48180
+
+#define mmTPC5_QM_CQ_PTR_LO_1                                        0xF48184
+
+#define mmTPC5_QM_CQ_PTR_HI_1                                        0xF48188
+
+#define mmTPC5_QM_CQ_TSIZE_1                                         0xF4818C
+
+#define mmTPC5_QM_CQ_CTL_1                                           0xF48190
+
+#define mmTPC5_QM_CQ_PTR_LO_2                                        0xF48194
+
+#define mmTPC5_QM_CQ_PTR_HI_2                                        0xF48198
+
+#define mmTPC5_QM_CQ_TSIZE_2                                         0xF4819C
+
+#define mmTPC5_QM_CQ_CTL_2                                           0xF481A0
+
+#define mmTPC5_QM_CQ_PTR_LO_3                                        0xF481A4
+
+#define mmTPC5_QM_CQ_PTR_HI_3                                        0xF481A8
+
+#define mmTPC5_QM_CQ_TSIZE_3                                         0xF481AC
+
+#define mmTPC5_QM_CQ_CTL_3                                           0xF481B0
+
+#define mmTPC5_QM_CQ_PTR_LO_4                                        0xF481B4
+
+#define mmTPC5_QM_CQ_PTR_HI_4                                        0xF481B8
+
+#define mmTPC5_QM_CQ_TSIZE_4                                         0xF481BC
+
+#define mmTPC5_QM_CQ_CTL_4                                           0xF481C0
+
+#define mmTPC5_QM_CQ_PTR_LO_STS_0                                    0xF481C4
+
+#define mmTPC5_QM_CQ_PTR_LO_STS_1                                    0xF481C8
+
+#define mmTPC5_QM_CQ_PTR_LO_STS_2                                    0xF481CC
+
+#define mmTPC5_QM_CQ_PTR_LO_STS_3                                    0xF481D0
+
+#define mmTPC5_QM_CQ_PTR_LO_STS_4                                    0xF481D4
+
+#define mmTPC5_QM_CQ_PTR_HI_STS_0                                    0xF481D8
+
+#define mmTPC5_QM_CQ_PTR_HI_STS_1                                    0xF481DC
+
+#define mmTPC5_QM_CQ_PTR_HI_STS_2                                    0xF481E0
+
+#define mmTPC5_QM_CQ_PTR_HI_STS_3                                    0xF481E4
+
+#define mmTPC5_QM_CQ_PTR_HI_STS_4                                    0xF481E8
+
+#define mmTPC5_QM_CQ_TSIZE_STS_0                                     0xF481EC
+
+#define mmTPC5_QM_CQ_TSIZE_STS_1                                     0xF481F0
+
+#define mmTPC5_QM_CQ_TSIZE_STS_2                                     0xF481F4
+
+#define mmTPC5_QM_CQ_TSIZE_STS_3                                     0xF481F8
+
+#define mmTPC5_QM_CQ_TSIZE_STS_4                                     0xF481FC
+
+#define mmTPC5_QM_CQ_CTL_STS_0                                       0xF48200
+
+#define mmTPC5_QM_CQ_CTL_STS_1                                       0xF48204
+
+#define mmTPC5_QM_CQ_CTL_STS_2                                       0xF48208
+
+#define mmTPC5_QM_CQ_CTL_STS_3                                       0xF4820C
+
+#define mmTPC5_QM_CQ_CTL_STS_4                                       0xF48210
+
+#define mmTPC5_QM_CQ_IFIFO_CNT_0                                     0xF48214
+
+#define mmTPC5_QM_CQ_IFIFO_CNT_1                                     0xF48218
+
+#define mmTPC5_QM_CQ_IFIFO_CNT_2                                     0xF4821C
+
+#define mmTPC5_QM_CQ_IFIFO_CNT_3                                     0xF48220
+
+#define mmTPC5_QM_CQ_IFIFO_CNT_4                                     0xF48224
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_0                             0xF48228
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_1                             0xF4822C
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_2                             0xF48230
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_3                             0xF48234
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_LO_4                             0xF48238
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_0                             0xF4823C
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_1                             0xF48240
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_2                             0xF48244
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_3                             0xF48248
+
+#define mmTPC5_QM_CP_MSG_BASE0_ADDR_HI_4                             0xF4824C
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_0                             0xF48250
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_1                             0xF48254
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_2                             0xF48258
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_3                             0xF4825C
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_LO_4                             0xF48260
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_0                             0xF48264
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_1                             0xF48268
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_2                             0xF4826C
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_3                             0xF48270
+
+#define mmTPC5_QM_CP_MSG_BASE1_ADDR_HI_4                             0xF48274
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_0                             0xF48278
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_1                             0xF4827C
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_2                             0xF48280
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_3                             0xF48284
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_LO_4                             0xF48288
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_0                             0xF4828C
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_1                             0xF48290
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_2                             0xF48294
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_3                             0xF48298
+
+#define mmTPC5_QM_CP_MSG_BASE2_ADDR_HI_4                             0xF4829C
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_0                             0xF482A0
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_1                             0xF482A4
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_2                             0xF482A8
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_3                             0xF482AC
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_LO_4                             0xF482B0
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_0                             0xF482B4
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_1                             0xF482B8
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_2                             0xF482BC
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_3                             0xF482C0
+
+#define mmTPC5_QM_CP_MSG_BASE3_ADDR_HI_4                             0xF482C4
+
+#define mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_0                             0xF482C8
+
+#define mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_1                             0xF482CC
+
+#define mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_2                             0xF482D0
+
+#define mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_3                             0xF482D4
+
+#define mmTPC5_QM_CP_LDMA_TSIZE_OFFSET_4                             0xF482D8
+
+#define mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xF482E0
+
+#define mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xF482E4
+
+#define mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xF482E8
+
+#define mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xF482EC
+
+#define mmTPC5_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xF482F0
+
+#define mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xF482F4
+
+#define mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xF482F8
+
+#define mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xF482FC
+
+#define mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xF48300
+
+#define mmTPC5_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xF48304
+
+#define mmTPC5_QM_CP_FENCE0_RDATA_0                                  0xF48308
+
+#define mmTPC5_QM_CP_FENCE0_RDATA_1                                  0xF4830C
+
+#define mmTPC5_QM_CP_FENCE0_RDATA_2                                  0xF48310
+
+#define mmTPC5_QM_CP_FENCE0_RDATA_3                                  0xF48314
+
+#define mmTPC5_QM_CP_FENCE0_RDATA_4                                  0xF48318
+
+#define mmTPC5_QM_CP_FENCE1_RDATA_0                                  0xF4831C
+
+#define mmTPC5_QM_CP_FENCE1_RDATA_1                                  0xF48320
+
+#define mmTPC5_QM_CP_FENCE1_RDATA_2                                  0xF48324
+
+#define mmTPC5_QM_CP_FENCE1_RDATA_3                                  0xF48328
+
+#define mmTPC5_QM_CP_FENCE1_RDATA_4                                  0xF4832C
+
+#define mmTPC5_QM_CP_FENCE2_RDATA_0                                  0xF48330
+
+#define mmTPC5_QM_CP_FENCE2_RDATA_1                                  0xF48334
+
+#define mmTPC5_QM_CP_FENCE2_RDATA_2                                  0xF48338
+
+#define mmTPC5_QM_CP_FENCE2_RDATA_3                                  0xF4833C
+
+#define mmTPC5_QM_CP_FENCE2_RDATA_4                                  0xF48340
+
+#define mmTPC5_QM_CP_FENCE3_RDATA_0                                  0xF48344
+
+#define mmTPC5_QM_CP_FENCE3_RDATA_1                                  0xF48348
+
+#define mmTPC5_QM_CP_FENCE3_RDATA_2                                  0xF4834C
+
+#define mmTPC5_QM_CP_FENCE3_RDATA_3                                  0xF48350
+
+#define mmTPC5_QM_CP_FENCE3_RDATA_4                                  0xF48354
+
+#define mmTPC5_QM_CP_FENCE0_CNT_0                                    0xF48358
+
+#define mmTPC5_QM_CP_FENCE0_CNT_1                                    0xF4835C
+
+#define mmTPC5_QM_CP_FENCE0_CNT_2                                    0xF48360
+
+#define mmTPC5_QM_CP_FENCE0_CNT_3                                    0xF48364
+
+#define mmTPC5_QM_CP_FENCE0_CNT_4                                    0xF48368
+
+#define mmTPC5_QM_CP_FENCE1_CNT_0                                    0xF4836C
+
+#define mmTPC5_QM_CP_FENCE1_CNT_1                                    0xF48370
+
+#define mmTPC5_QM_CP_FENCE1_CNT_2                                    0xF48374
+
+#define mmTPC5_QM_CP_FENCE1_CNT_3                                    0xF48378
+
+#define mmTPC5_QM_CP_FENCE1_CNT_4                                    0xF4837C
+
+#define mmTPC5_QM_CP_FENCE2_CNT_0                                    0xF48380
+
+#define mmTPC5_QM_CP_FENCE2_CNT_1                                    0xF48384
+
+#define mmTPC5_QM_CP_FENCE2_CNT_2                                    0xF48388
+
+#define mmTPC5_QM_CP_FENCE2_CNT_3                                    0xF4838C
+
+#define mmTPC5_QM_CP_FENCE2_CNT_4                                    0xF48390
+
+#define mmTPC5_QM_CP_FENCE3_CNT_0                                    0xF48394
+
+#define mmTPC5_QM_CP_FENCE3_CNT_1                                    0xF48398
+
+#define mmTPC5_QM_CP_FENCE3_CNT_2                                    0xF4839C
+
+#define mmTPC5_QM_CP_FENCE3_CNT_3                                    0xF483A0
+
+#define mmTPC5_QM_CP_FENCE3_CNT_4                                    0xF483A4
+
+#define mmTPC5_QM_CP_STS_0                                           0xF483A8
+
+#define mmTPC5_QM_CP_STS_1                                           0xF483AC
+
+#define mmTPC5_QM_CP_STS_2                                           0xF483B0
+
+#define mmTPC5_QM_CP_STS_3                                           0xF483B4
+
+#define mmTPC5_QM_CP_STS_4                                           0xF483B8
+
+#define mmTPC5_QM_CP_CURRENT_INST_LO_0                               0xF483BC
+
+#define mmTPC5_QM_CP_CURRENT_INST_LO_1                               0xF483C0
+
+#define mmTPC5_QM_CP_CURRENT_INST_LO_2                               0xF483C4
+
+#define mmTPC5_QM_CP_CURRENT_INST_LO_3                               0xF483C8
+
+#define mmTPC5_QM_CP_CURRENT_INST_LO_4                               0xF483CC
+
+#define mmTPC5_QM_CP_CURRENT_INST_HI_0                               0xF483D0
+
+#define mmTPC5_QM_CP_CURRENT_INST_HI_1                               0xF483D4
+
+#define mmTPC5_QM_CP_CURRENT_INST_HI_2                               0xF483D8
+
+#define mmTPC5_QM_CP_CURRENT_INST_HI_3                               0xF483DC
+
+#define mmTPC5_QM_CP_CURRENT_INST_HI_4                               0xF483E0
+
+#define mmTPC5_QM_CP_BARRIER_CFG_0                                   0xF483F4
+
+#define mmTPC5_QM_CP_BARRIER_CFG_1                                   0xF483F8
+
+#define mmTPC5_QM_CP_BARRIER_CFG_2                                   0xF483FC
+
+#define mmTPC5_QM_CP_BARRIER_CFG_3                                   0xF48400
+
+#define mmTPC5_QM_CP_BARRIER_CFG_4                                   0xF48404
+
+#define mmTPC5_QM_CP_DBG_0_0                                         0xF48408
+
+#define mmTPC5_QM_CP_DBG_0_1                                         0xF4840C
+
+#define mmTPC5_QM_CP_DBG_0_2                                         0xF48410
+
+#define mmTPC5_QM_CP_DBG_0_3                                         0xF48414
+
+#define mmTPC5_QM_CP_DBG_0_4                                         0xF48418
+
+#define mmTPC5_QM_CP_ARUSER_31_11_0                                  0xF4841C
+
+#define mmTPC5_QM_CP_ARUSER_31_11_1                                  0xF48420
+
+#define mmTPC5_QM_CP_ARUSER_31_11_2                                  0xF48424
+
+#define mmTPC5_QM_CP_ARUSER_31_11_3                                  0xF48428
+
+#define mmTPC5_QM_CP_ARUSER_31_11_4                                  0xF4842C
+
+#define mmTPC5_QM_CP_AWUSER_31_11_0                                  0xF48430
+
+#define mmTPC5_QM_CP_AWUSER_31_11_1                                  0xF48434
+
+#define mmTPC5_QM_CP_AWUSER_31_11_2                                  0xF48438
+
+#define mmTPC5_QM_CP_AWUSER_31_11_3                                  0xF4843C
+
+#define mmTPC5_QM_CP_AWUSER_31_11_4                                  0xF48440
+
+#define mmTPC5_QM_ARB_CFG_0                                          0xF48A00
+
+#define mmTPC5_QM_ARB_CHOISE_Q_PUSH                                  0xF48A04
+
+#define mmTPC5_QM_ARB_WRR_WEIGHT_0                                   0xF48A08
+
+#define mmTPC5_QM_ARB_WRR_WEIGHT_1                                   0xF48A0C
+
+#define mmTPC5_QM_ARB_WRR_WEIGHT_2                                   0xF48A10
+
+#define mmTPC5_QM_ARB_WRR_WEIGHT_3                                   0xF48A14
+
+#define mmTPC5_QM_ARB_CFG_1                                          0xF48A18
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_0                               0xF48A20
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_1                               0xF48A24
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_2                               0xF48A28
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_3                               0xF48A2C
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_4                               0xF48A30
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_5                               0xF48A34
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_6                               0xF48A38
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_7                               0xF48A3C
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_8                               0xF48A40
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_9                               0xF48A44
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_10                              0xF48A48
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_11                              0xF48A4C
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_12                              0xF48A50
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_13                              0xF48A54
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_14                              0xF48A58
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_15                              0xF48A5C
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_16                              0xF48A60
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_17                              0xF48A64
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_18                              0xF48A68
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_19                              0xF48A6C
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_20                              0xF48A70
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_21                              0xF48A74
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_22                              0xF48A78
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_23                              0xF48A7C
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_24                              0xF48A80
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_25                              0xF48A84
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_26                              0xF48A88
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_27                              0xF48A8C
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_28                              0xF48A90
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_29                              0xF48A94
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_30                              0xF48A98
+
+#define mmTPC5_QM_ARB_MST_AVAIL_CRED_31                              0xF48A9C
+
+#define mmTPC5_QM_ARB_MST_CRED_INC                                   0xF48AA0
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xF48AA4
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xF48AA8
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xF48AAC
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xF48AB0
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xF48AB4
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xF48AB8
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xF48ABC
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xF48AC0
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xF48AC4
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xF48AC8
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xF48ACC
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xF48AD0
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xF48AD4
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xF48AD8
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xF48ADC
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xF48AE0
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xF48AE4
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xF48AE8
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xF48AEC
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xF48AF0
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xF48AF4
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xF48AF8
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xF48AFC
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xF48B00
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xF48B04
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xF48B08
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xF48B0C
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xF48B10
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xF48B14
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xF48B18
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xF48B1C
+
+#define mmTPC5_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xF48B20
+
+#define mmTPC5_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xF48B28
+
+#define mmTPC5_QM_ARB_MST_SLAVE_EN                                   0xF48B2C
+
+#define mmTPC5_QM_ARB_MST_QUIET_PER                                  0xF48B34
+
+#define mmTPC5_QM_ARB_SLV_CHOISE_WDT                                 0xF48B38
+
+#define mmTPC5_QM_ARB_SLV_ID                                         0xF48B3C
+
+#define mmTPC5_QM_ARB_MSG_MAX_INFLIGHT                               0xF48B44
+
+#define mmTPC5_QM_ARB_MSG_AWUSER_31_11                               0xF48B48
+
+#define mmTPC5_QM_ARB_MSG_AWUSER_SEC_PROP                            0xF48B4C
+
+#define mmTPC5_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xF48B50
+
+#define mmTPC5_QM_ARB_BASE_LO                                        0xF48B54
+
+#define mmTPC5_QM_ARB_BASE_HI                                        0xF48B58
+
+#define mmTPC5_QM_ARB_STATE_STS                                      0xF48B80
+
+#define mmTPC5_QM_ARB_CHOISE_FULLNESS_STS                            0xF48B84
+
+#define mmTPC5_QM_ARB_MSG_STS                                        0xF48B88
+
+#define mmTPC5_QM_ARB_SLV_CHOISE_Q_HEAD                              0xF48B8C
+
+#define mmTPC5_QM_ARB_ERR_CAUSE                                      0xF48B9C
+
+#define mmTPC5_QM_ARB_ERR_MSG_EN                                     0xF48BA0
+
+#define mmTPC5_QM_ARB_ERR_STS_DRP                                    0xF48BA8
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_0                                 0xF48BB0
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_1                                 0xF48BB4
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_2                                 0xF48BB8
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_3                                 0xF48BBC
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_4                                 0xF48BC0
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_5                                 0xF48BC4
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_6                                 0xF48BC8
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_7                                 0xF48BCC
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_8                                 0xF48BD0
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_9                                 0xF48BD4
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_10                                0xF48BD8
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_11                                0xF48BDC
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_12                                0xF48BE0
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_13                                0xF48BE4
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_14                                0xF48BE8
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_15                                0xF48BEC
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_16                                0xF48BF0
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_17                                0xF48BF4
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_18                                0xF48BF8
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_19                                0xF48BFC
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_20                                0xF48C00
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_21                                0xF48C04
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_22                                0xF48C08
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_23                                0xF48C0C
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_24                                0xF48C10
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_25                                0xF48C14
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_26                                0xF48C18
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_27                                0xF48C1C
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_28                                0xF48C20
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_29                                0xF48C24
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_30                                0xF48C28
+
+#define mmTPC5_QM_ARB_MST_CRED_STS_31                                0xF48C2C
+
+#define mmTPC5_QM_CGM_CFG                                            0xF48C70
+
+#define mmTPC5_QM_CGM_STS                                            0xF48C74
+
+#define mmTPC5_QM_CGM_CFG1                                           0xF48C78
+
+#define mmTPC5_QM_LOCAL_RANGE_BASE                                   0xF48C80
+
+#define mmTPC5_QM_LOCAL_RANGE_SIZE                                   0xF48C84
+
+#define mmTPC5_QM_CSMR_STRICT_PRIO_CFG                               0xF48C90
+
+#define mmTPC5_QM_HBW_RD_RATE_LIM_CFG_1                              0xF48C94
+
+#define mmTPC5_QM_LBW_WR_RATE_LIM_CFG_0                              0xF48C98
+
+#define mmTPC5_QM_LBW_WR_RATE_LIM_CFG_1                              0xF48C9C
+
+#define mmTPC5_QM_HBW_RD_RATE_LIM_CFG_0                              0xF48CA0
+
+#define mmTPC5_QM_GLBL_AXCACHE                                       0xF48CA4
+
+#define mmTPC5_QM_IND_GW_APB_CFG                                     0xF48CB0
+
+#define mmTPC5_QM_IND_GW_APB_WDATA                                   0xF48CB4
+
+#define mmTPC5_QM_IND_GW_APB_RDATA                                   0xF48CB8
+
+#define mmTPC5_QM_IND_GW_APB_STATUS                                  0xF48CBC
+
+#define mmTPC5_QM_GLBL_ERR_ADDR_LO                                   0xF48CD0
+
+#define mmTPC5_QM_GLBL_ERR_ADDR_HI                                   0xF48CD4
+
+#define mmTPC5_QM_GLBL_ERR_WDATA                                     0xF48CD8
+
+#define mmTPC5_QM_GLBL_MEM_INIT_BUSY                                 0xF48D00
+
+#endif /* ASIC_REG_TPC5_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_cfg_regs.h
new file mode 100644
index 000000000000..eb251e72813f
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC6_CFG_REGS_H_
+#define ASIC_REG_TPC6_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC6_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xF86400
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xF86404
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xF86408
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xF8640C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xF86410
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xF86414
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xF86418
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xF8641C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xF86420
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xF86424
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xF86428
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xF8642C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xF86430
+
+#define mmTPC6_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xF86434
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xF86438
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xF8643C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xF86440
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xF86444
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xF86448
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xF8644C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xF86450
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xF86454
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xF86458
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xF8645C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xF86460
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xF86464
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xF86468
+
+#define mmTPC6_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xF8646C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xF86470
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xF86474
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xF86478
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xF8647C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xF86480
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xF86484
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xF86488
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xF8648C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xF86490
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xF86494
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xF86498
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xF8649C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xF864A0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xF864A4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xF864A8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xF864AC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xF864B0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xF864B4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xF864B8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xF864BC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xF864C0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xF864C4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xF864C8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xF864CC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xF864D0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xF864D4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xF864D8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xF864DC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xF864E0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xF864E4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xF864E8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xF864EC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xF864F0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xF864F4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xF864F8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xF864FC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xF86500
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xF86504
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xF86508
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xF8650C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xF86510
+
+#define mmTPC6_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xF86514
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xF86518
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xF8651C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xF86520
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xF86524
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xF86528
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xF8652C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xF86530
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xF86534
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xF86538
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xF8653C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xF86540
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xF86544
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xF86548
+
+#define mmTPC6_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xF8654C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xF86550
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xF86554
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xF86558
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xF8655C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xF86560
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xF86564
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xF86568
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xF8656C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xF86570
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xF86574
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xF86578
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xF8657C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xF86580
+
+#define mmTPC6_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xF86584
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xF86588
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xF8658C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xF86590
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xF86594
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xF86598
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xF8659C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xF865A0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xF865A4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xF865A8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xF865AC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xF865B0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xF865B4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xF865B8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xF865BC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xF865C0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xF865C4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xF865C8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xF865CC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xF865D0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xF865D4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xF865D8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xF865DC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xF865E0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xF865E4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xF865E8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xF865EC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xF865F0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xF865F4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xF865F8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xF865FC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xF86600
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xF86604
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xF86608
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xF8660C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xF86610
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xF86614
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xF86618
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xF8661C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xF86620
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xF86624
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xF86628
+
+#define mmTPC6_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xF8662C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xF86630
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xF86634
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xF86638
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xF8663C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xF86640
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xF86644
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xF86648
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xF8664C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xF86650
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xF86654
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xF86658
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xF8665C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xF86660
+
+#define mmTPC6_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xF86664
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xF86668
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xF8666C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xF86670
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xF86674
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xF86678
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xF8667C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xF86680
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xF86684
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xF86688
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xF8668C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xF86690
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xF86694
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xF86698
+
+#define mmTPC6_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xF8669C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xF866A0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xF866A4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xF866A8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xF866AC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xF866B0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xF866B4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xF866B8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xF866BC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xF866C0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xF866C4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xF866C8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xF866CC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xF866D0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xF866D4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xF866D8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xF866DC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xF866E0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xF866E4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xF866E8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xF866EC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xF866F0
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xF866F4
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xF866F8
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xF866FC
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xF86700
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xF86704
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xF86708
+
+#define mmTPC6_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xF8670C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xF86710
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xF86714
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xF86718
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xF8671C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xF86720
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xF86724
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xF86728
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xF8672C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xF86730
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xF86734
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xF86738
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xF8673C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xF86740
+
+#define mmTPC6_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xF86744
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xF86748
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xF8674C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xF86750
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xF86754
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xF86758
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xF8675C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xF86760
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xF86764
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xF86768
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xF8676C
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xF86770
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xF86774
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xF86778
+
+#define mmTPC6_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xF8677C
+
+#define mmTPC6_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xF86780
+
+#define mmTPC6_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xF86784
+
+#define mmTPC6_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xF86788
+
+#define mmTPC6_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xF8678C
+
+#define mmTPC6_CFG_KERNEL_TID_BASE_DIM_0                             0xF86790
+
+#define mmTPC6_CFG_KERNEL_TID_SIZE_DIM_0                             0xF86794
+
+#define mmTPC6_CFG_KERNEL_TID_BASE_DIM_1                             0xF86798
+
+#define mmTPC6_CFG_KERNEL_TID_SIZE_DIM_1                             0xF8679C
+
+#define mmTPC6_CFG_KERNEL_TID_BASE_DIM_2                             0xF867A0
+
+#define mmTPC6_CFG_KERNEL_TID_SIZE_DIM_2                             0xF867A4
+
+#define mmTPC6_CFG_KERNEL_TID_BASE_DIM_3                             0xF867A8
+
+#define mmTPC6_CFG_KERNEL_TID_SIZE_DIM_3                             0xF867AC
+
+#define mmTPC6_CFG_KERNEL_TID_BASE_DIM_4                             0xF867B0
+
+#define mmTPC6_CFG_KERNEL_TID_SIZE_DIM_4                             0xF867B4
+
+#define mmTPC6_CFG_KERNEL_KERNEL_CONFIG                              0xF867B8
+
+#define mmTPC6_CFG_KERNEL_KERNEL_ID                                  0xF867BC
+
+#define mmTPC6_CFG_KERNEL_SRF_0                                      0xF867C0
+
+#define mmTPC6_CFG_KERNEL_SRF_1                                      0xF867C4
+
+#define mmTPC6_CFG_KERNEL_SRF_2                                      0xF867C8
+
+#define mmTPC6_CFG_KERNEL_SRF_3                                      0xF867CC
+
+#define mmTPC6_CFG_KERNEL_SRF_4                                      0xF867D0
+
+#define mmTPC6_CFG_KERNEL_SRF_5                                      0xF867D4
+
+#define mmTPC6_CFG_KERNEL_SRF_6                                      0xF867D8
+
+#define mmTPC6_CFG_KERNEL_SRF_7                                      0xF867DC
+
+#define mmTPC6_CFG_KERNEL_SRF_8                                      0xF867E0
+
+#define mmTPC6_CFG_KERNEL_SRF_9                                      0xF867E4
+
+#define mmTPC6_CFG_KERNEL_SRF_10                                     0xF867E8
+
+#define mmTPC6_CFG_KERNEL_SRF_11                                     0xF867EC
+
+#define mmTPC6_CFG_KERNEL_SRF_12                                     0xF867F0
+
+#define mmTPC6_CFG_KERNEL_SRF_13                                     0xF867F4
+
+#define mmTPC6_CFG_KERNEL_SRF_14                                     0xF867F8
+
+#define mmTPC6_CFG_KERNEL_SRF_15                                     0xF867FC
+
+#define mmTPC6_CFG_KERNEL_SRF_16                                     0xF86800
+
+#define mmTPC6_CFG_KERNEL_SRF_17                                     0xF86804
+
+#define mmTPC6_CFG_KERNEL_SRF_18                                     0xF86808
+
+#define mmTPC6_CFG_KERNEL_SRF_19                                     0xF8680C
+
+#define mmTPC6_CFG_KERNEL_SRF_20                                     0xF86810
+
+#define mmTPC6_CFG_KERNEL_SRF_21                                     0xF86814
+
+#define mmTPC6_CFG_KERNEL_SRF_22                                     0xF86818
+
+#define mmTPC6_CFG_KERNEL_SRF_23                                     0xF8681C
+
+#define mmTPC6_CFG_KERNEL_SRF_24                                     0xF86820
+
+#define mmTPC6_CFG_KERNEL_SRF_25                                     0xF86824
+
+#define mmTPC6_CFG_KERNEL_SRF_26                                     0xF86828
+
+#define mmTPC6_CFG_KERNEL_SRF_27                                     0xF8682C
+
+#define mmTPC6_CFG_KERNEL_SRF_28                                     0xF86830
+
+#define mmTPC6_CFG_KERNEL_SRF_29                                     0xF86834
+
+#define mmTPC6_CFG_KERNEL_SRF_30                                     0xF86838
+
+#define mmTPC6_CFG_KERNEL_SRF_31                                     0xF8683C
+
+#define mmTPC6_CFG_ROUND_CSR                                         0xF868FC
+
+#define mmTPC6_CFG_PROT                                              0xF86900
+
+#define mmTPC6_CFG_SEMAPHORE                                         0xF86908
+
+#define mmTPC6_CFG_VFLAGS                                            0xF8690C
+
+#define mmTPC6_CFG_SFLAGS                                            0xF86910
+
+#define mmTPC6_CFG_LFSR_POLYNOM                                      0xF86918
+
+#define mmTPC6_CFG_STATUS                                            0xF8691C
+
+#define mmTPC6_CFG_CFG_BASE_ADDRESS_HIGH                             0xF86920
+
+#define mmTPC6_CFG_CFG_SUBTRACT_VALUE                                0xF86924
+
+#define mmTPC6_CFG_SM_BASE_ADDRESS_HIGH                              0xF8692C
+
+#define mmTPC6_CFG_TPC_CMD                                           0xF86930
+
+#define mmTPC6_CFG_TPC_EXECUTE                                       0xF86938
+
+#define mmTPC6_CFG_TPC_STALL                                         0xF8693C
+
+#define mmTPC6_CFG_ICACHE_BASE_ADDERESS_LOW                          0xF86940
+
+#define mmTPC6_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xF86944
+
+#define mmTPC6_CFG_RD_RATE_LIMIT                                     0xF86948
+
+#define mmTPC6_CFG_WR_RATE_LIMIT                                     0xF86950
+
+#define mmTPC6_CFG_MSS_CONFIG                                        0xF86954
+
+#define mmTPC6_CFG_TPC_INTR_CAUSE                                    0xF86958
+
+#define mmTPC6_CFG_TPC_INTR_MASK                                     0xF8695C
+
+#define mmTPC6_CFG_WQ_CREDITS                                        0xF86960
+
+#define mmTPC6_CFG_ARUSER_LO                                         0xF86964
+
+#define mmTPC6_CFG_ARUSER_HI                                         0xF86968
+
+#define mmTPC6_CFG_AWUSER_LO                                         0xF8696C
+
+#define mmTPC6_CFG_AWUSER_HI                                         0xF86970
+
+#define mmTPC6_CFG_OPCODE_EXEC                                       0xF86974
+
+#define mmTPC6_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xF86978
+
+#define mmTPC6_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xF8697C
+
+#define mmTPC6_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xF86980
+
+#define mmTPC6_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xF86984
+
+#define mmTPC6_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xF86988
+
+#define mmTPC6_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xF8698C
+
+#define mmTPC6_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xF86990
+
+#define mmTPC6_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xF86994
+
+#define mmTPC6_CFG_TSB_CFG_MAX_SIZE                                  0xF86998
+
+#define mmTPC6_CFG_TSB_CFG                                           0xF8699C
+
+#define mmTPC6_CFG_DBGMEM_ADD                                        0xF869A0
+
+#define mmTPC6_CFG_DBGMEM_DATA_WR                                    0xF869A4
+
+#define mmTPC6_CFG_DBGMEM_DATA_RD                                    0xF869A8
+
+#define mmTPC6_CFG_DBGMEM_CTRL                                       0xF869AC
+
+#define mmTPC6_CFG_DBGMEM_RC                                         0xF869B0
+
+#define mmTPC6_CFG_TSB_INFLIGHT_CNTR                                 0xF869B4
+
+#define mmTPC6_CFG_WQ_INFLIGHT_CNTR                                  0xF869B8
+
+#define mmTPC6_CFG_WQ_LBW_TOTAL_CNTR                                 0xF869BC
+
+#define mmTPC6_CFG_WQ_HBW_TOTAL_CNTR                                 0xF869C0
+
+#define mmTPC6_CFG_IRQ_OCCOUPY_CNTR                                  0xF869C4
+
+#define mmTPC6_CFG_FUNC_MBIST_CNTRL                                  0xF869D0
+
+#define mmTPC6_CFG_FUNC_MBIST_PAT                                    0xF869D4
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_0                                  0xF869D8
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_1                                  0xF869DC
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_2                                  0xF869E0
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_3                                  0xF869E4
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_4                                  0xF869E8
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_5                                  0xF869EC
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_6                                  0xF869F0
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_7                                  0xF869F4
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_8                                  0xF869F8
+
+#define mmTPC6_CFG_FUNC_MBIST_MEM_9                                  0xF869FC
+
+#define mmTPC6_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xF86A00
+
+#define mmTPC6_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xF86A04
+
+#define mmTPC6_CFG_QM_TENSOR_0_PADDING_VALUE                         0xF86A08
+
+#define mmTPC6_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xF86A0C
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xF86A10
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xF86A14
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xF86A18
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xF86A1C
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xF86A20
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xF86A24
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xF86A28
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xF86A2C
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xF86A30
+
+#define mmTPC6_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xF86A34
+
+#define mmTPC6_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xF86A38
+
+#define mmTPC6_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xF86A3C
+
+#define mmTPC6_CFG_QM_TENSOR_1_PADDING_VALUE                         0xF86A40
+
+#define mmTPC6_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xF86A44
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xF86A48
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xF86A4C
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xF86A50
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xF86A54
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xF86A58
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xF86A5C
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xF86A60
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xF86A64
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xF86A68
+
+#define mmTPC6_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xF86A6C
+
+#define mmTPC6_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xF86A70
+
+#define mmTPC6_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xF86A74
+
+#define mmTPC6_CFG_QM_TENSOR_2_PADDING_VALUE                         0xF86A78
+
+#define mmTPC6_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xF86A7C
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xF86A80
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xF86A84
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xF86A88
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xF86A8C
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xF86A90
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xF86A94
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xF86A98
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xF86A9C
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xF86AA0
+
+#define mmTPC6_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xF86AA4
+
+#define mmTPC6_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xF86AA8
+
+#define mmTPC6_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xF86AAC
+
+#define mmTPC6_CFG_QM_TENSOR_3_PADDING_VALUE                         0xF86AB0
+
+#define mmTPC6_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xF86AB4
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xF86AB8
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xF86ABC
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xF86AC0
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xF86AC4
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xF86AC8
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xF86ACC
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xF86AD0
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xF86AD4
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xF86AD8
+
+#define mmTPC6_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xF86ADC
+
+#define mmTPC6_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xF86AE0
+
+#define mmTPC6_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xF86AE4
+
+#define mmTPC6_CFG_QM_TENSOR_4_PADDING_VALUE                         0xF86AE8
+
+#define mmTPC6_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xF86AEC
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xF86AF0
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xF86AF4
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xF86AF8
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xF86AFC
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xF86B00
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xF86B04
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xF86B08
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xF86B0C
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xF86B10
+
+#define mmTPC6_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xF86B14
+
+#define mmTPC6_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xF86B18
+
+#define mmTPC6_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xF86B1C
+
+#define mmTPC6_CFG_QM_TENSOR_5_PADDING_VALUE                         0xF86B20
+
+#define mmTPC6_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xF86B24
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xF86B28
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xF86B2C
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xF86B30
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xF86B34
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xF86B38
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xF86B3C
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xF86B40
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xF86B44
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xF86B48
+
+#define mmTPC6_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xF86B4C
+
+#define mmTPC6_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xF86B50
+
+#define mmTPC6_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xF86B54
+
+#define mmTPC6_CFG_QM_TENSOR_6_PADDING_VALUE                         0xF86B58
+
+#define mmTPC6_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xF86B5C
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xF86B60
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xF86B64
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xF86B68
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xF86B6C
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xF86B70
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xF86B74
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xF86B78
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xF86B7C
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xF86B80
+
+#define mmTPC6_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xF86B84
+
+#define mmTPC6_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xF86B88
+
+#define mmTPC6_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xF86B8C
+
+#define mmTPC6_CFG_QM_TENSOR_7_PADDING_VALUE                         0xF86B90
+
+#define mmTPC6_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xF86B94
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xF86B98
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xF86B9C
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xF86BA0
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xF86BA4
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xF86BA8
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xF86BAC
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xF86BB0
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xF86BB4
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xF86BB8
+
+#define mmTPC6_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xF86BBC
+
+#define mmTPC6_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xF86BC0
+
+#define mmTPC6_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xF86BC4
+
+#define mmTPC6_CFG_QM_TENSOR_8_PADDING_VALUE                         0xF86BC8
+
+#define mmTPC6_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xF86BCC
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xF86BD0
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xF86BD4
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xF86BD8
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xF86BDC
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xF86BE0
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xF86BE4
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xF86BE8
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xF86BEC
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xF86BF0
+
+#define mmTPC6_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xF86BF4
+
+#define mmTPC6_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xF86BF8
+
+#define mmTPC6_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xF86BFC
+
+#define mmTPC6_CFG_QM_TENSOR_9_PADDING_VALUE                         0xF86C00
+
+#define mmTPC6_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xF86C04
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xF86C08
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xF86C0C
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xF86C10
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xF86C14
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xF86C18
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xF86C1C
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xF86C20
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xF86C24
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xF86C28
+
+#define mmTPC6_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xF86C2C
+
+#define mmTPC6_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xF86C30
+
+#define mmTPC6_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xF86C34
+
+#define mmTPC6_CFG_QM_TENSOR_10_PADDING_VALUE                        0xF86C38
+
+#define mmTPC6_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xF86C3C
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xF86C40
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xF86C44
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xF86C48
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xF86C4C
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xF86C50
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xF86C54
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xF86C58
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xF86C5C
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xF86C60
+
+#define mmTPC6_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xF86C64
+
+#define mmTPC6_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xF86C68
+
+#define mmTPC6_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xF86C6C
+
+#define mmTPC6_CFG_QM_TENSOR_11_PADDING_VALUE                        0xF86C70
+
+#define mmTPC6_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xF86C74
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xF86C78
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xF86C7C
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xF86C80
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xF86C84
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xF86C88
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xF86C8C
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xF86C90
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xF86C94
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xF86C98
+
+#define mmTPC6_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xF86C9C
+
+#define mmTPC6_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xF86CA0
+
+#define mmTPC6_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xF86CA4
+
+#define mmTPC6_CFG_QM_TENSOR_12_PADDING_VALUE                        0xF86CA8
+
+#define mmTPC6_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xF86CAC
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xF86CB0
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xF86CB4
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xF86CB8
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xF86CBC
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xF86CC0
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xF86CC4
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xF86CC8
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xF86CCC
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xF86CD0
+
+#define mmTPC6_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xF86CD4
+
+#define mmTPC6_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xF86CD8
+
+#define mmTPC6_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xF86CDC
+
+#define mmTPC6_CFG_QM_TENSOR_13_PADDING_VALUE                        0xF86CE0
+
+#define mmTPC6_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xF86CE4
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xF86CE8
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xF86CEC
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xF86CF0
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xF86CF4
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xF86CF8
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xF86CFC
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xF86D00
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xF86D04
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xF86D08
+
+#define mmTPC6_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xF86D0C
+
+#define mmTPC6_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xF86D10
+
+#define mmTPC6_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xF86D14
+
+#define mmTPC6_CFG_QM_TENSOR_14_PADDING_VALUE                        0xF86D18
+
+#define mmTPC6_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xF86D1C
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xF86D20
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xF86D24
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xF86D28
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xF86D2C
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xF86D30
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xF86D34
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xF86D38
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xF86D3C
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xF86D40
+
+#define mmTPC6_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xF86D44
+
+#define mmTPC6_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xF86D48
+
+#define mmTPC6_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xF86D4C
+
+#define mmTPC6_CFG_QM_TENSOR_15_PADDING_VALUE                        0xF86D50
+
+#define mmTPC6_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xF86D54
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xF86D58
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xF86D5C
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xF86D60
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xF86D64
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xF86D68
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xF86D6C
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xF86D70
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xF86D74
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xF86D78
+
+#define mmTPC6_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xF86D7C
+
+#define mmTPC6_CFG_QM_SYNC_OBJECT_MESSAGE                            0xF86D80
+
+#define mmTPC6_CFG_QM_SYNC_OBJECT_ADDR                               0xF86D84
+
+#define mmTPC6_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xF86D88
+
+#define mmTPC6_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xF86D8C
+
+#define mmTPC6_CFG_QM_TID_BASE_DIM_0                                 0xF86D90
+
+#define mmTPC6_CFG_QM_TID_SIZE_DIM_0                                 0xF86D94
+
+#define mmTPC6_CFG_QM_TID_BASE_DIM_1                                 0xF86D98
+
+#define mmTPC6_CFG_QM_TID_SIZE_DIM_1                                 0xF86D9C
+
+#define mmTPC6_CFG_QM_TID_BASE_DIM_2                                 0xF86DA0
+
+#define mmTPC6_CFG_QM_TID_SIZE_DIM_2                                 0xF86DA4
+
+#define mmTPC6_CFG_QM_TID_BASE_DIM_3                                 0xF86DA8
+
+#define mmTPC6_CFG_QM_TID_SIZE_DIM_3                                 0xF86DAC
+
+#define mmTPC6_CFG_QM_TID_BASE_DIM_4                                 0xF86DB0
+
+#define mmTPC6_CFG_QM_TID_SIZE_DIM_4                                 0xF86DB4
+
+#define mmTPC6_CFG_QM_KERNEL_CONFIG                                  0xF86DB8
+
+#define mmTPC6_CFG_QM_KERNEL_ID                                      0xF86DBC
+
+#define mmTPC6_CFG_QM_SRF_0                                          0xF86DC0
+
+#define mmTPC6_CFG_QM_SRF_1                                          0xF86DC4
+
+#define mmTPC6_CFG_QM_SRF_2                                          0xF86DC8
+
+#define mmTPC6_CFG_QM_SRF_3                                          0xF86DCC
+
+#define mmTPC6_CFG_QM_SRF_4                                          0xF86DD0
+
+#define mmTPC6_CFG_QM_SRF_5                                          0xF86DD4
+
+#define mmTPC6_CFG_QM_SRF_6                                          0xF86DD8
+
+#define mmTPC6_CFG_QM_SRF_7                                          0xF86DDC
+
+#define mmTPC6_CFG_QM_SRF_8                                          0xF86DE0
+
+#define mmTPC6_CFG_QM_SRF_9                                          0xF86DE4
+
+#define mmTPC6_CFG_QM_SRF_10                                         0xF86DE8
+
+#define mmTPC6_CFG_QM_SRF_11                                         0xF86DEC
+
+#define mmTPC6_CFG_QM_SRF_12                                         0xF86DF0
+
+#define mmTPC6_CFG_QM_SRF_13                                         0xF86DF4
+
+#define mmTPC6_CFG_QM_SRF_14                                         0xF86DF8
+
+#define mmTPC6_CFG_QM_SRF_15                                         0xF86DFC
+
+#define mmTPC6_CFG_QM_SRF_16                                         0xF86E00
+
+#define mmTPC6_CFG_QM_SRF_17                                         0xF86E04
+
+#define mmTPC6_CFG_QM_SRF_18                                         0xF86E08
+
+#define mmTPC6_CFG_QM_SRF_19                                         0xF86E0C
+
+#define mmTPC6_CFG_QM_SRF_20                                         0xF86E10
+
+#define mmTPC6_CFG_QM_SRF_21                                         0xF86E14
+
+#define mmTPC6_CFG_QM_SRF_22                                         0xF86E18
+
+#define mmTPC6_CFG_QM_SRF_23                                         0xF86E1C
+
+#define mmTPC6_CFG_QM_SRF_24                                         0xF86E20
+
+#define mmTPC6_CFG_QM_SRF_25                                         0xF86E24
+
+#define mmTPC6_CFG_QM_SRF_26                                         0xF86E28
+
+#define mmTPC6_CFG_QM_SRF_27                                         0xF86E2C
+
+#define mmTPC6_CFG_QM_SRF_28                                         0xF86E30
+
+#define mmTPC6_CFG_QM_SRF_29                                         0xF86E34
+
+#define mmTPC6_CFG_QM_SRF_30                                         0xF86E38
+
+#define mmTPC6_CFG_QM_SRF_31                                         0xF86E3C
+
+#endif /* ASIC_REG_TPC6_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_qm_regs.h
new file mode 100644
index 000000000000..e35ef7fd8b1c
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc6_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC6_QM_REGS_H_
+#define ASIC_REG_TPC6_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC6_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC6_QM_GLBL_CFG0                                          0xF88000
+
+#define mmTPC6_QM_GLBL_CFG1                                          0xF88004
+
+#define mmTPC6_QM_GLBL_PROT                                          0xF88008
+
+#define mmTPC6_QM_GLBL_ERR_CFG                                       0xF8800C
+
+#define mmTPC6_QM_GLBL_SECURE_PROPS_0                                0xF88010
+
+#define mmTPC6_QM_GLBL_SECURE_PROPS_1                                0xF88014
+
+#define mmTPC6_QM_GLBL_SECURE_PROPS_2                                0xF88018
+
+#define mmTPC6_QM_GLBL_SECURE_PROPS_3                                0xF8801C
+
+#define mmTPC6_QM_GLBL_SECURE_PROPS_4                                0xF88020
+
+#define mmTPC6_QM_GLBL_NON_SECURE_PROPS_0                            0xF88024
+
+#define mmTPC6_QM_GLBL_NON_SECURE_PROPS_1                            0xF88028
+
+#define mmTPC6_QM_GLBL_NON_SECURE_PROPS_2                            0xF8802C
+
+#define mmTPC6_QM_GLBL_NON_SECURE_PROPS_3                            0xF88030
+
+#define mmTPC6_QM_GLBL_NON_SECURE_PROPS_4                            0xF88034
+
+#define mmTPC6_QM_GLBL_STS0                                          0xF88038
+
+#define mmTPC6_QM_GLBL_STS1_0                                        0xF88040
+
+#define mmTPC6_QM_GLBL_STS1_1                                        0xF88044
+
+#define mmTPC6_QM_GLBL_STS1_2                                        0xF88048
+
+#define mmTPC6_QM_GLBL_STS1_3                                        0xF8804C
+
+#define mmTPC6_QM_GLBL_STS1_4                                        0xF88050
+
+#define mmTPC6_QM_GLBL_MSG_EN_0                                      0xF88054
+
+#define mmTPC6_QM_GLBL_MSG_EN_1                                      0xF88058
+
+#define mmTPC6_QM_GLBL_MSG_EN_2                                      0xF8805C
+
+#define mmTPC6_QM_GLBL_MSG_EN_3                                      0xF88060
+
+#define mmTPC6_QM_GLBL_MSG_EN_4                                      0xF88068
+
+#define mmTPC6_QM_PQ_BASE_LO_0                                       0xF88070
+
+#define mmTPC6_QM_PQ_BASE_LO_1                                       0xF88074
+
+#define mmTPC6_QM_PQ_BASE_LO_2                                       0xF88078
+
+#define mmTPC6_QM_PQ_BASE_LO_3                                       0xF8807C
+
+#define mmTPC6_QM_PQ_BASE_HI_0                                       0xF88080
+
+#define mmTPC6_QM_PQ_BASE_HI_1                                       0xF88084
+
+#define mmTPC6_QM_PQ_BASE_HI_2                                       0xF88088
+
+#define mmTPC6_QM_PQ_BASE_HI_3                                       0xF8808C
+
+#define mmTPC6_QM_PQ_SIZE_0                                          0xF88090
+
+#define mmTPC6_QM_PQ_SIZE_1                                          0xF88094
+
+#define mmTPC6_QM_PQ_SIZE_2                                          0xF88098
+
+#define mmTPC6_QM_PQ_SIZE_3                                          0xF8809C
+
+#define mmTPC6_QM_PQ_PI_0                                            0xF880A0
+
+#define mmTPC6_QM_PQ_PI_1                                            0xF880A4
+
+#define mmTPC6_QM_PQ_PI_2                                            0xF880A8
+
+#define mmTPC6_QM_PQ_PI_3                                            0xF880AC
+
+#define mmTPC6_QM_PQ_CI_0                                            0xF880B0
+
+#define mmTPC6_QM_PQ_CI_1                                            0xF880B4
+
+#define mmTPC6_QM_PQ_CI_2                                            0xF880B8
+
+#define mmTPC6_QM_PQ_CI_3                                            0xF880BC
+
+#define mmTPC6_QM_PQ_CFG0_0                                          0xF880C0
+
+#define mmTPC6_QM_PQ_CFG0_1                                          0xF880C4
+
+#define mmTPC6_QM_PQ_CFG0_2                                          0xF880C8
+
+#define mmTPC6_QM_PQ_CFG0_3                                          0xF880CC
+
+#define mmTPC6_QM_PQ_CFG1_0                                          0xF880D0
+
+#define mmTPC6_QM_PQ_CFG1_1                                          0xF880D4
+
+#define mmTPC6_QM_PQ_CFG1_2                                          0xF880D8
+
+#define mmTPC6_QM_PQ_CFG1_3                                          0xF880DC
+
+#define mmTPC6_QM_PQ_ARUSER_31_11_0                                  0xF880E0
+
+#define mmTPC6_QM_PQ_ARUSER_31_11_1                                  0xF880E4
+
+#define mmTPC6_QM_PQ_ARUSER_31_11_2                                  0xF880E8
+
+#define mmTPC6_QM_PQ_ARUSER_31_11_3                                  0xF880EC
+
+#define mmTPC6_QM_PQ_STS0_0                                          0xF880F0
+
+#define mmTPC6_QM_PQ_STS0_1                                          0xF880F4
+
+#define mmTPC6_QM_PQ_STS0_2                                          0xF880F8
+
+#define mmTPC6_QM_PQ_STS0_3                                          0xF880FC
+
+#define mmTPC6_QM_PQ_STS1_0                                          0xF88100
+
+#define mmTPC6_QM_PQ_STS1_1                                          0xF88104
+
+#define mmTPC6_QM_PQ_STS1_2                                          0xF88108
+
+#define mmTPC6_QM_PQ_STS1_3                                          0xF8810C
+
+#define mmTPC6_QM_CQ_CFG0_0                                          0xF88110
+
+#define mmTPC6_QM_CQ_CFG0_1                                          0xF88114
+
+#define mmTPC6_QM_CQ_CFG0_2                                          0xF88118
+
+#define mmTPC6_QM_CQ_CFG0_3                                          0xF8811C
+
+#define mmTPC6_QM_CQ_CFG0_4                                          0xF88120
+
+#define mmTPC6_QM_CQ_CFG1_0                                          0xF88124
+
+#define mmTPC6_QM_CQ_CFG1_1                                          0xF88128
+
+#define mmTPC6_QM_CQ_CFG1_2                                          0xF8812C
+
+#define mmTPC6_QM_CQ_CFG1_3                                          0xF88130
+
+#define mmTPC6_QM_CQ_CFG1_4                                          0xF88134
+
+#define mmTPC6_QM_CQ_ARUSER_31_11_0                                  0xF88138
+
+#define mmTPC6_QM_CQ_ARUSER_31_11_1                                  0xF8813C
+
+#define mmTPC6_QM_CQ_ARUSER_31_11_2                                  0xF88140
+
+#define mmTPC6_QM_CQ_ARUSER_31_11_3                                  0xF88144
+
+#define mmTPC6_QM_CQ_ARUSER_31_11_4                                  0xF88148
+
+#define mmTPC6_QM_CQ_STS0_0                                          0xF8814C
+
+#define mmTPC6_QM_CQ_STS0_1                                          0xF88150
+
+#define mmTPC6_QM_CQ_STS0_2                                          0xF88154
+
+#define mmTPC6_QM_CQ_STS0_3                                          0xF88158
+
+#define mmTPC6_QM_CQ_STS0_4                                          0xF8815C
+
+#define mmTPC6_QM_CQ_STS1_0                                          0xF88160
+
+#define mmTPC6_QM_CQ_STS1_1                                          0xF88164
+
+#define mmTPC6_QM_CQ_STS1_2                                          0xF88168
+
+#define mmTPC6_QM_CQ_STS1_3                                          0xF8816C
+
+#define mmTPC6_QM_CQ_STS1_4                                          0xF88170
+
+#define mmTPC6_QM_CQ_PTR_LO_0                                        0xF88174
+
+#define mmTPC6_QM_CQ_PTR_HI_0                                        0xF88178
+
+#define mmTPC6_QM_CQ_TSIZE_0                                         0xF8817C
+
+#define mmTPC6_QM_CQ_CTL_0                                           0xF88180
+
+#define mmTPC6_QM_CQ_PTR_LO_1                                        0xF88184
+
+#define mmTPC6_QM_CQ_PTR_HI_1                                        0xF88188
+
+#define mmTPC6_QM_CQ_TSIZE_1                                         0xF8818C
+
+#define mmTPC6_QM_CQ_CTL_1                                           0xF88190
+
+#define mmTPC6_QM_CQ_PTR_LO_2                                        0xF88194
+
+#define mmTPC6_QM_CQ_PTR_HI_2                                        0xF88198
+
+#define mmTPC6_QM_CQ_TSIZE_2                                         0xF8819C
+
+#define mmTPC6_QM_CQ_CTL_2                                           0xF881A0
+
+#define mmTPC6_QM_CQ_PTR_LO_3                                        0xF881A4
+
+#define mmTPC6_QM_CQ_PTR_HI_3                                        0xF881A8
+
+#define mmTPC6_QM_CQ_TSIZE_3                                         0xF881AC
+
+#define mmTPC6_QM_CQ_CTL_3                                           0xF881B0
+
+#define mmTPC6_QM_CQ_PTR_LO_4                                        0xF881B4
+
+#define mmTPC6_QM_CQ_PTR_HI_4                                        0xF881B8
+
+#define mmTPC6_QM_CQ_TSIZE_4                                         0xF881BC
+
+#define mmTPC6_QM_CQ_CTL_4                                           0xF881C0
+
+#define mmTPC6_QM_CQ_PTR_LO_STS_0                                    0xF881C4
+
+#define mmTPC6_QM_CQ_PTR_LO_STS_1                                    0xF881C8
+
+#define mmTPC6_QM_CQ_PTR_LO_STS_2                                    0xF881CC
+
+#define mmTPC6_QM_CQ_PTR_LO_STS_3                                    0xF881D0
+
+#define mmTPC6_QM_CQ_PTR_LO_STS_4                                    0xF881D4
+
+#define mmTPC6_QM_CQ_PTR_HI_STS_0                                    0xF881D8
+
+#define mmTPC6_QM_CQ_PTR_HI_STS_1                                    0xF881DC
+
+#define mmTPC6_QM_CQ_PTR_HI_STS_2                                    0xF881E0
+
+#define mmTPC6_QM_CQ_PTR_HI_STS_3                                    0xF881E4
+
+#define mmTPC6_QM_CQ_PTR_HI_STS_4                                    0xF881E8
+
+#define mmTPC6_QM_CQ_TSIZE_STS_0                                     0xF881EC
+
+#define mmTPC6_QM_CQ_TSIZE_STS_1                                     0xF881F0
+
+#define mmTPC6_QM_CQ_TSIZE_STS_2                                     0xF881F4
+
+#define mmTPC6_QM_CQ_TSIZE_STS_3                                     0xF881F8
+
+#define mmTPC6_QM_CQ_TSIZE_STS_4                                     0xF881FC
+
+#define mmTPC6_QM_CQ_CTL_STS_0                                       0xF88200
+
+#define mmTPC6_QM_CQ_CTL_STS_1                                       0xF88204
+
+#define mmTPC6_QM_CQ_CTL_STS_2                                       0xF88208
+
+#define mmTPC6_QM_CQ_CTL_STS_3                                       0xF8820C
+
+#define mmTPC6_QM_CQ_CTL_STS_4                                       0xF88210
+
+#define mmTPC6_QM_CQ_IFIFO_CNT_0                                     0xF88214
+
+#define mmTPC6_QM_CQ_IFIFO_CNT_1                                     0xF88218
+
+#define mmTPC6_QM_CQ_IFIFO_CNT_2                                     0xF8821C
+
+#define mmTPC6_QM_CQ_IFIFO_CNT_3                                     0xF88220
+
+#define mmTPC6_QM_CQ_IFIFO_CNT_4                                     0xF88224
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_0                             0xF88228
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_1                             0xF8822C
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_2                             0xF88230
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_3                             0xF88234
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_LO_4                             0xF88238
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_0                             0xF8823C
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_1                             0xF88240
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_2                             0xF88244
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_3                             0xF88248
+
+#define mmTPC6_QM_CP_MSG_BASE0_ADDR_HI_4                             0xF8824C
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_0                             0xF88250
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_1                             0xF88254
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_2                             0xF88258
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_3                             0xF8825C
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_LO_4                             0xF88260
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_0                             0xF88264
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_1                             0xF88268
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_2                             0xF8826C
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_3                             0xF88270
+
+#define mmTPC6_QM_CP_MSG_BASE1_ADDR_HI_4                             0xF88274
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_0                             0xF88278
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_1                             0xF8827C
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_2                             0xF88280
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_3                             0xF88284
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_LO_4                             0xF88288
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_0                             0xF8828C
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_1                             0xF88290
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_2                             0xF88294
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_3                             0xF88298
+
+#define mmTPC6_QM_CP_MSG_BASE2_ADDR_HI_4                             0xF8829C
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_0                             0xF882A0
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_1                             0xF882A4
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_2                             0xF882A8
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_3                             0xF882AC
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_LO_4                             0xF882B0
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_0                             0xF882B4
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_1                             0xF882B8
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_2                             0xF882BC
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_3                             0xF882C0
+
+#define mmTPC6_QM_CP_MSG_BASE3_ADDR_HI_4                             0xF882C4
+
+#define mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_0                             0xF882C8
+
+#define mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_1                             0xF882CC
+
+#define mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_2                             0xF882D0
+
+#define mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_3                             0xF882D4
+
+#define mmTPC6_QM_CP_LDMA_TSIZE_OFFSET_4                             0xF882D8
+
+#define mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xF882E0
+
+#define mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xF882E4
+
+#define mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xF882E8
+
+#define mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xF882EC
+
+#define mmTPC6_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xF882F0
+
+#define mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xF882F4
+
+#define mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xF882F8
+
+#define mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xF882FC
+
+#define mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xF88300
+
+#define mmTPC6_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xF88304
+
+#define mmTPC6_QM_CP_FENCE0_RDATA_0                                  0xF88308
+
+#define mmTPC6_QM_CP_FENCE0_RDATA_1                                  0xF8830C
+
+#define mmTPC6_QM_CP_FENCE0_RDATA_2                                  0xF88310
+
+#define mmTPC6_QM_CP_FENCE0_RDATA_3                                  0xF88314
+
+#define mmTPC6_QM_CP_FENCE0_RDATA_4                                  0xF88318
+
+#define mmTPC6_QM_CP_FENCE1_RDATA_0                                  0xF8831C
+
+#define mmTPC6_QM_CP_FENCE1_RDATA_1                                  0xF88320
+
+#define mmTPC6_QM_CP_FENCE1_RDATA_2                                  0xF88324
+
+#define mmTPC6_QM_CP_FENCE1_RDATA_3                                  0xF88328
+
+#define mmTPC6_QM_CP_FENCE1_RDATA_4                                  0xF8832C
+
+#define mmTPC6_QM_CP_FENCE2_RDATA_0                                  0xF88330
+
+#define mmTPC6_QM_CP_FENCE2_RDATA_1                                  0xF88334
+
+#define mmTPC6_QM_CP_FENCE2_RDATA_2                                  0xF88338
+
+#define mmTPC6_QM_CP_FENCE2_RDATA_3                                  0xF8833C
+
+#define mmTPC6_QM_CP_FENCE2_RDATA_4                                  0xF88340
+
+#define mmTPC6_QM_CP_FENCE3_RDATA_0                                  0xF88344
+
+#define mmTPC6_QM_CP_FENCE3_RDATA_1                                  0xF88348
+
+#define mmTPC6_QM_CP_FENCE3_RDATA_2                                  0xF8834C
+
+#define mmTPC6_QM_CP_FENCE3_RDATA_3                                  0xF88350
+
+#define mmTPC6_QM_CP_FENCE3_RDATA_4                                  0xF88354
+
+#define mmTPC6_QM_CP_FENCE0_CNT_0                                    0xF88358
+
+#define mmTPC6_QM_CP_FENCE0_CNT_1                                    0xF8835C
+
+#define mmTPC6_QM_CP_FENCE0_CNT_2                                    0xF88360
+
+#define mmTPC6_QM_CP_FENCE0_CNT_3                                    0xF88364
+
+#define mmTPC6_QM_CP_FENCE0_CNT_4                                    0xF88368
+
+#define mmTPC6_QM_CP_FENCE1_CNT_0                                    0xF8836C
+
+#define mmTPC6_QM_CP_FENCE1_CNT_1                                    0xF88370
+
+#define mmTPC6_QM_CP_FENCE1_CNT_2                                    0xF88374
+
+#define mmTPC6_QM_CP_FENCE1_CNT_3                                    0xF88378
+
+#define mmTPC6_QM_CP_FENCE1_CNT_4                                    0xF8837C
+
+#define mmTPC6_QM_CP_FENCE2_CNT_0                                    0xF88380
+
+#define mmTPC6_QM_CP_FENCE2_CNT_1                                    0xF88384
+
+#define mmTPC6_QM_CP_FENCE2_CNT_2                                    0xF88388
+
+#define mmTPC6_QM_CP_FENCE2_CNT_3                                    0xF8838C
+
+#define mmTPC6_QM_CP_FENCE2_CNT_4                                    0xF88390
+
+#define mmTPC6_QM_CP_FENCE3_CNT_0                                    0xF88394
+
+#define mmTPC6_QM_CP_FENCE3_CNT_1                                    0xF88398
+
+#define mmTPC6_QM_CP_FENCE3_CNT_2                                    0xF8839C
+
+#define mmTPC6_QM_CP_FENCE3_CNT_3                                    0xF883A0
+
+#define mmTPC6_QM_CP_FENCE3_CNT_4                                    0xF883A4
+
+#define mmTPC6_QM_CP_STS_0                                           0xF883A8
+
+#define mmTPC6_QM_CP_STS_1                                           0xF883AC
+
+#define mmTPC6_QM_CP_STS_2                                           0xF883B0
+
+#define mmTPC6_QM_CP_STS_3                                           0xF883B4
+
+#define mmTPC6_QM_CP_STS_4                                           0xF883B8
+
+#define mmTPC6_QM_CP_CURRENT_INST_LO_0                               0xF883BC
+
+#define mmTPC6_QM_CP_CURRENT_INST_LO_1                               0xF883C0
+
+#define mmTPC6_QM_CP_CURRENT_INST_LO_2                               0xF883C4
+
+#define mmTPC6_QM_CP_CURRENT_INST_LO_3                               0xF883C8
+
+#define mmTPC6_QM_CP_CURRENT_INST_LO_4                               0xF883CC
+
+#define mmTPC6_QM_CP_CURRENT_INST_HI_0                               0xF883D0
+
+#define mmTPC6_QM_CP_CURRENT_INST_HI_1                               0xF883D4
+
+#define mmTPC6_QM_CP_CURRENT_INST_HI_2                               0xF883D8
+
+#define mmTPC6_QM_CP_CURRENT_INST_HI_3                               0xF883DC
+
+#define mmTPC6_QM_CP_CURRENT_INST_HI_4                               0xF883E0
+
+#define mmTPC6_QM_CP_BARRIER_CFG_0                                   0xF883F4
+
+#define mmTPC6_QM_CP_BARRIER_CFG_1                                   0xF883F8
+
+#define mmTPC6_QM_CP_BARRIER_CFG_2                                   0xF883FC
+
+#define mmTPC6_QM_CP_BARRIER_CFG_3                                   0xF88400
+
+#define mmTPC6_QM_CP_BARRIER_CFG_4                                   0xF88404
+
+#define mmTPC6_QM_CP_DBG_0_0                                         0xF88408
+
+#define mmTPC6_QM_CP_DBG_0_1                                         0xF8840C
+
+#define mmTPC6_QM_CP_DBG_0_2                                         0xF88410
+
+#define mmTPC6_QM_CP_DBG_0_3                                         0xF88414
+
+#define mmTPC6_QM_CP_DBG_0_4                                         0xF88418
+
+#define mmTPC6_QM_CP_ARUSER_31_11_0                                  0xF8841C
+
+#define mmTPC6_QM_CP_ARUSER_31_11_1                                  0xF88420
+
+#define mmTPC6_QM_CP_ARUSER_31_11_2                                  0xF88424
+
+#define mmTPC6_QM_CP_ARUSER_31_11_3                                  0xF88428
+
+#define mmTPC6_QM_CP_ARUSER_31_11_4                                  0xF8842C
+
+#define mmTPC6_QM_CP_AWUSER_31_11_0                                  0xF88430
+
+#define mmTPC6_QM_CP_AWUSER_31_11_1                                  0xF88434
+
+#define mmTPC6_QM_CP_AWUSER_31_11_2                                  0xF88438
+
+#define mmTPC6_QM_CP_AWUSER_31_11_3                                  0xF8843C
+
+#define mmTPC6_QM_CP_AWUSER_31_11_4                                  0xF88440
+
+#define mmTPC6_QM_ARB_CFG_0                                          0xF88A00
+
+#define mmTPC6_QM_ARB_CHOISE_Q_PUSH                                  0xF88A04
+
+#define mmTPC6_QM_ARB_WRR_WEIGHT_0                                   0xF88A08
+
+#define mmTPC6_QM_ARB_WRR_WEIGHT_1                                   0xF88A0C
+
+#define mmTPC6_QM_ARB_WRR_WEIGHT_2                                   0xF88A10
+
+#define mmTPC6_QM_ARB_WRR_WEIGHT_3                                   0xF88A14
+
+#define mmTPC6_QM_ARB_CFG_1                                          0xF88A18
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_0                               0xF88A20
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_1                               0xF88A24
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_2                               0xF88A28
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_3                               0xF88A2C
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_4                               0xF88A30
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_5                               0xF88A34
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_6                               0xF88A38
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_7                               0xF88A3C
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_8                               0xF88A40
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_9                               0xF88A44
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_10                              0xF88A48
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_11                              0xF88A4C
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_12                              0xF88A50
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_13                              0xF88A54
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_14                              0xF88A58
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_15                              0xF88A5C
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_16                              0xF88A60
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_17                              0xF88A64
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_18                              0xF88A68
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_19                              0xF88A6C
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_20                              0xF88A70
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_21                              0xF88A74
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_22                              0xF88A78
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_23                              0xF88A7C
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_24                              0xF88A80
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_25                              0xF88A84
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_26                              0xF88A88
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_27                              0xF88A8C
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_28                              0xF88A90
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_29                              0xF88A94
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_30                              0xF88A98
+
+#define mmTPC6_QM_ARB_MST_AVAIL_CRED_31                              0xF88A9C
+
+#define mmTPC6_QM_ARB_MST_CRED_INC                                   0xF88AA0
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xF88AA4
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xF88AA8
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xF88AAC
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xF88AB0
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xF88AB4
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xF88AB8
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xF88ABC
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xF88AC0
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xF88AC4
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xF88AC8
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xF88ACC
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xF88AD0
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xF88AD4
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xF88AD8
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xF88ADC
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xF88AE0
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xF88AE4
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xF88AE8
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xF88AEC
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xF88AF0
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xF88AF4
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xF88AF8
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xF88AFC
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xF88B00
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xF88B04
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xF88B08
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xF88B0C
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xF88B10
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xF88B14
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xF88B18
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xF88B1C
+
+#define mmTPC6_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xF88B20
+
+#define mmTPC6_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xF88B28
+
+#define mmTPC6_QM_ARB_MST_SLAVE_EN                                   0xF88B2C
+
+#define mmTPC6_QM_ARB_MST_QUIET_PER                                  0xF88B34
+
+#define mmTPC6_QM_ARB_SLV_CHOISE_WDT                                 0xF88B38
+
+#define mmTPC6_QM_ARB_SLV_ID                                         0xF88B3C
+
+#define mmTPC6_QM_ARB_MSG_MAX_INFLIGHT                               0xF88B44
+
+#define mmTPC6_QM_ARB_MSG_AWUSER_31_11                               0xF88B48
+
+#define mmTPC6_QM_ARB_MSG_AWUSER_SEC_PROP                            0xF88B4C
+
+#define mmTPC6_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xF88B50
+
+#define mmTPC6_QM_ARB_BASE_LO                                        0xF88B54
+
+#define mmTPC6_QM_ARB_BASE_HI                                        0xF88B58
+
+#define mmTPC6_QM_ARB_STATE_STS                                      0xF88B80
+
+#define mmTPC6_QM_ARB_CHOISE_FULLNESS_STS                            0xF88B84
+
+#define mmTPC6_QM_ARB_MSG_STS                                        0xF88B88
+
+#define mmTPC6_QM_ARB_SLV_CHOISE_Q_HEAD                              0xF88B8C
+
+#define mmTPC6_QM_ARB_ERR_CAUSE                                      0xF88B9C
+
+#define mmTPC6_QM_ARB_ERR_MSG_EN                                     0xF88BA0
+
+#define mmTPC6_QM_ARB_ERR_STS_DRP                                    0xF88BA8
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_0                                 0xF88BB0
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_1                                 0xF88BB4
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_2                                 0xF88BB8
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_3                                 0xF88BBC
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_4                                 0xF88BC0
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_5                                 0xF88BC4
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_6                                 0xF88BC8
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_7                                 0xF88BCC
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_8                                 0xF88BD0
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_9                                 0xF88BD4
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_10                                0xF88BD8
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_11                                0xF88BDC
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_12                                0xF88BE0
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_13                                0xF88BE4
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_14                                0xF88BE8
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_15                                0xF88BEC
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_16                                0xF88BF0
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_17                                0xF88BF4
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_18                                0xF88BF8
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_19                                0xF88BFC
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_20                                0xF88C00
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_21                                0xF88C04
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_22                                0xF88C08
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_23                                0xF88C0C
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_24                                0xF88C10
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_25                                0xF88C14
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_26                                0xF88C18
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_27                                0xF88C1C
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_28                                0xF88C20
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_29                                0xF88C24
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_30                                0xF88C28
+
+#define mmTPC6_QM_ARB_MST_CRED_STS_31                                0xF88C2C
+
+#define mmTPC6_QM_CGM_CFG                                            0xF88C70
+
+#define mmTPC6_QM_CGM_STS                                            0xF88C74
+
+#define mmTPC6_QM_CGM_CFG1                                           0xF88C78
+
+#define mmTPC6_QM_LOCAL_RANGE_BASE                                   0xF88C80
+
+#define mmTPC6_QM_LOCAL_RANGE_SIZE                                   0xF88C84
+
+#define mmTPC6_QM_CSMR_STRICT_PRIO_CFG                               0xF88C90
+
+#define mmTPC6_QM_HBW_RD_RATE_LIM_CFG_1                              0xF88C94
+
+#define mmTPC6_QM_LBW_WR_RATE_LIM_CFG_0                              0xF88C98
+
+#define mmTPC6_QM_LBW_WR_RATE_LIM_CFG_1                              0xF88C9C
+
+#define mmTPC6_QM_HBW_RD_RATE_LIM_CFG_0                              0xF88CA0
+
+#define mmTPC6_QM_GLBL_AXCACHE                                       0xF88CA4
+
+#define mmTPC6_QM_IND_GW_APB_CFG                                     0xF88CB0
+
+#define mmTPC6_QM_IND_GW_APB_WDATA                                   0xF88CB4
+
+#define mmTPC6_QM_IND_GW_APB_RDATA                                   0xF88CB8
+
+#define mmTPC6_QM_IND_GW_APB_STATUS                                  0xF88CBC
+
+#define mmTPC6_QM_GLBL_ERR_ADDR_LO                                   0xF88CD0
+
+#define mmTPC6_QM_GLBL_ERR_ADDR_HI                                   0xF88CD4
+
+#define mmTPC6_QM_GLBL_ERR_WDATA                                     0xF88CD8
+
+#define mmTPC6_QM_GLBL_MEM_INIT_BUSY                                 0xF88D00
+
+#endif /* ASIC_REG_TPC6_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_cfg_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_cfg_regs.h
new file mode 100644
index 000000000000..1887b10e58e2
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_cfg_regs.h
@@ -0,0 +1,1226 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC7_CFG_REGS_H_
+#define ASIC_REG_TPC7_CFG_REGS_H_
+
+/*
+ *****************************************
+ *   TPC7_CFG (Prototype: TPC)
+ *****************************************
+ */
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_BASE_ADDR_LOW                     0xFC6400
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_BASE_ADDR_HIGH                    0xFC6404
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_PADDING_VALUE                     0xFC6408
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_TENSOR_CONFIG                     0xFC640C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_0_SIZE                        0xFC6410
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_0_STRIDE                      0xFC6414
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_1_SIZE                        0xFC6418
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_1_STRIDE                      0xFC641C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_2_SIZE                        0xFC6420
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_2_STRIDE                      0xFC6424
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_3_SIZE                        0xFC6428
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_3_STRIDE                      0xFC642C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_4_SIZE                        0xFC6430
+
+#define mmTPC7_CFG_KERNEL_TENSOR_0_DIM_4_STRIDE                      0xFC6434
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_BASE_ADDR_LOW                     0xFC6438
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_BASE_ADDR_HIGH                    0xFC643C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_PADDING_VALUE                     0xFC6440
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_TENSOR_CONFIG                     0xFC6444
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_0_SIZE                        0xFC6448
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_0_STRIDE                      0xFC644C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_1_SIZE                        0xFC6450
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_1_STRIDE                      0xFC6454
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_2_SIZE                        0xFC6458
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_2_STRIDE                      0xFC645C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_3_SIZE                        0xFC6460
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_3_STRIDE                      0xFC6464
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_4_SIZE                        0xFC6468
+
+#define mmTPC7_CFG_KERNEL_TENSOR_1_DIM_4_STRIDE                      0xFC646C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_BASE_ADDR_LOW                     0xFC6470
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_BASE_ADDR_HIGH                    0xFC6474
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_PADDING_VALUE                     0xFC6478
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_TENSOR_CONFIG                     0xFC647C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_0_SIZE                        0xFC6480
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_0_STRIDE                      0xFC6484
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_1_SIZE                        0xFC6488
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_1_STRIDE                      0xFC648C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_2_SIZE                        0xFC6490
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_2_STRIDE                      0xFC6494
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_3_SIZE                        0xFC6498
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_3_STRIDE                      0xFC649C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_4_SIZE                        0xFC64A0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_2_DIM_4_STRIDE                      0xFC64A4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_BASE_ADDR_LOW                     0xFC64A8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_BASE_ADDR_HIGH                    0xFC64AC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_PADDING_VALUE                     0xFC64B0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_TENSOR_CONFIG                     0xFC64B4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_0_SIZE                        0xFC64B8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_0_STRIDE                      0xFC64BC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_1_SIZE                        0xFC64C0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_1_STRIDE                      0xFC64C4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_2_SIZE                        0xFC64C8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_2_STRIDE                      0xFC64CC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_3_SIZE                        0xFC64D0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_3_STRIDE                      0xFC64D4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_4_SIZE                        0xFC64D8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_3_DIM_4_STRIDE                      0xFC64DC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_BASE_ADDR_LOW                     0xFC64E0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_BASE_ADDR_HIGH                    0xFC64E4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_PADDING_VALUE                     0xFC64E8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_TENSOR_CONFIG                     0xFC64EC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_0_SIZE                        0xFC64F0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_0_STRIDE                      0xFC64F4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_1_SIZE                        0xFC64F8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_1_STRIDE                      0xFC64FC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_2_SIZE                        0xFC6500
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_2_STRIDE                      0xFC6504
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_3_SIZE                        0xFC6508
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_3_STRIDE                      0xFC650C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_4_SIZE                        0xFC6510
+
+#define mmTPC7_CFG_KERNEL_TENSOR_4_DIM_4_STRIDE                      0xFC6514
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_BASE_ADDR_LOW                     0xFC6518
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_BASE_ADDR_HIGH                    0xFC651C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_PADDING_VALUE                     0xFC6520
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_TENSOR_CONFIG                     0xFC6524
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_0_SIZE                        0xFC6528
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_0_STRIDE                      0xFC652C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_1_SIZE                        0xFC6530
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_1_STRIDE                      0xFC6534
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_2_SIZE                        0xFC6538
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_2_STRIDE                      0xFC653C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_3_SIZE                        0xFC6540
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_3_STRIDE                      0xFC6544
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_4_SIZE                        0xFC6548
+
+#define mmTPC7_CFG_KERNEL_TENSOR_5_DIM_4_STRIDE                      0xFC654C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_BASE_ADDR_LOW                     0xFC6550
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_BASE_ADDR_HIGH                    0xFC6554
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_PADDING_VALUE                     0xFC6558
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_TENSOR_CONFIG                     0xFC655C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_0_SIZE                        0xFC6560
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_0_STRIDE                      0xFC6564
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_1_SIZE                        0xFC6568
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_1_STRIDE                      0xFC656C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_2_SIZE                        0xFC6570
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_2_STRIDE                      0xFC6574
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_3_SIZE                        0xFC6578
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_3_STRIDE                      0xFC657C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_4_SIZE                        0xFC6580
+
+#define mmTPC7_CFG_KERNEL_TENSOR_6_DIM_4_STRIDE                      0xFC6584
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_BASE_ADDR_LOW                     0xFC6588
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_BASE_ADDR_HIGH                    0xFC658C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_PADDING_VALUE                     0xFC6590
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_TENSOR_CONFIG                     0xFC6594
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_0_SIZE                        0xFC6598
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_0_STRIDE                      0xFC659C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_1_SIZE                        0xFC65A0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_1_STRIDE                      0xFC65A4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_2_SIZE                        0xFC65A8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_2_STRIDE                      0xFC65AC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_3_SIZE                        0xFC65B0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_3_STRIDE                      0xFC65B4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_4_SIZE                        0xFC65B8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_7_DIM_4_STRIDE                      0xFC65BC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_BASE_ADDR_LOW                     0xFC65C0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_BASE_ADDR_HIGH                    0xFC65C4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_PADDING_VALUE                     0xFC65C8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_TENSOR_CONFIG                     0xFC65CC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_0_SIZE                        0xFC65D0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_0_STRIDE                      0xFC65D4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_1_SIZE                        0xFC65D8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_1_STRIDE                      0xFC65DC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_2_SIZE                        0xFC65E0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_2_STRIDE                      0xFC65E4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_3_SIZE                        0xFC65E8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_3_STRIDE                      0xFC65EC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_4_SIZE                        0xFC65F0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_8_DIM_4_STRIDE                      0xFC65F4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_BASE_ADDR_LOW                     0xFC65F8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_BASE_ADDR_HIGH                    0xFC65FC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_PADDING_VALUE                     0xFC6600
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_TENSOR_CONFIG                     0xFC6604
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_0_SIZE                        0xFC6608
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_0_STRIDE                      0xFC660C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_1_SIZE                        0xFC6610
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_1_STRIDE                      0xFC6614
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_2_SIZE                        0xFC6618
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_2_STRIDE                      0xFC661C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_3_SIZE                        0xFC6620
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_3_STRIDE                      0xFC6624
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_4_SIZE                        0xFC6628
+
+#define mmTPC7_CFG_KERNEL_TENSOR_9_DIM_4_STRIDE                      0xFC662C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_BASE_ADDR_LOW                    0xFC6630
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_BASE_ADDR_HIGH                   0xFC6634
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_PADDING_VALUE                    0xFC6638
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_TENSOR_CONFIG                    0xFC663C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_0_SIZE                       0xFC6640
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_0_STRIDE                     0xFC6644
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_1_SIZE                       0xFC6648
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_1_STRIDE                     0xFC664C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_2_SIZE                       0xFC6650
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_2_STRIDE                     0xFC6654
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_3_SIZE                       0xFC6658
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_3_STRIDE                     0xFC665C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_4_SIZE                       0xFC6660
+
+#define mmTPC7_CFG_KERNEL_TENSOR_10_DIM_4_STRIDE                     0xFC6664
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_BASE_ADDR_LOW                    0xFC6668
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_BASE_ADDR_HIGH                   0xFC666C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_PADDING_VALUE                    0xFC6670
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_TENSOR_CONFIG                    0xFC6674
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_0_SIZE                       0xFC6678
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_0_STRIDE                     0xFC667C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_1_SIZE                       0xFC6680
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_1_STRIDE                     0xFC6684
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_2_SIZE                       0xFC6688
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_2_STRIDE                     0xFC668C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_3_SIZE                       0xFC6690
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_3_STRIDE                     0xFC6694
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_4_SIZE                       0xFC6698
+
+#define mmTPC7_CFG_KERNEL_TENSOR_11_DIM_4_STRIDE                     0xFC669C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_BASE_ADDR_LOW                    0xFC66A0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_BASE_ADDR_HIGH                   0xFC66A4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_PADDING_VALUE                    0xFC66A8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_TENSOR_CONFIG                    0xFC66AC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_0_SIZE                       0xFC66B0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_0_STRIDE                     0xFC66B4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_1_SIZE                       0xFC66B8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_1_STRIDE                     0xFC66BC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_2_SIZE                       0xFC66C0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_2_STRIDE                     0xFC66C4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_3_SIZE                       0xFC66C8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_3_STRIDE                     0xFC66CC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_4_SIZE                       0xFC66D0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_12_DIM_4_STRIDE                     0xFC66D4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_BASE_ADDR_LOW                    0xFC66D8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_BASE_ADDR_HIGH                   0xFC66DC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_PADDING_VALUE                    0xFC66E0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_TENSOR_CONFIG                    0xFC66E4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_0_SIZE                       0xFC66E8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_0_STRIDE                     0xFC66EC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_1_SIZE                       0xFC66F0
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_1_STRIDE                     0xFC66F4
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_2_SIZE                       0xFC66F8
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_2_STRIDE                     0xFC66FC
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_3_SIZE                       0xFC6700
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_3_STRIDE                     0xFC6704
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_4_SIZE                       0xFC6708
+
+#define mmTPC7_CFG_KERNEL_TENSOR_13_DIM_4_STRIDE                     0xFC670C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_BASE_ADDR_LOW                    0xFC6710
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_BASE_ADDR_HIGH                   0xFC6714
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_PADDING_VALUE                    0xFC6718
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_TENSOR_CONFIG                    0xFC671C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_0_SIZE                       0xFC6720
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_0_STRIDE                     0xFC6724
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_1_SIZE                       0xFC6728
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_1_STRIDE                     0xFC672C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_2_SIZE                       0xFC6730
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_2_STRIDE                     0xFC6734
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_3_SIZE                       0xFC6738
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_3_STRIDE                     0xFC673C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_4_SIZE                       0xFC6740
+
+#define mmTPC7_CFG_KERNEL_TENSOR_14_DIM_4_STRIDE                     0xFC6744
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_BASE_ADDR_LOW                    0xFC6748
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_BASE_ADDR_HIGH                   0xFC674C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_PADDING_VALUE                    0xFC6750
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_TENSOR_CONFIG                    0xFC6754
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_0_SIZE                       0xFC6758
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_0_STRIDE                     0xFC675C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_1_SIZE                       0xFC6760
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_1_STRIDE                     0xFC6764
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_2_SIZE                       0xFC6768
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_2_STRIDE                     0xFC676C
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_3_SIZE                       0xFC6770
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_3_STRIDE                     0xFC6774
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_4_SIZE                       0xFC6778
+
+#define mmTPC7_CFG_KERNEL_TENSOR_15_DIM_4_STRIDE                     0xFC677C
+
+#define mmTPC7_CFG_KERNEL_SYNC_OBJECT_MESSAGE                        0xFC6780
+
+#define mmTPC7_CFG_KERNEL_SYNC_OBJECT_ADDR                           0xFC6784
+
+#define mmTPC7_CFG_KERNEL_KERNEL_BASE_ADDRESS_LOW                    0xFC6788
+
+#define mmTPC7_CFG_KERNEL_KERNEL_BASE_ADDRESS_HIGH                   0xFC678C
+
+#define mmTPC7_CFG_KERNEL_TID_BASE_DIM_0                             0xFC6790
+
+#define mmTPC7_CFG_KERNEL_TID_SIZE_DIM_0                             0xFC6794
+
+#define mmTPC7_CFG_KERNEL_TID_BASE_DIM_1                             0xFC6798
+
+#define mmTPC7_CFG_KERNEL_TID_SIZE_DIM_1                             0xFC679C
+
+#define mmTPC7_CFG_KERNEL_TID_BASE_DIM_2                             0xFC67A0
+
+#define mmTPC7_CFG_KERNEL_TID_SIZE_DIM_2                             0xFC67A4
+
+#define mmTPC7_CFG_KERNEL_TID_BASE_DIM_3                             0xFC67A8
+
+#define mmTPC7_CFG_KERNEL_TID_SIZE_DIM_3                             0xFC67AC
+
+#define mmTPC7_CFG_KERNEL_TID_BASE_DIM_4                             0xFC67B0
+
+#define mmTPC7_CFG_KERNEL_TID_SIZE_DIM_4                             0xFC67B4
+
+#define mmTPC7_CFG_KERNEL_KERNEL_CONFIG                              0xFC67B8
+
+#define mmTPC7_CFG_KERNEL_KERNEL_ID                                  0xFC67BC
+
+#define mmTPC7_CFG_KERNEL_SRF_0                                      0xFC67C0
+
+#define mmTPC7_CFG_KERNEL_SRF_1                                      0xFC67C4
+
+#define mmTPC7_CFG_KERNEL_SRF_2                                      0xFC67C8
+
+#define mmTPC7_CFG_KERNEL_SRF_3                                      0xFC67CC
+
+#define mmTPC7_CFG_KERNEL_SRF_4                                      0xFC67D0
+
+#define mmTPC7_CFG_KERNEL_SRF_5                                      0xFC67D4
+
+#define mmTPC7_CFG_KERNEL_SRF_6                                      0xFC67D8
+
+#define mmTPC7_CFG_KERNEL_SRF_7                                      0xFC67DC
+
+#define mmTPC7_CFG_KERNEL_SRF_8                                      0xFC67E0
+
+#define mmTPC7_CFG_KERNEL_SRF_9                                      0xFC67E4
+
+#define mmTPC7_CFG_KERNEL_SRF_10                                     0xFC67E8
+
+#define mmTPC7_CFG_KERNEL_SRF_11                                     0xFC67EC
+
+#define mmTPC7_CFG_KERNEL_SRF_12                                     0xFC67F0
+
+#define mmTPC7_CFG_KERNEL_SRF_13                                     0xFC67F4
+
+#define mmTPC7_CFG_KERNEL_SRF_14                                     0xFC67F8
+
+#define mmTPC7_CFG_KERNEL_SRF_15                                     0xFC67FC
+
+#define mmTPC7_CFG_KERNEL_SRF_16                                     0xFC6800
+
+#define mmTPC7_CFG_KERNEL_SRF_17                                     0xFC6804
+
+#define mmTPC7_CFG_KERNEL_SRF_18                                     0xFC6808
+
+#define mmTPC7_CFG_KERNEL_SRF_19                                     0xFC680C
+
+#define mmTPC7_CFG_KERNEL_SRF_20                                     0xFC6810
+
+#define mmTPC7_CFG_KERNEL_SRF_21                                     0xFC6814
+
+#define mmTPC7_CFG_KERNEL_SRF_22                                     0xFC6818
+
+#define mmTPC7_CFG_KERNEL_SRF_23                                     0xFC681C
+
+#define mmTPC7_CFG_KERNEL_SRF_24                                     0xFC6820
+
+#define mmTPC7_CFG_KERNEL_SRF_25                                     0xFC6824
+
+#define mmTPC7_CFG_KERNEL_SRF_26                                     0xFC6828
+
+#define mmTPC7_CFG_KERNEL_SRF_27                                     0xFC682C
+
+#define mmTPC7_CFG_KERNEL_SRF_28                                     0xFC6830
+
+#define mmTPC7_CFG_KERNEL_SRF_29                                     0xFC6834
+
+#define mmTPC7_CFG_KERNEL_SRF_30                                     0xFC6838
+
+#define mmTPC7_CFG_KERNEL_SRF_31                                     0xFC683C
+
+#define mmTPC7_CFG_ROUND_CSR                                         0xFC68FC
+
+#define mmTPC7_CFG_PROT                                              0xFC6900
+
+#define mmTPC7_CFG_SEMAPHORE                                         0xFC6908
+
+#define mmTPC7_CFG_VFLAGS                                            0xFC690C
+
+#define mmTPC7_CFG_SFLAGS                                            0xFC6910
+
+#define mmTPC7_CFG_LFSR_POLYNOM                                      0xFC6918
+
+#define mmTPC7_CFG_STATUS                                            0xFC691C
+
+#define mmTPC7_CFG_CFG_BASE_ADDRESS_HIGH                             0xFC6920
+
+#define mmTPC7_CFG_CFG_SUBTRACT_VALUE                                0xFC6924
+
+#define mmTPC7_CFG_SM_BASE_ADDRESS_HIGH                              0xFC692C
+
+#define mmTPC7_CFG_TPC_CMD                                           0xFC6930
+
+#define mmTPC7_CFG_TPC_EXECUTE                                       0xFC6938
+
+#define mmTPC7_CFG_TPC_STALL                                         0xFC693C
+
+#define mmTPC7_CFG_ICACHE_BASE_ADDERESS_LOW                          0xFC6940
+
+#define mmTPC7_CFG_ICACHE_BASE_ADDERESS_HIGH                         0xFC6944
+
+#define mmTPC7_CFG_RD_RATE_LIMIT                                     0xFC6948
+
+#define mmTPC7_CFG_WR_RATE_LIMIT                                     0xFC6950
+
+#define mmTPC7_CFG_MSS_CONFIG                                        0xFC6954
+
+#define mmTPC7_CFG_TPC_INTR_CAUSE                                    0xFC6958
+
+#define mmTPC7_CFG_TPC_INTR_MASK                                     0xFC695C
+
+#define mmTPC7_CFG_WQ_CREDITS                                        0xFC6960
+
+#define mmTPC7_CFG_ARUSER_LO                                         0xFC6964
+
+#define mmTPC7_CFG_ARUSER_HI                                         0xFC6968
+
+#define mmTPC7_CFG_AWUSER_LO                                         0xFC696C
+
+#define mmTPC7_CFG_AWUSER_HI                                         0xFC6970
+
+#define mmTPC7_CFG_OPCODE_EXEC                                       0xFC6974
+
+#define mmTPC7_CFG_LUT_FUNC32_BASE_ADDR_LO                           0xFC6978
+
+#define mmTPC7_CFG_LUT_FUNC32_BASE_ADDR_HI                           0xFC697C
+
+#define mmTPC7_CFG_LUT_FUNC64_BASE_ADDR_LO                           0xFC6980
+
+#define mmTPC7_CFG_LUT_FUNC64_BASE_ADDR_HI                           0xFC6984
+
+#define mmTPC7_CFG_LUT_FUNC128_BASE_ADDR_LO                          0xFC6988
+
+#define mmTPC7_CFG_LUT_FUNC128_BASE_ADDR_HI                          0xFC698C
+
+#define mmTPC7_CFG_LUT_FUNC256_BASE_ADDR_LO                          0xFC6990
+
+#define mmTPC7_CFG_LUT_FUNC256_BASE_ADDR_HI                          0xFC6994
+
+#define mmTPC7_CFG_TSB_CFG_MAX_SIZE                                  0xFC6998
+
+#define mmTPC7_CFG_TSB_CFG                                           0xFC699C
+
+#define mmTPC7_CFG_DBGMEM_ADD                                        0xFC69A0
+
+#define mmTPC7_CFG_DBGMEM_DATA_WR                                    0xFC69A4
+
+#define mmTPC7_CFG_DBGMEM_DATA_RD                                    0xFC69A8
+
+#define mmTPC7_CFG_DBGMEM_CTRL                                       0xFC69AC
+
+#define mmTPC7_CFG_DBGMEM_RC                                         0xFC69B0
+
+#define mmTPC7_CFG_TSB_INFLIGHT_CNTR                                 0xFC69B4
+
+#define mmTPC7_CFG_WQ_INFLIGHT_CNTR                                  0xFC69B8
+
+#define mmTPC7_CFG_WQ_LBW_TOTAL_CNTR                                 0xFC69BC
+
+#define mmTPC7_CFG_WQ_HBW_TOTAL_CNTR                                 0xFC69C0
+
+#define mmTPC7_CFG_IRQ_OCCOUPY_CNTR                                  0xFC69C4
+
+#define mmTPC7_CFG_FUNC_MBIST_CNTRL                                  0xFC69D0
+
+#define mmTPC7_CFG_FUNC_MBIST_PAT                                    0xFC69D4
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_0                                  0xFC69D8
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_1                                  0xFC69DC
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_2                                  0xFC69E0
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_3                                  0xFC69E4
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_4                                  0xFC69E8
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_5                                  0xFC69EC
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_6                                  0xFC69F0
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_7                                  0xFC69F4
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_8                                  0xFC69F8
+
+#define mmTPC7_CFG_FUNC_MBIST_MEM_9                                  0xFC69FC
+
+#define mmTPC7_CFG_QM_TENSOR_0_BASE_ADDR_LOW                         0xFC6A00
+
+#define mmTPC7_CFG_QM_TENSOR_0_BASE_ADDR_HIGH                        0xFC6A04
+
+#define mmTPC7_CFG_QM_TENSOR_0_PADDING_VALUE                         0xFC6A08
+
+#define mmTPC7_CFG_QM_TENSOR_0_TENSOR_CONFIG                         0xFC6A0C
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_0_SIZE                            0xFC6A10
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_0_STRIDE                          0xFC6A14
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_1_SIZE                            0xFC6A18
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_1_STRIDE                          0xFC6A1C
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_2_SIZE                            0xFC6A20
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_2_STRIDE                          0xFC6A24
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_3_SIZE                            0xFC6A28
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_3_STRIDE                          0xFC6A2C
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_4_SIZE                            0xFC6A30
+
+#define mmTPC7_CFG_QM_TENSOR_0_DIM_4_STRIDE                          0xFC6A34
+
+#define mmTPC7_CFG_QM_TENSOR_1_BASE_ADDR_LOW                         0xFC6A38
+
+#define mmTPC7_CFG_QM_TENSOR_1_BASE_ADDR_HIGH                        0xFC6A3C
+
+#define mmTPC7_CFG_QM_TENSOR_1_PADDING_VALUE                         0xFC6A40
+
+#define mmTPC7_CFG_QM_TENSOR_1_TENSOR_CONFIG                         0xFC6A44
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_0_SIZE                            0xFC6A48
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_0_STRIDE                          0xFC6A4C
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_1_SIZE                            0xFC6A50
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_1_STRIDE                          0xFC6A54
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_2_SIZE                            0xFC6A58
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_2_STRIDE                          0xFC6A5C
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_3_SIZE                            0xFC6A60
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_3_STRIDE                          0xFC6A64
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_4_SIZE                            0xFC6A68
+
+#define mmTPC7_CFG_QM_TENSOR_1_DIM_4_STRIDE                          0xFC6A6C
+
+#define mmTPC7_CFG_QM_TENSOR_2_BASE_ADDR_LOW                         0xFC6A70
+
+#define mmTPC7_CFG_QM_TENSOR_2_BASE_ADDR_HIGH                        0xFC6A74
+
+#define mmTPC7_CFG_QM_TENSOR_2_PADDING_VALUE                         0xFC6A78
+
+#define mmTPC7_CFG_QM_TENSOR_2_TENSOR_CONFIG                         0xFC6A7C
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_0_SIZE                            0xFC6A80
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_0_STRIDE                          0xFC6A84
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_1_SIZE                            0xFC6A88
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_1_STRIDE                          0xFC6A8C
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_2_SIZE                            0xFC6A90
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_2_STRIDE                          0xFC6A94
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_3_SIZE                            0xFC6A98
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_3_STRIDE                          0xFC6A9C
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_4_SIZE                            0xFC6AA0
+
+#define mmTPC7_CFG_QM_TENSOR_2_DIM_4_STRIDE                          0xFC6AA4
+
+#define mmTPC7_CFG_QM_TENSOR_3_BASE_ADDR_LOW                         0xFC6AA8
+
+#define mmTPC7_CFG_QM_TENSOR_3_BASE_ADDR_HIGH                        0xFC6AAC
+
+#define mmTPC7_CFG_QM_TENSOR_3_PADDING_VALUE                         0xFC6AB0
+
+#define mmTPC7_CFG_QM_TENSOR_3_TENSOR_CONFIG                         0xFC6AB4
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_0_SIZE                            0xFC6AB8
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_0_STRIDE                          0xFC6ABC
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_1_SIZE                            0xFC6AC0
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_1_STRIDE                          0xFC6AC4
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_2_SIZE                            0xFC6AC8
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_2_STRIDE                          0xFC6ACC
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_3_SIZE                            0xFC6AD0
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_3_STRIDE                          0xFC6AD4
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_4_SIZE                            0xFC6AD8
+
+#define mmTPC7_CFG_QM_TENSOR_3_DIM_4_STRIDE                          0xFC6ADC
+
+#define mmTPC7_CFG_QM_TENSOR_4_BASE_ADDR_LOW                         0xFC6AE0
+
+#define mmTPC7_CFG_QM_TENSOR_4_BASE_ADDR_HIGH                        0xFC6AE4
+
+#define mmTPC7_CFG_QM_TENSOR_4_PADDING_VALUE                         0xFC6AE8
+
+#define mmTPC7_CFG_QM_TENSOR_4_TENSOR_CONFIG                         0xFC6AEC
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_0_SIZE                            0xFC6AF0
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_0_STRIDE                          0xFC6AF4
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_1_SIZE                            0xFC6AF8
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_1_STRIDE                          0xFC6AFC
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_2_SIZE                            0xFC6B00
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_2_STRIDE                          0xFC6B04
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_3_SIZE                            0xFC6B08
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_3_STRIDE                          0xFC6B0C
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_4_SIZE                            0xFC6B10
+
+#define mmTPC7_CFG_QM_TENSOR_4_DIM_4_STRIDE                          0xFC6B14
+
+#define mmTPC7_CFG_QM_TENSOR_5_BASE_ADDR_LOW                         0xFC6B18
+
+#define mmTPC7_CFG_QM_TENSOR_5_BASE_ADDR_HIGH                        0xFC6B1C
+
+#define mmTPC7_CFG_QM_TENSOR_5_PADDING_VALUE                         0xFC6B20
+
+#define mmTPC7_CFG_QM_TENSOR_5_TENSOR_CONFIG                         0xFC6B24
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_0_SIZE                            0xFC6B28
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_0_STRIDE                          0xFC6B2C
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_1_SIZE                            0xFC6B30
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_1_STRIDE                          0xFC6B34
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_2_SIZE                            0xFC6B38
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_2_STRIDE                          0xFC6B3C
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_3_SIZE                            0xFC6B40
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_3_STRIDE                          0xFC6B44
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_4_SIZE                            0xFC6B48
+
+#define mmTPC7_CFG_QM_TENSOR_5_DIM_4_STRIDE                          0xFC6B4C
+
+#define mmTPC7_CFG_QM_TENSOR_6_BASE_ADDR_LOW                         0xFC6B50
+
+#define mmTPC7_CFG_QM_TENSOR_6_BASE_ADDR_HIGH                        0xFC6B54
+
+#define mmTPC7_CFG_QM_TENSOR_6_PADDING_VALUE                         0xFC6B58
+
+#define mmTPC7_CFG_QM_TENSOR_6_TENSOR_CONFIG                         0xFC6B5C
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_0_SIZE                            0xFC6B60
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_0_STRIDE                          0xFC6B64
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_1_SIZE                            0xFC6B68
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_1_STRIDE                          0xFC6B6C
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_2_SIZE                            0xFC6B70
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_2_STRIDE                          0xFC6B74
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_3_SIZE                            0xFC6B78
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_3_STRIDE                          0xFC6B7C
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_4_SIZE                            0xFC6B80
+
+#define mmTPC7_CFG_QM_TENSOR_6_DIM_4_STRIDE                          0xFC6B84
+
+#define mmTPC7_CFG_QM_TENSOR_7_BASE_ADDR_LOW                         0xFC6B88
+
+#define mmTPC7_CFG_QM_TENSOR_7_BASE_ADDR_HIGH                        0xFC6B8C
+
+#define mmTPC7_CFG_QM_TENSOR_7_PADDING_VALUE                         0xFC6B90
+
+#define mmTPC7_CFG_QM_TENSOR_7_TENSOR_CONFIG                         0xFC6B94
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_0_SIZE                            0xFC6B98
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_0_STRIDE                          0xFC6B9C
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_1_SIZE                            0xFC6BA0
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_1_STRIDE                          0xFC6BA4
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_2_SIZE                            0xFC6BA8
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_2_STRIDE                          0xFC6BAC
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_3_SIZE                            0xFC6BB0
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_3_STRIDE                          0xFC6BB4
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_4_SIZE                            0xFC6BB8
+
+#define mmTPC7_CFG_QM_TENSOR_7_DIM_4_STRIDE                          0xFC6BBC
+
+#define mmTPC7_CFG_QM_TENSOR_8_BASE_ADDR_LOW                         0xFC6BC0
+
+#define mmTPC7_CFG_QM_TENSOR_8_BASE_ADDR_HIGH                        0xFC6BC4
+
+#define mmTPC7_CFG_QM_TENSOR_8_PADDING_VALUE                         0xFC6BC8
+
+#define mmTPC7_CFG_QM_TENSOR_8_TENSOR_CONFIG                         0xFC6BCC
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_0_SIZE                            0xFC6BD0
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_0_STRIDE                          0xFC6BD4
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_1_SIZE                            0xFC6BD8
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_1_STRIDE                          0xFC6BDC
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_2_SIZE                            0xFC6BE0
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_2_STRIDE                          0xFC6BE4
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_3_SIZE                            0xFC6BE8
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_3_STRIDE                          0xFC6BEC
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_4_SIZE                            0xFC6BF0
+
+#define mmTPC7_CFG_QM_TENSOR_8_DIM_4_STRIDE                          0xFC6BF4
+
+#define mmTPC7_CFG_QM_TENSOR_9_BASE_ADDR_LOW                         0xFC6BF8
+
+#define mmTPC7_CFG_QM_TENSOR_9_BASE_ADDR_HIGH                        0xFC6BFC
+
+#define mmTPC7_CFG_QM_TENSOR_9_PADDING_VALUE                         0xFC6C00
+
+#define mmTPC7_CFG_QM_TENSOR_9_TENSOR_CONFIG                         0xFC6C04
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_0_SIZE                            0xFC6C08
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_0_STRIDE                          0xFC6C0C
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_1_SIZE                            0xFC6C10
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_1_STRIDE                          0xFC6C14
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_2_SIZE                            0xFC6C18
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_2_STRIDE                          0xFC6C1C
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_3_SIZE                            0xFC6C20
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_3_STRIDE                          0xFC6C24
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_4_SIZE                            0xFC6C28
+
+#define mmTPC7_CFG_QM_TENSOR_9_DIM_4_STRIDE                          0xFC6C2C
+
+#define mmTPC7_CFG_QM_TENSOR_10_BASE_ADDR_LOW                        0xFC6C30
+
+#define mmTPC7_CFG_QM_TENSOR_10_BASE_ADDR_HIGH                       0xFC6C34
+
+#define mmTPC7_CFG_QM_TENSOR_10_PADDING_VALUE                        0xFC6C38
+
+#define mmTPC7_CFG_QM_TENSOR_10_TENSOR_CONFIG                        0xFC6C3C
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_0_SIZE                           0xFC6C40
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_0_STRIDE                         0xFC6C44
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_1_SIZE                           0xFC6C48
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_1_STRIDE                         0xFC6C4C
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_2_SIZE                           0xFC6C50
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_2_STRIDE                         0xFC6C54
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_3_SIZE                           0xFC6C58
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_3_STRIDE                         0xFC6C5C
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_4_SIZE                           0xFC6C60
+
+#define mmTPC7_CFG_QM_TENSOR_10_DIM_4_STRIDE                         0xFC6C64
+
+#define mmTPC7_CFG_QM_TENSOR_11_BASE_ADDR_LOW                        0xFC6C68
+
+#define mmTPC7_CFG_QM_TENSOR_11_BASE_ADDR_HIGH                       0xFC6C6C
+
+#define mmTPC7_CFG_QM_TENSOR_11_PADDING_VALUE                        0xFC6C70
+
+#define mmTPC7_CFG_QM_TENSOR_11_TENSOR_CONFIG                        0xFC6C74
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_0_SIZE                           0xFC6C78
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_0_STRIDE                         0xFC6C7C
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_1_SIZE                           0xFC6C80
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_1_STRIDE                         0xFC6C84
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_2_SIZE                           0xFC6C88
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_2_STRIDE                         0xFC6C8C
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_3_SIZE                           0xFC6C90
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_3_STRIDE                         0xFC6C94
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_4_SIZE                           0xFC6C98
+
+#define mmTPC7_CFG_QM_TENSOR_11_DIM_4_STRIDE                         0xFC6C9C
+
+#define mmTPC7_CFG_QM_TENSOR_12_BASE_ADDR_LOW                        0xFC6CA0
+
+#define mmTPC7_CFG_QM_TENSOR_12_BASE_ADDR_HIGH                       0xFC6CA4
+
+#define mmTPC7_CFG_QM_TENSOR_12_PADDING_VALUE                        0xFC6CA8
+
+#define mmTPC7_CFG_QM_TENSOR_12_TENSOR_CONFIG                        0xFC6CAC
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_0_SIZE                           0xFC6CB0
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_0_STRIDE                         0xFC6CB4
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_1_SIZE                           0xFC6CB8
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_1_STRIDE                         0xFC6CBC
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_2_SIZE                           0xFC6CC0
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_2_STRIDE                         0xFC6CC4
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_3_SIZE                           0xFC6CC8
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_3_STRIDE                         0xFC6CCC
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_4_SIZE                           0xFC6CD0
+
+#define mmTPC7_CFG_QM_TENSOR_12_DIM_4_STRIDE                         0xFC6CD4
+
+#define mmTPC7_CFG_QM_TENSOR_13_BASE_ADDR_LOW                        0xFC6CD8
+
+#define mmTPC7_CFG_QM_TENSOR_13_BASE_ADDR_HIGH                       0xFC6CDC
+
+#define mmTPC7_CFG_QM_TENSOR_13_PADDING_VALUE                        0xFC6CE0
+
+#define mmTPC7_CFG_QM_TENSOR_13_TENSOR_CONFIG                        0xFC6CE4
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_0_SIZE                           0xFC6CE8
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_0_STRIDE                         0xFC6CEC
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_1_SIZE                           0xFC6CF0
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_1_STRIDE                         0xFC6CF4
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_2_SIZE                           0xFC6CF8
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_2_STRIDE                         0xFC6CFC
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_3_SIZE                           0xFC6D00
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_3_STRIDE                         0xFC6D04
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_4_SIZE                           0xFC6D08
+
+#define mmTPC7_CFG_QM_TENSOR_13_DIM_4_STRIDE                         0xFC6D0C
+
+#define mmTPC7_CFG_QM_TENSOR_14_BASE_ADDR_LOW                        0xFC6D10
+
+#define mmTPC7_CFG_QM_TENSOR_14_BASE_ADDR_HIGH                       0xFC6D14
+
+#define mmTPC7_CFG_QM_TENSOR_14_PADDING_VALUE                        0xFC6D18
+
+#define mmTPC7_CFG_QM_TENSOR_14_TENSOR_CONFIG                        0xFC6D1C
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_0_SIZE                           0xFC6D20
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_0_STRIDE                         0xFC6D24
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_1_SIZE                           0xFC6D28
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_1_STRIDE                         0xFC6D2C
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_2_SIZE                           0xFC6D30
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_2_STRIDE                         0xFC6D34
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_3_SIZE                           0xFC6D38
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_3_STRIDE                         0xFC6D3C
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_4_SIZE                           0xFC6D40
+
+#define mmTPC7_CFG_QM_TENSOR_14_DIM_4_STRIDE                         0xFC6D44
+
+#define mmTPC7_CFG_QM_TENSOR_15_BASE_ADDR_LOW                        0xFC6D48
+
+#define mmTPC7_CFG_QM_TENSOR_15_BASE_ADDR_HIGH                       0xFC6D4C
+
+#define mmTPC7_CFG_QM_TENSOR_15_PADDING_VALUE                        0xFC6D50
+
+#define mmTPC7_CFG_QM_TENSOR_15_TENSOR_CONFIG                        0xFC6D54
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_0_SIZE                           0xFC6D58
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_0_STRIDE                         0xFC6D5C
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_1_SIZE                           0xFC6D60
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_1_STRIDE                         0xFC6D64
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_2_SIZE                           0xFC6D68
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_2_STRIDE                         0xFC6D6C
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_3_SIZE                           0xFC6D70
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_3_STRIDE                         0xFC6D74
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_4_SIZE                           0xFC6D78
+
+#define mmTPC7_CFG_QM_TENSOR_15_DIM_4_STRIDE                         0xFC6D7C
+
+#define mmTPC7_CFG_QM_SYNC_OBJECT_MESSAGE                            0xFC6D80
+
+#define mmTPC7_CFG_QM_SYNC_OBJECT_ADDR                               0xFC6D84
+
+#define mmTPC7_CFG_QM_KERNEL_BASE_ADDRESS_LOW                        0xFC6D88
+
+#define mmTPC7_CFG_QM_KERNEL_BASE_ADDRESS_HIGH                       0xFC6D8C
+
+#define mmTPC7_CFG_QM_TID_BASE_DIM_0                                 0xFC6D90
+
+#define mmTPC7_CFG_QM_TID_SIZE_DIM_0                                 0xFC6D94
+
+#define mmTPC7_CFG_QM_TID_BASE_DIM_1                                 0xFC6D98
+
+#define mmTPC7_CFG_QM_TID_SIZE_DIM_1                                 0xFC6D9C
+
+#define mmTPC7_CFG_QM_TID_BASE_DIM_2                                 0xFC6DA0
+
+#define mmTPC7_CFG_QM_TID_SIZE_DIM_2                                 0xFC6DA4
+
+#define mmTPC7_CFG_QM_TID_BASE_DIM_3                                 0xFC6DA8
+
+#define mmTPC7_CFG_QM_TID_SIZE_DIM_3                                 0xFC6DAC
+
+#define mmTPC7_CFG_QM_TID_BASE_DIM_4                                 0xFC6DB0
+
+#define mmTPC7_CFG_QM_TID_SIZE_DIM_4                                 0xFC6DB4
+
+#define mmTPC7_CFG_QM_KERNEL_CONFIG                                  0xFC6DB8
+
+#define mmTPC7_CFG_QM_KERNEL_ID                                      0xFC6DBC
+
+#define mmTPC7_CFG_QM_SRF_0                                          0xFC6DC0
+
+#define mmTPC7_CFG_QM_SRF_1                                          0xFC6DC4
+
+#define mmTPC7_CFG_QM_SRF_2                                          0xFC6DC8
+
+#define mmTPC7_CFG_QM_SRF_3                                          0xFC6DCC
+
+#define mmTPC7_CFG_QM_SRF_4                                          0xFC6DD0
+
+#define mmTPC7_CFG_QM_SRF_5                                          0xFC6DD4
+
+#define mmTPC7_CFG_QM_SRF_6                                          0xFC6DD8
+
+#define mmTPC7_CFG_QM_SRF_7                                          0xFC6DDC
+
+#define mmTPC7_CFG_QM_SRF_8                                          0xFC6DE0
+
+#define mmTPC7_CFG_QM_SRF_9                                          0xFC6DE4
+
+#define mmTPC7_CFG_QM_SRF_10                                         0xFC6DE8
+
+#define mmTPC7_CFG_QM_SRF_11                                         0xFC6DEC
+
+#define mmTPC7_CFG_QM_SRF_12                                         0xFC6DF0
+
+#define mmTPC7_CFG_QM_SRF_13                                         0xFC6DF4
+
+#define mmTPC7_CFG_QM_SRF_14                                         0xFC6DF8
+
+#define mmTPC7_CFG_QM_SRF_15                                         0xFC6DFC
+
+#define mmTPC7_CFG_QM_SRF_16                                         0xFC6E00
+
+#define mmTPC7_CFG_QM_SRF_17                                         0xFC6E04
+
+#define mmTPC7_CFG_QM_SRF_18                                         0xFC6E08
+
+#define mmTPC7_CFG_QM_SRF_19                                         0xFC6E0C
+
+#define mmTPC7_CFG_QM_SRF_20                                         0xFC6E10
+
+#define mmTPC7_CFG_QM_SRF_21                                         0xFC6E14
+
+#define mmTPC7_CFG_QM_SRF_22                                         0xFC6E18
+
+#define mmTPC7_CFG_QM_SRF_23                                         0xFC6E1C
+
+#define mmTPC7_CFG_QM_SRF_24                                         0xFC6E20
+
+#define mmTPC7_CFG_QM_SRF_25                                         0xFC6E24
+
+#define mmTPC7_CFG_QM_SRF_26                                         0xFC6E28
+
+#define mmTPC7_CFG_QM_SRF_27                                         0xFC6E2C
+
+#define mmTPC7_CFG_QM_SRF_28                                         0xFC6E30
+
+#define mmTPC7_CFG_QM_SRF_29                                         0xFC6E34
+
+#define mmTPC7_CFG_QM_SRF_30                                         0xFC6E38
+
+#define mmTPC7_CFG_QM_SRF_31                                         0xFC6E3C
+
+#endif /* ASIC_REG_TPC7_CFG_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_qm_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_qm_regs.h
new file mode 100644
index 000000000000..5c36c972c027
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/tpc7_qm_regs.h
@@ -0,0 +1,834 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_TPC7_QM_REGS_H_
+#define ASIC_REG_TPC7_QM_REGS_H_
+
+/*
+ *****************************************
+ *   TPC7_QM (Prototype: QMAN)
+ *****************************************
+ */
+
+#define mmTPC7_QM_GLBL_CFG0                                          0xFC8000
+
+#define mmTPC7_QM_GLBL_CFG1                                          0xFC8004
+
+#define mmTPC7_QM_GLBL_PROT                                          0xFC8008
+
+#define mmTPC7_QM_GLBL_ERR_CFG                                       0xFC800C
+
+#define mmTPC7_QM_GLBL_SECURE_PROPS_0                                0xFC8010
+
+#define mmTPC7_QM_GLBL_SECURE_PROPS_1                                0xFC8014
+
+#define mmTPC7_QM_GLBL_SECURE_PROPS_2                                0xFC8018
+
+#define mmTPC7_QM_GLBL_SECURE_PROPS_3                                0xFC801C
+
+#define mmTPC7_QM_GLBL_SECURE_PROPS_4                                0xFC8020
+
+#define mmTPC7_QM_GLBL_NON_SECURE_PROPS_0                            0xFC8024
+
+#define mmTPC7_QM_GLBL_NON_SECURE_PROPS_1                            0xFC8028
+
+#define mmTPC7_QM_GLBL_NON_SECURE_PROPS_2                            0xFC802C
+
+#define mmTPC7_QM_GLBL_NON_SECURE_PROPS_3                            0xFC8030
+
+#define mmTPC7_QM_GLBL_NON_SECURE_PROPS_4                            0xFC8034
+
+#define mmTPC7_QM_GLBL_STS0                                          0xFC8038
+
+#define mmTPC7_QM_GLBL_STS1_0                                        0xFC8040
+
+#define mmTPC7_QM_GLBL_STS1_1                                        0xFC8044
+
+#define mmTPC7_QM_GLBL_STS1_2                                        0xFC8048
+
+#define mmTPC7_QM_GLBL_STS1_3                                        0xFC804C
+
+#define mmTPC7_QM_GLBL_STS1_4                                        0xFC8050
+
+#define mmTPC7_QM_GLBL_MSG_EN_0                                      0xFC8054
+
+#define mmTPC7_QM_GLBL_MSG_EN_1                                      0xFC8058
+
+#define mmTPC7_QM_GLBL_MSG_EN_2                                      0xFC805C
+
+#define mmTPC7_QM_GLBL_MSG_EN_3                                      0xFC8060
+
+#define mmTPC7_QM_GLBL_MSG_EN_4                                      0xFC8068
+
+#define mmTPC7_QM_PQ_BASE_LO_0                                       0xFC8070
+
+#define mmTPC7_QM_PQ_BASE_LO_1                                       0xFC8074
+
+#define mmTPC7_QM_PQ_BASE_LO_2                                       0xFC8078
+
+#define mmTPC7_QM_PQ_BASE_LO_3                                       0xFC807C
+
+#define mmTPC7_QM_PQ_BASE_HI_0                                       0xFC8080
+
+#define mmTPC7_QM_PQ_BASE_HI_1                                       0xFC8084
+
+#define mmTPC7_QM_PQ_BASE_HI_2                                       0xFC8088
+
+#define mmTPC7_QM_PQ_BASE_HI_3                                       0xFC808C
+
+#define mmTPC7_QM_PQ_SIZE_0                                          0xFC8090
+
+#define mmTPC7_QM_PQ_SIZE_1                                          0xFC8094
+
+#define mmTPC7_QM_PQ_SIZE_2                                          0xFC8098
+
+#define mmTPC7_QM_PQ_SIZE_3                                          0xFC809C
+
+#define mmTPC7_QM_PQ_PI_0                                            0xFC80A0
+
+#define mmTPC7_QM_PQ_PI_1                                            0xFC80A4
+
+#define mmTPC7_QM_PQ_PI_2                                            0xFC80A8
+
+#define mmTPC7_QM_PQ_PI_3                                            0xFC80AC
+
+#define mmTPC7_QM_PQ_CI_0                                            0xFC80B0
+
+#define mmTPC7_QM_PQ_CI_1                                            0xFC80B4
+
+#define mmTPC7_QM_PQ_CI_2                                            0xFC80B8
+
+#define mmTPC7_QM_PQ_CI_3                                            0xFC80BC
+
+#define mmTPC7_QM_PQ_CFG0_0                                          0xFC80C0
+
+#define mmTPC7_QM_PQ_CFG0_1                                          0xFC80C4
+
+#define mmTPC7_QM_PQ_CFG0_2                                          0xFC80C8
+
+#define mmTPC7_QM_PQ_CFG0_3                                          0xFC80CC
+
+#define mmTPC7_QM_PQ_CFG1_0                                          0xFC80D0
+
+#define mmTPC7_QM_PQ_CFG1_1                                          0xFC80D4
+
+#define mmTPC7_QM_PQ_CFG1_2                                          0xFC80D8
+
+#define mmTPC7_QM_PQ_CFG1_3                                          0xFC80DC
+
+#define mmTPC7_QM_PQ_ARUSER_31_11_0                                  0xFC80E0
+
+#define mmTPC7_QM_PQ_ARUSER_31_11_1                                  0xFC80E4
+
+#define mmTPC7_QM_PQ_ARUSER_31_11_2                                  0xFC80E8
+
+#define mmTPC7_QM_PQ_ARUSER_31_11_3                                  0xFC80EC
+
+#define mmTPC7_QM_PQ_STS0_0                                          0xFC80F0
+
+#define mmTPC7_QM_PQ_STS0_1                                          0xFC80F4
+
+#define mmTPC7_QM_PQ_STS0_2                                          0xFC80F8
+
+#define mmTPC7_QM_PQ_STS0_3                                          0xFC80FC
+
+#define mmTPC7_QM_PQ_STS1_0                                          0xFC8100
+
+#define mmTPC7_QM_PQ_STS1_1                                          0xFC8104
+
+#define mmTPC7_QM_PQ_STS1_2                                          0xFC8108
+
+#define mmTPC7_QM_PQ_STS1_3                                          0xFC810C
+
+#define mmTPC7_QM_CQ_CFG0_0                                          0xFC8110
+
+#define mmTPC7_QM_CQ_CFG0_1                                          0xFC8114
+
+#define mmTPC7_QM_CQ_CFG0_2                                          0xFC8118
+
+#define mmTPC7_QM_CQ_CFG0_3                                          0xFC811C
+
+#define mmTPC7_QM_CQ_CFG0_4                                          0xFC8120
+
+#define mmTPC7_QM_CQ_CFG1_0                                          0xFC8124
+
+#define mmTPC7_QM_CQ_CFG1_1                                          0xFC8128
+
+#define mmTPC7_QM_CQ_CFG1_2                                          0xFC812C
+
+#define mmTPC7_QM_CQ_CFG1_3                                          0xFC8130
+
+#define mmTPC7_QM_CQ_CFG1_4                                          0xFC8134
+
+#define mmTPC7_QM_CQ_ARUSER_31_11_0                                  0xFC8138
+
+#define mmTPC7_QM_CQ_ARUSER_31_11_1                                  0xFC813C
+
+#define mmTPC7_QM_CQ_ARUSER_31_11_2                                  0xFC8140
+
+#define mmTPC7_QM_CQ_ARUSER_31_11_3                                  0xFC8144
+
+#define mmTPC7_QM_CQ_ARUSER_31_11_4                                  0xFC8148
+
+#define mmTPC7_QM_CQ_STS0_0                                          0xFC814C
+
+#define mmTPC7_QM_CQ_STS0_1                                          0xFC8150
+
+#define mmTPC7_QM_CQ_STS0_2                                          0xFC8154
+
+#define mmTPC7_QM_CQ_STS0_3                                          0xFC8158
+
+#define mmTPC7_QM_CQ_STS0_4                                          0xFC815C
+
+#define mmTPC7_QM_CQ_STS1_0                                          0xFC8160
+
+#define mmTPC7_QM_CQ_STS1_1                                          0xFC8164
+
+#define mmTPC7_QM_CQ_STS1_2                                          0xFC8168
+
+#define mmTPC7_QM_CQ_STS1_3                                          0xFC816C
+
+#define mmTPC7_QM_CQ_STS1_4                                          0xFC8170
+
+#define mmTPC7_QM_CQ_PTR_LO_0                                        0xFC8174
+
+#define mmTPC7_QM_CQ_PTR_HI_0                                        0xFC8178
+
+#define mmTPC7_QM_CQ_TSIZE_0                                         0xFC817C
+
+#define mmTPC7_QM_CQ_CTL_0                                           0xFC8180
+
+#define mmTPC7_QM_CQ_PTR_LO_1                                        0xFC8184
+
+#define mmTPC7_QM_CQ_PTR_HI_1                                        0xFC8188
+
+#define mmTPC7_QM_CQ_TSIZE_1                                         0xFC818C
+
+#define mmTPC7_QM_CQ_CTL_1                                           0xFC8190
+
+#define mmTPC7_QM_CQ_PTR_LO_2                                        0xFC8194
+
+#define mmTPC7_QM_CQ_PTR_HI_2                                        0xFC8198
+
+#define mmTPC7_QM_CQ_TSIZE_2                                         0xFC819C
+
+#define mmTPC7_QM_CQ_CTL_2                                           0xFC81A0
+
+#define mmTPC7_QM_CQ_PTR_LO_3                                        0xFC81A4
+
+#define mmTPC7_QM_CQ_PTR_HI_3                                        0xFC81A8
+
+#define mmTPC7_QM_CQ_TSIZE_3                                         0xFC81AC
+
+#define mmTPC7_QM_CQ_CTL_3                                           0xFC81B0
+
+#define mmTPC7_QM_CQ_PTR_LO_4                                        0xFC81B4
+
+#define mmTPC7_QM_CQ_PTR_HI_4                                        0xFC81B8
+
+#define mmTPC7_QM_CQ_TSIZE_4                                         0xFC81BC
+
+#define mmTPC7_QM_CQ_CTL_4                                           0xFC81C0
+
+#define mmTPC7_QM_CQ_PTR_LO_STS_0                                    0xFC81C4
+
+#define mmTPC7_QM_CQ_PTR_LO_STS_1                                    0xFC81C8
+
+#define mmTPC7_QM_CQ_PTR_LO_STS_2                                    0xFC81CC
+
+#define mmTPC7_QM_CQ_PTR_LO_STS_3                                    0xFC81D0
+
+#define mmTPC7_QM_CQ_PTR_LO_STS_4                                    0xFC81D4
+
+#define mmTPC7_QM_CQ_PTR_HI_STS_0                                    0xFC81D8
+
+#define mmTPC7_QM_CQ_PTR_HI_STS_1                                    0xFC81DC
+
+#define mmTPC7_QM_CQ_PTR_HI_STS_2                                    0xFC81E0
+
+#define mmTPC7_QM_CQ_PTR_HI_STS_3                                    0xFC81E4
+
+#define mmTPC7_QM_CQ_PTR_HI_STS_4                                    0xFC81E8
+
+#define mmTPC7_QM_CQ_TSIZE_STS_0                                     0xFC81EC
+
+#define mmTPC7_QM_CQ_TSIZE_STS_1                                     0xFC81F0
+
+#define mmTPC7_QM_CQ_TSIZE_STS_2                                     0xFC81F4
+
+#define mmTPC7_QM_CQ_TSIZE_STS_3                                     0xFC81F8
+
+#define mmTPC7_QM_CQ_TSIZE_STS_4                                     0xFC81FC
+
+#define mmTPC7_QM_CQ_CTL_STS_0                                       0xFC8200
+
+#define mmTPC7_QM_CQ_CTL_STS_1                                       0xFC8204
+
+#define mmTPC7_QM_CQ_CTL_STS_2                                       0xFC8208
+
+#define mmTPC7_QM_CQ_CTL_STS_3                                       0xFC820C
+
+#define mmTPC7_QM_CQ_CTL_STS_4                                       0xFC8210
+
+#define mmTPC7_QM_CQ_IFIFO_CNT_0                                     0xFC8214
+
+#define mmTPC7_QM_CQ_IFIFO_CNT_1                                     0xFC8218
+
+#define mmTPC7_QM_CQ_IFIFO_CNT_2                                     0xFC821C
+
+#define mmTPC7_QM_CQ_IFIFO_CNT_3                                     0xFC8220
+
+#define mmTPC7_QM_CQ_IFIFO_CNT_4                                     0xFC8224
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_0                             0xFC8228
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_1                             0xFC822C
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_2                             0xFC8230
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_3                             0xFC8234
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_LO_4                             0xFC8238
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_0                             0xFC823C
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_1                             0xFC8240
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_2                             0xFC8244
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_3                             0xFC8248
+
+#define mmTPC7_QM_CP_MSG_BASE0_ADDR_HI_4                             0xFC824C
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_0                             0xFC8250
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_1                             0xFC8254
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_2                             0xFC8258
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_3                             0xFC825C
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_LO_4                             0xFC8260
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_0                             0xFC8264
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_1                             0xFC8268
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_2                             0xFC826C
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_3                             0xFC8270
+
+#define mmTPC7_QM_CP_MSG_BASE1_ADDR_HI_4                             0xFC8274
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_0                             0xFC8278
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_1                             0xFC827C
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_2                             0xFC8280
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_3                             0xFC8284
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_LO_4                             0xFC8288
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_0                             0xFC828C
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_1                             0xFC8290
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_2                             0xFC8294
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_3                             0xFC8298
+
+#define mmTPC7_QM_CP_MSG_BASE2_ADDR_HI_4                             0xFC829C
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_0                             0xFC82A0
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_1                             0xFC82A4
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_2                             0xFC82A8
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_3                             0xFC82AC
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_LO_4                             0xFC82B0
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_0                             0xFC82B4
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_1                             0xFC82B8
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_2                             0xFC82BC
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_3                             0xFC82C0
+
+#define mmTPC7_QM_CP_MSG_BASE3_ADDR_HI_4                             0xFC82C4
+
+#define mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_0                             0xFC82C8
+
+#define mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_1                             0xFC82CC
+
+#define mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_2                             0xFC82D0
+
+#define mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_3                             0xFC82D4
+
+#define mmTPC7_QM_CP_LDMA_TSIZE_OFFSET_4                             0xFC82D8
+
+#define mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0                       0xFC82E0
+
+#define mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_1                       0xFC82E4
+
+#define mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_2                       0xFC82E8
+
+#define mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_3                       0xFC82EC
+
+#define mmTPC7_QM_CP_LDMA_SRC_BASE_LO_OFFSET_4                       0xFC82F0
+
+#define mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_0                       0xFC82F4
+
+#define mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_1                       0xFC82F8
+
+#define mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_2                       0xFC82FC
+
+#define mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_3                       0xFC8300
+
+#define mmTPC7_QM_CP_LDMA_DST_BASE_LO_OFFSET_4                       0xFC8304
+
+#define mmTPC7_QM_CP_FENCE0_RDATA_0                                  0xFC8308
+
+#define mmTPC7_QM_CP_FENCE0_RDATA_1                                  0xFC830C
+
+#define mmTPC7_QM_CP_FENCE0_RDATA_2                                  0xFC8310
+
+#define mmTPC7_QM_CP_FENCE0_RDATA_3                                  0xFC8314
+
+#define mmTPC7_QM_CP_FENCE0_RDATA_4                                  0xFC8318
+
+#define mmTPC7_QM_CP_FENCE1_RDATA_0                                  0xFC831C
+
+#define mmTPC7_QM_CP_FENCE1_RDATA_1                                  0xFC8320
+
+#define mmTPC7_QM_CP_FENCE1_RDATA_2                                  0xFC8324
+
+#define mmTPC7_QM_CP_FENCE1_RDATA_3                                  0xFC8328
+
+#define mmTPC7_QM_CP_FENCE1_RDATA_4                                  0xFC832C
+
+#define mmTPC7_QM_CP_FENCE2_RDATA_0                                  0xFC8330
+
+#define mmTPC7_QM_CP_FENCE2_RDATA_1                                  0xFC8334
+
+#define mmTPC7_QM_CP_FENCE2_RDATA_2                                  0xFC8338
+
+#define mmTPC7_QM_CP_FENCE2_RDATA_3                                  0xFC833C
+
+#define mmTPC7_QM_CP_FENCE2_RDATA_4                                  0xFC8340
+
+#define mmTPC7_QM_CP_FENCE3_RDATA_0                                  0xFC8344
+
+#define mmTPC7_QM_CP_FENCE3_RDATA_1                                  0xFC8348
+
+#define mmTPC7_QM_CP_FENCE3_RDATA_2                                  0xFC834C
+
+#define mmTPC7_QM_CP_FENCE3_RDATA_3                                  0xFC8350
+
+#define mmTPC7_QM_CP_FENCE3_RDATA_4                                  0xFC8354
+
+#define mmTPC7_QM_CP_FENCE0_CNT_0                                    0xFC8358
+
+#define mmTPC7_QM_CP_FENCE0_CNT_1                                    0xFC835C
+
+#define mmTPC7_QM_CP_FENCE0_CNT_2                                    0xFC8360
+
+#define mmTPC7_QM_CP_FENCE0_CNT_3                                    0xFC8364
+
+#define mmTPC7_QM_CP_FENCE0_CNT_4                                    0xFC8368
+
+#define mmTPC7_QM_CP_FENCE1_CNT_0                                    0xFC836C
+
+#define mmTPC7_QM_CP_FENCE1_CNT_1                                    0xFC8370
+
+#define mmTPC7_QM_CP_FENCE1_CNT_2                                    0xFC8374
+
+#define mmTPC7_QM_CP_FENCE1_CNT_3                                    0xFC8378
+
+#define mmTPC7_QM_CP_FENCE1_CNT_4                                    0xFC837C
+
+#define mmTPC7_QM_CP_FENCE2_CNT_0                                    0xFC8380
+
+#define mmTPC7_QM_CP_FENCE2_CNT_1                                    0xFC8384
+
+#define mmTPC7_QM_CP_FENCE2_CNT_2                                    0xFC8388
+
+#define mmTPC7_QM_CP_FENCE2_CNT_3                                    0xFC838C
+
+#define mmTPC7_QM_CP_FENCE2_CNT_4                                    0xFC8390
+
+#define mmTPC7_QM_CP_FENCE3_CNT_0                                    0xFC8394
+
+#define mmTPC7_QM_CP_FENCE3_CNT_1                                    0xFC8398
+
+#define mmTPC7_QM_CP_FENCE3_CNT_2                                    0xFC839C
+
+#define mmTPC7_QM_CP_FENCE3_CNT_3                                    0xFC83A0
+
+#define mmTPC7_QM_CP_FENCE3_CNT_4                                    0xFC83A4
+
+#define mmTPC7_QM_CP_STS_0                                           0xFC83A8
+
+#define mmTPC7_QM_CP_STS_1                                           0xFC83AC
+
+#define mmTPC7_QM_CP_STS_2                                           0xFC83B0
+
+#define mmTPC7_QM_CP_STS_3                                           0xFC83B4
+
+#define mmTPC7_QM_CP_STS_4                                           0xFC83B8
+
+#define mmTPC7_QM_CP_CURRENT_INST_LO_0                               0xFC83BC
+
+#define mmTPC7_QM_CP_CURRENT_INST_LO_1                               0xFC83C0
+
+#define mmTPC7_QM_CP_CURRENT_INST_LO_2                               0xFC83C4
+
+#define mmTPC7_QM_CP_CURRENT_INST_LO_3                               0xFC83C8
+
+#define mmTPC7_QM_CP_CURRENT_INST_LO_4                               0xFC83CC
+
+#define mmTPC7_QM_CP_CURRENT_INST_HI_0                               0xFC83D0
+
+#define mmTPC7_QM_CP_CURRENT_INST_HI_1                               0xFC83D4
+
+#define mmTPC7_QM_CP_CURRENT_INST_HI_2                               0xFC83D8
+
+#define mmTPC7_QM_CP_CURRENT_INST_HI_3                               0xFC83DC
+
+#define mmTPC7_QM_CP_CURRENT_INST_HI_4                               0xFC83E0
+
+#define mmTPC7_QM_CP_BARRIER_CFG_0                                   0xFC83F4
+
+#define mmTPC7_QM_CP_BARRIER_CFG_1                                   0xFC83F8
+
+#define mmTPC7_QM_CP_BARRIER_CFG_2                                   0xFC83FC
+
+#define mmTPC7_QM_CP_BARRIER_CFG_3                                   0xFC8400
+
+#define mmTPC7_QM_CP_BARRIER_CFG_4                                   0xFC8404
+
+#define mmTPC7_QM_CP_DBG_0_0                                         0xFC8408
+
+#define mmTPC7_QM_CP_DBG_0_1                                         0xFC840C
+
+#define mmTPC7_QM_CP_DBG_0_2                                         0xFC8410
+
+#define mmTPC7_QM_CP_DBG_0_3                                         0xFC8414
+
+#define mmTPC7_QM_CP_DBG_0_4                                         0xFC8418
+
+#define mmTPC7_QM_CP_ARUSER_31_11_0                                  0xFC841C
+
+#define mmTPC7_QM_CP_ARUSER_31_11_1                                  0xFC8420
+
+#define mmTPC7_QM_CP_ARUSER_31_11_2                                  0xFC8424
+
+#define mmTPC7_QM_CP_ARUSER_31_11_3                                  0xFC8428
+
+#define mmTPC7_QM_CP_ARUSER_31_11_4                                  0xFC842C
+
+#define mmTPC7_QM_CP_AWUSER_31_11_0                                  0xFC8430
+
+#define mmTPC7_QM_CP_AWUSER_31_11_1                                  0xFC8434
+
+#define mmTPC7_QM_CP_AWUSER_31_11_2                                  0xFC8438
+
+#define mmTPC7_QM_CP_AWUSER_31_11_3                                  0xFC843C
+
+#define mmTPC7_QM_CP_AWUSER_31_11_4                                  0xFC8440
+
+#define mmTPC7_QM_ARB_CFG_0                                          0xFC8A00
+
+#define mmTPC7_QM_ARB_CHOISE_Q_PUSH                                  0xFC8A04
+
+#define mmTPC7_QM_ARB_WRR_WEIGHT_0                                   0xFC8A08
+
+#define mmTPC7_QM_ARB_WRR_WEIGHT_1                                   0xFC8A0C
+
+#define mmTPC7_QM_ARB_WRR_WEIGHT_2                                   0xFC8A10
+
+#define mmTPC7_QM_ARB_WRR_WEIGHT_3                                   0xFC8A14
+
+#define mmTPC7_QM_ARB_CFG_1                                          0xFC8A18
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_0                               0xFC8A20
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_1                               0xFC8A24
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_2                               0xFC8A28
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_3                               0xFC8A2C
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_4                               0xFC8A30
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_5                               0xFC8A34
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_6                               0xFC8A38
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_7                               0xFC8A3C
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_8                               0xFC8A40
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_9                               0xFC8A44
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_10                              0xFC8A48
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_11                              0xFC8A4C
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_12                              0xFC8A50
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_13                              0xFC8A54
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_14                              0xFC8A58
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_15                              0xFC8A5C
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_16                              0xFC8A60
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_17                              0xFC8A64
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_18                              0xFC8A68
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_19                              0xFC8A6C
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_20                              0xFC8A70
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_21                              0xFC8A74
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_22                              0xFC8A78
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_23                              0xFC8A7C
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_24                              0xFC8A80
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_25                              0xFC8A84
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_26                              0xFC8A88
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_27                              0xFC8A8C
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_28                              0xFC8A90
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_29                              0xFC8A94
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_30                              0xFC8A98
+
+#define mmTPC7_QM_ARB_MST_AVAIL_CRED_31                              0xFC8A9C
+
+#define mmTPC7_QM_ARB_MST_CRED_INC                                   0xFC8AA0
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_0                         0xFC8AA4
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_1                         0xFC8AA8
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_2                         0xFC8AAC
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_3                         0xFC8AB0
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_4                         0xFC8AB4
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_5                         0xFC8AB8
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_6                         0xFC8ABC
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_7                         0xFC8AC0
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_8                         0xFC8AC4
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_9                         0xFC8AC8
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_10                        0xFC8ACC
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_11                        0xFC8AD0
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_12                        0xFC8AD4
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_13                        0xFC8AD8
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_14                        0xFC8ADC
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_15                        0xFC8AE0
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_16                        0xFC8AE4
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_17                        0xFC8AE8
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_18                        0xFC8AEC
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_19                        0xFC8AF0
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_20                        0xFC8AF4
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_21                        0xFC8AF8
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_22                        0xFC8AFC
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_23                        0xFC8B00
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_24                        0xFC8B04
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_25                        0xFC8B08
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_26                        0xFC8B0C
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_27                        0xFC8B10
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_28                        0xFC8B14
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_29                        0xFC8B18
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_30                        0xFC8B1C
+
+#define mmTPC7_QM_ARB_MST_CHOISE_PUSH_OFST_31                        0xFC8B20
+
+#define mmTPC7_QM_ARB_SLV_MASTER_INC_CRED_OFST                       0xFC8B28
+
+#define mmTPC7_QM_ARB_MST_SLAVE_EN                                   0xFC8B2C
+
+#define mmTPC7_QM_ARB_MST_QUIET_PER                                  0xFC8B34
+
+#define mmTPC7_QM_ARB_SLV_CHOISE_WDT                                 0xFC8B38
+
+#define mmTPC7_QM_ARB_SLV_ID                                         0xFC8B3C
+
+#define mmTPC7_QM_ARB_MSG_MAX_INFLIGHT                               0xFC8B44
+
+#define mmTPC7_QM_ARB_MSG_AWUSER_31_11                               0xFC8B48
+
+#define mmTPC7_QM_ARB_MSG_AWUSER_SEC_PROP                            0xFC8B4C
+
+#define mmTPC7_QM_ARB_MSG_AWUSER_NON_SEC_PROP                        0xFC8B50
+
+#define mmTPC7_QM_ARB_BASE_LO                                        0xFC8B54
+
+#define mmTPC7_QM_ARB_BASE_HI                                        0xFC8B58
+
+#define mmTPC7_QM_ARB_STATE_STS                                      0xFC8B80
+
+#define mmTPC7_QM_ARB_CHOISE_FULLNESS_STS                            0xFC8B84
+
+#define mmTPC7_QM_ARB_MSG_STS                                        0xFC8B88
+
+#define mmTPC7_QM_ARB_SLV_CHOISE_Q_HEAD                              0xFC8B8C
+
+#define mmTPC7_QM_ARB_ERR_CAUSE                                      0xFC8B9C
+
+#define mmTPC7_QM_ARB_ERR_MSG_EN                                     0xFC8BA0
+
+#define mmTPC7_QM_ARB_ERR_STS_DRP                                    0xFC8BA8
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_0                                 0xFC8BB0
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_1                                 0xFC8BB4
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_2                                 0xFC8BB8
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_3                                 0xFC8BBC
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_4                                 0xFC8BC0
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_5                                 0xFC8BC4
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_6                                 0xFC8BC8
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_7                                 0xFC8BCC
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_8                                 0xFC8BD0
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_9                                 0xFC8BD4
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_10                                0xFC8BD8
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_11                                0xFC8BDC
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_12                                0xFC8BE0
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_13                                0xFC8BE4
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_14                                0xFC8BE8
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_15                                0xFC8BEC
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_16                                0xFC8BF0
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_17                                0xFC8BF4
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_18                                0xFC8BF8
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_19                                0xFC8BFC
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_20                                0xFC8C00
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_21                                0xFC8C04
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_22                                0xFC8C08
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_23                                0xFC8C0C
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_24                                0xFC8C10
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_25                                0xFC8C14
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_26                                0xFC8C18
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_27                                0xFC8C1C
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_28                                0xFC8C20
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_29                                0xFC8C24
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_30                                0xFC8C28
+
+#define mmTPC7_QM_ARB_MST_CRED_STS_31                                0xFC8C2C
+
+#define mmTPC7_QM_CGM_CFG                                            0xFC8C70
+
+#define mmTPC7_QM_CGM_STS                                            0xFC8C74
+
+#define mmTPC7_QM_CGM_CFG1                                           0xFC8C78
+
+#define mmTPC7_QM_LOCAL_RANGE_BASE                                   0xFC8C80
+
+#define mmTPC7_QM_LOCAL_RANGE_SIZE                                   0xFC8C84
+
+#define mmTPC7_QM_CSMR_STRICT_PRIO_CFG                               0xFC8C90
+
+#define mmTPC7_QM_HBW_RD_RATE_LIM_CFG_1                              0xFC8C94
+
+#define mmTPC7_QM_LBW_WR_RATE_LIM_CFG_0                              0xFC8C98
+
+#define mmTPC7_QM_LBW_WR_RATE_LIM_CFG_1                              0xFC8C9C
+
+#define mmTPC7_QM_HBW_RD_RATE_LIM_CFG_0                              0xFC8CA0
+
+#define mmTPC7_QM_GLBL_AXCACHE                                       0xFC8CA4
+
+#define mmTPC7_QM_IND_GW_APB_CFG                                     0xFC8CB0
+
+#define mmTPC7_QM_IND_GW_APB_WDATA                                   0xFC8CB4
+
+#define mmTPC7_QM_IND_GW_APB_RDATA                                   0xFC8CB8
+
+#define mmTPC7_QM_IND_GW_APB_STATUS                                  0xFC8CBC
+
+#define mmTPC7_QM_GLBL_ERR_ADDR_LO                                   0xFC8CD0
+
+#define mmTPC7_QM_GLBL_ERR_ADDR_HI                                   0xFC8CD4
+
+#define mmTPC7_QM_GLBL_ERR_WDATA                                     0xFC8CD8
+
+#define mmTPC7_QM_GLBL_MEM_INIT_BUSY                                 0xFC8D00
+
+#endif /* ASIC_REG_TPC7_QM_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi.h b/drivers/misc/habanalabs/include/gaudi/gaudi.h
new file mode 100644
index 000000000000..8829891d3eef
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2018-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GAUDI_H
+#define GAUDI_H
+
+#define SRAM_BAR_ID		0
+#define CFG_BAR_ID		2
+#define HBM_BAR_ID		4
+
+#define SRAM_BAR_SIZE		0x4000000ull		/* 64MB */
+#define CFG_BAR_SIZE		0x8000000ull		/* 128MB */
+
+#define CFG_BASE		0x7FFC000000ull
+#define CFG_SIZE		0x4000000		/* 32MB CFG + 32MB DBG*/
+
+#define SRAM_BASE_ADDR		0x7FF0000000ull
+#define SRAM_SIZE		0x1400000		/* 20MB */
+
+#define SPI_FLASH_BASE_ADDR	0x7FF8000000ull
+
+#define PSOC_SCRATCHPAD_ADDR	0x7FFBFE0000ull
+#define PSOC_SCRATCHPAD_SIZE	0x10000			/* 64KB */
+
+#define PCIE_FW_SRAM_ADDR	0x7FFBFF0000ull
+#define PCIE_FW_SRAM_SIZE	0x8000			/* 32KB */
+
+#define DRAM_PHYS_BASE		0x0ull
+
+#define HOST_PHYS_BASE		0x8000000000ull		/* 0.5TB */
+#define HOST_PHYS_SIZE		0x1000000000000ull	/* 0.25PB (48 bits) */
+
+#define GAUDI_MSI_ENTRIES	32
+
+#define QMAN_PQ_ENTRY_SIZE	16			/* Bytes */
+
+#define MAX_ASID		1024
+
+#define PROT_BITS_OFFS		0xF80
+
+#define MME_NUMBER_OF_MASTER_ENGINES	2
+
+#define TPC_NUMBER_OF_ENGINES	8
+
+#define DMA_NUMBER_OF_CHANNELS	8
+
+#define NIC_NUMBER_OF_MACROS	5
+
+#define NIC_NUMBER_OF_ENGINES	(NIC_NUMBER_OF_MACROS * 2)
+
+#define NUMBER_OF_IF		8
+
+#define DEVICE_CACHE_LINE_SIZE	128
+
+#endif /* GAUDI_H */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h
new file mode 100644
index 000000000000..9ccba8437ec9
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2018-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef __GAUDI_ASYNC_EVENTS_H_
+#define __GAUDI_ASYNC_EVENTS_H_
+
+enum gaudi_async_event_id {
+	GAUDI_EVENT_PCIE_CORE_SERR = 32,
+	GAUDI_EVENT_PCIE_CORE_DERR = 33,
+	GAUDI_EVENT_PCIE_IF_SERR = 34,
+	GAUDI_EVENT_PCIE_IF_DERR = 35,
+	GAUDI_EVENT_PCIE_PHY_SERR = 36,
+	GAUDI_EVENT_PCIE_PHY_DERR = 37,
+	GAUDI_EVENT_TPC0_SERR = 38,
+	GAUDI_EVENT_TPC1_SERR = 39,
+	GAUDI_EVENT_TPC2_SERR = 40,
+	GAUDI_EVENT_TPC3_SERR = 41,
+	GAUDI_EVENT_TPC4_SERR = 42,
+	GAUDI_EVENT_TPC5_SERR = 43,
+	GAUDI_EVENT_TPC6_SERR = 44,
+	GAUDI_EVENT_TPC7_SERR = 45,
+	GAUDI_EVENT_TPC0_DERR = 46,
+	GAUDI_EVENT_TPC1_DERR = 47,
+	GAUDI_EVENT_TPC2_DERR = 48,
+	GAUDI_EVENT_TPC3_DERR = 49,
+	GAUDI_EVENT_TPC4_DERR = 50,
+	GAUDI_EVENT_TPC5_DERR = 51,
+	GAUDI_EVENT_TPC6_DERR = 52,
+	GAUDI_EVENT_TPC7_DERR = 53,
+	GAUDI_EVENT_MME0_ACC_SERR = 54,
+	GAUDI_EVENT_MME0_ACC_DERR = 55,
+	GAUDI_EVENT_MME0_SBAB_SERR = 56,
+	GAUDI_EVENT_MME0_SBAB_DERR = 57,
+	GAUDI_EVENT_MME1_ACC_SERR = 58,
+	GAUDI_EVENT_MME1_ACC_DERR = 59,
+	GAUDI_EVENT_MME1_SBAB_SERR = 60,
+	GAUDI_EVENT_MME1_SBAB_DERR = 61,
+	GAUDI_EVENT_MME2_ACC_SERR = 62,
+	GAUDI_EVENT_MME2_ACC_DERR = 63,
+	GAUDI_EVENT_MME2_SBAB_SERR = 64,
+	GAUDI_EVENT_MME2_SBAB_DERR = 65,
+	GAUDI_EVENT_MME3_ACC_SERR = 66,
+	GAUDI_EVENT_MME3_ACC_DERR = 67,
+	GAUDI_EVENT_MME3_SBAB_SERR = 68,
+	GAUDI_EVENT_MME3_SBAB_DERR = 69,
+	GAUDI_EVENT_DMA0_SERR_ECC = 70,
+	GAUDI_EVENT_DMA1_SERR_ECC = 71,
+	GAUDI_EVENT_DMA2_SERR_ECC = 72,
+	GAUDI_EVENT_DMA3_SERR_ECC = 73,
+	GAUDI_EVENT_DMA4_SERR_ECC = 74,
+	GAUDI_EVENT_DMA5_SERR_ECC = 75,
+	GAUDI_EVENT_DMA6_SERR_ECC = 76,
+	GAUDI_EVENT_DMA7_SERR_ECC = 77,
+	GAUDI_EVENT_DMA0_DERR_ECC = 78,
+	GAUDI_EVENT_DMA1_DERR_ECC = 79,
+	GAUDI_EVENT_DMA2_DERR_ECC = 80,
+	GAUDI_EVENT_DMA3_DERR_ECC = 81,
+	GAUDI_EVENT_DMA4_DERR_ECC = 82,
+	GAUDI_EVENT_DMA5_DERR_ECC = 83,
+	GAUDI_EVENT_DMA6_DERR_ECC = 84,
+	GAUDI_EVENT_DMA7_DERR_ECC = 85,
+	GAUDI_EVENT_CPU_IF_ECC_SERR = 86,
+	GAUDI_EVENT_CPU_IF_ECC_DERR = 87,
+	GAUDI_EVENT_PSOC_MEM_SERR = 88,
+	GAUDI_EVENT_PSOC_CORESIGHT_SERR = 89,
+	GAUDI_EVENT_PSOC_MEM_DERR = 90,
+	GAUDI_EVENT_PSOC_CORESIGHT_DERR = 91,
+	GAUDI_EVENT_SRAM0_SERR = 92,
+	GAUDI_EVENT_SRAM1_SERR = 93,
+	GAUDI_EVENT_SRAM2_SERR = 94,
+	GAUDI_EVENT_SRAM3_SERR = 95,
+	GAUDI_EVENT_SRAM7_SERR = 96,
+	GAUDI_EVENT_SRAM6_SERR = 97,
+	GAUDI_EVENT_SRAM5_SERR = 98,
+	GAUDI_EVENT_SRAM4_SERR = 99,
+	GAUDI_EVENT_SRAM8_SERR = 100,
+	GAUDI_EVENT_SRAM9_SERR = 101,
+	GAUDI_EVENT_SRAM10_SERR = 102,
+	GAUDI_EVENT_SRAM11_SERR = 103,
+	GAUDI_EVENT_SRAM15_SERR = 104,
+	GAUDI_EVENT_SRAM14_SERR = 105,
+	GAUDI_EVENT_SRAM13_SERR = 106,
+	GAUDI_EVENT_SRAM12_SERR = 107,
+	GAUDI_EVENT_SRAM16_SERR = 108,
+	GAUDI_EVENT_SRAM17_SERR = 109,
+	GAUDI_EVENT_SRAM18_SERR = 110,
+	GAUDI_EVENT_SRAM19_SERR = 111,
+	GAUDI_EVENT_SRAM23_SERR = 112,
+	GAUDI_EVENT_SRAM22_SERR = 113,
+	GAUDI_EVENT_SRAM21_SERR = 114,
+	GAUDI_EVENT_SRAM20_SERR = 115,
+	GAUDI_EVENT_SRAM24_SERR = 116,
+	GAUDI_EVENT_SRAM25_SERR = 117,
+	GAUDI_EVENT_SRAM26_SERR = 118,
+	GAUDI_EVENT_SRAM27_SERR = 119,
+	GAUDI_EVENT_SRAM31_SERR = 120,
+	GAUDI_EVENT_SRAM30_SERR = 121,
+	GAUDI_EVENT_SRAM29_SERR = 122,
+	GAUDI_EVENT_SRAM28_SERR = 123,
+	GAUDI_EVENT_SRAM0_DERR = 124,
+	GAUDI_EVENT_SRAM1_DERR = 125,
+	GAUDI_EVENT_SRAM2_DERR = 126,
+	GAUDI_EVENT_SRAM3_DERR = 127,
+	GAUDI_EVENT_SRAM7_DERR = 128,
+	GAUDI_EVENT_SRAM6_DERR = 129,
+	GAUDI_EVENT_SRAM5_DERR = 130,
+	GAUDI_EVENT_SRAM4_DERR = 131,
+	GAUDI_EVENT_SRAM8_DERR = 132,
+	GAUDI_EVENT_SRAM9_DERR = 133,
+	GAUDI_EVENT_SRAM10_DERR = 134,
+	GAUDI_EVENT_SRAM11_DERR = 135,
+	GAUDI_EVENT_SRAM15_DERR = 136,
+	GAUDI_EVENT_SRAM14_DERR = 137,
+	GAUDI_EVENT_SRAM13_DERR = 138,
+	GAUDI_EVENT_SRAM12_DERR = 139,
+	GAUDI_EVENT_SRAM16_DERR = 140,
+	GAUDI_EVENT_SRAM17_DERR = 141,
+	GAUDI_EVENT_SRAM18_DERR = 142,
+	GAUDI_EVENT_SRAM19_DERR = 143,
+	GAUDI_EVENT_SRAM23_DERR = 144,
+	GAUDI_EVENT_SRAM22_DERR = 145,
+	GAUDI_EVENT_SRAM21_DERR = 146,
+	GAUDI_EVENT_SRAM20_DERR = 147,
+	GAUDI_EVENT_SRAM24_DERR = 148,
+	GAUDI_EVENT_SRAM25_DERR = 149,
+	GAUDI_EVENT_SRAM26_DERR = 150,
+	GAUDI_EVENT_SRAM27_DERR = 151,
+	GAUDI_EVENT_SRAM31_DERR = 152,
+	GAUDI_EVENT_SRAM30_DERR = 153,
+	GAUDI_EVENT_SRAM29_DERR = 154,
+	GAUDI_EVENT_SRAM28_DERR = 155,
+	GAUDI_EVENT_NIC0_SERR = 156,
+	GAUDI_EVENT_NIC1_SERR = 157,
+	GAUDI_EVENT_NIC2_SERR = 158,
+	GAUDI_EVENT_NIC3_SERR = 159,
+	GAUDI_EVENT_NIC4_SERR = 160,
+	GAUDI_EVENT_NIC0_DERR = 166,
+	GAUDI_EVENT_NIC1_DERR = 167,
+	GAUDI_EVENT_NIC2_DERR = 168,
+	GAUDI_EVENT_NIC3_DERR = 169,
+	GAUDI_EVENT_NIC4_DERR = 170,
+	GAUDI_EVENT_DMA_IF0_SERR = 176,
+	GAUDI_EVENT_DMA_IF1_SERR = 177,
+	GAUDI_EVENT_DMA_IF2_SERR = 178,
+	GAUDI_EVENT_DMA_IF3_SERR = 179,
+	GAUDI_EVENT_DMA_IF0_DERR = 180,
+	GAUDI_EVENT_DMA_IF1_DERR = 181,
+	GAUDI_EVENT_DMA_IF2_DERR = 182,
+	GAUDI_EVENT_DMA_IF3_DERR = 183,
+	GAUDI_EVENT_GIC500 = 184,
+	GAUDI_EVENT_HBM_0_SERR = 185,
+	GAUDI_EVENT_HBM_1_SERR = 186,
+	GAUDI_EVENT_HBM_2_SERR = 187,
+	GAUDI_EVENT_HBM_3_SERR = 188,
+	GAUDI_EVENT_HBM_0_DERR = 189,
+	GAUDI_EVENT_HBM_1_DERR = 190,
+	GAUDI_EVENT_HBM_2_DERR = 191,
+	GAUDI_EVENT_HBM_3_DERR = 192,
+	GAUDI_EVENT_MMU_SERR = 193,
+	GAUDI_EVENT_MMU_DERR = 194,
+	GAUDI_EVENT_PCIE_DEC = 200,
+	GAUDI_EVENT_TPC0_DEC = 201,
+	GAUDI_EVENT_TPC1_DEC = 203,
+	GAUDI_EVENT_TPC2_DEC = 205,
+	GAUDI_EVENT_TPC3_DEC = 207,
+	GAUDI_EVENT_TPC4_DEC = 209,
+	GAUDI_EVENT_TPC5_DEC = 211,
+	GAUDI_EVENT_TPC6_DEC = 213,
+	GAUDI_EVENT_TPC7_DEC = 215,
+	GAUDI_EVENT_AXI_ECC = 217,
+	GAUDI_EVENT_L2_RAM_ECC = 218,
+	GAUDI_EVENT_MME0_WBC_RSP = 219,
+	GAUDI_EVENT_MME0_SBAB0_RSP = 220,
+	GAUDI_EVENT_MME1_WBC_RSP = 224,
+	GAUDI_EVENT_MME1_SBAB0_RSP = 225,
+	GAUDI_EVENT_MME2_WBC_RSP = 229,
+	GAUDI_EVENT_MME2_SBAB0_RSP = 230,
+	GAUDI_EVENT_MME3_WBC_RSP = 234,
+	GAUDI_EVENT_MME3_SBAB0_RSP = 235,
+	GAUDI_EVENT_PLL0 = 239,
+	GAUDI_EVENT_PLL1 = 240,
+	GAUDI_EVENT_PLL2 = 241,
+	GAUDI_EVENT_PLL3 = 242,
+	GAUDI_EVENT_PLL4 = 243,
+	GAUDI_EVENT_PLL5 = 244,
+	GAUDI_EVENT_PLL6 = 245,
+	GAUDI_EVENT_PLL7 = 246,
+	GAUDI_EVENT_PLL8 = 247,
+	GAUDI_EVENT_PLL9 = 248,
+	GAUDI_EVENT_PLL10 = 249,
+	GAUDI_EVENT_PLL11 = 250,
+	GAUDI_EVENT_PLL12 = 251,
+	GAUDI_EVENT_PLL13 = 252,
+	GAUDI_EVENT_PLL14 = 253,
+	GAUDI_EVENT_PLL15 = 254,
+	GAUDI_EVENT_PLL16 = 255,
+	GAUDI_EVENT_PLL17 = 256,
+	GAUDI_EVENT_CPU_AXI_SPLITTER = 257,
+	GAUDI_EVENT_PSOC_AXI_DEC = 262,
+	GAUDI_EVENT_PSOC_PRSTN_FALL = 263,
+	GAUDI_EVENT_NIC_SEI_0 = 264,
+	GAUDI_EVENT_NIC_SEI_1 = 265,
+	GAUDI_EVENT_NIC_SEI_2 = 266,
+	GAUDI_EVENT_NIC_SEI_3 = 267,
+	GAUDI_EVENT_NIC_SEI_4 = 268,
+	GAUDI_EVENT_PCIE_FLR = 290,
+	GAUDI_EVENT_TPC0_BMON_SPMU = 300,
+	GAUDI_EVENT_TPC0_KRN_ERR = 301,
+	GAUDI_EVENT_TPC1_BMON_SPMU = 306,
+	GAUDI_EVENT_TPC1_KRN_ERR = 307,
+	GAUDI_EVENT_TPC2_BMON_SPMU = 312,
+	GAUDI_EVENT_TPC2_KRN_ERR = 313,
+	GAUDI_EVENT_TPC3_BMON_SPMU = 318,
+	GAUDI_EVENT_TPC3_KRN_ERR = 319,
+	GAUDI_EVENT_TPC4_BMON_SPMU = 324,
+	GAUDI_EVENT_TPC4_KRN_ERR = 325,
+	GAUDI_EVENT_TPC5_BMON_SPMU = 330,
+	GAUDI_EVENT_TPC5_KRN_ERR = 331,
+	GAUDI_EVENT_TPC6_BMON_SPMU = 336,
+	GAUDI_EVENT_TPC6_KRN_ERR = 337,
+	GAUDI_EVENT_TPC7_BMON_SPMU = 342,
+	GAUDI_EVENT_TPC7_KRN_ERR = 343,
+	GAUDI_EVENT_MMU_PAGE_FAULT = 380,
+	GAUDI_EVENT_MMU_WR_PERM = 381,
+	GAUDI_EVENT_DMA_BM_CH0 = 383,
+	GAUDI_EVENT_DMA_BM_CH1 = 384,
+	GAUDI_EVENT_DMA_BM_CH2 = 385,
+	GAUDI_EVENT_DMA_BM_CH3 = 386,
+	GAUDI_EVENT_DMA_BM_CH4 = 387,
+	GAUDI_EVENT_DMA_BM_CH5 = 388,
+	GAUDI_EVENT_DMA_BM_CH6 = 389,
+	GAUDI_EVENT_DMA_BM_CH7 = 390,
+	GAUDI_EVENT_HBM0_SPI_0 = 395,
+	GAUDI_EVENT_HBM0_SPI_1 = 396,
+	GAUDI_EVENT_HBM1_SPI_0 = 399,
+	GAUDI_EVENT_HBM1_SPI_1 = 400,
+	GAUDI_EVENT_HBM2_SPI_0 = 403,
+	GAUDI_EVENT_HBM2_SPI_1 = 404,
+	GAUDI_EVENT_HBM3_SPI_0 = 407,
+	GAUDI_EVENT_HBM3_SPI_1 = 408,
+	GAUDI_EVENT_PSOC_GPIO_U16_0 = 421,
+	GAUDI_EVENT_PI_UPDATE = 484,
+	GAUDI_EVENT_HALT_MACHINE = 485,
+	GAUDI_EVENT_INTS_REGISTER = 486,
+	GAUDI_EVENT_SOFT_RESET = 487,
+	GAUDI_EVENT_RAZWI_OR_ADC = 548,
+	GAUDI_EVENT_TPC0_QM = 572,
+	GAUDI_EVENT_TPC1_QM = 573,
+	GAUDI_EVENT_TPC2_QM = 574,
+	GAUDI_EVENT_TPC3_QM = 575,
+	GAUDI_EVENT_TPC4_QM = 576,
+	GAUDI_EVENT_TPC5_QM = 577,
+	GAUDI_EVENT_TPC6_QM = 578,
+	GAUDI_EVENT_TPC7_QM = 579,
+	GAUDI_EVENT_MME0_QM = 581,
+	GAUDI_EVENT_MME2_QM = 582,
+	GAUDI_EVENT_DMA0_QM = 583,
+	GAUDI_EVENT_DMA1_QM = 584,
+	GAUDI_EVENT_DMA2_QM = 585,
+	GAUDI_EVENT_DMA3_QM = 586,
+	GAUDI_EVENT_DMA4_QM = 587,
+	GAUDI_EVENT_DMA5_QM = 588,
+	GAUDI_EVENT_DMA6_QM = 589,
+	GAUDI_EVENT_DMA7_QM = 590,
+	GAUDI_EVENT_NIC0_QM0 = 594,
+	GAUDI_EVENT_NIC0_QM1 = 595,
+	GAUDI_EVENT_NIC1_QM0 = 596,
+	GAUDI_EVENT_NIC1_QM1 = 597,
+	GAUDI_EVENT_NIC2_QM0 = 598,
+	GAUDI_EVENT_NIC2_QM1 = 599,
+	GAUDI_EVENT_NIC3_QM0 = 600,
+	GAUDI_EVENT_NIC3_QM1 = 601,
+	GAUDI_EVENT_NIC4_QM0 = 602,
+	GAUDI_EVENT_NIC4_QM1 = 603,
+	GAUDI_EVENT_DMA0_CORE = 604,
+	GAUDI_EVENT_DMA1_CORE = 605,
+	GAUDI_EVENT_DMA2_CORE = 606,
+	GAUDI_EVENT_DMA3_CORE = 607,
+	GAUDI_EVENT_DMA4_CORE = 608,
+	GAUDI_EVENT_DMA5_CORE = 609,
+	GAUDI_EVENT_DMA6_CORE = 610,
+	GAUDI_EVENT_DMA7_CORE = 611,
+	GAUDI_EVENT_NIC0_QP0 = 612,
+	GAUDI_EVENT_NIC0_QP1 = 613,
+	GAUDI_EVENT_NIC1_QP0 = 614,
+	GAUDI_EVENT_NIC1_QP1 = 615,
+	GAUDI_EVENT_NIC2_QP0 = 616,
+	GAUDI_EVENT_NIC2_QP1 = 617,
+	GAUDI_EVENT_NIC3_QP0 = 618,
+	GAUDI_EVENT_NIC3_QP1 = 619,
+	GAUDI_EVENT_NIC4_QP0 = 620,
+	GAUDI_EVENT_NIC4_QP1 = 621,
+	GAUDI_EVENT_FIX_POWER_ENV_S = 658,
+	GAUDI_EVENT_FIX_POWER_ENV_E = 659,
+	GAUDI_EVENT_FIX_THERMAL_ENV_S = 660,
+	GAUDI_EVENT_FIX_THERMAL_ENV_E = 661,
+	GAUDI_EVENT_RAZWI_OR_ADC_SW = 662,
+	GAUDI_EVENT_SIZE,
+};
+
+#endif /* __GAUDI_ASYNC_EVENTS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h
new file mode 100644
index 000000000000..737176ba06fb
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h
@@ -0,0 +1,694 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2018-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef __GAUDI_ASYNC_IDS_MAP_EVENTS_EXT_H_
+#define __GAUDI_ASYNC_IDS_MAP_EVENTS_EXT_H_
+
+struct gaudi_async_events_ids_map {
+	int fc_id;
+	int cpu_id;
+	int valid;
+	char name[64];
+};
+
+static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = {
+	{ .fc_id = 0, .cpu_id = 0, .valid = 0, .name = "" },
+	{ .fc_id = 1, .cpu_id = 1, .valid = 0, .name = "" },
+	{ .fc_id = 2, .cpu_id = 2, .valid = 0, .name = "" },
+	{ .fc_id = 3, .cpu_id = 3, .valid = 0, .name = "" },
+	{ .fc_id = 4, .cpu_id = 4, .valid = 0, .name = "" },
+	{ .fc_id = 5, .cpu_id = 5, .valid = 0, .name = "" },
+	{ .fc_id = 6, .cpu_id = 6, .valid = 0, .name = "" },
+	{ .fc_id = 7, .cpu_id = 7, .valid = 0, .name = "" },
+	{ .fc_id = 8, .cpu_id = 8, .valid = 0, .name = "" },
+	{ .fc_id = 9, .cpu_id = 9, .valid = 0, .name = "" },
+	{ .fc_id = 10, .cpu_id = 10, .valid = 0, .name = "" },
+	{ .fc_id = 11, .cpu_id = 11, .valid = 0, .name = "" },
+	{ .fc_id = 12, .cpu_id = 12, .valid = 0, .name = "" },
+	{ .fc_id = 13, .cpu_id = 13, .valid = 0, .name = "" },
+	{ .fc_id = 14, .cpu_id = 14, .valid = 0, .name = "" },
+	{ .fc_id = 15, .cpu_id = 15, .valid = 0, .name = "" },
+	{ .fc_id = 16, .cpu_id = 16, .valid = 0, .name = "" },
+	{ .fc_id = 17, .cpu_id = 17, .valid = 0, .name = "" },
+	{ .fc_id = 18, .cpu_id = 18, .valid = 0, .name = "" },
+	{ .fc_id = 19, .cpu_id = 19, .valid = 0, .name = "" },
+	{ .fc_id = 20, .cpu_id = 20, .valid = 0, .name = "" },
+	{ .fc_id = 21, .cpu_id = 21, .valid = 0, .name = "" },
+	{ .fc_id = 22, .cpu_id = 22, .valid = 0, .name = "" },
+	{ .fc_id = 23, .cpu_id = 23, .valid = 0, .name = "" },
+	{ .fc_id = 24, .cpu_id = 24, .valid = 0, .name = "" },
+	{ .fc_id = 25, .cpu_id = 25, .valid = 0, .name = "" },
+	{ .fc_id = 26, .cpu_id = 26, .valid = 0, .name = "" },
+	{ .fc_id = 27, .cpu_id = 27, .valid = 0, .name = "" },
+	{ .fc_id = 28, .cpu_id = 28, .valid = 0, .name = "" },
+	{ .fc_id = 29, .cpu_id = 29, .valid = 0, .name = "" },
+	{ .fc_id = 30, .cpu_id = 30, .valid = 0, .name = "" },
+	{ .fc_id = 31, .cpu_id = 31, .valid = 0, .name = "" },
+	{ .fc_id = 32, .cpu_id = 32, .valid = 1, .name = "PCIE_CORE_SERR" },
+	{ .fc_id = 33, .cpu_id = 33, .valid = 1, .name = "PCIE_CORE_DERR" },
+	{ .fc_id = 34, .cpu_id = 34, .valid = 1, .name = "PCIE_IF_SERR" },
+	{ .fc_id = 35, .cpu_id = 35, .valid = 1, .name = "PCIE_IF_DERR" },
+	{ .fc_id = 36, .cpu_id = 36, .valid = 1, .name = "PCIE_PHY_SERR" },
+	{ .fc_id = 37, .cpu_id = 37, .valid = 1, .name = "PCIE_PHY_DERR" },
+	{ .fc_id = 38, .cpu_id = 38, .valid = 1, .name = "TPC0_SERR" },
+	{ .fc_id = 39, .cpu_id = 38, .valid = 1, .name = "TPC1_SERR" },
+	{ .fc_id = 40, .cpu_id = 38, .valid = 1, .name = "TPC2_SERR" },
+	{ .fc_id = 41, .cpu_id = 38, .valid = 1, .name = "TPC3_SERR" },
+	{ .fc_id = 42, .cpu_id = 38, .valid = 1, .name = "TPC4_SERR" },
+	{ .fc_id = 43, .cpu_id = 38, .valid = 1, .name = "TPC5_SERR" },
+	{ .fc_id = 44, .cpu_id = 38, .valid = 1, .name = "TPC6_SERR" },
+	{ .fc_id = 45, .cpu_id = 38, .valid = 1, .name = "TPC7_SERR" },
+	{ .fc_id = 46, .cpu_id = 39, .valid = 1, .name = "TPC0_DERR" },
+	{ .fc_id = 47, .cpu_id = 39, .valid = 1, .name = "TPC1_DERR" },
+	{ .fc_id = 48, .cpu_id = 39, .valid = 1, .name = "TPC2_DERR" },
+	{ .fc_id = 49, .cpu_id = 39, .valid = 1, .name = "TPC3_DERR" },
+	{ .fc_id = 50, .cpu_id = 39, .valid = 1, .name = "TPC4_DERR" },
+	{ .fc_id = 51, .cpu_id = 39, .valid = 1, .name = "TPC5_DERR" },
+	{ .fc_id = 52, .cpu_id = 39, .valid = 1, .name = "TPC6_DERR" },
+	{ .fc_id = 53, .cpu_id = 39, .valid = 1, .name = "TPC7_DERR" },
+	{ .fc_id = 54, .cpu_id = 40, .valid = 1, .name = "MME0_ACC_SERR" },
+	{ .fc_id = 55, .cpu_id = 41, .valid = 1, .name = "MME0_ACC_DERR" },
+	{ .fc_id = 56, .cpu_id = 42, .valid = 1, .name = "MME0_SBAB_SERR" },
+	{ .fc_id = 57, .cpu_id = 43, .valid = 1, .name = "MME0_SBAB_DERR" },
+	{ .fc_id = 58, .cpu_id = 44, .valid = 1, .name = "MME1_ACC_SERR" },
+	{ .fc_id = 59, .cpu_id = 45, .valid = 1, .name = "MME1_ACC_DERR" },
+	{ .fc_id = 60, .cpu_id = 46, .valid = 1, .name = "MME1_SBAB_SERR" },
+	{ .fc_id = 61, .cpu_id = 47, .valid = 1, .name = "MME1_SBAB_DERR" },
+	{ .fc_id = 62, .cpu_id = 48, .valid = 1, .name = "MME2_ACC_SERR" },
+	{ .fc_id = 63, .cpu_id = 49, .valid = 1, .name = "MME2_ACC_DERR" },
+	{ .fc_id = 64, .cpu_id = 50, .valid = 1, .name = "MME2_SBAB_SERR" },
+	{ .fc_id = 65, .cpu_id = 51, .valid = 1, .name = "MME2_SBAB_DERR" },
+	{ .fc_id = 66, .cpu_id = 52, .valid = 1, .name = "MME3_ACC_SERR" },
+	{ .fc_id = 67, .cpu_id = 53, .valid = 1, .name = "MME3_ACC_DERR" },
+	{ .fc_id = 68, .cpu_id = 54, .valid = 1, .name = "MME3_SBAB_SERR" },
+	{ .fc_id = 69, .cpu_id = 55, .valid = 1, .name = "MME3_SBAB_DERR" },
+	{ .fc_id = 70, .cpu_id = 56, .valid = 1, .name = "DMA0_SERR_ECC" },
+	{ .fc_id = 71, .cpu_id = 56, .valid = 1, .name = "DMA1_SERR_ECC" },
+	{ .fc_id = 72, .cpu_id = 56, .valid = 1, .name = "DMA2_SERR_ECC" },
+	{ .fc_id = 73, .cpu_id = 56, .valid = 1, .name = "DMA3_SERR_ECC" },
+	{ .fc_id = 74, .cpu_id = 56, .valid = 1, .name = "DMA4_SERR_ECC" },
+	{ .fc_id = 75, .cpu_id = 56, .valid = 1, .name = "DMA5_SERR_ECC" },
+	{ .fc_id = 76, .cpu_id = 56, .valid = 1, .name = "DMA6_SERR_ECC" },
+	{ .fc_id = 77, .cpu_id = 56, .valid = 1, .name = "DMA7_SERR_ECC" },
+	{ .fc_id = 78, .cpu_id = 57, .valid = 1, .name = "DMA0_DERR_ECC" },
+	{ .fc_id = 79, .cpu_id = 57, .valid = 1, .name = "DMA1_DERR_ECC" },
+	{ .fc_id = 80, .cpu_id = 57, .valid = 1, .name = "DMA2_DERR_ECC" },
+	{ .fc_id = 81, .cpu_id = 57, .valid = 1, .name = "DMA3_DERR_ECC" },
+	{ .fc_id = 82, .cpu_id = 57, .valid = 1, .name = "DMA4_DERR_ECC" },
+	{ .fc_id = 83, .cpu_id = 57, .valid = 1, .name = "DMA5_DERR_ECC" },
+	{ .fc_id = 84, .cpu_id = 57, .valid = 1, .name = "DMA6_DERR_ECC" },
+	{ .fc_id = 85, .cpu_id = 57, .valid = 1, .name = "DMA7_DERR_ECC" },
+	{ .fc_id = 86, .cpu_id = 58, .valid = 1, .name = "CPU_IF_ECC_SERR" },
+	{ .fc_id = 87, .cpu_id = 59, .valid = 1, .name = "CPU_IF_ECC_DERR" },
+	{ .fc_id = 88, .cpu_id = 60, .valid = 1, .name = "PSOC_MEM_SERR" },
+	{ .fc_id = 89, .cpu_id = 61, .valid = 1,
+		.name = "PSOC_CORESIGHT_SERR" },
+	{ .fc_id = 90, .cpu_id = 62, .valid = 1, .name = "PSOC_MEM_DERR" },
+	{ .fc_id = 91, .cpu_id = 63, .valid = 1,
+		.name = "PSOC_CORESIGHT_DERR" },
+	{ .fc_id = 92, .cpu_id = 64, .valid = 1, .name = "SRAM0_SERR" },
+	{ .fc_id = 93, .cpu_id = 64, .valid = 1, .name = "SRAM1_SERR" },
+	{ .fc_id = 94, .cpu_id = 64, .valid = 1, .name = "SRAM2_SERR" },
+	{ .fc_id = 95, .cpu_id = 64, .valid = 1, .name = "SRAM3_SERR" },
+	{ .fc_id = 96, .cpu_id = 64, .valid = 1, .name = "SRAM7_SERR" },
+	{ .fc_id = 97, .cpu_id = 64, .valid = 1, .name = "SRAM6_SERR" },
+	{ .fc_id = 98, .cpu_id = 64, .valid = 1, .name = "SRAM5_SERR" },
+	{ .fc_id = 99, .cpu_id = 64, .valid = 1, .name = "SRAM4_SERR" },
+	{ .fc_id = 100, .cpu_id = 64, .valid = 1, .name = "SRAM8_SERR" },
+	{ .fc_id = 101, .cpu_id = 64, .valid = 1, .name = "SRAM9_SERR" },
+	{ .fc_id = 102, .cpu_id = 64, .valid = 1, .name = "SRAM10_SERR" },
+	{ .fc_id = 103, .cpu_id = 64, .valid = 1, .name = "SRAM11_SERR" },
+	{ .fc_id = 104, .cpu_id = 64, .valid = 1, .name = "SRAM15_SERR" },
+	{ .fc_id = 105, .cpu_id = 64, .valid = 1, .name = "SRAM14_SERR" },
+	{ .fc_id = 106, .cpu_id = 64, .valid = 1, .name = "SRAM13_SERR" },
+	{ .fc_id = 107, .cpu_id = 64, .valid = 1, .name = "SRAM12_SERR" },
+	{ .fc_id = 108, .cpu_id = 64, .valid = 1, .name = "SRAM16_SERR" },
+	{ .fc_id = 109, .cpu_id = 64, .valid = 1, .name = "SRAM17_SERR" },
+	{ .fc_id = 110, .cpu_id = 64, .valid = 1, .name = "SRAM18_SERR" },
+	{ .fc_id = 111, .cpu_id = 64, .valid = 1, .name = "SRAM19_SERR" },
+	{ .fc_id = 112, .cpu_id = 64, .valid = 1, .name = "SRAM23_SERR" },
+	{ .fc_id = 113, .cpu_id = 64, .valid = 1, .name = "SRAM22_SERR" },
+	{ .fc_id = 114, .cpu_id = 64, .valid = 1, .name = "SRAM21_SERR" },
+	{ .fc_id = 115, .cpu_id = 64, .valid = 1, .name = "SRAM20_SERR" },
+	{ .fc_id = 116, .cpu_id = 64, .valid = 1, .name = "SRAM24_SERR" },
+	{ .fc_id = 117, .cpu_id = 64, .valid = 1, .name = "SRAM25_SERR" },
+	{ .fc_id = 118, .cpu_id = 64, .valid = 1, .name = "SRAM26_SERR" },
+	{ .fc_id = 119, .cpu_id = 64, .valid = 1, .name = "SRAM27_SERR" },
+	{ .fc_id = 120, .cpu_id = 64, .valid = 1, .name = "SRAM31_SERR" },
+	{ .fc_id = 121, .cpu_id = 64, .valid = 1, .name = "SRAM30_SERR" },
+	{ .fc_id = 122, .cpu_id = 64, .valid = 1, .name = "SRAM29_SERR" },
+	{ .fc_id = 123, .cpu_id = 64, .valid = 1, .name = "SRAM28_SERR" },
+	{ .fc_id = 124, .cpu_id = 65, .valid = 1, .name = "SRAM0_DERR" },
+	{ .fc_id = 125, .cpu_id = 65, .valid = 1, .name = "SRAM1_DERR" },
+	{ .fc_id = 126, .cpu_id = 65, .valid = 1, .name = "SRAM2_DERR" },
+	{ .fc_id = 127, .cpu_id = 65, .valid = 1, .name = "SRAM3_DERR" },
+	{ .fc_id = 128, .cpu_id = 65, .valid = 1, .name = "SRAM7_DERR" },
+	{ .fc_id = 129, .cpu_id = 65, .valid = 1, .name = "SRAM6_DERR" },
+	{ .fc_id = 130, .cpu_id = 65, .valid = 1, .name = "SRAM5_DERR" },
+	{ .fc_id = 131, .cpu_id = 65, .valid = 1, .name = "SRAM4_DERR" },
+	{ .fc_id = 132, .cpu_id = 65, .valid = 1, .name = "SRAM8_DERR" },
+	{ .fc_id = 133, .cpu_id = 65, .valid = 1, .name = "SRAM9_DERR" },
+	{ .fc_id = 134, .cpu_id = 65, .valid = 1, .name = "SRAM10_DERR" },
+	{ .fc_id = 135, .cpu_id = 65, .valid = 1, .name = "SRAM11_DERR" },
+	{ .fc_id = 136, .cpu_id = 65, .valid = 1, .name = "SRAM15_DERR" },
+	{ .fc_id = 137, .cpu_id = 65, .valid = 1, .name = "SRAM14_DERR" },
+	{ .fc_id = 138, .cpu_id = 65, .valid = 1, .name = "SRAM13_DERR" },
+	{ .fc_id = 139, .cpu_id = 65, .valid = 1, .name = "SRAM12_DERR" },
+	{ .fc_id = 140, .cpu_id = 65, .valid = 1, .name = "SRAM16_DERR" },
+	{ .fc_id = 141, .cpu_id = 65, .valid = 1, .name = "SRAM17_DERR" },
+	{ .fc_id = 142, .cpu_id = 65, .valid = 1, .name = "SRAM18_DERR" },
+	{ .fc_id = 143, .cpu_id = 65, .valid = 1, .name = "SRAM19_DERR" },
+	{ .fc_id = 144, .cpu_id = 65, .valid = 1, .name = "SRAM23_DERR" },
+	{ .fc_id = 145, .cpu_id = 65, .valid = 1, .name = "SRAM22_DERR" },
+	{ .fc_id = 146, .cpu_id = 65, .valid = 1, .name = "SRAM21_DERR" },
+	{ .fc_id = 147, .cpu_id = 65, .valid = 1, .name = "SRAM20_DERR" },
+	{ .fc_id = 148, .cpu_id = 65, .valid = 1, .name = "SRAM24_DERR" },
+	{ .fc_id = 149, .cpu_id = 65, .valid = 1, .name = "SRAM25_DERR" },
+	{ .fc_id = 150, .cpu_id = 65, .valid = 1, .name = "SRAM26_DERR" },
+	{ .fc_id = 151, .cpu_id = 65, .valid = 1, .name = "SRAM27_DERR" },
+	{ .fc_id = 152, .cpu_id = 65, .valid = 1, .name = "SRAM31_DERR" },
+	{ .fc_id = 153, .cpu_id = 65, .valid = 1, .name = "SRAM30_DERR" },
+	{ .fc_id = 154, .cpu_id = 65, .valid = 1, .name = "SRAM29_DERR" },
+	{ .fc_id = 155, .cpu_id = 65, .valid = 1, .name = "SRAM28_DERR" },
+	{ .fc_id = 156, .cpu_id = 66, .valid = 1, .name = "NIC0_SERR" },
+	{ .fc_id = 157, .cpu_id = 66, .valid = 1, .name = "NIC1_SERR" },
+	{ .fc_id = 158, .cpu_id = 66, .valid = 1, .name = "NIC2_SERR" },
+	{ .fc_id = 159, .cpu_id = 66, .valid = 1, .name = "NIC3_SERR" },
+	{ .fc_id = 160, .cpu_id = 66, .valid = 1, .name = "NIC4_SERR" },
+	{ .fc_id = 161, .cpu_id = 66, .valid = 0, .name = "" },
+	{ .fc_id = 162, .cpu_id = 66, .valid = 0, .name = "" },
+	{ .fc_id = 163, .cpu_id = 66, .valid = 0, .name = "" },
+	{ .fc_id = 164, .cpu_id = 66, .valid = 0, .name = "" },
+	{ .fc_id = 165, .cpu_id = 66, .valid = 0, .name = "" },
+	{ .fc_id = 166, .cpu_id = 67, .valid = 1, .name = "NIC0_DERR" },
+	{ .fc_id = 167, .cpu_id = 67, .valid = 1, .name = "NIC1_DERR" },
+	{ .fc_id = 168, .cpu_id = 67, .valid = 1, .name = "NIC2_DERR" },
+	{ .fc_id = 169, .cpu_id = 67, .valid = 1, .name = "NIC3_DERR" },
+	{ .fc_id = 170, .cpu_id = 67, .valid = 1, .name = "NIC4_DERR" },
+	{ .fc_id = 171, .cpu_id = 67, .valid = 0, .name = "" },
+	{ .fc_id = 172, .cpu_id = 67, .valid = 0, .name = "" },
+	{ .fc_id = 173, .cpu_id = 67, .valid = 0, .name = "" },
+	{ .fc_id = 174, .cpu_id = 67, .valid = 0, .name = "" },
+	{ .fc_id = 175, .cpu_id = 67, .valid = 0, .name = "" },
+	{ .fc_id = 176, .cpu_id = 68, .valid = 1, .name = "DMA_IF0_SERR" },
+	{ .fc_id = 177, .cpu_id = 68, .valid = 1, .name = "DMA_IF1_SERR" },
+	{ .fc_id = 178, .cpu_id = 68, .valid = 1, .name = "DMA_IF2_SERR" },
+	{ .fc_id = 179, .cpu_id = 68, .valid = 1, .name = "DMA_IF3_SERR" },
+	{ .fc_id = 180, .cpu_id = 69, .valid = 1, .name = "DMA_IF0_DERR" },
+	{ .fc_id = 181, .cpu_id = 69, .valid = 1, .name = "DMA_IF1_DERR" },
+	{ .fc_id = 182, .cpu_id = 69, .valid = 1, .name = "DMA_IF2_DERR" },
+	{ .fc_id = 183, .cpu_id = 69, .valid = 1, .name = "DMA_IF3_DERR" },
+	{ .fc_id = 184, .cpu_id = 70, .valid = 1, .name = "GIC500" },
+	{ .fc_id = 185, .cpu_id = 71, .valid = 1, .name = "HBM_0_SERR" },
+	{ .fc_id = 186, .cpu_id = 71, .valid = 1, .name = "HBM_1_SERR" },
+	{ .fc_id = 187, .cpu_id = 71, .valid = 1, .name = "HBM_2_SERR" },
+	{ .fc_id = 188, .cpu_id = 71, .valid = 1, .name = "HBM_3_SERR" },
+	{ .fc_id = 189, .cpu_id = 72, .valid = 1, .name = "HBM_0_DERR" },
+	{ .fc_id = 190, .cpu_id = 72, .valid = 1, .name = "HBM_1_DERR" },
+	{ .fc_id = 191, .cpu_id = 72, .valid = 1, .name = "HBM_2_DERR" },
+	{ .fc_id = 192, .cpu_id = 72, .valid = 1, .name = "HBM_3_DERR" },
+	{ .fc_id = 193, .cpu_id = 73, .valid = 1, .name = "MMU_SERR" },
+	{ .fc_id = 194, .cpu_id = 74, .valid = 1, .name = "MMU_DERR" },
+	{ .fc_id = 195, .cpu_id = 75, .valid = 0, .name = "" },
+	{ .fc_id = 196, .cpu_id = 76, .valid = 0, .name = "" },
+	{ .fc_id = 197, .cpu_id = 77, .valid = 0, .name = "" },
+	{ .fc_id = 198, .cpu_id = 78, .valid = 0, .name = "" },
+	{ .fc_id = 199, .cpu_id = 79, .valid = 0, .name = "" },
+	{ .fc_id = 200, .cpu_id = 80, .valid = 1, .name = "PCIE_DEC" },
+	{ .fc_id = 201, .cpu_id = 81, .valid = 1, .name = "TPC0_DEC" },
+	{ .fc_id = 202, .cpu_id = 82, .valid = 0, .name = "" },
+	{ .fc_id = 203, .cpu_id = 83, .valid = 1, .name = "TPC1_DEC" },
+	{ .fc_id = 204, .cpu_id = 84, .valid = 0, .name = "" },
+	{ .fc_id = 205, .cpu_id = 85, .valid = 1, .name = "TPC2_DEC" },
+	{ .fc_id = 206, .cpu_id = 86, .valid = 0, .name = "" },
+	{ .fc_id = 207, .cpu_id = 87, .valid = 1, .name = "TPC3_DEC" },
+	{ .fc_id = 208, .cpu_id = 88, .valid = 0, .name = "" },
+	{ .fc_id = 209, .cpu_id = 89, .valid = 1, .name = "TPC4_DEC" },
+	{ .fc_id = 210, .cpu_id = 90, .valid = 0, .name = "" },
+	{ .fc_id = 211, .cpu_id = 91, .valid = 1, .name = "TPC5_DEC" },
+	{ .fc_id = 212, .cpu_id = 92, .valid = 0, .name = "" },
+	{ .fc_id = 213, .cpu_id = 93, .valid = 1, .name = "TPC6_DEC" },
+	{ .fc_id = 214, .cpu_id = 94, .valid = 0, .name = "" },
+	{ .fc_id = 215, .cpu_id = 95, .valid = 1, .name = "TPC7_DEC" },
+	{ .fc_id = 216, .cpu_id = 96, .valid = 0, .name = "" },
+	{ .fc_id = 217, .cpu_id = 97, .valid = 1, .name = "AXI_ECC" },
+	{ .fc_id = 218, .cpu_id = 98, .valid = 1, .name = "L2_RAM_ECC" },
+	{ .fc_id = 219, .cpu_id = 99, .valid = 1, .name = "MME0_WBC_RSP" },
+	{ .fc_id = 220, .cpu_id = 100, .valid = 1, .name = "MME0_SBAB0_RSP" },
+	{ .fc_id = 221, .cpu_id = 101, .valid = 0, .name = "" },
+	{ .fc_id = 222, .cpu_id = 102, .valid = 0, .name = "" },
+	{ .fc_id = 223, .cpu_id = 103, .valid = 0, .name = "" },
+	{ .fc_id = 224, .cpu_id = 104, .valid = 1, .name = "MME1_WBC_RSP" },
+	{ .fc_id = 225, .cpu_id = 105, .valid = 1, .name = "MME1_SBAB0_RSP" },
+	{ .fc_id = 226, .cpu_id = 106, .valid = 0, .name = "" },
+	{ .fc_id = 227, .cpu_id = 107, .valid = 0, .name = "" },
+	{ .fc_id = 228, .cpu_id = 108, .valid = 0, .name = "" },
+	{ .fc_id = 229, .cpu_id = 109, .valid = 1, .name = "MME2_WBC_RSP" },
+	{ .fc_id = 230, .cpu_id = 110, .valid = 1, .name = "MME2_SBAB0_RSP" },
+	{ .fc_id = 231, .cpu_id = 111, .valid = 0, .name = "" },
+	{ .fc_id = 232, .cpu_id = 112, .valid = 0, .name = "" },
+	{ .fc_id = 233, .cpu_id = 113, .valid = 0, .name = "" },
+	{ .fc_id = 234, .cpu_id = 114, .valid = 1, .name = "MME3_WBC_RSP" },
+	{ .fc_id = 235, .cpu_id = 115, .valid = 1, .name = "MME3_SBAB0_RSP" },
+	{ .fc_id = 236, .cpu_id = 116, .valid = 0, .name = "" },
+	{ .fc_id = 237, .cpu_id = 117, .valid = 0, .name = "" },
+	{ .fc_id = 238, .cpu_id = 118, .valid = 0, .name = "" },
+	{ .fc_id = 239, .cpu_id = 119, .valid = 1, .name = "PLL0" },
+	{ .fc_id = 240, .cpu_id = 119, .valid = 1, .name = "PLL1" },
+	{ .fc_id = 241, .cpu_id = 119, .valid = 1, .name = "PLL2" },
+	{ .fc_id = 242, .cpu_id = 119, .valid = 1, .name = "PLL3" },
+	{ .fc_id = 243, .cpu_id = 119, .valid = 1, .name = "PLL4" },
+	{ .fc_id = 244, .cpu_id = 119, .valid = 1, .name = "PLL5" },
+	{ .fc_id = 245, .cpu_id = 119, .valid = 1, .name = "PLL6" },
+	{ .fc_id = 246, .cpu_id = 119, .valid = 1, .name = "PLL7" },
+	{ .fc_id = 247, .cpu_id = 119, .valid = 1, .name = "PLL8" },
+	{ .fc_id = 248, .cpu_id = 119, .valid = 1, .name = "PLL9" },
+	{ .fc_id = 249, .cpu_id = 119, .valid = 1, .name = "PLL10" },
+	{ .fc_id = 250, .cpu_id = 119, .valid = 1, .name = "PLL11" },
+	{ .fc_id = 251, .cpu_id = 119, .valid = 1, .name = "PLL12" },
+	{ .fc_id = 252, .cpu_id = 119, .valid = 1, .name = "PLL13" },
+	{ .fc_id = 253, .cpu_id = 119, .valid = 1, .name = "PLL14" },
+	{ .fc_id = 254, .cpu_id = 119, .valid = 1, .name = "PLL15" },
+	{ .fc_id = 255, .cpu_id = 119, .valid = 1, .name = "PLL16" },
+	{ .fc_id = 256, .cpu_id = 119, .valid = 1, .name = "PLL17" },
+	{ .fc_id = 257, .cpu_id = 120, .valid = 1,
+		.name = "CPU_AXI_SPLITTER" },
+	{ .fc_id = 258, .cpu_id = 121, .valid = 0, .name = "" },
+	{ .fc_id = 259, .cpu_id = 122, .valid = 0, .name = "" },
+	{ .fc_id = 260, .cpu_id = 123, .valid = 0, .name = "" },
+	{ .fc_id = 261, .cpu_id = 124, .valid = 0, .name = "" },
+	{ .fc_id = 262, .cpu_id = 125, .valid = 1, .name = "PSOC_AXI_DEC" },
+	{ .fc_id = 263, .cpu_id = 126, .valid = 1, .name = "PSOC_PRSTN_FALL" },
+	{ .fc_id = 264, .cpu_id = 127, .valid = 1, .name = "NIC_SEI_0" },
+	{ .fc_id = 265, .cpu_id = 127, .valid = 1, .name = "NIC_SEI_1" },
+	{ .fc_id = 266, .cpu_id = 127, .valid = 1, .name = "NIC_SEI_2" },
+	{ .fc_id = 267, .cpu_id = 127, .valid = 1, .name = "NIC_SEI_3" },
+	{ .fc_id = 268, .cpu_id = 127, .valid = 1, .name = "NIC_SEI_4" },
+	{ .fc_id = 269, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 270, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 271, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 272, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 273, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 274, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 275, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 276, .cpu_id = 128, .valid = 0, .name = "" },
+	{ .fc_id = 277, .cpu_id = 129, .valid = 0, .name = "" },
+	{ .fc_id = 278, .cpu_id = 129, .valid = 0, .name = "" },
+	{ .fc_id = 279, .cpu_id = 129, .valid = 0, .name = "" },
+	{ .fc_id = 280, .cpu_id = 129, .valid = 0, .name = "" },
+	{ .fc_id = 281, .cpu_id = 130, .valid = 0, .name = "" },
+	{ .fc_id = 282, .cpu_id = 131, .valid = 0, .name = "" },
+	{ .fc_id = 283, .cpu_id = 132, .valid = 0, .name = "" },
+	{ .fc_id = 284, .cpu_id = 133, .valid = 0, .name = "" },
+	{ .fc_id = 285, .cpu_id = 134, .valid = 0, .name = "" },
+	{ .fc_id = 286, .cpu_id = 135, .valid = 0, .name = "" },
+	{ .fc_id = 287, .cpu_id = 136, .valid = 0, .name = "" },
+	{ .fc_id = 288, .cpu_id = 137, .valid = 0, .name = "" },
+	{ .fc_id = 289, .cpu_id = 138, .valid = 0, .name = "" },
+	{ .fc_id = 290, .cpu_id = 139, .valid = 1, .name = "PCIE_FLR" },
+	{ .fc_id = 291, .cpu_id = 140, .valid = 0, .name = "" },
+	{ .fc_id = 292, .cpu_id = 141, .valid = 0, .name = "" },
+	{ .fc_id = 293, .cpu_id = 142, .valid = 0, .name = "" },
+	{ .fc_id = 294, .cpu_id = 143, .valid = 0, .name = "" },
+	{ .fc_id = 295, .cpu_id = 144, .valid = 0, .name = "" },
+	{ .fc_id = 296, .cpu_id = 145, .valid = 0, .name = "" },
+	{ .fc_id = 297, .cpu_id = 146, .valid = 0, .name = "" },
+	{ .fc_id = 298, .cpu_id = 147, .valid = 0, .name = "" },
+	{ .fc_id = 299, .cpu_id = 148, .valid = 0, .name = "" },
+	{ .fc_id = 300, .cpu_id = 149, .valid = 1, .name = "TPC0_BMON_SPMU" },
+	{ .fc_id = 301, .cpu_id = 150, .valid = 1, .name = "TPC0_KRN_ERR" },
+	{ .fc_id = 302, .cpu_id = 151, .valid = 0, .name = "" },
+	{ .fc_id = 303, .cpu_id = 152, .valid = 0, .name = "" },
+	{ .fc_id = 304, .cpu_id = 153, .valid = 0, .name = "" },
+	{ .fc_id = 305, .cpu_id = 154, .valid = 0, .name = "" },
+	{ .fc_id = 306, .cpu_id = 155, .valid = 1, .name = "TPC1_BMON_SPMU" },
+	{ .fc_id = 307, .cpu_id = 156, .valid = 1, .name = "TPC1_KRN_ERR" },
+	{ .fc_id = 308, .cpu_id = 157, .valid = 0, .name = "" },
+	{ .fc_id = 309, .cpu_id = 158, .valid = 0, .name = "" },
+	{ .fc_id = 310, .cpu_id = 159, .valid = 0, .name = "" },
+	{ .fc_id = 311, .cpu_id = 160, .valid = 0, .name = "" },
+	{ .fc_id = 312, .cpu_id = 161, .valid = 1, .name = "TPC2_BMON_SPMU" },
+	{ .fc_id = 313, .cpu_id = 162, .valid = 1, .name = "TPC2_KRN_ERR" },
+	{ .fc_id = 314, .cpu_id = 163, .valid = 0, .name = "" },
+	{ .fc_id = 315, .cpu_id = 164, .valid = 0, .name = "" },
+	{ .fc_id = 316, .cpu_id = 165, .valid = 0, .name = "" },
+	{ .fc_id = 317, .cpu_id = 166, .valid = 0, .name = "" },
+	{ .fc_id = 318, .cpu_id = 167, .valid = 1, .name = "TPC3_BMON_SPMU" },
+	{ .fc_id = 319, .cpu_id = 168, .valid = 1, .name = "TPC3_KRN_ERR" },
+	{ .fc_id = 320, .cpu_id = 169, .valid = 0, .name = "" },
+	{ .fc_id = 321, .cpu_id = 170, .valid = 0, .name = "" },
+	{ .fc_id = 322, .cpu_id = 171, .valid = 0, .name = "" },
+	{ .fc_id = 323, .cpu_id = 172, .valid = 0, .name = "" },
+	{ .fc_id = 324, .cpu_id = 173, .valid = 1, .name = "TPC4_BMON_SPMU" },
+	{ .fc_id = 325, .cpu_id = 174, .valid = 1, .name = "TPC4_KRN_ERR" },
+	{ .fc_id = 326, .cpu_id = 175, .valid = 0, .name = "" },
+	{ .fc_id = 327, .cpu_id = 176, .valid = 0, .name = "" },
+	{ .fc_id = 328, .cpu_id = 177, .valid = 0, .name = "" },
+	{ .fc_id = 329, .cpu_id = 178, .valid = 0, .name = "" },
+	{ .fc_id = 330, .cpu_id = 179, .valid = 1, .name = "TPC5_BMON_SPMU" },
+	{ .fc_id = 331, .cpu_id = 180, .valid = 1, .name = "TPC5_KRN_ERR" },
+	{ .fc_id = 332, .cpu_id = 181, .valid = 0, .name = "" },
+	{ .fc_id = 333, .cpu_id = 182, .valid = 0, .name = "" },
+	{ .fc_id = 334, .cpu_id = 183, .valid = 0, .name = "" },
+	{ .fc_id = 335, .cpu_id = 184, .valid = 0, .name = "" },
+	{ .fc_id = 336, .cpu_id = 185, .valid = 1, .name = "TPC6_BMON_SPMU" },
+	{ .fc_id = 337, .cpu_id = 186, .valid = 1, .name = "TPC6_KRN_ERR" },
+	{ .fc_id = 338, .cpu_id = 187, .valid = 0, .name = "" },
+	{ .fc_id = 339, .cpu_id = 188, .valid = 0, .name = "" },
+	{ .fc_id = 340, .cpu_id = 189, .valid = 0, .name = "" },
+	{ .fc_id = 341, .cpu_id = 190, .valid = 0, .name = "" },
+	{ .fc_id = 342, .cpu_id = 191, .valid = 1, .name = "TPC7_BMON_SPMU" },
+	{ .fc_id = 343, .cpu_id = 192, .valid = 1, .name = "TPC7_KRN_ERR" },
+	{ .fc_id = 344, .cpu_id = 193, .valid = 0, .name = "" },
+	{ .fc_id = 345, .cpu_id = 194, .valid = 0, .name = "" },
+	{ .fc_id = 346, .cpu_id = 195, .valid = 0, .name = "" },
+	{ .fc_id = 347, .cpu_id = 196, .valid = 0, .name = "" },
+	{ .fc_id = 348, .cpu_id = 197, .valid = 0, .name = "" },
+	{ .fc_id = 349, .cpu_id = 198, .valid = 0, .name = "" },
+	{ .fc_id = 350, .cpu_id = 199, .valid = 0, .name = "" },
+	{ .fc_id = 351, .cpu_id = 200, .valid = 0, .name = "" },
+	{ .fc_id = 352, .cpu_id = 201, .valid = 0, .name = "" },
+	{ .fc_id = 353, .cpu_id = 202, .valid = 0, .name = "" },
+	{ .fc_id = 354, .cpu_id = 203, .valid = 0, .name = "" },
+	{ .fc_id = 355, .cpu_id = 204, .valid = 0, .name = "" },
+	{ .fc_id = 356, .cpu_id = 205, .valid = 0, .name = "" },
+	{ .fc_id = 357, .cpu_id = 206, .valid = 0, .name = "" },
+	{ .fc_id = 358, .cpu_id = 207, .valid = 0, .name = "" },
+	{ .fc_id = 359, .cpu_id = 208, .valid = 0, .name = "" },
+	{ .fc_id = 360, .cpu_id = 209, .valid = 0, .name = "" },
+	{ .fc_id = 361, .cpu_id = 210, .valid = 0, .name = "" },
+	{ .fc_id = 362, .cpu_id = 211, .valid = 0, .name = "" },
+	{ .fc_id = 363, .cpu_id = 212, .valid = 0, .name = "" },
+	{ .fc_id = 364, .cpu_id = 213, .valid = 0, .name = "" },
+	{ .fc_id = 365, .cpu_id = 214, .valid = 0, .name = "" },
+	{ .fc_id = 366, .cpu_id = 215, .valid = 0, .name = "" },
+	{ .fc_id = 367, .cpu_id = 216, .valid = 0, .name = "" },
+	{ .fc_id = 368, .cpu_id = 217, .valid = 0, .name = "" },
+	{ .fc_id = 369, .cpu_id = 218, .valid = 0, .name = "" },
+	{ .fc_id = 370, .cpu_id = 219, .valid = 0, .name = "" },
+	{ .fc_id = 371, .cpu_id = 220, .valid = 0, .name = "" },
+	{ .fc_id = 372, .cpu_id = 221, .valid = 0, .name = "" },
+	{ .fc_id = 373, .cpu_id = 222, .valid = 0, .name = "" },
+	{ .fc_id = 374, .cpu_id = 223, .valid = 0, .name = "" },
+	{ .fc_id = 375, .cpu_id = 224, .valid = 0, .name = "" },
+	{ .fc_id = 376, .cpu_id = 225, .valid = 0, .name = "" },
+	{ .fc_id = 377, .cpu_id = 226, .valid = 0, .name = "" },
+	{ .fc_id = 378, .cpu_id = 227, .valid = 0, .name = "" },
+	{ .fc_id = 379, .cpu_id = 228, .valid = 0, .name = "" },
+	{ .fc_id = 380, .cpu_id = 229, .valid = 1, .name = "MMU_PAGE_FAULT" },
+	{ .fc_id = 381, .cpu_id = 230, .valid = 1, .name = "MMU_WR_PERM" },
+	{ .fc_id = 382, .cpu_id = 231, .valid = 0, .name = "" },
+	{ .fc_id = 383, .cpu_id = 232, .valid = 1, .name = "DMA_BM_CH0" },
+	{ .fc_id = 384, .cpu_id = 233, .valid = 1, .name = "DMA_BM_CH1" },
+	{ .fc_id = 385, .cpu_id = 234, .valid = 1, .name = "DMA_BM_CH2" },
+	{ .fc_id = 386, .cpu_id = 235, .valid = 1, .name = "DMA_BM_CH3" },
+	{ .fc_id = 387, .cpu_id = 236, .valid = 1, .name = "DMA_BM_CH4" },
+	{ .fc_id = 388, .cpu_id = 237, .valid = 1, .name = "DMA_BM_CH5" },
+	{ .fc_id = 389, .cpu_id = 238, .valid = 1, .name = "DMA_BM_CH6" },
+	{ .fc_id = 390, .cpu_id = 239, .valid = 1, .name = "DMA_BM_CH7" },
+	{ .fc_id = 391, .cpu_id = 240, .valid = 0, .name = "" },
+	{ .fc_id = 392, .cpu_id = 241, .valid = 0, .name = "" },
+	{ .fc_id = 393, .cpu_id = 242, .valid = 0, .name = "" },
+	{ .fc_id = 394, .cpu_id = 243, .valid = 0, .name = "" },
+	{ .fc_id = 395, .cpu_id = 244, .valid = 1, .name = "HBM0_SPI_0" },
+	{ .fc_id = 396, .cpu_id = 245, .valid = 1, .name = "HBM0_SPI_1" },
+	{ .fc_id = 397, .cpu_id = 246, .valid = 0, .name = "" },
+	{ .fc_id = 398, .cpu_id = 247, .valid = 0, .name = "" },
+	{ .fc_id = 399, .cpu_id = 248, .valid = 1, .name = "HBM1_SPI_0" },
+	{ .fc_id = 400, .cpu_id = 249, .valid = 1, .name = "HBM1_SPI_1" },
+	{ .fc_id = 401, .cpu_id = 250, .valid = 0, .name = "" },
+	{ .fc_id = 402, .cpu_id = 251, .valid = 0, .name = "" },
+	{ .fc_id = 403, .cpu_id = 252, .valid = 1, .name = "HBM2_SPI_0" },
+	{ .fc_id = 404, .cpu_id = 253, .valid = 1, .name = "HBM2_SPI_1" },
+	{ .fc_id = 405, .cpu_id = 254, .valid = 0, .name = "" },
+	{ .fc_id = 406, .cpu_id = 255, .valid = 0, .name = "" },
+	{ .fc_id = 407, .cpu_id = 256, .valid = 1, .name = "HBM3_SPI_0" },
+	{ .fc_id = 408, .cpu_id = 257, .valid = 1, .name = "HBM3_SPI_1" },
+	{ .fc_id = 409, .cpu_id = 258, .valid = 0, .name = "" },
+	{ .fc_id = 410, .cpu_id = 259, .valid = 0, .name = "" },
+	{ .fc_id = 411, .cpu_id = 260, .valid = 0, .name = "" },
+	{ .fc_id = 412, .cpu_id = 261, .valid = 0, .name = "" },
+	{ .fc_id = 413, .cpu_id = 262, .valid = 0, .name = "" },
+	{ .fc_id = 414, .cpu_id = 263, .valid = 0, .name = "" },
+	{ .fc_id = 415, .cpu_id = 264, .valid = 0, .name = "" },
+	{ .fc_id = 416, .cpu_id = 265, .valid = 0, .name = "" },
+	{ .fc_id = 417, .cpu_id = 266, .valid = 0, .name = "" },
+	{ .fc_id = 418, .cpu_id = 267, .valid = 0, .name = "" },
+	{ .fc_id = 419, .cpu_id = 268, .valid = 0, .name = "" },
+	{ .fc_id = 420, .cpu_id = 269, .valid = 0, .name = "" },
+	{ .fc_id = 421, .cpu_id = 270, .valid = 1, .name = "PSOC_GPIO_U16_0" },
+	{ .fc_id = 422, .cpu_id = 271, .valid = 0, .name = "" },
+	{ .fc_id = 423, .cpu_id = 272, .valid = 0, .name = "" },
+	{ .fc_id = 424, .cpu_id = 273, .valid = 0, .name = "" },
+	{ .fc_id = 425, .cpu_id = 274, .valid = 0, .name = "" },
+	{ .fc_id = 426, .cpu_id = 275, .valid = 0, .name = "" },
+	{ .fc_id = 427, .cpu_id = 276, .valid = 0, .name = "" },
+	{ .fc_id = 428, .cpu_id = 277, .valid = 0, .name = "" },
+	{ .fc_id = 429, .cpu_id = 278, .valid = 0, .name = "" },
+	{ .fc_id = 430, .cpu_id = 279, .valid = 0, .name = "" },
+	{ .fc_id = 431, .cpu_id = 280, .valid = 0, .name = "" },
+	{ .fc_id = 432, .cpu_id = 281, .valid = 0, .name = "" },
+	{ .fc_id = 433, .cpu_id = 282, .valid = 0, .name = "" },
+	{ .fc_id = 434, .cpu_id = 283, .valid = 0, .name = "" },
+	{ .fc_id = 435, .cpu_id = 284, .valid = 0, .name = "" },
+	{ .fc_id = 436, .cpu_id = 285, .valid = 0, .name = "" },
+	{ .fc_id = 437, .cpu_id = 286, .valid = 0, .name = "" },
+	{ .fc_id = 438, .cpu_id = 287, .valid = 0, .name = "" },
+	{ .fc_id = 439, .cpu_id = 288, .valid = 0, .name = "" },
+	{ .fc_id = 440, .cpu_id = 289, .valid = 0, .name = "" },
+	{ .fc_id = 441, .cpu_id = 290, .valid = 0, .name = "" },
+	{ .fc_id = 442, .cpu_id = 291, .valid = 0, .name = "" },
+	{ .fc_id = 443, .cpu_id = 292, .valid = 0, .name = "" },
+	{ .fc_id = 444, .cpu_id = 293, .valid = 0, .name = "" },
+	{ .fc_id = 445, .cpu_id = 294, .valid = 0, .name = "" },
+	{ .fc_id = 446, .cpu_id = 295, .valid = 0, .name = "" },
+	{ .fc_id = 447, .cpu_id = 296, .valid = 0, .name = "" },
+	{ .fc_id = 448, .cpu_id = 297, .valid = 0, .name = "" },
+	{ .fc_id = 449, .cpu_id = 298, .valid = 0, .name = "" },
+	{ .fc_id = 450, .cpu_id = 299, .valid = 0, .name = "" },
+	{ .fc_id = 451, .cpu_id = 300, .valid = 0, .name = "" },
+	{ .fc_id = 452, .cpu_id = 301, .valid = 0, .name = "" },
+	{ .fc_id = 453, .cpu_id = 302, .valid = 0, .name = "" },
+	{ .fc_id = 454, .cpu_id = 303, .valid = 0, .name = "" },
+	{ .fc_id = 455, .cpu_id = 304, .valid = 0, .name = "" },
+	{ .fc_id = 456, .cpu_id = 305, .valid = 0, .name = "" },
+	{ .fc_id = 457, .cpu_id = 306, .valid = 0, .name = "" },
+	{ .fc_id = 458, .cpu_id = 307, .valid = 0, .name = "" },
+	{ .fc_id = 459, .cpu_id = 308, .valid = 0, .name = "" },
+	{ .fc_id = 460, .cpu_id = 309, .valid = 0, .name = "" },
+	{ .fc_id = 461, .cpu_id = 310, .valid = 0, .name = "" },
+	{ .fc_id = 462, .cpu_id = 311, .valid = 0, .name = "" },
+	{ .fc_id = 463, .cpu_id = 312, .valid = 0, .name = "" },
+	{ .fc_id = 464, .cpu_id = 313, .valid = 0, .name = "" },
+	{ .fc_id = 465, .cpu_id = 314, .valid = 0, .name = "" },
+	{ .fc_id = 466, .cpu_id = 315, .valid = 0, .name = "" },
+	{ .fc_id = 467, .cpu_id = 316, .valid = 0, .name = "" },
+	{ .fc_id = 468, .cpu_id = 317, .valid = 0, .name = "" },
+	{ .fc_id = 469, .cpu_id = 318, .valid = 0, .name = "" },
+	{ .fc_id = 470, .cpu_id = 319, .valid = 0, .name = "" },
+	{ .fc_id = 471, .cpu_id = 320, .valid = 0, .name = "" },
+	{ .fc_id = 472, .cpu_id = 321, .valid = 0, .name = "" },
+	{ .fc_id = 473, .cpu_id = 322, .valid = 0, .name = "" },
+	{ .fc_id = 474, .cpu_id = 323, .valid = 0, .name = "" },
+	{ .fc_id = 475, .cpu_id = 324, .valid = 0, .name = "" },
+	{ .fc_id = 476, .cpu_id = 325, .valid = 0, .name = "" },
+	{ .fc_id = 477, .cpu_id = 326, .valid = 0, .name = "" },
+	{ .fc_id = 478, .cpu_id = 327, .valid = 0, .name = "" },
+	{ .fc_id = 479, .cpu_id = 328, .valid = 0, .name = "" },
+	{ .fc_id = 480, .cpu_id = 329, .valid = 0, .name = "" },
+	{ .fc_id = 481, .cpu_id = 330, .valid = 0, .name = "" },
+	{ .fc_id = 482, .cpu_id = 331, .valid = 0, .name = "" },
+	{ .fc_id = 483, .cpu_id = 332, .valid = 0, .name = "" },
+	{ .fc_id = 484, .cpu_id = 333, .valid = 1, .name = "PI_UPDATE" },
+	{ .fc_id = 485, .cpu_id = 334, .valid = 1, .name = "HALT_MACHINE" },
+	{ .fc_id = 486, .cpu_id = 335, .valid = 1, .name = "INTS_REGISTER" },
+	{ .fc_id = 487, .cpu_id = 336, .valid = 1, .name = "SOFT_RESET" },
+	{ .fc_id = 488, .cpu_id = 337, .valid = 0, .name = "" },
+	{ .fc_id = 489, .cpu_id = 338, .valid = 0, .name = "" },
+	{ .fc_id = 490, .cpu_id = 339, .valid = 0, .name = "" },
+	{ .fc_id = 491, .cpu_id = 340, .valid = 0, .name = "" },
+	{ .fc_id = 492, .cpu_id = 341, .valid = 0, .name = "" },
+	{ .fc_id = 493, .cpu_id = 342, .valid = 0, .name = "" },
+	{ .fc_id = 494, .cpu_id = 343, .valid = 0, .name = "" },
+	{ .fc_id = 495, .cpu_id = 344, .valid = 0, .name = "" },
+	{ .fc_id = 496, .cpu_id = 345, .valid = 0, .name = "" },
+	{ .fc_id = 497, .cpu_id = 346, .valid = 0, .name = "" },
+	{ .fc_id = 498, .cpu_id = 347, .valid = 0, .name = "" },
+	{ .fc_id = 499, .cpu_id = 348, .valid = 0, .name = "" },
+	{ .fc_id = 500, .cpu_id = 349, .valid = 0, .name = "" },
+	{ .fc_id = 501, .cpu_id = 350, .valid = 0, .name = "" },
+	{ .fc_id = 502, .cpu_id = 351, .valid = 0, .name = "" },
+	{ .fc_id = 503, .cpu_id = 352, .valid = 0, .name = "" },
+	{ .fc_id = 504, .cpu_id = 353, .valid = 0, .name = "" },
+	{ .fc_id = 505, .cpu_id = 354, .valid = 0, .name = "" },
+	{ .fc_id = 506, .cpu_id = 355, .valid = 0, .name = "" },
+	{ .fc_id = 507, .cpu_id = 356, .valid = 0, .name = "" },
+	{ .fc_id = 508, .cpu_id = 357, .valid = 0, .name = "" },
+	{ .fc_id = 509, .cpu_id = 358, .valid = 0, .name = "" },
+	{ .fc_id = 510, .cpu_id = 359, .valid = 0, .name = "" },
+	{ .fc_id = 511, .cpu_id = 360, .valid = 0, .name = "" },
+	{ .fc_id = 512, .cpu_id = 361, .valid = 0, .name = "" },
+	{ .fc_id = 513, .cpu_id = 362, .valid = 0, .name = "" },
+	{ .fc_id = 514, .cpu_id = 363, .valid = 0, .name = "" },
+	{ .fc_id = 515, .cpu_id = 364, .valid = 0, .name = "" },
+	{ .fc_id = 516, .cpu_id = 365, .valid = 0, .name = "" },
+	{ .fc_id = 517, .cpu_id = 366, .valid = 0, .name = "" },
+	{ .fc_id = 518, .cpu_id = 367, .valid = 0, .name = "" },
+	{ .fc_id = 519, .cpu_id = 368, .valid = 0, .name = "" },
+	{ .fc_id = 520, .cpu_id = 369, .valid = 0, .name = "" },
+	{ .fc_id = 521, .cpu_id = 370, .valid = 0, .name = "" },
+	{ .fc_id = 522, .cpu_id = 371, .valid = 0, .name = "" },
+	{ .fc_id = 523, .cpu_id = 372, .valid = 0, .name = "" },
+	{ .fc_id = 524, .cpu_id = 373, .valid = 0, .name = "" },
+	{ .fc_id = 525, .cpu_id = 374, .valid = 0, .name = "" },
+	{ .fc_id = 526, .cpu_id = 375, .valid = 0, .name = "" },
+	{ .fc_id = 527, .cpu_id = 376, .valid = 0, .name = "" },
+	{ .fc_id = 528, .cpu_id = 377, .valid = 0, .name = "" },
+	{ .fc_id = 529, .cpu_id = 378, .valid = 0, .name = "" },
+	{ .fc_id = 530, .cpu_id = 379, .valid = 0, .name = "" },
+	{ .fc_id = 531, .cpu_id = 380, .valid = 0, .name = "" },
+	{ .fc_id = 532, .cpu_id = 381, .valid = 0, .name = "" },
+	{ .fc_id = 533, .cpu_id = 382, .valid = 0, .name = "" },
+	{ .fc_id = 534, .cpu_id = 383, .valid = 0, .name = "" },
+	{ .fc_id = 535, .cpu_id = 384, .valid = 0, .name = "" },
+	{ .fc_id = 536, .cpu_id = 385, .valid = 0, .name = "" },
+	{ .fc_id = 537, .cpu_id = 386, .valid = 0, .name = "" },
+	{ .fc_id = 538, .cpu_id = 387, .valid = 0, .name = "" },
+	{ .fc_id = 539, .cpu_id = 388, .valid = 0, .name = "" },
+	{ .fc_id = 540, .cpu_id = 389, .valid = 0, .name = "" },
+	{ .fc_id = 541, .cpu_id = 390, .valid = 0, .name = "" },
+	{ .fc_id = 542, .cpu_id = 391, .valid = 0, .name = "" },
+	{ .fc_id = 543, .cpu_id = 392, .valid = 0, .name = "" },
+	{ .fc_id = 544, .cpu_id = 393, .valid = 0, .name = "" },
+	{ .fc_id = 545, .cpu_id = 394, .valid = 0, .name = "" },
+	{ .fc_id = 546, .cpu_id = 395, .valid = 0, .name = "" },
+	{ .fc_id = 547, .cpu_id = 396, .valid = 0, .name = "" },
+	{ .fc_id = 548, .cpu_id = 397, .valid = 1, .name = "RAZWI_OR_ADC" },
+	{ .fc_id = 549, .cpu_id = 398, .valid = 0, .name = "" },
+	{ .fc_id = 550, .cpu_id = 399, .valid = 0, .name = "" },
+	{ .fc_id = 551, .cpu_id = 400, .valid = 0, .name = "" },
+	{ .fc_id = 552, .cpu_id = 401, .valid = 0, .name = "" },
+	{ .fc_id = 553, .cpu_id = 402, .valid = 0, .name = "" },
+	{ .fc_id = 554, .cpu_id = 403, .valid = 0, .name = "" },
+	{ .fc_id = 555, .cpu_id = 404, .valid = 0, .name = "" },
+	{ .fc_id = 556, .cpu_id = 405, .valid = 0, .name = "" },
+	{ .fc_id = 557, .cpu_id = 406, .valid = 0, .name = "" },
+	{ .fc_id = 558, .cpu_id = 407, .valid = 0, .name = "" },
+	{ .fc_id = 559, .cpu_id = 408, .valid = 0, .name = "" },
+	{ .fc_id = 560, .cpu_id = 409, .valid = 0, .name = "" },
+	{ .fc_id = 561, .cpu_id = 410, .valid = 0, .name = "" },
+	{ .fc_id = 562, .cpu_id = 411, .valid = 0, .name = "" },
+	{ .fc_id = 563, .cpu_id = 412, .valid = 0, .name = "" },
+	{ .fc_id = 564, .cpu_id = 413, .valid = 0, .name = "" },
+	{ .fc_id = 565, .cpu_id = 414, .valid = 0, .name = "" },
+	{ .fc_id = 566, .cpu_id = 415, .valid = 0, .name = "" },
+	{ .fc_id = 567, .cpu_id = 416, .valid = 0, .name = "" },
+	{ .fc_id = 568, .cpu_id = 417, .valid = 0, .name = "" },
+	{ .fc_id = 569, .cpu_id = 418, .valid = 0, .name = "" },
+	{ .fc_id = 570, .cpu_id = 419, .valid = 0, .name = "" },
+	{ .fc_id = 571, .cpu_id = 420, .valid = 0, .name = "" },
+	{ .fc_id = 572, .cpu_id = 421, .valid = 1, .name = "TPC0_QM" },
+	{ .fc_id = 573, .cpu_id = 422, .valid = 1, .name = "TPC1_QM" },
+	{ .fc_id = 574, .cpu_id = 423, .valid = 1, .name = "TPC2_QM" },
+	{ .fc_id = 575, .cpu_id = 424, .valid = 1, .name = "TPC3_QM" },
+	{ .fc_id = 576, .cpu_id = 425, .valid = 1, .name = "TPC4_QM" },
+	{ .fc_id = 577, .cpu_id = 426, .valid = 1, .name = "TPC5_QM" },
+	{ .fc_id = 578, .cpu_id = 427, .valid = 1, .name = "TPC6_QM" },
+	{ .fc_id = 579, .cpu_id = 428, .valid = 1, .name = "TPC7_QM" },
+	{ .fc_id = 580, .cpu_id = 429, .valid = 0, .name = "" },
+	{ .fc_id = 581, .cpu_id = 430, .valid = 1, .name = "MME0_QM" },
+	{ .fc_id = 582, .cpu_id = 431, .valid = 1, .name = "MME2_QM" },
+	{ .fc_id = 583, .cpu_id = 432, .valid = 1, .name = "DMA0_QM" },
+	{ .fc_id = 584, .cpu_id = 433, .valid = 1, .name = "DMA1_QM" },
+	{ .fc_id = 585, .cpu_id = 434, .valid = 1, .name = "DMA2_QM" },
+	{ .fc_id = 586, .cpu_id = 435, .valid = 1, .name = "DMA3_QM" },
+	{ .fc_id = 587, .cpu_id = 436, .valid = 1, .name = "DMA4_QM" },
+	{ .fc_id = 588, .cpu_id = 437, .valid = 1, .name = "DMA5_QM" },
+	{ .fc_id = 589, .cpu_id = 438, .valid = 1, .name = "DMA6_QM" },
+	{ .fc_id = 590, .cpu_id = 439, .valid = 1, .name = "DMA7_QM" },
+	{ .fc_id = 591, .cpu_id = 440, .valid = 0, .name = "" },
+	{ .fc_id = 592, .cpu_id = 441, .valid = 0, .name = "" },
+	{ .fc_id = 593, .cpu_id = 442, .valid = 0, .name = "" },
+	{ .fc_id = 594, .cpu_id = 443, .valid = 1, .name = "NIC0_QM0" },
+	{ .fc_id = 595, .cpu_id = 444, .valid = 1, .name = "NIC0_QM1" },
+	{ .fc_id = 596, .cpu_id = 445, .valid = 1, .name = "NIC1_QM0" },
+	{ .fc_id = 597, .cpu_id = 446, .valid = 1, .name = "NIC1_QM1" },
+	{ .fc_id = 598, .cpu_id = 447, .valid = 1, .name = "NIC2_QM0" },
+	{ .fc_id = 599, .cpu_id = 448, .valid = 1, .name = "NIC2_QM1" },
+	{ .fc_id = 600, .cpu_id = 449, .valid = 1, .name = "NIC3_QM0" },
+	{ .fc_id = 601, .cpu_id = 450, .valid = 1, .name = "NIC3_QM1" },
+	{ .fc_id = 602, .cpu_id = 451, .valid = 1, .name = "NIC4_QM0" },
+	{ .fc_id = 603, .cpu_id = 452, .valid = 1, .name = "NIC4_QM1" },
+	{ .fc_id = 604, .cpu_id = 453, .valid = 1, .name = "DMA0_CORE" },
+	{ .fc_id = 605, .cpu_id = 454, .valid = 1, .name = "DMA1_CORE" },
+	{ .fc_id = 606, .cpu_id = 455, .valid = 1, .name = "DMA2_CORE" },
+	{ .fc_id = 607, .cpu_id = 456, .valid = 1, .name = "DMA3_CORE" },
+	{ .fc_id = 608, .cpu_id = 457, .valid = 1, .name = "DMA4_CORE" },
+	{ .fc_id = 609, .cpu_id = 458, .valid = 1, .name = "DMA5_CORE" },
+	{ .fc_id = 610, .cpu_id = 459, .valid = 1, .name = "DMA6_CORE" },
+	{ .fc_id = 611, .cpu_id = 460, .valid = 1, .name = "DMA7_CORE" },
+	{ .fc_id = 612, .cpu_id = 461, .valid = 1, .name = "NIC0_QP0" },
+	{ .fc_id = 613, .cpu_id = 462, .valid = 1, .name = "NIC0_QP1" },
+	{ .fc_id = 614, .cpu_id = 463, .valid = 1, .name = "NIC1_QP0" },
+	{ .fc_id = 615, .cpu_id = 464, .valid = 1, .name = "NIC1_QP1" },
+	{ .fc_id = 616, .cpu_id = 465, .valid = 1, .name = "NIC2_QP0" },
+	{ .fc_id = 617, .cpu_id = 466, .valid = 1, .name = "NIC2_QP1" },
+	{ .fc_id = 618, .cpu_id = 467, .valid = 1, .name = "NIC3_QP0" },
+	{ .fc_id = 619, .cpu_id = 468, .valid = 1, .name = "NIC3_QP1" },
+	{ .fc_id = 620, .cpu_id = 469, .valid = 1, .name = "NIC4_QP0" },
+	{ .fc_id = 621, .cpu_id = 470, .valid = 1, .name = "NIC4_QP1" },
+	{ .fc_id = 622, .cpu_id = 471, .valid = 0, .name = "" },
+	{ .fc_id = 623, .cpu_id = 472, .valid = 0, .name = "" },
+	{ .fc_id = 624, .cpu_id = 473, .valid = 0, .name = "" },
+	{ .fc_id = 625, .cpu_id = 474, .valid = 0, .name = "" },
+	{ .fc_id = 626, .cpu_id = 475, .valid = 0, .name = "" },
+	{ .fc_id = 627, .cpu_id = 476, .valid = 0, .name = "" },
+	{ .fc_id = 628, .cpu_id = 477, .valid = 0, .name = "" },
+	{ .fc_id = 629, .cpu_id = 478, .valid = 0, .name = "" },
+	{ .fc_id = 630, .cpu_id = 479, .valid = 0, .name = "" },
+	{ .fc_id = 631, .cpu_id = 480, .valid = 0, .name = "" },
+	{ .fc_id = 632, .cpu_id = 481, .valid = 0, .name = "" },
+	{ .fc_id = 633, .cpu_id = 482, .valid = 0, .name = "" },
+	{ .fc_id = 634, .cpu_id = 483, .valid = 0, .name = "" },
+	{ .fc_id = 635, .cpu_id = 484, .valid = 0, .name = "" },
+	{ .fc_id = 636, .cpu_id = 485, .valid = 0, .name = "" },
+	{ .fc_id = 637, .cpu_id = 486, .valid = 0, .name = "" },
+	{ .fc_id = 638, .cpu_id = 487, .valid = 0, .name = "" },
+	{ .fc_id = 639, .cpu_id = 488, .valid = 0, .name = "" },
+	{ .fc_id = 640, .cpu_id = 489, .valid = 0, .name = "" },
+	{ .fc_id = 641, .cpu_id = 490, .valid = 0, .name = "" },
+	{ .fc_id = 642, .cpu_id = 491, .valid = 0, .name = "" },
+	{ .fc_id = 643, .cpu_id = 492, .valid = 0, .name = "" },
+	{ .fc_id = 644, .cpu_id = 493, .valid = 0, .name = "" },
+	{ .fc_id = 645, .cpu_id = 494, .valid = 0, .name = "" },
+	{ .fc_id = 646, .cpu_id = 495, .valid = 0, .name = "" },
+	{ .fc_id = 647, .cpu_id = 496, .valid = 0, .name = "" },
+	{ .fc_id = 648, .cpu_id = 497, .valid = 0, .name = "" },
+	{ .fc_id = 649, .cpu_id = 498, .valid = 0, .name = "" },
+	{ .fc_id = 650, .cpu_id = 499, .valid = 0, .name = "" },
+	{ .fc_id = 651, .cpu_id = 500, .valid = 0, .name = "" },
+	{ .fc_id = 652, .cpu_id = 501, .valid = 0, .name = "" },
+	{ .fc_id = 653, .cpu_id = 502, .valid = 0, .name = "" },
+	{ .fc_id = 654, .cpu_id = 503, .valid = 0, .name = "" },
+	{ .fc_id = 655, .cpu_id = 504, .valid = 0, .name = "" },
+	{ .fc_id = 656, .cpu_id = 505, .valid = 0, .name = "" },
+	{ .fc_id = 657, .cpu_id = 506, .valid = 0, .name = "" },
+	{ .fc_id = 658, .cpu_id = 507, .valid = 1, .name = "FIX_POWER_ENV_S" },
+	{ .fc_id = 659, .cpu_id = 508, .valid = 1, .name = "FIX_POWER_ENV_E" },
+	{ .fc_id = 660, .cpu_id = 509, .valid = 1,
+		.name = "FIX_THERMAL_ENV_S" },
+	{ .fc_id = 661, .cpu_id = 510, .valid = 1,
+		.name = "FIX_THERMAL_ENV_E" },
+	{ .fc_id = 662, .cpu_id = 511, .valid = 1, .name = "RAZWI_OR_ADC_SW" },
+};
+
+#endif /* __GAUDI_ASYNC_IDS_MAP_EVENTS_EXT_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_coresight.h b/drivers/misc/habanalabs/include/gaudi/gaudi_coresight.h
new file mode 100644
index 000000000000..c45cc7f4d4d7
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_coresight.h
@@ -0,0 +1,367 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GAUDI_CORESIGHT_H
+#define GAUDI_CORESIGHT_H
+
+enum gaudi_debug_stm_regs_index {
+	GAUDI_STM_FIRST = 0,
+	GAUDI_STM_MME0_ACC = GAUDI_STM_FIRST,
+	GAUDI_STM_MME0_SBAB,
+	GAUDI_STM_MME0_CTRL,
+	GAUDI_STM_MME1_ACC,
+	GAUDI_STM_MME1_SBAB,
+	GAUDI_STM_MME1_CTRL,
+	GAUDI_STM_MME2_ACC,
+	GAUDI_STM_MME2_SBAB,
+	GAUDI_STM_MME2_CTRL,
+	GAUDI_STM_MME3_ACC,
+	GAUDI_STM_MME3_SBAB,
+	GAUDI_STM_MME3_CTRL,
+	GAUDI_STM_DMA_IF_W_S,
+	GAUDI_STM_DMA_IF_E_S,
+	GAUDI_STM_DMA_IF_W_N,
+	GAUDI_STM_DMA_IF_E_N,
+	GAUDI_STM_CPU,
+	GAUDI_STM_DMA_CH_0_CS,
+	GAUDI_STM_DMA_CH_1_CS,
+	GAUDI_STM_DMA_CH_2_CS,
+	GAUDI_STM_DMA_CH_3_CS,
+	GAUDI_STM_DMA_CH_4_CS,
+	GAUDI_STM_DMA_CH_5_CS,
+	GAUDI_STM_DMA_CH_6_CS,
+	GAUDI_STM_DMA_CH_7_CS,
+	GAUDI_STM_PCIE,
+	GAUDI_STM_MMU_CS,
+	GAUDI_STM_PSOC,
+	GAUDI_STM_NIC0_0,
+	GAUDI_STM_NIC0_1,
+	GAUDI_STM_NIC1_0,
+	GAUDI_STM_NIC1_1,
+	GAUDI_STM_NIC2_0,
+	GAUDI_STM_NIC2_1,
+	GAUDI_STM_NIC3_0,
+	GAUDI_STM_NIC3_1,
+	GAUDI_STM_NIC4_0,
+	GAUDI_STM_NIC4_1,
+	GAUDI_STM_TPC0_EML,
+	GAUDI_STM_TPC1_EML,
+	GAUDI_STM_TPC2_EML,
+	GAUDI_STM_TPC3_EML,
+	GAUDI_STM_TPC4_EML,
+	GAUDI_STM_TPC5_EML,
+	GAUDI_STM_TPC6_EML,
+	GAUDI_STM_TPC7_EML,
+	GAUDI_STM_LAST = GAUDI_STM_TPC7_EML
+};
+
+enum gaudi_debug_etf_regs_index {
+	GAUDI_ETF_FIRST = 0,
+	GAUDI_ETF_MME0_ACC = GAUDI_ETF_FIRST,
+	GAUDI_ETF_MME0_SBAB,
+	GAUDI_ETF_MME0_CTRL,
+	GAUDI_ETF_MME1_ACC,
+	GAUDI_ETF_MME1_SBAB,
+	GAUDI_ETF_MME1_CTRL,
+	GAUDI_ETF_MME2_ACC,
+	GAUDI_ETF_MME2_SBAB,
+	GAUDI_ETF_MME2_CTRL,
+	GAUDI_ETF_MME3_ACC,
+	GAUDI_ETF_MME3_SBAB,
+	GAUDI_ETF_MME3_CTRL,
+	GAUDI_ETF_DMA_IF_W_S,
+	GAUDI_ETF_DMA_IF_E_S,
+	GAUDI_ETF_DMA_IF_W_N,
+	GAUDI_ETF_DMA_IF_E_N,
+	GAUDI_ETF_CPU_0,
+	GAUDI_ETF_CPU_1,
+	GAUDI_ETF_CPU_TRACE,
+	GAUDI_ETF_DMA_CH_0_CS,
+	GAUDI_ETF_DMA_CH_1_CS,
+	GAUDI_ETF_DMA_CH_2_CS,
+	GAUDI_ETF_DMA_CH_3_CS,
+	GAUDI_ETF_DMA_CH_4_CS,
+	GAUDI_ETF_DMA_CH_5_CS,
+	GAUDI_ETF_DMA_CH_6_CS,
+	GAUDI_ETF_DMA_CH_7_CS,
+	GAUDI_ETF_PCIE,
+	GAUDI_ETF_MMU_CS,
+	GAUDI_ETF_PSOC,
+	GAUDI_ETF_NIC0_0,
+	GAUDI_ETF_NIC0_1,
+	GAUDI_ETF_NIC1_0,
+	GAUDI_ETF_NIC1_1,
+	GAUDI_ETF_NIC2_0,
+	GAUDI_ETF_NIC2_1,
+	GAUDI_ETF_NIC3_0,
+	GAUDI_ETF_NIC3_1,
+	GAUDI_ETF_NIC4_0,
+	GAUDI_ETF_NIC4_1,
+	GAUDI_ETF_TPC0_EML,
+	GAUDI_ETF_TPC1_EML,
+	GAUDI_ETF_TPC2_EML,
+	GAUDI_ETF_TPC3_EML,
+	GAUDI_ETF_TPC4_EML,
+	GAUDI_ETF_TPC5_EML,
+	GAUDI_ETF_TPC6_EML,
+	GAUDI_ETF_TPC7_EML,
+	GAUDI_ETF_LAST = GAUDI_ETF_TPC7_EML
+};
+
+enum gaudi_debug_funnel_regs_index {
+	GAUDI_FUNNEL_FIRST = 0,
+	GAUDI_FUNNEL_MME0_ACC = GAUDI_FUNNEL_FIRST,
+	GAUDI_FUNNEL_MME1_ACC,
+	GAUDI_FUNNEL_MME2_ACC,
+	GAUDI_FUNNEL_MME3_ACC,
+	GAUDI_FUNNEL_SRAM_Y0_X0,
+	GAUDI_FUNNEL_SRAM_Y0_X1,
+	GAUDI_FUNNEL_SRAM_Y0_X2,
+	GAUDI_FUNNEL_SRAM_Y0_X3,
+	GAUDI_FUNNEL_SRAM_Y0_X4,
+	GAUDI_FUNNEL_SRAM_Y0_X5,
+	GAUDI_FUNNEL_SRAM_Y0_X6,
+	GAUDI_FUNNEL_SRAM_Y0_X7,
+	GAUDI_FUNNEL_SRAM_Y1_X0,
+	GAUDI_FUNNEL_SRAM_Y1_X1,
+	GAUDI_FUNNEL_SRAM_Y1_X2,
+	GAUDI_FUNNEL_SRAM_Y1_X3,
+	GAUDI_FUNNEL_SRAM_Y1_X4,
+	GAUDI_FUNNEL_SRAM_Y1_X5,
+	GAUDI_FUNNEL_SRAM_Y1_X6,
+	GAUDI_FUNNEL_SRAM_Y1_X7,
+	GAUDI_FUNNEL_SRAM_Y2_X0,
+	GAUDI_FUNNEL_SRAM_Y2_X1,
+	GAUDI_FUNNEL_SRAM_Y2_X2,
+	GAUDI_FUNNEL_SRAM_Y2_X3,
+	GAUDI_FUNNEL_SRAM_Y2_X4,
+	GAUDI_FUNNEL_SRAM_Y2_X5,
+	GAUDI_FUNNEL_SRAM_Y2_X6,
+	GAUDI_FUNNEL_SRAM_Y2_X7,
+	GAUDI_FUNNEL_SRAM_Y3_X0,
+	GAUDI_FUNNEL_SRAM_Y3_X1,
+	GAUDI_FUNNEL_SRAM_Y3_X2,
+	GAUDI_FUNNEL_SRAM_Y3_X4,
+	GAUDI_FUNNEL_SRAM_Y3_X3,
+	GAUDI_FUNNEL_SRAM_Y3_X5,
+	GAUDI_FUNNEL_SRAM_Y3_X6,
+	GAUDI_FUNNEL_SRAM_Y3_X7,
+	GAUDI_FUNNEL_SIF_0,
+	GAUDI_FUNNEL_SIF_1,
+	GAUDI_FUNNEL_SIF_2,
+	GAUDI_FUNNEL_SIF_3,
+	GAUDI_FUNNEL_SIF_4,
+	GAUDI_FUNNEL_SIF_5,
+	GAUDI_FUNNEL_SIF_6,
+	GAUDI_FUNNEL_SIF_7,
+	GAUDI_FUNNEL_NIF_0,
+	GAUDI_FUNNEL_NIF_1,
+	GAUDI_FUNNEL_NIF_2,
+	GAUDI_FUNNEL_NIF_3,
+	GAUDI_FUNNEL_NIF_4,
+	GAUDI_FUNNEL_NIF_5,
+	GAUDI_FUNNEL_NIF_6,
+	GAUDI_FUNNEL_NIF_7,
+	GAUDI_FUNNEL_DMA_IF_W_S,
+	GAUDI_FUNNEL_DMA_IF_E_S,
+	GAUDI_FUNNEL_DMA_IF_W_N,
+	GAUDI_FUNNEL_DMA_IF_E_N,
+	GAUDI_FUNNEL_CPU,
+	GAUDI_FUNNEL_NIC_TPC_W_S,
+	GAUDI_FUNNEL_NIC_TPC_E_S,
+	GAUDI_FUNNEL_NIC_TPC_W_N,
+	GAUDI_FUNNEL_NIC_TPC_E_N,
+	GAUDI_FUNNEL_PCIE,
+	GAUDI_FUNNEL_PSOC,
+	GAUDI_FUNNEL_NIC0,
+	GAUDI_FUNNEL_NIC1,
+	GAUDI_FUNNEL_NIC2,
+	GAUDI_FUNNEL_NIC3,
+	GAUDI_FUNNEL_NIC4,
+	GAUDI_FUNNEL_TPC0_EML,
+	GAUDI_FUNNEL_TPC1_EML,
+	GAUDI_FUNNEL_TPC2_EML,
+	GAUDI_FUNNEL_TPC3_EML,
+	GAUDI_FUNNEL_TPC4_EML,
+	GAUDI_FUNNEL_TPC5_EML,
+	GAUDI_FUNNEL_TPC6_EML,
+	GAUDI_FUNNEL_TPC7_EML,
+	GAUDI_FUNNEL_LAST = GAUDI_FUNNEL_TPC7_EML
+};
+
+enum gaudi_debug_bmon_regs_index {
+	GAUDI_BMON_FIRST = 0,
+	GAUDI_BMON_MME0_ACC_0 = GAUDI_BMON_FIRST,
+	GAUDI_BMON_MME0_SBAB_0,
+	GAUDI_BMON_MME0_SBAB_1,
+	GAUDI_BMON_MME0_CTRL_0,
+	GAUDI_BMON_MME0_CTRL_1,
+	GAUDI_BMON_MME1_ACC_0,
+	GAUDI_BMON_MME1_SBAB_0,
+	GAUDI_BMON_MME1_SBAB_1,
+	GAUDI_BMON_MME1_CTRL_0,
+	GAUDI_BMON_MME1_CTRL_1,
+	GAUDI_BMON_MME2_ACC_0,
+	GAUDI_BMON_MME2_SBAB_0,
+	GAUDI_BMON_MME2_SBAB_1,
+	GAUDI_BMON_MME2_CTRL_0,
+	GAUDI_BMON_MME2_CTRL_1,
+	GAUDI_BMON_MME3_ACC_0,
+	GAUDI_BMON_MME3_SBAB_0,
+	GAUDI_BMON_MME3_SBAB_1,
+	GAUDI_BMON_MME3_CTRL_0,
+	GAUDI_BMON_MME3_CTRL_1,
+	GAUDI_BMON_DMA_IF_W_S_SOB_WR,
+	GAUDI_BMON_DMA_IF_W_S_0_WR,
+	GAUDI_BMON_DMA_IF_W_S_0_RD,
+	GAUDI_BMON_DMA_IF_W_S_1_WR,
+	GAUDI_BMON_DMA_IF_W_S_1_RD,
+	GAUDI_BMON_DMA_IF_E_S_SOB_WR,
+	GAUDI_BMON_DMA_IF_E_S_0_WR,
+	GAUDI_BMON_DMA_IF_E_S_0_RD,
+	GAUDI_BMON_DMA_IF_E_S_1_WR,
+	GAUDI_BMON_DMA_IF_E_S_1_RD,
+	GAUDI_BMON_DMA_IF_W_N_SOB_WR,
+	GAUDI_BMON_DMA_IF_W_N_HBM0_WR,
+	GAUDI_BMON_DMA_IF_W_N_HBM0_RD,
+	GAUDI_BMON_DMA_IF_W_N_HBM1_WR,
+	GAUDI_BMON_DMA_IF_W_N_HBM1_RD,
+	GAUDI_BMON_DMA_IF_E_N_SOB_WR,
+	GAUDI_BMON_DMA_IF_E_N_HBM0_WR,
+	GAUDI_BMON_DMA_IF_E_N_HBM0_RD,
+	GAUDI_BMON_DMA_IF_E_N_HBM1_WR,
+	GAUDI_BMON_DMA_IF_E_N_HBM1_RD,
+	GAUDI_BMON_CPU_WR,
+	GAUDI_BMON_CPU_RD,
+	GAUDI_BMON_DMA_CH_0_0,
+	GAUDI_BMON_DMA_CH_0_1,
+	GAUDI_BMON_DMA_CH_1_0,
+	GAUDI_BMON_DMA_CH_1_1,
+	GAUDI_BMON_DMA_CH_2_0,
+	GAUDI_BMON_DMA_CH_2_1,
+	GAUDI_BMON_DMA_CH_3_0,
+	GAUDI_BMON_DMA_CH_3_1,
+	GAUDI_BMON_DMA_CH_4_0,
+	GAUDI_BMON_DMA_CH_4_1,
+	GAUDI_BMON_DMA_CH_5_0,
+	GAUDI_BMON_DMA_CH_5_1,
+	GAUDI_BMON_DMA_CH_6_0,
+	GAUDI_BMON_DMA_CH_6_1,
+	GAUDI_BMON_DMA_CH_7_0,
+	GAUDI_BMON_DMA_CH_7_1,
+	GAUDI_BMON_PCIE_MSTR_WR,
+	GAUDI_BMON_PCIE_MSTR_RD,
+	GAUDI_BMON_PCIE_SLV_WR,
+	GAUDI_BMON_PCIE_SLV_RD,
+	GAUDI_BMON_MMU_0,
+	GAUDI_BMON_MMU_1,
+	GAUDI_BMON_NIC0_0,
+	GAUDI_BMON_NIC0_1,
+	GAUDI_BMON_NIC0_2,
+	GAUDI_BMON_NIC0_3,
+	GAUDI_BMON_NIC0_4,
+	GAUDI_BMON_NIC1_0,
+	GAUDI_BMON_NIC1_1,
+	GAUDI_BMON_NIC1_2,
+	GAUDI_BMON_NIC1_3,
+	GAUDI_BMON_NIC1_4,
+	GAUDI_BMON_NIC2_0,
+	GAUDI_BMON_NIC2_1,
+	GAUDI_BMON_NIC2_2,
+	GAUDI_BMON_NIC2_3,
+	GAUDI_BMON_NIC2_4,
+	GAUDI_BMON_NIC3_0,
+	GAUDI_BMON_NIC3_1,
+	GAUDI_BMON_NIC3_2,
+	GAUDI_BMON_NIC3_3,
+	GAUDI_BMON_NIC3_4,
+	GAUDI_BMON_NIC4_0,
+	GAUDI_BMON_NIC4_1,
+	GAUDI_BMON_NIC4_2,
+	GAUDI_BMON_NIC4_3,
+	GAUDI_BMON_NIC4_4,
+	GAUDI_BMON_TPC0_EML_0,
+	GAUDI_BMON_TPC0_EML_1,
+	GAUDI_BMON_TPC0_EML_2,
+	GAUDI_BMON_TPC0_EML_3,
+	GAUDI_BMON_TPC1_EML_0,
+	GAUDI_BMON_TPC1_EML_1,
+	GAUDI_BMON_TPC1_EML_2,
+	GAUDI_BMON_TPC1_EML_3,
+	GAUDI_BMON_TPC2_EML_0,
+	GAUDI_BMON_TPC2_EML_1,
+	GAUDI_BMON_TPC2_EML_2,
+	GAUDI_BMON_TPC2_EML_3,
+	GAUDI_BMON_TPC3_EML_0,
+	GAUDI_BMON_TPC3_EML_1,
+	GAUDI_BMON_TPC3_EML_2,
+	GAUDI_BMON_TPC3_EML_3,
+	GAUDI_BMON_TPC4_EML_0,
+	GAUDI_BMON_TPC4_EML_1,
+	GAUDI_BMON_TPC4_EML_2,
+	GAUDI_BMON_TPC4_EML_3,
+	GAUDI_BMON_TPC5_EML_0,
+	GAUDI_BMON_TPC5_EML_1,
+	GAUDI_BMON_TPC5_EML_2,
+	GAUDI_BMON_TPC5_EML_3,
+	GAUDI_BMON_TPC6_EML_0,
+	GAUDI_BMON_TPC6_EML_1,
+	GAUDI_BMON_TPC6_EML_2,
+	GAUDI_BMON_TPC6_EML_3,
+	GAUDI_BMON_TPC7_EML_0,
+	GAUDI_BMON_TPC7_EML_1,
+	GAUDI_BMON_TPC7_EML_2,
+	GAUDI_BMON_TPC7_EML_3,
+	GAUDI_BMON_LAST = GAUDI_BMON_TPC7_EML_3
+};
+
+enum gaudi_debug_spmu_regs_index {
+	GAUDI_SPMU_FIRST = 0,
+	GAUDI_SPMU_MME0_ACC = GAUDI_SPMU_FIRST,
+	GAUDI_SPMU_MME0_SBAB,
+	GAUDI_SPMU_MME0_CTRL,
+	GAUDI_SPMU_MME1_ACC,
+	GAUDI_SPMU_MME1_SBAB,
+	GAUDI_SPMU_MME1_CTRL,
+	GAUDI_SPMU_MME2_MME2_ACC,
+	GAUDI_SPMU_MME2_SBAB,
+	GAUDI_SPMU_MME2_CTRL,
+	GAUDI_SPMU_MME3_ACC,
+	GAUDI_SPMU_MME3_SBAB,
+	GAUDI_SPMU_MME3_CTRL,
+	GAUDI_SPMU_DMA_CH_0_CS,
+	GAUDI_SPMU_DMA_CH_1_CS,
+	GAUDI_SPMU_DMA_CH_2_CS,
+	GAUDI_SPMU_DMA_CH_3_CS,
+	GAUDI_SPMU_DMA_CH_4_CS,
+	GAUDI_SPMU_DMA_CH_5_CS,
+	GAUDI_SPMU_DMA_CH_6_CS,
+	GAUDI_SPMU_DMA_CH_7_CS,
+	GAUDI_SPMU_PCIE,
+	GAUDI_SPMU_MMU_CS,
+	GAUDI_SPMU_NIC0_0,
+	GAUDI_SPMU_NIC0_1,
+	GAUDI_SPMU_NIC1_0,
+	GAUDI_SPMU_NIC1_1,
+	GAUDI_SPMU_NIC2_0,
+	GAUDI_SPMU_NIC2_1,
+	GAUDI_SPMU_NIC3_0,
+	GAUDI_SPMU_NIC3_1,
+	GAUDI_SPMU_NIC4_0,
+	GAUDI_SPMU_NIC4_1,
+	GAUDI_SPMU_TPC0_EML,
+	GAUDI_SPMU_TPC1_EML,
+	GAUDI_SPMU_TPC2_EML,
+	GAUDI_SPMU_TPC3_EML,
+	GAUDI_SPMU_TPC4_EML,
+	GAUDI_SPMU_TPC5_EML,
+	GAUDI_SPMU_TPC6_EML,
+	GAUDI_SPMU_TPC7_EML,
+	GAUDI_SPMU_LAST = GAUDI_SPMU_TPC7_EML
+};
+
+#endif /* GAUDI_CORESIGHT_H */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h
new file mode 100644
index 000000000000..8aadc6357da1
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2019-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GAUDI_FW_IF_H
+#define GAUDI_FW_IF_H
+
+#define GAUDI_EVENT_QUEUE_MSI_IDX	8
+#define GAUDI_NIC_PORT1_MSI_IDX		10
+#define GAUDI_NIC_PORT3_MSI_IDX		12
+#define GAUDI_NIC_PORT5_MSI_IDX		14
+#define GAUDI_NIC_PORT7_MSI_IDX		16
+#define GAUDI_NIC_PORT9_MSI_IDX		18
+
+#define UBOOT_FW_OFFSET			0x100000	/* 1MB in SRAM */
+#define LINUX_FW_OFFSET			0x800000	/* 8MB in HBM */
+
+enum gaudi_pll_index {
+	CPU_PLL = 0,
+	PCI_PLL,
+	SRAM_PLL,
+	HBM_PLL,
+	NIC_PLL,
+	DMA_PLL,
+	MESH_PLL,
+	MME_PLL,
+	TPC_PLL,
+	IF_PLL
+};
+
+#define GAUDI_PLL_FREQ_LOW		200000000 /* 200 MHz */
+
+#endif /* GAUDI_FW_IF_H */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h
new file mode 100644
index 000000000000..96f08050ef0f
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h
@@ -0,0 +1,458 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GAUDI_MASKS_H_
+#define GAUDI_MASKS_H_
+
+#include "asic_reg/gaudi_regs.h"
+
+/* Useful masks for bits in various registers */
+#define PCI_DMA_QMAN_ENABLE		(\
+	(0xF << DMA0_QM_GLBL_CFG0_PQF_EN_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_CFG0_CQF_EN_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_CFG0_CP_EN_SHIFT))
+
+#define QMAN_EXTERNAL_MAKE_TRUSTED	(\
+	(0xF << DMA0_QM_GLBL_PROT_PQF_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_PROT_CQF_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_PROT_CP_SHIFT) | \
+	(0x1 << DMA0_QM_GLBL_PROT_ERR_SHIFT))
+
+#define QMAN_INTERNAL_MAKE_TRUSTED	(\
+	(0xF << DMA0_QM_GLBL_PROT_PQF_SHIFT) | \
+	(0x1 << DMA0_QM_GLBL_PROT_ERR_SHIFT))
+
+#define HBM_DMA_QMAN_ENABLE		(\
+	(0xF << DMA0_QM_GLBL_CFG0_PQF_EN_SHIFT) | \
+	(0x1F << DMA0_QM_GLBL_CFG0_CQF_EN_SHIFT) | \
+	(0x1F << DMA0_QM_GLBL_CFG0_CP_EN_SHIFT))
+
+#define QMAN_MME_ENABLE		(\
+	(0xF << MME0_QM_GLBL_CFG0_PQF_EN_SHIFT) | \
+	(0x1F << MME0_QM_GLBL_CFG0_CQF_EN_SHIFT) | \
+	(0x1F << MME0_QM_GLBL_CFG0_CP_EN_SHIFT))
+
+#define QMAN_TPC_ENABLE		(\
+	(0xF << TPC0_QM_GLBL_CFG0_PQF_EN_SHIFT) | \
+	(0x1F << TPC0_QM_GLBL_CFG0_CQF_EN_SHIFT) | \
+	(0x1F << TPC0_QM_GLBL_CFG0_CP_EN_SHIFT))
+
+#define QMAN_UPPER_CP_CGM_PWR_GATE_EN	(\
+	(0x20 << DMA0_QM_CGM_CFG_IDLE_TH_SHIFT) | \
+	(0xA << DMA0_QM_CGM_CFG_G2F_TH_SHIFT) | \
+	(0x10 << DMA0_QM_CGM_CFG_CP_IDLE_MASK_SHIFT) | \
+	(1 << DMA0_QM_CGM_CFG_EN_SHIFT))
+
+#define QMAN_COMMON_CP_CGM_PWR_GATE_EN	(\
+	(0x20 << DMA0_QM_CGM_CFG_IDLE_TH_SHIFT) | \
+	(0xA << DMA0_QM_CGM_CFG_G2F_TH_SHIFT) | \
+	(0xF << DMA0_QM_CGM_CFG_CP_IDLE_MASK_SHIFT) | \
+	(1 << DMA0_QM_CGM_CFG_EN_SHIFT))
+
+#define PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK	(\
+	(0xF << DMA0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_SHIFT))
+
+#define PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK	(\
+	(0xF << DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT) | \
+	(0xF << DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT))
+
+#define HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK	(\
+	(0xF << DMA0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_SHIFT) | \
+	(0x1F << DMA0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_SHIFT) | \
+	(0x1F << DMA0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_SHIFT))
+
+#define HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK	(\
+	(0xF << DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT) | \
+	(0x1F << DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT) | \
+	(0x1F << DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT))
+
+#define TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK	(\
+	(0xF << TPC0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_SHIFT) | \
+	(0x1F << TPC0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_SHIFT) | \
+	(0x1F << TPC0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_SHIFT))
+
+#define TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK	(\
+	(0xF << TPC0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT) | \
+	(0x1F << TPC0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT) | \
+	(0x1F << TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT))
+
+#define MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK	(\
+	(0xF << MME0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_SHIFT) | \
+	(0x1F << MME0_QM_GLBL_ERR_CFG_CQF_ERR_MSG_EN_SHIFT) | \
+	(0x1F << MME0_QM_GLBL_ERR_CFG_CP_ERR_MSG_EN_SHIFT))
+
+#define MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK	(\
+	(0xF << MME0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT) | \
+	(0x1F << MME0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT) | \
+	(0x1F << MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT))
+
+#define QMAN_CGM1_PWR_GATE_EN	(0xA << DMA0_QM_CGM_CFG1_MASK_TH_SHIFT)
+
+/* RESET registers configuration */
+#define CFG_RST_L_PSOC_SHIFT		0
+#define CFG_RST_L_PCIE_SHIFT		1
+#define CFG_RST_L_PCIE_IF_SHIFT		2
+#define CFG_RST_L_HBM_S_PLL_SHIFT	3
+#define CFG_RST_L_TPC_S_PLL_SHIFT	4
+#define CFG_RST_L_MME_S_PLL_SHIFT	5
+#define CFG_RST_L_CPU_PLL_SHIFT		6
+#define CFG_RST_L_PCIE_PLL_SHIFT	7
+#define CFG_RST_L_NIC_S_PLL_SHIFT	8
+#define CFG_RST_L_HBM_N_PLL_SHIFT	9
+#define CFG_RST_L_TPC_N_PLL_SHIFT	10
+#define CFG_RST_L_MME_N_PLL_SHIFT	11
+#define CFG_RST_L_NIC_N_PLL_SHIFT	12
+#define CFG_RST_L_DMA_W_PLL_SHIFT	13
+#define CFG_RST_L_SIF_W_PLL_SHIFT	14
+#define CFG_RST_L_MESH_W_PLL_SHIFT	15
+#define CFG_RST_L_SRAM_W_PLL_SHIFT	16
+#define CFG_RST_L_DMA_E_PLL_SHIFT	17
+#define CFG_RST_L_SIF_E_PLL_SHIFT	18
+#define CFG_RST_L_MESH_E_PLL_SHIFT	19
+#define CFG_RST_L_SRAM_E_PLL_SHIFT	20
+#define CFG_RST_L_IF_1_SHIFT		21
+#define CFG_RST_L_IF_0_SHIFT		22
+#define CFG_RST_L_IF_2_SHIFT		23
+#define CFG_RST_L_IF_3_SHIFT		24
+#define CFG_RST_L_TPC_0_SHIFT		25
+#define CFG_RST_L_TPC_1_SHIFT		26
+#define CFG_RST_L_TPC_2_SHIFT		27
+#define CFG_RST_L_TPC_3_SHIFT		28
+#define CFG_RST_L_TPC_4_SHIFT		29
+#define CFG_RST_L_TPC_5_SHIFT		30
+#define CFG_RST_L_TPC_6_SHIFT		31
+#define CFG_RST_H_TPC_7_SHIFT		0
+#define CFG_RST_H_MME_0_SHIFT		1
+#define CFG_RST_H_MME_1_SHIFT		2
+#define CFG_RST_H_MME_2_SHIFT		3
+#define CFG_RST_H_MME_3_SHIFT		4
+#define CFG_RST_H_HBM_0_SHIFT		5
+#define CFG_RST_H_HBM_1_SHIFT		6
+#define CFG_RST_H_HBM_2_SHIFT		7
+#define CFG_RST_H_HBM_3_SHIFT		8
+#define CFG_RST_H_NIC_0_SHIFT		9
+#define CFG_RST_H_NIC_1_SHIFT		10
+#define CFG_RST_H_NIC_2_SHIFT		11
+#define CFG_RST_H_NIC_3_SHIFT		12
+#define CFG_RST_H_NIC_4_SHIFT		13
+#define CFG_RST_H_SM_0_SHIFT		14
+#define CFG_RST_H_SM_1_SHIFT		15
+#define CFG_RST_H_SM_2_SHIFT		16
+#define CFG_RST_H_SM_3_SHIFT		17
+#define CFG_RST_H_DMA_0_SHIFT		18
+#define CFG_RST_H_DMA_1_SHIFT		19
+#define CFG_RST_H_CPU_SHIFT		20
+#define CFG_RST_H_MMU_SHIFT		21
+
+
+#define CFG_RST_H_DMA_MASK		((1 << CFG_RST_H_DMA_0_SHIFT) | \
+					(1 << CFG_RST_H_DMA_1_SHIFT))
+
+#define CFG_RST_H_CPU_MASK		(1 << CFG_RST_H_CPU_SHIFT)
+#define CFG_RST_H_MMU_MASK		(1 << CFG_RST_H_MMU_SHIFT)
+
+#define CFG_RST_H_HBM_MASK		((1 << CFG_RST_H_HBM_0_SHIFT) | \
+					(1 << CFG_RST_H_HBM_1_SHIFT) | \
+					(1 << CFG_RST_H_HBM_2_SHIFT) | \
+					(1 << CFG_RST_H_HBM_3_SHIFT))
+
+#define CFG_RST_H_NIC_MASK		((1 << CFG_RST_H_NIC_0_SHIFT) | \
+					(1 << CFG_RST_H_NIC_1_SHIFT) | \
+					(1 << CFG_RST_H_NIC_2_SHIFT) | \
+					(1 << CFG_RST_H_NIC_3_SHIFT) | \
+					(1 << CFG_RST_H_NIC_4_SHIFT))
+
+#define CFG_RST_H_SM_MASK		((1 << CFG_RST_H_SM_0_SHIFT) | \
+					(1 << CFG_RST_H_SM_1_SHIFT) | \
+					(1 << CFG_RST_H_SM_2_SHIFT) | \
+					(1 << CFG_RST_H_SM_3_SHIFT))
+
+#define CFG_RST_H_MME_MASK		((1 << CFG_RST_H_MME_0_SHIFT) | \
+					(1 << CFG_RST_H_MME_1_SHIFT) | \
+					(1 << CFG_RST_H_MME_2_SHIFT) | \
+					(1 << CFG_RST_H_MME_3_SHIFT))
+
+#define CFG_RST_L_PSOC_MASK		(1 << CFG_RST_L_PSOC_SHIFT)
+
+#define CFG_RST_L_IF_MASK		((1 << CFG_RST_L_IF_0_SHIFT) | \
+					(1 << CFG_RST_L_IF_1_SHIFT) | \
+					(1 << CFG_RST_L_IF_2_SHIFT) | \
+					(1 << CFG_RST_L_IF_3_SHIFT))
+
+#define CFG_RST_L_TPC_MASK		((1 << CFG_RST_L_TPC_0_SHIFT) | \
+					(1 << CFG_RST_L_TPC_1_SHIFT) | \
+					(1 << CFG_RST_L_TPC_2_SHIFT) | \
+					(1 << CFG_RST_L_TPC_3_SHIFT) | \
+					(1 << CFG_RST_L_TPC_4_SHIFT) | \
+					(1 << CFG_RST_L_TPC_5_SHIFT) | \
+					(1 << CFG_RST_L_TPC_6_SHIFT))
+
+#define CFG_RST_H_TPC_MASK		(1 << CFG_RST_H_TPC_7_SHIFT)
+
+#define CA53_RESET			(1 << CFG_RST_H_CPU_SHIFT)
+
+#define UNIT_RST_L_PSOC_SHIFT		0
+#define UNIT_RST_L_PCIE_SHIFT		1
+#define UNIT_RST_L_PCIE_IF_SHIFT	2
+#define UNIT_RST_L_HBM_S_PLL_SHIFT	3
+#define UNIT_RST_L_TPC_S_PLL_SHIFT	4
+#define UNIT_RST_L_MME_S_PLL_SHIFT	5
+#define UNIT_RST_L_CPU_PLL_SHIFT	6
+#define UNIT_RST_L_PCIE_PLL_SHIFT	7
+#define UNIT_RST_L_NIC_S_PLL_SHIFT	8
+#define UNIT_RST_L_HBM_N_PLL_SHIFT	9
+#define UNIT_RST_L_TPC_N_PLL_SHIFT	10
+#define UNIT_RST_L_MME_N_PLL_SHIFT	11
+#define UNIT_RST_L_NIC_N_PLL_SHIFT	12
+#define UNIT_RST_L_DMA_W_PLL_SHIFT	13
+#define UNIT_RST_L_SIF_W_PLL_SHIFT	14
+#define UNIT_RST_L_MESH_W_PLL_SHIFT	15
+#define UNIT_RST_L_SRAM_W_PLL_SHIFT	16
+#define UNIT_RST_L_DMA_E_PLL_SHIFT	17
+#define UNIT_RST_L_SIF_E_PLL_SHIFT	18
+#define UNIT_RST_L_MESH_E_PLL_SHIFT	19
+#define UNIT_RST_L_SRAM_E_PLL_SHIFT	20
+#define UNIT_RST_L_TPC_0_SHIFT		21
+#define UNIT_RST_L_TPC_1_SHIFT		22
+#define UNIT_RST_L_TPC_2_SHIFT		23
+#define UNIT_RST_L_TPC_3_SHIFT		24
+#define UNIT_RST_L_TPC_4_SHIFT		25
+#define UNIT_RST_L_TPC_5_SHIFT		26
+#define UNIT_RST_L_TPC_6_SHIFT		27
+#define UNIT_RST_L_TPC_7_SHIFT		28
+#define UNIT_RST_L_MME_0_SHIFT		29
+#define UNIT_RST_L_MME_1_SHIFT		30
+#define UNIT_RST_L_MME_2_SHIFT		31
+
+#define UNIT_RST_H_MME_3_SHIFT		0
+#define UNIT_RST_H_HBM_0_SHIFT		1
+#define UNIT_RST_H_HBM_1_SHIFT		2
+#define UNIT_RST_H_HBM_2_SHIFT		3
+#define UNIT_RST_H_HBM_3_SHIFT		4
+#define UNIT_RST_H_NIC_0_SHIFT		5
+#define UNIT_RST_H_NIC_1_SHIFT		6
+#define UNIT_RST_H_NIC_2_SHIFT		7
+#define UNIT_RST_H_NIC_3_SHIFT		8
+#define UNIT_RST_H_NIC_4_SHIFT		9
+#define UNIT_RST_H_SM_0_SHIFT		10
+#define UNIT_RST_H_SM_1_SHIFT		11
+#define UNIT_RST_H_SM_2_SHIFT		12
+#define UNIT_RST_H_SM_3_SHIFT		13
+#define UNIT_RST_H_IF_0_SHIFT		14
+#define UNIT_RST_H_IF_1_SHIFT		15
+#define UNIT_RST_H_IF_2_SHIFT		16
+#define UNIT_RST_H_IF_3_SHIFT		17
+#define UNIT_RST_H_DMA_0_SHIFT		18
+#define UNIT_RST_H_DMA_1_SHIFT		19
+#define UNIT_RST_H_CPU_SHIFT		20
+#define UNIT_RST_H_MMU_SHIFT		21
+
+#define UNIT_RST_H_HBM_MASK		((1 << UNIT_RST_H_HBM_0_SHIFT) | \
+					(1 << UNIT_RST_H_HBM_1_SHIFT) | \
+					(1 << UNIT_RST_H_HBM_2_SHIFT) | \
+					(1 << UNIT_RST_H_HBM_3_SHIFT))
+
+#define UNIT_RST_H_NIC_MASK		((1 << UNIT_RST_H_NIC_0_SHIFT) | \
+					(1 << UNIT_RST_H_NIC_1_SHIFT) | \
+					(1 << UNIT_RST_H_NIC_2_SHIFT) | \
+					(1 << UNIT_RST_H_NIC_3_SHIFT) | \
+					(1 << UNIT_RST_H_NIC_4_SHIFT))
+
+#define UNIT_RST_H_SM_MASK		((1 << UNIT_RST_H_SM_0_SHIFT) | \
+					(1 << UNIT_RST_H_SM_1_SHIFT) | \
+					(1 << UNIT_RST_H_SM_2_SHIFT) | \
+					(1 << UNIT_RST_H_SM_3_SHIFT))
+
+#define UNIT_RST_H_MME_MASK		((1 << UNIT_RST_H_MME_0_SHIFT) | \
+					(1 << UNIT_RST_H_MME_1_SHIFT) | \
+					(1 << UNIT_RST_H_MME_2_SHIFT))
+
+#define UNIT_RST_L_MME_MASK		(1 << UNIT_RST_L_MME_3_SHIFT)
+
+#define UNIT_RST_L_IF_MASK		((1 << UNIT_RST_L_IF_0_SHIFT) | \
+					(1 << UNIT_RST_L_IF_1_SHIFT) | \
+					(1 << UNIT_RST_L_IF_2_SHIFT) | \
+					(1 << UNIT_RST_L_IF_3_SHIFT))
+
+#define UNIT_RST_L_TPC_MASK		((1 << UNIT_RST_L_TPC_0_SHIFT) | \
+					(1 << UNIT_RST_L_TPC_1_SHIFT) | \
+					(1 << UNIT_RST_L_TPC_2_SHIFT) | \
+					(1 << UNIT_RST_L_TPC_3_SHIFT) | \
+					(1 << UNIT_RST_L_TPC_4_SHIFT) | \
+					(1 << UNIT_RST_L_TPC_5_SHIFT) | \
+					(1 << UNIT_RST_L_TPC_6_SHIFT) | \
+					(1 << UNIT_RST_L_TPC_7_SHIFT))
+
+/* CPU_CA53_CFG_ARM_RST_CONTROL */
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NCPUPORESET_SHIFT               0
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NCPUPORESET_MASK                0x3
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NCORERESET_SHIFT                4
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NCORERESET_MASK                 0x30
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NL2RESET_SHIFT                  8
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NL2RESET_MASK                   0x100
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NPRESETDBG_SHIFT                12
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NPRESETDBG_MASK                 0x1000
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NMBISTRESET_SHIFT               16
+#define CPU_CA53_CFG_ARM_RST_CONTROL_NMBISTRESET_MASK                0x10000
+#define CPU_CA53_CFG_ARM_RST_CONTROL_WARMRSTREQ_SHIFT                20
+#define CPU_CA53_CFG_ARM_RST_CONTROL_WARMRSTREQ_MASK                 0x300000
+
+#define CPU_RESET_ASSERT	(\
+			1 << CPU_CA53_CFG_ARM_RST_CONTROL_NMBISTRESET_SHIFT)
+
+#define CPU_RESET_CORE0_DEASSERT	(\
+			1 << CPU_CA53_CFG_ARM_RST_CONTROL_NCPUPORESET_SHIFT |\
+			1 << CPU_CA53_CFG_ARM_RST_CONTROL_NCORERESET_SHIFT |\
+			1 << CPU_CA53_CFG_ARM_RST_CONTROL_NL2RESET_SHIFT |\
+			1 << CPU_CA53_CFG_ARM_RST_CONTROL_NMBISTRESET_SHIFT)
+
+/* QM_IDLE_MASK is valid for all engines QM idle check */
+#define QM_IDLE_MASK	(DMA0_QM_GLBL_STS0_PQF_IDLE_MASK | \
+			DMA0_QM_GLBL_STS0_CQF_IDLE_MASK | \
+			DMA0_QM_GLBL_STS0_CP_IDLE_MASK)
+
+/* CGM_IDLE_MASK is valid for all engines CGM idle check */
+#define CGM_IDLE_MASK	DMA0_QM_CGM_STS_AGENT_IDLE_MASK
+
+#define TPC_IDLE_MASK	((1 << TPC0_CFG_STATUS_SCALAR_PIPE_EMPTY_SHIFT) | \
+			(1 << TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_SHIFT) | \
+			(1 << TPC0_CFG_STATUS_IQ_EMPTY_SHIFT) | \
+			(1 << TPC0_CFG_STATUS_SB_EMPTY_SHIFT) | \
+			(1 << TPC0_CFG_STATUS_QM_IDLE_SHIFT) | \
+			(1 << TPC0_CFG_STATUS_QM_RDY_SHIFT))
+
+#define MME0_CTRL_ARCH_STATUS_SB_A_EMPTY_MASK                        0x80
+#define MME0_CTRL_ARCH_STATUS_SB_B_EMPTY_MASK                        0x100
+#define MME0_CTRL_ARCH_STATUS_WBC_AXI_IDLE_MASK                      0x1000
+
+#define MME_ARCH_IDLE_MASK	(MME0_CTRL_ARCH_STATUS_SB_A_EMPTY_MASK | \
+				MME0_CTRL_ARCH_STATUS_SB_B_EMPTY_MASK | \
+				MME0_CTRL_ARCH_STATUS_WBC_AXI_IDLE_MASK)
+
+#define IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) \
+	((((qm_glbl_sts0) & QM_IDLE_MASK) == QM_IDLE_MASK) && \
+			(((qm_cgm_sts) & CGM_IDLE_MASK) == CGM_IDLE_MASK))
+
+#define IS_DMA_IDLE(dma_core_sts0) \
+	!(dma_core_sts0 & DMA0_CORE_STS0_BUSY_MASK)
+
+#define IS_TPC_IDLE(tpc_cfg_sts) \
+	(((tpc_cfg_sts) & TPC_IDLE_MASK) == TPC_IDLE_MASK)
+
+#define IS_MME_IDLE(mme_arch_sts) \
+	(((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
+
+enum axi_id {
+	AXI_ID_MME,
+	AXI_ID_TPC,
+	AXI_ID_DMA,
+	AXI_ID_NIC,	/* Local NIC */
+	AXI_ID_PCI,
+	AXI_ID_CPU,
+	AXI_ID_PSOC,
+	AXI_ID_MMU,
+	AXI_ID_NIC_FT	/* Feed-Through NIC */
+};
+
+/* RAZWI initiator ID is built from the location in the chip and the AXI ID */
+
+#define RAZWI_INITIATOR_AXI_ID_SHIFT	20
+#define RAZWI_INITIATOR_AXI_ID_MASK	0xF
+#define RAZWI_INITIATOR_X_SHIFT		24
+#define RAZWI_INITIATOR_X_MASK		0xF
+#define RAZWI_INITIATOR_Y_SHIFT		28
+#define RAZWI_INITIATOR_Y_MASK		0x7
+
+#define RAZWI_INITIATOR_ID_AXI_ID(axi_id) \
+	(((axi_id) & RAZWI_INITIATOR_AXI_ID_MASK) << \
+		RAZWI_INITIATOR_AXI_ID_SHIFT)
+
+#define RAZWI_INITIATOR_ID_X_Y(x, y) \
+	((((y) & RAZWI_INITIATOR_Y_MASK) << RAZWI_INITIATOR_Y_SHIFT) | \
+		(((x) & RAZWI_INITIATOR_X_MASK) << RAZWI_INITIATOR_X_SHIFT))
+
+#define RAZWI_INITIATOR_ID_X_Y_TPC0_NIC0	RAZWI_INITIATOR_ID_X_Y(1, 0)
+#define RAZWI_INITIATOR_ID_X_Y_TPC1		RAZWI_INITIATOR_ID_X_Y(2, 0)
+#define RAZWI_INITIATOR_ID_X_Y_MME0_0		RAZWI_INITIATOR_ID_X_Y(3, 0)
+#define RAZWI_INITIATOR_ID_X_Y_MME0_1		RAZWI_INITIATOR_ID_X_Y(4, 0)
+#define RAZWI_INITIATOR_ID_X_Y_MME1_0		RAZWI_INITIATOR_ID_X_Y(5, 0)
+#define RAZWI_INITIATOR_ID_X_Y_MME1_1		RAZWI_INITIATOR_ID_X_Y(6, 0)
+#define RAZWI_INITIATOR_ID_X_Y_TPC2		RAZWI_INITIATOR_ID_X_Y(7, 0)
+#define RAZWI_INITIATOR_ID_X_Y_TPC3_PCI_CPU_PSOC \
+						RAZWI_INITIATOR_ID_X_Y(8, 0)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0	RAZWI_INITIATOR_ID_X_Y(0, 1)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0	RAZWI_INITIATOR_ID_X_Y(9, 1)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1	RAZWI_INITIATOR_ID_X_Y(0, 2)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1	RAZWI_INITIATOR_ID_X_Y(9, 2)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0	RAZWI_INITIATOR_ID_X_Y(0, 3)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0	RAZWI_INITIATOR_ID_X_Y(9, 3)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1	RAZWI_INITIATOR_ID_X_Y(0, 4)
+#define RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1	RAZWI_INITIATOR_ID_X_Y(9, 4)
+#define RAZWI_INITIATOR_ID_X_Y_TPC4_NIC1_NIC2	RAZWI_INITIATOR_ID_X_Y(1, 5)
+#define RAZWI_INITIATOR_ID_X_Y_TPC5		RAZWI_INITIATOR_ID_X_Y(2, 5)
+#define RAZWI_INITIATOR_ID_X_Y_MME2_0		RAZWI_INITIATOR_ID_X_Y(3, 5)
+#define RAZWI_INITIATOR_ID_X_Y_MME2_1		RAZWI_INITIATOR_ID_X_Y(4, 5)
+#define RAZWI_INITIATOR_ID_X_Y_MME3_0		RAZWI_INITIATOR_ID_X_Y(5, 5)
+#define RAZWI_INITIATOR_ID_X_Y_MME3_1		RAZWI_INITIATOR_ID_X_Y(6, 5)
+#define RAZWI_INITIATOR_ID_X_Y_TPC6		RAZWI_INITIATOR_ID_X_Y(7, 5)
+#define RAZWI_INITIATOR_ID_X_Y_TPC7_NIC4_NIC5	RAZWI_INITIATOR_ID_X_Y(8, 5)
+
+#define PSOC_ETR_AXICTL_PROTCTRLBIT1_SHIFT                           1
+
+/* STLB_CACHE_INV */
+#define STLB_CACHE_INV_PRODUCER_INDEX_SHIFT                          0
+#define STLB_CACHE_INV_PRODUCER_INDEX_MASK                           0xFF
+#define STLB_CACHE_INV_INDEX_MASK_SHIFT                              8
+#define STLB_CACHE_INV_INDEX_MASK_MASK                               0xFF00
+
+#define MME_ACC_ACC_STALL_R_SHIFT                                    0
+#define MME_SBAB_SB_STALL_R_SHIFT                                    0
+
+#define PCIE_WRAP_LBW_PROT_OVR_RD_EN_MASK                            0x700
+#define PCIE_WRAP_LBW_PROT_OVR_WR_EN_MASK                            0x7000
+
+#define PCIE_WRAP_LBW_DRAIN_CFG_EN_SHIFT                             0
+#define PCIE_WRAP_HBW_DRAIN_CFG_EN_SHIFT                             0
+
+/* DMA_IF_HBM_CRED_EN */
+#define DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT                      0
+#define DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_MASK                       0x1
+#define DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT                     1
+#define DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_MASK                      0x2
+
+#define DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT                      0
+#define DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT                       0
+#define DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT                         0
+#define DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT                         0
+
+#define IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT                          0
+#define IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT                           0
+
+#define IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT                             0
+#define IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT                             0
+
+/* MMU_UP_PAGE_ERROR_CAPTURE */
+#define MMU_UP_PAGE_ERROR_CAPTURE_VA_49_32_MASK                      0x3FFFF
+#define MMU_UP_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK                   0x40000
+
+/* MMU_UP_ACCESS_ERROR_CAPTURE */
+#define MMU_UP_ACCESS_ERROR_CAPTURE_VA_49_32_MASK                    0x3FFFF
+#define MMU_UP_ACCESS_ERROR_CAPTURE_ENTRY_VALID_MASK                 0x40000
+
+#define QM_ARB_ERR_MSG_EN_CHOISE_OVF_MASK                            0x1
+#define QM_ARB_ERR_MSG_EN_CHOISE_WDT_MASK                            0x2
+#define QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_MASK                           0x4
+
+#define QM_ARB_ERR_MSG_EN_MASK		(\
+					QM_ARB_ERR_MSG_EN_CHOISE_OVF_MASK |\
+					QM_ARB_ERR_MSG_EN_CHOISE_WDT_MASK |\
+					QM_ARB_ERR_MSG_EN_AXI_LBW_ERR_MASK)
+
+#endif /* GAUDI_MASKS_H_ */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h
new file mode 100644
index 000000000000..9a5800b0086b
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_packets.h
@@ -0,0 +1,212 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2017-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GAUDI_PACKETS_H
+#define GAUDI_PACKETS_H
+
+#include <linux/types.h>
+
+#define PACKET_HEADER_PACKET_ID_SHIFT		56
+#define PACKET_HEADER_PACKET_ID_MASK		0x1F00000000000000ull
+
+enum packet_id {
+	PACKET_WREG_32 = 0x1,
+	PACKET_WREG_BULK = 0x2,
+	PACKET_MSG_LONG = 0x3,
+	PACKET_MSG_SHORT = 0x4,
+	PACKET_CP_DMA = 0x5,
+	PACKET_REPEAT = 0x6,
+	PACKET_MSG_PROT = 0x7,
+	PACKET_FENCE = 0x8,
+	PACKET_LIN_DMA = 0x9,
+	PACKET_NOP = 0xA,
+	PACKET_STOP = 0xB,
+	PACKET_ARB_POINT = 0xC,
+	PACKET_WAIT = 0xD,
+	PACKET_LOAD_AND_EXE = 0xF,
+	MAX_PACKET_ID = (PACKET_HEADER_PACKET_ID_MASK >>
+				PACKET_HEADER_PACKET_ID_SHIFT) + 1
+};
+
+#define GAUDI_PKT_CTL_OPCODE_SHIFT	24
+#define GAUDI_PKT_CTL_OPCODE_MASK	0x1F000000
+
+#define GAUDI_PKT_CTL_EB_SHIFT		29
+#define GAUDI_PKT_CTL_EB_MASK		0x20000000
+
+#define GAUDI_PKT_CTL_RB_SHIFT		30
+#define GAUDI_PKT_CTL_RB_MASK		0x40000000
+
+#define GAUDI_PKT_CTL_MB_SHIFT		31
+#define GAUDI_PKT_CTL_MB_MASK		0x80000000
+
+/* All packets have, at least, an 8-byte header, which contains
+ * the packet type. The kernel driver uses the packet header for packet
+ * validation and to perform any necessary required preparation before
+ * sending them off to the hardware.
+ */
+struct gaudi_packet {
+	__le64 header;
+	/* The rest of the packet data follows. Use the corresponding
+	 * packet_XXX struct to deference the data, based on packet type
+	 */
+	u8 contents[0];
+};
+
+struct packet_nop {
+	__le32 reserved;
+	__le32 ctl;
+};
+
+struct packet_stop {
+	__le32 reserved;
+	__le32 ctl;
+};
+
+struct packet_wreg32 {
+	__le32 value;
+	__le32 ctl;
+};
+
+struct packet_wreg_bulk {
+	__le32 size64;
+	__le32 ctl;
+	__le64 values[0]; /* data starts here */
+};
+
+struct packet_msg_long {
+	__le32 value;
+	__le32 ctl;
+	__le64 addr;
+};
+
+#define GAUDI_PKT_SHORT_VAL_SOB_SYNC_VAL_SHIFT	0
+#define GAUDI_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK	0x0000EFFF
+
+#define GAUDI_PKT_SHORT_VAL_SOB_MOD_SHIFT	31
+#define GAUDI_PKT_SHORT_VAL_SOB_MOD_MASK	0x80000000
+
+#define GAUDI_PKT_SHORT_VAL_MON_SYNC_GID_SHIFT	0
+#define GAUDI_PKT_SHORT_VAL_MON_SYNC_GID_MASK	0x000000FF
+
+#define GAUDI_PKT_SHORT_VAL_MON_MASK_SHIFT	8
+#define GAUDI_PKT_SHORT_VAL_MON_MASK_MASK	0x0000FF00
+
+#define GAUDI_PKT_SHORT_VAL_MON_MODE_SHIFT	16
+#define GAUDI_PKT_SHORT_VAL_MON_MODE_MASK	0x00010000
+
+#define GAUDI_PKT_SHORT_VAL_MON_SYNC_VAL_SHIFT	17
+#define GAUDI_PKT_SHORT_VAL_MON_SYNC_VAL_MASK	0xFFFE0000
+
+#define GAUDI_PKT_SHORT_CTL_ADDR_SHIFT		0
+#define GAUDI_PKT_SHORT_CTL_ADDR_MASK		0x0000FFFF
+
+#define GAUDI_PKT_SHORT_CTL_OP_SHIFT		20
+#define GAUDI_PKT_SHORT_CTL_OP_MASK		0x00300000
+
+#define GAUDI_PKT_SHORT_CTL_BASE_SHIFT		22
+#define GAUDI_PKT_SHORT_CTL_BASE_MASK		0x00C00000
+
+#define GAUDI_PKT_SHORT_CTL_OPCODE_SHIFT	24
+#define GAUDI_PKT_SHORT_CTL_OPCODE_MASK		0x1F000000
+
+#define GAUDI_PKT_SHORT_CTL_EB_SHIFT		29
+#define GAUDI_PKT_SHORT_CTL_EB_MASK		0x20000000
+
+#define GAUDI_PKT_SHORT_CTL_RB_SHIFT		30
+#define GAUDI_PKT_SHORT_CTL_RB_MASK		0x40000000
+
+#define GAUDI_PKT_SHORT_CTL_MB_SHIFT		31
+#define GAUDI_PKT_SHORT_CTL_MB_MASK		0x80000000
+
+struct packet_msg_short {
+	__le32 value;
+	__le32 ctl;
+};
+
+struct packet_msg_prot {
+	__le32 value;
+	__le32 ctl;
+	__le64 addr;
+};
+
+#define GAUDI_PKT_FENCE_CFG_DEC_VAL_SHIFT	0
+#define GAUDI_PKT_FENCE_CFG_DEC_VAL_MASK	0x0000000F
+
+#define GAUDI_PKT_FENCE_CFG_TARGET_VAL_SHIFT	16
+#define GAUDI_PKT_FENCE_CFG_TARGET_VAL_MASK	0x00FF0000
+
+#define GAUDI_PKT_FENCE_CFG_ID_SHIFT		30
+#define GAUDI_PKT_FENCE_CFG_ID_MASK		0xC000000
+
+#define GAUDI_PKT_FENCE_CTL_PRED_SHIFT		0
+#define GAUDI_PKT_FENCE_CTL_PRED_MASK		0x0000001F
+
+#define GAUDI_PKT_FENCE_CTL_OPCODE_SHIFT	24
+#define GAUDI_PKT_FENCE_CTL_OPCODE_MASK		0x1F000000
+
+#define GAUDI_PKT_FENCE_CTL_EB_SHIFT		29
+#define GAUDI_PKT_FENCE_CTL_EB_MASK		0x20000000
+
+#define GAUDI_PKT_FENCE_CTL_RB_SHIFT		30
+#define GAUDI_PKT_FENCE_CTL_RB_MASK		0x40000000
+
+#define GAUDI_PKT_FENCE_CTL_MB_SHIFT		31
+#define GAUDI_PKT_FENCE_CTL_MB_MASK		0x80000000
+
+struct packet_fence {
+	__le32 cfg;
+	__le32 ctl;
+};
+
+#define GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_SHIFT	0
+#define GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK	0x00000001
+
+#define GAUDI_PKT_LIN_DMA_CTL_LIN_SHIFT		3
+#define GAUDI_PKT_LIN_DMA_CTL_LIN_MASK		0x00000008
+
+#define GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT	4
+#define GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK	0x00000010
+
+#define GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT	0
+#define GAUDI_PKT_LIN_DMA_DST_ADDR_MASK		0x00FFFFFFFFFFFFFFull
+
+struct packet_lin_dma {
+	__le32 tsize;
+	__le32 ctl;
+	__le64 src_addr;
+	__le64 dst_addr;
+};
+
+struct packet_arb_point {
+	__le32 cfg;
+	__le32 ctl;
+};
+
+struct packet_repeat {
+	__le32 cfg;
+	__le32 ctl;
+};
+
+struct packet_wait {
+	__le32 cfg;
+	__le32 ctl;
+};
+
+struct packet_load_and_exe {
+	__le32 cfg;
+	__le32 ctl;
+	__le64 src_addr;
+};
+
+struct packet_cp_dma {
+	__le32 tsize;
+	__le32 ctl;
+	__le64 src_addr;
+};
+
+#endif /* GAUDI_PACKETS_H */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h
new file mode 100644
index 000000000000..f25c60a2c243
--- /dev/null
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2019-2020 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef GAUDI_REG_MAP_H_
+#define GAUDI_REG_MAP_H_
+
+/*
+ * PSOC scratch-pad registers
+ */
+#define mmHW_STATE			mmPSOC_GLOBAL_CONF_SCRATCHPAD_0
+#define mmCPU_CMD_STATUS_TO_HOST	mmPSOC_GLOBAL_CONF_SCRATCHPAD_23
+#define mmCPU_BOOT_ERR0			mmPSOC_GLOBAL_CONF_SCRATCHPAD_24
+#define mmCPU_BOOT_ERR1			mmPSOC_GLOBAL_CONF_SCRATCHPAD_25
+#define mmUPD_STS			mmPSOC_GLOBAL_CONF_SCRATCHPAD_26
+#define mmUPD_CMD			mmPSOC_GLOBAL_CONF_SCRATCHPAD_27
+#define mmPREBOOT_VER_OFFSET		mmPSOC_GLOBAL_CONF_SCRATCHPAD_28
+#define mmUBOOT_VER_OFFSET		mmPSOC_GLOBAL_CONF_SCRATCHPAD_29
+#define mmRDWR_TEST			mmPSOC_GLOBAL_CONF_SCRATCHPAD_30
+#define mmBTL_ID			mmPSOC_GLOBAL_CONF_SCRATCHPAD_31
+#define mmPREBOOT_PCIE_EN		mmPSOC_GLOBAL_CONF_COLD_RST_FLOPS_1
+#define mmUPD_PENDING_STS		mmPSOC_GLOBAL_CONF_COLD_RST_FLOPS_3
+
+#endif /* GAUDI_REG_MAP_H_ */
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h
index 3c44ef3a23ed..067489bd048e 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/goya_masks.h
@@ -55,8 +55,7 @@
 	(1 << DMA_QM_0_GLBL_ERR_CFG_DMA_ERR_MSG_EN_SHIFT) | \
 	(1 << DMA_QM_0_GLBL_ERR_CFG_PQF_STOP_ON_ERR_SHIFT) | \
 	(1 << DMA_QM_0_GLBL_ERR_CFG_CQF_STOP_ON_ERR_SHIFT) | \
-	(1 << DMA_QM_0_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT) | \
-	(1 << DMA_QM_0_GLBL_ERR_CFG_DMA_STOP_ON_ERR_SHIFT))
+	(1 << DMA_QM_0_GLBL_ERR_CFG_CP_STOP_ON_ERR_SHIFT))
 
 #define QMAN_MME_ENABLE		(\
 	(1 << MME_QM_GLBL_CFG0_PQF_EN_SHIFT) | \
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h
index fce490e6a231..ce65c9da5c60 100644
--- a/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/goya_regs.h
@@ -18,6 +18,7 @@
 #include "psoc_mme_pll_regs.h"
 #include "psoc_pci_pll_regs.h"
 #include "psoc_emmc_pll_regs.h"
+#include "psoc_timestamp_regs.h"
 #include "cpu_if_regs.h"
 #include "cpu_ca53_cfg_regs.h"
 #include "cpu_pll_regs.h"
diff --git a/drivers/misc/habanalabs/include/goya/asic_reg/psoc_timestamp_regs.h b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_timestamp_regs.h
new file mode 100644
index 000000000000..9ce24597d4b0
--- /dev/null
+++ b/drivers/misc/habanalabs/include/goya/asic_reg/psoc_timestamp_regs.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2016-2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+/************************************
+ ** This is an auto-generated file **
+ **       DO NOT EDIT BELOW        **
+ ************************************/
+
+#ifndef ASIC_REG_PSOC_TIMESTAMP_REGS_H_
+#define ASIC_REG_PSOC_TIMESTAMP_REGS_H_
+
+/*
+ *****************************************
+ *   PSOC_TIMESTAMP (Prototype: TIMESTAMP)
+ *****************************************
+ */
+
+#define mmPSOC_TIMESTAMP_CNTCR                                       0xC49000
+
+#define mmPSOC_TIMESTAMP_CNTSR                                       0xC49004
+
+#define mmPSOC_TIMESTAMP_CNTCVL                                      0xC49008
+
+#define mmPSOC_TIMESTAMP_CNTCVU                                      0xC4900C
+
+#define mmPSOC_TIMESTAMP_CNTFID0                                     0xC49020
+
+#define mmPSOC_TIMESTAMP_PIDR4                                       0xC49FD0
+
+#define mmPSOC_TIMESTAMP_PIDR5                                       0xC49FD4
+
+#define mmPSOC_TIMESTAMP_PIDR6                                       0xC49FD8
+
+#define mmPSOC_TIMESTAMP_PIDR7                                       0xC49FDC
+
+#define mmPSOC_TIMESTAMP_PIDR0                                       0xC49FE0
+
+#define mmPSOC_TIMESTAMP_PIDR1                                       0xC49FE4
+
+#define mmPSOC_TIMESTAMP_PIDR2                                       0xC49FE8
+
+#define mmPSOC_TIMESTAMP_PIDR3                                       0xC49FEC
+
+#define mmPSOC_TIMESTAMP_CIDR0                                       0xC49FF0
+
+#define mmPSOC_TIMESTAMP_CIDR1                                       0xC49FF4
+
+#define mmPSOC_TIMESTAMP_CIDR2                                       0xC49FF8
+
+#define mmPSOC_TIMESTAMP_CIDR3                                       0xC49FFC
+
+#endif /* ASIC_REG_PSOC_TIMESTAMP_REGS_H_ */
diff --git a/drivers/misc/habanalabs/include/goya/goya_reg_map.h b/drivers/misc/habanalabs/include/goya/goya_reg_map.h
index 08061282cd9c..0195f62d7254 100644
--- a/drivers/misc/habanalabs/include/goya/goya_reg_map.h
+++ b/drivers/misc/habanalabs/include/goya/goya_reg_map.h
@@ -11,27 +11,30 @@
 /*
  * PSOC scratch-pad registers
  */
-#define mmCPU_PQ_BASE_ADDR_LOW			mmPSOC_GLOBAL_CONF_SCRATCHPAD_0
-#define mmCPU_PQ_BASE_ADDR_HIGH			mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
-#define mmCPU_EQ_BASE_ADDR_LOW			mmPSOC_GLOBAL_CONF_SCRATCHPAD_2
-#define mmCPU_EQ_BASE_ADDR_HIGH			mmPSOC_GLOBAL_CONF_SCRATCHPAD_3
-#define mmCPU_EQ_LENGTH				mmPSOC_GLOBAL_CONF_SCRATCHPAD_4
-#define mmCPU_PQ_LENGTH				mmPSOC_GLOBAL_CONF_SCRATCHPAD_5
-#define mmCPU_EQ_CI				mmPSOC_GLOBAL_CONF_SCRATCHPAD_6
-#define mmCPU_PQ_INIT_STATUS			mmPSOC_GLOBAL_CONF_SCRATCHPAD_7
-#define mmCPU_CQ_BASE_ADDR_LOW			mmPSOC_GLOBAL_CONF_SCRATCHPAD_8
-#define mmCPU_CQ_BASE_ADDR_HIGH			mmPSOC_GLOBAL_CONF_SCRATCHPAD_9
-#define mmCPU_CQ_LENGTH				mmPSOC_GLOBAL_CONF_SCRATCHPAD_10
-#define mmCPU_BOOT_ERR0				mmPSOC_GLOBAL_CONF_SCRATCHPAD_24
-#define mmCPU_BOOT_ERR1				mmPSOC_GLOBAL_CONF_SCRATCHPAD_25
-#define mmUPD_STS				mmPSOC_GLOBAL_CONF_SCRATCHPAD_26
-#define mmUPD_CMD				mmPSOC_GLOBAL_CONF_SCRATCHPAD_27
-#define mmPREBOOT_VER_OFFSET			mmPSOC_GLOBAL_CONF_SCRATCHPAD_28
-#define mmUBOOT_VER_OFFSET			mmPSOC_GLOBAL_CONF_SCRATCHPAD_29
-#define mmRDWR_TEST				mmPSOC_GLOBAL_CONF_SCRATCHPAD_30
-#define mmBTL_ID				mmPSOC_GLOBAL_CONF_SCRATCHPAD_31
+#define mmCPU_PQ_BASE_ADDR_LOW		mmPSOC_GLOBAL_CONF_SCRATCHPAD_0
+#define mmCPU_PQ_BASE_ADDR_HIGH		mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
+#define mmCPU_EQ_BASE_ADDR_LOW		mmPSOC_GLOBAL_CONF_SCRATCHPAD_2
+#define mmCPU_EQ_BASE_ADDR_HIGH		mmPSOC_GLOBAL_CONF_SCRATCHPAD_3
+#define mmCPU_EQ_LENGTH			mmPSOC_GLOBAL_CONF_SCRATCHPAD_4
+#define mmCPU_PQ_LENGTH			mmPSOC_GLOBAL_CONF_SCRATCHPAD_5
+#define mmCPU_EQ_CI			mmPSOC_GLOBAL_CONF_SCRATCHPAD_6
+#define mmCPU_PQ_INIT_STATUS		mmPSOC_GLOBAL_CONF_SCRATCHPAD_7
+#define mmCPU_CQ_BASE_ADDR_LOW		mmPSOC_GLOBAL_CONF_SCRATCHPAD_8
+#define mmCPU_CQ_BASE_ADDR_HIGH		mmPSOC_GLOBAL_CONF_SCRATCHPAD_9
+#define mmCPU_CQ_LENGTH			mmPSOC_GLOBAL_CONF_SCRATCHPAD_10
+#define mmCPU_CMD_STATUS_TO_HOST	mmPSOC_GLOBAL_CONF_SCRATCHPAD_23
+#define mmCPU_BOOT_ERR0			mmPSOC_GLOBAL_CONF_SCRATCHPAD_24
+#define mmCPU_BOOT_ERR1			mmPSOC_GLOBAL_CONF_SCRATCHPAD_25
+#define mmUPD_STS			mmPSOC_GLOBAL_CONF_SCRATCHPAD_26
+#define mmUPD_CMD			mmPSOC_GLOBAL_CONF_SCRATCHPAD_27
+#define mmPREBOOT_VER_OFFSET		mmPSOC_GLOBAL_CONF_SCRATCHPAD_28
+#define mmUBOOT_VER_OFFSET		mmPSOC_GLOBAL_CONF_SCRATCHPAD_29
+#define mmRDWR_TEST			mmPSOC_GLOBAL_CONF_SCRATCHPAD_30
+#define mmBTL_ID			mmPSOC_GLOBAL_CONF_SCRATCHPAD_31
 
-#define mmHW_STATE				mmPSOC_GLOBAL_CONF_APP_STATUS
+#define mmHW_STATE			mmPSOC_GLOBAL_CONF_APP_STATUS
 #define mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS	mmPSOC_GLOBAL_CONF_WARM_REBOOT
+#define mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU	mmPSOC_GLOBAL_CONF_UBOOT_MAGIC
+#define mmUPD_PENDING_STS		mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_3
 
 #endif /* GOYA_REG_MAP_H_ */
diff --git a/drivers/misc/habanalabs/include/hl_boot_if.h b/drivers/misc/habanalabs/include/hl_boot_if.h
index f7992a69fd3a..c22d134e73af 100644
--- a/drivers/misc/habanalabs/include/hl_boot_if.h
+++ b/drivers/misc/habanalabs/include/hl_boot_if.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0
  *
- * Copyright 2018 HabanaLabs, Ltd.
+ * Copyright 2018-2020 HabanaLabs, Ltd.
  * All Rights Reserved.
  *
  */
@@ -9,8 +9,47 @@
 #define HL_BOOT_IF_H
 
 #define LKD_HARD_RESET_MAGIC		0xED7BD694
+#define HL_POWER9_HOST_MAGIC		0x1DA30009
 
-/* CPU error bits in BOOT_ERROR registers */
+#define BOOT_FIT_SRAM_OFFSET		0x200000
+
+/*
+ * CPU error bits in BOOT_ERROR registers
+ *
+ * CPU_BOOT_ERR0_DRAM_INIT_FAIL		DRAM initialization failed.
+ *					DRAM is not reliable to use.
+ *
+ * CPU_BOOT_ERR0_FIT_CORRUPTED		FIT data integrity verification of the
+ *					image provided by the host has failed.
+ *
+ * CPU_BOOT_ERR0_TS_INIT_FAIL		Thermal Sensor initialization failed.
+ *					Boot continues as usual, but keep in
+ *					mind this is a warning.
+ *
+ * CPU_BOOT_ERR0_DRAM_SKIPPED		DRAM initialization has been skipped.
+ *					Skipping DRAM initialization has been
+ *					requested (e.g. strap, command, etc.)
+ *					and FW skipped the DRAM initialization.
+ *					Host can initialize the DRAM.
+ *
+ * CPU_BOOT_ERR0_BMC_WAIT_SKIPPED	Waiting for BMC data will be skipped.
+ *					Meaning the BMC data might not be
+ *					available until reset.
+ *
+ * CPU_BOOT_ERR0_NIC_DATA_NOT_RDY	NIC data from BMC is not ready.
+ *					BMC has not provided the NIC data yet.
+ *					Once provided this bit will be cleared.
+ *
+ * CPU_BOOT_ERR0_NIC_FW_FAIL		NIC FW loading failed.
+ *					The NIC FW loading and initialization
+ *					failed. This means NICs are not usable.
+ *
+ * CPU_BOOT_ERR0_ENABLED		Error registers enabled.
+ *					This is a main indication that the
+ *					running FW populates the error
+ *					registers. Meaning the error bits are
+ *					not garbage, but actual error statuses.
+ */
 #define CPU_BOOT_ERR0_DRAM_INIT_FAIL		(1 << 0)
 #define CPU_BOOT_ERR0_FIT_CORRUPTED		(1 << 1)
 #define CPU_BOOT_ERR0_TS_INIT_FAIL		(1 << 2)
@@ -27,22 +66,33 @@ enum cpu_boot_status {
 	CPU_BOOT_STATUS_SRAM_AVAIL = 3,
 	CPU_BOOT_STATUS_IN_BTL = 4,	/* BTL is H/W FSM */
 	CPU_BOOT_STATUS_IN_PREBOOT = 5,
-	CPU_BOOT_STATUS_IN_SPL = 6,
+	CPU_BOOT_STATUS_IN_SPL,		/* deprecated - not reported */
 	CPU_BOOT_STATUS_IN_UBOOT = 7,
 	CPU_BOOT_STATUS_DRAM_INIT_FAIL,	/* deprecated - will be removed */
 	CPU_BOOT_STATUS_FIT_CORRUPTED,	/* deprecated - will be removed */
+	/* U-Boot console prompt activated, commands are not processed */
 	CPU_BOOT_STATUS_UBOOT_NOT_READY = 10,
+	/* Finished NICs init, reported after DRAM and NICs */
 	CPU_BOOT_STATUS_NIC_FW_RDY = 11,
 	CPU_BOOT_STATUS_TS_INIT_FAIL,	/* deprecated - will be removed */
 	CPU_BOOT_STATUS_DRAM_SKIPPED,	/* deprecated - will be removed */
 	CPU_BOOT_STATUS_BMC_WAITING_SKIPPED, /* deprecated - will be removed */
+	/* Last boot loader progress status, ready to receive commands */
 	CPU_BOOT_STATUS_READY_TO_BOOT = 15,
+	CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT = 16,
 };
 
 enum kmd_msg {
 	KMD_MSG_NA = 0,
 	KMD_MSG_GOTO_WFE,
-	KMD_MSG_FIT_RDY
+	KMD_MSG_FIT_RDY,
+	KMD_MSG_SKIP_BMC,
+};
+
+enum cpu_msg_status {
+	CPU_MSG_CLR = 0,
+	CPU_MSG_OK,
+	CPU_MSG_ERR,
 };
 
 #endif /* HL_BOOT_IF_H */
diff --git a/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
index a6851a9d3f03..468bb045fbd1 100644
--- a/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
+++ b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_general.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0
  *
- * Copyright 2016-2018 HabanaLabs, Ltd.
+ * Copyright 2016-2020 HabanaLabs, Ltd.
  * All Rights Reserved.
  *
  */
diff --git a/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_v1_1.h b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_v1_1.h
new file mode 100644
index 000000000000..b2a9570583ac
--- /dev/null
+++ b/drivers/misc/habanalabs/include/hw_ip/mmu/mmu_v1_1.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright 2018 HabanaLabs, Ltd.
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef INCLUDE_MMU_V1_1_H_
+#define INCLUDE_MMU_V1_1_H_
+
+#define MMU_ASID		0xC12004
+#define MMU_HOP0_PA43_12	0xC12008
+#define MMU_HOP0_PA49_44	0xC1200C
+#define MMU_BUSY		0xC12000
+
+#endif /* INCLUDE_MMU_V1_1_H_ */
diff --git a/drivers/misc/habanalabs/memory.c b/drivers/misc/habanalabs/memory.c
index a72f766ca470..47da84a17719 100644
--- a/drivers/misc/habanalabs/memory.c
+++ b/drivers/misc/habanalabs/memory.c
@@ -886,6 +886,7 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
 
 		vm_type = (enum vm_type_t *) userptr;
 		hint_addr = args->map_host.hint_addr;
+		handle = phys_pg_pack->handle;
 	} else {
 		handle = lower_32_bits(args->map_device.handle);
 
@@ -954,10 +955,17 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
 		goto map_err;
 	}
 
-	hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type);
+	rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type);
 
 	mutex_unlock(&ctx->mmu_lock);
 
+	if (rc) {
+		dev_err(hdev->dev,
+			"mapping handle %u failed due to MMU cache invalidation\n",
+			handle);
+		goto map_err;
+	}
+
 	ret_vaddr += phys_pg_pack->offset;
 
 	hnode->ptr = vm_type;
@@ -1015,7 +1023,7 @@ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free)
 	struct hl_va_range *va_range;
 	enum vm_type_t *vm_type;
 	bool is_userptr;
-	int rc;
+	int rc = 0;
 
 	/* protect from double entrance */
 	mutex_lock(&ctx->mem_hash_lock);
@@ -1083,21 +1091,34 @@ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free)
 	 * at the loop end rather than for each iteration
 	 */
 	if (!ctx_free)
-		hdev->asic_funcs->mmu_invalidate_cache(hdev, true, *vm_type);
+		rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
+								*vm_type);
 
 	mutex_unlock(&ctx->mmu_lock);
 
 	/*
-	 * No point in maintaining the free VA block list if the context is
-	 * closing as the list will be freed anyway
+	 * If the context is closing we don't need to check for the MMU cache
+	 * invalidation return code and update the VA free list as in this flow
+	 * we invalidate the MMU cache outside of this unmap function and the VA
+	 * free list will be freed anyway.
 	 */
 	if (!ctx_free) {
-		rc = add_va_block(hdev, va_range, vaddr,
-					vaddr + phys_pg_pack->total_size - 1);
+		int tmp_rc;
+
 		if (rc)
+			dev_err(hdev->dev,
+				"unmapping vaddr 0x%llx failed due to MMU cache invalidation\n",
+				vaddr);
+
+		tmp_rc = add_va_block(hdev, va_range, vaddr,
+					vaddr + phys_pg_pack->total_size - 1);
+		if (tmp_rc) {
 			dev_warn(hdev->dev,
 					"add va block failed for vaddr: 0x%llx\n",
 					vaddr);
+			if (!rc)
+				rc = tmp_rc;
+		}
 	}
 
 	atomic_dec(&phys_pg_pack->mapping_cnt);
@@ -1108,7 +1129,7 @@ static int unmap_device_va(struct hl_ctx *ctx, u64 vaddr, bool ctx_free)
 		dma_unmap_host_va(hdev, userptr);
 	}
 
-	return 0;
+	return rc;
 
 mapping_cnt_err:
 	if (is_userptr)
diff --git a/drivers/misc/habanalabs/pci.c b/drivers/misc/habanalabs/pci.c
index c98d88c7a5c6..9f634ef6f5b3 100644
--- a/drivers/misc/habanalabs/pci.c
+++ b/drivers/misc/habanalabs/pci.c
@@ -267,6 +267,12 @@ int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
 	/* Enable + Bar match + match enable */
 	rc |= hl_pci_iatu_write(hdev, 0x104, 0xC0080000);
 
+	/* Return the DBI window to the default location */
+	rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
+	rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
+
+	hdev->asic_funcs->set_dma_mask_from_fw(hdev);
+
 	/* Point to DRAM */
 	if (!hdev->asic_funcs->set_dram_bar_base)
 		return -EINVAL;
@@ -274,7 +280,6 @@ int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
 								U64_MAX)
 		return -EIO;
 
-
 	/* Outbound Region 0 - Point to Host */
 	host_phys_end_addr = host_phys_base_address + host_phys_size - 1;
 	rc |= hl_pci_iatu_write(hdev, 0x008,
@@ -283,7 +288,12 @@ int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
 				upper_32_bits(host_phys_base_address));
 	rc |= hl_pci_iatu_write(hdev, 0x010, lower_32_bits(host_phys_end_addr));
 	rc |= hl_pci_iatu_write(hdev, 0x014, 0);
-	rc |= hl_pci_iatu_write(hdev, 0x018, 0);
+
+	if ((hdev->power9_64bit_dma_enable) && (hdev->dma_mask == 64))
+		rc |= hl_pci_iatu_write(hdev, 0x018, 0x08000000);
+	else
+		rc |= hl_pci_iatu_write(hdev, 0x018, 0);
+
 	rc |= hl_pci_iatu_write(hdev, 0x020, upper_32_bits(host_phys_end_addr));
 	/* Increase region size */
 	rc |= hl_pci_iatu_write(hdev, 0x000, 0x00002000);
@@ -310,41 +320,25 @@ int hl_pci_init_iatu(struct hl_device *hdev, u64 sram_base_address,
  *
  * Return: 0 on success, non-zero for failure.
  */
-int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask)
+static int hl_pci_set_dma_mask(struct hl_device *hdev)
 {
 	struct pci_dev *pdev = hdev->pdev;
 	int rc;
 
 	/* set DMA mask */
-	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
+	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask));
 	if (rc) {
-		dev_warn(hdev->dev,
+		dev_err(hdev->dev,
 			"Failed to set pci dma mask to %d bits, error %d\n",
-			dma_mask, rc);
-
-		dma_mask = hdev->dma_mask;
-
-		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
-		if (rc) {
-			dev_err(hdev->dev,
-				"Failed to set pci dma mask to %d bits, error %d\n",
-				dma_mask, rc);
-			return rc;
-		}
+			hdev->dma_mask, rc);
+		return rc;
 	}
 
-	/*
-	 * We managed to set the dma mask, so update the dma mask field. If
-	 * the set to the coherent mask will fail with that mask, we will
-	 * fail the entire function
-	 */
-	hdev->dma_mask = dma_mask;
-
-	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_mask));
+	rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask));
 	if (rc) {
 		dev_err(hdev->dev,
 			"Failed to set pci consistent dma mask to %d bits, error %d\n",
-			dma_mask, rc);
+			hdev->dma_mask, rc);
 		return rc;
 	}
 
@@ -354,21 +348,16 @@ int hl_pci_set_dma_mask(struct hl_device *hdev, u8 dma_mask)
 /**
  * hl_pci_init() - PCI initialization code.
  * @hdev: Pointer to hl_device structure.
- * @dma_mask: number of bits for the requested dma mask.
  *
  * Set DMA masks, initialize the PCI controller and map the PCI BARs.
  *
  * Return: 0 on success, non-zero for failure.
  */
-int hl_pci_init(struct hl_device *hdev, u8 dma_mask)
+int hl_pci_init(struct hl_device *hdev)
 {
 	struct pci_dev *pdev = hdev->pdev;
 	int rc;
 
-	rc = hl_pci_set_dma_mask(hdev, dma_mask);
-	if (rc)
-		return rc;
-
 	if (hdev->reset_pcilink)
 		hl_pci_reset_link_through_bridge(hdev);
 
@@ -380,18 +369,22 @@ int hl_pci_init(struct hl_device *hdev, u8 dma_mask)
 
 	pci_set_master(pdev);
 
-	rc = hdev->asic_funcs->init_iatu(hdev);
+	rc = hdev->asic_funcs->pci_bars_map(hdev);
 	if (rc) {
-		dev_err(hdev->dev, "Failed to initialize iATU\n");
+		dev_err(hdev->dev, "Failed to initialize PCI BARs\n");
 		goto disable_device;
 	}
 
-	rc = hdev->asic_funcs->pci_bars_map(hdev);
+	rc = hdev->asic_funcs->init_iatu(hdev);
 	if (rc) {
-		dev_err(hdev->dev, "Failed to initialize PCI BARs\n");
+		dev_err(hdev->dev, "Failed to initialize iATU\n");
 		goto disable_device;
 	}
 
+	rc = hl_pci_set_dma_mask(hdev);
+	if (rc)
+		goto disable_device;
+
 	return 0;
 
 disable_device:
diff --git a/drivers/misc/habanalabs/sysfs.c b/drivers/misc/habanalabs/sysfs.c
index 4cd622b017b9..5d78d5e1c782 100644
--- a/drivers/misc/habanalabs/sysfs.c
+++ b/drivers/misc/habanalabs/sysfs.c
@@ -183,6 +183,13 @@ static ssize_t soft_reset_store(struct device *dev,
 		goto out;
 	}
 
+	if (!hdev->supports_soft_reset) {
+		dev_err(hdev->dev, "Device does not support soft-reset\n");
+		goto out;
+	}
+
+	dev_warn(hdev->dev, "Soft-Reset requested through sysfs\n");
+
 	hl_device_reset(hdev, false, false);
 
 out:
@@ -204,6 +211,8 @@ static ssize_t hard_reset_store(struct device *dev,
 		goto out;
 	}
 
+	dev_warn(hdev->dev, "Hard-Reset requested through sysfs\n");
+
 	hl_device_reset(hdev, true, false);
 
 out:
@@ -220,6 +229,9 @@ static ssize_t device_type_show(struct device *dev,
 	case ASIC_GOYA:
 		str = "GOYA";
 		break;
+	case ASIC_GAUDI:
+		str = "GAUDI";
+		break;
 	default:
 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 				hdev->asic_type);
@@ -406,7 +418,10 @@ int hl_sysfs_init(struct hl_device *hdev)
 {
 	int rc;
 
-	hdev->pm_mng_profile = PM_AUTO;
+	if (hdev->asic_type == ASIC_GOYA)
+		hdev->pm_mng_profile = PM_AUTO;
+	else
+		hdev->pm_mng_profile = PM_MANUAL;
 	hdev->max_power = hdev->asic_prop.max_power_default;
 
 	hdev->asic_funcs->add_device_attr(hdev, &hl_dev_clks_attr_group);
diff --git a/drivers/misc/kgdbts.c b/drivers/misc/kgdbts.c
index bccd341e9ae1..d5d2af4d10e6 100644
--- a/drivers/misc/kgdbts.c
+++ b/drivers/misc/kgdbts.c
@@ -828,7 +828,7 @@ static void run_plant_and_detach_test(int is_early)
 	char before[BREAK_INSTR_SIZE];
 	char after[BREAK_INSTR_SIZE];
 
-	probe_kernel_read(before, (char *)kgdbts_break_test,
+	copy_from_kernel_nofault(before, (char *)kgdbts_break_test,
 	  BREAK_INSTR_SIZE);
 	init_simple_test();
 	ts.tst = plant_and_detach_test;
@@ -836,8 +836,8 @@ static void run_plant_and_detach_test(int is_early)
 	/* Activate test with initial breakpoint */
 	if (!is_early)
 		kgdb_breakpoint();
-	probe_kernel_read(after, (char *)kgdbts_break_test,
-	  BREAK_INSTR_SIZE);
+	copy_from_kernel_nofault(after, (char *)kgdbts_break_test,
+			BREAK_INSTR_SIZE);
 	if (memcmp(before, after, BREAK_INSTR_SIZE)) {
 		printk(KERN_CRIT "kgdbts: ERROR kgdb corrupted memory\n");
 		panic("kgdb memory corruption");
diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c
index 886459e0ddd9..736675f0a246 100644
--- a/drivers/misc/lkdtm/bugs.c
+++ b/drivers/misc/lkdtm/bugs.c
@@ -208,7 +208,7 @@ void lkdtm_OVERFLOW_UNSIGNED(void)
 	ignored = value;
 }
 
-/* Intentially using old-style flex array definition of 1 byte. */
+/* Intentionally using old-style flex array definition of 1 byte. */
 struct array_bounds_flex_array {
 	int one;
 	int two;
diff --git a/drivers/misc/mic/Kconfig b/drivers/misc/mic/Kconfig
index 3bfe72c59864..8f201d019f5a 100644
--- a/drivers/misc/mic/Kconfig
+++ b/drivers/misc/mic/Kconfig
@@ -116,7 +116,7 @@ config MIC_COSM
 
 config VOP
 	tristate "VOP Driver"
-	depends on VOP_BUS && VHOST_DPN
+	depends on VOP_BUS
 	select VHOST_RING
 	select VIRTIO
 	help
diff --git a/drivers/misc/mic/scif/scif_nodeqp.c b/drivers/misc/mic/scif/scif_nodeqp.c
index fcd999f50d14..ea084626fe11 100644
--- a/drivers/misc/mic/scif/scif_nodeqp.c
+++ b/drivers/misc/mic/scif/scif_nodeqp.c
@@ -660,7 +660,7 @@ int scif_nodeqp_send(struct scif_dev *scifdev, struct scifmsg *msg)
 	struct device *spdev = NULL;
 
 	if (msg->uop > SCIF_EXIT_ACK) {
-		/* Dont send messages once the exit flow has begun */
+		/* Don't send messages once the exit flow has begun */
 		if (OP_IDLE != scifdev->exit)
 			return -ENODEV;
 		spdev = scif_get_peer_dev(scifdev);
diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c
index 01e27682ea30..406cd5abfa72 100644
--- a/drivers/misc/mic/scif/scif_rma.c
+++ b/drivers/misc/mic/scif/scif_rma.c
@@ -113,14 +113,17 @@ static int scif_destroy_pinned_pages(struct scif_pinned_pages *pin)
 	int writeable = pin->prot & SCIF_PROT_WRITE;
 	int kernel = SCIF_MAP_KERNEL & pin->map_flags;
 
-	for (j = 0; j < pin->nr_pages; j++) {
-		if (pin->pages[j] && !kernel) {
-			if (writeable)
-				SetPageDirty(pin->pages[j]);
-			put_page(pin->pages[j]);
+	if (kernel) {
+		for (j = 0; j < pin->nr_pages; j++) {
+			if (pin->pages[j] && !kernel) {
+				if (writeable)
+					set_page_dirty_lock(pin->pages[j]);
+				put_page(pin->pages[j]);
+			}
 		}
-	}
-
+	} else
+		unpin_user_pages_dirty_lock(pin->pages, pin->nr_pages,
+					    writeable);
 	scif_free(pin->pages,
 		  pin->nr_pages * sizeof(*pin->pages));
 	scif_free(pin, sizeof(*pin));
@@ -1375,7 +1378,7 @@ retry:
 			}
 		}
 
-		pinned_pages->nr_pages = get_user_pages_fast(
+		pinned_pages->nr_pages = pin_user_pages_fast(
 				(u64)addr,
 				nr_pages,
 				(prot & SCIF_PROT_WRITE) ? FOLL_WRITE : 0,
@@ -1385,11 +1388,8 @@ retry:
 				if (ulimit)
 					__scif_dec_pinned_vm_lock(mm, nr_pages);
 				/* Roll back any pinned pages */
-				for (i = 0; i < pinned_pages->nr_pages; i++) {
-					if (pinned_pages->pages[i])
-						put_page(
-						pinned_pages->pages[i]);
-				}
+				unpin_user_pages(pinned_pages->pages,
+						 pinned_pages->nr_pages);
 				prot &= ~SCIF_PROT_WRITE;
 				try_upgrade = false;
 				goto retry;
diff --git a/drivers/misc/ocxl/context.c b/drivers/misc/ocxl/context.c
index de8a66b9d76b..c21f65a5c762 100644
--- a/drivers/misc/ocxl/context.c
+++ b/drivers/misc/ocxl/context.c
@@ -287,7 +287,7 @@ void ocxl_context_free(struct ocxl_context *ctx)
 
 	ocxl_afu_irq_free_all(ctx);
 	idr_destroy(&ctx->irq_idr);
-	/* reference to the AFU taken in ocxl_context_init */
+	/* reference to the AFU taken in ocxl_context_alloc() */
 	ocxl_afu_put(ctx->afu);
 	kfree(ctx);
 }
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index ef5a1af6bab7..41c40971979e 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -73,6 +73,8 @@
 #define is_am654_pci_dev(pdev)		\
 		((pdev)->device == PCI_DEVICE_ID_TI_AM654)
 
+#define PCI_DEVICE_ID_RENESAS_R8A774C0		0x002d
+
 static DEFINE_IDA(pci_endpoint_test_ida);
 
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
@@ -942,6 +944,8 @@ static const struct pci_device_id pci_endpoint_test_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
 	  .driver_data = (kernel_ulong_t)&am654_data
 	},
+	{ PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0),
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index 4b713a80b572..b1521112dbbd 100644
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -21,7 +21,6 @@
 #include <linux/uaccess.h>
 #include <linux/security.h>
 #include <linux/prefetch.h>
-#include <asm/pgtable.h>
 #include "gru.h"
 #include "grutables.h"
 #include "grulib.h"
@@ -43,7 +42,7 @@ static inline int is_gru_paddr(unsigned long paddr)
 }
 
 /*
- * Find the vma of a GRU segment. Caller must hold mmap_sem.
+ * Find the vma of a GRU segment. Caller must hold mmap_lock.
  */
 struct vm_area_struct *gru_find_vma(unsigned long vaddr)
 {
@@ -59,7 +58,7 @@ struct vm_area_struct *gru_find_vma(unsigned long vaddr)
  * Find and lock the gts that contains the specified user vaddr.
  *
  * Returns:
- * 	- *gts with the mmap_sem locked for read and the GTS locked.
+ * 	- *gts with the mmap_lock locked for read and the GTS locked.
  *	- NULL if vaddr invalid OR is not a valid GSEG vaddr.
  */
 
@@ -69,14 +68,14 @@ static struct gru_thread_state *gru_find_lock_gts(unsigned long vaddr)
 	struct vm_area_struct *vma;
 	struct gru_thread_state *gts = NULL;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	vma = gru_find_vma(vaddr);
 	if (vma)
 		gts = gru_find_thread_state(vma, TSID(vaddr, vma));
 	if (gts)
 		mutex_lock(&gts->ts_ctxlock);
 	else
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 	return gts;
 }
 
@@ -86,7 +85,7 @@ static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr)
 	struct vm_area_struct *vma;
 	struct gru_thread_state *gts = ERR_PTR(-EINVAL);
 
-	down_write(&mm->mmap_sem);
+	mmap_write_lock(mm);
 	vma = gru_find_vma(vaddr);
 	if (!vma)
 		goto err;
@@ -95,11 +94,11 @@ static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr)
 	if (IS_ERR(gts))
 		goto err;
 	mutex_lock(&gts->ts_ctxlock);
-	downgrade_write(&mm->mmap_sem);
+	mmap_write_downgrade(mm);
 	return gts;
 
 err:
-	up_write(&mm->mmap_sem);
+	mmap_write_unlock(mm);
 	return gts;
 }
 
@@ -109,7 +108,7 @@ err:
 static void gru_unlock_gts(struct gru_thread_state *gts)
 {
 	mutex_unlock(&gts->ts_ctxlock);
-	up_read(&current->mm->mmap_sem);
+	mmap_read_unlock(current->mm);
 }
 
 /*
@@ -199,7 +198,7 @@ static int non_atomic_pte_lookup(struct vm_area_struct *vma,
  * Only supports Intel large pages (2MB only) on x86_64.
  *	ZZZ - hugepage support is incomplete
  *
- * NOTE: mmap_sem is already held on entry to this function. This
+ * NOTE: mmap_lock is already held on entry to this function. This
  * guarantees existence of the page tables.
  */
 static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr,
@@ -570,14 +569,14 @@ static irqreturn_t gru_intr(int chiplet, int blade)
 		}
 
 		/*
-		 * This is running in interrupt context. Trylock the mmap_sem.
+		 * This is running in interrupt context. Trylock the mmap_lock.
 		 * If it fails, retry the fault in user context.
 		 */
 		gts->ustats.fmm_tlbmiss++;
 		if (!gts->ts_force_cch_reload &&
-					down_read_trylock(&gts->ts_mm->mmap_sem)) {
+					mmap_read_trylock(gts->ts_mm)) {
 			gru_try_dropin(gru, gts, tfh, NULL);
-			up_read(&gts->ts_mm->mmap_sem);
+			mmap_read_unlock(gts->ts_mm);
 		} else {
 			tfh_user_polling_mode(tfh);
 			STAT(intr_mm_lock_failed);
diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c
index 9d042310214f..93bb49ddda1f 100644
--- a/drivers/misc/sgi-gru/grufile.c
+++ b/drivers/misc/sgi-gru/grufile.c
@@ -135,7 +135,7 @@ static int gru_create_new_context(unsigned long arg)
 	if (!(req.options & GRU_OPT_MISS_MASK))
 		req.options |= GRU_OPT_MISS_FMM_INTR;
 
-	down_write(&current->mm->mmap_sem);
+	mmap_write_lock(current->mm);
 	vma = gru_find_vma(req.gseg);
 	if (vma) {
 		vdata = vma->vm_private_data;
@@ -146,7 +146,7 @@ static int gru_create_new_context(unsigned long arg)
 		vdata->vd_tlb_preload_count = req.tlb_preload_count;
 		ret = 0;
 	}
-	up_write(&current->mm->mmap_sem);
+	mmap_write_unlock(current->mm);
 
 	return ret;
 }
diff --git a/drivers/misc/sgi-xp/xpc_main.c b/drivers/misc/sgi-xp/xpc_main.c
index 79a963105983..d5e097cd556d 100644
--- a/drivers/misc/sgi-xp/xpc_main.c
+++ b/drivers/misc/sgi-xp/xpc_main.c
@@ -59,16 +59,16 @@
 
 /* define two XPC debug device structures to be used with dev_dbg() et al */
 
-struct device_driver xpc_dbg_name = {
+static struct device_driver xpc_dbg_name = {
 	.name = "xpc"
 };
 
-struct device xpc_part_dbg_subname = {
+static struct device xpc_part_dbg_subname = {
 	.init_name = "",	/* set to "part" at xpc_init() time */
 	.driver = &xpc_dbg_name
 };
 
-struct device xpc_chan_dbg_subname = {
+static struct device xpc_chan_dbg_subname = {
 	.init_name = "",	/* set to "chan" at xpc_init() time */
 	.driver = &xpc_dbg_name
 };
@@ -1217,7 +1217,7 @@ xpc_system_die(struct notifier_block *nb, unsigned long event, void *_die_args)
 	return NOTIFY_DONE;
 }
 
-int __init
+static int __init
 xpc_init(void)
 {
 	int ret;
@@ -1319,7 +1319,7 @@ out_1:
 
 module_init(xpc_init);
 
-void __exit
+static void __exit
 xpc_exit(void)
 {
 	xpc_do_exit(xpUnloading);
diff --git a/drivers/misc/sgi-xp/xpnet.c b/drivers/misc/sgi-xp/xpnet.c
index ada94e6a3c91..837d6c3fe69c 100644
--- a/drivers/misc/sgi-xp/xpnet.c
+++ b/drivers/misc/sgi-xp/xpnet.c
@@ -96,7 +96,7 @@ struct xpnet_pending_msg {
 	atomic_t use_count;
 };
 
-struct net_device *xpnet_device;
+static struct net_device *xpnet_device;
 
 /*
  * When we are notified of other partitions activating, we add them to
@@ -131,16 +131,16 @@ static DEFINE_SPINLOCK(xpnet_broadcast_lock);
 
 /* Define the XPNET debug device structures to be used with dev_dbg() et al */
 
-struct device_driver xpnet_dbg_name = {
+static struct device_driver xpnet_dbg_name = {
 	.name = "xpnet"
 };
 
-struct device xpnet_dbg_subname = {
+static struct device xpnet_dbg_subname = {
 	.init_name = "",	/* set to "" */
 	.driver = &xpnet_dbg_name
 };
 
-struct device *xpnet = &xpnet_dbg_subname;
+static struct device *xpnet = &xpnet_dbg_subname;
 
 /*
  * Packet was recevied by XPC and forwarded to us.
diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c
index d39307f060bd..107028e77ca3 100644
--- a/drivers/misc/uacce/uacce.c
+++ b/drivers/misc/uacce/uacce.c
@@ -90,109 +90,39 @@ static long uacce_fops_compat_ioctl(struct file *filep,
 }
 #endif
 
-static int uacce_sva_exit(struct device *dev, struct iommu_sva *handle,
-			  void *data)
+static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
 {
-	struct uacce_mm *uacce_mm = data;
-	struct uacce_queue *q;
-
-	/*
-	 * No new queue can be added concurrently because no caller can have a
-	 * reference to this mm. But there may be concurrent calls to
-	 * uacce_mm_put(), so we need the lock.
-	 */
-	mutex_lock(&uacce_mm->lock);
-	list_for_each_entry(q, &uacce_mm->queues, list)
-		uacce_put_queue(q);
-	uacce_mm->mm = NULL;
-	mutex_unlock(&uacce_mm->lock);
+	int pasid;
+	struct iommu_sva *handle;
 
-	return 0;
-}
-
-static struct iommu_sva_ops uacce_sva_ops = {
-	.mm_exit = uacce_sva_exit,
-};
-
-static struct uacce_mm *uacce_mm_get(struct uacce_device *uacce,
-				     struct uacce_queue *q,
-				     struct mm_struct *mm)
-{
-	struct uacce_mm *uacce_mm = NULL;
-	struct iommu_sva *handle = NULL;
-	int ret;
-
-	lockdep_assert_held(&uacce->mm_lock);
-
-	list_for_each_entry(uacce_mm, &uacce->mm_list, list) {
-		if (uacce_mm->mm == mm) {
-			mutex_lock(&uacce_mm->lock);
-			list_add(&q->list, &uacce_mm->queues);
-			mutex_unlock(&uacce_mm->lock);
-			return uacce_mm;
-		}
-	}
-
-	uacce_mm = kzalloc(sizeof(*uacce_mm), GFP_KERNEL);
-	if (!uacce_mm)
-		return NULL;
+	if (!(uacce->flags & UACCE_DEV_SVA))
+		return 0;
 
-	if (uacce->flags & UACCE_DEV_SVA) {
-		/*
-		 * Safe to pass an incomplete uacce_mm, since mm_exit cannot
-		 * fire while we hold a reference to the mm.
-		 */
-		handle = iommu_sva_bind_device(uacce->parent, mm, uacce_mm);
-		if (IS_ERR(handle))
-			goto err_free;
+	handle = iommu_sva_bind_device(uacce->parent, current->mm, NULL);
+	if (IS_ERR(handle))
+		return PTR_ERR(handle);
 
-		ret = iommu_sva_set_ops(handle, &uacce_sva_ops);
-		if (ret)
-			goto err_unbind;
-
-		uacce_mm->pasid = iommu_sva_get_pasid(handle);
-		if (uacce_mm->pasid == IOMMU_PASID_INVALID)
-			goto err_unbind;
+	pasid = iommu_sva_get_pasid(handle);
+	if (pasid == IOMMU_PASID_INVALID) {
+		iommu_sva_unbind_device(handle);
+		return -ENODEV;
 	}
 
-	uacce_mm->mm = mm;
-	uacce_mm->handle = handle;
-	INIT_LIST_HEAD(&uacce_mm->queues);
-	mutex_init(&uacce_mm->lock);
-	list_add(&q->list, &uacce_mm->queues);
-	list_add(&uacce_mm->list, &uacce->mm_list);
-
-	return uacce_mm;
-
-err_unbind:
-	if (handle)
-		iommu_sva_unbind_device(handle);
-err_free:
-	kfree(uacce_mm);
-	return NULL;
+	q->handle = handle;
+	q->pasid = pasid;
+	return 0;
 }
 
-static void uacce_mm_put(struct uacce_queue *q)
+static void uacce_unbind_queue(struct uacce_queue *q)
 {
-	struct uacce_mm *uacce_mm = q->uacce_mm;
-
-	lockdep_assert_held(&q->uacce->mm_lock);
-
-	mutex_lock(&uacce_mm->lock);
-	list_del(&q->list);
-	mutex_unlock(&uacce_mm->lock);
-
-	if (list_empty(&uacce_mm->queues)) {
-		if (uacce_mm->handle)
-			iommu_sva_unbind_device(uacce_mm->handle);
-		list_del(&uacce_mm->list);
-		kfree(uacce_mm);
-	}
+	if (!q->handle)
+		return;
+	iommu_sva_unbind_device(q->handle);
+	q->handle = NULL;
 }
 
 static int uacce_fops_open(struct inode *inode, struct file *filep)
 {
-	struct uacce_mm *uacce_mm = NULL;
 	struct uacce_device *uacce;
 	struct uacce_queue *q;
 	int ret = 0;
@@ -205,21 +135,16 @@ static int uacce_fops_open(struct inode *inode, struct file *filep)
 	if (!q)
 		return -ENOMEM;
 
-	mutex_lock(&uacce->mm_lock);
-	uacce_mm = uacce_mm_get(uacce, q, current->mm);
-	mutex_unlock(&uacce->mm_lock);
-	if (!uacce_mm) {
-		ret = -ENOMEM;
+	ret = uacce_bind_queue(uacce, q);
+	if (ret)
 		goto out_with_mem;
-	}
 
 	q->uacce = uacce;
-	q->uacce_mm = uacce_mm;
 
 	if (uacce->ops->get_queue) {
-		ret = uacce->ops->get_queue(uacce, uacce_mm->pasid, q);
+		ret = uacce->ops->get_queue(uacce, q->pasid, q);
 		if (ret < 0)
-			goto out_with_mm;
+			goto out_with_bond;
 	}
 
 	init_waitqueue_head(&q->wait);
@@ -227,12 +152,14 @@ static int uacce_fops_open(struct inode *inode, struct file *filep)
 	uacce->inode = inode;
 	q->state = UACCE_Q_INIT;
 
+	mutex_lock(&uacce->queues_lock);
+	list_add(&q->list, &uacce->queues);
+	mutex_unlock(&uacce->queues_lock);
+
 	return 0;
 
-out_with_mm:
-	mutex_lock(&uacce->mm_lock);
-	uacce_mm_put(q);
-	mutex_unlock(&uacce->mm_lock);
+out_with_bond:
+	uacce_unbind_queue(q);
 out_with_mem:
 	kfree(q);
 	return ret;
@@ -241,14 +168,12 @@ out_with_mem:
 static int uacce_fops_release(struct inode *inode, struct file *filep)
 {
 	struct uacce_queue *q = filep->private_data;
-	struct uacce_device *uacce = q->uacce;
 
+	mutex_lock(&q->uacce->queues_lock);
+	list_del(&q->list);
+	mutex_unlock(&q->uacce->queues_lock);
 	uacce_put_queue(q);
-
-	mutex_lock(&uacce->mm_lock);
-	uacce_mm_put(q);
-	mutex_unlock(&uacce->mm_lock);
-
+	uacce_unbind_queue(q);
 	kfree(q);
 
 	return 0;
@@ -513,8 +438,8 @@ struct uacce_device *uacce_alloc(struct device *parent,
 	if (ret < 0)
 		goto err_with_uacce;
 
-	INIT_LIST_HEAD(&uacce->mm_list);
-	mutex_init(&uacce->mm_lock);
+	INIT_LIST_HEAD(&uacce->queues);
+	mutex_init(&uacce->queues_lock);
 	device_initialize(&uacce->dev);
 	uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);
 	uacce->dev.class = uacce_class;
@@ -561,8 +486,7 @@ EXPORT_SYMBOL_GPL(uacce_register);
  */
 void uacce_remove(struct uacce_device *uacce)
 {
-	struct uacce_mm *uacce_mm;
-	struct uacce_queue *q;
+	struct uacce_queue *q, *next_q;
 
 	if (!uacce)
 		return;
@@ -574,24 +498,12 @@ void uacce_remove(struct uacce_device *uacce)
 		unmap_mapping_range(uacce->inode->i_mapping, 0, 0, 1);
 
 	/* ensure no open queue remains */
-	mutex_lock(&uacce->mm_lock);
-	list_for_each_entry(uacce_mm, &uacce->mm_list, list) {
-		/*
-		 * We don't take the uacce_mm->lock here. Since we hold the
-		 * device's mm_lock, no queue can be added to or removed from
-		 * this uacce_mm. We may run concurrently with mm_exit, but
-		 * uacce_put_queue() is serialized and iommu_sva_unbind_device()
-		 * waits for the lock that mm_exit is holding.
-		 */
-		list_for_each_entry(q, &uacce_mm->queues, list)
-			uacce_put_queue(q);
-
-		if (uacce->flags & UACCE_DEV_SVA) {
-			iommu_sva_unbind_device(uacce_mm->handle);
-			uacce_mm->handle = NULL;
-		}
+	mutex_lock(&uacce->queues_lock);
+	list_for_each_entry_safe(q, next_q, &uacce->queues, list) {
+		uacce_put_queue(q);
+		uacce_unbind_queue(q);
 	}
-	mutex_unlock(&uacce->mm_lock);
+	mutex_unlock(&uacce->queues_lock);
 
 	/* disable sva now since no opened queues */
 	if (uacce->flags & UACCE_DEV_SVA)
diff --git a/drivers/misc/vexpress-syscfg.c b/drivers/misc/vexpress-syscfg.c
deleted file mode 100644
index a431787c0898..000000000000
--- a/drivers/misc/vexpress-syscfg.c
+++ /dev/null
@@ -1,280 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *
- * Copyright (C) 2014 ARM Limited
- */
-
-#include <linux/delay.h>
-#include <linux/err.h>
-#include <linux/io.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/sched/signal.h>
-#include <linux/slab.h>
-#include <linux/syscore_ops.h>
-#include <linux/vexpress.h>
-
-
-#define SYS_CFGDATA		0x0
-
-#define SYS_CFGCTRL		0x4
-#define SYS_CFGCTRL_START	(1 << 31)
-#define SYS_CFGCTRL_WRITE	(1 << 30)
-#define SYS_CFGCTRL_DCC(n)	(((n) & 0xf) << 26)
-#define SYS_CFGCTRL_FUNC(n)	(((n) & 0x3f) << 20)
-#define SYS_CFGCTRL_SITE(n)	(((n) & 0x3) << 16)
-#define SYS_CFGCTRL_POSITION(n)	(((n) & 0xf) << 12)
-#define SYS_CFGCTRL_DEVICE(n)	(((n) & 0xfff) << 0)
-
-#define SYS_CFGSTAT		0x8
-#define SYS_CFGSTAT_ERR		(1 << 1)
-#define SYS_CFGSTAT_COMPLETE	(1 << 0)
-
-
-struct vexpress_syscfg {
-	struct device *dev;
-	void __iomem *base;
-	struct list_head funcs;
-};
-
-struct vexpress_syscfg_func {
-	struct list_head list;
-	struct vexpress_syscfg *syscfg;
-	struct regmap *regmap;
-	int num_templates;
-	u32 template[]; /* Keep it last! */
-};
-
-
-static int vexpress_syscfg_exec(struct vexpress_syscfg_func *func,
-		int index, bool write, u32 *data)
-{
-	struct vexpress_syscfg *syscfg = func->syscfg;
-	u32 command, status;
-	int tries;
-	long timeout;
-
-	if (WARN_ON(index >= func->num_templates))
-		return -EINVAL;
-
-	command = readl(syscfg->base + SYS_CFGCTRL);
-	if (WARN_ON(command & SYS_CFGCTRL_START))
-		return -EBUSY;
-
-	command = func->template[index];
-	command |= SYS_CFGCTRL_START;
-	command |= write ? SYS_CFGCTRL_WRITE : 0;
-
-	/* Use a canary for reads */
-	if (!write)
-		*data = 0xdeadbeef;
-
-	dev_dbg(syscfg->dev, "func %p, command %x, data %x\n",
-			func, command, *data);
-	writel(*data, syscfg->base + SYS_CFGDATA);
-	writel(0, syscfg->base + SYS_CFGSTAT);
-	writel(command, syscfg->base + SYS_CFGCTRL);
-	mb();
-
-	/* The operation can take ages... Go to sleep, 100us initially */
-	tries = 100;
-	timeout = 100;
-	do {
-		if (!irqs_disabled()) {
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(usecs_to_jiffies(timeout));
-			if (signal_pending(current))
-				return -EINTR;
-		} else {
-			udelay(timeout);
-		}
-
-		status = readl(syscfg->base + SYS_CFGSTAT);
-		if (status & SYS_CFGSTAT_ERR)
-			return -EFAULT;
-
-		if (timeout > 20)
-			timeout -= 20;
-	} while (--tries && !(status & SYS_CFGSTAT_COMPLETE));
-	if (WARN_ON_ONCE(!tries))
-		return -ETIMEDOUT;
-
-	if (!write) {
-		*data = readl(syscfg->base + SYS_CFGDATA);
-		dev_dbg(syscfg->dev, "func %p, read data %x\n", func, *data);
-	}
-
-	return 0;
-}
-
-static int vexpress_syscfg_read(void *context, unsigned int index,
-		unsigned int *val)
-{
-	struct vexpress_syscfg_func *func = context;
-
-	return vexpress_syscfg_exec(func, index, false, val);
-}
-
-static int vexpress_syscfg_write(void *context, unsigned int index,
-		unsigned int val)
-{
-	struct vexpress_syscfg_func *func = context;
-
-	return vexpress_syscfg_exec(func, index, true, &val);
-}
-
-static struct regmap_config vexpress_syscfg_regmap_config = {
-	.lock = vexpress_config_lock,
-	.unlock = vexpress_config_unlock,
-	.reg_bits = 32,
-	.val_bits = 32,
-	.reg_read = vexpress_syscfg_read,
-	.reg_write = vexpress_syscfg_write,
-	.reg_format_endian = REGMAP_ENDIAN_LITTLE,
-	.val_format_endian = REGMAP_ENDIAN_LITTLE,
-};
-
-
-static struct regmap *vexpress_syscfg_regmap_init(struct device *dev,
-		void *context)
-{
-	int err;
-	struct vexpress_syscfg *syscfg = context;
-	struct vexpress_syscfg_func *func;
-	struct property *prop;
-	const __be32 *val = NULL;
-	__be32 energy_quirk[4];
-	int num;
-	u32 site, position, dcc;
-	int i;
-
-	err = vexpress_config_get_topo(dev->of_node, &site,
-				&position, &dcc);
-	if (err)
-		return ERR_PTR(err);
-
-	prop = of_find_property(dev->of_node,
-			"arm,vexpress-sysreg,func", NULL);
-	if (!prop)
-		return ERR_PTR(-EINVAL);
-
-	num = prop->length / sizeof(u32) / 2;
-	val = prop->value;
-
-	/*
-	 * "arm,vexpress-energy" function used to be described
-	 * by its first device only, now it requires both
-	 */
-	if (num == 1 && of_device_is_compatible(dev->of_node,
-			"arm,vexpress-energy")) {
-		num = 2;
-		energy_quirk[0] = *val;
-		energy_quirk[2] = *val++;
-		energy_quirk[1] = *val;
-		energy_quirk[3] = cpu_to_be32(be32_to_cpup(val) + 1);
-		val = energy_quirk;
-	}
-
-	func = kzalloc(struct_size(func, template, num), GFP_KERNEL);
-	if (!func)
-		return ERR_PTR(-ENOMEM);
-
-	func->syscfg = syscfg;
-	func->num_templates = num;
-
-	for (i = 0; i < num; i++) {
-		u32 function, device;
-
-		function = be32_to_cpup(val++);
-		device = be32_to_cpup(val++);
-
-		dev_dbg(dev, "func %p: %u/%u/%u/%u/%u\n",
-				func, site, position, dcc,
-				function, device);
-
-		func->template[i] = SYS_CFGCTRL_DCC(dcc);
-		func->template[i] |= SYS_CFGCTRL_SITE(site);
-		func->template[i] |= SYS_CFGCTRL_POSITION(position);
-		func->template[i] |= SYS_CFGCTRL_FUNC(function);
-		func->template[i] |= SYS_CFGCTRL_DEVICE(device);
-	}
-
-	vexpress_syscfg_regmap_config.max_register = num - 1;
-
-	func->regmap = regmap_init(dev, NULL, func,
-			&vexpress_syscfg_regmap_config);
-
-	if (IS_ERR(func->regmap)) {
-		void *err = func->regmap;
-
-		kfree(func);
-		return err;
-	}
-
-	list_add(&func->list, &syscfg->funcs);
-
-	return func->regmap;
-}
-
-static void vexpress_syscfg_regmap_exit(struct regmap *regmap, void *context)
-{
-	struct vexpress_syscfg *syscfg = context;
-	struct vexpress_syscfg_func *func, *tmp;
-
-	regmap_exit(regmap);
-
-	list_for_each_entry_safe(func, tmp, &syscfg->funcs, list) {
-		if (func->regmap == regmap) {
-			list_del(&syscfg->funcs);
-			kfree(func);
-			break;
-		}
-	}
-}
-
-static struct vexpress_config_bridge_ops vexpress_syscfg_bridge_ops = {
-	.regmap_init = vexpress_syscfg_regmap_init,
-	.regmap_exit = vexpress_syscfg_regmap_exit,
-};
-
-
-static int vexpress_syscfg_probe(struct platform_device *pdev)
-{
-	struct vexpress_syscfg *syscfg;
-	struct resource *res;
-	struct device *bridge;
-
-	syscfg = devm_kzalloc(&pdev->dev, sizeof(*syscfg), GFP_KERNEL);
-	if (!syscfg)
-		return -ENOMEM;
-	syscfg->dev = &pdev->dev;
-	INIT_LIST_HEAD(&syscfg->funcs);
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	syscfg->base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(syscfg->base))
-		return PTR_ERR(syscfg->base);
-
-	/* Must use dev.parent (MFD), as that's where DT phandle points at... */
-	bridge = vexpress_config_bridge_register(pdev->dev.parent,
-			&vexpress_syscfg_bridge_ops, syscfg);
-
-	return PTR_ERR_OR_ZERO(bridge);
-}
-
-static const struct platform_device_id vexpress_syscfg_id_table[] = {
-	{ "vexpress-syscfg", },
-	{},
-};
-
-static struct platform_driver vexpress_syscfg_driver = {
-	.driver.name = "vexpress-syscfg",
-	.id_table = vexpress_syscfg_id_table,
-	.probe = vexpress_syscfg_probe,
-};
-
-static int __init vexpress_syscfg_init(void)
-{
-	return platform_driver_register(&vexpress_syscfg_driver);
-}
-core_initcall(vexpress_syscfg_init);
diff --git a/drivers/misc/xilinx_sdfec.c b/drivers/misc/xilinx_sdfec.c
index 71bbaa56bdb5..92291292756a 100644
--- a/drivers/misc/xilinx_sdfec.c
+++ b/drivers/misc/xilinx_sdfec.c
@@ -602,10 +602,10 @@ static int xsdfec_table_write(struct xsdfec_dev *xsdfec, u32 offset,
 			      const u32 depth)
 {
 	u32 reg = 0;
-	u32 res;
-	u32 n, i;
+	int res, i, nr_pages;
+	u32 n;
 	u32 *addr = NULL;
-	struct page *page[MAX_NUM_PAGES];
+	struct page *pages[MAX_NUM_PAGES];
 
 	/*
 	 * Writes that go beyond the length of
@@ -622,15 +622,21 @@ static int xsdfec_table_write(struct xsdfec_dev *xsdfec, u32 offset,
 	if ((len * XSDFEC_REG_WIDTH_JUMP) % PAGE_SIZE)
 		n += 1;
 
-	res = get_user_pages_fast((unsigned long)src_ptr, n, 0, page);
-	if (res < n) {
-		for (i = 0; i < res; i++)
-			put_page(page[i]);
+	if (WARN_ON_ONCE(n > INT_MAX))
+		return -EINVAL;
+
+	nr_pages = n;
+
+	res = pin_user_pages_fast((unsigned long)src_ptr, nr_pages, 0, pages);
+	if (res < nr_pages) {
+		if (res > 0)
+			unpin_user_pages(pages, res);
+
 		return -EINVAL;
 	}
 
-	for (i = 0; i < n; i++) {
-		addr = kmap(page[i]);
+	for (i = 0; i < nr_pages; i++) {
+		addr = kmap(pages[i]);
 		do {
 			xsdfec_regwrite(xsdfec,
 					base_addr + ((offset + reg) *
@@ -639,9 +645,9 @@ static int xsdfec_table_write(struct xsdfec_dev *xsdfec, u32 offset,
 			reg++;
 		} while ((reg < len) &&
 			 ((reg * XSDFEC_REG_WIDTH_JUMP) % PAGE_SIZE));
-		put_page(page[i]);
+		unpin_user_page(pages[i]);
 	}
-	return reg;
+	return 0;
 }
 
 static int xsdfec_add_ldpc(struct xsdfec_dev *xsdfec, void __user *arg)
@@ -649,14 +655,9 @@ static int xsdfec_add_ldpc(struct xsdfec_dev *xsdfec, void __user *arg)
 	struct xsdfec_ldpc_params *ldpc;
 	int ret, n;
 
-	ldpc = kzalloc(sizeof(*ldpc), GFP_KERNEL);
-	if (!ldpc)
-		return -ENOMEM;
-
-	if (copy_from_user(ldpc, arg, sizeof(*ldpc))) {
-		ret = -EFAULT;
-		goto err_out;
-	}
+	ldpc = memdup_user(arg, sizeof(*ldpc));
+	if (IS_ERR(ldpc))
+		return PTR_ERR(ldpc);
 
 	if (xsdfec->config.code == XSDFEC_TURBO_CODE) {
 		ret = -EIO;
@@ -720,8 +721,6 @@ static int xsdfec_add_ldpc(struct xsdfec_dev *xsdfec, void __user *arg)
 	ret = xsdfec_table_write(xsdfec, 4 * ldpc->qc_off, ldpc->qc_table,
 				 ldpc->nqc, XSDFEC_LDPC_QC_TABLE_ADDR_BASE,
 				 XSDFEC_QC_TABLE_DEPTH);
-	if (ret > 0)
-		ret = 0;
 err_out:
 	kfree(ldpc);
 	return ret;
@@ -1484,25 +1483,7 @@ static struct platform_driver xsdfec_driver = {
 	.remove =  xsdfec_remove,
 };
 
-static int __init xsdfec_init(void)
-{
-	int err;
-
-	err = platform_driver_register(&xsdfec_driver);
-	if (err < 0) {
-		pr_err("%s Unabled to register SDFEC driver", __func__);
-		return err;
-	}
-	return 0;
-}
-
-static void __exit xsdfec_exit(void)
-{
-	platform_driver_unregister(&xsdfec_driver);
-}
-
-module_init(xsdfec_init);
-module_exit(xsdfec_exit);
+module_platform_driver(xsdfec_driver);
 
 MODULE_AUTHOR("Xilinx, Inc");
 MODULE_DESCRIPTION("Xilinx SD-FEC16 Driver");
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 0ce332ad986b..3b706af35ec3 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -436,7 +436,7 @@ config MMC_MESON_MX_SDIO
 	tristate "Amlogic Meson6/Meson8/Meson8b SD/MMC Host Controller support"
 	depends on ARCH_MESON || COMPILE_TEST
 	depends on COMMON_CLK
-	depends on OF
+	depends on OF_ADDRESS
 	help
 	  This selects support for the SD/MMC Host Controller on
 	  Amlogic Meson6, Meson8 and Meson8b SoCs.
@@ -577,7 +577,7 @@ config MMC_MVSDIO
 	tristate "Marvell MMC/SD/SDIO host driver"
 	depends on PLAT_ORION
 	depends on OF
-	---help---
+	help
 	  This selects the Marvell SDIO host driver.
 	  SDIO may currently be found on the Kirkwood 88F6281 and 88F6192
 	  SoC controllers.
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index 2a4c8a2f3e64..db9b544465cd 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -123,10 +123,6 @@ struct sdhci_arasan_clk_data {
 	void		*clk_of_data;
 };
 
-struct sdhci_arasan_zynqmp_clk_data {
-	const struct zynqmp_eemi_ops *eemi_ops;
-};
-
 /**
  * struct sdhci_arasan_data - Arasan Controller Data
  *
@@ -599,9 +595,6 @@ static int sdhci_zynqmp_sdcardclk_set_phase(struct clk_hw *hw, int degrees)
 	struct sdhci_arasan_data *sdhci_arasan =
 		container_of(clk_data, struct sdhci_arasan_data, clk_data);
 	struct sdhci_host *host = sdhci_arasan->host;
-	struct sdhci_arasan_zynqmp_clk_data *zynqmp_clk_data =
-		clk_data->clk_of_data;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_clk_data->eemi_ops;
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 node_id = !strcmp(clk_name, "clk_out_sd0") ? NODE_SD_0 : NODE_SD_1;
 	u8 tap_delay, tap_max = 0;
@@ -641,8 +634,7 @@ static int sdhci_zynqmp_sdcardclk_set_phase(struct clk_hw *hw, int degrees)
 	tap_delay = (degrees * tap_max) / 360;
 
 	/* Set the Clock Phase */
-	ret = eemi_ops->ioctl(node_id, IOCTL_SET_SD_TAPDELAY,
-			      PM_TAPDELAY_OUTPUT, tap_delay, NULL);
+	ret = zynqmp_pm_set_sd_tapdelay(node_id, PM_TAPDELAY_OUTPUT, tap_delay);
 	if (ret)
 		pr_err("Error setting Output Tap Delay\n");
 
@@ -671,9 +663,6 @@ static int sdhci_zynqmp_sampleclk_set_phase(struct clk_hw *hw, int degrees)
 	struct sdhci_arasan_data *sdhci_arasan =
 		container_of(clk_data, struct sdhci_arasan_data, clk_data);
 	struct sdhci_host *host = sdhci_arasan->host;
-	struct sdhci_arasan_zynqmp_clk_data *zynqmp_clk_data =
-		clk_data->clk_of_data;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_clk_data->eemi_ops;
 	const char *clk_name = clk_hw_get_name(hw);
 	u32 node_id = !strcmp(clk_name, "clk_in_sd0") ? NODE_SD_0 : NODE_SD_1;
 	u8 tap_delay, tap_max = 0;
@@ -713,8 +702,7 @@ static int sdhci_zynqmp_sampleclk_set_phase(struct clk_hw *hw, int degrees)
 	tap_delay = (degrees * tap_max) / 360;
 
 	/* Set the Clock Phase */
-	ret = eemi_ops->ioctl(node_id, IOCTL_SET_SD_TAPDELAY,
-			      PM_TAPDELAY_INPUT, tap_delay, NULL);
+	ret = zynqmp_pm_set_sd_tapdelay(node_id, PM_TAPDELAY_INPUT, tap_delay);
 	if (ret)
 		pr_err("Error setting Input Tap Delay\n");
 
@@ -874,11 +862,6 @@ static const struct clk_ops versal_sampleclk_ops = {
 
 static void arasan_zynqmp_dll_reset(struct sdhci_host *host, u32 deviceid)
 {
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-	struct sdhci_arasan_data *sdhci_arasan = sdhci_pltfm_priv(pltfm_host);
-	struct sdhci_arasan_zynqmp_clk_data *zynqmp_clk_data =
-		sdhci_arasan->clk_data.clk_of_data;
-	const struct zynqmp_eemi_ops *eemi_ops = zynqmp_clk_data->eemi_ops;
 	u16 clk;
 
 	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
@@ -886,8 +869,7 @@ static void arasan_zynqmp_dll_reset(struct sdhci_host *host, u32 deviceid)
 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
 
 	/* Issue DLL Reset */
-	eemi_ops->ioctl(deviceid, IOCTL_SD_DLL_RESET,
-			PM_DLL_RESET_PULSE, 0, NULL);
+	zynqmp_pm_sd_dll_reset(deviceid, PM_DLL_RESET_PULSE);
 
 	clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL);
 
@@ -1617,20 +1599,6 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 		goto clk_disable_all;
 
 	if (of_device_is_compatible(np, "xlnx,zynqmp-8.9a")) {
-		struct sdhci_arasan_zynqmp_clk_data *zynqmp_clk_data;
-		const struct zynqmp_eemi_ops *eemi_ops;
-
-		zynqmp_clk_data = devm_kzalloc(&pdev->dev,
-					       sizeof(*zynqmp_clk_data),
-					       GFP_KERNEL);
-		eemi_ops = zynqmp_pm_get_eemi_ops();
-		if (IS_ERR(eemi_ops)) {
-			ret = PTR_ERR(eemi_ops);
-			goto unreg_clk;
-		}
-
-		zynqmp_clk_data->eemi_ops = eemi_ops;
-		sdhci_arasan->clk_data.clk_of_data = zynqmp_clk_data;
 		host->mmc_host_ops.execute_tuning =
 			arasan_zynqmp_execute_tuning;
 	}
diff --git a/drivers/mtd/chips/cfi_cmdset_0001.c b/drivers/mtd/chips/cfi_cmdset_0001.c
index 142c0f9485fe..42001c49833b 100644
--- a/drivers/mtd/chips/cfi_cmdset_0001.c
+++ b/drivers/mtd/chips/cfi_cmdset_0001.c
@@ -420,8 +420,9 @@ read_pri_intelext(struct map_info *map, __u16 adr)
 		extra_size = 0;
 
 		/* Protection Register info */
-		extra_size += (extp->NumProtectionFields - 1) *
-			      sizeof(struct cfi_intelext_otpinfo);
+		if (extp->NumProtectionFields)
+			extra_size += (extp->NumProtectionFields - 1) *
+				      sizeof(struct cfi_intelext_otpinfo);
 	}
 
 	if (extp->MinorVersion >= '1') {
@@ -695,14 +696,16 @@ static int cfi_intelext_partition_fixup(struct mtd_info *mtd,
 	 */
 	if (extp && extp->MajorVersion == '1' && extp->MinorVersion >= '3'
 	    && extp->FeatureSupport & (1 << 9)) {
+		int offs = 0;
 		struct cfi_private *newcfi;
 		struct flchip *chip;
 		struct flchip_shared *shared;
-		int offs, numregions, numparts, partshift, numvirtchips, i, j;
+		int numregions, numparts, partshift, numvirtchips, i, j;
 
 		/* Protection Register info */
-		offs = (extp->NumProtectionFields - 1) *
-		       sizeof(struct cfi_intelext_otpinfo);
+		if (extp->NumProtectionFields)
+			offs = (extp->NumProtectionFields - 1) *
+			       sizeof(struct cfi_intelext_otpinfo);
 
 		/* Burst Read info */
 		offs += extp->extra[offs+1]+2;
diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c
index eb0f4600efd1..a030792115bc 100644
--- a/drivers/mtd/devices/docg3.c
+++ b/drivers/mtd/devices/docg3.c
@@ -647,7 +647,7 @@ static int doc_ecc_bch_fix_data(struct docg3 *docg3, void *buf, u8 *hwecc)
 
 	for (i = 0; i < DOC_ECC_BCH_SIZE; i++)
 		ecc[i] = bitrev8(hwecc[i]);
-	numerrs = decode_bch(docg3->cascade->bch, NULL,
+	numerrs = bch_decode(docg3->cascade->bch, NULL,
 			     DOC_ECC_BCH_COVERED_BYTES,
 			     NULL, ecc, NULL, errorpos);
 	BUG_ON(numerrs == -EINVAL);
@@ -1984,8 +1984,8 @@ static int __init docg3_probe(struct platform_device *pdev)
 		return ret;
 	cascade->base = base;
 	mutex_init(&cascade->lock);
-	cascade->bch = init_bch(DOC_ECC_BCH_M, DOC_ECC_BCH_T,
-			     DOC_ECC_BCH_PRIMPOLY);
+	cascade->bch = bch_init(DOC_ECC_BCH_M, DOC_ECC_BCH_T,
+				DOC_ECC_BCH_PRIMPOLY, false);
 	if (!cascade->bch)
 		return ret;
 
@@ -2021,7 +2021,7 @@ notfound:
 	ret = -ENODEV;
 	dev_info(dev, "No supported DiskOnChip found\n");
 err_probe:
-	free_bch(cascade->bch);
+	bch_free(cascade->bch);
 	for (floor = 0; floor < DOC_MAX_NBFLOORS; floor++)
 		if (cascade->floors[floor])
 			doc_release_device(cascade->floors[floor]);
@@ -2045,7 +2045,7 @@ static int docg3_release(struct platform_device *pdev)
 		if (cascade->floors[floor])
 			doc_release_device(cascade->floors[floor]);
 
-	free_bch(docg3->cascade->bch);
+	bch_free(docg3->cascade->bch);
 	return 0;
 }
 
diff --git a/drivers/mtd/maps/physmap-gemini.c b/drivers/mtd/maps/physmap-gemini.c
index a289c8b5cabf..d4a46e159d38 100644
--- a/drivers/mtd/maps/physmap-gemini.c
+++ b/drivers/mtd/maps/physmap-gemini.c
@@ -46,11 +46,6 @@
 
 #define FLASH_PARALLEL_HIGH_PIN_CNT	(1 << 20)	/* else low pin cnt */
 
-static const struct of_device_id syscon_match[] = {
-	{ .compatible = "cortina,gemini-syscon" },
-	{ },
-};
-
 struct gemini_flash {
 	struct device *dev;
 	struct pinctrl *p;
diff --git a/drivers/mtd/mtdblock.c b/drivers/mtd/mtdblock.c
index 078e0f67377d..32e52d83b961 100644
--- a/drivers/mtd/mtdblock.c
+++ b/drivers/mtd/mtdblock.c
@@ -89,8 +89,6 @@ static int write_cached_data (struct mtdblk_dev *mtdblk)
 
 	ret = erase_write (mtd, mtdblk->cache_offset,
 			   mtdblk->cache_size, mtdblk->cache_data);
-	if (ret)
-		return ret;
 
 	/*
 	 * Here we could arguably set the cache state to STATE_CLEAN.
@@ -98,9 +96,14 @@ static int write_cached_data (struct mtdblk_dev *mtdblk)
 	 * be notified if this content is altered on the flash by other
 	 * means.  Let's declare it empty and leave buffering tasks to
 	 * the buffer cache instead.
+	 *
+	 * If this cache_offset points to a bad block, data cannot be
+	 * written to the device. Clear cache_state to avoid writing to
+	 * bad blocks repeatedly.
 	 */
-	mtdblk->cache_state = STATE_EMPTY;
-	return 0;
+	if (ret == 0 || ret == -EIO)
+		mtdblk->cache_state = STATE_EMPTY;
+	return ret;
 }
 
 
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index b47691e1b81c..76d832a88e0c 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -617,6 +617,19 @@ int add_mtd_device(struct mtd_info *mtd)
 		    !(mtd->flags & MTD_NO_ERASE)))
 		return -EINVAL;
 
+	/*
+	 * MTD_SLC_ON_MLC_EMULATION can only be set on partitions, when the
+	 * master is an MLC NAND and has a proper pairing scheme defined.
+	 * We also reject masters that implement ->_writev() for now, because
+	 * NAND controller drivers don't implement this hook, and adding the
+	 * SLC -> MLC address/length conversion to this path is useless if we
+	 * don't have a user.
+	 */
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION &&
+	    (!mtd_is_partition(mtd) || master->type != MTD_MLCNANDFLASH ||
+	     !master->pairing || master->_writev))
+		return -EINVAL;
+
 	mutex_lock(&mtd_table_mutex);
 
 	i = idr_alloc(&mtd_idr, mtd, 0, 0, GFP_KERNEL);
@@ -632,6 +645,14 @@ int add_mtd_device(struct mtd_info *mtd)
 	if (mtd->bitflip_threshold == 0)
 		mtd->bitflip_threshold = mtd->ecc_strength;
 
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION) {
+		int ngroups = mtd_pairing_groups(master);
+
+		mtd->erasesize /= ngroups;
+		mtd->size = (u64)mtd_div_by_eb(mtd->size, master) *
+			    mtd->erasesize;
+	}
+
 	if (is_power_of_2(mtd->erasesize))
 		mtd->erasesize_shift = ffs(mtd->erasesize) - 1;
 	else
@@ -1074,9 +1095,11 @@ int mtd_erase(struct mtd_info *mtd, struct erase_info *instr)
 {
 	struct mtd_info *master = mtd_get_master(mtd);
 	u64 mst_ofs = mtd_get_master_ofs(mtd, 0);
+	struct erase_info adjinstr;
 	int ret;
 
 	instr->fail_addr = MTD_FAIL_ADDR_UNKNOWN;
+	adjinstr = *instr;
 
 	if (!mtd->erasesize || !master->_erase)
 		return -ENOTSUPP;
@@ -1091,12 +1114,27 @@ int mtd_erase(struct mtd_info *mtd, struct erase_info *instr)
 
 	ledtrig_mtd_activity();
 
-	instr->addr += mst_ofs;
-	ret = master->_erase(master, instr);
-	if (instr->fail_addr != MTD_FAIL_ADDR_UNKNOWN)
-		instr->fail_addr -= mst_ofs;
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION) {
+		adjinstr.addr = (loff_t)mtd_div_by_eb(instr->addr, mtd) *
+				master->erasesize;
+		adjinstr.len = ((u64)mtd_div_by_eb(instr->addr + instr->len, mtd) *
+				master->erasesize) -
+			       adjinstr.addr;
+	}
+
+	adjinstr.addr += mst_ofs;
+
+	ret = master->_erase(master, &adjinstr);
+
+	if (adjinstr.fail_addr != MTD_FAIL_ADDR_UNKNOWN) {
+		instr->fail_addr = adjinstr.fail_addr - mst_ofs;
+		if (mtd->flags & MTD_SLC_ON_MLC_EMULATION) {
+			instr->fail_addr = mtd_div_by_eb(instr->fail_addr,
+							 master);
+			instr->fail_addr *= mtd->erasesize;
+		}
+	}
 
-	instr->addr -= mst_ofs;
 	return ret;
 }
 EXPORT_SYMBOL_GPL(mtd_erase);
@@ -1276,6 +1314,101 @@ static int mtd_check_oob_ops(struct mtd_info *mtd, loff_t offs,
 	return 0;
 }
 
+static int mtd_read_oob_std(struct mtd_info *mtd, loff_t from,
+			    struct mtd_oob_ops *ops)
+{
+	struct mtd_info *master = mtd_get_master(mtd);
+	int ret;
+
+	from = mtd_get_master_ofs(mtd, from);
+	if (master->_read_oob)
+		ret = master->_read_oob(master, from, ops);
+	else
+		ret = master->_read(master, from, ops->len, &ops->retlen,
+				    ops->datbuf);
+
+	return ret;
+}
+
+static int mtd_write_oob_std(struct mtd_info *mtd, loff_t to,
+			     struct mtd_oob_ops *ops)
+{
+	struct mtd_info *master = mtd_get_master(mtd);
+	int ret;
+
+	to = mtd_get_master_ofs(mtd, to);
+	if (master->_write_oob)
+		ret = master->_write_oob(master, to, ops);
+	else
+		ret = master->_write(master, to, ops->len, &ops->retlen,
+				     ops->datbuf);
+
+	return ret;
+}
+
+static int mtd_io_emulated_slc(struct mtd_info *mtd, loff_t start, bool read,
+			       struct mtd_oob_ops *ops)
+{
+	struct mtd_info *master = mtd_get_master(mtd);
+	int ngroups = mtd_pairing_groups(master);
+	int npairs = mtd_wunit_per_eb(master) / ngroups;
+	struct mtd_oob_ops adjops = *ops;
+	unsigned int wunit, oobavail;
+	struct mtd_pairing_info info;
+	int max_bitflips = 0;
+	u32 ebofs, pageofs;
+	loff_t base, pos;
+
+	ebofs = mtd_mod_by_eb(start, mtd);
+	base = (loff_t)mtd_div_by_eb(start, mtd) * master->erasesize;
+	info.group = 0;
+	info.pair = mtd_div_by_ws(ebofs, mtd);
+	pageofs = mtd_mod_by_ws(ebofs, mtd);
+	oobavail = mtd_oobavail(mtd, ops);
+
+	while (ops->retlen < ops->len || ops->oobretlen < ops->ooblen) {
+		int ret;
+
+		if (info.pair >= npairs) {
+			info.pair = 0;
+			base += master->erasesize;
+		}
+
+		wunit = mtd_pairing_info_to_wunit(master, &info);
+		pos = mtd_wunit_to_offset(mtd, base, wunit);
+
+		adjops.len = ops->len - ops->retlen;
+		if (adjops.len > mtd->writesize - pageofs)
+			adjops.len = mtd->writesize - pageofs;
+
+		adjops.ooblen = ops->ooblen - ops->oobretlen;
+		if (adjops.ooblen > oobavail - adjops.ooboffs)
+			adjops.ooblen = oobavail - adjops.ooboffs;
+
+		if (read) {
+			ret = mtd_read_oob_std(mtd, pos + pageofs, &adjops);
+			if (ret > 0)
+				max_bitflips = max(max_bitflips, ret);
+		} else {
+			ret = mtd_write_oob_std(mtd, pos + pageofs, &adjops);
+		}
+
+		if (ret < 0)
+			return ret;
+
+		max_bitflips = max(max_bitflips, ret);
+		ops->retlen += adjops.retlen;
+		ops->oobretlen += adjops.oobretlen;
+		adjops.datbuf += adjops.retlen;
+		adjops.oobbuf += adjops.oobretlen;
+		adjops.ooboffs = 0;
+		pageofs = 0;
+		info.pair++;
+	}
+
+	return max_bitflips;
+}
+
 int mtd_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops)
 {
 	struct mtd_info *master = mtd_get_master(mtd);
@@ -1294,12 +1427,10 @@ int mtd_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops)
 	if (!master->_read_oob && (!master->_read || ops->oobbuf))
 		return -EOPNOTSUPP;
 
-	from = mtd_get_master_ofs(mtd, from);
-	if (master->_read_oob)
-		ret_code = master->_read_oob(master, from, ops);
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION)
+		ret_code = mtd_io_emulated_slc(mtd, from, true, ops);
 	else
-		ret_code = master->_read(master, from, ops->len, &ops->retlen,
-					 ops->datbuf);
+		ret_code = mtd_read_oob_std(mtd, from, ops);
 
 	mtd_update_ecc_stats(mtd, master, &old_stats);
 
@@ -1338,13 +1469,10 @@ int mtd_write_oob(struct mtd_info *mtd, loff_t to,
 	if (!master->_write_oob && (!master->_write || ops->oobbuf))
 		return -EOPNOTSUPP;
 
-	to = mtd_get_master_ofs(mtd, to);
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION)
+		return mtd_io_emulated_slc(mtd, to, false, ops);
 
-	if (master->_write_oob)
-		return master->_write_oob(master, to, ops);
-	else
-		return master->_write(master, to, ops->len, &ops->retlen,
-				      ops->datbuf);
+	return mtd_write_oob_std(mtd, to, ops);
 }
 EXPORT_SYMBOL_GPL(mtd_write_oob);
 
@@ -1672,7 +1800,7 @@ EXPORT_SYMBOL_GPL(mtd_ooblayout_get_databytes);
  * @start: first ECC byte to set
  * @nbytes: number of ECC bytes to set
  *
- * Works like mtd_ooblayout_get_bytes(), except it acts on free bytes.
+ * Works like mtd_ooblayout_set_bytes(), except it acts on free bytes.
  *
  * Returns zero on success, a negative error code otherwise.
  */
@@ -1817,6 +1945,12 @@ int mtd_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 		return -EINVAL;
 	if (!len)
 		return 0;
+
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION) {
+		ofs = (loff_t)mtd_div_by_eb(ofs, mtd) * master->erasesize;
+		len = (u64)mtd_div_by_eb(len, mtd) * master->erasesize;
+	}
+
 	return master->_lock(master, mtd_get_master_ofs(mtd, ofs), len);
 }
 EXPORT_SYMBOL_GPL(mtd_lock);
@@ -1831,6 +1965,12 @@ int mtd_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 		return -EINVAL;
 	if (!len)
 		return 0;
+
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION) {
+		ofs = (loff_t)mtd_div_by_eb(ofs, mtd) * master->erasesize;
+		len = (u64)mtd_div_by_eb(len, mtd) * master->erasesize;
+	}
+
 	return master->_unlock(master, mtd_get_master_ofs(mtd, ofs), len);
 }
 EXPORT_SYMBOL_GPL(mtd_unlock);
@@ -1845,6 +1985,12 @@ int mtd_is_locked(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 		return -EINVAL;
 	if (!len)
 		return 0;
+
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION) {
+		ofs = (loff_t)mtd_div_by_eb(ofs, mtd) * master->erasesize;
+		len = (u64)mtd_div_by_eb(len, mtd) * master->erasesize;
+	}
+
 	return master->_is_locked(master, mtd_get_master_ofs(mtd, ofs), len);
 }
 EXPORT_SYMBOL_GPL(mtd_is_locked);
@@ -1857,6 +2003,10 @@ int mtd_block_isreserved(struct mtd_info *mtd, loff_t ofs)
 		return -EINVAL;
 	if (!master->_block_isreserved)
 		return 0;
+
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION)
+		ofs = (loff_t)mtd_div_by_eb(ofs, mtd) * master->erasesize;
+
 	return master->_block_isreserved(master, mtd_get_master_ofs(mtd, ofs));
 }
 EXPORT_SYMBOL_GPL(mtd_block_isreserved);
@@ -1869,6 +2019,10 @@ int mtd_block_isbad(struct mtd_info *mtd, loff_t ofs)
 		return -EINVAL;
 	if (!master->_block_isbad)
 		return 0;
+
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION)
+		ofs = (loff_t)mtd_div_by_eb(ofs, mtd) * master->erasesize;
+
 	return master->_block_isbad(master, mtd_get_master_ofs(mtd, ofs));
 }
 EXPORT_SYMBOL_GPL(mtd_block_isbad);
@@ -1885,6 +2039,9 @@ int mtd_block_markbad(struct mtd_info *mtd, loff_t ofs)
 	if (!(mtd->flags & MTD_WRITEABLE))
 		return -EROFS;
 
+	if (mtd->flags & MTD_SLC_ON_MLC_EMULATION)
+		ofs = (loff_t)mtd_div_by_eb(ofs, mtd) * master->erasesize;
+
 	ret = master->_block_markbad(master, mtd_get_master_ofs(mtd, ofs));
 	if (ret)
 		return ret;
diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
index 3f6025684f58..c3575b686f79 100644
--- a/drivers/mtd/mtdpart.c
+++ b/drivers/mtd/mtdpart.c
@@ -35,9 +35,12 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 					   const struct mtd_partition *part,
 					   int partno, uint64_t cur_offset)
 {
-	int wr_alignment = (parent->flags & MTD_NO_ERASE) ? parent->writesize :
-							    parent->erasesize;
-	struct mtd_info *child, *master = mtd_get_master(parent);
+	struct mtd_info *master = mtd_get_master(parent);
+	int wr_alignment = (parent->flags & MTD_NO_ERASE) ?
+			   master->writesize : master->erasesize;
+	u64 parent_size = mtd_is_partition(parent) ?
+			  parent->part.size : parent->size;
+	struct mtd_info *child;
 	u32 remainder;
 	char *name;
 	u64 tmp;
@@ -56,8 +59,9 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 	/* set up the MTD object for this partition */
 	child->type = parent->type;
 	child->part.flags = parent->flags & ~part->mask_flags;
+	child->part.flags |= part->add_flags;
 	child->flags = child->part.flags;
-	child->size = part->size;
+	child->part.size = part->size;
 	child->writesize = parent->writesize;
 	child->writebufsize = parent->writebufsize;
 	child->oobsize = parent->oobsize;
@@ -98,29 +102,29 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 	}
 	if (child->part.offset == MTDPART_OFS_RETAIN) {
 		child->part.offset = cur_offset;
-		if (parent->size - child->part.offset >= child->size) {
-			child->size = parent->size - child->part.offset -
-				      child->size;
+		if (parent_size - child->part.offset >= child->part.size) {
+			child->part.size = parent_size - child->part.offset -
+					   child->part.size;
 		} else {
 			printk(KERN_ERR "mtd partition \"%s\" doesn't have enough space: %#llx < %#llx, disabled\n",
-				part->name, parent->size - child->part.offset,
-				child->size);
+				part->name, parent_size - child->part.offset,
+				child->part.size);
 			/* register to preserve ordering */
 			goto out_register;
 		}
 	}
-	if (child->size == MTDPART_SIZ_FULL)
-		child->size = parent->size - child->part.offset;
+	if (child->part.size == MTDPART_SIZ_FULL)
+		child->part.size = parent_size - child->part.offset;
 
 	printk(KERN_NOTICE "0x%012llx-0x%012llx : \"%s\"\n",
-	       child->part.offset, child->part.offset + child->size,
+	       child->part.offset, child->part.offset + child->part.size,
 	       child->name);
 
 	/* let's do some sanity checks */
-	if (child->part.offset >= parent->size) {
+	if (child->part.offset >= parent_size) {
 		/* let's register it anyway to preserve ordering */
 		child->part.offset = 0;
-		child->size = 0;
+		child->part.size = 0;
 
 		/* Initialize ->erasesize to make add_mtd_device() happy. */
 		child->erasesize = parent->erasesize;
@@ -128,15 +132,16 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 			part->name);
 		goto out_register;
 	}
-	if (child->part.offset + child->size > parent->size) {
-		child->size = parent->size - child->part.offset;
+	if (child->part.offset + child->part.size > parent->size) {
+		child->part.size = parent_size - child->part.offset;
 		printk(KERN_WARNING"mtd: partition \"%s\" extends beyond the end of device \"%s\" -- size truncated to %#llx\n",
-			part->name, parent->name, child->size);
+			part->name, parent->name, child->part.size);
 	}
+
 	if (parent->numeraseregions > 1) {
 		/* Deal with variable erase size stuff */
 		int i, max = parent->numeraseregions;
-		u64 end = child->part.offset + child->size;
+		u64 end = child->part.offset + child->part.size;
 		struct mtd_erase_region_info *regions = parent->eraseregions;
 
 		/* Find the first erase regions which is part of this
@@ -156,7 +161,7 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 		BUG_ON(child->erasesize == 0);
 	} else {
 		/* Single erase size */
-		child->erasesize = parent->erasesize;
+		child->erasesize = master->erasesize;
 	}
 
 	/*
@@ -178,7 +183,7 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 			part->name);
 	}
 
-	tmp = mtd_get_master_ofs(child, 0) + child->size;
+	tmp = mtd_get_master_ofs(child, 0) + child->part.size;
 	remainder = do_div(tmp, wr_alignment);
 	if ((child->flags & MTD_WRITEABLE) && remainder) {
 		child->flags &= ~MTD_WRITEABLE;
@@ -186,6 +191,7 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 			part->name);
 	}
 
+	child->size = child->part.size;
 	child->ecc_step_size = parent->ecc_step_size;
 	child->ecc_strength = parent->ecc_strength;
 	child->bitflip_threshold = parent->bitflip_threshold;
@@ -193,7 +199,7 @@ static struct mtd_info *allocate_partition(struct mtd_info *parent,
 	if (master->_block_isbad) {
 		uint64_t offs = 0;
 
-		while (offs < child->size) {
+		while (offs < child->part.size) {
 			if (mtd_block_isreserved(child, offs))
 				child->ecc_stats.bbtblocks++;
 			else if (mtd_block_isbad(child, offs))
@@ -234,6 +240,8 @@ int mtd_add_partition(struct mtd_info *parent, const char *name,
 		      long long offset, long long length)
 {
 	struct mtd_info *master = mtd_get_master(parent);
+	u64 parent_size = mtd_is_partition(parent) ?
+			  parent->part.size : parent->size;
 	struct mtd_partition part;
 	struct mtd_info *child;
 	int ret = 0;
@@ -244,7 +252,7 @@ int mtd_add_partition(struct mtd_info *parent, const char *name,
 		return -EINVAL;
 
 	if (length == MTDPART_SIZ_FULL)
-		length = parent->size - offset;
+		length = parent_size - offset;
 
 	if (length <= 0)
 		return -EINVAL;
@@ -419,7 +427,7 @@ int add_mtd_partitions(struct mtd_info *parent,
 		/* Look for subpartitions */
 		parse_mtd_partitions(child, parts[i].types, NULL);
 
-		cur_offset = child->part.offset + child->size;
+		cur_offset = child->part.offset + child->part.size;
 	}
 
 	return 0;
diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig
index a80a46bb5b8b..113f61052269 100644
--- a/drivers/mtd/nand/raw/Kconfig
+++ b/drivers/mtd/nand/raw/Kconfig
@@ -213,10 +213,6 @@ config MTD_NAND_MLC_LPC32XX
 	  Please check the actual NAND chip connected and its support
 	  by the MLC NAND controller.
 
-config MTD_NAND_CM_X270
-	tristate "CM-X270 modules NAND controller"
-	depends on MACH_ARMCORE
-
 config MTD_NAND_PASEMI
 	tristate "PA Semi PWRficient NAND controller"
 	depends on PPC_PASEMI
@@ -457,6 +453,14 @@ config MTD_NAND_CADENCE
 	  Enable the driver for NAND flash on platforms using a Cadence NAND
 	  controller.
 
+config MTD_NAND_ARASAN
+	tristate "Support for Arasan NAND flash controller"
+	depends on HAS_IOMEM && HAS_DMA
+	select BCH
+	help
+	  Enables the driver for the Arasan NAND flash controller on
+	  Zynq Ultrascale+ MPSoC.
+
 comment "Misc"
 
 config MTD_SM_COMMON
diff --git a/drivers/mtd/nand/raw/Makefile b/drivers/mtd/nand/raw/Makefile
index 2d136b158fb7..2930f5b9015d 100644
--- a/drivers/mtd/nand/raw/Makefile
+++ b/drivers/mtd/nand/raw/Makefile
@@ -25,7 +25,6 @@ obj-$(CONFIG_MTD_NAND_GPIO)		+= gpio.o
 omap2_nand-objs := omap2.o
 obj-$(CONFIG_MTD_NAND_OMAP2) 		+= omap2_nand.o
 obj-$(CONFIG_MTD_NAND_OMAP_BCH_BUILD)	+= omap_elm.o
-obj-$(CONFIG_MTD_NAND_CM_X270)		+= cmx270_nand.o
 obj-$(CONFIG_MTD_NAND_MARVELL)		+= marvell_nand.o
 obj-$(CONFIG_MTD_NAND_TMIO)		+= tmio_nand.o
 obj-$(CONFIG_MTD_NAND_PLATFORM)		+= plat_nand.o
@@ -58,6 +57,7 @@ obj-$(CONFIG_MTD_NAND_TEGRA)		+= tegra_nand.o
 obj-$(CONFIG_MTD_NAND_STM32_FMC2)	+= stm32_fmc2_nand.o
 obj-$(CONFIG_MTD_NAND_MESON)		+= meson_nand.o
 obj-$(CONFIG_MTD_NAND_CADENCE)		+= cadence-nand-controller.o
+obj-$(CONFIG_MTD_NAND_ARASAN)		+= arasan-nand-controller.o
 
 nand-objs := nand_base.o nand_legacy.o nand_bbt.o nand_timings.o nand_ids.o
 nand-objs += nand_onfi.o
diff --git a/drivers/mtd/nand/raw/ams-delta.c b/drivers/mtd/nand/raw/ams-delta.c
index d66dab25df20..3711e7a0436c 100644
--- a/drivers/mtd/nand/raw/ams-delta.c
+++ b/drivers/mtd/nand/raw/ams-delta.c
@@ -387,12 +387,15 @@ static int gpio_nand_remove(struct platform_device *pdev)
 {
 	struct gpio_nand *priv = platform_get_drvdata(pdev);
 	struct mtd_info *mtd = nand_to_mtd(&priv->nand_chip);
+	int ret;
 
 	/* Apply write protection */
 	gpiod_set_value(priv->gpiod_nwp, 1);
 
 	/* Unregister device */
-	nand_release(mtd_to_nand(mtd));
+	ret = mtd_device_unregister(mtd);
+	WARN_ON(ret);
+	nand_cleanup(mtd_to_nand(mtd));
 
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/arasan-nand-controller.c b/drivers/mtd/nand/raw/arasan-nand-controller.c
new file mode 100644
index 000000000000..7141dcccba3c
--- /dev/null
+++ b/drivers/mtd/nand/raw/arasan-nand-controller.c
@@ -0,0 +1,1297 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Arasan NAND Flash Controller Driver
+ *
+ * Copyright (C) 2014 - 2020 Xilinx, Inc.
+ * Author:
+ *   Miquel Raynal <miquel.raynal@bootlin.com>
+ * Original work (fully rewritten):
+ *   Punnaiah Choudary Kalluri <punnaia@xilinx.com>
+ *   Naga Sureshkumar Relli <nagasure@xilinx.com>
+ */
+
+#include <linux/bch.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/mtd/mtd.h>
+#include <linux/mtd/partitions.h>
+#include <linux/mtd/rawnand.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define PKT_REG				0x00
+#define   PKT_SIZE(x)			FIELD_PREP(GENMASK(10, 0), (x))
+#define   PKT_STEPS(x)			FIELD_PREP(GENMASK(23, 12), (x))
+
+#define MEM_ADDR1_REG			0x04
+
+#define MEM_ADDR2_REG			0x08
+#define   ADDR2_STRENGTH(x)		FIELD_PREP(GENMASK(27, 25), (x))
+#define   ADDR2_CS(x)			FIELD_PREP(GENMASK(31, 30), (x))
+
+#define CMD_REG				0x0C
+#define   CMD_1(x)			FIELD_PREP(GENMASK(7, 0), (x))
+#define   CMD_2(x)			FIELD_PREP(GENMASK(15, 8), (x))
+#define   CMD_PAGE_SIZE(x)		FIELD_PREP(GENMASK(25, 23), (x))
+#define   CMD_DMA_ENABLE		BIT(27)
+#define   CMD_NADDRS(x)			FIELD_PREP(GENMASK(30, 28), (x))
+#define   CMD_ECC_ENABLE		BIT(31)
+
+#define PROG_REG			0x10
+#define   PROG_PGRD			BIT(0)
+#define   PROG_ERASE			BIT(2)
+#define   PROG_STATUS			BIT(3)
+#define   PROG_PGPROG			BIT(4)
+#define   PROG_RDID			BIT(6)
+#define   PROG_RDPARAM			BIT(7)
+#define   PROG_RST			BIT(8)
+#define   PROG_GET_FEATURE		BIT(9)
+#define   PROG_SET_FEATURE		BIT(10)
+
+#define INTR_STS_EN_REG			0x14
+#define INTR_SIG_EN_REG			0x18
+#define INTR_STS_REG			0x1C
+#define   WRITE_READY			BIT(0)
+#define   READ_READY			BIT(1)
+#define   XFER_COMPLETE			BIT(2)
+#define   DMA_BOUNDARY			BIT(6)
+#define   EVENT_MASK			GENMASK(7, 0)
+
+#define READY_STS_REG			0x20
+
+#define DMA_ADDR0_REG			0x50
+#define DMA_ADDR1_REG			0x24
+
+#define FLASH_STS_REG			0x28
+
+#define DATA_PORT_REG			0x30
+
+#define ECC_CONF_REG			0x34
+#define   ECC_CONF_COL(x)		FIELD_PREP(GENMASK(15, 0), (x))
+#define   ECC_CONF_LEN(x)		FIELD_PREP(GENMASK(26, 16), (x))
+#define   ECC_CONF_BCH_EN		BIT(27)
+
+#define ECC_ERR_CNT_REG			0x38
+#define   GET_PKT_ERR_CNT(x)		FIELD_GET(GENMASK(7, 0), (x))
+#define   GET_PAGE_ERR_CNT(x)		FIELD_GET(GENMASK(16, 8), (x))
+
+#define ECC_SP_REG			0x3C
+#define   ECC_SP_CMD1(x)		FIELD_PREP(GENMASK(7, 0), (x))
+#define   ECC_SP_CMD2(x)		FIELD_PREP(GENMASK(15, 8), (x))
+#define   ECC_SP_ADDRS(x)		FIELD_PREP(GENMASK(30, 28), (x))
+
+#define ECC_1ERR_CNT_REG		0x40
+#define ECC_2ERR_CNT_REG		0x44
+
+#define DATA_INTERFACE_REG		0x6C
+#define   DIFACE_SDR_MODE(x)		FIELD_PREP(GENMASK(2, 0), (x))
+#define   DIFACE_DDR_MODE(x)		FIELD_PREP(GENMASK(5, 3), (X))
+#define   DIFACE_SDR			0
+#define   DIFACE_NVDDR			BIT(9)
+
+#define ANFC_MAX_CS			2
+#define ANFC_DFLT_TIMEOUT_US		1000000
+#define ANFC_MAX_CHUNK_SIZE		SZ_1M
+#define ANFC_MAX_PARAM_SIZE		SZ_4K
+#define ANFC_MAX_STEPS			SZ_2K
+#define ANFC_MAX_PKT_SIZE		(SZ_2K - 1)
+#define ANFC_MAX_ADDR_CYC		5U
+#define ANFC_RSVD_ECC_BYTES		21
+
+#define ANFC_XLNX_SDR_DFLT_CORE_CLK	100000000
+#define ANFC_XLNX_SDR_HS_CORE_CLK	80000000
+
+/**
+ * struct anfc_op - Defines how to execute an operation
+ * @pkt_reg: Packet register
+ * @addr1_reg: Memory address 1 register
+ * @addr2_reg: Memory address 2 register
+ * @cmd_reg: Command register
+ * @prog_reg: Program register
+ * @steps: Number of "packets" to read/write
+ * @rdy_timeout_ms: Timeout for waits on Ready/Busy pin
+ * @len: Data transfer length
+ * @read: Data transfer direction from the controller point of view
+ */
+struct anfc_op {
+	u32 pkt_reg;
+	u32 addr1_reg;
+	u32 addr2_reg;
+	u32 cmd_reg;
+	u32 prog_reg;
+	int steps;
+	unsigned int rdy_timeout_ms;
+	unsigned int len;
+	bool read;
+	u8 *buf;
+};
+
+/**
+ * struct anand - Defines the NAND chip related information
+ * @node:		Used to store NAND chips into a list
+ * @chip:		NAND chip information structure
+ * @cs:			Chip select line
+ * @rb:			Ready-busy line
+ * @page_sz:		Register value of the page_sz field to use
+ * @clk:		Expected clock frequency to use
+ * @timings:		Data interface timing mode to use
+ * @ecc_conf:		Hardware ECC configuration value
+ * @strength:		Register value of the ECC strength
+ * @raddr_cycles:	Row address cycle information
+ * @caddr_cycles:	Column address cycle information
+ * @ecc_bits:		Exact number of ECC bits per syndrome
+ * @ecc_total:		Total number of ECC bytes
+ * @errloc:		Array of errors located with soft BCH
+ * @hw_ecc:		Buffer to store syndromes computed by hardware
+ * @bch:		BCH structure
+ */
+struct anand {
+	struct list_head node;
+	struct nand_chip chip;
+	unsigned int cs;
+	unsigned int rb;
+	unsigned int page_sz;
+	unsigned long clk;
+	u32 timings;
+	u32 ecc_conf;
+	u32 strength;
+	u16 raddr_cycles;
+	u16 caddr_cycles;
+	unsigned int ecc_bits;
+	unsigned int ecc_total;
+	unsigned int *errloc;
+	u8 *hw_ecc;
+	struct bch_control *bch;
+};
+
+/**
+ * struct arasan_nfc - Defines the Arasan NAND flash controller driver instance
+ * @dev:		Pointer to the device structure
+ * @base:		Remapped register area
+ * @controller_clk:		Pointer to the system clock
+ * @bus_clk:		Pointer to the flash clock
+ * @controller:		Base controller structure
+ * @chips:		List of all NAND chips attached to the controller
+ * @assigned_cs:	Bitmask describing already assigned CS lines
+ * @cur_clk:		Current clock rate
+ */
+struct arasan_nfc {
+	struct device *dev;
+	void __iomem *base;
+	struct clk *controller_clk;
+	struct clk *bus_clk;
+	struct nand_controller controller;
+	struct list_head chips;
+	unsigned long assigned_cs;
+	unsigned int cur_clk;
+};
+
+static struct anand *to_anand(struct nand_chip *nand)
+{
+	return container_of(nand, struct anand, chip);
+}
+
+static struct arasan_nfc *to_anfc(struct nand_controller *ctrl)
+{
+	return container_of(ctrl, struct arasan_nfc, controller);
+}
+
+static int anfc_wait_for_event(struct arasan_nfc *nfc, unsigned int event)
+{
+	u32 val;
+	int ret;
+
+	ret = readl_relaxed_poll_timeout(nfc->base + INTR_STS_REG, val,
+					 val & event, 0,
+					 ANFC_DFLT_TIMEOUT_US);
+	if (ret) {
+		dev_err(nfc->dev, "Timeout waiting for event 0x%x\n", event);
+		return -ETIMEDOUT;
+	}
+
+	writel_relaxed(event, nfc->base + INTR_STS_REG);
+
+	return 0;
+}
+
+static int anfc_wait_for_rb(struct arasan_nfc *nfc, struct nand_chip *chip,
+			    unsigned int timeout_ms)
+{
+	struct anand *anand = to_anand(chip);
+	u32 val;
+	int ret;
+
+	/* There is no R/B interrupt, we must poll a register */
+	ret = readl_relaxed_poll_timeout(nfc->base + READY_STS_REG, val,
+					 val & BIT(anand->rb),
+					 1, timeout_ms * 1000);
+	if (ret) {
+		dev_err(nfc->dev, "Timeout waiting for R/B 0x%x\n",
+			readl_relaxed(nfc->base + READY_STS_REG));
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void anfc_trigger_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
+{
+	writel_relaxed(nfc_op->pkt_reg, nfc->base + PKT_REG);
+	writel_relaxed(nfc_op->addr1_reg, nfc->base + MEM_ADDR1_REG);
+	writel_relaxed(nfc_op->addr2_reg, nfc->base + MEM_ADDR2_REG);
+	writel_relaxed(nfc_op->cmd_reg, nfc->base + CMD_REG);
+	writel_relaxed(nfc_op->prog_reg, nfc->base + PROG_REG);
+}
+
+static int anfc_pkt_len_config(unsigned int len, unsigned int *steps,
+			       unsigned int *pktsize)
+{
+	unsigned int nb, sz;
+
+	for (nb = 1; nb < ANFC_MAX_STEPS; nb *= 2) {
+		sz = len / nb;
+		if (sz <= ANFC_MAX_PKT_SIZE)
+			break;
+	}
+
+	if (sz * nb != len)
+		return -ENOTSUPP;
+
+	if (steps)
+		*steps = nb;
+
+	if (pktsize)
+		*pktsize = sz;
+
+	return 0;
+}
+
+/*
+ * When using the embedded hardware ECC engine, the controller is in charge of
+ * feeding the engine with, first, the ECC residue present in the data array.
+ * A typical read operation is:
+ * 1/ Assert the read operation by sending the relevant command/address cycles
+ *    but targeting the column of the first ECC bytes in the OOB area instead of
+ *    the main data directly.
+ * 2/ After having read the relevant number of ECC bytes, the controller uses
+ *    the RNDOUT/RNDSTART commands which are set into the "ECC Spare Command
+ *    Register" to move the pointer back at the beginning of the main data.
+ * 3/ It will read the content of the main area for a given size (pktsize) and
+ *    will feed the ECC engine with this buffer again.
+ * 4/ The ECC engine derives the ECC bytes for the given data and compare them
+ *    with the ones already received. It eventually trigger status flags and
+ *    then set the "Buffer Read Ready" flag.
+ * 5/ The corrected data is then available for reading from the data port
+ *    register.
+ *
+ * The hardware BCH ECC engine is known to be inconstent in BCH mode and never
+ * reports uncorrectable errors. Because of this bug, we have to use the
+ * software BCH implementation in the read path.
+ */
+static int anfc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
+				 int oob_required, int page)
+{
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	struct anand *anand = to_anand(chip);
+	unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
+	unsigned int max_bitflips = 0;
+	dma_addr_t dma_addr;
+	int step, ret;
+	struct anfc_op nfc_op = {
+		.pkt_reg =
+			PKT_SIZE(chip->ecc.size) |
+			PKT_STEPS(chip->ecc.steps),
+		.addr1_reg =
+			(page & 0xFF) << (8 * (anand->caddr_cycles)) |
+			(((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
+		.addr2_reg =
+			((page >> 16) & 0xFF) |
+			ADDR2_STRENGTH(anand->strength) |
+			ADDR2_CS(anand->cs),
+		.cmd_reg =
+			CMD_1(NAND_CMD_READ0) |
+			CMD_2(NAND_CMD_READSTART) |
+			CMD_PAGE_SIZE(anand->page_sz) |
+			CMD_DMA_ENABLE |
+			CMD_NADDRS(anand->caddr_cycles +
+				   anand->raddr_cycles),
+		.prog_reg = PROG_PGRD,
+	};
+
+	dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(nfc->dev, dma_addr)) {
+		dev_err(nfc->dev, "Buffer mapping error");
+		return -EIO;
+	}
+
+	writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
+	writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
+
+	anfc_trigger_op(nfc, &nfc_op);
+
+	ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
+	dma_unmap_single(nfc->dev, dma_addr, len, DMA_FROM_DEVICE);
+	if (ret) {
+		dev_err(nfc->dev, "Error reading page %d\n", page);
+		return ret;
+	}
+
+	/* Store the raw OOB bytes as well */
+	ret = nand_change_read_column_op(chip, mtd->writesize, chip->oob_poi,
+					 mtd->oobsize, 0);
+	if (ret)
+		return ret;
+
+	/*
+	 * For each step, compute by softare the BCH syndrome over the raw data.
+	 * Compare the theoretical amount of errors and compare with the
+	 * hardware engine feedback.
+	 */
+	for (step = 0; step < chip->ecc.steps; step++) {
+		u8 *raw_buf = &buf[step * chip->ecc.size];
+		unsigned int bit, byte;
+		int bf, i;
+
+		/* Extract the syndrome, it is not necessarily aligned */
+		memset(anand->hw_ecc, 0, chip->ecc.bytes);
+		nand_extract_bits(anand->hw_ecc, 0,
+				  &chip->oob_poi[mtd->oobsize - anand->ecc_total],
+				  anand->ecc_bits * step, anand->ecc_bits);
+
+		bf = bch_decode(anand->bch, raw_buf, chip->ecc.size,
+				anand->hw_ecc, NULL, NULL, anand->errloc);
+		if (!bf) {
+			continue;
+		} else if (bf > 0) {
+			for (i = 0; i < bf; i++) {
+				/* Only correct the data, not the syndrome */
+				if (anand->errloc[i] < (chip->ecc.size * 8)) {
+					bit = BIT(anand->errloc[i] & 7);
+					byte = anand->errloc[i] >> 3;
+					raw_buf[byte] ^= bit;
+				}
+			}
+
+			mtd->ecc_stats.corrected += bf;
+			max_bitflips = max_t(unsigned int, max_bitflips, bf);
+
+			continue;
+		}
+
+		bf = nand_check_erased_ecc_chunk(raw_buf, chip->ecc.size,
+						 NULL, 0, NULL, 0,
+						 chip->ecc.strength);
+		if (bf > 0) {
+			mtd->ecc_stats.corrected += bf;
+			max_bitflips = max_t(unsigned int, max_bitflips, bf);
+			memset(raw_buf, 0xFF, chip->ecc.size);
+		} else if (bf < 0) {
+			mtd->ecc_stats.failed++;
+		}
+	}
+
+	return 0;
+}
+
+static int anfc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
+				  int oob_required, int page)
+{
+	struct anand *anand = to_anand(chip);
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
+	dma_addr_t dma_addr;
+	int ret;
+	struct anfc_op nfc_op = {
+		.pkt_reg =
+			PKT_SIZE(chip->ecc.size) |
+			PKT_STEPS(chip->ecc.steps),
+		.addr1_reg =
+			(page & 0xFF) << (8 * (anand->caddr_cycles)) |
+			(((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
+		.addr2_reg =
+			((page >> 16) & 0xFF) |
+			ADDR2_STRENGTH(anand->strength) |
+			ADDR2_CS(anand->cs),
+		.cmd_reg =
+			CMD_1(NAND_CMD_SEQIN) |
+			CMD_2(NAND_CMD_PAGEPROG) |
+			CMD_PAGE_SIZE(anand->page_sz) |
+			CMD_DMA_ENABLE |
+			CMD_NADDRS(anand->caddr_cycles +
+				   anand->raddr_cycles) |
+			CMD_ECC_ENABLE,
+		.prog_reg = PROG_PGPROG,
+	};
+
+	writel_relaxed(anand->ecc_conf, nfc->base + ECC_CONF_REG);
+	writel_relaxed(ECC_SP_CMD1(NAND_CMD_RNDIN) |
+		       ECC_SP_ADDRS(anand->caddr_cycles),
+		       nfc->base + ECC_SP_REG);
+
+	dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_TO_DEVICE);
+	if (dma_mapping_error(nfc->dev, dma_addr)) {
+		dev_err(nfc->dev, "Buffer mapping error");
+		return -EIO;
+	}
+
+	writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
+	writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
+
+	anfc_trigger_op(nfc, &nfc_op);
+	ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
+	dma_unmap_single(nfc->dev, dma_addr, len, DMA_TO_DEVICE);
+	if (ret) {
+		dev_err(nfc->dev, "Error writing page %d\n", page);
+		return ret;
+	}
+
+	/* Spare data is not protected */
+	if (oob_required)
+		ret = nand_write_oob_std(chip, page);
+
+	return ret;
+}
+
+/* NAND framework ->exec_op() hooks and related helpers */
+static int anfc_parse_instructions(struct nand_chip *chip,
+				   const struct nand_subop *subop,
+				   struct anfc_op *nfc_op)
+{
+	struct anand *anand = to_anand(chip);
+	const struct nand_op_instr *instr = NULL;
+	bool first_cmd = true;
+	unsigned int op_id;
+	int ret, i;
+
+	memset(nfc_op, 0, sizeof(*nfc_op));
+	nfc_op->addr2_reg = ADDR2_CS(anand->cs);
+	nfc_op->cmd_reg = CMD_PAGE_SIZE(anand->page_sz);
+
+	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
+		unsigned int offset, naddrs, pktsize;
+		const u8 *addrs;
+		u8 *buf;
+
+		instr = &subop->instrs[op_id];
+
+		switch (instr->type) {
+		case NAND_OP_CMD_INSTR:
+			if (first_cmd)
+				nfc_op->cmd_reg |= CMD_1(instr->ctx.cmd.opcode);
+			else
+				nfc_op->cmd_reg |= CMD_2(instr->ctx.cmd.opcode);
+
+			first_cmd = false;
+			break;
+
+		case NAND_OP_ADDR_INSTR:
+			offset = nand_subop_get_addr_start_off(subop, op_id);
+			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
+			addrs = &instr->ctx.addr.addrs[offset];
+			nfc_op->cmd_reg |= CMD_NADDRS(naddrs);
+
+			for (i = 0; i < min(ANFC_MAX_ADDR_CYC, naddrs); i++) {
+				if (i < 4)
+					nfc_op->addr1_reg |= (u32)addrs[i] << i * 8;
+				else
+					nfc_op->addr2_reg |= addrs[i];
+			}
+
+			break;
+		case NAND_OP_DATA_IN_INSTR:
+			nfc_op->read = true;
+			fallthrough;
+		case NAND_OP_DATA_OUT_INSTR:
+			offset = nand_subop_get_data_start_off(subop, op_id);
+			buf = instr->ctx.data.buf.in;
+			nfc_op->buf = &buf[offset];
+			nfc_op->len = nand_subop_get_data_len(subop, op_id);
+			ret = anfc_pkt_len_config(nfc_op->len, &nfc_op->steps,
+						  &pktsize);
+			if (ret)
+				return ret;
+
+			/*
+			 * Number of DATA cycles must be aligned on 4, this
+			 * means the controller might read/write more than
+			 * requested. This is harmless most of the time as extra
+			 * DATA are discarded in the write path and read pointer
+			 * adjusted in the read path.
+			 *
+			 * FIXME: The core should mark operations where
+			 * reading/writing more is allowed so the exec_op()
+			 * implementation can take the right decision when the
+			 * alignment constraint is not met: adjust the number of
+			 * DATA cycles when it's allowed, reject the operation
+			 * otherwise.
+			 */
+			nfc_op->pkt_reg |= PKT_SIZE(round_up(pktsize, 4)) |
+					   PKT_STEPS(nfc_op->steps);
+			break;
+		case NAND_OP_WAITRDY_INSTR:
+			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int anfc_rw_pio_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
+{
+	unsigned int dwords = (nfc_op->len / 4) / nfc_op->steps;
+	unsigned int last_len = nfc_op->len % 4;
+	unsigned int offset, dir;
+	u8 *buf = nfc_op->buf;
+	int ret, i;
+
+	for (i = 0; i < nfc_op->steps; i++) {
+		dir = nfc_op->read ? READ_READY : WRITE_READY;
+		ret = anfc_wait_for_event(nfc, dir);
+		if (ret) {
+			dev_err(nfc->dev, "PIO %s ready signal not received\n",
+				nfc_op->read ? "Read" : "Write");
+			return ret;
+		}
+
+		offset = i * (dwords * 4);
+		if (nfc_op->read)
+			ioread32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
+				     dwords);
+		else
+			iowrite32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
+				      dwords);
+	}
+
+	if (last_len) {
+		u32 remainder;
+
+		offset = nfc_op->len - last_len;
+
+		if (nfc_op->read) {
+			remainder = readl_relaxed(nfc->base + DATA_PORT_REG);
+			memcpy(&buf[offset], &remainder, last_len);
+		} else {
+			memcpy(&remainder, &buf[offset], last_len);
+			writel_relaxed(remainder, nfc->base + DATA_PORT_REG);
+		}
+	}
+
+	return anfc_wait_for_event(nfc, XFER_COMPLETE);
+}
+
+static int anfc_misc_data_type_exec(struct nand_chip *chip,
+				    const struct nand_subop *subop,
+				    u32 prog_reg)
+{
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	struct anfc_op nfc_op = {};
+	int ret;
+
+	ret = anfc_parse_instructions(chip, subop, &nfc_op);
+	if (ret)
+		return ret;
+
+	nfc_op.prog_reg = prog_reg;
+	anfc_trigger_op(nfc, &nfc_op);
+
+	if (nfc_op.rdy_timeout_ms) {
+		ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
+		if (ret)
+			return ret;
+	}
+
+	return anfc_rw_pio_op(nfc, &nfc_op);
+}
+
+static int anfc_param_read_type_exec(struct nand_chip *chip,
+				     const struct nand_subop *subop)
+{
+	return anfc_misc_data_type_exec(chip, subop, PROG_RDPARAM);
+}
+
+static int anfc_data_read_type_exec(struct nand_chip *chip,
+				    const struct nand_subop *subop)
+{
+	return anfc_misc_data_type_exec(chip, subop, PROG_PGRD);
+}
+
+static int anfc_param_write_type_exec(struct nand_chip *chip,
+				      const struct nand_subop *subop)
+{
+	return anfc_misc_data_type_exec(chip, subop, PROG_SET_FEATURE);
+}
+
+static int anfc_data_write_type_exec(struct nand_chip *chip,
+				     const struct nand_subop *subop)
+{
+	return anfc_misc_data_type_exec(chip, subop, PROG_PGPROG);
+}
+
+static int anfc_misc_zerolen_type_exec(struct nand_chip *chip,
+				       const struct nand_subop *subop,
+				       u32 prog_reg)
+{
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	struct anfc_op nfc_op = {};
+	int ret;
+
+	ret = anfc_parse_instructions(chip, subop, &nfc_op);
+	if (ret)
+		return ret;
+
+	nfc_op.prog_reg = prog_reg;
+	anfc_trigger_op(nfc, &nfc_op);
+
+	ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
+	if (ret)
+		return ret;
+
+	if (nfc_op.rdy_timeout_ms)
+		ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
+
+	return ret;
+}
+
+static int anfc_status_type_exec(struct nand_chip *chip,
+				 const struct nand_subop *subop)
+{
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	u32 tmp;
+	int ret;
+
+	/* See anfc_check_op() for details about this constraint */
+	if (subop->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS)
+		return -ENOTSUPP;
+
+	ret = anfc_misc_zerolen_type_exec(chip, subop, PROG_STATUS);
+	if (ret)
+		return ret;
+
+	tmp = readl_relaxed(nfc->base + FLASH_STS_REG);
+	memcpy(subop->instrs[1].ctx.data.buf.in, &tmp, 1);
+
+	return 0;
+}
+
+static int anfc_reset_type_exec(struct nand_chip *chip,
+				const struct nand_subop *subop)
+{
+	return anfc_misc_zerolen_type_exec(chip, subop, PROG_RST);
+}
+
+static int anfc_erase_type_exec(struct nand_chip *chip,
+				const struct nand_subop *subop)
+{
+	return anfc_misc_zerolen_type_exec(chip, subop, PROG_ERASE);
+}
+
+static int anfc_wait_type_exec(struct nand_chip *chip,
+			       const struct nand_subop *subop)
+{
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	struct anfc_op nfc_op = {};
+	int ret;
+
+	ret = anfc_parse_instructions(chip, subop, &nfc_op);
+	if (ret)
+		return ret;
+
+	return anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
+}
+
+static const struct nand_op_parser anfc_op_parser = NAND_OP_PARSER(
+	NAND_OP_PARSER_PATTERN(
+		anfc_param_read_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
+		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
+	NAND_OP_PARSER_PATTERN(
+		anfc_param_write_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
+		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_PARAM_SIZE)),
+	NAND_OP_PARSER_PATTERN(
+		anfc_data_read_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
+		NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, ANFC_MAX_CHUNK_SIZE)),
+	NAND_OP_PARSER_PATTERN(
+		anfc_data_write_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
+		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_CHUNK_SIZE),
+		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
+	NAND_OP_PARSER_PATTERN(
+		anfc_reset_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
+	NAND_OP_PARSER_PATTERN(
+		anfc_erase_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
+	NAND_OP_PARSER_PATTERN(
+		anfc_status_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
+	NAND_OP_PARSER_PATTERN(
+		anfc_wait_type_exec,
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
+	);
+
+static int anfc_select_target(struct nand_chip *chip, int target)
+{
+	struct anand *anand = to_anand(chip);
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	int ret;
+
+	/* Update the controller timings and the potential ECC configuration */
+	writel_relaxed(anand->timings, nfc->base + DATA_INTERFACE_REG);
+
+	/* Update clock frequency */
+	if (nfc->cur_clk != anand->clk) {
+		clk_disable_unprepare(nfc->controller_clk);
+		ret = clk_set_rate(nfc->controller_clk, anand->clk);
+		if (ret) {
+			dev_err(nfc->dev, "Failed to change clock rate\n");
+			return ret;
+		}
+
+		ret = clk_prepare_enable(nfc->controller_clk);
+		if (ret) {
+			dev_err(nfc->dev,
+				"Failed to re-enable the controller clock\n");
+			return ret;
+		}
+
+		nfc->cur_clk = anand->clk;
+	}
+
+	return 0;
+}
+
+static int anfc_check_op(struct nand_chip *chip,
+			 const struct nand_operation *op)
+{
+	const struct nand_op_instr *instr;
+	int op_id;
+
+	/*
+	 * The controller abstracts all the NAND operations and do not support
+	 * data only operations.
+	 *
+	 * TODO: The nand_op_parser framework should be extended to
+	 * support custom checks on DATA instructions.
+	 */
+	for (op_id = 0; op_id < op->ninstrs; op_id++) {
+		instr = &op->instrs[op_id];
+
+		switch (instr->type) {
+		case NAND_OP_ADDR_INSTR:
+			if (instr->ctx.addr.naddrs > ANFC_MAX_ADDR_CYC)
+				return -ENOTSUPP;
+
+			break;
+		case NAND_OP_DATA_IN_INSTR:
+		case NAND_OP_DATA_OUT_INSTR:
+			if (instr->ctx.data.len > ANFC_MAX_CHUNK_SIZE)
+				return -ENOTSUPP;
+
+			if (anfc_pkt_len_config(instr->ctx.data.len, 0, 0))
+				return -ENOTSUPP;
+
+			break;
+		default:
+			break;
+		}
+	}
+
+	/*
+	 * The controller does not allow to proceed with a CMD+DATA_IN cycle
+	 * manually on the bus by reading data from the data register. Instead,
+	 * the controller abstract a status read operation with its own status
+	 * register after ordering a read status operation. Hence, we cannot
+	 * support any CMD+DATA_IN operation other than a READ STATUS.
+	 *
+	 * TODO: The nand_op_parser() framework should be extended to describe
+	 * fixed patterns instead of open-coding this check here.
+	 */
+	if (op->ninstrs == 2 &&
+	    op->instrs[0].type == NAND_OP_CMD_INSTR &&
+	    op->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS &&
+	    op->instrs[1].type == NAND_OP_DATA_IN_INSTR)
+		return -ENOTSUPP;
+
+	return nand_op_parser_exec_op(chip, &anfc_op_parser, op, true);
+}
+
+static int anfc_exec_op(struct nand_chip *chip,
+			const struct nand_operation *op,
+			bool check_only)
+{
+	int ret;
+
+	if (check_only)
+		return anfc_check_op(chip, op);
+
+	ret = anfc_select_target(chip, op->cs);
+	if (ret)
+		return ret;
+
+	return nand_op_parser_exec_op(chip, &anfc_op_parser, op, check_only);
+}
+
+static int anfc_setup_data_interface(struct nand_chip *chip, int target,
+				     const struct nand_data_interface *conf)
+{
+	struct anand *anand = to_anand(chip);
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	struct device_node *np = nfc->dev->of_node;
+
+	if (target < 0)
+		return 0;
+
+	anand->timings = DIFACE_SDR | DIFACE_SDR_MODE(conf->timings.mode);
+	anand->clk = ANFC_XLNX_SDR_DFLT_CORE_CLK;
+
+	/*
+	 * Due to a hardware bug in the ZynqMP SoC, SDR timing modes 0-1 work
+	 * with f > 90MHz (default clock is 100MHz) but signals are unstable
+	 * with higher modes. Hence we decrease a little bit the clock rate to
+	 * 80MHz when using modes 2-5 with this SoC.
+	 */
+	if (of_device_is_compatible(np, "xlnx,zynqmp-nand-controller") &&
+	    conf->timings.mode >= 2)
+		anand->clk = ANFC_XLNX_SDR_HS_CORE_CLK;
+
+	return 0;
+}
+
+static int anfc_calc_hw_ecc_bytes(int step_size, int strength)
+{
+	unsigned int bch_gf_mag, ecc_bits;
+
+	switch (step_size) {
+	case SZ_512:
+		bch_gf_mag = 13;
+		break;
+	case SZ_1K:
+		bch_gf_mag = 14;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ecc_bits = bch_gf_mag * strength;
+
+	return DIV_ROUND_UP(ecc_bits, 8);
+}
+
+static const int anfc_hw_ecc_512_strengths[] = {4, 8, 12};
+
+static const int anfc_hw_ecc_1024_strengths[] = {24};
+
+static const struct nand_ecc_step_info anfc_hw_ecc_step_infos[] = {
+	{
+		.stepsize = SZ_512,
+		.strengths = anfc_hw_ecc_512_strengths,
+		.nstrengths = ARRAY_SIZE(anfc_hw_ecc_512_strengths),
+	},
+	{
+		.stepsize = SZ_1K,
+		.strengths = anfc_hw_ecc_1024_strengths,
+		.nstrengths = ARRAY_SIZE(anfc_hw_ecc_1024_strengths),
+	},
+};
+
+static const struct nand_ecc_caps anfc_hw_ecc_caps = {
+	.stepinfos = anfc_hw_ecc_step_infos,
+	.nstepinfos = ARRAY_SIZE(anfc_hw_ecc_step_infos),
+	.calc_ecc_bytes = anfc_calc_hw_ecc_bytes,
+};
+
+static int anfc_init_hw_ecc_controller(struct arasan_nfc *nfc,
+				       struct nand_chip *chip)
+{
+	struct anand *anand = to_anand(chip);
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	struct nand_ecc_ctrl *ecc = &chip->ecc;
+	unsigned int bch_prim_poly = 0, bch_gf_mag = 0, ecc_offset;
+	int ret;
+
+	switch (mtd->writesize) {
+	case SZ_512:
+	case SZ_2K:
+	case SZ_4K:
+	case SZ_8K:
+	case SZ_16K:
+		break;
+	default:
+		dev_err(nfc->dev, "Unsupported page size %d\n", mtd->writesize);
+		return -EINVAL;
+	}
+
+	ret = nand_ecc_choose_conf(chip, &anfc_hw_ecc_caps, mtd->oobsize);
+	if (ret)
+		return ret;
+
+	switch (ecc->strength) {
+	case 12:
+		anand->strength = 0x1;
+		break;
+	case 8:
+		anand->strength = 0x2;
+		break;
+	case 4:
+		anand->strength = 0x3;
+		break;
+	case 24:
+		anand->strength = 0x4;
+		break;
+	default:
+		dev_err(nfc->dev, "Unsupported strength %d\n", ecc->strength);
+		return -EINVAL;
+	}
+
+	switch (ecc->size) {
+	case SZ_512:
+		bch_gf_mag = 13;
+		bch_prim_poly = 0x201b;
+		break;
+	case SZ_1K:
+		bch_gf_mag = 14;
+		bch_prim_poly = 0x4443;
+		break;
+	default:
+		dev_err(nfc->dev, "Unsupported step size %d\n", ecc->strength);
+		return -EINVAL;
+	}
+
+	mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
+
+	ecc->steps = mtd->writesize / ecc->size;
+	ecc->algo = NAND_ECC_BCH;
+	anand->ecc_bits = bch_gf_mag * ecc->strength;
+	ecc->bytes = DIV_ROUND_UP(anand->ecc_bits, 8);
+	anand->ecc_total = DIV_ROUND_UP(anand->ecc_bits * ecc->steps, 8);
+	ecc_offset = mtd->writesize + mtd->oobsize - anand->ecc_total;
+	anand->ecc_conf = ECC_CONF_COL(ecc_offset) |
+			  ECC_CONF_LEN(anand->ecc_total) |
+			  ECC_CONF_BCH_EN;
+
+	anand->errloc = devm_kmalloc_array(nfc->dev, ecc->strength,
+					   sizeof(*anand->errloc), GFP_KERNEL);
+	if (!anand->errloc)
+		return -ENOMEM;
+
+	anand->hw_ecc = devm_kmalloc(nfc->dev, ecc->bytes, GFP_KERNEL);
+	if (!anand->hw_ecc)
+		return -ENOMEM;
+
+	/* Enforce bit swapping to fit the hardware */
+	anand->bch = bch_init(bch_gf_mag, ecc->strength, bch_prim_poly, true);
+	if (!anand->bch)
+		return -EINVAL;
+
+	ecc->read_page = anfc_read_page_hw_ecc;
+	ecc->write_page = anfc_write_page_hw_ecc;
+
+	return 0;
+}
+
+static int anfc_attach_chip(struct nand_chip *chip)
+{
+	struct anand *anand = to_anand(chip);
+	struct arasan_nfc *nfc = to_anfc(chip->controller);
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int ret = 0;
+
+	if (mtd->writesize <= SZ_512)
+		anand->caddr_cycles = 1;
+	else
+		anand->caddr_cycles = 2;
+
+	if (chip->options & NAND_ROW_ADDR_3)
+		anand->raddr_cycles = 3;
+	else
+		anand->raddr_cycles = 2;
+
+	switch (mtd->writesize) {
+	case 512:
+		anand->page_sz = 0;
+		break;
+	case 1024:
+		anand->page_sz = 5;
+		break;
+	case 2048:
+		anand->page_sz = 1;
+		break;
+	case 4096:
+		anand->page_sz = 2;
+		break;
+	case 8192:
+		anand->page_sz = 3;
+		break;
+	case 16384:
+		anand->page_sz = 4;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* These hooks are valid for all ECC providers */
+	chip->ecc.read_page_raw = nand_monolithic_read_page_raw;
+	chip->ecc.write_page_raw = nand_monolithic_write_page_raw;
+
+	switch (chip->ecc.mode) {
+	case NAND_ECC_NONE:
+	case NAND_ECC_SOFT:
+	case NAND_ECC_ON_DIE:
+		break;
+	case NAND_ECC_HW:
+		ret = anfc_init_hw_ecc_controller(nfc, chip);
+		break;
+	default:
+		dev_err(nfc->dev, "Unsupported ECC mode: %d\n",
+			chip->ecc.mode);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static void anfc_detach_chip(struct nand_chip *chip)
+{
+	struct anand *anand = to_anand(chip);
+
+	if (anand->bch)
+		bch_free(anand->bch);
+}
+
+static const struct nand_controller_ops anfc_ops = {
+	.exec_op = anfc_exec_op,
+	.setup_data_interface = anfc_setup_data_interface,
+	.attach_chip = anfc_attach_chip,
+	.detach_chip = anfc_detach_chip,
+};
+
+static int anfc_chip_init(struct arasan_nfc *nfc, struct device_node *np)
+{
+	struct anand *anand;
+	struct nand_chip *chip;
+	struct mtd_info *mtd;
+	int cs, rb, ret;
+
+	anand = devm_kzalloc(nfc->dev, sizeof(*anand), GFP_KERNEL);
+	if (!anand)
+		return -ENOMEM;
+
+	/* We do not support multiple CS per chip yet */
+	if (of_property_count_elems_of_size(np, "reg", sizeof(u32)) != 1) {
+		dev_err(nfc->dev, "Invalid reg property\n");
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32(np, "reg", &cs);
+	if (ret)
+		return ret;
+
+	ret = of_property_read_u32(np, "nand-rb", &rb);
+	if (ret)
+		return ret;
+
+	if (cs >= ANFC_MAX_CS || rb >= ANFC_MAX_CS) {
+		dev_err(nfc->dev, "Wrong CS %d or RB %d\n", cs, rb);
+		return -EINVAL;
+	}
+
+	if (test_and_set_bit(cs, &nfc->assigned_cs)) {
+		dev_err(nfc->dev, "Already assigned CS %d\n", cs);
+		return -EINVAL;
+	}
+
+	anand->cs = cs;
+	anand->rb = rb;
+
+	chip = &anand->chip;
+	mtd = nand_to_mtd(chip);
+	mtd->dev.parent = nfc->dev;
+	chip->controller = &nfc->controller;
+	chip->options = NAND_BUSWIDTH_AUTO | NAND_NO_SUBPAGE_WRITE |
+			NAND_USES_DMA;
+
+	nand_set_flash_node(chip, np);
+	if (!mtd->name) {
+		dev_err(nfc->dev, "NAND label property is mandatory\n");
+		return -EINVAL;
+	}
+
+	ret = nand_scan(chip, 1);
+	if (ret) {
+		dev_err(nfc->dev, "Scan operation failed\n");
+		return ret;
+	}
+
+	ret = mtd_device_register(mtd, NULL, 0);
+	if (ret) {
+		nand_cleanup(chip);
+		return ret;
+	}
+
+	list_add_tail(&anand->node, &nfc->chips);
+
+	return 0;
+}
+
+static void anfc_chips_cleanup(struct arasan_nfc *nfc)
+{
+	struct anand *anand, *tmp;
+	struct nand_chip *chip;
+	int ret;
+
+	list_for_each_entry_safe(anand, tmp, &nfc->chips, node) {
+		chip = &anand->chip;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+		list_del(&anand->node);
+	}
+}
+
+static int anfc_chips_init(struct arasan_nfc *nfc)
+{
+	struct device_node *np = nfc->dev->of_node, *nand_np;
+	int nchips = of_get_child_count(np);
+	int ret;
+
+	if (!nchips || nchips > ANFC_MAX_CS) {
+		dev_err(nfc->dev, "Incorrect number of NAND chips (%d)\n",
+			nchips);
+		return -EINVAL;
+	}
+
+	for_each_child_of_node(np, nand_np) {
+		ret = anfc_chip_init(nfc, nand_np);
+		if (ret) {
+			of_node_put(nand_np);
+			anfc_chips_cleanup(nfc);
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static void anfc_reset(struct arasan_nfc *nfc)
+{
+	/* Disable interrupt signals */
+	writel_relaxed(0, nfc->base + INTR_SIG_EN_REG);
+
+	/* Enable interrupt status */
+	writel_relaxed(EVENT_MASK, nfc->base + INTR_STS_EN_REG);
+}
+
+static int anfc_probe(struct platform_device *pdev)
+{
+	struct arasan_nfc *nfc;
+	int ret;
+
+	nfc = devm_kzalloc(&pdev->dev, sizeof(*nfc), GFP_KERNEL);
+	if (!nfc)
+		return -ENOMEM;
+
+	nfc->dev = &pdev->dev;
+	nand_controller_init(&nfc->controller);
+	nfc->controller.ops = &anfc_ops;
+	INIT_LIST_HEAD(&nfc->chips);
+
+	nfc->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(nfc->base))
+		return PTR_ERR(nfc->base);
+
+	anfc_reset(nfc);
+
+	nfc->controller_clk = devm_clk_get(&pdev->dev, "controller");
+	if (IS_ERR(nfc->controller_clk))
+		return PTR_ERR(nfc->controller_clk);
+
+	nfc->bus_clk = devm_clk_get(&pdev->dev, "bus");
+	if (IS_ERR(nfc->bus_clk))
+		return PTR_ERR(nfc->bus_clk);
+
+	ret = clk_prepare_enable(nfc->controller_clk);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(nfc->bus_clk);
+	if (ret)
+		goto disable_controller_clk;
+
+	ret = anfc_chips_init(nfc);
+	if (ret)
+		goto disable_bus_clk;
+
+	platform_set_drvdata(pdev, nfc);
+
+	return 0;
+
+disable_bus_clk:
+	clk_disable_unprepare(nfc->bus_clk);
+
+disable_controller_clk:
+	clk_disable_unprepare(nfc->controller_clk);
+
+	return ret;
+}
+
+static int anfc_remove(struct platform_device *pdev)
+{
+	struct arasan_nfc *nfc = platform_get_drvdata(pdev);
+
+	anfc_chips_cleanup(nfc);
+
+	clk_disable_unprepare(nfc->bus_clk);
+	clk_disable_unprepare(nfc->controller_clk);
+
+	return 0;
+}
+
+static const struct of_device_id anfc_ids[] = {
+	{
+		.compatible = "xlnx,zynqmp-nand-controller",
+	},
+	{
+		.compatible = "arasan,nfc-v3p10",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, anfc_ids);
+
+static struct platform_driver anfc_driver = {
+	.driver = {
+		.name = "arasan-nand-controller",
+		.of_match_table = anfc_ids,
+	},
+	.probe = anfc_probe,
+	.remove = anfc_remove,
+};
+module_platform_driver(anfc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Punnaiah Choudary Kalluri <punnaia@xilinx.com>");
+MODULE_AUTHOR("Naga Sureshkumar Relli <nagasure@xilinx.com>");
+MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
+MODULE_DESCRIPTION("Arasan NAND Flash Controller Driver");
diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c
index 3ba17a98df4d..46a3724a788e 100644
--- a/drivers/mtd/nand/raw/atmel/nand-controller.c
+++ b/drivers/mtd/nand/raw/atmel/nand-controller.c
@@ -1494,7 +1494,7 @@ static void atmel_nand_init(struct atmel_nand_controller *nc,
 	 * suitable for DMA.
 	 */
 	if (nc->dmac)
-		chip->options |= NAND_USE_BOUNCE_BUFFER;
+		chip->options |= NAND_USES_DMA;
 
 	/* Default to HW ECC if pmecc is available. */
 	if (nc->pmecc)
diff --git a/drivers/mtd/nand/raw/au1550nd.c b/drivers/mtd/nand/raw/au1550nd.c
index 75eb3e97fae3..d865200ccd08 100644
--- a/drivers/mtd/nand/raw/au1550nd.c
+++ b/drivers/mtd/nand/raw/au1550nd.c
@@ -16,63 +16,16 @@
 
 
 struct au1550nd_ctx {
+	struct nand_controller controller;
 	struct nand_chip chip;
 
 	int cs;
 	void __iomem *base;
-	void (*write_byte)(struct nand_chip *, u_char);
 };
 
-/**
- * au_read_byte -  read one byte from the chip
- * @this:	NAND chip object
- *
- * read function for 8bit buswidth
- */
-static u_char au_read_byte(struct nand_chip *this)
-{
-	u_char ret = readb(this->legacy.IO_ADDR_R);
-	wmb(); /* drain writebuffer */
-	return ret;
-}
-
-/**
- * au_write_byte -  write one byte to the chip
- * @this:	NAND chip object
- * @byte:	pointer to data byte to write
- *
- * write function for 8it buswidth
- */
-static void au_write_byte(struct nand_chip *this, u_char byte)
-{
-	writeb(byte, this->legacy.IO_ADDR_W);
-	wmb(); /* drain writebuffer */
-}
-
-/**
- * au_read_byte16 -  read one byte endianness aware from the chip
- * @this:	NAND chip object
- *
- * read function for 16bit buswidth with endianness conversion
- */
-static u_char au_read_byte16(struct nand_chip *this)
-{
-	u_char ret = (u_char) cpu_to_le16(readw(this->legacy.IO_ADDR_R));
-	wmb(); /* drain writebuffer */
-	return ret;
-}
-
-/**
- * au_write_byte16 -  write one byte endianness aware to the chip
- * @this:	NAND chip object
- * @byte:	pointer to data byte to write
- *
- * write function for 16bit buswidth with endianness conversion
- */
-static void au_write_byte16(struct nand_chip *this, u_char byte)
+static struct au1550nd_ctx *chip_to_au_ctx(struct nand_chip *this)
 {
-	writew(le16_to_cpu((u16) byte), this->legacy.IO_ADDR_W);
-	wmb(); /* drain writebuffer */
+	return container_of(this, struct au1550nd_ctx, chip);
 }
 
 /**
@@ -83,12 +36,15 @@ static void au_write_byte16(struct nand_chip *this, u_char byte)
  *
  * write function for 8bit buswidth
  */
-static void au_write_buf(struct nand_chip *this, const u_char *buf, int len)
+static void au_write_buf(struct nand_chip *this, const void *buf,
+			 unsigned int len)
 {
+	struct au1550nd_ctx *ctx = chip_to_au_ctx(this);
+	const u8 *p = buf;
 	int i;
 
 	for (i = 0; i < len; i++) {
-		writeb(buf[i], this->legacy.IO_ADDR_W);
+		writeb(p[i], ctx->base + MEM_STNAND_DATA);
 		wmb(); /* drain writebuffer */
 	}
 }
@@ -101,12 +57,15 @@ static void au_write_buf(struct nand_chip *this, const u_char *buf, int len)
  *
  * read function for 8bit buswidth
  */
-static void au_read_buf(struct nand_chip *this, u_char *buf, int len)
+static void au_read_buf(struct nand_chip *this, void *buf,
+			unsigned int len)
 {
+	struct au1550nd_ctx *ctx = chip_to_au_ctx(this);
+	u8 *p = buf;
 	int i;
 
 	for (i = 0; i < len; i++) {
-		buf[i] = readb(this->legacy.IO_ADDR_R);
+		p[i] = readb(ctx->base + MEM_STNAND_DATA);
 		wmb(); /* drain writebuffer */
 	}
 }
@@ -119,17 +78,18 @@ static void au_read_buf(struct nand_chip *this, u_char *buf, int len)
  *
  * write function for 16bit buswidth
  */
-static void au_write_buf16(struct nand_chip *this, const u_char *buf, int len)
+static void au_write_buf16(struct nand_chip *this, const void *buf,
+			   unsigned int len)
 {
-	int i;
-	u16 *p = (u16 *) buf;
-	len >>= 1;
+	struct au1550nd_ctx *ctx = chip_to_au_ctx(this);
+	const u16 *p = buf;
+	unsigned int i;
 
+	len >>= 1;
 	for (i = 0; i < len; i++) {
-		writew(p[i], this->legacy.IO_ADDR_W);
+		writew(p[i], ctx->base + MEM_STNAND_DATA);
 		wmb(); /* drain writebuffer */
 	}
-
 }
 
 /**
@@ -140,239 +100,146 @@ static void au_write_buf16(struct nand_chip *this, const u_char *buf, int len)
  *
  * read function for 16bit buswidth
  */
-static void au_read_buf16(struct nand_chip *this, u_char *buf, int len)
+static void au_read_buf16(struct nand_chip *this, void *buf, unsigned int len)
 {
-	int i;
-	u16 *p = (u16 *) buf;
-	len >>= 1;
+	struct au1550nd_ctx *ctx = chip_to_au_ctx(this);
+	unsigned int i;
+	u16 *p = buf;
 
+	len >>= 1;
 	for (i = 0; i < len; i++) {
-		p[i] = readw(this->legacy.IO_ADDR_R);
+		p[i] = readw(ctx->base + MEM_STNAND_DATA);
 		wmb(); /* drain writebuffer */
 	}
 }
 
-/* Select the chip by setting nCE to low */
-#define NAND_CTL_SETNCE		1
-/* Deselect the chip by setting nCE to high */
-#define NAND_CTL_CLRNCE		2
-/* Select the command latch by setting CLE to high */
-#define NAND_CTL_SETCLE		3
-/* Deselect the command latch by setting CLE to low */
-#define NAND_CTL_CLRCLE		4
-/* Select the address latch by setting ALE to high */
-#define NAND_CTL_SETALE		5
-/* Deselect the address latch by setting ALE to low */
-#define NAND_CTL_CLRALE		6
-
-static void au1550_hwcontrol(struct mtd_info *mtd, int cmd)
+static int find_nand_cs(unsigned long nand_base)
 {
-	struct nand_chip *this = mtd_to_nand(mtd);
-	struct au1550nd_ctx *ctx = container_of(this, struct au1550nd_ctx,
-						chip);
+	void __iomem *base =
+			(void __iomem *)KSEG1ADDR(AU1000_STATIC_MEM_PHYS_ADDR);
+	unsigned long addr, staddr, start, mask, end;
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		addr = 0x1000 + (i * 0x10);			/* CSx */
+		staddr = __raw_readl(base + addr + 0x08);	/* STADDRx */
+		/* figure out the decoded range of this CS */
+		start = (staddr << 4) & 0xfffc0000;
+		mask = (staddr << 18) & 0xfffc0000;
+		end = (start | (start - 1)) & ~(start ^ mask);
+		if ((nand_base >= start) && (nand_base < end))
+			return i;
+	}
 
-	switch (cmd) {
+	return -ENODEV;
+}
 
-	case NAND_CTL_SETCLE:
-		this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_CMD;
-		break;
+static int au1550nd_waitrdy(struct nand_chip *this, unsigned int timeout_ms)
+{
+	unsigned long timeout_jiffies = jiffies;
+
+	timeout_jiffies += msecs_to_jiffies(timeout_ms) + 1;
+	do {
+		if (alchemy_rdsmem(AU1000_MEM_STSTAT) & 0x1)
+			return 0;
 
-	case NAND_CTL_CLRCLE:
-		this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_DATA;
+		usleep_range(10, 100);
+	} while (time_before(jiffies, timeout_jiffies));
+
+	return -ETIMEDOUT;
+}
+
+static int au1550nd_exec_instr(struct nand_chip *this,
+			       const struct nand_op_instr *instr)
+{
+	struct au1550nd_ctx *ctx = chip_to_au_ctx(this);
+	unsigned int i;
+	int ret = 0;
+
+	switch (instr->type) {
+	case NAND_OP_CMD_INSTR:
+		writeb(instr->ctx.cmd.opcode,
+		       ctx->base + MEM_STNAND_CMD);
+		/* Drain the writebuffer */
+		wmb();
 		break;
 
-	case NAND_CTL_SETALE:
-		this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_ADDR;
+	case NAND_OP_ADDR_INSTR:
+		for (i = 0; i < instr->ctx.addr.naddrs; i++) {
+			writeb(instr->ctx.addr.addrs[i],
+			       ctx->base + MEM_STNAND_ADDR);
+			/* Drain the writebuffer */
+			wmb();
+		}
 		break;
 
-	case NAND_CTL_CLRALE:
-		this->legacy.IO_ADDR_W = ctx->base + MEM_STNAND_DATA;
-		/* FIXME: Nobody knows why this is necessary,
-		 * but it works only that way */
-		udelay(1);
+	case NAND_OP_DATA_IN_INSTR:
+		if ((this->options & NAND_BUSWIDTH_16) &&
+		    !instr->ctx.data.force_8bit)
+			au_read_buf16(this, instr->ctx.data.buf.in,
+				      instr->ctx.data.len);
+		else
+			au_read_buf(this, instr->ctx.data.buf.in,
+				    instr->ctx.data.len);
 		break;
 
-	case NAND_CTL_SETNCE:
-		/* assert (force assert) chip enable */
-		alchemy_wrsmem((1 << (4 + ctx->cs)), AU1000_MEM_STNDCTL);
+	case NAND_OP_DATA_OUT_INSTR:
+		if ((this->options & NAND_BUSWIDTH_16) &&
+		    !instr->ctx.data.force_8bit)
+			au_write_buf16(this, instr->ctx.data.buf.out,
+				       instr->ctx.data.len);
+		else
+			au_write_buf(this, instr->ctx.data.buf.out,
+				     instr->ctx.data.len);
 		break;
 
-	case NAND_CTL_CLRNCE:
-		/* deassert chip enable */
-		alchemy_wrsmem(0, AU1000_MEM_STNDCTL);
+	case NAND_OP_WAITRDY_INSTR:
+		ret = au1550nd_waitrdy(this, instr->ctx.waitrdy.timeout_ms);
 		break;
+	default:
+		return -EINVAL;
 	}
 
-	this->legacy.IO_ADDR_R = this->legacy.IO_ADDR_W;
-
-	wmb(); /* Drain the writebuffer */
-}
-
-int au1550_device_ready(struct nand_chip *this)
-{
-	return (alchemy_rdsmem(AU1000_MEM_STSTAT) & 0x1) ? 1 : 0;
-}
+	if (instr->delay_ns)
+		ndelay(instr->delay_ns);
 
-/**
- * au1550_select_chip - control -CE line
- *	Forbid driving -CE manually permitting the NAND controller to do this.
- *	Keeping -CE asserted during the whole sector reads interferes with the
- *	NOR flash and PCMCIA drivers as it causes contention on the static bus.
- *	We only have to hold -CE low for the NAND read commands since the flash
- *	chip needs it to be asserted during chip not ready time but the NAND
- *	controller keeps it released.
- *
- * @this:	NAND chip object
- * @chip:	chipnumber to select, -1 for deselect
- */
-static void au1550_select_chip(struct nand_chip *this, int chip)
-{
+	return ret;
 }
 
-/**
- * au1550_command - Send command to NAND device
- * @this:	NAND chip object
- * @command:	the command to be sent
- * @column:	the column address for this command, -1 if none
- * @page_addr:	the page address for this command, -1 if none
- */
-static void au1550_command(struct nand_chip *this, unsigned command,
-			   int column, int page_addr)
+static int au1550nd_exec_op(struct nand_chip *this,
+			    const struct nand_operation *op,
+			    bool check_only)
 {
-	struct mtd_info *mtd = nand_to_mtd(this);
-	struct au1550nd_ctx *ctx = container_of(this, struct au1550nd_ctx,
-						chip);
-	int ce_override = 0, i;
-	unsigned long flags = 0;
-
-	/* Begin command latch cycle */
-	au1550_hwcontrol(mtd, NAND_CTL_SETCLE);
-	/*
-	 * Write out the command to the device.
-	 */
-	if (command == NAND_CMD_SEQIN) {
-		int readcmd;
-
-		if (column >= mtd->writesize) {
-			/* OOB area */
-			column -= mtd->writesize;
-			readcmd = NAND_CMD_READOOB;
-		} else if (column < 256) {
-			/* First 256 bytes --> READ0 */
-			readcmd = NAND_CMD_READ0;
-		} else {
-			column -= 256;
-			readcmd = NAND_CMD_READ1;
-		}
-		ctx->write_byte(this, readcmd);
-	}
-	ctx->write_byte(this, command);
+	struct au1550nd_ctx *ctx = chip_to_au_ctx(this);
+	unsigned int i;
+	int ret;
 
-	/* Set ALE and clear CLE to start address cycle */
-	au1550_hwcontrol(mtd, NAND_CTL_CLRCLE);
+	if (check_only)
+		return 0;
 
-	if (column != -1 || page_addr != -1) {
-		au1550_hwcontrol(mtd, NAND_CTL_SETALE);
+	/* assert (force assert) chip enable */
+	alchemy_wrsmem((1 << (4 + ctx->cs)), AU1000_MEM_STNDCTL);
+	/* Drain the writebuffer */
+	wmb();
 
-		/* Serially input address */
-		if (column != -1) {
-			/* Adjust columns for 16 bit buswidth */
-			if (this->options & NAND_BUSWIDTH_16 &&
-					!nand_opcode_8bits(command))
-				column >>= 1;
-			ctx->write_byte(this, column);
-		}
-		if (page_addr != -1) {
-			ctx->write_byte(this, (u8)(page_addr & 0xff));
-
-			if (command == NAND_CMD_READ0 ||
-			    command == NAND_CMD_READ1 ||
-			    command == NAND_CMD_READOOB) {
-				/*
-				 * NAND controller will release -CE after
-				 * the last address byte is written, so we'll
-				 * have to forcibly assert it. No interrupts
-				 * are allowed while we do this as we don't
-				 * want the NOR flash or PCMCIA drivers to
-				 * steal our precious bytes of data...
-				 */
-				ce_override = 1;
-				local_irq_save(flags);
-				au1550_hwcontrol(mtd, NAND_CTL_SETNCE);
-			}
-
-			ctx->write_byte(this, (u8)(page_addr >> 8));
-
-			if (this->options & NAND_ROW_ADDR_3)
-				ctx->write_byte(this,
-						((page_addr >> 16) & 0x0f));
-		}
-		/* Latch in address */
-		au1550_hwcontrol(mtd, NAND_CTL_CLRALE);
-	}
-
-	/*
-	 * Program and erase have their own busy handlers.
-	 * Status and sequential in need no delay.
-	 */
-	switch (command) {
-
-	case NAND_CMD_PAGEPROG:
-	case NAND_CMD_ERASE1:
-	case NAND_CMD_ERASE2:
-	case NAND_CMD_SEQIN:
-	case NAND_CMD_STATUS:
-		return;
-
-	case NAND_CMD_RESET:
-		break;
-
-	case NAND_CMD_READ0:
-	case NAND_CMD_READ1:
-	case NAND_CMD_READOOB:
-		/* Check if we're really driving -CE low (just in case) */
-		if (unlikely(!ce_override))
+	for (i = 0; i < op->ninstrs; i++) {
+		ret = au1550nd_exec_instr(this, &op->instrs[i]);
+		if (ret)
 			break;
-
-		/* Apply a short delay always to ensure that we do wait tWB. */
-		ndelay(100);
-		/* Wait for a chip to become ready... */
-		for (i = this->legacy.chip_delay;
-		     !this->legacy.dev_ready(this) && i > 0; --i)
-			udelay(1);
-
-		/* Release -CE and re-enable interrupts. */
-		au1550_hwcontrol(mtd, NAND_CTL_CLRNCE);
-		local_irq_restore(flags);
-		return;
 	}
-	/* Apply this short delay always to ensure that we do wait tWB. */
-	ndelay(100);
-
-	while(!this->legacy.dev_ready(this));
-}
 
-static int find_nand_cs(unsigned long nand_base)
-{
-	void __iomem *base =
-			(void __iomem *)KSEG1ADDR(AU1000_STATIC_MEM_PHYS_ADDR);
-	unsigned long addr, staddr, start, mask, end;
-	int i;
+	/* deassert chip enable */
+	alchemy_wrsmem(0, AU1000_MEM_STNDCTL);
+	/* Drain the writebuffer */
+	wmb();
 
-	for (i = 0; i < 4; i++) {
-		addr = 0x1000 + (i * 0x10);			/* CSx */
-		staddr = __raw_readl(base + addr + 0x08);	/* STADDRx */
-		/* figure out the decoded range of this CS */
-		start = (staddr << 4) & 0xfffc0000;
-		mask = (staddr << 18) & 0xfffc0000;
-		end = (start | (start - 1)) & ~(start ^ mask);
-		if ((nand_base >= start) && (nand_base < end))
-			return i;
-	}
-
-	return -ENODEV;
+	return ret;
 }
 
+static const struct nand_controller_ops au1550nd_ops = {
+	.exec_op = au1550nd_exec_op,
+};
+
 static int au1550nd_probe(struct platform_device *pdev)
 {
 	struct au1550nd_platdata *pd;
@@ -424,23 +291,15 @@ static int au1550nd_probe(struct platform_device *pdev)
 	}
 	ctx->cs = cs;
 
-	this->legacy.dev_ready = au1550_device_ready;
-	this->legacy.select_chip = au1550_select_chip;
-	this->legacy.cmdfunc = au1550_command;
-
-	/* 30 us command delay time */
-	this->legacy.chip_delay = 30;
+	nand_controller_init(&ctx->controller);
+	ctx->controller.ops = &au1550nd_ops;
+	this->controller = &ctx->controller;
 	this->ecc.mode = NAND_ECC_SOFT;
 	this->ecc.algo = NAND_ECC_HAMMING;
 
 	if (pd->devwidth)
 		this->options |= NAND_BUSWIDTH_16;
 
-	this->legacy.read_byte = (pd->devwidth) ? au_read_byte16 : au_read_byte;
-	ctx->write_byte = (pd->devwidth) ? au_write_byte16 : au_write_byte;
-	this->legacy.write_buf = (pd->devwidth) ? au_write_buf16 : au_write_buf;
-	this->legacy.read_buf = (pd->devwidth) ? au_read_buf16 : au_read_buf;
-
 	ret = nand_scan(this, 1);
 	if (ret) {
 		dev_err(&pdev->dev, "NAND scan failed with %d\n", ret);
@@ -466,8 +325,12 @@ static int au1550nd_remove(struct platform_device *pdev)
 {
 	struct au1550nd_ctx *ctx = platform_get_drvdata(pdev);
 	struct resource *r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	struct nand_chip *chip = &ctx->chip;
+	int ret;
 
-	nand_release(&ctx->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	iounmap(ctx->base);
 	release_mem_region(r->start, 0x1000);
 	kfree(ctx);
diff --git a/drivers/mtd/nand/raw/bcm47xxnflash/main.c b/drivers/mtd/nand/raw/bcm47xxnflash/main.c
index 8dae97c1dbe7..dcc70d9dc6e5 100644
--- a/drivers/mtd/nand/raw/bcm47xxnflash/main.c
+++ b/drivers/mtd/nand/raw/bcm47xxnflash/main.c
@@ -60,8 +60,12 @@ static int bcm47xxnflash_probe(struct platform_device *pdev)
 static int bcm47xxnflash_remove(struct platform_device *pdev)
 {
 	struct bcm47xxnflash *nflash = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &nflash->nand_chip;
+	int ret;
 
-	nand_release(&nflash->nand_chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
index 8f9ffb46a09f..44068e9eea03 100644
--- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c
+++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
@@ -4,7 +4,6 @@
  */
 
 #include <linux/clk.h>
-#include <linux/version.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/delay.h>
@@ -264,6 +263,7 @@ struct brcmnand_controller {
 	const unsigned int	*block_sizes;
 	unsigned int		max_page_size;
 	const unsigned int	*page_sizes;
+	unsigned int		page_size_shift;
 	unsigned int		max_oob;
 	u32			features;
 
@@ -338,8 +338,38 @@ enum brcmnand_reg {
 	BRCMNAND_FC_BASE,
 };
 
-/* BRCMNAND v4.0 */
-static const u16 brcmnand_regs_v40[] = {
+/* BRCMNAND v2.1-v2.2 */
+static const u16 brcmnand_regs_v21[] = {
+	[BRCMNAND_CMD_START]		=  0x04,
+	[BRCMNAND_CMD_EXT_ADDRESS]	=  0x08,
+	[BRCMNAND_CMD_ADDRESS]		=  0x0c,
+	[BRCMNAND_INTFC_STATUS]		=  0x5c,
+	[BRCMNAND_CS_SELECT]		=  0x14,
+	[BRCMNAND_CS_XOR]		=  0x18,
+	[BRCMNAND_LL_OP]		=     0,
+	[BRCMNAND_CS0_BASE]		=  0x40,
+	[BRCMNAND_CS1_BASE]		=     0,
+	[BRCMNAND_CORR_THRESHOLD]	=     0,
+	[BRCMNAND_CORR_THRESHOLD_EXT]	=     0,
+	[BRCMNAND_UNCORR_COUNT]		=     0,
+	[BRCMNAND_CORR_COUNT]		=     0,
+	[BRCMNAND_CORR_EXT_ADDR]	=  0x60,
+	[BRCMNAND_CORR_ADDR]		=  0x64,
+	[BRCMNAND_UNCORR_EXT_ADDR]	=  0x68,
+	[BRCMNAND_UNCORR_ADDR]		=  0x6c,
+	[BRCMNAND_SEMAPHORE]		=  0x50,
+	[BRCMNAND_ID]			=  0x54,
+	[BRCMNAND_ID_EXT]		=     0,
+	[BRCMNAND_LL_RDATA]		=     0,
+	[BRCMNAND_OOB_READ_BASE]	=  0x20,
+	[BRCMNAND_OOB_READ_10_BASE]	=     0,
+	[BRCMNAND_OOB_WRITE_BASE]	=  0x30,
+	[BRCMNAND_OOB_WRITE_10_BASE]	=     0,
+	[BRCMNAND_FC_BASE]		= 0x200,
+};
+
+/* BRCMNAND v3.3-v4.0 */
+static const u16 brcmnand_regs_v33[] = {
 	[BRCMNAND_CMD_START]		=  0x04,
 	[BRCMNAND_CMD_EXT_ADDRESS]	=  0x08,
 	[BRCMNAND_CMD_ADDRESS]		=  0x0c,
@@ -536,6 +566,9 @@ enum {
 	CFG_BUS_WIDTH			= BIT(CFG_BUS_WIDTH_SHIFT),
 	CFG_DEVICE_SIZE_SHIFT		= 24,
 
+	/* Only for v2.1 */
+	CFG_PAGE_SIZE_SHIFT_v2_1	= 30,
+
 	/* Only for pre-v7.1 (with no CFG_EXT register) */
 	CFG_PAGE_SIZE_SHIFT		= 20,
 	CFG_BLK_SIZE_SHIFT		= 28,
@@ -571,12 +604,16 @@ static int brcmnand_revision_init(struct brcmnand_controller *ctrl)
 {
 	static const unsigned int block_sizes_v6[] = { 8, 16, 128, 256, 512, 1024, 2048, 0 };
 	static const unsigned int block_sizes_v4[] = { 16, 128, 8, 512, 256, 1024, 2048, 0 };
-	static const unsigned int page_sizes[] = { 512, 2048, 4096, 8192, 0 };
+	static const unsigned int block_sizes_v2_2[] = { 16, 128, 8, 512, 256, 0 };
+	static const unsigned int block_sizes_v2_1[] = { 16, 128, 8, 512, 0 };
+	static const unsigned int page_sizes_v3_4[] = { 512, 2048, 4096, 8192, 0 };
+	static const unsigned int page_sizes_v2_2[] = { 512, 2048, 4096, 0 };
+	static const unsigned int page_sizes_v2_1[] = { 512, 2048, 0 };
 
 	ctrl->nand_version = nand_readreg(ctrl, 0) & 0xffff;
 
-	/* Only support v4.0+? */
-	if (ctrl->nand_version < 0x0400) {
+	/* Only support v2.1+ */
+	if (ctrl->nand_version < 0x0201) {
 		dev_err(ctrl->dev, "version %#x not supported\n",
 			ctrl->nand_version);
 		return -ENODEV;
@@ -591,8 +628,10 @@ static int brcmnand_revision_init(struct brcmnand_controller *ctrl)
 		ctrl->reg_offsets = brcmnand_regs_v60;
 	else if (ctrl->nand_version >= 0x0500)
 		ctrl->reg_offsets = brcmnand_regs_v50;
-	else if (ctrl->nand_version >= 0x0400)
-		ctrl->reg_offsets = brcmnand_regs_v40;
+	else if (ctrl->nand_version >= 0x0303)
+		ctrl->reg_offsets = brcmnand_regs_v33;
+	else if (ctrl->nand_version >= 0x0201)
+		ctrl->reg_offsets = brcmnand_regs_v21;
 
 	/* Chip-select stride */
 	if (ctrl->nand_version >= 0x0701)
@@ -606,8 +645,9 @@ static int brcmnand_revision_init(struct brcmnand_controller *ctrl)
 	} else {
 		ctrl->cs_offsets = brcmnand_cs_offsets;
 
-		/* v5.0 and earlier has a different CS0 offset layout */
-		if (ctrl->nand_version <= 0x0500)
+		/* v3.3-5.0 have a different CS0 offset layout */
+		if (ctrl->nand_version >= 0x0303 &&
+		    ctrl->nand_version <= 0x0500)
 			ctrl->cs0_offsets = brcmnand_cs_offsets_cs0;
 	}
 
@@ -617,14 +657,32 @@ static int brcmnand_revision_init(struct brcmnand_controller *ctrl)
 		ctrl->max_page_size = 16 * 1024;
 		ctrl->max_block_size = 2 * 1024 * 1024;
 	} else {
-		ctrl->page_sizes = page_sizes;
+		if (ctrl->nand_version >= 0x0304)
+			ctrl->page_sizes = page_sizes_v3_4;
+		else if (ctrl->nand_version >= 0x0202)
+			ctrl->page_sizes = page_sizes_v2_2;
+		else
+			ctrl->page_sizes = page_sizes_v2_1;
+
+		if (ctrl->nand_version >= 0x0202)
+			ctrl->page_size_shift = CFG_PAGE_SIZE_SHIFT;
+		else
+			ctrl->page_size_shift = CFG_PAGE_SIZE_SHIFT_v2_1;
+
 		if (ctrl->nand_version >= 0x0600)
 			ctrl->block_sizes = block_sizes_v6;
-		else
+		else if (ctrl->nand_version >= 0x0400)
 			ctrl->block_sizes = block_sizes_v4;
+		else if (ctrl->nand_version >= 0x0202)
+			ctrl->block_sizes = block_sizes_v2_2;
+		else
+			ctrl->block_sizes = block_sizes_v2_1;
 
 		if (ctrl->nand_version < 0x0400) {
-			ctrl->max_page_size = 4096;
+			if (ctrl->nand_version < 0x0202)
+				ctrl->max_page_size = 2048;
+			else
+				ctrl->max_page_size = 4096;
 			ctrl->max_block_size = 512 * 1024;
 		}
 	}
@@ -810,6 +868,9 @@ static void brcmnand_wr_corr_thresh(struct brcmnand_host *host, u8 val)
 	enum brcmnand_reg reg = BRCMNAND_CORR_THRESHOLD;
 	int cs = host->cs;
 
+	if (!ctrl->reg_offsets[reg])
+		return;
+
 	if (ctrl->nand_version == 0x0702)
 		bits = 7;
 	else if (ctrl->nand_version >= 0x0600)
@@ -868,8 +929,10 @@ static inline u32 brcmnand_spare_area_mask(struct brcmnand_controller *ctrl)
 		return GENMASK(7, 0);
 	else if (ctrl->nand_version >= 0x0600)
 		return GENMASK(6, 0);
-	else
+	else if (ctrl->nand_version >= 0x0303)
 		return GENMASK(5, 0);
+	else
+		return GENMASK(4, 0);
 }
 
 #define NAND_ACC_CONTROL_ECC_SHIFT	16
@@ -1100,30 +1163,30 @@ static int brcmnand_hamming_ooblayout_free(struct mtd_info *mtd, int section,
 	struct brcmnand_cfg *cfg = &host->hwcfg;
 	int sas = cfg->spare_area_size << cfg->sector_size_1k;
 	int sectors = cfg->page_size / (512 << cfg->sector_size_1k);
+	u32 next;
 
-	if (section >= sectors * 2)
+	if (section > sectors)
 		return -ERANGE;
 
-	oobregion->offset = (section / 2) * sas;
+	next = (section * sas);
+	if (section < sectors)
+		next += 6;
 
-	if (section & 1) {
-		oobregion->offset += 9;
-		oobregion->length = 7;
+	if (section) {
+		oobregion->offset = ((section - 1) * sas) + 9;
 	} else {
-		oobregion->length = 6;
-
-		/* First sector of each page may have BBI */
-		if (!section) {
-			/*
-			 * Small-page NAND use byte 6 for BBI while large-page
-			 * NAND use byte 0.
-			 */
-			if (cfg->page_size > 512)
-				oobregion->offset++;
-			oobregion->length--;
+		if (cfg->page_size > 512) {
+			/* Large page NAND uses first 2 bytes for BBI */
+			oobregion->offset = 2;
+		} else {
+			/* Small page NAND uses last byte before ECC for BBI */
+			oobregion->offset = 0;
+			next--;
 		}
 	}
 
+	oobregion->length = next - oobregion->offset;
+
 	return 0;
 }
 
@@ -2018,28 +2081,31 @@ static int brcmnand_read_by_pio(struct mtd_info *mtd, struct nand_chip *chip,
 static int brcmstb_nand_verify_erased_page(struct mtd_info *mtd,
 		  struct nand_chip *chip, void *buf, u64 addr)
 {
-	int i, sas;
-	void *oob = chip->oob_poi;
+	struct mtd_oob_region ecc;
+	int i;
 	int bitflips = 0;
 	int page = addr >> chip->page_shift;
 	int ret;
+	void *ecc_bytes;
 	void *ecc_chunk;
 
 	if (!buf)
 		buf = nand_get_data_buf(chip);
 
-	sas = mtd->oobsize / chip->ecc.steps;
-
 	/* read without ecc for verification */
 	ret = chip->ecc.read_page_raw(chip, buf, true, page);
 	if (ret)
 		return ret;
 
-	for (i = 0; i < chip->ecc.steps; i++, oob += sas) {
+	for (i = 0; i < chip->ecc.steps; i++) {
 		ecc_chunk = buf + chip->ecc.size * i;
-		ret = nand_check_erased_ecc_chunk(ecc_chunk,
-						  chip->ecc.size,
-						  oob, sas, NULL, 0,
+
+		mtd_ooblayout_ecc(mtd, i, &ecc);
+		ecc_bytes = chip->oob_poi + ecc.offset;
+
+		ret = nand_check_erased_ecc_chunk(ecc_chunk, chip->ecc.size,
+						  ecc_bytes, ecc.length,
+						  NULL, 0,
 						  chip->ecc.strength);
 		if (ret < 0)
 			return ret;
@@ -2377,7 +2443,7 @@ static int brcmnand_set_cfg(struct brcmnand_host *host,
 		(!!(cfg->device_width == 16) << CFG_BUS_WIDTH_SHIFT) |
 		(device_size << CFG_DEVICE_SIZE_SHIFT);
 	if (cfg_offs == cfg_ext_offs) {
-		tmp |= (page_size << CFG_PAGE_SIZE_SHIFT) |
+		tmp |= (page_size << ctrl->page_size_shift) |
 		       (block_size << CFG_BLK_SIZE_SHIFT);
 		nand_writereg(ctrl, cfg_offs, tmp);
 	} else {
@@ -2389,9 +2455,11 @@ static int brcmnand_set_cfg(struct brcmnand_host *host,
 
 	tmp = nand_readreg(ctrl, acc_control_offs);
 	tmp &= ~brcmnand_ecc_level_mask(ctrl);
-	tmp |= cfg->ecc_level << NAND_ACC_CONTROL_ECC_SHIFT;
 	tmp &= ~brcmnand_spare_area_mask(ctrl);
-	tmp |= cfg->spare_area_size;
+	if (ctrl->nand_version >= 0x0302) {
+		tmp |= cfg->ecc_level << NAND_ACC_CONTROL_ECC_SHIFT;
+		tmp |= cfg->spare_area_size;
+	}
 	nand_writereg(ctrl, acc_control_offs, tmp);
 
 	brcmnand_set_sector_size_1k(host, cfg->sector_size_1k);
@@ -2577,7 +2645,7 @@ static int brcmnand_attach_chip(struct nand_chip *chip)
 	 * to/from, and have nand_base pass us a bounce buffer instead, as
 	 * needed.
 	 */
-	chip->options |= NAND_USE_BOUNCE_BUFFER;
+	chip->options |= NAND_USES_DMA;
 
 	if (chip->bbt_options & NAND_BBT_USE_FLASH)
 		chip->bbt_options |= NAND_BBT_NO_OOB;
@@ -2764,6 +2832,8 @@ const struct dev_pm_ops brcmnand_pm_ops = {
 EXPORT_SYMBOL_GPL(brcmnand_pm_ops);
 
 static const struct of_device_id brcmnand_of_match[] = {
+	{ .compatible = "brcm,brcmnand-v2.1" },
+	{ .compatible = "brcm,brcmnand-v2.2" },
 	{ .compatible = "brcm,brcmnand-v4.0" },
 	{ .compatible = "brcm,brcmnand-v5.0" },
 	{ .compatible = "brcm,brcmnand-v6.0" },
@@ -3045,9 +3115,15 @@ int brcmnand_remove(struct platform_device *pdev)
 {
 	struct brcmnand_controller *ctrl = dev_get_drvdata(&pdev->dev);
 	struct brcmnand_host *host;
+	struct nand_chip *chip;
+	int ret;
 
-	list_for_each_entry(host, &ctrl->host_list, node)
-		nand_release(&host->chip);
+	list_for_each_entry(host, &ctrl->host_list, node) {
+		chip = &host->chip;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+	}
 
 	clk_disable_unprepare(ctrl->clk);
 
diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c
index efddc5c68afb..c405722adfe1 100644
--- a/drivers/mtd/nand/raw/cadence-nand-controller.c
+++ b/drivers/mtd/nand/raw/cadence-nand-controller.c
@@ -2223,10 +2223,12 @@ static int cadence_nand_exec_op(struct nand_chip *chip,
 				const struct nand_operation *op,
 				bool check_only)
 {
-	int status = cadence_nand_select_target(chip);
+	if (!check_only) {
+		int status = cadence_nand_select_target(chip);
 
-	if (status)
-		return status;
+		if (status)
+			return status;
+	}
 
 	return nand_op_parser_exec_op(chip, &cadence_nand_op_parser, op,
 				      check_only);
@@ -2592,7 +2594,7 @@ cadence_nand_setup_data_interface(struct nand_chip *chip, int chipnr,
 	return 0;
 }
 
-int cadence_nand_attach_chip(struct nand_chip *chip)
+static int cadence_nand_attach_chip(struct nand_chip *chip)
 {
 	struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller);
 	struct cdns_nand_chip *cdns_chip = to_cdns_nand_chip(chip);
@@ -2778,9 +2780,14 @@ static int cadence_nand_chip_init(struct cdns_nand_ctrl *cdns_ctrl,
 static void cadence_nand_chips_cleanup(struct cdns_nand_ctrl *cdns_ctrl)
 {
 	struct cdns_nand_chip *entry, *temp;
+	struct nand_chip *chip;
+	int ret;
 
 	list_for_each_entry_safe(entry, temp, &cdns_ctrl->chips, node) {
-		nand_release(&entry->chip);
+		chip = &entry->chip;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
 		list_del(&entry->node);
 	}
 }
diff --git a/drivers/mtd/nand/raw/cafe_nand.c b/drivers/mtd/nand/raw/cafe_nand.c
index 2d1c22dc88c1..92173790f20b 100644
--- a/drivers/mtd/nand/raw/cafe_nand.c
+++ b/drivers/mtd/nand/raw/cafe_nand.c
@@ -546,11 +546,6 @@ static int cafe_nand_write_page_lowlevel(struct nand_chip *chip,
 	return nand_prog_page_end_op(chip);
 }
 
-static int cafe_nand_block_bad(struct nand_chip *chip, loff_t ofs)
-{
-	return 0;
-}
-
 /* F_2[X]/(X**6+X+1)  */
 static unsigned short gf64_mul(u8 a, u8 b)
 {
@@ -718,10 +713,8 @@ static int cafe_nand_probe(struct pci_dev *pdev,
 	/* Enable the following for a flash based bad block table */
 	cafe->nand.bbt_options = NAND_BBT_USE_FLASH;
 
-	if (skipbbt) {
-		cafe->nand.options |= NAND_SKIP_BBTSCAN;
-		cafe->nand.legacy.block_bad = cafe_nand_block_bad;
-	}
+	if (skipbbt)
+		cafe->nand.options |= NAND_SKIP_BBTSCAN | NAND_NO_BBM_QUIRK;
 
 	if (numtimings && numtimings != 3) {
 		dev_warn(&cafe->pdev->dev, "%d timing register values ignored; precisely three are required\n", numtimings);
@@ -814,11 +807,14 @@ static void cafe_nand_remove(struct pci_dev *pdev)
 	struct mtd_info *mtd = pci_get_drvdata(pdev);
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct cafe_priv *cafe = nand_get_controller_data(chip);
+	int ret;
 
 	/* Disable NAND IRQ in global IRQ mask register */
 	cafe_writel(cafe, ~1 & cafe_readl(cafe, GLOBAL_IRQ_MASK), GLOBAL_IRQ_MASK);
 	free_irq(pdev->irq, mtd);
-	nand_release(chip);
+	ret = mtd_device_unregister(mtd);
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	free_rs(cafe->rs);
 	pci_iounmap(pdev, cafe->mmio);
 	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);
diff --git a/drivers/mtd/nand/raw/cmx270_nand.c b/drivers/mtd/nand/raw/cmx270_nand.c
deleted file mode 100644
index 045b6175ae79..000000000000
--- a/drivers/mtd/nand/raw/cmx270_nand.c
+++ /dev/null
@@ -1,236 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *  Copyright (C) 2006 Compulab, Ltd.
- *  Mike Rapoport <mike@compulab.co.il>
- *
- *  Derived from drivers/mtd/nand/h1910.c (removed in v3.10)
- *       Copyright (C) 2002 Marius Gröger (mag@sysgo.de)
- *       Copyright (c) 2001 Thomas Gleixner (gleixner@autronix.de)
- *
- *  Overview:
- *   This is a device driver for the NAND flash device found on the
- *   CM-X270 board.
- */
-
-#include <linux/mtd/rawnand.h>
-#include <linux/mtd/partitions.h>
-#include <linux/slab.h>
-#include <linux/gpio.h>
-#include <linux/module.h>
-
-#include <asm/io.h>
-#include <asm/irq.h>
-#include <asm/mach-types.h>
-
-#include <mach/pxa2xx-regs.h>
-
-#define GPIO_NAND_CS	(11)
-#define GPIO_NAND_RB	(89)
-
-/* MTD structure for CM-X270 board */
-static struct mtd_info *cmx270_nand_mtd;
-
-/* remaped IO address of the device */
-static void __iomem *cmx270_nand_io;
-
-/*
- * Define static partitions for flash device
- */
-static const struct mtd_partition partition_info[] = {
-	[0] = {
-		.name	= "cmx270-0",
-		.offset	= 0,
-		.size	= MTDPART_SIZ_FULL
-	}
-};
-#define NUM_PARTITIONS (ARRAY_SIZE(partition_info))
-
-static u_char cmx270_read_byte(struct nand_chip *this)
-{
-	return (readl(this->legacy.IO_ADDR_R) >> 16);
-}
-
-static void cmx270_write_buf(struct nand_chip *this, const u_char *buf,
-			     int len)
-{
-	int i;
-
-	for (i=0; i<len; i++)
-		writel((*buf++ << 16), this->legacy.IO_ADDR_W);
-}
-
-static void cmx270_read_buf(struct nand_chip *this, u_char *buf, int len)
-{
-	int i;
-
-	for (i=0; i<len; i++)
-		*buf++ = readl(this->legacy.IO_ADDR_R) >> 16;
-}
-
-static inline void nand_cs_on(void)
-{
-	gpio_set_value(GPIO_NAND_CS, 0);
-}
-
-static void nand_cs_off(void)
-{
-	dsb();
-
-	gpio_set_value(GPIO_NAND_CS, 1);
-}
-
-/*
- *	hardware specific access to control-lines
- */
-static void cmx270_hwcontrol(struct nand_chip *this, int dat,
-			     unsigned int ctrl)
-{
-	unsigned int nandaddr = (unsigned int)this->legacy.IO_ADDR_W;
-
-	dsb();
-
-	if (ctrl & NAND_CTRL_CHANGE) {
-		if ( ctrl & NAND_ALE )
-			nandaddr |=  (1 << 3);
-		else
-			nandaddr &= ~(1 << 3);
-		if ( ctrl & NAND_CLE )
-			nandaddr |=  (1 << 2);
-		else
-			nandaddr &= ~(1 << 2);
-		if ( ctrl & NAND_NCE )
-			nand_cs_on();
-		else
-			nand_cs_off();
-	}
-
-	dsb();
-	this->legacy.IO_ADDR_W = (void __iomem*)nandaddr;
-	if (dat != NAND_CMD_NONE)
-		writel((dat << 16), this->legacy.IO_ADDR_W);
-
-	dsb();
-}
-
-/*
- *	read device ready pin
- */
-static int cmx270_device_ready(struct nand_chip *this)
-{
-	dsb();
-
-	return (gpio_get_value(GPIO_NAND_RB));
-}
-
-/*
- * Main initialization routine
- */
-static int __init cmx270_init(void)
-{
-	struct nand_chip *this;
-	int ret;
-
-	if (!(machine_is_armcore() && cpu_is_pxa27x()))
-		return -ENODEV;
-
-	ret = gpio_request(GPIO_NAND_CS, "NAND CS");
-	if (ret) {
-		pr_warn("CM-X270: failed to request NAND CS gpio\n");
-		return ret;
-	}
-
-	gpio_direction_output(GPIO_NAND_CS, 1);
-
-	ret = gpio_request(GPIO_NAND_RB, "NAND R/B");
-	if (ret) {
-		pr_warn("CM-X270: failed to request NAND R/B gpio\n");
-		goto err_gpio_request;
-	}
-
-	gpio_direction_input(GPIO_NAND_RB);
-
-	/* Allocate memory for MTD device structure and private data */
-	this = kzalloc(sizeof(struct nand_chip), GFP_KERNEL);
-	if (!this) {
-		ret = -ENOMEM;
-		goto err_kzalloc;
-	}
-
-	cmx270_nand_io = ioremap(PXA_CS1_PHYS, 12);
-	if (!cmx270_nand_io) {
-		pr_debug("Unable to ioremap NAND device\n");
-		ret = -EINVAL;
-		goto err_ioremap;
-	}
-
-	cmx270_nand_mtd = nand_to_mtd(this);
-
-	/* Link the private data with the MTD structure */
-	cmx270_nand_mtd->owner = THIS_MODULE;
-
-	/* insert callbacks */
-	this->legacy.IO_ADDR_R = cmx270_nand_io;
-	this->legacy.IO_ADDR_W = cmx270_nand_io;
-	this->legacy.cmd_ctrl = cmx270_hwcontrol;
-	this->legacy.dev_ready = cmx270_device_ready;
-
-	/* 15 us command delay time */
-	this->legacy.chip_delay = 20;
-	this->ecc.mode = NAND_ECC_SOFT;
-	this->ecc.algo = NAND_ECC_HAMMING;
-
-	/* read/write functions */
-	this->legacy.read_byte = cmx270_read_byte;
-	this->legacy.read_buf = cmx270_read_buf;
-	this->legacy.write_buf = cmx270_write_buf;
-
-	/* Scan to find existence of the device */
-	ret = nand_scan(this, 1);
-	if (ret) {
-		pr_notice("No NAND device\n");
-		goto err_scan;
-	}
-
-	/* Register the partitions */
-	ret = mtd_device_register(cmx270_nand_mtd, partition_info,
-				  NUM_PARTITIONS);
-	if (ret)
-		goto err_scan;
-
-	/* Return happy */
-	return 0;
-
-err_scan:
-	iounmap(cmx270_nand_io);
-err_ioremap:
-	kfree(this);
-err_kzalloc:
-	gpio_free(GPIO_NAND_RB);
-err_gpio_request:
-	gpio_free(GPIO_NAND_CS);
-
-	return ret;
-
-}
-module_init(cmx270_init);
-
-/*
- * Clean up routine
- */
-static void __exit cmx270_cleanup(void)
-{
-	/* Release resources, unregister device */
-	nand_release(mtd_to_nand(cmx270_nand_mtd));
-
-	gpio_free(GPIO_NAND_RB);
-	gpio_free(GPIO_NAND_CS);
-
-	iounmap(cmx270_nand_io);
-
-	kfree(mtd_to_nand(cmx270_nand_mtd));
-}
-module_exit(cmx270_cleanup);
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Mike Rapoport <mike@compulab.co.il>");
-MODULE_DESCRIPTION("NAND flash driver for Compulab CM-X270 Module");
diff --git a/drivers/mtd/nand/raw/cs553x_nand.c b/drivers/mtd/nand/raw/cs553x_nand.c
index e2322cee3229..9472bf798ed5 100644
--- a/drivers/mtd/nand/raw/cs553x_nand.c
+++ b/drivers/mtd/nand/raw/cs553x_nand.c
@@ -21,9 +21,9 @@
 #include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
+#include <linux/iopoll.h>
 
 #include <asm/msr.h>
-#include <asm/io.h>
 
 #define NR_CS553X_CONTROLLERS	4
 
@@ -89,76 +89,151 @@
 #define CS_NAND_ECC_CLRECC	(1<<1)
 #define CS_NAND_ECC_ENECC	(1<<0)
 
-static void cs553x_read_buf(struct nand_chip *this, u_char *buf, int len)
+struct cs553x_nand_controller {
+	struct nand_controller base;
+	struct nand_chip chip;
+	void __iomem *mmio;
+};
+
+static struct cs553x_nand_controller *
+to_cs553x(struct nand_controller *controller)
+{
+	return container_of(controller, struct cs553x_nand_controller, base);
+}
+
+static int cs553x_write_ctrl_byte(struct cs553x_nand_controller *cs553x,
+				  u32 ctl, u8 data)
 {
+	u8 status;
+	int ret;
+
+	writeb(ctl, cs553x->mmio + MM_NAND_CTL);
+	writeb(data, cs553x->mmio + MM_NAND_IO);
+	ret = readb_poll_timeout_atomic(cs553x->mmio + MM_NAND_STS, status,
+					!(status & CS_NAND_CTLR_BUSY), 1,
+					100000);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void cs553x_data_in(struct cs553x_nand_controller *cs553x, void *buf,
+			   unsigned int len)
+{
+	writeb(0, cs553x->mmio + MM_NAND_CTL);
 	while (unlikely(len > 0x800)) {
-		memcpy_fromio(buf, this->legacy.IO_ADDR_R, 0x800);
+		memcpy_fromio(buf, cs553x->mmio, 0x800);
 		buf += 0x800;
 		len -= 0x800;
 	}
-	memcpy_fromio(buf, this->legacy.IO_ADDR_R, len);
+	memcpy_fromio(buf, cs553x->mmio, len);
 }
 
-static void cs553x_write_buf(struct nand_chip *this, const u_char *buf, int len)
+static void cs553x_data_out(struct cs553x_nand_controller *cs553x,
+			    const void *buf, unsigned int len)
 {
+	writeb(0, cs553x->mmio + MM_NAND_CTL);
 	while (unlikely(len > 0x800)) {
-		memcpy_toio(this->legacy.IO_ADDR_R, buf, 0x800);
+		memcpy_toio(cs553x->mmio, buf, 0x800);
 		buf += 0x800;
 		len -= 0x800;
 	}
-	memcpy_toio(this->legacy.IO_ADDR_R, buf, len);
+	memcpy_toio(cs553x->mmio, buf, len);
 }
 
-static unsigned char cs553x_read_byte(struct nand_chip *this)
+static int cs553x_wait_ready(struct cs553x_nand_controller *cs553x,
+			     unsigned int timeout_ms)
 {
-	return readb(this->legacy.IO_ADDR_R);
+	u8 mask = CS_NAND_CTLR_BUSY | CS_NAND_STS_FLASH_RDY;
+	u8 status;
+
+	return readb_poll_timeout(cs553x->mmio + MM_NAND_STS, status,
+				  (status & mask) == CS_NAND_STS_FLASH_RDY, 100,
+				  timeout_ms * 1000);
 }
 
-static void cs553x_write_byte(struct nand_chip *this, u_char byte)
+static int cs553x_exec_instr(struct cs553x_nand_controller *cs553x,
+			     const struct nand_op_instr *instr)
 {
-	int i = 100000;
+	unsigned int i;
+	int ret = 0;
+
+	switch (instr->type) {
+	case NAND_OP_CMD_INSTR:
+		ret = cs553x_write_ctrl_byte(cs553x, CS_NAND_CTL_CLE,
+					     instr->ctx.cmd.opcode);
+		break;
+
+	case NAND_OP_ADDR_INSTR:
+		for (i = 0; i < instr->ctx.addr.naddrs; i++) {
+			ret = cs553x_write_ctrl_byte(cs553x, CS_NAND_CTL_ALE,
+						     instr->ctx.addr.addrs[i]);
+			if (ret)
+				break;
+		}
+		break;
+
+	case NAND_OP_DATA_IN_INSTR:
+		cs553x_data_in(cs553x, instr->ctx.data.buf.in,
+			       instr->ctx.data.len);
+		break;
+
+	case NAND_OP_DATA_OUT_INSTR:
+		cs553x_data_out(cs553x, instr->ctx.data.buf.out,
+				instr->ctx.data.len);
+		break;
 
-	while (i && readb(this->legacy.IO_ADDR_R + MM_NAND_STS) & CS_NAND_CTLR_BUSY) {
-		udelay(1);
-		i--;
+	case NAND_OP_WAITRDY_INSTR:
+		ret = cs553x_wait_ready(cs553x, instr->ctx.waitrdy.timeout_ms);
+		break;
 	}
-	writeb(byte, this->legacy.IO_ADDR_W + 0x801);
+
+	if (instr->delay_ns)
+		ndelay(instr->delay_ns);
+
+	return ret;
 }
 
-static void cs553x_hwcontrol(struct nand_chip *this, int cmd,
-			     unsigned int ctrl)
+static int cs553x_exec_op(struct nand_chip *this,
+			  const struct nand_operation *op,
+			  bool check_only)
 {
-	void __iomem *mmio_base = this->legacy.IO_ADDR_R;
-	if (ctrl & NAND_CTRL_CHANGE) {
-		unsigned char ctl = (ctrl & ~NAND_CTRL_CHANGE ) ^ 0x01;
-		writeb(ctl, mmio_base + MM_NAND_CTL);
+	struct cs553x_nand_controller *cs553x = to_cs553x(this->controller);
+	unsigned int i;
+	int ret;
+
+	if (check_only)
+		return true;
+
+	/* De-assert the CE pin */
+	writeb(0, cs553x->mmio + MM_NAND_CTL);
+	for (i = 0; i < op->ninstrs; i++) {
+		ret = cs553x_exec_instr(cs553x, &op->instrs[i]);
+		if (ret)
+			break;
 	}
-	if (cmd != NAND_CMD_NONE)
-		cs553x_write_byte(this, cmd);
-}
 
-static int cs553x_device_ready(struct nand_chip *this)
-{
-	void __iomem *mmio_base = this->legacy.IO_ADDR_R;
-	unsigned char foo = readb(mmio_base + MM_NAND_STS);
+	/* Re-assert the CE pin. */
+	writeb(CS_NAND_CTL_CE, cs553x->mmio + MM_NAND_CTL);
 
-	return (foo & CS_NAND_STS_FLASH_RDY) && !(foo & CS_NAND_CTLR_BUSY);
+	return ret;
 }
 
 static void cs_enable_hwecc(struct nand_chip *this, int mode)
 {
-	void __iomem *mmio_base = this->legacy.IO_ADDR_R;
+	struct cs553x_nand_controller *cs553x = to_cs553x(this->controller);
 
-	writeb(0x07, mmio_base + MM_NAND_ECC_CTL);
+	writeb(0x07, cs553x->mmio + MM_NAND_ECC_CTL);
 }
 
 static int cs_calculate_ecc(struct nand_chip *this, const u_char *dat,
 			    u_char *ecc_code)
 {
+	struct cs553x_nand_controller *cs553x = to_cs553x(this->controller);
 	uint32_t ecc;
-	void __iomem *mmio_base = this->legacy.IO_ADDR_R;
 
-	ecc = readl(mmio_base + MM_NAND_STS);
+	ecc = readl(cs553x->mmio + MM_NAND_STS);
 
 	ecc_code[1] = ecc >> 8;
 	ecc_code[0] = ecc >> 16;
@@ -166,10 +241,15 @@ static int cs_calculate_ecc(struct nand_chip *this, const u_char *dat,
 	return 0;
 }
 
-static struct mtd_info *cs553x_mtd[4];
+static struct cs553x_nand_controller *controllers[4];
+
+static const struct nand_controller_ops cs553x_nand_controller_ops = {
+	.exec_op = cs553x_exec_op,
+};
 
 static int __init cs553x_init_one(int cs, int mmio, unsigned long adr)
 {
+	struct cs553x_nand_controller *controller;
 	int err = 0;
 	struct nand_chip *this;
 	struct mtd_info *new_mtd;
@@ -183,33 +263,29 @@ static int __init cs553x_init_one(int cs, int mmio, unsigned long adr)
 	}
 
 	/* Allocate memory for MTD device structure and private data */
-	this = kzalloc(sizeof(struct nand_chip), GFP_KERNEL);
-	if (!this) {
+	controller = kzalloc(sizeof(*controller), GFP_KERNEL);
+	if (!controller) {
 		err = -ENOMEM;
 		goto out;
 	}
 
+	this = &controller->chip;
+	nand_controller_init(&controller->base);
+	controller->base.ops = &cs553x_nand_controller_ops;
+	this->controller = &controller->base;
 	new_mtd = nand_to_mtd(this);
 
 	/* Link the private data with the MTD structure */
 	new_mtd->owner = THIS_MODULE;
 
 	/* map physical address */
-	this->legacy.IO_ADDR_R = this->legacy.IO_ADDR_W = ioremap(adr, 4096);
-	if (!this->legacy.IO_ADDR_R) {
+	controller->mmio = ioremap(adr, 4096);
+	if (!controller->mmio) {
 		pr_warn("ioremap cs553x NAND @0x%08lx failed\n", adr);
 		err = -EIO;
 		goto out_mtd;
 	}
 
-	this->legacy.cmd_ctrl = cs553x_hwcontrol;
-	this->legacy.dev_ready = cs553x_device_ready;
-	this->legacy.read_byte = cs553x_read_byte;
-	this->legacy.read_buf = cs553x_read_buf;
-	this->legacy.write_buf = cs553x_write_buf;
-
-	this->legacy.chip_delay = 0;
-
 	this->ecc.mode = NAND_ECC_HW;
 	this->ecc.size = 256;
 	this->ecc.bytes = 3;
@@ -232,15 +308,15 @@ static int __init cs553x_init_one(int cs, int mmio, unsigned long adr)
 	if (err)
 		goto out_free;
 
-	cs553x_mtd[cs] = new_mtd;
+	controllers[cs] = controller;
 	goto out;
 
 out_free:
 	kfree(new_mtd->name);
 out_ior:
-	iounmap(this->legacy.IO_ADDR_R);
+	iounmap(controller->mmio);
 out_mtd:
-	kfree(this);
+	kfree(controller);
 out:
 	return err;
 }
@@ -295,9 +371,10 @@ static int __init cs553x_init(void)
 	/* Register all devices together here. This means we can easily hack it to
 	   do mtdconcat etc. if we want to. */
 	for (i = 0; i < NR_CS553X_CONTROLLERS; i++) {
-		if (cs553x_mtd[i]) {
+		if (controllers[i]) {
 			/* If any devices registered, return success. Else the last error. */
-			mtd_device_register(cs553x_mtd[i], NULL, 0);
+			mtd_device_register(nand_to_mtd(&controllers[i]->chip),
+					    NULL, 0);
 			err = 0;
 		}
 	}
@@ -312,26 +389,26 @@ static void __exit cs553x_cleanup(void)
 	int i;
 
 	for (i = 0; i < NR_CS553X_CONTROLLERS; i++) {
-		struct mtd_info *mtd = cs553x_mtd[i];
-		struct nand_chip *this;
-		void __iomem *mmio_base;
+		struct cs553x_nand_controller *controller = controllers[i];
+		struct nand_chip *this = &controller->chip;
+		struct mtd_info *mtd = nand_to_mtd(this);
+		int ret;
 
 		if (!mtd)
 			continue;
 
-		this = mtd_to_nand(mtd);
-		mmio_base = this->legacy.IO_ADDR_R;
-
 		/* Release resources, unregister device */
-		nand_release(this);
+		ret = mtd_device_unregister(mtd);
+		WARN_ON(ret);
+		nand_cleanup(this);
 		kfree(mtd->name);
-		cs553x_mtd[i] = NULL;
+		controllers[i] = NULL;
 
 		/* unmap physical address */
-		iounmap(mmio_base);
+		iounmap(controller->mmio);
 
 		/* Free the MTD device structure */
-		kfree(this);
+		kfree(controller);
 	}
 }
 
diff --git a/drivers/mtd/nand/raw/davinci_nand.c b/drivers/mtd/nand/raw/davinci_nand.c
index 25c185bea50c..d975a62caaa5 100644
--- a/drivers/mtd/nand/raw/davinci_nand.c
+++ b/drivers/mtd/nand/raw/davinci_nand.c
@@ -14,7 +14,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/err.h>
-#include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/slab.h>
@@ -38,6 +38,7 @@
  * outputs in a "wire-AND" configuration, with no per-chip signals.
  */
 struct davinci_nand_info {
+	struct nand_controller	controller;
 	struct nand_chip	chip;
 
 	struct platform_device	*pdev;
@@ -81,46 +82,6 @@ static inline void davinci_nand_writel(struct davinci_nand_info *info,
 /*----------------------------------------------------------------------*/
 
 /*
- * Access to hardware control lines:  ALE, CLE, secondary chipselect.
- */
-
-static void nand_davinci_hwcontrol(struct nand_chip *nand, int cmd,
-				   unsigned int ctrl)
-{
-	struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(nand));
-	void __iomem			*addr = info->current_cs;
-
-	/* Did the control lines change? */
-	if (ctrl & NAND_CTRL_CHANGE) {
-		if ((ctrl & NAND_CTRL_CLE) == NAND_CTRL_CLE)
-			addr += info->mask_cle;
-		else if ((ctrl & NAND_CTRL_ALE) == NAND_CTRL_ALE)
-			addr += info->mask_ale;
-
-		nand->legacy.IO_ADDR_W = addr;
-	}
-
-	if (cmd != NAND_CMD_NONE)
-		iowrite8(cmd, nand->legacy.IO_ADDR_W);
-}
-
-static void nand_davinci_select_chip(struct nand_chip *nand, int chip)
-{
-	struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(nand));
-
-	info->current_cs = info->vaddr;
-
-	/* maybe kick in a second chipselect */
-	if (chip > 0)
-		info->current_cs += info->mask_chipsel;
-
-	info->chip.legacy.IO_ADDR_W = info->current_cs;
-	info->chip.legacy.IO_ADDR_R = info->chip.legacy.IO_ADDR_W;
-}
-
-/*----------------------------------------------------------------------*/
-
-/*
  * 1-bit hardware ECC ... context maintained for each core chipselect
  */
 
@@ -410,48 +371,75 @@ correct:
 	return corrected;
 }
 
-/*----------------------------------------------------------------------*/
-
-/*
- * NOTE:  NAND boot requires ALE == EM_A[1], CLE == EM_A[2], so that's
- * how these chips are normally wired.  This translates to both 8 and 16
- * bit busses using ALE == BIT(3) in byte addresses, and CLE == BIT(4).
+/**
+ * nand_read_page_hwecc_oob_first - hw ecc, read oob first
+ * @chip: nand chip info structure
+ * @buf: buffer to store read data
+ * @oob_required: caller requires OOB data read to chip->oob_poi
+ * @page: page number to read
  *
- * For now we assume that configuration, or any other one which ignores
- * the two LSBs for NAND access ... so we can issue 32-bit reads/writes
- * and have that transparently morphed into multiple NAND operations.
+ * Hardware ECC for large page chips, require OOB to be read first. For this
+ * ECC mode, the write_page method is re-used from ECC_HW. These methods
+ * read/write ECC from the OOB area, unlike the ECC_HW_SYNDROME support with
+ * multiple ECC steps, follows the "infix ECC" scheme and reads/writes ECC from
+ * the data area, by overwriting the NAND manufacturer bad block markings.
  */
-static void nand_davinci_read_buf(struct nand_chip *chip, uint8_t *buf,
-				  int len)
+static int nand_davinci_read_page_hwecc_oob_first(struct nand_chip *chip,
+						  uint8_t *buf,
+						  int oob_required, int page)
 {
-	if ((0x03 & ((uintptr_t)buf)) == 0 && (0x03 & len) == 0)
-		ioread32_rep(chip->legacy.IO_ADDR_R, buf, len >> 2);
-	else if ((0x01 & ((uintptr_t)buf)) == 0 && (0x01 & len) == 0)
-		ioread16_rep(chip->legacy.IO_ADDR_R, buf, len >> 1);
-	else
-		ioread8_rep(chip->legacy.IO_ADDR_R, buf, len);
-}
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int i, eccsize = chip->ecc.size, ret;
+	int eccbytes = chip->ecc.bytes;
+	int eccsteps = chip->ecc.steps;
+	uint8_t *p = buf;
+	uint8_t *ecc_code = chip->ecc.code_buf;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
+	unsigned int max_bitflips = 0;
+
+	/* Read the OOB area first */
+	ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
+	if (ret)
+		return ret;
 
-static void nand_davinci_write_buf(struct nand_chip *chip, const uint8_t *buf,
-				   int len)
-{
-	if ((0x03 & ((uintptr_t)buf)) == 0 && (0x03 & len) == 0)
-		iowrite32_rep(chip->legacy.IO_ADDR_R, buf, len >> 2);
-	else if ((0x01 & ((uintptr_t)buf)) == 0 && (0x01 & len) == 0)
-		iowrite16_rep(chip->legacy.IO_ADDR_R, buf, len >> 1);
-	else
-		iowrite8_rep(chip->legacy.IO_ADDR_R, buf, len);
-}
+	ret = nand_read_page_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
 
-/*
- * Check hardware register for wait status. Returns 1 if device is ready,
- * 0 if it is still busy.
- */
-static int nand_davinci_dev_ready(struct nand_chip *chip)
-{
-	struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(chip));
+	ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
+					 chip->ecc.total);
+	if (ret)
+		return ret;
+
+	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
+		int stat;
+
+		chip->ecc.hwctl(chip, NAND_ECC_READ);
 
-	return davinci_nand_readl(info, NANDFSR_OFFSET) & BIT(0);
+		ret = nand_read_data_op(chip, p, eccsize, false, false);
+		if (ret)
+			return ret;
+
+		chip->ecc.calculate(chip, p, &ecc_calc[i]);
+
+		stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL);
+		if (stat == -EBADMSG &&
+		    (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
+			/* check for empty pages with bitflips */
+			stat = nand_check_erased_ecc_chunk(p, eccsize,
+							   &ecc_code[i],
+							   eccbytes, NULL, 0,
+							   chip->ecc.strength);
+		}
+
+		if (stat < 0) {
+			mtd->ecc_stats.failed++;
+		} else {
+			mtd->ecc_stats.corrected += stat;
+			max_bitflips = max_t(unsigned int, max_bitflips, stat);
+		}
+	}
+	return max_bitflips;
 }
 
 /*----------------------------------------------------------------------*/
@@ -613,6 +601,13 @@ static int davinci_nand_attach_chip(struct nand_chip *chip)
 		break;
 	case NAND_ECC_HW:
 		if (pdata->ecc_bits == 4) {
+			int chunks = mtd->writesize / 512;
+
+			if (!chunks || mtd->oobsize < 16) {
+				dev_dbg(&info->pdev->dev, "too small\n");
+				return -EINVAL;
+			}
+
 			/*
 			 * No sanity checks:  CPUs must support this,
 			 * and the chips may not use NAND_BUSWIDTH_16.
@@ -635,6 +630,26 @@ static int davinci_nand_attach_chip(struct nand_chip *chip)
 			info->chip.ecc.bytes = 10;
 			info->chip.ecc.options = NAND_ECC_GENERIC_ERASED_CHECK;
 			info->chip.ecc.algo = NAND_ECC_BCH;
+
+			/*
+			 * Update ECC layout if needed ... for 1-bit HW ECC, the
+			 * default is OK, but it allocates 6 bytes when only 3
+			 * are needed (for each 512 bytes). For 4-bit HW ECC,
+			 * the default is not usable: 10 bytes needed, not 6.
+			 *
+			 * For small page chips, preserve the manufacturer's
+			 * badblock marking data ... and make sure a flash BBT
+			 * table marker fits in the free bytes.
+			 */
+			if (chunks == 1) {
+				mtd_set_ooblayout(mtd,
+						  &hwecc4_small_ooblayout_ops);
+			} else if (chunks == 4 || chunks == 8) {
+				mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
+				info->chip.ecc.read_page = nand_davinci_read_page_hwecc_oob_first;
+			} else {
+				return -EIO;
+			}
 		} else {
 			/* 1bit ecc hamming */
 			info->chip.ecc.calculate = nand_davinci_calculate_1bit;
@@ -650,39 +665,111 @@ static int davinci_nand_attach_chip(struct nand_chip *chip)
 		return -EINVAL;
 	}
 
-	/*
-	 * Update ECC layout if needed ... for 1-bit HW ECC, the default
-	 * is OK, but it allocates 6 bytes when only 3 are needed (for
-	 * each 512 bytes).  For the 4-bit HW ECC, that default is not
-	 * usable:  10 bytes are needed, not 6.
-	 */
-	if (pdata->ecc_bits == 4) {
-		int chunks = mtd->writesize / 512;
+	return ret;
+}
 
-		if (!chunks || mtd->oobsize < 16) {
-			dev_dbg(&info->pdev->dev, "too small\n");
-			return -EINVAL;
-		}
+static void nand_davinci_data_in(struct davinci_nand_info *info, void *buf,
+				 unsigned int len, bool force_8bit)
+{
+	u32 alignment = ((uintptr_t)buf | len) & 3;
 
-		/* For small page chips, preserve the manufacturer's
-		 * badblock marking data ... and make sure a flash BBT
-		 * table marker fits in the free bytes.
-		 */
-		if (chunks == 1) {
-			mtd_set_ooblayout(mtd, &hwecc4_small_ooblayout_ops);
-		} else if (chunks == 4 || chunks == 8) {
-			mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
-			info->chip.ecc.mode = NAND_ECC_HW_OOB_FIRST;
-		} else {
-			return -EIO;
+	if (force_8bit || (alignment & 1))
+		ioread8_rep(info->current_cs, buf, len);
+	else if (alignment & 3)
+		ioread16_rep(info->current_cs, buf, len >> 1);
+	else
+		ioread32_rep(info->current_cs, buf, len >> 2);
+}
+
+static void nand_davinci_data_out(struct davinci_nand_info *info,
+				  const void *buf, unsigned int len,
+				  bool force_8bit)
+{
+	u32 alignment = ((uintptr_t)buf | len) & 3;
+
+	if (force_8bit || (alignment & 1))
+		iowrite8_rep(info->current_cs, buf, len);
+	else if (alignment & 3)
+		iowrite16_rep(info->current_cs, buf, len >> 1);
+	else
+		iowrite32_rep(info->current_cs, buf, len >> 2);
+}
+
+static int davinci_nand_exec_instr(struct davinci_nand_info *info,
+				   const struct nand_op_instr *instr)
+{
+	unsigned int i, timeout_us;
+	u32 status;
+	int ret;
+
+	switch (instr->type) {
+	case NAND_OP_CMD_INSTR:
+		iowrite8(instr->ctx.cmd.opcode,
+			 info->current_cs + info->mask_cle);
+		break;
+
+	case NAND_OP_ADDR_INSTR:
+		for (i = 0; i < instr->ctx.addr.naddrs; i++) {
+			iowrite8(instr->ctx.addr.addrs[i],
+				 info->current_cs + info->mask_ale);
 		}
+		break;
+
+	case NAND_OP_DATA_IN_INSTR:
+		nand_davinci_data_in(info, instr->ctx.data.buf.in,
+				     instr->ctx.data.len,
+				     instr->ctx.data.force_8bit);
+		break;
+
+	case NAND_OP_DATA_OUT_INSTR:
+		nand_davinci_data_out(info, instr->ctx.data.buf.out,
+				      instr->ctx.data.len,
+				      instr->ctx.data.force_8bit);
+		break;
+
+	case NAND_OP_WAITRDY_INSTR:
+		timeout_us = instr->ctx.waitrdy.timeout_ms * 1000;
+		ret = readl_relaxed_poll_timeout(info->base + NANDFSR_OFFSET,
+						 status, status & BIT(0), 100,
+						 timeout_us);
+		if (ret)
+			return ret;
+
+		break;
 	}
 
-	return ret;
+	if (instr->delay_ns)
+		ndelay(instr->delay_ns);
+
+	return 0;
+}
+
+static int davinci_nand_exec_op(struct nand_chip *chip,
+				const struct nand_operation *op,
+				bool check_only)
+{
+	struct davinci_nand_info *info = to_davinci_nand(nand_to_mtd(chip));
+	unsigned int i;
+
+	if (check_only)
+		return 0;
+
+	info->current_cs = info->vaddr + (op->cs * info->mask_chipsel);
+
+	for (i = 0; i < op->ninstrs; i++) {
+		int ret;
+
+		ret = davinci_nand_exec_instr(info, &op->instrs[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static const struct nand_controller_ops davinci_nand_controller_ops = {
 	.attach_chip = davinci_nand_attach_chip,
+	.exec_op = davinci_nand_exec_op,
 };
 
 static int nand_davinci_probe(struct platform_device *pdev)
@@ -746,11 +833,6 @@ static int nand_davinci_probe(struct platform_device *pdev)
 	mtd->dev.parent		= &pdev->dev;
 	nand_set_flash_node(&info->chip, pdev->dev.of_node);
 
-	info->chip.legacy.IO_ADDR_R	= vaddr;
-	info->chip.legacy.IO_ADDR_W	= vaddr;
-	info->chip.legacy.chip_delay	= 0;
-	info->chip.legacy.select_chip	= nand_davinci_select_chip;
-
 	/* options such as NAND_BBT_USE_FLASH */
 	info->chip.bbt_options	= pdata->bbt_options;
 	/* options such as 16-bit widths */
@@ -767,14 +849,6 @@ static int nand_davinci_probe(struct platform_device *pdev)
 	info->mask_ale		= pdata->mask_ale ? : MASK_ALE;
 	info->mask_cle		= pdata->mask_cle ? : MASK_CLE;
 
-	/* Set address of hardware control function */
-	info->chip.legacy.cmd_ctrl	= nand_davinci_hwcontrol;
-	info->chip.legacy.dev_ready	= nand_davinci_dev_ready;
-
-	/* Speed up buffer I/O */
-	info->chip.legacy.read_buf     = nand_davinci_read_buf;
-	info->chip.legacy.write_buf    = nand_davinci_write_buf;
-
 	/* Use board-specific ECC config */
 	info->chip.ecc.mode	= pdata->ecc_mode;
 
@@ -788,7 +862,9 @@ static int nand_davinci_probe(struct platform_device *pdev)
 	spin_unlock_irq(&davinci_nand_lock);
 
 	/* Scan to find existence of the device(s) */
-	info->chip.legacy.dummy_controller.ops = &davinci_nand_controller_ops;
+	nand_controller_init(&info->controller);
+	info->controller.ops = &davinci_nand_controller_ops;
+	info->chip.controller = &info->controller;
 	ret = nand_scan(&info->chip, pdata->mask_chipsel ? 2 : 1);
 	if (ret < 0) {
 		dev_dbg(&pdev->dev, "no NAND chip(s) found\n");
@@ -817,13 +893,17 @@ err_cleanup_nand:
 static int nand_davinci_remove(struct platform_device *pdev)
 {
 	struct davinci_nand_info *info = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &info->chip;
+	int ret;
 
 	spin_lock_irq(&davinci_nand_lock);
 	if (info->chip.ecc.mode == NAND_ECC_HW_SYNDROME)
 		ecc4_busy = false;
 	spin_unlock_irq(&davinci_nand_lock);
 
-	nand_release(&info->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/denali.c b/drivers/mtd/nand/raw/denali.c
index 6a6c919b2569..4e6e1578aa2d 100644
--- a/drivers/mtd/nand/raw/denali.c
+++ b/drivers/mtd/nand/raw/denali.c
@@ -764,6 +764,7 @@ static int denali_write_page(struct nand_chip *chip, const u8 *buf,
 static int denali_setup_data_interface(struct nand_chip *chip, int chipnr,
 				       const struct nand_data_interface *conf)
 {
+	static const unsigned int data_setup_on_host = 10000;
 	struct denali_controller *denali = to_denali_controller(chip);
 	struct denali_chip_sel *sel;
 	const struct nand_sdr_timings *timings;
@@ -796,15 +797,6 @@ static int denali_setup_data_interface(struct nand_chip *chip, int chipnr,
 
 	sel = &to_denali_chip(chip)->sels[chipnr];
 
-	/* tREA -> ACC_CLKS */
-	acc_clks = DIV_ROUND_UP(timings->tREA_max, t_x);
-	acc_clks = min_t(int, acc_clks, ACC_CLKS__VALUE);
-
-	tmp = ioread32(denali->reg + ACC_CLKS);
-	tmp &= ~ACC_CLKS__VALUE;
-	tmp |= FIELD_PREP(ACC_CLKS__VALUE, acc_clks);
-	sel->acc_clks = tmp;
-
 	/* tRWH -> RE_2_WE */
 	re_2_we = DIV_ROUND_UP(timings->tRHW_min, t_x);
 	re_2_we = min_t(int, re_2_we, RE_2_WE__VALUE);
@@ -862,14 +854,45 @@ static int denali_setup_data_interface(struct nand_chip *chip, int chipnr,
 	tmp |= FIELD_PREP(RDWR_EN_HI_CNT__VALUE, rdwr_en_hi);
 	sel->rdwr_en_hi_cnt = tmp;
 
-	/* tRP, tWP -> RDWR_EN_LO_CNT */
+	/*
+	 * tREA -> ACC_CLKS
+	 * tRP, tWP, tRHOH, tRC, tWC -> RDWR_EN_LO_CNT
+	 */
+
+	/*
+	 * Determine the minimum of acc_clks to meet the setup timing when
+	 * capturing the incoming data.
+	 *
+	 * The delay on the chip side is well-defined as tREA, but we need to
+	 * take additional delay into account. This includes a certain degree
+	 * of unknowledge, such as signal propagation delays on the PCB and
+	 * in the SoC, load capacity of the I/O pins, etc.
+	 */
+	acc_clks = DIV_ROUND_UP(timings->tREA_max + data_setup_on_host, t_x);
+
+	/* Determine the minimum of rdwr_en_lo_cnt from RE#/WE# pulse width */
 	rdwr_en_lo = DIV_ROUND_UP(max(timings->tRP_min, timings->tWP_min), t_x);
+
+	/* Extend rdwr_en_lo to meet the data hold timing */
+	rdwr_en_lo = max_t(int, rdwr_en_lo,
+			   acc_clks - timings->tRHOH_min / t_x);
+
+	/* Extend rdwr_en_lo to meet the requirement for RE#/WE# cycle time */
 	rdwr_en_lo_hi = DIV_ROUND_UP(max(timings->tRC_min, timings->tWC_min),
 				     t_x);
-	rdwr_en_lo_hi = max_t(int, rdwr_en_lo_hi, mult_x);
 	rdwr_en_lo = max(rdwr_en_lo, rdwr_en_lo_hi - rdwr_en_hi);
 	rdwr_en_lo = min_t(int, rdwr_en_lo, RDWR_EN_LO_CNT__VALUE);
 
+	/* Center the data latch timing for extra safety */
+	acc_clks = (acc_clks + rdwr_en_lo +
+		    DIV_ROUND_UP(timings->tRHOH_min, t_x)) / 2;
+	acc_clks = min_t(int, acc_clks, ACC_CLKS__VALUE);
+
+	tmp = ioread32(denali->reg + ACC_CLKS);
+	tmp &= ~ACC_CLKS__VALUE;
+	tmp |= FIELD_PREP(ACC_CLKS__VALUE, acc_clks);
+	sel->acc_clks = tmp;
+
 	tmp = ioread32(denali->reg + RDWR_EN_LO_CNT);
 	tmp &= ~RDWR_EN_LO_CNT__VALUE;
 	tmp |= FIELD_PREP(RDWR_EN_LO_CNT__VALUE, rdwr_en_lo);
@@ -1203,7 +1226,7 @@ int denali_chip_init(struct denali_controller *denali,
 		mtd->name = "denali-nand";
 
 	if (denali->dma_avail) {
-		chip->options |= NAND_USE_BOUNCE_BUFFER;
+		chip->options |= NAND_USES_DMA;
 		chip->buf_align = 16;
 	}
 
@@ -1336,10 +1359,17 @@ EXPORT_SYMBOL(denali_init);
 
 void denali_remove(struct denali_controller *denali)
 {
-	struct denali_chip *dchip;
+	struct denali_chip *dchip, *tmp;
+	struct nand_chip *chip;
+	int ret;
 
-	list_for_each_entry(dchip, &denali->chips, node)
-		nand_release(&dchip->chip);
+	list_for_each_entry_safe(dchip, tmp, &denali->chips, node) {
+		chip = &dchip->chip;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+		list_del(&dchip->node);
+	}
 
 	denali_disable_irq(denali);
 }
diff --git a/drivers/mtd/nand/raw/diskonchip.c b/drivers/mtd/nand/raw/diskonchip.c
index c2a391ad2c35..43721863a0d8 100644
--- a/drivers/mtd/nand/raw/diskonchip.c
+++ b/drivers/mtd/nand/raw/diskonchip.c
@@ -58,6 +58,7 @@ static unsigned long doc_locations[] __initdata = {
 static struct mtd_info *doclist = NULL;
 
 struct doc_priv {
+	struct nand_controller base;
 	void __iomem *virtadr;
 	unsigned long physadr;
 	u_char ChipID;
@@ -69,6 +70,7 @@ struct doc_priv {
 	int mh1_page;
 	struct rs_control *rs_decoder;
 	struct mtd_info *nextdoc;
+	bool supports_32b_reads;
 
 	/* Handle the last stage of initialization (BBT scan, partitioning) */
 	int (*late_init)(struct mtd_info *mtd);
@@ -84,10 +86,6 @@ static u_char empty_write_ecc[6] = { 0x4b, 0x00, 0xe2, 0x0e, 0x93, 0xf7 };
 #define DoC_is_Millennium(doc) ((doc)->ChipID == DOC_ChipID_DocMil)
 #define DoC_is_2000(doc) ((doc)->ChipID == DOC_ChipID_Doc2k)
 
-static void doc200x_hwcontrol(struct nand_chip *this, int cmd,
-			      unsigned int bitmask);
-static void doc200x_select_chip(struct nand_chip *this, int chip);
-
 static int debug = 0;
 module_param(debug, int, 0);
 
@@ -302,20 +300,6 @@ static void doc2000_write_byte(struct nand_chip *this, u_char datum)
 	WriteDOC(datum, docptr, 2k_CDSN_IO);
 }
 
-static u_char doc2000_read_byte(struct nand_chip *this)
-{
-	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
-	u_char ret;
-
-	ReadDOC(docptr, CDSNSlowIO);
-	DoC_Delay(doc, 2);
-	ret = ReadDOC(docptr, 2k_CDSN_IO);
-	if (debug)
-		printk("read_byte returns %02x\n", ret);
-	return ret;
-}
-
 static void doc2000_writebuf(struct nand_chip *this, const u_char *buf,
 			     int len)
 {
@@ -337,33 +321,42 @@ static void doc2000_readbuf(struct nand_chip *this, u_char *buf, int len)
 {
 	struct doc_priv *doc = nand_get_controller_data(this);
 	void __iomem *docptr = doc->virtadr;
+	u32 *buf32 = (u32 *)buf;
 	int i;
 
 	if (debug)
 		printk("readbuf of %d bytes: ", len);
 
-	for (i = 0; i < len; i++)
-		buf[i] = ReadDOC(docptr, 2k_CDSN_IO + i);
+	if (!doc->supports_32b_reads ||
+	    ((((unsigned long)buf) | len) & 3)) {
+		for (i = 0; i < len; i++)
+			buf[i] = ReadDOC(docptr, 2k_CDSN_IO + i);
+	} else {
+		for (i = 0; i < len / 4; i++)
+			buf32[i] = readl(docptr + DoC_2k_CDSN_IO + i);
+	}
 }
 
-static void doc2000_readbuf_dword(struct nand_chip *this, u_char *buf, int len)
+/*
+ * We need our own readid() here because it's called before the NAND chip
+ * has been initialized, and calling nand_op_readid() would lead to a NULL
+ * pointer exception when dereferencing the NAND timings.
+ */
+static void doc200x_readid(struct nand_chip *this, unsigned int cs, u8 *id)
 {
-	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
-	int i;
+	u8 addr = 0;
+	struct nand_op_instr instrs[] = {
+		NAND_OP_CMD(NAND_CMD_READID, 0),
+		NAND_OP_ADDR(1, &addr, 50),
+		NAND_OP_8BIT_DATA_IN(2, id, 0),
+	};
 
-	if (debug)
-		printk("readbuf_dword of %d bytes: ", len);
+	struct nand_operation op = NAND_OPERATION(cs, instrs);
 
-	if (unlikely((((unsigned long)buf) | len) & 3)) {
-		for (i = 0; i < len; i++) {
-			*(uint8_t *) (&buf[i]) = ReadDOC(docptr, 2k_CDSN_IO + i);
-		}
-	} else {
-		for (i = 0; i < len; i += 4) {
-			*(uint32_t *) (&buf[i]) = readl(docptr + DoC_2k_CDSN_IO + i);
-		}
-	}
+	if (!id)
+		op.ninstrs--;
+
+	this->controller->ops->exec_op(this, &op, false);
 }
 
 static uint16_t __init doc200x_ident_chip(struct mtd_info *mtd, int nr)
@@ -371,20 +364,11 @@ static uint16_t __init doc200x_ident_chip(struct mtd_info *mtd, int nr)
 	struct nand_chip *this = mtd_to_nand(mtd);
 	struct doc_priv *doc = nand_get_controller_data(this);
 	uint16_t ret;
+	u8 id[2];
 
-	doc200x_select_chip(this, nr);
-	doc200x_hwcontrol(this, NAND_CMD_READID,
-			  NAND_CTRL_CLE | NAND_CTRL_CHANGE);
-	doc200x_hwcontrol(this, 0, NAND_CTRL_ALE | NAND_CTRL_CHANGE);
-	doc200x_hwcontrol(this, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE);
+	doc200x_readid(this, nr, id);
 
-	/* We can't use dev_ready here, but at least we wait for the
-	 * command to complete
-	 */
-	udelay(50);
-
-	ret = this->legacy.read_byte(this) << 8;
-	ret |= this->legacy.read_byte(this);
+	ret = ((u16)id[0] << 8) | id[1];
 
 	if (doc->ChipID == DOC_ChipID_Doc2k && try_dword && !nr) {
 		/* First chip probe. See if we get same results by 32-bit access */
@@ -394,18 +378,12 @@ static uint16_t __init doc200x_ident_chip(struct mtd_info *mtd, int nr)
 		} ident;
 		void __iomem *docptr = doc->virtadr;
 
-		doc200x_hwcontrol(this, NAND_CMD_READID,
-				  NAND_CTRL_CLE | NAND_CTRL_CHANGE);
-		doc200x_hwcontrol(this, 0, NAND_CTRL_ALE | NAND_CTRL_CHANGE);
-		doc200x_hwcontrol(this, NAND_CMD_NONE,
-				  NAND_NCE | NAND_CTRL_CHANGE);
-
-		udelay(50);
+		doc200x_readid(this, nr, NULL);
 
 		ident.dword = readl(docptr + DoC_2k_CDSN_IO);
 		if (((ident.byte[0] << 8) | ident.byte[1]) == ret) {
 			pr_info("DiskOnChip 2000 responds to DWORD access\n");
-			this->legacy.read_buf = &doc2000_readbuf_dword;
+			doc->supports_32b_reads = true;
 		}
 	}
 
@@ -434,20 +412,6 @@ static void __init doc2000_count_chips(struct mtd_info *mtd)
 	pr_debug("Detected %d chips per floor.\n", i);
 }
 
-static int doc200x_wait(struct nand_chip *this)
-{
-	struct doc_priv *doc = nand_get_controller_data(this);
-
-	int status;
-
-	DoC_WaitReady(doc);
-	nand_status_op(this, NULL);
-	DoC_WaitReady(doc);
-	status = (int)this->legacy.read_byte(this);
-
-	return status;
-}
-
 static void doc2001_write_byte(struct nand_chip *this, u_char datum)
 {
 	struct doc_priv *doc = nand_get_controller_data(this);
@@ -458,19 +422,6 @@ static void doc2001_write_byte(struct nand_chip *this, u_char datum)
 	WriteDOC(datum, docptr, WritePipeTerm);
 }
 
-static u_char doc2001_read_byte(struct nand_chip *this)
-{
-	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
-
-	//ReadDOC(docptr, CDSNSlowIO);
-	/* 11.4.5 -- delay twice to allow extended length cycle */
-	DoC_Delay(doc, 2);
-	ReadDOC(docptr, ReadPipeInit);
-	//return ReadDOC(docptr, Mil_CDSN_IO);
-	return ReadDOC(docptr, LastDataRead);
-}
-
 static void doc2001_writebuf(struct nand_chip *this, const u_char *buf, int len)
 {
 	struct doc_priv *doc = nand_get_controller_data(this);
@@ -499,20 +450,6 @@ static void doc2001_readbuf(struct nand_chip *this, u_char *buf, int len)
 	buf[i] = ReadDOC(docptr, LastDataRead);
 }
 
-static u_char doc2001plus_read_byte(struct nand_chip *this)
-{
-	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
-	u_char ret;
-
-	ReadDOC(docptr, Mplus_ReadPipeInit);
-	ReadDOC(docptr, Mplus_ReadPipeInit);
-	ret = ReadDOC(docptr, Mplus_LastDataRead);
-	if (debug)
-		printk("read_byte returns %02x\n", ret);
-	return ret;
-}
-
 static void doc2001plus_writebuf(struct nand_chip *this, const u_char *buf, int len)
 {
 	struct doc_priv *doc = nand_get_controller_data(this);
@@ -550,9 +487,12 @@ static void doc2001plus_readbuf(struct nand_chip *this, u_char *buf, int len)
 	}
 
 	/* Terminate read pipeline */
-	buf[len - 2] = ReadDOC(docptr, Mplus_LastDataRead);
-	if (debug && i < 16)
-		printk("%02x ", buf[len - 2]);
+	if (len >= 2) {
+		buf[len - 2] = ReadDOC(docptr, Mplus_LastDataRead);
+		if (debug && i < 16)
+			printk("%02x ", buf[len - 2]);
+	}
+
 	buf[len - 1] = ReadDOC(docptr, Mplus_LastDataRead);
 	if (debug && i < 16)
 		printk("%02x ", buf[len - 1]);
@@ -560,226 +500,163 @@ static void doc2001plus_readbuf(struct nand_chip *this, u_char *buf, int len)
 		printk("\n");
 }
 
-static void doc2001plus_select_chip(struct nand_chip *this, int chip)
+static void doc200x_write_control(struct doc_priv *doc, u8 value)
+{
+	WriteDOC(value, doc->virtadr, CDSNControl);
+	/* 11.4.3 -- 4 NOPs after CSDNControl write */
+	DoC_Delay(doc, 4);
+}
+
+static void doc200x_exec_instr(struct nand_chip *this,
+			       const struct nand_op_instr *instr)
 {
 	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
-	int floor = 0;
+	unsigned int i;
 
-	if (debug)
-		printk("select chip (%d)\n", chip);
+	switch (instr->type) {
+	case NAND_OP_CMD_INSTR:
+		doc200x_write_control(doc, CDSN_CTRL_CE | CDSN_CTRL_CLE);
+		doc2000_write_byte(this, instr->ctx.cmd.opcode);
+		break;
 
-	if (chip == -1) {
-		/* Disable flash internally */
-		WriteDOC(0, docptr, Mplus_FlashSelect);
-		return;
-	}
+	case NAND_OP_ADDR_INSTR:
+		doc200x_write_control(doc, CDSN_CTRL_CE | CDSN_CTRL_ALE);
+		for (i = 0; i < instr->ctx.addr.naddrs; i++) {
+			u8 addr = instr->ctx.addr.addrs[i];
 
-	floor = chip / doc->chips_per_floor;
-	chip -= (floor * doc->chips_per_floor);
+			if (DoC_is_2000(doc))
+				doc2000_write_byte(this, addr);
+			else
+				doc2001_write_byte(this, addr);
+		}
+		break;
 
-	/* Assert ChipEnable and deassert WriteProtect */
-	WriteDOC((DOC_FLASH_CE), docptr, Mplus_FlashSelect);
-	nand_reset_op(this);
+	case NAND_OP_DATA_IN_INSTR:
+		doc200x_write_control(doc, CDSN_CTRL_CE);
+		if (DoC_is_2000(doc))
+			doc2000_readbuf(this, instr->ctx.data.buf.in,
+					instr->ctx.data.len);
+		else
+			doc2001_readbuf(this, instr->ctx.data.buf.in,
+					instr->ctx.data.len);
+		break;
 
-	doc->curchip = chip;
-	doc->curfloor = floor;
+	case NAND_OP_DATA_OUT_INSTR:
+		doc200x_write_control(doc, CDSN_CTRL_CE);
+		if (DoC_is_2000(doc))
+			doc2000_writebuf(this, instr->ctx.data.buf.out,
+					 instr->ctx.data.len);
+		else
+			doc2001_writebuf(this, instr->ctx.data.buf.out,
+					 instr->ctx.data.len);
+		break;
+
+	case NAND_OP_WAITRDY_INSTR:
+		DoC_WaitReady(doc);
+		break;
+	}
+
+	if (instr->delay_ns)
+		ndelay(instr->delay_ns);
 }
 
-static void doc200x_select_chip(struct nand_chip *this, int chip)
+static int doc200x_exec_op(struct nand_chip *this,
+			   const struct nand_operation *op,
+			   bool check_only)
 {
 	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
-	int floor = 0;
+	unsigned int i;
 
-	if (debug)
-		printk("select chip (%d)\n", chip);
+	if (check_only)
+		return true;
 
-	if (chip == -1)
-		return;
+	doc->curchip = op->cs % doc->chips_per_floor;
+	doc->curfloor = op->cs / doc->chips_per_floor;
 
-	floor = chip / doc->chips_per_floor;
-	chip -= (floor * doc->chips_per_floor);
+	WriteDOC(doc->curfloor, doc->virtadr, FloorSelect);
+	WriteDOC(doc->curchip, doc->virtadr, CDSNDeviceSelect);
 
-	/* 11.4.4 -- deassert CE before changing chip */
-	doc200x_hwcontrol(this, NAND_CMD_NONE, 0 | NAND_CTRL_CHANGE);
+	/* Assert CE pin */
+	doc200x_write_control(doc, CDSN_CTRL_CE);
 
-	WriteDOC(floor, docptr, FloorSelect);
-	WriteDOC(chip, docptr, CDSNDeviceSelect);
+	for (i = 0; i < op->ninstrs; i++)
+		doc200x_exec_instr(this, &op->instrs[i]);
 
-	doc200x_hwcontrol(this, NAND_CMD_NONE, NAND_NCE | NAND_CTRL_CHANGE);
+	/* De-assert CE pin */
+	doc200x_write_control(doc, 0);
 
-	doc->curchip = chip;
-	doc->curfloor = floor;
+	return 0;
 }
 
-#define CDSN_CTRL_MSK (CDSN_CTRL_CE | CDSN_CTRL_CLE | CDSN_CTRL_ALE)
-
-static void doc200x_hwcontrol(struct nand_chip *this, int cmd,
-			      unsigned int ctrl)
+static void doc2001plus_write_pipe_term(struct doc_priv *doc)
 {
-	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
-
-	if (ctrl & NAND_CTRL_CHANGE) {
-		doc->CDSNControl &= ~CDSN_CTRL_MSK;
-		doc->CDSNControl |= ctrl & CDSN_CTRL_MSK;
-		if (debug)
-			printk("hwcontrol(%d): %02x\n", cmd, doc->CDSNControl);
-		WriteDOC(doc->CDSNControl, docptr, CDSNControl);
-		/* 11.4.3 -- 4 NOPs after CSDNControl write */
-		DoC_Delay(doc, 4);
-	}
-	if (cmd != NAND_CMD_NONE) {
-		if (DoC_is_2000(doc))
-			doc2000_write_byte(this, cmd);
-		else
-			doc2001_write_byte(this, cmd);
-	}
+	WriteDOC(0x00, doc->virtadr, Mplus_WritePipeTerm);
+	WriteDOC(0x00, doc->virtadr, Mplus_WritePipeTerm);
 }
 
-static void doc2001plus_command(struct nand_chip *this, unsigned command,
-				int column, int page_addr)
+static void doc2001plus_exec_instr(struct nand_chip *this,
+				   const struct nand_op_instr *instr)
 {
-	struct mtd_info *mtd = nand_to_mtd(this);
 	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
+	unsigned int i;
 
-	/*
-	 * Must terminate write pipeline before sending any commands
-	 * to the device.
-	 */
-	if (command == NAND_CMD_PAGEPROG) {
-		WriteDOC(0x00, docptr, Mplus_WritePipeTerm);
-		WriteDOC(0x00, docptr, Mplus_WritePipeTerm);
-	}
+	switch (instr->type) {
+	case NAND_OP_CMD_INSTR:
+		WriteDOC(instr->ctx.cmd.opcode, doc->virtadr, Mplus_FlashCmd);
+		doc2001plus_write_pipe_term(doc);
+		break;
 
-	/*
-	 * Write out the command to the device.
-	 */
-	if (command == NAND_CMD_SEQIN) {
-		int readcmd;
-
-		if (column >= mtd->writesize) {
-			/* OOB area */
-			column -= mtd->writesize;
-			readcmd = NAND_CMD_READOOB;
-		} else if (column < 256) {
-			/* First 256 bytes --> READ0 */
-			readcmd = NAND_CMD_READ0;
-		} else {
-			column -= 256;
-			readcmd = NAND_CMD_READ1;
-		}
-		WriteDOC(readcmd, docptr, Mplus_FlashCmd);
-	}
-	WriteDOC(command, docptr, Mplus_FlashCmd);
-	WriteDOC(0, docptr, Mplus_WritePipeTerm);
-	WriteDOC(0, docptr, Mplus_WritePipeTerm);
-
-	if (column != -1 || page_addr != -1) {
-		/* Serially input address */
-		if (column != -1) {
-			/* Adjust columns for 16 bit buswidth */
-			if (this->options & NAND_BUSWIDTH_16 &&
-					!nand_opcode_8bits(command))
-				column >>= 1;
-			WriteDOC(column, docptr, Mplus_FlashAddress);
-		}
-		if (page_addr != -1) {
-			WriteDOC((unsigned char)(page_addr & 0xff), docptr, Mplus_FlashAddress);
-			WriteDOC((unsigned char)((page_addr >> 8) & 0xff), docptr, Mplus_FlashAddress);
-			if (this->options & NAND_ROW_ADDR_3) {
-				WriteDOC((unsigned char)((page_addr >> 16) & 0x0f), docptr, Mplus_FlashAddress);
-				printk("high density\n");
-			}
+	case NAND_OP_ADDR_INSTR:
+		for (i = 0; i < instr->ctx.addr.naddrs; i++) {
+			u8 addr = instr->ctx.addr.addrs[i];
+
+			WriteDOC(addr, doc->virtadr, Mplus_FlashAddress);
 		}
-		WriteDOC(0, docptr, Mplus_WritePipeTerm);
-		WriteDOC(0, docptr, Mplus_WritePipeTerm);
+		doc2001plus_write_pipe_term(doc);
 		/* deassert ALE */
-		if (command == NAND_CMD_READ0 || command == NAND_CMD_READ1 ||
-		    command == NAND_CMD_READOOB || command == NAND_CMD_READID)
-			WriteDOC(0, docptr, Mplus_FlashControl);
-	}
-
-	/*
-	 * program and erase have their own busy handlers
-	 * status and sequential in needs no delay
-	 */
-	switch (command) {
-
-	case NAND_CMD_PAGEPROG:
-	case NAND_CMD_ERASE1:
-	case NAND_CMD_ERASE2:
-	case NAND_CMD_SEQIN:
-	case NAND_CMD_STATUS:
-		return;
+		WriteDOC(0, doc->virtadr, Mplus_FlashControl);
+		break;
 
-	case NAND_CMD_RESET:
-		if (this->legacy.dev_ready)
-			break;
-		udelay(this->legacy.chip_delay);
-		WriteDOC(NAND_CMD_STATUS, docptr, Mplus_FlashCmd);
-		WriteDOC(0, docptr, Mplus_WritePipeTerm);
-		WriteDOC(0, docptr, Mplus_WritePipeTerm);
-		while (!(this->legacy.read_byte(this) & 0x40)) ;
-		return;
-
-		/* This applies to read commands */
-	default:
-		/*
-		 * If we don't have access to the busy pin, we apply the given
-		 * command delay
-		 */
-		if (!this->legacy.dev_ready) {
-			udelay(this->legacy.chip_delay);
-			return;
-		}
+	case NAND_OP_DATA_IN_INSTR:
+		doc2001plus_readbuf(this, instr->ctx.data.buf.in,
+				    instr->ctx.data.len);
+		break;
+	case NAND_OP_DATA_OUT_INSTR:
+		doc2001plus_writebuf(this, instr->ctx.data.buf.out,
+				     instr->ctx.data.len);
+		doc2001plus_write_pipe_term(doc);
+		break;
+	case NAND_OP_WAITRDY_INSTR:
+		DoC_WaitReady(doc);
+		break;
 	}
 
-	/* Apply this short delay always to ensure that we do wait tWB in
-	 * any case on any machine. */
-	ndelay(100);
-	/* wait until command is processed */
-	while (!this->legacy.dev_ready(this)) ;
+	if (instr->delay_ns)
+		ndelay(instr->delay_ns);
 }
 
-static int doc200x_dev_ready(struct nand_chip *this)
+static int doc2001plus_exec_op(struct nand_chip *this,
+			       const struct nand_operation *op,
+			       bool check_only)
 {
 	struct doc_priv *doc = nand_get_controller_data(this);
-	void __iomem *docptr = doc->virtadr;
+	unsigned int i;
 
-	if (DoC_is_MillenniumPlus(doc)) {
-		/* 11.4.2 -- must NOP four times before checking FR/B# */
-		DoC_Delay(doc, 4);
-		if ((ReadDOC(docptr, Mplus_FlashControl) & CDSN_CTRL_FR_B_MASK) != CDSN_CTRL_FR_B_MASK) {
-			if (debug)
-				printk("not ready\n");
-			return 0;
-		}
-		if (debug)
-			printk("was ready\n");
-		return 1;
-	} else {
-		/* 11.4.2 -- must NOP four times before checking FR/B# */
-		DoC_Delay(doc, 4);
-		if (!(ReadDOC(docptr, CDSNControl) & CDSN_CTRL_FR_B)) {
-			if (debug)
-				printk("not ready\n");
-			return 0;
-		}
-		/* 11.4.2 -- Must NOP twice if it's ready */
-		DoC_Delay(doc, 2);
-		if (debug)
-			printk("was ready\n");
-		return 1;
-	}
-}
+	if (check_only)
+		return true;
+
+	doc->curchip = op->cs % doc->chips_per_floor;
+	doc->curfloor = op->cs / doc->chips_per_floor;
+
+	/* Assert ChipEnable and deassert WriteProtect */
+	WriteDOC(DOC_FLASH_CE, doc->virtadr, Mplus_FlashSelect);
+
+	for (i = 0; i < op->ninstrs; i++)
+		doc2001plus_exec_instr(this, &op->instrs[i]);
+
+	/* De-assert ChipEnable */
+	WriteDOC(0, doc->virtadr, Mplus_FlashSelect);
 
-static int doc200x_block_bad(struct nand_chip *this, loff_t ofs)
-{
-	/* This is our last resort if we couldn't find or create a BBT.  Just
-	   pretend all blocks are good. */
 	return 0;
 }
 
@@ -1344,9 +1221,6 @@ static inline int __init doc2000_init(struct mtd_info *mtd)
 	struct nand_chip *this = mtd_to_nand(mtd);
 	struct doc_priv *doc = nand_get_controller_data(this);
 
-	this->legacy.read_byte = doc2000_read_byte;
-	this->legacy.write_buf = doc2000_writebuf;
-	this->legacy.read_buf = doc2000_readbuf;
 	doc->late_init = nftl_scan_bbt;
 
 	doc->CDSNControl = CDSN_CTRL_FLASH_IO | CDSN_CTRL_ECC_IO;
@@ -1360,10 +1234,6 @@ static inline int __init doc2001_init(struct mtd_info *mtd)
 	struct nand_chip *this = mtd_to_nand(mtd);
 	struct doc_priv *doc = nand_get_controller_data(this);
 
-	this->legacy.read_byte = doc2001_read_byte;
-	this->legacy.write_buf = doc2001_writebuf;
-	this->legacy.read_buf = doc2001_readbuf;
-
 	ReadDOC(doc->virtadr, ChipID);
 	ReadDOC(doc->virtadr, ChipID);
 	ReadDOC(doc->virtadr, ChipID);
@@ -1390,13 +1260,7 @@ static inline int __init doc2001plus_init(struct mtd_info *mtd)
 	struct nand_chip *this = mtd_to_nand(mtd);
 	struct doc_priv *doc = nand_get_controller_data(this);
 
-	this->legacy.read_byte = doc2001plus_read_byte;
-	this->legacy.write_buf = doc2001plus_writebuf;
-	this->legacy.read_buf = doc2001plus_readbuf;
 	doc->late_init = inftl_scan_bbt;
-	this->legacy.cmd_ctrl = NULL;
-	this->legacy.select_chip = doc2001plus_select_chip;
-	this->legacy.cmdfunc = doc2001plus_command;
 	this->ecc.hwctl = doc2001plus_enable_hwecc;
 
 	doc->chips_per_floor = 1;
@@ -1405,6 +1269,14 @@ static inline int __init doc2001plus_init(struct mtd_info *mtd)
 	return 1;
 }
 
+static const struct nand_controller_ops doc200x_ops = {
+	.exec_op = doc200x_exec_op,
+};
+
+static const struct nand_controller_ops doc2001plus_ops = {
+	.exec_op = doc2001plus_exec_op,
+};
+
 static int __init doc_probe(unsigned long physadr)
 {
 	struct nand_chip *nand = NULL;
@@ -1548,7 +1420,6 @@ static int __init doc_probe(unsigned long physadr)
 		goto fail;
 	}
 
-
 	/*
 	 * Allocate a RS codec instance
 	 *
@@ -1566,6 +1437,12 @@ static int __init doc_probe(unsigned long physadr)
 		goto fail;
 	}
 
+	nand_controller_init(&doc->base);
+	if (ChipID == DOC_ChipID_DocMilPlus16)
+		doc->base.ops = &doc2001plus_ops;
+	else
+		doc->base.ops = &doc200x_ops;
+
 	mtd			= nand_to_mtd(nand);
 	nand->bbt_td		= (struct nand_bbt_descr *) (doc + 1);
 	nand->bbt_md		= nand->bbt_td + 1;
@@ -1573,12 +1450,8 @@ static int __init doc_probe(unsigned long physadr)
 	mtd->owner		= THIS_MODULE;
 	mtd_set_ooblayout(mtd, &doc200x_ooblayout_ops);
 
+	nand->controller	= &doc->base;
 	nand_set_controller_data(nand, doc);
-	nand->legacy.select_chip	= doc200x_select_chip;
-	nand->legacy.cmd_ctrl		= doc200x_hwcontrol;
-	nand->legacy.dev_ready	= doc200x_dev_ready;
-	nand->legacy.waitfunc	= doc200x_wait;
-	nand->legacy.block_bad	= doc200x_block_bad;
 	nand->ecc.hwctl		= doc200x_enable_hwecc;
 	nand->ecc.calculate	= doc200x_calculate_ecc;
 	nand->ecc.correct	= doc200x_correct_data;
@@ -1590,7 +1463,7 @@ static int __init doc_probe(unsigned long physadr)
 	nand->ecc.options	= NAND_ECC_GENERIC_ERASED_CHECK;
 	nand->bbt_options	= NAND_BBT_USE_FLASH;
 	/* Skip the automatic BBT scan so we can run it manually */
-	nand->options		|= NAND_SKIP_BBTSCAN;
+	nand->options		|= NAND_SKIP_BBTSCAN | NAND_NO_BBM_QUIRK;
 
 	doc->physadr		= physadr;
 	doc->virtadr		= virtadr;
@@ -1609,13 +1482,10 @@ static int __init doc_probe(unsigned long physadr)
 		numchips = doc2001_init(mtd);
 
 	if ((ret = nand_scan(nand, numchips)) || (ret = doc->late_init(mtd))) {
-		/* DBB note: i believe nand_release is necessary here, as
+		/* DBB note: i believe nand_cleanup is necessary here, as
 		   buffers may have been allocated in nand_base.  Check with
 		   Thomas. FIX ME! */
-		/* nand_release will call mtd_device_unregister, but we
-		   haven't yet added it.  This is handled without incident by
-		   mtd_device_unregister, as far as I can tell. */
-		nand_release(nand);
+		nand_cleanup(nand);
 		goto fail;
 	}
 
@@ -1644,13 +1514,16 @@ static void release_nanddoc(void)
 	struct mtd_info *mtd, *nextmtd;
 	struct nand_chip *nand;
 	struct doc_priv *doc;
+	int ret;
 
 	for (mtd = doclist; mtd; mtd = nextmtd) {
 		nand = mtd_to_nand(mtd);
 		doc = nand_get_controller_data(nand);
 
 		nextmtd = doc->nextdoc;
-		nand_release(nand);
+		ret = mtd_device_unregister(mtd);
+		WARN_ON(ret);
+		nand_cleanup(nand);
 		iounmap(doc->virtadr);
 		release_mem_region(doc->physadr, DOC_IOREMAP_LEN);
 		free_rs(doc->rs_decoder);
diff --git a/drivers/mtd/nand/raw/fsl_elbc_nand.c b/drivers/mtd/nand/raw/fsl_elbc_nand.c
index e1dc675b12bb..088692b2e27a 100644
--- a/drivers/mtd/nand/raw/fsl_elbc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_elbc_nand.c
@@ -956,8 +956,13 @@ static int fsl_elbc_nand_remove(struct platform_device *pdev)
 {
 	struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = fsl_lbc_ctrl_dev->nand;
 	struct fsl_elbc_mtd *priv = dev_get_drvdata(&pdev->dev);
+	struct nand_chip *chip = &priv->chip;
+	int ret;
+
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
-	nand_release(&priv->chip);
 	fsl_elbc_chip_remove(priv);
 
 	mutex_lock(&fsl_elbc_nand_mutex);
diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c
index 2af09edf405b..00ae7a910b03 100644
--- a/drivers/mtd/nand/raw/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c
@@ -1093,8 +1093,13 @@ err:
 static int fsl_ifc_nand_remove(struct platform_device *dev)
 {
 	struct fsl_ifc_mtd *priv = dev_get_drvdata(&dev->dev);
+	struct nand_chip *chip = &priv->chip;
+	int ret;
+
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
-	nand_release(&priv->chip);
 	fsl_ifc_chip_remove(priv);
 
 	mutex_lock(&fsl_ifc_nand_mutex);
diff --git a/drivers/mtd/nand/raw/fsl_upm.c b/drivers/mtd/nand/raw/fsl_upm.c
index f31fae3a4c68..627deb26db51 100644
--- a/drivers/mtd/nand/raw/fsl_upm.c
+++ b/drivers/mtd/nand/raw/fsl_upm.c
@@ -317,10 +317,13 @@ err1:
 static int fun_remove(struct platform_device *ofdev)
 {
 	struct fsl_upm_nand *fun = dev_get_drvdata(&ofdev->dev);
-	struct mtd_info *mtd = nand_to_mtd(&fun->chip);
-	int i;
+	struct nand_chip *chip = &fun->chip;
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int ret, i;
 
-	nand_release(&fun->chip);
+	ret = mtd_device_unregister(mtd);
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	kfree(mtd->name);
 
 	for (i = 0; i < fun->mchip_count; i++) {
diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c
index a6964feeec77..3909752b14c5 100644
--- a/drivers/mtd/nand/raw/fsmc_nand.c
+++ b/drivers/mtd/nand/raw/fsmc_nand.c
@@ -608,6 +608,9 @@ static int fsmc_exec_op(struct nand_chip *chip, const struct nand_operation *op,
 	unsigned int op_id;
 	int i;
 
+	if (check_only)
+		return 0;
+
 	pr_debug("Executing operation [%d instructions]:\n", op->ninstrs);
 
 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
@@ -691,7 +694,7 @@ static int fsmc_read_page_hwecc(struct nand_chip *chip, u8 *buf,
 	for (i = 0, s = 0; s < eccsteps; s++, i += eccbytes, p += eccsize) {
 		nand_read_page_op(chip, page, s * eccsize, NULL, 0);
 		chip->ecc.hwctl(chip, NAND_ECC_READ);
-		ret = nand_read_data_op(chip, p, eccsize, false);
+		ret = nand_read_data_op(chip, p, eccsize, false, false);
 		if (ret)
 			return ret;
 
@@ -809,11 +812,12 @@ static int fsmc_bch8_correct_data(struct nand_chip *chip, u8 *dat,
 
 	i = 0;
 	while (num_err--) {
-		change_bit(0, (unsigned long *)&err_idx[i]);
-		change_bit(1, (unsigned long *)&err_idx[i]);
+		err_idx[i] ^= 3;
 
 		if (err_idx[i] < chip->ecc.size * 8) {
-			change_bit(err_idx[i], (unsigned long *)dat);
+			int err = err_idx[i];
+
+			dat[err >> 3] ^= BIT(err & 7);
 			i++;
 		}
 	}
@@ -1132,7 +1136,12 @@ static int fsmc_nand_remove(struct platform_device *pdev)
 	struct fsmc_nand_data *host = platform_get_drvdata(pdev);
 
 	if (host) {
-		nand_release(&host->nand);
+		struct nand_chip *chip = &host->nand;
+		int ret;
+
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
 		fsmc_nand_disable(host);
 
 		if (host->mode == USE_DMA_ACCESS) {
diff --git a/drivers/mtd/nand/raw/gpio.c b/drivers/mtd/nand/raw/gpio.c
index f6b12354024f..938077e5c6a9 100644
--- a/drivers/mtd/nand/raw/gpio.c
+++ b/drivers/mtd/nand/raw/gpio.c
@@ -190,8 +190,12 @@ gpio_nand_get_io_sync(struct platform_device *pdev)
 static int gpio_nand_remove(struct platform_device *pdev)
 {
 	struct gpiomtd *gpiomtd = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &gpiomtd->nand_chip;
+	int ret;
 
-	nand_release(&gpiomtd->nand_chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
 	/* Enable write protection and disable the chip */
 	if (gpiomtd->nwp && !IS_ERR(gpiomtd->nwp))
diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
index 53b00c841aec..061a8ddda275 100644
--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
@@ -540,8 +540,10 @@ static int bch_set_geometry(struct gpmi_nand_data *this)
 		return ret;
 
 	ret = pm_runtime_get_sync(this->dev);
-	if (ret < 0)
+	if (ret < 0) {
+		pm_runtime_put_autosuspend(this->dev);
 		return ret;
+	}
 
 	/*
 	* Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this
@@ -834,158 +836,6 @@ map_fail:
 	return false;
 }
 
-/**
- * gpmi_copy_bits - copy bits from one memory region to another
- * @dst: destination buffer
- * @dst_bit_off: bit offset we're starting to write at
- * @src: source buffer
- * @src_bit_off: bit offset we're starting to read from
- * @nbits: number of bits to copy
- *
- * This functions copies bits from one memory region to another, and is used by
- * the GPMI driver to copy ECC sections which are not guaranteed to be byte
- * aligned.
- *
- * src and dst should not overlap.
- *
- */
-static void gpmi_copy_bits(u8 *dst, size_t dst_bit_off, const u8 *src,
-			   size_t src_bit_off, size_t nbits)
-{
-	size_t i;
-	size_t nbytes;
-	u32 src_buffer = 0;
-	size_t bits_in_src_buffer = 0;
-
-	if (!nbits)
-		return;
-
-	/*
-	 * Move src and dst pointers to the closest byte pointer and store bit
-	 * offsets within a byte.
-	 */
-	src += src_bit_off / 8;
-	src_bit_off %= 8;
-
-	dst += dst_bit_off / 8;
-	dst_bit_off %= 8;
-
-	/*
-	 * Initialize the src_buffer value with bits available in the first
-	 * byte of data so that we end up with a byte aligned src pointer.
-	 */
-	if (src_bit_off) {
-		src_buffer = src[0] >> src_bit_off;
-		if (nbits >= (8 - src_bit_off)) {
-			bits_in_src_buffer += 8 - src_bit_off;
-		} else {
-			src_buffer &= GENMASK(nbits - 1, 0);
-			bits_in_src_buffer += nbits;
-		}
-		nbits -= bits_in_src_buffer;
-		src++;
-	}
-
-	/* Calculate the number of bytes that can be copied from src to dst. */
-	nbytes = nbits / 8;
-
-	/* Try to align dst to a byte boundary. */
-	if (dst_bit_off) {
-		if (bits_in_src_buffer < (8 - dst_bit_off) && nbytes) {
-			src_buffer |= src[0] << bits_in_src_buffer;
-			bits_in_src_buffer += 8;
-			src++;
-			nbytes--;
-		}
-
-		if (bits_in_src_buffer >= (8 - dst_bit_off)) {
-			dst[0] &= GENMASK(dst_bit_off - 1, 0);
-			dst[0] |= src_buffer << dst_bit_off;
-			src_buffer >>= (8 - dst_bit_off);
-			bits_in_src_buffer -= (8 - dst_bit_off);
-			dst_bit_off = 0;
-			dst++;
-			if (bits_in_src_buffer > 7) {
-				bits_in_src_buffer -= 8;
-				dst[0] = src_buffer;
-				dst++;
-				src_buffer >>= 8;
-			}
-		}
-	}
-
-	if (!bits_in_src_buffer && !dst_bit_off) {
-		/*
-		 * Both src and dst pointers are byte aligned, thus we can
-		 * just use the optimized memcpy function.
-		 */
-		if (nbytes)
-			memcpy(dst, src, nbytes);
-	} else {
-		/*
-		 * src buffer is not byte aligned, hence we have to copy each
-		 * src byte to the src_buffer variable before extracting a byte
-		 * to store in dst.
-		 */
-		for (i = 0; i < nbytes; i++) {
-			src_buffer |= src[i] << bits_in_src_buffer;
-			dst[i] = src_buffer;
-			src_buffer >>= 8;
-		}
-	}
-	/* Update dst and src pointers */
-	dst += nbytes;
-	src += nbytes;
-
-	/*
-	 * nbits is the number of remaining bits. It should not exceed 8 as
-	 * we've already copied as much bytes as possible.
-	 */
-	nbits %= 8;
-
-	/*
-	 * If there's no more bits to copy to the destination and src buffer
-	 * was already byte aligned, then we're done.
-	 */
-	if (!nbits && !bits_in_src_buffer)
-		return;
-
-	/* Copy the remaining bits to src_buffer */
-	if (nbits)
-		src_buffer |= (*src & GENMASK(nbits - 1, 0)) <<
-			      bits_in_src_buffer;
-	bits_in_src_buffer += nbits;
-
-	/*
-	 * In case there were not enough bits to get a byte aligned dst buffer
-	 * prepare the src_buffer variable to match the dst organization (shift
-	 * src_buffer by dst_bit_off and retrieve the least significant bits
-	 * from dst).
-	 */
-	if (dst_bit_off)
-		src_buffer = (src_buffer << dst_bit_off) |
-			     (*dst & GENMASK(dst_bit_off - 1, 0));
-	bits_in_src_buffer += dst_bit_off;
-
-	/*
-	 * Keep most significant bits from dst if we end up with an unaligned
-	 * number of bits.
-	 */
-	nbytes = bits_in_src_buffer / 8;
-	if (bits_in_src_buffer % 8) {
-		src_buffer |= (dst[nbytes] &
-			       GENMASK(7, bits_in_src_buffer % 8)) <<
-			      (nbytes * 8);
-		nbytes++;
-	}
-
-	/* Copy the remaining bytes to dst */
-	for (i = 0; i < nbytes; i++) {
-		dst[i] = src_buffer;
-		src_buffer >>= 8;
-	}
-}
-
 /* add our owner bbt descriptor */
 static uint8_t scan_ff_pattern[] = { 0xff };
 static struct nand_bbt_descr gpmi_bbt_descr = {
@@ -1713,7 +1563,7 @@ static int gpmi_ecc_write_oob(struct nand_chip *chip, int page)
  * inline (interleaved with payload DATA), and do not align data chunk on
  * byte boundaries.
  * We thus need to take care moving the payload data and ECC bits stored in the
- * page into the provided buffers, which is why we're using gpmi_copy_bits.
+ * page into the provided buffers, which is why we're using nand_extract_bits().
  *
  * See set_geometry_by_ecc_info inline comments to have a full description
  * of the layout used by the GPMI controller.
@@ -1762,9 +1612,8 @@ static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
 	/* Extract interleaved payload data and ECC bits */
 	for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
 		if (buf)
-			gpmi_copy_bits(buf, step * eccsize * 8,
-				       tmp_buf, src_bit_off,
-				       eccsize * 8);
+			nand_extract_bits(buf, step * eccsize, tmp_buf,
+					  src_bit_off, eccsize * 8);
 		src_bit_off += eccsize * 8;
 
 		/* Align last ECC block to align a byte boundary */
@@ -1773,9 +1622,8 @@ static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
 			eccbits += 8 - ((oob_bit_off + eccbits) % 8);
 
 		if (oob_required)
-			gpmi_copy_bits(oob, oob_bit_off,
-				       tmp_buf, src_bit_off,
-				       eccbits);
+			nand_extract_bits(oob, oob_bit_off, tmp_buf,
+					  src_bit_off, eccbits);
 
 		src_bit_off += eccbits;
 		oob_bit_off += eccbits;
@@ -1800,7 +1648,7 @@ static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
  * inline (interleaved with payload DATA), and do not align data chunk on
  * byte boundaries.
  * We thus need to take care moving the OOB area at the right place in the
- * final page, which is why we're using gpmi_copy_bits.
+ * final page, which is why we're using nand_extract_bits().
  *
  * See set_geometry_by_ecc_info inline comments to have a full description
  * of the layout used by the GPMI controller.
@@ -1839,8 +1687,8 @@ static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
 	/* Interleave payload data and ECC bits */
 	for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
 		if (buf)
-			gpmi_copy_bits(tmp_buf, dst_bit_off,
-				       buf, step * eccsize * 8, eccsize * 8);
+			nand_extract_bits(tmp_buf, dst_bit_off, buf,
+					  step * eccsize * 8, eccsize * 8);
 		dst_bit_off += eccsize * 8;
 
 		/* Align last ECC block to align a byte boundary */
@@ -1849,8 +1697,8 @@ static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
 			eccbits += 8 - ((oob_bit_off + eccbits) % 8);
 
 		if (oob_required)
-			gpmi_copy_bits(tmp_buf, dst_bit_off,
-				       oob, oob_bit_off, eccbits);
+			nand_extract_bits(tmp_buf, dst_bit_off, oob,
+					  oob_bit_off, eccbits);
 
 		dst_bit_off += eccbits;
 		oob_bit_off += eccbits;
@@ -2408,6 +2256,9 @@ static int gpmi_nfc_exec_op(struct nand_chip *chip,
 	struct completion *completion;
 	unsigned long to;
 
+	if (check_only)
+		return 0;
+
 	this->ntransfers = 0;
 	for (i = 0; i < GPMI_MAX_TRANSFERS; i++)
 		this->transfers[i].direction = DMA_NONE;
@@ -2658,7 +2509,7 @@ static int gpmi_nand_probe(struct platform_device *pdev)
 
 	ret = __gpmi_enable_clk(this, true);
 	if (ret)
-		goto exit_nfc_init;
+		goto exit_acquire_resources;
 
 	pm_runtime_set_autosuspend_delay(&pdev->dev, 500);
 	pm_runtime_use_autosuspend(&pdev->dev);
@@ -2693,11 +2544,15 @@ exit_acquire_resources:
 static int gpmi_nand_remove(struct platform_device *pdev)
 {
 	struct gpmi_nand_data *this = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &this->nand;
+	int ret;
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
-	nand_release(&this->nand);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	gpmi_free_dma_buffer(this);
 	release_resources(this);
 	return 0;
diff --git a/drivers/mtd/nand/raw/hisi504_nand.c b/drivers/mtd/nand/raw/hisi504_nand.c
index 0b48be54ba6f..b84238e2268a 100644
--- a/drivers/mtd/nand/raw/hisi504_nand.c
+++ b/drivers/mtd/nand/raw/hisi504_nand.c
@@ -806,8 +806,12 @@ static int hisi_nfc_probe(struct platform_device *pdev)
 static int hisi_nfc_remove(struct platform_device *pdev)
 {
 	struct hinfc_host *host = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &host->chip;
+	int ret;
 
-	nand_release(&host->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
index 935c4902ada7..69423bb29adb 100644
--- a/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
+++ b/drivers/mtd/nand/raw/ingenic/ingenic_nand_drv.c
@@ -27,9 +27,6 @@
 
 #define DRV_NAME	"ingenic-nand"
 
-/* Command delay when there is no R/B pin. */
-#define RB_DELAY_US	100
-
 struct jz_soc_info {
 	unsigned long data_offset;
 	unsigned long addr_offset;
@@ -49,7 +46,6 @@ struct ingenic_nfc {
 	struct nand_controller controller;
 	unsigned int num_banks;
 	struct list_head chips;
-	int selected;
 	struct ingenic_nand_cs cs[];
 };
 
@@ -102,7 +98,7 @@ static int qi_lb60_ooblayout_free(struct mtd_info *mtd, int section,
 	return 0;
 }
 
-const struct mtd_ooblayout_ops qi_lb60_ooblayout_ops = {
+static const struct mtd_ooblayout_ops qi_lb60_ooblayout_ops = {
 	.ecc = qi_lb60_ooblayout_ecc,
 	.free = qi_lb60_ooblayout_free,
 };
@@ -142,51 +138,6 @@ static const struct mtd_ooblayout_ops jz4725b_ooblayout_ops = {
 	.free = jz4725b_ooblayout_free,
 };
 
-static void ingenic_nand_select_chip(struct nand_chip *chip, int chipnr)
-{
-	struct ingenic_nand *nand = to_ingenic_nand(nand_to_mtd(chip));
-	struct ingenic_nfc *nfc = to_ingenic_nfc(nand->chip.controller);
-	struct ingenic_nand_cs *cs;
-
-	/* Ensure the currently selected chip is deasserted. */
-	if (chipnr == -1 && nfc->selected >= 0) {
-		cs = &nfc->cs[nfc->selected];
-		jz4780_nemc_assert(nfc->dev, cs->bank, false);
-	}
-
-	nfc->selected = chipnr;
-}
-
-static void ingenic_nand_cmd_ctrl(struct nand_chip *chip, int cmd,
-				  unsigned int ctrl)
-{
-	struct ingenic_nand *nand = to_ingenic_nand(nand_to_mtd(chip));
-	struct ingenic_nfc *nfc = to_ingenic_nfc(nand->chip.controller);
-	struct ingenic_nand_cs *cs;
-
-	if (WARN_ON(nfc->selected < 0))
-		return;
-
-	cs = &nfc->cs[nfc->selected];
-
-	jz4780_nemc_assert(nfc->dev, cs->bank, ctrl & NAND_NCE);
-
-	if (cmd == NAND_CMD_NONE)
-		return;
-
-	if (ctrl & NAND_ALE)
-		writeb(cmd, cs->base + nfc->soc_info->addr_offset);
-	else if (ctrl & NAND_CLE)
-		writeb(cmd, cs->base + nfc->soc_info->cmd_offset);
-}
-
-static int ingenic_nand_dev_ready(struct nand_chip *chip)
-{
-	struct ingenic_nand *nand = to_ingenic_nand(nand_to_mtd(chip));
-
-	return !gpiod_get_value_cansleep(nand->busy_gpio);
-}
-
 static void ingenic_nand_ecc_hwctl(struct nand_chip *chip, int mode)
 {
 	struct ingenic_nand *nand = to_ingenic_nand(nand_to_mtd(chip));
@@ -298,8 +249,91 @@ static int ingenic_nand_attach_chip(struct nand_chip *chip)
 	return 0;
 }
 
+static int ingenic_nand_exec_instr(struct nand_chip *chip,
+				   struct ingenic_nand_cs *cs,
+				   const struct nand_op_instr *instr)
+{
+	struct ingenic_nand *nand = to_ingenic_nand(nand_to_mtd(chip));
+	struct ingenic_nfc *nfc = to_ingenic_nfc(chip->controller);
+	unsigned int i;
+
+	switch (instr->type) {
+	case NAND_OP_CMD_INSTR:
+		writeb(instr->ctx.cmd.opcode,
+		       cs->base + nfc->soc_info->cmd_offset);
+		return 0;
+	case NAND_OP_ADDR_INSTR:
+		for (i = 0; i < instr->ctx.addr.naddrs; i++)
+			writeb(instr->ctx.addr.addrs[i],
+			       cs->base + nfc->soc_info->addr_offset);
+		return 0;
+	case NAND_OP_DATA_IN_INSTR:
+		if (instr->ctx.data.force_8bit ||
+		    !(chip->options & NAND_BUSWIDTH_16))
+			ioread8_rep(cs->base + nfc->soc_info->data_offset,
+				    instr->ctx.data.buf.in,
+				    instr->ctx.data.len);
+		else
+			ioread16_rep(cs->base + nfc->soc_info->data_offset,
+				     instr->ctx.data.buf.in,
+				     instr->ctx.data.len);
+		return 0;
+	case NAND_OP_DATA_OUT_INSTR:
+		if (instr->ctx.data.force_8bit ||
+		    !(chip->options & NAND_BUSWIDTH_16))
+			iowrite8_rep(cs->base + nfc->soc_info->data_offset,
+				     instr->ctx.data.buf.out,
+				     instr->ctx.data.len);
+		else
+			iowrite16_rep(cs->base + nfc->soc_info->data_offset,
+				      instr->ctx.data.buf.out,
+				      instr->ctx.data.len);
+		return 0;
+	case NAND_OP_WAITRDY_INSTR:
+		if (!nand->busy_gpio)
+			return nand_soft_waitrdy(chip,
+						 instr->ctx.waitrdy.timeout_ms);
+
+		return nand_gpio_waitrdy(chip, nand->busy_gpio,
+					 instr->ctx.waitrdy.timeout_ms);
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static int ingenic_nand_exec_op(struct nand_chip *chip,
+				const struct nand_operation *op,
+				bool check_only)
+{
+	struct ingenic_nand *nand = to_ingenic_nand(nand_to_mtd(chip));
+	struct ingenic_nfc *nfc = to_ingenic_nfc(nand->chip.controller);
+	struct ingenic_nand_cs *cs;
+	unsigned int i;
+	int ret = 0;
+
+	if (check_only)
+		return 0;
+
+	cs = &nfc->cs[op->cs];
+	jz4780_nemc_assert(nfc->dev, cs->bank, true);
+	for (i = 0; i < op->ninstrs; i++) {
+		ret = ingenic_nand_exec_instr(chip, cs, &op->instrs[i]);
+		if (ret)
+			break;
+
+		if (op->instrs[i].delay_ns)
+			ndelay(op->instrs[i].delay_ns);
+	}
+	jz4780_nemc_assert(nfc->dev, cs->bank, false);
+
+	return ret;
+}
+
 static const struct nand_controller_ops ingenic_nand_controller_ops = {
 	.attach_chip = ingenic_nand_attach_chip,
+	.exec_op = ingenic_nand_exec_op,
 };
 
 static int ingenic_nand_init_chip(struct platform_device *pdev,
@@ -339,10 +373,20 @@ static int ingenic_nand_init_chip(struct platform_device *pdev,
 		ret = PTR_ERR(nand->busy_gpio);
 		dev_err(dev, "failed to request busy GPIO: %d\n", ret);
 		return ret;
-	} else if (nand->busy_gpio) {
-		nand->chip.legacy.dev_ready = ingenic_nand_dev_ready;
 	}
 
+	/*
+	 * The rb-gpios semantics was undocumented and qi,lb60 (along with
+	 * the ingenic driver) got it wrong. The active state encodes the
+	 * NAND ready state, which is high level. Since there's no signal
+	 * inverter on this board, it should be active-high. Let's fix that
+	 * here for older DTs so we can re-use the generic nand_gpio_waitrdy()
+	 * helper, and be consistent with what other drivers do.
+	 */
+	if (of_machine_is_compatible("qi,lb60") &&
+	    gpiod_is_active_low(nand->busy_gpio))
+		gpiod_toggle_active_low(nand->busy_gpio);
+
 	nand->wp_gpio = devm_gpiod_get_optional(dev, "wp", GPIOD_OUT_LOW);
 
 	if (IS_ERR(nand->wp_gpio)) {
@@ -359,12 +403,7 @@ static int ingenic_nand_init_chip(struct platform_device *pdev,
 		return -ENOMEM;
 	mtd->dev.parent = dev;
 
-	chip->legacy.IO_ADDR_R = cs->base + nfc->soc_info->data_offset;
-	chip->legacy.IO_ADDR_W = cs->base + nfc->soc_info->data_offset;
-	chip->legacy.chip_delay = RB_DELAY_US;
 	chip->options = NAND_NO_SUBPAGE_WRITE;
-	chip->legacy.select_chip = ingenic_nand_select_chip;
-	chip->legacy.cmd_ctrl = ingenic_nand_cmd_ctrl;
 	chip->ecc.mode = NAND_ECC_HW;
 	chip->controller = &nfc->controller;
 	nand_set_flash_node(chip, np);
@@ -376,7 +415,7 @@ static int ingenic_nand_init_chip(struct platform_device *pdev,
 
 	ret = mtd_device_register(mtd, NULL, 0);
 	if (ret) {
-		nand_release(chip);
+		nand_cleanup(chip);
 		return ret;
 	}
 
@@ -387,13 +426,18 @@ static int ingenic_nand_init_chip(struct platform_device *pdev,
 
 static void ingenic_nand_cleanup_chips(struct ingenic_nfc *nfc)
 {
-	struct ingenic_nand *chip;
+	struct ingenic_nand *ingenic_chip;
+	struct nand_chip *chip;
+	int ret;
 
 	while (!list_empty(&nfc->chips)) {
-		chip = list_first_entry(&nfc->chips,
-					struct ingenic_nand, chip_list);
-		nand_release(&chip->chip);
-		list_del(&chip->chip_list);
+		ingenic_chip = list_first_entry(&nfc->chips,
+						struct ingenic_nand, chip_list);
+		chip = &ingenic_chip->chip;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+		list_del(&ingenic_chip->chip_list);
 	}
 }
 
diff --git a/drivers/mtd/nand/raw/internals.h b/drivers/mtd/nand/raw/internals.h
index 9d0caadf940e..03866b0aadea 100644
--- a/drivers/mtd/nand/raw/internals.h
+++ b/drivers/mtd/nand/raw/internals.h
@@ -75,6 +75,9 @@ extern const struct nand_manufacturer_ops micron_nand_manuf_ops;
 extern const struct nand_manufacturer_ops samsung_nand_manuf_ops;
 extern const struct nand_manufacturer_ops toshiba_nand_manuf_ops;
 
+/* MLC pairing schemes */
+extern const struct mtd_pairing_scheme dist3_pairing_scheme;
+
 /* Core functions */
 const struct nand_manufacturer *nand_get_manufacturer(u8 id);
 int nand_bbm_get_next_page(struct nand_chip *chip, int page);
@@ -106,6 +109,15 @@ static inline bool nand_has_exec_op(struct nand_chip *chip)
 	return true;
 }
 
+static inline int nand_check_op(struct nand_chip *chip,
+				const struct nand_operation *op)
+{
+	if (!nand_has_exec_op(chip))
+		return 0;
+
+	return chip->controller->ops->exec_op(chip, op, true);
+}
+
 static inline int nand_exec_op(struct nand_chip *chip,
 			       const struct nand_operation *op)
 {
diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c
index 241b58b83240..7521038af2ef 100644
--- a/drivers/mtd/nand/raw/lpc32xx_mlc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c
@@ -826,8 +826,13 @@ free_gpio:
 static int lpc32xx_nand_remove(struct platform_device *pdev)
 {
 	struct lpc32xx_nand_host *host = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &host->nand_chip;
+	int ret;
+
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
-	nand_release(&host->nand_chip);
 	free_irq(host->irq, host);
 	if (use_dma)
 		dma_release_channel(host->dma_chan);
diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c
index 163f976353f8..b151fd000815 100644
--- a/drivers/mtd/nand/raw/lpc32xx_slc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_slc.c
@@ -947,8 +947,12 @@ static int lpc32xx_nand_remove(struct platform_device *pdev)
 {
 	uint32_t tmp;
 	struct lpc32xx_nand_host *host = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &host->nand_chip;
+	int ret;
 
-	nand_release(&host->nand_chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	dma_release_channel(host->dma_chan);
 
 	/* Force CE high */
diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c
index 179f0ca585f8..260a0430313e 100644
--- a/drivers/mtd/nand/raw/marvell_nand.c
+++ b/drivers/mtd/nand/raw/marvell_nand.c
@@ -707,7 +707,7 @@ static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
 	 * In case the interrupt was not served in the required time frame,
 	 * check if the ISR was not served or if something went actually wrong.
 	 */
-	if (ret && !pending) {
+	if (!ret && !pending) {
 		dev_err(nfc->dev, "Timeout waiting for RB signal\n");
 		return -ETIMEDOUT;
 	}
@@ -932,14 +932,14 @@ static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
 }
 
 /*
- * Check a chunk is correct or not according to hardware ECC engine.
+ * Check if a chunk is correct or not according to the hardware ECC engine.
  * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
  * mtd->ecc_stats.failure is not, the function will instead return a non-zero
  * value indicating that a check on the emptyness of the subpage must be
- * performed before declaring the subpage corrupted.
+ * performed before actually declaring the subpage as "corrupted".
  */
-static int marvell_nfc_hw_ecc_correct(struct nand_chip *chip,
-				      unsigned int *max_bitflips)
+static int marvell_nfc_hw_ecc_check_bitflips(struct nand_chip *chip,
+					     unsigned int *max_bitflips)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
@@ -1053,7 +1053,7 @@ static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
 	marvell_nfc_enable_hw_ecc(chip);
 	marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
 					    page);
-	ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
+	ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
 	marvell_nfc_disable_hw_ecc(chip);
 
 	if (!ret)
@@ -1224,12 +1224,12 @@ static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
 
 		/* Read spare bytes */
 		nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
-				  spare_len, false);
+				  spare_len, false, false);
 
 		/* Read ECC bytes */
 		nand_read_data_op(chip, oob + ecc_offset +
 				  (ALIGN(lt->ecc_bytes, 32) * chunk),
-				  ecc_len, false);
+				  ecc_len, false, false);
 	}
 
 	return 0;
@@ -1336,7 +1336,7 @@ static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
 		/* Read the chunk and detect number of bitflips */
 		marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
 						  spare, spare_len, page);
-		ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
+		ret = marvell_nfc_hw_ecc_check_bitflips(chip, &max_bitflips);
 		if (ret)
 			failure_mask |= BIT(chunk);
 
@@ -1358,10 +1358,9 @@ static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
 	 */
 
 	/*
-	 * In case there is any subpage read error reported by ->correct(), we
-	 * usually re-read only ECC bytes in raw mode and check if the whole
-	 * page is empty. In this case, it is normal that the ECC check failed
-	 * and we just ignore the error.
+	 * In case there is any subpage read error, we usually re-read only ECC
+	 * bytes in raw mode and check if the whole page is empty. In this case,
+	 * it is normal that the ECC check failed and we just ignore the error.
 	 *
 	 * However, it has been empirically observed that for some layouts (e.g
 	 * 2k page, 8b strength per 512B chunk), the controller tries to correct
@@ -2107,7 +2106,8 @@ static int marvell_nfc_exec_op(struct nand_chip *chip,
 {
 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
 
-	marvell_nfc_select_target(chip, op->cs);
+	if (!check_only)
+		marvell_nfc_select_target(chip, op->cs);
 
 	if (nfc->caps->is_nfcv2)
 		return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
@@ -2166,8 +2166,8 @@ static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
 	.free = marvell_nand_ooblayout_free,
 };
 
-static int marvell_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
-					 struct nand_ecc_ctrl *ecc)
+static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd,
+					       struct nand_ecc_ctrl *ecc)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
@@ -2261,7 +2261,7 @@ static int marvell_nand_ecc_init(struct mtd_info *mtd,
 
 	switch (ecc->mode) {
 	case NAND_ECC_HW:
-		ret = marvell_nand_hw_ecc_ctrl_init(mtd, ecc);
+		ret = marvell_nand_hw_ecc_controller_init(mtd, ecc);
 		if (ret)
 			return ret;
 		break;
@@ -2664,7 +2664,7 @@ static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
 		ret = mtd_device_register(mtd, NULL, 0);
 	if (ret) {
 		dev_err(dev, "failed to register mtd device: %d\n", ret);
-		nand_release(chip);
+		nand_cleanup(chip);
 		return ret;
 	}
 
@@ -2673,6 +2673,21 @@ static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
 	return 0;
 }
 
+static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
+{
+	struct marvell_nand_chip *entry, *temp;
+	struct nand_chip *chip;
+	int ret;
+
+	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
+		chip = &entry->chip;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+		list_del(&entry->node);
+	}
+}
+
 static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
 {
 	struct device_node *np = dev->of_node;
@@ -2707,21 +2722,16 @@ static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
 		ret = marvell_nand_chip_init(dev, nfc, nand_np);
 		if (ret) {
 			of_node_put(nand_np);
-			return ret;
+			goto cleanup_chips;
 		}
 	}
 
 	return 0;
-}
 
-static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
-{
-	struct marvell_nand_chip *entry, *temp;
+cleanup_chips:
+	marvell_nand_chips_cleanup(nfc);
 
-	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
-		nand_release(&entry->chip);
-		list_del(&entry->node);
-	}
+	return ret;
 }
 
 static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
@@ -2854,7 +2864,6 @@ static int marvell_nfc_init(struct marvell_nfc *nfc)
 static int marvell_nfc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct resource *r;
 	struct marvell_nfc *nfc;
 	int ret;
 	int irq;
@@ -2869,8 +2878,7 @@ static int marvell_nfc_probe(struct platform_device *pdev)
 	nfc->controller.ops = &marvell_nand_controller_ops;
 	INIT_LIST_HEAD(&nfc->chips);
 
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	nfc->regs = devm_ioremap_resource(dev, r);
+	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(nfc->regs))
 		return PTR_ERR(nfc->regs);
 
diff --git a/drivers/mtd/nand/raw/meson_nand.c b/drivers/mtd/nand/raw/meson_nand.c
index f6fb5c0e6255..3f376471f3f7 100644
--- a/drivers/mtd/nand/raw/meson_nand.c
+++ b/drivers/mtd/nand/raw/meson_nand.c
@@ -899,6 +899,9 @@ static int meson_nfc_exec_op(struct nand_chip *nand,
 	u32 op_id, delay_idle, cmd;
 	int i;
 
+	if (check_only)
+		return 0;
+
 	meson_nfc_select_chip(nand, op->cs);
 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
 		instr = &op->instrs[op_id];
@@ -1266,7 +1269,7 @@ meson_nfc_nand_chip_init(struct device *dev,
 	nand_set_flash_node(nand, np);
 	nand_set_controller_data(nand, nfc);
 
-	nand->options |= NAND_USE_BOUNCE_BUFFER;
+	nand->options |= NAND_USES_DMA;
 	mtd = nand_to_mtd(nand);
 	mtd->owner = THIS_MODULE;
 	mtd->dev.parent = dev;
diff --git a/drivers/mtd/nand/raw/mpc5121_nfc.c b/drivers/mtd/nand/raw/mpc5121_nfc.c
index a2fcb739e5f8..18ecb096a32d 100644
--- a/drivers/mtd/nand/raw/mpc5121_nfc.c
+++ b/drivers/mtd/nand/raw/mpc5121_nfc.c
@@ -805,8 +805,11 @@ static int mpc5121_nfc_remove(struct platform_device *op)
 {
 	struct device *dev = &op->dev;
 	struct mtd_info *mtd = dev_get_drvdata(dev);
+	int ret;
 
-	nand_release(mtd_to_nand(mtd));
+	ret = mtd_device_unregister(mtd);
+	WARN_ON(ret);
+	nand_cleanup(mtd_to_nand(mtd));
 	mpc5121_nfc_free(dev, mtd);
 
 	return 0;
diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c
index ef149e8b26d0..c1a6e31aabb8 100644
--- a/drivers/mtd/nand/raw/mtk_nand.c
+++ b/drivers/mtd/nand/raw/mtk_nand.c
@@ -1380,7 +1380,7 @@ static int mtk_nfc_nand_chip_init(struct device *dev, struct mtk_nfc *nfc,
 	nand_set_flash_node(nand, np);
 	nand_set_controller_data(nand, nfc);
 
-	nand->options |= NAND_USE_BOUNCE_BUFFER | NAND_SUBPAGE_READ;
+	nand->options |= NAND_USES_DMA | NAND_SUBPAGE_READ;
 	nand->legacy.dev_ready = mtk_nfc_dev_ready;
 	nand->legacy.select_chip = mtk_nfc_select_chip;
 	nand->legacy.write_byte = mtk_nfc_write_byte;
@@ -1419,7 +1419,7 @@ static int mtk_nfc_nand_chip_init(struct device *dev, struct mtk_nfc *nfc,
 	ret = mtd_device_register(mtd, NULL, 0);
 	if (ret) {
 		dev_err(dev, "mtd parse partition error\n");
-		nand_release(nand);
+		nand_cleanup(nand);
 		return ret;
 	}
 
@@ -1578,13 +1578,18 @@ release_ecc:
 static int mtk_nfc_remove(struct platform_device *pdev)
 {
 	struct mtk_nfc *nfc = platform_get_drvdata(pdev);
-	struct mtk_nfc_nand_chip *chip;
+	struct mtk_nfc_nand_chip *mtk_chip;
+	struct nand_chip *chip;
+	int ret;
 
 	while (!list_empty(&nfc->chips)) {
-		chip = list_first_entry(&nfc->chips, struct mtk_nfc_nand_chip,
-					node);
-		nand_release(&chip->nand);
-		list_del(&chip->node);
+		mtk_chip = list_first_entry(&nfc->chips,
+					    struct mtk_nfc_nand_chip, node);
+		chip = &mtk_chip->nand;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+		list_del(&mtk_chip->node);
 	}
 
 	mtk_ecc_release(nfc->ecc);
diff --git a/drivers/mtd/nand/raw/mxc_nand.c b/drivers/mtd/nand/raw/mxc_nand.c
index 59554c187e01..09dacb83cb5a 100644
--- a/drivers/mtd/nand/raw/mxc_nand.c
+++ b/drivers/mtd/nand/raw/mxc_nand.c
@@ -1919,8 +1919,12 @@ escan:
 static int mxcnd_remove(struct platform_device *pdev)
 {
 	struct mxc_nand_host *host = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &host->nand;
+	int ret;
 
-	nand_release(&host->nand);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	if (host->clk_act)
 		clk_disable_unprepare(host->clk);
 
diff --git a/drivers/mtd/nand/raw/mxic_nand.c b/drivers/mtd/nand/raw/mxic_nand.c
index ed7a4e021bf5..57f36721f4c6 100644
--- a/drivers/mtd/nand/raw/mxic_nand.c
+++ b/drivers/mtd/nand/raw/mxic_nand.c
@@ -393,6 +393,9 @@ static int mxic_nfc_exec_op(struct nand_chip *chip,
 	int ret = 0;
 	unsigned int op_id;
 
+	if (check_only)
+		return 0;
+
 	mxic_nfc_cs_enable(nfc);
 	init_completion(&nfc->complete);
 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
@@ -553,8 +556,13 @@ fail:
 static int mxic_nfc_remove(struct platform_device *pdev)
 {
 	struct mxic_nand_ctlr *nfc = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &nfc->chip;
+	int ret;
+
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
-	nand_release(&nfc->chip);
 	mxic_nfc_clk_disable(nfc);
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c
index c24e5e2ba130..45124dbb1835 100644
--- a/drivers/mtd/nand/raw/nand_base.c
+++ b/drivers/mtd/nand/raw/nand_base.c
@@ -205,6 +205,56 @@ static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = {
 	.free = nand_ooblayout_free_lp_hamming,
 };
 
+static int nand_pairing_dist3_get_info(struct mtd_info *mtd, int page,
+				       struct mtd_pairing_info *info)
+{
+	int lastpage = (mtd->erasesize / mtd->writesize) - 1;
+	int dist = 3;
+
+	if (page == lastpage)
+		dist = 2;
+
+	if (!page || (page & 1)) {
+		info->group = 0;
+		info->pair = (page + 1) / 2;
+	} else {
+		info->group = 1;
+		info->pair = (page + 1 - dist) / 2;
+	}
+
+	return 0;
+}
+
+static int nand_pairing_dist3_get_wunit(struct mtd_info *mtd,
+					const struct mtd_pairing_info *info)
+{
+	int lastpair = ((mtd->erasesize / mtd->writesize) - 1) / 2;
+	int page = info->pair * 2;
+	int dist = 3;
+
+	if (!info->group && !info->pair)
+		return 0;
+
+	if (info->pair == lastpair && info->group)
+		dist = 2;
+
+	if (!info->group)
+		page--;
+	else if (info->pair)
+		page += dist - 1;
+
+	if (page >= mtd->erasesize / mtd->writesize)
+		return -EINVAL;
+
+	return page;
+}
+
+const struct mtd_pairing_scheme dist3_pairing_scheme = {
+	.ngroups = 2,
+	.get_info = nand_pairing_dist3_get_info,
+	.get_wunit = nand_pairing_dist3_get_wunit,
+};
+
 static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len)
 {
 	int ret = 0;
@@ -225,6 +275,50 @@ static int check_offs_len(struct nand_chip *chip, loff_t ofs, uint64_t len)
 }
 
 /**
+ * nand_extract_bits - Copy unaligned bits from one buffer to another one
+ * @dst: destination buffer
+ * @dst_off: bit offset at which the writing starts
+ * @src: source buffer
+ * @src_off: bit offset at which the reading starts
+ * @nbits: number of bits to copy from @src to @dst
+ *
+ * Copy bits from one memory region to another (overlap authorized).
+ */
+void nand_extract_bits(u8 *dst, unsigned int dst_off, const u8 *src,
+		       unsigned int src_off, unsigned int nbits)
+{
+	unsigned int tmp, n;
+
+	dst += dst_off / 8;
+	dst_off %= 8;
+	src += src_off / 8;
+	src_off %= 8;
+
+	while (nbits) {
+		n = min3(8 - dst_off, 8 - src_off, nbits);
+
+		tmp = (*src >> src_off) & GENMASK(n - 1, 0);
+		*dst &= ~GENMASK(n - 1 + dst_off, dst_off);
+		*dst |= tmp << dst_off;
+
+		dst_off += n;
+		if (dst_off >= 8) {
+			dst++;
+			dst_off -= 8;
+		}
+
+		src_off += n;
+		if (src_off >= 8) {
+			src++;
+			src_off -= 8;
+		}
+
+		nbits -= n;
+	}
+}
+EXPORT_SYMBOL_GPL(nand_extract_bits);
+
+/**
  * nand_select_target() - Select a NAND target (A.K.A. die)
  * @chip: NAND chip object
  * @cs: the CS line to select. Note that this CS id is always from the chip
@@ -345,6 +439,9 @@ static int nand_block_bad(struct nand_chip *chip, loff_t ofs)
 
 static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs)
 {
+	if (chip->options & NAND_NO_BBM_QUIRK)
+		return 0;
+
 	if (chip->legacy.block_bad)
 		return chip->legacy.block_bad(chip, ofs);
 
@@ -690,7 +787,8 @@ int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms)
 	 */
 	timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
 	do {
-		ret = nand_read_data_op(chip, &status, sizeof(status), true);
+		ret = nand_read_data_op(chip, &status, sizeof(status), true,
+					false);
 		if (ret)
 			break;
 
@@ -736,8 +834,14 @@ EXPORT_SYMBOL_GPL(nand_soft_waitrdy);
 int nand_gpio_waitrdy(struct nand_chip *chip, struct gpio_desc *gpiod,
 		      unsigned long timeout_ms)
 {
-	/* Wait until R/B pin indicates chip is ready or timeout occurs */
-	timeout_ms = jiffies + msecs_to_jiffies(timeout_ms);
+
+	/*
+	 * Wait until R/B pin indicates chip is ready or timeout occurs.
+	 * +1 below is necessary because if we are now in the last fraction
+	 * of jiffy and msecs_to_jiffies is 1 then we will wait only that
+	 * small jiffy fraction - possibly leading to false timeout.
+	 */
+	timeout_ms = jiffies + msecs_to_jiffies(timeout_ms) + 1;
 	do {
 		if (gpiod_get_value_cansleep(gpiod))
 			return 0;
@@ -770,7 +874,7 @@ void panic_nand_wait(struct nand_chip *chip, unsigned long timeo)
 			u8 status;
 
 			ret = nand_read_data_op(chip, &status, sizeof(status),
-						true);
+						true, false);
 			if (ret)
 				return;
 
@@ -1868,6 +1972,8 @@ EXPORT_SYMBOL_GPL(nand_reset_op);
  * @buf: buffer used to store the data
  * @len: length of the buffer
  * @force_8bit: force 8-bit bus access
+ * @check_only: do not actually run the command, only checks if the
+ *              controller driver supports it
  *
  * This function does a raw data read on the bus. Usually used after launching
  * another NAND operation like nand_read_page_op().
@@ -1876,7 +1982,7 @@ EXPORT_SYMBOL_GPL(nand_reset_op);
  * Returns 0 on success, a negative error code otherwise.
  */
 int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len,
-		      bool force_8bit)
+		      bool force_8bit, bool check_only)
 {
 	if (!len || !buf)
 		return -EINVAL;
@@ -1889,9 +1995,15 @@ int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len,
 
 		instrs[0].ctx.data.force_8bit = force_8bit;
 
+		if (check_only)
+			return nand_check_op(chip, &op);
+
 		return nand_exec_op(chip, &op);
 	}
 
+	if (check_only)
+		return 0;
+
 	if (force_8bit) {
 		u8 *p = buf;
 		unsigned int i;
@@ -2112,7 +2224,7 @@ static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
 	char *prefix = "      ";
 	unsigned int i;
 
-	pr_debug("executing subop:\n");
+	pr_debug("executing subop (CS%d):\n", ctx->subop.cs);
 
 	for (i = 0; i < ctx->ninstrs; i++) {
 		instr = &ctx->instrs[i];
@@ -2176,6 +2288,7 @@ int nand_op_parser_exec_op(struct nand_chip *chip,
 			   const struct nand_operation *op, bool check_only)
 {
 	struct nand_op_parser_ctx ctx = {
+		.subop.cs = op->cs,
 		.subop.instrs = op->instrs,
 		.instrs = op->instrs,
 		.ninstrs = op->ninstrs,
@@ -2620,7 +2733,7 @@ int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required,
 
 	if (oob_required) {
 		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
-					false);
+					false, false);
 		if (ret)
 			return ret;
 	}
@@ -2630,6 +2743,47 @@ int nand_read_page_raw(struct nand_chip *chip, uint8_t *buf, int oob_required,
 EXPORT_SYMBOL(nand_read_page_raw);
 
 /**
+ * nand_monolithic_read_page_raw - Monolithic page read in raw mode
+ * @chip: NAND chip info structure
+ * @buf: buffer to store read data
+ * @oob_required: caller requires OOB data read to chip->oob_poi
+ * @page: page number to read
+ *
+ * This is a raw page read, ie. without any error detection/correction.
+ * Monolithic means we are requesting all the relevant data (main plus
+ * eventually OOB) to be loaded in the NAND cache and sent over the
+ * bus (from the NAND chip to the NAND controller) in a single
+ * operation. This is an alternative to nand_read_page_raw(), which
+ * first reads the main data, and if the OOB data is requested too,
+ * then reads more data on the bus.
+ */
+int nand_monolithic_read_page_raw(struct nand_chip *chip, u8 *buf,
+				  int oob_required, int page)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	unsigned int size = mtd->writesize;
+	u8 *read_buf = buf;
+	int ret;
+
+	if (oob_required) {
+		size += mtd->oobsize;
+
+		if (buf != chip->data_buf)
+			read_buf = nand_get_data_buf(chip);
+	}
+
+	ret = nand_read_page_op(chip, page, 0, read_buf, size);
+	if (ret)
+		return ret;
+
+	if (buf != chip->data_buf)
+		memcpy(buf, read_buf, mtd->writesize);
+
+	return 0;
+}
+EXPORT_SYMBOL(nand_monolithic_read_page_raw);
+
+/**
  * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
  * @chip: nand chip info structure
  * @buf: buffer to store read data
@@ -2652,7 +2806,7 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf,
 		return ret;
 
 	for (steps = chip->ecc.steps; steps > 0; steps--) {
-		ret = nand_read_data_op(chip, buf, eccsize, false);
+		ret = nand_read_data_op(chip, buf, eccsize, false, false);
 		if (ret)
 			return ret;
 
@@ -2660,14 +2814,14 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf,
 
 		if (chip->ecc.prepad) {
 			ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
-						false);
+						false, false);
 			if (ret)
 				return ret;
 
 			oob += chip->ecc.prepad;
 		}
 
-		ret = nand_read_data_op(chip, oob, eccbytes, false);
+		ret = nand_read_data_op(chip, oob, eccbytes, false, false);
 		if (ret)
 			return ret;
 
@@ -2675,7 +2829,7 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf,
 
 		if (chip->ecc.postpad) {
 			ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
-						false);
+						false, false);
 			if (ret)
 				return ret;
 
@@ -2685,7 +2839,7 @@ static int nand_read_page_raw_syndrome(struct nand_chip *chip, uint8_t *buf,
 
 	size = mtd->oobsize - (oob - chip->oob_poi);
 	if (size) {
-		ret = nand_read_data_op(chip, oob, size, false);
+		ret = nand_read_data_op(chip, oob, size, false, false);
 		if (ret)
 			return ret;
 	}
@@ -2878,14 +3032,15 @@ static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf,
 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
 		chip->ecc.hwctl(chip, NAND_ECC_READ);
 
-		ret = nand_read_data_op(chip, p, eccsize, false);
+		ret = nand_read_data_op(chip, p, eccsize, false, false);
 		if (ret)
 			return ret;
 
 		chip->ecc.calculate(chip, p, &ecc_calc[i]);
 	}
 
-	ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false);
+	ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
+				false);
 	if (ret)
 		return ret;
 
@@ -2921,76 +3076,6 @@ static int nand_read_page_hwecc(struct nand_chip *chip, uint8_t *buf,
 }
 
 /**
- * nand_read_page_hwecc_oob_first - [REPLACEABLE] hw ecc, read oob first
- * @chip: nand chip info structure
- * @buf: buffer to store read data
- * @oob_required: caller requires OOB data read to chip->oob_poi
- * @page: page number to read
- *
- * Hardware ECC for large page chips, require OOB to be read first. For this
- * ECC mode, the write_page method is re-used from ECC_HW. These methods
- * read/write ECC from the OOB area, unlike the ECC_HW_SYNDROME support with
- * multiple ECC steps, follows the "infix ECC" scheme and reads/writes ECC from
- * the data area, by overwriting the NAND manufacturer bad block markings.
- */
-static int nand_read_page_hwecc_oob_first(struct nand_chip *chip, uint8_t *buf,
-					  int oob_required, int page)
-{
-	struct mtd_info *mtd = nand_to_mtd(chip);
-	int i, eccsize = chip->ecc.size, ret;
-	int eccbytes = chip->ecc.bytes;
-	int eccsteps = chip->ecc.steps;
-	uint8_t *p = buf;
-	uint8_t *ecc_code = chip->ecc.code_buf;
-	uint8_t *ecc_calc = chip->ecc.calc_buf;
-	unsigned int max_bitflips = 0;
-
-	/* Read the OOB area first */
-	ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
-	if (ret)
-		return ret;
-
-	ret = nand_read_page_op(chip, page, 0, NULL, 0);
-	if (ret)
-		return ret;
-
-	ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
-					 chip->ecc.total);
-	if (ret)
-		return ret;
-
-	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
-		int stat;
-
-		chip->ecc.hwctl(chip, NAND_ECC_READ);
-
-		ret = nand_read_data_op(chip, p, eccsize, false);
-		if (ret)
-			return ret;
-
-		chip->ecc.calculate(chip, p, &ecc_calc[i]);
-
-		stat = chip->ecc.correct(chip, p, &ecc_code[i], NULL);
-		if (stat == -EBADMSG &&
-		    (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
-			/* check for empty pages with bitflips */
-			stat = nand_check_erased_ecc_chunk(p, eccsize,
-						&ecc_code[i], eccbytes,
-						NULL, 0,
-						chip->ecc.strength);
-		}
-
-		if (stat < 0) {
-			mtd->ecc_stats.failed++;
-		} else {
-			mtd->ecc_stats.corrected += stat;
-			max_bitflips = max_t(unsigned int, max_bitflips, stat);
-		}
-	}
-	return max_bitflips;
-}
-
-/**
  * nand_read_page_syndrome - [REPLACEABLE] hardware ECC syndrome based page read
  * @chip: nand chip info structure
  * @buf: buffer to store read data
@@ -3021,13 +3106,13 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf,
 
 		chip->ecc.hwctl(chip, NAND_ECC_READ);
 
-		ret = nand_read_data_op(chip, p, eccsize, false);
+		ret = nand_read_data_op(chip, p, eccsize, false, false);
 		if (ret)
 			return ret;
 
 		if (chip->ecc.prepad) {
 			ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
-						false);
+						false, false);
 			if (ret)
 				return ret;
 
@@ -3036,7 +3121,7 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf,
 
 		chip->ecc.hwctl(chip, NAND_ECC_READSYN);
 
-		ret = nand_read_data_op(chip, oob, eccbytes, false);
+		ret = nand_read_data_op(chip, oob, eccbytes, false, false);
 		if (ret)
 			return ret;
 
@@ -3046,7 +3131,7 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf,
 
 		if (chip->ecc.postpad) {
 			ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
-						false);
+						false, false);
 			if (ret)
 				return ret;
 
@@ -3074,7 +3159,7 @@ static int nand_read_page_syndrome(struct nand_chip *chip, uint8_t *buf,
 	/* Calculate remaining oob bytes */
 	i = mtd->oobsize - (oob - chip->oob_poi);
 	if (i) {
-		ret = nand_read_data_op(chip, oob, i, false);
+		ret = nand_read_data_op(chip, oob, i, false, false);
 		if (ret)
 			return ret;
 	}
@@ -3166,7 +3251,7 @@ static int nand_do_read_ops(struct nand_chip *chip, loff_t from,
 	uint32_t max_oobsize = mtd_oobavail(mtd, ops);
 
 	uint8_t *bufpoi, *oob, *buf;
-	int use_bufpoi;
+	int use_bounce_buf;
 	unsigned int max_bitflips = 0;
 	int retry_mode = 0;
 	bool ecc_fail = false;
@@ -3184,25 +3269,25 @@ static int nand_do_read_ops(struct nand_chip *chip, loff_t from,
 	oob_required = oob ? 1 : 0;
 
 	while (1) {
-		unsigned int ecc_failures = mtd->ecc_stats.failed;
+		struct mtd_ecc_stats ecc_stats = mtd->ecc_stats;
 
 		bytes = min(mtd->writesize - col, readlen);
 		aligned = (bytes == mtd->writesize);
 
 		if (!aligned)
-			use_bufpoi = 1;
-		else if (chip->options & NAND_USE_BOUNCE_BUFFER)
-			use_bufpoi = !virt_addr_valid(buf) ||
-				     !IS_ALIGNED((unsigned long)buf,
-						 chip->buf_align);
+			use_bounce_buf = 1;
+		else if (chip->options & NAND_USES_DMA)
+			use_bounce_buf = !virt_addr_valid(buf) ||
+					 !IS_ALIGNED((unsigned long)buf,
+						     chip->buf_align);
 		else
-			use_bufpoi = 0;
+			use_bounce_buf = 0;
 
 		/* Is the current page in the buffer? */
 		if (realpage != chip->pagecache.page || oob) {
-			bufpoi = use_bufpoi ? chip->data_buf : buf;
+			bufpoi = use_bounce_buf ? chip->data_buf : buf;
 
-			if (use_bufpoi && aligned)
+			if (use_bounce_buf && aligned)
 				pr_debug("%s: using read bounce buffer for buf@%p\n",
 						 __func__, buf);
 
@@ -3223,16 +3308,19 @@ read_retry:
 				ret = chip->ecc.read_page(chip, bufpoi,
 							  oob_required, page);
 			if (ret < 0) {
-				if (use_bufpoi)
+				if (use_bounce_buf)
 					/* Invalidate page cache */
 					chip->pagecache.page = -1;
 				break;
 			}
 
-			/* Transfer not aligned data */
-			if (use_bufpoi) {
+			/*
+			 * Copy back the data in the initial buffer when reading
+			 * partial pages or when a bounce buffer is required.
+			 */
+			if (use_bounce_buf) {
 				if (!NAND_HAS_SUBPAGE_READ(chip) && !oob &&
-				    !(mtd->ecc_stats.failed - ecc_failures) &&
+				    !(mtd->ecc_stats.failed - ecc_stats.failed) &&
 				    (ops->mode != MTD_OPS_RAW)) {
 					chip->pagecache.page = realpage;
 					chip->pagecache.bitflips = ret;
@@ -3240,7 +3328,7 @@ read_retry:
 					/* Invalidate page cache */
 					chip->pagecache.page = -1;
 				}
-				memcpy(buf, chip->data_buf + col, bytes);
+				memcpy(buf, bufpoi + col, bytes);
 			}
 
 			if (unlikely(oob)) {
@@ -3255,7 +3343,7 @@ read_retry:
 
 			nand_wait_readrdy(chip);
 
-			if (mtd->ecc_stats.failed - ecc_failures) {
+			if (mtd->ecc_stats.failed - ecc_stats.failed) {
 				if (retry_mode + 1 < chip->read_retries) {
 					retry_mode++;
 					ret = nand_setup_read_retry(chip,
@@ -3263,8 +3351,8 @@ read_retry:
 					if (ret < 0)
 						break;
 
-					/* Reset failures; retry */
-					mtd->ecc_stats.failed = ecc_failures;
+					/* Reset ecc_stats; retry */
+					mtd->ecc_stats = ecc_stats;
 					goto read_retry;
 				} else {
 					/* No more retry modes; real failure */
@@ -3373,7 +3461,7 @@ static int nand_read_oob_syndrome(struct nand_chip *chip, int page)
 			sndrnd = 1;
 		toread = min_t(int, length, chunk);
 
-		ret = nand_read_data_op(chip, bufpoi, toread, false);
+		ret = nand_read_data_op(chip, bufpoi, toread, false, false);
 		if (ret)
 			return ret;
 
@@ -3381,7 +3469,7 @@ static int nand_read_oob_syndrome(struct nand_chip *chip, int page)
 		length -= toread;
 	}
 	if (length > 0) {
-		ret = nand_read_data_op(chip, bufpoi, length, false);
+		ret = nand_read_data_op(chip, bufpoi, length, false, false);
 		if (ret)
 			return ret;
 	}
@@ -3634,6 +3722,42 @@ int nand_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
 EXPORT_SYMBOL(nand_write_page_raw);
 
 /**
+ * nand_monolithic_write_page_raw - Monolithic page write in raw mode
+ * @chip: NAND chip info structure
+ * @buf: data buffer to write
+ * @oob_required: must write chip->oob_poi to OOB
+ * @page: page number to write
+ *
+ * This is a raw page write, ie. without any error detection/correction.
+ * Monolithic means we are requesting all the relevant data (main plus
+ * eventually OOB) to be sent over the bus and effectively programmed
+ * into the NAND chip arrays in a single operation. This is an
+ * alternative to nand_write_page_raw(), which first sends the main
+ * data, then eventually send the OOB data by latching more data
+ * cycles on the NAND bus, and finally sends the program command to
+ * synchronyze the NAND chip cache.
+ */
+int nand_monolithic_write_page_raw(struct nand_chip *chip, const u8 *buf,
+				   int oob_required, int page)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	unsigned int size = mtd->writesize;
+	u8 *write_buf = (u8 *)buf;
+
+	if (oob_required) {
+		size += mtd->oobsize;
+
+		if (buf != chip->data_buf) {
+			write_buf = nand_get_data_buf(chip);
+			memcpy(write_buf, buf, mtd->writesize);
+		}
+	}
+
+	return nand_prog_page_op(chip, page, 0, write_buf, size);
+}
+EXPORT_SYMBOL(nand_monolithic_write_page_raw);
+
+/**
  * nand_write_page_raw_syndrome - [INTERN] raw page write function
  * @chip: nand chip info structure
  * @buf: data buffer
@@ -4012,20 +4136,23 @@ static int nand_do_write_ops(struct nand_chip *chip, loff_t to,
 	while (1) {
 		int bytes = mtd->writesize;
 		uint8_t *wbuf = buf;
-		int use_bufpoi;
+		int use_bounce_buf;
 		int part_pagewr = (column || writelen < mtd->writesize);
 
 		if (part_pagewr)
-			use_bufpoi = 1;
-		else if (chip->options & NAND_USE_BOUNCE_BUFFER)
-			use_bufpoi = !virt_addr_valid(buf) ||
-				     !IS_ALIGNED((unsigned long)buf,
-						 chip->buf_align);
+			use_bounce_buf = 1;
+		else if (chip->options & NAND_USES_DMA)
+			use_bounce_buf = !virt_addr_valid(buf) ||
+					 !IS_ALIGNED((unsigned long)buf,
+						     chip->buf_align);
 		else
-			use_bufpoi = 0;
+			use_bounce_buf = 0;
 
-		/* Partial page write?, or need to use bounce buffer */
-		if (use_bufpoi) {
+		/*
+		 * Copy the data from the initial buffer when doing partial page
+		 * writes or when a bounce buffer is required.
+		 */
+		if (use_bounce_buf) {
 			pr_debug("%s: using write bounce buffer for buf@%p\n",
 					 __func__, buf);
 			if (part_pagewr)
@@ -4883,7 +5010,6 @@ static const char * const nand_ecc_modes[] = {
 	[NAND_ECC_SOFT]		= "soft",
 	[NAND_ECC_HW]		= "hw",
 	[NAND_ECC_HW_SYNDROME]	= "hw_syndrome",
-	[NAND_ECC_HW_OOB_FIRST]	= "hw_oob_first",
 	[NAND_ECC_ON_DIE]	= "on-die",
 };
 
@@ -4896,14 +5022,14 @@ static int of_get_nand_ecc_mode(struct device_node *np)
 	if (err < 0)
 		return err;
 
-	for (i = 0; i < ARRAY_SIZE(nand_ecc_modes); i++)
+	for (i = NAND_ECC_NONE; i < ARRAY_SIZE(nand_ecc_modes); i++)
 		if (!strcasecmp(pm, nand_ecc_modes[i]))
 			return i;
 
 	/*
 	 * For backward compatibility we support few obsoleted values that don't
-	 * have their mappings into nand_ecc_modes_t anymore (they were merged
-	 * with other enums).
+	 * have their mappings into the nand_ecc_mode enum anymore (they were
+	 * merged with other enums).
 	 */
 	if (!strcasecmp(pm, "soft_bch"))
 		return NAND_ECC_SOFT;
@@ -4917,17 +5043,20 @@ static const char * const nand_ecc_algos[] = {
 	[NAND_ECC_RS]		= "rs",
 };
 
-static int of_get_nand_ecc_algo(struct device_node *np)
+static enum nand_ecc_algo of_get_nand_ecc_algo(struct device_node *np)
 {
+	enum nand_ecc_algo ecc_algo;
 	const char *pm;
-	int err, i;
+	int err;
 
 	err = of_property_read_string(np, "nand-ecc-algo", &pm);
 	if (!err) {
-		for (i = NAND_ECC_HAMMING; i < ARRAY_SIZE(nand_ecc_algos); i++)
-			if (!strcasecmp(pm, nand_ecc_algos[i]))
-				return i;
-		return -ENODEV;
+		for (ecc_algo = NAND_ECC_HAMMING;
+		     ecc_algo < ARRAY_SIZE(nand_ecc_algos);
+		     ecc_algo++) {
+			if (!strcasecmp(pm, nand_ecc_algos[ecc_algo]))
+				return ecc_algo;
+		}
 	}
 
 	/*
@@ -4935,15 +5064,14 @@ static int of_get_nand_ecc_algo(struct device_node *np)
 	 * for some obsoleted values that were specifying ECC algorithm.
 	 */
 	err = of_property_read_string(np, "nand-ecc-mode", &pm);
-	if (err < 0)
-		return err;
-
-	if (!strcasecmp(pm, "soft"))
-		return NAND_ECC_HAMMING;
-	else if (!strcasecmp(pm, "soft_bch"))
-		return NAND_ECC_BCH;
+	if (!err) {
+		if (!strcasecmp(pm, "soft"))
+			return NAND_ECC_HAMMING;
+		else if (!strcasecmp(pm, "soft_bch"))
+			return NAND_ECC_BCH;
+	}
 
-	return -ENODEV;
+	return NAND_ECC_UNKNOWN;
 }
 
 static int of_get_nand_ecc_step_size(struct device_node *np)
@@ -4988,7 +5116,8 @@ static bool of_get_nand_on_flash_bbt(struct device_node *np)
 static int nand_dt_init(struct nand_chip *chip)
 {
 	struct device_node *dn = nand_get_flash_node(chip);
-	int ecc_mode, ecc_algo, ecc_strength, ecc_step;
+	enum nand_ecc_algo ecc_algo;
+	int ecc_mode, ecc_strength, ecc_step;
 
 	if (!dn)
 		return 0;
@@ -5010,7 +5139,7 @@ static int nand_dt_init(struct nand_chip *chip)
 	if (ecc_mode >= 0)
 		chip->ecc.mode = ecc_mode;
 
-	if (ecc_algo >= 0)
+	if (ecc_algo != NAND_ECC_UNKNOWN)
 		chip->ecc.algo = ecc_algo;
 
 	if (ecc_strength >= 0)
@@ -5140,8 +5269,10 @@ static int nand_set_ecc_soft_ops(struct nand_chip *chip)
 		ecc->read_page = nand_read_page_swecc;
 		ecc->read_subpage = nand_read_subpage;
 		ecc->write_page = nand_write_page_swecc;
-		ecc->read_page_raw = nand_read_page_raw;
-		ecc->write_page_raw = nand_write_page_raw;
+		if (!ecc->read_page_raw)
+			ecc->read_page_raw = nand_read_page_raw;
+		if (!ecc->write_page_raw)
+			ecc->write_page_raw = nand_write_page_raw;
 		ecc->read_oob = nand_read_oob_std;
 		ecc->write_oob = nand_write_oob_std;
 		if (!ecc->size)
@@ -5163,8 +5294,10 @@ static int nand_set_ecc_soft_ops(struct nand_chip *chip)
 		ecc->read_page = nand_read_page_swecc;
 		ecc->read_subpage = nand_read_subpage;
 		ecc->write_page = nand_write_page_swecc;
-		ecc->read_page_raw = nand_read_page_raw;
-		ecc->write_page_raw = nand_write_page_raw;
+		if (!ecc->read_page_raw)
+			ecc->read_page_raw = nand_read_page_raw;
+		if (!ecc->write_page_raw)
+			ecc->write_page_raw = nand_write_page_raw;
 		ecc->read_oob = nand_read_oob_std;
 		ecc->write_oob = nand_write_oob_std;
 
@@ -5628,16 +5761,6 @@ static int nand_scan_tail(struct nand_chip *chip)
 	 */
 
 	switch (ecc->mode) {
-	case NAND_ECC_HW_OOB_FIRST:
-		/* Similar to NAND_ECC_HW, but a separate read_page handle */
-		if (!ecc->calculate || !ecc->correct || !ecc->hwctl) {
-			WARN(1, "No ECC functions supplied; hardware ECC not possible\n");
-			ret = -EINVAL;
-			goto err_nand_manuf_cleanup;
-		}
-		if (!ecc->read_page)
-			ecc->read_page = nand_read_page_hwecc_oob_first;
-		fallthrough;
 	case NAND_ECC_HW:
 		/* Use standard hwecc read page function? */
 		if (!ecc->read_page)
@@ -5781,8 +5904,10 @@ static int nand_scan_tail(struct nand_chip *chip)
 
 	/* ECC sanity check: warn if it's too weak */
 	if (!nand_ecc_strength_good(chip))
-		pr_warn("WARNING: %s: the ECC used on your system is too weak compared to the one required by the NAND chip\n",
-			mtd->name);
+		pr_warn("WARNING: %s: the ECC used on your system (%db/%dB) is too weak compared to the one required by the NAND chip (%db/%dB)\n",
+			mtd->name, chip->ecc.strength, chip->ecc.size,
+			chip->base.eccreq.strength,
+			chip->base.eccreq.step_size);
 
 	/* Allow subpage writes up to ecc.steps. Not possible for MLC flash */
 	if (!(chip->options & NAND_NO_SUBPAGE_WRITE) && nand_is_slc(chip)) {
@@ -5975,18 +6100,6 @@ void nand_cleanup(struct nand_chip *chip)
 
 EXPORT_SYMBOL_GPL(nand_cleanup);
 
-/**
- * nand_release - [NAND Interface] Unregister the MTD device and free resources
- *		  held by the NAND device
- * @chip: NAND chip object
- */
-void nand_release(struct nand_chip *chip)
-{
-	mtd_device_unregister(nand_to_mtd(chip));
-	nand_cleanup(chip);
-}
-EXPORT_SYMBOL_GPL(nand_release);
-
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Steven J. Hill <sjhill@realitydiluted.com>");
 MODULE_AUTHOR("Thomas Gleixner <tglx@linutronix.de>");
diff --git a/drivers/mtd/nand/raw/nand_bch.c b/drivers/mtd/nand/raw/nand_bch.c
index 17527310c3a1..d5af8c5fd02f 100644
--- a/drivers/mtd/nand/raw/nand_bch.c
+++ b/drivers/mtd/nand/raw/nand_bch.c
@@ -41,7 +41,7 @@ int nand_bch_calculate_ecc(struct nand_chip *chip, const unsigned char *buf,
 	unsigned int i;
 
 	memset(code, 0, chip->ecc.bytes);
-	encode_bch(nbc->bch, buf, chip->ecc.size, code);
+	bch_encode(nbc->bch, buf, chip->ecc.size, code);
 
 	/* apply mask so that an erased page is a valid codeword */
 	for (i = 0; i < chip->ecc.bytes; i++)
@@ -67,7 +67,7 @@ int nand_bch_correct_data(struct nand_chip *chip, unsigned char *buf,
 	unsigned int *errloc = nbc->errloc;
 	int i, count;
 
-	count = decode_bch(nbc->bch, NULL, chip->ecc.size, read_ecc, calc_ecc,
+	count = bch_decode(nbc->bch, NULL, chip->ecc.size, read_ecc, calc_ecc,
 			   NULL, errloc);
 	if (count > 0) {
 		for (i = 0; i < count; i++) {
@@ -130,7 +130,7 @@ struct nand_bch_control *nand_bch_init(struct mtd_info *mtd)
 	if (!nbc)
 		goto fail;
 
-	nbc->bch = init_bch(m, t, 0);
+	nbc->bch = bch_init(m, t, 0, false);
 	if (!nbc->bch)
 		goto fail;
 
@@ -182,7 +182,7 @@ struct nand_bch_control *nand_bch_init(struct mtd_info *mtd)
 		goto fail;
 
 	memset(erased_page, 0xff, eccsize);
-	encode_bch(nbc->bch, erased_page, eccsize, nbc->eccmask);
+	bch_encode(nbc->bch, erased_page, eccsize, nbc->eccmask);
 	kfree(erased_page);
 
 	for (i = 0; i < eccbytes; i++)
@@ -205,7 +205,7 @@ EXPORT_SYMBOL(nand_bch_init);
 void nand_bch_free(struct nand_bch_control *nbc)
 {
 	if (nbc) {
-		free_bch(nbc->bch);
+		bch_free(nbc->bch);
 		kfree(nbc->errloc);
 		kfree(nbc->eccmask);
 		kfree(nbc);
diff --git a/drivers/mtd/nand/raw/nand_jedec.c b/drivers/mtd/nand/raw/nand_jedec.c
index 9b540e76f84f..b15c42f48755 100644
--- a/drivers/mtd/nand/raw/nand_jedec.c
+++ b/drivers/mtd/nand/raw/nand_jedec.c
@@ -16,6 +16,8 @@
 
 #include "internals.h"
 
+#define JEDEC_PARAM_PAGES 3
+
 /*
  * Check if the NAND chip is JEDEC compliant, returns 1 if it is, 0 otherwise.
  */
@@ -25,9 +27,11 @@ int nand_jedec_detect(struct nand_chip *chip)
 	struct nand_memory_organization *memorg;
 	struct nand_jedec_params *p;
 	struct jedec_ecc_info *ecc;
+	bool use_datain = false;
 	int jedec_version = 0;
 	char id[5];
 	int i, val, ret;
+	u16 crc;
 
 	memorg = nanddev_get_memorg(&chip->base);
 
@@ -41,25 +45,31 @@ int nand_jedec_detect(struct nand_chip *chip)
 	if (!p)
 		return -ENOMEM;
 
-	ret = nand_read_param_page_op(chip, 0x40, NULL, 0);
-	if (ret) {
-		ret = 0;
-		goto free_jedec_param_page;
-	}
-
-	for (i = 0; i < 3; i++) {
-		ret = nand_read_data_op(chip, p, sizeof(*p), true);
+	if (!nand_has_exec_op(chip) ||
+	    !nand_read_data_op(chip, p, sizeof(*p), true, true))
+		use_datain = true;
+
+	for (i = 0; i < JEDEC_PARAM_PAGES; i++) {
+		if (!i)
+			ret = nand_read_param_page_op(chip, 0x40, p,
+						      sizeof(*p));
+		else if (use_datain)
+			ret = nand_read_data_op(chip, p, sizeof(*p), true,
+						false);
+		else
+			ret = nand_change_read_column_op(chip, sizeof(*p) * i,
+							 p, sizeof(*p), true);
 		if (ret) {
 			ret = 0;
 			goto free_jedec_param_page;
 		}
 
-		if (onfi_crc16(ONFI_CRC_BASE, (uint8_t *)p, 510) ==
-				le16_to_cpu(p->crc))
+		crc = onfi_crc16(ONFI_CRC_BASE, (u8 *)p, 510);
+		if (crc == le16_to_cpu(p->crc))
 			break;
 	}
 
-	if (i == 3) {
+	if (i == JEDEC_PARAM_PAGES) {
 		pr_err("Could not find valid JEDEC parameter page; aborting\n");
 		goto free_jedec_param_page;
 	}
diff --git a/drivers/mtd/nand/raw/nand_legacy.c b/drivers/mtd/nand/raw/nand_legacy.c
index f91e92e1b972..d64791c06a97 100644
--- a/drivers/mtd/nand/raw/nand_legacy.c
+++ b/drivers/mtd/nand/raw/nand_legacy.c
@@ -225,7 +225,8 @@ static void nand_wait_status_ready(struct nand_chip *chip, unsigned long timeo)
 	do {
 		u8 status;
 
-		ret = nand_read_data_op(chip, &status, sizeof(status), true);
+		ret = nand_read_data_op(chip, &status, sizeof(status), true,
+					false);
 		if (ret)
 			return;
 
@@ -552,7 +553,8 @@ static int nand_wait(struct nand_chip *chip)
 					break;
 			} else {
 				ret = nand_read_data_op(chip, &status,
-							sizeof(status), true);
+							sizeof(status), true,
+							false);
 				if (ret)
 					return ret;
 
@@ -563,7 +565,7 @@ static int nand_wait(struct nand_chip *chip)
 		} while (time_before(jiffies, timeo));
 	}
 
-	ret = nand_read_data_op(chip, &status, sizeof(status), true);
+	ret = nand_read_data_op(chip, &status, sizeof(status), true, false);
 	if (ret)
 		return ret;
 
diff --git a/drivers/mtd/nand/raw/nand_micron.c b/drivers/mtd/nand/raw/nand_micron.c
index 56654030ec7f..3589b4fce0d4 100644
--- a/drivers/mtd/nand/raw/nand_micron.c
+++ b/drivers/mtd/nand/raw/nand_micron.c
@@ -192,6 +192,7 @@ static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
 	struct micron_nand *micron = nand_get_manufacturer_data(chip);
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	unsigned int step, max_bitflips = 0;
+	bool use_datain = false;
 	int ret;
 
 	if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) {
@@ -211,8 +212,27 @@ static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status,
 	 * in non-raw mode, even if the user did not request those bytes.
 	 */
 	if (!oob_required) {
-		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
-					false);
+		/*
+		 * We first check which operation is supported by the controller
+		 * before running it. This trick makes it possible to support
+		 * all controllers, even the most constraints, without almost
+		 * any performance hit.
+		 *
+		 * TODO: could be enhanced to avoid repeating the same check
+		 * over and over in the fast path.
+		 */
+		if (!nand_has_exec_op(chip) ||
+		    !nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false,
+				       true))
+			use_datain = true;
+
+		if (use_datain)
+			ret = nand_read_data_op(chip, chip->oob_poi,
+						mtd->oobsize, false, false);
+		else
+			ret = nand_change_read_column_op(chip, mtd->writesize,
+							 chip->oob_poi,
+							 mtd->oobsize, false);
 		if (ret)
 			return ret;
 	}
@@ -285,6 +305,7 @@ micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,
 				 int oob_required, int page)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
+	bool use_datain = false;
 	u8 status;
 	int ret, max_bitflips = 0;
 
@@ -300,14 +321,36 @@ micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf,
 	if (ret)
 		goto out;
 
-	ret = nand_exit_status_op(chip);
-	if (ret)
-		goto out;
+	/*
+	 * We first check which operation is supported by the controller before
+	 * running it. This trick makes it possible to support all controllers,
+	 * even the most constraints, without almost any performance hit.
+	 *
+	 * TODO: could be enhanced to avoid repeating the same check over and
+	 * over in the fast path.
+	 */
+	if (!nand_has_exec_op(chip) ||
+	    !nand_read_data_op(chip, buf, mtd->writesize, false, true))
+		use_datain = true;
 
-	ret = nand_read_data_op(chip, buf, mtd->writesize, false);
-	if (!ret && oob_required)
-		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
+	if (use_datain) {
+		ret = nand_exit_status_op(chip);
+		if (ret)
+			goto out;
+
+		ret = nand_read_data_op(chip, buf, mtd->writesize, false,
 					false);
+		if (!ret && oob_required)
+			ret = nand_read_data_op(chip, chip->oob_poi,
+						mtd->oobsize, false, false);
+	} else {
+		ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize,
+						 false);
+		if (!ret && oob_required)
+			ret = nand_change_read_column_op(chip, mtd->writesize,
+							 chip->oob_poi,
+							 mtd->oobsize, false);
+	}
 
 	if (chip->ecc.strength == 4)
 		max_bitflips = micron_nand_on_die_ecc_status_4(chip, status,
@@ -508,8 +551,10 @@ static int micron_nand_init(struct nand_chip *chip)
 			chip->ecc.read_page_raw = nand_read_page_raw_notsupp;
 			chip->ecc.write_page_raw = nand_write_page_raw_notsupp;
 		} else {
-			chip->ecc.read_page_raw = nand_read_page_raw;
-			chip->ecc.write_page_raw = nand_write_page_raw;
+			if (!chip->ecc.read_page_raw)
+				chip->ecc.read_page_raw = nand_read_page_raw;
+			if (!chip->ecc.write_page_raw)
+				chip->ecc.write_page_raw = nand_write_page_raw;
 		}
 	}
 
diff --git a/drivers/mtd/nand/raw/nand_onfi.c b/drivers/mtd/nand/raw/nand_onfi.c
index 0b879bd0a68c..be3456627288 100644
--- a/drivers/mtd/nand/raw/nand_onfi.c
+++ b/drivers/mtd/nand/raw/nand_onfi.c
@@ -16,6 +16,8 @@
 
 #include "internals.h"
 
+#define ONFI_PARAM_PAGES 3
+
 u16 onfi_crc16(u16 crc, u8 const *p, size_t len)
 {
 	int i;
@@ -45,12 +47,10 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip,
 	if (!ep)
 		return -ENOMEM;
 
-	/* Send our own NAND_CMD_PARAM. */
-	ret = nand_read_param_page_op(chip, 0, NULL, 0);
-	if (ret)
-		goto ext_out;
-
-	/* Use the Change Read Column command to skip the ONFI param pages. */
+	/*
+	 * Use the Change Read Column command to skip the ONFI param pages and
+	 * ensure we read at the right location.
+	 */
 	ret = nand_change_read_column_op(chip,
 					 sizeof(*p) * p->num_of_param_pages,
 					 ep, len, true);
@@ -141,11 +141,13 @@ int nand_onfi_detect(struct nand_chip *chip)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	struct nand_memory_organization *memorg;
-	struct nand_onfi_params *p;
+	struct nand_onfi_params *p = NULL, *pbuf;
 	struct onfi_params *onfi;
+	bool use_datain = false;
 	int onfi_version = 0;
 	char id[4];
 	int i, ret, val;
+	u16 crc;
 
 	memorg = nanddev_get_memorg(&chip->base);
 
@@ -155,43 +157,54 @@ int nand_onfi_detect(struct nand_chip *chip)
 		return 0;
 
 	/* ONFI chip: allocate a buffer to hold its parameter page */
-	p = kzalloc((sizeof(*p) * 3), GFP_KERNEL);
-	if (!p)
+	pbuf = kzalloc((sizeof(*pbuf) * ONFI_PARAM_PAGES), GFP_KERNEL);
+	if (!pbuf)
 		return -ENOMEM;
 
-	ret = nand_read_param_page_op(chip, 0, NULL, 0);
-	if (ret) {
-		ret = 0;
-		goto free_onfi_param_page;
-	}
-
-	for (i = 0; i < 3; i++) {
-		ret = nand_read_data_op(chip, &p[i], sizeof(*p), true);
+	if (!nand_has_exec_op(chip) ||
+	    !nand_read_data_op(chip, &pbuf[0], sizeof(*pbuf), true, true))
+		use_datain = true;
+
+	for (i = 0; i < ONFI_PARAM_PAGES; i++) {
+		if (!i)
+			ret = nand_read_param_page_op(chip, 0, &pbuf[i],
+						      sizeof(*pbuf));
+		else if (use_datain)
+			ret = nand_read_data_op(chip, &pbuf[i], sizeof(*pbuf),
+						true, false);
+		else
+			ret = nand_change_read_column_op(chip, sizeof(*pbuf) * i,
+							 &pbuf[i], sizeof(*pbuf),
+							 true);
 		if (ret) {
 			ret = 0;
 			goto free_onfi_param_page;
 		}
 
-		if (onfi_crc16(ONFI_CRC_BASE, (u8 *)&p[i], 254) ==
-				le16_to_cpu(p->crc)) {
-			if (i)
-				memcpy(p, &p[i], sizeof(*p));
+		crc = onfi_crc16(ONFI_CRC_BASE, (u8 *)&pbuf[i], 254);
+		if (crc == le16_to_cpu(pbuf[i].crc)) {
+			p = &pbuf[i];
 			break;
 		}
 	}
 
-	if (i == 3) {
-		const void *srcbufs[3] = {p, p + 1, p + 2};
+	if (i == ONFI_PARAM_PAGES) {
+		const void *srcbufs[ONFI_PARAM_PAGES];
+		unsigned int j;
+
+		for (j = 0; j < ONFI_PARAM_PAGES; j++)
+			srcbufs[j] = pbuf + j;
 
 		pr_warn("Could not find a valid ONFI parameter page, trying bit-wise majority to recover it\n");
-		nand_bit_wise_majority(srcbufs, ARRAY_SIZE(srcbufs), p,
-				       sizeof(*p));
+		nand_bit_wise_majority(srcbufs, ONFI_PARAM_PAGES, pbuf,
+				       sizeof(*pbuf));
 
-		if (onfi_crc16(ONFI_CRC_BASE, (u8 *)p, 254) !=
-				le16_to_cpu(p->crc)) {
+		crc = onfi_crc16(ONFI_CRC_BASE, (u8 *)pbuf, 254);
+		if (crc != le16_to_cpu(pbuf->crc)) {
 			pr_err("ONFI parameter recovery failed, aborting\n");
 			goto free_onfi_param_page;
 		}
+		p = pbuf;
 	}
 
 	if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
@@ -299,14 +312,14 @@ int nand_onfi_detect(struct nand_chip *chip)
 	chip->parameters.onfi = onfi;
 
 	/* Identification done, free the full ONFI parameter page and exit */
-	kfree(p);
+	kfree(pbuf);
 
 	return 1;
 
 free_model:
 	kfree(chip->parameters.model);
 free_onfi_param_page:
-	kfree(p);
+	kfree(pbuf);
 
 	return ret;
 }
diff --git a/drivers/mtd/nand/raw/nand_timings.c b/drivers/mtd/nand/raw/nand_timings.c
index f64b06a71dfa..36d21be3dfe5 100644
--- a/drivers/mtd/nand/raw/nand_timings.c
+++ b/drivers/mtd/nand/raw/nand_timings.c
@@ -16,6 +16,7 @@ static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 0 */
 	{
 		.type = NAND_SDR_IFACE,
+		.timings.mode = 0,
 		.timings.sdr = {
 			.tCCS_min = 500000,
 			.tR_max = 200000000,
@@ -58,6 +59,7 @@ static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 1 */
 	{
 		.type = NAND_SDR_IFACE,
+		.timings.mode = 1,
 		.timings.sdr = {
 			.tCCS_min = 500000,
 			.tR_max = 200000000,
@@ -100,6 +102,7 @@ static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 2 */
 	{
 		.type = NAND_SDR_IFACE,
+		.timings.mode = 2,
 		.timings.sdr = {
 			.tCCS_min = 500000,
 			.tR_max = 200000000,
@@ -142,6 +145,7 @@ static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 3 */
 	{
 		.type = NAND_SDR_IFACE,
+		.timings.mode = 3,
 		.timings.sdr = {
 			.tCCS_min = 500000,
 			.tR_max = 200000000,
@@ -184,6 +188,7 @@ static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 4 */
 	{
 		.type = NAND_SDR_IFACE,
+		.timings.mode = 4,
 		.timings.sdr = {
 			.tCCS_min = 500000,
 			.tR_max = 200000000,
@@ -226,6 +231,7 @@ static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 5 */
 	{
 		.type = NAND_SDR_IFACE,
+		.timings.mode = 5,
 		.timings.sdr = {
 			.tCCS_min = 500000,
 			.tR_max = 200000000,
@@ -314,10 +320,9 @@ int onfi_fill_data_interface(struct nand_chip *chip,
 		/* microseconds -> picoseconds */
 		timings->tPROG_max = 1000000ULL * ONFI_DYN_TIMING_MAX;
 		timings->tBERS_max = 1000000ULL * ONFI_DYN_TIMING_MAX;
-		timings->tR_max = 1000000ULL * 200000000ULL;
 
-		/* nanoseconds -> picoseconds */
-		timings->tCCS_min = 1000UL * 500000;
+		timings->tR_max = 200000000;
+		timings->tCCS_min = 500000;
 	}
 
 	return 0;
diff --git a/drivers/mtd/nand/raw/nand_toshiba.c b/drivers/mtd/nand/raw/nand_toshiba.c
index f3dcd695b5db..ae069905d7e4 100644
--- a/drivers/mtd/nand/raw/nand_toshiba.c
+++ b/drivers/mtd/nand/raw/nand_toshiba.c
@@ -194,6 +194,17 @@ static void toshiba_nand_decode_id(struct nand_chip *chip)
 	}
 }
 
+static int tc58teg5dclta00_init(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	chip->onfi_timing_mode_default = 5;
+	chip->options |= NAND_NEED_SCRAMBLING;
+	mtd_set_pairing_scheme(mtd, &dist3_pairing_scheme);
+
+	return 0;
+}
+
 static int toshiba_nand_init(struct nand_chip *chip)
 {
 	if (nand_is_slc(chip))
@@ -204,6 +215,9 @@ static int toshiba_nand_init(struct nand_chip *chip)
 	    chip->id.data[4] & TOSHIBA_NAND_ID4_IS_BENAND)
 		toshiba_nand_benand_init(chip);
 
+	if (!strcmp("TC58TEG5DCLTA00", chip->parameters.model))
+		tc58teg5dclta00_init(chip);
+
 	return 0;
 }
 
diff --git a/drivers/mtd/nand/raw/nandsim.c b/drivers/mtd/nand/raw/nandsim.c
index 1de03bb34e84..0a5cb77966cc 100644
--- a/drivers/mtd/nand/raw/nandsim.c
+++ b/drivers/mtd/nand/raw/nandsim.c
@@ -353,6 +353,9 @@ struct nandsim {
 	void *file_buf;
 	struct page *held_pages[NS_MAX_HELD_PAGES];
 	int held_cnt;
+
+	/* debugfs entry */
+	struct dentry *dent;
 };
 
 /*
@@ -432,7 +435,7 @@ static unsigned long total_wear = 0;
 /* MTD structure for NAND controller */
 static struct mtd_info *nsmtd;
 
-static int nandsim_show(struct seq_file *m, void *private)
+static int ns_show(struct seq_file *m, void *private)
 {
 	unsigned long wmin = -1, wmax = 0, avg;
 	unsigned long deciles[10], decile_max[10], tot = 0;
@@ -483,19 +486,18 @@ static int nandsim_show(struct seq_file *m, void *private)
 
 	return 0;
 }
-DEFINE_SHOW_ATTRIBUTE(nandsim);
+DEFINE_SHOW_ATTRIBUTE(ns);
 
 /**
- * nandsim_debugfs_create - initialize debugfs
- * @dev: nandsim device description object
+ * ns_debugfs_create - initialize debugfs
+ * @ns: nandsim device description object
  *
  * This function creates all debugfs files for UBI device @ubi. Returns zero in
  * case of success and a negative error code in case of failure.
  */
-static int nandsim_debugfs_create(struct nandsim *dev)
+static int ns_debugfs_create(struct nandsim *ns)
 {
 	struct dentry *root = nsmtd->dbg.dfs_dir;
-	struct dentry *dent;
 
 	/*
 	 * Just skip debugfs initialization when the debugfs directory is
@@ -508,9 +510,9 @@ static int nandsim_debugfs_create(struct nandsim *dev)
 		return 0;
 	}
 
-	dent = debugfs_create_file("nandsim_wear_report", S_IRUSR,
-				   root, dev, &nandsim_fops);
-	if (IS_ERR_OR_NULL(dent)) {
+	ns->dent = debugfs_create_file("nandsim_wear_report", 0400, root, ns,
+				       &ns_fops);
+	if (IS_ERR_OR_NULL(ns->dent)) {
 		NS_ERR("cannot create \"nandsim_wear_report\" debugfs entry\n");
 		return -1;
 	}
@@ -518,13 +520,18 @@ static int nandsim_debugfs_create(struct nandsim *dev)
 	return 0;
 }
 
+static void ns_debugfs_remove(struct nandsim *ns)
+{
+	debugfs_remove_recursive(ns->dent);
+}
+
 /*
  * Allocate array of page pointers, create slab allocation for an array
  * and initialize the array by NULL pointers.
  *
  * RETURNS: 0 if success, -ENOMEM if memory alloc fails.
  */
-static int __init alloc_device(struct nandsim *ns)
+static int __init ns_alloc_device(struct nandsim *ns)
 {
 	struct file *cfile;
 	int i, err;
@@ -536,12 +543,12 @@ static int __init alloc_device(struct nandsim *ns)
 		if (!(cfile->f_mode & FMODE_CAN_READ)) {
 			NS_ERR("alloc_device: cache file not readable\n");
 			err = -EINVAL;
-			goto err_close;
+			goto err_close_filp;
 		}
 		if (!(cfile->f_mode & FMODE_CAN_WRITE)) {
 			NS_ERR("alloc_device: cache file not writeable\n");
 			err = -EINVAL;
-			goto err_close;
+			goto err_close_filp;
 		}
 		ns->pages_written =
 			vzalloc(array_size(sizeof(unsigned long),
@@ -549,16 +556,24 @@ static int __init alloc_device(struct nandsim *ns)
 		if (!ns->pages_written) {
 			NS_ERR("alloc_device: unable to allocate pages written array\n");
 			err = -ENOMEM;
-			goto err_close;
+			goto err_close_filp;
 		}
 		ns->file_buf = kmalloc(ns->geom.pgszoob, GFP_KERNEL);
 		if (!ns->file_buf) {
 			NS_ERR("alloc_device: unable to allocate file buf\n");
 			err = -ENOMEM;
-			goto err_free;
+			goto err_free_pw;
 		}
 		ns->cfile = cfile;
+
 		return 0;
+
+err_free_pw:
+		vfree(ns->pages_written);
+err_close_filp:
+		filp_close(cfile, NULL);
+
+		return err;
 	}
 
 	ns->pages = vmalloc(array_size(sizeof(union ns_mem), ns->geom.pgnum));
@@ -573,22 +588,22 @@ static int __init alloc_device(struct nandsim *ns)
 						ns->geom.pgszoob, 0, 0, NULL);
 	if (!ns->nand_pages_slab) {
 		NS_ERR("cache_create: unable to create kmem_cache\n");
-		return -ENOMEM;
+		err = -ENOMEM;
+		goto err_free_pg;
 	}
 
 	return 0;
 
-err_free:
-	vfree(ns->pages_written);
-err_close:
-	filp_close(cfile, NULL);
+err_free_pg:
+	vfree(ns->pages);
+
 	return err;
 }
 
 /*
  * Free any allocated pages, and free the array of page pointers.
  */
-static void free_device(struct nandsim *ns)
+static void ns_free_device(struct nandsim *ns)
 {
 	int i;
 
@@ -610,7 +625,7 @@ static void free_device(struct nandsim *ns)
 	}
 }
 
-static char __init *get_partition_name(int i)
+static char __init *ns_get_partition_name(int i)
 {
 	return kasprintf(GFP_KERNEL, "NAND simulator partition %d", i);
 }
@@ -620,7 +635,7 @@ static char __init *get_partition_name(int i)
  *
  * RETURNS: 0 if success, -ERRNO if failure.
  */
-static int __init init_nandsim(struct mtd_info *mtd)
+static int __init ns_init(struct mtd_info *mtd)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct nandsim   *ns   = nand_get_controller_data(chip);
@@ -693,7 +708,7 @@ static int __init init_nandsim(struct mtd_info *mtd)
 			NS_ERR("bad partition size.\n");
 			return -EINVAL;
 		}
-		ns->partitions[i].name   = get_partition_name(i);
+		ns->partitions[i].name = ns_get_partition_name(i);
 		if (!ns->partitions[i].name) {
 			NS_ERR("unable to allocate memory.\n");
 			return -ENOMEM;
@@ -707,12 +722,14 @@ static int __init init_nandsim(struct mtd_info *mtd)
 	if (remains) {
 		if (parts_num + 1 > ARRAY_SIZE(ns->partitions)) {
 			NS_ERR("too many partitions.\n");
-			return -EINVAL;
+			ret = -EINVAL;
+			goto free_partition_names;
 		}
-		ns->partitions[i].name   = get_partition_name(i);
+		ns->partitions[i].name = ns_get_partition_name(i);
 		if (!ns->partitions[i].name) {
 			NS_ERR("unable to allocate memory.\n");
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto free_partition_names;
 		}
 		ns->partitions[i].offset = next_offset;
 		ns->partitions[i].size   = remains;
@@ -739,33 +756,48 @@ static int __init init_nandsim(struct mtd_info *mtd)
 	printk("sector address bytes: %u\n",    ns->geom.secaddrbytes);
 	printk("options: %#x\n",                ns->options);
 
-	if ((ret = alloc_device(ns)) != 0)
-		return ret;
+	ret = ns_alloc_device(ns);
+	if (ret)
+		goto free_partition_names;
 
 	/* Allocate / initialize the internal buffer */
 	ns->buf.byte = kmalloc(ns->geom.pgszoob, GFP_KERNEL);
 	if (!ns->buf.byte) {
 		NS_ERR("init_nandsim: unable to allocate %u bytes for the internal buffer\n",
 			ns->geom.pgszoob);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto free_device;
 	}
 	memset(ns->buf.byte, 0xFF, ns->geom.pgszoob);
 
 	return 0;
+
+free_device:
+	ns_free_device(ns);
+free_partition_names:
+	for (i = 0; i < ARRAY_SIZE(ns->partitions); ++i)
+		kfree(ns->partitions[i].name);
+
+	return ret;
 }
 
 /*
  * Free the nandsim structure.
  */
-static void free_nandsim(struct nandsim *ns)
+static void ns_free(struct nandsim *ns)
 {
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ns->partitions); ++i)
+		kfree(ns->partitions[i].name);
+
 	kfree(ns->buf.byte);
-	free_device(ns);
+	ns_free_device(ns);
 
 	return;
 }
 
-static int parse_badblocks(struct nandsim *ns, struct mtd_info *mtd)
+static int ns_parse_badblocks(struct nandsim *ns, struct mtd_info *mtd)
 {
 	char *w;
 	int zero_ok;
@@ -793,7 +825,7 @@ static int parse_badblocks(struct nandsim *ns, struct mtd_info *mtd)
 	return 0;
 }
 
-static int parse_weakblocks(void)
+static int ns_parse_weakblocks(void)
 {
 	char *w;
 	int zero_ok;
@@ -830,7 +862,7 @@ static int parse_weakblocks(void)
 	return 0;
 }
 
-static int erase_error(unsigned int erase_block_no)
+static int ns_erase_error(unsigned int erase_block_no)
 {
 	struct weak_block *wb;
 
@@ -844,7 +876,7 @@ static int erase_error(unsigned int erase_block_no)
 	return 0;
 }
 
-static int parse_weakpages(void)
+static int ns_parse_weakpages(void)
 {
 	char *w;
 	int zero_ok;
@@ -881,7 +913,7 @@ static int parse_weakpages(void)
 	return 0;
 }
 
-static int write_error(unsigned int page_no)
+static int ns_write_error(unsigned int page_no)
 {
 	struct weak_page *wp;
 
@@ -895,7 +927,7 @@ static int write_error(unsigned int page_no)
 	return 0;
 }
 
-static int parse_gravepages(void)
+static int ns_parse_gravepages(void)
 {
 	char *g;
 	int zero_ok;
@@ -932,7 +964,7 @@ static int parse_gravepages(void)
 	return 0;
 }
 
-static int read_error(unsigned int page_no)
+static int ns_read_error(unsigned int page_no)
 {
 	struct grave_page *gp;
 
@@ -946,25 +978,7 @@ static int read_error(unsigned int page_no)
 	return 0;
 }
 
-static void free_lists(void)
-{
-	struct list_head *pos, *n;
-	list_for_each_safe(pos, n, &weak_blocks) {
-		list_del(pos);
-		kfree(list_entry(pos, struct weak_block, list));
-	}
-	list_for_each_safe(pos, n, &weak_pages) {
-		list_del(pos);
-		kfree(list_entry(pos, struct weak_page, list));
-	}
-	list_for_each_safe(pos, n, &grave_pages) {
-		list_del(pos);
-		kfree(list_entry(pos, struct grave_page, list));
-	}
-	kfree(erase_block_wear);
-}
-
-static int setup_wear_reporting(struct mtd_info *mtd)
+static int ns_setup_wear_reporting(struct mtd_info *mtd)
 {
 	size_t mem;
 
@@ -982,7 +996,7 @@ static int setup_wear_reporting(struct mtd_info *mtd)
 	return 0;
 }
 
-static void update_wear(unsigned int erase_block_no)
+static void ns_update_wear(unsigned int erase_block_no)
 {
 	if (!erase_block_wear)
 		return;
@@ -1001,7 +1015,7 @@ static void update_wear(unsigned int erase_block_no)
 /*
  * Returns the string representation of 'state' state.
  */
-static char *get_state_name(uint32_t state)
+static char *ns_get_state_name(uint32_t state)
 {
 	switch (NS_STATE(state)) {
 		case STATE_CMD_READ0:
@@ -1061,7 +1075,7 @@ static char *get_state_name(uint32_t state)
  *
  * RETURNS: 1 if wrong command, 0 if right.
  */
-static int check_command(int cmd)
+static int ns_check_command(int cmd)
 {
 	switch (cmd) {
 
@@ -1088,7 +1102,7 @@ static int check_command(int cmd)
 /*
  * Returns state after command is accepted by command number.
  */
-static uint32_t get_state_by_command(unsigned command)
+static uint32_t ns_get_state_by_command(unsigned command)
 {
 	switch (command) {
 		case NAND_CMD_READ0:
@@ -1126,7 +1140,7 @@ static uint32_t get_state_by_command(unsigned command)
 /*
  * Move an address byte to the correspondent internal register.
  */
-static inline void accept_addr_byte(struct nandsim *ns, u_char bt)
+static inline void ns_accept_addr_byte(struct nandsim *ns, u_char bt)
 {
 	uint byte = (uint)bt;
 
@@ -1144,9 +1158,10 @@ static inline void accept_addr_byte(struct nandsim *ns, u_char bt)
 /*
  * Switch to STATE_READY state.
  */
-static inline void switch_to_ready_state(struct nandsim *ns, u_char status)
+static inline void ns_switch_to_ready_state(struct nandsim *ns, u_char status)
 {
-	NS_DBG("switch_to_ready_state: switch to %s state\n", get_state_name(STATE_READY));
+	NS_DBG("switch_to_ready_state: switch to %s state\n",
+	       ns_get_state_name(STATE_READY));
 
 	ns->state       = STATE_READY;
 	ns->nxstate     = STATE_UNKNOWN;
@@ -1203,7 +1218,7 @@ static inline void switch_to_ready_state(struct nandsim *ns, u_char status)
  *          -1 - several matches.
  *           0 - operation is found.
  */
-static int find_operation(struct nandsim *ns, uint32_t flag)
+static int ns_find_operation(struct nandsim *ns, uint32_t flag)
 {
 	int opsfound = 0;
 	int i, j, idx = 0;
@@ -1256,7 +1271,8 @@ static int find_operation(struct nandsim *ns, uint32_t flag)
 		ns->state = ns->op[ns->stateidx];
 		ns->nxstate = ns->op[ns->stateidx + 1];
 		NS_DBG("find_operation: operation found, index: %d, state: %s, nxstate %s\n",
-				idx, get_state_name(ns->state), get_state_name(ns->nxstate));
+		       idx, ns_get_state_name(ns->state),
+		       ns_get_state_name(ns->nxstate));
 		return 0;
 	}
 
@@ -1264,13 +1280,13 @@ static int find_operation(struct nandsim *ns, uint32_t flag)
 		/* Nothing was found. Try to ignore previous commands (if any) and search again */
 		if (ns->npstates != 0) {
 			NS_DBG("find_operation: no operation found, try again with state %s\n",
-					get_state_name(ns->state));
+			       ns_get_state_name(ns->state));
 			ns->npstates = 0;
-			return find_operation(ns, 0);
+			return ns_find_operation(ns, 0);
 
 		}
 		NS_DBG("find_operation: no operations found\n");
-		switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+		ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 		return -2;
 	}
 
@@ -1287,7 +1303,7 @@ static int find_operation(struct nandsim *ns, uint32_t flag)
 	return -1;
 }
 
-static void put_pages(struct nandsim *ns)
+static void ns_put_pages(struct nandsim *ns)
 {
 	int i;
 
@@ -1296,7 +1312,8 @@ static void put_pages(struct nandsim *ns)
 }
 
 /* Get page cache pages in advance to provide NOFS memory allocation */
-static int get_pages(struct nandsim *ns, struct file *file, size_t count, loff_t pos)
+static int ns_get_pages(struct nandsim *ns, struct file *file, size_t count,
+			loff_t pos)
 {
 	pgoff_t index, start_index, end_index;
 	struct page *page;
@@ -1316,7 +1333,7 @@ static int get_pages(struct nandsim *ns, struct file *file, size_t count, loff_t
 				page = find_or_create_page(mapping, index, GFP_NOFS);
 			}
 			if (page == NULL) {
-				put_pages(ns);
+				ns_put_pages(ns);
 				return -ENOMEM;
 			}
 			unlock_page(page);
@@ -1326,35 +1343,37 @@ static int get_pages(struct nandsim *ns, struct file *file, size_t count, loff_t
 	return 0;
 }
 
-static ssize_t read_file(struct nandsim *ns, struct file *file, void *buf, size_t count, loff_t pos)
+static ssize_t ns_read_file(struct nandsim *ns, struct file *file, void *buf,
+			    size_t count, loff_t pos)
 {
 	ssize_t tx;
 	int err;
 	unsigned int noreclaim_flag;
 
-	err = get_pages(ns, file, count, pos);
+	err = ns_get_pages(ns, file, count, pos);
 	if (err)
 		return err;
 	noreclaim_flag = memalloc_noreclaim_save();
 	tx = kernel_read(file, buf, count, &pos);
 	memalloc_noreclaim_restore(noreclaim_flag);
-	put_pages(ns);
+	ns_put_pages(ns);
 	return tx;
 }
 
-static ssize_t write_file(struct nandsim *ns, struct file *file, void *buf, size_t count, loff_t pos)
+static ssize_t ns_write_file(struct nandsim *ns, struct file *file, void *buf,
+			     size_t count, loff_t pos)
 {
 	ssize_t tx;
 	int err;
 	unsigned int noreclaim_flag;
 
-	err = get_pages(ns, file, count, pos);
+	err = ns_get_pages(ns, file, count, pos);
 	if (err)
 		return err;
 	noreclaim_flag = memalloc_noreclaim_save();
 	tx = kernel_write(file, buf, count, &pos);
 	memalloc_noreclaim_restore(noreclaim_flag);
-	put_pages(ns);
+	ns_put_pages(ns);
 	return tx;
 }
 
@@ -1374,11 +1393,11 @@ static inline u_char *NS_PAGE_BYTE_OFF(struct nandsim *ns)
 	return NS_GET_PAGE(ns)->byte + ns->regs.column + ns->regs.off;
 }
 
-static int do_read_error(struct nandsim *ns, int num)
+static int ns_do_read_error(struct nandsim *ns, int num)
 {
 	unsigned int page_no = ns->regs.row;
 
-	if (read_error(page_no)) {
+	if (ns_read_error(page_no)) {
 		prandom_bytes(ns->buf.byte, num);
 		NS_WARN("simulating read error in page %u\n", page_no);
 		return 1;
@@ -1386,7 +1405,7 @@ static int do_read_error(struct nandsim *ns, int num)
 	return 0;
 }
 
-static void do_bit_flips(struct nandsim *ns, int num)
+static void ns_do_bit_flips(struct nandsim *ns, int num)
 {
 	if (bitflips && prandom_u32() < (1 << 22)) {
 		int flips = 1;
@@ -1406,7 +1425,7 @@ static void do_bit_flips(struct nandsim *ns, int num)
 /*
  * Fill the NAND buffer with data read from the specified page.
  */
-static void read_page(struct nandsim *ns, int num)
+static void ns_read_page(struct nandsim *ns, int num)
 {
 	union ns_mem *mypage;
 
@@ -1420,15 +1439,16 @@ static void read_page(struct nandsim *ns, int num)
 
 			NS_DBG("read_page: page %d written, reading from %d\n",
 				ns->regs.row, ns->regs.column + ns->regs.off);
-			if (do_read_error(ns, num))
+			if (ns_do_read_error(ns, num))
 				return;
 			pos = (loff_t)NS_RAW_OFFSET(ns) + ns->regs.off;
-			tx = read_file(ns, ns->cfile, ns->buf.byte, num, pos);
+			tx = ns_read_file(ns, ns->cfile, ns->buf.byte, num,
+					  pos);
 			if (tx != num) {
 				NS_ERR("read_page: read error for page %d ret %ld\n", ns->regs.row, (long)tx);
 				return;
 			}
-			do_bit_flips(ns, num);
+			ns_do_bit_flips(ns, num);
 		}
 		return;
 	}
@@ -1440,17 +1460,17 @@ static void read_page(struct nandsim *ns, int num)
 	} else {
 		NS_DBG("read_page: page %d allocated, reading from %d\n",
 			ns->regs.row, ns->regs.column + ns->regs.off);
-		if (do_read_error(ns, num))
+		if (ns_do_read_error(ns, num))
 			return;
 		memcpy(ns->buf.byte, NS_PAGE_BYTE_OFF(ns), num);
-		do_bit_flips(ns, num);
+		ns_do_bit_flips(ns, num);
 	}
 }
 
 /*
  * Erase all pages in the specified sector.
  */
-static void erase_sector(struct nandsim *ns)
+static void ns_erase_sector(struct nandsim *ns)
 {
 	union ns_mem *mypage;
 	int i;
@@ -1478,7 +1498,7 @@ static void erase_sector(struct nandsim *ns)
 /*
  * Program the specified page with the contents from the NAND buffer.
  */
-static int prog_page(struct nandsim *ns, int num)
+static int ns_prog_page(struct nandsim *ns, int num)
 {
 	int i;
 	union ns_mem *mypage;
@@ -1497,7 +1517,7 @@ static int prog_page(struct nandsim *ns, int num)
 			memset(ns->file_buf, 0xff, ns->geom.pgszoob);
 		} else {
 			all = 0;
-			tx = read_file(ns, ns->cfile, pg_off, num, off);
+			tx = ns_read_file(ns, ns->cfile, pg_off, num, off);
 			if (tx != num) {
 				NS_ERR("prog_page: read error for page %d ret %ld\n", ns->regs.row, (long)tx);
 				return -1;
@@ -1507,14 +1527,15 @@ static int prog_page(struct nandsim *ns, int num)
 			pg_off[i] &= ns->buf.byte[i];
 		if (all) {
 			loff_t pos = (loff_t)ns->regs.row * ns->geom.pgszoob;
-			tx = write_file(ns, ns->cfile, ns->file_buf, ns->geom.pgszoob, pos);
+			tx = ns_write_file(ns, ns->cfile, ns->file_buf,
+					   ns->geom.pgszoob, pos);
 			if (tx != ns->geom.pgszoob) {
 				NS_ERR("prog_page: write error for page %d ret %ld\n", ns->regs.row, (long)tx);
 				return -1;
 			}
 			__set_bit(ns->regs.row, ns->pages_written);
 		} else {
-			tx = write_file(ns, ns->cfile, pg_off, num, off);
+			tx = ns_write_file(ns, ns->cfile, pg_off, num, off);
 			if (tx != num) {
 				NS_ERR("prog_page: write error for page %d ret %ld\n", ns->regs.row, (long)tx);
 				return -1;
@@ -1552,7 +1573,7 @@ static int prog_page(struct nandsim *ns, int num)
  *
  * RETURNS: 0 if success, -1 if error.
  */
-static int do_state_action(struct nandsim *ns, uint32_t action)
+static int ns_do_state_action(struct nandsim *ns, uint32_t action)
 {
 	int num;
 	int busdiv = ns->busw == 8 ? 1 : 2;
@@ -1579,7 +1600,7 @@ static int do_state_action(struct nandsim *ns, uint32_t action)
 			break;
 		}
 		num = ns->geom.pgszoob - ns->regs.off - ns->regs.column;
-		read_page(ns, num);
+		ns_read_page(ns, num);
 
 		NS_DBG("do_state_action: (ACTION_CPY:) copy %d bytes to int buf, raw offset %d\n",
 			num, NS_RAW_OFFSET(ns) + ns->regs.off);
@@ -1622,14 +1643,14 @@ static int do_state_action(struct nandsim *ns, uint32_t action)
 				ns->regs.row, NS_RAW_OFFSET(ns));
 		NS_LOG("erase sector %u\n", erase_block_no);
 
-		erase_sector(ns);
+		ns_erase_sector(ns);
 
 		NS_MDELAY(erase_delay);
 
 		if (erase_block_wear)
-			update_wear(erase_block_no);
+			ns_update_wear(erase_block_no);
 
-		if (erase_error(erase_block_no)) {
+		if (ns_erase_error(erase_block_no)) {
 			NS_WARN("simulating erase failure in erase block %u\n", erase_block_no);
 			return -1;
 		}
@@ -1653,7 +1674,7 @@ static int do_state_action(struct nandsim *ns, uint32_t action)
 			return -1;
 		}
 
-		if (prog_page(ns, num) == -1)
+		if (ns_prog_page(ns, num) == -1)
 			return -1;
 
 		page_no = ns->regs.row;
@@ -1665,7 +1686,7 @@ static int do_state_action(struct nandsim *ns, uint32_t action)
 		NS_UDELAY(programm_delay);
 		NS_UDELAY(output_cycle * ns->geom.pgsz / 1000 / busdiv);
 
-		if (write_error(page_no)) {
+		if (ns_write_error(page_no)) {
 			NS_WARN("simulating write failure in page %u\n", page_no);
 			return -1;
 		}
@@ -1702,7 +1723,7 @@ static int do_state_action(struct nandsim *ns, uint32_t action)
 /*
  * Switch simulator's state.
  */
-static void switch_state(struct nandsim *ns)
+static void ns_switch_state(struct nandsim *ns)
 {
 	if (ns->op) {
 		/*
@@ -1716,11 +1737,13 @@ static void switch_state(struct nandsim *ns)
 
 		NS_DBG("switch_state: operation is known, switch to the next state, "
 			"state: %s, nxstate: %s\n",
-			get_state_name(ns->state), get_state_name(ns->nxstate));
+		       ns_get_state_name(ns->state),
+		       ns_get_state_name(ns->nxstate));
 
 		/* See, whether we need to do some action */
-		if ((ns->state & ACTION_MASK) && do_state_action(ns, ns->state) < 0) {
-			switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+		if ((ns->state & ACTION_MASK) &&
+		    ns_do_state_action(ns, ns->state) < 0) {
+			ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 			return;
 		}
 
@@ -1734,15 +1757,16 @@ static void switch_state(struct nandsim *ns)
 		 *  The only event causing the switch_state function to
 		 *  be called with yet unknown operation is new command.
 		 */
-		ns->state = get_state_by_command(ns->regs.command);
+		ns->state = ns_get_state_by_command(ns->regs.command);
 
 		NS_DBG("switch_state: operation is unknown, try to find it\n");
 
-		if (find_operation(ns, 0) != 0)
+		if (!ns_find_operation(ns, 0))
 			return;
 
-		if ((ns->state & ACTION_MASK) && do_state_action(ns, ns->state) < 0) {
-			switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+		if ((ns->state & ACTION_MASK) &&
+		    ns_do_state_action(ns, ns->state) < 0) {
+			ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 			return;
 		}
 	}
@@ -1770,7 +1794,7 @@ static void switch_state(struct nandsim *ns)
 
 		NS_DBG("switch_state: operation complete, switch to STATE_READY state\n");
 
-		switch_to_ready_state(ns, status);
+		ns_switch_to_ready_state(ns, status);
 
 		return;
 	} else if (ns->nxstate & (STATE_DATAIN_MASK | STATE_DATAOUT_MASK)) {
@@ -1784,7 +1808,8 @@ static void switch_state(struct nandsim *ns)
 
 		NS_DBG("switch_state: the next state is data I/O, switch, "
 			"state: %s, nxstate: %s\n",
-			get_state_name(ns->state), get_state_name(ns->nxstate));
+		       ns_get_state_name(ns->state),
+		       ns_get_state_name(ns->nxstate));
 
 		/*
 		 * Set the internal register to the count of bytes which
@@ -1862,8 +1887,8 @@ static u_char ns_nand_read_byte(struct nand_chip *chip)
 		return outb;
 	}
 	if (!(ns->state & STATE_DATAOUT_MASK)) {
-		NS_WARN("read_byte: unexpected data output cycle, state is %s "
-			"return %#x\n", get_state_name(ns->state), (uint)outb);
+		NS_WARN("read_byte: unexpected data output cycle, state is %s return %#x\n",
+			ns_get_state_name(ns->state), (uint)outb);
 		return outb;
 	}
 
@@ -1902,7 +1927,7 @@ static u_char ns_nand_read_byte(struct nand_chip *chip)
 		NS_DBG("read_byte: all bytes were read\n");
 
 		if (NS_STATE(ns->nxstate) == STATE_READY)
-			switch_state(ns);
+			ns_switch_state(ns);
 	}
 
 	return outb;
@@ -1929,12 +1954,12 @@ static void ns_nand_write_byte(struct nand_chip *chip, u_char byte)
 
 		if (byte == NAND_CMD_RESET) {
 			NS_LOG("reset chip\n");
-			switch_to_ready_state(ns, NS_STATUS_OK(ns));
+			ns_switch_to_ready_state(ns, NS_STATUS_OK(ns));
 			return;
 		}
 
 		/* Check that the command byte is correct */
-		if (check_command(byte)) {
+		if (ns_check_command(byte)) {
 			NS_ERR("write_byte: unknown command %#x\n", (uint)byte);
 			return;
 		}
@@ -1943,7 +1968,7 @@ static void ns_nand_write_byte(struct nand_chip *chip, u_char byte)
 			|| NS_STATE(ns->state) == STATE_DATAOUT) {
 			int row = ns->regs.row;
 
-			switch_state(ns);
+			ns_switch_state(ns);
 			if (byte == NAND_CMD_RNDOUT)
 				ns->regs.row = row;
 		}
@@ -1958,16 +1983,17 @@ static void ns_nand_write_byte(struct nand_chip *chip, u_char byte)
 				 * was expected but command was input. In this case ignore
 				 * previous command(s)/state(s) and accept the last one.
 				 */
-				NS_WARN("write_byte: command (%#x) wasn't expected, expected state is %s, "
-					"ignore previous states\n", (uint)byte, get_state_name(ns->nxstate));
+				NS_WARN("write_byte: command (%#x) wasn't expected, expected state is %s, ignore previous states\n",
+					(uint)byte,
+					ns_get_state_name(ns->nxstate));
 			}
-			switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+			ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 		}
 
 		NS_DBG("command byte corresponding to %s state accepted\n",
-			get_state_name(get_state_by_command(byte)));
+			ns_get_state_name(ns_get_state_by_command(byte)));
 		ns->regs.command = byte;
-		switch_state(ns);
+		ns_switch_state(ns);
 
 	} else if (ns->lines.ale == 1) {
 		/*
@@ -1978,11 +2004,13 @@ static void ns_nand_write_byte(struct nand_chip *chip, u_char byte)
 
 			NS_DBG("write_byte: operation isn't known yet, identify it\n");
 
-			if (find_operation(ns, 1) < 0)
+			if (ns_find_operation(ns, 1) < 0)
 				return;
 
-			if ((ns->state & ACTION_MASK) && do_state_action(ns, ns->state) < 0) {
-				switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+			if ((ns->state & ACTION_MASK) &&
+			    ns_do_state_action(ns, ns->state) < 0) {
+				ns_switch_to_ready_state(ns,
+							 NS_STATUS_FAILED(ns));
 				return;
 			}
 
@@ -2004,20 +2032,20 @@ static void ns_nand_write_byte(struct nand_chip *chip, u_char byte)
 
 		/* Check that chip is expecting address */
 		if (!(ns->nxstate & STATE_ADDR_MASK)) {
-			NS_ERR("write_byte: address (%#x) isn't expected, expected state is %s, "
-				"switch to STATE_READY\n", (uint)byte, get_state_name(ns->nxstate));
-			switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+			NS_ERR("write_byte: address (%#x) isn't expected, expected state is %s, switch to STATE_READY\n",
+			       (uint)byte, ns_get_state_name(ns->nxstate));
+			ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 			return;
 		}
 
 		/* Check if this is expected byte */
 		if (ns->regs.count == ns->regs.num) {
 			NS_ERR("write_byte: no more address bytes expected\n");
-			switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+			ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 			return;
 		}
 
-		accept_addr_byte(ns, byte);
+		ns_accept_addr_byte(ns, byte);
 
 		ns->regs.count += 1;
 
@@ -2026,7 +2054,7 @@ static void ns_nand_write_byte(struct nand_chip *chip, u_char byte)
 
 		if (ns->regs.count == ns->regs.num) {
 			NS_DBG("address (%#x, %#x) is accepted\n", ns->regs.row, ns->regs.column);
-			switch_state(ns);
+			ns_switch_state(ns);
 		}
 
 	} else {
@@ -2036,10 +2064,10 @@ static void ns_nand_write_byte(struct nand_chip *chip, u_char byte)
 
 		/* Check that chip is expecting data input */
 		if (!(ns->state & STATE_DATAIN_MASK)) {
-			NS_ERR("write_byte: data input (%#x) isn't expected, state is %s, "
-				"switch to %s\n", (uint)byte,
-				get_state_name(ns->state), get_state_name(STATE_READY));
-			switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+			NS_ERR("write_byte: data input (%#x) isn't expected, state is %s, switch to %s\n",
+			       (uint)byte, ns_get_state_name(ns->state),
+			       ns_get_state_name(STATE_READY));
+			ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 			return;
 		}
 
@@ -2069,16 +2097,16 @@ static void ns_nand_write_buf(struct nand_chip *chip, const u_char *buf,
 
 	/* Check that chip is expecting data input */
 	if (!(ns->state & STATE_DATAIN_MASK)) {
-		NS_ERR("write_buf: data input isn't expected, state is %s, "
-			"switch to STATE_READY\n", get_state_name(ns->state));
-		switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+		NS_ERR("write_buf: data input isn't expected, state is %s, switch to STATE_READY\n",
+		       ns_get_state_name(ns->state));
+		ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 		return;
 	}
 
 	/* Check if these are expected bytes */
 	if (ns->regs.count + len > ns->regs.num) {
 		NS_ERR("write_buf: too many input bytes\n");
-		switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+		ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 		return;
 	}
 
@@ -2105,7 +2133,7 @@ static void ns_nand_read_buf(struct nand_chip *chip, u_char *buf, int len)
 	}
 	if (!(ns->state & STATE_DATAOUT_MASK)) {
 		NS_WARN("read_buf: unexpected data output cycle, current state is %s\n",
-			get_state_name(ns->state));
+			ns_get_state_name(ns->state));
 		return;
 	}
 
@@ -2121,7 +2149,7 @@ static void ns_nand_read_buf(struct nand_chip *chip, u_char *buf, int len)
 	/* Check if these are expected bytes */
 	if (ns->regs.count + len > ns->regs.num) {
 		NS_ERR("read_buf: too many bytes to read\n");
-		switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
+		ns_switch_to_ready_state(ns, NS_STATUS_FAILED(ns));
 		return;
 	}
 
@@ -2130,7 +2158,7 @@ static void ns_nand_read_buf(struct nand_chip *chip, u_char *buf, int len)
 
 	if (ns->regs.count == ns->regs.num) {
 		if (NS_STATE(ns->nxstate) == STATE_READY)
-			switch_state(ns);
+			ns_switch_state(ns);
 	}
 
 	return;
@@ -2144,6 +2172,9 @@ static int ns_exec_op(struct nand_chip *chip, const struct nand_operation *op,
 	const struct nand_op_instr *instr = NULL;
 	struct nandsim *ns = nand_get_controller_data(chip);
 
+	if (check_only)
+		return 0;
+
 	ns->lines.ce = 1;
 
 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
@@ -2224,9 +2255,10 @@ static const struct nand_controller_ops ns_controller_ops = {
  */
 static int __init ns_init_module(void)
 {
+	struct list_head *pos, *n;
 	struct nand_chip *chip;
 	struct nandsim *ns;
-	int retval = -ENOMEM, i;
+	int ret;
 
 	if (bus_width != 8 && bus_width != 16) {
 		NS_ERR("wrong bus width (%d), use only 8 or 16\n", bus_width);
@@ -2259,8 +2291,8 @@ static int __init ns_init_module(void)
 		break;
 	default:
 		NS_ERR("bbt has to be 0..2\n");
-		retval = -EINVAL;
-		goto error;
+		ret = -EINVAL;
+		goto free_ns_struct;
 	}
 	/*
 	 * Perform minimum nandsim structure initialization to handle
@@ -2285,23 +2317,26 @@ static int __init ns_init_module(void)
 
 	nsmtd->owner = THIS_MODULE;
 
-	if ((retval = parse_weakblocks()) != 0)
-		goto error;
+	ret = ns_parse_weakblocks();
+	if (ret)
+		goto free_ns_struct;
 
-	if ((retval = parse_weakpages()) != 0)
-		goto error;
+	ret = ns_parse_weakpages();
+	if (ret)
+		goto free_wb_list;
 
-	if ((retval = parse_gravepages()) != 0)
-		goto error;
+	ret = ns_parse_gravepages();
+	if (ret)
+		goto free_wp_list;
 
 	nand_controller_init(&ns->base);
 	ns->base.ops = &ns_controller_ops;
 	chip->controller = &ns->base;
 
-	retval = nand_scan(chip, 1);
-	if (retval) {
+	ret = nand_scan(chip, 1);
+	if (ret) {
 		NS_ERR("Could not scan NAND Simulator device\n");
-		goto error;
+		goto free_gp_list;
 	}
 
 	if (overridesize) {
@@ -2313,8 +2348,8 @@ static int __init ns_init_module(void)
 
 		if (new_size >> overridesize != nsmtd->erasesize) {
 			NS_ERR("overridesize is too big\n");
-			retval = -EINVAL;
-			goto err_exit;
+			ret = -EINVAL;
+			goto cleanup_nand;
 		}
 
 		/* N.B. This relies on nand_scan not doing anything with the size before we change it */
@@ -2325,39 +2360,60 @@ static int __init ns_init_module(void)
 		chip->pagemask = (targetsize >> chip->page_shift) - 1;
 	}
 
-	if ((retval = setup_wear_reporting(nsmtd)) != 0)
-		goto err_exit;
+	ret = ns_setup_wear_reporting(nsmtd);
+	if (ret)
+		goto cleanup_nand;
 
-	if ((retval = init_nandsim(nsmtd)) != 0)
-		goto err_exit;
+	ret = ns_init(nsmtd);
+	if (ret)
+		goto free_ebw;
 
-	if ((retval = nand_create_bbt(chip)) != 0)
-		goto err_exit;
+	ret = nand_create_bbt(chip);
+	if (ret)
+		goto free_ns_object;
 
-	if ((retval = parse_badblocks(ns, nsmtd)) != 0)
-		goto err_exit;
+	ret = ns_parse_badblocks(ns, nsmtd);
+	if (ret)
+		goto free_ns_object;
 
 	/* Register NAND partitions */
-	retval = mtd_device_register(nsmtd, &ns->partitions[0],
-				     ns->nbparts);
-	if (retval != 0)
-		goto err_exit;
+	ret = mtd_device_register(nsmtd, &ns->partitions[0], ns->nbparts);
+	if (ret)
+		goto free_ns_object;
 
-	if ((retval = nandsim_debugfs_create(ns)) != 0)
-		goto err_exit;
+	ret = ns_debugfs_create(ns);
+	if (ret)
+		goto unregister_mtd;
 
         return 0;
 
-err_exit:
-	free_nandsim(ns);
-	nand_release(chip);
-	for (i = 0;i < ARRAY_SIZE(ns->partitions); ++i)
-		kfree(ns->partitions[i].name);
-error:
+unregister_mtd:
+	WARN_ON(mtd_device_unregister(nsmtd));
+free_ns_object:
+	ns_free(ns);
+free_ebw:
+	kfree(erase_block_wear);
+cleanup_nand:
+	nand_cleanup(chip);
+free_gp_list:
+	list_for_each_safe(pos, n, &grave_pages) {
+		list_del(pos);
+		kfree(list_entry(pos, struct grave_page, list));
+	}
+free_wp_list:
+	list_for_each_safe(pos, n, &weak_pages) {
+		list_del(pos);
+		kfree(list_entry(pos, struct weak_page, list));
+	}
+free_wb_list:
+	list_for_each_safe(pos, n, &weak_blocks) {
+		list_del(pos);
+		kfree(list_entry(pos, struct weak_block, list));
+	}
+free_ns_struct:
 	kfree(ns);
-	free_lists();
 
-	return retval;
+	return ret;
 }
 
 module_init(ns_init_module);
@@ -2369,14 +2425,30 @@ static void __exit ns_cleanup_module(void)
 {
 	struct nand_chip *chip = mtd_to_nand(nsmtd);
 	struct nandsim *ns = nand_get_controller_data(chip);
-	int i;
+	struct list_head *pos, *n;
 
-	free_nandsim(ns);    /* Free nandsim private resources */
-	nand_release(chip); /* Unregister driver */
-	for (i = 0;i < ARRAY_SIZE(ns->partitions); ++i)
-		kfree(ns->partitions[i].name);
-	kfree(ns);        /* Free other structures */
-	free_lists();
+	ns_debugfs_remove(ns);
+	WARN_ON(mtd_device_unregister(nsmtd));
+	ns_free(ns);
+	kfree(erase_block_wear);
+	nand_cleanup(chip);
+
+	list_for_each_safe(pos, n, &grave_pages) {
+		list_del(pos);
+		kfree(list_entry(pos, struct grave_page, list));
+	}
+
+	list_for_each_safe(pos, n, &weak_pages) {
+		list_del(pos);
+		kfree(list_entry(pos, struct weak_page, list));
+	}
+
+	list_for_each_safe(pos, n, &weak_blocks) {
+		list_del(pos);
+		kfree(list_entry(pos, struct weak_block, list));
+	}
+
+	kfree(ns);
 }
 
 module_exit(ns_cleanup_module);
diff --git a/drivers/mtd/nand/raw/ndfc.c b/drivers/mtd/nand/raw/ndfc.c
index d324396ab7ff..ed38338c1383 100644
--- a/drivers/mtd/nand/raw/ndfc.c
+++ b/drivers/mtd/nand/raw/ndfc.c
@@ -244,9 +244,13 @@ static int ndfc_probe(struct platform_device *ofdev)
 static int ndfc_remove(struct platform_device *ofdev)
 {
 	struct ndfc_controller *ndfc = dev_get_drvdata(&ofdev->dev);
-	struct mtd_info *mtd = nand_to_mtd(&ndfc->chip);
+	struct nand_chip *chip = &ndfc->chip;
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int ret;
 
-	nand_release(&ndfc->chip);
+	ret = mtd_device_unregister(mtd);
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	kfree(mtd->name);
 
 	return 0;
diff --git a/drivers/mtd/nand/raw/omap2.c b/drivers/mtd/nand/raw/omap2.c
index ad77c112a78a..eb7fcfd9276b 100644
--- a/drivers/mtd/nand/raw/omap2.c
+++ b/drivers/mtd/nand/raw/omap2.c
@@ -2283,14 +2283,18 @@ static int omap_nand_remove(struct platform_device *pdev)
 	struct mtd_info *mtd = platform_get_drvdata(pdev);
 	struct nand_chip *nand_chip = mtd_to_nand(mtd);
 	struct omap_nand_info *info = mtd_to_omap(mtd);
+	int ret;
+
 	if (nand_chip->ecc.priv) {
 		nand_bch_free(nand_chip->ecc.priv);
 		nand_chip->ecc.priv = NULL;
 	}
 	if (info->dma)
 		dma_release_channel(info->dma);
-	nand_release(nand_chip);
-	return 0;
+	ret = mtd_device_unregister(mtd);
+	WARN_ON(ret);
+	nand_cleanup(nand_chip);
+	return ret;
 }
 
 static const struct of_device_id omap_nand_ids[] = {
diff --git a/drivers/mtd/nand/raw/omap_elm.c b/drivers/mtd/nand/raw/omap_elm.c
index 3fa0e2cbbe53..078b1022ac2a 100644
--- a/drivers/mtd/nand/raw/omap_elm.c
+++ b/drivers/mtd/nand/raw/omap_elm.c
@@ -411,6 +411,7 @@ static int elm_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	if (pm_runtime_get_sync(&pdev->dev) < 0) {
 		ret = -EINVAL;
+		pm_runtime_put_sync(&pdev->dev);
 		pm_runtime_disable(&pdev->dev);
 		dev_err(&pdev->dev, "can't enable clock\n");
 		return ret;
diff --git a/drivers/mtd/nand/raw/orion_nand.c b/drivers/mtd/nand/raw/orion_nand.c
index d27b39a7223c..880b54ca1b41 100644
--- a/drivers/mtd/nand/raw/orion_nand.c
+++ b/drivers/mtd/nand/raw/orion_nand.c
@@ -180,7 +180,7 @@ static int __init orion_nand_probe(struct platform_device *pdev)
 	mtd->name = "orion_nand";
 	ret = mtd_device_register(mtd, board->parts, board->nr_parts);
 	if (ret) {
-		nand_release(nc);
+		nand_cleanup(nc);
 		goto no_dev;
 	}
 
@@ -195,8 +195,12 @@ static int orion_nand_remove(struct platform_device *pdev)
 {
 	struct orion_nand_info *info = platform_get_drvdata(pdev);
 	struct nand_chip *chip = &info->chip;
+	int ret;
 
-	nand_release(chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+
+	nand_cleanup(chip);
 
 	clk_disable_unprepare(info->clk);
 
diff --git a/drivers/mtd/nand/raw/oxnas_nand.c b/drivers/mtd/nand/raw/oxnas_nand.c
index c43cb4d92d3d..8d0d76ad319d 100644
--- a/drivers/mtd/nand/raw/oxnas_nand.c
+++ b/drivers/mtd/nand/raw/oxnas_nand.c
@@ -32,6 +32,7 @@ struct oxnas_nand_ctrl {
 	void __iomem *io_base;
 	struct clk *clk;
 	struct nand_chip *chips[OXNAS_NAND_MAX_CHIPS];
+	unsigned int nchips;
 };
 
 static uint8_t oxnas_nand_read_byte(struct nand_chip *chip)
@@ -79,9 +80,9 @@ static int oxnas_nand_probe(struct platform_device *pdev)
 	struct nand_chip *chip;
 	struct mtd_info *mtd;
 	struct resource *res;
-	int nchips = 0;
 	int count = 0;
 	int err = 0;
+	int i;
 
 	/* Allocate memory for the device structure (and zero it) */
 	oxnas = devm_kzalloc(&pdev->dev, sizeof(*oxnas),
@@ -140,17 +141,15 @@ static int oxnas_nand_probe(struct platform_device *pdev)
 			goto err_release_child;
 
 		err = mtd_device_register(mtd, NULL, 0);
-		if (err) {
-			nand_release(chip);
-			goto err_release_child;
-		}
+		if (err)
+			goto err_cleanup_nand;
 
-		oxnas->chips[nchips] = chip;
-		++nchips;
+		oxnas->chips[oxnas->nchips] = chip;
+		++oxnas->nchips;
 	}
 
 	/* Exit if no chips found */
-	if (!nchips) {
+	if (!oxnas->nchips) {
 		err = -ENODEV;
 		goto err_clk_unprepare;
 	}
@@ -159,8 +158,17 @@ static int oxnas_nand_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_cleanup_nand:
+	nand_cleanup(chip);
 err_release_child:
 	of_node_put(nand_np);
+
+	for (i = 0; i < oxnas->nchips; i++) {
+		chip = oxnas->chips[i];
+		WARN_ON(mtd_device_unregister(nand_to_mtd(chip)));
+		nand_cleanup(chip);
+	}
+
 err_clk_unprepare:
 	clk_disable_unprepare(oxnas->clk);
 	return err;
@@ -169,9 +177,14 @@ err_clk_unprepare:
 static int oxnas_nand_remove(struct platform_device *pdev)
 {
 	struct oxnas_nand_ctrl *oxnas = platform_get_drvdata(pdev);
+	struct nand_chip *chip;
+	int i;
 
-	if (oxnas->chips[0])
-		nand_release(oxnas->chips[0]);
+	for (i = 0; i < oxnas->nchips; i++) {
+		chip = oxnas->chips[i];
+		WARN_ON(mtd_device_unregister(nand_to_mtd(chip)));
+		nand_cleanup(chip);
+	}
 
 	clk_disable_unprepare(oxnas->clk);
 
diff --git a/drivers/mtd/nand/raw/pasemi_nand.c b/drivers/mtd/nand/raw/pasemi_nand.c
index 9cfe7395172a..d8eca8c3fdcd 100644
--- a/drivers/mtd/nand/raw/pasemi_nand.c
+++ b/drivers/mtd/nand/raw/pasemi_nand.c
@@ -146,7 +146,7 @@ static int pasemi_nand_probe(struct platform_device *ofdev)
 	if (mtd_device_register(pasemi_nand_mtd, NULL, 0)) {
 		dev_err(dev, "Unable to register MTD device\n");
 		err = -ENODEV;
-		goto out_lpc;
+		goto out_cleanup_nand;
 	}
 
 	dev_info(dev, "PA Semi NAND flash at %pR, control at I/O %x\n", &res,
@@ -154,6 +154,8 @@ static int pasemi_nand_probe(struct platform_device *ofdev)
 
 	return 0;
 
+ out_cleanup_nand:
+	nand_cleanup(chip);
  out_lpc:
 	release_region(lpcctl, 4);
  out_ior:
@@ -167,6 +169,7 @@ static int pasemi_nand_probe(struct platform_device *ofdev)
 static int pasemi_nand_remove(struct platform_device *ofdev)
 {
 	struct nand_chip *chip;
+	int ret;
 
 	if (!pasemi_nand_mtd)
 		return 0;
@@ -174,7 +177,9 @@ static int pasemi_nand_remove(struct platform_device *ofdev)
 	chip = mtd_to_nand(pasemi_nand_mtd);
 
 	/* Release resources, unregister device */
-	nand_release(chip);
+	ret = mtd_device_unregister(pasemi_nand_mtd);
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
 	release_region(lpcctl, 4);
 
diff --git a/drivers/mtd/nand/raw/plat_nand.c b/drivers/mtd/nand/raw/plat_nand.c
index dc0f3074ddbf..556182f26057 100644
--- a/drivers/mtd/nand/raw/plat_nand.c
+++ b/drivers/mtd/nand/raw/plat_nand.c
@@ -92,7 +92,7 @@ static int plat_nand_probe(struct platform_device *pdev)
 	if (!err)
 		return err;
 
-	nand_release(&data->chip);
+	nand_cleanup(&data->chip);
 out:
 	if (pdata->ctrl.remove)
 		pdata->ctrl.remove(pdev);
@@ -106,8 +106,12 @@ static int plat_nand_remove(struct platform_device *pdev)
 {
 	struct plat_nand_data *data = platform_get_drvdata(pdev);
 	struct platform_nand_data *pdata = dev_get_platdata(&pdev->dev);
+	struct nand_chip *chip = &data->chip;
+	int ret;
 
-	nand_release(&data->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	if (pdata->ctrl.remove)
 		pdata->ctrl.remove(pdev);
 
diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c
index 5b11c7061497..f1daf330951b 100644
--- a/drivers/mtd/nand/raw/qcom_nandc.c
+++ b/drivers/mtd/nand/raw/qcom_nandc.c
@@ -2836,7 +2836,7 @@ static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
 	chip->legacy.block_markbad	= qcom_nandc_block_markbad;
 
 	chip->controller = &nandc->controller;
-	chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
+	chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA |
 			 NAND_SKIP_BBTSCAN;
 
 	/* set up initial status value */
@@ -3005,10 +3005,15 @@ static int qcom_nandc_remove(struct platform_device *pdev)
 	struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
 	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	struct qcom_nand_host *host;
+	struct nand_chip *chip;
+	int ret;
 
-	list_for_each_entry(host, &nandc->host_list, node)
-		nand_release(&host->chip);
-
+	list_for_each_entry(host, &nandc->host_list, node) {
+		chip = &host->chip;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+	}
 
 	qcom_nandc_unalloc(nandc);
 
diff --git a/drivers/mtd/nand/raw/r852.c b/drivers/mtd/nand/raw/r852.c
index 77774250fb11..f865e3a47b01 100644
--- a/drivers/mtd/nand/raw/r852.c
+++ b/drivers/mtd/nand/raw/r852.c
@@ -651,7 +651,8 @@ static int r852_register_nand_device(struct r852_device *dev)
 	dev->card_registered = 1;
 	return 0;
 error3:
-	nand_release(dev->chip);
+	WARN_ON(mtd_device_unregister(nand_to_mtd(dev->chip)));
+	nand_cleanup(dev->chip);
 error1:
 	/* Force card redetect */
 	dev->card_detected = 0;
@@ -670,7 +671,8 @@ static void r852_unregister_nand_device(struct r852_device *dev)
 		return;
 
 	device_remove_file(&mtd->dev, &dev_attr_media_type);
-	nand_release(dev->chip);
+	WARN_ON(mtd_device_unregister(mtd));
+	nand_cleanup(dev->chip);
 	r852_engine_disable(dev);
 	dev->card_registered = 0;
 }
diff --git a/drivers/mtd/nand/raw/s3c2410.c b/drivers/mtd/nand/raw/s3c2410.c
index 0009c1820e21..f86dff311464 100644
--- a/drivers/mtd/nand/raw/s3c2410.c
+++ b/drivers/mtd/nand/raw/s3c2410.c
@@ -779,7 +779,8 @@ static int s3c24xx_nand_remove(struct platform_device *pdev)
 
 		for (mtdno = 0; mtdno < info->mtd_count; mtdno++, ptr++) {
 			pr_debug("releasing mtd %d (%p)\n", mtdno, ptr);
-			nand_release(&ptr->chip);
+			WARN_ON(mtd_device_unregister(nand_to_mtd(&ptr->chip)));
+			nand_cleanup(&ptr->chip);
 		}
 	}
 
diff --git a/drivers/mtd/nand/raw/sh_flctl.c b/drivers/mtd/nand/raw/sh_flctl.c
index 058e99d0cbcf..a661b8bb2dd5 100644
--- a/drivers/mtd/nand/raw/sh_flctl.c
+++ b/drivers/mtd/nand/raw/sh_flctl.c
@@ -1204,9 +1204,13 @@ err_chip:
 static int flctl_remove(struct platform_device *pdev)
 {
 	struct sh_flctl *flctl = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &flctl->chip;
+	int ret;
 
 	flctl_release_dma(flctl);
-	nand_release(&flctl->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	pm_runtime_disable(&pdev->dev);
 
 	return 0;
diff --git a/drivers/mtd/nand/raw/sharpsl.c b/drivers/mtd/nand/raw/sharpsl.c
index b47a9eaff89b..51286f7acf54 100644
--- a/drivers/mtd/nand/raw/sharpsl.c
+++ b/drivers/mtd/nand/raw/sharpsl.c
@@ -183,7 +183,7 @@ static int sharpsl_nand_probe(struct platform_device *pdev)
 	return 0;
 
 err_add:
-	nand_release(this);
+	nand_cleanup(this);
 
 err_scan:
 	iounmap(sharpsl->io);
@@ -199,13 +199,19 @@ err_get_res:
 static int sharpsl_nand_remove(struct platform_device *pdev)
 {
 	struct sharpsl_nand *sharpsl = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &sharpsl->chip;
+	int ret;
 
-	/* Release resources, unregister device */
-	nand_release(&sharpsl->chip);
+	/* Unregister device */
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+
+	/* Release resources */
+	nand_cleanup(chip);
 
 	iounmap(sharpsl->io);
 
-	/* Free the MTD device structure */
+	/* Free the driver's structure */
 	kfree(sharpsl);
 
 	return 0;
diff --git a/drivers/mtd/nand/raw/socrates_nand.c b/drivers/mtd/nand/raw/socrates_nand.c
index 20f40c0e812c..243b34cfbc1b 100644
--- a/drivers/mtd/nand/raw/socrates_nand.c
+++ b/drivers/mtd/nand/raw/socrates_nand.c
@@ -169,7 +169,7 @@ static int socrates_nand_probe(struct platform_device *ofdev)
 	if (!res)
 		return res;
 
-	nand_release(nand_chip);
+	nand_cleanup(nand_chip);
 
 out:
 	iounmap(host->io_base);
@@ -182,8 +182,12 @@ out:
 static int socrates_nand_remove(struct platform_device *ofdev)
 {
 	struct socrates_nand_host *host = dev_get_drvdata(&ofdev->dev);
+	struct nand_chip *chip = &host->nand_chip;
+	int ret;
 
-	nand_release(&host->nand_chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
 	iounmap(host->io_base);
 
diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c
index b6d45cd911ae..65c9d17b25a3 100644
--- a/drivers/mtd/nand/raw/stm32_fmc2_nand.c
+++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c
@@ -4,6 +4,7 @@
  * Author: Christophe Kerello <christophe.kerello@st.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
@@ -37,8 +38,7 @@
 /* Max ECC buffer length */
 #define FMC2_MAX_ECC_BUF_LEN		(FMC2_BCHDSRS_LEN * FMC2_MAX_SG)
 
-#define FMC2_TIMEOUT_US			1000
-#define FMC2_TIMEOUT_MS			1000
+#define FMC2_TIMEOUT_MS			5000
 
 /* Timings */
 #define FMC2_THIZ			1
@@ -85,20 +85,16 @@
 /* Register: FMC2_PCR */
 #define FMC2_PCR_PWAITEN		BIT(1)
 #define FMC2_PCR_PBKEN			BIT(2)
-#define FMC2_PCR_PWID_MASK		GENMASK(5, 4)
-#define FMC2_PCR_PWID(x)		(((x) & 0x3) << 4)
+#define FMC2_PCR_PWID			GENMASK(5, 4)
 #define FMC2_PCR_PWID_BUSWIDTH_8	0
 #define FMC2_PCR_PWID_BUSWIDTH_16	1
 #define FMC2_PCR_ECCEN			BIT(6)
 #define FMC2_PCR_ECCALG			BIT(8)
-#define FMC2_PCR_TCLR_MASK		GENMASK(12, 9)
-#define FMC2_PCR_TCLR(x)		(((x) & 0xf) << 9)
+#define FMC2_PCR_TCLR			GENMASK(12, 9)
 #define FMC2_PCR_TCLR_DEFAULT		0xf
-#define FMC2_PCR_TAR_MASK		GENMASK(16, 13)
-#define FMC2_PCR_TAR(x)			(((x) & 0xf) << 13)
+#define FMC2_PCR_TAR			GENMASK(16, 13)
 #define FMC2_PCR_TAR_DEFAULT		0xf
-#define FMC2_PCR_ECCSS_MASK		GENMASK(19, 17)
-#define FMC2_PCR_ECCSS(x)		(((x) & 0x7) << 17)
+#define FMC2_PCR_ECCSS			GENMASK(19, 17)
 #define FMC2_PCR_ECCSS_512		1
 #define FMC2_PCR_ECCSS_2048		3
 #define FMC2_PCR_BCHECC			BIT(24)
@@ -108,17 +104,17 @@
 #define FMC2_SR_NWRF			BIT(6)
 
 /* Register: FMC2_PMEM */
-#define FMC2_PMEM_MEMSET(x)		(((x) & 0xff) << 0)
-#define FMC2_PMEM_MEMWAIT(x)		(((x) & 0xff) << 8)
-#define FMC2_PMEM_MEMHOLD(x)		(((x) & 0xff) << 16)
-#define FMC2_PMEM_MEMHIZ(x)		(((x) & 0xff) << 24)
+#define FMC2_PMEM_MEMSET		GENMASK(7, 0)
+#define FMC2_PMEM_MEMWAIT		GENMASK(15, 8)
+#define FMC2_PMEM_MEMHOLD		GENMASK(23, 16)
+#define FMC2_PMEM_MEMHIZ		GENMASK(31, 24)
 #define FMC2_PMEM_DEFAULT		0x0a0a0a0a
 
 /* Register: FMC2_PATT */
-#define FMC2_PATT_ATTSET(x)		(((x) & 0xff) << 0)
-#define FMC2_PATT_ATTWAIT(x)		(((x) & 0xff) << 8)
-#define FMC2_PATT_ATTHOLD(x)		(((x) & 0xff) << 16)
-#define FMC2_PATT_ATTHIZ(x)		(((x) & 0xff) << 24)
+#define FMC2_PATT_ATTSET		GENMASK(7, 0)
+#define FMC2_PATT_ATTWAIT		GENMASK(15, 8)
+#define FMC2_PATT_ATTHOLD		GENMASK(23, 16)
+#define FMC2_PATT_ATTHIZ		GENMASK(31, 24)
 #define FMC2_PATT_DEFAULT		0x0a0a0a0a
 
 /* Register: FMC2_ISR */
@@ -133,9 +129,9 @@
 /* Register: FMC2_CSQCFGR1 */
 #define FMC2_CSQCFGR1_CMD2EN		BIT(1)
 #define FMC2_CSQCFGR1_DMADEN		BIT(2)
-#define FMC2_CSQCFGR1_ACYNBR(x)		(((x) & 0x7) << 4)
-#define FMC2_CSQCFGR1_CMD1(x)		(((x) & 0xff) << 8)
-#define FMC2_CSQCFGR1_CMD2(x)		(((x) & 0xff) << 16)
+#define FMC2_CSQCFGR1_ACYNBR		GENMASK(6, 4)
+#define FMC2_CSQCFGR1_CMD1		GENMASK(15, 8)
+#define FMC2_CSQCFGR1_CMD2		GENMASK(23, 16)
 #define FMC2_CSQCFGR1_CMD1T		BIT(24)
 #define FMC2_CSQCFGR1_CMD2T		BIT(25)
 
@@ -143,13 +139,13 @@
 #define FMC2_CSQCFGR2_SQSDTEN		BIT(0)
 #define FMC2_CSQCFGR2_RCMD2EN		BIT(1)
 #define FMC2_CSQCFGR2_DMASEN		BIT(2)
-#define FMC2_CSQCFGR2_RCMD1(x)		(((x) & 0xff) << 8)
-#define FMC2_CSQCFGR2_RCMD2(x)		(((x) & 0xff) << 16)
+#define FMC2_CSQCFGR2_RCMD1		GENMASK(15, 8)
+#define FMC2_CSQCFGR2_RCMD2		GENMASK(23, 16)
 #define FMC2_CSQCFGR2_RCMD1T		BIT(24)
 #define FMC2_CSQCFGR2_RCMD2T		BIT(25)
 
 /* Register: FMC2_CSQCFGR3 */
-#define FMC2_CSQCFGR3_SNBR(x)		(((x) & 0x1f) << 8)
+#define FMC2_CSQCFGR3_SNBR		GENMASK(13, 8)
 #define FMC2_CSQCFGR3_AC1T		BIT(16)
 #define FMC2_CSQCFGR3_AC2T		BIT(17)
 #define FMC2_CSQCFGR3_AC3T		BIT(18)
@@ -160,15 +156,15 @@
 #define FMC2_CSQCFGR3_RAC2T		BIT(23)
 
 /* Register: FMC2_CSQCAR1 */
-#define FMC2_CSQCAR1_ADDC1(x)		(((x) & 0xff) << 0)
-#define FMC2_CSQCAR1_ADDC2(x)		(((x) & 0xff) << 8)
-#define FMC2_CSQCAR1_ADDC3(x)		(((x) & 0xff) << 16)
-#define FMC2_CSQCAR1_ADDC4(x)		(((x) & 0xff) << 24)
+#define FMC2_CSQCAR1_ADDC1		GENMASK(7, 0)
+#define FMC2_CSQCAR1_ADDC2		GENMASK(15, 8)
+#define FMC2_CSQCAR1_ADDC3		GENMASK(23, 16)
+#define FMC2_CSQCAR1_ADDC4		GENMASK(31, 24)
 
 /* Register: FMC2_CSQCAR2 */
-#define FMC2_CSQCAR2_ADDC5(x)		(((x) & 0xff) << 0)
-#define FMC2_CSQCAR2_NANDCEN(x)		(((x) & 0x3) << 10)
-#define FMC2_CSQCAR2_SAO(x)		(((x) & 0xffff) << 16)
+#define FMC2_CSQCAR2_ADDC5		GENMASK(7, 0)
+#define FMC2_CSQCAR2_NANDCEN		GENMASK(11, 10)
+#define FMC2_CSQCAR2_SAO		GENMASK(31, 16)
 
 /* Register: FMC2_CSQIER */
 #define FMC2_CSQIER_TCIE		BIT(0)
@@ -189,28 +185,23 @@
 /* Register: FMC2_BCHDSR0 */
 #define FMC2_BCHDSR0_DUE		BIT(0)
 #define FMC2_BCHDSR0_DEF		BIT(1)
-#define FMC2_BCHDSR0_DEN_MASK		GENMASK(7, 4)
-#define FMC2_BCHDSR0_DEN_SHIFT		4
+#define FMC2_BCHDSR0_DEN		GENMASK(7, 4)
 
 /* Register: FMC2_BCHDSR1 */
-#define FMC2_BCHDSR1_EBP1_MASK		GENMASK(12, 0)
-#define FMC2_BCHDSR1_EBP2_MASK		GENMASK(28, 16)
-#define FMC2_BCHDSR1_EBP2_SHIFT		16
+#define FMC2_BCHDSR1_EBP1		GENMASK(12, 0)
+#define FMC2_BCHDSR1_EBP2		GENMASK(28, 16)
 
 /* Register: FMC2_BCHDSR2 */
-#define FMC2_BCHDSR2_EBP3_MASK		GENMASK(12, 0)
-#define FMC2_BCHDSR2_EBP4_MASK		GENMASK(28, 16)
-#define FMC2_BCHDSR2_EBP4_SHIFT		16
+#define FMC2_BCHDSR2_EBP3		GENMASK(12, 0)
+#define FMC2_BCHDSR2_EBP4		GENMASK(28, 16)
 
 /* Register: FMC2_BCHDSR3 */
-#define FMC2_BCHDSR3_EBP5_MASK		GENMASK(12, 0)
-#define FMC2_BCHDSR3_EBP6_MASK		GENMASK(28, 16)
-#define FMC2_BCHDSR3_EBP6_SHIFT		16
+#define FMC2_BCHDSR3_EBP5		GENMASK(12, 0)
+#define FMC2_BCHDSR3_EBP6		GENMASK(28, 16)
 
 /* Register: FMC2_BCHDSR4 */
-#define FMC2_BCHDSR4_EBP7_MASK		GENMASK(12, 0)
-#define FMC2_BCHDSR4_EBP8_MASK		GENMASK(28, 16)
-#define FMC2_BCHDSR4_EBP8_SHIFT		16
+#define FMC2_BCHDSR4_EBP7		GENMASK(12, 0)
+#define FMC2_BCHDSR4_EBP8		GENMASK(28, 16)
 
 enum stm32_fmc2_ecc {
 	FMC2_ECC_HAM = 1,
@@ -281,43 +272,41 @@ static inline struct stm32_fmc2_nfc *to_stm32_nfc(struct nand_controller *base)
 	return container_of(base, struct stm32_fmc2_nfc, base);
 }
 
-/* Timings configuration */
-static void stm32_fmc2_timings_init(struct nand_chip *chip)
+static void stm32_fmc2_nfc_timings_init(struct nand_chip *chip)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	struct stm32_fmc2_nand *nand = to_fmc2_nand(chip);
 	struct stm32_fmc2_timings *timings = &nand->timings;
-	u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR);
+	u32 pcr = readl_relaxed(nfc->io_base + FMC2_PCR);
 	u32 pmem, patt;
 
 	/* Set tclr/tar timings */
-	pcr &= ~FMC2_PCR_TCLR_MASK;
-	pcr |= FMC2_PCR_TCLR(timings->tclr);
-	pcr &= ~FMC2_PCR_TAR_MASK;
-	pcr |= FMC2_PCR_TAR(timings->tar);
+	pcr &= ~FMC2_PCR_TCLR;
+	pcr |= FIELD_PREP(FMC2_PCR_TCLR, timings->tclr);
+	pcr &= ~FMC2_PCR_TAR;
+	pcr |= FIELD_PREP(FMC2_PCR_TAR, timings->tar);
 
 	/* Set tset/twait/thold/thiz timings in common bank */
-	pmem = FMC2_PMEM_MEMSET(timings->tset_mem);
-	pmem |= FMC2_PMEM_MEMWAIT(timings->twait);
-	pmem |= FMC2_PMEM_MEMHOLD(timings->thold_mem);
-	pmem |= FMC2_PMEM_MEMHIZ(timings->thiz);
+	pmem = FIELD_PREP(FMC2_PMEM_MEMSET, timings->tset_mem);
+	pmem |= FIELD_PREP(FMC2_PMEM_MEMWAIT, timings->twait);
+	pmem |= FIELD_PREP(FMC2_PMEM_MEMHOLD, timings->thold_mem);
+	pmem |= FIELD_PREP(FMC2_PMEM_MEMHIZ, timings->thiz);
 
 	/* Set tset/twait/thold/thiz timings in attribut bank */
-	patt = FMC2_PATT_ATTSET(timings->tset_att);
-	patt |= FMC2_PATT_ATTWAIT(timings->twait);
-	patt |= FMC2_PATT_ATTHOLD(timings->thold_att);
-	patt |= FMC2_PATT_ATTHIZ(timings->thiz);
-
-	writel_relaxed(pcr, fmc2->io_base + FMC2_PCR);
-	writel_relaxed(pmem, fmc2->io_base + FMC2_PMEM);
-	writel_relaxed(patt, fmc2->io_base + FMC2_PATT);
+	patt = FIELD_PREP(FMC2_PATT_ATTSET, timings->tset_att);
+	patt |= FIELD_PREP(FMC2_PATT_ATTWAIT, timings->twait);
+	patt |= FIELD_PREP(FMC2_PATT_ATTHOLD, timings->thold_att);
+	patt |= FIELD_PREP(FMC2_PATT_ATTHIZ, timings->thiz);
+
+	writel_relaxed(pcr, nfc->io_base + FMC2_PCR);
+	writel_relaxed(pmem, nfc->io_base + FMC2_PMEM);
+	writel_relaxed(patt, nfc->io_base + FMC2_PATT);
 }
 
-/* Controller configuration */
-static void stm32_fmc2_setup(struct nand_chip *chip)
+static void stm32_fmc2_nfc_setup(struct nand_chip *chip)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
-	u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
+	u32 pcr = readl_relaxed(nfc->io_base + FMC2_PCR);
 
 	/* Configure ECC algorithm (default configuration is Hamming) */
 	pcr &= ~FMC2_PCR_ECCALG;
@@ -330,195 +319,182 @@ static void stm32_fmc2_setup(struct nand_chip *chip)
 	}
 
 	/* Set buswidth */
-	pcr &= ~FMC2_PCR_PWID_MASK;
+	pcr &= ~FMC2_PCR_PWID;
 	if (chip->options & NAND_BUSWIDTH_16)
-		pcr |= FMC2_PCR_PWID(FMC2_PCR_PWID_BUSWIDTH_16);
+		pcr |= FIELD_PREP(FMC2_PCR_PWID, FMC2_PCR_PWID_BUSWIDTH_16);
 
 	/* Set ECC sector size */
-	pcr &= ~FMC2_PCR_ECCSS_MASK;
-	pcr |= FMC2_PCR_ECCSS(FMC2_PCR_ECCSS_512);
+	pcr &= ~FMC2_PCR_ECCSS;
+	pcr |= FIELD_PREP(FMC2_PCR_ECCSS, FMC2_PCR_ECCSS_512);
 
-	writel_relaxed(pcr, fmc2->io_base + FMC2_PCR);
+	writel_relaxed(pcr, nfc->io_base + FMC2_PCR);
 }
 
-/* Select target */
-static int stm32_fmc2_select_chip(struct nand_chip *chip, int chipnr)
+static int stm32_fmc2_nfc_select_chip(struct nand_chip *chip, int chipnr)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	struct stm32_fmc2_nand *nand = to_fmc2_nand(chip);
 	struct dma_slave_config dma_cfg;
 	int ret;
 
-	if (nand->cs_used[chipnr] == fmc2->cs_sel)
+	if (nand->cs_used[chipnr] == nfc->cs_sel)
 		return 0;
 
-	fmc2->cs_sel = nand->cs_used[chipnr];
+	nfc->cs_sel = nand->cs_used[chipnr];
+	stm32_fmc2_nfc_setup(chip);
+	stm32_fmc2_nfc_timings_init(chip);
 
-	/* FMC2 setup routine */
-	stm32_fmc2_setup(chip);
-
-	/* Apply timings */
-	stm32_fmc2_timings_init(chip);
-
-	if (fmc2->dma_tx_ch && fmc2->dma_rx_ch) {
+	if (nfc->dma_tx_ch && nfc->dma_rx_ch) {
 		memset(&dma_cfg, 0, sizeof(dma_cfg));
-		dma_cfg.src_addr = fmc2->data_phys_addr[fmc2->cs_sel];
-		dma_cfg.dst_addr = fmc2->data_phys_addr[fmc2->cs_sel];
+		dma_cfg.src_addr = nfc->data_phys_addr[nfc->cs_sel];
+		dma_cfg.dst_addr = nfc->data_phys_addr[nfc->cs_sel];
 		dma_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 		dma_cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 		dma_cfg.src_maxburst = 32;
 		dma_cfg.dst_maxburst = 32;
 
-		ret = dmaengine_slave_config(fmc2->dma_tx_ch, &dma_cfg);
+		ret = dmaengine_slave_config(nfc->dma_tx_ch, &dma_cfg);
 		if (ret) {
-			dev_err(fmc2->dev, "tx DMA engine slave config failed\n");
+			dev_err(nfc->dev, "tx DMA engine slave config failed\n");
 			return ret;
 		}
 
-		ret = dmaengine_slave_config(fmc2->dma_rx_ch, &dma_cfg);
+		ret = dmaengine_slave_config(nfc->dma_rx_ch, &dma_cfg);
 		if (ret) {
-			dev_err(fmc2->dev, "rx DMA engine slave config failed\n");
+			dev_err(nfc->dev, "rx DMA engine slave config failed\n");
 			return ret;
 		}
 	}
 
-	if (fmc2->dma_ecc_ch) {
+	if (nfc->dma_ecc_ch) {
 		/*
 		 * Hamming: we read HECCR register
 		 * BCH4/BCH8: we read BCHDSRSx registers
 		 */
 		memset(&dma_cfg, 0, sizeof(dma_cfg));
-		dma_cfg.src_addr = fmc2->io_phys_addr;
+		dma_cfg.src_addr = nfc->io_phys_addr;
 		dma_cfg.src_addr += chip->ecc.strength == FMC2_ECC_HAM ?
 				    FMC2_HECCR : FMC2_BCHDSR0;
 		dma_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 
-		ret = dmaengine_slave_config(fmc2->dma_ecc_ch, &dma_cfg);
+		ret = dmaengine_slave_config(nfc->dma_ecc_ch, &dma_cfg);
 		if (ret) {
-			dev_err(fmc2->dev, "ECC DMA engine slave config failed\n");
+			dev_err(nfc->dev, "ECC DMA engine slave config failed\n");
 			return ret;
 		}
 
 		/* Calculate ECC length needed for one sector */
-		fmc2->dma_ecc_len = chip->ecc.strength == FMC2_ECC_HAM ?
-				    FMC2_HECCR_LEN : FMC2_BCHDSRS_LEN;
+		nfc->dma_ecc_len = chip->ecc.strength == FMC2_ECC_HAM ?
+				   FMC2_HECCR_LEN : FMC2_BCHDSRS_LEN;
 	}
 
 	return 0;
 }
 
-/* Set bus width to 16-bit or 8-bit */
-static void stm32_fmc2_set_buswidth_16(struct stm32_fmc2_nfc *fmc2, bool set)
+static void stm32_fmc2_nfc_set_buswidth_16(struct stm32_fmc2_nfc *nfc, bool set)
 {
-	u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR);
+	u32 pcr = readl_relaxed(nfc->io_base + FMC2_PCR);
 
-	pcr &= ~FMC2_PCR_PWID_MASK;
+	pcr &= ~FMC2_PCR_PWID;
 	if (set)
-		pcr |= FMC2_PCR_PWID(FMC2_PCR_PWID_BUSWIDTH_16);
-	writel_relaxed(pcr, fmc2->io_base + FMC2_PCR);
+		pcr |= FIELD_PREP(FMC2_PCR_PWID, FMC2_PCR_PWID_BUSWIDTH_16);
+	writel_relaxed(pcr, nfc->io_base + FMC2_PCR);
 }
 
-/* Enable/disable ECC */
-static void stm32_fmc2_set_ecc(struct stm32_fmc2_nfc *fmc2, bool enable)
+static void stm32_fmc2_nfc_set_ecc(struct stm32_fmc2_nfc *nfc, bool enable)
 {
-	u32 pcr = readl(fmc2->io_base + FMC2_PCR);
+	u32 pcr = readl(nfc->io_base + FMC2_PCR);
 
 	pcr &= ~FMC2_PCR_ECCEN;
 	if (enable)
 		pcr |= FMC2_PCR_ECCEN;
-	writel(pcr, fmc2->io_base + FMC2_PCR);
+	writel(pcr, nfc->io_base + FMC2_PCR);
 }
 
-/* Enable irq sources in case of the sequencer is used */
-static inline void stm32_fmc2_enable_seq_irq(struct stm32_fmc2_nfc *fmc2)
+static inline void stm32_fmc2_nfc_enable_seq_irq(struct stm32_fmc2_nfc *nfc)
 {
-	u32 csqier = readl_relaxed(fmc2->io_base + FMC2_CSQIER);
+	u32 csqier = readl_relaxed(nfc->io_base + FMC2_CSQIER);
 
 	csqier |= FMC2_CSQIER_TCIE;
 
-	fmc2->irq_state = FMC2_IRQ_SEQ;
+	nfc->irq_state = FMC2_IRQ_SEQ;
 
-	writel_relaxed(csqier, fmc2->io_base + FMC2_CSQIER);
+	writel_relaxed(csqier, nfc->io_base + FMC2_CSQIER);
 }
 
-/* Disable irq sources in case of the sequencer is used */
-static inline void stm32_fmc2_disable_seq_irq(struct stm32_fmc2_nfc *fmc2)
+static inline void stm32_fmc2_nfc_disable_seq_irq(struct stm32_fmc2_nfc *nfc)
 {
-	u32 csqier = readl_relaxed(fmc2->io_base + FMC2_CSQIER);
+	u32 csqier = readl_relaxed(nfc->io_base + FMC2_CSQIER);
 
 	csqier &= ~FMC2_CSQIER_TCIE;
 
-	writel_relaxed(csqier, fmc2->io_base + FMC2_CSQIER);
+	writel_relaxed(csqier, nfc->io_base + FMC2_CSQIER);
 
-	fmc2->irq_state = FMC2_IRQ_UNKNOWN;
+	nfc->irq_state = FMC2_IRQ_UNKNOWN;
 }
 
-/* Clear irq sources in case of the sequencer is used */
-static inline void stm32_fmc2_clear_seq_irq(struct stm32_fmc2_nfc *fmc2)
+static inline void stm32_fmc2_nfc_clear_seq_irq(struct stm32_fmc2_nfc *nfc)
 {
-	writel_relaxed(FMC2_CSQICR_CLEAR_IRQ, fmc2->io_base + FMC2_CSQICR);
+	writel_relaxed(FMC2_CSQICR_CLEAR_IRQ, nfc->io_base + FMC2_CSQICR);
 }
 
-/* Enable irq sources in case of bch is used */
-static inline void stm32_fmc2_enable_bch_irq(struct stm32_fmc2_nfc *fmc2,
-					     int mode)
+static inline void stm32_fmc2_nfc_enable_bch_irq(struct stm32_fmc2_nfc *nfc,
+						 int mode)
 {
-	u32 bchier = readl_relaxed(fmc2->io_base + FMC2_BCHIER);
+	u32 bchier = readl_relaxed(nfc->io_base + FMC2_BCHIER);
 
 	if (mode == NAND_ECC_WRITE)
 		bchier |= FMC2_BCHIER_EPBRIE;
 	else
 		bchier |= FMC2_BCHIER_DERIE;
 
-	fmc2->irq_state = FMC2_IRQ_BCH;
+	nfc->irq_state = FMC2_IRQ_BCH;
 
-	writel_relaxed(bchier, fmc2->io_base + FMC2_BCHIER);
+	writel_relaxed(bchier, nfc->io_base + FMC2_BCHIER);
 }
 
-/* Disable irq sources in case of bch is used */
-static inline void stm32_fmc2_disable_bch_irq(struct stm32_fmc2_nfc *fmc2)
+static inline void stm32_fmc2_nfc_disable_bch_irq(struct stm32_fmc2_nfc *nfc)
 {
-	u32 bchier = readl_relaxed(fmc2->io_base + FMC2_BCHIER);
+	u32 bchier = readl_relaxed(nfc->io_base + FMC2_BCHIER);
 
 	bchier &= ~FMC2_BCHIER_DERIE;
 	bchier &= ~FMC2_BCHIER_EPBRIE;
 
-	writel_relaxed(bchier, fmc2->io_base + FMC2_BCHIER);
+	writel_relaxed(bchier, nfc->io_base + FMC2_BCHIER);
 
-	fmc2->irq_state = FMC2_IRQ_UNKNOWN;
+	nfc->irq_state = FMC2_IRQ_UNKNOWN;
 }
 
-/* Clear irq sources in case of bch is used */
-static inline void stm32_fmc2_clear_bch_irq(struct stm32_fmc2_nfc *fmc2)
+static inline void stm32_fmc2_nfc_clear_bch_irq(struct stm32_fmc2_nfc *nfc)
 {
-	writel_relaxed(FMC2_BCHICR_CLEAR_IRQ, fmc2->io_base + FMC2_BCHICR);
+	writel_relaxed(FMC2_BCHICR_CLEAR_IRQ, nfc->io_base + FMC2_BCHICR);
 }
 
 /*
  * Enable ECC logic and reset syndrome/parity bits previously calculated
  * Syndrome/parity bits is cleared by setting the ECCEN bit to 0
  */
-static void stm32_fmc2_hwctl(struct nand_chip *chip, int mode)
+static void stm32_fmc2_nfc_hwctl(struct nand_chip *chip, int mode)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 
-	stm32_fmc2_set_ecc(fmc2, false);
+	stm32_fmc2_nfc_set_ecc(nfc, false);
 
 	if (chip->ecc.strength != FMC2_ECC_HAM) {
-		u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR);
+		u32 pcr = readl_relaxed(nfc->io_base + FMC2_PCR);
 
 		if (mode == NAND_ECC_WRITE)
 			pcr |= FMC2_PCR_WEN;
 		else
 			pcr &= ~FMC2_PCR_WEN;
-		writel_relaxed(pcr, fmc2->io_base + FMC2_PCR);
+		writel_relaxed(pcr, nfc->io_base + FMC2_PCR);
 
-		reinit_completion(&fmc2->complete);
-		stm32_fmc2_clear_bch_irq(fmc2);
-		stm32_fmc2_enable_bch_irq(fmc2, mode);
+		reinit_completion(&nfc->complete);
+		stm32_fmc2_nfc_clear_bch_irq(nfc);
+		stm32_fmc2_nfc_enable_bch_irq(nfc, mode);
 	}
 
-	stm32_fmc2_set_ecc(fmc2, true);
+	stm32_fmc2_nfc_set_ecc(nfc, true);
 }
 
 /*
@@ -526,40 +502,37 @@ static void stm32_fmc2_hwctl(struct nand_chip *chip, int mode)
  * ECC is 3 bytes for 512 bytes of data (supports error correction up to
  * max of 1-bit)
  */
-static inline void stm32_fmc2_ham_set_ecc(const u32 ecc_sta, u8 *ecc)
+static inline void stm32_fmc2_nfc_ham_set_ecc(const u32 ecc_sta, u8 *ecc)
 {
 	ecc[0] = ecc_sta;
 	ecc[1] = ecc_sta >> 8;
 	ecc[2] = ecc_sta >> 16;
 }
 
-static int stm32_fmc2_ham_calculate(struct nand_chip *chip, const u8 *data,
-				    u8 *ecc)
+static int stm32_fmc2_nfc_ham_calculate(struct nand_chip *chip, const u8 *data,
+					u8 *ecc)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	u32 sr, heccr;
 	int ret;
 
-	ret = readl_relaxed_poll_timeout(fmc2->io_base + FMC2_SR,
-					 sr, sr & FMC2_SR_NWRF, 10,
-					 FMC2_TIMEOUT_MS);
+	ret = readl_relaxed_poll_timeout(nfc->io_base + FMC2_SR,
+					 sr, sr & FMC2_SR_NWRF, 1,
+					 1000 * FMC2_TIMEOUT_MS);
 	if (ret) {
-		dev_err(fmc2->dev, "ham timeout\n");
+		dev_err(nfc->dev, "ham timeout\n");
 		return ret;
 	}
 
-	heccr = readl_relaxed(fmc2->io_base + FMC2_HECCR);
-
-	stm32_fmc2_ham_set_ecc(heccr, ecc);
-
-	/* Disable ECC */
-	stm32_fmc2_set_ecc(fmc2, false);
+	heccr = readl_relaxed(nfc->io_base + FMC2_HECCR);
+	stm32_fmc2_nfc_ham_set_ecc(heccr, ecc);
+	stm32_fmc2_nfc_set_ecc(nfc, false);
 
 	return 0;
 }
 
-static int stm32_fmc2_ham_correct(struct nand_chip *chip, u8 *dat,
-				  u8 *read_ecc, u8 *calc_ecc)
+static int stm32_fmc2_nfc_ham_correct(struct nand_chip *chip, u8 *dat,
+				      u8 *read_ecc, u8 *calc_ecc)
 {
 	u8 bit_position = 0, b0, b1, b2;
 	u32 byte_addr = 0, b;
@@ -615,28 +588,28 @@ static int stm32_fmc2_ham_correct(struct nand_chip *chip, u8 *dat,
  * ECC is 7/13 bytes for 512 bytes of data (supports error correction up to
  * max of 4-bit/8-bit)
  */
-static int stm32_fmc2_bch_calculate(struct nand_chip *chip, const u8 *data,
-				    u8 *ecc)
+static int stm32_fmc2_nfc_bch_calculate(struct nand_chip *chip, const u8 *data,
+					u8 *ecc)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	u32 bchpbr;
 
 	/* Wait until the BCH code is ready */
-	if (!wait_for_completion_timeout(&fmc2->complete,
+	if (!wait_for_completion_timeout(&nfc->complete,
 					 msecs_to_jiffies(FMC2_TIMEOUT_MS))) {
-		dev_err(fmc2->dev, "bch timeout\n");
-		stm32_fmc2_disable_bch_irq(fmc2);
+		dev_err(nfc->dev, "bch timeout\n");
+		stm32_fmc2_nfc_disable_bch_irq(nfc);
 		return -ETIMEDOUT;
 	}
 
 	/* Read parity bits */
-	bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR1);
+	bchpbr = readl_relaxed(nfc->io_base + FMC2_BCHPBR1);
 	ecc[0] = bchpbr;
 	ecc[1] = bchpbr >> 8;
 	ecc[2] = bchpbr >> 16;
 	ecc[3] = bchpbr >> 24;
 
-	bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR2);
+	bchpbr = readl_relaxed(nfc->io_base + FMC2_BCHPBR2);
 	ecc[4] = bchpbr;
 	ecc[5] = bchpbr >> 8;
 	ecc[6] = bchpbr >> 16;
@@ -644,24 +617,22 @@ static int stm32_fmc2_bch_calculate(struct nand_chip *chip, const u8 *data,
 	if (chip->ecc.strength == FMC2_ECC_BCH8) {
 		ecc[7] = bchpbr >> 24;
 
-		bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR3);
+		bchpbr = readl_relaxed(nfc->io_base + FMC2_BCHPBR3);
 		ecc[8] = bchpbr;
 		ecc[9] = bchpbr >> 8;
 		ecc[10] = bchpbr >> 16;
 		ecc[11] = bchpbr >> 24;
 
-		bchpbr = readl_relaxed(fmc2->io_base + FMC2_BCHPBR4);
+		bchpbr = readl_relaxed(nfc->io_base + FMC2_BCHPBR4);
 		ecc[12] = bchpbr;
 	}
 
-	/* Disable ECC */
-	stm32_fmc2_set_ecc(fmc2, false);
+	stm32_fmc2_nfc_set_ecc(nfc, false);
 
 	return 0;
 }
 
-/* BCH algorithm correction */
-static int stm32_fmc2_bch_decode(int eccsize, u8 *dat, u32 *ecc_sta)
+static int stm32_fmc2_nfc_bch_decode(int eccsize, u8 *dat, u32 *ecc_sta)
 {
 	u32 bchdsr0 = ecc_sta[0];
 	u32 bchdsr1 = ecc_sta[1];
@@ -680,16 +651,16 @@ static int stm32_fmc2_bch_decode(int eccsize, u8 *dat, u32 *ecc_sta)
 	if (unlikely(bchdsr0 & FMC2_BCHDSR0_DUE))
 		return -EBADMSG;
 
-	pos[0] = bchdsr1 & FMC2_BCHDSR1_EBP1_MASK;
-	pos[1] = (bchdsr1 & FMC2_BCHDSR1_EBP2_MASK) >> FMC2_BCHDSR1_EBP2_SHIFT;
-	pos[2] = bchdsr2 & FMC2_BCHDSR2_EBP3_MASK;
-	pos[3] = (bchdsr2 & FMC2_BCHDSR2_EBP4_MASK) >> FMC2_BCHDSR2_EBP4_SHIFT;
-	pos[4] = bchdsr3 & FMC2_BCHDSR3_EBP5_MASK;
-	pos[5] = (bchdsr3 & FMC2_BCHDSR3_EBP6_MASK) >> FMC2_BCHDSR3_EBP6_SHIFT;
-	pos[6] = bchdsr4 & FMC2_BCHDSR4_EBP7_MASK;
-	pos[7] = (bchdsr4 & FMC2_BCHDSR4_EBP8_MASK) >> FMC2_BCHDSR4_EBP8_SHIFT;
+	pos[0] = FIELD_GET(FMC2_BCHDSR1_EBP1, bchdsr1);
+	pos[1] = FIELD_GET(FMC2_BCHDSR1_EBP2, bchdsr1);
+	pos[2] = FIELD_GET(FMC2_BCHDSR2_EBP3, bchdsr2);
+	pos[3] = FIELD_GET(FMC2_BCHDSR2_EBP4, bchdsr2);
+	pos[4] = FIELD_GET(FMC2_BCHDSR3_EBP5, bchdsr3);
+	pos[5] = FIELD_GET(FMC2_BCHDSR3_EBP6, bchdsr3);
+	pos[6] = FIELD_GET(FMC2_BCHDSR4_EBP7, bchdsr4);
+	pos[7] = FIELD_GET(FMC2_BCHDSR4_EBP8, bchdsr4);
 
-	den = (bchdsr0 & FMC2_BCHDSR0_DEN_MASK) >> FMC2_BCHDSR0_DEN_SHIFT;
+	den = FIELD_GET(FMC2_BCHDSR0_DEN, bchdsr0);
 	for (i = 0; i < den; i++) {
 		if (pos[i] < eccsize * 8) {
 			change_bit(pos[i], (unsigned long *)dat);
@@ -700,34 +671,33 @@ static int stm32_fmc2_bch_decode(int eccsize, u8 *dat, u32 *ecc_sta)
 	return nb_errs;
 }
 
-static int stm32_fmc2_bch_correct(struct nand_chip *chip, u8 *dat,
-				  u8 *read_ecc, u8 *calc_ecc)
+static int stm32_fmc2_nfc_bch_correct(struct nand_chip *chip, u8 *dat,
+				      u8 *read_ecc, u8 *calc_ecc)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	u32 ecc_sta[5];
 
 	/* Wait until the decoding error is ready */
-	if (!wait_for_completion_timeout(&fmc2->complete,
+	if (!wait_for_completion_timeout(&nfc->complete,
 					 msecs_to_jiffies(FMC2_TIMEOUT_MS))) {
-		dev_err(fmc2->dev, "bch timeout\n");
-		stm32_fmc2_disable_bch_irq(fmc2);
+		dev_err(nfc->dev, "bch timeout\n");
+		stm32_fmc2_nfc_disable_bch_irq(nfc);
 		return -ETIMEDOUT;
 	}
 
-	ecc_sta[0] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR0);
-	ecc_sta[1] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR1);
-	ecc_sta[2] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR2);
-	ecc_sta[3] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR3);
-	ecc_sta[4] = readl_relaxed(fmc2->io_base + FMC2_BCHDSR4);
+	ecc_sta[0] = readl_relaxed(nfc->io_base + FMC2_BCHDSR0);
+	ecc_sta[1] = readl_relaxed(nfc->io_base + FMC2_BCHDSR1);
+	ecc_sta[2] = readl_relaxed(nfc->io_base + FMC2_BCHDSR2);
+	ecc_sta[3] = readl_relaxed(nfc->io_base + FMC2_BCHDSR3);
+	ecc_sta[4] = readl_relaxed(nfc->io_base + FMC2_BCHDSR4);
 
-	/* Disable ECC */
-	stm32_fmc2_set_ecc(fmc2, false);
+	stm32_fmc2_nfc_set_ecc(nfc, false);
 
-	return stm32_fmc2_bch_decode(chip->ecc.size, dat, ecc_sta);
+	return stm32_fmc2_nfc_bch_decode(chip->ecc.size, dat, ecc_sta);
 }
 
-static int stm32_fmc2_read_page(struct nand_chip *chip, u8 *buf,
-				int oob_required, int page)
+static int stm32_fmc2_nfc_read_page(struct nand_chip *chip, u8 *buf,
+				    int oob_required, int page)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	int ret, i, s, stat, eccsize = chip->ecc.size;
@@ -789,21 +759,21 @@ static int stm32_fmc2_read_page(struct nand_chip *chip, u8 *buf,
 }
 
 /* Sequencer read/write configuration */
-static void stm32_fmc2_rw_page_init(struct nand_chip *chip, int page,
-				    int raw, bool write_data)
+static void stm32_fmc2_nfc_rw_page_init(struct nand_chip *chip, int page,
+					int raw, bool write_data)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	u32 csqcfgr1, csqcfgr2, csqcfgr3;
 	u32 csqar1, csqar2;
 	u32 ecc_offset = mtd->writesize + FMC2_BBM_LEN;
-	u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR);
+	u32 pcr = readl_relaxed(nfc->io_base + FMC2_PCR);
 
 	if (write_data)
 		pcr |= FMC2_PCR_WEN;
 	else
 		pcr &= ~FMC2_PCR_WEN;
-	writel_relaxed(pcr, fmc2->io_base + FMC2_PCR);
+	writel_relaxed(pcr, nfc->io_base + FMC2_PCR);
 
 	/*
 	 * - Set Program Page/Page Read command
@@ -812,11 +782,11 @@ static void stm32_fmc2_rw_page_init(struct nand_chip *chip, int page,
 	 */
 	csqcfgr1 = FMC2_CSQCFGR1_DMADEN | FMC2_CSQCFGR1_CMD1T;
 	if (write_data)
-		csqcfgr1 |= FMC2_CSQCFGR1_CMD1(NAND_CMD_SEQIN);
+		csqcfgr1 |= FIELD_PREP(FMC2_CSQCFGR1_CMD1, NAND_CMD_SEQIN);
 	else
-		csqcfgr1 |= FMC2_CSQCFGR1_CMD1(NAND_CMD_READ0) |
+		csqcfgr1 |= FIELD_PREP(FMC2_CSQCFGR1_CMD1, NAND_CMD_READ0) |
 			    FMC2_CSQCFGR1_CMD2EN |
-			    FMC2_CSQCFGR1_CMD2(NAND_CMD_READSTART) |
+			    FIELD_PREP(FMC2_CSQCFGR1_CMD2, NAND_CMD_READSTART) |
 			    FMC2_CSQCFGR1_CMD2T;
 
 	/*
@@ -826,11 +796,12 @@ static void stm32_fmc2_rw_page_init(struct nand_chip *chip, int page,
 	 * - Set timings
 	 */
 	if (write_data)
-		csqcfgr2 = FMC2_CSQCFGR2_RCMD1(NAND_CMD_RNDIN);
+		csqcfgr2 = FIELD_PREP(FMC2_CSQCFGR2_RCMD1, NAND_CMD_RNDIN);
 	else
-		csqcfgr2 = FMC2_CSQCFGR2_RCMD1(NAND_CMD_RNDOUT) |
+		csqcfgr2 = FIELD_PREP(FMC2_CSQCFGR2_RCMD1, NAND_CMD_RNDOUT) |
 			   FMC2_CSQCFGR2_RCMD2EN |
-			   FMC2_CSQCFGR2_RCMD2(NAND_CMD_RNDOUTSTART) |
+			   FIELD_PREP(FMC2_CSQCFGR2_RCMD2,
+				      NAND_CMD_RNDOUTSTART) |
 			   FMC2_CSQCFGR2_RCMD1T |
 			   FMC2_CSQCFGR2_RCMD2T;
 	if (!raw) {
@@ -842,7 +813,7 @@ static void stm32_fmc2_rw_page_init(struct nand_chip *chip, int page,
 	 * - Set the number of sectors to be written
 	 * - Set timings
 	 */
-	csqcfgr3 = FMC2_CSQCFGR3_SNBR(chip->ecc.steps - 1);
+	csqcfgr3 = FIELD_PREP(FMC2_CSQCFGR3_SNBR, chip->ecc.steps - 1);
 	if (write_data) {
 		csqcfgr3 |= FMC2_CSQCFGR3_RAC2T;
 		if (chip->options & NAND_ROW_ADDR_3)
@@ -856,8 +827,8 @@ static void stm32_fmc2_rw_page_init(struct nand_chip *chip, int page,
 	 * Byte 1 and byte 2 => column, we start at 0x0
 	 * Byte 3 and byte 4 => page
 	 */
-	csqar1 = FMC2_CSQCAR1_ADDC3(page);
-	csqar1 |= FMC2_CSQCAR1_ADDC4(page >> 8);
+	csqar1 = FIELD_PREP(FMC2_CSQCAR1_ADDC3, page);
+	csqar1 |= FIELD_PREP(FMC2_CSQCAR1_ADDC4, page >> 8);
 
 	/*
 	 * - Set chip enable number
@@ -865,43 +836,44 @@ static void stm32_fmc2_rw_page_init(struct nand_chip *chip, int page,
 	 * - Calculate the number of address cycles to be issued
 	 * - Set byte 5 of address cycle if needed
 	 */
-	csqar2 = FMC2_CSQCAR2_NANDCEN(fmc2->cs_sel);
+	csqar2 = FIELD_PREP(FMC2_CSQCAR2_NANDCEN, nfc->cs_sel);
 	if (chip->options & NAND_BUSWIDTH_16)
-		csqar2 |= FMC2_CSQCAR2_SAO(ecc_offset >> 1);
+		csqar2 |= FIELD_PREP(FMC2_CSQCAR2_SAO, ecc_offset >> 1);
 	else
-		csqar2 |= FMC2_CSQCAR2_SAO(ecc_offset);
+		csqar2 |= FIELD_PREP(FMC2_CSQCAR2_SAO, ecc_offset);
 	if (chip->options & NAND_ROW_ADDR_3) {
-		csqcfgr1 |= FMC2_CSQCFGR1_ACYNBR(5);
-		csqar2 |= FMC2_CSQCAR2_ADDC5(page >> 16);
+		csqcfgr1 |= FIELD_PREP(FMC2_CSQCFGR1_ACYNBR, 5);
+		csqar2 |= FIELD_PREP(FMC2_CSQCAR2_ADDC5, page >> 16);
 	} else {
-		csqcfgr1 |= FMC2_CSQCFGR1_ACYNBR(4);
+		csqcfgr1 |= FIELD_PREP(FMC2_CSQCFGR1_ACYNBR, 4);
 	}
 
-	writel_relaxed(csqcfgr1, fmc2->io_base + FMC2_CSQCFGR1);
-	writel_relaxed(csqcfgr2, fmc2->io_base + FMC2_CSQCFGR2);
-	writel_relaxed(csqcfgr3, fmc2->io_base + FMC2_CSQCFGR3);
-	writel_relaxed(csqar1, fmc2->io_base + FMC2_CSQAR1);
-	writel_relaxed(csqar2, fmc2->io_base + FMC2_CSQAR2);
+	writel_relaxed(csqcfgr1, nfc->io_base + FMC2_CSQCFGR1);
+	writel_relaxed(csqcfgr2, nfc->io_base + FMC2_CSQCFGR2);
+	writel_relaxed(csqcfgr3, nfc->io_base + FMC2_CSQCFGR3);
+	writel_relaxed(csqar1, nfc->io_base + FMC2_CSQAR1);
+	writel_relaxed(csqar2, nfc->io_base + FMC2_CSQAR2);
 }
 
-static void stm32_fmc2_dma_callback(void *arg)
+static void stm32_fmc2_nfc_dma_callback(void *arg)
 {
 	complete((struct completion *)arg);
 }
 
 /* Read/write data from/to a page */
-static int stm32_fmc2_xfer(struct nand_chip *chip, const u8 *buf,
-			   int raw, bool write_data)
+static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf,
+			       int raw, bool write_data)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	struct dma_async_tx_descriptor *desc_data, *desc_ecc;
 	struct scatterlist *sg;
-	struct dma_chan *dma_ch = fmc2->dma_rx_ch;
+	struct dma_chan *dma_ch = nfc->dma_rx_ch;
 	enum dma_data_direction dma_data_dir = DMA_FROM_DEVICE;
 	enum dma_transfer_direction dma_transfer_dir = DMA_DEV_TO_MEM;
-	u32 csqcr = readl_relaxed(fmc2->io_base + FMC2_CSQCR);
+	u32 csqcr = readl_relaxed(nfc->io_base + FMC2_CSQCR);
 	int eccsteps = chip->ecc.steps;
 	int eccsize = chip->ecc.size;
+	unsigned long timeout = msecs_to_jiffies(FMC2_TIMEOUT_MS);
 	const u8 *p = buf;
 	int s, ret;
 
@@ -909,20 +881,20 @@ static int stm32_fmc2_xfer(struct nand_chip *chip, const u8 *buf,
 	if (write_data) {
 		dma_data_dir = DMA_TO_DEVICE;
 		dma_transfer_dir = DMA_MEM_TO_DEV;
-		dma_ch = fmc2->dma_tx_ch;
+		dma_ch = nfc->dma_tx_ch;
 	}
 
-	for_each_sg(fmc2->dma_data_sg.sgl, sg, eccsteps, s) {
+	for_each_sg(nfc->dma_data_sg.sgl, sg, eccsteps, s) {
 		sg_set_buf(sg, p, eccsize);
 		p += eccsize;
 	}
 
-	ret = dma_map_sg(fmc2->dev, fmc2->dma_data_sg.sgl,
+	ret = dma_map_sg(nfc->dev, nfc->dma_data_sg.sgl,
 			 eccsteps, dma_data_dir);
 	if (ret < 0)
 		return ret;
 
-	desc_data = dmaengine_prep_slave_sg(dma_ch, fmc2->dma_data_sg.sgl,
+	desc_data = dmaengine_prep_slave_sg(dma_ch, nfc->dma_data_sg.sgl,
 					    eccsteps, dma_transfer_dir,
 					    DMA_PREP_INTERRUPT);
 	if (!desc_data) {
@@ -930,10 +902,10 @@ static int stm32_fmc2_xfer(struct nand_chip *chip, const u8 *buf,
 		goto err_unmap_data;
 	}
 
-	reinit_completion(&fmc2->dma_data_complete);
-	reinit_completion(&fmc2->complete);
-	desc_data->callback = stm32_fmc2_dma_callback;
-	desc_data->callback_param = &fmc2->dma_data_complete;
+	reinit_completion(&nfc->dma_data_complete);
+	reinit_completion(&nfc->complete);
+	desc_data->callback = stm32_fmc2_nfc_dma_callback;
+	desc_data->callback_param = &nfc->dma_data_complete;
 	ret = dma_submit_error(dmaengine_submit(desc_data));
 	if (ret)
 		goto err_unmap_data;
@@ -942,19 +914,19 @@ static int stm32_fmc2_xfer(struct nand_chip *chip, const u8 *buf,
 
 	if (!write_data && !raw) {
 		/* Configure DMA ECC status */
-		p = fmc2->ecc_buf;
-		for_each_sg(fmc2->dma_ecc_sg.sgl, sg, eccsteps, s) {
-			sg_set_buf(sg, p, fmc2->dma_ecc_len);
-			p += fmc2->dma_ecc_len;
+		p = nfc->ecc_buf;
+		for_each_sg(nfc->dma_ecc_sg.sgl, sg, eccsteps, s) {
+			sg_set_buf(sg, p, nfc->dma_ecc_len);
+			p += nfc->dma_ecc_len;
 		}
 
-		ret = dma_map_sg(fmc2->dev, fmc2->dma_ecc_sg.sgl,
+		ret = dma_map_sg(nfc->dev, nfc->dma_ecc_sg.sgl,
 				 eccsteps, dma_data_dir);
 		if (ret < 0)
 			goto err_unmap_data;
 
-		desc_ecc = dmaengine_prep_slave_sg(fmc2->dma_ecc_ch,
-						   fmc2->dma_ecc_sg.sgl,
+		desc_ecc = dmaengine_prep_slave_sg(nfc->dma_ecc_ch,
+						   nfc->dma_ecc_sg.sgl,
 						   eccsteps, dma_transfer_dir,
 						   DMA_PREP_INTERRUPT);
 		if (!desc_ecc) {
@@ -962,76 +934,73 @@ static int stm32_fmc2_xfer(struct nand_chip *chip, const u8 *buf,
 			goto err_unmap_ecc;
 		}
 
-		reinit_completion(&fmc2->dma_ecc_complete);
-		desc_ecc->callback = stm32_fmc2_dma_callback;
-		desc_ecc->callback_param = &fmc2->dma_ecc_complete;
+		reinit_completion(&nfc->dma_ecc_complete);
+		desc_ecc->callback = stm32_fmc2_nfc_dma_callback;
+		desc_ecc->callback_param = &nfc->dma_ecc_complete;
 		ret = dma_submit_error(dmaengine_submit(desc_ecc));
 		if (ret)
 			goto err_unmap_ecc;
 
-		dma_async_issue_pending(fmc2->dma_ecc_ch);
+		dma_async_issue_pending(nfc->dma_ecc_ch);
 	}
 
-	stm32_fmc2_clear_seq_irq(fmc2);
-	stm32_fmc2_enable_seq_irq(fmc2);
+	stm32_fmc2_nfc_clear_seq_irq(nfc);
+	stm32_fmc2_nfc_enable_seq_irq(nfc);
 
 	/* Start the transfer */
 	csqcr |= FMC2_CSQCR_CSQSTART;
-	writel_relaxed(csqcr, fmc2->io_base + FMC2_CSQCR);
+	writel_relaxed(csqcr, nfc->io_base + FMC2_CSQCR);
 
 	/* Wait end of sequencer transfer */
-	if (!wait_for_completion_timeout(&fmc2->complete,
-					 msecs_to_jiffies(FMC2_TIMEOUT_MS))) {
-		dev_err(fmc2->dev, "seq timeout\n");
-		stm32_fmc2_disable_seq_irq(fmc2);
+	if (!wait_for_completion_timeout(&nfc->complete, timeout)) {
+		dev_err(nfc->dev, "seq timeout\n");
+		stm32_fmc2_nfc_disable_seq_irq(nfc);
 		dmaengine_terminate_all(dma_ch);
 		if (!write_data && !raw)
-			dmaengine_terminate_all(fmc2->dma_ecc_ch);
+			dmaengine_terminate_all(nfc->dma_ecc_ch);
 		ret = -ETIMEDOUT;
 		goto err_unmap_ecc;
 	}
 
 	/* Wait DMA data transfer completion */
-	if (!wait_for_completion_timeout(&fmc2->dma_data_complete,
-					 msecs_to_jiffies(FMC2_TIMEOUT_MS))) {
-		dev_err(fmc2->dev, "data DMA timeout\n");
+	if (!wait_for_completion_timeout(&nfc->dma_data_complete, timeout)) {
+		dev_err(nfc->dev, "data DMA timeout\n");
 		dmaengine_terminate_all(dma_ch);
 		ret = -ETIMEDOUT;
 	}
 
 	/* Wait DMA ECC transfer completion */
 	if (!write_data && !raw) {
-		if (!wait_for_completion_timeout(&fmc2->dma_ecc_complete,
-					msecs_to_jiffies(FMC2_TIMEOUT_MS))) {
-			dev_err(fmc2->dev, "ECC DMA timeout\n");
-			dmaengine_terminate_all(fmc2->dma_ecc_ch);
+		if (!wait_for_completion_timeout(&nfc->dma_ecc_complete,
+						 timeout)) {
+			dev_err(nfc->dev, "ECC DMA timeout\n");
+			dmaengine_terminate_all(nfc->dma_ecc_ch);
 			ret = -ETIMEDOUT;
 		}
 	}
 
 err_unmap_ecc:
 	if (!write_data && !raw)
-		dma_unmap_sg(fmc2->dev, fmc2->dma_ecc_sg.sgl,
+		dma_unmap_sg(nfc->dev, nfc->dma_ecc_sg.sgl,
 			     eccsteps, dma_data_dir);
 
 err_unmap_data:
-	dma_unmap_sg(fmc2->dev, fmc2->dma_data_sg.sgl, eccsteps, dma_data_dir);
+	dma_unmap_sg(nfc->dev, nfc->dma_data_sg.sgl, eccsteps, dma_data_dir);
 
 	return ret;
 }
 
-static int stm32_fmc2_sequencer_write(struct nand_chip *chip,
-				      const u8 *buf, int oob_required,
-				      int page, int raw)
+static int stm32_fmc2_nfc_seq_write(struct nand_chip *chip, const u8 *buf,
+				    int oob_required, int page, int raw)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	int ret;
 
 	/* Configure the sequencer */
-	stm32_fmc2_rw_page_init(chip, page, raw, true);
+	stm32_fmc2_nfc_rw_page_init(chip, page, raw, true);
 
 	/* Write the page */
-	ret = stm32_fmc2_xfer(chip, buf, raw, true);
+	ret = stm32_fmc2_nfc_xfer(chip, buf, raw, true);
 	if (ret)
 		return ret;
 
@@ -1047,55 +1016,50 @@ static int stm32_fmc2_sequencer_write(struct nand_chip *chip,
 	return nand_prog_page_end_op(chip);
 }
 
-static int stm32_fmc2_sequencer_write_page(struct nand_chip *chip,
-					   const u8 *buf,
-					   int oob_required,
-					   int page)
+static int stm32_fmc2_nfc_seq_write_page(struct nand_chip *chip, const u8 *buf,
+					 int oob_required, int page)
 {
 	int ret;
 
-	/* Select the target */
-	ret = stm32_fmc2_select_chip(chip, chip->cur_cs);
+	ret = stm32_fmc2_nfc_select_chip(chip, chip->cur_cs);
 	if (ret)
 		return ret;
 
-	return stm32_fmc2_sequencer_write(chip, buf, oob_required, page, false);
+	return stm32_fmc2_nfc_seq_write(chip, buf, oob_required, page, false);
 }
 
-static int stm32_fmc2_sequencer_write_page_raw(struct nand_chip *chip,
-					       const u8 *buf,
-					       int oob_required,
-					       int page)
+static int stm32_fmc2_nfc_seq_write_page_raw(struct nand_chip *chip,
+					     const u8 *buf, int oob_required,
+					     int page)
 {
 	int ret;
 
-	/* Select the target */
-	ret = stm32_fmc2_select_chip(chip, chip->cur_cs);
+	ret = stm32_fmc2_nfc_select_chip(chip, chip->cur_cs);
 	if (ret)
 		return ret;
 
-	return stm32_fmc2_sequencer_write(chip, buf, oob_required, page, true);
+	return stm32_fmc2_nfc_seq_write(chip, buf, oob_required, page, true);
 }
 
 /* Get a status indicating which sectors have errors */
-static inline u16 stm32_fmc2_get_mapping_status(struct stm32_fmc2_nfc *fmc2)
+static inline u16 stm32_fmc2_nfc_get_mapping_status(struct stm32_fmc2_nfc *nfc)
 {
-	u32 csqemsr = readl_relaxed(fmc2->io_base + FMC2_CSQEMSR);
+	u32 csqemsr = readl_relaxed(nfc->io_base + FMC2_CSQEMSR);
 
 	return csqemsr & FMC2_CSQEMSR_SEM;
 }
 
-static int stm32_fmc2_sequencer_correct(struct nand_chip *chip, u8 *dat,
-					u8 *read_ecc, u8 *calc_ecc)
+static int stm32_fmc2_nfc_seq_correct(struct nand_chip *chip, u8 *dat,
+				      u8 *read_ecc, u8 *calc_ecc)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
 	int eccstrength = chip->ecc.strength;
 	int i, s, eccsize = chip->ecc.size;
-	u32 *ecc_sta = (u32 *)fmc2->ecc_buf;
-	u16 sta_map = stm32_fmc2_get_mapping_status(fmc2);
+	u32 *ecc_sta = (u32 *)nfc->ecc_buf;
+	u16 sta_map = stm32_fmc2_nfc_get_mapping_status(nfc);
 	unsigned int max_bitflips = 0;
 
 	for (i = 0, s = 0; s < eccsteps; s++, i += eccbytes, dat += eccsize) {
@@ -1104,10 +1068,11 @@ static int stm32_fmc2_sequencer_correct(struct nand_chip *chip, u8 *dat,
 		if (eccstrength == FMC2_ECC_HAM) {
 			/* Ecc_sta = FMC2_HECCR */
 			if (sta_map & BIT(s)) {
-				stm32_fmc2_ham_set_ecc(*ecc_sta, &calc_ecc[i]);
-				stat = stm32_fmc2_ham_correct(chip, dat,
-							      &read_ecc[i],
-							      &calc_ecc[i]);
+				stm32_fmc2_nfc_ham_set_ecc(*ecc_sta,
+							   &calc_ecc[i]);
+				stat = stm32_fmc2_nfc_ham_correct(chip, dat,
+								  &read_ecc[i],
+								  &calc_ecc[i]);
 			}
 			ecc_sta++;
 		} else {
@@ -1119,8 +1084,8 @@ static int stm32_fmc2_sequencer_correct(struct nand_chip *chip, u8 *dat,
 			 * Ecc_sta[4] = FMC2_BCHDSR4
 			 */
 			if (sta_map & BIT(s))
-				stat = stm32_fmc2_bch_decode(eccsize, dat,
-							     ecc_sta);
+				stat = stm32_fmc2_nfc_bch_decode(eccsize, dat,
+								 ecc_sta);
 			ecc_sta += 5;
 		}
 
@@ -1143,30 +1108,29 @@ static int stm32_fmc2_sequencer_correct(struct nand_chip *chip, u8 *dat,
 	return max_bitflips;
 }
 
-static int stm32_fmc2_sequencer_read_page(struct nand_chip *chip, u8 *buf,
-					  int oob_required, int page)
+static int stm32_fmc2_nfc_seq_read_page(struct nand_chip *chip, u8 *buf,
+					int oob_required, int page)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	u8 *ecc_calc = chip->ecc.calc_buf;
 	u8 *ecc_code = chip->ecc.code_buf;
 	u16 sta_map;
 	int ret;
 
-	/* Select the target */
-	ret = stm32_fmc2_select_chip(chip, chip->cur_cs);
+	ret = stm32_fmc2_nfc_select_chip(chip, chip->cur_cs);
 	if (ret)
 		return ret;
 
 	/* Configure the sequencer */
-	stm32_fmc2_rw_page_init(chip, page, 0, false);
+	stm32_fmc2_nfc_rw_page_init(chip, page, 0, false);
 
 	/* Read the page */
-	ret = stm32_fmc2_xfer(chip, buf, 0, false);
+	ret = stm32_fmc2_nfc_xfer(chip, buf, 0, false);
 	if (ret)
 		return ret;
 
-	sta_map = stm32_fmc2_get_mapping_status(fmc2);
+	sta_map = stm32_fmc2_nfc_get_mapping_status(nfc);
 
 	/* Check if errors happen */
 	if (likely(!sta_map)) {
@@ -1193,22 +1157,21 @@ static int stm32_fmc2_sequencer_read_page(struct nand_chip *chip, u8 *buf,
 	return chip->ecc.correct(chip, buf, ecc_code, ecc_calc);
 }
 
-static int stm32_fmc2_sequencer_read_page_raw(struct nand_chip *chip, u8 *buf,
-					      int oob_required, int page)
+static int stm32_fmc2_nfc_seq_read_page_raw(struct nand_chip *chip, u8 *buf,
+					    int oob_required, int page)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	int ret;
 
-	/* Select the target */
-	ret = stm32_fmc2_select_chip(chip, chip->cur_cs);
+	ret = stm32_fmc2_nfc_select_chip(chip, chip->cur_cs);
 	if (ret)
 		return ret;
 
 	/* Configure the sequencer */
-	stm32_fmc2_rw_page_init(chip, page, 1, false);
+	stm32_fmc2_nfc_rw_page_init(chip, page, 1, false);
 
 	/* Read the page */
-	ret = stm32_fmc2_xfer(chip, buf, 1, false);
+	ret = stm32_fmc2_nfc_xfer(chip, buf, 1, false);
 	if (ret)
 		return ret;
 
@@ -1221,31 +1184,31 @@ static int stm32_fmc2_sequencer_read_page_raw(struct nand_chip *chip, u8 *buf,
 	return 0;
 }
 
-static irqreturn_t stm32_fmc2_irq(int irq, void *dev_id)
+static irqreturn_t stm32_fmc2_nfc_irq(int irq, void *dev_id)
 {
-	struct stm32_fmc2_nfc *fmc2 = (struct stm32_fmc2_nfc *)dev_id;
+	struct stm32_fmc2_nfc *nfc = (struct stm32_fmc2_nfc *)dev_id;
 
-	if (fmc2->irq_state == FMC2_IRQ_SEQ)
+	if (nfc->irq_state == FMC2_IRQ_SEQ)
 		/* Sequencer is used */
-		stm32_fmc2_disable_seq_irq(fmc2);
-	else if (fmc2->irq_state == FMC2_IRQ_BCH)
+		stm32_fmc2_nfc_disable_seq_irq(nfc);
+	else if (nfc->irq_state == FMC2_IRQ_BCH)
 		/* BCH is used */
-		stm32_fmc2_disable_bch_irq(fmc2);
+		stm32_fmc2_nfc_disable_bch_irq(nfc);
 
-	complete(&fmc2->complete);
+	complete(&nfc->complete);
 
 	return IRQ_HANDLED;
 }
 
-static void stm32_fmc2_read_data(struct nand_chip *chip, void *buf,
-				 unsigned int len, bool force_8bit)
+static void stm32_fmc2_nfc_read_data(struct nand_chip *chip, void *buf,
+				     unsigned int len, bool force_8bit)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
-	void __iomem *io_addr_r = fmc2->data_base[fmc2->cs_sel];
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
+	void __iomem *io_addr_r = nfc->data_base[nfc->cs_sel];
 
 	if (force_8bit && chip->options & NAND_BUSWIDTH_16)
 		/* Reconfigure bus width to 8-bit */
-		stm32_fmc2_set_buswidth_16(fmc2, false);
+		stm32_fmc2_nfc_set_buswidth_16(nfc, false);
 
 	if (!IS_ALIGNED((uintptr_t)buf, sizeof(u32))) {
 		if (!IS_ALIGNED((uintptr_t)buf, sizeof(u16)) && len) {
@@ -1281,18 +1244,18 @@ static void stm32_fmc2_read_data(struct nand_chip *chip, void *buf,
 
 	if (force_8bit && chip->options & NAND_BUSWIDTH_16)
 		/* Reconfigure bus width to 16-bit */
-		stm32_fmc2_set_buswidth_16(fmc2, true);
+		stm32_fmc2_nfc_set_buswidth_16(nfc, true);
 }
 
-static void stm32_fmc2_write_data(struct nand_chip *chip, const void *buf,
-				  unsigned int len, bool force_8bit)
+static void stm32_fmc2_nfc_write_data(struct nand_chip *chip, const void *buf,
+				      unsigned int len, bool force_8bit)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
-	void __iomem *io_addr_w = fmc2->data_base[fmc2->cs_sel];
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
+	void __iomem *io_addr_w = nfc->data_base[nfc->cs_sel];
 
 	if (force_8bit && chip->options & NAND_BUSWIDTH_16)
 		/* Reconfigure bus width to 8-bit */
-		stm32_fmc2_set_buswidth_16(fmc2, false);
+		stm32_fmc2_nfc_set_buswidth_16(nfc, false);
 
 	if (!IS_ALIGNED((uintptr_t)buf, sizeof(u32))) {
 		if (!IS_ALIGNED((uintptr_t)buf, sizeof(u16)) && len) {
@@ -1328,48 +1291,49 @@ static void stm32_fmc2_write_data(struct nand_chip *chip, const void *buf,
 
 	if (force_8bit && chip->options & NAND_BUSWIDTH_16)
 		/* Reconfigure bus width to 16-bit */
-		stm32_fmc2_set_buswidth_16(fmc2, true);
+		stm32_fmc2_nfc_set_buswidth_16(nfc, true);
 }
 
-static int stm32_fmc2_waitrdy(struct nand_chip *chip, unsigned long timeout_ms)
+static int stm32_fmc2_nfc_waitrdy(struct nand_chip *chip,
+				  unsigned long timeout_ms)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	const struct nand_sdr_timings *timings;
 	u32 isr, sr;
 
 	/* Check if there is no pending requests to the NAND flash */
-	if (readl_relaxed_poll_timeout_atomic(fmc2->io_base + FMC2_SR, sr,
+	if (readl_relaxed_poll_timeout_atomic(nfc->io_base + FMC2_SR, sr,
 					      sr & FMC2_SR_NWRF, 1,
-					      FMC2_TIMEOUT_US))
-		dev_warn(fmc2->dev, "Waitrdy timeout\n");
+					      1000 * FMC2_TIMEOUT_MS))
+		dev_warn(nfc->dev, "Waitrdy timeout\n");
 
 	/* Wait tWB before R/B# signal is low */
 	timings = nand_get_sdr_timings(&chip->data_interface);
 	ndelay(PSEC_TO_NSEC(timings->tWB_max));
 
 	/* R/B# signal is low, clear high level flag */
-	writel_relaxed(FMC2_ICR_CIHLF, fmc2->io_base + FMC2_ICR);
+	writel_relaxed(FMC2_ICR_CIHLF, nfc->io_base + FMC2_ICR);
 
 	/* Wait R/B# signal is high */
-	return readl_relaxed_poll_timeout_atomic(fmc2->io_base + FMC2_ISR,
+	return readl_relaxed_poll_timeout_atomic(nfc->io_base + FMC2_ISR,
 						 isr, isr & FMC2_ISR_IHLF,
 						 5, 1000 * timeout_ms);
 }
 
-static int stm32_fmc2_exec_op(struct nand_chip *chip,
-			      const struct nand_operation *op,
-			      bool check_only)
+static int stm32_fmc2_nfc_exec_op(struct nand_chip *chip,
+				  const struct nand_operation *op,
+				  bool check_only)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	const struct nand_op_instr *instr = NULL;
-	unsigned int op_id, i;
+	unsigned int op_id, i, timeout;
 	int ret;
 
-	ret = stm32_fmc2_select_chip(chip, op->cs);
-	if (ret)
-		return ret;
-
 	if (check_only)
+		return 0;
+
+	ret = stm32_fmc2_nfc_select_chip(chip, op->cs);
+	if (ret)
 		return ret;
 
 	for (op_id = 0; op_id < op->ninstrs; op_id++) {
@@ -1378,30 +1342,30 @@ static int stm32_fmc2_exec_op(struct nand_chip *chip,
 		switch (instr->type) {
 		case NAND_OP_CMD_INSTR:
 			writeb_relaxed(instr->ctx.cmd.opcode,
-				       fmc2->cmd_base[fmc2->cs_sel]);
+				       nfc->cmd_base[nfc->cs_sel]);
 			break;
 
 		case NAND_OP_ADDR_INSTR:
 			for (i = 0; i < instr->ctx.addr.naddrs; i++)
 				writeb_relaxed(instr->ctx.addr.addrs[i],
-					       fmc2->addr_base[fmc2->cs_sel]);
+					       nfc->addr_base[nfc->cs_sel]);
 			break;
 
 		case NAND_OP_DATA_IN_INSTR:
-			stm32_fmc2_read_data(chip, instr->ctx.data.buf.in,
-					     instr->ctx.data.len,
-					     instr->ctx.data.force_8bit);
+			stm32_fmc2_nfc_read_data(chip, instr->ctx.data.buf.in,
+						 instr->ctx.data.len,
+						 instr->ctx.data.force_8bit);
 			break;
 
 		case NAND_OP_DATA_OUT_INSTR:
-			stm32_fmc2_write_data(chip, instr->ctx.data.buf.out,
-					      instr->ctx.data.len,
-					      instr->ctx.data.force_8bit);
+			stm32_fmc2_nfc_write_data(chip, instr->ctx.data.buf.out,
+						  instr->ctx.data.len,
+						  instr->ctx.data.force_8bit);
 			break;
 
 		case NAND_OP_WAITRDY_INSTR:
-			ret = stm32_fmc2_waitrdy(chip,
-						 instr->ctx.waitrdy.timeout_ms);
+			timeout = instr->ctx.waitrdy.timeout_ms;
+			ret = stm32_fmc2_nfc_waitrdy(chip, timeout);
 			break;
 		}
 	}
@@ -1409,21 +1373,20 @@ static int stm32_fmc2_exec_op(struct nand_chip *chip,
 	return ret;
 }
 
-/* Controller initialization */
-static void stm32_fmc2_init(struct stm32_fmc2_nfc *fmc2)
+static void stm32_fmc2_nfc_init(struct stm32_fmc2_nfc *nfc)
 {
-	u32 pcr = readl_relaxed(fmc2->io_base + FMC2_PCR);
-	u32 bcr1 = readl_relaxed(fmc2->io_base + FMC2_BCR1);
+	u32 pcr = readl_relaxed(nfc->io_base + FMC2_PCR);
+	u32 bcr1 = readl_relaxed(nfc->io_base + FMC2_BCR1);
 
 	/* Set CS used to undefined */
-	fmc2->cs_sel = -1;
+	nfc->cs_sel = -1;
 
 	/* Enable wait feature and nand flash memory bank */
 	pcr |= FMC2_PCR_PWAITEN;
 	pcr |= FMC2_PCR_PBKEN;
 
 	/* Set buswidth to 8 bits mode for identification */
-	pcr &= ~FMC2_PCR_PWID_MASK;
+	pcr &= ~FMC2_PCR_PWID;
 
 	/* ECC logic is disabled */
 	pcr &= ~FMC2_PCR_ECCEN;
@@ -1434,32 +1397,31 @@ static void stm32_fmc2_init(struct stm32_fmc2_nfc *fmc2)
 	pcr &= ~FMC2_PCR_WEN;
 
 	/* Set default ECC sector size */
-	pcr &= ~FMC2_PCR_ECCSS_MASK;
-	pcr |= FMC2_PCR_ECCSS(FMC2_PCR_ECCSS_2048);
+	pcr &= ~FMC2_PCR_ECCSS;
+	pcr |= FIELD_PREP(FMC2_PCR_ECCSS, FMC2_PCR_ECCSS_2048);
 
 	/* Set default tclr/tar timings */
-	pcr &= ~FMC2_PCR_TCLR_MASK;
-	pcr |= FMC2_PCR_TCLR(FMC2_PCR_TCLR_DEFAULT);
-	pcr &= ~FMC2_PCR_TAR_MASK;
-	pcr |= FMC2_PCR_TAR(FMC2_PCR_TAR_DEFAULT);
+	pcr &= ~FMC2_PCR_TCLR;
+	pcr |= FIELD_PREP(FMC2_PCR_TCLR, FMC2_PCR_TCLR_DEFAULT);
+	pcr &= ~FMC2_PCR_TAR;
+	pcr |= FIELD_PREP(FMC2_PCR_TAR, FMC2_PCR_TAR_DEFAULT);
 
 	/* Enable FMC2 controller */
 	bcr1 |= FMC2_BCR1_FMC2EN;
 
-	writel_relaxed(bcr1, fmc2->io_base + FMC2_BCR1);
-	writel_relaxed(pcr, fmc2->io_base + FMC2_PCR);
-	writel_relaxed(FMC2_PMEM_DEFAULT, fmc2->io_base + FMC2_PMEM);
-	writel_relaxed(FMC2_PATT_DEFAULT, fmc2->io_base + FMC2_PATT);
+	writel_relaxed(bcr1, nfc->io_base + FMC2_BCR1);
+	writel_relaxed(pcr, nfc->io_base + FMC2_PCR);
+	writel_relaxed(FMC2_PMEM_DEFAULT, nfc->io_base + FMC2_PMEM);
+	writel_relaxed(FMC2_PATT_DEFAULT, nfc->io_base + FMC2_PATT);
 }
 
-/* Controller timings */
-static void stm32_fmc2_calc_timings(struct nand_chip *chip,
-				    const struct nand_sdr_timings *sdrt)
+static void stm32_fmc2_nfc_calc_timings(struct nand_chip *chip,
+					const struct nand_sdr_timings *sdrt)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	struct stm32_fmc2_nand *nand = to_fmc2_nand(chip);
 	struct stm32_fmc2_timings *tims = &nand->timings;
-	unsigned long hclk = clk_get_rate(fmc2->clk);
+	unsigned long hclk = clk_get_rate(nfc->clk);
 	unsigned long hclkp = NSEC_PER_SEC / (hclk / 1000);
 	unsigned long timing, tar, tclr, thiz, twait;
 	unsigned long tset_mem, tset_att, thold_mem, thold_att;
@@ -1583,8 +1545,8 @@ static void stm32_fmc2_calc_timings(struct nand_chip *chip,
 	tims->thold_att = clamp_val(timing, 1, FMC2_PMEM_PATT_TIMING_MASK);
 }
 
-static int stm32_fmc2_setup_interface(struct nand_chip *chip, int chipnr,
-				      const struct nand_data_interface *conf)
+static int stm32_fmc2_nfc_setup_interface(struct nand_chip *chip, int chipnr,
+					  const struct nand_data_interface *conf)
 {
 	const struct nand_sdr_timings *sdrt;
 
@@ -1595,71 +1557,67 @@ static int stm32_fmc2_setup_interface(struct nand_chip *chip, int chipnr,
 	if (chipnr == NAND_DATA_IFACE_CHECK_ONLY)
 		return 0;
 
-	stm32_fmc2_calc_timings(chip, sdrt);
-
-	/* Apply timings */
-	stm32_fmc2_timings_init(chip);
+	stm32_fmc2_nfc_calc_timings(chip, sdrt);
+	stm32_fmc2_nfc_timings_init(chip);
 
 	return 0;
 }
 
-/* DMA configuration */
-static int stm32_fmc2_dma_setup(struct stm32_fmc2_nfc *fmc2)
+static int stm32_fmc2_nfc_dma_setup(struct stm32_fmc2_nfc *nfc)
 {
 	int ret = 0;
 
-	fmc2->dma_tx_ch = dma_request_chan(fmc2->dev, "tx");
-	if (IS_ERR(fmc2->dma_tx_ch)) {
-		ret = PTR_ERR(fmc2->dma_tx_ch);
+	nfc->dma_tx_ch = dma_request_chan(nfc->dev, "tx");
+	if (IS_ERR(nfc->dma_tx_ch)) {
+		ret = PTR_ERR(nfc->dma_tx_ch);
 		if (ret != -ENODEV)
-			dev_err(fmc2->dev,
+			dev_err(nfc->dev,
 				"failed to request tx DMA channel: %d\n", ret);
-		fmc2->dma_tx_ch = NULL;
+		nfc->dma_tx_ch = NULL;
 		goto err_dma;
 	}
 
-	fmc2->dma_rx_ch = dma_request_chan(fmc2->dev, "rx");
-	if (IS_ERR(fmc2->dma_rx_ch)) {
-		ret = PTR_ERR(fmc2->dma_rx_ch);
+	nfc->dma_rx_ch = dma_request_chan(nfc->dev, "rx");
+	if (IS_ERR(nfc->dma_rx_ch)) {
+		ret = PTR_ERR(nfc->dma_rx_ch);
 		if (ret != -ENODEV)
-			dev_err(fmc2->dev,
+			dev_err(nfc->dev,
 				"failed to request rx DMA channel: %d\n", ret);
-		fmc2->dma_rx_ch = NULL;
+		nfc->dma_rx_ch = NULL;
 		goto err_dma;
 	}
 
-	fmc2->dma_ecc_ch = dma_request_chan(fmc2->dev, "ecc");
-	if (IS_ERR(fmc2->dma_ecc_ch)) {
-		ret = PTR_ERR(fmc2->dma_ecc_ch);
+	nfc->dma_ecc_ch = dma_request_chan(nfc->dev, "ecc");
+	if (IS_ERR(nfc->dma_ecc_ch)) {
+		ret = PTR_ERR(nfc->dma_ecc_ch);
 		if (ret != -ENODEV)
-			dev_err(fmc2->dev,
+			dev_err(nfc->dev,
 				"failed to request ecc DMA channel: %d\n", ret);
-		fmc2->dma_ecc_ch = NULL;
+		nfc->dma_ecc_ch = NULL;
 		goto err_dma;
 	}
 
-	ret = sg_alloc_table(&fmc2->dma_ecc_sg, FMC2_MAX_SG, GFP_KERNEL);
+	ret = sg_alloc_table(&nfc->dma_ecc_sg, FMC2_MAX_SG, GFP_KERNEL);
 	if (ret)
 		return ret;
 
 	/* Allocate a buffer to store ECC status registers */
-	fmc2->ecc_buf = devm_kzalloc(fmc2->dev, FMC2_MAX_ECC_BUF_LEN,
-				     GFP_KERNEL);
-	if (!fmc2->ecc_buf)
+	nfc->ecc_buf = devm_kzalloc(nfc->dev, FMC2_MAX_ECC_BUF_LEN, GFP_KERNEL);
+	if (!nfc->ecc_buf)
 		return -ENOMEM;
 
-	ret = sg_alloc_table(&fmc2->dma_data_sg, FMC2_MAX_SG, GFP_KERNEL);
+	ret = sg_alloc_table(&nfc->dma_data_sg, FMC2_MAX_SG, GFP_KERNEL);
 	if (ret)
 		return ret;
 
-	init_completion(&fmc2->dma_data_complete);
-	init_completion(&fmc2->dma_ecc_complete);
+	init_completion(&nfc->dma_data_complete);
+	init_completion(&nfc->dma_ecc_complete);
 
 	return 0;
 
 err_dma:
 	if (ret == -ENODEV) {
-		dev_warn(fmc2->dev,
+		dev_warn(nfc->dev,
 			 "DMAs not defined in the DT, polling mode is used\n");
 		ret = 0;
 	}
@@ -1667,35 +1625,34 @@ err_dma:
 	return ret;
 }
 
-/* NAND callbacks setup */
-static void stm32_fmc2_nand_callbacks_setup(struct nand_chip *chip)
+static void stm32_fmc2_nfc_nand_callbacks_setup(struct nand_chip *chip)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 
 	/*
 	 * Specific callbacks to read/write a page depending on
 	 * the mode (polling/sequencer) and the algo used (Hamming, BCH).
 	 */
-	if (fmc2->dma_tx_ch && fmc2->dma_rx_ch && fmc2->dma_ecc_ch) {
+	if (nfc->dma_tx_ch && nfc->dma_rx_ch && nfc->dma_ecc_ch) {
 		/* DMA => use sequencer mode callbacks */
-		chip->ecc.correct = stm32_fmc2_sequencer_correct;
-		chip->ecc.write_page = stm32_fmc2_sequencer_write_page;
-		chip->ecc.read_page = stm32_fmc2_sequencer_read_page;
-		chip->ecc.write_page_raw = stm32_fmc2_sequencer_write_page_raw;
-		chip->ecc.read_page_raw = stm32_fmc2_sequencer_read_page_raw;
+		chip->ecc.correct = stm32_fmc2_nfc_seq_correct;
+		chip->ecc.write_page = stm32_fmc2_nfc_seq_write_page;
+		chip->ecc.read_page = stm32_fmc2_nfc_seq_read_page;
+		chip->ecc.write_page_raw = stm32_fmc2_nfc_seq_write_page_raw;
+		chip->ecc.read_page_raw = stm32_fmc2_nfc_seq_read_page_raw;
 	} else {
 		/* No DMA => use polling mode callbacks */
-		chip->ecc.hwctl = stm32_fmc2_hwctl;
+		chip->ecc.hwctl = stm32_fmc2_nfc_hwctl;
 		if (chip->ecc.strength == FMC2_ECC_HAM) {
 			/* Hamming is used */
-			chip->ecc.calculate = stm32_fmc2_ham_calculate;
-			chip->ecc.correct = stm32_fmc2_ham_correct;
+			chip->ecc.calculate = stm32_fmc2_nfc_ham_calculate;
+			chip->ecc.correct = stm32_fmc2_nfc_ham_correct;
 			chip->ecc.options |= NAND_ECC_GENERIC_ERASED_CHECK;
 		} else {
 			/* BCH is used */
-			chip->ecc.calculate = stm32_fmc2_bch_calculate;
-			chip->ecc.correct = stm32_fmc2_bch_correct;
-			chip->ecc.read_page = stm32_fmc2_read_page;
+			chip->ecc.calculate = stm32_fmc2_nfc_bch_calculate;
+			chip->ecc.correct = stm32_fmc2_nfc_bch_correct;
+			chip->ecc.read_page = stm32_fmc2_nfc_read_page;
 		}
 	}
 
@@ -1708,9 +1665,8 @@ static void stm32_fmc2_nand_callbacks_setup(struct nand_chip *chip)
 		chip->ecc.bytes = chip->options & NAND_BUSWIDTH_16 ? 8 : 7;
 }
 
-/* FMC2 layout */
-static int stm32_fmc2_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
-					 struct mtd_oob_region *oobregion)
+static int stm32_fmc2_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
+					struct mtd_oob_region *oobregion)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
@@ -1724,8 +1680,8 @@ static int stm32_fmc2_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
 	return 0;
 }
 
-static int stm32_fmc2_nand_ooblayout_free(struct mtd_info *mtd, int section,
-					  struct mtd_oob_region *oobregion)
+static int stm32_fmc2_nfc_ooblayout_free(struct mtd_info *mtd, int section,
+					 struct mtd_oob_region *oobregion)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
@@ -1739,13 +1695,12 @@ static int stm32_fmc2_nand_ooblayout_free(struct mtd_info *mtd, int section,
 	return 0;
 }
 
-static const struct mtd_ooblayout_ops stm32_fmc2_nand_ooblayout_ops = {
-	.ecc = stm32_fmc2_nand_ooblayout_ecc,
-	.free = stm32_fmc2_nand_ooblayout_free,
+static const struct mtd_ooblayout_ops stm32_fmc2_nfc_ooblayout_ops = {
+	.ecc = stm32_fmc2_nfc_ooblayout_ecc,
+	.free = stm32_fmc2_nfc_ooblayout_free,
 };
 
-/* FMC2 caps */
-static int stm32_fmc2_calc_ecc_bytes(int step_size, int strength)
+static int stm32_fmc2_nfc_calc_ecc_bytes(int step_size, int strength)
 {
 	/* Hamming */
 	if (strength == FMC2_ECC_HAM)
@@ -1759,14 +1714,13 @@ static int stm32_fmc2_calc_ecc_bytes(int step_size, int strength)
 	return 8;
 }
 
-NAND_ECC_CAPS_SINGLE(stm32_fmc2_ecc_caps, stm32_fmc2_calc_ecc_bytes,
+NAND_ECC_CAPS_SINGLE(stm32_fmc2_nfc_ecc_caps, stm32_fmc2_nfc_calc_ecc_bytes,
 		     FMC2_ECC_STEP_SIZE,
 		     FMC2_ECC_HAM, FMC2_ECC_BCH4, FMC2_ECC_BCH8);
 
-/* FMC2 controller ops */
-static int stm32_fmc2_attach_chip(struct nand_chip *chip)
+static int stm32_fmc2_nfc_attach_chip(struct nand_chip *chip)
 {
-	struct stm32_fmc2_nfc *fmc2 = to_stm32_nfc(chip->controller);
+	struct stm32_fmc2_nfc *nfc = to_stm32_nfc(chip->controller);
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	int ret;
 
@@ -1778,49 +1732,45 @@ static int stm32_fmc2_attach_chip(struct nand_chip *chip)
 	 * ECC sector size = 512
 	 */
 	if (chip->ecc.mode != NAND_ECC_HW) {
-		dev_err(fmc2->dev, "nand_ecc_mode is not well defined in the DT\n");
+		dev_err(nfc->dev, "nand_ecc_mode is not well defined in the DT\n");
 		return -EINVAL;
 	}
 
-	ret = nand_ecc_choose_conf(chip, &stm32_fmc2_ecc_caps,
+	ret = nand_ecc_choose_conf(chip, &stm32_fmc2_nfc_ecc_caps,
 				   mtd->oobsize - FMC2_BBM_LEN);
 	if (ret) {
-		dev_err(fmc2->dev, "no valid ECC settings set\n");
+		dev_err(nfc->dev, "no valid ECC settings set\n");
 		return ret;
 	}
 
 	if (mtd->writesize / chip->ecc.size > FMC2_MAX_SG) {
-		dev_err(fmc2->dev, "nand page size is not supported\n");
+		dev_err(nfc->dev, "nand page size is not supported\n");
 		return -EINVAL;
 	}
 
 	if (chip->bbt_options & NAND_BBT_USE_FLASH)
 		chip->bbt_options |= NAND_BBT_NO_OOB;
 
-	/* NAND callbacks setup */
-	stm32_fmc2_nand_callbacks_setup(chip);
+	stm32_fmc2_nfc_nand_callbacks_setup(chip);
 
-	/* Define ECC layout */
-	mtd_set_ooblayout(mtd, &stm32_fmc2_nand_ooblayout_ops);
+	mtd_set_ooblayout(mtd, &stm32_fmc2_nfc_ooblayout_ops);
 
-	/* Configure bus width to 16-bit */
 	if (chip->options & NAND_BUSWIDTH_16)
-		stm32_fmc2_set_buswidth_16(fmc2, true);
+		stm32_fmc2_nfc_set_buswidth_16(nfc, true);
 
 	return 0;
 }
 
-static const struct nand_controller_ops stm32_fmc2_nand_controller_ops = {
-	.attach_chip = stm32_fmc2_attach_chip,
-	.exec_op = stm32_fmc2_exec_op,
-	.setup_data_interface = stm32_fmc2_setup_interface,
+static const struct nand_controller_ops stm32_fmc2_nfc_controller_ops = {
+	.attach_chip = stm32_fmc2_nfc_attach_chip,
+	.exec_op = stm32_fmc2_nfc_exec_op,
+	.setup_data_interface = stm32_fmc2_nfc_setup_interface,
 };
 
-/* FMC2 probe */
-static int stm32_fmc2_parse_child(struct stm32_fmc2_nfc *fmc2,
-				  struct device_node *dn)
+static int stm32_fmc2_nfc_parse_child(struct stm32_fmc2_nfc *nfc,
+				      struct device_node *dn)
 {
-	struct stm32_fmc2_nand *nand = &fmc2->nand;
+	struct stm32_fmc2_nand *nand = &nfc->nand;
 	u32 cs;
 	int ret, i;
 
@@ -1829,29 +1779,29 @@ static int stm32_fmc2_parse_child(struct stm32_fmc2_nfc *fmc2,
 
 	nand->ncs /= sizeof(u32);
 	if (!nand->ncs) {
-		dev_err(fmc2->dev, "invalid reg property size\n");
+		dev_err(nfc->dev, "invalid reg property size\n");
 		return -EINVAL;
 	}
 
 	for (i = 0; i < nand->ncs; i++) {
 		ret = of_property_read_u32_index(dn, "reg", i, &cs);
 		if (ret) {
-			dev_err(fmc2->dev, "could not retrieve reg property: %d\n",
+			dev_err(nfc->dev, "could not retrieve reg property: %d\n",
 				ret);
 			return ret;
 		}
 
 		if (cs > FMC2_MAX_CE) {
-			dev_err(fmc2->dev, "invalid reg value: %d\n", cs);
+			dev_err(nfc->dev, "invalid reg value: %d\n", cs);
 			return -EINVAL;
 		}
 
-		if (fmc2->cs_assigned & BIT(cs)) {
-			dev_err(fmc2->dev, "cs already assigned: %d\n", cs);
+		if (nfc->cs_assigned & BIT(cs)) {
+			dev_err(nfc->dev, "cs already assigned: %d\n", cs);
 			return -EINVAL;
 		}
 
-		fmc2->cs_assigned |= BIT(cs);
+		nfc->cs_assigned |= BIT(cs);
 		nand->cs_used[i] = cs;
 	}
 
@@ -1860,25 +1810,25 @@ static int stm32_fmc2_parse_child(struct stm32_fmc2_nfc *fmc2,
 	return 0;
 }
 
-static int stm32_fmc2_parse_dt(struct stm32_fmc2_nfc *fmc2)
+static int stm32_fmc2_nfc_parse_dt(struct stm32_fmc2_nfc *nfc)
 {
-	struct device_node *dn = fmc2->dev->of_node;
+	struct device_node *dn = nfc->dev->of_node;
 	struct device_node *child;
 	int nchips = of_get_child_count(dn);
 	int ret = 0;
 
 	if (!nchips) {
-		dev_err(fmc2->dev, "NAND chip not defined\n");
+		dev_err(nfc->dev, "NAND chip not defined\n");
 		return -EINVAL;
 	}
 
 	if (nchips > 1) {
-		dev_err(fmc2->dev, "too many NAND chips defined\n");
+		dev_err(nfc->dev, "too many NAND chips defined\n");
 		return -EINVAL;
 	}
 
 	for_each_child_of_node(dn, child) {
-		ret = stm32_fmc2_parse_child(fmc2, child);
+		ret = stm32_fmc2_nfc_parse_child(nfc, child);
 		if (ret < 0) {
 			of_node_put(child);
 			return ret;
@@ -1888,106 +1838,108 @@ static int stm32_fmc2_parse_dt(struct stm32_fmc2_nfc *fmc2)
 	return ret;
 }
 
-static int stm32_fmc2_probe(struct platform_device *pdev)
+static int stm32_fmc2_nfc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct reset_control *rstc;
-	struct stm32_fmc2_nfc *fmc2;
+	struct stm32_fmc2_nfc *nfc;
 	struct stm32_fmc2_nand *nand;
 	struct resource *res;
 	struct mtd_info *mtd;
 	struct nand_chip *chip;
 	int chip_cs, mem_region, ret, irq;
 
-	fmc2 = devm_kzalloc(dev, sizeof(*fmc2), GFP_KERNEL);
-	if (!fmc2)
+	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
+	if (!nfc)
 		return -ENOMEM;
 
-	fmc2->dev = dev;
-	nand_controller_init(&fmc2->base);
-	fmc2->base.ops = &stm32_fmc2_nand_controller_ops;
+	nfc->dev = dev;
+	nand_controller_init(&nfc->base);
+	nfc->base.ops = &stm32_fmc2_nfc_controller_ops;
 
-	ret = stm32_fmc2_parse_dt(fmc2);
+	ret = stm32_fmc2_nfc_parse_dt(nfc);
 	if (ret)
 		return ret;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	fmc2->io_base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(fmc2->io_base))
-		return PTR_ERR(fmc2->io_base);
+	nfc->io_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(nfc->io_base))
+		return PTR_ERR(nfc->io_base);
 
-	fmc2->io_phys_addr = res->start;
+	nfc->io_phys_addr = res->start;
 
 	for (chip_cs = 0, mem_region = 1; chip_cs < FMC2_MAX_CE;
 	     chip_cs++, mem_region += 3) {
-		if (!(fmc2->cs_assigned & BIT(chip_cs)))
+		if (!(nfc->cs_assigned & BIT(chip_cs)))
 			continue;
 
 		res = platform_get_resource(pdev, IORESOURCE_MEM, mem_region);
-		fmc2->data_base[chip_cs] = devm_ioremap_resource(dev, res);
-		if (IS_ERR(fmc2->data_base[chip_cs]))
-			return PTR_ERR(fmc2->data_base[chip_cs]);
+		nfc->data_base[chip_cs] = devm_ioremap_resource(dev, res);
+		if (IS_ERR(nfc->data_base[chip_cs]))
+			return PTR_ERR(nfc->data_base[chip_cs]);
 
-		fmc2->data_phys_addr[chip_cs] = res->start;
+		nfc->data_phys_addr[chip_cs] = res->start;
 
 		res = platform_get_resource(pdev, IORESOURCE_MEM,
 					    mem_region + 1);
-		fmc2->cmd_base[chip_cs] = devm_ioremap_resource(dev, res);
-		if (IS_ERR(fmc2->cmd_base[chip_cs]))
-			return PTR_ERR(fmc2->cmd_base[chip_cs]);
+		nfc->cmd_base[chip_cs] = devm_ioremap_resource(dev, res);
+		if (IS_ERR(nfc->cmd_base[chip_cs]))
+			return PTR_ERR(nfc->cmd_base[chip_cs]);
 
 		res = platform_get_resource(pdev, IORESOURCE_MEM,
 					    mem_region + 2);
-		fmc2->addr_base[chip_cs] = devm_ioremap_resource(dev, res);
-		if (IS_ERR(fmc2->addr_base[chip_cs]))
-			return PTR_ERR(fmc2->addr_base[chip_cs]);
+		nfc->addr_base[chip_cs] = devm_ioremap_resource(dev, res);
+		if (IS_ERR(nfc->addr_base[chip_cs]))
+			return PTR_ERR(nfc->addr_base[chip_cs]);
 	}
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
 
-	ret = devm_request_irq(dev, irq, stm32_fmc2_irq, 0,
-			       dev_name(dev), fmc2);
+	ret = devm_request_irq(dev, irq, stm32_fmc2_nfc_irq, 0,
+			       dev_name(dev), nfc);
 	if (ret) {
 		dev_err(dev, "failed to request irq\n");
 		return ret;
 	}
 
-	init_completion(&fmc2->complete);
+	init_completion(&nfc->complete);
 
-	fmc2->clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(fmc2->clk))
-		return PTR_ERR(fmc2->clk);
+	nfc->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(nfc->clk))
+		return PTR_ERR(nfc->clk);
 
-	ret = clk_prepare_enable(fmc2->clk);
+	ret = clk_prepare_enable(nfc->clk);
 	if (ret) {
 		dev_err(dev, "can not enable the clock\n");
 		return ret;
 	}
 
 	rstc = devm_reset_control_get(dev, NULL);
-	if (!IS_ERR(rstc)) {
+	if (IS_ERR(rstc)) {
+		ret = PTR_ERR(rstc);
+		if (ret == -EPROBE_DEFER)
+			goto err_clk_disable;
+	} else {
 		reset_control_assert(rstc);
 		reset_control_deassert(rstc);
 	}
 
-	/* DMA setup */
-	ret = stm32_fmc2_dma_setup(fmc2);
+	ret = stm32_fmc2_nfc_dma_setup(nfc);
 	if (ret)
-		return ret;
+		goto err_release_dma;
 
-	/* FMC2 init routine */
-	stm32_fmc2_init(fmc2);
+	stm32_fmc2_nfc_init(nfc);
 
-	nand = &fmc2->nand;
+	nand = &nfc->nand;
 	chip = &nand->chip;
 	mtd = nand_to_mtd(chip);
 	mtd->dev.parent = dev;
 
-	chip->controller = &fmc2->base;
+	chip->controller = &nfc->base;
 	chip->options |= NAND_BUSWIDTH_AUTO | NAND_NO_SUBPAGE_WRITE |
-			 NAND_USE_BOUNCE_BUFFER;
+			 NAND_USES_DMA;
 
 	/* Default ECC settings */
 	chip->ecc.mode = NAND_ECC_HW;
@@ -1997,86 +1949,91 @@ static int stm32_fmc2_probe(struct platform_device *pdev)
 	/* Scan to find existence of the device */
 	ret = nand_scan(chip, nand->ncs);
 	if (ret)
-		goto err_scan;
+		goto err_release_dma;
 
 	ret = mtd_device_register(mtd, NULL, 0);
 	if (ret)
-		goto err_device_register;
+		goto err_nand_cleanup;
 
-	platform_set_drvdata(pdev, fmc2);
+	platform_set_drvdata(pdev, nfc);
 
 	return 0;
 
-err_device_register:
+err_nand_cleanup:
 	nand_cleanup(chip);
 
-err_scan:
-	if (fmc2->dma_ecc_ch)
-		dma_release_channel(fmc2->dma_ecc_ch);
-	if (fmc2->dma_tx_ch)
-		dma_release_channel(fmc2->dma_tx_ch);
-	if (fmc2->dma_rx_ch)
-		dma_release_channel(fmc2->dma_rx_ch);
+err_release_dma:
+	if (nfc->dma_ecc_ch)
+		dma_release_channel(nfc->dma_ecc_ch);
+	if (nfc->dma_tx_ch)
+		dma_release_channel(nfc->dma_tx_ch);
+	if (nfc->dma_rx_ch)
+		dma_release_channel(nfc->dma_rx_ch);
 
-	sg_free_table(&fmc2->dma_data_sg);
-	sg_free_table(&fmc2->dma_ecc_sg);
+	sg_free_table(&nfc->dma_data_sg);
+	sg_free_table(&nfc->dma_ecc_sg);
 
-	clk_disable_unprepare(fmc2->clk);
+err_clk_disable:
+	clk_disable_unprepare(nfc->clk);
 
 	return ret;
 }
 
-static int stm32_fmc2_remove(struct platform_device *pdev)
+static int stm32_fmc2_nfc_remove(struct platform_device *pdev)
 {
-	struct stm32_fmc2_nfc *fmc2 = platform_get_drvdata(pdev);
-	struct stm32_fmc2_nand *nand = &fmc2->nand;
+	struct stm32_fmc2_nfc *nfc = platform_get_drvdata(pdev);
+	struct stm32_fmc2_nand *nand = &nfc->nand;
+	struct nand_chip *chip = &nand->chip;
+	int ret;
 
-	nand_release(&nand->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
-	if (fmc2->dma_ecc_ch)
-		dma_release_channel(fmc2->dma_ecc_ch);
-	if (fmc2->dma_tx_ch)
-		dma_release_channel(fmc2->dma_tx_ch);
-	if (fmc2->dma_rx_ch)
-		dma_release_channel(fmc2->dma_rx_ch);
+	if (nfc->dma_ecc_ch)
+		dma_release_channel(nfc->dma_ecc_ch);
+	if (nfc->dma_tx_ch)
+		dma_release_channel(nfc->dma_tx_ch);
+	if (nfc->dma_rx_ch)
+		dma_release_channel(nfc->dma_rx_ch);
 
-	sg_free_table(&fmc2->dma_data_sg);
-	sg_free_table(&fmc2->dma_ecc_sg);
+	sg_free_table(&nfc->dma_data_sg);
+	sg_free_table(&nfc->dma_ecc_sg);
 
-	clk_disable_unprepare(fmc2->clk);
+	clk_disable_unprepare(nfc->clk);
 
 	return 0;
 }
 
-static int __maybe_unused stm32_fmc2_suspend(struct device *dev)
+static int __maybe_unused stm32_fmc2_nfc_suspend(struct device *dev)
 {
-	struct stm32_fmc2_nfc *fmc2 = dev_get_drvdata(dev);
+	struct stm32_fmc2_nfc *nfc = dev_get_drvdata(dev);
 
-	clk_disable_unprepare(fmc2->clk);
+	clk_disable_unprepare(nfc->clk);
 
 	pinctrl_pm_select_sleep_state(dev);
 
 	return 0;
 }
 
-static int __maybe_unused stm32_fmc2_resume(struct device *dev)
+static int __maybe_unused stm32_fmc2_nfc_resume(struct device *dev)
 {
-	struct stm32_fmc2_nfc *fmc2 = dev_get_drvdata(dev);
-	struct stm32_fmc2_nand *nand = &fmc2->nand;
+	struct stm32_fmc2_nfc *nfc = dev_get_drvdata(dev);
+	struct stm32_fmc2_nand *nand = &nfc->nand;
 	int chip_cs, ret;
 
 	pinctrl_pm_select_default_state(dev);
 
-	ret = clk_prepare_enable(fmc2->clk);
+	ret = clk_prepare_enable(nfc->clk);
 	if (ret) {
 		dev_err(dev, "can not enable the clock\n");
 		return ret;
 	}
 
-	stm32_fmc2_init(fmc2);
+	stm32_fmc2_nfc_init(nfc);
 
 	for (chip_cs = 0; chip_cs < FMC2_MAX_CE; chip_cs++) {
-		if (!(fmc2->cs_assigned & BIT(chip_cs)))
+		if (!(nfc->cs_assigned & BIT(chip_cs)))
 			continue;
 
 		nand_reset(&nand->chip, chip_cs);
@@ -2085,27 +2042,27 @@ static int __maybe_unused stm32_fmc2_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(stm32_fmc2_pm_ops, stm32_fmc2_suspend,
-			 stm32_fmc2_resume);
+static SIMPLE_DEV_PM_OPS(stm32_fmc2_nfc_pm_ops, stm32_fmc2_nfc_suspend,
+			 stm32_fmc2_nfc_resume);
 
-static const struct of_device_id stm32_fmc2_match[] = {
+static const struct of_device_id stm32_fmc2_nfc_match[] = {
 	{.compatible = "st,stm32mp15-fmc2"},
 	{}
 };
-MODULE_DEVICE_TABLE(of, stm32_fmc2_match);
+MODULE_DEVICE_TABLE(of, stm32_fmc2_nfc_match);
 
-static struct platform_driver stm32_fmc2_driver = {
-	.probe	= stm32_fmc2_probe,
-	.remove	= stm32_fmc2_remove,
+static struct platform_driver stm32_fmc2_nfc_driver = {
+	.probe	= stm32_fmc2_nfc_probe,
+	.remove	= stm32_fmc2_nfc_remove,
 	.driver	= {
-		.name = "stm32_fmc2_nand",
-		.of_match_table = stm32_fmc2_match,
-		.pm = &stm32_fmc2_pm_ops,
+		.name = "stm32_fmc2_nfc",
+		.of_match_table = stm32_fmc2_nfc_match,
+		.pm = &stm32_fmc2_nfc_pm_ops,
 	},
 };
-module_platform_driver(stm32_fmc2_driver);
+module_platform_driver(stm32_fmc2_nfc_driver);
 
-MODULE_ALIAS("platform:stm32_fmc2_nand");
+MODULE_ALIAS("platform:stm32_fmc2_nfc");
 MODULE_AUTHOR("Christophe Kerello <christophe.kerello@st.com>");
-MODULE_DESCRIPTION("STMicroelectronics STM32 FMC2 nand driver");
+MODULE_DESCRIPTION("STMicroelectronics STM32 FMC2 NFC driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index 5f3e40b79fb1..ffbc1651fadc 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -1698,7 +1698,7 @@ static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
 		ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
 		ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
 		ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
-		nand->options |= NAND_USE_BOUNCE_BUFFER;
+		nand->options |= NAND_USES_DMA;
 	} else {
 		ecc->read_page = sunxi_nfc_hw_ecc_read_page;
 		ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
@@ -1907,7 +1907,8 @@ static int sunxi_nfc_exec_op(struct nand_chip *nand,
 	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 	const struct nand_op_parser *parser;
 
-	sunxi_nfc_select_chip(nand, op->cs);
+	if (!check_only)
+		sunxi_nfc_select_chip(nand, op->cs);
 
 	if (sunxi_nand->sels[op->cs].rb >= 0)
 		parser = &sunxi_nfc_op_parser;
@@ -2003,7 +2004,7 @@ static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
 	ret = mtd_device_register(mtd, NULL, 0);
 	if (ret) {
 		dev_err(dev, "failed to register mtd device: %d\n", ret);
-		nand_release(nand);
+		nand_cleanup(nand);
 		return ret;
 	}
 
@@ -2038,13 +2039,18 @@ static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
 {
 	struct sunxi_nand_chip *sunxi_nand;
+	struct nand_chip *chip;
+	int ret;
 
 	while (!list_empty(&nfc->chips)) {
 		sunxi_nand = list_first_entry(&nfc->chips,
 					      struct sunxi_nand_chip,
 					      node);
-		nand_release(&sunxi_nand->nand);
-		sunxi_nand_ecc_cleanup(&sunxi_nand->nand.ecc);
+		chip = &sunxi_nand->nand;
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
+		sunxi_nand_ecc_cleanup(&chip->ecc);
 		list_del(&sunxi_nand->node);
 	}
 }
diff --git a/drivers/mtd/nand/raw/tango_nand.c b/drivers/mtd/nand/raw/tango_nand.c
index 9acf2de37ee0..246871e01027 100644
--- a/drivers/mtd/nand/raw/tango_nand.c
+++ b/drivers/mtd/nand/raw/tango_nand.c
@@ -568,7 +568,7 @@ static int chip_init(struct device *dev, struct device_node *np)
 	chip->legacy.select_chip = tango_select_chip;
 	chip->legacy.cmd_ctrl = tango_cmd_ctrl;
 	chip->legacy.dev_ready = tango_dev_ready;
-	chip->options = NAND_USE_BOUNCE_BUFFER |
+	chip->options = NAND_USES_DMA |
 			NAND_NO_SUBPAGE_WRITE |
 			NAND_WAIT_TCCS;
 	chip->controller = &nfc->hw;
@@ -600,14 +600,19 @@ static int chip_init(struct device *dev, struct device_node *np)
 
 static int tango_nand_remove(struct platform_device *pdev)
 {
-	int cs;
 	struct tango_nfc *nfc = platform_get_drvdata(pdev);
+	struct nand_chip *chip;
+	int cs, ret;
 
 	dma_release_channel(nfc->chan);
 
 	for (cs = 0; cs < MAX_CS; ++cs) {
-		if (nfc->chips[cs])
-			nand_release(&nfc->chips[cs]->nand_chip);
+		if (nfc->chips[cs]) {
+			chip = &nfc->chips[cs]->nand_chip;
+			ret = mtd_device_unregister(nand_to_mtd(chip));
+			WARN_ON(ret);
+			nand_cleanup(chip);
+		}
 	}
 
 	return 0;
diff --git a/drivers/mtd/nand/raw/tegra_nand.c b/drivers/mtd/nand/raw/tegra_nand.c
index 3cc9a4c41443..f9d046b2cd3b 100644
--- a/drivers/mtd/nand/raw/tegra_nand.c
+++ b/drivers/mtd/nand/raw/tegra_nand.c
@@ -467,7 +467,9 @@ static int tegra_nand_exec_op(struct nand_chip *chip,
 			      const struct nand_operation *op,
 			      bool check_only)
 {
-	tegra_nand_select_target(chip, op->cs);
+	if (!check_only)
+		tegra_nand_select_target(chip, op->cs);
+
 	return nand_op_parser_exec_op(chip, &tegra_nand_op_parser, op,
 				      check_only);
 }
@@ -1113,7 +1115,7 @@ static int tegra_nand_chips_init(struct device *dev,
 	if (!mtd->name)
 		mtd->name = "tegra_nand";
 
-	chip->options = NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER;
+	chip->options = NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA;
 
 	ret = nand_scan(chip, 1);
 	if (ret)
diff --git a/drivers/mtd/nand/raw/tmio_nand.c b/drivers/mtd/nand/raw/tmio_nand.c
index db030f1701ee..843a8683b737 100644
--- a/drivers/mtd/nand/raw/tmio_nand.c
+++ b/drivers/mtd/nand/raw/tmio_nand.c
@@ -448,7 +448,7 @@ static int tmio_probe(struct platform_device *dev)
 	if (!retval)
 		return retval;
 
-	nand_release(nand_chip);
+	nand_cleanup(nand_chip);
 
 err_irq:
 	tmio_hw_stop(dev, tmio);
@@ -458,8 +458,12 @@ err_irq:
 static int tmio_remove(struct platform_device *dev)
 {
 	struct tmio_nand *tmio = platform_get_drvdata(dev);
+	struct nand_chip *chip = &tmio->chip;
+	int ret;
 
-	nand_release(&tmio->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	tmio_hw_stop(dev, tmio);
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/txx9ndfmc.c b/drivers/mtd/nand/raw/txx9ndfmc.c
index 2642d5bb3241..47d966871445 100644
--- a/drivers/mtd/nand/raw/txx9ndfmc.c
+++ b/drivers/mtd/nand/raw/txx9ndfmc.c
@@ -371,7 +371,7 @@ static int __init txx9ndfmc_probe(struct platform_device *dev)
 static int __exit txx9ndfmc_remove(struct platform_device *dev)
 {
 	struct txx9ndfmc_drvdata *drvdata = platform_get_drvdata(dev);
-	int i;
+	int ret, i;
 
 	if (!drvdata)
 		return 0;
@@ -385,7 +385,9 @@ static int __exit txx9ndfmc_remove(struct platform_device *dev)
 		chip = mtd_to_nand(mtd);
 		txx9_priv = nand_get_controller_data(chip);
 
-		nand_release(chip);
+		ret = mtd_device_unregister(nand_to_mtd(chip));
+		WARN_ON(ret);
+		nand_cleanup(chip);
 		kfree(txx9_priv->mtdname);
 		kfree(txx9_priv);
 	}
diff --git a/drivers/mtd/nand/raw/vf610_nfc.c b/drivers/mtd/nand/raw/vf610_nfc.c
index 6b399a75f9ae..7248c5901183 100644
--- a/drivers/mtd/nand/raw/vf610_nfc.c
+++ b/drivers/mtd/nand/raw/vf610_nfc.c
@@ -502,7 +502,9 @@ static int vf610_nfc_exec_op(struct nand_chip *chip,
 			     const struct nand_operation *op,
 			     bool check_only)
 {
-	vf610_nfc_select_target(chip, op->cs);
+	if (!check_only)
+		vf610_nfc_select_target(chip, op->cs);
+
 	return nand_op_parser_exec_op(chip, &vf610_nfc_op_parser, op,
 				      check_only);
 }
@@ -915,8 +917,12 @@ err_disable_clk:
 static int vf610_nfc_remove(struct platform_device *pdev)
 {
 	struct vf610_nfc *nfc = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &nfc->chip;
+	int ret;
 
-	nand_release(&nfc->chip);
+	ret = mtd_device_unregister(nand_to_mtd(chip));
+	WARN_ON(ret);
+	nand_cleanup(chip);
 	clk_disable_unprepare(nfc->clk);
 	return 0;
 }
diff --git a/drivers/mtd/nand/raw/xway_nand.c b/drivers/mtd/nand/raw/xway_nand.c
index 834f794816a9..94bfba994326 100644
--- a/drivers/mtd/nand/raw/xway_nand.c
+++ b/drivers/mtd/nand/raw/xway_nand.c
@@ -210,7 +210,7 @@ static int xway_nand_probe(struct platform_device *pdev)
 
 	err = mtd_device_register(mtd, NULL, 0);
 	if (err)
-		nand_release(&data->chip);
+		nand_cleanup(&data->chip);
 
 	return err;
 }
@@ -221,8 +221,12 @@ static int xway_nand_probe(struct platform_device *pdev)
 static int xway_nand_remove(struct platform_device *pdev)
 {
 	struct xway_nand_data *data = platform_get_drvdata(pdev);
+	struct nand_chip *chip = &data->chip;
+	int ret;
 
-	nand_release(&data->chip);
+	ret = mtd_device_unregister(mtd);
+	WARN_ON(ret);
+	nand_cleanup(chip);
 
 	return 0;
 }
diff --git a/drivers/mtd/parsers/cmdlinepart.c b/drivers/mtd/parsers/cmdlinepart.c
index c86f2db8c882..a79e4d866b08 100644
--- a/drivers/mtd/parsers/cmdlinepart.c
+++ b/drivers/mtd/parsers/cmdlinepart.c
@@ -9,7 +9,7 @@
  *
  * mtdparts=<mtddef>[;<mtddef]
  * <mtddef>  := <mtd-id>:<partdef>[,<partdef>]
- * <partdef> := <size>[@<offset>][<name>][ro][lk]
+ * <partdef> := <size>[@<offset>][<name>][ro][lk][slc]
  * <mtd-id>  := unique name used in mapping driver/device (mtd->name)
  * <size>    := standard linux memsize OR "-" to denote all remaining space
  *              size is automatically truncated at end of device
@@ -92,7 +92,7 @@ static struct mtd_partition * newpart(char *s,
 	int name_len;
 	unsigned char *extra_mem;
 	char delim;
-	unsigned int mask_flags;
+	unsigned int mask_flags, add_flags;
 
 	/* fetch the partition size */
 	if (*s == '-') {
@@ -109,6 +109,7 @@ static struct mtd_partition * newpart(char *s,
 
 	/* fetch partition name and flags */
 	mask_flags = 0; /* this is going to be a regular partition */
+	add_flags = 0;
 	delim = 0;
 
 	/* check for offset */
@@ -152,6 +153,12 @@ static struct mtd_partition * newpart(char *s,
 		s += 2;
 	}
 
+	/* if slc is found use emulated SLC mode on this partition*/
+	if (!strncmp(s, "slc", 3)) {
+		add_flags |= MTD_SLC_ON_MLC_EMULATION;
+		s += 3;
+	}
+
 	/* test if more partitions are following */
 	if (*s == ',') {
 		if (size == SIZE_REMAINING) {
@@ -184,6 +191,7 @@ static struct mtd_partition * newpart(char *s,
 	parts[this_part].size = size;
 	parts[this_part].offset = offset;
 	parts[this_part].mask_flags = mask_flags;
+	parts[this_part].add_flags = add_flags;
 	if (name)
 		strlcpy(extra_mem, name, name_len + 1);
 	else
@@ -218,12 +226,29 @@ static int mtdpart_setup_real(char *s)
 		struct cmdline_mtd_partition *this_mtd;
 		struct mtd_partition *parts;
 		int mtd_id_len, num_parts;
-		char *p, *mtd_id;
+		char *p, *mtd_id, *semicol;
+
+		/*
+		 * Replace the first ';' by a NULL char so strrchr can work
+		 * properly.
+		 */
+		semicol = strchr(s, ';');
+		if (semicol)
+			*semicol = '\0';
 
 		mtd_id = s;
 
-		/* fetch <mtd-id> */
-		p = strchr(s, ':');
+		/*
+		 * fetch <mtd-id>. We use strrchr to ignore all ':' that could
+		 * be present in the MTD name, only the last one is interpreted
+		 * as an <mtd-id>/<part-definition> separator.
+		 */
+		p = strrchr(s, ':');
+
+		/* Restore the ';' now. */
+		if (semicol)
+			*semicol = ';';
+
 		if (!p) {
 			pr_err("no mtd-id\n");
 			return -EINVAL;
diff --git a/drivers/mtd/parsers/ofpart.c b/drivers/mtd/parsers/ofpart.c
index 3caeabf27987..daf507c123e6 100644
--- a/drivers/mtd/parsers/ofpart.c
+++ b/drivers/mtd/parsers/ofpart.c
@@ -117,6 +117,9 @@ static int parse_fixed_partitions(struct mtd_info *master,
 		if (of_get_property(pp, "lock", &len))
 			parts[i].mask_flags |= MTD_POWERUP_LOCK;
 
+		if (of_property_read_bool(pp, "slc-mode"))
+			parts[i].add_flags |= MTD_SLC_ON_MLC_EMULATION;
+
 		i++;
 	}
 
diff --git a/drivers/mtd/spi-nor/Kconfig b/drivers/mtd/spi-nor/Kconfig
index 6e816eafb312..ffc4b380f2b1 100644
--- a/drivers/mtd/spi-nor/Kconfig
+++ b/drivers/mtd/spi-nor/Kconfig
@@ -1,12 +1,12 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig MTD_SPI_NOR
-	tristate "SPI-NOR device support"
+	tristate "SPI NOR device support"
 	depends on MTD
 	depends on MTD && SPI_MASTER
 	select SPI_MEM
 	help
 	  This is the framework for the SPI NOR which can be used by the SPI
-	  device drivers and the SPI-NOR device driver.
+	  device drivers and the SPI NOR device driver.
 
 if MTD_SPI_NOR
 
diff --git a/drivers/mtd/spi-nor/controllers/Kconfig b/drivers/mtd/spi-nor/controllers/Kconfig
index 10b86660b821..d89a5ea9446a 100644
--- a/drivers/mtd/spi-nor/controllers/Kconfig
+++ b/drivers/mtd/spi-nor/controllers/Kconfig
@@ -21,11 +21,11 @@ config SPI_CADENCE_QUADSPI
 	  Flash as an MTD device.
 
 config SPI_HISI_SFC
-	tristate "Hisilicon FMC SPI-NOR Flash Controller(SFC)"
+	tristate "Hisilicon FMC SPI NOR Flash Controller(SFC)"
 	depends on ARCH_HISI || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
-	  This enables support for HiSilicon FMC SPI-NOR flash controller.
+	  This enables support for HiSilicon FMC SPI NOR flash controller.
 
 config SPI_NXP_SPIFI
 	tristate "NXP SPI Flash Interface (SPIFI)"
diff --git a/drivers/mtd/spi-nor/controllers/aspeed-smc.c b/drivers/mtd/spi-nor/controllers/aspeed-smc.c
index ae85e4c0e114..7225870e8b18 100644
--- a/drivers/mtd/spi-nor/controllers/aspeed-smc.c
+++ b/drivers/mtd/spi-nor/controllers/aspeed-smc.c
@@ -727,7 +727,7 @@ static int aspeed_smc_chip_setup_finish(struct aspeed_smc_chip *chip)
 
 	/*
 	 * TODO: Adjust clocks if fast read is supported and interpret
-	 * SPI-NOR flags to adjust controller settings.
+	 * SPI NOR flags to adjust controller settings.
 	 */
 	if (chip->nor.read_proto == SNOR_PROTO_1_1_1) {
 		if (chip->nor.read_dummy == 0)
diff --git a/drivers/mtd/spi-nor/controllers/hisi-sfc.c b/drivers/mtd/spi-nor/controllers/hisi-sfc.c
index 6c7a4118752e..95c502173cbd 100644
--- a/drivers/mtd/spi-nor/controllers/hisi-sfc.c
+++ b/drivers/mtd/spi-nor/controllers/hisi-sfc.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * HiSilicon FMC SPI-NOR flash controller driver
+ * HiSilicon FMC SPI NOR flash controller driver
  *
  * Copyright (c) 2015-2016 HiSilicon Technologies Co., Ltd.
  */
diff --git a/drivers/mtd/spi-nor/controllers/nxp-spifi.c b/drivers/mtd/spi-nor/controllers/nxp-spifi.c
index 9a5b1a7c636a..5703e8313980 100644
--- a/drivers/mtd/spi-nor/controllers/nxp-spifi.c
+++ b/drivers/mtd/spi-nor/controllers/nxp-spifi.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * SPI-NOR driver for NXP SPI Flash Interface (SPIFI)
+ * SPI NOR driver for NXP SPI Flash Interface (SPIFI)
  *
  * Copyright (C) 2015 Joachim Eastwood <manabian@gmail.com>
  *
diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c
index cc68ea84318e..0369d98b2d12 100644
--- a/drivers/mtd/spi-nor/core.c
+++ b/drivers/mtd/spi-nor/core.c
@@ -499,7 +499,7 @@ int spi_nor_xread_sr(struct spi_nor *nor, u8 *sr)
  * the flash is ready for new commands.
  * @nor:	pointer to 'struct spi_nor'.
  *
- * Return: 0 on success, -errno otherwise.
+ * Return: 1 if ready, 0 if not ready, -errno on errors.
  */
 static int spi_nor_xsr_ready(struct spi_nor *nor)
 {
@@ -542,7 +542,7 @@ static void spi_nor_clear_sr(struct spi_nor *nor)
  * for new commands.
  * @nor:	pointer to 'struct spi_nor'.
  *
- * Return: 0 on success, -errno otherwise.
+ * Return: 1 if ready, 0 if not ready, -errno on errors.
  */
 static int spi_nor_sr_ready(struct spi_nor *nor)
 {
@@ -606,7 +606,7 @@ static void spi_nor_clear_fsr(struct spi_nor *nor)
  * ready for new commands.
  * @nor:	pointer to 'struct spi_nor'.
  *
- * Return: 0 on success, -errno otherwise.
+ * Return: 1 if ready, 0 if not ready, -errno on errors.
  */
 static int spi_nor_fsr_ready(struct spi_nor *nor)
 {
@@ -640,14 +640,14 @@ static int spi_nor_fsr_ready(struct spi_nor *nor)
 		return -EIO;
 	}
 
-	return nor->bouncebuf[0] & FSR_READY;
+	return !!(nor->bouncebuf[0] & FSR_READY);
 }
 
 /**
  * spi_nor_ready() - Query the flash to see if it is ready for new commands.
  * @nor:	pointer to 'struct spi_nor'.
  *
- * Return: 0 on success, -errno otherwise.
+ * Return: 1 if ready, 0 if not ready, -errno on errors.
  */
 static int spi_nor_ready(struct spi_nor *nor)
 {
@@ -2469,7 +2469,7 @@ static int spi_nor_select_read(struct spi_nor *nor,
 	nor->read_proto = read->proto;
 
 	/*
-	 * In the spi-nor framework, we don't need to make the difference
+	 * In the SPI NOR framework, we don't need to make the difference
 	 * between mode clock cycles and wait state clock cycles.
 	 * Indeed, the value of the mode clock cycles is used by a QSPI
 	 * flash memory to know whether it should enter or leave its 0-4-4
@@ -2675,7 +2675,7 @@ static int spi_nor_setup(struct spi_nor *nor,
 /**
  * spi_nor_manufacturer_init_params() - Initialize the flash's parameters and
  * settings based on MFR register and ->default_init() hook.
- * @nor:	pointer to a 'struct spi-nor'.
+ * @nor:	pointer to a 'struct spi_nor'.
  */
 static void spi_nor_manufacturer_init_params(struct spi_nor *nor)
 {
@@ -2690,7 +2690,7 @@ static void spi_nor_manufacturer_init_params(struct spi_nor *nor)
 /**
  * spi_nor_sfdp_init_params() - Initialize the flash's parameters and settings
  * based on JESD216 SFDP standard.
- * @nor:	pointer to a 'struct spi-nor'.
+ * @nor:	pointer to a 'struct spi_nor'.
  *
  * The method has a roll-back mechanism: in case the SFDP parsing fails, the
  * legacy flash parameters and settings will be restored.
@@ -2712,7 +2712,7 @@ static void spi_nor_sfdp_init_params(struct spi_nor *nor)
 /**
  * spi_nor_info_init_params() - Initialize the flash's parameters and settings
  * based on nor->info data.
- * @nor:	pointer to a 'struct spi-nor'.
+ * @nor:	pointer to a 'struct spi_nor'.
  */
 static void spi_nor_info_init_params(struct spi_nor *nor)
 {
@@ -2841,7 +2841,7 @@ static void spi_nor_late_init_params(struct spi_nor *nor)
 
 /**
  * spi_nor_init_params() - Initialize the flash's parameters and settings.
- * @nor:	pointer to a 'struct spi-nor'.
+ * @nor:	pointer to a 'struct spi_nor'.
  *
  * The flash parameters and settings are initialized based on a sequence of
  * calls that are ordered by priority:
@@ -3126,7 +3126,7 @@ int spi_nor_scan(struct spi_nor *nor, const char *name,
 	/*
 	 * Make sure the XSR_RDY flag is set before calling
 	 * spi_nor_wait_till_ready(). Xilinx S3AN share MFR
-	 * with Atmel spi-nor
+	 * with Atmel SPI NOR.
 	 */
 	if (info->flags & SPI_NOR_XSR_RDY)
 		nor->flags |=  SNOR_F_READY_XSR_RDY;
diff --git a/drivers/mtd/spi-nor/macronix.c b/drivers/mtd/spi-nor/macronix.c
index ab0f963d630c..96735d83c77c 100644
--- a/drivers/mtd/spi-nor/macronix.c
+++ b/drivers/mtd/spi-nor/macronix.c
@@ -63,10 +63,16 @@ static const struct flash_info macronix_parts[] = {
 		.fixups = &mx25l25635_fixups },
 	{ "mx25u25635f", INFO(0xc22539, 0, 64 * 1024, 512,
 			      SECT_4K | SPI_NOR_4B_OPCODES) },
+	{ "mx25u51245g", INFO(0xc2253a, 0, 64 * 1024, 1024,
+			      SECT_4K | SPI_NOR_DUAL_READ |
+			      SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) },
 	{ "mx25v8035f",  INFO(0xc22314, 0, 64 * 1024,  16,
 			      SECT_4K | SPI_NOR_DUAL_READ |
 			      SPI_NOR_QUAD_READ) },
 	{ "mx25l25655e", INFO(0xc22619, 0, 64 * 1024, 512, 0) },
+	{ "mx25l51245g", INFO(0xc2201a, 0, 64 * 1024, 1024,
+			      SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
+			      SPI_NOR_4B_OPCODES) },
 	{ "mx66l51235l", INFO(0xc2201a, 0, 64 * 1024, 1024,
 			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
 			      SPI_NOR_4B_OPCODES) },
diff --git a/drivers/mtd/spi-nor/micron-st.c b/drivers/mtd/spi-nor/micron-st.c
index 6c034b9718e2..3dca5b9af3b6 100644
--- a/drivers/mtd/spi-nor/micron-st.c
+++ b/drivers/mtd/spi-nor/micron-st.c
@@ -29,7 +29,9 @@ static const struct flash_info st_parts[] = {
 	{ "n25q064a",    INFO(0x20bb17, 0, 64 * 1024,  128,
 			      SECT_4K | SPI_NOR_QUAD_READ) },
 	{ "n25q128a11",  INFO(0x20bb18, 0, 64 * 1024,  256,
-			      SECT_4K | USE_FSR | SPI_NOR_QUAD_READ) },
+			      SECT_4K | USE_FSR | SPI_NOR_QUAD_READ |
+			      SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB |
+			      SPI_NOR_4BIT_BP | SPI_NOR_BP3_SR_BIT6) },
 	{ "n25q128a13",  INFO(0x20ba18, 0, 64 * 1024,  256,
 			      SECT_4K | USE_FSR | SPI_NOR_QUAD_READ) },
 	{ "mt25ql256a",  INFO6(0x20ba19, 0x104400, 64 * 1024,  512,
@@ -59,6 +61,8 @@ static const struct flash_info st_parts[] = {
 			      SPI_NOR_4BIT_BP | SPI_NOR_BP3_SR_BIT6) },
 	{ "n25q00",      INFO(0x20ba21, 0, 64 * 1024, 2048,
 			      SECT_4K | USE_FSR | SPI_NOR_QUAD_READ |
+			      SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB |
+			      SPI_NOR_4BIT_BP | SPI_NOR_BP3_SR_BIT6 |
 			      NO_CHIP_ERASE) },
 	{ "n25q00a",     INFO(0x20bb21, 0, 64 * 1024, 2048,
 			      SECT_4K | USE_FSR | SPI_NOR_QUAD_READ |
diff --git a/drivers/mtd/spi-nor/sfdp.c b/drivers/mtd/spi-nor/sfdp.c
index f6038d3a3684..55c0c508464b 100644
--- a/drivers/mtd/spi-nor/sfdp.c
+++ b/drivers/mtd/spi-nor/sfdp.c
@@ -21,10 +21,6 @@
 #define SFDP_4BAIT_ID		0xff84  /* 4-byte Address Instruction Table */
 
 #define SFDP_SIGNATURE		0x50444653U
-#define SFDP_JESD216_MAJOR	1
-#define SFDP_JESD216_MINOR	0
-#define SFDP_JESD216A_MINOR	5
-#define SFDP_JESD216B_MINOR	6
 
 struct sfdp_header {
 	u32		signature; /* Ox50444653U <=> "SFDP" */
@@ -437,7 +433,7 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 	struct sfdp_bfpt bfpt;
 	size_t len;
 	int i, cmd, err;
-	u32 addr;
+	u32 addr, val;
 	u16 half;
 	u8 erase_mask;
 
@@ -460,6 +456,7 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 	/* Number of address bytes. */
 	switch (bfpt.dwords[BFPT_DWORD(1)] & BFPT_DWORD1_ADDRESS_BYTES_MASK) {
 	case BFPT_DWORD1_ADDRESS_BYTES_3_ONLY:
+	case BFPT_DWORD1_ADDRESS_BYTES_3_OR_4:
 		nor->addr_width = 3;
 		break;
 
@@ -472,21 +469,21 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 	}
 
 	/* Flash Memory Density (in bits). */
-	params->size = bfpt.dwords[BFPT_DWORD(2)];
-	if (params->size & BIT(31)) {
-		params->size &= ~BIT(31);
+	val = bfpt.dwords[BFPT_DWORD(2)];
+	if (val & BIT(31)) {
+		val &= ~BIT(31);
 
 		/*
 		 * Prevent overflows on params->size. Anyway, a NOR of 2^64
 		 * bits is unlikely to exist so this error probably means
 		 * the BFPT we are reading is corrupted/wrong.
 		 */
-		if (params->size > 63)
+		if (val > 63)
 			return -EINVAL;
 
-		params->size = 1ULL << params->size;
+		params->size = 1ULL << val;
 	} else {
-		params->size++;
+		params->size = val + 1;
 	}
 	params->size >>= 3; /* Convert to bytes. */
 
@@ -548,15 +545,15 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 				  SNOR_ERASE_TYPE_MASK;
 
 	/* Stop here if not JESD216 rev A or later. */
-	if (bfpt_header->length < BFPT_DWORD_MAX)
+	if (bfpt_header->length == BFPT_DWORD_MAX_JESD216)
 		return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt,
 						params);
 
 	/* Page size: this field specifies 'N' so the page size = 2^N bytes. */
-	params->page_size = bfpt.dwords[BFPT_DWORD(11)];
-	params->page_size &= BFPT_DWORD11_PAGE_SIZE_MASK;
-	params->page_size >>= BFPT_DWORD11_PAGE_SIZE_SHIFT;
-	params->page_size = 1U << params->page_size;
+	val = bfpt.dwords[BFPT_DWORD(11)];
+	val &= BFPT_DWORD11_PAGE_SIZE_MASK;
+	val >>= BFPT_DWORD11_PAGE_SIZE_SHIFT;
+	params->page_size = 1U << val;
 
 	/* Quad Enable Requirements. */
 	switch (bfpt.dwords[BFPT_DWORD(15)] & BFPT_DWORD15_QER_MASK) {
@@ -604,6 +601,11 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 		return -EINVAL;
 	}
 
+	/* Stop here if not JESD216 rev C or later. */
+	if (bfpt_header->length == BFPT_DWORD_MAX_JESD216B)
+		return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt,
+						params);
+
 	return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt, params);
 }
 
diff --git a/drivers/mtd/spi-nor/sfdp.h b/drivers/mtd/spi-nor/sfdp.h
index e0a8ded04890..7f9846b3a1ad 100644
--- a/drivers/mtd/spi-nor/sfdp.h
+++ b/drivers/mtd/spi-nor/sfdp.h
@@ -7,14 +7,20 @@
 #ifndef __LINUX_MTD_SFDP_H
 #define __LINUX_MTD_SFDP_H
 
+/* SFDP revisions */
+#define SFDP_JESD216_MAJOR	1
+#define SFDP_JESD216_MINOR	0
+#define SFDP_JESD216A_MINOR	5
+#define SFDP_JESD216B_MINOR	6
+
 /* Basic Flash Parameter Table */
 
 /*
- * JESD216 rev B defines a Basic Flash Parameter Table of 16 DWORDs.
+ * JESD216 rev D defines a Basic Flash Parameter Table of 20 DWORDs.
  * They are indexed from 1 but C arrays are indexed from 0.
  */
 #define BFPT_DWORD(i)		((i) - 1)
-#define BFPT_DWORD_MAX		16
+#define BFPT_DWORD_MAX		20
 
 struct sfdp_bfpt {
 	u32	dwords[BFPT_DWORD_MAX];
@@ -22,6 +28,7 @@ struct sfdp_bfpt {
 
 /* The first version of JESD216 defined only 9 DWORDs. */
 #define BFPT_DWORD_MAX_JESD216			9
+#define BFPT_DWORD_MAX_JESD216B			16
 
 /* 1st DWORD. */
 #define BFPT_DWORD1_FAST_READ_1_1_2		BIT(16)
diff --git a/drivers/mtd/spi-nor/spansion.c b/drivers/mtd/spi-nor/spansion.c
index 6756202ace4b..e550cd5c9d3a 100644
--- a/drivers/mtd/spi-nor/spansion.c
+++ b/drivers/mtd/spi-nor/spansion.c
@@ -8,6 +8,27 @@
 
 #include "core.h"
 
+static int
+s25fs_s_post_bfpt_fixups(struct spi_nor *nor,
+			 const struct sfdp_parameter_header *bfpt_header,
+			 const struct sfdp_bfpt *bfpt,
+			 struct spi_nor_flash_parameter *params)
+{
+	/*
+	 * The S25FS-S chip family reports 512-byte pages in BFPT but
+	 * in reality the write buffer still wraps at the safe default
+	 * of 256 bytes.  Overwrite the page size advertised by BFPT
+	 * to get the writes working.
+	 */
+	params->page_size = 256;
+
+	return 0;
+}
+
+static struct spi_nor_fixups s25fs_s_fixups = {
+	.post_bfpt = s25fs_s_post_bfpt_fixups,
+};
+
 static const struct flash_info spansion_parts[] = {
 	/* Spansion/Cypress -- single (large) sector size only, at least
 	 * for the chips listed here (without boot sectors).
@@ -22,16 +43,27 @@ static const struct flash_info spansion_parts[] = {
 	{ "s25fl128s1", INFO6(0x012018, 0x4d0180, 64 * 1024, 256,
 			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
 			      USE_CLSR) },
-	{ "s25fl256s0", INFO(0x010219, 0x4d00, 256 * 1024, 128, USE_CLSR) },
-	{ "s25fl256s1", INFO(0x010219, 0x4d01,  64 * 1024, 512,
-			     SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
-			     USE_CLSR) },
+	{ "s25fl256s0", INFO6(0x010219, 0x4d0080, 256 * 1024, 128,
+			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
+			      USE_CLSR) },
+	{ "s25fl256s1", INFO6(0x010219, 0x4d0180, 64 * 1024, 512,
+			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
+			      USE_CLSR) },
 	{ "s25fl512s",  INFO6(0x010220, 0x4d0080, 256 * 1024, 256,
 			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
 			      SPI_NOR_HAS_LOCK | USE_CLSR) },
-	{ "s25fs512s",  INFO6(0x010220, 0x4d0081, 256 * 1024, 256,
+	{ "s25fs128s1", INFO6(0x012018, 0x4d0181, 64 * 1024, 256,
+			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR)
+	  .fixups = &s25fs_s_fixups, },
+	{ "s25fs256s0", INFO6(0x010219, 0x4d0081, 256 * 1024, 128,
+			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
+			      USE_CLSR) },
+	{ "s25fs256s1", INFO6(0x010219, 0x4d0181, 64 * 1024, 512,
 			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
 			      USE_CLSR) },
+	{ "s25fs512s",  INFO6(0x010220, 0x4d0081, 256 * 1024, 256,
+			      SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR)
+	  .fixups = &s25fs_s_fixups, },
 	{ "s70fl01gs",  INFO(0x010221, 0x4d00, 256 * 1024, 256, 0) },
 	{ "s25sl12800", INFO(0x012018, 0x0300, 256 * 1024,  64, 0) },
 	{ "s25sl12801", INFO(0x012018, 0x0301,  64 * 1024, 256, 0) },
@@ -70,6 +102,8 @@ static const struct flash_info spansion_parts[] = {
 	{ "s25fl256l",  INFO(0x016019,      0,  64 * 1024, 512,
 			     SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
 			     SPI_NOR_4B_OPCODES) },
+	{ "cy15x104q",  INFO6(0x042cc2, 0x7f7f7f, 512 * 1024, 1,
+			      SPI_NOR_NO_ERASE) },
 };
 
 static void spansion_post_sfdp_fixups(struct spi_nor *nor)
diff --git a/drivers/mtd/spi-nor/winbond.c b/drivers/mtd/spi-nor/winbond.c
index 17deabad57e1..5062af10f138 100644
--- a/drivers/mtd/spi-nor/winbond.c
+++ b/drivers/mtd/spi-nor/winbond.c
@@ -8,6 +8,31 @@
 
 #include "core.h"
 
+static int
+w25q256_post_bfpt_fixups(struct spi_nor *nor,
+			 const struct sfdp_parameter_header *bfpt_header,
+			 const struct sfdp_bfpt *bfpt,
+			 struct spi_nor_flash_parameter *params)
+{
+	/*
+	 * W25Q256JV supports 4B opcodes but W25Q256FV does not.
+	 * Unfortunately, Winbond has re-used the same JEDEC ID for both
+	 * variants which prevents us from defining a new entry in the parts
+	 * table.
+	 * To differentiate between W25Q256JV and W25Q256FV check SFDP header
+	 * version: only JV has JESD216A compliant structure (version 5).
+	 */
+	if (bfpt_header->major == SFDP_JESD216_MAJOR &&
+	    bfpt_header->minor == SFDP_JESD216A_MINOR)
+		nor->flags |= SNOR_F_4B_OPCODES;
+
+	return 0;
+}
+
+static struct spi_nor_fixups w25q256_fixups = {
+	.post_bfpt = w25q256_post_bfpt_fixups,
+};
+
 static const struct flash_info winbond_parts[] = {
 	/* Winbond -- w25x "blocks" are 64K, "sectors" are 4KiB */
 	{ "w25x05", INFO(0xef3010, 0, 64 * 1024,  1,  SECT_4K) },
@@ -53,8 +78,8 @@ static const struct flash_info winbond_parts[] = {
 	{ "w25q80bl", INFO(0xef4014, 0, 64 * 1024,  16, SECT_4K) },
 	{ "w25q128", INFO(0xef4018, 0, 64 * 1024, 256, SECT_4K) },
 	{ "w25q256", INFO(0xef4019, 0, 64 * 1024, 512,
-			  SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ |
-			  SPI_NOR_4B_OPCODES) },
+			  SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ)
+	  .fixups = &w25q256_fixups },
 	{ "w25q256jvm", INFO(0xef7019, 0, 64 * 1024, 512,
 			     SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
 	{ "w25q256jw", INFO(0xef6019, 0, 64 * 1024, 512,
diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c
index 12c02342149c..e85b04e9716b 100644
--- a/drivers/mtd/ubi/build.c
+++ b/drivers/mtd/ubi/build.c
@@ -867,8 +867,11 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
 	 * Both UBI and UBIFS have been designed for SLC NAND and NOR flashes.
 	 * MLC NAND is different and needs special care, otherwise UBI or UBIFS
 	 * will die soon and you will lose all your data.
+	 * Relax this rule if the partition we're attaching to operates in SLC
+	 * mode.
 	 */
-	if (mtd->type == MTD_MLCNANDFLASH) {
+	if (mtd->type == MTD_MLCNANDFLASH &&
+	    !(mtd->flags & MTD_SLC_ON_MLC_EMULATION)) {
 		pr_err("ubi: refuse attaching mtd%d - MLC NAND is not supported\n",
 			mtd->index);
 		return -EINVAL;
diff --git a/drivers/mtd/ubi/fastmap-wl.c b/drivers/mtd/ubi/fastmap-wl.c
index b486250923c5..83afc00e365a 100644
--- a/drivers/mtd/ubi/fastmap-wl.c
+++ b/drivers/mtd/ubi/fastmap-wl.c
@@ -116,6 +116,21 @@ void ubi_refill_pools(struct ubi_device *ubi)
 	wl_pool->size = 0;
 	pool->size = 0;
 
+	if (ubi->fm_anchor) {
+		wl_tree_add(ubi->fm_anchor, &ubi->free);
+		ubi->free_count++;
+	}
+	if (ubi->fm_next_anchor) {
+		wl_tree_add(ubi->fm_next_anchor, &ubi->free);
+		ubi->free_count++;
+	}
+
+	/* All available PEBs are in ubi->free, now is the time to get
+	 * the best anchor PEBs.
+	 */
+	ubi->fm_anchor = ubi_wl_get_fm_peb(ubi, 1);
+	ubi->fm_next_anchor = ubi_wl_get_fm_peb(ubi, 1);
+
 	for (;;) {
 		enough = 0;
 		if (pool->size < pool->max_size) {
@@ -271,26 +286,20 @@ static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi)
 int ubi_ensure_anchor_pebs(struct ubi_device *ubi)
 {
 	struct ubi_work *wrk;
-	struct ubi_wl_entry *anchor;
 
 	spin_lock(&ubi->wl_lock);
 
-	/* Do we already have an anchor? */
-	if (ubi->fm_anchor) {
-		spin_unlock(&ubi->wl_lock);
-		return 0;
-	}
-
-	/* See if we can find an anchor PEB on the list of free PEBs */
-	anchor = ubi_wl_get_fm_peb(ubi, 1);
-	if (anchor) {
-		ubi->fm_anchor = anchor;
-		spin_unlock(&ubi->wl_lock);
-		return 0;
+	/* Do we have a next anchor? */
+	if (!ubi->fm_next_anchor) {
+		ubi->fm_next_anchor = ubi_wl_get_fm_peb(ubi, 1);
+		if (!ubi->fm_next_anchor)
+			/* Tell wear leveling to produce a new anchor PEB */
+			ubi->fm_do_produce_anchor = 1;
 	}
 
-	/* No luck, trigger wear leveling to produce a new anchor PEB */
-	ubi->fm_do_produce_anchor = 1;
+	/* Do wear leveling to get a new anchor PEB or check the
+	 * existing next anchor candidate.
+	 */
 	if (ubi->wl_scheduled) {
 		spin_unlock(&ubi->wl_lock);
 		return 0;
diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c
index 53f448e7433a..022af59906aa 100644
--- a/drivers/mtd/ubi/fastmap.c
+++ b/drivers/mtd/ubi/fastmap.c
@@ -1220,6 +1220,17 @@ static int ubi_write_fastmap(struct ubi_device *ubi,
 		fm_pos += sizeof(*fec);
 		ubi_assert(fm_pos <= ubi->fm_size);
 	}
+	if (ubi->fm_next_anchor) {
+		fec = (struct ubi_fm_ec *)(fm_raw + fm_pos);
+
+		fec->pnum = cpu_to_be32(ubi->fm_next_anchor->pnum);
+		set_seen(ubi, ubi->fm_next_anchor->pnum, seen_pebs);
+		fec->ec = cpu_to_be32(ubi->fm_next_anchor->ec);
+
+		free_peb_count++;
+		fm_pos += sizeof(*fec);
+		ubi_assert(fm_pos <= ubi->fm_size);
+	}
 	fmh->free_peb_count = cpu_to_be32(free_peb_count);
 
 	ubi_for_each_used_peb(ubi, wl_e, tmp_rb) {
diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h
index 73c67e5c08f8..c2da77163f94 100644
--- a/drivers/mtd/ubi/ubi.h
+++ b/drivers/mtd/ubi/ubi.h
@@ -26,7 +26,7 @@
 #include <linux/notifier.h>
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/ubi.h>
-#include <asm/pgtable.h>
+#include <linux/pgtable.h>
 
 #include "ubi-media.h"
 
@@ -491,7 +491,8 @@ struct ubi_debug_info {
  * @fm_work: fastmap work queue
  * @fm_work_scheduled: non-zero if fastmap work was scheduled
  * @fast_attach: non-zero if UBI was attached by fastmap
- * @fm_anchor: The next anchor PEB to use for fastmap
+ * @fm_anchor: The new anchor PEB used during fastmap update
+ * @fm_next_anchor: An anchor PEB candidate for the next time fastmap is updated
  * @fm_do_produce_anchor: If true produce an anchor PEB in wl
  *
  * @used: RB-tree of used physical eraseblocks
@@ -602,6 +603,7 @@ struct ubi_device {
 	int fm_work_scheduled;
 	int fast_attach;
 	struct ubi_wl_entry *fm_anchor;
+	struct ubi_wl_entry *fm_next_anchor;
 	int fm_do_produce_anchor;
 
 	/* Wear-leveling sub-system's stuff */
diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c
index 5146cce5fe32..27636063ed1b 100644
--- a/drivers/mtd/ubi/wl.c
+++ b/drivers/mtd/ubi/wl.c
@@ -687,20 +687,27 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 	}
 
 #ifdef CONFIG_MTD_UBI_FASTMAP
+	e1 = find_anchor_wl_entry(&ubi->used);
+	if (e1 && ubi->fm_next_anchor &&
+	    (ubi->fm_next_anchor->ec - e1->ec >= UBI_WL_THRESHOLD)) {
+		ubi->fm_do_produce_anchor = 1;
+		/* fm_next_anchor is no longer considered a good anchor
+		 * candidate.
+		 * NULL assignment also prevents multiple wear level checks
+		 * of this PEB.
+		 */
+		wl_tree_add(ubi->fm_next_anchor, &ubi->free);
+		ubi->fm_next_anchor = NULL;
+		ubi->free_count++;
+	}
+
 	if (ubi->fm_do_produce_anchor) {
-		e1 = find_anchor_wl_entry(&ubi->used);
 		if (!e1)
 			goto out_cancel;
 		e2 = get_peb_for_wl(ubi);
 		if (!e2)
 			goto out_cancel;
 
-		/*
-		 * Anchor move within the anchor area is useless.
-		 */
-		if (e2->pnum < UBI_FM_MAX_START)
-			goto out_cancel;
-
 		self_check_in_wl_tree(ubi, e1, &ubi->used);
 		rb_erase(&e1->u.rb, &ubi->used);
 		dbg_wl("anchor-move PEB %d to PEB %d", e1->pnum, e2->pnum);
@@ -1079,8 +1086,11 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk)
 	if (!err) {
 		spin_lock(&ubi->wl_lock);
 
-		if (!ubi->fm_anchor && e->pnum < UBI_FM_MAX_START) {
-			ubi->fm_anchor = e;
+		if (!ubi->fm_next_anchor && e->pnum < UBI_FM_MAX_START) {
+			/* Abort anchor production, if needed it will be
+			 * enabled again in the wear leveling started below.
+			 */
+			ubi->fm_next_anchor = e;
 			ubi->fm_do_produce_anchor = 0;
 		} else {
 			wl_tree_add(e, &ubi->free);
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index c7d310ef1c83..9a49c4c2316b 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -7,7 +7,7 @@ menuconfig NETDEVICES
 	default y if UML
 	depends on NET
 	bool "Network device support"
-	---help---
+	help
 	  You can say N here if you don't intend to connect your Linux box to
 	  any other computer at all.
 
@@ -32,7 +32,7 @@ config MII
 config NET_CORE
 	default y
 	bool "Network core driver support"
-	---help---
+	help
 	  You can say N here if you do not intend to use any of the
 	  networking core drivers (i.e. VLAN, bridging, bonding, etc.)
 
@@ -42,7 +42,7 @@ config BONDING
 	tristate "Bonding driver support"
 	depends on INET
 	depends on IPV6 || IPV6=n
-	---help---
+	help
 	  Say 'Y' or 'M' if you wish to be able to 'bond' multiple Ethernet
 	  Channels together. This is called 'Etherchannel' by Cisco,
 	  'Trunking' by Sun, 802.3ad by the IEEE, and 'Bonding' in Linux.
@@ -58,7 +58,7 @@ config BONDING
 
 config DUMMY
 	tristate "Dummy net driver support"
-	---help---
+	help
 	  This is essentially a bit-bucket device (i.e. traffic you send to
 	  this device is consigned into oblivion) with a configurable IP
 	  address. It is most commonly used in order to make your currently
@@ -116,7 +116,7 @@ config WIREGUARD_DEBUG
 
 config EQUALIZER
 	tristate "EQL (serial line load balancing) support"
-	---help---
+	help
 	  If you have two serial connections to some other computer (this
 	  usually requires two modems and two telephone lines) and you use
 	  SLIP (the protocol for sending Internet traffic over telephone
@@ -150,7 +150,7 @@ config IFB
 	tristate "Intermediate Functional Block support"
 	depends on NET_CLS_ACT
 	select NET_REDIRECT
-	---help---
+	help
 	  This is an intermediate driver that allows sharing of
 	  resources.
 	  To compile this driver as a module, choose M here: the module
@@ -164,7 +164,7 @@ source "drivers/net/team/Kconfig"
 
 config MACVLAN
 	tristate "MAC-VLAN support"
-	---help---
+	help
 	  This allows one to create virtual interfaces that map packets to
 	  or from specific MAC addresses to a particular interface.
 
@@ -200,7 +200,7 @@ config IPVLAN
 	tristate "IP-VLAN support"
 	depends on INET
 	depends on IPV6 || !IPV6
-	---help---
+	help
 	  This allows one to create virtual devices off of a main interface
 	  and packets will be delivered based on the dest L3 (IPv6/IPv4 addr)
 	  on packets. All interfaces (including the main interface) share L2
@@ -219,7 +219,7 @@ config IPVTAP
 	depends on IPVLAN
 	depends on INET
 	select TAP
-	---help---
+	help
 	  This adds a specialized tap character device driver that is based
 	  on the IP-VLAN network interface, called ipvtap. An ipvtap device
 	  can be added in the same way as a ipvlan device, using 'type
@@ -233,7 +233,7 @@ config VXLAN
 	depends on INET
 	select NET_UDP_TUNNEL
 	select GRO_CELLS
-	---help---
+	help
 	  This allows one to create vxlan virtual interfaces that provide
 	  Layer 2 Networks over Layer 3 Networks. VXLAN is often used
 	  to tunnel virtual network infrastructure in virtualized environments.
@@ -249,7 +249,7 @@ config GENEVE
 	depends on IPV6 || !IPV6
 	select NET_UDP_TUNNEL
 	select GRO_CELLS
-	---help---
+	help
 	  This allows one to create geneve virtual interfaces that provide
 	  Layer 2 Networks over Layer 3 Networks. GENEVE is often used
 	  to tunnel virtual network infrastructure in virtualized environments.
@@ -276,7 +276,7 @@ config GTP
 	tristate "GPRS Tunneling Protocol datapath (GTP-U)"
 	depends on INET
 	select NET_UDP_TUNNEL
-	---help---
+	help
 	  This allows one to create gtp virtual interfaces that provide
 	  the GPRS Tunneling Protocol datapath (GTP-U). This tunneling protocol
 	  is used to prevent subscribers from accessing mobile carrier core
@@ -295,12 +295,12 @@ config MACSEC
 	select CRYPTO_AES
 	select CRYPTO_GCM
 	select GRO_CELLS
-	---help---
+	help
 	   MACsec is an encryption standard for Ethernet.
 
 config NETCONSOLE
 	tristate "Network console logging support"
-	---help---
+	help
 	  If you want to log kernel messages over the network, enable this.
 	  See <file:Documentation/networking/netconsole.rst> for details.
 
@@ -343,7 +343,7 @@ config TUN
 	tristate "Universal TUN/TAP device driver support"
 	depends on INET
 	select CRC32
-	---help---
+	help
 	  TUN/TAP provides packet reception and transmission for user space
 	  programs.  It can be viewed as a simple Point-to-Point or Ethernet
 	  device, which instead of receiving packets from a physical media,
@@ -365,14 +365,14 @@ config TUN
 
 config TAP
 	tristate
-	---help---
+	help
 	  This option is selected by any driver implementing tap user space
 	  interface for a virtual interface to re-use core tap functionality.
 
 config TUN_VNET_CROSS_LE
 	bool "Support for cross-endian vnet headers on little-endian kernels"
 	default n
-	---help---
+	help
 	  This option allows TUN/TAP and MACVTAP device drivers in a
 	  little-endian kernel to parse vnet headers that come from a
 	  big-endian legacy virtio device.
@@ -385,7 +385,7 @@ config TUN_VNET_CROSS_LE
 
 config VETH
 	tristate "Virtual ethernet pair device"
-	---help---
+	help
 	  This device is a local ethernet tunnel. Devices are created in pairs.
 	  When one end receives the packet it appears on its pair and vice
 	  versa.
@@ -394,13 +394,13 @@ config VIRTIO_NET
 	tristate "Virtio network driver"
 	depends on VIRTIO
 	select NET_FAILOVER
-	---help---
+	help
 	  This is the virtual network driver for virtio.  It can be used with
 	  QEMU based VMMs (like KVM or Xen).  Say Y or M.
 
 config NLMON
 	tristate "Virtual netlink monitoring device"
-	---help---
+	help
 	  This option enables a monitoring net device for netlink skbs. The
 	  purpose of this is to analyze netlink messages with packet sockets.
 	  Thus applications like tcpdump will be able to see local netlink
@@ -414,14 +414,14 @@ config NET_VRF
 	depends on NET_L3_MASTER_DEV
 	depends on IPV6 || IPV6=n
 	depends on IPV6_MULTIPLE_TABLES || IPV6=n
-	---help---
+	help
 	  This option enables the support for mapping interfaces into VRF's. The
 	  support enables VRF devices.
 
 config VSOCKMON
 	tristate "Virtual vsock monitoring device"
 	depends on VHOST_VSOCK
-	---help---
+	help
 	  This option enables a monitoring net device for vsock sockets. It is
 	  mostly intended for developers or support to debug vsock issues. If
 	  unsure, say N.
@@ -450,7 +450,7 @@ source "drivers/net/ipa/Kconfig"
 config NET_SB1000
 	tristate "General Instruments Surfboard 1000"
 	depends on PNP
-	---help---
+	help
 	  This is a driver for the General Instrument (also known as
 	  NextLevel) SURFboard 1000 internal
 	  cable modem. This is an ISA card which is used by a number of cable
diff --git a/drivers/net/appletalk/Kconfig b/drivers/net/appletalk/Kconfig
index 10589a82263b..09f94d4b14e2 100644
--- a/drivers/net/appletalk/Kconfig
+++ b/drivers/net/appletalk/Kconfig
@@ -5,7 +5,7 @@
 config ATALK
 	tristate "Appletalk protocol support"
 	select LLC
-	---help---
+	help
 	  AppleTalk is the protocol that Apple computers can use to communicate
 	  on a network.  If your Linux box is connected to such a network and you
 	  wish to connect to it, say Y.  You will need to use the netatalk package
@@ -79,7 +79,7 @@ config COPS_TANGENT
 config IPDDP
 	tristate "Appletalk-IP driver support"
 	depends on DEV_APPLETALK && ATALK
-	---help---
+	help
 	  This allows IP networking for users who only have AppleTalk
 	  networking available. This feature is experimental. With this
 	  driver, you can encapsulate IP inside AppleTalk (e.g. if your Linux
diff --git a/drivers/net/arcnet/Kconfig b/drivers/net/arcnet/Kconfig
index 43eef60653b2..a51b9dab6d3a 100644
--- a/drivers/net/arcnet/Kconfig
+++ b/drivers/net/arcnet/Kconfig
@@ -6,7 +6,7 @@
 menuconfig ARCNET
 	depends on NETDEVICES && (ISA || PCI || PCMCIA)
 	tristate "ARCnet support"
-	---help---
+	help
 	  If you have a network card of this type, say Y and check out the
 	  (arguably) beautiful poetry in
 	  <file:Documentation/networking/arcnet.rst>.
@@ -33,7 +33,7 @@ config ARCNET_1201
 
 config ARCNET_1051
 	tristate "Enable old ARCNet packet format (RFC 1051)"
-	---help---
+	help
 	  This allows you to use RFC1051 with your ARCnet card via the virtual
 	  arc0s device. You only need arc0s if you want to talk to ARCnet
 	  software complying with the "old" standard, specifically, the DOS
@@ -82,7 +82,7 @@ config ARCNET_COM90xx
 
 config ARCNET_COM90xxIO
 	tristate "ARCnet COM90xx (IO mapped) chipset driver"
-	---help---
+	help
 	  This is the chipset driver for the COM90xx cards, using them in
 	  IO-mapped mode instead of memory-mapped mode. This is slower than
 	  the normal driver. Only use it if your card doesn't support shared
@@ -93,7 +93,7 @@ config ARCNET_COM90xxIO
 
 config ARCNET_RIM_I
 	tristate "ARCnet COM90xx (RIM I) chipset driver"
-	---help---
+	help
 	  This is yet another chipset driver for the COM90xx cards, but this
 	  time only using memory-mapped mode, and no IO ports at all. This
 	  driver is completely untested, so if you have one of these cards,
diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c
index efd1a1d1f35e..5d3c691a1c66 100644
--- a/drivers/net/bareudp.c
+++ b/drivers/net/bareudp.c
@@ -552,6 +552,8 @@ static int bareudp_validate(struct nlattr *tb[], struct nlattr *data[],
 static int bareudp2info(struct nlattr *data[], struct bareudp_conf *conf,
 			struct netlink_ext_ack *extack)
 {
+	memset(conf, 0, sizeof(*conf));
+
 	if (!data[IFLA_BAREUDP_PORT]) {
 		NL_SET_ERR_MSG(extack, "port not specified");
 		return -EINVAL;
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index a25c65d4af71..004919aea5fb 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -3687,8 +3687,6 @@ static int bond_do_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cmd
 	case BOND_RELEASE_OLD:
 	case SIOCBONDRELEASE:
 		res = bond_release(bond_dev, slave_dev);
-		if (!res)
-			netdev_update_lockdep_key(slave_dev);
 		break;
 	case BOND_SETHWADDR_OLD:
 	case SIOCBONDSETHWADDR:
diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c
index 215c10923289..ddb3916d3506 100644
--- a/drivers/net/bonding/bond_options.c
+++ b/drivers/net/bonding/bond_options.c
@@ -1398,8 +1398,6 @@ static int bond_option_slaves_set(struct bonding *bond,
 	case '-':
 		slave_dbg(bond->dev, dev, "Releasing interface\n");
 		ret = bond_release(bond->dev, dev);
-		if (!ret)
-			netdev_update_lockdep_key(dev);
 		break;
 
 	default:
diff --git a/drivers/net/caif/Kconfig b/drivers/net/caif/Kconfig
index 1538ad194cf4..c245edddb48b 100644
--- a/drivers/net/caif/Kconfig
+++ b/drivers/net/caif/Kconfig
@@ -15,7 +15,7 @@ config CAIF_TTY
 	tristate "CAIF TTY transport driver"
 	depends on CAIF && TTY
 	default n
-	---help---
+	help
 	  The CAIF TTY transport driver is a Line Discipline (ldisc)
 	  identified as N_CAIF. When this ldisc is opened from user space
 	  it will redirect the TTY's traffic into the CAIF stack.
@@ -24,7 +24,7 @@ config CAIF_SPI_SLAVE
 	tristate "CAIF SPI transport driver for slave interface"
 	depends on CAIF && HAS_DMA
 	default n
-	---help---
+	help
 	  The CAIF Link layer SPI Protocol driver for Slave SPI interface.
 	  This driver implements a platform driver to accommodate for a
 	  platform specific SPI device. A sample CAIF SPI Platform device is
@@ -34,7 +34,7 @@ config CAIF_SPI_SYNC
 	bool "Next command and length in start of frame"
 	depends on CAIF_SPI_SLAVE
 	default n
-	---help---
+	help
 	  Putting the next command and length in the start of the frame can
 	  help to synchronize to the next transfer in case of over or under-runs.
 	  This option also needs to be enabled on the modem.
@@ -43,19 +43,19 @@ config CAIF_HSI
 	tristate "CAIF HSI transport driver"
 	depends on CAIF
 	default n
-	---help---
+	help
 	  The CAIF low level driver for CAIF over HSI.
 	  Be aware that if you enable this then you also need to
 	  enable a low-level HSI driver.
 
 config CAIF_VIRTIO
 	tristate "CAIF virtio transport driver"
-	depends on CAIF && HAS_DMA && VHOST_DPN
+	depends on CAIF && HAS_DMA
 	select VHOST_RING
 	select VIRTIO
 	select GENERIC_ALLOCATOR
 	default n
-	---help---
+	help
 	  The CAIF driver for CAIF over Virtio.
 
 endif # CAIF_DRIVERS
diff --git a/drivers/net/can/Kconfig b/drivers/net/can/Kconfig
index 17c166cc8482..f07012a76c0c 100644
--- a/drivers/net/can/Kconfig
+++ b/drivers/net/can/Kconfig
@@ -3,7 +3,7 @@ menu "CAN Device Drivers"
 
 config CAN_VCAN
 	tristate "Virtual Local CAN Interface (vcan)"
-	---help---
+	help
 	  Similar to the network loopback devices, vcan offers a
 	  virtual local CAN interface.
 
@@ -12,7 +12,7 @@ config CAN_VCAN
 
 config CAN_VXCAN
 	tristate "Virtual CAN Tunnel (vxcan)"
-	---help---
+	help
 	  Similar to the virtual ethernet driver veth, vxcan implements a
 	  local CAN traffic tunnel between two virtual CAN network devices.
 	  When creating a vxcan, two vxcan devices are created as pair.
@@ -31,7 +31,7 @@ config CAN_VXCAN
 config CAN_SLCAN
 	tristate "Serial / USB serial CAN Adaptors (slcan)"
 	depends on TTY
-	---help---
+	help
 	  CAN driver for several 'low cost' CAN interfaces that are attached
 	  via serial lines or via USB-to-serial adapters using the LAWICEL
 	  ASCII protocol. The driver implements the tty linediscipline N_SLCAN.
@@ -51,7 +51,7 @@ config CAN_SLCAN
 config CAN_DEV
 	tristate "Platform CAN drivers with Netlink support"
 	default y
-	---help---
+	help
 	  Enables the common framework for platform CAN drivers with Netlink
 	  support. This is the standard library for CAN drivers.
 	  If unsure, say Y.
@@ -61,7 +61,7 @@ if CAN_DEV
 config CAN_CALC_BITTIMING
 	bool "CAN bit-timing calculation"
 	default y
-	---help---
+	help
 	  If enabled, CAN bit-timing parameters will be calculated for the
 	  bit-rate specified via Netlink argument "bitrate" when the device
 	  get started. This works fine for the most common CAN controllers
@@ -81,7 +81,7 @@ config CAN_LEDS
 	# fulfills your needs instead of fixing this driver.
 	depends on BROKEN
 	select LEDS_TRIGGERS
-	---help---
+	help
 	  This option adds two LED triggers for packet receive and transmit
 	  events on each supported CAN device.
 
@@ -91,20 +91,20 @@ config CAN_LEDS
 config CAN_AT91
 	tristate "Atmel AT91 onchip CAN controller"
 	depends on (ARCH_AT91 || COMPILE_TEST) && HAS_IOMEM
-	---help---
+	help
 	  This is a driver for the SoC CAN controller in Atmel's AT91SAM9263
 	  and AT91SAM9X5 processors.
 
 config CAN_FLEXCAN
 	tristate "Support for Freescale FLEXCAN based chips"
 	depends on OF && HAS_IOMEM
-	---help---
+	help
 	  Say Y here if you want to support for Freescale FlexCAN.
 
 config CAN_GRCAN
 	tristate "Aeroflex Gaisler GRCAN and GRHCAN CAN devices"
 	depends on OF && HAS_DMA
-	---help---
+	help
 	  Say Y here if you want to use Aeroflex Gaisler GRCAN or GRHCAN.
 	  Note that the driver supports little endian, even though little
 	  endian syntheses of the cores would need some modifications on
@@ -113,7 +113,7 @@ config CAN_GRCAN
 config CAN_JANZ_ICAN3
 	tristate "Janz VMOD-ICAN3 Intelligent CAN controller"
 	depends on MFD_JANZ_CMODIO
-	---help---
+	help
 	  Driver for Janz VMOD-ICAN3 Intelligent CAN controller module, which
 	  connects to a MODULbus carrier board.
 
@@ -136,7 +136,7 @@ config CAN_KVASER_PCIEFD
 config CAN_SUN4I
 	tristate "Allwinner A10 CAN controller"
 	depends on MACH_SUN4I || MACH_SUN7I || COMPILE_TEST
-	---help---
+	help
 	  Say Y here if you want to use CAN controller found on Allwinner
 	  A10/A20 SoCs.
 
@@ -146,7 +146,7 @@ config CAN_SUN4I
 config CAN_TI_HECC
 	depends on ARM
 	tristate "TI High End CAN Controller"
-	---help---
+	help
 	  Driver for TI HECC (High End CAN Controller) module found on many
 	  TI devices. The device specifications are available from www.ti.com
 
@@ -154,14 +154,14 @@ config CAN_XILINXCAN
 	tristate "Xilinx CAN"
 	depends on ARCH_ZYNQ || ARM64 || MICROBLAZE || COMPILE_TEST
 	depends on COMMON_CLK && HAS_IOMEM
-	---help---
+	help
 	  Xilinx CAN driver. This driver supports both soft AXI CAN IP and
 	  Zynq CANPS IP.
 
 config PCH_CAN
 	tristate "Intel EG20T PCH CAN controller"
 	depends on PCI && (X86_32 || COMPILE_TEST)
-	---help---
+	help
 	  This driver is for PCH CAN of Topcliff (Intel EG20T PCH) which
 	  is an IOH for x86 embedded processor (Intel Atom E6xx series).
 	  This driver can access CAN bus.
@@ -182,7 +182,7 @@ endif
 
 config CAN_DEBUG_DEVICES
 	bool "CAN devices debugging messages"
-	---help---
+	help
 	  Say Y here if you want the CAN device drivers to produce a bunch of
 	  debug messages to the system log.  Select this if you are having
 	  a problem with CAN support and want to see more of what is going
diff --git a/drivers/net/can/c_can/Kconfig b/drivers/net/can/c_can/Kconfig
index b0f206d36f55..962725788b0a 100644
--- a/drivers/net/can/c_can/Kconfig
+++ b/drivers/net/can/c_can/Kconfig
@@ -7,7 +7,7 @@ if CAN_C_CAN
 
 config CAN_C_CAN_PLATFORM
 	tristate "Generic Platform Bus based C_CAN/D_CAN driver"
-	---help---
+	help
 	  This driver adds support for the C_CAN/D_CAN chips connected
 	  to the "platform bus" (Linux abstraction for directly to the
 	  processor attached devices) which can be found on various
@@ -18,7 +18,7 @@ config CAN_C_CAN_PLATFORM
 config CAN_C_CAN_PCI
 	tristate "Generic PCI Bus based C_CAN/D_CAN driver"
 	depends on PCI
-	---help---
+	help
 	  This driver adds support for the C_CAN/D_CAN chips connected
 	  to the PCI bus.
 endif
diff --git a/drivers/net/can/cc770/Kconfig b/drivers/net/can/cc770/Kconfig
index 13a4593a52df..9ef1359319f0 100644
--- a/drivers/net/can/cc770/Kconfig
+++ b/drivers/net/can/cc770/Kconfig
@@ -7,14 +7,14 @@ if CAN_CC770
 
 config CAN_CC770_ISA
 	tristate "ISA Bus based legacy CC770 driver"
-	---help---
+	help
 	  This driver adds legacy support for CC770 and AN82527 chips
 	  connected to the ISA bus using I/O port, memory mapped or
 	  indirect access.
 
 config CAN_CC770_PLATFORM
 	tristate "Generic Platform Bus based CC770 driver"
-	---help---
+	help
 	  This driver adds support for the CC770 and AN82527 chips
 	  connected to the "platform bus" (Linux abstraction for directly
 	  to the processor attached devices).
diff --git a/drivers/net/can/ifi_canfd/Kconfig b/drivers/net/can/ifi_canfd/Kconfig
index ce0197641a59..b5dd9c13d529 100644
--- a/drivers/net/can/ifi_canfd/Kconfig
+++ b/drivers/net/can/ifi_canfd/Kconfig
@@ -2,7 +2,7 @@
 config CAN_IFI_CANFD
 	depends on HAS_IOMEM
 	tristate "IFI CAN_FD IP"
-	---help---
+	help
 	  This driver adds support for the I/F/I CAN_FD soft IP block
 	  connected to the "platform bus" (Linux abstraction for directly
 	  to the processor attached devices). The CAN_FD is most often
diff --git a/drivers/net/can/m_can/Kconfig b/drivers/net/can/m_can/Kconfig
index 1ff0b7fe81d6..d9216147ca93 100644
--- a/drivers/net/can/m_can/Kconfig
+++ b/drivers/net/can/m_can/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config CAN_M_CAN
 	tristate "Bosch M_CAN support"
-	---help---
+	help
 	  Say Y here if you want support for Bosch M_CAN controller framework.
 	  This is common support for devices that embed the Bosch M_CAN IP.
 
@@ -9,7 +9,7 @@ config CAN_M_CAN_PLATFORM
 	tristate "Bosch M_CAN support for io-mapped devices"
 	depends on HAS_IOMEM
 	depends on CAN_M_CAN
-	---help---
+	help
 	  Say Y here if you want support for IO Mapped Bosch M_CAN controller.
 	  This support is for devices that have the Bosch M_CAN controller
 	  IP embedded into the device and the IP is IO Mapped to the processor.
@@ -18,7 +18,7 @@ config CAN_M_CAN_TCAN4X5X
 	depends on CAN_M_CAN
 	depends on REGMAP_SPI
 	tristate "TCAN4X5X M_CAN device"
-	---help---
+	help
 	  Say Y here if you want support for Texas Instruments TCAN4x5x
 	  M_CAN controller.  This device is a peripherial device that uses the
 	  SPI bus for communication.
diff --git a/drivers/net/can/mscan/Kconfig b/drivers/net/can/mscan/Kconfig
index 3a57a51be22e..dfe6bd9947bb 100644
--- a/drivers/net/can/mscan/Kconfig
+++ b/drivers/net/can/mscan/Kconfig
@@ -2,7 +2,7 @@
 config CAN_MSCAN
 	depends on PPC
 	tristate "Support for Freescale MSCAN based chips"
-	---help---
+	help
 	  The Motorola Scalable Controller Area Network (MSCAN) definition
 	  is based on the MSCAN12 definition which is the specific
 	  implementation of the Motorola Scalable CAN concept targeted for
@@ -13,7 +13,7 @@ if CAN_MSCAN
 config CAN_MPC5XXX
 	tristate "Freescale MPC5xxx onboard CAN controller"
 	depends on (PPC_MPC52xx || PPC_MPC512x)
-	---help---
+	help
 	  If you say yes here you get support for Freescale's MPC5xxx
 	  onboard CAN controller. Currently, the MPC5200, MPC5200B and
 	  MPC5121 (Rev. 2 and later) are supported.
diff --git a/drivers/net/can/peak_canfd/Kconfig b/drivers/net/can/peak_canfd/Kconfig
index c29ab2150794..f7e412766dbd 100644
--- a/drivers/net/can/peak_canfd/Kconfig
+++ b/drivers/net/can/peak_canfd/Kconfig
@@ -2,7 +2,7 @@
 config CAN_PEAK_PCIEFD
 	depends on PCI
 	tristate "PEAK-System PCAN-PCIe FD cards"
-	---help---
+	help
 	  This driver adds support for the PEAK-System PCI Express FD
 	  CAN-FD cards family.
 	  These 1x or 2x CAN-FD channels cards offer CAN 2.0 a/b as well as
diff --git a/drivers/net/can/peak_canfd/peak_pciefd_main.c b/drivers/net/can/peak_canfd/peak_pciefd_main.c
index d08a3d559114..6ad83a881039 100644
--- a/drivers/net/can/peak_canfd/peak_pciefd_main.c
+++ b/drivers/net/can/peak_canfd/peak_pciefd_main.c
@@ -146,7 +146,7 @@ struct pciefd_rx_dma {
 	__le32 irq_status;
 	__le32 sys_time_low;
 	__le32 sys_time_high;
-	struct pucan_rx_msg msg[0];
+	struct pucan_rx_msg msg[];
 } __packed __aligned(4);
 
 /* Tx Link record */
@@ -194,7 +194,7 @@ struct pciefd_board {
 	struct pci_dev *pci_dev;
 	int can_count;
 	spinlock_t cmd_lock;		/* 64-bits cmds must be atomic */
-	struct pciefd_can *can[0];	/* array of network devices */
+	struct pciefd_can *can[];	/* array of network devices */
 };
 
 /* supported device ids. */
diff --git a/drivers/net/can/rcar/Kconfig b/drivers/net/can/rcar/Kconfig
index bd5a8fcd83e1..8d36101b78e3 100644
--- a/drivers/net/can/rcar/Kconfig
+++ b/drivers/net/can/rcar/Kconfig
@@ -2,7 +2,7 @@
 config CAN_RCAR
 	tristate "Renesas R-Car CAN controller"
 	depends on ARCH_RENESAS || ARM
-	---help---
+	help
 	  Say Y here if you want to use CAN controller found on Renesas R-Car
 	  SoCs.
 
@@ -12,7 +12,7 @@ config CAN_RCAR
 config CAN_RCAR_CANFD
 	tristate "Renesas R-Car CAN FD controller"
 	depends on ARCH_RENESAS || ARM
-	---help---
+	help
 	  Say Y here if you want to use CAN FD controller found on
 	  Renesas R-Car SoCs. The driver puts the controller in CAN FD only
 	  mode, which can interoperate with CAN2.0 nodes but does not support
diff --git a/drivers/net/can/sja1000/Kconfig b/drivers/net/can/sja1000/Kconfig
index 32d242dc0d9f..110071b26921 100644
--- a/drivers/net/can/sja1000/Kconfig
+++ b/drivers/net/can/sja1000/Kconfig
@@ -9,7 +9,7 @@ if CAN_SJA1000
 config CAN_EMS_PCI
 	tristate "EMS CPC-PCI, CPC-PCIe and CPC-104P Card"
 	depends on PCI
-	---help---
+	help
 	  This driver is for the one, two or four channel CPC-PCI,
 	  CPC-PCIe and CPC-104P cards from EMS Dr. Thomas Wuensche
 	  (http://www.ems-wuensche.de).
@@ -17,7 +17,7 @@ config CAN_EMS_PCI
 config CAN_EMS_PCMCIA
 	tristate "EMS CPC-CARD Card"
 	depends on PCMCIA
-	---help---
+	help
 	  This driver is for the one or two channel CPC-CARD cards from
 	  EMS Dr. Thomas Wuensche (http://www.ems-wuensche.de).
 
@@ -34,14 +34,14 @@ config CAN_F81601
 config CAN_KVASER_PCI
 	tristate "Kvaser PCIcanx and Kvaser PCIcan PCI Cards"
 	depends on PCI
-	---help---
+	help
 	  This driver is for the PCIcanx and PCIcan cards (1, 2 or
 	  4 channel) from Kvaser (http://www.kvaser.com).
 
 config CAN_PEAK_PCI
 	tristate "PEAK PCAN-PCI/PCIe/miniPCI Cards"
 	depends on PCI
-	---help---
+	help
 	  This driver is for the PCAN-PCI/PCIe/miniPCI cards
 	  (1, 2, 3 or 4 channels) from PEAK-System Technik
 	  (http://www.peak-system.com).
@@ -52,7 +52,7 @@ config CAN_PEAK_PCIEC
 	select I2C
 	select I2C_ALGOBIT
 	default y
-	---help---
+	help
 	  Say Y here if you want to use a PCAN-ExpressCard from PEAK-System
 	  Technik. This will also automatically select I2C and I2C_ALGO
 	  configuration options.
@@ -61,7 +61,7 @@ config CAN_PEAK_PCMCIA
 	tristate "PEAK PCAN-PC Card"
 	depends on PCMCIA
 	depends on HAS_IOPORT_MAP
-	---help---
+	help
 	  This driver is for the PCAN-PC Card PCMCIA adapter (1 or 2 channels)
 	  from PEAK-System (http://www.peak-system.com). To compile this
 	  driver as a module, choose M here: the module will be called
@@ -70,7 +70,7 @@ config CAN_PEAK_PCMCIA
 config CAN_PLX_PCI
 	tristate "PLX90xx PCI-bridge based Cards"
 	depends on PCI
-	---help---
+	help
 	  This driver is for CAN interface cards based on
 	  the PLX90xx PCI bridge.
 	  Driver supports now:
@@ -87,13 +87,13 @@ config CAN_PLX_PCI
 
 config CAN_SJA1000_ISA
 	tristate "ISA Bus based legacy SJA1000 driver"
-	---help---
+	help
 	  This driver adds legacy support for SJA1000 chips connected to
 	  the ISA bus using I/O port, memory mapped or indirect access.
 
 config CAN_SJA1000_PLATFORM
 	tristate "Generic Platform Bus based SJA1000 driver"
-	---help---
+	help
 	  This driver adds support for the SJA1000 chips connected to
 	  the "platform bus" (Linux abstraction for directly to the
 	  processor attached devices).  Which can be found on various
diff --git a/drivers/net/can/softing/Kconfig b/drivers/net/can/softing/Kconfig
index 0f1708f99308..16b9eec63490 100644
--- a/drivers/net/can/softing/Kconfig
+++ b/drivers/net/can/softing/Kconfig
@@ -2,7 +2,7 @@
 config CAN_SOFTING
 	tristate "Softing Gmbh CAN generic support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  Support for CAN cards from Softing Gmbh & some cards
 	  from Vector Gmbh.
 	  Softing Gmbh CAN cards come with 1 or 2 physical busses.
@@ -20,7 +20,7 @@ config CAN_SOFTING_CS
 	tristate "Softing Gmbh CAN pcmcia cards"
 	depends on PCMCIA
 	depends on CAN_SOFTING
-	---help---
+	help
 	  Support for PCMCIA cards from Softing Gmbh & some cards
 	  from Vector Gmbh.
 	  You need firmware for these, which you can get at
diff --git a/drivers/net/can/spi/Kconfig b/drivers/net/can/spi/Kconfig
index 1c50788055cb..f780c79aac6f 100644
--- a/drivers/net/can/spi/Kconfig
+++ b/drivers/net/can/spi/Kconfig
@@ -5,13 +5,13 @@ menu "CAN SPI interfaces"
 config CAN_HI311X
 	tristate "Holt HI311x SPI CAN controllers"
 	depends on CAN_DEV && SPI && HAS_DMA
-	---help---
+	help
 	  Driver for the Holt HI311x SPI CAN controllers.
 
 config CAN_MCP251X
 	tristate "Microchip MCP251x and MCP25625 SPI CAN controllers"
 	depends on HAS_DMA
-	---help---
+	help
 	  Driver for the Microchip MCP251x and MCP25625 SPI CAN
 	  controllers.
 
diff --git a/drivers/net/can/usb/Kconfig b/drivers/net/can/usb/Kconfig
index b412f7ba4f89..77fa830fe7dd 100644
--- a/drivers/net/can/usb/Kconfig
+++ b/drivers/net/can/usb/Kconfig
@@ -4,25 +4,25 @@ menu "CAN USB interfaces"
 
 config CAN_8DEV_USB
 	tristate "8 devices USB2CAN interface"
-	---help---
+	help
 	  This driver supports the USB2CAN interface
 	  from 8 devices (http://www.8devices.com).
 
 config CAN_EMS_USB
 	tristate "EMS CPC-USB/ARM7 CAN/USB interface"
-	---help---
+	help
 	  This driver is for the one channel CPC-USB/ARM7 CAN/USB interface
 	  from EMS Dr. Thomas Wuensche (http://www.ems-wuensche.de).
 
 config CAN_ESD_USB2
 	tristate "ESD USB/2 CAN/USB interface"
-	---help---
+	help
 	  This driver supports the CAN-USB/2 interface
 	  from esd electronic system design gmbh (http://www.esd.eu).
 
 config CAN_GS_USB
 	tristate "Geschwister Schneider UG interfaces"
-	---help---
+	help
 	  This driver supports the Geschwister Schneider and bytewerk.org
 	  candleLight USB CAN interfaces USB/CAN devices
 	  If unsure choose N,
@@ -31,7 +31,7 @@ config CAN_GS_USB
 
 config CAN_KVASER_USB
 	tristate "Kvaser CAN/USB interface"
-	---help---
+	help
 	  This driver adds support for Kvaser CAN/USB devices like Kvaser
 	  Leaf Light, Kvaser USBcan II and Kvaser Memorator Pro 5xHS.
 
@@ -82,13 +82,13 @@ config CAN_KVASER_USB
 
 config CAN_MCBA_USB
 	tristate "Microchip CAN BUS Analyzer interface"
-	---help---
+	help
 	  This driver supports the CAN BUS Analyzer interface
 	  from Microchip (http://www.microchip.com/development-tools/).
 
 config CAN_PEAK_USB
 	tristate "PEAK PCAN-USB/USB Pro interfaces for CAN 2.0b/CAN-FD"
-	---help---
+	help
 	  This driver supports the PEAK-System Technik USB adapters that enable
 	  access to the CAN bus, with repect to the CAN 2.0b and/or CAN-FD
 	  standards, that is:
@@ -104,7 +104,7 @@ config CAN_PEAK_USB
 
 config CAN_UCAN
 	tristate "Theobroma Systems UCAN interface"
-	---help---
+	help
 	  This driver supports the Theobroma Systems
 	  UCAN USB-CAN interface.
 
diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
index 2d38dbc9dd8c..d0024cb30a7b 100644
--- a/drivers/net/dsa/Kconfig
+++ b/drivers/net/dsa/Kconfig
@@ -12,7 +12,7 @@ config NET_DSA_BCM_SF2
 	select BCM7XXX_PHY
 	select MDIO_BCM_UNIMAC
 	select B53
-	---help---
+	help
 	  This enables support for the Broadcom Starfighter 2 Ethernet
 	  switch chips.
 
@@ -20,7 +20,7 @@ config NET_DSA_LOOP
 	tristate "DSA mock-up Ethernet switch chip support"
 	depends on NET_DSA
 	select FIXED_PHY
-	---help---
+	help
 	  This enables support for a fake mock-up switch chip which
 	  exercises the DSA APIs.
 
@@ -28,7 +28,7 @@ config NET_DSA_LANTIQ_GSWIP
 	tristate "Lantiq / Intel GSWIP"
 	depends on HAS_IOMEM && NET_DSA
 	select NET_DSA_TAG_GSWIP
-	---help---
+	help
 	  This enables support for the Lantiq / Intel GSWIP 2.1 found in
 	  the xrx200 / VR9 SoC.
 
@@ -36,7 +36,7 @@ config NET_DSA_MT7530
 	tristate "Mediatek MT7530 Ethernet switch support"
 	depends on NET_DSA
 	select NET_DSA_TAG_MTK
-	---help---
+	help
 	  This enables support for the Mediatek MT7530 Ethernet switch
 	  chip.
 
@@ -44,7 +44,7 @@ config NET_DSA_MV88E6060
 	tristate "Marvell 88E6060 ethernet switch chip support"
 	depends on NET_DSA
 	select NET_DSA_TAG_TRAILER
-	---help---
+	help
 	  This enables support for the Marvell 88E6060 ethernet switch
 	  chip.
 
@@ -63,7 +63,7 @@ config NET_DSA_QCA8K
 	depends on NET_DSA
 	select NET_DSA_TAG_QCA
 	select REGMAP
-	---help---
+	help
 	  This enables support for the Qualcomm Atheros QCA8K Ethernet
 	  switch chips.
 
@@ -74,7 +74,7 @@ config NET_DSA_REALTEK_SMI
 	select IRQ_DOMAIN
 	select REALTEK_PHY
 	select REGMAP
-	---help---
+	help
 	  This enables support for the Realtek SMI-based switch
 	  chips, currently only RTL8366RB.
 
@@ -82,7 +82,7 @@ config NET_DSA_SMSC_LAN9303
 	tristate
 	select NET_DSA_TAG_LAN9303
 	select REGMAP
-	---help---
+	help
 	  This enables support for the SMSC/Microchip LAN9303 3 port ethernet
 	  switch chips.
 
@@ -91,7 +91,7 @@ config NET_DSA_SMSC_LAN9303_I2C
 	depends on NET_DSA && I2C
 	select NET_DSA_SMSC_LAN9303
 	select REGMAP_I2C
-	---help---
+	help
 	  Enable access functions if the SMSC/Microchip LAN9303 is configured
 	  for I2C managed mode.
 
@@ -99,7 +99,7 @@ config NET_DSA_SMSC_LAN9303_MDIO
 	tristate "SMSC/Microchip LAN9303 3-ports 10/100 ethernet switch in MDIO managed mode"
 	depends on NET_DSA
 	select NET_DSA_SMSC_LAN9303
-	---help---
+	help
 	  Enable access functions if the SMSC/Microchip LAN9303 is configured
 	  for MDIO managed mode.
 
@@ -109,7 +109,7 @@ config NET_DSA_VITESSE_VSC73XX
 	select FIXED_PHY
 	select VITESSE_PHY
 	select GPIOLIB
-	---help---
+	help
 	  This enables support for the Vitesse VSC7385, VSC7388,
 	  VSC7395 and VSC7398 SparX integrated ethernet switches.
 
@@ -118,7 +118,7 @@ config NET_DSA_VITESSE_VSC73XX_SPI
 	depends on NET_DSA
 	depends on SPI
 	select NET_DSA_VITESSE_VSC73XX
-	---help---
+	help
 	  This enables support for the Vitesse VSC7385, VSC7388, VSC7395
 	  and VSC7398 SparX integrated ethernet switches in SPI managed mode.
 
@@ -127,7 +127,7 @@ config NET_DSA_VITESSE_VSC73XX_PLATFORM
 	depends on NET_DSA
 	depends on HAS_IOMEM
 	select NET_DSA_VITESSE_VSC73XX
-	---help---
+	help
 	  This enables support for the Vitesse VSC7385, VSC7388, VSC7395
 	  and VSC7398 SparX integrated ethernet switches, connected over
 	  a CPU-attached address bus and work in memory-mapped I/O mode.
diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c
index cf6fa8fede33..521ebc072903 100644
--- a/drivers/net/dsa/lantiq_gswip.c
+++ b/drivers/net/dsa/lantiq_gswip.c
@@ -1452,7 +1452,8 @@ static void gswip_phylink_validate(struct dsa_switch *ds, int port,
 
 unsupported:
 	bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
-	dev_err(ds->dev, "Unsupported interface: %d\n", state->interface);
+	dev_err(ds->dev, "Unsupported interface '%s' for port %d\n",
+		phy_modes(state->interface), port);
 	return;
 }
 
diff --git a/drivers/net/dsa/qca/Kconfig b/drivers/net/dsa/qca/Kconfig
index e3c8d715a18f..13b7e679b8b5 100644
--- a/drivers/net/dsa/qca/Kconfig
+++ b/drivers/net/dsa/qca/Kconfig
@@ -4,6 +4,6 @@ config NET_DSA_AR9331
 	depends on NET_DSA
 	select NET_DSA_TAG_AR9331
 	select REGMAP
-	---help---
+	help
 	  This enables support for the Qualcomm Atheros AR9331 built-in Ethernet
 	  switch.
diff --git a/drivers/net/dsa/qca8k.c b/drivers/net/dsa/qca8k.c
index 9f4205b4439b..d2b5ab403e06 100644
--- a/drivers/net/dsa/qca8k.c
+++ b/drivers/net/dsa/qca8k.c
@@ -1079,8 +1079,7 @@ qca8k_sw_probe(struct mdio_device *mdiodev)
 	if (id != QCA8K_ID_QCA8337)
 		return -ENODEV;
 
-	priv->ds = devm_kzalloc(&mdiodev->dev, sizeof(*priv->ds),
-				QCA8K_NUM_PORTS);
+	priv->ds = devm_kzalloc(&mdiodev->dev, sizeof(*priv->ds), GFP_KERNEL);
 	if (!priv->ds)
 		return -ENOMEM;
 
diff --git a/drivers/net/dsa/sja1105/sja1105_ptp.c b/drivers/net/dsa/sja1105/sja1105_ptp.c
index bc0e47c1dbb9..177134596458 100644
--- a/drivers/net/dsa/sja1105/sja1105_ptp.c
+++ b/drivers/net/dsa/sja1105/sja1105_ptp.c
@@ -891,16 +891,16 @@ void sja1105_ptp_txtstamp_skb(struct dsa_switch *ds, int port,
 
 	mutex_lock(&ptp_data->lock);
 
-	rc = sja1105_ptpclkval_read(priv, &ticks, NULL);
+	rc = sja1105_ptpegr_ts_poll(ds, port, &ts);
 	if (rc < 0) {
-		dev_err(ds->dev, "Failed to read PTP clock: %d\n", rc);
+		dev_err(ds->dev, "timed out polling for tstamp\n");
 		kfree_skb(skb);
 		goto out;
 	}
 
-	rc = sja1105_ptpegr_ts_poll(ds, port, &ts);
+	rc = sja1105_ptpclkval_read(priv, &ticks, NULL);
 	if (rc < 0) {
-		dev_err(ds->dev, "timed out polling for tstamp\n");
+		dev_err(ds->dev, "Failed to read PTP clock: %d\n", rc);
 		kfree_skb(skb);
 		goto out;
 	}
diff --git a/drivers/net/ethernet/3com/Kconfig b/drivers/net/ethernet/3com/Kconfig
index 7cc259893cb9..074b0974769c 100644
--- a/drivers/net/ethernet/3com/Kconfig
+++ b/drivers/net/ethernet/3com/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_3COM
 	bool "3Com devices"
 	default y
 	depends on ISA || EISA || PCI || PCMCIA
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_3COM
 config EL3
 	tristate "3c509/3c579 \"EtherLink III\" support"
 	depends on (ISA || EISA)
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to the 3Com
 	  EtherLinkIII series, say Y here.
 
@@ -34,7 +34,7 @@ config EL3
 config 3C515
 	tristate "3c515 ISA \"Fast EtherLink\""
 	depends on ISA && ISA_DMA_API && !PPC32
-	---help---
+	help
 	  If you have a 3Com ISA EtherLink XL "Corkscrew" 3c515 Fast Ethernet
 	  network card, say Y here.
 
@@ -44,7 +44,7 @@ config 3C515
 config PCMCIA_3C574
 	tristate "3Com 3c574 PCMCIA support"
 	depends on PCMCIA
-	---help---
+	help
 	  Say Y here if you intend to attach a 3Com 3c574 or compatible PCMCIA
 	  (PC-card) Fast Ethernet card to your computer.
 
@@ -54,7 +54,7 @@ config PCMCIA_3C574
 config PCMCIA_3C589
 	tristate "3Com 3c589 PCMCIA support"
 	depends on PCMCIA
-	---help---
+	help
 	  Say Y here if you intend to attach a 3Com 3c589 or compatible PCMCIA
 	  (PC-card) Ethernet card to your computer.
 
@@ -65,7 +65,7 @@ config VORTEX
 	tristate "3c590/3c900 series (592/595/597) \"Vortex/Boomerang\" support"
 	depends on (PCI || EISA) && HAS_IOPORT_MAP
 	select MII
-	---help---
+	help
 	  This option enables driver support for a large number of 10Mbps and
 	  10/100Mbps EISA, PCI and Cardbus 3Com network cards:
 
@@ -86,7 +86,7 @@ config TYPHOON
 	tristate "3cr990 series \"Typhoon\" support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This option enables driver support for the 3cr990 series of cards:
 
 	  3C990-TX, 3CR990-TX-95, 3CR990-TX-97, 3CR990-FX-95, 3CR990-FX-97,
diff --git a/drivers/net/ethernet/8390/Kconfig b/drivers/net/ethernet/8390/Kconfig
index a9478577b495..9f4b302fd2ce 100644
--- a/drivers/net/ethernet/8390/Kconfig
+++ b/drivers/net/ethernet/8390/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_8390
 	bool "National Semiconductor 8390 devices"
 	default y
 	depends on NET_VENDOR_NATSEMI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_8390
 config PCMCIA_AXNET
 	tristate "Asix AX88190 PCMCIA support"
 	depends on PCMCIA
-	---help---
+	help
 	  Say Y here if you intend to attach an Asix AX88190-based PCMCIA
 	  (PC-card) Fast Ethernet card to your computer.  These cards are
 	  nearly NE2000 compatible but need a separate driver due to a few
@@ -35,7 +35,7 @@ config AX88796
 	select CRC32
 	select PHYLIB
 	select MDIO_BITBANG
-	---help---
+	help
 	  AX88796 driver, using platform bus to provide
 	  chip detection and resources
 
@@ -43,7 +43,7 @@ config AX88796_93CX6
 	bool "ASIX AX88796 external 93CX6 eeprom support"
 	depends on AX88796
 	select EEPROM_93CX6
-	---help---
+	help
 	  Select this if your platform comes with an external 93CX6 eeprom.
 
 config XSURF100
@@ -63,7 +63,7 @@ config HYDRA
 	tristate "Hydra support"
 	depends on ZORRO
 	select CRC32
-	---help---
+	help
 	  If you have a Hydra Ethernet adapter, say Y. Otherwise, say N.
 
 	  To compile this driver as a module, choose M here: the module
@@ -73,7 +73,7 @@ config ARM_ETHERH
 	tristate "I-cubed EtherH/ANT EtherM support"
 	depends on ARM && ARCH_ACORN
 	select CRC32
-	---help---
+	help
 	  If you have an Acorn system with one of these network cards, you
 	  should say Y to this option if you wish to use it with Linux.
 
@@ -81,7 +81,7 @@ config MAC8390
 	tristate "Macintosh NS 8390 based ethernet cards"
 	depends on MAC
 	select CRC32
-	---help---
+	help
 	  If you want to include a driver to support Nubus or LC-PDS
 	  Ethernet cards using an NS8390 chipset or its equivalent, say Y.
 
@@ -89,7 +89,7 @@ config MCF8390
 	tristate "ColdFire NS8390 based Ethernet support"
 	depends on COLDFIRE
 	select CRC32
-	---help---
+	help
 	  This driver is for Ethernet devices using an NS8390-compatible
 	  chipset on many common ColdFire CPU based boards. Many of the older
 	  Freescale dev boards use this, and some other common boards like
@@ -103,7 +103,7 @@ config NE2000
 	tristate "NE2000/NE1000 support"
 	depends on (ISA || (Q40 && m) || MACH_TX49XX || ATARI_ETHERNEC)
 	select CRC32
-	---help---
+	help
 	  If you have a network (Ethernet) card of this type, say Y here.
 	  Many Ethernet cards without a specific driver are compatible with
 	  the NE2000.
@@ -118,7 +118,7 @@ config NE2K_PCI
 	tristate "PCI NE2000 and clones support (see help)"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver is for NE2000 compatible PCI cards. It will not work
 	  with ISA NE2000 cards (they have their own driver, "NE2000/NE1000
 	  support" below). If you have a PCI NE2000 network (Ethernet) card,
@@ -136,7 +136,7 @@ config APNE
 	tristate "PCMCIA NE2000 support"
 	depends on AMIGA_PCMCIA
 	select CRC32
-	---help---
+	help
 	  If you have a PCMCIA NE2000 compatible adapter, say Y.  Otherwise,
 	  say N.
 
@@ -147,7 +147,7 @@ config PCMCIA_PCNET
 	tristate "NE2000 compatible PCMCIA support"
 	depends on PCMCIA
 	select CRC32
-	---help---
+	help
 	  Say Y here if you intend to attach an NE2000 compatible PCMCIA
 	  (PC-card) Ethernet or Fast Ethernet card to your computer.
 
@@ -158,7 +158,7 @@ config STNIC
 	tristate "National DP83902AV  support"
 	depends on SUPERH
 	select CRC32
-	---help---
+	help
 	  Support for cards based on the National Semiconductor DP83902AV
 	  ST-NIC Serial Network Interface Controller for Twisted Pair.  This
 	  is a 10Mbit/sec Ethernet controller.  Product overview and specs at
@@ -170,7 +170,7 @@ config ULTRA
 	tristate "SMC Ultra support"
 	depends on ISA
 	select CRC32
-	---help---
+	help
 	  If you have a network (Ethernet) card of this type, say Y here.
 
 	  Important: There have been many reports that, with some motherboards
@@ -187,7 +187,7 @@ config WD80x3
 	tristate "WD80*3 support"
 	depends on ISA
 	select CRC32
-	---help---
+	help
 	  If you have a network (Ethernet) card of this type, say Y here.
 
 	  To compile this driver as a module, choose M here. The module
@@ -197,7 +197,7 @@ config ZORRO8390
 	tristate "Zorro NS8390-based Ethernet support"
 	depends on ZORRO
 	select CRC32
-	---help---
+	help
 	  This driver is for Zorro Ethernet cards using an NS8390-compatible
 	  chipset, like the Village Tronic Ariadne II and the Individual
 	  Computers X-Surf Ethernet cards. If you have such a card, say Y.
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index 4ded81b27d0a..de50e8b9e656 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -7,7 +7,7 @@ menuconfig ETHERNET
 	bool "Ethernet driver support"
 	depends on NET
 	default y
-	---help---
+	help
 	  This section contains all the Ethernet device drivers.
 
 if ETHERNET
@@ -48,7 +48,7 @@ config CX_ECAT
 	tristate "Beckhoff CX5020 EtherCAT master support"
 	depends on PCI
 	depends on X86 || COMPILE_TEST
-	---help---
+	help
 	  Driver for EtherCAT master module located on CCAT FPGA
 	  that can be found on Beckhoff CX5020, and possibly other of CX
 	  Beckhoff CX series industrial PCs.
@@ -62,7 +62,7 @@ config DNET
 	tristate "Dave ethernet support (DNET)"
 	depends on HAS_IOMEM
 	select PHYLIB
-	---help---
+	help
 	  The Dave ethernet interface (DNET) is found on Qong Board FPGA.
 	  Say Y to include support for the DNET chip.
 
@@ -89,7 +89,7 @@ config JME
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver supports the PCI-Express gigabit ethernet adapters
 	  based on JMicron JMC250 chipset.
 
@@ -99,20 +99,20 @@ config JME
 config KORINA
 	tristate "Korina (IDT RC32434) Ethernet support"
 	depends on MIKROTIK_RB532
-	---help---
+	help
 	  If you have a Mikrotik RouterBoard 500 or IDT RC32434
 	  based system say Y. Otherwise say N.
 
 config LANTIQ_ETOP
 	tristate "Lantiq SoC ETOP driver"
 	depends on SOC_TYPE_XWAY
-	---help---
+	help
 	  Support for the MII0 inside the Lantiq SoC
 
 config LANTIQ_XRX200
 	tristate "Lantiq / Intel xRX200 PMAC network driver"
 	depends on SOC_TYPE_XWAY
-	---help---
+	help
 	  Support for the PMAC of the Gigabit switch (GSWIP) inside the
 	  Lantiq / Intel VRX200 VDSL SoC
 
@@ -130,7 +130,7 @@ config FEALNX
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  Say Y here to support the Myson MTD-800 family of PCI-based Ethernet
 	  cards. <http://www.myson.com.tw/>
 
@@ -150,7 +150,7 @@ config ETHOC
 	select PHYLIB
 	select CRC32
 	select BITREVERSE
-	---help---
+	help
 	  Say Y here if you want to use the OpenCores 10/100 Mbps Ethernet MAC.
 
 source "drivers/net/ethernet/packetengines/Kconfig"
diff --git a/drivers/net/ethernet/adaptec/Kconfig b/drivers/net/ethernet/adaptec/Kconfig
index 86e02da4f993..c96edc2e582f 100644
--- a/drivers/net/ethernet/adaptec/Kconfig
+++ b/drivers/net/ethernet/adaptec/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_ADAPTEC
 	bool "Adaptec devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config ADAPTEC_STARFIRE
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  Say Y here if you have an Adaptec Starfire (or DuraLAN) PCI network
 	  adapter. The DuraLAN chip is used on the 64 bit PCI boards from
 	  Adaptec e.g. the ANA-6922A. The older 32 bit boards use the tulip
diff --git a/drivers/net/ethernet/aeroflex/Kconfig b/drivers/net/ethernet/aeroflex/Kconfig
index 2fa0a317266b..ea4be67f787b 100644
--- a/drivers/net/ethernet/aeroflex/Kconfig
+++ b/drivers/net/ethernet/aeroflex/Kconfig
@@ -8,5 +8,5 @@ config GRETH
 	depends on SPARC
 	select PHYLIB
 	select CRC32
-	---help---
+	help
 	  Say Y here if you want to use the Aeroflex Gaisler GRETH Ethernet MAC.
diff --git a/drivers/net/ethernet/agere/Kconfig b/drivers/net/ethernet/agere/Kconfig
index 084c7190ce2f..d92516ae59cc 100644
--- a/drivers/net/ethernet/agere/Kconfig
+++ b/drivers/net/ethernet/agere/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_AGERE
 	bool "Agere devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config ET131X
 	tristate "Agere ET-1310 Gigabit Ethernet support"
 	depends on PCI
 	select PHYLIB
-	---help---
+	help
 	  This driver supports Agere ET-1310 ethernet adapters.
 
 	  To compile this driver as a module, choose M here. The module
diff --git a/drivers/net/ethernet/alacritech/Kconfig b/drivers/net/ethernet/alacritech/Kconfig
index 212f92c8e6a9..5f285e18faf7 100644
--- a/drivers/net/ethernet/alacritech/Kconfig
+++ b/drivers/net/ethernet/alacritech/Kconfig
@@ -2,7 +2,7 @@
 config NET_VENDOR_ALACRITECH
 	bool "Alacritech devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -16,7 +16,7 @@ config SLICOSS
 	tristate "Alacritech Slicoss support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver supports Gigabit Ethernet adapters based on the
 	  Session Layer Interface (SLIC) technology by Alacritech.
 
diff --git a/drivers/net/ethernet/allwinner/Kconfig b/drivers/net/ethernet/allwinner/Kconfig
index 264a482ec31d..3e81059f8693 100644
--- a/drivers/net/ethernet/allwinner/Kconfig
+++ b/drivers/net/ethernet/allwinner/Kconfig
@@ -8,7 +8,7 @@ config NET_VENDOR_ALLWINNER
 	default y
 
 	depends on ARCH_SUNXI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this
 	  class, say Y here.
 
@@ -28,7 +28,7 @@ config SUN4I_EMAC
 	select MII
 	select PHYLIB
 	select MDIO_SUN4I
-	---help---
+	help
 	  Support for Allwinner A10 EMAC ethernet driver.
 
 	  To compile this driver as a module, choose M here.  The module
diff --git a/drivers/net/ethernet/alteon/Kconfig b/drivers/net/ethernet/alteon/Kconfig
index c3f7067d2d10..cfe1f3159d61 100644
--- a/drivers/net/ethernet/alteon/Kconfig
+++ b/drivers/net/ethernet/alteon/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_ALTEON
 	bool "Alteon devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_ALTEON
 config ACENIC
 	tristate "Alteon AceNIC/3Com 3C985/NetGear GA620 Gigabit support"
 	depends on PCI
-	---help---
+	help
 	  Say Y here if you have an Alteon AceNIC, 3Com 3C985(B), NetGear
 	  GA620, SGI Gigabit or Farallon PN9000-SX PCI Gigabit Ethernet
 	  adapter. The driver allows for using the Jumbo Frame option (9000
@@ -34,7 +34,7 @@ config ACENIC
 config ACENIC_OMIT_TIGON_I
 	bool "Omit support for old Tigon I based AceNICs"
 	depends on ACENIC
-	---help---
+	help
 	  Say Y here if you only have Tigon II based AceNICs and want to leave
 	  out support for the older Tigon I based cards which are no longer
 	  being sold (ie. the original Alteon AceNIC and 3Com 3C985 (non B
diff --git a/drivers/net/ethernet/altera/Kconfig b/drivers/net/ethernet/altera/Kconfig
index 2690c398d2b2..914e56b91467 100644
--- a/drivers/net/ethernet/altera/Kconfig
+++ b/drivers/net/ethernet/altera/Kconfig
@@ -3,7 +3,7 @@ config ALTERA_TSE
 	tristate "Altera Triple-Speed Ethernet MAC support"
 	depends on HAS_DMA
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the Altera Triple-Speed (TSE) Ethernet MAC.
 
 	  To compile this driver as a module, choose M here. The module
diff --git a/drivers/net/ethernet/amazon/Kconfig b/drivers/net/ethernet/amazon/Kconfig
index cca72a75f551..c37fa393b99e 100644
--- a/drivers/net/ethernet/amazon/Kconfig
+++ b/drivers/net/ethernet/amazon/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_AMAZON
 	bool "Amazon Devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) device belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ config ENA_ETHERNET
 	tristate "Elastic Network Adapter (ENA) support"
 	depends on PCI_MSI && !CPU_BIG_ENDIAN
 	select DIMLIB
-	---help---
+	help
 	  This driver supports Elastic Network Adapter (ENA)"
 
 	  To compile this driver as a module, choose M here.
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index a0af74c93971..dda4b8fc9525 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -355,7 +355,7 @@ error_unmap_dma:
 	ena_unmap_tx_buff(xdp_ring, tx_info);
 	tx_info->xdpf = NULL;
 error_drop_packet:
-
+	__free_page(tx_info->xdp_rx_page);
 	return NETDEV_TX_OK;
 }
 
@@ -1646,11 +1646,9 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
 					 &next_to_clean);
 
 		if (unlikely(!skb)) {
-			if (xdp_verdict == XDP_TX) {
+			if (xdp_verdict == XDP_TX)
 				ena_free_rx_page(rx_ring,
 						 &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id]);
-				res_budget--;
-			}
 			for (i = 0; i < ena_rx_ctx.descs; i++) {
 				rx_ring->free_ids[next_to_clean] =
 					rx_ring->ena_bufs[i].req_id;
@@ -1658,8 +1656,10 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
 					ENA_RX_RING_IDX_NEXT(next_to_clean,
 							     rx_ring->ring_size);
 			}
-			if (xdp_verdict == XDP_TX || xdp_verdict == XDP_DROP)
+			if (xdp_verdict != XDP_PASS) {
+				res_budget--;
 				continue;
+			}
 			break;
 		}
 
diff --git a/drivers/net/ethernet/amd/7990.c b/drivers/net/ethernet/amd/7990.c
index 50fb66369415..ef512cf89abf 100644
--- a/drivers/net/ethernet/amd/7990.c
+++ b/drivers/net/ethernet/amd/7990.c
@@ -28,6 +28,7 @@
 #include <linux/route.h>
 #include <linux/string.h>
 #include <linux/skbuff.h>
+#include <linux/pgtable.h>
 #include <asm/irq.h>
 /* Used for the temporal inet entries and routing */
 #include <linux/socket.h>
@@ -35,7 +36,6 @@
 
 #include <asm/io.h>
 #include <asm/dma.h>
-#include <asm/pgtable.h>
 #ifdef CONFIG_HP300
 #include <asm/blinken.h>
 #endif
diff --git a/drivers/net/ethernet/amd/Kconfig b/drivers/net/ethernet/amd/Kconfig
index 9f965cdfff5c..db7d956a9c9b 100644
--- a/drivers/net/ethernet/amd/Kconfig
+++ b/drivers/net/ethernet/amd/Kconfig
@@ -9,7 +9,7 @@ config NET_VENDOR_AMD
 	depends on DIO || MACH_DECSTATION || MVME147 || ATARI || SUN3 || \
 		   SUN3X || SBUS || PCI || ZORRO || (ISA && ISA_DMA_API) || \
 		   (ARM && ARCH_EBSA110) || ISA || EISA || PCMCIA || ARM64
-	---help---
+	help
 	  If you have a network (Ethernet) chipset belonging to this class,
 	  say Y.
 
@@ -24,7 +24,7 @@ config A2065
 	tristate "A2065 support"
 	depends on ZORRO
 	select CRC32
-	---help---
+	help
 	  If you have a Commodore A2065 Ethernet adapter, say Y. Otherwise,
 	  say N.
 
@@ -36,7 +36,7 @@ config AMD8111_ETH
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  If you have an AMD 8111-based PCI LANCE ethernet card,
 	  answer Y here.
 
@@ -46,7 +46,7 @@ config AMD8111_ETH
 config LANCE
 	tristate "AMD LANCE and PCnet (AT1500 and NE2100) support"
 	depends on ISA && ISA_DMA_API && !ARM && !PPC32
-	---help---
+	help
 	  If you have a network (Ethernet) card of this type, say Y here.
 	  Some LinkSys cards are of this type.
 
@@ -58,7 +58,7 @@ config PCNET32
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  If you have a PCnet32 or PCnetPCI based network (Ethernet) card,
 	  answer Y here.
 
@@ -68,7 +68,7 @@ config PCNET32
 config ARIADNE
 	tristate "Ariadne support"
 	depends on ZORRO
-	---help---
+	help
 	  If you have a Village Tronic Ariadne Ethernet adapter, say Y.
 	  Otherwise, say N.
 
@@ -79,14 +79,14 @@ config ARM_AM79C961A
 	bool "ARM EBSA110 AM79C961A support"
 	depends on ARM && ARCH_EBSA110
 	select CRC32
-	---help---
+	help
 	  If you wish to compile a kernel for the EBSA-110, then you should
 	  always answer Y to this.
 
 config ATARILANCE
 	tristate "Atari LANCE support"
 	depends on ATARI
-	---help---
+	help
 	  Say Y to include support for several Atari Ethernet adapters based
 	  on the AMD LANCE chipset: RieblCard (with or without battery), or
 	  PAMCard VME (also the version by Rhotron, with different addresses).
@@ -95,7 +95,7 @@ config DECLANCE
 	tristate "DEC LANCE ethernet controller support"
 	depends on MACH_DECSTATION
 	select CRC32
-	---help---
+	help
 	  This driver is for the series of Ethernet controllers produced by
 	  DEC (now Compaq) based on the AMD LANCE chipset, including the
 	  DEPCA series.  (This chipset is better known via the NE2100 cards.)
@@ -104,7 +104,7 @@ config HPLANCE
 	tristate "HP on-board LANCE support"
 	depends on DIO
 	select CRC32
-	---help---
+	help
 	  If you want to use the builtin "LANCE" Ethernet controller on an
 	  HP300 machine, say Y here.
 
@@ -113,7 +113,7 @@ config MIPS_AU1X00_ENET
 	depends on MIPS_ALCHEMY
 	select PHYLIB
 	select CRC32
-	---help---
+	help
 	  If you have an Alchemy Semi AU1X00 based system
 	  say Y.  Otherwise, say N.
 
@@ -121,7 +121,7 @@ config MVME147_NET
 	tristate "MVME147 (LANCE) Ethernet support"
 	depends on MVME147
 	select CRC32
-	---help---
+	help
 	  Support for the on-board Ethernet interface on the Motorola MVME147
 	  single-board computer.  Say Y here to include the
 	  driver for this chip in your kernel.
@@ -140,7 +140,7 @@ config PCMCIA_NMCLAN
 config NI65
 	tristate "NI6510 support"
 	depends on ISA && ISA_DMA_API && !ARM && !PPC32
-	---help---
+	help
 	  If you have a network (Ethernet) card of this type, say Y here.
 
 	  To compile this driver as a module, choose M here. The module
@@ -149,7 +149,7 @@ config NI65
 config SUN3LANCE
 	tristate "Sun3/Sun3x on-board LANCE support"
 	depends on (SUN3 || SUN3X)
-	---help---
+	help
 	  Most Sun3 and Sun3x motherboards (including the 3/50, 3/60 and 3/80)
 	  featured an AMD LANCE 10Mbit Ethernet controller on board; say Y
 	  here to compile in the Linux driver for this and enable Ethernet.
@@ -163,7 +163,7 @@ config SUNLANCE
 	tristate "Sun LANCE support"
 	depends on SBUS
 	select CRC32
-	---help---
+	help
 	  This driver supports the "le" interface present on all 32-bit Sparc
 	  systems, on some older Ultra systems and as an Sbus option.  These
 	  cards are based on the AMD LANCE chipset, which is better known
@@ -181,7 +181,7 @@ config AMD_XGBE
 	select PHYLIB
 	select AMD_XGBE_HAVE_ECC if X86
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports the AMD 10GbE Ethernet device found on an
 	  AMD SoC.
 
@@ -192,7 +192,7 @@ config AMD_XGBE_DCB
 	bool "Data Center Bridging (DCB) support"
 	default n
 	depends on AMD_XGBE && DCB
-	---help---
+	help
 	  Say Y here to enable Data Center Bridging (DCB) support in the
 	  driver.
 
diff --git a/drivers/net/ethernet/amd/hplance.c b/drivers/net/ethernet/amd/hplance.c
index 1381a474063f..e10aceb2b767 100644
--- a/drivers/net/ethernet/amd/hplance.c
+++ b/drivers/net/ethernet/amd/hplance.c
@@ -15,6 +15,7 @@
 #include <linux/delay.h>
 #include <linux/init.h>
 #include <linux/errno.h>
+#include <linux/pgtable.h>
 /* Used for the temporal inet entries and routing */
 #include <linux/socket.h>
 #include <linux/route.h>
@@ -24,7 +25,6 @@
 #include <linux/skbuff.h>
 
 #include <asm/io.h>
-#include <asm/pgtable.h>
 
 #include "hplance.h"
 
diff --git a/drivers/net/ethernet/amd/mvme147.c b/drivers/net/ethernet/amd/mvme147.c
index 72abd3f82249..3f2e4cdd0b83 100644
--- a/drivers/net/ethernet/amd/mvme147.c
+++ b/drivers/net/ethernet/amd/mvme147.c
@@ -16,6 +16,7 @@
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/gfp.h>
+#include <linux/pgtable.h>
 /* Used for the temporal inet entries and routing */
 #include <linux/socket.h>
 #include <linux/route.h>
@@ -24,7 +25,6 @@
 #include <linux/skbuff.h>
 
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/mvme147hw.h>
 
 /* We have 32K of RAM for the init block and buffers. This places
diff --git a/drivers/net/ethernet/amd/sun3lance.c b/drivers/net/ethernet/amd/sun3lance.c
index da7e3d4f4166..e1fde585fd0d 100644
--- a/drivers/net/ethernet/amd/sun3lance.c
+++ b/drivers/net/ethernet/amd/sun3lance.c
@@ -37,12 +37,12 @@ static const char version[] =
 #include <linux/etherdevice.h>
 #include <linux/skbuff.h>
 #include <linux/bitops.h>
+#include <linux/pgtable.h>
 
 #include <asm/cacheflush.h>
 #include <asm/setup.h>
 #include <asm/irq.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/dvma.h>
 #include <asm/idprom.h>
 #include <asm/machines.h>
diff --git a/drivers/net/ethernet/amd/sunlance.c b/drivers/net/ethernet/amd/sunlance.c
index a21b2e60157e..ddece276ae23 100644
--- a/drivers/net/ethernet/amd/sunlance.c
+++ b/drivers/net/ethernet/amd/sunlance.c
@@ -94,10 +94,10 @@ static char lancestr[] = "LANCE";
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/gfp.h>
+#include <linux/pgtable.h>
 
 #include <asm/io.h>
 #include <asm/dma.h>
-#include <asm/pgtable.h>
 #include <asm/byteorder.h>	/* Used by the checksum routines */
 #include <asm/idprom.h>
 #include <asm/prom.h>
diff --git a/drivers/net/ethernet/apple/Kconfig b/drivers/net/ethernet/apple/Kconfig
index f78b9c841296..a4176d2ecec6 100644
--- a/drivers/net/ethernet/apple/Kconfig
+++ b/drivers/net/ethernet/apple/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_APPLE
 	bool "Apple devices"
 	default y
 	depends on (PPC_PMAC && PPC32) || MAC
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config MACE
 	tristate "MACE (Power Mac ethernet) support"
 	depends on PPC_PMAC && PPC32
 	select CRC32
-	---help---
+	help
 	  Power Macintoshes and clones with Ethernet built-in on the
 	  motherboard will usually use a MACE (Medium Access Control for
 	  Ethernet) interface. Say Y to include support for the MACE chip.
@@ -32,7 +32,7 @@ config MACE
 config MACE_AAUI_PORT
 	bool "Use AAUI port instead of TP by default"
 	depends on MACE
-	---help---
+	help
 	  Some Apple machines (notably the Apple Network Server) which use the
 	  MACE ethernet chip have an Apple AUI port (small 15-pin connector),
 	  instead of an 8-pin RJ45 connector for twisted-pair ethernet.  Say
@@ -44,7 +44,7 @@ config BMAC
 	tristate "BMAC (G3 ethernet) support"
 	depends on PPC_PMAC && PPC32
 	select CRC32
-	---help---
+	help
 	  Say Y for support of BMAC Ethernet interfaces. These are used on G3
 	  computers.
 
@@ -55,7 +55,7 @@ config MACMACE
 	tristate "Macintosh (AV) onboard MACE ethernet"
 	depends on MAC
 	select CRC32
-	---help---
+	help
 	  Support for the onboard AMD 79C940 MACE Ethernet controller used in
 	  the 660AV and 840AV Macintosh.  If you have one of these Macintoshes
 	  say Y here.
diff --git a/drivers/net/ethernet/apple/bmac.c b/drivers/net/ethernet/apple/bmac.c
index 3e3711b60d01..1e4e402f07d7 100644
--- a/drivers/net/ethernet/apple/bmac.c
+++ b/drivers/net/ethernet/apple/bmac.c
@@ -24,11 +24,11 @@
 #include <linux/bitrev.h>
 #include <linux/ethtool.h>
 #include <linux/slab.h>
+#include <linux/pgtable.h>
 #include <asm/prom.h>
 #include <asm/dbdma.h>
 #include <asm/io.h>
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/machdep.h>
 #include <asm/pmac_feature.h>
 #include <asm/macio.h>
diff --git a/drivers/net/ethernet/apple/mace.c b/drivers/net/ethernet/apple/mace.c
index b8ba2abf5b3a..9e5006e59215 100644
--- a/drivers/net/ethernet/apple/mace.c
+++ b/drivers/net/ethernet/apple/mace.c
@@ -19,10 +19,10 @@
 #include <linux/spinlock.h>
 #include <linux/bitrev.h>
 #include <linux/slab.h>
+#include <linux/pgtable.h>
 #include <asm/prom.h>
 #include <asm/dbdma.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/macio.h>
 
 #include "mace.h"
diff --git a/drivers/net/ethernet/aquantia/Kconfig b/drivers/net/ethernet/aquantia/Kconfig
index 76a44b2546ff..efb33c078a3c 100644
--- a/drivers/net/ethernet/aquantia/Kconfig
+++ b/drivers/net/ethernet/aquantia/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_AQUANTIA
 	bool "aQuantia devices"
 	default y
-	---help---
+	help
 	  Set this to y if you have an Ethernet network cards that uses the aQuantia
 	  AQC107/AQC108 chipset.
 
@@ -21,7 +21,7 @@ config AQTION
 	depends on PCI
 	depends on X86_64 || ARM64 || COMPILE_TEST
 	depends on MACSEC || MACSEC=n
-	---help---
+	help
 	  This enables the support for the aQuantia AQtion(tm) Ethernet card.
 
 endif # NET_VENDOR_AQUANTIA
diff --git a/drivers/net/ethernet/arc/Kconfig b/drivers/net/ethernet/arc/Kconfig
index 45c663d8b9aa..37a41773dd43 100644
--- a/drivers/net/ethernet/arc/Kconfig
+++ b/drivers/net/ethernet/arc/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_ARC
 	bool "ARC devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -27,7 +27,7 @@ config ARC_EMAC
 	select ARC_EMAC_CORE
 	depends on OF_IRQ && OF_NET
 	depends on ARC || COMPILE_TEST
-	---help---
+	help
 	  On some legacy ARC (Synopsys) FPGA boards such as ARCAngel4/ML50x
 	  non-standard on-chip ethernet device ARC EMAC 10/100 is used.
 	  Say Y here if you have such a board.  If unsure, say N.
@@ -37,7 +37,7 @@ config EMAC_ROCKCHIP
 	select ARC_EMAC_CORE
 	depends on OF_IRQ && OF_NET && REGULATOR
 	depends on ARCH_ROCKCHIP || COMPILE_TEST
-	---help---
+	help
 	  Support for Rockchip RK3036/RK3066/RK3188 EMAC ethernet controllers.
 	  This selects Rockchip SoC glue layer support for the
 	  emac device driver. This driver is used for RK3036/RK3066/RK3188
diff --git a/drivers/net/ethernet/atheros/Kconfig b/drivers/net/ethernet/atheros/Kconfig
index 2720bde5034e..fb803bf92ded 100644
--- a/drivers/net/ethernet/atheros/Kconfig
+++ b/drivers/net/ethernet/atheros/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_ATHEROS
 	bool "Atheros devices"
 	default y
 	depends on (PCI || ATH79)
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -30,7 +30,7 @@ config ATL2
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver supports the Atheros L2 fast ethernet adapter.
 
 	  To compile this driver as a module, choose M here.  The module
@@ -41,7 +41,7 @@ config ATL1
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver supports the Atheros/Attansic L1 gigabit ethernet
 	  adapter.
 
@@ -53,7 +53,7 @@ config ATL1E
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver supports the Atheros L1E gigabit ethernet adapter.
 
 	  To compile this driver as a module, choose M here.  The module
@@ -64,7 +64,7 @@ config ATL1C
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver supports the Atheros L1C gigabit ethernet adapter.
 
 	  To compile this driver as a module, choose M here.  The module
diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c
index b9b4edb913c1..9b7f1af5f574 100644
--- a/drivers/net/ethernet/atheros/alx/main.c
+++ b/drivers/net/ethernet/atheros/alx/main.c
@@ -1249,8 +1249,12 @@ out_disable_adv_intr:
 
 static void __alx_stop(struct alx_priv *alx)
 {
-	alx_halt(alx);
 	alx_free_irq(alx);
+
+	cancel_work_sync(&alx->link_check_wk);
+	cancel_work_sync(&alx->reset_wk);
+
+	alx_halt(alx);
 	alx_free_rings(alx);
 	alx_free_napis(alx);
 }
@@ -1855,9 +1859,6 @@ static void alx_remove(struct pci_dev *pdev)
 	struct alx_priv *alx = pci_get_drvdata(pdev);
 	struct alx_hw *hw = &alx->hw;
 
-	cancel_work_sync(&alx->link_check_wk);
-	cancel_work_sync(&alx->reset_wk);
-
 	/* restore permanent mac address */
 	alx_set_macaddr(hw, hw->perm_addr);
 
diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig
index 2a69c0d06f3c..7fb42f388d59 100644
--- a/drivers/net/ethernet/broadcom/Kconfig
+++ b/drivers/net/ethernet/broadcom/Kconfig
@@ -8,7 +8,7 @@ config NET_VENDOR_BROADCOM
 	default y
 	depends on (SSB_POSSIBLE && HAS_DMA) || PCI || BCM63XX || \
 		   SIBYTE_SB1xxx_SOC
-	---help---
+	help
 	  If you have a network (Ethernet) chipset belonging to this class,
 	  say Y.
 
@@ -25,7 +25,7 @@ config B44
 	select SSB
 	select MII
 	select PHYLIB
-	---help---
+	help
 	  If you have a network (Ethernet) controller of this type, say Y
 	  or M here.
 
@@ -79,7 +79,7 @@ config BNX2
 	depends on PCI
 	select CRC32
 	select FW_LOADER
-	---help---
+	help
 	  This driver supports QLogic bnx2 gigabit Ethernet cards.
 
 	  To compile this driver as a module, choose M here: the module
@@ -90,7 +90,7 @@ config CNIC
 	depends on PCI && (IPV6 || IPV6=n)
 	select BNX2
 	select UIO
-	---help---
+	help
 	  This driver supports offload features of QLogic bnx2 gigabit
 	  Ethernet cards.
 
@@ -101,7 +101,7 @@ config SB1250_MAC
 	tristate "SB1250 Gigabit Ethernet support"
 	depends on SIBYTE_SB1xxx_SOC
 	select PHYLIB
-	---help---
+	help
 	  This driver supports Gigabit Ethernet interfaces based on the
 	  Broadcom SiByte family of System-On-a-Chip parts.  They include
 	  the BCM1120, BCM1125, BCM1125H, BCM1250, BCM1255, BCM1280, BCM1455
@@ -115,7 +115,7 @@ config TIGON3
 	depends on PCI
 	select PHYLIB
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports Broadcom Tigon3 based gigabit Ethernet cards.
 
 	  To compile this driver as a module, choose M here: the module
@@ -125,7 +125,7 @@ config TIGON3_HWMON
 	bool "Broadcom Tigon3 HWMON support"
 	default y
 	depends on TIGON3 && HWMON && !(TIGON3=y && HWMON=m)
-	---help---
+	help
 	  Say Y if you want to expose the thermal sensor on Tigon3 devices.
 
 config BNX2X
@@ -136,7 +136,7 @@ config BNX2X
 	select ZLIB_INFLATE
 	select LIBCRC32C
 	select MDIO
-	---help---
+	help
 	  This driver supports Broadcom NetXtremeII 10 gigabit Ethernet cards.
 	  To compile this driver as a module, choose M here: the module
 	  will be called bnx2x.  This is recommended.
@@ -145,7 +145,7 @@ config BNX2X_SRIOV
 	bool "Broadcom 578xx and 57712 SR-IOV support"
 	depends on BNX2X && PCI_IOV
 	default y
-	---help---
+	help
 	  This configuration parameter enables Single Root Input Output
 	  Virtualization support in the 578xx and 57712 products. This
 	  allows for virtual function acceleration in virtual environments.
@@ -164,7 +164,7 @@ config BGMAC_BCMA
 	select BGMAC
 	select PHYLIB
 	select FIXED_PHY
-	---help---
+	help
 	  This driver supports GBit MAC and BCM4706 GBit MAC cores on BCMA bus.
 	  They can be found on BCM47xx SoCs and provide gigabit ethernet.
 	  In case of using this driver on BCM4706 it's also requires to enable
@@ -178,7 +178,7 @@ config BGMAC_PLATFORM
 	select PHYLIB
 	select FIXED_PHY
 	default ARCH_BCM_IPROC
-	---help---
+	help
 	  Say Y here if you want to use the Broadcom iProc Gigabit Ethernet
 	  controller through the generic platform interface
 
@@ -203,7 +203,7 @@ config BNXT
 	select NET_DEVLINK
 	select PAGE_POOL
 	select DIMLIB
-	---help---
+	help
 	  This driver supports Broadcom NetXtreme-C/E 10/25/40/50 gigabit
 	  Ethernet cards.  To compile this driver as a module, choose M here:
 	  the module will be called bnxt_en.  This is recommended.
@@ -212,7 +212,7 @@ config BNXT_SRIOV
 	bool "Broadcom NetXtreme-C/E SR-IOV support"
 	depends on BNXT && PCI_IOV
 	default y
-	---help---
+	help
 	  This configuration parameter enables Single Root Input Output
 	  Virtualization support in the NetXtreme-C/E products. This
 	  allows for virtual function acceleration in virtual environments.
@@ -221,7 +221,7 @@ config BNXT_FLOWER_OFFLOAD
 	bool "TC Flower offload support for NetXtreme-C/E"
 	depends on BNXT
 	default y
-	---help---
+	help
 	  This configuration parameter enables TC Flower packet classifier
 	  offload for eswitch.  This option enables SR-IOV switchdev eswitch
 	  offload.
@@ -230,7 +230,7 @@ config BNXT_DCB
 	bool "Data Center Bridging (DCB) Support"
 	default n
 	depends on BNXT && DCB
-	---help---
+	help
 	  Say Y here if you want to use Data Center Bridging (DCB) in the
 	  driver.
 
@@ -240,7 +240,7 @@ config BNXT_HWMON
 	bool "Broadcom NetXtreme-C/E HWMON support"
 	default y
 	depends on BNXT && HWMON && !(BNXT=y && HWMON=m)
-	---help---
+	help
 	  Say Y if you want to expose the thermal sensor data on NetXtreme-C/E
 	  devices, via the hwmon sysfs interface.
 
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index c62589c266b2..b93e05f91d77 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -10037,7 +10037,7 @@ static void bnxt_timer(struct timer_list *t)
 	struct bnxt *bp = from_timer(bp, t, timer);
 	struct net_device *dev = bp->dev;
 
-	if (!netif_running(dev))
+	if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
 		return;
 
 	if (atomic_read(&bp->intr_sem) != 0)
@@ -12133,19 +12133,9 @@ static int bnxt_resume(struct device *device)
 		goto resume_exit;
 	}
 
-	if (bnxt_hwrm_queue_qportcfg(bp)) {
-		rc = -ENODEV;
+	rc = bnxt_hwrm_func_qcaps(bp);
+	if (rc)
 		goto resume_exit;
-	}
-
-	if (bp->hwrm_spec_code >= 0x10803) {
-		if (bnxt_alloc_ctx_mem(bp)) {
-			rc = -ENODEV;
-			goto resume_exit;
-		}
-	}
-	if (BNXT_NEW_RM(bp))
-		bnxt_hwrm_func_resc_qcaps(bp, false);
 
 	if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
 		rc = -ENODEV;
@@ -12161,6 +12151,8 @@ static int bnxt_resume(struct device *device)
 
 resume_exit:
 	bnxt_ulp_start(bp, rc);
+	if (!rc)
+		bnxt_reenable_sriov(bp);
 	rtnl_unlock();
 	return rc;
 }
@@ -12204,6 +12196,9 @@ static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
 		bnxt_close(netdev);
 
 	pci_disable_device(pdev);
+	bnxt_free_ctx_mem(bp);
+	kfree(bp->ctx);
+	bp->ctx = NULL;
 	rtnl_unlock();
 
 	/* Request a slot slot reset. */
@@ -12237,12 +12232,16 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
 		pci_set_master(pdev);
 
 		err = bnxt_hwrm_func_reset(bp);
-		if (!err && netif_running(netdev))
-			err = bnxt_open(netdev);
-
-		if (!err)
-			result = PCI_ERS_RESULT_RECOVERED;
+		if (!err) {
+			err = bnxt_hwrm_func_qcaps(bp);
+			if (!err && netif_running(netdev))
+				err = bnxt_open(netdev);
+		}
 		bnxt_ulp_start(bp, err);
+		if (!err) {
+			bnxt_reenable_sriov(bp);
+			result = PCI_ERS_RESULT_RECOVERED;
+		}
 	}
 
 	if (result != PCI_ERS_RESULT_RECOVERED) {
diff --git a/drivers/net/ethernet/brocade/Kconfig b/drivers/net/ethernet/brocade/Kconfig
index d4564c7a279c..fb4c3cdf7233 100644
--- a/drivers/net/ethernet/brocade/Kconfig
+++ b/drivers/net/ethernet/brocade/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_BROCADE
 	bool "QLogic BR-series devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/ethernet/brocade/bna/Kconfig b/drivers/net/ethernet/brocade/bna/Kconfig
index b124a628f86a..0a48ad93ed6c 100644
--- a/drivers/net/ethernet/brocade/bna/Kconfig
+++ b/drivers/net/ethernet/brocade/bna/Kconfig
@@ -6,7 +6,7 @@
 config BNA
 	tristate "QLogic BR-series 1010/1020/1860 10Gb Ethernet Driver support"
 	depends on PCI
-	---help---
+	help
 	  This driver supports QLogic BR-series 1010/1020/1860 10Gb CEE capable
 	  Ethernet cards.
 	  To compile this driver as a module, choose M here: the module
diff --git a/drivers/net/ethernet/cadence/Kconfig b/drivers/net/ethernet/cadence/Kconfig
index 2c4c12b03502..85858163bac5 100644
--- a/drivers/net/ethernet/cadence/Kconfig
+++ b/drivers/net/ethernet/cadence/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_CADENCE
 	bool "Cadence devices"
 	depends on HAS_IOMEM
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  If unsure, say Y.
@@ -23,7 +23,7 @@ config MACB
 	tristate "Cadence MACB/GEM support"
 	depends on HAS_DMA && COMMON_CLK
 	select PHYLINK
-	---help---
+	help
 	  The Cadence MACB ethernet interface is found on many Atmel AT32 and
 	  AT91 parts.  This driver also supports the Cadence GEM (Gigabit
 	  Ethernet MAC found in some ARM SoC devices).  Say Y to include
@@ -37,13 +37,13 @@ config MACB_USE_HWSTAMP
 	depends on MACB
 	depends on PTP_1588_CLOCK
 	default y
-	---help---
+	help
 	  Enable IEEE 1588 Precision Time Protocol (PTP) support for MACB.
 
 config MACB_PCI
 	tristate "Cadence PCI MACB/GEM support"
 	depends on MACB && PCI
-	---help---
+	help
 	  This is PCI wrapper for MACB driver.
 
 	  To compile this driver as a module, choose M here: the module
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index 36290a8e2a84..67933079aeea 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -2558,19 +2558,21 @@ static int macb_open(struct net_device *dev)
 
 	err = macb_phylink_connect(bp);
 	if (err)
-		goto pm_exit;
+		goto napi_exit;
 
 	netif_tx_start_all_queues(dev);
 
 	if (bp->ptp_info)
 		bp->ptp_info->ptp_init(dev);
 
-pm_exit:
-	if (err) {
-		pm_runtime_put_sync(&bp->pdev->dev);
-		return err;
-	}
 	return 0;
+
+napi_exit:
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
+		napi_disable(&queue->napi);
+pm_exit:
+	pm_runtime_put_sync(&bp->pdev->dev);
+	return err;
 }
 
 static int macb_close(struct net_device *dev)
diff --git a/drivers/net/ethernet/cavium/Kconfig b/drivers/net/ethernet/cavium/Kconfig
index 4520e7ee00fe..4875cdae622e 100644
--- a/drivers/net/ethernet/cavium/Kconfig
+++ b/drivers/net/ethernet/cavium/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_CAVIUM
 	bool "Cavium ethernet drivers"
 	default y
-	---help---
+	help
 	  Select this option if you want enable Cavium network support.
 
 	  If you have a Cavium SoC or network adapter, say Y.
@@ -17,7 +17,7 @@ config THUNDER_NIC_PF
 	tristate "Thunder Physical function driver"
 	depends on 64BIT && PCI
 	select THUNDER_NIC_BGX
-	---help---
+	help
 	  This driver supports Thunder's NIC physical function.
 	  The NIC provides the controller and DMA engines to
 	  move network traffic to/from the memory. The NIC
@@ -29,7 +29,7 @@ config THUNDER_NIC_VF
 	tristate "Thunder Virtual function driver"
 	imply CAVIUM_PTP
 	depends on 64BIT && PCI
-	---help---
+	help
 	  This driver supports Thunder's NIC virtual function
 
 config	THUNDER_NIC_BGX
@@ -38,7 +38,7 @@ config	THUNDER_NIC_BGX
 	select PHYLIB
 	select MDIO_THUNDER if PCI
 	select THUNDER_NIC_RGX
-	---help---
+	help
 	  This driver supports programming and controlling of MAC
 	  interface from NIC physical function driver.
 
@@ -47,7 +47,7 @@ config	THUNDER_NIC_RGX
 	depends on 64BIT && PCI
 	select PHYLIB
 	select MDIO_THUNDER if PCI
-	---help---
+	help
 	  This driver supports configuring XCV block of RGX interface
 	  present on CN81XX chip.
 
@@ -55,7 +55,7 @@ config CAVIUM_PTP
 	tristate "Cavium PTP coprocessor as PTP clock"
 	depends on 64BIT && PCI
 	depends on PTP_1588_CLOCK
-	---help---
+	help
 	  This driver adds support for the Precision Time Protocol Clocks and
 	  Timestamping coprocessor (PTP) found on Cavium processors.
 	  PTP provides timestamping mechanism that is suitable for use in IEEE 1588
@@ -70,7 +70,7 @@ config LIQUIDIO
 	select FW_LOADER
 	select LIBCRC32C
 	select NET_DEVLINK
-	---help---
+	help
 	  This driver supports Cavium LiquidIO Intelligent Server Adapters
 	  based on CN66XX, CN68XX and CN23XX chips.
 
@@ -92,7 +92,7 @@ config LIQUIDIO_VF
 	tristate "Cavium LiquidIO VF support"
 	depends on 64BIT && PCI_MSI
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports Cavium LiquidIO Intelligent Server Adapter
 	  based on CN23XX chips.
 
diff --git a/drivers/net/ethernet/chelsio/Kconfig b/drivers/net/ethernet/chelsio/Kconfig
index 82cdfa51ce37..b7f43254598f 100644
--- a/drivers/net/ethernet/chelsio/Kconfig
+++ b/drivers/net/ethernet/chelsio/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_CHELSIO
 	bool "Chelsio devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config CHELSIO_T1
 	depends on PCI
 	select CRC32
 	select MDIO
-	---help---
+	help
 	  This driver supports Chelsio gigabit and 10-gigabit
 	  Ethernet cards. More information about adapter features and
 	  performance tuning is in
@@ -42,7 +42,7 @@ config CHELSIO_T1
 config CHELSIO_T1_1G
 	bool "Chelsio gigabit Ethernet support"
 	depends on CHELSIO_T1
-	---help---
+	help
 	  Enables support for Chelsio's gigabit Ethernet PCI cards.  If you
 	  are using only 10G cards say 'N' here.
 
@@ -51,7 +51,7 @@ config CHELSIO_T3
 	depends on PCI && INET
 	select FW_LOADER
 	select MDIO
-	---help---
+	help
 	  This driver supports Chelsio T3-based gigabit and 10Gb Ethernet
 	  adapters.
 
@@ -72,7 +72,7 @@ config CHELSIO_T4
 	select FW_LOADER
 	select MDIO
 	select ZLIB_DEFLATE
-	---help---
+	help
 	  This driver supports Chelsio T4, T5 & T6 based gigabit, 10Gb Ethernet
 	  adapter and T5/T6 based 40Gb and T6 based 25Gb, 50Gb and 100Gb
 	  Ethernet adapters.
@@ -92,7 +92,7 @@ config CHELSIO_T4_DCB
 	bool "Data Center Bridging (DCB) Support for Chelsio T4/T5/T6 cards"
 	default n
 	depends on CHELSIO_T4 && DCB
-	---help---
+	help
 	  Enable DCB support through rtNetlink interface.
 	  Say Y here if you want to enable Data Center Bridging (DCB) support
 	  in the driver.
@@ -103,7 +103,7 @@ config CHELSIO_T4_FCOE
 	bool "Fibre Channel over Ethernet (FCoE) Support for Chelsio T5 cards"
 	default n
 	depends on CHELSIO_T4 && CHELSIO_T4_DCB && FCOE
-	---help---
+	help
 	  Enable FCoE offload features.
 	  Say Y here if you want to enable Fibre Channel over Ethernet (FCoE) support
 	  in the driver.
@@ -113,7 +113,7 @@ config CHELSIO_T4_FCOE
 config CHELSIO_T4VF
 	tristate "Chelsio Communications T4/T5/T6 Virtual Function Ethernet support"
 	depends on PCI
-	---help---
+	help
 	  This driver supports Chelsio T4, T5 & T6 based gigabit, 10Gb Ethernet
 	  adapters and T5/T6 based 40Gb and T6 based 25Gb, 50Gb and 100Gb
 	  Ethernet adapters with PCI-E SR-IOV Virtual Functions.
@@ -131,7 +131,7 @@ config CHELSIO_T4VF
 
 config CHELSIO_LIB
 	tristate
-	---help---
+	help
 	Common library for Chelsio drivers.
 
 endif # NET_VENDOR_CHELSIO
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index 41315712deb8..828499256004 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -3357,7 +3357,7 @@ static ssize_t blocked_fl_read(struct file *filp, char __user *ubuf,
 		       adap->sge.egr_sz, adap->sge.blocked_fl);
 	len += sprintf(buf + len, "\n");
 	size = simple_read_from_buffer(ubuf, count, ppos, buf, len);
-	kvfree(buf);
+	kfree(buf);
 	return size;
 }
 
@@ -3374,12 +3374,12 @@ static ssize_t blocked_fl_write(struct file *filp, const char __user *ubuf,
 
 	err = bitmap_parse_user(ubuf, count, t, adap->sge.egr_sz);
 	if (err) {
-		kvfree(t);
+		kfree(t);
 		return err;
 	}
 
 	bitmap_copy(adap->sge.blocked_fl, t, adap->sge.egr_sz);
-	kvfree(t);
+	kfree(t);
 	return count;
 }
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
index 0307e9c69a47..08439e215efe 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -663,6 +663,7 @@ static int uld_attach(struct adapter *adap, unsigned int uld)
 	return 0;
 }
 
+#ifdef CONFIG_CHELSIO_TLS_DEVICE
 static bool cxgb4_uld_in_use(struct adapter *adap)
 {
 	const struct tid_info *t = &adap->tids;
@@ -670,7 +671,6 @@ static bool cxgb4_uld_in_use(struct adapter *adap)
 	return (atomic_read(&t->conns_in_use) || t->stids_in_use);
 }
 
-#ifdef CONFIG_CHELSIO_TLS_DEVICE
 /* cxgb4_set_ktls_feature: request FW to enable/disable ktls settings.
  * @adap: adapter info
  * @enable: 1 to enable / 0 to disable ktls settings.
diff --git a/drivers/net/ethernet/cirrus/Kconfig b/drivers/net/ethernet/cirrus/Kconfig
index 8d845f5ee0c5..7cf4526596f3 100644
--- a/drivers/net/ethernet/cirrus/Kconfig
+++ b/drivers/net/ethernet/cirrus/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_CIRRUS
 	bool "Cirrus devices"
 	default y
 	depends on ISA || EISA || ARM || MAC
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config CS89x0
 	tristate "CS89x0 support"
 	depends on ISA || EISA || ARM
 	depends on !PPC32
-	---help---
+	help
 	  Support for CS89x0 chipset based Ethernet cards. If you have a
 	  network (Ethernet) card of this type, say Y and read the file
 	  <file:Documentation/networking/device_drivers/cirrus/cs89x0.rst>.
@@ -51,7 +51,7 @@ config EP93XX_ETH
 config MAC89x0
 	tristate "Macintosh CS89x0 based ethernet cards"
 	depends on MAC
-	---help---
+	help
 	  Support for CS89x0 chipset based Ethernet cards.  If you have a
 	  Nubus or LC-PDS network (Ethernet) card of this type, say Y here.
 
diff --git a/drivers/net/ethernet/cisco/Kconfig b/drivers/net/ethernet/cisco/Kconfig
index ee5b7b3868c7..18c3a0718d6f 100644
--- a/drivers/net/ethernet/cisco/Kconfig
+++ b/drivers/net/ethernet/cisco/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_CISCO
 	bool "Cisco devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/ethernet/cisco/enic/Kconfig b/drivers/net/ethernet/cisco/enic/Kconfig
index edaae706a102..ad80c0fa96a6 100644
--- a/drivers/net/ethernet/cisco/enic/Kconfig
+++ b/drivers/net/ethernet/cisco/enic/Kconfig
@@ -6,5 +6,5 @@
 config ENIC
 	tristate "Cisco VIC Ethernet NIC Support"
 	depends on PCI
-	---help---
+	help
 	  This enables the support for the Cisco VIC Ethernet card.
diff --git a/drivers/net/ethernet/cortina/Kconfig b/drivers/net/ethernet/cortina/Kconfig
index 89bc4579724d..ac8cb5744a87 100644
--- a/drivers/net/ethernet/cortina/Kconfig
+++ b/drivers/net/ethernet/cortina/Kconfig
@@ -4,7 +4,7 @@
 config NET_VENDOR_CORTINA
 	bool "Cortina Gemini devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y
 	  and read the Ethernet-HOWTO, available from
 	  <http://www.tldp.org/docs.html#howto>.
@@ -17,7 +17,7 @@ config GEMINI_ETHERNET
 	depends on HAS_IOMEM
 	select PHYLIB
 	select CRC32
-	---help---
+	help
 	  This driver supports StorLink SL351x (Gemini) dual Gigabit Ethernet.
 
 endif # NET_VENDOR_CORTINA
diff --git a/drivers/net/ethernet/davicom/Kconfig b/drivers/net/ethernet/davicom/Kconfig
index a321a7144fb0..76247103e566 100644
--- a/drivers/net/ethernet/davicom/Kconfig
+++ b/drivers/net/ethernet/davicom/Kconfig
@@ -8,7 +8,7 @@ config DM9000
 	depends on ARM || MIPS || COLDFIRE || NIOS2
 	select CRC32
 	select MII
-	---help---
+	help
 	  Support for DM9000 chipset.
 
 	  To compile this driver as a module, choose M here.  The module
@@ -17,7 +17,7 @@ config DM9000
 config DM9000_FORCE_SIMPLE_PHY_POLL
 	bool "Force simple NSR based PHY polling"
 	depends on DM9000
-	---help---
+	help
 	  This configuration forces the DM9000 to use the NSR's LinkStatus
 	  bit to determine if the link is up or down instead of the more
 	  costly MII PHY reads. Note, this will not work if the chip is
diff --git a/drivers/net/ethernet/dec/Kconfig b/drivers/net/ethernet/dec/Kconfig
index df1eeb04d5ea..9e5e5f10bd19 100644
--- a/drivers/net/ethernet/dec/Kconfig
+++ b/drivers/net/ethernet/dec/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_DEC
 	bool "Digital Equipment devices"
 	default y
 	depends on PCI || EISA || CARDBUS
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/ethernet/dec/tulip/Kconfig b/drivers/net/ethernet/dec/tulip/Kconfig
index 177f36f4b89d..8afc5942179a 100644
--- a/drivers/net/ethernet/dec/tulip/Kconfig
+++ b/drivers/net/ethernet/dec/tulip/Kconfig
@@ -6,7 +6,7 @@
 config NET_TULIP
 	bool "DEC - Tulip devices"
 	depends on (PCI || EISA || CARDBUS)
-	---help---
+	help
 	  This selects the "Tulip" family of EISA/PCI network cards.
 
 if NET_TULIP
@@ -15,7 +15,7 @@ config DE2104X
 	tristate "Early DECchip Tulip (dc2104x) PCI support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver is developed for the SMC EtherPower series Ethernet
 	  cards and also works with cards based on the DECchip
 	  21040 (Tulip series) chips.  Some LinkSys PCI cards are
@@ -32,7 +32,7 @@ config DE2104X_DSL
 	depends on DE2104X
 	range 0 31
 	default 0
-	---help---
+	help
 	  Setting this value allows to align ring buffer descriptors into their
 	  own cache lines. Value of 4 corresponds to the typical 32 byte line
 	  (the descriptor is 16 bytes). This is necessary on systems that lack
@@ -43,7 +43,7 @@ config TULIP
 	tristate "DECchip Tulip (dc2114x) PCI support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver is developed for the SMC EtherPower series Ethernet
 	  cards and also works with cards based on the DECchip 
 	  21140 (Tulip series) chips.  Some LinkSys PCI cards are
@@ -58,7 +58,7 @@ config TULIP
 config TULIP_MWI
 	bool "New bus configuration"
 	depends on TULIP
-	---help---
+	help
 	  This configures your Tulip card specifically for the card and
 	  system cache line size type you are using.
 
@@ -69,7 +69,7 @@ config TULIP_MWI
 config TULIP_MMIO
 	bool "Use PCI shared mem for NIC registers"
 	depends on TULIP
-	---help---
+	help
 	  Use PCI shared memory for the NIC registers, rather than going through
 	  the Tulip's PIO (programmed I/O ports).  Faster, but could produce
 	  obscure bugs if your mainboard has memory controller timing issues.
@@ -78,7 +78,7 @@ config TULIP_MMIO
 config TULIP_NAPI
 	bool "Use RX polling (NAPI)"
 	depends on TULIP
-	---help---
+	help
 	  NAPI is a new driver API designed to reduce CPU and interrupt load
 	  when the driver is receiving lots of packets from the card. It is
 	  still somewhat experimental and thus not yet enabled by default.
@@ -92,7 +92,7 @@ config TULIP_NAPI
 config TULIP_NAPI_HW_MITIGATION
 	bool "Use Interrupt Mitigation"
 	depends on TULIP_NAPI
-	---help---
+	help
 	  Use HW to reduce RX interrupts. Not strictly necessary since NAPI
 	  reduces RX interrupts by itself. Interrupt mitigation reduces RX
 	  interrupts even at low levels of traffic at the cost of a small
@@ -109,7 +109,7 @@ config DE4X5
 	depends on (PCI || EISA)
 	depends on VIRT_TO_BUS || ALPHA || PPC || SPARC
 	select CRC32
-	---help---
+	help
 	  This is support for the DIGITAL series of PCI/EISA Ethernet cards.
 	  These include the DE425, DE434, DE435, DE450 and DE500 models.  If
 	  you have a network card of this type, say Y.  More specific
@@ -124,7 +124,7 @@ config WINBOND_840
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver is for the Winbond W89c840 chip.  It also works with 
 	  the TX9882 chip on the Compex RL100-ATX board.
 	  More specific information and updates are available from
@@ -134,7 +134,7 @@ config DM9102
 	tristate "Davicom DM910x/DM980x support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver is for DM9102(A)/DM9132/DM9801 compatible PCI cards from
 	  Davicom (<http://www.davicom.com.tw/>).  If you have such a network
 	  (Ethernet) card, say Y.  Some information is contained in the file
@@ -147,7 +147,7 @@ config ULI526X
 	tristate "ULi M526x controller support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver is for ULi M5261/M5263 10/100M Ethernet Controller
 	  (<http://www.nvidia.com/page/uli_drivers.html>).
 
@@ -157,7 +157,7 @@ config ULI526X
 config PCMCIA_XIRCOM
 	tristate "Xircom CardBus support"
 	depends on CARDBUS
-	---help---
+	help
 	  This driver is for the Digital "Tulip" Ethernet CardBus adapters.
 	  It should work with most DEC 21*4*-based chips/ethercards, as well
 	  as with work-alike chips from Lite-On (PNIC) and Macronix (MXIC) and
diff --git a/drivers/net/ethernet/dlink/Kconfig b/drivers/net/ethernet/dlink/Kconfig
index 1362658a3030..0d77f84c8e7b 100644
--- a/drivers/net/ethernet/dlink/Kconfig
+++ b/drivers/net/ethernet/dlink/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_DLINK
 	bool "D-Link devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config DL2K
 	tristate "DL2000/TC902x/IP1000A-based Gigabit Ethernet support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver supports DL2000/TC902x/IP1000A-based Gigabit ethernet cards,
 	  which includes
 	  D-Link DGE-550T Gigabit Ethernet Adapter.
@@ -37,7 +37,7 @@ config SUNDANCE
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver is for the Sundance "Alta" chip.
 	  More specific information and updates are available from
 	  <http://www.scyld.com/network/sundance.html>.
@@ -45,7 +45,7 @@ config SUNDANCE
 config SUNDANCE_MMIO
 	bool "Use MMIO instead of PIO"
 	depends on SUNDANCE
-	---help---
+	help
 	  Enable memory-mapped I/O for interaction with Sundance NIC registers.
 	  Do NOT enable this by default, PIO (enabled when MMIO is disabled)
 	  is known to solve bugs on certain chips.
diff --git a/drivers/net/ethernet/emulex/Kconfig b/drivers/net/ethernet/emulex/Kconfig
index 22c143f2d787..5797a76dc731 100644
--- a/drivers/net/ethernet/emulex/Kconfig
+++ b/drivers/net/ethernet/emulex/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_EMULEX
 	bool "Emulex devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/ethernet/ezchip/Kconfig b/drivers/net/ethernet/ezchip/Kconfig
index 6db75fd2f9af..38aa824efb25 100644
--- a/drivers/net/ethernet/ezchip/Kconfig
+++ b/drivers/net/ethernet/ezchip/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_EZCHIP
 	bool "EZchip devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) device belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ config EZCHIP_NPS_MANAGEMENT_ENET
 	tristate "EZchip NPS management enet support"
 	depends on OF_IRQ && OF_NET
 	depends on HAS_IOMEM
-	---help---
+	help
 	  Simple LAN device for debug or management purposes.
 	  Device supports interrupts for RX and TX(completion).
 	  Device does not have DMA ability.
diff --git a/drivers/net/ethernet/faraday/Kconfig b/drivers/net/ethernet/faraday/Kconfig
index 73e4f2648e49..c2677ec0564d 100644
--- a/drivers/net/ethernet/faraday/Kconfig
+++ b/drivers/net/ethernet/faraday/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_FARADAY
 	bool "Faraday devices"
 	default y
 	depends on ARM || NDS32 || COMPILE_TEST
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config FTMAC100
 	depends on ARM || NDS32 || COMPILE_TEST
 	depends on !64BIT || BROKEN
 	select MII
-	---help---
+	help
 	  This driver supports the FTMAC100 10/100 Ethernet controller
 	  from Faraday. It is used on Faraday A320, Andes AG101 and some
 	  other ARM/NDS32 SoC's.
@@ -33,7 +33,7 @@ config FTGMAC100
 	depends on !64BIT || BROKEN
 	select PHYLIB
 	select MDIO_ASPEED if MACH_ASPEED_G6
-	---help---
+	help
 	  This driver supports the FTGMAC100 Gigabit Ethernet controller
 	  from Faraday. It is used on Faraday A369, Andes AG102 and some
 	  other ARM/NDS32 SoC's.
diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig
index bfc6bfe94d0a..a1d53ddf1593 100644
--- a/drivers/net/ethernet/freescale/Kconfig
+++ b/drivers/net/ethernet/freescale/Kconfig
@@ -10,7 +10,7 @@ config NET_VENDOR_FREESCALE
 		   M523x || M527x || M5272 || M528x || M520x || M532x || \
 		   ARCH_MXC || ARCH_MXS || (PPC_MPC52xx && PPC_BESTCOMM) || \
 		   ARCH_LAYERSCAPE || COMPILE_TEST
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -27,7 +27,7 @@ config FEC
 	default ARCH_MXC || SOC_IMX28 if ARM
 	select PHYLIB
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  Say Y here if you want to use the built-in 10/100 Fast ethernet
 	  controller on some Motorola ColdFire and Freescale i.MX processors.
 
@@ -37,7 +37,7 @@ config FEC_MPC52xx
 	select CRC32
 	select PHYLIB
 	select PPC_BESTCOMM_FEC
-	---help---
+	help
 	  This option enables support for the MPC5200's on-chip
 	  Fast Ethernet Controller
 	  If compiled as module, it will be called fec_mpc52xx.
@@ -46,7 +46,7 @@ config FEC_MPC52xx_MDIO
 	bool "FEC MPC52xx MDIO bus driver"
 	depends on FEC_MPC52xx
 	default y
-	---help---
+	help
 	  The MPC5200's FEC can connect to the Ethernet either with
 	  an external MII PHY chip or 10 Mbps 7-wire interface
 	  (Motorola? industry standard).
@@ -60,7 +60,7 @@ source "drivers/net/ethernet/freescale/fman/Kconfig"
 config FSL_PQ_MDIO
 	tristate "Freescale PQ MDIO"
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the MDIO bus used by the gianfar and UCC drivers.
 
 config FSL_XGMAC_MDIO
@@ -68,7 +68,7 @@ config FSL_XGMAC_MDIO
 	select PHYLIB
 	depends on OF
 	select OF_MDIO
-	---help---
+	help
 	  This driver supports the MDIO bus on the Fman 10G Ethernet MACs, and
 	  on the FMan mEMAC (which supports both Clauses 22 and 45)
 
@@ -78,7 +78,7 @@ config UCC_GETH
 	select FSL_PQ_MDIO
 	select PHYLIB
 	select FIXED_PHY
-	---help---
+	help
 	  This driver supports the Gigabit Ethernet mode of the QUICC Engine,
 	  which is available on some Freescale SOCs.
 
@@ -93,7 +93,7 @@ config GIANFAR
 	select PHYLIB
 	select FIXED_PHY
 	select CRC32
-	---help---
+	help
 	  This driver supports the Gigabit TSEC on the MPC83xx, MPC85xx,
 	  and MPC86xx family of chips, the eTSEC on LS1021A and the FEC
 	  on the 8540.
diff --git a/drivers/net/ethernet/freescale/dpaa/Kconfig b/drivers/net/ethernet/freescale/dpaa/Kconfig
index 0a54c7e0e4ae..626ec58a0afc 100644
--- a/drivers/net/ethernet/freescale/dpaa/Kconfig
+++ b/drivers/net/ethernet/freescale/dpaa/Kconfig
@@ -5,7 +5,7 @@ menuconfig FSL_DPAA_ETH
 	select PHYLIB
 	select FIXED_PHY
 	select FSL_FMAN_MAC
-	---help---
+	help
 	  Data Path Acceleration Architecture Ethernet driver,
 	  supporting the Freescale QorIQ chips.
 	  Depends on Freescale Buffer Manager and Queue Manager
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
index 8fb48de5d18c..f150cd454fa4 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
@@ -2907,8 +2907,9 @@ static int setup_dpni(struct fsl_mc_device *ls_dev)
 	if (err && err != -EOPNOTSUPP)
 		goto close;
 
-	priv->cls_rules = devm_kzalloc(dev, sizeof(struct dpaa2_eth_cls_rule) *
-				       dpaa2_eth_fs_count(priv), GFP_KERNEL);
+	priv->cls_rules = devm_kcalloc(dev, dpaa2_eth_fs_count(priv),
+				       sizeof(struct dpaa2_eth_cls_rule),
+				       GFP_KERNEL);
 	if (!priv->cls_rules) {
 		err = -ENOMEM;
 		goto close;
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
index ce85feaac357..b0d4b1984a70 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
@@ -40,9 +40,9 @@
 #include <linux/of_platform.h>
 #include <linux/of_gpio.h>
 #include <linux/of_net.h>
+#include <linux/pgtable.h>
 
 #include <linux/vmalloc.h>
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <linux/uaccess.h>
 
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
index 6e64989f8478..b47490be872c 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-fcc.c
@@ -35,12 +35,12 @@
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/gfp.h>
+#include <linux/pgtable.h>
 
 #include <asm/immap_cpm2.h>
 #include <asm/mpc8260.h>
 #include <asm/cpm2.h>
 
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <linux/uaccess.h>
 
diff --git a/drivers/net/ethernet/freescale/fs_enet/mii-fec.c b/drivers/net/ethernet/freescale/fs_enet/mii-fec.c
index 1582d82483ec..8b51ee142fa3 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mii-fec.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mii-fec.c
@@ -32,8 +32,8 @@
 #include <linux/platform_device.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
+#include <linux/pgtable.h>
 
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <linux/uaccess.h>
 #include <asm/mpc5xxx.h>
diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c
index 552e7554a9f8..db791f60b884 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.c
+++ b/drivers/net/ethernet/freescale/ucc_geth.c
@@ -42,7 +42,6 @@
 #include <soc/fsl/qe/ucc.h>
 #include <soc/fsl/qe/ucc_fast.h>
 #include <asm/machdep.h>
-#include <net/sch_generic.h>
 
 #include "ucc_geth.h"
 
diff --git a/drivers/net/ethernet/fujitsu/Kconfig b/drivers/net/ethernet/fujitsu/Kconfig
index cee99f20d2c2..0a1400cb410a 100644
--- a/drivers/net/ethernet/fujitsu/Kconfig
+++ b/drivers/net/ethernet/fujitsu/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_FUJITSU
 	bool "Fujitsu devices"
 	default y
 	depends on PCMCIA
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ config PCMCIA_FMVJ18X
 	tristate "Fujitsu FMV-J18x PCMCIA support"
 	depends on PCMCIA
 	select CRC32
-	---help---
+	help
 	  Say Y here if you intend to attach a Fujitsu FMV-J18x or compatible
 	  PCMCIA (PC-card) Ethernet card to your computer.
 
diff --git a/drivers/net/ethernet/hisilicon/Kconfig b/drivers/net/ethernet/hisilicon/Kconfig
index 2fff43509098..44f9279cdde1 100644
--- a/drivers/net/ethernet/hisilicon/Kconfig
+++ b/drivers/net/ethernet/hisilicon/Kconfig
@@ -8,7 +8,7 @@ config NET_VENDOR_HISILICON
 	default y
 	depends on OF || ACPI
 	depends on ARM || ARM64 || COMPILE_TEST
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -42,7 +42,7 @@ config HIP04_ETH
 	select MARVELL_PHY
 	select MFD_SYSCON
 	select HNS_MDIO
-	---help---
+	help
 	  If you wish to compile a kernel for a hardware with hisilicon p04 SoC and
 	  want to use the internal ethernet then you should answer Y to this.
 
@@ -59,13 +59,13 @@ config HI13X1_GMAC
 config HNS_MDIO
 	tristate
 	select PHYLIB
-	---help---
+	help
 	  This selects the HNS MDIO support. It is needed by HNS_DSAF to access
 	  the PHY
 
 config HNS
 	tristate
-	---help---
+	help
 	  This selects the framework support for Hisilicon Network Subsystem. It
 	  is needed by any driver which provides HNS acceleration engine or make
 	  use of the engine
@@ -74,7 +74,7 @@ config HNS_DSAF
 	tristate "Hisilicon HNS DSAF device Support"
 	select HNS
 	select HNS_MDIO
-	---help---
+	help
 	  This selects the DSAF (Distributed System Area Frabric) network
 	  acceleration engine support. The engine is used in Hisilicon hip05,
 	  Hi1610 and further ICT SoC
@@ -83,14 +83,14 @@ config HNS_ENET
 	tristate "Hisilicon HNS Ethernet Device Support"
 	select PHYLIB
 	select HNS
-	---help---
+	help
 	  This selects the general ethernet driver for HNS.  This module make
 	  use of any HNS AE driver, such as HNS_DSAF
 
 config HNS3
 	tristate "Hisilicon Network Subsystem Support HNS3 (Framework)"
 	depends on PCI
-	---help---
+	help
 	  This selects the framework support for Hisilicon Network Subsystem 3.
 	  This layer facilitates clients like ENET, RoCE and user-space ethernet
 	  drivers(like ODP)to register with HNAE devices and their associated
@@ -102,7 +102,7 @@ config HNS3_HCLGE
 	tristate "Hisilicon HNS3 HCLGE Acceleration Engine & Compatibility Layer Support"
 	default m
 	depends on PCI_MSI
-	---help---
+	help
 	  This selects the HNS3_HCLGE network acceleration engine & its hardware
 	  compatibility layer. The engine would be used in Hisilicon hip08 family of
 	  SoCs and further upcoming SoCs.
@@ -111,7 +111,7 @@ config HNS3_DCB
 	bool "Hisilicon HNS3 Data Center Bridge Support"
 	default n
 	depends on HNS3_HCLGE && DCB
-	---help---
+	help
 	  Say Y here if you want to use Data Center Bridging (DCB) in the HNS3 driver.
 
 	  If unsure, say N.
@@ -120,7 +120,7 @@ config HNS3_HCLGEVF
 	tristate "Hisilicon HNS3VF Acceleration Engine & Compatibility Layer Support"
 	depends on PCI_MSI
 	depends on HNS3_HCLGE
-    ---help---
+	help
 	  This selects the HNS3 VF drivers network acceleration engine & its hardware
 	  compatibility layer. The engine would be used in Hisilicon hip08 family of
 	  SoCs and further upcoming SoCs.
@@ -130,7 +130,7 @@ config HNS3_ENET
 	default m
 	depends on 64BIT && PCI
 	depends on INET
-	---help---
+	help
 	  This selects the Ethernet Driver for Hisilicon Network Subsystem 3 for hip08
 	  family of SoCs. This module depends upon HNAE3 driver to access the HNAE3
 	  devices and their associated operations.
diff --git a/drivers/net/ethernet/huawei/Kconfig b/drivers/net/ethernet/huawei/Kconfig
index bdcbface62d7..c05fce15eb51 100644
--- a/drivers/net/ethernet/huawei/Kconfig
+++ b/drivers/net/ethernet/huawei/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_HUAWEI
 	bool "Huawei devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 	  Note that the answer to this question doesn't directly affect the
 	  kernel: saying N will just cause the configurator to skip all
diff --git a/drivers/net/ethernet/huawei/hinic/Kconfig b/drivers/net/ethernet/huawei/hinic/Kconfig
index cabc2f72d9d7..936e2dd3bb13 100644
--- a/drivers/net/ethernet/huawei/hinic/Kconfig
+++ b/drivers/net/ethernet/huawei/hinic/Kconfig
@@ -6,7 +6,7 @@
 config HINIC
 	tristate "Huawei Intelligent PCIE Network Interface Card"
 	depends on (PCI_MSI && (X86 || ARM64))
-	---help---
+	help
 	  This driver supports HiNIC PCIE Ethernet cards.
 	  To compile this driver as part of the kernel, choose Y here.
 	  If unsure, choose N.
diff --git a/drivers/net/ethernet/i825xx/82596.c b/drivers/net/ethernet/i825xx/82596.c
index bef676d93339..fc8c7cd67471 100644
--- a/drivers/net/ethernet/i825xx/82596.c
+++ b/drivers/net/ethernet/i825xx/82596.c
@@ -53,10 +53,10 @@
 #include <linux/init.h>
 #include <linux/bitops.h>
 #include <linux/gfp.h>
+#include <linux/pgtable.h>
 
 #include <asm/io.h>
 #include <asm/dma.h>
-#include <asm/pgtable.h>
 #include <asm/cacheflush.h>
 
 static char version[] __initdata =
diff --git a/drivers/net/ethernet/i825xx/Kconfig b/drivers/net/ethernet/i825xx/Kconfig
index 33faff985438..3b5fab123824 100644
--- a/drivers/net/ethernet/i825xx/Kconfig
+++ b/drivers/net/ethernet/i825xx/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_I825XX
 	bool "Intel (82586/82593/82596) devices"
 	default y
 	depends on NET_VENDOR_INTEL
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question does not directly affect the
@@ -20,14 +20,14 @@ if NET_VENDOR_I825XX
 config ARM_ETHER1
 	tristate "Acorn Ether1 support"
 	depends on ARM && ARCH_ACORN
-	---help---
+	help
 	  If you have an Acorn system with one of these (AKA25) network cards,
 	  you should say Y to this option if you wish to use it with Linux.
 
 config BVME6000_NET
 	tristate "BVME6000 Ethernet support"
 	depends on BVME6000
-	---help---
+	help
 	  This is the driver for the Ethernet interface on BVME4000 and
 	  BVME6000 VME boards.  Say Y here to include the driver for this chip
 	  in your kernel.
@@ -36,14 +36,14 @@ config BVME6000_NET
 config LASI_82596
 	tristate "Lasi ethernet"
 	depends on GSC
-	---help---
+	help
 	  Say Y here to support the builtin Intel 82596 ethernet controller
 	  found in Hewlett-Packard PA-RISC machines with 10Mbit ethernet.
 
 config MVME16x_NET
 	tristate "MVME16x Ethernet support"
 	depends on MVME16x
-	---help---
+	help
 	  This is the driver for the Ethernet interface on the Motorola
 	  MVME162, 166, 167, 172 and 177 boards.  Say Y here to include the
 	  driver for this chip in your kernel.
@@ -52,14 +52,14 @@ config MVME16x_NET
 config SNI_82596
 	tristate "SNI RM ethernet"
 	depends on SNI_RM
-	---help---
+	help
 	  Say Y here to support the on-board Intel 82596 ethernet controller
 	  built into SNI RM machines.
 
 config SUN3_82586
 	bool "Sun3 on-board Intel 82586 support"
 	depends on SUN3
-	---help---
+	help
 	  This driver enables support for the on-board Intel 82586 based
 	  Ethernet adapter found on Sun 3/1xx and 3/2xx motherboards.  Note
 	  that this driver does not support 82586-based adapters on additional
diff --git a/drivers/net/ethernet/ibm/Kconfig b/drivers/net/ethernet/ibm/Kconfig
index a95d941360f8..c0c112d95b89 100644
--- a/drivers/net/ethernet/ibm/Kconfig
+++ b/drivers/net/ethernet/ibm/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_IBM
 	bool "IBM devices"
 	default y
 	depends on PPC_PSERIES || PPC_DCR || (IBMEBUS && SPARSEMEM)
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_IBM
 config IBMVETH
 	tristate "IBM LAN Virtual Ethernet support"
 	depends on PPC_PSERIES
-	---help---
+	help
 	  This driver supports virtual ethernet adapters on newer IBM iSeries
 	  and pSeries systems.
 
@@ -32,7 +32,7 @@ source "drivers/net/ethernet/ibm/emac/Kconfig"
 config EHEA
 	tristate "eHEA Ethernet support"
 	depends on IBMEBUS && SPARSEMEM
-	---help---
+	help
 	  This driver supports the IBM pSeries eHEA ethernet adapter.
 
 	  To compile the driver as a module, choose M here. The module
@@ -41,7 +41,7 @@ config EHEA
 config IBMVNIC
 	tristate "IBM Virtual NIC support"
 	depends on PPC_PSERIES
-	---help---
+	help
 	  This driver supports Virtual NIC adapters on IBM i and IBM System p
 	  systems.
 
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 197dc5b2c090..2baf7b3ff4cb 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -842,12 +842,13 @@ static int ibmvnic_login(struct net_device *netdev)
 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 	unsigned long timeout = msecs_to_jiffies(30000);
 	int retry_count = 0;
+	int retries = 10;
 	bool retry;
 	int rc;
 
 	do {
 		retry = false;
-		if (retry_count > IBMVNIC_MAX_QUEUES) {
+		if (retry_count > retries) {
 			netdev_warn(netdev, "Login attempts exceeded\n");
 			return -1;
 		}
@@ -862,11 +863,23 @@ static int ibmvnic_login(struct net_device *netdev)
 
 		if (!wait_for_completion_timeout(&adapter->init_done,
 						 timeout)) {
-			netdev_warn(netdev, "Login timed out\n");
-			return -1;
+			netdev_warn(netdev, "Login timed out, retrying...\n");
+			retry = true;
+			adapter->init_done_rc = 0;
+			retry_count++;
+			continue;
 		}
 
-		if (adapter->init_done_rc == PARTIALSUCCESS) {
+		if (adapter->init_done_rc == ABORTED) {
+			netdev_warn(netdev, "Login aborted, retrying...\n");
+			retry = true;
+			adapter->init_done_rc = 0;
+			retry_count++;
+			/* FW or device may be busy, so
+			 * wait a bit before retrying login
+			 */
+			msleep(500);
+		} else if (adapter->init_done_rc == PARTIALSUCCESS) {
 			retry_count++;
 			release_sub_crqs(adapter, 1);
 
@@ -5184,6 +5197,9 @@ static int ibmvnic_remove(struct vio_dev *dev)
 	adapter->state = VNIC_REMOVING;
 	spin_unlock_irqrestore(&adapter->state_lock, flags);
 
+	flush_work(&adapter->ibmvnic_reset);
+	flush_delayed_work(&adapter->ibmvnic_delayed_reset);
+
 	rtnl_lock();
 	unregister_netdevice(netdev);
 
diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig
index ad34e4335df2..48a8f9aa1dd0 100644
--- a/drivers/net/ethernet/intel/Kconfig
+++ b/drivers/net/ethernet/intel/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_INTEL
 	bool "Intel devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ config E100
 	tristate "Intel(R) PRO/100+ support"
 	depends on PCI
 	select MII
-	---help---
+	help
 	  This driver supports Intel(R) PRO/100 family of adapters.
 	  To verify that your adapter is supported, find the board ID number
 	  on the adapter. Look for a label that has a barcode and a number
@@ -42,7 +42,7 @@ config E100
 config E1000
 	tristate "Intel(R) PRO/1000 Gigabit Ethernet support"
 	depends on PCI
-	---help---
+	help
 	  This driver supports Intel(R) PRO/1000 gigabit ethernet family of
 	  adapters.  For more information on how to identify your adapter, go
 	  to the Adapter & Driver ID Guide that can be located at:
@@ -60,7 +60,7 @@ config E1000E
 	depends on PCI && (!SPARC32 || BROKEN)
 	select CRC32
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports the PCI-Express Intel(R) PRO/1000 gigabit
 	  ethernet family of adapters. For PCI or PCI-X e1000 adapters,
 	  use the regular e1000 driver For more information on how to
@@ -79,7 +79,7 @@ config E1000E_HWTS
 	bool "Support HW cross-timestamp on PCH devices"
 	default y
 	depends on E1000E && X86
-	---help---
+	help
 	 Say Y to enable hardware supported cross-timestamping on PCH
 	 devices. The cross-timestamp is available through the PTP clock
 	 driver precise cross-timestamp ioctl (PTP_SYS_OFFSET_PRECISE).
@@ -90,7 +90,7 @@ config IGB
 	imply PTP_1588_CLOCK
 	select I2C
 	select I2C_ALGOBIT
-	---help---
+	help
 	  This driver supports Intel(R) 82575/82576 gigabit ethernet family of
 	  adapters.  For more information on how to identify your adapter, go
 	  to the Adapter & Driver ID Guide that can be located at:
@@ -107,7 +107,7 @@ config IGB_HWMON
 	bool "Intel(R) PCI-Express Gigabit adapters HWMON support"
 	default y
 	depends on IGB && HWMON && !(IGB=y && HWMON=m)
-	---help---
+	help
 	  Say Y if you want to expose thermal sensor data on Intel devices.
 
 	  Some of our devices contain thermal sensors, both external and internal.
@@ -118,7 +118,7 @@ config IGB_DCA
 	bool "Direct Cache Access (DCA) Support"
 	default y
 	depends on IGB && DCA && !(IGB=y && DCA=m)
-	---help---
+	help
 	  Say Y here if you want to use Direct Cache Access (DCA) in the
 	  driver.  DCA is a method for warming the CPU cache before data
 	  is used, with the intent of lessening the impact of cache misses.
@@ -126,7 +126,7 @@ config IGB_DCA
 config IGBVF
 	tristate "Intel(R) 82576 Virtual Function Ethernet support"
 	depends on PCI
-	---help---
+	help
 	  This driver supports Intel(R) 82576 virtual functions.  For more
 	  information on how to identify your adapter, go to the Adapter &
 	  Driver ID Guide that can be located at:
@@ -142,7 +142,7 @@ config IGBVF
 config IXGB
 	tristate "Intel(R) PRO/10GbE support"
 	depends on PCI
-	---help---
+	help
 	  This driver supports Intel(R) PRO/10GbE family of adapters for
 	  PCI-X type cards. For PCI-E type cards, use the "ixgbe" driver
 	  instead. For more information on how to identify your adapter, go
@@ -162,7 +162,7 @@ config IXGBE
 	select MDIO
 	select PHYLIB
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports Intel(R) 10GbE PCI Express family of
 	  adapters.  For more information on how to identify your adapter, go
 	  to the Adapter & Driver ID Guide that can be located at:
@@ -179,7 +179,7 @@ config IXGBE_HWMON
 	bool "Intel(R) 10GbE PCI Express adapters HWMON support"
 	default y
 	depends on IXGBE && HWMON && !(IXGBE=y && HWMON=m)
-	---help---
+	help
 	  Say Y if you want to expose the thermal sensor data on some of
 	  our cards, via a hwmon sysfs interface.
 
@@ -187,7 +187,7 @@ config IXGBE_DCA
 	bool "Direct Cache Access (DCA) Support"
 	default y
 	depends on IXGBE && DCA && !(IXGBE=y && DCA=m)
-	---help---
+	help
 	  Say Y here if you want to use Direct Cache Access (DCA) in the
 	  driver.  DCA is a method for warming the CPU cache before data
 	  is used, with the intent of lessening the impact of cache misses.
@@ -196,7 +196,7 @@ config IXGBE_DCB
 	bool "Data Center Bridging (DCB) Support"
 	default n
 	depends on IXGBE && DCB
-	---help---
+	help
 	  Say Y here if you want to use Data Center Bridging (DCB) in the
 	  driver.
 
@@ -208,13 +208,13 @@ config IXGBE_IPSEC
 	depends on XFRM_OFFLOAD
 	default y
 	select XFRM_ALGO
-	---help---
+	help
 	  Enable support for IPSec offload in ixgbe.ko
 
 config IXGBEVF
 	tristate "Intel(R) 10GbE PCI Express Virtual Function Ethernet support"
 	depends on PCI_MSI
-	---help---
+	help
 	  This driver supports Intel(R) PCI Express virtual functions for the
 	  Intel(R) ixgbe driver.  For more information on how to identify your
 	  adapter, go to the Adapter & Driver ID Guide that can be located at:
@@ -234,14 +234,14 @@ config IXGBEVF_IPSEC
 	depends on XFRM_OFFLOAD
 	default y
 	select XFRM_ALGO
-	---help---
+	help
 	  Enable support for IPSec offload in ixgbevf.ko
 
 config I40E
 	tristate "Intel(R) Ethernet Controller XL710 Family support"
 	imply PTP_1588_CLOCK
 	depends on PCI
-	---help---
+	help
 	  This driver supports Intel(R) Ethernet Controller XL710 Family of
 	  devices.  For more information on how to identify your adapter, go
 	  to the Adapter & Driver ID Guide that can be located at:
@@ -258,7 +258,7 @@ config I40E_DCB
 	bool "Data Center Bridging (DCB) Support"
 	default n
 	depends on I40E && DCB
-	---help---
+	help
 	  Say Y here if you want to use Data Center Bridging (DCB) in the
 	  driver.
 
@@ -272,7 +272,7 @@ config I40EVF
 	tristate "Intel(R) Ethernet Adaptive Virtual Function support"
 	select IAVF
 	depends on PCI_MSI
-	---help---
+	help
 	  This driver supports virtual functions for Intel XL710,
 	  X710, X722, XXV710, and all devices advertising support for
 	  Intel Ethernet Adaptive Virtual Function devices. For more
@@ -295,7 +295,7 @@ config ICE
 	default n
 	depends on PCI_MSI
 	select NET_DEVLINK
-	---help---
+	help
 	  This driver supports Intel(R) Ethernet Connection E800 Series of
 	  devices.  For more information on how to identify your adapter, go
 	  to the Adapter & Driver ID Guide that can be located at:
@@ -313,7 +313,7 @@ config FM10K
 	default n
 	depends on PCI_MSI
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports Intel(R) FM10000 Ethernet Switch Host
 	  Interface.  For more information on how to identify your adapter,
 	  go to the Adapter & Driver ID Guide that can be located at:
@@ -330,7 +330,7 @@ config IGC
 	tristate "Intel(R) Ethernet Controller I225-LM/I225-V support"
 	default n
 	depends on PCI
-	---help---
+	help
 	  This driver supports Intel(R) Ethernet Controller I225-LM/I225-V
 	  family of adapters.
 
diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c
index d9fa4600f745..4b2de08137be 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_main.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_main.c
@@ -151,10 +151,8 @@ static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
 				  __be16 proto, u16 vid);
 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 
-#ifdef CONFIG_PM
-static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
-static int e1000_resume(struct pci_dev *pdev);
-#endif
+static int __maybe_unused e1000_suspend(struct device *dev);
+static int __maybe_unused e1000_resume(struct device *dev);
 static void e1000_shutdown(struct pci_dev *pdev);
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
@@ -179,16 +177,16 @@ static const struct pci_error_handlers e1000_err_handler = {
 	.resume = e1000_io_resume,
 };
 
+static SIMPLE_DEV_PM_OPS(e1000_pm_ops, e1000_suspend, e1000_resume);
+
 static struct pci_driver e1000_driver = {
 	.name     = e1000_driver_name,
 	.id_table = e1000_pci_tbl,
 	.probe    = e1000_probe,
 	.remove   = e1000_remove,
-#ifdef CONFIG_PM
-	/* Power Management Hooks */
-	.suspend  = e1000_suspend,
-	.resume   = e1000_resume,
-#endif
+	.driver = {
+		.pm = &e1000_pm_ops,
+	},
 	.shutdown = e1000_shutdown,
 	.err_handler = &e1000_err_handler
 };
@@ -5060,9 +5058,6 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
 	struct e1000_hw *hw = &adapter->hw;
 	u32 ctrl, ctrl_ext, rctl, status;
 	u32 wufc = adapter->wol;
-#ifdef CONFIG_PM
-	int retval = 0;
-#endif
 
 	netif_device_detach(netdev);
 
@@ -5076,12 +5071,6 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
 		e1000_down(adapter);
 	}
 
-#ifdef CONFIG_PM
-	retval = pci_save_state(pdev);
-	if (retval)
-		return retval;
-#endif
-
 	status = er32(STATUS);
 	if (status & E1000_STATUS_LU)
 		wufc &= ~E1000_WUFC_LNKC;
@@ -5142,37 +5131,26 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
 	return 0;
 }
 
-#ifdef CONFIG_PM
-static int e1000_suspend(struct pci_dev *pdev, pm_message_t state)
+static int __maybe_unused e1000_suspend(struct device *dev)
 {
 	int retval;
+	struct pci_dev *pdev = to_pci_dev(dev);
 	bool wake;
 
 	retval = __e1000_shutdown(pdev, &wake);
-	if (retval)
-		return retval;
-
-	if (wake) {
-		pci_prepare_to_sleep(pdev);
-	} else {
-		pci_wake_from_d3(pdev, false);
-		pci_set_power_state(pdev, PCI_D3hot);
-	}
+	device_set_wakeup_enable(dev, wake);
 
-	return 0;
+	return retval;
 }
 
-static int e1000_resume(struct pci_dev *pdev)
+static int __maybe_unused e1000_resume(struct device *dev)
 {
+	struct pci_dev *pdev = to_pci_dev(dev);
 	struct net_device *netdev = pci_get_drvdata(pdev);
 	struct e1000_adapter *adapter = netdev_priv(netdev);
 	struct e1000_hw *hw = &adapter->hw;
 	u32 err;
 
-	pci_set_power_state(pdev, PCI_D0);
-	pci_restore_state(pdev);
-	pci_save_state(pdev);
-
 	if (adapter->need_ioport)
 		err = pci_enable_device(pdev);
 	else
@@ -5209,7 +5187,6 @@ static int e1000_resume(struct pci_dev *pdev)
 
 	return 0;
 }
-#endif
 
 static void e1000_shutdown(struct pci_dev *pdev)
 {
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index a279f4fa9962..6f6479ca1267 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -6349,7 +6349,6 @@ fl_out:
 	pm_runtime_put_sync(netdev->dev.parent);
 }
 
-#ifdef CONFIG_PM_SLEEP
 /* S0ix implementation */
 static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter)
 {
@@ -6571,7 +6570,6 @@ static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter)
 	mac_data &= ~E1000_CTRL_EXT_FORCE_SMBUS;
 	ew32(CTRL_EXT, mac_data);
 }
-#endif /* CONFIG_PM_SLEEP */
 
 static int e1000e_pm_freeze(struct device *dev)
 {
@@ -6611,11 +6609,17 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool runtime)
 	struct net_device *netdev = pci_get_drvdata(pdev);
 	struct e1000_adapter *adapter = netdev_priv(netdev);
 	struct e1000_hw *hw = &adapter->hw;
-	u32 ctrl, ctrl_ext, rctl, status;
-	/* Runtime suspend should only enable wakeup for link changes */
-	u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
+	u32 ctrl, ctrl_ext, rctl, status, wufc;
 	int retval = 0;
 
+	/* Runtime suspend should only enable wakeup for link changes */
+	if (runtime)
+		wufc = E1000_WUFC_LNKC;
+	else if (device_may_wakeup(&pdev->dev))
+		wufc = adapter->wol;
+	else
+		wufc = 0;
+
 	status = er32(STATUS);
 	if (status & E1000_STATUS_LU)
 		wufc &= ~E1000_WUFC_LNKC;
@@ -6672,7 +6676,7 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool runtime)
 	if (adapter->hw.phy.type == e1000_phy_igp_3) {
 		e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
 	} else if (hw->mac.type >= e1000_pch_lpt) {
-		if (!(wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC)))
+		if (wufc && !(wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC)))
 			/* ULP does not support wake from unicast, multicast
 			 * or broadcast.
 			 */
@@ -6869,7 +6873,6 @@ err_irq:
 	return rc;
 }
 
-#ifdef CONFIG_PM
 static int __e1000_resume(struct pci_dev *pdev)
 {
 	struct net_device *netdev = pci_get_drvdata(pdev);
@@ -6935,8 +6938,7 @@ static int __e1000_resume(struct pci_dev *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int e1000e_pm_suspend(struct device *dev)
+static __maybe_unused int e1000e_pm_suspend(struct device *dev)
 {
 	struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev));
 	struct e1000_adapter *adapter = netdev_priv(netdev);
@@ -6960,7 +6962,7 @@ static int e1000e_pm_suspend(struct device *dev)
 	return rc;
 }
 
-static int e1000e_pm_resume(struct device *dev)
+static __maybe_unused int e1000e_pm_resume(struct device *dev)
 {
 	struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev));
 	struct e1000_adapter *adapter = netdev_priv(netdev);
@@ -6979,9 +6981,8 @@ static int e1000e_pm_resume(struct device *dev)
 
 	return e1000e_pm_thaw(dev);
 }
-#endif /* CONFIG_PM_SLEEP */
 
-static int e1000e_pm_runtime_idle(struct device *dev)
+static __maybe_unused int e1000e_pm_runtime_idle(struct device *dev)
 {
 	struct net_device *netdev = dev_get_drvdata(dev);
 	struct e1000_adapter *adapter = netdev_priv(netdev);
@@ -6997,7 +6998,7 @@ static int e1000e_pm_runtime_idle(struct device *dev)
 	return -EBUSY;
 }
 
-static int e1000e_pm_runtime_resume(struct device *dev)
+static __maybe_unused int e1000e_pm_runtime_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct net_device *netdev = pci_get_drvdata(pdev);
@@ -7014,7 +7015,7 @@ static int e1000e_pm_runtime_resume(struct device *dev)
 	return rc;
 }
 
-static int e1000e_pm_runtime_suspend(struct device *dev)
+static __maybe_unused int e1000e_pm_runtime_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct net_device *netdev = pci_get_drvdata(pdev);
@@ -7039,7 +7040,6 @@ static int e1000e_pm_runtime_suspend(struct device *dev)
 
 	return 0;
 }
-#endif /* CONFIG_PM */
 
 static void e1000_shutdown(struct pci_dev *pdev)
 {
diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h
index bcd11b4b29df..10b805ba03ee 100644
--- a/drivers/net/ethernet/intel/iavf/iavf.h
+++ b/drivers/net/ethernet/intel/iavf/iavf.h
@@ -87,6 +87,10 @@ struct iavf_vsi {
 #define IAVF_HLUT_ARRAY_SIZE ((IAVF_VFQF_HLUT_MAX_INDEX + 1) * 4)
 #define IAVF_MBPS_DIVISOR	125000 /* divisor to convert to Mbps */
 
+#define IAVF_VIRTCHNL_VF_RESOURCE_SIZE (sizeof(struct virtchnl_vf_resource) + \
+					(IAVF_MAX_VF_VSI * \
+					 sizeof(struct virtchnl_vsi_resource)))
+
 /* MAX_MSIX_Q_VECTORS of these are allocated,
  * but we only use one per queue-specific vector.
  */
@@ -215,6 +219,10 @@ struct iavf_cloud_filter {
 	bool add;		/* filter needs to be added */
 };
 
+#define IAVF_RESET_WAIT_MS 10
+#define IAVF_RESET_WAIT_DETECTED_COUNT 500
+#define IAVF_RESET_WAIT_COMPLETE_COUNT 2000
+
 /* board specific private data structure */
 struct iavf_adapter {
 	struct work_struct reset_task;
@@ -306,6 +314,14 @@ struct iavf_adapter {
 	bool netdev_registered;
 	bool link_up;
 	enum virtchnl_link_speed link_speed;
+	/* This is only populated if the VIRTCHNL_VF_CAP_ADV_LINK_SPEED is set
+	 * in vf_res->vf_cap_flags. Use ADV_LINK_SUPPORT macro to determine if
+	 * this field is valid. This field should be used going forward and the
+	 * enum virtchnl_link_speed above should be considered the legacy way of
+	 * storing/communicating link speeds.
+	 */
+	u32 link_speed_mbps;
+
 	enum virtchnl_ops current_op;
 #define CLIENT_ALLOWED(_a) ((_a)->vf_res ? \
 			    (_a)->vf_res->vf_cap_flags & \
@@ -322,6 +338,8 @@ struct iavf_adapter {
 			VIRTCHNL_VF_OFFLOAD_RSS_PF)))
 #define VLAN_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \
 			  VIRTCHNL_VF_OFFLOAD_VLAN)
+#define ADV_LINK_SUPPORT(_a) ((_a)->vf_res->vf_cap_flags & \
+			      VIRTCHNL_VF_CAP_ADV_LINK_SPEED)
 	struct virtchnl_vf_resource *vf_res; /* incl. all VSIs */
 	struct virtchnl_vsi_resource *vsi_res; /* our LAN VSI */
 	struct virtchnl_version_info pf_version;
diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
index 2c39d46b6138..181573822942 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
@@ -278,35 +278,46 @@ static int iavf_get_link_ksettings(struct net_device *netdev,
 	ethtool_link_ksettings_zero_link_mode(cmd, supported);
 	cmd->base.autoneg = AUTONEG_DISABLE;
 	cmd->base.port = PORT_NONE;
-	/* Set speed and duplex */
+	cmd->base.duplex = DUPLEX_FULL;
+
+	if (ADV_LINK_SUPPORT(adapter)) {
+		if (adapter->link_speed_mbps &&
+		    adapter->link_speed_mbps < U32_MAX)
+			cmd->base.speed = adapter->link_speed_mbps;
+		else
+			cmd->base.speed = SPEED_UNKNOWN;
+
+		return 0;
+	}
+
 	switch (adapter->link_speed) {
-	case IAVF_LINK_SPEED_40GB:
+	case VIRTCHNL_LINK_SPEED_40GB:
 		cmd->base.speed = SPEED_40000;
 		break;
-	case IAVF_LINK_SPEED_25GB:
-#ifdef SPEED_25000
+	case VIRTCHNL_LINK_SPEED_25GB:
 		cmd->base.speed = SPEED_25000;
-#else
-		netdev_info(netdev,
-			    "Speed is 25G, display not supported by this version of ethtool.\n");
-#endif
 		break;
-	case IAVF_LINK_SPEED_20GB:
+	case VIRTCHNL_LINK_SPEED_20GB:
 		cmd->base.speed = SPEED_20000;
 		break;
-	case IAVF_LINK_SPEED_10GB:
+	case VIRTCHNL_LINK_SPEED_10GB:
 		cmd->base.speed = SPEED_10000;
 		break;
-	case IAVF_LINK_SPEED_1GB:
+	case VIRTCHNL_LINK_SPEED_5GB:
+		cmd->base.speed = SPEED_5000;
+		break;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		cmd->base.speed = SPEED_2500;
+		break;
+	case VIRTCHNL_LINK_SPEED_1GB:
 		cmd->base.speed = SPEED_1000;
 		break;
-	case IAVF_LINK_SPEED_100MB:
+	case VIRTCHNL_LINK_SPEED_100MB:
 		cmd->base.speed = SPEED_100;
 		break;
 	default:
 		break;
 	}
-	cmd->base.duplex = DUPLEX_FULL;
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
index 2050649848ba..fa82768e5eda 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -1756,17 +1756,17 @@ static int iavf_init_get_resources(struct iavf_adapter *adapter)
 	struct net_device *netdev = adapter->netdev;
 	struct pci_dev *pdev = adapter->pdev;
 	struct iavf_hw *hw = &adapter->hw;
-	int err = 0, bufsz;
+	int err;
 
 	WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
 	/* aq msg sent, awaiting reply */
 	if (!adapter->vf_res) {
-		bufsz = sizeof(struct virtchnl_vf_resource) +
-			(IAVF_MAX_VF_VSI *
-			sizeof(struct virtchnl_vsi_resource));
-		adapter->vf_res = kzalloc(bufsz, GFP_KERNEL);
-		if (!adapter->vf_res)
+		adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
+					  GFP_KERNEL);
+		if (!adapter->vf_res) {
+			err = -ENOMEM;
 			goto err;
+		}
 	}
 	err = iavf_get_vf_config(adapter);
 	if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK) {
@@ -2036,7 +2036,7 @@ static void iavf_disable_vf(struct iavf_adapter *adapter)
 	iavf_reset_interrupt_capability(adapter);
 	iavf_free_queues(adapter);
 	iavf_free_q_vectors(adapter);
-	kfree(adapter->vf_res);
+	memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
 	iavf_shutdown_adminq(&adapter->hw);
 	adapter->netdev->flags &= ~IFF_UP;
 	clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section);
@@ -2046,8 +2046,6 @@ static void iavf_disable_vf(struct iavf_adapter *adapter)
 	dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
 }
 
-#define IAVF_RESET_WAIT_MS 10
-#define IAVF_RESET_WAIT_COUNT 500
 /**
  * iavf_reset_task - Call-back task to handle hardware reset
  * @work: pointer to work_struct
@@ -2101,20 +2099,20 @@ static void iavf_reset_task(struct work_struct *work)
 	adapter->flags |= IAVF_FLAG_RESET_PENDING;
 
 	/* poll until we see the reset actually happen */
-	for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) {
+	for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
 		reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
 			  IAVF_VF_ARQLEN1_ARQENABLE_MASK;
 		if (!reg_val)
 			break;
 		usleep_range(5000, 10000);
 	}
-	if (i == IAVF_RESET_WAIT_COUNT) {
+	if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
 		dev_info(&adapter->pdev->dev, "Never saw reset\n");
 		goto continue_reset; /* act like the reset happened */
 	}
 
 	/* wait until the reset is complete and the PF is responding to us */
-	for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) {
+	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
 		/* sleep first to make sure a minimum wait time is met */
 		msleep(IAVF_RESET_WAIT_MS);
 
@@ -2126,7 +2124,7 @@ static void iavf_reset_task(struct work_struct *work)
 
 	pci_set_master(adapter->pdev);
 
-	if (i == IAVF_RESET_WAIT_COUNT) {
+	if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
 		dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
 			reg_val);
 		iavf_disable_vf(adapter);
@@ -2487,29 +2485,46 @@ static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
 {
 	int speed = 0, ret = 0;
 
+	if (ADV_LINK_SUPPORT(adapter)) {
+		if (adapter->link_speed_mbps < U32_MAX) {
+			speed = adapter->link_speed_mbps;
+			goto validate_bw;
+		} else {
+			dev_err(&adapter->pdev->dev, "Unknown link speed\n");
+			return -EINVAL;
+		}
+	}
+
 	switch (adapter->link_speed) {
-	case IAVF_LINK_SPEED_40GB:
-		speed = 40000;
+	case VIRTCHNL_LINK_SPEED_40GB:
+		speed = SPEED_40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		speed = SPEED_25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		speed = SPEED_20000;
 		break;
-	case IAVF_LINK_SPEED_25GB:
-		speed = 25000;
+	case VIRTCHNL_LINK_SPEED_10GB:
+		speed = SPEED_10000;
 		break;
-	case IAVF_LINK_SPEED_20GB:
-		speed = 20000;
+	case VIRTCHNL_LINK_SPEED_5GB:
+		speed = SPEED_5000;
 		break;
-	case IAVF_LINK_SPEED_10GB:
-		speed = 10000;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		speed = SPEED_2500;
 		break;
-	case IAVF_LINK_SPEED_1GB:
-		speed = 1000;
+	case VIRTCHNL_LINK_SPEED_1GB:
+		speed = SPEED_1000;
 		break;
-	case IAVF_LINK_SPEED_100MB:
-		speed = 100;
+	case VIRTCHNL_LINK_SPEED_100MB:
+		speed = SPEED_100;
 		break;
 	default:
 		break;
 	}
 
+validate_bw:
 	if (max_tx_rate > speed) {
 		dev_err(&adapter->pdev->dev,
 			"Invalid tx rate specified\n");
@@ -3412,7 +3427,7 @@ static int iavf_check_reset_complete(struct iavf_hw *hw)
 	u32 rstat;
 	int i;
 
-	for (i = 0; i < 100; i++) {
+	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
 		rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
 			     IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
 		if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
index 7a30d5d5ef53..e091bab7e770 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c
@@ -379,19 +379,19 @@ static inline unsigned int iavf_itr_divisor(struct iavf_q_vector *q_vector)
 	unsigned int divisor;
 
 	switch (q_vector->adapter->link_speed) {
-	case IAVF_LINK_SPEED_40GB:
+	case VIRTCHNL_LINK_SPEED_40GB:
 		divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 1024;
 		break;
-	case IAVF_LINK_SPEED_25GB:
-	case IAVF_LINK_SPEED_20GB:
+	case VIRTCHNL_LINK_SPEED_25GB:
+	case VIRTCHNL_LINK_SPEED_20GB:
 		divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 512;
 		break;
 	default:
-	case IAVF_LINK_SPEED_10GB:
+	case VIRTCHNL_LINK_SPEED_10GB:
 		divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 256;
 		break;
-	case IAVF_LINK_SPEED_1GB:
-	case IAVF_LINK_SPEED_100MB:
+	case VIRTCHNL_LINK_SPEED_1GB:
+	case VIRTCHNL_LINK_SPEED_100MB:
 		divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 32;
 		break;
 	}
diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
index d58374c2c33d..ed08ace4f05a 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c
@@ -139,7 +139,8 @@ int iavf_send_vf_config_msg(struct iavf_adapter *adapter)
 	       VIRTCHNL_VF_OFFLOAD_ENCAP |
 	       VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM |
 	       VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
-	       VIRTCHNL_VF_OFFLOAD_ADQ;
+	       VIRTCHNL_VF_OFFLOAD_ADQ |
+	       VIRTCHNL_VF_CAP_ADV_LINK_SPEED;
 
 	adapter->current_op = VIRTCHNL_OP_GET_VF_RESOURCES;
 	adapter->aq_required &= ~IAVF_FLAG_AQ_GET_CONFIG;
@@ -891,6 +892,8 @@ void iavf_disable_vlan_stripping(struct iavf_adapter *adapter)
 	iavf_send_pf_msg(adapter, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, NULL, 0);
 }
 
+#define IAVF_MAX_SPEED_STRLEN	13
+
 /**
  * iavf_print_link_message - print link up or down
  * @adapter: adapter structure
@@ -900,37 +903,105 @@ void iavf_disable_vlan_stripping(struct iavf_adapter *adapter)
 static void iavf_print_link_message(struct iavf_adapter *adapter)
 {
 	struct net_device *netdev = adapter->netdev;
-	char *speed = "Unknown ";
+	int link_speed_mbps;
+	char *speed;
 
 	if (!adapter->link_up) {
 		netdev_info(netdev, "NIC Link is Down\n");
 		return;
 	}
 
+	speed = kcalloc(1, IAVF_MAX_SPEED_STRLEN, GFP_KERNEL);
+	if (!speed)
+		return;
+
+	if (ADV_LINK_SUPPORT(adapter)) {
+		link_speed_mbps = adapter->link_speed_mbps;
+		goto print_link_msg;
+	}
+
 	switch (adapter->link_speed) {
-	case IAVF_LINK_SPEED_40GB:
-		speed = "40 G";
+	case VIRTCHNL_LINK_SPEED_40GB:
+		link_speed_mbps = SPEED_40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		link_speed_mbps = SPEED_25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		link_speed_mbps = SPEED_20000;
 		break;
-	case IAVF_LINK_SPEED_25GB:
-		speed = "25 G";
+	case VIRTCHNL_LINK_SPEED_10GB:
+		link_speed_mbps = SPEED_10000;
 		break;
-	case IAVF_LINK_SPEED_20GB:
-		speed = "20 G";
+	case VIRTCHNL_LINK_SPEED_5GB:
+		link_speed_mbps = SPEED_5000;
 		break;
-	case IAVF_LINK_SPEED_10GB:
-		speed = "10 G";
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		link_speed_mbps = SPEED_2500;
 		break;
-	case IAVF_LINK_SPEED_1GB:
-		speed = "1000 M";
+	case VIRTCHNL_LINK_SPEED_1GB:
+		link_speed_mbps = SPEED_1000;
 		break;
-	case IAVF_LINK_SPEED_100MB:
-		speed = "100 M";
+	case VIRTCHNL_LINK_SPEED_100MB:
+		link_speed_mbps = SPEED_100;
 		break;
 	default:
+		link_speed_mbps = SPEED_UNKNOWN;
 		break;
 	}
 
-	netdev_info(netdev, "NIC Link is Up %sbps Full Duplex\n", speed);
+print_link_msg:
+	if (link_speed_mbps > SPEED_1000) {
+		if (link_speed_mbps == SPEED_2500)
+			snprintf(speed, IAVF_MAX_SPEED_STRLEN, "2.5 Gbps");
+		else
+			/* convert to Gbps inline */
+			snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%d %s",
+				 link_speed_mbps / 1000, "Gbps");
+	} else if (link_speed_mbps == SPEED_UNKNOWN) {
+		snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%s", "Unknown Mbps");
+	} else {
+		snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%u %s",
+			 link_speed_mbps, "Mbps");
+	}
+
+	netdev_info(netdev, "NIC Link is Up Speed is %s Full Duplex\n", speed);
+	kfree(speed);
+}
+
+/**
+ * iavf_get_vpe_link_status
+ * @adapter: adapter structure
+ * @vpe: virtchnl_pf_event structure
+ *
+ * Helper function for determining the link status
+ **/
+static bool
+iavf_get_vpe_link_status(struct iavf_adapter *adapter,
+			 struct virtchnl_pf_event *vpe)
+{
+	if (ADV_LINK_SUPPORT(adapter))
+		return vpe->event_data.link_event_adv.link_status;
+	else
+		return vpe->event_data.link_event.link_status;
+}
+
+/**
+ * iavf_set_adapter_link_speed_from_vpe
+ * @adapter: adapter structure for which we are setting the link speed
+ * @vpe: virtchnl_pf_event structure that contains the link speed we are setting
+ *
+ * Helper function for setting iavf_adapter link speed
+ **/
+static void
+iavf_set_adapter_link_speed_from_vpe(struct iavf_adapter *adapter,
+				     struct virtchnl_pf_event *vpe)
+{
+	if (ADV_LINK_SUPPORT(adapter))
+		adapter->link_speed_mbps =
+			vpe->event_data.link_event_adv.link_speed;
+	else
+		adapter->link_speed = vpe->event_data.link_event.link_speed;
 }
 
 /**
@@ -1160,12 +1231,11 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter,
 	if (v_opcode == VIRTCHNL_OP_EVENT) {
 		struct virtchnl_pf_event *vpe =
 			(struct virtchnl_pf_event *)msg;
-		bool link_up = vpe->event_data.link_event.link_status;
+		bool link_up = iavf_get_vpe_link_status(adapter, vpe);
 
 		switch (vpe->event) {
 		case VIRTCHNL_EVENT_LINK_CHANGE:
-			adapter->link_speed =
-				vpe->event_data.link_event.link_speed;
+			iavf_set_adapter_link_speed_from_vpe(adapter, vpe);
 
 			/* we've already got the right link status, bail */
 			if (adapter->link_up == link_up)
diff --git a/drivers/net/ethernet/korina.c b/drivers/net/ethernet/korina.c
index f1d84921e42b..03e034918d14 100644
--- a/drivers/net/ethernet/korina.c
+++ b/drivers/net/ethernet/korina.c
@@ -53,10 +53,10 @@
 #include <linux/mii.h>
 #include <linux/ethtool.h>
 #include <linux/crc32.h>
+#include <linux/pgtable.h>
 
 #include <asm/bootinfo.h>
 #include <asm/bitops.h>
-#include <asm/pgtable.h>
 #include <asm/io.h>
 #include <asm/dma.h>
 
diff --git a/drivers/net/ethernet/marvell/Kconfig b/drivers/net/ethernet/marvell/Kconfig
index 3d5caea096fb..cd8ddd1ef6f2 100644
--- a/drivers/net/ethernet/marvell/Kconfig
+++ b/drivers/net/ethernet/marvell/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_MARVELL
 	bool "Marvell devices"
 	default y
 	depends on PCI || CPU_PXA168 || MV64X60 || PPC32 || PLAT_ORION || INET || COMPILE_TEST
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config MV643XX_ETH
 	depends on INET
 	select PHYLIB
 	select MVMDIO
-	---help---
+	help
 	  This driver supports the gigabit ethernet MACs in the
 	  Marvell Discovery PPC/MIPS chipset family (MV643XX) and
 	  in the Marvell Orion ARM SoC family.
@@ -35,7 +35,7 @@ config MVMDIO
 	tristate "Marvell MDIO interface support"
 	depends on HAS_IOMEM
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the MDIO interface found in the network
 	  interface units of the Marvell EBU SoCs (Kirkwood, Orion5x,
 	  Dove, Armada 370 and Armada XP).
@@ -46,7 +46,7 @@ config MVNETA_BM_ENABLE
 	tristate "Marvell Armada 38x/XP network interface BM support"
 	depends on MVNETA
 	depends on !64BIT
-	---help---
+	help
 	  This driver supports auxiliary block of the network
 	  interface units in the Marvell ARMADA XP and ARMADA 38x SoC
 	  family, which is called buffer manager.
@@ -62,7 +62,7 @@ config MVNETA
 	select MVMDIO
 	select PHYLINK
 	select PAGE_POOL
-	---help---
+	help
 	  This driver supports the network interface units in the
 	  Marvell ARMADA XP, ARMADA 370, ARMADA 38x and
 	  ARMADA 37xx SoC family.
@@ -87,7 +87,7 @@ config MVPP2
 	depends on ARCH_MVEBU || COMPILE_TEST
 	select MVMDIO
 	select PHYLINK
-	---help---
+	help
 	  This driver supports the network interface units in the
 	  Marvell ARMADA 375, 7K and 8K SoCs.
 
@@ -96,7 +96,7 @@ config PXA168_ETH
 	depends on HAS_IOMEM
 	depends on CPU_PXA168 || ARCH_BERLIN || COMPILE_TEST
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the pxa168 Ethernet ports.
 
 	  To compile this driver as a module, choose M here. The module
@@ -106,7 +106,7 @@ config SKGE
 	tristate "Marvell Yukon Gigabit Ethernet support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver support the Marvell Yukon or SysKonnect SK-98xx/SK-95xx
 	  and related Gigabit Ethernet adapters. It is a new smaller driver
 	  with better performance and more complete ethtool support.
@@ -127,7 +127,7 @@ config SKGE
 config SKGE_DEBUG
 	bool "Debugging interface"
 	depends on SKGE && DEBUG_FS
-	---help---
+	help
 	  This option adds the ability to dump driver state for debugging.
 	  The file /sys/kernel/debug/skge/ethX displays the state of the internal
 	  transmit and receive rings.
@@ -137,7 +137,7 @@ config SKGE_DEBUG
 config SKGE_GENESIS
 	bool "Support for older SysKonnect Genesis boards"
 	depends on SKGE
-	---help---
+	help
 	 This enables support for the older and uncommon SysKonnect Genesis
 	 chips, which support MII via an external transceiver, instead of
 	 an internal one. Disabling this option will save some memory
@@ -147,7 +147,7 @@ config SKY2
 	tristate "Marvell Yukon 2 support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver supports Gigabit Ethernet adapters based on the
 	  Marvell Yukon 2 chipset:
 	  Marvell 88E8021/88E8022/88E8035/88E8036/88E8038/88E8050/88E8052/
@@ -162,7 +162,7 @@ config SKY2
 config SKY2_DEBUG
 	bool "Debugging interface"
 	depends on SKY2 && DEBUG_FS
-	---help---
+	help
 	  This option adds the ability to dump driver state for debugging.
 	  The file /sys/kernel/debug/sky2/ethX displays the state of the internal
 	  transmit and receive rings.
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index 011cd26953d9..946925bbcb2d 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -325,7 +325,7 @@
 	      cache_line_size())
 
 /* Driver assumes that the last 3 bits are 0 */
-#define MVNETA_SKB_HEADROOM	(max(XDP_PACKET_HEADROOM, NET_SKB_PAD) & ~0x7)
+#define MVNETA_SKB_HEADROOM	ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8)
 #define MVNETA_SKB_PAD	(SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
 			 MVNETA_SKB_HEADROOM))
 #define MVNETA_SKB_SIZE(len)	(SKB_DATA_ALIGN(len) + MVNETA_SKB_PAD)
@@ -452,11 +452,17 @@ struct mvneta_pcpu_port {
 	u32			cause_rx_tx;
 };
 
+enum {
+	__MVNETA_DOWN,
+};
+
 struct mvneta_port {
 	u8 id;
 	struct mvneta_pcpu_port __percpu	*ports;
 	struct mvneta_pcpu_stats __percpu	*stats;
 
+	unsigned long state;
+
 	int pkt_size;
 	void __iomem *base;
 	struct mvneta_rx_queue *rxqs;
@@ -2113,6 +2119,9 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame,
 	struct netdev_queue *nq;
 	u32 ret;
 
+	if (unlikely(test_bit(__MVNETA_DOWN, &pp->state)))
+		return -ENETDOWN;
+
 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 		return -EINVAL;
 
@@ -3568,12 +3577,16 @@ static void mvneta_start_dev(struct mvneta_port *pp)
 
 	phylink_start(pp->phylink);
 	netif_tx_start_all_queues(pp->dev);
+
+	clear_bit(__MVNETA_DOWN, &pp->state);
 }
 
 static void mvneta_stop_dev(struct mvneta_port *pp)
 {
 	unsigned int cpu;
 
+	set_bit(__MVNETA_DOWN, &pp->state);
+
 	phylink_stop(pp->phylink);
 
 	if (!pp->neta_armada3700) {
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
index 2b5dad2ec650..24f4d8e0da98 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
@@ -1544,7 +1544,7 @@ static void mvpp2_read_stats(struct mvpp2_port *port)
 	for (q = 0; q < port->ntxqs; q++)
 		for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_txq_regs); i++)
 			*pstats++ += mvpp2_read_index(port->priv,
-						      MVPP22_CTRS_TX_CTR(port->id, i),
+						      MVPP22_CTRS_TX_CTR(port->id, q),
 						      mvpp2_ethtool_txq_regs[i].offset);
 
 	/* Rxqs are numbered from 0 from the user standpoint, but not from the
@@ -1553,7 +1553,7 @@ static void mvpp2_read_stats(struct mvpp2_port *port)
 	for (q = 0; q < port->nrxqs; q++)
 		for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_rxq_regs); i++)
 			*pstats++ += mvpp2_read_index(port->priv,
-						      port->first_rxq + i,
+						      port->first_rxq + q,
 						      mvpp2_ethtool_rxq_regs[i].offset);
 }
 
@@ -5983,8 +5983,8 @@ static int mvpp2_remove(struct platform_device *pdev)
 {
 	struct mvpp2 *priv = platform_get_drvdata(pdev);
 	struct fwnode_handle *fwnode = pdev->dev.fwnode;
+	int i = 0, poolnum = MVPP2_BM_POOLS_NUM;
 	struct fwnode_handle *port_fwnode;
-	int i = 0;
 
 	mvpp2_dbgfs_cleanup(priv);
 
@@ -5998,7 +5998,10 @@ static int mvpp2_remove(struct platform_device *pdev)
 
 	destroy_workqueue(priv->stats_queue);
 
-	for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
+	if (priv->percpu_pools)
+		poolnum = mvpp2_get_nrxqs(priv) * 2;
+
+	for (i = 0; i < poolnum; i++) {
 		struct mvpp2_bm_pool *bm_pool = &priv->bm_pools[i];
 
 		mvpp2_bm_pool_destroy(&pdev->dev, priv, bm_pool);
diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig
index d9dfb614daa6..543a1d047567 100644
--- a/drivers/net/ethernet/marvell/octeontx2/Kconfig
+++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig
@@ -21,7 +21,7 @@ config NDC_DIS_DYNAMIC_CACHING
 	bool "Disable caching of dynamic entries in NDC"
 	depends on OCTEONTX2_AF
 	default n
-	---help---
+	help
 	  This config option disables caching of dynamic entries such as NIX SQEs
 	  , NPA stack pages etc in NDC. Also locks down NIX SQ/CQ/RQ/RSS and
 	  NPA Aura/Pool contexts.
diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c
index 17243bb5ba91..eb8cf60ecf12 100644
--- a/drivers/net/ethernet/marvell/pxa168_eth.c
+++ b/drivers/net/ethernet/marvell/pxa168_eth.c
@@ -31,8 +31,8 @@
 #include <linux/types.h>
 #include <linux/udp.h>
 #include <linux/workqueue.h>
+#include <linux/pgtable.h>
 
-#include <asm/pgtable.h>
 #include <asm/cacheflush.h>
 
 #define DRIVER_NAME	"pxa168-eth"
diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig
index 500c15e7ea4a..62a820b1eb16 100644
--- a/drivers/net/ethernet/mediatek/Kconfig
+++ b/drivers/net/ethernet/mediatek/Kconfig
@@ -2,7 +2,7 @@
 config NET_VENDOR_MEDIATEK
 	bool "MediaTek devices"
 	depends on ARCH_MEDIATEK || SOC_MT7621 || SOC_MT7620
-	---help---
+	help
 	  If you have a Mediatek SoC with ethernet, say Y.
 
 if NET_VENDOR_MEDIATEK
@@ -10,7 +10,7 @@ if NET_VENDOR_MEDIATEK
 config NET_MEDIATEK_SOC
 	tristate "MediaTek SoC Gigabit Ethernet support"
 	select PHYLINK
-	---help---
+	help
 	  This driver supports the gigabit ethernet MACs in the
 	  MediaTek SoC family.
 
diff --git a/drivers/net/ethernet/mediatek/mtk_star_emac.c b/drivers/net/ethernet/mediatek/mtk_star_emac.c
index f1ace4fec19f..3e765bdcf9e1 100644
--- a/drivers/net/ethernet/mediatek/mtk_star_emac.c
+++ b/drivers/net/ethernet/mediatek/mtk_star_emac.c
@@ -24,7 +24,6 @@
 #include <linux/regmap.h>
 #include <linux/skbuff.h>
 #include <linux/spinlock.h>
-#include <linux/workqueue.h>
 
 #define MTK_STAR_DRVNAME			"mtk_star_emac"
 
@@ -262,7 +261,6 @@ struct mtk_star_priv {
 	spinlock_t lock;
 
 	struct rtnl_link_stats64 stats;
-	struct work_struct stats_work;
 };
 
 static struct device *mtk_star_get_dev(struct mtk_star_priv *priv)
@@ -432,42 +430,6 @@ static void mtk_star_intr_disable(struct mtk_star_priv *priv)
 	regmap_write(priv->regs, MTK_STAR_REG_INT_MASK, ~0);
 }
 
-static void mtk_star_intr_enable_tx(struct mtk_star_priv *priv)
-{
-	regmap_clear_bits(priv->regs, MTK_STAR_REG_INT_MASK,
-			  MTK_STAR_BIT_INT_STS_TNTC);
-}
-
-static void mtk_star_intr_enable_rx(struct mtk_star_priv *priv)
-{
-	regmap_clear_bits(priv->regs, MTK_STAR_REG_INT_MASK,
-			  MTK_STAR_BIT_INT_STS_FNRC);
-}
-
-static void mtk_star_intr_enable_stats(struct mtk_star_priv *priv)
-{
-	regmap_clear_bits(priv->regs, MTK_STAR_REG_INT_MASK,
-			  MTK_STAR_REG_INT_STS_MIB_CNT_TH);
-}
-
-static void mtk_star_intr_disable_tx(struct mtk_star_priv *priv)
-{
-	regmap_set_bits(priv->regs, MTK_STAR_REG_INT_MASK,
-			MTK_STAR_BIT_INT_STS_TNTC);
-}
-
-static void mtk_star_intr_disable_rx(struct mtk_star_priv *priv)
-{
-	regmap_set_bits(priv->regs, MTK_STAR_REG_INT_MASK,
-			MTK_STAR_BIT_INT_STS_FNRC);
-}
-
-static void mtk_star_intr_disable_stats(struct mtk_star_priv *priv)
-{
-	regmap_set_bits(priv->regs, MTK_STAR_REG_INT_MASK,
-			MTK_STAR_REG_INT_STS_MIB_CNT_TH);
-}
-
 static unsigned int mtk_star_intr_read(struct mtk_star_priv *priv)
 {
 	unsigned int val;
@@ -663,20 +625,6 @@ static void mtk_star_update_stats(struct mtk_star_priv *priv)
 	stats->rx_errors += stats->rx_fifo_errors;
 }
 
-/* This runs in process context and parallel TX and RX paths executing in
- * napi context may result in losing some stats data but this should happen
- * seldom enough to be acceptable.
- */
-static void mtk_star_update_stats_work(struct work_struct *work)
-{
-	struct mtk_star_priv *priv = container_of(work, struct mtk_star_priv,
-						 stats_work);
-
-	mtk_star_update_stats(priv);
-	mtk_star_reset_counters(priv);
-	mtk_star_intr_enable_stats(priv);
-}
-
 static struct sk_buff *mtk_star_alloc_skb(struct net_device *ndev)
 {
 	uintptr_t tail, offset;
@@ -767,42 +715,25 @@ static void mtk_star_free_tx_skbs(struct mtk_star_priv *priv)
 	mtk_star_ring_free_skbs(priv, ring, mtk_star_dma_unmap_tx);
 }
 
-/* All processing for TX and RX happens in the napi poll callback. */
+/* All processing for TX and RX happens in the napi poll callback.
+ *
+ * FIXME: The interrupt handling should be more fine-grained with each
+ * interrupt enabled/disabled independently when needed. Unfortunatly this
+ * turned out to impact the driver's stability and until we have something
+ * working properly, we're disabling all interrupts during TX & RX processing
+ * or when resetting the counter registers.
+ */
 static irqreturn_t mtk_star_handle_irq(int irq, void *data)
 {
 	struct mtk_star_priv *priv;
 	struct net_device *ndev;
-	bool need_napi = false;
-	unsigned int status;
 
 	ndev = data;
 	priv = netdev_priv(ndev);
 
 	if (netif_running(ndev)) {
-		status = mtk_star_intr_read(priv);
-
-		if (status & MTK_STAR_BIT_INT_STS_TNTC) {
-			mtk_star_intr_disable_tx(priv);
-			need_napi = true;
-		}
-
-		if (status & MTK_STAR_BIT_INT_STS_FNRC) {
-			mtk_star_intr_disable_rx(priv);
-			need_napi = true;
-		}
-
-		if (need_napi)
-			napi_schedule(&priv->napi);
-
-		/* One of the counters reached 0x8000000 - update stats and
-		 * reset all counters.
-		 */
-		if (unlikely(status & MTK_STAR_REG_INT_STS_MIB_CNT_TH)) {
-			mtk_star_intr_disable_stats(priv);
-			schedule_work(&priv->stats_work);
-		}
-
-		mtk_star_intr_ack_all(priv);
+		mtk_star_intr_disable(priv);
+		napi_schedule(&priv->napi);
 	}
 
 	return IRQ_HANDLED;
@@ -1169,8 +1100,6 @@ static void mtk_star_tx_complete_all(struct mtk_star_priv *priv)
 	if (wake && netif_queue_stopped(ndev))
 		netif_wake_queue(ndev);
 
-	mtk_star_intr_enable_tx(priv);
-
 	spin_unlock(&priv->lock);
 }
 
@@ -1332,20 +1261,32 @@ static int mtk_star_process_rx(struct mtk_star_priv *priv, int budget)
 static int mtk_star_poll(struct napi_struct *napi, int budget)
 {
 	struct mtk_star_priv *priv;
+	unsigned int status;
 	int received = 0;
 
 	priv = container_of(napi, struct mtk_star_priv, napi);
 
-	/* Clean-up all TX descriptors. */
-	mtk_star_tx_complete_all(priv);
-	/* Receive up to $budget packets. */
-	received = mtk_star_process_rx(priv, budget);
+	status = mtk_star_intr_read(priv);
+	mtk_star_intr_ack_all(priv);
 
-	if (received < budget) {
-		napi_complete_done(napi, received);
-		mtk_star_intr_enable_rx(priv);
+	if (status & MTK_STAR_BIT_INT_STS_TNTC)
+		/* Clean-up all TX descriptors. */
+		mtk_star_tx_complete_all(priv);
+
+	if (status & MTK_STAR_BIT_INT_STS_FNRC)
+		/* Receive up to $budget packets. */
+		received = mtk_star_process_rx(priv, budget);
+
+	if (unlikely(status & MTK_STAR_REG_INT_STS_MIB_CNT_TH)) {
+		mtk_star_update_stats(priv);
+		mtk_star_reset_counters(priv);
 	}
 
+	if (received < budget)
+		napi_complete_done(napi, received);
+
+	mtk_star_intr_enable(priv);
+
 	return received;
 }
 
@@ -1532,7 +1473,6 @@ static int mtk_star_probe(struct platform_device *pdev)
 	ndev->max_mtu = MTK_STAR_MAX_FRAME_SIZE;
 
 	spin_lock_init(&priv->lock);
-	INIT_WORK(&priv->stats_work, mtk_star_update_stats_work);
 
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
diff --git a/drivers/net/ethernet/mellanox/Kconfig b/drivers/net/ethernet/mellanox/Kconfig
index 23cf7917a0c9..ff6613a5cdd3 100644
--- a/drivers/net/ethernet/mellanox/Kconfig
+++ b/drivers/net/ethernet/mellanox/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_MELLANOX
 	bool "Mellanox devices"
 	default y
 	depends on PCI || I2C
-	---help---
+	help
 	  If you have a network (Ethernet or RDMA) device belonging to this
 	  class, say Y.
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/Kconfig b/drivers/net/ethernet/mellanox/mlx4/Kconfig
index e69c3c31e701..400e611ba041 100644
--- a/drivers/net/ethernet/mellanox/mlx4/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx4/Kconfig
@@ -8,7 +8,7 @@ config MLX4_EN
 	depends on PCI && NETDEVICES && ETHERNET && INET
 	select MLX4_CORE
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports Mellanox Technologies ConnectX Ethernet
 	  devices.
 
@@ -16,7 +16,7 @@ config MLX4_EN_DCB
 	bool "Data Center Bridging (DCB) Support"
 	default y
 	depends on MLX4_EN && DCB
-	---help---
+	help
 	  Say Y here if you want to use Data Center Bridging (DCB) in the
 	  driver.
 	  If set to N, will not be able to configure QoS and ratelimit attributes.
@@ -34,7 +34,7 @@ config MLX4_DEBUG
 	bool "Verbose debugging output" if (MLX4_CORE && EXPERT)
 	depends on MLX4_CORE
 	default y
-	---help---
+	help
 	  This option causes debugging code to be compiled into the
 	  mlx4_core driver.  The output can be turned on via the
 	  debug_level module parameter (which can also be set after
@@ -44,6 +44,6 @@ config MLX4_CORE_GEN2
 	bool "Support for old gen2 Mellanox PCI IDs" if (MLX4_CORE)
 	depends on MLX4_CORE
 	default y
-	---help---
+	help
 	  Say Y here if you want to use old gen2 Mellanox devices in the
 	  driver.
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index c72c4e1ea383..3d9aa7da95e9 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -2345,8 +2345,6 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
 			goto out_free;
 		}
 
-		dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1;
-
 		if (enable_4k_uar || !dev->persist->num_vfs) {
 			init_hca->log_uar_sz = ilog2(dev->caps.num_uars) +
 						    PAGE_SHIFT - DEFAULT_UAR_PAGE_SHIFT;
diff --git a/drivers/net/ethernet/mellanox/mlx4/mr.c b/drivers/net/ethernet/mellanox/mlx4/mr.c
index 1a11bc0e1612..d2986f1f2db0 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mr.c
+++ b/drivers/net/ethernet/mellanox/mlx4/mr.c
@@ -966,189 +966,6 @@ void mlx4_cleanup_mr_table(struct mlx4_dev *dev)
 	mlx4_bitmap_cleanup(&mr_table->mpt_bitmap);
 }
 
-static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list,
-				  int npages, u64 iova)
-{
-	int i, page_mask;
-
-	if (npages > fmr->max_pages)
-		return -EINVAL;
-
-	page_mask = (1 << fmr->page_shift) - 1;
-
-	/* We are getting page lists, so va must be page aligned. */
-	if (iova & page_mask)
-		return -EINVAL;
-
-	/* Trust the user not to pass misaligned data in page_list */
-	if (0)
-		for (i = 0; i < npages; ++i) {
-			if (page_list[i] & ~page_mask)
-				return -EINVAL;
-		}
-
-	if (fmr->maps >= fmr->max_maps)
-		return -EINVAL;
-
-	return 0;
-}
-
-int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,
-		      int npages, u64 iova, u32 *lkey, u32 *rkey)
-{
-	u32 key;
-	int i, err;
-
-	err = mlx4_check_fmr(fmr, page_list, npages, iova);
-	if (err)
-		return err;
-
-	++fmr->maps;
-
-	key = key_to_hw_index(fmr->mr.key);
-	key += dev->caps.num_mpts;
-	*lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
-
-	*(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
-
-	/* Make sure MPT status is visible before writing MTT entries */
-	wmb();
-
-	dma_sync_single_for_cpu(&dev->persist->pdev->dev, fmr->dma_handle,
-				npages * sizeof(u64), DMA_TO_DEVICE);
-
-	for (i = 0; i < npages; ++i)
-		fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT);
-
-	dma_sync_single_for_device(&dev->persist->pdev->dev, fmr->dma_handle,
-				   npages * sizeof(u64), DMA_TO_DEVICE);
-
-	fmr->mpt->key    = cpu_to_be32(key);
-	fmr->mpt->lkey   = cpu_to_be32(key);
-	fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift));
-	fmr->mpt->start  = cpu_to_be64(iova);
-
-	/* Make MTT entries are visible before setting MPT status */
-	wmb();
-
-	*(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW;
-
-	/* Make sure MPT status is visible before consumer can use FMR */
-	wmb();
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr);
-
-int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
-		   int max_maps, u8 page_shift, struct mlx4_fmr *fmr)
-{
-	struct mlx4_priv *priv = mlx4_priv(dev);
-	int err = -ENOMEM;
-
-	if (max_maps > dev->caps.max_fmr_maps)
-		return -EINVAL;
-
-	if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32)
-		return -EINVAL;
-
-	/* All MTTs must fit in the same page */
-	if (max_pages * sizeof(*fmr->mtts) > PAGE_SIZE)
-		return -EINVAL;
-
-	fmr->page_shift = page_shift;
-	fmr->max_pages  = max_pages;
-	fmr->max_maps   = max_maps;
-	fmr->maps = 0;
-
-	err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages,
-			    page_shift, &fmr->mr);
-	if (err)
-		return err;
-
-	fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table,
-				    fmr->mr.mtt.offset,
-				    &fmr->dma_handle);
-
-	if (!fmr->mtts) {
-		err = -ENOMEM;
-		goto err_free;
-	}
-
-	return 0;
-
-err_free:
-	(void) mlx4_mr_free(dev, &fmr->mr);
-	return err;
-}
-EXPORT_SYMBOL_GPL(mlx4_fmr_alloc);
-
-int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
-{
-	struct mlx4_priv *priv = mlx4_priv(dev);
-	int err;
-
-	err = mlx4_mr_enable(dev, &fmr->mr);
-	if (err)
-		return err;
-
-	fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table,
-				    key_to_hw_index(fmr->mr.key), NULL);
-	if (!fmr->mpt)
-		return -ENOMEM;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(mlx4_fmr_enable);
-
-void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
-		    u32 *lkey, u32 *rkey)
-{
-	if (!fmr->maps)
-		return;
-
-	/* To unmap: it is sufficient to take back ownership from HW */
-	*(u8 *)fmr->mpt = MLX4_MPT_STATUS_SW;
-
-	/* Make sure MPT status is visible */
-	wmb();
-
-	fmr->maps = 0;
-}
-EXPORT_SYMBOL_GPL(mlx4_fmr_unmap);
-
-int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
-{
-	int ret;
-
-	if (fmr->maps)
-		return -EBUSY;
-	if (fmr->mr.enabled == MLX4_MPT_EN_HW) {
-		/* In case of FMR was enabled and unmapped
-		 * make sure to give ownership of MPT back to HW
-		 * so HW2SW_MPT command will success.
-		 */
-		*(u8 *)fmr->mpt = MLX4_MPT_STATUS_SW;
-		/* Make sure MPT status is visible before changing MPT fields */
-		wmb();
-		fmr->mpt->length = 0;
-		fmr->mpt->start  = 0;
-		/* Make sure MPT data is visible after changing MPT status */
-		wmb();
-		*(u8 *)fmr->mpt = MLX4_MPT_STATUS_HW;
-		/* make sure MPT status is visible */
-		wmb();
-	}
-
-	ret = mlx4_mr_free(dev, &fmr->mr);
-	if (ret)
-		return ret;
-	fmr->mr.enabled = MLX4_MPT_DISABLED;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(mlx4_fmr_free);
-
 int mlx4_SYNC_TPT(struct mlx4_dev *dev)
 {
 	return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index b6ffd1622cfd..4dfdbb82ea9d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -12,7 +12,7 @@ config MLX5_CORE
 	depends on PTP_1588_CLOCK || !PTP_1588_CLOCK
 	depends on PCI_HYPERV_INTERFACE || !PCI_HYPERV_INTERFACE
 	default n
-	---help---
+	help
 	  Core driver for low level functionality of the ConnectX-4 and
 	  Connect-IB cards by Mellanox Technologies.
 
@@ -23,7 +23,7 @@ config MLX5_FPGA
 	bool "Mellanox Technologies Innova support"
 	depends on MLX5_CORE
 	select MLX5_ACCEL
-	---help---
+	help
 	  Build support for the Innova family of network cards by Mellanox
 	  Technologies. Innova network cards are comprised of a ConnectX chip
 	  and an FPGA chip on one board. If you select this option, the
@@ -36,14 +36,14 @@ config MLX5_CORE_EN
 	select PAGE_POOL
 	select DIMLIB
 	default n
-	---help---
+	help
 	  Ethernet support in Mellanox Technologies ConnectX-4 NIC.
 
 config MLX5_EN_ARFS
 	bool "Mellanox MLX5 ethernet accelerated receive flow steering (ARFS) support"
 	depends on MLX5_CORE_EN && RFS_ACCEL
 	default y
-	---help---
+	help
 	  Mellanox MLX5 ethernet hardware-accelerated receive flow steering support,
 	  Enables ethernet netdevice arfs support and ntuple filtering.
 
@@ -51,7 +51,7 @@ config MLX5_EN_RXNFC
 	bool "Mellanox MLX5 ethernet rx nfc flow steering support"
 	depends on MLX5_CORE_EN
 	default y
-	---help---
+	help
 	  Mellanox MLX5 ethernet rx nfc flow steering support
 	  Enables ethtool receive network flow classification, which allows user defined
 	  flow rules to direct traffic into arbitrary rx queue via ethtool set/get_rxnfc
@@ -61,7 +61,7 @@ config MLX5_MPFS
 	bool "Mellanox Technologies MLX5 MPFS support"
 	depends on MLX5_CORE_EN
 	default y
-	---help---
+	help
 	  Mellanox Technologies Ethernet Multi-Physical Function Switch (MPFS)
 	  support in ConnectX NIC. MPFs is required for when multi-PF configuration
 	  is enabled to allow passing user configured unicast MAC addresses to the
@@ -71,7 +71,7 @@ config MLX5_ESWITCH
 	bool "Mellanox Technologies MLX5 SRIOV E-Switch support"
 	depends on MLX5_CORE_EN && NET_SWITCHDEV
 	default y
-	---help---
+	help
 	  Mellanox Technologies Ethernet SRIOV E-Switch support in ConnectX NIC.
 	  E-Switch provides internal SRIOV packet steering and switching for the
 	  enabled VFs and PF in two available modes:
@@ -107,7 +107,7 @@ config MLX5_CORE_EN_DCB
 	bool "Data Center Bridging (DCB) Support"
 	default y
 	depends on MLX5_CORE_EN && DCB
-	---help---
+	help
 	  Say Y here if you want to use Data Center Bridging (DCB) in the
 	  driver.
 	  If set to N, will not be able to configure QoS and ratelimit attributes.
@@ -119,7 +119,7 @@ config MLX5_CORE_IPOIB
 	bool "Mellanox 5th generation network adapters (connectX series) IPoIB offloads support"
 	depends on MLX5_CORE_EN
 	default n
-	---help---
+	help
 	  MLX5 IPoIB offloads & acceleration support.
 
 config MLX5_FPGA_IPSEC
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index e94f0c4d74a7..a99fe4b02b9b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -283,7 +283,6 @@ int mlx5_devlink_register(struct devlink *devlink, struct device *dev)
 		goto params_reg_err;
 	mlx5_devlink_set_params_init_values(devlink);
 	devlink_params_publish(devlink);
-	devlink_reload_enable(devlink);
 	return 0;
 
 params_reg_err:
@@ -293,7 +292,6 @@ params_reg_err:
 
 void mlx5_devlink_unregister(struct devlink *devlink)
 {
-	devlink_reload_disable(devlink);
 	devlink_params_unregister(devlink, mlx5_devlink_params,
 				  ARRAY_SIZE(mlx5_devlink_params));
 	devlink_unregister(devlink);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
index afc19dca1f5f..430025550fad 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
@@ -328,21 +328,21 @@ mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act,
 
 	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 		MLX5_SET(set_action_in, modact, length, 0);
-		if (offset == offsetof(struct ipv6hdr, saddr))
+		if (offset == offsetof(struct ipv6hdr, saddr) + 12)
 			field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0;
-		else if (offset == offsetof(struct ipv6hdr, saddr) + 4)
-			field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32;
 		else if (offset == offsetof(struct ipv6hdr, saddr) + 8)
+			field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32;
+		else if (offset == offsetof(struct ipv6hdr, saddr) + 4)
 			field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64;
-		else if (offset == offsetof(struct ipv6hdr, saddr) + 12)
+		else if (offset == offsetof(struct ipv6hdr, saddr))
 			field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96;
-		else if (offset == offsetof(struct ipv6hdr, daddr))
+		else if (offset == offsetof(struct ipv6hdr, daddr) + 12)
 			field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0;
-		else if (offset == offsetof(struct ipv6hdr, daddr) + 4)
-			field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32;
 		else if (offset == offsetof(struct ipv6hdr, daddr) + 8)
+			field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32;
+		else if (offset == offsetof(struct ipv6hdr, daddr) + 4)
 			field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64;
-		else if (offset == offsetof(struct ipv6hdr, daddr) + 12)
+		else if (offset == offsetof(struct ipv6hdr, daddr))
 			field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96;
 		else
 			return -EOPNOTSUPP;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
index c28cbae42331..2c80205dc939 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
@@ -152,6 +152,10 @@ void mlx5e_close_xsk(struct mlx5e_channel *c)
 	mlx5e_close_cq(&c->xskicosq.cq);
 	mlx5e_close_xdpsq(&c->xsksq);
 	mlx5e_close_cq(&c->xsksq.cq);
+
+	memset(&c->xskrq, 0, sizeof(c->xskrq));
+	memset(&c->xsksq, 0, sizeof(c->xsksq));
+	memset(&c->xskicosq, 0, sizeof(c->xskicosq));
 }
 
 void mlx5e_activate_xsk(struct mlx5e_channel *c)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 3ef2525e8de9..ec5658bbe3c5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -1173,7 +1173,8 @@ int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir,
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5e_rss_params *rss = &priv->rss_params;
 	int inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
-	bool hash_changed = false;
+	bool refresh_tirs = false;
+	bool refresh_rqt = false;
 	void *in;
 
 	if ((hfunc != ETH_RSS_HASH_NO_CHANGE) &&
@@ -1189,36 +1190,38 @@ int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir,
 
 	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != rss->hfunc) {
 		rss->hfunc = hfunc;
-		hash_changed = true;
+		refresh_rqt = true;
+		refresh_tirs = true;
 	}
 
 	if (indir) {
 		memcpy(rss->indirection_rqt, indir,
 		       sizeof(rss->indirection_rqt));
-
-		if (test_bit(MLX5E_STATE_OPENED, &priv->state)) {
-			u32 rqtn = priv->indir_rqt.rqtn;
-			struct mlx5e_redirect_rqt_param rrp = {
-				.is_rss = true,
-				{
-					.rss = {
-						.hfunc = rss->hfunc,
-						.channels  = &priv->channels,
-					},
-				},
-			};
-
-			mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
-		}
+		refresh_rqt = true;
 	}
 
 	if (key) {
 		memcpy(rss->toeplitz_hash_key, key,
 		       sizeof(rss->toeplitz_hash_key));
-		hash_changed = hash_changed || rss->hfunc == ETH_RSS_HASH_TOP;
+		refresh_tirs = refresh_tirs || rss->hfunc == ETH_RSS_HASH_TOP;
+	}
+
+	if (refresh_rqt && test_bit(MLX5E_STATE_OPENED, &priv->state)) {
+		struct mlx5e_redirect_rqt_param rrp = {
+			.is_rss = true,
+			{
+				.rss = {
+					.hfunc = rss->hfunc,
+					.channels  = &priv->channels,
+				},
+			},
+		};
+		u32 rqtn = priv->indir_rqt.rqtn;
+
+		mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
 	}
 
-	if (hash_changed)
+	if (refresh_tirs)
 		mlx5e_modify_tirs_hash(priv, in);
 
 	mutex_unlock(&priv->state_lock);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c
index 9bda4fe2eafa..5dc335e621c5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/ingress_lgcy.c
@@ -162,10 +162,12 @@ int esw_acl_ingress_lgcy_setup(struct mlx5_eswitch *esw,
 
 	if (MLX5_CAP_ESW_INGRESS_ACL(esw->dev, flow_counter)) {
 		counter = mlx5_fc_create(esw->dev, false);
-		if (IS_ERR(counter))
+		if (IS_ERR(counter)) {
 			esw_warn(esw->dev,
 				 "vport[%d] configure ingress drop rule counter failed\n",
 				 vport->vport);
+			counter = NULL;
+		}
 		vport->ingress.legacy.drop_counter = counter;
 	}
 
@@ -272,7 +274,7 @@ void esw_acl_ingress_lgcy_cleanup(struct mlx5_eswitch *esw,
 	esw_acl_ingress_table_destroy(vport);
 
 clean_drop_counter:
-	if (!IS_ERR_OR_NULL(vport->ingress.legacy.drop_counter)) {
+	if (vport->ingress.legacy.drop_counter) {
 		mlx5_fc_destroy(esw->dev, vport->ingress.legacy.drop_counter);
 		vport->ingress.legacy.drop_counter = NULL;
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index c0cfbab15fe9..b31f769d2df9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -192,15 +192,23 @@ static bool reset_fw_if_needed(struct mlx5_core_dev *dev)
 
 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
 {
+	bool err_detected = false;
+
+	/* Mark the device as fatal in order to abort FW commands */
+	if ((check_fatal_sensors(dev) || force) &&
+	    dev->state == MLX5_DEVICE_STATE_UP) {
+		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
+		err_detected = true;
+	}
 	mutex_lock(&dev->intf_state_mutex);
-	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
-		goto unlock;
+	if (!err_detected && dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
+		goto unlock;/* a previous error is still being handled */
 	if (dev->state == MLX5_DEVICE_STATE_UNINITIALIZED) {
 		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
 		goto unlock;
 	}
 
-	if (check_fatal_sensors(dev) || force) {
+	if (check_fatal_sensors(dev) || force) { /* protected state setting */
 		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
 		mlx5_cmd_flush(dev);
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index df46b1fce3a7..8b658908f044 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -785,6 +785,11 @@ err_disable:
 
 static void mlx5_pci_close(struct mlx5_core_dev *dev)
 {
+	/* health work might still be active, and it needs pci bar in
+	 * order to know the NIC state. Therefore, drain the health WQ
+	 * before removing the pci bars
+	 */
+	mlx5_drain_health_wq(dev);
 	iounmap(dev->iseg);
 	pci_clear_master(dev->pdev);
 	release_bar(dev->pdev);
@@ -1194,23 +1199,22 @@ int mlx5_load_one(struct mlx5_core_dev *dev, bool boot)
 	if (err)
 		goto err_load;
 
+	set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
+
 	if (boot) {
 		err = mlx5_devlink_register(priv_to_devlink(dev), dev->device);
 		if (err)
 			goto err_devlink_reg;
-	}
-
-	if (mlx5_device_registered(dev))
-		mlx5_attach_device(dev);
-	else
 		mlx5_register_device(dev);
-
-	set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
+	} else {
+		mlx5_attach_device(dev);
+	}
 
 	mutex_unlock(&dev->intf_state_mutex);
 	return 0;
 
 err_devlink_reg:
+	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
 	mlx5_unload(dev);
 err_load:
 	if (boot)
@@ -1226,10 +1230,15 @@ out:
 
 void mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup)
 {
-	if (cleanup)
+	mutex_lock(&dev->intf_state_mutex);
+
+	if (cleanup) {
 		mlx5_unregister_device(dev);
+		mlx5_devlink_unregister(priv_to_devlink(dev));
+	} else {
+		mlx5_detach_device(dev);
+	}
 
-	mutex_lock(&dev->intf_state_mutex);
 	if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
 		mlx5_core_warn(dev, "%s: interface is down, NOP\n",
 			       __func__);
@@ -1240,9 +1249,6 @@ void mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup)
 
 	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
 
-	if (mlx5_device_registered(dev))
-		mlx5_detach_device(dev);
-
 	mlx5_unload(dev);
 
 	if (cleanup)
@@ -1275,11 +1281,6 @@ static int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
 
 	priv->dbg_root = debugfs_create_dir(dev_name(dev->device),
 					    mlx5_debugfs_root);
-	if (!priv->dbg_root) {
-		dev_err(dev->device, "mlx5_core: error, Cannot create debugfs dir, aborting\n");
-		goto err_dbg_root;
-	}
-
 	err = mlx5_health_init(dev);
 	if (err)
 		goto err_health_init;
@@ -1294,7 +1295,6 @@ err_pagealloc_init:
 	mlx5_health_cleanup(dev);
 err_health_init:
 	debugfs_remove(dev->priv.dbg_root);
-err_dbg_root:
 	mutex_destroy(&priv->pgdir_mutex);
 	mutex_destroy(&priv->alloc_mutex);
 	mutex_destroy(&priv->bfregs.wc_head.lock);
@@ -1362,6 +1362,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		dev_err(&pdev->dev, "mlx5_crdump_enable failed with error code %d\n", err);
 
 	pci_save_state(pdev);
+	devlink_reload_enable(devlink);
 	return 0;
 
 err_load_one:
@@ -1379,9 +1380,8 @@ static void remove_one(struct pci_dev *pdev)
 	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
 	struct devlink *devlink = priv_to_devlink(dev);
 
+	devlink_reload_disable(devlink);
 	mlx5_crdump_disable(dev);
-	mlx5_devlink_unregister(devlink);
-
 	mlx5_drain_health_wq(dev);
 	mlx5_unload_one(dev, true);
 	mlx5_pci_close(dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c
index f421013b0b54..2ca79b9bde1f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_send.c
@@ -179,7 +179,7 @@ static struct mlx5dr_qp *dr_create_rc_qp(struct mlx5_core_dev *mdev,
 	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
 	err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
 	dr_qp->qpn = MLX5_GET(create_qp_out, out, qpn);
-	kfree(in);
+	kvfree(in);
 	if (err)
 		goto err_in;
 	dr_qp->uar = attr->uar;
diff --git a/drivers/net/ethernet/mellanox/mlxfw/Kconfig b/drivers/net/ethernet/mellanox/mlxfw/Kconfig
index 5b604501f33e..c339f3c46e45 100644
--- a/drivers/net/ethernet/mellanox/mlxfw/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlxfw/Kconfig
@@ -5,7 +5,7 @@
 
 config MLXFW
 	tristate "Mellanox Technologies firmware flash module"
-	---help---
+	help
 	  This driver supports Mellanox Technologies Firmware
 	  flashing common logic.
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/Kconfig b/drivers/net/ethernet/mellanox/mlxsw/Kconfig
index f458fd1ce9f8..872e9910bb7c 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlxsw/Kconfig
@@ -6,7 +6,7 @@
 config MLXSW_CORE
 	tristate "Mellanox Technologies Switch ASICs support"
 	select NET_DEVLINK
-	---help---
+	help
 	  This driver supports Mellanox Technologies Switch ASICs family.
 
 	  To compile this driver as a module, choose M here: the
@@ -17,14 +17,14 @@ config MLXSW_CORE_HWMON
 	depends on MLXSW_CORE && HWMON
 	depends on !(MLXSW_CORE=y && HWMON=m)
 	default y
-	---help---
+	help
 	  Say Y here if you want to expose HWMON interface on mlxsw devices.
 
 config MLXSW_CORE_THERMAL
 	bool "Thermal zone support for Mellanox Technologies Switch ASICs"
 	depends on MLXSW_CORE && THERMAL
 	default y
-	---help---
+	help
 	 Say Y here if you want to automatically control fans speed according
 	 ambient temperature reported by ASIC.
 
@@ -32,7 +32,7 @@ config MLXSW_PCI
 	tristate "PCI bus implementation for Mellanox Technologies Switch ASICs"
 	depends on PCI && HAS_IOMEM && MLXSW_CORE
 	default m
-	---help---
+	help
 	  This is PCI bus implementation for Mellanox Technologies Switch ASICs.
 
 	  To compile this driver as a module, choose M here: the
@@ -42,7 +42,7 @@ config MLXSW_I2C
 	tristate "I2C bus implementation for Mellanox Technologies Switch ASICs"
 	depends on I2C && MLXSW_CORE
 	default m
-	---help---
+	help
 	  This is I2C bus implementation for Mellanox Technologies Switch ASICs.
 
 	  To compile this driver as a module, choose M here: the
@@ -52,7 +52,7 @@ config MLXSW_SWITCHIB
 	tristate "Mellanox Technologies SwitchIB and SwitchIB-2 support"
 	depends on MLXSW_CORE && MLXSW_PCI && NET_SWITCHDEV
 	default m
-	---help---
+	help
 	  This driver supports Mellanox Technologies SwitchIB and SwitchIB-2
 	  Infiniband Switch ASICs.
 
@@ -63,7 +63,7 @@ config MLXSW_SWITCHX2
 	tristate "Mellanox Technologies SwitchX-2 support"
 	depends on MLXSW_CORE && MLXSW_PCI && NET_SWITCHDEV
 	default m
-	---help---
+	help
 	  This driver supports Mellanox Technologies SwitchX-2 Ethernet
 	  Switch ASICs.
 
@@ -86,7 +86,7 @@ config MLXSW_SPECTRUM
 	imply PTP_1588_CLOCK
 	select NET_PTP_CLASSIFY if PTP_1588_CLOCK
 	default m
-	---help---
+	help
 	  This driver supports Mellanox Technologies
 	  Spectrum/Spectrum-2/Spectrum-3 Ethernet Switch ASICs.
 
@@ -97,7 +97,7 @@ config MLXSW_SPECTRUM_DCB
 	bool "Data Center Bridging (DCB) support"
 	depends on MLXSW_SPECTRUM && DCB
 	default y
-	---help---
+	help
 	  Say Y here if you want to use Data Center Bridging (DCB) in the
 	  driver.
 
@@ -105,7 +105,7 @@ config MLXSW_MINIMAL
 	tristate "Mellanox Technologies minimal I2C support"
 	depends on MLXSW_CORE && MLXSW_I2C
 	default m
-	---help---
+	help
 	  This driver supports I2C access for Mellanox Technologies Switch
 	  ASICs.
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
index ce0a6837daa3..05f8d5a92862 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
@@ -391,8 +391,7 @@ static int mlxsw_thermal_set_trip_hyst(struct thermal_zone_device *tzdev,
 static int mlxsw_thermal_trend_get(struct thermal_zone_device *tzdev,
 				   int trip, enum thermal_trend *trend)
 {
-	struct mlxsw_thermal_module *tz = tzdev->devdata;
-	struct mlxsw_thermal *thermal = tz->parent;
+	struct mlxsw_thermal *thermal = tzdev->devdata;
 
 	if (trip < 0 || trip >= MLXSW_THERMAL_NUM_TRIPS)
 		return -EINVAL;
@@ -593,6 +592,22 @@ mlxsw_thermal_module_trip_hyst_set(struct thermal_zone_device *tzdev, int trip,
 	return 0;
 }
 
+static int mlxsw_thermal_module_trend_get(struct thermal_zone_device *tzdev,
+					  int trip, enum thermal_trend *trend)
+{
+	struct mlxsw_thermal_module *tz = tzdev->devdata;
+	struct mlxsw_thermal *thermal = tz->parent;
+
+	if (trip < 0 || trip >= MLXSW_THERMAL_NUM_TRIPS)
+		return -EINVAL;
+
+	if (tzdev == thermal->tz_highest_dev)
+		return 1;
+
+	*trend = THERMAL_TREND_STABLE;
+	return 0;
+}
+
 static struct thermal_zone_device_ops mlxsw_thermal_module_ops = {
 	.bind		= mlxsw_thermal_module_bind,
 	.unbind		= mlxsw_thermal_module_unbind,
@@ -604,7 +619,7 @@ static struct thermal_zone_device_ops mlxsw_thermal_module_ops = {
 	.set_trip_temp	= mlxsw_thermal_module_trip_temp_set,
 	.get_trip_hyst	= mlxsw_thermal_module_trip_hyst_get,
 	.set_trip_hyst	= mlxsw_thermal_module_trip_hyst_set,
-	.get_trend	= mlxsw_thermal_trend_get,
+	.get_trend	= mlxsw_thermal_module_trend_get,
 };
 
 static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev,
@@ -643,7 +658,7 @@ static struct thermal_zone_device_ops mlxsw_thermal_gearbox_ops = {
 	.set_trip_temp	= mlxsw_thermal_module_trip_temp_set,
 	.get_trip_hyst	= mlxsw_thermal_module_trip_hyst_get,
 	.set_trip_hyst	= mlxsw_thermal_module_trip_hyst_set,
-	.get_trend	= mlxsw_thermal_trend_get,
+	.get_trend	= mlxsw_thermal_module_trend_get,
 };
 
 static int mlxsw_thermal_get_max_state(struct thermal_cooling_device *cdev,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index 5ffa32b75e5f..55af877763ed 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -978,8 +978,10 @@ int __mlxsw_sp_port_headroom_set(struct mlxsw_sp_port *mlxsw_sp_port, int mtu,
 
 		lossy = !(pfc || pause_en);
 		thres_cells = mlxsw_sp_pg_buf_threshold_get(mlxsw_sp, mtu);
+		mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, &thres_cells);
 		delay_cells = mlxsw_sp_pg_buf_delay_get(mlxsw_sp, mtu, delay,
 							pfc, pause_en);
+		mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, &delay_cells);
 		total_cells = thres_cells + delay_cells;
 
 		taken_headroom_cells += total_cells;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index 6f96ca50c9ba..6e87457dd635 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -374,6 +374,19 @@ mlxsw_sp_port_vlan_find_by_vid(const struct mlxsw_sp_port *mlxsw_sp_port,
 	return NULL;
 }
 
+static inline void
+mlxsw_sp_port_headroom_8x_adjust(const struct mlxsw_sp_port *mlxsw_sp_port,
+				 u16 *p_size)
+{
+	/* Ports with eight lanes use two headroom buffers between which the
+	 * configured headroom size is split. Therefore, multiply the calculated
+	 * headroom size by two.
+	 */
+	if (mlxsw_sp_port->mapping.width != 8)
+		return;
+	*p_size *= 2;
+}
+
 enum mlxsw_sp_flood_type {
 	MLXSW_SP_FLOOD_TYPE_UC,
 	MLXSW_SP_FLOOD_TYPE_BC,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
index 21bfb2f6a6f0..f25a8b084b4b 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
@@ -312,6 +312,7 @@ static int mlxsw_sp_port_pb_init(struct mlxsw_sp_port *mlxsw_sp_port)
 
 		if (i == MLXSW_SP_PB_UNUSED)
 			continue;
+		mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, &size);
 		mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl, i, size);
 	}
 	mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
index 304eb8c3d8bd..f843545d3478 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
@@ -782,6 +782,7 @@ mlxsw_sp_span_port_buffer_update(struct mlxsw_sp_port *mlxsw_sp_port, u16 mtu)
 		speed = 0;
 
 	buffsize = mlxsw_sp_span_buffsize_get(mlxsw_sp, speed, mtu);
+	mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, (u16 *) &buffsize);
 	mlxsw_reg_sbib_pack(sbib_pl, mlxsw_sp_port->local_port, buffsize);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbib), sbib_pl);
 }
diff --git a/drivers/net/ethernet/micrel/Kconfig b/drivers/net/ethernet/micrel/Kconfig
index 09f35209d43d..42bc014136fe 100644
--- a/drivers/net/ethernet/micrel/Kconfig
+++ b/drivers/net/ethernet/micrel/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_MICREL
 	bool "Micrel devices"
 	default y
 	depends on (HAS_IOMEM && DMA_ENGINE) || SPI || PCI || HAS_IOMEM
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_MICREL
 config KS8842
 	tristate "Micrel KSZ8841/42 with generic bus interface"
 	depends on HAS_IOMEM && DMA_ENGINE
-	---help---
+	help
 	  This platform driver is for KSZ8841(1-port) / KS8842(2-port)
 	  ethernet switch chip (managed, VLAN, QoS) from Micrel or
 	  Timberdale(FPGA).
@@ -31,7 +31,7 @@ config KS8851
 	select MII
 	select CRC32
 	select EEPROM_93CX6
-	---help---
+	help
 	  SPI driver for Micrel KS8851 SPI attached network chip.
 
 config KS8851_MLL
@@ -40,7 +40,7 @@ config KS8851_MLL
 	select MII
 	select CRC32
 	select EEPROM_93CX6
-	---help---
+	help
 	  This platform driver is for Micrel KS8851 Address/data bus
 	  multiplexed network chip.
 
@@ -49,7 +49,7 @@ config KSZ884X_PCI
 	depends on PCI
 	select MII
 	select CRC32
-	---help---
+	help
 	  This PCI driver is for Micrel KSZ8841/KSZ8842 PCI Ethernet chip.
 
 	  To compile this driver as a module, choose M here. The module
diff --git a/drivers/net/ethernet/microchip/Kconfig b/drivers/net/ethernet/microchip/Kconfig
index 45fe41f3d9f3..31f9a82dc113 100644
--- a/drivers/net/ethernet/microchip/Kconfig
+++ b/drivers/net/ethernet/microchip/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_MICROCHIP
 	bool "Microchip devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ config ENC28J60
 	tristate "ENC28J60 support"
 	depends on SPI
 	select CRC32
-	---help---
+	help
 	  Support for the Microchip EN28J60 ethernet chip.
 
 	  To compile this driver as a module, choose M here. The module will be
@@ -29,14 +29,14 @@ config ENC28J60
 config ENC28J60_WRITEVERIFY
 	bool "Enable write verify"
 	depends on ENC28J60
-	---help---
+	help
 	  Enable the verify after the buffer write useful for debugging purpose.
 	  If unsure, say N.
 
 config ENCX24J600
     tristate "ENCX24J600 support"
     depends on SPI
-    ---help---
+	help
       Support for the Microchip ENC424J600/624J600 ethernet chip.
 
       To compile this driver as a module, choose M here. The module will be
@@ -47,7 +47,7 @@ config LAN743X
 	depends on PCI
 	select PHYLIB
 	select CRC16
-	---help---
+	help
 	  Support for the Microchip LAN743x PCI Express Gigabit Ethernet chip
 
 	  To compile this driver as a module, choose M here. The module will be
diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c
index 36624e3c633b..f1711ac86d0c 100644
--- a/drivers/net/ethernet/microchip/lan743x_main.c
+++ b/drivers/net/ethernet/microchip/lan743x_main.c
@@ -985,7 +985,7 @@ static void lan743x_phy_link_status_change(struct net_device *netdev)
 		break;
 		case SPEED_1000:
 			data |= MAC_CR_CFG_H_;
-			data |= MAC_CR_CFG_L_;
+			data &= ~MAC_CR_CFG_L_;
 		break;
 		}
 		lan743x_csr_write(adapter, MAC_CR, data);
@@ -3091,6 +3091,8 @@ static const struct pci_device_id lan743x_pcidev_tbl[] = {
 	{ 0, }
 };
 
+MODULE_DEVICE_TABLE(pci, lan743x_pcidev_tbl);
+
 static struct pci_driver lan743x_pcidev_driver = {
 	.name     = DRIVER_NAME,
 	.id_table = lan743x_pcidev_tbl,
diff --git a/drivers/net/ethernet/moxa/Kconfig b/drivers/net/ethernet/moxa/Kconfig
index 1a7cacbc0c59..134802b521cb 100644
--- a/drivers/net/ethernet/moxa/Kconfig
+++ b/drivers/net/ethernet/moxa/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_MOXART
 	bool "MOXA ART devices"
 	default y
 	depends on (ARM && ARCH_MOXART)
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config ARM_MOXART_ETHER
 	tristate "MOXART Ethernet support"
 	depends on ARM && ARCH_MOXART
 	select NET_CORE
-	---help---
+	help
 	  If you wish to compile a kernel for a hardware with MOXA ART SoC and
 	  want to use the internal ethernet then you should answer Y to this.
 
diff --git a/drivers/net/ethernet/myricom/Kconfig b/drivers/net/ethernet/myricom/Kconfig
index 6bc993eae4c4..81267fd72dbf 100644
--- a/drivers/net/ethernet/myricom/Kconfig
+++ b/drivers/net/ethernet/myricom/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_MYRI
 	bool "Myricom devices"
 	default y
 	depends on PCI && INET
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config MYRI10GE
 	depends on PCI && INET
 	select FW_LOADER
 	select CRC32
-	---help---
+	help
 	  This driver supports Myricom Myri-10G Dual Protocol interface in
 	  Ethernet mode. If the eeprom on your board is not recent enough,
 	  you will need a newer firmware image.
@@ -37,7 +37,7 @@ config MYRI10GE_DCA
 	bool "Direct Cache Access (DCA) Support"
 	default y
 	depends on MYRI10GE && DCA && !(MYRI10GE=y && DCA=m)
-	---help---
+	help
 	  Say Y here if you want to use Direct Cache Access (DCA) in the
 	  driver.  DCA is a method for warming the CPU cache before data
 	  is used, with the intent of lessening the impact of cache misses.
diff --git a/drivers/net/ethernet/natsemi/Kconfig b/drivers/net/ethernet/natsemi/Kconfig
index c519c1f30225..0a92101aa3f1 100644
--- a/drivers/net/ethernet/natsemi/Kconfig
+++ b/drivers/net/ethernet/natsemi/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_NATSEMI
 	bool "National Semiconductor devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -19,7 +19,7 @@ if NET_VENDOR_NATSEMI
 config MACSONIC
 	tristate "Macintosh SONIC based ethernet (onboard, NuBus, LC, CS)"
 	depends on MAC
-	---help---
+	help
 	  Support for NatSemi SONIC based Ethernet devices.  This includes
 	  the onboard Ethernet in many Quadras as well as some LC-PDS,
 	  a few Nubus and all known Comm Slot Ethernet cards.  If you have
@@ -31,7 +31,7 @@ config MACSONIC
 config MIPS_JAZZ_SONIC
 	tristate "MIPS JAZZ onboard SONIC Ethernet support"
 	depends on MACH_JAZZ
-	---help---
+	help
 	  This is the driver for the onboard card of MIPS Magnum 4000,
 	  Acer PICA, Olivetti M700-10 and a few other identical OEM systems.
 
@@ -39,7 +39,7 @@ config NATSEMI
 	tristate "National Semiconductor DP8381x series PCI Ethernet support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This driver is for the National Semiconductor DP83810 series,
 	  which is used in cards from PureData, NetGear, Linksys
 	  and others, including the 83815 chip.
@@ -49,7 +49,7 @@ config NATSEMI
 config NS83820
 	tristate "National Semiconductor DP83820 support"
 	depends on PCI
-	---help---
+	help
 	  This is a driver for the National Semiconductor DP83820 series
 	  of gigabit ethernet MACs.  Cards using this chipset include
 	  the D-Link DGE-500T, PureData's PDP8023Z-TG, SMC's SMC9462TX,
@@ -59,7 +59,7 @@ config NS83820
 config XTENSA_XT2000_SONIC
 	tristate "Xtensa XT2000 onboard SONIC Ethernet support"
 	depends on XTENSA_PLATFORM_XT2000
-	---help---
+	help
 	  This is the driver for the onboard card of the Xtensa XT2000 board.
 
 endif # NET_VENDOR_NATSEMI
diff --git a/drivers/net/ethernet/natsemi/jazzsonic.c b/drivers/net/ethernet/natsemi/jazzsonic.c
index 8b018ed37b1b..ce3eca5d152b 100644
--- a/drivers/net/ethernet/natsemi/jazzsonic.c
+++ b/drivers/net/ethernet/natsemi/jazzsonic.c
@@ -36,9 +36,9 @@
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
+#include <linux/pgtable.h>
 
 #include <asm/bootinfo.h>
-#include <asm/pgtable.h>
 #include <asm/io.h>
 #include <asm/dma.h>
 #include <asm/jazz.h>
diff --git a/drivers/net/ethernet/natsemi/macsonic.c b/drivers/net/ethernet/natsemi/macsonic.c
index 1b5559aacb38..776b7d264dc3 100644
--- a/drivers/net/ethernet/natsemi/macsonic.c
+++ b/drivers/net/ethernet/natsemi/macsonic.c
@@ -51,8 +51,8 @@
 #include <linux/dma-mapping.h>
 #include <linux/bitrev.h>
 #include <linux/slab.h>
+#include <linux/pgtable.h>
 
-#include <asm/pgtable.h>
 #include <asm/io.h>
 #include <asm/hwtest.h>
 #include <asm/dma.h>
diff --git a/drivers/net/ethernet/natsemi/xtsonic.c b/drivers/net/ethernet/natsemi/xtsonic.c
index dda9ec7d9cee..afa166ff7aef 100644
--- a/drivers/net/ethernet/natsemi/xtsonic.c
+++ b/drivers/net/ethernet/natsemi/xtsonic.c
@@ -35,9 +35,9 @@
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
+#include <linux/pgtable.h>
 
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/dma.h>
 
 static char xtsonic_string[] = "xtsonic";
diff --git a/drivers/net/ethernet/neterion/Kconfig b/drivers/net/ethernet/neterion/Kconfig
index a82a37094579..5484f18f272e 100644
--- a/drivers/net/ethernet/neterion/Kconfig
+++ b/drivers/net/ethernet/neterion/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_NETERION
 	bool "Neterion (Exar) devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_NETERION
 config S2IO
 	tristate "Neterion (Exar) Xframe 10Gb Ethernet Adapter"
 	depends on PCI
-	---help---
+	help
 	  This driver supports Exar Corp's Xframe Series 10Gb Ethernet Adapters.
 	  These were originally released from S2IO, which renamed itself
 	  Neterion.  So, the adapters might be labeled as either one, depending
@@ -35,7 +35,7 @@ config S2IO
 config VXGE
 	tristate "Neterion (Exar) X3100 Series 10GbE PCIe Server Adapter"
 	depends on PCI
-	---help---
+	help
 	  This driver supports Exar Corp's X3100 Series 10 GbE PCIe
 	  I/O Virtualized Server Adapter.  These were originally released from
 	  Neterion, which was later acquired by Exar.  So, the adapters might be
@@ -51,7 +51,7 @@ config VXGE_DEBUG_TRACE_ALL
 	bool "Enabling All Debug trace statements in driver"
 	default n
 	depends on VXGE
-	---help---
+	help
 	  Say Y here if you want to enabling all the debug trace statements in
 	  the vxge driver. By default only few debug trace statements are
 	  enabled.
diff --git a/drivers/net/ethernet/netronome/Kconfig b/drivers/net/ethernet/netronome/Kconfig
index a3f68a718813..d8b99d6a0356 100644
--- a/drivers/net/ethernet/netronome/Kconfig
+++ b/drivers/net/ethernet/netronome/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_NETRONOME
 	bool "Netronome(R) devices"
 	default y
-	---help---
+	help
 	  If you have a Netronome(R) network (Ethernet) card or device, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config NFP
 	depends on VXLAN || VXLAN=n
 	depends on TLS && TLS_DEVICE || TLS_DEVICE=n
 	select NET_DEVLINK
-	---help---
+	help
 	  This driver supports the Netronome(R) NFP4000/NFP6000 based
 	  cards working as a advanced Ethernet NIC.  It works with both
 	  SR-IOV physical and virtual functions.
@@ -33,7 +33,7 @@ config NFP_APP_FLOWER
 	depends on NET_SWITCHDEV
 	depends on IPV6!=m || NFP=m
 	default y
-	---help---
+	help
 	  Enable driver support for TC Flower offload on NFP4000 and NFP6000.
 	  Say Y, if you are planning to make use of TC Flower offload
 	  either directly, with Open vSwitch, or any other way.  Note that
@@ -55,7 +55,7 @@ config NFP_APP_ABM_NIC
 config NFP_DEBUG
 	bool "Debug support for Netronome(R) NFP4000/NFP6000 NIC drivers"
 	depends on NFP
-	---help---
+	help
 	  Enable extra sanity checks and debugfs support in
 	  Netronome(R) NFP4000/NFP6000 NIC drivers.
 	  Note: selecting this option may adversely impact
diff --git a/drivers/net/ethernet/nvidia/Kconfig b/drivers/net/ethernet/nvidia/Kconfig
index faacbd129c44..c653786b1d05 100644
--- a/drivers/net/ethernet/nvidia/Kconfig
+++ b/drivers/net/ethernet/nvidia/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_NVIDIA
 	bool "NVIDIA devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_NVIDIA
 config FORCEDETH
 	tristate "nForce Ethernet support"
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) controller of this type, say Y here.
 
 	  To compile this driver as a module, choose M here. The module
diff --git a/drivers/net/ethernet/oki-semi/Kconfig b/drivers/net/ethernet/oki-semi/Kconfig
index 1c455c645bce..c2fff04ba7b6 100644
--- a/drivers/net/ethernet/oki-semi/Kconfig
+++ b/drivers/net/ethernet/oki-semi/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_OKI
 	bool "OKI Semiconductor devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/Kconfig b/drivers/net/ethernet/oki-semi/pch_gbe/Kconfig
index 69e11d19bdc6..af84f72bf08e 100644
--- a/drivers/net/ethernet/oki-semi/pch_gbe/Kconfig
+++ b/drivers/net/ethernet/oki-semi/pch_gbe/Kconfig
@@ -9,7 +9,7 @@ config PCH_GBE
 	select MII
 	select PTP_1588_CLOCK_PCH
 	select NET_PTP_CLASSIFY
-	---help---
+	help
 	  This is a gigabit ethernet driver for EG20T PCH.
 	  EG20T PCH is the platform controller hub that is used in Intel's
 	  general embedded platform.  EG20T PCH has Gigabit Ethernet interface.
diff --git a/drivers/net/ethernet/packetengines/Kconfig b/drivers/net/ethernet/packetengines/Kconfig
index ead3750b4489..de91331dcb7d 100644
--- a/drivers/net/ethernet/packetengines/Kconfig
+++ b/drivers/net/ethernet/packetengines/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_PACKET_ENGINES
 	bool "Packet Engines devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config HAMACHI
 	tristate "Packet Engines Hamachi GNIC-II support"
 	depends on PCI
 	select MII
-	---help---
+	help
 	  If you have a Gigabit Ethernet card of this type, say Y here.
 
 	  To compile this driver as a module, choose M here. The module will be
@@ -31,7 +31,7 @@ config YELLOWFIN
 	tristate "Packet Engines Yellowfin Gigabit-NIC support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  Say Y here if you have a Packet Engines G-NIC PCI Gigabit Ethernet
 	  adapter or the SYM53C885 Ethernet controller. The Gigabit adapter is
 	  used by the Beowulf Linux cluster project.  See
diff --git a/drivers/net/ethernet/pasemi/Kconfig b/drivers/net/ethernet/pasemi/Kconfig
index f4562243d4a0..cd68ebcf3e47 100644
--- a/drivers/net/ethernet/pasemi/Kconfig
+++ b/drivers/net/ethernet/pasemi/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_PASEMI
 	bool "PA Semi devices"
 	default y
 	depends on PPC_PASEMI && PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config PASEMI_MAC
 	tristate "PA Semi 1/10Gbit MAC"
 	depends on PPC_PASEMI && PCI
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the on-chip 1/10Gbit Ethernet controller on
 	  PA Semi's PWRficient line of chips.
 
diff --git a/drivers/net/ethernet/pensando/ionic/ionic.h b/drivers/net/ethernet/pensando/ionic/ionic.h
index 23ccc0da2341..f5a910c458ba 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic.h
@@ -17,7 +17,6 @@ struct ionic_lif;
 
 #define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF	0x1002
 #define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF	0x1003
-#define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_MGMT	0x1004
 
 #define DEVCMD_TIMEOUT  10
 
@@ -42,7 +41,6 @@ struct ionic {
 	struct dentry *dentry;
 	struct ionic_dev_bar bars[IONIC_BARS_MAX];
 	unsigned int num_bars;
-	bool is_mgmt_nic;
 	struct ionic_identity ident;
 	struct list_head lifs;
 	struct ionic_lif *master_lif;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
index 60fc191a35e5..2924cde440aa 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
@@ -15,7 +15,6 @@
 static const struct pci_device_id ionic_id_table[] = {
 	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF) },
 	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF) },
-	{ PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_IONIC_ETH_MGMT) },
 	{ 0, }	/* end of table */
 };
 MODULE_DEVICE_TABLE(pci, ionic_id_table);
@@ -225,9 +224,6 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	pci_set_drvdata(pdev, ionic);
 	mutex_init(&ionic->dev_cmd_lock);
 
-	ionic->is_mgmt_nic =
-		ent->device == PCI_DEVICE_ID_PENSANDO_IONIC_ETH_MGMT;
-
 	/* Query system for DMA addressing limitation for the device. */
 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(IONIC_ADDR_LEN));
 	if (err) {
@@ -252,8 +248,7 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	pci_set_master(pdev);
-	if (!ionic->is_mgmt_nic)
-		pcie_print_link_status(pdev);
+	pcie_print_link_status(pdev);
 
 	err = ionic_map_bars(ionic);
 	if (err)
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_devlink.c b/drivers/net/ethernet/pensando/ionic/ionic_devlink.c
index 273c889faaad..2d590e571133 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_devlink.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_devlink.c
@@ -77,10 +77,6 @@ int ionic_devlink_register(struct ionic *ionic)
 		return err;
 	}
 
-	/* don't register the mgmt_nic as a port */
-	if (ionic->is_mgmt_nic)
-		return 0;
-
 	devlink_port_attrs_set(&ionic->dl_port, DEVLINK_PORT_FLAVOUR_PHYSICAL,
 			       0, false, 0, NULL, 0);
 	err = devlink_port_register(dl, &ionic->dl_port, 0);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index 7321a92f8395..9d8c969f21cb 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -99,9 +99,6 @@ static void ionic_link_status_check(struct ionic_lif *lif)
 	if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 		return;
 
-	if (lif->ionic->is_mgmt_nic)
-		return;
-
 	link_status = le16_to_cpu(lif->info->status.link_status);
 	link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 
@@ -116,7 +113,7 @@ static void ionic_link_status_check(struct ionic_lif *lif)
 			netif_carrier_on(netdev);
 		}
 
-		if (netif_running(lif->netdev))
+		if (lif->netdev->flags & IFF_UP && netif_running(lif->netdev))
 			ionic_start_queues(lif);
 	} else {
 		if (netif_carrier_ok(netdev)) {
@@ -124,7 +121,7 @@ static void ionic_link_status_check(struct ionic_lif *lif)
 			netif_carrier_off(netdev);
 		}
 
-		if (netif_running(lif->netdev))
+		if (lif->netdev->flags & IFF_UP && netif_running(lif->netdev))
 			ionic_stop_queues(lif);
 	}
 
@@ -1193,10 +1190,6 @@ static int ionic_init_nic_features(struct ionic_lif *lif)
 	netdev_features_t features;
 	int err;
 
-	/* no netdev features on the management device */
-	if (lif->ionic->is_mgmt_nic)
-		return 0;
-
 	/* set up what we expect to support by default */
 	features = NETIF_F_HW_VLAN_CTAG_TX |
 		   NETIF_F_HW_VLAN_CTAG_RX |
@@ -2594,12 +2587,6 @@ int ionic_lifs_register(struct ionic *ionic)
 {
 	int err;
 
-	/* the netdev is not registered on the management device, it is
-	 * only used as a vehicle for napi operations on the adminq
-	 */
-	if (ionic->is_mgmt_nic)
-		return 0;
-
 	INIT_WORK(&ionic->nb_work, ionic_lif_notify_work);
 
 	ionic->nb.notifier_call = ionic_lif_notify;
diff --git a/drivers/net/ethernet/qlogic/Kconfig b/drivers/net/ethernet/qlogic/Kconfig
index 55a29ec76680..8f743d80760b 100644
--- a/drivers/net/ethernet/qlogic/Kconfig
+++ b/drivers/net/ethernet/qlogic/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_QLOGIC
 	bool "QLogic devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_QLOGIC
 config QLA3XXX
 	tristate "QLogic QLA3XXX Network Driver Support"
 	depends on PCI
-	---help---
+	help
 	  This driver supports QLogic ISP3XXX gigabit Ethernet cards.
 
 	  To compile this driver as a module, choose M here: the module
@@ -30,7 +30,7 @@ config QLCNIC
 	tristate "QLOGIC QLCNIC 1/10Gb Converged Ethernet NIC Support"
 	depends on PCI
 	select FW_LOADER
-	---help---
+	help
 	  This driver supports QLogic QLE8240 and QLE8242 Converged Ethernet
 	  devices.
 
@@ -38,7 +38,7 @@ config QLCNIC_SRIOV
 	bool "QLOGIC QLCNIC 83XX family SR-IOV Support"
 	depends on QLCNIC && PCI_IOV
 	default y
-	---help---
+	help
 	  This configuration parameter enables Single Root Input Output
 	  Virtualization support for QLE83XX Converged Ethernet devices.
 	  This allows for virtual function acceleration in virtualized
@@ -48,7 +48,7 @@ config QLCNIC_DCB
 	bool "QLOGIC QLCNIC 82XX and 83XX family DCB Support"
 	depends on QLCNIC && DCB
 	default y
-	---help---
+	help
 	  This configuration parameter enables DCB support in QLE83XX
 	  and QLE82XX Converged Ethernet devices. This allows for DCB
 	  get operations support through rtNetlink interface. Only CEE
@@ -59,7 +59,7 @@ config QLCNIC_HWMON
 	bool "QLOGIC QLCNIC 82XX and 83XX family HWMON support"
 	depends on QLCNIC && HWMON && !(QLCNIC=y && HWMON=m)
 	default y
-	---help---
+	help
 	  This configuration parameter can be used to read the
 	  board temperature in Converged Ethernet devices
 	  supported by qlcnic.
@@ -70,7 +70,7 @@ config NETXEN_NIC
 	tristate "NetXen Multi port (1/10) Gigabit Ethernet NIC"
 	depends on PCI
 	select FW_LOADER
-	---help---
+	help
 	  This enables the support for NetXen's Gigabit Ethernet card.
 
 config QED
@@ -79,7 +79,7 @@ config QED
 	select ZLIB_INFLATE
 	select CRC8
 	select NET_DEVLINK
-	---help---
+	help
 	  This enables the support for ...
 
 config QED_LL2
@@ -89,7 +89,7 @@ config QED_SRIOV
 	bool "QLogic QED 25/40/100Gb SR-IOV support"
 	depends on QED && PCI_IOV
 	default y
-	---help---
+	help
 	  This configuration parameter enables Single Root Input Output
 	  Virtualization support for QED devices.
 	  This allows for virtual function acceleration in virtualized
@@ -99,7 +99,7 @@ config QEDE
 	tristate "QLogic QED 25/40/100Gb Ethernet NIC"
 	depends on QED
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This enables the support for ...
 
 config QED_RDMA
diff --git a/drivers/net/ethernet/qlogic/qed/qed.h b/drivers/net/ethernet/qlogic/qed/qed.h
index 66ed39d6f357..a49743d56b9c 100644
--- a/drivers/net/ethernet/qlogic/qed/qed.h
+++ b/drivers/net/ethernet/qlogic/qed/qed.h
@@ -1014,6 +1014,7 @@ int qed_device_num_ports(struct qed_dev *cdev);
 int qed_fill_dev_info(struct qed_dev *cdev,
 		      struct qed_dev_info *dev_info);
 void qed_link_update(struct qed_hwfn *hwfn, struct qed_ptt *ptt);
+void qed_bw_update(struct qed_hwfn *hwfn, struct qed_ptt *ptt);
 u32 qed_unzip_data(struct qed_hwfn *p_hwfn,
 		   u32 input_len, u8 *input_buf,
 		   u32 max_size, u8 *unzip_buf);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c
index 83e798d4eebb..11367a248d55 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_main.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_main.c
@@ -1949,6 +1949,15 @@ void qed_link_update(struct qed_hwfn *hwfn, struct qed_ptt *ptt)
 		op->link_update(cookie, &if_link);
 }
 
+void qed_bw_update(struct qed_hwfn *hwfn, struct qed_ptt *ptt)
+{
+	void *cookie = hwfn->cdev->ops_cookie;
+	struct qed_common_cb_ops *op = hwfn->cdev->protocol_ops.common;
+
+	if (IS_LEAD_HWFN(hwfn) && cookie && op && op->bw_update)
+		op->bw_update(cookie);
+}
+
 static int qed_drain(struct qed_dev *cdev)
 {
 	struct qed_hwfn *hwfn;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_rdma.c b/drivers/net/ethernet/qlogic/qed/qed_rdma.c
index 98455f698f53..19c0c8864da1 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_rdma.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_rdma.c
@@ -526,7 +526,6 @@ static void qed_rdma_init_devinfo(struct qed_hwfn *p_hwfn,
 		dev->max_cqe = QED_RDMA_MAX_CQE_16_BIT;
 
 	dev->max_mw = 0;
-	dev->max_fmr = QED_RDMA_MAX_FMR;
 	dev->max_mr_mw_fmr_pbl = (PAGE_SIZE / 8) * (PAGE_SIZE / 8);
 	dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * PAGE_SIZE;
 	dev->max_pkey = QED_RDMA_MAX_P_KEY;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_rdma.h b/drivers/net/ethernet/qlogic/qed/qed_rdma.h
index 3898cae61e7a..1e69d5bb0a70 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_rdma.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_rdma.h
@@ -45,7 +45,6 @@
 #include "qed_iwarp.h"
 #include "qed_roce.h"
 
-#define QED_RDMA_MAX_FMR                    (RDMA_MAX_TIDS)
 #define QED_RDMA_MAX_P_KEY                  (1)
 #define QED_RDMA_MAX_WQE                    (0x7FFF)
 #define QED_RDMA_MAX_SRQ_WQE_ELEM           (0x7FFF)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.c b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
index 66876af814c4..20679fd4204b 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
@@ -33,6 +33,7 @@
 #include <linux/etherdevice.h>
 #include <linux/crc32.h>
 #include <linux/vmalloc.h>
+#include <linux/crash_dump.h>
 #include <linux/qed/qed_iov_if.h>
 #include "qed_cxt.h"
 #include "qed_hsi.h"
@@ -607,6 +608,9 @@ int qed_iov_hw_info(struct qed_hwfn *p_hwfn)
 	int pos;
 	int rc;
 
+	if (is_kdump_kernel())
+		return 0;
+
 	if (IS_VF(p_hwfn->cdev))
 		return 0;
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.h b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
index aabeaf03135e..368e88565783 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
@@ -32,7 +32,6 @@
 
 #ifndef _QED_SRIOV_H
 #define _QED_SRIOV_H
-#include <linux/crash_dump.h>
 #include <linux/types.h>
 #include "qed_vf.h"
 
@@ -41,12 +40,9 @@
 #define QED_VF_ARRAY_LENGTH (3)
 
 #ifdef CONFIG_QED_SRIOV
-#define IS_VF(cdev)             (is_kdump_kernel() ? \
-				 (0) : ((cdev)->b_is_vf))
-#define IS_PF(cdev)             (is_kdump_kernel() ? \
-				 (1) : !((cdev)->b_is_vf))
-#define IS_PF_SRIOV(p_hwfn)     (is_kdump_kernel() ? \
-				 (0) : !!((p_hwfn)->cdev->p_iov_info))
+#define IS_VF(cdev)             ((cdev)->b_is_vf)
+#define IS_PF(cdev)             (!((cdev)->b_is_vf))
+#define IS_PF_SRIOV(p_hwfn)     (!!((p_hwfn)->cdev->p_iov_info))
 #else
 #define IS_VF(cdev)             (0)
 #define IS_PF(cdev)             (1)
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index b2d154258b07..756c05eb96f3 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -1265,7 +1265,7 @@ static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	case QEDE_PRIVATE_VF:
 		if (debug & QED_LOG_VERBOSE_MASK)
 			dev_err(&pdev->dev, "Probing a VF\n");
-		is_vf = is_kdump_kernel() ? false : true;
+		is_vf = true;
 		break;
 	default:
 		if (debug & QED_LOG_VERBOSE_MASK)
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
index 8d7b9bb910f2..10037639ac2c 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
@@ -269,7 +269,7 @@ static ssize_t qlcnic_sysfs_read_crb(struct file *filp, struct kobject *kobj,
 				     struct bin_attribute *attr, char *buf,
 				     loff_t offset, size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	int ret;
 
@@ -286,7 +286,7 @@ static ssize_t qlcnic_sysfs_write_crb(struct file *filp, struct kobject *kobj,
 				      struct bin_attribute *attr, char *buf,
 				      loff_t offset, size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	int ret;
 
@@ -315,7 +315,7 @@ static ssize_t qlcnic_sysfs_read_mem(struct file *filp, struct kobject *kobj,
 				     struct bin_attribute *attr, char *buf,
 				     loff_t offset, size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	u64 data;
 	int ret;
@@ -337,7 +337,7 @@ static ssize_t qlcnic_sysfs_write_mem(struct file *filp, struct kobject *kobj,
 				      struct bin_attribute *attr, char *buf,
 				      loff_t offset, size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	u64 data;
 	int ret;
@@ -402,7 +402,7 @@ static ssize_t qlcnic_sysfs_write_pm_config(struct file *filp,
 					    char *buf, loff_t offset,
 					    size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_pm_func_cfg *pm_cfg;
 	u32 id, action, pci_func;
@@ -452,7 +452,7 @@ static ssize_t qlcnic_sysfs_read_pm_config(struct file *filp,
 					   char *buf, loff_t offset,
 					   size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_pm_func_cfg *pm_cfg;
 	u8 pci_func;
@@ -545,7 +545,7 @@ static ssize_t qlcnic_sysfs_write_esw_config(struct file *file,
 					     char *buf, loff_t offset,
 					     size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_esw_func_cfg *esw_cfg;
 	struct qlcnic_npar_info *npar;
@@ -629,7 +629,7 @@ static ssize_t qlcnic_sysfs_read_esw_config(struct file *file,
 					    char *buf, loff_t offset,
 					    size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_esw_func_cfg *esw_cfg;
 	u8 pci_func;
@@ -681,7 +681,7 @@ static ssize_t qlcnic_sysfs_write_npar_config(struct file *file,
 					      char *buf, loff_t offset,
 					      size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_info nic_info;
 	struct qlcnic_npar_func_cfg *np_cfg;
@@ -728,7 +728,7 @@ static ssize_t qlcnic_sysfs_read_npar_config(struct file *file,
 					     char *buf, loff_t offset,
 					     size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_npar_func_cfg *np_cfg;
 	struct qlcnic_info nic_info;
@@ -775,7 +775,7 @@ static ssize_t qlcnic_sysfs_get_port_stats(struct file *file,
 					   char *buf, loff_t offset,
 					   size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_esw_statistics port_stats;
 	int ret;
@@ -810,7 +810,7 @@ static ssize_t qlcnic_sysfs_get_esw_stats(struct file *file,
 					  char *buf, loff_t offset,
 					  size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_esw_statistics esw_stats;
 	int ret;
@@ -845,7 +845,7 @@ static ssize_t qlcnic_sysfs_clear_esw_stats(struct file *file,
 					    char *buf, loff_t offset,
 					    size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	int ret;
 
@@ -875,7 +875,7 @@ static ssize_t qlcnic_sysfs_clear_port_stats(struct file *file,
 					     size_t size)
 {
 
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	int ret;
 
@@ -904,7 +904,7 @@ static ssize_t qlcnic_sysfs_read_pci_config(struct file *file,
 					    char *buf, loff_t offset,
 					    size_t size)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 	struct qlcnic_pci_func_cfg *pci_cfg;
 	struct qlcnic_pci_info *pci_info;
@@ -946,7 +946,7 @@ static ssize_t qlcnic_83xx_sysfs_flash_read_handler(struct file *filp,
 {
 	unsigned char *p_read_buf;
 	int  ret, count;
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 
 	if (!size)
@@ -1124,7 +1124,7 @@ static ssize_t qlcnic_83xx_sysfs_flash_write_handler(struct file *filp,
 	int  ret;
 	static int flash_mode;
 	unsigned long data;
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct qlcnic_adapter *adapter = dev_get_drvdata(dev);
 
 	ret = kstrtoul(buf, 16, &data);
diff --git a/drivers/net/ethernet/qualcomm/Kconfig b/drivers/net/ethernet/qualcomm/Kconfig
index 09a678a94634..a4434eb38950 100644
--- a/drivers/net/ethernet/qualcomm/Kconfig
+++ b/drivers/net/ethernet/qualcomm/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_QUALCOMM
 	bool "Qualcomm devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -25,7 +25,7 @@ config QCA7000_SPI
 	tristate "Qualcomm Atheros QCA7000 SPI support"
 	select QCA7000
 	depends on SPI_MASTER && OF
-	---help---
+	help
 	  This SPI protocol driver supports the Qualcomm Atheros QCA7000.
 
 	  To compile this driver as a module, choose M here. The module
@@ -35,7 +35,7 @@ config QCA7000_UART
 	tristate "Qualcomm Atheros QCA7000 UART support"
 	select QCA7000
 	depends on SERIAL_DEV_BUS && OF
-	---help---
+	help
 	  This UART protocol driver supports the Qualcomm Atheros QCA7000.
 
 	  Currently the driver assumes these device UART settings:
@@ -52,7 +52,7 @@ config QCOM_EMAC
 	depends on HAS_DMA && HAS_IOMEM
 	select CRC32
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the Qualcomm Technologies, Inc. Gigabit
 	  Ethernet Media Access Controller (EMAC). The controller
 	  supports IEEE 802.3-2002, half-duplex mode at 10/100 Mb/s,
diff --git a/drivers/net/ethernet/qualcomm/rmnet/Kconfig b/drivers/net/ethernet/qualcomm/rmnet/Kconfig
index 9f9279575e2e..b4a575eb0175 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/Kconfig
+++ b/drivers/net/ethernet/qualcomm/rmnet/Kconfig
@@ -7,7 +7,7 @@ menuconfig RMNET
 	tristate "RmNet MAP driver"
 	default n
 	select GRO_CELLS
-	---help---
+	help
 	  If you select this, you will enable the RMNET module which is used
 	  for handling data in the multiplexing and aggregation protocol (MAP)
 	  format in the embedded data path. RMNET devices can be attached to
diff --git a/drivers/net/ethernet/rdc/Kconfig b/drivers/net/ethernet/rdc/Kconfig
index 76df60c2f4ac..6884c7864bb9 100644
--- a/drivers/net/ethernet/rdc/Kconfig
+++ b/drivers/net/ethernet/rdc/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_RDC
 	bool "RDC devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config R6040
 	select CRC32
 	select MII
 	select PHYLIB
-	---help---
+	help
 	  This is a driver for the R6040 Fast Ethernet MACs found in the
 	  the RDC R-321x System-on-chips.
 
diff --git a/drivers/net/ethernet/realtek/Kconfig b/drivers/net/ethernet/realtek/Kconfig
index 5e0b9d2f14f7..93d9df55b361 100644
--- a/drivers/net/ethernet/realtek/Kconfig
+++ b/drivers/net/ethernet/realtek/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_REALTEK
 	bool "Realtek devices"
 	default y
 	depends on PCI || (PARPORT && X86)
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config ATP
 	tristate "AT-LAN-TEC/RealTek pocket adapter support"
 	depends on PARPORT && X86
 	select CRC32
-	---help---
+	help
 	  This is a network (Ethernet) device which attaches to your parallel
 	  port. Read the file <file:drivers/net/ethernet/realtek/atp.c>
 	  if you want to use this.  If you intend to use this driver, you
@@ -36,7 +36,7 @@ config 8139CP
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This is a driver for the Fast Ethernet PCI network cards based on
 	  the RTL8139C+ chips. If you have one of those, say Y here.
 
@@ -48,7 +48,7 @@ config 8139TOO
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This is a driver for the Fast Ethernet PCI network cards based on
 	  the RTL 8129/8130/8139 chips. If you have one of those, say Y here.
 
@@ -59,7 +59,7 @@ config 8139TOO_PIO
 	bool "Use PIO instead of MMIO"
 	default y
 	depends on 8139TOO
-	---help---
+	help
 	  This instructs the driver to use programmed I/O ports (PIO) instead
 	  of PCI shared memory (MMIO).  This can possibly solve some problems
 	  in case your mainboard has memory consistency issues.  If unsure,
@@ -68,7 +68,7 @@ config 8139TOO_PIO
 config 8139TOO_TUNE_TWISTER
 	bool "Support for uncommon RTL-8139 rev. K (automatic channel equalization)"
 	depends on 8139TOO
-	---help---
+	help
 	  This implements a function which might come in handy in case you
 	  are using low quality on long cabling. It is required for RealTek
 	  RTL-8139 revision K boards, and totally unused otherwise.  It tries
@@ -79,7 +79,7 @@ config 8139TOO_TUNE_TWISTER
 config 8139TOO_8129
 	bool "Support for older RTL-8129/8130 boards"
 	depends on 8139TOO
-	---help---
+	help
 	  This enables support for the older and uncommon RTL-8129 and
 	  RTL-8130 chips, which support MII via an external transceiver,
 	  instead of an internal one.  Disabling this option will save some
@@ -88,7 +88,7 @@ config 8139TOO_8129
 config 8139_OLD_RX_RESET
 	bool "Use older RX-reset method"
 	depends on 8139TOO
-	---help---
+	help
 	  The 8139too driver was recently updated to contain a more rapid
 	  reset sequence, in the face of severe receive errors.  This "new"
 	  RX-reset method should be adequate for all boards.  But if you
@@ -102,7 +102,7 @@ config R8169
 	select CRC32
 	select PHYLIB
 	select REALTEK_PHY
-	---help---
+	help
 	  Say Y here if you have a Realtek Ethernet adapter belonging to
 	  the following families:
 	  RTL8169 Gigabit Ethernet
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index 4d2ec9742cee..dad84ecf5a77 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -3928,7 +3928,7 @@ static void rtl8169_tx_clear(struct rtl8169_private *tp)
 	netdev_reset_queue(tp->dev);
 }
 
-static void rtl8169_hw_reset(struct rtl8169_private *tp)
+static void rtl8169_hw_reset(struct rtl8169_private *tp, bool going_down)
 {
 	/* Give a racing hard_start_xmit a few cycles to complete. */
 	synchronize_rcu();
@@ -3938,6 +3938,9 @@ static void rtl8169_hw_reset(struct rtl8169_private *tp)
 
 	rtl_rx_close(tp);
 
+	if (going_down && tp->dev->wol_enabled)
+		goto no_reset;
+
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_27:
 	case RTL_GIGA_MAC_VER_28:
@@ -3959,7 +3962,7 @@ static void rtl8169_hw_reset(struct rtl8169_private *tp)
 	}
 
 	rtl_hw_reset(tp);
-
+no_reset:
 	rtl8169_tx_clear(tp);
 	rtl8169_init_ring_indexes(tp);
 }
@@ -3972,7 +3975,7 @@ static void rtl_reset_work(struct rtl8169_private *tp)
 	napi_disable(&tp->napi);
 	netif_stop_queue(dev);
 
-	rtl8169_hw_reset(tp);
+	rtl8169_hw_reset(tp, false);
 
 	for (i = 0; i < NUM_RX_DESC; i++)
 		rtl8169_mark_to_asic(tp->RxDescArray + i);
@@ -4637,7 +4640,7 @@ static void rtl8169_down(struct rtl8169_private *tp)
 	phy_stop(tp->phydev);
 	napi_disable(&tp->napi);
 
-	rtl8169_hw_reset(tp);
+	rtl8169_hw_reset(tp, true);
 
 	rtl_pll_power_down(tp);
 
@@ -4942,8 +4945,6 @@ static void rtl_shutdown(struct pci_dev *pdev)
 	/* Restore original MAC address */
 	rtl_rar_set(tp, tp->dev->perm_addr);
 
-	rtl8169_hw_reset(tp);
-
 	if (system_state == SYSTEM_POWER_OFF) {
 		if (tp->saved_wolopts) {
 			rtl_wol_suspend_quirk(tp);
diff --git a/drivers/net/ethernet/renesas/Kconfig b/drivers/net/ethernet/renesas/Kconfig
index bb0ebdfd4459..5a2a4af31812 100644
--- a/drivers/net/ethernet/renesas/Kconfig
+++ b/drivers/net/ethernet/renesas/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_RENESAS
 	bool "Renesas devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config SH_ETH
 	select MII
 	select MDIO_BITBANG
 	select PHYLIB
-	---help---
+	help
 	  Renesas SuperH Ethernet device driver.
 	  This driver supporting CPUs are:
 		- SH7619, SH7710, SH7712, SH7724, SH7734, SH7763, SH7757,
diff --git a/drivers/net/ethernet/rocker/Kconfig b/drivers/net/ethernet/rocker/Kconfig
index 1083de99830d..99e1290e0307 100644
--- a/drivers/net/ethernet/rocker/Kconfig
+++ b/drivers/net/ethernet/rocker/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_ROCKER
 	bool "Rocker devices"
 	default y
-	---help---
+	help
 	  If you have a network device belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -19,7 +19,7 @@ if NET_VENDOR_ROCKER
 config ROCKER
 	tristate "Rocker switch driver (EXPERIMENTAL)"
 	depends on PCI && NET_SWITCHDEV && BRIDGE
-	---help---
+	help
 	  This driver supports Rocker switch device.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index 7585cd2270ba..fc99e7118e49 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -647,10 +647,10 @@ static int rocker_dma_rings_init(struct rocker *rocker)
 err_dma_event_ring_bufs_alloc:
 	rocker_dma_ring_destroy(rocker, &rocker->event_ring);
 err_dma_event_ring_create:
+	rocker_dma_cmd_ring_waits_free(rocker);
+err_dma_cmd_ring_waits_alloc:
 	rocker_dma_ring_bufs_free(rocker, &rocker->cmd_ring,
 				  PCI_DMA_BIDIRECTIONAL);
-err_dma_cmd_ring_waits_alloc:
-	rocker_dma_cmd_ring_waits_free(rocker);
 err_dma_cmd_ring_bufs_alloc:
 	rocker_dma_ring_destroy(rocker, &rocker->cmd_ring);
 	return err;
diff --git a/drivers/net/ethernet/samsung/Kconfig b/drivers/net/ethernet/samsung/Kconfig
index e92a178a76df..0582e110b1c0 100644
--- a/drivers/net/ethernet/samsung/Kconfig
+++ b/drivers/net/ethernet/samsung/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_SAMSUNG
 	bool "Samsung Ethernet devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) chipset belonging to this class,
 	  say Y.
 
@@ -23,7 +23,7 @@ config SXGBE_ETH
 	select PHYLIB
 	select CRC32
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This is the driver for the SXGBE 10G Ethernet IP block found on
 	  Samsung platforms.
 
diff --git a/drivers/net/ethernet/seeq/Kconfig b/drivers/net/ethernet/seeq/Kconfig
index f3ac9cba5770..ad1df37571dd 100644
--- a/drivers/net/ethernet/seeq/Kconfig
+++ b/drivers/net/ethernet/seeq/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_SEEQ
 	bool "SEEQ devices"
 	default y
 	depends on HAS_IOMEM
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,14 +20,14 @@ if NET_VENDOR_SEEQ
 config ARM_ETHER3
 	tristate "Acorn/ANT Ether3 support"
 	depends on ARM && ARCH_ACORN
-	---help---
+	help
 	  If you have an Acorn system with one of these network cards, you
 	  should say Y to this option if you wish to use it with Linux.
 
 config SGISEEQ
 	tristate "SGI Seeq ethernet controller support"
 	depends on SGI_HAS_SEEQ
-	---help---
+	help
 	  Say Y here if you have an Seeq based Ethernet network card. This is
 	  used in many Silicon Graphics machines.
 
diff --git a/drivers/net/ethernet/sfc/Kconfig b/drivers/net/ethernet/sfc/Kconfig
index ea5a9220196c..81b0f7d3a025 100644
--- a/drivers/net/ethernet/sfc/Kconfig
+++ b/drivers/net/ethernet/sfc/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_SOLARFLARE
 	bool "Solarflare devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config SFC
 	select MDIO
 	select CRC32
 	imply PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports 10/40-gigabit Ethernet cards based on
 	  the Solarflare SFC9000-family and SFC9100-family controllers.
 
@@ -32,7 +32,7 @@ config SFC_MTD
 	bool "Solarflare SFC9000/SFC9100-family MTD support"
 	depends on SFC && MTD && !(SFC=y && MTD=m)
 	default y
-	---help---
+	help
 	  This exposes the on-board flash and/or EEPROM as MTD devices
 	  (e.g. /dev/mtd1).  This is required to update the firmware or
 	  the boot configuration under Linux.
@@ -40,14 +40,14 @@ config SFC_MCDI_MON
 	bool "Solarflare SFC9000/SFC9100-family hwmon support"
 	depends on SFC && HWMON && !(SFC=y && HWMON=m)
 	default y
-	---help---
+	help
 	  This exposes the on-board firmware-managed sensors as a
 	  hardware monitor device.
 config SFC_SRIOV
 	bool "Solarflare SFC9000-family SR-IOV support"
 	depends on SFC && PCI_IOV
 	default y
-	---help---
+	help
 	  This enables support for the SFC9000 I/O Virtualization
 	  features, allowing accelerated network performance in
 	  virtualized environments.
@@ -55,7 +55,7 @@ config SFC_MCDI_LOGGING
 	bool "Solarflare SFC9000/SFC9100-family MCDI logging support"
 	depends on SFC
 	default y
-	---help---
+	help
 	  This enables support for tracing of MCDI (Management-Controller-to-
 	  Driver-Interface) commands and responses, allowing debugging of
 	  driver/firmware interaction.  The tracing is actually enabled by
diff --git a/drivers/net/ethernet/sfc/falcon/Kconfig b/drivers/net/ethernet/sfc/falcon/Kconfig
index 20e361950f7d..071154045dc2 100644
--- a/drivers/net/ethernet/sfc/falcon/Kconfig
+++ b/drivers/net/ethernet/sfc/falcon/Kconfig
@@ -6,7 +6,7 @@ config SFC_FALCON
 	select CRC32
 	select I2C
 	select I2C_ALGOBIT
-	---help---
+	help
 	  This driver supports 10-gigabit Ethernet cards based on
 	  the Solarflare SFC4000 controller.
 
@@ -16,7 +16,7 @@ config SFC_FALCON_MTD
 	bool "Solarflare SFC4000 MTD support"
 	depends on SFC_FALCON && MTD && !(SFC_FALCON=y && MTD=m)
 	default y
-	---help---
+	help
 	  This exposes the on-board flash and/or EEPROM as MTD devices
 	  (e.g. /dev/mtd1).  This is required to update the boot
 	  configuration under Linux.
diff --git a/drivers/net/ethernet/sgi/Kconfig b/drivers/net/ethernet/sgi/Kconfig
index bc26fa0d196f..af66bb0a20d1 100644
--- a/drivers/net/ethernet/sgi/Kconfig
+++ b/drivers/net/ethernet/sgi/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_SGI
 	bool "SGI devices"
 	default y
 	depends on (PCI && SGI_MFD_IOC3) ||  SGI_IP32
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config SGI_IOC3_ETH
 	select CRC16
 	select CRC32
 	select MII
-	---help---
+	help
 	  If you have a network (Ethernet) card of this type, say Y here.
 
 config SGI_O2MACE_ETH
diff --git a/drivers/net/ethernet/silan/Kconfig b/drivers/net/ethernet/silan/Kconfig
index 71929d148c3c..7ed08d588ac2 100644
--- a/drivers/net/ethernet/silan/Kconfig
+++ b/drivers/net/ethernet/silan/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_SILAN
 	bool "Silan devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config SC92031
 	tristate "Silan SC92031 PCI Fast Ethernet Adapter driver"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This is a driver for the Fast Ethernet PCI network cards based on
 	  the Silan SC92031 chip (sometimes also called Rsltek 8139D). If you
 	  have one of these, say Y here.
diff --git a/drivers/net/ethernet/sis/Kconfig b/drivers/net/ethernet/sis/Kconfig
index d848ab0349a7..775d76d9890e 100644
--- a/drivers/net/ethernet/sis/Kconfig
+++ b/drivers/net/ethernet/sis/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_SIS
 	bool "Silicon Integrated Systems (SiS) devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config SIS900
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This is a driver for the Fast Ethernet PCI network cards based on
 	  the SiS 900 and SiS 7016 chips. The SiS 900 core is also embedded in
 	  SiS 630 and SiS 540 chipsets.
@@ -38,7 +38,7 @@ config SIS190
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  Say Y here if you have a SiS 190 PCI Fast Ethernet adapter or
 	  a SiS 191 PCI Gigabit Ethernet adapter. Both are expected to
 	  appear in lan on motherboard designs which are based on SiS 965
diff --git a/drivers/net/ethernet/smsc/Kconfig b/drivers/net/ethernet/smsc/Kconfig
index 4d2d91ec8b41..6293b1e6c772 100644
--- a/drivers/net/ethernet/smsc/Kconfig
+++ b/drivers/net/ethernet/smsc/Kconfig
@@ -9,7 +9,7 @@ config NET_VENDOR_SMSC
 	depends on ARM || ARM64 || ATARI_ETHERNAT || COLDFIRE || \
 		   ISA || MAC || MIPS || NIOS2 || PCI || \
 		   PCMCIA || SUPERH || XTENSA || H8300
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config SMC9194
 	tristate "SMC 9194 support"
 	depends on ISA
 	select CRC32
-	---help---
+	help
 	  This is support for the SMC9xxx based Ethernet cards. Choose this
 	  option if you have a DELL laptop with the docking station, or
 	  another SMC9192/9194 based chipset.  Say Y if you want it compiled
@@ -40,7 +40,7 @@ config SMC91X
 	depends on !OF || GPIOLIB
 	depends on ARM || ARM64 || ATARI_ETHERNAT || COLDFIRE || \
 		   MIPS || NIOS2 || SUPERH || XTENSA || H8300
-	---help---
+	help
 	  This is a driver for SMC's 91x series of Ethernet chipsets,
 	  including the SMC91C94 and the SMC91C111. Say Y if you want it
 	  compiled into the kernel, and read the file
@@ -56,7 +56,7 @@ config PCMCIA_SMC91C92
 	depends on PCMCIA
 	select CRC32
 	select MII
-	---help---
+	help
 	  Say Y here if you intend to attach an SMC 91Cxx compatible PCMCIA
 	  (PC-card) Ethernet or Fast Ethernet card to your computer.
 
@@ -68,7 +68,7 @@ config EPIC100
 	depends on PCI
 	select CRC32
 	select MII
-	---help---
+	help
 	  This driver is for the SMC EtherPower II 9432 PCI Ethernet NIC,
 	  which is based on the SMC83c17x (EPIC/100).
 	  More specific information and updates are available from
@@ -79,7 +79,7 @@ config SMC911X
 	select CRC32
 	select MII
 	depends on (ARM || SUPERH)
-	---help---
+	help
 	  This is a driver for SMSC's LAN911x series of Ethernet chipsets
 	  including the new LAN9115, LAN9116, LAN9117, and LAN9118.
 	  Say Y here if you want it compiled into the kernel.
@@ -94,7 +94,7 @@ config SMSC911X
 	select CRC32
 	select MII
 	select PHYLIB
-	---help---
+	help
 	  Say Y here if you want support for SMSC LAN911x and LAN921x families
 	  of ethernet controllers.
 
@@ -104,7 +104,7 @@ config SMSC911X
 config SMSC911X_ARCH_HOOKS
 	def_bool n
 	depends on SMSC911X
-	---help---
+	help
 	  If the arch enables this, it allows the arch to implement various
 	  hooks for more comprehensive interrupt control and also to override
 	  the source of the MAC address.
@@ -115,7 +115,7 @@ config SMSC9420
 	select CRC32
 	select PHYLIB
 	select SMSC_PHY
-	---help---
+	help
 	  This is a driver for SMSC's LAN9420 PCI ethernet adapter.
 	  Say Y here if you want it compiled into the kernel.
 
diff --git a/drivers/net/ethernet/socionext/Kconfig b/drivers/net/ethernet/socionext/Kconfig
index 95e99baf3f45..48298389851d 100644
--- a/drivers/net/ethernet/socionext/Kconfig
+++ b/drivers/net/ethernet/socionext/Kconfig
@@ -2,7 +2,7 @@
 config NET_VENDOR_SOCIONEXT
 	bool "Socionext ethernet drivers"
 	default y
-	---help---
+	help
 	  Option to select ethernet drivers for Socionext platforms.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -18,7 +18,7 @@ config SNI_AVE
 	depends on HAS_IOMEM
 	select MFD_SYSCON
 	select PHYLIB
-	---help---
+	help
 	  Driver for gigabit ethernet MACs, called AVE, in the
 	  Socionext UniPhier family.
 
@@ -28,7 +28,7 @@ config SNI_NETSEC
 	select PHYLIB
 	select PAGE_POOL
 	select MII
-	---help---
+	help
 	  Enable to add support for the SocioNext NetSec Gigabit Ethernet
 	  controller + PHY, as found on the Synquacer SC2A11 SoC
 
diff --git a/drivers/net/ethernet/stmicro/Kconfig b/drivers/net/ethernet/stmicro/Kconfig
index 39ef86360417..cc136b4c9afd 100644
--- a/drivers/net/ethernet/stmicro/Kconfig
+++ b/drivers/net/ethernet/stmicro/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_STMICRO
 	bool "STMicroelectronics devices"
 	default y
 	depends on HAS_IOMEM
-	---help---
+	help
 	  If you have a network (Ethernet) card based on Synopsys Ethernet IP
 	  Cores, say Y.
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index 36bd2e18f23b..9a47c5aec91a 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -9,7 +9,7 @@ config STMMAC_ETH
 	select CRC32
 	imply PTP_1588_CLOCK
 	select RESET_CONTROLLER
-	---help---
+	help
 	  This is the driver for the Ethernet IPs built around a
 	  Synopsys IP Core.
 
@@ -20,7 +20,7 @@ config STMMAC_SELFTESTS
 	depends on INET
 	depends on STMMAC_ETH
 	default n
-	---help---
+	help
 	  This adds support for STMMAC Selftests using ethtool. Enable this
 	  feature if you are facing problems with your HW and submit the test
 	  results to the netdev Mailing List.
@@ -30,7 +30,7 @@ config STMMAC_PLATFORM
 	depends on STMMAC_ETH
 	select MFD_SYSCON
 	default y
-	---help---
+	help
 	  This selects the platform specific bus support for the stmmac driver.
 	  This is the driver used on several SoCs:
 	  STi, Allwinner, Amlogic Meson, Altera SOCFPGA.
@@ -52,7 +52,7 @@ config DWMAC_DWC_QOS_ETH
 config DWMAC_GENERIC
 	tristate "Generic driver for DWMAC"
 	default STMMAC_PLATFORM
-	---help---
+	help
 	  Generic DWMAC driver for platforms that don't require any
 	  platform specific code to function or is using platform
 	  data for setup.
@@ -84,7 +84,7 @@ config DWMAC_LPC18XX
 	default ARCH_LPC18XX
 	depends on OF && (ARCH_LPC18XX || COMPILE_TEST)
 	select MFD_SYSCON
-	---help---
+	help
 	  Support for NXP LPC18xx/43xx DWMAC Ethernet.
 
 config DWMAC_MEDIATEK
@@ -155,7 +155,7 @@ config DWMAC_STI
 	default ARCH_STI
 	depends on OF && (ARCH_STI || COMPILE_TEST)
 	select MFD_SYSCON
-	---help---
+	help
 	  Support for ethernet controller on STi SOCs.
 
 	  This selects STi SoC glue layer support for the stmmac
@@ -167,7 +167,7 @@ config DWMAC_STM32
 	default ARCH_STM32
 	depends on OF && HAS_IOMEM && (ARCH_STM32 || COMPILE_TEST)
 	select MFD_SYSCON
-	---help---
+	help
 	  Support for ethernet controller on STM32 SOCs.
 
 	  This selects STM32 SoC glue layer support for the stmmac
@@ -178,7 +178,7 @@ config DWMAC_SUNXI
 	tristate "Allwinner GMAC support"
 	default ARCH_SUNXI
 	depends on OF && (ARCH_SUNXI || COMPILE_TEST)
-	---help---
+	help
 	  Support for Allwinner A20/A31 GMAC ethernet controllers.
 
 	  This selects Allwinner SoC glue layer support for the
@@ -190,7 +190,7 @@ config DWMAC_SUN8I
 	default ARCH_SUNXI
 	depends on OF && (ARCH_SUNXI || COMPILE_TEST)
 	select MDIO_BUS_MUX
-	---help---
+	help
 	  Support for Allwinner H3 A83T A64 EMAC ethernet controllers.
 
 	  This selects Allwinner SoC glue layer support for the
@@ -202,7 +202,7 @@ config DWMAC_IMX8
 	default ARCH_MXC
 	depends on OF && (ARCH_MXC || COMPILE_TEST)
 	select MFD_SYSCON
-	---help---
+	help
 	  Support for ethernet controller on NXP i.MX8 SOCs.
 
 	  This selects NXP SoC glue layer support for the stmmac
@@ -216,7 +216,7 @@ config DWMAC_INTEL
 	default X86
 	depends on X86 && STMMAC_ETH && PCI
 	depends on COMMON_CLK
-	---help---
+	help
 	  This selects the Intel platform specific bus support for the
 	  stmmac driver. This driver is used for Intel Quark/EHL/TGL.
 
@@ -224,7 +224,7 @@ config STMMAC_PCI
 	tristate "STMMAC PCI bus support"
 	depends on STMMAC_ETH && PCI
 	depends on COMMON_CLK
-	---help---
+	help
 	  This selects the platform specific bus support for the stmmac driver.
 	  This driver was tested on XLINX XC2V3000 FF1152AMT0221
 	  D1215994A VIRTEX FPGA board and SNPS QoS IPK Prototyping Kit.
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
index 5010af7dab4a..3c5df5eeed6c 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c
@@ -225,7 +225,7 @@ static int imx_dwmac_probe(struct platform_device *pdev)
 
 	dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL);
 	if (!dwmac)
-		return PTR_ERR(dwmac);
+		return -ENOMEM;
 
 	plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac);
 	if (IS_ERR(plat_dat))
diff --git a/drivers/net/ethernet/sun/Kconfig b/drivers/net/ethernet/sun/Kconfig
index 7b982e02ea3a..309de38a7530 100644
--- a/drivers/net/ethernet/sun/Kconfig
+++ b/drivers/net/ethernet/sun/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_SUN
 	bool "Sun devices"
 	default y
 	depends on SUN3 || SBUS || PCI || SUN_LDOMS
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config HAPPYMEAL
 	tristate "Sun Happy Meal 10/100baseT support"
 	depends on (SBUS || PCI)
 	select CRC32
-	---help---
+	help
 	  This driver supports the "hme" interface present on most Ultra
 	  systems and as an option on older Sbus systems. This driver supports
 	  both PCI and Sbus devices. This driver also supports the "qfe" quad
@@ -34,7 +34,7 @@ config SUNBMAC
 	tristate "Sun BigMAC 10/100baseT support"
 	depends on SBUS
 	select CRC32
-	---help---
+	help
 	  This driver supports the "be" interface available as an Sbus option.
 	  This is Sun's older 100baseT Ethernet device.
 
@@ -45,7 +45,7 @@ config SUNQE
 	tristate "Sun QuadEthernet support"
 	depends on SBUS
 	select CRC32
-	---help---
+	help
 	  This driver supports the "qe" 10baseT Ethernet device, available as
 	  an Sbus option. Note that this is not the same as Quad FastEthernet
 	  "qfe" which is supported by the Happy Meal driver instead.
@@ -58,7 +58,7 @@ config SUNGEM
 	depends on PCI
 	select CRC32
 	select SUNGEM_PHY
-	---help---
+	help
 	  Support for the Sun GEM chip, aka Sun GigabitEthernet/P 2.0.  See also
 	  <http://docs.oracle.com/cd/E19455-01/806-3985-10/806-3985-10.pdf>.
 
@@ -66,7 +66,7 @@ config CASSINI
 	tristate "Sun Cassini support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  Support for the Sun Cassini chip, aka Sun GigaSwift Ethernet. See also
 	  <http://docs.oracle.com/cd/E19113-01/giga.ether.pci/817-4341-10/817-4341-10.pdf>.
 
@@ -80,7 +80,7 @@ config SUNVNET
 	default m
 	depends on SUN_LDOMS
 	depends on SUNVNET_COMMON
-	---help---
+	help
 	  Support for virtual network devices under Sun Logical Domains.
 
 config LDMVSW
@@ -88,7 +88,7 @@ config LDMVSW
 	default m
 	depends on SUN_LDOMS
 	depends on SUNVNET_COMMON
-	---help---
+	help
 	  Support for virtual switch devices under Sun4v Logical Domains.
 	  This driver adds a network interface for every vsw-port node
 	  found in the machine description of a service domain.
@@ -100,7 +100,7 @@ config NIU
 	tristate "Sun Neptune 10Gbit Ethernet support"
 	depends on PCI
 	select CRC32
-	---help---
+	help
 	  This enables support for cards based upon Sun's
 	  Neptune chipset.
 
diff --git a/drivers/net/ethernet/sun/sunbmac.c b/drivers/net/ethernet/sun/sunbmac.c
index c5add0b45eed..34fdbc6d6031 100644
--- a/drivers/net/ethernet/sun/sunbmac.c
+++ b/drivers/net/ethernet/sun/sunbmac.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/pgtable.h>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
@@ -34,7 +35,6 @@
 #include <asm/io.h>
 #include <asm/openprom.h>
 #include <asm/oplib.h>
-#include <asm/pgtable.h>
 
 #include "sunbmac.h"
 
diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c
index f0fe7bb2a750..54b53dbdb33c 100644
--- a/drivers/net/ethernet/sun/sunhme.c
+++ b/drivers/net/ethernet/sun/sunhme.c
@@ -52,7 +52,6 @@
 #endif
 #include <linux/uaccess.h>
 
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 
 #ifdef CONFIG_PCI
diff --git a/drivers/net/ethernet/sun/sunqe.c b/drivers/net/ethernet/sun/sunqe.c
index 2102b95ec347..577cd9753d8e 100644
--- a/drivers/net/ethernet/sun/sunqe.c
+++ b/drivers/net/ethernet/sun/sunqe.c
@@ -28,6 +28,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/pgtable.h>
 
 #include <asm/io.h>
 #include <asm/dma.h>
@@ -36,7 +37,6 @@
 #include <asm/openprom.h>
 #include <asm/oplib.h>
 #include <asm/auxio.h>
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 
 #include "sunqe.h"
diff --git a/drivers/net/ethernet/synopsys/Kconfig b/drivers/net/ethernet/synopsys/Kconfig
index 9e199772c1d7..f2a4287c48b8 100644
--- a/drivers/net/ethernet/synopsys/Kconfig
+++ b/drivers/net/ethernet/synopsys/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_SYNOPSYS
 	bool "Synopsys devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) device belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config DWC_XLGMAC
 	depends on HAS_IOMEM && HAS_DMA
 	select BITREVERSE
 	select CRC32
-	---help---
+	help
 	  This driver supports the Synopsys DesignWare Cores Enterprise
 	  Ethernet (dwc-xlgmac).
 
@@ -30,7 +30,7 @@ if DWC_XLGMAC
 config DWC_XLGMAC_PCI
 	tristate "XLGMAC PCI bus support"
 	depends on DWC_XLGMAC && PCI
-	---help---
+	help
 	  This selects the pci bus support for the dwc-xlgmac driver.
 	  This driver was tested on Synopsys XLGMAC IP Prototyping Kit.
 
diff --git a/drivers/net/ethernet/tehuti/Kconfig b/drivers/net/ethernet/tehuti/Kconfig
index 8ad1526f4bdd..8735633765a1 100644
--- a/drivers/net/ethernet/tehuti/Kconfig
+++ b/drivers/net/ethernet/tehuti/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_TEHUTI
 	bool "Tehuti devices"
 	default y
 	depends on PCI
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_TEHUTI
 config TEHUTI
 	tristate "Tehuti Networks 10G Ethernet"
 	depends on PCI
-	---help---
+	help
 	  Tehuti Networks 10G Ethernet NIC
 
 endif # NET_VENDOR_TEHUTI
diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig
index 182d10f171f6..50f55364d4fb 100644
--- a/drivers/net/ethernet/ti/Kconfig
+++ b/drivers/net/ethernet/ti/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_TI
 	bool "Texas Instruments (TI) devices"
 	default y
 	depends on PCI || EISA || AR7 || ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || ARCH_K3
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config TI_DAVINCI_EMAC
 	select TI_DAVINCI_MDIO
 	select PHYLIB
 	select GENERIC_ALLOCATOR
-	---help---
+	help
 	  This driver supports TI's DaVinci Ethernet .
 
 	  To compile this driver as a module, choose M here: the module
@@ -33,7 +33,7 @@ config TI_DAVINCI_MDIO
 	tristate "TI DaVinci MDIO Support"
 	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
 	select PHYLIB
-	---help---
+	help
 	  This driver supports TI's DaVinci MDIO module.
 
 	  To compile this driver as a module, choose M here: the module
@@ -42,7 +42,7 @@ config TI_DAVINCI_MDIO
 config TI_CPSW_PHY_SEL
 	bool "TI CPSW Phy mode Selection (DEPRECATED)"
 	default n
-	---help---
+	help
 	  This driver supports configuring of the phy mode connected to
 	  the CPSW. DEPRECATED: use PHY_TI_GMII_SEL.
 
@@ -55,7 +55,7 @@ config TI_CPSW
 	select PAGE_POOL
 	select REGMAP
 	imply PHY_TI_GMII_SEL
-	---help---
+	help
 	  This driver supports TI's CPSW Ethernet Switch.
 
 	  To compile this driver as a module, choose M here: the module
@@ -83,7 +83,7 @@ config TI_CPTS
 	depends on ARCH_OMAP2PLUS || ARCH_KEYSTONE || COMPILE_TEST
 	depends on COMMON_CLK
 	depends on PTP_1588_CLOCK
-	---help---
+	help
 	  This driver supports the Common Platform Time Sync unit of
 	  the CPSW Ethernet Switch and Keystone 2 1g/10g Switch Subsystem.
 	  The unit can time stamp PTP UDP/IPv4 and Layer 2 packets, and the
@@ -133,7 +133,7 @@ config TI_KEYSTONE_NETCP
 	depends on OF
 	depends on KEYSTONE_NAVIGATOR_DMA && KEYSTONE_NAVIGATOR_QMSS
 	depends on TI_CPTS || !TI_CPTS
-	---help---
+	help
 	  This driver supports TI's Keystone NETCP Core.
 
 	  To compile this driver as a module, choose M here: the module
@@ -142,7 +142,7 @@ config TI_KEYSTONE_NETCP
 config TI_KEYSTONE_NETCP_ETHSS
 	depends on TI_KEYSTONE_NETCP
 	tristate "TI Keystone NETCP Ethernet subsystem Support"
-	---help---
+	help
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called keystone_netcp_ethss.
@@ -150,7 +150,7 @@ config TI_KEYSTONE_NETCP_ETHSS
 config TLAN
 	tristate "TI ThunderLAN support"
 	depends on (PCI || EISA)
-	---help---
+	help
 	  If you have a PCI Ethernet network card based on the ThunderLAN chip
 	  which is supported by this driver, say Y here.
 
@@ -168,7 +168,7 @@ config CPMAC
 	tristate "TI AR7 CPMAC Ethernet support"
 	depends on AR7
 	select PHYLIB
-	---help---
+	help
 	  TI AR7 CPMAC Ethernet support
 
 endif # NET_VENDOR_TI
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index 87a4775ed53a..1492648247d9 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -1981,7 +1981,7 @@ MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
 
 static int am65_cpsw_nuss_probe(struct platform_device *pdev)
 {
-	struct cpsw_ale_params ale_params;
+	struct cpsw_ale_params ale_params = { 0 };
 	const struct of_device_id *of_id;
 	struct device *dev = &pdev->dev;
 	struct am65_cpsw_common *common;
diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c
index 8dc6be11b2ff..9ad872bfae3a 100644
--- a/drivers/net/ethernet/ti/cpsw_ale.c
+++ b/drivers/net/ethernet/ti/cpsw_ale.c
@@ -604,10 +604,44 @@ void cpsw_ale_set_unreg_mcast(struct cpsw_ale *ale, int unreg_mcast_mask,
 	}
 }
 
+static void cpsw_ale_vlan_set_unreg_mcast(struct cpsw_ale *ale, u32 *ale_entry,
+					  int allmulti)
+{
+	int unreg_mcast;
+
+	unreg_mcast =
+		cpsw_ale_get_vlan_unreg_mcast(ale_entry,
+					      ale->vlan_field_bits);
+	if (allmulti)
+		unreg_mcast |= ALE_PORT_HOST;
+	else
+		unreg_mcast &= ~ALE_PORT_HOST;
+	cpsw_ale_set_vlan_unreg_mcast(ale_entry, unreg_mcast,
+				      ale->vlan_field_bits);
+}
+
+static void
+cpsw_ale_vlan_set_unreg_mcast_idx(struct cpsw_ale *ale, u32 *ale_entry,
+				  int allmulti)
+{
+	int unreg_mcast;
+	int idx;
+
+	idx = cpsw_ale_get_vlan_unreg_mcast_idx(ale_entry);
+
+	unreg_mcast = readl(ale->params.ale_regs + ALE_VLAN_MASK_MUX(idx));
+
+	if (allmulti)
+		unreg_mcast |= ALE_PORT_HOST;
+	else
+		unreg_mcast &= ~ALE_PORT_HOST;
+
+	writel(unreg_mcast, ale->params.ale_regs + ALE_VLAN_MASK_MUX(idx));
+}
+
 void cpsw_ale_set_allmulti(struct cpsw_ale *ale, int allmulti, int port)
 {
 	u32 ale_entry[ALE_ENTRY_WORDS];
-	int unreg_mcast = 0;
 	int type, idx;
 
 	for (idx = 0; idx < ale->params.ale_entries; idx++) {
@@ -624,15 +658,12 @@ void cpsw_ale_set_allmulti(struct cpsw_ale *ale, int allmulti, int port)
 		if (port != -1 && !(vlan_members & BIT(port)))
 			continue;
 
-		unreg_mcast =
-			cpsw_ale_get_vlan_unreg_mcast(ale_entry,
-						      ale->vlan_field_bits);
-		if (allmulti)
-			unreg_mcast |= ALE_PORT_HOST;
+		if (!ale->params.nu_switch_ale)
+			cpsw_ale_vlan_set_unreg_mcast(ale, ale_entry, allmulti);
 		else
-			unreg_mcast &= ~ALE_PORT_HOST;
-		cpsw_ale_set_vlan_unreg_mcast(ale_entry, unreg_mcast,
-					      ale->vlan_field_bits);
+			cpsw_ale_vlan_set_unreg_mcast_idx(ale, ale_entry,
+							  allmulti);
+
 		cpsw_ale_write(ale, idx, ale_entry);
 	}
 }
diff --git a/drivers/net/ethernet/toshiba/Kconfig b/drivers/net/ethernet/toshiba/Kconfig
index 9ccdf032404e..701e9b7c1c3b 100644
--- a/drivers/net/ethernet/toshiba/Kconfig
+++ b/drivers/net/ethernet/toshiba/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_TOSHIBA
 	bool "Toshiba devices"
 	default y
 	depends on PCI && (PPC_IBM_CELL_BLADE || MIPS) || PPC_PS3
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -21,7 +21,7 @@ config GELIC_NET
 	tristate "PS3 Gigabit Ethernet driver"
 	depends on PPC_PS3
 	select PS3_SYS_MANAGER
-	---help---
+	help
 	  This driver supports the network device on the PS3 game
 	  console.  This driver has built-in support for Ethernet.
 
@@ -32,7 +32,7 @@ config GELIC_WIRELESS
 	bool "PS3 Wireless support"
 	depends on GELIC_NET && WLAN
 	select WIRELESS_EXT
-	---help---
+	help
 	  This option adds the support for the wireless feature of PS3.
 	  If you have the wireless-less model of PS3 or have no plan to
 	  use wireless feature, disabling this option saves memory.  As
@@ -44,7 +44,7 @@ config SPIDER_NET
 	depends on PCI && PPC_IBM_CELL_BLADE
 	select FW_LOADER
 	select SUNGEM_PHY
-	---help---
+	help
 	  This driver supports the Gigabit Ethernet chips present on the
 	  Cell Processor-Based Blades from IBM.
 
diff --git a/drivers/net/ethernet/toshiba/ps3_gelic_net.c b/drivers/net/ethernet/toshiba/ps3_gelic_net.c
index 310e6839c6e5..d9a5722f561b 100644
--- a/drivers/net/ethernet/toshiba/ps3_gelic_net.c
+++ b/drivers/net/ethernet/toshiba/ps3_gelic_net.c
@@ -382,8 +382,6 @@ static int gelic_descr_prepare_rx(struct gelic_card *card,
 	descr->skb = dev_alloc_skb(bufsize + GELIC_NET_RXBUF_ALIGN - 1);
 	if (!descr->skb) {
 		descr->buf_addr = 0; /* tell DMAC don't touch memory */
-		dev_info(ctodev(card),
-			 "%s:allocate skb failed !!\n", __func__);
 		return -ENOMEM;
 	}
 	descr->buf_size = cpu_to_be32(bufsize);
diff --git a/drivers/net/ethernet/tundra/Kconfig b/drivers/net/ethernet/tundra/Kconfig
index 5c909df0c3b9..edd52b2bd135 100644
--- a/drivers/net/ethernet/tundra/Kconfig
+++ b/drivers/net/ethernet/tundra/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_TUNDRA
 	bool "Tundra devices"
 	default y
 	depends on TSI108_BRIDGE
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_TUNDRA
 config TSI108_ETH
 	tristate "Tundra TSI108 gigabit Ethernet support"
 	depends on TSI108_BRIDGE
-	---help---
+	help
 	  This driver supports Tundra TSI108 gigabit Ethernet ports.
 	  To compile this driver as a module, choose M here: the module
 	  will be called tsi108_eth.
diff --git a/drivers/net/ethernet/via/Kconfig b/drivers/net/ethernet/via/Kconfig
index 6cff5f7d57c4..da287ef65be7 100644
--- a/drivers/net/ethernet/via/Kconfig
+++ b/drivers/net/ethernet/via/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_VIA
 	bool "VIA devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config VIA_RHINE
 	depends on HAS_DMA
 	select CRC32
 	select MII
-	---help---
+	help
 	  If you have a VIA "Rhine" based network card (Rhine-I (VT86C100A),
 	  Rhine-II (VT6102), or Rhine-III (VT6105)), say Y here. Rhine-type
 	  Ethernet functions can also be found integrated on South Bridges
@@ -35,7 +35,7 @@ config VIA_RHINE
 config VIA_RHINE_MMIO
 	bool "Use MMIO instead of PIO"
 	depends on VIA_RHINE
-	---help---
+	help
 	  This instructs the driver to use PCI shared memory (MMIO) instead of
 	  programmed I/O ports (PIO). Enabling this gives an improvement in
 	  processing time in parts of the driver.
@@ -49,7 +49,7 @@ config VIA_VELOCITY
 	select CRC32
 	select CRC_CCITT
 	select MII
-	---help---
+	help
 	  If you have a VIA "Velocity" based network card say Y here.
 
 	  To compile this driver as a module, choose M here. The module
diff --git a/drivers/net/ethernet/wiznet/Kconfig b/drivers/net/ethernet/wiznet/Kconfig
index 0422775e1659..4bac2ad2d6a1 100644
--- a/drivers/net/ethernet/wiznet/Kconfig
+++ b/drivers/net/ethernet/wiznet/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_WIZNET
 	bool "WIZnet devices"
 	depends on HAS_IOMEM
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_WIZNET
 config WIZNET_W5100
 	tristate "WIZnet W5100 Ethernet support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  Support for WIZnet W5100 chips.
 
 	  W5100 is a single chip with integrated 10/100 Ethernet MAC,
@@ -33,7 +33,7 @@ config WIZNET_W5100
 config WIZNET_W5300
 	tristate "WIZnet W5300 Ethernet support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  Support for WIZnet W5300 chips.
 
 	  W5300 is a single chip with integrated 10/100 Ethernet MAC,
@@ -50,20 +50,20 @@ choice
 
 config WIZNET_BUS_DIRECT
 	bool "Direct address bus mode"
-	---help---
+	help
 	  In direct address mode host system can directly access all registers
 	  after mapping to Memory-Mapped I/O space.
 
 config WIZNET_BUS_INDIRECT
 	bool "Indirect address bus mode"
-	---help---
+	help
 	  In indirect address mode host system indirectly accesses registers
 	  using Indirect Mode Address Register and Indirect Mode Data Register,
 	  which are directly mapped to Memory-Mapped I/O space.
 
 config WIZNET_BUS_ANY
 	bool "Select interface mode in runtime"
-	---help---
+	help
 	  If interface mode is unknown in compile time, it can be selected
 	  in runtime from board/platform resources configuration.
 
@@ -74,7 +74,7 @@ config WIZNET_W5100_SPI
 	tristate "WIZnet W5100/W5200/W5500 Ethernet support for SPI mode"
 	depends on WIZNET_BUS_ANY && WIZNET_W5100
 	depends on SPI
-	---help---
+	help
 	  In SPI mode host system accesses registers using SPI protocol
 	  (mode 0) on the SPI bus.
 
diff --git a/drivers/net/ethernet/xilinx/Kconfig b/drivers/net/ethernet/xilinx/Kconfig
index 0810af8193cb..d0d0d4fe9d40 100644
--- a/drivers/net/ethernet/xilinx/Kconfig
+++ b/drivers/net/ethernet/xilinx/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_XILINX
 	bool "Xilinx devices"
 	default y
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,20 +20,20 @@ config XILINX_EMACLITE
 	tristate "Xilinx 10/100 Ethernet Lite support"
 	depends on PPC32 || MICROBLAZE || ARCH_ZYNQ || MIPS
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the 10/100 Ethernet Lite from Xilinx.
 
 config XILINX_AXI_EMAC
 	tristate "Xilinx 10/100/1000 AXI Ethernet support"
 	select PHYLINK
-	---help---
+	help
 	  This driver supports the 10/100/1000 Ethernet from Xilinx for the
 	  AXI bus interface used in Xilinx Virtex FPGAs and Soc's.
 
 config XILINX_LL_TEMAC
 	tristate "Xilinx LL TEMAC (LocalLink Tri-mode Ethernet MAC) driver"
 	select PHYLIB
-	---help---
+	help
 	  This driver supports the Xilinx 10/100/1000 LocalLink TEMAC
 	  core used in Xilinx Spartan and Virtex FPGAs
 
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h
index fbaf3c987d9c..f34c7903ff52 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet.h
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h
@@ -186,7 +186,7 @@
 #define XAE_RAF_TXVSTRPMODE_MASK	0x00000180 /* Tx VLAN STRIP mode */
 #define XAE_RAF_RXVSTRPMODE_MASK	0x00000600 /* Rx VLAN STRIP mode */
 #define XAE_RAF_NEWFNCENBL_MASK		0x00000800 /* New function mode */
-/* Exteneded Multicast Filtering mode */
+/* Extended Multicast Filtering mode */
 #define XAE_RAF_EMULTIFLTRENBL_MASK	0x00001000
 #define XAE_RAF_STATSRST_MASK		0x00002000 /* Stats. Counter Reset */
 #define XAE_RAF_RXBADFRMEN_MASK		0x00004000 /* Recv Bad Frame Enable */
diff --git a/drivers/net/ethernet/xircom/Kconfig b/drivers/net/ethernet/xircom/Kconfig
index ad5390079b13..7497b9bea511 100644
--- a/drivers/net/ethernet/xircom/Kconfig
+++ b/drivers/net/ethernet/xircom/Kconfig
@@ -7,7 +7,7 @@ config NET_VENDOR_XIRCOM
 	bool "Xircom devices"
 	default y
 	depends on PCMCIA
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ if NET_VENDOR_XIRCOM
 config PCMCIA_XIRC2PS
 	tristate "Xircom 16-bit PCMCIA support"
 	depends on PCMCIA
-	---help---
+	help
 	  Say Y here if you intend to attach a Xircom 16-bit PCMCIA (PC-card)
 	  Ethernet or Fast Ethernet card to your computer.
 
diff --git a/drivers/net/ethernet/xscale/Kconfig b/drivers/net/ethernet/xscale/Kconfig
index 98aa7b8ddb06..7b83a6e5d894 100644
--- a/drivers/net/ethernet/xscale/Kconfig
+++ b/drivers/net/ethernet/xscale/Kconfig
@@ -8,7 +8,7 @@ config NET_VENDOR_XSCALE
 	default y
 	depends on NET_VENDOR_INTEL && (ARM && ARCH_IXP4XX && \
 		   IXP4XX_NPE && IXP4XX_QMGR)
-	---help---
+	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question does not directly affect the
@@ -23,7 +23,7 @@ config IXP4XX_ETH
 	depends on ARM && ARCH_IXP4XX && IXP4XX_NPE && IXP4XX_QMGR
 	select PHYLIB
 	select NET_PTP_CLASSIFY
-	---help---
+	help
 	  Say Y here if you want to use built-in Ethernet ports
 	  on IXP4xx processor.
 
diff --git a/drivers/net/fddi/Kconfig b/drivers/net/fddi/Kconfig
index da4f58eed08f..60cc7524520c 100644
--- a/drivers/net/fddi/Kconfig
+++ b/drivers/net/fddi/Kconfig
@@ -6,7 +6,7 @@
 config FDDI
 	tristate "FDDI driver support"
 	depends on PCI || EISA || TC
-	---help---
+	help
 	  Fiber Distributed Data Interface is a high speed local area network
 	  design; essentially a replacement for high speed Ethernet. FDDI can
 	  run over copper or fiber. If you are connected to such a network and
@@ -30,7 +30,7 @@ config DEFZA
 config DEFXX
 	tristate "Digital DEFTA/DEFEA/DEFPA adapter support"
 	depends on FDDI && (PCI || EISA || TC)
-	---help---
+	help
 	  This is support for the DIGITAL series of TURBOchannel (DEFTA),
 	  EISA (DEFEA) and PCI (DEFPA) controllers which can connect you
 	  to a local FDDI network.
@@ -44,7 +44,7 @@ config DEFXX_MMIO
 	depends on DEFXX
 	default n if PCI || EISA
 	default y
-	---help---
+	help
 	  This instructs the driver to use EISA or PCI memory-mapped I/O
 	  (MMIO) as appropriate instead of programmed I/O ports (PIO).
 	  Enabling this gives an improvement in processing time in parts
@@ -58,7 +58,7 @@ config SKFP
 	tristate "SysKonnect FDDI PCI support"
 	depends on FDDI && PCI
 	select BITREVERSE
-	---help---
+	help
 	  Say Y here if you have a SysKonnect FDDI PCI adapter.
 	  The following adapters are supported by this driver:
 	  - SK-5521 (SK-NET FDDI-UP)
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index 6b461be1820b..75266580b586 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -987,9 +987,10 @@ static netdev_tx_t geneve_xmit(struct sk_buff *skb, struct net_device *dev)
 	if (geneve->collect_md) {
 		info = skb_tunnel_info(skb);
 		if (unlikely(!info || !(info->mode & IP_TUNNEL_INFO_TX))) {
-			err = -EINVAL;
 			netdev_dbg(dev, "no tunnel metadata\n");
-			goto tx_error;
+			dev_kfree_skb(skb);
+			dev->stats.tx_dropped++;
+			return NETDEV_TX_OK;
 		}
 	} else {
 		info = &geneve->info;
@@ -1006,7 +1007,7 @@ static netdev_tx_t geneve_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	if (likely(!err))
 		return NETDEV_TX_OK;
-tx_error:
+
 	dev_kfree_skb(skb);
 
 	if (err == -ELOOP)
diff --git a/drivers/net/hamradio/Kconfig b/drivers/net/hamradio/Kconfig
index f4500f04147d..70f754abbeca 100644
--- a/drivers/net/hamradio/Kconfig
+++ b/drivers/net/hamradio/Kconfig
@@ -3,7 +3,7 @@ config MKISS
 	tristate "Serial port KISS driver"
 	depends on AX25 && TTY
 	select CRC16
-	---help---
+	help
 	  KISS is a protocol used for the exchange of data between a computer
 	  and a Terminal Node Controller (a small embedded system commonly
 	  used for networking over AX.25 amateur radio connections; it
@@ -20,7 +20,7 @@ config MKISS
 config 6PACK
 	tristate "Serial port 6PACK driver"
 	depends on AX25 && TTY
-	---help---
+	help
 	  6pack is a transmission protocol for the data exchange between your
 	  PC and your TNC (the Terminal Node Controller acts as a kind of
 	  modem connecting your computer's serial port to your radio's
@@ -48,7 +48,7 @@ config BPQETHER
 config DMASCC
 	tristate "High-speed (DMA) SCC driver for AX.25"
 	depends on ISA && AX25 && BROKEN_ON_SMP && ISA_DMA_API
-	---help---
+	help
 	  This is a driver for high-speed SCC boards, i.e. those supporting
 	  DMA on one port. You usually use those boards to connect your
 	  computer to an amateur radio modem (such as the WA4DSY 56kbps
@@ -81,7 +81,7 @@ config DMASCC
 config SCC
 	tristate "Z8530 SCC driver"
 	depends on ISA && AX25 && ISA_DMA_API
-	---help---
+	help
 	  These cards are used to connect your Linux box to an amateur radio
 	  in order to communicate with other computers. If you want to use
 	  this, read <file:Documentation/networking/z8530drv.rst> and the
@@ -116,7 +116,7 @@ config BAYCOM_SER_FDX
 	tristate "BAYCOM ser12 fullduplex driver for AX.25"
 	depends on AX25 && !S390
 	select CRC_CCITT
-	---help---
+	help
 	  This is one of two drivers for Baycom style simple amateur radio
 	  modems that connect to a serial interface. The driver supports the
 	  ser12 design in full-duplex mode. In addition, it allows the
@@ -136,7 +136,7 @@ config BAYCOM_SER_HDX
 	tristate "BAYCOM ser12 halfduplex driver for AX.25"
 	depends on AX25 && !S390
 	select CRC_CCITT
-	---help---
+	help
 	  This is one of two drivers for Baycom style simple amateur radio
 	  modems that connect to a serial interface. The driver supports the
 	  ser12 design in half-duplex mode. This is the old driver.  It is
@@ -154,7 +154,7 @@ config BAYCOM_PAR
 	tristate "BAYCOM picpar and par96 driver for AX.25"
 	depends on PARPORT && AX25
 	select CRC_CCITT
-	---help---
+	help
 	  This is a driver for Baycom style simple amateur radio modems that
 	  connect to a parallel interface. The driver supports the picpar and
 	  par96 designs. To configure the driver, use the sethdlc utility
@@ -169,7 +169,7 @@ config BAYCOM_EPP
 	tristate "BAYCOM epp driver for AX.25"
 	depends on PARPORT && AX25 && !64BIT
 	select CRC_CCITT
-	---help---
+	help
 	  This is a driver for Baycom style simple amateur radio modems that
 	  connect to a parallel interface. The driver supports the EPP
 	  designs. To configure the driver, use the sethdlc utility available
diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c
index 60dcaf2a04a9..1ad6085994b1 100644
--- a/drivers/net/hamradio/bpqether.c
+++ b/drivers/net/hamradio/bpqether.c
@@ -113,6 +113,7 @@ static LIST_HEAD(bpq_devices);
  * off into a separate class since they always nest.
  */
 static struct lock_class_key bpq_netdev_xmit_lock_key;
+static struct lock_class_key bpq_netdev_addr_lock_key;
 
 static void bpq_set_lockdep_class_one(struct net_device *dev,
 				      struct netdev_queue *txq,
@@ -123,6 +124,7 @@ static void bpq_set_lockdep_class_one(struct net_device *dev,
 
 static void bpq_set_lockdep_class(struct net_device *dev)
 {
+	lockdep_set_class(&dev->addr_list_lock, &bpq_netdev_addr_lock_key);
 	netdev_for_each_tx_queue(dev, bpq_set_lockdep_class_one, NULL);
 }
 
diff --git a/drivers/net/hamradio/yam.c b/drivers/net/hamradio/yam.c
index 71cdef9fb56b..5ab53e9942f3 100644
--- a/drivers/net/hamradio/yam.c
+++ b/drivers/net/hamradio/yam.c
@@ -1133,6 +1133,7 @@ static int __init yam_init_driver(void)
 		err = register_netdev(dev);
 		if (err) {
 			printk(KERN_WARNING "yam: cannot register net device %s\n", dev->name);
+			free_netdev(dev);
 			goto error;
 		}
 		yam_devs[i] = dev;
diff --git a/drivers/net/hippi/Kconfig b/drivers/net/hippi/Kconfig
index de78504a5798..46b911bb8493 100644
--- a/drivers/net/hippi/Kconfig
+++ b/drivers/net/hippi/Kconfig
@@ -6,7 +6,7 @@
 config HIPPI
 	bool "HIPPI driver support"
 	depends on INET && PCI
-	---help---
+	help
 	  HIgh Performance Parallel Interface (HIPPI) is a 800Mbit/sec and
 	  1600Mbit/sec dual-simplex switched or point-to-point network. HIPPI
 	  can run over copper (25m) or fiber (300m on multi-mode or 10km on
@@ -21,7 +21,7 @@ if HIPPI
 config ROADRUNNER
 	tristate "Essential RoadRunner HIPPI PCI adapter support"
 	depends on PCI
-	---help---
+	help
 	  Say Y here if this is your PCI HIPPI network card.
 
 	  To compile this driver as a module, choose M here: the module
@@ -30,7 +30,7 @@ config ROADRUNNER
 config ROADRUNNER_LARGE_RINGS
 	bool "Use large TX/RX rings"
 	depends on ROADRUNNER
-	---help---
+	help
 	  If you say Y here, the RoadRunner driver will preallocate up to 2 MB
 	  of additional memory to allow for fastest operation, both for
 	  transmitting and receiving. This memory cannot be used by any other
diff --git a/drivers/net/ieee802154/Kconfig b/drivers/net/ieee802154/Kconfig
index c92a62dbf398..0f7c6dc2ed15 100644
--- a/drivers/net/ieee802154/Kconfig
+++ b/drivers/net/ieee802154/Kconfig
@@ -3,7 +3,7 @@ menuconfig IEEE802154_DRIVERS
 	tristate "IEEE 802.15.4 drivers"
 	depends on NETDEVICES && IEEE802154
 	default y
-	---help---
+	help
 	  Say Y here to get to see options for IEEE 802.15.4 Low-Rate
 	  Wireless Personal Area Network device drivers. This option alone
 	  does not add any kernel code.
@@ -14,7 +14,7 @@ menuconfig IEEE802154_DRIVERS
 config IEEE802154_FAKELB
 	depends on IEEE802154_DRIVERS && MAC802154
 	tristate "IEEE 802.15.4 loopback driver"
-	---help---
+	help
 	  Say Y here to enable the fake driver that can emulate a net
 	  of several interconnected radio devices.
 
@@ -26,7 +26,7 @@ config IEEE802154_AT86RF230
 	tristate "AT86RF230/231/233/212 transceiver driver"
 	depends on SPI
 	select REGMAP_SPI
-	---help---
+	help
 	  Say Y here to enable the at86rf230/231/233/212 SPI 802.15.4 wireless
 	  controller.
 
@@ -37,7 +37,7 @@ config IEEE802154_AT86RF230_DEBUGFS
 	depends on IEEE802154_AT86RF230
 	bool "AT86RF230 debugfs interface"
 	depends on DEBUG_FS
-	---help---
+	help
 	  This option compiles debugfs code for the at86rf230 driver.
 
 config IEEE802154_MRF24J40
@@ -45,7 +45,7 @@ config IEEE802154_MRF24J40
 	depends on IEEE802154_DRIVERS && MAC802154
 	depends on SPI
 	select REGMAP_SPI
-	---help---
+	help
 	  Say Y here to enable the MRF24J20 SPI 802.15.4 wireless
 	  controller.
 
@@ -56,7 +56,7 @@ config IEEE802154_CC2520
 	depends on IEEE802154_DRIVERS && MAC802154
 	tristate "CC2520 transceiver driver"
 	depends on SPI
-	---help---
+	help
 	  Say Y here to enable the CC2520 SPI 802.15.4 wireless
 	  controller.
 
@@ -66,7 +66,7 @@ config IEEE802154_CC2520
 config IEEE802154_ATUSB
 	tristate "ATUSB transceiver driver"
 	depends on IEEE802154_DRIVERS && MAC802154 && USB
-	---help---
+	help
 	  Say Y here to enable the ATUSB IEEE 802.15.4 wireless
 	  controller.
 
@@ -77,7 +77,7 @@ config IEEE802154_ADF7242
 	tristate "ADF7242 transceiver driver"
 	depends on IEEE802154_DRIVERS && MAC802154
 	depends on SPI
-	---help---
+	help
 	  Say Y here to enable the ADF7242 SPI 802.15.4 wireless
 	  controller.
 
@@ -89,7 +89,7 @@ config IEEE802154_CA8210
 	depends on IEEE802154_DRIVERS && MAC802154
 	depends on COMMON_CLK
 	depends on SPI
-	---help---
+	help
 	  Say Y here to enable the CA8210 SPI 802.15.4 wireless
 	  controller.
 
@@ -100,7 +100,7 @@ config IEEE802154_CA8210_DEBUGFS
 	bool "CA8210 debugfs interface"
 	depends on IEEE802154_CA8210
 	depends on DEBUG_FS
-	---help---
+	help
 	  This option compiles debugfs code for the ca8210 driver. This
 	  exposes a debugfs node for each CA8210 instance which allows
 	  direct use of the Cascoda API, exposing the 802.15.4 MAC
@@ -110,7 +110,7 @@ config IEEE802154_MCR20A
 	tristate "MCR20A transceiver driver"
 	depends on IEEE802154_DRIVERS && MAC802154
 	depends on SPI
-	---help---
+	help
 	  Say Y here to enable the MCR20A SPI 802.15.4 wireless
 	  controller.
 
@@ -120,7 +120,7 @@ config IEEE802154_MCR20A
 config IEEE802154_HWSIM
 	depends on IEEE802154_DRIVERS && MAC802154
 	tristate "Simulated radio testing tool for mac802154"
-	---help---
+	help
 	  This driver is a developer testing tool that can be used to test
 	  IEEE 802.15.4 networking stack (mac802154) functionality. This is not
 	  needed for normal wpan usage and is only for testing.
diff --git a/drivers/net/ipa/ipa_data-sc7180.c b/drivers/net/ipa/ipa_data-sc7180.c
index 43faa35ae726..d4c2bc7ad24b 100644
--- a/drivers/net/ipa/ipa_data-sc7180.c
+++ b/drivers/net/ipa/ipa_data-sc7180.c
@@ -106,7 +106,7 @@ static const struct ipa_gsi_endpoint_data ipa_gsi_endpoint_data[] = {
 	[IPA_ENDPOINT_MODEM_LAN_RX] = {
 		.ee_id		= GSI_EE_MODEM,
 		.channel_id	= 3,
-		.endpoint_id	= 13,
+		.endpoint_id	= 11,
 		.toward_ipa	= false,
 	},
 	[IPA_ENDPOINT_MODEM_AP_TX] = {
diff --git a/drivers/net/ipa/ipa_endpoint.c b/drivers/net/ipa/ipa_endpoint.c
index 66649a806dd1..9f50d0d11704 100644
--- a/drivers/net/ipa/ipa_endpoint.c
+++ b/drivers/net/ipa/ipa_endpoint.c
@@ -32,6 +32,9 @@
 /* The amount of RX buffer space consumed by standard skb overhead */
 #define IPA_RX_BUFFER_OVERHEAD	(PAGE_SIZE - SKB_MAX_ORDER(NET_SKB_PAD, 0))
 
+/* Where to find the QMAP mux_id for a packet within modem-supplied metadata */
+#define IPA_ENDPOINT_QMAP_METADATA_MASK		0x000000ff /* host byte order */
+
 #define IPA_ENDPOINT_RESET_AGGR_RETRY_MAX	3
 #define IPA_AGGR_TIME_LIMIT_DEFAULT		1000	/* microseconds */
 
@@ -433,6 +436,24 @@ static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint)
 	iowrite32(val, endpoint->ipa->reg_virt + offset);
 }
 
+/**
+ * We program QMAP endpoints so each packet received is preceded by a QMAP
+ * header structure.  The QMAP header contains a 1-byte mux_id and 2-byte
+ * packet size field, and we have the IPA hardware populate both for each
+ * received packet.  The header is configured (in the HDR_EXT register)
+ * to use big endian format.
+ *
+ * The packet size is written into the QMAP header's pkt_len field.  That
+ * location is defined here using the HDR_OFST_PKT_SIZE field.
+ *
+ * The mux_id comes from a 4-byte metadata value supplied with each packet
+ * by the modem.  It is *not* a QMAP header, but it does contain the mux_id
+ * value that we want, in its low-order byte.  A bitmask defined in the
+ * endpoint's METADATA_MASK register defines which byte within the modem
+ * metadata contains the mux_id.  And the OFST_METADATA field programmed
+ * here indicates where the extracted byte should be placed within the QMAP
+ * header.
+ */
 static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint)
 {
 	u32 offset = IPA_REG_ENDP_INIT_HDR_N_OFFSET(endpoint->endpoint_id);
@@ -441,25 +462,31 @@ static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint)
 	if (endpoint->data->qmap) {
 		size_t header_size = sizeof(struct rmnet_map_header);
 
+		/* We might supply a checksum header after the QMAP header */
 		if (endpoint->toward_ipa && endpoint->data->checksum)
 			header_size += sizeof(struct rmnet_map_ul_csum_header);
-
 		val |= u32_encode_bits(header_size, HDR_LEN_FMASK);
-		/* metadata is the 4 byte rmnet_map header itself */
-		val |= HDR_OFST_METADATA_VALID_FMASK;
-		val |= u32_encode_bits(0, HDR_OFST_METADATA_FMASK);
-		/* HDR_ADDITIONAL_CONST_LEN is 0; (IPA->AP only) */
+
+		/* Define how to fill fields in a received QMAP header */
 		if (!endpoint->toward_ipa) {
-			u32 size_offset = offsetof(struct rmnet_map_header,
-						   pkt_len);
+			u32 off;	/* Field offset within header */
+
+			/* Where IPA will write the metadata value */
+			off = offsetof(struct rmnet_map_header, mux_id);
+			val |= u32_encode_bits(off, HDR_OFST_METADATA_FMASK);
 
+			/* Where IPA will write the length */
+			off = offsetof(struct rmnet_map_header, pkt_len);
 			val |= HDR_OFST_PKT_SIZE_VALID_FMASK;
-			val |= u32_encode_bits(size_offset,
-					       HDR_OFST_PKT_SIZE_FMASK);
+			val |= u32_encode_bits(off, HDR_OFST_PKT_SIZE_FMASK);
 		}
+		/* For QMAP TX, metadata offset is 0 (modem assumes this) */
+		val |= HDR_OFST_METADATA_VALID_FMASK;
+
+		/* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */
 		/* HDR_A5_MUX is 0 */
 		/* HDR_LEN_INC_DEAGG_HDR is 0 */
-		/* HDR_METADATA_REG_VALID is 0; (AP->IPA only) */
+		/* HDR_METADATA_REG_VALID is 0 (TX only) */
 	}
 
 	iowrite32(val, endpoint->ipa->reg_virt + offset);
@@ -472,38 +499,27 @@ static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint)
 	u32 val = 0;
 
 	val |= HDR_ENDIANNESS_FMASK;		/* big endian */
-	val |= HDR_TOTAL_LEN_OR_PAD_VALID_FMASK;
-	/* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */
+
+	/* A QMAP header contains a 6 bit pad field at offset 0.  The RMNet
+	 * driver assumes this field is meaningful in packets it receives,
+	 * and assumes the header's payload length includes that padding.
+	 * The RMNet driver does *not* pad packets it sends, however, so
+	 * the pad field (although 0) should be ignored.
+	 */
+	if (endpoint->data->qmap && !endpoint->toward_ipa) {
+		val |= HDR_TOTAL_LEN_OR_PAD_VALID_FMASK;
+		/* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */
+		val |= HDR_PAYLOAD_LEN_INC_PADDING_FMASK;
+		/* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */
+	}
+
 	/* HDR_PAYLOAD_LEN_INC_PADDING is 0 */
-	/* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */
 	if (!endpoint->toward_ipa)
 		val |= u32_encode_bits(pad_align, HDR_PAD_TO_ALIGNMENT_FMASK);
 
 	iowrite32(val, endpoint->ipa->reg_virt + offset);
 }
 
-/**
- * Generate a metadata mask value that will select only the mux_id
- * field in an rmnet_map header structure.  The mux_id is at offset
- * 1 byte from the beginning of the structure, but the metadata
- * value is treated as a 4-byte unit.  So this mask must be computed
- * with endianness in mind.  Note that ipa_endpoint_init_hdr_metadata_mask()
- * will convert this value to the proper byte order.
- *
- * Marked __always_inline because this is really computing a
- * constant value.
- */
-static __always_inline __be32 ipa_rmnet_mux_id_metadata_mask(void)
-{
-	size_t mux_id_offset = offsetof(struct rmnet_map_header, mux_id);
-	u32 mux_id_mask = 0;
-	u8 *bytes;
-
-	bytes = (u8 *)&mux_id_mask;
-	bytes[mux_id_offset] = 0xff;	/* mux_id is 1 byte */
-
-	return cpu_to_be32(mux_id_mask);
-}
 
 static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint)
 {
@@ -513,8 +529,9 @@ static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint)
 
 	offset = IPA_REG_ENDP_INIT_HDR_METADATA_MASK_N_OFFSET(endpoint_id);
 
+	/* Note that HDR_ENDIANNESS indicates big endian header fields */
 	if (!endpoint->toward_ipa && endpoint->data->qmap)
-		val = ipa_rmnet_mux_id_metadata_mask();
+		val = cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK);
 
 	iowrite32(val, endpoint->ipa->reg_virt + offset);
 }
@@ -693,10 +710,12 @@ static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint)
 	u32 seq_type = endpoint->seq_type;
 	u32 val = 0;
 
+	/* Sequencer type is made up of four nibbles */
 	val |= u32_encode_bits(seq_type & 0xf, HPS_SEQ_TYPE_FMASK);
 	val |= u32_encode_bits((seq_type >> 4) & 0xf, DPS_SEQ_TYPE_FMASK);
-	/* HPS_REP_SEQ_TYPE is 0 */
-	/* DPS_REP_SEQ_TYPE is 0 */
+	/* The second two apply to replicated packets */
+	val |= u32_encode_bits((seq_type >> 8) & 0xf, HPS_REP_SEQ_TYPE_FMASK);
+	val |= u32_encode_bits((seq_type >> 12) & 0xf, DPS_REP_SEQ_TYPE_FMASK);
 
 	iowrite32(val, endpoint->ipa->reg_virt + offset);
 }
diff --git a/drivers/net/ipa/ipa_reg.h b/drivers/net/ipa/ipa_reg.h
index 3b8106aa277a..0a688d8c1d7c 100644
--- a/drivers/net/ipa/ipa_reg.h
+++ b/drivers/net/ipa/ipa_reg.h
@@ -455,6 +455,8 @@ enum ipa_mode {
  *	second packet processing pass + no decipher + microcontroller
  * @IPA_SEQ_DMA_DEC:		DMA + cipher/decipher
  * @IPA_SEQ_DMA_COMP_DECOMP:	DMA + compression/decompression
+ * @IPA_SEQ_PKT_PROCESS_NO_DEC_NO_UCP_DMAP:
+ *	packet processing + no decipher + no uCP + HPS REP DMA parser
  * @IPA_SEQ_INVALID:		invalid sequencer type
  *
  * The values defined here are broken into 4-bit nibbles that are written
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index 20b53e255f68..e56547bfdac9 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -3999,6 +3999,8 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
 	return 0;
 }
 
+static struct lock_class_key macsec_netdev_addr_lock_key;
+
 static int macsec_newlink(struct net *net, struct net_device *dev,
 			  struct nlattr *tb[], struct nlattr *data[],
 			  struct netlink_ext_ack *extack)
@@ -4050,6 +4052,9 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 		return err;
 
 	netdev_lockdep_set_classes(dev);
+	lockdep_set_class_and_subclass(&dev->addr_list_lock,
+				       &macsec_netdev_addr_lock_key,
+				       dev->lower_level);
 
 	err = netdev_upper_dev_link(real_dev, dev, extack);
 	if (err < 0)
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 563aed5b3d9f..6a6cc9f75307 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -860,6 +860,8 @@ static int macvlan_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
  * "super class" of normal network devices; split their locks off into a
  * separate class since they always nest.
  */
+static struct lock_class_key macvlan_netdev_addr_lock_key;
+
 #define ALWAYS_ON_OFFLOADS \
 	(NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE | \
 	 NETIF_F_GSO_ROBUST | NETIF_F_GSO_ENCAP_ALL)
@@ -875,6 +877,14 @@ static int macvlan_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 #define MACVLAN_STATE_MASK \
 	((1<<__LINK_STATE_NOCARRIER) | (1<<__LINK_STATE_DORMANT))
 
+static void macvlan_set_lockdep_class(struct net_device *dev)
+{
+	netdev_lockdep_set_classes(dev);
+	lockdep_set_class_and_subclass(&dev->addr_list_lock,
+				       &macvlan_netdev_addr_lock_key,
+				       dev->lower_level);
+}
+
 static int macvlan_init(struct net_device *dev)
 {
 	struct macvlan_dev *vlan = netdev_priv(dev);
@@ -892,8 +902,7 @@ static int macvlan_init(struct net_device *dev)
 	dev->gso_max_size	= lowerdev->gso_max_size;
 	dev->gso_max_segs	= lowerdev->gso_max_segs;
 	dev->hard_header_len	= lowerdev->hard_header_len;
-
-	netdev_lockdep_set_classes(dev);
+	macvlan_set_lockdep_class(dev);
 
 	vlan->pcpu_stats = netdev_alloc_pcpu_stats(struct vlan_pcpu_stats);
 	if (!vlan->pcpu_stats)
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index 047c27087b10..f25702386d83 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -134,7 +134,7 @@ config MDIO_GPIO
 	tristate "GPIO lib-based bitbanged MDIO buses"
 	depends on MDIO_BITBANG
 	depends on GPIOLIB || COMPILE_TEST
-	---help---
+	help
 	  Supports GPIO lib-based MDIO busses.
 
 	  To compile this driver as a module, choose M here: the module
@@ -265,7 +265,7 @@ config SWPHY
 config LED_TRIGGER_PHY
 	bool "Support LED triggers for tracking link state"
 	depends on LEDS_TRIGGERS
-	---help---
+	help
 	  Adds support for a set of LED trigger events per-PHY.  Link
 	  state change will trigger the events, for consumption by an
 	  LED class driver.  There are triggers for each link speed currently
@@ -298,12 +298,12 @@ config ADIN_PHY
 
 config AMD_PHY
 	tristate "AMD PHYs"
-	---help---
+	help
 	  Currently supports the am79c874
 
 config AQUANTIA_PHY
 	tristate "Aquantia PHYs"
-	---help---
+	help
 	  Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
 
 config AX88796B_PHY
@@ -316,13 +316,13 @@ config BCM63XX_PHY
 	tristate "Broadcom 63xx SOCs internal PHY"
 	depends on BCM63XX || COMPILE_TEST
 	select BCM_NET_PHYLIB
-	---help---
+	help
 	  Currently supports the 6348 and 6358 PHYs.
 
 config BCM7XXX_PHY
 	tristate "Broadcom 7xxx SOCs internal PHYs"
 	select BCM_NET_PHYLIB
-	---help---
+	help
 	  Currently supports the BCM7366, BCM7439, BCM7445, and
 	  40nm and 65nm generation of BCM7xxx Set Top Box SoCs.
 
@@ -336,7 +336,7 @@ config BCM_CYGNUS_PHY
 	depends on ARCH_BCM_IPROC || COMPILE_TEST
 	depends on MDIO_BCM_IPROC
 	select BCM_NET_PHYLIB
-	---help---
+	help
 	  This PHY driver is for the 1G internal PHYs of the Broadcom
 	  Cygnus and Omega Family SoC.
 
@@ -350,7 +350,7 @@ config BCM_NET_PHYLIB
 config BROADCOM_PHY
 	tristate "Broadcom PHYs"
 	select BCM_NET_PHYLIB
-	---help---
+	help
 	  Currently supports the BCM5411, BCM5421, BCM5461, BCM54616S, BCM5464,
 	  BCM5481, BCM54810 and BCM5482 PHYs.
 
@@ -368,48 +368,48 @@ config BCM54140_PHY
 config BCM84881_PHY
 	tristate "Broadcom BCM84881 PHY"
 	depends on PHYLIB
-	---help---
+	help
 	  Support the Broadcom BCM84881 PHY.
 
 config CICADA_PHY
 	tristate "Cicada PHYs"
-	---help---
+	help
 	  Currently supports the cis8204
 
 config CORTINA_PHY
 	tristate "Cortina EDC CDR 10G Ethernet PHY"
-	---help---
+	help
 	  Currently supports the CS4340 phy.
 
 config DAVICOM_PHY
 	tristate "Davicom PHYs"
-	---help---
+	help
 	  Currently supports dm9161e and dm9131
 
 config DP83822_PHY
 	tristate "Texas Instruments DP83822/825/826 PHYs"
-	---help---
+	help
 	  Supports the DP83822, DP83825I, DP83825CM, DP83825CS, DP83825S,
 	  DP83826C and DP83826NC PHYs.
 
 config DP83TC811_PHY
 	tristate "Texas Instruments DP83TC811 PHY"
-	---help---
+	help
 	  Supports the DP83TC811 PHY.
 
 config DP83848_PHY
 	tristate "Texas Instruments DP83848 PHY"
-	---help---
+	help
 	  Supports the DP83848 PHY.
 
 config DP83867_PHY
 	tristate "Texas Instruments DP83867 Gigabit PHY"
-	---help---
+	help
 	  Currently supports the DP83867 PHY.
 
 config DP83869_PHY
 	tristate "Texas Instruments DP83869 Gigabit PHY"
-	---help---
+	help
 	  Currently supports the DP83869 PHY.  This PHY supports copper and
 	  fiber connections.
 
@@ -417,7 +417,7 @@ config FIXED_PHY
 	tristate "MDIO Bus/PHY emulation with fixed speed/link PHYs"
 	depends on PHYLIB
 	select SWPHY
-	---help---
+	help
 	  Adds the platform "fixed" MDIO Bus to cover the boards that use
 	  PHYs that are not connected to the real MDIO bus.
 
@@ -425,12 +425,12 @@ config FIXED_PHY
 
 config ICPLUS_PHY
 	tristate "ICPlus PHYs"
-	---help---
+	help
 	  Currently supports the IP175C and IP1001 PHYs.
 
 config INTEL_XWAY_PHY
 	tristate "Intel XWAY PHYs"
-	---help---
+	help
 	  Supports the Intel XWAY (former Lantiq) 11G and 22E PHYs.
 	  These PHYs are marked as standalone chips under the names
 	  PEF 7061, PEF 7071 and PEF 7072 or integrated into the Intel
@@ -438,33 +438,33 @@ config INTEL_XWAY_PHY
 
 config LSI_ET1011C_PHY
 	tristate "LSI ET1011C PHY"
-	---help---
+	help
 	  Supports the LSI ET1011C PHY.
 
 config LXT_PHY
 	tristate "Intel LXT PHYs"
-	---help---
+	help
 	  Currently supports the lxt970, lxt971
 
 config MARVELL_PHY
 	tristate "Marvell PHYs"
-	---help---
+	help
 	  Currently has a driver for the 88E1011S
 
 config MARVELL_10G_PHY
 	tristate "Marvell Alaska 10Gbit PHYs"
-	---help---
+	help
 	  Support for the Marvell Alaska MV88X3310 and compatible PHYs.
 
 config MESON_GXL_PHY
 	tristate "Amlogic Meson GXL Internal PHY"
 	depends on ARCH_MESON || COMPILE_TEST
-	---help---
+	help
 	  Currently has a driver for the Amlogic Meson GXL Internal PHY
 
 config MICREL_PHY
 	tristate "Micrel PHYs"
-	---help---
+	help
 	  Supports the KSZ9021, VSC8201, KS8001 PHYs.
 
 config MICROCHIP_PHY
@@ -474,7 +474,7 @@ config MICROCHIP_PHY
 
 config MICROCHIP_T1_PHY
 	tristate "Microchip T1 PHYs"
-	---help---
+	help
 	  Supports the LAN87XX PHYs.
 
 config MICROSEMI_PHY
@@ -482,18 +482,18 @@ config MICROSEMI_PHY
 	depends on MACSEC || MACSEC=n
 	select CRYPTO_AES
 	select CRYPTO_ECB
-	---help---
+	help
 	  Currently supports VSC8514, VSC8530, VSC8531, VSC8540 and VSC8541 PHYs
 
 config NATIONAL_PHY
 	tristate "National Semiconductor PHYs"
-	---help---
+	help
 	  Currently supports the DP83865 PHY.
 
 config NXP_TJA11XX_PHY
 	tristate "NXP TJA11xx PHYs support"
 	depends on HWMON
-	---help---
+	help
 	  Currently supports the NXP TJA1100 and TJA1101 PHY.
 
 config AT803X_PHY
@@ -504,47 +504,47 @@ config AT803X_PHY
 
 config QSEMI_PHY
 	tristate "Quality Semiconductor PHYs"
-	---help---
+	help
 	  Currently supports the qs6612
 
 config REALTEK_PHY
 	tristate "Realtek PHYs"
-	---help---
+	help
 	  Supports the Realtek 821x PHY.
 
 config RENESAS_PHY
 	tristate "Driver for Renesas PHYs"
-	---help---
+	help
 	  Supports the Renesas PHYs uPD60620 and uPD60620A.
 
 config ROCKCHIP_PHY
 	tristate "Driver for Rockchip Ethernet PHYs"
-	---help---
+	help
 	  Currently supports the integrated Ethernet PHY.
 
 config SMSC_PHY
 	tristate "SMSC PHYs"
-	---help---
+	help
 	  Currently supports the LAN83C185, LAN8187 and LAN8700 PHYs
 
 config STE10XP
 	tristate "STMicroelectronics STe10Xp PHYs"
-	---help---
+	help
 	  This is the driver for the STe100p and STe101p PHYs.
 
 config TERANETICS_PHY
 	tristate "Teranetics PHYs"
-	---help---
+	help
 	  Currently supports the Teranetics TN2020
 
 config VITESSE_PHY
 	tristate "Vitesse PHYs"
-	---help---
+	help
 	  Currently supports the vsc8244
 
 config XILINX_GMII2RGMII
 	tristate "Xilinx GMII2RGMII converter driver"
-	---help---
+	help
 	  This driver support xilinx GMII to RGMII IP core it provides
 	  the Reduced Gigabit Media Independent Interface(RGMII) between
 	  Ethernet physical media devices and the Gigabit Ethernet controller.
diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c
index 4017ae1692d8..f3c04981b8da 100644
--- a/drivers/net/phy/dp83867.c
+++ b/drivers/net/phy/dp83867.c
@@ -488,7 +488,7 @@ static int dp83867_verify_rgmii_cfg(struct phy_device *phydev)
 	return 0;
 }
 
-#ifdef CONFIG_OF_MDIO
+#if IS_ENABLED(CONFIG_OF_MDIO)
 static int dp83867_of_init(struct phy_device *phydev)
 {
 	struct dp83867_private *dp83867 = phydev->priv;
diff --git a/drivers/net/phy/dp83869.c b/drivers/net/phy/dp83869.c
index cfb22a21a2e6..53ed3abc26c9 100644
--- a/drivers/net/phy/dp83869.c
+++ b/drivers/net/phy/dp83869.c
@@ -176,7 +176,7 @@ static int dp83869_set_strapped_mode(struct phy_device *phydev)
 	return 0;
 }
 
-#ifdef CONFIG_OF_MDIO
+#if IS_ENABLED(CONFIG_OF_MDIO)
 static int dp83869_of_init(struct phy_device *phydev)
 {
 	struct dp83869_private *dp83869 = phydev->priv;
@@ -218,10 +218,13 @@ static int dp83869_of_init(struct phy_device *phydev)
 		ret = phy_read_mmd(phydev, DP83869_DEVADDR, DP83869_STRAP_STS1);
 		if (ret < 0)
 			return ret;
+
 		if (ret & DP83869_STRAP_MIRROR_ENABLED)
 			dp83869->port_mirroring = DP83869_PORT_MIRRORING_EN;
 		else
 			dp83869->port_mirroring = DP83869_PORT_MIRRORING_DIS;
+
+		ret = 0;
 	}
 
 	if (of_property_read_u32(of_node, "rx-fifo-depth",
diff --git a/drivers/net/phy/fixed_phy.c b/drivers/net/phy/fixed_phy.c
index 4a3d34f40cb9..c4641b1704d6 100644
--- a/drivers/net/phy/fixed_phy.c
+++ b/drivers/net/phy/fixed_phy.c
@@ -19,7 +19,6 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/gpio/consumer.h>
-#include <linux/seqlock.h>
 #include <linux/idr.h>
 #include <linux/netdevice.h>
 #include <linux/linkmode.h>
@@ -34,7 +33,6 @@ struct fixed_mdio_bus {
 struct fixed_phy {
 	int addr;
 	struct phy_device *phydev;
-	seqcount_t seqcount;
 	struct fixed_phy_status status;
 	bool no_carrier;
 	int (*link_update)(struct net_device *, struct fixed_phy_status *);
@@ -80,19 +78,17 @@ static int fixed_mdio_read(struct mii_bus *bus, int phy_addr, int reg_num)
 	list_for_each_entry(fp, &fmb->phys, node) {
 		if (fp->addr == phy_addr) {
 			struct fixed_phy_status state;
-			int s;
-
-			do {
-				s = read_seqcount_begin(&fp->seqcount);
-				fp->status.link = !fp->no_carrier;
-				/* Issue callback if user registered it. */
-				if (fp->link_update)
-					fp->link_update(fp->phydev->attached_dev,
-							&fp->status);
-				/* Check the GPIO for change in status */
-				fixed_phy_update(fp);
-				state = fp->status;
-			} while (read_seqcount_retry(&fp->seqcount, s));
+
+			fp->status.link = !fp->no_carrier;
+
+			/* Issue callback if user registered it. */
+			if (fp->link_update)
+				fp->link_update(fp->phydev->attached_dev,
+						&fp->status);
+
+			/* Check the GPIO for change in status */
+			fixed_phy_update(fp);
+			state = fp->status;
 
 			return swphy_read_reg(reg_num, &state);
 		}
@@ -150,8 +146,6 @@ static int fixed_phy_add_gpiod(unsigned int irq, int phy_addr,
 	if (!fp)
 		return -ENOMEM;
 
-	seqcount_init(&fp->seqcount);
-
 	if (irq != PHY_POLL)
 		fmb->mii_bus->irq[phy_addr] = irq;
 
diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
index 4ea226566cec..c9ecf3c8c3fd 100644
--- a/drivers/net/phy/marvell.c
+++ b/drivers/net/phy/marvell.c
@@ -429,7 +429,7 @@ static int m88e1101_config_aneg(struct phy_device *phydev)
 	return marvell_config_aneg(phydev);
 }
 
-#ifdef CONFIG_OF_MDIO
+#if IS_ENABLED(CONFIG_OF_MDIO)
 /* Set and/or override some configuration registers based on the
  * marvell,reg-init property stored in the of_node for the phydev.
  *
diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c
index 255fdfcc13a6..6ceee82b2839 100644
--- a/drivers/net/phy/mdio_bus.c
+++ b/drivers/net/phy/mdio_bus.c
@@ -764,6 +764,7 @@ EXPORT_SYMBOL(mdiobus_scan);
 
 static void mdiobus_stats_acct(struct mdio_bus_stats *stats, bool op, int ret)
 {
+	preempt_disable();
 	u64_stats_update_begin(&stats->syncp);
 
 	u64_stats_inc(&stats->transfers);
@@ -778,6 +779,7 @@ static void mdiobus_stats_acct(struct mdio_bus_stats *stats, bool op, int ret)
 		u64_stats_inc(&stats->writes);
 out:
 	u64_stats_update_end(&stats->syncp);
+	preempt_enable();
 }
 
 /**
diff --git a/drivers/net/phy/mscc/mscc.h b/drivers/net/phy/mscc/mscc.h
index f828c917b9f7..fbcee5fce7b2 100644
--- a/drivers/net/phy/mscc/mscc.h
+++ b/drivers/net/phy/mscc/mscc.h
@@ -374,7 +374,7 @@ struct vsc8531_private {
 #endif
 };
 
-#ifdef CONFIG_OF_MDIO
+#if IS_ENABLED(CONFIG_OF_MDIO)
 struct vsc8531_edge_rate_table {
 	u32 vddmac;
 	u32 slowdown[8];
diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c
index 7ed0285206d0..5ddc44f87eaf 100644
--- a/drivers/net/phy/mscc/mscc_main.c
+++ b/drivers/net/phy/mscc/mscc_main.c
@@ -98,7 +98,7 @@ static const struct vsc85xx_hw_stat vsc8584_hw_stats[] = {
 	},
 };
 
-#ifdef CONFIG_OF_MDIO
+#if IS_ENABLED(CONFIG_OF_MDIO)
 static const struct vsc8531_edge_rate_table edge_table[] = {
 	{MSCC_VDDMAC_3300, { 0, 2,  4,  7, 10, 17, 29, 53} },
 	{MSCC_VDDMAC_2500, { 0, 3,  6, 10, 14, 23, 37, 63} },
@@ -382,7 +382,7 @@ out_unlock:
 	mutex_unlock(&phydev->lock);
 }
 
-#ifdef CONFIG_OF_MDIO
+#if IS_ENABLED(CONFIG_OF_MDIO)
 static int vsc85xx_edge_rate_magic_get(struct phy_device *phydev)
 {
 	u32 vdd, sd;
@@ -1396,7 +1396,7 @@ static int vsc8584_config_init(struct phy_device *phydev)
 
 	/* Disable SerDes for 100Base-FX */
 	ret = vsc8584_cmd(phydev, PROC_CMD_FIBER_MEDIA_CONF |
-			  PROC_CMD_FIBER_PORT(vsc8531->base_addr) |
+			  PROC_CMD_FIBER_PORT(vsc8531->addr) |
 			  PROC_CMD_FIBER_DISABLE |
 			  PROC_CMD_READ_MOD_WRITE_PORT |
 			  PROC_CMD_RST_CONF_PORT | PROC_CMD_FIBER_100BASE_FX);
@@ -1405,7 +1405,7 @@ static int vsc8584_config_init(struct phy_device *phydev)
 
 	/* Disable SerDes for 1000Base-X */
 	ret = vsc8584_cmd(phydev, PROC_CMD_FIBER_MEDIA_CONF |
-			  PROC_CMD_FIBER_PORT(vsc8531->base_addr) |
+			  PROC_CMD_FIBER_PORT(vsc8531->addr) |
 			  PROC_CMD_FIBER_DISABLE |
 			  PROC_CMD_READ_MOD_WRITE_PORT |
 			  PROC_CMD_RST_CONF_PORT | PROC_CMD_FIBER_1000BASE_X);
diff --git a/drivers/net/plip/Kconfig b/drivers/net/plip/Kconfig
index e03556d1d0c2..67bfb01de6f7 100644
--- a/drivers/net/plip/Kconfig
+++ b/drivers/net/plip/Kconfig
@@ -6,7 +6,7 @@
 config PLIP
 	tristate "PLIP (parallel port) support"
 	depends on PARPORT
-	---help---
+	help
 	  PLIP (Parallel Line Internet Protocol) is used to create a
 	  reasonably fast mini network consisting of two (or, rarely, more)
 	  local machines.  A PLIP link from a Linux box is a popular means to
diff --git a/drivers/net/ppp/Kconfig b/drivers/net/ppp/Kconfig
index 1a2e2f7629f3..ac4d162d9455 100644
--- a/drivers/net/ppp/Kconfig
+++ b/drivers/net/ppp/Kconfig
@@ -6,7 +6,7 @@
 config PPP
 	tristate "PPP (point-to-point protocol) support"
 	select SLHC
-	---help---
+	help
 	  PPP (Point to Point Protocol) is a newer and better SLIP.  It serves
 	  the same purpose: sending Internet traffic over telephone (and other
 	  serial) lines.  Ask your access provider if they support it, because
@@ -38,7 +38,7 @@ if PPP
 config PPP_BSDCOMP
 	tristate "PPP BSD-Compress compression"
 	depends on PPP
-	---help---
+	help
 	  Support for the BSD-Compress compression method for PPP, which uses
 	  the LZW compression method to compress each PPP packet before it is
 	  sent over the wire. The machine at the other end of the PPP link
@@ -59,7 +59,7 @@ config PPP_DEFLATE
 	depends on PPP
 	select ZLIB_INFLATE
 	select ZLIB_DEFLATE
-	---help---
+	help
 	  Support for the Deflate compression method for PPP, which uses the
 	  Deflate algorithm (the same algorithm that gzip uses) to compress
 	  each PPP packet before it is sent over the wire.  The machine at the
@@ -72,7 +72,7 @@ config PPP_DEFLATE
 config PPP_FILTER
 	bool "PPP filtering"
 	depends on PPP
-	---help---
+	help
 	  Say Y here if you want to be able to filter the packets passing over
 	  PPP interfaces.  This allows you to control which packets count as
 	  activity (i.e. which packets will reset the idle timer or bring up
@@ -88,7 +88,7 @@ config PPP_MPPE
 	select CRYPTO
 	select CRYPTO_SHA1
 	select CRYPTO_LIB_ARC4
-	---help---
+	help
 	  Support for the MPPE Encryption protocol, as employed by the
 	  Microsoft Point-to-Point Tunneling Protocol.
 
@@ -98,7 +98,7 @@ config PPP_MPPE
 config PPP_MULTILINK
 	bool "PPP multilink support"
 	depends on PPP
-	---help---
+	help
 	  PPP multilink is a protocol (defined in RFC 1990) which allows you
 	  to combine several (logical or physical) lines into one logical PPP
 	  connection, so that you can utilize your full bandwidth.
@@ -111,7 +111,7 @@ config PPP_MULTILINK
 config PPPOATM
 	tristate "PPP over ATM"
 	depends on ATM && PPP
-	---help---
+	help
 	  Support PPP (Point to Point Protocol) encapsulated in ATM frames.
 	  This implementation does not yet comply with section 8 of RFC2364,
 	  which can lead to bad results if the ATM peer loses state and
@@ -120,7 +120,7 @@ config PPPOATM
 config PPPOE
 	tristate "PPP over Ethernet"
 	depends on PPP
-	---help---
+	help
 	  Support for PPP over Ethernet.
 
 	  This driver requires the latest version of pppd from the CVS
@@ -132,7 +132,7 @@ config PPPOE
 config PPTP
 	tristate "PPP over IPv4 (PPTP)"
 	depends on PPP && NET_IPGRE_DEMUX
-	---help---
+	help
 	  Support for PPP over IPv4.(Point-to-Point Tunneling Protocol)
 
 	  This driver requires pppd plugin to work in client mode or
@@ -143,7 +143,7 @@ config PPTP
 config PPPOL2TP
 	tristate "PPP over L2TP"
 	depends on L2TP && PPP
-	---help---
+	help
 	  Support for PPP-over-L2TP socket family. L2TP is a protocol
 	  used by ISPs and enterprises to tunnel PPP traffic over UDP
 	  tunnels. L2TP is replacing PPTP for VPN uses.
@@ -153,7 +153,7 @@ config PPP_ASYNC
 	tristate "PPP support for async serial ports"
 	depends on PPP
 	select CRC_CCITT
-	---help---
+	help
 	  Say Y (or M) here if you want to be able to use PPP over standard
 	  asynchronous serial ports, such as COM1 or COM2 on a PC.  If you use
 	  a modem (not a synchronous or ISDN modem) to contact your ISP, you
@@ -166,7 +166,7 @@ config PPP_ASYNC
 config PPP_SYNC_TTY
 	tristate "PPP support for sync tty ports"
 	depends on PPP
-	---help---
+	help
 	  Say Y (or M) here if you want to be able to use PPP over synchronous
 	  (HDLC) tty devices, such as the SyncLink adapter. These devices
 	  are often used for high-speed leased lines like T1/E1.
diff --git a/drivers/net/slip/Kconfig b/drivers/net/slip/Kconfig
index 30bbafb0e3c7..ea68fc15c945 100644
--- a/drivers/net/slip/Kconfig
+++ b/drivers/net/slip/Kconfig
@@ -6,7 +6,7 @@
 config SLIP
 	tristate "SLIP (serial line) support"
 	depends on TTY
-	---help---
+	help
 	  Say Y if you intend to use SLIP or CSLIP (compressed SLIP) to
 	  connect to your Internet service provider or to connect to some
 	  other local Unix box or if you want to configure your Linux box as a
@@ -36,7 +36,7 @@ config SLIP
 
 config SLHC
 	tristate
-	---help---
+	help
 	  This option enables Van Jacobsen serial line header compression
 	  routines.
 
@@ -46,7 +46,7 @@ config SLIP_COMPRESSED
 	bool "CSLIP compressed headers"
 	depends on SLIP
 	select SLHC
-	---help---
+	help
 	  This protocol is faster than SLIP because it uses compression on the
 	  TCP/IP headers (not on the data itself), but it has to be supported
 	  on both ends. Ask your access provider if you are not sure and
@@ -61,7 +61,7 @@ config SLIP_COMPRESSED
 config SLIP_SMART
 	bool "Keepalive and linefill"
 	depends on SLIP
-	---help---
+	help
 	  Adds additional capabilities to the SLIP driver to support the
 	  RELCOM line fill and keepalive monitoring. Ideal on poor quality
 	  analogue lines.
@@ -69,7 +69,7 @@ config SLIP_SMART
 config SLIP_MODE_SLIP6
 	bool "Six bit SLIP encapsulation"
 	depends on SLIP
-	---help---
+	help
 	  Just occasionally you may need to run IP over hostile serial
 	  networks that don't pass all control characters or are only seven
 	  bit. Saying Y here adds an extra mode you can use with SLIP:
diff --git a/drivers/net/team/Kconfig b/drivers/net/team/Kconfig
index 2aa9fd7b57df..4d449845dac7 100644
--- a/drivers/net/team/Kconfig
+++ b/drivers/net/team/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig NET_TEAM
 	tristate "Ethernet team driver support"
-	---help---
+	help
 	  This allows one to create virtual interfaces that teams together
 	  multiple ethernet devices.
 
@@ -18,7 +18,7 @@ if NET_TEAM
 config NET_TEAM_MODE_BROADCAST
 	tristate "Broadcast mode support"
 	depends on NET_TEAM
-	---help---
+	help
 	  Basic mode where packets are transmitted always by all suitable ports.
 
 	  All added ports are setup to have team's device address.
@@ -29,7 +29,7 @@ config NET_TEAM_MODE_BROADCAST
 config NET_TEAM_MODE_ROUNDROBIN
 	tristate "Round-robin mode support"
 	depends on NET_TEAM
-	---help---
+	help
 	  Basic mode where port used for transmitting packets is selected in
 	  round-robin fashion using packet counter.
 
@@ -41,7 +41,7 @@ config NET_TEAM_MODE_ROUNDROBIN
 config NET_TEAM_MODE_RANDOM
 	tristate "Random mode support"
 	depends on NET_TEAM
-	---help---
+	help
 	  Basic mode where port used for transmitting packets is selected
 	  randomly.
 
@@ -53,7 +53,7 @@ config NET_TEAM_MODE_RANDOM
 config NET_TEAM_MODE_ACTIVEBACKUP
 	tristate "Active-backup mode support"
 	depends on NET_TEAM
-	---help---
+	help
 	  Only one port is active at a time and the rest of ports are used
 	  for backup.
 
@@ -66,7 +66,7 @@ config NET_TEAM_MODE_ACTIVEBACKUP
 config NET_TEAM_MODE_LOADBALANCE
 	tristate "Load-balance mode support"
 	depends on NET_TEAM
-	---help---
+	help
 	  This mode provides load balancing functionality. Tx port selection
 	  is done using BPF function set up from userspace (bpf_hash_func
 	  option)
diff --git a/drivers/net/usb/Kconfig b/drivers/net/usb/Kconfig
index 05bdcc5917f6..a7fbc3ccd29e 100644
--- a/drivers/net/usb/Kconfig
+++ b/drivers/net/usb/Kconfig
@@ -15,7 +15,7 @@ if USB_NET_DRIVERS
 config USB_CATC
 	tristate "USB CATC NetMate-based Ethernet device support"
 	select CRC32
-	---help---
+	help
 	  Say Y if you want to use one of the following 10Mbps USB Ethernet
 	  device based on the EL1210A chip. Supported devices are:
 	  Belkin F5U011
@@ -33,7 +33,7 @@ config USB_CATC
 
 config USB_KAWETH
 	tristate "USB KLSI KL5USB101-based ethernet device support"
-	---help---
+	help
 	  Say Y here if you want to use one of the following 10Mbps only
 	  USB Ethernet adapters based on the KLSI KL5KUSB101B chipset:
 	  3Com 3C19250
@@ -73,7 +73,7 @@ config USB_KAWETH
 config USB_PEGASUS
 	tristate "USB Pegasus/Pegasus-II based ethernet device support"
 	select MII
-	---help---
+	help
 	  Say Y here if you know you have Pegasus or Pegasus-II based adapter.
 	  If in doubt then look at <file:drivers/net/usb/pegasus.h> for the
 	  complete list of supported devices.
@@ -128,7 +128,7 @@ config USB_LAN78XX
 config USB_USBNET
 	tristate "Multi-purpose USB Networking Framework"
 	select MII
-	---help---
+	help
 	  This driver supports several kinds of network links over USB,
 	  with "minidrivers" built around a common network driver core
 	  that supports deep queues for efficient transfers.  (This gives
@@ -321,7 +321,7 @@ config USB_NET_SR9800
 	tristate "CoreChip-sz SR9800 based USB 2.0 10/100 ethernet devices"
 	depends on USB_USBNET
 	select CRC32
-	---help---
+	help
 	  Say Y if you want to use one of the following 100Mbps USB Ethernet
 	  device based on the CoreChip-sz SR9800 chip.
 
@@ -570,7 +570,7 @@ config USB_CDC_PHONET
 config USB_IPHETH
 	tristate "Apple iPhone USB Ethernet driver"
 	default n
-	---help---
+	help
 	  Module used to share Internet connection (tethering) from your
 	  iPhone (Original, 3G and 3GS) to your system.
 	  Note that you need userspace libraries and programs that are needed
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index 5bb448ae6c9c..e8085ab6d484 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -857,7 +857,6 @@ static int vxlan_fdb_nh_update(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
 			       u32 nhid, struct netlink_ext_ack *extack)
 {
 	struct nexthop *old_nh = rtnl_dereference(fdb->nh);
-	struct nh_group *nhg;
 	struct nexthop *nh;
 	int err = -EINVAL;
 
@@ -876,13 +875,12 @@ static int vxlan_fdb_nh_update(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
 			nh = NULL;
 			goto err_inval;
 		}
-		if (!nh->is_fdb_nh) {
+		if (!nexthop_is_fdb(nh)) {
 			NL_SET_ERR_MSG(extack, "Nexthop is not a fdb nexthop");
 			goto err_inval;
 		}
 
-		nhg = rtnl_dereference(nh->nh_grp);
-		if (!nh->is_group || !nhg->mpath) {
+		if (!nexthop_is_multipath(nh)) {
 			NL_SET_ERR_MSG(extack, "Nexthop is not a multipath group");
 			goto err_inval;
 		}
@@ -890,14 +888,14 @@ static int vxlan_fdb_nh_update(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
 		/* check nexthop group family */
 		switch (vxlan->default_dst.remote_ip.sa.sa_family) {
 		case AF_INET:
-			if (!nhg->has_v4) {
+			if (!nexthop_has_v4(nh)) {
 				err = -EAFNOSUPPORT;
 				NL_SET_ERR_MSG(extack, "Nexthop group family not supported");
 				goto err_inval;
 			}
 			break;
 		case AF_INET6:
-			if (nhg->has_v4) {
+			if (nexthop_has_v4(nh)) {
 				err = -EAFNOSUPPORT;
 				NL_SET_ERR_MSG(extack, "Nexthop group family not supported");
 				goto err_inval;
@@ -4245,10 +4243,8 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],
 		mod_timer(&vxlan->age_timer, jiffies);
 
 	netdev_adjacent_change_commit(dst->remote_dev, lowerdev, dev);
-	if (lowerdev && lowerdev != dst->remote_dev) {
+	if (lowerdev && lowerdev != dst->remote_dev)
 		dst->remote_dev = lowerdev;
-		netdev_update_lockdep_key(lowerdev);
-	}
 	vxlan_config_apply(dev, &conf, lowerdev, vxlan->net, true);
 	return 0;
 }
diff --git a/drivers/net/wan/Kconfig b/drivers/net/wan/Kconfig
index 3e21726c36e8..39e5ab261d7c 100644
--- a/drivers/net/wan/Kconfig
+++ b/drivers/net/wan/Kconfig
@@ -5,7 +5,7 @@
 
 menuconfig WAN
 	bool "Wan interfaces support"
-	---help---
+	help
 	  Wide Area Networks (WANs), such as X.25, Frame Relay and leased
 	  lines, are used to interconnect Local Area Networks (LANs) over vast
 	  distances with data transfer rates significantly higher than those
@@ -39,7 +39,7 @@ config HOSTESS_SV11
 config COSA
 	tristate "COSA/SRP sync serial boards support"
 	depends on ISA && m && ISA_DMA_API && HDLC && VIRT_TO_BUS
-	---help---
+	help
 	  Driver for COSA and SRP synchronous serial boards.
 
 	  These boards allow to connect synchronous serial devices (for example
@@ -63,7 +63,7 @@ config COSA
 config LANMEDIA
 	tristate "LanMedia Corp. SSI/V.35, T1/E1, HSSI, T3 boards"
 	depends on PCI && VIRT_TO_BUS && HDLC
-	---help---
+	help
 	  Driver for the following Lan Media family of serial boards:
 
 	  - LMC 1000 board allows you to connect synchronous serial devices
@@ -251,7 +251,7 @@ config C101
 config FARSYNC
 	tristate "FarSync T-Series support"
 	depends on HDLC && PCI
-	---help---
+	help
 	  Support for the FarSync T-Series X.21 (and V.35/V.24) cards by
 	  FarSite Communications Ltd.
 
@@ -323,7 +323,7 @@ config IXP4XX_HSS
 
 config DLCI
 	tristate "Frame Relay DLCI support"
-	---help---
+	help
 	  Support for the Frame Relay protocol.
 
 	  Frame Relay is a fast low-cost way to connect to a remote Internet
@@ -370,7 +370,7 @@ config SDLA
 config LAPBETHER
 	tristate "LAPB over Ethernet driver"
 	depends on LAPB && X25
-	---help---
+	help
 	  Driver for a pseudo device (typically called /dev/lapb0) which allows
 	  you to open an LAPB point-to-point connection to some other computer
 	  on your Ethernet network.
@@ -386,7 +386,7 @@ config LAPBETHER
 config X25_ASY
 	tristate "X.25 async driver"
 	depends on LAPB && X25 && TTY
-	---help---
+	help
 	  Send and receive X.25 frames over regular asynchronous serial
 	  lines such as telephone lines equipped with ordinary modems.
 
@@ -401,7 +401,7 @@ config X25_ASY
 config SBNI
 	tristate "Granch SBNI12 Leased Line adapter support"
 	depends on X86
-	---help---
+	help
 	  Driver for ISA SBNI12-xx cards which are low cost alternatives to
 	  leased line modems.
 
diff --git a/drivers/net/wireless/Kconfig b/drivers/net/wireless/Kconfig
index 15b0ad171f4c..8ab62bb6b853 100644
--- a/drivers/net/wireless/Kconfig
+++ b/drivers/net/wireless/Kconfig
@@ -9,7 +9,7 @@ menuconfig WLAN
 	depends on NET
 	select WIRELESS
 	default y
-	---help---
+	help
 	  This section contains all the pre 802.11 and 802.11 wireless
 	  device drivers. For a complete list of drivers and documentation
 	  on them refer to the wireless wiki:
@@ -54,7 +54,7 @@ config PCMCIA_RAYCS
 	select WIRELESS_EXT
 	select WEXT_SPY
 	select WEXT_PRIV
-	---help---
+	help
 	  Say Y here if you intend to attach an Aviator/Raytheon PCMCIA
 	  (PC-card) wireless Ethernet networking card to your computer.
 	  Please read the file <file:Documentation/networking/ray_cs.rst> for
@@ -76,7 +76,7 @@ config PCMCIA_WL3501
 config MAC80211_HWSIM
 	tristate "Simulated radio testing tool for mac80211"
 	depends on MAC80211
-	---help---
+	help
 	  This driver is a developer testing tool that can be used to test
 	  IEEE 802.11 networking stack (mac80211) functionality. This is not
 	  needed for normal wireless LAN usage and is only for testing. See
@@ -94,7 +94,7 @@ config USB_NET_RNDIS_WLAN
 	select USB_USBNET
 	select USB_NET_CDCETHER
 	select USB_NET_RNDIS_HOST
-	---help---
+	help
 	  This is a driver for wireless RNDIS devices.
 	  These are USB based adapters found in devices such as:
 
@@ -118,7 +118,7 @@ config USB_NET_RNDIS_WLAN
 config VIRT_WIFI
 	tristate "Wifi wrapper for ethernet drivers"
 	depends on CFG80211
-	---help---
+	help
 	  This option adds support for ethernet connections to appear as if they
 	  are wifi connections through a special rtnetlink device.
 
diff --git a/drivers/net/wireless/admtek/Kconfig b/drivers/net/wireless/admtek/Kconfig
index a91cc1459c96..678979a1d987 100644
--- a/drivers/net/wireless/admtek/Kconfig
+++ b/drivers/net/wireless/admtek/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_ADMTEK
 	bool "ADMtek devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -17,7 +17,7 @@ config ADM8211
 	depends on MAC80211 && PCI
 	select CRC32
 	select EEPROM_93CX6
-	---help---
+	help
 	  This driver is for ADM8211A, ADM8211B, and ADM8211C based cards.
 	  These are PCI/mini-PCI/Cardbus 802.11b chips found in cards such as:
 
diff --git a/drivers/net/wireless/ath/Kconfig b/drivers/net/wireless/ath/Kconfig
index b10972b6cba4..6e9d46b96555 100644
--- a/drivers/net/wireless/ath/Kconfig
+++ b/drivers/net/wireless/ath/Kconfig
@@ -5,7 +5,7 @@ config ATH_COMMON
 config WLAN_VENDOR_ATH
 	bool "Atheros/Qualcomm devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -25,7 +25,7 @@ if WLAN_VENDOR_ATH
 
 config ATH_DEBUG
 	bool "Atheros wireless debugging"
-	---help---
+	help
 	  Say Y, if you want to debug atheros wireless drivers.
 	  Right now only ath9k makes use of this.
 
@@ -33,7 +33,7 @@ config ATH_TRACEPOINTS
 	bool "Atheros wireless tracing"
 	depends on ATH_DEBUG
 	depends on EVENT_TRACING
-	---help---
+	help
 	  This option enables tracepoints for atheros wireless drivers.
 	  Currently, ath9k makes use of this facility.
 
@@ -41,7 +41,7 @@ config ATH_REG_DYNAMIC_USER_REG_HINTS
 	bool "Atheros dynamic user regulatory hints"
 	depends on CFG80211_CERTIFICATION_ONUS
 	default n
-	---help---
+	help
 	  Say N. This should only be enabled in countries where
 	  this feature is explicitly allowed and only on cards that
 	  specifically have been tested for this.
@@ -50,7 +50,7 @@ config ATH_REG_DYNAMIC_USER_CERT_TESTING
 	bool "Atheros dynamic user regulatory testing"
 	depends on ATH_REG_DYNAMIC_USER_REG_HINTS && CFG80211_CERTIFICATION_ONUS
 	default n
-	---help---
+	help
 	  Say N. This should only be enabled on systems
 	  undergoing certification testing.
 
diff --git a/drivers/net/wireless/ath/ar5523/Kconfig b/drivers/net/wireless/ath/ar5523/Kconfig
index e82df5f1ea67..0d838c1e7b12 100644
--- a/drivers/net/wireless/ath/ar5523/Kconfig
+++ b/drivers/net/wireless/ath/ar5523/Kconfig
@@ -4,6 +4,6 @@ config AR5523
 	depends on MAC80211 && USB
 	select ATH_COMMON
 	select FW_LOADER
-	---help---
+	help
 	  This module add support for AR5523 based USB dongles such as D-Link
 	  DWL-G132, Netgear WPN111 and many more.
diff --git a/drivers/net/wireless/ath/ath10k/Kconfig b/drivers/net/wireless/ath/ath10k/Kconfig
index b99fd0eff994..40f91bc8514d 100644
--- a/drivers/net/wireless/ath/ath10k/Kconfig
+++ b/drivers/net/wireless/ath/ath10k/Kconfig
@@ -6,7 +6,7 @@ config ATH10K
 	select CRC32
 	select WANT_DEV_COREDUMP
 	select ATH10K_CE
-	---help---
+	help
 	  This module adds support for wireless adapters based on
 	  Atheros IEEE 802.11ac family of chipsets.
 
@@ -18,25 +18,25 @@ config ATH10K_CE
 config ATH10K_PCI
 	tristate "Atheros ath10k PCI support"
 	depends on ATH10K && PCI
-	---help---
+	help
 	  This module adds support for PCIE bus
 
 config ATH10K_AHB
 	bool "Atheros ath10k AHB support"
 	depends on ATH10K_PCI && OF && RESET_CONTROLLER
-	---help---
+	help
 	  This module adds support for AHB bus
 
 config ATH10K_SDIO
 	tristate "Atheros ath10k SDIO support"
 	depends on ATH10K && MMC
-	---help---
+	help
 	  This module adds support for SDIO/MMC bus.
 
 config ATH10K_USB
 	tristate "Atheros ath10k USB support (EXPERIMENTAL)"
 	depends on ATH10K && USB
-	---help---
+	help
 	  This module adds experimental support for USB bus. Currently
 	  work in progress and will not fully work.
 
@@ -45,14 +45,14 @@ config ATH10K_SNOC
 	depends on ATH10K
 	depends on ARCH_QCOM || COMPILE_TEST
 	select QCOM_QMI_HELPERS
-	---help---
+	help
 	  This module adds support for integrated WCN3990 chip connected
 	  to system NOC(SNOC).
 
 config ATH10K_DEBUG
 	bool "Atheros ath10k debugging"
 	depends on ATH10K
-	---help---
+	help
 	  Enables debug support
 
 	  If unsure, say Y to make it easier to debug problems.
@@ -60,7 +60,7 @@ config ATH10K_DEBUG
 config ATH10K_DEBUGFS
 	bool "Atheros ath10k debugfs support"
 	depends on ATH10K && DEBUG_FS
-	---help---
+	help
 	  Enabled debugfs support
 
 	  If unsure, say Y to make it easier to debug problems.
@@ -70,20 +70,20 @@ config ATH10K_SPECTRAL
 	depends on ATH10K_DEBUGFS
 	select RELAY
 	default n
-	---help---
+	help
 	  Say Y to enable access to the FFT/spectral data via debugfs.
 
 config ATH10K_TRACING
 	bool "Atheros ath10k tracing support"
 	depends on ATH10K
 	depends on EVENT_TRACING
-	---help---
+	help
 	  Select this to ath10k use tracing infrastructure.
 
 config ATH10K_DFS_CERTIFIED
 	bool "Atheros DFS support for certified platforms"
 	depends on ATH10K && CFG80211_CERTIFICATION_ONUS
 	default n
-	---help---
+	help
 	This option enables DFS support for initiating radiation on
 	ath10k.
diff --git a/drivers/net/wireless/ath/ath11k/Kconfig b/drivers/net/wireless/ath/ath11k/Kconfig
index 738f99090d83..7acb42e2e6e2 100644
--- a/drivers/net/wireless/ath/ath11k/Kconfig
+++ b/drivers/net/wireless/ath/ath11k/Kconfig
@@ -7,7 +7,7 @@ config ATH11K
 	depends on ARCH_QCOM || COMPILE_TEST
 	select ATH_COMMON
 	select QCOM_QMI_HELPERS
-	---help---
+	help
 	  This module adds support for Qualcomm Technologies 802.11ax family of
 	  chipsets.
 
@@ -16,7 +16,7 @@ config ATH11K
 config ATH11K_DEBUG
 	bool "QCA ath11k debugging"
 	depends on ATH11K
-	---help---
+	help
 	  Enables debug support
 
 	  If unsure, say Y to make it easier to debug problems.
@@ -24,7 +24,7 @@ config ATH11K_DEBUG
 config ATH11K_DEBUGFS
 	bool "QCA ath11k debugfs support"
 	depends on ATH11K && DEBUG_FS && MAC80211_DEBUGFS
-	---help---
+	help
 	  Enable ath11k debugfs support
 
 	  If unsure, say Y to make it easier to debug problems.
@@ -32,5 +32,5 @@ config ATH11K_DEBUGFS
 config ATH11K_TRACING
 	bool "ath11k tracing support"
 	depends on ATH11K && EVENT_TRACING
-	---help---
+	help
 	  Select this to use ath11k tracing infrastructure.
diff --git a/drivers/net/wireless/ath/ath5k/Kconfig b/drivers/net/wireless/ath/ath5k/Kconfig
index 96010d4b00e7..f35cd8de228e 100644
--- a/drivers/net/wireless/ath/ath5k/Kconfig
+++ b/drivers/net/wireless/ath/ath5k/Kconfig
@@ -8,7 +8,7 @@ config ATH5K
 	select NEW_LEDS
 	select ATH5K_AHB if ATH25
 	select ATH5K_PCI if !ATH25
-	---help---
+	help
 	  This module adds support for wireless adapters based on
 	  Atheros 5xxx chipset.
 
@@ -25,7 +25,7 @@ config ATH5K
 config ATH5K_DEBUG
 	bool "Atheros 5xxx debugging"
 	depends on ATH5K
-	---help---
+	help
 	  Atheros 5xxx debugging messages.
 
 	  Say Y, if and you will get debug options for ath5k.
@@ -45,7 +45,7 @@ config ATH5K_TRACER
 	bool "Atheros 5xxx tracer"
 	depends on ATH5K
 	depends on EVENT_TRACING
-	---help---
+	help
 	  Say Y here to enable tracepoints for the ath5k driver
 	  using the kernel tracing infrastructure.  Select this
 	  option if you are interested in debugging the driver.
@@ -55,21 +55,21 @@ config ATH5K_TRACER
 config ATH5K_AHB
 	bool "Atheros 5xxx AHB bus support"
 	depends on ATH25 && ATH5K
-	---help---
+	help
 	  This adds support for WiSoC type chipsets of the 5xxx Atheros
 	  family.
 
 config ATH5K_PCI
 	bool "Atheros 5xxx PCI bus support"
 	depends on (!ATH25 && PCI)
-	---help---
+	help
 	  This adds support for PCI type chipsets of the 5xxx Atheros
 	  family.
 
 config ATH5K_TEST_CHANNELS
 	bool "Enables testing channels on ath5k"
 	depends on ATH5K && CFG80211_CERTIFICATION_ONUS
-	---help---
+	help
 	  This enables non-standard IEEE 802.11 channels on ath5k, which
 	  can be used for research purposes. This option should be disabled
 	  unless doing research.
diff --git a/drivers/net/wireless/ath/ath6kl/Kconfig b/drivers/net/wireless/ath/ath6kl/Kconfig
index 62c22fdcca38..cd96cf8d9940 100644
--- a/drivers/net/wireless/ath/ath6kl/Kconfig
+++ b/drivers/net/wireless/ath/ath6kl/Kconfig
@@ -2,7 +2,7 @@
 config ATH6KL
 	tristate "Atheros mobile chipsets support"
 	depends on CFG80211
-	---help---
+	help
 	  This module adds core support for wireless adapters based on
 	  Atheros AR6003 and AR6004 chipsets. You still need separate
 	  bus drivers for USB and SDIO to be able to use real devices.
@@ -15,7 +15,7 @@ config ATH6KL_SDIO
 	tristate "Atheros ath6kl SDIO support"
 	depends on ATH6KL
 	depends on MMC
-	---help---
+	help
 	  This module adds support for wireless adapters based on
 	  Atheros AR6003 and AR6004 chipsets running over SDIO. If you
 	  choose to build it as a module, it will be called ath6kl_sdio.
@@ -26,7 +26,7 @@ config ATH6KL_USB
 	tristate "Atheros ath6kl USB support"
 	depends on ATH6KL
 	depends on USB
-	---help---
+	help
 	  This module adds support for wireless adapters based on
 	  Atheros AR6004 chipset and chipsets based on it running over
 	  USB. If you choose to build it as a module, it will be
@@ -35,7 +35,7 @@ config ATH6KL_USB
 config ATH6KL_DEBUG
 	bool "Atheros ath6kl debugging"
 	depends on ATH6KL
-	---help---
+	help
 	  Enables ath6kl debug support, including debug messages
 	  enabled with debug_mask module parameter and debugfs
 	  interface.
@@ -46,7 +46,7 @@ config ATH6KL_TRACING
 	bool "Atheros ath6kl tracing support"
 	depends on ATH6KL
 	depends on EVENT_TRACING
-	---help---
+	help
 	  Select this to ath6kl use tracing infrastructure which, for
 	  example, can be enabled with help of trace-cmd. All debug
 	  messages and commands are delivered to using individually
@@ -58,7 +58,7 @@ config ATH6KL_REGDOMAIN
 	bool "Atheros ath6kl regdomain support"
 	depends on ATH6KL
 	depends on CFG80211_CERTIFICATION_ONUS
-	---help---
+	help
 	  Enabling this makes it possible to change the regdomain in
 	  the firmware. This can be only enabled if regulatory requirements
 	  are taken into account.
diff --git a/drivers/net/wireless/ath/ath9k/Kconfig b/drivers/net/wireless/ath/ath9k/Kconfig
index 78620c6b64a2..42bfdb4a6214 100644
--- a/drivers/net/wireless/ath/ath9k/Kconfig
+++ b/drivers/net/wireless/ath/ath9k/Kconfig
@@ -14,7 +14,7 @@ config ATH9K_BTCOEX_SUPPORT
 	bool "Atheros bluetooth coexistence support"
 	depends on (ATH9K || ATH9K_HTC)
 	default y
-	---help---
+	help
 	  Say Y, if you want to use the ath9k/ath9k_htc radios together with
 	  Bluetooth modules in the same system.
 
@@ -26,7 +26,7 @@ config ATH9K
 	select LEDS_CLASS
 	select NEW_LEDS
 	select ATH9K_COMMON
-	---help---
+	help
 	  This module adds support for wireless adapters based on
 	  Atheros IEEE 802.11n AR5008, AR9001 and AR9002 family
 	  of chipsets. For a specific list of supported external
@@ -42,7 +42,7 @@ config ATH9K_PCI
 	bool "Atheros ath9k PCI/PCIe bus support"
 	default y
 	depends on ATH9K && PCI
-	---help---
+	help
 	  This option enables the PCI bus support in ath9k.
 
 	  Say Y, if you have a compatible PCI/PCIe wireless card.
@@ -51,7 +51,7 @@ config ATH9K_AHB
 	bool "Atheros ath9k AHB bus support"
 	depends on ATH9K
 	default n
-	---help---
+	help
 	  This option enables the AHB bus support in ath9k.
 
 	  Say Y, if you have a SoC with a compatible built-in
@@ -62,7 +62,7 @@ config ATH9K_DEBUGFS
 	depends on ATH9K && DEBUG_FS
 	select MAC80211_DEBUGFS
 	select ATH9K_COMMON_DEBUG
-	---help---
+	help
 	  Say Y, if you need access to ath9k's statistics for
 	  interrupts, rate control, etc.
 
@@ -74,14 +74,14 @@ config ATH9K_STATION_STATISTICS
 	depends on ATH9K && ATH9K_DEBUGFS && DEBUG_FS
 	select MAC80211_DEBUGFS
 	default n
-	---help---
+	help
 	  This option enables detailed statistics for association stations.
 
 config ATH9K_TX99
 	bool "Atheros ath9k TX99 testing support"
 	depends on ATH9K_DEBUGFS && CFG80211_CERTIFICATION_ONUS
 	default n
-	---help---
+	help
 	  Say N. This should only be enabled on systems undergoing
 	  certification testing and evaluation in a controlled environment.
 	  Enabling this will only enable TX99 support, all other modes of
@@ -101,7 +101,7 @@ config ATH9K_DFS_CERTIFIED
 	bool "Atheros DFS support for certified platforms"
 	depends on ATH9K && CFG80211_CERTIFICATION_ONUS
 	default n
-	---help---
+	help
 	  This option enables DFS support for initiating radiation on
 	  ath9k. There is no way to dynamically detect if a card was DFS
 	  certified and as such this is left as a build time option. This
@@ -120,7 +120,7 @@ config ATH9K_DYNACK
 	bool "Atheros ath9k ACK timeout estimation algorithm"
 	depends on ATH9K
 	default n
-	---help---
+	help
 	  This option enables ath9k dynamic ACK timeout estimation algorithm
 	  based on ACK frame RX timestamp, TX frame timestamp and frame
 	  duration
@@ -129,7 +129,7 @@ config ATH9K_WOW
 	bool "Wake on Wireless LAN support (EXPERIMENTAL)"
 	depends on ATH9K && PM
 	default n
-	---help---
+	help
 	  This option enables Wake on Wireless LAN support for certain cards.
 	  Currently, AR9462 is supported.
 
@@ -147,7 +147,7 @@ config ATH9K_CHANNEL_CONTEXT
 	bool "Channel Context support"
 	depends on ATH9K
 	default n
-	---help---
+	help
 	  This option enables channel context support in ath9k, which is needed
 	  for multi-channel concurrency. Enable this if P2P PowerSave support
 	  is required.
@@ -181,7 +181,7 @@ config ATH9K_HTC
 	select LEDS_CLASS
 	select NEW_LEDS
 	select ATH9K_COMMON
-	---help---
+	help
 	  Support for Atheros HTC based cards.
 	  Chipsets supported: AR9271
 
@@ -193,7 +193,7 @@ config ATH9K_HTC_DEBUGFS
 	bool "Atheros ath9k_htc debugging"
 	depends on ATH9K_HTC && DEBUG_FS
 	select ATH9K_COMMON_DEBUG
-	---help---
+	help
 	  Say Y, if you need access to ath9k_htc's statistics.
 	  As well as access to the FFT/spectral data.
 
@@ -201,7 +201,7 @@ config ATH9K_HWRNG
 	bool "Random number generator support"
 	depends on ATH9K && (HW_RANDOM = y || HW_RANDOM = ATH9K)
 	default n
-	---help---
+	help
 	  This option incorporates the ADC register output as a source of
 	  randomness into Linux entropy pool (/dev/urandom and /dev/random)
 
@@ -213,5 +213,5 @@ config ATH9K_COMMON_SPECTRAL
 	depends on ATH9K_DEBUGFS || ATH9K_HTC_DEBUGFS
 	select RELAY
 	default n
-	---help---
+	help
 	  Say Y to enable access to the FFT/spectral data via debugfs.
diff --git a/drivers/net/wireless/ath/wcn36xx/Kconfig b/drivers/net/wireless/ath/wcn36xx/Kconfig
index a4b153470a2c..5832c7ef9352 100644
--- a/drivers/net/wireless/ath/wcn36xx/Kconfig
+++ b/drivers/net/wireless/ath/wcn36xx/Kconfig
@@ -4,7 +4,7 @@ config WCN36XX
 	depends on MAC80211 && HAS_DMA
 	depends on QCOM_WCNSS_CTRL || QCOM_WCNSS_CTRL=n
 	depends on RPMSG || RPMSG=n
-	---help---
+	help
 	  This module adds support for wireless adapters based on
 	  Qualcomm Atheros WCN3660 and WCN3680 mobile chipsets.
 
@@ -13,7 +13,7 @@ config WCN36XX
 config WCN36XX_DEBUGFS
 	bool "WCN36XX debugfs support"
 	depends on WCN36XX
-	---help---
+	help
 	  Enabled debugfs support
 
 	  If unsure, say Y to make it easier to debug problems.
diff --git a/drivers/net/wireless/ath/wil6210/Kconfig b/drivers/net/wireless/ath/wil6210/Kconfig
index 0d1a8dab30ed..dadba2d41bbb 100644
--- a/drivers/net/wireless/ath/wil6210/Kconfig
+++ b/drivers/net/wireless/ath/wil6210/Kconfig
@@ -5,7 +5,7 @@ config WIL6210
 	depends on CFG80211
 	depends on PCI
 	default n
-	---help---
+	help
 	  This module adds support for wireless adapter based on
 	  wil6210 chip by Wilocity. It supports operation on the
 	  60 GHz band, covered by the IEEE802.11ad standard.
@@ -19,7 +19,7 @@ config WIL6210_ISR_COR
 	bool "Use Clear-On-Read mode for ISR registers for wil6210"
 	depends on WIL6210
 	default y
-	---help---
+	help
 	  ISR registers on wil6210 chip may operate in either
 	  COR (Clear-On-Read) or W1C (Write-1-to-Clear) mode.
 	  For production code, use COR (say y); is default since
@@ -35,7 +35,7 @@ config WIL6210_TRACING
 	depends on WIL6210
 	depends on EVENT_TRACING
 	default n
-	---help---
+	help
 	  Say Y here to enable tracepoints for the wil6210 driver
 	  using the kernel tracing infrastructure.  Select this
 	  option if you are interested in debugging the driver.
@@ -47,7 +47,7 @@ config WIL6210_DEBUGFS
 	depends on WIL6210
 	depends on DEBUG_FS
 	default y
-	---help---
+	help
 	  Say Y here to enable wil6210 debugfs support, using the
 	  kernel debugfs infrastructure. Select this
 	  option if you are interested in debugging the driver.
diff --git a/drivers/net/wireless/atmel/Kconfig b/drivers/net/wireless/atmel/Kconfig
index c2142c70f25d..ca45a1021cf4 100644
--- a/drivers/net/wireless/atmel/Kconfig
+++ b/drivers/net/wireless/atmel/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_ATMEL
 	bool "Atmel devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -19,7 +19,7 @@ config ATMEL
 	select WEXT_PRIV
 	select FW_LOADER
 	select CRC32
-	---help---
+	help
 	  A driver 802.11b wireless cards based on the Atmel fast-vnet
 	  chips. This driver supports standard Linux wireless extensions.
 
@@ -33,7 +33,7 @@ config ATMEL
 config PCI_ATMEL
 	tristate "Atmel at76c506 PCI cards"
 	depends on ATMEL && PCI
-	---help---
+	help
 	  Enable support for PCI and mini-PCI cards containing the
 	  Atmel at76c506 chip.
 
@@ -43,7 +43,7 @@ config PCMCIA_ATMEL
 	select WIRELESS_EXT
 	select FW_LOADER
 	select CRC32
-	---help---
+	help
 	  Enable support for PCMCIA cards containing the
 	  Atmel at76c502 and at76c504 chips.
 
@@ -51,7 +51,7 @@ config AT76C50X_USB
 	tristate "Atmel at76c503/at76c505/at76c505a USB cards"
 	depends on MAC80211 && USB
 	select FW_LOADER
-	---help---
+	help
 	  Enable support for USB Wireless devices using Atmel at76c503,
 	  at76c505 or at76c505a chips.
 
diff --git a/drivers/net/wireless/broadcom/Kconfig b/drivers/net/wireless/broadcom/Kconfig
index bb1cb402a919..f74333366a90 100644
--- a/drivers/net/wireless/broadcom/Kconfig
+++ b/drivers/net/wireless/broadcom/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_BROADCOM
 	bool "Broadcom devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/wireless/broadcom/b43/Kconfig b/drivers/net/wireless/broadcom/b43/Kconfig
index 3b582e76762b..4559549b80fe 100644
--- a/drivers/net/wireless/broadcom/b43/Kconfig
+++ b/drivers/net/wireless/broadcom/b43/Kconfig
@@ -6,7 +6,7 @@ config B43
 	select SSB if B43_SSB
 	select FW_LOADER
 	select CORDIC
-	---help---
+	help
 	  b43 is a driver for the Broadcom 43xx series wireless devices.
 
 	  Check "lspci" for something like
@@ -77,7 +77,7 @@ config B43_SDIO
 	bool "Broadcom 43xx SDIO device support"
 	depends on B43 && B43_SSB && SSB_SDIOHOST_POSSIBLE
 	select SSB_SDIOHOST
-	---help---
+	help
 	  Broadcom 43xx device support for Soft-MAC SDIO devices.
 
 	  With this config option you can drive Soft-MAC b43 cards with a
@@ -108,7 +108,7 @@ config B43_PHY_G
 	bool "Support for G-PHY (802.11g) devices"
 	depends on B43 && B43_SSB
 	default y
-	---help---
+	help
 	  This PHY type can be found in the following chipsets:
 	  PCI: BCM4306, BCM4311, BCM4318
 	  SoC: BCM4712, BCM5352E
@@ -117,7 +117,7 @@ config B43_PHY_N
 	bool "Support for N-PHY (the main 802.11n series) devices"
 	depends on B43
 	default y
-	---help---
+	help
 	  This PHY type can be found in the following chipsets:
 	  PCI: BCM4321, BCM4322,
 	       BCM43222, BCM43224, BCM43225,
@@ -128,7 +128,7 @@ config B43_PHY_LP
 	bool "Support for LP-PHY (low-power 802.11g) devices"
 	depends on B43 && B43_SSB
 	default y
-	---help---
+	help
 	  The LP-PHY is a low-power PHY built into some notebooks
 	  and embedded devices. It supports 802.11a/b/g
 	  (802.11a support is optional, and currently disabled).
@@ -137,13 +137,13 @@ config B43_PHY_HT
 	bool "Support for HT-PHY (high throughput 802.11n) devices"
 	depends on B43 && B43_BCMA
 	default y
-	---help---
+	help
 	  This PHY type with 3x3:3 MIMO can be found in the BCM4331 PCI chipset.
 
 config B43_PHY_LCN
 	bool "Support for LCN-PHY devices (BROKEN)"
 	depends on B43 && BROKEN
-	---help---
+	help
 	  Support for the LCN-PHY.
 
 	  Say N, this is BROKEN and crashes driver.
@@ -151,7 +151,7 @@ config B43_PHY_LCN
 config B43_PHY_AC
 	bool "Support for AC-PHY (802.11ac) devices (BROKEN)"
 	depends on B43 && B43_BCMA && BROKEN
-	---help---
+	help
 	  This PHY type can be found in the following chipsets:
 	  PCI: BCM4352, BCM4360
 
@@ -174,7 +174,7 @@ config B43_HWRNG
 config B43_DEBUG
 	bool "Broadcom 43xx debugging"
 	depends on B43
-	---help---
+	help
 	  Broadcom 43xx debugging.
 
 	  This adds additional runtime sanity checks and statistics to the driver.
diff --git a/drivers/net/wireless/broadcom/b43legacy/Kconfig b/drivers/net/wireless/broadcom/b43legacy/Kconfig
index bfac34142ab5..e4da34ec4f5b 100644
--- a/drivers/net/wireless/broadcom/b43legacy/Kconfig
+++ b/drivers/net/wireless/broadcom/b43legacy/Kconfig
@@ -4,7 +4,7 @@ config B43LEGACY
 	depends on SSB_POSSIBLE && MAC80211 && HAS_DMA
 	select SSB
 	select FW_LOADER
-	---help---
+	help
 	  b43legacy is a driver for 802.11b devices from Broadcom (BCM4301 and
 	  BCM4303) and early model 802.11g chips (BCM4306 Ver. 2) used in the
 	  Linksys WPC54G V1 PCMCIA devices.
@@ -54,7 +54,7 @@ config B43LEGACY_DEBUG
 	bool "Broadcom 43xx-legacy debugging"
 	depends on B43LEGACY
 	default y
-	---help---
+	help
 	  Say Y, because this information will help you get the driver running.
 	  This option generates a minimum of log output.
 
@@ -75,7 +75,7 @@ config B43LEGACY_DMA_AND_PIO_MODE
 	bool "DMA + PIO"
 	select B43LEGACY_DMA
 	select B43LEGACY_PIO
-	---help---
+	help
 	  Include both, Direct Memory Access (DMA) and Programmed I/O (PIO)
 	  data transfer modes. The mode actually used is selectable through
 	  the module parameter "pio". With pio=0 as a module parameter, the
@@ -86,14 +86,14 @@ config B43LEGACY_DMA_AND_PIO_MODE
 config B43LEGACY_DMA_MODE
 	bool "DMA (Direct Memory Access) only"
 	select B43LEGACY_DMA
-	---help---
+	help
 	  Only include Direct Memory Access (DMA).
 	  This reduces the size of the driver module, by omitting the PIO code.
 
 config B43LEGACY_PIO_MODE
 	bool "PIO (Programmed I/O) only"
 	select B43LEGACY_PIO
-	---help---
+	help
 	  Only include Programmed I/O (PIO).
 	  This reduces the size of the driver module, by omitting the DMA code.
 	  Please note that PIO transfers are slow (compared to DMA).
diff --git a/drivers/net/wireless/broadcom/brcm80211/Kconfig b/drivers/net/wireless/broadcom/brcm80211/Kconfig
index a5bf16c4f495..5bf2318763c5 100644
--- a/drivers/net/wireless/broadcom/brcm80211/Kconfig
+++ b/drivers/net/wireless/broadcom/brcm80211/Kconfig
@@ -12,7 +12,7 @@ config BRCMSMAC
 	select BRCMUTIL
 	select FW_LOADER
 	select CORDIC
-	---help---
+	help
 	  This module adds support for PCIe wireless adapters based on Broadcom
 	  IEEE802.11n SoftMAC chipsets. It also has WLAN led support, which will
 	  be available if you select BCMA_DRIVER_GPIO. If you choose to build a
@@ -23,7 +23,7 @@ source "drivers/net/wireless/broadcom/brcm80211/brcmfmac/Kconfig"
 config BRCM_TRACING
 	bool "Broadcom device tracing"
 	depends on BRCMSMAC || BRCMFMAC
-	---help---
+	help
 	  If you say Y here, the Broadcom wireless drivers will register
 	  with ftrace to dump event information into the trace ringbuffer.
 	  Tracing can be enabled at runtime to aid in debugging wireless
@@ -35,5 +35,5 @@ config BRCMDBG
 	bool "Broadcom driver debug functions"
 	depends on BRCMSMAC || BRCMFMAC
 	select WANT_DEV_COREDUMP if BRCMFMAC
-	---help---
+	help
 	  Selecting this enables additional code for debug purposes.
diff --git a/drivers/net/wireless/cisco/Kconfig b/drivers/net/wireless/cisco/Kconfig
index 7a3b3bb2ce15..681bfc2d740a 100644
--- a/drivers/net/wireless/cisco/Kconfig
+++ b/drivers/net/wireless/cisco/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_CISCO
 	bool "Cisco devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -20,7 +20,7 @@ config AIRO
 	select CRYPTO_SKCIPHER
 	select WEXT_SPY
 	select WEXT_PRIV
-	---help---
+	help
 	  This is the standard Linux driver to support Cisco/Aironet ISA and
 	  PCI 802.11 wireless cards.
 	  It supports the new 802.11b cards from Cisco (Cisco 34X, Cisco 35X
@@ -42,7 +42,7 @@ config AIRO_CS
 	select CRYPTO
 	select CRYPTO_AES
 	select CRYPTO_CTR
-	---help---
+	help
 	  This is the standard Linux driver to support Cisco/Aironet PCMCIA
 	  802.11 wireless cards.  This driver is the same as the Aironet
 	  driver part of the Linux Pcmcia package.
diff --git a/drivers/net/wireless/intel/Kconfig b/drivers/net/wireless/intel/Kconfig
index 6ec42f67d0f2..3d1d7d11a396 100644
--- a/drivers/net/wireless/intel/Kconfig
+++ b/drivers/net/wireless/intel/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_INTEL
 	bool "Intel devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/wireless/intel/ipw2x00/Kconfig b/drivers/net/wireless/intel/ipw2x00/Kconfig
index f42b3cdce611..d00386915a9d 100644
--- a/drivers/net/wireless/intel/ipw2x00/Kconfig
+++ b/drivers/net/wireless/intel/ipw2x00/Kconfig
@@ -12,7 +12,7 @@ config IPW2100
 	select FW_LOADER
 	select LIB80211
 	select LIBIPW
-	---help---
+	help
 	  A driver for the Intel PRO/Wireless 2100 Network
 	  Connection 802.11b wireless network adapter.
 
@@ -41,7 +41,7 @@ config IPW2100
 config IPW2100_MONITOR
 	bool "Enable promiscuous mode"
 	depends on IPW2100
-	---help---
+	help
 	  Enables promiscuous/monitor mode support for the ipw2100 driver.
 	  With this feature compiled into the driver, you can switch to
 	  promiscuous mode via the Wireless Tool's Monitor mode.  While in this
@@ -50,7 +50,7 @@ config IPW2100_MONITOR
 config IPW2100_DEBUG
 	bool "Enable full debugging output in IPW2100 module."
 	depends on IPW2100
-	---help---
+	help
 	  This option will enable debug tracing output for the IPW2100.
 
 	  This will result in the kernel module being ~60k larger.  You can
@@ -74,7 +74,7 @@ config IPW2200
 	select FW_LOADER
 	select LIB80211
 	select LIBIPW
-	---help---
+	help
 	  A driver for the Intel PRO/Wireless 2200BG and 2915ABG Network
 	  Connection adapters.
 
@@ -103,7 +103,7 @@ config IPW2200
 config IPW2200_MONITOR
 	bool "Enable promiscuous mode"
 	depends on IPW2200
-	---help---
+	help
 	  Enables promiscuous/monitor mode support for the ipw2200 driver.
 	  With this feature compiled into the driver, you can switch to
 	  promiscuous mode via the Wireless Tool's Monitor mode.  While in this
@@ -117,7 +117,7 @@ config IPW2200_PROMISCUOUS
 	bool "Enable creation of a RF radiotap promiscuous interface"
 	depends on IPW2200_MONITOR
 	select IPW2200_RADIOTAP
-	---help---
+	help
 	  Enables the creation of a second interface prefixed 'rtap'.
 	  This second interface will provide every received in radiotap
 	  format.
@@ -144,7 +144,7 @@ config IPW2200_QOS
 config IPW2200_DEBUG
 	bool "Enable full debugging output in IPW2200 module."
 	depends on IPW2200
-	---help---
+	help
 	  This option will enable low level debug tracing output for IPW2200.
 
 	  Note, normal debug code is already compiled in. This low level
@@ -170,7 +170,7 @@ config LIBIPW
 	select LIB80211_CRYPT_WEP
 	select LIB80211_CRYPT_TKIP
 	select LIB80211_CRYPT_CCMP
-	---help---
+	help
 	This option enables the hardware independent IEEE 802.11
 	networking stack.  This component is deprecated in favor of the
 	mac80211 component.
@@ -178,7 +178,7 @@ config LIBIPW
 config LIBIPW_DEBUG
 	bool "Full debugging output for the LIBIPW component"
 	depends on LIBIPW
-	---help---
+	help
 	  This option will enable debug tracing output for the
 	  libipw component.
 
diff --git a/drivers/net/wireless/intel/iwlegacy/Kconfig b/drivers/net/wireless/intel/iwlegacy/Kconfig
index 100f55858b13..24fe3f63c321 100644
--- a/drivers/net/wireless/intel/iwlegacy/Kconfig
+++ b/drivers/net/wireless/intel/iwlegacy/Kconfig
@@ -11,7 +11,7 @@ config IWL4965
 	tristate "Intel Wireless WiFi 4965AGN (iwl4965)"
 	depends on PCI && MAC80211
 	select IWLEGACY
-	---help---
+	help
 	  This option enables support for
 
 	  Select to build the driver supporting the:
@@ -39,7 +39,7 @@ config IWL3945
 	tristate "Intel PRO/Wireless 3945ABG/BG Network Connection (iwl3945)"
 	depends on PCI && MAC80211
 	select IWLEGACY
-	---help---
+	help
 	  Select to build the driver supporting the:
 
 	  Intel PRO/Wireless 3945ABG/BG Network Connection
@@ -67,7 +67,7 @@ menu "iwl3945 / iwl4965 Debugging Options"
 config IWLEGACY_DEBUG
 	bool "Enable full debugging output in iwlegacy (iwl 3945/4965) drivers"
 	depends on IWLEGACY
-	---help---
+	help
 	  This option will enable debug tracing output for the iwlegacy
 	  drivers.
 
@@ -93,7 +93,7 @@ config IWLEGACY_DEBUG
 config IWLEGACY_DEBUGFS
 	bool "iwlegacy (iwl 3945/4965) debugfs support"
 	depends on IWLEGACY && MAC80211_DEBUGFS
-	---help---
+	help
 	  Enable creation of debugfs files for the iwlegacy drivers. This
 	  is a low-impact option that allows getting insight into the
 	  driver's state at runtime.
diff --git a/drivers/net/wireless/intel/iwlwifi/Kconfig b/drivers/net/wireless/intel/iwlwifi/Kconfig
index 091d621ad25f..36153fa5f96a 100644
--- a/drivers/net/wireless/intel/iwlwifi/Kconfig
+++ b/drivers/net/wireless/intel/iwlwifi/Kconfig
@@ -3,7 +3,7 @@ config IWLWIFI
 	tristate "Intel Wireless WiFi Next Gen AGN - Wireless-N/Advanced-N/Ultimate-N (iwlwifi) "
 	depends on PCI && HAS_IOMEM && CFG80211
 	select FW_LOADER
-	---help---
+	help
 	  Select to build the driver supporting the:
 
 	  Intel Wireless WiFi Link Next-Gen AGN
@@ -96,7 +96,7 @@ menu "Debugging Options"
 
 config IWLWIFI_DEBUG
 	bool "Enable full debugging output in the iwlwifi driver"
-	---help---
+	help
 	  This option will enable debug tracing output for the iwlwifi drivers
 
 	  This will result in the kernel module being ~100k larger.  You can
@@ -121,7 +121,7 @@ config IWLWIFI_DEBUG
 config IWLWIFI_DEBUGFS
 	bool "iwlwifi debugfs support"
 	depends on MAC80211_DEBUGFS
-	---help---
+	help
 	  Enable creation of debugfs files for the iwlwifi drivers. This
 	  is a low-impact option that allows getting insight into the
 	  driver's state at runtime.
diff --git a/drivers/net/wireless/intersil/Kconfig b/drivers/net/wireless/intersil/Kconfig
index 4e968912e27c..6a6ce9d4aeee 100644
--- a/drivers/net/wireless/intersil/Kconfig
+++ b/drivers/net/wireless/intersil/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_INTERSIL
 	bool "Intersil devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -23,7 +23,7 @@ config PRISM54
 	select WEXT_SPY
 	select WEXT_PRIV
 	select FW_LOADER
-	---help---
+	help
 	  This enables support for FullMAC PCI/Cardbus prism54 devices. This
 	  driver is now deprecated in favor for the SoftMAC driver, p54pci.
 	  p54pci supports FullMAC PCI/Cardbus devices as well.
diff --git a/drivers/net/wireless/intersil/hostap/Kconfig b/drivers/net/wireless/intersil/hostap/Kconfig
index c70dc168dc63..6ad88299432f 100644
--- a/drivers/net/wireless/intersil/hostap/Kconfig
+++ b/drivers/net/wireless/intersil/hostap/Kconfig
@@ -15,7 +15,7 @@ config HOSTAP
 	select LIB80211_CRYPT_WEP
 	select LIB80211_CRYPT_TKIP
 	select LIB80211_CRYPT_CCMP
-	---help---
+	help
 	Shared driver code for IEEE 802.11b wireless cards based on
 	Intersil Prism2/2.5/3 chipset. This driver supports so called
 	Host AP mode that allows the card to act as an IEEE 802.11
@@ -36,7 +36,7 @@ config HOSTAP
 config HOSTAP_FIRMWARE
 	bool "Support downloading firmware images with Host AP driver"
 	depends on HOSTAP
-	---help---
+	help
 	Configure Host AP driver to include support for firmware image
 	download. This option by itself only enables downloading to the
 	volatile memory, i.e. the card RAM. This option is required to
@@ -49,7 +49,7 @@ config HOSTAP_FIRMWARE
 config HOSTAP_FIRMWARE_NVRAM
 	bool "Support for non-volatile firmware download"
 	depends on HOSTAP_FIRMWARE
-	---help---
+	help
 	Allow Host AP driver to write firmware images to the non-volatile
 	card memory, i.e. flash memory that survives power cycling.
 	Enable this option if you want to be able to change card firmware
@@ -61,7 +61,7 @@ config HOSTAP_FIRMWARE_NVRAM
 config HOSTAP_PLX
 	tristate "Host AP driver for Prism2/2.5/3 in PLX9052 PCI adaptors"
 	depends on PCI && HOSTAP
-	---help---
+	help
 	Host AP driver's version for Prism2/2.5/3 PC Cards in PLX9052 based
 	PCI adaptors.
 
@@ -75,7 +75,7 @@ config HOSTAP_PLX
 config HOSTAP_PCI
 	tristate "Host AP driver for Prism2.5 PCI adaptors"
 	depends on PCI && HOSTAP
-	---help---
+	help
 	Host AP driver's version for Prism2.5 PCI adaptors.
 
 	"Host AP support for Prism2/2.5/3 IEEE 802.11b" is required for this
@@ -88,7 +88,7 @@ config HOSTAP_PCI
 config HOSTAP_CS
 	tristate "Host AP driver for Prism2/2.5/3 PC Cards"
 	depends on PCMCIA && HOSTAP
-	---help---
+	help
 	Host AP driver's version for Prism2/2.5/3 PC Cards.
 
 	"Host AP support for Prism2/2.5/3 IEEE 802.11b" is required for this
diff --git a/drivers/net/wireless/intersil/hostap/hostap_hw.c b/drivers/net/wireless/intersil/hostap/hostap_hw.c
index aadf3dec5bf3..2ab34cf74ecc 100644
--- a/drivers/net/wireless/intersil/hostap/hostap_hw.c
+++ b/drivers/net/wireless/intersil/hostap/hostap_hw.c
@@ -3048,6 +3048,7 @@ static void prism2_clear_set_tim_queue(local_info_t *local)
  * This is a natural nesting, which needs a split lock type.
  */
 static struct lock_class_key hostap_netdev_xmit_lock_key;
+static struct lock_class_key hostap_netdev_addr_lock_key;
 
 static void prism2_set_lockdep_class_one(struct net_device *dev,
 					 struct netdev_queue *txq,
@@ -3059,6 +3060,8 @@ static void prism2_set_lockdep_class_one(struct net_device *dev,
 
 static void prism2_set_lockdep_class(struct net_device *dev)
 {
+	lockdep_set_class(&dev->addr_list_lock,
+			  &hostap_netdev_addr_lock_key);
 	netdev_for_each_tx_queue(dev, prism2_set_lockdep_class_one, NULL);
 }
 
diff --git a/drivers/net/wireless/intersil/orinoco/Kconfig b/drivers/net/wireless/intersil/orinoco/Kconfig
index c430d7a46730..c470ee23673f 100644
--- a/drivers/net/wireless/intersil/orinoco/Kconfig
+++ b/drivers/net/wireless/intersil/orinoco/Kconfig
@@ -10,7 +10,7 @@ config HERMES
 	select FW_LOADER
 	select CRYPTO
 	select CRYPTO_MICHAEL_MIC
-	---help---
+	help
 	  A driver for 802.11b wireless cards based on the "Hermes" or
 	  Intersil HFA384x (Prism 2) MAC controller.  This includes the vast
 	  majority of the PCMCIA 802.11b cards (which are nearly all rebadges)
@@ -32,7 +32,7 @@ config HERMES
 config HERMES_PRISM
 	bool "Support Prism 2/2.5 chipset"
 	depends on HERMES
-	---help---
+	help
 
 	  Say Y to enable support for Prism 2 and 2.5 chipsets.  These
 	  chipsets are better handled by the hostap driver.  This driver
@@ -44,7 +44,7 @@ config HERMES_CACHE_FW_ON_INIT
 	bool "Cache Hermes firmware on driver initialisation"
 	depends on HERMES
 	default y
-	---help---
+	help
 	  Say Y to cache any firmware required by the Hermes drivers
 	  on startup.  The firmware will remain cached until the
 	  driver is unloaded.  The cache uses 64K of RAM.
@@ -110,7 +110,7 @@ config PCI_HERMES
 config PCMCIA_HERMES
 	tristate "Hermes PCMCIA card support"
 	depends on PCMCIA && HERMES && HAS_IOPORT_MAP
-	---help---
+	help
 	  A driver for "Hermes" chipset based PCMCIA wireless adaptors, such
 	  as the Lucent WavelanIEEE/Orinoco cards and their OEM (Cabletron/
 	  EnteraSys RoamAbout 802.11, ELSA Airlancer, Melco Buffalo and
@@ -125,7 +125,7 @@ config PCMCIA_HERMES
 config PCMCIA_SPECTRUM
 	tristate "Symbol Spectrum24 Trilogy PCMCIA card support"
 	depends on PCMCIA && HERMES && HAS_IOPORT_MAP
-	---help---
+	help
 
 	  This is a driver for 802.11b cards using RAM-loadable Symbol
 	  firmware, such as Symbol Wireless Networker LA4100, CompactFlash
@@ -139,5 +139,5 @@ config ORINOCO_USB
 	tristate "Agere Orinoco USB support"
 	depends on USB && HERMES
 	select FW_LOADER
-	---help---
+	help
 	  This driver is for USB versions of the Agere Orinoco card.
diff --git a/drivers/net/wireless/intersil/p54/Kconfig b/drivers/net/wireless/intersil/p54/Kconfig
index 26cd80732afa..024be5507daf 100644
--- a/drivers/net/wireless/intersil/p54/Kconfig
+++ b/drivers/net/wireless/intersil/p54/Kconfig
@@ -4,7 +4,7 @@ config P54_COMMON
 	depends on MAC80211
 	select FW_LOADER
 	select CRC_CCITT
-	---help---
+	help
 	  This is common code for isl38xx/stlc45xx based modules.
 	  This module does nothing by itself - the USB/PCI/SPI front-ends
 	  also need to be enabled in order to support any devices.
@@ -18,7 +18,7 @@ config P54_USB
 	tristate "Prism54 USB support"
 	depends on P54_COMMON && USB
 	select CRC32
-	---help---
+	help
 	  This driver is for USB isl38xx based wireless cards.
 
 	  These devices require softmac firmware which can be found at
@@ -29,7 +29,7 @@ config P54_USB
 config P54_PCI
 	tristate "Prism54 PCI support"
 	depends on P54_COMMON && PCI
-	---help---
+	help
 	  This driver is for PCI isl38xx based wireless cards.
 	  This driver supports most devices that are supported by the
 	  fullmac prism54 driver plus many devices which are not
@@ -43,7 +43,7 @@ config P54_PCI
 config P54_SPI
 	tristate "Prism54 SPI (stlc45xx) support"
 	depends on P54_COMMON && SPI_MASTER
-	---help---
+	help
 	  This driver is for stlc4550 or stlc4560 based wireless chips
 	  such as Nokia's N800/N810 Portable Internet Tablet.
 
@@ -53,7 +53,7 @@ config P54_SPI_DEFAULT_EEPROM
 	bool "Include fallback EEPROM blob"
 	depends on P54_SPI
 	default n
-	---help---
+	help
 	 Unlike the PCI or USB devices, the SPI variants don't have
 	 a dedicated EEPROM chip to store all device specific values
 	 for calibration, country and interface settings.
diff --git a/drivers/net/wireless/marvell/Kconfig b/drivers/net/wireless/marvell/Kconfig
index dff82fdbea78..864bc0f6d4d4 100644
--- a/drivers/net/wireless/marvell/Kconfig
+++ b/drivers/net/wireless/marvell/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_MARVELL
 	bool "Marvell devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -19,7 +19,7 @@ source "drivers/net/wireless/marvell/mwifiex/Kconfig"
 config MWL8K
 	tristate "Marvell 88W8xxx PCI/PCIe Wireless support"
 	depends on MAC80211 && PCI
-	---help---
+	help
 	  This driver supports Marvell TOPDOG 802.11 wireless cards.
 
 	  To compile this driver as a module, choose M here: the module
diff --git a/drivers/net/wireless/marvell/libertas/Kconfig b/drivers/net/wireless/marvell/libertas/Kconfig
index b9fe598130c3..6d62ab49aa8d 100644
--- a/drivers/net/wireless/marvell/libertas/Kconfig
+++ b/drivers/net/wireless/marvell/libertas/Kconfig
@@ -6,37 +6,37 @@ config LIBERTAS
 	select WEXT_SPY
 	select LIB80211
 	select FW_LOADER
-	---help---
+	help
 	  A library for Marvell Libertas 8xxx devices.
 
 config LIBERTAS_USB
 	tristate "Marvell Libertas 8388 USB 802.11b/g cards"
 	depends on LIBERTAS && USB
-	---help---
+	help
 	  A driver for Marvell Libertas 8388 USB devices.
 
 config LIBERTAS_CS
 	tristate "Marvell Libertas 8385 CompactFlash 802.11b/g cards"
 	depends on LIBERTAS && PCMCIA && HAS_IOPORT_MAP
-	---help---
+	help
 	  A driver for Marvell Libertas 8385 CompactFlash devices.
 
 config LIBERTAS_SDIO
 	tristate "Marvell Libertas 8385/8686/8688 SDIO 802.11b/g cards"
 	depends on LIBERTAS && MMC
-	---help---
+	help
 	  A driver for Marvell Libertas 8385/8686/8688 SDIO devices.
 
 config LIBERTAS_SPI
 	tristate "Marvell Libertas 8686 SPI 802.11b/g cards"
 	depends on LIBERTAS && SPI
-	---help---
+	help
 	  A driver for Marvell Libertas 8686 SPI devices.
 
 config LIBERTAS_DEBUG
 	bool "Enable full debugging output in the Libertas module."
 	depends on LIBERTAS
-	---help---
+	help
 	  Debugging support.
 
 config LIBERTAS_MESH
diff --git a/drivers/net/wireless/marvell/libertas_tf/Kconfig b/drivers/net/wireless/marvell/libertas_tf/Kconfig
index aa40d65f611a..345da4a6ee8d 100644
--- a/drivers/net/wireless/marvell/libertas_tf/Kconfig
+++ b/drivers/net/wireless/marvell/libertas_tf/Kconfig
@@ -3,17 +3,17 @@ config LIBERTAS_THINFIRM
 	tristate "Marvell 8xxx Libertas WLAN driver support with thin firmware"
 	depends on MAC80211
 	select FW_LOADER
-	---help---
+	help
 	  A library for Marvell Libertas 8xxx devices using thinfirm.
 
 config LIBERTAS_THINFIRM_DEBUG
 	bool "Enable full debugging output in the Libertas thin firmware module."
 	depends on LIBERTAS_THINFIRM
-	---help---
+	help
 	  Debugging support.
 
 config LIBERTAS_THINFIRM_USB
 	tristate "Marvell Libertas 8388 USB 802.11b/g cards with thin firmware"
 	depends on LIBERTAS_THINFIRM && USB
-	---help---
+	help
 	  A driver for Marvell Libertas 8388 USB devices using thinfirm.
diff --git a/drivers/net/wireless/marvell/mwifiex/Kconfig b/drivers/net/wireless/marvell/mwifiex/Kconfig
index 64d8a11ad1e7..2b4ff2b78a7e 100644
--- a/drivers/net/wireless/marvell/mwifiex/Kconfig
+++ b/drivers/net/wireless/marvell/mwifiex/Kconfig
@@ -2,7 +2,7 @@
 config MWIFIEX
 	tristate "Marvell WiFi-Ex Driver"
 	depends on CFG80211
-	---help---
+	help
 	  This adds support for wireless adapters based on Marvell
 	  802.11n/ac chipsets.
 
@@ -14,7 +14,7 @@ config MWIFIEX_SDIO
 	depends on MWIFIEX && MMC
 	select FW_LOADER
 	select WANT_DEV_COREDUMP
-	---help---
+	help
 	  This adds support for wireless adapters based on Marvell
 	  8786/8787/8797/8887/8897/8977/8987/8997 chipsets with SDIO interface.
 
@@ -26,7 +26,7 @@ config MWIFIEX_PCIE
 	depends on MWIFIEX && PCI
 	select FW_LOADER
 	select WANT_DEV_COREDUMP
-	---help---
+	help
 	  This adds support for wireless adapters based on Marvell
 	  8766/8897/8997 chipsets with PCIe interface.
 
@@ -37,7 +37,7 @@ config MWIFIEX_USB
 	tristate "Marvell WiFi-Ex Driver for USB8766/8797/8997"
 	depends on MWIFIEX && USB
 	select FW_LOADER
-	---help---
+	help
 	  This adds support for wireless adapters based on Marvell
 	  8797/8997 chipset with USB interface.
 
diff --git a/drivers/net/wireless/mediatek/Kconfig b/drivers/net/wireless/mediatek/Kconfig
index 02d1120f148f..bd4db128248e 100644
--- a/drivers/net/wireless/mediatek/Kconfig
+++ b/drivers/net/wireless/mediatek/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_MEDIATEK
 	bool "MediaTek devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/wireless/mediatek/mt7601u/Kconfig b/drivers/net/wireless/mediatek/mt7601u/Kconfig
index 0b230f303d53..4a8b96280670 100644
--- a/drivers/net/wireless/mediatek/mt7601u/Kconfig
+++ b/drivers/net/wireless/mediatek/mt7601u/Kconfig
@@ -3,5 +3,5 @@ config MT7601U
 	tristate "MediaTek MT7601U (USB) support"
 	depends on MAC80211
 	depends on USB
-	---help---
+	help
 	  This adds support for MT7601U-based wireless USB dongles.
diff --git a/drivers/net/wireless/ralink/Kconfig b/drivers/net/wireless/ralink/Kconfig
index 92eec8fff8fd..135a191c435d 100644
--- a/drivers/net/wireless/ralink/Kconfig
+++ b/drivers/net/wireless/ralink/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_RALINK
 	bool "Ralink devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/wireless/ralink/rt2x00/Kconfig b/drivers/net/wireless/ralink/rt2x00/Kconfig
index d4969d617822..dcccc290a7f5 100644
--- a/drivers/net/wireless/ralink/rt2x00/Kconfig
+++ b/drivers/net/wireless/ralink/rt2x00/Kconfig
@@ -2,7 +2,7 @@
 menuconfig RT2X00
 	tristate "Ralink driver support"
 	depends on MAC80211 && HAS_DMA
-	---help---
+	help
 	  This will enable the support for the Ralink drivers,
 	  developed in the rt2x00 project <http://rt2x00.serialmonkey.com>.
 
@@ -24,7 +24,7 @@ config RT2400PCI
 	select RT2X00_LIB_MMIO
 	select RT2X00_LIB_PCI
 	select EEPROM_93CX6
-	---help---
+	help
 	  This adds support for rt2400 wireless chipset family.
 	  Supported chips: RT2460.
 
@@ -36,7 +36,7 @@ config RT2500PCI
 	select RT2X00_LIB_MMIO
 	select RT2X00_LIB_PCI
 	select EEPROM_93CX6
-	---help---
+	help
 	  This adds support for rt2500 wireless chipset family.
 	  Supported chips: RT2560.
 
@@ -51,7 +51,7 @@ config RT61PCI
 	select RT2X00_LIB_CRYPTO
 	select CRC_ITU_T
 	select EEPROM_93CX6
-	---help---
+	help
 	  This adds support for rt2501 wireless chipset family.
 	  Supported chips: RT2561, RT2561S & RT2661.
 
@@ -68,7 +68,7 @@ config RT2800PCI
 	select RT2X00_LIB_CRYPTO
 	select CRC_CCITT
 	select EEPROM_93CX6
-	---help---
+	help
 	  This adds support for rt27xx/rt28xx/rt30xx wireless chipset family.
 	  Supported chips: RT2760, RT2790, RT2860, RT2880, RT2890, RT3052,
 			   RT3090, RT3091 & RT3092
@@ -80,7 +80,7 @@ if RT2800PCI
 config RT2800PCI_RT33XX
 	bool "rt2800pci - Include support for rt33xx devices"
 	default y
-	---help---
+	help
 	  This adds support for rt33xx wireless chipset family to the
 	  rt2800pci driver.
 	  Supported chips: RT3390
@@ -88,7 +88,7 @@ config RT2800PCI_RT33XX
 config RT2800PCI_RT35XX
 	bool "rt2800pci - Include support for rt35xx devices (EXPERIMENTAL)"
 	default y
-	---help---
+	help
 	  This adds support for rt35xx wireless chipset family to the
 	  rt2800pci driver.
 	  Supported chips: RT3060, RT3062, RT3562, RT3592
@@ -97,7 +97,7 @@ config RT2800PCI_RT35XX
 config RT2800PCI_RT53XX
 	bool "rt2800pci - Include support for rt53xx devices (EXPERIMENTAL)"
 	default y
-	---help---
+	help
 	  This adds support for rt53xx wireless chipset family to the
 	  rt2800pci driver.
 	  Supported chips: RT5390
@@ -105,7 +105,7 @@ config RT2800PCI_RT53XX
 config RT2800PCI_RT3290
 	bool "rt2800pci - Include support for rt3290 devices (EXPERIMENTAL)"
 	default y
-	---help---
+	help
 	  This adds support for rt3290 wireless chipset family to the
 	  rt2800pci driver.
 	  Supported chips: RT3290
@@ -116,7 +116,7 @@ config RT2500USB
 	depends on USB
 	select RT2X00_LIB_USB
 	select RT2X00_LIB_CRYPTO
-	---help---
+	help
 	  This adds support for rt2500 wireless chipset family.
 	  Supported chips: RT2571 & RT2572.
 
@@ -129,7 +129,7 @@ config RT73USB
 	select RT2X00_LIB_FIRMWARE
 	select RT2X00_LIB_CRYPTO
 	select CRC_ITU_T
-	---help---
+	help
 	  This adds support for rt2501 wireless chipset family.
 	  Supported chips: RT2571W, RT2573 & RT2671.
 
@@ -143,7 +143,7 @@ config RT2800USB
 	select RT2X00_LIB_FIRMWARE
 	select RT2X00_LIB_CRYPTO
 	select CRC_CCITT
-	---help---
+	help
 	  This adds support for rt27xx/rt28xx/rt30xx wireless chipset family.
 	  Supported chips: RT2770, RT2870 & RT3070, RT3071 & RT3072
 
@@ -154,7 +154,7 @@ if RT2800USB
 config RT2800USB_RT33XX
 	bool "rt2800usb - Include support for rt33xx devices"
 	default y
-	---help---
+	help
 	  This adds support for rt33xx wireless chipset family to the
 	  rt2800usb driver.
 	  Supported chips: RT3370
@@ -162,27 +162,27 @@ config RT2800USB_RT33XX
 config RT2800USB_RT35XX
 	bool "rt2800usb - Include support for rt35xx devices (EXPERIMENTAL)"
 	default y
-	---help---
+	help
 	  This adds support for rt35xx wireless chipset family to the
 	  rt2800usb driver.
 	  Supported chips: RT3572
 
 config RT2800USB_RT3573
 	bool "rt2800usb - Include support for rt3573 devices (EXPERIMENTAL)"
-	---help---
+	help
 	  This enables support for RT3573 chipset based wireless USB devices
 	  in the rt2800usb driver.
 
 config RT2800USB_RT53XX
 	bool "rt2800usb - Include support for rt53xx devices (EXPERIMENTAL)"
-	---help---
+	help
 	  This adds support for rt53xx wireless chipset family to the
 	  rt2800usb driver.
 	  Supported chips: RT5370
 
 config RT2800USB_RT55XX
 	bool "rt2800usb - Include support for rt55xx devices (EXPERIMENTAL)"
-	---help---
+	help
 	  This adds support for rt55xx wireless chipset family to the
 	  rt2800usb driver.
 	  Supported chips: RT5572
@@ -190,7 +190,7 @@ config RT2800USB_RT55XX
 config RT2800USB_UNKNOWN
 	bool "rt2800usb - Include support for unknown (USB) devices"
 	default n
-	---help---
+	help
 	  This adds support for rt2800usb devices that are known to
 	  have a rt28xx family compatible chipset, but for which the exact
 	  chipset is unknown.
@@ -209,7 +209,7 @@ config RT2800SOC
 	select RT2X00_LIB_FIRMWARE
 	select RT2800_LIB
 	select RT2800_LIB_MMIO
-	---help---
+	help
 	  This adds support for Ralink WiSoC devices.
 	  Supported chips: RT2880, RT3050, RT3052, RT3350, RT3352.
 
@@ -256,7 +256,7 @@ config RT2X00_LIB_LEDS
 config RT2X00_LIB_DEBUGFS
 	bool "Ralink debugfs support"
 	depends on RT2X00_LIB && MAC80211_DEBUGFS
-	---help---
+	help
 	  Enable creation of debugfs files for the rt2x00 drivers.
 	  These debugfs files support both reading and writing of the
 	  most important register types of the rt2x00 hardware.
@@ -264,7 +264,7 @@ config RT2X00_LIB_DEBUGFS
 config RT2X00_DEBUG
 	bool "Ralink debug output"
 	depends on RT2X00_LIB
-	---help---
+	help
 	  Enable debugging output for all rt2x00 modules
 
 endif
diff --git a/drivers/net/wireless/realtek/Kconfig b/drivers/net/wireless/realtek/Kconfig
index 8ea2d8d7e356..474843277fa1 100644
--- a/drivers/net/wireless/realtek/Kconfig
+++ b/drivers/net/wireless/realtek/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_REALTEK
 	bool "Realtek devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/wireless/realtek/rtl818x/Kconfig b/drivers/net/wireless/realtek/rtl818x/Kconfig
index e1aa3fc71e66..603812c9a43e 100644
--- a/drivers/net/wireless/realtek/rtl818x/Kconfig
+++ b/drivers/net/wireless/realtek/rtl818x/Kconfig
@@ -6,7 +6,7 @@ config RTL8180
 	tristate "Realtek 8180/8185/8187SE PCI support"
 	depends on MAC80211 && PCI
 	select EEPROM_93CX6
-	---help---
+	help
 	  This is a driver for RTL8180, RTL8185 and RTL8187SE based cards.
 	  These are PCI based chips found in cards such as:
 
@@ -62,7 +62,7 @@ config RTL8187
 	tristate "Realtek 8187 and 8187B USB support"
 	depends on MAC80211 && USB
 	select EEPROM_93CX6
-	---help---
+	help
 	  This is a driver for RTL8187 and RTL8187B based cards.
 	  These are USB based chips found in devices such as:
 
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/Kconfig b/drivers/net/wireless/realtek/rtl8xxxu/Kconfig
index 32d151cde618..a263507a77a6 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/Kconfig
+++ b/drivers/net/wireless/realtek/rtl8xxxu/Kconfig
@@ -5,7 +5,7 @@
 config RTL8XXXU
 	tristate "RTL8723AU/RTL8188[CR]U/RTL819[12]CU (mac80211) support"
 	depends on MAC80211 && USB
-	---help---
+	help
 	  This is an alternative driver for various Realtek RTL8XXX
 	  parts written to utilize the Linux mac80211 stack.
 	  The driver is known to work with a number of RTL8723AU,
@@ -27,7 +27,7 @@ config RTL8XXXU
 config RTL8XXXU_UNTESTED
 	bool "Include support for untested Realtek 8xxx USB devices (EXPERIMENTAL)"
 	depends on RTL8XXXU
-	---help---
+	help
 	  This option enables detection of Realtek 8723/8188/8191/8192 WiFi
 	  USB devices which have not been tested directly by the driver
 	  author or reported to be working by third parties.
diff --git a/drivers/net/wireless/realtek/rtlwifi/Kconfig b/drivers/net/wireless/realtek/rtlwifi/Kconfig
index 28c247f7f6f8..9f6a4e35543c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/Kconfig
+++ b/drivers/net/wireless/realtek/rtlwifi/Kconfig
@@ -3,7 +3,7 @@ menuconfig RTL_CARDS
 	tristate "Realtek rtlwifi family of devices"
 	depends on MAC80211 && (PCI || USB)
 	default y
-	---help---
+	help
 	  This option will enable support for the Realtek mac80211-based
 	  wireless drivers. Drivers rtl8192ce, rtl8192cu, rtl8192se, rtl8192de,
 	  rtl8723ae, rtl8723be, rtl8188ee, rtl8192ee, and rtl8821ae share
@@ -17,7 +17,7 @@ config RTL8192CE
 	select RTL8192C_COMMON
 	select RTLWIFI
 	select RTLWIFI_PCI
-	---help---
+	help
 	This is the driver for Realtek RTL8192CE/RTL8188CE 802.11n PCIe
 	wireless network adapters.
 
@@ -28,7 +28,7 @@ config RTL8192SE
 	depends on PCI
 	select RTLWIFI
 	select RTLWIFI_PCI
-	---help---
+	help
 	This is the driver for Realtek RTL8192SE/RTL8191SE 802.11n PCIe
 	wireless network adapters.
 
@@ -39,7 +39,7 @@ config RTL8192DE
 	depends on PCI
 	select RTLWIFI
 	select RTLWIFI_PCI
-	---help---
+	help
 	This is the driver for Realtek RTL8192DE/RTL8188DE 802.11n PCIe
 	wireless network adapters.
 
@@ -52,7 +52,7 @@ config RTL8723AE
 	select RTLWIFI_PCI
 	select RTL8723_COMMON
 	select RTLBTCOEXIST
-	---help---
+	help
 	This is the driver for Realtek RTL8723AE 802.11n PCIe
 	wireless network adapters.
 
@@ -65,7 +65,7 @@ config RTL8723BE
 	select RTLWIFI_PCI
 	select RTL8723_COMMON
 	select RTLBTCOEXIST
-	---help---
+	help
 	This is the driver for Realtek RTL8723BE 802.11n PCIe
 	wireless network adapters.
 
@@ -76,7 +76,7 @@ config RTL8188EE
 	depends on PCI
 	select RTLWIFI
 	select RTLWIFI_PCI
-	---help---
+	help
 	This is the driver for Realtek RTL8188EE 802.11n PCIe
 	wireless network adapters.
 
@@ -88,7 +88,7 @@ config RTL8192EE
 	select RTLWIFI
 	select RTLWIFI_PCI
 	select RTLBTCOEXIST
-	---help---
+	help
 	This is the driver for Realtek RTL8192EE 802.11n PCIe
 	wireless network adapters.
 
@@ -100,7 +100,7 @@ config RTL8821AE
 	select RTLWIFI
 	select RTLWIFI_PCI
 	select RTLBTCOEXIST
-	---help---
+	help
 	This is the driver for Realtek RTL8821AE/RTL8812AE 802.11ac PCIe
 	wireless network adapters.
 
@@ -112,7 +112,7 @@ config RTL8192CU
 	select RTLWIFI
 	select RTLWIFI_USB
 	select RTL8192C_COMMON
-	---help---
+	help
 	This is the driver for Realtek RTL8192CU/RTL8188CU 802.11n USB
 	wireless network adapters.
 
@@ -132,7 +132,7 @@ config RTLWIFI_DEBUG
 	bool "Debugging output for rtlwifi driver family"
 	depends on RTLWIFI
 	default y
-	---help---
+	help
 	To use the module option that sets the dynamic-debugging level for,
 	the front-end driver, this parameter must be "Y". For memory-limited
 	systems, choose "N". If in doubt, choose "Y".
diff --git a/drivers/net/wireless/rsi/Kconfig b/drivers/net/wireless/rsi/Kconfig
index ad5d34350cf9..09a592409204 100644
--- a/drivers/net/wireless/rsi/Kconfig
+++ b/drivers/net/wireless/rsi/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_RSI
 	bool "Redpine Signals Inc devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -16,7 +16,7 @@ config RSI_91X
 	tristate "Redpine Signals Inc 91x WLAN driver support"
 	select BT_HCIRSI if RSI_COEX
 	depends on MAC80211
-	---help---
+	help
 	  This option enabes support for RSI 1x1 devices.
 	  Select M (recommended), if you have a RSI 1x1 wireless module.
 
@@ -24,7 +24,7 @@ config RSI_DEBUGFS
 	bool "Redpine Signals Inc debug support"
 	depends on RSI_91X
 	default y
-	---help---
+	help
 	 Say Y, if you would like to enable debug support. This option
 	 creates debugfs entries
 
@@ -32,7 +32,7 @@ config RSI_SDIO
 	tristate "Redpine Signals SDIO bus support"
 	depends on MMC && RSI_91X
 	default m
-	---help---
+	help
 	  This option enables the SDIO bus support in rsi drivers.
 	  Select M (recommended), if you have a RSI 1x1 wireless module.
 
@@ -40,7 +40,7 @@ config RSI_USB
 	tristate "Redpine Signals USB bus support"
 	depends on USB && RSI_91X
 	default m
-	---help---
+	help
 	  This option enables the USB bus support in rsi drivers.
 	  Select M (recommended), if you have a RSI 1x1 wireless module.
 
@@ -49,7 +49,7 @@ config RSI_COEX
 	depends on BT && RSI_91X
 	depends on !(BT=m && RSI_91X=y)
 	default y
-	---help---
+	help
 	  This option enables the WLAN BT coex support in rsi drivers.
 	  Select M (recommended), if you have want to use this feature
 	  and you have RS9113 module.
diff --git a/drivers/net/wireless/st/Kconfig b/drivers/net/wireless/st/Kconfig
index 441d1b8e7b80..cd647a533d0e 100644
--- a/drivers/net/wireless/st/Kconfig
+++ b/drivers/net/wireless/st/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_ST
 	bool "STMicroelectronics devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
diff --git a/drivers/net/wireless/ti/Kconfig b/drivers/net/wireless/ti/Kconfig
index b81f2e41a63a..7c0b17a76fe2 100644
--- a/drivers/net/wireless/ti/Kconfig
+++ b/drivers/net/wireless/ti/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_TI
 	bool "Texas Instrument devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -22,7 +22,7 @@ config WILINK_PLATFORM_DATA
 	bool "TI WiLink platform data"
 	depends on WLCORE_SDIO || WL1251_SDIO
 	default y
-	---help---
+	help
 	Small platform data bit needed to pass data to the sdio modules.
 
 
diff --git a/drivers/net/wireless/ti/wl1251/Kconfig b/drivers/net/wireless/ti/wl1251/Kconfig
index 7d39f0a4ba5b..de1c70e10cda 100644
--- a/drivers/net/wireless/ti/wl1251/Kconfig
+++ b/drivers/net/wireless/ti/wl1251/Kconfig
@@ -4,7 +4,7 @@ config WL1251
 	depends on MAC80211
 	select FW_LOADER
 	select CRC7
-	---help---
+	help
 	  This will enable TI wl1251 driver support. The drivers make
 	  use of the mac80211 stack.
 
@@ -14,7 +14,7 @@ config WL1251
 config WL1251_SPI
 	tristate "TI wl1251 SPI support"
 	depends on WL1251 && SPI_MASTER
-	---help---
+	help
 	  This module adds support for the SPI interface of adapters using
 	  TI wl1251 chipset.  Select this if your platform is using
 	  the SPI bus.
@@ -25,7 +25,7 @@ config WL1251_SPI
 config WL1251_SDIO
 	tristate "TI wl1251 SDIO support"
 	depends on WL1251 && MMC
-	---help---
+	help
 	  This module adds support for the SDIO interface of adapters using
 	  TI wl1251 chipset.  Select this if your platform is using
 	  the SDIO bus.
diff --git a/drivers/net/wireless/ti/wl12xx/Kconfig b/drivers/net/wireless/ti/wl12xx/Kconfig
index 9c4511604b67..786fff288505 100644
--- a/drivers/net/wireless/ti/wl12xx/Kconfig
+++ b/drivers/net/wireless/ti/wl12xx/Kconfig
@@ -3,7 +3,7 @@ config WL12XX
 	tristate "TI wl12xx support"
 	depends on MAC80211
 	select WLCORE
-	---help---
+	help
 	  This module adds support for wireless adapters based on TI wl1271,
 	  wl1273, wl1281 and wl1283 chipsets. This module does *not* include
 	  support for wl1251.  For wl1251 support, use the separate homonymous
diff --git a/drivers/net/wireless/ti/wl18xx/Kconfig b/drivers/net/wireless/ti/wl18xx/Kconfig
index e29aa2a3ba8a..98fe188dfb41 100644
--- a/drivers/net/wireless/ti/wl18xx/Kconfig
+++ b/drivers/net/wireless/ti/wl18xx/Kconfig
@@ -3,6 +3,6 @@ config WL18XX
 	tristate "TI wl18xx support"
 	depends on MAC80211
 	select WLCORE
-	---help---
+	help
 	  This module adds support for wireless adapters based on TI
 	  WiLink 8 chipsets.
diff --git a/drivers/net/wireless/ti/wlcore/Kconfig b/drivers/net/wireless/ti/wlcore/Kconfig
index a9db1288221c..8094323256fb 100644
--- a/drivers/net/wireless/ti/wlcore/Kconfig
+++ b/drivers/net/wireless/ti/wlcore/Kconfig
@@ -3,7 +3,7 @@ config WLCORE
 	tristate "TI wlcore support"
 	depends on MAC80211
 	select FW_LOADER
-	---help---
+	help
 	  This module contains the main code for TI WLAN chips.  It abstracts
 	  hardware-specific differences among different chipset families.
 	  Each chipset family needs to implement its own lower-level module
@@ -16,7 +16,7 @@ config WLCORE_SPI
 	tristate "TI wlcore SPI support"
 	depends on WLCORE && SPI_MASTER && OF
 	select CRC7
-	---help---
+	help
 	  This module adds support for the SPI interface of adapters using
 	  TI WLAN chipsets.  Select this if your platform is using
 	  the SPI bus.
@@ -27,7 +27,7 @@ config WLCORE_SPI
 config WLCORE_SDIO
 	tristate "TI wlcore SDIO support"
 	depends on WLCORE && MMC
-	---help---
+	help
 	  This module adds support for the SDIO interface of adapters using
 	  TI WLAN chipsets.  Select this if your platform is using
 	  the SDIO bus.
diff --git a/drivers/net/wireless/zydas/Kconfig b/drivers/net/wireless/zydas/Kconfig
index 78a45cc4c014..08574433df66 100644
--- a/drivers/net/wireless/zydas/Kconfig
+++ b/drivers/net/wireless/zydas/Kconfig
@@ -2,7 +2,7 @@
 config WLAN_VENDOR_ZYDAS
 	bool "ZyDAS devices"
 	default y
-	---help---
+	help
 	  If you have a wireless card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
@@ -18,7 +18,7 @@ config USB_ZD1201
 	select WIRELESS_EXT
 	select WEXT_PRIV
 	select FW_LOADER
-	---help---
+	help
 	  Say Y if you want to use wireless LAN adapters based on the ZyDAS
 	  ZD1201 chip.
 
diff --git a/drivers/net/wireless/zydas/zd1211rw/Kconfig b/drivers/net/wireless/zydas/zd1211rw/Kconfig
index 0b7f1810f6e2..f9da8c0685d4 100644
--- a/drivers/net/wireless/zydas/zd1211rw/Kconfig
+++ b/drivers/net/wireless/zydas/zd1211rw/Kconfig
@@ -3,7 +3,7 @@ config ZD1211RW
 	tristate "ZyDAS ZD1211/ZD1211B USB-wireless support"
 	depends on USB && MAC80211
 	select FW_LOADER
-	---help---
+	help
 	  This is a driver for the ZyDAS ZD1211/ZD1211B wireless
 	  chip, present in many USB-wireless adapters.
 
@@ -13,7 +13,7 @@ config ZD1211RW
 config ZD1211RW_DEBUG
 	bool "ZyDAS ZD1211 debugging"
 	depends on ZD1211RW
-	---help---
+	help
 	  ZD1211 debugging messages. Choosing Y will result in additional debug
 	  messages being saved to your kernel logs, which may help debug any
 	  problems.
diff --git a/drivers/nfc/fdp/Kconfig b/drivers/nfc/fdp/Kconfig
index f575d2aec553..6c51f4af4256 100644
--- a/drivers/nfc/fdp/Kconfig
+++ b/drivers/nfc/fdp/Kconfig
@@ -4,7 +4,7 @@ config NFC_FDP
 	depends on NFC_NCI
 	select CRC_CCITT
 	default n
-	---help---
+	help
 	  Intel Fields Peak NFC controller core driver.
 	  This is a driver based on the NCI NFC kernel layers.
 
@@ -15,7 +15,7 @@ config NFC_FDP
 config NFC_FDP_I2C
 	tristate "NFC FDP i2c support"
 	depends on NFC_FDP && I2C
-	---help---
+	help
 	  This module adds support for the Intel Fields Peak NFC controller
 	  i2c interface.
 	  Select this if your platform is using the i2c bus.
diff --git a/drivers/nfc/microread/Kconfig b/drivers/nfc/microread/Kconfig
index 07be6d48a644..21a682b19613 100644
--- a/drivers/nfc/microread/Kconfig
+++ b/drivers/nfc/microread/Kconfig
@@ -2,7 +2,7 @@
 config NFC_MICROREAD
 	tristate
 	select CRC_CCITT
-	---help---
+	help
 	  This module contains the main code for Inside Secure microread
 	  NFC chipsets. It implements the chipset HCI logic and hooks into
 	  the NFC kernel APIs. Physical layers will register against it.
@@ -11,7 +11,7 @@ config NFC_MICROREAD_I2C
 	tristate "Inside Secure Microread device support (I2C)"
 	depends on NFC_HCI && I2C && NFC_SHDLC
 	select NFC_MICROREAD
-	---help---
+	help
 	  This module adds support for the i2c interface of adapters using
 	  Inside microread chipsets.  Select this if your platform is using
 	  the i2c bus.
@@ -23,7 +23,7 @@ config NFC_MICROREAD_MEI
 	tristate "Inside Secure Microread device support (MEI)"
 	depends on NFC_HCI && NFC_MEI_PHY
 	select NFC_MICROREAD
-	---help---
+	help
 	  This module adds support for the mei interface of adapters using
 	  Inside microread chipsets.  Select this if your microread chipset
 	  is handled by Intel's Management Engine Interface on your platform.
diff --git a/drivers/nfc/nxp-nci/Kconfig b/drivers/nfc/nxp-nci/Kconfig
index e1f71deab6fc..ab6db98b495d 100644
--- a/drivers/nfc/nxp-nci/Kconfig
+++ b/drivers/nfc/nxp-nci/Kconfig
@@ -2,7 +2,7 @@
 config NFC_NXP_NCI
 	tristate "NXP-NCI NFC driver"
 	depends on NFC_NCI
-	---help---
+	help
 	  Generic core driver for NXP NCI chips such as the NPC100 (PN547),
 	  NPC300 (PN548) or PN7150 families.
 	  This is a driver based on the NCI NFC kernel layers and
@@ -15,7 +15,7 @@ config NFC_NXP_NCI
 config NFC_NXP_NCI_I2C
 	tristate "NXP-NCI I2C support"
 	depends on NFC_NXP_NCI && I2C
-	---help---
+	help
 	  This module adds support for an I2C interface to the NXP NCI
 	  chips.
 	  Select this if your platform is using the I2C bus.
diff --git a/drivers/nfc/pn533/Kconfig b/drivers/nfc/pn533/Kconfig
index 7fe1bbe26568..31822b432d51 100644
--- a/drivers/nfc/pn533/Kconfig
+++ b/drivers/nfc/pn533/Kconfig
@@ -9,7 +9,7 @@ config NFC_PN533_USB
 	tristate "NFC PN533 device support (USB)"
 	depends on USB
 	select NFC_PN533
-	---help---
+	help
 	  This module adds support for the NXP pn533 USB interface.
 	  Select this if your platform is using the USB bus.
 
@@ -20,7 +20,7 @@ config NFC_PN533_I2C
 	tristate "NFC PN533 device support (I2C)"
 	depends on I2C
 	select NFC_PN533
-	---help---
+	help
 	  This module adds support for the NXP pn533 I2C interface.
 	  Select this if your platform is using the I2C bus.
 
@@ -31,7 +31,7 @@ config NFC_PN532_UART
 	tristate "NFC PN532 device support (UART)"
 	depends on SERIAL_DEV_BUS
 	select NFC_PN533
-	---help---
+	help
 	  This module adds support for the NXP pn532 UART interface.
 	  Select this if your platform is using the UART bus.
 
diff --git a/drivers/nfc/pn544/Kconfig b/drivers/nfc/pn544/Kconfig
index 47ef5e3f25ff..0fdc3108c528 100644
--- a/drivers/nfc/pn544/Kconfig
+++ b/drivers/nfc/pn544/Kconfig
@@ -2,7 +2,7 @@
 config NFC_PN544
 	tristate
 	select CRC_CCITT
-	---help---
+	help
 	  NXP PN544 core driver.
 	  This is a driver based on the HCI NFC kernel layers and
 	  will thus not work with NXP libnfc library.
@@ -11,7 +11,7 @@ config NFC_PN544_I2C
 	tristate "NXP PN544 device support (I2C)"
 	depends on NFC_HCI && I2C && NFC_SHDLC
 	select NFC_PN544
-	---help---
+	help
 	  This module adds support for the NXP pn544 i2c interface.
 	  Select this if your platform is using the i2c bus.
 
@@ -22,7 +22,7 @@ config NFC_PN544_MEI
 	tristate "NXP PN544 device support (MEI)"
 	depends on NFC_HCI && NFC_MEI_PHY
 	select NFC_PN544
-	---help---
+	help
 	  This module adds support for the mei interface of adapters using
 	  NXP pn544 chipsets.  Select this if your pn544 chipset
 	  is handled by Intel's Management Engine Interface on your platform.
diff --git a/drivers/nfc/s3fwrn5/Kconfig b/drivers/nfc/s3fwrn5/Kconfig
index c4e86df392a4..af9d18690afe 100644
--- a/drivers/nfc/s3fwrn5/Kconfig
+++ b/drivers/nfc/s3fwrn5/Kconfig
@@ -2,7 +2,7 @@
 config NFC_S3FWRN5
 	tristate
 	select CRYPTO
-	---help---
+	help
 	  Core driver for Samsung S3FWRN5 NFC chip. Contains core utilities
 	  of chip. It's intended to be used by PHYs to avoid duplicating lots
 	  of common code.
@@ -12,7 +12,7 @@ config NFC_S3FWRN5_I2C
 	depends on NFC_NCI && I2C
 	select NFC_S3FWRN5
 	default n
-	---help---
+	help
 	  This module adds support for an I2C interface to the S3FWRN5 chip.
 	  Select this if your platform is using the I2C bus.
 
diff --git a/drivers/nfc/st-nci/Kconfig b/drivers/nfc/st-nci/Kconfig
index 8fe53da46280..0e6909206999 100644
--- a/drivers/nfc/st-nci/Kconfig
+++ b/drivers/nfc/st-nci/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config NFC_ST_NCI
 	tristate
-	---help---
+	help
 	  STMicroelectronics NFC NCI chips core driver. It implements the chipset
 	  NCI logic and hooks into the NFC kernel APIs. Physical layers will
 	  register against it.
@@ -10,7 +10,7 @@ config NFC_ST_NCI_I2C
 	tristate "STMicroelectronics ST NCI NFC driver (I2C)"
 	depends on NFC_NCI && I2C
 	select NFC_ST_NCI
-	---help---
+	help
 	  This module adds support for an I2C interface to the
 	  STMicroelectronics NFC NCI chips family.
 	  Select this if your platform is using the i2c bus.
@@ -22,7 +22,7 @@ config NFC_ST_NCI_SPI
 	tristate "STMicroelectronics ST NCI NFC driver (SPI)"
 	depends on NFC_NCI && SPI
 	select NFC_ST_NCI
-	---help---
+	help
 	  This module adds support for an SPI interface to the
 	  STMicroelectronics NFC NCI chips family.
 	  Select this if your platform is using the spi bus.
diff --git a/drivers/nfc/st21nfca/Kconfig b/drivers/nfc/st21nfca/Kconfig
index ab20724afe17..3407e9cfa140 100644
--- a/drivers/nfc/st21nfca/Kconfig
+++ b/drivers/nfc/st21nfca/Kconfig
@@ -2,7 +2,7 @@
 config NFC_ST21NFCA
 	tristate
 	select CRC_CCITT
-	---help---
+	help
 	  STMicroelectronics ST21NFCA core driver. It implements the chipset
 	  HCI logic and hooks into the NFC kernel APIs. Physical layers will
 	  register against it.
@@ -11,7 +11,7 @@ config NFC_ST21NFCA_I2C
 	tristate "STMicroelectronics ST21NFCA NFC driver (I2C)"
 	depends on NFC_HCI && I2C && NFC_SHDLC
 	select NFC_ST21NFCA
-	---help---
+	help
 	  This module adds support for the STMicroelectronics st21nfca i2c interface.
 	  Select this if your platform is using the i2c bus.
 
diff --git a/drivers/ntb/core.c b/drivers/ntb/core.c
index 2581ab724c34..f8f75a504a58 100644
--- a/drivers/ntb/core.c
+++ b/drivers/ntb/core.c
@@ -214,10 +214,8 @@ int ntb_default_port_number(struct ntb_dev *ntb)
 	case NTB_TOPO_B2B_DSD:
 		return NTB_PORT_SEC_DSD;
 	default:
-		break;
+		return 0;
 	}
-
-	return -EINVAL;
 }
 EXPORT_SYMBOL(ntb_default_port_number);
 
@@ -240,10 +238,8 @@ int ntb_default_peer_port_number(struct ntb_dev *ntb, int pidx)
 	case NTB_TOPO_B2B_DSD:
 		return NTB_PORT_PRI_USD;
 	default:
-		break;
+		return 0;
 	}
-
-	return -EINVAL;
 }
 EXPORT_SYMBOL(ntb_default_peer_port_number);
 
@@ -315,4 +311,3 @@ static void __exit ntb_driver_exit(void)
 	bus_unregister(&ntb_bus);
 }
 module_exit(ntb_driver_exit);
-
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 9e310e1ad4d0..88e1db65be02 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -1191,10 +1191,6 @@ static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
 			goto err_dma_mask;
 		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
 	}
-	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
-					  dma_get_mask(&pdev->dev));
-	if (rc)
-		goto err_dma_mask;
 
 	ndev->self_mmio = pci_iomap(pdev, 0, 0);
 	if (!ndev->self_mmio) {
diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c
index edae52384b8a..d54261f50851 100644
--- a/drivers/ntb/hw/idt/ntb_hw_idt.c
+++ b/drivers/ntb/hw/idt/ntb_hw_idt.c
@@ -2660,12 +2660,6 @@ static int idt_init_pci(struct idt_ntb_dev *ndev)
 		dev_warn(&pdev->dev,
 			"Cannot set consistent DMA highmem bit mask\n");
 	}
-	ret = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
-					   dma_get_mask(&pdev->dev));
-	if (ret != 0) {
-		dev_err(&pdev->dev, "Failed to set NTB device DMA bit mask\n");
-		return ret;
-	}
 
 	/*
 	 * Enable the device advanced error reporting. It's not critical to
diff --git a/drivers/ntb/hw/intel/Makefile b/drivers/ntb/hw/intel/Makefile
index 60ec8a773eea..f80da0ba15b2 100644
--- a/drivers/ntb/hw/intel/Makefile
+++ b/drivers/ntb/hw/intel/Makefile
@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-$(CONFIG_NTB_INTEL) += ntb_hw_intel.o
-ntb_hw_intel-y := ntb_hw_gen1.o ntb_hw_gen3.o
+ntb_hw_intel-y := ntb_hw_gen1.o ntb_hw_gen3.o ntb_hw_gen4.o
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.c b/drivers/ntb/hw/intel/ntb_hw_gen1.c
index bb57ec239029..423f9b8fbbcf 100644
--- a/drivers/ntb/hw/intel/ntb_hw_gen1.c
+++ b/drivers/ntb/hw/intel/ntb_hw_gen1.c
@@ -60,6 +60,7 @@
 #include "ntb_hw_intel.h"
 #include "ntb_hw_gen1.h"
 #include "ntb_hw_gen3.h"
+#include "ntb_hw_gen4.h"
 
 #define NTB_NAME	"ntb_hw_intel"
 #define NTB_DESC	"Intel(R) PCI-E Non-Transparent Bridge Driver"
@@ -762,6 +763,8 @@ static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 		return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
 	else if (pdev_is_gen3(ndev->ntb.pdev))
 		return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
+	else if (pdev_is_gen4(ndev->ntb.pdev))
+		return ndev_ntb4_debugfs_read(filp, ubuf, count, offp);
 
 	return -ENXIO;
 }
@@ -1783,10 +1786,6 @@ static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
 			goto err_dma_mask;
 		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
 	}
-	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
-					  dma_get_mask(&pdev->dev));
-	if (rc)
-		goto err_dma_mask;
 
 	ndev->self_mmio = pci_iomap(pdev, 0, 0);
 	if (!ndev->self_mmio) {
@@ -1858,16 +1857,15 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
 	int rc, node;
 
 	node = dev_to_node(&pdev->dev);
+	ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
+	if (!ndev) {
+		rc = -ENOMEM;
+		goto err_ndev;
+	}
 
-	if (pdev_is_gen1(pdev)) {
-		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
-		if (!ndev) {
-			rc = -ENOMEM;
-			goto err_ndev;
-		}
-
-		ndev_init_struct(ndev, pdev);
+	ndev_init_struct(ndev, pdev);
 
+	if (pdev_is_gen1(pdev)) {
 		rc = intel_ntb_init_pci(ndev, pdev);
 		if (rc)
 			goto err_init_pci;
@@ -1875,17 +1873,8 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
 		rc = xeon_init_dev(ndev);
 		if (rc)
 			goto err_init_dev;
-
 	} else if (pdev_is_gen3(pdev)) {
-		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
-		if (!ndev) {
-			rc = -ENOMEM;
-			goto err_ndev;
-		}
-
-		ndev_init_struct(ndev, pdev);
 		ndev->ntb.ops = &intel_ntb3_ops;
-
 		rc = intel_ntb_init_pci(ndev, pdev);
 		if (rc)
 			goto err_init_pci;
@@ -1893,7 +1882,15 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
 		rc = gen3_init_dev(ndev);
 		if (rc)
 			goto err_init_dev;
+	} else if (pdev_is_gen4(pdev)) {
+		ndev->ntb.ops = &intel_ntb4_ops;
+		rc = intel_ntb_init_pci(ndev, pdev);
+		if (rc)
+			goto err_init_pci;
 
+		rc = gen4_init_dev(ndev);
+		if (rc)
+			goto err_init_dev;
 	} else {
 		rc = -EINVAL;
 		goto err_ndev;
@@ -1915,7 +1912,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
 
 err_register:
 	ndev_deinit_debugfs(ndev);
-	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
+	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) || pdev_is_gen4(pdev))
 		xeon_deinit_dev(ndev);
 err_init_dev:
 	intel_ntb_deinit_pci(ndev);
@@ -1931,7 +1928,7 @@ static void intel_ntb_pci_remove(struct pci_dev *pdev)
 
 	ntb_unregister_device(&ndev->ntb);
 	ndev_deinit_debugfs(ndev);
-	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
+	if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) || pdev_is_gen4(pdev))
 		xeon_deinit_dev(ndev);
 	intel_ntb_deinit_pci(ndev);
 	kfree(ndev);
@@ -2036,6 +2033,7 @@ static const struct file_operations intel_ntb_debugfs_info = {
 };
 
 static const struct pci_device_id intel_ntb_pci_tbl[] = {
+	/* GEN1 */
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
@@ -2051,7 +2049,12 @@ static const struct pci_device_id intel_ntb_pci_tbl[] = {
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
+
+	/* GEN3 */
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
+
+	/* GEN4 */
+	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_ICX)},
 	{0}
 };
 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.h b/drivers/ntb/hw/intel/ntb_hw_gen1.h
index 544cf5c06f4d..1b759942d8af 100644
--- a/drivers/ntb/hw/intel/ntb_hw_gen1.h
+++ b/drivers/ntb/hw/intel/ntb_hw_gen1.h
@@ -140,6 +140,7 @@
 #define NTB_HWERR_SB01BASE_LOCKUP	BIT_ULL(1)
 #define NTB_HWERR_B2BDOORBELL_BIT14	BIT_ULL(2)
 #define NTB_HWERR_MSIX_VECTOR32_BAD	BIT_ULL(3)
+#define NTB_HWERR_BAR_ALIGN		BIT_ULL(4)
 
 extern struct intel_b2b_addr xeon_b2b_usd_addr;
 extern struct intel_b2b_addr xeon_b2b_dsd_addr;
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.c b/drivers/ntb/hw/intel/ntb_hw_gen3.c
index c3397160db7f..ffcfc3e02c35 100644
--- a/drivers/ntb/hw/intel/ntb_hw_gen3.c
+++ b/drivers/ntb/hw/intel/ntb_hw_gen3.c
@@ -415,9 +415,8 @@ ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
 	return ret;
 }
 
-static int intel_ntb3_link_enable(struct ntb_dev *ntb,
-				  enum ntb_speed max_speed,
-				  enum ntb_width max_width)
+int intel_ntb3_link_enable(struct ntb_dev *ntb, enum ntb_speed max_speed,
+		enum ntb_width max_width)
 {
 	struct intel_ntb_dev *ndev;
 	u32 ntb_ctl;
@@ -532,7 +531,7 @@ static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
 	return 0;
 }
 
-static int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
+int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
 				   resource_size_t *db_size,
 				   u64 *db_data, int db_bit)
 {
@@ -563,7 +562,7 @@ static int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
 	return 0;
 }
 
-static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
+int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 {
 	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 	int bit;
@@ -581,7 +580,7 @@ static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 	return 0;
 }
 
-static u64 intel_ntb3_db_read(struct ntb_dev *ntb)
+u64 intel_ntb3_db_read(struct ntb_dev *ntb)
 {
 	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
@@ -590,7 +589,7 @@ static u64 intel_ntb3_db_read(struct ntb_dev *ntb)
 			    ndev->self_reg->db_clear);
 }
 
-static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits)
+int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits)
 {
 	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.h b/drivers/ntb/hw/intel/ntb_hw_gen3.h
index 75fb86ca27bb..2bc5d8356045 100644
--- a/drivers/ntb/hw/intel/ntb_hw_gen3.h
+++ b/drivers/ntb/hw/intel/ntb_hw_gen3.h
@@ -104,6 +104,14 @@ static inline void gen3_db_iowrite(u64 bits, void __iomem *mmio)
 ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
 				      size_t count, loff_t *offp);
 int gen3_init_dev(struct intel_ntb_dev *ndev);
+int intel_ntb3_link_enable(struct ntb_dev *ntb, enum ntb_speed max_speed,
+		enum ntb_width max_width);
+u64 intel_ntb3_db_read(struct ntb_dev *ntb);
+int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits);
+int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits);
+int intel_ntb3_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
+				resource_size_t *db_size,
+				u64 *db_data, int db_bit);
 
 extern const struct ntb_dev_ops intel_ntb3_ops;
 
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen4.c b/drivers/ntb/hw/intel/ntb_hw_gen4.c
new file mode 100644
index 000000000000..bc4541cbf8c6
--- /dev/null
+++ b/drivers/ntb/hw/intel/ntb_hw_gen4.c
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/ntb.h>
+#include <linux/log2.h>
+
+#include "ntb_hw_intel.h"
+#include "ntb_hw_gen1.h"
+#include "ntb_hw_gen3.h"
+#include "ntb_hw_gen4.h"
+
+static int gen4_poll_link(struct intel_ntb_dev *ndev);
+static int gen4_link_is_up(struct intel_ntb_dev *ndev);
+
+static const struct intel_ntb_reg gen4_reg = {
+	.poll_link		= gen4_poll_link,
+	.link_is_up		= gen4_link_is_up,
+	.db_ioread		= gen3_db_ioread,
+	.db_iowrite		= gen3_db_iowrite,
+	.db_size		= sizeof(u32),
+	.ntb_ctl		= GEN4_NTBCNTL_OFFSET,
+	.mw_bar			= {2, 4},
+};
+
+static const struct intel_ntb_alt_reg gen4_pri_reg = {
+	.db_clear		= GEN4_IM_INT_STATUS_OFFSET,
+	.db_mask		= GEN4_IM_INT_DISABLE_OFFSET,
+	.spad			= GEN4_IM_SPAD_OFFSET,
+};
+
+static const struct intel_ntb_xlat_reg gen4_sec_xlat = {
+	.bar2_limit		= GEN4_IM23XLMT_OFFSET,
+	.bar2_xlat		= GEN4_IM23XBASE_OFFSET,
+	.bar2_idx		= GEN4_IM23XBASEIDX_OFFSET,
+};
+
+static const struct intel_ntb_alt_reg gen4_b2b_reg = {
+	.db_bell		= GEN4_IM_DOORBELL_OFFSET,
+	.spad			= GEN4_EM_SPAD_OFFSET,
+};
+
+static int gen4_poll_link(struct intel_ntb_dev *ndev)
+{
+	u16 reg_val;
+
+	/*
+	 * We need to write to DLLSCS bit in the SLOTSTS before we
+	 * can clear the hardware link interrupt on ICX NTB.
+	 */
+	iowrite16(GEN4_SLOTSTS_DLLSCS, ndev->self_mmio + GEN4_SLOTSTS);
+	ndev->reg->db_iowrite(ndev->db_link_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_clear);
+
+	reg_val = ioread16(ndev->self_mmio + GEN4_LINK_STATUS_OFFSET);
+	if (reg_val == ndev->lnk_sta)
+		return 0;
+
+	ndev->lnk_sta = reg_val;
+
+	return 1;
+}
+
+static int gen4_link_is_up(struct intel_ntb_dev *ndev)
+{
+	return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
+}
+
+static int gen4_init_isr(struct intel_ntb_dev *ndev)
+{
+	int i;
+
+	/*
+	 * The MSIX vectors and the interrupt status bits are not lined up
+	 * on Gen3 (Skylake) and Gen4. By default the link status bit is bit
+	 * 32, however it is by default MSIX vector0. We need to fixup to
+	 * line them up. The vectors at reset is 1-32,0. We need to reprogram
+	 * to 0-32.
+	 */
+	for (i = 0; i < GEN4_DB_MSIX_VECTOR_COUNT; i++)
+		iowrite8(i, ndev->self_mmio + GEN4_INTVEC_OFFSET + i);
+
+	return ndev_init_isr(ndev, GEN4_DB_MSIX_VECTOR_COUNT,
+			     GEN4_DB_MSIX_VECTOR_COUNT,
+			     GEN4_DB_MSIX_VECTOR_SHIFT,
+			     GEN4_DB_TOTAL_SHIFT);
+}
+
+static int gen4_setup_b2b_mw(struct intel_ntb_dev *ndev,
+			    const struct intel_b2b_addr *addr,
+			    const struct intel_b2b_addr *peer_addr)
+{
+	struct pci_dev *pdev;
+	void __iomem *mmio;
+	phys_addr_t bar_addr;
+
+	pdev = ndev->ntb.pdev;
+	mmio = ndev->self_mmio;
+
+	/* setup incoming bar limits == base addrs (zero length windows) */
+	bar_addr = addr->bar2_addr64;
+	iowrite64(bar_addr, mmio + GEN4_IM23XLMT_OFFSET);
+	bar_addr = ioread64(mmio + GEN4_IM23XLMT_OFFSET);
+	dev_dbg(&pdev->dev, "IM23XLMT %#018llx\n", bar_addr);
+
+	bar_addr = addr->bar4_addr64;
+	iowrite64(bar_addr, mmio + GEN4_IM45XLMT_OFFSET);
+	bar_addr = ioread64(mmio + GEN4_IM45XLMT_OFFSET);
+	dev_dbg(&pdev->dev, "IM45XLMT %#018llx\n", bar_addr);
+
+	/* zero incoming translation addrs */
+	iowrite64(0, mmio + GEN4_IM23XBASE_OFFSET);
+	iowrite64(0, mmio + GEN4_IM45XBASE_OFFSET);
+
+	ndev->peer_mmio = ndev->self_mmio;
+
+	return 0;
+}
+
+static int gen4_init_ntb(struct intel_ntb_dev *ndev)
+{
+	int rc;
+
+
+	ndev->mw_count = XEON_MW_COUNT;
+	ndev->spad_count = GEN4_SPAD_COUNT;
+	ndev->db_count = GEN4_DB_COUNT;
+	ndev->db_link_mask = GEN4_DB_LINK_BIT;
+
+	ndev->self_reg = &gen4_pri_reg;
+	ndev->xlat_reg = &gen4_sec_xlat;
+	ndev->peer_reg = &gen4_b2b_reg;
+
+	if (ndev->ntb.topo == NTB_TOPO_B2B_USD)
+		rc = gen4_setup_b2b_mw(ndev, &xeon_b2b_dsd_addr,
+				&xeon_b2b_usd_addr);
+	else
+		rc = gen4_setup_b2b_mw(ndev, &xeon_b2b_usd_addr,
+				&xeon_b2b_dsd_addr);
+	if (rc)
+		return rc;
+
+	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+
+	ndev->reg->db_iowrite(ndev->db_valid_mask,
+			      ndev->self_mmio +
+			      ndev->self_reg->db_mask);
+
+	return 0;
+}
+
+static enum ntb_topo gen4_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
+{
+	switch (ppd & GEN4_PPD_TOPO_MASK) {
+	case GEN4_PPD_TOPO_B2B_USD:
+		return NTB_TOPO_B2B_USD;
+	case GEN4_PPD_TOPO_B2B_DSD:
+		return NTB_TOPO_B2B_DSD;
+	}
+
+	return NTB_TOPO_NONE;
+}
+
+int gen4_init_dev(struct intel_ntb_dev *ndev)
+{
+	struct pci_dev *pdev = ndev->ntb.pdev;
+	u32 ppd1/*, ppd0*/;
+	u16 lnkctl;
+	int rc;
+
+	ndev->reg = &gen4_reg;
+
+	if (pdev_is_ICX(pdev))
+		ndev->hwerr_flags |= NTB_HWERR_BAR_ALIGN;
+
+	ppd1 = ioread32(ndev->self_mmio + GEN4_PPD1_OFFSET);
+	ndev->ntb.topo = gen4_ppd_topo(ndev, ppd1);
+	dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd1,
+		ntb_topo_string(ndev->ntb.topo));
+	if (ndev->ntb.topo == NTB_TOPO_NONE)
+		return -EINVAL;
+
+	rc = gen4_init_ntb(ndev);
+	if (rc)
+		return rc;
+
+	/* init link setup */
+	lnkctl = ioread16(ndev->self_mmio + GEN4_LINK_CTRL_OFFSET);
+	lnkctl |= GEN4_LINK_CTRL_LINK_DISABLE;
+	iowrite16(lnkctl, ndev->self_mmio + GEN4_LINK_CTRL_OFFSET);
+
+	return gen4_init_isr(ndev);
+}
+
+ssize_t ndev_ntb4_debugfs_read(struct file *filp, char __user *ubuf,
+				      size_t count, loff_t *offp)
+{
+	struct intel_ntb_dev *ndev;
+	void __iomem *mmio;
+	char *buf;
+	size_t buf_size;
+	ssize_t ret, off;
+	union { u64 v64; u32 v32; u16 v16; } u;
+
+	ndev = filp->private_data;
+	mmio = ndev->self_mmio;
+
+	buf_size = min(count, 0x800ul);
+
+	buf = kmalloc(buf_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	off = 0;
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "NTB Device Information:\n");
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Connection Topology -\t%s\n",
+			 ntb_topo_string(ndev->ntb.topo));
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
+	off += scnprintf(buf + off, buf_size - off,
+			 "LNK STA (cached) -\t\t%#06x\n", ndev->lnk_sta);
+
+	if (!ndev->reg->link_is_up(ndev))
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tDown\n");
+	else {
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Status -\t\tUp\n");
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Speed -\t\tPCI-E Gen %u\n",
+				 NTB_LNK_STA_SPEED(ndev->lnk_sta));
+		off += scnprintf(buf + off, buf_size - off,
+				 "Link Width -\t\tx%u\n",
+				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
+	}
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Memory Window Count -\t%u\n", ndev->mw_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Scratchpad Count -\t%u\n", ndev->spad_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Count -\t%u\n", ndev->db_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
+
+	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Doorbell Mask -\t\t%#llx\n", u.v64);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Incoming XLAT:\n");
+
+	u.v64 = ioread64(mmio + GEN4_IM23XBASE_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IM23XBASE -\t\t%#018llx\n", u.v64);
+
+	u.v64 = ioread64(mmio + GEN4_IM45XBASE_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IM45XBASE -\t\t%#018llx\n", u.v64);
+
+	u.v64 = ioread64(mmio + GEN4_IM23XLMT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IM23XLMT -\t\t\t%#018llx\n", u.v64);
+
+	u.v64 = ioread64(mmio + GEN4_IM45XLMT_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			 "IM45XLMT -\t\t\t%#018llx\n", u.v64);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Statistics:\n");
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "\nNTB Hardware Errors:\n");
+
+	if (!pci_read_config_word(ndev->ntb.pdev,
+				  GEN4_DEVSTS_OFFSET, &u.v16))
+		off += scnprintf(buf + off, buf_size - off,
+				"DEVSTS -\t\t%#06x\n", u.v16);
+
+	u.v16 = ioread16(mmio + GEN4_LINK_STATUS_OFFSET);
+	off += scnprintf(buf + off, buf_size - off,
+			"LNKSTS -\t\t%#06x\n", u.v16);
+
+	if (!pci_read_config_dword(ndev->ntb.pdev,
+				   GEN4_UNCERRSTS_OFFSET, &u.v32))
+		off += scnprintf(buf + off, buf_size - off,
+				 "UNCERRSTS -\t\t%#06x\n", u.v32);
+
+	if (!pci_read_config_dword(ndev->ntb.pdev,
+				   GEN4_CORERRSTS_OFFSET, &u.v32))
+		off += scnprintf(buf + off, buf_size - off,
+				 "CORERRSTS -\t\t%#06x\n", u.v32);
+
+	ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
+	kfree(buf);
+	return ret;
+}
+
+static int intel_ntb4_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
+				   dma_addr_t addr, resource_size_t size)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+	unsigned long xlat_reg, limit_reg, idx_reg;
+	unsigned short base_idx, reg_val16;
+	resource_size_t bar_size, mw_size;
+	void __iomem *mmio;
+	u64 base, limit, reg_val;
+	int bar;
+
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+		idx += 1;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
+
+	if (idx == ndev->b2b_idx)
+		mw_size = bar_size - ndev->b2b_off;
+	else
+		mw_size = bar_size;
+
+	if (ndev->hwerr_flags & NTB_HWERR_BAR_ALIGN) {
+		/* hardware requires that addr is aligned to bar size */
+		if (addr & (bar_size - 1))
+			return -EINVAL;
+	} else {
+		if (addr & (PAGE_SIZE - 1))
+			return -EINVAL;
+	}
+
+	/* make sure the range fits in the usable mw size */
+	if (size > mw_size)
+		return -EINVAL;
+
+	mmio = ndev->self_mmio;
+	xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10);
+	limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10);
+	base = pci_resource_start(ndev->ntb.pdev, bar);
+
+	/* Set the limit if supported, if size is not mw_size */
+	if (limit_reg && size != mw_size) {
+		limit = base + size;
+		base_idx = __ilog2_u64(size);
+	} else {
+		limit = base + mw_size;
+		base_idx = __ilog2_u64(mw_size);
+	}
+
+
+	/* set and verify setting the translation address */
+	iowrite64(addr, mmio + xlat_reg);
+	reg_val = ioread64(mmio + xlat_reg);
+	if (reg_val != addr) {
+		iowrite64(0, mmio + xlat_reg);
+		return -EIO;
+	}
+
+	dev_dbg(&ntb->pdev->dev, "BAR %d IMXBASE: %#Lx\n", bar, reg_val);
+
+	/* set and verify setting the limit */
+	iowrite64(limit, mmio + limit_reg);
+	reg_val = ioread64(mmio + limit_reg);
+	if (reg_val != limit) {
+		iowrite64(base, mmio + limit_reg);
+		iowrite64(0, mmio + xlat_reg);
+		return -EIO;
+	}
+
+	dev_dbg(&ntb->pdev->dev, "BAR %d IMXLMT: %#Lx\n", bar, reg_val);
+
+	if (ndev->hwerr_flags & NTB_HWERR_BAR_ALIGN) {
+		idx_reg = ndev->xlat_reg->bar2_idx + (idx * 0x2);
+		iowrite16(base_idx, mmio + idx_reg);
+		reg_val16 = ioread16(mmio + idx_reg);
+		if (reg_val16 != base_idx) {
+			iowrite64(base, mmio + limit_reg);
+			iowrite64(0, mmio + xlat_reg);
+			iowrite16(0, mmio + idx_reg);
+			return -EIO;
+		}
+		dev_dbg(&ntb->pdev->dev, "BAR %d IMBASEIDX: %#x\n", bar, reg_val16);
+	}
+
+
+	return 0;
+}
+
+static int intel_ntb4_link_enable(struct ntb_dev *ntb,
+		enum ntb_speed max_speed, enum ntb_width max_width)
+{
+	struct intel_ntb_dev *ndev;
+	u32 ntb_ctl, ppd0;
+	u16 lnkctl;
+
+	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+	dev_dbg(&ntb->pdev->dev,
+			"Enabling link with max_speed %d max_width %d\n",
+			max_speed, max_width);
+
+	if (max_speed != NTB_SPEED_AUTO)
+		dev_dbg(&ntb->pdev->dev,
+				"ignoring max_speed %d\n", max_speed);
+	if (max_width != NTB_WIDTH_AUTO)
+		dev_dbg(&ntb->pdev->dev,
+				"ignoring max_width %d\n", max_width);
+
+	ntb_ctl = NTB_CTL_E2I_BAR23_SNOOP | NTB_CTL_I2E_BAR23_SNOOP;
+	ntb_ctl |= NTB_CTL_E2I_BAR45_SNOOP | NTB_CTL_I2E_BAR45_SNOOP;
+	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+	lnkctl = ioread16(ndev->self_mmio + GEN4_LINK_CTRL_OFFSET);
+	lnkctl &= ~GEN4_LINK_CTRL_LINK_DISABLE;
+	iowrite16(lnkctl, ndev->self_mmio + GEN4_LINK_CTRL_OFFSET);
+
+	/* start link training in PPD0 */
+	ppd0 = ioread32(ndev->self_mmio + GEN4_PPD0_OFFSET);
+	ppd0 |= GEN4_PPD_LINKTRN;
+	iowrite32(ppd0, ndev->self_mmio + GEN4_PPD0_OFFSET);
+
+	/* make sure link training has started */
+	ppd0 = ioread32(ndev->self_mmio + GEN4_PPD0_OFFSET);
+	if (!(ppd0 & GEN4_PPD_LINKTRN)) {
+		dev_warn(&ntb->pdev->dev, "Link is not training\n");
+		return -ENXIO;
+	}
+
+	ndev->dev_up = 1;
+
+	return 0;
+}
+
+static int intel_ntb4_link_disable(struct ntb_dev *ntb)
+{
+	struct intel_ntb_dev *ndev;
+	u32 ntb_cntl;
+	u16 lnkctl;
+
+	ndev = container_of(ntb, struct intel_ntb_dev, ntb);
+
+	dev_dbg(&ntb->pdev->dev, "Disabling link\n");
+
+	/* clear the snoop bits */
+	ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+	ntb_cntl &= ~(NTB_CTL_E2I_BAR23_SNOOP | NTB_CTL_I2E_BAR23_SNOOP);
+	ntb_cntl &= ~(NTB_CTL_E2I_BAR45_SNOOP | NTB_CTL_I2E_BAR45_SNOOP);
+	iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
+
+	lnkctl = ioread16(ndev->self_mmio + GEN4_LINK_CTRL_OFFSET);
+	lnkctl |= GEN4_LINK_CTRL_LINK_DISABLE;
+	iowrite16(lnkctl, ndev->self_mmio + GEN4_LINK_CTRL_OFFSET);
+
+	ndev->dev_up = 0;
+
+	return 0;
+}
+
+static int intel_ntb4_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
+				   resource_size_t *addr_align,
+				   resource_size_t *size_align,
+				   resource_size_t *size_max)
+{
+	struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+	resource_size_t bar_size, mw_size;
+	int bar;
+
+	if (pidx != NTB_DEF_PEER_IDX)
+		return -EINVAL;
+
+	if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+		idx += 1;
+
+	bar = ndev_mw_to_bar(ndev, idx);
+	if (bar < 0)
+		return bar;
+
+	bar_size = pci_resource_len(ndev->ntb.pdev, bar);
+
+	if (idx == ndev->b2b_idx)
+		mw_size = bar_size - ndev->b2b_off;
+	else
+		mw_size = bar_size;
+
+	if (addr_align) {
+		if (ndev->hwerr_flags & NTB_HWERR_BAR_ALIGN)
+			*addr_align = pci_resource_len(ndev->ntb.pdev, bar);
+		else
+			*addr_align = PAGE_SIZE;
+	}
+
+	if (size_align)
+		*size_align = 1;
+
+	if (size_max)
+		*size_max = mw_size;
+
+	return 0;
+}
+
+const struct ntb_dev_ops intel_ntb4_ops = {
+	.mw_count		= intel_ntb_mw_count,
+	.mw_get_align		= intel_ntb4_mw_get_align,
+	.mw_set_trans		= intel_ntb4_mw_set_trans,
+	.peer_mw_count		= intel_ntb_peer_mw_count,
+	.peer_mw_get_addr	= intel_ntb_peer_mw_get_addr,
+	.link_is_up		= intel_ntb_link_is_up,
+	.link_enable		= intel_ntb4_link_enable,
+	.link_disable		= intel_ntb4_link_disable,
+	.db_valid_mask		= intel_ntb_db_valid_mask,
+	.db_vector_count	= intel_ntb_db_vector_count,
+	.db_vector_mask		= intel_ntb_db_vector_mask,
+	.db_read		= intel_ntb3_db_read,
+	.db_clear		= intel_ntb3_db_clear,
+	.db_set_mask		= intel_ntb_db_set_mask,
+	.db_clear_mask		= intel_ntb_db_clear_mask,
+	.peer_db_addr		= intel_ntb3_peer_db_addr,
+	.peer_db_set		= intel_ntb3_peer_db_set,
+	.spad_is_unsafe		= intel_ntb_spad_is_unsafe,
+	.spad_count		= intel_ntb_spad_count,
+	.spad_read		= intel_ntb_spad_read,
+	.spad_write		= intel_ntb_spad_write,
+	.peer_spad_addr		= intel_ntb_peer_spad_addr,
+	.peer_spad_read		= intel_ntb_peer_spad_read,
+	.peer_spad_write	= intel_ntb_peer_spad_write,
+};
+
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen4.h b/drivers/ntb/hw/intel/ntb_hw_gen4.h
new file mode 100644
index 000000000000..a868c788de02
--- /dev/null
+++ b/drivers/ntb/hw/intel/ntb_hw_gen4.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)          */
+/* Copyright(c) 2020 Intel Corporation. All rights reserved.   */
+#ifndef _NTB_INTEL_GEN4_H_
+#define _NTB_INTEL_GEN4_H_
+
+#include "ntb_hw_intel.h"
+
+/* Supported PCI device revision range for ICX */
+#define PCI_DEVICE_REVISION_ICX_MIN	0x2
+#define PCI_DEVICE_REVISION_ICX_MAX	0xF
+
+/* Intel Gen4 NTB hardware */
+/* PCIe config space */
+#define GEN4_IMBAR23SZ_OFFSET		0x00c4
+#define GEN4_IMBAR45SZ_OFFSET		0x00c5
+#define GEN4_EMBAR23SZ_OFFSET		0x00c6
+#define GEN4_EMBAR45SZ_OFFSET		0x00c7
+#define GEN4_DEVCTRL_OFFSET		0x0048
+#define GEN4_DEVSTS_OFFSET		0x004a
+#define GEN4_UNCERRSTS_OFFSET		0x0104
+#define GEN4_CORERRSTS_OFFSET		0x0110
+
+/* BAR0 MMIO */
+#define GEN4_NTBCNTL_OFFSET		0x0000
+#define GEN4_IM23XBASE_OFFSET		0x0010	/* IMBAR1XBASE */
+#define GEN4_IM23XLMT_OFFSET		0x0018  /* IMBAR1XLMT */
+#define GEN4_IM45XBASE_OFFSET		0x0020	/* IMBAR2XBASE */
+#define GEN4_IM45XLMT_OFFSET		0x0028  /* IMBAR2XLMT */
+#define GEN4_IM_INT_STATUS_OFFSET	0x0040
+#define GEN4_IM_INT_DISABLE_OFFSET	0x0048
+#define GEN4_INTVEC_OFFSET		0x0050  /* 0-32 vecs */
+#define GEN4_IM23XBASEIDX_OFFSET	0x0074
+#define GEN4_IM45XBASEIDX_OFFSET	0x0076
+#define GEN4_IM_SPAD_OFFSET		0x0080  /* 0-15 SPADs */
+#define GEN4_IM_SPAD_SEM_OFFSET		0x00c0	/* SPAD hw semaphore */
+#define GEN4_IM_SPAD_STICKY_OFFSET	0x00c4  /* sticky SPAD */
+#define GEN4_IM_DOORBELL_OFFSET		0x0100  /* 0-31 doorbells */
+#define GEN4_EM_SPAD_OFFSET		0x8080
+/* note, link status is now in MMIO and not config space for NTB */
+#define GEN4_LINK_CTRL_OFFSET		0xb050
+#define GEN4_LINK_STATUS_OFFSET		0xb052
+#define GEN4_PPD0_OFFSET		0xb0d4
+#define GEN4_PPD1_OFFSET		0xb4c0
+#define GEN4_LTSSMSTATEJMP		0xf040
+
+#define GEN4_PPD_CLEAR_TRN		0x0001
+#define GEN4_PPD_LINKTRN		0x0008
+#define GEN4_PPD_CONN_MASK		0x0300
+#define GEN4_PPD_CONN_B2B		0x0200
+#define GEN4_PPD_DEV_MASK		0x1000
+#define GEN4_PPD_DEV_DSD		0x1000
+#define GEN4_PPD_DEV_USD		0x0000
+#define GEN4_LINK_CTRL_LINK_DISABLE	0x0010
+
+#define GEN4_SLOTSTS			0xb05a
+#define GEN4_SLOTSTS_DLLSCS		0x100
+
+#define GEN4_PPD_TOPO_MASK	(GEN4_PPD_CONN_MASK | GEN4_PPD_DEV_MASK)
+#define GEN4_PPD_TOPO_B2B_USD	(GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_USD)
+#define GEN4_PPD_TOPO_B2B_DSD	(GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_DSD)
+
+#define GEN4_DB_COUNT			32
+#define GEN4_DB_LINK			32
+#define GEN4_DB_LINK_BIT		BIT_ULL(GEN4_DB_LINK)
+#define GEN4_DB_MSIX_VECTOR_COUNT	33
+#define GEN4_DB_MSIX_VECTOR_SHIFT	1
+#define GEN4_DB_TOTAL_SHIFT		33
+#define GEN4_SPAD_COUNT			16
+
+#define NTB_CTL_E2I_BAR23_SNOOP		0x000004
+#define NTB_CTL_E2I_BAR23_NOSNOOP	0x000008
+#define NTB_CTL_I2E_BAR23_SNOOP		0x000010
+#define NTB_CTL_I2E_BAR23_NOSNOOP	0x000020
+#define NTB_CTL_E2I_BAR45_SNOOP		0x000040
+#define NTB_CTL_E2I_BAR45_NOSNOO	0x000080
+#define NTB_CTL_I2E_BAR45_SNOOP		0x000100
+#define NTB_CTL_I2E_BAR45_NOSNOOP	0x000200
+#define NTB_CTL_BUSNO_DIS_INC		0x000400
+#define NTB_CTL_LINK_DOWN		0x010000
+
+#define NTB_SJC_FORCEDETECT		0x000004
+
+ssize_t ndev_ntb4_debugfs_read(struct file *filp, char __user *ubuf,
+				      size_t count, loff_t *offp);
+int gen4_init_dev(struct intel_ntb_dev *ndev);
+ssize_t ndev_ntb4_debugfs_read(struct file *filp, char __user *ubuf,
+				      size_t count, loff_t *offp);
+
+extern const struct ntb_dev_ops intel_ntb4_ops;
+
+static inline int pdev_is_ICX(struct pci_dev *pdev)
+{
+	if (pdev_is_gen4(pdev) &&
+	    pdev->revision >= PCI_DEVICE_REVISION_ICX_MIN &&
+	    pdev->revision <= PCI_DEVICE_REVISION_ICX_MAX)
+		return 1;
+	return 0;
+}
+
+#endif
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h
index e071e28bca3f..d61fcd91714b 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -72,6 +72,7 @@
 #define PCI_DEVICE_ID_INTEL_NTB_PS_BDX	0x6F0E
 #define PCI_DEVICE_ID_INTEL_NTB_SS_BDX	0x6F0F
 #define PCI_DEVICE_ID_INTEL_NTB_B2B_SKX	0x201C
+#define PCI_DEVICE_ID_INTEL_NTB_B2B_ICX	0x347e
 
 /* Ntb control and link status */
 #define NTB_CTL_CFG_LOCK		BIT(0)
@@ -120,6 +121,7 @@ struct intel_ntb_xlat_reg {
 	unsigned long			bar0_base;
 	unsigned long			bar2_xlat;
 	unsigned long			bar2_limit;
+	unsigned short			bar2_idx;
 };
 
 struct intel_b2b_addr {
@@ -182,6 +184,9 @@ struct intel_ntb_dev {
 
 	struct dentry			*debugfs_dir;
 	struct dentry			*debugfs_info;
+
+	/* gen4 entries */
+	int				dev_up;
 };
 
 #define ntb_ndev(__ntb) container_of(__ntb, struct intel_ntb_dev, ntb)
@@ -219,4 +224,11 @@ static inline int pdev_is_gen3(struct pci_dev *pdev)
 	return 0;
 }
 
+static inline int pdev_is_gen4(struct pci_dev *pdev)
+{
+	if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_ICX)
+		return 1;
+
+	return 0;
+}
 #endif
diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index 972f6d984f6d..89df1350fefd 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -101,8 +101,8 @@ MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
 #define DMA_MDELAY		10
 
 #define MSG_TRIES		1000
-#define MSG_UDELAY_LOW		1000
-#define MSG_UDELAY_HIGH		2000
+#define MSG_UDELAY_LOW		1000000
+#define MSG_UDELAY_HIGH		2000000
 
 #define PERF_BUF_LEN 1024
 
@@ -159,6 +159,8 @@ struct perf_peer {
 	/* NTB connection setup service */
 	struct work_struct	service;
 	unsigned long		sts;
+
+	struct completion init_comp;
 };
 #define to_peer_service(__work) \
 	container_of(__work, struct perf_peer, service)
@@ -547,6 +549,7 @@ static int perf_setup_outbuf(struct perf_peer *peer)
 
 	/* Initialization is finally done */
 	set_bit(PERF_STS_DONE, &peer->sts);
+	complete_all(&peer->init_comp);
 
 	return 0;
 }
@@ -557,7 +560,7 @@ static void perf_free_inbuf(struct perf_peer *peer)
 		return;
 
 	(void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
-	dma_free_coherent(&peer->perf->ntb->dev, peer->inbuf_size,
+	dma_free_coherent(&peer->perf->ntb->pdev->dev, peer->inbuf_size,
 			  peer->inbuf, peer->inbuf_xlat);
 	peer->inbuf = NULL;
 }
@@ -586,8 +589,9 @@ static int perf_setup_inbuf(struct perf_peer *peer)
 
 	perf_free_inbuf(peer);
 
-	peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,
-					 &peer->inbuf_xlat, GFP_KERNEL);
+	peer->inbuf = dma_alloc_coherent(&perf->ntb->pdev->dev,
+					 peer->inbuf_size, &peer->inbuf_xlat,
+					 GFP_KERNEL);
 	if (!peer->inbuf) {
 		dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
 			&peer->inbuf_size);
@@ -637,6 +641,7 @@ static void perf_service_work(struct work_struct *work)
 		perf_setup_outbuf(peer);
 
 	if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {
+		init_completion(&peer->init_comp);
 		clear_bit(PERF_STS_DONE, &peer->sts);
 		if (test_bit(0, &peer->perf->busy_flag) &&
 		    peer == peer->perf->test_peer) {
@@ -653,7 +658,7 @@ static int perf_init_service(struct perf_ctx *perf)
 {
 	u64 mask;
 
-	if (ntb_peer_mw_count(perf->ntb) < perf->pcnt + 1) {
+	if (ntb_peer_mw_count(perf->ntb) < perf->pcnt) {
 		dev_err(&perf->ntb->dev, "Not enough memory windows\n");
 		return -EINVAL;
 	}
@@ -803,7 +808,7 @@ static int perf_copy_chunk(struct perf_thread *pthr,
 	dst_vaddr = dst;
 	dst_dma_addr = peer->dma_dst_addr + (dst_vaddr - vbase);
 
-	unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
+	unmap = dmaengine_get_unmap_data(dma_dev, 1, GFP_NOWAIT);
 	if (!unmap)
 		return -ENOMEM;
 
@@ -816,15 +821,8 @@ static int perf_copy_chunk(struct perf_thread *pthr,
 	}
 	unmap->to_cnt = 1;
 
-	unmap->addr[1] = dst_dma_addr;
-	if (dma_mapping_error(dma_dev, unmap->addr[1])) {
-		ret = -EIO;
-		goto err_free_resource;
-	}
-	unmap->from_cnt = 1;
-
 	do {
-		tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, unmap->addr[1],
+		tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, dst_dma_addr,
 			unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 		if (!tx)
 			msleep(DMA_MDELAY);
@@ -1010,8 +1008,8 @@ static void perf_clear_test(struct perf_thread *pthr)
 				   pthr->perf->test_peer->dma_dst_addr,
 				   pthr->perf->test_peer->outbuf_size,
 				   DMA_FROM_DEVICE, 0);
-	if (pthr->dma_chan)
-		dma_release_channel(pthr->dma_chan);
+
+	dma_release_channel(pthr->dma_chan);
 
 no_dma_notify:
 	atomic_dec(&perf->tsync);
@@ -1083,8 +1081,9 @@ static int perf_submit_test(struct perf_peer *peer)
 	struct perf_thread *pthr;
 	int tidx, ret;
 
-	if (!test_bit(PERF_STS_DONE, &peer->sts))
-		return -ENOLINK;
+	ret = wait_for_completion_interruptible(&peer->init_comp);
+	if (ret < 0)
+		return ret;
 
 	if (test_and_set_bit_lock(0, &perf->busy_flag))
 		return -EBUSY;
@@ -1455,10 +1454,21 @@ static int perf_init_peers(struct perf_ctx *perf)
 			peer->gidx = pidx;
 		}
 		INIT_WORK(&peer->service, perf_service_work);
+		init_completion(&peer->init_comp);
 	}
 	if (perf->gidx == -1)
 		perf->gidx = pidx;
 
+	/*
+	 * Hardware with only two ports may not have unique port
+	 * numbers. In this case, the gidxs should all be zero.
+	 */
+	if (perf->pcnt == 1 &&  ntb_port_number(perf->ntb) == 0 &&
+	    ntb_peer_port_number(perf->ntb, 0) == 0) {
+		perf->gidx = 0;
+		perf->peers[0].gidx = 0;
+	}
+
 	for (pidx = 0; pidx < perf->pcnt; pidx++) {
 		ret = perf_setup_peer_mw(&perf->peers[pidx]);
 		if (ret)
@@ -1554,4 +1564,3 @@ static void __exit perf_exit(void)
 	destroy_workqueue(perf_wq);
 }
 module_exit(perf_exit);
-
diff --git a/drivers/ntb/test/ntb_pingpong.c b/drivers/ntb/test/ntb_pingpong.c
index 04dd46647db3..2164e8492772 100644
--- a/drivers/ntb/test/ntb_pingpong.c
+++ b/drivers/ntb/test/ntb_pingpong.c
@@ -121,15 +121,14 @@ static int pp_find_next_peer(struct pp_ctx *pp)
 	link = ntb_link_is_up(pp->ntb, NULL, NULL);
 
 	/* Find next available peer */
-	if (link & pp->nmask) {
+	if (link & pp->nmask)
 		pidx = __ffs64(link & pp->nmask);
-		out_db = BIT_ULL(pidx + 1);
-	} else if (link & pp->pmask) {
+	else if (link & pp->pmask)
 		pidx = __ffs64(link & pp->pmask);
-		out_db = BIT_ULL(pidx);
-	} else {
+	else
 		return -ENODEV;
-	}
+
+	out_db = BIT_ULL(ntb_peer_port_number(pp->ntb, pidx));
 
 	spin_lock(&pp->lock);
 	pp->out_pidx = pidx;
@@ -303,7 +302,7 @@ static void pp_init_flds(struct pp_ctx *pp)
 			break;
 	}
 
-	pp->in_db = BIT_ULL(pidx);
+	pp->in_db = BIT_ULL(lport);
 	pp->pmask = GENMASK_ULL(pidx, 0) >> 1;
 	pp->nmask = GENMASK_ULL(pcnt - 1, pidx);
 
@@ -432,4 +431,3 @@ static void __exit pp_exit(void)
 	debugfs_remove_recursive(pp_dbgfs_topdir);
 }
 module_exit(pp_exit);
-
diff --git a/drivers/ntb/test/ntb_tool.c b/drivers/ntb/test/ntb_tool.c
index 69da758fe64c..b7bf3f863d79 100644
--- a/drivers/ntb/test/ntb_tool.c
+++ b/drivers/ntb/test/ntb_tool.c
@@ -504,7 +504,7 @@ static ssize_t tool_peer_link_read(struct file *filep, char __user *ubuf,
 	buf[1] = '\n';
 	buf[2] = '\0';
 
-	return simple_read_from_buffer(ubuf, size, offp, buf, 3);
+	return simple_read_from_buffer(ubuf, size, offp, buf, 2);
 }
 
 static TOOL_FOPS_RDWR(tool_peer_link_fops,
@@ -590,7 +590,7 @@ static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
 	inmw->size = min_t(resource_size_t, req_size, size);
 	inmw->size = round_up(inmw->size, addr_align);
 	inmw->size = round_up(inmw->size, size_align);
-	inmw->mm_base = dma_alloc_coherent(&tc->ntb->dev, inmw->size,
+	inmw->mm_base = dma_alloc_coherent(&tc->ntb->pdev->dev, inmw->size,
 					   &inmw->dma_base, GFP_KERNEL);
 	if (!inmw->mm_base)
 		return -ENOMEM;
@@ -612,7 +612,7 @@ static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
 	return 0;
 
 err_free_dma:
-	dma_free_coherent(&tc->ntb->dev, inmw->size, inmw->mm_base,
+	dma_free_coherent(&tc->ntb->pdev->dev, inmw->size, inmw->mm_base,
 			  inmw->dma_base);
 	inmw->mm_base = NULL;
 	inmw->dma_base = 0;
@@ -629,7 +629,7 @@ static void tool_free_mw(struct tool_ctx *tc, int pidx, int widx)
 
 	if (inmw->mm_base != NULL) {
 		ntb_mw_clear_trans(tc->ntb, pidx, widx);
-		dma_free_coherent(&tc->ntb->dev, inmw->size,
+		dma_free_coherent(&tc->ntb->pdev->dev, inmw->size,
 				  inmw->mm_base, inmw->dma_base);
 	}
 
@@ -1690,4 +1690,3 @@ static void __exit tool_exit(void)
 	debugfs_remove_recursive(tool_dbgfs_topdir);
 }
 module_exit(tool_exit);
-
diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c
index 036e23aef9b0..39030a324d7f 100644
--- a/drivers/nvdimm/blk.c
+++ b/drivers/nvdimm/blk.c
@@ -165,7 +165,7 @@ static int nsblk_do_bvec(struct nd_namespace_blk *nsblk,
 static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio)
 {
 	struct bio_integrity_payload *bip;
-	struct nd_namespace_blk *nsblk;
+	struct nd_namespace_blk *nsblk = bio->bi_disk->private_data;
 	struct bvec_iter iter;
 	unsigned long start;
 	struct bio_vec bvec;
@@ -176,7 +176,6 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio)
 		return BLK_QC_T_NONE;
 
 	bip = bio_integrity(bio);
-	nsblk = q->queuedata;
 	rw = bio_data_dir(bio);
 	do_acct = blk_queue_io_stat(bio->bi_disk->queue);
 	if (do_acct)
@@ -260,7 +259,6 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk)
 	blk_queue_max_hw_sectors(q, UINT_MAX);
 	blk_queue_logical_block_size(q, nsblk_sector_size(nsblk));
 	blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
-	q->queuedata = nsblk;
 
 	disk = alloc_disk(0);
 	if (!disk)
@@ -270,6 +268,7 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk)
 	disk->fops		= &nd_blk_fops;
 	disk->queue		= q;
 	disk->flags		= GENHD_FL_EXT_DEVT;
+	disk->private_data	= nsblk;
 	nvdimm_namespace_disk_name(&nsblk->common, disk->disk_name);
 
 	if (devm_add_action_or_reset(dev, nd_blk_release_disk, disk))
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index 90c0c4bbe77b..48e9d169b6f9 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -1442,7 +1442,7 @@ static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip,
 static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio)
 {
 	struct bio_integrity_payload *bip = bio_integrity(bio);
-	struct btt *btt = q->queuedata;
+	struct btt *btt = bio->bi_disk->private_data;
 	struct bvec_iter iter;
 	unsigned long start;
 	struct bio_vec bvec;
@@ -1545,7 +1545,6 @@ static int btt_blk_init(struct btt *btt)
 	blk_queue_logical_block_size(btt->btt_queue, btt->sector_size);
 	blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX);
 	blk_queue_flag_set(QUEUE_FLAG_NONROT, btt->btt_queue);
-	btt->btt_queue->queuedata = btt;
 
 	if (btt_meta_size(btt)) {
 		int rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt));
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index 97f948f8f4e6..d25e66fd942d 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -7,7 +7,6 @@
  * Copyright (c) 2015, Boaz Harrosh <boaz@plexistor.com>.
  */
 
-#include <asm/cacheflush.h>
 #include <linux/blkdev.h>
 #include <linux/hdreg.h>
 #include <linux/init.h>
@@ -25,6 +24,8 @@
 #include <linux/dax.h>
 #include <linux/nd.h>
 #include <linux/backing-dev.h>
+#include <linux/mm.h>
+#include <asm/cacheflush.h>
 #include "pmem.h"
 #include "pfn.h"
 #include "nd.h"
@@ -196,7 +197,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
 	unsigned long start;
 	struct bio_vec bvec;
 	struct bvec_iter iter;
-	struct pmem_device *pmem = q->queuedata;
+	struct pmem_device *pmem = bio->bi_disk->private_data;
 	struct nd_region *nd_region = to_region(pmem);
 
 	if (bio->bi_opf & REQ_PREFLUSH)
@@ -233,7 +234,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
 static int pmem_rw_page(struct block_device *bdev, sector_t sector,
 		       struct page *page, unsigned int op)
 {
-	struct pmem_device *pmem = bdev->bd_queue->queuedata;
+	struct pmem_device *pmem = bdev->bd_disk->private_data;
 	blk_status_t rc;
 
 	if (op_is_write(op))
@@ -466,7 +467,6 @@ static int pmem_attach_disk(struct device *dev,
 	blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
 	if (pmem->pfn_flags & PFN_MAP)
 		blk_queue_flag_set(QUEUE_FLAG_DAX, q);
-	q->queuedata = pmem;
 
 	disk = alloc_disk_node(0, nid);
 	if (!disk)
@@ -476,6 +476,7 @@ static int pmem_attach_disk(struct device *dev,
 	disk->fops		= &pmem_fops;
 	disk->queue		= q;
 	disk->flags		= GENHD_FL_EXT_DEVT;
+	disk->private_data	= pmem;
 	disk->queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO;
 	nvdimm_namespace_disk_name(ndns, disk->disk_name);
 	set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)
diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 9c17ed32be64..3ed9786b88d8 100644
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -7,7 +7,7 @@ config BLK_DEV_NVME
 	tristate "NVM Express block device"
 	depends on PCI && BLOCK
 	select NVME_CORE
-	---help---
+	help
 	  The NVM Express driver is for solid state drives directly
 	  connected to the PCI or PCI Express bus.  If you know you
 	  don't have one of these, it is safe to answer N.
@@ -18,7 +18,7 @@ config BLK_DEV_NVME
 config NVME_MULTIPATH
 	bool "NVMe multipath support"
 	depends on NVME_CORE
-	---help---
+	help
 	   This option enables support for multipath access to NVMe
 	   subsystems.  If this option is enabled only a single
 	   /dev/nvmeXnY device will show up for each NVMe namespaces,
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 0585efa47d8f..c2c5bc4fb702 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -3669,7 +3669,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 	ns->disk = disk;
 
 	if (__nvme_revalidate_disk(disk, id))
-		goto out_free_disk;
+		goto out_put_disk;
 
 	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
 		ret = nvme_nvm_register(ns, disk_name, node);
@@ -3696,8 +3696,6 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 	/* prevent double queue cleanup */
 	ns->disk->queue = NULL;
 	put_disk(ns->disk);
- out_free_disk:
-	del_gendisk(ns->disk);
  out_unlink_ns:
 	mutex_lock(&ctrl->subsys->lock);
 	list_del_rcu(&ns->siblings);
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index cb0007592c12..e999a8c4b7e8 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -2634,10 +2634,11 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
 		opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
 		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
 
-		if (!(op->flags & FCOP_FLAGS_AEN))
+		if (!(op->flags & FCOP_FLAGS_AEN)) {
 			nvme_fc_unmap_data(ctrl, op->rq, op);
+			nvme_cleanup_cmd(op->rq);
+		}
 
-		nvme_cleanup_cmd(op->rq);
 		nvme_fc_ctrl_put(ctrl);
 
 		if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index fa5c75501049..c0f4226d3299 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -599,8 +599,7 @@ static inline void nvme_trace_bio_complete(struct request *req,
 	struct nvme_ns *ns = req->q->queuedata;
 
 	if (req->cmd_flags & REQ_NVME_MPATH)
-		trace_block_bio_complete(ns->head->disk->queue,
-					 req->bio, status);
+		trace_block_bio_complete(ns->head->disk->queue, req->bio);
 }
 
 extern struct device_attribute dev_attr_ana_grpid;
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index d690d5593a80..e2bacd369a88 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -2950,9 +2950,15 @@ static int nvme_suspend(struct device *dev)
 	 * the PCI bus layer to put it into D3 in order to take the PCIe link
 	 * down, so as to allow the platform to achieve its minimum low-power
 	 * state (which may not be possible if the link is up).
+	 *
+	 * If a host memory buffer is enabled, shut down the device as the NVMe
+	 * specification allows the device to access the host memory buffer in
+	 * host DRAM from all power states, but hosts will fail access to DRAM
+	 * during S3.
 	 */
 	if (pm_suspend_via_firmware() || !ctrl->npss ||
 	    !pcie_aspm_enabled(pdev) ||
+	    ndev->nr_host_mem_descs ||
 	    (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND))
 		return nvme_disable_prepare_reset(ndev, true);
 
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 1843110ec34f..3345ec7efaff 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -131,8 +131,8 @@ struct nvme_tcp_ctrl {
 static LIST_HEAD(nvme_tcp_ctrl_list);
 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
 static struct workqueue_struct *nvme_tcp_wq;
-static struct blk_mq_ops nvme_tcp_mq_ops;
-static struct blk_mq_ops nvme_tcp_admin_mq_ops;
+static const struct blk_mq_ops nvme_tcp_mq_ops;
+static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
 
 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
@@ -2301,7 +2301,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx)
 	return queue->nr_cqe;
 }
 
-static struct blk_mq_ops nvme_tcp_mq_ops = {
+static const struct blk_mq_ops nvme_tcp_mq_ops = {
 	.queue_rq	= nvme_tcp_queue_rq,
 	.complete	= nvme_complete_rq,
 	.init_request	= nvme_tcp_init_request,
@@ -2312,7 +2312,7 @@ static struct blk_mq_ops nvme_tcp_mq_ops = {
 	.poll		= nvme_tcp_poll,
 };
 
-static struct blk_mq_ops nvme_tcp_admin_mq_ops = {
+static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
 	.queue_rq	= nvme_tcp_queue_rq,
 	.complete	= nvme_complete_rq,
 	.init_request	= nvme_tcp_init_request,
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 6392bcd30bd7..6e2f623e472e 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -129,7 +129,22 @@ static u32 nvmet_async_event_result(struct nvmet_async_event *aen)
 	return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16);
 }
 
-static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status)
+static void nvmet_async_events_failall(struct nvmet_ctrl *ctrl)
+{
+	u16 status = NVME_SC_INTERNAL | NVME_SC_DNR;
+	struct nvmet_req *req;
+
+	mutex_lock(&ctrl->lock);
+	while (ctrl->nr_async_event_cmds) {
+		req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
+		mutex_unlock(&ctrl->lock);
+		nvmet_req_complete(req, status);
+		mutex_lock(&ctrl->lock);
+	}
+	mutex_unlock(&ctrl->lock);
+}
+
+static void nvmet_async_events_process(struct nvmet_ctrl *ctrl)
 {
 	struct nvmet_async_event *aen;
 	struct nvmet_req *req;
@@ -139,15 +154,14 @@ static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status)
 		aen = list_first_entry(&ctrl->async_events,
 				       struct nvmet_async_event, entry);
 		req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds];
-		if (status == 0)
-			nvmet_set_result(req, nvmet_async_event_result(aen));
+		nvmet_set_result(req, nvmet_async_event_result(aen));
 
 		list_del(&aen->entry);
 		kfree(aen);
 
 		mutex_unlock(&ctrl->lock);
 		trace_nvmet_async_event(ctrl, req->cqe->result.u32);
-		nvmet_req_complete(req, status);
+		nvmet_req_complete(req, 0);
 		mutex_lock(&ctrl->lock);
 	}
 	mutex_unlock(&ctrl->lock);
@@ -170,7 +184,7 @@ static void nvmet_async_event_work(struct work_struct *work)
 	struct nvmet_ctrl *ctrl =
 		container_of(work, struct nvmet_ctrl, async_event_work);
 
-	nvmet_async_events_process(ctrl, 0);
+	nvmet_async_events_process(ctrl);
 }
 
 void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
@@ -779,7 +793,6 @@ static void nvmet_confirm_sq(struct percpu_ref *ref)
 
 void nvmet_sq_destroy(struct nvmet_sq *sq)
 {
-	u16 status = NVME_SC_INTERNAL | NVME_SC_DNR;
 	struct nvmet_ctrl *ctrl = sq->ctrl;
 
 	/*
@@ -787,7 +800,7 @@ void nvmet_sq_destroy(struct nvmet_sq *sq)
 	 * queue doesn't have outstanding requests on it.
 	 */
 	if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq)
-		nvmet_async_events_process(ctrl, status);
+		nvmet_async_events_failall(ctrl);
 	percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq);
 	wait_for_completion(&sq->confirm_done);
 	wait_for_completion(&sq->free_done);
diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index d5141780592e..76ea23a2c2be 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -20,6 +20,7 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/rdma_cm.h>
 #include <rdma/rw.h>
+#include <rdma/ib_cm.h>
 
 #include <linux/nvme-rdma.h>
 #include "nvmet.h"
@@ -1403,7 +1404,8 @@ static int nvmet_rdma_cm_reject(struct rdma_cm_id *cm_id,
 	rej.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
 	rej.sts = cpu_to_le16(status);
 
-	return rdma_reject(cm_id, (void *)&rej, sizeof(rej));
+	return rdma_reject(cm_id, (void *)&rej, sizeof(rej),
+			   IB_CM_REJ_CONSUMER_DEFINED);
 }
 
 static struct nvmet_rdma_queue *
diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 1669177cd26c..de9217cfd22d 100644
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -153,7 +153,7 @@ static LIST_HEAD(nvmet_tcp_queue_list);
 static DEFINE_MUTEX(nvmet_tcp_queue_mutex);
 
 static struct workqueue_struct *nvmet_tcp_wq;
-static struct nvmet_fabrics_ops nvmet_tcp_ops;
+static const struct nvmet_fabrics_ops nvmet_tcp_ops;
 static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
 static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd);
 
@@ -1713,7 +1713,7 @@ static void nvmet_tcp_disc_port_addr(struct nvmet_req *req,
 	}
 }
 
-static struct nvmet_fabrics_ops nvmet_tcp_ops = {
+static const struct nvmet_fabrics_ops nvmet_tcp_ops = {
 	.owner			= THIS_MODULE,
 	.type			= NVMF_TRTYPE_TCP,
 	.msdbd			= 1,
diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
index 05c6ae4b0b97..927eb5f6003f 100644
--- a/drivers/nvmem/core.c
+++ b/drivers/nvmem/core.c
@@ -66,6 +66,30 @@ static LIST_HEAD(nvmem_lookup_list);
 
 static BLOCKING_NOTIFIER_HEAD(nvmem_notifier);
 
+static int nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset,
+			  void *val, size_t bytes)
+{
+	if (nvmem->reg_read)
+		return nvmem->reg_read(nvmem->priv, offset, val, bytes);
+
+	return -EINVAL;
+}
+
+static int nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset,
+			   void *val, size_t bytes)
+{
+	int ret;
+
+	if (nvmem->reg_write) {
+		gpiod_set_value_cansleep(nvmem->wp_gpio, 0);
+		ret = nvmem->reg_write(nvmem->priv, offset, val, bytes);
+		gpiod_set_value_cansleep(nvmem->wp_gpio, 1);
+		return ret;
+	}
+
+	return -EINVAL;
+}
+
 #ifdef CONFIG_NVMEM_SYSFS
 static const char * const nvmem_type_str[] = {
 	[NVMEM_TYPE_UNKNOWN] = "Unknown",
@@ -122,7 +146,7 @@ static ssize_t bin_attr_nvmem_read(struct file *filp, struct kobject *kobj,
 	if (!nvmem->reg_read)
 		return -EPERM;
 
-	rc = nvmem->reg_read(nvmem->priv, pos, buf, count);
+	rc = nvmem_reg_read(nvmem, pos, buf, count);
 
 	if (rc)
 		return rc;
@@ -159,7 +183,7 @@ static ssize_t bin_attr_nvmem_write(struct file *filp, struct kobject *kobj,
 	if (!nvmem->reg_write)
 		return -EPERM;
 
-	rc = nvmem->reg_write(nvmem->priv, pos, buf, count);
+	rc = nvmem_reg_write(nvmem, pos, buf, count);
 
 	if (rc)
 		return rc;
@@ -167,11 +191,8 @@ static ssize_t bin_attr_nvmem_write(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static umode_t nvmem_bin_attr_is_visible(struct kobject *kobj,
-					 struct bin_attribute *attr, int i)
+static umode_t nvmem_bin_attr_get_umode(struct nvmem_device *nvmem)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
-	struct nvmem_device *nvmem = to_nvmem_device(dev);
 	umode_t mode = 0400;
 
 	if (!nvmem->root_only)
@@ -189,6 +210,15 @@ static umode_t nvmem_bin_attr_is_visible(struct kobject *kobj,
 	return mode;
 }
 
+static umode_t nvmem_bin_attr_is_visible(struct kobject *kobj,
+					 struct bin_attribute *attr, int i)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct nvmem_device *nvmem = to_nvmem_device(dev);
+
+	return nvmem_bin_attr_get_umode(nvmem);
+}
+
 /* default read/write permissions */
 static struct bin_attribute bin_attr_rw_nvmem = {
 	.attr	= {
@@ -215,34 +245,14 @@ static const struct attribute_group *nvmem_dev_groups[] = {
 	NULL,
 };
 
-/* read only permission */
-static struct bin_attribute bin_attr_ro_nvmem = {
+static struct bin_attribute bin_attr_nvmem_eeprom_compat = {
 	.attr	= {
-		.name	= "nvmem",
-		.mode	= 0444,
-	},
-	.read	= bin_attr_nvmem_read,
-};
-
-/* default read/write permissions, root only */
-static struct bin_attribute bin_attr_rw_root_nvmem = {
-	.attr	= {
-		.name	= "nvmem",
-		.mode	= 0600,
+		.name	= "eeprom",
 	},
 	.read	= bin_attr_nvmem_read,
 	.write	= bin_attr_nvmem_write,
 };
 
-/* read only permission, root only */
-static struct bin_attribute bin_attr_ro_root_nvmem = {
-	.attr	= {
-		.name	= "nvmem",
-		.mode	= 0400,
-	},
-	.read	= bin_attr_nvmem_read,
-};
-
 /*
  * nvmem_setup_compat() - Create an additional binary entry in
  * drivers sys directory, to be backwards compatible with the older
@@ -259,18 +269,8 @@ static int nvmem_sysfs_setup_compat(struct nvmem_device *nvmem,
 	if (!config->base_dev)
 		return -EINVAL;
 
-	if (nvmem->read_only) {
-		if (config->root_only)
-			nvmem->eeprom = bin_attr_ro_root_nvmem;
-		else
-			nvmem->eeprom = bin_attr_ro_nvmem;
-	} else {
-		if (config->root_only)
-			nvmem->eeprom = bin_attr_rw_root_nvmem;
-		else
-			nvmem->eeprom = bin_attr_rw_nvmem;
-	}
-	nvmem->eeprom.attr.name = "eeprom";
+	nvmem->eeprom = bin_attr_nvmem_eeprom_compat;
+	nvmem->eeprom.attr.mode = nvmem_bin_attr_get_umode(nvmem);
 	nvmem->eeprom.size = nvmem->size;
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	nvmem->eeprom.attr.key = &eeprom_lock_key;
@@ -311,30 +311,6 @@ static void nvmem_sysfs_remove_compat(struct nvmem_device *nvmem,
 
 #endif /* CONFIG_NVMEM_SYSFS */
 
-static int nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset,
-			  void *val, size_t bytes)
-{
-	if (nvmem->reg_read)
-		return nvmem->reg_read(nvmem->priv, offset, val, bytes);
-
-	return -EINVAL;
-}
-
-static int nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset,
-			   void *val, size_t bytes)
-{
-	int ret;
-
-	if (nvmem->reg_write) {
-		gpiod_set_value_cansleep(nvmem->wp_gpio, 0);
-		ret = nvmem->reg_write(nvmem->priv, offset, val, bytes);
-		gpiod_set_value_cansleep(nvmem->wp_gpio, 1);
-		return ret;
-	}
-
-	return -EINVAL;
-}
-
 static void nvmem_release(struct device *dev)
 {
 	struct nvmem_device *nvmem = to_nvmem_device(dev);
diff --git a/drivers/nvmem/imx-ocotp.c b/drivers/nvmem/imx-ocotp.c
index 50bea2aadc1b..7a1ebd6fd08b 100644
--- a/drivers/nvmem/imx-ocotp.c
+++ b/drivers/nvmem/imx-ocotp.c
@@ -196,7 +196,6 @@ static int imx_ocotp_read(void *context, unsigned int offset,
 		if (*(buf - 1) == IMX_OCOTP_READ_LOCKED_VAL)
 			imx_ocotp_clr_err_if_set(priv);
 	}
-	ret = 0;
 
 read_end:
 	clk_disable_unprepare(priv->clk);
@@ -435,17 +434,13 @@ static int imx_ocotp_write(void *context, unsigned int offset, void *val,
 	       priv->base + IMX_OCOTP_ADDR_CTRL_SET);
 	ret = imx_ocotp_wait_for_busy(priv,
 				      priv->params->ctrl.bm_rel_shadows);
-	if (ret < 0) {
+	if (ret < 0)
 		dev_err(priv->dev, "timeout during shadow register reload\n");
-		goto write_end;
-	}
 
 write_end:
 	clk_disable_unprepare(priv->clk);
 	mutex_unlock(&ocotp_mutex);
-	if (ret < 0)
-		return ret;
-	return bytes;
+	return ret < 0 ? ret : bytes;
 }
 
 static struct nvmem_config imx_ocotp_nvmem_config = {
diff --git a/drivers/nvmem/jz4780-efuse.c b/drivers/nvmem/jz4780-efuse.c
index 512e1872ba36..0b01b840edd9 100644
--- a/drivers/nvmem/jz4780-efuse.c
+++ b/drivers/nvmem/jz4780-efuse.c
@@ -211,10 +211,8 @@ static int jz4780_efuse_probe(struct platform_device *pdev)
 	cfg.priv = efuse;
 
 	nvmem = devm_nvmem_register(dev, &cfg);
-	if (IS_ERR(nvmem))
-		return PTR_ERR(nvmem);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(nvmem);
 }
 
 static const struct of_device_id jz4780_efuse_match[] = {
diff --git a/drivers/nvmem/qfprom.c b/drivers/nvmem/qfprom.c
index d057f1bfb2e9..8a91717600be 100644
--- a/drivers/nvmem/qfprom.c
+++ b/drivers/nvmem/qfprom.c
@@ -27,25 +27,11 @@ static int qfprom_reg_read(void *context,
 	return 0;
 }
 
-static int qfprom_reg_write(void *context,
-			 unsigned int reg, void *_val, size_t bytes)
-{
-	struct qfprom_priv *priv = context;
-	u8 *val = _val;
-	int i = 0, words = bytes;
-
-	while (words--)
-		writeb(*val++, priv->base + reg + i++);
-
-	return 0;
-}
-
 static struct nvmem_config econfig = {
 	.name = "qfprom",
 	.stride = 1,
 	.word_size = 1,
 	.reg_read = qfprom_reg_read,
-	.reg_write = qfprom_reg_write,
 };
 
 static int qfprom_probe(struct platform_device *pdev)
diff --git a/drivers/nvmem/zynqmp_nvmem.c b/drivers/nvmem/zynqmp_nvmem.c
index 5893543918c8..e28d7b133e11 100644
--- a/drivers/nvmem/zynqmp_nvmem.c
+++ b/drivers/nvmem/zynqmp_nvmem.c
@@ -16,8 +16,6 @@ struct zynqmp_nvmem_data {
 	struct nvmem_device *nvmem;
 };
 
-static const struct zynqmp_eemi_ops *eemi_ops;
-
 static int zynqmp_nvmem_read(void *context, unsigned int offset,
 			     void *val, size_t bytes)
 {
@@ -25,10 +23,7 @@ static int zynqmp_nvmem_read(void *context, unsigned int offset,
 	int idcode, version;
 	struct zynqmp_nvmem_data *priv = context;
 
-	if (!eemi_ops->get_chipid)
-		return -ENXIO;
-
-	ret = eemi_ops->get_chipid(&idcode, &version);
+	ret = zynqmp_pm_get_chipid(&idcode, &version);
 	if (ret < 0)
 		return ret;
 
@@ -61,10 +56,6 @@ static int zynqmp_nvmem_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
-	eemi_ops = zynqmp_pm_get_eemi_ops();
-	if (IS_ERR(eemi_ops))
-		return PTR_ERR(eemi_ops);
-
 	priv->dev = dev;
 	econfig.dev = dev;
 	econfig.reg_read = zynqmp_nvmem_read;
diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c
index 08fd823edac9..fe64430b438a 100644
--- a/drivers/of/dynamic.c
+++ b/drivers/of/dynamic.c
@@ -286,7 +286,6 @@ int of_detach_node(struct device_node *np)
 {
 	struct of_reconfig_data rd;
 	unsigned long flags;
-	int rc = 0;
 
 	memset(&rd, 0, sizeof(rd));
 	rd.dn = np;
@@ -301,7 +300,7 @@ int of_detach_node(struct device_node *np)
 
 	of_reconfig_notify(OF_RECONFIG_DETACH_NODE, &rd);
 
-	return rc;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(of_detach_node);
 
diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
index 2cdf64d2456f..4602e467ca8b 100644
--- a/drivers/of/fdt.c
+++ b/drivers/of/fdt.c
@@ -471,7 +471,7 @@ void *initial_boot_params __ro_after_init;
 static u32 of_fdt_crc32;
 
 /**
- * res_mem_reserve_reg() - reserve all memory described in 'reg' property
+ * __reserved_mem_reserve_reg() - reserve all memory described in 'reg' property
  */
 static int __init __reserved_mem_reserve_reg(unsigned long node,
 					     const char *uname)
@@ -643,8 +643,6 @@ int __init of_scan_flat_dt(int (*it)(unsigned long node,
 	     offset = fdt_next_node(blob, offset, &depth)) {
 
 		pathp = fdt_get_name(blob, offset, NULL);
-		if (*pathp == '/')
-			pathp = kbasename(pathp);
 		rc = it(offset, pathp, depth, data);
 	}
 	return rc;
@@ -671,8 +669,6 @@ int __init of_scan_flat_dt_subnodes(unsigned long parent,
 		int rc;
 
 		pathp = fdt_get_name(blob, node, NULL);
-		if (*pathp == '/')
-			pathp = kbasename(pathp);
 		rc = it(node, pathp, data);
 		if (rc)
 			return rc;
@@ -1078,7 +1074,7 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname,
 #endif
 #endif /* CONFIG_CMDLINE */
 
-	pr_debug("Command line is: %s\n", (char*)data);
+	pr_debug("Command line is: %s\n", (char *)data);
 
 	rng_seed = of_get_flat_dt_prop(node, "rng-seed", &l);
 	if (rng_seed && l > 0) {
diff --git a/drivers/of/kobj.c b/drivers/of/kobj.c
index c72eef988041..a32e60b024b8 100644
--- a/drivers/of/kobj.c
+++ b/drivers/of/kobj.c
@@ -134,8 +134,6 @@ int __of_attach_node_sysfs(struct device_node *np)
 	if (!name)
 		return -ENOMEM;
 
-	of_node_get(np);
-
 	rc = kobject_add(&np->kobj, parent, "%s", name);
 	kfree(name);
 	if (rc)
@@ -144,6 +142,7 @@ int __of_attach_node_sysfs(struct device_node *np)
 	for_each_property_of_node(np, pp)
 		__of_add_property_sysfs(np, pp);
 
+	of_node_get(np);
 	return 0;
 }
 
diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c
index 1a84bc0d5fa8..6877080c8af9 100644
--- a/drivers/of/of_reserved_mem.c
+++ b/drivers/of/of_reserved_mem.c
@@ -46,7 +46,7 @@ static int __init early_init_dt_alloc_reserved_memory_arch(phys_addr_t size,
 }
 
 /**
- * res_mem_save_node() - save fdt node for second pass initialization
+ * fdt_reserved_mem_save_node() - save fdt node for second pass initialization
  */
 void __init fdt_reserved_mem_save_node(unsigned long node, const char *uname,
 				      phys_addr_t base, phys_addr_t size)
@@ -68,8 +68,8 @@ void __init fdt_reserved_mem_save_node(unsigned long node, const char *uname,
 }
 
 /**
- * res_mem_alloc_size() - allocate reserved memory described by 'size', 'align'
- *			  and 'alloc-ranges' properties
+ * __reserved_mem_alloc_size() - allocate reserved memory described by
+ *	'size', 'align'  and 'alloc-ranges' properties.
  */
 static int __init __reserved_mem_alloc_size(unsigned long node,
 	const char *uname, phys_addr_t *res_base, phys_addr_t *res_size)
@@ -165,7 +165,7 @@ static const struct of_device_id __rmem_of_table_sentinel
 	__used __section(__reservedmem_of_table_end);
 
 /**
- * res_mem_init_node() - call region specific reserved memory init code
+ * __reserved_mem_init_node() - call region specific reserved memory init code
  */
 static int __init __reserved_mem_init_node(struct reserved_mem *rmem)
 {
@@ -232,7 +232,7 @@ static void __init __rmem_check_for_overlap(void)
 }
 
 /**
- * fdt_init_reserved_mem - allocate and init all saved reserved memory regions
+ * fdt_init_reserved_mem() - allocate and init all saved reserved memory regions
  */
 void __init fdt_init_reserved_mem(void)
 {
@@ -358,6 +358,25 @@ int of_reserved_mem_device_init_by_idx(struct device *dev,
 EXPORT_SYMBOL_GPL(of_reserved_mem_device_init_by_idx);
 
 /**
+ * of_reserved_mem_device_init_by_name() - assign named reserved memory region
+ *					   to given device
+ * @dev: pointer to the device to configure
+ * @np: pointer to the device node with 'memory-region' property
+ * @name: name of the selected memory region
+ *
+ * Returns: 0 on success or a negative error-code on failure.
+ */
+int of_reserved_mem_device_init_by_name(struct device *dev,
+					struct device_node *np,
+					const char *name)
+{
+	int idx = of_property_match_string(np, "memory-region-names", name);
+
+	return of_reserved_mem_device_init_by_idx(dev, np, idx);
+}
+EXPORT_SYMBOL_GPL(of_reserved_mem_device_init_by_name);
+
+/**
  * of_reserved_mem_device_release() - release reserved memory device structures
  * @dev:	Pointer to the device to deconfigure
  *
@@ -366,24 +385,22 @@ EXPORT_SYMBOL_GPL(of_reserved_mem_device_init_by_idx);
  */
 void of_reserved_mem_device_release(struct device *dev)
 {
-	struct rmem_assigned_device *rd;
-	struct reserved_mem *rmem = NULL;
+	struct rmem_assigned_device *rd, *tmp;
+	LIST_HEAD(release_list);
 
 	mutex_lock(&of_rmem_assigned_device_mutex);
-	list_for_each_entry(rd, &of_rmem_assigned_device_list, list) {
-		if (rd->dev == dev) {
-			rmem = rd->rmem;
-			list_del(&rd->list);
-			kfree(rd);
-			break;
-		}
+	list_for_each_entry_safe(rd, tmp, &of_rmem_assigned_device_list, list) {
+		if (rd->dev == dev)
+			list_move_tail(&rd->list, &release_list);
 	}
 	mutex_unlock(&of_rmem_assigned_device_mutex);
 
-	if (!rmem || !rmem->ops || !rmem->ops->device_release)
-		return;
+	list_for_each_entry_safe(rd, tmp, &release_list, list) {
+		if (rd->rmem && rd->rmem->ops && rd->rmem->ops->device_release)
+			rd->rmem->ops->device_release(rd->rmem, dev);
 
-	rmem->ops->device_release(rmem, dev);
+		kfree(rd);
+	}
 }
 EXPORT_SYMBOL_GPL(of_reserved_mem_device_release);
 
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index 3371e4a06248..071f04da32c8 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -291,7 +291,7 @@ static struct amba_device *of_amba_device_create(struct device_node *node,
 #endif /* CONFIG_ARM_AMBA */
 
 /**
- * of_devname_lookup() - Given a device node, lookup the preferred Linux name
+ * of_dev_lookup() - Given a device node, lookup the preferred Linux name
  */
 static const struct of_dev_auxdata *of_dev_lookup(const struct of_dev_auxdata *lookup,
 				 struct device_node *np)
@@ -538,7 +538,9 @@ static int __init of_platform_default_populate_init(void)
 	}
 
 	/* Populate everything else. */
+	fw_devlink_pause();
 	of_platform_default_populate(NULL, NULL, NULL);
+	fw_devlink_resume();
 
 	return 0;
 }
diff --git a/drivers/of/property.c b/drivers/of/property.c
index b4916dcc9e72..1f2086f4e7ce 100644
--- a/drivers/of/property.c
+++ b/drivers/of/property.c
@@ -1045,8 +1045,20 @@ static int of_link_to_phandle(struct device *dev, struct device_node *sup_np,
 	 * Find the device node that contains the supplier phandle.  It may be
 	 * @sup_np or it may be an ancestor of @sup_np.
 	 */
-	while (sup_np && !of_find_property(sup_np, "compatible", NULL))
+	while (sup_np) {
+
+		/* Don't allow linking to a disabled supplier */
+		if (!of_device_is_available(sup_np)) {
+			of_node_put(sup_np);
+			sup_np = NULL;
+		}
+
+		if (of_find_property(sup_np, "compatible", NULL))
+			break;
+
 		sup_np = of_get_next_parent(sup_np);
+	}
+
 	if (!sup_np) {
 		dev_dbg(dev, "Not linking to %pOFP - No device\n", tmp_np);
 		return -ENODEV;
@@ -1074,7 +1086,7 @@ static int of_link_to_phandle(struct device *dev, struct device_node *sup_np,
 		return -EAGAIN;
 	}
 	if (!device_link_add(dev, sup_dev, dl_flags))
-		ret = -EAGAIN;
+		ret = -EINVAL;
 	put_device(sup_dev);
 	return ret;
 }
@@ -1206,6 +1218,7 @@ DEFINE_SIMPLE_PROP(interrupt_parent, "interrupt-parent", NULL)
 DEFINE_SIMPLE_PROP(dmas, "dmas", "#dma-cells")
 DEFINE_SIMPLE_PROP(power_domains, "power-domains", "#power-domain-cells")
 DEFINE_SIMPLE_PROP(hwlocks, "hwlocks", "#hwlock-cells")
+DEFINE_SIMPLE_PROP(extcon, "extcon", NULL)
 DEFINE_SUFFIX_PROP(regulators, "-supply", NULL)
 DEFINE_SUFFIX_PROP(gpio, "-gpio", "#gpio-cells")
 DEFINE_SUFFIX_PROP(gpios, "-gpios", "#gpio-cells")
@@ -1230,6 +1243,7 @@ static const struct supplier_bindings of_supplier_bindings[] = {
 	{ .parse_prop = parse_dmas, },
 	{ .parse_prop = parse_power_domains, },
 	{ .parse_prop = parse_hwlocks, },
+	{ .parse_prop = parse_extcon, },
 	{ .parse_prop = parse_regulators, },
 	{ .parse_prop = parse_gpio, },
 	{ .parse_prop = parse_gpios, },
@@ -1296,7 +1310,7 @@ static int of_link_to_suppliers(struct device *dev,
 		if (of_link_property(dev, con_np, p->name))
 			ret = -ENODEV;
 
-	for_each_child_of_node(con_np, child)
+	for_each_available_child_of_node(con_np, child)
 		if (of_link_to_suppliers(dev, child) && !ret)
 			ret = -EAGAIN;
 
diff --git a/drivers/opp/Kconfig b/drivers/opp/Kconfig
index 35dfc7e80f92..e8ce47b32735 100644
--- a/drivers/opp/Kconfig
+++ b/drivers/opp/Kconfig
@@ -2,7 +2,7 @@
 config PM_OPP
 	bool
 	select SRCU
-	---help---
+	help
 	  SOCs have a standard set of tuples consisting of frequency and
 	  voltage pairs that the device will support per voltage domain. This
 	  is called Operating Performance Point or OPP. The actual definitions
diff --git a/drivers/opp/core.c b/drivers/opp/core.c
index e4f01e7771a2..dfbd3d10410c 100644
--- a/drivers/opp/core.c
+++ b/drivers/opp/core.c
@@ -664,7 +664,7 @@ static inline int _generic_set_opp_clk_only(struct device *dev, struct clk *clk,
 	return ret;
 }
 
-static int _generic_set_opp_regulator(const struct opp_table *opp_table,
+static int _generic_set_opp_regulator(struct opp_table *opp_table,
 				      struct device *dev,
 				      unsigned long old_freq,
 				      unsigned long freq,
@@ -699,6 +699,18 @@ static int _generic_set_opp_regulator(const struct opp_table *opp_table,
 			goto restore_freq;
 	}
 
+	/*
+	 * Enable the regulator after setting its voltages, otherwise it breaks
+	 * some boot-enabled regulators.
+	 */
+	if (unlikely(!opp_table->regulator_enabled)) {
+		ret = regulator_enable(reg);
+		if (ret < 0)
+			dev_warn(dev, "Failed to enable regulator: %d", ret);
+		else
+			opp_table->regulator_enabled = true;
+	}
+
 	return 0;
 
 restore_freq:
@@ -713,6 +725,34 @@ restore_voltage:
 	return ret;
 }
 
+static int _set_opp_bw(const struct opp_table *opp_table,
+		       struct dev_pm_opp *opp, struct device *dev, bool remove)
+{
+	u32 avg, peak;
+	int i, ret;
+
+	if (!opp_table->paths)
+		return 0;
+
+	for (i = 0; i < opp_table->path_count; i++) {
+		if (remove) {
+			avg = 0;
+			peak = 0;
+		} else {
+			avg = opp->bandwidth[i].avg;
+			peak = opp->bandwidth[i].peak;
+		}
+		ret = icc_set_bw(opp_table->paths[i], avg, peak);
+		if (ret) {
+			dev_err(dev, "Failed to %s bandwidth[%d]: %d\n",
+				remove ? "remove" : "set", i, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
 static int _set_opp_custom(const struct opp_table *opp_table,
 			   struct device *dev, unsigned long old_freq,
 			   unsigned long freq,
@@ -817,15 +857,31 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 	}
 
 	if (unlikely(!target_freq)) {
-		if (opp_table->required_opp_tables) {
-			ret = _set_required_opps(dev, opp_table, NULL);
-		} else if (!_get_opp_count(opp_table)) {
+		/*
+		 * Some drivers need to support cases where some platforms may
+		 * have OPP table for the device, while others don't and
+		 * opp_set_rate() just needs to behave like clk_set_rate().
+		 */
+		if (!_get_opp_count(opp_table))
 			return 0;
-		} else {
+
+		if (!opp_table->required_opp_tables && !opp_table->regulators &&
+		    !opp_table->paths) {
 			dev_err(dev, "target frequency can't be 0\n");
 			ret = -EINVAL;
+			goto put_opp_table;
+		}
+
+		ret = _set_opp_bw(opp_table, NULL, dev, true);
+		if (ret)
+			return ret;
+
+		if (opp_table->regulator_enabled) {
+			regulator_disable(opp_table->regulators[0]);
+			opp_table->regulator_enabled = false;
 		}
 
+		ret = _set_required_opps(dev, opp_table, NULL);
 		goto put_opp_table;
 	}
 
@@ -909,6 +965,9 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
 			dev_err(dev, "Failed to set required opps: %d\n", ret);
 	}
 
+	if (!ret)
+		ret = _set_opp_bw(opp_table, opp, dev, false);
+
 put_opp:
 	dev_pm_opp_put(opp);
 put_old_opp:
@@ -999,6 +1058,12 @@ static struct opp_table *_allocate_opp_table(struct device *dev, int index)
 				ret);
 	}
 
+	/* Find interconnect path(s) for the device */
+	ret = dev_pm_opp_of_find_icc_paths(dev, opp_table);
+	if (ret)
+		dev_warn(dev, "%s: Error finding interconnect paths: %d\n",
+			 __func__, ret);
+
 	BLOCKING_INIT_NOTIFIER_HEAD(&opp_table->head);
 	INIT_LIST_HEAD(&opp_table->opp_list);
 	kref_init(&opp_table->kref);
@@ -1057,6 +1122,7 @@ static void _opp_table_kref_release(struct kref *kref)
 {
 	struct opp_table *opp_table = container_of(kref, struct opp_table, kref);
 	struct opp_device *opp_dev, *temp;
+	int i;
 
 	_of_clear_opp_table(opp_table);
 
@@ -1064,6 +1130,12 @@ static void _opp_table_kref_release(struct kref *kref)
 	if (!IS_ERR(opp_table->clk))
 		clk_put(opp_table->clk);
 
+	if (opp_table->paths) {
+		for (i = 0; i < opp_table->path_count; i++)
+			icc_put(opp_table->paths[i]);
+		kfree(opp_table->paths);
+	}
+
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
 	list_for_each_entry_safe(opp_dev, temp, &opp_table->dev_list, node) {
@@ -1243,19 +1315,23 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_remove_all_dynamic);
 struct dev_pm_opp *_opp_allocate(struct opp_table *table)
 {
 	struct dev_pm_opp *opp;
-	int count, supply_size;
+	int supply_count, supply_size, icc_size;
 
 	/* Allocate space for at least one supply */
-	count = table->regulator_count > 0 ? table->regulator_count : 1;
-	supply_size = sizeof(*opp->supplies) * count;
+	supply_count = table->regulator_count > 0 ? table->regulator_count : 1;
+	supply_size = sizeof(*opp->supplies) * supply_count;
+	icc_size = sizeof(*opp->bandwidth) * table->path_count;
 
 	/* allocate new OPP node and supplies structures */
-	opp = kzalloc(sizeof(*opp) + supply_size, GFP_KERNEL);
+	opp = kzalloc(sizeof(*opp) + supply_size + icc_size, GFP_KERNEL);
+
 	if (!opp)
 		return NULL;
 
 	/* Put the supplies at the end of the OPP structure as an empty array */
 	opp->supplies = (struct dev_pm_opp_supply *)(opp + 1);
+	if (icc_size)
+		opp->bandwidth = (struct dev_pm_opp_icc_bw *)(opp->supplies + supply_count);
 	INIT_LIST_HEAD(&opp->node);
 
 	return opp;
@@ -1286,11 +1362,24 @@ static bool _opp_supported_by_regulators(struct dev_pm_opp *opp,
 	return true;
 }
 
+int _opp_compare_key(struct dev_pm_opp *opp1, struct dev_pm_opp *opp2)
+{
+	if (opp1->rate != opp2->rate)
+		return opp1->rate < opp2->rate ? -1 : 1;
+	if (opp1->bandwidth && opp2->bandwidth &&
+	    opp1->bandwidth[0].peak != opp2->bandwidth[0].peak)
+		return opp1->bandwidth[0].peak < opp2->bandwidth[0].peak ? -1 : 1;
+	if (opp1->level != opp2->level)
+		return opp1->level < opp2->level ? -1 : 1;
+	return 0;
+}
+
 static int _opp_is_duplicate(struct device *dev, struct dev_pm_opp *new_opp,
 			     struct opp_table *opp_table,
 			     struct list_head **head)
 {
 	struct dev_pm_opp *opp;
+	int opp_cmp;
 
 	/*
 	 * Insert new OPP in order of increasing frequency and discard if
@@ -1301,12 +1390,13 @@ static int _opp_is_duplicate(struct device *dev, struct dev_pm_opp *new_opp,
 	 * loop.
 	 */
 	list_for_each_entry(opp, &opp_table->opp_list, node) {
-		if (new_opp->rate > opp->rate) {
+		opp_cmp = _opp_compare_key(new_opp, opp);
+		if (opp_cmp > 0) {
 			*head = &opp->node;
 			continue;
 		}
 
-		if (new_opp->rate < opp->rate)
+		if (opp_cmp < 0)
 			return 0;
 
 		/* Duplicate OPPs */
@@ -1670,6 +1760,13 @@ void dev_pm_opp_put_regulators(struct opp_table *opp_table)
 	/* Make sure there are no concurrent readers while updating opp_table */
 	WARN_ON(!list_empty(&opp_table->opp_list));
 
+	if (opp_table->regulator_enabled) {
+		for (i = opp_table->regulator_count - 1; i >= 0; i--)
+			regulator_disable(opp_table->regulators[i]);
+
+		opp_table->regulator_enabled = false;
+	}
+
 	for (i = opp_table->regulator_count - 1; i >= 0; i--)
 		regulator_put(opp_table->regulators[i]);
 
diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c
index 609665e339b6..596c185b5dda 100644
--- a/drivers/opp/debugfs.c
+++ b/drivers/opp/debugfs.c
@@ -32,6 +32,47 @@ void opp_debug_remove_one(struct dev_pm_opp *opp)
 	debugfs_remove_recursive(opp->dentry);
 }
 
+static ssize_t bw_name_read(struct file *fp, char __user *userbuf,
+			    size_t count, loff_t *ppos)
+{
+	struct icc_path *path = fp->private_data;
+	char buf[64];
+	int i;
+
+	i = scnprintf(buf, sizeof(buf), "%.62s\n", icc_get_name(path));
+
+	return simple_read_from_buffer(userbuf, count, ppos, buf, i);
+}
+
+static const struct file_operations bw_name_fops = {
+	.open = simple_open,
+	.read = bw_name_read,
+	.llseek = default_llseek,
+};
+
+static void opp_debug_create_bw(struct dev_pm_opp *opp,
+				struct opp_table *opp_table,
+				struct dentry *pdentry)
+{
+	struct dentry *d;
+	char name[11];
+	int i;
+
+	for (i = 0; i < opp_table->path_count; i++) {
+		snprintf(name, sizeof(name), "icc-path-%.1d", i);
+
+		/* Create per-path directory */
+		d = debugfs_create_dir(name, pdentry);
+
+		debugfs_create_file("name", S_IRUGO, d, opp_table->paths[i],
+				    &bw_name_fops);
+		debugfs_create_u32("peak_bw", S_IRUGO, d,
+				   &opp->bandwidth[i].peak);
+		debugfs_create_u32("avg_bw", S_IRUGO, d,
+				   &opp->bandwidth[i].avg);
+	}
+}
+
 static void opp_debug_create_supplies(struct dev_pm_opp *opp,
 				      struct opp_table *opp_table,
 				      struct dentry *pdentry)
@@ -94,6 +135,7 @@ void opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table)
 			     &opp->clock_latency_ns);
 
 	opp_debug_create_supplies(opp, opp_table, d);
+	opp_debug_create_bw(opp, opp_table, d);
 
 	opp->dentry = d;
 }
diff --git a/drivers/opp/of.c b/drivers/opp/of.c
index 9cd8f0adacae..9a5873591a40 100644
--- a/drivers/opp/of.c
+++ b/drivers/opp/of.c
@@ -332,6 +332,105 @@ free_required_opps:
 	return ret;
 }
 
+static int _bandwidth_supported(struct device *dev, struct opp_table *opp_table)
+{
+	struct device_node *np, *opp_np;
+	struct property *prop;
+
+	if (!opp_table) {
+		np = of_node_get(dev->of_node);
+		if (!np)
+			return -ENODEV;
+
+		opp_np = _opp_of_get_opp_desc_node(np, 0);
+		of_node_put(np);
+	} else {
+		opp_np = of_node_get(opp_table->np);
+	}
+
+	/* Lets not fail in case we are parsing opp-v1 bindings */
+	if (!opp_np)
+		return 0;
+
+	/* Checking only first OPP is sufficient */
+	np = of_get_next_available_child(opp_np, NULL);
+	if (!np) {
+		dev_err(dev, "OPP table empty\n");
+		return -EINVAL;
+	}
+	of_node_put(opp_np);
+
+	prop = of_find_property(np, "opp-peak-kBps", NULL);
+	of_node_put(np);
+
+	if (!prop || !prop->length)
+		return 0;
+
+	return 1;
+}
+
+int dev_pm_opp_of_find_icc_paths(struct device *dev,
+				 struct opp_table *opp_table)
+{
+	struct device_node *np;
+	int ret, i, count, num_paths;
+	struct icc_path **paths;
+
+	ret = _bandwidth_supported(dev, opp_table);
+	if (ret <= 0)
+		return ret;
+
+	ret = 0;
+
+	np = of_node_get(dev->of_node);
+	if (!np)
+		return 0;
+
+	count = of_count_phandle_with_args(np, "interconnects",
+					   "#interconnect-cells");
+	of_node_put(np);
+	if (count < 0)
+		return 0;
+
+	/* two phandles when #interconnect-cells = <1> */
+	if (count % 2) {
+		dev_err(dev, "%s: Invalid interconnects values\n", __func__);
+		return -EINVAL;
+	}
+
+	num_paths = count / 2;
+	paths = kcalloc(num_paths, sizeof(*paths), GFP_KERNEL);
+	if (!paths)
+		return -ENOMEM;
+
+	for (i = 0; i < num_paths; i++) {
+		paths[i] = of_icc_get_by_index(dev, i);
+		if (IS_ERR(paths[i])) {
+			ret = PTR_ERR(paths[i]);
+			if (ret != -EPROBE_DEFER) {
+				dev_err(dev, "%s: Unable to get path%d: %d\n",
+					__func__, i, ret);
+			}
+			goto err;
+		}
+	}
+
+	if (opp_table) {
+		opp_table->paths = paths;
+		opp_table->path_count = num_paths;
+		return 0;
+	}
+
+err:
+	while (i--)
+		icc_put(paths[i]);
+
+	kfree(paths);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(dev_pm_opp_of_find_icc_paths);
+
 static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table,
 			      struct device_node *np)
 {
@@ -521,6 +620,90 @@ void dev_pm_opp_of_remove_table(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table);
 
+static int _read_bw(struct dev_pm_opp *new_opp, struct opp_table *table,
+		    struct device_node *np, bool peak)
+{
+	const char *name = peak ? "opp-peak-kBps" : "opp-avg-kBps";
+	struct property *prop;
+	int i, count, ret;
+	u32 *bw;
+
+	prop = of_find_property(np, name, NULL);
+	if (!prop)
+		return -ENODEV;
+
+	count = prop->length / sizeof(u32);
+	if (table->path_count != count) {
+		pr_err("%s: Mismatch between %s and paths (%d %d)\n",
+				__func__, name, count, table->path_count);
+		return -EINVAL;
+	}
+
+	bw = kmalloc_array(count, sizeof(*bw), GFP_KERNEL);
+	if (!bw)
+		return -ENOMEM;
+
+	ret = of_property_read_u32_array(np, name, bw, count);
+	if (ret) {
+		pr_err("%s: Error parsing %s: %d\n", __func__, name, ret);
+		goto out;
+	}
+
+	for (i = 0; i < count; i++) {
+		if (peak)
+			new_opp->bandwidth[i].peak = kBps_to_icc(bw[i]);
+		else
+			new_opp->bandwidth[i].avg = kBps_to_icc(bw[i]);
+	}
+
+out:
+	kfree(bw);
+	return ret;
+}
+
+static int _read_opp_key(struct dev_pm_opp *new_opp, struct opp_table *table,
+			 struct device_node *np, bool *rate_not_available)
+{
+	bool found = false;
+	u64 rate;
+	int ret;
+
+	ret = of_property_read_u64(np, "opp-hz", &rate);
+	if (!ret) {
+		/*
+		 * Rate is defined as an unsigned long in clk API, and so
+		 * casting explicitly to its type. Must be fixed once rate is 64
+		 * bit guaranteed in clk API.
+		 */
+		new_opp->rate = (unsigned long)rate;
+		found = true;
+	}
+	*rate_not_available = !!ret;
+
+	/*
+	 * Bandwidth consists of peak and average (optional) values:
+	 * opp-peak-kBps = <path1_value path2_value>;
+	 * opp-avg-kBps = <path1_value path2_value>;
+	 */
+	ret = _read_bw(new_opp, table, np, true);
+	if (!ret) {
+		found = true;
+		ret = _read_bw(new_opp, table, np, false);
+	}
+
+	/* The properties were found but we failed to parse them */
+	if (ret && ret != -ENODEV)
+		return ret;
+
+	if (!of_property_read_u32(np, "opp-level", &new_opp->level))
+		found = true;
+
+	if (found)
+		return 0;
+
+	return ret;
+}
+
 /**
  * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings)
  * @opp_table:	OPP table
@@ -558,26 +741,12 @@ static struct dev_pm_opp *_opp_add_static_v2(struct opp_table *opp_table,
 	if (!new_opp)
 		return ERR_PTR(-ENOMEM);
 
-	ret = of_property_read_u64(np, "opp-hz", &rate);
-	if (ret < 0) {
-		/* "opp-hz" is optional for devices like power domains. */
-		if (!opp_table->is_genpd) {
-			dev_err(dev, "%s: opp-hz not found\n", __func__);
-			goto free_opp;
-		}
-
-		rate_not_available = true;
-	} else {
-		/*
-		 * Rate is defined as an unsigned long in clk API, and so
-		 * casting explicitly to its type. Must be fixed once rate is 64
-		 * bit guaranteed in clk API.
-		 */
-		new_opp->rate = (unsigned long)rate;
+	ret = _read_opp_key(new_opp, opp_table, np, &rate_not_available);
+	if (ret < 0 && !opp_table->is_genpd) {
+		dev_err(dev, "%s: opp key field not found\n", __func__);
+		goto free_opp;
 	}
 
-	of_property_read_u32(np, "opp-level", &new_opp->level);
-
 	/* Check if the OPP supports hardware's hierarchy of versions or not */
 	if (!_opp_is_supported(dev, opp_table, np)) {
 		dev_dbg(dev, "OPP not supported by hardware: %llu\n", rate);
diff --git a/drivers/opp/opp.h b/drivers/opp/opp.h
index d14e27102730..e51646ff279e 100644
--- a/drivers/opp/opp.h
+++ b/drivers/opp/opp.h
@@ -12,6 +12,7 @@
 #define __DRIVER_OPP_H__
 
 #include <linux/device.h>
+#include <linux/interconnect.h>
 #include <linux/kernel.h>
 #include <linux/kref.h>
 #include <linux/list.h>
@@ -59,6 +60,7 @@ extern struct list_head opp_tables;
  * @rate:	Frequency in hertz
  * @level:	Performance level
  * @supplies:	Power supplies voltage/current values
+ * @bandwidth:	Interconnect bandwidth values
  * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's
  *		frequency from any other OPP's frequency.
  * @required_opps: List of OPPs that are required by this OPP.
@@ -81,6 +83,7 @@ struct dev_pm_opp {
 	unsigned int level;
 
 	struct dev_pm_opp_supply *supplies;
+	struct dev_pm_opp_icc_bw *bandwidth;
 
 	unsigned long clock_latency_ns;
 
@@ -144,8 +147,11 @@ enum opp_table_access {
  * @clk: Device's clock handle
  * @regulators: Supply regulators
  * @regulator_count: Number of power supply regulators. Its value can be -1
+ * @regulator_enabled: Set to true if regulators were previously enabled.
  * (uninitialized), 0 (no opp-microvolt property) or > 0 (has opp-microvolt
  * property).
+ * @paths: Interconnect path handles
+ * @path_count: Number of interconnect paths
  * @genpd_performance_state: Device's power domain support performance state.
  * @is_genpd: Marks if the OPP table belongs to a genpd.
  * @set_opp: Platform specific set_opp callback
@@ -189,6 +195,9 @@ struct opp_table {
 	struct clk *clk;
 	struct regulator **regulators;
 	int regulator_count;
+	bool regulator_enabled;
+	struct icc_path **paths;
+	unsigned int path_count;
 	bool genpd_performance_state;
 	bool is_genpd;
 
@@ -211,6 +220,7 @@ struct opp_device *_add_opp_dev(const struct device *dev, struct opp_table *opp_
 void _dev_pm_opp_find_and_remove_table(struct device *dev);
 struct dev_pm_opp *_opp_allocate(struct opp_table *opp_table);
 void _opp_free(struct dev_pm_opp *opp);
+int _opp_compare_key(struct dev_pm_opp *opp1, struct dev_pm_opp *opp2);
 int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, struct opp_table *opp_table, bool rate_not_available);
 int _opp_add_v1(struct opp_table *opp_table, struct device *dev, unsigned long freq, long u_volt, bool dynamic);
 void _dev_pm_opp_cpumask_remove_table(const struct cpumask *cpumask, int last_cpu);
diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c
index ac27f3d3fbb4..4d7695289eda 100644
--- a/drivers/oprofile/buffer_sync.c
+++ b/drivers/oprofile/buffer_sync.c
@@ -91,11 +91,11 @@ munmap_notify(struct notifier_block *self, unsigned long val, void *data)
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *mpnt;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 
 	mpnt = find_vma(mm, addr);
 	if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) {
-		up_read(&mm->mmap_sem);
+		mmap_read_unlock(mm);
 		/* To avoid latency problems, we only process the current CPU,
 		 * hoping that most samples for the task are on this CPU
 		 */
@@ -103,7 +103,7 @@ munmap_notify(struct notifier_block *self, unsigned long val, void *data)
 		return 0;
 	}
 
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	return 0;
 }
 
@@ -256,7 +256,7 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset)
 	unsigned long cookie = NO_COOKIE;
 	struct vm_area_struct *vma;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
 
 		if (addr < vma->vm_start || addr >= vma->vm_end)
@@ -276,7 +276,7 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset)
 
 	if (!vma)
 		cookie = INVALID_COOKIE;
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 
 	return cookie;
 }
@@ -486,7 +486,7 @@ typedef enum {
 
 /* Sync one of the CPU's buffers into the global event buffer.
  * Here we need to go through each batch of samples punctuated
- * by context switch notes, taking the task's mmap_sem and doing
+ * by context switch notes, taking the task's mmap_lock and doing
  * lookup in task->mm->mmap to convert EIP into dcookie/offset
  * value.
  */
diff --git a/drivers/oprofile/cpu_buffer.h b/drivers/oprofile/cpu_buffer.h
index e1d097e250ae..31478c0cff87 100644
--- a/drivers/oprofile/cpu_buffer.h
+++ b/drivers/oprofile/cpu_buffer.h
@@ -33,7 +33,7 @@ void flush_cpu_work(void);
 struct op_sample {
 	unsigned long eip;
 	unsigned long event;
-	unsigned long data[0];
+	unsigned long data[];
 };
 
 struct op_entry;
diff --git a/drivers/parport/Kconfig b/drivers/parport/Kconfig
index 1791830e7a71..e78a9f0302c7 100644
--- a/drivers/parport/Kconfig
+++ b/drivers/parport/Kconfig
@@ -15,7 +15,7 @@ config ARCH_MIGHT_HAVE_PC_PARPORT
 menuconfig PARPORT
 	tristate "Parallel port support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  If you want to use devices connected to your machine's parallel port
 	  (the connector at the computer with 25 holes), e.g. printer, ZIP
 	  drive, PLIP link (Parallel Line Internet Protocol is mainly used to
diff --git a/drivers/parport/daisy.c b/drivers/parport/daisy.c
index 3b00e2c8e2e9..6d78ec3a762f 100644
--- a/drivers/parport/daisy.c
+++ b/drivers/parport/daisy.c
@@ -30,12 +30,6 @@
 
 #undef DEBUG
 
-#ifdef DEBUG
-#define DPRINTK(stuff...) printk(stuff)
-#else
-#define DPRINTK(stuff...)
-#endif
-
 static struct daisydev {
 	struct daisydev *next;
 	struct parport *port;
@@ -145,8 +139,7 @@ again:
 	    ((num_ports = num_mux_ports(port)) == 2 || num_ports == 4)) {
 		/* Leave original as port zero. */
 		port->muxport = 0;
-		printk(KERN_INFO
-			"%s: 1st (default) port of %d-way multiplexor\n",
+		pr_info("%s: 1st (default) port of %d-way multiplexor\n",
 			port->name, num_ports);
 		for (i = 1; i < num_ports; i++) {
 			/* Clone the port. */
@@ -159,8 +152,7 @@ again:
 				continue;
 			}
 
-			printk(KERN_INFO
-				"%s: %d%s port of %d-way multiplexor on %s\n",
+			pr_info("%s: %d%s port of %d-way multiplexor on %s\n",
 				extra->name, i + 1, th[i + 1], num_ports,
 				port->name);
 
@@ -323,8 +315,7 @@ static int cpp_daisy(struct parport *port, int cmd)
 		  | PARPORT_STATUS_PAPEROUT
 		  | PARPORT_STATUS_SELECT
 		  | PARPORT_STATUS_ERROR)) {
-		DPRINTK(KERN_DEBUG "%s: cpp_daisy: aa5500ff(%02x)\n",
-			 port->name, s);
+		pr_debug("%s: cpp_daisy: aa5500ff(%02x)\n", port->name, s);
 		return -ENXIO;
 	}
 
@@ -334,8 +325,7 @@ static int cpp_daisy(struct parport *port, int cmd)
 					  | PARPORT_STATUS_SELECT
 					  | PARPORT_STATUS_ERROR);
 	if (s != (PARPORT_STATUS_SELECT | PARPORT_STATUS_ERROR)) {
-		DPRINTK(KERN_DEBUG "%s: cpp_daisy: aa5500ff87(%02x)\n",
-			 port->name, s);
+		pr_debug("%s: cpp_daisy: aa5500ff87(%02x)\n", port->name, s);
 		return -ENXIO;
 	}
 
@@ -370,7 +360,7 @@ static int cpp_mux(struct parport *port, int cmd)
 
 	s = parport_read_status(port);
 	if (!(s & PARPORT_STATUS_ACK)) {
-		DPRINTK(KERN_DEBUG "%s: cpp_mux: aa55f00f52ad%02x(%02x)\n",
+		pr_debug("%s: cpp_mux: aa55f00f52ad%02x(%02x)\n",
 			 port->name, cmd, s);
 		return -EIO;
 	}
@@ -456,8 +446,7 @@ static int assign_addrs(struct parport *port)
 		  | PARPORT_STATUS_PAPEROUT
 		  | PARPORT_STATUS_SELECT
 		  | PARPORT_STATUS_ERROR)) {
-		DPRINTK(KERN_DEBUG "%s: assign_addrs: aa5500ff(%02x)\n",
-			 port->name, s);
+		pr_debug("%s: assign_addrs: aa5500ff(%02x)\n", port->name, s);
 		return 0;
 	}
 
@@ -467,8 +456,7 @@ static int assign_addrs(struct parport *port)
 					  | PARPORT_STATUS_SELECT
 					  | PARPORT_STATUS_ERROR);
 	if (s != (PARPORT_STATUS_SELECT | PARPORT_STATUS_ERROR)) {
-		DPRINTK(KERN_DEBUG "%s: assign_addrs: aa5500ff87(%02x)\n",
-			 port->name, s);
+		pr_debug("%s: assign_addrs: aa5500ff87(%02x)\n", port->name, s);
 		return 0;
 	}
 
@@ -505,8 +493,7 @@ static int assign_addrs(struct parport *port)
 
 	parport_write_data(port, 0xff); udelay(2);
 	detected = numdevs - thisdev;
-	DPRINTK(KERN_DEBUG "%s: Found %d daisy-chained devices\n", port->name,
-		 detected);
+	pr_debug("%s: Found %d daisy-chained devices\n", port->name, detected);
 
 	/* Ask the new devices to introduce themselves. */
 	deviceid = kmalloc(1024, GFP_KERNEL);
diff --git a/drivers/parport/ieee1284.c b/drivers/parport/ieee1284.c
index 90fb73575495..f28d6a3c5a68 100644
--- a/drivers/parport/ieee1284.c
+++ b/drivers/parport/ieee1284.c
@@ -31,12 +31,6 @@
 #undef DEBUG /* Don't want a garbled console */
 #endif
 
-#ifdef DEBUG
-#define DPRINTK(stuff...) printk (stuff)
-#else
-#define DPRINTK(stuff...)
-#endif
-
 /* Make parport_wait_peripheral wake up.
  * It will be useful to call this from an interrupt handler. */
 static void parport_ieee1284_wakeup (struct parport *port)
@@ -258,12 +252,11 @@ static void parport_ieee1284_terminate (struct parport *port)
 						     PARPORT_STATUS_PAPEROUT,
 						     PARPORT_STATUS_PAPEROUT);
 			if (r)
-				DPRINTK (KERN_INFO "%s: Timeout at event 49\n",
+				pr_debug("%s: Timeout at event 49\n",
 					 port->name);
 
 			parport_data_forward (port);
-			DPRINTK (KERN_DEBUG "%s: ECP direction: forward\n",
-				 port->name);
+			pr_debug("%s: ECP direction: forward\n", port->name);
 			port->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
 		}
 
@@ -281,8 +274,7 @@ static void parport_ieee1284_terminate (struct parport *port)
 		/* Event 24: nAck goes low */
 		r = parport_wait_peripheral (port, PARPORT_STATUS_ACK, 0);
 		if (r)
-			DPRINTK (KERN_INFO "%s: Timeout at event 24\n",
-				 port->name);
+			pr_debug("%s: Timeout at event 24\n", port->name);
 
 		/* Event 25: Set nAutoFd low */
 		parport_frob_control (port,
@@ -294,8 +286,7 @@ static void parport_ieee1284_terminate (struct parport *port)
 					     PARPORT_STATUS_ACK, 
 					     PARPORT_STATUS_ACK);
 		if (r)
-			DPRINTK (KERN_INFO "%s: Timeout at event 27\n",
-				 port->name);
+			pr_debug("%s: Timeout at event 27\n", port->name);
 
 		/* Event 29: Set nAutoFd high */
 		parport_frob_control (port, PARPORT_CONTROL_AUTOFD, 0);
@@ -304,8 +295,7 @@ static void parport_ieee1284_terminate (struct parport *port)
 	port->ieee1284.mode = IEEE1284_MODE_COMPAT;
 	port->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
 
-	DPRINTK (KERN_DEBUG "%s: In compatibility (forward idle) mode\n",
-		 port->name);
+	pr_debug("%s: In compatibility (forward idle) mode\n", port->name);
 }		
 #endif /* IEEE1284 support */
 
@@ -329,7 +319,7 @@ int parport_negotiate (struct parport *port, int mode)
 #ifndef CONFIG_PARPORT_1284
 	if (mode == IEEE1284_MODE_COMPAT)
 		return 0;
-	printk (KERN_ERR "parport: IEEE1284 not supported in this kernel\n");
+	pr_err("parport: IEEE1284 not supported in this kernel\n");
 	return -1;
 #else
 	int m = mode & ~IEEE1284_ADDR;
@@ -406,8 +396,7 @@ int parport_negotiate (struct parport *port, int mode)
 				      PARPORT_CONTROL_SELECT
 				      | PARPORT_CONTROL_AUTOFD,
 				      PARPORT_CONTROL_SELECT);
-		DPRINTK (KERN_DEBUG
-			 "%s: Peripheral not IEEE1284 compliant (0x%02X)\n",
+		pr_debug("%s: Peripheral not IEEE1284 compliant (0x%02X)\n",
 			 port->name, parport_read_status (port));
 		port->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
 		return -1; /* Not IEEE1284 compliant */
@@ -430,8 +419,7 @@ int parport_negotiate (struct parport *port, int mode)
 				     PARPORT_STATUS_ACK,
 				     PARPORT_STATUS_ACK)) {
 		/* This shouldn't really happen with a compliant device. */
-		DPRINTK (KERN_DEBUG
-			 "%s: Mode 0x%02x not supported? (0x%02x)\n",
+		pr_debug("%s: Mode 0x%02x not supported? (0x%02x)\n",
 			 port->name, mode, port->ops->read_status (port));
 		parport_ieee1284_terminate (port);
 		return 1;
@@ -442,7 +430,7 @@ int parport_negotiate (struct parport *port, int mode)
 	/* xflag should be high for all modes other than nibble (0). */
 	if (mode && !xflag) {
 		/* Mode not supported. */
-		DPRINTK (KERN_DEBUG "%s: Mode 0x%02x rejected by peripheral\n",
+		pr_debug("%s: Mode 0x%02x rejected by peripheral\n",
 			 port->name, mode);
 		parport_ieee1284_terminate (port);
 		return 1;
@@ -463,9 +451,7 @@ int parport_negotiate (struct parport *port, int mode)
 		/* Event 52: nAck goes low */
 		if (parport_wait_peripheral (port, PARPORT_STATUS_ACK, 0)) {
 			/* This peripheral is _very_ slow. */
-			DPRINTK (KERN_DEBUG
-				 "%s: Event 52 didn't happen\n",
-				 port->name);
+			pr_debug("%s: Event 52 didn't happen\n", port->name);
 			parport_ieee1284_terminate (port);
 			return 1;
 		}
@@ -481,10 +467,9 @@ int parport_negotiate (struct parport *port, int mode)
 					     PARPORT_STATUS_ACK)) {
 			/* This shouldn't really happen with a compliant
 			 * device. */
-			DPRINTK (KERN_DEBUG
-				 "%s: Mode 0x%02x not supported? (0x%02x)\n",
+			pr_debug("%s: Mode 0x%02x not supported? (0x%02x)\n",
 				 port->name, mode,
-				 port->ops->read_status (port));
+				 port->ops->read_status(port));
 			parport_ieee1284_terminate (port);
 			return 1;
 		}
@@ -495,8 +480,8 @@ int parport_negotiate (struct parport *port, int mode)
 		/* xflag should be high. */
 		if (!xflag) {
 			/* Extended mode not supported. */
-			DPRINTK (KERN_DEBUG "%s: Extended mode 0x%02x not "
-				 "supported\n", port->name, mode);
+			pr_debug("%s: Extended mode 0x%02x not supported\n",
+				 port->name, mode);
 			parport_ieee1284_terminate (port);
 			return 1;
 		}
@@ -505,7 +490,7 @@ int parport_negotiate (struct parport *port, int mode)
 	}
 
 	/* Mode is supported */
-	DPRINTK (KERN_DEBUG "%s: In mode 0x%02x\n", port->name, mode);
+	pr_debug("%s: In mode 0x%02x\n", port->name, mode);
 	port->ieee1284.mode = mode;
 
 	/* But ECP is special */
@@ -522,13 +507,11 @@ int parport_negotiate (struct parport *port, int mode)
 					     PARPORT_STATUS_PAPEROUT,
 					     PARPORT_STATUS_PAPEROUT);
 		if (r) {
-			DPRINTK (KERN_INFO "%s: Timeout at event 31\n",
-				port->name);
+			pr_debug("%s: Timeout at event 31\n", port->name);
 		}
 
 		port->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
-		DPRINTK (KERN_DEBUG "%s: ECP direction: forward\n",
-			 port->name);
+		pr_debug("%s: ECP direction: forward\n", port->name);
 	} else switch (mode) {
 	case IEEE1284_MODE_NIBBLE:
 	case IEEE1284_MODE_BYTE:
@@ -573,7 +556,7 @@ void parport_ieee1284_interrupt (void *handle)
 	if (port->ieee1284.phase == IEEE1284_PH_REV_IDLE) {
 		/* An interrupt in this phase means that data
 		 * is now available. */
-		DPRINTK (KERN_DEBUG "%s: Data available\n", port->name);
+		pr_debug("%s: Data available\n", port->name);
 		parport_ieee1284_ack_data_avail (port);
 	}
 #endif /* IEEE1284 support */
@@ -617,13 +600,12 @@ ssize_t parport_write (struct parport *port, const void *buffer, size_t len)
 		parport_negotiate (port, IEEE1284_MODE_COMPAT);
 		/* fall through */
 	case IEEE1284_MODE_COMPAT:
-		DPRINTK (KERN_DEBUG "%s: Using compatibility mode\n",
-			 port->name);
+		pr_debug("%s: Using compatibility mode\n", port->name);
 		fn = port->ops->compat_write_data;
 		break;
 
 	case IEEE1284_MODE_EPP:
-		DPRINTK (KERN_DEBUG "%s: Using EPP mode\n", port->name);
+		pr_debug("%s: Using EPP mode\n", port->name);
 		if (addr) {
 			fn = port->ops->epp_write_addr;
 		} else {
@@ -631,8 +613,7 @@ ssize_t parport_write (struct parport *port, const void *buffer, size_t len)
 		}
 		break;
 	case IEEE1284_MODE_EPPSWE:
-		DPRINTK (KERN_DEBUG "%s: Using software-emulated EPP mode\n",
-			port->name);
+		pr_debug("%s: Using software-emulated EPP mode\n", port->name);
 		if (addr) {
 			fn = parport_ieee1284_epp_write_addr;
 		} else {
@@ -641,7 +622,7 @@ ssize_t parport_write (struct parport *port, const void *buffer, size_t len)
 		break;
 	case IEEE1284_MODE_ECP:
 	case IEEE1284_MODE_ECPRLE:
-		DPRINTK (KERN_DEBUG "%s: Using ECP mode\n", port->name);
+		pr_debug("%s: Using ECP mode\n", port->name);
 		if (addr) {
 			fn = port->ops->ecp_write_addr;
 		} else {
@@ -650,8 +631,7 @@ ssize_t parport_write (struct parport *port, const void *buffer, size_t len)
 		break;
 
 	case IEEE1284_MODE_ECPSWE:
-		DPRINTK (KERN_DEBUG "%s: Using software-emulated ECP mode\n",
-			 port->name);
+		pr_debug("%s: Using software-emulated ECP mode\n", port->name);
 		/* The caller has specified that it must be emulated,
 		 * even if we have ECP hardware! */
 		if (addr) {
@@ -662,13 +642,13 @@ ssize_t parport_write (struct parport *port, const void *buffer, size_t len)
 		break;
 
 	default:
-		DPRINTK (KERN_DEBUG "%s: Unknown mode 0x%02x\n", port->name,
-			port->ieee1284.mode);
+		pr_debug("%s: Unknown mode 0x%02x\n",
+			 port->name, port->ieee1284.mode);
 		return -ENOSYS;
 	}
 
 	retval = (*fn) (port, buffer, len, 0);
-	DPRINTK (KERN_DEBUG "%s: wrote %d/%d bytes\n", port->name, retval, len);
+	pr_debug("%s: wrote %zd/%zu bytes\n", port->name, retval, len);
 	return retval;
 #endif /* IEEE1284 support */
 }
@@ -694,7 +674,7 @@ ssize_t parport_write (struct parport *port, const void *buffer, size_t len)
 ssize_t parport_read (struct parport *port, void *buffer, size_t len)
 {
 #ifndef CONFIG_PARPORT_1284
-	printk (KERN_ERR "parport: IEEE1284 not supported in this kernel\n");
+	pr_err("parport: IEEE1284 not supported in this kernel\n");
 	return -ENODEV;
 #else
 	int mode = port->physport->ieee1284.mode;
@@ -715,7 +695,7 @@ ssize_t parport_read (struct parport *port, void *buffer, size_t len)
 		if ((port->physport->modes & PARPORT_MODE_TRISTATE) &&
 		    !parport_negotiate (port, IEEE1284_MODE_BYTE)) {
 			/* got into BYTE mode OK */
-			DPRINTK (KERN_DEBUG "%s: Using byte mode\n", port->name);
+			pr_debug("%s: Using byte mode\n", port->name);
 			fn = port->ops->byte_read_data;
 			break;
 		}
@@ -724,17 +704,17 @@ ssize_t parport_read (struct parport *port, void *buffer, size_t len)
 		}
 		/* fall through - to NIBBLE */
 	case IEEE1284_MODE_NIBBLE:
-		DPRINTK (KERN_DEBUG "%s: Using nibble mode\n", port->name);
+		pr_debug("%s: Using nibble mode\n", port->name);
 		fn = port->ops->nibble_read_data;
 		break;
 
 	case IEEE1284_MODE_BYTE:
-		DPRINTK (KERN_DEBUG "%s: Using byte mode\n", port->name);
+		pr_debug("%s: Using byte mode\n", port->name);
 		fn = port->ops->byte_read_data;
 		break;
 
 	case IEEE1284_MODE_EPP:
-		DPRINTK (KERN_DEBUG "%s: Using EPP mode\n", port->name);
+		pr_debug("%s: Using EPP mode\n", port->name);
 		if (addr) {
 			fn = port->ops->epp_read_addr;
 		} else {
@@ -742,8 +722,7 @@ ssize_t parport_read (struct parport *port, void *buffer, size_t len)
 		}
 		break;
 	case IEEE1284_MODE_EPPSWE:
-		DPRINTK (KERN_DEBUG "%s: Using software-emulated EPP mode\n",
-			port->name);
+		pr_debug("%s: Using software-emulated EPP mode\n", port->name);
 		if (addr) {
 			fn = parport_ieee1284_epp_read_addr;
 		} else {
@@ -752,19 +731,18 @@ ssize_t parport_read (struct parport *port, void *buffer, size_t len)
 		break;
 	case IEEE1284_MODE_ECP:
 	case IEEE1284_MODE_ECPRLE:
-		DPRINTK (KERN_DEBUG "%s: Using ECP mode\n", port->name);
+		pr_debug("%s: Using ECP mode\n", port->name);
 		fn = port->ops->ecp_read_data;
 		break;
 
 	case IEEE1284_MODE_ECPSWE:
-		DPRINTK (KERN_DEBUG "%s: Using software-emulated ECP mode\n",
-			 port->name);
+		pr_debug("%s: Using software-emulated ECP mode\n", port->name);
 		fn = parport_ieee1284_ecp_read_data;
 		break;
 
 	default:
-		DPRINTK (KERN_DEBUG "%s: Unknown mode 0x%02x\n", port->name,
-			 port->physport->ieee1284.mode);
+		pr_debug("%s: Unknown mode 0x%02x\n",
+			 port->name, port->physport->ieee1284.mode);
 		return -ENOSYS;
 	}
 
diff --git a/drivers/parport/ieee1284_ops.c b/drivers/parport/ieee1284_ops.c
index 5d41dda6da4e..2c11bd3fe1fd 100644
--- a/drivers/parport/ieee1284_ops.c
+++ b/drivers/parport/ieee1284_ops.c
@@ -27,12 +27,6 @@
 #undef DEBUG /* Don't want a garbled console */
 #endif
 
-#ifdef DEBUG
-#define DPRINTK(stuff...) printk (stuff)
-#else
-#define DPRINTK(stuff...)
-#endif
-
 /***                                *
  * One-way data transfer functions. *
  *                                ***/
@@ -115,7 +109,7 @@ size_t parport_ieee1284_write_compat (struct parport *port,
 		if (signal_pending (current))
 			break;
 
-		DPRINTK (KERN_DEBUG "%s: Timed out\n", port->name);
+		pr_debug("%s: Timed out\n", port->name);
 		break;
 
 	ready:
@@ -178,9 +172,8 @@ size_t parport_ieee1284_read_nibble (struct parport *port,
 		if (parport_wait_peripheral (port,
 					     PARPORT_STATUS_ACK, 0)) {
 			/* Timeout -- no more data? */
-			DPRINTK (KERN_DEBUG
-				 "%s: Nibble timeout at event 9 (%d bytes)\n",
-				 port->name, i/2);
+			pr_debug("%s: Nibble timeout at event 9 (%d bytes)\n",
+				 port->name, i / 2);
 			parport_frob_control (port, PARPORT_CONTROL_AUTOFD, 0);
 			break;
 		}
@@ -201,8 +194,7 @@ size_t parport_ieee1284_read_nibble (struct parport *port,
 					     PARPORT_STATUS_ACK,
 					     PARPORT_STATUS_ACK)) {
 			/* Timeout -- no more data? */
-			DPRINTK (KERN_DEBUG
-				 "%s: Nibble timeout at event 11\n",
+			pr_debug("%s: Nibble timeout at event 11\n",
 				 port->name);
 			break;
 		}
@@ -219,9 +211,8 @@ size_t parport_ieee1284_read_nibble (struct parport *port,
 		/* Read the last nibble without checking data avail. */
 		if (parport_read_status (port) & PARPORT_STATUS_ERROR) {
 		end_of_data:
-			DPRINTK (KERN_DEBUG
-				"%s: No more nibble data (%d bytes)\n",
-				port->name, i/2);
+			pr_debug("%s: No more nibble data (%d bytes)\n",
+				 port->name, i / 2);
 
 			/* Go to reverse idle phase. */
 			parport_frob_control (port,
@@ -272,8 +263,7 @@ size_t parport_ieee1284_read_byte (struct parport *port,
 			/* Timeout -- no more data? */
 			parport_frob_control (port, PARPORT_CONTROL_AUTOFD,
 						 0);
-			DPRINTK (KERN_DEBUG "%s: Byte timeout at event 9\n",
-				 port->name);
+			pr_debug("%s: Byte timeout at event 9\n", port->name);
 			break;
 		}
 
@@ -288,8 +278,7 @@ size_t parport_ieee1284_read_byte (struct parport *port,
 					     PARPORT_STATUS_ACK,
 					     PARPORT_STATUS_ACK)) {
 			/* Timeout -- no more data? */
-			DPRINTK (KERN_DEBUG "%s: Byte timeout at event 11\n",
-				 port->name);
+			pr_debug("%s: Byte timeout at event 11\n", port->name);
 			break;
 		}
 
@@ -307,8 +296,7 @@ size_t parport_ieee1284_read_byte (struct parport *port,
 		/* Read the last byte without checking data avail. */
 		if (parport_read_status (port) & PARPORT_STATUS_ERROR) {
 		end_of_data:
-			DPRINTK (KERN_DEBUG
-				 "%s: No more byte data (%zd bytes)\n",
+			pr_debug("%s: No more byte data (%zd bytes)\n",
 				 port->name, count);
 
 			/* Go to reverse idle phase. */
@@ -353,12 +341,10 @@ int ecp_forward_to_reverse (struct parport *port)
 					  PARPORT_STATUS_PAPEROUT, 0);
 
 	if (!retval) {
-		DPRINTK (KERN_DEBUG "%s: ECP direction: reverse\n",
-			 port->name);
+		pr_debug("%s: ECP direction: reverse\n", port->name);
 		port->ieee1284.phase = IEEE1284_PH_REV_IDLE;
 	} else {
-		DPRINTK (KERN_DEBUG "%s: ECP direction: failed to reverse\n",
-			 port->name);
+		pr_debug("%s: ECP direction: failed to reverse\n", port->name);
 		port->ieee1284.phase = IEEE1284_PH_ECP_DIR_UNKNOWN;
 	}
 
@@ -384,12 +370,10 @@ int ecp_reverse_to_forward (struct parport *port)
 
 	if (!retval) {
 		parport_data_forward (port);
-		DPRINTK (KERN_DEBUG "%s: ECP direction: forward\n",
-			 port->name);
+		pr_debug("%s: ECP direction: forward\n", port->name);
 		port->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
 	} else {
-		DPRINTK (KERN_DEBUG
-			 "%s: ECP direction: failed to switch forward\n",
+		pr_debug("%s: ECP direction: failed to switch forward\n",
 			 port->name);
 		port->ieee1284.phase = IEEE1284_PH_ECP_DIR_UNKNOWN;
 	}
@@ -450,7 +434,7 @@ size_t parport_ieee1284_ecp_write_data (struct parport *port,
 		}
 
 		/* Time for Host Transfer Recovery (page 41 of IEEE1284) */
-		DPRINTK (KERN_DEBUG "%s: ECP transfer stalled!\n", port->name);
+		pr_debug("%s: ECP transfer stalled!\n", port->name);
 
 		parport_frob_control (port, PARPORT_CONTROL_INIT,
 				      PARPORT_CONTROL_INIT);
@@ -466,8 +450,7 @@ size_t parport_ieee1284_ecp_write_data (struct parport *port,
 		if (!(parport_read_status (port) & PARPORT_STATUS_PAPEROUT))
 			break;
 
-		DPRINTK (KERN_DEBUG "%s: Host transfer recovered\n",
-			 port->name);
+		pr_debug("%s: Host transfer recovered\n", port->name);
 
 		if (time_after_eq (jiffies, expire)) break;
 		goto try_again;
@@ -565,23 +548,20 @@ size_t parport_ieee1284_ecp_read_data (struct parport *port,
                    command or a normal data byte, don't accept it. */
 		if (command) {
 			if (byte & 0x80) {
-				DPRINTK (KERN_DEBUG "%s: stopping short at "
-					 "channel command (%02x)\n",
+				pr_debug("%s: stopping short at channel command (%02x)\n",
 					 port->name, byte);
 				goto out;
 			}
 			else if (port->ieee1284.mode != IEEE1284_MODE_ECPRLE)
-				DPRINTK (KERN_DEBUG "%s: device illegally "
-					 "using RLE; accepting anyway\n",
+				pr_debug("%s: device illegally using RLE; accepting anyway\n",
 					 port->name);
 
 			rle_count = byte + 1;
 
 			/* Are we allowed to read that many bytes? */
 			if (rle_count > (len - count)) {
-				DPRINTK (KERN_DEBUG "%s: leaving %d RLE bytes "
-					 "for next time\n", port->name,
-					 rle_count);
+				pr_debug("%s: leaving %d RLE bytes for next time\n",
+					 port->name, rle_count);
 				break;
 			}
 
@@ -596,11 +576,10 @@ size_t parport_ieee1284_ecp_read_data (struct parport *port,
 					     PARPORT_STATUS_ACK)) {
 			/* It's gone wrong.  Return what data we have
                            to the caller. */
-			DPRINTK (KERN_DEBUG "ECP read timed out at 45\n");
+			pr_debug("ECP read timed out at 45\n");
 
 			if (command)
-				printk (KERN_WARNING
-					"%s: command ignored (%02x)\n",
+				pr_warn("%s: command ignored (%02x)\n",
 					port->name, byte);
 
 			break;
@@ -620,7 +599,7 @@ size_t parport_ieee1284_ecp_read_data (struct parport *port,
 			memset (buf, byte, rle_count);
 			buf += rle_count;
 			count += rle_count;
-			DPRINTK (KERN_DEBUG "%s: decompressed to %d bytes\n",
+			pr_debug("%s: decompressed to %d bytes\n",
 				 port->name, rle_count);
 		} else {
 			/* Normal data byte. */
@@ -686,7 +665,7 @@ size_t parport_ieee1284_ecp_write_addr (struct parport *port,
 		}
 
 		/* Time for Host Transfer Recovery (page 41 of IEEE1284) */
-		DPRINTK (KERN_DEBUG "%s: ECP transfer stalled!\n", port->name);
+		pr_debug("%s: ECP transfer stalled!\n", port->name);
 
 		parport_frob_control (port, PARPORT_CONTROL_INIT,
 				      PARPORT_CONTROL_INIT);
@@ -702,8 +681,7 @@ size_t parport_ieee1284_ecp_write_addr (struct parport *port,
 		if (!(parport_read_status (port) & PARPORT_STATUS_PAPEROUT))
 			break;
 
-		DPRINTK (KERN_DEBUG "%s: Host transfer recovered\n",
-			 port->name);
+		pr_debug("%s: Host transfer recovered\n", port->name);
 
 		if (time_after_eq (jiffies, expire)) break;
 		goto try_again;
diff --git a/drivers/parport/parport_amiga.c b/drivers/parport/parport_amiga.c
index 3301861f69fa..1e88bcfe0d7b 100644
--- a/drivers/parport/parport_amiga.c
+++ b/drivers/parport/parport_amiga.c
@@ -28,16 +28,10 @@
 #include <asm/amigaints.h>
 
 #undef DEBUG
-#ifdef DEBUG
-#define DPRINTK printk
-#else
-#define DPRINTK(x...)	do { } while (0)
-#endif
-
 
 static void amiga_write_data(struct parport *p, unsigned char data)
 {
-	DPRINTK(KERN_DEBUG "write_data %c\n",data);
+	pr_debug("write_data %c\n", data);
 	/* Triggers also /STROBE. This behavior cannot be changed */
 	ciaa.prb = data;
 	mb();
@@ -59,13 +53,13 @@ static unsigned char control_amiga_to_pc(unsigned char control)
 
 static void amiga_write_control(struct parport *p, unsigned char control)
 {
-	DPRINTK(KERN_DEBUG "write_control %02x\n",control);
+	pr_debug("write_control %02x\n", control);
 	/* No implementation possible */
 }
 	
 static unsigned char amiga_read_control( struct parport *p)
 {
-	DPRINTK(KERN_DEBUG "read_control \n");
+	pr_debug("read_control\n");
 	return control_amiga_to_pc(0);
 }
 
@@ -73,7 +67,7 @@ static unsigned char amiga_frob_control( struct parport *p, unsigned char mask,
 {
 	unsigned char old;
 
-	DPRINTK(KERN_DEBUG "frob_control mask %02x, value %02x\n",mask,val);
+	pr_debug("frob_control mask %02x, value %02x\n", mask, val);
 	old = amiga_read_control(p);
 	amiga_write_control(p, (old & ~mask) ^ val);
 	return old;
@@ -99,7 +93,7 @@ static unsigned char amiga_read_status(struct parport *p)
 	unsigned char status;
 
 	status = status_amiga_to_pc(ciab.pra & 7);
-	DPRINTK(KERN_DEBUG "read_status %02x\n", status);
+	pr_debug("read_status %02x\n", status);
 	return status;
 }
 
@@ -115,14 +109,14 @@ static void amiga_disable_irq(struct parport *p)
 
 static void amiga_data_forward(struct parport *p)
 {
-	DPRINTK(KERN_DEBUG "forward\n");
+	pr_debug("forward\n");
 	ciaa.ddrb = 0xff; /* all pins output */
 	mb();
 }
 
 static void amiga_data_reverse(struct parport *p)
 {
-	DPRINTK(KERN_DEBUG "reverse\n");
+	pr_debug("reverse\n");
 	ciaa.ddrb = 0; /* all pins input */
 	mb();
 }
@@ -212,7 +206,7 @@ static int __init amiga_parallel_probe(struct platform_device *pdev)
 	if (err)
 		goto out_irq;
 
-	printk(KERN_INFO "%s: Amiga built-in port using irq\n", p->name);
+	pr_info("%s: Amiga built-in port using irq\n", p->name);
 	/* XXX: set operating mode */
 	parport_announce_port(p);
 
diff --git a/drivers/parport/parport_atari.c b/drivers/parport/parport_atari.c
index f8dd368bfdbb..2ff0fe053e6e 100644
--- a/drivers/parport/parport_atari.c
+++ b/drivers/parport/parport_atari.c
@@ -200,7 +200,7 @@ static int __init parport_atari_init(void)
 		}
 
 		this_port = p;
-		printk(KERN_INFO "%s: Atari built-in port using irq\n", p->name);
+		pr_info("%s: Atari built-in port using irq\n", p->name);
 		parport_announce_port (p);
 
 		return 0;
diff --git a/drivers/parport/parport_cs.c b/drivers/parport/parport_cs.c
index e77044c2bf62..8e7e3ac4bb87 100644
--- a/drivers/parport/parport_cs.c
+++ b/drivers/parport/parport_cs.c
@@ -142,10 +142,8 @@ static int parport_config(struct pcmcia_device *link)
 			      link->irq, PARPORT_DMA_NONE,
 			      &link->dev, IRQF_SHARED);
     if (p == NULL) {
-	printk(KERN_NOTICE "parport_cs: parport_pc_probe_port() at "
-	       "0x%3x, irq %u failed\n",
-	       (unsigned int) link->resource[0]->start,
-	       link->irq);
+	    pr_notice("parport_cs: parport_pc_probe_port() at 0x%3x, irq %u failed\n",
+		      (unsigned int)link->resource[0]->start, link->irq);
 	goto failed;
     }
 
diff --git a/drivers/parport/parport_gsc.c b/drivers/parport/parport_gsc.c
index 922535a118ba..9228e8f90309 100644
--- a/drivers/parport/parport_gsc.c
+++ b/drivers/parport/parport_gsc.c
@@ -238,14 +238,14 @@ struct parport *parport_gsc_probe_port(unsigned long base,
 
 	priv = kzalloc (sizeof (struct parport_gsc_private), GFP_KERNEL);
 	if (!priv) {
-		printk (KERN_DEBUG "parport (0x%lx): no memory!\n", base);
+		printk(KERN_DEBUG "parport (0x%lx): no memory!\n", base);
 		return NULL;
 	}
 	ops = kmemdup(&parport_gsc_ops, sizeof(struct parport_operations),
 		      GFP_KERNEL);
 	if (!ops) {
-		printk (KERN_DEBUG "parport (0x%lx): no memory for ops!\n",
-			base);
+		printk(KERN_DEBUG "parport (0x%lx): no memory for ops!\n",
+		       base);
 		kfree (priv);
 		return NULL;
 	}
@@ -282,7 +282,7 @@ struct parport *parport_gsc_probe_port(unsigned long base,
 	p->size = (p->modes & PARPORT_MODE_EPP)?8:3;
 	p->private_data = priv;
 
-	printk(KERN_INFO "%s: PC-style at 0x%lx", p->name, p->base);
+	pr_info("%s: PC-style at 0x%lx", p->name, p->base);
 	p->irq = irq;
 	if (p->irq == PARPORT_IRQ_AUTO) {
 		p->irq = PARPORT_IRQ_NONE;
@@ -299,12 +299,16 @@ struct parport *parport_gsc_probe_port(unsigned long base,
 		p->dma = PARPORT_DMA_NONE;
 
 	pr_cont(" [");
-#define printmode(x) {if(p->modes&PARPORT_MODE_##x){pr_cont("%s%s",f?",":"",#x);f++;}}
+#define printmode(x)							\
+do {									\
+	if (p->modes & PARPORT_MODE_##x)				\
+		pr_cont("%s%s", f++ ? "," : "", #x);			\
+} while (0)
 	{
 		int f = 0;
 		printmode(PCSPP);
 		printmode(TRISTATE);
-		printmode(COMPAT)
+		printmode(COMPAT);
 		printmode(EPP);
 //		printmode(ECP);
 //		printmode(DMA);
@@ -315,8 +319,7 @@ struct parport *parport_gsc_probe_port(unsigned long base,
 	if (p->irq != PARPORT_IRQ_NONE) {
 		if (request_irq (p->irq, parport_irq_handler,
 				 0, p->name, p)) {
-			printk (KERN_WARNING "%s: irq %d in use, "
-				"resorting to polled operation\n",
+			pr_warn("%s: irq %d in use, resorting to polled operation\n",
 				p->name, p->irq);
 			p->irq = PARPORT_IRQ_NONE;
 			p->dma = PARPORT_DMA_NONE;
@@ -347,7 +350,7 @@ static int __init parport_init_chip(struct parisc_device *dev)
 	unsigned long port;
 
 	if (!dev->irq) {
-		printk(KERN_WARNING "IRQ not found for parallel device at 0x%llx\n",
+		pr_warn("IRQ not found for parallel device at 0x%llx\n",
 			(unsigned long long)dev->hpa.start);
 		return -ENODEV;
 	}
@@ -360,11 +363,11 @@ static int __init parport_init_chip(struct parisc_device *dev)
 	if (boot_cpu_data.cpu_type > pcxt && !pdc_add_valid(port+4)) {
 
 		/* Initialize bidirectional-mode (0x10) & data-tranfer-mode #1 (0x20) */
-		printk("%s: initialize bidirectional-mode.\n", __func__);
+		pr_info("%s: initialize bidirectional-mode\n", __func__);
 		parport_writeb ( (0x10 + 0x20), port + 4);
 
 	} else {
-		printk("%s: enhanced parport-modes not supported.\n", __func__);
+		pr_info("%s: enhanced parport-modes not supported\n", __func__);
 	}
 	
 	p = parport_gsc_probe_port(port, 0, dev->irq,
diff --git a/drivers/parport/parport_gsc.h b/drivers/parport/parport_gsc.h
index 4c4d3c6cd77e..9301217edf12 100644
--- a/drivers/parport/parport_gsc.h
+++ b/drivers/parport/parport_gsc.h
@@ -71,7 +71,7 @@ struct parport_gsc_private {
 static inline void parport_gsc_write_data(struct parport *p, unsigned char d)
 {
 #ifdef DEBUG_PARPORT
-	printk (KERN_DEBUG "parport_gsc_write_data(%p,0x%02x)\n", p, d);
+	printk(KERN_DEBUG "%s(%p,0x%02x)\n", __func__, p, d);
 #endif
 	parport_writeb(d, DATA(p));
 }
@@ -80,8 +80,7 @@ static inline unsigned char parport_gsc_read_data(struct parport *p)
 {
 	unsigned char val = parport_readb (DATA (p));
 #ifdef DEBUG_PARPORT
-	printk (KERN_DEBUG "parport_gsc_read_data(%p) = 0x%02x\n",
-		p, val);
+	printk(KERN_DEBUG "%s(%p) = 0x%02x\n", __func__, p, val);
 #endif
 	return val;
 }
@@ -95,9 +94,9 @@ static inline unsigned char __parport_gsc_frob_control(struct parport *p,
 	struct parport_gsc_private *priv = p->physport->private_data;
 	unsigned char ctr = priv->ctr;
 #ifdef DEBUG_PARPORT
-	printk (KERN_DEBUG
-		"__parport_gsc_frob_control(%02x,%02x): %02x -> %02x\n",
-		mask, val, ctr, ((ctr & ~mask) ^ val) & priv->ctr_writable);
+	printk(KERN_DEBUG "%s(%02x,%02x): %02x -> %02x\n",
+	       __func__, mask, val,
+	       ctr, ((ctr & ~mask) ^ val) & priv->ctr_writable);
 #endif
 	ctr = (ctr & ~mask) ^ val;
 	ctr &= priv->ctr_writable; /* only write writable bits. */
@@ -126,8 +125,8 @@ static inline void parport_gsc_write_control(struct parport *p,
 
 	/* Take this out when drivers have adapted to newer interface. */
 	if (d & 0x20) {
-		printk (KERN_DEBUG "%s (%s): use data_reverse for this!\n",
-			p->name, p->cad->name);
+		printk(KERN_DEBUG "%s (%s): use data_reverse for this!\n",
+		       p->name, p->cad->name);
 		parport_gsc_data_reverse (p);
 	}
 
@@ -155,9 +154,9 @@ static inline unsigned char parport_gsc_frob_control(struct parport *p,
 
 	/* Take this out when drivers have adapted to newer interface. */
 	if (mask & 0x20) {
-		printk (KERN_DEBUG "%s (%s): use data_%s for this!\n",
-			p->name, p->cad->name,
-			(val & 0x20) ? "reverse" : "forward");
+		printk(KERN_DEBUG "%s (%s): use data_%s for this!\n",
+		       p->name, p->cad->name,
+		       (val & 0x20) ? "reverse" : "forward");
 		if (val & 0x20)
 			parport_gsc_data_reverse (p);
 		else
diff --git a/drivers/parport/parport_ip32.c b/drivers/parport/parport_ip32.c
index ab215b650f41..48b084e86dc6 100644
--- a/drivers/parport/parport_ip32.c
+++ b/drivers/parport/parport_ip32.c
@@ -328,19 +328,19 @@ static void parport_ip32_dump_state(struct parport *p, char *str,
 						     "TST", "CFG"};
 		unsigned int ecr = readb(priv->regs.ecr);
 		printk(KERN_DEBUG PPIP32 "    ecr=0x%02x", ecr);
-		printk(" %s",
-		       ecr_modes[(ecr & ECR_MODE_MASK) >> ECR_MODE_SHIFT]);
+		pr_cont(" %s",
+			ecr_modes[(ecr & ECR_MODE_MASK) >> ECR_MODE_SHIFT]);
 		if (ecr & ECR_nERRINTR)
-			printk(",nErrIntrEn");
+			pr_cont(",nErrIntrEn");
 		if (ecr & ECR_DMAEN)
-			printk(",dmaEn");
+			pr_cont(",dmaEn");
 		if (ecr & ECR_SERVINTR)
-			printk(",serviceIntr");
+			pr_cont(",serviceIntr");
 		if (ecr & ECR_F_FULL)
-			printk(",f_full");
+			pr_cont(",f_full");
 		if (ecr & ECR_F_EMPTY)
-			printk(",f_empty");
-		printk("\n");
+			pr_cont(",f_empty");
+		pr_cont("\n");
 	}
 	if (show_ecp_config) {
 		unsigned int oecr, cnfgA, cnfgB;
@@ -352,52 +352,53 @@ static void parport_ip32_dump_state(struct parport *p, char *str,
 		writeb(ECR_MODE_PS2, priv->regs.ecr);
 		writeb(oecr, priv->regs.ecr);
 		printk(KERN_DEBUG PPIP32 "    cnfgA=0x%02x", cnfgA);
-		printk(" ISA-%s", (cnfgA & CNFGA_IRQ) ? "Level" : "Pulses");
+		pr_cont(" ISA-%s", (cnfgA & CNFGA_IRQ) ? "Level" : "Pulses");
 		switch (cnfgA & CNFGA_ID_MASK) {
 		case CNFGA_ID_8:
-			printk(",8 bits");
+			pr_cont(",8 bits");
 			break;
 		case CNFGA_ID_16:
-			printk(",16 bits");
+			pr_cont(",16 bits");
 			break;
 		case CNFGA_ID_32:
-			printk(",32 bits");
+			pr_cont(",32 bits");
 			break;
 		default:
-			printk(",unknown ID");
+			pr_cont(",unknown ID");
 			break;
 		}
 		if (!(cnfgA & CNFGA_nBYTEINTRANS))
-			printk(",ByteInTrans");
+			pr_cont(",ByteInTrans");
 		if ((cnfgA & CNFGA_ID_MASK) != CNFGA_ID_8)
-			printk(",%d byte%s left", cnfgA & CNFGA_PWORDLEFT,
-			       ((cnfgA & CNFGA_PWORDLEFT) > 1) ? "s" : "");
-		printk("\n");
+			pr_cont(",%d byte%s left",
+				cnfgA & CNFGA_PWORDLEFT,
+				((cnfgA & CNFGA_PWORDLEFT) > 1) ? "s" : "");
+		pr_cont("\n");
 		printk(KERN_DEBUG PPIP32 "    cnfgB=0x%02x", cnfgB);
-		printk(" irq=%u,dma=%u",
-		       (cnfgB & CNFGB_IRQ_MASK) >> CNFGB_IRQ_SHIFT,
-		       (cnfgB & CNFGB_DMA_MASK) >> CNFGB_DMA_SHIFT);
-		printk(",intrValue=%d", !!(cnfgB & CNFGB_INTRVAL));
+		pr_cont(" irq=%u,dma=%u",
+			(cnfgB & CNFGB_IRQ_MASK) >> CNFGB_IRQ_SHIFT,
+			(cnfgB & CNFGB_DMA_MASK) >> CNFGB_DMA_SHIFT);
+		pr_cont(",intrValue=%d", !!(cnfgB & CNFGB_INTRVAL));
 		if (cnfgB & CNFGB_COMPRESS)
-			printk(",compress");
-		printk("\n");
+			pr_cont(",compress");
+		pr_cont("\n");
 	}
 	for (i = 0; i < 2; i++) {
 		unsigned int dcr = i ? priv->dcr_cache : readb(priv->regs.dcr);
 		printk(KERN_DEBUG PPIP32 "    dcr(%s)=0x%02x",
 		       i ? "soft" : "hard", dcr);
-		printk(" %s", (dcr & DCR_DIR) ? "rev" : "fwd");
+		pr_cont(" %s", (dcr & DCR_DIR) ? "rev" : "fwd");
 		if (dcr & DCR_IRQ)
-			printk(",ackIntEn");
+			pr_cont(",ackIntEn");
 		if (!(dcr & DCR_SELECT))
-			printk(",nSelectIn");
+			pr_cont(",nSelectIn");
 		if (dcr & DCR_nINIT)
-			printk(",nInit");
+			pr_cont(",nInit");
 		if (!(dcr & DCR_AUTOFD))
-			printk(",nAutoFD");
+			pr_cont(",nAutoFD");
 		if (!(dcr & DCR_STROBE))
-			printk(",nStrobe");
-		printk("\n");
+			pr_cont(",nStrobe");
+		pr_cont("\n");
 	}
 #define sep (f++ ? ',' : ' ')
 	{
@@ -405,20 +406,20 @@ static void parport_ip32_dump_state(struct parport *p, char *str,
 		unsigned int dsr = readb(priv->regs.dsr);
 		printk(KERN_DEBUG PPIP32 "    dsr=0x%02x", dsr);
 		if (!(dsr & DSR_nBUSY))
-			printk("%cBusy", sep);
+			pr_cont("%cBusy", sep);
 		if (dsr & DSR_nACK)
-			printk("%cnAck", sep);
+			pr_cont("%cnAck", sep);
 		if (dsr & DSR_PERROR)
-			printk("%cPError", sep);
+			pr_cont("%cPError", sep);
 		if (dsr & DSR_SELECT)
-			printk("%cSelect", sep);
+			pr_cont("%cSelect", sep);
 		if (dsr & DSR_nFAULT)
-			printk("%cnFault", sep);
+			pr_cont("%cnFault", sep);
 		if (!(dsr & DSR_nPRINT))
-			printk("%c(Print)", sep);
+			pr_cont("%c(Print)", sep);
 		if (dsr & DSR_TIMEOUT)
-			printk("%cTimeout", sep);
-		printk("\n");
+			pr_cont("%cTimeout", sep);
+		pr_cont("\n");
 	}
 #undef sep
 }
@@ -1337,9 +1338,8 @@ static unsigned int parport_ip32_fwp_wait_interrupt(struct parport *p)
 			ecr = parport_ip32_read_econtrol(p);
 			if ((ecr & ECR_F_EMPTY) && !(ecr & ECR_SERVINTR)
 			    && !lost_interrupt) {
-				printk(KERN_WARNING PPIP32
-				       "%s: lost interrupt in %s\n",
-				       p->name, __func__);
+				pr_warn(PPIP32 "%s: lost interrupt in %s\n",
+					p->name, __func__);
 				lost_interrupt = 1;
 			}
 		}
@@ -1643,8 +1643,8 @@ static size_t parport_ip32_compat_write_data(struct parport *p,
 				       DSR_nBUSY | DSR_nFAULT)) {
 		/* Avoid to flood the logs */
 		if (ready_before)
-			printk(KERN_INFO PPIP32 "%s: not ready in %s\n",
-			       p->name, __func__);
+			pr_info(PPIP32 "%s: not ready in %s\n",
+				p->name, __func__);
 		ready_before = 0;
 		goto stop;
 	}
@@ -1704,7 +1704,7 @@ static size_t parport_ip32_ecp_write_data(struct parport *p,
 
 		/* Event 49: PError goes high. */
 		if (parport_wait_peripheral(p, DSR_PERROR, DSR_PERROR)) {
-			printk(KERN_DEBUG PPIP32 "%s: PError timeout in %s",
+			printk(KERN_DEBUG PPIP32 "%s: PError timeout in %s\n",
 			       p->name, __func__);
 			physport->ieee1284.phase = IEEE1284_PH_ECP_DIR_UNKNOWN;
 			return 0;
@@ -1724,8 +1724,8 @@ static size_t parport_ip32_ecp_write_data(struct parport *p,
 				       DSR_nBUSY | DSR_nFAULT)) {
 		/* Avoid to flood the logs */
 		if (ready_before)
-			printk(KERN_INFO PPIP32 "%s: not ready in %s\n",
-			       p->name, __func__);
+			pr_info(PPIP32 "%s: not ready in %s\n",
+				p->name, __func__);
 		ready_before = 0;
 		goto stop;
 	}
@@ -2064,8 +2064,7 @@ static __init struct parport *parport_ip32_probe_port(void)
 	p->modes |= PARPORT_MODE_TRISTATE;
 
 	if (!parport_ip32_fifo_supported(p)) {
-		printk(KERN_WARNING PPIP32
-		       "%s: error: FIFO disabled\n", p->name);
+		pr_warn(PPIP32 "%s: error: FIFO disabled\n", p->name);
 		/* Disable hardware modes depending on a working FIFO. */
 		features &= ~PARPORT_IP32_ENABLE_SPP;
 		features &= ~PARPORT_IP32_ENABLE_ECP;
@@ -2077,8 +2076,7 @@ static __init struct parport *parport_ip32_probe_port(void)
 	if (features & PARPORT_IP32_ENABLE_IRQ) {
 		int irq = MACEISA_PARALLEL_IRQ;
 		if (request_irq(irq, parport_ip32_interrupt, 0, p->name, p)) {
-			printk(KERN_WARNING PPIP32
-			       "%s: error: IRQ disabled\n", p->name);
+			pr_warn(PPIP32 "%s: error: IRQ disabled\n", p->name);
 			/* DMA cannot work without interrupts. */
 			features &= ~PARPORT_IP32_ENABLE_DMA;
 		} else {
@@ -2091,8 +2089,7 @@ static __init struct parport *parport_ip32_probe_port(void)
 	/* Allocate DMA resources */
 	if (features & PARPORT_IP32_ENABLE_DMA) {
 		if (parport_ip32_dma_register())
-			printk(KERN_WARNING PPIP32
-			       "%s: error: DMA disabled\n", p->name);
+			pr_warn(PPIP32 "%s: error: DMA disabled\n", p->name);
 		else {
 			pr_probe(p, "DMA support enabled\n");
 			p->dma = 0; /* arbitrary value != PARPORT_DMA_NONE */
@@ -2134,13 +2131,15 @@ static __init struct parport *parport_ip32_probe_port(void)
 	parport_ip32_dump_state(p, "end init", 0);
 
 	/* Print out what we found */
-	printk(KERN_INFO "%s: SGI IP32 at 0x%lx (0x%lx)",
-	       p->name, p->base, p->base_hi);
+	pr_info("%s: SGI IP32 at 0x%lx (0x%lx)", p->name, p->base, p->base_hi);
 	if (p->irq != PARPORT_IRQ_NONE)
-		printk(", irq %d", p->irq);
-	printk(" [");
-#define printmode(x)	if (p->modes & PARPORT_MODE_##x)		\
-				printk("%s%s", f++ ? "," : "", #x)
+		pr_cont(", irq %d", p->irq);
+	pr_cont(" [");
+#define printmode(x)							\
+do {									\
+	if (p->modes & PARPORT_MODE_##x)				\
+		pr_cont("%s%s", f++ ? "," : "", #x);			\
+} while (0)
 	{
 		unsigned int f = 0;
 		printmode(PCSPP);
@@ -2151,7 +2150,7 @@ static __init struct parport *parport_ip32_probe_port(void)
 		printmode(DMA);
 	}
 #undef printmode
-	printk("]\n");
+	pr_cont("]\n");
 
 	parport_announce_port(p);
 	return p;
diff --git a/drivers/parport/parport_mfc3.c b/drivers/parport/parport_mfc3.c
index 9f87faf939e3..d6bbe8446301 100644
--- a/drivers/parport/parport_mfc3.c
+++ b/drivers/parport/parport_mfc3.c
@@ -70,11 +70,6 @@
 #define MAX_MFC 5
 
 #undef DEBUG
-#ifdef DEBUG
-#define DPRINTK printk
-#else
-static inline int DPRINTK(void *nothing, ...) {return 0;}
-#endif
 
 static struct parport *this_port[MAX_MFC] = {NULL, };
 static volatile int dummy; /* for trigger readds */
@@ -84,7 +79,7 @@ static struct parport_operations pp_mfc3_ops;
 
 static void mfc3_write_data(struct parport *p, unsigned char data)
 {
-DPRINTK(KERN_DEBUG "write_data %c\n",data);
+	pr_debug("write_data %c\n", data);
 
 	dummy = pia(p)->pprb; /* clears irq bit */
 	/* Triggers also /STROBE.*/
@@ -128,13 +123,13 @@ static unsigned char control_mfc3_to_pc(unsigned char control)
 
 static void mfc3_write_control(struct parport *p, unsigned char control)
 {
-DPRINTK(KERN_DEBUG "write_control %02x\n",control);
+	pr_debug("write_control %02x\n", control);
 	pia(p)->ppra = (pia(p)->ppra & 0x1f) | control_pc_to_mfc3(control);
 }
 	
 static unsigned char mfc3_read_control( struct parport *p)
 {
-DPRINTK(KERN_DEBUG "read_control \n");
+	pr_debug("read_control\n");
 	return control_mfc3_to_pc(pia(p)->ppra & 0xe0);
 }
 
@@ -142,7 +137,7 @@ static unsigned char mfc3_frob_control( struct parport *p, unsigned char mask, u
 {
 	unsigned char old;
 
-DPRINTK(KERN_DEBUG "frob_control mask %02x, value %02x\n",mask,val);
+	pr_debug("frob_control mask %02x, value %02x\n", mask, val);
 	old = mfc3_read_control(p);
 	mfc3_write_control(p, (old & ~mask) ^ val);
 	return old;
@@ -171,7 +166,7 @@ static unsigned char mfc3_read_status(struct parport *p)
 	unsigned char status;
 
 	status = status_mfc3_to_pc(pia(p)->ppra & 0x1f);
-DPRINTK(KERN_DEBUG "read_status %02x\n", status);
+	pr_debug("read_status %02x\n", status);
 	return status;
 }
 
@@ -202,7 +197,7 @@ static void mfc3_disable_irq(struct parport *p)
 
 static void mfc3_data_forward(struct parport *p)
 {
-	DPRINTK(KERN_DEBUG "forward\n");
+	pr_debug("forward\n");
 	pia(p)->crb &= ~PIA_DDR; /* make data direction register visible */
 	pia(p)->pddrb = 255; /* all pins output */
 	pia(p)->crb |= PIA_DDR; /* make data register visible - default */
@@ -210,7 +205,7 @@ static void mfc3_data_forward(struct parport *p)
 
 static void mfc3_data_reverse(struct parport *p)
 {
-	DPRINTK(KERN_DEBUG "reverse\n");
+	pr_debug("reverse\n");
 	pia(p)->crb &= ~PIA_DDR; /* make data direction register visible */
 	pia(p)->pddrb = 0; /* all pins input */
 	pia(p)->crb |= PIA_DDR; /* make data register visible - default */
@@ -325,7 +320,7 @@ static int __init parport_mfc3_init(void)
 		p->dev = &z->dev;
 
 		this_port[pias++] = p;
-		printk(KERN_INFO "%s: Multiface III port using irq\n", p->name);
+		pr_info("%s: Multiface III port using irq\n", p->name);
 		/* XXX: set operating mode */
 
 		p->private_data = (void *)piabase;
diff --git a/drivers/parport/parport_pc.c b/drivers/parport/parport_pc.c
index 1f17a39eabe8..77e37e3cb3a0 100644
--- a/drivers/parport/parport_pc.c
+++ b/drivers/parport/parport_pc.c
@@ -87,13 +87,6 @@
 
 #undef DEBUG
 
-#ifdef DEBUG
-#define DPRINTK  printk
-#else
-#define DPRINTK(stuff...)
-#endif
-
-
 #define NR_SUPERIOS 3
 static struct superio_struct {	/* For Super-IO chips autodetection */
 	int io;
@@ -118,8 +111,8 @@ static void frob_econtrol(struct parport *pb, unsigned char m,
 	if (m != 0xff)
 		ectr = inb(ECONTROL(pb));
 
-	DPRINTK(KERN_DEBUG "frob_econtrol(%02x,%02x): %02x -> %02x\n",
-		m, v, ectr, (ectr & ~m) ^ v);
+	pr_debug("frob_econtrol(%02x,%02x): %02x -> %02x\n",
+		 m, v, ectr, (ectr & ~m) ^ v);
 
 	outb((ectr & ~m) ^ v, ECONTROL(pb));
 }
@@ -142,7 +135,7 @@ static int change_mode(struct parport *p, int m)
 	unsigned char oecr;
 	int mode;
 
-	DPRINTK(KERN_INFO "parport change_mode ECP-ISA to mode 0x%02x\n", m);
+	pr_debug("parport change_mode ECP-ISA to mode 0x%02x\n", m);
 
 	if (!priv->ecr) {
 		printk(KERN_DEBUG "change_mode: but there's no ECR!\n");
@@ -298,8 +291,8 @@ static size_t parport_pc_epp_read_data(struct parport *port, void *buf,
 			status = inb(STATUS(port));
 			if (status & 0x01) {
 				/* EPP timeout should never occur... */
-				printk(KERN_DEBUG
-"%s: EPP timeout occurred while talking to w91284pic (should not have done)\n", port->name);
+				printk(KERN_DEBUG "%s: EPP timeout occurred while talking to w91284pic (should not have done)\n",
+				       port->name);
 				clear_epp_timeout(port);
 			}
 		}
@@ -727,7 +720,7 @@ static size_t parport_pc_compat_write_block_pio(struct parport *port,
 	r = change_mode(port, ECR_PPF); /* Parallel port FIFO */
 	if (r)
 		printk(KERN_DEBUG "%s: Warning change_mode ECR_PPF failed\n",
-								port->name);
+		       port->name);
 
 	port->physport->ieee1284.phase = IEEE1284_PH_FWD_DATA;
 
@@ -770,9 +763,8 @@ static size_t parport_pc_compat_write_block_pio(struct parport *port,
 				     PARPORT_STATUS_BUSY,
 				     PARPORT_STATUS_BUSY);
 	if (r)
-		printk(KERN_DEBUG
-			"%s: BUSY timeout (%d) in compat_write_block_pio\n",
-			port->name, r);
+		printk(KERN_DEBUG "%s: BUSY timeout (%d) in compat_write_block_pio\n",
+		       port->name, r);
 
 	port->physport->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
 
@@ -810,8 +802,8 @@ static size_t parport_pc_ecp_write_block_pio(struct parport *port,
 					     PARPORT_STATUS_PAPEROUT,
 					     PARPORT_STATUS_PAPEROUT);
 		if (r) {
-			printk(KERN_DEBUG "%s: PError timeout (%d) "
-				"in ecp_write_block_pio\n", port->name, r);
+			printk(KERN_DEBUG "%s: PError timeout (%d) in ecp_write_block_pio\n",
+			       port->name, r);
 		}
 	}
 
@@ -824,7 +816,7 @@ static size_t parport_pc_ecp_write_block_pio(struct parport *port,
 	r = change_mode(port, ECR_ECP); /* ECP FIFO */
 	if (r)
 		printk(KERN_DEBUG "%s: Warning change_mode ECR_ECP failed\n",
-								port->name);
+		       port->name);
 	port->physport->ieee1284.phase = IEEE1284_PH_FWD_DATA;
 
 	/* Write the data to the FIFO. */
@@ -867,8 +859,8 @@ static size_t parport_pc_ecp_write_block_pio(struct parport *port,
 		parport_frob_control(port, PARPORT_CONTROL_INIT, 0);
 		r = parport_wait_peripheral(port, PARPORT_STATUS_PAPEROUT, 0);
 		if (r)
-			printk(KERN_DEBUG "%s: PE,1 timeout (%d) "
-				"in ecp_write_block_pio\n", port->name, r);
+			printk(KERN_DEBUG "%s: PE,1 timeout (%d) in ecp_write_block_pio\n",
+			       port->name, r);
 
 		parport_frob_control(port,
 				      PARPORT_CONTROL_INIT,
@@ -877,17 +869,16 @@ static size_t parport_pc_ecp_write_block_pio(struct parport *port,
 					     PARPORT_STATUS_PAPEROUT,
 					     PARPORT_STATUS_PAPEROUT);
 		if (r)
-			printk(KERN_DEBUG "%s: PE,2 timeout (%d) "
-				"in ecp_write_block_pio\n", port->name, r);
+			printk(KERN_DEBUG "%s: PE,2 timeout (%d) in ecp_write_block_pio\n",
+			       port->name, r);
 	}
 
 	r = parport_wait_peripheral(port,
 				     PARPORT_STATUS_BUSY,
 				     PARPORT_STATUS_BUSY);
 	if (r)
-		printk(KERN_DEBUG
-			"%s: BUSY timeout (%d) in ecp_write_block_pio\n",
-			port->name, r);
+		printk(KERN_DEBUG "%s: BUSY timeout (%d) in ecp_write_block_pio\n",
+		       port->name, r);
 
 	port->physport->ieee1284.phase = IEEE1284_PH_FWD_IDLE;
 
@@ -982,28 +973,24 @@ static void show_parconfig_smsc37c669(int io, int key)
 	outb(0xaa, io);
 
 	if (verbose_probing) {
-		printk(KERN_INFO
-			"SMSC 37c669 LPT Config: cr_1=0x%02x, 4=0x%02x, "
-			"A=0x%2x, 23=0x%02x, 26=0x%02x, 27=0x%02x\n",
+		pr_info("SMSC 37c669 LPT Config: cr_1=0x%02x, 4=0x%02x, A=0x%2x, 23=0x%02x, 26=0x%02x, 27=0x%02x\n",
 			cr1, cr4, cra, cr23, cr26, cr27);
 
 		/* The documentation calls DMA and IRQ-Lines by letters, so
 		   the board maker can/will wire them
 		   appropriately/randomly...  G=reserved H=IDE-irq, */
-		printk(KERN_INFO
-	"SMSC LPT Config: io=0x%04x, irq=%c, dma=%c, fifo threshold=%d\n",
-				cr23 * 4,
-				(cr27 & 0x0f) ? 'A' - 1 + (cr27 & 0x0f) : '-',
-				(cr26 & 0x0f) ? 'A' - 1 + (cr26 & 0x0f) : '-',
-				cra & 0x0f);
-		printk(KERN_INFO "SMSC LPT Config: enabled=%s power=%s\n",
-		       (cr23 * 4 >= 0x100) ? "yes" : "no",
-		       (cr1 & 4) ? "yes" : "no");
-		printk(KERN_INFO
-			"SMSC LPT Config: Port mode=%s, EPP version =%s\n",
-				(cr1 & 0x08) ? "Standard mode only (SPP)"
-					      : modes[cr4 & 0x03],
-				(cr4 & 0x40) ? "1.7" : "1.9");
+		pr_info("SMSC LPT Config: io=0x%04x, irq=%c, dma=%c, fifo threshold=%d\n",
+			cr23 * 4,
+			(cr27 & 0x0f) ? 'A' - 1 + (cr27 & 0x0f) : '-',
+			(cr26 & 0x0f) ? 'A' - 1 + (cr26 & 0x0f) : '-',
+			cra & 0x0f);
+		pr_info("SMSC LPT Config: enabled=%s power=%s\n",
+			(cr23 * 4 >= 0x100) ? "yes" : "no",
+			(cr1 & 4) ? "yes" : "no");
+		pr_info("SMSC LPT Config: Port mode=%s, EPP version =%s\n",
+			(cr1 & 0x08) ? "Standard mode only (SPP)"
+			: modes[cr4 & 0x03],
+			(cr4 & 0x40) ? "1.7" : "1.9");
 	}
 
 	/* Heuristics !  BIOS setup for this mainboard device limits
@@ -1013,7 +1000,7 @@ static void show_parconfig_smsc37c669(int io, int key)
 	if (cr23 * 4 >= 0x100) { /* if active */
 		s = find_free_superio();
 		if (s == NULL)
-			printk(KERN_INFO "Super-IO: too many chips!\n");
+			pr_info("Super-IO: too many chips!\n");
 		else {
 			int d;
 			switch (cr23 * 4) {
@@ -1078,26 +1065,24 @@ static void show_parconfig_winbond(int io, int key)
 	outb(0xaa, io);
 
 	if (verbose_probing) {
-		printk(KERN_INFO
-    "Winbond LPT Config: cr_30=%02x 60,61=%02x%02x 70=%02x 74=%02x, f0=%02x\n",
-					cr30, cr60, cr61, cr70, cr74, crf0);
-		printk(KERN_INFO "Winbond LPT Config: active=%s, io=0x%02x%02x irq=%d, ",
-		       (cr30 & 0x01) ? "yes" : "no", cr60, cr61, cr70 & 0x0f);
+		pr_info("Winbond LPT Config: cr_30=%02x 60,61=%02x%02x 70=%02x 74=%02x, f0=%02x\n",
+			cr30, cr60, cr61, cr70, cr74, crf0);
+		pr_info("Winbond LPT Config: active=%s, io=0x%02x%02x irq=%d, ",
+			(cr30 & 0x01) ? "yes" : "no", cr60, cr61, cr70 & 0x0f);
 		if ((cr74 & 0x07) > 3)
 			pr_cont("dma=none\n");
 		else
 			pr_cont("dma=%d\n", cr74 & 0x07);
-		printk(KERN_INFO
-		    "Winbond LPT Config: irqtype=%s, ECP fifo threshold=%d\n",
-					irqtypes[crf0>>7], (crf0>>3)&0x0f);
-		printk(KERN_INFO "Winbond LPT Config: Port mode=%s\n",
-					modes[crf0 & 0x07]);
+		pr_info("Winbond LPT Config: irqtype=%s, ECP fifo threshold=%d\n",
+			irqtypes[crf0 >> 7], (crf0 >> 3) & 0x0f);
+		pr_info("Winbond LPT Config: Port mode=%s\n",
+			modes[crf0 & 0x07]);
 	}
 
 	if (cr30 & 0x01) { /* the settings can be interrogated later ... */
 		s = find_free_superio();
 		if (s == NULL)
-			printk(KERN_INFO "Super-IO: too many chips!\n");
+			pr_info("Super-IO: too many chips!\n");
 		else {
 			s->io = (cr60 << 8) | cr61;
 			s->irq = cr70 & 0x0f;
@@ -1151,9 +1136,8 @@ static void decode_winbond(int efer, int key, int devid, int devrev, int oldid)
 		progif = 0;
 
 	if (verbose_probing)
-		printk(KERN_INFO "Winbond chip at EFER=0x%x key=0x%02x "
-		       "devid=%02x devrev=%02x oldid=%02x type=%s\n",
-		       efer, key, devid, devrev, oldid, type);
+		pr_info("Winbond chip at EFER=0x%x key=0x%02x devid=%02x devrev=%02x oldid=%02x type=%s\n",
+			efer, key, devid, devrev, oldid, type);
 
 	if (progif == 2)
 		show_parconfig_winbond(efer, key);
@@ -1184,9 +1168,8 @@ static void decode_smsc(int efer, int key, int devid, int devrev)
 		type = "37c666GT";
 
 	if (verbose_probing)
-		printk(KERN_INFO "SMSC chip at EFER=0x%x "
-		       "key=0x%02x devid=%02x devrev=%02x type=%s\n",
-		       efer, key, devid, devrev, type);
+		pr_info("SMSC chip at EFER=0x%x key=0x%02x devid=%02x devrev=%02x type=%s\n",
+			efer, key, devid, devrev, type);
 
 	if (func)
 		func(efer, key);
@@ -1358,7 +1341,7 @@ static void detect_and_report_it87(void)
 	dev |= inb(0x2f);
 	if (dev == 0x8712 || dev == 0x8705 || dev == 0x8715 ||
 	    dev == 0x8716 || dev == 0x8718 || dev == 0x8726) {
-		printk(KERN_INFO "IT%04X SuperIO detected.\n", dev);
+		pr_info("IT%04X SuperIO detected\n", dev);
 		outb(0x07, 0x2E);	/* Parallel Port */
 		outb(0x03, 0x2F);
 		outb(0xF0, 0x2E);	/* BOOT 0x80 off */
@@ -1445,8 +1428,8 @@ static int parport_SPP_supported(struct parport *pb)
 	if (user_specified)
 		/* That didn't work, but the user thinks there's a
 		 * port here. */
-		printk(KERN_INFO "parport 0x%lx (WARNING): CTR: "
-			"wrote 0x%02x, read 0x%02x\n", pb->base, w, r);
+		pr_info("parport 0x%lx (WARNING): CTR: wrote 0x%02x, read 0x%02x\n",
+			pb->base, w, r);
 
 	/* Try the data register.  The data lines aren't tri-stated at
 	 * this stage, so we expect back what we wrote. */
@@ -1464,10 +1447,9 @@ static int parport_SPP_supported(struct parport *pb)
 	if (user_specified) {
 		/* Didn't work, but the user is convinced this is the
 		 * place. */
-		printk(KERN_INFO "parport 0x%lx (WARNING): DATA: "
-			"wrote 0x%02x, read 0x%02x\n", pb->base, w, r);
-		printk(KERN_INFO "parport 0x%lx: You gave this address, "
-			"but there is probably no parallel port there!\n",
+		pr_info("parport 0x%lx (WARNING): DATA: wrote 0x%02x, read 0x%02x\n",
+			pb->base, w, r);
+		pr_info("parport 0x%lx: You gave this address, but there is probably no parallel port there!\n",
 			pb->base);
 	}
 
@@ -1620,7 +1602,7 @@ static int parport_ECP_supported(struct parport *pb)
 	if (i <= priv->fifo_depth) {
 		if (verbose_probing)
 			printk(KERN_DEBUG "0x%lx: writeIntrThreshold is %d\n",
-				pb->base, i);
+			       pb->base, i);
 	} else
 		/* Number of bytes we know we can write if we get an
 		   interrupt. */
@@ -1642,7 +1624,7 @@ static int parport_ECP_supported(struct parport *pb)
 
 	if (i <= priv->fifo_depth) {
 		if (verbose_probing)
-			printk(KERN_INFO "0x%lx: readIntrThreshold is %d\n",
+			pr_info("0x%lx: readIntrThreshold is %d\n",
 				pb->base, i);
 	} else
 		/* Number of bytes we can read if we get an interrupt. */
@@ -1657,17 +1639,14 @@ static int parport_ECP_supported(struct parport *pb)
 	switch (pword) {
 	case 0:
 		pword = 2;
-		printk(KERN_WARNING "0x%lx: Unsupported pword size!\n",
-			pb->base);
+		pr_warn("0x%lx: Unsupported pword size!\n", pb->base);
 		break;
 	case 2:
 		pword = 4;
-		printk(KERN_WARNING "0x%lx: Unsupported pword size!\n",
-			pb->base);
+		pr_warn("0x%lx: Unsupported pword size!\n", pb->base);
 		break;
 	default:
-		printk(KERN_WARNING "0x%lx: Unknown implementation ID\n",
-			pb->base);
+		pr_warn("0x%lx: Unknown implementation ID\n", pb->base);
 		/* Fall through - Assume 1 */
 	case 1:
 		pword = 1;
@@ -1676,14 +1655,14 @@ static int parport_ECP_supported(struct parport *pb)
 
 	if (verbose_probing) {
 		printk(KERN_DEBUG "0x%lx: PWord is %d bits\n",
-			pb->base, 8 * pword);
+		       pb->base, 8 * pword);
 
-		printk(KERN_DEBUG "0x%lx: Interrupts are ISA-%s\n", pb->base,
-			config & 0x80 ? "Level" : "Pulses");
+		printk(KERN_DEBUG "0x%lx: Interrupts are ISA-%s\n",
+		       pb->base, config & 0x80 ? "Level" : "Pulses");
 
 		configb = inb(CONFIGB(pb));
 		printk(KERN_DEBUG "0x%lx: ECP port cfgA=0x%02x cfgB=0x%02x\n",
-			pb->base, config, configb);
+		       pb->base, config, configb);
 		printk(KERN_DEBUG "0x%lx: ECP settings irq=", pb->base);
 		if ((configb >> 3) & 0x07)
 			pr_cont("%d", intrline[(configb >> 3) & 0x07]);
@@ -2107,9 +2086,9 @@ struct parport *parport_pc_probe_port(unsigned long int base,
 
 	p->size = (p->modes & PARPORT_MODE_EPP) ? 8 : 3;
 
-	printk(KERN_INFO "%s: PC-style at 0x%lx", p->name, p->base);
+	pr_info("%s: PC-style at 0x%lx", p->name, p->base);
 	if (p->base_hi && priv->ecr)
-		printk(KERN_CONT " (0x%lx)", p->base_hi);
+		pr_cont(" (0x%lx)", p->base_hi);
 	if (p->irq == PARPORT_IRQ_AUTO) {
 		p->irq = PARPORT_IRQ_NONE;
 		parport_irq_probe(p);
@@ -2120,7 +2099,7 @@ struct parport *parport_pc_probe_port(unsigned long int base,
 		p->irq = PARPORT_IRQ_NONE;
 	}
 	if (p->irq != PARPORT_IRQ_NONE) {
-		printk(KERN_CONT ", irq %d", p->irq);
+		pr_cont(", irq %d", p->irq);
 		priv->ctr_writable |= 0x10;
 
 		if (p->dma == PARPORT_DMA_AUTO) {
@@ -2144,41 +2123,39 @@ struct parport *parport_pc_probe_port(unsigned long int base,
 		/* p->ops->ecp_read_data = parport_pc_ecp_read_block_pio; */
 #endif /* IEEE 1284 support */
 		if (p->dma != PARPORT_DMA_NONE) {
-			printk(KERN_CONT ", dma %d", p->dma);
+			pr_cont(", dma %d", p->dma);
 			p->modes |= PARPORT_MODE_DMA;
 		} else
-			printk(KERN_CONT ", using FIFO");
+			pr_cont(", using FIFO");
 	} else
 		/* We can't use the DMA channel after all. */
 		p->dma = PARPORT_DMA_NONE;
 #endif /* Allowed to use FIFO/DMA */
 
-	printk(KERN_CONT " [");
+	pr_cont(" [");
 
-#define printmode(x) \
-	{\
-		if (p->modes & PARPORT_MODE_##x) {\
-			printk(KERN_CONT "%s%s", f ? "," : "", #x);\
-			f++;\
-		} \
-	}
+#define printmode(x)							\
+do {									\
+	if (p->modes & PARPORT_MODE_##x)				\
+		pr_cont("%s%s", f++ ? "," : "", #x);			\
+} while (0)
 
 	{
 		int f = 0;
 		printmode(PCSPP);
 		printmode(TRISTATE);
-		printmode(COMPAT)
+		printmode(COMPAT);
 		printmode(EPP);
 		printmode(ECP);
 		printmode(DMA);
 	}
 #undef printmode
 #ifndef CONFIG_PARPORT_1284
-	printk(KERN_CONT "(,...)");
+	pr_cont("(,...)");
 #endif /* CONFIG_PARPORT_1284 */
-	printk(KERN_CONT "]\n");
+	pr_cont("]\n");
 	if (probedirq != PARPORT_IRQ_NONE)
-		printk(KERN_INFO "%s: irq %d detected\n", p->name, probedirq);
+		pr_info("%s: irq %d detected\n", p->name, probedirq);
 
 	/* If No ECP release the ports grabbed above. */
 	if (ECR_res && (p->modes & PARPORT_MODE_ECP) == 0) {
@@ -2193,8 +2170,7 @@ struct parport *parport_pc_probe_port(unsigned long int base,
 	if (p->irq != PARPORT_IRQ_NONE) {
 		if (request_irq(p->irq, parport_irq_handler,
 				 irqflags, p->name, p)) {
-			printk(KERN_WARNING "%s: irq %d in use, "
-				"resorting to polled operation\n",
+			pr_warn("%s: irq %d in use, resorting to polled operation\n",
 				p->name, p->irq);
 			p->irq = PARPORT_IRQ_NONE;
 			p->dma = PARPORT_DMA_NONE;
@@ -2204,8 +2180,7 @@ struct parport *parport_pc_probe_port(unsigned long int base,
 #ifdef HAS_DMA
 		if (p->dma != PARPORT_DMA_NONE) {
 			if (request_dma(p->dma, p->name)) {
-				printk(KERN_WARNING "%s: dma %d in use, "
-					"resorting to PIO operation\n",
+				pr_warn("%s: dma %d in use, resorting to PIO operation\n",
 					p->name, p->dma);
 				p->dma = PARPORT_DMA_NONE;
 			} else {
@@ -2215,9 +2190,7 @@ struct parport *parport_pc_probe_port(unsigned long int base,
 						       &priv->dma_handle,
 						       GFP_KERNEL);
 				if (!priv->dma_buf) {
-					printk(KERN_WARNING "%s: "
-						"cannot get buffer for DMA, "
-						"resorting to PIO operation\n",
+					pr_warn("%s: cannot get buffer for DMA, resorting to PIO operation\n",
 						p->name);
 					free_dma(p->dma);
 					p->dma = PARPORT_DMA_NONE;
@@ -2313,7 +2286,7 @@ static int sio_ite_8872_probe(struct pci_dev *pdev, int autoirq, int autodma,
 	int irq;
 	int i;
 
-	DPRINTK(KERN_DEBUG "sio_ite_8872_probe()\n");
+	pr_debug("sio_ite_8872_probe()\n");
 
 	/* make sure which one chip */
 	for (i = 0; i < 5; i++) {
@@ -2330,7 +2303,7 @@ static int sio_ite_8872_probe(struct pci_dev *pdev, int autoirq, int autodma,
 		}
 	}
 	if (i >= 5) {
-		printk(KERN_INFO "parport_pc: cannot find ITE8872 INTA\n");
+		pr_info("parport_pc: cannot find ITE8872 INTA\n");
 		return 0;
 	}
 
@@ -2339,29 +2312,28 @@ static int sio_ite_8872_probe(struct pci_dev *pdev, int autoirq, int autodma,
 
 	switch (type) {
 	case 0x2:
-		printk(KERN_INFO "parport_pc: ITE8871 found (1P)\n");
+		pr_info("parport_pc: ITE8871 found (1P)\n");
 		ite8872set = 0x64200000;
 		break;
 	case 0xa:
-		printk(KERN_INFO "parport_pc: ITE8875 found (1P)\n");
+		pr_info("parport_pc: ITE8875 found (1P)\n");
 		ite8872set = 0x64200000;
 		break;
 	case 0xe:
-		printk(KERN_INFO "parport_pc: ITE8872 found (2S1P)\n");
+		pr_info("parport_pc: ITE8872 found (2S1P)\n");
 		ite8872set = 0x64e00000;
 		break;
 	case 0x6:
-		printk(KERN_INFO "parport_pc: ITE8873 found (1S)\n");
+		pr_info("parport_pc: ITE8873 found (1S)\n");
 		release_region(inta_addr[i], 32);
 		return 0;
 	case 0x8:
-		printk(KERN_INFO "parport_pc: ITE8874 found (2S)\n");
+		pr_info("parport_pc: ITE8874 found (2S)\n");
 		release_region(inta_addr[i], 32);
 		return 0;
 	default:
-		printk(KERN_INFO "parport_pc: unknown ITE887x\n");
-		printk(KERN_INFO "parport_pc: please mail 'lspci -nvv' "
-			"output to Rich.Liu@ite.com.tw\n");
+		pr_info("parport_pc: unknown ITE887x\n");
+		pr_info("parport_pc: please mail 'lspci -nvv' output to Rich.Liu@ite.com.tw\n");
 		release_region(inta_addr[i], 32);
 		return 0;
 	}
@@ -2379,11 +2351,9 @@ static int sio_ite_8872_probe(struct pci_dev *pdev, int autoirq, int autodma,
 	pci_write_config_dword(pdev, 0x9c,
 				ite8872set | (ite8872_irq * 0x11111));
 
-	DPRINTK(KERN_DEBUG "ITE887x: The IRQ is %d.\n", ite8872_irq);
-	DPRINTK(KERN_DEBUG "ITE887x: The PARALLEL I/O port is 0x%x.\n",
-		 ite8872_lpt);
-	DPRINTK(KERN_DEBUG "ITE887x: The PARALLEL I/O porthi is 0x%x.\n",
-		 ite8872_lpthi);
+	pr_debug("ITE887x: The IRQ is %d\n", ite8872_irq);
+	pr_debug("ITE887x: The PARALLEL I/O port is 0x%x\n", ite8872_lpt);
+	pr_debug("ITE887x: The PARALLEL I/O porthi is 0x%x\n", ite8872_lpthi);
 
 	/* Let the user (or defaults) steer us away from interrupts */
 	irq = ite8872_irq;
@@ -2396,9 +2366,8 @@ static int sio_ite_8872_probe(struct pci_dev *pdev, int autoirq, int autodma,
 	release_region(inta_addr[i], 32);
 	if (parport_pc_probe_port(ite8872_lpt, ite8872_lpthi,
 				   irq, PARPORT_DMA_NONE, &pdev->dev, 0)) {
-		printk(KERN_INFO
-			"parport_pc: ITE 8872 parallel port: io=0x%X",
-								ite8872_lpt);
+		pr_info("parport_pc: ITE 8872 parallel port: io=0x%X",
+			ite8872_lpt);
 		if (irq != PARPORT_IRQ_NONE)
 			pr_cont(", irq=%d", irq);
 		pr_cont("\n");
@@ -2471,8 +2440,7 @@ static int sio_via_probe(struct pci_dev *pdev, int autoirq, int autodma,
 		have_epp = 1;
 		break;
 	default:
-		printk(KERN_DEBUG
-			"parport_pc: probing current configuration\n");
+		printk(KERN_DEBUG "parport_pc: probing current configuration\n");
 		siofunc = VIA_FUNCTION_PROBE;
 		break;
 	}
@@ -2508,12 +2476,11 @@ static int sio_via_probe(struct pci_dev *pdev, int autoirq, int autodma,
 	port1 = inb(VIA_CONFIG_DATA) << 2;
 
 	printk(KERN_DEBUG "parport_pc: Current parallel port base: 0x%X\n",
-									port1);
+	       port1);
 	if (port1 == 0x3BC && have_epp) {
 		outb(via->viacfg_parport_base, VIA_CONFIG_INDEX);
 		outb((0x378 >> 2), VIA_CONFIG_DATA);
-		printk(KERN_DEBUG
-			"parport_pc: Parallel port base changed to 0x378\n");
+		printk(KERN_DEBUG "parport_pc: Parallel port base changed to 0x378\n");
 		port1 = 0x378;
 	}
 
@@ -2525,7 +2492,7 @@ static int sio_via_probe(struct pci_dev *pdev, int autoirq, int autodma,
 	pci_write_config_byte(pdev, via->via_pci_superio_config_reg, tmp);
 
 	if (siofunc == VIA_FUNCTION_PARPORT_DISABLE) {
-		printk(KERN_INFO "parport_pc: VIA parallel port disabled in BIOS\n");
+		pr_info("parport_pc: VIA parallel port disabled in BIOS\n");
 		return 0;
 	}
 
@@ -2558,9 +2525,8 @@ static int sio_via_probe(struct pci_dev *pdev, int autoirq, int autodma,
 	case 0x278:
 		port2 = 0x678; break;
 	default:
-		printk(KERN_INFO
-			"parport_pc: Weird VIA parport base 0x%X, ignoring\n",
-									port1);
+		pr_info("parport_pc: Weird VIA parport base 0x%X, ignoring\n",
+			port1);
 		return 0;
 	}
 
@@ -2579,8 +2545,7 @@ static int sio_via_probe(struct pci_dev *pdev, int autoirq, int autodma,
 
 	/* finally, do the probe with values obtained */
 	if (parport_pc_probe_port(port1, port2, irq, dma, &pdev->dev, 0)) {
-		printk(KERN_INFO
-			"parport_pc: VIA parallel port: io=0x%X", port1);
+		pr_info("parport_pc: VIA parallel port: io=0x%X", port1);
 		if (irq != PARPORT_IRQ_NONE)
 			pr_cont(", irq=%d", irq);
 		if (dma != PARPORT_DMA_NONE)
@@ -2589,7 +2554,7 @@ static int sio_via_probe(struct pci_dev *pdev, int autoirq, int autodma,
 		return 1;
 	}
 
-	printk(KERN_WARNING "parport_pc: Strange, can't probe VIA parallel port: io=0x%X, irq=%d, dma=%d\n",
+	pr_warn("parport_pc: Strange, can't probe VIA parallel port: io=0x%X, irq=%d, dma=%d\n",
 		port1, irq, dma);
 	return 0;
 }
@@ -2854,14 +2819,12 @@ static int parport_pc_pci_probe(struct pci_dev *dev,
 		/* TODO: test if sharing interrupts works */
 		irq = dev->irq;
 		if (irq == IRQ_NONE) {
-			printk(KERN_DEBUG
-	"PCI parallel port detected: %04x:%04x, I/O at %#lx(%#lx)\n",
-				id->vendor, id->device, io_lo, io_hi);
+			printk(KERN_DEBUG "PCI parallel port detected: %04x:%04x, I/O at %#lx(%#lx)\n",
+			       id->vendor, id->device, io_lo, io_hi);
 			irq = PARPORT_IRQ_NONE;
 		} else {
-			printk(KERN_DEBUG
-	"PCI parallel port detected: %04x:%04x, I/O at %#lx(%#lx), IRQ %d\n",
-				id->vendor, id->device, io_lo, io_hi, irq);
+			printk(KERN_DEBUG "PCI parallel port detected: %04x:%04x, I/O at %#lx(%#lx), IRQ %d\n",
+			       id->vendor, id->device, io_lo, io_hi, irq);
 		}
 		data->ports[count] =
 			parport_pc_probe_port(io_lo, io_hi, irq,
@@ -3111,7 +3074,7 @@ static int __init parport_parse_param(const char *s, int *val,
 		if (ep != s)
 			*val = r;
 		else {
-			printk(KERN_ERR "parport: bad specifier `%s'\n", s);
+			pr_err("parport: bad specifier `%s'\n", s);
 			return -1;
 		}
 	}
@@ -3133,8 +3096,8 @@ static int __init parport_parse_dma(const char *dmastr, int *val)
 #ifdef CONFIG_PCI
 static int __init parport_init_mode_setup(char *str)
 {
-	printk(KERN_DEBUG
-	     "parport_pc.c: Specified parameter parport_init_mode=%s\n", str);
+	printk(KERN_DEBUG "parport_pc.c: Specified parameter parport_init_mode=%s\n",
+	       str);
 
 	if (!strcmp(str, "spp"))
 		parport_init_mode = 1;
@@ -3201,10 +3164,7 @@ static int __init parse_parport_params(void)
 				irqval[0] = val;
 				break;
 			default:
-				printk(KERN_WARNING
-					"parport_pc: irq specified "
-					"without base address.  Use 'io=' "
-					"to specify one\n");
+				pr_warn("parport_pc: irq specified without base address.  Use 'io=' to specify one\n");
 			}
 
 		if (dma[0] && !parport_parse_dma(dma[0], &val))
@@ -3214,10 +3174,7 @@ static int __init parse_parport_params(void)
 				dmaval[0] = val;
 				break;
 			default:
-				printk(KERN_WARNING
-					"parport_pc: dma specified "
-					"without base address.  Use 'io=' "
-					"to specify one\n");
+				pr_warn("parport_pc: dma specified without base address.  Use 'io=' to specify one\n");
 			}
 	}
 	return 0;
@@ -3256,12 +3213,12 @@ static int __init parport_setup(char *str)
 
 	val = simple_strtoul(str, &endptr, 0);
 	if (endptr == str) {
-		printk(KERN_WARNING "parport=%s not understood\n", str);
+		pr_warn("parport=%s not understood\n", str);
 		return 1;
 	}
 
 	if (parport_setup_ptr == PARPORT_PC_MAX_PORTS) {
-		printk(KERN_ERR "parport=%s ignored, too many ports\n", str);
+		pr_err("parport=%s ignored, too many ports\n", str);
 		return 1;
 	}
 
diff --git a/drivers/parport/parport_sunbpp.c b/drivers/parport/parport_sunbpp.c
index d5a669b60c27..e840c1b5ab90 100644
--- a/drivers/parport/parport_sunbpp.c
+++ b/drivers/parport/parport_sunbpp.c
@@ -314,7 +314,7 @@ static int bpp_probe(struct platform_device *op)
 	value_tcr &= ~P_TCR_DIR;
 	sbus_writeb(value_tcr, &regs->p_tcr);
 
-	printk(KERN_INFO "%s: sunbpp at 0x%lx\n", p->name, p->base);
+	pr_info("%s: sunbpp at 0x%lx\n", p->name, p->base);
 
 	dev_set_drvdata(&op->dev, p);
 
diff --git a/drivers/parport/probe.c b/drivers/parport/probe.c
index e5e6a463a941..7e6d713fa5ac 100644
--- a/drivers/parport/probe.c
+++ b/drivers/parport/probe.c
@@ -38,16 +38,16 @@ static void pretty_print(struct parport *port, int device)
 {
 	struct parport_device_info *info = &port->probe_info[device + 1];
 
-	printk(KERN_INFO "%s", port->name);
+	pr_info("%s", port->name);
 
 	if (device >= 0)
-		printk (" (addr %d)", device);
+		pr_cont(" (addr %d)", device);
 
-	printk (": %s", classes[info->class].descr);
+	pr_cont(": %s", classes[info->class].descr);
 	if (info->class)
-		printk(", %s %s", info->mfr, info->model);
+		pr_cont(", %s %s", info->mfr, info->model);
 
-	printk("\n");
+	pr_cont("\n");
 }
 
 static void parse_data(struct parport *port, int device, char *str)
@@ -58,7 +58,7 @@ static void parse_data(struct parport *port, int device, char *str)
 	struct parport_device_info *info = &port->probe_info[device + 1];
 
 	if (!txt) {
-		printk(KERN_WARNING "%s probe: memory squeeze\n", port->name);
+		pr_warn("%s probe: memory squeeze\n", port->name);
 		return;
 	}
 	strcpy(txt, str);
@@ -98,7 +98,8 @@ static void parse_data(struct parport *port, int device, char *str)
 						goto rock_on;
 					}
 				}
-				printk(KERN_WARNING "%s probe: warning, class '%s' not understood.\n", port->name, sep);
+				pr_warn("%s probe: warning, class '%s' not understood\n",
+					port->name, sep);
 				info->class = PARPORT_CLASS_OTHER;
 			} else if (!strcmp(p, "CMD") ||
 				   !strcmp(p, "COMMAND SET")) {
@@ -177,9 +178,8 @@ static ssize_t parport_read_device_id (struct parport *port, char *buffer,
 		 * just return constant nibble forever. This catches
 		 * also those cases. */
 		if (idlens[0] == 0 || idlens[0] > 0xFFF) {
-			printk (KERN_DEBUG "%s: reported broken Device ID"
-				" length of %#zX bytes\n",
-				port->name, idlens[0]);
+			printk(KERN_DEBUG "%s: reported broken Device ID length of %#zX bytes\n",
+			       port->name, idlens[0]);
 			return -EIO;
 		}
 		numidlens = 2;
@@ -201,10 +201,8 @@ static ssize_t parport_read_device_id (struct parport *port, char *buffer,
 
 		if (port->physport->ieee1284.phase != IEEE1284_PH_HBUSY_DAVAIL) {
 			if (belen != len) {
-				printk (KERN_DEBUG "%s: Device ID was %zd bytes"
-					" while device told it would be %d"
-					" bytes\n",
-					port->name, len, belen);
+				printk(KERN_DEBUG "%s: Device ID was %zd bytes while device told it would be %d bytes\n",
+				       port->name, len, belen);
 			}
 			goto done;
 		}
@@ -214,11 +212,9 @@ static ssize_t parport_read_device_id (struct parport *port, char *buffer,
 		 * the first 256 bytes or so that we must have read so
 		 * far. */
 		if (buffer[len-1] == ';') {
- 			printk (KERN_DEBUG "%s: Device ID reading stopped"
-				" before device told data not available. "
-				"Current idlen %u of %u, len bytes %02X %02X\n",
-				port->name, current_idlen, numidlens,
-				length[0], length[1]);
+			printk(KERN_DEBUG "%s: Device ID reading stopped before device told data not available. Current idlen %u of %u, len bytes %02X %02X\n",
+			       port->name, current_idlen, numidlens,
+			       length[0], length[1]);
 			goto done;
 		}
 	}
diff --git a/drivers/parport/procfs.c b/drivers/parport/procfs.c
index ee7b5daabfd4..d740eba3c099 100644
--- a/drivers/parport/procfs.c
+++ b/drivers/parport/procfs.c
@@ -210,7 +210,11 @@ static int do_hardware_modes(struct ctl_table *table, int write,
 		return -EACCES;
 
 	{
-#define printmode(x) {if(port->modes&PARPORT_MODE_##x){len+=sprintf(buffer+len,"%s%s",f?",":"",#x);f++;}}
+#define printmode(x)							\
+do {									\
+	if (port->modes & PARPORT_MODE_##x)				\
+		len += sprintf(buffer + len, "%s%s", f++ ? "," : "", #x); \
+} while (0)
 		int f = 0;
 		printmode(PCSPP);
 		printmode(TRISTATE);
diff --git a/drivers/parport/share.c b/drivers/parport/share.c
index d6920ebeabcd..7fec4fefe151 100644
--- a/drivers/parport/share.c
+++ b/drivers/parport/share.c
@@ -278,46 +278,32 @@ static int port_detect(struct device *dev, void *dev_drv)
 int __parport_register_driver(struct parport_driver *drv, struct module *owner,
 			      const char *mod_name)
 {
-	if (drv->devmodel) {
-		/* using device model */
-		int ret;
-
-		/* initialize common driver fields */
-		drv->driver.name = drv->name;
-		drv->driver.bus = &parport_bus_type;
-		drv->driver.owner = owner;
-		drv->driver.mod_name = mod_name;
-		ret = driver_register(&drv->driver);
-		if (ret)
-			return ret;
+	/* using device model */
+	int ret;
 
-		/*
-		 * check if bus has any parallel port registered, if
-		 * none is found then load the lowlevel driver.
-		 */
-		ret = bus_for_each_dev(&parport_bus_type, NULL, NULL,
-				       port_detect);
-		if (!ret)
-			get_lowlevel_driver();
-
-		mutex_lock(&registration_lock);
-		if (drv->match_port)
-			bus_for_each_dev(&parport_bus_type, NULL, drv,
-					 port_check);
-		mutex_unlock(&registration_lock);
-	} else {
-		struct parport *port;
-
-		drv->devmodel = false;
-
-		if (list_empty(&portlist))
-			get_lowlevel_driver();
-		mutex_lock(&registration_lock);
-		list_for_each_entry(port, &portlist, list)
-			drv->attach(port);
-		list_add(&drv->list, &drivers);
-		mutex_unlock(&registration_lock);
-	}
+	/* initialize common driver fields */
+	drv->driver.name = drv->name;
+	drv->driver.bus = &parport_bus_type;
+	drv->driver.owner = owner;
+	drv->driver.mod_name = mod_name;
+	ret = driver_register(&drv->driver);
+	if (ret)
+		return ret;
+
+	/*
+	 * check if bus has any parallel port registered, if
+	 * none is found then load the lowlevel driver.
+	 */
+	ret = bus_for_each_dev(&parport_bus_type, NULL, NULL,
+			       port_detect);
+	if (!ret)
+		get_lowlevel_driver();
+
+	mutex_lock(&registration_lock);
+	if (drv->match_port)
+		bus_for_each_dev(&parport_bus_type, NULL, drv,
+				 port_check);
+	mutex_unlock(&registration_lock);
 
 	return 0;
 }
@@ -352,17 +338,9 @@ static int port_detach(struct device *dev, void *_drv)
 
 void parport_unregister_driver(struct parport_driver *drv)
 {
-	struct parport *port;
-
 	mutex_lock(&registration_lock);
-	if (drv->devmodel) {
-		bus_for_each_dev(&parport_bus_type, NULL, drv, port_detach);
-		driver_unregister(&drv->driver);
-	} else {
-		list_del_init(&drv->list);
-		list_for_each_entry(port, &portlist, list)
-			drv->detach(port);
-	}
+	bus_for_each_dev(&parport_bus_type, NULL, drv, port_detach);
+	driver_unregister(&drv->driver);
 	mutex_unlock(&registration_lock);
 }
 EXPORT_SYMBOL(parport_unregister_driver);
@@ -554,8 +532,8 @@ void parport_announce_port(struct parport *port)
 #endif
 
 	if (!port->dev)
-		printk(KERN_WARNING "%s: fix this legacy no-device port driver!\n",
-		       port->name);
+		pr_warn("%s: fix this legacy no-device port driver!\n",
+			port->name);
 
 	parport_proc_register(port);
 	mutex_lock(&registration_lock);
@@ -641,47 +619,48 @@ void parport_remove_port(struct parport *port)
 }
 EXPORT_SYMBOL(parport_remove_port);
 
+static void free_pardevice(struct device *dev)
+{
+	struct pardevice *par_dev = to_pardevice(dev);
+
+	kfree(par_dev->name);
+	kfree(par_dev);
+}
+
 /**
- *	parport_register_device - register a device on a parallel port
+ *	parport_register_dev_model - register a device on a parallel port
  *	@port: port to which the device is attached
  *	@name: a name to refer to the device
- *	@pf: preemption callback
- *	@kf: kick callback (wake-up)
- *	@irq_func: interrupt handler
- *	@flags: registration flags
- *	@handle: data for callback functions
+ *	@par_dev_cb: struct containing callbacks
+ *	@id: device number to be given to the device
  *
  *	This function, called by parallel port device drivers,
  *	declares that a device is connected to a port, and tells the
  *	system all it needs to know.
  *
- *	The @name is allocated by the caller and must not be
- *	deallocated until the caller calls @parport_unregister_device
- *	for that device.
- *
- *	The preemption callback function, @pf, is called when this
- *	device driver has claimed access to the port but another
- *	device driver wants to use it.  It is given @handle as its
- *	parameter, and should return zero if it is willing for the
- *	system to release the port to another driver on its behalf.
- *	If it wants to keep control of the port it should return
- *	non-zero, and no action will be taken.  It is good manners for
- *	the driver to try to release the port at the earliest
- *	opportunity after its preemption callback rejects a preemption
- *	attempt.  Note that if a preemption callback is happy for
- *	preemption to go ahead, there is no need to release the port;
- *	it is done automatically.  This function may not block, as it
- *	may be called from interrupt context.  If the device driver
- *	does not support preemption, @pf can be %NULL.
+ *	The struct pardev_cb contains pointer to callbacks. preemption
+ *	callback function, @preempt, is called when this device driver
+ *	has claimed access to the port but another device driver wants
+ *	to use it.  It is given, @private, as its parameter, and should
+ *	return zero if it is willing for the system to release the port
+ *	to another driver on its behalf. If it wants to keep control of
+ *	the port it should return non-zero, and no action will be taken.
+ *	It is good manners for the driver to try to release the port at
+ *	the earliest opportunity after its preemption callback rejects a
+ *	preemption attempt. Note that if a preemption callback is happy
+ *	for preemption to go ahead, there is no need to release the
+ *	port; it is done automatically. This function may not block, as
+ *	it may be called from interrupt context. If the device driver
+ *	does not support preemption, @preempt can be %NULL.
  *
- *	The wake-up ("kick") callback function, @kf, is called when
+ *	The wake-up ("kick") callback function, @wakeup, is called when
  *	the port is available to be claimed for exclusive access; that
  *	is, parport_claim() is guaranteed to succeed when called from
  *	inside the wake-up callback function.  If the driver wants to
  *	claim the port it should do so; otherwise, it need not take
  *	any action.  This function may not block, as it may be called
  *	from interrupt context.  If the device driver does not want to
- *	be explicitly invited to claim the port in this way, @kf can
+ *	be explicitly invited to claim the port in this way, @wakeup can
  *	be %NULL.
  *
  *	The interrupt handler, @irq_func, is called when an interrupt
@@ -711,138 +690,6 @@ EXPORT_SYMBOL(parport_remove_port);
  **/
 
 struct pardevice *
-parport_register_device(struct parport *port, const char *name,
-			int (*pf)(void *), void (*kf)(void *),
-			void (*irq_func)(void *),
-			int flags, void *handle)
-{
-	struct pardevice *tmp;
-
-	if (port->physport->flags & PARPORT_FLAG_EXCL) {
-		/* An exclusive device is registered. */
-		printk(KERN_DEBUG "%s: no more devices allowed\n",
-			port->name);
-		return NULL;
-	}
-
-	if (flags & PARPORT_DEV_LURK) {
-		if (!pf || !kf) {
-			printk(KERN_INFO "%s: refused to register lurking device (%s) without callbacks\n", port->name, name);
-			return NULL;
-		}
-	}
-
-	if (flags & PARPORT_DEV_EXCL) {
-		if (port->physport->devices) {
-			/*
-			 * If a device is already registered and this new
-			 * device wants exclusive access, then no need to
-			 * continue as we can not grant exclusive access to
-			 * this device.
-			 */
-			pr_err("%s: cannot grant exclusive access for device %s\n",
-			       port->name, name);
-			return NULL;
-		}
-	}
-
-	/*
-	 * We up our own module reference count, and that of the port
-	 * on which a device is to be registered, to ensure that
-	 * neither of us gets unloaded while we sleep in (e.g.)
-	 * kmalloc.
-	 */
-	if (!try_module_get(port->ops->owner))
-		return NULL;
-
-	parport_get_port(port);
-
-	tmp = kmalloc(sizeof(struct pardevice), GFP_KERNEL);
-	if (!tmp)
-		goto out;
-
-	tmp->state = kmalloc(sizeof(struct parport_state), GFP_KERNEL);
-	if (!tmp->state)
-		goto out_free_pardevice;
-
-	tmp->name = name;
-	tmp->port = port;
-	tmp->daisy = -1;
-	tmp->preempt = pf;
-	tmp->wakeup = kf;
-	tmp->private = handle;
-	tmp->flags = flags;
-	tmp->irq_func = irq_func;
-	tmp->waiting = 0;
-	tmp->timeout = 5 * HZ;
-	tmp->devmodel = false;
-
-	/* Chain this onto the list */
-	tmp->prev = NULL;
-	/*
-	 * This function must not run from an irq handler so we don' t need
-	 * to clear irq on the local CPU. -arca
-	 */
-	spin_lock(&port->physport->pardevice_lock);
-
-	if (flags & PARPORT_DEV_EXCL) {
-		if (port->physport->devices) {
-			spin_unlock(&port->physport->pardevice_lock);
-			printk(KERN_DEBUG
-				"%s: cannot grant exclusive access for device %s\n",
-				port->name, name);
-			goto out_free_all;
-		}
-		port->flags |= PARPORT_FLAG_EXCL;
-	}
-
-	tmp->next = port->physport->devices;
-	wmb(); /*
-		* Make sure that tmp->next is written before it's
-		* added to the list; see comments marked 'no locking
-		* required'
-		*/
-	if (port->physport->devices)
-		port->physport->devices->prev = tmp;
-	port->physport->devices = tmp;
-	spin_unlock(&port->physport->pardevice_lock);
-
-	init_waitqueue_head(&tmp->wait_q);
-	tmp->timeslice = parport_default_timeslice;
-	tmp->waitnext = tmp->waitprev = NULL;
-
-	/*
-	 * This has to be run as last thing since init_state may need other
-	 * pardevice fields. -arca
-	 */
-	port->ops->init_state(tmp, tmp->state);
-	if (!test_and_set_bit(PARPORT_DEVPROC_REGISTERED, &port->devflags)) {
-		port->proc_device = tmp;
-		parport_device_proc_register(tmp);
-	}
-	return tmp;
-
- out_free_all:
-	kfree(tmp->state);
- out_free_pardevice:
-	kfree(tmp);
- out:
-	parport_put_port(port);
-	module_put(port->ops->owner);
-
-	return NULL;
-}
-EXPORT_SYMBOL(parport_register_device);
-
-static void free_pardevice(struct device *dev)
-{
-	struct pardevice *par_dev = to_pardevice(dev);
-
-	kfree(par_dev->name);
-	kfree(par_dev);
-}
-
-struct pardevice *
 parport_register_dev_model(struct parport *port, const char *name,
 			   const struct pardev_cb *par_dev_cb, int id)
 {
@@ -996,7 +843,7 @@ void parport_unregister_device(struct pardevice *dev)
 
 #ifdef PARPORT_PARANOID
 	if (!dev) {
-		printk(KERN_ERR "parport_unregister_device: passed NULL\n");
+		pr_err("%s: passed NULL\n", __func__);
 		return;
 	}
 #endif
@@ -1046,10 +893,7 @@ void parport_unregister_device(struct pardevice *dev)
 	spin_unlock_irq(&port->waitlist_lock);
 
 	kfree(dev->state);
-	if (dev->devmodel)
-		device_unregister(&dev->dev);
-	else
-		kfree(dev);
+	device_unregister(&dev->dev);
 
 	module_put(port->ops->owner);
 	parport_put_port(port);
@@ -1137,8 +981,7 @@ int parport_claim(struct pardevice *dev)
 	unsigned long flags;
 
 	if (port->cad == dev) {
-		printk(KERN_INFO "%s: %s already owner\n",
-		       dev->port->name,dev->name);
+		pr_info("%s: %s already owner\n", dev->port->name, dev->name);
 		return 0;
 	}
 
@@ -1158,9 +1001,8 @@ int parport_claim(struct pardevice *dev)
 			 * I think we'll actually deadlock rather than
 			 * get here, but just in case..
 			 */
-			printk(KERN_WARNING
-			       "%s: %s released port when preempted!\n",
-			       port->name, oldcad->name);
+			pr_warn("%s: %s released port when preempted!\n",
+				port->name, oldcad->name);
 			if (port->cad)
 				goto blocked;
 		}
@@ -1260,7 +1102,8 @@ int parport_claim_or_block(struct pardevice *dev)
 	r = parport_claim(dev);
 	if (r == -EAGAIN) {
 #ifdef PARPORT_DEBUG_SHARING
-		printk(KERN_DEBUG "%s: parport_claim() returned -EAGAIN\n", dev->name);
+		printk(KERN_DEBUG "%s: parport_claim() returned -EAGAIN\n",
+		       dev->name);
 #endif
 		/*
 		 * FIXME!!! Use the proper locking for dev->waiting,
@@ -1293,7 +1136,7 @@ int parport_claim_or_block(struct pardevice *dev)
 		if (dev->port->physport->cad != dev)
 			printk(KERN_DEBUG "%s: exiting parport_claim_or_block but %s owns port!\n",
 			       dev->name, dev->port->physport->cad ?
-			       dev->port->physport->cad->name:"nobody");
+			       dev->port->physport->cad->name : "nobody");
 #endif
 	}
 	dev->waiting = 0;
@@ -1320,8 +1163,8 @@ void parport_release(struct pardevice *dev)
 	write_lock_irqsave(&port->cad_lock, flags);
 	if (port->cad != dev) {
 		write_unlock_irqrestore(&port->cad_lock, flags);
-		printk(KERN_WARNING "%s: %s tried to release parport when not owner\n",
-		       port->name, dev->name);
+		pr_warn("%s: %s tried to release parport when not owner\n",
+			port->name, dev->name);
 		return;
 	}
 
@@ -1361,7 +1204,8 @@ void parport_release(struct pardevice *dev)
 			if (dev->port->cad) /* racy but no matter */
 				return;
 		} else {
-			printk(KERN_ERR "%s: don't know how to wake %s\n", port->name, pd->name);
+			pr_err("%s: don't know how to wake %s\n",
+			       port->name, pd->name);
 		}
 	}
 
diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c
index 390e92f2d8d1..b761c1f72f67 100644
--- a/drivers/pci/ats.c
+++ b/drivers/pci/ats.c
@@ -31,6 +31,22 @@ void pci_ats_init(struct pci_dev *dev)
 }
 
 /**
+ * pci_ats_supported - check if the device can use ATS
+ * @dev: the PCI device
+ *
+ * Returns true if the device supports ATS and is allowed to use it, false
+ * otherwise.
+ */
+bool pci_ats_supported(struct pci_dev *dev)
+{
+	if (!dev->ats_cap)
+		return false;
+
+	return (dev->untrusted == 0);
+}
+EXPORT_SYMBOL_GPL(pci_ats_supported);
+
+/**
  * pci_enable_ats - enable the ATS capability
  * @dev: the PCI device
  * @ps: the IOMMU page shift
@@ -42,7 +58,7 @@ int pci_enable_ats(struct pci_dev *dev, int ps)
 	u16 ctrl;
 	struct pci_dev *pdev;
 
-	if (!dev->ats_cap)
+	if (!pci_ats_supported(dev))
 		return -EINVAL;
 
 	if (WARN_ON(dev->ats_enabled))
diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index ae36edb1d7db..adddf21fa381 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -58,15 +58,33 @@ config PCIE_RCAR
 	bool "Renesas R-Car PCIe controller"
 	depends on ARCH_RENESAS || COMPILE_TEST
 	depends on PCI_MSI_IRQ_DOMAIN
+	select PCIE_RCAR_HOST
 	help
 	  Say Y here if you want PCIe controller support on R-Car SoCs.
+	  This option will be removed after arm64 defconfig is updated.
+
+config PCIE_RCAR_HOST
+	bool "Renesas R-Car PCIe host controller"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on PCI_MSI_IRQ_DOMAIN
+	help
+	  Say Y here if you want PCIe controller support on R-Car SoCs in host
+	  mode.
+
+config PCIE_RCAR_EP
+	bool "Renesas R-Car PCIe endpoint controller"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on PCI_ENDPOINT
+	help
+	  Say Y here if you want PCIe controller support on R-Car SoCs in
+	  endpoint mode.
 
 config PCI_HOST_COMMON
-	bool
+	tristate
 	select PCI_ECAM
 
 config PCI_HOST_GENERIC
-	bool "Generic PCI host controller"
+	tristate "Generic PCI host controller"
 	depends on OF
 	select PCI_HOST_COMMON
 	select IRQ_DOMAIN
@@ -230,7 +248,7 @@ config PCIE_TANGO_SMP8759
 config VMD
 	depends on PCI_MSI && X86_64 && SRCU
 	tristate "Intel Volume Management Device Driver"
-	---help---
+	help
 	  Adds support for the Intel Volume Management Device (VMD). VMD is a
 	  secondary PCI host bridge that allows PCI Express root ports,
 	  and devices attached to them, to be removed from the default
diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile
index fbac4b0190a0..efd9733ead26 100644
--- a/drivers/pci/controller/Makefile
+++ b/drivers/pci/controller/Makefile
@@ -7,7 +7,8 @@ obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o
 obj-$(CONFIG_PCI_AARDVARK) += pci-aardvark.o
 obj-$(CONFIG_PCI_TEGRA) += pci-tegra.o
 obj-$(CONFIG_PCI_RCAR_GEN2) += pci-rcar-gen2.o
-obj-$(CONFIG_PCIE_RCAR) += pcie-rcar.o
+obj-$(CONFIG_PCIE_RCAR_HOST) += pcie-rcar.o pcie-rcar-host.o
+obj-$(CONFIG_PCIE_RCAR_EP) += pcie-rcar.o pcie-rcar-ep.o
 obj-$(CONFIG_PCI_HOST_COMMON) += pci-host-common.o
 obj-$(CONFIG_PCI_HOST_GENERIC) += pci-host-generic.o
 obj-$(CONFIG_PCIE_XILINX) += pcie-xilinx.o
diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c
index 1c173dad67d1..1c15c8352125 100644
--- a/drivers/pci/controller/cadence/pcie-cadence-ep.c
+++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c
@@ -450,7 +450,7 @@ int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep)
 		epc->max_functions = 1;
 
 	ret = pci_epc_mem_init(epc, pcie->mem_res->start,
-			       resource_size(pcie->mem_res));
+			       resource_size(pcie->mem_res), PAGE_SIZE);
 	if (ret < 0) {
 		dev_err(dev, "failed to initialize the memory space\n");
 		goto err_init;
diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c
index 9b1c3966414b..8c2543f28ba0 100644
--- a/drivers/pci/controller/cadence/pcie-cadence-host.c
+++ b/drivers/pci/controller/cadence/pcie-cadence-host.c
@@ -140,9 +140,6 @@ static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc)
 	for_each_of_pci_range(&parser, &range) {
 		bool is_io;
 
-		if (r >= rc->max_regions)
-			break;
-
 		if ((range.flags & IORESOURCE_TYPE_BITS) == IORESOURCE_MEM)
 			is_io = false;
 		else if ((range.flags & IORESOURCE_TYPE_BITS) == IORESOURCE_IO)
@@ -219,17 +216,14 @@ int cdns_pcie_host_setup(struct cdns_pcie_rc *rc)
 	pcie = &rc->pcie;
 	pcie->is_rc = true;
 
-	rc->max_regions = 32;
-	of_property_read_u32(np, "cdns,max-outbound-regions", &rc->max_regions);
-
 	rc->no_bar_nbits = 32;
 	of_property_read_u32(np, "cdns,no-bar-match-nbits", &rc->no_bar_nbits);
 
 	rc->vendor_id = 0xffff;
-	of_property_read_u16(np, "vendor-id", &rc->vendor_id);
+	of_property_read_u32(np, "vendor-id", &rc->vendor_id);
 
 	rc->device_id = 0xffff;
-	of_property_read_u16(np, "device-id", &rc->device_id);
+	of_property_read_u32(np, "device-id", &rc->device_id);
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "reg");
 	pcie->reg_base = devm_ioremap_resource(dev, res);
diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h
index a2b28b912ca4..df14ad002fe9 100644
--- a/drivers/pci/controller/cadence/pcie-cadence.h
+++ b/drivers/pci/controller/cadence/pcie-cadence.h
@@ -251,7 +251,6 @@ struct cdns_pcie {
  * @bus_range: first/last buses behind the PCIe host controller
  * @cfg_base: IO mapped window to access the PCI configuration space of a
  *            single function at a time
- * @max_regions: maximum number of regions supported by the hardware
  * @no_bar_nbits: Number of bits to keep for inbound (PCIe -> CPU) address
  *                translation (nbits sets into the "no BAR match" register)
  * @vendor_id: PCI vendor ID
@@ -262,10 +261,9 @@ struct cdns_pcie_rc {
 	struct resource		*cfg_res;
 	struct resource		*bus_range;
 	void __iomem		*cfg_base;
-	u32			max_regions;
 	u32			no_bar_nbits;
-	u16			vendor_id;
-	u16			device_id;
+	u32			vendor_id;
+	u32			device_id;
 };
 
 /**
diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index 03dcaf65d159..044a3761c44f 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -26,7 +26,7 @@ config PCI_DRA7XX_HOST
 	depends on OF && HAS_IOMEM && TI_PIPE3
 	select PCIE_DW_HOST
 	select PCI_DRA7XX
-	default y
+	default y if SOC_DRA7XX
 	help
 	  Enables support for the PCIe controller in the DRA7xx SoC to work in
 	  host mode. There are two instances of PCIe controller in DRA7xx.
@@ -111,7 +111,6 @@ config PCI_KEYSTONE_HOST
 	depends on PCI_MSI_IRQ_DOMAIN
 	select PCIE_DW_HOST
 	select PCI_KEYSTONE
-	default y
 	help
 	  Enables support for the PCIe controller in the Keystone SoC to
 	  work in host mode. The PCI controller on Keystone is based on
@@ -281,15 +280,25 @@ config PCIE_TEGRA194_EP
 	  selected. This uses the DesignWare core.
 
 config PCIE_UNIPHIER
-	bool "Socionext UniPhier PCIe controllers"
+	bool "Socionext UniPhier PCIe host controllers"
 	depends on ARCH_UNIPHIER || COMPILE_TEST
 	depends on OF && HAS_IOMEM
 	depends on PCI_MSI_IRQ_DOMAIN
 	select PCIE_DW_HOST
 	help
-	  Say Y here if you want PCIe controller support on UniPhier SoCs.
+	  Say Y here if you want PCIe host controller support on UniPhier SoCs.
 	  This driver supports LD20 and PXs3 SoCs.
 
+config PCIE_UNIPHIER_EP
+	bool "Socionext UniPhier PCIe endpoint controllers"
+	depends on ARCH_UNIPHIER || COMPILE_TEST
+	depends on OF && HAS_IOMEM
+	depends on PCI_ENDPOINT
+	select PCIE_DW_EP
+	help
+	  Say Y here if you want PCIe endpoint controller support on
+	  UniPhier SoCs. This driver supports Pro5 SoC.
+
 config PCIE_AL
 	bool "Amazon Annapurna Labs PCIe controller"
 	depends on OF && (ARM64 || COMPILE_TEST)
diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile
index 8a637cfcf6e9..a751553fa0db 100644
--- a/drivers/pci/controller/dwc/Makefile
+++ b/drivers/pci/controller/dwc/Makefile
@@ -19,6 +19,7 @@ obj-$(CONFIG_PCIE_HISI_STB) += pcie-histb.o
 obj-$(CONFIG_PCI_MESON) += pci-meson.o
 obj-$(CONFIG_PCIE_TEGRA194) += pcie-tegra194.o
 obj-$(CONFIG_PCIE_UNIPHIER) += pcie-uniphier.o
+obj-$(CONFIG_PCIE_UNIPHIER_EP) += pcie-uniphier-ep.o
 
 # The following drivers are for devices that use the generic ACPI
 # pci_root.c driver but don't support standard ECAM config access.
diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
index 3b0e58f2de58..6184ebc9392d 100644
--- a/drivers/pci/controller/dwc/pci-dra7xx.c
+++ b/drivers/pci/controller/dwc/pci-dra7xx.c
@@ -840,7 +840,6 @@ static int __init dra7xx_pcie_probe(struct platform_device *pdev)
 	struct phy **phy;
 	struct device_link **link;
 	void __iomem *base;
-	struct resource *res;
 	struct dw_pcie *pci;
 	struct dra7xx_pcie *dra7xx;
 	struct device *dev = &pdev->dev;
@@ -877,10 +876,9 @@ static int __init dra7xx_pcie_probe(struct platform_device *pdev)
 		return irq;
 	}
 
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ti_conf");
-	base = devm_ioremap(dev, res->start, resource_size(res));
-	if (!base)
-		return -ENOMEM;
+	base = devm_platform_ioremap_resource_byname(pdev, "ti_conf");
+	if (IS_ERR(base))
+		return PTR_ERR(base);
 
 	phy_count = of_property_count_strings(np, "phy-names");
 	if (phy_count < 0) {
diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c
index acfbd34032a8..8f08ae53f53e 100644
--- a/drivers/pci/controller/dwc/pci-imx6.c
+++ b/drivers/pci/controller/dwc/pci-imx6.c
@@ -868,9 +868,9 @@ static int imx6_add_pcie_port(struct imx6_pcie *imx6_pcie,
 
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
 		pp->msi_irq = platform_get_irq_byname(pdev, "msi");
-		if (pp->msi_irq <= 0) {
+		if (pp->msi_irq < 0) {
 			dev_err(dev, "failed to get MSI irq\n");
-			return -ENODEV;
+			return pp->msi_irq;
 		}
 	}
 
diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c
index 3715dceca1bf..ca59ba9e0ecd 100644
--- a/drivers/pci/controller/dwc/pci-meson.c
+++ b/drivers/pci/controller/dwc/pci-meson.c
@@ -289,11 +289,11 @@ static void meson_pcie_init_dw(struct meson_pcie *mp)
 	meson_cfg_writel(mp, val, PCIE_CFG0);
 
 	val = meson_elb_readl(mp, PCIE_PORT_LINK_CTRL_OFF);
-	val &= ~LINK_CAPABLE_MASK;
+	val &= ~(LINK_CAPABLE_MASK | FAST_LINK_MODE);
 	meson_elb_writel(mp, val, PCIE_PORT_LINK_CTRL_OFF);
 
 	val = meson_elb_readl(mp, PCIE_PORT_LINK_CTRL_OFF);
-	val |= LINK_CAPABLE_X1 | FAST_LINK_MODE;
+	val |= LINK_CAPABLE_X1;
 	meson_elb_writel(mp, val, PCIE_PORT_LINK_CTRL_OFF);
 
 	val = meson_elb_readl(mp, PCIE_GEN2_CTRL_OFF);
diff --git a/drivers/pci/controller/dwc/pcie-al.c b/drivers/pci/controller/dwc/pcie-al.c
index 1eeda2f6371f..270868f3859a 100644
--- a/drivers/pci/controller/dwc/pcie-al.c
+++ b/drivers/pci/controller/dwc/pcie-al.c
@@ -80,7 +80,7 @@ static int al_pcie_init(struct pci_config_window *cfg)
 	return 0;
 }
 
-struct pci_ecam_ops al_pcie_ops = {
+const struct pci_ecam_ops al_pcie_ops = {
 	.bus_shift    = 20,
 	.init         =  al_pcie_init,
 	.pci_ops      = {
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 1cdcbd102ce8..5e5b8821bed8 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -412,11 +412,11 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 		reg = ep->msi_cap + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
-	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1);
+	aligned_offset = msg_addr_lower & (epc->mem->window.page_size - 1);
 	msg_addr = ((u64)msg_addr_upper) << 32 |
 			(msg_addr_lower & ~aligned_offset);
 	ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr,
-				  epc->mem->page_size);
+				  epc->mem->window.page_size);
 	if (ret)
 		return ret;
 
@@ -433,7 +433,6 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epf_msix_tbl *msix_tbl;
 	struct pci_epc *epc = ep->epc;
-	struct pci_epf_bar *epf_bar;
 	u32 reg, msg_data, vec_ctrl;
 	unsigned int aligned_offset;
 	u32 tbl_offset;
@@ -446,10 +445,7 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;
 
-	epf_bar = ep->epf_bar[bir];
-	msix_tbl = epf_bar->addr;
-	msix_tbl = (struct pci_epf_msix_tbl *)((char *)msix_tbl + tbl_offset);
-
+	msix_tbl = ep->epf_bar[bir]->addr + tbl_offset;
 	msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr;
 	msg_data = msix_tbl[(interrupt_num - 1)].msg_data;
 	vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl;
@@ -459,9 +455,9 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 		return -EPERM;
 	}
 
-	aligned_offset = msg_addr & (epc->mem->page_size - 1);
+	aligned_offset = msg_addr & (epc->mem->window.page_size - 1);
 	ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys,  msg_addr,
-				  epc->mem->page_size);
+				  epc->mem->window.page_size);
 	if (ret)
 		return ret;
 
@@ -477,7 +473,7 @@ void dw_pcie_ep_exit(struct dw_pcie_ep *ep)
 	struct pci_epc *epc = ep->epc;
 
 	pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem,
-			      epc->mem->page_size);
+			      epc->mem->window.page_size);
 
 	pci_epc_mem_exit(epc);
 }
@@ -610,15 +606,15 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	if (ret < 0)
 		epc->max_functions = 1;
 
-	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
-				 ep->page_size);
+	ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
+			       ep->page_size);
 	if (ret < 0) {
 		dev_err(dev, "Failed to initialize address space\n");
 		return ret;
 	}
 
 	ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys,
-					     epc->mem->page_size);
+					     epc->mem->window.page_size);
 	if (!ep->msi_mem) {
 		dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n");
 		return -ENOMEM;
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 395feb8ca051..0a4a5aa6fe46 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -236,7 +236,7 @@ static void dw_pcie_irq_domain_free(struct irq_domain *domain,
 				    unsigned int virq, unsigned int nr_irqs)
 {
 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
-	struct pcie_port *pp = irq_data_get_irq_chip_data(d);
+	struct pcie_port *pp = domain->host_data;
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&pp->lock, flags);
@@ -264,6 +264,8 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
 		return -ENOMEM;
 	}
 
+	irq_domain_update_bus_token(pp->irq_domain, DOMAIN_BUS_NEXUS);
+
 	pp->msi_domain = pci_msi_create_irq_domain(fwnode,
 						   &dw_pcie_msi_domain_info,
 						   pp->irq_domain);
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index 681548c88282..c92496e36fd5 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -244,13 +244,16 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
 					     u64 pci_addr, u32 size)
 {
 	u32 retries, val;
+	u64 limit_addr = cpu_addr + size - 1;
 
 	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_BASE,
 				 lower_32_bits(cpu_addr));
 	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_BASE,
 				 upper_32_bits(cpu_addr));
-	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
-				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_LIMIT,
+				 lower_32_bits(limit_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_LIMIT,
+				 upper_32_bits(limit_addr));
 	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_TARGET,
 				 lower_32_bits(pci_addr));
 	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET,
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index d6e1f397e6b0..656e00f8fbeb 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -112,9 +112,10 @@
 #define PCIE_ATU_UNR_REGION_CTRL2	0x04
 #define PCIE_ATU_UNR_LOWER_BASE		0x08
 #define PCIE_ATU_UNR_UPPER_BASE		0x0C
-#define PCIE_ATU_UNR_LIMIT		0x10
+#define PCIE_ATU_UNR_LOWER_LIMIT	0x10
 #define PCIE_ATU_UNR_LOWER_TARGET	0x14
 #define PCIE_ATU_UNR_UPPER_TARGET	0x18
+#define PCIE_ATU_UNR_UPPER_LIMIT	0x20
 
 /*
  * The default address offset between dbi_base and atu_base. Root controller
diff --git a/drivers/pci/controller/dwc/pcie-hisi.c b/drivers/pci/controller/dwc/pcie-hisi.c
index 6d9e1b2b8f7b..0ad4e07dd4c2 100644
--- a/drivers/pci/controller/dwc/pcie-hisi.c
+++ b/drivers/pci/controller/dwc/pcie-hisi.c
@@ -104,7 +104,7 @@ static int hisi_pcie_init(struct pci_config_window *cfg)
 	return 0;
 }
 
-struct pci_ecam_ops hisi_pcie_ops = {
+const struct pci_ecam_ops hisi_pcie_ops = {
 	.bus_shift    = 20,
 	.init         =  hisi_pcie_init,
 	.pci_ops      = {
@@ -332,15 +332,6 @@ static struct platform_driver hisi_pcie_driver = {
 };
 builtin_platform_driver(hisi_pcie_driver);
 
-static int hisi_pcie_almost_ecam_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct pci_ecam_ops *ops;
-
-	ops = (struct pci_ecam_ops *)of_device_get_match_data(dev);
-	return pci_host_common_probe(pdev, ops);
-}
-
 static int hisi_pcie_platform_init(struct pci_config_window *cfg)
 {
 	struct device *dev = cfg->parent;
@@ -362,7 +353,7 @@ static int hisi_pcie_platform_init(struct pci_config_window *cfg)
 	return 0;
 }
 
-struct pci_ecam_ops hisi_pcie_platform_ops = {
+static const struct pci_ecam_ops hisi_pcie_platform_ops = {
 	.bus_shift    = 20,
 	.init         =  hisi_pcie_platform_init,
 	.pci_ops      = {
@@ -375,17 +366,17 @@ struct pci_ecam_ops hisi_pcie_platform_ops = {
 static const struct of_device_id hisi_pcie_almost_ecam_of_match[] = {
 	{
 		.compatible =  "hisilicon,hip06-pcie-ecam",
-		.data	    = (void *) &hisi_pcie_platform_ops,
+		.data	    =  &hisi_pcie_platform_ops,
 	},
 	{
 		.compatible =  "hisilicon,hip07-pcie-ecam",
-		.data       = (void *) &hisi_pcie_platform_ops,
+		.data       =  &hisi_pcie_platform_ops,
 	},
 	{},
 };
 
 static struct platform_driver hisi_pcie_almost_ecam_driver = {
-	.probe  = hisi_pcie_almost_ecam_probe,
+	.probe  = pci_host_common_probe,
 	.driver = {
 		   .name = "hisi-pcie-almost-ecam",
 		   .of_match_table = hisi_pcie_almost_ecam_of_match,
diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c
index fc2a12212dec..2d8dbb318087 100644
--- a/drivers/pci/controller/dwc/pcie-intel-gw.c
+++ b/drivers/pci/controller/dwc/pcie-intel-gw.c
@@ -453,7 +453,7 @@ static int intel_pcie_msi_init(struct pcie_port *pp)
 	return 0;
 }
 
-u64 intel_pcie_cpu_addr(struct dw_pcie *pcie, u64 cpu_addr)
+static u64 intel_pcie_cpu_addr(struct dw_pcie *pcie, u64 cpu_addr)
 {
 	return cpu_addr + BUS_IATU_OFFSET;
 }
diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
index ae30a2fd3716..92b77f7d8354 100644
--- a/drivers/pci/controller/dwc/pcie-tegra194.c
+++ b/drivers/pci/controller/dwc/pcie-tegra194.c
@@ -1623,7 +1623,7 @@ static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
 	ret = pinctrl_pm_select_default_state(dev);
 	if (ret < 0) {
 		dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
-		goto fail_pinctrl;
+		goto fail_pm_get_sync;
 	}
 
 	tegra_pcie_init_controller(pcie);
@@ -1650,9 +1650,8 @@ static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
 
 fail_host_init:
 	tegra_pcie_deinit_controller(pcie);
-fail_pinctrl:
-	pm_runtime_put_sync(dev);
 fail_pm_get_sync:
+	pm_runtime_put_sync(dev);
 	pm_runtime_disable(dev);
 	return ret;
 }
@@ -2190,9 +2189,9 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev)
 	}
 
 	pp->irq = platform_get_irq_byname(pdev, "intr");
-	if (!pp->irq) {
+	if (pp->irq < 0) {
 		dev_err(dev, "Failed to get \"intr\" interrupt\n");
-		return -ENODEV;
+		return pp->irq;
 	}
 
 	pcie->bpmp = tegra_bpmp_get(dev);
diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c
new file mode 100644
index 000000000000..148355960061
--- /dev/null
+++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c
@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PCIe endpoint controller driver for UniPhier SoCs
+ * Copyright 2018 Socionext Inc.
+ * Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/of_device.h>
+#include <linux/pci.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+#include "pcie-designware.h"
+
+/* Link Glue registers */
+#define PCL_RSTCTRL0			0x0010
+#define PCL_RSTCTRL_AXI_REG		BIT(3)
+#define PCL_RSTCTRL_AXI_SLAVE		BIT(2)
+#define PCL_RSTCTRL_AXI_MASTER		BIT(1)
+#define PCL_RSTCTRL_PIPE3		BIT(0)
+
+#define PCL_RSTCTRL1			0x0020
+#define PCL_RSTCTRL_PERST		BIT(0)
+
+#define PCL_RSTCTRL2			0x0024
+#define PCL_RSTCTRL_PHY_RESET		BIT(0)
+
+#define PCL_MODE			0x8000
+#define PCL_MODE_REGEN			BIT(8)
+#define PCL_MODE_REGVAL			BIT(0)
+
+#define PCL_APP_CLK_CTRL		0x8004
+#define PCL_APP_CLK_REQ			BIT(0)
+
+#define PCL_APP_READY_CTRL		0x8008
+#define PCL_APP_LTSSM_ENABLE		BIT(0)
+
+#define PCL_APP_MSI0			0x8040
+#define PCL_APP_VEN_MSI_TC_MASK		GENMASK(10, 8)
+#define PCL_APP_VEN_MSI_VECTOR_MASK	GENMASK(4, 0)
+
+#define PCL_APP_MSI1			0x8044
+#define PCL_APP_MSI_REQ			BIT(0)
+
+#define PCL_APP_INTX			0x8074
+#define PCL_APP_INTX_SYS_INT		BIT(0)
+
+/* assertion time of INTx in usec */
+#define PCL_INTX_WIDTH_USEC		30
+
+struct uniphier_pcie_ep_priv {
+	void __iomem *base;
+	struct dw_pcie pci;
+	struct clk *clk, *clk_gio;
+	struct reset_control *rst, *rst_gio;
+	struct phy *phy;
+	const struct pci_epc_features *features;
+};
+
+#define to_uniphier_pcie(x)	dev_get_drvdata((x)->dev)
+
+static void uniphier_pcie_ltssm_enable(struct uniphier_pcie_ep_priv *priv,
+				       bool enable)
+{
+	u32 val;
+
+	val = readl(priv->base + PCL_APP_READY_CTRL);
+	if (enable)
+		val |= PCL_APP_LTSSM_ENABLE;
+	else
+		val &= ~PCL_APP_LTSSM_ENABLE;
+	writel(val, priv->base + PCL_APP_READY_CTRL);
+}
+
+static void uniphier_pcie_phy_reset(struct uniphier_pcie_ep_priv *priv,
+				    bool assert)
+{
+	u32 val;
+
+	val = readl(priv->base + PCL_RSTCTRL2);
+	if (assert)
+		val |= PCL_RSTCTRL_PHY_RESET;
+	else
+		val &= ~PCL_RSTCTRL_PHY_RESET;
+	writel(val, priv->base + PCL_RSTCTRL2);
+}
+
+static void uniphier_pcie_init_ep(struct uniphier_pcie_ep_priv *priv)
+{
+	u32 val;
+
+	/* set EP mode */
+	val = readl(priv->base + PCL_MODE);
+	val |= PCL_MODE_REGEN | PCL_MODE_REGVAL;
+	writel(val, priv->base + PCL_MODE);
+
+	/* clock request */
+	val = readl(priv->base + PCL_APP_CLK_CTRL);
+	val &= ~PCL_APP_CLK_REQ;
+	writel(val, priv->base + PCL_APP_CLK_CTRL);
+
+	/* deassert PIPE3 and AXI reset */
+	val = readl(priv->base + PCL_RSTCTRL0);
+	val |= PCL_RSTCTRL_AXI_REG | PCL_RSTCTRL_AXI_SLAVE
+		| PCL_RSTCTRL_AXI_MASTER | PCL_RSTCTRL_PIPE3;
+	writel(val, priv->base + PCL_RSTCTRL0);
+
+	uniphier_pcie_ltssm_enable(priv, false);
+
+	msleep(100);
+}
+
+static int uniphier_pcie_start_link(struct dw_pcie *pci)
+{
+	struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci);
+
+	uniphier_pcie_ltssm_enable(priv, true);
+
+	return 0;
+}
+
+static void uniphier_pcie_stop_link(struct dw_pcie *pci)
+{
+	struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci);
+
+	uniphier_pcie_ltssm_enable(priv, false);
+}
+
+static void uniphier_pcie_ep_init(struct dw_pcie_ep *ep)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	enum pci_barno bar;
+
+	for (bar = BAR_0; bar <= BAR_5; bar++)
+		dw_pcie_ep_reset_bar(pci, bar);
+}
+
+static int uniphier_pcie_ep_raise_legacy_irq(struct dw_pcie_ep *ep)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci);
+	u32 val;
+
+	/*
+	 * This makes pulse signal to send INTx to the RC, so this should
+	 * be cleared as soon as possible. This sequence is covered with
+	 * mutex in pci_epc_raise_irq().
+	 */
+	/* assert INTx */
+	val = readl(priv->base + PCL_APP_INTX);
+	val |= PCL_APP_INTX_SYS_INT;
+	writel(val, priv->base + PCL_APP_INTX);
+
+	udelay(PCL_INTX_WIDTH_USEC);
+
+	/* deassert INTx */
+	val &= ~PCL_APP_INTX_SYS_INT;
+	writel(val, priv->base + PCL_APP_INTX);
+
+	return 0;
+}
+
+static int uniphier_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep,
+					  u8 func_no, u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci);
+	u32 val;
+
+	val = FIELD_PREP(PCL_APP_VEN_MSI_TC_MASK, func_no)
+		| FIELD_PREP(PCL_APP_VEN_MSI_VECTOR_MASK, interrupt_num - 1);
+	writel(val, priv->base + PCL_APP_MSI0);
+
+	val = readl(priv->base + PCL_APP_MSI1);
+	val |= PCL_APP_MSI_REQ;
+	writel(val, priv->base + PCL_APP_MSI1);
+
+	return 0;
+}
+
+static int uniphier_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
+				      enum pci_epc_irq_type type,
+				      u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+
+	switch (type) {
+	case PCI_EPC_IRQ_LEGACY:
+		return uniphier_pcie_ep_raise_legacy_irq(ep);
+	case PCI_EPC_IRQ_MSI:
+		return uniphier_pcie_ep_raise_msi_irq(ep, func_no,
+						      interrupt_num);
+	default:
+		dev_err(pci->dev, "UNKNOWN IRQ type (%d)\n", type);
+	}
+
+	return 0;
+}
+
+static const struct pci_epc_features*
+uniphier_pcie_get_features(struct dw_pcie_ep *ep)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci);
+
+	return priv->features;
+}
+
+static const struct dw_pcie_ep_ops uniphier_pcie_ep_ops = {
+	.ep_init = uniphier_pcie_ep_init,
+	.raise_irq = uniphier_pcie_ep_raise_irq,
+	.get_features = uniphier_pcie_get_features,
+};
+
+static int uniphier_add_pcie_ep(struct uniphier_pcie_ep_priv *priv,
+				struct platform_device *pdev)
+{
+	struct dw_pcie *pci = &priv->pci;
+	struct dw_pcie_ep *ep = &pci->ep;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int ret;
+
+	ep->ops = &uniphier_pcie_ep_ops;
+
+	pci->dbi_base2 = devm_platform_ioremap_resource_byname(pdev, "dbi2");
+	if (IS_ERR(pci->dbi_base2))
+		return PTR_ERR(pci->dbi_base2);
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
+	if (!res)
+		return -EINVAL;
+
+	ep->phys_base = res->start;
+	ep->addr_size = resource_size(res);
+
+	ret = dw_pcie_ep_init(ep);
+	if (ret)
+		dev_err(dev, "Failed to initialize endpoint (%d)\n", ret);
+
+	return ret;
+}
+
+static int uniphier_pcie_ep_enable(struct uniphier_pcie_ep_priv *priv)
+{
+	int ret;
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(priv->clk_gio);
+	if (ret)
+		goto out_clk_disable;
+
+	ret = reset_control_deassert(priv->rst);
+	if (ret)
+		goto out_clk_gio_disable;
+
+	ret = reset_control_deassert(priv->rst_gio);
+	if (ret)
+		goto out_rst_assert;
+
+	uniphier_pcie_init_ep(priv);
+
+	uniphier_pcie_phy_reset(priv, true);
+
+	ret = phy_init(priv->phy);
+	if (ret)
+		goto out_rst_gio_assert;
+
+	uniphier_pcie_phy_reset(priv, false);
+
+	return 0;
+
+out_rst_gio_assert:
+	reset_control_assert(priv->rst_gio);
+out_rst_assert:
+	reset_control_assert(priv->rst);
+out_clk_gio_disable:
+	clk_disable_unprepare(priv->clk_gio);
+out_clk_disable:
+	clk_disable_unprepare(priv->clk);
+
+	return ret;
+}
+
+static const struct dw_pcie_ops dw_pcie_ops = {
+	.start_link = uniphier_pcie_start_link,
+	.stop_link = uniphier_pcie_stop_link,
+};
+
+static int uniphier_pcie_ep_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct uniphier_pcie_ep_priv *priv;
+	struct resource *res;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->features = of_device_get_match_data(dev);
+	if (WARN_ON(!priv->features))
+		return -EINVAL;
+
+	priv->pci.dev = dev;
+	priv->pci.ops = &dw_pcie_ops;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
+	priv->pci.dbi_base = devm_pci_remap_cfg_resource(dev, res);
+	if (IS_ERR(priv->pci.dbi_base))
+		return PTR_ERR(priv->pci.dbi_base);
+
+	priv->base = devm_platform_ioremap_resource_byname(pdev, "link");
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	priv->clk_gio = devm_clk_get(dev, "gio");
+	if (IS_ERR(priv->clk_gio))
+		return PTR_ERR(priv->clk_gio);
+
+	priv->rst_gio = devm_reset_control_get_shared(dev, "gio");
+	if (IS_ERR(priv->rst_gio))
+		return PTR_ERR(priv->rst_gio);
+
+	priv->clk = devm_clk_get(dev, "link");
+	if (IS_ERR(priv->clk))
+		return PTR_ERR(priv->clk);
+
+	priv->rst = devm_reset_control_get_shared(dev, "link");
+	if (IS_ERR(priv->rst))
+		return PTR_ERR(priv->rst);
+
+	priv->phy = devm_phy_optional_get(dev, "pcie-phy");
+	if (IS_ERR(priv->phy)) {
+		ret = PTR_ERR(priv->phy);
+		dev_err(dev, "Failed to get phy (%d)\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, priv);
+
+	ret = uniphier_pcie_ep_enable(priv);
+	if (ret)
+		return ret;
+
+	return uniphier_add_pcie_ep(priv, pdev);
+}
+
+static const struct pci_epc_features uniphier_pro5_data = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = false,
+	.align = 1 << 16,
+	.bar_fixed_64bit = BIT(BAR_0) | BIT(BAR_2) | BIT(BAR_4),
+	.reserved_bar =  BIT(BAR_4),
+};
+
+static const struct of_device_id uniphier_pcie_ep_match[] = {
+	{
+		.compatible = "socionext,uniphier-pro5-pcie-ep",
+		.data = &uniphier_pro5_data,
+	},
+	{ /* sentinel */ },
+};
+
+static struct platform_driver uniphier_pcie_ep_driver = {
+	.probe  = uniphier_pcie_ep_probe,
+	.driver = {
+		.name = "uniphier-pcie-ep",
+		.of_match_table = uniphier_pcie_ep_match,
+		.suppress_bind_attrs = true,
+	},
+};
+builtin_platform_driver(uniphier_pcie_ep_driver);
diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c
index a94be264240f..5907baa9b1f2 100644
--- a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c
+++ b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c
@@ -522,9 +522,9 @@ static int mobiveil_pcie_integrated_interrupt_init(struct mobiveil_pcie *pcie)
 	mobiveil_pcie_enable_msi(pcie);
 
 	rp->irq = platform_get_irq(pdev, 0);
-	if (rp->irq <= 0) {
+	if (rp->irq < 0) {
 		dev_err(dev, "failed to map IRQ: %d\n", rp->irq);
-		return -ENODEV;
+		return rp->irq;
 	}
 
 	/* initialize the IRQ domains */
diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c
index 2a20b649f40c..90ff291c24f0 100644
--- a/drivers/pci/controller/pci-aardvark.c
+++ b/drivers/pci/controller/pci-aardvark.c
@@ -9,15 +9,18 @@
  */
 
 #include <linux/delay.h>
+#include <linux/gpio.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/init.h>
+#include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/msi.h>
 #include <linux/of_address.h>
+#include <linux/of_gpio.h>
 #include <linux/of_pci.h>
 
 #include "../pci.h"
@@ -31,16 +34,6 @@
 #define     PCIE_CORE_CMD_MEM_IO_REQ_EN				BIT(2)
 #define PCIE_CORE_DEV_REV_REG					0x8
 #define PCIE_CORE_PCIEXP_CAP					0xc0
-#define PCIE_CORE_DEV_CTRL_STATS_REG				0xc8
-#define     PCIE_CORE_DEV_CTRL_STATS_RELAX_ORDER_DISABLE	(0 << 4)
-#define     PCIE_CORE_DEV_CTRL_STATS_MAX_PAYLOAD_SZ_SHIFT	5
-#define     PCIE_CORE_DEV_CTRL_STATS_SNOOP_DISABLE		(0 << 11)
-#define     PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SIZE_SHIFT	12
-#define     PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SZ		0x2
-#define PCIE_CORE_LINK_CTRL_STAT_REG				0xd0
-#define     PCIE_CORE_LINK_L0S_ENTRY				BIT(0)
-#define     PCIE_CORE_LINK_TRAINING				BIT(5)
-#define     PCIE_CORE_LINK_WIDTH_SHIFT				20
 #define PCIE_CORE_ERR_CAPCTL_REG				0x118
 #define     PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX			BIT(5)
 #define     PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN			BIT(6)
@@ -101,6 +94,8 @@
 #define     PCIE_CORE_CTRL2_STRICT_ORDER_ENABLE	BIT(5)
 #define     PCIE_CORE_CTRL2_OB_WIN_ENABLE	BIT(6)
 #define     PCIE_CORE_CTRL2_MSI_ENABLE		BIT(10)
+#define PCIE_CORE_REF_CLK_REG			(CONTROL_BASE_ADDR + 0x14)
+#define     PCIE_CORE_REF_CLK_TX_ENABLE		BIT(1)
 #define PCIE_MSG_LOG_REG			(CONTROL_BASE_ADDR + 0x30)
 #define PCIE_ISR0_REG				(CONTROL_BASE_ADDR + 0x40)
 #define PCIE_MSG_PM_PME_MASK			BIT(7)
@@ -201,7 +196,10 @@ struct advk_pcie {
 	struct mutex msi_used_lock;
 	u16 msi_msg;
 	int root_bus_nr;
+	int link_gen;
 	struct pci_bridge_emul bridge;
+	struct gpio_desc *reset_gpio;
+	struct phy *phy;
 };
 
 static inline void advk_writel(struct advk_pcie *pcie, u32 val, u64 reg)
@@ -214,6 +212,11 @@ static inline u32 advk_readl(struct advk_pcie *pcie, u64 reg)
 	return readl(pcie->base + reg);
 }
 
+static inline u16 advk_read16(struct advk_pcie *pcie, u64 reg)
+{
+	return advk_readl(pcie, (reg & ~0x3)) >> ((reg & 0x3) * 8);
+}
+
 static int advk_pcie_link_up(struct advk_pcie *pcie)
 {
 	u32 val, ltssm_state;
@@ -225,20 +228,16 @@ static int advk_pcie_link_up(struct advk_pcie *pcie)
 
 static int advk_pcie_wait_for_link(struct advk_pcie *pcie)
 {
-	struct device *dev = &pcie->pdev->dev;
 	int retries;
 
 	/* check if the link is up or not */
 	for (retries = 0; retries < LINK_WAIT_MAX_RETRIES; retries++) {
-		if (advk_pcie_link_up(pcie)) {
-			dev_info(dev, "link up\n");
+		if (advk_pcie_link_up(pcie))
 			return 0;
-		}
 
 		usleep_range(LINK_WAIT_USLEEP_MIN, LINK_WAIT_USLEEP_MAX);
 	}
 
-	dev_err(dev, "link never came up\n");
 	return -ETIMEDOUT;
 }
 
@@ -253,10 +252,115 @@ static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie)
 	}
 }
 
+static int advk_pcie_train_at_gen(struct advk_pcie *pcie, int gen)
+{
+	int ret, neg_gen;
+	u32 reg;
+
+	/* Setup link speed */
+	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+	reg &= ~PCIE_GEN_SEL_MSK;
+	if (gen == 3)
+		reg |= SPEED_GEN_3;
+	else if (gen == 2)
+		reg |= SPEED_GEN_2;
+	else
+		reg |= SPEED_GEN_1;
+	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+	/*
+	 * Enable link training. This is not needed in every call to this
+	 * function, just once suffices, but it does not break anything either.
+	 */
+	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+	reg |= LINK_TRAINING_EN;
+	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+	/*
+	 * Start link training immediately after enabling it.
+	 * This solves problems for some buggy cards.
+	 */
+	reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL);
+	reg |= PCI_EXP_LNKCTL_RL;
+	advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL);
+
+	ret = advk_pcie_wait_for_link(pcie);
+	if (ret)
+		return ret;
+
+	reg = advk_read16(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKSTA);
+	neg_gen = reg & PCI_EXP_LNKSTA_CLS;
+
+	return neg_gen;
+}
+
+static void advk_pcie_train_link(struct advk_pcie *pcie)
+{
+	struct device *dev = &pcie->pdev->dev;
+	int neg_gen = -1, gen;
+
+	/*
+	 * Try link training at link gen specified by device tree property
+	 * 'max-link-speed'. If this fails, iteratively train at lower gen.
+	 */
+	for (gen = pcie->link_gen; gen > 0; --gen) {
+		neg_gen = advk_pcie_train_at_gen(pcie, gen);
+		if (neg_gen > 0)
+			break;
+	}
+
+	if (neg_gen < 0)
+		goto err;
+
+	/*
+	 * After successful training if negotiated gen is lower than requested,
+	 * train again on negotiated gen. This solves some stability issues for
+	 * some buggy gen1 cards.
+	 */
+	if (neg_gen < gen) {
+		gen = neg_gen;
+		neg_gen = advk_pcie_train_at_gen(pcie, gen);
+	}
+
+	if (neg_gen == gen) {
+		dev_info(dev, "link up at gen %i\n", gen);
+		return;
+	}
+
+err:
+	dev_err(dev, "link never came up\n");
+}
+
+static void advk_pcie_issue_perst(struct advk_pcie *pcie)
+{
+	u32 reg;
+
+	if (!pcie->reset_gpio)
+		return;
+
+	/* PERST does not work for some cards when link training is enabled */
+	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
+	reg &= ~LINK_TRAINING_EN;
+	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
+
+	/* 10ms delay is needed for some cards */
+	dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n");
+	gpiod_set_value_cansleep(pcie->reset_gpio, 1);
+	usleep_range(10000, 11000);
+	gpiod_set_value_cansleep(pcie->reset_gpio, 0);
+}
+
 static void advk_pcie_setup_hw(struct advk_pcie *pcie)
 {
 	u32 reg;
 
+	advk_pcie_issue_perst(pcie);
+
+	/* Enable TX */
+	reg = advk_readl(pcie, PCIE_CORE_REF_CLK_REG);
+	reg |= PCIE_CORE_REF_CLK_TX_ENABLE;
+	advk_writel(pcie, reg, PCIE_CORE_REF_CLK_REG);
+
 	/* Set to Direct mode */
 	reg = advk_readl(pcie, CTRL_CONFIG_REG);
 	reg &= ~(CTRL_MODE_MASK << CTRL_MODE_SHIFT);
@@ -275,36 +379,26 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie)
 		PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV;
 	advk_writel(pcie, reg, PCIE_CORE_ERR_CAPCTL_REG);
 
-	/* Set PCIe Device Control and Status 1 PF0 register */
-	reg = PCIE_CORE_DEV_CTRL_STATS_RELAX_ORDER_DISABLE |
-		(7 << PCIE_CORE_DEV_CTRL_STATS_MAX_PAYLOAD_SZ_SHIFT) |
-		PCIE_CORE_DEV_CTRL_STATS_SNOOP_DISABLE |
-		(PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SZ <<
-		 PCIE_CORE_DEV_CTRL_STATS_MAX_RD_REQ_SIZE_SHIFT);
-	advk_writel(pcie, reg, PCIE_CORE_DEV_CTRL_STATS_REG);
+	/* Set PCIe Device Control register */
+	reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
+	reg &= ~PCI_EXP_DEVCTL_RELAX_EN;
+	reg &= ~PCI_EXP_DEVCTL_NOSNOOP_EN;
+	reg &= ~PCI_EXP_DEVCTL_READRQ;
+	reg |= PCI_EXP_DEVCTL_PAYLOAD; /* Set max payload size */
+	reg |= PCI_EXP_DEVCTL_READRQ_512B;
+	advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL);
 
 	/* Program PCIe Control 2 to disable strict ordering */
 	reg = PCIE_CORE_CTRL2_RESERVED |
 		PCIE_CORE_CTRL2_TD_ENABLE;
 	advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG);
 
-	/* Set GEN2 */
-	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
-	reg &= ~PCIE_GEN_SEL_MSK;
-	reg |= SPEED_GEN_2;
-	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
-
 	/* Set lane X1 */
 	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
 	reg &= ~LANE_CNT_MSK;
 	reg |= LANE_COUNT_1;
 	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
 
-	/* Enable link training */
-	reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG);
-	reg |= LINK_TRAINING_EN;
-	advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG);
-
 	/* Enable MSI */
 	reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG);
 	reg |= PCIE_CORE_CTRL2_MSI_ENABLE;
@@ -340,23 +434,22 @@ static void advk_pcie_setup_hw(struct advk_pcie *pcie)
 
 	/*
 	 * PERST# signal could have been asserted by pinctrl subsystem before
-	 * probe() callback has been called, making the endpoint going into
+	 * probe() callback has been called or issued explicitly by reset gpio
+	 * function advk_pcie_issue_perst(), making the endpoint going into
 	 * fundamental reset. As required by PCI Express spec a delay for at
 	 * least 100ms after such a reset before link training is needed.
 	 */
 	msleep(PCI_PM_D3COLD_WAIT);
 
-	/* Start link training */
-	reg = advk_readl(pcie, PCIE_CORE_LINK_CTRL_STAT_REG);
-	reg |= PCIE_CORE_LINK_TRAINING;
-	advk_writel(pcie, reg, PCIE_CORE_LINK_CTRL_STAT_REG);
-
-	advk_pcie_wait_for_link(pcie);
-
-	reg = PCIE_CORE_LINK_L0S_ENTRY |
-		(1 << PCIE_CORE_LINK_WIDTH_SHIFT);
-	advk_writel(pcie, reg, PCIE_CORE_LINK_CTRL_STAT_REG);
+	advk_pcie_train_link(pcie);
 
+	/*
+	 * FIXME: The following register update is suspicious. This register is
+	 * applicable only when the PCI controller is configured for Endpoint
+	 * mode, not as a Root Complex. But apparently when this code is
+	 * removed, some cards stop working. This should be investigated and
+	 * a comment explaining this should be put here.
+	 */
 	reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG);
 	reg |= PCIE_CORE_CMD_MEM_ACCESS_EN |
 		PCIE_CORE_CMD_IO_ACCESS_EN |
@@ -952,6 +1045,62 @@ static irqreturn_t advk_pcie_irq_handler(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
+static void __maybe_unused advk_pcie_disable_phy(struct advk_pcie *pcie)
+{
+	phy_power_off(pcie->phy);
+	phy_exit(pcie->phy);
+}
+
+static int advk_pcie_enable_phy(struct advk_pcie *pcie)
+{
+	int ret;
+
+	if (!pcie->phy)
+		return 0;
+
+	ret = phy_init(pcie->phy);
+	if (ret)
+		return ret;
+
+	ret = phy_set_mode(pcie->phy, PHY_MODE_PCIE);
+	if (ret) {
+		phy_exit(pcie->phy);
+		return ret;
+	}
+
+	ret = phy_power_on(pcie->phy);
+	if (ret) {
+		phy_exit(pcie->phy);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int advk_pcie_setup_phy(struct advk_pcie *pcie)
+{
+	struct device *dev = &pcie->pdev->dev;
+	struct device_node *node = dev->of_node;
+	int ret = 0;
+
+	pcie->phy = devm_of_phy_get(dev, node, NULL);
+	if (IS_ERR(pcie->phy) && (PTR_ERR(pcie->phy) == -EPROBE_DEFER))
+		return PTR_ERR(pcie->phy);
+
+	/* Old bindings miss the PHY handle */
+	if (IS_ERR(pcie->phy)) {
+		dev_warn(dev, "PHY unavailable (%ld)\n", PTR_ERR(pcie->phy));
+		pcie->phy = NULL;
+		return 0;
+	}
+
+	ret = advk_pcie_enable_phy(pcie);
+	if (ret)
+		dev_err(dev, "Failed to initialize PHY (%d)\n", ret);
+
+	return ret;
+}
+
 static int advk_pcie_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -973,6 +1122,9 @@ static int advk_pcie_probe(struct platform_device *pdev)
 		return PTR_ERR(pcie->base);
 
 	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
 	ret = devm_request_irq(dev, irq, advk_pcie_irq_handler,
 			       IRQF_SHARED | IRQF_NO_THREAD, "advk-pcie",
 			       pcie);
@@ -989,6 +1141,32 @@ static int advk_pcie_probe(struct platform_device *pdev)
 	}
 	pcie->root_bus_nr = bus->start;
 
+	pcie->reset_gpio = devm_gpiod_get_from_of_node(dev, dev->of_node,
+						       "reset-gpios", 0,
+						       GPIOD_OUT_LOW,
+						       "pcie1-reset");
+	ret = PTR_ERR_OR_ZERO(pcie->reset_gpio);
+	if (ret) {
+		if (ret == -ENOENT) {
+			pcie->reset_gpio = NULL;
+		} else {
+			if (ret != -EPROBE_DEFER)
+				dev_err(dev, "Failed to get reset-gpio: %i\n",
+					ret);
+			return ret;
+		}
+	}
+
+	ret = of_pci_get_max_link_speed(dev->of_node);
+	if (ret <= 0 || ret > 3)
+		pcie->link_gen = 3;
+	else
+		pcie->link_gen = ret;
+
+	ret = advk_pcie_setup_phy(pcie);
+	if (ret)
+		return ret;
+
 	advk_pcie_setup_hw(pcie);
 
 	advk_sw_pci_bridge_init(pcie);
diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c
index 250a3fc80ec6..953de57f6c57 100644
--- a/drivers/pci/controller/pci-host-common.c
+++ b/drivers/pci/controller/pci-host-common.c
@@ -8,7 +8,9 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/of_pci.h>
 #include <linux/pci-ecam.h>
 #include <linux/platform_device.h>
@@ -19,7 +21,7 @@ static void gen_pci_unmap_cfg(void *ptr)
 }
 
 static struct pci_config_window *gen_pci_init(struct device *dev,
-		struct list_head *resources, struct pci_ecam_ops *ops)
+		struct list_head *resources, const struct pci_ecam_ops *ops)
 {
 	int err;
 	struct resource cfgres;
@@ -54,15 +56,19 @@ err_out:
 	return ERR_PTR(err);
 }
 
-int pci_host_common_probe(struct platform_device *pdev,
-			  struct pci_ecam_ops *ops)
+int pci_host_common_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct pci_host_bridge *bridge;
 	struct pci_config_window *cfg;
 	struct list_head resources;
+	const struct pci_ecam_ops *ops;
 	int ret;
 
+	ops = of_device_get_match_data(&pdev->dev);
+	if (!ops)
+		return -ENODEV;
+
 	bridge = devm_pci_alloc_host_bridge(dev, 0);
 	if (!bridge)
 		return -ENOMEM;
@@ -82,7 +88,7 @@ int pci_host_common_probe(struct platform_device *pdev,
 	bridge->dev.parent = dev;
 	bridge->sysdata = cfg;
 	bridge->busnr = cfg->busr.start;
-	bridge->ops = &ops->pci_ops;
+	bridge->ops = (struct pci_ops *)&ops->pci_ops;
 	bridge->map_irq = of_irq_parse_and_map_pci;
 	bridge->swizzle_irq = pci_common_swizzle;
 
@@ -95,6 +101,7 @@ int pci_host_common_probe(struct platform_device *pdev,
 	platform_set_drvdata(pdev, bridge->bus);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(pci_host_common_probe);
 
 int pci_host_common_remove(struct platform_device *pdev)
 {
@@ -107,3 +114,6 @@ int pci_host_common_remove(struct platform_device *pdev)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(pci_host_common_remove);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/controller/pci-host-generic.c b/drivers/pci/controller/pci-host-generic.c
index 75a2fb930d4b..b51977abfdf1 100644
--- a/drivers/pci/controller/pci-host-generic.c
+++ b/drivers/pci/controller/pci-host-generic.c
@@ -10,12 +10,11 @@
 
 #include <linux/kernel.h>
 #include <linux/init.h>
-#include <linux/of_address.h>
-#include <linux/of_pci.h>
+#include <linux/module.h>
 #include <linux/pci-ecam.h>
 #include <linux/platform_device.h>
 
-static struct pci_ecam_ops gen_pci_cfg_cam_bus_ops = {
+static const struct pci_ecam_ops gen_pci_cfg_cam_bus_ops = {
 	.bus_shift	= 16,
 	.pci_ops	= {
 		.map_bus	= pci_ecam_map_bus,
@@ -49,7 +48,7 @@ static void __iomem *pci_dw_ecam_map_bus(struct pci_bus *bus,
 	return pci_ecam_map_bus(bus, devfn, where);
 }
 
-static struct pci_ecam_ops pci_dw_ecam_bus_ops = {
+static const struct pci_ecam_ops pci_dw_ecam_bus_ops = {
 	.bus_shift	= 20,
 	.pci_ops	= {
 		.map_bus	= pci_dw_ecam_map_bus,
@@ -76,25 +75,16 @@ static const struct of_device_id gen_pci_of_match[] = {
 
 	{ },
 };
-
-static int gen_pci_probe(struct platform_device *pdev)
-{
-	const struct of_device_id *of_id;
-	struct pci_ecam_ops *ops;
-
-	of_id = of_match_node(gen_pci_of_match, pdev->dev.of_node);
-	ops = (struct pci_ecam_ops *)of_id->data;
-
-	return pci_host_common_probe(pdev, ops);
-}
+MODULE_DEVICE_TABLE(of, gen_pci_of_match);
 
 static struct platform_driver gen_pci_driver = {
 	.driver = {
 		.name = "pci-host-generic",
 		.of_match_table = gen_pci_of_match,
-		.suppress_bind_attrs = true,
 	},
-	.probe = gen_pci_probe,
+	.probe = pci_host_common_probe,
 	.remove = pci_host_common_remove,
 };
-builtin_platform_driver(gen_pci_driver);
+module_platform_driver(gen_pci_driver);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
index 222ff5639ebe..bf40ff09c99d 100644
--- a/drivers/pci/controller/pci-hyperv.c
+++ b/drivers/pci/controller/pci-hyperv.c
@@ -480,6 +480,9 @@ struct hv_pcibus_device {
 
 	struct workqueue_struct *wq;
 
+	/* Highest slot of child device with resources allocated */
+	int wslot_res_allocated;
+
 	/* hypercall arg, must not cross page boundary */
 	struct hv_retarget_device_interrupt retarget_msi_interrupt_params;
 
@@ -2210,10 +2213,8 @@ static void hv_pci_devices_present(struct hv_pcibus_device *hbus,
 	struct hv_dr_state *dr;
 	int i;
 
-	dr = kzalloc(offsetof(struct hv_dr_state, func) +
-		     (sizeof(struct hv_pcidev_description) *
-		      (relations->device_count)), GFP_NOWAIT);
-
+	dr = kzalloc(struct_size(dr, func, relations->device_count),
+		     GFP_NOWAIT);
 	if (!dr)
 		return;
 
@@ -2247,10 +2248,8 @@ static void hv_pci_devices_present2(struct hv_pcibus_device *hbus,
 	struct hv_dr_state *dr;
 	int i;
 
-	dr = kzalloc(offsetof(struct hv_dr_state, func) +
-		     (sizeof(struct hv_pcidev_description) *
-		      (relations->device_count)), GFP_NOWAIT);
-
+	dr = kzalloc(struct_size(dr, func, relations->device_count),
+		     GFP_NOWAIT);
 	if (!dr)
 		return;
 
@@ -2444,9 +2443,8 @@ static void hv_pci_onchannelcallback(void *context)
 
 				bus_rel = (struct pci_bus_relations *)buffer;
 				if (bytes_recvd <
-				    offsetof(struct pci_bus_relations, func) +
-				    (sizeof(struct pci_function_description) *
-				     (bus_rel->device_count))) {
+					struct_size(bus_rel, func,
+						    bus_rel->device_count)) {
 					dev_err(&hbus->hdev->device,
 						"bus relations too small\n");
 					break;
@@ -2459,9 +2457,8 @@ static void hv_pci_onchannelcallback(void *context)
 
 				bus_rel2 = (struct pci_bus_relations2 *)buffer;
 				if (bytes_recvd <
-				    offsetof(struct pci_bus_relations2, func) +
-				    (sizeof(struct pci_function_description2) *
-				     (bus_rel2->device_count))) {
+					struct_size(bus_rel2, func,
+						    bus_rel2->device_count)) {
 					dev_err(&hbus->hdev->device,
 						"bus relations v2 too small\n");
 					break;
@@ -2748,6 +2745,8 @@ static void hv_free_config_window(struct hv_pcibus_device *hbus)
 	vmbus_free_mmio(hbus->mem_config->start, PCI_CONFIG_MMIO_LENGTH);
 }
 
+static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs);
+
 /**
  * hv_pci_enter_d0() - Bring the "bus" into the D0 power state
  * @hdev:	VMBus's tracking struct for this root PCI bus
@@ -2760,8 +2759,10 @@ static int hv_pci_enter_d0(struct hv_device *hdev)
 	struct pci_bus_d0_entry *d0_entry;
 	struct hv_pci_compl comp_pkt;
 	struct pci_packet *pkt;
+	bool retry = true;
 	int ret;
 
+enter_d0_retry:
 	/*
 	 * Tell the host that the bus is ready to use, and moved into the
 	 * powered-on state.  This includes telling the host which region
@@ -2788,6 +2789,38 @@ static int hv_pci_enter_d0(struct hv_device *hdev)
 	if (ret)
 		goto exit;
 
+	/*
+	 * In certain case (Kdump) the pci device of interest was
+	 * not cleanly shut down and resource is still held on host
+	 * side, the host could return invalid device status.
+	 * We need to explicitly request host to release the resource
+	 * and try to enter D0 again.
+	 */
+	if (comp_pkt.completion_status < 0 && retry) {
+		retry = false;
+
+		dev_err(&hdev->device, "Retrying D0 Entry\n");
+
+		/*
+		 * Hv_pci_bus_exit() calls hv_send_resource_released()
+		 * to free up resources of its child devices.
+		 * In the kdump kernel we need to set the
+		 * wslot_res_allocated to 255 so it scans all child
+		 * devices to release resources allocated in the
+		 * normal kernel before panic happened.
+		 */
+		hbus->wslot_res_allocated = 255;
+
+		ret = hv_pci_bus_exit(hdev, true);
+
+		if (ret == 0) {
+			kfree(pkt);
+			goto enter_d0_retry;
+		}
+		dev_err(&hdev->device,
+			"Retrying D0 failed with ret %d\n", ret);
+	}
+
 	if (comp_pkt.completion_status < 0) {
 		dev_err(&hdev->device,
 			"PCI Pass-through VSP failed D0 Entry with status %x\n",
@@ -2859,7 +2892,7 @@ static int hv_send_resources_allocated(struct hv_device *hdev)
 	struct hv_pci_dev *hpdev;
 	struct pci_packet *pkt;
 	size_t size_res;
-	u32 wslot;
+	int wslot;
 	int ret;
 
 	size_res = (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2)
@@ -2912,6 +2945,8 @@ static int hv_send_resources_allocated(struct hv_device *hdev)
 				comp_pkt.completion_status);
 			break;
 		}
+
+		hbus->wslot_res_allocated = wslot;
 	}
 
 	kfree(pkt);
@@ -2930,10 +2965,10 @@ static int hv_send_resources_released(struct hv_device *hdev)
 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
 	struct pci_child_message pkt;
 	struct hv_pci_dev *hpdev;
-	u32 wslot;
+	int wslot;
 	int ret;
 
-	for (wslot = 0; wslot < 256; wslot++) {
+	for (wslot = hbus->wslot_res_allocated; wslot >= 0; wslot--) {
 		hpdev = get_pcichild_wslot(hbus, wslot);
 		if (!hpdev)
 			continue;
@@ -2948,8 +2983,12 @@ static int hv_send_resources_released(struct hv_device *hdev)
 				       VM_PKT_DATA_INBAND, 0);
 		if (ret)
 			return ret;
+
+		hbus->wslot_res_allocated = wslot - 1;
 	}
 
+	hbus->wslot_res_allocated = -1;
+
 	return 0;
 }
 
@@ -3049,6 +3088,7 @@ static int hv_pci_probe(struct hv_device *hdev,
 	if (!hbus)
 		return -ENOMEM;
 	hbus->state = hv_pcibus_init;
+	hbus->wslot_res_allocated = -1;
 
 	/*
 	 * The PCI bus "domain" is what is called "segment" in ACPI and other
@@ -3148,7 +3188,7 @@ static int hv_pci_probe(struct hv_device *hdev,
 
 	ret = hv_pci_allocate_bridge_windows(hbus);
 	if (ret)
-		goto free_irq_domain;
+		goto exit_d0;
 
 	ret = hv_send_resources_allocated(hdev);
 	if (ret)
@@ -3166,6 +3206,8 @@ static int hv_pci_probe(struct hv_device *hdev,
 
 free_windows:
 	hv_pci_free_bridge_windows(hbus);
+exit_d0:
+	(void) hv_pci_bus_exit(hdev, true);
 free_irq_domain:
 	irq_domain_remove(hbus->irq_domain);
 free_fwnode:
@@ -3185,7 +3227,7 @@ free_bus:
 	return ret;
 }
 
-static int hv_pci_bus_exit(struct hv_device *hdev, bool hibernating)
+static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs)
 {
 	struct hv_pcibus_device *hbus = hv_get_drvdata(hdev);
 	struct {
@@ -3203,7 +3245,7 @@ static int hv_pci_bus_exit(struct hv_device *hdev, bool hibernating)
 	if (hdev->channel->rescind)
 		return 0;
 
-	if (!hibernating) {
+	if (!keep_devs) {
 		/* Delete any children which might still exist. */
 		dr = kzalloc(sizeof(*dr), GFP_KERNEL);
 		if (dr && hv_pci_start_relations_work(hbus, dr))
diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c
index 3e64ba6a36a8..235b456698fc 100644
--- a/drivers/pci/controller/pci-tegra.c
+++ b/drivers/pci/controller/pci-tegra.c
@@ -2219,8 +2219,8 @@ static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
 			if (PTR_ERR(rp->reset_gpio) == -ENOENT) {
 				rp->reset_gpio = NULL;
 			} else {
-				dev_err(dev, "failed to get reset GPIO: %d\n",
-					err);
+				dev_err(dev, "failed to get reset GPIO: %ld\n",
+					PTR_ERR(rp->reset_gpio));
 				return PTR_ERR(rp->reset_gpio);
 			}
 		}
@@ -2712,7 +2712,7 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 	err = pm_runtime_get_sync(pcie->dev);
 	if (err < 0) {
 		dev_err(dev, "fail to enable pcie controller: %d\n", err);
-		goto teardown_msi;
+		goto pm_runtime_put;
 	}
 
 	host->busnr = bus->start;
@@ -2746,7 +2746,6 @@ static int tegra_pcie_probe(struct platform_device *pdev)
 pm_runtime_put:
 	pm_runtime_put_sync(pcie->dev);
 	pm_runtime_disable(pcie->dev);
-teardown_msi:
 	tegra_pcie_msi_teardown(pcie);
 put_resources:
 	tegra_pcie_put_resources(pcie);
diff --git a/drivers/pci/controller/pci-thunder-ecam.c b/drivers/pci/controller/pci-thunder-ecam.c
index 32d1d7b81ef4..7e8835fee5f7 100644
--- a/drivers/pci/controller/pci-thunder-ecam.c
+++ b/drivers/pci/controller/pci-thunder-ecam.c
@@ -345,7 +345,7 @@ static int thunder_ecam_config_write(struct pci_bus *bus, unsigned int devfn,
 	return pci_generic_config_write(bus, devfn, where, size, val);
 }
 
-struct pci_ecam_ops pci_thunder_ecam_ops = {
+const struct pci_ecam_ops pci_thunder_ecam_ops = {
 	.bus_shift	= 20,
 	.pci_ops	= {
 		.map_bus        = pci_ecam_map_bus,
@@ -357,22 +357,20 @@ struct pci_ecam_ops pci_thunder_ecam_ops = {
 #ifdef CONFIG_PCI_HOST_THUNDER_ECAM
 
 static const struct of_device_id thunder_ecam_of_match[] = {
-	{ .compatible = "cavium,pci-host-thunder-ecam" },
+	{
+		.compatible = "cavium,pci-host-thunder-ecam",
+		.data = &pci_thunder_ecam_ops,
+	},
 	{ },
 };
 
-static int thunder_ecam_probe(struct platform_device *pdev)
-{
-	return pci_host_common_probe(pdev, &pci_thunder_ecam_ops);
-}
-
 static struct platform_driver thunder_ecam_driver = {
 	.driver = {
 		.name = KBUILD_MODNAME,
 		.of_match_table = thunder_ecam_of_match,
 		.suppress_bind_attrs = true,
 	},
-	.probe = thunder_ecam_probe,
+	.probe = pci_host_common_probe,
 };
 builtin_platform_driver(thunder_ecam_driver);
 
diff --git a/drivers/pci/controller/pci-thunder-pem.c b/drivers/pci/controller/pci-thunder-pem.c
index 9491e266b1ea..3f847969143e 100644
--- a/drivers/pci/controller/pci-thunder-pem.c
+++ b/drivers/pci/controller/pci-thunder-pem.c
@@ -403,7 +403,7 @@ static int thunder_pem_acpi_init(struct pci_config_window *cfg)
 	return thunder_pem_init(dev, cfg, res_pem);
 }
 
-struct pci_ecam_ops thunder_pem_ecam_ops = {
+const struct pci_ecam_ops thunder_pem_ecam_ops = {
 	.bus_shift	= 24,
 	.init		= thunder_pem_acpi_init,
 	.pci_ops	= {
@@ -440,7 +440,7 @@ static int thunder_pem_platform_init(struct pci_config_window *cfg)
 	return thunder_pem_init(dev, cfg, res_pem);
 }
 
-static struct pci_ecam_ops pci_thunder_pem_ops = {
+static const struct pci_ecam_ops pci_thunder_pem_ops = {
 	.bus_shift	= 24,
 	.init		= thunder_pem_platform_init,
 	.pci_ops	= {
@@ -451,22 +451,20 @@ static struct pci_ecam_ops pci_thunder_pem_ops = {
 };
 
 static const struct of_device_id thunder_pem_of_match[] = {
-	{ .compatible = "cavium,pci-host-thunder-pem" },
+	{
+		.compatible = "cavium,pci-host-thunder-pem",
+		.data = &pci_thunder_pem_ops,
+	},
 	{ },
 };
 
-static int thunder_pem_probe(struct platform_device *pdev)
-{
-	return pci_host_common_probe(pdev, &pci_thunder_pem_ops);
-}
-
 static struct platform_driver thunder_pem_driver = {
 	.driver = {
 		.name = KBUILD_MODNAME,
 		.of_match_table = thunder_pem_of_match,
 		.suppress_bind_attrs = true,
 	},
-	.probe = thunder_pem_probe,
+	.probe = pci_host_common_probe,
 };
 builtin_platform_driver(thunder_pem_driver);
 
diff --git a/drivers/pci/controller/pci-v3-semi.c b/drivers/pci/controller/pci-v3-semi.c
index bd05221f5a22..3681e5af3878 100644
--- a/drivers/pci/controller/pci-v3-semi.c
+++ b/drivers/pci/controller/pci-v3-semi.c
@@ -720,7 +720,7 @@ static int v3_pci_probe(struct platform_device *pdev)
 	int irq;
 	int ret;
 
-	host = pci_alloc_host_bridge(sizeof(*v3));
+	host = devm_pci_alloc_host_bridge(dev, sizeof(*v3));
 	if (!host)
 		return -ENOMEM;
 
@@ -777,9 +777,9 @@ static int v3_pci_probe(struct platform_device *pdev)
 
 	/* Get and request error IRQ resource */
 	irq = platform_get_irq(pdev, 0);
-	if (irq <= 0) {
+	if (irq < 0) {
 		dev_err(dev, "unable to obtain PCIv3 error IRQ\n");
-		return -ENODEV;
+		return irq;
 	}
 	ret = devm_request_irq(dev, irq, v3_irq, 0,
 			"PCIv3 error", v3);
diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c
index de195fd430dc..d1efa8ffbae1 100644
--- a/drivers/pci/controller/pci-xgene.c
+++ b/drivers/pci/controller/pci-xgene.c
@@ -256,7 +256,7 @@ static int xgene_v1_pcie_ecam_init(struct pci_config_window *cfg)
 	return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_1);
 }
 
-struct pci_ecam_ops xgene_v1_pcie_ecam_ops = {
+const struct pci_ecam_ops xgene_v1_pcie_ecam_ops = {
 	.bus_shift	= 16,
 	.init		= xgene_v1_pcie_ecam_init,
 	.pci_ops	= {
@@ -271,7 +271,7 @@ static int xgene_v2_pcie_ecam_init(struct pci_config_window *cfg)
 	return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_2);
 }
 
-struct pci_ecam_ops xgene_v2_pcie_ecam_ops = {
+const struct pci_ecam_ops xgene_v2_pcie_ecam_ops = {
 	.bus_shift	= 16,
 	.init		= xgene_v2_pcie_ecam_init,
 	.pci_ops	= {
diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c
index b447c3e4abad..24cb1c331058 100644
--- a/drivers/pci/controller/pcie-altera.c
+++ b/drivers/pci/controller/pcie-altera.c
@@ -193,7 +193,7 @@ static bool altera_pcie_valid_device(struct altera_pcie *pcie,
 	if (bus->number == pcie->root_bus_nr && dev > 0)
 		return false;
 
-	 return true;
+	return true;
 }
 
 static int tlp_read_packet(struct altera_pcie *pcie, u32 *value)
diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c
index 6d79d14527a6..7730ea845ff2 100644
--- a/drivers/pci/controller/pcie-brcmstb.c
+++ b/drivers/pci/controller/pcie-brcmstb.c
@@ -28,6 +28,8 @@
 #include <linux/string.h>
 #include <linux/types.h>
 
+#include <soc/bcm2835/raspberrypi-firmware.h>
+
 #include "../pci.h"
 
 /* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */
@@ -41,6 +43,9 @@
 #define PCIE_RC_CFG_PRIV1_ID_VAL3			0x043c
 #define  PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK	0xffffff
 
+#define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY			0x04dc
+#define  PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK	0xc00
+
 #define PCIE_RC_DL_MDIO_ADDR				0x1100
 #define PCIE_RC_DL_MDIO_WR_DATA				0x1104
 #define PCIE_RC_DL_MDIO_RD_DATA				0x1108
@@ -54,11 +59,11 @@
 
 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO		0x400c
 #define PCIE_MEM_WIN0_LO(win)	\
-		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 4)
+		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8)
 
 #define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI		0x4010
 #define PCIE_MEM_WIN0_HI(win)	\
-		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 4)
+		PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8)
 
 #define PCIE_MISC_RC_BAR1_CONFIG_LO			0x402c
 #define  PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK		0x1f
@@ -693,10 +698,11 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie)
 	int num_out_wins = 0;
 	u16 nlw, cls, lnksta;
 	int i, ret;
-	u32 tmp;
+	u32 tmp, aspm_support;
 
 	/* Reset the bridge */
 	brcm_pcie_bridge_sw_init_set(pcie, 1);
+	brcm_pcie_perst_set(pcie, 1);
 
 	usleep_range(100, 200);
 
@@ -803,6 +809,15 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie)
 		num_out_wins++;
 	}
 
+	/* Don't advertise L0s capability if 'aspm-no-l0s' */
+	aspm_support = PCIE_LINK_STATE_L1;
+	if (!of_property_read_bool(pcie->np, "aspm-no-l0s"))
+		aspm_support |= PCIE_LINK_STATE_L0S;
+	tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
+	u32p_replace_bits(&tmp, aspm_support,
+		PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK);
+	writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY);
+
 	/*
 	 * For config space accesses on the RC, show the right class for
 	 * a PCIe-PCIe bridge (the default setting is to be EP mode).
@@ -899,7 +914,6 @@ static void __brcm_pcie_remove(struct brcm_pcie *pcie)
 	brcm_msi_remove(pcie);
 	brcm_pcie_turn_off(pcie);
 	clk_disable_unprepare(pcie->clk);
-	clk_put(pcie->clk);
 }
 
 static int brcm_pcie_remove(struct platform_device *pdev)
@@ -917,11 +931,26 @@ static int brcm_pcie_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node, *msi_np;
 	struct pci_host_bridge *bridge;
+	struct device_node *fw_np;
 	struct brcm_pcie *pcie;
 	struct pci_bus *child;
 	struct resource *res;
 	int ret;
 
+	/*
+	 * We have to wait for Raspberry Pi's firmware interface to be up as a
+	 * PCI fixup, rpi_firmware_init_vl805(), depends on it. This driver's
+	 * probe can race with the firmware interface's (see
+	 * drivers/firmware/raspberrypi.c) and potentially break the PCI fixup.
+	 */
+	fw_np = of_find_compatible_node(NULL, NULL,
+					"raspberrypi,bcm2835-firmware");
+	if (fw_np && !rpi_firmware_get(fw_np)) {
+		of_node_put(fw_np);
+		return -EPROBE_DEFER;
+	}
+	of_node_put(fw_np);
+
 	bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie));
 	if (!bridge)
 		return -ENOMEM;
diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
index cb982891b22b..ebfa7d5a4e2d 100644
--- a/drivers/pci/controller/pcie-mediatek.c
+++ b/drivers/pci/controller/pcie-mediatek.c
@@ -651,6 +651,9 @@ static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
 	}
 
 	port->irq = platform_get_irq(pdev, port->slot);
+	if (port->irq < 0)
+		return port->irq;
+
 	irq_set_chained_handler_and_data(port->irq,
 					 mtk_pcie_intr_handler, port);
 
diff --git a/drivers/pci/controller/pcie-rcar-ep.c b/drivers/pci/controller/pcie-rcar-ep.c
new file mode 100644
index 000000000000..b4a288e24aaf
--- /dev/null
+++ b/drivers/pci/controller/pcie-rcar-ep.c
@@ -0,0 +1,563 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PCIe endpoint driver for Renesas R-Car SoCs
+ *  Copyright (c) 2020 Renesas Electronics Europe GmbH
+ *
+ * Author: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/pci.h>
+#include <linux/pci-epc.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+#include "pcie-rcar.h"
+
+#define RCAR_EPC_MAX_FUNCTIONS		1
+
+/* Structure representing the PCIe interface */
+struct rcar_pcie_endpoint {
+	struct rcar_pcie	pcie;
+	phys_addr_t		*ob_mapped_addr;
+	struct pci_epc_mem_window *ob_window;
+	u8			max_functions;
+	unsigned int		bar_to_atu[MAX_NR_INBOUND_MAPS];
+	unsigned long		*ib_window_map;
+	u32			num_ib_windows;
+	u32			num_ob_windows;
+};
+
+static void rcar_pcie_ep_hw_init(struct rcar_pcie *pcie)
+{
+	u32 val;
+
+	rcar_pci_write_reg(pcie, 0, PCIETCTLR);
+
+	/* Set endpoint mode */
+	rcar_pci_write_reg(pcie, 0, PCIEMSR);
+
+	/* Initialize default capabilities. */
+	rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP);
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS),
+		   PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ENDPOINT << 4);
+	rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f,
+		   PCI_HEADER_TYPE_NORMAL);
+
+	/* Write out the physical slot number = 0 */
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0);
+
+	val = rcar_pci_read_reg(pcie, EXPCAP(1));
+	/* device supports fixed 128 bytes MPSS */
+	val &= ~GENMASK(2, 0);
+	rcar_pci_write_reg(pcie, val, EXPCAP(1));
+
+	val = rcar_pci_read_reg(pcie, EXPCAP(2));
+	/* read requests size 128 bytes */
+	val &= ~GENMASK(14, 12);
+	/* payload size 128 bytes */
+	val &= ~GENMASK(7, 5);
+	rcar_pci_write_reg(pcie, val, EXPCAP(2));
+
+	/* Set target link speed to 5.0 GT/s */
+	rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS,
+		   PCI_EXP_LNKSTA_CLS_5_0GB);
+
+	/* Set the completion timer timeout to the maximum 50ms. */
+	rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50);
+
+	/* Terminate list of capabilities (Next Capability Offset=0) */
+	rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0);
+
+	/* flush modifications */
+	wmb();
+}
+
+static int rcar_pcie_ep_get_window(struct rcar_pcie_endpoint *ep,
+				   phys_addr_t addr)
+{
+	int i;
+
+	for (i = 0; i < ep->num_ob_windows; i++)
+		if (ep->ob_window[i].phys_base == addr)
+			return i;
+
+	return -EINVAL;
+}
+
+static int rcar_pcie_parse_outbound_ranges(struct rcar_pcie_endpoint *ep,
+					   struct platform_device *pdev)
+{
+	struct rcar_pcie *pcie = &ep->pcie;
+	char outbound_name[10];
+	struct resource *res;
+	unsigned int i = 0;
+
+	ep->num_ob_windows = 0;
+	for (i = 0; i < RCAR_PCI_MAX_RESOURCES; i++) {
+		sprintf(outbound_name, "memory%u", i);
+		res = platform_get_resource_byname(pdev,
+						   IORESOURCE_MEM,
+						   outbound_name);
+		if (!res) {
+			dev_err(pcie->dev, "missing outbound window %u\n", i);
+			return -EINVAL;
+		}
+		if (!devm_request_mem_region(&pdev->dev, res->start,
+					     resource_size(res),
+					     outbound_name)) {
+			dev_err(pcie->dev, "Cannot request memory region %s.\n",
+				outbound_name);
+			return -EIO;
+		}
+
+		ep->ob_window[i].phys_base = res->start;
+		ep->ob_window[i].size = resource_size(res);
+		/* controller doesn't support multiple allocation
+		 * from same window, so set page_size to window size
+		 */
+		ep->ob_window[i].page_size = resource_size(res);
+	}
+	ep->num_ob_windows = i;
+
+	return 0;
+}
+
+static int rcar_pcie_ep_get_pdata(struct rcar_pcie_endpoint *ep,
+				  struct platform_device *pdev)
+{
+	struct rcar_pcie *pcie = &ep->pcie;
+	struct pci_epc_mem_window *window;
+	struct device *dev = pcie->dev;
+	struct resource res;
+	int err;
+
+	err = of_address_to_resource(dev->of_node, 0, &res);
+	if (err)
+		return err;
+	pcie->base = devm_ioremap_resource(dev, &res);
+	if (IS_ERR(pcie->base))
+		return PTR_ERR(pcie->base);
+
+	ep->ob_window = devm_kcalloc(dev, RCAR_PCI_MAX_RESOURCES,
+				     sizeof(*window), GFP_KERNEL);
+	if (!ep->ob_window)
+		return -ENOMEM;
+
+	rcar_pcie_parse_outbound_ranges(ep, pdev);
+
+	err = of_property_read_u8(dev->of_node, "max-functions",
+				  &ep->max_functions);
+	if (err < 0 || ep->max_functions > RCAR_EPC_MAX_FUNCTIONS)
+		ep->max_functions = RCAR_EPC_MAX_FUNCTIONS;
+
+	return 0;
+}
+
+static int rcar_pcie_ep_write_header(struct pci_epc *epc, u8 fn,
+				     struct pci_epf_header *hdr)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+	struct rcar_pcie *pcie = &ep->pcie;
+	u32 val;
+
+	if (!fn)
+		val = hdr->vendorid;
+	else
+		val = rcar_pci_read_reg(pcie, IDSETR0);
+	val |= hdr->deviceid << 16;
+	rcar_pci_write_reg(pcie, val, IDSETR0);
+
+	val = hdr->revid;
+	val |= hdr->progif_code << 8;
+	val |= hdr->subclass_code << 16;
+	val |= hdr->baseclass_code << 24;
+	rcar_pci_write_reg(pcie, val, IDSETR1);
+
+	if (!fn)
+		val = hdr->subsys_vendor_id;
+	else
+		val = rcar_pci_read_reg(pcie, SUBIDSETR);
+	val |= hdr->subsys_id << 16;
+	rcar_pci_write_reg(pcie, val, SUBIDSETR);
+
+	if (hdr->interrupt_pin > PCI_INTERRUPT_INTA)
+		return -EINVAL;
+	val = rcar_pci_read_reg(pcie, PCICONF(15));
+	val |= (hdr->interrupt_pin << 8);
+	rcar_pci_write_reg(pcie, val, PCICONF(15));
+
+	return 0;
+}
+
+static int rcar_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
+				struct pci_epf_bar *epf_bar)
+{
+	int flags = epf_bar->flags | LAR_ENABLE | LAM_64BIT;
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+	u64 size = 1ULL << fls64(epf_bar->size - 1);
+	dma_addr_t cpu_addr = epf_bar->phys_addr;
+	enum pci_barno bar = epf_bar->barno;
+	struct rcar_pcie *pcie = &ep->pcie;
+	u32 mask;
+	int idx;
+	int err;
+
+	idx = find_first_zero_bit(ep->ib_window_map, ep->num_ib_windows);
+	if (idx >= ep->num_ib_windows) {
+		dev_err(pcie->dev, "no free inbound window\n");
+		return -EINVAL;
+	}
+
+	if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO)
+		flags |= IO_SPACE;
+
+	ep->bar_to_atu[bar] = idx;
+	/* use 64-bit BARs */
+	set_bit(idx, ep->ib_window_map);
+	set_bit(idx + 1, ep->ib_window_map);
+
+	if (cpu_addr > 0) {
+		unsigned long nr_zeros = __ffs64(cpu_addr);
+		u64 alignment = 1ULL << nr_zeros;
+
+		size = min(size, alignment);
+	}
+
+	size = min(size, 1ULL << 32);
+
+	mask = roundup_pow_of_two(size) - 1;
+	mask &= ~0xf;
+
+	rcar_pcie_set_inbound(pcie, cpu_addr,
+			      0x0, mask | flags, idx, false);
+
+	err = rcar_pcie_wait_for_phyrdy(pcie);
+	if (err) {
+		dev_err(pcie->dev, "phy not ready\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void rcar_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn,
+				   struct pci_epf_bar *epf_bar)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+	enum pci_barno bar = epf_bar->barno;
+	u32 atu_index = ep->bar_to_atu[bar];
+
+	rcar_pcie_set_inbound(&ep->pcie, 0x0, 0x0, 0x0, bar, false);
+
+	clear_bit(atu_index, ep->ib_window_map);
+	clear_bit(atu_index + 1, ep->ib_window_map);
+}
+
+static int rcar_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 interrupts)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+	struct rcar_pcie *pcie = &ep->pcie;
+	u32 flags;
+
+	flags = rcar_pci_read_reg(pcie, MSICAP(fn));
+	flags |= interrupts << MSICAP0_MMESCAP_OFFSET;
+	rcar_pci_write_reg(pcie, flags, MSICAP(fn));
+
+	return 0;
+}
+
+static int rcar_pcie_ep_get_msi(struct pci_epc *epc, u8 fn)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+	struct rcar_pcie *pcie = &ep->pcie;
+	u32 flags;
+
+	flags = rcar_pci_read_reg(pcie, MSICAP(fn));
+	if (!(flags & MSICAP0_MSIE))
+		return -EINVAL;
+
+	return ((flags & MSICAP0_MMESE_MASK) >> MSICAP0_MMESE_OFFSET);
+}
+
+static int rcar_pcie_ep_map_addr(struct pci_epc *epc, u8 fn,
+				 phys_addr_t addr, u64 pci_addr, size_t size)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+	struct rcar_pcie *pcie = &ep->pcie;
+	struct resource_entry win;
+	struct resource res;
+	int window;
+	int err;
+
+	/* check if we have a link. */
+	err = rcar_pcie_wait_for_dl(pcie);
+	if (err) {
+		dev_err(pcie->dev, "link not up\n");
+		return err;
+	}
+
+	window = rcar_pcie_ep_get_window(ep, addr);
+	if (window < 0) {
+		dev_err(pcie->dev, "failed to get corresponding window\n");
+		return -EINVAL;
+	}
+
+	memset(&win, 0x0, sizeof(win));
+	memset(&res, 0x0, sizeof(res));
+	res.start = pci_addr;
+	res.end = pci_addr + size - 1;
+	res.flags = IORESOURCE_MEM;
+	win.res = &res;
+
+	rcar_pcie_set_outbound(pcie, window, &win);
+
+	ep->ob_mapped_addr[window] = addr;
+
+	return 0;
+}
+
+static void rcar_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn,
+				    phys_addr_t addr)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+	struct resource_entry win;
+	struct resource res;
+	int idx;
+
+	for (idx = 0; idx < ep->num_ob_windows; idx++)
+		if (ep->ob_mapped_addr[idx] == addr)
+			break;
+
+	if (idx >= ep->num_ob_windows)
+		return;
+
+	memset(&win, 0x0, sizeof(win));
+	memset(&res, 0x0, sizeof(res));
+	win.res = &res;
+	rcar_pcie_set_outbound(&ep->pcie, idx, &win);
+
+	ep->ob_mapped_addr[idx] = 0;
+}
+
+static int rcar_pcie_ep_assert_intx(struct rcar_pcie_endpoint *ep,
+				    u8 fn, u8 intx)
+{
+	struct rcar_pcie *pcie = &ep->pcie;
+	u32 val;
+
+	val = rcar_pci_read_reg(pcie, PCIEMSITXR);
+	if ((val & PCI_MSI_FLAGS_ENABLE)) {
+		dev_err(pcie->dev, "MSI is enabled, cannot assert INTx\n");
+		return -EINVAL;
+	}
+
+	val = rcar_pci_read_reg(pcie, PCICONF(1));
+	if ((val & INTDIS)) {
+		dev_err(pcie->dev, "INTx message transmission is disabled\n");
+		return -EINVAL;
+	}
+
+	val = rcar_pci_read_reg(pcie, PCIEINTXR);
+	if ((val & ASTINTX)) {
+		dev_err(pcie->dev, "INTx is already asserted\n");
+		return -EINVAL;
+	}
+
+	val |= ASTINTX;
+	rcar_pci_write_reg(pcie, val, PCIEINTXR);
+	usleep_range(1000, 1001);
+	val = rcar_pci_read_reg(pcie, PCIEINTXR);
+	val &= ~ASTINTX;
+	rcar_pci_write_reg(pcie, val, PCIEINTXR);
+
+	return 0;
+}
+
+static int rcar_pcie_ep_assert_msi(struct rcar_pcie *pcie,
+				   u8 fn, u8 interrupt_num)
+{
+	u16 msi_count;
+	u32 val;
+
+	/* Check MSI enable bit */
+	val = rcar_pci_read_reg(pcie, MSICAP(fn));
+	if (!(val & MSICAP0_MSIE))
+		return -EINVAL;
+
+	/* Get MSI numbers from MME */
+	msi_count = ((val & MSICAP0_MMESE_MASK) >> MSICAP0_MMESE_OFFSET);
+	msi_count = 1 << msi_count;
+
+	if (!interrupt_num || interrupt_num > msi_count)
+		return -EINVAL;
+
+	val = rcar_pci_read_reg(pcie, PCIEMSITXR);
+	rcar_pci_write_reg(pcie, val | (interrupt_num - 1), PCIEMSITXR);
+
+	return 0;
+}
+
+static int rcar_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn,
+				  enum pci_epc_irq_type type,
+				  u16 interrupt_num)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+
+	switch (type) {
+	case PCI_EPC_IRQ_LEGACY:
+		return rcar_pcie_ep_assert_intx(ep, fn, 0);
+
+	case PCI_EPC_IRQ_MSI:
+		return rcar_pcie_ep_assert_msi(&ep->pcie, fn, interrupt_num);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int rcar_pcie_ep_start(struct pci_epc *epc)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+
+	rcar_pci_write_reg(&ep->pcie, MACCTLR_INIT_VAL, MACCTLR);
+	rcar_pci_write_reg(&ep->pcie, CFINIT, PCIETCTLR);
+
+	return 0;
+}
+
+static void rcar_pcie_ep_stop(struct pci_epc *epc)
+{
+	struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc);
+
+	rcar_pci_write_reg(&ep->pcie, 0, PCIETCTLR);
+}
+
+static const struct pci_epc_features rcar_pcie_epc_features = {
+	.linkup_notifier = false,
+	.msi_capable = true,
+	.msix_capable = false,
+	/* use 64-bit BARs so mark BAR[1,3,5] as reserved */
+	.reserved_bar = 1 << BAR_1 | 1 << BAR_3 | 1 << BAR_5,
+	.bar_fixed_64bit = 1 << BAR_0 | 1 << BAR_2 | 1 << BAR_4,
+	.bar_fixed_size[0] = 128,
+	.bar_fixed_size[2] = 256,
+	.bar_fixed_size[4] = 256,
+};
+
+static const struct pci_epc_features*
+rcar_pcie_ep_get_features(struct pci_epc *epc, u8 func_no)
+{
+	return &rcar_pcie_epc_features;
+}
+
+static const struct pci_epc_ops rcar_pcie_epc_ops = {
+	.write_header	= rcar_pcie_ep_write_header,
+	.set_bar	= rcar_pcie_ep_set_bar,
+	.clear_bar	= rcar_pcie_ep_clear_bar,
+	.set_msi	= rcar_pcie_ep_set_msi,
+	.get_msi	= rcar_pcie_ep_get_msi,
+	.map_addr	= rcar_pcie_ep_map_addr,
+	.unmap_addr	= rcar_pcie_ep_unmap_addr,
+	.raise_irq	= rcar_pcie_ep_raise_irq,
+	.start		= rcar_pcie_ep_start,
+	.stop		= rcar_pcie_ep_stop,
+	.get_features	= rcar_pcie_ep_get_features,
+};
+
+static const struct of_device_id rcar_pcie_ep_of_match[] = {
+	{ .compatible = "renesas,r8a774c0-pcie-ep", },
+	{ .compatible = "renesas,rcar-gen3-pcie-ep" },
+	{ },
+};
+
+static int rcar_pcie_ep_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rcar_pcie_endpoint *ep;
+	struct rcar_pcie *pcie;
+	struct pci_epc *epc;
+	int err;
+
+	ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL);
+	if (!ep)
+		return -ENOMEM;
+
+	pcie = &ep->pcie;
+	pcie->dev = dev;
+
+	pm_runtime_enable(dev);
+	err = pm_runtime_get_sync(dev);
+	if (err < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed\n");
+		goto err_pm_disable;
+	}
+
+	err = rcar_pcie_ep_get_pdata(ep, pdev);
+	if (err < 0) {
+		dev_err(dev, "failed to request resources: %d\n", err);
+		goto err_pm_put;
+	}
+
+	ep->num_ib_windows = MAX_NR_INBOUND_MAPS;
+	ep->ib_window_map =
+			devm_kcalloc(dev, BITS_TO_LONGS(ep->num_ib_windows),
+				     sizeof(long), GFP_KERNEL);
+	if (!ep->ib_window_map) {
+		err = -ENOMEM;
+		dev_err(dev, "failed to allocate memory for inbound map\n");
+		goto err_pm_put;
+	}
+
+	ep->ob_mapped_addr = devm_kcalloc(dev, ep->num_ob_windows,
+					  sizeof(*ep->ob_mapped_addr),
+					  GFP_KERNEL);
+	if (!ep->ob_mapped_addr) {
+		err = -ENOMEM;
+		dev_err(dev, "failed to allocate memory for outbound memory pointers\n");
+		goto err_pm_put;
+	}
+
+	epc = devm_pci_epc_create(dev, &rcar_pcie_epc_ops);
+	if (IS_ERR(epc)) {
+		dev_err(dev, "failed to create epc device\n");
+		err = PTR_ERR(epc);
+		goto err_pm_put;
+	}
+
+	epc->max_functions = ep->max_functions;
+	epc_set_drvdata(epc, ep);
+
+	rcar_pcie_ep_hw_init(pcie);
+
+	err = pci_epc_multi_mem_init(epc, ep->ob_window, ep->num_ob_windows);
+	if (err < 0) {
+		dev_err(dev, "failed to initialize the epc memory space\n");
+		goto err_pm_put;
+	}
+
+	return 0;
+
+err_pm_put:
+	pm_runtime_put(dev);
+
+err_pm_disable:
+	pm_runtime_disable(dev);
+
+	return err;
+}
+
+static struct platform_driver rcar_pcie_ep_driver = {
+	.driver = {
+		.name = "rcar-pcie-ep",
+		.of_match_table = rcar_pcie_ep_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe = rcar_pcie_ep_probe,
+};
+builtin_platform_driver(rcar_pcie_ep_driver);
diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c
new file mode 100644
index 000000000000..d210a36561be
--- /dev/null
+++ b/drivers/pci/controller/pcie-rcar-host.c
@@ -0,0 +1,1130 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PCIe driver for Renesas R-Car SoCs
+ *  Copyright (C) 2014-2020 Renesas Electronics Europe Ltd
+ *
+ * Based on:
+ *  arch/sh/drivers/pci/pcie-sh7786.c
+ *  arch/sh/drivers/pci/ops-sh7786.c
+ *  Copyright (C) 2009 - 2011  Paul Mundt
+ *
+ * Author: Phil Edworthy <phil.edworthy@renesas.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/msi.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_pci.h>
+#include <linux/of_platform.h>
+#include <linux/pci.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+
+#include "pcie-rcar.h"
+
+struct rcar_msi {
+	DECLARE_BITMAP(used, INT_PCI_MSI_NR);
+	struct irq_domain *domain;
+	struct msi_controller chip;
+	unsigned long pages;
+	struct mutex lock;
+	int irq1;
+	int irq2;
+};
+
+static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip)
+{
+	return container_of(chip, struct rcar_msi, chip);
+}
+
+/* Structure representing the PCIe interface */
+struct rcar_pcie_host {
+	struct rcar_pcie	pcie;
+	struct device		*dev;
+	struct phy		*phy;
+	void __iomem		*base;
+	struct list_head	resources;
+	int			root_bus_nr;
+	struct clk		*bus_clk;
+	struct			rcar_msi msi;
+	int			(*phy_init_fn)(struct rcar_pcie_host *host);
+};
+
+static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
+{
+	unsigned int shift = BITS_PER_BYTE * (where & 3);
+	u32 val = rcar_pci_read_reg(pcie, where & ~3);
+
+	return val >> shift;
+}
+
+/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
+static int rcar_pcie_config_access(struct rcar_pcie_host *host,
+		unsigned char access_type, struct pci_bus *bus,
+		unsigned int devfn, int where, u32 *data)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+	unsigned int dev, func, reg, index;
+
+	dev = PCI_SLOT(devfn);
+	func = PCI_FUNC(devfn);
+	reg = where & ~3;
+	index = reg / 4;
+
+	/*
+	 * While each channel has its own memory-mapped extended config
+	 * space, it's generally only accessible when in endpoint mode.
+	 * When in root complex mode, the controller is unable to target
+	 * itself with either type 0 or type 1 accesses, and indeed, any
+	 * controller initiated target transfer to its own config space
+	 * result in a completer abort.
+	 *
+	 * Each channel effectively only supports a single device, but as
+	 * the same channel <-> device access works for any PCI_SLOT()
+	 * value, we cheat a bit here and bind the controller's config
+	 * space to devfn 0 in order to enable self-enumeration. In this
+	 * case the regular ECAR/ECDR path is sidelined and the mangled
+	 * config access itself is initiated as an internal bus transaction.
+	 */
+	if (pci_is_root_bus(bus)) {
+		if (dev != 0)
+			return PCIBIOS_DEVICE_NOT_FOUND;
+
+		if (access_type == RCAR_PCI_ACCESS_READ) {
+			*data = rcar_pci_read_reg(pcie, PCICONF(index));
+		} else {
+			/* Keep an eye out for changes to the root bus number */
+			if (pci_is_root_bus(bus) && (reg == PCI_PRIMARY_BUS))
+				host->root_bus_nr = *data & 0xff;
+
+			rcar_pci_write_reg(pcie, *data, PCICONF(index));
+		}
+
+		return PCIBIOS_SUCCESSFUL;
+	}
+
+	if (host->root_bus_nr < 0)
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	/* Clear errors */
+	rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
+
+	/* Set the PIO address */
+	rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) |
+		PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR);
+
+	/* Enable the configuration access */
+	if (bus->parent->number == host->root_bus_nr)
+		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
+	else
+		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
+
+	/* Check for errors */
+	if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	/* Check for master and target aborts */
+	if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
+		(PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT))
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	if (access_type == RCAR_PCI_ACCESS_READ)
+		*data = rcar_pci_read_reg(pcie, PCIECDR);
+	else
+		rcar_pci_write_reg(pcie, *data, PCIECDR);
+
+	/* Disable the configuration access */
+	rcar_pci_write_reg(pcie, 0, PCIECCTLR);
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn,
+			       int where, int size, u32 *val)
+{
+	struct rcar_pcie_host *host = bus->sysdata;
+	int ret;
+
+	ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_READ,
+				      bus, devfn, where, val);
+	if (ret != PCIBIOS_SUCCESSFUL) {
+		*val = 0xffffffff;
+		return ret;
+	}
+
+	if (size == 1)
+		*val = (*val >> (BITS_PER_BYTE * (where & 3))) & 0xff;
+	else if (size == 2)
+		*val = (*val >> (BITS_PER_BYTE * (where & 2))) & 0xffff;
+
+	dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n",
+		bus->number, devfn, where, size, *val);
+
+	return ret;
+}
+
+/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
+static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn,
+				int where, int size, u32 val)
+{
+	struct rcar_pcie_host *host = bus->sysdata;
+	unsigned int shift;
+	u32 data;
+	int ret;
+
+	ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_READ,
+				      bus, devfn, where, &data);
+	if (ret != PCIBIOS_SUCCESSFUL)
+		return ret;
+
+	dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n",
+		bus->number, devfn, where, size, val);
+
+	if (size == 1) {
+		shift = BITS_PER_BYTE * (where & 3);
+		data &= ~(0xff << shift);
+		data |= ((val & 0xff) << shift);
+	} else if (size == 2) {
+		shift = BITS_PER_BYTE * (where & 2);
+		data &= ~(0xffff << shift);
+		data |= ((val & 0xffff) << shift);
+	} else
+		data = val;
+
+	ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_WRITE,
+				      bus, devfn, where, &data);
+
+	return ret;
+}
+
+static struct pci_ops rcar_pcie_ops = {
+	.read	= rcar_pcie_read_conf,
+	.write	= rcar_pcie_write_conf,
+};
+
+static int rcar_pcie_setup(struct list_head *resource,
+			   struct rcar_pcie_host *host)
+{
+	struct resource_entry *win;
+	int i = 0;
+
+	/* Setup PCI resources */
+	resource_list_for_each_entry(win, &host->resources) {
+		struct resource *res = win->res;
+
+		if (!res->flags)
+			continue;
+
+		switch (resource_type(res)) {
+		case IORESOURCE_IO:
+		case IORESOURCE_MEM:
+			rcar_pcie_set_outbound(&host->pcie, i, win);
+			i++;
+			break;
+		case IORESOURCE_BUS:
+			host->root_bus_nr = res->start;
+			break;
+		default:
+			continue;
+		}
+
+		pci_add_resource(resource, res);
+	}
+
+	return 1;
+}
+
+static void rcar_pcie_force_speedup(struct rcar_pcie *pcie)
+{
+	struct device *dev = pcie->dev;
+	unsigned int timeout = 1000;
+	u32 macsr;
+
+	if ((rcar_pci_read_reg(pcie, MACS2R) & LINK_SPEED) != LINK_SPEED_5_0GTS)
+		return;
+
+	if (rcar_pci_read_reg(pcie, MACCTLR) & SPEED_CHANGE) {
+		dev_err(dev, "Speed change already in progress\n");
+		return;
+	}
+
+	macsr = rcar_pci_read_reg(pcie, MACSR);
+	if ((macsr & LINK_SPEED) == LINK_SPEED_5_0GTS)
+		goto done;
+
+	/* Set target link speed to 5.0 GT/s */
+	rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS,
+		   PCI_EXP_LNKSTA_CLS_5_0GB);
+
+	/* Set speed change reason as intentional factor */
+	rcar_rmw32(pcie, MACCGSPSETR, SPCNGRSN, 0);
+
+	/* Clear SPCHGFIN, SPCHGSUC, and SPCHGFAIL */
+	if (macsr & (SPCHGFIN | SPCHGSUC | SPCHGFAIL))
+		rcar_pci_write_reg(pcie, macsr, MACSR);
+
+	/* Start link speed change */
+	rcar_rmw32(pcie, MACCTLR, SPEED_CHANGE, SPEED_CHANGE);
+
+	while (timeout--) {
+		macsr = rcar_pci_read_reg(pcie, MACSR);
+		if (macsr & SPCHGFIN) {
+			/* Clear the interrupt bits */
+			rcar_pci_write_reg(pcie, macsr, MACSR);
+
+			if (macsr & SPCHGFAIL)
+				dev_err(dev, "Speed change failed\n");
+
+			goto done;
+		}
+
+		msleep(1);
+	}
+
+	dev_err(dev, "Speed change timed out\n");
+
+done:
+	dev_info(dev, "Current link speed is %s GT/s\n",
+		 (macsr & LINK_SPEED) == LINK_SPEED_5_0GTS ? "5" : "2.5");
+}
+
+static void rcar_pcie_hw_enable(struct rcar_pcie_host *host)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+	struct resource_entry *win;
+	LIST_HEAD(res);
+	int i = 0;
+
+	/* Try setting 5 GT/s link speed */
+	rcar_pcie_force_speedup(pcie);
+
+	/* Setup PCI resources */
+	resource_list_for_each_entry(win, &host->resources) {
+		struct resource *res = win->res;
+
+		if (!res->flags)
+			continue;
+
+		switch (resource_type(res)) {
+		case IORESOURCE_IO:
+		case IORESOURCE_MEM:
+			rcar_pcie_set_outbound(pcie, i, win);
+			i++;
+			break;
+		}
+	}
+}
+
+static int rcar_pcie_enable(struct rcar_pcie_host *host)
+{
+	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host);
+	struct rcar_pcie *pcie = &host->pcie;
+	struct device *dev = pcie->dev;
+	struct pci_bus *bus, *child;
+	int ret;
+
+	/* Try setting 5 GT/s link speed */
+	rcar_pcie_force_speedup(pcie);
+
+	rcar_pcie_setup(&bridge->windows, host);
+
+	pci_add_flags(PCI_REASSIGN_ALL_BUS);
+
+	bridge->dev.parent = dev;
+	bridge->sysdata = host;
+	bridge->busnr = host->root_bus_nr;
+	bridge->ops = &rcar_pcie_ops;
+	bridge->map_irq = of_irq_parse_and_map_pci;
+	bridge->swizzle_irq = pci_common_swizzle;
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		bridge->msi = &host->msi.chip;
+
+	ret = pci_scan_root_bus_bridge(bridge);
+	if (ret < 0)
+		return ret;
+
+	bus = bridge->bus;
+
+	pci_bus_size_bridges(bus);
+	pci_bus_assign_resources(bus);
+
+	list_for_each_entry(child, &bus->children, node)
+		pcie_bus_configure_settings(child);
+
+	pci_bus_add_devices(bus);
+
+	return 0;
+}
+
+static int phy_wait_for_ack(struct rcar_pcie *pcie)
+{
+	struct device *dev = pcie->dev;
+	unsigned int timeout = 100;
+
+	while (timeout--) {
+		if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK)
+			return 0;
+
+		udelay(100);
+	}
+
+	dev_err(dev, "Access to PCIe phy timed out\n");
+
+	return -ETIMEDOUT;
+}
+
+static void phy_write_reg(struct rcar_pcie *pcie,
+			  unsigned int rate, u32 addr,
+			  unsigned int lane, u32 data)
+{
+	u32 phyaddr;
+
+	phyaddr = WRITE_CMD |
+		((rate & 1) << RATE_POS) |
+		((lane & 0xf) << LANE_POS) |
+		((addr & 0xff) << ADR_POS);
+
+	/* Set write data */
+	rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR);
+	rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR);
+
+	/* Ignore errors as they will be dealt with if the data link is down */
+	phy_wait_for_ack(pcie);
+
+	/* Clear command */
+	rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR);
+	rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR);
+
+	/* Ignore errors as they will be dealt with if the data link is down */
+	phy_wait_for_ack(pcie);
+}
+
+static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
+{
+	int err;
+
+	/* Begin initialization */
+	rcar_pci_write_reg(pcie, 0, PCIETCTLR);
+
+	/* Set mode */
+	rcar_pci_write_reg(pcie, 1, PCIEMSR);
+
+	err = rcar_pcie_wait_for_phyrdy(pcie);
+	if (err)
+		return err;
+
+	/*
+	 * Initial header for port config space is type 1, set the device
+	 * class to match. Hardware takes care of propagating the IDSETR
+	 * settings, so there is no need to bother with a quirk.
+	 */
+	rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1);
+
+	/*
+	 * Setup Secondary Bus Number & Subordinate Bus Number, even though
+	 * they aren't used, to avoid bridge being detected as broken.
+	 */
+	rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1);
+	rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1);
+
+	/* Initialize default capabilities. */
+	rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP);
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS),
+		PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4);
+	rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f,
+		PCI_HEADER_TYPE_BRIDGE);
+
+	/* Enable data link layer active state reporting */
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC,
+		PCI_EXP_LNKCAP_DLLLARC);
+
+	/* Write out the physical slot number = 0 */
+	rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0);
+
+	/* Set the completion timer timeout to the maximum 50ms. */
+	rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50);
+
+	/* Terminate list of capabilities (Next Capability Offset=0) */
+	rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0);
+
+	/* Enable MSI */
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		rcar_pci_write_reg(pcie, 0x801f0000, PCIEMSITXR);
+
+	rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR);
+
+	/* Finish initialization - establish a PCI Express link */
+	rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
+
+	/* This will timeout if we don't have a link. */
+	err = rcar_pcie_wait_for_dl(pcie);
+	if (err)
+		return err;
+
+	/* Enable INTx interrupts */
+	rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8);
+
+	wmb();
+
+	return 0;
+}
+
+static int rcar_pcie_phy_init_h1(struct rcar_pcie_host *host)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+
+	/* Initialize the phy */
+	phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191);
+	phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180);
+	phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188);
+	phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188);
+	phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014);
+	phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014);
+	phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0);
+	phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB);
+	phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062);
+	phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000);
+	phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000);
+	phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806);
+
+	phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5);
+	phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F);
+	phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000);
+
+	return 0;
+}
+
+static int rcar_pcie_phy_init_gen2(struct rcar_pcie_host *host)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+
+	/*
+	 * These settings come from the R-Car Series, 2nd Generation User's
+	 * Manual, section 50.3.1 (2) Initialization of the physical layer.
+	 */
+	rcar_pci_write_reg(pcie, 0x000f0030, GEN2_PCIEPHYADDR);
+	rcar_pci_write_reg(pcie, 0x00381203, GEN2_PCIEPHYDATA);
+	rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
+	rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
+
+	rcar_pci_write_reg(pcie, 0x000f0054, GEN2_PCIEPHYADDR);
+	/* The following value is for DC connection, no termination resistor */
+	rcar_pci_write_reg(pcie, 0x13802007, GEN2_PCIEPHYDATA);
+	rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
+	rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
+
+	return 0;
+}
+
+static int rcar_pcie_phy_init_gen3(struct rcar_pcie_host *host)
+{
+	int err;
+
+	err = phy_init(host->phy);
+	if (err)
+		return err;
+
+	err = phy_power_on(host->phy);
+	if (err)
+		phy_exit(host->phy);
+
+	return err;
+}
+
+static int rcar_msi_alloc(struct rcar_msi *chip)
+{
+	int msi;
+
+	mutex_lock(&chip->lock);
+
+	msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
+	if (msi < INT_PCI_MSI_NR)
+		set_bit(msi, chip->used);
+	else
+		msi = -ENOSPC;
+
+	mutex_unlock(&chip->lock);
+
+	return msi;
+}
+
+static int rcar_msi_alloc_region(struct rcar_msi *chip, int no_irqs)
+{
+	int msi;
+
+	mutex_lock(&chip->lock);
+	msi = bitmap_find_free_region(chip->used, INT_PCI_MSI_NR,
+				      order_base_2(no_irqs));
+	mutex_unlock(&chip->lock);
+
+	return msi;
+}
+
+static void rcar_msi_free(struct rcar_msi *chip, unsigned long irq)
+{
+	mutex_lock(&chip->lock);
+	clear_bit(irq, chip->used);
+	mutex_unlock(&chip->lock);
+}
+
+static irqreturn_t rcar_pcie_msi_irq(int irq, void *data)
+{
+	struct rcar_pcie_host *host = data;
+	struct rcar_pcie *pcie = &host->pcie;
+	struct rcar_msi *msi = &host->msi;
+	struct device *dev = pcie->dev;
+	unsigned long reg;
+
+	reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
+
+	/* MSI & INTx share an interrupt - we only handle MSI here */
+	if (!reg)
+		return IRQ_NONE;
+
+	while (reg) {
+		unsigned int index = find_first_bit(&reg, 32);
+		unsigned int msi_irq;
+
+		/* clear the interrupt */
+		rcar_pci_write_reg(pcie, 1 << index, PCIEMSIFR);
+
+		msi_irq = irq_find_mapping(msi->domain, index);
+		if (msi_irq) {
+			if (test_bit(index, msi->used))
+				generic_handle_irq(msi_irq);
+			else
+				dev_info(dev, "unhandled MSI\n");
+		} else {
+			/* Unknown MSI, just clear it */
+			dev_dbg(dev, "unexpected MSI\n");
+		}
+
+		/* see if there's any more pending in this vector */
+		reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
+			      struct msi_desc *desc)
+{
+	struct rcar_msi *msi = to_rcar_msi(chip);
+	struct rcar_pcie_host *host = container_of(chip, struct rcar_pcie_host,
+						   msi.chip);
+	struct rcar_pcie *pcie = &host->pcie;
+	struct msi_msg msg;
+	unsigned int irq;
+	int hwirq;
+
+	hwirq = rcar_msi_alloc(msi);
+	if (hwirq < 0)
+		return hwirq;
+
+	irq = irq_find_mapping(msi->domain, hwirq);
+	if (!irq) {
+		rcar_msi_free(msi, hwirq);
+		return -EINVAL;
+	}
+
+	irq_set_msi_desc(irq, desc);
+
+	msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
+	msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
+	msg.data = hwirq;
+
+	pci_write_msi_msg(irq, &msg);
+
+	return 0;
+}
+
+static int rcar_msi_setup_irqs(struct msi_controller *chip,
+			       struct pci_dev *pdev, int nvec, int type)
+{
+	struct rcar_msi *msi = to_rcar_msi(chip);
+	struct rcar_pcie_host *host = container_of(chip, struct rcar_pcie_host,
+						   msi.chip);
+	struct rcar_pcie *pcie = &host->pcie;
+	struct msi_desc *desc;
+	struct msi_msg msg;
+	unsigned int irq;
+	int hwirq;
+	int i;
+
+	/* MSI-X interrupts are not supported */
+	if (type == PCI_CAP_ID_MSIX)
+		return -EINVAL;
+
+	WARN_ON(!list_is_singular(&pdev->dev.msi_list));
+	desc = list_entry(pdev->dev.msi_list.next, struct msi_desc, list);
+
+	hwirq = rcar_msi_alloc_region(msi, nvec);
+	if (hwirq < 0)
+		return -ENOSPC;
+
+	irq = irq_find_mapping(msi->domain, hwirq);
+	if (!irq)
+		return -ENOSPC;
+
+	for (i = 0; i < nvec; i++) {
+		/*
+		 * irq_create_mapping() called from rcar_pcie_probe() pre-
+		 * allocates descs,  so there is no need to allocate descs here.
+		 * We can therefore assume that if irq_find_mapping() above
+		 * returns non-zero, then the descs are also successfully
+		 * allocated.
+		 */
+		if (irq_set_msi_desc_off(irq, i, desc)) {
+			/* TODO: clear */
+			return -EINVAL;
+		}
+	}
+
+	desc->nvec_used = nvec;
+	desc->msi_attrib.multiple = order_base_2(nvec);
+
+	msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
+	msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
+	msg.data = hwirq;
+
+	pci_write_msi_msg(irq, &msg);
+
+	return 0;
+}
+
+static void rcar_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
+{
+	struct rcar_msi *msi = to_rcar_msi(chip);
+	struct irq_data *d = irq_get_irq_data(irq);
+
+	rcar_msi_free(msi, d->hwirq);
+}
+
+static struct irq_chip rcar_msi_irq_chip = {
+	.name = "R-Car PCIe MSI",
+	.irq_enable = pci_msi_unmask_irq,
+	.irq_disable = pci_msi_mask_irq,
+	.irq_mask = pci_msi_mask_irq,
+	.irq_unmask = pci_msi_unmask_irq,
+};
+
+static int rcar_msi_map(struct irq_domain *domain, unsigned int irq,
+			irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(irq, &rcar_msi_irq_chip, handle_simple_irq);
+	irq_set_chip_data(irq, domain->host_data);
+
+	return 0;
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+	.map = rcar_msi_map,
+};
+
+static void rcar_pcie_unmap_msi(struct rcar_pcie_host *host)
+{
+	struct rcar_msi *msi = &host->msi;
+	int i, irq;
+
+	for (i = 0; i < INT_PCI_MSI_NR; i++) {
+		irq = irq_find_mapping(msi->domain, i);
+		if (irq > 0)
+			irq_dispose_mapping(irq);
+	}
+
+	irq_domain_remove(msi->domain);
+}
+
+static void rcar_pcie_hw_enable_msi(struct rcar_pcie_host *host)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+	struct rcar_msi *msi = &host->msi;
+	unsigned long base;
+
+	/* setup MSI data target */
+	base = virt_to_phys((void *)msi->pages);
+
+	rcar_pci_write_reg(pcie, lower_32_bits(base) | MSIFE, PCIEMSIALR);
+	rcar_pci_write_reg(pcie, upper_32_bits(base), PCIEMSIAUR);
+
+	/* enable all MSI interrupts */
+	rcar_pci_write_reg(pcie, 0xffffffff, PCIEMSIIER);
+}
+
+static int rcar_pcie_enable_msi(struct rcar_pcie_host *host)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+	struct device *dev = pcie->dev;
+	struct rcar_msi *msi = &host->msi;
+	int err, i;
+
+	mutex_init(&msi->lock);
+
+	msi->chip.dev = dev;
+	msi->chip.setup_irq = rcar_msi_setup_irq;
+	msi->chip.setup_irqs = rcar_msi_setup_irqs;
+	msi->chip.teardown_irq = rcar_msi_teardown_irq;
+
+	msi->domain = irq_domain_add_linear(dev->of_node, INT_PCI_MSI_NR,
+					    &msi_domain_ops, &msi->chip);
+	if (!msi->domain) {
+		dev_err(dev, "failed to create IRQ domain\n");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < INT_PCI_MSI_NR; i++)
+		irq_create_mapping(msi->domain, i);
+
+	/* Two irqs are for MSI, but they are also used for non-MSI irqs */
+	err = devm_request_irq(dev, msi->irq1, rcar_pcie_msi_irq,
+			       IRQF_SHARED | IRQF_NO_THREAD,
+			       rcar_msi_irq_chip.name, host);
+	if (err < 0) {
+		dev_err(dev, "failed to request IRQ: %d\n", err);
+		goto err;
+	}
+
+	err = devm_request_irq(dev, msi->irq2, rcar_pcie_msi_irq,
+			       IRQF_SHARED | IRQF_NO_THREAD,
+			       rcar_msi_irq_chip.name, host);
+	if (err < 0) {
+		dev_err(dev, "failed to request IRQ: %d\n", err);
+		goto err;
+	}
+
+	/* setup MSI data target */
+	msi->pages = __get_free_pages(GFP_KERNEL, 0);
+	rcar_pcie_hw_enable_msi(host);
+
+	return 0;
+
+err:
+	rcar_pcie_unmap_msi(host);
+	return err;
+}
+
+static void rcar_pcie_teardown_msi(struct rcar_pcie_host *host)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+	struct rcar_msi *msi = &host->msi;
+
+	/* Disable all MSI interrupts */
+	rcar_pci_write_reg(pcie, 0, PCIEMSIIER);
+
+	/* Disable address decoding of the MSI interrupt, MSIFE */
+	rcar_pci_write_reg(pcie, 0, PCIEMSIALR);
+
+	free_pages(msi->pages, 0);
+
+	rcar_pcie_unmap_msi(host);
+}
+
+static int rcar_pcie_get_resources(struct rcar_pcie_host *host)
+{
+	struct rcar_pcie *pcie = &host->pcie;
+	struct device *dev = pcie->dev;
+	struct resource res;
+	int err, i;
+
+	host->phy = devm_phy_optional_get(dev, "pcie");
+	if (IS_ERR(host->phy))
+		return PTR_ERR(host->phy);
+
+	err = of_address_to_resource(dev->of_node, 0, &res);
+	if (err)
+		return err;
+
+	pcie->base = devm_ioremap_resource(dev, &res);
+	if (IS_ERR(pcie->base))
+		return PTR_ERR(pcie->base);
+
+	host->bus_clk = devm_clk_get(dev, "pcie_bus");
+	if (IS_ERR(host->bus_clk)) {
+		dev_err(dev, "cannot get pcie bus clock\n");
+		return PTR_ERR(host->bus_clk);
+	}
+
+	i = irq_of_parse_and_map(dev->of_node, 0);
+	if (!i) {
+		dev_err(dev, "cannot get platform resources for msi interrupt\n");
+		err = -ENOENT;
+		goto err_irq1;
+	}
+	host->msi.irq1 = i;
+
+	i = irq_of_parse_and_map(dev->of_node, 1);
+	if (!i) {
+		dev_err(dev, "cannot get platform resources for msi interrupt\n");
+		err = -ENOENT;
+		goto err_irq2;
+	}
+	host->msi.irq2 = i;
+
+	return 0;
+
+err_irq2:
+	irq_dispose_mapping(host->msi.irq1);
+err_irq1:
+	return err;
+}
+
+static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie,
+				    struct resource_entry *entry,
+				    int *index)
+{
+	u64 restype = entry->res->flags;
+	u64 cpu_addr = entry->res->start;
+	u64 cpu_end = entry->res->end;
+	u64 pci_addr = entry->res->start - entry->offset;
+	u32 flags = LAM_64BIT | LAR_ENABLE;
+	u64 mask;
+	u64 size = resource_size(entry->res);
+	int idx = *index;
+
+	if (restype & IORESOURCE_PREFETCH)
+		flags |= LAM_PREFETCH;
+
+	while (cpu_addr < cpu_end) {
+		if (idx >= MAX_NR_INBOUND_MAPS - 1) {
+			dev_err(pcie->dev, "Failed to map inbound regions!\n");
+			return -EINVAL;
+		}
+		/*
+		 * If the size of the range is larger than the alignment of
+		 * the start address, we have to use multiple entries to
+		 * perform the mapping.
+		 */
+		if (cpu_addr > 0) {
+			unsigned long nr_zeros = __ffs64(cpu_addr);
+			u64 alignment = 1ULL << nr_zeros;
+
+			size = min(size, alignment);
+		}
+		/* Hardware supports max 4GiB inbound region */
+		size = min(size, 1ULL << 32);
+
+		mask = roundup_pow_of_two(size) - 1;
+		mask &= ~0xf;
+
+		rcar_pcie_set_inbound(pcie, cpu_addr, pci_addr,
+				      lower_32_bits(mask) | flags, idx, true);
+
+		pci_addr += size;
+		cpu_addr += size;
+		idx += 2;
+	}
+	*index = idx;
+
+	return 0;
+}
+
+static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie_host *host)
+{
+	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host);
+	struct resource_entry *entry;
+	int index = 0, err = 0;
+
+	resource_list_for_each_entry(entry, &bridge->dma_ranges) {
+		err = rcar_pcie_inbound_ranges(&host->pcie, entry, &index);
+		if (err)
+			break;
+	}
+
+	return err;
+}
+
+static const struct of_device_id rcar_pcie_of_match[] = {
+	{ .compatible = "renesas,pcie-r8a7779",
+	  .data = rcar_pcie_phy_init_h1 },
+	{ .compatible = "renesas,pcie-r8a7790",
+	  .data = rcar_pcie_phy_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7791",
+	  .data = rcar_pcie_phy_init_gen2 },
+	{ .compatible = "renesas,pcie-rcar-gen2",
+	  .data = rcar_pcie_phy_init_gen2 },
+	{ .compatible = "renesas,pcie-r8a7795",
+	  .data = rcar_pcie_phy_init_gen3 },
+	{ .compatible = "renesas,pcie-rcar-gen3",
+	  .data = rcar_pcie_phy_init_gen3 },
+	{},
+};
+
+static int rcar_pcie_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rcar_pcie_host *host;
+	struct rcar_pcie *pcie;
+	u32 data;
+	int err;
+	struct pci_host_bridge *bridge;
+
+	bridge = pci_alloc_host_bridge(sizeof(*host));
+	if (!bridge)
+		return -ENOMEM;
+
+	host = pci_host_bridge_priv(bridge);
+	pcie = &host->pcie;
+	pcie->dev = dev;
+	platform_set_drvdata(pdev, host);
+
+	err = pci_parse_request_of_pci_ranges(dev, &host->resources,
+					      &bridge->dma_ranges, NULL);
+	if (err)
+		goto err_free_bridge;
+
+	pm_runtime_enable(pcie->dev);
+	err = pm_runtime_get_sync(pcie->dev);
+	if (err < 0) {
+		dev_err(pcie->dev, "pm_runtime_get_sync failed\n");
+		goto err_pm_disable;
+	}
+
+	err = rcar_pcie_get_resources(host);
+	if (err < 0) {
+		dev_err(dev, "failed to request resources: %d\n", err);
+		goto err_pm_put;
+	}
+
+	err = clk_prepare_enable(host->bus_clk);
+	if (err) {
+		dev_err(dev, "failed to enable bus clock: %d\n", err);
+		goto err_unmap_msi_irqs;
+	}
+
+	err = rcar_pcie_parse_map_dma_ranges(host);
+	if (err)
+		goto err_clk_disable;
+
+	host->phy_init_fn = of_device_get_match_data(dev);
+	err = host->phy_init_fn(host);
+	if (err) {
+		dev_err(dev, "failed to init PCIe PHY\n");
+		goto err_clk_disable;
+	}
+
+	/* Failure to get a link might just be that no cards are inserted */
+	if (rcar_pcie_hw_init(pcie)) {
+		dev_info(dev, "PCIe link down\n");
+		err = -ENODEV;
+		goto err_phy_shutdown;
+	}
+
+	data = rcar_pci_read_reg(pcie, MACSR);
+	dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);
+
+	if (IS_ENABLED(CONFIG_PCI_MSI)) {
+		err = rcar_pcie_enable_msi(host);
+		if (err < 0) {
+			dev_err(dev,
+				"failed to enable MSI support: %d\n",
+				err);
+			goto err_phy_shutdown;
+		}
+	}
+
+	err = rcar_pcie_enable(host);
+	if (err)
+		goto err_msi_teardown;
+
+	return 0;
+
+err_msi_teardown:
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		rcar_pcie_teardown_msi(host);
+
+err_phy_shutdown:
+	if (host->phy) {
+		phy_power_off(host->phy);
+		phy_exit(host->phy);
+	}
+
+err_clk_disable:
+	clk_disable_unprepare(host->bus_clk);
+
+err_unmap_msi_irqs:
+	irq_dispose_mapping(host->msi.irq2);
+	irq_dispose_mapping(host->msi.irq1);
+
+err_pm_put:
+	pm_runtime_put(dev);
+
+err_pm_disable:
+	pm_runtime_disable(dev);
+	pci_free_resource_list(&host->resources);
+
+err_free_bridge:
+	pci_free_host_bridge(bridge);
+
+	return err;
+}
+
+static int __maybe_unused rcar_pcie_resume(struct device *dev)
+{
+	struct rcar_pcie_host *host = dev_get_drvdata(dev);
+	struct rcar_pcie *pcie = &host->pcie;
+	unsigned int data;
+	int err;
+
+	err = rcar_pcie_parse_map_dma_ranges(host);
+	if (err)
+		return 0;
+
+	/* Failure to get a link might just be that no cards are inserted */
+	err = host->phy_init_fn(host);
+	if (err) {
+		dev_info(dev, "PCIe link down\n");
+		return 0;
+	}
+
+	data = rcar_pci_read_reg(pcie, MACSR);
+	dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);
+
+	/* Enable MSI */
+	if (IS_ENABLED(CONFIG_PCI_MSI))
+		rcar_pcie_hw_enable_msi(host);
+
+	rcar_pcie_hw_enable(host);
+
+	return 0;
+}
+
+static int rcar_pcie_resume_noirq(struct device *dev)
+{
+	struct rcar_pcie_host *host = dev_get_drvdata(dev);
+	struct rcar_pcie *pcie = &host->pcie;
+
+	if (rcar_pci_read_reg(pcie, PMSR) &&
+	    !(rcar_pci_read_reg(pcie, PCIETCTLR) & DL_DOWN))
+		return 0;
+
+	/* Re-establish the PCIe link */
+	rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR);
+	rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
+	return rcar_pcie_wait_for_dl(pcie);
+}
+
+static const struct dev_pm_ops rcar_pcie_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(NULL, rcar_pcie_resume)
+	.resume_noirq = rcar_pcie_resume_noirq,
+};
+
+static struct platform_driver rcar_pcie_driver = {
+	.driver = {
+		.name = "rcar-pcie",
+		.of_match_table = rcar_pcie_of_match,
+		.pm = &rcar_pcie_pm_ops,
+		.suppress_bind_attrs = true,
+	},
+	.probe = rcar_pcie_probe,
+};
+builtin_platform_driver(rcar_pcie_driver);
diff --git a/drivers/pci/controller/pcie-rcar.c b/drivers/pci/controller/pcie-rcar.c
index 759c6542c5c8..7583699ef7b6 100644
--- a/drivers/pci/controller/pcie-rcar.c
+++ b/drivers/pci/controller/pcie-rcar.c
@@ -1,177 +1,27 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * PCIe driver for Renesas R-Car SoCs
- *  Copyright (C) 2014 Renesas Electronics Europe Ltd
- *
- * Based on:
- *  arch/sh/drivers/pci/pcie-sh7786.c
- *  arch/sh/drivers/pci/ops-sh7786.c
- *  Copyright (C) 2009 - 2011  Paul Mundt
+ *  Copyright (C) 2014-2020 Renesas Electronics Europe Ltd
  *
  * Author: Phil Edworthy <phil.edworthy@renesas.com>
  */
 
-#include <linux/bitops.h>
-#include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/irqdomain.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/msi.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/of_pci.h>
-#include <linux/of_platform.h>
 #include <linux/pci.h>
-#include <linux/phy/phy.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/slab.h>
-
-#define PCIECAR			0x000010
-#define PCIECCTLR		0x000018
-#define  CONFIG_SEND_ENABLE	BIT(31)
-#define  TYPE0			(0 << 8)
-#define  TYPE1			BIT(8)
-#define PCIECDR			0x000020
-#define PCIEMSR			0x000028
-#define PCIEINTXR		0x000400
-#define PCIEPHYSR		0x0007f0
-#define  PHYRDY			BIT(0)
-#define PCIEMSITXR		0x000840
-
-/* Transfer control */
-#define PCIETCTLR		0x02000
-#define  DL_DOWN		BIT(3)
-#define  CFINIT			BIT(0)
-#define PCIETSTR		0x02004
-#define  DATA_LINK_ACTIVE	BIT(0)
-#define PCIEERRFR		0x02020
-#define  UNSUPPORTED_REQUEST	BIT(4)
-#define PCIEMSIFR		0x02044
-#define PCIEMSIALR		0x02048
-#define  MSIFE			BIT(0)
-#define PCIEMSIAUR		0x0204c
-#define PCIEMSIIER		0x02050
-
-/* root port address */
-#define PCIEPRAR(x)		(0x02080 + ((x) * 0x4))
-
-/* local address reg & mask */
-#define PCIELAR(x)		(0x02200 + ((x) * 0x20))
-#define PCIELAMR(x)		(0x02208 + ((x) * 0x20))
-#define  LAM_PREFETCH		BIT(3)
-#define  LAM_64BIT		BIT(2)
-#define  LAR_ENABLE		BIT(1)
-
-/* PCIe address reg & mask */
-#define PCIEPALR(x)		(0x03400 + ((x) * 0x20))
-#define PCIEPAUR(x)		(0x03404 + ((x) * 0x20))
-#define PCIEPAMR(x)		(0x03408 + ((x) * 0x20))
-#define PCIEPTCTLR(x)		(0x0340c + ((x) * 0x20))
-#define  PAR_ENABLE		BIT(31)
-#define  IO_SPACE		BIT(8)
-
-/* Configuration */
-#define PCICONF(x)		(0x010000 + ((x) * 0x4))
-#define PMCAP(x)		(0x010040 + ((x) * 0x4))
-#define EXPCAP(x)		(0x010070 + ((x) * 0x4))
-#define VCCAP(x)		(0x010100 + ((x) * 0x4))
-
-/* link layer */
-#define IDSETR1			0x011004
-#define TLCTLR			0x011048
-#define MACSR			0x011054
-#define  SPCHGFIN		BIT(4)
-#define  SPCHGFAIL		BIT(6)
-#define  SPCHGSUC		BIT(7)
-#define  LINK_SPEED		(0xf << 16)
-#define  LINK_SPEED_2_5GTS	(1 << 16)
-#define  LINK_SPEED_5_0GTS	(2 << 16)
-#define MACCTLR			0x011058
-#define  MACCTLR_NFTS_MASK	GENMASK(23, 16)	/* The name is from SH7786 */
-#define  SPEED_CHANGE		BIT(24)
-#define  SCRAMBLE_DISABLE	BIT(27)
-#define  LTSMDIS		BIT(31)
-#define  MACCTLR_INIT_VAL	(LTSMDIS | MACCTLR_NFTS_MASK)
-#define PMSR			0x01105c
-#define MACS2R			0x011078
-#define MACCGSPSETR		0x011084
-#define  SPCNGRSN		BIT(31)
-
-/* R-Car H1 PHY */
-#define H1_PCIEPHYADRR		0x04000c
-#define  WRITE_CMD		BIT(16)
-#define  PHY_ACK		BIT(24)
-#define  RATE_POS		12
-#define  LANE_POS		8
-#define  ADR_POS		0
-#define H1_PCIEPHYDOUTR		0x040014
-
-/* R-Car Gen2 PHY */
-#define GEN2_PCIEPHYADDR	0x780
-#define GEN2_PCIEPHYDATA	0x784
-#define GEN2_PCIEPHYCTRL	0x78c
-
-#define INT_PCI_MSI_NR		32
-
-#define RCONF(x)		(PCICONF(0) + (x))
-#define RPMCAP(x)		(PMCAP(0) + (x))
-#define REXPCAP(x)		(EXPCAP(0) + (x))
-#define RVCCAP(x)		(VCCAP(0) + (x))
 
-#define PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
-#define PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
-#define PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)
+#include "pcie-rcar.h"
 
-#define RCAR_PCI_MAX_RESOURCES	4
-#define MAX_NR_INBOUND_MAPS	6
-
-struct rcar_msi {
-	DECLARE_BITMAP(used, INT_PCI_MSI_NR);
-	struct irq_domain *domain;
-	struct msi_controller chip;
-	unsigned long pages;
-	struct mutex lock;
-	int irq1;
-	int irq2;
-};
-
-static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip)
-{
-	return container_of(chip, struct rcar_msi, chip);
-}
-
-/* Structure representing the PCIe interface */
-struct rcar_pcie {
-	struct device		*dev;
-	struct phy		*phy;
-	void __iomem		*base;
-	struct list_head	resources;
-	int			root_bus_nr;
-	struct clk		*bus_clk;
-	struct			rcar_msi msi;
-};
-
-static void rcar_pci_write_reg(struct rcar_pcie *pcie, u32 val,
-			       unsigned int reg)
+void rcar_pci_write_reg(struct rcar_pcie *pcie, u32 val, unsigned int reg)
 {
 	writel(val, pcie->base + reg);
 }
 
-static u32 rcar_pci_read_reg(struct rcar_pcie *pcie, unsigned int reg)
+u32 rcar_pci_read_reg(struct rcar_pcie *pcie, unsigned int reg)
 {
 	return readl(pcie->base + reg);
 }
 
-enum {
-	RCAR_PCI_ACCESS_READ,
-	RCAR_PCI_ACCESS_WRITE,
-};
-
-static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data)
+void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data)
 {
 	unsigned int shift = BITS_PER_BYTE * (where & 3);
 	u32 val = rcar_pci_read_reg(pcie, where & ~3);
@@ -181,163 +31,42 @@ static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data)
 	rcar_pci_write_reg(pcie, val, where & ~3);
 }
 
-static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
+int rcar_pcie_wait_for_phyrdy(struct rcar_pcie *pcie)
 {
-	unsigned int shift = BITS_PER_BYTE * (where & 3);
-	u32 val = rcar_pci_read_reg(pcie, where & ~3);
-
-	return val >> shift;
-}
-
-/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
-static int rcar_pcie_config_access(struct rcar_pcie *pcie,
-		unsigned char access_type, struct pci_bus *bus,
-		unsigned int devfn, int where, u32 *data)
-{
-	unsigned int dev, func, reg, index;
-
-	dev = PCI_SLOT(devfn);
-	func = PCI_FUNC(devfn);
-	reg = where & ~3;
-	index = reg / 4;
-
-	/*
-	 * While each channel has its own memory-mapped extended config
-	 * space, it's generally only accessible when in endpoint mode.
-	 * When in root complex mode, the controller is unable to target
-	 * itself with either type 0 or type 1 accesses, and indeed, any
-	 * controller initiated target transfer to its own config space
-	 * result in a completer abort.
-	 *
-	 * Each channel effectively only supports a single device, but as
-	 * the same channel <-> device access works for any PCI_SLOT()
-	 * value, we cheat a bit here and bind the controller's config
-	 * space to devfn 0 in order to enable self-enumeration. In this
-	 * case the regular ECAR/ECDR path is sidelined and the mangled
-	 * config access itself is initiated as an internal bus transaction.
-	 */
-	if (pci_is_root_bus(bus)) {
-		if (dev != 0)
-			return PCIBIOS_DEVICE_NOT_FOUND;
-
-		if (access_type == RCAR_PCI_ACCESS_READ) {
-			*data = rcar_pci_read_reg(pcie, PCICONF(index));
-		} else {
-			/* Keep an eye out for changes to the root bus number */
-			if (pci_is_root_bus(bus) && (reg == PCI_PRIMARY_BUS))
-				pcie->root_bus_nr = *data & 0xff;
-
-			rcar_pci_write_reg(pcie, *data, PCICONF(index));
-		}
-
-		return PCIBIOS_SUCCESSFUL;
-	}
-
-	if (pcie->root_bus_nr < 0)
-		return PCIBIOS_DEVICE_NOT_FOUND;
-
-	/* Clear errors */
-	rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
-
-	/* Set the PIO address */
-	rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) |
-		PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR);
-
-	/* Enable the configuration access */
-	if (bus->parent->number == pcie->root_bus_nr)
-		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
-	else
-		rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
-
-	/* Check for errors */
-	if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
-		return PCIBIOS_DEVICE_NOT_FOUND;
-
-	/* Check for master and target aborts */
-	if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
-		(PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT))
-		return PCIBIOS_DEVICE_NOT_FOUND;
-
-	if (access_type == RCAR_PCI_ACCESS_READ)
-		*data = rcar_pci_read_reg(pcie, PCIECDR);
-	else
-		rcar_pci_write_reg(pcie, *data, PCIECDR);
-
-	/* Disable the configuration access */
-	rcar_pci_write_reg(pcie, 0, PCIECCTLR);
-
-	return PCIBIOS_SUCCESSFUL;
-}
+	unsigned int timeout = 10;
 
-static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn,
-			       int where, int size, u32 *val)
-{
-	struct rcar_pcie *pcie = bus->sysdata;
-	int ret;
+	while (timeout--) {
+		if (rcar_pci_read_reg(pcie, PCIEPHYSR) & PHYRDY)
+			return 0;
 
-	ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
-				      bus, devfn, where, val);
-	if (ret != PCIBIOS_SUCCESSFUL) {
-		*val = 0xffffffff;
-		return ret;
+		msleep(5);
 	}
 
-	if (size == 1)
-		*val = (*val >> (BITS_PER_BYTE * (where & 3))) & 0xff;
-	else if (size == 2)
-		*val = (*val >> (BITS_PER_BYTE * (where & 2))) & 0xffff;
-
-	dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n",
-		bus->number, devfn, where, size, *val);
-
-	return ret;
+	return -ETIMEDOUT;
 }
 
-/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */
-static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn,
-				int where, int size, u32 val)
+int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
 {
-	struct rcar_pcie *pcie = bus->sysdata;
-	unsigned int shift;
-	u32 data;
-	int ret;
-
-	ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
-				      bus, devfn, where, &data);
-	if (ret != PCIBIOS_SUCCESSFUL)
-		return ret;
-
-	dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n",
-		bus->number, devfn, where, size, val);
+	unsigned int timeout = 10000;
 
-	if (size == 1) {
-		shift = BITS_PER_BYTE * (where & 3);
-		data &= ~(0xff << shift);
-		data |= ((val & 0xff) << shift);
-	} else if (size == 2) {
-		shift = BITS_PER_BYTE * (where & 2);
-		data &= ~(0xffff << shift);
-		data |= ((val & 0xffff) << shift);
-	} else
-		data = val;
+	while (timeout--) {
+		if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
+			return 0;
 
-	ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_WRITE,
-				      bus, devfn, where, &data);
+		udelay(5);
+		cpu_relax();
+	}
 
-	return ret;
+	return -ETIMEDOUT;
 }
 
-static struct pci_ops rcar_pcie_ops = {
-	.read	= rcar_pcie_read_conf,
-	.write	= rcar_pcie_write_conf,
-};
-
-static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie,
-				   struct resource *res)
+void rcar_pcie_set_outbound(struct rcar_pcie *pcie, int win,
+			    struct resource_entry *window)
 {
 	/* Setup PCIe address space mappings for each resource */
-	resource_size_t size;
+	struct resource *res = window->res;
 	resource_size_t res_start;
+	resource_size_t size;
 	u32 mask;
 
 	rcar_pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win));
@@ -347,13 +76,16 @@ static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie,
 	 * keeps things pretty simple.
 	 */
 	size = resource_size(res);
-	mask = (roundup_pow_of_two(size) / SZ_128) - 1;
+	if (size > 128)
+		mask = (roundup_pow_of_two(size) / SZ_128) - 1;
+	else
+		mask = 0x0;
 	rcar_pci_write_reg(pcie, mask << 7, PCIEPAMR(win));
 
 	if (res->flags & IORESOURCE_IO)
-		res_start = pci_pio_to_address(res->start);
+		res_start = pci_pio_to_address(res->start) - window->offset;
 	else
-		res_start = res->start;
+		res_start = res->start - window->offset;
 
 	rcar_pci_write_reg(pcie, upper_32_bits(res_start), PCIEPAUR(win));
 	rcar_pci_write_reg(pcie, lower_32_bits(res_start) & ~0x7F,
@@ -367,883 +99,22 @@ static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie,
 	rcar_pci_write_reg(pcie, mask, PCIEPTCTLR(win));
 }
 
-static int rcar_pcie_setup(struct list_head *resource, struct rcar_pcie *pci)
-{
-	struct resource_entry *win;
-	int i = 0;
-
-	/* Setup PCI resources */
-	resource_list_for_each_entry(win, &pci->resources) {
-		struct resource *res = win->res;
-
-		if (!res->flags)
-			continue;
-
-		switch (resource_type(res)) {
-		case IORESOURCE_IO:
-		case IORESOURCE_MEM:
-			rcar_pcie_setup_window(i, pci, res);
-			i++;
-			break;
-		case IORESOURCE_BUS:
-			pci->root_bus_nr = res->start;
-			break;
-		default:
-			continue;
-		}
-
-		pci_add_resource(resource, res);
-	}
-
-	return 1;
-}
-
-static void rcar_pcie_force_speedup(struct rcar_pcie *pcie)
-{
-	struct device *dev = pcie->dev;
-	unsigned int timeout = 1000;
-	u32 macsr;
-
-	if ((rcar_pci_read_reg(pcie, MACS2R) & LINK_SPEED) != LINK_SPEED_5_0GTS)
-		return;
-
-	if (rcar_pci_read_reg(pcie, MACCTLR) & SPEED_CHANGE) {
-		dev_err(dev, "Speed change already in progress\n");
-		return;
-	}
-
-	macsr = rcar_pci_read_reg(pcie, MACSR);
-	if ((macsr & LINK_SPEED) == LINK_SPEED_5_0GTS)
-		goto done;
-
-	/* Set target link speed to 5.0 GT/s */
-	rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS,
-		   PCI_EXP_LNKSTA_CLS_5_0GB);
-
-	/* Set speed change reason as intentional factor */
-	rcar_rmw32(pcie, MACCGSPSETR, SPCNGRSN, 0);
-
-	/* Clear SPCHGFIN, SPCHGSUC, and SPCHGFAIL */
-	if (macsr & (SPCHGFIN | SPCHGSUC | SPCHGFAIL))
-		rcar_pci_write_reg(pcie, macsr, MACSR);
-
-	/* Start link speed change */
-	rcar_rmw32(pcie, MACCTLR, SPEED_CHANGE, SPEED_CHANGE);
-
-	while (timeout--) {
-		macsr = rcar_pci_read_reg(pcie, MACSR);
-		if (macsr & SPCHGFIN) {
-			/* Clear the interrupt bits */
-			rcar_pci_write_reg(pcie, macsr, MACSR);
-
-			if (macsr & SPCHGFAIL)
-				dev_err(dev, "Speed change failed\n");
-
-			goto done;
-		}
-
-		msleep(1);
-	}
-
-	dev_err(dev, "Speed change timed out\n");
-
-done:
-	dev_info(dev, "Current link speed is %s GT/s\n",
-		 (macsr & LINK_SPEED) == LINK_SPEED_5_0GTS ? "5" : "2.5");
-}
-
-static int rcar_pcie_enable(struct rcar_pcie *pcie)
-{
-	struct device *dev = pcie->dev;
-	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
-	struct pci_bus *bus, *child;
-	int ret;
-
-	/* Try setting 5 GT/s link speed */
-	rcar_pcie_force_speedup(pcie);
-
-	rcar_pcie_setup(&bridge->windows, pcie);
-
-	pci_add_flags(PCI_REASSIGN_ALL_BUS);
-
-	bridge->dev.parent = dev;
-	bridge->sysdata = pcie;
-	bridge->busnr = pcie->root_bus_nr;
-	bridge->ops = &rcar_pcie_ops;
-	bridge->map_irq = of_irq_parse_and_map_pci;
-	bridge->swizzle_irq = pci_common_swizzle;
-	if (IS_ENABLED(CONFIG_PCI_MSI))
-		bridge->msi = &pcie->msi.chip;
-
-	ret = pci_scan_root_bus_bridge(bridge);
-	if (ret < 0)
-		return ret;
-
-	bus = bridge->bus;
-
-	pci_bus_size_bridges(bus);
-	pci_bus_assign_resources(bus);
-
-	list_for_each_entry(child, &bus->children, node)
-		pcie_bus_configure_settings(child);
-
-	pci_bus_add_devices(bus);
-
-	return 0;
-}
-
-static int phy_wait_for_ack(struct rcar_pcie *pcie)
-{
-	struct device *dev = pcie->dev;
-	unsigned int timeout = 100;
-
-	while (timeout--) {
-		if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK)
-			return 0;
-
-		udelay(100);
-	}
-
-	dev_err(dev, "Access to PCIe phy timed out\n");
-
-	return -ETIMEDOUT;
-}
-
-static void phy_write_reg(struct rcar_pcie *pcie,
-			  unsigned int rate, u32 addr,
-			  unsigned int lane, u32 data)
-{
-	u32 phyaddr;
-
-	phyaddr = WRITE_CMD |
-		((rate & 1) << RATE_POS) |
-		((lane & 0xf) << LANE_POS) |
-		((addr & 0xff) << ADR_POS);
-
-	/* Set write data */
-	rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR);
-	rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR);
-
-	/* Ignore errors as they will be dealt with if the data link is down */
-	phy_wait_for_ack(pcie);
-
-	/* Clear command */
-	rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR);
-	rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR);
-
-	/* Ignore errors as they will be dealt with if the data link is down */
-	phy_wait_for_ack(pcie);
-}
-
-static int rcar_pcie_wait_for_phyrdy(struct rcar_pcie *pcie)
-{
-	unsigned int timeout = 10;
-
-	while (timeout--) {
-		if (rcar_pci_read_reg(pcie, PCIEPHYSR) & PHYRDY)
-			return 0;
-
-		msleep(5);
-	}
-
-	return -ETIMEDOUT;
-}
-
-static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
-{
-	unsigned int timeout = 10000;
-
-	while (timeout--) {
-		if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
-			return 0;
-
-		udelay(5);
-		cpu_relax();
-	}
-
-	return -ETIMEDOUT;
-}
-
-static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
-{
-	int err;
-
-	/* Begin initialization */
-	rcar_pci_write_reg(pcie, 0, PCIETCTLR);
-
-	/* Set mode */
-	rcar_pci_write_reg(pcie, 1, PCIEMSR);
-
-	err = rcar_pcie_wait_for_phyrdy(pcie);
-	if (err)
-		return err;
-
-	/*
-	 * Initial header for port config space is type 1, set the device
-	 * class to match. Hardware takes care of propagating the IDSETR
-	 * settings, so there is no need to bother with a quirk.
-	 */
-	rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1);
-
-	/*
-	 * Setup Secondary Bus Number & Subordinate Bus Number, even though
-	 * they aren't used, to avoid bridge being detected as broken.
-	 */
-	rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1);
-	rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1);
-
-	/* Initialize default capabilities. */
-	rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP);
-	rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS),
-		PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4);
-	rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f,
-		PCI_HEADER_TYPE_BRIDGE);
-
-	/* Enable data link layer active state reporting */
-	rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC,
-		PCI_EXP_LNKCAP_DLLLARC);
-
-	/* Write out the physical slot number = 0 */
-	rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0);
-
-	/* Set the completion timer timeout to the maximum 50ms. */
-	rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50);
-
-	/* Terminate list of capabilities (Next Capability Offset=0) */
-	rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0);
-
-	/* Enable MSI */
-	if (IS_ENABLED(CONFIG_PCI_MSI))
-		rcar_pci_write_reg(pcie, 0x801f0000, PCIEMSITXR);
-
-	rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR);
-
-	/* Finish initialization - establish a PCI Express link */
-	rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
-
-	/* This will timeout if we don't have a link. */
-	err = rcar_pcie_wait_for_dl(pcie);
-	if (err)
-		return err;
-
-	/* Enable INTx interrupts */
-	rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8);
-
-	wmb();
-
-	return 0;
-}
-
-static int rcar_pcie_phy_init_h1(struct rcar_pcie *pcie)
-{
-	/* Initialize the phy */
-	phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191);
-	phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180);
-	phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188);
-	phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188);
-	phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014);
-	phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014);
-	phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0);
-	phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB);
-	phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062);
-	phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000);
-	phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000);
-	phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806);
-
-	phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5);
-	phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F);
-	phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000);
-
-	return 0;
-}
-
-static int rcar_pcie_phy_init_gen2(struct rcar_pcie *pcie)
+void rcar_pcie_set_inbound(struct rcar_pcie *pcie, u64 cpu_addr,
+			   u64 pci_addr, u64 flags, int idx, bool host)
 {
 	/*
-	 * These settings come from the R-Car Series, 2nd Generation User's
-	 * Manual, section 50.3.1 (2) Initialization of the physical layer.
+	 * Set up 64-bit inbound regions as the range parser doesn't
+	 * distinguish between 32 and 64-bit types.
 	 */
-	rcar_pci_write_reg(pcie, 0x000f0030, GEN2_PCIEPHYADDR);
-	rcar_pci_write_reg(pcie, 0x00381203, GEN2_PCIEPHYDATA);
-	rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
-	rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
-
-	rcar_pci_write_reg(pcie, 0x000f0054, GEN2_PCIEPHYADDR);
-	/* The following value is for DC connection, no termination resistor */
-	rcar_pci_write_reg(pcie, 0x13802007, GEN2_PCIEPHYDATA);
-	rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
-	rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
-
-	return 0;
-}
-
-static int rcar_pcie_phy_init_gen3(struct rcar_pcie *pcie)
-{
-	int err;
-
-	err = phy_init(pcie->phy);
-	if (err)
-		return err;
-
-	err = phy_power_on(pcie->phy);
-	if (err)
-		phy_exit(pcie->phy);
-
-	return err;
-}
-
-static int rcar_msi_alloc(struct rcar_msi *chip)
-{
-	int msi;
-
-	mutex_lock(&chip->lock);
-
-	msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
-	if (msi < INT_PCI_MSI_NR)
-		set_bit(msi, chip->used);
-	else
-		msi = -ENOSPC;
-
-	mutex_unlock(&chip->lock);
-
-	return msi;
-}
-
-static int rcar_msi_alloc_region(struct rcar_msi *chip, int no_irqs)
-{
-	int msi;
-
-	mutex_lock(&chip->lock);
-	msi = bitmap_find_free_region(chip->used, INT_PCI_MSI_NR,
-				      order_base_2(no_irqs));
-	mutex_unlock(&chip->lock);
-
-	return msi;
-}
-
-static void rcar_msi_free(struct rcar_msi *chip, unsigned long irq)
-{
-	mutex_lock(&chip->lock);
-	clear_bit(irq, chip->used);
-	mutex_unlock(&chip->lock);
-}
-
-static irqreturn_t rcar_pcie_msi_irq(int irq, void *data)
-{
-	struct rcar_pcie *pcie = data;
-	struct rcar_msi *msi = &pcie->msi;
-	struct device *dev = pcie->dev;
-	unsigned long reg;
-
-	reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
-
-	/* MSI & INTx share an interrupt - we only handle MSI here */
-	if (!reg)
-		return IRQ_NONE;
-
-	while (reg) {
-		unsigned int index = find_first_bit(&reg, 32);
-		unsigned int msi_irq;
-
-		/* clear the interrupt */
-		rcar_pci_write_reg(pcie, 1 << index, PCIEMSIFR);
-
-		msi_irq = irq_find_mapping(msi->domain, index);
-		if (msi_irq) {
-			if (test_bit(index, msi->used))
-				generic_handle_irq(msi_irq);
-			else
-				dev_info(dev, "unhandled MSI\n");
-		} else {
-			/* Unknown MSI, just clear it */
-			dev_dbg(dev, "unexpected MSI\n");
-		}
-
-		/* see if there's any more pending in this vector */
-		reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
-	}
-
-	return IRQ_HANDLED;
-}
-
-static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
-			      struct msi_desc *desc)
-{
-	struct rcar_msi *msi = to_rcar_msi(chip);
-	struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
-	struct msi_msg msg;
-	unsigned int irq;
-	int hwirq;
-
-	hwirq = rcar_msi_alloc(msi);
-	if (hwirq < 0)
-		return hwirq;
-
-	irq = irq_find_mapping(msi->domain, hwirq);
-	if (!irq) {
-		rcar_msi_free(msi, hwirq);
-		return -EINVAL;
-	}
-
-	irq_set_msi_desc(irq, desc);
-
-	msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
-	msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
-	msg.data = hwirq;
-
-	pci_write_msi_msg(irq, &msg);
-
-	return 0;
-}
-
-static int rcar_msi_setup_irqs(struct msi_controller *chip,
-			       struct pci_dev *pdev, int nvec, int type)
-{
-	struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
-	struct rcar_msi *msi = to_rcar_msi(chip);
-	struct msi_desc *desc;
-	struct msi_msg msg;
-	unsigned int irq;
-	int hwirq;
-	int i;
-
-	/* MSI-X interrupts are not supported */
-	if (type == PCI_CAP_ID_MSIX)
-		return -EINVAL;
-
-	WARN_ON(!list_is_singular(&pdev->dev.msi_list));
-	desc = list_entry(pdev->dev.msi_list.next, struct msi_desc, list);
-
-	hwirq = rcar_msi_alloc_region(msi, nvec);
-	if (hwirq < 0)
-		return -ENOSPC;
-
-	irq = irq_find_mapping(msi->domain, hwirq);
-	if (!irq)
-		return -ENOSPC;
-
-	for (i = 0; i < nvec; i++) {
-		/*
-		 * irq_create_mapping() called from rcar_pcie_probe() pre-
-		 * allocates descs,  so there is no need to allocate descs here.
-		 * We can therefore assume that if irq_find_mapping() above
-		 * returns non-zero, then the descs are also successfully
-		 * allocated.
-		 */
-		if (irq_set_msi_desc_off(irq, i, desc)) {
-			/* TODO: clear */
-			return -EINVAL;
-		}
-	}
-
-	desc->nvec_used = nvec;
-	desc->msi_attrib.multiple = order_base_2(nvec);
-
-	msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
-	msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
-	msg.data = hwirq;
-
-	pci_write_msi_msg(irq, &msg);
-
-	return 0;
-}
-
-static void rcar_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
-{
-	struct rcar_msi *msi = to_rcar_msi(chip);
-	struct irq_data *d = irq_get_irq_data(irq);
-
-	rcar_msi_free(msi, d->hwirq);
-}
-
-static struct irq_chip rcar_msi_irq_chip = {
-	.name = "R-Car PCIe MSI",
-	.irq_enable = pci_msi_unmask_irq,
-	.irq_disable = pci_msi_mask_irq,
-	.irq_mask = pci_msi_mask_irq,
-	.irq_unmask = pci_msi_unmask_irq,
-};
-
-static int rcar_msi_map(struct irq_domain *domain, unsigned int irq,
-			irq_hw_number_t hwirq)
-{
-	irq_set_chip_and_handler(irq, &rcar_msi_irq_chip, handle_simple_irq);
-	irq_set_chip_data(irq, domain->host_data);
-
-	return 0;
-}
-
-static const struct irq_domain_ops msi_domain_ops = {
-	.map = rcar_msi_map,
-};
-
-static void rcar_pcie_unmap_msi(struct rcar_pcie *pcie)
-{
-	struct rcar_msi *msi = &pcie->msi;
-	int i, irq;
-
-	for (i = 0; i < INT_PCI_MSI_NR; i++) {
-		irq = irq_find_mapping(msi->domain, i);
-		if (irq > 0)
-			irq_dispose_mapping(irq);
-	}
-
-	irq_domain_remove(msi->domain);
-}
-
-static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
-{
-	struct device *dev = pcie->dev;
-	struct rcar_msi *msi = &pcie->msi;
-	phys_addr_t base;
-	int err, i;
-
-	mutex_init(&msi->lock);
-
-	msi->chip.dev = dev;
-	msi->chip.setup_irq = rcar_msi_setup_irq;
-	msi->chip.setup_irqs = rcar_msi_setup_irqs;
-	msi->chip.teardown_irq = rcar_msi_teardown_irq;
-
-	msi->domain = irq_domain_add_linear(dev->of_node, INT_PCI_MSI_NR,
-					    &msi_domain_ops, &msi->chip);
-	if (!msi->domain) {
-		dev_err(dev, "failed to create IRQ domain\n");
-		return -ENOMEM;
-	}
-
-	for (i = 0; i < INT_PCI_MSI_NR; i++)
-		irq_create_mapping(msi->domain, i);
-
-	/* Two irqs are for MSI, but they are also used for non-MSI irqs */
-	err = devm_request_irq(dev, msi->irq1, rcar_pcie_msi_irq,
-			       IRQF_SHARED | IRQF_NO_THREAD,
-			       rcar_msi_irq_chip.name, pcie);
-	if (err < 0) {
-		dev_err(dev, "failed to request IRQ: %d\n", err);
-		goto err;
-	}
-
-	err = devm_request_irq(dev, msi->irq2, rcar_pcie_msi_irq,
-			       IRQF_SHARED | IRQF_NO_THREAD,
-			       rcar_msi_irq_chip.name, pcie);
-	if (err < 0) {
-		dev_err(dev, "failed to request IRQ: %d\n", err);
-		goto err;
-	}
-
-	/* setup MSI data target */
-	msi->pages = __get_free_pages(GFP_KERNEL, 0);
-	if (!msi->pages) {
-		err = -ENOMEM;
-		goto err;
-	}
-	base = virt_to_phys((void *)msi->pages);
-
-	rcar_pci_write_reg(pcie, lower_32_bits(base) | MSIFE, PCIEMSIALR);
-	rcar_pci_write_reg(pcie, upper_32_bits(base), PCIEMSIAUR);
-
-	/* enable all MSI interrupts */
-	rcar_pci_write_reg(pcie, 0xffffffff, PCIEMSIIER);
-
-	return 0;
-
-err:
-	rcar_pcie_unmap_msi(pcie);
-	return err;
-}
-
-static void rcar_pcie_teardown_msi(struct rcar_pcie *pcie)
-{
-	struct rcar_msi *msi = &pcie->msi;
-
-	/* Disable all MSI interrupts */
-	rcar_pci_write_reg(pcie, 0, PCIEMSIIER);
-
-	/* Disable address decoding of the MSI interrupt, MSIFE */
-	rcar_pci_write_reg(pcie, 0, PCIEMSIALR);
-
-	free_pages(msi->pages, 0);
-
-	rcar_pcie_unmap_msi(pcie);
-}
-
-static int rcar_pcie_get_resources(struct rcar_pcie *pcie)
-{
-	struct device *dev = pcie->dev;
-	struct resource res;
-	int err, i;
-
-	pcie->phy = devm_phy_optional_get(dev, "pcie");
-	if (IS_ERR(pcie->phy))
-		return PTR_ERR(pcie->phy);
-
-	err = of_address_to_resource(dev->of_node, 0, &res);
-	if (err)
-		return err;
-
-	pcie->base = devm_ioremap_resource(dev, &res);
-	if (IS_ERR(pcie->base))
-		return PTR_ERR(pcie->base);
-
-	pcie->bus_clk = devm_clk_get(dev, "pcie_bus");
-	if (IS_ERR(pcie->bus_clk)) {
-		dev_err(dev, "cannot get pcie bus clock\n");
-		return PTR_ERR(pcie->bus_clk);
-	}
-
-	i = irq_of_parse_and_map(dev->of_node, 0);
-	if (!i) {
-		dev_err(dev, "cannot get platform resources for msi interrupt\n");
-		err = -ENOENT;
-		goto err_irq1;
-	}
-	pcie->msi.irq1 = i;
-
-	i = irq_of_parse_and_map(dev->of_node, 1);
-	if (!i) {
-		dev_err(dev, "cannot get platform resources for msi interrupt\n");
-		err = -ENOENT;
-		goto err_irq2;
-	}
-	pcie->msi.irq2 = i;
-
-	return 0;
-
-err_irq2:
-	irq_dispose_mapping(pcie->msi.irq1);
-err_irq1:
-	return err;
-}
-
-static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie,
-				    struct resource_entry *entry,
-				    int *index)
-{
-	u64 restype = entry->res->flags;
-	u64 cpu_addr = entry->res->start;
-	u64 cpu_end = entry->res->end;
-	u64 pci_addr = entry->res->start - entry->offset;
-	u32 flags = LAM_64BIT | LAR_ENABLE;
-	u64 mask;
-	u64 size = resource_size(entry->res);
-	int idx = *index;
-
-	if (restype & IORESOURCE_PREFETCH)
-		flags |= LAM_PREFETCH;
-
-	while (cpu_addr < cpu_end) {
-		if (idx >= MAX_NR_INBOUND_MAPS - 1) {
-			dev_err(pcie->dev, "Failed to map inbound regions!\n");
-			return -EINVAL;
-		}
-		/*
-		 * If the size of the range is larger than the alignment of
-		 * the start address, we have to use multiple entries to
-		 * perform the mapping.
-		 */
-		if (cpu_addr > 0) {
-			unsigned long nr_zeros = __ffs64(cpu_addr);
-			u64 alignment = 1ULL << nr_zeros;
-
-			size = min(size, alignment);
-		}
-		/* Hardware supports max 4GiB inbound region */
-		size = min(size, 1ULL << 32);
-
-		mask = roundup_pow_of_two(size) - 1;
-		mask &= ~0xf;
-
-		/*
-		 * Set up 64-bit inbound regions as the range parser doesn't
-		 * distinguish between 32 and 64-bit types.
-		 */
+	if (host)
 		rcar_pci_write_reg(pcie, lower_32_bits(pci_addr),
 				   PCIEPRAR(idx));
-		rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx));
-		rcar_pci_write_reg(pcie, lower_32_bits(mask) | flags,
-				   PCIELAMR(idx));
+	rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx));
+	rcar_pci_write_reg(pcie, flags, PCIELAMR(idx));
 
+	if (host)
 		rcar_pci_write_reg(pcie, upper_32_bits(pci_addr),
 				   PCIEPRAR(idx + 1));
-		rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr),
-				   PCIELAR(idx + 1));
-		rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1));
-
-		pci_addr += size;
-		cpu_addr += size;
-		idx += 2;
-	}
-	*index = idx;
-
-	return 0;
-}
-
-static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie)
-{
-	struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
-	struct resource_entry *entry;
-	int index = 0, err = 0;
-
-	resource_list_for_each_entry(entry, &bridge->dma_ranges) {
-		err = rcar_pcie_inbound_ranges(pcie, entry, &index);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
-static const struct of_device_id rcar_pcie_of_match[] = {
-	{ .compatible = "renesas,pcie-r8a7779",
-	  .data = rcar_pcie_phy_init_h1 },
-	{ .compatible = "renesas,pcie-r8a7790",
-	  .data = rcar_pcie_phy_init_gen2 },
-	{ .compatible = "renesas,pcie-r8a7791",
-	  .data = rcar_pcie_phy_init_gen2 },
-	{ .compatible = "renesas,pcie-rcar-gen2",
-	  .data = rcar_pcie_phy_init_gen2 },
-	{ .compatible = "renesas,pcie-r8a7795",
-	  .data = rcar_pcie_phy_init_gen3 },
-	{ .compatible = "renesas,pcie-rcar-gen3",
-	  .data = rcar_pcie_phy_init_gen3 },
-	{},
-};
-
-static int rcar_pcie_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct rcar_pcie *pcie;
-	u32 data;
-	int err;
-	int (*phy_init_fn)(struct rcar_pcie *);
-	struct pci_host_bridge *bridge;
-
-	bridge = pci_alloc_host_bridge(sizeof(*pcie));
-	if (!bridge)
-		return -ENOMEM;
-
-	pcie = pci_host_bridge_priv(bridge);
-
-	pcie->dev = dev;
-	platform_set_drvdata(pdev, pcie);
-
-	err = pci_parse_request_of_pci_ranges(dev, &pcie->resources,
-					      &bridge->dma_ranges, NULL);
-	if (err)
-		goto err_free_bridge;
-
-	pm_runtime_enable(pcie->dev);
-	err = pm_runtime_get_sync(pcie->dev);
-	if (err < 0) {
-		dev_err(pcie->dev, "pm_runtime_get_sync failed\n");
-		goto err_pm_disable;
-	}
-
-	err = rcar_pcie_get_resources(pcie);
-	if (err < 0) {
-		dev_err(dev, "failed to request resources: %d\n", err);
-		goto err_pm_put;
-	}
-
-	err = clk_prepare_enable(pcie->bus_clk);
-	if (err) {
-		dev_err(dev, "failed to enable bus clock: %d\n", err);
-		goto err_unmap_msi_irqs;
-	}
-
-	err = rcar_pcie_parse_map_dma_ranges(pcie);
-	if (err)
-		goto err_clk_disable;
-
-	phy_init_fn = of_device_get_match_data(dev);
-	err = phy_init_fn(pcie);
-	if (err) {
-		dev_err(dev, "failed to init PCIe PHY\n");
-		goto err_clk_disable;
-	}
-
-	/* Failure to get a link might just be that no cards are inserted */
-	if (rcar_pcie_hw_init(pcie)) {
-		dev_info(dev, "PCIe link down\n");
-		err = -ENODEV;
-		goto err_phy_shutdown;
-	}
-
-	data = rcar_pci_read_reg(pcie, MACSR);
-	dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);
-
-	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		err = rcar_pcie_enable_msi(pcie);
-		if (err < 0) {
-			dev_err(dev,
-				"failed to enable MSI support: %d\n",
-				err);
-			goto err_phy_shutdown;
-		}
-	}
-
-	err = rcar_pcie_enable(pcie);
-	if (err)
-		goto err_msi_teardown;
-
-	return 0;
-
-err_msi_teardown:
-	if (IS_ENABLED(CONFIG_PCI_MSI))
-		rcar_pcie_teardown_msi(pcie);
-
-err_phy_shutdown:
-	if (pcie->phy) {
-		phy_power_off(pcie->phy);
-		phy_exit(pcie->phy);
-	}
-
-err_clk_disable:
-	clk_disable_unprepare(pcie->bus_clk);
-
-err_unmap_msi_irqs:
-	irq_dispose_mapping(pcie->msi.irq2);
-	irq_dispose_mapping(pcie->msi.irq1);
-
-err_pm_put:
-	pm_runtime_put(dev);
-
-err_pm_disable:
-	pm_runtime_disable(dev);
-	pci_free_resource_list(&pcie->resources);
-
-err_free_bridge:
-	pci_free_host_bridge(bridge);
-
-	return err;
+	rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx + 1));
+	rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1));
 }
-
-static int rcar_pcie_resume_noirq(struct device *dev)
-{
-	struct rcar_pcie *pcie = dev_get_drvdata(dev);
-
-	if (rcar_pci_read_reg(pcie, PMSR) &&
-	    !(rcar_pci_read_reg(pcie, PCIETCTLR) & DL_DOWN))
-		return 0;
-
-	/* Re-establish the PCIe link */
-	rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR);
-	rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
-	return rcar_pcie_wait_for_dl(pcie);
-}
-
-static const struct dev_pm_ops rcar_pcie_pm_ops = {
-	.resume_noirq = rcar_pcie_resume_noirq,
-};
-
-static struct platform_driver rcar_pcie_driver = {
-	.driver = {
-		.name = "rcar-pcie",
-		.of_match_table = rcar_pcie_of_match,
-		.pm = &rcar_pcie_pm_ops,
-		.suppress_bind_attrs = true,
-	},
-	.probe = rcar_pcie_probe,
-};
-builtin_platform_driver(rcar_pcie_driver);
diff --git a/drivers/pci/controller/pcie-rcar.h b/drivers/pci/controller/pcie-rcar.h
new file mode 100644
index 000000000000..d4c698b5f821
--- /dev/null
+++ b/drivers/pci/controller/pcie-rcar.h
@@ -0,0 +1,140 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * PCIe driver for Renesas R-Car SoCs
+ *  Copyright (C) 2014-2020 Renesas Electronics Europe Ltd
+ *
+ * Author: Phil Edworthy <phil.edworthy@renesas.com>
+ */
+
+#ifndef _PCIE_RCAR_H
+#define _PCIE_RCAR_H
+
+#define PCIECAR			0x000010
+#define PCIECCTLR		0x000018
+#define  CONFIG_SEND_ENABLE	BIT(31)
+#define  TYPE0			(0 << 8)
+#define  TYPE1			BIT(8)
+#define PCIECDR			0x000020
+#define PCIEMSR			0x000028
+#define PCIEINTXR		0x000400
+#define  ASTINTX		BIT(16)
+#define PCIEPHYSR		0x0007f0
+#define  PHYRDY			BIT(0)
+#define PCIEMSITXR		0x000840
+
+/* Transfer control */
+#define PCIETCTLR		0x02000
+#define  DL_DOWN		BIT(3)
+#define  CFINIT			BIT(0)
+#define PCIETSTR		0x02004
+#define  DATA_LINK_ACTIVE	BIT(0)
+#define PCIEERRFR		0x02020
+#define  UNSUPPORTED_REQUEST	BIT(4)
+#define PCIEMSIFR		0x02044
+#define PCIEMSIALR		0x02048
+#define  MSIFE			BIT(0)
+#define PCIEMSIAUR		0x0204c
+#define PCIEMSIIER		0x02050
+
+/* root port address */
+#define PCIEPRAR(x)		(0x02080 + ((x) * 0x4))
+
+/* local address reg & mask */
+#define PCIELAR(x)		(0x02200 + ((x) * 0x20))
+#define PCIELAMR(x)		(0x02208 + ((x) * 0x20))
+#define  LAM_PREFETCH		BIT(3)
+#define  LAM_64BIT		BIT(2)
+#define  LAR_ENABLE		BIT(1)
+
+/* PCIe address reg & mask */
+#define PCIEPALR(x)		(0x03400 + ((x) * 0x20))
+#define PCIEPAUR(x)		(0x03404 + ((x) * 0x20))
+#define PCIEPAMR(x)		(0x03408 + ((x) * 0x20))
+#define PCIEPTCTLR(x)		(0x0340c + ((x) * 0x20))
+#define  PAR_ENABLE		BIT(31)
+#define  IO_SPACE		BIT(8)
+
+/* Configuration */
+#define PCICONF(x)		(0x010000 + ((x) * 0x4))
+#define  INTDIS			BIT(10)
+#define PMCAP(x)		(0x010040 + ((x) * 0x4))
+#define MSICAP(x)		(0x010050 + ((x) * 0x4))
+#define  MSICAP0_MSIE		BIT(16)
+#define  MSICAP0_MMESCAP_OFFSET	17
+#define  MSICAP0_MMESE_OFFSET	20
+#define  MSICAP0_MMESE_MASK	GENMASK(22, 20)
+#define EXPCAP(x)		(0x010070 + ((x) * 0x4))
+#define VCCAP(x)		(0x010100 + ((x) * 0x4))
+
+/* link layer */
+#define IDSETR0			0x011000
+#define IDSETR1			0x011004
+#define SUBIDSETR		0x011024
+#define TLCTLR			0x011048
+#define MACSR			0x011054
+#define  SPCHGFIN		BIT(4)
+#define  SPCHGFAIL		BIT(6)
+#define  SPCHGSUC		BIT(7)
+#define  LINK_SPEED		(0xf << 16)
+#define  LINK_SPEED_2_5GTS	(1 << 16)
+#define  LINK_SPEED_5_0GTS	(2 << 16)
+#define MACCTLR			0x011058
+#define  MACCTLR_NFTS_MASK	GENMASK(23, 16)	/* The name is from SH7786 */
+#define  SPEED_CHANGE		BIT(24)
+#define  SCRAMBLE_DISABLE	BIT(27)
+#define  LTSMDIS		BIT(31)
+#define  MACCTLR_INIT_VAL	(LTSMDIS | MACCTLR_NFTS_MASK)
+#define PMSR			0x01105c
+#define MACS2R			0x011078
+#define MACCGSPSETR		0x011084
+#define  SPCNGRSN		BIT(31)
+
+/* R-Car H1 PHY */
+#define H1_PCIEPHYADRR		0x04000c
+#define  WRITE_CMD		BIT(16)
+#define  PHY_ACK		BIT(24)
+#define  RATE_POS		12
+#define  LANE_POS		8
+#define  ADR_POS		0
+#define H1_PCIEPHYDOUTR		0x040014
+
+/* R-Car Gen2 PHY */
+#define GEN2_PCIEPHYADDR	0x780
+#define GEN2_PCIEPHYDATA	0x784
+#define GEN2_PCIEPHYCTRL	0x78c
+
+#define INT_PCI_MSI_NR		32
+
+#define RCONF(x)		(PCICONF(0) + (x))
+#define RPMCAP(x)		(PMCAP(0) + (x))
+#define REXPCAP(x)		(EXPCAP(0) + (x))
+#define RVCCAP(x)		(VCCAP(0) + (x))
+
+#define PCIE_CONF_BUS(b)	(((b) & 0xff) << 24)
+#define PCIE_CONF_DEV(d)	(((d) & 0x1f) << 19)
+#define PCIE_CONF_FUNC(f)	(((f) & 0x7) << 16)
+
+#define RCAR_PCI_MAX_RESOURCES	4
+#define MAX_NR_INBOUND_MAPS	6
+
+struct rcar_pcie {
+	struct device		*dev;
+	void __iomem		*base;
+};
+
+enum {
+	RCAR_PCI_ACCESS_READ,
+	RCAR_PCI_ACCESS_WRITE,
+};
+
+void rcar_pci_write_reg(struct rcar_pcie *pcie, u32 val, unsigned int reg);
+u32 rcar_pci_read_reg(struct rcar_pcie *pcie, unsigned int reg);
+void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data);
+int rcar_pcie_wait_for_phyrdy(struct rcar_pcie *pcie);
+int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie);
+void rcar_pcie_set_outbound(struct rcar_pcie *pcie, int win,
+			    struct resource_entry *window);
+void rcar_pcie_set_inbound(struct rcar_pcie *pcie, u64 cpu_addr,
+			   u64 pci_addr, u64 flags, int idx, bool host);
+
+#endif
diff --git a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c
index d743b0a48988..5eaf36629a75 100644
--- a/drivers/pci/controller/pcie-rockchip-ep.c
+++ b/drivers/pci/controller/pcie-rockchip-ep.c
@@ -615,7 +615,7 @@ static int rockchip_pcie_ep_probe(struct platform_device *pdev)
 	rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG);
 
 	err = pci_epc_mem_init(epc, rockchip->mem_res->start,
-			       resource_size(rockchip->mem_res));
+			       resource_size(rockchip->mem_res), PAGE_SIZE);
 	if (err < 0) {
 		dev_err(dev, "failed to initialize the memory space\n");
 		goto err_uninit_port;
diff --git a/drivers/pci/controller/pcie-tango.c b/drivers/pci/controller/pcie-tango.c
index 21a208da3f59..8f640c70f936 100644
--- a/drivers/pci/controller/pcie-tango.c
+++ b/drivers/pci/controller/pcie-tango.c
@@ -207,7 +207,7 @@ static int smp8759_config_write(struct pci_bus *bus, unsigned int devfn,
 	return ret;
 }
 
-static struct pci_ecam_ops smp8759_ecam_ops = {
+static const struct pci_ecam_ops smp8759_ecam_ops = {
 	.bus_shift	= 20,
 	.pci_ops	= {
 		.map_bus	= pci_ecam_map_bus,
@@ -273,9 +273,9 @@ static int tango_pcie_probe(struct platform_device *pdev)
 		writel_relaxed(0, pcie->base + SMP8759_ENABLE + offset);
 
 	virq = platform_get_irq(pdev, 1);
-	if (virq <= 0) {
+	if (virq < 0) {
 		dev_err(dev, "Failed to map IRQ\n");
-		return -ENXIO;
+		return virq;
 	}
 
 	irq_dom = irq_domain_create_linear(fwnode, MSI_MAX, &dom_ops, pcie);
@@ -295,11 +295,14 @@ static int tango_pcie_probe(struct platform_device *pdev)
 	spin_lock_init(&pcie->used_msi_lock);
 	irq_set_chained_handler_and_data(virq, tango_msi_isr, pcie);
 
-	return pci_host_common_probe(pdev, &smp8759_ecam_ops);
+	return pci_host_common_probe(pdev);
 }
 
 static const struct of_device_id tango_pcie_ids[] = {
-	{ .compatible = "sigma,smp8759-pcie" },
+	{
+		.compatible = "sigma,smp8759-pcie",
+		.data = &smp8759_ecam_ops,
+	},
 	{ },
 };
 
diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index dac91d60701d..e386d4eac407 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -445,9 +445,11 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)
 			if (!membar2)
 				return -ENOMEM;
 			offset[0] = vmd->dev->resource[VMD_MEMBAR1].start -
-					readq(membar2 + MB2_SHADOW_OFFSET);
+					(readq(membar2 + MB2_SHADOW_OFFSET) &
+					 PCI_BASE_ADDRESS_MEM_MASK);
 			offset[1] = vmd->dev->resource[VMD_MEMBAR2].start -
-					readq(membar2 + MB2_SHADOW_OFFSET + 8);
+					(readq(membar2 + MB2_SHADOW_OFFSET + 8) &
+					 PCI_BASE_ADDRESS_MEM_MASK);
 			pci_iounmap(vmd->dev, membar2);
 		}
 	}
diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c
index 1a81af0ba961..8f065a42fc1a 100644
--- a/drivers/pci/ecam.c
+++ b/drivers/pci/ecam.c
@@ -26,7 +26,7 @@ static const bool per_bus_mapping = !IS_ENABLED(CONFIG_64BIT);
  */
 struct pci_config_window *pci_ecam_create(struct device *dev,
 		struct resource *cfgres, struct resource *busr,
-		struct pci_ecam_ops *ops)
+		const struct pci_ecam_ops *ops)
 {
 	struct pci_config_window *cfg;
 	unsigned int bus_range, bus_range_max, bsz;
@@ -101,6 +101,7 @@ err_exit:
 	pci_ecam_free(cfg);
 	return ERR_PTR(err);
 }
+EXPORT_SYMBOL_GPL(pci_ecam_create);
 
 void pci_ecam_free(struct pci_config_window *cfg)
 {
@@ -121,6 +122,7 @@ void pci_ecam_free(struct pci_config_window *cfg)
 		release_resource(&cfg->res);
 	kfree(cfg);
 }
+EXPORT_SYMBOL_GPL(pci_ecam_free);
 
 /*
  * Function to implement the pci_ops ->map_bus method
@@ -143,9 +145,10 @@ void __iomem *pci_ecam_map_bus(struct pci_bus *bus, unsigned int devfn,
 		base = cfg->win + (busn << cfg->ops->bus_shift);
 	return base + (devfn << devfn_shift) + where;
 }
+EXPORT_SYMBOL_GPL(pci_ecam_map_bus);
 
 /* ECAM ops */
-struct pci_ecam_ops pci_generic_ecam_ops = {
+const struct pci_ecam_ops pci_generic_ecam_ops = {
 	.bus_shift	= 20,
 	.pci_ops	= {
 		.map_bus	= pci_ecam_map_bus,
@@ -153,10 +156,11 @@ struct pci_ecam_ops pci_generic_ecam_ops = {
 		.write		= pci_generic_config_write,
 	}
 };
+EXPORT_SYMBOL_GPL(pci_generic_ecam_ops);
 
 #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)
 /* ECAM ops for 32-bit access only (non-compliant) */
-struct pci_ecam_ops pci_32b_ops = {
+const struct pci_ecam_ops pci_32b_ops = {
 	.bus_shift	= 20,
 	.pci_ops	= {
 		.map_bus	= pci_ecam_map_bus,
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index 60330f3e3751..c89a9561439f 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -187,6 +187,9 @@ static int pci_epf_test_init_dma_chan(struct pci_epf_test *epf_test)
  */
 static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test)
 {
+	if (!epf_test->dma_supported)
+		return;
+
 	dma_release_channel(epf_test->dma_chan);
 	epf_test->dma_chan = NULL;
 }
diff --git a/drivers/pci/endpoint/pci-epc-mem.c b/drivers/pci/endpoint/pci-epc-mem.c
index abfac1109a13..80c46f3a4590 100644
--- a/drivers/pci/endpoint/pci-epc-mem.c
+++ b/drivers/pci/endpoint/pci-epc-mem.c
@@ -23,7 +23,7 @@
 static int pci_epc_mem_get_order(struct pci_epc_mem *mem, size_t size)
 {
 	int order;
-	unsigned int page_shift = ilog2(mem->page_size);
+	unsigned int page_shift = ilog2(mem->window.page_size);
 
 	size--;
 	size >>= page_shift;
@@ -36,62 +36,97 @@ static int pci_epc_mem_get_order(struct pci_epc_mem *mem, size_t size)
 }
 
 /**
- * __pci_epc_mem_init() - initialize the pci_epc_mem structure
+ * pci_epc_multi_mem_init() - initialize the pci_epc_mem structure
  * @epc: the EPC device that invoked pci_epc_mem_init
- * @phys_base: the physical address of the base
- * @size: the size of the address space
- * @page_size: size of each page
+ * @windows: pointer to windows supported by the device
+ * @num_windows: number of windows device supports
  *
  * Invoke to initialize the pci_epc_mem structure used by the
  * endpoint functions to allocate mapped PCI address.
  */
-int __pci_epc_mem_init(struct pci_epc *epc, phys_addr_t phys_base, size_t size,
-		       size_t page_size)
+int pci_epc_multi_mem_init(struct pci_epc *epc,
+			   struct pci_epc_mem_window *windows,
+			   unsigned int num_windows)
 {
-	int ret;
-	struct pci_epc_mem *mem;
-	unsigned long *bitmap;
+	struct pci_epc_mem *mem = NULL;
+	unsigned long *bitmap = NULL;
 	unsigned int page_shift;
-	int pages;
+	size_t page_size;
 	int bitmap_size;
+	int pages;
+	int ret;
+	int i;
 
-	if (page_size < PAGE_SIZE)
-		page_size = PAGE_SIZE;
+	epc->num_windows = 0;
 
-	page_shift = ilog2(page_size);
-	pages = size >> page_shift;
-	bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
+	if (!windows || !num_windows)
+		return -EINVAL;
 
-	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
-	if (!mem) {
-		ret = -ENOMEM;
-		goto err;
-	}
+	epc->windows = kcalloc(num_windows, sizeof(*epc->windows), GFP_KERNEL);
+	if (!epc->windows)
+		return -ENOMEM;
 
-	bitmap = kzalloc(bitmap_size, GFP_KERNEL);
-	if (!bitmap) {
-		ret = -ENOMEM;
-		goto err_mem;
-	}
+	for (i = 0; i < num_windows; i++) {
+		page_size = windows[i].page_size;
+		if (page_size < PAGE_SIZE)
+			page_size = PAGE_SIZE;
+		page_shift = ilog2(page_size);
+		pages = windows[i].size >> page_shift;
+		bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
 
-	mem->bitmap = bitmap;
-	mem->phys_base = phys_base;
-	mem->page_size = page_size;
-	mem->pages = pages;
-	mem->size = size;
-	mutex_init(&mem->lock);
+		mem = kzalloc(sizeof(*mem), GFP_KERNEL);
+		if (!mem) {
+			ret = -ENOMEM;
+			i--;
+			goto err_mem;
+		}
+
+		bitmap = kzalloc(bitmap_size, GFP_KERNEL);
+		if (!bitmap) {
+			ret = -ENOMEM;
+			kfree(mem);
+			i--;
+			goto err_mem;
+		}
+
+		mem->window.phys_base = windows[i].phys_base;
+		mem->window.size = windows[i].size;
+		mem->window.page_size = page_size;
+		mem->bitmap = bitmap;
+		mem->pages = pages;
+		mutex_init(&mem->lock);
+		epc->windows[i] = mem;
+	}
 
-	epc->mem = mem;
+	epc->mem = epc->windows[0];
+	epc->num_windows = num_windows;
 
 	return 0;
 
 err_mem:
-	kfree(mem);
+	for (; i >= 0; i--) {
+		mem = epc->windows[i];
+		kfree(mem->bitmap);
+		kfree(mem);
+	}
+	kfree(epc->windows);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_epc_multi_mem_init);
+
+int pci_epc_mem_init(struct pci_epc *epc, phys_addr_t base,
+		     size_t size, size_t page_size)
+{
+	struct pci_epc_mem_window mem_window;
+
+	mem_window.phys_base = base;
+	mem_window.size = size;
+	mem_window.page_size = page_size;
 
-err:
-return ret;
+	return pci_epc_multi_mem_init(epc, &mem_window, 1);
 }
-EXPORT_SYMBOL_GPL(__pci_epc_mem_init);
+EXPORT_SYMBOL_GPL(pci_epc_mem_init);
 
 /**
  * pci_epc_mem_exit() - cleanup the pci_epc_mem structure
@@ -102,11 +137,22 @@ EXPORT_SYMBOL_GPL(__pci_epc_mem_init);
  */
 void pci_epc_mem_exit(struct pci_epc *epc)
 {
-	struct pci_epc_mem *mem = epc->mem;
+	struct pci_epc_mem *mem;
+	int i;
+
+	if (!epc->num_windows)
+		return;
 
+	for (i = 0; i < epc->num_windows; i++) {
+		mem = epc->windows[i];
+		kfree(mem->bitmap);
+		kfree(mem);
+	}
+	kfree(epc->windows);
+
+	epc->windows = NULL;
 	epc->mem = NULL;
-	kfree(mem->bitmap);
-	kfree(mem);
+	epc->num_windows = 0;
 }
 EXPORT_SYMBOL_GPL(pci_epc_mem_exit);
 
@@ -122,31 +168,60 @@ EXPORT_SYMBOL_GPL(pci_epc_mem_exit);
 void __iomem *pci_epc_mem_alloc_addr(struct pci_epc *epc,
 				     phys_addr_t *phys_addr, size_t size)
 {
-	int pageno;
 	void __iomem *virt_addr = NULL;
-	struct pci_epc_mem *mem = epc->mem;
-	unsigned int page_shift = ilog2(mem->page_size);
+	struct pci_epc_mem *mem;
+	unsigned int page_shift;
+	size_t align_size;
+	int pageno;
 	int order;
+	int i;
 
-	size = ALIGN(size, mem->page_size);
-	order = pci_epc_mem_get_order(mem, size);
-
-	mutex_lock(&mem->lock);
-	pageno = bitmap_find_free_region(mem->bitmap, mem->pages, order);
-	if (pageno < 0)
-		goto ret;
+	for (i = 0; i < epc->num_windows; i++) {
+		mem = epc->windows[i];
+		mutex_lock(&mem->lock);
+		align_size = ALIGN(size, mem->window.page_size);
+		order = pci_epc_mem_get_order(mem, align_size);
 
-	*phys_addr = mem->phys_base + ((phys_addr_t)pageno << page_shift);
-	virt_addr = ioremap(*phys_addr, size);
-	if (!virt_addr)
-		bitmap_release_region(mem->bitmap, pageno, order);
+		pageno = bitmap_find_free_region(mem->bitmap, mem->pages,
+						 order);
+		if (pageno >= 0) {
+			page_shift = ilog2(mem->window.page_size);
+			*phys_addr = mem->window.phys_base +
+				((phys_addr_t)pageno << page_shift);
+			virt_addr = ioremap(*phys_addr, align_size);
+			if (!virt_addr) {
+				bitmap_release_region(mem->bitmap,
+						      pageno, order);
+				mutex_unlock(&mem->lock);
+				continue;
+			}
+			mutex_unlock(&mem->lock);
+			return virt_addr;
+		}
+		mutex_unlock(&mem->lock);
+	}
 
-ret:
-	mutex_unlock(&mem->lock);
 	return virt_addr;
 }
 EXPORT_SYMBOL_GPL(pci_epc_mem_alloc_addr);
 
+static struct pci_epc_mem *pci_epc_get_matching_window(struct pci_epc *epc,
+						       phys_addr_t phys_addr)
+{
+	struct pci_epc_mem *mem;
+	int i;
+
+	for (i = 0; i < epc->num_windows; i++) {
+		mem = epc->windows[i];
+
+		if (phys_addr >= mem->window.phys_base &&
+		    phys_addr < (mem->window.phys_base + mem->window.size))
+			return mem;
+	}
+
+	return NULL;
+}
+
 /**
  * pci_epc_mem_free_addr() - free the allocated memory address
  * @epc: the EPC device on which memory was allocated
@@ -159,14 +234,23 @@ EXPORT_SYMBOL_GPL(pci_epc_mem_alloc_addr);
 void pci_epc_mem_free_addr(struct pci_epc *epc, phys_addr_t phys_addr,
 			   void __iomem *virt_addr, size_t size)
 {
+	struct pci_epc_mem *mem;
+	unsigned int page_shift;
+	size_t page_size;
 	int pageno;
-	struct pci_epc_mem *mem = epc->mem;
-	unsigned int page_shift = ilog2(mem->page_size);
 	int order;
 
+	mem = pci_epc_get_matching_window(epc, phys_addr);
+	if (!mem) {
+		pr_err("failed to get matching window\n");
+		return;
+	}
+
+	page_size = mem->window.page_size;
+	page_shift = ilog2(page_size);
 	iounmap(virt_addr);
-	pageno = (phys_addr - mem->phys_base) >> page_shift;
-	size = ALIGN(size, mem->page_size);
+	pageno = (phys_addr - mem->window.phys_base) >> page_shift;
+	size = ALIGN(size, page_size);
 	order = pci_epc_mem_get_order(mem, size);
 	mutex_lock(&mem->lock);
 	bitmap_release_region(mem->bitmap, pageno, order);
diff --git a/drivers/pci/hotplug/Kconfig b/drivers/pci/hotplug/Kconfig
index 32455a79372d..840a84bb5ee2 100644
--- a/drivers/pci/hotplug/Kconfig
+++ b/drivers/pci/hotplug/Kconfig
@@ -6,7 +6,7 @@
 menuconfig HOTPLUG_PCI
 	bool "Support for PCI Hotplug"
 	depends on PCI && SYSFS
-	---help---
+	help
 	  Say Y here if you have a motherboard with a PCI Hotplug controller.
 	  This allows you to add and remove PCI cards while the machine is
 	  powered up and running.
diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h
index ae44f46d1bf3..4fd200d8b0a9 100644
--- a/drivers/pci/hotplug/pciehp.h
+++ b/drivers/pci/hotplug/pciehp.h
@@ -148,8 +148,6 @@ struct controller {
 #define MRL_SENS(ctrl)		((ctrl)->slot_cap & PCI_EXP_SLTCAP_MRLSP)
 #define ATTN_LED(ctrl)		((ctrl)->slot_cap & PCI_EXP_SLTCAP_AIP)
 #define PWR_LED(ctrl)		((ctrl)->slot_cap & PCI_EXP_SLTCAP_PIP)
-#define HP_SUPR_RM(ctrl)	((ctrl)->slot_cap & PCI_EXP_SLTCAP_HPS)
-#define EMI(ctrl)		((ctrl)->slot_cap & PCI_EXP_SLTCAP_EIP)
 #define NO_CMD_CMPL(ctrl)	((ctrl)->slot_cap & PCI_EXP_SLTCAP_NCCS)
 #define PSN(ctrl)		(((ctrl)->slot_cap & PCI_EXP_SLTCAP_PSN) >> 19)
 
diff --git a/drivers/pci/hotplug/rpaphp_core.c b/drivers/pci/hotplug/rpaphp_core.c
index 6504869efabc..9887c9de08c3 100644
--- a/drivers/pci/hotplug/rpaphp_core.c
+++ b/drivers/pci/hotplug/rpaphp_core.c
@@ -435,7 +435,7 @@ static int rpaphp_drc_add_slot(struct device_node *dn)
  */
 int rpaphp_add_slot(struct device_node *dn)
 {
-	if (!dn->name || strcmp(dn->name, "pci"))
+	if (!of_node_name_eq(dn, "pci"))
 		return 0;
 
 	if (of_find_property(dn, "ibm,drc-info", NULL))
diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c
index 39295d88f670..b59f84918fe0 100644
--- a/drivers/pci/hotplug/s390_pci_hpc.c
+++ b/drivers/pci/hotplug/s390_pci_hpc.c
@@ -52,6 +52,7 @@ static int enable_slot(struct hotplug_slot *hotplug_slot)
 {
 	struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev,
 					     hotplug_slot);
+	struct zpci_bus *zbus = zdev->zbus;
 	int rc;
 
 	if (zdev->state != ZPCI_FN_STATE_STANDBY)
@@ -65,9 +66,9 @@ static int enable_slot(struct hotplug_slot *hotplug_slot)
 	if (rc)
 		goto out_deconfigure;
 
-	pci_scan_slot(zdev->bus, ZPCI_DEVFN);
+	pci_scan_slot(zbus->bus, zdev->devfn);
 	pci_lock_rescan_remove();
-	pci_bus_add_devices(zdev->bus);
+	pci_bus_add_devices(zbus->bus);
 	pci_unlock_rescan_remove();
 
 	return rc;
@@ -82,13 +83,17 @@ static int disable_slot(struct hotplug_slot *hotplug_slot)
 	struct zpci_dev *zdev = container_of(hotplug_slot, struct zpci_dev,
 					     hotplug_slot);
 	struct pci_dev *pdev;
+	struct zpci_bus *zbus = zdev->zbus;
 	int rc;
 
 	if (!zpci_fn_configured(zdev->state))
 		return -EIO;
 
-	pdev = pci_get_slot(zdev->bus, ZPCI_DEVFN);
+	pdev = pci_get_slot(zbus->bus, zdev->devfn);
 	if (pdev) {
+		if (pci_num_vf(pdev))
+			return -EBUSY;
+
 		pci_stop_and_remove_bus_device_locked(pdev);
 		pci_dev_put(pdev);
 	}
@@ -133,12 +138,13 @@ static const struct hotplug_slot_ops s390_hotplug_slot_ops = {
 int zpci_init_slot(struct zpci_dev *zdev)
 {
 	char name[SLOT_NAME_SIZE];
+	struct zpci_bus *zbus = zdev->zbus;
 
 	zdev->hotplug_slot.ops = &s390_hotplug_slot_ops;
 
 	snprintf(name, SLOT_NAME_SIZE, "%08x", zdev->fid);
-	return pci_hp_register(&zdev->hotplug_slot, zdev->bus,
-			       ZPCI_DEVFN, name);
+	return pci_hp_register(&zdev->hotplug_slot, zbus->bus,
+			       zdev->devfn, name);
 }
 
 void zpci_exit_slot(struct zpci_dev *zdev)
diff --git a/drivers/pci/hotplug/shpchp.h b/drivers/pci/hotplug/shpchp.h
index f7f13ee5d06e..6e85885b554c 100644
--- a/drivers/pci/hotplug/shpchp.h
+++ b/drivers/pci/hotplug/shpchp.h
@@ -164,7 +164,7 @@ u8 shpchp_handle_switch_change(u8 hp_slot, struct controller *ctrl);
 u8 shpchp_handle_presence_change(u8 hp_slot, struct controller *ctrl);
 u8 shpchp_handle_power_fault(u8 hp_slot, struct controller *ctrl);
 int shpchp_configure_device(struct slot *p_slot);
-int shpchp_unconfigure_device(struct slot *p_slot);
+void shpchp_unconfigure_device(struct slot *p_slot);
 void cleanup_slots(struct controller *ctrl);
 void shpchp_queue_pushbutton_work(struct work_struct *work);
 int shpc_init(struct controller *ctrl, struct pci_dev *pdev);
diff --git a/drivers/pci/hotplug/shpchp_ctrl.c b/drivers/pci/hotplug/shpchp_ctrl.c
index 078003dcde5b..afdc52d1cae7 100644
--- a/drivers/pci/hotplug/shpchp_ctrl.c
+++ b/drivers/pci/hotplug/shpchp_ctrl.c
@@ -341,8 +341,7 @@ static int remove_board(struct slot *p_slot)
 	u8 hp_slot;
 	int rc;
 
-	if (shpchp_unconfigure_device(p_slot))
-		return(1);
+	shpchp_unconfigure_device(p_slot);
 
 	hp_slot = p_slot->device - ctrl->slot_device_offset;
 	p_slot = shpchp_find_slot(ctrl, hp_slot + ctrl->slot_device_offset);
diff --git a/drivers/pci/hotplug/shpchp_pci.c b/drivers/pci/hotplug/shpchp_pci.c
index 115701301487..36db0c3c4ea6 100644
--- a/drivers/pci/hotplug/shpchp_pci.c
+++ b/drivers/pci/hotplug/shpchp_pci.c
@@ -61,9 +61,8 @@ int shpchp_configure_device(struct slot *p_slot)
 	return ret;
 }
 
-int shpchp_unconfigure_device(struct slot *p_slot)
+void shpchp_unconfigure_device(struct slot *p_slot)
 {
-	int rc = 0;
 	struct pci_bus *parent = p_slot->ctrl->pci_dev->subordinate;
 	struct pci_dev *dev, *temp;
 	struct controller *ctrl = p_slot->ctrl;
@@ -83,6 +82,4 @@ int shpchp_unconfigure_device(struct slot *p_slot)
 	}
 
 	pci_unlock_rescan_remove();
-	return rc;
 }
-
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 4d1f392b05f9..b37e08c4f9d1 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -133,12 +133,35 @@ static void pci_read_vf_config_common(struct pci_dev *virtfn)
 			     &physfn->sriov->subsystem_device);
 }
 
+int pci_iov_sysfs_link(struct pci_dev *dev,
+		struct pci_dev *virtfn, int id)
+{
+	char buf[VIRTFN_ID_LEN];
+	int rc;
+
+	sprintf(buf, "virtfn%u", id);
+	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
+	if (rc)
+		goto failed;
+	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
+	if (rc)
+		goto failed1;
+
+	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
+
+	return 0;
+
+failed1:
+	sysfs_remove_link(&dev->dev.kobj, buf);
+failed:
+	return rc;
+}
+
 int pci_iov_add_virtfn(struct pci_dev *dev, int id)
 {
 	int i;
 	int rc = -ENOMEM;
 	u64 size;
-	char buf[VIRTFN_ID_LEN];
 	struct pci_dev *virtfn;
 	struct resource *res;
 	struct pci_sriov *iov = dev->sriov;
@@ -182,23 +205,14 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id)
 	}
 
 	pci_device_add(virtfn, virtfn->bus);
-
-	sprintf(buf, "virtfn%u", id);
-	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
+	rc = pci_iov_sysfs_link(dev, virtfn, id);
 	if (rc)
 		goto failed1;
-	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
-	if (rc)
-		goto failed2;
-
-	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
 
 	pci_bus_add_device(virtfn);
 
 	return 0;
 
-failed2:
-	sysfs_remove_link(&dev->dev.kobj, buf);
 failed1:
 	pci_stop_and_remove_bus_device(virtfn);
 	pci_dev_put(dev);
@@ -557,9 +571,6 @@ static void sriov_del_vfs(struct pci_dev *dev)
 	struct pci_sriov *iov = dev->sriov;
 	int i;
 
-	if (dev->no_vf_scan)
-		return;
-
 	for (i = 0; i < iov->num_VFs; i++)
 		pci_iov_remove_virtfn(dev, i);
 }
diff --git a/drivers/pci/of.c b/drivers/pci/of.c
index 81ceeaa6f1d5..27839cd2459f 100644
--- a/drivers/pci/of.c
+++ b/drivers/pci/of.c
@@ -592,7 +592,7 @@ int of_pci_get_max_link_speed(struct device_node *node)
 	u32 max_link_speed;
 
 	if (of_property_read_u32(node, "max-link-speed", &max_link_speed) ||
-	    max_link_speed > 4)
+	    max_link_speed == 0 || max_link_speed > 4)
 		return -EINVAL;
 
 	return max_link_speed;
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index b73b10bce0df..e8e444eeb1cd 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -282,6 +282,8 @@ static const struct pci_p2pdma_whitelist_entry {
 } pci_p2pdma_whitelist[] = {
 	/* AMD ZEN */
 	{PCI_VENDOR_ID_AMD,	0x1450,	0},
+	{PCI_VENDOR_ID_AMD,	0x15d0,	0},
+	{PCI_VENDOR_ID_AMD,	0x1630,	0},
 
 	/* Intel Xeon E5/Core i7 */
 	{PCI_VENDOR_ID_INTEL,	0x3c00, REQ_SAME_HOST_BRIDGE},
diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c
index d21969fba6ab..7224b1e5f2a8 100644
--- a/drivers/pci/pci-acpi.c
+++ b/drivers/pci/pci-acpi.c
@@ -948,7 +948,7 @@ static bool acpi_pci_bridge_d3(struct pci_dev *dev)
 	 * Look for a special _DSD property for the root port and if it
 	 * is set we know the hierarchy behind it supports D3 just fine.
 	 */
-	root = pci_find_pcie_root_port(dev);
+	root = pcie_find_root_port(dev);
 	if (!root)
 		return false;
 
@@ -1128,7 +1128,7 @@ void acpi_pci_add_bus(struct pci_bus *bus)
 		return;
 
 	obj = acpi_evaluate_dsm(ACPI_HANDLE(bus->bridge), &pci_acpi_dsm_guid, 3,
-				RESET_DELAY_DSM, NULL);
+				DSM_PCI_POWER_ON_RESET_DELAY, NULL);
 	if (!obj)
 		return;
 
@@ -1193,7 +1193,7 @@ static void pci_acpi_optimize_delay(struct pci_dev *pdev,
 		pdev->d3cold_delay = 0;
 
 	obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 3,
-				FUNCTION_DELAY_DSM, NULL);
+				DSM_PCI_DEVICE_READINESS_DURATIONS, NULL);
 	if (!obj)
 		return;
 
diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c
index 4f4f54bc732e..ccf26d12ec61 100644
--- a/drivers/pci/pci-bridge-emul.c
+++ b/drivers/pci/pci-bridge-emul.c
@@ -24,6 +24,17 @@
 #define PCI_CAP_PCIE_START	PCI_BRIDGE_CONF_END
 #define PCI_CAP_PCIE_END	(PCI_CAP_PCIE_START + PCI_EXP_SLTSTA2 + 2)
 
+/**
+ * struct pci_bridge_reg_behavior - register bits behaviors
+ * @ro:		Read-Only bits
+ * @rw:		Read-Write bits
+ * @w1c:	Write-1-to-Clear bits
+ *
+ * Reads and Writes will be filtered by specified behavior. All other bits not
+ * declared are assumed 'Reserved' and will return 0 on reads, per PCIe 5.0:
+ * "Reserved register fields must be read only and must return 0 (all 0's for
+ * multi-bit fields) when read".
+ */
 struct pci_bridge_reg_behavior {
 	/* Read-only bits */
 	u32 ro;
@@ -33,9 +44,6 @@ struct pci_bridge_reg_behavior {
 
 	/* Write-1-to-clear bits */
 	u32 w1c;
-
-	/* Reserved bits (hardwired to 0) */
-	u32 rsvd;
 };
 
 static const struct pci_bridge_reg_behavior pci_regs_behavior[] = {
@@ -49,7 +57,6 @@ static const struct pci_bridge_reg_behavior pci_regs_behavior[] = {
 			PCI_COMMAND_FAST_BACK) |
 		       (PCI_STATUS_CAP_LIST | PCI_STATUS_66MHZ |
 			PCI_STATUS_FAST_BACK | PCI_STATUS_DEVSEL_MASK) << 16),
-		.rsvd = GENMASK(15, 10) | ((BIT(6) | GENMASK(3, 0)) << 16),
 		.w1c = PCI_STATUS_ERROR_BITS << 16,
 	},
 	[PCI_CLASS_REVISION / 4] = { .ro = ~0 },
@@ -96,8 +103,6 @@ static const struct pci_bridge_reg_behavior pci_regs_behavior[] = {
 		       GENMASK(11, 8) | GENMASK(3, 0)),
 
 		.w1c = PCI_STATUS_ERROR_BITS << 16,
-
-		.rsvd = ((BIT(6) | GENMASK(4, 0)) << 16),
 	},
 
 	[PCI_MEMORY_BASE / 4] = {
@@ -130,12 +135,10 @@ static const struct pci_bridge_reg_behavior pci_regs_behavior[] = {
 
 	[PCI_CAPABILITY_LIST / 4] = {
 		.ro = GENMASK(7, 0),
-		.rsvd = GENMASK(31, 8),
 	},
 
 	[PCI_ROM_ADDRESS1 / 4] = {
 		.rw = GENMASK(31, 11) | BIT(0),
-		.rsvd = GENMASK(10, 1),
 	},
 
 	/*
@@ -158,8 +161,6 @@ static const struct pci_bridge_reg_behavior pci_regs_behavior[] = {
 		.ro = (GENMASK(15, 8) | ((PCI_BRIDGE_CTL_FAST_BACK) << 16)),
 
 		.w1c = BIT(10) << 16,
-
-		.rsvd = (GENMASK(15, 12) | BIT(4)) << 16,
 	},
 };
 
@@ -181,31 +182,29 @@ static const struct pci_bridge_reg_behavior pcie_cap_regs_behavior[] = {
 		.rw = GENMASK(15, 0),
 
 		/*
-		 * Device status register has 4 bits W1C, then 2 bits
-		 * RO, the rest is reserved
+		 * Device status register has bits 6 and [3:0] W1C, [5:4] RO,
+		 * the rest is reserved
 		 */
-		.w1c = GENMASK(19, 16),
-		.ro = GENMASK(20, 19),
-		.rsvd = GENMASK(31, 21),
+		.w1c = (BIT(6) | GENMASK(3, 0)) << 16,
+		.ro = GENMASK(5, 4) << 16,
 	},
 
 	[PCI_EXP_LNKCAP / 4] = {
 		/* All bits are RO, except bit 23 which is reserved */
 		.ro = lower_32_bits(~BIT(23)),
-		.rsvd = BIT(23),
 	},
 
 	[PCI_EXP_LNKCTL / 4] = {
 		/*
-		 * Link control has bits [1:0] and [11:3] RW, the
-		 * other bits are reserved.
-		 * Link status has bits [13:0] RO, and bits [14:15]
+		 * Link control has bits [15:14], [11:3] and [1:0] RW, the
+		 * rest is reserved.
+		 *
+		 * Link status has bits [13:0] RO, and bits [15:14]
 		 * W1C.
 		 */
-		.rw = GENMASK(11, 3) | GENMASK(1, 0),
+		.rw = GENMASK(15, 14) | GENMASK(11, 3) | GENMASK(1, 0),
 		.ro = GENMASK(13, 0) << 16,
 		.w1c = GENMASK(15, 14) << 16,
-		.rsvd = GENMASK(15, 12) | BIT(2),
 	},
 
 	[PCI_EXP_SLTCAP / 4] = {
@@ -214,19 +213,18 @@ static const struct pci_bridge_reg_behavior pcie_cap_regs_behavior[] = {
 
 	[PCI_EXP_SLTCTL / 4] = {
 		/*
-		 * Slot control has bits [12:0] RW, the rest is
+		 * Slot control has bits [14:0] RW, the rest is
 		 * reserved.
 		 *
-		 * Slot status has a mix of W1C and RO bits, as well
-		 * as reserved bits.
+		 * Slot status has bits 8 and [4:0] W1C, bits [7:5] RO, the
+		 * rest is reserved.
 		 */
-		.rw = GENMASK(12, 0),
+		.rw = GENMASK(14, 0),
 		.w1c = (PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
 			PCI_EXP_SLTSTA_MRLSC | PCI_EXP_SLTSTA_PDC |
 			PCI_EXP_SLTSTA_CC | PCI_EXP_SLTSTA_DLLSC) << 16,
 		.ro = (PCI_EXP_SLTSTA_MRLSS | PCI_EXP_SLTSTA_PDS |
 		       PCI_EXP_SLTSTA_EIS) << 16,
-		.rsvd = GENMASK(15, 12) | (GENMASK(15, 9) << 16),
 	},
 
 	[PCI_EXP_RTCTL / 4] = {
@@ -234,19 +232,21 @@ static const struct pci_bridge_reg_behavior pcie_cap_regs_behavior[] = {
 		 * Root control has bits [4:0] RW, the rest is
 		 * reserved.
 		 *
-		 * Root status has bit 0 RO, the rest is reserved.
+		 * Root capabilities has bit 0 RO, the rest is reserved.
 		 */
 		.rw = (PCI_EXP_RTCTL_SECEE | PCI_EXP_RTCTL_SENFEE |
 		       PCI_EXP_RTCTL_SEFEE | PCI_EXP_RTCTL_PMEIE |
 		       PCI_EXP_RTCTL_CRSSVE),
 		.ro = PCI_EXP_RTCAP_CRSVIS << 16,
-		.rsvd = GENMASK(15, 5) | (GENMASK(15, 1) << 16),
 	},
 
 	[PCI_EXP_RTSTA / 4] = {
+		/*
+		 * Root status has bits 17 and [15:0] RO, bit 16 W1C, the rest
+		 * is reserved.
+		 */
 		.ro = GENMASK(15, 0) | PCI_EXP_RTSTA_PENDING,
 		.w1c = PCI_EXP_RTSTA_PME,
-		.rsvd = GENMASK(31, 18),
 	},
 };
 
@@ -354,7 +354,8 @@ int pci_bridge_emul_conf_read(struct pci_bridge_emul *bridge, int where,
 	 * Make sure we never return any reserved bit with a value
 	 * different from 0.
 	 */
-	*value &= ~behavior[reg / 4].rsvd;
+	*value &= behavior[reg / 4].ro | behavior[reg / 4].rw |
+		  behavior[reg / 4].w1c;
 
 	if (size == 1)
 		*value = (*value >> (8 * (where & 3))) & 0xff;
diff --git a/drivers/pci/pci-label.c b/drivers/pci/pci-label.c
index a5910f942857..707dd9808676 100644
--- a/drivers/pci/pci-label.c
+++ b/drivers/pci/pci-label.c
@@ -178,7 +178,7 @@ static int dsm_get_label(struct device *dev, char *buf,
 		return -1;
 
 	obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 0x2,
-				DEVICE_LABEL_DSM, NULL);
+				DSM_PCI_DEVICE_NAME, NULL);
 	if (!obj)
 		return -1;
 
@@ -218,7 +218,7 @@ static bool device_has_dsm(struct device *dev)
 		return false;
 
 	return !!acpi_check_dsm(handle, &pci_acpi_dsm_guid, 0x2,
-				1 << DEVICE_LABEL_DSM);
+				1 << DSM_PCI_DEVICE_NAME);
 }
 
 static umode_t acpi_index_string_exist(struct kobject *kobj,
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 595fcf59843f..ce096272f52b 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -752,30 +752,6 @@ struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
 EXPORT_SYMBOL(pci_find_resource);
 
 /**
- * pci_find_pcie_root_port - return PCIe Root Port
- * @dev: PCI device to query
- *
- * Traverse up the parent chain and return the PCIe Root Port PCI Device
- * for a given PCI Device.
- */
-struct pci_dev *pci_find_pcie_root_port(struct pci_dev *dev)
-{
-	struct pci_dev *bridge, *highest_pcie_bridge = dev;
-
-	bridge = pci_upstream_bridge(dev);
-	while (bridge && pci_is_pcie(bridge)) {
-		highest_pcie_bridge = bridge;
-		bridge = pci_upstream_bridge(bridge);
-	}
-
-	if (pci_pcie_type(highest_pcie_bridge) != PCI_EXP_TYPE_ROOT_PORT)
-		return NULL;
-
-	return highest_pcie_bridge;
-}
-EXPORT_SYMBOL(pci_find_pcie_root_port);
-
-/**
  * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
  * @dev: the PCI device to operate on
  * @pos: config space offset of status word
@@ -868,7 +844,9 @@ static inline bool platform_pci_need_resume(struct pci_dev *dev)
 
 static inline bool platform_pci_bridge_d3(struct pci_dev *dev)
 {
-	return pci_platform_pm ? pci_platform_pm->bridge_d3(dev) : false;
+	if (pci_platform_pm && pci_platform_pm->bridge_d3)
+		return pci_platform_pm->bridge_d3(dev);
+	return false;
 }
 
 /**
@@ -1578,7 +1556,7 @@ EXPORT_SYMBOL(pci_restore_state);
 
 struct pci_saved_state {
 	u32 config_space[16];
-	struct pci_cap_saved_data cap[0];
+	struct pci_cap_saved_data cap[];
 };
 
 /**
@@ -4660,7 +4638,8 @@ static int pci_pm_reset(struct pci_dev *dev, int probe)
  * pcie_wait_for_link_delay - Wait until link is active or inactive
  * @pdev: Bridge device
  * @active: waiting for active or inactive?
- * @delay: Delay to wait after link has become active (in ms)
+ * @delay: Delay to wait after link has become active (in ms). Specify %0
+ *	   for no delay.
  *
  * Use this to wait till link becomes active or inactive.
  */
@@ -4673,10 +4652,10 @@ static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
 
 	/*
 	 * Some controllers might not implement link active reporting. In this
-	 * case, we wait for 1000 + 100 ms.
+	 * case, we wait for 1000 ms + any delay requested by the caller.
 	 */
 	if (!pdev->link_active_reporting) {
-		msleep(1100);
+		msleep(timeout + delay);
 		return true;
 	}
 
@@ -4701,7 +4680,7 @@ static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
 		msleep(10);
 		timeout -= 10;
 	}
-	if (active && ret)
+	if (active && ret && delay)
 		msleep(delay);
 	else if (ret != active)
 		pci_info(pdev, "Data Link Layer Link Active not %s in 1000 msec\n",
@@ -4822,17 +4801,28 @@ void pci_bridge_wait_for_secondary_bus(struct pci_dev *dev)
 	if (!pcie_downstream_port(dev))
 		return;
 
-	if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) {
-		pci_dbg(dev, "waiting %d ms for downstream link\n", delay);
-		msleep(delay);
-	} else {
-		pci_dbg(dev, "waiting %d ms for downstream link, after activation\n",
-			delay);
-		if (!pcie_wait_for_link_delay(dev, true, delay)) {
+	/*
+	 * Per PCIe r5.0, sec 6.6.1, for downstream ports that support
+	 * speeds > 5 GT/s, we must wait for link training to complete
+	 * before the mandatory delay.
+	 *
+	 * We can only tell when link training completes via DLL Link
+	 * Active, which is required for downstream ports that support
+	 * speeds > 5 GT/s (sec 7.5.3.6).  Unfortunately some common
+	 * devices do not implement Link Active reporting even when it's
+	 * required, so we'll check for that directly instead of checking
+	 * the supported link speed.  We assume devices without Link Active
+	 * reporting can train in 100 ms regardless of speed.
+	 */
+	if (dev->link_active_reporting) {
+		pci_dbg(dev, "waiting for link to train\n");
+		if (!pcie_wait_for_link_delay(dev, true, 0)) {
 			/* Did not train, no need to wait any further */
 			return;
 		}
 	}
+	pci_dbg(child, "waiting %d ms to become accessible\n", delay);
+	msleep(delay);
 
 	if (!pci_device_is_present(child)) {
 		pci_dbg(child, "waiting additional %d ms to become accessible\n", delay);
diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig
index 66386811cfde..9cd31331aee9 100644
--- a/drivers/pci/pcie/Kconfig
+++ b/drivers/pci/pcie/Kconfig
@@ -25,7 +25,6 @@ config PCIEAER
 	bool "PCI Express Advanced Error Reporting support"
 	depends on PCIEPORTBUS
 	select RAS
-	default y
 	help
 	  This enables PCI Express Root Port Advanced Error Reporting
 	  (AER) driver support. Error reporting messages sent to Root
diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c
index f4274d301235..3acf56683915 100644
--- a/drivers/pci/pcie/aer.c
+++ b/drivers/pci/pcie/aer.c
@@ -136,22 +136,18 @@ static const char * const ecrc_policy_str[] = {
  */
 static int enable_ecrc_checking(struct pci_dev *dev)
 {
-	int pos;
+	int aer = dev->aer_cap;
 	u32 reg32;
 
-	if (!pci_is_pcie(dev))
+	if (!aer)
 		return -ENODEV;
 
-	pos = dev->aer_cap;
-	if (!pos)
-		return -ENODEV;
-
-	pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
+	pci_read_config_dword(dev, aer + PCI_ERR_CAP, &reg32);
 	if (reg32 & PCI_ERR_CAP_ECRC_GENC)
 		reg32 |= PCI_ERR_CAP_ECRC_GENE;
 	if (reg32 & PCI_ERR_CAP_ECRC_CHKC)
 		reg32 |= PCI_ERR_CAP_ECRC_CHKE;
-	pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
+	pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
 
 	return 0;
 }
@@ -164,19 +160,15 @@ static int enable_ecrc_checking(struct pci_dev *dev)
  */
 static int disable_ecrc_checking(struct pci_dev *dev)
 {
-	int pos;
+	int aer = dev->aer_cap;
 	u32 reg32;
 
-	if (!pci_is_pcie(dev))
+	if (!aer)
 		return -ENODEV;
 
-	pos = dev->aer_cap;
-	if (!pos)
-		return -ENODEV;
-
-	pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
+	pci_read_config_dword(dev, aer + PCI_ERR_CAP, &reg32);
 	reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
-	pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
+	pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
 
 	return 0;
 }
@@ -217,142 +209,22 @@ void pcie_ecrc_get_policy(char *str)
 }
 #endif	/* CONFIG_PCIE_ECRC */
 
-#ifdef CONFIG_ACPI_APEI
-static inline int hest_match_pci(struct acpi_hest_aer_common *p,
-				 struct pci_dev *pci)
-{
-	return   ACPI_HEST_SEGMENT(p->bus) == pci_domain_nr(pci->bus) &&
-		 ACPI_HEST_BUS(p->bus)     == pci->bus->number &&
-		 p->device                 == PCI_SLOT(pci->devfn) &&
-		 p->function               == PCI_FUNC(pci->devfn);
-}
-
-static inline bool hest_match_type(struct acpi_hest_header *hest_hdr,
-				struct pci_dev *dev)
-{
-	u16 hest_type = hest_hdr->type;
-	u8 pcie_type = pci_pcie_type(dev);
-
-	if ((hest_type == ACPI_HEST_TYPE_AER_ROOT_PORT &&
-		pcie_type == PCI_EXP_TYPE_ROOT_PORT) ||
-	    (hest_type == ACPI_HEST_TYPE_AER_ENDPOINT &&
-		pcie_type == PCI_EXP_TYPE_ENDPOINT) ||
-	    (hest_type == ACPI_HEST_TYPE_AER_BRIDGE &&
-		(dev->class >> 16) == PCI_BASE_CLASS_BRIDGE))
-		return true;
-	return false;
-}
-
-struct aer_hest_parse_info {
-	struct pci_dev *pci_dev;
-	int firmware_first;
-};
-
-static int hest_source_is_pcie_aer(struct acpi_hest_header *hest_hdr)
-{
-	if (hest_hdr->type == ACPI_HEST_TYPE_AER_ROOT_PORT ||
-	    hest_hdr->type == ACPI_HEST_TYPE_AER_ENDPOINT ||
-	    hest_hdr->type == ACPI_HEST_TYPE_AER_BRIDGE)
-		return 1;
-	return 0;
-}
-
-static int aer_hest_parse(struct acpi_hest_header *hest_hdr, void *data)
-{
-	struct aer_hest_parse_info *info = data;
-	struct acpi_hest_aer_common *p;
-	int ff;
-
-	if (!hest_source_is_pcie_aer(hest_hdr))
-		return 0;
-
-	p = (struct acpi_hest_aer_common *)(hest_hdr + 1);
-	ff = !!(p->flags & ACPI_HEST_FIRMWARE_FIRST);
-
-	/*
-	 * If no specific device is supplied, determine whether
-	 * FIRMWARE_FIRST is set for *any* PCIe device.
-	 */
-	if (!info->pci_dev) {
-		info->firmware_first |= ff;
-		return 0;
-	}
-
-	/* Otherwise, check the specific device */
-	if (p->flags & ACPI_HEST_GLOBAL) {
-		if (hest_match_type(hest_hdr, info->pci_dev))
-			info->firmware_first = ff;
-	} else
-		if (hest_match_pci(p, info->pci_dev))
-			info->firmware_first = ff;
-
-	return 0;
-}
-
-static void aer_set_firmware_first(struct pci_dev *pci_dev)
-{
-	int rc;
-	struct aer_hest_parse_info info = {
-		.pci_dev	= pci_dev,
-		.firmware_first	= 0,
-	};
-
-	rc = apei_hest_parse(aer_hest_parse, &info);
-
-	if (rc)
-		pci_dev->__aer_firmware_first = 0;
-	else
-		pci_dev->__aer_firmware_first = info.firmware_first;
-	pci_dev->__aer_firmware_first_valid = 1;
-}
+#define	PCI_EXP_AER_FLAGS	(PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \
+				 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE)
 
-int pcie_aer_get_firmware_first(struct pci_dev *dev)
+int pcie_aer_is_native(struct pci_dev *dev)
 {
-	if (!pci_is_pcie(dev))
-		return 0;
+	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
 
-	if (pcie_ports_native)
+	if (!dev->aer_cap)
 		return 0;
 
-	if (!dev->__aer_firmware_first_valid)
-		aer_set_firmware_first(dev);
-	return dev->__aer_firmware_first;
-}
-
-static bool aer_firmware_first;
-
-/**
- * aer_acpi_firmware_first - Check if APEI should control AER.
- */
-bool aer_acpi_firmware_first(void)
-{
-	static bool parsed = false;
-	struct aer_hest_parse_info info = {
-		.pci_dev	= NULL,	/* Check all PCIe devices */
-		.firmware_first	= 0,
-	};
-
-	if (pcie_ports_native)
-		return false;
-
-	if (!parsed) {
-		apei_hest_parse(aer_hest_parse, &info);
-		aer_firmware_first = info.firmware_first;
-		parsed = true;
-	}
-	return aer_firmware_first;
+	return pcie_ports_native || host->native_aer;
 }
-#endif
-
-#define	PCI_EXP_AER_FLAGS	(PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \
-				 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE)
 
 int pci_enable_pcie_error_reporting(struct pci_dev *dev)
 {
-	if (pcie_aer_get_firmware_first(dev))
-		return -EIO;
-
-	if (!dev->aer_cap)
+	if (!pcie_aer_is_native(dev))
 		return -EIO;
 
 	return pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
@@ -361,7 +233,7 @@ EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting);
 
 int pci_disable_pcie_error_reporting(struct pci_dev *dev)
 {
-	if (pcie_aer_get_firmware_first(dev))
+	if (!pcie_aer_is_native(dev))
 		return -EIO;
 
 	return pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
@@ -379,22 +251,18 @@ void pci_aer_clear_device_status(struct pci_dev *dev)
 
 int pci_aer_clear_nonfatal_status(struct pci_dev *dev)
 {
-	int pos;
+	int aer = dev->aer_cap;
 	u32 status, sev;
 
-	pos = dev->aer_cap;
-	if (!pos)
-		return -EIO;
-
-	if (pcie_aer_get_firmware_first(dev))
+	if (!pcie_aer_is_native(dev))
 		return -EIO;
 
 	/* Clear status bits for ERR_NONFATAL errors only */
-	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
-	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &sev);
+	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
+	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
 	status &= ~sev;
 	if (status)
-		pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
+		pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
 
 	return 0;
 }
@@ -402,22 +270,18 @@ EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status);
 
 void pci_aer_clear_fatal_status(struct pci_dev *dev)
 {
-	int pos;
+	int aer = dev->aer_cap;
 	u32 status, sev;
 
-	pos = dev->aer_cap;
-	if (!pos)
-		return;
-
-	if (pcie_aer_get_firmware_first(dev))
+	if (!pcie_aer_is_native(dev))
 		return;
 
 	/* Clear status bits for ERR_FATAL errors only */
-	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
-	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &sev);
+	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
+	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
 	status &= sev;
 	if (status)
-		pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
+		pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
 }
 
 /**
@@ -431,35 +295,31 @@ void pci_aer_clear_fatal_status(struct pci_dev *dev)
  */
 int pci_aer_raw_clear_status(struct pci_dev *dev)
 {
-	int pos;
+	int aer = dev->aer_cap;
 	u32 status;
 	int port_type;
 
-	if (!pci_is_pcie(dev))
-		return -ENODEV;
-
-	pos = dev->aer_cap;
-	if (!pos)
+	if (!aer)
 		return -EIO;
 
 	port_type = pci_pcie_type(dev);
 	if (port_type == PCI_EXP_TYPE_ROOT_PORT) {
-		pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, &status);
-		pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, status);
+		pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status);
+		pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status);
 	}
 
-	pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
-	pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, status);
+	pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
+	pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status);
 
-	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
-	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
+	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
+	pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
 
 	return 0;
 }
 
 int pci_aer_clear_status(struct pci_dev *dev)
 {
-	if (pcie_aer_get_firmware_first(dev))
+	if (!pcie_aer_is_native(dev))
 		return -EIO;
 
 	return pci_aer_raw_clear_status(dev);
@@ -467,12 +327,11 @@ int pci_aer_clear_status(struct pci_dev *dev)
 
 void pci_save_aer_state(struct pci_dev *dev)
 {
+	int aer = dev->aer_cap;
 	struct pci_cap_saved_state *save_state;
 	u32 *cap;
-	int pos;
 
-	pos = dev->aer_cap;
-	if (!pos)
+	if (!aer)
 		return;
 
 	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
@@ -480,22 +339,21 @@ void pci_save_aer_state(struct pci_dev *dev)
 		return;
 
 	cap = &save_state->cap.data[0];
-	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, cap++);
-	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, cap++);
-	pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, cap++);
-	pci_read_config_dword(dev, pos + PCI_ERR_CAP, cap++);
+	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++);
+	pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++);
+	pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++);
+	pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++);
 	if (pcie_cap_has_rtctl(dev))
-		pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, cap++);
+		pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++);
 }
 
 void pci_restore_aer_state(struct pci_dev *dev)
 {
+	int aer = dev->aer_cap;
 	struct pci_cap_saved_state *save_state;
 	u32 *cap;
-	int pos;
 
-	pos = dev->aer_cap;
-	if (!pos)
+	if (!aer)
 		return;
 
 	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
@@ -503,12 +361,12 @@ void pci_restore_aer_state(struct pci_dev *dev)
 		return;
 
 	cap = &save_state->cap.data[0];
-	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, *cap++);
-	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, *cap++);
-	pci_write_config_dword(dev, pos + PCI_ERR_COR_MASK, *cap++);
-	pci_write_config_dword(dev, pos + PCI_ERR_CAP, *cap++);
+	pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++);
+	pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++);
+	pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++);
+	pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++);
 	if (pcie_cap_has_rtctl(dev))
-		pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, *cap++);
+		pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++);
 }
 
 void pci_aer_init(struct pci_dev *dev)
@@ -939,7 +797,7 @@ static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
  */
 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
 {
-	int pos;
+	int aer = dev->aer_cap;
 	u32 status, mask;
 	u16 reg16;
 
@@ -974,17 +832,16 @@ static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
 	if (!(reg16 & PCI_EXP_AER_FLAGS))
 		return false;
 
-	pos = dev->aer_cap;
-	if (!pos)
+	if (!aer)
 		return false;
 
 	/* Check if error is recorded */
 	if (e_info->severity == AER_CORRECTABLE) {
-		pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
-		pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, &mask);
+		pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
+		pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);
 	} else {
-		pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
-		pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, &mask);
+		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
+		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);
 	}
 	if (status & ~mask)
 		return true;
@@ -1055,16 +912,15 @@ static bool find_source_device(struct pci_dev *parent,
  */
 static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info)
 {
-	int pos;
+	int aer = dev->aer_cap;
 
 	if (info->severity == AER_CORRECTABLE) {
 		/*
 		 * Correctable error does not need software intervention.
 		 * No need to go through error recovery process.
 		 */
-		pos = dev->aer_cap;
-		if (pos)
-			pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
+		if (aer)
+			pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS,
 					info->status);
 		pci_aer_clear_device_status(dev);
 	} else if (info->severity == AER_NONFATAL)
@@ -1155,22 +1011,21 @@ EXPORT_SYMBOL_GPL(aer_recover_queue);
  */
 int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 {
-	int pos, temp;
+	int aer = dev->aer_cap;
+	int temp;
 
 	/* Must reset in this function */
 	info->status = 0;
 	info->tlp_header_valid = 0;
 
-	pos = dev->aer_cap;
-
 	/* The device might not support AER */
-	if (!pos)
+	if (!aer)
 		return 0;
 
 	if (info->severity == AER_CORRECTABLE) {
-		pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS,
+		pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS,
 			&info->status);
-		pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK,
+		pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK,
 			&info->mask);
 		if (!(info->status & ~info->mask))
 			return 0;
@@ -1179,27 +1034,27 @@ int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 		   info->severity == AER_NONFATAL) {
 
 		/* Link is still healthy for IO reads */
-		pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS,
+		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS,
 			&info->status);
-		pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK,
+		pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK,
 			&info->mask);
 		if (!(info->status & ~info->mask))
 			return 0;
 
 		/* Get First Error Pointer */
-		pci_read_config_dword(dev, pos + PCI_ERR_CAP, &temp);
+		pci_read_config_dword(dev, aer + PCI_ERR_CAP, &temp);
 		info->first_error = PCI_ERR_CAP_FEP(temp);
 
 		if (info->status & AER_LOG_TLP_MASKS) {
 			info->tlp_header_valid = 1;
 			pci_read_config_dword(dev,
-				pos + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
+				aer + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
 			pci_read_config_dword(dev,
-				pos + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
+				aer + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
 			pci_read_config_dword(dev,
-				pos + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
+				aer + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
 			pci_read_config_dword(dev,
-				pos + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
+				aer + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
 		}
 	}
 
@@ -1305,15 +1160,15 @@ static irqreturn_t aer_irq(int irq, void *context)
 	struct pcie_device *pdev = (struct pcie_device *)context;
 	struct aer_rpc *rpc = get_service_data(pdev);
 	struct pci_dev *rp = rpc->rpd;
+	int aer = rp->aer_cap;
 	struct aer_err_source e_src = {};
-	int pos = rp->aer_cap;
 
-	pci_read_config_dword(rp, pos + PCI_ERR_ROOT_STATUS, &e_src.status);
+	pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status);
 	if (!(e_src.status & (PCI_ERR_ROOT_UNCOR_RCV|PCI_ERR_ROOT_COR_RCV)))
 		return IRQ_NONE;
 
-	pci_read_config_dword(rp, pos + PCI_ERR_ROOT_ERR_SRC, &e_src.id);
-	pci_write_config_dword(rp, pos + PCI_ERR_ROOT_STATUS, e_src.status);
+	pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id);
+	pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status);
 
 	if (!kfifo_put(&rpc->aer_fifo, e_src))
 		return IRQ_HANDLED;
@@ -1365,7 +1220,7 @@ static void set_downstream_devices_error_reporting(struct pci_dev *dev,
 static void aer_enable_rootport(struct aer_rpc *rpc)
 {
 	struct pci_dev *pdev = rpc->rpd;
-	int aer_pos;
+	int aer = pdev->aer_cap;
 	u16 reg16;
 	u32 reg32;
 
@@ -1377,14 +1232,13 @@ static void aer_enable_rootport(struct aer_rpc *rpc)
 	pcie_capability_clear_word(pdev, PCI_EXP_RTCTL,
 				   SYSTEM_ERROR_INTR_ON_MESG_MASK);
 
-	aer_pos = pdev->aer_cap;
 	/* Clear error status */
-	pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, &reg32);
-	pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32);
-	pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, &reg32);
-	pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32);
-	pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, &reg32);
-	pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32);
+	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, &reg32);
+	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
+	pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, &reg32);
+	pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32);
+	pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, &reg32);
+	pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32);
 
 	/*
 	 * Enable error reporting for the root port device and downstream port
@@ -1393,9 +1247,9 @@ static void aer_enable_rootport(struct aer_rpc *rpc)
 	set_downstream_devices_error_reporting(pdev, true);
 
 	/* Enable Root Port's interrupt in response to error messages */
-	pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, &reg32);
+	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
 	reg32 |= ROOT_PORT_INTR_ON_MESG_MASK;
-	pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, reg32);
+	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
 }
 
 /**
@@ -1407,8 +1261,8 @@ static void aer_enable_rootport(struct aer_rpc *rpc)
 static void aer_disable_rootport(struct aer_rpc *rpc)
 {
 	struct pci_dev *pdev = rpc->rpd;
+	int aer = pdev->aer_cap;
 	u32 reg32;
-	int pos;
 
 	/*
 	 * Disable error reporting for the root port device and downstream port
@@ -1416,15 +1270,14 @@ static void aer_disable_rootport(struct aer_rpc *rpc)
 	 */
 	set_downstream_devices_error_reporting(pdev, false);
 
-	pos = pdev->aer_cap;
 	/* Disable Root's interrupt in response to error messages */
-	pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
+	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
 	reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
-	pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, reg32);
+	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
 
 	/* Clear Root's error status reg */
-	pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, &reg32);
-	pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32);
+	pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, &reg32);
+	pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
 }
 
 /**
@@ -1481,28 +1334,27 @@ static int aer_probe(struct pcie_device *dev)
  */
 static pci_ers_result_t aer_root_reset(struct pci_dev *dev)
 {
+	int aer = dev->aer_cap;
 	u32 reg32;
-	int pos;
 	int rc;
 
-	pos = dev->aer_cap;
 
 	/* Disable Root's interrupt in response to error messages */
-	pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
+	pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
 	reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
-	pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32);
+	pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, reg32);
 
 	rc = pci_bus_error_reset(dev);
 	pci_info(dev, "Root Port link has been reset\n");
 
 	/* Clear Root Error Status */
-	pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, &reg32);
-	pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, reg32);
+	pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &reg32);
+	pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, reg32);
 
 	/* Enable Root Port's interrupt in response to error messages */
-	pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
+	pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
 	reg32 |= ROOT_PORT_INTR_ON_MESG_MASK;
-	pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32);
+	pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, reg32);
 
 	return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
 }
@@ -1523,7 +1375,7 @@ static struct pcie_port_service_driver aerdriver = {
  */
 int __init pcie_aer_init(void)
 {
-	if (!pci_aer_available() || aer_acpi_firmware_first())
+	if (!pci_aer_available())
 		return -ENXIO;
 	return pcie_port_service_register(&aerdriver);
 }
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 2378ed692534..b17e5ffd31b1 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -628,16 +628,6 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
 
 	/* Setup initial capable state. Will be updated later */
 	link->aspm_capable = link->aspm_support;
-	/*
-	 * If the downstream component has pci bridge function, don't
-	 * do ASPM for now.
-	 */
-	list_for_each_entry(child, &linkbus->devices, bus_list) {
-		if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) {
-			link->aspm_disable = ASPM_STATE_ALL;
-			break;
-		}
-	}
 
 	/* Get and check endpoint acceptable latencies */
 	list_for_each_entry(child, &linkbus->devices, bus_list) {
diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c
index 762170423fdd..daa9a4153776 100644
--- a/drivers/pci/pcie/dpc.c
+++ b/drivers/pci/pcie/dpc.c
@@ -284,7 +284,7 @@ static int dpc_probe(struct pcie_device *dev)
 	int status;
 	u16 ctl, cap;
 
-	if (pcie_aer_get_firmware_first(pdev) && !pcie_ports_dpc_native)
+	if (!pcie_aer_is_native(pdev) && !pcie_ports_dpc_native)
 		return -ENOTSUPP;
 
 	status = devm_request_threaded_irq(device, dev->irq, dpc_irq,
@@ -301,6 +301,7 @@ static int dpc_probe(struct pcie_device *dev)
 
 	ctl = (ctl & 0xfff4) | PCI_EXP_DPC_CTL_EN_FATAL | PCI_EXP_DPC_CTL_INT_EN;
 	pci_write_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CTL, ctl);
+	pci_info(pdev, "enabled with IRQ %d\n", dev->irq);
 
 	pci_info(pdev, "error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
 		 cap & PCI_EXP_DPC_IRQ, FLAG(cap, PCI_EXP_DPC_CAP_RP_EXT),
diff --git a/drivers/pci/pcie/edr.c b/drivers/pci/pcie/edr.c
index 594622a6cb16..a6b9b479b97a 100644
--- a/drivers/pci/pcie/edr.c
+++ b/drivers/pci/pcie/edr.c
@@ -148,11 +148,11 @@ static void edr_handle_event(acpi_handle handle, u32 event, void *data)
 	pci_ers_result_t estate = PCI_ERS_RESULT_DISCONNECT;
 	u16 status;
 
-	pci_info(pdev, "ACPI event %#x received\n", event);
-
 	if (event != ACPI_NOTIFY_DISCONNECT_RECOVER)
 		return;
 
+	pci_info(pdev, "EDR event received\n");
+
 	/* Locate the port which issued EDR event */
 	edev = acpi_dpc_port_get(pdev);
 	if (!edev) {
diff --git a/drivers/pci/pcie/pme.c b/drivers/pci/pcie/pme.c
index f38e6c19dd50..6a32970bb731 100644
--- a/drivers/pci/pcie/pme.c
+++ b/drivers/pci/pcie/pme.c
@@ -408,7 +408,7 @@ static int pcie_pme_suspend(struct pcie_device *srv)
 
 /**
  * pcie_pme_resume - Resume PCIe PME service device.
- * @srv - PCIe service device to resume.
+ * @srv: PCIe service device to resume.
  */
 static int pcie_pme_resume(struct pcie_device *srv)
 {
@@ -431,7 +431,7 @@ static int pcie_pme_resume(struct pcie_device *srv)
 
 /**
  * pcie_pme_remove - Prepare PCIe PME service device for removal.
- * @srv - PCIe service device to remove.
+ * @srv: PCIe service device to remove.
  */
 static void pcie_pme_remove(struct pcie_device *srv)
 {
diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h
index 64b5e081cdb2..af7cf237432a 100644
--- a/drivers/pci/pcie/portdrv.h
+++ b/drivers/pci/pcie/portdrv.h
@@ -29,8 +29,10 @@ extern bool pcie_ports_dpc_native;
 
 #ifdef CONFIG_PCIEAER
 int pcie_aer_init(void);
+int pcie_aer_is_native(struct pci_dev *dev);
 #else
 static inline int pcie_aer_init(void) { return 0; }
+static inline int pcie_aer_is_native(struct pci_dev *dev) { return 0; }
 #endif
 
 #ifdef CONFIG_HOTPLUG_PCI_PCIE
@@ -147,16 +149,5 @@ static inline bool pcie_pme_no_msi(void) { return false; }
 static inline void pcie_pme_interrupt_enable(struct pci_dev *dev, bool en) {}
 #endif /* !CONFIG_PCIE_PME */
 
-#ifdef CONFIG_ACPI_APEI
-int pcie_aer_get_firmware_first(struct pci_dev *pci_dev);
-#else
-static inline int pcie_aer_get_firmware_first(struct pci_dev *pci_dev)
-{
-	if (pci_dev->__aer_firmware_first_valid)
-		return pci_dev->__aer_firmware_first;
-	return 0;
-}
-#endif
-
 struct device *pcie_port_find_device(struct pci_dev *dev, u32 service);
 #endif /* _PORTDRV_H_ */
diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c
index 9361f3aa26ab..357a454cafa0 100644
--- a/drivers/pci/pcie/ptm.c
+++ b/drivers/pci/pcie/ptm.c
@@ -39,10 +39,6 @@ void pci_ptm_init(struct pci_dev *dev)
 	if (!pci_is_pcie(dev))
 		return;
 
-	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
-	if (!pos)
-		return;
-
 	/*
 	 * Enable PTM only on interior devices (root ports, switch ports,
 	 * etc.) on the assumption that it causes no link traffic until an
@@ -52,6 +48,23 @@ void pci_ptm_init(struct pci_dev *dev)
 	     pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END))
 		return;
 
+	/*
+	 * Switch Downstream Ports are not permitted to have a PTM
+	 * capability; their PTM behavior is controlled by the Upstream
+	 * Port (PCIe r5.0, sec 7.9.16).
+	 */
+	ups = pci_upstream_bridge(dev);
+	if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM &&
+	    ups && ups->ptm_enabled) {
+		dev->ptm_granularity = ups->ptm_granularity;
+		dev->ptm_enabled = 1;
+		return;
+	}
+
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM);
+	if (!pos)
+		return;
+
 	pci_read_config_dword(dev, pos + PCI_PTM_CAP, &cap);
 	local_clock = (cap & PCI_PTM_GRANULARITY_MASK) >> 8;
 
@@ -61,7 +74,6 @@ void pci_ptm_init(struct pci_dev *dev)
 	 * the spec recommendation (PCIe r3.1, sec 7.32.3), select the
 	 * furthest upstream Time Source as the PTM Root.
 	 */
-	ups = pci_upstream_bridge(dev);
 	if (ups && ups->ptm_enabled) {
 		ctrl = PCI_PTM_CTRL_ENABLE;
 		if (ups->ptm_granularity == 0)
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index d9c2c3301a8a..2f66988cea25 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -565,7 +565,7 @@ static struct pci_bus *pci_alloc_bus(struct pci_bus *parent)
 	return b;
 }
 
-static void devm_pci_release_host_bridge_dev(struct device *dev)
+static void pci_release_host_bridge_dev(struct device *dev)
 {
 	struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
 
@@ -574,12 +574,7 @@ static void devm_pci_release_host_bridge_dev(struct device *dev)
 
 	pci_free_resource_list(&bridge->windows);
 	pci_free_resource_list(&bridge->dma_ranges);
-}
-
-static void pci_release_host_bridge_dev(struct device *dev)
-{
-	devm_pci_release_host_bridge_dev(dev);
-	kfree(to_pci_host_bridge(dev));
+	kfree(bridge);
 }
 
 static void pci_init_host_bridge(struct pci_host_bridge *bridge)
@@ -599,6 +594,8 @@ static void pci_init_host_bridge(struct pci_host_bridge *bridge)
 	bridge->native_pme = 1;
 	bridge->native_ltr = 1;
 	bridge->native_dpc = 1;
+
+	device_initialize(&bridge->dev);
 }
 
 struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
@@ -616,17 +613,25 @@ struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
 }
 EXPORT_SYMBOL(pci_alloc_host_bridge);
 
+static void devm_pci_alloc_host_bridge_release(void *data)
+{
+	pci_free_host_bridge(data);
+}
+
 struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
 						   size_t priv)
 {
+	int ret;
 	struct pci_host_bridge *bridge;
 
-	bridge = devm_kzalloc(dev, sizeof(*bridge) + priv, GFP_KERNEL);
+	bridge = pci_alloc_host_bridge(priv);
 	if (!bridge)
 		return NULL;
 
-	pci_init_host_bridge(bridge);
-	bridge->dev.release = devm_pci_release_host_bridge_dev;
+	ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
+				       bridge);
+	if (ret)
+		return NULL;
 
 	return bridge;
 }
@@ -634,10 +639,7 @@ EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
 
 void pci_free_host_bridge(struct pci_host_bridge *bridge)
 {
-	pci_free_resource_list(&bridge->windows);
-	pci_free_resource_list(&bridge->dma_ranges);
-
-	kfree(bridge);
+	put_device(&bridge->dev);
 }
 EXPORT_SYMBOL(pci_free_host_bridge);
 
@@ -908,10 +910,11 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 	if (err)
 		goto free;
 
-	err = device_register(&bridge->dev);
-	if (err)
+	err = device_add(&bridge->dev);
+	if (err) {
 		put_device(&bridge->dev);
-
+		goto free;
+	}
 	bus->bridge = get_device(&bridge->dev);
 	device_enable_async_suspend(bus->bridge);
 	pci_set_bus_of_node(bus);
@@ -977,7 +980,7 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 
 unregister:
 	put_device(&bridge->dev);
-	device_unregister(&bridge->dev);
+	device_del(&bridge->dev);
 
 free:
 	kfree(bus);
@@ -1934,13 +1937,33 @@ static void pci_configure_mps(struct pci_dev *dev)
 	struct pci_dev *bridge = pci_upstream_bridge(dev);
 	int mps, mpss, p_mps, rc;
 
-	if (!pci_is_pcie(dev) || !bridge || !pci_is_pcie(bridge))
+	if (!pci_is_pcie(dev))
 		return;
 
 	/* MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out */
 	if (dev->is_virtfn)
 		return;
 
+	/*
+	 * For Root Complex Integrated Endpoints, program the maximum
+	 * supported value unless limited by the PCIE_BUS_PEER2PEER case.
+	 */
+	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
+		if (pcie_bus_config == PCIE_BUS_PEER2PEER)
+			mps = 128;
+		else
+			mps = 128 << dev->pcie_mpss;
+		rc = pcie_set_mps(dev, mps);
+		if (rc) {
+			pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
+				 mps);
+		}
+		return;
+	}
+
+	if (!bridge || !pci_is_pcie(bridge))
+		return;
+
 	mps = pcie_get_mps(dev);
 	p_mps = pcie_get_mps(bridge);
 
@@ -2056,7 +2079,7 @@ static void pci_configure_relaxed_ordering(struct pci_dev *dev)
 	 * For now, we only deal with Relaxed Ordering issues with Root
 	 * Ports. Peer-to-Peer DMA is another can of worms.
 	 */
-	root = pci_find_pcie_root_port(dev);
+	root = pcie_find_root_port(dev);
 	if (!root)
 		return;
 
@@ -2952,7 +2975,7 @@ struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
 	return bridge->bus;
 
 err_out:
-	kfree(bridge);
+	put_device(&bridge->dev);
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(pci_create_root_bus);
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index ca9ed5774eb1..812bfc32ecb8 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -4319,7 +4319,7 @@ DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_AMD, 0x1a02, PCI_CLASS_NOT_DEFINED,
  */
 static void quirk_disable_root_port_attributes(struct pci_dev *pdev)
 {
-	struct pci_dev *root_port = pci_find_pcie_root_port(pdev);
+	struct pci_dev *root_port = pcie_find_root_port(pdev);
 
 	if (!root_port) {
 		pci_warn(pdev, "PCIe Completion erratum may cause device errors\n");
@@ -4682,6 +4682,20 @@ static int pci_quirk_mf_endpoint_acs(struct pci_dev *dev, u16 acs_flags)
 		PCI_ACS_CR | PCI_ACS_UF | PCI_ACS_DT);
 }
 
+static int pci_quirk_rciep_acs(struct pci_dev *dev, u16 acs_flags)
+{
+	/*
+	 * Intel RCiEP's are required to allow p2p only on translated
+	 * addresses.  Refer to Intel VT-d specification, r3.1, sec 3.16,
+	 * "Root-Complex Peer to Peer Considerations".
+	 */
+	if (pci_pcie_type(dev) != PCI_EXP_TYPE_RC_END)
+		return -ENOTTY;
+
+	return pci_acs_ctrl_enabled(acs_flags,
+		PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF);
+}
+
 static int pci_quirk_brcm_acs(struct pci_dev *dev, u16 acs_flags)
 {
 	/*
@@ -4764,6 +4778,7 @@ static const struct pci_dev_acs_enabled {
 	/* I219 */
 	{ PCI_VENDOR_ID_INTEL, 0x15b7, pci_quirk_mf_endpoint_acs },
 	{ PCI_VENDOR_ID_INTEL, 0x15b8, pci_quirk_mf_endpoint_acs },
+	{ PCI_VENDOR_ID_INTEL, PCI_ANY_ID, pci_quirk_rciep_acs },
 	/* QCOM QDF2xxx root ports */
 	{ PCI_VENDOR_ID_QCOM, 0x0400, pci_quirk_qcom_rp_acs },
 	{ PCI_VENDOR_ID_QCOM, 0x0401, pci_quirk_qcom_rp_acs },
@@ -5129,13 +5144,25 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev)
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap);
 
-/* FLR may cause some 82579 devices to hang */
-static void quirk_intel_no_flr(struct pci_dev *dev)
+/*
+ * FLR may cause the following to devices to hang:
+ *
+ * AMD Starship/Matisse HD Audio Controller 0x1487
+ * AMD Starship USB 3.0 Host Controller 0x148c
+ * AMD Matisse USB 3.0 Host Controller 0x149c
+ * Intel 82579LM Gigabit Ethernet Controller 0x1502
+ * Intel 82579V Gigabit Ethernet Controller 0x1503
+ *
+ */
+static void quirk_no_flr(struct pci_dev *dev)
 {
 	dev->dev_flags |= PCI_DEV_FLAGS_NO_FLR_RESET;
 }
-DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_intel_no_flr);
-DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_intel_no_flr);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x1487, quirk_no_flr);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x148c, quirk_no_flr);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x149c, quirk_no_flr);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_no_flr);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_no_flr);
 
 static void quirk_no_ext_tags(struct pci_dev *pdev)
 {
@@ -5568,6 +5595,19 @@ static void pci_fixup_no_d0_pme(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ASMEDIA, 0x2142, pci_fixup_no_d0_pme);
 
+/*
+ * Device [12d8:0x400e] and [12d8:0x400f]
+ * These devices advertise PME# support in all power states but don't
+ * reliably assert it.
+ */
+static void pci_fixup_no_pme(struct pci_dev *dev)
+{
+	pci_info(dev, "PME# is unreliable, disabling it\n");
+	dev->pme_support = 0;
+}
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_PERICOM, 0x400e, pci_fixup_no_pme);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_PERICOM, 0x400f, pci_fixup_no_pme);
+
 static void apex_pci_fixup_class(struct pci_dev *pdev)
 {
 	pdev->class = (PCI_CLASS_SYSTEM_OTHER << 8) | pdev->class;
diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c
index e9c6b120cf45..95dec03d9f2a 100644
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -160,6 +160,6 @@ void pci_remove_root_bus(struct pci_bus *bus)
 	host_bridge->bus = NULL;
 
 	/* remove the host bridge */
-	device_unregister(&host_bridge->dev);
+	device_del(&host_bridge->dev);
 }
 EXPORT_SYMBOL_GPL(pci_remove_root_bus);
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index bbcef1a053ab..9b94b1f16d80 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -26,6 +26,7 @@
 #include "pci.h"
 
 unsigned int pci_flags;
+EXPORT_SYMBOL_GPL(pci_flags);
 
 struct pci_dev_resource {
 	struct list_head list;
@@ -583,7 +584,7 @@ static void pci_setup_bridge_io(struct pci_dev *bridge)
 		io_mask = PCI_IO_1K_RANGE_MASK;
 
 	/* Set up the top and bottom of the PCI I/O segment for this bus */
-	res = &bridge->resource[PCI_BRIDGE_RESOURCES + 0];
+	res = &bridge->resource[PCI_BRIDGE_IO_WINDOW];
 	pcibios_resource_to_bus(bridge->bus, &region, res);
 	if (res->flags & IORESOURCE_IO) {
 		pci_read_config_word(bridge, PCI_IO_BASE, &l);
@@ -613,7 +614,7 @@ static void pci_setup_bridge_mmio(struct pci_dev *bridge)
 	u32 l;
 
 	/* Set up the top and bottom of the PCI Memory segment for this bus */
-	res = &bridge->resource[PCI_BRIDGE_RESOURCES + 1];
+	res = &bridge->resource[PCI_BRIDGE_MEM_WINDOW];
 	pcibios_resource_to_bus(bridge->bus, &region, res);
 	if (res->flags & IORESOURCE_MEM) {
 		l = (region.start >> 16) & 0xfff0;
@@ -640,7 +641,7 @@ static void pci_setup_bridge_mmio_pref(struct pci_dev *bridge)
 
 	/* Set up PREF base/limit */
 	bu = lu = 0;
-	res = &bridge->resource[PCI_BRIDGE_RESOURCES + 2];
+	res = &bridge->resource[PCI_BRIDGE_PREF_MEM_WINDOW];
 	pcibios_resource_to_bus(bridge->bus, &region, res);
 	if (res->flags & IORESOURCE_PREFETCH) {
 		l = (region.start >> 16) & 0xfff0;
@@ -707,14 +708,14 @@ int pci_claim_bridge_resource(struct pci_dev *bridge, int i)
 	if (!pci_bus_clip_resource(bridge, i))
 		return -EINVAL;	/* Clipping didn't change anything */
 
-	switch (i - PCI_BRIDGE_RESOURCES) {
-	case 0:
+	switch (i) {
+	case PCI_BRIDGE_IO_WINDOW:
 		pci_setup_bridge_io(bridge);
 		break;
-	case 1:
+	case PCI_BRIDGE_MEM_WINDOW:
 		pci_setup_bridge_mmio(bridge);
 		break;
-	case 2:
+	case PCI_BRIDGE_PREF_MEM_WINDOW:
 		pci_setup_bridge_mmio_pref(bridge);
 		break;
 	default:
@@ -735,18 +736,22 @@ int pci_claim_bridge_resource(struct pci_dev *bridge, int i)
 static void pci_bridge_check_ranges(struct pci_bus *bus)
 {
 	struct pci_dev *bridge = bus->self;
-	struct resource *b_res = &bridge->resource[PCI_BRIDGE_RESOURCES];
+	struct resource *b_res;
 
-	b_res[1].flags |= IORESOURCE_MEM;
+	b_res = &bridge->resource[PCI_BRIDGE_MEM_WINDOW];
+	b_res->flags |= IORESOURCE_MEM;
 
-	if (bridge->io_window)
-		b_res[0].flags |= IORESOURCE_IO;
+	if (bridge->io_window) {
+		b_res = &bridge->resource[PCI_BRIDGE_IO_WINDOW];
+		b_res->flags |= IORESOURCE_IO;
+	}
 
 	if (bridge->pref_window) {
-		b_res[2].flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
+		b_res = &bridge->resource[PCI_BRIDGE_PREF_MEM_WINDOW];
+		b_res->flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
 		if (bridge->pref_64_window) {
-			b_res[2].flags |= IORESOURCE_MEM_64;
-			b_res[2].flags |= PCI_PREF_RANGE_TYPE_64;
+			b_res->flags |= IORESOURCE_MEM_64 |
+					PCI_PREF_RANGE_TYPE_64;
 		}
 	}
 }
@@ -1105,35 +1110,37 @@ static void pci_bus_size_cardbus(struct pci_bus *bus,
 				 struct list_head *realloc_head)
 {
 	struct pci_dev *bridge = bus->self;
-	struct resource *b_res = &bridge->resource[PCI_BRIDGE_RESOURCES];
+	struct resource *b_res;
 	resource_size_t b_res_3_size = pci_cardbus_mem_size * 2;
 	u16 ctrl;
 
-	if (b_res[0].parent)
+	b_res = &bridge->resource[PCI_CB_BRIDGE_IO_0_WINDOW];
+	if (b_res->parent)
 		goto handle_b_res_1;
 	/*
 	 * Reserve some resources for CardBus.  We reserve a fixed amount
 	 * of bus space for CardBus bridges.
 	 */
-	b_res[0].start = pci_cardbus_io_size;
-	b_res[0].end = b_res[0].start + pci_cardbus_io_size - 1;
-	b_res[0].flags |= IORESOURCE_IO | IORESOURCE_STARTALIGN;
+	b_res->start = pci_cardbus_io_size;
+	b_res->end = b_res->start + pci_cardbus_io_size - 1;
+	b_res->flags |= IORESOURCE_IO | IORESOURCE_STARTALIGN;
 	if (realloc_head) {
-		b_res[0].end -= pci_cardbus_io_size;
+		b_res->end -= pci_cardbus_io_size;
 		add_to_list(realloc_head, bridge, b_res, pci_cardbus_io_size,
-				pci_cardbus_io_size);
+			    pci_cardbus_io_size);
 	}
 
 handle_b_res_1:
-	if (b_res[1].parent)
+	b_res = &bridge->resource[PCI_CB_BRIDGE_IO_1_WINDOW];
+	if (b_res->parent)
 		goto handle_b_res_2;
-	b_res[1].start = pci_cardbus_io_size;
-	b_res[1].end = b_res[1].start + pci_cardbus_io_size - 1;
-	b_res[1].flags |= IORESOURCE_IO | IORESOURCE_STARTALIGN;
+	b_res->start = pci_cardbus_io_size;
+	b_res->end = b_res->start + pci_cardbus_io_size - 1;
+	b_res->flags |= IORESOURCE_IO | IORESOURCE_STARTALIGN;
 	if (realloc_head) {
-		b_res[1].end -= pci_cardbus_io_size;
-		add_to_list(realloc_head, bridge, b_res+1, pci_cardbus_io_size,
-				 pci_cardbus_io_size);
+		b_res->end -= pci_cardbus_io_size;
+		add_to_list(realloc_head, bridge, b_res, pci_cardbus_io_size,
+			    pci_cardbus_io_size);
 	}
 
 handle_b_res_2:
@@ -1153,21 +1160,22 @@ handle_b_res_2:
 		pci_read_config_word(bridge, PCI_CB_BRIDGE_CONTROL, &ctrl);
 	}
 
-	if (b_res[2].parent)
+	b_res = &bridge->resource[PCI_CB_BRIDGE_MEM_0_WINDOW];
+	if (b_res->parent)
 		goto handle_b_res_3;
 	/*
 	 * If we have prefetchable memory support, allocate two regions.
 	 * Otherwise, allocate one region of twice the size.
 	 */
 	if (ctrl & PCI_CB_BRIDGE_CTL_PREFETCH_MEM0) {
-		b_res[2].start = pci_cardbus_mem_size;
-		b_res[2].end = b_res[2].start + pci_cardbus_mem_size - 1;
-		b_res[2].flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH |
-				  IORESOURCE_STARTALIGN;
+		b_res->start = pci_cardbus_mem_size;
+		b_res->end = b_res->start + pci_cardbus_mem_size - 1;
+		b_res->flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH |
+				    IORESOURCE_STARTALIGN;
 		if (realloc_head) {
-			b_res[2].end -= pci_cardbus_mem_size;
-			add_to_list(realloc_head, bridge, b_res+2,
-				 pci_cardbus_mem_size, pci_cardbus_mem_size);
+			b_res->end -= pci_cardbus_mem_size;
+			add_to_list(realloc_head, bridge, b_res,
+				    pci_cardbus_mem_size, pci_cardbus_mem_size);
 		}
 
 		/* Reduce that to half */
@@ -1175,15 +1183,16 @@ handle_b_res_2:
 	}
 
 handle_b_res_3:
-	if (b_res[3].parent)
+	b_res = &bridge->resource[PCI_CB_BRIDGE_MEM_1_WINDOW];
+	if (b_res->parent)
 		goto handle_done;
-	b_res[3].start = pci_cardbus_mem_size;
-	b_res[3].end = b_res[3].start + b_res_3_size - 1;
-	b_res[3].flags |= IORESOURCE_MEM | IORESOURCE_STARTALIGN;
+	b_res->start = pci_cardbus_mem_size;
+	b_res->end = b_res->start + b_res_3_size - 1;
+	b_res->flags |= IORESOURCE_MEM | IORESOURCE_STARTALIGN;
 	if (realloc_head) {
-		b_res[3].end -= b_res_3_size;
-		add_to_list(realloc_head, bridge, b_res+3, b_res_3_size,
-				 pci_cardbus_mem_size);
+		b_res->end -= b_res_3_size;
+		add_to_list(realloc_head, bridge, b_res, b_res_3_size,
+			    pci_cardbus_mem_size);
 	}
 
 handle_done:
@@ -1227,7 +1236,7 @@ void __pci_bus_size_bridges(struct pci_bus *bus, struct list_head *realloc_head)
 				break;
 		hdr_type = -1;	/* Intentionally invalid - not a PCI device. */
 	} else {
-		pref = &bus->self->resource[PCI_BRIDGE_RESOURCES + 2];
+		pref = &bus->self->resource[PCI_BRIDGE_PREF_MEM_WINDOW];
 		hdr_type = bus->self->hdr_type;
 	}
 
@@ -1885,9 +1894,9 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus,
 	struct pci_dev *dev, *bridge = bus->self;
 	resource_size_t io_per_hp, mmio_per_hp, mmio_pref_per_hp, align;
 
-	io_res = &bridge->resource[PCI_BRIDGE_RESOURCES + 0];
-	mmio_res = &bridge->resource[PCI_BRIDGE_RESOURCES + 1];
-	mmio_pref_res = &bridge->resource[PCI_BRIDGE_RESOURCES + 2];
+	io_res = &bridge->resource[PCI_BRIDGE_IO_WINDOW];
+	mmio_res = &bridge->resource[PCI_BRIDGE_MEM_WINDOW];
+	mmio_pref_res = &bridge->resource[PCI_BRIDGE_PREF_MEM_WINDOW];
 
 	/*
 	 * The alignment of this bridge is yet to be considered, hence it must
@@ -1960,21 +1969,21 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus,
 		 * Reduce the available resource space by what the
 		 * bridge and devices below it occupy.
 		 */
-		res = &dev->resource[PCI_BRIDGE_RESOURCES + 0];
+		res = &dev->resource[PCI_BRIDGE_IO_WINDOW];
 		align = pci_resource_alignment(dev, res);
 		align = align ? ALIGN(io.start, align) - io.start : 0;
 		used_size = align + resource_size(res);
 		if (!res->parent)
 			io.start = min(io.start + used_size, io.end + 1);
 
-		res = &dev->resource[PCI_BRIDGE_RESOURCES + 1];
+		res = &dev->resource[PCI_BRIDGE_MEM_WINDOW];
 		align = pci_resource_alignment(dev, res);
 		align = align ? ALIGN(mmio.start, align) - mmio.start : 0;
 		used_size = align + resource_size(res);
 		if (!res->parent)
 			mmio.start = min(mmio.start + used_size, mmio.end + 1);
 
-		res = &dev->resource[PCI_BRIDGE_RESOURCES + 2];
+		res = &dev->resource[PCI_BRIDGE_PREF_MEM_WINDOW];
 		align = pci_resource_alignment(dev, res);
 		align = align ? ALIGN(mmio_pref.start, align) -
 			mmio_pref.start : 0;
@@ -2027,9 +2036,9 @@ static void pci_bridge_distribute_available_resources(struct pci_dev *bridge,
 		return;
 
 	/* Take the initial extra resources from the hotplug port */
-	available_io = bridge->resource[PCI_BRIDGE_RESOURCES + 0];
-	available_mmio = bridge->resource[PCI_BRIDGE_RESOURCES + 1];
-	available_mmio_pref = bridge->resource[PCI_BRIDGE_RESOURCES + 2];
+	available_io = bridge->resource[PCI_BRIDGE_IO_WINDOW];
+	available_mmio = bridge->resource[PCI_BRIDGE_MEM_WINDOW];
+	available_mmio_pref = bridge->resource[PCI_BRIDGE_PREF_MEM_WINDOW];
 
 	pci_bus_distribute_available_resources(bridge->subordinate,
 					       add_list, available_io,
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index d8ca40a97693..d21fa04fa44d 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -439,10 +439,11 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
 	res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
 
 	/* Check if the new config works by trying to assign everything. */
-	ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
-	if (ret)
-		goto error_resize;
-
+	if (dev->bus->self) {
+		ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
+		if (ret)
+			goto error_resize;
+	}
 	return 0;
 
 error_resize:
diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c
index e69cac84b605..850cfeb74608 100644
--- a/drivers/pci/switch/switchtec.c
+++ b/drivers/pci/switch/switchtec.c
@@ -25,7 +25,7 @@ static int max_devices = 16;
 module_param(max_devices, int, 0644);
 MODULE_PARM_DESC(max_devices, "max number of switchtec device instances");
 
-static bool use_dma_mrpc = 1;
+static bool use_dma_mrpc = true;
 module_param(use_dma_mrpc, bool, 0644);
 MODULE_PARM_DESC(use_dma_mrpc,
 		 "Enable the use of the DMA MRPC feature");
diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c
index d1b16cf3403f..fab267e359e7 100644
--- a/drivers/pci/xen-pcifront.c
+++ b/drivers/pci/xen-pcifront.c
@@ -77,9 +77,6 @@ static inline void pcifront_init_sd(struct pcifront_sd *sd,
 static DEFINE_SPINLOCK(pcifront_dev_lock);
 static struct pcifront_device *pcifront_dev;
 
-static int verbose_request;
-module_param(verbose_request, int, 0644);
-
 static int errno_to_pcibios_err(int errno)
 {
 	switch (errno) {
@@ -190,18 +187,16 @@ static int pcifront_bus_read(struct pci_bus *bus, unsigned int devfn,
 	struct pcifront_sd *sd = bus->sysdata;
 	struct pcifront_device *pdev = pcifront_get_pdev(sd);
 
-	if (verbose_request)
-		dev_info(&pdev->xdev->dev,
-			 "read dev=%04x:%02x:%02x.%d - offset %x size %d\n",
-			 pci_domain_nr(bus), bus->number, PCI_SLOT(devfn),
-			 PCI_FUNC(devfn), where, size);
+	dev_dbg(&pdev->xdev->dev,
+		"read dev=%04x:%02x:%02x.%d - offset %x size %d\n",
+		pci_domain_nr(bus), bus->number, PCI_SLOT(devfn),
+		PCI_FUNC(devfn), where, size);
 
 	err = do_pci_op(pdev, &op);
 
 	if (likely(!err)) {
-		if (verbose_request)
-			dev_info(&pdev->xdev->dev, "read got back value %x\n",
-				 op.value);
+		dev_dbg(&pdev->xdev->dev, "read got back value %x\n",
+			op.value);
 
 		*val = op.value;
 	} else if (err == -ENODEV) {
@@ -229,12 +224,10 @@ static int pcifront_bus_write(struct pci_bus *bus, unsigned int devfn,
 	struct pcifront_sd *sd = bus->sysdata;
 	struct pcifront_device *pdev = pcifront_get_pdev(sd);
 
-	if (verbose_request)
-		dev_info(&pdev->xdev->dev,
-			 "write dev=%04x:%02x:%02x.%d - "
-			 "offset %x size %d val %x\n",
-			 pci_domain_nr(bus), bus->number,
-			 PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val);
+	dev_dbg(&pdev->xdev->dev,
+		"write dev=%04x:%02x:%02x.%d - offset %x size %d val %x\n",
+		pci_domain_nr(bus), bus->number,
+		PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val);
 
 	return errno_to_pcibios_err(do_pci_op(pdev, &op));
 }
diff --git a/drivers/pcmcia/Kconfig b/drivers/pcmcia/Kconfig
index e004d8da03dc..82d10b6661c7 100644
--- a/drivers/pcmcia/Kconfig
+++ b/drivers/pcmcia/Kconfig
@@ -6,7 +6,7 @@
 menuconfig PCCARD
 	tristate "PCCard (PCMCIA/CardBus) support"
 	depends on !UML
-	---help---
+	help
 	  Say Y here if you want to attach PCMCIA- or PC-cards to your Linux
 	  computer.  These are credit-card size devices such as network cards,
 	  modems or hard drives often used with laptops computers.  There are
@@ -22,7 +22,7 @@ config PCMCIA
 	tristate "16-bit PCMCIA support"
 	select CRC32
 	default y
-	---help---
+	help
 	   This option enables support for 16-bit PCMCIA cards. Most older
 	   PC-cards are such 16-bit PCMCIA cards, so unless you know you're
 	   only using 32-bit CardBus cards, say Y or M here.
@@ -54,7 +54,7 @@ config CARDBUS
 	bool "32-bit CardBus support"
 	depends on PCI
 	default y
-	---help---
+	help
 	  CardBus is a bus mastering architecture for PC-cards, which allows
 	  for 32 bit PC-cards (the original PCMCIA standard specifies only
 	  a 16 bit wide bus). Many newer PC-cards are actually CardBus cards.
@@ -75,7 +75,7 @@ config YENTA
 	depends on PCI
 	select CARDBUS if !EXPERT
 	select PCCARD_NONSTATIC if PCMCIA != n
-	---help---
+	help
 	  This option enables support for CardBus host bridges.  Virtually
 	  all modern PCMCIA bridges are CardBus compatible.  A "bridge" is
 	  the hardware inside your computer that PCMCIA cards are plugged
diff --git a/drivers/pcmcia/cs_internal.h b/drivers/pcmcia/cs_internal.h
index fb9b17fa0fb5..580369f3c0b0 100644
--- a/drivers/pcmcia/cs_internal.h
+++ b/drivers/pcmcia/cs_internal.h
@@ -164,12 +164,6 @@ int pcmcia_replace_cis(struct pcmcia_socket *s,
 int pccard_validate_cis(struct pcmcia_socket *s, unsigned int *count);
 int verify_cis_cache(struct pcmcia_socket *s);
 
-int pccard_loop_tuple(struct pcmcia_socket *s, unsigned int function,
-		      cisdata_t code, cisparse_t *parse, void *priv_data,
-		      int (*loop_tuple) (tuple_t *tuple,
-					 cisparse_t *parse,
-					 void *priv_data));
-
 int pccard_get_first_tuple(struct pcmcia_socket *s, unsigned int function,
 			tuple_t *tuple);
 
diff --git a/drivers/pcmcia/pcmcia_cis.c b/drivers/pcmcia/pcmcia_cis.c
index b553f7ab532f..e4c4daf92038 100644
--- a/drivers/pcmcia/pcmcia_cis.c
+++ b/drivers/pcmcia/pcmcia_cis.c
@@ -78,9 +78,9 @@ done:
  * calls the @loop_tuple function for each entry. If the call to @loop_tuple
  * returns 0, the loop exits. Returns 0 on success or errorcode otherwise.
  */
-int pccard_loop_tuple(struct pcmcia_socket *s, unsigned int function,
-		      cisdata_t code, cisparse_t *parse, void *priv_data,
-		      int (*loop_tuple) (tuple_t *tuple,
+static int pccard_loop_tuple(struct pcmcia_socket *s, unsigned int function,
+			     cisdata_t code, cisparse_t *parse, void *priv_data,
+			     int (*loop_tuple) (tuple_t *tuple,
 					 cisparse_t *parse,
 					 void *priv_data))
 {
diff --git a/drivers/pcmcia/yenta_socket.c b/drivers/pcmcia/yenta_socket.c
index bf6529b0b5b0..84bfc0e85d6b 100644
--- a/drivers/pcmcia/yenta_socket.c
+++ b/drivers/pcmcia/yenta_socket.c
@@ -694,7 +694,7 @@ static int yenta_allocate_res(struct yenta_socket *socket, int nr, unsigned type
 	struct pci_bus_region region;
 	unsigned mask;
 
-	res = dev->resource + PCI_BRIDGE_RESOURCES + nr;
+	res = &dev->resource[nr];
 	/* Already allocated? */
 	if (res->parent)
 		return 0;
@@ -711,7 +711,7 @@ static int yenta_allocate_res(struct yenta_socket *socket, int nr, unsigned type
 	region.end = config_readl(socket, addr_end) | ~mask;
 	if (region.start && region.end > region.start && !override_bios) {
 		pcibios_bus_to_resource(dev->bus, res, &region);
-		if (pci_claim_resource(dev, PCI_BRIDGE_RESOURCES + nr) == 0)
+		if (pci_claim_resource(dev, nr) == 0)
 			return 0;
 		dev_info(&dev->dev,
 			 "Preassigned resource %d busy or not available, reconfiguring...\n",
@@ -745,19 +745,35 @@ static int yenta_allocate_res(struct yenta_socket *socket, int nr, unsigned type
 	return 0;
 }
 
+static void yenta_free_res(struct yenta_socket *socket, int nr)
+{
+	struct pci_dev *dev = socket->dev;
+	struct resource *res;
+
+	res = &dev->resource[nr];
+	if (res->start != 0 && res->end != 0)
+		release_resource(res);
+
+	res->start = res->end = res->flags = 0;
+}
+
 /*
  * Allocate the bridge mappings for the device..
  */
 static void yenta_allocate_resources(struct yenta_socket *socket)
 {
 	int program = 0;
-	program += yenta_allocate_res(socket, 0, IORESOURCE_IO,
+	program += yenta_allocate_res(socket, PCI_CB_BRIDGE_IO_0_WINDOW,
+			   IORESOURCE_IO,
 			   PCI_CB_IO_BASE_0, PCI_CB_IO_LIMIT_0);
-	program += yenta_allocate_res(socket, 1, IORESOURCE_IO,
+	program += yenta_allocate_res(socket, PCI_CB_BRIDGE_IO_1_WINDOW,
+			   IORESOURCE_IO,
 			   PCI_CB_IO_BASE_1, PCI_CB_IO_LIMIT_1);
-	program += yenta_allocate_res(socket, 2, IORESOURCE_MEM|IORESOURCE_PREFETCH,
+	program += yenta_allocate_res(socket, PCI_CB_BRIDGE_MEM_0_WINDOW,
+			   IORESOURCE_MEM | IORESOURCE_PREFETCH,
 			   PCI_CB_MEMORY_BASE_0, PCI_CB_MEMORY_LIMIT_0);
-	program += yenta_allocate_res(socket, 3, IORESOURCE_MEM,
+	program += yenta_allocate_res(socket, PCI_CB_BRIDGE_MEM_1_WINDOW,
+			   IORESOURCE_MEM,
 			   PCI_CB_MEMORY_BASE_1, PCI_CB_MEMORY_LIMIT_1);
 	if (program)
 		pci_setup_cardbus(socket->dev->subordinate);
@@ -769,14 +785,10 @@ static void yenta_allocate_resources(struct yenta_socket *socket)
  */
 static void yenta_free_resources(struct yenta_socket *socket)
 {
-	int i;
-	for (i = 0; i < 4; i++) {
-		struct resource *res;
-		res = socket->dev->resource + PCI_BRIDGE_RESOURCES + i;
-		if (res->start != 0 && res->end != 0)
-			release_resource(res);
-		res->start = res->end = res->flags = 0;
-	}
+	yenta_free_res(socket, PCI_CB_BRIDGE_IO_0_WINDOW);
+	yenta_free_res(socket, PCI_CB_BRIDGE_IO_1_WINDOW);
+	yenta_free_res(socket, PCI_CB_BRIDGE_MEM_0_WINDOW);
+	yenta_free_res(socket, PCI_CB_BRIDGE_MEM_1_WINDOW);
 }
 
 
diff --git a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c
index 8dd1278bec04..7719ae4e2c56 100644
--- a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c
+++ b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c
@@ -35,7 +35,7 @@
 /* L3C has 8-counters */
 #define L3C_NR_COUNTERS		0x8
 
-#define L3C_PERF_CTRL_EN	0x20000
+#define L3C_PERF_CTRL_EN	0x10000
 #define L3C_EVTYPE_NONE		0xff
 
 /*
diff --git a/drivers/phy/amlogic/Kconfig b/drivers/phy/amlogic/Kconfig
index 71801e30d601..617cf073e9aa 100644
--- a/drivers/phy/amlogic/Kconfig
+++ b/drivers/phy/amlogic/Kconfig
@@ -3,12 +3,13 @@
 # Phy drivers for Amlogic platforms
 #
 config PHY_MESON8B_USB2
-	tristate "Meson8, Meson8b and GXBB USB2 PHY driver"
+	tristate "Meson8, Meson8b, Meson8m2 and GXBB USB2 PHY driver"
 	default ARCH_MESON
 	depends on OF && (ARCH_MESON || COMPILE_TEST)
 	depends on USB_SUPPORT
 	select USB_COMMON
 	select GENERIC_PHY
+	select REGMAP_MMIO
 	help
 	  Enable this to support the Meson USB2 PHYs found in Meson8,
 	  Meson8b and GXBB SoCs.
@@ -26,18 +27,6 @@ config PHY_MESON_GXL_USB2
 	  GXL and GXM SoCs.
 	  If unsure, say N.
 
-config PHY_MESON_GXL_USB3
-	tristate "Meson GXL and GXM USB3 PHY drivers"
-	default ARCH_MESON
-	depends on OF && (ARCH_MESON || COMPILE_TEST)
-	depends on USB_SUPPORT
-	select GENERIC_PHY
-	select REGMAP_MMIO
-	help
-	  Enable this to support the Meson USB3 PHY and OTG detection
-	  IP block found in Meson GXL and GXM SoCs.
-	  If unsure, say N.
-
 config PHY_MESON_G12A_USB2
 	tristate "Meson G12A USB2 PHY driver"
 	default ARCH_MESON
diff --git a/drivers/phy/amlogic/Makefile b/drivers/phy/amlogic/Makefile
index e2baa133f7af..99702a45e9be 100644
--- a/drivers/phy/amlogic/Makefile
+++ b/drivers/phy/amlogic/Makefile
@@ -2,7 +2,6 @@
 obj-$(CONFIG_PHY_MESON8B_USB2)			+= phy-meson8b-usb2.o
 obj-$(CONFIG_PHY_MESON_GXL_USB2)		+= phy-meson-gxl-usb2.o
 obj-$(CONFIG_PHY_MESON_G12A_USB2)		+= phy-meson-g12a-usb2.o
-obj-$(CONFIG_PHY_MESON_GXL_USB3)		+= phy-meson-gxl-usb3.o
 obj-$(CONFIG_PHY_MESON_G12A_USB3_PCIE)		+= phy-meson-g12a-usb3-pcie.o
 obj-$(CONFIG_PHY_MESON_AXG_PCIE)		+= phy-meson-axg-pcie.o
 obj-$(CONFIG_PHY_MESON_AXG_MIPI_PCIE_ANALOG)	+= phy-meson-axg-mipi-pcie-analog.o
diff --git a/drivers/phy/amlogic/phy-meson-gxl-usb3.c b/drivers/phy/amlogic/phy-meson-gxl-usb3.c
deleted file mode 100644
index c0e9e4c16149..000000000000
--- a/drivers/phy/amlogic/phy-meson-gxl-usb3.c
+++ /dev/null
@@ -1,283 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Meson GXL USB3 PHY and OTG mode detection driver
- *
- * Copyright (C) 2018 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
- */
-
-#include <linux/bitfield.h>
-#include <linux/bitops.h>
-#include <linux/clk.h>
-#include <linux/module.h>
-#include <linux/of_device.h>
-#include <linux/phy/phy.h>
-#include <linux/regmap.h>
-#include <linux/reset.h>
-#include <linux/platform_device.h>
-
-#define USB_R0							0x00
-	#define USB_R0_P30_FSEL_MASK				GENMASK(5, 0)
-	#define USB_R0_P30_PHY_RESET				BIT(6)
-	#define USB_R0_P30_TEST_POWERDOWN_HSP			BIT(7)
-	#define USB_R0_P30_TEST_POWERDOWN_SSP			BIT(8)
-	#define USB_R0_P30_ACJT_LEVEL_MASK			GENMASK(13, 9)
-	#define USB_R0_P30_TX_BOOST_LEVEL_MASK			GENMASK(16, 14)
-	#define USB_R0_P30_LANE0_TX2RX_LOOPBACK			BIT(17)
-	#define USB_R0_P30_LANE0_EXT_PCLK_REQ			BIT(18)
-	#define USB_R0_P30_PCS_RX_LOS_MASK_VAL_MASK		GENMASK(28, 19)
-	#define USB_R0_U2D_SS_SCALEDOWN_MODE_MASK		GENMASK(30, 29)
-	#define USB_R0_U2D_ACT					BIT(31)
-
-#define USB_R1							0x04
-	#define USB_R1_U3H_BIGENDIAN_GS				BIT(0)
-	#define USB_R1_U3H_PME_ENABLE				BIT(1)
-	#define USB_R1_U3H_HUB_PORT_OVERCURRENT_MASK		GENMASK(6, 2)
-	#define USB_R1_U3H_HUB_PORT_PERM_ATTACH_MASK		GENMASK(11, 7)
-	#define USB_R1_U3H_HOST_U2_PORT_DISABLE_MASK		GENMASK(15, 12)
-	#define USB_R1_U3H_HOST_U3_PORT_DISABLE			BIT(16)
-	#define USB_R1_U3H_HOST_PORT_POWER_CONTROL_PRESENT	BIT(17)
-	#define USB_R1_U3H_HOST_MSI_ENABLE			BIT(18)
-	#define USB_R1_U3H_FLADJ_30MHZ_REG_MASK			GENMASK(24, 19)
-	#define USB_R1_P30_PCS_TX_SWING_FULL_MASK		GENMASK(31, 25)
-
-#define USB_R2							0x08
-	#define USB_R2_P30_CR_DATA_IN_MASK			GENMASK(15, 0)
-	#define USB_R2_P30_CR_READ				BIT(16)
-	#define USB_R2_P30_CR_WRITE				BIT(17)
-	#define USB_R2_P30_CR_CAP_ADDR				BIT(18)
-	#define USB_R2_P30_CR_CAP_DATA				BIT(19)
-	#define USB_R2_P30_PCS_TX_DEEMPH_3P5DB_MASK		GENMASK(25, 20)
-	#define USB_R2_P30_PCS_TX_DEEMPH_6DB_MASK		GENMASK(31, 26)
-
-#define USB_R3							0x0c
-	#define USB_R3_P30_SSC_ENABLE				BIT(0)
-	#define USB_R3_P30_SSC_RANGE_MASK			GENMASK(3, 1)
-	#define USB_R3_P30_SSC_REF_CLK_SEL_MASK			GENMASK(12, 4)
-	#define USB_R3_P30_REF_SSP_EN				BIT(13)
-	#define USB_R3_P30_LOS_BIAS_MASK			GENMASK(18, 16)
-	#define USB_R3_P30_LOS_LEVEL_MASK			GENMASK(23, 19)
-	#define USB_R3_P30_MPLL_MULTIPLIER_MASK			GENMASK(30, 24)
-
-#define USB_R4							0x10
-	#define USB_R4_P21_PORT_RESET_0				BIT(0)
-	#define USB_R4_P21_SLEEP_M0				BIT(1)
-	#define USB_R4_MEM_PD_MASK				GENMASK(3, 2)
-	#define USB_R4_P21_ONLY					BIT(4)
-
-#define USB_R5							0x14
-	#define USB_R5_ID_DIG_SYNC				BIT(0)
-	#define USB_R5_ID_DIG_REG				BIT(1)
-	#define USB_R5_ID_DIG_CFG_MASK				GENMASK(3, 2)
-	#define USB_R5_ID_DIG_EN_0				BIT(4)
-	#define USB_R5_ID_DIG_EN_1				BIT(5)
-	#define USB_R5_ID_DIG_CURR				BIT(6)
-	#define USB_R5_ID_DIG_IRQ				BIT(7)
-	#define USB_R5_ID_DIG_TH_MASK				GENMASK(15, 8)
-	#define USB_R5_ID_DIG_CNT_MASK				GENMASK(23, 16)
-
-/* read-only register */
-#define USB_R6							0x18
-	#define USB_R6_P30_CR_DATA_OUT_MASK			GENMASK(15, 0)
-	#define USB_R6_P30_CR_ACK				BIT(16)
-
-struct phy_meson_gxl_usb3_priv {
-	struct regmap		*regmap;
-	enum phy_mode		mode;
-	struct clk		*clk_phy;
-	struct clk		*clk_peripheral;
-	struct reset_control	*reset;
-};
-
-static const struct regmap_config phy_meson_gxl_usb3_regmap_conf = {
-	.reg_bits = 8,
-	.val_bits = 32,
-	.reg_stride = 4,
-	.max_register = USB_R6,
-};
-
-static int phy_meson_gxl_usb3_power_on(struct phy *phy)
-{
-	struct phy_meson_gxl_usb3_priv *priv = phy_get_drvdata(phy);
-
-	regmap_update_bits(priv->regmap, USB_R5, USB_R5_ID_DIG_EN_0,
-			   USB_R5_ID_DIG_EN_0);
-	regmap_update_bits(priv->regmap, USB_R5, USB_R5_ID_DIG_EN_1,
-			   USB_R5_ID_DIG_EN_1);
-	regmap_update_bits(priv->regmap, USB_R5, USB_R5_ID_DIG_TH_MASK,
-			   FIELD_PREP(USB_R5_ID_DIG_TH_MASK, 0xff));
-
-	return 0;
-}
-
-static int phy_meson_gxl_usb3_power_off(struct phy *phy)
-{
-	struct phy_meson_gxl_usb3_priv *priv = phy_get_drvdata(phy);
-
-	regmap_update_bits(priv->regmap, USB_R5, USB_R5_ID_DIG_EN_0, 0);
-	regmap_update_bits(priv->regmap, USB_R5, USB_R5_ID_DIG_EN_1, 0);
-
-	return 0;
-}
-
-static int phy_meson_gxl_usb3_set_mode(struct phy *phy,
-				       enum phy_mode mode, int submode)
-{
-	struct phy_meson_gxl_usb3_priv *priv = phy_get_drvdata(phy);
-
-	switch (mode) {
-	case PHY_MODE_USB_HOST:
-		regmap_update_bits(priv->regmap, USB_R0, USB_R0_U2D_ACT, 0);
-		regmap_update_bits(priv->regmap, USB_R4, USB_R4_P21_SLEEP_M0,
-				   0);
-		break;
-
-	case PHY_MODE_USB_DEVICE:
-		regmap_update_bits(priv->regmap, USB_R0, USB_R0_U2D_ACT,
-				   USB_R0_U2D_ACT);
-		regmap_update_bits(priv->regmap, USB_R4, USB_R4_P21_SLEEP_M0,
-				   USB_R4_P21_SLEEP_M0);
-		break;
-
-	default:
-		dev_err(&phy->dev, "unsupported PHY mode %d\n", mode);
-		return -EINVAL;
-	}
-
-	priv->mode = mode;
-
-	return 0;
-}
-
-static int phy_meson_gxl_usb3_init(struct phy *phy)
-{
-	struct phy_meson_gxl_usb3_priv *priv = phy_get_drvdata(phy);
-	int ret;
-
-	ret = reset_control_reset(priv->reset);
-	if (ret)
-		goto err;
-
-	ret = clk_prepare_enable(priv->clk_phy);
-	if (ret)
-		goto err;
-
-	ret = clk_prepare_enable(priv->clk_peripheral);
-	if (ret)
-		goto err_disable_clk_phy;
-
-	ret = phy_meson_gxl_usb3_set_mode(phy, priv->mode, 0);
-	if (ret)
-		goto err_disable_clk_peripheral;
-
-	regmap_update_bits(priv->regmap, USB_R1,
-			   USB_R1_U3H_FLADJ_30MHZ_REG_MASK,
-			   FIELD_PREP(USB_R1_U3H_FLADJ_30MHZ_REG_MASK, 0x20));
-
-	return 0;
-
-err_disable_clk_peripheral:
-	clk_disable_unprepare(priv->clk_peripheral);
-err_disable_clk_phy:
-	clk_disable_unprepare(priv->clk_phy);
-err:
-	return ret;
-}
-
-static int phy_meson_gxl_usb3_exit(struct phy *phy)
-{
-	struct phy_meson_gxl_usb3_priv *priv = phy_get_drvdata(phy);
-
-	clk_disable_unprepare(priv->clk_peripheral);
-	clk_disable_unprepare(priv->clk_phy);
-
-	return 0;
-}
-
-static const struct phy_ops phy_meson_gxl_usb3_ops = {
-	.power_on	= phy_meson_gxl_usb3_power_on,
-	.power_off	= phy_meson_gxl_usb3_power_off,
-	.set_mode	= phy_meson_gxl_usb3_set_mode,
-	.init		= phy_meson_gxl_usb3_init,
-	.exit		= phy_meson_gxl_usb3_exit,
-	.owner		= THIS_MODULE,
-};
-
-static int phy_meson_gxl_usb3_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct device_node *np = dev->of_node;
-	struct phy_meson_gxl_usb3_priv *priv;
-	struct resource *res;
-	struct phy *phy;
-	struct phy_provider *phy_provider;
-	void __iomem *base;
-	int ret;
-
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(base))
-		return PTR_ERR(base);
-
-	priv->regmap = devm_regmap_init_mmio(dev, base,
-					     &phy_meson_gxl_usb3_regmap_conf);
-	if (IS_ERR(priv->regmap))
-		return PTR_ERR(priv->regmap);
-
-	priv->clk_phy = devm_clk_get(dev, "phy");
-	if (IS_ERR(priv->clk_phy))
-		return PTR_ERR(priv->clk_phy);
-
-	priv->clk_peripheral = devm_clk_get(dev, "peripheral");
-	if (IS_ERR(priv->clk_peripheral))
-		return PTR_ERR(priv->clk_peripheral);
-
-	priv->reset = devm_reset_control_array_get_shared(dev);
-	if (IS_ERR(priv->reset))
-		return PTR_ERR(priv->reset);
-
-	/*
-	 * default to host mode as hardware defaults and/or boot-loader
-	 * behavior can result in this PHY starting up in device mode. this
-	 * default and the initialization in phy_meson_gxl_usb3_init ensure
-	 * that we reproducibly start in a known mode on all devices.
-	 */
-	priv->mode = PHY_MODE_USB_HOST;
-
-	phy = devm_phy_create(dev, np, &phy_meson_gxl_usb3_ops);
-	if (IS_ERR(phy)) {
-		ret = PTR_ERR(phy);
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "failed to create PHY\n");
-
-		return ret;
-	}
-
-	phy_set_drvdata(phy, priv);
-
-	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
-
-	return PTR_ERR_OR_ZERO(phy_provider);
-}
-
-static const struct of_device_id phy_meson_gxl_usb3_of_match[] = {
-	{ .compatible = "amlogic,meson-gxl-usb3-phy", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, phy_meson_gxl_usb3_of_match);
-
-static struct platform_driver phy_meson_gxl_usb3_driver = {
-	.probe	= phy_meson_gxl_usb3_probe,
-	.driver	= {
-		.name		= "phy-meson-gxl-usb3",
-		.of_match_table	= phy_meson_gxl_usb3_of_match,
-	},
-};
-module_platform_driver(phy_meson_gxl_usb3_driver);
-
-MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
-MODULE_DESCRIPTION("Meson GXL USB3 PHY and OTG detection driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/amlogic/phy-meson8b-usb2.c b/drivers/phy/amlogic/phy-meson8b-usb2.c
index bd66bd723e4a..03c061dd5f0d 100644
--- a/drivers/phy/amlogic/phy-meson8b-usb2.c
+++ b/drivers/phy/amlogic/phy-meson8b-usb2.c
@@ -10,6 +10,8 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
 #include <linux/reset.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
@@ -76,6 +78,17 @@
 	#define REG_ADP_BC_ACA_PIN_FLOAT		BIT(26)
 
 #define REG_DBG_UART					0x10
+	#define REG_DBG_UART_BYPASS_SEL			BIT(0)
+	#define REG_DBG_UART_BYPASS_DM_EN		BIT(1)
+	#define REG_DBG_UART_BYPASS_DP_EN		BIT(2)
+	#define REG_DBG_UART_BYPASS_DM_DATA		BIT(3)
+	#define REG_DBG_UART_BYPASS_DP_DATA		BIT(4)
+	#define REG_DBG_UART_FSV_MINUS			BIT(5)
+	#define REG_DBG_UART_FSV_PLUS			BIT(6)
+	#define REG_DBG_UART_FSV_BURN_IN_TEST		BIT(7)
+	#define REG_DBG_UART_LOOPBACK_EN_B		BIT(8)
+	#define REG_DBG_UART_SET_IDDQ			BIT(9)
+	#define REG_DBG_UART_ATE_RESET			BIT(10)
 
 #define REG_TEST					0x14
 	#define REG_TEST_DATA_IN_MASK			GENMASK(3, 0)
@@ -104,35 +117,30 @@
 #define RESET_COMPLETE_TIME				500
 #define ACA_ENABLE_COMPLETE_TIME			50
 
-struct phy_meson8b_usb2_priv {
-	void __iomem		*regs;
-	enum usb_dr_mode	dr_mode;
-	struct clk		*clk_usb_general;
-	struct clk		*clk_usb;
-	struct reset_control	*reset;
+struct phy_meson8b_usb2_match_data {
+	bool			host_enable_aca;
 };
 
-static u32 phy_meson8b_usb2_read(struct phy_meson8b_usb2_priv *phy_priv,
-				 u32 reg)
-{
-	return readl(phy_priv->regs + reg);
-}
-
-static void phy_meson8b_usb2_mask_bits(struct phy_meson8b_usb2_priv *phy_priv,
-				       u32 reg, u32 mask, u32 value)
-{
-	u32 data;
-
-	data = phy_meson8b_usb2_read(phy_priv, reg);
-	data &= ~mask;
-	data |= (value & mask);
+struct phy_meson8b_usb2_priv {
+	struct regmap					*regmap;
+	enum usb_dr_mode				dr_mode;
+	struct clk					*clk_usb_general;
+	struct clk					*clk_usb;
+	struct reset_control				*reset;
+	const struct phy_meson8b_usb2_match_data	*match;
+};
 
-	writel(data, phy_priv->regs + reg);
-}
+static const struct regmap_config phy_meson8b_usb2_regmap_conf = {
+	.reg_bits = 8,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = REG_TUNE,
+};
 
 static int phy_meson8b_usb2_power_on(struct phy *phy)
 {
 	struct phy_meson8b_usb2_priv *priv = phy_get_drvdata(phy);
+	u32 reg;
 	int ret;
 
 	if (!IS_ERR_OR_NULL(priv->reset)) {
@@ -156,38 +164,43 @@ static int phy_meson8b_usb2_power_on(struct phy *phy)
 		return ret;
 	}
 
-	phy_meson8b_usb2_mask_bits(priv, REG_CONFIG, REG_CONFIG_CLK_32k_ALTSEL,
-				   REG_CONFIG_CLK_32k_ALTSEL);
+	regmap_update_bits(priv->regmap, REG_CONFIG, REG_CONFIG_CLK_32k_ALTSEL,
+			   REG_CONFIG_CLK_32k_ALTSEL);
 
-	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_REF_CLK_SEL_MASK,
-				   0x2 << REG_CTRL_REF_CLK_SEL_SHIFT);
+	regmap_update_bits(priv->regmap, REG_CTRL, REG_CTRL_REF_CLK_SEL_MASK,
+			   0x2 << REG_CTRL_REF_CLK_SEL_SHIFT);
 
-	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_FSEL_MASK,
-				   0x5 << REG_CTRL_FSEL_SHIFT);
+	regmap_update_bits(priv->regmap, REG_CTRL, REG_CTRL_FSEL_MASK,
+			   0x5 << REG_CTRL_FSEL_SHIFT);
 
 	/* reset the PHY */
-	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_POWER_ON_RESET,
-				   REG_CTRL_POWER_ON_RESET);
+	regmap_update_bits(priv->regmap, REG_CTRL, REG_CTRL_POWER_ON_RESET,
+			   REG_CTRL_POWER_ON_RESET);
 	udelay(RESET_COMPLETE_TIME);
-	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_POWER_ON_RESET, 0);
+	regmap_update_bits(priv->regmap, REG_CTRL, REG_CTRL_POWER_ON_RESET, 0);
 	udelay(RESET_COMPLETE_TIME);
 
-	phy_meson8b_usb2_mask_bits(priv, REG_CTRL, REG_CTRL_SOF_TOGGLE_OUT,
-				   REG_CTRL_SOF_TOGGLE_OUT);
+	regmap_update_bits(priv->regmap, REG_CTRL, REG_CTRL_SOF_TOGGLE_OUT,
+			   REG_CTRL_SOF_TOGGLE_OUT);
 
 	if (priv->dr_mode == USB_DR_MODE_HOST) {
-		phy_meson8b_usb2_mask_bits(priv, REG_ADP_BC,
+		regmap_update_bits(priv->regmap, REG_DBG_UART,
+				   REG_DBG_UART_SET_IDDQ, 0);
+
+		if (priv->match->host_enable_aca) {
+			regmap_update_bits(priv->regmap, REG_ADP_BC,
 					   REG_ADP_BC_ACA_ENABLE,
 					   REG_ADP_BC_ACA_ENABLE);
 
-		udelay(ACA_ENABLE_COMPLETE_TIME);
+			udelay(ACA_ENABLE_COMPLETE_TIME);
 
-		if (phy_meson8b_usb2_read(priv, REG_ADP_BC) &
-			REG_ADP_BC_ACA_PIN_FLOAT) {
-			dev_warn(&phy->dev, "USB ID detect failed!\n");
-			clk_disable_unprepare(priv->clk_usb);
-			clk_disable_unprepare(priv->clk_usb_general);
-			return -EINVAL;
+			regmap_read(priv->regmap, REG_ADP_BC, &reg);
+			if (reg & REG_ADP_BC_ACA_PIN_FLOAT) {
+				dev_warn(&phy->dev, "USB ID detect failed!\n");
+				clk_disable_unprepare(priv->clk_usb);
+				clk_disable_unprepare(priv->clk_usb_general);
+				return -EINVAL;
+			}
 		}
 	}
 
@@ -198,6 +211,11 @@ static int phy_meson8b_usb2_power_off(struct phy *phy)
 {
 	struct phy_meson8b_usb2_priv *priv = phy_get_drvdata(phy);
 
+	if (priv->dr_mode == USB_DR_MODE_HOST)
+		regmap_update_bits(priv->regmap, REG_DBG_UART,
+				   REG_DBG_UART_SET_IDDQ,
+				   REG_DBG_UART_SET_IDDQ);
+
 	clk_disable_unprepare(priv->clk_usb);
 	clk_disable_unprepare(priv->clk_usb_general);
 
@@ -213,18 +231,26 @@ static const struct phy_ops phy_meson8b_usb2_ops = {
 static int phy_meson8b_usb2_probe(struct platform_device *pdev)
 {
 	struct phy_meson8b_usb2_priv *priv;
-	struct resource *res;
 	struct phy *phy;
 	struct phy_provider *phy_provider;
+	void __iomem *base;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	priv->regs = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(priv->regs))
-		return PTR_ERR(priv->regs);
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	priv->match = device_get_match_data(&pdev->dev);
+	if (!priv->match)
+		return -ENODEV;
+
+	priv->regmap = devm_regmap_init_mmio(&pdev->dev, base,
+					     &phy_meson8b_usb2_regmap_conf);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
 
 	priv->clk_usb_general = devm_clk_get(&pdev->dev, "usb_general");
 	if (IS_ERR(priv->clk_usb_general))
@@ -259,11 +285,32 @@ static int phy_meson8b_usb2_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(phy_provider);
 }
 
+static const struct phy_meson8b_usb2_match_data phy_meson8_usb2_match_data = {
+	.host_enable_aca = false,
+};
+
+static const struct phy_meson8b_usb2_match_data phy_meson8b_usb2_match_data = {
+	.host_enable_aca = true,
+};
+
 static const struct of_device_id phy_meson8b_usb2_of_match[] = {
-	{ .compatible = "amlogic,meson8-usb2-phy", },
-	{ .compatible = "amlogic,meson8b-usb2-phy", },
-	{ .compatible = "amlogic,meson-gxbb-usb2-phy", },
-	{ },
+	{
+		.compatible = "amlogic,meson8-usb2-phy",
+		.data = &phy_meson8_usb2_match_data
+	},
+	{
+		.compatible = "amlogic,meson8b-usb2-phy",
+		.data = &phy_meson8b_usb2_match_data
+	},
+	{
+		.compatible = "amlogic,meson8m2-usb2-phy",
+		.data = &phy_meson8b_usb2_match_data
+	},
+	{
+		.compatible = "amlogic,meson-gxbb-usb2-phy",
+		.data = &phy_meson8b_usb2_match_data
+	},
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, phy_meson8b_usb2_of_match);
 
@@ -277,5 +324,5 @@ static struct platform_driver phy_meson8b_usb2_driver = {
 module_platform_driver(phy_meson8b_usb2_driver);
 
 MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
-MODULE_DESCRIPTION("Meson8, Meson8b and GXBB USB2 PHY driver");
+MODULE_DESCRIPTION("Meson8, Meson8b, Meson8m2 and GXBB USB2 PHY driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c b/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c
index 7ceea5ae2704..527625912b78 100644
--- a/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c
+++ b/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c
@@ -279,7 +279,7 @@ static irqreturn_t gpio_irq_handler(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static struct phy_ops ops = {
+static const struct phy_ops ops = {
 	.init		= ns2_drd_phy_init,
 	.power_on	= ns2_drd_phy_poweron,
 	.power_off	= ns2_drd_phy_poweroff,
diff --git a/drivers/phy/broadcom/phy-bcm-sr-usb.c b/drivers/phy/broadcom/phy-bcm-sr-usb.c
index fe6c58910e4c..77c025a0720c 100644
--- a/drivers/phy/broadcom/phy-bcm-sr-usb.c
+++ b/drivers/phy/broadcom/phy-bcm-sr-usb.c
@@ -16,8 +16,6 @@ enum bcm_usb_phy_version {
 };
 
 enum bcm_usb_phy_reg {
-	PLL_NDIV_FRAC,
-	PLL_NDIV_INT,
 	PLL_CTRL,
 	PHY_CTRL,
 	PHY_PLL_CTRL,
@@ -31,18 +29,11 @@ static const u8 bcm_usb_combo_phy_ss[] = {
 };
 
 static const u8 bcm_usb_combo_phy_hs[] = {
-	[PLL_NDIV_FRAC]	= 0x04,
-	[PLL_NDIV_INT]	= 0x08,
 	[PLL_CTRL]	= 0x0c,
 	[PHY_CTRL]	= 0x10,
 };
 
-#define HSPLL_NDIV_INT_VAL	0x13
-#define HSPLL_NDIV_FRAC_VAL	0x1005
-
 static const u8 bcm_usb_hs_phy[] = {
-	[PLL_NDIV_FRAC]	= 0x0,
-	[PLL_NDIV_INT]	= 0x4,
 	[PLL_CTRL]	= 0x8,
 	[PHY_CTRL]	= 0xc,
 };
@@ -52,7 +43,6 @@ enum pll_ctrl_bits {
 	SSPLL_SUSPEND_EN,
 	PLL_SEQ_START,
 	PLL_LOCK,
-	PLL_PDIV,
 };
 
 static const u8 u3pll_ctrl[] = {
@@ -66,29 +56,17 @@ static const u8 u3pll_ctrl[] = {
 #define HSPLL_PDIV_VAL		0x1
 
 static const u8 u2pll_ctrl[] = {
-	[PLL_PDIV]	= 1,
 	[PLL_RESETB]	= 5,
 	[PLL_LOCK]	= 6,
 };
 
 enum bcm_usb_phy_ctrl_bits {
 	CORERDY,
-	AFE_LDO_PWRDWNB,
-	AFE_PLL_PWRDWNB,
-	AFE_BG_PWRDWNB,
-	PHY_ISO,
 	PHY_RESETB,
 	PHY_PCTL,
 };
 
 #define PHY_PCTL_MASK	0xffff
-/*
- * 0x0806 of PCTL_VAL has below bits set
- * BIT-8 : refclk divider 1
- * BIT-3:2: device mode; mode is not effect
- * BIT-1: soft reset active low
- */
-#define HSPHY_PCTL_VAL	0x0806
 #define SSPHY_PCTL_VAL	0x0006
 
 static const u8 u3phy_ctrl[] = {
@@ -98,10 +76,6 @@ static const u8 u3phy_ctrl[] = {
 
 static const u8 u2phy_ctrl[] = {
 	[CORERDY]		= 0,
-	[AFE_LDO_PWRDWNB]	= 1,
-	[AFE_PLL_PWRDWNB]	= 2,
-	[AFE_BG_PWRDWNB]	= 3,
-	[PHY_ISO]		= 4,
 	[PHY_RESETB]		= 5,
 	[PHY_PCTL]		= 6,
 };
@@ -186,38 +160,13 @@ static int bcm_usb_hs_phy_init(struct bcm_usb_phy_cfg *phy_cfg)
 	int ret = 0;
 	void __iomem *regs = phy_cfg->regs;
 	const u8 *offset;
-	u32 rd_data;
 
 	offset = phy_cfg->offset;
 
-	writel(HSPLL_NDIV_INT_VAL, regs + offset[PLL_NDIV_INT]);
-	writel(HSPLL_NDIV_FRAC_VAL, regs + offset[PLL_NDIV_FRAC]);
-
-	rd_data = readl(regs + offset[PLL_CTRL]);
-	rd_data &= ~(HSPLL_PDIV_MASK << u2pll_ctrl[PLL_PDIV]);
-	rd_data |= (HSPLL_PDIV_VAL << u2pll_ctrl[PLL_PDIV]);
-	writel(rd_data, regs + offset[PLL_CTRL]);
-
-	/* Set Core Ready high */
-	bcm_usb_reg32_setbits(regs + offset[PHY_CTRL],
-			      BIT(u2phy_ctrl[CORERDY]));
-
-	/* Maximum timeout for Core Ready done */
-	msleep(30);
-
+	bcm_usb_reg32_clrbits(regs + offset[PLL_CTRL],
+			      BIT(u2pll_ctrl[PLL_RESETB]));
 	bcm_usb_reg32_setbits(regs + offset[PLL_CTRL],
 			      BIT(u2pll_ctrl[PLL_RESETB]));
-	bcm_usb_reg32_setbits(regs + offset[PHY_CTRL],
-			      BIT(u2phy_ctrl[PHY_RESETB]));
-
-
-	rd_data = readl(regs + offset[PHY_CTRL]);
-	rd_data &= ~(PHY_PCTL_MASK << u2phy_ctrl[PHY_PCTL]);
-	rd_data |= (HSPHY_PCTL_VAL << u2phy_ctrl[PHY_PCTL]);
-	writel(rd_data, regs + offset[PHY_CTRL]);
-
-	/* Maximum timeout for PLL reset done */
-	msleep(30);
 
 	ret = bcm_usb_pll_lock_check(regs + offset[PLL_CTRL],
 				     BIT(u2pll_ctrl[PLL_LOCK]));
@@ -256,7 +205,7 @@ static int bcm_usb_phy_init(struct phy *phy)
 	return ret;
 }
 
-static struct phy_ops sr_phy_ops = {
+static const struct phy_ops sr_phy_ops = {
 	.init		= bcm_usb_phy_init,
 	.reset		= bcm_usb_phy_reset,
 	.owner		= THIS_MODULE,
diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c
index 491bbd46c5b3..99fbc7e4138b 100644
--- a/drivers/phy/broadcom/phy-brcm-usb.c
+++ b/drivers/phy/broadcom/phy-brcm-usb.c
@@ -39,14 +39,14 @@ struct match_chip_info {
 	u8 optional_reg;
 };
 
-static struct value_to_name_map brcm_dr_mode_to_name[] = {
+static const struct value_to_name_map brcm_dr_mode_to_name[] = {
 	{ USB_CTLR_MODE_HOST, "host" },
 	{ USB_CTLR_MODE_DEVICE, "peripheral" },
 	{ USB_CTLR_MODE_DRD, "drd" },
 	{ USB_CTLR_MODE_TYPEC_PD, "typec-pd" }
 };
 
-static struct value_to_name_map brcm_dual_mode_to_name[] = {
+static const struct value_to_name_map brcm_dual_mode_to_name[] = {
 	{ 0, "host" },
 	{ 1, "device" },
 	{ 2, "auto" },
@@ -138,7 +138,7 @@ static int brcm_usb_phy_exit(struct phy *gphy)
 	return 0;
 }
 
-static struct phy_ops brcm_usb_phy_ops = {
+static const struct phy_ops brcm_usb_phy_ops = {
 	.init		= brcm_usb_phy_init,
 	.exit		= brcm_usb_phy_exit,
 	.owner		= THIS_MODULE,
@@ -170,7 +170,7 @@ static struct phy *brcm_usb_phy_xlate(struct device *dev,
 	return ERR_PTR(-ENODEV);
 }
 
-static int name_to_value(struct value_to_name_map *table, int count,
+static int name_to_value(const struct value_to_name_map *table, int count,
 			 const char *name, int *value)
 {
 	int x;
@@ -185,7 +185,7 @@ static int name_to_value(struct value_to_name_map *table, int count,
 	return -EINVAL;
 }
 
-static const char *value_to_name(struct value_to_name_map *table, int count,
+static const char *value_to_name(const struct value_to_name_map *table, int count,
 				 int value)
 {
 	if (value >= count)
@@ -252,7 +252,7 @@ static const struct attribute_group brcm_usb_phy_group = {
 	.attrs = brcm_usb_phy_attrs,
 };
 
-static struct match_chip_info chip_info_7216 = {
+static const struct match_chip_info chip_info_7216 = {
 	.init_func = &brcm_usb_dvr_init_7216,
 	.required_regs = {
 		BRCM_REGS_CTRL,
@@ -262,7 +262,7 @@ static struct match_chip_info chip_info_7216 = {
 	},
 };
 
-static struct match_chip_info chip_info_7211b0 = {
+static const struct match_chip_info chip_info_7211b0 = {
 	.init_func = &brcm_usb_dvr_init_7211b0,
 	.required_regs = {
 		BRCM_REGS_CTRL,
@@ -275,7 +275,7 @@ static struct match_chip_info chip_info_7211b0 = {
 	.optional_reg = BRCM_REGS_BDC_EC,
 };
 
-static struct match_chip_info chip_info_7445 = {
+static const struct match_chip_info chip_info_7445 = {
 	.init_func = &brcm_usb_dvr_init_7445,
 	.required_regs = {
 		BRCM_REGS_CTRL,
diff --git a/drivers/phy/cadence/Kconfig b/drivers/phy/cadence/Kconfig
index 459545871608..432832bdbd16 100644
--- a/drivers/phy/cadence/Kconfig
+++ b/drivers/phy/cadence/Kconfig
@@ -27,3 +27,12 @@ config PHY_CADENCE_SIERRA
 	select GENERIC_PHY
 	help
 	  Enable this to support the Cadence Sierra PHY driver
+
+config PHY_CADENCE_SALVO
+	tristate "Cadence Salvo PHY Driver"
+	depends on OF && HAS_IOMEM
+	select GENERIC_PHY
+	help
+	  Enable this to support the Cadence SALVO PHY driver,
+	  this PHY is a legacy PHY, and only are used for USB3
+	  and USB2.
diff --git a/drivers/phy/cadence/Makefile b/drivers/phy/cadence/Makefile
index 6a7ffc6ea599..26e16bd34efe 100644
--- a/drivers/phy/cadence/Makefile
+++ b/drivers/phy/cadence/Makefile
@@ -2,3 +2,4 @@
 obj-$(CONFIG_PHY_CADENCE_TORRENT)	+= phy-cadence-torrent.o
 obj-$(CONFIG_PHY_CADENCE_DPHY)	+= cdns-dphy.o
 obj-$(CONFIG_PHY_CADENCE_SIERRA)	+= phy-cadence-sierra.o
+obj-$(CONFIG_PHY_CADENCE_SALVO)	+= phy-cadence-salvo.o
diff --git a/drivers/phy/cadence/phy-cadence-salvo.c b/drivers/phy/cadence/phy-cadence-salvo.c
new file mode 100644
index 000000000000..1ecbb964cd21
--- /dev/null
+++ b/drivers/phy/cadence/phy-cadence-salvo.c
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Salvo PHY is a 28nm PHY, it is a legacy PHY, and only
+ * for USB3 and USB2.
+ *
+ * Copyright (c) 2019-2020 NXP
+ */
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+
+/* PHY register definition */
+#define PHY_PMA_CMN_CTRL1			0xC800
+#define TB_ADDR_CMN_DIAG_HSCLK_SEL		0x01e0
+#define TB_ADDR_CMN_PLL0_VCOCAL_INIT_TMR	0x0084
+#define TB_ADDR_CMN_PLL0_VCOCAL_ITER_TMR	0x0085
+#define TB_ADDR_CMN_PLL0_INTDIV	                0x0094
+#define TB_ADDR_CMN_PLL0_FRACDIV		0x0095
+#define TB_ADDR_CMN_PLL0_HIGH_THR		0x0096
+#define TB_ADDR_CMN_PLL0_SS_CTRL1		0x0098
+#define TB_ADDR_CMN_PLL0_SS_CTRL2		0x0099
+#define TB_ADDR_CMN_PLL0_DSM_DIAG		0x0097
+#define TB_ADDR_CMN_DIAG_PLL0_OVRD		0x01c2
+#define TB_ADDR_CMN_DIAG_PLL0_FBH_OVRD		0x01c0
+#define TB_ADDR_CMN_DIAG_PLL0_FBL_OVRD		0x01c1
+#define TB_ADDR_CMN_DIAG_PLL0_V2I_TUNE          0x01C5
+#define TB_ADDR_CMN_DIAG_PLL0_CP_TUNE           0x01C6
+#define TB_ADDR_CMN_DIAG_PLL0_LF_PROG           0x01C7
+#define TB_ADDR_CMN_DIAG_PLL0_TEST_MODE		0x01c4
+#define TB_ADDR_CMN_PSM_CLK_CTRL		0x0061
+#define TB_ADDR_XCVR_DIAG_RX_LANE_CAL_RST_TMR	0x40ea
+#define TB_ADDR_XCVR_PSM_RCTRL	                0x4001
+#define TB_ADDR_TX_PSC_A0		        0x4100
+#define TB_ADDR_TX_PSC_A1		        0x4101
+#define TB_ADDR_TX_PSC_A2		        0x4102
+#define TB_ADDR_TX_PSC_A3		        0x4103
+#define TB_ADDR_TX_DIAG_ECTRL_OVRD		0x41f5
+#define TB_ADDR_TX_PSC_CAL		        0x4106
+#define TB_ADDR_TX_PSC_RDY		        0x4107
+#define TB_ADDR_RX_PSC_A0	                0x8000
+#define TB_ADDR_RX_PSC_A1	                0x8001
+#define TB_ADDR_RX_PSC_A2	                0x8002
+#define TB_ADDR_RX_PSC_A3	                0x8003
+#define TB_ADDR_RX_PSC_CAL	                0x8006
+#define TB_ADDR_RX_PSC_RDY	                0x8007
+#define TB_ADDR_TX_TXCC_MGNLS_MULT_000		0x4058
+#define TB_ADDR_TX_DIAG_BGREF_PREDRV_DELAY	0x41e7
+#define TB_ADDR_RX_SLC_CU_ITER_TMR		0x80e3
+#define TB_ADDR_RX_SIGDET_HL_FILT_TMR		0x8090
+#define TB_ADDR_RX_SAMP_DAC_CTRL		0x8058
+#define TB_ADDR_RX_DIAG_SIGDET_TUNE		0x81dc
+#define TB_ADDR_RX_DIAG_LFPSDET_TUNE2		0x81df
+#define TB_ADDR_RX_DIAG_BS_TM	                0x81f5
+#define TB_ADDR_RX_DIAG_DFE_CTRL1		0x81d3
+#define TB_ADDR_RX_DIAG_ILL_IQE_TRIM4		0x81c7
+#define TB_ADDR_RX_DIAG_ILL_E_TRIM0		0x81c2
+#define TB_ADDR_RX_DIAG_ILL_IQ_TRIM0		0x81c1
+#define TB_ADDR_RX_DIAG_ILL_IQE_TRIM6		0x81c9
+#define TB_ADDR_RX_DIAG_RXFE_TM3		0x81f8
+#define TB_ADDR_RX_DIAG_RXFE_TM4		0x81f9
+#define TB_ADDR_RX_DIAG_LFPSDET_TUNE		0x81dd
+#define TB_ADDR_RX_DIAG_DFE_CTRL3		0x81d5
+#define TB_ADDR_RX_DIAG_SC2C_DELAY		0x81e1
+#define TB_ADDR_RX_REE_VGA_GAIN_NODFE		0x81bf
+#define TB_ADDR_XCVR_PSM_CAL_TMR		0x4002
+#define TB_ADDR_XCVR_PSM_A0BYP_TMR		0x4004
+#define TB_ADDR_XCVR_PSM_A0IN_TMR		0x4003
+#define TB_ADDR_XCVR_PSM_A1IN_TMR		0x4005
+#define TB_ADDR_XCVR_PSM_A2IN_TMR		0x4006
+#define TB_ADDR_XCVR_PSM_A3IN_TMR		0x4007
+#define TB_ADDR_XCVR_PSM_A4IN_TMR		0x4008
+#define TB_ADDR_XCVR_PSM_A5IN_TMR		0x4009
+#define TB_ADDR_XCVR_PSM_A0OUT_TMR		0x400a
+#define TB_ADDR_XCVR_PSM_A1OUT_TMR		0x400b
+#define TB_ADDR_XCVR_PSM_A2OUT_TMR		0x400c
+#define TB_ADDR_XCVR_PSM_A3OUT_TMR		0x400d
+#define TB_ADDR_XCVR_PSM_A4OUT_TMR		0x400e
+#define TB_ADDR_XCVR_PSM_A5OUT_TMR		0x400f
+#define TB_ADDR_TX_RCVDET_EN_TMR	        0x4122
+#define TB_ADDR_TX_RCVDET_ST_TMR	        0x4123
+#define TB_ADDR_XCVR_DIAG_LANE_FCM_EN_MGN_TMR	0x40f2
+#define TB_ADDR_TX_RCVDETSC_CTRL	        0x4124
+
+/* TB_ADDR_TX_RCVDETSC_CTRL */
+#define RXDET_IN_P3_32KHZ			BIT(1)
+
+struct cdns_reg_pairs {
+	u16 val;
+	u32 off;
+};
+
+struct cdns_salvo_data {
+	u8 reg_offset_shift;
+	struct cdns_reg_pairs *init_sequence_val;
+	u8 init_sequence_length;
+};
+
+struct cdns_salvo_phy {
+	struct phy *phy;
+	struct clk *clk;
+	void __iomem *base;
+	struct cdns_salvo_data *data;
+};
+
+static const struct of_device_id cdns_salvo_phy_of_match[];
+static u16 cdns_salvo_read(struct cdns_salvo_phy *salvo_phy, u32 reg)
+{
+	return (u16)readl(salvo_phy->base +
+		reg * (1 << salvo_phy->data->reg_offset_shift));
+}
+
+static void cdns_salvo_write(struct cdns_salvo_phy *salvo_phy,
+			     u32 reg, u16 val)
+{
+	writel(val, salvo_phy->base +
+		reg * (1 << salvo_phy->data->reg_offset_shift));
+}
+
+/*
+ * Below bringup sequence pair are from Cadence PHY's User Guide
+ * and NXP platform tuning results.
+ */
+static struct cdns_reg_pairs cdns_nxp_sequence_pair[] = {
+	{0x0830, PHY_PMA_CMN_CTRL1},
+	{0x0010, TB_ADDR_CMN_DIAG_HSCLK_SEL},
+	{0x00f0, TB_ADDR_CMN_PLL0_VCOCAL_INIT_TMR},
+	{0x0018, TB_ADDR_CMN_PLL0_VCOCAL_ITER_TMR},
+	{0x00d0, TB_ADDR_CMN_PLL0_INTDIV},
+	{0x4aaa, TB_ADDR_CMN_PLL0_FRACDIV},
+	{0x0034, TB_ADDR_CMN_PLL0_HIGH_THR},
+	{0x01ee, TB_ADDR_CMN_PLL0_SS_CTRL1},
+	{0x7f03, TB_ADDR_CMN_PLL0_SS_CTRL2},
+	{0x0020, TB_ADDR_CMN_PLL0_DSM_DIAG},
+	{0x0000, TB_ADDR_CMN_DIAG_PLL0_OVRD},
+	{0x0000, TB_ADDR_CMN_DIAG_PLL0_FBH_OVRD},
+	{0x0000, TB_ADDR_CMN_DIAG_PLL0_FBL_OVRD},
+	{0x0007, TB_ADDR_CMN_DIAG_PLL0_V2I_TUNE},
+	{0x0027, TB_ADDR_CMN_DIAG_PLL0_CP_TUNE},
+	{0x0008, TB_ADDR_CMN_DIAG_PLL0_LF_PROG},
+	{0x0022, TB_ADDR_CMN_DIAG_PLL0_TEST_MODE},
+	{0x000a, TB_ADDR_CMN_PSM_CLK_CTRL},
+	{0x0139, TB_ADDR_XCVR_DIAG_RX_LANE_CAL_RST_TMR},
+	{0xbefc, TB_ADDR_XCVR_PSM_RCTRL},
+
+	{0x7799, TB_ADDR_TX_PSC_A0},
+	{0x7798, TB_ADDR_TX_PSC_A1},
+	{0x509b, TB_ADDR_TX_PSC_A2},
+	{0x0003, TB_ADDR_TX_DIAG_ECTRL_OVRD},
+	{0x509b, TB_ADDR_TX_PSC_A3},
+	{0x2090, TB_ADDR_TX_PSC_CAL},
+	{0x2090, TB_ADDR_TX_PSC_RDY},
+
+	{0xA6FD, TB_ADDR_RX_PSC_A0},
+	{0xA6FD, TB_ADDR_RX_PSC_A1},
+	{0xA410, TB_ADDR_RX_PSC_A2},
+	{0x2410, TB_ADDR_RX_PSC_A3},
+
+	{0x23FF, TB_ADDR_RX_PSC_CAL},
+	{0x2010, TB_ADDR_RX_PSC_RDY},
+
+	{0x0020, TB_ADDR_TX_TXCC_MGNLS_MULT_000},
+	{0x00ff, TB_ADDR_TX_DIAG_BGREF_PREDRV_DELAY},
+	{0x0002, TB_ADDR_RX_SLC_CU_ITER_TMR},
+	{0x0013, TB_ADDR_RX_SIGDET_HL_FILT_TMR},
+	{0x0000, TB_ADDR_RX_SAMP_DAC_CTRL},
+	{0x1004, TB_ADDR_RX_DIAG_SIGDET_TUNE},
+	{0x4041, TB_ADDR_RX_DIAG_LFPSDET_TUNE2},
+	{0x0480, TB_ADDR_RX_DIAG_BS_TM},
+	{0x8006, TB_ADDR_RX_DIAG_DFE_CTRL1},
+	{0x003f, TB_ADDR_RX_DIAG_ILL_IQE_TRIM4},
+	{0x543f, TB_ADDR_RX_DIAG_ILL_E_TRIM0},
+	{0x543f, TB_ADDR_RX_DIAG_ILL_IQ_TRIM0},
+	{0x0000, TB_ADDR_RX_DIAG_ILL_IQE_TRIM6},
+	{0x8000, TB_ADDR_RX_DIAG_RXFE_TM3},
+	{0x0003, TB_ADDR_RX_DIAG_RXFE_TM4},
+	{0x2408, TB_ADDR_RX_DIAG_LFPSDET_TUNE},
+	{0x05ca, TB_ADDR_RX_DIAG_DFE_CTRL3},
+	{0x0258, TB_ADDR_RX_DIAG_SC2C_DELAY},
+	{0x1fff, TB_ADDR_RX_REE_VGA_GAIN_NODFE},
+
+	{0x02c6, TB_ADDR_XCVR_PSM_CAL_TMR},
+	{0x0002, TB_ADDR_XCVR_PSM_A0BYP_TMR},
+	{0x02c6, TB_ADDR_XCVR_PSM_A0IN_TMR},
+	{0x0010, TB_ADDR_XCVR_PSM_A1IN_TMR},
+	{0x0010, TB_ADDR_XCVR_PSM_A2IN_TMR},
+	{0x0010, TB_ADDR_XCVR_PSM_A3IN_TMR},
+	{0x0010, TB_ADDR_XCVR_PSM_A4IN_TMR},
+	{0x0010, TB_ADDR_XCVR_PSM_A5IN_TMR},
+
+	{0x0002, TB_ADDR_XCVR_PSM_A0OUT_TMR},
+	{0x0002, TB_ADDR_XCVR_PSM_A1OUT_TMR},
+	{0x0002, TB_ADDR_XCVR_PSM_A2OUT_TMR},
+	{0x0002, TB_ADDR_XCVR_PSM_A3OUT_TMR},
+	{0x0002, TB_ADDR_XCVR_PSM_A4OUT_TMR},
+	{0x0002, TB_ADDR_XCVR_PSM_A5OUT_TMR},
+	/* Change rx detect parameter */
+	{0x0960, TB_ADDR_TX_RCVDET_EN_TMR},
+	{0x01e0, TB_ADDR_TX_RCVDET_ST_TMR},
+	{0x0090, TB_ADDR_XCVR_DIAG_LANE_FCM_EN_MGN_TMR},
+};
+
+static int cdns_salvo_phy_init(struct phy *phy)
+{
+	struct cdns_salvo_phy *salvo_phy = phy_get_drvdata(phy);
+	struct cdns_salvo_data *data = salvo_phy->data;
+	int ret, i;
+	u16 value;
+
+	ret = clk_prepare_enable(salvo_phy->clk);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < data->init_sequence_length; i++) {
+		struct cdns_reg_pairs *reg_pair = data->init_sequence_val + i;
+
+		cdns_salvo_write(salvo_phy, reg_pair->off, reg_pair->val);
+	}
+
+	/* RXDET_IN_P3_32KHZ, Receiver detect slow clock enable */
+	value = cdns_salvo_read(salvo_phy, TB_ADDR_TX_RCVDETSC_CTRL);
+	value |= RXDET_IN_P3_32KHZ;
+	cdns_salvo_write(salvo_phy, TB_ADDR_TX_RCVDETSC_CTRL,
+			 RXDET_IN_P3_32KHZ);
+
+	udelay(10);
+
+	clk_disable_unprepare(salvo_phy->clk);
+
+	return ret;
+}
+
+static int cdns_salvo_phy_power_on(struct phy *phy)
+{
+	struct cdns_salvo_phy *salvo_phy = phy_get_drvdata(phy);
+
+	return clk_prepare_enable(salvo_phy->clk);
+}
+
+static int cdns_salvo_phy_power_off(struct phy *phy)
+{
+	struct cdns_salvo_phy *salvo_phy = phy_get_drvdata(phy);
+
+	clk_disable_unprepare(salvo_phy->clk);
+
+	return 0;
+}
+
+static struct phy_ops cdns_salvo_phy_ops = {
+	.init		= cdns_salvo_phy_init,
+	.power_on	= cdns_salvo_phy_power_on,
+	.power_off	= cdns_salvo_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static int cdns_salvo_phy_probe(struct platform_device *pdev)
+{
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	struct cdns_salvo_phy *salvo_phy;
+	struct resource *res;
+	const struct of_device_id *match;
+	struct cdns_salvo_data *data;
+
+	match = of_match_device(cdns_salvo_phy_of_match, dev);
+	if (!match)
+		return -EINVAL;
+
+	data = (struct cdns_salvo_data *)match->data;
+	salvo_phy = devm_kzalloc(dev, sizeof(*salvo_phy), GFP_KERNEL);
+	if (!salvo_phy)
+		return -ENOMEM;
+
+	salvo_phy->data = data;
+	salvo_phy->clk = devm_clk_get_optional(dev, "salvo_phy_clk");
+	if (IS_ERR(salvo_phy->clk))
+		return PTR_ERR(salvo_phy->clk);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	salvo_phy->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(salvo_phy->base))
+		return PTR_ERR(salvo_phy->base);
+
+	salvo_phy->phy = devm_phy_create(dev, NULL, &cdns_salvo_phy_ops);
+	if (IS_ERR(salvo_phy->phy))
+		return PTR_ERR(salvo_phy->phy);
+
+	phy_set_drvdata(salvo_phy->phy, salvo_phy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static const struct cdns_salvo_data cdns_nxp_salvo_data = {
+	2,
+	cdns_nxp_sequence_pair,
+	ARRAY_SIZE(cdns_nxp_sequence_pair),
+};
+
+static const struct of_device_id cdns_salvo_phy_of_match[] = {
+	{
+		.compatible = "nxp,salvo-phy",
+		.data = &cdns_nxp_salvo_data,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, cdns_salvo_phy_of_match);
+
+static struct platform_driver cdns_salvo_phy_driver = {
+	.probe	= cdns_salvo_phy_probe,
+	.driver = {
+		.name	= "cdns-salvo-phy",
+		.of_match_table	= cdns_salvo_phy_of_match,
+	}
+};
+module_platform_driver(cdns_salvo_phy_driver);
+
+MODULE_AUTHOR("Peter Chen <peter.chen@nxp.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Cadence SALVO PHY Driver");
diff --git a/drivers/phy/cadence/phy-cadence-sierra.c b/drivers/phy/cadence/phy-cadence-sierra.c
index a5c08e5bd2bf..faed652b73f7 100644
--- a/drivers/phy/cadence/phy-cadence-sierra.c
+++ b/drivers/phy/cadence/phy-cadence-sierra.c
@@ -685,10 +685,10 @@ static struct cdns_reg_pairs cdns_usb_cmn_regs_ext_ssc[] = {
 static struct cdns_reg_pairs cdns_usb_ln_regs_ext_ssc[] = {
 	{0xFE0A, SIERRA_DET_STANDEC_A_PREG},
 	{0x000F, SIERRA_DET_STANDEC_B_PREG},
-	{0x00A5, SIERRA_DET_STANDEC_C_PREG},
+	{0x55A5, SIERRA_DET_STANDEC_C_PREG},
 	{0x69ad, SIERRA_DET_STANDEC_D_PREG},
 	{0x0241, SIERRA_DET_STANDEC_E_PREG},
-	{0x0010, SIERRA_PSM_LANECAL_DLY_A1_RESETS_PREG},
+	{0x0110, SIERRA_PSM_LANECAL_DLY_A1_RESETS_PREG},
 	{0x0014, SIERRA_PSM_A0IN_TMR_PREG},
 	{0xCF00, SIERRA_PSM_DIAG_PREG},
 	{0x001F, SIERRA_PSC_TX_A0_PREG},
@@ -696,7 +696,7 @@ static struct cdns_reg_pairs cdns_usb_ln_regs_ext_ssc[] = {
 	{0x0003, SIERRA_PSC_TX_A2_PREG},
 	{0x0003, SIERRA_PSC_TX_A3_PREG},
 	{0x0FFF, SIERRA_PSC_RX_A0_PREG},
-	{0x0619, SIERRA_PSC_RX_A1_PREG},
+	{0x0003, SIERRA_PSC_RX_A1_PREG},
 	{0x0003, SIERRA_PSC_RX_A2_PREG},
 	{0x0001, SIERRA_PSC_RX_A3_PREG},
 	{0x0001, SIERRA_PLLCTRL_SUBRATE_PREG},
@@ -705,19 +705,19 @@ static struct cdns_reg_pairs cdns_usb_ln_regs_ext_ssc[] = {
 	{0x00CA, SIERRA_CLKPATH_BIASTRIM_PREG},
 	{0x2512, SIERRA_DFE_BIASTRIM_PREG},
 	{0x0000, SIERRA_DRVCTRL_ATTEN_PREG},
-	{0x873E, SIERRA_CLKPATHCTRL_TMR_PREG},
-	{0x03CF, SIERRA_RX_CREQ_FLTR_A_MODE1_PREG},
-	{0x01CE, SIERRA_RX_CREQ_FLTR_A_MODE0_PREG},
+	{0x823E, SIERRA_CLKPATHCTRL_TMR_PREG},
+	{0x078F, SIERRA_RX_CREQ_FLTR_A_MODE1_PREG},
+	{0x078F, SIERRA_RX_CREQ_FLTR_A_MODE0_PREG},
 	{0x7B3C, SIERRA_CREQ_CCLKDET_MODE01_PREG},
-	{0x033F, SIERRA_RX_CTLE_MAINTENANCE_PREG},
+	{0x023C, SIERRA_RX_CTLE_MAINTENANCE_PREG},
 	{0x3232, SIERRA_CREQ_FSMCLK_SEL_PREG},
 	{0x0000, SIERRA_CREQ_EQ_CTRL_PREG},
-	{0x8000, SIERRA_CREQ_SPARE_PREG},
+	{0x0000, SIERRA_CREQ_SPARE_PREG},
 	{0xCC44, SIERRA_CREQ_EQ_OPEN_EYE_THRESH_PREG},
-	{0x8453, SIERRA_CTLELUT_CTRL_PREG},
-	{0x4110, SIERRA_DFE_ECMP_RATESEL_PREG},
-	{0x4110, SIERRA_DFE_SMP_RATESEL_PREG},
-	{0x0002, SIERRA_DEQ_PHALIGN_CTRL},
+	{0x8452, SIERRA_CTLELUT_CTRL_PREG},
+	{0x4121, SIERRA_DFE_ECMP_RATESEL_PREG},
+	{0x4121, SIERRA_DFE_SMP_RATESEL_PREG},
+	{0x0003, SIERRA_DEQ_PHALIGN_CTRL},
 	{0x3200, SIERRA_DEQ_CONCUR_CTRL1_PREG},
 	{0x5064, SIERRA_DEQ_CONCUR_CTRL2_PREG},
 	{0x0030, SIERRA_DEQ_EPIPWR_CTRL2_PREG},
@@ -725,7 +725,7 @@ static struct cdns_reg_pairs cdns_usb_ln_regs_ext_ssc[] = {
 	{0x5A5A, SIERRA_DEQ_ERRCMP_CTRL_PREG},
 	{0x02F5, SIERRA_DEQ_OFFSET_CTRL_PREG},
 	{0x02F5, SIERRA_DEQ_GAIN_CTRL_PREG},
-	{0x9A8A, SIERRA_DEQ_VGATUNE_CTRL_PREG},
+	{0x9999, SIERRA_DEQ_VGATUNE_CTRL_PREG},
 	{0x0014, SIERRA_DEQ_GLUT0},
 	{0x0014, SIERRA_DEQ_GLUT1},
 	{0x0014, SIERRA_DEQ_GLUT2},
@@ -772,6 +772,7 @@ static struct cdns_reg_pairs cdns_usb_ln_regs_ext_ssc[] = {
 	{0x000F, SIERRA_LFPSFILT_NS_PREG},
 	{0x0009, SIERRA_LFPSFILT_RD_PREG},
 	{0x0001, SIERRA_LFPSFILT_MP_PREG},
+	{0x6013, SIERRA_SIGDET_SUPPORT_PREG},
 	{0x8013, SIERRA_SDFILT_H2L_A_PREG},
 	{0x8009, SIERRA_SDFILT_L2H_PREG},
 	{0x0024, SIERRA_RXBUFFER_CTLECTRL_PREG},
diff --git a/drivers/phy/intel/Kconfig b/drivers/phy/intel/Kconfig
index 4ea6a8897cd7..7b47682a4e0e 100644
--- a/drivers/phy/intel/Kconfig
+++ b/drivers/phy/intel/Kconfig
@@ -2,8 +2,23 @@
 #
 # Phy drivers for Intel Lightning Mountain(LGM) platform
 #
+config PHY_INTEL_COMBO
+	bool "Intel ComboPHY driver"
+	depends on X86 || COMPILE_TEST
+	depends on OF && HAS_IOMEM
+	select MFD_SYSCON
+	select GENERIC_PHY
+	select REGMAP
+	help
+	  Enable this to support Intel ComboPhy.
+
+	  This driver configures ComboPhy subsystem on Intel gateway
+	  chipsets which provides PHYs for various controllers, EMAC,
+	  SATA and PCIe.
+
 config PHY_INTEL_EMMC
 	tristate "Intel EMMC PHY driver"
+	depends on X86 || COMPILE_TEST
 	select GENERIC_PHY
 	help
 	  Enable this to support the Intel EMMC PHY
diff --git a/drivers/phy/intel/Makefile b/drivers/phy/intel/Makefile
index 6b876a75599d..233d530dadde 100644
--- a/drivers/phy/intel/Makefile
+++ b/drivers/phy/intel/Makefile
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_PHY_INTEL_COMBO)		+= phy-intel-combo.o
 obj-$(CONFIG_PHY_INTEL_EMMC)            += phy-intel-emmc.o
diff --git a/drivers/phy/intel/phy-intel-combo.c b/drivers/phy/intel/phy-intel-combo.c
new file mode 100644
index 000000000000..c2a35be4cdfb
--- /dev/null
+++ b/drivers/phy/intel/phy-intel-combo.c
@@ -0,0 +1,632 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Intel Combo-PHY driver
+ *
+ * Copyright (C) 2019-2020 Intel Corporation.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/iopoll.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include <dt-bindings/phy/phy.h>
+
+#define PCIE_PHY_GEN_CTRL	0x00
+#define PCIE_PHY_CLK_PAD	BIT(17)
+
+#define PAD_DIS_CFG		0x174
+
+#define PCS_XF_ATE_OVRD_IN_2	0x3008
+#define ADAPT_REQ_MSK		GENMASK(5, 4)
+
+#define PCS_XF_RX_ADAPT_ACK	0x3010
+#define RX_ADAPT_ACK_BIT	BIT(0)
+
+#define CR_ADDR(addr, lane)	(((addr) + (lane) * 0x100) << 2)
+#define REG_COMBO_MODE(x)	((x) * 0x200)
+#define REG_CLK_DISABLE(x)	((x) * 0x200 + 0x124)
+
+#define COMBO_PHY_ID(x)		((x)->parent->id)
+#define PHY_ID(x)		((x)->id)
+
+#define CLK_100MHZ		100000000
+#define CLK_156_25MHZ		156250000
+
+static const unsigned long intel_iphy_clk_rates[] = {
+	CLK_100MHZ, CLK_156_25MHZ, CLK_100MHZ,
+};
+
+enum {
+	PHY_0,
+	PHY_1,
+	PHY_MAX_NUM
+};
+
+/*
+ * Clock Register bit fields to enable clocks
+ * for ComboPhy according to the mode.
+ */
+enum intel_phy_mode {
+	PHY_PCIE_MODE = 0,
+	PHY_XPCS_MODE,
+	PHY_SATA_MODE,
+};
+
+/* ComboPhy mode Register values */
+enum intel_combo_mode {
+	PCIE0_PCIE1_MODE = 0,
+	PCIE_DL_MODE,
+	RXAUI_MODE,
+	XPCS0_XPCS1_MODE,
+	SATA0_SATA1_MODE,
+};
+
+enum aggregated_mode {
+	PHY_SL_MODE,
+	PHY_DL_MODE,
+};
+
+struct intel_combo_phy;
+
+struct intel_cbphy_iphy {
+	struct phy		*phy;
+	struct intel_combo_phy	*parent;
+	struct reset_control	*app_rst;
+	u32			id;
+};
+
+struct intel_combo_phy {
+	struct device		*dev;
+	struct clk		*core_clk;
+	unsigned long		clk_rate;
+	void __iomem		*app_base;
+	void __iomem		*cr_base;
+	struct regmap		*syscfg;
+	struct regmap		*hsiocfg;
+	u32			id;
+	u32			bid;
+	struct reset_control	*phy_rst;
+	struct reset_control	*core_rst;
+	struct intel_cbphy_iphy	iphy[PHY_MAX_NUM];
+	enum intel_phy_mode	phy_mode;
+	enum aggregated_mode	aggr_mode;
+	u32			init_cnt;
+	struct mutex		lock;
+};
+
+static int intel_cbphy_iphy_enable(struct intel_cbphy_iphy *iphy, bool set)
+{
+	struct intel_combo_phy *cbphy = iphy->parent;
+	u32 mask = BIT(cbphy->phy_mode * 2 + iphy->id);
+	u32 val;
+
+	/* Register: 0 is enable, 1 is disable */
+	val = set ? 0 : mask;
+
+	return regmap_update_bits(cbphy->hsiocfg, REG_CLK_DISABLE(cbphy->bid),
+				  mask, val);
+}
+
+static int intel_cbphy_pcie_refclk_cfg(struct intel_cbphy_iphy *iphy, bool set)
+{
+	struct intel_combo_phy *cbphy = iphy->parent;
+	u32 mask = BIT(cbphy->id * 2 + iphy->id);
+	u32 val;
+
+	/* Register: 0 is enable, 1 is disable */
+	val = set ? 0 : mask;
+
+	return regmap_update_bits(cbphy->syscfg, PAD_DIS_CFG, mask, val);
+}
+
+static inline void combo_phy_w32_off_mask(void __iomem *base, unsigned int reg,
+					  u32 mask, u32 val)
+{
+	u32 reg_val;
+
+	reg_val = readl(base + reg);
+	reg_val &= ~mask;
+	reg_val |= FIELD_PREP(mask, val);
+	writel(reg_val, base + reg);
+}
+
+static int intel_cbphy_iphy_cfg(struct intel_cbphy_iphy *iphy,
+				int (*phy_cfg)(struct intel_cbphy_iphy *))
+{
+	struct intel_combo_phy *cbphy = iphy->parent;
+	int ret;
+
+	ret = phy_cfg(iphy);
+	if (ret)
+		return ret;
+
+	if (cbphy->aggr_mode != PHY_DL_MODE)
+		return 0;
+
+	return phy_cfg(&cbphy->iphy[PHY_1]);
+}
+
+static int intel_cbphy_pcie_en_pad_refclk(struct intel_cbphy_iphy *iphy)
+{
+	struct intel_combo_phy *cbphy = iphy->parent;
+	int ret;
+
+	ret = intel_cbphy_pcie_refclk_cfg(iphy, true);
+	if (ret) {
+		dev_err(cbphy->dev, "Failed to enable PCIe pad refclk\n");
+		return ret;
+	}
+
+	if (cbphy->init_cnt)
+		return 0;
+
+	combo_phy_w32_off_mask(cbphy->app_base, PCIE_PHY_GEN_CTRL,
+			       PCIE_PHY_CLK_PAD, 0);
+
+	/* Delay for stable clock PLL */
+	usleep_range(50, 100);
+
+	return 0;
+}
+
+static int intel_cbphy_pcie_dis_pad_refclk(struct intel_cbphy_iphy *iphy)
+{
+	struct intel_combo_phy *cbphy = iphy->parent;
+	int ret;
+
+	ret = intel_cbphy_pcie_refclk_cfg(iphy, false);
+	if (ret) {
+		dev_err(cbphy->dev, "Failed to disable PCIe pad refclk\n");
+		return ret;
+	}
+
+	if (cbphy->init_cnt)
+		return 0;
+
+	combo_phy_w32_off_mask(cbphy->app_base, PCIE_PHY_GEN_CTRL,
+			       PCIE_PHY_CLK_PAD, 1);
+
+	return 0;
+}
+
+static int intel_cbphy_set_mode(struct intel_combo_phy *cbphy)
+{
+	enum intel_combo_mode cb_mode = PHY_PCIE_MODE;
+	enum aggregated_mode aggr = cbphy->aggr_mode;
+	struct device *dev = cbphy->dev;
+	enum intel_phy_mode mode;
+	int ret;
+
+	mode = cbphy->phy_mode;
+
+	switch (mode) {
+	case PHY_PCIE_MODE:
+		cb_mode = (aggr == PHY_DL_MODE) ? PCIE_DL_MODE : PCIE0_PCIE1_MODE;
+		break;
+
+	case PHY_XPCS_MODE:
+		cb_mode = (aggr == PHY_DL_MODE) ? RXAUI_MODE : XPCS0_XPCS1_MODE;
+		break;
+
+	case PHY_SATA_MODE:
+		if (aggr == PHY_DL_MODE) {
+			dev_err(dev, "Mode:%u not support dual lane!\n", mode);
+			return -EINVAL;
+		}
+
+		cb_mode = SATA0_SATA1_MODE;
+		break;
+	}
+
+	ret = regmap_write(cbphy->hsiocfg, REG_COMBO_MODE(cbphy->bid), cb_mode);
+	if (ret)
+		dev_err(dev, "Failed to set ComboPhy mode: %d\n", ret);
+
+	return ret;
+}
+
+static void intel_cbphy_rst_assert(struct intel_combo_phy *cbphy)
+{
+	reset_control_assert(cbphy->core_rst);
+	reset_control_assert(cbphy->phy_rst);
+}
+
+static void intel_cbphy_rst_deassert(struct intel_combo_phy *cbphy)
+{
+	reset_control_deassert(cbphy->core_rst);
+	reset_control_deassert(cbphy->phy_rst);
+	/* Delay to ensure reset process is done */
+	usleep_range(10, 20);
+}
+
+static int intel_cbphy_iphy_power_on(struct intel_cbphy_iphy *iphy)
+{
+	struct intel_combo_phy *cbphy = iphy->parent;
+	int ret;
+
+	if (!cbphy->init_cnt) {
+		ret = clk_prepare_enable(cbphy->core_clk);
+		if (ret) {
+			dev_err(cbphy->dev, "Clock enable failed!\n");
+			return ret;
+		}
+
+		ret = clk_set_rate(cbphy->core_clk, cbphy->clk_rate);
+		if (ret) {
+			dev_err(cbphy->dev, "Clock freq set to %lu failed!\n",
+				cbphy->clk_rate);
+			goto clk_err;
+		}
+
+		intel_cbphy_rst_assert(cbphy);
+		intel_cbphy_rst_deassert(cbphy);
+		ret = intel_cbphy_set_mode(cbphy);
+		if (ret)
+			goto clk_err;
+	}
+
+	ret = intel_cbphy_iphy_enable(iphy, true);
+	if (ret) {
+		dev_err(cbphy->dev, "Failed enabling PHY core\n");
+		goto clk_err;
+	}
+
+	ret = reset_control_deassert(iphy->app_rst);
+	if (ret) {
+		dev_err(cbphy->dev, "PHY(%u:%u) reset deassert failed!\n",
+			COMBO_PHY_ID(iphy), PHY_ID(iphy));
+		goto clk_err;
+	}
+
+	/* Delay to ensure reset process is done */
+	udelay(1);
+
+	return 0;
+
+clk_err:
+	clk_disable_unprepare(cbphy->core_clk);
+
+	return ret;
+}
+
+static int intel_cbphy_iphy_power_off(struct intel_cbphy_iphy *iphy)
+{
+	struct intel_combo_phy *cbphy = iphy->parent;
+	int ret;
+
+	ret = reset_control_assert(iphy->app_rst);
+	if (ret) {
+		dev_err(cbphy->dev, "PHY(%u:%u) reset assert failed!\n",
+			COMBO_PHY_ID(iphy), PHY_ID(iphy));
+		return ret;
+	}
+
+	ret = intel_cbphy_iphy_enable(iphy, false);
+	if (ret) {
+		dev_err(cbphy->dev, "Failed disabling PHY core\n");
+		return ret;
+	}
+
+	if (cbphy->init_cnt)
+		return 0;
+
+	clk_disable_unprepare(cbphy->core_clk);
+	intel_cbphy_rst_assert(cbphy);
+
+	return 0;
+}
+
+static int intel_cbphy_init(struct phy *phy)
+{
+	struct intel_cbphy_iphy *iphy = phy_get_drvdata(phy);
+	struct intel_combo_phy *cbphy = iphy->parent;
+	int ret;
+
+	mutex_lock(&cbphy->lock);
+	ret = intel_cbphy_iphy_cfg(iphy, intel_cbphy_iphy_power_on);
+	if (ret)
+		goto err;
+
+	if (cbphy->phy_mode == PHY_PCIE_MODE) {
+		ret = intel_cbphy_iphy_cfg(iphy, intel_cbphy_pcie_en_pad_refclk);
+		if (ret)
+			goto err;
+	}
+
+	cbphy->init_cnt++;
+
+err:
+	mutex_unlock(&cbphy->lock);
+
+	return ret;
+}
+
+static int intel_cbphy_exit(struct phy *phy)
+{
+	struct intel_cbphy_iphy *iphy = phy_get_drvdata(phy);
+	struct intel_combo_phy *cbphy = iphy->parent;
+	int ret;
+
+	mutex_lock(&cbphy->lock);
+	cbphy->init_cnt--;
+	if (cbphy->phy_mode == PHY_PCIE_MODE) {
+		ret = intel_cbphy_iphy_cfg(iphy, intel_cbphy_pcie_dis_pad_refclk);
+		if (ret)
+			goto err;
+	}
+
+	ret = intel_cbphy_iphy_cfg(iphy, intel_cbphy_iphy_power_off);
+
+err:
+	mutex_unlock(&cbphy->lock);
+
+	return ret;
+}
+
+static int intel_cbphy_calibrate(struct phy *phy)
+{
+	struct intel_cbphy_iphy *iphy = phy_get_drvdata(phy);
+	struct intel_combo_phy *cbphy = iphy->parent;
+	void __iomem *cr_base = cbphy->cr_base;
+	int val, ret, id;
+
+	if (cbphy->phy_mode != PHY_XPCS_MODE)
+		return 0;
+
+	id = PHY_ID(iphy);
+
+	/* trigger auto RX adaptation */
+	combo_phy_w32_off_mask(cr_base, CR_ADDR(PCS_XF_ATE_OVRD_IN_2, id),
+			       ADAPT_REQ_MSK, 3);
+	/* Wait RX adaptation to finish */
+	ret = readl_poll_timeout(cr_base + CR_ADDR(PCS_XF_RX_ADAPT_ACK, id),
+				 val, val & RX_ADAPT_ACK_BIT, 10, 5000);
+	if (ret)
+		dev_err(cbphy->dev, "RX Adaptation failed!\n");
+	else
+		dev_dbg(cbphy->dev, "RX Adaptation success!\n");
+
+	/* Stop RX adaptation */
+	combo_phy_w32_off_mask(cr_base, CR_ADDR(PCS_XF_ATE_OVRD_IN_2, id),
+			       ADAPT_REQ_MSK, 0);
+
+	return ret;
+}
+
+static int intel_cbphy_fwnode_parse(struct intel_combo_phy *cbphy)
+{
+	struct device *dev = cbphy->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	struct fwnode_handle *fwnode = dev_fwnode(dev);
+	struct fwnode_reference_args ref;
+	int ret;
+	u32 val;
+
+	cbphy->core_clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(cbphy->core_clk)) {
+		ret = PTR_ERR(cbphy->core_clk);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Get clk failed:%d!\n", ret);
+		return ret;
+	}
+
+	cbphy->core_rst = devm_reset_control_get_optional(dev, "core");
+	if (IS_ERR(cbphy->core_rst)) {
+		ret = PTR_ERR(cbphy->core_rst);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Get core reset control err: %d!\n", ret);
+		return ret;
+	}
+
+	cbphy->phy_rst = devm_reset_control_get_optional(dev, "phy");
+	if (IS_ERR(cbphy->phy_rst)) {
+		ret = PTR_ERR(cbphy->phy_rst);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Get PHY reset control err: %d!\n", ret);
+		return ret;
+	}
+
+	cbphy->iphy[0].app_rst = devm_reset_control_get_optional(dev, "iphy0");
+	if (IS_ERR(cbphy->iphy[0].app_rst)) {
+		ret = PTR_ERR(cbphy->iphy[0].app_rst);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Get phy0 reset control err: %d!\n", ret);
+		return ret;
+	}
+
+	cbphy->iphy[1].app_rst = devm_reset_control_get_optional(dev, "iphy1");
+	if (IS_ERR(cbphy->iphy[1].app_rst)) {
+		ret = PTR_ERR(cbphy->iphy[1].app_rst);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Get phy1 reset control err: %d!\n", ret);
+		return ret;
+	}
+
+	cbphy->app_base = devm_platform_ioremap_resource_byname(pdev, "app");
+	if (IS_ERR(cbphy->app_base))
+		return PTR_ERR(cbphy->app_base);
+
+	cbphy->cr_base = devm_platform_ioremap_resource_byname(pdev, "core");
+	if (IS_ERR(cbphy->cr_base))
+		return PTR_ERR(cbphy->cr_base);
+
+	/*
+	 * syscfg and hsiocfg variables stores the handle of the registers set
+	 * in which ComboPhy subsytem specific registers are subset. Using
+	 * Register map framework to access the registers set.
+	 */
+	ret = fwnode_property_get_reference_args(fwnode, "intel,syscfg", NULL,
+						 1, 0, &ref);
+	if (ret < 0)
+		return ret;
+
+	cbphy->id = ref.args[0];
+	cbphy->syscfg = device_node_to_regmap(to_of_node(ref.fwnode));
+	fwnode_handle_put(ref.fwnode);
+
+	ret = fwnode_property_get_reference_args(fwnode, "intel,hsio", NULL, 1,
+						 0, &ref);
+	if (ret < 0)
+		return ret;
+
+	cbphy->bid = ref.args[0];
+	cbphy->hsiocfg = device_node_to_regmap(to_of_node(ref.fwnode));
+	fwnode_handle_put(ref.fwnode);
+
+	ret = fwnode_property_read_u32_array(fwnode, "intel,phy-mode", &val, 1);
+	if (ret)
+		return ret;
+
+	switch (val) {
+	case PHY_TYPE_PCIE:
+		cbphy->phy_mode = PHY_PCIE_MODE;
+		break;
+
+	case PHY_TYPE_SATA:
+		cbphy->phy_mode = PHY_SATA_MODE;
+		break;
+
+	case PHY_TYPE_XPCS:
+		cbphy->phy_mode = PHY_XPCS_MODE;
+		break;
+
+	default:
+		dev_err(dev, "Invalid PHY mode: %u\n", val);
+		return -EINVAL;
+	}
+
+	cbphy->clk_rate = intel_iphy_clk_rates[cbphy->phy_mode];
+
+	if (fwnode_property_present(fwnode, "intel,aggregation"))
+		cbphy->aggr_mode = PHY_DL_MODE;
+	else
+		cbphy->aggr_mode = PHY_SL_MODE;
+
+	return 0;
+}
+
+static const struct phy_ops intel_cbphy_ops = {
+	.init		= intel_cbphy_init,
+	.exit		= intel_cbphy_exit,
+	.calibrate	= intel_cbphy_calibrate,
+	.owner		= THIS_MODULE,
+};
+
+static struct phy *intel_cbphy_xlate(struct device *dev,
+				     struct of_phandle_args *args)
+{
+	struct intel_combo_phy *cbphy = dev_get_drvdata(dev);
+	u32 iphy_id;
+
+	if (args->args_count < 1) {
+		dev_err(dev, "Invalid number of arguments\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	iphy_id = args->args[0];
+	if (iphy_id >= PHY_MAX_NUM) {
+		dev_err(dev, "Invalid phy instance %d\n", iphy_id);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (cbphy->aggr_mode == PHY_DL_MODE && iphy_id == PHY_1) {
+		dev_err(dev, "Invalid. ComboPhy is in Dual lane mode %d\n", iphy_id);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return cbphy->iphy[iphy_id].phy;
+}
+
+static int intel_cbphy_create(struct intel_combo_phy *cbphy)
+{
+	struct phy_provider *phy_provider;
+	struct device *dev = cbphy->dev;
+	struct intel_cbphy_iphy *iphy;
+	int i;
+
+	for (i = 0; i < PHY_MAX_NUM; i++) {
+		iphy = &cbphy->iphy[i];
+		iphy->parent = cbphy;
+		iphy->id = i;
+
+		/* In dual lane mode skip phy creation for the second phy */
+		if (cbphy->aggr_mode == PHY_DL_MODE && iphy->id == PHY_1)
+			continue;
+
+		iphy->phy = devm_phy_create(dev, NULL, &intel_cbphy_ops);
+		if (IS_ERR(iphy->phy)) {
+			dev_err(dev, "PHY[%u:%u]: create PHY instance failed!\n",
+				COMBO_PHY_ID(iphy), PHY_ID(iphy));
+
+			return PTR_ERR(iphy->phy);
+		}
+
+		phy_set_drvdata(iphy->phy, iphy);
+	}
+
+	dev_set_drvdata(dev, cbphy);
+	phy_provider = devm_of_phy_provider_register(dev, intel_cbphy_xlate);
+	if (IS_ERR(phy_provider))
+		dev_err(dev, "Register PHY provider failed!\n");
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static int intel_cbphy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct intel_combo_phy *cbphy;
+	int ret;
+
+	cbphy = devm_kzalloc(dev, sizeof(*cbphy), GFP_KERNEL);
+	if (!cbphy)
+		return -ENOMEM;
+
+	cbphy->dev = dev;
+	cbphy->init_cnt = 0;
+	mutex_init(&cbphy->lock);
+	ret = intel_cbphy_fwnode_parse(cbphy);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, cbphy);
+
+	return intel_cbphy_create(cbphy);
+}
+
+static int intel_cbphy_remove(struct platform_device *pdev)
+{
+	struct intel_combo_phy *cbphy = platform_get_drvdata(pdev);
+
+	intel_cbphy_rst_assert(cbphy);
+	clk_disable_unprepare(cbphy->core_clk);
+	return 0;
+}
+
+static const struct of_device_id of_intel_cbphy_match[] = {
+	{ .compatible = "intel,combo-phy" },
+	{ .compatible = "intel,combophy-lgm" },
+	{}
+};
+
+static struct platform_driver intel_cbphy_driver = {
+	.probe = intel_cbphy_probe,
+	.remove = intel_cbphy_remove,
+	.driver = {
+		.name = "intel-combo-phy",
+		.of_match_table = of_intel_cbphy_match,
+	}
+};
+
+module_platform_driver(intel_cbphy_driver);
+
+MODULE_DESCRIPTION("Intel Combo-phy driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c
index 12e71a315a2c..089db0dea703 100644
--- a/drivers/phy/motorola/phy-cpcap-usb.c
+++ b/drivers/phy/motorola/phy-cpcap-usb.c
@@ -122,7 +122,6 @@ enum cpcap_gpio_mode {
 struct cpcap_phy_ddata {
 	struct regmap *reg;
 	struct device *dev;
-	struct clk *refclk;
 	struct usb_phy phy;
 	struct delayed_work detect_work;
 	struct pinctrl *pins;
@@ -707,7 +706,6 @@ static int cpcap_usb_phy_remove(struct platform_device *pdev)
 
 	usb_remove_phy(&ddata->phy);
 	cancel_delayed_work_sync(&ddata->detect_work);
-	clk_unprepare(ddata->refclk);
 	regulator_disable(ddata->vusb);
 
 	return 0;
diff --git a/drivers/phy/qualcomm/Kconfig b/drivers/phy/qualcomm/Kconfig
index 98674ed094d9..ca9ce7e84a5c 100644
--- a/drivers/phy/qualcomm/Kconfig
+++ b/drivers/phy/qualcomm/Kconfig
@@ -18,6 +18,13 @@ config PHY_QCOM_APQ8064_SATA
 	depends on OF
 	select GENERIC_PHY
 
+config PHY_QCOM_IPQ4019_USB
+	tristate "Qualcomm IPQ4019 USB PHY driver"
+	depends on OF && (ARCH_QCOM || COMPILE_TEST)
+	select GENERIC_PHY
+	help
+	  Support for the USB PHY-s on Qualcomm IPQ40xx SoC-s.
+
 config PHY_QCOM_IPQ806X_SATA
 	tristate "Qualcomm IPQ806x SATA SerDes/PHY driver"
 	depends on ARCH_QCOM
@@ -85,6 +92,16 @@ config PHY_QCOM_USB_HS
 	  Support for the USB high-speed ULPI compliant phy on Qualcomm
 	  chipsets.
 
+config PHY_QCOM_USB_SNPS_FEMTO_V2
+	tristate "Qualcomm SNPS FEMTO USB HS PHY V2 module"
+	depends on OF && (ARCH_QCOM || COMPILE_TEST)
+	select GENERIC_PHY
+	help
+	  Enable support for the USB high-speed SNPS Femto phy on Qualcomm
+	  chipsets.  This PHY has differences in the register map compared
+	  to the V1 variants.  The PHY is paired with a Synopsys DWC3 USB
+	  controller on Qualcomm SOCs.
+
 config PHY_QCOM_USB_HSIC
 	tristate "Qualcomm USB HSIC ULPI PHY module"
 	depends on USB_ULPI_BUS
diff --git a/drivers/phy/qualcomm/Makefile b/drivers/phy/qualcomm/Makefile
index 1f14aeacbd70..86fb32efab79 100644
--- a/drivers/phy/qualcomm/Makefile
+++ b/drivers/phy/qualcomm/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_PHY_ATH79_USB)		+= phy-ath79-usb.o
 obj-$(CONFIG_PHY_QCOM_APQ8064_SATA)	+= phy-qcom-apq8064-sata.o
+obj-$(CONFIG_PHY_QCOM_IPQ4019_USB)	+= phy-qcom-ipq4019-usb.o
 obj-$(CONFIG_PHY_QCOM_IPQ806X_SATA)	+= phy-qcom-ipq806x-sata.o
 obj-$(CONFIG_PHY_QCOM_PCIE2)		+= phy-qcom-pcie2.o
 obj-$(CONFIG_PHY_QCOM_QMP)		+= phy-qcom-qmp.o
@@ -12,3 +13,4 @@ obj-$(CONFIG_PHY_QCOM_USB_HS) 		+= phy-qcom-usb-hs.o
 obj-$(CONFIG_PHY_QCOM_USB_HSIC) 	+= phy-qcom-usb-hsic.o
 obj-$(CONFIG_PHY_QCOM_USB_HS_28NM)	+= phy-qcom-usb-hs-28nm.o
 obj-$(CONFIG_PHY_QCOM_USB_SS)		+= phy-qcom-usb-ss.o
+obj-$(CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2)+= phy-qcom-snps-femto-v2.o
diff --git a/drivers/phy/qualcomm/phy-qcom-ipq4019-usb.c b/drivers/phy/qualcomm/phy-qcom-ipq4019-usb.c
new file mode 100644
index 000000000000..b8ef331e1545
--- /dev/null
+++ b/drivers/phy/qualcomm/phy-qcom-ipq4019-usb.c
@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2018 John Crispin <john@phrozen.org>
+ *
+ * Based on code from
+ * Allwinner Technology Co., Ltd. <www.allwinnertech.com>
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_platform.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+struct ipq4019_usb_phy {
+	struct device		*dev;
+	struct phy		*phy;
+	void __iomem		*base;
+	struct reset_control	*por_rst;
+	struct reset_control	*srif_rst;
+};
+
+static int ipq4019_ss_phy_power_off(struct phy *_phy)
+{
+	struct ipq4019_usb_phy *phy = phy_get_drvdata(_phy);
+
+	reset_control_assert(phy->por_rst);
+	msleep(10);
+
+	return 0;
+}
+
+static int ipq4019_ss_phy_power_on(struct phy *_phy)
+{
+	struct ipq4019_usb_phy *phy = phy_get_drvdata(_phy);
+
+	ipq4019_ss_phy_power_off(_phy);
+
+	reset_control_deassert(phy->por_rst);
+
+	return 0;
+}
+
+static struct phy_ops ipq4019_usb_ss_phy_ops = {
+	.power_on	= ipq4019_ss_phy_power_on,
+	.power_off	= ipq4019_ss_phy_power_off,
+};
+
+static int ipq4019_hs_phy_power_off(struct phy *_phy)
+{
+	struct ipq4019_usb_phy *phy = phy_get_drvdata(_phy);
+
+	reset_control_assert(phy->por_rst);
+	msleep(10);
+
+	reset_control_assert(phy->srif_rst);
+	msleep(10);
+
+	return 0;
+}
+
+static int ipq4019_hs_phy_power_on(struct phy *_phy)
+{
+	struct ipq4019_usb_phy *phy = phy_get_drvdata(_phy);
+
+	ipq4019_hs_phy_power_off(_phy);
+
+	reset_control_deassert(phy->srif_rst);
+	msleep(10);
+
+	reset_control_deassert(phy->por_rst);
+
+	return 0;
+}
+
+static struct phy_ops ipq4019_usb_hs_phy_ops = {
+	.power_on	= ipq4019_hs_phy_power_on,
+	.power_off	= ipq4019_hs_phy_power_off,
+};
+
+static const struct of_device_id ipq4019_usb_phy_of_match[] = {
+	{ .compatible = "qcom,usb-hs-ipq4019-phy", .data = &ipq4019_usb_hs_phy_ops},
+	{ .compatible = "qcom,usb-ss-ipq4019-phy", .data = &ipq4019_usb_ss_phy_ops},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ipq4019_usb_phy_of_match);
+
+static int ipq4019_usb_phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	struct phy_provider *phy_provider;
+	struct ipq4019_usb_phy *phy;
+
+	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
+	if (!phy)
+		return -ENOMEM;
+
+	phy->dev = &pdev->dev;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	phy->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(phy->base)) {
+		dev_err(dev, "failed to remap register memory\n");
+		return PTR_ERR(phy->base);
+	}
+
+	phy->por_rst = devm_reset_control_get(phy->dev, "por_rst");
+	if (IS_ERR(phy->por_rst)) {
+		if (PTR_ERR(phy->por_rst) != -EPROBE_DEFER)
+			dev_err(dev, "POR reset is missing\n");
+		return PTR_ERR(phy->por_rst);
+	}
+
+	phy->srif_rst = devm_reset_control_get_optional(phy->dev, "srif_rst");
+	if (IS_ERR(phy->srif_rst))
+		return PTR_ERR(phy->srif_rst);
+
+	phy->phy = devm_phy_create(dev, NULL, of_device_get_match_data(dev));
+	if (IS_ERR(phy->phy)) {
+		dev_err(dev, "failed to create PHY\n");
+		return PTR_ERR(phy->phy);
+	}
+	phy_set_drvdata(phy->phy, phy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static struct platform_driver ipq4019_usb_phy_driver = {
+	.probe	= ipq4019_usb_phy_probe,
+	.driver = {
+		.of_match_table	= ipq4019_usb_phy_of_match,
+		.name  = "ipq4019-usb-phy",
+	}
+};
+module_platform_driver(ipq4019_usb_phy_driver);
+
+MODULE_DESCRIPTION("QCOM/IPQ4019 USB phy driver");
+MODULE_AUTHOR("John Crispin <john@phrozen.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c
index c190406246ab..e91040af3394 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.c
@@ -119,14 +119,17 @@ enum qphy_reg_layout {
 	QPHY_PCS_AUTONOMOUS_MODE_CTRL,
 	QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR,
 	QPHY_PCS_LFPS_RXTERM_IRQ_STATUS,
+	QPHY_PCS_POWER_DOWN_CONTROL,
+	/* Keep last to ensure regs_layout arrays are properly initialized */
+	QPHY_LAYOUT_SIZE
 };
 
-static const unsigned int msm8996_ufsphy_regs_layout[] = {
+static const unsigned int msm8996_ufsphy_regs_layout[QPHY_LAYOUT_SIZE] = {
 	[QPHY_START_CTRL]		= 0x00,
 	[QPHY_PCS_READY_STATUS]		= 0x168,
 };
 
-static const unsigned int pciephy_regs_layout[] = {
+static const unsigned int pciephy_regs_layout[QPHY_LAYOUT_SIZE] = {
 	[QPHY_COM_SW_RESET]		= 0x400,
 	[QPHY_COM_POWER_DOWN_CONTROL]	= 0x404,
 	[QPHY_COM_START_CONTROL]	= 0x408,
@@ -142,7 +145,7 @@ static const unsigned int pciephy_regs_layout[] = {
 	[QPHY_PCS_STATUS]		= 0x174,
 };
 
-static const unsigned int usb3phy_regs_layout[] = {
+static const unsigned int usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = {
 	[QPHY_FLL_CNTRL1]		= 0xc0,
 	[QPHY_FLL_CNTRL2]		= 0xc4,
 	[QPHY_FLL_CNT_VAL_L]		= 0xc8,
@@ -156,7 +159,7 @@ static const unsigned int usb3phy_regs_layout[] = {
 	[QPHY_PCS_LFPS_RXTERM_IRQ_STATUS] = 0x178,
 };
 
-static const unsigned int qmp_v3_usb3phy_regs_layout[] = {
+static const unsigned int qmp_v3_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = {
 	[QPHY_SW_RESET]			= 0x00,
 	[QPHY_START_CTRL]		= 0x08,
 	[QPHY_PCS_STATUS]		= 0x174,
@@ -165,27 +168,34 @@ static const unsigned int qmp_v3_usb3phy_regs_layout[] = {
 	[QPHY_PCS_LFPS_RXTERM_IRQ_STATUS] = 0x170,
 };
 
-static const unsigned int sdm845_qmp_pciephy_regs_layout[] = {
+static const unsigned int sdm845_qmp_pciephy_regs_layout[QPHY_LAYOUT_SIZE] = {
 	[QPHY_SW_RESET]			= 0x00,
 	[QPHY_START_CTRL]		= 0x08,
 	[QPHY_PCS_STATUS]		= 0x174,
 };
 
-static const unsigned int sdm845_qhp_pciephy_regs_layout[] = {
+static const unsigned int sdm845_qhp_pciephy_regs_layout[QPHY_LAYOUT_SIZE] = {
 	[QPHY_SW_RESET]			= 0x00,
 	[QPHY_START_CTRL]		= 0x08,
 	[QPHY_PCS_STATUS]		= 0x2ac,
 };
 
-static const unsigned int sdm845_ufsphy_regs_layout[] = {
+static const unsigned int qmp_v4_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = {
+	[QPHY_SW_RESET]			= 0x00,
+	[QPHY_START_CTRL]		= 0x44,
+	[QPHY_PCS_STATUS]		= 0x14,
+	[QPHY_PCS_POWER_DOWN_CONTROL]	= 0x40,
+};
+
+static const unsigned int sdm845_ufsphy_regs_layout[QPHY_LAYOUT_SIZE] = {
 	[QPHY_START_CTRL]		= 0x00,
 	[QPHY_PCS_READY_STATUS]		= 0x160,
 };
 
-static const unsigned int sm8150_ufsphy_regs_layout[] = {
-	[QPHY_START_CTRL]		= QPHY_V4_PHY_START,
-	[QPHY_PCS_READY_STATUS]		= QPHY_V4_PCS_READY_STATUS,
-	[QPHY_SW_RESET]			= QPHY_V4_SW_RESET,
+static const unsigned int sm8150_ufsphy_regs_layout[QPHY_LAYOUT_SIZE] = {
+	[QPHY_START_CTRL]		= QPHY_V4_PCS_UFS_PHY_START,
+	[QPHY_PCS_READY_STATUS]		= QPHY_V4_PCS_UFS_READY_STATUS,
+	[QPHY_SW_RESET]			= QPHY_V4_PCS_UFS_SW_RESET,
 };
 
 static const struct qmp_phy_init_tbl msm8996_pcie_serdes_tbl[] = {
@@ -1272,13 +1282,121 @@ static const struct qmp_phy_init_tbl sm8150_ufsphy_rx_tbl[] = {
 };
 
 static const struct qmp_phy_init_tbl sm8150_ufsphy_pcs_tbl[] = {
-	QMP_PHY_INIT_CFG(QPHY_V4_RX_SIGDET_CTRL2, 0x6d),
-	QMP_PHY_INIT_CFG(QPHY_V4_TX_LARGE_AMP_DRV_LVL, 0x0a),
-	QMP_PHY_INIT_CFG(QPHY_V4_TX_SMALL_AMP_DRV_LVL, 0x02),
-	QMP_PHY_INIT_CFG(QPHY_V4_TX_MID_TERM_CTRL1, 0x43),
-	QMP_PHY_INIT_CFG(QPHY_V4_DEBUG_BUS_CLKSEL, 0x1f),
-	QMP_PHY_INIT_CFG(QPHY_V4_RX_MIN_HIBERN8_TIME, 0xff),
-	QMP_PHY_INIT_CFG(QPHY_V4_MULTI_LANE_CTRL1, 0x02),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_UFS_RX_SIGDET_CTRL2, 0x6d),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0a),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_UFS_TX_SMALL_AMP_DRV_LVL, 0x02),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_UFS_TX_MID_TERM_CTRL1, 0x43),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_UFS_DEBUG_BUS_CLKSEL, 0x1f),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_UFS_RX_MIN_HIBERN8_TIME, 0xff),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_UFS_MULTI_LANE_CTRL1, 0x02),
+};
+
+static const struct qmp_phy_init_tbl sm8150_usb3_serdes_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SSC_STEP_SIZE1_MODE0, 0xde),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SSC_STEP_SIZE2_MODE0, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SSC_STEP_SIZE1_MODE1, 0xde),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SSC_STEP_SIZE2_MODE1, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SYSCLK_BUF_ENABLE, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_CMN_IPTRIM, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_CP_CTRL_MODE0, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_CP_CTRL_MODE1, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_RCTRL_MODE1, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_CCTRL_MODE0, 0x36),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_CCTRL_MODE1, 0x36),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_SYSCLK_EN_SEL, 0x1a),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_LOCK_CMP_EN, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_LOCK_CMP1_MODE0, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_LOCK_CMP2_MODE0, 0x34),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_LOCK_CMP1_MODE1, 0x34),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_LOCK_CMP2_MODE1, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DEC_START_MODE1, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DIV_FRAC_START1_MODE0, 0xab),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DIV_FRAC_START2_MODE0, 0xea),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DIV_FRAC_START3_MODE0, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_VCO_TUNE_MAP, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DIV_FRAC_START1_MODE1, 0xab),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DIV_FRAC_START2_MODE1, 0xea),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_DIV_FRAC_START3_MODE1, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_VCO_TUNE1_MODE0, 0x24),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_VCO_TUNE1_MODE1, 0x24),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_VCO_TUNE2_MODE1, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_HSCLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_CORECLK_DIV_MODE1, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0xca),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x1e),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE1_MODE1, 0xca),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE2_MODE1, 0x1e),
+	QMP_PHY_INIT_CFG(QSERDES_V4_COM_BIN_VCOCAL_HSCLK_SEL, 0x11),
+};
+
+static const struct qmp_phy_init_tbl sm8150_usb3_tx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V4_TX_RES_CODE_LANE_TX, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V4_TX_RES_CODE_LANE_RX, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V4_TX_LANE_MODE_1, 0xd5),
+	QMP_PHY_INIT_CFG(QSERDES_V4_TX_RCV_DETECT_LVL_2, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V4_TX_PI_QEC_CTRL, 0x20),
+};
+
+static const struct qmp_phy_init_tbl sm8150_usb3_rx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SO_GAIN, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x7f),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_PI_CONTROLS, 0x99),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_THRESH1, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_THRESH2, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_GAIN1, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_UCDR_SB2_GAIN2, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_VGA_CAL_CNTRL1, 0x54),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_VGA_CAL_CNTRL2, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4a),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_IDAC_TSETTLE_LOW, 0xc0),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_IDAC_TSETTLE_HIGH, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_SIGDET_CNTRL, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_SIGDET_DEGLITCH_CNTRL, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_LOW, 0xbf),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH, 0xbf),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH2, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH3, 0x7f),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_00_HIGH4, 0x94),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_LOW, 0xdc),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH, 0xdc),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH2, 0x5c),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH3, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_RX_MODE_01_HIGH4, 0xb3),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_DFE_EN_TIMER, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_AUX_DATA_TCOARSE_TFINE, 0xa0),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_DCC_CTRL1, 0x0c),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_GM_CAL, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V4_RX_VTH_CODE, 0x10),
+};
+
+static const struct qmp_phy_init_tbl sm8150_usb3_pcs_tbl[] = {
+	/* Lock Det settings */
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG1, 0xd0),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG2, 0x07),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_LOCK_DETECT_CONFIG6, 0x13),
+
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_REFGEN_REQ_CONFIG1, 0x21),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_RX_SIGDET_LVL, 0xaa),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_CDR_RESET_TIME, 0x0a),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG1, 0x88),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_ALIGN_DETECT_CONFIG2, 0x13),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_PCS_TX_RX_CONFIG, 0x0c),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG1, 0x4b),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_EQ_CONFIG5, 0x10),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL, 0xf8),
+	QMP_PHY_INIT_CFG(QPHY_V4_PCS_USB3_RXEQTRAINING_DFE_TIME_S2, 0x07),
 };
 
 /* struct qmp_phy_cfg - per-PHY initialization config */
@@ -1445,6 +1563,10 @@ static const char * const sdm845_pciephy_clk_l[] = {
 	"aux", "cfg_ahb", "ref", "refgen",
 };
 
+static const char * const qmp_v4_phy_clk_l[] = {
+	"aux", "ref_clk_src", "ref", "com_aux",
+};
+
 static const char * const sdm845_ufs_phy_clk_l[] = {
 	"ref", "ref_aux",
 };
@@ -1458,6 +1580,10 @@ static const char * const msm8996_usb3phy_reset_l[] = {
 	"phy", "common",
 };
 
+static const char * const sc7180_usb3phy_reset_l[] = {
+	"phy",
+};
+
 static const char * const sdm845_pciephy_reset_l[] = {
 	"phy",
 };
@@ -1671,6 +1797,37 @@ static const struct qmp_phy_cfg qmp_v3_usb3phy_cfg = {
 	.is_dual_lane_phy	= true,
 };
 
+static const struct qmp_phy_cfg sc7180_usb3phy_cfg = {
+	.type			= PHY_TYPE_USB3,
+	.nlanes			= 1,
+
+	.serdes_tbl		= qmp_v3_usb3_serdes_tbl,
+	.serdes_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_serdes_tbl),
+	.tx_tbl			= qmp_v3_usb3_tx_tbl,
+	.tx_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_tx_tbl),
+	.rx_tbl			= qmp_v3_usb3_rx_tbl,
+	.rx_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_rx_tbl),
+	.pcs_tbl		= qmp_v3_usb3_pcs_tbl,
+	.pcs_tbl_num		= ARRAY_SIZE(qmp_v3_usb3_pcs_tbl),
+	.clk_list		= qmp_v3_phy_clk_l,
+	.num_clks		= ARRAY_SIZE(qmp_v3_phy_clk_l),
+	.reset_list		= sc7180_usb3phy_reset_l,
+	.num_resets		= ARRAY_SIZE(sc7180_usb3phy_reset_l),
+	.vreg_list		= qmp_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(qmp_phy_vreg_l),
+	.regs			= qmp_v3_usb3phy_regs_layout,
+
+	.start_ctrl		= SERDES_START | PCS_START,
+	.pwrdn_ctrl		= SW_PWRDN,
+
+	.has_pwrdn_delay	= true,
+	.pwrdn_delay_min	= POWER_DOWN_DELAY_US_MIN,
+	.pwrdn_delay_max	= POWER_DOWN_DELAY_US_MAX,
+
+	.has_phy_dp_com_ctrl	= true,
+	.is_dual_lane_phy	= true,
+};
+
 static const struct qmp_phy_cfg qmp_v3_usb3_uniphy_cfg = {
 	.type			= PHY_TYPE_USB3,
 	.nlanes			= 1,
@@ -1798,6 +1955,37 @@ static const struct qmp_phy_cfg sm8150_ufsphy_cfg = {
 	.is_dual_lane_phy	= true,
 };
 
+static const struct qmp_phy_cfg sm8150_usb3phy_cfg = {
+	.type			= PHY_TYPE_USB3,
+	.nlanes			= 1,
+
+	.serdes_tbl		= sm8150_usb3_serdes_tbl,
+	.serdes_tbl_num		= ARRAY_SIZE(sm8150_usb3_serdes_tbl),
+	.tx_tbl			= sm8150_usb3_tx_tbl,
+	.tx_tbl_num		= ARRAY_SIZE(sm8150_usb3_tx_tbl),
+	.rx_tbl			= sm8150_usb3_rx_tbl,
+	.rx_tbl_num		= ARRAY_SIZE(sm8150_usb3_rx_tbl),
+	.pcs_tbl		= sm8150_usb3_pcs_tbl,
+	.pcs_tbl_num		= ARRAY_SIZE(sm8150_usb3_pcs_tbl),
+	.clk_list		= qmp_v4_phy_clk_l,
+	.num_clks		= ARRAY_SIZE(qmp_v4_phy_clk_l),
+	.reset_list		= msm8996_usb3phy_reset_l,
+	.num_resets		= ARRAY_SIZE(msm8996_usb3phy_reset_l),
+	.vreg_list		= qmp_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(qmp_phy_vreg_l),
+	.regs			= qmp_v4_usb3phy_regs_layout,
+
+	.start_ctrl		= SERDES_START | PCS_START,
+	.pwrdn_ctrl		= SW_PWRDN,
+
+	.has_pwrdn_delay	= true,
+	.pwrdn_delay_min	= POWER_DOWN_DELAY_US_MIN,
+	.pwrdn_delay_max	= POWER_DOWN_DELAY_US_MAX,
+
+	.has_phy_dp_com_ctrl	= true,
+	.is_dual_lane_phy	= true,
+};
+
 static void qcom_qmp_phy_configure(void __iomem *base,
 				   const unsigned int *regs,
 				   const struct qmp_phy_init_tbl tbl[],
@@ -1880,11 +2068,18 @@ static int qcom_qmp_phy_com_init(struct qmp_phy *qphy)
 			     SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET);
 	}
 
-	if (cfg->has_phy_com_ctrl)
+	if (cfg->has_phy_com_ctrl) {
 		qphy_setbits(serdes, cfg->regs[QPHY_COM_POWER_DOWN_CONTROL],
 			     SW_PWRDN);
-	else
-		qphy_setbits(pcs, QPHY_POWER_DOWN_CONTROL, cfg->pwrdn_ctrl);
+	} else {
+		if (cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL])
+			qphy_setbits(pcs,
+					cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL],
+					cfg->pwrdn_ctrl);
+		else
+			qphy_setbits(pcs, QPHY_POWER_DOWN_CONTROL,
+					cfg->pwrdn_ctrl);
+	}
 
 	/* Serdes configuration */
 	qcom_qmp_phy_configure(serdes, cfg->regs, cfg->serdes_tbl,
@@ -2110,7 +2305,13 @@ static int qcom_qmp_phy_disable(struct phy *phy)
 	qphy_clrbits(qphy->pcs, cfg->regs[QPHY_START_CTRL], cfg->start_ctrl);
 
 	/* Put PHY into POWER DOWN state: active low */
-	qphy_clrbits(qphy->pcs, QPHY_POWER_DOWN_CONTROL, cfg->pwrdn_ctrl);
+	if (cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL]) {
+		qphy_clrbits(qphy->pcs, cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL],
+			     cfg->pwrdn_ctrl);
+	} else {
+		qphy_clrbits(qphy->pcs, QPHY_POWER_DOWN_CONTROL,
+				cfg->pwrdn_ctrl);
+	}
 
 	if (cfg->has_lane_rst)
 		reset_control_assert(qphy->lane_rst);
@@ -2516,6 +2717,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = {
 		.compatible = "qcom,ipq8074-qmp-pcie-phy",
 		.data = &ipq8074_pciephy_cfg,
 	}, {
+		.compatible = "qcom,sc7180-qmp-usb3-phy",
+		.data = &sc7180_usb3phy_cfg,
+	}, {
 		.compatible = "qcom,sdm845-qhp-pcie-phy",
 		.data = &sdm845_qhp_pciephy_cfg,
 	}, {
@@ -2536,6 +2740,12 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = {
 	}, {
 		.compatible = "qcom,sm8150-qmp-ufs-phy",
 		.data = &sm8150_ufsphy_cfg,
+	}, {
+		.compatible = "qcom,sm8250-qmp-ufs-phy",
+		.data = &sm8150_ufsphy_cfg,
+	}, {
+		.compatible = "qcom,sm8150-qmp-usb3-phy",
+		.data = &sm8150_usb3phy_cfg,
 	},
 	{ },
 };
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h
index dece0e67704b..6d017a0c0c8d 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.h
@@ -125,7 +125,7 @@
 #define QPHY_L1SS_WAKEUP_DLY_TIME_AUXCLK_LSB		0x1DC
 #define QPHY_L1SS_WAKEUP_DLY_TIME_AUXCLK_MSB		0x1E0
 
-/* Only for QMP V3 PHY - DP COM registers */
+/* Only for QMP V3 & V4 PHY - DP COM registers */
 #define QPHY_V3_DP_COM_PHY_MODE_CTRL			0x00
 #define QPHY_V3_DP_COM_SW_RESET				0x04
 #define QPHY_V3_DP_COM_POWER_DOWN_CTRL			0x08
@@ -314,6 +314,14 @@
 #define QPHY_V3_PCS_MISC_OSC_DTCT_MODE2_CONFIG5		0x60
 
 /* Only for QMP V4 PHY - QSERDES COM registers */
+#define QSERDES_V4_COM_SSC_EN_CENTER			0x010
+#define QSERDES_V4_COM_SSC_PER1				0x01c
+#define QSERDES_V4_COM_SSC_PER2				0x020
+#define QSERDES_V4_COM_SSC_STEP_SIZE1_MODE0		0x024
+#define QSERDES_V4_COM_SSC_STEP_SIZE2_MODE0		0x028
+#define QSERDES_V4_COM_SSC_STEP_SIZE1_MODE1		0x030
+#define QSERDES_V4_COM_SSC_STEP_SIZE2_MODE1		0x034
+#define QSERDES_V4_COM_SYSCLK_BUF_ENABLE		0x050
 #define QSERDES_V4_COM_PLL_IVCO				0x058
 #define QSERDES_V4_COM_CMN_IPTRIM			0x060
 #define QSERDES_V4_COM_CP_CTRL_MODE0			0x074
@@ -330,10 +338,22 @@
 #define QSERDES_V4_COM_DEC_START_MODE0			0x0bc
 #define QSERDES_V4_COM_LOCK_CMP2_MODE1			0x0b8
 #define QSERDES_V4_COM_DEC_START_MODE1			0x0c4
+#define QSERDES_V4_COM_DIV_FRAC_START1_MODE0		0x0cc
+#define QSERDES_V4_COM_DIV_FRAC_START2_MODE0		0x0d0
+#define QSERDES_V4_COM_DIV_FRAC_START3_MODE0		0x0d4
+#define QSERDES_V4_COM_DIV_FRAC_START1_MODE1		0x0d8
+#define QSERDES_V4_COM_DIV_FRAC_START2_MODE1		0x0dc
+#define QSERDES_V4_COM_DIV_FRAC_START3_MODE1		0x0e0
 #define QSERDES_V4_COM_VCO_TUNE_MAP			0x10c
+#define QSERDES_V4_COM_VCO_TUNE1_MODE0			0x110
+#define QSERDES_V4_COM_VCO_TUNE2_MODE0			0x114
+#define QSERDES_V4_COM_VCO_TUNE1_MODE1			0x118
+#define QSERDES_V4_COM_VCO_TUNE2_MODE1			0x11c
 #define QSERDES_V4_COM_VCO_TUNE_INITVAL2		0x124
 #define QSERDES_V4_COM_HSCLK_SEL			0x158
 #define QSERDES_V4_COM_HSCLK_HS_SWITCH_SEL		0x15c
+#define QSERDES_V4_COM_CORECLK_DIV_MODE1		0x16c
+#define QSERDES_V4_COM_SVS_MODE_CLK_SEL			0x184
 #define QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE1_MODE0	0x1ac
 #define QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE2_MODE0	0x1b0
 #define QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE1_MODE1	0x1b4
@@ -341,12 +361,16 @@
 #define QSERDES_V4_COM_BIN_VCOCAL_CMP_CODE2_MODE1	0x1b8
 
 /* Only for QMP V4 PHY - TX registers */
+#define QSERDES_V4_TX_RES_CODE_LANE_TX			0x34
+#define QSERDES_V4_TX_RES_CODE_LANE_RX			0x38
 #define QSERDES_V4_TX_LANE_MODE_1			0x84
+#define QSERDES_V4_TX_RCV_DETECT_LVL_2			0x9c
 #define QSERDES_V4_TX_PWM_GEAR_1_DIVIDER_BAND0_1	0xd8
 #define QSERDES_V4_TX_PWM_GEAR_2_DIVIDER_BAND0_1	0xdC
 #define QSERDES_V4_TX_PWM_GEAR_3_DIVIDER_BAND0_1	0xe0
 #define QSERDES_V4_TX_PWM_GEAR_4_DIVIDER_BAND0_1	0xe4
 #define QSERDES_V4_TX_TRAN_DRVR_EMP_EN			0xb8
+#define QSERDES_V4_TX_PI_QEC_CTRL		0x104
 
 /* Only for QMP V4 PHY - RX registers */
 #define QSERDES_V4_RX_UCDR_FO_GAIN			0x008
@@ -354,17 +378,27 @@
 #define QSERDES_V4_RX_UCDR_FASTLOCK_FO_GAIN		0x030
 #define QSERDES_V4_RX_UCDR_SO_SATURATION_AND_ENABLE	0x034
 #define QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_LOW		0x03c
+#define QSERDES_V4_RX_UCDR_FASTLOCK_COUNT_HIGH		0x040
 #define QSERDES_V4_RX_UCDR_PI_CONTROLS			0x044
 #define QSERDES_V4_RX_UCDR_PI_CTRL2			0x048
+#define QSERDES_V4_RX_UCDR_SB2_THRESH1			0x04c
+#define QSERDES_V4_RX_UCDR_SB2_THRESH2			0x050
+#define QSERDES_V4_RX_UCDR_SB2_GAIN1			0x054
+#define QSERDES_V4_RX_UCDR_SB2_GAIN2			0x058
+#define QSERDES_V4_RX_AUX_DATA_TCOARSE_TFINE			0x060
 #define QSERDES_V4_RX_AC_JTAG_ENABLE			0x068
 #define QSERDES_V4_RX_AC_JTAG_MODE			0x078
 #define QSERDES_V4_RX_RX_TERM_BW			0x080
+#define QSERDES_V4_RX_VGA_CAL_CNTRL1			0x0d4
+#define QSERDES_V4_RX_VGA_CAL_CNTRL2			0x0d8
+#define QSERDES_V4_RX_GM_CAL				0x0dc
 #define QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL2		0x0ec
 #define QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL3		0x0f0
 #define QSERDES_V4_RX_RX_EQU_ADAPTOR_CNTRL4		0x0f4
 #define QSERDES_V4_RX_RX_IDAC_TSETTLE_LOW		0x0f8
 #define QSERDES_V4_RX_RX_IDAC_TSETTLE_HIGH		0x0fc
 #define QSERDES_V4_RX_RX_IDAC_MEASURE_TIME		0x100
+#define QSERDES_V4_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1	0x110
 #define QSERDES_V4_RX_RX_OFFSET_ADAPTOR_CNTRL2		0x114
 #define QSERDES_V4_RX_SIGDET_CNTRL			0x11c
 #define QSERDES_V4_RX_SIGDET_LVL			0x120
@@ -385,29 +419,32 @@
 #define QSERDES_V4_RX_RX_MODE_10_HIGH2			0x1a0
 #define QSERDES_V4_RX_RX_MODE_10_HIGH3			0x1a4
 #define QSERDES_V4_RX_RX_MODE_10_HIGH4			0x1a8
+#define QSERDES_V4_RX_DFE_EN_TIMER			0x1b4
+#define QSERDES_V4_RX_DFE_CTLE_POST_CAL_OFFSET		0x1b8
 #define QSERDES_V4_RX_DCC_CTRL1				0x1bc
+#define QSERDES_V4_RX_VTH_CODE				0x1c4
 
-/* Only for QMP V4 PHY - PCS registers */
-#define QPHY_V4_PHY_START				0x000
-#define QPHY_V4_POWER_DOWN_CONTROL			0x004
-#define QPHY_V4_SW_RESET				0x008
-#define QPHY_V4_TIMER_20US_CORECLK_STEPS_MSB		0x00c
-#define QPHY_V4_TIMER_20US_CORECLK_STEPS_LSB		0x010
-#define QPHY_V4_PLL_CNTL				0x02c
-#define QPHY_V4_TX_LARGE_AMP_DRV_LVL			0x030
-#define QPHY_V4_TX_SMALL_AMP_DRV_LVL			0x038
-#define QPHY_V4_BIST_FIXED_PAT_CTRL			0x060
-#define QPHY_V4_TX_HSGEAR_CAPABILITY			0x074
-#define QPHY_V4_RX_HSGEAR_CAPABILITY			0x0b4
-#define QPHY_V4_DEBUG_BUS_CLKSEL			0x124
-#define QPHY_V4_LINECFG_DISABLE				0x148
-#define QPHY_V4_RX_MIN_HIBERN8_TIME			0x150
-#define QPHY_V4_RX_SIGDET_CTRL2				0x158
-#define QPHY_V4_TX_PWM_GEAR_BAND			0x160
-#define QPHY_V4_TX_HS_GEAR_BAND				0x168
-#define QPHY_V4_PCS_READY_STATUS			0x180
-#define QPHY_V4_TX_MID_TERM_CTRL1			0x1d8
-#define QPHY_V4_MULTI_LANE_CTRL1			0x1e0
+/* Only for QMP V4 PHY - UFS PCS registers */
+#define QPHY_V4_PCS_UFS_PHY_START				0x000
+#define QPHY_V4_PCS_UFS_POWER_DOWN_CONTROL			0x004
+#define QPHY_V4_PCS_UFS_SW_RESET				0x008
+#define QPHY_V4_PCS_UFS_TIMER_20US_CORECLK_STEPS_MSB		0x00c
+#define QPHY_V4_PCS_UFS_TIMER_20US_CORECLK_STEPS_LSB		0x010
+#define QPHY_V4_PCS_UFS_PLL_CNTL				0x02c
+#define QPHY_V4_PCS_UFS_TX_LARGE_AMP_DRV_LVL			0x030
+#define QPHY_V4_PCS_UFS_TX_SMALL_AMP_DRV_LVL			0x038
+#define QPHY_V4_PCS_UFS_BIST_FIXED_PAT_CTRL			0x060
+#define QPHY_V4_PCS_UFS_TX_HSGEAR_CAPABILITY			0x074
+#define QPHY_V4_PCS_UFS_RX_HSGEAR_CAPABILITY			0x0b4
+#define QPHY_V4_PCS_UFS_DEBUG_BUS_CLKSEL			0x124
+#define QPHY_V4_PCS_UFS_LINECFG_DISABLE				0x148
+#define QPHY_V4_PCS_UFS_RX_MIN_HIBERN8_TIME			0x150
+#define QPHY_V4_PCS_UFS_RX_SIGDET_CTRL2				0x158
+#define QPHY_V4_PCS_UFS_TX_PWM_GEAR_BAND			0x160
+#define QPHY_V4_PCS_UFS_TX_HS_GEAR_BAND				0x168
+#define QPHY_V4_PCS_UFS_READY_STATUS			0x180
+#define QPHY_V4_PCS_UFS_TX_MID_TERM_CTRL1			0x1d8
+#define QPHY_V4_PCS_UFS_MULTI_LANE_CTRL1			0x1e0
 
 /* PCIE GEN3 COM registers */
 #define PCIE_GEN3_QHP_COM_SSC_EN_CENTER			0x14
@@ -523,4 +560,161 @@
 #define PCIE_GEN3_QHP_PHY_POWER_STATE_CONFIG5		0x16c
 #define PCIE_GEN3_QHP_PHY_PCS_TX_RX_CONFIG		0x174
 
+/* Only for QMP V4 PHY - USB/PCIe PCS registers */
+#define QPHY_V4_PCS_SW_RESET				0x000
+#define QPHY_V4_PCS_REVISION_ID0			0x004
+#define QPHY_V4_PCS_REVISION_ID1			0x008
+#define QPHY_V4_PCS_REVISION_ID2			0x00c
+#define QPHY_V4_PCS_REVISION_ID3			0x010
+#define QPHY_V4_PCS_PCS_STATUS1				0x014
+#define QPHY_V4_PCS_PCS_STATUS2				0x018
+#define QPHY_V4_PCS_PCS_STATUS3				0x01c
+#define QPHY_V4_PCS_PCS_STATUS4				0x020
+#define QPHY_V4_PCS_PCS_STATUS5				0x024
+#define QPHY_V4_PCS_PCS_STATUS6				0x028
+#define QPHY_V4_PCS_PCS_STATUS7				0x02c
+#define QPHY_V4_PCS_DEBUG_BUS_0_STATUS			0x030
+#define QPHY_V4_PCS_DEBUG_BUS_1_STATUS			0x034
+#define QPHY_V4_PCS_DEBUG_BUS_2_STATUS			0x038
+#define QPHY_V4_PCS_DEBUG_BUS_3_STATUS			0x03c
+#define QPHY_V4_PCS_POWER_DOWN_CONTROL			0x040
+#define QPHY_V4_PCS_START_CONTROL			0x044
+#define QPHY_V4_PCS_INSIG_SW_CTRL1			0x048
+#define QPHY_V4_PCS_INSIG_SW_CTRL2			0x04c
+#define QPHY_V4_PCS_INSIG_SW_CTRL3			0x050
+#define QPHY_V4_PCS_INSIG_SW_CTRL4			0x054
+#define QPHY_V4_PCS_INSIG_SW_CTRL5			0x058
+#define QPHY_V4_PCS_INSIG_SW_CTRL6			0x05c
+#define QPHY_V4_PCS_INSIG_SW_CTRL7			0x060
+#define QPHY_V4_PCS_INSIG_SW_CTRL8			0x064
+#define QPHY_V4_PCS_INSIG_MX_CTRL1			0x068
+#define QPHY_V4_PCS_INSIG_MX_CTRL2			0x06c
+#define QPHY_V4_PCS_INSIG_MX_CTRL3			0x070
+#define QPHY_V4_PCS_INSIG_MX_CTRL4			0x074
+#define QPHY_V4_PCS_INSIG_MX_CTRL5			0x078
+#define QPHY_V4_PCS_INSIG_MX_CTRL7			0x07c
+#define QPHY_V4_PCS_INSIG_MX_CTRL8			0x080
+#define QPHY_V4_PCS_OUTSIG_SW_CTRL1			0x084
+#define QPHY_V4_PCS_OUTSIG_MX_CTRL1			0x088
+#define QPHY_V4_PCS_CLAMP_ENABLE			0x08c
+#define QPHY_V4_PCS_POWER_STATE_CONFIG1			0x090
+#define QPHY_V4_PCS_POWER_STATE_CONFIG2			0x094
+#define QPHY_V4_PCS_FLL_CNTRL1				0x098
+#define QPHY_V4_PCS_FLL_CNTRL2				0x09c
+#define QPHY_V4_PCS_FLL_CNT_VAL_L			0x0a0
+#define QPHY_V4_PCS_FLL_CNT_VAL_H_TOL			0x0a4
+#define QPHY_V4_PCS_FLL_MAN_CODE			0x0a8
+#define QPHY_V4_PCS_TEST_CONTROL1			0x0ac
+#define QPHY_V4_PCS_TEST_CONTROL2			0x0b0
+#define QPHY_V4_PCS_TEST_CONTROL3			0x0b4
+#define QPHY_V4_PCS_TEST_CONTROL4			0x0b8
+#define QPHY_V4_PCS_TEST_CONTROL5			0x0bc
+#define QPHY_V4_PCS_TEST_CONTROL6			0x0c0
+#define QPHY_V4_PCS_LOCK_DETECT_CONFIG1			0x0c4
+#define QPHY_V4_PCS_LOCK_DETECT_CONFIG2			0x0c8
+#define QPHY_V4_PCS_LOCK_DETECT_CONFIG3			0x0cc
+#define QPHY_V4_PCS_LOCK_DETECT_CONFIG4			0x0d0
+#define QPHY_V4_PCS_LOCK_DETECT_CONFIG5			0x0d4
+#define QPHY_V4_PCS_LOCK_DETECT_CONFIG6			0x0d8
+#define QPHY_V4_PCS_REFGEN_REQ_CONFIG1			0x0dc
+#define QPHY_V4_PCS_REFGEN_REQ_CONFIG2			0x0e0
+#define QPHY_V4_PCS_REFGEN_REQ_CONFIG3			0x0e4
+#define QPHY_V4_PCS_BIST_CTRL				0x0e8
+#define QPHY_V4_PCS_PRBS_POLY0				0x0ec
+#define QPHY_V4_PCS_PRBS_POLY1				0x0f0
+#define QPHY_V4_PCS_FIXED_PAT0				0x0f4
+#define QPHY_V4_PCS_FIXED_PAT1				0x0f8
+#define QPHY_V4_PCS_FIXED_PAT2				0x0fc
+#define QPHY_V4_PCS_FIXED_PAT3				0x100
+#define QPHY_V4_PCS_FIXED_PAT4				0x104
+#define QPHY_V4_PCS_FIXED_PAT5				0x108
+#define QPHY_V4_PCS_FIXED_PAT6				0x10c
+#define QPHY_V4_PCS_FIXED_PAT7				0x110
+#define QPHY_V4_PCS_FIXED_PAT8				0x114
+#define QPHY_V4_PCS_FIXED_PAT9				0x118
+#define QPHY_V4_PCS_FIXED_PAT10				0x11c
+#define QPHY_V4_PCS_FIXED_PAT11				0x120
+#define QPHY_V4_PCS_FIXED_PAT12				0x124
+#define QPHY_V4_PCS_FIXED_PAT13				0x128
+#define QPHY_V4_PCS_FIXED_PAT14				0x12c
+#define QPHY_V4_PCS_FIXED_PAT15				0x130
+#define QPHY_V4_PCS_TXMGN_CONFIG			0x134
+#define QPHY_V4_PCS_G12S1_TXMGN_V0			0x138
+#define QPHY_V4_PCS_G12S1_TXMGN_V1			0x13c
+#define QPHY_V4_PCS_G12S1_TXMGN_V2			0x140
+#define QPHY_V4_PCS_G12S1_TXMGN_V3			0x144
+#define QPHY_V4_PCS_G12S1_TXMGN_V4			0x148
+#define QPHY_V4_PCS_G12S1_TXMGN_V0_RS			0x14c
+#define QPHY_V4_PCS_G12S1_TXMGN_V1_RS			0x150
+#define QPHY_V4_PCS_G12S1_TXMGN_V2_RS			0x154
+#define QPHY_V4_PCS_G12S1_TXMGN_V3_RS			0x158
+#define QPHY_V4_PCS_G12S1_TXMGN_V4_RS			0x15c
+#define QPHY_V4_PCS_G3S2_TXMGN_MAIN			0x160
+#define QPHY_V4_PCS_G3S2_TXMGN_MAIN_RS			0x164
+#define QPHY_V4_PCS_G12S1_TXDEEMPH_M6DB			0x168
+#define QPHY_V4_PCS_G12S1_TXDEEMPH_M3P5DB		0x16c
+#define QPHY_V4_PCS_G3S2_PRE_GAIN			0x170
+#define QPHY_V4_PCS_G3S2_POST_GAIN			0x174
+#define QPHY_V4_PCS_G3S2_PRE_POST_OFFSET		0x178
+#define QPHY_V4_PCS_G3S2_PRE_GAIN_RS			0x17c
+#define QPHY_V4_PCS_G3S2_POST_GAIN_RS			0x180
+#define QPHY_V4_PCS_G3S2_PRE_POST_OFFSET_RS		0x184
+#define QPHY_V4_PCS_RX_SIGDET_LVL			0x188
+#define QPHY_V4_PCS_RX_SIGDET_DTCT_CNTRL		0x18c
+#define QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_L		0x190
+#define QPHY_V4_PCS_RCVR_DTCT_DLY_P1U2_H		0x194
+#define QPHY_V4_PCS_RATE_SLEW_CNTRL1			0x198
+#define QPHY_V4_PCS_RATE_SLEW_CNTRL2			0x19c
+#define QPHY_V4_PCS_PWRUP_RESET_DLY_TIME_AUXCLK		0x1a0
+#define QPHY_V4_PCS_P2U3_WAKEUP_DLY_TIME_AUXCLK_L	0x1a4
+#define QPHY_V4_PCS_P2U3_WAKEUP_DLY_TIME_AUXCLK_H	0x1a8
+#define QPHY_V4_PCS_TSYNC_RSYNC_TIME			0x1ac
+#define QPHY_V4_PCS_CDR_RESET_TIME			0x1b0
+#define QPHY_V4_PCS_TSYNC_DLY_TIME			0x1b4
+#define QPHY_V4_PCS_ELECIDLE_DLY_SEL			0x1b8
+#define QPHY_V4_PCS_CMN_ACK_OUT_SEL			0x1bc
+#define QPHY_V4_PCS_ALIGN_DETECT_CONFIG1		0x1c0
+#define QPHY_V4_PCS_ALIGN_DETECT_CONFIG2		0x1c4
+#define QPHY_V4_PCS_ALIGN_DETECT_CONFIG3		0x1c8
+#define QPHY_V4_PCS_ALIGN_DETECT_CONFIG4		0x1cc
+#define QPHY_V4_PCS_PCS_TX_RX_CONFIG			0x1d0
+#define QPHY_V4_PCS_RX_IDLE_DTCT_CNTRL			0x1d4
+#define QPHY_V4_PCS_RX_DCC_CAL_CONFIG			0x1d8
+#define QPHY_V4_PCS_EQ_CONFIG1				0x1dc
+#define QPHY_V4_PCS_EQ_CONFIG2				0x1e0
+#define QPHY_V4_PCS_EQ_CONFIG3				0x1e4
+#define QPHY_V4_PCS_EQ_CONFIG4				0x1e8
+#define QPHY_V4_PCS_EQ_CONFIG5				0x1ec
+#define QPHY_V4_PCS_USB3_POWER_STATE_CONFIG1		0x300
+#define QPHY_V4_PCS_USB3_AUTONOMOUS_MODE_STATUS		0x304
+#define QPHY_V4_PCS_USB3_AUTONOMOUS_MODE_CTRL		0x308
+#define QPHY_V4_PCS_USB3_AUTONOMOUS_MODE_CTRL2		0x30c
+#define QPHY_V4_PCS_USB3_LFPS_RXTERM_IRQ_SOURCE_STATUS	0x310
+#define QPHY_V4_PCS_USB3_LFPS_RXTERM_IRQ_CLEAR		0x314
+#define QPHY_V4_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL	0x318
+#define QPHY_V4_PCS_USB3_LFPS_TX_ECSTART		0x31c
+#define QPHY_V4_PCS_USB3_LFPS_PER_TIMER_VAL		0x320
+#define QPHY_V4_PCS_USB3_LFPS_TX_END_CNT_U3_START	0x324
+#define QPHY_V4_PCS_USB3_RXEQTRAINING_LOCK_TIME		0x328
+#define QPHY_V4_PCS_USB3_RXEQTRAINING_WAIT_TIME		0x32c
+#define QPHY_V4_PCS_USB3_RXEQTRAINING_CTLE_TIME		0x330
+#define QPHY_V4_PCS_USB3_RXEQTRAINING_WAIT_TIME_S2	0x334
+#define QPHY_V4_PCS_USB3_RXEQTRAINING_DFE_TIME_S2	0x338
+#define QPHY_V4_PCS_USB3_RCVR_DTCT_DLY_U3_L		0x33c
+#define QPHY_V4_PCS_USB3_RCVR_DTCT_DLY_U3_H		0x340
+#define QPHY_V4_PCS_USB3_ARCVR_DTCT_EN_PERIOD		0x344
+#define QPHY_V4_PCS_USB3_ARCVR_DTCT_CM_DLY		0x348
+#define QPHY_V4_PCS_USB3_TXONESZEROS_RUN_LENGTH		0x34c
+#define QPHY_V4_PCS_USB3_ALFPS_DEGLITCH_VAL		0x350
+#define QPHY_V4_PCS_USB3_SIGDET_STARTUP_TIMER_VAL	0x354
+#define QPHY_V4_PCS_USB3_TEST_CONTROL			0x358
+
+/* Only for QMP V4 PHY - PCS_MISC registers */
+#define QPHY_V4_PCS_MISC_TYPEC_CTRL			0x00
+#define QPHY_V4_PCS_MISC_TYPEC_PWRDN_CTRL		0x04
+#define QPHY_V4_PCS_MISC_PCS_MISC_CONFIG1		0x08
+#define QPHY_V4_PCS_MISC_CLAMP_ENABLE			0x0c
+#define QPHY_V4_PCS_MISC_TYPEC_STATUS			0x10
+#define QPHY_V4_PCS_MISC_PLACEHOLDER_STATUS		0x14
+
 #endif
diff --git a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c
new file mode 100644
index 000000000000..4d74045271eb
--- /dev/null
+++ b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c
@@ -0,0 +1,287 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+
+#define USB2_PHY_USB_PHY_UTMI_CTRL0		(0x3c)
+#define SLEEPM					BIT(0)
+#define OPMODE_MASK				GENMASK(4, 3)
+#define OPMODE_NORMAL				(0x00)
+#define OPMODE_NONDRIVING			BIT(3)
+#define TERMSEL					BIT(5)
+
+#define USB2_PHY_USB_PHY_UTMI_CTRL1		(0x40)
+#define XCVRSEL					BIT(0)
+
+#define USB2_PHY_USB_PHY_UTMI_CTRL5		(0x50)
+#define POR					BIT(1)
+
+#define USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON0	(0x54)
+#define RETENABLEN				BIT(3)
+#define FSEL_MASK				GENMASK(7, 5)
+#define FSEL_DEFAULT				(0x3 << 4)
+
+#define USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON1	(0x58)
+#define VBUSVLDEXTSEL0				BIT(4)
+#define PLLBTUNE				BIT(5)
+
+#define USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON2	(0x5c)
+#define VREGBYPASS				BIT(0)
+
+#define USB2_PHY_USB_PHY_HS_PHY_CTRL1		(0x60)
+#define VBUSVLDEXT0				BIT(0)
+
+#define USB2_PHY_USB_PHY_HS_PHY_CTRL2		(0x64)
+#define USB2_AUTO_RESUME			BIT(0)
+#define USB2_SUSPEND_N				BIT(2)
+#define USB2_SUSPEND_N_SEL			BIT(3)
+
+#define USB2_PHY_USB_PHY_CFG0			(0x94)
+#define UTMI_PHY_DATAPATH_CTRL_OVERRIDE_EN	BIT(0)
+#define UTMI_PHY_CMN_CTRL_OVERRIDE_EN		BIT(1)
+
+#define USB2_PHY_USB_PHY_REFCLK_CTRL		(0xa0)
+#define REFCLK_SEL_MASK				GENMASK(1, 0)
+#define REFCLK_SEL_DEFAULT			(0x2 << 0)
+
+static const char * const qcom_snps_hsphy_vreg_names[] = {
+	"vdda-pll", "vdda33", "vdda18",
+};
+
+#define SNPS_HS_NUM_VREGS		ARRAY_SIZE(qcom_snps_hsphy_vreg_names)
+
+/**
+ * struct qcom_snps_hsphy - snps hs phy attributes
+ *
+ * @phy: generic phy
+ * @base: iomapped memory space for snps hs phy
+ *
+ * @cfg_ahb_clk: AHB2PHY interface clock
+ * @ref_clk: phy reference clock
+ * @iface_clk: phy interface clock
+ * @phy_reset: phy reset control
+ * @vregs: regulator supplies bulk data
+ * @phy_initialized: if PHY has been initialized correctly
+ */
+struct qcom_snps_hsphy {
+	struct phy *phy;
+	void __iomem *base;
+
+	struct clk *cfg_ahb_clk;
+	struct clk *ref_clk;
+	struct reset_control *phy_reset;
+	struct regulator_bulk_data vregs[SNPS_HS_NUM_VREGS];
+
+	bool phy_initialized;
+};
+
+static inline void qcom_snps_hsphy_write_mask(void __iomem *base, u32 offset,
+						u32 mask, u32 val)
+{
+	u32 reg;
+
+	reg = readl_relaxed(base + offset);
+	reg &= ~mask;
+	reg |= val & mask;
+	writel_relaxed(reg, base + offset);
+
+	/* Ensure above write is completed */
+	readl_relaxed(base + offset);
+}
+
+static int qcom_snps_hsphy_init(struct phy *phy)
+{
+	struct qcom_snps_hsphy *hsphy = phy_get_drvdata(phy);
+	int ret;
+
+	dev_vdbg(&phy->dev, "%s(): Initializing SNPS HS phy\n", __func__);
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(hsphy->vregs), hsphy->vregs);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(hsphy->cfg_ahb_clk);
+	if (ret) {
+		dev_err(&phy->dev, "failed to enable cfg ahb clock, %d\n", ret);
+		goto poweroff_phy;
+	}
+
+	ret = reset_control_assert(hsphy->phy_reset);
+	if (ret) {
+		dev_err(&phy->dev, "failed to assert phy_reset, %d\n", ret);
+		goto disable_ahb_clk;
+	}
+
+	usleep_range(100, 150);
+
+	ret = reset_control_deassert(hsphy->phy_reset);
+	if (ret) {
+		dev_err(&phy->dev, "failed to de-assert phy_reset, %d\n", ret);
+		goto disable_ahb_clk;
+	}
+
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_CFG0,
+					UTMI_PHY_CMN_CTRL_OVERRIDE_EN,
+					UTMI_PHY_CMN_CTRL_OVERRIDE_EN);
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_UTMI_CTRL5,
+							POR, POR);
+	qcom_snps_hsphy_write_mask(hsphy->base,
+					USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON0,
+					FSEL_MASK, 0);
+	qcom_snps_hsphy_write_mask(hsphy->base,
+					USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON1,
+					PLLBTUNE, PLLBTUNE);
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_REFCLK_CTRL,
+					REFCLK_SEL_DEFAULT, REFCLK_SEL_MASK);
+	qcom_snps_hsphy_write_mask(hsphy->base,
+					USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON1,
+					VBUSVLDEXTSEL0, VBUSVLDEXTSEL0);
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_HS_PHY_CTRL1,
+					VBUSVLDEXT0, VBUSVLDEXT0);
+
+	qcom_snps_hsphy_write_mask(hsphy->base,
+					USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON2,
+					VREGBYPASS, VREGBYPASS);
+
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_HS_PHY_CTRL2,
+					USB2_SUSPEND_N_SEL | USB2_SUSPEND_N,
+					USB2_SUSPEND_N_SEL | USB2_SUSPEND_N);
+
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_UTMI_CTRL0,
+					SLEEPM, SLEEPM);
+
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_UTMI_CTRL5,
+					POR, 0);
+
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_HS_PHY_CTRL2,
+					USB2_SUSPEND_N_SEL, 0);
+
+	qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_CFG0,
+					UTMI_PHY_CMN_CTRL_OVERRIDE_EN, 0);
+
+	hsphy->phy_initialized = true;
+
+	return 0;
+
+disable_ahb_clk:
+	clk_disable_unprepare(hsphy->cfg_ahb_clk);
+poweroff_phy:
+	regulator_bulk_disable(ARRAY_SIZE(hsphy->vregs), hsphy->vregs);
+
+	return ret;
+}
+
+static int qcom_snps_hsphy_exit(struct phy *phy)
+{
+	struct qcom_snps_hsphy *hsphy = phy_get_drvdata(phy);
+
+	reset_control_assert(hsphy->phy_reset);
+	clk_disable_unprepare(hsphy->cfg_ahb_clk);
+	regulator_bulk_disable(ARRAY_SIZE(hsphy->vregs), hsphy->vregs);
+	hsphy->phy_initialized = false;
+
+	return 0;
+}
+
+static const struct phy_ops qcom_snps_hsphy_gen_ops = {
+	.init		= qcom_snps_hsphy_init,
+	.exit		= qcom_snps_hsphy_exit,
+	.owner		= THIS_MODULE,
+};
+
+static const struct of_device_id qcom_snps_hsphy_of_match_table[] = {
+	{ .compatible	= "qcom,sm8150-usb-hs-phy", },
+	{ .compatible	= "qcom,usb-snps-hs-7nm-phy", },
+	{ .compatible	= "qcom,usb-snps-femto-v2-phy",	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qcom_snps_hsphy_of_match_table);
+
+static int qcom_snps_hsphy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct qcom_snps_hsphy *hsphy;
+	struct phy_provider *phy_provider;
+	struct phy *generic_phy;
+	int ret, i;
+	int num;
+
+	hsphy = devm_kzalloc(dev, sizeof(*hsphy), GFP_KERNEL);
+	if (!hsphy)
+		return -ENOMEM;
+
+	hsphy->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(hsphy->base))
+		return PTR_ERR(hsphy->base);
+
+	hsphy->ref_clk = devm_clk_get(dev, "ref");
+	if (IS_ERR(hsphy->ref_clk)) {
+		ret = PTR_ERR(hsphy->ref_clk);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to get ref clk, %d\n", ret);
+		return ret;
+	}
+
+	hsphy->phy_reset = devm_reset_control_get_exclusive(&pdev->dev, NULL);
+	if (IS_ERR(hsphy->phy_reset)) {
+		dev_err(dev, "failed to get phy core reset\n");
+		return PTR_ERR(hsphy->phy_reset);
+	}
+
+	num = ARRAY_SIZE(hsphy->vregs);
+	for (i = 0; i < num; i++)
+		hsphy->vregs[i].supply = qcom_snps_hsphy_vreg_names[i];
+
+	ret = devm_regulator_bulk_get(dev, num, hsphy->vregs);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to get regulator supplies: %d\n",
+				ret);
+		return ret;
+	}
+
+	generic_phy = devm_phy_create(dev, NULL, &qcom_snps_hsphy_gen_ops);
+	if (IS_ERR(generic_phy)) {
+		ret = PTR_ERR(generic_phy);
+		dev_err(dev, "failed to create phy, %d\n", ret);
+		return ret;
+	}
+	hsphy->phy = generic_phy;
+
+	dev_set_drvdata(dev, hsphy);
+	phy_set_drvdata(generic_phy, hsphy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (!IS_ERR(phy_provider))
+		dev_dbg(dev, "Registered Qcom-SNPS HS phy\n");
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static struct platform_driver qcom_snps_hsphy_driver = {
+	.probe		= qcom_snps_hsphy_probe,
+	.driver = {
+		.name	= "qcom-snps-hs-femto-v2-phy",
+		.of_match_table = qcom_snps_hsphy_of_match_table,
+	},
+};
+
+module_platform_driver(qcom_snps_hsphy_driver);
+
+MODULE_DESCRIPTION("Qualcomm SNPS FEMTO USB HS PHY V2 driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/samsung/phy-s5pv210-usb2.c b/drivers/phy/samsung/phy-s5pv210-usb2.c
index 56a5083fe6f9..32be62e49804 100644
--- a/drivers/phy/samsung/phy-s5pv210-usb2.c
+++ b/drivers/phy/samsung/phy-s5pv210-usb2.c
@@ -139,6 +139,10 @@ static void s5pv210_phy_pwr(struct samsung_usb2_phy_instance *inst, bool on)
 		udelay(10);
 		rst &= ~rstbits;
 		writel(rst, drv->reg_phy + S5PV210_UPHYRST);
+		/* The following delay is necessary for the reset sequence to be
+		 * completed
+		 */
+		udelay(80);
 	} else {
 		pwr = readl(drv->reg_phy + S5PV210_UPHYPWR);
 		pwr |= phypwr;
diff --git a/drivers/phy/samsung/phy-samsung-usb2.h b/drivers/phy/samsung/phy-samsung-usb2.h
index 2c1a7d71142b..77fb23bc218f 100644
--- a/drivers/phy/samsung/phy-samsung-usb2.h
+++ b/drivers/phy/samsung/phy-samsung-usb2.h
@@ -43,7 +43,7 @@ struct samsung_usb2_phy_driver {
 	struct regmap *reg_pmu;
 	struct regmap *reg_sys;
 	spinlock_t lock;
-	struct samsung_usb2_phy_instance instances[0];
+	struct samsung_usb2_phy_instance instances[];
 };
 
 struct samsung_usb2_common_phy {
diff --git a/drivers/phy/ti/phy-am654-serdes.c b/drivers/phy/ti/phy-am654-serdes.c
index 88a047b9fa6f..0a166d5a6414 100644
--- a/drivers/phy/ti/phy-am654-serdes.c
+++ b/drivers/phy/ti/phy-am654-serdes.c
@@ -77,6 +77,7 @@ static struct regmap_config serdes_am654_regmap_config = {
 	.val_bits = 32,
 	.reg_stride = 4,
 	.fast_io = true,
+	.max_register = 0x1ffc,
 };
 
 static const struct reg_field cmu_master_cdn_o = REG_FIELD(CMU_R07C, 24, 24);
@@ -200,9 +201,91 @@ static int serdes_am654_power_off(struct phy *x)
 	return 0;
 }
 
-static int serdes_am654_init(struct phy *x)
+#define SERDES_AM654_CFG(offset, a, b, val) \
+	regmap_update_bits(phy->regmap, (offset),\
+			   GENMASK((a), (b)), (val) << (b))
+
+static int serdes_am654_usb3_init(struct serdes_am654 *phy)
+{
+	SERDES_AM654_CFG(0x0000, 31, 24, 0x17);
+	SERDES_AM654_CFG(0x0004, 15, 8, 0x02);
+	SERDES_AM654_CFG(0x0004, 7, 0, 0x0e);
+	SERDES_AM654_CFG(0x0008, 23, 16, 0x2e);
+	SERDES_AM654_CFG(0x0008, 31, 24, 0x2e);
+	SERDES_AM654_CFG(0x0060, 7, 0, 0x4b);
+	SERDES_AM654_CFG(0x0060, 15, 8, 0x98);
+	SERDES_AM654_CFG(0x0060, 23, 16, 0x60);
+	SERDES_AM654_CFG(0x00d0, 31, 24, 0x45);
+	SERDES_AM654_CFG(0x00e8, 15, 8, 0x0e);
+	SERDES_AM654_CFG(0x0220, 7, 0, 0x34);
+	SERDES_AM654_CFG(0x0220, 15, 8, 0x34);
+	SERDES_AM654_CFG(0x0220, 31, 24, 0x37);
+	SERDES_AM654_CFG(0x0224, 7, 0, 0x37);
+	SERDES_AM654_CFG(0x0224, 15, 8, 0x37);
+	SERDES_AM654_CFG(0x0228, 23, 16, 0x37);
+	SERDES_AM654_CFG(0x0228, 31, 24, 0x37);
+	SERDES_AM654_CFG(0x022c, 7, 0, 0x37);
+	SERDES_AM654_CFG(0x022c, 15, 8, 0x37);
+	SERDES_AM654_CFG(0x0230, 15, 8, 0x2a);
+	SERDES_AM654_CFG(0x0230, 23, 16, 0x2a);
+	SERDES_AM654_CFG(0x0240, 23, 16, 0x10);
+	SERDES_AM654_CFG(0x0240, 31, 24, 0x34);
+	SERDES_AM654_CFG(0x0244, 7, 0, 0x40);
+	SERDES_AM654_CFG(0x0244, 23, 16, 0x34);
+	SERDES_AM654_CFG(0x0248, 15, 8, 0x0d);
+	SERDES_AM654_CFG(0x0258, 15, 8, 0x16);
+	SERDES_AM654_CFG(0x0258, 23, 16, 0x84);
+	SERDES_AM654_CFG(0x0258, 31, 24, 0xf2);
+	SERDES_AM654_CFG(0x025c, 7, 0, 0x21);
+	SERDES_AM654_CFG(0x0260, 7, 0, 0x27);
+	SERDES_AM654_CFG(0x0260, 15, 8, 0x04);
+	SERDES_AM654_CFG(0x0268, 15, 8, 0x04);
+	SERDES_AM654_CFG(0x0288, 15, 8, 0x2c);
+	SERDES_AM654_CFG(0x0330, 31, 24, 0xa0);
+	SERDES_AM654_CFG(0x0338, 23, 16, 0x03);
+	SERDES_AM654_CFG(0x0338, 31, 24, 0x00);
+	SERDES_AM654_CFG(0x033c, 7, 0, 0x00);
+	SERDES_AM654_CFG(0x0344, 31, 24, 0x18);
+	SERDES_AM654_CFG(0x034c, 7, 0, 0x18);
+	SERDES_AM654_CFG(0x039c, 23, 16, 0x3b);
+	SERDES_AM654_CFG(0x0a04, 7, 0, 0x03);
+	SERDES_AM654_CFG(0x0a14, 31, 24, 0x3c);
+	SERDES_AM654_CFG(0x0a18, 15, 8, 0x3c);
+	SERDES_AM654_CFG(0x0a38, 7, 0, 0x3e);
+	SERDES_AM654_CFG(0x0a38, 15, 8, 0x3e);
+	SERDES_AM654_CFG(0x0ae0, 7, 0, 0x07);
+	SERDES_AM654_CFG(0x0b6c, 23, 16, 0xcd);
+	SERDES_AM654_CFG(0x0b6c, 31, 24, 0x04);
+	SERDES_AM654_CFG(0x0b98, 23, 16, 0x03);
+	SERDES_AM654_CFG(0x1400, 7, 0, 0x3f);
+	SERDES_AM654_CFG(0x1404, 23, 16, 0x6f);
+	SERDES_AM654_CFG(0x1404, 31, 24, 0x6f);
+	SERDES_AM654_CFG(0x140c, 7, 0, 0x6f);
+	SERDES_AM654_CFG(0x140c, 15, 8, 0x6f);
+	SERDES_AM654_CFG(0x1410, 15, 8, 0x27);
+	SERDES_AM654_CFG(0x1414, 7, 0, 0x0c);
+	SERDES_AM654_CFG(0x1414, 23, 16, 0x07);
+	SERDES_AM654_CFG(0x1418, 23, 16, 0x40);
+	SERDES_AM654_CFG(0x141c, 7, 0, 0x00);
+	SERDES_AM654_CFG(0x141c, 15, 8, 0x1f);
+	SERDES_AM654_CFG(0x1428, 31, 24, 0x08);
+	SERDES_AM654_CFG(0x1434, 31, 24, 0x00);
+	SERDES_AM654_CFG(0x1444, 7, 0, 0x94);
+	SERDES_AM654_CFG(0x1460, 31, 24, 0x7f);
+	SERDES_AM654_CFG(0x1464, 7, 0, 0x43);
+	SERDES_AM654_CFG(0x1464, 23, 16, 0x6f);
+	SERDES_AM654_CFG(0x1464, 31, 24, 0x43);
+	SERDES_AM654_CFG(0x1484, 23, 16, 0x8f);
+	SERDES_AM654_CFG(0x1498, 7, 0, 0x4f);
+	SERDES_AM654_CFG(0x1498, 23, 16, 0x4f);
+	SERDES_AM654_CFG(0x007c, 31, 24, 0x0d);
+	SERDES_AM654_CFG(0x0b90, 15, 8, 0x0f);
+
+	return 0;
+}
+
+static int serdes_am654_pcie_init(struct serdes_am654 *phy)
 {
-	struct serdes_am654 *phy = phy_get_drvdata(x);
 	int ret;
 
 	ret = regmap_field_write(phy->config_version, VERSION);
@@ -220,11 +303,28 @@ static int serdes_am654_init(struct phy *x)
 	return 0;
 }
 
+static int serdes_am654_init(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+
+	switch (phy->type) {
+	case PHY_TYPE_PCIE:
+		return serdes_am654_pcie_init(phy);
+	case PHY_TYPE_USB3:
+		return serdes_am654_usb3_init(phy);
+	default:
+		return -EINVAL;
+	}
+}
+
 static int serdes_am654_reset(struct phy *x)
 {
 	struct serdes_am654 *phy = phy_get_drvdata(x);
 	int ret;
 
+	serdes_am654_disable_pll(phy);
+	serdes_am654_disable_txrx(phy);
+
 	ret = regmap_field_write(phy->por_en, 0x1);
 	if (ret)
 		return ret;
diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c
index 7b51045df783..30ea5b207285 100644
--- a/drivers/phy/ti/phy-j721e-wiz.c
+++ b/drivers/phy/ti/phy-j721e-wiz.c
@@ -20,6 +20,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/reset-controller.h>
+#include <dt-bindings/phy/phy.h>
 
 #define WIZ_SERDES_CTRL		0x404
 #define WIZ_SERDES_TOP_CTRL	0x408
@@ -78,6 +79,8 @@ static const struct reg_field p_enable[WIZ_MAX_LANES] = {
 	REG_FIELD(WIZ_LANECTL(3), 30, 31),
 };
 
+enum p_enable { P_ENABLE = 2, P_ENABLE_FORCE = 1, P_ENABLE_DISABLE = 0 };
+
 static const struct reg_field p_align[WIZ_MAX_LANES] = {
 	REG_FIELD(WIZ_LANECTL(0), 29, 29),
 	REG_FIELD(WIZ_LANECTL(1), 29, 29),
@@ -220,6 +223,7 @@ struct wiz {
 	struct reset_controller_dev wiz_phy_reset_dev;
 	struct gpio_desc	*gpio_typec_dir;
 	int			typec_dir_delay;
+	u32 lane_phy_type[WIZ_MAX_LANES];
 };
 
 static int wiz_reset(struct wiz *wiz)
@@ -242,12 +246,17 @@ static int wiz_reset(struct wiz *wiz)
 static int wiz_mode_select(struct wiz *wiz)
 {
 	u32 num_lanes = wiz->num_lanes;
+	enum wiz_lane_standard_mode mode;
 	int ret;
 	int i;
 
 	for (i = 0; i < num_lanes; i++) {
-		ret = regmap_field_write(wiz->p_standard_mode[i],
-					 LANE_MODE_GEN4);
+		if (wiz->lane_phy_type[i] == PHY_TYPE_DP)
+			mode = LANE_MODE_GEN1;
+		else
+			mode = LANE_MODE_GEN4;
+
+		ret = regmap_field_write(wiz->p_standard_mode[i], mode);
 		if (ret)
 			return ret;
 	}
@@ -707,7 +716,7 @@ static int wiz_phy_reset_assert(struct reset_controller_dev *rcdev,
 		return ret;
 	}
 
-	ret = regmap_field_write(wiz->p_enable[id - 1], false);
+	ret = regmap_field_write(wiz->p_enable[id - 1], P_ENABLE_DISABLE);
 	return ret;
 }
 
@@ -734,7 +743,11 @@ static int wiz_phy_reset_deassert(struct reset_controller_dev *rcdev,
 		return ret;
 	}
 
-	ret = regmap_field_write(wiz->p_enable[id - 1], true);
+	if (wiz->lane_phy_type[id - 1] == PHY_TYPE_DP)
+		ret = regmap_field_write(wiz->p_enable[id - 1], P_ENABLE);
+	else
+		ret = regmap_field_write(wiz->p_enable[id - 1], P_ENABLE_FORCE);
+
 	return ret;
 }
 
@@ -761,6 +774,40 @@ static const struct of_device_id wiz_id_table[] = {
 };
 MODULE_DEVICE_TABLE(of, wiz_id_table);
 
+static int wiz_get_lane_phy_types(struct device *dev, struct wiz *wiz)
+{
+	struct device_node *serdes, *subnode;
+
+	serdes = of_get_child_by_name(dev->of_node, "serdes");
+	if (!serdes) {
+		dev_err(dev, "%s: Getting \"serdes\"-node failed\n", __func__);
+		return -EINVAL;
+	}
+
+	for_each_child_of_node(serdes, subnode) {
+		u32 reg, num_lanes = 1, phy_type = PHY_NONE;
+		int ret, i;
+
+		ret = of_property_read_u32(subnode, "reg", &reg);
+		if (ret) {
+			dev_err(dev,
+				"%s: Reading \"reg\" from \"%s\" failed: %d\n",
+				__func__, subnode->name, ret);
+			return ret;
+		}
+		of_property_read_u32(subnode, "cdns,num-lanes", &num_lanes);
+		of_property_read_u32(subnode, "cdns,phy-type", &phy_type);
+
+		dev_dbg(dev, "%s: Lanes %u-%u have phy-type %u\n", __func__,
+			reg, reg + num_lanes - 1, phy_type);
+
+		for (i = reg; i < reg + num_lanes; i++)
+			wiz->lane_phy_type[i] = phy_type;
+	}
+
+	return 0;
+}
+
 static int wiz_probe(struct platform_device *pdev)
 {
 	struct reset_controller_dev *phy_reset_dev;
@@ -794,8 +841,10 @@ static int wiz_probe(struct platform_device *pdev)
 	}
 
 	base = devm_ioremap(dev, res.start, resource_size(&res));
-	if (!base)
+	if (!base) {
+		ret = -ENOMEM;
 		goto err_addr_to_resource;
+	}
 
 	regmap = devm_regmap_init_mmio(dev, base, &wiz_regmap_config);
 	if (IS_ERR(regmap)) {
@@ -812,6 +861,7 @@ static int wiz_probe(struct platform_device *pdev)
 
 	if (num_lanes > WIZ_MAX_LANES) {
 		dev_err(dev, "Cannot support %d lanes\n", num_lanes);
+		ret = -ENODEV;
 		goto err_addr_to_resource;
 	}
 
@@ -844,6 +894,10 @@ static int wiz_probe(struct platform_device *pdev)
 		}
 	}
 
+	ret = wiz_get_lane_phy_types(dev, wiz);
+	if (ret)
+		return ret;
+
 	wiz->dev = dev;
 	wiz->regmap = regmap;
 	wiz->num_lanes = num_lanes;
@@ -897,6 +951,7 @@ static int wiz_probe(struct platform_device *pdev)
 	serdes_pdev = of_platform_device_create(child_node, NULL, dev);
 	if (!serdes_pdev) {
 		dev_WARN(dev, "Unable to create SERDES platform device\n");
+		ret = -ENOMEM;
 		goto err_pdev_create;
 	}
 	wiz->serdes_pdev = serdes_pdev;
diff --git a/drivers/phy/ti/phy-omap-usb2.c b/drivers/phy/ti/phy-omap-usb2.c
index 3d74629d7423..cb2dd3230fa7 100644
--- a/drivers/phy/ti/phy-omap-usb2.c
+++ b/drivers/phy/ti/phy-omap-usb2.c
@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * omap-usb2.c - USB PHY, talking to musb controller in OMAP.
+ * omap-usb2.c - USB PHY, talking to USB controller on TI SoCs.
  *
- * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com
+ * Copyright (C) 2012-2020 Texas Instruments Incorporated - http://www.ti.com
  * Author: Kishon Vijay Abraham I <kishon@ti.com>
  */
 
@@ -23,13 +23,65 @@
 #include <linux/regmap.h>
 #include <linux/of_platform.h>
 
-#define USB2PHY_DISCON_BYP_LATCH (1 << 31)
-#define USB2PHY_ANA_CONFIG1 0x4c
+#define USB2PHY_ANA_CONFIG1		0x4c
+#define USB2PHY_DISCON_BYP_LATCH	BIT(31)
 
+/* SoC Specific USB2_OTG register definitions */
 #define AM654_USB2_OTG_PD		BIT(8)
 #define AM654_USB2_VBUS_DET_EN		BIT(5)
 #define AM654_USB2_VBUSVALID_DET_EN	BIT(4)
 
+#define OMAP_DEV_PHY_PD		BIT(0)
+#define OMAP_USB2_PHY_PD	BIT(28)
+
+#define AM437X_USB2_PHY_PD		BIT(0)
+#define AM437X_USB2_OTG_PD		BIT(1)
+#define AM437X_USB2_OTGVDET_EN		BIT(19)
+#define AM437X_USB2_OTGSESSEND_EN	BIT(20)
+
+/* Driver Flags */
+#define OMAP_USB2_HAS_START_SRP			BIT(0)
+#define OMAP_USB2_HAS_SET_VBUS			BIT(1)
+#define OMAP_USB2_CALIBRATE_FALSE_DISCONNECT	BIT(2)
+
+struct omap_usb {
+	struct usb_phy		phy;
+	struct phy_companion	*comparator;
+	void __iomem		*pll_ctrl_base;
+	void __iomem		*phy_base;
+	struct device		*dev;
+	struct device		*control_dev;
+	struct clk		*wkupclk;
+	struct clk		*optclk;
+	u8			flags;
+	struct regmap		*syscon_phy_power; /* ctrl. reg. acces */
+	unsigned int		power_reg; /* power reg. index within syscon */
+	u32			mask;
+	u32			power_on;
+	u32			power_off;
+};
+
+#define	phy_to_omapusb(x)	container_of((x), struct omap_usb, phy)
+
+struct usb_phy_data {
+	const char *label;
+	u8 flags;
+	u32 mask;
+	u32 power_on;
+	u32 power_off;
+};
+
+static inline u32 omap_usb_readl(void __iomem *addr, unsigned int offset)
+{
+	return __raw_readl(addr + offset);
+}
+
+static inline void omap_usb_writel(void __iomem *addr, unsigned int offset,
+				   u32 data)
+{
+	__raw_writel(data, addr + offset);
+}
+
 /**
  * omap_usb2_set_comparator - links the comparator present in the sytem with
  *	this phy
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 834c59950d1c..8828613c4e0e 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -82,7 +82,7 @@ config PINCTRL_AT91
 config PINCTRL_AT91PIO4
 	bool "AT91 PIO4 pinctrl driver"
 	depends on OF
-	depends on ARCH_AT91
+	depends on ARCH_AT91 || COMPILE_TEST
 	select PINMUX
 	select GENERIC_PINCONF
 	select GPIOLIB
@@ -95,6 +95,7 @@ config PINCTRL_AT91PIO4
 config PINCTRL_AMD
 	tristate "AMD GPIO pin control"
 	depends on HAS_IOMEM
+	depends on ACPI || COMPILE_TEST
 	select GPIOLIB
 	select GPIOLIB_IRQCHIP
 	select PINMUX
@@ -172,15 +173,22 @@ config PINCTRL_GEMINI
 	select GENERIC_PINCONF
 	select MFD_SYSCON
 
+config PINCTRL_MCP23S08_I2C
+	tristate
+	select REGMAP_I2C
+
+config PINCTRL_MCP23S08_SPI
+	tristate
+	select REGMAP_SPI
+
 config PINCTRL_MCP23S08
 	tristate "Microchip MCP23xxx I/O expander"
 	depends on SPI_MASTER || I2C
-	depends on I2C || I2C=n
 	select GPIOLIB
 	select GPIOLIB_IRQCHIP
-	select REGMAP_I2C if I2C
-	select REGMAP_SPI if SPI_MASTER
 	select GENERIC_PINCONF
+	select PINCTRL_MCP23S08_I2C if I2C
+	select PINCTRL_MCP23S08_SPI if SPI_MASTER
 	help
 	  SPI/I2C driver for Microchip MCP23S08 / MCP23S17 / MCP23S18 /
 	  MCP23008 / MCP23017 / MCP23018 I/O expanders.
@@ -435,6 +443,7 @@ config PINCTRL_TB10X
 config PINCTRL_EQUILIBRIUM
 	tristate "Generic pinctrl and GPIO driver for Intel Lightning Mountain SoC"
 	depends on OF && HAS_IOMEM
+	depends on X86 || COMPILE_TEST
 	select PINMUX
 	select PINCONF
 	select GPIOLIB
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index 0b36a1cfca8a..1731b2154df9 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -21,6 +21,8 @@ obj-$(CONFIG_PINCTRL_DIGICOLOR)	+= pinctrl-digicolor.o
 obj-$(CONFIG_PINCTRL_FALCON)	+= pinctrl-falcon.o
 obj-$(CONFIG_PINCTRL_GEMINI)	+= pinctrl-gemini.o
 obj-$(CONFIG_PINCTRL_MAX77620)	+= pinctrl-max77620.o
+obj-$(CONFIG_PINCTRL_MCP23S08_I2C)	+= pinctrl-mcp23s08_i2c.o
+obj-$(CONFIG_PINCTRL_MCP23S08_SPI)	+= pinctrl-mcp23s08_spi.o
 obj-$(CONFIG_PINCTRL_MCP23S08)	+= pinctrl-mcp23s08.o
 obj-$(CONFIG_PINCTRL_MESON)	+= meson/
 obj-$(CONFIG_PINCTRL_OXNAS)	+= pinctrl-oxnas.o
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
index f690fc5cd688..71e666178300 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
@@ -1406,7 +1406,7 @@ static int __init bcm281xx_pinctrl_probe(struct platform_device *pdev)
 	pdata->reg_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(pdata->reg_base)) {
 		dev_err(&pdev->dev, "Failed to ioremap MEM resource\n");
-		return -ENODEV;
+		return PTR_ERR(pdata->reg_base);
 	}
 
 	/* Initialize the dynamic part of pinctrl_desc */
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
index 06bd2b70af3c..1d21129f7751 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
@@ -19,6 +19,7 @@
 #include <linux/irq.h>
 #include <linux/irqdesc.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/of_address.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
@@ -76,6 +77,7 @@
 struct bcm2835_pinctrl {
 	struct device *dev;
 	void __iomem *base;
+	int *wake_irq;
 
 	/* note: locking assumes each bank will have its own unsigned long */
 	unsigned long enabled_irq_map[BCM2835_NUM_BANKS];
@@ -435,6 +437,11 @@ static void bcm2835_gpio_irq_handler(struct irq_desc *desc)
 	chained_irq_exit(host_chip, desc);
 }
 
+static irqreturn_t bcm2835_gpio_wake_irq_handler(int irq, void *dev_id)
+{
+	return IRQ_HANDLED;
+}
+
 static inline void __bcm2835_gpio_irq_config(struct bcm2835_pinctrl *pc,
 	unsigned reg, unsigned offset, bool enable)
 {
@@ -634,6 +641,34 @@ static void bcm2835_gpio_irq_ack(struct irq_data *data)
 	bcm2835_gpio_set_bit(pc, GPEDS0, gpio);
 }
 
+static int bcm2835_gpio_irq_set_wake(struct irq_data *data, unsigned int on)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct bcm2835_pinctrl *pc = gpiochip_get_data(chip);
+	unsigned gpio = irqd_to_hwirq(data);
+	unsigned int irqgroup;
+	int ret = -EINVAL;
+
+	if (!pc->wake_irq)
+		return ret;
+
+	if (gpio <= 27)
+		irqgroup = 0;
+	else if (gpio >= 28 && gpio <= 45)
+		irqgroup = 1;
+	else if (gpio >= 46 && gpio <= 57)
+		irqgroup = 2;
+	else
+		return ret;
+
+	if (on)
+		ret = enable_irq_wake(pc->wake_irq[irqgroup]);
+	else
+		ret = disable_irq_wake(pc->wake_irq[irqgroup]);
+
+	return ret;
+}
+
 static struct irq_chip bcm2835_gpio_irq_chip = {
 	.name = MODULE_NAME,
 	.irq_enable = bcm2835_gpio_irq_enable,
@@ -642,6 +677,8 @@ static struct irq_chip bcm2835_gpio_irq_chip = {
 	.irq_ack = bcm2835_gpio_irq_ack,
 	.irq_mask = bcm2835_gpio_irq_disable,
 	.irq_unmask = bcm2835_gpio_irq_enable,
+	.irq_set_wake = bcm2835_gpio_irq_set_wake,
+	.flags = IRQCHIP_MASK_ON_SUSPEND,
 };
 
 static int bcm2835_pctl_get_groups_count(struct pinctrl_dev *pctldev)
@@ -1137,6 +1174,10 @@ static const struct of_device_id bcm2835_pinctrl_match[] = {
 		.compatible = "brcm,bcm2711-gpio",
 		.data = &bcm2711_plat_data,
 	},
+	{
+		.compatible = "brcm,bcm7211-gpio",
+		.data = &bcm2711_plat_data,
+	},
 	{}
 };
 
@@ -1150,6 +1191,7 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 	struct resource iomem;
 	int err, i;
 	const struct of_device_id *match;
+	int is_7211 = 0;
 
 	BUILD_BUG_ON(ARRAY_SIZE(bcm2835_gpio_pins) != BCM2711_NUM_GPIOS);
 	BUILD_BUG_ON(ARRAY_SIZE(bcm2835_gpio_groups) != BCM2711_NUM_GPIOS);
@@ -1176,6 +1218,7 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 		return -EINVAL;
 
 	pdata = match->data;
+	is_7211 = of_device_is_compatible(np, "brcm,bcm7211-gpio");
 
 	pc->gpio_chip = *pdata->gpio_chip;
 	pc->gpio_chip.parent = dev;
@@ -1210,6 +1253,15 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 				     GFP_KERNEL);
 	if (!girq->parents)
 		return -ENOMEM;
+
+	if (is_7211) {
+		pc->wake_irq = devm_kcalloc(dev, BCM2835_NUM_IRQS,
+					    sizeof(*pc->wake_irq),
+					    GFP_KERNEL);
+		if (!pc->wake_irq)
+			return -ENOMEM;
+	}
+
 	/*
 	 * Use the same handler for all groups: this is necessary
 	 * since we use one gpiochip to cover all lines - the
@@ -1217,8 +1269,34 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 	 * bank that was firing the IRQ and look up the per-group
 	 * and bank data.
 	 */
-	for (i = 0; i < BCM2835_NUM_IRQS; i++)
+	for (i = 0; i < BCM2835_NUM_IRQS; i++) {
+		int len;
+		char *name;
+
 		girq->parents[i] = irq_of_parse_and_map(np, i);
+		if (!is_7211)
+			continue;
+
+		/* Skip over the all banks interrupts */
+		pc->wake_irq[i] = irq_of_parse_and_map(np, i +
+						       BCM2835_NUM_IRQS + 1);
+
+		len = strlen(dev_name(pc->dev)) + 16;
+		name = devm_kzalloc(pc->dev, len, GFP_KERNEL);
+		if (!name)
+			return -ENOMEM;
+
+		snprintf(name, len, "%s:bank%d", dev_name(pc->dev), i);
+
+		/* These are optional interrupts */
+		err = devm_request_irq(dev, pc->wake_irq[i],
+				       bcm2835_gpio_wake_irq_handler,
+				       IRQF_SHARED, name, pc);
+		if (err)
+			dev_warn(dev, "unable to request wake IRQ %d\n",
+				 pc->wake_irq[i]);
+	}
+
 	girq->default_type = IRQ_TYPE_NONE;
 	girq->handler = handle_level_irq;
 
diff --git a/drivers/pinctrl/freescale/Kconfig b/drivers/pinctrl/freescale/Kconfig
index c784663b00ad..4ca44dd69e53 100644
--- a/drivers/pinctrl/freescale/Kconfig
+++ b/drivers/pinctrl/freescale/Kconfig
@@ -165,6 +165,13 @@ config PINCTRL_IMX8QXP
 	help
 	  Say Y here to enable the imx8qxp pinctrl driver
 
+config PINCTRL_IMX8DXL
+	bool "IMX8DXL pinctrl driver"
+	depends on IMX_SCU && ARCH_MXC && ARM64
+	select PINCTRL_IMX_SCU
+	help
+	  Say Y here to enable the imx8dxl pinctrl driver
+
 config PINCTRL_VF610
 	bool "Freescale Vybrid VF610 pinctrl driver"
 	depends on SOC_VF610
diff --git a/drivers/pinctrl/freescale/Makefile b/drivers/pinctrl/freescale/Makefile
index 0ebd3af21e4d..c61722565289 100644
--- a/drivers/pinctrl/freescale/Makefile
+++ b/drivers/pinctrl/freescale/Makefile
@@ -24,6 +24,7 @@ obj-$(CONFIG_PINCTRL_IMX8MP)	+= pinctrl-imx8mp.o
 obj-$(CONFIG_PINCTRL_IMX8MQ)	+= pinctrl-imx8mq.o
 obj-$(CONFIG_PINCTRL_IMX8QM)	+= pinctrl-imx8qm.o
 obj-$(CONFIG_PINCTRL_IMX8QXP)	+= pinctrl-imx8qxp.o
+obj-$(CONFIG_PINCTRL_IMX8DXL)	+= pinctrl-imx8dxl.o
 obj-$(CONFIG_PINCTRL_VF610)	+= pinctrl-vf610.o
 obj-$(CONFIG_PINCTRL_MXS)	+= pinctrl-mxs.o
 obj-$(CONFIG_PINCTRL_IMX23)	+= pinctrl-imx23.o
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c
index 9f42036c5fbb..cb7e0f08d2cf 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx.c
@@ -774,16 +774,6 @@ static int imx_pinctrl_probe_dt(struct platform_device *pdev,
 	return 0;
 }
 
-/*
- * imx_free_resources() - free memory used by this driver
- * @info: info driver instance
- */
-static void imx_free_resources(struct imx_pinctrl *ipctl)
-{
-	if (ipctl->pctl)
-		pinctrl_unregister(ipctl->pctl);
-}
-
 int imx_pinctrl_probe(struct platform_device *pdev,
 		      const struct imx_pinctrl_soc_info *info)
 {
@@ -834,12 +824,13 @@ int imx_pinctrl_probe(struct platform_device *pdev,
 				return -EINVAL;
 			}
 
-			ipctl->input_sel_base = of_iomap(np, 0);
+			ipctl->input_sel_base = devm_of_iomap(&pdev->dev, np,
+							      0, NULL);
 			of_node_put(np);
-			if (!ipctl->input_sel_base) {
+			if (IS_ERR(ipctl->input_sel_base)) {
 				dev_err(&pdev->dev,
 					"iomuxc input select base address not found\n");
-				return -ENOMEM;
+				return PTR_ERR(ipctl->input_sel_base);
 			}
 		}
 	}
@@ -874,23 +865,18 @@ int imx_pinctrl_probe(struct platform_device *pdev,
 					     &ipctl->pctl);
 	if (ret) {
 		dev_err(&pdev->dev, "could not register IMX pinctrl driver\n");
-		goto free;
+		return ret;
 	}
 
 	ret = imx_pinctrl_probe_dt(pdev, ipctl);
 	if (ret) {
 		dev_err(&pdev->dev, "fail to probe dt properties\n");
-		goto free;
+		return ret;
 	}
 
 	dev_info(&pdev->dev, "initialized IMX pinctrl driver\n");
 
 	return pinctrl_enable(ipctl->pctl);
-
-free:
-	imx_free_resources(ipctl);
-
-	return ret;
 }
 
 static int __maybe_unused imx_pinctrl_suspend(struct device *dev)
diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
index c00d0022d311..08d110078c43 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
@@ -60,7 +60,7 @@ struct imx1_pinctrl {
 
 /*
  * IMX1 IOMUXC manages the pins based on ports. Each port has 32 pins. IOMUX
- * control register are seperated into function, output configuration, input
+ * control registers are separated into function, output configuration, input
  * configuration A, input configuration B, GPIO in use and data direction.
  *
  * Those controls that are represented by 1 bit have a direct mapping between
@@ -638,7 +638,6 @@ int imx1_pinctrl_core_probe(struct platform_device *pdev,
 
 	ret = of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
 	if (ret) {
-		pinctrl_unregister(ipctl->pctl);
 		dev_err(&pdev->dev, "Failed to populate subdevices\n");
 		return ret;
 	}
diff --git a/drivers/pinctrl/freescale/pinctrl-imx8dxl.c b/drivers/pinctrl/freescale/pinctrl-imx8dxl.c
new file mode 100644
index 000000000000..7f32e57b7f6a
--- /dev/null
+++ b/drivers/pinctrl/freescale/pinctrl-imx8dxl.c
@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019~2020 NXP
+ */
+
+#include <dt-bindings/pinctrl/pads-imx8dxl.h>
+#include <linux/err.h>
+#include <linux/firmware/imx/sci.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-imx.h"
+
+static const struct pinctrl_pin_desc imx8dxl_pinctrl_pads[] = {
+	IMX_PINCTRL_PIN(IMX8DXL_PCIE_CTRL0_PERST_B),
+	IMX_PINCTRL_PIN(IMX8DXL_PCIE_CTRL0_CLKREQ_B),
+	IMX_PINCTRL_PIN(IMX8DXL_PCIE_CTRL0_WAKE_B),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_PCIESEP),
+	IMX_PINCTRL_PIN(IMX8DXL_USB_SS3_TC0),
+	IMX_PINCTRL_PIN(IMX8DXL_USB_SS3_TC1),
+	IMX_PINCTRL_PIN(IMX8DXL_USB_SS3_TC2),
+	IMX_PINCTRL_PIN(IMX8DXL_USB_SS3_TC3),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_3V3_USB3IO),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_CLK),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_CMD),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA0),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA1),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA2),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA3),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA4),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA5),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA6),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_DATA7),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_STROBE),
+	IMX_PINCTRL_PIN(IMX8DXL_EMMC0_RESET_B),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_SD1FIX0),
+	IMX_PINCTRL_PIN(IMX8DXL_USDHC1_RESET_B),
+	IMX_PINCTRL_PIN(IMX8DXL_USDHC1_VSELECT),
+	IMX_PINCTRL_PIN(IMX8DXL_CTL_NAND_RE_P_N),
+	IMX_PINCTRL_PIN(IMX8DXL_USDHC1_WP),
+	IMX_PINCTRL_PIN(IMX8DXL_USDHC1_CD_B),
+	IMX_PINCTRL_PIN(IMX8DXL_CTL_NAND_DQS_P_N),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_VSELSEP),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_TXC),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_TX_CTL),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_TXD0),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_TXD1),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_TXD2),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_TXD3),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_ENET_ENETB0),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_RXC),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_RX_CTL),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_RXD0),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_RXD1),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_RXD2),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_RGMII_RXD3),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_ENET_ENETB1),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_REFCLK_125M_25M),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_MDIO),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET0_MDC),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIOCT),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_TXC),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_TXD2),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_TX_CTL),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_TXD3),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_RXC),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_RXD3),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_RXD2),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_RXD1),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_TXD0),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_TXD1),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_RXD0),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_RGMII_RX_CTL),
+	IMX_PINCTRL_PIN(IMX8DXL_ENET1_REFCLK_125M_25M),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHB),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI3_SCK),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI3_SDO),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI3_SDI),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI3_CS0),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI3_CS1),
+	IMX_PINCTRL_PIN(IMX8DXL_MCLK_IN1),
+	IMX_PINCTRL_PIN(IMX8DXL_MCLK_IN0),
+	IMX_PINCTRL_PIN(IMX8DXL_MCLK_OUT0),
+	IMX_PINCTRL_PIN(IMX8DXL_UART1_TX),
+	IMX_PINCTRL_PIN(IMX8DXL_UART1_RX),
+	IMX_PINCTRL_PIN(IMX8DXL_UART1_RTS_B),
+	IMX_PINCTRL_PIN(IMX8DXL_UART1_CTS_B),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHK),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI0_SCK),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI0_SDI),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI0_SDO),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI0_CS1),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI0_CS0),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHT),
+	IMX_PINCTRL_PIN(IMX8DXL_ADC_IN1),
+	IMX_PINCTRL_PIN(IMX8DXL_ADC_IN0),
+	IMX_PINCTRL_PIN(IMX8DXL_ADC_IN3),
+	IMX_PINCTRL_PIN(IMX8DXL_ADC_IN2),
+	IMX_PINCTRL_PIN(IMX8DXL_ADC_IN5),
+	IMX_PINCTRL_PIN(IMX8DXL_ADC_IN4),
+	IMX_PINCTRL_PIN(IMX8DXL_FLEXCAN0_RX),
+	IMX_PINCTRL_PIN(IMX8DXL_FLEXCAN0_TX),
+	IMX_PINCTRL_PIN(IMX8DXL_FLEXCAN1_RX),
+	IMX_PINCTRL_PIN(IMX8DXL_FLEXCAN1_TX),
+	IMX_PINCTRL_PIN(IMX8DXL_FLEXCAN2_RX),
+	IMX_PINCTRL_PIN(IMX8DXL_FLEXCAN2_TX),
+	IMX_PINCTRL_PIN(IMX8DXL_UART0_RX),
+	IMX_PINCTRL_PIN(IMX8DXL_UART0_TX),
+	IMX_PINCTRL_PIN(IMX8DXL_UART2_TX),
+	IMX_PINCTRL_PIN(IMX8DXL_UART2_RX),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIOLH),
+	IMX_PINCTRL_PIN(IMX8DXL_JTAG_TRST_B),
+	IMX_PINCTRL_PIN(IMX8DXL_PMIC_I2C_SCL),
+	IMX_PINCTRL_PIN(IMX8DXL_PMIC_I2C_SDA),
+	IMX_PINCTRL_PIN(IMX8DXL_PMIC_INT_B),
+	IMX_PINCTRL_PIN(IMX8DXL_SCU_GPIO0_00),
+	IMX_PINCTRL_PIN(IMX8DXL_SCU_GPIO0_01),
+	IMX_PINCTRL_PIN(IMX8DXL_SCU_PMIC_STANDBY),
+	IMX_PINCTRL_PIN(IMX8DXL_SCU_BOOT_MODE1),
+	IMX_PINCTRL_PIN(IMX8DXL_SCU_BOOT_MODE0),
+	IMX_PINCTRL_PIN(IMX8DXL_SCU_BOOT_MODE2),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_OUT1),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_OUT2),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_OUT3),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_OUT4),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_IN0),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_IN1),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_IN2),
+	IMX_PINCTRL_PIN(IMX8DXL_SNVS_TAMPER_IN3),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI1_SCK),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI1_SDO),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI1_SDI),
+	IMX_PINCTRL_PIN(IMX8DXL_SPI1_CS0),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHD),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0A_DATA1),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0A_DATA0),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0A_DATA3),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0A_DATA2),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0A_SS0_B),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0A_DQS),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0A_SCLK),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_QSPI0A),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0B_SCLK),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0B_DQS),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0B_DATA1),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0B_DATA0),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0B_DATA3),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0B_DATA2),
+	IMX_PINCTRL_PIN(IMX8DXL_QSPI0B_SS0_B),
+	IMX_PINCTRL_PIN(IMX8DXL_COMP_CTL_GPIO_1V8_3V3_QSPI0B)
+};
+
+
+static struct imx_pinctrl_soc_info imx8dxl_pinctrl_info = {
+	.pins = imx8dxl_pinctrl_pads,
+	.npins = ARRAY_SIZE(imx8dxl_pinctrl_pads),
+	.flags = IMX_USE_SCU,
+};
+
+static const struct of_device_id imx8dxl_pinctrl_of_match[] = {
+	{ .compatible = "fsl,imx8dxl-iomuxc", },
+	{ /* sentinel */ }
+};
+
+static int imx8dxl_pinctrl_probe(struct platform_device *pdev)
+{
+	int ret;
+
+	ret = imx_pinctrl_sc_ipc_init(pdev);
+	if (ret)
+		return ret;
+
+	return imx_pinctrl_probe(pdev, &imx8dxl_pinctrl_info);
+}
+
+static struct platform_driver imx8dxl_pinctrl_driver = {
+	.driver = {
+		.name = "fsl,imx8dxl-iomuxc",
+		.of_match_table = of_match_ptr(imx8dxl_pinctrl_of_match),
+		.suppress_bind_attrs = true,
+	},
+	.probe = imx8dxl_pinctrl_probe,
+};
+
+static int __init imx8dxl_pinctrl_init(void)
+{
+	return platform_driver_register(&imx8dxl_pinctrl_driver);
+}
+arch_initcall(imx8dxl_pinctrl_init);
diff --git a/drivers/pinctrl/intel/Kconfig b/drivers/pinctrl/intel/Kconfig
index ee440ec4c94c..787833e343a4 100644
--- a/drivers/pinctrl/intel/Kconfig
+++ b/drivers/pinctrl/intel/Kconfig
@@ -111,6 +111,14 @@ config PINCTRL_ICELAKE
 	  This pinctrl driver provides an interface that allows configuring
 	  of Intel Ice Lake PCH pins and using them as GPIOs.
 
+config PINCTRL_JASPERLAKE
+	tristate "Intel Jasper Lake PCH pinctrl and GPIO driver"
+	depends on ACPI
+	select PINCTRL_INTEL
+	help
+	  This pinctrl driver provides an interface that allows configuring
+	  of Intel Jasper Lake PCH pins and using them as GPIOs.
+
 config PINCTRL_LEWISBURG
 	tristate "Intel Lewisburg pinctrl and GPIO driver"
 	depends on ACPI
diff --git a/drivers/pinctrl/intel/Makefile b/drivers/pinctrl/intel/Makefile
index f60f99cfa7aa..f6f63eb8100f 100644
--- a/drivers/pinctrl/intel/Makefile
+++ b/drivers/pinctrl/intel/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_PINCTRL_CEDARFORK)		+= pinctrl-cedarfork.o
 obj-$(CONFIG_PINCTRL_DENVERTON)		+= pinctrl-denverton.o
 obj-$(CONFIG_PINCTRL_GEMINILAKE)	+= pinctrl-geminilake.o
 obj-$(CONFIG_PINCTRL_ICELAKE)		+= pinctrl-icelake.o
+obj-$(CONFIG_PINCTRL_JASPERLAKE)	+= pinctrl-jasperlake.o
 obj-$(CONFIG_PINCTRL_LEWISBURG)		+= pinctrl-lewisburg.o
 obj-$(CONFIG_PINCTRL_SUNRISEPOINT)	+= pinctrl-sunrisepoint.o
 obj-$(CONFIG_PINCTRL_TIGERLAKE)		+= pinctrl-tigerlake.o
diff --git a/drivers/pinctrl/intel/pinctrl-baytrail.c b/drivers/pinctrl/intel/pinctrl-baytrail.c
index 9b821c9cbd16..0ff7c55173da 100644
--- a/drivers/pinctrl/intel/pinctrl-baytrail.c
+++ b/drivers/pinctrl/intel/pinctrl-baytrail.c
@@ -1506,8 +1506,7 @@ static int byt_gpio_probe(struct intel_pinctrl *vg)
 {
 	struct platform_device *pdev = to_platform_device(vg->dev);
 	struct gpio_chip *gc;
-	struct resource *irq_rc;
-	int ret;
+	int irq, ret;
 
 	/* Set up gpio chip */
 	vg->chip	= byt_gpio_chip;
@@ -1527,8 +1526,8 @@ static int byt_gpio_probe(struct intel_pinctrl *vg)
 #endif
 
 	/* set up interrupts  */
-	irq_rc = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (irq_rc && irq_rc->start) {
+	irq = platform_get_irq_optional(pdev, 0);
+	if (irq > 0) {
 		struct gpio_irq_chip *girq;
 
 		vg->irqchip.name = "BYT-GPIO",
@@ -1548,7 +1547,7 @@ static int byt_gpio_probe(struct intel_pinctrl *vg)
 					     sizeof(*girq->parents), GFP_KERNEL);
 		if (!girq->parents)
 			return -ENOMEM;
-		girq->parents[0] = (unsigned int)irq_rc->start;
+		girq->parents[0] = irq;
 		girq->default_type = IRQ_TYPE_NONE;
 		girq->handler = handle_bad_irq;
 	}
diff --git a/drivers/pinctrl/intel/pinctrl-cannonlake.c b/drivers/pinctrl/intel/pinctrl-cannonlake.c
index f51b27bbf9f1..515f57a0d180 100644
--- a/drivers/pinctrl/intel/pinctrl-cannonlake.c
+++ b/drivers/pinctrl/intel/pinctrl-cannonlake.c
@@ -30,8 +30,6 @@
 		.gpio_base = (g),			\
 	}
 
-#define CNL_NO_GPIO	-1
-
 #define CNL_COMMUNITY(b, s, e, o, g)			\
 	{						\
 		.barno = (b),				\
@@ -377,27 +375,27 @@ static const struct intel_padgroup cnlh_community0_gpps[] = {
 };
 
 static const struct intel_padgroup cnlh_community1_gpps[] = {
-	CNL_GPP(0, 51, 74, 64),			/* GPP_C */
-	CNL_GPP(1, 75, 98, 96),			/* GPP_D */
-	CNL_GPP(2, 99, 106, 128),		/* GPP_G */
-	CNL_GPP(3, 107, 114, CNL_NO_GPIO),	/* AZA */
-	CNL_GPP(4, 115, 146, 160),		/* vGPIO_0 */
-	CNL_GPP(5, 147, 154, CNL_NO_GPIO),	/* vGPIO_1 */
+	CNL_GPP(0, 51, 74, 64),				/* GPP_C */
+	CNL_GPP(1, 75, 98, 96),				/* GPP_D */
+	CNL_GPP(2, 99, 106, 128),			/* GPP_G */
+	CNL_GPP(3, 107, 114, INTEL_GPIO_BASE_NOMAP),	/* AZA */
+	CNL_GPP(4, 115, 146, 160),			/* vGPIO_0 */
+	CNL_GPP(5, 147, 154, INTEL_GPIO_BASE_NOMAP),	/* vGPIO_1 */
 };
 
 static const struct intel_padgroup cnlh_community3_gpps[] = {
-	CNL_GPP(0, 155, 178, 192),		/* GPP_K */
-	CNL_GPP(1, 179, 202, 224),		/* GPP_H */
-	CNL_GPP(2, 203, 215, 256),		/* GPP_E */
-	CNL_GPP(3, 216, 239, 288),		/* GPP_F */
-	CNL_GPP(4, 240, 248, CNL_NO_GPIO),	/* SPI */
+	CNL_GPP(0, 155, 178, 192),			/* GPP_K */
+	CNL_GPP(1, 179, 202, 224),			/* GPP_H */
+	CNL_GPP(2, 203, 215, 256),			/* GPP_E */
+	CNL_GPP(3, 216, 239, 288),			/* GPP_F */
+	CNL_GPP(4, 240, 248, INTEL_GPIO_BASE_NOMAP),	/* SPI */
 };
 
 static const struct intel_padgroup cnlh_community4_gpps[] = {
-	CNL_GPP(0, 249, 259, CNL_NO_GPIO),	/* CPU */
-	CNL_GPP(1, 260, 268, CNL_NO_GPIO),	/* JTAG */
-	CNL_GPP(2, 269, 286, 320),		/* GPP_I */
-	CNL_GPP(3, 287, 298, 352),		/* GPP_J */
+	CNL_GPP(0, 249, 259, INTEL_GPIO_BASE_NOMAP),	/* CPU */
+	CNL_GPP(1, 260, 268, INTEL_GPIO_BASE_NOMAP),	/* JTAG */
+	CNL_GPP(2, 269, 286, 320),			/* GPP_I */
+	CNL_GPP(3, 287, 298, 352),			/* GPP_J */
 };
 
 static const unsigned int cnlh_spi0_pins[] = { 40, 41, 42, 43 };
@@ -790,25 +788,25 @@ static const struct intel_function cnllp_functions[] = {
 };
 
 static const struct intel_padgroup cnllp_community0_gpps[] = {
-	CNL_GPP(0, 0, 24, 0),			/* GPP_A */
-	CNL_GPP(1, 25, 50, 32),			/* GPP_B */
-	CNL_GPP(2, 51, 58, 64),			/* GPP_G */
-	CNL_GPP(3, 59, 67, CNL_NO_GPIO),	/* SPI */
+	CNL_GPP(0, 0, 24, 0),				/* GPP_A */
+	CNL_GPP(1, 25, 50, 32),				/* GPP_B */
+	CNL_GPP(2, 51, 58, 64),				/* GPP_G */
+	CNL_GPP(3, 59, 67, INTEL_GPIO_BASE_NOMAP),	/* SPI */
 };
 
 static const struct intel_padgroup cnllp_community1_gpps[] = {
-	CNL_GPP(0, 68, 92, 96),			/* GPP_D */
-	CNL_GPP(1, 93, 116, 128),		/* GPP_F */
-	CNL_GPP(2, 117, 140, 160),		/* GPP_H */
-	CNL_GPP(3, 141, 172, 192),		/* vGPIO */
-	CNL_GPP(4, 173, 180, 224),		/* vGPIO */
+	CNL_GPP(0, 68, 92, 96),				/* GPP_D */
+	CNL_GPP(1, 93, 116, 128),			/* GPP_F */
+	CNL_GPP(2, 117, 140, 160),			/* GPP_H */
+	CNL_GPP(3, 141, 172, 192),			/* vGPIO */
+	CNL_GPP(4, 173, 180, 224),			/* vGPIO */
 };
 
 static const struct intel_padgroup cnllp_community4_gpps[] = {
-	CNL_GPP(0, 181, 204, 256),		/* GPP_C */
-	CNL_GPP(1, 205, 228, 288),		/* GPP_E */
-	CNL_GPP(2, 229, 237, CNL_NO_GPIO),	/* JTAG */
-	CNL_GPP(3, 238, 243, CNL_NO_GPIO),	/* HVCMOS */
+	CNL_GPP(0, 181, 204, 256),			/* GPP_C */
+	CNL_GPP(1, 205, 228, 288),			/* GPP_E */
+	CNL_GPP(2, 229, 237, INTEL_GPIO_BASE_NOMAP),	/* JTAG */
+	CNL_GPP(3, 238, 243, INTEL_GPIO_BASE_NOMAP),	/* HVCMOS */
 };
 
 static const struct intel_community cnllp_communities[] = {
diff --git a/drivers/pinctrl/intel/pinctrl-cherryview.c b/drivers/pinctrl/intel/pinctrl-cherryview.c
index 1093a6105d40..8e3953a223d0 100644
--- a/drivers/pinctrl/intel/pinctrl-cherryview.c
+++ b/drivers/pinctrl/intel/pinctrl-cherryview.c
@@ -35,18 +35,18 @@
 
 #define CHV_PADCTRL0			0x000
 #define CHV_PADCTRL0_INTSEL_SHIFT	28
-#define CHV_PADCTRL0_INTSEL_MASK	(0xf << CHV_PADCTRL0_INTSEL_SHIFT)
+#define CHV_PADCTRL0_INTSEL_MASK	GENMASK(31, 28)
 #define CHV_PADCTRL0_TERM_UP		BIT(23)
 #define CHV_PADCTRL0_TERM_SHIFT		20
-#define CHV_PADCTRL0_TERM_MASK		(7 << CHV_PADCTRL0_TERM_SHIFT)
+#define CHV_PADCTRL0_TERM_MASK		GENMASK(22, 20)
 #define CHV_PADCTRL0_TERM_20K		1
 #define CHV_PADCTRL0_TERM_5K		2
 #define CHV_PADCTRL0_TERM_1K		4
 #define CHV_PADCTRL0_PMODE_SHIFT	16
-#define CHV_PADCTRL0_PMODE_MASK		(0xf << CHV_PADCTRL0_PMODE_SHIFT)
+#define CHV_PADCTRL0_PMODE_MASK		GENMASK(19, 16)
 #define CHV_PADCTRL0_GPIOEN		BIT(15)
 #define CHV_PADCTRL0_GPIOCFG_SHIFT	8
-#define CHV_PADCTRL0_GPIOCFG_MASK	(7 << CHV_PADCTRL0_GPIOCFG_SHIFT)
+#define CHV_PADCTRL0_GPIOCFG_MASK	GENMASK(10, 8)
 #define CHV_PADCTRL0_GPIOCFG_GPIO	0
 #define CHV_PADCTRL0_GPIOCFG_GPO	1
 #define CHV_PADCTRL0_GPIOCFG_GPI	2
@@ -57,58 +57,17 @@
 #define CHV_PADCTRL1			0x004
 #define CHV_PADCTRL1_CFGLOCK		BIT(31)
 #define CHV_PADCTRL1_INVRXTX_SHIFT	4
-#define CHV_PADCTRL1_INVRXTX_MASK	(0xf << CHV_PADCTRL1_INVRXTX_SHIFT)
-#define CHV_PADCTRL1_INVRXTX_TXENABLE	(2 << CHV_PADCTRL1_INVRXTX_SHIFT)
+#define CHV_PADCTRL1_INVRXTX_MASK	GENMASK(7, 4)
+#define CHV_PADCTRL1_INVRXTX_RXDATA	BIT(6)
+#define CHV_PADCTRL1_INVRXTX_TXENABLE	BIT(5)
 #define CHV_PADCTRL1_ODEN		BIT(3)
-#define CHV_PADCTRL1_INVRXTX_RXDATA	(4 << CHV_PADCTRL1_INVRXTX_SHIFT)
-#define CHV_PADCTRL1_INTWAKECFG_MASK	7
+#define CHV_PADCTRL1_INTWAKECFG_MASK	GENMASK(2, 0)
 #define CHV_PADCTRL1_INTWAKECFG_FALLING	1
 #define CHV_PADCTRL1_INTWAKECFG_RISING	2
 #define CHV_PADCTRL1_INTWAKECFG_BOTH	3
 #define CHV_PADCTRL1_INTWAKECFG_LEVEL	4
 
 /**
- * struct chv_alternate_function - A per group or per pin alternate function
- * @pin: Pin number (only used in per pin configs)
- * @mode: Mode the pin should be set in
- * @invert_oe: Invert OE for this pin
- */
-struct chv_alternate_function {
-	unsigned int pin;
-	u8 mode;
-	bool invert_oe;
-};
-
-/**
- * struct chv_pincgroup - describes a CHV pin group
- * @name: Name of the group
- * @pins: An array of pins in this group
- * @npins: Number of pins in this group
- * @altfunc: Alternate function applied to all pins in this group
- * @overrides: Alternate function override per pin or %NULL if not used
- * @noverrides: Number of per pin alternate function overrides if
- *              @overrides != NULL.
- */
-struct chv_pingroup {
-	const char *name;
-	const unsigned int *pins;
-	size_t npins;
-	struct chv_alternate_function altfunc;
-	const struct chv_alternate_function *overrides;
-	size_t noverrides;
-};
-
-/**
- * struct chv_gpio_pinrange - A range of pins that can be used as GPIOs
- * @base: Start pin number
- * @npins: Number of pins in this range
- */
-struct chv_gpio_pinrange {
-	unsigned int base;
-	unsigned int npins;
-};
-
-/**
  * struct chv_community - A community specific configuration
  * @uid: ACPI _UID used to match the community
  * @pins: All pins in this community
@@ -117,8 +76,8 @@ struct chv_gpio_pinrange {
  * @ngroups: Number of groups
  * @functions: All functions in this community
  * @nfunctions: Number of functions
- * @gpio_ranges: An array of GPIO ranges in this community
- * @ngpio_ranges: Number of GPIO ranges
+ * @gpps: Pad groups
+ * @ngpps: Number of pad groups in this community
  * @nirqs: Total number of IRQs this community can generate
  * @acpi_space_id: An address space ID for ACPI OpRegion handler
  */
@@ -126,12 +85,12 @@ struct chv_community {
 	const char *uid;
 	const struct pinctrl_pin_desc *pins;
 	size_t npins;
-	const struct chv_pingroup *groups;
+	const struct intel_pingroup *groups;
 	size_t ngroups;
 	const struct intel_function *functions;
 	size_t nfunctions;
-	const struct chv_gpio_pinrange *gpio_ranges;
-	size_t ngpio_ranges;
+	const struct intel_padgroup *gpps;
+	size_t ngpps;
 	size_t nirqs;
 	acpi_adr_space_type acpi_space_id;
 };
@@ -173,37 +132,14 @@ struct chv_pinctrl {
 	struct chv_pin_context *saved_pin_context;
 };
 
-#define ALTERNATE_FUNCTION(p, m, i)		\
-	{					\
-		.pin = (p),			\
-		.mode = (m),			\
-		.invert_oe = (i),		\
-	}
+#define	PINMODE_INVERT_OE	BIT(15)
 
-#define PIN_GROUP_WITH_ALT(n, p, m, i)		\
-	{					\
-		.name = (n),			\
-		.pins = (p),			\
-		.npins = ARRAY_SIZE((p)),	\
-		.altfunc.mode = (m),		\
-		.altfunc.invert_oe = (i),	\
-	}
+#define PINMODE(m, i)		((m) | ((i) * PINMODE_INVERT_OE))
 
-#define PIN_GROUP_WITH_OVERRIDE(n, p, m, i, o)	\
-	{					\
-		.name = (n),			\
-		.pins = (p),			\
-		.npins = ARRAY_SIZE((p)),	\
-		.altfunc.mode = (m),		\
-		.altfunc.invert_oe = (i),	\
-		.overrides = (o),		\
-		.noverrides = ARRAY_SIZE((o)),	\
-	}
-
-#define GPIO_PINRANGE(start, end)		\
+#define CHV_GPP(start, end)		\
 	{					\
 		.base = (start),		\
-		.npins = (end) - (start) + 1,	\
+		.size = (end) - (start) + 1,	\
 	}
 
 static const struct pinctrl_pin_desc southwest_pins[] = {
@@ -288,40 +224,37 @@ static const unsigned southwest_i2c6_pins[] = { 47, 51 };
 static const unsigned southwest_i2c_nfc_pins[] = { 49, 52 };
 static const unsigned southwest_spi3_pins[] = { 76, 79, 80, 81, 82 };
 
-/* LPE I2S TXD pins need to have invert_oe set */
-static const struct chv_alternate_function southwest_lpe_altfuncs[] = {
-	ALTERNATE_FUNCTION(30, 1, true),
-	ALTERNATE_FUNCTION(34, 1, true),
-	ALTERNATE_FUNCTION(97, 1, true),
+/* Some of LPE I2S TXD pins need to have OE inversion set */
+static const unsigned int southwest_lpe_altfuncs[] = {
+	PINMODE(1, 1), PINMODE(1, 0), PINMODE(1, 0), PINMODE(1, 0), /* 30, 31, 32, 33 */
+	PINMODE(1, 1), PINMODE(1, 0), PINMODE(1, 0), PINMODE(1, 0), /* 34, 35, 36, 37 */
+	PINMODE(1, 0), PINMODE(1, 0), PINMODE(1, 0), PINMODE(1, 1), /* 92, 94, 96, 97 */
 };
 
 /*
  * Two spi3 chipselects are available in different mode than the main spi3
- * functionality, which is using mode 1.
+ * functionality, which is using mode 2.
  */
-static const struct chv_alternate_function southwest_spi3_altfuncs[] = {
-	ALTERNATE_FUNCTION(76, 3, false),
-	ALTERNATE_FUNCTION(80, 3, false),
+static const unsigned int southwest_spi3_altfuncs[] = {
+	PINMODE(3, 0), PINMODE(2, 0), PINMODE(3, 0), PINMODE(2, 0), /* 76, 79, 80, 81 */
+	PINMODE(2, 0),						    /* 82 */
 };
 
-static const struct chv_pingroup southwest_groups[] = {
-	PIN_GROUP_WITH_ALT("uart0_grp", southwest_uart0_pins, 2, false),
-	PIN_GROUP_WITH_ALT("uart1_grp", southwest_uart1_pins, 1, false),
-	PIN_GROUP_WITH_ALT("uart2_grp", southwest_uart2_pins, 1, false),
-	PIN_GROUP_WITH_ALT("hda_grp", southwest_hda_pins, 2, false),
-	PIN_GROUP_WITH_ALT("i2c0_grp", southwest_i2c0_pins, 1, true),
-	PIN_GROUP_WITH_ALT("i2c1_grp", southwest_i2c1_pins, 1, true),
-	PIN_GROUP_WITH_ALT("i2c2_grp", southwest_i2c2_pins, 1, true),
-	PIN_GROUP_WITH_ALT("i2c3_grp", southwest_i2c3_pins, 1, true),
-	PIN_GROUP_WITH_ALT("i2c4_grp", southwest_i2c4_pins, 1, true),
-	PIN_GROUP_WITH_ALT("i2c5_grp", southwest_i2c5_pins, 1, true),
-	PIN_GROUP_WITH_ALT("i2c6_grp", southwest_i2c6_pins, 1, true),
-	PIN_GROUP_WITH_ALT("i2c_nfc_grp", southwest_i2c_nfc_pins, 2, true),
-
-	PIN_GROUP_WITH_OVERRIDE("lpe_grp", southwest_lpe_pins, 1, false,
-				southwest_lpe_altfuncs),
-	PIN_GROUP_WITH_OVERRIDE("spi3_grp", southwest_spi3_pins, 2, false,
-				southwest_spi3_altfuncs),
+static const struct intel_pingroup southwest_groups[] = {
+	PIN_GROUP("uart0_grp", southwest_uart0_pins, PINMODE(2, 0)),
+	PIN_GROUP("uart1_grp", southwest_uart1_pins, PINMODE(1, 0)),
+	PIN_GROUP("uart2_grp", southwest_uart2_pins, PINMODE(1, 0)),
+	PIN_GROUP("hda_grp", southwest_hda_pins, PINMODE(2, 0)),
+	PIN_GROUP("i2c0_grp", southwest_i2c0_pins, PINMODE(1, 1)),
+	PIN_GROUP("i2c1_grp", southwest_i2c1_pins, PINMODE(1, 1)),
+	PIN_GROUP("i2c2_grp", southwest_i2c2_pins, PINMODE(1, 1)),
+	PIN_GROUP("i2c3_grp", southwest_i2c3_pins, PINMODE(1, 1)),
+	PIN_GROUP("i2c4_grp", southwest_i2c4_pins, PINMODE(1, 1)),
+	PIN_GROUP("i2c5_grp", southwest_i2c5_pins, PINMODE(1, 1)),
+	PIN_GROUP("i2c6_grp", southwest_i2c6_pins, PINMODE(1, 1)),
+	PIN_GROUP("i2c_nfc_grp", southwest_i2c_nfc_pins, PINMODE(2, 1)),
+	PIN_GROUP("lpe_grp", southwest_lpe_pins, southwest_lpe_altfuncs),
+	PIN_GROUP("spi3_grp", southwest_spi3_pins, southwest_spi3_altfuncs),
 };
 
 static const char * const southwest_uart0_groups[] = { "uart0_grp" };
@@ -360,14 +293,14 @@ static const struct intel_function southwest_functions[] = {
 	FUNCTION("spi3", southwest_spi3_groups),
 };
 
-static const struct chv_gpio_pinrange southwest_gpio_ranges[] = {
-	GPIO_PINRANGE(0, 7),
-	GPIO_PINRANGE(15, 22),
-	GPIO_PINRANGE(30, 37),
-	GPIO_PINRANGE(45, 52),
-	GPIO_PINRANGE(60, 67),
-	GPIO_PINRANGE(75, 82),
-	GPIO_PINRANGE(90, 97),
+static const struct intel_padgroup southwest_gpps[] = {
+	CHV_GPP(0, 7),
+	CHV_GPP(15, 22),
+	CHV_GPP(30, 37),
+	CHV_GPP(45, 52),
+	CHV_GPP(60, 67),
+	CHV_GPP(75, 82),
+	CHV_GPP(90, 97),
 };
 
 static const struct chv_community southwest_community = {
@@ -378,8 +311,8 @@ static const struct chv_community southwest_community = {
 	.ngroups = ARRAY_SIZE(southwest_groups),
 	.functions = southwest_functions,
 	.nfunctions = ARRAY_SIZE(southwest_functions),
-	.gpio_ranges = southwest_gpio_ranges,
-	.ngpio_ranges = ARRAY_SIZE(southwest_gpio_ranges),
+	.gpps = southwest_gpps,
+	.ngpps = ARRAY_SIZE(southwest_gpps),
 	/*
 	 * Southwest community can generate GPIO interrupts only for the
 	 * first 8 interrupts. The upper half (8-15) can only be used to
@@ -455,20 +388,20 @@ static const struct pinctrl_pin_desc north_pins[] = {
 	PINCTRL_PIN(72, "PANEL0_VDDEN"),
 };
 
-static const struct chv_gpio_pinrange north_gpio_ranges[] = {
-	GPIO_PINRANGE(0, 8),
-	GPIO_PINRANGE(15, 27),
-	GPIO_PINRANGE(30, 41),
-	GPIO_PINRANGE(45, 56),
-	GPIO_PINRANGE(60, 72),
+static const struct intel_padgroup north_gpps[] = {
+	CHV_GPP(0, 8),
+	CHV_GPP(15, 27),
+	CHV_GPP(30, 41),
+	CHV_GPP(45, 56),
+	CHV_GPP(60, 72),
 };
 
 static const struct chv_community north_community = {
 	.uid = "2",
 	.pins = north_pins,
 	.npins = ARRAY_SIZE(north_pins),
-	.gpio_ranges = north_gpio_ranges,
-	.ngpio_ranges = ARRAY_SIZE(north_gpio_ranges),
+	.gpps = north_gpps,
+	.ngpps = ARRAY_SIZE(north_gpps),
 	/*
 	 * North community can generate GPIO interrupts only for the first
 	 * 8 interrupts. The upper half (8-15) can only be used to trigger
@@ -506,17 +439,17 @@ static const struct pinctrl_pin_desc east_pins[] = {
 	PINCTRL_PIN(26, "MF_ISH_I2C1_SDA"),
 };
 
-static const struct chv_gpio_pinrange east_gpio_ranges[] = {
-	GPIO_PINRANGE(0, 11),
-	GPIO_PINRANGE(15, 26),
+static const struct intel_padgroup east_gpps[] = {
+	CHV_GPP(0, 11),
+	CHV_GPP(15, 26),
 };
 
 static const struct chv_community east_community = {
 	.uid = "3",
 	.pins = east_pins,
 	.npins = ARRAY_SIZE(east_pins),
-	.gpio_ranges = east_gpio_ranges,
-	.ngpio_ranges = ARRAY_SIZE(east_gpio_ranges),
+	.gpps = east_gpps,
+	.ngpps = ARRAY_SIZE(east_gpps),
 	.nirqs = 16,
 	.acpi_space_id = 0x93,
 };
@@ -596,14 +529,14 @@ static const unsigned southeast_sdmmc3_pins[] = {
 static const unsigned southeast_spi1_pins[] = { 60, 61, 62, 64, 66 };
 static const unsigned southeast_spi2_pins[] = { 2, 3, 4, 6, 7 };
 
-static const struct chv_pingroup southeast_groups[] = {
-	PIN_GROUP_WITH_ALT("pwm0_grp", southeast_pwm0_pins, 1, false),
-	PIN_GROUP_WITH_ALT("pwm1_grp", southeast_pwm1_pins, 1, false),
-	PIN_GROUP_WITH_ALT("sdmmc1_grp", southeast_sdmmc1_pins, 1, false),
-	PIN_GROUP_WITH_ALT("sdmmc2_grp", southeast_sdmmc2_pins, 1, false),
-	PIN_GROUP_WITH_ALT("sdmmc3_grp", southeast_sdmmc3_pins, 1, false),
-	PIN_GROUP_WITH_ALT("spi1_grp", southeast_spi1_pins, 1, false),
-	PIN_GROUP_WITH_ALT("spi2_grp", southeast_spi2_pins, 4, false),
+static const struct intel_pingroup southeast_groups[] = {
+	PIN_GROUP("pwm0_grp", southeast_pwm0_pins, PINMODE(1, 0)),
+	PIN_GROUP("pwm1_grp", southeast_pwm1_pins, PINMODE(1, 0)),
+	PIN_GROUP("sdmmc1_grp", southeast_sdmmc1_pins, PINMODE(1, 0)),
+	PIN_GROUP("sdmmc2_grp", southeast_sdmmc2_pins, PINMODE(1, 0)),
+	PIN_GROUP("sdmmc3_grp", southeast_sdmmc3_pins, PINMODE(1, 0)),
+	PIN_GROUP("spi1_grp", southeast_spi1_pins, PINMODE(1, 0)),
+	PIN_GROUP("spi2_grp", southeast_spi2_pins, PINMODE(4, 0)),
 };
 
 static const char * const southeast_pwm0_groups[] = { "pwm0_grp" };
@@ -624,13 +557,13 @@ static const struct intel_function southeast_functions[] = {
 	FUNCTION("spi2", southeast_spi2_groups),
 };
 
-static const struct chv_gpio_pinrange southeast_gpio_ranges[] = {
-	GPIO_PINRANGE(0, 7),
-	GPIO_PINRANGE(15, 26),
-	GPIO_PINRANGE(30, 35),
-	GPIO_PINRANGE(45, 52),
-	GPIO_PINRANGE(60, 69),
-	GPIO_PINRANGE(75, 85),
+static const struct intel_padgroup southeast_gpps[] = {
+	CHV_GPP(0, 7),
+	CHV_GPP(15, 26),
+	CHV_GPP(30, 35),
+	CHV_GPP(45, 52),
+	CHV_GPP(60, 69),
+	CHV_GPP(75, 85),
 };
 
 static const struct chv_community southeast_community = {
@@ -641,8 +574,8 @@ static const struct chv_community southeast_community = {
 	.ngroups = ARRAY_SIZE(southeast_groups),
 	.functions = southeast_functions,
 	.nfunctions = ARRAY_SIZE(southeast_functions),
-	.gpio_ranges = southeast_gpio_ranges,
-	.ngpio_ranges = ARRAY_SIZE(southeast_gpio_ranges),
+	.gpps = southeast_gpps,
+	.ngpps = ARRAY_SIZE(southeast_gpps),
 	.nirqs = 16,
 	.acpi_space_id = 0x94,
 };
@@ -789,7 +722,7 @@ static int chv_pinmux_set_mux(struct pinctrl_dev *pctldev,
 			      unsigned int function, unsigned int group)
 {
 	struct chv_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
-	const struct chv_pingroup *grp;
+	const struct intel_pingroup *grp;
 	unsigned long flags;
 	int i;
 
@@ -808,22 +741,21 @@ static int chv_pinmux_set_mux(struct pinctrl_dev *pctldev,
 	}
 
 	for (i = 0; i < grp->npins; i++) {
-		const struct chv_alternate_function *altfunc = &grp->altfunc;
 		int pin = grp->pins[i];
 		void __iomem *reg;
+		unsigned int mode;
+		bool invert_oe;
 		u32 value;
 
 		/* Check if there is pin-specific config */
-		if (grp->overrides) {
-			int j;
-
-			for (j = 0; j < grp->noverrides; j++) {
-				if (grp->overrides[j].pin == pin) {
-					altfunc = &grp->overrides[j];
-					break;
-				}
-			}
-		}
+		if (grp->modes)
+			mode = grp->modes[i];
+		else
+			mode = grp->mode;
+
+		/* Extract OE inversion */
+		invert_oe = mode & PINMODE_INVERT_OE;
+		mode &= ~PINMODE_INVERT_OE;
 
 		reg = chv_padreg(pctrl, pin, CHV_PADCTRL0);
 		value = readl(reg);
@@ -831,18 +763,18 @@ static int chv_pinmux_set_mux(struct pinctrl_dev *pctldev,
 		value &= ~CHV_PADCTRL0_GPIOEN;
 		/* Set to desired mode */
 		value &= ~CHV_PADCTRL0_PMODE_MASK;
-		value |= altfunc->mode << CHV_PADCTRL0_PMODE_SHIFT;
+		value |= mode << CHV_PADCTRL0_PMODE_SHIFT;
 		chv_writel(value, reg);
 
 		/* Update for invert_oe */
 		reg = chv_padreg(pctrl, pin, CHV_PADCTRL1);
 		value = readl(reg) & ~CHV_PADCTRL1_INVRXTX_MASK;
-		if (altfunc->invert_oe)
+		if (invert_oe)
 			value |= CHV_PADCTRL1_INVRXTX_TXENABLE;
 		chv_writel(value, reg);
 
 		dev_dbg(pctrl->dev, "configured pin %u mode %u OE %sinverted\n",
-			pin, altfunc->mode, altfunc->invert_oe ? "" : "not ");
+			pin, mode, invert_oe ? "" : "not ");
 	}
 
 	raw_spin_unlock_irqrestore(&chv_lock, flags);
@@ -1594,14 +1526,14 @@ static int chv_gpio_add_pin_ranges(struct gpio_chip *chip)
 {
 	struct chv_pinctrl *pctrl = gpiochip_get_data(chip);
 	const struct chv_community *community = pctrl->community;
-	const struct chv_gpio_pinrange *range;
+	const struct intel_padgroup *gpp;
 	int ret, i;
 
-	for (i = 0; i < community->ngpio_ranges; i++) {
-		range = &community->gpio_ranges[i];
+	for (i = 0; i < community->ngpps; i++) {
+		gpp = &community->gpps[i];
 		ret = gpiochip_add_pin_range(chip, dev_name(pctrl->dev),
-					     range->base, range->base,
-					     range->npins);
+					     gpp->base, gpp->base,
+					     gpp->size);
 		if (ret) {
 			dev_err(pctrl->dev, "failed to add GPIO pin range\n");
 			return ret;
@@ -1613,7 +1545,7 @@ static int chv_gpio_add_pin_ranges(struct gpio_chip *chip)
 
 static int chv_gpio_probe(struct chv_pinctrl *pctrl, int irq)
 {
-	const struct chv_gpio_pinrange *range;
+	const struct intel_padgroup *gpp;
 	struct gpio_chip *chip = &pctrl->chip;
 	bool need_valid_mask = !dmi_check_system(chv_no_valid_mask);
 	const struct chv_community *community = pctrl->community;
@@ -1661,12 +1593,12 @@ static int chv_gpio_probe(struct chv_pinctrl *pctrl, int irq)
 	}
 
 	if (!need_valid_mask) {
-		for (i = 0; i < community->ngpio_ranges; i++) {
-			range = &community->gpio_ranges[i];
+		for (i = 0; i < community->ngpps; i++) {
+			gpp = &community->gpps[i];
 
 			irq_domain_associate_many(chip->irq.domain, irq_base,
-						  range->base, range->npins);
-			irq_base += range->npins;
+						  gpp->base, gpp->size);
+			irq_base += gpp->size;
 		}
 	}
 
diff --git a/drivers/pinctrl/intel/pinctrl-icelake.c b/drivers/pinctrl/intel/pinctrl-icelake.c
index 6489e9bbb61f..429b5a83acf0 100644
--- a/drivers/pinctrl/intel/pinctrl-icelake.c
+++ b/drivers/pinctrl/intel/pinctrl-icelake.c
@@ -29,8 +29,6 @@
 		.gpio_base = (g),			\
 	}
 
-#define ICL_NO_GPIO	-1
-
 #define ICL_COMMUNITY(b, s, e, g)			\
 	{						\
 		.barno = (b),				\
@@ -305,29 +303,29 @@ static const struct pinctrl_pin_desc icllp_pins[] = {
 };
 
 static const struct intel_padgroup icllp_community0_gpps[] = {
-	ICL_GPP(0, 0, 7, 0),			/* GPP_G */
-	ICL_GPP(1, 8, 33, 32),			/* GPP_B */
-	ICL_GPP(2, 34, 58, 64),			/* GPP_A */
+	ICL_GPP(0, 0, 7, 0),				/* GPP_G */
+	ICL_GPP(1, 8, 33, 32),				/* GPP_B */
+	ICL_GPP(2, 34, 58, 64),				/* GPP_A */
 };
 
 static const struct intel_padgroup icllp_community1_gpps[] = {
-	ICL_GPP(0, 59, 82, 96),			/* GPP_H */
-	ICL_GPP(1, 83, 103, 128),		/* GPP_D */
-	ICL_GPP(2, 104, 123, 160),		/* GPP_F */
-	ICL_GPP(3, 124, 152, 192),		/* vGPIO */
+	ICL_GPP(0, 59, 82, 96),				/* GPP_H */
+	ICL_GPP(1, 83, 103, 128),			/* GPP_D */
+	ICL_GPP(2, 104, 123, 160),			/* GPP_F */
+	ICL_GPP(3, 124, 152, 192),			/* vGPIO */
 };
 
 static const struct intel_padgroup icllp_community4_gpps[] = {
-	ICL_GPP(0, 153, 176, 224),		/* GPP_C */
-	ICL_GPP(1, 177, 182, ICL_NO_GPIO),	/* HVCMOS */
-	ICL_GPP(2, 183, 206, 256),		/* GPP_E */
-	ICL_GPP(3, 207, 215, ICL_NO_GPIO),	/* JTAG */
+	ICL_GPP(0, 153, 176, 224),			/* GPP_C */
+	ICL_GPP(1, 177, 182, INTEL_GPIO_BASE_NOMAP),	/* HVCMOS */
+	ICL_GPP(2, 183, 206, 256),			/* GPP_E */
+	ICL_GPP(3, 207, 215, INTEL_GPIO_BASE_NOMAP),	/* JTAG */
 };
 
 static const struct intel_padgroup icllp_community5_gpps[] = {
-	ICL_GPP(0, 216, 223, 288),		/* GPP_R */
-	ICL_GPP(1, 224, 231, 320),		/* GPP_S */
-	ICL_GPP(2, 232, 240, ICL_NO_GPIO),	/* SPI */
+	ICL_GPP(0, 216, 223, 288),			/* GPP_R */
+	ICL_GPP(1, 224, 231, 320),			/* GPP_S */
+	ICL_GPP(2, 232, 240, INTEL_GPIO_BASE_NOMAP),	/* SPI */
 };
 
 static const struct intel_community icllp_communities[] = {
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 74fdfd2b9ff5..6a274e20d926 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -798,7 +798,7 @@ static int intel_gpio_to_pin(struct intel_pinctrl *pctrl, unsigned int offset,
 		for (j = 0; j < comm->ngpps; j++) {
 			const struct intel_padgroup *pgrp = &comm->gpps[j];
 
-			if (pgrp->gpio_base < 0)
+			if (pgrp->gpio_base == INTEL_GPIO_BASE_NOMAP)
 				continue;
 
 			if (offset >= pgrp->gpio_base &&
@@ -1138,7 +1138,7 @@ static int intel_gpio_add_community_ranges(struct intel_pinctrl *pctrl,
 	for (i = 0; i < community->ngpps; i++) {
 		const struct intel_padgroup *gpp = &community->gpps[i];
 
-		if (gpp->gpio_base < 0)
+		if (gpp->gpio_base == INTEL_GPIO_BASE_NOMAP)
 			continue;
 
 		ret = gpiochip_add_pin_range(&pctrl->chip, dev_name(pctrl->dev),
@@ -1180,7 +1180,7 @@ static unsigned int intel_gpio_ngpio(const struct intel_pinctrl *pctrl)
 		for (j = 0; j < community->ngpps; j++) {
 			const struct intel_padgroup *gpp = &community->gpps[j];
 
-			if (gpp->gpio_base < 0)
+			if (gpp->gpio_base == INTEL_GPIO_BASE_NOMAP)
 				continue;
 
 			if (gpp->gpio_base + gpp->size > ngpio)
@@ -1276,8 +1276,18 @@ static int intel_pinctrl_add_padgroups(struct intel_pinctrl *pctrl,
 		if (gpps[i].size > 32)
 			return -EINVAL;
 
-		if (!gpps[i].gpio_base)
-			gpps[i].gpio_base = gpps[i].base;
+		/* Special treatment for GPIO base */
+		switch (gpps[i].gpio_base) {
+			case INTEL_GPIO_BASE_MATCH:
+				gpps[i].gpio_base = gpps[i].base;
+				break;
+			case INTEL_GPIO_BASE_ZERO:
+				gpps[i].gpio_base = 0;
+				break;
+			case INTEL_GPIO_BASE_NOMAP:
+			default:
+				break;
+		}
 
 		gpps[i].padown_num = padown_num;
 
@@ -1596,7 +1606,7 @@ static void intel_restore_hostown(struct intel_pinctrl *pctrl, unsigned int c,
 	struct device *dev = pctrl->dev;
 	u32 requested;
 
-	if (padgrp->gpio_base < 0)
+	if (padgrp->gpio_base == INTEL_GPIO_BASE_NOMAP)
 		return;
 
 	requested = intel_gpio_is_requested(&pctrl->chip, padgrp->gpio_base, padgrp->size);
diff --git a/drivers/pinctrl/intel/pinctrl-intel.h b/drivers/pinctrl/intel/pinctrl-intel.h
index c6f066f6d3fb..cc78c483518f 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.h
+++ b/drivers/pinctrl/intel/pinctrl-intel.h
@@ -53,8 +53,7 @@ struct intel_function {
  * @reg_num: GPI_IS register number
  * @base: Starting pin of this group
  * @size: Size of this group (maximum is 32).
- * @gpio_base: Starting GPIO base of this group (%0 if matches with @base,
- *	       and %-1 if no GPIO mapping should be created)
+ * @gpio_base: Starting GPIO base of this group
  * @padown_num: PAD_OWN register number (assigned by the core driver)
  *
  * If pad groups of a community are not the same size, use this structure
@@ -69,6 +68,19 @@ struct intel_padgroup {
 };
 
 /**
+ * enum - Special treatment for GPIO base in pad group
+ *
+ * @INTEL_GPIO_BASE_ZERO:	force GPIO base to be 0
+ * @INTEL_GPIO_BASE_NOMAP:	no GPIO mapping should be created
+ * @INTEL_GPIO_BASE_MATCH:	matches with starting pin number
+ */
+enum {
+	INTEL_GPIO_BASE_ZERO	= -2,
+	INTEL_GPIO_BASE_NOMAP	= -1,
+	INTEL_GPIO_BASE_MATCH	= 0,
+};
+
+/**
  * struct intel_community - Intel pin community description
  * @barno: MMIO BAR number where registers for this community reside
  * @padown_offset: Register offset of PAD_OWN register from @regs. If %0
@@ -82,20 +94,20 @@ struct intel_padgroup {
  * @ie_offset: Register offset of GPI_IE from @regs.
  * @features: Additional features supported by the hardware
  * @pin_base: Starting pin of pins in this community
+ * @npins: Number of pins in this community
  * @gpp_size: Maximum number of pads in each group, such as PADCFGLOCK,
- *            HOSTSW_OWN,  GPI_IS, GPI_IE, etc. Used when @gpps is %NULL.
+ *            HOSTSW_OWN, GPI_IS, GPI_IE. Used when @gpps is %NULL.
  * @gpp_num_padown_regs: Number of pad registers each pad group consumes at
  *			 minimum. Use %0 if the number of registers can be
  *			 determined by the size of the group.
- * @npins: Number of pins in this community
  * @gpps: Pad groups if the controller has variable size pad groups
  * @ngpps: Number of pad groups in this community
  * @pad_map: Optional non-linear mapping of the pads
  * @regs: Community specific common registers (reserved for core driver)
  * @pad_regs: Community specific pad registers (reserved for core driver)
  *
- * Most Intel GPIO host controllers this driver supports each pad group is
- * of equal size (except the last one). In that case the driver can just
+ * In some of Intel GPIO host controllers this driver supports each pad group
+ * is of equal size (except the last one). In that case the driver can just
  * fill in @gpp_size field and let the core driver to handle the rest. If
  * the controller has pad groups of variable size the client driver can
  * pass custom @gpps and @ngpps instead.
@@ -109,12 +121,13 @@ struct intel_community {
 	unsigned int ie_offset;
 	unsigned int features;
 	unsigned int pin_base;
+	size_t npins;
 	unsigned int gpp_size;
 	unsigned int gpp_num_padown_regs;
-	size_t npins;
 	const struct intel_padgroup *gpps;
 	size_t ngpps;
 	const unsigned int *pad_map;
+
 	/* Reserved for the core driver */
 	void __iomem *regs;
 	void __iomem *pad_regs;
diff --git a/drivers/pinctrl/intel/pinctrl-jasperlake.c b/drivers/pinctrl/intel/pinctrl-jasperlake.c
new file mode 100644
index 000000000000..9bd0e8e6310c
--- /dev/null
+++ b/drivers/pinctrl/intel/pinctrl-jasperlake.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Intel Jasper Lake PCH pinctrl/GPIO driver
+ *
+ * Copyright (C) 2020, Intel Corporation
+ * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+ */
+
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-intel.h"
+
+#define JSL_PAD_OWN	0x020
+#define JSL_PADCFGLOCK	0x080
+#define JSL_HOSTSW_OWN	0x0b0
+#define JSL_GPI_IS	0x100
+#define JSL_GPI_IE	0x120
+
+#define JSL_GPP(r, s, e, g)				\
+	{						\
+		.reg_num = (r),				\
+		.base = (s),				\
+		.size = ((e) - (s) + 1),		\
+		.gpio_base = (g),			\
+	}
+
+#define JSL_COMMUNITY(b, s, e, g)			\
+	{						\
+		.barno = (b),				\
+		.padown_offset = JSL_PAD_OWN,		\
+		.padcfglock_offset = JSL_PADCFGLOCK,	\
+		.hostown_offset = JSL_HOSTSW_OWN,	\
+		.is_offset = JSL_GPI_IS,		\
+		.ie_offset = JSL_GPI_IE,		\
+		.pin_base = (s),			\
+		.npins = ((e) - (s) + 1),		\
+		.gpps = (g),				\
+		.ngpps = ARRAY_SIZE(g),			\
+	}
+
+/* Jasper Lake */
+static const struct pinctrl_pin_desc jsl_pins[] = {
+	/* GPP_F */
+	PINCTRL_PIN(0, "CNV_BRI_DT_UART0_RTSB"),
+	PINCTRL_PIN(1, "CNV_BRI_RSP_UART0_RXD"),
+	PINCTRL_PIN(2, "EMMC_HIP_MON"),
+	PINCTRL_PIN(3, "CNV_RGI_RSP_UART0_CTSB"),
+	PINCTRL_PIN(4, "CNV_RF_RESET_B"),
+	PINCTRL_PIN(5, "MODEM_CLKREQ"),
+	PINCTRL_PIN(6, "CNV_PA_BLANKING"),
+	PINCTRL_PIN(7, "EMMC_CMD"),
+	PINCTRL_PIN(8, "EMMC_DATA0"),
+	PINCTRL_PIN(9, "EMMC_DATA1"),
+	PINCTRL_PIN(10, "EMMC_DATA2"),
+	PINCTRL_PIN(11, "EMMC_DATA3"),
+	PINCTRL_PIN(12, "EMMC_DATA4"),
+	PINCTRL_PIN(13, "EMMC_DATA5"),
+	PINCTRL_PIN(14, "EMMC_DATA6"),
+	PINCTRL_PIN(15, "EMMC_DATA7"),
+	PINCTRL_PIN(16, "EMMC_RCLK"),
+	PINCTRL_PIN(17, "EMMC_CLK"),
+	PINCTRL_PIN(18, "EMMC_RESETB"),
+	PINCTRL_PIN(19, "A4WP_PRESENT"),
+	/* GPP_B */
+	PINCTRL_PIN(20, "CORE_VID_0"),
+	PINCTRL_PIN(21, "CORE_VID_1"),
+	PINCTRL_PIN(22, "VRALERTB"),
+	PINCTRL_PIN(23, "CPU_GP_2"),
+	PINCTRL_PIN(24, "CPU_GP_3"),
+	PINCTRL_PIN(25, "SRCCLKREQB_0"),
+	PINCTRL_PIN(26, "SRCCLKREQB_1"),
+	PINCTRL_PIN(27, "SRCCLKREQB_2"),
+	PINCTRL_PIN(28, "SRCCLKREQB_3"),
+	PINCTRL_PIN(29, "SRCCLKREQB_4"),
+	PINCTRL_PIN(30, "SRCCLKREQB_5"),
+	PINCTRL_PIN(31, "PMCALERTB"),
+	PINCTRL_PIN(32, "SLP_S0B"),
+	PINCTRL_PIN(33, "PLTRSTB"),
+	PINCTRL_PIN(34, "SPKR"),
+	PINCTRL_PIN(35, "GSPI0_CS0B"),
+	PINCTRL_PIN(36, "GSPI0_CLK"),
+	PINCTRL_PIN(37, "GSPI0_MISO"),
+	PINCTRL_PIN(38, "GSPI0_MOSI"),
+	PINCTRL_PIN(39, "GSPI1_CS0B"),
+	PINCTRL_PIN(40, "GSPI1_CLK"),
+	PINCTRL_PIN(41, "GSPI1_MISO"),
+	PINCTRL_PIN(42, "GSPI1_MOSI"),
+	PINCTRL_PIN(43, "DDSP_HPD_A"),
+	PINCTRL_PIN(44, "GSPI0_CLK_LOOPBK"),
+	PINCTRL_PIN(45, "GSPI1_CLK_LOOPBK"),
+	/* GPP_A */
+	PINCTRL_PIN(46, "ESPI_IO_0"),
+	PINCTRL_PIN(47, "ESPI_IO_1"),
+	PINCTRL_PIN(48, "ESPI_IO_2"),
+	PINCTRL_PIN(49, "ESPI_IO_3"),
+	PINCTRL_PIN(50, "ESPI_CSB"),
+	PINCTRL_PIN(51, "ESPI_CLK"),
+	PINCTRL_PIN(52, "ESPI_RESETB"),
+	PINCTRL_PIN(53, "SMBCLK"),
+	PINCTRL_PIN(54, "SMBDATA"),
+	PINCTRL_PIN(55, "SMBALERTB"),
+	PINCTRL_PIN(56, "CPU_GP_0"),
+	PINCTRL_PIN(57, "CPU_GP_1"),
+	PINCTRL_PIN(58, "USB2_OCB_1"),
+	PINCTRL_PIN(59, "USB2_OCB_2"),
+	PINCTRL_PIN(60, "USB2_OCB_3"),
+	PINCTRL_PIN(61, "DDSP_HPD_A_TIME_SYNC_0"),
+	PINCTRL_PIN(62, "DDSP_HPD_B"),
+	PINCTRL_PIN(63, "DDSP_HPD_C"),
+	PINCTRL_PIN(64, "USB2_OCB_0"),
+	PINCTRL_PIN(65, "PCHHOTB"),
+	PINCTRL_PIN(66, "ESPI_CLK_LOOPBK"),
+	/* GPP_S */
+	PINCTRL_PIN(67, "SNDW1_CLK"),
+	PINCTRL_PIN(68, "SNDW1_DATA"),
+	PINCTRL_PIN(69, "SNDW2_CLK"),
+	PINCTRL_PIN(70, "SNDW2_DATA"),
+	PINCTRL_PIN(71, "SNDW1_CLK"),
+	PINCTRL_PIN(72, "SNDW1_DATA"),
+	PINCTRL_PIN(73, "SNDW4_CLK_DMIC_CLK_0"),
+	PINCTRL_PIN(74, "SNDW4_DATA_DMIC_DATA_0"),
+	/* GPP_R */
+	PINCTRL_PIN(75, "HDA_BCLK"),
+	PINCTRL_PIN(76, "HDA_SYNC"),
+	PINCTRL_PIN(77, "HDA_SDO"),
+	PINCTRL_PIN(78, "HDA_SDI_0"),
+	PINCTRL_PIN(79, "HDA_RSTB"),
+	PINCTRL_PIN(80, "HDA_SDI_1"),
+	PINCTRL_PIN(81, "I2S1_SFRM"),
+	PINCTRL_PIN(82, "I2S1_TXD"),
+	/* GPP_H */
+	PINCTRL_PIN(83, "GPPC_H_0"),
+	PINCTRL_PIN(84, "SD_PWR_EN_B"),
+	PINCTRL_PIN(85, "MODEM_CLKREQ"),
+	PINCTRL_PIN(86, "SX_EXIT_HOLDOFFB"),
+	PINCTRL_PIN(87, "I2C2_SDA"),
+	PINCTRL_PIN(88, "I2C2_SCL"),
+	PINCTRL_PIN(89, "I2C3_SDA"),
+	PINCTRL_PIN(90, "I2C3_SCL"),
+	PINCTRL_PIN(91, "I2C4_SDA"),
+	PINCTRL_PIN(92, "I2C4_SCL"),
+	PINCTRL_PIN(93, "CPU_VCCIO_PWR_GATEB"),
+	PINCTRL_PIN(94, "I2S2_SCLK"),
+	PINCTRL_PIN(95, "I2S2_SFRM"),
+	PINCTRL_PIN(96, "I2S2_TXD"),
+	PINCTRL_PIN(97, "I2S2_RXD"),
+	PINCTRL_PIN(98, "I2S1_SCLK"),
+	PINCTRL_PIN(99, "GPPC_H_16"),
+	PINCTRL_PIN(100, "GPPC_H_17"),
+	PINCTRL_PIN(101, "GPPC_H_18"),
+	PINCTRL_PIN(102, "GPPC_H_19"),
+	PINCTRL_PIN(103, "GPPC_H_20"),
+	PINCTRL_PIN(104, "GPPC_H_21"),
+	PINCTRL_PIN(105, "GPPC_H_22"),
+	PINCTRL_PIN(106, "GPPC_H_23"),
+	/* GPP_D */
+	PINCTRL_PIN(107, "SPI1_CSB"),
+	PINCTRL_PIN(108, "SPI1_CLK"),
+	PINCTRL_PIN(109, "SPI1_MISO_IO_1"),
+	PINCTRL_PIN(110, "SPI1_MOSI_IO_0"),
+	PINCTRL_PIN(111, "ISH_I2C0_SDA"),
+	PINCTRL_PIN(112, "ISH_I2C0_SCL"),
+	PINCTRL_PIN(113, "ISH_I2C1_SDA"),
+	PINCTRL_PIN(114, "ISH_I2C1_SCL"),
+	PINCTRL_PIN(115, "ISH_SPI_CSB"),
+	PINCTRL_PIN(116, "ISH_SPI_CLK"),
+	PINCTRL_PIN(117, "ISH_SPI_MISO"),
+	PINCTRL_PIN(118, "ISH_SPI_MOSI"),
+	PINCTRL_PIN(119, "ISH_UART0_RXD"),
+	PINCTRL_PIN(120, "ISH_UART0_TXD"),
+	PINCTRL_PIN(121, "ISH_UART0_RTSB"),
+	PINCTRL_PIN(122, "ISH_UART0_CTSB"),
+	PINCTRL_PIN(123, "SPI1_IO_2"),
+	PINCTRL_PIN(124, "SPI1_IO_3"),
+	PINCTRL_PIN(125, "I2S_MCLK"),
+	PINCTRL_PIN(126, "CNV_MFUART2_RXD"),
+	PINCTRL_PIN(127, "CNV_MFUART2_TXD"),
+	PINCTRL_PIN(128, "CNV_PA_BLANKING"),
+	PINCTRL_PIN(129, "I2C5_SDA"),
+	PINCTRL_PIN(130, "I2C5_SCL"),
+	PINCTRL_PIN(131, "GSPI2_CLK_LOOPBK"),
+	PINCTRL_PIN(132, "SPI1_CLK_LOOPBK"),
+	/* vGPIO */
+	PINCTRL_PIN(133, "CNV_BTEN"),
+	PINCTRL_PIN(134, "CNV_WCEN"),
+	PINCTRL_PIN(135, "CNV_BT_HOST_WAKEB"),
+	PINCTRL_PIN(136, "CNV_BT_IF_SELECT"),
+	PINCTRL_PIN(137, "vCNV_BT_UART_TXD"),
+	PINCTRL_PIN(138, "vCNV_BT_UART_RXD"),
+	PINCTRL_PIN(139, "vCNV_BT_UART_CTS_B"),
+	PINCTRL_PIN(140, "vCNV_BT_UART_RTS_B"),
+	PINCTRL_PIN(141, "vCNV_MFUART1_TXD"),
+	PINCTRL_PIN(142, "vCNV_MFUART1_RXD"),
+	PINCTRL_PIN(143, "vCNV_MFUART1_CTS_B"),
+	PINCTRL_PIN(144, "vCNV_MFUART1_RTS_B"),
+	PINCTRL_PIN(145, "vUART0_TXD"),
+	PINCTRL_PIN(146, "vUART0_RXD"),
+	PINCTRL_PIN(147, "vUART0_CTS_B"),
+	PINCTRL_PIN(148, "vUART0_RTS_B"),
+	PINCTRL_PIN(149, "vISH_UART0_TXD"),
+	PINCTRL_PIN(150, "vISH_UART0_RXD"),
+	PINCTRL_PIN(151, "vISH_UART0_CTS_B"),
+	PINCTRL_PIN(152, "vISH_UART0_RTS_B"),
+	PINCTRL_PIN(153, "vCNV_BT_I2S_BCLK"),
+	PINCTRL_PIN(154, "vCNV_BT_I2S_WS_SYNC"),
+	PINCTRL_PIN(155, "vCNV_BT_I2S_SDO"),
+	PINCTRL_PIN(156, "vCNV_BT_I2S_SDI"),
+	PINCTRL_PIN(157, "vI2S2_SCLK"),
+	PINCTRL_PIN(158, "vI2S2_SFRM"),
+	PINCTRL_PIN(159, "vI2S2_TXD"),
+	PINCTRL_PIN(160, "vI2S2_RXD"),
+	PINCTRL_PIN(161, "vSD3_CD_B"),
+	/* GPP_C */
+	PINCTRL_PIN(162, "GPPC_C_0"),
+	PINCTRL_PIN(163, "GPPC_C_1"),
+	PINCTRL_PIN(164, "GPPC_C_2"),
+	PINCTRL_PIN(165, "GPPC_C_3"),
+	PINCTRL_PIN(166, "GPPC_C_4"),
+	PINCTRL_PIN(167, "GPPC_C_5"),
+	PINCTRL_PIN(168, "SUSWARNB_SUSPWRDNACK"),
+	PINCTRL_PIN(169, "SUSACKB"),
+	PINCTRL_PIN(170, "UART0_RXD"),
+	PINCTRL_PIN(171, "UART0_TXD"),
+	PINCTRL_PIN(172, "UART0_RTSB"),
+	PINCTRL_PIN(173, "UART0_CTSB"),
+	PINCTRL_PIN(174, "UART1_RXD"),
+	PINCTRL_PIN(175, "UART1_TXD"),
+	PINCTRL_PIN(176, "UART1_RTSB"),
+	PINCTRL_PIN(177, "UART1_CTSB"),
+	PINCTRL_PIN(178, "I2C0_SDA"),
+	PINCTRL_PIN(179, "I2C0_SCL"),
+	PINCTRL_PIN(180, "I2C1_SDA"),
+	PINCTRL_PIN(181, "I2C1_SCL"),
+	PINCTRL_PIN(182, "UART2_RXD"),
+	PINCTRL_PIN(183, "UART2_TXD"),
+	PINCTRL_PIN(184, "UART2_RTSB"),
+	PINCTRL_PIN(185, "UART2_CTSB"),
+	/* HVCMOS */
+	PINCTRL_PIN(186, "L_BKLTEN"),
+	PINCTRL_PIN(187, "L_BKLTCTL"),
+	PINCTRL_PIN(188, "L_VDDEN"),
+	PINCTRL_PIN(189, "SYS_PWROK"),
+	PINCTRL_PIN(190, "SYS_RESETB"),
+	PINCTRL_PIN(191, "MLK_RSTB"),
+	/* GPP_E */
+	PINCTRL_PIN(192, "ISH_GP_0"),
+	PINCTRL_PIN(193, "ISH_GP_1"),
+	PINCTRL_PIN(194, "IMGCLKOUT_1"),
+	PINCTRL_PIN(195, "ISH_GP_2"),
+	PINCTRL_PIN(196, "IMGCLKOUT_2"),
+	PINCTRL_PIN(197, "SATA_LEDB"),
+	PINCTRL_PIN(198, "IMGCLKOUT_3"),
+	PINCTRL_PIN(199, "ISH_GP_3"),
+	PINCTRL_PIN(200, "ISH_GP_4"),
+	PINCTRL_PIN(201, "ISH_GP_5"),
+	PINCTRL_PIN(202, "ISH_GP_6"),
+	PINCTRL_PIN(203, "ISH_GP_7"),
+	PINCTRL_PIN(204, "IMGCLKOUT_4"),
+	PINCTRL_PIN(205, "DDPA_CTRLCLK"),
+	PINCTRL_PIN(206, "DDPA_CTRLDATA"),
+	PINCTRL_PIN(207, "DDPB_CTRLCLK"),
+	PINCTRL_PIN(208, "DDPB_CTRLDATA"),
+	PINCTRL_PIN(209, "DDPC_CTRLCLK"),
+	PINCTRL_PIN(210, "DDPC_CTRLDATA"),
+	PINCTRL_PIN(211, "IMGCLKOUT_5"),
+	PINCTRL_PIN(212, "CNV_BRI_DT"),
+	PINCTRL_PIN(213, "CNV_BRI_RSP"),
+	PINCTRL_PIN(214, "CNV_RGI_DT"),
+	PINCTRL_PIN(215, "CNV_RGI_RSP"),
+	/* GPP_G */
+	PINCTRL_PIN(216, "SD3_CMD"),
+	PINCTRL_PIN(217, "SD3_D0"),
+	PINCTRL_PIN(218, "SD3_D1"),
+	PINCTRL_PIN(219, "SD3_D2"),
+	PINCTRL_PIN(220, "SD3_D3"),
+	PINCTRL_PIN(221, "SD3_CDB"),
+	PINCTRL_PIN(222, "SD3_CLK"),
+	PINCTRL_PIN(223, "SD3_WP"),
+};
+
+static const struct intel_padgroup jsl_community0_gpps[] = {
+	JSL_GPP(0, 0, 19, 320),				/* GPP_F */
+	JSL_GPP(1, 20, 45, 32),				/* GPP_B */
+	JSL_GPP(2, 46, 66, 64),				/* GPP_A */
+	JSL_GPP(3, 67, 74, 96),				/* GPP_S */
+	JSL_GPP(4, 75, 82, 128),			/* GPP_R */
+};
+
+static const struct intel_padgroup jsl_community1_gpps[] = {
+	JSL_GPP(0, 83, 106, 160),			/* GPP_H */
+	JSL_GPP(1, 107, 132, 192),			/* GPP_D */
+	JSL_GPP(2, 133, 161, 224),			/* vGPIO */
+	JSL_GPP(3, 162, 185, 256),			/* GPP_C */
+};
+
+static const struct intel_padgroup jsl_community4_gpps[] = {
+	JSL_GPP(0, 186, 191, INTEL_GPIO_BASE_NOMAP),	/* HVCMOS */
+	JSL_GPP(1, 192, 215, 288),			/* GPP_E */
+};
+
+static const struct intel_padgroup jsl_community5_gpps[] = {
+	JSL_GPP(0, 216, 223, INTEL_GPIO_BASE_ZERO),	/* GPP_G */
+};
+
+static const struct intel_community jsl_communities[] = {
+	JSL_COMMUNITY(0, 0, 82, jsl_community0_gpps),
+	JSL_COMMUNITY(1, 83, 185, jsl_community1_gpps),
+	JSL_COMMUNITY(2, 186, 215, jsl_community4_gpps),
+	JSL_COMMUNITY(3, 216, 223, jsl_community5_gpps),
+};
+
+static const struct intel_pinctrl_soc_data jsl_soc_data = {
+	.pins = jsl_pins,
+	.npins = ARRAY_SIZE(jsl_pins),
+	.communities = jsl_communities,
+	.ncommunities = ARRAY_SIZE(jsl_communities),
+};
+
+static const struct acpi_device_id jsl_pinctrl_acpi_match[] = {
+	{ "INT34C8", (kernel_ulong_t)&jsl_soc_data },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, jsl_pinctrl_acpi_match);
+
+static INTEL_PINCTRL_PM_OPS(jsl_pinctrl_pm_ops);
+
+static struct platform_driver jsl_pinctrl_driver = {
+	.probe = intel_pinctrl_probe_by_hid,
+	.driver = {
+		.name = "jasperlake-pinctrl",
+		.acpi_match_table = jsl_pinctrl_acpi_match,
+		.pm = &jsl_pinctrl_pm_ops,
+	},
+};
+
+module_platform_driver(jsl_pinctrl_driver);
+
+MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
+MODULE_DESCRIPTION("Intel Jasper Lake PCH pinctrl/GPIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/intel/pinctrl-lynxpoint.c b/drivers/pinctrl/intel/pinctrl-lynxpoint.c
index e928742c7181..a45b8f2182fd 100644
--- a/drivers/pinctrl/intel/pinctrl-lynxpoint.c
+++ b/drivers/pinctrl/intel/pinctrl-lynxpoint.c
@@ -794,11 +794,11 @@ static int lp_gpio_probe(struct platform_device *pdev)
 	const struct intel_pinctrl_soc_data *soc;
 	struct intel_pinctrl *lg;
 	struct gpio_chip *gc;
-	struct resource *io_rc, *irq_rc;
 	struct device *dev = &pdev->dev;
+	struct resource *io_rc;
 	void __iomem *regs;
 	unsigned int i;
-	int ret;
+	int irq, ret;
 
 	soc = (const struct intel_pinctrl_soc_data *)device_get_match_data(dev);
 	if (!soc)
@@ -870,8 +870,8 @@ static int lp_gpio_probe(struct platform_device *pdev)
 	gc->parent = dev;
 
 	/* set up interrupts  */
-	irq_rc = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (irq_rc && irq_rc->start) {
+	irq = platform_get_irq_optional(pdev, 0);
+	if (irq > 0) {
 		struct gpio_irq_chip *girq;
 
 		girq = &gc->irq;
@@ -884,7 +884,7 @@ static int lp_gpio_probe(struct platform_device *pdev)
 					     GFP_KERNEL);
 		if (!girq->parents)
 			return -ENOMEM;
-		girq->parents[0] = (unsigned int)irq_rc->start;
+		girq->parents[0] = irq;
 		girq->default_type = IRQ_TYPE_NONE;
 		girq->handler = handle_bad_irq;
 	}
diff --git a/drivers/pinctrl/intel/pinctrl-tigerlake.c b/drivers/pinctrl/intel/pinctrl-tigerlake.c
index 08a86f6fdea6..bcfd7548e282 100644
--- a/drivers/pinctrl/intel/pinctrl-tigerlake.c
+++ b/drivers/pinctrl/intel/pinctrl-tigerlake.c
@@ -21,8 +21,6 @@
 #define TGL_GPI_IS	0x100
 #define TGL_GPI_IE	0x120
 
-#define TGL_NO_GPIO	-1
-
 #define TGL_GPP(r, s, e, g)				\
 	{						\
 		.reg_num = (r),				\
@@ -342,30 +340,30 @@ static const struct pinctrl_pin_desc tgllp_pins[] = {
 };
 
 static const struct intel_padgroup tgllp_community0_gpps[] = {
-	TGL_GPP(0, 0, 25, 0),			/* GPP_B */
-	TGL_GPP(1, 26, 41, 32),			/* GPP_T */
-	TGL_GPP(2, 42, 66, 64),			/* GPP_A */
+	TGL_GPP(0, 0, 25, 0),				/* GPP_B */
+	TGL_GPP(1, 26, 41, 32),				/* GPP_T */
+	TGL_GPP(2, 42, 66, 64),				/* GPP_A */
 };
 
 static const struct intel_padgroup tgllp_community1_gpps[] = {
-	TGL_GPP(0, 67, 74, 96),			/* GPP_S */
-	TGL_GPP(1, 75, 98, 128),		/* GPP_H */
-	TGL_GPP(2, 99, 119, 160),		/* GPP_D */
-	TGL_GPP(3, 120, 143, 192),		/* GPP_U */
-	TGL_GPP(4, 144, 170, 224),		/* vGPIO */
+	TGL_GPP(0, 67, 74, 96),				/* GPP_S */
+	TGL_GPP(1, 75, 98, 128),			/* GPP_H */
+	TGL_GPP(2, 99, 119, 160),			/* GPP_D */
+	TGL_GPP(3, 120, 143, 192),			/* GPP_U */
+	TGL_GPP(4, 144, 170, 224),			/* vGPIO */
 };
 
 static const struct intel_padgroup tgllp_community4_gpps[] = {
-	TGL_GPP(0, 171, 194, 256),		/* GPP_C */
-	TGL_GPP(1, 195, 219, 288),		/* GPP_F */
-	TGL_GPP(2, 220, 225, TGL_NO_GPIO),	/* HVCMOS */
-	TGL_GPP(3, 226, 250, 320),		/* GPP_E */
-	TGL_GPP(4, 251, 259, TGL_NO_GPIO),	/* JTAG */
+	TGL_GPP(0, 171, 194, 256),			/* GPP_C */
+	TGL_GPP(1, 195, 219, 288),			/* GPP_F */
+	TGL_GPP(2, 220, 225, INTEL_GPIO_BASE_NOMAP),	/* HVCMOS */
+	TGL_GPP(3, 226, 250, 320),			/* GPP_E */
+	TGL_GPP(4, 251, 259, INTEL_GPIO_BASE_NOMAP),	/* JTAG */
 };
 
 static const struct intel_padgroup tgllp_community5_gpps[] = {
-	TGL_GPP(0, 260, 267, 352),		/* GPP_R */
-	TGL_GPP(1, 268, 276, TGL_NO_GPIO),	/* SPI */
+	TGL_GPP(0, 260, 267, 352),			/* GPP_R */
+	TGL_GPP(1, 268, 276, INTEL_GPIO_BASE_NOMAP),	/* SPI */
 };
 
 static const struct intel_community tgllp_communities[] = {
diff --git a/drivers/pinctrl/mediatek/Kconfig b/drivers/pinctrl/mediatek/Kconfig
index 701f9af63f5e..f32d3644c509 100644
--- a/drivers/pinctrl/mediatek/Kconfig
+++ b/drivers/pinctrl/mediatek/Kconfig
@@ -3,10 +3,12 @@ menu "MediaTek pinctrl drivers"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 
 config EINT_MTK
-	bool "MediaTek External Interrupt Support"
+	tristate "MediaTek External Interrupt Support"
 	depends on PINCTRL_MTK || PINCTRL_MTK_MOORE || PINCTRL_MTK_PARIS || COMPILE_TEST
 	select GPIOLIB
 	select IRQ_DOMAIN
+	default y if PINCTRL_MTK || PINCTRL_MTK_MOORE
+	default PINCTRL_MTK_PARIS
 
 config PINCTRL_MTK
 	bool
@@ -17,6 +19,9 @@ config PINCTRL_MTK
 	select EINT_MTK
 	select OF_GPIO
 
+config PINCTRL_MTK_V2
+	tristate
+
 config PINCTRL_MTK_MOORE
 	bool
 	depends on OF
@@ -25,15 +30,17 @@ config PINCTRL_MTK_MOORE
 	select GENERIC_PINMUX_FUNCTIONS
 	select GPIOLIB
 	select OF_GPIO
+	select PINCTRL_MTK_V2
 
 config PINCTRL_MTK_PARIS
-	bool
+	tristate
 	depends on OF
 	select PINMUX
 	select GENERIC_PINCONF
 	select GPIOLIB
 	select EINT_MTK
 	select OF_GPIO
+	select PINCTRL_MTK_V2
 
 # For ARMv7 SoCs
 config PINCTRL_MT2701
@@ -80,7 +87,7 @@ config PINCTRL_MT2712
 	select PINCTRL_MTK
 
 config PINCTRL_MT6765
-	bool "Mediatek MT6765 pin control"
+	tristate "Mediatek MT6765 pin control"
 	depends on OF
 	depends on ARM64 || COMPILE_TEST
 	default ARM64 && ARCH_MEDIATEK
diff --git a/drivers/pinctrl/mediatek/Makefile b/drivers/pinctrl/mediatek/Makefile
index a74325abd877..4b7132876e71 100644
--- a/drivers/pinctrl/mediatek/Makefile
+++ b/drivers/pinctrl/mediatek/Makefile
@@ -2,8 +2,9 @@
 # Core
 obj-$(CONFIG_EINT_MTK)		+= mtk-eint.o
 obj-$(CONFIG_PINCTRL_MTK)	+= pinctrl-mtk-common.o
-obj-$(CONFIG_PINCTRL_MTK_MOORE) += pinctrl-moore.o pinctrl-mtk-common-v2.o
-obj-$(CONFIG_PINCTRL_MTK_PARIS) += pinctrl-paris.o pinctrl-mtk-common-v2.o
+obj-$(CONFIG_PINCTRL_MTK_V2)	+= pinctrl-mtk-common-v2.o
+obj-$(CONFIG_PINCTRL_MTK_MOORE) += pinctrl-moore.o
+obj-$(CONFIG_PINCTRL_MTK_PARIS) += pinctrl-paris.o
 
 # SoC Drivers
 obj-$(CONFIG_PINCTRL_MT2701)	+= pinctrl-mt2701.o
diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c
index 7e526bcf5e0b..22736f60c16c 100644
--- a/drivers/pinctrl/mediatek/mtk-eint.c
+++ b/drivers/pinctrl/mediatek/mtk-eint.c
@@ -15,6 +15,7 @@
 #include <linux/io.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
+#include <linux/module.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 
@@ -379,6 +380,7 @@ int mtk_eint_do_suspend(struct mtk_eint *eint)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_eint_do_suspend);
 
 int mtk_eint_do_resume(struct mtk_eint *eint)
 {
@@ -386,6 +388,7 @@ int mtk_eint_do_resume(struct mtk_eint *eint)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_eint_do_resume);
 
 int mtk_eint_set_debounce(struct mtk_eint *eint, unsigned long eint_num,
 			  unsigned int debounce)
@@ -440,6 +443,7 @@ int mtk_eint_set_debounce(struct mtk_eint *eint, unsigned long eint_num,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_eint_set_debounce);
 
 int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n)
 {
@@ -451,6 +455,7 @@ int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n)
 
 	return irq;
 }
+EXPORT_SYMBOL_GPL(mtk_eint_find_irq);
 
 int mtk_eint_do_init(struct mtk_eint *eint)
 {
@@ -495,3 +500,7 @@ int mtk_eint_do_init(struct mtk_eint *eint)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_eint_do_init);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("MediaTek EINT Driver");
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt6765.c b/drivers/pinctrl/mediatek/pinctrl-mt6765.c
index 905dae8c3fd8..2c59d3936256 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt6765.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt6765.c
@@ -6,6 +6,7 @@
  *
  */
 
+#include <linux/module.h>
 #include "pinctrl-mtk-mt6765.h"
 #include "pinctrl-paris.h"
 
@@ -1103,3 +1104,6 @@ static int __init mt6765_pinctrl_init(void)
 	return platform_driver_register(&mt6765_pinctrl_driver);
 }
 arch_initcall(mt6765_pinctrl_init);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("MediaTek MT6765 Pinctrl Driver");
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
index d3169a87e1b3..b77b18fe5adc 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
@@ -12,6 +12,7 @@
 #include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
+#include <linux/module.h>
 #include <linux/of_irq.h>
 
 #include "mtk-eint.h"
@@ -204,6 +205,7 @@ int mtk_hw_set_value(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_hw_set_value);
 
 int mtk_hw_get_value(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc,
 		     int field, int *value)
@@ -223,6 +225,7 @@ int mtk_hw_get_value(struct mtk_pinctrl *hw, const struct mtk_pin_desc *desc,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_hw_get_value);
 
 static int mtk_xt_find_eint_num(struct mtk_pinctrl *hw, unsigned long eint_n)
 {
@@ -361,6 +364,7 @@ int mtk_build_eint(struct mtk_pinctrl *hw, struct platform_device *pdev)
 
 	return mtk_eint_do_init(hw->eint);
 }
+EXPORT_SYMBOL_GPL(mtk_build_eint);
 
 /* Revision 0 */
 int mtk_pinconf_bias_disable_set(struct mtk_pinctrl *hw,
@@ -380,6 +384,7 @@ int mtk_pinconf_bias_disable_set(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_disable_set);
 
 int mtk_pinconf_bias_disable_get(struct mtk_pinctrl *hw,
 				 const struct mtk_pin_desc *desc, int *res)
@@ -402,6 +407,7 @@ int mtk_pinconf_bias_disable_get(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_disable_get);
 
 int mtk_pinconf_bias_set(struct mtk_pinctrl *hw,
 			 const struct mtk_pin_desc *desc, bool pullup)
@@ -421,6 +427,7 @@ int mtk_pinconf_bias_set(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_set);
 
 int mtk_pinconf_bias_get(struct mtk_pinctrl *hw,
 			 const struct mtk_pin_desc *desc, bool pullup, int *res)
@@ -440,6 +447,7 @@ int mtk_pinconf_bias_get(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_get);
 
 /* Revision 1 */
 int mtk_pinconf_bias_disable_set_rev1(struct mtk_pinctrl *hw,
@@ -454,6 +462,7 @@ int mtk_pinconf_bias_disable_set_rev1(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_disable_set_rev1);
 
 int mtk_pinconf_bias_disable_get_rev1(struct mtk_pinctrl *hw,
 				      const struct mtk_pin_desc *desc, int *res)
@@ -471,6 +480,7 @@ int mtk_pinconf_bias_disable_get_rev1(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_disable_get_rev1);
 
 int mtk_pinconf_bias_set_rev1(struct mtk_pinctrl *hw,
 			      const struct mtk_pin_desc *desc, bool pullup)
@@ -490,6 +500,7 @@ int mtk_pinconf_bias_set_rev1(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_set_rev1);
 
 int mtk_pinconf_bias_get_rev1(struct mtk_pinctrl *hw,
 			      const struct mtk_pin_desc *desc, bool pullup,
@@ -515,6 +526,7 @@ int mtk_pinconf_bias_get_rev1(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_get_rev1);
 
 /* Combo for the following pull register type:
  * 1. PU + PD
@@ -715,6 +727,7 @@ int mtk_pinconf_bias_set_combo(struct mtk_pinctrl *hw,
 out:
 	return err;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_set_combo);
 
 int mtk_pinconf_bias_get_combo(struct mtk_pinctrl *hw,
 			      const struct mtk_pin_desc *desc,
@@ -735,6 +748,7 @@ int mtk_pinconf_bias_get_combo(struct mtk_pinctrl *hw,
 out:
 	return err;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_bias_get_combo);
 
 /* Revision 0 */
 int mtk_pinconf_drive_set(struct mtk_pinctrl *hw,
@@ -764,6 +778,7 @@ int mtk_pinconf_drive_set(struct mtk_pinctrl *hw,
 
 	return err;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_drive_set);
 
 int mtk_pinconf_drive_get(struct mtk_pinctrl *hw,
 			  const struct mtk_pin_desc *desc, int *val)
@@ -788,6 +803,7 @@ int mtk_pinconf_drive_get(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_drive_get);
 
 /* Revision 1 */
 int mtk_pinconf_drive_set_rev1(struct mtk_pinctrl *hw,
@@ -809,6 +825,7 @@ int mtk_pinconf_drive_set_rev1(struct mtk_pinctrl *hw,
 
 	return err;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_drive_set_rev1);
 
 int mtk_pinconf_drive_get_rev1(struct mtk_pinctrl *hw,
 			       const struct mtk_pin_desc *desc, int *val)
@@ -826,18 +843,21 @@ int mtk_pinconf_drive_get_rev1(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_drive_get_rev1);
 
 int mtk_pinconf_drive_set_raw(struct mtk_pinctrl *hw,
 			       const struct mtk_pin_desc *desc, u32 arg)
 {
 	return mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DRV, arg);
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_drive_set_raw);
 
 int mtk_pinconf_drive_get_raw(struct mtk_pinctrl *hw,
 			       const struct mtk_pin_desc *desc, int *val)
 {
 	return mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DRV, val);
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_drive_get_raw);
 
 int mtk_pinconf_adv_pull_set(struct mtk_pinctrl *hw,
 			     const struct mtk_pin_desc *desc, bool pullup,
@@ -878,6 +898,7 @@ int mtk_pinconf_adv_pull_set(struct mtk_pinctrl *hw,
 
 	return err;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_adv_pull_set);
 
 int mtk_pinconf_adv_pull_get(struct mtk_pinctrl *hw,
 			     const struct mtk_pin_desc *desc, bool pullup,
@@ -920,6 +941,7 @@ int mtk_pinconf_adv_pull_get(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_adv_pull_get);
 
 int mtk_pinconf_adv_drive_set(struct mtk_pinctrl *hw,
 			      const struct mtk_pin_desc *desc, u32 arg)
@@ -946,6 +968,7 @@ int mtk_pinconf_adv_drive_set(struct mtk_pinctrl *hw,
 
 	return err;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_adv_drive_set);
 
 int mtk_pinconf_adv_drive_get(struct mtk_pinctrl *hw,
 			      const struct mtk_pin_desc *desc, u32 *val)
@@ -969,3 +992,8 @@ int mtk_pinconf_adv_drive_get(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_pinconf_adv_drive_get);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
+MODULE_DESCRIPTION("Pin configuration library module for mediatek SoCs");
diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c
index ee305f140400..90a432bf9fed 100644
--- a/drivers/pinctrl/mediatek/pinctrl-paris.c
+++ b/drivers/pinctrl/mediatek/pinctrl-paris.c
@@ -10,6 +10,7 @@
  */
 
 #include <linux/gpio/driver.h>
+#include <linux/module.h>
 #include <dt-bindings/pinctrl/mt65xx.h>
 #include "pinctrl-paris.h"
 
@@ -631,6 +632,7 @@ ssize_t mtk_pctrl_show_one_pin(struct mtk_pinctrl *hw,
 
 	return len;
 }
+EXPORT_SYMBOL_GPL(mtk_pctrl_show_one_pin);
 
 #define PIN_DBG_BUF_SZ 96
 static void mtk_pctrl_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s,
@@ -1019,6 +1021,7 @@ int mtk_paris_pinctrl_probe(struct platform_device *pdev,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mtk_paris_pinctrl_probe);
 
 static int mtk_paris_pinctrl_suspend(struct device *device)
 {
@@ -1038,3 +1041,6 @@ const struct dev_pm_ops mtk_paris_pinctrl_pm_ops = {
 	.suspend_noirq = mtk_paris_pinctrl_suspend,
 	.resume_noirq = mtk_paris_pinctrl_resume,
 };
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("MediaTek Pinctrl Common Driver V2 Paris");
diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c
index bbc919bef2bf..079f8ee8d353 100644
--- a/drivers/pinctrl/meson/pinctrl-meson.c
+++ b/drivers/pinctrl/meson/pinctrl-meson.c
@@ -549,6 +549,18 @@ static const struct pinconf_ops meson_pinconf_ops = {
 	.is_generic		= true,
 };
 
+static int meson_gpio_get_direction(struct gpio_chip *chip, unsigned gpio)
+{
+	struct meson_pinctrl *pc = gpiochip_get_data(chip);
+	int ret;
+
+	ret = meson_pinconf_get_output(pc, gpio);
+	if (ret < 0)
+		return ret;
+
+	return ret ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
+}
+
 static int meson_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
 	return meson_pinconf_set_output(gpiochip_get_data(chip), gpio, false);
@@ -591,6 +603,8 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc)
 	pc->chip.parent = pc->dev;
 	pc->chip.request = gpiochip_generic_request;
 	pc->chip.free = gpiochip_generic_free;
+	pc->chip.set_config = gpiochip_generic_config;
+	pc->chip.get_direction = meson_gpio_get_direction;
 	pc->chip.direction_input = meson_gpio_direction_input;
 	pc->chip.direction_output = meson_gpio_direction_output;
 	pc->chip.get = meson_gpio_get;
diff --git a/drivers/pinctrl/nomadik/pinctrl-ab8505.c b/drivers/pinctrl/nomadik/pinctrl-ab8505.c
index 5e6e7d28390a..b93af1fb37f0 100644
--- a/drivers/pinctrl/nomadik/pinctrl-ab8505.c
+++ b/drivers/pinctrl/nomadik/pinctrl-ab8505.c
@@ -178,6 +178,7 @@ static const struct abx500_pingroup ab8505_groups[] = {
 	AB8505_PIN_GROUP(gpio40_a_1, ABX500_ALT_A),
 	AB8505_PIN_GROUP(gpio41_a_1, ABX500_ALT_A),
 	AB8505_PIN_GROUP(uartrxdata_a_1, ABX500_ALT_A),
+	AB8505_PIN_GROUP(gpio50_a_1, ABX500_ALT_A),
 	AB8505_PIN_GROUP(gpio52_a_1, ABX500_ALT_A),
 	AB8505_PIN_GROUP(gpio53_a_1, ABX500_ALT_A),
 	AB8505_PIN_GROUP(pdmdata_b_1, ABX500_ALT_B),
diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c
index b9246e0b4fe2..acad3887cc74 100644
--- a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c
+++ b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c
@@ -691,18 +691,21 @@ static const struct nmk_pingroup nmk_db8500_groups[] = {
 	DB8500_PIN_GROUP(lcd_d8_d11_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(lcd_d12_d23_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(kp_a_1, NMK_GPIO_ALT_A),
+	DB8500_PIN_GROUP(kpskaskb_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(mc2_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(ssp1_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(ssp0_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(ipgpio0_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(ipgpio1_a_1, NMK_GPIO_ALT_A),
+	DB8500_PIN_GROUP(modem_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(kp_a_2, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(msp2sck_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(msp2_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(mc4_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(mc1_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(mc1_a_2, NMK_GPIO_ALT_A),
+	DB8500_PIN_GROUP(mc1dir_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(hsir_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(hsit_a_1, NMK_GPIO_ALT_A),
 	DB8500_PIN_GROUP(hsit_a_2, NMK_GPIO_ALT_A),
@@ -760,7 +763,7 @@ static const struct nmk_pingroup nmk_db8500_groups[] = {
 	DB8500_PIN_GROUP(u0_c_1, NMK_GPIO_ALT_C),
 	DB8500_PIN_GROUP(ipgpio4_c_1, NMK_GPIO_ALT_C),
 	DB8500_PIN_GROUP(ipgpio5_c_1, NMK_GPIO_ALT_C),
-	DB8500_PIN_GROUP(ipgpio6_c_1, NMK_GPIO_ALT_C),
+	DB8500_PIN_GROUP(ipgpio6_c_2, NMK_GPIO_ALT_C),
 	DB8500_PIN_GROUP(ipgpio7_c_1, NMK_GPIO_ALT_C),
 	DB8500_PIN_GROUP(smcleale_c_1, NMK_GPIO_ALT_C),
 	DB8500_PIN_GROUP(stmape_c_1, NMK_GPIO_ALT_C),
@@ -955,6 +958,7 @@ static const struct nmk_function nmk_db8500_functions[] = {
 	FUNCTION(spi0),
 	FUNCTION(spi2),
 	FUNCTION(remap),
+	FUNCTION(sbag),
 	FUNCTION(ptm),
 	FUNCTION(rf),
 	FUNCTION(hx),
diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
index ca7bbe4164c0..ba25c4654391 100644
--- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c
+++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
@@ -1343,8 +1343,6 @@ static const struct nmk_cfg_param nmk_cfg_params[] = {
 
 static int nmk_dt_pin_config(int index, int val, unsigned long *config)
 {
-	int ret = 0;
-
 	if (nmk_cfg_params[index].choice == NULL)
 		*config = nmk_cfg_params[index].config;
 	else {
@@ -1354,7 +1352,7 @@ static int nmk_dt_pin_config(int index, int val, unsigned long *config)
 				nmk_cfg_params[index].choice[val];
 		}
 	}
-	return ret;
+	return 0;
 }
 
 static const char *nmk_find_pin_name(struct pinctrl_dev *pctldev, const char *pin_name)
diff --git a/drivers/pinctrl/pinctrl-at91-pio4.c b/drivers/pinctrl/pinctrl-at91-pio4.c
index 694912409fd9..54222ccddfb1 100644
--- a/drivers/pinctrl/pinctrl-at91-pio4.c
+++ b/drivers/pinctrl/pinctrl-at91-pio4.c
@@ -1019,7 +1019,7 @@ static int atmel_pinctrl_probe(struct platform_device *pdev)
 
 	atmel_pioctrl->reg_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(atmel_pioctrl->reg_base))
-		return -EINVAL;
+		return PTR_ERR(atmel_pioctrl->reg_base);
 
 	atmel_pioctrl->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(atmel_pioctrl->clk)) {
diff --git a/drivers/pinctrl/pinctrl-bm1880.c b/drivers/pinctrl/pinctrl-bm1880.c
index f7dff4f14101..d1a7d9836787 100644
--- a/drivers/pinctrl/pinctrl-bm1880.c
+++ b/drivers/pinctrl/pinctrl-bm1880.c
@@ -408,6 +408,7 @@ static const struct bm1880_pctrl_group bm1880_pctrl_groups[] = {
 	BM1880_PINCTRL_GRP(pwm34),
 	BM1880_PINCTRL_GRP(pwm35),
 	BM1880_PINCTRL_GRP(pwm36),
+	BM1880_PINCTRL_GRP(pwm37),
 	BM1880_PINCTRL_GRP(i2c0),
 	BM1880_PINCTRL_GRP(i2c1),
 	BM1880_PINCTRL_GRP(i2c2),
diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c
index e5dcf77fe43d..6a8d44504f94 100644
--- a/drivers/pinctrl/pinctrl-ingenic.c
+++ b/drivers/pinctrl/pinctrl-ingenic.c
@@ -1977,6 +1977,25 @@ static const struct pinctrl_ops ingenic_pctlops = {
 	.dt_free_map = pinconf_generic_dt_free_map,
 };
 
+static int ingenic_gpio_irq_request(struct irq_data *data)
+{
+	struct gpio_chip *gpio_chip = irq_data_get_irq_chip_data(data);
+	int ret;
+
+	ret = ingenic_gpio_direction_input(gpio_chip, data->hwirq);
+	if (ret)
+		return ret;
+
+	return gpiochip_reqres_irq(gpio_chip, data->hwirq);
+}
+
+static void ingenic_gpio_irq_release(struct irq_data *data)
+{
+	struct gpio_chip *gpio_chip = irq_data_get_irq_chip_data(data);
+
+	return gpiochip_relres_irq(gpio_chip, data->hwirq);
+}
+
 static int ingenic_pinmux_set_pin_fn(struct ingenic_pinctrl *jzpc,
 		int pin, int func)
 {
@@ -2338,6 +2357,8 @@ static int __init ingenic_gpio_probe(struct ingenic_pinctrl *jzpc,
 	jzgc->irq_chip.irq_ack = ingenic_gpio_irq_ack;
 	jzgc->irq_chip.irq_set_type = ingenic_gpio_irq_set_type;
 	jzgc->irq_chip.irq_set_wake = ingenic_gpio_irq_set_wake;
+	jzgc->irq_chip.irq_request_resources = ingenic_gpio_irq_request;
+	jzgc->irq_chip.irq_release_resources = ingenic_gpio_irq_release;
 	jzgc->irq_chip.flags = IRQCHIP_MASK_ON_SUSPEND;
 
 	girq = &jzgc->gc.irq;
diff --git a/drivers/pinctrl/pinctrl-lantiq.c b/drivers/pinctrl/pinctrl-lantiq.c
index aa92f141b865..626e02d7a1ba 100644
--- a/drivers/pinctrl/pinctrl-lantiq.c
+++ b/drivers/pinctrl/pinctrl-lantiq.c
@@ -221,7 +221,7 @@ static int match_mux(const struct ltq_mfp_pin *mfp, unsigned mux)
 	return i;
 }
 
-/* dont assume .mfp is linearly mapped. find the mfp with the correct .pin */
+/* don't assume .mfp is linearly mapped. find the mfp with the correct .pin */
 static int match_mfp(const struct ltq_pinmux_info *info, int pin)
 {
 	int i;
diff --git a/drivers/pinctrl/pinctrl-mcp23s08.c b/drivers/pinctrl/pinctrl-mcp23s08.c
index 3a235487e38d..151931b593f6 100644
--- a/drivers/pinctrl/pinctrl-mcp23s08.c
+++ b/drivers/pinctrl/pinctrl-mcp23s08.c
@@ -1,34 +1,23 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /* MCP23S08 SPI/I2C GPIO driver */
 
+#include <linux/bitops.h>
 #include <linux/kernel.h>
 #include <linux/device.h>
 #include <linux/mutex.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/export.h>
 #include <linux/gpio/driver.h>
-#include <linux/i2c.h>
-#include <linux/spi/spi.h>
-#include <linux/spi/mcp23s08.h>
 #include <linux/slab.h>
 #include <asm/byteorder.h>
 #include <linux/interrupt.h>
-#include <linux/of_device.h>
 #include <linux/regmap.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/pinctrl/pinconf.h>
 #include <linux/pinctrl/pinconf-generic.h>
 
-/*
- * MCP types supported by driver
- */
-#define MCP_TYPE_S08	0
-#define MCP_TYPE_S17	1
-#define MCP_TYPE_008	2
-#define MCP_TYPE_017	3
-#define MCP_TYPE_S18    4
-#define MCP_TYPE_018    5
-
-#define MCP_MAX_DEV_PER_CS	8
+#include "pinctrl-mcp23s08.h"
 
 /* Registers are all 8 bits wide.
  *
@@ -53,31 +42,6 @@
 #define MCP_GPIO	0x09
 #define MCP_OLAT	0x0a
 
-struct mcp23s08;
-
-struct mcp23s08 {
-	u8			addr;
-	bool			irq_active_high;
-	bool			reg_shift;
-
-	u16			irq_rise;
-	u16			irq_fall;
-	int			irq;
-	bool			irq_controller;
-	int			cached_gpio;
-	/* lock protects regmap access with bypass/cache flags */
-	struct mutex		lock;
-
-	struct gpio_chip	chip;
-	struct irq_chip		irq_chip;
-
-	struct regmap		*regmap;
-	struct device		*dev;
-
-	struct pinctrl_dev	*pctldev;
-	struct pinctrl_desc	pinctrl_desc;
-};
-
 static const struct reg_default mcp23x08_defaults[] = {
 	{.reg = MCP_IODIR,		.def = 0xff},
 	{.reg = MCP_IPOL,		.def = 0x00},
@@ -109,7 +73,7 @@ static const struct regmap_access_table mcp23x08_precious_table = {
 	.n_yes_ranges = 1,
 };
 
-static const struct regmap_config mcp23x08_regmap = {
+const struct regmap_config mcp23x08_regmap = {
 	.reg_bits = 8,
 	.val_bits = 8,
 
@@ -121,6 +85,7 @@ static const struct regmap_config mcp23x08_regmap = {
 	.cache_type = REGCACHE_FLAT,
 	.max_register = MCP_OLAT,
 };
+EXPORT_SYMBOL_GPL(mcp23x08_regmap);
 
 static const struct reg_default mcp23x16_defaults[] = {
 	{.reg = MCP_IODIR << 1,		.def = 0xffff},
@@ -153,7 +118,7 @@ static const struct regmap_access_table mcp23x16_precious_table = {
 	.n_yes_ranges = 1,
 };
 
-static const struct regmap_config mcp23x17_regmap = {
+const struct regmap_config mcp23x17_regmap = {
 	.reg_bits = 8,
 	.val_bits = 16,
 
@@ -166,6 +131,7 @@ static const struct regmap_config mcp23x17_regmap = {
 	.cache_type = REGCACHE_FLAT,
 	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
+EXPORT_SYMBOL_GPL(mcp23x17_regmap);
 
 static int mcp_read(struct mcp23s08 *mcp, unsigned int reg, unsigned int *val)
 {
@@ -309,80 +275,6 @@ static const struct pinconf_ops mcp_pinconf_ops = {
 
 /*----------------------------------------------------------------------*/
 
-#ifdef CONFIG_SPI_MASTER
-
-static int mcp23sxx_spi_write(void *context, const void *data, size_t count)
-{
-	struct mcp23s08 *mcp = context;
-	struct spi_device *spi = to_spi_device(mcp->dev);
-	struct spi_message m;
-	struct spi_transfer t[2] = { { .tx_buf = &mcp->addr, .len = 1, },
-				     { .tx_buf = data, .len = count, }, };
-
-	spi_message_init(&m);
-	spi_message_add_tail(&t[0], &m);
-	spi_message_add_tail(&t[1], &m);
-
-	return spi_sync(spi, &m);
-}
-
-static int mcp23sxx_spi_gather_write(void *context,
-				const void *reg, size_t reg_size,
-				const void *val, size_t val_size)
-{
-	struct mcp23s08 *mcp = context;
-	struct spi_device *spi = to_spi_device(mcp->dev);
-	struct spi_message m;
-	struct spi_transfer t[3] = { { .tx_buf = &mcp->addr, .len = 1, },
-				     { .tx_buf = reg, .len = reg_size, },
-				     { .tx_buf = val, .len = val_size, }, };
-
-	spi_message_init(&m);
-	spi_message_add_tail(&t[0], &m);
-	spi_message_add_tail(&t[1], &m);
-	spi_message_add_tail(&t[2], &m);
-
-	return spi_sync(spi, &m);
-}
-
-static int mcp23sxx_spi_read(void *context, const void *reg, size_t reg_size,
-				void *val, size_t val_size)
-{
-	struct mcp23s08 *mcp = context;
-	struct spi_device *spi = to_spi_device(mcp->dev);
-	u8 tx[2];
-
-	if (reg_size != 1)
-		return -EINVAL;
-
-	tx[0] = mcp->addr | 0x01;
-	tx[1] = *((u8 *) reg);
-
-	return spi_write_then_read(spi, tx, sizeof(tx), val, val_size);
-}
-
-static const struct regmap_bus mcp23sxx_spi_regmap = {
-	.write = mcp23sxx_spi_write,
-	.gather_write = mcp23sxx_spi_gather_write,
-	.read = mcp23sxx_spi_read,
-};
-
-#endif /* CONFIG_SPI_MASTER */
-
-/*----------------------------------------------------------------------*/
-
-/* A given spi_device can represent up to eight mcp23sxx chips
- * sharing the same chipselect but using different addresses
- * (e.g. chips #0 and #3 might be populated, but not #1 or $2).
- * Driver data holds all the per-chip data.
- */
-struct mcp23s08_driver_data {
-	unsigned		ngpio;
-	struct mcp23s08		*mcp[8];
-	struct mcp23s08		chip[];
-};
-
-
 static int mcp23s08_direction_input(struct gpio_chip *chip, unsigned offset)
 {
 	struct mcp23s08	*mcp = gpiochip_get_data(chip);
@@ -562,7 +454,6 @@ static int mcp23s08_irq_set_type(struct irq_data *data, unsigned int type)
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
 	struct mcp23s08 *mcp = gpiochip_get_data(gc);
 	unsigned int pos = data->hwirq;
-	int status = 0;
 
 	if ((type & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH) {
 		mcp_set_bit(mcp, MCP_INTCON, pos, false);
@@ -585,7 +476,7 @@ static int mcp23s08_irq_set_type(struct irq_data *data, unsigned int type)
 	} else
 		return -EINVAL;
 
-	return status;
+	return 0;
 }
 
 static void mcp23s08_irq_bus_lock(struct irq_data *data)
@@ -656,21 +547,25 @@ static int mcp23s08_irqchip_setup(struct mcp23s08 *mcp)
 
 /*----------------------------------------------------------------------*/
 
-static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
-			      void *data, unsigned addr, unsigned type,
-			      unsigned int base, int cs)
+int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
+		       unsigned int addr, unsigned int type, unsigned int base)
 {
 	int status, ret;
 	bool mirror = false;
 	bool open_drain = false;
-	struct regmap_config *one_regmap_config = NULL;
-	int raw_chip_address = (addr & ~0x40) >> 1;
 
 	mutex_init(&mcp->lock);
 
 	mcp->dev = dev;
 	mcp->addr = addr;
+
 	mcp->irq_active_high = false;
+	mcp->irq_chip.name = dev_name(dev);
+	mcp->irq_chip.irq_mask = mcp23s08_irq_mask;
+	mcp->irq_chip.irq_unmask = mcp23s08_irq_unmask;
+	mcp->irq_chip.irq_set_type = mcp23s08_irq_set_type;
+	mcp->irq_chip.irq_bus_lock = mcp23s08_irq_bus_lock;
+	mcp->irq_chip.irq_bus_sync_unlock = mcp23s08_irq_bus_unlock;
 
 	mcp->chip.direction_input = mcp23s08_direction_input;
 	mcp->chip.get = mcp23s08_get;
@@ -681,83 +576,6 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
 	mcp->chip.of_node = dev->of_node;
 #endif
 
-	switch (type) {
-#ifdef CONFIG_SPI_MASTER
-	case MCP_TYPE_S08:
-	case MCP_TYPE_S17:
-		switch (type) {
-		case MCP_TYPE_S08:
-			one_regmap_config =
-				devm_kmemdup(dev, &mcp23x08_regmap,
-					sizeof(struct regmap_config), GFP_KERNEL);
-			mcp->reg_shift = 0;
-			mcp->chip.ngpio = 8;
-			mcp->chip.label = devm_kasprintf(dev, GFP_KERNEL,
-					"mcp23s08.%d", raw_chip_address);
-			break;
-		case MCP_TYPE_S17:
-			one_regmap_config =
-				devm_kmemdup(dev, &mcp23x17_regmap,
-					sizeof(struct regmap_config), GFP_KERNEL);
-			mcp->reg_shift = 1;
-			mcp->chip.ngpio = 16;
-			mcp->chip.label = devm_kasprintf(dev, GFP_KERNEL,
-					"mcp23s17.%d", raw_chip_address);
-			break;
-		}
-		if (!one_regmap_config)
-			return -ENOMEM;
-
-		one_regmap_config->name = devm_kasprintf(dev, GFP_KERNEL, "%d", raw_chip_address);
-		mcp->regmap = devm_regmap_init(dev, &mcp23sxx_spi_regmap, mcp,
-					       one_regmap_config);
-		break;
-
-	case MCP_TYPE_S18:
-		one_regmap_config =
-			devm_kmemdup(dev, &mcp23x17_regmap,
-				sizeof(struct regmap_config), GFP_KERNEL);
-		if (!one_regmap_config)
-			return -ENOMEM;
-		mcp->regmap = devm_regmap_init(dev, &mcp23sxx_spi_regmap, mcp,
-					       one_regmap_config);
-		mcp->reg_shift = 1;
-		mcp->chip.ngpio = 16;
-		mcp->chip.label = "mcp23s18";
-		break;
-#endif /* CONFIG_SPI_MASTER */
-
-#if IS_ENABLED(CONFIG_I2C)
-	case MCP_TYPE_008:
-		mcp->regmap = devm_regmap_init_i2c(data, &mcp23x08_regmap);
-		mcp->reg_shift = 0;
-		mcp->chip.ngpio = 8;
-		mcp->chip.label = "mcp23008";
-		break;
-
-	case MCP_TYPE_017:
-		mcp->regmap = devm_regmap_init_i2c(data, &mcp23x17_regmap);
-		mcp->reg_shift = 1;
-		mcp->chip.ngpio = 16;
-		mcp->chip.label = "mcp23017";
-		break;
-
-	case MCP_TYPE_018:
-		mcp->regmap = devm_regmap_init_i2c(data, &mcp23x17_regmap);
-		mcp->reg_shift = 1;
-		mcp->chip.ngpio = 16;
-		mcp->chip.label = "mcp23018";
-		break;
-#endif /* CONFIG_I2C */
-
-	default:
-		dev_err(dev, "invalid device type (%d)\n", type);
-		return -EINVAL;
-	}
-
-	if (IS_ERR(mcp->regmap))
-		return PTR_ERR(mcp->regmap);
-
 	mcp->chip.base = base;
 	mcp->chip.can_sleep = true;
 	mcp->chip.parent = dev;
@@ -816,14 +634,6 @@ static int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
 			goto fail;
 	}
 
-	if (one_regmap_config) {
-		mcp->pinctrl_desc.name = devm_kasprintf(dev, GFP_KERNEL,
-				"mcp23xxx-pinctrl.%d", raw_chip_address);
-		if (!mcp->pinctrl_desc.name)
-			return -ENOMEM;
-	} else {
-		mcp->pinctrl_desc.name = "mcp23xxx-pinctrl";
-	}
 	mcp->pinctrl_desc.pctlops = &mcp_pinctrl_ops;
 	mcp->pinctrl_desc.confops = &mcp_pinconf_ops;
 	mcp->pinctrl_desc.npins = mcp->chip.ngpio;
@@ -847,291 +657,5 @@ fail:
 		dev_dbg(dev, "can't setup chip %d, --> %d\n", addr, ret);
 	return ret;
 }
-
-/*----------------------------------------------------------------------*/
-
-#ifdef CONFIG_OF
-#ifdef CONFIG_SPI_MASTER
-static const struct of_device_id mcp23s08_spi_of_match[] = {
-	{
-		.compatible = "microchip,mcp23s08",
-		.data = (void *) MCP_TYPE_S08,
-	},
-	{
-		.compatible = "microchip,mcp23s17",
-		.data = (void *) MCP_TYPE_S17,
-	},
-	{
-		.compatible = "microchip,mcp23s18",
-		.data = (void *) MCP_TYPE_S18,
-	},
-/* NOTE: The use of the mcp prefix is deprecated and will be removed. */
-	{
-		.compatible = "mcp,mcp23s08",
-		.data = (void *) MCP_TYPE_S08,
-	},
-	{
-		.compatible = "mcp,mcp23s17",
-		.data = (void *) MCP_TYPE_S17,
-	},
-	{ },
-};
-MODULE_DEVICE_TABLE(of, mcp23s08_spi_of_match);
-#endif
-
-#if IS_ENABLED(CONFIG_I2C)
-static const struct of_device_id mcp23s08_i2c_of_match[] = {
-	{
-		.compatible = "microchip,mcp23008",
-		.data = (void *) MCP_TYPE_008,
-	},
-	{
-		.compatible = "microchip,mcp23017",
-		.data = (void *) MCP_TYPE_017,
-	},
-	{
-		.compatible = "microchip,mcp23018",
-		.data = (void *) MCP_TYPE_018,
-	},
-/* NOTE: The use of the mcp prefix is deprecated and will be removed. */
-	{
-		.compatible = "mcp,mcp23008",
-		.data = (void *) MCP_TYPE_008,
-	},
-	{
-		.compatible = "mcp,mcp23017",
-		.data = (void *) MCP_TYPE_017,
-	},
-	{ },
-};
-MODULE_DEVICE_TABLE(of, mcp23s08_i2c_of_match);
-#endif
-#endif /* CONFIG_OF */
-
-
-#if IS_ENABLED(CONFIG_I2C)
-
-static int mcp230xx_probe(struct i2c_client *client,
-				    const struct i2c_device_id *id)
-{
-	struct mcp23s08_platform_data *pdata, local_pdata;
-	struct mcp23s08 *mcp;
-	int status;
-
-	pdata = dev_get_platdata(&client->dev);
-	if (!pdata) {
-		pdata = &local_pdata;
-		pdata->base = -1;
-	}
-
-	mcp = devm_kzalloc(&client->dev, sizeof(*mcp), GFP_KERNEL);
-	if (!mcp)
-		return -ENOMEM;
-
-	mcp->irq = client->irq;
-	mcp->irq_chip.name = dev_name(&client->dev);
-	mcp->irq_chip.irq_mask = mcp23s08_irq_mask;
-	mcp->irq_chip.irq_unmask = mcp23s08_irq_unmask;
-	mcp->irq_chip.irq_set_type = mcp23s08_irq_set_type;
-	mcp->irq_chip.irq_bus_lock = mcp23s08_irq_bus_lock;
-	mcp->irq_chip.irq_bus_sync_unlock = mcp23s08_irq_bus_unlock;
-
-	status = mcp23s08_probe_one(mcp, &client->dev, client, client->addr,
-				    id->driver_data, pdata->base, 0);
-	if (status)
-		return status;
-
-	i2c_set_clientdata(client, mcp);
-
-	return 0;
-}
-
-static const struct i2c_device_id mcp230xx_id[] = {
-	{ "mcp23008", MCP_TYPE_008 },
-	{ "mcp23017", MCP_TYPE_017 },
-	{ "mcp23018", MCP_TYPE_018 },
-	{ },
-};
-MODULE_DEVICE_TABLE(i2c, mcp230xx_id);
-
-static struct i2c_driver mcp230xx_driver = {
-	.driver = {
-		.name	= "mcp230xx",
-		.of_match_table = of_match_ptr(mcp23s08_i2c_of_match),
-	},
-	.probe		= mcp230xx_probe,
-	.id_table	= mcp230xx_id,
-};
-
-static int __init mcp23s08_i2c_init(void)
-{
-	return i2c_add_driver(&mcp230xx_driver);
-}
-
-static void mcp23s08_i2c_exit(void)
-{
-	i2c_del_driver(&mcp230xx_driver);
-}
-
-#else
-
-static int __init mcp23s08_i2c_init(void) { return 0; }
-static void mcp23s08_i2c_exit(void) { }
-
-#endif /* CONFIG_I2C */
-
-/*----------------------------------------------------------------------*/
-
-#ifdef CONFIG_SPI_MASTER
-
-static int mcp23s08_probe(struct spi_device *spi)
-{
-	struct mcp23s08_platform_data	*pdata, local_pdata;
-	unsigned			addr;
-	int				chips = 0;
-	struct mcp23s08_driver_data	*data;
-	int				status, type;
-	unsigned			ngpio = 0;
-	const struct			of_device_id *match;
-
-	match = of_match_device(of_match_ptr(mcp23s08_spi_of_match), &spi->dev);
-	if (match)
-		type = (int)(uintptr_t)match->data;
-	else
-		type = spi_get_device_id(spi)->driver_data;
-
-	pdata = dev_get_platdata(&spi->dev);
-	if (!pdata) {
-		pdata = &local_pdata;
-		pdata->base = -1;
-
-		status = device_property_read_u32(&spi->dev,
-			"microchip,spi-present-mask", &pdata->spi_present_mask);
-		if (status) {
-			status = device_property_read_u32(&spi->dev,
-				"mcp,spi-present-mask",
-				&pdata->spi_present_mask);
-
-			if (status) {
-				dev_err(&spi->dev, "missing spi-present-mask");
-				return -ENODEV;
-			}
-		}
-	}
-
-	if (!pdata->spi_present_mask || pdata->spi_present_mask > 0xff) {
-		dev_err(&spi->dev, "invalid spi-present-mask");
-		return -ENODEV;
-	}
-
-	for (addr = 0; addr < MCP_MAX_DEV_PER_CS; addr++) {
-		if (pdata->spi_present_mask & BIT(addr))
-			chips++;
-	}
-
-	if (!chips)
-		return -ENODEV;
-
-	data = devm_kzalloc(&spi->dev,
-			    struct_size(data, chip, chips), GFP_KERNEL);
-	if (!data)
-		return -ENOMEM;
-
-	spi_set_drvdata(spi, data);
-
-	for (addr = 0; addr < MCP_MAX_DEV_PER_CS; addr++) {
-		if (!(pdata->spi_present_mask & BIT(addr)))
-			continue;
-		chips--;
-		data->mcp[addr] = &data->chip[chips];
-		data->mcp[addr]->irq = spi->irq;
-		data->mcp[addr]->irq_chip.name = dev_name(&spi->dev);
-		data->mcp[addr]->irq_chip.irq_mask = mcp23s08_irq_mask;
-		data->mcp[addr]->irq_chip.irq_unmask = mcp23s08_irq_unmask;
-		data->mcp[addr]->irq_chip.irq_set_type = mcp23s08_irq_set_type;
-		data->mcp[addr]->irq_chip.irq_bus_lock = mcp23s08_irq_bus_lock;
-		data->mcp[addr]->irq_chip.irq_bus_sync_unlock =
-			mcp23s08_irq_bus_unlock;
-		status = mcp23s08_probe_one(data->mcp[addr], &spi->dev, spi,
-					    0x40 | (addr << 1), type,
-					    pdata->base, addr);
-		if (status < 0)
-			return status;
-
-		if (pdata->base != -1)
-			pdata->base += data->mcp[addr]->chip.ngpio;
-		ngpio += data->mcp[addr]->chip.ngpio;
-	}
-	data->ngpio = ngpio;
-
-	return 0;
-}
-
-static const struct spi_device_id mcp23s08_ids[] = {
-	{ "mcp23s08", MCP_TYPE_S08 },
-	{ "mcp23s17", MCP_TYPE_S17 },
-	{ "mcp23s18", MCP_TYPE_S18 },
-	{ },
-};
-MODULE_DEVICE_TABLE(spi, mcp23s08_ids);
-
-static struct spi_driver mcp23s08_driver = {
-	.probe		= mcp23s08_probe,
-	.id_table	= mcp23s08_ids,
-	.driver = {
-		.name	= "mcp23s08",
-		.of_match_table = of_match_ptr(mcp23s08_spi_of_match),
-	},
-};
-
-static int __init mcp23s08_spi_init(void)
-{
-	return spi_register_driver(&mcp23s08_driver);
-}
-
-static void mcp23s08_spi_exit(void)
-{
-	spi_unregister_driver(&mcp23s08_driver);
-}
-
-#else
-
-static int __init mcp23s08_spi_init(void) { return 0; }
-static void mcp23s08_spi_exit(void) { }
-
-#endif /* CONFIG_SPI_MASTER */
-
-/*----------------------------------------------------------------------*/
-
-static int __init mcp23s08_init(void)
-{
-	int ret;
-
-	ret = mcp23s08_spi_init();
-	if (ret)
-		goto spi_fail;
-
-	ret = mcp23s08_i2c_init();
-	if (ret)
-		goto i2c_fail;
-
-	return 0;
-
- i2c_fail:
-	mcp23s08_spi_exit();
- spi_fail:
-	return ret;
-}
-/* register after spi/i2c postcore initcall and before
- * subsys initcalls that may rely on these GPIOs
- */
-subsys_initcall(mcp23s08_init);
-
-static void __exit mcp23s08_exit(void)
-{
-	mcp23s08_spi_exit();
-	mcp23s08_i2c_exit();
-}
-module_exit(mcp23s08_exit);
-
+EXPORT_SYMBOL_GPL(mcp23s08_probe_one);
 MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/pinctrl-mcp23s08.h b/drivers/pinctrl/pinctrl-mcp23s08.h
new file mode 100644
index 000000000000..90dc27081a3c
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-mcp23s08.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* MCP23S08 SPI/I2C GPIO driver */
+
+#include <linux/gpio/driver.h>
+#include <linux/irq.h>
+#include <linux/mutex.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/types.h>
+
+/*
+ * MCP types supported by driver
+ */
+#define MCP_TYPE_S08	1
+#define MCP_TYPE_S17	2
+#define MCP_TYPE_008	3
+#define MCP_TYPE_017	4
+#define MCP_TYPE_S18	5
+#define MCP_TYPE_018	6
+
+struct device;
+struct regmap;
+
+struct pinctrl_dev;
+
+struct mcp23s08 {
+	u8			addr;
+	bool			irq_active_high;
+	bool			reg_shift;
+
+	u16			irq_rise;
+	u16			irq_fall;
+	int			irq;
+	bool			irq_controller;
+	int			cached_gpio;
+	/* lock protects regmap access with bypass/cache flags */
+	struct mutex		lock;
+
+	struct gpio_chip	chip;
+	struct irq_chip		irq_chip;
+
+	struct regmap		*regmap;
+	struct device		*dev;
+
+	struct pinctrl_dev	*pctldev;
+	struct pinctrl_desc	pinctrl_desc;
+};
+
+extern const struct regmap_config mcp23x08_regmap;
+extern const struct regmap_config mcp23x17_regmap;
+
+int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev,
+		       unsigned int addr, unsigned int type, unsigned int base);
diff --git a/drivers/pinctrl/pinctrl-mcp23s08_i2c.c b/drivers/pinctrl/pinctrl-mcp23s08_i2c.c
new file mode 100644
index 000000000000..e0b001c8c08c
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-mcp23s08_i2c.c
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* MCP23S08 I2C GPIO driver */
+
+#include <linux/i2c.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#include "pinctrl-mcp23s08.h"
+
+static int mcp230xx_probe(struct i2c_client *client, const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	unsigned int type = id->driver_data;
+	struct mcp23s08 *mcp;
+	int ret;
+
+	mcp = devm_kzalloc(dev, sizeof(*mcp), GFP_KERNEL);
+	if (!mcp)
+		return -ENOMEM;
+
+	switch (type) {
+	case MCP_TYPE_008:
+		mcp->regmap = devm_regmap_init_i2c(client, &mcp23x08_regmap);
+		mcp->reg_shift = 0;
+		mcp->chip.ngpio = 8;
+		mcp->chip.label = "mcp23008";
+		break;
+
+	case MCP_TYPE_017:
+		mcp->regmap = devm_regmap_init_i2c(client, &mcp23x17_regmap);
+		mcp->reg_shift = 1;
+		mcp->chip.ngpio = 16;
+		mcp->chip.label = "mcp23017";
+		break;
+
+	case MCP_TYPE_018:
+		mcp->regmap = devm_regmap_init_i2c(client, &mcp23x17_regmap);
+		mcp->reg_shift = 1;
+		mcp->chip.ngpio = 16;
+		mcp->chip.label = "mcp23018";
+		break;
+
+	default:
+		dev_err(dev, "invalid device type (%d)\n", type);
+		return -EINVAL;
+	}
+
+	if (IS_ERR(mcp->regmap))
+		return PTR_ERR(mcp->regmap);
+
+	mcp->irq = client->irq;
+	mcp->pinctrl_desc.name = "mcp23xxx-pinctrl";
+
+	ret = mcp23s08_probe_one(mcp, dev, client->addr, type, -1);
+	if (ret)
+		return ret;
+
+	i2c_set_clientdata(client, mcp);
+
+	return 0;
+}
+
+static const struct i2c_device_id mcp230xx_id[] = {
+	{ "mcp23008", MCP_TYPE_008 },
+	{ "mcp23017", MCP_TYPE_017 },
+	{ "mcp23018", MCP_TYPE_018 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, mcp230xx_id);
+
+static const struct of_device_id mcp23s08_i2c_of_match[] = {
+	{
+		.compatible = "microchip,mcp23008",
+		.data = (void *) MCP_TYPE_008,
+	},
+	{
+		.compatible = "microchip,mcp23017",
+		.data = (void *) MCP_TYPE_017,
+	},
+	{
+		.compatible = "microchip,mcp23018",
+		.data = (void *) MCP_TYPE_018,
+	},
+/* NOTE: The use of the mcp prefix is deprecated and will be removed. */
+	{
+		.compatible = "mcp,mcp23008",
+		.data = (void *) MCP_TYPE_008,
+	},
+	{
+		.compatible = "mcp,mcp23017",
+		.data = (void *) MCP_TYPE_017,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mcp23s08_i2c_of_match);
+
+static struct i2c_driver mcp230xx_driver = {
+	.driver = {
+		.name	= "mcp230xx",
+		.of_match_table = mcp23s08_i2c_of_match,
+	},
+	.probe		= mcp230xx_probe,
+	.id_table	= mcp230xx_id,
+};
+
+static int __init mcp23s08_i2c_init(void)
+{
+	return i2c_add_driver(&mcp230xx_driver);
+}
+
+/*
+ * Register after I²C postcore initcall and before
+ * subsys initcalls that may rely on these GPIOs.
+ */
+subsys_initcall(mcp23s08_i2c_init);
+
+static void mcp23s08_i2c_exit(void)
+{
+	i2c_del_driver(&mcp230xx_driver);
+}
+module_exit(mcp23s08_i2c_exit);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/pinctrl-mcp23s08_spi.c b/drivers/pinctrl/pinctrl-mcp23s08_spi.c
new file mode 100644
index 000000000000..e06fb885fd2b
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-mcp23s08_spi.c
@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* MCP23S08 SPI GPIO driver */
+
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+
+#include "pinctrl-mcp23s08.h"
+
+#define MCP_MAX_DEV_PER_CS	8
+
+/*
+ * A given spi_device can represent up to eight mcp23sxx chips
+ * sharing the same chipselect but using different addresses
+ * (e.g. chips #0 and #3 might be populated, but not #1 or #2).
+ * Driver data holds all the per-chip data.
+ */
+struct mcp23s08_driver_data {
+	unsigned		ngpio;
+	struct mcp23s08		*mcp[8];
+	struct mcp23s08		chip[];
+};
+
+static int mcp23sxx_spi_write(void *context, const void *data, size_t count)
+{
+	struct mcp23s08 *mcp = context;
+	struct spi_device *spi = to_spi_device(mcp->dev);
+	struct spi_message m;
+	struct spi_transfer t[2] = { { .tx_buf = &mcp->addr, .len = 1, },
+				     { .tx_buf = data, .len = count, }, };
+
+	spi_message_init(&m);
+	spi_message_add_tail(&t[0], &m);
+	spi_message_add_tail(&t[1], &m);
+
+	return spi_sync(spi, &m);
+}
+
+static int mcp23sxx_spi_gather_write(void *context,
+				const void *reg, size_t reg_size,
+				const void *val, size_t val_size)
+{
+	struct mcp23s08 *mcp = context;
+	struct spi_device *spi = to_spi_device(mcp->dev);
+	struct spi_message m;
+	struct spi_transfer t[3] = { { .tx_buf = &mcp->addr, .len = 1, },
+				     { .tx_buf = reg, .len = reg_size, },
+				     { .tx_buf = val, .len = val_size, }, };
+
+	spi_message_init(&m);
+	spi_message_add_tail(&t[0], &m);
+	spi_message_add_tail(&t[1], &m);
+	spi_message_add_tail(&t[2], &m);
+
+	return spi_sync(spi, &m);
+}
+
+static int mcp23sxx_spi_read(void *context, const void *reg, size_t reg_size,
+				void *val, size_t val_size)
+{
+	struct mcp23s08 *mcp = context;
+	struct spi_device *spi = to_spi_device(mcp->dev);
+	u8 tx[2];
+
+	if (reg_size != 1)
+		return -EINVAL;
+
+	tx[0] = mcp->addr | 0x01;
+	tx[1] = *((u8 *) reg);
+
+	return spi_write_then_read(spi, tx, sizeof(tx), val, val_size);
+}
+
+static const struct regmap_bus mcp23sxx_spi_regmap = {
+	.write = mcp23sxx_spi_write,
+	.gather_write = mcp23sxx_spi_gather_write,
+	.read = mcp23sxx_spi_read,
+};
+
+static int mcp23s08_spi_regmap_init(struct mcp23s08 *mcp, struct device *dev,
+				    unsigned int addr, unsigned int type)
+{
+	const struct regmap_config *config;
+	struct regmap_config *copy;
+	const char *name;
+
+	switch (type) {
+	case MCP_TYPE_S08:
+		mcp->reg_shift = 0;
+		mcp->chip.ngpio = 8;
+		mcp->chip.label = devm_kasprintf(dev, GFP_KERNEL, "mcp23s08.%d", addr);
+
+		config = &mcp23x08_regmap;
+		name = devm_kasprintf(dev, GFP_KERNEL, "%d", addr);
+		break;
+
+	case MCP_TYPE_S17:
+		mcp->reg_shift = 1;
+		mcp->chip.ngpio = 16;
+		mcp->chip.label = devm_kasprintf(dev, GFP_KERNEL, "mcp23s17.%d", addr);
+
+		config = &mcp23x17_regmap;
+		name = devm_kasprintf(dev, GFP_KERNEL, "%d", addr);
+		break;
+
+	case MCP_TYPE_S18:
+		mcp->reg_shift = 1;
+		mcp->chip.ngpio = 16;
+		mcp->chip.label = "mcp23s18";
+
+		config = &mcp23x17_regmap;
+		name = config->name;
+		break;
+
+	default:
+		dev_err(dev, "invalid device type (%d)\n", type);
+		return -EINVAL;
+	}
+
+	copy = devm_kmemdup(dev, &config, sizeof(config), GFP_KERNEL);
+	if (!copy)
+		return -ENOMEM;
+
+	copy->name = name;
+
+	mcp->regmap = devm_regmap_init(dev, &mcp23sxx_spi_regmap, mcp, copy);
+	if (IS_ERR(mcp->regmap))
+		return PTR_ERR(mcp->regmap);
+
+	return 0;
+}
+
+static int mcp23s08_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct mcp23s08_driver_data *data;
+	unsigned long spi_present_mask;
+	const void *match;
+	unsigned int addr;
+	unsigned int ngpio = 0;
+	int chips;
+	int type;
+	int ret;
+	u32 v;
+
+	match = device_get_match_data(dev);
+	if (match)
+		type = (int)(uintptr_t)match;
+	else
+		type = spi_get_device_id(spi)->driver_data;
+
+	ret = device_property_read_u32(dev, "microchip,spi-present-mask", &v);
+	if (ret) {
+		ret = device_property_read_u32(dev, "mcp,spi-present-mask", &v);
+		if (ret) {
+			dev_err(dev, "missing spi-present-mask");
+			return ret;
+		}
+	}
+	spi_present_mask = v;
+
+	if (!spi_present_mask || spi_present_mask >= BIT(MCP_MAX_DEV_PER_CS)) {
+		dev_err(dev, "invalid spi-present-mask");
+		return -ENODEV;
+	}
+
+	chips = hweight_long(spi_present_mask);
+
+	data = devm_kzalloc(dev, struct_size(data, chip, chips), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, data);
+
+	for_each_set_bit(addr, &spi_present_mask, MCP_MAX_DEV_PER_CS) {
+		data->mcp[addr] = &data->chip[--chips];
+		data->mcp[addr]->irq = spi->irq;
+
+		ret = mcp23s08_spi_regmap_init(data->mcp[addr], dev, addr, type);
+		if (ret)
+			return ret;
+
+		data->mcp[addr]->pinctrl_desc.name = devm_kasprintf(dev, GFP_KERNEL,
+								    "mcp23xxx-pinctrl.%d",
+								    addr);
+		if (!data->mcp[addr]->pinctrl_desc.name)
+			return -ENOMEM;
+
+		ret = mcp23s08_probe_one(data->mcp[addr], dev, 0x40 | (addr << 1), type, -1);
+		if (ret < 0)
+			return ret;
+
+		ngpio += data->mcp[addr]->chip.ngpio;
+	}
+	data->ngpio = ngpio;
+
+	return 0;
+}
+
+static const struct spi_device_id mcp23s08_ids[] = {
+	{ "mcp23s08", MCP_TYPE_S08 },
+	{ "mcp23s17", MCP_TYPE_S17 },
+	{ "mcp23s18", MCP_TYPE_S18 },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, mcp23s08_ids);
+
+static const struct of_device_id mcp23s08_spi_of_match[] = {
+	{
+		.compatible = "microchip,mcp23s08",
+		.data = (void *) MCP_TYPE_S08,
+	},
+	{
+		.compatible = "microchip,mcp23s17",
+		.data = (void *) MCP_TYPE_S17,
+	},
+	{
+		.compatible = "microchip,mcp23s18",
+		.data = (void *) MCP_TYPE_S18,
+	},
+/* NOTE: The use of the mcp prefix is deprecated and will be removed. */
+	{
+		.compatible = "mcp,mcp23s08",
+		.data = (void *) MCP_TYPE_S08,
+	},
+	{
+		.compatible = "mcp,mcp23s17",
+		.data = (void *) MCP_TYPE_S17,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mcp23s08_spi_of_match);
+
+static struct spi_driver mcp23s08_driver = {
+	.probe		= mcp23s08_probe,
+	.id_table	= mcp23s08_ids,
+	.driver = {
+		.name	= "mcp23s08",
+		.of_match_table = mcp23s08_spi_of_match,
+	},
+};
+
+static int __init mcp23s08_spi_init(void)
+{
+	return spi_register_driver(&mcp23s08_driver);
+}
+
+/*
+ * Register after SPI postcore initcall and before
+ * subsys initcalls that may rely on these GPIOs.
+ */
+subsys_initcall(mcp23s08_spi_init);
+
+static void mcp23s08_spi_exit(void)
+{
+	spi_unregister_driver(&mcp23s08_driver);
+}
+module_exit(mcp23s08_spi_exit);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/pinctrl-ocelot.c b/drivers/pinctrl/pinctrl-ocelot.c
index ed8eac6c1494..95c225bc7572 100644
--- a/drivers/pinctrl/pinctrl-ocelot.c
+++ b/drivers/pinctrl/pinctrl-ocelot.c
@@ -46,32 +46,15 @@ enum {
 	FUNC_IRQ0_OUT,
 	FUNC_IRQ1_IN,
 	FUNC_IRQ1_OUT,
-	FUNC_MIIM1,
-	FUNC_MIIM2,
+	FUNC_MIIM,
 	FUNC_PCI_WAKE,
 	FUNC_PTP0,
 	FUNC_PTP1,
 	FUNC_PTP2,
 	FUNC_PTP3,
 	FUNC_PWM,
-	FUNC_RECO_CLK0,
-	FUNC_RECO_CLK1,
-	FUNC_SFP0,
-	FUNC_SFP1,
-	FUNC_SFP2,
-	FUNC_SFP3,
-	FUNC_SFP4,
-	FUNC_SFP5,
-	FUNC_SFP6,
-	FUNC_SFP7,
-	FUNC_SFP8,
-	FUNC_SFP9,
-	FUNC_SFP10,
-	FUNC_SFP11,
-	FUNC_SFP12,
-	FUNC_SFP13,
-	FUNC_SFP14,
-	FUNC_SFP15,
+	FUNC_RECO_CLK,
+	FUNC_SFP,
 	FUNC_SG0,
 	FUNC_SG1,
 	FUNC_SG2,
@@ -92,32 +75,15 @@ static const char *const ocelot_function_names[] = {
 	[FUNC_IRQ0_OUT]		= "irq0_out",
 	[FUNC_IRQ1_IN]		= "irq1_in",
 	[FUNC_IRQ1_OUT]		= "irq1_out",
-	[FUNC_MIIM1]		= "miim1",
-	[FUNC_MIIM2]		= "miim2",
+	[FUNC_MIIM]		= "miim",
 	[FUNC_PCI_WAKE]		= "pci_wake",
 	[FUNC_PTP0]		= "ptp0",
 	[FUNC_PTP1]		= "ptp1",
 	[FUNC_PTP2]		= "ptp2",
 	[FUNC_PTP3]		= "ptp3",
 	[FUNC_PWM]		= "pwm",
-	[FUNC_RECO_CLK0]	= "reco_clk0",
-	[FUNC_RECO_CLK1]	= "reco_clk1",
-	[FUNC_SFP0]		= "sfp0",
-	[FUNC_SFP1]		= "sfp1",
-	[FUNC_SFP2]		= "sfp2",
-	[FUNC_SFP3]		= "sfp3",
-	[FUNC_SFP4]		= "sfp4",
-	[FUNC_SFP5]		= "sfp5",
-	[FUNC_SFP6]		= "sfp6",
-	[FUNC_SFP7]		= "sfp7",
-	[FUNC_SFP8]		= "sfp8",
-	[FUNC_SFP9]		= "sfp9",
-	[FUNC_SFP10]		= "sfp10",
-	[FUNC_SFP11]		= "sfp11",
-	[FUNC_SFP12]		= "sfp12",
-	[FUNC_SFP13]		= "sfp13",
-	[FUNC_SFP14]		= "sfp14",
-	[FUNC_SFP15]		= "sfp15",
+	[FUNC_RECO_CLK]		= "reco_clk",
+	[FUNC_SFP]		= "sfp",
 	[FUNC_SG0]		= "sg0",
 	[FUNC_SG1]		= "sg1",
 	[FUNC_SG2]		= "sg2",
@@ -168,18 +134,18 @@ OCELOT_P(6,  UART,      TWI_SCL_M, NONE);
 OCELOT_P(7,  UART,      TWI_SCL_M, NONE);
 OCELOT_P(8,  SI,        TWI_SCL_M, IRQ0_OUT);
 OCELOT_P(9,  SI,        TWI_SCL_M, IRQ1_OUT);
-OCELOT_P(10, PTP2,      TWI_SCL_M, SFP0);
-OCELOT_P(11, PTP3,      TWI_SCL_M, SFP1);
-OCELOT_P(12, UART2,     TWI_SCL_M, SFP2);
-OCELOT_P(13, UART2,     TWI_SCL_M, SFP3);
-OCELOT_P(14, MIIM1,     TWI_SCL_M, SFP4);
-OCELOT_P(15, MIIM1,     TWI_SCL_M, SFP5);
+OCELOT_P(10, PTP2,      TWI_SCL_M, SFP);
+OCELOT_P(11, PTP3,      TWI_SCL_M, SFP);
+OCELOT_P(12, UART2,     TWI_SCL_M, SFP);
+OCELOT_P(13, UART2,     TWI_SCL_M, SFP);
+OCELOT_P(14, MIIM,      TWI_SCL_M, SFP);
+OCELOT_P(15, MIIM,      TWI_SCL_M, SFP);
 OCELOT_P(16, TWI,       NONE,      SI);
 OCELOT_P(17, TWI,       TWI_SCL_M, SI);
 OCELOT_P(18, PTP0,      TWI_SCL_M, NONE);
 OCELOT_P(19, PTP1,      TWI_SCL_M, NONE);
-OCELOT_P(20, RECO_CLK0, TACHO,     NONE);
-OCELOT_P(21, RECO_CLK1, PWM,       NONE);
+OCELOT_P(20, RECO_CLK,  TACHO,     TWI_SCL_M);
+OCELOT_P(21, RECO_CLK,  PWM,       TWI_SCL_M);
 
 #define OCELOT_PIN(n) {						\
 	.number = n,						\
@@ -264,22 +230,22 @@ JAGUAR2_P(40, NONE,      TWI_SCL_M);
 JAGUAR2_P(41, NONE,      TWI_SCL_M);
 JAGUAR2_P(42, NONE,      TWI_SCL_M);
 JAGUAR2_P(43, NONE,      TWI_SCL_M);
-JAGUAR2_P(44, NONE,      SFP8);
-JAGUAR2_P(45, NONE,      SFP9);
-JAGUAR2_P(46, NONE,      SFP10);
-JAGUAR2_P(47, NONE,      SFP11);
-JAGUAR2_P(48, SFP0,      NONE);
-JAGUAR2_P(49, SFP1,      SI);
-JAGUAR2_P(50, SFP2,      SI);
-JAGUAR2_P(51, SFP3,      SI);
-JAGUAR2_P(52, SFP4,      NONE);
-JAGUAR2_P(53, SFP5,      NONE);
-JAGUAR2_P(54, SFP6,      NONE);
-JAGUAR2_P(55, SFP7,      NONE);
-JAGUAR2_P(56, MIIM1,     SFP12);
-JAGUAR2_P(57, MIIM1,     SFP13);
-JAGUAR2_P(58, MIIM2,     SFP14);
-JAGUAR2_P(59, MIIM2,     SFP15);
+JAGUAR2_P(44, NONE,      SFP);
+JAGUAR2_P(45, NONE,      SFP);
+JAGUAR2_P(46, NONE,      SFP);
+JAGUAR2_P(47, NONE,      SFP);
+JAGUAR2_P(48, SFP,       NONE);
+JAGUAR2_P(49, SFP,       SI);
+JAGUAR2_P(50, SFP,       SI);
+JAGUAR2_P(51, SFP,       SI);
+JAGUAR2_P(52, SFP,       NONE);
+JAGUAR2_P(53, SFP,       NONE);
+JAGUAR2_P(54, SFP,       NONE);
+JAGUAR2_P(55, SFP,       NONE);
+JAGUAR2_P(56, MIIM,      SFP);
+JAGUAR2_P(57, MIIM,      SFP);
+JAGUAR2_P(58, MIIM,      SFP);
+JAGUAR2_P(59, MIIM,      SFP);
 JAGUAR2_P(60, NONE,      NONE);
 JAGUAR2_P(61, NONE,      NONE);
 JAGUAR2_P(62, NONE,      NONE);
@@ -714,11 +680,12 @@ static void ocelot_irq_handler(struct irq_desc *desc)
 	struct irq_chip *parent_chip = irq_desc_get_chip(desc);
 	struct gpio_chip *chip = irq_desc_get_handler_data(desc);
 	struct ocelot_pinctrl *info = gpiochip_get_data(chip);
+	unsigned int id_reg = OCELOT_GPIO_INTR_IDENT * info->stride;
 	unsigned int reg = 0, irq, i;
 	unsigned long irqs;
 
 	for (i = 0; i < info->stride; i++) {
-		regmap_read(info->map, OCELOT_GPIO_INTR_IDENT + 4 * i, &reg);
+		regmap_read(info->map, id_reg + 4 * i, &reg);
 		if (!reg)
 			continue;
 
@@ -751,21 +718,21 @@ static int ocelot_gpiochip_register(struct platform_device *pdev,
 	gc->of_node = info->dev->of_node;
 	gc->label = "ocelot-gpio";
 
-	irq = irq_of_parse_and_map(pdev->dev.of_node, 0);
-	if (irq <= 0)
-		return irq;
-
-	girq = &gc->irq;
-	girq->chip = &ocelot_irqchip;
-	girq->parent_handler = ocelot_irq_handler;
-	girq->num_parents = 1;
-	girq->parents = devm_kcalloc(&pdev->dev, 1, sizeof(*girq->parents),
-				     GFP_KERNEL);
-	if (!girq->parents)
-		return -ENOMEM;
-	girq->parents[0] = irq;
-	girq->default_type = IRQ_TYPE_NONE;
-	girq->handler = handle_edge_irq;
+	irq = irq_of_parse_and_map(gc->of_node, 0);
+	if (irq) {
+		girq = &gc->irq;
+		girq->chip = &ocelot_irqchip;
+		girq->parent_handler = ocelot_irq_handler;
+		girq->num_parents = 1;
+		girq->parents = devm_kcalloc(&pdev->dev, 1,
+					     sizeof(*girq->parents),
+					     GFP_KERNEL);
+		if (!girq->parents)
+			return -ENOMEM;
+		girq->parents[0] = irq;
+		girq->default_type = IRQ_TYPE_NONE;
+		girq->handler = handle_edge_irq;
+	}
 
 	ret = devm_gpiochip_add_data(&pdev->dev, gc, info);
 	if (ret)
diff --git a/drivers/pinctrl/pinctrl-rk805.c b/drivers/pinctrl/pinctrl-rk805.c
index cccbe072274e..c6f4229eb106 100644
--- a/drivers/pinctrl/pinctrl-rk805.c
+++ b/drivers/pinctrl/pinctrl-rk805.c
@@ -73,7 +73,7 @@ struct rk805_pctrl_info {
 	int num_pin_groups;
 	const struct pinctrl_pin_desc *pins;
 	unsigned int num_pins;
-	struct rk805_pin_config *pin_cfg;
+	const struct rk805_pin_config *pin_cfg;
 };
 
 enum rk805_pinmux_option {
@@ -121,7 +121,7 @@ static const struct rk805_pin_group rk805_pin_groups[] = {
 #define RK805_GPIO0_VAL_MSK	BIT(0)
 #define RK805_GPIO1_VAL_MSK	BIT(1)
 
-static struct rk805_pin_config rk805_gpio_cfgs[] = {
+static const struct rk805_pin_config rk805_gpio_cfgs[] = {
 	{
 		.reg = RK805_OUT_REG,
 		.val_msk = RK805_GPIO0_VAL_MSK,
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index 098951346339..c07324d1f265 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -508,8 +508,8 @@ static int rockchip_dt_node_to_map(struct pinctrl_dev *pctldev,
 	}
 
 	map_num += grp->npins;
-	new_map = devm_kcalloc(pctldev->dev, map_num, sizeof(*new_map),
-								GFP_KERNEL);
+
+	new_map = kcalloc(map_num, sizeof(*new_map), GFP_KERNEL);
 	if (!new_map)
 		return -ENOMEM;
 
@@ -519,7 +519,7 @@ static int rockchip_dt_node_to_map(struct pinctrl_dev *pctldev,
 	/* create mux map */
 	parent = of_get_parent(np);
 	if (!parent) {
-		devm_kfree(pctldev->dev, new_map);
+		kfree(new_map);
 		return -EINVAL;
 	}
 	new_map[0].type = PIN_MAP_TYPE_MUX_GROUP;
@@ -546,6 +546,7 @@ static int rockchip_dt_node_to_map(struct pinctrl_dev *pctldev,
 static void rockchip_dt_free_map(struct pinctrl_dev *pctldev,
 				    struct pinctrl_map *map, unsigned num_maps)
 {
+	kfree(map);
 }
 
 static const struct pinctrl_ops rockchip_pctrl_ops = {
@@ -2940,14 +2941,14 @@ static int rockchip_pinctrl_parse_dt(struct platform_device *pdev,
 					      sizeof(struct rockchip_pmx_func),
 					      GFP_KERNEL);
 	if (!info->functions)
-		return -EINVAL;
+		return -ENOMEM;
 
 	info->groups = devm_kcalloc(dev,
 					    info->ngroups,
 					    sizeof(struct rockchip_pin_group),
 					    GFP_KERNEL);
 	if (!info->groups)
-		return -EINVAL;
+		return -ENOMEM;
 
 	i = 0;
 
diff --git a/drivers/pinctrl/pinctrl-rza1.c b/drivers/pinctrl/pinctrl-rza1.c
index da2d8365c690..38a14bbced5f 100644
--- a/drivers/pinctrl/pinctrl-rza1.c
+++ b/drivers/pinctrl/pinctrl-rza1.c
@@ -93,7 +93,7 @@ struct rza1_bidir_entry {
 };
 
 /**
- * rza1_swio_pin - describe a single pin that needs bidir flag applied.
+ * rza1_swio_pin - describe a single pin that needs swio flag applied.
  */
 struct rza1_swio_pin {
 	u16 pin: 4;
@@ -418,7 +418,7 @@ static const struct rza1_bidir_entry rza1l_bidir_entries[RZA1_NPORTS] = {
 };
 
 static const struct rza1_swio_entry rza1l_swio_entries[] = {
-	[0] = { ARRAY_SIZE(rza1h_swio_pins), rza1h_swio_pins },
+	[0] = { ARRAY_SIZE(rza1l_swio_pins), rza1l_swio_pins },
 };
 
 /* RZ/A1L (r7s72102x) pinmux flags table */
diff --git a/drivers/pinctrl/pinctrl-stmfx.c b/drivers/pinctrl/pinctrl-stmfx.c
index 60100b45f5e5..1aae803c12cd 100644
--- a/drivers/pinctrl/pinctrl-stmfx.c
+++ b/drivers/pinctrl/pinctrl-stmfx.c
@@ -288,7 +288,7 @@ static int stmfx_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 	struct pinctrl_gpio_range *range;
 	enum pin_config_param param;
 	u32 arg;
-	int dir, i, ret;
+	int i, ret;
 
 	range = pinctrl_find_gpio_range_from_pin_nolock(pctldev, pin);
 	if (!range) {
@@ -296,10 +296,6 @@ static int stmfx_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 		return -EINVAL;
 	}
 
-	dir = stmfx_gpio_get_direction(&pctl->gpio_chip, pin);
-	if (dir < 0)
-		return dir;
-
 	for (i = 0; i < num_configs; i++) {
 		param = pinconf_to_config_param(configs[i]);
 		arg = pinconf_to_config_argument(configs[i]);
diff --git a/drivers/pinctrl/pinctrl-sx150x.c b/drivers/pinctrl/pinctrl-sx150x.c
index 6e74bd87d959..708bc91862fe 100644
--- a/drivers/pinctrl/pinctrl-sx150x.c
+++ b/drivers/pinctrl/pinctrl-sx150x.c
@@ -988,7 +988,7 @@ static unsigned int sx150x_maybe_swizzle(struct sx150x_pinctrl *pctl,
 /*
  * In order to mask the differences between 16 and 8 bit expander
  * devices we set up a sligthly ficticious regmap that pretends to be
- * a set of 32-bit (to accomodate RegSenseLow/RegSenseHigh
+ * a set of 32-bit (to accommodate RegSenseLow/RegSenseHigh
  * pair/quartet) registers and transparently reconstructs those
  * registers via multiple I2C/SMBus reads
  *
diff --git a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c
index bddf2c5dd3bf..eab029a21643 100644
--- a/drivers/pinctrl/pxa/pinctrl-pxa2xx.c
+++ b/drivers/pinctrl/pxa/pinctrl-pxa2xx.c
@@ -425,15 +425,6 @@ int pxa2xx_pinctrl_init(struct platform_device *pdev,
 }
 EXPORT_SYMBOL_GPL(pxa2xx_pinctrl_init);
 
-int pxa2xx_pinctrl_exit(struct platform_device *pdev)
-{
-	struct pxa_pinctrl *pctl = platform_get_drvdata(pdev);
-
-	pinctrl_unregister(pctl->pctl_dev);
-	return 0;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_pinctrl_exit);
-
 MODULE_AUTHOR("Robert Jarzmik <robert.jarzmik@free.fr>");
 MODULE_DESCRIPTION("Marvell PXA2xx pinctrl driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig
index c5d4428f1f94..ff1ee159dca2 100644
--- a/drivers/pinctrl/qcom/Kconfig
+++ b/drivers/pinctrl/qcom/Kconfig
@@ -216,4 +216,13 @@ config PINCTRL_SM8150
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM8150 platform.
 
+config PINCTRL_SM8250
+	tristate "Qualcomm Technologies Inc SM8250 pin controller driver"
+	depends on GPIOLIB && OF
+	select PINCTRL_MSM
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
+	  Technologies Inc SM8250 platform.
+
 endif
diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile
index d9e09045a776..061ec9fb659b 100644
--- a/drivers/pinctrl/qcom/Makefile
+++ b/drivers/pinctrl/qcom/Makefile
@@ -26,3 +26,4 @@ obj-$(CONFIG_PINCTRL_SC7180)	+= pinctrl-sc7180.o
 obj-$(CONFIG_PINCTRL_SDM660)   += pinctrl-sdm660.o
 obj-$(CONFIG_PINCTRL_SDM845) += pinctrl-sdm845.o
 obj-$(CONFIG_PINCTRL_SM8150) += pinctrl-sm8150.o
+obj-$(CONFIG_PINCTRL_SM8250) += pinctrl-sm8250.o
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index 85858c1d56d0..83b7d64bc4c1 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -23,7 +23,6 @@
 #include <linux/pm.h>
 #include <linux/log2.h>
 #include <linux/qcom_scm.h>
-#include <linux/io.h>
 
 #include <linux/soc/qcom/irq.h>
 
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8250.c b/drivers/pinctrl/qcom/pinctrl-sm8250.c
new file mode 100644
index 000000000000..a660f1274b66
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-sm8250.c
@@ -0,0 +1,1361 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-msm.h"
+
+static const char * const sm8250_tiles[] = {
+	"west",
+	"south",
+	"north",
+};
+
+enum {
+	WEST,
+	SOUTH,
+	NORTH,
+};
+
+#define FUNCTION(fname)					\
+	[msm_mux_##fname] = {				\
+		.name = #fname,				\
+		.groups = fname##_groups,		\
+		.ngroups = ARRAY_SIZE(fname##_groups),	\
+	}
+
+#define REG_SIZE 0x1000
+#define PINGROUP(id, _tile, f1, f2, f3, f4, f5, f6, f7, f8, f9) \
+	{						\
+		.name = "gpio" #id,			\
+		.pins = gpio##id##_pins,		\
+		.npins = (unsigned int)ARRAY_SIZE(gpio##id##_pins),	\
+		.funcs = (int[]){			\
+			msm_mux_gpio, /* gpio mode */	\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9			\
+		},					\
+		.nfuncs = 10,				\
+		.ctl_reg = REG_SIZE * id,		\
+		.io_reg = REG_SIZE * id + 0x4,		\
+		.intr_cfg_reg = REG_SIZE * id + 0x8,	\
+		.intr_status_reg = REG_SIZE * id + 0xc,	\
+		.intr_target_reg = REG_SIZE * id + 0x8,	\
+		.tile = _tile,				\
+		.mux_bit = 2,				\
+		.pull_bit = 0,				\
+		.drv_bit = 6,				\
+		.oe_bit = 9,				\
+		.in_bit = 0,				\
+		.out_bit = 1,				\
+		.intr_enable_bit = 0,			\
+		.intr_status_bit = 0,			\
+		.intr_target_bit = 5,			\
+		.intr_target_kpss_val = 3,		\
+		.intr_raw_status_bit = 4,		\
+		.intr_polarity_bit = 1,			\
+		.intr_detection_bit = 2,		\
+		.intr_detection_width = 2,		\
+	}
+
+#define SDC_PINGROUP(pg_name, ctl, pull, drv)	\
+	{						\
+		.name = #pg_name,			\
+		.pins = pg_name##_pins,			\
+		.npins = (unsigned int)ARRAY_SIZE(pg_name##_pins),	\
+		.ctl_reg = ctl,				\
+		.io_reg = 0,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.intr_target_reg = 0,			\
+		.tile = NORTH,				\
+		.mux_bit = -1,				\
+		.pull_bit = pull,			\
+		.drv_bit = drv,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = -1,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+#define UFS_RESET(pg_name, offset)				\
+	{						\
+		.name = #pg_name,			\
+		.pins = pg_name##_pins,			\
+		.npins = (unsigned int)ARRAY_SIZE(pg_name##_pins),	\
+		.ctl_reg = offset,			\
+		.io_reg = offset + 0x4,			\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.intr_target_reg = 0,			\
+		.tile = SOUTH,				\
+		.mux_bit = -1,				\
+		.pull_bit = 3,				\
+		.drv_bit = 0,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = 0,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+static const struct pinctrl_pin_desc sm8250_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),
+	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),
+	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),
+	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),
+	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),
+	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),
+	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),
+	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),
+	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),
+	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),
+	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),
+	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),
+	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),
+	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),
+	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),
+	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),
+	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),
+	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),
+	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),
+	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),
+	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),
+	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),
+	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),
+	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),
+	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),
+	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),
+	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),
+	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),
+	PINCTRL_PIN(109, "GPIO_109"),
+	PINCTRL_PIN(110, "GPIO_110"),
+	PINCTRL_PIN(111, "GPIO_111"),
+	PINCTRL_PIN(112, "GPIO_112"),
+	PINCTRL_PIN(113, "GPIO_113"),
+	PINCTRL_PIN(114, "GPIO_114"),
+	PINCTRL_PIN(115, "GPIO_115"),
+	PINCTRL_PIN(116, "GPIO_116"),
+	PINCTRL_PIN(117, "GPIO_117"),
+	PINCTRL_PIN(118, "GPIO_118"),
+	PINCTRL_PIN(119, "GPIO_119"),
+	PINCTRL_PIN(120, "GPIO_120"),
+	PINCTRL_PIN(121, "GPIO_121"),
+	PINCTRL_PIN(122, "GPIO_122"),
+	PINCTRL_PIN(123, "GPIO_123"),
+	PINCTRL_PIN(124, "GPIO_124"),
+	PINCTRL_PIN(125, "GPIO_125"),
+	PINCTRL_PIN(126, "GPIO_126"),
+	PINCTRL_PIN(127, "GPIO_127"),
+	PINCTRL_PIN(128, "GPIO_128"),
+	PINCTRL_PIN(129, "GPIO_129"),
+	PINCTRL_PIN(130, "GPIO_130"),
+	PINCTRL_PIN(131, "GPIO_131"),
+	PINCTRL_PIN(132, "GPIO_132"),
+	PINCTRL_PIN(133, "GPIO_133"),
+	PINCTRL_PIN(134, "GPIO_134"),
+	PINCTRL_PIN(135, "GPIO_135"),
+	PINCTRL_PIN(136, "GPIO_136"),
+	PINCTRL_PIN(137, "GPIO_137"),
+	PINCTRL_PIN(138, "GPIO_138"),
+	PINCTRL_PIN(139, "GPIO_139"),
+	PINCTRL_PIN(140, "GPIO_140"),
+	PINCTRL_PIN(141, "GPIO_141"),
+	PINCTRL_PIN(142, "GPIO_142"),
+	PINCTRL_PIN(143, "GPIO_143"),
+	PINCTRL_PIN(144, "GPIO_144"),
+	PINCTRL_PIN(145, "GPIO_145"),
+	PINCTRL_PIN(146, "GPIO_146"),
+	PINCTRL_PIN(147, "GPIO_147"),
+	PINCTRL_PIN(148, "GPIO_148"),
+	PINCTRL_PIN(149, "GPIO_149"),
+	PINCTRL_PIN(150, "GPIO_150"),
+	PINCTRL_PIN(151, "GPIO_151"),
+	PINCTRL_PIN(152, "GPIO_152"),
+	PINCTRL_PIN(153, "GPIO_153"),
+	PINCTRL_PIN(154, "GPIO_154"),
+	PINCTRL_PIN(155, "GPIO_155"),
+	PINCTRL_PIN(156, "GPIO_156"),
+	PINCTRL_PIN(157, "GPIO_157"),
+	PINCTRL_PIN(158, "GPIO_158"),
+	PINCTRL_PIN(159, "GPIO_159"),
+	PINCTRL_PIN(160, "GPIO_160"),
+	PINCTRL_PIN(161, "GPIO_161"),
+	PINCTRL_PIN(162, "GPIO_162"),
+	PINCTRL_PIN(163, "GPIO_163"),
+	PINCTRL_PIN(164, "GPIO_164"),
+	PINCTRL_PIN(165, "GPIO_165"),
+	PINCTRL_PIN(166, "GPIO_166"),
+	PINCTRL_PIN(167, "GPIO_167"),
+	PINCTRL_PIN(168, "GPIO_168"),
+	PINCTRL_PIN(169, "GPIO_169"),
+	PINCTRL_PIN(170, "GPIO_170"),
+	PINCTRL_PIN(171, "GPIO_171"),
+	PINCTRL_PIN(172, "GPIO_172"),
+	PINCTRL_PIN(173, "GPIO_173"),
+	PINCTRL_PIN(174, "GPIO_174"),
+	PINCTRL_PIN(175, "GPIO_175"),
+	PINCTRL_PIN(176, "GPIO_176"),
+	PINCTRL_PIN(177, "GPIO_177"),
+	PINCTRL_PIN(178, "GPIO_178"),
+	PINCTRL_PIN(179, "GPIO_179"),
+	PINCTRL_PIN(180, "SDC2_CLK"),
+	PINCTRL_PIN(181, "SDC2_CMD"),
+	PINCTRL_PIN(182, "SDC2_DATA"),
+	PINCTRL_PIN(183, "UFS_RESET"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+DECLARE_MSM_GPIO_PINS(110);
+DECLARE_MSM_GPIO_PINS(111);
+DECLARE_MSM_GPIO_PINS(112);
+DECLARE_MSM_GPIO_PINS(113);
+DECLARE_MSM_GPIO_PINS(114);
+DECLARE_MSM_GPIO_PINS(115);
+DECLARE_MSM_GPIO_PINS(116);
+DECLARE_MSM_GPIO_PINS(117);
+DECLARE_MSM_GPIO_PINS(118);
+DECLARE_MSM_GPIO_PINS(119);
+DECLARE_MSM_GPIO_PINS(120);
+DECLARE_MSM_GPIO_PINS(121);
+DECLARE_MSM_GPIO_PINS(122);
+DECLARE_MSM_GPIO_PINS(123);
+DECLARE_MSM_GPIO_PINS(124);
+DECLARE_MSM_GPIO_PINS(125);
+DECLARE_MSM_GPIO_PINS(126);
+DECLARE_MSM_GPIO_PINS(127);
+DECLARE_MSM_GPIO_PINS(128);
+DECLARE_MSM_GPIO_PINS(129);
+DECLARE_MSM_GPIO_PINS(130);
+DECLARE_MSM_GPIO_PINS(131);
+DECLARE_MSM_GPIO_PINS(132);
+DECLARE_MSM_GPIO_PINS(133);
+DECLARE_MSM_GPIO_PINS(134);
+DECLARE_MSM_GPIO_PINS(135);
+DECLARE_MSM_GPIO_PINS(136);
+DECLARE_MSM_GPIO_PINS(137);
+DECLARE_MSM_GPIO_PINS(138);
+DECLARE_MSM_GPIO_PINS(139);
+DECLARE_MSM_GPIO_PINS(140);
+DECLARE_MSM_GPIO_PINS(141);
+DECLARE_MSM_GPIO_PINS(142);
+DECLARE_MSM_GPIO_PINS(143);
+DECLARE_MSM_GPIO_PINS(144);
+DECLARE_MSM_GPIO_PINS(145);
+DECLARE_MSM_GPIO_PINS(146);
+DECLARE_MSM_GPIO_PINS(147);
+DECLARE_MSM_GPIO_PINS(148);
+DECLARE_MSM_GPIO_PINS(149);
+DECLARE_MSM_GPIO_PINS(150);
+DECLARE_MSM_GPIO_PINS(151);
+DECLARE_MSM_GPIO_PINS(152);
+DECLARE_MSM_GPIO_PINS(153);
+DECLARE_MSM_GPIO_PINS(154);
+DECLARE_MSM_GPIO_PINS(155);
+DECLARE_MSM_GPIO_PINS(156);
+DECLARE_MSM_GPIO_PINS(157);
+DECLARE_MSM_GPIO_PINS(158);
+DECLARE_MSM_GPIO_PINS(159);
+DECLARE_MSM_GPIO_PINS(160);
+DECLARE_MSM_GPIO_PINS(161);
+DECLARE_MSM_GPIO_PINS(162);
+DECLARE_MSM_GPIO_PINS(163);
+DECLARE_MSM_GPIO_PINS(164);
+DECLARE_MSM_GPIO_PINS(165);
+DECLARE_MSM_GPIO_PINS(166);
+DECLARE_MSM_GPIO_PINS(167);
+DECLARE_MSM_GPIO_PINS(168);
+DECLARE_MSM_GPIO_PINS(169);
+DECLARE_MSM_GPIO_PINS(170);
+DECLARE_MSM_GPIO_PINS(171);
+DECLARE_MSM_GPIO_PINS(172);
+DECLARE_MSM_GPIO_PINS(173);
+DECLARE_MSM_GPIO_PINS(174);
+DECLARE_MSM_GPIO_PINS(175);
+DECLARE_MSM_GPIO_PINS(176);
+DECLARE_MSM_GPIO_PINS(177);
+DECLARE_MSM_GPIO_PINS(178);
+DECLARE_MSM_GPIO_PINS(179);
+
+static const unsigned int ufs_reset_pins[] = { 180 };
+static const unsigned int sdc2_clk_pins[] = { 181 };
+static const unsigned int sdc2_cmd_pins[] = { 182 };
+static const unsigned int sdc2_data_pins[] = { 183 };
+
+enum sm8250_functions {
+	msm_mux_aoss_cti,
+	msm_mux_atest,
+	msm_mux_audio_ref,
+	msm_mux_cam_mclk,
+	msm_mux_cci_async,
+	msm_mux_cci_i2c,
+	msm_mux_cci_timer0,
+	msm_mux_cci_timer1,
+	msm_mux_cci_timer2,
+	msm_mux_cci_timer3,
+	msm_mux_cci_timer4,
+	msm_mux_cri_trng,
+	msm_mux_cri_trng0,
+	msm_mux_cri_trng1,
+	msm_mux_dbg_out,
+	msm_mux_ddr_bist,
+	msm_mux_ddr_pxi0,
+	msm_mux_ddr_pxi1,
+	msm_mux_ddr_pxi2,
+	msm_mux_ddr_pxi3,
+	msm_mux_dp_hot,
+	msm_mux_dp_lcd,
+	msm_mux_gcc_gp1,
+	msm_mux_gcc_gp2,
+	msm_mux_gcc_gp3,
+	msm_mux_gpio,
+	msm_mux_ibi_i3c,
+	msm_mux_jitter_bist,
+	msm_mux_lpass_slimbus,
+	msm_mux_mdp_vsync,
+	msm_mux_mdp_vsync0,
+	msm_mux_mdp_vsync1,
+	msm_mux_mdp_vsync2,
+	msm_mux_mdp_vsync3,
+	msm_mux_mi2s0_data0,
+	msm_mux_mi2s0_data1,
+	msm_mux_mi2s0_sck,
+	msm_mux_mi2s0_ws,
+	msm_mux_mi2s1_data0,
+	msm_mux_mi2s1_data1,
+	msm_mux_mi2s1_sck,
+	msm_mux_mi2s1_ws,
+	msm_mux_mi2s2_data0,
+	msm_mux_mi2s2_data1,
+	msm_mux_mi2s2_sck,
+	msm_mux_mi2s2_ws,
+	msm_mux_pci_e0,
+	msm_mux_pci_e1,
+	msm_mux_pci_e2,
+	msm_mux_phase_flag,
+	msm_mux_pll_bist,
+	msm_mux_pll_bypassnl,
+	msm_mux_pll_clk,
+	msm_mux_pll_reset,
+	msm_mux_pri_mi2s,
+	msm_mux_prng_rosc,
+	msm_mux_qdss_cti,
+	msm_mux_qdss_gpio,
+	msm_mux_qspi0,
+	msm_mux_qspi1,
+	msm_mux_qspi2,
+	msm_mux_qspi3,
+	msm_mux_qspi_clk,
+	msm_mux_qspi_cs,
+	msm_mux_qup0,
+	msm_mux_qup1,
+	msm_mux_qup10,
+	msm_mux_qup11,
+	msm_mux_qup12,
+	msm_mux_qup13,
+	msm_mux_qup14,
+	msm_mux_qup15,
+	msm_mux_qup16,
+	msm_mux_qup17,
+	msm_mux_qup18,
+	msm_mux_qup19,
+	msm_mux_qup2,
+	msm_mux_qup3,
+	msm_mux_qup4,
+	msm_mux_qup5,
+	msm_mux_qup6,
+	msm_mux_qup7,
+	msm_mux_qup8,
+	msm_mux_qup9,
+	msm_mux_qup_l4,
+	msm_mux_qup_l5,
+	msm_mux_qup_l6,
+	msm_mux_sd_write,
+	msm_mux_sdc40,
+	msm_mux_sdc41,
+	msm_mux_sdc42,
+	msm_mux_sdc43,
+	msm_mux_sdc4_clk,
+	msm_mux_sdc4_cmd,
+	msm_mux_sec_mi2s,
+	msm_mux_sp_cmu,
+	msm_mux_tgu_ch0,
+	msm_mux_tgu_ch1,
+	msm_mux_tgu_ch2,
+	msm_mux_tgu_ch3,
+	msm_mux_tsense_pwm1,
+	msm_mux_tsense_pwm2,
+	msm_mux_tsif0_clk,
+	msm_mux_tsif0_data,
+	msm_mux_tsif0_en,
+	msm_mux_tsif0_error,
+	msm_mux_tsif0_sync,
+	msm_mux_tsif1_clk,
+	msm_mux_tsif1_data,
+	msm_mux_tsif1_en,
+	msm_mux_tsif1_error,
+	msm_mux_tsif1_sync,
+	msm_mux_usb2phy_ac,
+	msm_mux_usb_phy,
+	msm_mux_vsense_trigger,
+	msm_mux__,
+};
+
+static const char * const tsif1_data_groups[] = {
+	"gpio75",
+};
+static const char * const sdc41_groups[] = {
+	"gpio75",
+};
+static const char * const tsif1_sync_groups[] = {
+	"gpio76",
+};
+static const char * const sdc40_groups[] = {
+	"gpio76",
+};
+static const char * const aoss_cti_groups[] = {
+	"gpio77",
+};
+static const char * const phase_flag_groups[] = {
+	"gpio45", "gpio46", "gpio47", "gpio48", "gpio49", "gpio50", "gpio51",
+	"gpio69", "gpio70", "gpio71", "gpio72", "gpio73", "gpio74", "gpio77",
+	"gpio78", "gpio79", "gpio80", "gpio81", "gpio82", "gpio83", "gpio84",
+	"gpio103", "gpio104", "gpio115", "gpio116", "gpio117", "gpio118",
+	"gpio119", "gpio120", "gpio122", "gpio124", "gpio125",
+};
+static const char * const sd_write_groups[] = {
+	"gpio78",
+};
+static const char * const pci_e0_groups[] = {
+	"gpio79", "gpio80",
+};
+static const char * const pci_e1_groups[] = {
+	"gpio82", "gpio83",
+};
+static const char * const pci_e2_groups[] = {
+	"gpio85", "gpio86",
+};
+static const char * const tgu_ch0_groups[] = {
+	"gpio85",
+};
+static const char * const atest_groups[] = {
+	"gpio24", "gpio25", "gpio26", "gpio27", "gpio32", "gpio33", "gpio34",
+	"gpio35", "gpio36", "gpio37", "gpio85", "gpio86", "gpio87", "gpio88",
+	"gpio89",
+};
+static const char * const tgu_ch3_groups[] = {
+	"gpio86",
+};
+static const char * const tsif1_error_groups[] = {
+	"gpio90",
+};
+static const char * const tgu_ch1_groups[] = {
+	"gpio90",
+};
+static const char * const tsif0_error_groups[] = {
+	"gpio91",
+};
+static const char * const tgu_ch2_groups[] = {
+	"gpio91",
+};
+static const char * const cam_mclk_groups[] = {
+	"gpio94", "gpio95", "gpio96", "gpio97", "gpio98", "gpio99", "gpio100",
+};
+static const char * const ddr_bist_groups[] = {
+	"gpio94", "gpio95", "gpio143", "gpio144",
+};
+static const char * const pll_bypassnl_groups[] = {
+	"gpio96",
+};
+static const char * const pll_reset_groups[] = {
+	"gpio97",
+};
+static const char * const cci_i2c_groups[] = {
+	"gpio101", "gpio102", "gpio103", "gpio104", "gpio105", "gpio106",
+	"gpio107", "gpio108",
+};
+static const char * const qdss_gpio_groups[] = {
+	"gpio94", "gpio95", "gpio96", "gpio97", "gpio98", "gpio99", "gpio100",
+	"gpio101", "gpio102", "gpio103", "gpio104", "gpio105", "gpio106",
+	"gpio107", "gpio108", "gpio109", "gpio110", "gpio111", "gpio160",
+	"gpio161", "gpio162", "gpio163", "gpio164", "gpio165", "gpio166",
+	"gpio167", "gpio168", "gpio169", "gpio170", "gpio171", "gpio172",
+	"gpio173", "gpio174", "gpio175", "gpio176", "gpio177",
+};
+static const char * const gcc_gp1_groups[] = {
+	"gpio106", "gpio136",
+};
+static const char * const gcc_gp2_groups[] = {
+	"gpio107", "gpio137",
+};
+static const char * const gcc_gp3_groups[] = {
+	"gpio108", "gpio138",
+};
+static const char * const cci_timer0_groups[] = {
+	"gpio109",
+};
+static const char * const cci_timer1_groups[] = {
+	"gpio110",
+};
+static const char * const cci_timer2_groups[] = {
+	"gpio111",
+};
+static const char * const cci_timer3_groups[] = {
+	"gpio112",
+};
+static const char * const cci_async_groups[] = {
+	"gpio112", "gpio113", "gpio114",
+};
+static const char * const cci_timer4_groups[] = {
+	"gpio113",
+};
+static const char * const qup2_groups[] = {
+	"gpio115", "gpio116", "gpio117", "gpio118",
+};
+static const char * const qup3_groups[] = {
+	"gpio119", "gpio120", "gpio121", "gpio122",
+};
+static const char * const tsense_pwm1_groups[] = {
+	"gpio123",
+};
+static const char * const tsense_pwm2_groups[] = {
+	"gpio123",
+};
+static const char * const qup9_groups[] = {
+	"gpio125", "gpio126", "gpio127", "gpio128",
+};
+static const char * const qup10_groups[] = {
+	"gpio129", "gpio130", "gpio131", "gpio132",
+};
+static const char * const mi2s2_sck_groups[] = {
+	"gpio133",
+};
+static const char * const mi2s2_data0_groups[] = {
+	"gpio134",
+};
+static const char * const mi2s2_ws_groups[] = {
+	"gpio135",
+};
+static const char * const pri_mi2s_groups[] = {
+	"gpio136",
+};
+static const char * const sec_mi2s_groups[] = {
+	"gpio137",
+};
+static const char * const audio_ref_groups[] = {
+	"gpio137",
+};
+static const char * const mi2s2_data1_groups[] = {
+	"gpio137",
+};
+static const char * const mi2s0_sck_groups[] = {
+	"gpio138",
+};
+static const char * const mi2s0_data0_groups[] = {
+	"gpio139",
+};
+static const char * const mi2s0_data1_groups[] = {
+	"gpio140",
+};
+static const char * const mi2s0_ws_groups[] = {
+	"gpio141",
+};
+static const char * const lpass_slimbus_groups[] = {
+	"gpio142", "gpio143", "gpio144", "gpio145",
+};
+static const char * const mi2s1_sck_groups[] = {
+	"gpio142",
+};
+static const char * const mi2s1_data0_groups[] = {
+	"gpio143",
+};
+static const char * const mi2s1_data1_groups[] = {
+	"gpio144",
+};
+static const char * const mi2s1_ws_groups[] = {
+	"gpio145",
+};
+static const char * const cri_trng0_groups[] = {
+	"gpio159",
+};
+static const char * const cri_trng1_groups[] = {
+	"gpio160",
+};
+static const char * const cri_trng_groups[] = {
+	"gpio161",
+};
+static const char * const sp_cmu_groups[] = {
+	"gpio162",
+};
+static const char * const prng_rosc_groups[] = {
+	"gpio163",
+};
+static const char * const qup19_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3",
+};
+static const char * const gpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7",
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14",
+	"gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20", "gpio21",
+	"gpio22", "gpio23", "gpio24", "gpio25", "gpio26", "gpio27", "gpio28",
+	"gpio29", "gpio30", "gpio31", "gpio32", "gpio33", "gpio34", "gpio35",
+	"gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41", "gpio42",
+	"gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio48", "gpio49",
+	"gpio50", "gpio51", "gpio52", "gpio53", "gpio54", "gpio55", "gpio56",
+	"gpio57", "gpio58", "gpio59", "gpio60", "gpio61", "gpio62", "gpio63",
+	"gpio64", "gpio65", "gpio66", "gpio67", "gpio68", "gpio69", "gpio70",
+	"gpio71", "gpio72", "gpio73", "gpio74", "gpio75", "gpio76", "gpio77",
+	"gpio78", "gpio79", "gpio80", "gpio81", "gpio82", "gpio83", "gpio84",
+	"gpio85", "gpio86", "gpio87", "gpio88", "gpio89", "gpio90", "gpio91",
+	"gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio97", "gpio98",
+	"gpio99", "gpio100", "gpio101", "gpio102", "gpio103", "gpio104",
+	"gpio105", "gpio106", "gpio107", "gpio108", "gpio109", "gpio110",
+	"gpio111", "gpio112", "gpio113", "gpio114", "gpio115", "gpio116",
+	"gpio117", "gpio118", "gpio119", "gpio120", "gpio121", "gpio122",
+	"gpio123", "gpio124", "gpio125", "gpio126", "gpio127", "gpio128",
+	"gpio129", "gpio130", "gpio131", "gpio132", "gpio133", "gpio134",
+	"gpio135", "gpio136", "gpio137", "gpio138", "gpio139", "gpio140",
+	"gpio141", "gpio142", "gpio143", "gpio144", "gpio145", "gpio146",
+	"gpio147", "gpio148", "gpio149", "gpio150", "gpio151", "gpio152",
+	"gpio153", "gpio154", "gpio155", "gpio156", "gpio157", "gpio158",
+	"gpio159", "gpio160", "gpio161", "gpio162", "gpio163", "gpio164",
+	"gpio165", "gpio166", "gpio167", "gpio168", "gpio169", "gpio170",
+	"gpio171", "gpio172", "gpio173", "gpio174", "gpio175", "gpio176",
+	"gpio177", "gpio178", "gpio179",
+};
+static const char * const qdss_cti_groups[] = {
+	"gpio0", "gpio2", "gpio2", "gpio44", "gpio45", "gpio46", "gpio92",
+	"gpio93",
+};
+static const char * const qup1_groups[] = {
+	"gpio4", "gpio5", "gpio6", "gpio7",
+};
+static const char * const ibi_i3c_groups[] = {
+	"gpio4", "gpio5", "gpio24", "gpio25", "gpio28", "gpio29", "gpio40",
+	"gpio41",
+};
+static const char * const qup_l4_groups[] = {
+	"gpio6", "gpio14", "gpio46", "gpio123",
+};
+static const char * const qup_l5_groups[] = {
+	"gpio7", "gpio15", "gpio47", "gpio124",
+};
+static const char * const qup4_groups[] = {
+	"gpio8", "gpio9", "gpio10", "gpio11",
+};
+static const char * const qup5_groups[] = {
+	"gpio12", "gpio13", "gpio14", "gpio15",
+};
+static const char * const qup6_groups[] = {
+	"gpio16", "gpio17", "gpio18", "gpio19",
+};
+static const char * const qup7_groups[] = {
+	"gpio20", "gpio21", "gpio22", "gpio23",
+};
+static const char * const qup8_groups[] = {
+	"gpio24", "gpio25", "gpio26", "gpio27",
+};
+static const char * const qup0_groups[] = {
+	"gpio28", "gpio29", "gpio30", "gpio31",
+};
+static const char * const qup12_groups[] = {
+	"gpio32", "gpio33", "gpio34", "gpio35",
+};
+static const char * const qup13_groups[] = {
+	"gpio36", "gpio37", "gpio38", "gpio39",
+};
+static const char * const qup14_groups[] = {
+	"gpio40", "gpio41", "gpio42", "gpio43",
+};
+static const char * const ddr_pxi3_groups[] = {
+	"gpio40", "gpio43",
+};
+static const char * const ddr_pxi1_groups[] = {
+	"gpio41", "gpio42",
+};
+static const char * const vsense_trigger_groups[] = {
+	"gpio42",
+};
+static const char * const qup15_groups[] = {
+	"gpio44", "gpio45", "gpio46", "gpio47",
+};
+static const char * const dbg_out_groups[] = {
+	"gpio44",
+};
+static const char * const qup16_groups[] = {
+	"gpio48", "gpio49", "gpio50", "gpio51",
+};
+static const char * const qup17_groups[] = {
+	"gpio52", "gpio53", "gpio54", "gpio55",
+};
+static const char * const ddr_pxi0_groups[] = {
+	"gpio52", "gpio53",
+};
+static const char * const jitter_bist_groups[] = {
+	"gpio54",
+};
+static const char * const pll_bist_groups[] = {
+	"gpio55",
+};
+static const char * const ddr_pxi2_groups[] = {
+	"gpio55", "gpio56",
+};
+static const char * const qup18_groups[] = {
+	"gpio56", "gpio57", "gpio58", "gpio59",
+};
+static const char * const qup11_groups[] = {
+	"gpio60", "gpio61", "gpio62", "gpio63",
+};
+static const char * const usb2phy_ac_groups[] = {
+	"gpio64", "gpio90",
+};
+static const char * const qup_l6_groups[] = {
+	"gpio64", "gpio77", "gpio92", "gpio93",
+};
+static const char * const usb_phy_groups[] = {
+	"gpio65",
+};
+static const char * const pll_clk_groups[] = {
+	"gpio65",
+};
+static const char * const mdp_vsync_groups[] = {
+	"gpio66", "gpio67", "gpio68", "gpio122", "gpio124",
+};
+static const char * const dp_lcd_groups[] = {
+	"gpio67",
+};
+static const char * const dp_hot_groups[] = {
+	"gpio68",
+};
+static const char * const qspi_cs_groups[] = {
+	"gpio69", "gpio75",
+};
+static const char * const tsif0_clk_groups[] = {
+	"gpio69",
+};
+static const char * const qspi0_groups[] = {
+	"gpio70",
+};
+static const char * const tsif0_en_groups[] = {
+	"gpio70",
+};
+static const char * const mdp_vsync0_groups[] = {
+	"gpio70",
+};
+static const char * const mdp_vsync1_groups[] = {
+	"gpio70",
+};
+static const char * const mdp_vsync2_groups[] = {
+	"gpio70",
+};
+static const char * const mdp_vsync3_groups[] = {
+	"gpio70",
+};
+static const char * const qspi1_groups[] = {
+	"gpio71",
+};
+static const char * const tsif0_data_groups[] = {
+	"gpio71",
+};
+static const char * const sdc4_cmd_groups[] = {
+	"gpio71",
+};
+static const char * const qspi2_groups[] = {
+	"gpio72",
+};
+static const char * const tsif0_sync_groups[] = {
+	"gpio72",
+};
+static const char * const sdc43_groups[] = {
+	"gpio72",
+};
+static const char * const qspi_clk_groups[] = {
+	"gpio73",
+};
+static const char * const tsif1_clk_groups[] = {
+	"gpio73",
+};
+static const char * const sdc4_clk_groups[] = {
+	"gpio73",
+};
+static const char * const qspi3_groups[] = {
+	"gpio74",
+};
+static const char * const tsif1_en_groups[] = {
+	"gpio74",
+};
+static const char * const sdc42_groups[] = {
+	"gpio74",
+};
+
+static const struct msm_function sm8250_functions[] = {
+	FUNCTION(aoss_cti),
+	FUNCTION(atest),
+	FUNCTION(audio_ref),
+	FUNCTION(cam_mclk),
+	FUNCTION(cci_async),
+	FUNCTION(cci_i2c),
+	FUNCTION(cci_timer0),
+	FUNCTION(cci_timer1),
+	FUNCTION(cci_timer2),
+	FUNCTION(cci_timer3),
+	FUNCTION(cci_timer4),
+	FUNCTION(cri_trng),
+	FUNCTION(cri_trng0),
+	FUNCTION(cri_trng1),
+	FUNCTION(dbg_out),
+	FUNCTION(ddr_bist),
+	FUNCTION(ddr_pxi0),
+	FUNCTION(ddr_pxi1),
+	FUNCTION(ddr_pxi2),
+	FUNCTION(ddr_pxi3),
+	FUNCTION(dp_hot),
+	FUNCTION(dp_lcd),
+	FUNCTION(gcc_gp1),
+	FUNCTION(gcc_gp2),
+	FUNCTION(gcc_gp3),
+	FUNCTION(gpio),
+	FUNCTION(ibi_i3c),
+	FUNCTION(jitter_bist),
+	FUNCTION(lpass_slimbus),
+	FUNCTION(mdp_vsync),
+	FUNCTION(mdp_vsync0),
+	FUNCTION(mdp_vsync1),
+	FUNCTION(mdp_vsync2),
+	FUNCTION(mdp_vsync3),
+	FUNCTION(mi2s0_data0),
+	FUNCTION(mi2s0_data1),
+	FUNCTION(mi2s0_sck),
+	FUNCTION(mi2s0_ws),
+	FUNCTION(mi2s1_data0),
+	FUNCTION(mi2s1_data1),
+	FUNCTION(mi2s1_sck),
+	FUNCTION(mi2s1_ws),
+	FUNCTION(mi2s2_data0),
+	FUNCTION(mi2s2_data1),
+	FUNCTION(mi2s2_sck),
+	FUNCTION(mi2s2_ws),
+	FUNCTION(pci_e0),
+	FUNCTION(pci_e1),
+	FUNCTION(pci_e2),
+	FUNCTION(phase_flag),
+	FUNCTION(pll_bist),
+	FUNCTION(pll_bypassnl),
+	FUNCTION(pll_clk),
+	FUNCTION(pll_reset),
+	FUNCTION(pri_mi2s),
+	FUNCTION(prng_rosc),
+	FUNCTION(qdss_cti),
+	FUNCTION(qdss_gpio),
+	FUNCTION(qspi0),
+	FUNCTION(qspi1),
+	FUNCTION(qspi2),
+	FUNCTION(qspi3),
+	FUNCTION(qspi_clk),
+	FUNCTION(qspi_cs),
+	FUNCTION(qup0),
+	FUNCTION(qup1),
+	FUNCTION(qup10),
+	FUNCTION(qup11),
+	FUNCTION(qup12),
+	FUNCTION(qup13),
+	FUNCTION(qup14),
+	FUNCTION(qup15),
+	FUNCTION(qup16),
+	FUNCTION(qup17),
+	FUNCTION(qup18),
+	FUNCTION(qup19),
+	FUNCTION(qup2),
+	FUNCTION(qup3),
+	FUNCTION(qup4),
+	FUNCTION(qup5),
+	FUNCTION(qup6),
+	FUNCTION(qup7),
+	FUNCTION(qup8),
+	FUNCTION(qup9),
+	FUNCTION(qup_l4),
+	FUNCTION(qup_l5),
+	FUNCTION(qup_l6),
+	FUNCTION(sd_write),
+	FUNCTION(sdc40),
+	FUNCTION(sdc41),
+	FUNCTION(sdc42),
+	FUNCTION(sdc43),
+	FUNCTION(sdc4_clk),
+	FUNCTION(sdc4_cmd),
+	FUNCTION(sec_mi2s),
+	FUNCTION(sp_cmu),
+	FUNCTION(tgu_ch0),
+	FUNCTION(tgu_ch1),
+	FUNCTION(tgu_ch2),
+	FUNCTION(tgu_ch3),
+	FUNCTION(tsense_pwm1),
+	FUNCTION(tsense_pwm2),
+	FUNCTION(tsif0_clk),
+	FUNCTION(tsif0_data),
+	FUNCTION(tsif0_en),
+	FUNCTION(tsif0_error),
+	FUNCTION(tsif0_sync),
+	FUNCTION(tsif1_clk),
+	FUNCTION(tsif1_data),
+	FUNCTION(tsif1_en),
+	FUNCTION(tsif1_error),
+	FUNCTION(tsif1_sync),
+	FUNCTION(usb2phy_ac),
+	FUNCTION(usb_phy),
+	FUNCTION(vsense_trigger),
+};
+
+/* Every pin is maintained as a single group, and missing or non-existing pin
+ * would be maintained as dummy group to synchronize pin group index with
+ * pin descriptor registered with pinctrl core.
+ * Clients would not be able to request these dummy pin groups.
+ */
+static const struct msm_pingroup sm8250_groups[] = {
+	[0] = PINGROUP(0, SOUTH, qup19, qdss_cti, _, _, _, _, _, _, _),
+	[1] = PINGROUP(1, SOUTH, qup19, _, _, _, _, _, _, _, _),
+	[2] = PINGROUP(2, SOUTH, qup19, qdss_cti, qdss_cti, _, _, _, _, _, _),
+	[3] = PINGROUP(3, SOUTH, qup19, _, _, _, _, _, _, _, _),
+	[4] = PINGROUP(4, NORTH, qup1, ibi_i3c, _, _, _, _, _, _, _),
+	[5] = PINGROUP(5, NORTH, qup1, ibi_i3c, _, _, _, _, _, _, _),
+	[6] = PINGROUP(6, NORTH, qup1, qup_l4, _, _, _, _, _, _, _),
+	[7] = PINGROUP(7, NORTH, qup1, qup_l5, _, _, _, _, _, _, _),
+	[8] = PINGROUP(8, NORTH, qup4, _, _, _, _, _, _, _, _),
+	[9] = PINGROUP(9, NORTH, qup4, _, _, _, _, _, _, _, _),
+	[10] = PINGROUP(10, NORTH, qup4, _, _, _, _, _, _, _, _),
+	[11] = PINGROUP(11, NORTH, qup4, _, _, _, _, _, _, _, _),
+	[12] = PINGROUP(12, NORTH, qup5, _, _, _, _, _, _, _, _),
+	[13] = PINGROUP(13, NORTH, qup5, _, _, _, _, _, _, _, _),
+	[14] = PINGROUP(14, NORTH, qup5, qup_l4, _, _, _, _, _, _, _),
+	[15] = PINGROUP(15, NORTH, qup5, qup_l5, _, _, _, _, _, _, _),
+	[16] = PINGROUP(16, NORTH, qup6, _, _, _, _, _, _, _, _),
+	[17] = PINGROUP(17, NORTH, qup6, _, _, _, _, _, _, _, _),
+	[18] = PINGROUP(18, NORTH, qup6, _, _, _, _, _, _, _, _),
+	[19] = PINGROUP(19, NORTH, qup6, _, _, _, _, _, _, _, _),
+	[20] = PINGROUP(20, NORTH, qup7, _, _, _, _, _, _, _, _),
+	[21] = PINGROUP(21, NORTH, qup7, _, _, _, _, _, _, _, _),
+	[22] = PINGROUP(22, NORTH, qup7, _, _, _, _, _, _, _, _),
+	[23] = PINGROUP(23, NORTH, qup7, _, _, _, _, _, _, _, _),
+	[24] = PINGROUP(24, SOUTH, qup8, ibi_i3c, atest, _, _, _, _, _, _),
+	[25] = PINGROUP(25, SOUTH, qup8, ibi_i3c, atest, _, _, _, _, _, _),
+	[26] = PINGROUP(26, SOUTH, qup8, atest, _, _, _, _, _, _, _),
+	[27] = PINGROUP(27, SOUTH, qup8, atest, _, _, _, _, _, _, _),
+	[28] = PINGROUP(28, NORTH, qup0, ibi_i3c, _, _, _, _, _, _, _),
+	[29] = PINGROUP(29, NORTH, qup0, ibi_i3c, _, _, _, _, _, _, _),
+	[30] = PINGROUP(30, NORTH, qup0, _, _, _, _, _, _, _, _),
+	[31] = PINGROUP(31, NORTH, qup0, _, _, _, _, _, _, _, _),
+	[32] = PINGROUP(32, SOUTH, qup12, _, atest, _, _, _, _, _, _),
+	[33] = PINGROUP(33, SOUTH, qup12, atest, _, _, _, _, _, _, _),
+	[34] = PINGROUP(34, SOUTH, qup12, atest, _, _, _, _, _, _, _),
+	[35] = PINGROUP(35, SOUTH, qup12, atest, _, _, _, _, _, _, _),
+	[36] = PINGROUP(36, SOUTH, qup13, atest, _, _, _, _, _, _, _),
+	[37] = PINGROUP(37, SOUTH, qup13, atest, _, _, _, _, _, _, _),
+	[38] = PINGROUP(38, SOUTH, qup13, _, _, _, _, _, _, _, _),
+	[39] = PINGROUP(39, SOUTH, qup13, _, _, _, _, _, _, _, _),
+	[40] = PINGROUP(40, SOUTH, qup14, ibi_i3c, _, ddr_pxi3, _, _, _, _, _),
+	[41] = PINGROUP(41, SOUTH, qup14, ibi_i3c, _, ddr_pxi1, _, _, _, _, _),
+	[42] = PINGROUP(42, SOUTH, qup14, vsense_trigger, ddr_pxi1, _, _, _, _, _, _),
+	[43] = PINGROUP(43, SOUTH, qup14, ddr_pxi3, _, _, _, _, _, _, _),
+	[44] = PINGROUP(44, SOUTH, qup15, qdss_cti, dbg_out, _, _, _, _, _, _),
+	[45] = PINGROUP(45, SOUTH, qup15, qdss_cti, phase_flag, _, _, _, _, _, _),
+	[46] = PINGROUP(46, SOUTH, qup15, qup_l4, qdss_cti, phase_flag, _, _, _, _, _),
+	[47] = PINGROUP(47, SOUTH, qup15, qup_l5, phase_flag, _, _, _, _, _, _),
+	[48] = PINGROUP(48, SOUTH, qup16, phase_flag, _, _, _, _, _, _, _),
+	[49] = PINGROUP(49, SOUTH, qup16, phase_flag, _, _, _, _, _, _, _),
+	[50] = PINGROUP(50, SOUTH, qup16, phase_flag, _, _, _, _, _, _, _),
+	[51] = PINGROUP(51, SOUTH, qup16, phase_flag, _, _, _, _, _, _, _),
+	[52] = PINGROUP(52, SOUTH, qup17, ddr_pxi0, _, _, _, _, _, _, _),
+	[53] = PINGROUP(53, SOUTH, qup17, ddr_pxi0, _, _, _, _, _, _, _),
+	[54] = PINGROUP(54, SOUTH, qup17, jitter_bist, _, _, _, _, _, _, _),
+	[55] = PINGROUP(55, SOUTH, qup17, pll_bist, ddr_pxi2, _, _, _, _, _, _),
+	[56] = PINGROUP(56, SOUTH, qup18, ddr_pxi2, _, _, _, _, _, _, _),
+	[57] = PINGROUP(57, SOUTH, qup18, _, _, _, _, _, _, _, _),
+	[58] = PINGROUP(58, SOUTH, qup18, _, _, _, _, _, _, _, _),
+	[59] = PINGROUP(59, SOUTH, qup18, _, _, _, _, _, _, _, _),
+	[60] = PINGROUP(60, SOUTH, qup11, _, _, _, _, _, _, _, _),
+	[61] = PINGROUP(61, SOUTH, qup11, _, _, _, _, _, _, _, _),
+	[62] = PINGROUP(62, SOUTH, qup11, _, _, _, _, _, _, _, _),
+	[63] = PINGROUP(63, SOUTH, qup11, _, _, _, _, _, _, _, _),
+	[64] = PINGROUP(64, SOUTH, usb2phy_ac, qup_l6, _, _, _, _, _, _, _),
+	[65] = PINGROUP(65, SOUTH, usb_phy, pll_clk, _, _, _, _, _, _, _),
+	[66] = PINGROUP(66, NORTH, mdp_vsync, _, _, _, _, _, _, _, _),
+	[67] = PINGROUP(67, NORTH, mdp_vsync, dp_lcd, _, _, _, _, _, _, _),
+	[68] = PINGROUP(68, NORTH, mdp_vsync, dp_hot, _, _, _, _, _, _, _),
+	[69] = PINGROUP(69, SOUTH, qspi_cs, tsif0_clk, phase_flag, _, _, _, _, _, _),
+	[70] = PINGROUP(70, SOUTH, qspi0, tsif0_en, mdp_vsync0, mdp_vsync1, mdp_vsync2, mdp_vsync3, phase_flag, _, _),
+	[71] = PINGROUP(71, SOUTH, qspi1, tsif0_data, sdc4_cmd, phase_flag, _, _, _, _, _),
+	[72] = PINGROUP(72, SOUTH, qspi2, tsif0_sync, sdc43, phase_flag, _, _, _, _, _),
+	[73] = PINGROUP(73, SOUTH, qspi_clk, tsif1_clk, sdc4_clk, phase_flag, _, _, _, _, _),
+	[74] = PINGROUP(74, SOUTH, qspi3, tsif1_en, sdc42, phase_flag, _, _, _, _, _),
+	[75] = PINGROUP(75, SOUTH, qspi_cs, tsif1_data, sdc41, _, _, _, _, _, _),
+	[76] = PINGROUP(76, SOUTH, tsif1_sync, sdc40, _, _, _, _, _, _, _),
+	[77] = PINGROUP(77, NORTH, qup_l6, aoss_cti, phase_flag, _, _, _, _, _, _),
+	[78] = PINGROUP(78, NORTH, sd_write, phase_flag, _, _, _, _, _, _, _),
+	[79] = PINGROUP(79, NORTH, pci_e0, phase_flag, _, _, _, _, _, _, _),
+	[80] = PINGROUP(80, NORTH, pci_e0, phase_flag, _, _, _, _, _, _, _),
+	[81] = PINGROUP(81, NORTH, phase_flag, _, _, _, _, _, _, _, _),
+	[82] = PINGROUP(82, NORTH, pci_e1, phase_flag, _, _, _, _, _, _, _),
+	[83] = PINGROUP(83, NORTH, pci_e1, phase_flag, _, _, _, _, _, _, _),
+	[84] = PINGROUP(84, NORTH, phase_flag, _, _, _, _, _, _, _, _),
+	[85] = PINGROUP(85, SOUTH, pci_e2, tgu_ch0, atest, _, _, _, _, _, _),
+	[86] = PINGROUP(86, SOUTH, pci_e2, tgu_ch3, atest, _, _, _, _, _, _),
+	[87] = PINGROUP(87, SOUTH, atest, _, _, _, _, _, _, _, _),
+	[88] = PINGROUP(88, SOUTH, _, atest, _, _, _, _, _, _, _),
+	[89] = PINGROUP(89, SOUTH, _, atest, _, _, _, _, _, _, _),
+	[90] = PINGROUP(90, SOUTH, tsif1_error, usb2phy_ac, tgu_ch1, _, _, _, _, _, _),
+	[91] = PINGROUP(91, SOUTH, tsif0_error, tgu_ch2, _, _, _, _, _, _, _),
+	[92] = PINGROUP(92, NORTH, qup_l6, qdss_cti, _, _, _, _, _, _, _),
+	[93] = PINGROUP(93, NORTH, qup_l6, qdss_cti, _, _, _, _, _, _, _),
+	[94] = PINGROUP(94, NORTH, cam_mclk, ddr_bist, qdss_gpio, _, _, _, _, _, _),
+	[95] = PINGROUP(95, NORTH, cam_mclk, ddr_bist, qdss_gpio, _, _, _, _, _, _),
+	[96] = PINGROUP(96, NORTH, cam_mclk, pll_bypassnl, qdss_gpio, _, _, _, _, _, _),
+	[97] = PINGROUP(97, NORTH, cam_mclk, pll_reset, qdss_gpio, _, _, _, _, _, _),
+	[98] = PINGROUP(98, NORTH, cam_mclk, qdss_gpio, _, _, _, _, _, _, _),
+	[99] = PINGROUP(99, NORTH, cam_mclk, qdss_gpio, _, _, _, _, _, _, _),
+	[100] = PINGROUP(100, NORTH, cam_mclk, qdss_gpio, _, _, _, _, _, _, _),
+	[101] = PINGROUP(101, NORTH, cci_i2c, qdss_gpio, _, _, _, _, _, _, _),
+	[102] = PINGROUP(102, NORTH, cci_i2c, qdss_gpio, _, _, _, _, _, _, _),
+	[103] = PINGROUP(103, NORTH, cci_i2c, phase_flag, _, qdss_gpio, _, _, _, _, _),
+	[104] = PINGROUP(104, NORTH, cci_i2c, phase_flag, _, qdss_gpio, _, _, _, _, _),
+	[105] = PINGROUP(105, NORTH, cci_i2c, qdss_gpio, _, _, _, _, _, _, _),
+	[106] = PINGROUP(106, NORTH, cci_i2c, gcc_gp1, qdss_gpio, _, _, _, _, _, _),
+	[107] = PINGROUP(107, NORTH, cci_i2c, gcc_gp2, qdss_gpio, _, _, _, _, _, _),
+	[108] = PINGROUP(108, NORTH, cci_i2c, gcc_gp3, qdss_gpio, _, _, _, _, _, _),
+	[109] = PINGROUP(109, NORTH, cci_timer0, qdss_gpio, _, _, _, _, _, _, _),
+	[110] = PINGROUP(110, NORTH, cci_timer1, qdss_gpio, _, _, _, _, _, _, _),
+	[111] = PINGROUP(111, NORTH, cci_timer2, qdss_gpio, _, _, _, _, _, _, _),
+	[112] = PINGROUP(112, NORTH, cci_timer3, cci_async, _, _, _, _, _, _, _),
+	[113] = PINGROUP(113, NORTH, cci_timer4, cci_async, _, _, _, _, _, _, _),
+	[114] = PINGROUP(114, NORTH, cci_async, _, _, _, _, _, _, _, _),
+	[115] = PINGROUP(115, NORTH, qup2, phase_flag, _, _, _, _, _, _, _),
+	[116] = PINGROUP(116, NORTH, qup2, phase_flag, _, _, _, _, _, _, _),
+	[117] = PINGROUP(117, NORTH, qup2, phase_flag, _, _, _, _, _, _, _),
+	[118] = PINGROUP(118, NORTH, qup2, phase_flag, _, _, _, _, _, _, _),
+	[119] = PINGROUP(119, NORTH, qup3, phase_flag, _, _, _, _, _, _, _),
+	[120] = PINGROUP(120, NORTH, qup3, phase_flag, _, _, _, _, _, _, _),
+	[121] = PINGROUP(121, NORTH, qup3, _, _, _, _, _, _, _, _),
+	[122] = PINGROUP(122, NORTH, qup3, mdp_vsync, phase_flag, _, _, _, _, _, _),
+	[123] = PINGROUP(123, NORTH, qup_l4, tsense_pwm1, tsense_pwm2, _, _, _, _, _, _),
+	[124] = PINGROUP(124, NORTH, qup_l5, mdp_vsync, phase_flag, _, _, _, _, _, _),
+	[125] = PINGROUP(125, SOUTH, qup9, phase_flag, _, _, _, _, _, _, _),
+	[126] = PINGROUP(126, SOUTH, qup9, _, _, _, _, _, _, _, _),
+	[127] = PINGROUP(127, SOUTH, qup9, _, _, _, _, _, _, _, _),
+	[128] = PINGROUP(128, SOUTH, qup9, _, _, _, _, _, _, _, _),
+	[129] = PINGROUP(129, SOUTH, qup10, _, _, _, _, _, _, _, _),
+	[130] = PINGROUP(130, SOUTH, qup10, _, _, _, _, _, _, _, _),
+	[131] = PINGROUP(131, SOUTH, qup10, _, _, _, _, _, _, _, _),
+	[132] = PINGROUP(132, SOUTH, qup10, _, _, _, _, _, _, _, _),
+	[133] = PINGROUP(133, WEST, mi2s2_sck, _, _, _, _, _, _, _, _),
+	[134] = PINGROUP(134, WEST, mi2s2_data0, _, _, _, _, _, _, _, _),
+	[135] = PINGROUP(135, WEST, mi2s2_ws, _, _, _, _, _, _, _, _),
+	[136] = PINGROUP(136, WEST, pri_mi2s, gcc_gp1, _, _, _, _, _, _, _),
+	[137] = PINGROUP(137, WEST, sec_mi2s, audio_ref, mi2s2_data1, gcc_gp2, _, _, _, _, _),
+	[138] = PINGROUP(138, WEST, mi2s0_sck, gcc_gp3, _, _, _, _, _, _, _),
+	[139] = PINGROUP(139, WEST, mi2s0_data0, _, _, _, _, _, _, _, _),
+	[140] = PINGROUP(140, WEST, mi2s0_data1, _, _, _, _, _, _, _, _),
+	[141] = PINGROUP(141, WEST, mi2s0_ws, _, _, _, _, _, _, _, _),
+	[142] = PINGROUP(142, WEST, lpass_slimbus, mi2s1_sck, _, _, _, _, _, _, _),
+	[143] = PINGROUP(143, WEST, lpass_slimbus, mi2s1_data0, ddr_bist, _, _, _, _, _, _),
+	[144] = PINGROUP(144, WEST, lpass_slimbus, mi2s1_data1, ddr_bist, _, _, _, _, _, _),
+	[145] = PINGROUP(145, WEST, lpass_slimbus, mi2s1_ws, _, _, _, _, _, _, _),
+	[146] = PINGROUP(146, WEST, _, _, _, _, _, _, _, _, _),
+	[147] = PINGROUP(147, WEST, _, _, _, _, _, _, _, _, _),
+	[148] = PINGROUP(148, WEST, _, _, _, _, _, _, _, _, _),
+	[149] = PINGROUP(149, WEST, _, _, _, _, _, _, _, _, _),
+	[150] = PINGROUP(150, WEST, _, _, _, _, _, _, _, _, _),
+	[151] = PINGROUP(151, WEST, _, _, _, _, _, _, _, _, _),
+	[152] = PINGROUP(152, WEST, _, _, _, _, _, _, _, _, _),
+	[153] = PINGROUP(153, WEST, _, _, _, _, _, _, _, _, _),
+	[154] = PINGROUP(154, WEST, _, _, _, _, _, _, _, _, _),
+	[155] = PINGROUP(155, WEST, _, _, _, _, _, _, _, _, _),
+	[156] = PINGROUP(156, WEST, _, _, _, _, _, _, _, _, _),
+	[157] = PINGROUP(157, WEST, _, _, _, _, _, _, _, _, _),
+	[158] = PINGROUP(158, WEST, _, _, _, _, _, _, _, _, _),
+	[159] = PINGROUP(159, WEST, cri_trng0, _, _, _, _, _, _, _, _),
+	[160] = PINGROUP(160, WEST, cri_trng1, qdss_gpio, _, _, _, _, _, _, _),
+	[161] = PINGROUP(161, WEST, cri_trng, qdss_gpio, _, _, _, _, _, _, _),
+	[162] = PINGROUP(162, WEST, sp_cmu, qdss_gpio, _, _, _, _, _, _, _),
+	[163] = PINGROUP(163, WEST, prng_rosc, qdss_gpio, _, _, _, _, _, _, _),
+	[164] = PINGROUP(164, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[165] = PINGROUP(165, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[166] = PINGROUP(166, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[167] = PINGROUP(167, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[168] = PINGROUP(168, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[169] = PINGROUP(169, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[170] = PINGROUP(170, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[171] = PINGROUP(171, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[172] = PINGROUP(172, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[173] = PINGROUP(173, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[174] = PINGROUP(174, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[175] = PINGROUP(175, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[176] = PINGROUP(176, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[177] = PINGROUP(177, WEST, qdss_gpio, _, _, _, _, _, _, _, _),
+	[178] = PINGROUP(178, WEST, _, _, _, _, _, _, _, _, _),
+	[179] = PINGROUP(179, WEST, _, _, _, _, _, _, _, _, _),
+	[180] = UFS_RESET(ufs_reset, 0xb8000),
+	[181] = SDC_PINGROUP(sdc2_clk, 0x7000, 14, 6),
+	[182] = SDC_PINGROUP(sdc2_cmd, 0xb7000, 11, 3),
+	[183] = SDC_PINGROUP(sdc2_data, 0xb7000, 9, 0),
+};
+
+static const struct msm_pinctrl_soc_data sm8250_pinctrl = {
+	.pins = sm8250_pins,
+	.npins = ARRAY_SIZE(sm8250_pins),
+	.functions = sm8250_functions,
+	.nfunctions = ARRAY_SIZE(sm8250_functions),
+	.groups = sm8250_groups,
+	.ngroups = ARRAY_SIZE(sm8250_groups),
+	.ngpios = 181,
+	.tiles = sm8250_tiles,
+	.ntiles = ARRAY_SIZE(sm8250_tiles),
+};
+
+static int sm8250_pinctrl_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &sm8250_pinctrl);
+}
+
+static const struct of_device_id sm8250_pinctrl_of_match[] = {
+	{ .compatible = "qcom,sm8250-pinctrl", },
+	{ },
+};
+
+static struct platform_driver sm8250_pinctrl_driver = {
+	.driver = {
+		.name = "sm8250-pinctrl",
+		.of_match_table = sm8250_pinctrl_of_match,
+	},
+	.probe = sm8250_pinctrl_probe,
+	.remove = msm_pinctrl_remove,
+};
+
+static int __init sm8250_pinctrl_init(void)
+{
+	return platform_driver_register(&sm8250_pinctrl_driver);
+}
+arch_initcall(sm8250_pinctrl_init);
+
+static void __exit sm8250_pinctrl_exit(void)
+{
+	platform_driver_unregister(&sm8250_pinctrl_driver);
+}
+module_exit(sm8250_pinctrl_exit);
+
+MODULE_DESCRIPTION("QTI sm8250 pinctrl driver");
+MODULE_LICENSE("GPL v2");
+MODULE_DEVICE_TABLE(of, sm8250_pinctrl_of_match);
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c
index 0599f5127b01..84501c785473 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.c
@@ -40,6 +40,8 @@ struct exynos_irq_chip {
 	u32 eint_pend;
 	u32 eint_wake_mask_value;
 	u32 eint_wake_mask_reg;
+	void (*set_eint_wakeup_mask)(struct samsung_pinctrl_drv_data *drvdata,
+				     struct exynos_irq_chip *irq_chip);
 };
 
 static inline struct exynos_irq_chip *to_exynos_irq_chip(struct irq_chip *chip)
@@ -265,6 +267,7 @@ struct exynos_eint_gpio_save {
 	u32 eint_con;
 	u32 eint_fltcon0;
 	u32 eint_fltcon1;
+	u32 eint_mask;
 };
 
 /*
@@ -342,6 +345,47 @@ static int exynos_wkup_irq_set_wake(struct irq_data *irqd, unsigned int on)
 	return 0;
 }
 
+static void
+exynos_pinctrl_set_eint_wakeup_mask(struct samsung_pinctrl_drv_data *drvdata,
+				    struct exynos_irq_chip *irq_chip)
+{
+	struct regmap *pmu_regs;
+
+	if (!drvdata->retention_ctrl || !drvdata->retention_ctrl->priv) {
+		dev_warn(drvdata->dev,
+			 "No retention data configured bank with external wakeup interrupt. Wake-up mask will not be set.\n");
+		return;
+	}
+
+	pmu_regs = drvdata->retention_ctrl->priv;
+	dev_info(drvdata->dev,
+		 "Setting external wakeup interrupt mask: 0x%x\n",
+		 irq_chip->eint_wake_mask_value);
+
+	regmap_write(pmu_regs, irq_chip->eint_wake_mask_reg,
+		     irq_chip->eint_wake_mask_value);
+}
+
+static void
+s5pv210_pinctrl_set_eint_wakeup_mask(struct samsung_pinctrl_drv_data *drvdata,
+				    struct exynos_irq_chip *irq_chip)
+
+{
+	void __iomem *clk_base;
+
+	if (!drvdata->retention_ctrl || !drvdata->retention_ctrl->priv) {
+		dev_warn(drvdata->dev,
+			 "No retention data configured bank with external wakeup interrupt. Wake-up mask will not be set.\n");
+		return;
+	}
+
+
+	clk_base = (void __iomem *) drvdata->retention_ctrl->priv;
+
+	__raw_writel(irq_chip->eint_wake_mask_value,
+		     clk_base + irq_chip->eint_wake_mask_reg);
+}
+
 /*
  * irq_chip for wakeup interrupts
  */
@@ -360,8 +404,9 @@ static const struct exynos_irq_chip s5pv210_wkup_irq_chip __initconst = {
 	.eint_mask = EXYNOS_WKUP_EMASK_OFFSET,
 	.eint_pend = EXYNOS_WKUP_EPEND_OFFSET,
 	.eint_wake_mask_value = EXYNOS_EINT_WAKEUP_MASK_DISABLED,
-	/* Only difference with exynos4210_wkup_irq_chip: */
+	/* Only differences with exynos4210_wkup_irq_chip: */
 	.eint_wake_mask_reg = S5PV210_EINT_WAKEUP_MASK,
+	.set_eint_wakeup_mask = s5pv210_pinctrl_set_eint_wakeup_mask,
 };
 
 static const struct exynos_irq_chip exynos4210_wkup_irq_chip __initconst = {
@@ -380,6 +425,7 @@ static const struct exynos_irq_chip exynos4210_wkup_irq_chip __initconst = {
 	.eint_pend = EXYNOS_WKUP_EPEND_OFFSET,
 	.eint_wake_mask_value = EXYNOS_EINT_WAKEUP_MASK_DISABLED,
 	.eint_wake_mask_reg = EXYNOS_EINT_WAKEUP_MASK,
+	.set_eint_wakeup_mask = exynos_pinctrl_set_eint_wakeup_mask,
 };
 
 static const struct exynos_irq_chip exynos7_wkup_irq_chip __initconst = {
@@ -398,6 +444,7 @@ static const struct exynos_irq_chip exynos7_wkup_irq_chip __initconst = {
 	.eint_pend = EXYNOS7_WKUP_EPEND_OFFSET,
 	.eint_wake_mask_value = EXYNOS_EINT_WAKEUP_MASK_DISABLED,
 	.eint_wake_mask_reg = EXYNOS5433_EINT_WAKEUP_MASK,
+	.set_eint_wakeup_mask = exynos_pinctrl_set_eint_wakeup_mask,
 };
 
 /* list of external wakeup controllers supported */
@@ -574,27 +621,6 @@ int exynos_eint_wkup_init(struct samsung_pinctrl_drv_data *d)
 	return 0;
 }
 
-static void
-exynos_pinctrl_set_eint_wakeup_mask(struct samsung_pinctrl_drv_data *drvdata,
-				    struct exynos_irq_chip *irq_chip)
-{
-	struct regmap *pmu_regs;
-
-	if (!drvdata->retention_ctrl || !drvdata->retention_ctrl->priv) {
-		dev_warn(drvdata->dev,
-			 "No retention data configured bank with external wakeup interrupt. Wake-up mask will not be set.\n");
-		return;
-	}
-
-	pmu_regs = drvdata->retention_ctrl->priv;
-	dev_info(drvdata->dev,
-		 "Setting external wakeup interrupt mask: 0x%x\n",
-		 irq_chip->eint_wake_mask_value);
-
-	regmap_write(pmu_regs, irq_chip->eint_wake_mask_reg,
-		     irq_chip->eint_wake_mask_value);
-}
-
 static void exynos_pinctrl_suspend_bank(
 				struct samsung_pinctrl_drv_data *drvdata,
 				struct samsung_pin_bank *bank)
@@ -608,10 +634,13 @@ static void exynos_pinctrl_suspend_bank(
 						+ 2 * bank->eint_offset);
 	save->eint_fltcon1 = readl(regs + EXYNOS_GPIO_EFLTCON_OFFSET
 						+ 2 * bank->eint_offset + 4);
+	save->eint_mask = readl(regs + bank->irq_chip->eint_mask
+						+ bank->eint_offset);
 
 	pr_debug("%s: save     con %#010x\n", bank->name, save->eint_con);
 	pr_debug("%s: save fltcon0 %#010x\n", bank->name, save->eint_fltcon0);
 	pr_debug("%s: save fltcon1 %#010x\n", bank->name, save->eint_fltcon1);
+	pr_debug("%s: save    mask %#010x\n", bank->name, save->eint_mask);
 }
 
 void exynos_pinctrl_suspend(struct samsung_pinctrl_drv_data *drvdata)
@@ -626,8 +655,8 @@ void exynos_pinctrl_suspend(struct samsung_pinctrl_drv_data *drvdata)
 		else if (bank->eint_type == EINT_TYPE_WKUP) {
 			if (!irq_chip) {
 				irq_chip = bank->irq_chip;
-				exynos_pinctrl_set_eint_wakeup_mask(drvdata,
-								    irq_chip);
+				irq_chip->set_eint_wakeup_mask(drvdata,
+							       irq_chip);
 			} else if (bank->irq_chip != irq_chip) {
 				dev_warn(drvdata->dev,
 					 "More than one external wakeup interrupt chip configured (bank: %s). This is not supported by hardware nor by driver.\n",
@@ -653,6 +682,9 @@ static void exynos_pinctrl_resume_bank(
 	pr_debug("%s: fltcon1 %#010x => %#010x\n", bank->name,
 			readl(regs + EXYNOS_GPIO_EFLTCON_OFFSET
 			+ 2 * bank->eint_offset + 4), save->eint_fltcon1);
+	pr_debug("%s:    mask %#010x => %#010x\n", bank->name,
+			readl(regs + bank->irq_chip->eint_mask
+			+ bank->eint_offset), save->eint_mask);
 
 	writel(save->eint_con, regs + EXYNOS_GPIO_ECON_OFFSET
 						+ bank->eint_offset);
@@ -660,6 +692,8 @@ static void exynos_pinctrl_resume_bank(
 						+ 2 * bank->eint_offset);
 	writel(save->eint_fltcon1, regs + EXYNOS_GPIO_EFLTCON_OFFSET
 						+ 2 * bank->eint_offset + 4);
+	writel(save->eint_mask, regs + bank->irq_chip->eint_mask
+						+ bank->eint_offset);
 }
 
 void exynos_pinctrl_resume(struct samsung_pinctrl_drv_data *drvdata)
diff --git a/drivers/pinctrl/sh-pfc/Kconfig b/drivers/pinctrl/sh-pfc/Kconfig
index 9552851b96f1..c461a2f1927a 100644
--- a/drivers/pinctrl/sh-pfc/Kconfig
+++ b/drivers/pinctrl/sh-pfc/Kconfig
@@ -12,6 +12,7 @@ config PINCTRL_SH_PFC
 	select PINCTRL_PFC_EMEV2 if ARCH_EMEV2
 	select PINCTRL_PFC_R8A73A4 if ARCH_R8A73A4
 	select PINCTRL_PFC_R8A7740 if ARCH_R8A7740
+	select PINCTRL_PFC_R8A7742 if ARCH_R8A7742
 	select PINCTRL_PFC_R8A7743 if ARCH_R8A7743
 	select PINCTRL_PFC_R8A7744 if ARCH_R8A7744
 	select PINCTRL_PFC_R8A7745 if ARCH_R8A7745
@@ -74,6 +75,9 @@ config PINCTRL_PFC_R8A7740
 	bool "R-Mobile A1 pin control support" if COMPILE_TEST
 	select PINCTRL_SH_PFC_GPIO
 
+config PINCTRL_PFC_R8A7742
+	bool "RZ/G1H pin control support" if COMPILE_TEST
+
 config PINCTRL_PFC_R8A7743
 	bool "RZ/G1M pin control support" if COMPILE_TEST
 
diff --git a/drivers/pinctrl/sh-pfc/Makefile b/drivers/pinctrl/sh-pfc/Makefile
index 9ebe321d24c4..3855d82069c9 100644
--- a/drivers/pinctrl/sh-pfc/Makefile
+++ b/drivers/pinctrl/sh-pfc/Makefile
@@ -4,6 +4,7 @@ obj-$(CONFIG_PINCTRL_SH_PFC_GPIO)	+= gpio.o
 obj-$(CONFIG_PINCTRL_PFC_EMEV2)	+= pfc-emev2.o
 obj-$(CONFIG_PINCTRL_PFC_R8A73A4)	+= pfc-r8a73a4.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7740)	+= pfc-r8a7740.o
+obj-$(CONFIG_PINCTRL_PFC_R8A7742)	+= pfc-r8a7790.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7743)	+= pfc-r8a7791.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7744)	+= pfc-r8a7791.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7745)	+= pfc-r8a7794.o
diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c
index a2e19efa26e3..f368383cba61 100644
--- a/drivers/pinctrl/sh-pfc/core.c
+++ b/drivers/pinctrl/sh-pfc/core.c
@@ -485,6 +485,12 @@ static const struct of_device_id sh_pfc_of_table[] = {
 		.data = &r8a7740_pinmux_info,
 	},
 #endif
+#ifdef CONFIG_PINCTRL_PFC_R8A7742
+	{
+		.compatible = "renesas,pfc-r8a7742",
+		.data = &r8a7742_pinmux_info,
+	},
+#endif
 #ifdef CONFIG_PINCTRL_PFC_R8A7743
 	{
 		.compatible = "renesas,pfc-r8a7743",
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7790.c b/drivers/pinctrl/sh-pfc/pfc-r8a7790.c
index 3366ed561cce..f524401fec5f 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7790.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7790.c
@@ -3938,297 +3938,304 @@ static const unsigned int vin3_clk_mux[] = {
 	VI3_CLK_MARK,
 };
 
-static const struct sh_pfc_pin_group pinmux_groups[] = {
-	SH_PFC_PIN_GROUP(audio_clk_a),
-	SH_PFC_PIN_GROUP(audio_clk_b),
-	SH_PFC_PIN_GROUP(audio_clk_c),
-	SH_PFC_PIN_GROUP(audio_clkout),
-	SH_PFC_PIN_GROUP(audio_clkout_b),
-	SH_PFC_PIN_GROUP(audio_clkout_c),
-	SH_PFC_PIN_GROUP(audio_clkout_d),
-	SH_PFC_PIN_GROUP(avb_link),
-	SH_PFC_PIN_GROUP(avb_magic),
-	SH_PFC_PIN_GROUP(avb_phy_int),
-	SH_PFC_PIN_GROUP(avb_mdio),
-	SH_PFC_PIN_GROUP(avb_mii),
-	SH_PFC_PIN_GROUP(avb_gmii),
-	SH_PFC_PIN_GROUP(du_rgb666),
-	SH_PFC_PIN_GROUP(du_rgb888),
-	SH_PFC_PIN_GROUP(du_clk_out_0),
-	SH_PFC_PIN_GROUP(du_clk_out_1),
-	SH_PFC_PIN_GROUP(du_sync_0),
-	SH_PFC_PIN_GROUP(du_sync_1),
-	SH_PFC_PIN_GROUP(du_cde),
-	SH_PFC_PIN_GROUP(du0_clk_in),
-	SH_PFC_PIN_GROUP(du1_clk_in),
-	SH_PFC_PIN_GROUP(du2_clk_in),
-	SH_PFC_PIN_GROUP(eth_link),
-	SH_PFC_PIN_GROUP(eth_magic),
-	SH_PFC_PIN_GROUP(eth_mdio),
-	SH_PFC_PIN_GROUP(eth_rmii),
-	SH_PFC_PIN_GROUP(hscif0_data),
-	SH_PFC_PIN_GROUP(hscif0_clk),
-	SH_PFC_PIN_GROUP(hscif0_ctrl),
-	SH_PFC_PIN_GROUP(hscif0_data_b),
-	SH_PFC_PIN_GROUP(hscif0_ctrl_b),
-	SH_PFC_PIN_GROUP(hscif0_data_c),
-	SH_PFC_PIN_GROUP(hscif0_ctrl_c),
-	SH_PFC_PIN_GROUP(hscif0_data_d),
-	SH_PFC_PIN_GROUP(hscif0_ctrl_d),
-	SH_PFC_PIN_GROUP(hscif0_data_e),
-	SH_PFC_PIN_GROUP(hscif0_ctrl_e),
-	SH_PFC_PIN_GROUP(hscif0_data_f),
-	SH_PFC_PIN_GROUP(hscif0_ctrl_f),
-	SH_PFC_PIN_GROUP(hscif1_data),
-	SH_PFC_PIN_GROUP(hscif1_clk),
-	SH_PFC_PIN_GROUP(hscif1_ctrl),
-	SH_PFC_PIN_GROUP(hscif1_data_b),
-	SH_PFC_PIN_GROUP(hscif1_clk_b),
-	SH_PFC_PIN_GROUP(hscif1_ctrl_b),
-	SH_PFC_PIN_GROUP(i2c0),
-	SH_PFC_PIN_GROUP(i2c1),
-	SH_PFC_PIN_GROUP(i2c1_b),
-	SH_PFC_PIN_GROUP(i2c1_c),
-	SH_PFC_PIN_GROUP(i2c2),
-	SH_PFC_PIN_GROUP(i2c2_b),
-	SH_PFC_PIN_GROUP(i2c2_c),
-	SH_PFC_PIN_GROUP(i2c2_d),
-	SH_PFC_PIN_GROUP(i2c2_e),
-	SH_PFC_PIN_GROUP(i2c3),
-	SH_PFC_PIN_GROUP(iic0),
-	SH_PFC_PIN_GROUP(iic1),
-	SH_PFC_PIN_GROUP(iic1_b),
-	SH_PFC_PIN_GROUP(iic1_c),
-	SH_PFC_PIN_GROUP(iic2),
-	SH_PFC_PIN_GROUP(iic2_b),
-	SH_PFC_PIN_GROUP(iic2_c),
-	SH_PFC_PIN_GROUP(iic2_d),
-	SH_PFC_PIN_GROUP(iic2_e),
-	SH_PFC_PIN_GROUP(iic3),
-	SH_PFC_PIN_GROUP(intc_irq0),
-	SH_PFC_PIN_GROUP(intc_irq1),
-	SH_PFC_PIN_GROUP(intc_irq2),
-	SH_PFC_PIN_GROUP(intc_irq3),
-	SH_PFC_PIN_GROUP(mlb_3pin),
-	SH_PFC_PIN_GROUP(mmc0_data1),
-	SH_PFC_PIN_GROUP(mmc0_data4),
-	SH_PFC_PIN_GROUP(mmc0_data8),
-	SH_PFC_PIN_GROUP(mmc0_ctrl),
-	SH_PFC_PIN_GROUP(mmc1_data1),
-	SH_PFC_PIN_GROUP(mmc1_data4),
-	SH_PFC_PIN_GROUP(mmc1_data8),
-	SH_PFC_PIN_GROUP(mmc1_ctrl),
-	SH_PFC_PIN_GROUP(msiof0_clk),
-	SH_PFC_PIN_GROUP(msiof0_sync),
-	SH_PFC_PIN_GROUP(msiof0_ss1),
-	SH_PFC_PIN_GROUP(msiof0_ss2),
-	SH_PFC_PIN_GROUP(msiof0_rx),
-	SH_PFC_PIN_GROUP(msiof0_tx),
-	SH_PFC_PIN_GROUP(msiof0_clk_b),
-	SH_PFC_PIN_GROUP(msiof0_ss1_b),
-	SH_PFC_PIN_GROUP(msiof0_ss2_b),
-	SH_PFC_PIN_GROUP(msiof0_rx_b),
-	SH_PFC_PIN_GROUP(msiof0_tx_b),
-	SH_PFC_PIN_GROUP(msiof1_clk),
-	SH_PFC_PIN_GROUP(msiof1_sync),
-	SH_PFC_PIN_GROUP(msiof1_ss1),
-	SH_PFC_PIN_GROUP(msiof1_ss2),
-	SH_PFC_PIN_GROUP(msiof1_rx),
-	SH_PFC_PIN_GROUP(msiof1_tx),
-	SH_PFC_PIN_GROUP(msiof1_clk_b),
-	SH_PFC_PIN_GROUP(msiof1_ss1_b),
-	SH_PFC_PIN_GROUP(msiof1_ss2_b),
-	SH_PFC_PIN_GROUP(msiof1_rx_b),
-	SH_PFC_PIN_GROUP(msiof1_tx_b),
-	SH_PFC_PIN_GROUP(msiof2_clk),
-	SH_PFC_PIN_GROUP(msiof2_sync),
-	SH_PFC_PIN_GROUP(msiof2_ss1),
-	SH_PFC_PIN_GROUP(msiof2_ss2),
-	SH_PFC_PIN_GROUP(msiof2_rx),
-	SH_PFC_PIN_GROUP(msiof2_tx),
-	SH_PFC_PIN_GROUP(msiof3_clk),
-	SH_PFC_PIN_GROUP(msiof3_sync),
-	SH_PFC_PIN_GROUP(msiof3_ss1),
-	SH_PFC_PIN_GROUP(msiof3_ss2),
-	SH_PFC_PIN_GROUP(msiof3_rx),
-	SH_PFC_PIN_GROUP(msiof3_tx),
-	SH_PFC_PIN_GROUP(msiof3_clk_b),
-	SH_PFC_PIN_GROUP(msiof3_sync_b),
-	SH_PFC_PIN_GROUP(msiof3_rx_b),
-	SH_PFC_PIN_GROUP(msiof3_tx_b),
-	SH_PFC_PIN_GROUP(pwm0),
-	SH_PFC_PIN_GROUP(pwm0_b),
-	SH_PFC_PIN_GROUP(pwm1),
-	SH_PFC_PIN_GROUP(pwm1_b),
-	SH_PFC_PIN_GROUP(pwm2),
-	SH_PFC_PIN_GROUP(pwm3),
-	SH_PFC_PIN_GROUP(pwm4),
-	SH_PFC_PIN_GROUP(pwm5),
-	SH_PFC_PIN_GROUP(pwm6),
-	SH_PFC_PIN_GROUP(qspi_ctrl),
-	SH_PFC_PIN_GROUP(qspi_data2),
-	SH_PFC_PIN_GROUP(qspi_data4),
-	SH_PFC_PIN_GROUP(scif0_data),
-	SH_PFC_PIN_GROUP(scif0_clk),
-	SH_PFC_PIN_GROUP(scif0_ctrl),
-	SH_PFC_PIN_GROUP(scif0_data_b),
-	SH_PFC_PIN_GROUP(scif1_data),
-	SH_PFC_PIN_GROUP(scif1_clk),
-	SH_PFC_PIN_GROUP(scif1_ctrl),
-	SH_PFC_PIN_GROUP(scif1_data_b),
-	SH_PFC_PIN_GROUP(scif1_data_c),
-	SH_PFC_PIN_GROUP(scif1_data_d),
-	SH_PFC_PIN_GROUP(scif1_clk_d),
-	SH_PFC_PIN_GROUP(scif1_data_e),
-	SH_PFC_PIN_GROUP(scif1_clk_e),
-	SH_PFC_PIN_GROUP(scif2_data),
-	SH_PFC_PIN_GROUP(scif2_clk),
-	SH_PFC_PIN_GROUP(scif2_data_b),
-	SH_PFC_PIN_GROUP(scifa0_data),
-	SH_PFC_PIN_GROUP(scifa0_clk),
-	SH_PFC_PIN_GROUP(scifa0_ctrl),
-	SH_PFC_PIN_GROUP(scifa0_data_b),
-	SH_PFC_PIN_GROUP(scifa0_clk_b),
-	SH_PFC_PIN_GROUP(scifa0_ctrl_b),
-	SH_PFC_PIN_GROUP(scifa1_data),
-	SH_PFC_PIN_GROUP(scifa1_clk),
-	SH_PFC_PIN_GROUP(scifa1_ctrl),
-	SH_PFC_PIN_GROUP(scifa1_data_b),
-	SH_PFC_PIN_GROUP(scifa1_clk_b),
-	SH_PFC_PIN_GROUP(scifa1_ctrl_b),
-	SH_PFC_PIN_GROUP(scifa1_data_c),
-	SH_PFC_PIN_GROUP(scifa1_clk_c),
-	SH_PFC_PIN_GROUP(scifa1_ctrl_c),
-	SH_PFC_PIN_GROUP(scifa1_data_d),
-	SH_PFC_PIN_GROUP(scifa1_clk_d),
-	SH_PFC_PIN_GROUP(scifa1_ctrl_d),
-	SH_PFC_PIN_GROUP(scifa2_data),
-	SH_PFC_PIN_GROUP(scifa2_clk),
-	SH_PFC_PIN_GROUP(scifa2_ctrl),
-	SH_PFC_PIN_GROUP(scifa2_data_b),
-	SH_PFC_PIN_GROUP(scifa2_data_c),
-	SH_PFC_PIN_GROUP(scifa2_clk_c),
-	SH_PFC_PIN_GROUP(scifb0_data),
-	SH_PFC_PIN_GROUP(scifb0_clk),
-	SH_PFC_PIN_GROUP(scifb0_ctrl),
-	SH_PFC_PIN_GROUP(scifb0_data_b),
-	SH_PFC_PIN_GROUP(scifb0_clk_b),
-	SH_PFC_PIN_GROUP(scifb0_ctrl_b),
-	SH_PFC_PIN_GROUP(scifb0_data_c),
-	SH_PFC_PIN_GROUP(scifb1_data),
-	SH_PFC_PIN_GROUP(scifb1_clk),
-	SH_PFC_PIN_GROUP(scifb1_ctrl),
-	SH_PFC_PIN_GROUP(scifb1_data_b),
-	SH_PFC_PIN_GROUP(scifb1_clk_b),
-	SH_PFC_PIN_GROUP(scifb1_ctrl_b),
-	SH_PFC_PIN_GROUP(scifb1_data_c),
-	SH_PFC_PIN_GROUP(scifb1_data_d),
-	SH_PFC_PIN_GROUP(scifb1_data_e),
-	SH_PFC_PIN_GROUP(scifb1_clk_e),
-	SH_PFC_PIN_GROUP(scifb1_data_f),
-	SH_PFC_PIN_GROUP(scifb1_data_g),
-	SH_PFC_PIN_GROUP(scifb1_clk_g),
-	SH_PFC_PIN_GROUP(scifb2_data),
-	SH_PFC_PIN_GROUP(scifb2_clk),
-	SH_PFC_PIN_GROUP(scifb2_ctrl),
-	SH_PFC_PIN_GROUP(scifb2_data_b),
-	SH_PFC_PIN_GROUP(scifb2_clk_b),
-	SH_PFC_PIN_GROUP(scifb2_ctrl_b),
-	SH_PFC_PIN_GROUP(scifb2_data_c),
-	SH_PFC_PIN_GROUP(scif_clk),
-	SH_PFC_PIN_GROUP(scif_clk_b),
-	SH_PFC_PIN_GROUP(sdhi0_data1),
-	SH_PFC_PIN_GROUP(sdhi0_data4),
-	SH_PFC_PIN_GROUP(sdhi0_ctrl),
-	SH_PFC_PIN_GROUP(sdhi0_cd),
-	SH_PFC_PIN_GROUP(sdhi0_wp),
-	SH_PFC_PIN_GROUP(sdhi1_data1),
-	SH_PFC_PIN_GROUP(sdhi1_data4),
-	SH_PFC_PIN_GROUP(sdhi1_ctrl),
-	SH_PFC_PIN_GROUP(sdhi1_cd),
-	SH_PFC_PIN_GROUP(sdhi1_wp),
-	SH_PFC_PIN_GROUP(sdhi2_data1),
-	SH_PFC_PIN_GROUP(sdhi2_data4),
-	SH_PFC_PIN_GROUP(sdhi2_ctrl),
-	SH_PFC_PIN_GROUP(sdhi2_cd),
-	SH_PFC_PIN_GROUP(sdhi2_wp),
-	SH_PFC_PIN_GROUP(sdhi3_data1),
-	SH_PFC_PIN_GROUP(sdhi3_data4),
-	SH_PFC_PIN_GROUP(sdhi3_ctrl),
-	SH_PFC_PIN_GROUP(sdhi3_cd),
-	SH_PFC_PIN_GROUP(sdhi3_wp),
-	SH_PFC_PIN_GROUP(ssi0_data),
-	SH_PFC_PIN_GROUP(ssi0129_ctrl),
-	SH_PFC_PIN_GROUP(ssi1_data),
-	SH_PFC_PIN_GROUP(ssi1_ctrl),
-	SH_PFC_PIN_GROUP(ssi2_data),
-	SH_PFC_PIN_GROUP(ssi2_ctrl),
-	SH_PFC_PIN_GROUP(ssi3_data),
-	SH_PFC_PIN_GROUP(ssi34_ctrl),
-	SH_PFC_PIN_GROUP(ssi4_data),
-	SH_PFC_PIN_GROUP(ssi4_ctrl),
-	SH_PFC_PIN_GROUP(ssi5),
-	SH_PFC_PIN_GROUP(ssi5_b),
-	SH_PFC_PIN_GROUP(ssi5_c),
-	SH_PFC_PIN_GROUP(ssi6),
-	SH_PFC_PIN_GROUP(ssi6_b),
-	SH_PFC_PIN_GROUP(ssi7_data),
-	SH_PFC_PIN_GROUP(ssi7_b_data),
-	SH_PFC_PIN_GROUP(ssi7_c_data),
-	SH_PFC_PIN_GROUP(ssi78_ctrl),
-	SH_PFC_PIN_GROUP(ssi78_b_ctrl),
-	SH_PFC_PIN_GROUP(ssi78_c_ctrl),
-	SH_PFC_PIN_GROUP(ssi8_data),
-	SH_PFC_PIN_GROUP(ssi8_b_data),
-	SH_PFC_PIN_GROUP(ssi8_c_data),
-	SH_PFC_PIN_GROUP(ssi9_data),
-	SH_PFC_PIN_GROUP(ssi9_ctrl),
-	SH_PFC_PIN_GROUP(tpu0_to0),
-	SH_PFC_PIN_GROUP(tpu0_to1),
-	SH_PFC_PIN_GROUP(tpu0_to2),
-	SH_PFC_PIN_GROUP(tpu0_to3),
-	SH_PFC_PIN_GROUP(usb0),
-	SH_PFC_PIN_GROUP(usb0_ovc_vbus),
-	SH_PFC_PIN_GROUP(usb1),
-	SH_PFC_PIN_GROUP(usb2),
-	VIN_DATA_PIN_GROUP(vin0_data, 24),
-	VIN_DATA_PIN_GROUP(vin0_data, 20),
-	SH_PFC_PIN_GROUP(vin0_data18),
-	VIN_DATA_PIN_GROUP(vin0_data, 16),
-	VIN_DATA_PIN_GROUP(vin0_data, 12),
-	VIN_DATA_PIN_GROUP(vin0_data, 10),
-	VIN_DATA_PIN_GROUP(vin0_data, 8),
-	VIN_DATA_PIN_GROUP(vin0_data, 4),
-	SH_PFC_PIN_GROUP(vin0_sync),
-	SH_PFC_PIN_GROUP(vin0_field),
-	SH_PFC_PIN_GROUP(vin0_clkenb),
-	SH_PFC_PIN_GROUP(vin0_clk),
-	VIN_DATA_PIN_GROUP(vin1_data, 24),
-	VIN_DATA_PIN_GROUP(vin1_data, 20),
-	SH_PFC_PIN_GROUP(vin1_data18),
-	VIN_DATA_PIN_GROUP(vin1_data, 16),
-	VIN_DATA_PIN_GROUP(vin1_data, 12),
-	VIN_DATA_PIN_GROUP(vin1_data, 10),
-	VIN_DATA_PIN_GROUP(vin1_data, 8),
-	VIN_DATA_PIN_GROUP(vin1_data, 4),
-	SH_PFC_PIN_GROUP(vin1_sync),
-	SH_PFC_PIN_GROUP(vin1_field),
-	SH_PFC_PIN_GROUP(vin1_clkenb),
-	SH_PFC_PIN_GROUP(vin1_clk),
-	VIN_DATA_PIN_GROUP(vin2_data, 24),
-	SH_PFC_PIN_GROUP(vin2_data18),
-	VIN_DATA_PIN_GROUP(vin2_data, 16),
-	VIN_DATA_PIN_GROUP(vin2_data, 8),
-	VIN_DATA_PIN_GROUP(vin2_data, 4),
-	SH_PFC_PIN_GROUP(vin2_sync),
-	SH_PFC_PIN_GROUP(vin2_field),
-	SH_PFC_PIN_GROUP(vin2_clkenb),
-	SH_PFC_PIN_GROUP(vin2_clk),
-	SH_PFC_PIN_GROUP(vin3_data8),
-	SH_PFC_PIN_GROUP(vin3_sync),
-	SH_PFC_PIN_GROUP(vin3_field),
-	SH_PFC_PIN_GROUP(vin3_clkenb),
-	SH_PFC_PIN_GROUP(vin3_clk),
+static const struct {
+	struct sh_pfc_pin_group common[289];
+	struct sh_pfc_pin_group automotive[1];
+} pinmux_groups = {
+	.common = {
+		SH_PFC_PIN_GROUP(audio_clk_a),
+		SH_PFC_PIN_GROUP(audio_clk_b),
+		SH_PFC_PIN_GROUP(audio_clk_c),
+		SH_PFC_PIN_GROUP(audio_clkout),
+		SH_PFC_PIN_GROUP(audio_clkout_b),
+		SH_PFC_PIN_GROUP(audio_clkout_c),
+		SH_PFC_PIN_GROUP(audio_clkout_d),
+		SH_PFC_PIN_GROUP(avb_link),
+		SH_PFC_PIN_GROUP(avb_magic),
+		SH_PFC_PIN_GROUP(avb_phy_int),
+		SH_PFC_PIN_GROUP(avb_mdio),
+		SH_PFC_PIN_GROUP(avb_mii),
+		SH_PFC_PIN_GROUP(avb_gmii),
+		SH_PFC_PIN_GROUP(du_rgb666),
+		SH_PFC_PIN_GROUP(du_rgb888),
+		SH_PFC_PIN_GROUP(du_clk_out_0),
+		SH_PFC_PIN_GROUP(du_clk_out_1),
+		SH_PFC_PIN_GROUP(du_sync_0),
+		SH_PFC_PIN_GROUP(du_sync_1),
+		SH_PFC_PIN_GROUP(du_cde),
+		SH_PFC_PIN_GROUP(du0_clk_in),
+		SH_PFC_PIN_GROUP(du1_clk_in),
+		SH_PFC_PIN_GROUP(du2_clk_in),
+		SH_PFC_PIN_GROUP(eth_link),
+		SH_PFC_PIN_GROUP(eth_magic),
+		SH_PFC_PIN_GROUP(eth_mdio),
+		SH_PFC_PIN_GROUP(eth_rmii),
+		SH_PFC_PIN_GROUP(hscif0_data),
+		SH_PFC_PIN_GROUP(hscif0_clk),
+		SH_PFC_PIN_GROUP(hscif0_ctrl),
+		SH_PFC_PIN_GROUP(hscif0_data_b),
+		SH_PFC_PIN_GROUP(hscif0_ctrl_b),
+		SH_PFC_PIN_GROUP(hscif0_data_c),
+		SH_PFC_PIN_GROUP(hscif0_ctrl_c),
+		SH_PFC_PIN_GROUP(hscif0_data_d),
+		SH_PFC_PIN_GROUP(hscif0_ctrl_d),
+		SH_PFC_PIN_GROUP(hscif0_data_e),
+		SH_PFC_PIN_GROUP(hscif0_ctrl_e),
+		SH_PFC_PIN_GROUP(hscif0_data_f),
+		SH_PFC_PIN_GROUP(hscif0_ctrl_f),
+		SH_PFC_PIN_GROUP(hscif1_data),
+		SH_PFC_PIN_GROUP(hscif1_clk),
+		SH_PFC_PIN_GROUP(hscif1_ctrl),
+		SH_PFC_PIN_GROUP(hscif1_data_b),
+		SH_PFC_PIN_GROUP(hscif1_clk_b),
+		SH_PFC_PIN_GROUP(hscif1_ctrl_b),
+		SH_PFC_PIN_GROUP(i2c0),
+		SH_PFC_PIN_GROUP(i2c1),
+		SH_PFC_PIN_GROUP(i2c1_b),
+		SH_PFC_PIN_GROUP(i2c1_c),
+		SH_PFC_PIN_GROUP(i2c2),
+		SH_PFC_PIN_GROUP(i2c2_b),
+		SH_PFC_PIN_GROUP(i2c2_c),
+		SH_PFC_PIN_GROUP(i2c2_d),
+		SH_PFC_PIN_GROUP(i2c2_e),
+		SH_PFC_PIN_GROUP(i2c3),
+		SH_PFC_PIN_GROUP(iic0),
+		SH_PFC_PIN_GROUP(iic1),
+		SH_PFC_PIN_GROUP(iic1_b),
+		SH_PFC_PIN_GROUP(iic1_c),
+		SH_PFC_PIN_GROUP(iic2),
+		SH_PFC_PIN_GROUP(iic2_b),
+		SH_PFC_PIN_GROUP(iic2_c),
+		SH_PFC_PIN_GROUP(iic2_d),
+		SH_PFC_PIN_GROUP(iic2_e),
+		SH_PFC_PIN_GROUP(iic3),
+		SH_PFC_PIN_GROUP(intc_irq0),
+		SH_PFC_PIN_GROUP(intc_irq1),
+		SH_PFC_PIN_GROUP(intc_irq2),
+		SH_PFC_PIN_GROUP(intc_irq3),
+		SH_PFC_PIN_GROUP(mmc0_data1),
+		SH_PFC_PIN_GROUP(mmc0_data4),
+		SH_PFC_PIN_GROUP(mmc0_data8),
+		SH_PFC_PIN_GROUP(mmc0_ctrl),
+		SH_PFC_PIN_GROUP(mmc1_data1),
+		SH_PFC_PIN_GROUP(mmc1_data4),
+		SH_PFC_PIN_GROUP(mmc1_data8),
+		SH_PFC_PIN_GROUP(mmc1_ctrl),
+		SH_PFC_PIN_GROUP(msiof0_clk),
+		SH_PFC_PIN_GROUP(msiof0_sync),
+		SH_PFC_PIN_GROUP(msiof0_ss1),
+		SH_PFC_PIN_GROUP(msiof0_ss2),
+		SH_PFC_PIN_GROUP(msiof0_rx),
+		SH_PFC_PIN_GROUP(msiof0_tx),
+		SH_PFC_PIN_GROUP(msiof0_clk_b),
+		SH_PFC_PIN_GROUP(msiof0_ss1_b),
+		SH_PFC_PIN_GROUP(msiof0_ss2_b),
+		SH_PFC_PIN_GROUP(msiof0_rx_b),
+		SH_PFC_PIN_GROUP(msiof0_tx_b),
+		SH_PFC_PIN_GROUP(msiof1_clk),
+		SH_PFC_PIN_GROUP(msiof1_sync),
+		SH_PFC_PIN_GROUP(msiof1_ss1),
+		SH_PFC_PIN_GROUP(msiof1_ss2),
+		SH_PFC_PIN_GROUP(msiof1_rx),
+		SH_PFC_PIN_GROUP(msiof1_tx),
+		SH_PFC_PIN_GROUP(msiof1_clk_b),
+		SH_PFC_PIN_GROUP(msiof1_ss1_b),
+		SH_PFC_PIN_GROUP(msiof1_ss2_b),
+		SH_PFC_PIN_GROUP(msiof1_rx_b),
+		SH_PFC_PIN_GROUP(msiof1_tx_b),
+		SH_PFC_PIN_GROUP(msiof2_clk),
+		SH_PFC_PIN_GROUP(msiof2_sync),
+		SH_PFC_PIN_GROUP(msiof2_ss1),
+		SH_PFC_PIN_GROUP(msiof2_ss2),
+		SH_PFC_PIN_GROUP(msiof2_rx),
+		SH_PFC_PIN_GROUP(msiof2_tx),
+		SH_PFC_PIN_GROUP(msiof3_clk),
+		SH_PFC_PIN_GROUP(msiof3_sync),
+		SH_PFC_PIN_GROUP(msiof3_ss1),
+		SH_PFC_PIN_GROUP(msiof3_ss2),
+		SH_PFC_PIN_GROUP(msiof3_rx),
+		SH_PFC_PIN_GROUP(msiof3_tx),
+		SH_PFC_PIN_GROUP(msiof3_clk_b),
+		SH_PFC_PIN_GROUP(msiof3_sync_b),
+		SH_PFC_PIN_GROUP(msiof3_rx_b),
+		SH_PFC_PIN_GROUP(msiof3_tx_b),
+		SH_PFC_PIN_GROUP(pwm0),
+		SH_PFC_PIN_GROUP(pwm0_b),
+		SH_PFC_PIN_GROUP(pwm1),
+		SH_PFC_PIN_GROUP(pwm1_b),
+		SH_PFC_PIN_GROUP(pwm2),
+		SH_PFC_PIN_GROUP(pwm3),
+		SH_PFC_PIN_GROUP(pwm4),
+		SH_PFC_PIN_GROUP(pwm5),
+		SH_PFC_PIN_GROUP(pwm6),
+		SH_PFC_PIN_GROUP(qspi_ctrl),
+		SH_PFC_PIN_GROUP(qspi_data2),
+		SH_PFC_PIN_GROUP(qspi_data4),
+		SH_PFC_PIN_GROUP(scif0_data),
+		SH_PFC_PIN_GROUP(scif0_clk),
+		SH_PFC_PIN_GROUP(scif0_ctrl),
+		SH_PFC_PIN_GROUP(scif0_data_b),
+		SH_PFC_PIN_GROUP(scif1_data),
+		SH_PFC_PIN_GROUP(scif1_clk),
+		SH_PFC_PIN_GROUP(scif1_ctrl),
+		SH_PFC_PIN_GROUP(scif1_data_b),
+		SH_PFC_PIN_GROUP(scif1_data_c),
+		SH_PFC_PIN_GROUP(scif1_data_d),
+		SH_PFC_PIN_GROUP(scif1_clk_d),
+		SH_PFC_PIN_GROUP(scif1_data_e),
+		SH_PFC_PIN_GROUP(scif1_clk_e),
+		SH_PFC_PIN_GROUP(scif2_data),
+		SH_PFC_PIN_GROUP(scif2_clk),
+		SH_PFC_PIN_GROUP(scif2_data_b),
+		SH_PFC_PIN_GROUP(scifa0_data),
+		SH_PFC_PIN_GROUP(scifa0_clk),
+		SH_PFC_PIN_GROUP(scifa0_ctrl),
+		SH_PFC_PIN_GROUP(scifa0_data_b),
+		SH_PFC_PIN_GROUP(scifa0_clk_b),
+		SH_PFC_PIN_GROUP(scifa0_ctrl_b),
+		SH_PFC_PIN_GROUP(scifa1_data),
+		SH_PFC_PIN_GROUP(scifa1_clk),
+		SH_PFC_PIN_GROUP(scifa1_ctrl),
+		SH_PFC_PIN_GROUP(scifa1_data_b),
+		SH_PFC_PIN_GROUP(scifa1_clk_b),
+		SH_PFC_PIN_GROUP(scifa1_ctrl_b),
+		SH_PFC_PIN_GROUP(scifa1_data_c),
+		SH_PFC_PIN_GROUP(scifa1_clk_c),
+		SH_PFC_PIN_GROUP(scifa1_ctrl_c),
+		SH_PFC_PIN_GROUP(scifa1_data_d),
+		SH_PFC_PIN_GROUP(scifa1_clk_d),
+		SH_PFC_PIN_GROUP(scifa1_ctrl_d),
+		SH_PFC_PIN_GROUP(scifa2_data),
+		SH_PFC_PIN_GROUP(scifa2_clk),
+		SH_PFC_PIN_GROUP(scifa2_ctrl),
+		SH_PFC_PIN_GROUP(scifa2_data_b),
+		SH_PFC_PIN_GROUP(scifa2_data_c),
+		SH_PFC_PIN_GROUP(scifa2_clk_c),
+		SH_PFC_PIN_GROUP(scifb0_data),
+		SH_PFC_PIN_GROUP(scifb0_clk),
+		SH_PFC_PIN_GROUP(scifb0_ctrl),
+		SH_PFC_PIN_GROUP(scifb0_data_b),
+		SH_PFC_PIN_GROUP(scifb0_clk_b),
+		SH_PFC_PIN_GROUP(scifb0_ctrl_b),
+		SH_PFC_PIN_GROUP(scifb0_data_c),
+		SH_PFC_PIN_GROUP(scifb1_data),
+		SH_PFC_PIN_GROUP(scifb1_clk),
+		SH_PFC_PIN_GROUP(scifb1_ctrl),
+		SH_PFC_PIN_GROUP(scifb1_data_b),
+		SH_PFC_PIN_GROUP(scifb1_clk_b),
+		SH_PFC_PIN_GROUP(scifb1_ctrl_b),
+		SH_PFC_PIN_GROUP(scifb1_data_c),
+		SH_PFC_PIN_GROUP(scifb1_data_d),
+		SH_PFC_PIN_GROUP(scifb1_data_e),
+		SH_PFC_PIN_GROUP(scifb1_clk_e),
+		SH_PFC_PIN_GROUP(scifb1_data_f),
+		SH_PFC_PIN_GROUP(scifb1_data_g),
+		SH_PFC_PIN_GROUP(scifb1_clk_g),
+		SH_PFC_PIN_GROUP(scifb2_data),
+		SH_PFC_PIN_GROUP(scifb2_clk),
+		SH_PFC_PIN_GROUP(scifb2_ctrl),
+		SH_PFC_PIN_GROUP(scifb2_data_b),
+		SH_PFC_PIN_GROUP(scifb2_clk_b),
+		SH_PFC_PIN_GROUP(scifb2_ctrl_b),
+		SH_PFC_PIN_GROUP(scifb2_data_c),
+		SH_PFC_PIN_GROUP(scif_clk),
+		SH_PFC_PIN_GROUP(scif_clk_b),
+		SH_PFC_PIN_GROUP(sdhi0_data1),
+		SH_PFC_PIN_GROUP(sdhi0_data4),
+		SH_PFC_PIN_GROUP(sdhi0_ctrl),
+		SH_PFC_PIN_GROUP(sdhi0_cd),
+		SH_PFC_PIN_GROUP(sdhi0_wp),
+		SH_PFC_PIN_GROUP(sdhi1_data1),
+		SH_PFC_PIN_GROUP(sdhi1_data4),
+		SH_PFC_PIN_GROUP(sdhi1_ctrl),
+		SH_PFC_PIN_GROUP(sdhi1_cd),
+		SH_PFC_PIN_GROUP(sdhi1_wp),
+		SH_PFC_PIN_GROUP(sdhi2_data1),
+		SH_PFC_PIN_GROUP(sdhi2_data4),
+		SH_PFC_PIN_GROUP(sdhi2_ctrl),
+		SH_PFC_PIN_GROUP(sdhi2_cd),
+		SH_PFC_PIN_GROUP(sdhi2_wp),
+		SH_PFC_PIN_GROUP(sdhi3_data1),
+		SH_PFC_PIN_GROUP(sdhi3_data4),
+		SH_PFC_PIN_GROUP(sdhi3_ctrl),
+		SH_PFC_PIN_GROUP(sdhi3_cd),
+		SH_PFC_PIN_GROUP(sdhi3_wp),
+		SH_PFC_PIN_GROUP(ssi0_data),
+		SH_PFC_PIN_GROUP(ssi0129_ctrl),
+		SH_PFC_PIN_GROUP(ssi1_data),
+		SH_PFC_PIN_GROUP(ssi1_ctrl),
+		SH_PFC_PIN_GROUP(ssi2_data),
+		SH_PFC_PIN_GROUP(ssi2_ctrl),
+		SH_PFC_PIN_GROUP(ssi3_data),
+		SH_PFC_PIN_GROUP(ssi34_ctrl),
+		SH_PFC_PIN_GROUP(ssi4_data),
+		SH_PFC_PIN_GROUP(ssi4_ctrl),
+		SH_PFC_PIN_GROUP(ssi5),
+		SH_PFC_PIN_GROUP(ssi5_b),
+		SH_PFC_PIN_GROUP(ssi5_c),
+		SH_PFC_PIN_GROUP(ssi6),
+		SH_PFC_PIN_GROUP(ssi6_b),
+		SH_PFC_PIN_GROUP(ssi7_data),
+		SH_PFC_PIN_GROUP(ssi7_b_data),
+		SH_PFC_PIN_GROUP(ssi7_c_data),
+		SH_PFC_PIN_GROUP(ssi78_ctrl),
+		SH_PFC_PIN_GROUP(ssi78_b_ctrl),
+		SH_PFC_PIN_GROUP(ssi78_c_ctrl),
+		SH_PFC_PIN_GROUP(ssi8_data),
+		SH_PFC_PIN_GROUP(ssi8_b_data),
+		SH_PFC_PIN_GROUP(ssi8_c_data),
+		SH_PFC_PIN_GROUP(ssi9_data),
+		SH_PFC_PIN_GROUP(ssi9_ctrl),
+		SH_PFC_PIN_GROUP(tpu0_to0),
+		SH_PFC_PIN_GROUP(tpu0_to1),
+		SH_PFC_PIN_GROUP(tpu0_to2),
+		SH_PFC_PIN_GROUP(tpu0_to3),
+		SH_PFC_PIN_GROUP(usb0),
+		SH_PFC_PIN_GROUP(usb0_ovc_vbus),
+		SH_PFC_PIN_GROUP(usb1),
+		SH_PFC_PIN_GROUP(usb2),
+		VIN_DATA_PIN_GROUP(vin0_data, 24),
+		VIN_DATA_PIN_GROUP(vin0_data, 20),
+		SH_PFC_PIN_GROUP(vin0_data18),
+		VIN_DATA_PIN_GROUP(vin0_data, 16),
+		VIN_DATA_PIN_GROUP(vin0_data, 12),
+		VIN_DATA_PIN_GROUP(vin0_data, 10),
+		VIN_DATA_PIN_GROUP(vin0_data, 8),
+		VIN_DATA_PIN_GROUP(vin0_data, 4),
+		SH_PFC_PIN_GROUP(vin0_sync),
+		SH_PFC_PIN_GROUP(vin0_field),
+		SH_PFC_PIN_GROUP(vin0_clkenb),
+		SH_PFC_PIN_GROUP(vin0_clk),
+		VIN_DATA_PIN_GROUP(vin1_data, 24),
+		VIN_DATA_PIN_GROUP(vin1_data, 20),
+		SH_PFC_PIN_GROUP(vin1_data18),
+		VIN_DATA_PIN_GROUP(vin1_data, 16),
+		VIN_DATA_PIN_GROUP(vin1_data, 12),
+		VIN_DATA_PIN_GROUP(vin1_data, 10),
+		VIN_DATA_PIN_GROUP(vin1_data, 8),
+		VIN_DATA_PIN_GROUP(vin1_data, 4),
+		SH_PFC_PIN_GROUP(vin1_sync),
+		SH_PFC_PIN_GROUP(vin1_field),
+		SH_PFC_PIN_GROUP(vin1_clkenb),
+		SH_PFC_PIN_GROUP(vin1_clk),
+		VIN_DATA_PIN_GROUP(vin2_data, 24),
+		SH_PFC_PIN_GROUP(vin2_data18),
+		VIN_DATA_PIN_GROUP(vin2_data, 16),
+		VIN_DATA_PIN_GROUP(vin2_data, 8),
+		VIN_DATA_PIN_GROUP(vin2_data, 4),
+		SH_PFC_PIN_GROUP(vin2_sync),
+		SH_PFC_PIN_GROUP(vin2_field),
+		SH_PFC_PIN_GROUP(vin2_clkenb),
+		SH_PFC_PIN_GROUP(vin2_clk),
+		SH_PFC_PIN_GROUP(vin3_data8),
+		SH_PFC_PIN_GROUP(vin3_sync),
+		SH_PFC_PIN_GROUP(vin3_field),
+		SH_PFC_PIN_GROUP(vin3_clkenb),
+		SH_PFC_PIN_GROUP(vin3_clk),
+	},
+	.automotive = {
+		SH_PFC_PIN_GROUP(mlb_3pin),
+	}
 };
 
 static const char * const audio_clk_groups[] = {
@@ -4689,63 +4696,70 @@ static const char * const vin3_groups[] = {
 	"vin3_clk",
 };
 
-static const struct sh_pfc_function pinmux_functions[] = {
-	SH_PFC_FUNCTION(audio_clk),
-	SH_PFC_FUNCTION(avb),
-	SH_PFC_FUNCTION(du),
-	SH_PFC_FUNCTION(du0),
-	SH_PFC_FUNCTION(du1),
-	SH_PFC_FUNCTION(du2),
-	SH_PFC_FUNCTION(eth),
-	SH_PFC_FUNCTION(hscif0),
-	SH_PFC_FUNCTION(hscif1),
-	SH_PFC_FUNCTION(i2c0),
-	SH_PFC_FUNCTION(i2c1),
-	SH_PFC_FUNCTION(i2c2),
-	SH_PFC_FUNCTION(i2c3),
-	SH_PFC_FUNCTION(iic0),
-	SH_PFC_FUNCTION(iic1),
-	SH_PFC_FUNCTION(iic2),
-	SH_PFC_FUNCTION(iic3),
-	SH_PFC_FUNCTION(intc),
-	SH_PFC_FUNCTION(mlb),
-	SH_PFC_FUNCTION(mmc0),
-	SH_PFC_FUNCTION(mmc1),
-	SH_PFC_FUNCTION(msiof0),
-	SH_PFC_FUNCTION(msiof1),
-	SH_PFC_FUNCTION(msiof2),
-	SH_PFC_FUNCTION(msiof3),
-	SH_PFC_FUNCTION(pwm0),
-	SH_PFC_FUNCTION(pwm1),
-	SH_PFC_FUNCTION(pwm2),
-	SH_PFC_FUNCTION(pwm3),
-	SH_PFC_FUNCTION(pwm4),
-	SH_PFC_FUNCTION(pwm5),
-	SH_PFC_FUNCTION(pwm6),
-	SH_PFC_FUNCTION(qspi),
-	SH_PFC_FUNCTION(scif0),
-	SH_PFC_FUNCTION(scif1),
-	SH_PFC_FUNCTION(scif2),
-	SH_PFC_FUNCTION(scifa0),
-	SH_PFC_FUNCTION(scifa1),
-	SH_PFC_FUNCTION(scifa2),
-	SH_PFC_FUNCTION(scifb0),
-	SH_PFC_FUNCTION(scifb1),
-	SH_PFC_FUNCTION(scifb2),
-	SH_PFC_FUNCTION(scif_clk),
-	SH_PFC_FUNCTION(sdhi0),
-	SH_PFC_FUNCTION(sdhi1),
-	SH_PFC_FUNCTION(sdhi2),
-	SH_PFC_FUNCTION(sdhi3),
-	SH_PFC_FUNCTION(ssi),
-	SH_PFC_FUNCTION(tpu0),
-	SH_PFC_FUNCTION(usb0),
-	SH_PFC_FUNCTION(usb1),
-	SH_PFC_FUNCTION(usb2),
-	SH_PFC_FUNCTION(vin0),
-	SH_PFC_FUNCTION(vin1),
-	SH_PFC_FUNCTION(vin2),
-	SH_PFC_FUNCTION(vin3),
+static const struct {
+	struct sh_pfc_function common[55];
+	struct sh_pfc_function automotive[1];
+} pinmux_functions = {
+	.common = {
+		SH_PFC_FUNCTION(audio_clk),
+		SH_PFC_FUNCTION(avb),
+		SH_PFC_FUNCTION(du),
+		SH_PFC_FUNCTION(du0),
+		SH_PFC_FUNCTION(du1),
+		SH_PFC_FUNCTION(du2),
+		SH_PFC_FUNCTION(eth),
+		SH_PFC_FUNCTION(hscif0),
+		SH_PFC_FUNCTION(hscif1),
+		SH_PFC_FUNCTION(i2c0),
+		SH_PFC_FUNCTION(i2c1),
+		SH_PFC_FUNCTION(i2c2),
+		SH_PFC_FUNCTION(i2c3),
+		SH_PFC_FUNCTION(iic0),
+		SH_PFC_FUNCTION(iic1),
+		SH_PFC_FUNCTION(iic2),
+		SH_PFC_FUNCTION(iic3),
+		SH_PFC_FUNCTION(intc),
+		SH_PFC_FUNCTION(mmc0),
+		SH_PFC_FUNCTION(mmc1),
+		SH_PFC_FUNCTION(msiof0),
+		SH_PFC_FUNCTION(msiof1),
+		SH_PFC_FUNCTION(msiof2),
+		SH_PFC_FUNCTION(msiof3),
+		SH_PFC_FUNCTION(pwm0),
+		SH_PFC_FUNCTION(pwm1),
+		SH_PFC_FUNCTION(pwm2),
+		SH_PFC_FUNCTION(pwm3),
+		SH_PFC_FUNCTION(pwm4),
+		SH_PFC_FUNCTION(pwm5),
+		SH_PFC_FUNCTION(pwm6),
+		SH_PFC_FUNCTION(qspi),
+		SH_PFC_FUNCTION(scif0),
+		SH_PFC_FUNCTION(scif1),
+		SH_PFC_FUNCTION(scif2),
+		SH_PFC_FUNCTION(scifa0),
+		SH_PFC_FUNCTION(scifa1),
+		SH_PFC_FUNCTION(scifa2),
+		SH_PFC_FUNCTION(scifb0),
+		SH_PFC_FUNCTION(scifb1),
+		SH_PFC_FUNCTION(scifb2),
+		SH_PFC_FUNCTION(scif_clk),
+		SH_PFC_FUNCTION(sdhi0),
+		SH_PFC_FUNCTION(sdhi1),
+		SH_PFC_FUNCTION(sdhi2),
+		SH_PFC_FUNCTION(sdhi3),
+		SH_PFC_FUNCTION(ssi),
+		SH_PFC_FUNCTION(tpu0),
+		SH_PFC_FUNCTION(usb0),
+		SH_PFC_FUNCTION(usb1),
+		SH_PFC_FUNCTION(usb2),
+		SH_PFC_FUNCTION(vin0),
+		SH_PFC_FUNCTION(vin1),
+		SH_PFC_FUNCTION(vin2),
+		SH_PFC_FUNCTION(vin3),
+	},
+	.automotive = {
+		SH_PFC_FUNCTION(mlb),
+	}
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
@@ -5736,6 +5750,29 @@ static const struct sh_pfc_soc_operations r8a7790_pinmux_ops = {
 	.pin_to_pocctrl = r8a7790_pin_to_pocctrl,
 };
 
+#ifdef CONFIG_PINCTRL_PFC_R8A7742
+const struct sh_pfc_soc_info r8a7742_pinmux_info = {
+	.name = "r8a77420_pfc",
+	.ops = &r8a7790_pinmux_ops,
+	.unlock_reg = 0xe6060000, /* PMMR */
+
+	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
+
+	.pins = pinmux_pins,
+	.nr_pins = ARRAY_SIZE(pinmux_pins),
+	.groups = pinmux_groups.common,
+	.nr_groups = ARRAY_SIZE(pinmux_groups.common),
+	.functions = pinmux_functions.common,
+	.nr_functions = ARRAY_SIZE(pinmux_functions.common),
+
+	.cfg_regs = pinmux_config_regs,
+
+	.pinmux_data = pinmux_data,
+	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
+};
+#endif
+
+#ifdef CONFIG_PINCTRL_PFC_R8A7790
 const struct sh_pfc_soc_info r8a7790_pinmux_info = {
 	.name = "r8a77900_pfc",
 	.ops = &r8a7790_pinmux_ops,
@@ -5745,13 +5782,16 @@ const struct sh_pfc_soc_info r8a7790_pinmux_info = {
 
 	.pins = pinmux_pins,
 	.nr_pins = ARRAY_SIZE(pinmux_pins),
-	.groups = pinmux_groups,
-	.nr_groups = ARRAY_SIZE(pinmux_groups),
-	.functions = pinmux_functions,
-	.nr_functions = ARRAY_SIZE(pinmux_functions),
+	.groups = pinmux_groups.common,
+	.nr_groups = ARRAY_SIZE(pinmux_groups.common) +
+		ARRAY_SIZE(pinmux_groups.automotive),
+	.functions = pinmux_functions.common,
+	.nr_functions = ARRAY_SIZE(pinmux_functions.common) +
+		ARRAY_SIZE(pinmux_functions.automotive),
 
 	.cfg_regs = pinmux_config_regs,
 
 	.pinmux_data = pinmux_data,
 	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
 };
+#endif
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7269.c b/drivers/pinctrl/sh-pfc/pfc-sh7269.c
index d20974a55d93..e2916aaa8304 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7269.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7269.c
@@ -1963,8 +1963,9 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	/* "name" addr register_size Field_Width */
 
-	/* where Field_Width is 1 for single mode registers or 4 for upto 16
-	   mode registers and modes are described in assending order [0..16] */
+	/* where Field_Width is 1 for single mode registers or 4 for up to 16
+	 * mode registers and modes are described in assending order [0..15]
+	 */
 
 	{ PINMUX_CFG_REG("PAIOR0", 0xfffe3812, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
diff --git a/drivers/pinctrl/sh-pfc/sh_pfc.h b/drivers/pinctrl/sh-pfc/sh_pfc.h
index d57e633e99c0..0f013827baf9 100644
--- a/drivers/pinctrl/sh-pfc/sh_pfc.h
+++ b/drivers/pinctrl/sh-pfc/sh_pfc.h
@@ -304,6 +304,7 @@ struct sh_pfc_soc_info {
 extern const struct sh_pfc_soc_info emev2_pinmux_info;
 extern const struct sh_pfc_soc_info r8a73a4_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7740_pinmux_info;
+extern const struct sh_pfc_soc_info r8a7742_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7743_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7744_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7745_pinmux_info;
diff --git a/drivers/pinctrl/sirf/pinctrl-sirf.c b/drivers/pinctrl/sirf/pinctrl-sirf.c
index 1ebcb957c654..63a287d5795f 100644
--- a/drivers/pinctrl/sirf/pinctrl-sirf.c
+++ b/drivers/pinctrl/sirf/pinctrl-sirf.c
@@ -794,13 +794,17 @@ static int sirfsoc_gpio_probe(struct device_node *np)
 		return -ENODEV;
 
 	sgpio = devm_kzalloc(&pdev->dev, sizeof(*sgpio), GFP_KERNEL);
-	if (!sgpio)
-		return -ENOMEM;
+	if (!sgpio) {
+		err = -ENOMEM;
+		goto out_put_device;
+	}
 	spin_lock_init(&sgpio->lock);
 
 	regs = of_iomap(np, 0);
-	if (!regs)
-		return -ENOMEM;
+	if (!regs) {
+		err = -ENOMEM;
+		goto out_put_device;
+	}
 
 	sgpio->chip.gc.request = sirfsoc_gpio_request;
 	sgpio->chip.gc.free = sirfsoc_gpio_free;
@@ -824,8 +828,10 @@ static int sirfsoc_gpio_probe(struct device_node *np)
 	girq->parents = devm_kcalloc(&pdev->dev, SIRFSOC_GPIO_NO_OF_BANKS,
 				     sizeof(*girq->parents),
 				     GFP_KERNEL);
-	if (!girq->parents)
-		return -ENOMEM;
+	if (!girq->parents) {
+		err = -ENOMEM;
+		goto out_put_device;
+	}
 	for (i = 0; i < SIRFSOC_GPIO_NO_OF_BANKS; i++) {
 		bank = &sgpio->sgpio_bank[i];
 		spin_lock_init(&bank->lock);
@@ -868,6 +874,8 @@ out_no_range:
 	gpiochip_remove(&sgpio->chip.gc);
 out:
 	iounmap(regs);
+out_put_device:
+	put_device(&pdev->dev);
 	return err;
 }
 
diff --git a/drivers/pinctrl/sprd/pinctrl-sprd.c b/drivers/pinctrl/sprd/pinctrl-sprd.c
index 48cbf2a2837f..08dc1931b358 100644
--- a/drivers/pinctrl/sprd/pinctrl-sprd.c
+++ b/drivers/pinctrl/sprd/pinctrl-sprd.c
@@ -68,8 +68,8 @@
 #define SLEEP_PULL_UP_MASK		0x1
 #define SLEEP_PULL_UP_SHIFT		3
 
-#define PULL_UP_20K			(BIT(12) | BIT(7))
-#define PULL_UP_4_7K			BIT(12)
+#define PULL_UP_4_7K			(BIT(12) | BIT(7))
+#define PULL_UP_20K			BIT(7)
 #define PULL_UP_MASK			0x21
 #define PULL_UP_SHIFT			7
 
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
index 8a08c4afc6a8..9e5b61449999 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
@@ -103,8 +103,11 @@ static int sun8i_a23_r_pinctrl_probe(struct platform_device *pdev)
 
 	rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(rstc)) {
-		dev_err(&pdev->dev, "Reset controller missing\n");
-		return PTR_ERR(rstc);
+		ret = PTR_ERR(rstc);
+		if (ret == -EPROBE_DEFER)
+			return ret;
+		dev_err(&pdev->dev, "Reset controller missing err=%d\n", ret);
+		return ret;
 	}
 
 	ret = reset_control_deassert(rstc);
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c b/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c
index 6f7b3767f453..43922ab81666 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c
+++ b/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c
@@ -123,7 +123,7 @@ static int tegra_xusb_padctl_get_group_pins(struct pinctrl_dev *pinctrl,
 					    unsigned *num_pins)
 {
 	/*
-	 * For the tegra-xusb pad controller groups are synonomous
+	 * For the tegra-xusb pad controller groups are synonymous
 	 * with lanes/pins and there is always one lane/pin per group.
 	 */
 	*pins = &pinctrl->desc->pins[group].number;
diff --git a/drivers/pinctrl/zte/pinctrl-zx.c b/drivers/pinctrl/zte/pinctrl-zx.c
index 786bf89487d6..80d00ab8c110 100644
--- a/drivers/pinctrl/zte/pinctrl-zx.c
+++ b/drivers/pinctrl/zte/pinctrl-zx.c
@@ -94,7 +94,7 @@ static int zx_set_mux(struct pinctrl_dev *pctldev, unsigned int func_selector,
 	if (data->aon_pin) {
 		/*
 		 * It's an AON pin, whose mux register offset and bit position
-		 * can be caluculated from pin number.  Each register covers 16
+		 * can be calculated from pin number.  Each register covers 16
 		 * pins, and each pin occupies 2 bits.
 		 */
 		u16 aoffset = pindesc->number / 16 * 4;
diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig
index 03ea5129ed0c..cf072153bdc5 100644
--- a/drivers/platform/chrome/Kconfig
+++ b/drivers/platform/chrome/Kconfig
@@ -16,7 +16,7 @@ config MFD_CROS_EC
 menuconfig CHROME_PLATFORMS
 	bool "Platform support for Chrome hardware"
 	depends on X86 || ARM || ARM64 || COMPILE_TEST
-	---help---
+	help
 	  Say Y here to get to see options for platform support for
 	  various Chromebooks and Chromeboxes. This option alone does
 	  not add any kernel code.
@@ -28,7 +28,7 @@ if CHROME_PLATFORMS
 config CHROMEOS_LAPTOP
 	tristate "Chrome OS Laptop"
 	depends on I2C && DMI && X86
-	---help---
+	help
 	  This driver instantiates i2c and smbus devices such as
 	  light sensors and touchpads.
 
@@ -38,7 +38,7 @@ config CHROMEOS_LAPTOP
 config CHROMEOS_PSTORE
 	tristate "Chrome OS pstore support"
 	depends on X86
-	---help---
+	help
 	  This module instantiates the persistent storage on x86 ChromeOS
 	  devices. It can be used to store away console logs and crash
 	  information across reboots.
@@ -112,7 +112,7 @@ config CROS_EC_SPI
 	tristate "ChromeOS Embedded Controller (SPI)"
 	depends on CROS_EC && SPI
 
-	---help---
+	help
 	  If you say Y here, you get support for talking to the ChromeOS EC
 	  through a SPI bus, using a byte-level protocol. Since the EC's
 	  response time cannot be guaranteed, we support ignoring
@@ -217,6 +217,7 @@ config CROS_EC_SYSFS
 config CROS_EC_TYPEC
 	tristate "ChromeOS EC Type-C Connector Control"
 	depends on MFD_CROS_EC_DEV && TYPEC
+	depends on CROS_USBPD_NOTIFY
 	default MFD_CROS_EC_DEV
 	help
 	  If you say Y here, you get support for accessing Type C connector
diff --git a/drivers/platform/chrome/chromeos_pstore.c b/drivers/platform/chrome/chromeos_pstore.c
index fa51153688b4..f37c0ef4af1f 100644
--- a/drivers/platform/chrome/chromeos_pstore.c
+++ b/drivers/platform/chrome/chromeos_pstore.c
@@ -57,6 +57,7 @@ static struct ramoops_platform_data chromeos_ramoops_data = {
 	.record_size	= 0x40000,
 	.console_size	= 0x20000,
 	.ftrace_size	= 0x20000,
+	.pmsg_size	= 0x20000,
 	.max_reason	= KMSG_DUMP_OOPS,
 };
 
diff --git a/drivers/platform/chrome/cros_ec_i2c.c b/drivers/platform/chrome/cros_ec_i2c.c
index 6119eccd8a18..30c8938c27d5 100644
--- a/drivers/platform/chrome/cros_ec_i2c.c
+++ b/drivers/platform/chrome/cros_ec_i2c.c
@@ -16,7 +16,7 @@
 
 #include "cros_ec.h"
 
-/**
+/*
  * Request format for protocol v3
  * byte 0	0xda (EC_COMMAND_PROTOCOL_3)
  * byte 1-8	struct ec_host_request
diff --git a/drivers/platform/chrome/cros_ec_ishtp.c b/drivers/platform/chrome/cros_ec_ishtp.c
index 93a71e93a2f1..ed794a7ddba9 100644
--- a/drivers/platform/chrome/cros_ec_ishtp.c
+++ b/drivers/platform/chrome/cros_ec_ishtp.c
@@ -48,7 +48,8 @@ static const guid_t cros_ish_guid =
 struct header {
 	u8 channel;
 	u8 status;
-	u8 reserved[2];
+	u8 token;
+	u8 reserved;
 } __packed;
 
 struct cros_ish_out_msg {
@@ -90,6 +91,7 @@ static DECLARE_RWSEM(init_lock);
  * data exceeds this value, we log an error.
  * @size: Actual size of data received from firmware.
  * @error: 0 for success, negative error code for a failure in process_recv().
+ * @token: Expected token for response that we are waiting on.
  * @received: Set to true on receiving a valid firmware	response to host command
  * @wait_queue: Wait queue for host to wait for firmware response.
  */
@@ -98,6 +100,7 @@ struct response_info {
 	size_t max_size;
 	size_t size;
 	int error;
+	u8 token;
 	bool received;
 	wait_queue_head_t wait_queue;
 };
@@ -162,6 +165,7 @@ static int ish_send(struct ishtp_cl_data *client_data,
 		    u8 *out_msg, size_t out_size,
 		    u8 *in_msg, size_t in_size)
 {
+	static u8 next_token;
 	int rv;
 	struct header *out_hdr = (struct header *)out_msg;
 	struct ishtp_cl *cros_ish_cl = client_data->cros_ish_cl;
@@ -174,8 +178,11 @@ static int ish_send(struct ishtp_cl_data *client_data,
 	client_data->response.data = in_msg;
 	client_data->response.max_size = in_size;
 	client_data->response.error = 0;
+	client_data->response.token = next_token++;
 	client_data->response.received = false;
 
+	out_hdr->token = client_data->response.token;
+
 	rv = ishtp_cl_send(cros_ish_cl, out_msg, out_size);
 	if (rv) {
 		dev_err(cl_data_to_dev(client_data),
@@ -249,17 +256,23 @@ static void process_recv(struct ishtp_cl *cros_ish_cl,
 
 	switch (in_msg->hdr.channel) {
 	case CROS_EC_COMMAND:
-		/* Sanity check */
-		if (!client_data->response.data) {
+		if (client_data->response.received) {
 			dev_err(dev,
-				"Receiving buffer is null. Should be allocated by calling function\n");
-			client_data->response.error = -EINVAL;
-			goto error_wake_up;
+				"Previous firmware message not yet processed\n");
+			goto end_error;
 		}
 
-		if (client_data->response.received) {
+		if (client_data->response.token != in_msg->hdr.token) {
+			dev_err_ratelimited(dev,
+					    "Dropping old response token %d\n",
+					    in_msg->hdr.token);
+			goto end_error;
+		}
+
+		/* Sanity check */
+		if (!client_data->response.data) {
 			dev_err(dev,
-				"Previous firmware message not yet processed\n");
+				"Receiving buffer is null. Should be allocated by calling function\n");
 			client_data->response.error = -EINVAL;
 			goto error_wake_up;
 		}
@@ -289,21 +302,28 @@ static void process_recv(struct ishtp_cl *cros_ish_cl,
 		memcpy(client_data->response.data,
 		       rb_in_proc->buffer.data, data_len);
 
+error_wake_up:
+		/* Free the buffer since we copied data or didn't need it */
+		ishtp_cl_io_rb_recycle(rb_in_proc);
+		rb_in_proc = NULL;
+
 		/* Set flag before waking up the caller */
 		client_data->response.received = true;
-error_wake_up:
+
 		/* Wake the calling thread */
 		wake_up_interruptible(&client_data->response.wait_queue);
 
 		break;
 
 	case CROS_MKBP_EVENT:
+		/* Free the buffer. This is just an event without data */
+		ishtp_cl_io_rb_recycle(rb_in_proc);
+		rb_in_proc = NULL;
 		/*
 		 * Set timestamp from beginning of function since we actually
 		 * got an incoming MKBP event
 		 */
 		client_data->ec_dev->last_event_time = timestamp;
-		/* The event system doesn't send any data in buffer */
 		schedule_work(&client_data->work_ec_evt);
 
 		break;
@@ -313,8 +333,9 @@ error_wake_up:
 	}
 
 end_error:
-	/* Free the buffer */
-	ishtp_cl_io_rb_recycle(rb_in_proc);
+	/* Free the buffer if we already haven't */
+	if (rb_in_proc)
+		ishtp_cl_io_rb_recycle(rb_in_proc);
 
 	up_read(&init_lock);
 }
diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c
index 874269c07073..66b8d21092af 100644
--- a/drivers/platform/chrome/cros_ec_typec.c
+++ b/drivers/platform/chrome/cros_ec_typec.c
@@ -11,11 +11,22 @@
 #include <linux/of.h>
 #include <linux/platform_data/cros_ec_commands.h>
 #include <linux/platform_data/cros_ec_proto.h>
+#include <linux/platform_data/cros_usbpd_notify.h>
 #include <linux/platform_device.h>
 #include <linux/usb/typec.h>
 
 #define DRV_NAME "cros-ec-typec"
 
+/* Per port data. */
+struct cros_typec_port {
+	struct typec_port *port;
+	/* Initial capabilities for the port. */
+	struct typec_capability caps;
+	struct typec_partner *partner;
+	/* Port partner PD identity info. */
+	struct usb_pd_identity p_identity;
+};
+
 /* Platform-specific data for the Chrome OS EC Type C controller. */
 struct cros_typec_data {
 	struct device *dev;
@@ -23,9 +34,8 @@ struct cros_typec_data {
 	int num_ports;
 	unsigned int cmd_ver;
 	/* Array of ports, indexed by port number. */
-	struct typec_port *ports[EC_USB_PD_MAX_PORTS];
-	/* Initial capabilities for each port. */
-	struct typec_capability *caps[EC_USB_PD_MAX_PORTS];
+	struct cros_typec_port *ports[EC_USB_PD_MAX_PORTS];
+	struct notifier_block nb;
 };
 
 static int cros_typec_parse_port_props(struct typec_capability *cap,
@@ -74,14 +84,25 @@ static int cros_typec_parse_port_props(struct typec_capability *cap,
 	return 0;
 }
 
+static void cros_unregister_ports(struct cros_typec_data *typec)
+{
+	int i;
+
+	for (i = 0; i < typec->num_ports; i++) {
+		if (!typec->ports[i])
+			continue;
+		typec_unregister_port(typec->ports[i]->port);
+	}
+}
+
 static int cros_typec_init_ports(struct cros_typec_data *typec)
 {
 	struct device *dev = typec->dev;
 	struct typec_capability *cap;
 	struct fwnode_handle *fwnode;
+	struct cros_typec_port *cros_port;
 	const char *port_prop;
 	int ret;
-	int i;
 	int nports;
 	u32 port_num = 0;
 
@@ -113,22 +134,23 @@ static int cros_typec_init_ports(struct cros_typec_data *typec)
 
 		dev_dbg(dev, "Registering port %d\n", port_num);
 
-		cap = devm_kzalloc(dev, sizeof(*cap), GFP_KERNEL);
-		if (!cap) {
+		cros_port = devm_kzalloc(dev, sizeof(*cros_port), GFP_KERNEL);
+		if (!cros_port) {
 			ret = -ENOMEM;
 			goto unregister_ports;
 		}
 
-		typec->caps[port_num] = cap;
+		typec->ports[port_num] = cros_port;
+		cap = &cros_port->caps;
 
 		ret = cros_typec_parse_port_props(cap, fwnode, dev);
 		if (ret < 0)
 			goto unregister_ports;
 
-		typec->ports[port_num] = typec_register_port(dev, cap);
-		if (IS_ERR(typec->ports[port_num])) {
+		cros_port->port = typec_register_port(dev, cap);
+		if (IS_ERR(cros_port->port)) {
 			dev_err(dev, "Failed to register port %d\n", port_num);
-			ret = PTR_ERR(typec->ports[port_num]);
+			ret = PTR_ERR(cros_port->port);
 			goto unregister_ports;
 		}
 	}
@@ -136,8 +158,7 @@ static int cros_typec_init_ports(struct cros_typec_data *typec)
 	return 0;
 
 unregister_ports:
-	for (i = 0; i < typec->num_ports; i++)
-		typec_unregister_port(typec->ports[i]);
+	cros_unregister_ports(typec);
 	return ret;
 }
 
@@ -172,10 +193,34 @@ static int cros_typec_ec_command(struct cros_typec_data *typec,
 	return ret;
 }
 
+static int cros_typec_add_partner(struct cros_typec_data *typec, int port_num,
+				  bool pd_en)
+{
+	struct cros_typec_port *port = typec->ports[port_num];
+	struct typec_partner_desc p_desc = {
+		.usb_pd = pd_en,
+	};
+	int ret = 0;
+
+	/*
+	 * Fill an initial PD identity, which will then be updated with info
+	 * from the EC.
+	 */
+	p_desc.identity = &port->p_identity;
+
+	port->partner = typec_register_partner(port->port, &p_desc);
+	if (IS_ERR(port->partner)) {
+		ret = PTR_ERR(port->partner);
+		port->partner = NULL;
+	}
+
+	return ret;
+}
+
 static void cros_typec_set_port_params_v0(struct cros_typec_data *typec,
 		int port_num, struct ec_response_usb_pd_control *resp)
 {
-	struct typec_port *port = typec->ports[port_num];
+	struct typec_port *port = typec->ports[port_num]->port;
 	enum typec_orientation polarity;
 
 	if (!resp->enabled)
@@ -192,8 +237,10 @@ static void cros_typec_set_port_params_v0(struct cros_typec_data *typec,
 static void cros_typec_set_port_params_v1(struct cros_typec_data *typec,
 		int port_num, struct ec_response_usb_pd_control_v1 *resp)
 {
-	struct typec_port *port = typec->ports[port_num];
+	struct typec_port *port = typec->ports[port_num]->port;
 	enum typec_orientation polarity;
+	bool pd_en;
+	int ret;
 
 	if (!(resp->enabled & PD_CTRL_RESP_ENABLED_CONNECTED))
 		polarity = TYPEC_ORIENTATION_NONE;
@@ -208,6 +255,25 @@ static void cros_typec_set_port_params_v1(struct cros_typec_data *typec,
 			TYPEC_SOURCE : TYPEC_SINK);
 	typec_set_vconn_role(port, resp->role & PD_CTRL_RESP_ROLE_VCONN ?
 			TYPEC_SOURCE : TYPEC_SINK);
+
+	/* Register/remove partners when a connect/disconnect occurs. */
+	if (resp->enabled & PD_CTRL_RESP_ENABLED_CONNECTED) {
+		if (typec->ports[port_num]->partner)
+			return;
+
+		pd_en = resp->enabled & PD_CTRL_RESP_ENABLED_PD_CAPABLE;
+		ret = cros_typec_add_partner(typec, port_num, pd_en);
+		if (ret)
+			dev_warn(typec->dev,
+				 "Failed to register partner on port: %d\n",
+				 port_num);
+	} else {
+		if (!typec->ports[port_num]->partner)
+			return;
+
+		typec_unregister_partner(typec->ports[port_num]->partner);
+		typec->ports[port_num]->partner = NULL;
+	}
 }
 
 static int cros_typec_port_update(struct cros_typec_data *typec, int port_num)
@@ -272,6 +338,22 @@ static int cros_typec_get_cmd_version(struct cros_typec_data *typec)
 	return 0;
 }
 
+static int cros_ec_typec_event(struct notifier_block *nb,
+			       unsigned long host_event, void *_notify)
+{
+	struct cros_typec_data *typec = container_of(nb, struct cros_typec_data,
+						     nb);
+	int ret, i;
+
+	for (i = 0; i < typec->num_ports; i++) {
+		ret = cros_typec_port_update(typec, i);
+		if (ret < 0)
+			dev_warn(typec->dev, "Update failed for port: %d\n", i);
+	}
+
+	return NOTIFY_OK;
+}
+
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id cros_typec_acpi_id[] = {
 	{ "GOOG0014", 0 },
@@ -332,12 +414,15 @@ static int cros_typec_probe(struct platform_device *pdev)
 			goto unregister_ports;
 	}
 
+	typec->nb.notifier_call = cros_ec_typec_event;
+	ret = cros_usbpd_register_notify(&typec->nb);
+	if (ret < 0)
+		goto unregister_ports;
+
 	return 0;
 
 unregister_ports:
-	for (i = 0; i < typec->num_ports; i++)
-		if (typec->ports[i])
-			typec_unregister_port(typec->ports[i]);
+	cros_unregister_ports(typec);
 	return ret;
 }
 
diff --git a/drivers/platform/chrome/cros_usbpd_logger.c b/drivers/platform/chrome/cros_usbpd_logger.c
index 7de3ea75ef46..d16931203d82 100644
--- a/drivers/platform/chrome/cros_usbpd_logger.c
+++ b/drivers/platform/chrome/cros_usbpd_logger.c
@@ -46,6 +46,7 @@ static const char * const fault_names[] = {
 	"---", "OCP", "fast OCP", "OVP", "Discharge"
 };
 
+__printf(3, 4)
 static int append_str(char *buf, int pos, const char *fmt, ...)
 {
 	va_list args;
diff --git a/drivers/platform/chrome/wilco_ec/debugfs.c b/drivers/platform/chrome/wilco_ec/debugfs.c
index df5a5f6c3ec6..a812788a0bdc 100644
--- a/drivers/platform/chrome/wilco_ec/debugfs.c
+++ b/drivers/platform/chrome/wilco_ec/debugfs.c
@@ -208,7 +208,12 @@ static int send_ec_cmd(struct wilco_ec_device *ec, u8 sub_cmd, u8 *out_val)
  */
 static int h1_gpio_get(void *arg, u64 *val)
 {
-	return send_ec_cmd(arg, SUB_CMD_H1_GPIO, (u8 *)val);
+	int ret;
+
+	ret = send_ec_cmd(arg, SUB_CMD_H1_GPIO, (u8 *)val);
+	if (ret == 0)
+		*val &= 0xFF;
+	return ret;
 }
 
 DEFINE_DEBUGFS_ATTRIBUTE(fops_h1_gpio, h1_gpio_get, NULL, "0x%02llx\n");
diff --git a/drivers/platform/mellanox/Kconfig b/drivers/platform/mellanox/Kconfig
index 56e037623b29..916b39dc11bc 100644
--- a/drivers/platform/mellanox/Kconfig
+++ b/drivers/platform/mellanox/Kconfig
@@ -6,7 +6,7 @@
 menuconfig MELLANOX_PLATFORM
 	bool "Platform support for Mellanox hardware"
 	depends on X86 || ARM || ARM64 || COMPILE_TEST
-	---help---
+	help
 	  Say Y here to get to see options for platform support for
 	  Mellanox systems. This option alone does not add any kernel code.
 
@@ -19,7 +19,7 @@ config MLXREG_HOTPLUG
 	depends on REGMAP
 	depends on HWMON
 	depends on I2C
-	---help---
+	help
 	  This driver handles hot-plug events for the power suppliers, power
 	  cables and fans on the wide range Mellanox IB and Ethernet systems.
 
diff --git a/drivers/platform/mellanox/mlxreg-hotplug.c b/drivers/platform/mellanox/mlxreg-hotplug.c
index 77be37a1fbcf..ed48917af162 100644
--- a/drivers/platform/mellanox/mlxreg-hotplug.c
+++ b/drivers/platform/mellanox/mlxreg-hotplug.c
@@ -101,6 +101,7 @@ static int mlxreg_hotplug_device_create(struct mlxreg_hotplug_priv_data *priv,
 					struct mlxreg_core_data *data)
 {
 	struct mlxreg_core_hotplug_platform_data *pdata;
+	struct i2c_client *client;
 
 	/* Notify user by sending hwmon uevent. */
 	kobject_uevent(&priv->hwmon->kobj, KOBJ_CHANGE);
@@ -121,18 +122,20 @@ static int mlxreg_hotplug_device_create(struct mlxreg_hotplug_priv_data *priv,
 		return -EFAULT;
 	}
 
-	data->hpdev.client = i2c_new_device(data->hpdev.adapter,
-					    data->hpdev.brdinfo);
-	if (!data->hpdev.client) {
+	client = i2c_new_client_device(data->hpdev.adapter,
+				       data->hpdev.brdinfo);
+	if (IS_ERR(client)) {
 		dev_err(priv->dev, "Failed to create client %s at bus %d at addr 0x%02x\n",
 			data->hpdev.brdinfo->type, data->hpdev.nr +
 			pdata->shift_nr, data->hpdev.brdinfo->addr);
 
 		i2c_put_adapter(data->hpdev.adapter);
 		data->hpdev.adapter = NULL;
-		return -EFAULT;
+		return PTR_ERR(client);
 	}
 
+	data->hpdev.client = client;
+
 	return 0;
 }
 
diff --git a/drivers/platform/olpc/olpc-xo175-ec.c b/drivers/platform/olpc/olpc-xo175-ec.c
index 83ed1fbf73cf..5e1d14e35f20 100644
--- a/drivers/platform/olpc/olpc-xo175-ec.c
+++ b/drivers/platform/olpc/olpc-xo175-ec.c
@@ -410,7 +410,7 @@ static void olpc_xo175_ec_complete(void *arg)
 		dev_dbg(dev, "got event %.2x\n", byte);
 		switch (byte) {
 		case EVENT_AC_CHANGE:
-			psy = power_supply_get_by_name("olpc-ac");
+			psy = power_supply_get_by_name("olpc_ac");
 			if (psy) {
 				power_supply_changed(psy);
 				power_supply_put(psy);
@@ -420,7 +420,7 @@ static void olpc_xo175_ec_complete(void *arg)
 		case EVENT_BATTERY_CRITICAL:
 		case EVENT_BATTERY_SOC_CHANGE:
 		case EVENT_BATTERY_ERROR:
-			psy = power_supply_get_by_name("olpc-battery");
+			psy = power_supply_get_by_name("olpc_battery");
 			if (psy) {
 				power_supply_changed(psy);
 				power_supply_put(psy);
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index ee93dcd065b8..0581a54cf562 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -7,7 +7,7 @@ menuconfig X86_PLATFORM_DEVICES
 	bool "X86 Platform Specific Device Drivers"
 	default y
 	depends on X86
-	---help---
+	help
 	  Say Y here to get to see options for device drivers for various
 	  x86 platforms, including vendor-specific laptop extension drivers.
 	  This option alone does not add any kernel code.
@@ -41,7 +41,7 @@ config WMI_BMOF
 	tristate "WMI embedded Binary MOF driver"
 	depends on ACPI_WMI
 	default ACPI_WMI
-	---help---
+	help
 	  Say Y here if you want to be able to read a firmware-embedded
 	  WMI Binary MOF data. Using this requires userspace tools and may be
 	  rather tedious.
@@ -55,7 +55,7 @@ config ALIENWARE_WMI
 	depends on LEDS_CLASS
 	depends on NEW_LEDS
 	depends on ACPI_WMI
-	---help---
+	help
 	 This is a driver for controlling Alienware BIOS driven
 	 features.  It exposes an interface for controlling the AlienFX
 	 zones on Alienware machines that don't contain a dedicated AlienFX
@@ -91,7 +91,7 @@ config INTEL_WMI_SBL_FW_UPDATE
 config INTEL_WMI_THUNDERBOLT
 	tristate "Intel WMI thunderbolt force power driver"
 	depends on ACPI_WMI
-	---help---
+	help
 	  Say Y here if you want to be able to use the WMI interface on select
 	  systems to force the power control of Intel Thunderbolt controllers.
 	  This is useful for updating the firmware when devices are not plugged
@@ -103,7 +103,7 @@ config INTEL_WMI_THUNDERBOLT
 config MXM_WMI
        tristate "WMI support for MXM Laptop Graphics"
        depends on ACPI_WMI
-       ---help---
+	help
           MXM is a standard for laptop graphics cards, the WMI interface
 	  is required for switchable nvidia graphics machines
 
@@ -128,7 +128,7 @@ config ACERHDF
 	tristate "Acer Aspire One temperature and fan driver"
 	depends on ACPI && THERMAL
 	select THERMAL_GOV_BANG_BANG
-	---help---
+	help
 	  This is a driver for Acer Aspire One netbooks. It allows to access
 	  the temperature sensor and to control the fan.
 
@@ -149,7 +149,7 @@ config ACER_WIRELESS
         tristate "Acer Wireless Radio Control Driver"
         depends on ACPI
         depends on INPUT
-        ---help---
+	help
           The Acer Wireless Radio Control handles the airplane mode hotkey
           present on new Acer laptops.
 
@@ -172,7 +172,7 @@ config ACER_WMI
 	select INPUT_SPARSEKMAP
 	# Acer WMI depends on ACPI_VIDEO when ACPI is enabled
         select ACPI_VIDEO if ACPI
-	---help---
+	help
 	  This is a driver for newer Acer (and Wistron) laptops. It adds
 	  wireless radio and bluetooth control, and on some laptops,
 	  exposes the mail LED and LCD backlight.
@@ -187,7 +187,7 @@ config APPLE_GMUX
 	depends on BACKLIGHT_CLASS_DEVICE
 	depends on BACKLIGHT_APPLE=n || BACKLIGHT_APPLE
 	depends on ACPI_VIDEO=n || ACPI_VIDEO
-	---help---
+	help
 	  This driver provides support for the gmux device found on many
 	  Apple laptops, which controls the display mux for the hybrid
 	  graphics as well as the backlight. Currently only backlight
@@ -203,7 +203,7 @@ config ASUS_LAPTOP
 	depends on RFKILL || RFKILL = n
 	depends on ACPI_VIDEO || ACPI_VIDEO = n
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  This is a driver for Asus laptops, Lenovo SL and the Pegatron
 	  Lucid tablet. It may also support some MEDION, JVC or VICTOR
 	  laptops. It makes all the extra buttons generate standard
@@ -223,7 +223,7 @@ config ASUS_WIRELESS
 	depends on INPUT
 	select NEW_LEDS
 	select LEDS_CLASS
-	---help---
+	help
 	  The Asus Wireless Radio Control handles the airplane mode hotkey
 	  present on some Asus laptops.
 
@@ -246,7 +246,7 @@ config ASUS_WMI
 	select INPUT_SPARSEKMAP
 	select LEDS_CLASS
 	select NEW_LEDS
-	---help---
+	help
 	  Say Y here if you have a WMI aware Asus laptop (like Eee PCs or new
 	  Asus Notebooks).
 
@@ -257,7 +257,7 @@ config ASUS_NB_WMI
 	tristate "Asus Notebook WMI Driver"
 	depends on ASUS_WMI
 	depends on SERIO_I8042 || SERIO_I8042 = n
-	---help---
+	help
 	  This is a driver for newer Asus notebooks. It adds extra features
 	  like wireless radio and bluetooth control, leds, hotkeys, backlight...
 
@@ -279,7 +279,7 @@ config EEEPC_LAPTOP
 	select LEDS_CLASS
 	select NEW_LEDS
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  This driver supports the Fn-Fx keys on Eee PC laptops.
 
 	  It  also gives access to some extra laptop functionalities like
@@ -292,7 +292,7 @@ config EEEPC_LAPTOP
 config EEEPC_WMI
 	tristate "Eee PC WMI Driver"
 	depends on ASUS_WMI
-	---help---
+	help
 	  This is a driver for newer Eee PC laptops. It adds extra features
 	  like wireless radio and bluetooth control, leds, hotkeys, backlight...
 
@@ -327,7 +327,7 @@ config DELL_SMBIOS
 	tristate "Dell SMBIOS driver"
 	depends on DCDBAS || DCDBAS=n
 	depends on ACPI_WMI || ACPI_WMI=n
-	---help---
+	help
 	This provides support for the Dell SMBIOS calling interface.
 	If you have a Dell computer you should enable this option.
 
@@ -339,7 +339,7 @@ config DELL_SMBIOS_WMI
 	depends on ACPI_WMI
 	select DELL_WMI_DESCRIPTOR
 	depends on DELL_SMBIOS
-	---help---
+	help
 	This provides an implementation for the Dell SMBIOS calling interface
 	communicated over ACPI-WMI.
 
@@ -352,7 +352,7 @@ config DELL_SMBIOS_SMM
 	default y
 	depends on DCDBAS
 	depends on DELL_SMBIOS
-	---help---
+	help
 	This provides an implementation for the Dell SMBIOS calling interface
 	communicated over SMI/SMM.
 
@@ -373,7 +373,7 @@ config DELL_LAPTOP
 	select NEW_LEDS
 	select LEDS_TRIGGERS
 	select LEDS_TRIGGER_AUDIO
-	---help---
+	help
 	This driver adds support for rfkill and backlight control to Dell
 	laptops (except for some models covered by the Compal driver).
 
@@ -382,7 +382,7 @@ config DELL_RBTN
 	depends on ACPI
 	depends on INPUT
 	depends on RFKILL
-	---help---
+	help
 	  Say Y here if you want to support Dell Airplane Mode Switch ACPI
 	  device on Dell laptops. Sometimes it has names: DELLABCE or DELRBTN.
 	  This driver register rfkill device or input hotkey device depending
@@ -408,7 +408,7 @@ config DELL_RBU
 config DELL_SMO8800
 	tristate "Dell Latitude freefall driver (ACPI SMO88XX)"
 	depends on ACPI
-	---help---
+	help
 	  Say Y here if you want to support SMO88XX freefall devices
 	  on Dell Latitude laptops.
 
@@ -424,7 +424,7 @@ config DELL_WMI
 	depends on DELL_SMBIOS
 	select DELL_WMI_DESCRIPTOR
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  Say Y here if you want to support WMI-based hotkeys on Dell laptops.
 
 	  To compile this driver as a module, choose M here: the module will
@@ -439,7 +439,7 @@ config DELL_WMI_AIO
 	depends on ACPI_WMI
 	depends on INPUT
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  Say Y here if you want to support WMI-based hotkeys on Dell
 	  All-In-One machines.
 
@@ -458,7 +458,7 @@ config AMILO_RFKILL
 	tristate "Fujitsu-Siemens Amilo rfkill support"
 	depends on RFKILL
 	depends on SERIO_I8042
-	---help---
+	help
 	  This is a driver for enabling wifi on some Fujitsu-Siemens Amilo
 	  laptops.
 
@@ -470,7 +470,7 @@ config FUJITSU_LAPTOP
 	depends on ACPI_VIDEO || ACPI_VIDEO = n
 	select INPUT_SPARSEKMAP
 	select LEDS_CLASS
-	---help---
+	help
 	  This is a driver for laptops built by Fujitsu:
 
 	    * P2xxx/P5xxx/S6xxx/S7xxx series Lifebooks
@@ -485,7 +485,7 @@ config FUJITSU_TABLET
        tristate "Fujitsu Tablet Extras"
        depends on ACPI
        depends on INPUT
-       ---help---
+	help
          This is a driver for tablets built by Fujitsu:
 
            * Lifebook P1510/P1610/P1620/Txxxx
@@ -502,7 +502,7 @@ config GPD_POCKET_FAN
 	tristate "GPD Pocket Fan Controller support"
 	depends on ACPI
 	depends on THERMAL
-	---help---
+	help
 	  Driver for the GPD Pocket vendor specific FAN02501 ACPI device
 	  which controls the fan speed on the GPD Pocket.
 
@@ -558,14 +558,14 @@ config TC1100_WMI
 	depends on !X86_64
 	depends on ACPI
 	depends on ACPI_WMI
-	---help---
+	help
 	  This is a driver for the WMI extensions (wireless and bluetooth power
 	  control) of the HP Compaq TC1100 tablet.
 
 config IBM_RTL
 	tristate "Device driver to enable PRTL support"
 	depends on PCI
-	---help---
+	help
 	 Enable support for IBM Premium Real Time Mode (PRTM).
 	 This module will allow you the enter and exit PRTM in the BIOS via
 	 sysfs on platforms that support this feature.  System in PRTM will
@@ -623,7 +623,7 @@ config THINKPAD_ACPI
 	select LEDS_CLASS
 	select LEDS_TRIGGERS
 	select LEDS_TRIGGER_AUDIO
-	---help---
+	help
 	  This is a driver for the IBM and Lenovo ThinkPad laptops. It adds
 	  support for Fn-Fx key combinations, Bluetooth control, video
 	  output switching, ThinkLight control, UltraBay eject and more.
@@ -646,7 +646,7 @@ config THINKPAD_ACPI_ALSA_SUPPORT
 	depends on SND
 	depends on SND = y || THINKPAD_ACPI = SND
 	default y
-	---help---
+	help
 	  Enables monitoring of the built-in console audio output control
 	  (headphone and speakers), which is operated by the mute and (in
 	  some ThinkPad models) volume hotkeys.
@@ -666,7 +666,7 @@ config THINKPAD_ACPI_ALSA_SUPPORT
 config THINKPAD_ACPI_DEBUGFACILITIES
 	bool "Maintainer debug facilities"
 	depends on THINKPAD_ACPI
-	---help---
+	help
 	  Enables extra stuff in the thinkpad-acpi which is completely useless
 	  for normal use.  Read the driver source to find out what it does.
 
@@ -676,7 +676,7 @@ config THINKPAD_ACPI_DEBUGFACILITIES
 config THINKPAD_ACPI_DEBUG
 	bool "Verbose debug mode"
 	depends on THINKPAD_ACPI
-	---help---
+	help
 	  Enables extra debugging information, at the expense of a slightly
 	  increase in driver size.
 
@@ -685,7 +685,7 @@ config THINKPAD_ACPI_DEBUG
 config THINKPAD_ACPI_UNSAFE_LEDS
 	bool "Allow control of important LEDs (unsafe)"
 	depends on THINKPAD_ACPI
-	---help---
+	help
 	  Overriding LED state on ThinkPads can mask important
 	  firmware alerts (like critical battery condition), or misled
 	  the user into damaging the hardware (undocking or ejecting
@@ -709,7 +709,7 @@ config THINKPAD_ACPI_VIDEO
 	bool "Video output control support"
 	depends on THINKPAD_ACPI
 	default y
-	---help---
+	help
 	  Allows the thinkpad_acpi driver to provide an interface to control
 	  the various video output ports.
 
@@ -732,7 +732,7 @@ config THINKPAD_ACPI_HOTKEY_POLL
 	bool "Support NVRAM polling for hot keys"
 	depends on THINKPAD_ACPI
 	default y
-	---help---
+	help
 	  Some thinkpad models benefit from NVRAM polling to detect a few of
 	  the hot key press events.  If you know your ThinkPad model does not
 	  need to do NVRAM polling to support any of the hot keys you use,
@@ -767,7 +767,7 @@ config INTEL_CHT_INT33FE
 	depends on CHARGER_BQ24190=y || (CHARGER_BQ24190=m && m)
 	depends on USB_ROLES_INTEL_XHCI=y || (USB_ROLES_INTEL_XHCI=m && m)
 	depends on TYPEC_MUX_PI3USB30532=y || (TYPEC_MUX_PI3USB30532=m && m)
-	---help---
+	help
 	  This driver add support for the INT33FE ACPI device found on
 	  some Intel Cherry Trail devices.
 
@@ -801,7 +801,7 @@ config INTEL_INT0002_VGPIO
 	tristate "Intel ACPI INT0002 Virtual GPIO driver"
 	depends on GPIOLIB && ACPI
 	select GPIOLIB_IRQCHIP
-	---help---
+	help
 	  Some peripherals on Bay Trail and Cherry Trail platforms signal a
 	  Power Management Event (PME) to the Power Management Controller (PMC)
 	  to wakeup the system. When this happens software needs to explicitly
@@ -820,7 +820,7 @@ config INTEL_MENLOW
 	tristate "Thermal Management driver for Intel menlow platform"
 	depends on ACPI_THERMAL
 	select THERMAL
-	---help---
+	help
 	  ACPI thermal management enhancement driver on
 	  Intel Menlow platform.
 
@@ -831,7 +831,7 @@ config INTEL_OAKTRAIL
 	depends on ACPI
 	depends on ACPI_VIDEO || ACPI_VIDEO = n
 	depends on RFKILL && BACKLIGHT_CLASS_DEVICE && ACPI
-	---help---
+	help
 	  Intel Oaktrail platform need this driver to provide interfaces to
 	  enable/disable the Camera, WiFi, BT etc. devices. If in doubt, say Y
 	  here; it will only load on supported platforms.
@@ -854,7 +854,7 @@ config SURFACE3_WMI
 	depends on DMI
 	depends on INPUT
 	depends on SPI
-	---help---
+	help
 	  Say Y here if you have a Surface 3.
 
 	  To compile this driver as a module, choose M here: the module will
@@ -863,7 +863,7 @@ config SURFACE3_WMI
 config SURFACE_3_BUTTON
 	tristate "Power/home/volume buttons driver for Microsoft Surface 3 tablet"
 	depends on ACPI && KEYBOARD_GPIO && I2C
-	---help---
+	help
 	  This driver handles the power/home/volume buttons on the Microsoft Surface 3 tablet.
 
 config SURFACE_3_POWER_OPREGION
@@ -876,7 +876,7 @@ config SURFACE_3_POWER_OPREGION
 config SURFACE_PRO3_BUTTON
 	tristate "Power/home/volume buttons driver for Microsoft Surface Pro 3/4 tablet"
 	depends on ACPI && INPUT
-	---help---
+	help
 	  This driver handles the power/home/volume buttons on the Microsoft Surface Pro 3/4 tablet.
 
 config MSI_LAPTOP
@@ -887,7 +887,7 @@ config MSI_LAPTOP
 	depends on RFKILL
 	depends on INPUT && SERIO_I8042
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  This is a driver for laptops built by MSI (MICRO-STAR
 	  INTERNATIONAL):
 
@@ -918,7 +918,7 @@ config XO15_EBOOK
 	tristate "OLPC XO-1.5 ebook switch"
 	depends on OLPC || COMPILE_TEST
 	depends on ACPI && INPUT
-	---help---
+	help
 	  Support for the ebook switch on the OLPC XO-1.5 laptop.
 
 	  This switch is triggered as the screen is rotated and folded down to
@@ -928,7 +928,7 @@ config XO1_RFKILL
 	tristate "OLPC XO-1 software RF kill switch"
 	depends on OLPC || COMPILE_TEST
 	depends on RFKILL
-	---help---
+	help
 	  Support for enabling/disabling the WLAN interface on the OLPC XO-1
 	  laptop.
 
@@ -953,7 +953,7 @@ config SAMSUNG_LAPTOP
 	depends on BACKLIGHT_CLASS_DEVICE
 	select LEDS_CLASS
 	select NEW_LEDS
-	---help---
+	help
 	  This module implements a driver for a wide range of different
 	  Samsung laptops.  It offers control over the different
 	  function keys, wireless LED, LCD backlight level.
@@ -968,7 +968,7 @@ config SAMSUNG_Q10
 	tristate "Samsung Q10 Extras"
 	depends on ACPI
 	select BACKLIGHT_CLASS_DEVICE
-	---help---
+	help
 	  This driver provides support for backlight control on Samsung Q10
 	  and related laptops, including Dell Latitude X200.
 
@@ -985,7 +985,7 @@ config ACPI_TOSHIBA
 	depends on RFKILL || RFKILL = n
 	depends on IIO
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  This driver adds support for access to certain system settings
 	  on "legacy free" Toshiba laptops.  These laptops can be recognized by
 	  their lack of a BIOS setup menu and APM support.
@@ -1012,7 +1012,7 @@ config TOSHIBA_BT_RFKILL
 	tristate "Toshiba Bluetooth RFKill switch support"
 	depends on ACPI
 	depends on RFKILL || RFKILL = n
-	---help---
+	help
 	  This driver adds support for Bluetooth events for the RFKill
 	  switch on modern Toshiba laptops with full ACPI support and
 	  an RFKill switch.
@@ -1027,7 +1027,7 @@ config TOSHIBA_BT_RFKILL
 config TOSHIBA_HAPS
 	tristate "Toshiba HDD Active Protection Sensor"
 	depends on ACPI
-	---help---
+	help
 	  This driver adds support for the built-in accelerometer
 	  found on recent Toshiba laptops equipped with HID TOS620A
 	  device.
@@ -1048,7 +1048,7 @@ config TOSHIBA_WMI
 	depends on ACPI_WMI
 	depends on INPUT
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  This driver adds hotkey monitoring support to some Toshiba models
 	  that manage the hotkeys via WMI events.
 
@@ -1078,7 +1078,7 @@ config COMPAL_LAPTOP
 	depends on RFKILL
 	depends on HWMON
 	depends on POWER_SUPPLY
-	---help---
+	help
 	  This is a driver for laptops built by Compal, and some models by
 	  other brands (e.g. Dell, Toshiba).
 
@@ -1104,7 +1104,7 @@ config PANASONIC_LAPTOP
 	depends on INPUT && ACPI
 	depends on BACKLIGHT_CLASS_DEVICE
 	select INPUT_SPARSEKMAP
-	---help---
+	help
 	  This driver adds support for access to backlight control and hotkeys
 	  on Panasonic Let's Note laptops.
 
@@ -1118,7 +1118,7 @@ config SONY_LAPTOP
 	depends on BACKLIGHT_CLASS_DEVICE
 	depends on INPUT
 	depends on RFKILL
-	---help---
+	help
 	  This mini-driver drives the SNC and SPIC devices present in the ACPI
 	  BIOS of the Sony Vaio laptops.
 
@@ -1131,7 +1131,7 @@ config SONY_LAPTOP
 config SONYPI_COMPAT
 	bool "Sonypi compatibility"
 	depends on SONY_LAPTOP
-	---help---
+	help
 	  Build the sonypi driver compatibility code into the sony-laptop driver.
 
 config SYSTEM76_ACPI
@@ -1154,7 +1154,7 @@ config TOPSTAR_LAPTOP
 	select INPUT_SPARSEKMAP
 	select LEDS_CLASS
 	select NEW_LEDS
-	---help---
+	help
 	  This driver adds support for hotkeys found on Topstar laptops.
 
 	  If you have a Topstar laptop, say Y or M here.
@@ -1173,7 +1173,7 @@ config I2C_MULTI_INSTANTIATE
 config MLX_PLATFORM
 	tristate "Mellanox Technologies platform support"
 	depends on I2C && REGMAP
-	---help---
+	help
 	  This option enables system support for the Mellanox Technologies
 	  platform. The Mellanox systems provide data center networking
 	  solutions based on Virtual Protocol Interconnect (VPI) technology
@@ -1186,7 +1186,7 @@ config TOUCHSCREEN_DMI
 	bool "DMI based touchscreen configuration info"
 	depends on ACPI && DMI && I2C=y && TOUCHSCREEN_SILEAD
 	select EFI_EMBEDDED_FIRMWARE if EFI
-	---help---
+	help
 	  Certain ACPI based tablets with e.g. Silead or Chipone touchscreens
 	  do not have enough data in ACPI tables for the touchscreen driver to
 	  handle the touchscreen properly, as OEMs expect the data to be baked
@@ -1197,7 +1197,7 @@ config TOUCHSCREEN_DMI
 config INTEL_IMR
 	bool "Intel Isolated Memory Region support"
 	depends on X86_INTEL_QUARK && IOSF_MBI
-	---help---
+	help
 	  This option provides a means to manipulate Isolated Memory Regions.
 	  IMRs are a set of registers that define read and write access masks
 	  to prohibit certain system agents from accessing memory with 1 KiB
@@ -1221,7 +1221,7 @@ config INTEL_IMR
 config INTEL_IPS
 	tristate "Intel Intelligent Power Sharing"
 	depends on ACPI && PCI
-	---help---
+	help
 	  Intel Calpella platforms support dynamic power sharing between the
 	  CPU and GPU, maximizing performance in a given TDP.  This driver,
 	  along with the CPU frequency and i915 drivers, provides that
@@ -1231,7 +1231,7 @@ config INTEL_IPS
 config INTEL_RST
         tristate "Intel Rapid Start Technology Driver"
 	depends on ACPI
-	---help---
+	help
 	  This driver provides support for modifying parameters on systems
 	  equipped with Intel's Rapid Start Technology. When put in an ACPI
 	  sleep state, these devices will wake after either a configured
@@ -1243,7 +1243,7 @@ config INTEL_RST
 config INTEL_SMARTCONNECT
         tristate "Intel Smart Connect disabling driver"
 	depends on ACPI
-	---help---
+	help
 	  Intel Smart Connect is a technology intended to permit devices to
 	  update state by resuming for a short period of time at regular
 	  intervals. If a user enables this functionality under Windows and
@@ -1259,7 +1259,7 @@ source "drivers/platform/x86/intel_speed_select_if/Kconfig"
 config INTEL_TURBO_MAX_3
 	bool "Intel Turbo Boost Max Technology 3.0 enumeration driver"
 	depends on X86_64 && SCHED_MC_PRIO
-	---help---
+	help
 	  This driver reads maximum performance ratio of each CPU and set up
 	  the scheduler priority metrics. In this way scheduler can prefer
 	  CPU with higher performance to schedule tasks.
@@ -1282,7 +1282,7 @@ config INTEL_BXTWC_PMIC_TMU
 	depends on REGMAP
 	depends on MFD_INTEL_PMC_BXT
 	depends on INTEL_SOC_PMIC_BXTWC
-	---help---
+	help
 	  Select this driver to use Intel BXT Whiskey Cove PMIC TMU feature.
 	  This driver enables the alarm wakeup functionality in the TMU unit
 	  of Whiskey Cove PMIC.
@@ -1291,7 +1291,7 @@ config INTEL_CHTDC_TI_PWRBTN
 	tristate "Intel Cherry Trail Dollar Cove TI power button driver"
 	depends on INTEL_SOC_PMIC_CHTDC_TI
 	depends on INPUT
-	---help---
+	help
 	  This option adds a power button driver driver for Dollar Cove TI
 	  PMIC on Intel Cherry Trail devices.
 
@@ -1317,7 +1317,7 @@ config INTEL_MRFLD_PWRBTN
 	tristate "Intel Merrifield Basin Cove power button driver"
 	depends on INTEL_SOC_PMIC_MRFLD
 	depends on INPUT
-	---help---
+	help
 	  This option adds a power button driver for Basin Cove PMIC
 	  on Intel Merrifield devices.
 
@@ -1327,7 +1327,7 @@ config INTEL_MRFLD_PWRBTN
 config INTEL_PMC_CORE
 	tristate "Intel PMC Core driver"
 	depends on PCI
-	---help---
+	help
 	  The Intel Platform Controller Hub for Intel Core SoCs provides access
 	  to Power Management Controller registers via a PCI interface. This
 	  driver can utilize debugging capabilities and supported features as
@@ -1341,7 +1341,7 @@ config INTEL_PMC_CORE
 
 config INTEL_PUNIT_IPC
 	tristate "Intel P-Unit IPC Driver"
-	---help---
+	help
 	  This driver provides support for Intel P-Unit Mailbox IPC mechanism,
 	  which is used to bridge the communications between kernel and P-Unit.
 
@@ -1379,7 +1379,7 @@ config INTEL_SCU_PLATFORM
 config INTEL_SCU_IPC_UTIL
 	tristate "Intel SCU IPC utility driver"
 	depends on INTEL_SCU
-	---help---
+	help
 	  The IPC Util driver provides an interface with the SCU enabling
 	  low level access for debug work and updating the firmware. Say
 	  N unless you will be doing this on an Intel MID platform.
@@ -1389,7 +1389,7 @@ config INTEL_TELEMETRY
 	depends on X86_64
 	depends on MFD_INTEL_PMC_BXT
 	depends on INTEL_PUNIT_IPC
-	---help---
+	help
 	  This driver provides interfaces to configure and use
 	  telemetry for INTEL SoC from APL onwards. It is also
 	  used to get various SoC events and parameters
diff --git a/drivers/pnp/Kconfig b/drivers/pnp/Kconfig
index 39191bebcb28..a72141dffa29 100644
--- a/drivers/pnp/Kconfig
+++ b/drivers/pnp/Kconfig
@@ -7,7 +7,7 @@ menuconfig PNP
 	bool "Plug and Play support"
 	depends on HAS_IOMEM
 	depends on ISA || ACPI
-	---help---
+	help
 	  Plug and Play (PnP) is a standard for peripherals which allows those
 	  peripherals to be configured by software, e.g. assign IRQ's or other
 	  parameters. No jumpers on the cards are needed, instead the values
diff --git a/drivers/pnp/pnpbios/Kconfig b/drivers/pnp/pnpbios/Kconfig
index 7da992b70945..bcdac269af33 100644
--- a/drivers/pnp/pnpbios/Kconfig
+++ b/drivers/pnp/pnpbios/Kconfig
@@ -6,7 +6,7 @@ config PNPBIOS
 	bool "Plug and Play BIOS support"
 	depends on ISA && X86_32
 	default n
-	---help---
+	help
 	  Linux uses the PNPBIOS as defined in "Plug and Play BIOS
 	  Specification Version 1.0A May 5, 1994" to autodetect built-in
 	  mainboard resources (e.g. parallel port resources).
@@ -26,7 +26,7 @@ config PNPBIOS
 config PNPBIOS_PROC_FS
 	bool "Plug and Play BIOS /proc interface"
 	depends on PNPBIOS && PROC_FS
-	---help---
+	help
 	  If you say Y here and to "/proc file system support", you will be
 	  able to directly access the PNPBIOS.  This includes resource
 	  allocation, ESCD, and other PNPBIOS services.  Using this
diff --git a/drivers/power/reset/Kconfig b/drivers/power/reset/Kconfig
index 890380302080..f07b982c8dff 100644
--- a/drivers/power/reset/Kconfig
+++ b/drivers/power/reset/Kconfig
@@ -123,6 +123,13 @@ config POWER_RESET_OCELOT_RESET
 	help
 	  This driver supports restart for Microsemi Ocelot SoC.
 
+config POWER_RESET_OXNAS
+	bool "OXNAS SoC restart driver"
+	depends on ARCH_OXNAS
+	default MACH_OX820
+	help
+	  Restart support for OXNAS/PLXTECH OX820 SoC.
+
 config POWER_RESET_PIIX4_POWEROFF
 	tristate "Intel PIIX4 power-off driver"
 	depends on PCI
@@ -184,7 +191,7 @@ config POWER_RESET_VERSATILE
 config POWER_RESET_VEXPRESS
 	bool "ARM Versatile Express power-off and reset driver"
 	depends on ARM || ARM64
-	depends on VEXPRESS_CONFIG
+	depends on VEXPRESS_CONFIG=y
 	help
 	  Power off and reset support for the ARM Ltd. Versatile
 	  Express boards.
diff --git a/drivers/power/reset/Makefile b/drivers/power/reset/Makefile
index da37f8b851dc..5710ca469517 100644
--- a/drivers/power/reset/Makefile
+++ b/drivers/power/reset/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_POWER_RESET_GPIO_RESTART) += gpio-restart.o
 obj-$(CONFIG_POWER_RESET_HISI) += hisi-reboot.o
 obj-$(CONFIG_POWER_RESET_MSM) += msm-poweroff.o
 obj-$(CONFIG_POWER_RESET_MT6323) += mt6323-poweroff.o
+obj-$(CONFIG_POWER_RESET_OXNAS) += oxnas-restart.o
 obj-$(CONFIG_POWER_RESET_QCOM_PON) += qcom-pon.o
 obj-$(CONFIG_POWER_RESET_OCELOT_RESET) += ocelot-reset.o
 obj-$(CONFIG_POWER_RESET_PIIX4_POWEROFF) += piix4-poweroff.o
diff --git a/drivers/power/reset/gpio-poweroff.c b/drivers/power/reset/gpio-poweroff.c
index 6a4bbb506551..c5067eb75370 100644
--- a/drivers/power/reset/gpio-poweroff.c
+++ b/drivers/power/reset/gpio-poweroff.c
@@ -54,7 +54,7 @@ static int gpio_poweroff_probe(struct platform_device *pdev)
 	/* If a pm_power_off function has already been added, leave it alone */
 	if (pm_power_off != NULL) {
 		dev_err(&pdev->dev,
-			"%s: pm_power_off function already registered",
+			"%s: pm_power_off function already registered\n",
 		       __func__);
 		return -EBUSY;
 	}
diff --git a/drivers/power/reset/ltc2952-poweroff.c b/drivers/power/reset/ltc2952-poweroff.c
index e4a0cc45b3d1..318927938b05 100644
--- a/drivers/power/reset/ltc2952-poweroff.c
+++ b/drivers/power/reset/ltc2952-poweroff.c
@@ -94,7 +94,6 @@ static enum hrtimer_restart ltc2952_poweroff_timer_wde(struct hrtimer *timer)
 {
 	ktime_t now;
 	int state;
-	unsigned long overruns;
 	struct ltc2952_poweroff *data = to_ltc2952(timer, timer_wde);
 
 	if (data->kernel_panic)
@@ -104,7 +103,7 @@ static enum hrtimer_restart ltc2952_poweroff_timer_wde(struct hrtimer *timer)
 	gpiod_set_value(data->gpio_watchdog, !state);
 
 	now = hrtimer_cb_get_time(timer);
-	overruns = hrtimer_forward(timer, now, data->wde_interval);
+	hrtimer_forward(timer, now, data->wde_interval);
 
 	return HRTIMER_RESTART;
 }
diff --git a/drivers/power/reset/mt6323-poweroff.c b/drivers/power/reset/mt6323-poweroff.c
index 1caf43d9e46d..0532803e6cbc 100644
--- a/drivers/power/reset/mt6323-poweroff.c
+++ b/drivers/power/reset/mt6323-poweroff.c
@@ -30,7 +30,7 @@ static void mt6323_do_pwroff(void)
 	int ret;
 
 	regmap_write(pwrc->regmap, pwrc->base + RTC_BBPU, RTC_BBPU_KEY);
-	regmap_write(pwrc->regmap, pwrc->base + RTC_WRTGR, 1);
+	regmap_write(pwrc->regmap, pwrc->base + RTC_WRTGR_MT6323, 1);
 
 	ret = regmap_read_poll_timeout(pwrc->regmap,
 					pwrc->base + RTC_BBPU, val,
diff --git a/drivers/power/reset/oxnas-restart.c b/drivers/power/reset/oxnas-restart.c
new file mode 100644
index 000000000000..13090bec058a
--- /dev/null
+++ b/drivers/power/reset/oxnas-restart.c
@@ -0,0 +1,233 @@
+// SPDX-License-Identifier: (GPL-2.0)
+/*
+ * oxnas SoC reset driver
+ * based on:
+ * Microsemi MIPS SoC reset driver
+ * and ox820_assert_system_reset() written by Ma Hajun <mahaijuns@gmail.com>
+ *
+ * Copyright (c) 2013 Ma Hajun <mahaijuns@gmail.com>
+ * Copyright (c) 2017 Microsemi Corporation
+ * Copyright (c) 2020 Daniel Golle <daniel@makrotopia.org>
+ */
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/notifier.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/reboot.h>
+#include <linux/regmap.h>
+
+/* bit numbers of reset control register */
+#define OX820_SYS_CTRL_RST_SCU                0
+#define OX820_SYS_CTRL_RST_COPRO              1
+#define OX820_SYS_CTRL_RST_ARM0               2
+#define OX820_SYS_CTRL_RST_ARM1               3
+#define OX820_SYS_CTRL_RST_USBHS              4
+#define OX820_SYS_CTRL_RST_USBHSPHYA          5
+#define OX820_SYS_CTRL_RST_MACA               6
+#define OX820_SYS_CTRL_RST_MAC                OX820_SYS_CTRL_RST_MACA
+#define OX820_SYS_CTRL_RST_PCIEA              7
+#define OX820_SYS_CTRL_RST_SGDMA              8
+#define OX820_SYS_CTRL_RST_CIPHER             9
+#define OX820_SYS_CTRL_RST_DDR                10
+#define OX820_SYS_CTRL_RST_SATA               11
+#define OX820_SYS_CTRL_RST_SATA_LINK          12
+#define OX820_SYS_CTRL_RST_SATA_PHY           13
+#define OX820_SYS_CTRL_RST_PCIEPHY            14
+#define OX820_SYS_CTRL_RST_STATIC             15
+#define OX820_SYS_CTRL_RST_GPIO               16
+#define OX820_SYS_CTRL_RST_UART1              17
+#define OX820_SYS_CTRL_RST_UART2              18
+#define OX820_SYS_CTRL_RST_MISC               19
+#define OX820_SYS_CTRL_RST_I2S                20
+#define OX820_SYS_CTRL_RST_SD                 21
+#define OX820_SYS_CTRL_RST_MACB               22
+#define OX820_SYS_CTRL_RST_PCIEB              23
+#define OX820_SYS_CTRL_RST_VIDEO              24
+#define OX820_SYS_CTRL_RST_DDR_PHY            25
+#define OX820_SYS_CTRL_RST_USBHSPHYB          26
+#define OX820_SYS_CTRL_RST_USBDEV             27
+#define OX820_SYS_CTRL_RST_ARMDBG             29
+#define OX820_SYS_CTRL_RST_PLLA               30
+#define OX820_SYS_CTRL_RST_PLLB               31
+
+/* bit numbers of clock control register */
+#define OX820_SYS_CTRL_CLK_COPRO              0
+#define OX820_SYS_CTRL_CLK_DMA                1
+#define OX820_SYS_CTRL_CLK_CIPHER             2
+#define OX820_SYS_CTRL_CLK_SD                 3
+#define OX820_SYS_CTRL_CLK_SATA               4
+#define OX820_SYS_CTRL_CLK_I2S                5
+#define OX820_SYS_CTRL_CLK_USBHS              6
+#define OX820_SYS_CTRL_CLK_MACA               7
+#define OX820_SYS_CTRL_CLK_MAC                OX820_SYS_CTRL_CLK_MACA
+#define OX820_SYS_CTRL_CLK_PCIEA              8
+#define OX820_SYS_CTRL_CLK_STATIC             9
+#define OX820_SYS_CTRL_CLK_MACB               10
+#define OX820_SYS_CTRL_CLK_PCIEB              11
+#define OX820_SYS_CTRL_CLK_REF600             12
+#define OX820_SYS_CTRL_CLK_USBDEV             13
+#define OX820_SYS_CTRL_CLK_DDR                14
+#define OX820_SYS_CTRL_CLK_DDRPHY             15
+#define OX820_SYS_CTRL_CLK_DDRCK              16
+
+/* Regmap offsets */
+#define OX820_CLK_SET_REGOFFSET               0x2c
+#define OX820_CLK_CLR_REGOFFSET               0x30
+#define OX820_RST_SET_REGOFFSET               0x34
+#define OX820_RST_CLR_REGOFFSET               0x38
+#define OX820_SECONDARY_SEL_REGOFFSET         0x14
+#define OX820_TERTIARY_SEL_REGOFFSET          0x8c
+#define OX820_QUATERNARY_SEL_REGOFFSET        0x94
+#define OX820_DEBUG_SEL_REGOFFSET             0x9c
+#define OX820_ALTERNATIVE_SEL_REGOFFSET       0xa4
+#define OX820_PULLUP_SEL_REGOFFSET            0xac
+#define OX820_SEC_SECONDARY_SEL_REGOFFSET     0x100014
+#define OX820_SEC_TERTIARY_SEL_REGOFFSET      0x10008c
+#define OX820_SEC_QUATERNARY_SEL_REGOFFSET    0x100094
+#define OX820_SEC_DEBUG_SEL_REGOFFSET         0x10009c
+#define OX820_SEC_ALTERNATIVE_SEL_REGOFFSET   0x1000a4
+#define OX820_SEC_PULLUP_SEL_REGOFFSET        0x1000ac
+
+struct oxnas_restart_context {
+	struct regmap *sys_ctrl;
+	struct notifier_block restart_handler;
+};
+
+static int ox820_restart_handle(struct notifier_block *this,
+				 unsigned long mode, void *cmd)
+{
+	struct oxnas_restart_context *ctx = container_of(this, struct
+							oxnas_restart_context,
+							restart_handler);
+	u32 value;
+
+	/*
+	 * Assert reset to cores as per power on defaults
+	 * Don't touch the DDR interface as things will come to an impromptu
+	 * stop NB Possibly should be asserting reset for PLLB, but there are
+	 * timing concerns here according to the docs
+	 */
+	value = BIT(OX820_SYS_CTRL_RST_COPRO)		|
+		BIT(OX820_SYS_CTRL_RST_USBHS)		|
+		BIT(OX820_SYS_CTRL_RST_USBHSPHYA)	|
+		BIT(OX820_SYS_CTRL_RST_MACA)		|
+		BIT(OX820_SYS_CTRL_RST_PCIEA)		|
+		BIT(OX820_SYS_CTRL_RST_SGDMA)		|
+		BIT(OX820_SYS_CTRL_RST_CIPHER)		|
+		BIT(OX820_SYS_CTRL_RST_SATA)		|
+		BIT(OX820_SYS_CTRL_RST_SATA_LINK)	|
+		BIT(OX820_SYS_CTRL_RST_SATA_PHY)	|
+		BIT(OX820_SYS_CTRL_RST_PCIEPHY)		|
+		BIT(OX820_SYS_CTRL_RST_STATIC)		|
+		BIT(OX820_SYS_CTRL_RST_UART1)		|
+		BIT(OX820_SYS_CTRL_RST_UART2)		|
+		BIT(OX820_SYS_CTRL_RST_MISC)		|
+		BIT(OX820_SYS_CTRL_RST_I2S)		|
+		BIT(OX820_SYS_CTRL_RST_SD)		|
+		BIT(OX820_SYS_CTRL_RST_MACB)		|
+		BIT(OX820_SYS_CTRL_RST_PCIEB)		|
+		BIT(OX820_SYS_CTRL_RST_VIDEO)		|
+		BIT(OX820_SYS_CTRL_RST_USBHSPHYB)	|
+		BIT(OX820_SYS_CTRL_RST_USBDEV);
+
+	regmap_write(ctx->sys_ctrl, OX820_RST_SET_REGOFFSET, value);
+
+	/* Release reset to cores as per power on defaults */
+	regmap_write(ctx->sys_ctrl, OX820_RST_CLR_REGOFFSET,
+			BIT(OX820_SYS_CTRL_RST_GPIO));
+
+	/*
+	 * Disable clocks to cores as per power-on defaults - must leave DDR
+	 * related clocks enabled otherwise we'll stop rather abruptly.
+	 */
+	value = BIT(OX820_SYS_CTRL_CLK_COPRO)		|
+		BIT(OX820_SYS_CTRL_CLK_DMA)		|
+		BIT(OX820_SYS_CTRL_CLK_CIPHER)		|
+		BIT(OX820_SYS_CTRL_CLK_SD)		|
+		BIT(OX820_SYS_CTRL_CLK_SATA)		|
+		BIT(OX820_SYS_CTRL_CLK_I2S)		|
+		BIT(OX820_SYS_CTRL_CLK_USBHS)		|
+		BIT(OX820_SYS_CTRL_CLK_MAC)		|
+		BIT(OX820_SYS_CTRL_CLK_PCIEA)		|
+		BIT(OX820_SYS_CTRL_CLK_STATIC)		|
+		BIT(OX820_SYS_CTRL_CLK_MACB)		|
+		BIT(OX820_SYS_CTRL_CLK_PCIEB)		|
+		BIT(OX820_SYS_CTRL_CLK_REF600)		|
+		BIT(OX820_SYS_CTRL_CLK_USBDEV);
+
+	regmap_write(ctx->sys_ctrl, OX820_CLK_CLR_REGOFFSET, value);
+
+	/* Enable clocks to cores as per power-on defaults */
+
+	/* Set sys-control pin mux'ing as per power-on defaults */
+	regmap_write(ctx->sys_ctrl, OX820_SECONDARY_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_TERTIARY_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_QUATERNARY_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_DEBUG_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_ALTERNATIVE_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_PULLUP_SEL_REGOFFSET, 0);
+
+	regmap_write(ctx->sys_ctrl, OX820_SEC_SECONDARY_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_SEC_TERTIARY_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_SEC_QUATERNARY_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_SEC_DEBUG_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_SEC_ALTERNATIVE_SEL_REGOFFSET, 0);
+	regmap_write(ctx->sys_ctrl, OX820_SEC_PULLUP_SEL_REGOFFSET, 0);
+
+	/*
+	 * No need to save any state, as the ROM loader can determine whether
+	 * reset is due to power cycling or programatic action, just hit the
+	 * (self-clearing) CPU reset bit of the block reset register
+	 */
+	value =
+		BIT(OX820_SYS_CTRL_RST_SCU) |
+		BIT(OX820_SYS_CTRL_RST_ARM0) |
+		BIT(OX820_SYS_CTRL_RST_ARM1);
+
+	regmap_write(ctx->sys_ctrl, OX820_RST_SET_REGOFFSET, value);
+
+	pr_emerg("Unable to restart system\n");
+	return NOTIFY_DONE;
+}
+
+static int ox820_restart_probe(struct platform_device *pdev)
+{
+	struct oxnas_restart_context *ctx;
+	struct regmap *sys_ctrl;
+	struct device *dev = &pdev->dev;
+	int err = 0;
+
+	sys_ctrl = syscon_node_to_regmap(pdev->dev.of_node);
+	if (IS_ERR(sys_ctrl))
+		return PTR_ERR(sys_ctrl);
+
+	ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->sys_ctrl = sys_ctrl;
+	ctx->restart_handler.notifier_call = ox820_restart_handle;
+	ctx->restart_handler.priority = 192;
+	err = register_restart_handler(&ctx->restart_handler);
+	if (err)
+		dev_err(dev, "can't register restart notifier (err=%d)\n", err);
+
+	return err;
+}
+
+static const struct of_device_id ox820_restart_of_match[] = {
+	{ .compatible = "oxsemi,ox820-sys-ctrl" },
+	{}
+};
+
+static struct platform_driver ox820_restart_driver = {
+	.probe = ox820_restart_probe,
+	.driver = {
+		.name = "ox820-chip-reset",
+		.of_match_table = ox820_restart_of_match,
+	},
+};
+builtin_platform_driver(ox820_restart_driver);
diff --git a/drivers/power/reset/qcom-pon.c b/drivers/power/reset/qcom-pon.c
index 22a743a0bf28..4a688741a88a 100644
--- a/drivers/power/reset/qcom-pon.c
+++ b/drivers/power/reset/qcom-pon.c
@@ -34,7 +34,8 @@ static int pm8916_reboot_mode_write(struct reboot_mode_driver *reboot,
 
 	ret = regmap_update_bits(pon->regmap,
 				 pon->baseaddr + PON_SOFT_RB_SPARE,
-				 0xfc, magic << pon->reason_shift);
+				 GENMASK(7, pon->reason_shift),
+				 magic << pon->reason_shift);
 	if (ret < 0)
 		dev_err(pon->dev, "update reboot mode bits failed\n");
 
diff --git a/drivers/power/reset/syscon-reboot.c b/drivers/power/reset/syscon-reboot.c
index 62fbba0df971..510e363381ca 100644
--- a/drivers/power/reset/syscon-reboot.c
+++ b/drivers/power/reset/syscon-reboot.c
@@ -51,8 +51,11 @@ static int syscon_reboot_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ctx->map = syscon_regmap_lookup_by_phandle(dev->of_node, "regmap");
-	if (IS_ERR(ctx->map))
-		return PTR_ERR(ctx->map);
+	if (IS_ERR(ctx->map)) {
+		ctx->map = syscon_node_to_regmap(dev->parent->of_node);
+		if (IS_ERR(ctx->map))
+			return PTR_ERR(ctx->map);
+	}
 
 	if (of_property_read_u32(pdev->dev.of_node, "offset", &ctx->offset))
 		return -EINVAL;
diff --git a/drivers/power/reset/vexpress-poweroff.c b/drivers/power/reset/vexpress-poweroff.c
index 90cbaa8341e3..1fdbcbd95fc2 100644
--- a/drivers/power/reset/vexpress-poweroff.c
+++ b/drivers/power/reset/vexpress-poweroff.c
@@ -143,11 +143,7 @@ static struct platform_driver vexpress_reset_driver = {
 	.driver = {
 		.name = "vexpress-reset",
 		.of_match_table = vexpress_reset_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
-
-static int __init vexpress_reset_init(void)
-{
-	return platform_driver_register(&vexpress_reset_driver);
-}
-device_initcall(vexpress_reset_init);
+builtin_platform_driver(vexpress_reset_driver);
diff --git a/drivers/power/supply/88pm860x_battery.c b/drivers/power/supply/88pm860x_battery.c
index 5ca047b3f58f..1308f3a185f3 100644
--- a/drivers/power/supply/88pm860x_battery.c
+++ b/drivers/power/supply/88pm860x_battery.c
@@ -919,16 +919,12 @@ static int pm860x_battery_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	info->irq_cc = platform_get_irq(pdev, 0);
-	if (info->irq_cc <= 0) {
-		dev_err(&pdev->dev, "No IRQ resource!\n");
+	if (info->irq_cc <= 0)
 		return -EINVAL;
-	}
 
 	info->irq_batt = platform_get_irq(pdev, 1);
-	if (info->irq_batt <= 0) {
-		dev_err(&pdev->dev, "No IRQ resource!\n");
+	if (info->irq_batt <= 0)
 		return -EINVAL;
-	}
 
 	info->chip = chip;
 	info->i2c =
diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig
index f3424fdce341..44d3c8512fb8 100644
--- a/drivers/power/supply/Kconfig
+++ b/drivers/power/supply/Kconfig
@@ -116,6 +116,17 @@ config BATTERY_CPCAP
 	  Say Y here to enable support for battery on Motorola
 	  phones and tablets such as droid 4.
 
+config BATTERY_CW2015
+	tristate "CW2015 Battery driver"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say Y here to enable support for the cellwise cw2015
+	  battery fuel gauge (used in the Pinebook Pro & others)
+
+	  This driver can also be built as a module. If so, the module will be
+	  called cw2015_battery.
+
 config BATTERY_DS2760
 	tristate "DS2760 battery driver (HP iPAQ & others)"
 	depends on W1
@@ -415,7 +426,7 @@ config CHARGER_PCF50633
 	tristate "NXP PCF50633 MBC"
 	depends on MFD_PCF50633
 	help
-	 Say Y to include support for NXP PCF50633 Main Battery Charger.
+	  Say Y to include support for NXP PCF50633 Main Battery Charger.
 
 config BATTERY_RX51
 	tristate "Nokia RX-51 (N900) battery driver"
@@ -541,6 +552,16 @@ config CHARGER_MAX8998
 	  Say Y to enable support for the battery charger control sysfs and
 	  platform data of MAX8998/LP3974 PMICs.
 
+config CHARGER_MP2629
+	tristate "Monolithic power system MP2629 Battery charger"
+	depends on MFD_MP2629
+	depends on MP2629_ADC
+	depends on IIO
+	help
+	  Select this option to enable support for Monolithic power system
+	  Battery charger. This driver provides Battery charger power management
+	  functions on the systems.
+
 config CHARGER_QCOM_SMBB
 	tristate "Qualcomm Switch-Mode Battery Charger and Boost"
 	depends on MFD_SPMI_PMIC || COMPILE_TEST
@@ -577,7 +598,7 @@ config CHARGER_BQ24257
 	tristate "TI BQ24250/24251/24257 battery charger driver"
 	depends on I2C
 	depends on GPIOLIB || COMPILE_TEST
-	depends on REGMAP_I2C
+	select REGMAP_I2C
 	help
 	  Say Y to enable support for the TI BQ24250, BQ24251, and BQ24257 battery
 	  chargers.
@@ -609,15 +630,15 @@ config CHARGER_TPS65090
 	tristate "TPS65090 battery charger driver"
 	depends on MFD_TPS65090
 	help
-	 Say Y here to enable support for battery charging with TPS65090
-	 PMIC chips.
+	  Say Y here to enable support for battery charging with TPS65090
+	  PMIC chips.
 
 config CHARGER_TPS65217
 	tristate "TPS65217 battery charger driver"
 	depends on MFD_TPS65217
 	help
-	 Say Y here to enable support for battery charging with TPS65217
-	 PMIC chips.
+	  Say Y here to enable support for battery charging with TPS65217
+	  PMIC chips.
 
 config BATTERY_GAUGE_LTC2941
 	tristate "LTC2941/LTC2943 Battery Gauge Driver"
@@ -660,7 +681,6 @@ config CHARGER_RT9455
 config CHARGER_CROS_USBPD
 	tristate "ChromeOS EC based USBPD charger"
 	depends on CROS_USBPD_NOTIFY
-	default n
 	help
 	  Say Y here to enable ChromeOS EC based USBPD charger
 	  driver. This driver gets various bits of information about
@@ -671,16 +691,16 @@ config CHARGER_SC2731
 	tristate "Spreadtrum SC2731 charger driver"
 	depends on MFD_SC27XX_PMIC || COMPILE_TEST
 	help
-	 Say Y here to enable support for battery charging with SC2731
-	 PMIC chips.
+	  Say Y here to enable support for battery charging with SC2731
+	  PMIC chips.
 
 config FUEL_GAUGE_SC27XX
 	tristate "Spreadtrum SC27XX fuel gauge driver"
 	depends on MFD_SC27XX_PMIC || COMPILE_TEST
 	depends on IIO
 	help
-	 Say Y here to enable support for fuel gauge with SC27XX
-	 PMIC chips.
+	  Say Y here to enable support for fuel gauge with SC27XX
+	  PMIC chips.
 
 config CHARGER_UCS1002
 	tristate "Microchip UCS1002 USB Port Power Controller"
@@ -695,11 +715,20 @@ config CHARGER_UCS1002
 config CHARGER_BD70528
 	tristate "ROHM bd70528 charger driver"
 	depends on MFD_ROHM_BD70528
-	default n
+	select LINEAR_RANGES
+	help
+	  Say Y here to enable support for getting battery status
+	  information and altering charger configurations from charger
+	  block of the ROHM BD70528 Power Management IC.
+
+config CHARGER_BD99954
+	tristate "ROHM bd99954 charger driver"
+	depends on I2C
+	select LINEAR_RANGES
 	help
-	 Say Y here to enable support for getting battery status
-	 information and altering charger configurations from charger
-	 block of the ROHM BD70528 Power Management IC.
+	  Say Y here to enable support for getting battery and charger
+	  information and altering charger configurations from the ROHM
+	  BD99954 charger IC.
 
 config CHARGER_WILCO
 	tristate "Wilco EC based charger for ChromeOS"
diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile
index 6c7da920ea83..b9644663e435 100644
--- a/drivers/power/supply/Makefile
+++ b/drivers/power/supply/Makefile
@@ -24,6 +24,7 @@ obj-$(CONFIG_BATTERY_ACT8945A)	+= act8945a_charger.o
 obj-$(CONFIG_BATTERY_AXP20X)	+= axp20x_battery.o
 obj-$(CONFIG_CHARGER_AXP20X)	+= axp20x_ac_power.o
 obj-$(CONFIG_BATTERY_CPCAP)	+= cpcap-battery.o
+obj-$(CONFIG_BATTERY_CW2015)	+= cw2015_battery.o
 obj-$(CONFIG_BATTERY_DS2760)	+= ds2760_battery.o
 obj-$(CONFIG_BATTERY_DS2780)	+= ds2780_battery.o
 obj-$(CONFIG_BATTERY_DS2781)	+= ds2781_battery.o
@@ -75,6 +76,7 @@ obj-$(CONFIG_CHARGER_MAX77650)	+= max77650-charger.o
 obj-$(CONFIG_CHARGER_MAX77693)	+= max77693_charger.o
 obj-$(CONFIG_CHARGER_MAX8997)	+= max8997_charger.o
 obj-$(CONFIG_CHARGER_MAX8998)	+= max8998_charger.o
+obj-$(CONFIG_CHARGER_MP2629)	+= mp2629_charger.o
 obj-$(CONFIG_CHARGER_QCOM_SMBB)	+= qcom_smbb.o
 obj-$(CONFIG_CHARGER_BQ2415X)	+= bq2415x_charger.o
 obj-$(CONFIG_CHARGER_BQ24190)	+= bq24190_charger.o
@@ -91,4 +93,5 @@ obj-$(CONFIG_CHARGER_SC2731)	+= sc2731_charger.o
 obj-$(CONFIG_FUEL_GAUGE_SC27XX)	+= sc27xx_fuel_gauge.o
 obj-$(CONFIG_CHARGER_UCS1002)	+= ucs1002_power.o
 obj-$(CONFIG_CHARGER_BD70528)	+= bd70528-charger.o
+obj-$(CONFIG_CHARGER_BD99954)	+= bd99954-charger.o
 obj-$(CONFIG_CHARGER_WILCO)	+= wilco-charger.o
diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c
index b96f90a82ecf..751c4f6c7487 100644
--- a/drivers/power/supply/ab8500_fg.c
+++ b/drivers/power/supply/ab8500_fg.c
@@ -2399,7 +2399,7 @@ static void ab8500_fg_reinit_work(struct work_struct *work)
 	struct ab8500_fg *di = container_of(work, struct ab8500_fg,
 		fg_reinit_work.work);
 
-	if (di->flags.calibrate == false) {
+	if (!di->flags.calibrate) {
 		dev_dbg(di->dev, "Resetting FG state machine to init.\n");
 		ab8500_fg_clear_cap_samples(di);
 		ab8500_fg_calc_cap_discharge_voltage(di, true);
diff --git a/drivers/power/supply/axp288_charger.c b/drivers/power/supply/axp288_charger.c
index cf4c67b2d235..9d981b76c1e7 100644
--- a/drivers/power/supply/axp288_charger.c
+++ b/drivers/power/supply/axp288_charger.c
@@ -880,10 +880,9 @@ static int axp288_charger_probe(struct platform_device *pdev)
 	/* Register charger interrupts */
 	for (i = 0; i < CHRG_INTR_END; i++) {
 		pirq = platform_get_irq(info->pdev, i);
-		if (pirq < 0) {
-			dev_err(&pdev->dev, "Failed to get IRQ: %d\n", pirq);
+		if (pirq < 0)
 			return pirq;
-		}
+
 		info->irq[i] = regmap_irq_get_virq(info->regmap_irqc, pirq);
 		if (info->irq[i] < 0) {
 			dev_warn(&info->pdev->dev,
diff --git a/drivers/power/supply/axp288_fuel_gauge.c b/drivers/power/supply/axp288_fuel_gauge.c
index f40fa0e63b6e..148eb8105803 100644
--- a/drivers/power/supply/axp288_fuel_gauge.c
+++ b/drivers/power/supply/axp288_fuel_gauge.c
@@ -718,6 +718,12 @@ static const struct dmi_system_id axp288_fuel_gauge_blacklist[] = {
 		},
 	},
 	{
+		/* Meegopad T02 */
+		.matches = {
+			DMI_MATCH(DMI_PRODUCT_NAME, "MEEGOPAD T02"),
+		},
+	},
+	{
 		/* Meegopad T08 */
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Default string"),
diff --git a/drivers/power/supply/bd70528-charger.c b/drivers/power/supply/bd70528-charger.c
index 3b820110ecfa..7c1f0b99c71b 100644
--- a/drivers/power/supply/bd70528-charger.c
+++ b/drivers/power/supply/bd70528-charger.c
@@ -72,6 +72,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/power_supply.h>
+#include <linux/linear_range.h>
 
 #define CHG_STAT_SUSPEND	0x0
 #define CHG_STAT_TRICKLE	0x1
@@ -335,38 +336,37 @@ static int bd70528_get_present(struct bd70528_psy *bdpsy, int *val)
 	return 0;
 }
 
-struct bd70528_linear_range {
-	int min;
-	int step;
-	int vals;
-	int low_sel;
-};
-
-static const struct bd70528_linear_range current_limit_ranges[] = {
+static const struct linear_range current_limit_ranges[] = {
 	{
 		.min = 5,
 		.step = 1,
-		.vals = 36,
-		.low_sel = 0,
+		.min_sel = 0,
+		.max_sel = 0x22,
 	},
 	{
 		.min = 40,
 		.step = 5,
-		.vals = 5,
-		.low_sel = 0x23,
+		.min_sel = 0x23,
+		.max_sel = 0x26,
 	},
 	{
 		.min = 60,
 		.step = 20,
-		.vals = 8,
-		.low_sel = 0x27,
+		.min_sel = 0x27,
+		.max_sel = 0x2d,
 	},
 	{
 		.min = 200,
 		.step = 50,
-		.vals = 7,
-		.low_sel = 0x2e,
-	}
+		.min_sel = 0x2e,
+		.max_sel = 0x34,
+	},
+	{
+		.min = 500,
+		.step = 0,
+		.min_sel = 0x35,
+		.max_sel = 0x3f,
+	},
 };
 
 /*
@@ -374,18 +374,18 @@ static const struct bd70528_linear_range current_limit_ranges[] = {
  * voltage for low temperatures. The driver currently only reads
  * the charge current at room temperature. We do set both though.
  */
-static const struct bd70528_linear_range warm_charge_curr[] = {
+static const struct linear_range warm_charge_curr[] = {
 	{
 		.min = 10,
 		.step = 10,
-		.vals = 20,
-		.low_sel = 0,
+		.min_sel = 0,
+		.max_sel = 0x12,
 	},
 	{
 		.min = 200,
 		.step = 25,
-		.vals = 13,
-		.low_sel = 0x13,
+		.min_sel = 0x13,
+		.max_sel = 0x1f,
 	},
 };
 
@@ -398,56 +398,6 @@ static const struct bd70528_linear_range warm_charge_curr[] = {
 #define MAX_WARM_CHG_CURR_SEL 0x1f
 #define MIN_CHG_CURR_SEL 0x0
 
-static int find_value_for_selector_low(const struct bd70528_linear_range *r,
-				       int selectors, unsigned int sel,
-				       unsigned int *val)
-{
-	int i;
-
-	for (i = 0; i < selectors; i++) {
-		if (r[i].low_sel <= sel && r[i].low_sel + r[i].vals >= sel) {
-			*val = r[i].min + (sel - r[i].low_sel) * r[i].step;
-			return 0;
-		}
-	}
-	return -EINVAL;
-}
-
-/*
- * For BD70528 voltage/current limits we happily accept any value which
- * belongs the range. We could check if value matching the selector is
- * desired by computing the range min + (sel - sel_low) * range step - but
- * I guess it is enough if we use voltage/current which is closest (below)
- * the requested?
- */
-static int find_selector_for_value_low(const struct bd70528_linear_range *r,
-				       int selectors, unsigned int val,
-				       unsigned int *sel, bool *found)
-{
-	int i;
-	int ret = -EINVAL;
-
-	*found = false;
-	for (i = 0; i < selectors; i++) {
-		if (r[i].min <= val) {
-			if (r[i].min + r[i].step * r[i].vals >= val) {
-				*found = true;
-				*sel = r[i].low_sel + (val - r[i].min) /
-				       r[i].step;
-				ret = 0;
-				break;
-			}
-			/*
-			 * If the range max is smaller than requested
-			 * we can set the max supported value from range
-			 */
-			*sel = r[i].low_sel + r[i].vals;
-			ret = 0;
-		}
-	}
-	return ret;
-}
-
 static int get_charge_current(struct bd70528_psy *bdpsy, int *ma)
 {
 	unsigned int sel;
@@ -463,9 +413,9 @@ static int get_charge_current(struct bd70528_psy *bdpsy, int *ma)
 
 	sel &= BD70528_MASK_CHG_CHG_CURR;
 
-	ret = find_value_for_selector_low(&warm_charge_curr[0],
-					  ARRAY_SIZE(warm_charge_curr), sel,
-					  ma);
+	ret = linear_range_get_value_array(&warm_charge_curr[0],
+					   ARRAY_SIZE(warm_charge_curr),
+					   sel, ma);
 	if (ret) {
 		dev_err(bdpsy->dev,
 			"Unknown charge current value 0x%x\n",
@@ -491,10 +441,9 @@ static int get_current_limit(struct bd70528_psy *bdpsy, int *ma)
 
 	sel &= BD70528_MASK_CHG_DCIN_ILIM;
 
-	ret = find_value_for_selector_low(&current_limit_ranges[0],
-					  ARRAY_SIZE(current_limit_ranges), sel,
-					  ma);
-
+	ret = linear_range_get_value_array(&current_limit_ranges[0],
+					   ARRAY_SIZE(current_limit_ranges),
+					   sel, ma);
 	if (ret) {
 		/* Unspecified values mean 500 mA */
 		*ma = 500;
@@ -588,15 +537,28 @@ static int set_charge_current(struct bd70528_psy *bdpsy, int ma)
 		goto set;
 	}
 
-	ret = find_selector_for_value_low(&warm_charge_curr[0],
-					  ARRAY_SIZE(warm_charge_curr), ma,
-					  &reg, &found);
+/*
+ * For BD70528 voltage/current limits we happily accept any value which
+ * belongs the range. We could check if value matching the selector is
+ * desired by computing the range min + (sel - sel_low) * range step - but
+ * I guess it is enough if we use voltage/current which is closest (below)
+ * the requested?
+ */
+
+	ret = linear_range_get_selector_low_array(warm_charge_curr,
+						  ARRAY_SIZE(warm_charge_curr),
+						  ma, &reg, &found);
 	if (ret) {
+		dev_err(bdpsy->dev,
+			 "Unsupported charge current %u mA\n", ma);
 		reg = MIN_CHG_CURR_SEL;
 		goto set;
 	}
 	if (!found) {
-		/* There was a gap in supported values and we hit it */
+		/*
+		 * There was a gap in supported values and we hit it.
+		 * Yet a smaller value was found so we use it.
+		 */
 		dev_warn(bdpsy->dev,
 			 "Unsupported charge current %u mA\n", ma);
 	}
@@ -648,17 +610,21 @@ static int set_current_limit(struct bd70528_psy *bdpsy, int ma)
 		goto set;
 	}
 
-	ret = find_selector_for_value_low(&current_limit_ranges[0],
-					  ARRAY_SIZE(current_limit_ranges), ma,
-					  &reg, &found);
+	ret = linear_range_get_selector_low_array(current_limit_ranges,
+					ARRAY_SIZE(current_limit_ranges),
+					ma, &reg, &found);
 	if (ret) {
+		dev_err(bdpsy->dev, "Unsupported current limit %umA\n", ma);
 		reg = MIN_CURR_LIMIT_SEL;
 		goto set;
 	}
 	if (!found) {
-		/* There was a gap in supported values and we hit it ?*/
-		dev_warn(bdpsy->dev, "Unsupported current limit %umA\n",
-			 ma);
+		/*
+		 * There was a gap in supported values and we hit it.
+		 * We found a smaller value from ranges and use it.
+		 * Warn user though.
+		 */
+		dev_warn(bdpsy->dev, "Unsupported current limit %umA\n", ma);
 	}
 
 set:
diff --git a/drivers/power/supply/bd99954-charger.c b/drivers/power/supply/bd99954-charger.c
new file mode 100644
index 000000000000..ffd8bfa08179
--- /dev/null
+++ b/drivers/power/supply/bd99954-charger.c
@@ -0,0 +1,1142 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ROHM BD99954 charger driver
+ *
+ * Copyright (C) 2020 Rohm Semiconductors
+ *	Originally written by:
+ *		Mikko Mutanen <mikko.mutanen@fi.rohmeurope.com>
+ *		Markus Laine <markus.laine@fi.rohmeurope.com>
+ *	Bugs added by:
+ *		Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>
+ */
+
+/*
+ *   The battery charging profile of BD99954.
+ *
+ *   Curve (1) represents charging current.
+ *   Curve (2) represents battery voltage.
+ *
+ *   The BD99954 data sheet divides charging to three phases.
+ *   a) Trickle-charge with constant current (8).
+ *   b) pre-charge with constant current (6)
+ *   c) fast-charge, first with constant current (5) phase. After
+ *      the battery voltage has reached target level (4) we have constant
+ *      voltage phase until charging current has dropped to termination
+ *      level (7)
+ *
+ *    V ^                                                        ^ I
+ *      .                                                        .
+ *      .                                                        .
+ *(4)` `.` ` ` ` ` ` ` ` ` ` ` ` ` ` ----------------------------.
+ *      .                           :/                           .
+ *      .                     o----+/:/ ` ` ` ` ` ` ` ` ` ` ` ` `.` ` (5)
+ *      .                     +   ::  +                          .
+ *      .                     +  /-   --                         .
+ *      .                     +`/-     +                         .
+ *      .                     o/-      -:                        .
+ *      .                    .s.        +`                       .
+ *      .                  .--+         `/                       .
+ *      .               ..``  +          .:                      .
+ *      .             -`      +           --                     .
+ *      .    (2)  ...``       +            :-                    .
+ *      .    ...``            +             -:                   .
+ *(3)` `.`.""  ` ` ` `+-------- ` ` ` ` ` ` `.:` ` ` ` ` ` ` ` ` .` ` (6)
+ *      .             +                       `:.                .
+ *      .             +                         -:               .
+ *      .             +                           -:.            .
+ *      .             +                             .--.         .
+ *      .   (1)       +                                `.+` ` ` `.` ` (7)
+ *      -..............` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` + ` ` ` .` ` (8)
+ *      .                                                +       -
+ *      -------------------------------------------------+++++++++-->
+ *      |   trickle   |  pre  |          fast            |
+ *
+ * Details of DT properties for different limits can be found from BD99954
+ * device tree binding documentation.
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/linear_range.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/power_supply.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#include "bd99954-charger.h"
+
+struct battery_data {
+	u16 precharge_current;	/* Trickle-charge Current */
+	u16 fc_reg_voltage;	/* Fast Charging Regulation Voltage */
+	u16 voltage_min;
+	u16 voltage_max;
+};
+
+/* Initial field values, converted to initial register values */
+struct bd9995x_init_data {
+	u16 vsysreg_set;	/* VSYS Regulation Setting */
+	u16 ibus_lim_set;	/* VBUS input current limitation */
+	u16 icc_lim_set;	/* VCC/VACP Input Current Limit Setting */
+	u16 itrich_set;		/* Trickle-charge Current Setting */
+	u16 iprech_set;		/* Pre-Charge Current Setting */
+	u16 ichg_set;		/* Fast-Charge constant current */
+	u16 vfastchg_reg_set1;	/* Fast Charging Regulation Voltage */
+	u16 vprechg_th_set;	/* Pre-charge Voltage Threshold Setting */
+	u16 vrechg_set;		/* Re-charge Battery Voltage Setting */
+	u16 vbatovp_set;	/* Battery Over Voltage Threshold Setting */
+	u16 iterm_set;		/* Charging termination current */
+};
+
+struct bd9995x_state {
+	u8 online;
+	u16 chgstm_status;
+	u16 vbat_vsys_status;
+	u16 vbus_vcc_status;
+};
+
+struct bd9995x_device {
+	struct i2c_client *client;
+	struct device *dev;
+	struct power_supply *charger;
+
+	struct regmap *rmap;
+	struct regmap_field *rmap_fields[F_MAX_FIELDS];
+
+	int chip_id;
+	int chip_rev;
+	struct bd9995x_init_data init_data;
+	struct bd9995x_state state;
+
+	struct mutex lock; /* Protect state data */
+};
+
+static const struct regmap_range bd9995x_readonly_reg_ranges[] = {
+	regmap_reg_range(CHGSTM_STATUS, SEL_ILIM_VAL),
+	regmap_reg_range(IOUT_DACIN_VAL, IOUT_DACIN_VAL),
+	regmap_reg_range(VCC_UCD_STATUS, VCC_IDD_STATUS),
+	regmap_reg_range(VBUS_UCD_STATUS, VBUS_IDD_STATUS),
+	regmap_reg_range(CHIP_ID, CHIP_REV),
+	regmap_reg_range(SYSTEM_STATUS, SYSTEM_STATUS),
+	regmap_reg_range(IBATP_VAL, VBAT_AVE_VAL),
+	regmap_reg_range(VTH_VAL, EXTIADP_AVE_VAL),
+};
+
+static const struct regmap_access_table bd9995x_writeable_regs = {
+	.no_ranges = bd9995x_readonly_reg_ranges,
+	.n_no_ranges = ARRAY_SIZE(bd9995x_readonly_reg_ranges),
+};
+
+static const struct regmap_range bd9995x_volatile_reg_ranges[] = {
+	regmap_reg_range(CHGSTM_STATUS, WDT_STATUS),
+	regmap_reg_range(VCC_UCD_STATUS, VCC_IDD_STATUS),
+	regmap_reg_range(VBUS_UCD_STATUS, VBUS_IDD_STATUS),
+	regmap_reg_range(INT0_STATUS, INT7_STATUS),
+	regmap_reg_range(SYSTEM_STATUS, SYSTEM_CTRL_SET),
+	regmap_reg_range(IBATP_VAL, EXTIADP_AVE_VAL), /* Measurement regs */
+};
+
+static const struct regmap_access_table bd9995x_volatile_regs = {
+	.yes_ranges = bd9995x_volatile_reg_ranges,
+	.n_yes_ranges = ARRAY_SIZE(bd9995x_volatile_reg_ranges),
+};
+
+static const struct regmap_range_cfg regmap_range_cfg[] = {
+	{
+	.selector_reg     = MAP_SET,
+	.selector_mask    = 0xFFFF,
+	.selector_shift   = 0,
+	.window_start     = 0,
+	.window_len       = 0x100,
+	.range_min        = 0 * 0x100,
+	.range_max        = 3 * 0x100,
+	},
+};
+
+static const struct regmap_config bd9995x_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.reg_stride = 1,
+
+	.max_register = 3 * 0x100,
+	.cache_type = REGCACHE_RBTREE,
+
+	.ranges = regmap_range_cfg,
+	.num_ranges = ARRAY_SIZE(regmap_range_cfg),
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+	.wr_table = &bd9995x_writeable_regs,
+	.volatile_table = &bd9995x_volatile_regs,
+};
+
+enum bd9995x_chrg_fault {
+	CHRG_FAULT_NORMAL,
+	CHRG_FAULT_INPUT,
+	CHRG_FAULT_THERMAL_SHUTDOWN,
+	CHRG_FAULT_TIMER_EXPIRED,
+};
+
+static int bd9995x_get_prop_batt_health(struct bd9995x_device *bd)
+{
+	int ret, tmp;
+
+	ret = regmap_field_read(bd->rmap_fields[F_BATTEMP], &tmp);
+	if (ret)
+		return POWER_SUPPLY_HEALTH_UNKNOWN;
+
+	/* TODO: Check these against datasheet page 34 */
+
+	switch (tmp) {
+	case ROOM:
+		return POWER_SUPPLY_HEALTH_GOOD;
+	case HOT1:
+	case HOT2:
+	case HOT3:
+		return POWER_SUPPLY_HEALTH_OVERHEAT;
+	case COLD1:
+	case COLD2:
+		return POWER_SUPPLY_HEALTH_COLD;
+	case TEMP_DIS:
+	case BATT_OPEN:
+	default:
+		return POWER_SUPPLY_HEALTH_UNKNOWN;
+	}
+}
+
+static int bd9995x_get_prop_charge_type(struct bd9995x_device *bd)
+{
+	int ret, tmp;
+
+	ret = regmap_field_read(bd->rmap_fields[F_CHGSTM_STATE], &tmp);
+	if (ret)
+		return POWER_SUPPLY_CHARGE_TYPE_UNKNOWN;
+
+	switch (tmp) {
+	case CHGSTM_TRICKLE_CHARGE:
+	case CHGSTM_PRE_CHARGE:
+		return POWER_SUPPLY_CHARGE_TYPE_TRICKLE;
+	case CHGSTM_FAST_CHARGE:
+		return POWER_SUPPLY_CHARGE_TYPE_FAST;
+	case CHGSTM_TOP_OFF:
+	case CHGSTM_DONE:
+	case CHGSTM_SUSPEND:
+		return POWER_SUPPLY_CHARGE_TYPE_NONE;
+	default: /* Rest of the states are error related, no charging */
+		return POWER_SUPPLY_CHARGE_TYPE_NONE;
+	}
+}
+
+static bool bd9995x_get_prop_batt_present(struct bd9995x_device *bd)
+{
+	int ret, tmp;
+
+	ret = regmap_field_read(bd->rmap_fields[F_BATTEMP], &tmp);
+	if (ret)
+		return false;
+
+	return tmp != BATT_OPEN;
+}
+
+static int bd9995x_get_prop_batt_voltage(struct bd9995x_device *bd)
+{
+	int ret, tmp;
+
+	ret = regmap_field_read(bd->rmap_fields[F_VBAT_VAL], &tmp);
+	if (ret)
+		return 0;
+
+	tmp = min(tmp, 19200);
+
+	return tmp * 1000;
+}
+
+static int bd9995x_get_prop_batt_current(struct bd9995x_device *bd)
+{
+	int ret, tmp;
+
+	ret = regmap_field_read(bd->rmap_fields[F_IBATP_VAL], &tmp);
+	if (ret)
+		return 0;
+
+	return tmp * 1000;
+}
+
+#define DEFAULT_BATTERY_TEMPERATURE 250
+
+static int bd9995x_get_prop_batt_temp(struct bd9995x_device *bd)
+{
+	int ret, tmp;
+
+	ret = regmap_field_read(bd->rmap_fields[F_THERM_VAL], &tmp);
+	if (ret)
+		return DEFAULT_BATTERY_TEMPERATURE;
+
+	return (200 - tmp) * 10;
+}
+
+static int bd9995x_power_supply_get_property(struct power_supply *psy,
+					     enum power_supply_property psp,
+					     union power_supply_propval *val)
+{
+	int ret, tmp;
+	struct bd9995x_device *bd = power_supply_get_drvdata(psy);
+	struct bd9995x_state state;
+
+	mutex_lock(&bd->lock);
+	state = bd->state;
+	mutex_unlock(&bd->lock);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_STATUS:
+		switch (state.chgstm_status) {
+		case CHGSTM_TRICKLE_CHARGE:
+		case CHGSTM_PRE_CHARGE:
+		case CHGSTM_FAST_CHARGE:
+		case CHGSTM_TOP_OFF:
+			val->intval = POWER_SUPPLY_STATUS_CHARGING;
+			break;
+
+		case CHGSTM_DONE:
+			val->intval = POWER_SUPPLY_STATUS_FULL;
+			break;
+
+		case CHGSTM_SUSPEND:
+		case CHGSTM_TEMPERATURE_ERROR_1:
+		case CHGSTM_TEMPERATURE_ERROR_2:
+		case CHGSTM_TEMPERATURE_ERROR_3:
+		case CHGSTM_TEMPERATURE_ERROR_4:
+		case CHGSTM_TEMPERATURE_ERROR_5:
+		case CHGSTM_TEMPERATURE_ERROR_6:
+		case CHGSTM_TEMPERATURE_ERROR_7:
+		case CHGSTM_THERMAL_SHUT_DOWN_1:
+		case CHGSTM_THERMAL_SHUT_DOWN_2:
+		case CHGSTM_THERMAL_SHUT_DOWN_3:
+		case CHGSTM_THERMAL_SHUT_DOWN_4:
+		case CHGSTM_THERMAL_SHUT_DOWN_5:
+		case CHGSTM_THERMAL_SHUT_DOWN_6:
+		case CHGSTM_THERMAL_SHUT_DOWN_7:
+		case CHGSTM_BATTERY_ERROR:
+			val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING;
+			break;
+
+		default:
+			val->intval = POWER_SUPPLY_STATUS_UNKNOWN;
+			break;
+		}
+		break;
+
+	case POWER_SUPPLY_PROP_MANUFACTURER:
+		val->strval = BD9995X_MANUFACTURER;
+		break;
+
+	case POWER_SUPPLY_PROP_ONLINE:
+		val->intval = state.online;
+		break;
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
+		ret = regmap_field_read(bd->rmap_fields[F_IBATP_VAL], &tmp);
+		if (ret)
+			return ret;
+		val->intval = tmp * 1000;
+		break;
+
+	case POWER_SUPPLY_PROP_CHARGE_AVG:
+		ret = regmap_field_read(bd->rmap_fields[F_IBATP_AVE_VAL], &tmp);
+		if (ret)
+			return ret;
+		val->intval = tmp * 1000;
+		break;
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX:
+		/*
+		 * Currently the DT uses this property to give the
+		 * target current for fast-charging constant current phase.
+		 * I think it is correct in a sense.
+		 *
+		 * Yet, this prop we read and return here is the programmed
+		 * safety limit for combined input currents. This feels
+		 * also correct in a sense.
+		 *
+		 * However, this results a mismatch to DT value and value
+		 * read from sysfs.
+		 */
+		ret = regmap_field_read(bd->rmap_fields[F_SEL_ILIM_VAL], &tmp);
+		if (ret)
+			return ret;
+		val->intval = tmp * 1000;
+		break;
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
+		if (!state.online) {
+			val->intval = 0;
+			break;
+		}
+
+		ret = regmap_field_read(bd->rmap_fields[F_VFASTCHG_REG_SET1],
+					&tmp);
+		if (ret)
+			return ret;
+
+		/*
+		 * The actual range : 2560 to 19200 mV. No matter what the
+		 * register says
+		 */
+		val->intval = clamp_val(tmp << 4, 2560, 19200);
+		val->intval *= 1000;
+		break;
+
+	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+		ret = regmap_field_read(bd->rmap_fields[F_ITERM_SET], &tmp);
+		if (ret)
+			return ret;
+		/* Start step is 64 mA */
+		val->intval = tmp << 6;
+		/* Maximum is 1024 mA - no matter what register says */
+		val->intval = min(val->intval, 1024);
+		val->intval *= 1000;
+		break;
+
+	/* Battery properties which we access through charger */
+	case POWER_SUPPLY_PROP_PRESENT:
+		val->intval = bd9995x_get_prop_batt_present(bd);
+		break;
+
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		val->intval = bd9995x_get_prop_batt_voltage(bd);
+		break;
+
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		val->intval = bd9995x_get_prop_batt_current(bd);
+		break;
+
+	case POWER_SUPPLY_PROP_CHARGE_TYPE:
+		val->intval = bd9995x_get_prop_charge_type(bd);
+		break;
+
+	case POWER_SUPPLY_PROP_HEALTH:
+		val->intval = bd9995x_get_prop_batt_health(bd);
+		break;
+
+	case POWER_SUPPLY_PROP_TEMP:
+		val->intval = bd9995x_get_prop_batt_temp(bd);
+		break;
+
+	case POWER_SUPPLY_PROP_TECHNOLOGY:
+		val->intval = POWER_SUPPLY_TECHNOLOGY_LION;
+		break;
+
+	case POWER_SUPPLY_PROP_MODEL_NAME:
+		val->strval = "bd99954";
+		break;
+
+	default:
+		return -EINVAL;
+
+	}
+
+	return 0;
+}
+
+static int bd9995x_get_chip_state(struct bd9995x_device *bd,
+				  struct bd9995x_state *state)
+{
+	int i, ret, tmp;
+	struct {
+		struct regmap_field *id;
+		u16 *data;
+	} state_fields[] = {
+		{
+			bd->rmap_fields[F_CHGSTM_STATE], &state->chgstm_status,
+		}, {
+			bd->rmap_fields[F_VBAT_VSYS_STATUS],
+			&state->vbat_vsys_status,
+		}, {
+			bd->rmap_fields[F_VBUS_VCC_STATUS],
+			&state->vbus_vcc_status,
+		},
+	};
+
+
+	for (i = 0; i < ARRAY_SIZE(state_fields); i++) {
+		ret = regmap_field_read(state_fields[i].id, &tmp);
+		if (ret)
+			return ret;
+
+		*state_fields[i].data = tmp;
+	}
+
+	if (state->vbus_vcc_status & STATUS_VCC_DET ||
+	    state->vbus_vcc_status & STATUS_VBUS_DET)
+		state->online = 1;
+	else
+		state->online = 0;
+
+	return 0;
+}
+
+static irqreturn_t bd9995x_irq_handler_thread(int irq, void *private)
+{
+	struct bd9995x_device *bd = private;
+	int ret, status, mask, i;
+	unsigned long tmp;
+	struct bd9995x_state state;
+
+	/*
+	 * The bd9995x does not seem to generate big amount of interrupts.
+	 * The logic regarding which interrupts can cause relevant
+	 * status changes seem to be pretty complex.
+	 *
+	 * So lets implement really simple and hopefully bullet-proof handler:
+	 * It does not really matter which IRQ we handle, we just go and
+	 * re-read all interesting statuses + give the framework a nudge.
+	 *
+	 * Other option would be building a _complex_ and error prone logic
+	 * trying to decide what could have been changed (resulting this IRQ
+	 * we are now handling). During the normal operation the BD99954 does
+	 * not seem to be generating much of interrupts so benefit from such
+	 * logic would probably be minimal.
+	 */
+
+	ret = regmap_read(bd->rmap, INT0_STATUS, &status);
+	if (ret) {
+		dev_err(bd->dev, "Failed to read IRQ status\n");
+		return IRQ_NONE;
+	}
+
+	ret = regmap_field_read(bd->rmap_fields[F_INT0_SET], &mask);
+	if (ret) {
+		dev_err(bd->dev, "Failed to read IRQ mask\n");
+		return IRQ_NONE;
+	}
+
+	/* Handle only IRQs that are not masked */
+	status &= mask;
+	tmp = status;
+
+	/* Lowest bit does not represent any sub-registers */
+	tmp >>= 1;
+
+	/*
+	 * Mask and ack IRQs we will handle (+ the idiot bit)
+	 */
+	ret = regmap_field_write(bd->rmap_fields[F_INT0_SET], 0);
+	if (ret) {
+		dev_err(bd->dev, "Failed to mask F_INT0\n");
+		return IRQ_NONE;
+	}
+
+	ret = regmap_write(bd->rmap, INT0_STATUS, status);
+	if (ret) {
+		dev_err(bd->dev, "Failed to ack F_INT0\n");
+		goto err_umask;
+	}
+
+	for_each_set_bit(i, &tmp, 7) {
+		int sub_status, sub_mask;
+		int sub_status_reg[] = {
+			INT1_STATUS, INT2_STATUS, INT3_STATUS, INT4_STATUS,
+			INT5_STATUS, INT6_STATUS, INT7_STATUS,
+		};
+		struct regmap_field *sub_mask_f[] = {
+			bd->rmap_fields[F_INT1_SET],
+			bd->rmap_fields[F_INT2_SET],
+			bd->rmap_fields[F_INT3_SET],
+			bd->rmap_fields[F_INT4_SET],
+			bd->rmap_fields[F_INT5_SET],
+			bd->rmap_fields[F_INT6_SET],
+			bd->rmap_fields[F_INT7_SET],
+		};
+
+		/* Clear sub IRQs */
+		ret = regmap_read(bd->rmap, sub_status_reg[i], &sub_status);
+		if (ret) {
+			dev_err(bd->dev, "Failed to read IRQ sub-status\n");
+			goto err_umask;
+		}
+
+		ret = regmap_field_read(sub_mask_f[i], &sub_mask);
+		if (ret) {
+			dev_err(bd->dev, "Failed to read IRQ sub-mask\n");
+			goto err_umask;
+		}
+
+		/* Ack active sub-statuses */
+		sub_status &= sub_mask;
+
+		ret = regmap_write(bd->rmap, sub_status_reg[i], sub_status);
+		if (ret) {
+			dev_err(bd->dev, "Failed to ack sub-IRQ\n");
+			goto err_umask;
+		}
+	}
+
+	ret = regmap_field_write(bd->rmap_fields[F_INT0_SET], mask);
+	if (ret)
+		/* May as well retry once */
+		goto err_umask;
+
+	/* Read whole chip state */
+	ret = bd9995x_get_chip_state(bd, &state);
+	if (ret < 0) {
+		dev_err(bd->dev, "Failed to read chip state\n");
+	} else {
+		mutex_lock(&bd->lock);
+		bd->state = state;
+		mutex_unlock(&bd->lock);
+
+		power_supply_changed(bd->charger);
+	}
+
+	return IRQ_HANDLED;
+
+err_umask:
+	ret = regmap_field_write(bd->rmap_fields[F_INT0_SET], mask);
+	if (ret)
+		dev_err(bd->dev,
+		"Failed to un-mask F_INT0 - IRQ permanently disabled\n");
+
+	return IRQ_NONE;
+}
+
+static int __bd9995x_chip_reset(struct bd9995x_device *bd)
+{
+	int ret, state;
+	int rst_check_counter = 10;
+	u16 tmp = ALLRST | OTPLD;
+
+	ret = regmap_raw_write(bd->rmap, SYSTEM_CTRL_SET, &tmp, 2);
+	if (ret < 0)
+		return ret;
+
+	do {
+		ret = regmap_field_read(bd->rmap_fields[F_OTPLD_STATE], &state);
+		if (ret)
+			return ret;
+
+		msleep(10);
+	} while (state == 0 && --rst_check_counter);
+
+	if (!rst_check_counter) {
+		dev_err(bd->dev, "chip reset not completed\n");
+		return -ETIMEDOUT;
+	}
+
+	tmp = 0;
+	ret = regmap_raw_write(bd->rmap, SYSTEM_CTRL_SET, &tmp, 2);
+
+	return ret;
+}
+
+static int bd9995x_hw_init(struct bd9995x_device *bd)
+{
+	int ret;
+	int i;
+	struct bd9995x_state state;
+	struct bd9995x_init_data *id = &bd->init_data;
+
+	const struct {
+		enum bd9995x_fields id;
+		u16 value;
+	} init_data[] = {
+		/* Enable the charging trigger after SDP charger attached */
+		{F_SDP_CHG_TRIG_EN,	1},
+		/* Enable charging trigger after SDP charger attached */
+		{F_SDP_CHG_TRIG,	1},
+		/* Disable charging trigger by BC1.2 detection */
+		{F_VBUS_BC_DISEN,	1},
+		/* Disable charging trigger by BC1.2 detection */
+		{F_VCC_BC_DISEN,	1},
+		/* Disable automatic limitation of the input current */
+		{F_ILIM_AUTO_DISEN,	1},
+		/* Select current limitation when SDP charger attached*/
+		{F_SDP_500_SEL,		1},
+		/* Select current limitation when DCP charger attached */
+		{F_DCP_2500_SEL,	1},
+		{F_VSYSREG_SET,		id->vsysreg_set},
+		/* Activate USB charging and DC/DC converter */
+		{F_USB_SUS,		0},
+		/* DCDC clock: 1200 kHz*/
+		{F_DCDC_CLK_SEL,	3},
+		/* Enable charging */
+		{F_CHG_EN,		1},
+		/* Disable Input current Limit setting voltage measurement */
+		{F_EXTIADPEN,		0},
+		/* Disable input current limiting */
+		{F_VSYS_PRIORITY,	1},
+		{F_IBUS_LIM_SET,	id->ibus_lim_set},
+		{F_ICC_LIM_SET,		id->icc_lim_set},
+		/* Charge Termination Current Setting to 0*/
+		{F_ITERM_SET,		id->iterm_set},
+		/* Trickle-charge Current Setting */
+		{F_ITRICH_SET,		id->itrich_set},
+		/* Pre-charge Current setting */
+		{F_IPRECH_SET,		id->iprech_set},
+		/* Fast Charge Current for constant current phase */
+		{F_ICHG_SET,		id->ichg_set},
+		/* Fast Charge Voltage Regulation Setting */
+		{F_VFASTCHG_REG_SET1,	id->vfastchg_reg_set1},
+		/* Set Pre-charge Voltage Threshold for trickle charging. */
+		{F_VPRECHG_TH_SET,	id->vprechg_th_set},
+		{F_VRECHG_SET,		id->vrechg_set},
+		{F_VBATOVP_SET,		id->vbatovp_set},
+		/* Reverse buck boost voltage Setting */
+		{F_VRBOOST_SET,		0},
+		/* Disable fast-charging watchdog */
+		{F_WDT_FST,		0},
+		/* Disable pre-charging watchdog */
+		{F_WDT_PRE,		0},
+		/* Power save off */
+		{F_POWER_SAVE_MODE,	0},
+		{F_INT1_SET,		INT1_ALL},
+		{F_INT2_SET,		INT2_ALL},
+		{F_INT3_SET,		INT3_ALL},
+		{F_INT4_SET,		INT4_ALL},
+		{F_INT5_SET,		INT5_ALL},
+		{F_INT6_SET,		INT6_ALL},
+		{F_INT7_SET,		INT7_ALL},
+	};
+
+	/*
+	 * Currently we initialize charger to a known state at startup.
+	 * If we want to allow for example the boot code to initialize
+	 * charger we should get rid of this.
+	 */
+	ret = __bd9995x_chip_reset(bd);
+	if (ret < 0)
+		return ret;
+
+	/* Initialize currents/voltages and other parameters */
+	for (i = 0; i < ARRAY_SIZE(init_data); i++) {
+		ret = regmap_field_write(bd->rmap_fields[init_data[i].id],
+					 init_data[i].value);
+		if (ret) {
+			dev_err(bd->dev, "failed to initialize charger (%d)\n",
+				ret);
+			return ret;
+		}
+	}
+
+	ret = bd9995x_get_chip_state(bd, &state);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&bd->lock);
+	bd->state = state;
+	mutex_unlock(&bd->lock);
+
+	return 0;
+}
+
+static enum power_supply_property bd9995x_power_supply_props[] = {
+	POWER_SUPPLY_PROP_MANUFACTURER,
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_ONLINE,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT,
+	POWER_SUPPLY_PROP_CHARGE_AVG,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE,
+	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
+	/* Battery props we access through charger */
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CHARGE_TYPE,
+	POWER_SUPPLY_PROP_HEALTH,
+	POWER_SUPPLY_PROP_TEMP,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_MODEL_NAME,
+};
+
+static const struct power_supply_desc bd9995x_power_supply_desc = {
+	.name = "bd9995x-charger",
+	.type = POWER_SUPPLY_TYPE_USB,
+	.properties = bd9995x_power_supply_props,
+	.num_properties = ARRAY_SIZE(bd9995x_power_supply_props),
+	.get_property = bd9995x_power_supply_get_property,
+};
+
+/*
+ * Limit configurations for vbus-input-current and vcc-vacp-input-current
+ * Minimum limit is 0 uA. Max is 511 * 32000 uA = 16352000 uA. This is
+ * configured by writing a register so that each increment in register
+ * value equals to 32000 uA limit increment.
+ *
+ * Eg, value 0x0 is limit 0, value 0x1 is limit 32000, ...
+ * Describe the setting in linear_range table.
+ */
+static const struct linear_range input_current_limit_ranges[] = {
+	{
+		.min = 0,
+		.step = 32000,
+		.min_sel = 0x0,
+		.max_sel = 0x1ff,
+	},
+};
+
+/* Possible trickle, pre-charging and termination current values */
+static const struct linear_range charging_current_ranges[] = {
+	{
+		.min = 0,
+		.step = 64000,
+		.min_sel = 0x0,
+		.max_sel = 0x10,
+	}, {
+		.min = 1024000,
+		.step = 0,
+		.min_sel = 0x11,
+		.max_sel = 0x1f,
+	},
+};
+
+/*
+ * Fast charging voltage regulation, starting re-charging limit
+ * and battery over voltage protection have same possible values
+ */
+static const struct linear_range charge_voltage_regulation_ranges[] = {
+	{
+		.min = 2560000,
+		.step = 0,
+		.min_sel = 0,
+		.max_sel = 0xA0,
+	}, {
+		.min = 2560000,
+		.step = 16000,
+		.min_sel = 0xA0,
+		.max_sel = 0x4B0,
+	}, {
+		.min = 19200000,
+		.step = 0,
+		.min_sel = 0x4B0,
+		.max_sel = 0x7FF,
+	},
+};
+
+/* Possible VSYS voltage regulation values */
+static const struct linear_range vsys_voltage_regulation_ranges[] = {
+	{
+		.min = 2560000,
+		.step = 0,
+		.min_sel = 0,
+		.max_sel = 0x28,
+	}, {
+		.min = 2560000,
+		.step = 64000,
+		.min_sel = 0x28,
+		.max_sel = 0x12C,
+	}, {
+		.min = 19200000,
+		.step = 0,
+		.min_sel = 0x12C,
+		.max_sel = 0x1FF,
+	},
+};
+
+/* Possible settings for switching from trickle to pre-charging limits */
+static const struct linear_range trickle_to_pre_threshold_ranges[] = {
+	{
+		.min = 2048000,
+		.step = 0,
+		.min_sel = 0,
+		.max_sel = 0x20,
+	}, {
+		.min = 2048000,
+		.step = 64000,
+		.min_sel = 0x20,
+		.max_sel = 0x12C,
+	}, {
+		.min = 19200000,
+		.step = 0,
+		.min_sel = 0x12C,
+		.max_sel = 0x1FF
+	}
+};
+
+/* Possible current values for fast-charging constant current phase */
+static const struct linear_range fast_charge_current_ranges[] = {
+	{
+		.min = 0,
+		.step = 64000,
+		.min_sel = 0,
+		.max_sel = 0xFF,
+	}
+};
+
+struct battery_init {
+	const char *name;
+	int *info_data;
+	const struct linear_range *range;
+	int ranges;
+	u16 *data;
+};
+
+struct dt_init {
+	char *prop;
+	const struct linear_range *range;
+	int ranges;
+	u16 *data;
+};
+
+static int bd9995x_fw_probe(struct bd9995x_device *bd)
+{
+	int ret;
+	struct power_supply_battery_info info;
+	u32 property;
+	int i;
+	int regval;
+	bool found;
+	struct bd9995x_init_data *init = &bd->init_data;
+	struct battery_init battery_inits[] = {
+		{
+			.name = "trickle-charging current",
+			.info_data = &info.tricklecharge_current_ua,
+			.range = &charging_current_ranges[0],
+			.ranges = 2,
+			.data = &init->itrich_set,
+		}, {
+			.name = "pre-charging current",
+			.info_data = &info.precharge_current_ua,
+			.range = &charging_current_ranges[0],
+			.ranges = 2,
+			.data = &init->iprech_set,
+		}, {
+			.name = "pre-to-trickle charge voltage threshold",
+			.info_data = &info.precharge_voltage_max_uv,
+			.range = &trickle_to_pre_threshold_ranges[0],
+			.ranges = 2,
+			.data = &init->vprechg_th_set,
+		}, {
+			.name = "charging termination current",
+			.info_data = &info.charge_term_current_ua,
+			.range = &charging_current_ranges[0],
+			.ranges = 2,
+			.data = &init->iterm_set,
+		}, {
+			.name = "charging re-start voltage",
+			.info_data = &info.charge_restart_voltage_uv,
+			.range = &charge_voltage_regulation_ranges[0],
+			.ranges = 2,
+			.data = &init->vrechg_set,
+		}, {
+			.name = "battery overvoltage limit",
+			.info_data = &info.overvoltage_limit_uv,
+			.range = &charge_voltage_regulation_ranges[0],
+			.ranges = 2,
+			.data = &init->vbatovp_set,
+		}, {
+			.name = "fast-charging max current",
+			.info_data = &info.constant_charge_current_max_ua,
+			.range = &fast_charge_current_ranges[0],
+			.ranges = 1,
+			.data = &init->ichg_set,
+		}, {
+			.name = "fast-charging voltage",
+			.info_data = &info.constant_charge_voltage_max_uv,
+			.range = &charge_voltage_regulation_ranges[0],
+			.ranges = 2,
+			.data = &init->vfastchg_reg_set1,
+		},
+	};
+	struct dt_init props[] = {
+		{
+			.prop = "rohm,vsys-regulation-microvolt",
+			.range = &vsys_voltage_regulation_ranges[0],
+			.ranges = 2,
+			.data = &init->vsysreg_set,
+		}, {
+			.prop = "rohm,vbus-input-current-limit-microamp",
+			.range = &input_current_limit_ranges[0],
+			.ranges = 1,
+			.data = &init->ibus_lim_set,
+		}, {
+			.prop = "rohm,vcc-input-current-limit-microamp",
+			.range = &input_current_limit_ranges[0],
+			.ranges = 1,
+			.data = &init->icc_lim_set,
+		},
+	};
+
+	/*
+	 * The power_supply_get_battery_info() does not support getting values
+	 * from ACPI. Let's fix it if ACPI is required here.
+	 */
+	ret = power_supply_get_battery_info(bd->charger, &info);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < ARRAY_SIZE(battery_inits); i++) {
+		int val = *battery_inits[i].info_data;
+		const struct linear_range *range = battery_inits[i].range;
+		int ranges = battery_inits[i].ranges;
+
+		if (val == -EINVAL)
+			continue;
+
+		ret = linear_range_get_selector_low_array(range, ranges, val,
+							  &regval, &found);
+		if (ret) {
+			dev_err(bd->dev, "Unsupported value for %s\n",
+				battery_inits[i].name);
+
+			power_supply_put_battery_info(bd->charger, &info);
+			return -EINVAL;
+		}
+		if (!found) {
+			dev_warn(bd->dev,
+				 "Unsupported value for %s - using smaller\n",
+				 battery_inits[i].name);
+		}
+		*(battery_inits[i].data) = regval;
+	}
+
+	power_supply_put_battery_info(bd->charger, &info);
+
+	for (i = 0; i < ARRAY_SIZE(props); i++) {
+		ret = device_property_read_u32(bd->dev, props[i].prop,
+					       &property);
+		if (ret < 0) {
+			dev_err(bd->dev, "failed to read %s", props[i].prop);
+
+			return ret;
+		}
+
+		ret = linear_range_get_selector_low_array(props[i].range,
+							  props[i].ranges,
+							  property, &regval,
+							  &found);
+		if (ret) {
+			dev_err(bd->dev, "Unsupported value for '%s'\n",
+				props[i].prop);
+
+			return -EINVAL;
+		}
+
+		if (!found) {
+			dev_warn(bd->dev,
+				 "Unsupported value for '%s' - using smaller\n",
+				 props[i].prop);
+		}
+
+		*(props[i].data) = regval;
+	}
+
+	return 0;
+}
+
+static void bd9995x_chip_reset(void *bd)
+{
+	__bd9995x_chip_reset(bd);
+}
+
+static int bd9995x_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct bd9995x_device *bd;
+	struct power_supply_config psy_cfg = {};
+	int ret;
+	int i;
+
+	bd = devm_kzalloc(dev, sizeof(*bd), GFP_KERNEL);
+	if (!bd)
+		return -ENOMEM;
+
+	bd->client = client;
+	bd->dev = dev;
+	psy_cfg.drv_data = bd;
+	psy_cfg.of_node = dev->of_node;
+
+	mutex_init(&bd->lock);
+
+	bd->rmap = devm_regmap_init_i2c(client, &bd9995x_regmap_config);
+	if (IS_ERR(bd->rmap)) {
+		dev_err(dev, "Failed to setup register access via i2c\n");
+		return PTR_ERR(bd->rmap);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(bd9995x_reg_fields); i++) {
+		const struct reg_field *reg_fields = bd9995x_reg_fields;
+
+		bd->rmap_fields[i] = devm_regmap_field_alloc(dev, bd->rmap,
+							     reg_fields[i]);
+		if (IS_ERR(bd->rmap_fields[i])) {
+			dev_err(dev, "cannot allocate regmap field\n");
+			return PTR_ERR(bd->rmap_fields[i]);
+		}
+	}
+
+	i2c_set_clientdata(client, bd);
+
+	ret = regmap_field_read(bd->rmap_fields[F_CHIP_ID], &bd->chip_id);
+	if (ret) {
+		dev_err(dev, "Cannot read chip ID.\n");
+		return ret;
+	}
+
+	if (bd->chip_id != BD99954_ID) {
+		dev_err(dev, "Chip with ID=0x%x, not supported!\n",
+			bd->chip_id);
+		return -ENODEV;
+	}
+
+	ret = regmap_field_read(bd->rmap_fields[F_CHIP_REV], &bd->chip_rev);
+	if (ret) {
+		dev_err(dev, "Cannot read revision.\n");
+		return ret;
+	}
+
+	dev_info(bd->dev, "Found BD99954 chip rev %d\n", bd->chip_rev);
+
+	/*
+	 * We need to init the psy before we can call
+	 * power_supply_get_battery_info() for it
+	 */
+	bd->charger = devm_power_supply_register(bd->dev,
+						 &bd9995x_power_supply_desc,
+						&psy_cfg);
+	if (IS_ERR(bd->charger)) {
+		dev_err(dev, "Failed to register power supply\n");
+		return PTR_ERR(bd->charger);
+	}
+
+	ret = bd9995x_fw_probe(bd);
+	if (ret < 0) {
+		dev_err(dev, "Cannot read device properties.\n");
+		return ret;
+	}
+
+	ret = bd9995x_hw_init(bd);
+	if (ret < 0) {
+		dev_err(dev, "Cannot initialize the chip.\n");
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(dev, bd9995x_chip_reset, bd);
+	if (ret)
+		return ret;
+
+	return devm_request_threaded_irq(dev, client->irq, NULL,
+					 bd9995x_irq_handler_thread,
+					 IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+					 BD9995X_IRQ_PIN, bd);
+}
+
+static const struct of_device_id bd9995x_of_match[] = {
+	{ .compatible = "rohm,bd99954", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, bd9995x_of_match);
+
+static struct i2c_driver bd9995x_driver = {
+	.driver = {
+		.name = "bd9995x-charger",
+		.of_match_table = bd9995x_of_match,
+	},
+	.probe_new = bd9995x_probe,
+};
+module_i2c_driver(bd9995x_driver);
+
+MODULE_AUTHOR("Laine Markus <markus.laine@fi.rohmeurope.com>");
+MODULE_DESCRIPTION("ROHM BD99954 charger driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/bd99954-charger.h b/drivers/power/supply/bd99954-charger.h
new file mode 100644
index 000000000000..f58897925383
--- /dev/null
+++ b/drivers/power/supply/bd99954-charger.h
@@ -0,0 +1,1075 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2020 ROHM Semiconductors */
+#ifndef BD99954_CHARGER_H
+#define BD99954_CHARGER_H
+
+#include <linux/regmap.h>
+
+#define BD9995X_MANUFACTURER "Rohm Semiconductor"
+#define BD9995X_IRQ_PIN      "bd9995x_irq"
+
+#define BD9995X_VSYS_PRECHARGE_OFFSET_MV 200
+
+#define BD99954_ID            0x346
+#define BD99955_ID            0x221
+#define BD99956_ID            0x331
+
+/* Battery Charger Commands */
+#define    CHARGING_CURRENT   0x14
+#define    CHARGING_VOLTAGE   0x15
+#define    PROTECT_SET        0x3E
+#define    MAP_SET            0x3F
+
+/* Extended commands */
+#define    CHGSTM_STATUS       0x100
+#define    VBAT_VSYS_STATUS    0x101
+#define    VBUS_VCC_STATUS     0x102
+#define    CHGOP_STATUS        0x103
+#define    WDT_STATUS          0x104
+#define    CUR_ILIM_VAL        0x105
+#define    SEL_ILIM_VAL        0x106
+#define    IBUS_LIM_SET        0x107
+#define    ICC_LIM_SET         0x108
+#define    IOTG_LIM_SET        0x109
+#define    VIN_CTRL_SET        0x10A
+#define    CHGOP_SET1          0x10B
+#define    CHGOP_SET2          0x10C
+#define    VBUSCLPS_TH_SET     0x10D
+#define    VCCCLPS_TH_SET      0x10E
+#define    CHGWDT_SET          0x10F
+#define    BATTWDT_SET         0x110
+#define    VSYSREG_SET         0x111
+#define    VSYSVAL_THH_SET     0x112
+#define    VSYSVAL_THL_SET     0x113
+#define    ITRICH_SET          0x114
+#define    IPRECH_SET          0x115
+#define    ICHG_SET            0x116
+#define    ITERM_SET           0x117
+#define    VPRECHG_TH_SET      0x118
+#define    VRBOOST_SET         0x119
+#define    VFASTCHG_REG_SET1   0x11A
+#define    VFASTCHG_REG_SET2   0x11B
+#define    VFASTCHG_REG_SET3   0x11C
+#define    VRECHG_SET          0x11D
+#define    VBATOVP_SET         0x11E
+#define    IBATSHORT_SET       0x11F
+#define    PROCHOT_CTRL_SET    0x120
+#define    PROCHOT_ICRIT_SET   0x121
+#define    PROCHOT_INORM_SET   0x122
+#define    PROCHOT_IDCHG_SET   0x123
+#define    PROCHOT_VSYS_SET    0x124
+#define    PMON_IOUT_CTRL_SET  0x125
+#define    PMON_DACIN_VAL      0x126
+#define    IOUT_DACIN_VAL      0x127
+#define    VCC_UCD_SET         0x128
+#define    VCC_UCD_STATUS      0x129
+#define    VCC_IDD_STATUS      0x12A
+#define    VCC_UCD_FCTRL_SET   0x12B
+#define    VCC_UCD_FCTRL_EN    0x12C
+#define    VBUS_UCD_SET        0x130
+#define    VBUS_UCD_STATUS     0x131
+#define    VBUS_IDD_STATUS     0x132
+#define    VBUS_UCD_FCTRL_SET  0x133
+#define    VBUS_UCD_FCTRL_EN   0x134
+#define    CHIP_ID             0x138
+#define    CHIP_REV            0x139
+#define    IC_SET1             0x13A
+#define    IC_SET2             0x13B
+#define    SYSTEM_STATUS       0x13C
+#define    SYSTEM_CTRL_SET     0x13D
+#define    VM_CTRL_SET         0x140
+#define    THERM_WINDOW_SET1   0x141
+#define    THERM_WINDOW_SET2   0x142
+#define    THERM_WINDOW_SET3   0x143
+#define    THERM_WINDOW_SET4   0x144
+#define    THERM_WINDOW_SET5   0x145
+#define    IBATP_TH_SET        0x146
+#define    IBATM_TH_SET        0x147
+#define    VBAT_TH_SET         0x148
+#define    THERM_TH_SET        0x149
+#define    IACP_TH_SET         0x14A
+#define    VACP_TH_SET         0x14B
+#define    VBUS_TH_SET         0x14C
+#define    VCC_TH_SET          0x14D
+#define    VSYS_TH_SET         0x14E
+#define    EXTIADP_TH_SET      0x14F
+#define    IBATP_VAL           0x150
+#define    IBATP_AVE_VAL       0x151
+#define    IBATM_VAL           0x152
+#define    IBATM_AVE_VAL       0x153
+#define    VBAT_VAL            0x154
+#define    VBAT_AVE_VAL        0x155
+#define    THERM_VAL           0x156
+#define    VTH_VAL             0x157
+#define    IACP_VAL            0x158
+#define    IACP_AVE_VAL        0x159
+#define    VACP_VAL            0x15A
+#define    VACP_AVE_VAL        0x15B
+#define    VBUS_VAL            0x15C
+#define    VBUS_AVE_VAL        0x15D
+#define    VCC_VAL             0x15E
+#define    VCC_AVE_VAL         0x15F
+#define    VSYS_VAL            0x160
+#define    VSYS_AVE_VAL        0x161
+#define    EXTIADP_VAL         0x162
+#define    EXTIADP_AVE_VAL     0x163
+#define    VACPCLPS_TH_SET     0x164
+#define    INT0_SET            0x168
+#define    INT1_SET            0x169
+#define    INT2_SET            0x16A
+#define    INT3_SET            0x16B
+#define    INT4_SET            0x16C
+#define    INT5_SET            0x16D
+#define    INT6_SET            0x16E
+#define    INT7_SET            0x16F
+#define    INT0_STATUS         0x170
+#define    INT1_STATUS         0x171
+#define    INT2_STATUS         0x172
+#define    INT3_STATUS         0x173
+#define    INT4_STATUS         0x174
+#define    INT5_STATUS         0x175
+#define    INT6_STATUS         0x176
+#define    INT7_STATUS         0x177
+#define    OTPREG0             0x17A
+#define    OTPREG1             0x17B
+#define    SMBREG              0x17C
+#define    DEBUG_MODE_SET      0x17F
+#define    DEBUG0x14           0x214
+#define    DEBUG0x1A           0x21A
+
+enum bd9995x_fields {
+	F_PREV_CHGSTM_STATE, F_CHGSTM_STATE,
+	F_VBAT_VSYS_STATUS,
+	F_VBUS_VCC_STATUS,
+	F_BATTEMP, F_VRECHG_DET, F_RBOOST_UV, F_RBOOSTS,
+	F_THERMWDT_VAL, F_CHGWDT_VAL,
+	F_CUR_ILIM_VAL,
+	F_SEL_ILIM_VAL,
+	F_IBUS_LIM_SET,
+	F_ICC_LIM_SET,
+	F_IOTG_LIM_SET,
+	F_OTG_BOTH_EN,
+	F_VRBOOST_TRIG,
+	F_VRBOOST_EN,
+	F_PP_BOTH_THRU,
+	F_VIN_ORD,
+	F_VBUS_EN,
+	F_VCC_EN,
+	F_VSYS_PRIORITY,
+	F_PPC_SUB_CAP,
+	F_PPC_CAP,
+	F_DCP_2500_SEL,
+	F_SDP_500_SEL,
+	F_ILIM_AUTO_DISEN,
+	F_VCC_BC_DISEN,
+	F_VBUS_BC_DISEN,
+	F_SDP_CHG_TRIG_EN,
+	F_SDP_CHG_TRIG,
+	F_AUTO_TOF,
+	F_AUTO_FST,
+	F_AUTO_RECH,
+	F_ILIM_RESET_EN,
+	F_DCDC_1MS_SEL,
+	F_SEL_ILIM_DIV,
+	F_BATT_LEARN,
+	F_CHG_EN,
+	F_USB_SUS,
+	F_CHOP_SS_INIT,
+	F_CHOP_ALL_INIT,
+	F_DCDC_CLK_SEL,
+	F_CHOP_SS,
+	F_CHOP_ALL,
+	F_VBUSCLPS_TH_SET,
+	F_VCCCLPS_TH_SET,
+	F_WDT_FST,
+	F_WDT_PRE,
+	F_WDT_IBAT_SHORT,
+	F_WDT_THERM,
+	F_VSYSREG_SET,
+	F_VSYSVAL_THH_SET,
+	F_VSYSVAL_THL_SET,
+	F_ITRICH_SET,
+	F_IPRECH_SET,
+	F_ICHG_SET,
+	F_ITERM_SET,
+	F_VPRECHG_TH_SET,
+	F_VRBOOST_SET,
+	F_VFASTCHG_REG_SET1,
+	F_VFASTCHG_REG_SET2,
+	F_VFASTCHG_REG_SET3,
+	F_VRECHG_SET,
+	F_VBATOVP_SET,
+	F_IBATM_SHORT_SET,
+	F_PROCHOT_DG_SET,
+	F_PROCHOT_ICRIT_DG_SET,
+	F_PROCHOT_IDCHG_DG_SET,
+	F_PROCHOT_EN,
+	F_PROCHOT_ICRIT_SET,
+	F_PROCHOT_INORM_SET,
+	F_PROCHOT_IDCHG_SET,
+	F_PROCHOT_VSYS_SET,
+	F_IMON_INSEL,
+	F_PMON_INSEL,
+	F_IOUT_OUT_EN,
+	F_IOUT_SOURCE_SEL,
+	F_IOUT_GAIN_SET,
+	F_PMON_OUT_EN,
+	F_PMON_GAIN_SET,
+	F_PMON_DACIN_VAL,
+	F_IOUT_DACIN_VAL,
+	F_VCC_BCSRETRY,
+	F_VCC_ADCRTRY,
+	F_VCC_USBDETEN,
+	F_VCC_IDRDETEN,
+	F_VCC_ENUMRDY,
+	F_VCC_ADCPOLEN,
+	F_VCC_DCDMODE,
+	F_VCC_USB_SW_EN,
+	F_VCC_USB_SW,
+	F_VCC_DCDFAIL,
+	F_VCC_CHGPORT,
+	F_VCC_PUPDET,
+	F_VCC_VBUS_VLD,
+	F_VCC_CHGDET,
+	F_VCC_OTGDET,
+	F_VCC_VBINOP,
+	F_VCC_EXTID,
+	F_VCC_IDRDET,
+	F_VCC_INDO,
+	F_VCC_UCDSWEN,
+	F_VCC_RREF_EN,
+	F_VCC_DPPU_EN,
+	F_VCC_DPREF_EN,
+	F_VCC_DMREF_EN,
+	F_VCC_DPDET_EN,
+	F_VCC_DMDET_EN,
+	F_VCC_DPSINK_EN,
+	F_VCC_DMSINK_EN,
+	F_VCC_DP_BUFF_EN,
+	F_VCC_DM_BUFF_EN,
+	F_VCC_EXTCLKENBL,
+	F_VCC_PLSTESTEN,
+	F_VCC_UCDSWEN_TSTENB,
+	F_VCC_RREF_EN_TSTENB,
+	F_VCC_DPPU_EN_TSTENB,
+	F_VCC_DPREF_EN_TSTENB,
+	F_VCC_DMREF_EN_TSTENB,
+	F_VCC_DPDET_EN_TSTENB,
+	F_VCC_DMDET_EN_TSTENB,
+	F_VCC_DPSINK_EN_TSTENB,
+	F_VCC_DMSINK_EN_TSTENB,
+	F_VCC_DP_BUFF_EN_TSTENB,
+	F_VCC_DM_BUFF_EN_TSTENB,
+	F_VBUS_BCSRETRY,
+	F_VBUS_ADCRTRY,
+	F_VBUS_USBDETEN,
+	F_VBUS_IDRDETEN,
+	F_VBUS_ENUMRDY,
+	F_VBUS_ADCPOLEN,
+	F_VBUS_DCDMODE,
+	F_VBUS_USB_SW_EN,
+	F_VBUS_USB_SW,
+	F_VBUS_DCDFAIL,
+	F_VBUS_CHGPORT,
+	F_VBUS_PUPDET,
+	F_VBUS_VBUS_VLD,
+	F_VBUS_CHGDET,
+	F_VBUS_OTGDET,
+	F_VBUS_VBINOP,
+	F_VBUS_EXTID,
+	F_VBUS_IDRDET,
+	F_VBUS_INDO,
+	F_VBUS_UCDSWEN,
+	F_VBUS_RREF_EN,
+	F_VBUS_DPPU_EN,
+	F_VBUS_DPREF_EN,
+	F_VBUS_DMREF_EN,
+	F_VBUS_DPDET_EN,
+	F_VBUS_DMDET_EN,
+	F_VBUS_DPSINK_EN,
+	F_VBUS_DMSINK_EN,
+	F_VBUS_DP_BUFF_EN,
+	F_VBUS_DM_BUFF_EN,
+	F_VBUS_EXTCLKENBL,
+	F_VBUS_PLSTESTEN,
+	F_VBUS_UCDSWEN_TSTENB,
+	F_VBUS_RREF_EN_TSTENB,
+	F_VBUS_DPPU_EN_TSTENB,
+	F_VBUS_DPREF_EN_TSTENB,
+	F_VBUS_DMREF_EN_TSTENB,
+	F_VBUS_DPDET_EN_TSTENB,
+	F_VBUS_DMDET_EN_TSTENB,
+	F_VBUS_DPSINK_EN_TSTENB,
+	F_VBUS_DMSINK_EN_TSTENB,
+	F_VBUS_DP_BUFF_EN_TSTENB,
+	F_VBUS_DM_BUFF_EN_TSTENB,
+	F_CHIP_ID,
+	F_CHIP_REV,
+	F_ONE_CELL_MODE,
+	F_cell,
+	F_VACP_AUTO_DISCHG,
+	F_VACP_LOAD,
+	F_ACOK_POL,
+	F_ACOK_DISEN,
+	F_DEBUG_SET1,
+	F_DEBUG_SET0,
+	F_MONRST_STATE,
+	F_ALMRST_STATE,
+	F_CHGRST_STATE,
+	F_OTPLD_STATE,
+	F_ALLRST_STATE,
+	F_PROTECT_SET,
+	F_MAP_SET,
+	F_ADCINTERVAL,
+	F_ADCMOD,
+	F_ADCTMOD,
+	F_EXTIADPEN,
+	F_VSYSENB,
+	F_VCCENB,
+	F_VBUSENB,
+	F_VACPENB,
+	F_IACPENB,
+	F_THERMENB,
+	F_VBATENB,
+	F_IBATMENB,
+	F_IBATPENB,
+	F_TMPTHR1B,
+	F_TMPTHR1A,
+	F_TMPTHR2B,
+	F_TMPTHR2A,
+	F_TMPTHR3B,
+	F_TMPTHR3A,
+	F_TMPTHR4B,
+	F_TMPTHR4A,
+	F_TMPTHR5B,
+	F_TMPTHR5A,
+	F_IBATP_TH_SET,
+	F_IBATM_TH_SET,
+	F_VBAT_TH_SET,
+	F_THERM_TH_SET,
+	F_IACP_TH_SET,
+	F_VACP_TH_SET,
+	F_VBUS_TH_SET,
+	F_VCC_TH_SET,
+	F_VSYS_TH_SET,
+	F_EXTIADP_TH_SET,
+	F_IBATP_VAL,
+	F_IBATP_AVE_VAL,
+	F_IBATM_VAL,
+	F_IBATM_AVE_VAL,
+	F_VBAT_VAL,
+	F_VBAT_AVE_VAL,
+	F_THERM_VAL,
+	F_VTH_VAL,
+	F_IACP_VAL,
+	F_IACP_AVE_VAL,
+	F_VACP_VAL,
+	F_VACP_AVE_VAL,
+	F_VBUS_VAL,
+	F_VBUS_AVE_VAL,
+	F_VCC_VAL,
+	F_VCC_AVE_VAL,
+	F_VSYS_VAL,
+	F_VSYS_AVE_VAL,
+	F_EXTIADP_VAL,
+	F_EXTIADP_AVE_VAL,
+	F_VACPCLPS_TH_SET,
+	F_INT7_SET,
+	F_INT6_SET,
+	F_INT5_SET,
+	F_INT4_SET,
+	F_INT3_SET,
+	F_INT2_SET,
+	F_INT1_SET,
+	F_INT0_SET,
+	F_VBUS_RBUV_DET,
+	F_VBUS_RBUV_RES,
+	F_VBUS_TH_DET,
+	F_VBUS_TH_RES,
+	F_VBUS_IIN_MOD,
+	F_VBUS_OV_DET,
+	F_VBUS_OV_RES,
+	F_VBUS_CLPS_DET,
+	F_VBUS_CLPS,
+	F_VBUS_DET,
+	F_VBUS_RES,
+	F_VCC_RBUV_DET,
+	F_VCC_RBUV_RES,
+	F_VCC_TH_DET,
+	F_VCC_TH_RES,
+	F_VCC_IIN_MOD,
+	F_VCC_OVP_DET,
+	F_VCC_OVP_RES,
+	F_VCC_CLPS_DET,
+	F_VCC_CLPS_RES,
+	F_VCC_DET,
+	F_VCC_RES,
+	F_TH_DET,
+	F_TH_RMV,
+	F_TMP_OUT_DET,
+	F_TMP_OUT_RES,
+	F_VBAT_TH_DET,
+	F_VBAT_TH_RES,
+	F_IBAT_SHORT_DET,
+	F_IBAT_SHORT_RES,
+	F_VBAT_OV_DET,
+	F_VBAT_OV_RES,
+	F_BAT_ASSIST_DET,
+	F_BAT_ASSIST_RES,
+	F_VSYS_TH_DET,
+	F_VSYS_TH_RES,
+	F_VSYS_OV_DET,
+	F_VSYS_OV_RES,
+	F_VSYS_SHT_DET,
+	F_VSYS_SHT_RES,
+	F_VSYS_UV_DET,
+	F_VSYS_UV_RES,
+	F_OTP_LOAD_DONE,
+	F_PWR_ON,
+	F_EXTIADP_TRNS,
+	F_EXTIADP_TH_DET,
+	F_EXIADP_TH_RES,
+	F_BAT_MNT_DET,
+	F_BAT_MNT_RES,
+	F_TSD_DET,
+	F_TSD_RES,
+	F_CHGWDT_EXP,
+	F_THERMWDT_EXP,
+	F_TMP_TRNS,
+	F_CHG_TRNS,
+	F_VBUS_UCD_PORT_DET,
+	F_VBUS_UCD_UCHG_DET,
+	F_VBUS_UCD_URID_RMV,
+	F_VBUS_UCD_OTG_DET,
+	F_VBUS_UCD_URID_MOD,
+	F_VCC_UCD_PORT_DET,
+	F_VCC_UCD_UCHG_DET,
+	F_VCC_UCD_URID_RMV,
+	F_VCC_UCD_OTG_DET,
+	F_VCC_UCD_URID_MOD,
+	F_PROCHOT_DET,
+	F_PROCHOT_RES,
+	F_VACP_DET,
+	F_VACP_RES,
+	F_VACP_TH_DET,
+	F_VACP_TH_RES,
+	F_IACP_TH_DET,
+	F_IACP_THE_RES,
+	F_THERM_TH_DET,
+	F_THERM_TH_RES,
+	F_IBATM_TH_DET,
+	F_IBATM_TH_RES,
+	F_IBATP_TH_DET,
+	F_IBATP_TH_RES,
+	F_INT7_STATUS,
+	F_INT6_STATUS,
+	F_INT5_STATUS,
+	F_INT4_STATUS,
+	F_INT3_STATUS,
+	F_INT2_STATUS,
+	F_INT1_STATUS,
+	F_INT0_STATUS,
+	F_ILIM_DECREASE,
+	F_RESERVE_OTPREG1,
+	F_POWER_SAVE_MODE,
+	F_DEBUG_MODE_SET,
+	F_DEBUG0x14,
+	F_DEBUG0x1A,
+	F_MAX_FIELDS
+};
+
+static const struct reg_field bd9995x_reg_fields[] = {
+	    [F_PREV_CHGSTM_STATE] = REG_FIELD(CHGSTM_STATUS, 8, 14),
+	    [F_CHGSTM_STATE] = REG_FIELD(CHGSTM_STATUS, 0, 6),
+	    [F_VBAT_VSYS_STATUS] = REG_FIELD(VBAT_VSYS_STATUS, 0, 15),
+	    [F_VBUS_VCC_STATUS] = REG_FIELD(VBUS_VCC_STATUS, 0, 12),
+	    [F_BATTEMP] = REG_FIELD(CHGOP_STATUS, 8, 10),
+	    [F_VRECHG_DET] = REG_FIELD(CHGOP_STATUS, 6, 6),
+	    [F_RBOOST_UV] = REG_FIELD(CHGOP_STATUS, 1, 1),
+	    [F_RBOOSTS] = REG_FIELD(CHGOP_STATUS, 0, 0),
+	    [F_THERMWDT_VAL] = REG_FIELD(WDT_STATUS, 8, 15),
+	    [F_CHGWDT_VAL] = REG_FIELD(WDT_STATUS, 0, 7),
+	    [F_CUR_ILIM_VAL] = REG_FIELD(CUR_ILIM_VAL, 0, 13),
+	    [F_SEL_ILIM_VAL] = REG_FIELD(SEL_ILIM_VAL, 0, 13),
+	    [F_IBUS_LIM_SET] = REG_FIELD(IBUS_LIM_SET, 5, 13),
+	    [F_ICC_LIM_SET] = REG_FIELD(ICC_LIM_SET, 5, 13),
+	    [F_IOTG_LIM_SET] = REG_FIELD(IOTG_LIM_SET, 5, 13),
+	    [F_OTG_BOTH_EN] = REG_FIELD(VIN_CTRL_SET, 15, 15),
+	    [F_VRBOOST_TRIG] = REG_FIELD(VIN_CTRL_SET, 14, 14),
+	    [F_VRBOOST_EN] = REG_FIELD(VIN_CTRL_SET, 12, 13),
+	    [F_PP_BOTH_THRU] = REG_FIELD(VIN_CTRL_SET, 11, 11),
+	    [F_VIN_ORD] = REG_FIELD(VIN_CTRL_SET, 7, 7),
+	    [F_VBUS_EN] = REG_FIELD(VIN_CTRL_SET, 6, 6),
+	    [F_VCC_EN] = REG_FIELD(VIN_CTRL_SET, 5, 5),
+	    [F_VSYS_PRIORITY] = REG_FIELD(VIN_CTRL_SET, 4, 4),
+	    [F_PPC_SUB_CAP] = REG_FIELD(VIN_CTRL_SET, 2, 3),
+	    [F_PPC_CAP] = REG_FIELD(VIN_CTRL_SET, 0, 1),
+	    [F_DCP_2500_SEL] = REG_FIELD(CHGOP_SET1, 15, 15),
+	    [F_SDP_500_SEL] = REG_FIELD(CHGOP_SET1, 14, 14),
+	    [F_ILIM_AUTO_DISEN] = REG_FIELD(CHGOP_SET1, 13, 13),
+	    [F_VCC_BC_DISEN] = REG_FIELD(CHGOP_SET1, 11, 11),
+	    [F_VBUS_BC_DISEN] = REG_FIELD(CHGOP_SET1, 10, 10),
+	    [F_SDP_CHG_TRIG_EN] = REG_FIELD(CHGOP_SET1, 9, 9),
+	    [F_SDP_CHG_TRIG] = REG_FIELD(CHGOP_SET1, 8, 8),
+	    [F_AUTO_TOF] = REG_FIELD(CHGOP_SET1, 6, 6),
+	    [F_AUTO_FST] = REG_FIELD(CHGOP_SET1, 5, 5),
+	    [F_AUTO_RECH] = REG_FIELD(CHGOP_SET1, 3, 3),
+	    [F_ILIM_RESET_EN] = REG_FIELD(CHGOP_SET2, 14, 14),
+	    [F_DCDC_1MS_SEL] = REG_FIELD(CHGOP_SET2, 12, 13),
+	    [F_SEL_ILIM_DIV] = REG_FIELD(CHGOP_SET2, 10, 10),
+	    [F_BATT_LEARN] = REG_FIELD(CHGOP_SET2, 8, 8),
+	    [F_CHG_EN] = REG_FIELD(CHGOP_SET2, 7, 7),
+	    [F_USB_SUS] = REG_FIELD(CHGOP_SET2, 6, 6),
+	    [F_CHOP_SS_INIT] = REG_FIELD(CHGOP_SET2, 5, 5),
+	    [F_CHOP_ALL_INIT] = REG_FIELD(CHGOP_SET2, 4, 4),
+	    [F_DCDC_CLK_SEL] = REG_FIELD(CHGOP_SET2, 2, 3),
+	    [F_CHOP_SS] = REG_FIELD(CHGOP_SET2, 1, 1),
+	    [F_CHOP_ALL] = REG_FIELD(CHGOP_SET2, 0, 0),
+	    [F_VBUSCLPS_TH_SET] = REG_FIELD(VBUSCLPS_TH_SET, 7, 14),
+	    [F_VCCCLPS_TH_SET] = REG_FIELD(VCCCLPS_TH_SET, 7, 14),
+	    [F_WDT_FST] = REG_FIELD(CHGWDT_SET, 8, 15),
+	    [F_WDT_PRE] = REG_FIELD(CHGWDT_SET, 0, 7),
+	    [F_WDT_IBAT_SHORT] = REG_FIELD(BATTWDT_SET, 8, 15),
+	    [F_WDT_THERM] = REG_FIELD(BATTWDT_SET, 0, 7),
+	    [F_VSYSREG_SET] = REG_FIELD(VSYSREG_SET, 6, 14),
+	    [F_VSYSVAL_THH_SET] = REG_FIELD(VSYSVAL_THH_SET, 6, 14),
+	    [F_VSYSVAL_THL_SET] = REG_FIELD(VSYSVAL_THL_SET, 6, 14),
+	    [F_ITRICH_SET] = REG_FIELD(ITRICH_SET, 6, 10),
+	    [F_IPRECH_SET] = REG_FIELD(IPRECH_SET, 6, 10),
+	    [F_ICHG_SET] = REG_FIELD(ICHG_SET, 6, 13),
+	    [F_ITERM_SET] = REG_FIELD(ITERM_SET, 6, 10),
+	    [F_VPRECHG_TH_SET] = REG_FIELD(VPRECHG_TH_SET, 6, 14),
+	    [F_VRBOOST_SET] = REG_FIELD(VRBOOST_SET, 6, 14),
+	    [F_VFASTCHG_REG_SET1] = REG_FIELD(VFASTCHG_REG_SET1, 4, 14),
+	    [F_VFASTCHG_REG_SET2] = REG_FIELD(VFASTCHG_REG_SET2, 4, 14),
+	    [F_VFASTCHG_REG_SET3] = REG_FIELD(VFASTCHG_REG_SET3, 4, 14),
+	    [F_VRECHG_SET] = REG_FIELD(VRECHG_SET, 4, 14),
+	    [F_VBATOVP_SET] = REG_FIELD(VBATOVP_SET, 4, 14),
+	    [F_IBATM_SHORT_SET] = REG_FIELD(IBATSHORT_SET, 0, 14),
+	    [F_PROCHOT_DG_SET] = REG_FIELD(PROCHOT_CTRL_SET, 14, 15),
+	    [F_PROCHOT_ICRIT_DG_SET] = REG_FIELD(PROCHOT_CTRL_SET, 10, 11),
+	    [F_PROCHOT_IDCHG_DG_SET] = REG_FIELD(PROCHOT_CTRL_SET, 8, 9),
+	    [F_PROCHOT_EN] = REG_FIELD(PROCHOT_CTRL_SET, 0, 4),
+	    [F_PROCHOT_ICRIT_SET] = REG_FIELD(PROCHOT_ICRIT_SET, 0, 14),
+	    [F_PROCHOT_INORM_SET] = REG_FIELD(PROCHOT_INORM_SET, 0, 14),
+	    [F_PROCHOT_IDCHG_SET] = REG_FIELD(PROCHOT_IDCHG_SET, 0, 14),
+	    [F_PROCHOT_VSYS_SET] = REG_FIELD(PROCHOT_VSYS_SET, 0, 14),
+	    [F_IMON_INSEL] = REG_FIELD(PMON_IOUT_CTRL_SET, 9, 9),
+	    [F_PMON_INSEL] = REG_FIELD(PMON_IOUT_CTRL_SET, 8, 8),
+	    [F_IOUT_OUT_EN] = REG_FIELD(PMON_IOUT_CTRL_SET, 7, 7),
+	    [F_IOUT_SOURCE_SEL] = REG_FIELD(PMON_IOUT_CTRL_SET, 6, 6),
+	    [F_IOUT_GAIN_SET] = REG_FIELD(PMON_IOUT_CTRL_SET, 4, 5),
+	    [F_PMON_OUT_EN] = REG_FIELD(PMON_IOUT_CTRL_SET, 3, 3),
+	    [F_PMON_GAIN_SET] = REG_FIELD(PMON_IOUT_CTRL_SET, 0, 2),
+	    [F_PMON_DACIN_VAL] = REG_FIELD(PMON_DACIN_VAL, 0, 9),
+	    [F_IOUT_DACIN_VAL] = REG_FIELD(IOUT_DACIN_VAL, 0, 11),
+	    [F_VCC_BCSRETRY] = REG_FIELD(VCC_UCD_SET, 12, 12),
+	    [F_VCC_ADCRTRY] = REG_FIELD(VCC_UCD_SET, 8, 8),
+	    [F_VCC_USBDETEN] = REG_FIELD(VCC_UCD_SET, 7, 7),
+	    [F_VCC_IDRDETEN] = REG_FIELD(VCC_UCD_SET, 6, 6),
+	    [F_VCC_ENUMRDY] = REG_FIELD(VCC_UCD_SET, 5, 5),
+	    [F_VCC_ADCPOLEN] = REG_FIELD(VCC_UCD_SET, 4, 4),
+	    [F_VCC_DCDMODE] = REG_FIELD(VCC_UCD_SET, 3, 3),
+	    [F_VCC_USB_SW_EN] = REG_FIELD(VCC_UCD_SET, 1, 1),
+	    [F_VCC_USB_SW] = REG_FIELD(VCC_UCD_SET, 0, 0),
+	    [F_VCC_DCDFAIL] = REG_FIELD(VCC_UCD_STATUS, 15, 15),
+	    [F_VCC_CHGPORT] = REG_FIELD(VCC_UCD_STATUS, 12, 13),
+	    [F_VCC_PUPDET] = REG_FIELD(VCC_UCD_STATUS, 11, 11),
+	    [F_VCC_VBUS_VLD] = REG_FIELD(VCC_UCD_STATUS, 7, 7),
+	    [F_VCC_CHGDET] = REG_FIELD(VCC_UCD_STATUS, 6, 6),
+	    [F_VCC_OTGDET] = REG_FIELD(VCC_UCD_STATUS, 3, 3),
+	    [F_VCC_VBINOP] = REG_FIELD(VCC_IDD_STATUS, 6, 6),
+	    [F_VCC_EXTID] = REG_FIELD(VCC_IDD_STATUS, 5, 5),
+	    [F_VCC_IDRDET] = REG_FIELD(VCC_IDD_STATUS, 4, 4),
+	    [F_VCC_INDO] = REG_FIELD(VCC_IDD_STATUS, 0, 3),
+	    [F_VCC_UCDSWEN] = REG_FIELD(VCC_UCD_FCTRL_SET, 10, 10),
+	    [F_VCC_RREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 9, 9),
+	    [F_VCC_DPPU_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 8, 8),
+	    [F_VCC_DPREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 7, 7),
+	    [F_VCC_DMREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 6, 6),
+	    [F_VCC_DPDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 5, 5),
+	    [F_VCC_DMDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 4, 4),
+	    [F_VCC_DPSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 3, 3),
+	    [F_VCC_DMSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 2, 2),
+	    [F_VCC_DP_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 1, 1),
+	    [F_VCC_DM_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 0, 0),
+	    [F_VCC_EXTCLKENBL] = REG_FIELD(VCC_UCD_FCTRL_EN, 15, 15),
+	    [F_VCC_PLSTESTEN] = REG_FIELD(VCC_UCD_FCTRL_EN, 14, 14),
+	    [F_VCC_UCDSWEN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 10, 10),
+	    [F_VCC_RREF_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 9, 9),
+	    [F_VCC_DPPU_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 8, 8),
+	    [F_VCC_DPREF_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 7, 7),
+	    [F_VCC_DMREF_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 6, 6),
+	    [F_VCC_DPDET_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 5, 5),
+	    [F_VCC_DMDET_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 4, 4),
+	    [F_VCC_DPSINK_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 3, 3),
+	    [F_VCC_DMSINK_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 2, 2),
+	    [F_VCC_DP_BUFF_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 1, 1),
+	    [F_VCC_DM_BUFF_EN_TSTENB] = REG_FIELD(VCC_UCD_FCTRL_EN, 0, 0),
+
+	    [F_VBUS_BCSRETRY] = REG_FIELD(VBUS_UCD_SET, 12, 12),
+	    [F_VBUS_ADCRTRY] = REG_FIELD(VBUS_UCD_SET, 8, 8),
+	    [F_VBUS_USBDETEN] = REG_FIELD(VBUS_UCD_SET, 7, 7),
+	    [F_VBUS_IDRDETEN] = REG_FIELD(VBUS_UCD_SET, 6, 6),
+	    [F_VBUS_ENUMRDY] = REG_FIELD(VBUS_UCD_SET, 5, 5),
+	    [F_VBUS_ADCPOLEN] = REG_FIELD(VBUS_UCD_SET, 4, 4),
+	    [F_VBUS_DCDMODE] = REG_FIELD(VBUS_UCD_SET, 3, 3),
+	    [F_VBUS_USB_SW_EN] = REG_FIELD(VBUS_UCD_SET, 1, 1),
+	    [F_VBUS_USB_SW] = REG_FIELD(VBUS_UCD_SET, 0, 0),
+	    [F_VBUS_DCDFAIL] = REG_FIELD(VBUS_UCD_STATUS, 15, 15),
+	    [F_VBUS_CHGPORT] = REG_FIELD(VBUS_UCD_STATUS, 12, 13),
+	    [F_VBUS_PUPDET] = REG_FIELD(VBUS_UCD_STATUS, 11, 11),
+	    [F_VBUS_VBUS_VLD] = REG_FIELD(VBUS_UCD_STATUS, 7, 7),
+	    [F_VBUS_CHGDET] = REG_FIELD(VBUS_UCD_STATUS, 6, 6),
+	    [F_VBUS_OTGDET] = REG_FIELD(VBUS_UCD_STATUS, 3, 3),
+	    [F_VBUS_VBINOP] = REG_FIELD(VBUS_IDD_STATUS, 6, 6),
+	    [F_VBUS_EXTID] = REG_FIELD(VBUS_IDD_STATUS, 5, 5),
+	    [F_VBUS_IDRDET] = REG_FIELD(VBUS_IDD_STATUS, 4, 4),
+	    [F_VBUS_INDO] = REG_FIELD(VBUS_IDD_STATUS, 0, 3),
+	    [F_VBUS_UCDSWEN] = REG_FIELD(VCC_UCD_FCTRL_SET, 10, 10),
+	    [F_VBUS_RREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 9, 9),
+	    [F_VBUS_DPPU_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 8, 8),
+	    [F_VBUS_DPREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 7, 7),
+	    [F_VBUS_DMREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 6, 6),
+	    [F_VBUS_DPDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 5, 5),
+	    [F_VBUS_DMDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 4, 4),
+	    [F_VBUS_DPSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 3, 3),
+	    [F_VBUS_DMSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 2, 2),
+	    [F_VBUS_DP_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 1, 1),
+	    [F_VBUS_DM_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 0, 0),
+
+	    [F_VBUS_EXTCLKENBL] = REG_FIELD(VBUS_UCD_FCTRL_EN, 15, 15),
+	    [F_VBUS_PLSTESTEN] = REG_FIELD(VBUS_UCD_FCTRL_EN, 14, 14),
+	    [F_VBUS_UCDSWEN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 10, 10),
+	    [F_VBUS_RREF_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 9, 9),
+	    [F_VBUS_DPPU_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 8, 8),
+	    [F_VBUS_DPREF_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 7, 7),
+	    [F_VBUS_DMREF_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 6, 6),
+	    [F_VBUS_DPDET_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 5, 5),
+	    [F_VBUS_DMDET_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 4, 4),
+	    [F_VBUS_DPSINK_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 3, 3),
+	    [F_VBUS_DMSINK_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 2, 2),
+	    [F_VBUS_DP_BUFF_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 1, 1),
+	    [F_VBUS_DM_BUFF_EN_TSTENB] = REG_FIELD(VBUS_UCD_FCTRL_EN, 0, 0),
+
+	    [F_CHIP_ID] = REG_FIELD(CHIP_ID, 0, 15),
+	    [F_CHIP_REV] = REG_FIELD(CHIP_REV, 0, 15),
+	    [F_ONE_CELL_MODE] = REG_FIELD(IC_SET1, 11, 11),
+	    [F_cell] = REG_FIELD(IC_SET1, 1, 1),
+	    [F_VACP_AUTO_DISCHG] = REG_FIELD(IC_SET1, 9, 9),
+	    [F_VACP_LOAD] = REG_FIELD(IC_SET1, 8, 8),
+	    [F_ACOK_POL] = REG_FIELD(IC_SET1, 1, 1),
+	    [F_ACOK_DISEN] = REG_FIELD(IC_SET1, 0, 0),
+	    [F_DEBUG_SET1] = REG_FIELD(IC_SET2, 4, 8),
+	    [F_DEBUG_SET0] = REG_FIELD(IC_SET2, 0, 0),
+	    [F_MONRST_STATE] = REG_FIELD(SYSTEM_STATUS, 6, 6),
+	    [F_ALMRST_STATE] = REG_FIELD(SYSTEM_STATUS, 5, 5),
+	    [F_CHGRST_STATE] = REG_FIELD(SYSTEM_STATUS, 4, 4),
+	    [F_OTPLD_STATE] = REG_FIELD(SYSTEM_STATUS, 1, 1),
+	    [F_ALLRST_STATE] = REG_FIELD(SYSTEM_STATUS, 0, 0),
+	    [F_PROTECT_SET] = REG_FIELD(PROTECT_SET, 0, 15),
+	    [F_MAP_SET] = REG_FIELD(MAP_SET, 0, 15),
+	    [F_ADCINTERVAL] = REG_FIELD(VM_CTRL_SET, 14, 15),
+	    [F_ADCMOD] = REG_FIELD(VM_CTRL_SET, 12, 13),
+	    [F_ADCTMOD] = REG_FIELD(VM_CTRL_SET, 10, 11),
+	    [F_EXTIADPEN] = REG_FIELD(VM_CTRL_SET, 9, 9),
+	    [F_VSYSENB] = REG_FIELD(VM_CTRL_SET, 8, 8),
+	    [F_VCCENB] = REG_FIELD(VM_CTRL_SET, 7, 7),
+	    [F_VBUSENB] = REG_FIELD(VM_CTRL_SET, 6, 6),
+	    [F_VACPENB] = REG_FIELD(VM_CTRL_SET, 5, 5),
+	    [F_IACPENB] = REG_FIELD(VM_CTRL_SET, 4, 4),
+	    [F_THERMENB] = REG_FIELD(VM_CTRL_SET, 3, 3),
+	    [F_VBATENB] = REG_FIELD(VM_CTRL_SET, 2, 2),
+	    [F_IBATMENB] = REG_FIELD(VM_CTRL_SET, 1, 1),
+	    [F_IBATPENB] = REG_FIELD(VM_CTRL_SET, 0, 0),
+	    [F_TMPTHR1B] = REG_FIELD(THERM_WINDOW_SET1, 8, 15),
+	    [F_TMPTHR1A] = REG_FIELD(THERM_WINDOW_SET1, 0, 7),
+	    [F_TMPTHR2B] = REG_FIELD(THERM_WINDOW_SET2, 8, 15),
+	    [F_TMPTHR2A] = REG_FIELD(THERM_WINDOW_SET2, 0, 7),
+	    [F_TMPTHR3B] = REG_FIELD(THERM_WINDOW_SET3, 8, 15),
+	    [F_TMPTHR3A] = REG_FIELD(THERM_WINDOW_SET3, 0, 7),
+	    [F_TMPTHR4B] = REG_FIELD(THERM_WINDOW_SET4, 8, 15),
+	    [F_TMPTHR4A] = REG_FIELD(THERM_WINDOW_SET4, 0, 7),
+	    [F_TMPTHR5B] = REG_FIELD(THERM_WINDOW_SET5, 8, 15),
+	    [F_TMPTHR5A] = REG_FIELD(THERM_WINDOW_SET5, 0, 7),
+	    [F_IBATP_TH_SET] = REG_FIELD(IBATP_TH_SET, 0, 14),
+	    [F_IBATM_TH_SET] = REG_FIELD(IBATM_TH_SET, 0, 14),
+	    [F_VBAT_TH_SET] = REG_FIELD(VBAT_TH_SET, 0, 14),
+	    [F_THERM_TH_SET] = REG_FIELD(THERM_TH_SET, 0, 7),
+	    [F_IACP_TH_SET] = REG_FIELD(IACP_TH_SET, 0, 14),
+	    [F_VACP_TH_SET] = REG_FIELD(VACP_TH_SET, 0, 14),
+	    [F_VBUS_TH_SET] = REG_FIELD(VBUS_TH_SET, 0, 14),
+	    [F_VCC_TH_SET] = REG_FIELD(VCC_TH_SET, 0, 14),
+	    [F_VSYS_TH_SET] = REG_FIELD(VSYS_TH_SET, 0, 14),
+	    [F_EXTIADP_TH_SET] = REG_FIELD(EXTIADP_TH_SET, 0, 11),
+	    [F_IBATP_VAL] = REG_FIELD(IBATP_VAL, 0, 14),
+	    [F_IBATP_AVE_VAL] = REG_FIELD(IBATP_AVE_VAL, 0, 14),
+	    [F_IBATM_VAL] = REG_FIELD(IBATM_VAL, 0, 14),
+	    [F_IBATM_AVE_VAL] = REG_FIELD(IBATM_AVE_VAL, 0, 14),
+	    [F_VBAT_VAL] = REG_FIELD(VBAT_VAL, 0, 14),
+	    [F_VBAT_AVE_VAL] = REG_FIELD(VBAT_AVE_VAL, 0, 14),
+	    [F_THERM_VAL] = REG_FIELD(THERM_VAL, 0, 7),
+	    [F_VTH_VAL] = REG_FIELD(VTH_VAL, 0, 11),
+	    [F_IACP_VAL] = REG_FIELD(IACP_VAL, 0, 14),
+	    [F_IACP_AVE_VAL] = REG_FIELD(IACP_AVE_VAL, 0, 14),
+	    [F_VACP_VAL] = REG_FIELD(VACP_VAL, 0, 14),
+	    [F_VACP_AVE_VAL] = REG_FIELD(VACP_AVE_VAL, 0, 14),
+	    [F_VBUS_VAL] = REG_FIELD(VBUS_VAL, 0, 14),
+	    [F_VBUS_AVE_VAL] = REG_FIELD(VBUS_AVE_VAL, 0, 14),
+	    [F_VCC_VAL] = REG_FIELD(VCC_VAL, 0, 14),
+	    [F_VCC_AVE_VAL] = REG_FIELD(VCC_AVE_VAL, 0, 14),
+	    [F_VSYS_VAL] = REG_FIELD(VSYS_VAL, 0, 14),
+	    [F_VSYS_AVE_VAL] = REG_FIELD(VSYS_AVE_VAL, 0, 14),
+	    [F_EXTIADP_VAL] = REG_FIELD(EXTIADP_VAL, 0, 11),
+	    [F_EXTIADP_AVE_VAL] = REG_FIELD(EXTIADP_AVE_VAL, 0, 11),
+	    [F_VACPCLPS_TH_SET] = REG_FIELD(VACPCLPS_TH_SET, 7, 14),
+	    [F_INT7_SET] = REG_FIELD(INT7_SET, 0, 15),
+	    [F_INT6_SET] = REG_FIELD(INT6_SET, 0, 13),
+	    [F_INT5_SET] = REG_FIELD(INT5_SET, 0, 13),
+	    [F_INT4_SET] = REG_FIELD(INT4_SET, 0, 9),
+	    [F_INT3_SET] = REG_FIELD(INT3_SET, 0, 15),
+	    [F_INT2_SET] = REG_FIELD(INT2_SET, 0, 15),
+	    [F_INT1_SET] = REG_FIELD(INT1_SET, 0, 15),
+	    [F_INT0_SET] = REG_FIELD(INT0_SET, 0, 7),
+	    [F_VBUS_RBUV_DET] = REG_FIELD(INT1_SET, 15, 15),
+	    [F_VBUS_RBUV_RES] = REG_FIELD(INT1_SET, 14, 14),
+	    [F_VBUS_TH_DET] = REG_FIELD(INT1_SET, 9, 9),
+	    [F_VBUS_TH_RES] = REG_FIELD(INT1_SET, 8, 8),
+	    [F_VBUS_IIN_MOD] = REG_FIELD(INT1_SET, 6, 6),
+	    [F_VBUS_OV_DET] = REG_FIELD(INT1_SET, 5, 5),
+	    [F_VBUS_OV_RES] = REG_FIELD(INT1_SET, 4, 4),
+	    [F_VBUS_CLPS_DET] = REG_FIELD(INT1_SET, 3, 3),
+	    [F_VBUS_CLPS] = REG_FIELD(INT1_SET, 2, 2),
+	    [F_VBUS_DET] = REG_FIELD(INT1_SET, 1, 1),
+	    [F_VBUS_RES] = REG_FIELD(INT1_SET, 0, 0),
+	    [F_VCC_RBUV_DET] = REG_FIELD(INT2_SET, 15, 15),
+	    [F_VCC_RBUV_RES] = REG_FIELD(INT2_SET, 14, 14),
+	    [F_VCC_TH_DET] = REG_FIELD(INT2_SET, 9, 9),
+	    [F_VCC_TH_RES] = REG_FIELD(INT2_SET, 8, 8),
+	    [F_VCC_IIN_MOD] = REG_FIELD(INT2_SET, 6, 6),
+	    [F_VCC_OVP_DET] = REG_FIELD(INT2_SET, 5, 5),
+	    [F_VCC_OVP_RES] = REG_FIELD(INT2_SET, 4, 4),
+	    [F_VCC_CLPS_DET] = REG_FIELD(INT2_SET, 3, 3),
+	    [F_VCC_CLPS_RES] = REG_FIELD(INT2_SET, 2, 2),
+	    [F_VCC_DET] = REG_FIELD(INT2_SET, 1, 1),
+	    [F_VCC_RES] = REG_FIELD(INT2_SET, 0, 0),
+	    [F_TH_DET] = REG_FIELD(INT3_SET, 15, 15),
+	    [F_TH_RMV] = REG_FIELD(INT3_SET, 14, 14),
+	    [F_TMP_OUT_DET] = REG_FIELD(INT3_SET, 11, 11),
+	    [F_TMP_OUT_RES] = REG_FIELD(INT3_SET, 10, 10),
+	    [F_VBAT_TH_DET] = REG_FIELD(INT3_SET, 9, 9),
+	    [F_VBAT_TH_RES] = REG_FIELD(INT3_SET, 8, 8),
+	    [F_IBAT_SHORT_DET] = REG_FIELD(INT3_SET, 7, 7),
+	    [F_IBAT_SHORT_RES] = REG_FIELD(INT3_SET, 6, 6),
+	    [F_VBAT_OV_DET] = REG_FIELD(INT3_SET, 5, 5),
+	    [F_VBAT_OV_RES] = REG_FIELD(INT3_SET, 4, 4),
+	    [F_BAT_ASSIST_DET] = REG_FIELD(INT3_SET, 3, 3),
+	    [F_BAT_ASSIST_RES] = REG_FIELD(INT3_SET, 2, 2),
+	    [F_VSYS_TH_DET] = REG_FIELD(INT4_SET, 9, 9),
+	    [F_VSYS_TH_RES] = REG_FIELD(INT4_SET, 8, 8),
+	    [F_VSYS_OV_DET] = REG_FIELD(INT4_SET, 5, 5),
+	    [F_VSYS_OV_RES] = REG_FIELD(INT4_SET, 4, 4),
+	    [F_VSYS_SHT_DET] = REG_FIELD(INT4_SET, 3, 3),
+	    [F_VSYS_SHT_RES] = REG_FIELD(INT4_SET, 2, 2),
+	    [F_VSYS_UV_DET] = REG_FIELD(INT4_SET, 1, 1),
+	    [F_VSYS_UV_RES] = REG_FIELD(INT4_SET, 0, 0),
+	    [F_OTP_LOAD_DONE] = REG_FIELD(INT5_SET, 13, 13),
+	    [F_PWR_ON] = REG_FIELD(INT5_SET, 12, 12),
+	    [F_EXTIADP_TRNS] = REG_FIELD(INT5_SET, 11, 11),
+	    [F_EXTIADP_TH_DET] = REG_FIELD(INT5_SET, 9, 9),
+	    [F_EXIADP_TH_RES] = REG_FIELD(INT5_SET, 8, 8),
+	    [F_BAT_MNT_DET] = REG_FIELD(INT5_SET, 7, 7),
+	    [F_BAT_MNT_RES] = REG_FIELD(INT5_SET, 6, 6),
+	    [F_TSD_DET] = REG_FIELD(INT5_SET, 5, 5),
+	    [F_TSD_RES] = REG_FIELD(INT5_SET, 4, 4),
+	    [F_CHGWDT_EXP] = REG_FIELD(INT5_SET, 3, 3),
+	    [F_THERMWDT_EXP] = REG_FIELD(INT5_SET, 2, 2),
+	    [F_TMP_TRNS] = REG_FIELD(INT5_SET, 1, 1),
+	    [F_CHG_TRNS] = REG_FIELD(INT5_SET, 0, 0),
+	    [F_VBUS_UCD_PORT_DET] = REG_FIELD(INT6_SET, 13, 13),
+	    [F_VBUS_UCD_UCHG_DET] = REG_FIELD(INT6_SET, 12, 12),
+	    [F_VBUS_UCD_URID_RMV] = REG_FIELD(INT6_SET, 11, 11),
+	    [F_VBUS_UCD_OTG_DET] = REG_FIELD(INT6_SET, 10, 10),
+	    [F_VBUS_UCD_URID_MOD] = REG_FIELD(INT6_SET, 8, 8),
+	    [F_VCC_UCD_PORT_DET] = REG_FIELD(INT6_SET, 5, 5),
+	    [F_VCC_UCD_UCHG_DET] = REG_FIELD(INT6_SET, 4, 4),
+	    [F_VCC_UCD_URID_RMV] = REG_FIELD(INT6_SET, 3, 3),
+	    [F_VCC_UCD_OTG_DET] = REG_FIELD(INT6_SET, 2, 2),
+	    [F_VCC_UCD_URID_MOD] = REG_FIELD(INT6_SET, 0, 0),
+	    [F_PROCHOT_DET] = REG_FIELD(INT7_SET, 15, 15),
+	    [F_PROCHOT_RES] = REG_FIELD(INT7_SET, 14, 14),
+	    [F_VACP_DET] = REG_FIELD(INT7_SET, 11, 11),
+	    [F_VACP_RES] = REG_FIELD(INT7_SET, 10, 10),
+	    [F_VACP_TH_DET] = REG_FIELD(INT7_SET, 9, 9),
+	    [F_VACP_TH_RES] = REG_FIELD(INT7_SET, 8, 8),
+	    [F_IACP_TH_DET] = REG_FIELD(INT7_SET, 7, 7),
+	    [F_IACP_THE_RES] = REG_FIELD(INT7_SET, 6, 6),
+	    [F_THERM_TH_DET] = REG_FIELD(INT7_SET, 5, 5),
+	    [F_THERM_TH_RES] = REG_FIELD(INT7_SET, 4, 4),
+	    [F_IBATM_TH_DET] = REG_FIELD(INT7_SET, 3, 3),
+	    [F_IBATM_TH_RES] = REG_FIELD(INT7_SET, 2, 2),
+	    [F_IBATP_TH_DET] = REG_FIELD(INT7_SET, 1, 1),
+	    [F_IBATP_TH_RES] = REG_FIELD(INT7_SET, 0, 0),
+	    [F_INT7_STATUS] = REG_FIELD(INT7_STATUS, 0, 15),
+	    [F_INT6_STATUS] = REG_FIELD(INT6_STATUS, 0, 13),
+	    [F_INT5_STATUS] = REG_FIELD(INT5_STATUS, 0, 13),
+	    [F_INT4_STATUS] = REG_FIELD(INT4_STATUS, 0, 9),
+	    [F_INT3_STATUS] = REG_FIELD(INT3_STATUS, 0, 15),
+	    [F_INT2_STATUS] = REG_FIELD(INT2_STATUS, 0, 15),
+	    [F_INT1_STATUS] = REG_FIELD(INT1_STATUS, 0, 15),
+	    [F_INT0_STATUS] = REG_FIELD(INT0_STATUS, 0, 7),
+	    [F_ILIM_DECREASE] = REG_FIELD(OTPREG0, 0, 15),
+	    [F_RESERVE_OTPREG1] = REG_FIELD(OTPREG1, 0, 15),
+	    [F_POWER_SAVE_MODE] = REG_FIELD(SMBREG, 0, 15),
+	    [F_DEBUG_MODE_SET] = REG_FIELD(DEBUG_MODE_SET, 0, 15),
+	    [F_DEBUG0x14] = REG_FIELD(DEBUG0x14, 0, 15),
+	    [F_DEBUG0x1A] = REG_FIELD(DEBUG0x1A, 0, 15),
+};
+
+/* CHGSTM_STATEs */
+#define CHGSTM_SUSPEND 0x00
+#define CHGSTM_TRICKLE_CHARGE 0x01
+#define CHGSTM_PRE_CHARGE 0x02
+#define CHGSTM_FAST_CHARGE 0x03
+#define CHGSTM_TOP_OFF 0x04
+#define CHGSTM_DONE 0x05
+#define CHGSTM_OTG 0x08
+#define CHGSTM_OTG_DONE 0x09
+#define CHGSTM_TEMPERATURE_ERROR_1 0x10
+#define CHGSTM_TEMPERATURE_ERROR_2 0x11
+#define CHGSTM_TEMPERATURE_ERROR_3 0x12
+#define CHGSTM_TEMPERATURE_ERROR_4 0x13
+#define CHGSTM_TEMPERATURE_ERROR_5 0x14
+#define CHGSTM_TEMPERATURE_ERROR_6 0x15
+#define CHGSTM_TEMPERATURE_ERROR_7 0x18
+#define CHGSTM_THERMAL_SHUT_DOWN_1 0x20
+#define CHGSTM_THERMAL_SHUT_DOWN_2 0x21
+#define CHGSTM_THERMAL_SHUT_DOWN_3 0x22
+#define CHGSTM_THERMAL_SHUT_DOWN_4 0x23
+#define CHGSTM_THERMAL_SHUT_DOWN_5 0x24
+#define CHGSTM_THERMAL_SHUT_DOWN_6 0x25
+#define CHGSTM_THERMAL_SHUT_DOWN_7 0x28
+#define CHGSTM_BATTERY_ERROR 0x40
+
+/* VBAT_VSYS_STATUS */
+#define STATUS_VSYS_OV BIT(15)
+#define STATUS_VSYS_SSD BIT(14)
+#define STATUS_VSYS_SCP BIT(13)
+#define STATUS_VSYS_UVN BIT(12)
+#define STATUS_IBAT_SHORT BIT(6)
+#define STATUS_VBAT_OV BIT(3)
+#define STATUS_DEAD_BAT BIT(0)
+
+/* VBUS_VCC_STATUS */
+#define STATUS_VACP_DET BIT(12)
+#define STATUS_VCC_OVP BIT(11)
+#define STATUS_ILIM_VCC_MOD BIT(10)
+#define STATUS_VCC_CLPS BIT(9)
+#define STATUS_VCC_DET BIT(8)
+#define STATUS_VBUS_OVP BIT(3)
+#define STATUS_ILIM_VBUS_MOD BIT(2)
+#define STATUS_VBUS_CLPS BIT(1)
+#define STATUS_VBUS_DET BIT(0)
+
+/* Interrupt set/status definitions */
+
+/* INT 0 */
+#define INT0_INT7_STATUS BIT(7)
+#define INT0_INT6_STATUS BIT(6)
+#define INT0_INT5_STATUS BIT(5)
+#define INT0_INT4_STATUS BIT(4)
+#define INT0_INT3_STATUS BIT(3)
+#define INT0_INT2_STATUS BIT(2)
+#define INT0_INT1_STATUS BIT(1)
+#define INT0_INT0_STATUS BIT(0)
+#define INT0_ALL 0xff
+
+/* INT 1 */
+#define VBUS_RBUV_DET BIT(15)
+#define VBUS_RBUV_RES BIT(14)
+#define VBUS_TH_DET BIT(9)
+#define VBUS_TH_RES BIT(8)
+#define VBUS_IIN_MOD BIT(6)
+#define VBUS_OV_DET BIT(5)
+#define VBUS_OV_RES BIT(4)
+#define VBUS_CLPS_DET BIT(3)
+#define VBUS_CLPS BIT(2)
+#define VBUS_DET BIT(1)
+#define VBUS_RES BIT(0)
+#define INT1_ALL (VBUS_RBUV_DET|\
+		 VBUS_RBUV_RES|\
+		 VBUS_TH_DET |\
+		 VBUS_TH_RES |\
+		 VBUS_IIN_MOD|\
+		 VBUS_OV_DET |\
+		 VBUS_OV_RES |\
+		 VBUS_CLPS_DET |\
+		 VBUS_CLPS |\
+		 VBUS_DET |\
+		 VBUS_RES)
+
+/* INT 2 */
+#define VCC_RBUV_DET BIT(15)
+#define VCC_RBUV_RES BIT(14)
+#define VCC_TH_DET BIT(9)
+#define VCC_TH_RES BIT(8)
+#define VCC_IIN_MOD BIT(6)
+#define VCC_OVP_DET BIT(5)
+#define VCC_OVP_RES BIT(4)
+#define VCC_CLPS_DET BIT(3)
+#define VCC_CLPS_RES BIT(2)
+#define VCC_DET BIT(1)
+#define VCC_RES BIT(0)
+#define INT2_ALL (VCC_RBUV_DET |\
+		 VCC_RBUV_RES |\
+		 VCC_TH_DET |\
+		 VCC_TH_RES |\
+		 VCC_IIN_MOD |\
+		 VCC_OVP_DET |\
+		 VCC_OVP_RES |\
+		 VCC_CLPS_DET |\
+		 VCC_CLPS_RES |\
+		 VCC_DET |\
+		 VCC_RES)
+/* INT 3 */
+#define TH_DET BIT(15)
+#define TH_RMV BIT(14)
+#define TMP_OUT_DET BIT(11)
+#define TMP_OUT_RES BIT(10)
+#define VBAT_TH_DET BIT(9)
+#define VBAT_TH_RES BIT(8)
+#define IBAT_SHORT_DET BIT(7)
+#define IBAT_SHORT_RES BIT(6)
+#define VBAT_OV_DET BIT(5)
+#define VBAT_OV_RES BIT(4)
+#define BAT_ASSIST_DET BIT(3)
+#define BAT_ASSIST_RES BIT(2)
+#define INT3_ALL (TH_DET |\
+		 TH_RMV |\
+		 TMP_OUT_DET |\
+		 TMP_OUT_RES |\
+		 VBAT_TH_DET |\
+		 VBAT_TH_RES |\
+		 IBAT_SHORT_DET |\
+		 IBAT_SHORT_RES |\
+		 VBAT_OV_DET |\
+		 VBAT_OV_RES |\
+		 BAT_ASSIST_DET |\
+		 BAT_ASSIST_RES)
+
+/* INT 4 */
+#define VSYS_TH_DET BIT(9)
+#define VSYS_TH_RES BIT(8)
+#define VSYS_OV_DET BIT(5)
+#define VSYS_OV_RES BIT(4)
+#define VSYS_SHT_DET BIT(3)
+#define VSYS_SHT_RES BIT(2)
+#define VSYS_UV_DET BIT(1)
+#define VSYS_UV_RES BIT(0)
+#define INT4_ALL (VSYS_TH_DET |\
+		 VSYS_TH_RES |\
+		 VSYS_OV_DET |\
+		 VSYS_OV_RES |\
+		 VSYS_SHT_DET |\
+		 VSYS_SHT_RES |\
+		 VSYS_UV_DET |\
+		 VSYS_UV_RES)
+
+/* INT 5*/
+#define OTP_LOAD_DONE BIT(13)
+#define PWR_ON BIT(12)
+#define EXTIADP_TRNS BIT(11)
+#define EXTIADP_TH_DET BIT(9)
+#define EXIADP_TH_RES BIT(8)
+#define BAT_MNT_DET BIT(7)
+#define BAT_MNT_RES BIT(6)
+#define TSD_DET BIT(5)
+#define TSD_RES BIT(4)
+#define CHGWDT_EXP BIT(3)
+#define THERMWDT_EXP BIT(2)
+#define TMP_TRNS BIT(1)
+#define CHG_TRNS BIT(0)
+#define INT5_ALL (OTP_LOAD_DONE |\
+		 PWR_ON |\
+		 EXTIADP_TRNS |\
+		 EXTIADP_TH_DET |\
+		 EXIADP_TH_RES |\
+		 BAT_MNT_DET |\
+		 BAT_MNT_RES |\
+		 TSD_DET |\
+		 TSD_RES |\
+		 CHGWDT_EXP |\
+		 THERMWDT_EXP |\
+		 TMP_TRNS |\
+		 CHG_TRNS)
+
+/* INT 6*/
+#define VBUS_UCD_PORT_DET BIT(13)
+#define VBUS_UCD_UCHG_DET BIT(12)
+#define VBUS_UCD_URID_RMV BIT(11)
+#define VBUS_UCD_OTG_DET BIT(10)
+#define VBUS_UCD_URID_MOD BIT(8)
+#define VCC_UCD_PORT_DET BIT(5)
+#define VCC_UCD_UCHG_DET BIT(4)
+#define VCC_UCD_URID_RMV BIT(3)
+#define VCC_UCD_OTG_DET BIT(2)
+#define VCC_UCD_URID_MOD BIT(0)
+#define INT6_ALL (VBUS_UCD_PORT_DET |\
+		 VBUS_UCD_UCHG_DET |\
+		 VBUS_UCD_URID_RMV |\
+		 VBUS_UCD_OTG_DET |\
+		 VBUS_UCD_URID_MOD |\
+		 VCC_UCD_PORT_DET |\
+		 VCC_UCD_UCHG_DET |\
+		 VCC_UCD_URID_RMV |\
+		 VCC_UCD_OTG_DET |\
+		 VCC_UCD_URID_MOD)
+
+/* INT 7 */
+#define PROCHOT_DET BIT(15)
+#define PROCHOT_RES BIT(14)
+#define VACP_DET BIT(11)
+#define VACP_RES BIT(10)
+#define VACP_TH_DET BIT(9)
+#define VACP_TH_RES BIT(8)
+#define IACP_TH_DET BIT(7)
+#define IACP_THE_RES BIT(6)
+#define THERM_TH_DET BIT(5)
+#define THERM_TH_RES BIT(4)
+#define IBATM_TH_DET BIT(3)
+#define IBATM_TH_RES BIT(2)
+#define IBATP_TH_DET BIT(1)
+#define IBATP_TH_RES BIT(0)
+#define INT7_ALL (PROCHOT_DET |\
+		 PROCHOT_RES |\
+		 VACP_DET |\
+		 VACP_RES |\
+		 VACP_TH_DET |\
+		 VACP_TH_RES |\
+		 IACP_TH_DET |\
+		 IACP_THE_RES |\
+		 THERM_TH_DET |\
+		 THERM_TH_RES |\
+		 IBATM_TH_DET |\
+		 IBATM_TH_RES |\
+		 IBATP_TH_DET |\
+		 IBATP_TH_RES)
+
+/* SYSTEM_CTRL_SET*/
+#define MONRST BIT(6)
+#define ALMRST BIT(5)
+#define CHGRST BIT(4)
+#define OTPLD  BIT(1)
+#define ALLRST BIT(0)
+
+/* F_BATTEMP */
+#define ROOM		0x0
+#define HOT1		0x1
+#define HOT2		0x2
+#define HOT3		0x3
+#define COLD1		0x4
+#define COLD2		0x5
+#define TEMP_DIS	0x6
+#define BATT_OPEN	0x7
+
+#endif
diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
index 453d6332d43a..4540e913057f 100644
--- a/drivers/power/supply/bq24190_charger.c
+++ b/drivers/power/supply/bq24190_charger.c
@@ -673,7 +673,7 @@ static int bq24190_register_reset(struct bq24190_dev_info *bdi)
 	 *   { .type = "bq24190", .addr = 0x6b, .properties = pe, .irq = irq };
 	 * struct i2c_adapter ad = { ... };
 	 * i2c_add_adapter(&ad);
-	 * i2c_new_device(&ad, &bi);
+	 * i2c_new_client_device(&ad, &bi);
 	 */
 	if (device_property_read_bool(bdi->dev, "disable-reset"))
 		return 0;
diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c
index aebd1253dbc9..77150667e36b 100644
--- a/drivers/power/supply/bq25890_charger.c
+++ b/drivers/power/supply/bq25890_charger.c
@@ -32,6 +32,13 @@ enum bq25890_chip_version {
 	BQ25896,
 };
 
+static const char *const bq25890_chip_name[] = {
+	"BQ25890",
+	"BQ25892",
+	"BQ25895",
+	"BQ25896",
+};
+
 enum bq25890_fields {
 	F_EN_HIZ, F_EN_ILIM, F_IILIM,				     /* Reg00 */
 	F_BHOT, F_BCOLD, F_VINDPM_OFS,				     /* Reg01 */
@@ -119,6 +126,7 @@ static const struct regmap_access_table bq25890_writeable_regs = {
 
 static const struct regmap_range bq25890_volatile_reg_ranges[] = {
 	regmap_reg_range(0x00, 0x00),
+	regmap_reg_range(0x02, 0x02),
 	regmap_reg_range(0x09, 0x09),
 	regmap_reg_range(0x0b, 0x14),
 };
@@ -246,6 +254,7 @@ enum bq25890_table_ids {
 	/* range tables */
 	TBL_ICHG,
 	TBL_ITERM,
+	TBL_IILIM,
 	TBL_VREG,
 	TBL_BOOSTV,
 	TBL_SYSVMIN,
@@ -286,6 +295,7 @@ static const union {
 	/* TODO: BQ25896 has max ICHG 3008 mA */
 	[TBL_ICHG] =	{ .rt = {0,	  5056000, 64000} },	 /* uA */
 	[TBL_ITERM] =	{ .rt = {64000,   1024000, 64000} },	 /* uA */
+	[TBL_IILIM] =   { .rt = {50000,   3200000, 50000} },	 /* uA */
 	[TBL_VREG] =	{ .rt = {3840000, 4608000, 16000} },	 /* uV */
 	[TBL_BOOSTV] =	{ .rt = {4550000, 5510000, 64000} },	 /* uV */
 	[TBL_SYSVMIN] = { .rt = {3000000, 3700000, 100000} },	 /* uV */
@@ -367,18 +377,42 @@ enum bq25890_chrg_fault {
 	CHRG_FAULT_TIMER_EXPIRED,
 };
 
+static bool bq25890_is_adc_property(enum power_supply_property psp)
+{
+	switch (psp) {
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+static irqreturn_t __bq25890_handle_irq(struct bq25890_device *bq);
+
 static int bq25890_power_supply_get_property(struct power_supply *psy,
 					     enum power_supply_property psp,
 					     union power_supply_propval *val)
 {
-	int ret;
 	struct bq25890_device *bq = power_supply_get_drvdata(psy);
 	struct bq25890_state state;
+	bool do_adc_conv;
+	int ret;
 
 	mutex_lock(&bq->lock);
+	/* update state in case we lost an interrupt */
+	__bq25890_handle_irq(bq);
 	state = bq->state;
+	do_adc_conv = !state.online && bq25890_is_adc_property(psp);
+	if (do_adc_conv)
+		bq25890_field_write(bq, F_CONV_START, 1);
 	mutex_unlock(&bq->lock);
 
+	if (do_adc_conv)
+		regmap_field_read_poll_timeout(bq->rmap_fields[F_CONV_START],
+			ret, !ret, 25000, 1000000);
+
 	switch (psp) {
 	case POWER_SUPPLY_PROP_STATUS:
 		if (!state.online)
@@ -395,22 +429,24 @@ static int bq25890_power_supply_get_property(struct power_supply *psy,
 
 		break;
 
+	case POWER_SUPPLY_PROP_CHARGE_TYPE:
+		if (!state.online || state.chrg_status == STATUS_NOT_CHARGING ||
+		    state.chrg_status == STATUS_TERMINATION_DONE)
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_NONE;
+		else if (state.chrg_status == STATUS_PRE_CHARGING)
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_STANDARD;
+		else if (state.chrg_status == STATUS_FAST_CHARGING)
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_FAST;
+		else /* unreachable */
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_UNKNOWN;
+		break;
+
 	case POWER_SUPPLY_PROP_MANUFACTURER:
 		val->strval = BQ25890_MANUFACTURER;
 		break;
 
 	case POWER_SUPPLY_PROP_MODEL_NAME:
-		if (bq->chip_version == BQ25890)
-			val->strval = "BQ25890";
-		else if (bq->chip_version == BQ25892)
-			val->strval = "BQ25892";
-		else if (bq->chip_version == BQ25895)
-			val->strval = "BQ25895";
-		else if (bq->chip_version == BQ25896)
-			val->strval = "BQ25896";
-		else
-			val->strval = "UNKNOWN";
-
+		val->strval = bq25890_chip_name[bq->chip_version];
 		break;
 
 	case POWER_SUPPLY_PROP_ONLINE:
@@ -430,15 +466,6 @@ static int bq25890_power_supply_get_property(struct power_supply *psy,
 			val->intval = POWER_SUPPLY_HEALTH_UNSPEC_FAILURE;
 		break;
 
-	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
-		ret = bq25890_field_read(bq, F_ICHGR); /* read measured value */
-		if (ret < 0)
-			return ret;
-
-		/* converted_val = ADC_val * 50mA (table 10.3.19) */
-		val->intval = ret * 50000;
-		break;
-
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX:
 		val->intval = bq25890_find_val(bq->init_data.ichg, TBL_ICHG);
 		break;
@@ -461,10 +488,22 @@ static int bq25890_power_supply_get_property(struct power_supply *psy,
 		val->intval = bq25890_find_val(bq->init_data.vreg, TBL_VREG);
 		break;
 
+	case POWER_SUPPLY_PROP_PRECHARGE_CURRENT:
+		val->intval = bq25890_find_val(bq->init_data.iprechg, TBL_ITERM);
+		break;
+
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
 		val->intval = bq25890_find_val(bq->init_data.iterm, TBL_ITERM);
 		break;
 
+	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+		ret = bq25890_field_read(bq, F_IILIM);
+		if (ret < 0)
+			return ret;
+
+		val->intval = bq25890_find_val(ret, TBL_IILIM);
+		break;
+
 	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
 		ret = bq25890_field_read(bq, F_SYSV); /* read measured value */
 		if (ret < 0)
@@ -474,6 +513,15 @@ static int bq25890_power_supply_get_property(struct power_supply *psy,
 		val->intval = 2304000 + ret * 20000;
 		break;
 
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		ret = bq25890_field_read(bq, F_ICHGR); /* read measured value */
+		if (ret < 0)
+			return ret;
+
+		/* converted_val = ADC_val * 50mA (table 10.3.19) */
+		val->intval = ret * -50000;
+		break;
+
 	default:
 		return -EINVAL;
 	}
@@ -513,74 +561,50 @@ static int bq25890_get_chip_state(struct bq25890_device *bq,
 	return 0;
 }
 
-static bool bq25890_state_changed(struct bq25890_device *bq,
-				  struct bq25890_state *new_state)
-{
-	struct bq25890_state old_state;
-
-	mutex_lock(&bq->lock);
-	old_state = bq->state;
-	mutex_unlock(&bq->lock);
-
-	return (old_state.chrg_status != new_state->chrg_status ||
-		old_state.chrg_fault != new_state->chrg_fault	||
-		old_state.online != new_state->online		||
-		old_state.bat_fault != new_state->bat_fault	||
-		old_state.boost_fault != new_state->boost_fault ||
-		old_state.vsys_status != new_state->vsys_status);
-}
-
-static void bq25890_handle_state_change(struct bq25890_device *bq,
-					struct bq25890_state *new_state)
+static irqreturn_t __bq25890_handle_irq(struct bq25890_device *bq)
 {
+	struct bq25890_state new_state;
 	int ret;
-	struct bq25890_state old_state;
 
-	mutex_lock(&bq->lock);
-	old_state = bq->state;
-	mutex_unlock(&bq->lock);
+	ret = bq25890_get_chip_state(bq, &new_state);
+	if (ret < 0)
+		return IRQ_NONE;
+
+	if (!memcmp(&bq->state, &new_state, sizeof(new_state)))
+		return IRQ_NONE;
 
-	if (!new_state->online) {			     /* power removed */
+	if (!new_state.online && bq->state.online) {	    /* power removed */
 		/* disable ADC */
 		ret = bq25890_field_write(bq, F_CONV_START, 0);
 		if (ret < 0)
 			goto error;
-	} else if (!old_state.online) {			    /* power inserted */
+	} else if (new_state.online && !bq->state.online) { /* power inserted */
 		/* enable ADC, to have control of charge current/voltage */
 		ret = bq25890_field_write(bq, F_CONV_START, 1);
 		if (ret < 0)
 			goto error;
 	}
 
-	return;
+	bq->state = new_state;
+	power_supply_changed(bq->charger);
 
+	return IRQ_HANDLED;
 error:
-	dev_err(bq->dev, "Error communicating with the chip.\n");
+	dev_err(bq->dev, "Error communicating with the chip: %pe\n",
+		ERR_PTR(ret));
+	return IRQ_HANDLED;
 }
 
 static irqreturn_t bq25890_irq_handler_thread(int irq, void *private)
 {
 	struct bq25890_device *bq = private;
-	int ret;
-	struct bq25890_state state;
-
-	ret = bq25890_get_chip_state(bq, &state);
-	if (ret < 0)
-		goto handled;
-
-	if (!bq25890_state_changed(bq, &state))
-		goto handled;
-
-	bq25890_handle_state_change(bq, &state);
+	irqreturn_t ret;
 
 	mutex_lock(&bq->lock);
-	bq->state = state;
+	ret = __bq25890_handle_irq(bq);
 	mutex_unlock(&bq->lock);
 
-	power_supply_changed(bq->charger);
-
-handled:
-	return IRQ_HANDLED;
+	return ret;
 }
 
 static int bq25890_chip_reset(struct bq25890_device *bq)
@@ -610,7 +634,6 @@ static int bq25890_hw_init(struct bq25890_device *bq)
 {
 	int ret;
 	int i;
-	struct bq25890_state state;
 
 	const struct {
 		enum bq25890_fields id;
@@ -651,38 +674,37 @@ static int bq25890_hw_init(struct bq25890_device *bq)
 		}
 	}
 
-	/* Configure ADC for continuous conversions. This does not enable it. */
-	ret = bq25890_field_write(bq, F_CONV_RATE, 1);
+	/* Configure ADC for continuous conversions when charging */
+	ret = bq25890_field_write(bq, F_CONV_RATE, !!bq->state.online);
 	if (ret < 0) {
 		dev_dbg(bq->dev, "Config ADC failed %d\n", ret);
 		return ret;
 	}
 
-	ret = bq25890_get_chip_state(bq, &state);
+	ret = bq25890_get_chip_state(bq, &bq->state);
 	if (ret < 0) {
 		dev_dbg(bq->dev, "Get state failed %d\n", ret);
 		return ret;
 	}
 
-	mutex_lock(&bq->lock);
-	bq->state = state;
-	mutex_unlock(&bq->lock);
-
 	return 0;
 }
 
-static enum power_supply_property bq25890_power_supply_props[] = {
+static const enum power_supply_property bq25890_power_supply_props[] = {
 	POWER_SUPPLY_PROP_MANUFACTURER,
 	POWER_SUPPLY_PROP_MODEL_NAME,
 	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_CHARGE_TYPE,
 	POWER_SUPPLY_PROP_ONLINE,
 	POWER_SUPPLY_PROP_HEALTH,
-	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX,
+	POWER_SUPPLY_PROP_PRECHARGE_CURRENT,
 	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
+	POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
 };
 
 static char *bq25890_charger_supplied_to[] = {
@@ -881,17 +903,11 @@ static int bq25890_fw_probe(struct bq25890_device *bq)
 static int bq25890_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
-	struct i2c_adapter *adapter = client->adapter;
 	struct device *dev = &client->dev;
 	struct bq25890_device *bq;
 	int ret;
 	int i;
 
-	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA)) {
-		dev_err(dev, "No support for SMBUS_BYTE_DATA\n");
-		return -ENODEV;
-	}
-
 	bq = devm_kzalloc(dev, sizeof(*bq), GFP_KERNEL);
 	if (!bq)
 		return -ENOMEM;
@@ -1004,34 +1020,34 @@ static int bq25890_suspend(struct device *dev)
 	 * If charger is removed, while in suspend, make sure ADC is diabled
 	 * since it consumes slightly more power.
 	 */
-	return bq25890_field_write(bq, F_CONV_START, 0);
+	return bq25890_field_write(bq, F_CONV_RATE, 0);
 }
 
 static int bq25890_resume(struct device *dev)
 {
 	int ret;
-	struct bq25890_state state;
 	struct bq25890_device *bq = dev_get_drvdata(dev);
 
-	ret = bq25890_get_chip_state(bq, &state);
-	if (ret < 0)
-		return ret;
-
 	mutex_lock(&bq->lock);
-	bq->state = state;
-	mutex_unlock(&bq->lock);
+
+	ret = bq25890_get_chip_state(bq, &bq->state);
+	if (ret < 0)
+		goto unlock;
 
 	/* Re-enable ADC only if charger is plugged in. */
-	if (state.online) {
-		ret = bq25890_field_write(bq, F_CONV_START, 1);
+	if (bq->state.online) {
+		ret = bq25890_field_write(bq, F_CONV_RATE, 1);
 		if (ret < 0)
-			return ret;
+			goto unlock;
 	}
 
 	/* signal userspace, maybe state changed while suspended */
 	power_supply_changed(bq->charger);
 
-	return 0;
+unlock:
+	mutex_unlock(&bq->lock);
+
+	return ret;
 }
 #endif
 
diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c
index a21e1a2673f8..2ef53dc1f2fb 100644
--- a/drivers/power/supply/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
@@ -1422,7 +1422,9 @@ static int charger_manager_prepare_sysfs(struct charger_manager *cm)
 }
 
 static int cm_init_thermal_data(struct charger_manager *cm,
-		struct power_supply *fuel_gauge)
+		struct power_supply *fuel_gauge,
+		enum power_supply_property *properties,
+		size_t *num_properties)
 {
 	struct charger_desc *desc = cm->desc;
 	union power_supply_propval val;
@@ -1433,9 +1435,8 @@ static int cm_init_thermal_data(struct charger_manager *cm,
 					POWER_SUPPLY_PROP_TEMP, &val);
 
 	if (!ret) {
-		cm->charger_psy_desc.properties[cm->charger_psy_desc.num_properties] =
-				POWER_SUPPLY_PROP_TEMP;
-		cm->charger_psy_desc.num_properties++;
+		properties[*num_properties] = POWER_SUPPLY_PROP_TEMP;
+		(*num_properties)++;
 		cm->desc->measure_battery_temp = true;
 	}
 #ifdef CONFIG_THERMAL
@@ -1446,9 +1447,8 @@ static int cm_init_thermal_data(struct charger_manager *cm,
 			return PTR_ERR(cm->tzd_batt);
 
 		/* Use external thermometer */
-		cm->charger_psy_desc.properties[cm->charger_psy_desc.num_properties] =
-				POWER_SUPPLY_PROP_TEMP_AMBIENT;
-		cm->charger_psy_desc.num_properties++;
+		properties[*num_properties] = POWER_SUPPLY_PROP_TEMP_AMBIENT;
+		(*num_properties)++;
 		cm->desc->measure_battery_temp = true;
 		ret = 0;
 	}
@@ -1621,6 +1621,8 @@ static int charger_manager_probe(struct platform_device *pdev)
 	int j = 0;
 	union power_supply_propval val;
 	struct power_supply *fuel_gauge;
+	enum power_supply_property *properties;
+	size_t num_properties;
 	struct power_supply_config psy_cfg = {};
 
 	if (IS_ERR(desc)) {
@@ -1717,18 +1719,17 @@ static int charger_manager_probe(struct platform_device *pdev)
 	cm->charger_psy_desc.name = cm->psy_name_buf;
 
 	/* Allocate for psy properties because they may vary */
-	cm->charger_psy_desc.properties =
-		devm_kcalloc(&pdev->dev,
+	properties = devm_kcalloc(&pdev->dev,
 			     ARRAY_SIZE(default_charger_props) +
 				NUM_CHARGER_PSY_OPTIONAL,
-			     sizeof(enum power_supply_property), GFP_KERNEL);
-	if (!cm->charger_psy_desc.properties)
+			     sizeof(*properties), GFP_KERNEL);
+	if (!properties)
 		return -ENOMEM;
 
-	memcpy(cm->charger_psy_desc.properties, default_charger_props,
+	memcpy(properties, default_charger_props,
 		sizeof(enum power_supply_property) *
 		ARRAY_SIZE(default_charger_props));
-	cm->charger_psy_desc.num_properties = psy_default.num_properties;
+	num_properties = ARRAY_SIZE(default_charger_props);
 
 	/* Find which optional psy-properties are available */
 	fuel_gauge = power_supply_get_by_name(desc->psy_fuel_gauge);
@@ -1739,25 +1740,28 @@ static int charger_manager_probe(struct platform_device *pdev)
 	}
 	if (!power_supply_get_property(fuel_gauge,
 					  POWER_SUPPLY_PROP_CHARGE_NOW, &val)) {
-		cm->charger_psy_desc.properties[cm->charger_psy_desc.num_properties] =
+		properties[num_properties] =
 				POWER_SUPPLY_PROP_CHARGE_NOW;
-		cm->charger_psy_desc.num_properties++;
+		num_properties++;
 	}
 	if (!power_supply_get_property(fuel_gauge,
 					  POWER_SUPPLY_PROP_CURRENT_NOW,
 					  &val)) {
-		cm->charger_psy_desc.properties[cm->charger_psy_desc.num_properties] =
+		properties[num_properties] =
 				POWER_SUPPLY_PROP_CURRENT_NOW;
-		cm->charger_psy_desc.num_properties++;
+		num_properties++;
 	}
 
-	ret = cm_init_thermal_data(cm, fuel_gauge);
+	ret = cm_init_thermal_data(cm, fuel_gauge, properties, &num_properties);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to initialize thermal data\n");
 		cm->desc->measure_battery_temp = false;
 	}
 	power_supply_put(fuel_gauge);
 
+	cm->charger_psy_desc.properties = properties;
+	cm->charger_psy_desc.num_properties = num_properties;
+
 	INIT_DELAYED_WORK(&cm->fullbatt_vchk_work, fullbatt_vchk);
 
 	/* Register sysfs entry for charger(regulator) */
diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c
new file mode 100644
index 000000000000..0146f1bfc29b
--- /dev/null
+++ b/drivers/power/supply/cw2015_battery.c
@@ -0,0 +1,750 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Fuel gauge driver for CellWise 2013 / 2015
+ *
+ * Copyright (C) 2012, RockChip
+ * Copyright (C) 2020, Tobias Schramm
+ *
+ * Authors: xuhuicong <xhc@rock-chips.com>
+ * Authors: Tobias Schramm <t.schramm@manjaro.org>
+ */
+
+#include <linux/bits.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/gfp.h>
+#include <linux/gpio/consumer.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/power_supply.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/time.h>
+#include <linux/workqueue.h>
+
+#define CW2015_SIZE_BATINFO		64
+
+#define CW2015_RESET_TRIES		5
+
+#define CW2015_REG_VERSION		0x00
+#define CW2015_REG_VCELL		0x02
+#define CW2015_REG_SOC			0x04
+#define CW2015_REG_RRT_ALERT		0x06
+#define CW2015_REG_CONFIG		0x08
+#define CW2015_REG_MODE			0x0A
+#define CW2015_REG_BATINFO		0x10
+
+#define CW2015_MODE_SLEEP_MASK		GENMASK(7, 6)
+#define CW2015_MODE_SLEEP		(0x03 << 6)
+#define CW2015_MODE_NORMAL		(0x00 << 6)
+#define CW2015_MODE_QUICK_START		(0x03 << 4)
+#define CW2015_MODE_RESTART		(0x0f << 0)
+
+#define CW2015_CONFIG_UPDATE_FLG	(0x01 << 1)
+#define CW2015_ATHD(x)			((x) << 3)
+#define CW2015_MASK_ATHD		GENMASK(7, 3)
+#define CW2015_MASK_SOC			GENMASK(12, 0)
+
+/* reset gauge of no valid state of charge could be polled for 40s */
+#define CW2015_BAT_SOC_ERROR_MS		(40 * MSEC_PER_SEC)
+/* reset gauge if state of charge stuck for half an hour during charging */
+#define CW2015_BAT_CHARGING_STUCK_MS	(1800 * MSEC_PER_SEC)
+
+/* poll interval from CellWise GPL Android driver example */
+#define CW2015_DEFAULT_POLL_INTERVAL_MS		8000
+
+#define CW2015_AVERAGING_SAMPLES		3
+
+struct cw_battery {
+	struct device *dev;
+	struct workqueue_struct *battery_workqueue;
+	struct delayed_work battery_delay_work;
+	struct regmap *regmap;
+	struct power_supply *rk_bat;
+	struct power_supply_battery_info battery;
+	u8 *bat_profile;
+
+	bool charger_attached;
+	bool battery_changed;
+
+	int soc;
+	int voltage_mv;
+	int status;
+	int time_to_empty;
+	int charge_count;
+
+	u32 poll_interval_ms;
+	u8 alert_level;
+
+	unsigned int read_errors;
+	unsigned int charge_stuck_cnt;
+};
+
+static int cw_read_word(struct cw_battery *cw_bat, u8 reg, u16 *val)
+{
+	__be16 value;
+	int ret;
+
+	ret = regmap_bulk_read(cw_bat->regmap, reg, &value, sizeof(value));
+	if (ret)
+		return ret;
+
+	*val = be16_to_cpu(value);
+	return 0;
+}
+
+static int cw_update_profile(struct cw_battery *cw_bat)
+{
+	int ret;
+	unsigned int reg_val;
+	u8 reset_val;
+
+	/* make sure gauge is not in sleep mode */
+	ret = regmap_read(cw_bat->regmap, CW2015_REG_MODE, &reg_val);
+	if (ret)
+		return ret;
+
+	reset_val = reg_val;
+	if ((reg_val & CW2015_MODE_SLEEP_MASK) == CW2015_MODE_SLEEP) {
+		dev_err(cw_bat->dev,
+			"Gauge is in sleep mode, can't update battery info\n");
+		return -EINVAL;
+	}
+
+	/* write new battery info */
+	ret = regmap_raw_write(cw_bat->regmap, CW2015_REG_BATINFO,
+			       cw_bat->bat_profile,
+			       CW2015_SIZE_BATINFO);
+	if (ret)
+		return ret;
+
+	/* set config update flag  */
+	reg_val |= CW2015_CONFIG_UPDATE_FLG;
+	reg_val &= ~CW2015_MASK_ATHD;
+	reg_val |= CW2015_ATHD(cw_bat->alert_level);
+	ret = regmap_write(cw_bat->regmap, CW2015_REG_CONFIG, reg_val);
+	if (ret)
+		return ret;
+
+	/* reset gauge to apply new battery profile */
+	reset_val &= ~CW2015_MODE_RESTART;
+	reg_val = reset_val | CW2015_MODE_RESTART;
+	ret = regmap_write(cw_bat->regmap, CW2015_REG_MODE, reg_val);
+	if (ret)
+		return ret;
+
+	/* wait for gauge to reset */
+	msleep(20);
+
+	/* clear reset flag */
+	ret = regmap_write(cw_bat->regmap, CW2015_REG_MODE, reset_val);
+	if (ret)
+		return ret;
+
+	/* wait for gauge to become ready */
+	ret = regmap_read_poll_timeout(cw_bat->regmap, CW2015_REG_SOC,
+				       reg_val, reg_val <= 100,
+				       10 * USEC_PER_MSEC, 10 * USEC_PER_SEC);
+	if (ret)
+		dev_err(cw_bat->dev,
+			"Gauge did not become ready after profile upload\n");
+	else
+		dev_dbg(cw_bat->dev, "Battery profile updated\n");
+
+	return ret;
+}
+
+static int cw_init(struct cw_battery *cw_bat)
+{
+	int ret;
+	unsigned int reg_val = CW2015_MODE_SLEEP;
+
+	if ((reg_val & CW2015_MODE_SLEEP_MASK) == CW2015_MODE_SLEEP) {
+		reg_val = CW2015_MODE_NORMAL;
+		ret = regmap_write(cw_bat->regmap, CW2015_REG_MODE, reg_val);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_read(cw_bat->regmap, CW2015_REG_CONFIG, &reg_val);
+	if (ret)
+		return ret;
+
+	if ((reg_val & CW2015_MASK_ATHD) != CW2015_ATHD(cw_bat->alert_level)) {
+		dev_dbg(cw_bat->dev, "Setting new alert level\n");
+		reg_val &= ~CW2015_MASK_ATHD;
+		reg_val |= ~CW2015_ATHD(cw_bat->alert_level);
+		ret = regmap_write(cw_bat->regmap, CW2015_REG_CONFIG, reg_val);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_read(cw_bat->regmap, CW2015_REG_CONFIG, &reg_val);
+	if (ret)
+		return ret;
+
+	if (!(reg_val & CW2015_CONFIG_UPDATE_FLG)) {
+		dev_dbg(cw_bat->dev,
+			"Battery profile not present, uploading battery profile\n");
+		if (cw_bat->bat_profile) {
+			ret = cw_update_profile(cw_bat);
+			if (ret) {
+				dev_err(cw_bat->dev,
+					"Failed to upload battery profile\n");
+				return ret;
+			}
+		} else {
+			dev_warn(cw_bat->dev,
+				 "No profile specified, continuing without profile\n");
+		}
+	} else if (cw_bat->bat_profile) {
+		u8 bat_info[CW2015_SIZE_BATINFO];
+
+		ret = regmap_raw_read(cw_bat->regmap, CW2015_REG_BATINFO,
+				      bat_info, CW2015_SIZE_BATINFO);
+		if (ret) {
+			dev_err(cw_bat->dev,
+				"Failed to read stored battery profile\n");
+			return ret;
+		}
+
+		if (memcmp(bat_info, cw_bat->bat_profile, CW2015_SIZE_BATINFO)) {
+			dev_warn(cw_bat->dev, "Replacing stored battery profile\n");
+			ret = cw_update_profile(cw_bat);
+			if (ret)
+				return ret;
+		}
+	} else {
+		dev_warn(cw_bat->dev,
+			 "Can't check current battery profile, no profile provided\n");
+	}
+
+	dev_dbg(cw_bat->dev, "Battery profile configured\n");
+	return 0;
+}
+
+static int cw_power_on_reset(struct cw_battery *cw_bat)
+{
+	int ret;
+	unsigned char reset_val;
+
+	reset_val = CW2015_MODE_SLEEP;
+	ret = regmap_write(cw_bat->regmap, CW2015_REG_MODE, reset_val);
+	if (ret)
+		return ret;
+
+	/* wait for gauge to enter sleep */
+	msleep(20);
+
+	reset_val = CW2015_MODE_NORMAL;
+	ret = regmap_write(cw_bat->regmap, CW2015_REG_MODE, reset_val);
+	if (ret)
+		return ret;
+
+	ret = cw_init(cw_bat);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+#define HYSTERESIS(current, previous, up, down) \
+	(((current) < (previous) + (up)) && ((current) > (previous) - (down)))
+
+static int cw_get_soc(struct cw_battery *cw_bat)
+{
+	unsigned int soc;
+	int ret;
+
+	ret = regmap_read(cw_bat->regmap, CW2015_REG_SOC, &soc);
+	if (ret)
+		return ret;
+
+	if (soc > 100) {
+		int max_error_cycles =
+			CW2015_BAT_SOC_ERROR_MS / cw_bat->poll_interval_ms;
+
+		dev_err(cw_bat->dev, "Invalid SoC %d%%\n", soc);
+		cw_bat->read_errors++;
+		if (cw_bat->read_errors > max_error_cycles) {
+			dev_warn(cw_bat->dev,
+				 "Too many invalid SoC reports, resetting gauge\n");
+			cw_power_on_reset(cw_bat);
+			cw_bat->read_errors = 0;
+		}
+		return cw_bat->soc;
+	}
+	cw_bat->read_errors = 0;
+
+	/* Reset gauge if stuck while charging */
+	if (cw_bat->status == POWER_SUPPLY_STATUS_CHARGING && soc == cw_bat->soc) {
+		int max_stuck_cycles =
+			CW2015_BAT_CHARGING_STUCK_MS / cw_bat->poll_interval_ms;
+
+		cw_bat->charge_stuck_cnt++;
+		if (cw_bat->charge_stuck_cnt > max_stuck_cycles) {
+			dev_warn(cw_bat->dev,
+				 "SoC stuck @%u%%, resetting gauge\n", soc);
+			cw_power_on_reset(cw_bat);
+			cw_bat->charge_stuck_cnt = 0;
+		}
+	} else {
+		cw_bat->charge_stuck_cnt = 0;
+	}
+
+	/* Ignore voltage dips during charge */
+	if (cw_bat->charger_attached && HYSTERESIS(soc, cw_bat->soc, 0, 3))
+		soc = cw_bat->soc;
+
+	/* Ignore voltage spikes during discharge */
+	if (!cw_bat->charger_attached && HYSTERESIS(soc, cw_bat->soc, 3, 0))
+		soc = cw_bat->soc;
+
+	return soc;
+}
+
+static int cw_get_voltage(struct cw_battery *cw_bat)
+{
+	int ret, i, voltage_mv;
+	u16 reg_val;
+	u32 avg = 0;
+
+	for (i = 0; i < CW2015_AVERAGING_SAMPLES; i++) {
+		ret = cw_read_word(cw_bat, CW2015_REG_VCELL, &reg_val);
+		if (ret)
+			return ret;
+
+		avg += reg_val;
+	}
+	avg /= CW2015_AVERAGING_SAMPLES;
+
+	/*
+	 * 305 uV per ADC step
+	 * Use 312 / 1024  as efficient approximation of 305 / 1000
+	 * Negligible error of 0.1%
+	 */
+	voltage_mv = avg * 312 / 1024;
+
+	dev_dbg(cw_bat->dev, "Read voltage: %d mV, raw=0x%04x\n",
+		voltage_mv, reg_val);
+	return voltage_mv;
+}
+
+static int cw_get_time_to_empty(struct cw_battery *cw_bat)
+{
+	int ret;
+	u16 value16;
+
+	ret = cw_read_word(cw_bat, CW2015_REG_RRT_ALERT, &value16);
+	if (ret)
+		return ret;
+
+	return value16 & CW2015_MASK_SOC;
+}
+
+static void cw_update_charge_status(struct cw_battery *cw_bat)
+{
+	int ret;
+
+	ret = power_supply_am_i_supplied(cw_bat->rk_bat);
+	if (ret < 0) {
+		dev_warn(cw_bat->dev, "Failed to get supply state: %d\n", ret);
+	} else {
+		bool charger_attached;
+
+		charger_attached = !!ret;
+		if (cw_bat->charger_attached != charger_attached) {
+			cw_bat->battery_changed = true;
+			if (charger_attached)
+				cw_bat->charge_count++;
+		}
+		cw_bat->charger_attached = charger_attached;
+	}
+}
+
+static void cw_update_soc(struct cw_battery *cw_bat)
+{
+	int soc;
+
+	soc = cw_get_soc(cw_bat);
+	if (soc < 0)
+		dev_err(cw_bat->dev, "Failed to get SoC from gauge: %d\n", soc);
+	else if (cw_bat->soc != soc) {
+		cw_bat->soc = soc;
+		cw_bat->battery_changed = true;
+	}
+}
+
+static void cw_update_voltage(struct cw_battery *cw_bat)
+{
+	int voltage_mv;
+
+	voltage_mv = cw_get_voltage(cw_bat);
+	if (voltage_mv < 0)
+		dev_err(cw_bat->dev, "Failed to get voltage from gauge: %d\n",
+			voltage_mv);
+	else
+		cw_bat->voltage_mv = voltage_mv;
+}
+
+static void cw_update_status(struct cw_battery *cw_bat)
+{
+	int status = POWER_SUPPLY_STATUS_DISCHARGING;
+
+	if (cw_bat->charger_attached) {
+		if (cw_bat->soc >= 100)
+			status = POWER_SUPPLY_STATUS_FULL;
+		else
+			status = POWER_SUPPLY_STATUS_CHARGING;
+	}
+
+	if (cw_bat->status != status)
+		cw_bat->battery_changed = true;
+	cw_bat->status = status;
+}
+
+static void cw_update_time_to_empty(struct cw_battery *cw_bat)
+{
+	int time_to_empty;
+
+	time_to_empty = cw_get_time_to_empty(cw_bat);
+	if (time_to_empty < 0)
+		dev_err(cw_bat->dev, "Failed to get time to empty from gauge: %d\n",
+			time_to_empty);
+	else if (cw_bat->time_to_empty != time_to_empty) {
+		cw_bat->time_to_empty = time_to_empty;
+		cw_bat->battery_changed = true;
+	}
+}
+
+static void cw_bat_work(struct work_struct *work)
+{
+	struct delayed_work *delay_work;
+	struct cw_battery *cw_bat;
+	int ret;
+	unsigned int reg_val;
+
+	delay_work = to_delayed_work(work);
+	cw_bat = container_of(delay_work, struct cw_battery, battery_delay_work);
+	ret = regmap_read(cw_bat->regmap, CW2015_REG_MODE, &reg_val);
+	if (ret) {
+		dev_err(cw_bat->dev, "Failed to read mode from gauge: %d\n", ret);
+	} else {
+		if ((reg_val & CW2015_MODE_SLEEP_MASK) == CW2015_MODE_SLEEP) {
+			int i;
+
+			for (i = 0; i < CW2015_RESET_TRIES; i++) {
+				if (!cw_power_on_reset(cw_bat))
+					break;
+			}
+		}
+		cw_update_soc(cw_bat);
+		cw_update_voltage(cw_bat);
+		cw_update_charge_status(cw_bat);
+		cw_update_status(cw_bat);
+		cw_update_time_to_empty(cw_bat);
+	}
+	dev_dbg(cw_bat->dev, "charger_attached = %d\n", cw_bat->charger_attached);
+	dev_dbg(cw_bat->dev, "status = %d\n", cw_bat->status);
+	dev_dbg(cw_bat->dev, "soc = %d%%\n", cw_bat->soc);
+	dev_dbg(cw_bat->dev, "voltage = %dmV\n", cw_bat->voltage_mv);
+
+	if (cw_bat->battery_changed)
+		power_supply_changed(cw_bat->rk_bat);
+	cw_bat->battery_changed = false;
+
+	queue_delayed_work(cw_bat->battery_workqueue,
+			   &cw_bat->battery_delay_work,
+			   msecs_to_jiffies(cw_bat->poll_interval_ms));
+}
+
+static bool cw_battery_valid_time_to_empty(struct cw_battery *cw_bat)
+{
+	return	cw_bat->time_to_empty > 0 &&
+		cw_bat->time_to_empty < CW2015_MASK_SOC &&
+		cw_bat->status == POWER_SUPPLY_STATUS_DISCHARGING;
+}
+
+static int cw_battery_get_property(struct power_supply *psy,
+				   enum power_supply_property psp,
+				   union power_supply_propval *val)
+{
+	struct cw_battery *cw_bat;
+
+	cw_bat = power_supply_get_drvdata(psy);
+	switch (psp) {
+	case POWER_SUPPLY_PROP_CAPACITY:
+		val->intval = cw_bat->soc;
+		break;
+
+	case POWER_SUPPLY_PROP_STATUS:
+		val->intval = cw_bat->status;
+		break;
+
+	case POWER_SUPPLY_PROP_PRESENT:
+		val->intval = !!cw_bat->voltage_mv;
+		break;
+
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		val->intval = cw_bat->voltage_mv * 1000;
+		break;
+
+	case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW:
+		if (cw_battery_valid_time_to_empty(cw_bat))
+			val->intval = cw_bat->time_to_empty;
+		else
+			val->intval = 0;
+		break;
+
+	case POWER_SUPPLY_PROP_TECHNOLOGY:
+		val->intval = POWER_SUPPLY_TECHNOLOGY_LION;
+		break;
+
+	case POWER_SUPPLY_PROP_CHARGE_COUNTER:
+		val->intval = cw_bat->charge_count;
+		break;
+
+	case POWER_SUPPLY_PROP_CHARGE_FULL:
+	case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN:
+		if (cw_bat->battery.charge_full_design_uah > 0)
+			val->intval = cw_bat->battery.charge_full_design_uah;
+		else
+			val->intval = 0;
+		break;
+
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		if (cw_battery_valid_time_to_empty(cw_bat) &&
+		    cw_bat->battery.charge_full_design_uah > 0) {
+			/* calculate remaining capacity */
+			val->intval = cw_bat->battery.charge_full_design_uah;
+			val->intval = val->intval * cw_bat->soc / 100;
+
+			/* estimate current based on time to empty */
+			val->intval = 60 * val->intval / cw_bat->time_to_empty;
+		} else {
+			val->intval = 0;
+		}
+
+		break;
+
+	default:
+		break;
+	}
+	return 0;
+}
+
+static enum power_supply_property cw_battery_properties[] = {
+	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_CHARGE_COUNTER,
+	POWER_SUPPLY_PROP_CHARGE_FULL,
+	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+};
+
+static const struct power_supply_desc cw2015_bat_desc = {
+	.name		= "cw2015-battery",
+	.type		= POWER_SUPPLY_TYPE_BATTERY,
+	.properties	= cw_battery_properties,
+	.num_properties	= ARRAY_SIZE(cw_battery_properties),
+	.get_property	= cw_battery_get_property,
+};
+
+static int cw2015_parse_properties(struct cw_battery *cw_bat)
+{
+	struct device *dev = cw_bat->dev;
+	int length;
+	int ret;
+
+	length = device_property_count_u8(dev, "cellwise,battery-profile");
+	if (length < 0) {
+		dev_warn(cw_bat->dev,
+			 "No battery-profile found, using current flash contents\n");
+	} else if (length != CW2015_SIZE_BATINFO) {
+		dev_err(cw_bat->dev, "battery-profile must be %d bytes\n",
+			CW2015_SIZE_BATINFO);
+		return -EINVAL;
+	} else {
+		cw_bat->bat_profile = devm_kzalloc(dev, length, GFP_KERNEL);
+		if (!cw_bat->bat_profile)
+			return -ENOMEM;
+
+		ret = device_property_read_u8_array(dev,
+						"cellwise,battery-profile",
+						cw_bat->bat_profile,
+						length);
+		if (ret)
+			return ret;
+	}
+
+	ret = device_property_read_u32(dev, "cellwise,monitor-interval-ms",
+				       &cw_bat->poll_interval_ms);
+	if (ret) {
+		dev_dbg(cw_bat->dev, "Using default poll interval\n");
+		cw_bat->poll_interval_ms = CW2015_DEFAULT_POLL_INTERVAL_MS;
+	}
+
+	return 0;
+}
+
+static const struct regmap_range regmap_ranges_rd_yes[] = {
+	regmap_reg_range(CW2015_REG_VERSION, CW2015_REG_VERSION),
+	regmap_reg_range(CW2015_REG_VCELL, CW2015_REG_CONFIG),
+	regmap_reg_range(CW2015_REG_MODE, CW2015_REG_MODE),
+	regmap_reg_range(CW2015_REG_BATINFO,
+			CW2015_REG_BATINFO + CW2015_SIZE_BATINFO - 1),
+};
+
+static const struct regmap_access_table regmap_rd_table = {
+	.yes_ranges = regmap_ranges_rd_yes,
+	.n_yes_ranges = 4,
+};
+
+static const struct regmap_range regmap_ranges_wr_yes[] = {
+	regmap_reg_range(CW2015_REG_RRT_ALERT, CW2015_REG_CONFIG),
+	regmap_reg_range(CW2015_REG_MODE, CW2015_REG_MODE),
+	regmap_reg_range(CW2015_REG_BATINFO,
+			CW2015_REG_BATINFO + CW2015_SIZE_BATINFO - 1),
+};
+
+static const struct regmap_access_table regmap_wr_table = {
+	.yes_ranges = regmap_ranges_wr_yes,
+	.n_yes_ranges = 3,
+};
+
+static const struct regmap_range regmap_ranges_vol_yes[] = {
+	regmap_reg_range(CW2015_REG_VCELL, CW2015_REG_SOC + 1),
+};
+
+static const struct regmap_access_table regmap_vol_table = {
+	.yes_ranges = regmap_ranges_vol_yes,
+	.n_yes_ranges = 1,
+};
+
+static const struct regmap_config cw2015_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.rd_table = &regmap_rd_table,
+	.wr_table = &regmap_wr_table,
+	.volatile_table = &regmap_vol_table,
+	.max_register = CW2015_REG_BATINFO + CW2015_SIZE_BATINFO - 1,
+};
+
+static int cw_bat_probe(struct i2c_client *client)
+{
+	int ret;
+	struct cw_battery *cw_bat;
+	struct power_supply_config psy_cfg = { 0 };
+
+	cw_bat = devm_kzalloc(&client->dev, sizeof(*cw_bat), GFP_KERNEL);
+	if (!cw_bat)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, cw_bat);
+	cw_bat->dev = &client->dev;
+	cw_bat->soc = 1;
+
+	ret = cw2015_parse_properties(cw_bat);
+	if (ret) {
+		dev_err(cw_bat->dev, "Failed to parse cw2015 properties\n");
+		return ret;
+	}
+
+	cw_bat->regmap = devm_regmap_init_i2c(client, &cw2015_regmap_config);
+	if (IS_ERR(cw_bat->regmap)) {
+		dev_err(cw_bat->dev, "Failed to allocate regmap: %ld\n",
+			PTR_ERR(cw_bat->regmap));
+		return PTR_ERR(cw_bat->regmap);
+	}
+
+	ret = cw_init(cw_bat);
+	if (ret) {
+		dev_err(cw_bat->dev, "Init failed: %d\n", ret);
+		return ret;
+	}
+
+	psy_cfg.drv_data = cw_bat;
+	psy_cfg.fwnode = dev_fwnode(cw_bat->dev);
+
+	cw_bat->rk_bat = devm_power_supply_register(&client->dev,
+						    &cw2015_bat_desc,
+						    &psy_cfg);
+	if (IS_ERR(cw_bat->rk_bat)) {
+		dev_err(cw_bat->dev, "Failed to register power supply\n");
+		return PTR_ERR(cw_bat->rk_bat);
+	}
+
+	ret = power_supply_get_battery_info(cw_bat->rk_bat, &cw_bat->battery);
+	if (ret) {
+		dev_warn(cw_bat->dev,
+			 "No monitored battery, some properties will be missing\n");
+	}
+
+	cw_bat->battery_workqueue = create_singlethread_workqueue("rk_battery");
+	INIT_DELAYED_WORK(&cw_bat->battery_delay_work, cw_bat_work);
+	queue_delayed_work(cw_bat->battery_workqueue,
+			   &cw_bat->battery_delay_work, msecs_to_jiffies(10));
+	return 0;
+}
+
+static int __maybe_unused cw_bat_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct cw_battery *cw_bat = i2c_get_clientdata(client);
+
+	cancel_delayed_work_sync(&cw_bat->battery_delay_work);
+	return 0;
+}
+
+static int __maybe_unused cw_bat_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct cw_battery *cw_bat = i2c_get_clientdata(client);
+
+	queue_delayed_work(cw_bat->battery_workqueue,
+			   &cw_bat->battery_delay_work, 0);
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(cw_bat_pm_ops, cw_bat_suspend, cw_bat_resume);
+
+static int cw_bat_remove(struct i2c_client *client)
+{
+	struct cw_battery *cw_bat = i2c_get_clientdata(client);
+
+	cancel_delayed_work_sync(&cw_bat->battery_delay_work);
+	power_supply_put_battery_info(cw_bat->rk_bat, &cw_bat->battery);
+	return 0;
+}
+
+static const struct i2c_device_id cw_bat_id_table[] = {
+	{ "cw2015", 0 },
+	{ }
+};
+
+static const struct of_device_id cw2015_of_match[] = {
+	{ .compatible = "cellwise,cw2015" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, cw2015_of_match);
+
+static struct i2c_driver cw_bat_driver = {
+	.driver = {
+		.name = "cw2015",
+		.of_match_table = cw2015_of_match,
+		.pm = &cw_bat_pm_ops,
+	},
+	.probe_new = cw_bat_probe,
+	.remove = cw_bat_remove,
+	.id_table = cw_bat_id_table,
+};
+
+module_i2c_driver(cw_bat_driver);
+
+MODULE_AUTHOR("xhc<xhc@rock-chips.com>");
+MODULE_AUTHOR("Tobias Schramm <t.schramm@manjaro.org>");
+MODULE_DESCRIPTION("cw2015/cw2013 battery driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/generic-adc-battery.c b/drivers/power/supply/generic-adc-battery.c
index bc462d1ec963..caa829738ef7 100644
--- a/drivers/power/supply/generic-adc-battery.c
+++ b/drivers/power/supply/generic-adc-battery.c
@@ -241,6 +241,7 @@ static int gab_probe(struct platform_device *pdev)
 	struct power_supply_desc *psy_desc;
 	struct power_supply_config psy_cfg = {};
 	struct gab_platform_data *pdata = pdev->dev.platform_data;
+	enum power_supply_property *properties;
 	int ret = 0;
 	int chan;
 	int index = ARRAY_SIZE(gab_props);
@@ -268,16 +269,16 @@ static int gab_probe(struct platform_device *pdev)
 	 * copying the static properties and allocating extra memory for holding
 	 * the extra configurable properties received from platform data.
 	 */
-	psy_desc->properties = kcalloc(ARRAY_SIZE(gab_props) +
-					ARRAY_SIZE(gab_chan_name),
-					sizeof(*psy_desc->properties),
-					GFP_KERNEL);
-	if (!psy_desc->properties) {
+	properties = kcalloc(ARRAY_SIZE(gab_props) +
+			     ARRAY_SIZE(gab_chan_name),
+			     sizeof(*properties),
+			     GFP_KERNEL);
+	if (!properties) {
 		ret = -ENOMEM;
 		goto first_mem_fail;
 	}
 
-	memcpy(psy_desc->properties, gab_props, sizeof(gab_props));
+	memcpy(properties, gab_props, sizeof(gab_props));
 
 	/*
 	 * getting channel from iio and copying the battery properties
@@ -294,13 +295,11 @@ static int gab_probe(struct platform_device *pdev)
 			int index2;
 
 			for (index2 = 0; index2 < index; index2++) {
-				if (psy_desc->properties[index2] ==
-				    gab_dyn_props[chan])
+				if (properties[index2] == gab_dyn_props[chan])
 					break;	/* already known */
 			}
 			if (index2 == index)	/* really new */
-				psy_desc->properties[index++] =
-					gab_dyn_props[chan];
+				properties[index++] = gab_dyn_props[chan];
 			any = true;
 		}
 	}
@@ -317,6 +316,7 @@ static int gab_probe(struct platform_device *pdev)
 	 * as come channels may be not be supported by the device.So
 	 * we need to take care of that.
 	 */
+	psy_desc->properties = properties;
 	psy_desc->num_properties = index;
 
 	adc_bat->psy = power_supply_register(&pdev->dev, psy_desc, &psy_cfg);
@@ -358,7 +358,7 @@ err_reg_fail:
 			iio_channel_release(adc_bat->channel[chan]);
 	}
 second_mem_fail:
-	kfree(psy_desc->properties);
+	kfree(properties);
 first_mem_fail:
 	return ret;
 }
diff --git a/drivers/power/supply/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c
index 84a206f42a8e..e7931ffb7151 100644
--- a/drivers/power/supply/lp8788-charger.c
+++ b/drivers/power/supply/lp8788-charger.c
@@ -572,27 +572,14 @@ static void lp8788_setup_adc_channel(struct device *dev,
 		return;
 
 	/* ADC channel for battery voltage */
-	chan = iio_channel_get(dev, pdata->adc_vbatt);
+	chan = devm_iio_channel_get(dev, pdata->adc_vbatt);
 	pchg->chan[LP8788_VBATT] = IS_ERR(chan) ? NULL : chan;
 
 	/* ADC channel for battery temperature */
-	chan = iio_channel_get(dev, pdata->adc_batt_temp);
+	chan = devm_iio_channel_get(dev, pdata->adc_batt_temp);
 	pchg->chan[LP8788_BATT_TEMP] = IS_ERR(chan) ? NULL : chan;
 }
 
-static void lp8788_release_adc_channel(struct lp8788_charger *pchg)
-{
-	int i;
-
-	for (i = 0; i < LP8788_NUM_CHG_ADC; i++) {
-		if (!pchg->chan[i])
-			continue;
-
-		iio_channel_release(pchg->chan[i]);
-		pchg->chan[i] = NULL;
-	}
-}
-
 static ssize_t lp8788_show_charger_status(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
@@ -735,7 +722,6 @@ static int lp8788_charger_remove(struct platform_device *pdev)
 	flush_work(&pchg->charger_work);
 	lp8788_irq_unregister(pdev, pchg);
 	lp8788_psy_unregister(pchg);
-	lp8788_release_adc_channel(pchg);
 
 	return 0;
 }
diff --git a/drivers/power/supply/max14577_charger.c b/drivers/power/supply/max14577_charger.c
index 8a59feac6468..dcedae18d7be 100644
--- a/drivers/power/supply/max14577_charger.c
+++ b/drivers/power/supply/max14577_charger.c
@@ -623,9 +623,19 @@ static const struct platform_device_id max14577_charger_id[] = {
 };
 MODULE_DEVICE_TABLE(platform, max14577_charger_id);
 
+static const struct of_device_id of_max14577_charger_dt_match[] = {
+	{ .compatible = "maxim,max14577-charger",
+	  .data = (void *)MAXIM_DEVICE_TYPE_MAX14577, },
+	{ .compatible = "maxim,max77836-charger",
+	  .data = (void *)MAXIM_DEVICE_TYPE_MAX77836, },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, of_max14577_charger_dt_match);
+
 static struct platform_driver max14577_charger_driver = {
 	.driver = {
 		.name	= "max14577-charger",
+		.of_match_table = of_max14577_charger_dt_match,
 	},
 	.probe		= max14577_charger_probe,
 	.remove		= max14577_charger_remove,
diff --git a/drivers/power/supply/max14656_charger_detector.c b/drivers/power/supply/max14656_charger_detector.c
index 3bbb8b4c8ae7..137f9fafce8c 100644
--- a/drivers/power/supply/max14656_charger_detector.c
+++ b/drivers/power/supply/max14656_charger_detector.c
@@ -139,10 +139,9 @@ static void max14656_irq_worker(struct work_struct *work)
 
 	u8 buf[REG_TOTAL_NUM];
 	u8 chg_type;
-	int ret = 0;
 
-	ret = max14656_read_block_reg(chip->client, MAX14656_DEVICE_ID,
-				      REG_TOTAL_NUM, buf);
+	max14656_read_block_reg(chip->client, MAX14656_DEVICE_ID,
+				REG_TOTAL_NUM, buf);
 
 	if ((buf[MAX14656_STATUS_1] & STATUS1_VB_VALID_MASK) &&
 		(buf[MAX14656_STATUS_1] & STATUS1_CHG_TYPE_MASK)) {
diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c
index 8a1f0ee493aa..48aa44665e2f 100644
--- a/drivers/power/supply/max17040_battery.c
+++ b/drivers/power/supply/max17040_battery.c
@@ -126,7 +126,7 @@ static void max17040_get_vcell(struct i2c_client *client)
 
 	vcell = max17040_read_reg(client, MAX17040_VCELL);
 
-	chip->vcell = vcell;
+	chip->vcell = (vcell >> 4) * 1250;
 }
 
 static void max17040_get_soc(struct i2c_client *client)
diff --git a/drivers/power/supply/max17042_battery.c b/drivers/power/supply/max17042_battery.c
index 69ec4295d55d..f284547913d6 100644
--- a/drivers/power/supply/max17042_battery.c
+++ b/drivers/power/supply/max17042_battery.c
@@ -87,6 +87,7 @@ static enum power_supply_property max17042_battery_props[] = {
 	POWER_SUPPLY_PROP_SCOPE,
 	POWER_SUPPLY_PROP_CURRENT_NOW,
 	POWER_SUPPLY_PROP_CURRENT_AVG,
+	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
 };
 
 static int max17042_get_temperature(struct max17042_chip *chip, int *temp)
@@ -411,6 +412,13 @@ static int max17042_get_property(struct power_supply *psy,
 			return -EINVAL;
 		}
 		break;
+	case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW:
+		ret = regmap_read(map, MAX17042_TTE, &data);
+		if (ret < 0)
+			return ret;
+
+		val->intval = data * 5625 / 1000;
+		break;
 	default:
 		return -EINVAL;
 	}
diff --git a/drivers/power/supply/mp2629_charger.c b/drivers/power/supply/mp2629_charger.c
new file mode 100644
index 000000000000..bdf924b73e47
--- /dev/null
+++ b/drivers/power/supply/mp2629_charger.c
@@ -0,0 +1,669 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * MP2629 battery charger driver
+ *
+ * Copyright 2020 Monolithic Power Systems, Inc
+ *
+ * Author: Saravanan Sekar <sravanhome@gmail.com>
+ */
+
+#include <linux/bits.h>
+#include <linux/iio/consumer.h>
+#include <linux/iio/types.h>
+#include <linux/interrupt.h>
+#include <linux/mfd/mp2629.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+#include <linux/power_supply.h>
+#include <linux/regmap.h>
+
+#define MP2629_REG_INPUT_ILIM		0x00
+#define MP2629_REG_INPUT_VLIM		0x01
+#define MP2629_REG_CHARGE_CTRL		0x04
+#define MP2629_REG_CHARGE_ILIM		0x05
+#define MP2629_REG_PRECHARGE		0x06
+#define MP2629_REG_TERM_CURRENT		0x06
+#define MP2629_REG_CHARGE_VLIM		0x07
+#define MP2629_REG_TIMER_CTRL		0x08
+#define MP2629_REG_IMPEDANCE_COMP	0x09
+#define MP2629_REG_INTERRUPT		0x0b
+#define MP2629_REG_STATUS		0x0c
+#define MP2629_REG_FAULT		0x0d
+
+#define MP2629_MASK_INPUT_TYPE		GENMASK(7, 5)
+#define MP2629_MASK_CHARGE_TYPE		GENMASK(4, 3)
+#define MP2629_MASK_CHARGE_CTRL		GENMASK(5, 4)
+#define MP2629_MASK_WDOG_CTRL		GENMASK(5, 4)
+#define MP2629_MASK_IMPEDANCE		GENMASK(7, 4)
+
+#define MP2629_INPUTSOURCE_CHANGE	GENMASK(7, 5)
+#define MP2629_CHARGING_CHANGE		GENMASK(4, 3)
+#define MP2629_FAULT_BATTERY		BIT(3)
+#define MP2629_FAULT_THERMAL		BIT(4)
+#define MP2629_FAULT_INPUT		BIT(5)
+#define MP2629_FAULT_OTG		BIT(6)
+
+#define MP2629_MAX_BATT_CAPACITY	100
+
+#define MP2629_PROPS(_idx, _min, _max, _step)		\
+	[_idx] = {					\
+		.min	= _min,				\
+		.max	= _max,				\
+		.step	= _step,			\
+}
+
+enum mp2629_source_type {
+	MP2629_SOURCE_TYPE_NO_INPUT,
+	MP2629_SOURCE_TYPE_NON_STD,
+	MP2629_SOURCE_TYPE_SDP,
+	MP2629_SOURCE_TYPE_CDP,
+	MP2629_SOURCE_TYPE_DCP,
+	MP2629_SOURCE_TYPE_OTG = 7,
+};
+
+enum mp2629_field {
+	INPUT_ILIM,
+	INPUT_VLIM,
+	CHARGE_ILIM,
+	CHARGE_VLIM,
+	PRECHARGE,
+	TERM_CURRENT,
+	MP2629_MAX_FIELD
+};
+
+struct mp2629_charger {
+	struct device *dev;
+	int status;
+	int fault;
+
+	struct regmap *regmap;
+	struct regmap_field *regmap_fields[MP2629_MAX_FIELD];
+	struct mutex lock;
+	struct power_supply *usb;
+	struct power_supply *battery;
+	struct iio_channel *iiochan[MP2629_ADC_CHAN_END];
+};
+
+struct mp2629_prop {
+	int reg;
+	int mask;
+	int min;
+	int max;
+	int step;
+	int shift;
+};
+
+static enum power_supply_usb_type mp2629_usb_types[] = {
+	POWER_SUPPLY_USB_TYPE_SDP,
+	POWER_SUPPLY_USB_TYPE_DCP,
+	POWER_SUPPLY_USB_TYPE_CDP,
+	POWER_SUPPLY_USB_TYPE_PD_DRP,
+	POWER_SUPPLY_USB_TYPE_UNKNOWN
+};
+
+static enum power_supply_property mp2629_charger_usb_props[] = {
+	POWER_SUPPLY_PROP_ONLINE,
+	POWER_SUPPLY_PROP_USB_TYPE,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,
+	POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT,
+};
+
+static enum power_supply_property mp2629_charger_bat_props[] = {
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_HEALTH,
+	POWER_SUPPLY_PROP_CHARGE_TYPE,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_PRECHARGE_CURRENT,
+	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX,
+};
+
+static struct mp2629_prop props[] = {
+	MP2629_PROPS(INPUT_ILIM, 100000, 3250000, 50000),
+	MP2629_PROPS(INPUT_VLIM, 3800000, 5300000, 100000),
+	MP2629_PROPS(CHARGE_ILIM, 320000, 4520000, 40000),
+	MP2629_PROPS(CHARGE_VLIM, 3400000, 4670000, 10000),
+	MP2629_PROPS(PRECHARGE, 120000, 720000, 40000),
+	MP2629_PROPS(TERM_CURRENT, 80000, 680000, 40000),
+};
+
+static const struct reg_field mp2629_reg_fields[] = {
+	[INPUT_ILIM]	= REG_FIELD(MP2629_REG_INPUT_ILIM, 0, 5),
+	[INPUT_VLIM]	= REG_FIELD(MP2629_REG_INPUT_VLIM, 0, 3),
+	[CHARGE_ILIM]	= REG_FIELD(MP2629_REG_CHARGE_ILIM, 0, 6),
+	[CHARGE_VLIM]	= REG_FIELD(MP2629_REG_CHARGE_VLIM, 1, 7),
+	[PRECHARGE]	= REG_FIELD(MP2629_REG_PRECHARGE, 4, 7),
+	[TERM_CURRENT]	= REG_FIELD(MP2629_REG_TERM_CURRENT, 0, 3),
+};
+
+static char *adc_chan_name[] = {
+	"mp2629-batt-volt",
+	"mp2629-system-volt",
+	"mp2629-input-volt",
+	"mp2629-batt-current",
+	"mp2629-input-current",
+};
+
+static int mp2629_read_adc(struct mp2629_charger *charger,
+			   enum mp2629_adc_chan ch,
+			   union power_supply_propval *val)
+{
+	int ret;
+	int chval;
+
+	ret = iio_read_channel_processed(charger->iiochan[ch], &chval);
+	if (ret)
+		return ret;
+
+	val->intval = chval * 1000;
+
+	return 0;
+}
+
+static int mp2629_get_prop(struct mp2629_charger *charger,
+			   enum mp2629_field fld,
+			   union power_supply_propval *val)
+{
+	int ret;
+	unsigned int rval;
+
+	ret = regmap_field_read(charger->regmap_fields[fld], &rval);
+	if (ret)
+		return ret;
+
+	val->intval = rval * props[fld].step + props[fld].min;
+
+	return 0;
+}
+
+static int mp2629_set_prop(struct mp2629_charger *charger,
+			   enum mp2629_field fld,
+			   const union power_supply_propval *val)
+{
+	unsigned int rval;
+
+	if (val->intval < props[fld].min || val->intval > props[fld].max)
+		return -EINVAL;
+
+	rval = (val->intval - props[fld].min) / props[fld].step;
+	return regmap_field_write(charger->regmap_fields[fld], rval);
+}
+
+static int mp2629_get_battery_capacity(struct mp2629_charger *charger,
+				       union power_supply_propval *val)
+{
+	union power_supply_propval vnow, vlim;
+	int ret;
+
+	ret = mp2629_read_adc(charger, MP2629_BATT_VOLT, &vnow);
+	if (ret)
+		return ret;
+
+	ret = mp2629_get_prop(charger, CHARGE_VLIM, &vlim);
+	if (ret)
+		return ret;
+
+	val->intval = (vnow.intval * 100) / vlim.intval;
+	val->intval = min(val->intval, MP2629_MAX_BATT_CAPACITY);
+
+	return 0;
+}
+
+static int mp2629_charger_battery_get_prop(struct power_supply *psy,
+					enum power_supply_property psp,
+					union power_supply_propval *val)
+{
+	struct mp2629_charger *charger = dev_get_drvdata(psy->dev.parent);
+	unsigned int rval;
+	int ret = 0;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		ret = mp2629_read_adc(charger, MP2629_BATT_VOLT, val);
+		break;
+
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		ret = mp2629_read_adc(charger, MP2629_BATT_CURRENT, val);
+		break;
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX:
+		val->intval = 4520000;
+		break;
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX:
+		val->intval = 4670000;
+		break;
+
+	case POWER_SUPPLY_PROP_CAPACITY:
+		ret = mp2629_get_battery_capacity(charger, val);
+		break;
+
+	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+		ret = mp2629_get_prop(charger, TERM_CURRENT, val);
+		break;
+
+	case POWER_SUPPLY_PROP_PRECHARGE_CURRENT:
+		ret = mp2629_get_prop(charger, PRECHARGE, val);
+		break;
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
+		ret = mp2629_get_prop(charger, CHARGE_VLIM, val);
+		break;
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
+		ret = mp2629_get_prop(charger, CHARGE_ILIM, val);
+		break;
+
+	case POWER_SUPPLY_PROP_HEALTH:
+		if (!charger->fault)
+			val->intval = POWER_SUPPLY_HEALTH_GOOD;
+		if (MP2629_FAULT_BATTERY & charger->fault)
+			val->intval = POWER_SUPPLY_HEALTH_OVERVOLTAGE;
+		else if (MP2629_FAULT_THERMAL & charger->fault)
+			val->intval = POWER_SUPPLY_HEALTH_OVERHEAT;
+		else if (MP2629_FAULT_INPUT & charger->fault)
+			val->intval = POWER_SUPPLY_HEALTH_OVERVOLTAGE;
+		break;
+
+	case POWER_SUPPLY_PROP_STATUS:
+		ret = regmap_read(charger->regmap, MP2629_REG_STATUS, &rval);
+		if (ret)
+			break;
+
+		rval = (rval & MP2629_MASK_CHARGE_TYPE) >> 3;
+		switch (rval) {
+		case 0x00:
+			val->intval = POWER_SUPPLY_STATUS_DISCHARGING;
+			break;
+		case 0x01:
+		case 0x10:
+			val->intval = POWER_SUPPLY_STATUS_CHARGING;
+			break;
+		case 0x11:
+			val->intval = POWER_SUPPLY_STATUS_FULL;
+		}
+		break;
+
+	case POWER_SUPPLY_PROP_CHARGE_TYPE:
+		ret = regmap_read(charger->regmap, MP2629_REG_STATUS, &rval);
+		if (ret)
+			break;
+
+		rval = (rval & MP2629_MASK_CHARGE_TYPE) >> 3;
+		switch (rval) {
+		case 0x00:
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_NONE;
+			break;
+		case 0x01:
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_TRICKLE;
+			break;
+		case 0x10:
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_STANDARD;
+			break;
+		default:
+			val->intval = POWER_SUPPLY_CHARGE_TYPE_UNKNOWN;
+		}
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int mp2629_charger_battery_set_prop(struct power_supply *psy,
+					enum power_supply_property psp,
+					const union power_supply_propval *val)
+{
+	struct mp2629_charger *charger = dev_get_drvdata(psy->dev.parent);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+		return mp2629_set_prop(charger, TERM_CURRENT, val);
+
+	case POWER_SUPPLY_PROP_PRECHARGE_CURRENT:
+		return mp2629_set_prop(charger, PRECHARGE, val);
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
+		return mp2629_set_prop(charger, CHARGE_VLIM, val);
+
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
+		return mp2629_set_prop(charger, CHARGE_ILIM, val);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int mp2629_charger_usb_get_prop(struct power_supply *psy,
+				enum power_supply_property psp,
+				union power_supply_propval *val)
+{
+	struct mp2629_charger *charger = dev_get_drvdata(psy->dev.parent);
+	unsigned int rval;
+	int ret;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_ONLINE:
+		ret = regmap_read(charger->regmap, MP2629_REG_STATUS, &rval);
+		if (ret)
+			break;
+
+		val->intval = !!(rval & MP2629_MASK_INPUT_TYPE);
+		break;
+
+	case POWER_SUPPLY_PROP_USB_TYPE:
+		ret = regmap_read(charger->regmap, MP2629_REG_STATUS, &rval);
+		if (ret)
+			break;
+
+		rval = (rval & MP2629_MASK_INPUT_TYPE) >> 5;
+		switch (rval) {
+		case MP2629_SOURCE_TYPE_SDP:
+			val->intval = POWER_SUPPLY_USB_TYPE_SDP;
+			break;
+		case MP2629_SOURCE_TYPE_CDP:
+			val->intval = POWER_SUPPLY_USB_TYPE_CDP;
+			break;
+		case MP2629_SOURCE_TYPE_DCP:
+			val->intval = POWER_SUPPLY_USB_TYPE_DCP;
+			break;
+		case MP2629_SOURCE_TYPE_OTG:
+			val->intval = POWER_SUPPLY_USB_TYPE_PD_DRP;
+			break;
+		default:
+			val->intval = POWER_SUPPLY_USB_TYPE_UNKNOWN;
+			break;
+		}
+		break;
+
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		ret = mp2629_read_adc(charger, MP2629_INPUT_VOLT, val);
+		break;
+
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		ret = mp2629_read_adc(charger, MP2629_INPUT_CURRENT, val);
+		break;
+
+	case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT:
+		ret = mp2629_get_prop(charger, INPUT_VLIM, val);
+		break;
+
+	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+		ret = mp2629_get_prop(charger, INPUT_ILIM, val);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int mp2629_charger_usb_set_prop(struct power_supply *psy,
+				enum power_supply_property psp,
+				const union power_supply_propval *val)
+{
+	struct mp2629_charger *charger = dev_get_drvdata(psy->dev.parent);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT:
+		return mp2629_set_prop(charger, INPUT_VLIM, val);
+
+	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+		return mp2629_set_prop(charger, INPUT_ILIM, val);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int mp2629_charger_battery_prop_writeable(struct power_supply *psy,
+				     enum power_supply_property psp)
+{
+	return (psp == POWER_SUPPLY_PROP_PRECHARGE_CURRENT) ||
+	       (psp == POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT) ||
+	       (psp == POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT) ||
+	       (psp == POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE);
+}
+
+static int mp2629_charger_usb_prop_writeable(struct power_supply *psy,
+				     enum power_supply_property psp)
+{
+	return (psp == POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT) ||
+	       (psp == POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT);
+}
+
+static irqreturn_t mp2629_irq_handler(int irq, void *dev_id)
+{
+	struct mp2629_charger *charger = dev_id;
+	unsigned int rval;
+	int ret;
+
+	mutex_lock(&charger->lock);
+
+	ret = regmap_read(charger->regmap, MP2629_REG_FAULT, &rval);
+	if (ret)
+		goto unlock;
+
+	if (rval) {
+		charger->fault = rval;
+		if (MP2629_FAULT_BATTERY & rval)
+			dev_err(charger->dev, "Battery fault OVP\n");
+		else if (MP2629_FAULT_THERMAL & rval)
+			dev_err(charger->dev, "Thermal shutdown fault\n");
+		else if (MP2629_FAULT_INPUT & rval)
+			dev_err(charger->dev, "no input or input OVP\n");
+		else if (MP2629_FAULT_OTG & rval)
+			dev_err(charger->dev, "VIN overloaded\n");
+
+		goto unlock;
+	}
+
+	ret = regmap_read(charger->regmap, MP2629_REG_STATUS, &rval);
+	if (ret)
+		goto unlock;
+
+	if (rval & MP2629_INPUTSOURCE_CHANGE)
+		power_supply_changed(charger->usb);
+	else if (rval & MP2629_CHARGING_CHANGE)
+		power_supply_changed(charger->battery);
+
+unlock:
+	mutex_unlock(&charger->lock);
+
+	return IRQ_HANDLED;
+}
+
+static const struct power_supply_desc mp2629_usb_desc = {
+	.name		= "mp2629_usb",
+	.type		= POWER_SUPPLY_TYPE_USB,
+	.usb_types      = mp2629_usb_types,
+	.num_usb_types  = ARRAY_SIZE(mp2629_usb_types),
+	.properties	= mp2629_charger_usb_props,
+	.num_properties	= ARRAY_SIZE(mp2629_charger_usb_props),
+	.get_property	= mp2629_charger_usb_get_prop,
+	.set_property	= mp2629_charger_usb_set_prop,
+	.property_is_writeable = mp2629_charger_usb_prop_writeable,
+};
+
+static const struct power_supply_desc mp2629_battery_desc = {
+	.name		= "mp2629_battery",
+	.type		= POWER_SUPPLY_TYPE_BATTERY,
+	.properties	= mp2629_charger_bat_props,
+	.num_properties	= ARRAY_SIZE(mp2629_charger_bat_props),
+	.get_property	= mp2629_charger_battery_get_prop,
+	.set_property	= mp2629_charger_battery_set_prop,
+	.property_is_writeable = mp2629_charger_battery_prop_writeable,
+};
+
+static ssize_t batt_impedance_compensation_show(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	struct mp2629_charger *charger = dev_get_drvdata(dev->parent);
+	unsigned int rval;
+	int ret;
+
+	ret = regmap_read(charger->regmap, MP2629_REG_IMPEDANCE_COMP, &rval);
+	if (ret)
+		return ret;
+
+	rval = (rval >> 4) * 10;
+	return sprintf(buf, "%d mohm\n", rval);
+}
+
+static ssize_t batt_impedance_compensation_store(struct device *dev,
+					    struct device_attribute *attr,
+					    const char *buf,
+					    size_t count)
+{
+	struct mp2629_charger *charger = dev_get_drvdata(dev->parent);
+	unsigned int val;
+	int ret;
+
+	ret = kstrtouint(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val > 140)
+		return -ERANGE;
+
+	/* multiples of 10 mohm so round off */
+	val = val / 10;
+	ret = regmap_update_bits(charger->regmap, MP2629_REG_IMPEDANCE_COMP,
+					MP2629_MASK_IMPEDANCE, val << 4);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(batt_impedance_compensation);
+
+static struct attribute *mp2629_charger_sysfs_attrs[] = {
+	&dev_attr_batt_impedance_compensation.attr,
+	NULL
+};
+ATTRIBUTE_GROUPS(mp2629_charger_sysfs);
+
+static void mp2629_charger_disable(void *data)
+{
+	struct mp2629_charger *charger = data;
+
+	regmap_update_bits(charger->regmap, MP2629_REG_CHARGE_CTRL,
+					MP2629_MASK_CHARGE_CTRL, 0);
+}
+
+static int mp2629_charger_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mp2629_data *ddata = dev_get_drvdata(dev->parent);
+	struct mp2629_charger *charger;
+	struct power_supply_config psy_cfg = {};
+	int ret, i, irq;
+
+	charger = devm_kzalloc(dev, sizeof(*charger), GFP_KERNEL);
+	if (!charger)
+		return -ENOMEM;
+
+	charger->regmap = ddata->regmap;
+	charger->dev = dev;
+	platform_set_drvdata(pdev, charger);
+
+	irq = platform_get_irq_optional(to_platform_device(dev->parent), 0);
+	if (irq < 0) {
+		dev_err(dev, "get irq fail: %d\n", irq);
+		return irq;
+	}
+
+	for (i = 0; i < MP2629_MAX_FIELD; i++) {
+		charger->regmap_fields[i] = devm_regmap_field_alloc(dev,
+					charger->regmap, mp2629_reg_fields[i]);
+		if (IS_ERR(charger->regmap_fields[i])) {
+			dev_err(dev, "regmap field alloc fail %d\n", i);
+			return PTR_ERR(charger->regmap_fields[i]);
+		}
+	}
+
+	for (i = 0; i < MP2629_ADC_CHAN_END; i++) {
+		charger->iiochan[i] = devm_iio_channel_get(dev,
+							adc_chan_name[i]);
+		if (IS_ERR(charger->iiochan[i])) {
+			dev_err(dev, "iio chan get %s err\n", adc_chan_name[i]);
+			return PTR_ERR(charger->iiochan[i]);
+		}
+	}
+
+	ret = devm_add_action_or_reset(dev, mp2629_charger_disable, charger);
+	if (ret)
+		return ret;
+
+	charger->usb = devm_power_supply_register(dev, &mp2629_usb_desc, NULL);
+	if (IS_ERR(charger->usb)) {
+		dev_err(dev, "power supply register usb failed\n");
+		return PTR_ERR(charger->usb);
+	}
+
+	psy_cfg.drv_data = charger;
+	psy_cfg.attr_grp = mp2629_charger_sysfs_groups;
+	charger->battery = devm_power_supply_register(dev,
+					 &mp2629_battery_desc, &psy_cfg);
+	if (IS_ERR(charger->battery)) {
+		dev_err(dev, "power supply register battery failed\n");
+		return PTR_ERR(charger->battery);
+	}
+
+	ret = regmap_update_bits(charger->regmap, MP2629_REG_CHARGE_CTRL,
+					MP2629_MASK_CHARGE_CTRL, BIT(4));
+	if (ret) {
+		dev_err(dev, "enable charge fail: %d\n", ret);
+		return ret;
+	}
+
+	regmap_update_bits(charger->regmap, MP2629_REG_TIMER_CTRL,
+					MP2629_MASK_WDOG_CTRL, 0);
+
+	mutex_init(&charger->lock);
+
+	ret = devm_request_threaded_irq(dev, irq, NULL,	mp2629_irq_handler,
+					IRQF_ONESHOT | IRQF_TRIGGER_RISING,
+					"mp2629-charger", charger);
+	if (ret) {
+		dev_err(dev, "failed to request gpio IRQ\n");
+		return ret;
+	}
+
+	regmap_update_bits(charger->regmap, MP2629_REG_INTERRUPT,
+				GENMASK(6, 5), BIT(6) | BIT(5));
+
+	return 0;
+}
+
+static const struct of_device_id mp2629_charger_of_match[] = {
+	{ .compatible = "mps,mp2629_charger"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, mp2629_charger_of_match);
+
+static struct platform_driver mp2629_charger_driver = {
+	.driver = {
+		.name = "mp2629_charger",
+		.of_match_table = mp2629_charger_of_match,
+	},
+	.probe		= mp2629_charger_probe,
+};
+module_platform_driver(mp2629_charger_driver);
+
+MODULE_AUTHOR("Saravanan Sekar <sravanhome@gmail.com>");
+MODULE_DESCRIPTION("MP2629 Charger driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/olpc_battery.c b/drivers/power/supply/olpc_battery.c
index ad0e9e0edb3f..e0476ec06601 100644
--- a/drivers/power/supply/olpc_battery.c
+++ b/drivers/power/supply/olpc_battery.c
@@ -88,7 +88,7 @@ static enum power_supply_property olpc_ac_props[] = {
 };
 
 static const struct power_supply_desc olpc_ac_desc = {
-	.name = "olpc-ac",
+	.name = "olpc_ac",
 	.type = POWER_SUPPLY_TYPE_MAINS,
 	.properties = olpc_ac_props,
 	.num_properties = ARRAY_SIZE(olpc_ac_props),
@@ -605,7 +605,7 @@ static const struct attribute_group *olpc_bat_sysfs_groups[] = {
  *********************************************************************/
 
 static struct power_supply_desc olpc_bat_desc = {
-	.name = "olpc-battery",
+	.name = "olpc_battery",
 	.get_property = olpc_bat_get_property,
 	.use_for_apm = 1,
 };
diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index 1a9a9fae73d3..02b37fe6061c 100644
--- a/drivers/power/supply/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
@@ -620,10 +620,18 @@ int power_supply_get_battery_info(struct power_supply *psy,
 			     &info->voltage_min_design_uv);
 	of_property_read_u32(battery_np, "voltage-max-design-microvolt",
 			     &info->voltage_max_design_uv);
+	of_property_read_u32(battery_np, "trickle-charge-current-microamp",
+			     &info->tricklecharge_current_ua);
 	of_property_read_u32(battery_np, "precharge-current-microamp",
 			     &info->precharge_current_ua);
+	of_property_read_u32(battery_np, "precharge-upper-limit-microvolt",
+			     &info->precharge_voltage_max_uv);
 	of_property_read_u32(battery_np, "charge-term-current-microamp",
 			     &info->charge_term_current_ua);
+	of_property_read_u32(battery_np, "re-charge-voltage-microvolt",
+			     &info->charge_restart_voltage_uv);
+	of_property_read_u32(battery_np, "over-voltage-threshold-microvolt",
+			     &info->overvoltage_limit_uv);
 	of_property_read_u32(battery_np, "constant-charge-current-max-microamp",
 			     &info->constant_charge_current_max_ua);
 	of_property_read_u32(battery_np, "constant-charge-voltage-max-microvolt",
diff --git a/drivers/power/supply/power_supply_hwmon.c b/drivers/power/supply/power_supply_hwmon.c
index 75cf861ba492..7fe4b6b6ddc8 100644
--- a/drivers/power/supply/power_supply_hwmon.c
+++ b/drivers/power/supply/power_supply_hwmon.c
@@ -13,6 +13,11 @@ struct power_supply_hwmon {
 	unsigned long *props;
 };
 
+static const char *const ps_temp_label[] = {
+	"temp",
+	"ambient temp",
+};
+
 static int power_supply_hwmon_in_to_property(u32 attr)
 {
 	switch (attr) {
@@ -98,6 +103,39 @@ static bool power_supply_hwmon_is_a_label(enum hwmon_sensor_types type,
 	return type == hwmon_temp && attr == hwmon_temp_label;
 }
 
+struct hwmon_type_attr_list {
+	const u32 *attrs;
+	size_t n_attrs;
+};
+
+static const u32 ps_temp_attrs[] = {
+	hwmon_temp_input,
+	hwmon_temp_min, hwmon_temp_max,
+	hwmon_temp_min_alarm, hwmon_temp_max_alarm,
+};
+
+static const struct hwmon_type_attr_list ps_type_attrs[hwmon_max] = {
+	[hwmon_temp] = { ps_temp_attrs, ARRAY_SIZE(ps_temp_attrs) },
+};
+
+static bool power_supply_hwmon_has_input(
+	const struct power_supply_hwmon *psyhw,
+	enum hwmon_sensor_types type, int channel)
+{
+	const struct hwmon_type_attr_list *attr_list = &ps_type_attrs[type];
+	size_t i;
+
+	for (i = 0; i < attr_list->n_attrs; ++i) {
+		int prop = power_supply_hwmon_to_property(type,
+			attr_list->attrs[i], channel);
+
+		if (prop >= 0 && test_bit(prop, psyhw->props))
+			return true;
+	}
+
+	return false;
+}
+
 static bool power_supply_hwmon_is_writable(enum hwmon_sensor_types type,
 					   u32 attr)
 {
@@ -124,9 +162,12 @@ static umode_t power_supply_hwmon_is_visible(const void *data,
 	const struct power_supply_hwmon *psyhw = data;
 	int prop;
 
-
-	if (power_supply_hwmon_is_a_label(type, attr))
-		return 0444;
+	if (power_supply_hwmon_is_a_label(type, attr)) {
+		if (power_supply_hwmon_has_input(psyhw, type, channel))
+			return 0444;
+		else
+			return 0;
+	}
 
 	prop = power_supply_hwmon_to_property(type, attr, channel);
 	if (prop < 0 || !test_bit(prop, psyhw->props))
@@ -144,7 +185,20 @@ static int power_supply_hwmon_read_string(struct device *dev,
 					  u32 attr, int channel,
 					  const char **str)
 {
-	*str = channel ? "temp" : "temp ambient";
+	switch (type) {
+	case hwmon_temp:
+		*str = ps_temp_label[channel];
+		break;
+	default:
+		/* unreachable, but see:
+		 * gcc bug #51513 [1] and clang bug #978 [2]
+		 *
+		 * [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51513
+		 * [2] https://github.com/ClangBuiltLinux/linux/issues/978
+		 */
+		break;
+	}
+
 	return 0;
 }
 
@@ -304,7 +358,7 @@ int power_supply_add_hwmon_sysfs(struct power_supply *psy)
 		goto error;
 	}
 
-	ret = devm_add_action(dev, power_supply_hwmon_bitmap_free,
+	ret = devm_add_action_or_reset(dev, power_supply_hwmon_bitmap_free,
 			      psyhw->props);
 	if (ret)
 		goto error;
diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c
index f37ad4eae60b..bc79560229b5 100644
--- a/drivers/power/supply/power_supply_sysfs.c
+++ b/drivers/power/supply/power_supply_sysfs.c
@@ -18,68 +18,211 @@
 
 #include "power_supply.h"
 
-/*
- * This is because the name "current" breaks the device attr macro.
- * The "current" word resolves to "(get_current())" so instead of
- * "current" "(get_current())" appears in the sysfs.
- *
- * The source of this definition is the device.h which calls __ATTR
- * macro in sysfs.h which calls the __stringify macro.
- *
- * Only modification that the name is not tried to be resolved
- * (as a macro let's say).
- */
+#define MAX_PROP_NAME_LEN 30
+
+struct power_supply_attr {
+	const char *prop_name;
+	char attr_name[MAX_PROP_NAME_LEN + 1];
+	struct device_attribute dev_attr;
+	const char * const *text_values;
+	int text_values_len;
+};
 
-#define POWER_SUPPLY_ATTR(_name)					\
-{									\
-	.attr = { .name = #_name },					\
-	.show = power_supply_show_property,				\
-	.store = power_supply_store_property,				\
+#define _POWER_SUPPLY_ATTR(_name, _text, _len)	\
+[POWER_SUPPLY_PROP_ ## _name] =			\
+{						\
+	.prop_name = #_name,			\
+	.attr_name = #_name "\0",		\
+	.text_values = _text,			\
+	.text_values_len = _len,		\
 }
 
-static struct device_attribute power_supply_attrs[];
+#define POWER_SUPPLY_ATTR(_name) _POWER_SUPPLY_ATTR(_name, NULL, 0)
+#define _POWER_SUPPLY_ENUM_ATTR(_name, _text)	\
+	_POWER_SUPPLY_ATTR(_name, _text, ARRAY_SIZE(_text))
+#define POWER_SUPPLY_ENUM_ATTR(_name)	\
+	_POWER_SUPPLY_ENUM_ATTR(_name, POWER_SUPPLY_ ## _name ## _TEXT)
+
+static const char * const POWER_SUPPLY_TYPE_TEXT[] = {
+	[POWER_SUPPLY_TYPE_UNKNOWN]		= "Unknown",
+	[POWER_SUPPLY_TYPE_BATTERY]		= "Battery",
+	[POWER_SUPPLY_TYPE_UPS]			= "UPS",
+	[POWER_SUPPLY_TYPE_MAINS]		= "Mains",
+	[POWER_SUPPLY_TYPE_USB]			= "USB",
+	[POWER_SUPPLY_TYPE_USB_DCP]		= "USB_DCP",
+	[POWER_SUPPLY_TYPE_USB_CDP]		= "USB_CDP",
+	[POWER_SUPPLY_TYPE_USB_ACA]		= "USB_ACA",
+	[POWER_SUPPLY_TYPE_USB_TYPE_C]		= "USB_C",
+	[POWER_SUPPLY_TYPE_USB_PD]		= "USB_PD",
+	[POWER_SUPPLY_TYPE_USB_PD_DRP]		= "USB_PD_DRP",
+	[POWER_SUPPLY_TYPE_APPLE_BRICK_ID]	= "BrickID",
+};
 
-static const char * const power_supply_type_text[] = {
-	"Unknown", "Battery", "UPS", "Mains", "USB",
-	"USB_DCP", "USB_CDP", "USB_ACA", "USB_C",
-	"USB_PD", "USB_PD_DRP", "BrickID"
+static const char * const POWER_SUPPLY_USB_TYPE_TEXT[] = {
+	[POWER_SUPPLY_USB_TYPE_UNKNOWN]		= "Unknown",
+	[POWER_SUPPLY_USB_TYPE_SDP]		= "SDP",
+	[POWER_SUPPLY_USB_TYPE_DCP]		= "DCP",
+	[POWER_SUPPLY_USB_TYPE_CDP]		= "CDP",
+	[POWER_SUPPLY_USB_TYPE_ACA]		= "ACA",
+	[POWER_SUPPLY_USB_TYPE_C]		= "C",
+	[POWER_SUPPLY_USB_TYPE_PD]		= "PD",
+	[POWER_SUPPLY_USB_TYPE_PD_DRP]		= "PD_DRP",
+	[POWER_SUPPLY_USB_TYPE_PD_PPS]		= "PD_PPS",
+	[POWER_SUPPLY_USB_TYPE_APPLE_BRICK_ID]	= "BrickID",
 };
 
-static const char * const power_supply_usb_type_text[] = {
-	"Unknown", "SDP", "DCP", "CDP", "ACA", "C",
-	"PD", "PD_DRP", "PD_PPS", "BrickID"
+static const char * const POWER_SUPPLY_STATUS_TEXT[] = {
+	[POWER_SUPPLY_STATUS_UNKNOWN]		= "Unknown",
+	[POWER_SUPPLY_STATUS_CHARGING]		= "Charging",
+	[POWER_SUPPLY_STATUS_DISCHARGING]	= "Discharging",
+	[POWER_SUPPLY_STATUS_NOT_CHARGING]	= "Not charging",
+	[POWER_SUPPLY_STATUS_FULL]		= "Full",
 };
 
-static const char * const power_supply_status_text[] = {
-	"Unknown", "Charging", "Discharging", "Not charging", "Full"
+static const char * const POWER_SUPPLY_CHARGE_TYPE_TEXT[] = {
+	[POWER_SUPPLY_CHARGE_TYPE_UNKNOWN]	= "Unknown",
+	[POWER_SUPPLY_CHARGE_TYPE_NONE]		= "N/A",
+	[POWER_SUPPLY_CHARGE_TYPE_TRICKLE]	= "Trickle",
+	[POWER_SUPPLY_CHARGE_TYPE_FAST]		= "Fast",
+	[POWER_SUPPLY_CHARGE_TYPE_STANDARD]	= "Standard",
+	[POWER_SUPPLY_CHARGE_TYPE_ADAPTIVE]	= "Adaptive",
+	[POWER_SUPPLY_CHARGE_TYPE_CUSTOM]	= "Custom",
 };
 
-static const char * const power_supply_charge_type_text[] = {
-	"Unknown", "N/A", "Trickle", "Fast", "Standard", "Adaptive", "Custom"
+static const char * const POWER_SUPPLY_HEALTH_TEXT[] = {
+	[POWER_SUPPLY_HEALTH_UNKNOWN]		    = "Unknown",
+	[POWER_SUPPLY_HEALTH_GOOD]		    = "Good",
+	[POWER_SUPPLY_HEALTH_OVERHEAT]		    = "Overheat",
+	[POWER_SUPPLY_HEALTH_DEAD]		    = "Dead",
+	[POWER_SUPPLY_HEALTH_OVERVOLTAGE]	    = "Over voltage",
+	[POWER_SUPPLY_HEALTH_UNSPEC_FAILURE]	    = "Unspecified failure",
+	[POWER_SUPPLY_HEALTH_COLD]		    = "Cold",
+	[POWER_SUPPLY_HEALTH_WATCHDOG_TIMER_EXPIRE] = "Watchdog timer expire",
+	[POWER_SUPPLY_HEALTH_SAFETY_TIMER_EXPIRE]   = "Safety timer expire",
+	[POWER_SUPPLY_HEALTH_OVERCURRENT]	    = "Over current",
+	[POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED]  = "Calibration required",
 };
 
-static const char * const power_supply_health_text[] = {
-	"Unknown", "Good", "Overheat", "Dead", "Over voltage",
-	"Unspecified failure", "Cold", "Watchdog timer expire",
-	"Safety timer expire", "Over current"
+static const char * const POWER_SUPPLY_TECHNOLOGY_TEXT[] = {
+	[POWER_SUPPLY_TECHNOLOGY_UNKNOWN]	= "Unknown",
+	[POWER_SUPPLY_TECHNOLOGY_NiMH]		= "NiMH",
+	[POWER_SUPPLY_TECHNOLOGY_LION]		= "Li-ion",
+	[POWER_SUPPLY_TECHNOLOGY_LIPO]		= "Li-poly",
+	[POWER_SUPPLY_TECHNOLOGY_LiFe]		= "LiFe",
+	[POWER_SUPPLY_TECHNOLOGY_NiCd]		= "NiCd",
+	[POWER_SUPPLY_TECHNOLOGY_LiMn]		= "LiMn",
 };
 
-static const char * const power_supply_technology_text[] = {
-	"Unknown", "NiMH", "Li-ion", "Li-poly", "LiFe", "NiCd",
-	"LiMn"
+static const char * const POWER_SUPPLY_CAPACITY_LEVEL_TEXT[] = {
+	[POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN]	= "Unknown",
+	[POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL]	= "Critical",
+	[POWER_SUPPLY_CAPACITY_LEVEL_LOW]	= "Low",
+	[POWER_SUPPLY_CAPACITY_LEVEL_NORMAL]	= "Normal",
+	[POWER_SUPPLY_CAPACITY_LEVEL_HIGH]	= "High",
+	[POWER_SUPPLY_CAPACITY_LEVEL_FULL]	= "Full",
 };
 
-static const char * const power_supply_capacity_level_text[] = {
-	"Unknown", "Critical", "Low", "Normal", "High", "Full"
+static const char * const POWER_SUPPLY_SCOPE_TEXT[] = {
+	[POWER_SUPPLY_SCOPE_UNKNOWN]	= "Unknown",
+	[POWER_SUPPLY_SCOPE_SYSTEM]	= "System",
+	[POWER_SUPPLY_SCOPE_DEVICE]	= "Device",
 };
 
-static const char * const power_supply_scope_text[] = {
-	"Unknown", "System", "Device"
+static struct power_supply_attr power_supply_attrs[] = {
+	/* Properties of type `int' */
+	POWER_SUPPLY_ENUM_ATTR(STATUS),
+	POWER_SUPPLY_ENUM_ATTR(CHARGE_TYPE),
+	POWER_SUPPLY_ENUM_ATTR(HEALTH),
+	POWER_SUPPLY_ATTR(PRESENT),
+	POWER_SUPPLY_ATTR(ONLINE),
+	POWER_SUPPLY_ATTR(AUTHENTIC),
+	POWER_SUPPLY_ENUM_ATTR(TECHNOLOGY),
+	POWER_SUPPLY_ATTR(CYCLE_COUNT),
+	POWER_SUPPLY_ATTR(VOLTAGE_MAX),
+	POWER_SUPPLY_ATTR(VOLTAGE_MIN),
+	POWER_SUPPLY_ATTR(VOLTAGE_MAX_DESIGN),
+	POWER_SUPPLY_ATTR(VOLTAGE_MIN_DESIGN),
+	POWER_SUPPLY_ATTR(VOLTAGE_NOW),
+	POWER_SUPPLY_ATTR(VOLTAGE_AVG),
+	POWER_SUPPLY_ATTR(VOLTAGE_OCV),
+	POWER_SUPPLY_ATTR(VOLTAGE_BOOT),
+	POWER_SUPPLY_ATTR(CURRENT_MAX),
+	POWER_SUPPLY_ATTR(CURRENT_NOW),
+	POWER_SUPPLY_ATTR(CURRENT_AVG),
+	POWER_SUPPLY_ATTR(CURRENT_BOOT),
+	POWER_SUPPLY_ATTR(POWER_NOW),
+	POWER_SUPPLY_ATTR(POWER_AVG),
+	POWER_SUPPLY_ATTR(CHARGE_FULL_DESIGN),
+	POWER_SUPPLY_ATTR(CHARGE_EMPTY_DESIGN),
+	POWER_SUPPLY_ATTR(CHARGE_FULL),
+	POWER_SUPPLY_ATTR(CHARGE_EMPTY),
+	POWER_SUPPLY_ATTR(CHARGE_NOW),
+	POWER_SUPPLY_ATTR(CHARGE_AVG),
+	POWER_SUPPLY_ATTR(CHARGE_COUNTER),
+	POWER_SUPPLY_ATTR(CONSTANT_CHARGE_CURRENT),
+	POWER_SUPPLY_ATTR(CONSTANT_CHARGE_CURRENT_MAX),
+	POWER_SUPPLY_ATTR(CONSTANT_CHARGE_VOLTAGE),
+	POWER_SUPPLY_ATTR(CONSTANT_CHARGE_VOLTAGE_MAX),
+	POWER_SUPPLY_ATTR(CHARGE_CONTROL_LIMIT),
+	POWER_SUPPLY_ATTR(CHARGE_CONTROL_LIMIT_MAX),
+	POWER_SUPPLY_ATTR(CHARGE_CONTROL_START_THRESHOLD),
+	POWER_SUPPLY_ATTR(CHARGE_CONTROL_END_THRESHOLD),
+	POWER_SUPPLY_ATTR(INPUT_CURRENT_LIMIT),
+	POWER_SUPPLY_ATTR(INPUT_VOLTAGE_LIMIT),
+	POWER_SUPPLY_ATTR(INPUT_POWER_LIMIT),
+	POWER_SUPPLY_ATTR(ENERGY_FULL_DESIGN),
+	POWER_SUPPLY_ATTR(ENERGY_EMPTY_DESIGN),
+	POWER_SUPPLY_ATTR(ENERGY_FULL),
+	POWER_SUPPLY_ATTR(ENERGY_EMPTY),
+	POWER_SUPPLY_ATTR(ENERGY_NOW),
+	POWER_SUPPLY_ATTR(ENERGY_AVG),
+	POWER_SUPPLY_ATTR(CAPACITY),
+	POWER_SUPPLY_ATTR(CAPACITY_ALERT_MIN),
+	POWER_SUPPLY_ATTR(CAPACITY_ALERT_MAX),
+	POWER_SUPPLY_ATTR(CAPACITY_ERROR_MARGIN),
+	POWER_SUPPLY_ENUM_ATTR(CAPACITY_LEVEL),
+	POWER_SUPPLY_ATTR(TEMP),
+	POWER_SUPPLY_ATTR(TEMP_MAX),
+	POWER_SUPPLY_ATTR(TEMP_MIN),
+	POWER_SUPPLY_ATTR(TEMP_ALERT_MIN),
+	POWER_SUPPLY_ATTR(TEMP_ALERT_MAX),
+	POWER_SUPPLY_ATTR(TEMP_AMBIENT),
+	POWER_SUPPLY_ATTR(TEMP_AMBIENT_ALERT_MIN),
+	POWER_SUPPLY_ATTR(TEMP_AMBIENT_ALERT_MAX),
+	POWER_SUPPLY_ATTR(TIME_TO_EMPTY_NOW),
+	POWER_SUPPLY_ATTR(TIME_TO_EMPTY_AVG),
+	POWER_SUPPLY_ATTR(TIME_TO_FULL_NOW),
+	POWER_SUPPLY_ATTR(TIME_TO_FULL_AVG),
+	POWER_SUPPLY_ENUM_ATTR(TYPE),
+	POWER_SUPPLY_ATTR(USB_TYPE),
+	POWER_SUPPLY_ENUM_ATTR(SCOPE),
+	POWER_SUPPLY_ATTR(PRECHARGE_CURRENT),
+	POWER_SUPPLY_ATTR(CHARGE_TERM_CURRENT),
+	POWER_SUPPLY_ATTR(CALIBRATE),
+	POWER_SUPPLY_ATTR(MANUFACTURE_YEAR),
+	POWER_SUPPLY_ATTR(MANUFACTURE_MONTH),
+	POWER_SUPPLY_ATTR(MANUFACTURE_DAY),
+	/* Properties of type `const char *' */
+	POWER_SUPPLY_ATTR(MODEL_NAME),
+	POWER_SUPPLY_ATTR(MANUFACTURER),
+	POWER_SUPPLY_ATTR(SERIAL_NUMBER),
 };
 
+static struct attribute *
+__power_supply_attrs[ARRAY_SIZE(power_supply_attrs) + 1];
+
+static struct power_supply_attr *to_ps_attr(struct device_attribute *attr)
+{
+	return container_of(attr, struct power_supply_attr, dev_attr);
+}
+
+static enum power_supply_property dev_attr_psp(struct device_attribute *attr)
+{
+	return  to_ps_attr(attr) - power_supply_attrs;
+}
+
 static ssize_t power_supply_show_usb_type(struct device *dev,
-					  enum power_supply_usb_type *usb_types,
-					  ssize_t num_usb_types,
+					  const struct power_supply_desc *desc,
 					  union power_supply_propval *value,
 					  char *buf)
 {
@@ -88,16 +231,16 @@ static ssize_t power_supply_show_usb_type(struct device *dev,
 	bool match = false;
 	int i;
 
-	for (i = 0; i < num_usb_types; ++i) {
-		usb_type = usb_types[i];
+	for (i = 0; i < desc->num_usb_types; ++i) {
+		usb_type = desc->usb_types[i];
 
 		if (value->intval == usb_type) {
 			count += sprintf(buf + count, "[%s] ",
-					 power_supply_usb_type_text[usb_type]);
+					 POWER_SUPPLY_USB_TYPE_TEXT[usb_type]);
 			match = true;
 		} else {
 			count += sprintf(buf + count, "%s ",
-					 power_supply_usb_type_text[usb_type]);
+					 POWER_SUPPLY_USB_TYPE_TEXT[usb_type]);
 		}
 	}
 
@@ -117,7 +260,8 @@ static ssize_t power_supply_show_property(struct device *dev,
 					  char *buf) {
 	ssize_t ret;
 	struct power_supply *psy = dev_get_drvdata(dev);
-	enum power_supply_property psp = attr - power_supply_attrs;
+	struct power_supply_attr *ps_attr = to_ps_attr(attr);
+	enum power_supply_property psp = dev_attr_psp(attr);
 	union power_supply_propval value;
 
 	if (psp == POWER_SUPPLY_PROP_TYPE) {
@@ -137,39 +281,15 @@ static ssize_t power_supply_show_property(struct device *dev,
 		}
 	}
 
+	if (ps_attr->text_values_len > 0 &&
+	    value.intval < ps_attr->text_values_len && value.intval >= 0) {
+		return sprintf(buf, "%s\n", ps_attr->text_values[value.intval]);
+	}
+
 	switch (psp) {
-	case POWER_SUPPLY_PROP_STATUS:
-		ret = sprintf(buf, "%s\n",
-			      power_supply_status_text[value.intval]);
-		break;
-	case POWER_SUPPLY_PROP_CHARGE_TYPE:
-		ret = sprintf(buf, "%s\n",
-			      power_supply_charge_type_text[value.intval]);
-		break;
-	case POWER_SUPPLY_PROP_HEALTH:
-		ret = sprintf(buf, "%s\n",
-			      power_supply_health_text[value.intval]);
-		break;
-	case POWER_SUPPLY_PROP_TECHNOLOGY:
-		ret = sprintf(buf, "%s\n",
-			      power_supply_technology_text[value.intval]);
-		break;
-	case POWER_SUPPLY_PROP_CAPACITY_LEVEL:
-		ret = sprintf(buf, "%s\n",
-			      power_supply_capacity_level_text[value.intval]);
-		break;
-	case POWER_SUPPLY_PROP_TYPE:
-		ret = sprintf(buf, "%s\n",
-			      power_supply_type_text[value.intval]);
-		break;
 	case POWER_SUPPLY_PROP_USB_TYPE:
-		ret = power_supply_show_usb_type(dev, psy->desc->usb_types,
-						 psy->desc->num_usb_types,
-						 &value, buf);
-		break;
-	case POWER_SUPPLY_PROP_SCOPE:
-		ret = sprintf(buf, "%s\n",
-			      power_supply_scope_text[value.intval]);
+		ret = power_supply_show_usb_type(dev, psy->desc,
+						&value, buf);
 		break;
 	case POWER_SUPPLY_PROP_MODEL_NAME ... POWER_SUPPLY_PROP_SERIAL_NUMBER:
 		ret = sprintf(buf, "%s\n", value.strval);
@@ -186,30 +306,14 @@ static ssize_t power_supply_store_property(struct device *dev,
 					   const char *buf, size_t count) {
 	ssize_t ret;
 	struct power_supply *psy = dev_get_drvdata(dev);
-	enum power_supply_property psp = attr - power_supply_attrs;
+	struct power_supply_attr *ps_attr = to_ps_attr(attr);
+	enum power_supply_property psp = dev_attr_psp(attr);
 	union power_supply_propval value;
 
-	switch (psp) {
-	case POWER_SUPPLY_PROP_STATUS:
-		ret = sysfs_match_string(power_supply_status_text, buf);
-		break;
-	case POWER_SUPPLY_PROP_CHARGE_TYPE:
-		ret = sysfs_match_string(power_supply_charge_type_text, buf);
-		break;
-	case POWER_SUPPLY_PROP_HEALTH:
-		ret = sysfs_match_string(power_supply_health_text, buf);
-		break;
-	case POWER_SUPPLY_PROP_TECHNOLOGY:
-		ret = sysfs_match_string(power_supply_technology_text, buf);
-		break;
-	case POWER_SUPPLY_PROP_CAPACITY_LEVEL:
-		ret = sysfs_match_string(power_supply_capacity_level_text, buf);
-		break;
-	case POWER_SUPPLY_PROP_SCOPE:
-		ret = sysfs_match_string(power_supply_scope_text, buf);
-		break;
-	default:
-		ret = -EINVAL;
+	ret = -EINVAL;
+	if (ps_attr->text_values_len > 0) {
+		ret = __sysfs_match_string(ps_attr->text_values,
+					   ps_attr->text_values_len, buf);
 	}
 
 	/*
@@ -235,86 +339,6 @@ static ssize_t power_supply_store_property(struct device *dev,
 	return count;
 }
 
-/* Must be in the same order as POWER_SUPPLY_PROP_* */
-static struct device_attribute power_supply_attrs[] = {
-	/* Properties of type `int' */
-	POWER_SUPPLY_ATTR(status),
-	POWER_SUPPLY_ATTR(charge_type),
-	POWER_SUPPLY_ATTR(health),
-	POWER_SUPPLY_ATTR(present),
-	POWER_SUPPLY_ATTR(online),
-	POWER_SUPPLY_ATTR(authentic),
-	POWER_SUPPLY_ATTR(technology),
-	POWER_SUPPLY_ATTR(cycle_count),
-	POWER_SUPPLY_ATTR(voltage_max),
-	POWER_SUPPLY_ATTR(voltage_min),
-	POWER_SUPPLY_ATTR(voltage_max_design),
-	POWER_SUPPLY_ATTR(voltage_min_design),
-	POWER_SUPPLY_ATTR(voltage_now),
-	POWER_SUPPLY_ATTR(voltage_avg),
-	POWER_SUPPLY_ATTR(voltage_ocv),
-	POWER_SUPPLY_ATTR(voltage_boot),
-	POWER_SUPPLY_ATTR(current_max),
-	POWER_SUPPLY_ATTR(current_now),
-	POWER_SUPPLY_ATTR(current_avg),
-	POWER_SUPPLY_ATTR(current_boot),
-	POWER_SUPPLY_ATTR(power_now),
-	POWER_SUPPLY_ATTR(power_avg),
-	POWER_SUPPLY_ATTR(charge_full_design),
-	POWER_SUPPLY_ATTR(charge_empty_design),
-	POWER_SUPPLY_ATTR(charge_full),
-	POWER_SUPPLY_ATTR(charge_empty),
-	POWER_SUPPLY_ATTR(charge_now),
-	POWER_SUPPLY_ATTR(charge_avg),
-	POWER_SUPPLY_ATTR(charge_counter),
-	POWER_SUPPLY_ATTR(constant_charge_current),
-	POWER_SUPPLY_ATTR(constant_charge_current_max),
-	POWER_SUPPLY_ATTR(constant_charge_voltage),
-	POWER_SUPPLY_ATTR(constant_charge_voltage_max),
-	POWER_SUPPLY_ATTR(charge_control_limit),
-	POWER_SUPPLY_ATTR(charge_control_limit_max),
-	POWER_SUPPLY_ATTR(charge_control_start_threshold),
-	POWER_SUPPLY_ATTR(charge_control_end_threshold),
-	POWER_SUPPLY_ATTR(input_current_limit),
-	POWER_SUPPLY_ATTR(input_voltage_limit),
-	POWER_SUPPLY_ATTR(input_power_limit),
-	POWER_SUPPLY_ATTR(energy_full_design),
-	POWER_SUPPLY_ATTR(energy_empty_design),
-	POWER_SUPPLY_ATTR(energy_full),
-	POWER_SUPPLY_ATTR(energy_empty),
-	POWER_SUPPLY_ATTR(energy_now),
-	POWER_SUPPLY_ATTR(energy_avg),
-	POWER_SUPPLY_ATTR(capacity),
-	POWER_SUPPLY_ATTR(capacity_alert_min),
-	POWER_SUPPLY_ATTR(capacity_alert_max),
-	POWER_SUPPLY_ATTR(capacity_level),
-	POWER_SUPPLY_ATTR(temp),
-	POWER_SUPPLY_ATTR(temp_max),
-	POWER_SUPPLY_ATTR(temp_min),
-	POWER_SUPPLY_ATTR(temp_alert_min),
-	POWER_SUPPLY_ATTR(temp_alert_max),
-	POWER_SUPPLY_ATTR(temp_ambient),
-	POWER_SUPPLY_ATTR(temp_ambient_alert_min),
-	POWER_SUPPLY_ATTR(temp_ambient_alert_max),
-	POWER_SUPPLY_ATTR(time_to_empty_now),
-	POWER_SUPPLY_ATTR(time_to_empty_avg),
-	POWER_SUPPLY_ATTR(time_to_full_now),
-	POWER_SUPPLY_ATTR(time_to_full_avg),
-	POWER_SUPPLY_ATTR(type),
-	POWER_SUPPLY_ATTR(usb_type),
-	POWER_SUPPLY_ATTR(scope),
-	POWER_SUPPLY_ATTR(precharge_current),
-	POWER_SUPPLY_ATTR(charge_term_current),
-	POWER_SUPPLY_ATTR(calibrate),
-	/* Properties of type `const char *' */
-	POWER_SUPPLY_ATTR(model_name),
-	POWER_SUPPLY_ATTR(manufacturer),
-	POWER_SUPPLY_ATTR(serial_number),
-};
-
-static struct attribute *
-__power_supply_attrs[ARRAY_SIZE(power_supply_attrs) + 1];
-
 static umode_t power_supply_attr_is_visible(struct kobject *kobj,
 					   struct attribute *attr,
 					   int attrno)
@@ -324,6 +348,9 @@ static umode_t power_supply_attr_is_visible(struct kobject *kobj,
 	umode_t mode = S_IRUSR | S_IRGRP | S_IROTH;
 	int i;
 
+	if (!power_supply_attrs[attrno].prop_name)
+		return 0;
+
 	if (attrno == POWER_SUPPLY_PROP_TYPE)
 		return mode;
 
@@ -352,31 +379,69 @@ static const struct attribute_group *power_supply_attr_groups[] = {
 	NULL,
 };
 
+static void str_to_lower(char *str)
+{
+	while (*str) {
+		*str = tolower(*str);
+		str++;
+	}
+}
+
 void power_supply_init_attrs(struct device_type *dev_type)
 {
 	int i;
 
 	dev_type->groups = power_supply_attr_groups;
 
-	for (i = 0; i < ARRAY_SIZE(power_supply_attrs); i++)
-		__power_supply_attrs[i] = &power_supply_attrs[i].attr;
-}
+	for (i = 0; i < ARRAY_SIZE(power_supply_attrs); i++) {
+		struct device_attribute *attr;
 
-static char *kstruprdup(const char *str, gfp_t gfp)
-{
-	char *ret, *ustr;
+		if (!power_supply_attrs[i].prop_name) {
+			pr_warn("%s: Property %d skipped because is is missing from power_supply_attrs\n",
+				__func__, i);
+			sprintf(power_supply_attrs[i].attr_name, "_err_%d", i);
+		} else {
+			str_to_lower(power_supply_attrs[i].attr_name);
+		}
 
-	ustr = ret = kmalloc(strlen(str) + 1, gfp);
+		attr = &power_supply_attrs[i].dev_attr;
 
-	if (!ret)
-		return NULL;
+		attr->attr.name = power_supply_attrs[i].attr_name;
+		attr->show = power_supply_show_property;
+		attr->store = power_supply_store_property;
+		__power_supply_attrs[i] = &attr->attr;
+	}
+}
+
+static int add_prop_uevent(struct device *dev, struct kobj_uevent_env *env,
+			   enum power_supply_property prop, char *prop_buf)
+{
+	int ret = 0;
+	struct power_supply_attr *pwr_attr;
+	struct device_attribute *dev_attr;
+	char *line;
+
+	pwr_attr = &power_supply_attrs[prop];
+	dev_attr = &pwr_attr->dev_attr;
+
+	ret = power_supply_show_property(dev, dev_attr, prop_buf);
+	if (ret == -ENODEV || ret == -ENODATA) {
+		/*
+		 * When a battery is absent, we expect -ENODEV. Don't abort;
+		 * send the uevent with at least the the PRESENT=0 property
+		 */
+		return 0;
+	}
 
-	while (*str)
-		*ustr++ = toupper(*str++);
+	if (ret < 0)
+		return ret;
 
-	*ustr = 0;
+	line = strchr(prop_buf, '\n');
+	if (line)
+		*line = 0;
 
-	return ret;
+	return add_uevent_var(env, "POWER_SUPPLY_%s=%s",
+			      pwr_attr->prop_name, prop_buf);
 }
 
 int power_supply_uevent(struct device *dev, struct kobj_uevent_env *env)
@@ -384,7 +449,6 @@ int power_supply_uevent(struct device *dev, struct kobj_uevent_env *env)
 	struct power_supply *psy = dev_get_drvdata(dev);
 	int ret = 0, j;
 	char *prop_buf;
-	char *attrname;
 
 	if (!psy || !psy->desc) {
 		dev_dbg(dev, "No power supply yet\n");
@@ -399,35 +463,13 @@ int power_supply_uevent(struct device *dev, struct kobj_uevent_env *env)
 	if (!prop_buf)
 		return -ENOMEM;
 
-	for (j = 0; j < psy->desc->num_properties; j++) {
-		struct device_attribute *attr;
-		char *line;
-
-		attr = &power_supply_attrs[psy->desc->properties[j]];
-
-		ret = power_supply_show_property(dev, attr, prop_buf);
-		if (ret == -ENODEV || ret == -ENODATA) {
-			/* When a battery is absent, we expect -ENODEV. Don't abort;
-			   send the uevent with at least the the PRESENT=0 property */
-			ret = 0;
-			continue;
-		}
-
-		if (ret < 0)
-			goto out;
-
-		line = strchr(prop_buf, '\n');
-		if (line)
-			*line = 0;
-
-		attrname = kstruprdup(attr->attr.name, GFP_KERNEL);
-		if (!attrname) {
-			ret = -ENOMEM;
-			goto out;
-		}
+	ret = add_prop_uevent(dev, env, POWER_SUPPLY_PROP_TYPE, prop_buf);
+	if (ret)
+		goto out;
 
-		ret = add_uevent_var(env, "POWER_SUPPLY_%s=%s", attrname, prop_buf);
-		kfree(attrname);
+	for (j = 0; j < psy->desc->num_properties; j++) {
+		ret = add_prop_uevent(dev, env, psy->desc->properties[j],
+				      prop_buf);
 		if (ret)
 			goto out;
 	}
diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c
index 6acd242eed48..83b9924033bd 100644
--- a/drivers/power/supply/sbs-battery.c
+++ b/drivers/power/supply/sbs-battery.c
@@ -14,7 +14,7 @@
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/of_device.h>
 #include <linux/power/sbs-battery.h>
 #include <linux/power_supply.h>
@@ -23,9 +23,12 @@
 
 enum {
 	REG_MANUFACTURER_DATA,
+	REG_BATTERY_MODE,
 	REG_TEMPERATURE,
 	REG_VOLTAGE,
-	REG_CURRENT,
+	REG_CURRENT_NOW,
+	REG_CURRENT_AVG,
+	REG_MAX_ERR,
 	REG_CAPACITY,
 	REG_TIME_TO_EMPTY,
 	REG_TIME_TO_FULL,
@@ -41,10 +44,15 @@ enum {
 	REG_DESIGN_CAPACITY_CHARGE,
 	REG_DESIGN_VOLTAGE_MIN,
 	REG_DESIGN_VOLTAGE_MAX,
+	REG_CHEMISTRY,
 	REG_MANUFACTURER,
 	REG_MODEL_NAME,
+	REG_CHARGE_CURRENT,
+	REG_CHARGE_VOLTAGE,
 };
 
+#define REG_ADDR_MANUFACTURE_DATE	0x1B
+
 /* Battery Mode defines */
 #define BATTERY_MODE_OFFSET		0x03
 #define BATTERY_MODE_CAPACITY_MASK	BIT(15)
@@ -52,6 +60,7 @@ enum sbs_capacity_mode {
 	CAPACITY_MODE_AMPS = 0,
 	CAPACITY_MODE_WATTS = BATTERY_MODE_CAPACITY_MASK
 };
+#define BATTERY_MODE_CHARGER_MASK	(1<<14)
 
 /* manufacturer access defines */
 #define MANUFACTURER_ACCESS_STATUS	0x0006
@@ -79,12 +88,18 @@ static const struct chip_data {
 } sbs_data[] = {
 	[REG_MANUFACTURER_DATA] =
 		SBS_DATA(POWER_SUPPLY_PROP_PRESENT, 0x00, 0, 65535),
+	[REG_BATTERY_MODE] =
+		SBS_DATA(-1, 0x03, 0, 65535),
 	[REG_TEMPERATURE] =
 		SBS_DATA(POWER_SUPPLY_PROP_TEMP, 0x08, 0, 65535),
 	[REG_VOLTAGE] =
 		SBS_DATA(POWER_SUPPLY_PROP_VOLTAGE_NOW, 0x09, 0, 20000),
-	[REG_CURRENT] =
+	[REG_CURRENT_NOW] =
 		SBS_DATA(POWER_SUPPLY_PROP_CURRENT_NOW, 0x0A, -32768, 32767),
+	[REG_CURRENT_AVG] =
+		SBS_DATA(POWER_SUPPLY_PROP_CURRENT_AVG, 0x0B, -32768, 32767),
+	[REG_MAX_ERR] =
+		SBS_DATA(POWER_SUPPLY_PROP_CAPACITY_ERROR_MARGIN, 0x0c, 0, 100),
 	[REG_CAPACITY] =
 		SBS_DATA(POWER_SUPPLY_PROP_CAPACITY, 0x0D, 0, 100),
 	[REG_REMAINING_CAPACITY] =
@@ -99,6 +114,10 @@ static const struct chip_data {
 		SBS_DATA(POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG, 0x12, 0, 65535),
 	[REG_TIME_TO_FULL] =
 		SBS_DATA(POWER_SUPPLY_PROP_TIME_TO_FULL_AVG, 0x13, 0, 65535),
+	[REG_CHARGE_CURRENT] =
+		SBS_DATA(POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, 0x14, 0, 65535),
+	[REG_CHARGE_VOLTAGE] =
+		SBS_DATA(POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX, 0x15, 0, 65535),
 	[REG_STATUS] =
 		SBS_DATA(POWER_SUPPLY_PROP_STATUS, 0x16, 0, 65535),
 	[REG_CAPACITY_LEVEL] =
@@ -119,10 +138,12 @@ static const struct chip_data {
 	[REG_MANUFACTURER] =
 		SBS_DATA(POWER_SUPPLY_PROP_MANUFACTURER, 0x20, 0, 65535),
 	[REG_MODEL_NAME] =
-		SBS_DATA(POWER_SUPPLY_PROP_MODEL_NAME, 0x21, 0, 65535)
+		SBS_DATA(POWER_SUPPLY_PROP_MODEL_NAME, 0x21, 0, 65535),
+	[REG_CHEMISTRY] =
+		SBS_DATA(POWER_SUPPLY_PROP_TECHNOLOGY, 0x22, 0, 65535)
 };
 
-static enum power_supply_property sbs_properties[] = {
+static const enum power_supply_property sbs_properties[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_HEALTH,
@@ -131,7 +152,9 @@ static enum power_supply_property sbs_properties[] = {
 	POWER_SUPPLY_PROP_CYCLE_COUNT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
 	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CURRENT_AVG,
 	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_CAPACITY_ERROR_MARGIN,
 	POWER_SUPPLY_PROP_TEMP,
 	POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG,
 	POWER_SUPPLY_PROP_TIME_TO_FULL_AVG,
@@ -144,13 +167,18 @@ static enum power_supply_property sbs_properties[] = {
 	POWER_SUPPLY_PROP_CHARGE_NOW,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX,
+	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX,
+	POWER_SUPPLY_PROP_MANUFACTURE_YEAR,
+	POWER_SUPPLY_PROP_MANUFACTURE_MONTH,
+	POWER_SUPPLY_PROP_MANUFACTURE_DAY,
 	/* Properties of type `const char *' */
 	POWER_SUPPLY_PROP_MANUFACTURER,
 	POWER_SUPPLY_PROP_MODEL_NAME
 };
 
-/* Supports special manufacturer commands from TI BQ20Z75 IC. */
-#define SBS_FLAGS_TI_BQ20Z75		BIT(0)
+/* Supports special manufacturer commands from TI BQ20Z65 and BQ20Z75 IC. */
+#define SBS_FLAGS_TI_BQ20ZX5		BIT(0)
 
 struct sbs_info {
 	struct i2c_client		*client;
@@ -158,6 +186,7 @@ struct sbs_info {
 	bool				is_present;
 	struct gpio_desc		*gpio_detect;
 	bool				enable_detection;
+	bool				charger_broadcasts;
 	int				last_state;
 	int				poll_time;
 	u32				i2c_retry_count;
@@ -169,8 +198,48 @@ struct sbs_info {
 
 static char model_name[I2C_SMBUS_BLOCK_MAX + 1];
 static char manufacturer[I2C_SMBUS_BLOCK_MAX + 1];
+static char chemistry[I2C_SMBUS_BLOCK_MAX + 1];
 static bool force_load;
 
+static int sbs_read_word_data(struct i2c_client *client, u8 address);
+static int sbs_write_word_data(struct i2c_client *client, u8 address, u16 value);
+
+static void sbs_disable_charger_broadcasts(struct sbs_info *chip)
+{
+	int val = sbs_read_word_data(chip->client, BATTERY_MODE_OFFSET);
+	if (val < 0)
+		goto exit;
+
+	val |= BATTERY_MODE_CHARGER_MASK;
+
+	val = sbs_write_word_data(chip->client, BATTERY_MODE_OFFSET, val);
+
+exit:
+	if (val < 0)
+		dev_err(&chip->client->dev,
+			"Failed to disable charger broadcasting: %d\n", val);
+	else
+		dev_dbg(&chip->client->dev, "%s\n", __func__);
+}
+
+static int sbs_update_presence(struct sbs_info *chip, bool is_present)
+{
+	if (chip->is_present == is_present)
+		return 0;
+
+	if (!is_present) {
+		chip->is_present = false;
+		return 0;
+	}
+
+	if (!chip->is_present && is_present && !chip->charger_broadcasts)
+		sbs_disable_charger_broadcasts(chip);
+
+	chip->is_present = true;
+
+	return 0;
+}
+
 static int sbs_read_word_data(struct i2c_client *client, u8 address)
 {
 	struct sbs_info *chip = i2c_get_clientdata(client);
@@ -288,15 +357,15 @@ static int sbs_status_correct(struct i2c_client *client, int *intval)
 {
 	int ret;
 
-	ret = sbs_read_word_data(client, sbs_data[REG_CURRENT].addr);
+	ret = sbs_read_word_data(client, sbs_data[REG_CURRENT_NOW].addr);
 	if (ret < 0)
 		return ret;
 
 	ret = (s16)ret;
 
-	/* Not drawing current means full (cannot be not charging) */
-	if (ret == 0)
-		*intval = POWER_SUPPLY_STATUS_FULL;
+	/* Not drawing current -> not charging (i.e. idle) */
+	if (*intval != POWER_SUPPLY_STATUS_FULL && ret == 0)
+		*intval = POWER_SUPPLY_STATUS_NOT_CHARGING;
 
 	if (*intval == POWER_SUPPLY_STATUS_FULL) {
 		/* Drawing or providing current when full */
@@ -309,6 +378,17 @@ static int sbs_status_correct(struct i2c_client *client, int *intval)
 	return 0;
 }
 
+static bool sbs_bat_needs_calibration(struct i2c_client *client)
+{
+	int ret;
+
+	ret = sbs_read_word_data(client, sbs_data[REG_BATTERY_MODE].addr);
+	if (ret < 0)
+		return false;
+
+	return !!(ret & BIT(7));
+}
+
 static int sbs_get_battery_presence_and_health(
 	struct i2c_client *client, enum power_supply_property psp,
 	union power_supply_propval *val)
@@ -328,9 +408,14 @@ static int sbs_get_battery_presence_and_health(
 
 	if (psp == POWER_SUPPLY_PROP_PRESENT)
 		val->intval = 1; /* battery present */
-	else /* POWER_SUPPLY_PROP_HEALTH */
-		/* SBS spec doesn't have a general health command. */
-		val->intval = POWER_SUPPLY_HEALTH_UNKNOWN;
+	else { /* POWER_SUPPLY_PROP_HEALTH */
+		if (sbs_bat_needs_calibration(client)) {
+			val->intval = POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED;
+		} else {
+			/* SBS spec doesn't have a general health command. */
+			val->intval = POWER_SUPPLY_HEALTH_UNKNOWN;
+		}
+	}
 
 	return 0;
 }
@@ -384,6 +469,8 @@ static int sbs_get_ti_battery_presence_and_health(
 			val->intval = POWER_SUPPLY_HEALTH_OVERHEAT;
 		else if (ret == 0x0C)
 			val->intval = POWER_SUPPLY_HEALTH_DEAD;
+		else if (sbs_bat_needs_calibration(client))
+			val->intval = POWER_SUPPLY_HEALTH_CALIBRATION_REQUIRED;
 		else
 			val->intval = POWER_SUPPLY_HEALTH_GOOD;
 	}
@@ -492,7 +579,10 @@ static void  sbs_unit_adjustment(struct i2c_client *client,
 	case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN:
 	case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN:
 	case POWER_SUPPLY_PROP_CURRENT_NOW:
+	case POWER_SUPPLY_PROP_CURRENT_AVG:
 	case POWER_SUPPLY_PROP_CHARGE_NOW:
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX:
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX:
 	case POWER_SUPPLY_PROP_CHARGE_FULL:
 	case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN:
 		val->intval *= BASE_UNIT_CONVERSION;
@@ -602,6 +692,70 @@ static int sbs_get_property_index(struct i2c_client *client,
 	return -EINVAL;
 }
 
+static int sbs_get_chemistry(struct i2c_client *client,
+		union power_supply_propval *val)
+{
+	enum power_supply_property psp = POWER_SUPPLY_PROP_TECHNOLOGY;
+	int ret;
+
+	ret = sbs_get_property_index(client, psp);
+	if (ret < 0)
+		return ret;
+
+	ret = sbs_get_battery_string_property(client, ret, psp,
+					      chemistry);
+	if (ret < 0)
+		return ret;
+
+	if (!strncasecmp(chemistry, "LION", 4))
+		val->intval = POWER_SUPPLY_TECHNOLOGY_LION;
+	else if (!strncasecmp(chemistry, "LiP", 3))
+		val->intval = POWER_SUPPLY_TECHNOLOGY_LIPO;
+	else if (!strncasecmp(chemistry, "NiCd", 4))
+		val->intval = POWER_SUPPLY_TECHNOLOGY_NiCd;
+	else if (!strncasecmp(chemistry, "NiMH", 4))
+		val->intval = POWER_SUPPLY_TECHNOLOGY_NiMH;
+	else
+		val->intval = POWER_SUPPLY_TECHNOLOGY_UNKNOWN;
+
+	if (val->intval == POWER_SUPPLY_TECHNOLOGY_UNKNOWN)
+		dev_warn(&client->dev, "Unknown chemistry: %s\n", chemistry);
+
+	return 0;
+}
+
+static int sbs_get_battery_manufacture_date(struct i2c_client *client,
+	enum power_supply_property psp,
+	union power_supply_propval *val)
+{
+	int ret;
+	u16 day, month, year;
+
+	ret = sbs_read_word_data(client, REG_ADDR_MANUFACTURE_DATE);
+	if (ret < 0)
+		return ret;
+
+	day   = ret   & GENMASK(4,  0);
+	month = (ret  & GENMASK(8,  5)) >> 5;
+	year  = ((ret & GENMASK(15, 9)) >> 9) + 1980;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_MANUFACTURE_YEAR:
+		val->intval = year;
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURE_MONTH:
+		val->intval = month;
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURE_DAY:
+		val->intval = day;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int sbs_get_property(struct power_supply *psy,
 	enum power_supply_property psp,
 	union power_supply_propval *val)
@@ -616,7 +770,7 @@ static int sbs_get_property(struct power_supply *psy,
 			return ret;
 		if (psp == POWER_SUPPLY_PROP_PRESENT) {
 			val->intval = ret;
-			chip->is_present = val->intval;
+			sbs_update_presence(chip, ret);
 			return 0;
 		}
 		if (ret == 0)
@@ -626,7 +780,7 @@ static int sbs_get_property(struct power_supply *psy,
 	switch (psp) {
 	case POWER_SUPPLY_PROP_PRESENT:
 	case POWER_SUPPLY_PROP_HEALTH:
-		if (chip->flags & SBS_FLAGS_TI_BQ20Z75)
+		if (chip->flags & SBS_FLAGS_TI_BQ20ZX5)
 			ret = sbs_get_ti_battery_presence_and_health(client,
 								     psp, val);
 		else
@@ -639,7 +793,10 @@ static int sbs_get_property(struct power_supply *psy,
 		break;
 
 	case POWER_SUPPLY_PROP_TECHNOLOGY:
-		val->intval = POWER_SUPPLY_TECHNOLOGY_LION;
+		ret = sbs_get_chemistry(client, val);
+		if (ret < 0)
+			break;
+
 		goto done; /* don't trigger power_supply_changed()! */
 
 	case POWER_SUPPLY_PROP_ENERGY_NOW:
@@ -670,12 +827,16 @@ static int sbs_get_property(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_CYCLE_COUNT:
 	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
 	case POWER_SUPPLY_PROP_CURRENT_NOW:
+	case POWER_SUPPLY_PROP_CURRENT_AVG:
 	case POWER_SUPPLY_PROP_TEMP:
 	case POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG:
 	case POWER_SUPPLY_PROP_TIME_TO_FULL_AVG:
 	case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN:
 	case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN:
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX:
+	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX:
 	case POWER_SUPPLY_PROP_CAPACITY:
+	case POWER_SUPPLY_PROP_CAPACITY_ERROR_MARGIN:
 		ret = sbs_get_property_index(client, psp);
 		if (ret < 0)
 			break;
@@ -703,6 +864,12 @@ static int sbs_get_property(struct power_supply *psy,
 		val->strval = manufacturer;
 		break;
 
+	case POWER_SUPPLY_PROP_MANUFACTURE_YEAR:
+	case POWER_SUPPLY_PROP_MANUFACTURE_MONTH:
+	case POWER_SUPPLY_PROP_MANUFACTURE_DAY:
+		ret = sbs_get_battery_manufacture_date(client, psp, val);
+		break;
+
 	default:
 		dev_err(&client->dev,
 			"%s: INVALID property\n", __func__);
@@ -714,7 +881,7 @@ static int sbs_get_property(struct power_supply *psy,
 
 	if (!chip->gpio_detect &&
 		chip->is_present != (ret >= 0)) {
-		chip->is_present = (ret >= 0);
+		sbs_update_presence(chip, (ret >= 0));
 		power_supply_changed(chip->power_supply);
 	}
 
@@ -745,7 +912,7 @@ static void sbs_supply_changed(struct sbs_info *chip)
 	ret = gpiod_get_value_cansleep(chip->gpio_detect);
 	if (ret < 0)
 		return;
-	chip->is_present = ret;
+	sbs_update_presence(chip, ret);
 	power_supply_changed(battery);
 }
 
@@ -815,8 +982,7 @@ static const struct power_supply_desc sbs_default_desc = {
 	.external_power_changed = sbs_external_power_changed,
 };
 
-static int sbs_probe(struct i2c_client *client,
-	const struct i2c_device_id *id)
+static int sbs_probe(struct i2c_client *client)
 {
 	struct sbs_info *chip;
 	struct power_supply_desc *sbs_desc;
@@ -839,7 +1005,7 @@ static int sbs_probe(struct i2c_client *client,
 	if (!chip)
 		return -ENOMEM;
 
-	chip->flags = (u32)(uintptr_t)of_device_get_match_data(&client->dev);
+	chip->flags = (u32)(uintptr_t)device_get_match_data(&client->dev);
 	chip->client = client;
 	chip->enable_detection = false;
 	psy_cfg.of_node = client->dev.of_node;
@@ -850,13 +1016,13 @@ static int sbs_probe(struct i2c_client *client,
 	/* use pdata if available, fall back to DT properties,
 	 * or hardcoded defaults if not
 	 */
-	rc = of_property_read_u32(client->dev.of_node, "sbs,i2c-retry-count",
-				  &chip->i2c_retry_count);
+	rc = device_property_read_u32(&client->dev, "sbs,i2c-retry-count",
+				      &chip->i2c_retry_count);
 	if (rc)
 		chip->i2c_retry_count = 0;
 
-	rc = of_property_read_u32(client->dev.of_node, "sbs,poll-retry-count",
-				  &chip->poll_retry_count);
+	rc = device_property_read_u32(&client->dev, "sbs,poll-retry-count",
+				      &chip->poll_retry_count);
 	if (rc)
 		chip->poll_retry_count = 0;
 
@@ -866,6 +1032,9 @@ static int sbs_probe(struct i2c_client *client,
 	}
 	chip->i2c_retry_count = chip->i2c_retry_count + 1;
 
+	chip->charger_broadcasts = !device_property_read_bool(&client->dev,
+					"sbs,disable-charger-broadcasts");
+
 	chip->gpio_detect = devm_gpiod_get_optional(&client->dev,
 			"sbs,battery-detect", GPIOD_IN);
 	if (IS_ERR(chip->gpio_detect)) {
@@ -950,7 +1119,7 @@ static int sbs_suspend(struct device *dev)
 	if (chip->poll_time > 0)
 		cancel_delayed_work_sync(&chip->work);
 
-	if (chip->flags & SBS_FLAGS_TI_BQ20Z75) {
+	if (chip->flags & SBS_FLAGS_TI_BQ20ZX5) {
 		/* Write to manufacturer access with sleep command. */
 		ret = sbs_write_word_data(client,
 					  sbs_data[REG_MANUFACTURER_DATA].addr,
@@ -970,6 +1139,7 @@ static SIMPLE_DEV_PM_OPS(sbs_pm_ops, sbs_suspend, NULL);
 #endif
 
 static const struct i2c_device_id sbs_id[] = {
+	{ "bq20z65", 0 },
 	{ "bq20z75", 0 },
 	{ "sbs-battery", 1 },
 	{}
@@ -979,15 +1149,19 @@ MODULE_DEVICE_TABLE(i2c, sbs_id);
 static const struct of_device_id sbs_dt_ids[] = {
 	{ .compatible = "sbs,sbs-battery" },
 	{
+		.compatible = "ti,bq20z65",
+		.data = (void *)SBS_FLAGS_TI_BQ20ZX5,
+	},
+	{
 		.compatible = "ti,bq20z75",
-		.data = (void *)SBS_FLAGS_TI_BQ20Z75,
+		.data = (void *)SBS_FLAGS_TI_BQ20ZX5,
 	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sbs_dt_ids);
 
 static struct i2c_driver sbs_battery_driver = {
-	.probe		= sbs_probe,
+	.probe_new	= sbs_probe,
 	.remove		= sbs_remove,
 	.alert		= sbs_alert,
 	.id_table	= sbs_id,
diff --git a/drivers/power/supply/sc27xx_fuel_gauge.c b/drivers/power/supply/sc27xx_fuel_gauge.c
index a7c8a8453db1..be42e814ea34 100644
--- a/drivers/power/supply/sc27xx_fuel_gauge.c
+++ b/drivers/power/supply/sc27xx_fuel_gauge.c
@@ -42,6 +42,8 @@
 #define SC27XX_FGU_USER_AREA_SET	0xa0
 #define SC27XX_FGU_USER_AREA_CLEAR	0xa4
 #define SC27XX_FGU_USER_AREA_STATUS	0xa8
+#define SC27XX_FGU_VOLTAGE_BUF		0xd0
+#define SC27XX_FGU_CURRENT_BUF		0xf0
 
 #define SC27XX_WRITE_SELCLB_EN		BIT(0)
 #define SC27XX_FGU_CLBCNT_MASK		GENMASK(15, 0)
@@ -82,6 +84,7 @@
  * @init_clbcnt: the initial coulomb counter
  * @max_volt: the maximum constant input voltage in millivolt
  * @min_volt: the minimum drained battery voltage in microvolt
+ * @boot_volt: the voltage measured during boot in microvolt
  * @table_len: the capacity table length
  * @resist_table_len: the resistance table length
  * @cur_1000ma_adc: ADC value corresponding to 1000 mA
@@ -107,6 +110,7 @@ struct sc27xx_fgu_data {
 	int init_clbcnt;
 	int max_volt;
 	int min_volt;
+	int boot_volt;
 	int table_len;
 	int resist_table_len;
 	int cur_1000ma_adc;
@@ -319,6 +323,7 @@ static int sc27xx_fgu_get_boot_capacity(struct sc27xx_fgu_data *data, int *cap)
 
 	volt = sc27xx_fgu_adc_to_voltage(data, volt);
 	ocv = volt * 1000 - oci * data->internal_resist;
+	data->boot_volt = ocv;
 
 	/*
 	 * Parse the capacity table to look up the correct capacity percent
@@ -376,6 +381,44 @@ static int sc27xx_fgu_get_clbcnt(struct sc27xx_fgu_data *data, int *clb_cnt)
 	return 0;
 }
 
+static int sc27xx_fgu_get_vol_now(struct sc27xx_fgu_data *data, int *val)
+{
+	int ret;
+	u32 vol;
+
+	ret = regmap_read(data->regmap, data->base + SC27XX_FGU_VOLTAGE_BUF,
+			  &vol);
+	if (ret)
+		return ret;
+
+	/*
+	 * It is ADC values reading from registers which need to convert to
+	 * corresponding voltage values.
+	 */
+	*val = sc27xx_fgu_adc_to_voltage(data, vol);
+
+	return 0;
+}
+
+static int sc27xx_fgu_get_cur_now(struct sc27xx_fgu_data *data, int *val)
+{
+	int ret;
+	u32 cur;
+
+	ret = regmap_read(data->regmap, data->base + SC27XX_FGU_CURRENT_BUF,
+			  &cur);
+	if (ret)
+		return ret;
+
+	/*
+	 * It is ADC values reading from registers which need to convert to
+	 * corresponding current values.
+	 */
+	*val = sc27xx_fgu_adc_to_current(data, cur - SC27XX_FGU_CUR_BASIC_ADC);
+
+	return 0;
+}
+
 static int sc27xx_fgu_get_capacity(struct sc27xx_fgu_data *data, int *cap)
 {
 	int ret, cur_clbcnt, delta_clbcnt, delta_cap, temp;
@@ -577,7 +620,7 @@ static int sc27xx_fgu_get_property(struct power_supply *psy,
 		val->intval = value;
 		break;
 
-	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+	case POWER_SUPPLY_PROP_VOLTAGE_AVG:
 		ret = sc27xx_fgu_get_vbat_vol(data, &value);
 		if (ret)
 			goto error;
@@ -601,7 +644,6 @@ static int sc27xx_fgu_get_property(struct power_supply *psy,
 		val->intval = value;
 		break;
 
-	case POWER_SUPPLY_PROP_CURRENT_NOW:
 	case POWER_SUPPLY_PROP_CURRENT_AVG:
 		ret = sc27xx_fgu_get_current(data, &value);
 		if (ret)
@@ -625,6 +667,26 @@ static int sc27xx_fgu_get_property(struct power_supply *psy,
 
 		break;
 
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		ret = sc27xx_fgu_get_vol_now(data, &value);
+		if (ret)
+			goto error;
+
+		val->intval = value * 1000;
+		break;
+
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		ret = sc27xx_fgu_get_cur_now(data, &value);
+		if (ret)
+			goto error;
+
+		val->intval = value * 1000;
+		break;
+
+	case POWER_SUPPLY_PROP_VOLTAGE_BOOT:
+		val->intval = data->boot_volt;
+		break;
+
 	default:
 		ret = -EINVAL;
 		break;
@@ -656,6 +718,11 @@ static int sc27xx_fgu_set_property(struct power_supply *psy,
 		ret = 0;
 		break;
 
+	case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN:
+		data->total_cap = val->intval / 1000;
+		ret = 0;
+		break;
+
 	default:
 		ret = -EINVAL;
 	}
@@ -676,7 +743,8 @@ static int sc27xx_fgu_property_is_writeable(struct power_supply *psy,
 					    enum power_supply_property psp)
 {
 	return psp == POWER_SUPPLY_PROP_CAPACITY ||
-		psp == POWER_SUPPLY_PROP_CALIBRATE;
+		psp == POWER_SUPPLY_PROP_CALIBRATE ||
+		psp == POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN;
 }
 
 static enum power_supply_property sc27xx_fgu_props[] = {
@@ -688,6 +756,8 @@ static enum power_supply_property sc27xx_fgu_props[] = {
 	POWER_SUPPLY_PROP_CAPACITY,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
 	POWER_SUPPLY_PROP_VOLTAGE_OCV,
+	POWER_SUPPLY_PROP_VOLTAGE_AVG,
+	POWER_SUPPLY_PROP_VOLTAGE_BOOT,
 	POWER_SUPPLY_PROP_CURRENT_NOW,
 	POWER_SUPPLY_PROP_CURRENT_AVG,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE,
@@ -705,6 +775,7 @@ static const struct power_supply_desc sc27xx_fgu_desc = {
 	.set_property		= sc27xx_fgu_set_property,
 	.external_power_changed	= sc27xx_fgu_external_power_changed,
 	.property_is_writeable	= sc27xx_fgu_property_is_writeable,
+	.no_thermal		= true,
 };
 
 static void sc27xx_fgu_adjust_cap(struct sc27xx_fgu_data *data, int cap)
diff --git a/drivers/power/supply/smb347-charger.c b/drivers/power/supply/smb347-charger.c
index c1d124b8be0c..f99026d81f2a 100644
--- a/drivers/power/supply/smb347-charger.c
+++ b/drivers/power/supply/smb347-charger.c
@@ -8,6 +8,7 @@
  *          Mika Westerberg <mika.westerberg@linux.intel.com>
  */
 
+#include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/gpio.h>
 #include <linux/kernel.h>
@@ -708,6 +709,9 @@ static irqreturn_t smb347_interrupt(int irq, void *data)
 	bool handled = false;
 	int ret;
 
+	/* SMB347 it needs at least 20ms for setting IRQSTAT_E_*IN_UV_IRQ */
+	usleep_range(25000, 35000);
+
 	ret = regmap_read(smb->regmap, STAT_C, &stat_c);
 	if (ret < 0) {
 		dev_warn(smb->dev, "reading STAT_C failed\n");
@@ -1138,6 +1142,7 @@ static bool smb347_volatile_reg(struct device *dev, unsigned int reg)
 	switch (reg) {
 	case IRQSTAT_A:
 	case IRQSTAT_C:
+	case IRQSTAT_D:
 	case IRQSTAT_E:
 	case IRQSTAT_F:
 	case STAT_A:
diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig
index dc1c1381d7fa..ebc4d4578339 100644
--- a/drivers/powercap/Kconfig
+++ b/drivers/powercap/Kconfig
@@ -23,7 +23,7 @@ config INTEL_RAPL
 	tristate "Intel RAPL Support via MSR Interface"
 	depends on X86 && IOSF_MBI
 	select INTEL_RAPL_CORE
-	---help---
+	help
 	  This enables support for the Intel Running Average Power Limit (RAPL)
 	  technology via MSR interface, which allows power limits to be enforced
 	  and monitored on modern Intel processors (Sandy Bridge and later).
diff --git a/drivers/powercap/idle_inject.c b/drivers/powercap/idle_inject.c
index e9bbd3c42eef..c90f0990968b 100644
--- a/drivers/powercap/idle_inject.c
+++ b/drivers/powercap/idle_inject.c
@@ -61,12 +61,14 @@ struct idle_inject_thread {
  * @timer: idle injection period timer
  * @idle_duration_us: duration of CPU idle time to inject
  * @run_duration_us: duration of CPU run time to allow
+ * @latency_us: max allowed latency
  * @cpumask: mask of CPUs affected by idle injection
  */
 struct idle_inject_device {
 	struct hrtimer timer;
 	unsigned int idle_duration_us;
 	unsigned int run_duration_us;
+	unsigned int latency_us;
 	unsigned long cpumask[];
 };
 
@@ -138,7 +140,8 @@ static void idle_inject_fn(unsigned int cpu)
 	 */
 	iit->should_run = 0;
 
-	play_idle(READ_ONCE(ii_dev->idle_duration_us));
+	play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
+			  READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
 }
 
 /**
@@ -170,6 +173,16 @@ void idle_inject_get_duration(struct idle_inject_device *ii_dev,
 }
 
 /**
+ * idle_inject_set_latency - set the maximum latency allowed
+ * @latency_us: set the latency requirement for the idle state
+ */
+void idle_inject_set_latency(struct idle_inject_device *ii_dev,
+			     unsigned int latency_us)
+{
+	WRITE_ONCE(ii_dev->latency_us, latency_us);
+}
+
+/**
  * idle_inject_start - start idle injections
  * @ii_dev: idle injection control device structure
  *
@@ -297,6 +310,7 @@ struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
 	cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
 	hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	ii_dev->timer.function = idle_inject_timer_fn;
+	ii_dev->latency_us = UINT_MAX;
 
 	for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
 
diff --git a/drivers/pps/Kconfig b/drivers/pps/Kconfig
index afbf5e2b06b7..e1651d51cfc9 100644
--- a/drivers/pps/Kconfig
+++ b/drivers/pps/Kconfig
@@ -5,7 +5,7 @@
 
 menuconfig PPS
 	tristate "PPS support"
-	---help---
+	help
 	  PPS (Pulse Per Second) is a special pulse provided by some GPS
 	  antennae. Userland can use it to get a high-precision time
 	  reference.
diff --git a/drivers/ps3/ps3-lpm.c b/drivers/ps3/ps3-lpm.c
index 83c45659bc9d..e54aa2d82f50 100644
--- a/drivers/ps3/ps3-lpm.c
+++ b/drivers/ps3/ps3-lpm.c
@@ -1096,8 +1096,8 @@ int ps3_lpm_open(enum ps3_lpm_tb_type tb_type, void *tb_cache,
 		lpm_priv->tb_cache_internal = NULL;
 		lpm_priv->tb_cache = NULL;
 	} else if (tb_cache) {
-		if (tb_cache != (void *)_ALIGN_UP((unsigned long)tb_cache, 128)
-			|| tb_cache_size != _ALIGN_UP(tb_cache_size, 128)) {
+		if (tb_cache != (void *)ALIGN((unsigned long)tb_cache, 128)
+			|| tb_cache_size != ALIGN(tb_cache_size, 128)) {
 			dev_err(sbd_core(), "%s:%u: unaligned tb_cache\n",
 				__func__, __LINE__);
 			result = -EINVAL;
@@ -1111,12 +1111,10 @@ int ps3_lpm_open(enum ps3_lpm_tb_type tb_type, void *tb_cache,
 		lpm_priv->tb_cache_internal = kzalloc(
 			lpm_priv->tb_cache_size + 127, GFP_KERNEL);
 		if (!lpm_priv->tb_cache_internal) {
-			dev_err(sbd_core(), "%s:%u: alloc internal tb_cache "
-				"failed\n", __func__, __LINE__);
 			result = -ENOMEM;
 			goto fail_malloc;
 		}
-		lpm_priv->tb_cache = (void *)_ALIGN_UP(
+		lpm_priv->tb_cache = (void *)ALIGN(
 			(unsigned long)lpm_priv->tb_cache_internal, 128);
 	}
 
diff --git a/drivers/ps3/ps3-vuart.c b/drivers/ps3/ps3-vuart.c
index ddaa5ea5801a..4ed131eaff51 100644
--- a/drivers/ps3/ps3-vuart.c
+++ b/drivers/ps3/ps3-vuart.c
@@ -858,13 +858,13 @@ static int ps3_vuart_handle_port_interrupt(struct ps3_system_bus_device *dev)
 	return 0;
 }
 
-struct vuart_bus_priv {
+static struct vuart_bus_priv {
 	struct ports_bmp *bmp;
 	unsigned int virq;
 	struct mutex probe_mutex;
 	int use_count;
 	struct ps3_system_bus_device *devices[PORT_COUNT];
-} static vuart_bus_priv;
+} vuart_bus_priv;
 
 /**
  * ps3_vuart_irq_handler - first stage interrupt handler
@@ -917,7 +917,6 @@ static int ps3_vuart_bus_interrupt_get(void)
 	vuart_bus_priv.bmp = kzalloc(sizeof(struct ports_bmp), GFP_KERNEL);
 
 	if (!vuart_bus_priv.bmp) {
-		pr_debug("%s:%d: kzalloc failed.\n", __func__, __LINE__);
 		result = -ENOMEM;
 		goto fail_bmp_malloc;
 	}
diff --git a/drivers/ptp/Kconfig b/drivers/ptp/Kconfig
index 86400c708150..942f72d8151d 100644
--- a/drivers/ptp/Kconfig
+++ b/drivers/ptp/Kconfig
@@ -64,7 +64,7 @@ config DP83640_PHY
 	depends on NETWORK_PHY_TIMESTAMPING
 	depends on PHYLIB
 	depends on PTP_1588_CLOCK
-	---help---
+	help
 	  Supports the DP83640 PHYTER with IEEE 1588 features.
 
 	  This driver adds support for using the DP83640 as a PTP
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index eebbc917ac97..cb8d739067d2 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -232,9 +232,19 @@ config PWM_IMX_TPM
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-imx-tpm.
 
+config PWM_IQS620A
+	tristate "Azoteq IQS620A PWM support"
+	depends on MFD_IQS62X || COMPILE_TEST
+	help
+	  Generic PWM framework driver for the Azoteq IQS620A multi-function
+	  sensor.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called pwm-iqs620a.
+
 config PWM_JZ4740
 	tristate "Ingenic JZ47xx PWM support"
-	depends on MACH_INGENIC
+	depends on MIPS
 	depends on COMMON_CLK
 	select MFD_SYSCON
 	help
diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile
index 9a475073dafc..a59c710e98c7 100644
--- a/drivers/pwm/Makefile
+++ b/drivers/pwm/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_PWM_IMG)		+= pwm-img.o
 obj-$(CONFIG_PWM_IMX1)		+= pwm-imx1.o
 obj-$(CONFIG_PWM_IMX27)		+= pwm-imx27.o
 obj-$(CONFIG_PWM_IMX_TPM)	+= pwm-imx-tpm.o
+obj-$(CONFIG_PWM_IQS620A)	+= pwm-iqs620a.o
 obj-$(CONFIG_PWM_JZ4740)	+= pwm-jz4740.o
 obj-$(CONFIG_PWM_LP3943)	+= pwm-lp3943.o
 obj-$(CONFIG_PWM_LPC18XX_SCT)	+= pwm-lpc18xx-sct.o
diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c
index 9973c442b455..004b2ea9b5fd 100644
--- a/drivers/pwm/core.c
+++ b/drivers/pwm/core.c
@@ -121,7 +121,7 @@ static int pwm_device_request(struct pwm_device *pwm, const char *label)
 		pwm->chip->ops->get_state(pwm->chip, pwm, &pwm->state);
 		trace_pwm_get(pwm, &pwm->state);
 
-		if (IS_ENABLED(PWM_DEBUG))
+		if (IS_ENABLED(CONFIG_PWM_DEBUG))
 			pwm->last = pwm->state;
 	}
 
@@ -537,7 +537,7 @@ static void pwm_apply_state_debug(struct pwm_device *pwm,
 
 	if (!state->enabled && s2.enabled && s2.duty_cycle > 0)
 		dev_warn(chip->dev,
-			 "requested disabled, but yielded enabled with duty > 0");
+			 "requested disabled, but yielded enabled with duty > 0\n");
 
 	/* reapply the state that the driver reported being configured. */
 	err = chip->ops->apply(chip, pwm, &s1);
diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c
index c9e57bd109fb..599a0f66a384 100644
--- a/drivers/pwm/pwm-img.c
+++ b/drivers/pwm/pwm-img.c
@@ -129,8 +129,10 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	duty = DIV_ROUND_UP(timebase * duty_ns, period_ns);
 
 	ret = pm_runtime_get_sync(chip->dev);
-	if (ret < 0)
+	if (ret < 0) {
+		pm_runtime_put_autosuspend(chip->dev);
 		return ret;
+	}
 
 	val = img_pwm_readl(pwm_chip, PWM_CTRL_CFG);
 	val &= ~(PWM_CTRL_CFG_DIV_MASK << PWM_CTRL_CFG_DIV_SHIFT(pwm->hwpwm));
@@ -331,8 +333,10 @@ static int img_pwm_remove(struct platform_device *pdev)
 	int ret;
 
 	ret = pm_runtime_get_sync(&pdev->dev);
-	if (ret < 0)
+	if (ret < 0) {
+		pm_runtime_put(&pdev->dev);
 		return ret;
+	}
 
 	for (i = 0; i < pwm_chip->chip.npwm; i++) {
 		val = img_pwm_readl(pwm_chip, PWM_CTRL_CFG);
diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c
index a6e40d4c485f..732a6f3701e8 100644
--- a/drivers/pwm/pwm-imx27.c
+++ b/drivers/pwm/pwm-imx27.c
@@ -150,13 +150,12 @@ static void pwm_imx27_get_state(struct pwm_chip *chip,
 
 	prescaler = MX3_PWMCR_PRESCALER_GET(val);
 	pwm_clk = clk_get_rate(imx->clk_per);
-	pwm_clk = DIV_ROUND_CLOSEST_ULL(pwm_clk, prescaler);
 	val = readl(imx->mmio_base + MX3_PWMPR);
 	period = val >= MX3_PWMPR_MAX ? MX3_PWMPR_MAX : val;
 
 	/* PWMOUT (Hz) = PWMCLK / (PWMPR + 2) */
-	tmp = NSEC_PER_SEC * (u64)(period + 2);
-	state->period = DIV_ROUND_CLOSEST_ULL(tmp, pwm_clk);
+	tmp = NSEC_PER_SEC * (u64)(period + 2) * prescaler;
+	state->period = DIV_ROUND_UP_ULL(tmp, pwm_clk);
 
 	/*
 	 * PWMSAR can be read only if PWM is enabled. If the PWM is disabled,
@@ -167,8 +166,8 @@ static void pwm_imx27_get_state(struct pwm_chip *chip,
 	else
 		val = imx->duty_cycle;
 
-	tmp = NSEC_PER_SEC * (u64)(val);
-	state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, pwm_clk);
+	tmp = NSEC_PER_SEC * (u64)(val) * prescaler;
+	state->duty_cycle = DIV_ROUND_UP_ULL(tmp, pwm_clk);
 
 	pwm_imx27_clk_disable_unprepare(imx);
 }
@@ -220,22 +219,23 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 	struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip);
 	struct pwm_state cstate;
 	unsigned long long c;
+	unsigned long long clkrate;
 	int ret;
 	u32 cr;
 
 	pwm_get_state(pwm, &cstate);
 
-	c = clk_get_rate(imx->clk_per);
-	c *= state->period;
+	clkrate = clk_get_rate(imx->clk_per);
+	c = clkrate * state->period;
 
-	do_div(c, 1000000000);
+	do_div(c, NSEC_PER_SEC);
 	period_cycles = c;
 
 	prescale = period_cycles / 0x10000 + 1;
 
 	period_cycles /= prescale;
-	c = (unsigned long long)period_cycles * state->duty_cycle;
-	do_div(c, state->period);
+	c = clkrate * state->duty_cycle;
+	do_div(c, NSEC_PER_SEC * prescale);
 	duty_cycles = c;
 
 	/*
diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c
new file mode 100644
index 000000000000..674f0e238ba0
--- /dev/null
+++ b/drivers/pwm/pwm-iqs620a.c
@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Azoteq IQS620A PWM Generator
+ *
+ * Copyright (C) 2019 Jeff LaBundy <jeff@labundy.com>
+ *
+ * Limitations:
+ * - The period is fixed to 1 ms and is generated continuously despite changes
+ *   to the duty cycle or enable/disable state.
+ * - Changes to the duty cycle or enable/disable state take effect immediately
+ *   and may result in a glitch during the period in which the change is made.
+ * - The device cannot generate a 0% duty cycle. For duty cycles below 1 / 256
+ *   ms, the output is disabled and relies upon an external pull-down resistor
+ *   to hold the GPIO3/LTX pin low.
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/mfd/iqs62x.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/notifier.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define IQS620_PWR_SETTINGS			0xD2
+#define IQS620_PWR_SETTINGS_PWM_OUT		BIT(7)
+
+#define IQS620_PWM_DUTY_CYCLE			0xD8
+
+#define IQS620_PWM_PERIOD_NS			1000000
+
+struct iqs620_pwm_private {
+	struct iqs62x_core *iqs62x;
+	struct pwm_chip chip;
+	struct notifier_block notifier;
+	struct mutex lock;
+	bool out_en;
+	u8 duty_val;
+};
+
+static int iqs620_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
+			    const struct pwm_state *state)
+{
+	struct iqs620_pwm_private *iqs620_pwm;
+	struct iqs62x_core *iqs62x;
+	int duty_scale, ret;
+
+	if (state->polarity != PWM_POLARITY_NORMAL)
+		return -ENOTSUPP;
+
+	if (state->period < IQS620_PWM_PERIOD_NS)
+		return -EINVAL;
+
+	iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip);
+	iqs62x = iqs620_pwm->iqs62x;
+
+	/*
+	 * The duty cycle generated by the device is calculated as follows:
+	 *
+	 * duty_cycle = (IQS620_PWM_DUTY_CYCLE + 1) / 256 * 1 ms
+	 *
+	 * ...where IQS620_PWM_DUTY_CYCLE is a register value between 0 and 255
+	 * (inclusive). Therefore the lowest duty cycle the device can generate
+	 * while the output is enabled is 1 / 256 ms.
+	 *
+	 * For lower duty cycles (e.g. 0), the PWM output is simply disabled to
+	 * allow an external pull-down resistor to hold the GPIO3/LTX pin low.
+	 */
+	duty_scale = state->duty_cycle * 256 / IQS620_PWM_PERIOD_NS;
+
+	mutex_lock(&iqs620_pwm->lock);
+
+	if (!state->enabled || !duty_scale) {
+		ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS,
+					 IQS620_PWR_SETTINGS_PWM_OUT, 0);
+		if (ret)
+			goto err_mutex;
+	}
+
+	if (duty_scale) {
+		u8 duty_val = min(duty_scale - 1, 0xFF);
+
+		ret = regmap_write(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE,
+				   duty_val);
+		if (ret)
+			goto err_mutex;
+
+		iqs620_pwm->duty_val = duty_val;
+	}
+
+	if (state->enabled && duty_scale) {
+		ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS,
+					 IQS620_PWR_SETTINGS_PWM_OUT, 0xFF);
+		if (ret)
+			goto err_mutex;
+	}
+
+	iqs620_pwm->out_en = state->enabled;
+
+err_mutex:
+	mutex_unlock(&iqs620_pwm->lock);
+
+	return ret;
+}
+
+static void iqs620_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
+				 struct pwm_state *state)
+{
+	struct iqs620_pwm_private *iqs620_pwm;
+
+	iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip);
+
+	mutex_lock(&iqs620_pwm->lock);
+
+	/*
+	 * Since the device cannot generate a 0% duty cycle, requests to do so
+	 * cause subsequent calls to iqs620_pwm_get_state to report the output
+	 * as disabled with duty cycle equal to that which was in use prior to
+	 * the request. This is not ideal, but is the best compromise based on
+	 * the capabilities of the device.
+	 */
+	state->enabled = iqs620_pwm->out_en;
+	state->duty_cycle = DIV_ROUND_UP((iqs620_pwm->duty_val + 1) *
+					 IQS620_PWM_PERIOD_NS, 256);
+
+	mutex_unlock(&iqs620_pwm->lock);
+
+	state->period = IQS620_PWM_PERIOD_NS;
+}
+
+static int iqs620_pwm_notifier(struct notifier_block *notifier,
+			       unsigned long event_flags, void *context)
+{
+	struct iqs620_pwm_private *iqs620_pwm;
+	struct iqs62x_core *iqs62x;
+	int ret;
+
+	if (!(event_flags & BIT(IQS62X_EVENT_SYS_RESET)))
+		return NOTIFY_DONE;
+
+	iqs620_pwm = container_of(notifier, struct iqs620_pwm_private,
+				  notifier);
+	iqs62x = iqs620_pwm->iqs62x;
+
+	mutex_lock(&iqs620_pwm->lock);
+
+	/*
+	 * The parent MFD driver already prints an error message in the event
+	 * of a device reset, so nothing else is printed here unless there is
+	 * an additional failure.
+	 */
+	ret = regmap_write(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE,
+			   iqs620_pwm->duty_val);
+	if (ret)
+		goto err_mutex;
+
+	ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS,
+				 IQS620_PWR_SETTINGS_PWM_OUT,
+				 iqs620_pwm->out_en ? 0xFF : 0);
+
+err_mutex:
+	mutex_unlock(&iqs620_pwm->lock);
+
+	if (ret) {
+		dev_err(iqs620_pwm->chip.dev,
+			"Failed to re-initialize device: %d\n", ret);
+		return NOTIFY_BAD;
+	}
+
+	return NOTIFY_OK;
+}
+
+static const struct pwm_ops iqs620_pwm_ops = {
+	.apply = iqs620_pwm_apply,
+	.get_state = iqs620_pwm_get_state,
+	.owner = THIS_MODULE,
+};
+
+static void iqs620_pwm_notifier_unregister(void *context)
+{
+	struct iqs620_pwm_private *iqs620_pwm = context;
+	int ret;
+
+	ret = blocking_notifier_chain_unregister(&iqs620_pwm->iqs62x->nh,
+						 &iqs620_pwm->notifier);
+	if (ret)
+		dev_err(iqs620_pwm->chip.dev,
+			"Failed to unregister notifier: %d\n", ret);
+}
+
+static int iqs620_pwm_probe(struct platform_device *pdev)
+{
+	struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent);
+	struct iqs620_pwm_private *iqs620_pwm;
+	unsigned int val;
+	int ret;
+
+	iqs620_pwm = devm_kzalloc(&pdev->dev, sizeof(*iqs620_pwm), GFP_KERNEL);
+	if (!iqs620_pwm)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, iqs620_pwm);
+	iqs620_pwm->iqs62x = iqs62x;
+
+	ret = regmap_read(iqs62x->regmap, IQS620_PWR_SETTINGS, &val);
+	if (ret)
+		return ret;
+	iqs620_pwm->out_en = val & IQS620_PWR_SETTINGS_PWM_OUT;
+
+	ret = regmap_read(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, &val);
+	if (ret)
+		return ret;
+	iqs620_pwm->duty_val = val;
+
+	iqs620_pwm->chip.dev = &pdev->dev;
+	iqs620_pwm->chip.ops = &iqs620_pwm_ops;
+	iqs620_pwm->chip.base = -1;
+	iqs620_pwm->chip.npwm = 1;
+
+	mutex_init(&iqs620_pwm->lock);
+
+	iqs620_pwm->notifier.notifier_call = iqs620_pwm_notifier;
+	ret = blocking_notifier_chain_register(&iqs620_pwm->iqs62x->nh,
+					       &iqs620_pwm->notifier);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(&pdev->dev,
+				       iqs620_pwm_notifier_unregister,
+				       iqs620_pwm);
+	if (ret)
+		return ret;
+
+	ret = pwmchip_add(&iqs620_pwm->chip);
+	if (ret)
+		dev_err(&pdev->dev, "Failed to add device: %d\n", ret);
+
+	return ret;
+}
+
+static int iqs620_pwm_remove(struct platform_device *pdev)
+{
+	struct iqs620_pwm_private *iqs620_pwm = platform_get_drvdata(pdev);
+	int ret;
+
+	ret = pwmchip_remove(&iqs620_pwm->chip);
+	if (ret)
+		dev_err(&pdev->dev, "Failed to remove device: %d\n", ret);
+
+	return ret;
+}
+
+static struct platform_driver iqs620_pwm_platform_driver = {
+	.driver = {
+		.name = "iqs620a-pwm",
+	},
+	.probe = iqs620_pwm_probe,
+	.remove = iqs620_pwm_remove,
+};
+module_platform_driver(iqs620_pwm_platform_driver);
+
+MODULE_AUTHOR("Jeff LaBundy <jeff@labundy.com>");
+MODULE_DESCRIPTION("Azoteq IQS620A PWM Generator");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:iqs620a-pwm");
diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c
index 3cd5c054ad9a..5830ac2bdf6a 100644
--- a/drivers/pwm/pwm-jz4740.c
+++ b/drivers/pwm/pwm-jz4740.c
@@ -6,7 +6,6 @@
  * Limitations:
  * - The .apply callback doesn't complete the currently running period before
  *   reconfiguring the hardware.
- * - Each period starts with the inactive part.
  */
 
 #include <linux/clk.h>
@@ -21,7 +20,9 @@
 #include <linux/pwm.h>
 #include <linux/regmap.h>
 
-#define NUM_PWM 8
+struct soc_info {
+	unsigned int num_pwms;
+};
 
 struct jz4740_pwm_chip {
 	struct pwm_chip chip;
@@ -37,7 +38,7 @@ static bool jz4740_pwm_can_use_chn(struct jz4740_pwm_chip *jz,
 				   unsigned int channel)
 {
 	/* Enable all TCU channels for PWM use by default except channels 0/1 */
-	u32 pwm_channels_mask = GENMASK(NUM_PWM - 1, 2);
+	u32 pwm_channels_mask = GENMASK(jz->chip.npwm - 1, 2);
 
 	device_property_read_u32(jz->chip.dev->parent,
 				 "ingenic,pwm-channels-mask",
@@ -158,12 +159,12 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 	/* Calculate period value */
 	tmp = (unsigned long long)rate * state->period;
 	do_div(tmp, NSEC_PER_SEC);
-	period = (unsigned long)tmp;
+	period = tmp;
 
 	/* Calculate duty value */
-	tmp = (unsigned long long)period * state->duty_cycle;
-	do_div(tmp, state->period);
-	duty = period - tmp;
+	tmp = (unsigned long long)rate * state->duty_cycle;
+	do_div(tmp, NSEC_PER_SEC);
+	duty = tmp;
 
 	if (duty >= period)
 		duty = period - 1;
@@ -189,18 +190,26 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 	regmap_update_bits(jz4740->map, TCU_REG_TCSRc(pwm->hwpwm),
 			   TCU_TCSR_PWM_SD, TCU_TCSR_PWM_SD);
 
-	/* Set polarity */
-	switch (state->polarity) {
-	case PWM_POLARITY_NORMAL:
+	/*
+	 * Set polarity.
+	 *
+	 * The PWM starts in inactive state until the internal timer reaches the
+	 * duty value, then becomes active until the timer reaches the period
+	 * value. In theory, we should then use (period - duty) as the real duty
+	 * value, as a high duty value would otherwise result in the PWM pin
+	 * being inactive most of the time.
+	 *
+	 * Here, we don't do that, and instead invert the polarity of the PWM
+	 * when it is active. This trick makes the PWM start with its active
+	 * state instead of its inactive state.
+	 */
+	if ((state->polarity == PWM_POLARITY_NORMAL) ^ state->enabled)
 		regmap_update_bits(jz4740->map, TCU_REG_TCSRc(pwm->hwpwm),
 				   TCU_TCSR_PWM_INITL_HIGH, 0);
-		break;
-	case PWM_POLARITY_INVERSED:
+	else
 		regmap_update_bits(jz4740->map, TCU_REG_TCSRc(pwm->hwpwm),
 				   TCU_TCSR_PWM_INITL_HIGH,
 				   TCU_TCSR_PWM_INITL_HIGH);
-		break;
-	}
 
 	if (state->enabled)
 		jz4740_pwm_enable(chip, pwm);
@@ -219,6 +228,11 @@ static int jz4740_pwm_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct jz4740_pwm_chip *jz4740;
+	const struct soc_info *info;
+
+	info = device_get_match_data(dev);
+	if (!info)
+		return -EINVAL;
 
 	jz4740 = devm_kzalloc(dev, sizeof(*jz4740), GFP_KERNEL);
 	if (!jz4740)
@@ -232,7 +246,7 @@ static int jz4740_pwm_probe(struct platform_device *pdev)
 
 	jz4740->chip.dev = dev;
 	jz4740->chip.ops = &jz4740_pwm_ops;
-	jz4740->chip.npwm = NUM_PWM;
+	jz4740->chip.npwm = info->num_pwms;
 	jz4740->chip.base = -1;
 	jz4740->chip.of_xlate = of_pwm_xlate_with_flags;
 	jz4740->chip.of_pwm_n_cells = 3;
@@ -249,9 +263,18 @@ static int jz4740_pwm_remove(struct platform_device *pdev)
 	return pwmchip_remove(&jz4740->chip);
 }
 
+static const struct soc_info __maybe_unused jz4740_soc_info = {
+	.num_pwms = 8,
+};
+
+static const struct soc_info __maybe_unused jz4725b_soc_info = {
+	.num_pwms = 6,
+};
+
 #ifdef CONFIG_OF
 static const struct of_device_id jz4740_pwm_dt_ids[] = {
-	{ .compatible = "ingenic,jz4740-pwm", },
+	{ .compatible = "ingenic,jz4740-pwm", .data = &jz4740_soc_info },
+	{ .compatible = "ingenic,jz4725b-pwm", .data = &jz4725b_soc_info },
 	{},
 };
 MODULE_DEVICE_TABLE(of, jz4740_pwm_dt_ids);
diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c
index 75bbfe5f3bc2..9d965ffe66d1 100644
--- a/drivers/pwm/pwm-lpss.c
+++ b/drivers/pwm/pwm-lpss.c
@@ -158,7 +158,6 @@ static int pwm_lpss_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 	return 0;
 }
 
-/* This function gets called once from pwmchip_add to get the initial state */
 static void pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
 			       struct pwm_state *state)
 {
@@ -167,6 +166,8 @@ static void pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
 	unsigned long long base_unit, freq, on_time_div;
 	u32 ctrl;
 
+	pm_runtime_get_sync(chip->dev);
+
 	base_unit_range = BIT(lpwm->info->base_unit_bits);
 
 	ctrl = pwm_lpss_read(pwm);
@@ -187,8 +188,7 @@ static void pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
 	state->polarity = PWM_POLARITY_NORMAL;
 	state->enabled = !!(ctrl & PWM_ENABLE);
 
-	if (state->enabled)
-		pm_runtime_get(chip->dev);
+	pm_runtime_put(chip->dev);
 }
 
 static const struct pwm_ops pwm_lpss_ops = {
@@ -202,7 +202,8 @@ struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r,
 {
 	struct pwm_lpss_chip *lpwm;
 	unsigned long c;
-	int ret;
+	int i, ret;
+	u32 ctrl;
 
 	if (WARN_ON(info->npwm > MAX_PWMS))
 		return ERR_PTR(-ENODEV);
@@ -232,6 +233,12 @@ struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r,
 		return ERR_PTR(ret);
 	}
 
+	for (i = 0; i < lpwm->info->npwm; i++) {
+		ctrl = pwm_lpss_read(&lpwm->chip.pwms[i]);
+		if (ctrl & PWM_ENABLE)
+			pm_runtime_get(dev);
+	}
+
 	return lpwm;
 }
 EXPORT_SYMBOL_GPL(pwm_lpss_probe);
diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c
index 73352e6fbccb..eb8c9cb645a6 100644
--- a/drivers/pwm/pwm-rockchip.c
+++ b/drivers/pwm/pwm-rockchip.c
@@ -83,12 +83,7 @@ static void rockchip_pwm_get_state(struct pwm_chip *chip,
 	state->duty_cycle =  DIV_ROUND_CLOSEST_ULL(tmp, clk_rate);
 
 	val = readl_relaxed(pc->base + pc->data->regs.ctrl);
-	if (pc->data->supports_polarity)
-		state->enabled = ((val & enable_conf) != enable_conf) ?
-				 false : true;
-	else
-		state->enabled = ((val & enable_conf) == enable_conf) ?
-				 true : false;
+	state->enabled = (val & enable_conf) == enable_conf;
 
 	if (pc->data->supports_polarity && !(val & PWM_DUTY_POSITIVE))
 		state->polarity = PWM_POLARITY_INVERSED;
diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c
index 5c677c563349..18fbbe3277d0 100644
--- a/drivers/pwm/pwm-sun4i.c
+++ b/drivers/pwm/pwm-sun4i.c
@@ -352,6 +352,12 @@ static const struct sun4i_pwm_data sun4i_pwm_single_bypass = {
 	.npwm = 1,
 };
 
+static const struct sun4i_pwm_data sun50i_a64_pwm_data = {
+	.has_prescaler_bypass = true,
+	.has_direct_mod_clk_output = true,
+	.npwm = 1,
+};
+
 static const struct sun4i_pwm_data sun50i_h6_pwm_data = {
 	.has_prescaler_bypass = true,
 	.has_direct_mod_clk_output = true,
@@ -375,6 +381,9 @@ static const struct of_device_id sun4i_pwm_dt_ids[] = {
 		.compatible = "allwinner,sun8i-h3-pwm",
 		.data = &sun4i_pwm_single_bypass,
 	}, {
+		.compatible = "allwinner,sun50i-a64-pwm",
+		.data = &sun50i_a64_pwm_data,
+	}, {
 		.compatible = "allwinner,sun50i-h6-pwm",
 		.data = &sun50i_h6_pwm_data,
 	}, {
diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c
index d26ed8f579ff..1daf591025c0 100644
--- a/drivers/pwm/pwm-tegra.c
+++ b/drivers/pwm/pwm-tegra.c
@@ -4,8 +4,36 @@
  *
  * Tegra pulse-width-modulation controller driver
  *
- * Copyright (c) 2010, NVIDIA Corporation.
+ * Copyright (c) 2010-2020, NVIDIA Corporation.
  * Based on arch/arm/plat-mxc/pwm.c by Sascha Hauer <s.hauer@pengutronix.de>
+ *
+ * Overview of Tegra Pulse Width Modulator Register:
+ * 1. 13-bit: Frequency division (SCALE)
+ * 2. 8-bit : Pulse division (DUTY)
+ * 3. 1-bit : Enable bit
+ *
+ * The PWM clock frequency is divided by 256 before subdividing it based
+ * on the programmable frequency division value to generate the required
+ * frequency for PWM output. The maximum output frequency that can be
+ * achieved is (max rate of source clock) / 256.
+ * e.g. if source clock rate is 408 MHz, maximum output frequency can be:
+ * 408 MHz/256 = 1.6 MHz.
+ * This 1.6 MHz frequency can further be divided using SCALE value in PWM.
+ *
+ * PWM pulse width: 8 bits are usable [23:16] for varying pulse width.
+ * To achieve 100% duty cycle, program Bit [24] of this register to
+ * 1’b1. In which case the other bits [23:16] are set to don't care.
+ *
+ * Limitations:
+ * -	When PWM is disabled, the output is driven to inactive.
+ * -	It does not allow the current PWM period to complete and
+ *	stops abruptly.
+ *
+ * -	If the register is reconfigured while PWM is running,
+ *	it does not complete the currently running period.
+ *
+ * -	If the user input duty is beyond acceptible limits,
+ *	-EINVAL is returned.
  */
 
 #include <linux/clk.h>
@@ -41,6 +69,7 @@ struct tegra_pwm_chip {
 	struct reset_control*rst;
 
 	unsigned long clk_rate;
+	unsigned long min_period_ns;
 
 	void __iomem *regs;
 
@@ -68,7 +97,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 {
 	struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip);
 	unsigned long long c = duty_ns, hz;
-	unsigned long rate;
+	unsigned long rate, required_clk_rate;
 	u32 val = 0;
 	int err;
 
@@ -83,9 +112,47 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	val = (u32)c << PWM_DUTY_SHIFT;
 
 	/*
+	 *  min period = max clock limit >> PWM_DUTY_WIDTH
+	 */
+	if (period_ns < pc->min_period_ns)
+		return -EINVAL;
+
+	/*
 	 * Compute the prescaler value for which (1 << PWM_DUTY_WIDTH)
 	 * cycles at the PWM clock rate will take period_ns nanoseconds.
+	 *
+	 * num_channels: If single instance of PWM controller has multiple
+	 * channels (e.g. Tegra210 or older) then it is not possible to
+	 * configure separate clock rates to each of the channels, in such
+	 * case the value stored during probe will be referred.
+	 *
+	 * If every PWM controller instance has one channel respectively, i.e.
+	 * nums_channels == 1 then only the clock rate can be modified
+	 * dynamically (e.g. Tegra186 or Tegra194).
 	 */
+	if (pc->soc->num_channels == 1) {
+		/*
+		 * Rate is multiplied with 2^PWM_DUTY_WIDTH so that it matches
+		 * with the maximum possible rate that the controller can
+		 * provide. Any further lower value can be derived by setting
+		 * PFM bits[0:12].
+		 *
+		 * required_clk_rate is a reference rate for source clock and
+		 * it is derived based on user requested period. By setting the
+		 * source clock rate as required_clk_rate, PWM controller will
+		 * be able to configure the requested period.
+		 */
+		required_clk_rate =
+			(NSEC_PER_SEC / period_ns) << PWM_DUTY_WIDTH;
+
+		err = clk_set_rate(pc->clk, required_clk_rate);
+		if (err < 0)
+			return -EINVAL;
+
+		/* Store the new rate for further references */
+		pc->clk_rate = clk_get_rate(pc->clk);
+	}
+
 	rate = pc->clk_rate >> PWM_DUTY_WIDTH;
 
 	/* Consider precision in PWM_SCALE_WIDTH rate calculation */
@@ -94,7 +161,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 
 	/*
 	 * Since the actual PWM divider is the register's frequency divider
-	 * field minus 1, we need to decrement to get the correct value to
+	 * field plus 1, we need to decrement to get the correct value to
 	 * write to the register.
 	 */
 	if (rate > 0)
@@ -205,6 +272,10 @@ static int tegra_pwm_probe(struct platform_device *pdev)
 	 */
 	pwm->clk_rate = clk_get_rate(pwm->clk);
 
+	/* Set minimum limit of PWM period for the IP */
+	pwm->min_period_ns =
+	    (NSEC_PER_SEC / (pwm->soc->max_frequency >> PWM_DUTY_WIDTH)) + 1;
+
 	pwm->rst = devm_reset_control_get_exclusive(&pdev->dev, "pwm");
 	if (IS_ERR(pwm->rst)) {
 		ret = PTR_ERR(pwm->rst);
@@ -312,5 +383,6 @@ static struct platform_driver tegra_pwm_driver = {
 module_platform_driver(tegra_pwm_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("NVIDIA Corporation");
+MODULE_AUTHOR("Sandipan Patra <spatra@nvidia.com>");
+MODULE_DESCRIPTION("Tegra PWM controller driver");
 MODULE_ALIAS("platform:tegra-pwm");
diff --git a/drivers/rapidio/Kconfig b/drivers/rapidio/Kconfig
index 677d1aff61b7..e4c422d806be 100644
--- a/drivers/rapidio/Kconfig
+++ b/drivers/rapidio/Kconfig
@@ -19,14 +19,14 @@ config RAPIDIO_DISC_TIMEOUT
 	int "Discovery timeout duration (seconds)"
 	depends on RAPIDIO
 	default "30"
-	---help---
+	help
 	  Amount of time a discovery node waits for a host to complete
 	  enumeration before giving up.
 
 config RAPIDIO_ENABLE_RX_TX_PORTS
 	bool "Enable RapidIO Input/Output Ports"
 	depends on RAPIDIO
-	---help---
+	help
 	  The RapidIO specification describes a Output port transmit
 	  enable and a Input port receive enable. The recommended state
 	  for Input ports and Output ports should be disabled. When
diff --git a/drivers/rapidio/devices/Kconfig b/drivers/rapidio/devices/Kconfig
index 9a195654b0ca..c416531ad7c8 100644
--- a/drivers/rapidio/devices/Kconfig
+++ b/drivers/rapidio/devices/Kconfig
@@ -7,5 +7,5 @@ config RAPIDIO_TSI721
 	tristate "IDT Tsi721 PCI Express SRIO Controller support"
 	depends on RAPIDIO && PCIEPORTBUS
 	default "n"
-	---help---
+	help
 	  Include support for IDT Tsi721 PCI Express Serial RapidIO controller.
diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c
index 10af330153b5..451608e960a1 100644
--- a/drivers/rapidio/devices/rio_mport_cdev.c
+++ b/drivers/rapidio/devices/rio_mport_cdev.c
@@ -572,14 +572,12 @@ static void dma_req_free(struct kref *ref)
 	struct mport_dma_req *req = container_of(ref, struct mport_dma_req,
 			refcount);
 	struct mport_cdev_priv *priv = req->priv;
-	unsigned int i;
 
 	dma_unmap_sg(req->dmach->device->dev,
 		     req->sgt.sgl, req->sgt.nents, req->dir);
 	sg_free_table(&req->sgt);
 	if (req->page_list) {
-		for (i = 0; i < req->nr_pages; i++)
-			put_page(req->page_list[i]);
+		unpin_user_pages(req->page_list, req->nr_pages);
 		kfree(req->page_list);
 	}
 
@@ -815,7 +813,7 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode,
 	struct mport_dma_req *req;
 	struct mport_dev *md = priv->md;
 	struct dma_chan *chan;
-	int i, ret;
+	int ret;
 	int nents;
 
 	if (xfer->length == 0)
@@ -862,7 +860,7 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode,
 			goto err_req;
 		}
 
-		pinned = get_user_pages_fast(
+		pinned = pin_user_pages_fast(
 				(unsigned long)xfer->loc_addr & PAGE_MASK,
 				nr_pages,
 				dir == DMA_FROM_DEVICE ? FOLL_WRITE : 0,
@@ -870,7 +868,7 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode,
 
 		if (pinned != nr_pages) {
 			if (pinned < 0) {
-				rmcd_error("get_user_pages_unlocked err=%ld",
+				rmcd_error("pin_user_pages_fast err=%ld",
 					   pinned);
 				nr_pages = 0;
 			} else
@@ -951,8 +949,7 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode,
 
 err_pg:
 	if (!req->page_list) {
-		for (i = 0; i < nr_pages; i++)
-			put_page(page_list[i]);
+		unpin_user_pages(page_list, nr_pages);
 		kfree(page_list);
 	}
 err_req:
@@ -2384,13 +2381,6 @@ static struct mport_dev *mport_cdev_add(struct rio_mport *mport)
 	cdev_init(&md->cdev, &mport_fops);
 	md->cdev.owner = THIS_MODULE;
 
-	ret = cdev_device_add(&md->cdev, &md->dev);
-	if (ret) {
-		rmcd_error("Failed to register mport %d (err=%d)",
-		       mport->id, ret);
-		goto err_cdev;
-	}
-
 	INIT_LIST_HEAD(&md->doorbells);
 	spin_lock_init(&md->db_lock);
 	INIT_LIST_HEAD(&md->portwrites);
@@ -2410,6 +2400,13 @@ static struct mport_dev *mport_cdev_add(struct rio_mport *mport)
 #else
 	md->properties.transfer_mode |= RIO_TRANSFER_MODE_TRANSFER;
 #endif
+
+	ret = cdev_device_add(&md->cdev, &md->dev);
+	if (ret) {
+		rmcd_error("Failed to register mport %d (err=%d)",
+		       mport->id, ret);
+		goto err_cdev;
+	}
 	ret = rio_query_mport(mport, &attr);
 	if (!ret) {
 		md->properties.flags = attr.flags;
diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c
index 0e90c5d4bb2b..eb8ed28533f8 100644
--- a/drivers/rapidio/rio-scan.c
+++ b/drivers/rapidio/rio-scan.c
@@ -39,7 +39,7 @@ struct rio_id_table {
 	u16 start;	/* logical minimal id */
 	u32 max;	/* max number of IDs in table */
 	spinlock_t lock;
-	unsigned long table[0];
+	unsigned long table[];
 };
 
 static int next_destid = 0;
diff --git a/drivers/rapidio/switches/Kconfig b/drivers/rapidio/switches/Kconfig
index c1eb9cb899b1..3e18f9c51e29 100644
--- a/drivers/rapidio/switches/Kconfig
+++ b/drivers/rapidio/switches/Kconfig
@@ -4,28 +4,28 @@
 #
 config RAPIDIO_TSI57X
 	tristate "IDT Tsi57x SRIO switches support"
-	---help---
+	help
 	  Includes support for IDT Tsi57x family of serial RapidIO switches.
 
 config RAPIDIO_CPS_XX
 	tristate "IDT CPS-xx SRIO switches support"
-	---help---
+	help
 	  Includes support for IDT CPS-16/12/10/8 serial RapidIO switches.
 
 config RAPIDIO_TSI568
 	tristate "Tsi568 SRIO switch support"
 	default n
-	---help---
+	help
 	  Includes support for IDT Tsi568 serial RapidIO switch.
 
 config RAPIDIO_CPS_GEN2
 	tristate "IDT CPS Gen.2 SRIO switch support"
 	default n
-	---help---
+	help
 	  Includes support for ITD CPS Gen.2 serial RapidIO switches.
 
 config RAPIDIO_RXS_GEN3
 	tristate "IDT RXS Gen.3 SRIO switch support"
 	default n
-	---help---
+	help
 	  Includes support for ITD RXS Gen.3 serial RapidIO switches.
diff --git a/drivers/ras/cec.c b/drivers/ras/cec.c
index c09cf55e2d20..569d9ad2c594 100644
--- a/drivers/ras/cec.c
+++ b/drivers/ras/cec.c
@@ -309,7 +309,7 @@ static bool sanity_check(struct ce_array *ca)
 	return ret;
 }
 
-int cec_add_elem(u64 pfn)
+static int cec_add_elem(u64 pfn)
 {
 	struct ce_array *ca = &ce_arr;
 	unsigned int to = 0;
@@ -527,7 +527,33 @@ err:
 	return 1;
 }
 
-void __init cec_init(void)
+static int cec_notifier(struct notifier_block *nb, unsigned long val,
+			void *data)
+{
+	struct mce *m = (struct mce *)data;
+
+	if (!m)
+		return NOTIFY_DONE;
+
+	/* We eat only correctable DRAM errors with usable addresses. */
+	if (mce_is_memory_error(m) &&
+	    mce_is_correctable(m)  &&
+	    mce_usable_address(m)) {
+		if (!cec_add_elem(m->addr >> PAGE_SHIFT)) {
+			m->kflags |= MCE_HANDLED_CEC;
+			return NOTIFY_OK;
+		}
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block cec_nb = {
+	.notifier_call	= cec_notifier,
+	.priority	= MCE_PRIO_CEC,
+};
+
+static void __init cec_init(void)
 {
 	if (ce_arr.disabled)
 		return;
@@ -546,8 +572,11 @@ void __init cec_init(void)
 	INIT_DELAYED_WORK(&cec_work, cec_work_fn);
 	schedule_delayed_work(&cec_work, CEC_DECAY_DEFAULT_INTERVAL);
 
+	mce_register_decode_chain(&cec_nb);
+
 	pr_info("Correctable Errors collector initialized.\n");
 }
+late_initcall(cec_init);
 
 int __init parse_cec_param(char *str)
 {
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index fbaed079b299..c4d1731295eb 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -23,6 +23,15 @@ config IMX_REMOTEPROC
 
 	  It's safe to say N here.
 
+config INGENIC_VPU_RPROC
+	tristate "Ingenic JZ47xx VPU remoteproc support"
+	depends on MIPS || COMPILE_TEST
+	help
+	  Say y or m here to support the VPU in the JZ47xx SoCs from Ingenic.
+
+	  This can be either built-in or a loadable module.
+	  If unsure say N.
+
 config MTK_SCP
 	tristate "Mediatek SCP support"
 	depends on ARCH_MEDIATEK
diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
index 0effd3825035..e8b886e511f0 100644
--- a/drivers/remoteproc/Makefile
+++ b/drivers/remoteproc/Makefile
@@ -10,6 +10,7 @@ remoteproc-y				+= remoteproc_sysfs.o
 remoteproc-y				+= remoteproc_virtio.o
 remoteproc-y				+= remoteproc_elf_loader.o
 obj-$(CONFIG_IMX_REMOTEPROC)		+= imx_rproc.o
+obj-$(CONFIG_INGENIC_VPU_RPROC)		+= ingenic_rproc.o
 obj-$(CONFIG_MTK_SCP)			+= mtk_scp.o mtk_scp_ipi.o
 obj-$(CONFIG_OMAP_REMOTEPROC)		+= omap_remoteproc.o
 obj-$(CONFIG_WKUP_M3_RPROC)		+= wkup_m3_rproc.o
diff --git a/drivers/remoteproc/ingenic_rproc.c b/drivers/remoteproc/ingenic_rproc.c
new file mode 100644
index 000000000000..189020d77b25
--- /dev/null
+++ b/drivers/remoteproc/ingenic_rproc.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Ingenic JZ47xx remoteproc driver
+ * Copyright 2019, Paul Cercueil <paul@crapouillou.net>
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/remoteproc.h>
+
+#include "remoteproc_internal.h"
+
+#define REG_AUX_CTRL		0x0
+#define REG_AUX_MSG_ACK		0x10
+#define REG_AUX_MSG		0x14
+#define REG_CORE_MSG_ACK	0x18
+#define REG_CORE_MSG		0x1C
+
+#define AUX_CTRL_SLEEP		BIT(31)
+#define AUX_CTRL_MSG_IRQ_EN	BIT(3)
+#define AUX_CTRL_NMI_RESETS	BIT(2)
+#define AUX_CTRL_NMI		BIT(1)
+#define AUX_CTRL_SW_RESET	BIT(0)
+
+struct vpu_mem_map {
+	const char *name;
+	unsigned int da;
+};
+
+struct vpu_mem_info {
+	const struct vpu_mem_map *map;
+	unsigned long len;
+	void __iomem *base;
+};
+
+static const struct vpu_mem_map vpu_mem_map[] = {
+	{ "tcsm0", 0x132b0000 },
+	{ "tcsm1", 0xf4000000 },
+	{ "sram",  0x132f0000 },
+};
+
+/**
+ * struct vpu - Ingenic VPU remoteproc private structure
+ * @irq: interrupt number
+ * @clks: pointers to the VPU and AUX clocks
+ * @aux_base: raw pointer to the AUX interface registers
+ * @mem_info: array of struct vpu_mem_info, which contain the mapping info of
+ *            each of the external memories
+ * @dev: private pointer to the device
+ */
+struct vpu {
+	int irq;
+	struct clk_bulk_data clks[2];
+	void __iomem *aux_base;
+	struct vpu_mem_info mem_info[ARRAY_SIZE(vpu_mem_map)];
+	struct device *dev;
+};
+
+static int ingenic_rproc_start(struct rproc *rproc)
+{
+	struct vpu *vpu = rproc->priv;
+	u32 ctrl;
+
+	enable_irq(vpu->irq);
+
+	/* Reset the AUX and enable message IRQ */
+	ctrl = AUX_CTRL_NMI_RESETS | AUX_CTRL_NMI | AUX_CTRL_MSG_IRQ_EN;
+	writel(ctrl, vpu->aux_base + REG_AUX_CTRL);
+
+	return 0;
+}
+
+static int ingenic_rproc_stop(struct rproc *rproc)
+{
+	struct vpu *vpu = rproc->priv;
+
+	disable_irq(vpu->irq);
+
+	/* Keep AUX in reset mode */
+	writel(AUX_CTRL_SW_RESET, vpu->aux_base + REG_AUX_CTRL);
+
+	return 0;
+}
+
+static void ingenic_rproc_kick(struct rproc *rproc, int vqid)
+{
+	struct vpu *vpu = rproc->priv;
+
+	writel(vqid, vpu->aux_base + REG_CORE_MSG);
+}
+
+static void *ingenic_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len)
+{
+	struct vpu *vpu = rproc->priv;
+	void __iomem *va = NULL;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vpu_mem_map); i++) {
+		const struct vpu_mem_info *info = &vpu->mem_info[i];
+		const struct vpu_mem_map *map = info->map;
+
+		if (da >= map->da && (da + len) < (map->da + info->len)) {
+			va = info->base + (da - map->da);
+			break;
+		}
+	}
+
+	return (__force void *)va;
+}
+
+static struct rproc_ops ingenic_rproc_ops = {
+	.start = ingenic_rproc_start,
+	.stop = ingenic_rproc_stop,
+	.kick = ingenic_rproc_kick,
+	.da_to_va = ingenic_rproc_da_to_va,
+};
+
+static irqreturn_t vpu_interrupt(int irq, void *data)
+{
+	struct rproc *rproc = data;
+	struct vpu *vpu = rproc->priv;
+	u32 vring;
+
+	vring = readl(vpu->aux_base + REG_AUX_MSG);
+
+	/* Ack the interrupt */
+	writel(0, vpu->aux_base + REG_AUX_MSG_ACK);
+
+	return rproc_vq_interrupt(rproc, vring);
+}
+
+static void ingenic_rproc_disable_clks(void *data)
+{
+	struct vpu *vpu = data;
+
+	pm_runtime_resume(vpu->dev);
+	pm_runtime_disable(vpu->dev);
+
+	clk_bulk_disable_unprepare(ARRAY_SIZE(vpu->clks), vpu->clks);
+}
+
+static int ingenic_rproc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *mem;
+	struct rproc *rproc;
+	struct vpu *vpu;
+	unsigned int i;
+	int ret;
+
+	rproc = devm_rproc_alloc(dev, "ingenic-vpu",
+				 &ingenic_rproc_ops, NULL, sizeof(*vpu));
+	if (!rproc)
+		return -ENOMEM;
+
+	vpu = rproc->priv;
+	vpu->dev = &pdev->dev;
+	platform_set_drvdata(pdev, vpu);
+
+	mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "aux");
+	vpu->aux_base = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(vpu->aux_base)) {
+		dev_err(dev, "Failed to ioremap\n");
+		return PTR_ERR(vpu->aux_base);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(vpu_mem_map); i++) {
+		mem = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   vpu_mem_map[i].name);
+
+		vpu->mem_info[i].base = devm_ioremap_resource(dev, mem);
+		if (IS_ERR(vpu->mem_info[i].base)) {
+			ret = PTR_ERR(vpu->mem_info[i].base);
+			dev_err(dev, "Failed to ioremap\n");
+			return ret;
+		}
+
+		vpu->mem_info[i].len = resource_size(mem);
+		vpu->mem_info[i].map = &vpu_mem_map[i];
+	}
+
+	vpu->clks[0].id = "vpu";
+	vpu->clks[1].id = "aux";
+
+	ret = devm_clk_bulk_get(dev, ARRAY_SIZE(vpu->clks), vpu->clks);
+	if (ret) {
+		dev_err(dev, "Failed to get clocks\n");
+		return ret;
+	}
+
+	vpu->irq = platform_get_irq(pdev, 0);
+	if (vpu->irq < 0)
+		return vpu->irq;
+
+	ret = devm_request_irq(dev, vpu->irq, vpu_interrupt, 0, "VPU", rproc);
+	if (ret < 0) {
+		dev_err(dev, "Failed to request IRQ\n");
+		return ret;
+	}
+
+	disable_irq(vpu->irq);
+
+	/* The clocks must be enabled for the firmware to be loaded in TCSM */
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(vpu->clks), vpu->clks);
+	if (ret) {
+		dev_err(dev, "Unable to start clocks\n");
+		return ret;
+	}
+
+	pm_runtime_irq_safe(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+	pm_runtime_get_sync(dev);
+	pm_runtime_use_autosuspend(dev);
+
+	ret = devm_add_action_or_reset(dev, ingenic_rproc_disable_clks, vpu);
+	if (ret) {
+		dev_err(dev, "Unable to register action\n");
+		goto out_pm_put;
+	}
+
+	ret = devm_rproc_add(dev, rproc);
+	if (ret) {
+		dev_err(dev, "Failed to register remote processor\n");
+		goto out_pm_put;
+	}
+
+out_pm_put:
+	pm_runtime_put_autosuspend(dev);
+
+	return ret;
+}
+
+static const struct of_device_id ingenic_rproc_of_matches[] = {
+	{ .compatible = "ingenic,jz4770-vpu-rproc", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ingenic_rproc_of_matches);
+
+static int __maybe_unused ingenic_rproc_suspend(struct device *dev)
+{
+	struct vpu *vpu = dev_get_drvdata(dev);
+
+	clk_bulk_disable(ARRAY_SIZE(vpu->clks), vpu->clks);
+
+	return 0;
+}
+
+static int __maybe_unused ingenic_rproc_resume(struct device *dev)
+{
+	struct vpu *vpu = dev_get_drvdata(dev);
+
+	return clk_bulk_enable(ARRAY_SIZE(vpu->clks), vpu->clks);
+}
+
+static const struct dev_pm_ops __maybe_unused ingenic_rproc_pm = {
+	SET_RUNTIME_PM_OPS(ingenic_rproc_suspend, ingenic_rproc_resume, NULL)
+};
+
+static struct platform_driver ingenic_rproc_driver = {
+	.probe = ingenic_rproc_probe,
+	.driver = {
+		.name = "ingenic-vpu",
+#ifdef CONFIG_PM
+		.pm = &ingenic_rproc_pm,
+#endif
+		.of_match_table = ingenic_rproc_of_matches,
+	},
+};
+module_platform_driver(ingenic_rproc_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Paul Cercueil <paul@crapouillou.net>");
+MODULE_DESCRIPTION("Ingenic JZ47xx Remote Processor control driver");
diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c
index 2bead57c9cf9..ac13e7b046a6 100644
--- a/drivers/remoteproc/mtk_scp.c
+++ b/drivers/remoteproc/mtk_scp.c
@@ -132,8 +132,8 @@ static int scp_ipi_init(struct mtk_scp *scp)
 		(struct mtk_share_obj __iomem *)(scp->sram_base + recv_offset);
 	scp->send_buf =
 		(struct mtk_share_obj __iomem *)(scp->sram_base + send_offset);
-	memset_io(scp->recv_buf, 0, sizeof(scp->recv_buf));
-	memset_io(scp->send_buf, 0, sizeof(scp->send_buf));
+	memset_io(scp->recv_buf, 0, sizeof(*scp->recv_buf));
+	memset_io(scp->send_buf, 0, sizeof(*scp->send_buf));
 
 	return 0;
 }
diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c
index 60650bcc8c67..9028cea2d81e 100644
--- a/drivers/remoteproc/qcom_common.c
+++ b/drivers/remoteproc/qcom_common.c
@@ -42,12 +42,21 @@ static void glink_subdev_stop(struct rproc_subdev *subdev, bool crashed)
 	glink->edge = NULL;
 }
 
+static void glink_subdev_unprepare(struct rproc_subdev *subdev)
+{
+	struct qcom_rproc_glink *glink = to_glink_subdev(subdev);
+
+	qcom_glink_ssr_notify(glink->ssr_name);
+}
+
 /**
  * qcom_add_glink_subdev() - try to add a GLINK subdevice to rproc
  * @rproc:	rproc handle to parent the subdevice
  * @glink:	reference to a GLINK subdev context
+ * @ssr_name:	identifier of the associated remoteproc for ssr notifications
  */
-void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink)
+void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink,
+			   const char *ssr_name)
 {
 	struct device *dev = &rproc->dev;
 
@@ -55,9 +64,14 @@ void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink)
 	if (!glink->node)
 		return;
 
+	glink->ssr_name = kstrdup_const(ssr_name, GFP_KERNEL);
+	if (!glink->ssr_name)
+		return;
+
 	glink->dev = dev;
 	glink->subdev.start = glink_subdev_start;
 	glink->subdev.stop = glink_subdev_stop;
+	glink->subdev.unprepare = glink_subdev_unprepare;
 
 	rproc_add_subdev(rproc, &glink->subdev);
 }
@@ -74,6 +88,7 @@ void qcom_remove_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glin
 		return;
 
 	rproc_remove_subdev(rproc, &glink->subdev);
+	kfree_const(glink->ssr_name);
 	of_node_put(glink->node);
 }
 EXPORT_SYMBOL_GPL(qcom_remove_glink_subdev);
diff --git a/drivers/remoteproc/qcom_common.h b/drivers/remoteproc/qcom_common.h
index 58de71e4781c..34e5188187dc 100644
--- a/drivers/remoteproc/qcom_common.h
+++ b/drivers/remoteproc/qcom_common.h
@@ -11,6 +11,8 @@ struct qcom_sysmon;
 struct qcom_rproc_glink {
 	struct rproc_subdev subdev;
 
+	const char *ssr_name;
+
 	struct device *dev;
 	struct device_node *node;
 	struct qcom_glink *edge;
@@ -30,7 +32,8 @@ struct qcom_rproc_ssr {
 	const char *name;
 };
 
-void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink);
+void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink,
+			   const char *ssr_name);
 void qcom_remove_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink);
 
 int qcom_register_dump_segments(struct rproc *rproc, const struct firmware *fw);
diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c
index 24a3db961d5e..d2a2574dcf35 100644
--- a/drivers/remoteproc/qcom_q6v5_adsp.c
+++ b/drivers/remoteproc/qcom_q6v5_adsp.c
@@ -431,6 +431,7 @@ static int adsp_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "unable to allocate remoteproc\n");
 		return -ENOMEM;
 	}
+	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
 	adsp = (struct qcom_adsp *)rproc->priv;
 	adsp->dev = &pdev->dev;
@@ -460,7 +461,7 @@ static int adsp_probe(struct platform_device *pdev)
 	if (ret)
 		goto disable_pm;
 
-	qcom_add_glink_subdev(rproc, &adsp->glink_subdev);
+	qcom_add_glink_subdev(rproc, &adsp->glink_subdev, desc->ssr_name);
 	qcom_add_ssr_subdev(rproc, &adsp->ssr_subdev, desc->ssr_name);
 	adsp->sysmon = qcom_add_sysmon_subdev(rproc,
 					      desc->sysmon_name,
diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c
index 5475d4f808a8..feb70283b6a2 100644
--- a/drivers/remoteproc/qcom_q6v5_mss.c
+++ b/drivers/remoteproc/qcom_q6v5_mss.c
@@ -69,13 +69,9 @@
 #define AXI_HALTREQ_REG			0x0
 #define AXI_HALTACK_REG			0x4
 #define AXI_IDLE_REG			0x8
-#define NAV_AXI_HALTREQ_BIT		BIT(0)
-#define NAV_AXI_HALTACK_BIT		BIT(1)
-#define NAV_AXI_IDLE_BIT		BIT(2)
 #define AXI_GATING_VALID_OVERRIDE	BIT(0)
 
 #define HALT_ACK_TIMEOUT_US		100000
-#define NAV_HALT_ACK_TIMEOUT_US		200
 
 /* QDSP6SS_RESET */
 #define Q6SS_STOP_CORE			BIT(0)
@@ -143,7 +139,7 @@ struct rproc_hexagon_res {
 	int version;
 	bool need_mem_protection;
 	bool has_alt_reset;
-	bool has_halt_nav;
+	bool has_spare_reg;
 };
 
 struct q6v5 {
@@ -154,13 +150,11 @@ struct q6v5 {
 	void __iomem *rmb_base;
 
 	struct regmap *halt_map;
-	struct regmap *halt_nav_map;
 	struct regmap *conn_map;
 
 	u32 halt_q6;
 	u32 halt_modem;
 	u32 halt_nc;
-	u32 halt_nav;
 	u32 conn_box;
 
 	struct reset_control *mss_restart;
@@ -196,7 +190,6 @@ struct q6v5 {
 
 	phys_addr_t mpss_phys;
 	phys_addr_t mpss_reloc;
-	void *mpss_region;
 	size_t mpss_size;
 
 	struct qcom_rproc_glink glink_subdev;
@@ -206,7 +199,7 @@ struct q6v5 {
 	struct qcom_sysmon *sysmon;
 	bool need_mem_protection;
 	bool has_alt_reset;
-	bool has_halt_nav;
+	bool has_spare_reg;
 	int mpss_perm;
 	int mba_perm;
 	const char *hexagon_mdt_image;
@@ -427,21 +420,19 @@ static int q6v5_reset_assert(struct q6v5 *qproc)
 		reset_control_assert(qproc->pdc_reset);
 		ret = reset_control_reset(qproc->mss_restart);
 		reset_control_deassert(qproc->pdc_reset);
-	} else if (qproc->has_halt_nav) {
+	} else if (qproc->has_spare_reg) {
 		/*
 		 * When the AXI pipeline is being reset with the Q6 modem partly
 		 * operational there is possibility of AXI valid signal to
 		 * glitch, leading to spurious transactions and Q6 hangs. A work
 		 * around is employed by asserting the AXI_GATING_VALID_OVERRIDE
-		 * BIT before triggering Q6 MSS reset. Both the HALTREQ and
-		 * AXI_GATING_VALID_OVERRIDE are withdrawn post MSS assert
-		 * followed by a MSS deassert, while holding the PDC reset.
+		 * BIT before triggering Q6 MSS reset. AXI_GATING_VALID_OVERRIDE
+		 * is withdrawn post MSS assert followed by a MSS deassert,
+		 * while holding the PDC reset.
 		 */
 		reset_control_assert(qproc->pdc_reset);
 		regmap_update_bits(qproc->conn_map, qproc->conn_box,
 				   AXI_GATING_VALID_OVERRIDE, 1);
-		regmap_update_bits(qproc->halt_nav_map, qproc->halt_nav,
-				   NAV_AXI_HALTREQ_BIT, 0);
 		reset_control_assert(qproc->mss_restart);
 		reset_control_deassert(qproc->pdc_reset);
 		regmap_update_bits(qproc->conn_map, qproc->conn_box,
@@ -464,7 +455,7 @@ static int q6v5_reset_deassert(struct q6v5 *qproc)
 		ret = reset_control_reset(qproc->mss_restart);
 		writel(0, qproc->rmb_base + RMB_MBA_ALT_RESET);
 		reset_control_deassert(qproc->pdc_reset);
-	} else if (qproc->has_halt_nav) {
+	} else if (qproc->has_spare_reg) {
 		ret = reset_control_reset(qproc->mss_restart);
 	} else {
 		ret = reset_control_deassert(qproc->mss_restart);
@@ -761,32 +752,6 @@ static void q6v5proc_halt_axi_port(struct q6v5 *qproc,
 	regmap_write(halt_map, offset + AXI_HALTREQ_REG, 0);
 }
 
-static void q6v5proc_halt_nav_axi_port(struct q6v5 *qproc,
-				       struct regmap *halt_map,
-				       u32 offset)
-{
-	unsigned int val;
-	int ret;
-
-	/* Check if we're already idle */
-	ret = regmap_read(halt_map, offset, &val);
-	if (!ret && (val & NAV_AXI_IDLE_BIT))
-		return;
-
-	/* Assert halt request */
-	regmap_update_bits(halt_map, offset, NAV_AXI_HALTREQ_BIT,
-			   NAV_AXI_HALTREQ_BIT);
-
-	/* Wait for halt ack*/
-	regmap_read_poll_timeout(halt_map, offset, val,
-				 (val & NAV_AXI_HALTACK_BIT),
-				 5, NAV_HALT_ACK_TIMEOUT_US);
-
-	ret = regmap_read(halt_map, offset, &val);
-	if (ret || !(val & NAV_AXI_IDLE_BIT))
-		dev_err(qproc->dev, "port failed halt\n");
-}
-
 static int q6v5_mpss_init_image(struct q6v5 *qproc, const struct firmware *fw)
 {
 	unsigned long dma_attrs = DMA_ATTR_FORCE_CONTIGUOUS;
@@ -951,9 +916,6 @@ static int q6v5_mba_load(struct q6v5 *qproc)
 halt_axi_ports:
 	q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_q6);
 	q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_modem);
-	if (qproc->has_halt_nav)
-		q6v5proc_halt_nav_axi_port(qproc, qproc->halt_nav_map,
-					   qproc->halt_nav);
 	q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_nc);
 
 reclaim_mba:
@@ -1001,9 +963,6 @@ static void q6v5_mba_reclaim(struct q6v5 *qproc)
 
 	q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_q6);
 	q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_modem);
-	if (qproc->has_halt_nav)
-		q6v5proc_halt_nav_axi_port(qproc, qproc->halt_nav_map,
-					   qproc->halt_nav);
 	q6v5proc_halt_axi_port(qproc, qproc->halt_map, qproc->halt_nc);
 	if (qproc->version == MSS_MSM8996) {
 		/*
@@ -1156,7 +1115,13 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
 			goto release_firmware;
 		}
 
-		ptr = qproc->mpss_region + offset;
+		ptr = ioremap_wc(qproc->mpss_phys + offset, phdr->p_memsz);
+		if (!ptr) {
+			dev_err(qproc->dev,
+				"unable to map memory region: %pa+%zx-%x\n",
+				&qproc->mpss_phys, offset, phdr->p_memsz);
+			goto release_firmware;
+		}
 
 		if (phdr->p_filesz && phdr->p_offset < fw->size) {
 			/* Firmware is large enough to be non-split */
@@ -1165,6 +1130,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
 					"failed to load segment %d from truncated file %s\n",
 					i, fw_name);
 				ret = -EINVAL;
+				iounmap(ptr);
 				goto release_firmware;
 			}
 
@@ -1175,6 +1141,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
 			ret = request_firmware(&seg_fw, fw_name, qproc->dev);
 			if (ret) {
 				dev_err(qproc->dev, "failed to load %s\n", fw_name);
+				iounmap(ptr);
 				goto release_firmware;
 			}
 
@@ -1187,6 +1154,7 @@ static int q6v5_mpss_load(struct q6v5 *qproc)
 			memset(ptr + phdr->p_filesz, 0,
 			       phdr->p_memsz - phdr->p_filesz);
 		}
+		iounmap(ptr);
 		size += phdr->p_memsz;
 
 		code_length = readl(qproc->rmb_base + RMB_PMI_CODE_LENGTH_REG);
@@ -1236,7 +1204,8 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc,
 	int ret = 0;
 	struct q6v5 *qproc = rproc->priv;
 	unsigned long mask = BIT((unsigned long)segment->priv);
-	void *ptr = rproc_da_to_va(rproc, segment->da, segment->size);
+	int offset = segment->da - qproc->mpss_reloc;
+	void *ptr = NULL;
 
 	/* Unlock mba before copying segments */
 	if (!qproc->dump_mba_loaded) {
@@ -1250,10 +1219,15 @@ static void qcom_q6v5_dump_segment(struct rproc *rproc,
 		}
 	}
 
-	if (!ptr || ret)
-		memset(dest, 0xff, segment->size);
-	else
+	if (!ret)
+		ptr = ioremap_wc(qproc->mpss_phys + offset, segment->size);
+
+	if (ptr) {
 		memcpy(dest, ptr, segment->size);
+		iounmap(ptr);
+	} else {
+		memset(dest, 0xff, segment->size);
+	}
 
 	qproc->dump_segment_mask |= mask;
 
@@ -1327,18 +1301,6 @@ static int q6v5_stop(struct rproc *rproc)
 	return 0;
 }
 
-static void *q6v5_da_to_va(struct rproc *rproc, u64 da, size_t len)
-{
-	struct q6v5 *qproc = rproc->priv;
-	int offset;
-
-	offset = da - qproc->mpss_reloc;
-	if (offset < 0 || offset + len > qproc->mpss_size)
-		return NULL;
-
-	return qproc->mpss_region + offset;
-}
-
 static int qcom_q6v5_register_dump_segments(struct rproc *rproc,
 					    const struct firmware *mba_fw)
 {
@@ -1357,6 +1319,8 @@ static int qcom_q6v5_register_dump_segments(struct rproc *rproc,
 		return ret;
 	}
 
+	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
+
 	ehdr = (struct elf32_hdr *)fw->data;
 	phdrs = (struct elf32_phdr *)(ehdr + 1);
 	qproc->dump_complete_mask = 0;
@@ -1384,7 +1348,6 @@ static int qcom_q6v5_register_dump_segments(struct rproc *rproc,
 static const struct rproc_ops q6v5_ops = {
 	.start = q6v5_start,
 	.stop = q6v5_stop,
-	.da_to_va = q6v5_da_to_va,
 	.parse_fw = qcom_q6v5_register_dump_segments,
 	.load = q6v5_load,
 };
@@ -1432,36 +1395,12 @@ static int q6v5_init_mem(struct q6v5 *qproc, struct platform_device *pdev)
 	qproc->halt_modem = args.args[1];
 	qproc->halt_nc = args.args[2];
 
-	if (qproc->has_halt_nav) {
-		struct platform_device *nav_pdev;
-
+	if (qproc->has_spare_reg) {
 		ret = of_parse_phandle_with_fixed_args(pdev->dev.of_node,
-						       "qcom,halt-nav-regs",
+						       "qcom,spare-regs",
 						       1, 0, &args);
 		if (ret < 0) {
-			dev_err(&pdev->dev, "failed to parse halt-nav-regs\n");
-			return -EINVAL;
-		}
-
-		nav_pdev = of_find_device_by_node(args.np);
-		of_node_put(args.np);
-		if (!nav_pdev) {
-			dev_err(&pdev->dev, "failed to get mss clock device\n");
-			return -EPROBE_DEFER;
-		}
-
-		qproc->halt_nav_map = dev_get_regmap(&nav_pdev->dev, NULL);
-		if (!qproc->halt_nav_map) {
-			dev_err(&pdev->dev, "failed to get map from device\n");
-			return -EINVAL;
-		}
-		qproc->halt_nav = args.args[0];
-
-		ret = of_parse_phandle_with_fixed_args(pdev->dev.of_node,
-						       "qcom,halt-nav-regs",
-						       1, 1, &args);
-		if (ret < 0) {
-			dev_err(&pdev->dev, "failed to parse halt-nav-regs\n");
+			dev_err(&pdev->dev, "failed to parse spare-regs\n");
 			return -EINVAL;
 		}
 
@@ -1547,7 +1486,7 @@ static int q6v5_init_reset(struct q6v5 *qproc)
 		return PTR_ERR(qproc->mss_restart);
 	}
 
-	if (qproc->has_alt_reset || qproc->has_halt_nav) {
+	if (qproc->has_alt_reset || qproc->has_spare_reg) {
 		qproc->pdc_reset = devm_reset_control_get_exclusive(qproc->dev,
 								    "pdc_reset");
 		if (IS_ERR(qproc->pdc_reset)) {
@@ -1566,8 +1505,17 @@ static int q6v5_alloc_memory_region(struct q6v5 *qproc)
 	struct resource r;
 	int ret;
 
+	/*
+	 * In the absence of mba/mpss sub-child, extract the mba and mpss
+	 * reserved memory regions from device's memory-region property.
+	 */
 	child = of_get_child_by_name(qproc->dev->of_node, "mba");
-	node = of_parse_phandle(child, "memory-region", 0);
+	if (!child)
+		node = of_parse_phandle(qproc->dev->of_node,
+					"memory-region", 0);
+	else
+		node = of_parse_phandle(child, "memory-region", 0);
+
 	ret = of_address_to_resource(node, 0, &r);
 	if (ret) {
 		dev_err(qproc->dev, "unable to resolve mba region\n");
@@ -1584,8 +1532,14 @@ static int q6v5_alloc_memory_region(struct q6v5 *qproc)
 		return -EBUSY;
 	}
 
-	child = of_get_child_by_name(qproc->dev->of_node, "mpss");
-	node = of_parse_phandle(child, "memory-region", 0);
+	if (!child) {
+		node = of_parse_phandle(qproc->dev->of_node,
+					"memory-region", 1);
+	} else {
+		child = of_get_child_by_name(qproc->dev->of_node, "mpss");
+		node = of_parse_phandle(child, "memory-region", 0);
+	}
+
 	ret = of_address_to_resource(node, 0, &r);
 	if (ret) {
 		dev_err(qproc->dev, "unable to resolve mpss region\n");
@@ -1595,12 +1549,6 @@ static int q6v5_alloc_memory_region(struct q6v5 *qproc)
 
 	qproc->mpss_phys = qproc->mpss_reloc = r.start;
 	qproc->mpss_size = resource_size(&r);
-	qproc->mpss_region = devm_ioremap_wc(qproc->dev, qproc->mpss_phys, qproc->mpss_size);
-	if (!qproc->mpss_region) {
-		dev_err(qproc->dev, "unable to map memory region: %pa+%zx\n",
-			&r.start, qproc->mpss_size);
-		return -EBUSY;
-	}
 
 	return 0;
 }
@@ -1667,6 +1615,7 @@ static int q6v5_probe(struct platform_device *pdev)
 	}
 
 	rproc->auto_boot = false;
+	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
 	qproc = (struct q6v5 *)rproc->priv;
 	qproc->dev = &pdev->dev;
@@ -1679,7 +1628,7 @@ static int q6v5_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, qproc);
 
-	qproc->has_halt_nav = desc->has_halt_nav;
+	qproc->has_spare_reg = desc->has_spare_reg;
 	ret = q6v5_init_mem(qproc, pdev);
 	if (ret)
 		goto free_rproc;
@@ -1759,7 +1708,7 @@ static int q6v5_probe(struct platform_device *pdev)
 
 	qproc->mpss_perm = BIT(QCOM_SCM_VMID_HLOS);
 	qproc->mba_perm = BIT(QCOM_SCM_VMID_HLOS);
-	qcom_add_glink_subdev(rproc, &qproc->glink_subdev);
+	qcom_add_glink_subdev(rproc, &qproc->glink_subdev, "mpss");
 	qcom_add_smd_subdev(rproc, &qproc->smd_subdev);
 	qcom_add_ssr_subdev(rproc, &qproc->ssr_subdev, "mpss");
 	qcom_add_ipa_notify_subdev(rproc, &qproc->ipa_notify_subdev);
@@ -1828,8 +1777,6 @@ static const struct rproc_hexagon_res sc7180_mss = {
 	.active_clk_names = (char*[]){
 		"mnoc_axi",
 		"nav",
-		"mss_nav",
-		"mss_crypto",
 		NULL
 	},
 	.active_pd_names = (char*[]){
@@ -1844,7 +1791,7 @@ static const struct rproc_hexagon_res sc7180_mss = {
 	},
 	.need_mem_protection = true,
 	.has_alt_reset = false,
-	.has_halt_nav = true,
+	.has_spare_reg = true,
 	.version = MSS_SC7180,
 };
 
@@ -1879,7 +1826,7 @@ static const struct rproc_hexagon_res sdm845_mss = {
 	},
 	.need_mem_protection = true,
 	.has_alt_reset = true,
-	.has_halt_nav = false,
+	.has_spare_reg = false,
 	.version = MSS_SDM845,
 };
 
@@ -1906,7 +1853,7 @@ static const struct rproc_hexagon_res msm8998_mss = {
 	},
 	.need_mem_protection = true,
 	.has_alt_reset = false,
-	.has_halt_nav = false,
+	.has_spare_reg = false,
 	.version = MSS_MSM8998,
 };
 
@@ -1936,7 +1883,7 @@ static const struct rproc_hexagon_res msm8996_mss = {
 	},
 	.need_mem_protection = true,
 	.has_alt_reset = false,
-	.has_halt_nav = false,
+	.has_spare_reg = false,
 	.version = MSS_MSM8996,
 };
 
@@ -1969,7 +1916,7 @@ static const struct rproc_hexagon_res msm8916_mss = {
 	},
 	.need_mem_protection = false,
 	.has_alt_reset = false,
-	.has_halt_nav = false,
+	.has_spare_reg = false,
 	.version = MSS_MSM8916,
 };
 
@@ -2010,7 +1957,7 @@ static const struct rproc_hexagon_res msm8974_mss = {
 	},
 	.need_mem_protection = false,
 	.has_alt_reset = false,
-	.has_halt_nav = false,
+	.has_spare_reg = false,
 	.version = MSS_MSM8974,
 };
 
diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c
index 7a63efb85405..61791a03f648 100644
--- a/drivers/remoteproc/qcom_q6v5_pas.c
+++ b/drivers/remoteproc/qcom_q6v5_pas.c
@@ -398,6 +398,7 @@ static int adsp_probe(struct platform_device *pdev)
 	}
 
 	rproc->auto_boot = desc->auto_boot;
+	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
 	adsp = (struct qcom_adsp *)rproc->priv;
 	adsp->dev = &pdev->dev;
@@ -406,6 +407,8 @@ static int adsp_probe(struct platform_device *pdev)
 	adsp->has_aggre2_clk = desc->has_aggre2_clk;
 	platform_set_drvdata(pdev, adsp);
 
+	device_wakeup_enable(adsp->dev);
+
 	ret = adsp_alloc_memory_region(adsp);
 	if (ret)
 		goto free_rproc;
@@ -435,7 +438,7 @@ static int adsp_probe(struct platform_device *pdev)
 	if (ret)
 		goto detach_proxy_pds;
 
-	qcom_add_glink_subdev(rproc, &adsp->glink_subdev);
+	qcom_add_glink_subdev(rproc, &adsp->glink_subdev, desc->ssr_name);
 	qcom_add_smd_subdev(rproc, &adsp->smd_subdev);
 	qcom_add_ssr_subdev(rproc, &adsp->ssr_subdev, desc->ssr_name);
 	adsp->sysmon = qcom_add_sysmon_subdev(rproc,
@@ -507,6 +510,26 @@ static const struct adsp_data sm8150_adsp_resource = {
 		.ssctl_id = 0x14,
 };
 
+static const struct adsp_data sm8250_adsp_resource = {
+	.crash_reason_smem = 423,
+	.firmware_name = "adsp.mdt",
+	.pas_id = 1,
+	.has_aggre2_clk = false,
+	.auto_boot = true,
+	.active_pd_names = (char*[]){
+		"load_state",
+		NULL
+	},
+	.proxy_pd_names = (char*[]){
+		"lcx",
+		"lmx",
+		NULL
+	},
+	.ssr_name = "lpass",
+	.sysmon_name = "adsp",
+	.ssctl_id = 0x14,
+};
+
 static const struct adsp_data msm8998_adsp_resource = {
 		.crash_reason_smem = 423,
 		.firmware_name = "adsp.mdt",
@@ -552,6 +575,25 @@ static const struct adsp_data sm8150_cdsp_resource = {
 	.ssctl_id = 0x17,
 };
 
+static const struct adsp_data sm8250_cdsp_resource = {
+	.crash_reason_smem = 601,
+	.firmware_name = "cdsp.mdt",
+	.pas_id = 18,
+	.has_aggre2_clk = false,
+	.auto_boot = true,
+	.active_pd_names = (char*[]){
+		"load_state",
+		NULL
+	},
+	.proxy_pd_names = (char*[]){
+		"cx",
+		NULL
+	},
+	.ssr_name = "cdsp",
+	.sysmon_name = "cdsp",
+	.ssctl_id = 0x17,
+};
+
 static const struct adsp_data mpss_resource_init = {
 	.crash_reason_smem = 421,
 	.firmware_name = "modem.mdt",
@@ -603,6 +645,26 @@ static const struct adsp_data sm8150_slpi_resource = {
 		.ssctl_id = 0x16,
 };
 
+static const struct adsp_data sm8250_slpi_resource = {
+	.crash_reason_smem = 424,
+	.firmware_name = "slpi.mdt",
+	.pas_id = 12,
+	.has_aggre2_clk = false,
+	.auto_boot = true,
+	.active_pd_names = (char*[]){
+		"load_state",
+		NULL
+	},
+	.proxy_pd_names = (char*[]){
+		"lcx",
+		"lmx",
+		NULL
+	},
+	.ssr_name = "dsps",
+	.sysmon_name = "slpi",
+	.ssctl_id = 0x16,
+};
+
 static const struct adsp_data msm8998_slpi_resource = {
 		.crash_reason_smem = 424,
 		.firmware_name = "slpi.mdt",
@@ -637,12 +699,16 @@ static const struct of_device_id adsp_of_match[] = {
 	{ .compatible = "qcom,qcs404-adsp-pas", .data = &adsp_resource_init },
 	{ .compatible = "qcom,qcs404-cdsp-pas", .data = &cdsp_resource_init },
 	{ .compatible = "qcom,qcs404-wcss-pas", .data = &wcss_resource_init },
+	{ .compatible = "qcom,sc7180-mpss-pas", .data = &mpss_resource_init},
 	{ .compatible = "qcom,sdm845-adsp-pas", .data = &adsp_resource_init},
 	{ .compatible = "qcom,sdm845-cdsp-pas", .data = &cdsp_resource_init},
 	{ .compatible = "qcom,sm8150-adsp-pas", .data = &sm8150_adsp_resource},
 	{ .compatible = "qcom,sm8150-cdsp-pas", .data = &sm8150_cdsp_resource},
 	{ .compatible = "qcom,sm8150-mpss-pas", .data = &mpss_resource_init},
 	{ .compatible = "qcom,sm8150-slpi-pas", .data = &sm8150_slpi_resource},
+	{ .compatible = "qcom,sm8250-adsp-pas", .data = &sm8250_adsp_resource},
+	{ .compatible = "qcom,sm8250-cdsp-pas", .data = &sm8250_cdsp_resource},
+	{ .compatible = "qcom,sm8250-slpi-pas", .data = &sm8250_slpi_resource},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, adsp_of_match);
diff --git a/drivers/remoteproc/qcom_q6v5_wcss.c b/drivers/remoteproc/qcom_q6v5_wcss.c
index f1924b740a10..88c76b9417fa 100644
--- a/drivers/remoteproc/qcom_q6v5_wcss.c
+++ b/drivers/remoteproc/qcom_q6v5_wcss.c
@@ -91,6 +91,9 @@ struct q6v5_wcss {
 	phys_addr_t mem_reloc;
 	void *mem_region;
 	size_t mem_size;
+
+	struct qcom_rproc_glink glink_subdev;
+	struct qcom_rproc_ssr ssr_subdev;
 };
 
 static int q6v5_wcss_reset(struct q6v5_wcss *wcss)
@@ -557,6 +560,9 @@ static int q6v5_wcss_probe(struct platform_device *pdev)
 	if (ret)
 		goto free_rproc;
 
+	qcom_add_glink_subdev(rproc, &wcss->glink_subdev, "q6wcss");
+	qcom_add_ssr_subdev(rproc, &wcss->ssr_subdev, "q6wcss");
+
 	ret = rproc_add(rproc);
 	if (ret)
 		goto free_rproc;
diff --git a/drivers/remoteproc/qcom_sysmon.c b/drivers/remoteproc/qcom_sysmon.c
index faf3822d8791..8d8996d714f0 100644
--- a/drivers/remoteproc/qcom_sysmon.c
+++ b/drivers/remoteproc/qcom_sysmon.c
@@ -46,6 +46,25 @@ struct qcom_sysmon {
 	struct sockaddr_qrtr ssctl;
 };
 
+enum {
+	SSCTL_SSR_EVENT_BEFORE_POWERUP,
+	SSCTL_SSR_EVENT_AFTER_POWERUP,
+	SSCTL_SSR_EVENT_BEFORE_SHUTDOWN,
+	SSCTL_SSR_EVENT_AFTER_SHUTDOWN,
+};
+
+static const char * const sysmon_state_string[] = {
+	[SSCTL_SSR_EVENT_BEFORE_POWERUP]	= "before_powerup",
+	[SSCTL_SSR_EVENT_AFTER_POWERUP]		= "after_powerup",
+	[SSCTL_SSR_EVENT_BEFORE_SHUTDOWN]	= "before_shutdown",
+	[SSCTL_SSR_EVENT_AFTER_SHUTDOWN]	= "after_shutdown",
+};
+
+struct sysmon_event {
+	const char *subsys_name;
+	u32 ssr_event;
+};
+
 static DEFINE_MUTEX(sysmon_lock);
 static LIST_HEAD(sysmon_list);
 
@@ -54,13 +73,15 @@ static LIST_HEAD(sysmon_list);
  * @sysmon:	sysmon context
  * @name:	other remote's name
  */
-static void sysmon_send_event(struct qcom_sysmon *sysmon, const char *name)
+static void sysmon_send_event(struct qcom_sysmon *sysmon,
+			      const struct sysmon_event *event)
 {
 	char req[50];
 	int len;
 	int ret;
 
-	len = snprintf(req, sizeof(req), "ssr:%s:before_shutdown", name);
+	len = snprintf(req, sizeof(req), "ssr:%s:%s", event->subsys_name,
+		       sysmon_state_string[event->ssr_event]);
 	if (len >= sizeof(req))
 		return;
 
@@ -149,13 +170,6 @@ static int sysmon_callback(struct rpmsg_device *rpdev, void *data, int count,
 #define SSCTL_SUBSYS_NAME_LENGTH	15
 
 enum {
-	SSCTL_SSR_EVENT_BEFORE_POWERUP,
-	SSCTL_SSR_EVENT_AFTER_POWERUP,
-	SSCTL_SSR_EVENT_BEFORE_SHUTDOWN,
-	SSCTL_SSR_EVENT_AFTER_SHUTDOWN,
-};
-
-enum {
 	SSCTL_SSR_EVENT_FORCED,
 	SSCTL_SSR_EVENT_GRACEFUL,
 };
@@ -331,7 +345,8 @@ static void ssctl_request_shutdown(struct qcom_sysmon *sysmon)
  * @sysmon:	sysmon context
  * @name:	other remote's name
  */
-static void ssctl_send_event(struct qcom_sysmon *sysmon, const char *name)
+static void ssctl_send_event(struct qcom_sysmon *sysmon,
+			     const struct sysmon_event *event)
 {
 	struct ssctl_subsys_event_resp resp;
 	struct ssctl_subsys_event_req req;
@@ -346,9 +361,9 @@ static void ssctl_send_event(struct qcom_sysmon *sysmon, const char *name)
 	}
 
 	memset(&req, 0, sizeof(req));
-	strlcpy(req.subsys_name, name, sizeof(req.subsys_name));
+	strlcpy(req.subsys_name, event->subsys_name, sizeof(req.subsys_name));
 	req.subsys_name_len = strlen(req.subsys_name);
-	req.event = SSCTL_SSR_EVENT_BEFORE_SHUTDOWN;
+	req.event = event->ssr_event;
 	req.evt_driven_valid = true;
 	req.evt_driven = SSCTL_SSR_EVENT_FORCED;
 
@@ -424,16 +439,68 @@ static const struct qmi_ops ssctl_ops = {
 	.del_server = ssctl_del_server,
 };
 
+static int sysmon_prepare(struct rproc_subdev *subdev)
+{
+	struct qcom_sysmon *sysmon = container_of(subdev, struct qcom_sysmon,
+						  subdev);
+	struct sysmon_event event = {
+		.subsys_name = sysmon->name,
+		.ssr_event = SSCTL_SSR_EVENT_BEFORE_POWERUP
+	};
+
+	blocking_notifier_call_chain(&sysmon_notifiers, 0, (void *)&event);
+
+	return 0;
+}
+
+/**
+ * sysmon_start() - start callback for the sysmon remoteproc subdevice
+ * @subdev:	instance of the sysmon subdevice
+ *
+ * Inform all the listners of sysmon notifications that the rproc associated
+ * to @subdev has booted up. The rproc that booted up also needs to know
+ * which rprocs are already up and running, so send start notifications
+ * on behalf of all the online rprocs.
+ */
 static int sysmon_start(struct rproc_subdev *subdev)
 {
+	struct qcom_sysmon *sysmon = container_of(subdev, struct qcom_sysmon,
+						  subdev);
+	struct qcom_sysmon *target;
+	struct sysmon_event event = {
+		.subsys_name = sysmon->name,
+		.ssr_event = SSCTL_SSR_EVENT_AFTER_POWERUP
+	};
+
+	blocking_notifier_call_chain(&sysmon_notifiers, 0, (void *)&event);
+
+	mutex_lock(&sysmon_lock);
+	list_for_each_entry(target, &sysmon_list, node) {
+		if (target == sysmon ||
+		    target->rproc->state != RPROC_RUNNING)
+			continue;
+
+		event.subsys_name = target->name;
+
+		if (sysmon->ssctl_version == 2)
+			ssctl_send_event(sysmon, &event);
+		else if (sysmon->ept)
+			sysmon_send_event(sysmon, &event);
+	}
+	mutex_unlock(&sysmon_lock);
+
 	return 0;
 }
 
 static void sysmon_stop(struct rproc_subdev *subdev, bool crashed)
 {
 	struct qcom_sysmon *sysmon = container_of(subdev, struct qcom_sysmon, subdev);
+	struct sysmon_event event = {
+		.subsys_name = sysmon->name,
+		.ssr_event = SSCTL_SSR_EVENT_BEFORE_SHUTDOWN
+	};
 
-	blocking_notifier_call_chain(&sysmon_notifiers, 0, (void *)sysmon->name);
+	blocking_notifier_call_chain(&sysmon_notifiers, 0, (void *)&event);
 
 	/* Don't request graceful shutdown if we've crashed */
 	if (crashed)
@@ -445,6 +512,18 @@ static void sysmon_stop(struct rproc_subdev *subdev, bool crashed)
 		sysmon_request_shutdown(sysmon);
 }
 
+static void sysmon_unprepare(struct rproc_subdev *subdev)
+{
+	struct qcom_sysmon *sysmon = container_of(subdev, struct qcom_sysmon,
+						  subdev);
+	struct sysmon_event event = {
+		.subsys_name = sysmon->name,
+		.ssr_event = SSCTL_SSR_EVENT_AFTER_SHUTDOWN
+	};
+
+	blocking_notifier_call_chain(&sysmon_notifiers, 0, (void *)&event);
+}
+
 /**
  * sysmon_notify() - notify sysmon target of another's SSR
  * @nb:		notifier_block associated with sysmon instance
@@ -456,19 +535,20 @@ static int sysmon_notify(struct notifier_block *nb, unsigned long event,
 {
 	struct qcom_sysmon *sysmon = container_of(nb, struct qcom_sysmon, nb);
 	struct rproc *rproc = sysmon->rproc;
-	const char *ssr_name = data;
+	struct sysmon_event *sysmon_event = data;
 
 	/* Skip non-running rprocs and the originating instance */
-	if (rproc->state != RPROC_RUNNING || !strcmp(data, sysmon->name)) {
+	if (rproc->state != RPROC_RUNNING ||
+	    !strcmp(sysmon_event->subsys_name, sysmon->name)) {
 		dev_dbg(sysmon->dev, "not notifying %s\n", sysmon->name);
 		return NOTIFY_DONE;
 	}
 
 	/* Only SSCTL version 2 supports SSR events */
 	if (sysmon->ssctl_version == 2)
-		ssctl_send_event(sysmon, ssr_name);
+		ssctl_send_event(sysmon, sysmon_event);
 	else if (sysmon->ept)
-		sysmon_send_event(sysmon, ssr_name);
+		sysmon_send_event(sysmon, sysmon_event);
 
 	return NOTIFY_DONE;
 }
@@ -543,8 +623,10 @@ struct qcom_sysmon *qcom_add_sysmon_subdev(struct rproc *rproc,
 
 	qmi_add_lookup(&sysmon->qmi, 43, 0, 0);
 
+	sysmon->subdev.prepare = sysmon_prepare;
 	sysmon->subdev.start = sysmon_start;
 	sysmon->subdev.stop = sysmon_stop;
+	sysmon->subdev.unprepare = sysmon_unprepare;
 
 	rproc_add_subdev(rproc, &sysmon->subdev);
 
diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c
index 0c7afd038f0d..5d65e1a9329a 100644
--- a/drivers/remoteproc/qcom_wcnss.c
+++ b/drivers/remoteproc/qcom_wcnss.c
@@ -480,6 +480,7 @@ static int wcnss_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "unable to allocate remoteproc\n");
 		return -ENOMEM;
 	}
+	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 
 	wcnss = (struct qcom_wcnss *)rproc->priv;
 	wcnss->dev = &pdev->dev;
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index e12a54e67588..9f04c30c4aaf 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -29,6 +29,7 @@
 #include <linux/devcoredump.h>
 #include <linux/rculist.h>
 #include <linux/remoteproc.h>
+#include <linux/pm_runtime.h>
 #include <linux/iommu.h>
 #include <linux/idr.h>
 #include <linux/elf.h>
@@ -517,7 +518,7 @@ static int rproc_handle_vdev(struct rproc *rproc, struct fw_rsc_vdev *rsc,
 
 	/* Initialise vdev subdevice */
 	snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index);
-	rvdev->dev.parent = rproc->dev.parent;
+	rvdev->dev.parent = &rproc->dev;
 	rvdev->dev.dma_pfn_offset = rproc->dev.parent->dma_pfn_offset;
 	rvdev->dev.release = rproc_rvdev_release;
 	dev_set_name(&rvdev->dev, "%s#%s", dev_name(rvdev->dev.parent), name);
@@ -1382,6 +1383,12 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 	if (ret)
 		return ret;
 
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "pm_runtime_get_sync failed: %d\n", ret);
+		return ret;
+	}
+
 	dev_info(dev, "Booting fw image %s, size %zd\n", name, fw->size);
 
 	/*
@@ -1391,7 +1398,14 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 	ret = rproc_enable_iommu(rproc);
 	if (ret) {
 		dev_err(dev, "can't enable iommu: %d\n", ret);
-		return ret;
+		goto put_pm_runtime;
+	}
+
+	/* Prepare rproc for firmware loading if needed */
+	ret = rproc_prepare_device(rproc);
+	if (ret) {
+		dev_err(dev, "can't prepare rproc %s: %d\n", rproc->name, ret);
+		goto disable_iommu;
 	}
 
 	rproc->bootaddr = rproc_get_boot_addr(rproc, fw);
@@ -1399,7 +1413,7 @@ static int rproc_fw_boot(struct rproc *rproc, const struct firmware *fw)
 	/* Load resource table, core dump segment list etc from the firmware */
 	ret = rproc_parse_fw(rproc, fw);
 	if (ret)
-		goto disable_iommu;
+		goto unprepare_rproc;
 
 	/* reset max_notifyid */
 	rproc->max_notifyid = -1;
@@ -1433,8 +1447,13 @@ clean_up_resources:
 	kfree(rproc->cached_table);
 	rproc->cached_table = NULL;
 	rproc->table_ptr = NULL;
+unprepare_rproc:
+	/* release HW resources if needed */
+	rproc_unprepare_device(rproc);
 disable_iommu:
 	rproc_disable_iommu(rproc);
+put_pm_runtime:
+	pm_runtime_put(dev);
 	return ret;
 }
 
@@ -1566,6 +1585,28 @@ int rproc_coredump_add_custom_segment(struct rproc *rproc,
 EXPORT_SYMBOL(rproc_coredump_add_custom_segment);
 
 /**
+ * rproc_coredump_set_elf_info() - set coredump elf information
+ * @rproc:	handle of a remote processor
+ * @class:	elf class for coredump elf file
+ * @machine:	elf machine for coredump elf file
+ *
+ * Set elf information which will be used for coredump elf file.
+ *
+ * Return: 0 on success, negative errno on error.
+ */
+int rproc_coredump_set_elf_info(struct rproc *rproc, u8 class, u16 machine)
+{
+	if (class != ELFCLASS64 && class != ELFCLASS32)
+		return -EINVAL;
+
+	rproc->elf_class = class;
+	rproc->elf_machine = machine;
+
+	return 0;
+}
+EXPORT_SYMBOL(rproc_coredump_set_elf_info);
+
+/**
  * rproc_coredump() - perform coredump
  * @rproc:	rproc handle
  *
@@ -1587,6 +1628,11 @@ static void rproc_coredump(struct rproc *rproc)
 	if (list_empty(&rproc->dump_segments))
 		return;
 
+	if (class == ELFCLASSNONE) {
+		dev_err(&rproc->dev, "Elf class is not set\n");
+		return;
+	}
+
 	data_size = elf_size_of_hdr(class);
 	list_for_each_entry(segment, &rproc->dump_segments, node) {
 		data_size += elf_size_of_phdr(class) + segment->size;
@@ -1605,7 +1651,7 @@ static void rproc_coredump(struct rproc *rproc)
 	elf_hdr_init_ident(ehdr, class);
 
 	elf_hdr_set_e_type(class, ehdr, ET_CORE);
-	elf_hdr_set_e_machine(class, ehdr, EM_NONE);
+	elf_hdr_set_e_machine(class, ehdr, rproc->elf_machine);
 	elf_hdr_set_e_version(class, ehdr, EV_CURRENT);
 	elf_hdr_set_e_entry(class, ehdr, rproc->bootaddr);
 	elf_hdr_set_e_phoff(class, ehdr, elf_size_of_hdr(class));
@@ -1729,6 +1775,8 @@ static void rproc_crash_handler_work(struct work_struct *work)
 
 	if (!rproc->recovery_disabled)
 		rproc_trigger_recovery(rproc);
+
+	pm_relax(rproc->dev.parent);
 }
 
 /**
@@ -1838,8 +1886,13 @@ void rproc_shutdown(struct rproc *rproc)
 	/* clean up all acquired resources */
 	rproc_resource_cleanup(rproc);
 
+	/* release HW resources if needed */
+	rproc_unprepare_device(rproc);
+
 	rproc_disable_iommu(rproc);
 
+	pm_runtime_put(dev);
+
 	/* Free the copy of the resource table */
 	kfree(rproc->cached_table);
 	rproc->cached_table = NULL;
@@ -1949,6 +2002,33 @@ int rproc_add(struct rproc *rproc)
 }
 EXPORT_SYMBOL(rproc_add);
 
+static void devm_rproc_remove(void *rproc)
+{
+	rproc_del(rproc);
+}
+
+/**
+ * devm_rproc_add() - resource managed rproc_add()
+ * @dev: the underlying device
+ * @rproc: the remote processor handle to register
+ *
+ * This function performs like rproc_add() but the registered rproc device will
+ * automatically be removed on driver detach.
+ *
+ * Returns: 0 on success, negative errno on failure
+ */
+int devm_rproc_add(struct device *dev, struct rproc *rproc)
+{
+	int err;
+
+	err = rproc_add(rproc);
+	if (err)
+		return err;
+
+	return devm_add_action_or_reset(dev, devm_rproc_remove, rproc);
+}
+EXPORT_SYMBOL(devm_rproc_add);
+
 /**
  * rproc_type_release() - release a remote processor instance
  * @dev: the rproc's device
@@ -1969,7 +2049,8 @@ static void rproc_type_release(struct device *dev)
 	if (rproc->index >= 0)
 		ida_simple_remove(&rproc_dev_index, rproc->index);
 
-	kfree(rproc->firmware);
+	kfree_const(rproc->firmware);
+	kfree_const(rproc->name);
 	kfree(rproc->ops);
 	kfree(rproc);
 }
@@ -1979,6 +2060,47 @@ static const struct device_type rproc_type = {
 	.release	= rproc_type_release,
 };
 
+static int rproc_alloc_firmware(struct rproc *rproc,
+				const char *name, const char *firmware)
+{
+	const char *p;
+
+	/*
+	 * Allocate a firmware name if the caller gave us one to work
+	 * with.  Otherwise construct a new one using a default pattern.
+	 */
+	if (firmware)
+		p = kstrdup_const(firmware, GFP_KERNEL);
+	else
+		p = kasprintf(GFP_KERNEL, "rproc-%s-fw", name);
+
+	if (!p)
+		return -ENOMEM;
+
+	rproc->firmware = p;
+
+	return 0;
+}
+
+static int rproc_alloc_ops(struct rproc *rproc, const struct rproc_ops *ops)
+{
+	rproc->ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL);
+	if (!rproc->ops)
+		return -ENOMEM;
+
+	if (rproc->ops->load)
+		return 0;
+
+	/* Default to ELF loader if no load function is specified */
+	rproc->ops->load = rproc_elf_load_segments;
+	rproc->ops->parse_fw = rproc_elf_load_rsc_table;
+	rproc->ops->find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table;
+	rproc->ops->sanity_check = rproc_elf_sanity_check;
+	rproc->ops->get_boot_addr = rproc_elf_get_boot_addr;
+
+	return 0;
+}
+
 /**
  * rproc_alloc() - allocate a remote processor handle
  * @dev: the underlying device
@@ -2007,79 +2129,49 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 			  const char *firmware, int len)
 {
 	struct rproc *rproc;
-	char *p, *template = "rproc-%s-fw";
-	int name_len;
 
 	if (!dev || !name || !ops)
 		return NULL;
 
-	if (!firmware) {
-		/*
-		 * If the caller didn't pass in a firmware name then
-		 * construct a default name.
-		 */
-		name_len = strlen(name) + strlen(template) - 2 + 1;
-		p = kmalloc(name_len, GFP_KERNEL);
-		if (!p)
-			return NULL;
-		snprintf(p, name_len, template, name);
-	} else {
-		p = kstrdup(firmware, GFP_KERNEL);
-		if (!p)
-			return NULL;
-	}
-
 	rproc = kzalloc(sizeof(struct rproc) + len, GFP_KERNEL);
-	if (!rproc) {
-		kfree(p);
-		return NULL;
-	}
-
-	rproc->ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL);
-	if (!rproc->ops) {
-		kfree(p);
-		kfree(rproc);
+	if (!rproc)
 		return NULL;
-	}
 
-	rproc->firmware = p;
-	rproc->name = name;
 	rproc->priv = &rproc[1];
 	rproc->auto_boot = true;
-	rproc->elf_class = ELFCLASS32;
+	rproc->elf_class = ELFCLASSNONE;
+	rproc->elf_machine = EM_NONE;
 
 	device_initialize(&rproc->dev);
 	rproc->dev.parent = dev;
 	rproc->dev.type = &rproc_type;
 	rproc->dev.class = &rproc_class;
 	rproc->dev.driver_data = rproc;
+	idr_init(&rproc->notifyids);
+
+	rproc->name = kstrdup_const(name, GFP_KERNEL);
+	if (!rproc->name)
+		goto put_device;
+
+	if (rproc_alloc_firmware(rproc, name, firmware))
+		goto put_device;
+
+	if (rproc_alloc_ops(rproc, ops))
+		goto put_device;
 
 	/* Assign a unique device index and name */
 	rproc->index = ida_simple_get(&rproc_dev_index, 0, 0, GFP_KERNEL);
 	if (rproc->index < 0) {
 		dev_err(dev, "ida_simple_get failed: %d\n", rproc->index);
-		put_device(&rproc->dev);
-		return NULL;
+		goto put_device;
 	}
 
 	dev_set_name(&rproc->dev, "remoteproc%d", rproc->index);
 
 	atomic_set(&rproc->power, 0);
 
-	/* Default to ELF loader if no load function is specified */
-	if (!rproc->ops->load) {
-		rproc->ops->load = rproc_elf_load_segments;
-		rproc->ops->parse_fw = rproc_elf_load_rsc_table;
-		rproc->ops->find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table;
-		if (!rproc->ops->sanity_check)
-			rproc->ops->sanity_check = rproc_elf32_sanity_check;
-		rproc->ops->get_boot_addr = rproc_elf_get_boot_addr;
-	}
-
 	mutex_init(&rproc->lock);
 
-	idr_init(&rproc->notifyids);
-
 	INIT_LIST_HEAD(&rproc->carveouts);
 	INIT_LIST_HEAD(&rproc->mappings);
 	INIT_LIST_HEAD(&rproc->traces);
@@ -2091,7 +2183,14 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 
 	rproc->state = RPROC_OFFLINE;
 
+	pm_runtime_no_callbacks(&rproc->dev);
+	pm_runtime_enable(&rproc->dev);
+
 	return rproc;
+
+put_device:
+	put_device(&rproc->dev);
+	return NULL;
 }
 EXPORT_SYMBOL(rproc_alloc);
 
@@ -2106,6 +2205,7 @@ EXPORT_SYMBOL(rproc_alloc);
  */
 void rproc_free(struct rproc *rproc)
 {
+	pm_runtime_disable(&rproc->dev);
 	put_device(&rproc->dev);
 }
 EXPORT_SYMBOL(rproc_free);
@@ -2171,6 +2271,46 @@ int rproc_del(struct rproc *rproc)
 }
 EXPORT_SYMBOL(rproc_del);
 
+static void devm_rproc_free(struct device *dev, void *res)
+{
+	rproc_free(*(struct rproc **)res);
+}
+
+/**
+ * devm_rproc_alloc() - resource managed rproc_alloc()
+ * @dev: the underlying device
+ * @name: name of this remote processor
+ * @ops: platform-specific handlers (mainly start/stop)
+ * @firmware: name of firmware file to load, can be NULL
+ * @len: length of private data needed by the rproc driver (in bytes)
+ *
+ * This function performs like rproc_alloc() but the acquired rproc device will
+ * automatically be released on driver detach.
+ *
+ * Returns: new rproc instance, or NULL on failure
+ */
+struct rproc *devm_rproc_alloc(struct device *dev, const char *name,
+			       const struct rproc_ops *ops,
+			       const char *firmware, int len)
+{
+	struct rproc **ptr, *rproc;
+
+	ptr = devres_alloc(devm_rproc_free, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return NULL;
+
+	rproc = rproc_alloc(dev, name, ops, firmware, len);
+	if (rproc) {
+		*ptr = rproc;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
+	}
+
+	return rproc;
+}
+EXPORT_SYMBOL(devm_rproc_alloc);
+
 /**
  * rproc_add_subdev() - add a subdevice to a remoteproc
  * @rproc: rproc handle to add the subdevice to
@@ -2230,6 +2370,9 @@ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type)
 		return;
 	}
 
+	/* Prevent suspend while the remoteproc is being recovered */
+	pm_stay_awake(rproc->dev.parent);
+
 	dev_err(&rproc->dev, "crash detected in %s: type %s\n",
 		rproc->name, rproc_crash_to_string(type));
 
diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c
index d734cadb16e3..732770e92b99 100644
--- a/drivers/remoteproc/remoteproc_debugfs.c
+++ b/drivers/remoteproc/remoteproc_debugfs.c
@@ -269,17 +269,7 @@ static int rproc_rsc_table_show(struct seq_file *seq, void *p)
 	return 0;
 }
 
-static int rproc_rsc_table_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, rproc_rsc_table_show, inode->i_private);
-}
-
-static const struct file_operations rproc_rsc_table_ops = {
-	.open		= rproc_rsc_table_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(rproc_rsc_table);
 
 /* Expose carveout content via debugfs */
 static int rproc_carveouts_show(struct seq_file *seq, void *p)
@@ -299,17 +289,7 @@ static int rproc_carveouts_show(struct seq_file *seq, void *p)
 	return 0;
 }
 
-static int rproc_carveouts_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, rproc_carveouts_show, inode->i_private);
-}
-
-static const struct file_operations rproc_carveouts_ops = {
-	.open		= rproc_carveouts_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(rproc_carveouts);
 
 void rproc_remove_trace_file(struct dentry *tfile)
 {
@@ -354,9 +334,9 @@ void rproc_create_debug_dir(struct rproc *rproc)
 	debugfs_create_file("crash", 0200, rproc->dbg_dir,
 			    rproc, &rproc_crash_ops);
 	debugfs_create_file("resource_table", 0400, rproc->dbg_dir,
-			    rproc, &rproc_rsc_table_ops);
+			    rproc, &rproc_rsc_table_fops);
 	debugfs_create_file("carveout_memories", 0400, rproc->dbg_dir,
-			    rproc, &rproc_carveouts_ops);
+			    rproc, &rproc_carveouts_fops);
 }
 
 void __init rproc_init_debugfs(void)
diff --git a/drivers/remoteproc/remoteproc_elf_loader.c b/drivers/remoteproc/remoteproc_elf_loader.c
index 16e2c496fd45..df68d87752e4 100644
--- a/drivers/remoteproc/remoteproc_elf_loader.c
+++ b/drivers/remoteproc/remoteproc_elf_loader.c
@@ -113,27 +113,6 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw)
 EXPORT_SYMBOL(rproc_elf_sanity_check);
 
 /**
- * rproc_elf_sanity_check() - Sanity Check ELF32 firmware image
- * @rproc: the remote processor handle
- * @fw: the ELF32 firmware image
- *
- * Make sure this fw image is sane.
- */
-int rproc_elf32_sanity_check(struct rproc *rproc, const struct firmware *fw)
-{
-	int ret = rproc_elf_sanity_check(rproc, fw);
-
-	if (ret)
-		return ret;
-
-	if (fw_elf_get_class(fw) == ELFCLASS32)
-		return 0;
-
-	return -EINVAL;
-}
-EXPORT_SYMBOL(rproc_elf32_sanity_check);
-
-/**
  * rproc_elf_get_boot_addr() - Get rproc's boot address.
  * @rproc: the remote processor handle
  * @fw: the ELF firmware image
@@ -248,9 +227,6 @@ int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
 			memset(ptr + filesz, 0, memsz - filesz);
 	}
 
-	if (ret == 0)
-		rproc->elf_class = class;
-
 	return ret;
 }
 EXPORT_SYMBOL(rproc_elf_load_segments);
diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h
index b389dc79da81..4ba7cb59d3e8 100644
--- a/drivers/remoteproc/remoteproc_internal.h
+++ b/drivers/remoteproc/remoteproc_internal.h
@@ -54,7 +54,6 @@ void *rproc_da_to_va(struct rproc *rproc, u64 da, size_t len);
 phys_addr_t rproc_va_to_pa(void *cpu_addr);
 int rproc_trigger_recovery(struct rproc *rproc);
 
-int rproc_elf32_sanity_check(struct rproc *rproc, const struct firmware *fw);
 int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw);
 u64 rproc_elf_get_boot_addr(struct rproc *rproc, const struct firmware *fw);
 int rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw);
@@ -64,6 +63,22 @@ struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc,
 struct rproc_mem_entry *
 rproc_find_carveout_by_name(struct rproc *rproc, const char *name, ...);
 
+static inline int rproc_prepare_device(struct rproc *rproc)
+{
+	if (rproc->ops->prepare)
+		return rproc->ops->prepare(rproc);
+
+	return 0;
+}
+
+static inline int rproc_unprepare_device(struct rproc *rproc)
+{
+	if (rproc->ops->unprepare)
+		return rproc->ops->unprepare(rproc);
+
+	return 0;
+}
+
 static inline
 int rproc_fw_sanity_check(struct rproc *rproc, const struct firmware *fw)
 {
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index e61d738d9b47..dfd3808c34fd 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -337,8 +337,7 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id)
 
 	if (rproc->ops->kick == NULL) {
 		ret = -EINVAL;
-		dev_err(dev, ".kick method not defined for %s",
-				rproc->name);
+		dev_err(dev, ".kick method not defined for %s\n", rproc->name);
 		goto out;
 	}
 
@@ -376,6 +375,18 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id)
 				goto out;
 			}
 		}
+	} else {
+		struct device_node *np = rproc->dev.parent->of_node;
+
+		/*
+		 * If we don't have dedicated buffer, just attempt to re-assign
+		 * the reserved memory from our parent. A default memory-region
+		 * at index 0 from the parent's memory-regions is assigned for
+		 * the rvdev dev to allocate from. Failure is non-critical and
+		 * the allocations will fall back to global pools, so don't
+		 * check return value either.
+		 */
+		of_reserved_mem_device_init_by_idx(dev, np, 0);
 	}
 
 	/* Allocate virtio device */
diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c
index a6cbfa452764..a3268d95a50e 100644
--- a/drivers/remoteproc/st_remoteproc.c
+++ b/drivers/remoteproc/st_remoteproc.c
@@ -233,7 +233,7 @@ static const struct rproc_ops st_rproc_ops = {
 	.parse_fw		= st_rproc_parse_fw,
 	.load			= rproc_elf_load_segments,
 	.find_loaded_rsc_table	= rproc_elf_find_loaded_rsc_table,
-	.sanity_check		= rproc_elf32_sanity_check,
+	.sanity_check		= rproc_elf_sanity_check,
 	.get_boot_addr		= rproc_elf_get_boot_addr,
 };
 
diff --git a/drivers/remoteproc/st_slim_rproc.c b/drivers/remoteproc/st_slim_rproc.c
index 3cca8b65a8db..09bcb4d8b9e0 100644
--- a/drivers/remoteproc/st_slim_rproc.c
+++ b/drivers/remoteproc/st_slim_rproc.c
@@ -203,7 +203,7 @@ static const struct rproc_ops slim_rproc_ops = {
 	.da_to_va       = slim_rproc_da_to_va,
 	.get_boot_addr	= rproc_elf_get_boot_addr,
 	.load		= rproc_elf_load_segments,
-	.sanity_check	= rproc_elf32_sanity_check,
+	.sanity_check	= rproc_elf_sanity_check,
 };
 
 /**
diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c
index 0bdd56f02f18..062797a447c6 100644
--- a/drivers/remoteproc/stm32_rproc.c
+++ b/drivers/remoteproc/stm32_rproc.c
@@ -506,7 +506,7 @@ static struct rproc_ops st_rproc_ops = {
 	.load		= rproc_elf_load_segments,
 	.parse_fw	= stm32_rproc_parse_fw,
 	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
-	.sanity_check	= rproc_elf32_sanity_check,
+	.sanity_check	= rproc_elf_sanity_check,
 	.get_boot_addr	= rproc_elf_get_boot_addr,
 };
 
@@ -626,6 +626,7 @@ static int stm32_rproc_probe(struct platform_device *pdev)
 	if (!rproc)
 		return -ENOMEM;
 
+	rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_NONE);
 	rproc->has_iommu = false;
 	ddata = rproc->priv;
 	ddata->workqueue = create_workqueue(dev_name(dev));
diff --git a/drivers/reset/hisilicon/hi6220_reset.c b/drivers/reset/hisilicon/hi6220_reset.c
index 24e6d420b26b..19926506d033 100644
--- a/drivers/reset/hisilicon/hi6220_reset.c
+++ b/drivers/reset/hisilicon/hi6220_reset.c
@@ -33,6 +33,7 @@
 enum hi6220_reset_ctrl_type {
 	PERIPHERAL,
 	MEDIA,
+	AO,
 };
 
 struct hi6220_reset_data {
@@ -92,6 +93,65 @@ static const struct reset_control_ops hi6220_media_reset_ops = {
 	.deassert = hi6220_media_deassert,
 };
 
+#define AO_SCTRL_SC_PW_CLKEN0     0x800
+#define AO_SCTRL_SC_PW_CLKDIS0    0x804
+
+#define AO_SCTRL_SC_PW_RSTEN0     0x810
+#define AO_SCTRL_SC_PW_RSTDIS0    0x814
+
+#define AO_SCTRL_SC_PW_ISOEN0     0x820
+#define AO_SCTRL_SC_PW_ISODIS0    0x824
+#define AO_MAX_INDEX              12
+
+static int hi6220_ao_assert(struct reset_controller_dev *rc_dev,
+			       unsigned long idx)
+{
+	struct hi6220_reset_data *data = to_reset_data(rc_dev);
+	struct regmap *regmap = data->regmap;
+	int ret;
+
+	ret = regmap_write(regmap, AO_SCTRL_SC_PW_RSTEN0, BIT(idx));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(regmap, AO_SCTRL_SC_PW_ISOEN0, BIT(idx));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(regmap, AO_SCTRL_SC_PW_CLKDIS0, BIT(idx));
+	return ret;
+}
+
+static int hi6220_ao_deassert(struct reset_controller_dev *rc_dev,
+				 unsigned long idx)
+{
+	struct hi6220_reset_data *data = to_reset_data(rc_dev);
+	struct regmap *regmap = data->regmap;
+	int ret;
+
+	/*
+	 * It was suggested to disable isolation before enabling
+	 * the clocks and deasserting reset, to avoid glitches.
+	 * But this order is preserved to keep it matching the
+	 * vendor code.
+	 */
+	ret = regmap_write(regmap, AO_SCTRL_SC_PW_RSTDIS0, BIT(idx));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(regmap, AO_SCTRL_SC_PW_ISODIS0, BIT(idx));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(regmap, AO_SCTRL_SC_PW_CLKEN0, BIT(idx));
+	return ret;
+}
+
+static const struct reset_control_ops hi6220_ao_reset_ops = {
+	.assert = hi6220_ao_assert,
+	.deassert = hi6220_ao_deassert,
+};
+
 static int hi6220_reset_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -117,9 +177,12 @@ static int hi6220_reset_probe(struct platform_device *pdev)
 	if (type == MEDIA) {
 		data->rc_dev.ops = &hi6220_media_reset_ops;
 		data->rc_dev.nr_resets = MEDIA_MAX_INDEX;
-	} else {
+	} else if (type == PERIPHERAL) {
 		data->rc_dev.ops = &hi6220_peripheral_reset_ops;
 		data->rc_dev.nr_resets = PERIPH_MAX_INDEX;
+	} else {
+		data->rc_dev.ops = &hi6220_ao_reset_ops;
+		data->rc_dev.nr_resets = AO_MAX_INDEX;
 	}
 
 	return reset_controller_register(&data->rc_dev);
@@ -134,6 +197,10 @@ static const struct of_device_id hi6220_reset_match[] = {
 		.compatible = "hisilicon,hi6220-mediactrl",
 		.data = (void *)MEDIA,
 	},
+	{
+		.compatible = "hisilicon,hi6220-aoctrl",
+		.data = (void *)AO,
+	},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, hi6220_reset_match);
diff --git a/drivers/reset/reset-imx7.c b/drivers/reset/reset-imx7.c
index 1443a55a0c29..d170fe663210 100644
--- a/drivers/reset/reset-imx7.c
+++ b/drivers/reset/reset-imx7.c
@@ -15,6 +15,7 @@
 #include <linux/regmap.h>
 #include <dt-bindings/reset/imx7-reset.h>
 #include <dt-bindings/reset/imx8mq-reset.h>
+#include <dt-bindings/reset/imx8mp-reset.h>
 
 struct imx7_src_signal {
 	unsigned int offset, bit;
@@ -145,6 +146,18 @@ enum imx8mq_src_registers {
 	SRC_DDRC2_RCR		= 0x1004,
 };
 
+enum imx8mp_src_registers {
+	SRC_SUPERMIX_RCR	= 0x0018,
+	SRC_AUDIOMIX_RCR	= 0x001c,
+	SRC_MLMIX_RCR		= 0x0028,
+	SRC_GPU2D_RCR		= 0x0038,
+	SRC_GPU3D_RCR		= 0x003c,
+	SRC_VPU_G1_RCR		= 0x0048,
+	SRC_VPU_G2_RCR		= 0x004c,
+	SRC_VPUVC8KE_RCR	= 0x0050,
+	SRC_NOC_RCR		= 0x0054,
+};
+
 static const struct imx7_src_signal imx8mq_src_signals[IMX8MQ_RESET_NUM] = {
 	[IMX8MQ_RESET_A53_CORE_POR_RESET0]	= { SRC_A53RCR0, BIT(0) },
 	[IMX8MQ_RESET_A53_CORE_POR_RESET1]	= { SRC_A53RCR0, BIT(1) },
@@ -253,6 +266,93 @@ static const struct imx7_src_variant variant_imx8mq = {
 	},
 };
 
+static const struct imx7_src_signal imx8mp_src_signals[IMX8MP_RESET_NUM] = {
+	[IMX8MP_RESET_A53_CORE_POR_RESET0]	= { SRC_A53RCR0, BIT(0) },
+	[IMX8MP_RESET_A53_CORE_POR_RESET1]	= { SRC_A53RCR0, BIT(1) },
+	[IMX8MP_RESET_A53_CORE_POR_RESET2]	= { SRC_A53RCR0, BIT(2) },
+	[IMX8MP_RESET_A53_CORE_POR_RESET3]	= { SRC_A53RCR0, BIT(3) },
+	[IMX8MP_RESET_A53_CORE_RESET0]		= { SRC_A53RCR0, BIT(4) },
+	[IMX8MP_RESET_A53_CORE_RESET1]		= { SRC_A53RCR0, BIT(5) },
+	[IMX8MP_RESET_A53_CORE_RESET2]		= { SRC_A53RCR0, BIT(6) },
+	[IMX8MP_RESET_A53_CORE_RESET3]		= { SRC_A53RCR0, BIT(7) },
+	[IMX8MP_RESET_A53_DBG_RESET0]		= { SRC_A53RCR0, BIT(8) },
+	[IMX8MP_RESET_A53_DBG_RESET1]		= { SRC_A53RCR0, BIT(9) },
+	[IMX8MP_RESET_A53_DBG_RESET2]		= { SRC_A53RCR0, BIT(10) },
+	[IMX8MP_RESET_A53_DBG_RESET3]		= { SRC_A53RCR0, BIT(11) },
+	[IMX8MP_RESET_A53_ETM_RESET0]		= { SRC_A53RCR0, BIT(12) },
+	[IMX8MP_RESET_A53_ETM_RESET1]		= { SRC_A53RCR0, BIT(13) },
+	[IMX8MP_RESET_A53_ETM_RESET2]		= { SRC_A53RCR0, BIT(14) },
+	[IMX8MP_RESET_A53_ETM_RESET3]		= { SRC_A53RCR0, BIT(15) },
+	[IMX8MP_RESET_A53_SOC_DBG_RESET]	= { SRC_A53RCR0, BIT(20) },
+	[IMX8MP_RESET_A53_L2RESET]		= { SRC_A53RCR0, BIT(21) },
+	[IMX8MP_RESET_SW_NON_SCLR_M7C_RST]	= { SRC_M4RCR, BIT(0) },
+	[IMX8MP_RESET_OTG1_PHY_RESET]		= { SRC_USBOPHY1_RCR, BIT(0) },
+	[IMX8MP_RESET_OTG2_PHY_RESET]		= { SRC_USBOPHY2_RCR, BIT(0) },
+	[IMX8MP_RESET_SUPERMIX_RESET]		= { SRC_SUPERMIX_RCR, BIT(0) },
+	[IMX8MP_RESET_AUDIOMIX_RESET]		= { SRC_AUDIOMIX_RCR, BIT(0) },
+	[IMX8MP_RESET_MLMIX_RESET]		= { SRC_MLMIX_RCR, BIT(0) },
+	[IMX8MP_RESET_PCIEPHY]			= { SRC_PCIEPHY_RCR, BIT(2) },
+	[IMX8MP_RESET_PCIEPHY_PERST]		= { SRC_PCIEPHY_RCR, BIT(3) },
+	[IMX8MP_RESET_PCIE_CTRL_APPS_EN]	= { SRC_PCIEPHY_RCR, BIT(6) },
+	[IMX8MP_RESET_PCIE_CTRL_APPS_TURNOFF]	= { SRC_PCIEPHY_RCR, BIT(11) },
+	[IMX8MP_RESET_HDMI_PHY_APB_RESET]	= { SRC_HDMI_RCR, BIT(0) },
+	[IMX8MP_RESET_MEDIA_RESET]		= { SRC_DISP_RCR, BIT(0) },
+	[IMX8MP_RESET_GPU2D_RESET]		= { SRC_GPU2D_RCR, BIT(0) },
+	[IMX8MP_RESET_GPU3D_RESET]		= { SRC_GPU3D_RCR, BIT(0) },
+	[IMX8MP_RESET_GPU_RESET]		= { SRC_GPU_RCR, BIT(0) },
+	[IMX8MP_RESET_VPU_RESET]		= { SRC_VPU_RCR, BIT(0) },
+	[IMX8MP_RESET_VPU_G1_RESET]		= { SRC_VPU_G1_RCR, BIT(0) },
+	[IMX8MP_RESET_VPU_G2_RESET]		= { SRC_VPU_G2_RCR, BIT(0) },
+	[IMX8MP_RESET_VPUVC8KE_RESET]		= { SRC_VPUVC8KE_RCR, BIT(0) },
+	[IMX8MP_RESET_NOC_RESET]		= { SRC_NOC_RCR, BIT(0) },
+};
+
+static int imx8mp_reset_set(struct reset_controller_dev *rcdev,
+			    unsigned long id, bool assert)
+{
+	struct imx7_src *imx7src = to_imx7_src(rcdev);
+	const unsigned int bit = imx7src->signals[id].bit;
+	unsigned int value = assert ? bit : 0;
+
+	switch (id) {
+	case IMX8MP_RESET_PCIEPHY:
+		/*
+		 * wait for more than 10us to release phy g_rst and
+		 * btnrst
+		 */
+		if (!assert)
+			udelay(10);
+		break;
+
+	case IMX8MP_RESET_PCIE_CTRL_APPS_EN:
+		value = assert ? 0 : bit;
+		break;
+	}
+
+	return imx7_reset_update(imx7src, id, value);
+}
+
+static int imx8mp_reset_assert(struct reset_controller_dev *rcdev,
+			       unsigned long id)
+{
+	return imx8mp_reset_set(rcdev, id, true);
+}
+
+static int imx8mp_reset_deassert(struct reset_controller_dev *rcdev,
+				 unsigned long id)
+{
+	return imx8mp_reset_set(rcdev, id, false);
+}
+
+static const struct imx7_src_variant variant_imx8mp = {
+	.signals = imx8mp_src_signals,
+	.signals_num = ARRAY_SIZE(imx8mp_src_signals),
+	.ops = {
+		.assert   = imx8mp_reset_assert,
+		.deassert = imx8mp_reset_deassert,
+	},
+};
+
 static int imx7_reset_probe(struct platform_device *pdev)
 {
 	struct imx7_src *imx7src;
@@ -283,6 +383,7 @@ static int imx7_reset_probe(struct platform_device *pdev)
 static const struct of_device_id imx7_reset_dt_ids[] = {
 	{ .compatible = "fsl,imx7d-src", .data = &variant_imx7 },
 	{ .compatible = "fsl,imx8mq-src", .data = &variant_imx8mq },
+	{ .compatible = "fsl,imx8mp-src", .data = &variant_imx8mp },
 	{ /* sentinel */ },
 };
 
diff --git a/drivers/reset/reset-zynqmp.c b/drivers/reset/reset-zynqmp.c
index 0144075b11a6..373ea8d4f7a1 100644
--- a/drivers/reset/reset-zynqmp.c
+++ b/drivers/reset/reset-zynqmp.c
@@ -15,7 +15,6 @@
 
 struct zynqmp_reset_data {
 	struct reset_controller_dev rcdev;
-	const struct zynqmp_eemi_ops *eemi_ops;
 };
 
 static inline struct zynqmp_reset_data *
@@ -27,28 +26,23 @@ to_zynqmp_reset_data(struct reset_controller_dev *rcdev)
 static int zynqmp_reset_assert(struct reset_controller_dev *rcdev,
 			       unsigned long id)
 {
-	struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev);
-
-	return priv->eemi_ops->reset_assert(ZYNQMP_RESET_ID + id,
-					    PM_RESET_ACTION_ASSERT);
+	return zynqmp_pm_reset_assert(ZYNQMP_RESET_ID + id,
+				      PM_RESET_ACTION_ASSERT);
 }
 
 static int zynqmp_reset_deassert(struct reset_controller_dev *rcdev,
 				 unsigned long id)
 {
-	struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev);
-
-	return priv->eemi_ops->reset_assert(ZYNQMP_RESET_ID + id,
-					    PM_RESET_ACTION_RELEASE);
+	return zynqmp_pm_reset_assert(ZYNQMP_RESET_ID + id,
+				      PM_RESET_ACTION_RELEASE);
 }
 
 static int zynqmp_reset_status(struct reset_controller_dev *rcdev,
 			       unsigned long id)
 {
-	struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev);
 	int val, err;
 
-	err = priv->eemi_ops->reset_get_status(ZYNQMP_RESET_ID + id, &val);
+	err = zynqmp_pm_reset_get_status(ZYNQMP_RESET_ID + id, &val);
 	if (err)
 		return err;
 
@@ -58,10 +52,8 @@ static int zynqmp_reset_status(struct reset_controller_dev *rcdev,
 static int zynqmp_reset_reset(struct reset_controller_dev *rcdev,
 			      unsigned long id)
 {
-	struct zynqmp_reset_data *priv = to_zynqmp_reset_data(rcdev);
-
-	return priv->eemi_ops->reset_assert(ZYNQMP_RESET_ID + id,
-					    PM_RESET_ACTION_PULSE);
+	return zynqmp_pm_reset_assert(ZYNQMP_RESET_ID + id,
+				      PM_RESET_ACTION_PULSE);
 }
 
 static const struct reset_control_ops zynqmp_reset_ops = {
@@ -79,10 +71,6 @@ static int zynqmp_reset_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
-	priv->eemi_ops = zynqmp_pm_get_eemi_ops();
-	if (IS_ERR(priv->eemi_ops))
-		return PTR_ERR(priv->eemi_ops);
-
 	platform_set_drvdata(pdev, priv);
 
 	priv->rcdev.ops = &zynqmp_reset_ops;
diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig
index a9108ff563dc..f96716893c2a 100644
--- a/drivers/rpmsg/Kconfig
+++ b/drivers/rpmsg/Kconfig
@@ -24,13 +24,13 @@ config RPMSG_MTK_SCP
 	  remote processors in MediaTek platforms.
 	  This use IPI and IPC to communicate with remote processors.
 
-config RPMSG_QCOM_GLINK_NATIVE
+config RPMSG_QCOM_GLINK
 	tristate
 	select RPMSG
 
 config RPMSG_QCOM_GLINK_RPM
 	tristate "Qualcomm RPM Glink driver"
-	select RPMSG_QCOM_GLINK_NATIVE
+	select RPMSG_QCOM_GLINK
 	depends on HAS_IOMEM
 	depends on MAILBOX
 	help
@@ -40,7 +40,7 @@ config RPMSG_QCOM_GLINK_RPM
 
 config RPMSG_QCOM_GLINK_SMEM
 	tristate "Qualcomm SMEM Glink driver"
-	select RPMSG_QCOM_GLINK_NATIVE
+	select RPMSG_QCOM_GLINK
 	depends on MAILBOX
 	depends on QCOM_SMEM
 	help
diff --git a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile
index ae92a7fb08f6..ffe932ef6050 100644
--- a/drivers/rpmsg/Makefile
+++ b/drivers/rpmsg/Makefile
@@ -2,8 +2,9 @@
 obj-$(CONFIG_RPMSG)		+= rpmsg_core.o
 obj-$(CONFIG_RPMSG_CHAR)	+= rpmsg_char.o
 obj-$(CONFIG_RPMSG_MTK_SCP)	+= mtk_rpmsg.o
+qcom_glink-objs			:= qcom_glink_native.o qcom_glink_ssr.o
+obj-$(CONFIG_RPMSG_QCOM_GLINK) += qcom_glink.o
 obj-$(CONFIG_RPMSG_QCOM_GLINK_RPM) += qcom_glink_rpm.o
-obj-$(CONFIG_RPMSG_QCOM_GLINK_NATIVE) += qcom_glink_native.o
 obj-$(CONFIG_RPMSG_QCOM_GLINK_SMEM) += qcom_glink_smem.o
 obj-$(CONFIG_RPMSG_QCOM_SMD)	+= qcom_smd.o
 obj-$(CONFIG_RPMSG_VIRTIO)	+= virtio_rpmsg_bus.o
diff --git a/drivers/soc/qcom/glink_ssr.c b/drivers/rpmsg/qcom_glink_ssr.c
index d7babe3d67bc..dcd1ce616974 100644
--- a/drivers/soc/qcom/glink_ssr.c
+++ b/drivers/rpmsg/qcom_glink_ssr.c
@@ -54,6 +54,19 @@ struct glink_ssr {
 	struct completion completion;
 };
 
+/* Notifier list for all registered glink_ssr instances */
+static BLOCKING_NOTIFIER_HEAD(ssr_notifiers);
+
+/**
+ * qcom_glink_ssr_notify() - notify GLINK SSR about stopped remoteproc
+ * @ssr_name:	name of the remoteproc that has been stopped
+ */
+void qcom_glink_ssr_notify(const char *ssr_name)
+{
+	blocking_notifier_call_chain(&ssr_notifiers, 0, (void *)ssr_name);
+}
+EXPORT_SYMBOL_GPL(qcom_glink_ssr_notify);
+
 static int qcom_glink_ssr_callback(struct rpmsg_device *rpdev,
 				   void *data, int len, void *priv, u32 addr)
 {
@@ -81,8 +94,9 @@ static int qcom_glink_ssr_callback(struct rpmsg_device *rpdev,
 	return 0;
 }
 
-static int qcom_glink_ssr_notify(struct notifier_block *nb, unsigned long event,
-				 void *data)
+static int qcom_glink_ssr_notifier_call(struct notifier_block *nb,
+					unsigned long event,
+					void *data)
 {
 	struct glink_ssr *ssr = container_of(nb, struct glink_ssr, nb);
 	struct do_cleanup_msg msg;
@@ -121,18 +135,18 @@ static int qcom_glink_ssr_probe(struct rpmsg_device *rpdev)
 
 	ssr->dev = &rpdev->dev;
 	ssr->ept = rpdev->ept;
-	ssr->nb.notifier_call = qcom_glink_ssr_notify;
+	ssr->nb.notifier_call = qcom_glink_ssr_notifier_call;
 
 	dev_set_drvdata(&rpdev->dev, ssr);
 
-	return qcom_register_ssr_notifier(&ssr->nb);
+	return blocking_notifier_chain_register(&ssr_notifiers, &ssr->nb);
 }
 
 static void qcom_glink_ssr_remove(struct rpmsg_device *rpdev)
 {
 	struct glink_ssr *ssr = dev_get_drvdata(&rpdev->dev);
 
-	qcom_unregister_ssr_notifier(&ssr->nb);
+	blocking_notifier_chain_unregister(&ssr_notifiers, &ssr->nb);
 }
 
 static const struct rpmsg_device_id qcom_glink_ssr_match[] = {
@@ -150,7 +164,3 @@ static struct rpmsg_driver qcom_glink_ssr_driver = {
 	},
 };
 module_rpmsg_driver(qcom_glink_ssr_driver);
-
-MODULE_ALIAS("rpmsg:glink_ssr");
-MODULE_DESCRIPTION("Qualcomm GLINK SSR notifier");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/rpmsg/rpmsg_core.c b/drivers/rpmsg/rpmsg_core.c
index e330ec4dfc33..a6361cad608b 100644
--- a/drivers/rpmsg/rpmsg_core.c
+++ b/drivers/rpmsg/rpmsg_core.c
@@ -284,7 +284,7 @@ int rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, u32 dst,
 EXPORT_SYMBOL(rpmsg_trysend_offchannel);
 
 /*
- * match an rpmsg channel with a channel info struct.
+ * match a rpmsg channel with a channel info struct.
  * this is used to make sure we're not creating rpmsg devices for channels
  * that already exist.
  */
diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
index 376ebbf880d6..07d4f3374098 100644
--- a/drivers/rpmsg/virtio_rpmsg_bus.c
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c
@@ -89,7 +89,7 @@ struct rpmsg_hdr {
 	u32 reserved;
 	u16 len;
 	u16 flags;
-	u8 data[0];
+	u8 data[];
 } __packed;
 
 /**
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index ec873f09c763..b54d87d45c89 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -1516,7 +1516,7 @@ config RTC_DRV_GENERIC
 	tristate "Generic RTC support"
 	# Please consider writing a new RTC driver instead of using the generic
 	# RTC abstraction
-	depends on PARISC || M68K || PPC || SUPERH32 || COMPILE_TEST
+	depends on PARISC || M68K || PPC || SUPERH || COMPILE_TEST
 	help
 	  Say Y or M here to enable RTC support on systems using the generic
 	  RTC abstraction. If you do not know what you are doing, you should
@@ -1680,6 +1680,7 @@ config RTC_DRV_MPC5121
 config RTC_DRV_JZ4740
 	tristate "Ingenic JZ4740 SoC"
 	depends on MIPS || COMPILE_TEST
+	depends on OF
 	help
 	  If you say yes here you get support for the Ingenic JZ47xx SoCs RTC
 	  controllers.
diff --git a/drivers/rtc/rtc-88pm860x.c b/drivers/rtc/rtc-88pm860x.c
index cc9b14ef90f1..c90457d001e9 100644
--- a/drivers/rtc/rtc-88pm860x.c
+++ b/drivers/rtc/rtc-88pm860x.c
@@ -106,12 +106,6 @@ static int pm860x_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	unsigned char buf[4];
 	unsigned long ticks, base, data;
 
-	if (tm->tm_year > 206) {
-		dev_dbg(info->dev, "Set time %d out of range. "
-			"Please set time between 1970 to 2106.\n",
-			1900 + tm->tm_year);
-		return -EINVAL;
-	}
 	ticks = rtc_tm_to_time64(tm);
 
 	/* load 32-bit read-only counter */
diff --git a/drivers/rtc/rtc-abx80x.c b/drivers/rtc/rtc-abx80x.c
index 3521d8e8dc38..803725b3a02c 100644
--- a/drivers/rtc/rtc-abx80x.c
+++ b/drivers/rtc/rtc-abx80x.c
@@ -13,6 +13,7 @@
 #include <linux/bcd.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/rtc.h>
 #include <linux/watchdog.h>
 
@@ -554,8 +555,9 @@ static const struct rtc_class_ops abx80x_rtc_ops = {
 	.ioctl		= abx80x_ioctl,
 };
 
-static int abx80x_dt_trickle_cfg(struct device_node *np)
+static int abx80x_dt_trickle_cfg(struct i2c_client *client)
 {
+	struct device_node *np = client->dev.of_node;
 	const char *diode;
 	int trickle_cfg = 0;
 	int i, ret;
@@ -565,12 +567,14 @@ static int abx80x_dt_trickle_cfg(struct device_node *np)
 	if (ret)
 		return ret;
 
-	if (!strcmp(diode, "standard"))
+	if (!strcmp(diode, "standard")) {
 		trickle_cfg |= ABX8XX_TRICKLE_STANDARD_DIODE;
-	else if (!strcmp(diode, "schottky"))
+	} else if (!strcmp(diode, "schottky")) {
 		trickle_cfg |= ABX8XX_TRICKLE_SCHOTTKY_DIODE;
-	else
+	} else {
+		dev_dbg(&client->dev, "Invalid tc-diode value: %s\n", diode);
 		return -EINVAL;
+	}
 
 	ret = of_property_read_u32(np, "abracon,tc-resistor", &tmp);
 	if (ret)
@@ -580,8 +584,10 @@ static int abx80x_dt_trickle_cfg(struct device_node *np)
 		if (trickle_resistors[i] == tmp)
 			break;
 
-	if (i == sizeof(trickle_resistors))
+	if (i == sizeof(trickle_resistors)) {
+		dev_dbg(&client->dev, "Invalid tc-resistor value: %u\n", tmp);
 		return -EINVAL;
+	}
 
 	return (trickle_cfg | i);
 }
@@ -793,7 +799,7 @@ static int abx80x_probe(struct i2c_client *client,
 	}
 
 	if (np && abx80x_caps[part].has_tc)
-		trickle_cfg = abx80x_dt_trickle_cfg(np);
+		trickle_cfg = abx80x_dt_trickle_cfg(client);
 
 	if (trickle_cfg > 0) {
 		dev_info(&client->dev, "Enabling trickle charger: %02x\n",
@@ -863,9 +869,57 @@ static const struct i2c_device_id abx80x_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, abx80x_id);
 
+#ifdef CONFIG_OF
+static const struct of_device_id abx80x_of_match[] = {
+	{
+		.compatible = "abracon,abx80x",
+		.data = (void *)ABX80X
+	},
+	{
+		.compatible = "abracon,ab0801",
+		.data = (void *)AB0801
+	},
+	{
+		.compatible = "abracon,ab0803",
+		.data = (void *)AB0803
+	},
+	{
+		.compatible = "abracon,ab0804",
+		.data = (void *)AB0804
+	},
+	{
+		.compatible = "abracon,ab0805",
+		.data = (void *)AB0805
+	},
+	{
+		.compatible = "abracon,ab1801",
+		.data = (void *)AB1801
+	},
+	{
+		.compatible = "abracon,ab1803",
+		.data = (void *)AB1803
+	},
+	{
+		.compatible = "abracon,ab1804",
+		.data = (void *)AB1804
+	},
+	{
+		.compatible = "abracon,ab1805",
+		.data = (void *)AB1805
+	},
+	{
+		.compatible = "microcrystal,rv1805",
+		.data = (void *)RV1805
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, abx80x_of_match);
+#endif
+
 static struct i2c_driver abx80x_driver = {
 	.driver		= {
 		.name	= "rtc-abx80x",
+		.of_match_table = of_match_ptr(abx80x_of_match),
 	},
 	.probe		= abx80x_probe,
 	.id_table	= abx80x_id,
diff --git a/drivers/rtc/rtc-fsl-ftm-alarm.c b/drivers/rtc/rtc-fsl-ftm-alarm.c
index 756af62b0486..68f0a1801a2e 100644
--- a/drivers/rtc/rtc-fsl-ftm-alarm.c
+++ b/drivers/rtc/rtc-fsl-ftm-alarm.c
@@ -21,6 +21,7 @@
 #include <linux/rtc.h>
 #include <linux/time.h>
 #include <linux/acpi.h>
+#include <linux/pm_wakeirq.h>
 
 #define FTM_SC_CLK(c)		((c) << FTM_SC_CLK_MASK_SHIFT)
 
@@ -268,13 +269,11 @@ static int ftm_rtc_probe(struct platform_device *pdev)
 	}
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "can't get irq number\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	ret = devm_request_irq(&pdev->dev, irq, ftm_rtc_alarm_interrupt,
-			       IRQF_NO_SUSPEND, dev_name(&pdev->dev), rtc);
+			       0, dev_name(&pdev->dev), rtc);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "failed to request irq\n");
 		return ret;
@@ -287,6 +286,9 @@ static int ftm_rtc_probe(struct platform_device *pdev)
 	rtc->rtc_dev->ops = &ftm_rtc_ops;
 
 	device_init_wakeup(&pdev->dev, true);
+	ret = dev_pm_set_wake_irq(&pdev->dev, irq);
+	if (ret)
+		dev_err(&pdev->dev, "failed to enable irq wake\n");
 
 	ret = rtc_register_device(rtc->rtc_dev);
 	if (ret) {
diff --git a/drivers/rtc/rtc-goldfish.c b/drivers/rtc/rtc-goldfish.c
index cb6b0ad7ec3f..27797157fcb3 100644
--- a/drivers/rtc/rtc-goldfish.c
+++ b/drivers/rtc/rtc-goldfish.c
@@ -174,7 +174,7 @@ static int goldfish_rtc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, rtcdrv);
 	rtcdrv->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(rtcdrv->base))
-		return -ENODEV;
+		return PTR_ERR(rtcdrv->base);
 
 	rtcdrv->irq = platform_get_irq(pdev, 0);
 	if (rtcdrv->irq < 0)
diff --git a/drivers/rtc/rtc-jz4740.c b/drivers/rtc/rtc-jz4740.c
index e4c719085c31..9607e6b6e0b3 100644
--- a/drivers/rtc/rtc-jz4740.c
+++ b/drivers/rtc/rtc-jz4740.c
@@ -55,14 +55,8 @@ struct jz4740_rtc {
 	enum jz4740_rtc_type type;
 
 	struct rtc_device *rtc;
-	struct clk *clk;
-
-	int irq;
 
 	spinlock_t lock;
-
-	unsigned int min_wakeup_pin_assert_time;
-	unsigned int reset_pin_assert_time;
 };
 
 static struct device *dev_for_power_off;
@@ -259,156 +253,157 @@ static void jz4740_rtc_poweroff(struct device *dev)
 
 static void jz4740_rtc_power_off(void)
 {
-	struct jz4740_rtc *rtc = dev_get_drvdata(dev_for_power_off);
-	unsigned long rtc_rate;
-	unsigned long wakeup_filter_ticks;
-	unsigned long reset_counter_ticks;
+	jz4740_rtc_poweroff(dev_for_power_off);
+	kernel_halt();
+}
 
-	clk_prepare_enable(rtc->clk);
+static void jz4740_rtc_clk_disable(void *data)
+{
+	clk_disable_unprepare(data);
+}
 
-	rtc_rate = clk_get_rate(rtc->clk);
+static const struct of_device_id jz4740_rtc_of_match[] = {
+	{ .compatible = "ingenic,jz4740-rtc", .data = (void *)ID_JZ4740 },
+	{ .compatible = "ingenic,jz4760-rtc", .data = (void *)ID_JZ4760 },
+	{ .compatible = "ingenic,jz4780-rtc", .data = (void *)ID_JZ4780 },
+	{},
+};
+MODULE_DEVICE_TABLE(of, jz4740_rtc_of_match);
+
+static void jz4740_rtc_set_wakeup_params(struct jz4740_rtc *rtc,
+					 struct device_node *np,
+					 unsigned long rate)
+{
+	unsigned long wakeup_ticks, reset_ticks;
+	unsigned int min_wakeup_pin_assert_time = 60; /* Default: 60ms */
+	unsigned int reset_pin_assert_time = 100; /* Default: 100ms */
+
+	of_property_read_u32(np, "ingenic,reset-pin-assert-time-ms",
+			     &reset_pin_assert_time);
+	of_property_read_u32(np, "ingenic,min-wakeup-pin-assert-time-ms",
+			     &min_wakeup_pin_assert_time);
 
 	/*
 	 * Set minimum wakeup pin assertion time: 100 ms.
 	 * Range is 0 to 2 sec if RTC is clocked at 32 kHz.
 	 */
-	wakeup_filter_ticks =
-		(rtc->min_wakeup_pin_assert_time * rtc_rate) / 1000;
-	if (wakeup_filter_ticks < JZ_RTC_WAKEUP_FILTER_MASK)
-		wakeup_filter_ticks &= JZ_RTC_WAKEUP_FILTER_MASK;
+	wakeup_ticks = (min_wakeup_pin_assert_time * rate) / 1000;
+	if (wakeup_ticks < JZ_RTC_WAKEUP_FILTER_MASK)
+		wakeup_ticks &= JZ_RTC_WAKEUP_FILTER_MASK;
 	else
-		wakeup_filter_ticks = JZ_RTC_WAKEUP_FILTER_MASK;
-	jz4740_rtc_reg_write(rtc,
-			     JZ_REG_RTC_WAKEUP_FILTER, wakeup_filter_ticks);
+		wakeup_ticks = JZ_RTC_WAKEUP_FILTER_MASK;
+	jz4740_rtc_reg_write(rtc, JZ_REG_RTC_WAKEUP_FILTER, wakeup_ticks);
 
 	/*
 	 * Set reset pin low-level assertion time after wakeup: 60 ms.
 	 * Range is 0 to 125 ms if RTC is clocked at 32 kHz.
 	 */
-	reset_counter_ticks = (rtc->reset_pin_assert_time * rtc_rate) / 1000;
-	if (reset_counter_ticks < JZ_RTC_RESET_COUNTER_MASK)
-		reset_counter_ticks &= JZ_RTC_RESET_COUNTER_MASK;
+	reset_ticks = (reset_pin_assert_time * rate) / 1000;
+	if (reset_ticks < JZ_RTC_RESET_COUNTER_MASK)
+		reset_ticks &= JZ_RTC_RESET_COUNTER_MASK;
 	else
-		reset_counter_ticks = JZ_RTC_RESET_COUNTER_MASK;
-	jz4740_rtc_reg_write(rtc,
-			     JZ_REG_RTC_RESET_COUNTER, reset_counter_ticks);
-
-	jz4740_rtc_poweroff(dev_for_power_off);
-	kernel_halt();
+		reset_ticks = JZ_RTC_RESET_COUNTER_MASK;
+	jz4740_rtc_reg_write(rtc, JZ_REG_RTC_RESET_COUNTER, reset_ticks);
 }
 
-static const struct of_device_id jz4740_rtc_of_match[] = {
-	{ .compatible = "ingenic,jz4740-rtc", .data = (void *)ID_JZ4740 },
-	{ .compatible = "ingenic,jz4760-rtc", .data = (void *)ID_JZ4760 },
-	{ .compatible = "ingenic,jz4780-rtc", .data = (void *)ID_JZ4780 },
-	{},
-};
-MODULE_DEVICE_TABLE(of, jz4740_rtc_of_match);
-
 static int jz4740_rtc_probe(struct platform_device *pdev)
 {
-	int ret;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
 	struct jz4740_rtc *rtc;
-	const struct platform_device_id *id = platform_get_device_id(pdev);
-	const struct of_device_id *of_id = of_match_device(
-			jz4740_rtc_of_match, &pdev->dev);
-	struct device_node *np = pdev->dev.of_node;
+	unsigned long rate;
+	struct clk *clk;
+	int ret, irq;
 
-	rtc = devm_kzalloc(&pdev->dev, sizeof(*rtc), GFP_KERNEL);
+	rtc = devm_kzalloc(dev, sizeof(*rtc), GFP_KERNEL);
 	if (!rtc)
 		return -ENOMEM;
 
-	if (of_id)
-		rtc->type = (enum jz4740_rtc_type)of_id->data;
-	else
-		rtc->type = id->driver_data;
+	rtc->type = (enum jz4740_rtc_type)device_get_match_data(dev);
 
-	rtc->irq = platform_get_irq(pdev, 0);
-	if (rtc->irq < 0)
-		return -ENOENT;
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
 
 	rtc->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(rtc->base))
 		return PTR_ERR(rtc->base);
 
-	rtc->clk = devm_clk_get(&pdev->dev, "rtc");
-	if (IS_ERR(rtc->clk)) {
-		dev_err(&pdev->dev, "Failed to get RTC clock\n");
-		return PTR_ERR(rtc->clk);
+	clk = devm_clk_get(dev, "rtc");
+	if (IS_ERR(clk)) {
+		dev_err(dev, "Failed to get RTC clock\n");
+		return PTR_ERR(clk);
+	}
+
+	ret = clk_prepare_enable(clk);
+	if (ret) {
+		dev_err(dev, "Failed to enable clock\n");
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(dev, jz4740_rtc_clk_disable, clk);
+	if (ret) {
+		dev_err(dev, "Failed to register devm action\n");
+		return ret;
 	}
 
 	spin_lock_init(&rtc->lock);
 
 	platform_set_drvdata(pdev, rtc);
 
-	device_init_wakeup(&pdev->dev, 1);
+	device_init_wakeup(dev, 1);
 
-	ret = dev_pm_set_wake_irq(&pdev->dev, rtc->irq);
+	ret = dev_pm_set_wake_irq(dev, irq);
 	if (ret) {
-		dev_err(&pdev->dev, "Failed to set wake irq: %d\n", ret);
+		dev_err(dev, "Failed to set wake irq: %d\n", ret);
 		return ret;
 	}
 
-	rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
+	rtc->rtc = devm_rtc_allocate_device(dev);
 	if (IS_ERR(rtc->rtc)) {
 		ret = PTR_ERR(rtc->rtc);
-		dev_err(&pdev->dev, "Failed to allocate rtc device: %d\n", ret);
+		dev_err(dev, "Failed to allocate rtc device: %d\n", ret);
 		return ret;
 	}
 
 	rtc->rtc->ops = &jz4740_rtc_ops;
 	rtc->rtc->range_max = U32_MAX;
 
+	rate = clk_get_rate(clk);
+	jz4740_rtc_set_wakeup_params(rtc, np, rate);
+
+	/* Each 1 Hz pulse should happen after (rate) ticks */
+	jz4740_rtc_reg_write(rtc, JZ_REG_RTC_REGULATOR, rate - 1);
+
 	ret = rtc_register_device(rtc->rtc);
 	if (ret)
 		return ret;
 
-	ret = devm_request_irq(&pdev->dev, rtc->irq, jz4740_rtc_irq, 0,
-				pdev->name, rtc);
+	ret = devm_request_irq(dev, irq, jz4740_rtc_irq, 0,
+			       pdev->name, rtc);
 	if (ret) {
-		dev_err(&pdev->dev, "Failed to request rtc irq: %d\n", ret);
+		dev_err(dev, "Failed to request rtc irq: %d\n", ret);
 		return ret;
 	}
 
-	if (np && of_device_is_system_power_controller(np)) {
-		if (!pm_power_off) {
-			/* Default: 60ms */
-			rtc->reset_pin_assert_time = 60;
-			of_property_read_u32(np,
-					     "ingenic,reset-pin-assert-time-ms",
-					     &rtc->reset_pin_assert_time);
-
-			/* Default: 100ms */
-			rtc->min_wakeup_pin_assert_time = 100;
-			of_property_read_u32(np,
-					     "ingenic,min-wakeup-pin-assert-time-ms",
-					     &rtc->min_wakeup_pin_assert_time);
-
-			dev_for_power_off = &pdev->dev;
+	if (of_device_is_system_power_controller(np)) {
+		dev_for_power_off = dev;
+
+		if (!pm_power_off)
 			pm_power_off = jz4740_rtc_power_off;
-		} else {
-			dev_warn(&pdev->dev,
-				 "Poweroff handler already present!\n");
-		}
+		else
+			dev_warn(dev, "Poweroff handler already present!\n");
 	}
 
 	return 0;
 }
 
-static const struct platform_device_id jz4740_rtc_ids[] = {
-	{ "jz4740-rtc", ID_JZ4740 },
-	{ "jz4780-rtc", ID_JZ4780 },
-	{}
-};
-MODULE_DEVICE_TABLE(platform, jz4740_rtc_ids);
-
 static struct platform_driver jz4740_rtc_driver = {
 	.probe	 = jz4740_rtc_probe,
 	.driver	 = {
 		.name  = "jz4740-rtc",
-		.of_match_table = of_match_ptr(jz4740_rtc_of_match),
+		.of_match_table = jz4740_rtc_of_match,
 	},
-	.id_table = jz4740_rtc_ids,
 };
 
 module_platform_driver(jz4740_rtc_driver);
diff --git a/drivers/rtc/rtc-lpc24xx.c b/drivers/rtc/rtc-lpc24xx.c
index 00ef16ba9480..eec881a81067 100644
--- a/drivers/rtc/rtc-lpc24xx.c
+++ b/drivers/rtc/rtc-lpc24xx.c
@@ -205,10 +205,8 @@ static int lpc24xx_rtc_probe(struct platform_device *pdev)
 		return PTR_ERR(rtc->rtc_base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_warn(&pdev->dev, "can't get interrupt resource\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	rtc->clk_rtc = devm_clk_get(&pdev->dev, "rtc");
 	if (IS_ERR(rtc->clk_rtc)) {
diff --git a/drivers/rtc/rtc-max77686.c b/drivers/rtc/rtc-max77686.c
index d5a0e27dd0a0..03ebcf1c0f3d 100644
--- a/drivers/rtc/rtc-max77686.c
+++ b/drivers/rtc/rtc-max77686.c
@@ -78,6 +78,8 @@ struct max77686_rtc_driver_data {
 	int			alarm_pending_status_reg;
 	/* RTC IRQ CHIP for regmap */
 	const struct regmap_irq_chip *rtc_irq_chip;
+	/* regmap configuration for the chip */
+	const struct regmap_config *regmap_config;
 };
 
 struct max77686_rtc_info {
@@ -182,6 +184,11 @@ static const struct regmap_irq_chip max77686_rtc_irq_chip = {
 	.num_irqs	= ARRAY_SIZE(max77686_rtc_irqs),
 };
 
+static const struct regmap_config max77686_rtc_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
 static const struct max77686_rtc_driver_data max77686_drv_data = {
 	.delay = 16000,
 	.mask  = 0x7f,
@@ -191,6 +198,13 @@ static const struct max77686_rtc_driver_data max77686_drv_data = {
 	.alarm_pending_status_reg = MAX77686_REG_STATUS2,
 	.rtc_i2c_addr = MAX77686_I2C_ADDR_RTC,
 	.rtc_irq_chip = &max77686_rtc_irq_chip,
+	.regmap_config = &max77686_rtc_regmap_config,
+};
+
+static const struct regmap_config max77620_rtc_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.use_single_write = true,
 };
 
 static const struct max77686_rtc_driver_data max77620_drv_data = {
@@ -202,6 +216,7 @@ static const struct max77686_rtc_driver_data max77620_drv_data = {
 	.alarm_pending_status_reg = MAX77686_INVALID_REG,
 	.rtc_i2c_addr = MAX77620_I2C_ADDR_RTC,
 	.rtc_irq_chip = &max77686_rtc_irq_chip,
+	.regmap_config = &max77620_rtc_regmap_config,
 };
 
 static const unsigned int max77802_map[REG_RTC_END] = {
@@ -658,11 +673,6 @@ static int max77686_rtc_init_reg(struct max77686_rtc_info *info)
 	return ret;
 }
 
-static const struct regmap_config max77686_rtc_regmap_config = {
-	.reg_bits = 8,
-	.val_bits = 8,
-};
-
 static int max77686_init_rtc_regmap(struct max77686_rtc_info *info)
 {
 	struct device *parent = info->dev->parent;
@@ -698,7 +708,7 @@ static int max77686_init_rtc_regmap(struct max77686_rtc_info *info)
 	}
 
 	info->rtc_regmap = devm_regmap_init_i2c(info->rtc,
-						&max77686_rtc_regmap_config);
+						info->drv_data->regmap_config);
 	if (IS_ERR(info->rtc_regmap)) {
 		ret = PTR_ERR(info->rtc_regmap);
 		dev_err(info->dev, "Failed to allocate RTC regmap: %d\n", ret);
diff --git a/drivers/rtc/rtc-mc13xxx.c b/drivers/rtc/rtc-mc13xxx.c
index afce2c0b4bd6..d6802e6191cb 100644
--- a/drivers/rtc/rtc-mc13xxx.c
+++ b/drivers/rtc/rtc-mc13xxx.c
@@ -308,8 +308,10 @@ static int __init mc13xxx_rtc_probe(struct platform_device *pdev)
 	mc13xxx_unlock(mc13xxx);
 
 	ret = rtc_register_device(priv->rtc);
-	if (ret)
+	if (ret) {
+		mc13xxx_lock(mc13xxx);
 		goto err_irq_request;
+	}
 
 	return 0;
 
diff --git a/drivers/rtc/rtc-mpc5121.c b/drivers/rtc/rtc-mpc5121.c
index 3040844129ce..5c2ce71aa044 100644
--- a/drivers/rtc/rtc-mpc5121.c
+++ b/drivers/rtc/rtc-mpc5121.c
@@ -316,7 +316,7 @@ static int mpc5121_rtc_probe(struct platform_device *op)
 	rtc->regs = devm_platform_ioremap_resource(op, 0);
 	if (IS_ERR(rtc->regs)) {
 		dev_err(&op->dev, "%s: couldn't map io space\n", __func__);
-		return -ENOSYS;
+		return PTR_ERR(rtc->regs);
 	}
 
 	device_init_wakeup(&op->dev, 1);
diff --git a/drivers/rtc/rtc-mt2712.c b/drivers/rtc/rtc-mt2712.c
index 581b8731fb8a..d5f691c8a035 100644
--- a/drivers/rtc/rtc-mt2712.c
+++ b/drivers/rtc/rtc-mt2712.c
@@ -310,7 +310,6 @@ static const struct rtc_class_ops mt2712_rtc_ops = {
 
 static int mt2712_rtc_probe(struct platform_device *pdev)
 {
-	struct resource *res;
 	struct mt2712_rtc *mt2712_rtc;
 	int ret;
 
@@ -319,8 +318,7 @@ static int mt2712_rtc_probe(struct platform_device *pdev)
 	if (!mt2712_rtc)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	mt2712_rtc->base = devm_ioremap_resource(&pdev->dev, res);
+	mt2712_rtc->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(mt2712_rtc->base))
 		return PTR_ERR(mt2712_rtc->base);
 
@@ -328,10 +326,8 @@ static int mt2712_rtc_probe(struct platform_device *pdev)
 	mt2712_rtc_hw_init(mt2712_rtc);
 
 	mt2712_rtc->irq = platform_get_irq(pdev, 0);
-	if (mt2712_rtc->irq < 0) {
-		dev_err(&pdev->dev, "No IRQ resource\n");
+	if (mt2712_rtc->irq < 0)
 		return mt2712_rtc->irq;
-	}
 
 	platform_set_drvdata(pdev, mt2712_rtc);
 
@@ -356,13 +352,7 @@ static int mt2712_rtc_probe(struct platform_device *pdev)
 	mt2712_rtc->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
 	mt2712_rtc->rtc->range_max = MT2712_RTC_TIMESTAMP_END_2127;
 
-	ret = rtc_register_device(mt2712_rtc->rtc);
-	if (ret) {
-		dev_err(&pdev->dev, "register rtc device failed\n");
-		return ret;
-	}
-
-	return 0;
+	return rtc_register_device(mt2712_rtc->rtc);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/rtc/rtc-mt6397.c b/drivers/rtc/rtc-mt6397.c
index cda238dfe69b..f8b1353777ba 100644
--- a/drivers/rtc/rtc-mt6397.c
+++ b/drivers/rtc/rtc-mt6397.c
@@ -9,6 +9,7 @@
 #include <linux/mfd/mt6397/core.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 #include <linux/rtc.h>
@@ -20,7 +21,7 @@ static int mtk_rtc_write_trigger(struct mt6397_rtc *rtc)
 	int ret;
 	u32 data;
 
-	ret = regmap_write(rtc->regmap, rtc->addr_base + RTC_WRTGR, 1);
+	ret = regmap_write(rtc->regmap, rtc->addr_base + rtc->data->wrtgr, 1);
 	if (ret < 0)
 		return ret;
 
@@ -269,6 +270,8 @@ static int mtk_rtc_probe(struct platform_device *pdev)
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	rtc->addr_base = res->start;
 
+	rtc->data = of_device_get_match_data(&pdev->dev);
+
 	rtc->irq = platform_get_irq(pdev, 0);
 	if (rtc->irq < 0)
 		return rtc->irq;
@@ -325,9 +328,18 @@ static int mt6397_rtc_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(mt6397_pm_ops, mt6397_rtc_suspend,
 			mt6397_rtc_resume);
 
+static const struct mtk_rtc_data mt6358_rtc_data = {
+	.wrtgr = RTC_WRTGR_MT6358,
+};
+
+static const struct mtk_rtc_data mt6397_rtc_data = {
+	.wrtgr = RTC_WRTGR_MT6397,
+};
+
 static const struct of_device_id mt6397_rtc_of_match[] = {
-	{ .compatible = "mediatek,mt6323-rtc", },
-	{ .compatible = "mediatek,mt6397-rtc", },
+	{ .compatible = "mediatek,mt6323-rtc", .data = &mt6397_rtc_data },
+	{ .compatible = "mediatek,mt6358-rtc", .data = &mt6358_rtc_data },
+	{ .compatible = "mediatek,mt6397-rtc", .data = &mt6397_rtc_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, mt6397_rtc_of_match);
diff --git a/drivers/rtc/rtc-pcf2127.c b/drivers/rtc/rtc-pcf2127.c
index 4e50d6768f13..9c5670776c68 100644
--- a/drivers/rtc/rtc-pcf2127.c
+++ b/drivers/rtc/rtc-pcf2127.c
@@ -137,8 +137,7 @@ static int pcf2127_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	tm->tm_wday = buf[PCF2127_REG_DW] & 0x07;
 	tm->tm_mon = bcd2bin(buf[PCF2127_REG_MO] & 0x1F) - 1; /* rtc mn 1-12 */
 	tm->tm_year = bcd2bin(buf[PCF2127_REG_YR]);
-	if (tm->tm_year < 70)
-		tm->tm_year += 100;	/* assume we are in 1970...2069 */
+	tm->tm_year += 100;
 
 	dev_dbg(dev, "%s: tm is secs=%d, mins=%d, hours=%d, "
 		"mday=%d, mon=%d, year=%d, wday=%d\n",
@@ -172,7 +171,7 @@ static int pcf2127_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	buf[i++] = bin2bcd(tm->tm_mon + 1);
 
 	/* year */
-	buf[i++] = bin2bcd(tm->tm_year % 100);
+	buf[i++] = bin2bcd(tm->tm_year - 100);
 
 	/* write register's data */
 	err = regmap_bulk_write(pcf2127->regmap, PCF2127_REG_SC, buf, i);
@@ -185,30 +184,35 @@ static int pcf2127_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	return 0;
 }
 
-#ifdef CONFIG_RTC_INTF_DEV
 static int pcf2127_rtc_ioctl(struct device *dev,
 				unsigned int cmd, unsigned long arg)
 {
 	struct pcf2127 *pcf2127 = dev_get_drvdata(dev);
-	int touser;
+	int val, touser = 0;
 	int ret;
 
 	switch (cmd) {
 	case RTC_VL_READ:
-		ret = regmap_read(pcf2127->regmap, PCF2127_REG_CTRL3, &touser);
+		ret = regmap_read(pcf2127->regmap, PCF2127_REG_CTRL3, &val);
 		if (ret)
 			return ret;
 
-		touser = touser & PCF2127_BIT_CTRL3_BLF ? RTC_VL_BACKUP_LOW : 0;
+		if (val & PCF2127_BIT_CTRL3_BLF)
+			touser |= RTC_VL_BACKUP_LOW;
+
+		if (val & PCF2127_BIT_CTRL3_BF)
+			touser |= RTC_VL_BACKUP_SWITCH;
 
 		return put_user(touser, (unsigned int __user *)arg);
+
+	case RTC_VL_CLR:
+		return regmap_update_bits(pcf2127->regmap, PCF2127_REG_CTRL3,
+					  PCF2127_BIT_CTRL3_BF, 0);
+
 	default:
 		return -ENOIOCTLCMD;
 	}
 }
-#else
-#define pcf2127_rtc_ioctl NULL
-#endif
 
 static const struct rtc_class_ops pcf2127_rtc_ops = {
 	.ioctl		= pcf2127_rtc_ioctl,
@@ -433,6 +437,9 @@ static int pcf2127_probe(struct device *dev, struct regmap *regmap,
 		return PTR_ERR(pcf2127->rtc);
 
 	pcf2127->rtc->ops = &pcf2127_rtc_ops;
+	pcf2127->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	pcf2127->rtc->range_max = RTC_TIMESTAMP_END_2099;
+	pcf2127->rtc->set_start_time = true; /* Sets actual start to 1970 */
 
 	pcf2127->wdd.parent = dev;
 	pcf2127->wdd.info = &pcf2127_wdt_info;
@@ -441,6 +448,7 @@ static int pcf2127_probe(struct device *dev, struct regmap *regmap,
 	pcf2127->wdd.max_timeout = PCF2127_WD_VAL_MAX;
 	pcf2127->wdd.timeout = PCF2127_WD_VAL_DEFAULT;
 	pcf2127->wdd.min_hw_heartbeat_ms = 500;
+	pcf2127->wdd.status = WATCHDOG_NOWAYOUT_INIT_STATUS;
 
 	watchdog_set_drvdata(&pcf2127->wdd, pcf2127);
 
@@ -495,7 +503,6 @@ static int pcf2127_probe(struct device *dev, struct regmap *regmap,
 	 */
 	ret = regmap_update_bits(pcf2127->regmap, PCF2127_REG_CTRL3,
 				 PCF2127_BIT_CTRL3_BTSE |
-				 PCF2127_BIT_CTRL3_BF |
 				 PCF2127_BIT_CTRL3_BIE |
 				 PCF2127_BIT_CTRL3_BLIE, 0);
 	if (ret) {
@@ -636,6 +643,7 @@ static int pcf2127_i2c_probe(struct i2c_client *client,
 	static const struct regmap_config config = {
 		.reg_bits = 8,
 		.val_bits = 8,
+		.max_register = 0x1d,
 	};
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
@@ -703,6 +711,7 @@ static int pcf2127_spi_probe(struct spi_device *spi)
 		.val_bits = 8,
 		.read_flag_mask = 0xa0,
 		.write_flag_mask = 0x20,
+		.max_register = 0x1d,
 	};
 	struct regmap *regmap;
 
diff --git a/drivers/rtc/rtc-rc5t619.c b/drivers/rtc/rtc-rc5t619.c
index 24e386ecbc7e..dd1a20977478 100644
--- a/drivers/rtc/rtc-rc5t619.c
+++ b/drivers/rtc/rtc-rc5t619.c
@@ -356,10 +356,8 @@ static int rc5t619_rtc_probe(struct platform_device *pdev)
 	int err;
 
 	rtc = devm_kzalloc(dev, sizeof(*rtc), GFP_KERNEL);
-	if (IS_ERR(rtc)) {
-		err = PTR_ERR(rtc);
+	if (!rtc)
 		return -ENOMEM;
-	}
 
 	rtc->rn5t618 = rn5t618;
 
diff --git a/drivers/rtc/rtc-rv3028.c b/drivers/rtc/rtc-rv3028.c
index a0ddc86c975a..ec84db0b3d7a 100644
--- a/drivers/rtc/rtc-rv3028.c
+++ b/drivers/rtc/rtc-rv3028.c
@@ -755,6 +755,8 @@ static int rv3028_probe(struct i2c_client *client)
 		return -ENOMEM;
 
 	rv3028->regmap = devm_regmap_init_i2c(client, &regmap_config);
+	if (IS_ERR(rv3028->regmap))
+		return PTR_ERR(rv3028->regmap);
 
 	i2c_set_clientdata(client, rv3028);
 
diff --git a/drivers/rtc/rtc-snvs.c b/drivers/rtc/rtc-snvs.c
index 35ee08aa7584..0263d996b8a8 100644
--- a/drivers/rtc/rtc-snvs.c
+++ b/drivers/rtc/rtc-snvs.c
@@ -148,10 +148,21 @@ static int snvs_rtc_enable(struct snvs_rtc_data *data, bool enable)
 static int snvs_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
 	struct snvs_rtc_data *data = dev_get_drvdata(dev);
-	unsigned long time = rtc_read_lp_counter(data);
+	unsigned long time;
+	int ret;
+
+	if (data->clk) {
+		ret = clk_enable(data->clk);
+		if (ret)
+			return ret;
+	}
 
+	time = rtc_read_lp_counter(data);
 	rtc_time64_to_tm(time, tm);
 
+	if (data->clk)
+		clk_disable(data->clk);
+
 	return 0;
 }
 
@@ -161,6 +172,12 @@ static int snvs_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	unsigned long time = rtc_tm_to_time64(tm);
 	int ret;
 
+	if (data->clk) {
+		ret = clk_enable(data->clk);
+		if (ret)
+			return ret;
+	}
+
 	/* Disable RTC first */
 	ret = snvs_rtc_enable(data, false);
 	if (ret)
@@ -173,6 +190,9 @@ static int snvs_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	/* Enable RTC again */
 	ret = snvs_rtc_enable(data, true);
 
+	if (data->clk)
+		clk_disable(data->clk);
+
 	return ret;
 }
 
@@ -180,6 +200,13 @@ static int snvs_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
 	struct snvs_rtc_data *data = dev_get_drvdata(dev);
 	u32 lptar, lpsr;
+	int ret;
+
+	if (data->clk) {
+		ret = clk_enable(data->clk);
+		if (ret)
+			return ret;
+	}
 
 	regmap_read(data->regmap, data->offset + SNVS_LPTAR, &lptar);
 	rtc_time64_to_tm(lptar, &alrm->time);
@@ -187,18 +214,33 @@ static int snvs_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	regmap_read(data->regmap, data->offset + SNVS_LPSR, &lpsr);
 	alrm->pending = (lpsr & SNVS_LPSR_LPTA) ? 1 : 0;
 
+	if (data->clk)
+		clk_disable(data->clk);
+
 	return 0;
 }
 
 static int snvs_rtc_alarm_irq_enable(struct device *dev, unsigned int enable)
 {
 	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+	int ret;
+
+	if (data->clk) {
+		ret = clk_enable(data->clk);
+		if (ret)
+			return ret;
+	}
 
 	regmap_update_bits(data->regmap, data->offset + SNVS_LPCR,
 			   (SNVS_LPCR_LPTA_EN | SNVS_LPCR_LPWUI_EN),
 			   enable ? (SNVS_LPCR_LPTA_EN | SNVS_LPCR_LPWUI_EN) : 0);
 
-	return rtc_write_sync_lp(data);
+	ret = rtc_write_sync_lp(data);
+
+	if (data->clk)
+		clk_disable(data->clk);
+
+	return ret;
 }
 
 static int snvs_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
@@ -207,6 +249,12 @@ static int snvs_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	unsigned long time = rtc_tm_to_time64(&alrm->time);
 	int ret;
 
+	if (data->clk) {
+		ret = clk_enable(data->clk);
+		if (ret)
+			return ret;
+	}
+
 	regmap_update_bits(data->regmap, data->offset + SNVS_LPCR, SNVS_LPCR_LPTA_EN, 0);
 	ret = rtc_write_sync_lp(data);
 	if (ret)
@@ -216,6 +264,9 @@ static int snvs_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	/* Clear alarm interrupt status bit */
 	regmap_write(data->regmap, data->offset + SNVS_LPSR, SNVS_LPSR_LPTA);
 
+	if (data->clk)
+		clk_disable(data->clk);
+
 	return snvs_rtc_alarm_irq_enable(dev, alrm->enabled);
 }
 
@@ -362,7 +413,7 @@ static int __maybe_unused snvs_rtc_suspend_noirq(struct device *dev)
 	struct snvs_rtc_data *data = dev_get_drvdata(dev);
 
 	if (data->clk)
-		clk_disable_unprepare(data->clk);
+		clk_disable(data->clk);
 
 	return 0;
 }
@@ -372,7 +423,7 @@ static int __maybe_unused snvs_rtc_resume_noirq(struct device *dev)
 	struct snvs_rtc_data *data = dev_get_drvdata(dev);
 
 	if (data->clk)
-		return clk_prepare_enable(data->clk);
+		return clk_enable(data->clk);
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-stmp3xxx.c b/drivers/rtc/rtc-stmp3xxx.c
index ff6488be385f..c9bc3d4a1e66 100644
--- a/drivers/rtc/rtc-stmp3xxx.c
+++ b/drivers/rtc/rtc-stmp3xxx.c
@@ -416,5 +416,5 @@ module_platform_driver(stmp3xxx_rtcdrv);
 
 MODULE_DESCRIPTION("STMP3xxx RTC Driver");
 MODULE_AUTHOR("dmitry pervushin <dpervushin@embeddedalley.com> and "
-		"Wolfram Sang <w.sang@pengutronix.de>");
+		"Wolfram Sang <kernel@pengutronix.de>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/s390/cio/Makefile b/drivers/s390/cio/Makefile
index 23eae4188876..a9235f111e79 100644
--- a/drivers/s390/cio/Makefile
+++ b/drivers/s390/cio/Makefile
@@ -21,5 +21,5 @@ qdio-objs := qdio_main.o qdio_thinint.o qdio_debug.o qdio_setup.o
 obj-$(CONFIG_QDIO) += qdio.o
 
 vfio_ccw-objs += vfio_ccw_drv.o vfio_ccw_cp.o vfio_ccw_ops.o vfio_ccw_fsm.o \
-	vfio_ccw_async.o vfio_ccw_trace.o
+	vfio_ccw_async.o vfio_ccw_trace.o vfio_ccw_chp.o
 obj-$(CONFIG_VFIO_CCW) += vfio_ccw.o
diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c
index 1ca73c2e5a8f..c314e9495c1b 100644
--- a/drivers/s390/cio/chsc.c
+++ b/drivers/s390/cio/chsc.c
@@ -57,6 +57,7 @@ int chsc_error_from_response(int response)
 	case 0x0104:
 		return -EINVAL;
 	case 0x0004:
+	case 0x0106:		/* "Wrong Channel Parm" for the op 0x003d */
 		return -EOPNOTSUPP;
 	case 0x000b:
 	case 0x0107:		/* "Channel busy" for the op 0x003d */
@@ -1336,36 +1337,35 @@ out:
 EXPORT_SYMBOL_GPL(chsc_scm_info);
 
 /**
- * chsc_pnso_brinfo() - Perform Network-Subchannel Operation, Bridge Info.
+ * chsc_pnso() - Perform Network-Subchannel Operation
  * @schid:		id of the subchannel on which PNSO is performed
- * @brinfo_area:	request and response block for the operation
+ * @pnso_area:		request and response block for the operation
  * @resume_token:	resume token for multiblock response
  * @cnc:		Boolean change-notification control
  *
- * brinfo_area must be allocated by the caller with get_zeroed_page(GFP_KERNEL)
+ * pnso_area must be allocated by the caller with get_zeroed_page(GFP_KERNEL)
  *
  * Returns 0 on success.
  */
-int chsc_pnso_brinfo(struct subchannel_id schid,
-		struct chsc_pnso_area *brinfo_area,
-		struct chsc_brinfo_resume_token resume_token,
-		int cnc)
+int chsc_pnso(struct subchannel_id schid,
+	      struct chsc_pnso_area *pnso_area,
+	      struct chsc_pnso_resume_token resume_token,
+	      int cnc)
 {
-	memset(brinfo_area, 0, sizeof(*brinfo_area));
-	brinfo_area->request.length = 0x0030;
-	brinfo_area->request.code = 0x003d; /* network-subchannel operation */
-	brinfo_area->m	   = schid.m;
-	brinfo_area->ssid  = schid.ssid;
-	brinfo_area->sch   = schid.sch_no;
-	brinfo_area->cssid = schid.cssid;
-	brinfo_area->oc    = 0; /* Store-network-bridging-information list */
-	brinfo_area->resume_token = resume_token;
-	brinfo_area->n	   = (cnc != 0);
-	if (chsc(brinfo_area))
+	memset(pnso_area, 0, sizeof(*pnso_area));
+	pnso_area->request.length = 0x0030;
+	pnso_area->request.code = 0x003d; /* network-subchannel operation */
+	pnso_area->m	   = schid.m;
+	pnso_area->ssid  = schid.ssid;
+	pnso_area->sch	 = schid.sch_no;
+	pnso_area->cssid = schid.cssid;
+	pnso_area->oc	 = 0; /* Store-network-bridging-information list */
+	pnso_area->resume_token = resume_token;
+	pnso_area->n	   = (cnc != 0);
+	if (chsc(pnso_area))
 		return -EIO;
-	return chsc_error_from_response(brinfo_area->response.code);
+	return chsc_error_from_response(pnso_area->response.code);
 }
-EXPORT_SYMBOL_GPL(chsc_pnso_brinfo);
 
 int chsc_sgib(u32 origin)
 {
diff --git a/drivers/s390/cio/chsc.h b/drivers/s390/cio/chsc.h
index 34de6d77442c..7ecf7e4c402e 100644
--- a/drivers/s390/cio/chsc.h
+++ b/drivers/s390/cio/chsc.h
@@ -205,52 +205,10 @@ struct chsc_scm_info {
 
 int chsc_scm_info(struct chsc_scm_info *scm_area, u64 token);
 
-struct chsc_brinfo_resume_token {
-	u64 t1;
-	u64 t2;
-} __packed;
-
-struct chsc_brinfo_naihdr {
-	struct chsc_brinfo_resume_token resume_token;
-	u32:32;
-	u32 instance;
-	u32:24;
-	u8 naids;
-	u32 reserved[3];
-} __packed;
-
-struct chsc_pnso_area {
-	struct chsc_header request;
-	u8:2;
-	u8 m:1;
-	u8:5;
-	u8:2;
-	u8 ssid:2;
-	u8 fmt:4;
-	u16 sch;
-	u8:8;
-	u8 cssid;
-	u16:16;
-	u8 oc;
-	u32:24;
-	struct chsc_brinfo_resume_token resume_token;
-	u32 n:1;
-	u32:31;
-	u32 reserved[3];
-	struct chsc_header response;
-	u32:32;
-	struct chsc_brinfo_naihdr naihdr;
-	union {
-		struct qdio_brinfo_entry_l3_ipv6 l3_ipv6[0];
-		struct qdio_brinfo_entry_l3_ipv4 l3_ipv4[0];
-		struct qdio_brinfo_entry_l2	 l2[0];
-	} entries;
-} __packed __aligned(PAGE_SIZE);
-
-int chsc_pnso_brinfo(struct subchannel_id schid,
-		struct chsc_pnso_area *brinfo_area,
-		struct chsc_brinfo_resume_token resume_token,
-		int cnc);
+int chsc_pnso(struct subchannel_id schid,
+	      struct chsc_pnso_area *pnso_area,
+	      struct chsc_pnso_resume_token resume_token,
+	      int cnc);
 
 int __init chsc_get_cssid(int idx);
 
diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c
index ccecf6b9504e..963fcc9054c6 100644
--- a/drivers/s390/cio/device_ops.c
+++ b/drivers/s390/cio/device_ops.c
@@ -710,6 +710,29 @@ void ccw_device_get_schid(struct ccw_device *cdev, struct subchannel_id *schid)
 }
 EXPORT_SYMBOL_GPL(ccw_device_get_schid);
 
+/**
+ * ccw_device_pnso() - Perform Network-Subchannel Operation
+ * @cdev:		device on which PNSO is performed
+ * @pnso_area:		request and response block for the operation
+ * @resume_token:	resume token for multiblock response
+ * @cnc:		Boolean change-notification control
+ *
+ * pnso_area must be allocated by the caller with get_zeroed_page(GFP_KERNEL)
+ *
+ * Returns 0 on success.
+ */
+int ccw_device_pnso(struct ccw_device *cdev,
+		    struct chsc_pnso_area *pnso_area,
+		    struct chsc_pnso_resume_token resume_token,
+		    int cnc)
+{
+	struct subchannel_id schid;
+
+	ccw_device_get_schid(cdev, &schid);
+	return chsc_pnso(schid, pnso_area, resume_token, cnc);
+}
+EXPORT_SYMBOL_GPL(ccw_device_pnso);
+
 /*
  * Allocate zeroed dma coherent 31 bit addressable memory using
  * the subchannels dma pool. Maximal size of allocation supported
diff --git a/drivers/s390/cio/idset.c b/drivers/s390/cio/idset.c
index 77d0ea7b381b..45f9c0736be4 100644
--- a/drivers/s390/cio/idset.c
+++ b/drivers/s390/cio/idset.c
@@ -59,18 +59,6 @@ static inline int idset_contains(struct idset *set, int ssid, int id)
 	return test_bit(ssid * set->num_id + id, set->bitmap);
 }
 
-static inline int idset_get_first(struct idset *set, int *ssid, int *id)
-{
-	int bitnum;
-
-	bitnum = find_first_bit(set->bitmap, set->num_ssid * set->num_id);
-	if (bitnum >= set->num_ssid * set->num_id)
-		return 0;
-	*ssid = bitnum / set->num_id;
-	*id = bitnum % set->num_id;
-	return 1;
-}
-
 struct idset *idset_sch_new(void)
 {
 	return idset_new(max_ssid + 1, __MAX_SUBCHANNEL + 1);
diff --git a/drivers/s390/cio/qdio.h b/drivers/s390/cio/qdio.h
index b8453b594679..eb13c479e11d 100644
--- a/drivers/s390/cio/qdio.h
+++ b/drivers/s390/cio/qdio.h
@@ -221,9 +221,6 @@ struct qdio_q {
 	 */
 	int first_to_check;
 
-	/* beginning position for calling the program */
-	int first_to_kick;
-
 	/* number of buffers in use by the adapter */
 	atomic_t nr_buf_used;
 
@@ -292,6 +289,8 @@ struct qdio_irq {
 
 	struct qdio_q *input_qs[QDIO_MAX_QUEUES_PER_IRQ];
 	struct qdio_q *output_qs[QDIO_MAX_QUEUES_PER_IRQ];
+	unsigned int max_input_qs;
+	unsigned int max_output_qs;
 
 	void (*irq_poll)(struct ccw_device *cdev, unsigned long data);
 	unsigned long poll_state;
@@ -364,16 +363,13 @@ static inline int multicast_outbound(struct qdio_q *q)
 extern u64 last_ai_time;
 
 /* prototypes for thin interrupt */
-void qdio_setup_thinint(struct qdio_irq *irq_ptr);
 int qdio_establish_thinint(struct qdio_irq *irq_ptr);
 void qdio_shutdown_thinint(struct qdio_irq *irq_ptr);
 void tiqdio_add_device(struct qdio_irq *irq_ptr);
 void tiqdio_remove_device(struct qdio_irq *irq_ptr);
 void tiqdio_inbound_processing(unsigned long q);
-int tiqdio_allocate_memory(void);
-void tiqdio_free_memory(void);
-int tiqdio_register_thinints(void);
-void tiqdio_unregister_thinints(void);
+int qdio_thinint_init(void);
+void qdio_thinint_exit(void);
 int test_nonshared_ind(struct qdio_irq *);
 
 /* prototypes for setup */
@@ -389,8 +385,10 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr,
 			struct subchannel_id *schid,
 			struct qdio_ssqd_desc *data);
 int qdio_setup_irq(struct qdio_irq *irq_ptr, struct qdio_initialize *init_data);
+void qdio_shutdown_irq(struct qdio_irq *irq);
 void qdio_print_subchannel_info(struct qdio_irq *irq_ptr);
-void qdio_release_memory(struct qdio_irq *irq_ptr);
+void qdio_free_queues(struct qdio_irq *irq_ptr);
+void qdio_free_async_data(struct qdio_irq *irq_ptr);
 int qdio_setup_init(void);
 void qdio_setup_exit(void);
 int qdio_enable_async_operation(struct qdio_output_q *q);
diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c
index bcc3ab14e72d..610c05f59589 100644
--- a/drivers/s390/cio/qdio_main.c
+++ b/drivers/s390/cio/qdio_main.c
@@ -143,7 +143,7 @@ again:
 		DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
 		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,
-			   q->first_to_kick, count, q->irq_ptr->int_parm);
+			   q->first_to_check, count, q->irq_ptr->int_parm);
 		return 0;
 	}
 }
@@ -191,7 +191,7 @@ again:
 		DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
 		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,
-			   q->first_to_kick, count, q->irq_ptr->int_parm);
+			   q->first_to_check, count, q->irq_ptr->int_parm);
 		return 0;
 	}
 }
@@ -438,15 +438,12 @@ static void process_buffer_error(struct qdio_q *q, unsigned int start,
 		  q->sbal[start]->element[15].sflags);
 }
 
-static inline void inbound_primed(struct qdio_q *q, unsigned int start,
-				  int count)
+static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,
+				       int count, bool auto_ack)
 {
 	int new;
 
-	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, count);
-
-	/* for QEBSM the ACK was already set by EQBS */
-	if (is_qebsm(q)) {
+	if (auto_ack) {
 		if (!q->u.in.ack_count) {
 			q->u.in.ack_count = count;
 			q->u.in.ack_start = start;
@@ -466,15 +463,14 @@ static inline void inbound_primed(struct qdio_q *q, unsigned int start,
 	 * or by the next inbound run.
 	 */
 	new = add_buf(start, count - 1);
-	if (q->u.in.ack_count) {
-		/* reset the previous ACK but first set the new one */
-		set_buf_state(q, new, SLSB_P_INPUT_ACK);
-		set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
-	} else {
-		q->u.in.ack_count = 1;
-		set_buf_state(q, new, SLSB_P_INPUT_ACK);
-	}
+	set_buf_state(q, new, SLSB_P_INPUT_ACK);
+
+	/* delete the previous ACKs */
+	if (q->u.in.ack_count)
+		set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
+			       q->u.in.ack_count);
 
+	q->u.in.ack_count = 1;
 	q->u.in.ack_start = new;
 	count--;
 	if (!count)
@@ -508,19 +504,21 @@ static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start)
 
 	switch (state) {
 	case SLSB_P_INPUT_PRIMED:
-		inbound_primed(q, start, count);
+		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,
+			      count);
+
+		inbound_handle_work(q, start, count, is_qebsm(q));
 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
 			qperf_inc(q, inbound_queue_full);
 		if (q->irq_ptr->perf_stat_enabled)
 			account_sbals(q, count);
 		return count;
 	case SLSB_P_INPUT_ERROR:
+		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,
+			      count);
+
 		process_buffer_error(q, start, count);
-		/*
-		 * Interrupts may be avoided as long as the error is present
-		 * so change the buffer state immediately to avoid starvation.
-		 */
-		set_buf_states(q, start, SLSB_P_INPUT_NOT_INIT, count);
+		inbound_handle_work(q, start, count, false);
 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
 			qperf_inc(q, inbound_queue_full);
 		if (q->irq_ptr->perf_stat_enabled)
@@ -624,10 +622,9 @@ static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q,
 	return phys_aob;
 }
 
-static void qdio_kick_handler(struct qdio_q *q, unsigned int count)
+static void qdio_kick_handler(struct qdio_q *q, unsigned int start,
+			      unsigned int count)
 {
-	int start = q->first_to_kick;
-
 	if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
 		return;
 
@@ -644,7 +641,6 @@ static void qdio_kick_handler(struct qdio_q *q, unsigned int count)
 		   q->irq_ptr->int_parm);
 
 	/* for the next time */
-	q->first_to_kick = add_buf(start, count);
 	q->qdio_error = 0;
 }
 
@@ -668,9 +664,9 @@ static void __qdio_inbound_processing(struct qdio_q *q)
 	if (count == 0)
 		return;
 
+	qdio_kick_handler(q, start, count);
 	start = add_buf(start, count);
 	q->first_to_check = start;
-	qdio_kick_handler(q, count);
 
 	if (!qdio_inbound_q_done(q, start)) {
 		/* means poll time is not yet over */
@@ -826,7 +822,7 @@ static void __qdio_outbound_processing(struct qdio_q *q)
 	count = qdio_outbound_q_moved(q, start);
 	if (count) {
 		q->first_to_check = add_buf(start, count);
-		qdio_kick_handler(q, count);
+		qdio_kick_handler(q, start, count);
 	}
 
 	if (queue_type(q) == QDIO_ZFCP_QFMT && !pci_out_supported(q->irq_ptr) &&
@@ -880,47 +876,17 @@ static inline void qdio_check_outbound_pci_queues(struct qdio_irq *irq)
 			qdio_tasklet_schedule(out);
 }
 
-static void __tiqdio_inbound_processing(struct qdio_q *q)
+void tiqdio_inbound_processing(unsigned long data)
 {
-	unsigned int start = q->first_to_check;
-	int count;
+	struct qdio_q *q = (struct qdio_q *)data;
 
-	qperf_inc(q, tasklet_inbound);
 	if (need_siga_sync(q) && need_siga_sync_after_ai(q))
 		qdio_sync_queues(q);
 
 	/* The interrupt could be caused by a PCI request: */
 	qdio_check_outbound_pci_queues(q->irq_ptr);
 
-	count = qdio_inbound_q_moved(q, start);
-	if (count == 0)
-		return;
-
-	start = add_buf(start, count);
-	q->first_to_check = start;
-	qdio_kick_handler(q, count);
-
-	if (!qdio_inbound_q_done(q, start)) {
-		qperf_inc(q, tasklet_inbound_resched);
-		if (!qdio_tasklet_schedule(q))
-			return;
-	}
-
-	qdio_stop_polling(q);
-	/*
-	 * We need to check again to not lose initiative after
-	 * resetting the ACK state.
-	 */
-	if (!qdio_inbound_q_done(q, start)) {
-		qperf_inc(q, tasklet_inbound_resched2);
-		qdio_tasklet_schedule(q);
-	}
-}
-
-void tiqdio_inbound_processing(unsigned long data)
-{
-	struct qdio_q *q = (struct qdio_q *)data;
-	__tiqdio_inbound_processing(q);
+	__qdio_inbound_processing(q);
 }
 
 static inline void qdio_set_state(struct qdio_irq *irq_ptr,
@@ -977,7 +943,6 @@ static void qdio_handle_activate_check(struct ccw_device *cdev,
 {
 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
 	struct qdio_q *q;
-	int count;
 
 	DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
 	DBF_ERROR("intp :%lx", intparm);
@@ -992,9 +957,8 @@ static void qdio_handle_activate_check(struct ccw_device *cdev,
 		goto no_handler;
 	}
 
-	count = sub_buf(q->first_to_check, q->first_to_kick);
 	q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE,
-		   q->nr, q->first_to_kick, count, irq_ptr->int_parm);
+		   q->nr, q->first_to_check, 0, irq_ptr->int_parm);
 no_handler:
 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
 	/*
@@ -1154,35 +1118,27 @@ int qdio_shutdown(struct ccw_device *cdev, int how)
 
 	/* cleanup subchannel */
 	spin_lock_irq(get_ccwdev_lock(cdev));
-
+	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
 	if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
 		rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
 	else
 		/* default behaviour is halt */
 		rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
+	spin_unlock_irq(get_ccwdev_lock(cdev));
 	if (rc) {
 		DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no);
 		DBF_ERROR("rc:%4d", rc);
 		goto no_cleanup;
 	}
 
-	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
-	spin_unlock_irq(get_ccwdev_lock(cdev));
 	wait_event_interruptible_timeout(cdev->private->wait_q,
 		irq_ptr->state == QDIO_IRQ_STATE_INACTIVE ||
 		irq_ptr->state == QDIO_IRQ_STATE_ERR,
 		10 * HZ);
-	spin_lock_irq(get_ccwdev_lock(cdev));
 
 no_cleanup:
 	qdio_shutdown_thinint(irq_ptr);
-
-	/* restore interrupt handler */
-	if ((void *)cdev->handler == (void *)qdio_int_handler) {
-		cdev->handler = irq_ptr->orig_handler;
-		cdev->private->intparm = 0;
-	}
-	spin_unlock_irq(get_ccwdev_lock(cdev));
+	qdio_shutdown_irq(irq_ptr);
 
 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
 	mutex_unlock(&irq_ptr->setup_mutex);
@@ -1213,7 +1169,11 @@ int qdio_free(struct ccw_device *cdev)
 	cdev->private->qdio_data = NULL;
 	mutex_unlock(&irq_ptr->setup_mutex);
 
-	qdio_release_memory(irq_ptr);
+	qdio_free_async_data(irq_ptr);
+	qdio_free_queues(irq_ptr);
+	free_page((unsigned long) irq_ptr->qdr);
+	free_page(irq_ptr->chsc_page);
+	free_page((unsigned long) irq_ptr);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(qdio_free);
@@ -1229,6 +1189,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 {
 	struct subchannel_id schid;
 	struct qdio_irq *irq_ptr;
+	int rc = -ENOMEM;
 
 	ccw_device_get_schid(cdev, &schid);
 	DBF_EVENT("qallocate:%4x", schid.sch_no);
@@ -1240,12 +1201,12 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 	/* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
 	irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!irq_ptr)
-		goto out_err;
+		return -ENOMEM;
 
 	irq_ptr->cdev = cdev;
 	mutex_init(&irq_ptr->setup_mutex);
 	if (qdio_allocate_dbf(irq_ptr))
-		goto out_rel;
+		goto err_dbf;
 
 	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,
 		      no_output_qs);
@@ -1258,24 +1219,30 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 	 */
 	irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
 	if (!irq_ptr->chsc_page)
-		goto out_rel;
+		goto err_chsc;
 
 	/* qdr is used in ccw1.cda which is u32 */
 	irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!irq_ptr->qdr)
-		goto out_rel;
+		goto err_qdr;
 
-	if (qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs))
-		goto out_rel;
+	rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs);
+	if (rc)
+		goto err_queues;
 
 	INIT_LIST_HEAD(&irq_ptr->entry);
 	cdev->private->qdio_data = irq_ptr;
 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
 	return 0;
-out_rel:
-	qdio_release_memory(irq_ptr);
-out_err:
-	return -ENOMEM;
+
+err_queues:
+	free_page((unsigned long) irq_ptr->qdr);
+err_qdr:
+	free_page(irq_ptr->chsc_page);
+err_chsc:
+err_dbf:
+	free_page((unsigned long) irq_ptr);
+	return rc;
 }
 EXPORT_SYMBOL_GPL(qdio_allocate);
 
@@ -1338,6 +1305,10 @@ int qdio_establish(struct ccw_device *cdev,
 	if (!irq_ptr)
 		return -ENODEV;
 
+	if (init_data->no_input_qs > irq_ptr->max_input_qs ||
+	    init_data->no_output_qs > irq_ptr->max_output_qs)
+		return -EINVAL;
+
 	if ((init_data->no_input_qs && !init_data->input_handler) ||
 	    (init_data->no_output_qs && !init_data->output_handler))
 		return -EINVAL;
@@ -1352,8 +1323,8 @@ int qdio_establish(struct ccw_device *cdev,
 
 	rc = qdio_establish_thinint(irq_ptr);
 	if (rc) {
+		qdio_shutdown_irq(irq_ptr);
 		mutex_unlock(&irq_ptr->setup_mutex);
-		qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
 		return rc;
 	}
 
@@ -1371,8 +1342,9 @@ int qdio_establish(struct ccw_device *cdev,
 	if (rc) {
 		DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
 		DBF_ERROR("rc:%4x", rc);
+		qdio_shutdown_thinint(irq_ptr);
+		qdio_shutdown_irq(irq_ptr);
 		mutex_unlock(&irq_ptr->setup_mutex);
-		qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
 		return rc;
 	}
 
@@ -1460,25 +1432,6 @@ out:
 }
 EXPORT_SYMBOL_GPL(qdio_activate);
 
-static inline int buf_in_between(int bufnr, int start, int count)
-{
-	int end = add_buf(start, count);
-
-	if (end > start) {
-		if (bufnr >= start && bufnr < end)
-			return 1;
-		else
-			return 0;
-	}
-
-	/* wrap-around case */
-	if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) ||
-	    (bufnr < end))
-		return 1;
-	else
-		return 0;
-}
-
 /**
  * handle_inbound - reset processed input buffers
  * @q: queue containing the buffers
@@ -1489,36 +1442,18 @@ static inline int buf_in_between(int bufnr, int start, int count)
 static int handle_inbound(struct qdio_q *q, unsigned int callflags,
 			  int bufnr, int count)
 {
-	int diff;
+	int overlap;
 
 	qperf_inc(q, inbound_call);
 
-	if (!q->u.in.ack_count)
-		goto set;
-
-	/* protect against stop polling setting an ACK for an emptied slsb */
-	if (count == QDIO_MAX_BUFFERS_PER_Q) {
-		/* overwriting everything, just delete polling status */
-		q->u.in.ack_count = 0;
-		goto set;
-	} else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {
-		if (is_qebsm(q)) {
-			/* partial overwrite, just update ack_start */
-			diff = add_buf(bufnr, count);
-			diff = sub_buf(diff, q->u.in.ack_start);
-			q->u.in.ack_count -= diff;
-			if (q->u.in.ack_count <= 0) {
-				q->u.in.ack_count = 0;
-				goto set;
-			}
-			q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);
-		} else {
-			/* the only ACK will be deleted */
-			q->u.in.ack_count = 0;
-		}
+	/* If any ACKed SBALs are returned to HW, adjust ACK tracking: */
+	overlap = min(count - sub_buf(q->u.in.ack_start, bufnr),
+		      q->u.in.ack_count);
+	if (overlap > 0) {
+		q->u.in.ack_start = add_buf(q->u.in.ack_start, overlap);
+		q->u.in.ack_count -= overlap;
 	}
 
-set:
 	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
 	atomic_add(count, &q->nr_buf_used);
 
@@ -1627,7 +1562,7 @@ int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
 EXPORT_SYMBOL_GPL(do_QDIO);
 
 /**
- * qdio_start_irq - process input buffers
+ * qdio_start_irq - enable interrupt processing for the device
  * @cdev: associated ccw_device for the qdio subchannel
  *
  * Return codes
@@ -1770,94 +1705,6 @@ int qdio_stop_irq(struct ccw_device *cdev)
 }
 EXPORT_SYMBOL(qdio_stop_irq);
 
-/**
- * qdio_pnso_brinfo() - perform network subchannel op #0 - bridge info.
- * @schid:		Subchannel ID.
- * @cnc:		Boolean Change-Notification Control
- * @response:		Response code will be stored at this address
- * @cb: 		Callback function will be executed for each element
- *			of the address list
- * @priv:		Pointer to pass to the callback function.
- *
- * Performs "Store-network-bridging-information list" operation and calls
- * the callback function for every entry in the list. If "change-
- * notification-control" is set, further changes in the address list
- * will be reported via the IPA command.
- */
-int qdio_pnso_brinfo(struct subchannel_id schid,
-		int cnc, u16 *response,
-		void (*cb)(void *priv, enum qdio_brinfo_entry_type type,
-				void *entry),
-		void *priv)
-{
-	struct chsc_pnso_area *rr;
-	int rc;
-	u32 prev_instance = 0;
-	int isfirstblock = 1;
-	int i, size, elems;
-
-	rr = (struct chsc_pnso_area *)get_zeroed_page(GFP_KERNEL);
-	if (rr == NULL)
-		return -ENOMEM;
-	do {
-		/* on the first iteration, naihdr.resume_token will be zero */
-		rc = chsc_pnso_brinfo(schid, rr, rr->naihdr.resume_token, cnc);
-		if (rc != 0 && rc != -EBUSY)
-			goto out;
-		if (rr->response.code != 1) {
-			rc = -EIO;
-			continue;
-		} else
-			rc = 0;
-
-		if (cb == NULL)
-			continue;
-
-		size = rr->naihdr.naids;
-		elems = (rr->response.length -
-				sizeof(struct chsc_header) -
-				sizeof(struct chsc_brinfo_naihdr)) /
-				size;
-
-		if (!isfirstblock && (rr->naihdr.instance != prev_instance)) {
-			/* Inform the caller that they need to scrap */
-			/* the data that was already reported via cb */
-				rc = -EAGAIN;
-				break;
-		}
-		isfirstblock = 0;
-		prev_instance = rr->naihdr.instance;
-		for (i = 0; i < elems; i++)
-			switch (size) {
-			case sizeof(struct qdio_brinfo_entry_l3_ipv6):
-				(*cb)(priv, l3_ipv6_addr,
-						&rr->entries.l3_ipv6[i]);
-				break;
-			case sizeof(struct qdio_brinfo_entry_l3_ipv4):
-				(*cb)(priv, l3_ipv4_addr,
-						&rr->entries.l3_ipv4[i]);
-				break;
-			case sizeof(struct qdio_brinfo_entry_l2):
-				(*cb)(priv, l2_addr_lnid,
-						&rr->entries.l2[i]);
-				break;
-			default:
-				WARN_ON_ONCE(1);
-				rc = -EIO;
-				goto out;
-			}
-	} while (rr->response.code == 0x0107 ||  /* channel busy */
-		  (rr->response.code == 1 && /* list stored */
-		   /* resume token is non-zero => list incomplete */
-		   (rr->naihdr.resume_token.t1 || rr->naihdr.resume_token.t2)));
-	(*response) = rr->response.code;
-
-out:
-	free_page((unsigned long)rr);
-	return rc;
-}
-EXPORT_SYMBOL_GPL(qdio_pnso_brinfo);
-
 static int __init init_QDIO(void)
 {
 	int rc;
@@ -1868,16 +1715,11 @@ static int __init init_QDIO(void)
 	rc = qdio_setup_init();
 	if (rc)
 		goto out_debug;
-	rc = tiqdio_allocate_memory();
+	rc = qdio_thinint_init();
 	if (rc)
 		goto out_cache;
-	rc = tiqdio_register_thinints();
-	if (rc)
-		goto out_ti;
 	return 0;
 
-out_ti:
-	tiqdio_free_memory();
 out_cache:
 	qdio_setup_exit();
 out_debug:
@@ -1887,8 +1729,7 @@ out_debug:
 
 static void __exit exit_QDIO(void)
 {
-	tiqdio_unregister_thinints();
-	tiqdio_free_memory();
+	qdio_thinint_exit();
 	qdio_setup_exit();
 	qdio_debug_exit();
 }
diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c
index 3083edd61f0c..2c5cc6ec668e 100644
--- a/drivers/s390/cio/qdio_setup.c
+++ b/drivers/s390/cio/qdio_setup.c
@@ -135,6 +135,27 @@ output:
 	}
 }
 
+static void __qdio_free_queues(struct qdio_q **queues, unsigned int count)
+{
+	struct qdio_q *q;
+	unsigned int i;
+
+	for (i = 0; i < count; i++) {
+		q = queues[i];
+		free_page((unsigned long) q->slib);
+		kmem_cache_free(qdio_q_cache, q);
+	}
+}
+
+void qdio_free_queues(struct qdio_irq *irq_ptr)
+{
+	__qdio_free_queues(irq_ptr->input_qs, irq_ptr->max_input_qs);
+	irq_ptr->max_input_qs = 0;
+
+	__qdio_free_queues(irq_ptr->output_qs, irq_ptr->max_output_qs);
+	irq_ptr->max_output_qs = 0;
+}
+
 static int __qdio_allocate_qs(struct qdio_q **irq_ptr_qs, int nr_queues)
 {
 	struct qdio_q *q;
@@ -142,12 +163,15 @@ static int __qdio_allocate_qs(struct qdio_q **irq_ptr_qs, int nr_queues)
 
 	for (i = 0; i < nr_queues; i++) {
 		q = kmem_cache_zalloc(qdio_q_cache, GFP_KERNEL);
-		if (!q)
+		if (!q) {
+			__qdio_free_queues(irq_ptr_qs, i);
 			return -ENOMEM;
+		}
 
 		q->slib = (struct slib *) __get_free_page(GFP_KERNEL);
 		if (!q->slib) {
 			kmem_cache_free(qdio_q_cache, q);
+			__qdio_free_queues(irq_ptr_qs, i);
 			return -ENOMEM;
 		}
 		irq_ptr_qs[i] = q;
@@ -162,8 +186,16 @@ int qdio_allocate_qs(struct qdio_irq *irq_ptr, int nr_input_qs, int nr_output_qs
 	rc = __qdio_allocate_qs(irq_ptr->input_qs, nr_input_qs);
 	if (rc)
 		return rc;
+
 	rc = __qdio_allocate_qs(irq_ptr->output_qs, nr_output_qs);
-	return rc;
+	if (rc) {
+		__qdio_free_queues(irq_ptr->input_qs, nr_input_qs);
+		return rc;
+	}
+
+	irq_ptr->max_input_qs = nr_input_qs;
+	irq_ptr->max_output_qs = nr_output_qs;
+	return 0;
 }
 
 static void setup_queues_misc(struct qdio_q *q, struct qdio_irq *irq_ptr,
@@ -347,45 +379,28 @@ void qdio_setup_ssqd_info(struct qdio_irq *irq_ptr)
 	DBF_EVENT("3:%4x qib:%4x", irq_ptr->ssqd_desc.qdioac3, irq_ptr->qib.ac);
 }
 
-void qdio_release_memory(struct qdio_irq *irq_ptr)
+void qdio_free_async_data(struct qdio_irq *irq_ptr)
 {
 	struct qdio_q *q;
 	int i;
 
-	/*
-	 * Must check queue array manually since irq_ptr->nr_input_queues /
-	 * irq_ptr->nr_input_queues may not yet be set.
-	 */
-	for (i = 0; i < QDIO_MAX_QUEUES_PER_IRQ; i++) {
-		q = irq_ptr->input_qs[i];
-		if (q) {
-			free_page((unsigned long) q->slib);
-			kmem_cache_free(qdio_q_cache, q);
-		}
-	}
-	for (i = 0; i < QDIO_MAX_QUEUES_PER_IRQ; i++) {
+	for (i = 0; i < irq_ptr->max_output_qs; i++) {
 		q = irq_ptr->output_qs[i];
-		if (q) {
-			if (q->u.out.use_cq) {
-				int n;
-
-				for (n = 0; n < QDIO_MAX_BUFFERS_PER_Q; ++n) {
-					struct qaob *aob = q->u.out.aobs[n];
-					if (aob) {
-						qdio_release_aob(aob);
-						q->u.out.aobs[n] = NULL;
-					}
-				}
+		if (q->u.out.use_cq) {
+			unsigned int n;
+
+			for (n = 0; n < QDIO_MAX_BUFFERS_PER_Q; n++) {
+				struct qaob *aob = q->u.out.aobs[n];
 
-				qdio_disable_async_operation(&q->u.out);
+				if (aob) {
+					qdio_release_aob(aob);
+					q->u.out.aobs[n] = NULL;
+				}
 			}
-			free_page((unsigned long) q->slib);
-			kmem_cache_free(qdio_q_cache, q);
+
+			qdio_disable_async_operation(&q->u.out);
 		}
 	}
-	free_page((unsigned long) irq_ptr->qdr);
-	free_page(irq_ptr->chsc_page);
-	free_page((unsigned long) irq_ptr);
 }
 
 static void __qdio_allocate_fill_qdr(struct qdio_irq *irq_ptr,
@@ -480,7 +495,6 @@ int qdio_setup_irq(struct qdio_irq *irq_ptr, struct qdio_initialize *init_data)
 	}
 
 	setup_qib(irq_ptr, init_data);
-	qdio_setup_thinint(irq_ptr);
 	set_impl_params(irq_ptr, init_data->qib_param_field_format,
 			init_data->qib_param_field,
 			init_data->input_slib_elements,
@@ -491,6 +505,12 @@ int qdio_setup_irq(struct qdio_irq *irq_ptr, struct qdio_initialize *init_data)
 
 	/* qdr, qib, sls, slsbs, slibs, sbales are filled now */
 
+	/* set our IRQ handler */
+	spin_lock_irq(get_ccwdev_lock(cdev));
+	irq_ptr->orig_handler = cdev->handler;
+	cdev->handler = qdio_int_handler;
+	spin_unlock_irq(get_ccwdev_lock(cdev));
+
 	/* get qdio commands */
 	ciw = ccw_device_get_ciw(cdev, CIW_TYPE_EQUEUE);
 	if (!ciw) {
@@ -506,12 +526,18 @@ int qdio_setup_irq(struct qdio_irq *irq_ptr, struct qdio_initialize *init_data)
 	}
 	irq_ptr->aqueue = *ciw;
 
-	/* set new interrupt handler */
+	return 0;
+}
+
+void qdio_shutdown_irq(struct qdio_irq *irq)
+{
+	struct ccw_device *cdev = irq->cdev;
+
+	/* restore IRQ handler */
 	spin_lock_irq(get_ccwdev_lock(cdev));
-	irq_ptr->orig_handler = cdev->handler;
-	cdev->handler = qdio_int_handler;
+	cdev->handler = irq->orig_handler;
+	cdev->private->intparm = 0;
 	spin_unlock_irq(get_ccwdev_lock(cdev));
-	return 0;
 }
 
 void qdio_print_subchannel_info(struct qdio_irq *irq_ptr)
diff --git a/drivers/s390/cio/qdio_thinint.c b/drivers/s390/cio/qdio_thinint.c
index ae50373617cd..7a440e4328cd 100644
--- a/drivers/s390/cio/qdio_thinint.c
+++ b/drivers/s390/cio/qdio_thinint.c
@@ -197,47 +197,21 @@ out:
 	return rc;
 }
 
-/* allocate non-shared indicators and shared indicator */
-int __init tiqdio_allocate_memory(void)
-{
-	q_indicators = kcalloc(TIQDIO_NR_INDICATORS,
-			       sizeof(struct indicator_t),
-			       GFP_KERNEL);
-	if (!q_indicators)
-		return -ENOMEM;
-	return 0;
-}
-
-void tiqdio_free_memory(void)
-{
-	kfree(q_indicators);
-}
-
-int __init tiqdio_register_thinints(void)
+int qdio_establish_thinint(struct qdio_irq *irq_ptr)
 {
 	int rc;
 
-	rc = register_adapter_interrupt(&tiqdio_airq);
-	if (rc) {
-		DBF_EVENT("RTI:%x", rc);
-		return rc;
-	}
-	return 0;
-}
-
-int qdio_establish_thinint(struct qdio_irq *irq_ptr)
-{
 	if (!is_thinint_irq(irq_ptr))
 		return 0;
-	return set_subchannel_ind(irq_ptr, 0);
-}
 
-void qdio_setup_thinint(struct qdio_irq *irq_ptr)
-{
-	if (!is_thinint_irq(irq_ptr))
-		return;
 	irq_ptr->dsci = get_indicator();
 	DBF_HEX(&irq_ptr->dsci, sizeof(void *));
+
+	rc = set_subchannel_ind(irq_ptr, 0);
+	if (rc)
+		put_indicator(irq_ptr->dsci);
+
+	return rc;
 }
 
 void qdio_shutdown_thinint(struct qdio_irq *irq_ptr)
@@ -250,8 +224,27 @@ void qdio_shutdown_thinint(struct qdio_irq *irq_ptr)
 	put_indicator(irq_ptr->dsci);
 }
 
-void __exit tiqdio_unregister_thinints(void)
+int __init qdio_thinint_init(void)
+{
+	int rc;
+
+	q_indicators = kcalloc(TIQDIO_NR_INDICATORS, sizeof(struct indicator_t),
+			       GFP_KERNEL);
+	if (!q_indicators)
+		return -ENOMEM;
+
+	rc = register_adapter_interrupt(&tiqdio_airq);
+	if (rc) {
+		DBF_EVENT("RTI:%x", rc);
+		kfree(q_indicators);
+		return rc;
+	}
+	return 0;
+}
+
+void __exit qdio_thinint_exit(void)
 {
 	WARN_ON(!list_empty(&tiq_list));
 	unregister_adapter_interrupt(&tiqdio_airq);
+	kfree(q_indicators);
 }
diff --git a/drivers/s390/cio/vfio_ccw_chp.c b/drivers/s390/cio/vfio_ccw_chp.c
new file mode 100644
index 000000000000..a646fc81c872
--- /dev/null
+++ b/drivers/s390/cio/vfio_ccw_chp.c
@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Channel path related status regions for vfio_ccw
+ *
+ * Copyright IBM Corp. 2020
+ *
+ * Author(s): Farhan Ali <alifm@linux.ibm.com>
+ *            Eric Farman <farman@linux.ibm.com>
+ */
+
+#include <linux/vfio.h>
+#include "vfio_ccw_private.h"
+
+static ssize_t vfio_ccw_schib_region_read(struct vfio_ccw_private *private,
+					  char __user *buf, size_t count,
+					  loff_t *ppos)
+{
+	unsigned int i = VFIO_CCW_OFFSET_TO_INDEX(*ppos) - VFIO_CCW_NUM_REGIONS;
+	loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
+	struct ccw_schib_region *region;
+	int ret;
+
+	if (pos + count > sizeof(*region))
+		return -EINVAL;
+
+	mutex_lock(&private->io_mutex);
+	region = private->region[i].data;
+
+	if (cio_update_schib(private->sch)) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	memcpy(region, &private->sch->schib, sizeof(*region));
+
+	if (copy_to_user(buf, (void *)region + pos, count)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	ret = count;
+
+out:
+	mutex_unlock(&private->io_mutex);
+	return ret;
+}
+
+static ssize_t vfio_ccw_schib_region_write(struct vfio_ccw_private *private,
+					   const char __user *buf, size_t count,
+					   loff_t *ppos)
+{
+	return -EINVAL;
+}
+
+
+static void vfio_ccw_schib_region_release(struct vfio_ccw_private *private,
+					  struct vfio_ccw_region *region)
+{
+
+}
+
+static const struct vfio_ccw_regops vfio_ccw_schib_region_ops = {
+	.read = vfio_ccw_schib_region_read,
+	.write = vfio_ccw_schib_region_write,
+	.release = vfio_ccw_schib_region_release,
+};
+
+int vfio_ccw_register_schib_dev_regions(struct vfio_ccw_private *private)
+{
+	return vfio_ccw_register_dev_region(private,
+					    VFIO_REGION_SUBTYPE_CCW_SCHIB,
+					    &vfio_ccw_schib_region_ops,
+					    sizeof(struct ccw_schib_region),
+					    VFIO_REGION_INFO_FLAG_READ,
+					    private->schib_region);
+}
+
+static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private,
+					char __user *buf, size_t count,
+					loff_t *ppos)
+{
+	unsigned int i = VFIO_CCW_OFFSET_TO_INDEX(*ppos) - VFIO_CCW_NUM_REGIONS;
+	loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK;
+	struct ccw_crw_region *region;
+	struct vfio_ccw_crw *crw;
+	int ret;
+
+	if (pos + count > sizeof(*region))
+		return -EINVAL;
+
+	crw = list_first_entry_or_null(&private->crw,
+				       struct vfio_ccw_crw, next);
+
+	if (crw)
+		list_del(&crw->next);
+
+	mutex_lock(&private->io_mutex);
+	region = private->region[i].data;
+
+	if (crw)
+		memcpy(&region->crw, &crw->crw, sizeof(region->crw));
+
+	if (copy_to_user(buf, (void *)region + pos, count))
+		ret = -EFAULT;
+	else
+		ret = count;
+
+	region->crw = 0;
+
+	mutex_unlock(&private->io_mutex);
+
+	kfree(crw);
+
+	/* Notify the guest if more CRWs are on our queue */
+	if (!list_empty(&private->crw) && private->crw_trigger)
+		eventfd_signal(private->crw_trigger, 1);
+
+	return ret;
+}
+
+static ssize_t vfio_ccw_crw_region_write(struct vfio_ccw_private *private,
+					 const char __user *buf, size_t count,
+					 loff_t *ppos)
+{
+	return -EINVAL;
+}
+
+static void vfio_ccw_crw_region_release(struct vfio_ccw_private *private,
+					struct vfio_ccw_region *region)
+{
+
+}
+
+static const struct vfio_ccw_regops vfio_ccw_crw_region_ops = {
+	.read = vfio_ccw_crw_region_read,
+	.write = vfio_ccw_crw_region_write,
+	.release = vfio_ccw_crw_region_release,
+};
+
+int vfio_ccw_register_crw_dev_regions(struct vfio_ccw_private *private)
+{
+	return vfio_ccw_register_dev_region(private,
+					    VFIO_REGION_SUBTYPE_CCW_CRW,
+					    &vfio_ccw_crw_region_ops,
+					    sizeof(struct ccw_crw_region),
+					    VFIO_REGION_INFO_FLAG_READ,
+					    private->crw_region);
+}
diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c
index 3645d1720c4b..b9febc581b1f 100644
--- a/drivers/s390/cio/vfio_ccw_cp.c
+++ b/drivers/s390/cio/vfio_ccw_cp.c
@@ -8,6 +8,7 @@
  *            Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
  */
 
+#include <linux/ratelimit.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
 #include <linux/iommu.h>
@@ -625,23 +626,27 @@ static int ccwchain_fetch_one(struct ccwchain *chain,
  * the target channel program from @orb->cmd.iova to the new ccwchain(s).
  *
  * Limitations:
- * 1. Supports only prefetch enabled mode.
- * 2. Supports idal(c64) ccw chaining.
- * 3. Supports 4k idaw.
+ * 1. Supports idal(c64) ccw chaining.
+ * 2. Supports 4k idaw.
  *
  * Returns:
  *   %0 on success and a negative error value on failure.
  */
 int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb)
 {
+	/* custom ratelimit used to avoid flood during guest IPL */
+	static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 1);
 	int ret;
 
 	/*
-	 * XXX:
-	 * Only support prefetch enable mode now.
+	 * We only support prefetching the channel program. We assume all channel
+	 * programs executed by supported guests likewise support prefetching.
+	 * Executing a channel program that does not specify prefetching will
+	 * typically not cause an error, but a warning is issued to help identify
+	 * the problem if something does break.
 	 */
-	if (!orb->cmd.pfch)
-		return -EOPNOTSUPP;
+	if (!orb->cmd.pfch && __ratelimit(&ratelimit_state))
+		dev_warn(mdev, "Prefetching channel program even though prefetch not specified in ORB");
 
 	INIT_LIST_HEAD(&cp->ccwchain_list);
 	memcpy(&cp->orb, orb, sizeof(*orb));
diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c
index 339a6bc0339b..8c625b530035 100644
--- a/drivers/s390/cio/vfio_ccw_drv.c
+++ b/drivers/s390/cio/vfio_ccw_drv.c
@@ -19,6 +19,7 @@
 
 #include <asm/isc.h>
 
+#include "chp.h"
 #include "ioasm.h"
 #include "css.h"
 #include "vfio_ccw_private.h"
@@ -26,6 +27,8 @@
 struct workqueue_struct *vfio_ccw_work_q;
 static struct kmem_cache *vfio_ccw_io_region;
 static struct kmem_cache *vfio_ccw_cmd_region;
+static struct kmem_cache *vfio_ccw_schib_region;
+static struct kmem_cache *vfio_ccw_crw_region;
 
 debug_info_t *vfio_ccw_debug_msg_id;
 debug_info_t *vfio_ccw_debug_trace_id;
@@ -105,6 +108,16 @@ static void vfio_ccw_sch_io_todo(struct work_struct *work)
 		eventfd_signal(private->io_trigger, 1);
 }
 
+static void vfio_ccw_crw_todo(struct work_struct *work)
+{
+	struct vfio_ccw_private *private;
+
+	private = container_of(work, struct vfio_ccw_private, crw_work);
+
+	if (!list_empty(&private->crw) && private->crw_trigger)
+		eventfd_signal(private->crw_trigger, 1);
+}
+
 /*
  * Css driver callbacks
  */
@@ -116,6 +129,18 @@ static void vfio_ccw_sch_irq(struct subchannel *sch)
 	vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_INTERRUPT);
 }
 
+static void vfio_ccw_free_regions(struct vfio_ccw_private *private)
+{
+	if (private->crw_region)
+		kmem_cache_free(vfio_ccw_crw_region, private->crw_region);
+	if (private->schib_region)
+		kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
+	if (private->cmd_region)
+		kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
+	if (private->io_region)
+		kmem_cache_free(vfio_ccw_io_region, private->io_region);
+}
+
 static int vfio_ccw_sch_probe(struct subchannel *sch)
 {
 	struct pmcw *pmcw = &sch->schib.pmcw;
@@ -147,6 +172,18 @@ static int vfio_ccw_sch_probe(struct subchannel *sch)
 	if (!private->cmd_region)
 		goto out_free;
 
+	private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region,
+						  GFP_KERNEL | GFP_DMA);
+
+	if (!private->schib_region)
+		goto out_free;
+
+	private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region,
+						GFP_KERNEL | GFP_DMA);
+
+	if (!private->crw_region)
+		goto out_free;
+
 	private->sch = sch;
 	dev_set_drvdata(&sch->dev, private);
 	mutex_init(&private->io_mutex);
@@ -159,7 +196,9 @@ static int vfio_ccw_sch_probe(struct subchannel *sch)
 	if (ret)
 		goto out_free;
 
+	INIT_LIST_HEAD(&private->crw);
 	INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
+	INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
 	atomic_set(&private->avail, 1);
 	private->state = VFIO_CCW_STATE_STANDBY;
 
@@ -181,10 +220,7 @@ out_disable:
 	cio_disable_subchannel(sch);
 out_free:
 	dev_set_drvdata(&sch->dev, NULL);
-	if (private->cmd_region)
-		kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
-	if (private->io_region)
-		kmem_cache_free(vfio_ccw_io_region, private->io_region);
+	vfio_ccw_free_regions(private);
 	kfree(private->cp.guest_cp);
 	kfree(private);
 	return ret;
@@ -193,15 +229,20 @@ out_free:
 static int vfio_ccw_sch_remove(struct subchannel *sch)
 {
 	struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
+	struct vfio_ccw_crw *crw, *temp;
 
 	vfio_ccw_sch_quiesce(sch);
 
+	list_for_each_entry_safe(crw, temp, &private->crw, next) {
+		list_del(&crw->next);
+		kfree(crw);
+	}
+
 	vfio_ccw_mdev_unreg(sch);
 
 	dev_set_drvdata(&sch->dev, NULL);
 
-	kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
-	kmem_cache_free(vfio_ccw_io_region, private->io_region);
+	vfio_ccw_free_regions(private);
 	kfree(private->cp.guest_cp);
 	kfree(private);
 
@@ -258,6 +299,83 @@ out_unlock:
 	return rc;
 }
 
+static void vfio_ccw_queue_crw(struct vfio_ccw_private *private,
+			       unsigned int rsc,
+			       unsigned int erc,
+			       unsigned int rsid)
+{
+	struct vfio_ccw_crw *crw;
+
+	/*
+	 * If unable to allocate a CRW, just drop the event and
+	 * carry on.  The guest will either see a later one or
+	 * learn when it issues its own store subchannel.
+	 */
+	crw = kzalloc(sizeof(*crw), GFP_ATOMIC);
+	if (!crw)
+		return;
+
+	/*
+	 * Build the CRW based on the inputs given to us.
+	 */
+	crw->crw.rsc = rsc;
+	crw->crw.erc = erc;
+	crw->crw.rsid = rsid;
+
+	list_add_tail(&crw->next, &private->crw);
+	queue_work(vfio_ccw_work_q, &private->crw_work);
+}
+
+static int vfio_ccw_chp_event(struct subchannel *sch,
+			      struct chp_link *link, int event)
+{
+	struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
+	int mask = chp_ssd_get_mask(&sch->ssd_info, link);
+	int retry = 255;
+
+	if (!private || !mask)
+		return 0;
+
+	trace_vfio_ccw_chp_event(private->sch->schid, mask, event);
+	VFIO_CCW_MSG_EVENT(2, "%pUl (%x.%x.%04x): mask=0x%x event=%d\n",
+			   mdev_uuid(private->mdev), sch->schid.cssid,
+			   sch->schid.ssid, sch->schid.sch_no,
+			   mask, event);
+
+	if (cio_update_schib(sch))
+		return -ENODEV;
+
+	switch (event) {
+	case CHP_VARY_OFF:
+		/* Path logically turned off */
+		sch->opm &= ~mask;
+		sch->lpm &= ~mask;
+		if (sch->schib.pmcw.lpum & mask)
+			cio_cancel_halt_clear(sch, &retry);
+		break;
+	case CHP_OFFLINE:
+		/* Path is gone */
+		if (sch->schib.pmcw.lpum & mask)
+			cio_cancel_halt_clear(sch, &retry);
+		vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_PERRN,
+				   link->chpid.id);
+		break;
+	case CHP_VARY_ON:
+		/* Path logically turned on */
+		sch->opm |= mask;
+		sch->lpm |= mask;
+		break;
+	case CHP_ONLINE:
+		/* Path became available */
+		sch->lpm |= mask & sch->opm;
+		vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_INIT,
+				   link->chpid.id);
+		break;
+	}
+
+	return 0;
+}
+
 static struct css_device_id vfio_ccw_sch_ids[] = {
 	{ .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, },
 	{ /* end of list */ },
@@ -275,6 +393,7 @@ static struct css_driver vfio_ccw_sch_driver = {
 	.remove = vfio_ccw_sch_remove,
 	.shutdown = vfio_ccw_sch_shutdown,
 	.sch_event = vfio_ccw_sch_event,
+	.chp_event = vfio_ccw_chp_event,
 };
 
 static int __init vfio_ccw_debug_init(void)
@@ -304,6 +423,14 @@ static void vfio_ccw_debug_exit(void)
 	debug_unregister(vfio_ccw_debug_trace_id);
 }
 
+static void vfio_ccw_destroy_regions(void)
+{
+	kmem_cache_destroy(vfio_ccw_crw_region);
+	kmem_cache_destroy(vfio_ccw_schib_region);
+	kmem_cache_destroy(vfio_ccw_cmd_region);
+	kmem_cache_destroy(vfio_ccw_io_region);
+}
+
 static int __init vfio_ccw_sch_init(void)
 {
 	int ret;
@@ -336,6 +463,26 @@ static int __init vfio_ccw_sch_init(void)
 		goto out_err;
 	}
 
+	vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region",
+					sizeof(struct ccw_schib_region), 0,
+					SLAB_ACCOUNT, 0,
+					sizeof(struct ccw_schib_region), NULL);
+
+	if (!vfio_ccw_schib_region) {
+		ret = -ENOMEM;
+		goto out_err;
+	}
+
+	vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region",
+					sizeof(struct ccw_crw_region), 0,
+					SLAB_ACCOUNT, 0,
+					sizeof(struct ccw_crw_region), NULL);
+
+	if (!vfio_ccw_crw_region) {
+		ret = -ENOMEM;
+		goto out_err;
+	}
+
 	isc_register(VFIO_CCW_ISC);
 	ret = css_driver_register(&vfio_ccw_sch_driver);
 	if (ret) {
@@ -346,8 +493,7 @@ static int __init vfio_ccw_sch_init(void)
 	return ret;
 
 out_err:
-	kmem_cache_destroy(vfio_ccw_cmd_region);
-	kmem_cache_destroy(vfio_ccw_io_region);
+	vfio_ccw_destroy_regions();
 	destroy_workqueue(vfio_ccw_work_q);
 	vfio_ccw_debug_exit();
 	return ret;
@@ -357,8 +503,7 @@ static void __exit vfio_ccw_sch_exit(void)
 {
 	css_driver_unregister(&vfio_ccw_sch_driver);
 	isc_unregister(VFIO_CCW_ISC);
-	kmem_cache_destroy(vfio_ccw_io_region);
-	kmem_cache_destroy(vfio_ccw_cmd_region);
+	vfio_ccw_destroy_regions();
 	destroy_workqueue(vfio_ccw_work_q);
 	vfio_ccw_debug_exit();
 }
diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c
index f0d71ab77c50..8b3ed5b45277 100644
--- a/drivers/s390/cio/vfio_ccw_ops.c
+++ b/drivers/s390/cio/vfio_ccw_ops.c
@@ -172,8 +172,22 @@ static int vfio_ccw_mdev_open(struct mdev_device *mdev)
 
 	ret = vfio_ccw_register_async_dev_regions(private);
 	if (ret)
-		vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
-					 &private->nb);
+		goto out_unregister;
+
+	ret = vfio_ccw_register_schib_dev_regions(private);
+	if (ret)
+		goto out_unregister;
+
+	ret = vfio_ccw_register_crw_dev_regions(private);
+	if (ret)
+		goto out_unregister;
+
+	return ret;
+
+out_unregister:
+	vfio_ccw_unregister_dev_regions(private);
+	vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
+				 &private->nb);
 	return ret;
 }
 
@@ -181,7 +195,6 @@ static void vfio_ccw_mdev_release(struct mdev_device *mdev)
 {
 	struct vfio_ccw_private *private =
 		dev_get_drvdata(mdev_parent_dev(mdev));
-	int i;
 
 	if ((private->state != VFIO_CCW_STATE_NOT_OPER) &&
 	    (private->state != VFIO_CCW_STATE_STANDBY)) {
@@ -191,15 +204,9 @@ static void vfio_ccw_mdev_release(struct mdev_device *mdev)
 	}
 
 	cp_free(&private->cp);
+	vfio_ccw_unregister_dev_regions(private);
 	vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
 				 &private->nb);
-
-	for (i = 0; i < private->num_regions; i++)
-		private->region[i].ops->release(private, &private->region[i]);
-
-	private->num_regions = 0;
-	kfree(private->region);
-	private->region = NULL;
 }
 
 static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private,
@@ -384,17 +391,22 @@ static int vfio_ccw_mdev_get_region_info(struct vfio_region_info *info,
 
 static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info)
 {
-	if (info->index != VFIO_CCW_IO_IRQ_INDEX)
+	switch (info->index) {
+	case VFIO_CCW_IO_IRQ_INDEX:
+	case VFIO_CCW_CRW_IRQ_INDEX:
+		info->count = 1;
+		info->flags = VFIO_IRQ_INFO_EVENTFD;
+		break;
+	default:
 		return -EINVAL;
-
-	info->count = 1;
-	info->flags = VFIO_IRQ_INFO_EVENTFD;
+	}
 
 	return 0;
 }
 
 static int vfio_ccw_mdev_set_irqs(struct mdev_device *mdev,
 				  uint32_t flags,
+				  uint32_t index,
 				  void __user *data)
 {
 	struct vfio_ccw_private *private;
@@ -404,7 +416,17 @@ static int vfio_ccw_mdev_set_irqs(struct mdev_device *mdev,
 		return -EINVAL;
 
 	private = dev_get_drvdata(mdev_parent_dev(mdev));
-	ctx = &private->io_trigger;
+
+	switch (index) {
+	case VFIO_CCW_IO_IRQ_INDEX:
+		ctx = &private->io_trigger;
+		break;
+	case VFIO_CCW_CRW_IRQ_INDEX:
+		ctx = &private->crw_trigger;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	switch (flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
 	case VFIO_IRQ_SET_DATA_NONE:
@@ -482,6 +504,17 @@ int vfio_ccw_register_dev_region(struct vfio_ccw_private *private,
 	return 0;
 }
 
+void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private)
+{
+	int i;
+
+	for (i = 0; i < private->num_regions; i++)
+		private->region[i].ops->release(private, &private->region[i]);
+	private->num_regions = 0;
+	kfree(private->region);
+	private->region = NULL;
+}
+
 static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev,
 				   unsigned int cmd,
 				   unsigned long arg)
@@ -565,7 +598,7 @@ static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev,
 			return ret;
 
 		data = (void __user *)(arg + minsz);
-		return vfio_ccw_mdev_set_irqs(mdev, hdr.flags, data);
+		return vfio_ccw_mdev_set_irqs(mdev, hdr.flags, hdr.index, data);
 	}
 	case VFIO_DEVICE_RESET:
 		return vfio_ccw_mdev_reset(mdev);
diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h
index 9b9bb4982972..8723156b29ea 100644
--- a/drivers/s390/cio/vfio_ccw_private.h
+++ b/drivers/s390/cio/vfio_ccw_private.h
@@ -17,6 +17,7 @@
 #include <linux/eventfd.h>
 #include <linux/workqueue.h>
 #include <linux/vfio_ccw.h>
+#include <asm/crw.h>
 #include <asm/debug.h>
 
 #include "css.h"
@@ -53,8 +54,16 @@ int vfio_ccw_register_dev_region(struct vfio_ccw_private *private,
 				 unsigned int subtype,
 				 const struct vfio_ccw_regops *ops,
 				 size_t size, u32 flags, void *data);
+void vfio_ccw_unregister_dev_regions(struct vfio_ccw_private *private);
 
 int vfio_ccw_register_async_dev_regions(struct vfio_ccw_private *private);
+int vfio_ccw_register_schib_dev_regions(struct vfio_ccw_private *private);
+int vfio_ccw_register_crw_dev_regions(struct vfio_ccw_private *private);
+
+struct vfio_ccw_crw {
+	struct list_head	next;
+	struct crw		crw;
+};
 
 /**
  * struct vfio_ccw_private
@@ -68,6 +77,8 @@ int vfio_ccw_register_async_dev_regions(struct vfio_ccw_private *private);
  * @io_mutex: protect against concurrent update of I/O regions
  * @region: additional regions for other subchannel operations
  * @cmd_region: MMIO region for asynchronous I/O commands other than START
+ * @schib_region: MMIO region for SCHIB information
+ * @crw_region: MMIO region for getting channel report words
  * @num_regions: number of additional regions
  * @cp: channel program for the current I/O operation
  * @irb: irb info received from interrupt
@@ -86,14 +97,19 @@ struct vfio_ccw_private {
 	struct mutex		io_mutex;
 	struct vfio_ccw_region *region;
 	struct ccw_cmd_region	*cmd_region;
+	struct ccw_schib_region *schib_region;
+	struct ccw_crw_region	*crw_region;
 	int num_regions;
 
 	struct channel_program	cp;
 	struct irb		irb;
 	union scsw		scsw;
+	struct list_head	crw;
 
 	struct eventfd_ctx	*io_trigger;
+	struct eventfd_ctx	*crw_trigger;
 	struct work_struct	io_work;
+	struct work_struct	crw_work;
 } __aligned(8);
 
 extern int vfio_ccw_mdev_reg(struct subchannel *sch);
diff --git a/drivers/s390/cio/vfio_ccw_trace.c b/drivers/s390/cio/vfio_ccw_trace.c
index 8c671d2519f6..4a0205905afc 100644
--- a/drivers/s390/cio/vfio_ccw_trace.c
+++ b/drivers/s390/cio/vfio_ccw_trace.c
@@ -9,6 +9,7 @@
 #define CREATE_TRACE_POINTS
 #include "vfio_ccw_trace.h"
 
+EXPORT_TRACEPOINT_SYMBOL(vfio_ccw_chp_event);
 EXPORT_TRACEPOINT_SYMBOL(vfio_ccw_fsm_async_request);
 EXPORT_TRACEPOINT_SYMBOL(vfio_ccw_fsm_event);
 EXPORT_TRACEPOINT_SYMBOL(vfio_ccw_fsm_io_request);
diff --git a/drivers/s390/cio/vfio_ccw_trace.h b/drivers/s390/cio/vfio_ccw_trace.h
index f5d31887d413..62fb30598d47 100644
--- a/drivers/s390/cio/vfio_ccw_trace.h
+++ b/drivers/s390/cio/vfio_ccw_trace.h
@@ -17,6 +17,36 @@
 
 #include <linux/tracepoint.h>
 
+TRACE_EVENT(vfio_ccw_chp_event,
+	TP_PROTO(struct subchannel_id schid,
+		 int mask,
+		 int event),
+	TP_ARGS(schid, mask, event),
+
+	TP_STRUCT__entry(
+		__field(u8, cssid)
+		__field(u8, ssid)
+		__field(u16, sch_no)
+		__field(int, mask)
+		__field(int, event)
+	),
+
+	TP_fast_assign(
+		__entry->cssid = schid.cssid;
+		__entry->ssid = schid.ssid;
+		__entry->sch_no = schid.sch_no;
+		__entry->mask = mask;
+		__entry->event = event;
+	),
+
+	TP_printk("schid=%x.%x.%04x mask=0x%x event=%d",
+		  __entry->cssid,
+		  __entry->ssid,
+		  __entry->sch_no,
+		  __entry->mask,
+		  __entry->event)
+);
+
 TRACE_EVENT(vfio_ccw_fsm_async_request,
 	TP_PROTO(struct subchannel_id schid,
 		 int command,
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index 35064443e748..e71ca4a719a5 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -62,8 +62,10 @@ MODULE_PARM_DESC(aqmask, "AP bus domain mask.");
 
 static struct device *ap_root_device;
 
-DEFINE_SPINLOCK(ap_list_lock);
-LIST_HEAD(ap_card_list);
+/* Hashtable of all queue devices on the AP bus */
+DEFINE_HASHTABLE(ap_queues, 8);
+/* lock used for the ap_queues hashtable */
+DEFINE_SPINLOCK(ap_queues_lock);
 
 /* Default permissions (ioctl, card and domain masking) */
 struct ap_perms ap_perms;
@@ -414,7 +416,7 @@ static void ap_interrupt_handler(struct airq_struct *airq, bool floating)
  */
 static void ap_tasklet_fn(unsigned long dummy)
 {
-	struct ap_card *ac;
+	int bkt;
 	struct ap_queue *aq;
 	enum ap_wait wait = AP_WAIT_NONE;
 
@@ -425,34 +427,30 @@ static void ap_tasklet_fn(unsigned long dummy)
 	if (ap_using_interrupts())
 		xchg(ap_airq.lsi_ptr, 0);
 
-	spin_lock_bh(&ap_list_lock);
-	for_each_ap_card(ac) {
-		for_each_ap_queue(aq, ac) {
-			spin_lock_bh(&aq->lock);
-			wait = min(wait, ap_sm_event_loop(aq, AP_EVENT_POLL));
-			spin_unlock_bh(&aq->lock);
-		}
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode) {
+		spin_lock_bh(&aq->lock);
+		wait = min(wait, ap_sm_event_loop(aq, AP_EVENT_POLL));
+		spin_unlock_bh(&aq->lock);
 	}
-	spin_unlock_bh(&ap_list_lock);
+	spin_unlock_bh(&ap_queues_lock);
 
 	ap_wait(wait);
 }
 
 static int ap_pending_requests(void)
 {
-	struct ap_card *ac;
+	int bkt;
 	struct ap_queue *aq;
 
-	spin_lock_bh(&ap_list_lock);
-	for_each_ap_card(ac) {
-		for_each_ap_queue(aq, ac) {
-			if (aq->queue_count == 0)
-				continue;
-			spin_unlock_bh(&ap_list_lock);
-			return 1;
-		}
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode) {
+		if (aq->queue_count == 0)
+			continue;
+		spin_unlock_bh(&ap_queues_lock);
+		return 1;
 	}
-	spin_unlock_bh(&ap_list_lock);
+	spin_unlock_bh(&ap_queues_lock);
 	return 0;
 }
 
@@ -683,24 +681,20 @@ static int ap_device_probe(struct device *dev)
 	}
 
 	/* Add queue/card to list of active queues/cards */
-	spin_lock_bh(&ap_list_lock);
-	if (is_card_dev(dev))
-		list_add(&to_ap_card(dev)->list, &ap_card_list);
-	else
-		list_add(&to_ap_queue(dev)->list,
-			 &to_ap_queue(dev)->card->queues);
-	spin_unlock_bh(&ap_list_lock);
+	spin_lock_bh(&ap_queues_lock);
+	if (is_queue_dev(dev))
+		hash_add(ap_queues, &to_ap_queue(dev)->hnode,
+			 to_ap_queue(dev)->qid);
+	spin_unlock_bh(&ap_queues_lock);
 
 	ap_dev->drv = ap_drv;
 	rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
 
 	if (rc) {
-		spin_lock_bh(&ap_list_lock);
-		if (is_card_dev(dev))
-			list_del_init(&to_ap_card(dev)->list);
-		else
-			list_del_init(&to_ap_queue(dev)->list);
-		spin_unlock_bh(&ap_list_lock);
+		spin_lock_bh(&ap_queues_lock);
+		if (is_queue_dev(dev))
+			hash_del(&to_ap_queue(dev)->hnode);
+		spin_unlock_bh(&ap_queues_lock);
 		ap_dev->drv = NULL;
 	}
 
@@ -725,16 +719,33 @@ static int ap_device_remove(struct device *dev)
 		ap_queue_remove(to_ap_queue(dev));
 
 	/* Remove queue/card from list of active queues/cards */
-	spin_lock_bh(&ap_list_lock);
-	if (is_card_dev(dev))
-		list_del_init(&to_ap_card(dev)->list);
-	else
-		list_del_init(&to_ap_queue(dev)->list);
-	spin_unlock_bh(&ap_list_lock);
+	spin_lock_bh(&ap_queues_lock);
+	if (is_queue_dev(dev))
+		hash_del(&to_ap_queue(dev)->hnode);
+	spin_unlock_bh(&ap_queues_lock);
 
 	return 0;
 }
 
+struct ap_queue *ap_get_qdev(ap_qid_t qid)
+{
+	int bkt;
+	struct ap_queue *aq;
+
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode) {
+		if (aq->qid == qid) {
+			get_device(&aq->ap_dev.device);
+			spin_unlock_bh(&ap_queues_lock);
+			return aq;
+		}
+	}
+	spin_unlock_bh(&ap_queues_lock);
+
+	return NULL;
+}
+EXPORT_SYMBOL(ap_get_qdev);
+
 int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
 		       char *name)
 {
@@ -1506,6 +1517,9 @@ static int __init ap_module_init(void)
 		return -ENODEV;
 	}
 
+	/* init ap_queue hashtable */
+	hash_init(ap_queues);
+
 	/* set up the AP permissions (ioctls, ap and aq masks) */
 	ap_perms_init();
 
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index 8e8e37b6c0ee..053cc34d2ca2 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -15,6 +15,7 @@
 
 #include <linux/device.h>
 #include <linux/types.h>
+#include <linux/hashtable.h>
 #include <asm/isc.h>
 #include <asm/ap.h>
 
@@ -27,8 +28,8 @@
 
 extern int ap_domain_index;
 
-extern spinlock_t ap_list_lock;
-extern struct list_head ap_card_list;
+extern DECLARE_HASHTABLE(ap_queues, 8);
+extern spinlock_t ap_queues_lock;
 
 static inline int ap_test_bit(unsigned int *ptr, unsigned int nr)
 {
@@ -152,8 +153,6 @@ struct ap_device {
 
 struct ap_card {
 	struct ap_device ap_dev;
-	struct list_head list;		/* Private list of AP cards. */
-	struct list_head queues;	/* List of assoc. AP queues */
 	void *private;			/* ap driver private pointer. */
 	int raw_hwtype;			/* AP raw hardware type. */
 	unsigned int functions;		/* AP device function bitfield. */
@@ -166,7 +165,7 @@ struct ap_card {
 
 struct ap_queue {
 	struct ap_device ap_dev;
-	struct list_head list;		/* Private list of AP queues. */
+	struct hlist_node hnode;	/* Node for the ap_queues hashtable */
 	struct ap_card *card;		/* Ptr to assoc. AP card. */
 	spinlock_t lock;		/* Per device lock. */
 	void *private;			/* ap driver private pointer. */
@@ -223,12 +222,6 @@ static inline void ap_release_message(struct ap_message *ap_msg)
 	kzfree(ap_msg->private);
 }
 
-#define for_each_ap_card(_ac) \
-	list_for_each_entry(_ac, &ap_card_list, list)
-
-#define for_each_ap_queue(_aq, _ac) \
-	list_for_each_entry(_aq, &(_ac)->queues, list)
-
 /*
  * Note: don't use ap_send/ap_recv after using ap_queue_message
  * for the first time. Otherwise the ap message queue will get
@@ -270,6 +263,16 @@ extern struct ap_perms ap_perms;
 extern struct mutex ap_perms_mutex;
 
 /*
+ * Get ap_queue device for this qid.
+ * Returns ptr to the struct ap_queue device or NULL if there
+ * was no ap_queue device with this qid found. When something is
+ * found, the reference count of the embedded device is increased.
+ * So the caller has to decrease the reference count after use
+ * with a call to put_device(&aq->ap_dev.device).
+ */
+struct ap_queue *ap_get_qdev(ap_qid_t qid);
+
+/*
  * check APQN for owned/reserved by ap bus and default driver(s).
  * Checks if this APQN is or will be in use by the ap bus
  * and the default set of drivers.
diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c
index 0a39dfdb6a1d..6588713319ba 100644
--- a/drivers/s390/crypto/ap_card.c
+++ b/drivers/s390/crypto/ap_card.c
@@ -66,9 +66,9 @@ static ssize_t request_count_show(struct device *dev,
 	u64 req_cnt;
 
 	req_cnt = 0;
-	spin_lock_bh(&ap_list_lock);
+	spin_lock_bh(&ap_queues_lock);
 	req_cnt = atomic64_read(&ac->total_request_count);
-	spin_unlock_bh(&ap_list_lock);
+	spin_unlock_bh(&ap_queues_lock);
 	return scnprintf(buf, PAGE_SIZE, "%llu\n", req_cnt);
 }
 
@@ -76,13 +76,15 @@ static ssize_t request_count_store(struct device *dev,
 				   struct device_attribute *attr,
 				   const char *buf, size_t count)
 {
-	struct ap_card *ac = to_ap_card(dev);
+	int bkt;
 	struct ap_queue *aq;
+	struct ap_card *ac = to_ap_card(dev);
 
-	spin_lock_bh(&ap_list_lock);
-	for_each_ap_queue(aq, ac)
-		aq->total_request_count = 0;
-	spin_unlock_bh(&ap_list_lock);
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode)
+		if (ac == aq->card)
+			aq->total_request_count = 0;
+	spin_unlock_bh(&ap_queues_lock);
 	atomic64_set(&ac->total_request_count, 0);
 
 	return count;
@@ -93,15 +95,17 @@ static DEVICE_ATTR_RW(request_count);
 static ssize_t requestq_count_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
-	struct ap_card *ac = to_ap_card(dev);
+	int bkt;
 	struct ap_queue *aq;
 	unsigned int reqq_cnt;
+	struct ap_card *ac = to_ap_card(dev);
 
 	reqq_cnt = 0;
-	spin_lock_bh(&ap_list_lock);
-	for_each_ap_queue(aq, ac)
-		reqq_cnt += aq->requestq_count;
-	spin_unlock_bh(&ap_list_lock);
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode)
+		if (ac == aq->card)
+			reqq_cnt += aq->requestq_count;
+	spin_unlock_bh(&ap_queues_lock);
 	return scnprintf(buf, PAGE_SIZE, "%d\n", reqq_cnt);
 }
 
@@ -110,15 +114,17 @@ static DEVICE_ATTR_RO(requestq_count);
 static ssize_t pendingq_count_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
-	struct ap_card *ac = to_ap_card(dev);
+	int bkt;
 	struct ap_queue *aq;
 	unsigned int penq_cnt;
+	struct ap_card *ac = to_ap_card(dev);
 
 	penq_cnt = 0;
-	spin_lock_bh(&ap_list_lock);
-	for_each_ap_queue(aq, ac)
-		penq_cnt += aq->pendingq_count;
-	spin_unlock_bh(&ap_list_lock);
+	spin_lock_bh(&ap_queues_lock);
+	hash_for_each(ap_queues, bkt, aq, hnode)
+		if (ac == aq->card)
+			penq_cnt += aq->pendingq_count;
+	spin_unlock_bh(&ap_queues_lock);
 	return scnprintf(buf, PAGE_SIZE, "%d\n", penq_cnt);
 }
 
@@ -163,11 +169,6 @@ static void ap_card_device_release(struct device *dev)
 {
 	struct ap_card *ac = to_ap_card(dev);
 
-	if (!list_empty(&ac->list)) {
-		spin_lock_bh(&ap_list_lock);
-		list_del_init(&ac->list);
-		spin_unlock_bh(&ap_list_lock);
-	}
 	kfree(ac);
 }
 
@@ -179,8 +180,6 @@ struct ap_card *ap_card_create(int id, int queue_depth, int raw_type,
 	ac = kzalloc(sizeof(*ac), GFP_KERNEL);
 	if (!ac)
 		return NULL;
-	INIT_LIST_HEAD(&ac->list);
-	INIT_LIST_HEAD(&ac->queues);
 	ac->ap_dev.device.release = ap_card_device_release;
 	ac->ap_dev.device.type = &ap_card_type;
 	ac->ap_dev.device_type = comp_type;
diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
index 0eaf1d04e8df..73b077dca3e6 100644
--- a/drivers/s390/crypto/ap_queue.c
+++ b/drivers/s390/crypto/ap_queue.c
@@ -568,11 +568,10 @@ static void ap_queue_device_release(struct device *dev)
 {
 	struct ap_queue *aq = to_ap_queue(dev);
 
-	if (!list_empty(&aq->list)) {
-		spin_lock_bh(&ap_list_lock);
-		list_del_init(&aq->list);
-		spin_unlock_bh(&ap_list_lock);
-	}
+	spin_lock_bh(&ap_queues_lock);
+	hash_del(&aq->hnode);
+	spin_unlock_bh(&ap_queues_lock);
+
 	kfree(aq);
 }
 
@@ -590,7 +589,6 @@ struct ap_queue *ap_queue_create(ap_qid_t qid, int device_type)
 	aq->state = AP_STATE_UNBOUND;
 	aq->interrupt = AP_INTR_DISABLED;
 	spin_lock_init(&aq->lock);
-	INIT_LIST_HEAD(&aq->list);
 	INIT_LIST_HEAD(&aq->pendingq);
 	INIT_LIST_HEAD(&aq->requestq);
 	timer_setup(&aq->timeout, ap_request_timeout, 0);
diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c
index da47e423e1b1..2d3bca3c0141 100644
--- a/drivers/s390/net/qeth_l2_main.c
+++ b/drivers/s390/net/qeth_l2_main.c
@@ -20,6 +20,7 @@
 #include <linux/list.h>
 #include <linux/hash.h>
 #include <linux/hashtable.h>
+#include <asm/chsc.h>
 #include <asm/setup.h>
 #include "qeth_core.h"
 #include "qeth_l2.h"
@@ -27,8 +28,8 @@
 static void qeth_bridgeport_query_support(struct qeth_card *card);
 static void qeth_bridge_state_change(struct qeth_card *card,
 					struct qeth_ipa_cmd *cmd);
-static void qeth_bridge_host_event(struct qeth_card *card,
-					struct qeth_ipa_cmd *cmd);
+static void qeth_addr_change_event(struct qeth_card *card,
+				   struct qeth_ipa_cmd *cmd);
 static void qeth_l2_vnicc_set_defaults(struct qeth_card *card);
 static void qeth_l2_vnicc_init(struct qeth_card *card);
 static bool qeth_l2_vnicc_recover_timeout(struct qeth_card *card, u32 vnicc,
@@ -629,6 +630,72 @@ static void qeth_l2_set_rx_mode(struct net_device *dev)
 	schedule_work(&card->rx_mode_work);
 }
 
+/**
+ *	qeth_l2_pnso() - perform network subchannel operation
+ *	@card: qeth_card structure pointer
+ *	@cnc: Boolean Change-Notification Control
+ *	@cb: Callback function will be executed for each element
+ *		of the address list
+ *	@priv: Pointer to pass to the callback function.
+ *
+ *	Collects network information in a network address list and calls the
+ *	callback function for every entry in the list. If "change-notification-
+ *	control" is set, further changes in the address list will be reported
+ *	via the IPA command.
+ */
+static int qeth_l2_pnso(struct qeth_card *card, int cnc,
+			void (*cb)(void *priv, struct chsc_pnso_naid_l2 *entry),
+			void *priv)
+{
+	struct ccw_device *ddev = CARD_DDEV(card);
+	struct chsc_pnso_area *rr;
+	u32 prev_instance = 0;
+	int isfirstblock = 1;
+	int i, size, elems;
+	int rc;
+
+	QETH_CARD_TEXT(card, 2, "PNSO");
+	rr = (struct chsc_pnso_area *)get_zeroed_page(GFP_KERNEL);
+	if (rr == NULL)
+		return -ENOMEM;
+	do {
+		/* on the first iteration, naihdr.resume_token will be zero */
+		rc = ccw_device_pnso(ddev, rr, rr->naihdr.resume_token, cnc);
+		if (rc)
+			continue;
+		if (cb == NULL)
+			continue;
+
+		size = rr->naihdr.naids;
+		if (size != sizeof(struct chsc_pnso_naid_l2)) {
+			WARN_ON_ONCE(1);
+			continue;
+		}
+
+		elems = (rr->response.length - sizeof(struct chsc_header) -
+			 sizeof(struct chsc_pnso_naihdr)) / size;
+
+		if (!isfirstblock && (rr->naihdr.instance != prev_instance)) {
+			/* Inform the caller that they need to scrap */
+			/* the data that was already reported via cb */
+			rc = -EAGAIN;
+			break;
+		}
+		isfirstblock = 0;
+		prev_instance = rr->naihdr.instance;
+		for (i = 0; i < elems; i++)
+			(*cb)(priv, &rr->entries[i]);
+	} while ((rc == -EBUSY) || (!rc && /* list stored */
+		   /* resume token is non-zero => list incomplete */
+		   (rr->naihdr.resume_token.t1 || rr->naihdr.resume_token.t2)));
+
+	if (rc)
+		QETH_CARD_TEXT_(card, 2, "PNrp%04x", rr->response.code);
+
+	free_page((unsigned long)rr);
+	return rc;
+}
+
 static const struct net_device_ops qeth_l2_netdev_ops = {
 	.ndo_open		= qeth_open,
 	.ndo_stop		= qeth_stop,
@@ -856,7 +923,7 @@ static int qeth_l2_control_event(struct qeth_card *card,
 		} else
 			return 1;
 	case IPA_CMD_ADDRESS_CHANGE_NOTIF:
-		qeth_bridge_host_event(card, cmd);
+		qeth_addr_change_event(card, cmd);
 		return 0;
 	default:
 		return 1;
@@ -973,8 +1040,10 @@ enum qeth_an_event_type {anev_reg_unreg, anev_abort, anev_reset};
  * for all currently registered addresses.
  */
 static void qeth_bridge_emit_host_event(struct qeth_card *card,
-	enum qeth_an_event_type evtype,
-	u8 code, struct net_if_token *token, struct mac_addr_lnid *addr_lnid)
+					enum qeth_an_event_type evtype,
+					u8 code,
+					struct net_if_token *token,
+					struct mac_addr_lnid *addr_lnid)
 {
 	char str[7][32];
 	char *env[8];
@@ -1091,74 +1160,76 @@ static void qeth_bridge_state_change(struct qeth_card *card,
 	queue_work(card->event_wq, &data->worker);
 }
 
-struct qeth_bridge_host_data {
+struct qeth_addr_change_data {
 	struct work_struct worker;
 	struct qeth_card *card;
-	struct qeth_ipacmd_addr_change hostevs;
+	struct qeth_ipacmd_addr_change ac_event;
 };
 
-static void qeth_bridge_host_event_worker(struct work_struct *work)
+static void qeth_addr_change_event_worker(struct work_struct *work)
 {
-	struct qeth_bridge_host_data *data =
-		container_of(work, struct qeth_bridge_host_data, worker);
+	struct qeth_addr_change_data *data =
+		container_of(work, struct qeth_addr_change_data, worker);
 	int i;
 
-	if (data->hostevs.lost_event_mask) {
+	QETH_CARD_TEXT(data->card, 4, "adrchgew");
+	if (data->ac_event.lost_event_mask) {
 		dev_info(&data->card->gdev->dev,
-"Address notification from the Bridge Port stopped %s (%s)\n",
-			data->card->dev->name,
-			(data->hostevs.lost_event_mask == 0x01)
+			 "Address change notification stopped on %s (%s)\n",
+			 data->card->dev->name,
+			(data->ac_event.lost_event_mask == 0x01)
 			? "Overflow"
-			: (data->hostevs.lost_event_mask == 0x02)
+			: (data->ac_event.lost_event_mask == 0x02)
 			? "Bridge port state change"
 			: "Unknown reason");
 		mutex_lock(&data->card->sbp_lock);
 		data->card->options.sbp.hostnotification = 0;
 		mutex_unlock(&data->card->sbp_lock);
 		qeth_bridge_emit_host_event(data->card, anev_abort,
-			0, NULL, NULL);
+					    0, NULL, NULL);
 	} else
-		for (i = 0; i < data->hostevs.num_entries; i++) {
+		for (i = 0; i < data->ac_event.num_entries; i++) {
 			struct qeth_ipacmd_addr_change_entry *entry =
-					&data->hostevs.entry[i];
+					&data->ac_event.entry[i];
 			qeth_bridge_emit_host_event(data->card,
-					anev_reg_unreg,
-					entry->change_code,
-					&entry->token, &entry->addr_lnid);
+						    anev_reg_unreg,
+						    entry->change_code,
+						    &entry->token,
+						    &entry->addr_lnid);
 		}
 	kfree(data);
 }
 
-static void qeth_bridge_host_event(struct qeth_card *card,
-					struct qeth_ipa_cmd *cmd)
+static void qeth_addr_change_event(struct qeth_card *card,
+				   struct qeth_ipa_cmd *cmd)
 {
 	struct qeth_ipacmd_addr_change *hostevs =
 		 &cmd->data.addrchange;
-	struct qeth_bridge_host_data *data;
+	struct qeth_addr_change_data *data;
 	int extrasize;
 
-	QETH_CARD_TEXT(card, 2, "brhostev");
+	QETH_CARD_TEXT(card, 4, "adrchgev");
 	if (cmd->hdr.return_code != 0x0000) {
 		if (cmd->hdr.return_code == 0x0010) {
 			if (hostevs->lost_event_mask == 0x00)
 				hostevs->lost_event_mask = 0xff;
 		} else {
-			QETH_CARD_TEXT_(card, 2, "BPHe%04x",
+			QETH_CARD_TEXT_(card, 2, "ACHN%04x",
 				cmd->hdr.return_code);
 			return;
 		}
 	}
 	extrasize = sizeof(struct qeth_ipacmd_addr_change_entry) *
 						hostevs->num_entries;
-	data = kzalloc(sizeof(struct qeth_bridge_host_data) + extrasize,
-		GFP_ATOMIC);
+	data = kzalloc(sizeof(struct qeth_addr_change_data) + extrasize,
+		       GFP_ATOMIC);
 	if (!data) {
-		QETH_CARD_TEXT(card, 2, "BPHalloc");
+		QETH_CARD_TEXT(card, 2, "ACNalloc");
 		return;
 	}
-	INIT_WORK(&data->worker, qeth_bridge_host_event_worker);
+	INIT_WORK(&data->worker, qeth_addr_change_event_worker);
 	data->card = card;
-	memcpy(&data->hostevs, hostevs,
+	memcpy(&data->ac_event, hostevs,
 			sizeof(struct qeth_ipacmd_addr_change) + extrasize);
 	queue_work(card->event_wq, &data->worker);
 }
@@ -1448,63 +1519,18 @@ int qeth_bridgeport_setrole(struct qeth_card *card, enum qeth_sbp_roles role)
 	return qeth_send_ipa_cmd(card, iob, qeth_bridgeport_set_cb, NULL);
 }
 
-/**
- * qeth_anset_makerc() - derive "traditional" error from hardware codes.
- * @card:		      qeth_card structure pointer, for debug messages.
- *
- * Returns negative errno-compatible error indication or 0 on success.
- */
-static int qeth_anset_makerc(struct qeth_card *card, int pnso_rc, u16 response)
-{
-	int rc;
-
-	if (pnso_rc == 0)
-		switch (response) {
-		case 0x0001:
-			rc = 0;
-			break;
-		case 0x0004:
-		case 0x0100:
-		case 0x0106:
-			rc = -EOPNOTSUPP;
-			dev_err(&card->gdev->dev,
-				"Setting address notification failed\n");
-			break;
-		case 0x0107:
-			rc = -EAGAIN;
-			break;
-		default:
-			rc = -EIO;
-		}
-	else
-		rc = -EIO;
-
-	if (rc) {
-		QETH_CARD_TEXT_(card, 2, "SBPp%04x", pnso_rc);
-		QETH_CARD_TEXT_(card, 2, "SBPr%04x", response);
-	}
-	return rc;
-}
-
 static void qeth_bridgeport_an_set_cb(void *priv,
-		enum qdio_brinfo_entry_type type, void *entry)
+				      struct chsc_pnso_naid_l2 *entry)
 {
 	struct qeth_card *card = (struct qeth_card *)priv;
-	struct qdio_brinfo_entry_l2 *l2entry;
 	u8 code;
 
-	if (type != l2_addr_lnid) {
-		WARN_ON_ONCE(1);
-		return;
-	}
-
-	l2entry = (struct qdio_brinfo_entry_l2 *)entry;
 	code = IPA_ADDR_CHANGE_CODE_MACADDR;
-	if (l2entry->addr_lnid.lnid < VLAN_N_VID)
+	if (entry->addr_lnid.lnid < VLAN_N_VID)
 		code |= IPA_ADDR_CHANGE_CODE_VLANID;
 	qeth_bridge_emit_host_event(card, anev_reg_unreg, code,
-		(struct net_if_token *)&l2entry->nit,
-		(struct mac_addr_lnid *)&l2entry->addr_lnid);
+				    (struct net_if_token *)&entry->nit,
+				    (struct mac_addr_lnid *)&entry->addr_lnid);
 }
 
 /**
@@ -1520,22 +1546,16 @@ static void qeth_bridgeport_an_set_cb(void *priv,
 int qeth_bridgeport_an_set(struct qeth_card *card, int enable)
 {
 	int rc;
-	u16 response;
-	struct ccw_device *ddev;
-	struct subchannel_id schid;
 
 	if (!card->options.sbp.supported_funcs)
 		return -EOPNOTSUPP;
-	ddev = CARD_DDEV(card);
-	ccw_device_get_schid(ddev, &schid);
 
 	if (enable) {
 		qeth_bridge_emit_host_event(card, anev_reset, 0, NULL, NULL);
-		rc = qdio_pnso_brinfo(schid, 1, &response,
-			qeth_bridgeport_an_set_cb, card);
+		rc = qeth_l2_pnso(card, 1, qeth_bridgeport_an_set_cb, card);
 	} else
-		rc = qdio_pnso_brinfo(schid, 0, &response, NULL, NULL);
-	return qeth_anset_makerc(card, rc, response);
+		rc = qeth_l2_pnso(card, 0, NULL, NULL);
+	return rc;
 }
 
 static bool qeth_bridgeport_is_in_use(struct qeth_card *card)
diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c
index 09ec846fe01d..18b713a616de 100644
--- a/drivers/s390/scsi/zfcp_aux.c
+++ b/drivers/s390/scsi/zfcp_aux.c
@@ -4,7 +4,7 @@
  *
  * Module interface and handling of zfcp data structures.
  *
- * Copyright IBM Corp. 2002, 2018
+ * Copyright IBM Corp. 2002, 2020
  */
 
 /*
@@ -415,8 +415,7 @@ struct zfcp_adapter *zfcp_adapter_enqueue(struct ccw_device *ccw_device)
 
 	adapter->stat_read_buf_num = FSF_STATUS_READS_RECOM;
 
-	if (!zfcp_scsi_adapter_register(adapter))
-		return adapter;
+	return adapter;
 
 failed:
 	zfcp_adapter_unregister(adapter);
diff --git a/drivers/s390/scsi/zfcp_diag.h b/drivers/s390/scsi/zfcp_diag.h
index b9c93d15f67c..3852367f15f6 100644
--- a/drivers/s390/scsi/zfcp_diag.h
+++ b/drivers/s390/scsi/zfcp_diag.h
@@ -4,7 +4,7 @@
  *
  * Definitions for handling diagnostics in the the zfcp device driver.
  *
- * Copyright IBM Corp. 2018
+ * Copyright IBM Corp. 2018, 2020
  */
 
 #ifndef ZFCP_DIAG_H
@@ -56,11 +56,11 @@ struct zfcp_diag_adapter {
 
 	unsigned long	max_age;
 
-	struct {
+	struct zfcp_diag_adapter_port_data {
 		struct zfcp_diag_header		header;
 		struct fsf_qtcb_bottom_port	data;
 	} port_data;
-	struct {
+	struct zfcp_diag_adapter_config_data {
 		struct zfcp_diag_header		header;
 		struct fsf_qtcb_bottom_config	data;
 	} config_data;
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 3d0bc000f500..db320dab1fee 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -4,7 +4,7 @@
  *
  * Error Recovery Procedures (ERP).
  *
- * Copyright IBM Corp. 2002, 2017
+ * Copyright IBM Corp. 2002, 2020
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -14,6 +14,7 @@
 #include <linux/bug.h>
 #include "zfcp_ext.h"
 #include "zfcp_reqlist.h"
+#include "zfcp_diag.h"
 
 #define ZFCP_MAX_ERPS                   3
 
@@ -768,10 +769,14 @@ static enum zfcp_erp_act_result zfcp_erp_adapter_strat_fsf_xconf(
 	if (!(atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_XCONFIG_OK))
 		return ZFCP_ERP_FAILED;
 
+	return ZFCP_ERP_SUCCEEDED;
+}
+
+static void
+zfcp_erp_adapter_strategy_open_ptp_port(struct zfcp_adapter *const adapter)
+{
 	if (fc_host_port_type(adapter->scsi_host) == FC_PORTTYPE_PTP)
 		zfcp_erp_enqueue_ptp_port(adapter);
-
-	return ZFCP_ERP_SUCCEEDED;
 }
 
 static enum zfcp_erp_act_result zfcp_erp_adapter_strategy_open_fsf_xport(
@@ -800,6 +805,59 @@ static enum zfcp_erp_act_result zfcp_erp_adapter_strategy_open_fsf_xport(
 	return ZFCP_ERP_SUCCEEDED;
 }
 
+static enum zfcp_erp_act_result
+zfcp_erp_adapter_strategy_alloc_shost(struct zfcp_adapter *const adapter)
+{
+	struct zfcp_diag_adapter_config_data *const config_data =
+		&adapter->diagnostics->config_data;
+	struct zfcp_diag_adapter_port_data *const port_data =
+		&adapter->diagnostics->port_data;
+	unsigned long flags;
+	int rc;
+
+	rc = zfcp_scsi_adapter_register(adapter);
+	if (rc == -EEXIST)
+		return ZFCP_ERP_SUCCEEDED;
+	else if (rc)
+		return ZFCP_ERP_FAILED;
+
+	/*
+	 * We allocated the shost for the first time. Before it was NULL,
+	 * and so we deferred all updates in the xconf- and xport-data
+	 * handlers. We need to make up for that now, and make all the updates
+	 * that would have been done before.
+	 *
+	 * We can be sure that xconf- and xport-data succeeded, because
+	 * otherwise this function is not called. But they might have been
+	 * incomplete.
+	 */
+
+	spin_lock_irqsave(&config_data->header.access_lock, flags);
+	zfcp_scsi_shost_update_config_data(adapter, &config_data->data,
+					   !!config_data->header.incomplete);
+	spin_unlock_irqrestore(&config_data->header.access_lock, flags);
+
+	if (adapter->adapter_features & FSF_FEATURE_HBAAPI_MANAGEMENT) {
+		spin_lock_irqsave(&port_data->header.access_lock, flags);
+		zfcp_scsi_shost_update_port_data(adapter, &port_data->data);
+		spin_unlock_irqrestore(&port_data->header.access_lock, flags);
+	}
+
+	/*
+	 * There is a remote possibility that the 'Exchange Port Data' request
+	 * reports a different connectivity status than 'Exchange Config Data'.
+	 * But any change to the connectivity status of the local optic that
+	 * happens after the initial xconf request is expected to be reported
+	 * to us, as soon as we post Status Read Buffers to the FCP channel
+	 * firmware after this function. So any resulting inconsistency will
+	 * only be momentary.
+	 */
+	if (config_data->header.incomplete)
+		zfcp_fsf_fc_host_link_down(adapter);
+
+	return ZFCP_ERP_SUCCEEDED;
+}
+
 static enum zfcp_erp_act_result zfcp_erp_adapter_strategy_open_fsf(
 	struct zfcp_erp_action *act)
 {
@@ -809,6 +867,12 @@ static enum zfcp_erp_act_result zfcp_erp_adapter_strategy_open_fsf(
 	if (zfcp_erp_adapter_strategy_open_fsf_xport(act) == ZFCP_ERP_FAILED)
 		return ZFCP_ERP_FAILED;
 
+	if (zfcp_erp_adapter_strategy_alloc_shost(act->adapter) ==
+	    ZFCP_ERP_FAILED)
+		return ZFCP_ERP_FAILED;
+
+	zfcp_erp_adapter_strategy_open_ptp_port(act->adapter);
+
 	if (mempool_resize(act->adapter->pool.sr_data,
 			   act->adapter->stat_read_buf_num))
 		return ZFCP_ERP_FAILED;
@@ -1636,6 +1700,13 @@ void zfcp_erp_set_adapter_status(struct zfcp_adapter *adapter, u32 mask)
 		atomic_or(common_mask, &port->status);
 	read_unlock_irqrestore(&adapter->port_list_lock, flags);
 
+	/*
+	 * if `scsi_host` is missing, xconfig/xport data has never completed
+	 * yet, so we can't access it, but there are also no SDEVs yet
+	 */
+	if (adapter->scsi_host == NULL)
+		return;
+
 	spin_lock_irqsave(adapter->scsi_host->host_lock, flags);
 	__shost_for_each_device(sdev, adapter->scsi_host)
 		atomic_or(common_mask, &sdev_to_zfcp(sdev)->status);
@@ -1673,6 +1744,13 @@ void zfcp_erp_clear_adapter_status(struct zfcp_adapter *adapter, u32 mask)
 	}
 	read_unlock_irqrestore(&adapter->port_list_lock, flags);
 
+	/*
+	 * if `scsi_host` is missing, xconfig/xport data has never completed
+	 * yet, so we can't access it, but there are also no SDEVs yet
+	 */
+	if (adapter->scsi_host == NULL)
+		return;
+
 	spin_lock_irqsave(adapter->scsi_host->host_lock, flags);
 	__shost_for_each_device(sdev, adapter->scsi_host) {
 		atomic_andnot(common_mask, &sdev_to_zfcp(sdev)->status);
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index 88294ca0e2ea..3ef5d74331c3 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -125,6 +125,7 @@ extern int zfcp_fsf_exchange_config_data_sync(struct zfcp_qdio *,
 extern int zfcp_fsf_exchange_port_data(struct zfcp_erp_action *);
 extern int zfcp_fsf_exchange_port_data_sync(struct zfcp_qdio *,
 					    struct fsf_qtcb_bottom_port *);
+extern u32 zfcp_fsf_convert_portspeed(u32 fsf_speed);
 extern void zfcp_fsf_req_dismiss_all(struct zfcp_adapter *);
 extern int zfcp_fsf_status_read(struct zfcp_qdio *);
 extern int zfcp_status_read_refill(struct zfcp_adapter *adapter);
@@ -134,6 +135,7 @@ extern int zfcp_fsf_send_els(struct zfcp_adapter *, u32,
 			     struct zfcp_fsf_ct_els *, unsigned int);
 extern int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *);
 extern void zfcp_fsf_req_free(struct zfcp_fsf_req *);
+extern void zfcp_fsf_fc_host_link_down(struct zfcp_adapter *adapter);
 extern struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_device *sdev,
 						   u8 tm_flags);
 extern struct zfcp_fsf_req *zfcp_fsf_abort_fcp_cmnd(struct scsi_cmnd *);
@@ -153,6 +155,8 @@ extern int zfcp_qdio_sbal_get(struct zfcp_qdio *);
 extern int zfcp_qdio_send(struct zfcp_qdio *, struct zfcp_qdio_req *);
 extern int zfcp_qdio_sbals_from_sg(struct zfcp_qdio *, struct zfcp_qdio_req *,
 				   struct scatterlist *);
+extern void zfcp_qdio_shost_update(struct zfcp_adapter *const adapter,
+				   const struct zfcp_qdio *const qdio);
 extern int zfcp_qdio_open(struct zfcp_qdio *);
 extern void zfcp_qdio_close(struct zfcp_qdio *);
 extern void zfcp_qdio_siosl(struct zfcp_adapter *);
@@ -169,6 +173,13 @@ extern void zfcp_scsi_schedule_rport_block(struct zfcp_port *);
 extern void zfcp_scsi_schedule_rports_block(struct zfcp_adapter *);
 extern void zfcp_scsi_set_prot(struct zfcp_adapter *);
 extern void zfcp_scsi_dif_sense_error(struct scsi_cmnd *, int);
+extern void zfcp_scsi_shost_update_config_data(
+	struct zfcp_adapter *const adapter,
+	const struct fsf_qtcb_bottom_config *const bottom,
+	const bool bottom_incomplete);
+extern void zfcp_scsi_shost_update_port_data(
+	struct zfcp_adapter *const adapter,
+	const struct fsf_qtcb_bottom_port *const bottom);
 
 /* zfcp_sysfs.c */
 extern const struct attribute_group *zfcp_unit_attr_groups[];
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index 111fe3fc32d7..c795f22249d8 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -120,21 +120,25 @@ static void zfcp_fsf_status_read_port_closed(struct zfcp_fsf_req *req)
 	read_unlock_irqrestore(&adapter->port_list_lock, flags);
 }
 
-static void zfcp_fsf_fc_host_link_down(struct zfcp_adapter *adapter)
+void zfcp_fsf_fc_host_link_down(struct zfcp_adapter *adapter)
 {
 	struct Scsi_Host *shost = adapter->scsi_host;
 
+	adapter->hydra_version = 0;
+	adapter->peer_wwpn = 0;
+	adapter->peer_wwnn = 0;
+	adapter->peer_d_id = 0;
+
+	/* if there is no shost yet, we have nothing to zero-out */
+	if (shost == NULL)
+		return;
+
 	fc_host_port_id(shost) = 0;
 	fc_host_fabric_name(shost) = 0;
 	fc_host_speed(shost) = FC_PORTSPEED_UNKNOWN;
 	fc_host_port_type(shost) = FC_PORTTYPE_UNKNOWN;
-	adapter->hydra_version = 0;
 	snprintf(fc_host_model(shost), FC_SYMBOLIC_NAME_SIZE, "0x%04x", 0);
 	memset(fc_host_active_fc4s(shost), 0, FC_FC4_LIST_SIZE);
-
-	adapter->peer_wwpn = 0;
-	adapter->peer_wwnn = 0;
-	adapter->peer_d_id = 0;
 }
 
 static void zfcp_fsf_link_down_info_eval(struct zfcp_fsf_req *req,
@@ -479,7 +483,7 @@ void zfcp_fsf_req_dismiss_all(struct zfcp_adapter *adapter)
 #define ZFCP_FSF_PORTSPEED_128GBIT	(1 <<  8)
 #define ZFCP_FSF_PORTSPEED_NOT_NEGOTIATED (1 << 15)
 
-static u32 zfcp_fsf_convert_portspeed(u32 fsf_speed)
+u32 zfcp_fsf_convert_portspeed(u32 fsf_speed)
 {
 	u32 fdmi_speed = 0;
 	if (fsf_speed & ZFCP_FSF_PORTSPEED_1GBIT)
@@ -509,64 +513,36 @@ static int zfcp_fsf_exchange_config_evaluate(struct zfcp_fsf_req *req)
 {
 	struct fsf_qtcb_bottom_config *bottom = &req->qtcb->bottom.config;
 	struct zfcp_adapter *adapter = req->adapter;
-	struct Scsi_Host *shost = adapter->scsi_host;
-	struct fc_els_flogi *nsp, *plogi;
+	struct fc_els_flogi *plogi;
 
 	/* adjust pointers for missing command code */
-	nsp = (struct fc_els_flogi *) ((u8 *)&bottom->nport_serv_param
-					- sizeof(u32));
 	plogi = (struct fc_els_flogi *) ((u8 *)&bottom->plogi_payload
 					- sizeof(u32));
 
 	if (req->data)
 		memcpy(req->data, bottom, sizeof(*bottom));
 
-	snprintf(fc_host_manufacturer(shost), FC_SERIAL_NUMBER_SIZE, "%s",
-		 "IBM");
-	fc_host_port_name(shost) = be64_to_cpu(nsp->fl_wwpn);
-	fc_host_node_name(shost) = be64_to_cpu(nsp->fl_wwnn);
-	fc_host_supported_classes(shost) = FC_COS_CLASS2 | FC_COS_CLASS3;
-
 	adapter->timer_ticks = bottom->timer_interval & ZFCP_FSF_TIMER_INT_MASK;
 	adapter->stat_read_buf_num = max(bottom->status_read_buf_num,
 					 (u16)FSF_STATUS_READS_RECOM);
 
-	zfcp_scsi_set_prot(adapter);
-
 	/* no error return above here, otherwise must fix call chains */
 	/* do not evaluate invalid fields */
 	if (req->qtcb->header.fsf_status == FSF_EXCHANGE_CONFIG_DATA_INCOMPLETE)
 		return 0;
 
-	fc_host_port_id(shost) = ntoh24(bottom->s_id);
-	fc_host_speed(shost) =
-		zfcp_fsf_convert_portspeed(bottom->fc_link_speed);
-
 	adapter->hydra_version = bottom->adapter_type;
-	snprintf(fc_host_model(shost), FC_SYMBOLIC_NAME_SIZE, "0x%04x",
-		 bottom->adapter_type);
 
 	switch (bottom->fc_topology) {
 	case FSF_TOPO_P2P:
 		adapter->peer_d_id = ntoh24(bottom->peer_d_id);
 		adapter->peer_wwpn = be64_to_cpu(plogi->fl_wwpn);
 		adapter->peer_wwnn = be64_to_cpu(plogi->fl_wwnn);
-		fc_host_port_type(shost) = FC_PORTTYPE_PTP;
-		fc_host_fabric_name(shost) = 0;
 		break;
 	case FSF_TOPO_FABRIC:
-		fc_host_fabric_name(shost) = be64_to_cpu(plogi->fl_wwnn);
-		if (bottom->connection_features & FSF_FEATURE_NPIV_MODE)
-			fc_host_port_type(shost) = FC_PORTTYPE_NPIV;
-		else
-			fc_host_port_type(shost) = FC_PORTTYPE_NPORT;
 		break;
 	case FSF_TOPO_AL:
-		fc_host_port_type(shost) = FC_PORTTYPE_NLPORT;
-		fc_host_fabric_name(shost) = 0;
-		fallthrough;
 	default:
-		fc_host_fabric_name(shost) = 0;
 		dev_err(&adapter->ccw_device->dev,
 			"Unknown or unsupported arbitrated loop "
 			"fibre channel topology detected\n");
@@ -584,13 +560,10 @@ static void zfcp_fsf_exchange_config_data_handler(struct zfcp_fsf_req *req)
 		&adapter->diagnostics->config_data.header;
 	struct fsf_qtcb *qtcb = req->qtcb;
 	struct fsf_qtcb_bottom_config *bottom = &qtcb->bottom.config;
-	struct Scsi_Host *shost = adapter->scsi_host;
 
 	if (req->status & ZFCP_STATUS_FSFREQ_ERROR)
 		return;
 
-	snprintf(fc_host_firmware_version(shost), FC_VERSION_STRING_SIZE,
-		 "0x%08x", bottom->lic_version);
 	adapter->fsf_lic_version = bottom->lic_version;
 	adapter->adapter_features = bottom->adapter_features;
 	adapter->connection_features = bottom->connection_features;
@@ -606,6 +579,7 @@ static void zfcp_fsf_exchange_config_data_handler(struct zfcp_fsf_req *req)
 		 */
 		zfcp_diag_update_xdata(diag_hdr, bottom, false);
 
+		zfcp_scsi_shost_update_config_data(adapter, bottom, false);
 		if (zfcp_fsf_exchange_config_evaluate(req))
 			return;
 
@@ -630,6 +604,8 @@ static void zfcp_fsf_exchange_config_data_handler(struct zfcp_fsf_req *req)
 				&adapter->status);
 		zfcp_fsf_link_down_info_eval(req,
 			&qtcb->header.fsf_status_qual.link_down_info);
+
+		zfcp_scsi_shost_update_config_data(adapter, bottom, true);
 		if (zfcp_fsf_exchange_config_evaluate(req))
 			return;
 		break;
@@ -638,16 +614,8 @@ static void zfcp_fsf_exchange_config_data_handler(struct zfcp_fsf_req *req)
 		return;
 	}
 
-	if (adapter->adapter_features & FSF_FEATURE_HBAAPI_MANAGEMENT) {
+	if (adapter->adapter_features & FSF_FEATURE_HBAAPI_MANAGEMENT)
 		adapter->hardware_version = bottom->hardware_version;
-		snprintf(fc_host_hardware_version(shost),
-			 FC_VERSION_STRING_SIZE,
-			 "0x%08x", bottom->hardware_version);
-		memcpy(fc_host_serial_number(shost), bottom->serial_number,
-		       min(FC_SERIAL_NUMBER_SIZE, 17));
-		EBCASC(fc_host_serial_number(shost),
-		       min(FC_SERIAL_NUMBER_SIZE, 17));
-	}
 
 	if (FSF_QTCB_CURRENT_VERSION < bottom->low_qtcb_version) {
 		dev_err(&adapter->ccw_device->dev,
@@ -761,19 +729,10 @@ static void zfcp_fsf_exchange_port_evaluate(struct zfcp_fsf_req *req)
 {
 	struct zfcp_adapter *adapter = req->adapter;
 	struct fsf_qtcb_bottom_port *bottom = &req->qtcb->bottom.port;
-	struct Scsi_Host *shost = adapter->scsi_host;
 
 	if (req->data)
 		memcpy(req->data, bottom, sizeof(*bottom));
 
-	fc_host_permanent_port_name(shost) = bottom->wwpn;
-	fc_host_maxframe_size(shost) = bottom->maximum_frame_size;
-	fc_host_supported_speeds(shost) =
-		zfcp_fsf_convert_portspeed(bottom->supported_speed);
-	memcpy(fc_host_supported_fc4s(shost), bottom->supported_fc4_types,
-	       FC_FC4_LIST_SIZE);
-	memcpy(fc_host_active_fc4s(shost), bottom->active_fc4_types,
-	       FC_FC4_LIST_SIZE);
 	if (adapter->adapter_features & FSF_FEATURE_FC_SECURITY)
 		adapter->fc_security_algorithms =
 			bottom->fc_security_algorithms;
@@ -800,6 +759,7 @@ static void zfcp_fsf_exchange_port_data_handler(struct zfcp_fsf_req *req)
 		 */
 		zfcp_diag_update_xdata(diag_hdr, bottom, false);
 
+		zfcp_scsi_shost_update_port_data(req->adapter, bottom);
 		zfcp_fsf_exchange_port_evaluate(req);
 		break;
 	case FSF_EXCHANGE_CONFIG_DATA_INCOMPLETE:
@@ -808,6 +768,8 @@ static void zfcp_fsf_exchange_port_data_handler(struct zfcp_fsf_req *req)
 
 		zfcp_fsf_link_down_info_eval(req,
 			&qtcb->header.fsf_status_qual.link_down_info);
+
+		zfcp_scsi_shost_update_port_data(req->adapter, bottom);
 		zfcp_fsf_exchange_port_evaluate(req);
 		break;
 	}
diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c
index 26702b56a7ab..3a7f3374d10a 100644
--- a/drivers/s390/scsi/zfcp_qdio.c
+++ b/drivers/s390/scsi/zfcp_qdio.c
@@ -4,7 +4,7 @@
  *
  * Setup and helper functions to access QDIO.
  *
- * Copyright IBM Corp. 2002, 2017
+ * Copyright IBM Corp. 2002, 2020
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -342,6 +342,18 @@ void zfcp_qdio_close(struct zfcp_qdio *qdio)
 	atomic_set(&qdio->req_q_free, 0);
 }
 
+void zfcp_qdio_shost_update(struct zfcp_adapter *const adapter,
+			    const struct zfcp_qdio *const qdio)
+{
+	struct Scsi_Host *const shost = adapter->scsi_host;
+
+	if (shost == NULL)
+		return;
+
+	shost->sg_tablesize = qdio->max_sbale_per_req;
+	shost->max_sectors = qdio->max_sbale_per_req * 8;
+}
+
 /**
  * zfcp_qdio_open - prepare and initialize response queue
  * @qdio: pointer to struct zfcp_qdio
@@ -420,10 +432,7 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio)
 	atomic_set(&qdio->req_q_free, QDIO_MAX_BUFFERS_PER_Q);
 	atomic_or(ZFCP_STATUS_ADAPTER_QDIOUP, &qdio->adapter->status);
 
-	if (adapter->scsi_host) {
-		adapter->scsi_host->sg_tablesize = qdio->max_sbale_per_req;
-		adapter->scsi_host->max_sectors = qdio->max_sbale_per_req * 8;
-	}
+	zfcp_qdio_shost_update(adapter, qdio);
 
 	return 0;
 
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index 13d873f806e4..d58bf79892f2 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -4,7 +4,7 @@
  *
  * Interface to Linux SCSI midlayer.
  *
- * Copyright IBM Corp. 2002, 2018
+ * Copyright IBM Corp. 2002, 2020
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -451,26 +451,39 @@ static struct scsi_host_template zfcp_scsi_host_template = {
 };
 
 /**
- * zfcp_scsi_adapter_register - Register SCSI and FC host with SCSI midlayer
+ * zfcp_scsi_adapter_register() - Allocate and register SCSI and FC host with
+ *				  SCSI midlayer
  * @adapter: The zfcp adapter to register with the SCSI midlayer
+ *
+ * Allocates the SCSI host object for the given adapter, sets basic properties
+ * (such as the transport template, QDIO limits, ...), and registers it with
+ * the midlayer.
+ *
+ * During registration with the midlayer the corresponding FC host object for
+ * the referenced transport class is also implicitely allocated.
+ *
+ * Upon success adapter->scsi_host is set, and upon failure it remains NULL. If
+ * adapter->scsi_host is already set, nothing is done.
+ *
+ * Return:
+ * * 0	     - Allocation and registration was successful
+ * * -EEXIST - SCSI and FC host did already exist, nothing was done, nothing
+ *	       was changed
+ * * -EIO    - Allocation or registration failed
  */
 int zfcp_scsi_adapter_register(struct zfcp_adapter *adapter)
 {
 	struct ccw_dev_id dev_id;
 
 	if (adapter->scsi_host)
-		return 0;
+		return -EEXIST;
 
 	ccw_device_get_id(adapter->ccw_device, &dev_id);
 	/* register adapter as SCSI host with mid layer of SCSI stack */
 	adapter->scsi_host = scsi_host_alloc(&zfcp_scsi_host_template,
 					     sizeof (struct zfcp_adapter *));
-	if (!adapter->scsi_host) {
-		dev_err(&adapter->ccw_device->dev,
-			"Registering the FCP device with the "
-			"SCSI stack failed\n");
-		return -EIO;
-	}
+	if (!adapter->scsi_host)
+		goto err_out;
 
 	/* tell the SCSI stack some characteristics of this adapter */
 	adapter->scsi_host->max_id = 511;
@@ -480,14 +493,23 @@ int zfcp_scsi_adapter_register(struct zfcp_adapter *adapter)
 	adapter->scsi_host->max_cmd_len = 16; /* in struct fcp_cmnd */
 	adapter->scsi_host->transportt = zfcp_scsi_transport_template;
 
+	/* make all basic properties known at registration time */
+	zfcp_qdio_shost_update(adapter, adapter->qdio);
+	zfcp_scsi_set_prot(adapter);
+
 	adapter->scsi_host->hostdata[0] = (unsigned long) adapter;
 
 	if (scsi_add_host(adapter->scsi_host, &adapter->ccw_device->dev)) {
 		scsi_host_put(adapter->scsi_host);
-		return -EIO;
+		goto err_out;
 	}
 
 	return 0;
+err_out:
+	adapter->scsi_host = NULL;
+	dev_err(&adapter->ccw_device->dev,
+		"Registering the FCP device with the SCSI stack failed\n");
+	return -EIO;
 }
 
 /**
@@ -841,6 +863,95 @@ void zfcp_scsi_dif_sense_error(struct scsi_cmnd *scmd, int ascq)
 	set_host_byte(scmd, DID_SOFT_ERROR);
 }
 
+void zfcp_scsi_shost_update_config_data(
+	struct zfcp_adapter *const adapter,
+	const struct fsf_qtcb_bottom_config *const bottom,
+	const bool bottom_incomplete)
+{
+	struct Scsi_Host *const shost = adapter->scsi_host;
+	const struct fc_els_flogi *nsp, *plogi;
+
+	if (shost == NULL)
+		return;
+
+	snprintf(fc_host_firmware_version(shost), FC_VERSION_STRING_SIZE,
+		 "0x%08x", bottom->lic_version);
+
+	if (adapter->adapter_features & FSF_FEATURE_HBAAPI_MANAGEMENT) {
+		snprintf(fc_host_hardware_version(shost),
+			 FC_VERSION_STRING_SIZE,
+			 "0x%08x", bottom->hardware_version);
+		memcpy(fc_host_serial_number(shost), bottom->serial_number,
+		       min(FC_SERIAL_NUMBER_SIZE, 17));
+		EBCASC(fc_host_serial_number(shost),
+		       min(FC_SERIAL_NUMBER_SIZE, 17));
+	}
+
+	/* adjust pointers for missing command code */
+	nsp = (struct fc_els_flogi *) ((u8 *)&bottom->nport_serv_param
+					- sizeof(u32));
+	plogi = (struct fc_els_flogi *) ((u8 *)&bottom->plogi_payload
+					- sizeof(u32));
+
+	snprintf(fc_host_manufacturer(shost), FC_SERIAL_NUMBER_SIZE, "%s",
+		 "IBM");
+	fc_host_port_name(shost) = be64_to_cpu(nsp->fl_wwpn);
+	fc_host_node_name(shost) = be64_to_cpu(nsp->fl_wwnn);
+	fc_host_supported_classes(shost) = FC_COS_CLASS2 | FC_COS_CLASS3;
+
+	zfcp_scsi_set_prot(adapter);
+
+	/* do not evaluate invalid fields */
+	if (bottom_incomplete)
+		return;
+
+	fc_host_port_id(shost) = ntoh24(bottom->s_id);
+	fc_host_speed(shost) =
+		zfcp_fsf_convert_portspeed(bottom->fc_link_speed);
+
+	snprintf(fc_host_model(shost), FC_SYMBOLIC_NAME_SIZE, "0x%04x",
+		 bottom->adapter_type);
+
+	switch (bottom->fc_topology) {
+	case FSF_TOPO_P2P:
+		fc_host_port_type(shost) = FC_PORTTYPE_PTP;
+		fc_host_fabric_name(shost) = 0;
+		break;
+	case FSF_TOPO_FABRIC:
+		fc_host_fabric_name(shost) = be64_to_cpu(plogi->fl_wwnn);
+		if (bottom->connection_features & FSF_FEATURE_NPIV_MODE)
+			fc_host_port_type(shost) = FC_PORTTYPE_NPIV;
+		else
+			fc_host_port_type(shost) = FC_PORTTYPE_NPORT;
+		break;
+	case FSF_TOPO_AL:
+		fc_host_port_type(shost) = FC_PORTTYPE_NLPORT;
+		fallthrough;
+	default:
+		fc_host_fabric_name(shost) = 0;
+		break;
+	}
+}
+
+void zfcp_scsi_shost_update_port_data(
+	struct zfcp_adapter *const adapter,
+	const struct fsf_qtcb_bottom_port *const bottom)
+{
+	struct Scsi_Host *const shost = adapter->scsi_host;
+
+	if (shost == NULL)
+		return;
+
+	fc_host_permanent_port_name(shost) = bottom->wwpn;
+	fc_host_maxframe_size(shost) = bottom->maximum_frame_size;
+	fc_host_supported_speeds(shost) =
+		zfcp_fsf_convert_portspeed(bottom->supported_speed);
+	memcpy(fc_host_supported_fc4s(shost), bottom->supported_fc4_types,
+	       FC_FC4_LIST_SIZE);
+	memcpy(fc_host_active_fc4s(shost), bottom->active_fc4_types,
+	       FC_FC4_LIST_SIZE);
+}
+
 struct fc_function_template zfcp_transport_functions = {
 	.show_starget_port_id = 1,
 	.show_starget_port_name = 1,
diff --git a/drivers/s390/scsi/zfcp_sysfs.c b/drivers/s390/scsi/zfcp_sysfs.c
index 7ec30ded0169..8d9662e8b717 100644
--- a/drivers/s390/scsi/zfcp_sysfs.c
+++ b/drivers/s390/scsi/zfcp_sysfs.c
@@ -216,20 +216,32 @@ static ssize_t zfcp_sysfs_port_rescan_store(struct device *dev,
 {
 	struct ccw_device *cdev = to_ccwdev(dev);
 	struct zfcp_adapter *adapter = zfcp_ccw_adapter_by_cdev(cdev);
+	int retval = 0;
 
 	if (!adapter)
 		return -ENODEV;
 
 	/*
+	 * If `scsi_host` is missing, we can't schedule `scan_work`, as it
+	 * makes use of the corresponding fc_host object. But this state is
+	 * only possible if xconfig/xport data has never completed yet,
+	 * and we couldn't successfully scan for ports anyway.
+	 */
+	if (adapter->scsi_host == NULL) {
+		retval = -ENODEV;
+		goto out;
+	}
+
+	/*
 	 * Users wish is our command: immediately schedule and flush a
 	 * worker to conduct a synchronous port scan, that is, neither
 	 * a random delay nor a rate limit is applied here.
 	 */
 	queue_delayed_work(adapter->work_queue, &adapter->scan_work, 0);
 	flush_delayed_work(&adapter->scan_work);
+out:
 	zfcp_ccw_adapter_put(adapter);
-
-	return (ssize_t) count;
+	return retval ? retval : (ssize_t) count;
 }
 static ZFCP_DEV_ATTR(adapter, port_rescan, S_IWUSR, NULL,
 		     zfcp_sysfs_port_rescan_store);
diff --git a/drivers/sbus/char/Kconfig b/drivers/sbus/char/Kconfig
index cebfbbb643a2..7c0a308e4959 100644
--- a/drivers/sbus/char/Kconfig
+++ b/drivers/sbus/char/Kconfig
@@ -53,7 +53,7 @@ config ENVCTRL
 config DISPLAY7SEG
 	tristate "7-Segment Display support"
 	depends on PCI && SPARC64
-	---help---
+	help
 	  This is the driver for the 7-segment display and LED present on
 	  Sun Microsystems CompactPCI models CP1400 and CP1500.
 
diff --git a/drivers/sbus/char/flash.c b/drivers/sbus/char/flash.c
index 4147d22fd448..3adfef210d8e 100644
--- a/drivers/sbus/char/flash.c
+++ b/drivers/sbus/char/flash.c
@@ -17,7 +17,6 @@
 #include <linux/of_device.h>
 
 #include <linux/uaccess.h>
-#include <asm/pgtable.h>
 #include <asm/io.h>
 #include <asm/upa.h>
 
diff --git a/drivers/sbus/char/oradax.c b/drivers/sbus/char/oradax.c
index 8af216287a84..21b7cb6e7e70 100644
--- a/drivers/sbus/char/oradax.c
+++ b/drivers/sbus/char/oradax.c
@@ -410,9 +410,7 @@ static void dax_unlock_pages(struct dax_ctx *ctx, int ccb_index, int nelem)
 
 			if (p) {
 				dax_dbg("freeing page %p", p);
-				if (j == OUT)
-					set_page_dirty(p);
-				put_page(p);
+				unpin_user_pages_dirty_lock(&p, 1, j == OUT);
 				ctx->pages[i][j] = NULL;
 			}
 		}
@@ -425,13 +423,13 @@ static int dax_lock_page(void *va, struct page **p)
 
 	dax_dbg("uva %p", va);
 
-	ret = get_user_pages_fast((unsigned long)va, 1, FOLL_WRITE, p);
+	ret = pin_user_pages_fast((unsigned long)va, 1, FOLL_WRITE, p);
 	if (ret == 1) {
 		dax_dbg("locked page %p, for VA %p", *p, va);
 		return 0;
 	}
 
-	dax_dbg("get_user_pages failed, va=%p, ret=%d", va, ret);
+	dax_dbg("pin_user_pages failed, va=%p, ret=%d", va, ret);
 	return -1;
 }
 
diff --git a/drivers/sbus/char/uctrl.c b/drivers/sbus/char/uctrl.c
index 37d252f2548d..05de0ce79cb9 100644
--- a/drivers/sbus/char/uctrl.c
+++ b/drivers/sbus/char/uctrl.c
@@ -21,7 +21,6 @@
 #include <asm/oplib.h>
 #include <asm/irq.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 
 #define DEBUG 1
 #ifdef DEBUG
diff --git a/drivers/scsi/53c700.c b/drivers/scsi/53c700.c
index 0068963bb933..461b3babb601 100644
--- a/drivers/scsi/53c700.c
+++ b/drivers/scsi/53c700.c
@@ -116,9 +116,9 @@
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/device.h>
+#include <linux/pgtable.h>
 #include <asm/dma.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/byteorder.h>
 
 #include <scsi/scsi.h>
diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c
index b5b3154e2c28..bb49d83cadc7 100644
--- a/drivers/scsi/BusLogic.c
+++ b/drivers/scsi/BusLogic.c
@@ -2237,7 +2237,7 @@ static bool __init blogic_inquiry(struct blogic_adapter *adapter)
 					"INQUIRE INSTALLED DEVICES ID 0 TO 7");
 		for (tgt_id = 0; tgt_id < 8; tgt_id++)
 			adapter->tgt_flags[tgt_id].tgt_exists =
-				(installed_devs0to7[tgt_id] != 0 ? true : false);
+				installed_devs0to7[tgt_id] != 0;
 	}
 	/*
 	   Issue the Inquire Setup Information command.
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 2017c43dac1b..e9ff4cd5fbe9 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -11,7 +11,7 @@ config RAID_ATTRS
 	default n
 	depends on BLOCK
 	depends on SCSI_MOD
-	---help---
+	help
 	  Provides RAID
 
 config SCSI
@@ -20,7 +20,7 @@ config SCSI
 	select SCSI_DMA if HAS_DMA
 	select SG_POOL
 	select BLK_SCSI_REQUEST
-	---help---
+	help
 	  If you want to use a SCSI hard disk, SCSI tape drive, SCSI CD-ROM or
 	  any other SCSI device under Linux, say Y and make sure that you know
 	  the name of your SCSI host adapter (the card inside your computer
@@ -55,7 +55,7 @@ config SCSI_PROC_FS
 	bool "legacy /proc/scsi/ support"
 	depends on SCSI && PROC_FS
 	default y
-	---help---
+	help
 	  This option enables support for the various files in
 	  /proc/scsi.  In Linux 2.6 this has been superseded by
 	  files in sysfs but many legacy applications rely on this.
@@ -69,7 +69,7 @@ config BLK_DEV_SD
 	tristate "SCSI disk support"
 	depends on SCSI
 	select BLK_DEV_INTEGRITY_T10 if BLK_DEV_INTEGRITY
-	---help---
+	help
 	  If you want to use SCSI hard disks, Fibre Channel disks,
 	  Serial ATA (SATA) or Parallel ATA (PATA) hard disks,
 	  USB storage or the SCSI or parallel port version of
@@ -90,7 +90,7 @@ config BLK_DEV_SD
 config CHR_DEV_ST
 	tristate "SCSI tape support"
 	depends on SCSI
-	---help---
+	help
 	  If you want to use a SCSI tape drive under Linux, say Y and read the
 	  SCSI-HOWTO, available from
 	  <http://www.tldp.org/docs.html#howto>, and
@@ -104,7 +104,7 @@ config BLK_DEV_SR
 	tristate "SCSI CDROM support"
 	depends on SCSI && BLK_DEV
 	select CDROM
-	---help---
+	help
 	  If you want to use a CD or DVD drive attached to your computer
 	  by SCSI, FireWire, USB or ATAPI, say Y and read the SCSI-HOWTO
 	  and the CDROM-HOWTO at <http://www.tldp.org/docs.html#howto>.
@@ -118,7 +118,7 @@ config BLK_DEV_SR
 config CHR_DEV_SG
 	tristate "SCSI generic support"
 	depends on SCSI
-	---help---
+	help
 	  If you want to use SCSI scanners, synthesizers or CD-writers or just
 	  about anything having "SCSI" in its name other than hard disks,
 	  CD-ROMs or tapes, say Y here. These won't be supported by the kernel
@@ -143,7 +143,7 @@ config CHR_DEV_SG
 config CHR_DEV_SCH
 	tristate "SCSI media changer support"
 	depends on SCSI
-	---help---
+	help
 	  This is a driver for SCSI media changers.  Most common devices are
 	  tape libraries and MOD/CDROM jukeboxes.  *Real* jukeboxes, you
 	  don't need this for those tiny 6-slot cdrom changers.  Media
@@ -178,7 +178,7 @@ config SCSI_CONSTANTS
 config SCSI_LOGGING
 	bool "SCSI logging facility"
 	depends on SCSI
-	---help---
+	help
 	  This turns on a logging facility that can be used to debug a number
 	  of SCSI related problems.
 
@@ -376,7 +376,7 @@ config SCSI_AHA152X
 	depends on ISA && SCSI
 	select SCSI_SPI_ATTRS
 	select CHECK_SIGNATURE
-	---help---
+	help
 	  This is a driver for the AHA-1510, AHA-1520, AHA-1522, and AHA-2825
 	  SCSI host adapters. It also works for the AVA-1505, but the IRQ etc.
 	  must be manually specified in this case.
@@ -391,7 +391,7 @@ config SCSI_AHA152X
 config SCSI_AHA1542
 	tristate "Adaptec AHA1542 support"
 	depends on ISA && SCSI && ISA_DMA_API
-	---help---
+	help
 	  This is support for a SCSI host adapter.  It is explained in section
 	  3.4 of the SCSI-HOWTO, available from
 	  <http://www.tldp.org/docs.html#howto>.  Note that Trantor was
@@ -405,7 +405,7 @@ config SCSI_AHA1542
 config SCSI_AHA1740
 	tristate "Adaptec AHA1740 support"
 	depends on EISA && SCSI
-	---help---
+	help
 	  This is support for a SCSI host adapter.  It is explained in section
 	  3.5 of the SCSI-HOWTO, available from
 	  <http://www.tldp.org/docs.html#howto>.  If it doesn't work out
@@ -498,7 +498,7 @@ config SCSI_HPTIOP
 config SCSI_BUSLOGIC
 	tristate "BusLogic SCSI support"
 	depends on (PCI || ISA) && SCSI && ISA_DMA_API && VIRT_TO_BUS
-	---help---
+	help
 	  This is support for BusLogic MultiMaster and FlashPoint SCSI Host
 	  Adapters. Consult the SCSI-HOWTO, available from
 	  <http://www.tldp.org/docs.html#howto>, and the files
@@ -580,20 +580,20 @@ config LIBFC
 	tristate "LibFC module"
 	depends on SCSI_FC_ATTRS
 	select CRC32
-	---help---
+	help
 	  Fibre Channel library module
 
 config LIBFCOE
 	tristate "LibFCoE module"
 	depends on LIBFC
-	---help---
+	help
 	  Library for Fibre Channel over Ethernet module
 
 config FCOE
 	tristate "FCoE module"
 	depends on PCI
 	depends on LIBFCOE
-	---help---
+	help
 	  Fibre Channel over Ethernet module
 
 config FCOE_FNIC
@@ -672,7 +672,7 @@ config SCSI_FDOMAIN_ISA
 config SCSI_GDTH
 	tristate "Intel/ICP (former GDT SCSI Disk Array) RAID Controller support"
 	depends on PCI && SCSI
-	---help---
+	help
 	  Formerly called GDT SCSI Disk Array Controller Support.
 
 	  This is a driver for RAID/SCSI Disk Array Controllers (EISA/ISA/PCI) 
@@ -688,7 +688,7 @@ config SCSI_ISCI
 	depends on PCI && SCSI
 	depends on X86
 	select SCSI_SAS_LIBSAS
-	---help---
+	help
 	  This driver supports the 6Gb/s SAS capabilities of the storage
 	  control unit found in the Intel(R) C600 series chipset.
 
@@ -696,7 +696,7 @@ config SCSI_GENERIC_NCR5380
 	tristate "Generic NCR5380/53c400 SCSI ISA card support"
 	depends on ISA && SCSI && HAS_IOPORT_MAP
 	select SCSI_SPI_ATTRS
-	---help---
+	help
 	  This is a driver for old ISA card SCSI controllers based on a
 	  NCR 5380, 53C80, 53C400, 53C400A, or DTC 436 device.
 	  Most boards such as the Trantor T130 fit this category, as do
@@ -708,7 +708,7 @@ config SCSI_GENERIC_NCR5380
 config SCSI_IPS
 	tristate "IBM ServeRAID support"
 	depends on PCI && SCSI
-	---help---
+	help
 	  This is support for the IBM ServeRAID hardware RAID controllers.
 	  See <http://www.developer.ibm.com/welcome/netfinity/serveraid.html>
 	  and <http://www-947.ibm.com/support/entry/portal/docdisplay?brand=5000008&lndocid=SERV-RAID>
@@ -790,7 +790,7 @@ config SCSI_INIA100
 config SCSI_PPA
 	tristate "IOMEGA parallel port (ppa - older drives)"
 	depends on SCSI && PARPORT_PC
-	---help---
+	help
 	  This driver supports older versions of IOMEGA's parallel port ZIP
 	  drive (a 100 MB removable media device).
 
@@ -817,7 +817,7 @@ config SCSI_PPA
 config SCSI_IMM
 	tristate "IOMEGA parallel port (imm - newer drives)"
 	depends on SCSI && PARPORT_PC
-	---help---
+	help
 	  This driver supports newer versions of IOMEGA's parallel port ZIP
 	  drive (a 100 MB removable media device).
 
@@ -844,7 +844,7 @@ config SCSI_IMM
 config SCSI_IZIP_EPP16
 	bool "ppa/imm option - Use slow (but safe) EPP-16"
 	depends on SCSI_PPA || SCSI_IMM
-	---help---
+	help
 	  EPP (Enhanced Parallel Port) is a standard for parallel ports which
 	  allows them to act as expansion buses that can handle up to 64
 	  peripheral devices.
@@ -896,7 +896,7 @@ config 53C700_LE_ON_BE
 config SCSI_STEX
 	tristate "Promise SuperTrak EX Series support"
 	depends on PCI && SCSI
-	---help---
+	help
 	  This driver supports Promise SuperTrak EX series storage controllers.
 
 	  Promise provides Linux RAID configuration utility for these
@@ -914,7 +914,7 @@ config SCSI_SYM53C8XX_2
 	tristate "SYM53C8XX Version 2 SCSI support"
 	depends on PCI && SCSI
 	select SCSI_SPI_ATTRS
-	---help---
+	help
 	  This driver supports the whole NCR53C8XX/SYM53C8XX family of
 	  PCI-SCSI controllers.  It also supports the subset of LSI53C10XX
 	  Ultra-160 controllers that are based on the SYM53C8XX SCRIPTS
@@ -928,7 +928,7 @@ config SCSI_SYM53C8XX_DMA_ADDRESSING_MODE
 	int "DMA addressing mode"
 	depends on SCSI_SYM53C8XX_2
 	default "1"
-	---help---
+	help
 	  This option only applies to PCI-SCSI chips that are PCI DAC
 	  capable (875A, 895A, 896, 1010-33, 1010-66, 1000).
 
@@ -984,7 +984,7 @@ config SCSI_IPR
 	select FW_LOADER
 	select IRQ_POLL
 	select SGL_ALLOC
-	---help---
+	help
 	  This driver supports the IBM Power Linux family RAID adapters.
 	  This includes IBM pSeries 5712, 5703, 5709, and 570A, as well
 	  as IBM iSeries 5702, 5703, 5709, and 570A.
@@ -1022,7 +1022,7 @@ config SCSI_NCR53C8XX_DEFAULT_TAGS
 	int "default tagged command queue depth"
 	depends on SCSI_ZALON
 	default "8"
-	---help---
+	help
 	  "Tagged command queuing" is a feature of SCSI-2 which improves
 	  performance: the host adapter can send several SCSI commands to a
 	  device's queue even if previous commands haven't finished yet.
@@ -1048,7 +1048,7 @@ config SCSI_NCR53C8XX_MAX_TAGS
 	int "maximum number of queued commands"
 	depends on SCSI_ZALON
 	default "32"
-	---help---
+	help
 	  This option allows you to specify the maximum number of commands
 	  that can be queued to any device, when tagged command queuing is
 	  possible. The default value is 32. Minimum is 2, maximum is 64.
@@ -1065,7 +1065,7 @@ config SCSI_NCR53C8XX_SYNC
 	int "synchronous transfers frequency in MHz"
 	depends on SCSI_ZALON
 	default "20"
-	---help---
+	help
 	  The SCSI Parallel Interface-2 Standard defines 5 classes of transfer
 	  rates: FAST-5, FAST-10, FAST-20, FAST-40 and FAST-80.  The numbers
 	  are respectively the maximum data transfer rates in mega-transfers
@@ -1108,7 +1108,7 @@ config SCSI_NCR53C8XX_NO_DISCONNECT
 config SCSI_QLOGIC_FAS
 	tristate "Qlogic FAS SCSI support"
 	depends on ISA && SCSI
-	---help---
+	help
 	  This is a driver for the ISA, VLB, and PCMCIA versions of the Qlogic
 	  FastSCSI! cards as well as any other card based on the FASXX chip
 	  (including the Control Concepts SCSI/IDE/SIO/PIO/FDC cards).
@@ -1158,14 +1158,14 @@ config SCSI_LPFC
 	depends on NVME_TARGET_FC || NVME_TARGET_FC=n
 	depends on NVME_FC || NVME_FC=n
 	select CRC_T10DIF
-	---help---
+	help
           This lpfc driver supports the Emulex LightPulse
           Family of Fibre Channel PCI host adapters.
 
 config SCSI_LPFC_DEBUG_FS
 	bool "Emulex LightPulse Fibre Channel debugfs Support"
 	depends on SCSI_LPFC && DEBUG_FS
-	---help---
+	help
 	  This makes debugging information from the lpfc driver
 	  available via the debugfs filesystem.
 
@@ -1173,7 +1173,7 @@ config SCSI_SIM710
 	tristate "Simple 53c710 SCSI support (Compaq, NCR machines)"
 	depends on EISA && SCSI
 	select SCSI_SPI_ATTRS
-	---help---
+	help
 	  This driver is for NCR53c710 based SCSI host adapters.
 
 	  It currently supports Compaq EISA cards.
@@ -1181,7 +1181,7 @@ config SCSI_SIM710
 config SCSI_DC395x
 	tristate "Tekram DC395(U/UW/F) and DC315(U) SCSI support"
 	depends on PCI && SCSI
-	---help---
+	help
 	  This driver supports PCI SCSI host adapters based on the ASIC
 	  TRM-S1040 chip, e.g Tekram DC395(U/UW/F) and DC315(U) variants.
 
@@ -1197,7 +1197,7 @@ config SCSI_AM53C974
 	tristate "Tekram DC390(T) and Am53/79C974 SCSI support (new driver)"
 	depends on PCI && SCSI
 	select SCSI_SPI_ATTRS
-	---help---
+	help
 	  This driver supports PCI SCSI host adapters based on the Am53C974A
 	  chip, e.g. Tekram DC390(T), DawiControl 2974 and some onboard
 	  PCscsi/PCnet (Am53/79C974) solutions.
@@ -1224,7 +1224,7 @@ config SCSI_WD719X
 	tristate "Western Digital WD7193/7197/7296 support"
 	depends on PCI && SCSI
 	select EEPROM_93CX6
-	---help---
+	help
 	  This is a driver for Western Digital WD7193, WD7197 and WD7296 PCI
 	  SCSI controllers (based on WD33C296A chip).
 
@@ -1318,7 +1318,7 @@ config A2091_SCSI
 config GVP11_SCSI
 	tristate "GVP Series II WD33C93A support"
 	depends on ZORRO && SCSI
-	---help---
+	help
 	  If you have a Great Valley Products Series II SCSI controller,
 	  answer Y. Also say Y if you have a later model of GVP SCSI
 	  controller (such as the GVP A4008 or a Combo board). Otherwise,
@@ -1374,7 +1374,7 @@ config ATARI_SCSI
 	tristate "Atari native SCSI support"
 	depends on ATARI && SCSI
 	select SCSI_SPI_ATTRS
-	---help---
+	help
 	  If you have an Atari with built-in NCR5380 SCSI controller (TT,
 	  Falcon, ...) say Y to get it supported. Of course also, if you have
 	  a compatible SCSI controller (e.g. for Medusa).
@@ -1486,7 +1486,7 @@ config SCSI_PMCRAID
 	tristate "PMC SIERRA Linux MaxRAID adapter support"
 	depends on PCI && SCSI && NET
 	select SGL_ALLOC
-	---help---
+	help
 	  This driver supports the PMC SIERRA MaxRAID adapters.
 
 config SCSI_PM8001
diff --git a/drivers/scsi/a2091.c b/drivers/scsi/a2091.c
index 564b35473672..5853db36eceb 100644
--- a/drivers/scsi/a2091.c
+++ b/drivers/scsi/a2091.c
@@ -9,7 +9,6 @@
 #include <linux/module.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/amigaints.h>
 #include <asm/amigahw.h>
 
diff --git a/drivers/scsi/a3000.c b/drivers/scsi/a3000.c
index b6a0432f305a..86f1da22aaa5 100644
--- a/drivers/scsi/a3000.c
+++ b/drivers/scsi/a3000.c
@@ -10,7 +10,6 @@
 #include <linux/module.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/amigaints.h>
 #include <asm/amigahw.h>
 
diff --git a/drivers/scsi/aacraid/aachba.c b/drivers/scsi/aacraid/aachba.c
index eb72ac8136c3..2b868f8db8ff 100644
--- a/drivers/scsi/aacraid/aachba.c
+++ b/drivers/scsi/aacraid/aachba.c
@@ -814,7 +814,6 @@ int aac_probe_container(struct aac_dev *dev, int cid)
 		kfree(scsidev);
 		return -ENOMEM;
 	}
-	scsicmd->list.next = NULL;
 	scsicmd->scsi_done = aac_probe_container_scsi_done;
 
 	scsicmd->device = scsidev;
diff --git a/drivers/scsi/aacraid/commctrl.c b/drivers/scsi/aacraid/commctrl.c
index ffe41bc111fc..34e65dea992e 100644
--- a/drivers/scsi/aacraid/commctrl.c
+++ b/drivers/scsi/aacraid/commctrl.c
@@ -513,15 +513,10 @@ static int aac_send_raw_srb(struct aac_dev* dev, void __user * arg)
 		goto cleanup;
 	}
 
-	user_srbcmd = kmalloc(fibsize, GFP_KERNEL);
-	if (!user_srbcmd) {
-		dprintk((KERN_DEBUG"aacraid: Could not make a copy of the srb\n"));
-		rcode = -ENOMEM;
-		goto cleanup;
-	}
-	if(copy_from_user(user_srbcmd, user_srb,fibsize)){
-		dprintk((KERN_DEBUG"aacraid: Could not copy srb from user\n"));
-		rcode = -EFAULT;
+	user_srbcmd = memdup_user(user_srb, fibsize);
+	if (IS_ERR(user_srbcmd)) {
+		rcode = PTR_ERR(user_srbcmd);
+		user_srbcmd = NULL;
 		goto cleanup;
 	}
 
diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c
index ddd73f6798af..8ee4e1abe568 100644
--- a/drivers/scsi/aacraid/commsup.c
+++ b/drivers/scsi/aacraid/commsup.c
@@ -2351,7 +2351,7 @@ fib_free_out:
 	goto out;
 }
 
-int aac_send_safw_hostttime(struct aac_dev *dev, struct timespec64 *now)
+static int aac_send_safw_hostttime(struct aac_dev *dev, struct timespec64 *now)
 {
 	struct tm cur_tm;
 	char wellness_str[] = "<HW>TD\010\0\0\0\0\0\0\0\0\0DW\0\0ZZ";
@@ -2380,7 +2380,7 @@ out:
 	return ret;
 }
 
-int aac_send_hosttime(struct aac_dev *dev, struct timespec64 *now)
+static int aac_send_hosttime(struct aac_dev *dev, struct timespec64 *now)
 {
 	int ret = -ENOMEM;
 	struct fib *fibptr;
diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c
index 83a60b0a8cd8..a308e86a97f1 100644
--- a/drivers/scsi/aacraid/linit.c
+++ b/drivers/scsi/aacraid/linit.c
@@ -864,7 +864,7 @@ static u8 aac_eh_tmf_hard_reset_fib(struct aac_hba_map_info *info,
        return HBA_IU_TYPE_SATA_REQ;
 }
 
-void aac_tmf_callback(void *context, struct fib *fibptr)
+static void aac_tmf_callback(void *context, struct fib *fibptr)
 {
 	struct aac_hba_resp *err =
 		&((struct aac_native_hba *)fibptr->hw_fib_va)->resp.err;
@@ -1078,7 +1078,7 @@ static int aac_eh_bus_reset(struct scsi_cmnd* cmd)
  *	@scsi_cmd:	SCSI command block causing the reset
  *
  */
-int aac_eh_host_reset(struct scsi_cmnd *cmd)
+static int aac_eh_host_reset(struct scsi_cmnd *cmd)
 {
 	struct scsi_device * dev = cmd->device;
 	struct Scsi_Host * host = dev->host;
@@ -1632,7 +1632,7 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	struct Scsi_Host *shost;
 	struct aac_dev *aac;
 	struct list_head *insert = &aac_devices;
-	int error = -ENODEV;
+	int error;
 	int unique_id = 0;
 	u64 dmamask;
 	int mask_bits = 0;
@@ -1657,7 +1657,6 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	error = pci_enable_device(pdev);
 	if (error)
 		goto out;
-	error = -ENODEV;
 
 	if (!(aac_drivers[index].quirks & AAC_QUIRK_SRC)) {
 		error = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
@@ -1689,8 +1688,10 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	pci_set_master(pdev);
 
 	shost = scsi_host_alloc(&aac_driver_template, sizeof(struct aac_dev));
-	if (!shost)
+	if (!shost) {
+		error = -ENOMEM;
 		goto out_disable_pdev;
+	}
 
 	shost->irq = pdev->irq;
 	shost->unique_id = unique_id;
@@ -1714,8 +1715,11 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	aac->fibs = kcalloc(shost->can_queue + AAC_NUM_MGT_FIB,
 			    sizeof(struct fib),
 			    GFP_KERNEL);
-	if (!aac->fibs)
+	if (!aac->fibs) {
+		error = -ENOMEM;
 		goto out_free_host;
+	}
+
 	spin_lock_init(&aac->fib_lock);
 
 	mutex_init(&aac->ioctl_mutex);
diff --git a/drivers/scsi/aic7xxx/Kconfig.aic79xx b/drivers/scsi/aic7xxx/Kconfig.aic79xx
index d4c50b8fce29..a47dbd500e9a 100644
--- a/drivers/scsi/aic7xxx/Kconfig.aic79xx
+++ b/drivers/scsi/aic7xxx/Kconfig.aic79xx
@@ -15,7 +15,7 @@ config AIC79XX_CMDS_PER_DEVICE
 	int "Maximum number of TCQ commands per device"
 	depends on SCSI_AIC79XX
 	default "32"
-	---help---
+	help
 	Specify the number of commands you would like to allocate per SCSI
 	device when Tagged Command Queueing (TCQ) is enabled on that device.
 
@@ -38,7 +38,7 @@ config AIC79XX_RESET_DELAY_MS
 	int "Initial bus reset delay in milli-seconds"
 	depends on SCSI_AIC79XX
 	default "5000"
-	---help---
+	help
 	The number of milliseconds to delay after an initial bus reset.
 	The bus settle delay following all error recovery actions is
 	dictated by the SCSI layer and is not affected by this value.
diff --git a/drivers/scsi/aic7xxx/Kconfig.aic7xxx b/drivers/scsi/aic7xxx/Kconfig.aic7xxx
index 9d027549d698..0cfd92ce750a 100644
--- a/drivers/scsi/aic7xxx/Kconfig.aic7xxx
+++ b/drivers/scsi/aic7xxx/Kconfig.aic7xxx
@@ -7,7 +7,7 @@ config SCSI_AIC7XXX
 	tristate "Adaptec AIC7xxx Fast -> U160 support"
 	depends on (PCI || EISA) && SCSI
 	select SCSI_SPI_ATTRS
-	---help---
+	help
 	This driver supports all of Adaptec's Fast through Ultra 160 PCI
 	based SCSI controllers as well as the aic7770 based EISA and VLB
 	SCSI controllers (the 274x and 284x series).  For AAA and ARO based
@@ -20,7 +20,7 @@ config AIC7XXX_CMDS_PER_DEVICE
 	int "Maximum number of TCQ commands per device"
 	depends on SCSI_AIC7XXX
 	default "32"
-	---help---
+	help
 	Specify the number of commands you would like to allocate per SCSI
 	device when Tagged Command Queueing (TCQ) is enabled on that device.
 
@@ -43,7 +43,7 @@ config AIC7XXX_RESET_DELAY_MS
 	int "Initial bus reset delay in milli-seconds"
 	depends on SCSI_AIC7XXX
 	default "5000"
-	---help---
+	help
 	The number of milliseconds to delay after an initial bus reset.
 	The bus settle delay following all error recovery actions is
 	dictated by the SCSI layer and is not affected by this value.
diff --git a/drivers/scsi/aic7xxx/aic79xx_core.c b/drivers/scsi/aic7xxx/aic79xx_core.c
index a336a458c978..e4a09b93d00c 100644
--- a/drivers/scsi/aic7xxx/aic79xx_core.c
+++ b/drivers/scsi/aic7xxx/aic79xx_core.c
@@ -3662,8 +3662,7 @@ ahd_free_tstate(struct ahd_softc *ahd, u_int scsi_id, char channel, int force)
 		return;
 
 	tstate = ahd->enabled_targets[scsi_id];
-	if (tstate != NULL)
-		kfree(tstate);
+	kfree(tstate);
 	ahd->enabled_targets[scsi_id] = NULL;
 }
 #endif
@@ -6054,14 +6053,13 @@ ahd_alloc(void *platform_arg, char *name)
 {
 	struct  ahd_softc *ahd;
 
-	ahd = kmalloc(sizeof(*ahd), GFP_ATOMIC);
+	ahd = kzalloc(sizeof(*ahd), GFP_ATOMIC);
 	if (!ahd) {
 		printk("aic7xxx: cannot malloc softc!\n");
 		kfree(name);
 		return NULL;
 	}
 
-	memset(ahd, 0, sizeof(*ahd));
 	ahd->seep_config = kmalloc(sizeof(*ahd->seep_config), GFP_ATOMIC);
 	if (ahd->seep_config == NULL) {
 		kfree(ahd);
@@ -6120,8 +6118,7 @@ ahd_set_unit(struct ahd_softc *ahd, int unit)
 void
 ahd_set_name(struct ahd_softc *ahd, char *name)
 {
-	if (ahd->name != NULL)
-		kfree(ahd->name);
+	kfree(ahd->name);
 	ahd->name = name;
 }
 
@@ -6182,12 +6179,9 @@ ahd_free(struct ahd_softc *ahd)
 		kfree(ahd->black_hole);
 	}
 #endif
-	if (ahd->name != NULL)
-		kfree(ahd->name);
-	if (ahd->seep_config != NULL)
-		kfree(ahd->seep_config);
-	if (ahd->saved_stack != NULL)
-		kfree(ahd->saved_stack);
+	kfree(ahd->name);
+	kfree(ahd->seep_config);
+	kfree(ahd->saved_stack);
 	kfree(ahd);
 	return;
 }
diff --git a/drivers/scsi/aic7xxx/aic7xxx_core.c b/drivers/scsi/aic7xxx/aic7xxx_core.c
index 84fc499cb1e6..3d4df906fa4f 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_core.c
+++ b/drivers/scsi/aic7xxx/aic7xxx_core.c
@@ -2178,8 +2178,7 @@ ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
 	if (channel == 'B')
 		scsi_id += 8;
 	tstate = ahc->enabled_targets[scsi_id];
-	if (tstate != NULL)
-		kfree(tstate);
+	kfree(tstate);
 	ahc->enabled_targets[scsi_id] = NULL;
 }
 #endif
@@ -4384,13 +4383,13 @@ ahc_alloc(void *platform_arg, char *name)
 	struct  ahc_softc *ahc;
 	int	i;
 
-	ahc = kmalloc(sizeof(*ahc), GFP_ATOMIC);
+	ahc = kzalloc(sizeof(*ahc), GFP_ATOMIC);
 	if (!ahc) {
 		printk("aic7xxx: cannot malloc softc!\n");
 		kfree(name);
 		return NULL;
 	}
-	memset(ahc, 0, sizeof(*ahc));
+
 	ahc->seep_config = kmalloc(sizeof(*ahc->seep_config), GFP_ATOMIC);
 	if (ahc->seep_config == NULL) {
 		kfree(ahc);
@@ -4453,8 +4452,7 @@ ahc_set_unit(struct ahc_softc *ahc, int unit)
 void
 ahc_set_name(struct ahc_softc *ahc, char *name)
 {
-	if (ahc->name != NULL)
-		kfree(ahc->name);
+	kfree(ahc->name);
 	ahc->name = name;
 }
 
@@ -4515,10 +4513,8 @@ ahc_free(struct ahc_softc *ahc)
 		kfree(ahc->black_hole);
 	}
 #endif
-	if (ahc->name != NULL)
-		kfree(ahc->name);
-	if (ahc->seep_config != NULL)
-		kfree(ahc->seep_config);
+	kfree(ahc->name);
+	kfree(ahc->seep_config);
 	kfree(ahc);
 	return;
 }
@@ -4927,8 +4923,7 @@ ahc_fini_scbdata(struct ahc_softc *ahc)
 	case 0:
 		break;
 	}
-	if (scb_data->scbarray != NULL)
-		kfree(scb_data->scbarray);
+	kfree(scb_data->scbarray);
 }
 
 static void
diff --git a/drivers/scsi/aic94xx/aic94xx_sds.c b/drivers/scsi/aic94xx/aic94xx_sds.c
index 3ddc8852bc32..105adba559a1 100644
--- a/drivers/scsi/aic94xx/aic94xx_sds.c
+++ b/drivers/scsi/aic94xx/aic94xx_sds.c
@@ -406,7 +406,7 @@ struct asd_manuf_sec {
 	u8    sas_addr[SAS_ADDR_SIZE];
 	u8    pcba_sn[ASD_PCBA_SN_SIZE];
 	/* Here start the other segments */
-	u8    linked_list[0];
+	u8    linked_list[];
 } __attribute__ ((packed));
 
 struct asd_manuf_phy_desc {
@@ -449,7 +449,7 @@ struct asd_ms_sb_desc {
 	u8    type;
 	u8    node_desc_index;
 	u8    conn_desc_index;
-	u8    _recvd[0];
+	u8    _recvd[];
 } __attribute__ ((packed));
 
 #if 0
@@ -478,12 +478,12 @@ struct asd_ms_conn_desc {
 	u8    size_sideband_desc;
 	u32   _resvd;
 	u8    name[16];
-	struct asd_ms_sb_desc sb_desc[0];
+	struct asd_ms_sb_desc sb_desc[];
 } __attribute__ ((packed));
 
 struct asd_nd_phy_desc {
 	u8    vp_attch_type;
-	u8    attch_specific[0];
+	u8    attch_specific[];
 } __attribute__ ((packed));
 
 #if 0
@@ -503,7 +503,7 @@ struct asd_ms_node_desc {
 	u8    size_phy_desc;
 	u8    _resvd;
 	u8    name[16];
-	struct asd_nd_phy_desc phy_desc[0];
+	struct asd_nd_phy_desc phy_desc[];
 } __attribute__ ((packed));
 
 struct asd_ms_conn_map {
@@ -518,7 +518,7 @@ struct asd_ms_conn_map {
 	u8    usage_model_id;
 	u32   _resvd;
 	struct asd_ms_conn_desc conn_desc[0];
-	struct asd_ms_node_desc node_desc[0];
+	struct asd_ms_node_desc node_desc[];
 } __attribute__ ((packed));
 
 struct asd_ctrla_phy_entry {
@@ -542,7 +542,7 @@ struct asd_ll_el {
 	u8   id0;
 	u8   id1;
 	__le16  next;
-	u8   something_here[0];
+	u8   something_here[];
 } __attribute__ ((packed));
 
 static int asd_poll_flash(struct asd_ha_struct *asd_ha)
diff --git a/drivers/scsi/arm/acornscsi.c b/drivers/scsi/arm/acornscsi.c
index ddb52e7ba622..9a912fd0f70b 100644
--- a/drivers/scsi/arm/acornscsi.c
+++ b/drivers/scsi/arm/acornscsi.c
@@ -2911,8 +2911,10 @@ static int acornscsi_probe(struct expansion_card *ec, const struct ecard_id *id)
 
 	ashost->base = ecardm_iomap(ec, ECARD_RES_MEMC, 0, 0);
 	ashost->fast = ecardm_iomap(ec, ECARD_RES_IOCFAST, 0, 0);
-	if (!ashost->base || !ashost->fast)
+	if (!ashost->base || !ashost->fast) {
+		ret = -ENOMEM;
 		goto out_put;
+	}
 
 	host->irq = ec->irq;
 	ashost->host = host;
diff --git a/drivers/scsi/arm/cumana_2.c b/drivers/scsi/arm/cumana_2.c
index a1f3e9ee4e63..65691c21f133 100644
--- a/drivers/scsi/arm/cumana_2.c
+++ b/drivers/scsi/arm/cumana_2.c
@@ -23,11 +23,11 @@
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/dma-mapping.h>
+#include <linux/pgtable.h>
 
 #include <asm/dma.h>
 #include <asm/ecard.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 
 #include "../scsi.h"
 #include <scsi/scsi_host.h>
diff --git a/drivers/scsi/arm/eesox.c b/drivers/scsi/arm/eesox.c
index 134f040d58e2..6e204a2e0c8d 100644
--- a/drivers/scsi/arm/eesox.c
+++ b/drivers/scsi/arm/eesox.c
@@ -29,11 +29,11 @@
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/dma-mapping.h>
+#include <linux/pgtable.h>
 
 #include <asm/io.h>
 #include <asm/dma.h>
 #include <asm/ecard.h>
-#include <asm/pgtable.h>
 
 #include "../scsi.h"
 #include <scsi/scsi_host.h>
diff --git a/drivers/scsi/arm/powertec.c b/drivers/scsi/arm/powertec.c
index c795537a671c..772a13e5fd91 100644
--- a/drivers/scsi/arm/powertec.c
+++ b/drivers/scsi/arm/powertec.c
@@ -14,11 +14,11 @@
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/dma-mapping.h>
+#include <linux/pgtable.h>
 
 #include <asm/dma.h>
 #include <asm/ecard.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 
 #include "../scsi.h"
 #include <scsi/scsi_host.h>
diff --git a/drivers/scsi/bfa/bfa_core.c b/drivers/scsi/bfa/bfa_core.c
index 0f554ebb8f2c..fb4c469bd89f 100644
--- a/drivers/scsi/bfa/bfa_core.c
+++ b/drivers/scsi/bfa/bfa_core.c
@@ -708,7 +708,7 @@ bfa_reqq_resume(struct bfa_s *bfa, int qid)
 	}
 }
 
-bfa_boolean_t
+static bfa_boolean_t
 bfa_isr_rspq(struct bfa_s *bfa, int qid)
 {
 	struct bfi_msg_s *m;
diff --git a/drivers/scsi/bfa/bfa_fcpim.c b/drivers/scsi/bfa/bfa_fcpim.c
index 284baa3b0c8e..766f2b5ed2ab 100644
--- a/drivers/scsi/bfa/bfa_fcpim.c
+++ b/drivers/scsi/bfa/bfa_fcpim.c
@@ -436,7 +436,7 @@ bfa_fcpim_port_iostats(struct bfa_s *bfa,
 	return BFA_STATUS_OK;
 }
 
-void
+static void
 bfa_ioim_profile_comp(struct bfa_ioim_s *ioim)
 {
 	struct bfa_itnim_latency_s *io_lat =
@@ -453,7 +453,7 @@ bfa_ioim_profile_comp(struct bfa_ioim_s *ioim)
 	io_lat->avg[idx] += val;
 }
 
-void
+static void
 bfa_ioim_profile_start(struct bfa_ioim_s *ioim)
 {
 	ioim->start_time = jiffies;
diff --git a/drivers/scsi/bfa/bfa_fcs_lport.c b/drivers/scsi/bfa/bfa_fcs_lport.c
index 7c3eadc58b98..297a77f5806c 100644
--- a/drivers/scsi/bfa/bfa_fcs_lport.c
+++ b/drivers/scsi/bfa/bfa_fcs_lport.c
@@ -1283,7 +1283,7 @@ bfa_fcs_lport_n2n_offline(struct bfa_fcs_lport_s *port)
 	n2n_port->reply_oxid = 0;
 }
 
-void
+static void
 bfa_fcport_get_loop_attr(struct bfa_fcs_lport_s *port)
 {
 	int i = 0, j = 0, bit = 0, alpa_bit = 0;
@@ -4358,7 +4358,7 @@ bfa_fcs_lport_ns_sm_online(struct bfa_fcs_lport_ns_s *ns,
 			bfa_sm_set_state(ns,
 				bfa_fcs_lport_ns_sm_sending_gid_ft);
 			bfa_fcs_lport_ns_send_gid_ft(ns, NULL);
-		};
+		}
 		break;
 
 	default:
diff --git a/drivers/scsi/bfa/bfa_fcs_rport.c b/drivers/scsi/bfa/bfa_fcs_rport.c
index 82801b366500..fc294e1950a6 100644
--- a/drivers/scsi/bfa/bfa_fcs_rport.c
+++ b/drivers/scsi/bfa/bfa_fcs_rport.c
@@ -1575,7 +1575,7 @@ bfa_fcs_rport_sm_nsdisc_sent(struct bfa_fcs_rport_s *rport,
 			bfa_timer_start(rport->fcs->bfa, &rport->timer,
 					bfa_fcs_rport_timeout, rport,
 					bfa_fcs_rport_del_timeout);
-		};
+		}
 		break;
 
 	case RPSM_EVENT_DELETE:
@@ -2449,7 +2449,7 @@ bfa_fcs_rport_hal_online_action(struct bfa_fcs_rport_s *rport)
 		bfa_fcs_itnim_brp_online(rport->itnim);
 		if (!BFA_FCS_PID_IS_WKA(rport->pid))
 			bfa_fcs_rpf_rport_online(rport);
-	};
+	}
 
 	wwn2str(lpwwn_buf, bfa_fcs_lport_get_pwwn(port));
 	wwn2str(rpwwn_buf, rport->pwwn);
diff --git a/drivers/scsi/bfa/bfa_ioc_ct.c b/drivers/scsi/bfa/bfa_ioc_ct.c
index 18b58b2f304f..6fd3383ee538 100644
--- a/drivers/scsi/bfa/bfa_ioc_ct.c
+++ b/drivers/scsi/bfa/bfa_ioc_ct.c
@@ -364,7 +364,7 @@ bfa_ioc_ct_isr_mode_set(struct bfa_ioc_s *ioc, bfa_boolean_t msix)
 	writel(r32, rb + FNC_PERS_REG);
 }
 
-bfa_boolean_t
+static bfa_boolean_t
 bfa_ioc_ct2_lpu_read_stat(struct bfa_ioc_s *ioc)
 {
 	u32	r32;
@@ -744,7 +744,7 @@ bfa_ioc_ct2_mem_init(void __iomem *rb)
 	writel(0, (rb + CT2_MBIST_CTL_REG));
 }
 
-void
+static void
 bfa_ioc_ct2_mac_reset(void __iomem *rb)
 {
 	/* put port0, port1 MAC & AHB in reset */
diff --git a/drivers/scsi/bfa/bfa_svc.c b/drivers/scsi/bfa/bfa_svc.c
index 6d2131441f0a..0b7d2e8f4a66 100644
--- a/drivers/scsi/bfa/bfa_svc.c
+++ b/drivers/scsi/bfa/bfa_svc.c
@@ -4284,7 +4284,7 @@ bfa_fcport_dportdisable(struct bfa_s *bfa)
 	bfa_port_set_dportenabled(&bfa->modules.port, BFA_FALSE);
 }
 
-void
+static void
 bfa_fcport_ddportenable(struct bfa_s *bfa)
 {
 	/*
@@ -4293,7 +4293,7 @@ bfa_fcport_ddportenable(struct bfa_s *bfa)
 	bfa_sm_send_event(BFA_FCPORT_MOD(bfa), BFA_FCPORT_SM_DDPORTENABLE);
 }
 
-void
+static void
 bfa_fcport_ddportdisable(struct bfa_s *bfa)
 {
 	/*
@@ -5517,7 +5517,6 @@ uf_recv(struct bfa_s *bfa, struct bfi_uf_frm_rcvd_s *m)
 	struct bfa_uf_s *uf = &ufm->uf_list[uf_tag];
 	struct bfa_uf_buf_s *uf_buf;
 	uint8_t *buf;
-	struct fchs_s *fchs;
 
 	uf_buf = (struct bfa_uf_buf_s *)
 			bfa_mem_get_dmabuf_kva(ufm, uf_tag, uf->pb_len);
@@ -5526,8 +5525,6 @@ uf_recv(struct bfa_s *bfa, struct bfi_uf_frm_rcvd_s *m)
 	m->frm_len = be16_to_cpu(m->frm_len);
 	m->xfr_len = be16_to_cpu(m->xfr_len);
 
-	fchs = (struct fchs_s *)uf_buf;
-
 	list_del(&uf->qe);	/* dequeue from posted queue */
 
 	uf->data_ptr = buf;
diff --git a/drivers/scsi/bfa/bfad.c b/drivers/scsi/bfa/bfad.c
index eb0c76338295..bc5d84f87d8f 100644
--- a/drivers/scsi/bfa/bfad.c
+++ b/drivers/scsi/bfa/bfad.c
@@ -50,7 +50,7 @@ int		pcie_max_read_reqsz;
 int		bfa_debugfs_enable = 1;
 int		msix_disable_cb = 0, msix_disable_ct = 0;
 int		max_xfer_size = BFAD_MAX_SECTORS >> 1;
-int		max_rport_logins = BFA_FCS_MAX_RPORT_LOGINS;
+static int	max_rport_logins = BFA_FCS_MAX_RPORT_LOGINS;
 
 /* Firmware releated */
 u32	bfi_image_cb_size, bfi_image_ct_size, bfi_image_ct2_size;
diff --git a/drivers/scsi/bfa/bfad_attr.c b/drivers/scsi/bfa/bfad_attr.c
index fbfce02e5b93..5ae1e3f78910 100644
--- a/drivers/scsi/bfa/bfad_attr.c
+++ b/drivers/scsi/bfa/bfad_attr.c
@@ -437,7 +437,7 @@ bfad_im_vport_create(struct fc_vport *fc_vport, bool disable)
 	return status;
 }
 
-int
+static int
 bfad_im_issue_fc_host_lip(struct Scsi_Host *shost)
 {
 	struct bfad_im_port_s *im_port =
@@ -562,7 +562,7 @@ bfad_im_vport_disable(struct fc_vport *fc_vport, bool disable)
 	return 0;
 }
 
-void
+static void
 bfad_im_vport_set_symbolic_name(struct fc_vport *fc_vport)
 {
 	struct bfad_vport_s *vport = (struct bfad_vport_s *)fc_vport->dd_data;
diff --git a/drivers/scsi/bfa/bfad_bsg.c b/drivers/scsi/bfa/bfad_bsg.c
index a76c968dbac5..412dbe125e10 100644
--- a/drivers/scsi/bfa/bfad_bsg.c
+++ b/drivers/scsi/bfa/bfad_bsg.c
@@ -136,7 +136,7 @@ bfad_iocmd_ioc_get_attr(struct bfad_s *bfad, void *cmd)
 	return 0;
 }
 
-int
+static int
 bfad_iocmd_ioc_get_stats(struct bfad_s *bfad, void *cmd)
 {
 	struct bfa_bsg_ioc_stats_s *iocmd = (struct bfa_bsg_ioc_stats_s *)cmd;
diff --git a/drivers/scsi/bnx2fc/Kconfig b/drivers/scsi/bnx2fc/Kconfig
index e0ccb48ec961..3cf7e08df809 100644
--- a/drivers/scsi/bnx2fc/Kconfig
+++ b/drivers/scsi/bnx2fc/Kconfig
@@ -9,5 +9,5 @@ config SCSI_BNX2X_FCOE
 	select ETHERNET
 	select NET_VENDOR_BROADCOM
 	select CNIC
-	---help---
+	help
 	This driver supports FCoE offload for the QLogic devices.
diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
index 1cbb431fa682..0e33324e16f5 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
@@ -945,7 +945,7 @@ static void bnx2fc_indicate_netevent(void *context, unsigned long event,
 				 */
 				if (interface->enabled)
 					fcoe_ctlr_link_up(ctlr);
-			};
+			}
 		} else if (fcoe_ctlr_link_down(ctlr)) {
 			switch (cdev->enabled) {
 			case FCOE_CTLR_DISABLED:
@@ -965,7 +965,7 @@ static void bnx2fc_indicate_netevent(void *context, unsigned long event,
 				put_cpu();
 				fcoe_clean_pending_queue(lport);
 				wait_for_upload = 1;
-			};
+			}
 		}
 	}
 	mutex_unlock(&bnx2fc_dev_lock);
diff --git a/drivers/scsi/bnx2fc/bnx2fc_io.c b/drivers/scsi/bnx2fc/bnx2fc_io.c
index 2b070f0835df..1aba5897ccb0 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_io.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_io.c
@@ -1081,6 +1081,7 @@ int bnx2fc_eh_device_reset(struct scsi_cmnd *sc_cmd)
 }
 
 static int bnx2fc_abts_cleanup(struct bnx2fc_cmd *io_req)
+	__must_hold(&tgt->tgt_lock)
 {
 	struct bnx2fc_rport *tgt = io_req->tgt;
 	unsigned int time_left;
diff --git a/drivers/scsi/bnx2i/Kconfig b/drivers/scsi/bnx2i/Kconfig
index 702dc82c9501..75ace2302fed 100644
--- a/drivers/scsi/bnx2i/Kconfig
+++ b/drivers/scsi/bnx2i/Kconfig
@@ -9,6 +9,6 @@ config SCSI_BNX2_ISCSI
 	select ETHERNET
 	select NET_VENDOR_BROADCOM
 	select CNIC
-	---help---
+	help
 	This driver supports iSCSI offload for the QLogic NetXtreme II
 	devices.
diff --git a/drivers/scsi/cxgbi/cxgb3i/Kconfig b/drivers/scsi/cxgbi/cxgb3i/Kconfig
index 3e4b644249cb..e20e6f3bfe64 100644
--- a/drivers/scsi/cxgbi/cxgb3i/Kconfig
+++ b/drivers/scsi/cxgbi/cxgb3i/Kconfig
@@ -8,5 +8,5 @@ config SCSI_CXGB3_ISCSI
 	select CHELSIO_T3
 	select CHELSIO_LIB
 	select SCSI_ISCSI_ATTRS
-	---help---
+	help
 	  This driver supports iSCSI offload for the Chelsio T3 devices.
diff --git a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
index 524cdbcd29aa..ec7d01f6e2d5 100644
--- a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
+++ b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
@@ -959,6 +959,7 @@ static int init_act_open(struct cxgbi_sock *csk)
 	struct net_device *ndev = cdev->ports[csk->port_id];
 	struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
 	struct sk_buff *skb = NULL;
+	int ret;
 
 	log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 		"csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
@@ -979,16 +980,16 @@ static int init_act_open(struct cxgbi_sock *csk)
 	csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
 	if (csk->atid < 0) {
 		pr_err("NO atid available.\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto put_sock;
 	}
 	cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
 	cxgbi_sock_get(csk);
 
 	skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
 	if (!skb) {
-		cxgb3_free_atid(t3dev, csk->atid);
-		cxgbi_sock_put(csk);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto free_atid;
 	}
 	skb->sk = (struct sock *)csk;
 	set_arp_failure_handler(skb, act_open_arp_failure);
@@ -1010,6 +1011,15 @@ static int init_act_open(struct cxgbi_sock *csk)
 	cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
 	send_act_open_req(csk, skb, csk->l2t);
 	return 0;
+
+free_atid:
+	cxgb3_free_atid(t3dev, csk->atid);
+put_sock:
+	cxgbi_sock_put(csk);
+	l2t_release(t3dev, csk->l2t);
+	csk->l2t = NULL;
+
+	return ret;
 }
 
 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
diff --git a/drivers/scsi/cxgbi/cxgb4i/Kconfig b/drivers/scsi/cxgbi/cxgb4i/Kconfig
index d1f1baba3285..b206e266b4e7 100644
--- a/drivers/scsi/cxgbi/cxgb4i/Kconfig
+++ b/drivers/scsi/cxgbi/cxgb4i/Kconfig
@@ -8,5 +8,5 @@ config SCSI_CXGB4_ISCSI
 	select CHELSIO_T4
 	select CHELSIO_LIB
 	select SCSI_ISCSI_ATTRS
-	---help---
+	help
 	  This driver supports iSCSI offload for the Chelsio T4 devices.
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
index bc1086ae6835..8ce8592f6a64 100644
--- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -1127,10 +1127,9 @@ static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
 	if (!csk)
 		goto rel_skb;
 
-	if (csk)
-		pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
-			       (&csk->saddr), (&csk->daddr), csk,
-			       csk->state, csk->flags, csk->tid, rpl->status);
+	pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
+		       (&csk->saddr), (&csk->daddr), csk,
+		       csk->state, csk->flags, csk->tid, rpl->status);
 
 	if (rpl->status == CPL_ERR_ABORT_FAILED)
 		goto rel_skb;
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c
index fbd2ae40dab4..94250ebe9e80 100644
--- a/drivers/scsi/cxlflash/main.c
+++ b/drivers/scsi/cxlflash/main.c
@@ -47,9 +47,6 @@ static void process_cmd_err(struct afu_cmd *cmd, struct scsi_cmnd *scp)
 	struct sisl_ioasa *ioasa;
 	u32 resid;
 
-	if (unlikely(!cmd))
-		return;
-
 	ioasa = &(cmd->sa);
 
 	if (ioasa->rc.flags & SISL_RC_FLAGS_UNDERRUN) {
@@ -3744,6 +3741,7 @@ static int cxlflash_probe(struct pci_dev *pdev,
 	cfg->afu_cookie = cfg->ops->create_afu(pdev);
 	if (unlikely(!cfg->afu_cookie)) {
 		dev_err(dev, "%s: create_afu failed\n", __func__);
+		rc = -ENOMEM;
 		goto out_remove;
 	}
 
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index 02dff3a684e0..0497ef6a9453 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -27,6 +27,7 @@
 /*#define UARTDELAY 1 */
 
 #include <linux/module.h>
+#include <linux/pgtable.h>
 
 MODULE_AUTHOR("Deanna Bonds, with _lots_ of help from Mark Salyzyn");
 MODULE_DESCRIPTION("Adaptec I2O RAID Driver");
@@ -55,7 +56,6 @@ MODULE_DESCRIPTION("Adaptec I2O RAID Driver");
 #include <linux/mutex.h>
 
 #include <asm/processor.h>	/* for boot_cpu_data */
-#include <asm/pgtable.h>
 #include <asm/io.h>		/* for virt_to_bus, etc. */
 
 #include <scsi/scsi.h>
@@ -1120,7 +1120,7 @@ static struct adpt_device* adpt_find_device(adpt_hba* pHba, u32 chan, u32 id, u6
 {
 	struct adpt_device* d;
 
-	if(chan < 0 || chan >= MAX_CHANNEL)
+	if (chan >= MAX_CHANNEL)
 		return NULL;
 	
 	d = pHba->channel[chan].device[id];
diff --git a/drivers/scsi/esas2r/Kconfig b/drivers/scsi/esas2r/Kconfig
index 19f6d3029658..c9b43f7fce31 100644
--- a/drivers/scsi/esas2r/Kconfig
+++ b/drivers/scsi/esas2r/Kconfig
@@ -2,5 +2,5 @@
 config SCSI_ESAS2R
 	tristate "ATTO Technology's ExpressSAS RAID adapter driver"
 	depends on PCI && SCSI
-	---help---
+	help
 	  This driver supports the ATTO ExpressSAS R6xx SAS/SATA RAID controllers.
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 25dae9f0b205..cb41d166e0c0 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -1915,7 +1915,7 @@ static int fcoe_device_notification(struct notifier_block *notifier,
 		case FCOE_CTLR_ENABLED:
 		case FCOE_CTLR_UNUSED:
 			fcoe_ctlr_link_up(ctlr);
-		};
+		}
 	} else if (fcoe_ctlr_link_down(ctlr)) {
 		switch (cdev->enabled) {
 		case FCOE_CTLR_DISABLED:
@@ -1927,7 +1927,7 @@ static int fcoe_device_notification(struct notifier_block *notifier,
 			stats->LinkFailureCount++;
 			put_cpu();
 			fcoe_clean_pending_queue(lport);
-		};
+		}
 	}
 out:
 	return rc;
diff --git a/drivers/scsi/fnic/fnic_main.c b/drivers/scsi/fnic/fnic_main.c
index 18584ab27c32..7910b573bacb 100644
--- a/drivers/scsi/fnic/fnic_main.c
+++ b/drivers/scsi/fnic/fnic_main.c
@@ -49,8 +49,8 @@
 
 static struct kmem_cache *fnic_sgl_cache[FNIC_SGL_NUM_CACHES];
 static struct kmem_cache *fnic_io_req_cache;
-LIST_HEAD(fnic_list);
-DEFINE_SPINLOCK(fnic_list_lock);
+static LIST_HEAD(fnic_list);
+static DEFINE_SPINLOCK(fnic_list_lock);
 
 /* Supported devices by fnic module */
 static struct pci_device_id fnic_id_table[] = {
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
index b60795893994..27535c90b248 100644
--- a/drivers/scsi/fnic/fnic_scsi.c
+++ b/drivers/scsi/fnic/fnic_scsi.c
@@ -2624,8 +2624,8 @@ int fnic_host_reset(struct scsi_cmnd *sc)
 	unsigned long flags;
 
 	spin_lock_irqsave(&fnic->fnic_lock, flags);
-	if (fnic->internal_reset_inprogress == 0) {
-		fnic->internal_reset_inprogress = 1;
+	if (!fnic->internal_reset_inprogress) {
+		fnic->internal_reset_inprogress = true;
 	} else {
 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
@@ -2654,7 +2654,7 @@ int fnic_host_reset(struct scsi_cmnd *sc)
 	}
 
 	spin_lock_irqsave(&fnic->fnic_lock, flags);
-	fnic->internal_reset_inprogress = 0;
+	fnic->internal_reset_inprogress = false;
 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);
 	return ret;
 }
diff --git a/drivers/scsi/fnic/vnic_dev.c b/drivers/scsi/fnic/vnic_dev.c
index 1b88a3b53eee..a2beee6e09f0 100644
--- a/drivers/scsi/fnic/vnic_dev.c
+++ b/drivers/scsi/fnic/vnic_dev.c
@@ -254,7 +254,7 @@ void vnic_dev_free_desc_ring(struct vnic_dev *vdev, struct vnic_dev_ring *ring)
 	}
 }
 
-int vnic_dev_cmd1(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, int wait)
+static int vnic_dev_cmd1(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, int wait)
 {
 	struct vnic_devcmd __iomem *devcmd = vdev->devcmd;
 	int delay;
@@ -316,7 +316,7 @@ int vnic_dev_cmd1(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd, int wait)
 	return -ETIMEDOUT;
 }
 
-int vnic_dev_cmd2(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd,
+static int vnic_dev_cmd2(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd,
 		int wait)
 {
 	struct devcmd2_controller *dc2c = vdev->devcmd2;
@@ -411,7 +411,7 @@ int vnic_dev_cmd2(struct vnic_dev *vdev, enum vnic_devcmd_cmd cmd,
 }
 
 
-int vnic_dev_init_devcmd1(struct vnic_dev *vdev)
+static int vnic_dev_init_devcmd1(struct vnic_dev *vdev)
 {
 	vdev->devcmd = vnic_dev_get_res(vdev, RES_TYPE_DEVCMD, 0);
 	if (!vdev->devcmd)
@@ -422,7 +422,7 @@ int vnic_dev_init_devcmd1(struct vnic_dev *vdev)
 }
 
 
-int vnic_dev_init_devcmd2(struct vnic_dev *vdev)
+static int vnic_dev_init_devcmd2(struct vnic_dev *vdev)
 {
 	int err;
 	unsigned int fetch_index;
@@ -492,7 +492,7 @@ err_free_devcmd2:
 }
 
 
-void vnic_dev_deinit_devcmd2(struct vnic_dev *vdev)
+static void vnic_dev_deinit_devcmd2(struct vnic_dev *vdev)
 {
 	vnic_dev_free_desc_ring(vdev, &vdev->devcmd2->results_ring);
 	vnic_wq_disable(&vdev->devcmd2->wq);
@@ -503,7 +503,7 @@ void vnic_dev_deinit_devcmd2(struct vnic_dev *vdev)
 }
 
 
-int vnic_dev_cmd_no_proxy(struct vnic_dev *vdev,
+static int vnic_dev_cmd_no_proxy(struct vnic_dev *vdev,
 	enum vnic_devcmd_cmd cmd, u64 *a0, u64 *a1, int wait)
 {
 	int err;
diff --git a/drivers/scsi/fnic/vnic_wq.c b/drivers/scsi/fnic/vnic_wq.c
index 015af2cdabaf..442972c04e65 100644
--- a/drivers/scsi/fnic/vnic_wq.c
+++ b/drivers/scsi/fnic/vnic_wq.c
@@ -25,7 +25,7 @@
 #include "vnic_wq.h"
 
 
-int vnic_wq_get_ctrl(struct vnic_dev *vdev, struct vnic_wq *wq,
+static int vnic_wq_get_ctrl(struct vnic_dev *vdev, struct vnic_wq *wq,
 		unsigned int index, enum vnic_res_type res_type)
 {
 	wq->ctrl = vnic_dev_get_res(vdev, res_type, index);
@@ -37,7 +37,7 @@ int vnic_wq_get_ctrl(struct vnic_dev *vdev, struct vnic_wq *wq,
 }
 
 
-int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq,
+static int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq,
 		unsigned int desc_count, unsigned int desc_size)
 {
 	return vnic_dev_alloc_desc_ring(vdev, &wq->ring, desc_count, desc_size);
diff --git a/drivers/scsi/gdth.c b/drivers/scsi/gdth.c
index fe03410268e6..7f150d52b4a6 100644
--- a/drivers/scsi/gdth.c
+++ b/drivers/scsi/gdth.c
@@ -329,8 +329,8 @@ static void gdth_scsi_done(struct scsi_cmnd *scp)
 		scp->scsi_done(scp);
 }
 
-int __gdth_execute(struct scsi_device *sdev, gdth_cmd_str *gdtcmd, char *cmnd,
-                   int timeout, u32 *info)
+static int __gdth_execute(struct scsi_device *sdev, gdth_cmd_str *gdtcmd,
+			  char *cmnd, int timeout, u32 *info)
 {
     gdth_ha_str *ha = shost_priv(sdev->host);
     struct scsi_cmnd *scp;
diff --git a/drivers/scsi/gvp11.c b/drivers/scsi/gvp11.c
index 11df0eca0293..727f8c8f30b5 100644
--- a/drivers/scsi/gvp11.c
+++ b/drivers/scsi/gvp11.c
@@ -9,7 +9,6 @@
 #include <linux/module.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/amigaints.h>
 #include <asm/amigahw.h>
 
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 9a6deb21fe4d..11caa4b0d797 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -898,8 +898,11 @@ void hisi_sas_phy_oob_ready(struct hisi_hba *hisi_hba, int phy_no)
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	struct device *dev = hisi_hba->dev;
 
+	dev_dbg(dev, "phy%d OOB ready\n", phy_no);
+	if (phy->phy_attached)
+		return;
+
 	if (!timer_pending(&phy->timer)) {
-		dev_dbg(dev, "phy%d OOB ready\n", phy_no);
 		phy->timer.expires = jiffies + HISI_SAS_WAIT_PHYUP_TIMEOUT * HZ;
 		add_timer(&phy->timer);
 	}
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index fa25766502a2..2e1718f9ade2 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -1175,15 +1175,14 @@ static void slot_err_v1_hw(struct hisi_hba *hisi_hba,
 
 }
 
-static int slot_complete_v1_hw(struct hisi_hba *hisi_hba,
-			       struct hisi_sas_slot *slot)
+static void slot_complete_v1_hw(struct hisi_hba *hisi_hba,
+				struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_device *sas_dev;
 	struct device *dev = hisi_hba->dev;
 	struct task_status_struct *ts;
 	struct domain_device *device;
-	enum exec_status sts;
 	struct hisi_sas_complete_v1_hdr *complete_queue =
 			hisi_hba->complete_hdr[slot->cmplt_queue];
 	struct hisi_sas_complete_v1_hdr *complete_hdr;
@@ -1194,7 +1193,7 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba,
 	cmplt_hdr_data = le32_to_cpu(complete_hdr->data);
 
 	if (unlikely(!task || !task->lldd_task || !task->dev))
-		return -EINVAL;
+		return;
 
 	ts = &task->task_status;
 	device = task->dev;
@@ -1260,7 +1259,7 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba,
 
 		slot_err_v1_hw(hisi_hba, task, slot);
 		if (unlikely(slot->abort))
-			return ts->stat;
+			return;
 		goto out;
 	}
 
@@ -1309,12 +1308,9 @@ static int slot_complete_v1_hw(struct hisi_hba *hisi_hba,
 
 out:
 	hisi_sas_slot_task_free(hisi_hba, task, slot);
-	sts = ts->stat;
 
 	if (task->task_done)
 		task->task_done(task);
-
-	return sts;
 }
 
 /* Interrupts */
@@ -1757,6 +1753,7 @@ static struct device_attribute *host_attrs_v1_hw[] = {
 
 static struct scsi_host_template sht_v1_hw = {
 	.name			= DRV_NAME,
+	.proc_name		= DRV_NAME,
 	.module			= THIS_MODULE,
 	.queuecommand		= sas_queuecommand,
 	.target_alloc		= sas_target_alloc,
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index e05faf315dcd..e7e7849a4c14 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -2318,8 +2318,8 @@ static void slot_err_v2_hw(struct hisi_hba *hisi_hba,
 	}
 }
 
-static int
-slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
+static void slot_complete_v2_hw(struct hisi_hba *hisi_hba,
+				struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_device *sas_dev;
@@ -2327,7 +2327,6 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	struct task_status_struct *ts;
 	struct domain_device *device;
 	struct sas_ha_struct *ha;
-	enum exec_status sts;
 	struct hisi_sas_complete_v2_hdr *complete_queue =
 			hisi_hba->complete_hdr[slot->cmplt_queue];
 	struct hisi_sas_complete_v2_hdr *complete_hdr =
@@ -2337,7 +2336,7 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	u32 dw0;
 
 	if (unlikely(!task || !task->lldd_task || !task->dev))
-		return -EINVAL;
+		return;
 
 	ts = &task->task_status;
 	device = task->dev;
@@ -2406,7 +2405,7 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 				 error_info[2], error_info[3]);
 
 		if (unlikely(slot->abort))
-			return ts->stat;
+			return;
 		goto out;
 	}
 
@@ -2456,12 +2455,11 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	}
 
 out:
-	sts = ts->stat;
 	spin_lock_irqsave(&task->task_state_lock, flags);
 	if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
 		spin_unlock_irqrestore(&task->task_state_lock, flags);
 		dev_info(dev, "slot complete: task(%pK) aborted\n", task);
-		return SAS_ABORTED_TASK;
+		return;
 	}
 	task->task_state_flags |= SAS_TASK_STATE_DONE;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
@@ -2473,15 +2471,13 @@ out:
 			spin_unlock_irqrestore(&device->done_lock, flags);
 			dev_info(dev, "slot complete: task(%pK) ignored\n",
 				 task);
-			return sts;
+			return;
 		}
 		spin_unlock_irqrestore(&device->done_lock, flags);
 	}
 
 	if (task->task_done)
 		task->task_done(task);
-
-	return sts;
 }
 
 static void prep_ata_v2_hw(struct hisi_hba *hisi_hba,
@@ -3533,6 +3529,7 @@ static struct device_attribute *host_attrs_v2_hw[] = {
 
 static struct scsi_host_template sht_v2_hw = {
 	.name			= DRV_NAME,
+	.proc_name		= DRV_NAME,
 	.module			= THIS_MODULE,
 	.queuecommand		= sas_queuecommand,
 	.target_alloc		= sas_target_alloc,
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index 374885aa8d77..3e6b78a1f993 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -912,11 +912,15 @@ static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 		return -EINVAL;
 	}
 
-	/* Switch over to MSI handling , from PCI AER default */
+	/*
+	 * This DSM handles some hardware-related configurations:
+	 * 1. Switch over to MSI error handling in kernel
+	 * 2. BIOS *may* reset some register values through this method
+	 */
 	obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid, 0,
 				DSM_FUNC_ERR_HANDLE_MSI, NULL);
 	if (!obj)
-		dev_warn(dev, "Switch over to MSI handling failed\n");
+		dev_warn(dev, "can not find DSM method, ignore\n");
 	else
 		ACPI_FREE(obj);
 
@@ -2152,8 +2156,8 @@ slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
 	}
 }
 
-static int
-slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
+static void slot_complete_v3_hw(struct hisi_hba *hisi_hba,
+				struct hisi_sas_slot *slot)
 {
 	struct sas_task *task = slot->task;
 	struct hisi_sas_device *sas_dev;
@@ -2161,7 +2165,6 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	struct task_status_struct *ts;
 	struct domain_device *device;
 	struct sas_ha_struct *ha;
-	enum exec_status sts;
 	struct hisi_sas_complete_v3_hdr *complete_queue =
 			hisi_hba->complete_hdr[slot->cmplt_queue];
 	struct hisi_sas_complete_v3_hdr *complete_hdr =
@@ -2171,7 +2174,7 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	u32 dw0, dw1, dw3;
 
 	if (unlikely(!task || !task->lldd_task || !task->dev))
-		return -EINVAL;
+		return;
 
 	ts = &task->task_status;
 	device = task->dev;
@@ -2233,7 +2236,7 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 				 error_info[0], error_info[1],
 				 error_info[2], error_info[3]);
 		if (unlikely(slot->abort))
-			return ts->stat;
+			return;
 		goto out;
 	}
 
@@ -2278,12 +2281,11 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 	}
 
 out:
-	sts = ts->stat;
 	spin_lock_irqsave(&task->task_state_lock, flags);
 	if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
 		spin_unlock_irqrestore(&task->task_state_lock, flags);
 		dev_info(dev, "slot complete: task(%pK) aborted\n", task);
-		return SAS_ABORTED_TASK;
+		return;
 	}
 	task->task_state_flags |= SAS_TASK_STATE_DONE;
 	spin_unlock_irqrestore(&task->task_state_lock, flags);
@@ -2295,15 +2297,13 @@ out:
 			spin_unlock_irqrestore(&device->done_lock, flags);
 			dev_info(dev, "slot complete: task(%pK) ignored\n ",
 				 task);
-			return sts;
+			return;
 		}
 		spin_unlock_irqrestore(&device->done_lock, flags);
 	}
 
 	if (task->task_done)
 		task->task_done(task);
-
-	return sts;
 }
 
 static irqreturn_t  cq_thread_v3_hw(int irq_no, void *p)
@@ -2897,6 +2897,7 @@ static const struct hisi_sas_debugfs_reg debugfs_axi_reg = {
 };
 
 static const struct hisi_sas_debugfs_reg_lu debugfs_ras_reg_lu[] = {
+	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0),
 	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1),
 	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR0_MASK),
 	HISI_SAS_DEBUGFS_REG(SAS_RAS_INTR1_MASK),
@@ -3071,6 +3072,7 @@ static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable)
 
 static struct scsi_host_template sht_v3_hw = {
 	.name			= DRV_NAME,
+	.proc_name		= DRV_NAME,
 	.module			= THIS_MODULE,
 	.queuecommand		= sas_queuecommand,
 	.target_alloc		= sas_target_alloc,
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index 1e9302e99d05..81d0414e2117 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -254,6 +254,10 @@ static irqreturn_t do_hpsa_intr_intx(int irq, void *dev_id);
 static irqreturn_t do_hpsa_intr_msi(int irq, void *dev_id);
 static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd,
 		      void __user *arg);
+static int hpsa_passthru_ioctl(struct ctlr_info *h,
+			       IOCTL_Command_struct *iocommand);
+static int hpsa_big_passthru_ioctl(struct ctlr_info *h,
+				   BIG_IOCTL_Command_struct *ioc);
 
 #ifdef CONFIG_COMPAT
 static int hpsa_compat_ioctl(struct scsi_device *dev, unsigned int cmd,
@@ -6217,75 +6221,63 @@ static void cmd_free(struct ctlr_info *h, struct CommandList *c)
 static int hpsa_ioctl32_passthru(struct scsi_device *dev, unsigned int cmd,
 	void __user *arg)
 {
-	IOCTL32_Command_struct __user *arg32 =
-	    (IOCTL32_Command_struct __user *) arg;
+	struct ctlr_info *h = sdev_to_hba(dev);
+	IOCTL32_Command_struct __user *arg32 = arg;
 	IOCTL_Command_struct arg64;
-	IOCTL_Command_struct __user *p = compat_alloc_user_space(sizeof(arg64));
 	int err;
 	u32 cp;
 
-	memset(&arg64, 0, sizeof(arg64));
-	err = 0;
-	err |= copy_from_user(&arg64.LUN_info, &arg32->LUN_info,
-			   sizeof(arg64.LUN_info));
-	err |= copy_from_user(&arg64.Request, &arg32->Request,
-			   sizeof(arg64.Request));
-	err |= copy_from_user(&arg64.error_info, &arg32->error_info,
-			   sizeof(arg64.error_info));
-	err |= get_user(arg64.buf_size, &arg32->buf_size);
-	err |= get_user(cp, &arg32->buf);
-	arg64.buf = compat_ptr(cp);
-	err |= copy_to_user(p, &arg64, sizeof(arg64));
+	if (!arg)
+		return -EINVAL;
 
-	if (err)
+	memset(&arg64, 0, sizeof(arg64));
+	if (copy_from_user(&arg64, arg32, offsetof(IOCTL_Command_struct, buf)))
+		return -EFAULT;
+	if (get_user(cp, &arg32->buf))
 		return -EFAULT;
+	arg64.buf = compat_ptr(cp);
 
-	err = hpsa_ioctl(dev, CCISS_PASSTHRU, p);
+	if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0)
+		return -EAGAIN;
+	err = hpsa_passthru_ioctl(h, &arg64);
+	atomic_inc(&h->passthru_cmds_avail);
 	if (err)
 		return err;
-	err |= copy_in_user(&arg32->error_info, &p->error_info,
-			 sizeof(arg32->error_info));
-	if (err)
+	if (copy_to_user(&arg32->error_info, &arg64.error_info,
+			 sizeof(arg32->error_info)))
 		return -EFAULT;
-	return err;
+	return 0;
 }
 
 static int hpsa_ioctl32_big_passthru(struct scsi_device *dev,
 	unsigned int cmd, void __user *arg)
 {
-	BIG_IOCTL32_Command_struct __user *arg32 =
-	    (BIG_IOCTL32_Command_struct __user *) arg;
+	struct ctlr_info *h = sdev_to_hba(dev);
+	BIG_IOCTL32_Command_struct __user *arg32 = arg;
 	BIG_IOCTL_Command_struct arg64;
-	BIG_IOCTL_Command_struct __user *p =
-	    compat_alloc_user_space(sizeof(arg64));
 	int err;
 	u32 cp;
 
+	if (!arg)
+		return -EINVAL;
 	memset(&arg64, 0, sizeof(arg64));
-	err = 0;
-	err |= copy_from_user(&arg64.LUN_info, &arg32->LUN_info,
-			   sizeof(arg64.LUN_info));
-	err |= copy_from_user(&arg64.Request, &arg32->Request,
-			   sizeof(arg64.Request));
-	err |= copy_from_user(&arg64.error_info, &arg32->error_info,
-			   sizeof(arg64.error_info));
-	err |= get_user(arg64.buf_size, &arg32->buf_size);
-	err |= get_user(arg64.malloc_size, &arg32->malloc_size);
-	err |= get_user(cp, &arg32->buf);
-	arg64.buf = compat_ptr(cp);
-	err |= copy_to_user(p, &arg64, sizeof(arg64));
-
-	if (err)
+	if (copy_from_user(&arg64, arg32,
+			   offsetof(BIG_IOCTL32_Command_struct, buf)))
+		return -EFAULT;
+	if (get_user(cp, &arg32->buf))
 		return -EFAULT;
+	arg64.buf = compat_ptr(cp);
 
-	err = hpsa_ioctl(dev, CCISS_BIG_PASSTHRU, p);
+	if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0)
+		return -EAGAIN;
+	err = hpsa_big_passthru_ioctl(h, &arg64);
+	atomic_inc(&h->passthru_cmds_avail);
 	if (err)
 		return err;
-	err |= copy_in_user(&arg32->error_info, &p->error_info,
-			 sizeof(arg32->error_info));
-	if (err)
+	if (copy_to_user(&arg32->error_info, &arg64.error_info,
+			 sizeof(arg32->error_info)))
 		return -EFAULT;
-	return err;
+	return 0;
 }
 
 static int hpsa_compat_ioctl(struct scsi_device *dev, unsigned int cmd,
@@ -6358,37 +6350,33 @@ static int hpsa_getdrivver_ioctl(struct ctlr_info *h, void __user *argp)
 	return 0;
 }
 
-static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp)
+static int hpsa_passthru_ioctl(struct ctlr_info *h,
+			       IOCTL_Command_struct *iocommand)
 {
-	IOCTL_Command_struct iocommand;
 	struct CommandList *c;
 	char *buff = NULL;
 	u64 temp64;
 	int rc = 0;
 
-	if (!argp)
-		return -EINVAL;
 	if (!capable(CAP_SYS_RAWIO))
 		return -EPERM;
-	if (copy_from_user(&iocommand, argp, sizeof(iocommand)))
-		return -EFAULT;
-	if ((iocommand.buf_size < 1) &&
-	    (iocommand.Request.Type.Direction != XFER_NONE)) {
+	if ((iocommand->buf_size < 1) &&
+	    (iocommand->Request.Type.Direction != XFER_NONE)) {
 		return -EINVAL;
 	}
-	if (iocommand.buf_size > 0) {
-		buff = kmalloc(iocommand.buf_size, GFP_KERNEL);
+	if (iocommand->buf_size > 0) {
+		buff = kmalloc(iocommand->buf_size, GFP_KERNEL);
 		if (buff == NULL)
 			return -ENOMEM;
-		if (iocommand.Request.Type.Direction & XFER_WRITE) {
+		if (iocommand->Request.Type.Direction & XFER_WRITE) {
 			/* Copy the data into the buffer we created */
-			if (copy_from_user(buff, iocommand.buf,
-				iocommand.buf_size)) {
+			if (copy_from_user(buff, iocommand->buf,
+				iocommand->buf_size)) {
 				rc = -EFAULT;
 				goto out_kfree;
 			}
 		} else {
-			memset(buff, 0, iocommand.buf_size);
+			memset(buff, 0, iocommand->buf_size);
 		}
 	}
 	c = cmd_alloc(h);
@@ -6398,23 +6386,23 @@ static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp)
 	c->scsi_cmd = SCSI_CMD_BUSY;
 	/* Fill in Command Header */
 	c->Header.ReplyQueue = 0; /* unused in simple mode */
-	if (iocommand.buf_size > 0) {	/* buffer to fill */
+	if (iocommand->buf_size > 0) {	/* buffer to fill */
 		c->Header.SGList = 1;
 		c->Header.SGTotal = cpu_to_le16(1);
 	} else	{ /* no buffers to fill */
 		c->Header.SGList = 0;
 		c->Header.SGTotal = cpu_to_le16(0);
 	}
-	memcpy(&c->Header.LUN, &iocommand.LUN_info, sizeof(c->Header.LUN));
+	memcpy(&c->Header.LUN, &iocommand->LUN_info, sizeof(c->Header.LUN));
 
 	/* Fill in Request block */
-	memcpy(&c->Request, &iocommand.Request,
+	memcpy(&c->Request, &iocommand->Request,
 		sizeof(c->Request));
 
 	/* Fill in the scatter gather information */
-	if (iocommand.buf_size > 0) {
+	if (iocommand->buf_size > 0) {
 		temp64 = dma_map_single(&h->pdev->dev, buff,
-			iocommand.buf_size, DMA_BIDIRECTIONAL);
+			iocommand->buf_size, DMA_BIDIRECTIONAL);
 		if (dma_mapping_error(&h->pdev->dev, (dma_addr_t) temp64)) {
 			c->SG[0].Addr = cpu_to_le64(0);
 			c->SG[0].Len = cpu_to_le32(0);
@@ -6422,12 +6410,12 @@ static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp)
 			goto out;
 		}
 		c->SG[0].Addr = cpu_to_le64(temp64);
-		c->SG[0].Len = cpu_to_le32(iocommand.buf_size);
+		c->SG[0].Len = cpu_to_le32(iocommand->buf_size);
 		c->SG[0].Ext = cpu_to_le32(HPSA_SG_LAST); /* not chaining */
 	}
 	rc = hpsa_scsi_do_simple_cmd(h, c, DEFAULT_REPLY_QUEUE,
 					NO_TIMEOUT);
-	if (iocommand.buf_size > 0)
+	if (iocommand->buf_size > 0)
 		hpsa_pci_unmap(h->pdev, c, 1, DMA_BIDIRECTIONAL);
 	check_ioctl_unit_attention(h, c);
 	if (rc) {
@@ -6436,16 +6424,12 @@ static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp)
 	}
 
 	/* Copy the error information out */
-	memcpy(&iocommand.error_info, c->err_info,
-		sizeof(iocommand.error_info));
-	if (copy_to_user(argp, &iocommand, sizeof(iocommand))) {
-		rc = -EFAULT;
-		goto out;
-	}
-	if ((iocommand.Request.Type.Direction & XFER_READ) &&
-		iocommand.buf_size > 0) {
+	memcpy(&iocommand->error_info, c->err_info,
+		sizeof(iocommand->error_info));
+	if ((iocommand->Request.Type.Direction & XFER_READ) &&
+		iocommand->buf_size > 0) {
 		/* Copy the data out of the buffer we created */
-		if (copy_to_user(iocommand.buf, buff, iocommand.buf_size)) {
+		if (copy_to_user(iocommand->buf, buff, iocommand->buf_size)) {
 			rc = -EFAULT;
 			goto out;
 		}
@@ -6457,9 +6441,9 @@ out_kfree:
 	return rc;
 }
 
-static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp)
+static int hpsa_big_passthru_ioctl(struct ctlr_info *h,
+				   BIG_IOCTL_Command_struct *ioc)
 {
-	BIG_IOCTL_Command_struct *ioc;
 	struct CommandList *c;
 	unsigned char **buff = NULL;
 	int *buff_size = NULL;
@@ -6470,29 +6454,17 @@ static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp)
 	u32 sz;
 	BYTE __user *data_ptr;
 
-	if (!argp)
-		return -EINVAL;
 	if (!capable(CAP_SYS_RAWIO))
 		return -EPERM;
-	ioc = vmemdup_user(argp, sizeof(*ioc));
-	if (IS_ERR(ioc)) {
-		status = PTR_ERR(ioc);
-		goto cleanup1;
-	}
+
 	if ((ioc->buf_size < 1) &&
-	    (ioc->Request.Type.Direction != XFER_NONE)) {
-		status = -EINVAL;
-		goto cleanup1;
-	}
+	    (ioc->Request.Type.Direction != XFER_NONE))
+		return -EINVAL;
 	/* Check kmalloc limits  using all SGs */
-	if (ioc->malloc_size > MAX_KMALLOC_SIZE) {
-		status = -EINVAL;
-		goto cleanup1;
-	}
-	if (ioc->buf_size > ioc->malloc_size * SG_ENTRIES_IN_CMD) {
-		status = -EINVAL;
-		goto cleanup1;
-	}
+	if (ioc->malloc_size > MAX_KMALLOC_SIZE)
+		return -EINVAL;
+	if (ioc->buf_size > ioc->malloc_size * SG_ENTRIES_IN_CMD)
+		return -EINVAL;
 	buff = kcalloc(SG_ENTRIES_IN_CMD, sizeof(char *), GFP_KERNEL);
 	if (!buff) {
 		status = -ENOMEM;
@@ -6565,10 +6537,6 @@ static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp)
 
 	/* Copy the error information out */
 	memcpy(&ioc->error_info, c->err_info, sizeof(ioc->error_info));
-	if (copy_to_user(argp, ioc, sizeof(*ioc))) {
-		status = -EFAULT;
-		goto cleanup0;
-	}
 	if ((ioc->Request.Type.Direction & XFER_READ) && ioc->buf_size > 0) {
 		int i;
 
@@ -6594,7 +6562,6 @@ cleanup1:
 		kfree(buff);
 	}
 	kfree(buff_size);
-	kvfree(ioc);
 	return status;
 }
 
@@ -6610,14 +6577,11 @@ static void check_ioctl_unit_attention(struct ctlr_info *h,
  * ioctl
  */
 static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd,
-		      void __user *arg)
+		      void __user *argp)
 {
-	struct ctlr_info *h;
-	void __user *argp = (void __user *)arg;
+	struct ctlr_info *h = sdev_to_hba(dev);
 	int rc;
 
-	h = sdev_to_hba(dev);
-
 	switch (cmd) {
 	case CCISS_DEREGDISK:
 	case CCISS_REGNEWDISK:
@@ -6628,18 +6592,35 @@ static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd,
 		return hpsa_getpciinfo_ioctl(h, argp);
 	case CCISS_GETDRIVVER:
 		return hpsa_getdrivver_ioctl(h, argp);
-	case CCISS_PASSTHRU:
+	case CCISS_PASSTHRU: {
+		IOCTL_Command_struct iocommand;
+
+		if (!argp)
+			return -EINVAL;
+		if (copy_from_user(&iocommand, argp, sizeof(iocommand)))
+			return -EFAULT;
 		if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0)
 			return -EAGAIN;
-		rc = hpsa_passthru_ioctl(h, argp);
+		rc = hpsa_passthru_ioctl(h, &iocommand);
 		atomic_inc(&h->passthru_cmds_avail);
+		if (!rc && copy_to_user(argp, &iocommand, sizeof(iocommand)))
+			rc = -EFAULT;
 		return rc;
-	case CCISS_BIG_PASSTHRU:
+	}
+	case CCISS_BIG_PASSTHRU: {
+		BIG_IOCTL_Command_struct ioc;
+		if (!argp)
+			return -EINVAL;
+		if (copy_from_user(&ioc, argp, sizeof(ioc)))
+			return -EFAULT;
 		if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0)
 			return -EAGAIN;
-		rc = hpsa_big_passthru_ioctl(h, argp);
+		rc = hpsa_big_passthru_ioctl(h, &ioc);
 		atomic_inc(&h->passthru_cmds_avail);
+		if (!rc && copy_to_user(argp, &ioc, sizeof(ioc)))
+			rc = -EFAULT;
 		return rc;
+	}
 	default:
 		return -ENOTTY;
 	}
diff --git a/drivers/scsi/ibmvscsi/ibmvscsi.c b/drivers/scsi/ibmvscsi/ibmvscsi.c
index 59f0f1030c54..14f687e9b1f4 100644
--- a/drivers/scsi/ibmvscsi/ibmvscsi.c
+++ b/drivers/scsi/ibmvscsi/ibmvscsi.c
@@ -415,6 +415,8 @@ static int ibmvscsi_reenable_crq_queue(struct crq_queue *queue,
 	int rc = 0;
 	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
 
+	set_adapter_info(hostdata);
+
 	/* Re-enable the CRQ */
 	do {
 		if (rc)
@@ -2384,7 +2386,7 @@ static struct vio_driver ibmvscsi_driver = {
 static struct srp_function_template ibmvscsi_transport_functions = {
 };
 
-int __init ibmvscsi_module_init(void)
+static int __init ibmvscsi_module_init(void)
 {
 	int ret;
 
@@ -2406,7 +2408,7 @@ int __init ibmvscsi_module_init(void)
 	return ret;
 }
 
-void __exit ibmvscsi_module_exit(void)
+static void __exit ibmvscsi_module_exit(void)
 {
 	vio_unregister_driver(&ibmvscsi_driver);
 	srp_release_transport(ibmvscsi_transport_template);
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index d48a8fa997b9..7d77997d26d4 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -1164,7 +1164,7 @@ static void ipr_update_ata_class(struct ipr_resource_entry *res, unsigned int pr
 	default:
 		res->ata_class = ATA_DEV_UNKNOWN;
 		break;
-	};
+	}
 }
 
 /**
@@ -9529,8 +9529,7 @@ static void ipr_free_cmd_blks(struct ipr_ioa_cfg *ioa_cfg)
 		}
 	}
 
-	if (ioa_cfg->ipr_cmd_pool)
-		dma_pool_destroy(ioa_cfg->ipr_cmd_pool);
+	dma_pool_destroy(ioa_cfg->ipr_cmd_pool);
 
 	kfree(ioa_cfg->ipr_cmnd_list);
 	kfree(ioa_cfg->ipr_cmnd_list_dma);
diff --git a/drivers/scsi/isci/isci.h b/drivers/scsi/isci/isci.h
index 680e30947671..4e6b1decbca7 100644
--- a/drivers/scsi/isci/isci.h
+++ b/drivers/scsi/isci/isci.h
@@ -500,19 +500,19 @@ struct sci_timer {
 static inline
 void sci_init_timer(struct sci_timer *tmr, void (*fn)(struct timer_list *t))
 {
-	tmr->cancel = 0;
+	tmr->cancel = false;
 	timer_setup(&tmr->timer, fn, 0);
 }
 
 static inline void sci_mod_timer(struct sci_timer *tmr, unsigned long msec)
 {
-	tmr->cancel = 0;
+	tmr->cancel = false;
 	mod_timer(&tmr->timer, jiffies + msecs_to_jiffies(msec));
 }
 
 static inline void sci_del_timer(struct sci_timer *tmr)
 {
-	tmr->cancel = 1;
+	tmr->cancel = true;
 	del_timer(&tmr->timer);
 }
 
diff --git a/drivers/scsi/iscsi_boot_sysfs.c b/drivers/scsi/iscsi_boot_sysfs.c
index e4857b728033..a64abe38db2d 100644
--- a/drivers/scsi/iscsi_boot_sysfs.c
+++ b/drivers/scsi/iscsi_boot_sysfs.c
@@ -352,7 +352,7 @@ iscsi_boot_create_kobj(struct iscsi_boot_kset *boot_kset,
 	boot_kobj->kobj.kset = boot_kset->kset;
 	if (kobject_init_and_add(&boot_kobj->kobj, &iscsi_boot_ktype,
 				 NULL, name, index)) {
-		kfree(boot_kobj);
+		kobject_put(&boot_kobj->kobj);
 		return NULL;
 	}
 	boot_kobj->data = data;
diff --git a/drivers/scsi/lasi700.c b/drivers/scsi/lasi700.c
index c48a73a0f517..de71d240a56f 100644
--- a/drivers/scsi/lasi700.c
+++ b/drivers/scsi/lasi700.c
@@ -31,7 +31,6 @@
 #include <linux/slab.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <asm/hardware.h>
 #include <asm/parisc-device.h>
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 874dd4beed10..e5a64d4f255c 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -2627,7 +2627,9 @@ struct Scsi_Host *iscsi_host_alloc(struct scsi_host_template *sht,
 	if (xmit_can_sleep) {
 		snprintf(ihost->workq_name, sizeof(ihost->workq_name),
 			"iscsi_q_%d", shost->host_no);
-		ihost->workq = create_singlethread_workqueue(ihost->workq_name);
+		ihost->workq = alloc_workqueue("%s",
+			WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_UNBOUND,
+			2, ihost->workq_name);
 		if (!ihost->workq)
 			goto free_host;
 	}
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index c5a828a041e0..5d716d388707 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -160,6 +160,7 @@ qc_already_gone:
 }
 
 static unsigned int sas_ata_qc_issue(struct ata_queued_cmd *qc)
+	__must_hold(ap->lock)
 {
 	struct sas_task *task;
 	struct scatterlist *sg;
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index 62e96d4fdcc6..c3ceb6e5b061 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -627,6 +627,19 @@ struct lpfc_ras_fwlog {
 	enum ras_state state;    /* RAS logging running state */
 };
 
+enum lpfc_irq_chann_mode {
+	/* Assign IRQs to all possible cpus that have hardware queues */
+	NORMAL_MODE,
+
+	/* Assign IRQs only to cpus on the same numa node as HBA */
+	NUMA_MODE,
+
+	/* Assign IRQs only on non-hyperthreaded CPUs. This is the
+	 * same as normal_mode, but assign IRQS only on physical CPUs.
+	 */
+	NHT_MODE,
+};
+
 struct lpfc_hba {
 	/* SCSI interface function jump table entries */
 	struct lpfc_io_buf * (*lpfc_get_scsi_buf)
@@ -835,7 +848,6 @@ struct lpfc_hba {
 	uint32_t cfg_fcp_mq_threshold;
 	uint32_t cfg_hdw_queue;
 	uint32_t cfg_irq_chann;
-	uint32_t cfg_irq_numa;
 	uint32_t cfg_suppress_rsp;
 	uint32_t cfg_nvme_oas;
 	uint32_t cfg_nvme_embed_cmd;
@@ -1003,6 +1015,7 @@ struct lpfc_hba {
 	mempool_t *active_rrq_pool;
 
 	struct fc_host_statistics link_stats;
+	enum lpfc_irq_chann_mode irq_chann_mode;
 	enum intr_type_t intr_type;
 	uint32_t intr_mode;
 #define LPFC_INTR_ERROR	0xFFFFFFFF
@@ -1314,19 +1327,19 @@ lpfc_phba_elsring(struct lpfc_hba *phba)
 }
 
 /**
- * lpfc_next_online_numa_cpu - Finds next online CPU on NUMA node
- * @numa_mask: Pointer to phba's numa_mask member.
+ * lpfc_next_online_cpu - Finds next online CPU on cpumask
+ * @mask: Pointer to phba's cpumask member.
  * @start: starting cpu index
  *
  * Note: If no valid cpu found, then nr_cpu_ids is returned.
  *
  **/
 static inline unsigned int
-lpfc_next_online_numa_cpu(const struct cpumask *numa_mask, unsigned int start)
+lpfc_next_online_cpu(const struct cpumask *mask, unsigned int start)
 {
 	unsigned int cpu_it;
 
-	for_each_cpu_wrap(cpu_it, numa_mask, start) {
+	for_each_cpu_wrap(cpu_it, mask, start) {
 		if (cpu_online(cpu_it))
 			break;
 	}
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index f089867674cb..a62c60ca6477 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -4874,7 +4874,7 @@ lpfc_request_firmware_upgrade_store(struct device *dev,
 	struct Scsi_Host *shost = class_to_shost(dev);
 	struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata;
 	struct lpfc_hba *phba = vport->phba;
-	int val = 0, rc = -EINVAL;
+	int val = 0, rc;
 
 	/* Sanity check on user data */
 	if (!isdigit(buf[0]))
@@ -5701,17 +5701,69 @@ LPFC_ATTR_R(hdw_queue,
 	    LPFC_HBA_HDWQ_MIN, LPFC_HBA_HDWQ_MAX,
 	    "Set the number of I/O Hardware Queues");
 
-static inline void
-lpfc_assign_default_irq_numa(struct lpfc_hba *phba)
+#if IS_ENABLED(CONFIG_X86)
+/**
+ * lpfc_cpumask_irq_mode_init - initalizes cpumask of phba based on
+ *				irq_chann_mode
+ * @phba: Pointer to HBA context object.
+ **/
+static void
+lpfc_cpumask_irq_mode_init(struct lpfc_hba *phba)
+{
+	unsigned int cpu, first_cpu, numa_node = NUMA_NO_NODE;
+	const struct cpumask *sibling_mask;
+	struct cpumask *aff_mask = &phba->sli4_hba.irq_aff_mask;
+
+	cpumask_clear(aff_mask);
+
+	if (phba->irq_chann_mode == NUMA_MODE) {
+		/* Check if we're a NUMA architecture */
+		numa_node = dev_to_node(&phba->pcidev->dev);
+		if (numa_node == NUMA_NO_NODE) {
+			phba->irq_chann_mode = NORMAL_MODE;
+			return;
+		}
+	}
+
+	for_each_possible_cpu(cpu) {
+		switch (phba->irq_chann_mode) {
+		case NUMA_MODE:
+			if (cpu_to_node(cpu) == numa_node)
+				cpumask_set_cpu(cpu, aff_mask);
+			break;
+		case NHT_MODE:
+			sibling_mask = topology_sibling_cpumask(cpu);
+			first_cpu = cpumask_first(sibling_mask);
+			if (first_cpu < nr_cpu_ids)
+				cpumask_set_cpu(first_cpu, aff_mask);
+			break;
+		default:
+			break;
+		}
+	}
+}
+#endif
+
+static void
+lpfc_assign_default_irq_chann(struct lpfc_hba *phba)
 {
 #if IS_ENABLED(CONFIG_X86)
-	/* If AMD architecture, then default is LPFC_IRQ_CHANN_NUMA */
-	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
-		phba->cfg_irq_numa = 1;
-	else
-		phba->cfg_irq_numa = 0;
+	switch (boot_cpu_data.x86_vendor) {
+	case X86_VENDOR_AMD:
+		/* If AMD architecture, then default is NUMA_MODE */
+		phba->irq_chann_mode = NUMA_MODE;
+		break;
+	case X86_VENDOR_INTEL:
+		/* If Intel architecture, then default is no hyperthread mode */
+		phba->irq_chann_mode = NHT_MODE;
+		break;
+	default:
+		phba->irq_chann_mode = NORMAL_MODE;
+		break;
+	}
+	lpfc_cpumask_irq_mode_init(phba);
 #else
-	phba->cfg_irq_numa = 0;
+	phba->irq_chann_mode = NORMAL_MODE;
 #endif
 }
 
@@ -5723,6 +5775,7 @@ lpfc_assign_default_irq_numa(struct lpfc_hba *phba)
  *
  *	0		= Configure number of IRQ Channels to:
  *			  if AMD architecture, number of CPUs on HBA's NUMA node
+ *			  if Intel architecture, number of physical CPUs.
  *			  otherwise, number of active CPUs.
  *	[1,256]		= Manually specify how many IRQ Channels to use.
  *
@@ -5748,35 +5801,44 @@ MODULE_PARM_DESC(lpfc_irq_chann, "Set number of interrupt vectors to allocate");
 static int
 lpfc_irq_chann_init(struct lpfc_hba *phba, uint32_t val)
 {
-	const struct cpumask *numa_mask;
+	const struct cpumask *aff_mask;
 
 	if (phba->cfg_use_msi != 2) {
 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
 				"8532 use_msi = %u ignoring cfg_irq_numa\n",
 				phba->cfg_use_msi);
-		phba->cfg_irq_numa = 0;
-		phba->cfg_irq_chann = LPFC_IRQ_CHANN_MIN;
+		phba->irq_chann_mode = NORMAL_MODE;
+		phba->cfg_irq_chann = LPFC_IRQ_CHANN_DEF;
 		return 0;
 	}
 
 	/* Check if default setting was passed */
 	if (val == LPFC_IRQ_CHANN_DEF)
-		lpfc_assign_default_irq_numa(phba);
+		lpfc_assign_default_irq_chann(phba);
 
-	if (phba->cfg_irq_numa) {
-		numa_mask = &phba->sli4_hba.numa_mask;
+	if (phba->irq_chann_mode != NORMAL_MODE) {
+		aff_mask = &phba->sli4_hba.irq_aff_mask;
 
-		if (cpumask_empty(numa_mask)) {
+		if (cpumask_empty(aff_mask)) {
 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-					"8533 Could not identify NUMA node, "
-					"ignoring cfg_irq_numa\n");
-			phba->cfg_irq_numa = 0;
-			phba->cfg_irq_chann = LPFC_IRQ_CHANN_MIN;
+					"8533 Could not identify CPUS for "
+					"mode %d, ignoring\n",
+					phba->irq_chann_mode);
+			phba->irq_chann_mode = NORMAL_MODE;
+			phba->cfg_irq_chann = LPFC_IRQ_CHANN_DEF;
 		} else {
-			phba->cfg_irq_chann = cpumask_weight(numa_mask);
+			phba->cfg_irq_chann = cpumask_weight(aff_mask);
+
+			/* If no hyperthread mode, then set hdwq count to
+			 * aff_mask weight as well
+			 */
+			if (phba->irq_chann_mode == NHT_MODE)
+				phba->cfg_hdw_queue = phba->cfg_irq_chann;
+
 			lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
 					"8543 lpfc_irq_chann set to %u "
-					"(numa)\n", phba->cfg_irq_chann);
+					"(mode: %d)\n", phba->cfg_irq_chann,
+					phba->irq_chann_mode);
 		}
 	} else {
 		if (val > LPFC_IRQ_CHANN_MAX) {
@@ -5787,7 +5849,7 @@ lpfc_irq_chann_init(struct lpfc_hba *phba, uint32_t val)
 					val,
 					LPFC_IRQ_CHANN_MIN,
 					LPFC_IRQ_CHANN_MAX);
-			phba->cfg_irq_chann = LPFC_IRQ_CHANN_MIN;
+			phba->cfg_irq_chann = LPFC_IRQ_CHANN_DEF;
 			return -EINVAL;
 		}
 		phba->cfg_irq_chann = val;
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 196f6ae9952e..69d4710d95a0 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -461,7 +461,6 @@ lpfc_prep_node_fc4type(struct lpfc_vport *vport, uint32_t Did, uint8_t fc4_type)
 	struct lpfc_nodelist *ndlp;
 
 	if ((vport->port_type != LPFC_NPIV_PORT) ||
-	    (fc4_type == FC_TYPE_FCP) ||
 	    !(vport->ct_flags & FC_CT_RFF_ID) || !vport->cfg_restrict_login) {
 
 		ndlp = lpfc_setup_disc_node(vport, Did);
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index 4daae90e0c99..ae0a8252128c 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -2429,7 +2429,8 @@ lpfc_debugfs_dif_err_write(struct file *file, const char __user *buf,
 		return 0;
 
 	if (dent == phba->debug_InjErrLBA) {
-		if ((buf[0] == 'o') && (buf[1] == 'f') && (buf[2] == 'f'))
+		if ((dstbuf[0] == 'o') && (dstbuf[1] == 'f') &&
+		    (dstbuf[2] == 'f'))
 			tmp = (uint64_t)(-1);
 	}
 
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 80d1e661b0d4..3d670568a276 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -7936,19 +7936,13 @@ lpfc_els_timeout_handler(struct lpfc_vport *vport)
 	if (unlikely(!pring))
 		return;
 
-	if ((phba->pport->load_flag & FC_UNLOADING))
+	if (phba->pport->load_flag & FC_UNLOADING)
 		return;
+
 	spin_lock_irq(&phba->hbalock);
 	if (phba->sli_rev == LPFC_SLI_REV4)
 		spin_lock(&pring->ring_lock);
 
-	if ((phba->pport->load_flag & FC_UNLOADING)) {
-		if (phba->sli_rev == LPFC_SLI_REV4)
-			spin_unlock(&pring->ring_lock);
-		spin_unlock_irq(&phba->hbalock);
-		return;
-	}
-
 	list_for_each_entry_safe(piocb, tmp_iocb, &pring->txcmplq, list) {
 		cmd = &piocb->iocb;
 
@@ -8514,6 +8508,8 @@ lpfc_els_unsol_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	spin_lock_irq(shost->host_lock);
 	if (ndlp->nlp_flag & NLP_IN_DEV_LOSS) {
 		spin_unlock_irq(shost->host_lock);
+		if (newnode)
+			lpfc_nlp_put(ndlp);
 		goto dropit;
 	}
 	spin_unlock_irq(shost->host_lock);
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index f5952f8cd4b5..4084f7f2b821 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -1360,14 +1360,14 @@ lpfc_vlan_id_match(uint16_t curr_vlan_id, uint16_t new_vlan_id)
 }
 
 /**
- * lpfc_update_fcf_record - Update driver fcf record
  * __lpfc_update_fcf_record_pri - update the lpfc_fcf_pri record.
  * @phba: pointer to lpfc hba data structure.
  * @fcf_index: Index for the lpfc_fcf_record.
  * @new_fcf_record: pointer to hba fcf record.
  *
  * This routine updates the driver FCF priority record from the new HBA FCF
- * record. This routine is called with the host lock held.
+ * record. The hbalock is asserted held in the code path calling this
+ * routine.
  **/
 static void
 __lpfc_update_fcf_record_pri(struct lpfc_hba *phba, uint16_t fcf_index,
@@ -1376,8 +1376,6 @@ __lpfc_update_fcf_record_pri(struct lpfc_hba *phba, uint16_t fcf_index,
 {
 	struct lpfc_fcf_pri *fcf_pri;
 
-	lockdep_assert_held(&phba->hbalock);
-
 	fcf_pri = &phba->fcf.fcf_pri[fcf_index];
 	fcf_pri->fcf_rec.fcf_index = fcf_index;
 	/* FCF record priority */
@@ -1455,7 +1453,7 @@ lpfc_copy_fcf_record(struct lpfc_fcf_rec *fcf_rec,
  *
  * This routine updates the driver FCF record from the new HBA FCF record
  * together with the address mode, vlan_id, and other informations. This
- * routine is called with the host lock held.
+ * routine is called with the hbalock held.
  **/
 static void
 __lpfc_update_fcf_record(struct lpfc_hba *phba, struct lpfc_fcf_rec *fcf_rec,
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index 10c5d1c3122e..6dfff0376547 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -3541,7 +3541,7 @@ struct lpfc_mbx_set_feature {
 #define lpfc_mbx_set_feature_UER_SHIFT  0
 #define lpfc_mbx_set_feature_UER_MASK   0x00000001
 #define lpfc_mbx_set_feature_UER_WORD   word6
-#define lpfc_mbx_set_feature_mds_SHIFT  0
+#define lpfc_mbx_set_feature_mds_SHIFT  2
 #define lpfc_mbx_set_feature_mds_MASK   0x00000001
 #define lpfc_mbx_set_feature_mds_WORD   word6
 #define lpfc_mbx_set_feature_mds_deep_loopbk_SHIFT  1
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index ea99483345f2..69a5249e007a 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -6020,29 +6020,6 @@ static void lpfc_log_intr_mode(struct lpfc_hba *phba, uint32_t intr_mode)
 }
 
 /**
- * lpfc_cpumask_of_node_init - initalizes cpumask of phba's NUMA node
- * @phba: Pointer to HBA context object.
- *
- **/
-static void
-lpfc_cpumask_of_node_init(struct lpfc_hba *phba)
-{
-	unsigned int cpu, numa_node;
-	struct cpumask *numa_mask = &phba->sli4_hba.numa_mask;
-
-	cpumask_clear(numa_mask);
-
-	/* Check if we're a NUMA architecture */
-	numa_node = dev_to_node(&phba->pcidev->dev);
-	if (numa_node == NUMA_NO_NODE)
-		return;
-
-	for_each_possible_cpu(cpu)
-		if (cpu_to_node(cpu) == numa_node)
-			cpumask_set_cpu(cpu, numa_mask);
-}
-
-/**
  * lpfc_enable_pci_dev - Enable a generic PCI device.
  * @phba: pointer to lpfc hba data structure.
  *
@@ -6480,7 +6457,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 	phba->sli4_hba.num_present_cpu = lpfc_present_cpu;
 	phba->sli4_hba.num_possible_cpu = cpumask_last(cpu_possible_mask) + 1;
 	phba->sli4_hba.curr_disp_cpu = 0;
-	lpfc_cpumask_of_node_init(phba);
 
 	/* Get all the module params for configuring this host */
 	lpfc_get_cfgparam(phba);
@@ -6688,6 +6664,13 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 #endif
 				/* Not supported for NVMET */
 				phba->cfg_xri_rebalancing = 0;
+				if (phba->irq_chann_mode == NHT_MODE) {
+					phba->cfg_irq_chann =
+						phba->sli4_hba.num_present_cpu;
+					phba->cfg_hdw_queue =
+						phba->sli4_hba.num_present_cpu;
+					phba->irq_chann_mode = NORMAL_MODE;
+				}
 				break;
 			}
 		}
@@ -7029,7 +7012,7 @@ lpfc_sli4_driver_resource_unset(struct lpfc_hba *phba)
 	phba->sli4_hba.num_possible_cpu = 0;
 	phba->sli4_hba.num_present_cpu = 0;
 	phba->sli4_hba.curr_disp_cpu = 0;
-	cpumask_clear(&phba->sli4_hba.numa_mask);
+	cpumask_clear(&phba->sli4_hba.irq_aff_mask);
 
 	/* Free memory allocated for fast-path work queue handles */
 	kfree(phba->sli4_hba.hba_eq_hdl);
@@ -11284,11 +11267,12 @@ lpfc_irq_clear_aff(struct lpfc_hba_eq_hdl *eqhdl)
  * @offline: true, cpu is going offline. false, cpu is coming online.
  *
  * If cpu is going offline, we'll try our best effort to find the next
- * online cpu on the phba's NUMA node and migrate all offlining IRQ affinities.
+ * online cpu on the phba's original_mask and migrate all offlining IRQ
+ * affinities.
  *
- * If cpu is coming online, reaffinitize the IRQ back to the onlineng cpu.
+ * If cpu is coming online, reaffinitize the IRQ back to the onlining cpu.
  *
- * Note: Call only if cfg_irq_numa is enabled, otherwise rely on
+ * Note: Call only if NUMA or NHT mode is enabled, otherwise rely on
  *	 PCI_IRQ_AFFINITY to auto-manage IRQ affinity.
  *
  **/
@@ -11298,14 +11282,14 @@ lpfc_irq_rebalance(struct lpfc_hba *phba, unsigned int cpu, bool offline)
 	struct lpfc_vector_map_info *cpup;
 	struct cpumask *aff_mask;
 	unsigned int cpu_select, cpu_next, idx;
-	const struct cpumask *numa_mask;
+	const struct cpumask *orig_mask;
 
-	if (!phba->cfg_irq_numa)
+	if (phba->irq_chann_mode == NORMAL_MODE)
 		return;
 
-	numa_mask = &phba->sli4_hba.numa_mask;
+	orig_mask = &phba->sli4_hba.irq_aff_mask;
 
-	if (!cpumask_test_cpu(cpu, numa_mask))
+	if (!cpumask_test_cpu(cpu, orig_mask))
 		return;
 
 	cpup = &phba->sli4_hba.cpu_map[cpu];
@@ -11314,9 +11298,9 @@ lpfc_irq_rebalance(struct lpfc_hba *phba, unsigned int cpu, bool offline)
 		return;
 
 	if (offline) {
-		/* Find next online CPU on NUMA node */
-		cpu_next = cpumask_next_wrap(cpu, numa_mask, cpu, true);
-		cpu_select = lpfc_next_online_numa_cpu(numa_mask, cpu_next);
+		/* Find next online CPU on original mask */
+		cpu_next = cpumask_next_wrap(cpu, orig_mask, cpu, true);
+		cpu_select = lpfc_next_online_cpu(orig_mask, cpu_next);
 
 		/* Found a valid CPU */
 		if ((cpu_select < nr_cpu_ids) && (cpu_select != cpu)) {
@@ -11431,7 +11415,7 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba)
 {
 	int vectors, rc, index;
 	char *name;
-	const struct cpumask *numa_mask = NULL;
+	const struct cpumask *aff_mask = NULL;
 	unsigned int cpu = 0, cpu_cnt = 0, cpu_select = nr_cpu_ids;
 	struct lpfc_hba_eq_hdl *eqhdl;
 	const struct cpumask *maskp;
@@ -11441,16 +11425,18 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba)
 	/* Set up MSI-X multi-message vectors */
 	vectors = phba->cfg_irq_chann;
 
-	if (phba->cfg_irq_numa) {
-		numa_mask = &phba->sli4_hba.numa_mask;
-		cpu_cnt = cpumask_weight(numa_mask);
+	if (phba->irq_chann_mode != NORMAL_MODE)
+		aff_mask = &phba->sli4_hba.irq_aff_mask;
+
+	if (aff_mask) {
+		cpu_cnt = cpumask_weight(aff_mask);
 		vectors = min(phba->cfg_irq_chann, cpu_cnt);
 
-		/* cpu: iterates over numa_mask including offline or online
-		 * cpu_select: iterates over online numa_mask to set affinity
+		/* cpu: iterates over aff_mask including offline or online
+		 * cpu_select: iterates over online aff_mask to set affinity
 		 */
-		cpu = cpumask_first(numa_mask);
-		cpu_select = lpfc_next_online_numa_cpu(numa_mask, cpu);
+		cpu = cpumask_first(aff_mask);
+		cpu_select = lpfc_next_online_cpu(aff_mask, cpu);
 	} else {
 		flags |= PCI_IRQ_AFFINITY;
 	}
@@ -11484,7 +11470,7 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba)
 
 		eqhdl->irq = pci_irq_vector(phba->pcidev, index);
 
-		if (phba->cfg_irq_numa) {
+		if (aff_mask) {
 			/* If found a neighboring online cpu, set affinity */
 			if (cpu_select < nr_cpu_ids)
 				lpfc_irq_set_aff(eqhdl, cpu_select);
@@ -11494,11 +11480,11 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba)
 						LPFC_CPU_FIRST_IRQ,
 						cpu);
 
-			/* Iterate to next offline or online cpu in numa_mask */
-			cpu = cpumask_next(cpu, numa_mask);
+			/* Iterate to next offline or online cpu in aff_mask */
+			cpu = cpumask_next(cpu, aff_mask);
 
-			/* Find next online cpu in numa_mask to set affinity */
-			cpu_select = lpfc_next_online_numa_cpu(numa_mask, cpu);
+			/* Find next online cpu in aff_mask to set affinity */
+			cpu_select = lpfc_next_online_cpu(aff_mask, cpu);
 		} else if (vectors == 1) {
 			cpu = cpumask_first(cpu_present_mask);
 			lpfc_assign_eq_map_info(phba, index, LPFC_CPU_FIRST_IRQ,
diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c
index e35b52b66d6c..e34e0f11bfdd 100644
--- a/drivers/scsi/lpfc/lpfc_mbox.c
+++ b/drivers/scsi/lpfc/lpfc_mbox.c
@@ -1378,7 +1378,8 @@ lpfc_config_port(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 	 */
 
 	if (phba->cfg_hostmem_hgp && phba->sli_rev != 3) {
-		phba->host_gp = &phba->mbox->us.s2.host[0];
+		phba->host_gp = (struct lpfc_hgp __iomem *)
+				 &phba->mbox->us.s2.host[0];
 		phba->hbq_put = NULL;
 		offset = (uint8_t *)&phba->mbox->us.s2.host -
 			(uint8_t *)phba->slim2p.virt;
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index b46ba70f78da..b16c087ba272 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -1654,11 +1654,6 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 
 	phba = vport->phba;
 
-	if (vport->load_flag & FC_UNLOADING) {
-		ret = -ENODEV;
-		goto out_fail;
-	}
-
 	if (unlikely(vport->load_flag & FC_UNLOADING)) {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
 				 "6124 Fail IO, Driver unload\n");
@@ -2491,38 +2486,6 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 		spin_unlock_irq(&vport->phba->hbalock);
 		rport = remote_port->private;
 		if (oldrport) {
-			/* New remoteport record does not guarantee valid
-			 * host private memory area.
-			 */
-			if (oldrport == remote_port->private) {
-				/* Same remoteport - ndlp should match.
-				 * Just reuse.
-				 */
-				lpfc_printf_vlog(ndlp->vport, KERN_INFO,
-						 LOG_NVME_DISC,
-						 "6014 Rebind lport to current "
-						 "remoteport x%px wwpn 0x%llx, "
-						 "Data: x%x x%x x%px x%px x%x "
-						 " x%06x\n",
-						 remote_port,
-						 remote_port->port_name,
-						 remote_port->port_id,
-						 remote_port->port_role,
-						 oldrport->ndlp,
-						 ndlp,
-						 ndlp->nlp_type,
-						 ndlp->nlp_DID);
-
-				/* It's a complete rebind only if the driver
-				 * is registering with the same ndlp. Otherwise
-				 * the driver likely executed a node swap
-				 * prior to this registration and the ndlp to
-				 * remoteport binding needs to be redone.
-				 */
-				if (prev_ndlp == ndlp)
-					return 0;
-
-			}
 
 			/* Sever the ndlp<->rport association
 			 * before dropping the ndlp ref from
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c
index 32eb5e873e9b..88760416a8cb 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.c
+++ b/drivers/scsi/lpfc/lpfc_nvmet.c
@@ -1030,11 +1030,6 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
 		goto aerr;
 	}
 
-	if (phba->pport->load_flag & FC_UNLOADING) {
-		rc = -ENODEV;
-		goto aerr;
-	}
-
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 	if (ctxp->ts_cmd_nvme) {
 		if (rsp->op == NVMET_FCOP_RSP)
@@ -1157,9 +1152,6 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
 	if (phba->pport->load_flag & FC_UNLOADING)
 		return;
 
-	if (phba->pport->load_flag & FC_UNLOADING)
-		return;
-
 	if (!ctxp->hdwq)
 		ctxp->hdwq = &phba->sli4_hba.hdwq[0];
 
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 9e21c4f3b009..25653baba367 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -535,7 +535,7 @@ lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq,
 	if (count > eq->EQ_max_eqe)
 		eq->EQ_max_eqe = count;
 
-	eq->queue_claimed = 0;
+	xchg(&eq->queue_claimed, 0);
 
 rearm_and_exit:
 	/* Always clear the EQ. */
@@ -1245,8 +1245,8 @@ lpfc_sli_get_iocbq(struct lpfc_hba *phba)
  * @phba: Pointer to HBA context object.
  * @iocbq: Pointer to driver iocb object.
  *
- * This function is called with hbalock held to release driver
- * iocb object to the iocb pool. The iotag in the iocb object
+ * This function is called to release the driver iocb object
+ * to the iocb pool. The iotag in the iocb object
  * does not change for each use of the iocb object. This function
  * clears all other fields of the iocb object when it is freed.
  * The sqlq structure that holds the xritag and phys and virtual
@@ -1256,7 +1256,8 @@ lpfc_sli_get_iocbq(struct lpfc_hba *phba)
  * this IO was aborted then the sglq entry it put on the
  * lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the
  * IO has good status or fails for any other reason then the sglq
- * entry is added to the free list (lpfc_els_sgl_list).
+ * entry is added to the free list (lpfc_els_sgl_list). The hbalock is
+ *  asserted held in the code path calling this routine.
  **/
 static void
 __lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
@@ -1266,8 +1267,6 @@ __lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
 	unsigned long iflag = 0;
 	struct lpfc_sli_ring *pring;
 
-	lockdep_assert_held(&phba->hbalock);
-
 	if (iocbq->sli4_xritag == NO_XRI)
 		sglq = NULL;
 	else
@@ -1330,18 +1329,17 @@ out:
  * @phba: Pointer to HBA context object.
  * @iocbq: Pointer to driver iocb object.
  *
- * This function is called with hbalock held to release driver
- * iocb object to the iocb pool. The iotag in the iocb object
- * does not change for each use of the iocb object. This function
- * clears all other fields of the iocb object when it is freed.
+ * This function is called to release the driver iocb object to the
+ * iocb pool. The iotag in the iocb object does not change for each
+ * use of the iocb object. This function clears all other fields of
+ * the iocb object when it is freed. The hbalock is asserted held in
+ * the code path calling this routine.
  **/
 static void
 __lpfc_sli_release_iocbq_s3(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
 {
 	size_t start_clean = offsetof(struct lpfc_iocbq, iocb);
 
-	lockdep_assert_held(&phba->hbalock);
-
 	/*
 	 * Clean all volatile data fields, preserve iotag and node struct.
 	 */
@@ -1786,17 +1784,17 @@ lpfc_sli_next_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq)
  * @nextiocb: Pointer to driver iocb object which need to be
  *            posted to firmware.
  *
- * This function is called with hbalock held to post a new iocb to
- * the firmware. This function copies the new iocb to ring iocb slot and
- * updates the ring pointers. It adds the new iocb to txcmplq if there is
+ * This function is called to post a new iocb to the firmware. This
+ * function copies the new iocb to ring iocb slot and updates the
+ * ring pointers. It adds the new iocb to txcmplq if there is
  * a completion call back for this iocb else the function will free the
- * iocb object.
+ * iocb object.  The hbalock is asserted held in the code path calling
+ * this routine.
  **/
 static void
 lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 		IOCB_t *iocb, struct lpfc_iocbq *nextiocb)
 {
-	lockdep_assert_held(&phba->hbalock);
 	/*
 	 * Set up an iotag
 	 */
@@ -11284,6 +11282,7 @@ lpfc_ignore_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
  * request, this function issues abort out unconditionally. This function is
  * called with hbalock held. The function returns 0 when it fails due to
  * memory allocation failure or when the command iocb is an abort request.
+ * The hbalock is asserted held in the code path calling this routine.
  **/
 static int
 lpfc_sli_abort_iotag_issue(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
@@ -11297,8 +11296,6 @@ lpfc_sli_abort_iotag_issue(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	unsigned long iflags;
 	struct lpfc_nodelist *ndlp;
 
-	lockdep_assert_held(&phba->hbalock);
-
 	/*
 	 * There are certain command types we don't want to abort.  And we
 	 * don't want to abort commands that are already in the process of
@@ -13808,7 +13805,7 @@ __lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq,
 				"0369 No entry from completion queue "
 				"qid=%d\n", cq->queue_id);
 
-	cq->queue_claimed = 0;
+	xchg(&cq->queue_claimed, 0);
 
 rearm_and_exit:
 	phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed,
@@ -14389,7 +14386,6 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
 	int ecount = 0;
 	int hba_eqidx;
 	struct lpfc_eq_intr_info *eqi;
-	uint32_t icnt;
 
 	/* Get the driver's phba structure from the dev_id */
 	hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id;
@@ -14417,11 +14413,12 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
 		return IRQ_NONE;
 	}
 
-	eqi = phba->sli4_hba.eq_info;
-	icnt = this_cpu_inc_return(eqi->icnt);
+	eqi = this_cpu_ptr(phba->sli4_hba.eq_info);
+	eqi->icnt++;
+
 	fpeq->last_cpu = raw_smp_processor_id();
 
-	if (icnt > LPFC_EQD_ISR_TRIGGER &&
+	if (eqi->icnt > LPFC_EQD_ISR_TRIGGER &&
 	    fpeq->q_flag & HBA_EQ_DELAY_CHK &&
 	    phba->cfg_auto_imax &&
 	    fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY &&
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index 8da7429e385a..4decb53d81c3 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -920,7 +920,7 @@ struct lpfc_sli4_hba {
 	struct lpfc_vector_map_info *cpu_map;
 	uint16_t num_possible_cpu;
 	uint16_t num_present_cpu;
-	struct cpumask numa_mask;
+	struct cpumask irq_aff_mask;
 	uint16_t curr_disp_cpu;
 	struct lpfc_eq_intr_info __percpu *eq_info;
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index ca40c47cfbe0..ab0bc26c098d 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "12.8.0.0"
+#define LPFC_DRIVER_VERSION "12.8.0.1"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/mac53c94.c b/drivers/scsi/mac53c94.c
index 35d3e322d6d5..43edf83fdb62 100644
--- a/drivers/scsi/mac53c94.c
+++ b/drivers/scsi/mac53c94.c
@@ -20,9 +20,9 @@
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/pgtable.h>
 #include <asm/dbdma.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/prom.h>
 #include <asm/macio.h>
 
diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c
index 8443f2f35be2..8f918df631bf 100644
--- a/drivers/scsi/megaraid/megaraid_mbox.c
+++ b/drivers/scsi/megaraid/megaraid_mbox.c
@@ -302,8 +302,8 @@ static struct pci_driver megaraid_pci_driver = {
 // definitions for the device attributes for exporting logical drive number
 // for a scsi address (Host, Channel, Id, Lun)
 
-DEVICE_ATTR(megaraid_mbox_app_hndl, S_IRUSR, megaraid_sysfs_show_app_hndl,
-		NULL);
+static DEVICE_ATTR(megaraid_mbox_app_hndl, S_IRUSR, megaraid_sysfs_show_app_hndl,
+		   NULL);
 
 // Host template initializer for megaraid mbox sysfs device attributes
 static struct device_attribute *megaraid_shost_attrs[] = {
@@ -312,7 +312,7 @@ static struct device_attribute *megaraid_shost_attrs[] = {
 };
 
 
-DEVICE_ATTR(megaraid_mbox_ld, S_IRUSR, megaraid_sysfs_show_ldnum, NULL);
+static DEVICE_ATTR(megaraid_mbox_ld, S_IRUSR, megaraid_sysfs_show_ldnum, NULL);
 
 // Host template initializer for megaraid mbox sysfs device attributes
 static struct device_attribute *megaraid_sdev_attrs[] = {
diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index 83d8c4cb1ad5..af2c7a2a9565 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -21,8 +21,8 @@
 /*
  * MegaRAID SAS Driver meta data
  */
-#define MEGASAS_VERSION				"07.713.01.00-rc1"
-#define MEGASAS_RELDATE				"Dec 27, 2019"
+#define MEGASAS_VERSION				"07.714.04.00-rc1"
+#define MEGASAS_RELDATE				"Apr 14, 2020"
 
 #define MEGASAS_MSIX_NAME_LEN			32
 
@@ -511,7 +511,7 @@ union MR_PROGRESS {
  */
 struct MR_PD_PROGRESS {
 	struct {
-#ifndef MFI_BIG_ENDIAN
+#ifndef __BIG_ENDIAN_BITFIELD
 		u32     rbld:1;
 		u32     patrol:1;
 		u32     clear:1;
@@ -537,7 +537,7 @@ struct MR_PD_PROGRESS {
 	};
 
 	struct {
-#ifndef MFI_BIG_ENDIAN
+#ifndef __BIG_ENDIAN_BITFIELD
 		u32     rbld:1;
 		u32     patrol:1;
 		u32     clear:1;
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index babe85d7b537..00668335c2af 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -81,7 +81,7 @@ int smp_affinity_enable = 1;
 module_param(smp_affinity_enable, int, 0444);
 MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
 
-int rdpq_enable = 1;
+static int rdpq_enable = 1;
 module_param(rdpq_enable, int, 0444);
 MODULE_PARM_DESC(rdpq_enable, "Allocate reply queue in chunks for large queue depth enable/disable Default: enable(1)");
 
@@ -89,7 +89,7 @@ unsigned int dual_qdepth_disable;
 module_param(dual_qdepth_disable, int, 0444);
 MODULE_PARM_DESC(dual_qdepth_disable, "Disable dual queue depth feature. Default: 0");
 
-unsigned int scmd_timeout = MEGASAS_DEFAULT_CMD_TIMEOUT;
+static unsigned int scmd_timeout = MEGASAS_DEFAULT_CMD_TIMEOUT;
 module_param(scmd_timeout, int, 0444);
 MODULE_PARM_DESC(scmd_timeout, "scsi command timeout (10-90s), default 90s. See megasas_reset_timer.");
 
@@ -1982,9 +1982,9 @@ static void megasas_set_fw_assisted_qd(struct scsi_device *sdev,
 
 	if (is_target_prop) {
 		tgt_device_qd = le32_to_cpu(instance->tgt_prop->device_qdepth);
-		if (tgt_device_qd &&
-		    (tgt_device_qd <= instance->host->can_queue))
-			device_qd = tgt_device_qd;
+		if (tgt_device_qd)
+			device_qd = min(instance->host->can_queue,
+					(int)tgt_device_qd);
 	}
 
 	if (instance->enable_sdev_max_qd && interface_type != UNKNOWN_DRIVE)
diff --git a/drivers/scsi/megaraid/megaraid_sas_fp.c b/drivers/scsi/megaraid/megaraid_sas_fp.c
index 89c3685f5163..3b3d04d7671f 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fp.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fp.c
@@ -85,7 +85,7 @@ u32 mega_mod64(u64 dividend, u32 divisor)
  *
  * @return quotient
  **/
-u64 mega_div64_32(uint64_t dividend, uint32_t divisor)
+static u64 mega_div64_32(uint64_t dividend, uint32_t divisor)
 {
 	u32 remainder;
 	u64 d;
@@ -367,7 +367,7 @@ u8 MR_ValidateMapInfo(struct megasas_instance *instance, u64 map_id)
 	return 1;
 }
 
-u32 MR_GetSpanBlock(u32 ld, u64 row, u64 *span_blk,
+static u32 MR_GetSpanBlock(u32 ld, u64 row, u64 *span_blk,
 		    struct MR_DRV_RAID_MAP_ALL *map)
 {
 	struct MR_SPAN_BLOCK_INFO *pSpanBlock = MR_LdSpanInfoGet(ld, map);
@@ -417,7 +417,7 @@ u32 MR_GetSpanBlock(u32 ld, u64 row, u64 *span_blk,
 *    div_error	   - Devide error code.
 */
 
-u32 mr_spanset_get_span_block(struct megasas_instance *instance,
+static u32 mr_spanset_get_span_block(struct megasas_instance *instance,
 		u32 ld, u64 row, u64 *span_blk, struct MR_DRV_RAID_MAP_ALL *map)
 {
 	struct fusion_context *fusion = instance->ctrl_context;
@@ -642,7 +642,7 @@ static u32 get_arm_from_strip(struct megasas_instance *instance,
 }
 
 /* This Function will return Phys arm */
-u8 get_arm(struct megasas_instance *instance, u32 ld, u8 span, u64 stripe,
+static u8 get_arm(struct megasas_instance *instance, u32 ld, u8 span, u64 stripe,
 		struct MR_DRV_RAID_MAP_ALL *map)
 {
 	struct MR_LD_RAID  *raid = MR_LdRaidGet(ld, map);
@@ -785,7 +785,7 @@ static u8 mr_spanset_get_phy_params(struct megasas_instance *instance, u32 ld,
 *    span          - Span number
 *    block         - Absolute Block number in the physical disk
 */
-u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow,
+static u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow,
 		u16 stripRef, struct IO_REQUEST_INFO *io_info,
 		struct RAID_CONTEXT *pRAID_Context,
 		struct MR_DRV_RAID_MAP_ALL *map)
@@ -1342,7 +1342,7 @@ void mr_update_load_balance_params(struct MR_DRV_RAID_MAP_ALL *drv_map,
 	}
 }
 
-u8 megasas_get_best_arm_pd(struct megasas_instance *instance,
+static u8 megasas_get_best_arm_pd(struct megasas_instance *instance,
 			   struct LD_LOAD_BALANCE_INFO *lbInfo,
 			   struct IO_REQUEST_INFO *io_info,
 			   struct MR_DRV_RAID_MAP_ALL *drv_map)
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index b2ad96564484..319f241da4b6 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -176,7 +176,7 @@ static inline bool megasas_check_same_4gb_region
  * megasas_enable_intr_fusion -	Enables interrupts
  * @regs:			MFI register set
  */
-void
+static void
 megasas_enable_intr_fusion(struct megasas_instance *instance)
 {
 	struct megasas_register_set __iomem *regs;
@@ -198,7 +198,7 @@ megasas_enable_intr_fusion(struct megasas_instance *instance)
  * megasas_disable_intr_fusion - Disables interrupt
  * @regs:			 MFI register set
  */
-void
+static void
 megasas_disable_intr_fusion(struct megasas_instance *instance)
 {
 	u32 mask = 0xFFFFFFFF;
@@ -2070,7 +2070,6 @@ static bool
 megasas_is_prp_possible(struct megasas_instance *instance,
 			struct scsi_cmnd *scmd, int sge_count)
 {
-	int i;
 	u32 data_length = 0;
 	struct scatterlist *sg_scmd;
 	bool build_prp = false;
@@ -2099,63 +2098,6 @@ megasas_is_prp_possible(struct megasas_instance *instance,
 			build_prp = true;
 	}
 
-/*
- * Below code detects gaps/holes in IO data buffers.
- * What does holes/gaps mean?
- * Any SGE except first one in a SGL starts at non NVME page size
- * aligned address OR Any SGE except last one in a SGL ends at
- * non NVME page size boundary.
- *
- * Driver has already informed block layer by setting boundary rules for
- * bio merging done at NVME page size boundary calling kernel API
- * blk_queue_virt_boundary inside slave_config.
- * Still there is possibility of IO coming with holes to driver because of
- * IO merging done by IO scheduler.
- *
- * With SCSI BLK MQ enabled, there will be no IO with holes as there is no
- * IO scheduling so no IO merging.
- *
- * With SCSI BLK MQ disabled, IO scheduler may attempt to merge IOs and
- * then sending IOs with holes.
- *
- * Though driver can request block layer to disable IO merging by calling-
- * blk_queue_flag_set(QUEUE_FLAG_NOMERGES, sdev->request_queue) but
- * user may tune sysfs parameter- nomerges again to 0 or 1.
- *
- * If in future IO scheduling is enabled with SCSI BLK MQ,
- * this algorithm to detect holes will be required in driver
- * for SCSI BLK MQ enabled case as well.
- *
- *
- */
-	scsi_for_each_sg(scmd, sg_scmd, sge_count, i) {
-		if ((i != 0) && (i != (sge_count - 1))) {
-			if (mega_mod64(sg_dma_len(sg_scmd), mr_nvme_pg_size) ||
-			    mega_mod64(sg_dma_address(sg_scmd),
-				       mr_nvme_pg_size)) {
-				build_prp = false;
-				break;
-			}
-		}
-
-		if ((sge_count > 1) && (i == 0)) {
-			if ((mega_mod64((sg_dma_address(sg_scmd) +
-					sg_dma_len(sg_scmd)),
-					mr_nvme_pg_size))) {
-				build_prp = false;
-				break;
-			}
-		}
-
-		if ((sge_count > 1) && (i == (sge_count - 1))) {
-			if (mega_mod64(sg_dma_address(sg_scmd),
-				       mr_nvme_pg_size)) {
-				build_prp = false;
-				break;
-			}
-		}
-	}
-
 	return build_prp;
 }
 
@@ -4230,7 +4172,7 @@ void  megasas_reset_reply_desc(struct megasas_instance *instance)
  * megasas_refire_mgmt_cmd :	Re-fire management commands
  * @instance:				Controller's soft instance
 */
-void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
+static void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
 			     bool return_ioctl)
 {
 	int j;
@@ -4238,8 +4180,9 @@ void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
 	struct fusion_context *fusion;
 	struct megasas_cmd *cmd_mfi;
 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
+	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
 	u16 smid;
-	bool refire_cmd = 0;
+	bool refire_cmd = false;
 	u8 result;
 	u32 opcode = 0;
 
@@ -4305,6 +4248,11 @@ void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
 			result = COMPLETE_CMD;
 		}
 
+		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
+				cmd_fusion->io_request;
+		if (scsi_io_req->Function == MPI2_FUNCTION_SCSI_TASK_MGMT)
+			result = RETURN_CMD;
+
 		switch (result) {
 		case REFIRE_CMD:
 			megasas_fire_cmd_fusion(instance, req_desc);
@@ -4533,7 +4481,6 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
 	if (!timeleft) {
 		dev_err(&instance->pdev->dev,
 			"task mgmt type 0x%x timed out\n", type);
-		cmd_mfi->flags |= DRV_DCMD_SKIP_REFIRE;
 		mutex_unlock(&instance->reset_mutex);
 		rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR);
 		mutex_lock(&instance->reset_mutex);
@@ -4713,12 +4660,12 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
 		"attempting task abort! scmd(0x%p) tm_dev_handle 0x%x\n",
 		scmd, devhandle);
 
-	mr_device_priv_data->tm_busy = 1;
+	mr_device_priv_data->tm_busy = true;
 	ret = megasas_issue_tm(instance, devhandle,
 			scmd->device->channel, scmd->device->id, smid,
 			MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
 			mr_device_priv_data);
-	mr_device_priv_data->tm_busy = 0;
+	mr_device_priv_data->tm_busy = false;
 
 	mutex_unlock(&instance->reset_mutex);
 	scmd_printk(KERN_INFO, scmd, "task abort %s!! scmd(0x%p)\n",
@@ -4783,12 +4730,12 @@ int megasas_reset_target_fusion(struct scsi_cmnd *scmd)
 	sdev_printk(KERN_INFO, scmd->device,
 		"attempting target reset! scmd(0x%p) tm_dev_handle: 0x%x\n",
 		scmd, devhandle);
-	mr_device_priv_data->tm_busy = 1;
+	mr_device_priv_data->tm_busy = true;
 	ret = megasas_issue_tm(instance, devhandle,
 			scmd->device->channel, scmd->device->id, 0,
 			MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET,
 			mr_device_priv_data);
-	mr_device_priv_data->tm_busy = 0;
+	mr_device_priv_data->tm_busy = false;
 	mutex_unlock(&instance->reset_mutex);
 	scmd_printk(KERN_NOTICE, scmd, "target reset %s!!\n",
 		(ret == SUCCESS) ? "SUCCESS" : "FAILED");
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.h b/drivers/scsi/megaraid/megaraid_sas_fusion.h
index d57ecc7f88d8..30de4b01f703 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.h
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.h
@@ -774,7 +774,7 @@ struct MR_SPAN_BLOCK_INFO {
 struct MR_CPU_AFFINITY_MASK {
 	union {
 		struct {
-#ifndef MFI_BIG_ENDIAN
+#ifndef __BIG_ENDIAN_BITFIELD
 		u8 hw_path:1;
 		u8 cpu0:1;
 		u8 cpu1:1;
@@ -866,7 +866,7 @@ struct MR_LD_RAID {
 	__le16     seqNum;
 
 struct {
-#ifndef MFI_BIG_ENDIAN
+#ifndef __BIG_ENDIAN_BITFIELD
 	u32 ldSyncRequired:1;
 	u32 regTypeReqOnReadIsValid:1;
 	u32 isEPD:1;
@@ -889,7 +889,7 @@ struct {
 	/* 0x30 - 0x33, Logical block size for the LD */
 	u32 logical_block_length;
 	struct {
-#ifndef MFI_BIG_ENDIAN
+#ifndef __BIG_ENDIAN_BITFIELD
 	/* 0x34, P_I_EXPONENT from READ CAPACITY 16 */
 	u32 ld_pi_exp:4;
 	/* 0x34, LOGICAL BLOCKS PER PHYSICAL
diff --git a/drivers/scsi/mesh.c b/drivers/scsi/mesh.c
index 74fb50644678..f9f8f4921654 100644
--- a/drivers/scsi/mesh.c
+++ b/drivers/scsi/mesh.c
@@ -31,9 +31,9 @@
 #include <linux/reboot.h>
 #include <linux/spinlock.h>
 #include <linux/pci.h>
+#include <linux/pgtable.h>
 #include <asm/dbdma.h>
 #include <asm/io.h>
-#include <asm/pgtable.h>
 #include <asm/prom.h>
 #include <asm/irq.h>
 #include <asm/hydra.h>
diff --git a/drivers/scsi/mpt3sas/Kconfig b/drivers/scsi/mpt3sas/Kconfig
index a072187875df..86209455172d 100644
--- a/drivers/scsi/mpt3sas/Kconfig
+++ b/drivers/scsi/mpt3sas/Kconfig
@@ -46,7 +46,7 @@ config SCSI_MPT3SAS
 	select SCSI_SAS_ATTRS
 	select RAID_ATTRS
 	select IRQ_POLL
-	---help---
+	help
 	This driver supports PCI-Express SAS 12Gb/s Host Adapters.
 
 config SCSI_MPT2SAS_MAX_SGE
@@ -54,7 +54,7 @@ config SCSI_MPT2SAS_MAX_SGE
 	depends on PCI && SCSI && SCSI_MPT3SAS
 	default "128"
 	range 16 256
-	---help---
+	help
 	This option allows you to specify the maximum number of scatter-
 	gather entries per I/O. The driver default is 128, which matches
 	MAX_PHYS_SEGMENTS in most kernels.  However in SuSE kernels this
@@ -66,7 +66,7 @@ config SCSI_MPT3SAS_MAX_SGE
 	depends on PCI && SCSI && SCSI_MPT3SAS
 	default "128"
 	range 16 256
-	---help---
+	help
 	This option allows you to specify the maximum number of scatter-
 	gather entries per I/O. The driver default is 128, which matches
 	MAX_PHYS_SEGMENTS in most kernels.  However in SuSE kernels this
@@ -78,6 +78,6 @@ config SCSI_MPT2SAS
 	default n
 	select SCSI_MPT3SAS
 	depends on PCI && SCSI
-	---help---
+	help
 	Dummy config option for backwards compatiblity: configure the MPT3SAS
 	driver instead.
diff --git a/drivers/scsi/mpt3sas/Makefile b/drivers/scsi/mpt3sas/Makefile
index 84fb3fbdb0ca..e76d994dbed3 100644
--- a/drivers/scsi/mpt3sas/Makefile
+++ b/drivers/scsi/mpt3sas/Makefile
@@ -7,4 +7,5 @@ mpt3sas-y +=  mpt3sas_base.o     \
 		mpt3sas_transport.o     \
 		mpt3sas_ctl.o	\
 		mpt3sas_trigger_diag.o \
-		mpt3sas_warpdrive.o
+		mpt3sas_warpdrive.o \
+		mpt3sas_debugfs.o \
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 663782bb790d..96b78fdc6b8a 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -413,7 +413,7 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 {
 	Mpi2SGESimple32_t *sgel, *sgel_next;
 	u32  sgl_flags, sge_chain_count = 0;
-	bool is_write = 0;
+	bool is_write = false;
 	u16 i = 0;
 	void __iomem *buffer_iomem;
 	phys_addr_t buffer_iomem_phys;
@@ -482,7 +482,7 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
 
 	if (le32_to_cpu(sgel->FlagsLength) &
 			(MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
-		is_write = 1;
+		is_write = true;
 
 	for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
 
@@ -2806,58 +2806,38 @@ _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,
 static int
 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
 {
-	u64 required_mask, coherent_mask;
 	struct sysinfo s;
-	/* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */
-	int dma_mask = (ioc->hba_mpi_version_belonged > MPI2_VERSION) ? 63 : 64;
-
-	if (ioc->is_mcpu_endpoint)
-		goto try_32bit;
+	int dma_mask;
 
-	required_mask = dma_get_required_mask(&pdev->dev);
-	if (sizeof(dma_addr_t) == 4 || required_mask == 32)
-		goto try_32bit;
-
-	if (ioc->dma_mask)
-		coherent_mask = DMA_BIT_MASK(dma_mask);
+	if (ioc->is_mcpu_endpoint ||
+	    sizeof(dma_addr_t) == 4 || ioc->use_32bit_dma ||
+	    dma_get_required_mask(&pdev->dev) <= 32)
+		dma_mask = 32;
+	/* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */
+	else if (ioc->hba_mpi_version_belonged > MPI2_VERSION)
+		dma_mask = 63;
 	else
-		coherent_mask = DMA_BIT_MASK(32);
+		dma_mask = 64;
 
 	if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(dma_mask)) ||
-	    dma_set_coherent_mask(&pdev->dev, coherent_mask))
-		goto try_32bit;
-
-	ioc->base_add_sg_single = &_base_add_sg_single_64;
-	ioc->sge_size = sizeof(Mpi2SGESimple64_t);
-	ioc->dma_mask = dma_mask;
-	goto out;
-
- try_32bit:
-	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)))
+	    dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(dma_mask)))
 		return -ENODEV;
 
-	ioc->base_add_sg_single = &_base_add_sg_single_32;
-	ioc->sge_size = sizeof(Mpi2SGESimple32_t);
-	ioc->dma_mask = 32;
- out:
+	if (dma_mask > 32) {
+		ioc->base_add_sg_single = &_base_add_sg_single_64;
+		ioc->sge_size = sizeof(Mpi2SGESimple64_t);
+	} else {
+		ioc->base_add_sg_single = &_base_add_sg_single_32;
+		ioc->sge_size = sizeof(Mpi2SGESimple32_t);
+	}
+
 	si_meminfo(&s);
 	ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",
-		 ioc->dma_mask, convert_to_kb(s.totalram));
+		dma_mask, convert_to_kb(s.totalram));
 
 	return 0;
 }
 
-static int
-_base_change_consistent_dma_mask(struct MPT3SAS_ADAPTER *ioc,
-				      struct pci_dev *pdev)
-{
-	if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(ioc->dma_mask))) {
-		if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)))
-			return -ENODEV;
-	}
-	return 0;
-}
-
 /**
  * _base_check_enable_msix - checks MSIX capabable.
  * @ioc: per adapter object
@@ -4827,8 +4807,9 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 {
 	int i = 0;
 	int j = 0;
+	int dma_alloc_count = 0;
 	struct chain_tracker *ct;
-	struct reply_post_struct *rps;
+	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;
 
 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
 
@@ -4870,29 +4851,34 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	}
 
 	if (ioc->reply_post) {
-		do {
-			rps = &ioc->reply_post[i];
-			if (rps->reply_post_free) {
-				dma_pool_free(
-				    ioc->reply_post_free_dma_pool,
-				    rps->reply_post_free,
-				    rps->reply_post_free_dma);
-				dexitprintk(ioc,
-					    ioc_info(ioc, "reply_post_free_pool(0x%p): free\n",
-						     rps->reply_post_free));
-				rps->reply_post_free = NULL;
+		dma_alloc_count = DIV_ROUND_UP(count,
+				RDPQ_MAX_INDEX_IN_ONE_CHUNK);
+		for (i = 0; i < count; i++) {
+			if (i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0
+			    && dma_alloc_count) {
+				if (ioc->reply_post[i].reply_post_free) {
+					dma_pool_free(
+					    ioc->reply_post_free_dma_pool,
+					    ioc->reply_post[i].reply_post_free,
+					ioc->reply_post[i].reply_post_free_dma);
+					dexitprintk(ioc, ioc_info(ioc,
+					   "reply_post_free_pool(0x%p): free\n",
+					   ioc->reply_post[i].reply_post_free));
+					ioc->reply_post[i].reply_post_free =
+									NULL;
+				}
+				--dma_alloc_count;
 			}
-		} while (ioc->rdpq_array_enable &&
-			   (++i < ioc->reply_queue_count));
+		}
+		dma_pool_destroy(ioc->reply_post_free_dma_pool);
 		if (ioc->reply_post_free_array &&
 			ioc->rdpq_array_enable) {
 			dma_pool_free(ioc->reply_post_free_array_dma_pool,
-				ioc->reply_post_free_array,
-				ioc->reply_post_free_array_dma);
+			    ioc->reply_post_free_array,
+			    ioc->reply_post_free_array_dma);
 			ioc->reply_post_free_array = NULL;
 		}
 		dma_pool_destroy(ioc->reply_post_free_array_dma_pool);
-		dma_pool_destroy(ioc->reply_post_free_dma_pool);
 		kfree(ioc->reply_post);
 	}
 
@@ -4902,8 +4888,7 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 					ioc->pcie_sg_lookup[i].pcie_sgl,
 					ioc->pcie_sg_lookup[i].pcie_sgl_dma);
 		}
-		if (ioc->pcie_sgl_dma_pool)
-			dma_pool_destroy(ioc->pcie_sgl_dma_pool);
+		dma_pool_destroy(ioc->pcie_sgl_dma_pool);
 	}
 
 	if (ioc->config_page) {
@@ -4915,7 +4900,9 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	}
 
 	kfree(ioc->hpr_lookup);
+	ioc->hpr_lookup = NULL;
 	kfree(ioc->internal_lookup);
+	ioc->internal_lookup = NULL;
 	if (ioc->chain_lookup) {
 		for (i = 0; i < ioc->scsiio_depth; i++) {
 			for (j = ioc->chains_per_prp_buffer;
@@ -4935,7 +4922,7 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 }
 
 /**
- * is_MSB_are_same - checks whether all reply queues in a set are
+ * mpt3sas_check_same_4gb_region - checks whether all reply queues in a set are
  *	having same upper 32bits in their base memory address.
  * @reply_pool_start_address: Base address of a reply queue set
  * @pool_sz: Size of single Reply Descriptor Post Queues pool size
@@ -4945,7 +4932,7 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
  */
 
 static int
-is_MSB_are_same(long reply_pool_start_address, u32 pool_sz)
+mpt3sas_check_same_4gb_region(long reply_pool_start_address, u32 pool_sz)
 {
 	long reply_pool_end_address;
 
@@ -4959,6 +4946,85 @@ is_MSB_are_same(long reply_pool_start_address, u32 pool_sz)
 }
 
 /**
+ * base_alloc_rdpq_dma_pool - Allocating DMA'able memory
+ *                     for reply queues.
+ * @ioc: per adapter object
+ * @sz: DMA Pool size
+ * Return: 0 for success, non-zero for failure.
+ */
+static int
+base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz)
+{
+	int i = 0;
+	u32 dma_alloc_count = 0;
+	int reply_post_free_sz = ioc->reply_post_queue_depth *
+		sizeof(Mpi2DefaultReplyDescriptor_t);
+	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;
+
+	ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct),
+			GFP_KERNEL);
+	if (!ioc->reply_post)
+		return -ENOMEM;
+	/*
+	 *  For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
+	 *  VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should
+	 *  be within 4GB boundary i.e reply queues in a set must have same
+	 *  upper 32-bits in their memory address. so here driver is allocating
+	 *  the DMA'able memory for reply queues according.
+	 *  Driver uses limitation of
+	 *  VENTURA_SERIES to manage INVADER_SERIES as well.
+	 */
+	dma_alloc_count = DIV_ROUND_UP(count,
+				RDPQ_MAX_INDEX_IN_ONE_CHUNK);
+	ioc->reply_post_free_dma_pool =
+		dma_pool_create("reply_post_free pool",
+		    &ioc->pdev->dev, sz, 16, 0);
+	if (!ioc->reply_post_free_dma_pool)
+		return -ENOMEM;
+	for (i = 0; i < count; i++) {
+		if ((i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0) && dma_alloc_count) {
+			ioc->reply_post[i].reply_post_free =
+			    dma_pool_zalloc(ioc->reply_post_free_dma_pool,
+				GFP_KERNEL,
+				&ioc->reply_post[i].reply_post_free_dma);
+			if (!ioc->reply_post[i].reply_post_free)
+				return -ENOMEM;
+			/*
+			 * Each set of RDPQ pool must satisfy 4gb boundary
+			 * restriction.
+			 * 1) Check if allocated resources for RDPQ pool are in
+			 *	the same 4GB range.
+			 * 2) If #1 is true, continue with 64 bit DMA.
+			 * 3) If #1 is false, return 1. which means free all the
+			 * resources and set DMA mask to 32 and allocate.
+			 */
+			if (!mpt3sas_check_same_4gb_region(
+				(long)ioc->reply_post[i].reply_post_free, sz)) {
+				dinitprintk(ioc,
+				    ioc_err(ioc, "bad Replypost free pool(0x%p)"
+				    "reply_post_free_dma = (0x%llx)\n",
+				    ioc->reply_post[i].reply_post_free,
+				    (unsigned long long)
+				    ioc->reply_post[i].reply_post_free_dma));
+				return -EAGAIN;
+			}
+			dma_alloc_count--;
+
+		} else {
+			ioc->reply_post[i].reply_post_free =
+			    (Mpi2ReplyDescriptorsUnion_t *)
+			    ((long)ioc->reply_post[i-1].reply_post_free
+			    + reply_post_free_sz);
+			ioc->reply_post[i].reply_post_free_dma =
+			    (dma_addr_t)
+			    (ioc->reply_post[i-1].reply_post_free_dma +
+			    reply_post_free_sz);
+		}
+	}
+	return 0;
+}
+
+/**
  * _base_allocate_memory_pools - allocate start of day memory pools
  * @ioc: per adapter object
  *
@@ -4972,10 +5038,12 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	u16 chains_needed_per_io;
 	u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz;
 	u32 retry_sz;
+	u32 rdpq_sz = 0;
 	u16 max_request_credit, nvme_blocks_needed;
 	unsigned short sg_tablesize;
 	u16 sge_size;
 	int i, j;
+	int ret = 0;
 	struct chain_tracker *ct;
 
 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
@@ -5129,54 +5197,28 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	/* reply post queue, 16 byte align */
 	reply_post_free_sz = ioc->reply_post_queue_depth *
 	    sizeof(Mpi2DefaultReplyDescriptor_t);
-
-	sz = reply_post_free_sz;
+	rdpq_sz = reply_post_free_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
 	if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)
-		sz *= ioc->reply_queue_count;
-
-	ioc->reply_post = kcalloc((ioc->rdpq_array_enable) ?
-	    (ioc->reply_queue_count):1,
-	    sizeof(struct reply_post_struct), GFP_KERNEL);
-
-	if (!ioc->reply_post) {
-		ioc_err(ioc, "reply_post_free pool: kcalloc failed\n");
-		goto out;
-	}
-	ioc->reply_post_free_dma_pool = dma_pool_create("reply_post_free pool",
-	    &ioc->pdev->dev, sz, 16, 0);
-	if (!ioc->reply_post_free_dma_pool) {
-		ioc_err(ioc, "reply_post_free pool: dma_pool_create failed\n");
-		goto out;
-	}
-	i = 0;
-	do {
-		ioc->reply_post[i].reply_post_free =
-		    dma_pool_zalloc(ioc->reply_post_free_dma_pool,
-		    GFP_KERNEL,
-		    &ioc->reply_post[i].reply_post_free_dma);
-		if (!ioc->reply_post[i].reply_post_free) {
-			ioc_err(ioc, "reply_post_free pool: dma_pool_alloc failed\n");
-			goto out;
-		}
-		dinitprintk(ioc,
-			    ioc_info(ioc, "reply post free pool (0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n",
-				     ioc->reply_post[i].reply_post_free,
-				     ioc->reply_post_queue_depth,
-				     8, sz / 1024));
-		dinitprintk(ioc,
-			    ioc_info(ioc, "reply_post_free_dma = (0x%llx)\n",
-				     (u64)ioc->reply_post[i].reply_post_free_dma));
-		total_sz += sz;
-	} while (ioc->rdpq_array_enable && (++i < ioc->reply_queue_count));
-
-	if (ioc->dma_mask > 32) {
-		if (_base_change_consistent_dma_mask(ioc, ioc->pdev) != 0) {
-			ioc_warn(ioc, "no suitable consistent DMA mask for %s\n",
-				 pci_name(ioc->pdev));
-			goto out;
+		rdpq_sz = reply_post_free_sz * ioc->reply_queue_count;
+	ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz);
+	if (ret == -EAGAIN) {
+		/*
+		 * Free allocated bad RDPQ memory pools.
+		 * Change dma coherent mask to 32 bit and reallocate RDPQ
+		 */
+		_base_release_memory_pools(ioc);
+		ioc->use_32bit_dma = true;
+		if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {
+			ioc_err(ioc,
+			    "32 DMA mask failed %s\n", pci_name(ioc->pdev));
+			return -ENODEV;
 		}
-	}
-
+		if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz))
+			return -ENOMEM;
+	} else if (ret == -ENOMEM)
+		return -ENOMEM;
+	total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 :
+	    DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK));
 	ioc->scsiio_depth = ioc->hba_queue_depth -
 	    ioc->hi_priority_depth - ioc->internal_depth;
 
@@ -5188,7 +5230,6 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 		    ioc_info(ioc, "scsi host: can_queue depth (%d)\n",
 			     ioc->shost->can_queue));
 
-
 	/* contiguous pool for request and chains, 16 byte align, one extra "
 	 * "frame for smid=0
 	 */
@@ -5405,7 +5446,7 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	 * Actual requirement is not alignment, but we need start and end of
 	 * DMA address must have same upper 32 bit address.
 	 */
-	if (!is_MSB_are_same((long)ioc->sense, sz)) {
+	if (!mpt3sas_check_same_4gb_region((long)ioc->sense, sz)) {
 		//Release Sense pool & Reallocate
 		dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
 		dma_pool_destroy(ioc->sense_dma_pool);
@@ -7158,7 +7199,7 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	ioc->smp_affinity_enable = smp_affinity_enable;
 
 	ioc->rdpq_array_enable_assigned = 0;
-	ioc->dma_mask = 0;
+	ioc->use_32bit_dma = false;
 	if (ioc->is_aero_ioc)
 		ioc->base_readl = &_base_readl_aero;
 	else
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index e7197150721f..4fca3939c034 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -76,8 +76,8 @@
 #define MPT3SAS_DRIVER_NAME		"mpt3sas"
 #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>"
 #define MPT3SAS_DESCRIPTION	"LSI MPT Fusion SAS 3.0 Device Driver"
-#define MPT3SAS_DRIVER_VERSION		"33.100.00.00"
-#define MPT3SAS_MAJOR_VERSION		33
+#define MPT3SAS_DRIVER_VERSION		"34.100.00.00"
+#define MPT3SAS_MAJOR_VERSION		34
 #define MPT3SAS_MINOR_VERSION		100
 #define MPT3SAS_BUILD_VERSION		0
 #define MPT3SAS_RELEASE_VERSION	00
@@ -367,6 +367,7 @@ struct mpt3sas_nvme_cmd {
 #define MPT3SAS_HIGH_IOPS_REPLY_QUEUES		8
 #define MPT3SAS_HIGH_IOPS_BATCH_COUNT		16
 #define MPT3SAS_GEN35_MAX_MSIX_QUEUES		128
+#define RDPQ_MAX_INDEX_IN_ONE_CHUNK		16
 
 /* OEM Specific Flags will come from OEM specific header files */
 struct Mpi2ManufacturingPage10_t {
@@ -1026,7 +1027,6 @@ typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc);
  * @ir_firmware: IR firmware present
  * @bars: bitmask of BAR's that must be configured
  * @mask_interrupts: ignore interrupt
- * @dma_mask: used to set the consistent dma mask
  * @pci_access_mutex: Mutex to synchronize ioctl, sysfs show path and
  *			pci resource handling
  * @fault_reset_work_q_name: fw fault work queue
@@ -1064,6 +1064,7 @@ typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc);
  * @thresh_hold: Max number of reply descriptors processed
  *				before updating Host Index
  * @drv_support_bitmap: driver's supported feature bit map
+ * @use_32bit_dma: Flag to use 32 bit consistent dma mask
  * @scsi_io_cb_idx: shost generated commands
  * @tm_cb_idx: task management commands
  * @scsih_cb_idx: scsih internal commands
@@ -1205,7 +1206,6 @@ struct MPT3SAS_ADAPTER {
 	u8		ir_firmware;
 	int		bars;
 	u8		mask_interrupts;
-	int		dma_mask;
 
 	/* fw fault handler */
 	char		fault_reset_work_q_name[20];
@@ -1254,6 +1254,7 @@ struct MPT3SAS_ADAPTER {
 	u8		high_iops_queues;
 	u32		drv_support_bitmap;
 	bool		enable_sdev_max_qd;
+	bool		use_32bit_dma;
 
 	/* internal commands, callback index */
 	u8		scsi_io_cb_idx;
@@ -1471,6 +1472,8 @@ struct MPT3SAS_ADAPTER {
 	u16		device_remove_in_progress_sz;
 	u8		is_gen35_ioc;
 	u8		is_aero_ioc;
+	struct dentry	*debugfs_root;
+	struct dentry	*ioc_dump;
 	PUT_SMID_IO_FP_HIP put_smid_scsi_io;
 	PUT_SMID_IO_FP_HIP put_smid_fast_path;
 	PUT_SMID_IO_FP_HIP put_smid_hi_priority;
@@ -1478,6 +1481,11 @@ struct MPT3SAS_ADAPTER {
 	GET_MSIX_INDEX get_msix_index_for_smlio;
 };
 
+struct mpt3sas_debugfs_buffer {
+	void	*buf;
+	u32	len;
+};
+
 #define MPT_DRV_SUPPORT_BITMAP_MEMMOVE 0x00000001
 
 typedef u8 (*MPT_CALLBACK)(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
@@ -1781,6 +1789,11 @@ mpt3sas_setup_direct_io(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
 /* NCQ Prio Handling Check */
 bool scsih_ncq_prio_supp(struct scsi_device *sdev);
 
+void mpt3sas_setup_debugfs(struct MPT3SAS_ADAPTER *ioc);
+void mpt3sas_destroy_debugfs(struct MPT3SAS_ADAPTER *ioc);
+void mpt3sas_init_debugfs(void);
+void mpt3sas_exit_debugfs(void);
+
 /**
  * _scsih_is_pcie_scsi_device - determines if device is an pcie scsi device
  * @device_info: bitfield providing information about the device.
diff --git a/drivers/scsi/mpt3sas/mpt3sas_debugfs.c b/drivers/scsi/mpt3sas/mpt3sas_debugfs.c
new file mode 100644
index 000000000000..a6ab1db81167
--- /dev/null
+++ b/drivers/scsi/mpt3sas/mpt3sas_debugfs.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Debugfs interface Support for MPT (Message Passing Technology) based
+ * controllers.
+ *
+ * Copyright (C) 2020  Broadcom Inc.
+ *
+ * Authors: Broadcom Inc.
+ * Sreekanth Reddy  <sreekanth.reddy@broadcom.com>
+ * Suganath Prabu <suganath-prabu.subramani@broadcom.com>
+ *
+ * Send feedback to : MPT-FusionLinux.pdl@broadcom.com)
+ *
+ **/
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/compat.h>
+#include <linux/uio.h>
+
+#include <scsi/scsi.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_host.h>
+#include "mpt3sas_base.h"
+#include <linux/debugfs.h>
+
+static struct dentry *mpt3sas_debugfs_root;
+
+/*
+ * _debugfs_iocdump_read - copy ioc dump from debugfs buffer
+ * @filep:	File Pointer
+ * @ubuf:	Buffer to fill data
+ * @cnt:	Length of the buffer
+ * @ppos:	Offset in the file
+ */
+
+static ssize_t
+_debugfs_iocdump_read(struct file *filp, char __user *ubuf, size_t cnt,
+	loff_t *ppos)
+
+{
+	struct mpt3sas_debugfs_buffer *debug = filp->private_data;
+
+	if (!debug || !debug->buf)
+		return 0;
+
+	return simple_read_from_buffer(ubuf, cnt, ppos, debug->buf, debug->len);
+}
+
+/*
+ * _debugfs_iocdump_open :	open the ioc_dump debugfs attribute file
+ */
+static int
+_debugfs_iocdump_open(struct inode *inode, struct file *file)
+{
+	struct MPT3SAS_ADAPTER *ioc = inode->i_private;
+	struct mpt3sas_debugfs_buffer *debug;
+
+	debug = kzalloc(sizeof(struct mpt3sas_debugfs_buffer), GFP_KERNEL);
+	if (!debug)
+		return -ENOMEM;
+
+	debug->buf = (void *)ioc;
+	debug->len = sizeof(struct MPT3SAS_ADAPTER);
+	file->private_data = debug;
+	return 0;
+}
+
+/*
+ * _debugfs_iocdump_release :	release the ioc_dump debugfs attribute
+ * @inode: inode structure to the corresponds device
+ * @file: File pointer
+ */
+static int
+_debugfs_iocdump_release(struct inode *inode, struct file *file)
+{
+	struct mpt3sas_debugfs_buffer *debug = file->private_data;
+
+	if (!debug)
+		return 0;
+
+	file->private_data = NULL;
+	kfree(debug);
+	return 0;
+}
+
+static const struct file_operations mpt3sas_debugfs_iocdump_fops = {
+	.owner		= THIS_MODULE,
+	.open           = _debugfs_iocdump_open,
+	.read           = _debugfs_iocdump_read,
+	.release        = _debugfs_iocdump_release,
+};
+
+/*
+ * mpt3sas_init_debugfs :	Create debugfs root for mpt3sas driver
+ */
+void mpt3sas_init_debugfs(void)
+{
+	mpt3sas_debugfs_root = debugfs_create_dir("mpt3sas", NULL);
+	if (!mpt3sas_debugfs_root)
+		pr_info("mpt3sas: Cannot create debugfs root\n");
+}
+
+/*
+ * mpt3sas_exit_debugfs :	Remove debugfs root for mpt3sas driver
+ */
+void mpt3sas_exit_debugfs(void)
+{
+	debugfs_remove_recursive(mpt3sas_debugfs_root);
+}
+
+/*
+ * mpt3sas_setup_debugfs :	Setup debugfs per HBA adapter
+ * ioc:				MPT3SAS_ADAPTER object
+ */
+void
+mpt3sas_setup_debugfs(struct MPT3SAS_ADAPTER *ioc)
+{
+	char name[64];
+
+	snprintf(name, sizeof(name), "scsi_host%d", ioc->shost->host_no);
+	if (!ioc->debugfs_root) {
+		ioc->debugfs_root =
+		    debugfs_create_dir(name, mpt3sas_debugfs_root);
+		if (!ioc->debugfs_root) {
+			dev_err(&ioc->pdev->dev,
+			    "Cannot create per adapter debugfs directory\n");
+			return;
+		}
+	}
+
+	snprintf(name, sizeof(name), "ioc_dump");
+	ioc->ioc_dump =	debugfs_create_file(name, 0444,
+	    ioc->debugfs_root, ioc, &mpt3sas_debugfs_iocdump_fops);
+	if (!ioc->ioc_dump) {
+		dev_err(&ioc->pdev->dev,
+		    "Cannot create ioc_dump debugfs file\n");
+		debugfs_remove(ioc->debugfs_root);
+		return;
+	}
+
+	snprintf(name, sizeof(name), "host_recovery");
+	debugfs_create_u8(name, 0444, ioc->debugfs_root, &ioc->shost_recovery);
+
+}
+
+/*
+ * mpt3sas_destroy_debugfs :	Destroy debugfs per HBA adapter
+ * @ioc:	MPT3SAS_ADAPTER object
+ */
+void mpt3sas_destroy_debugfs(struct MPT3SAS_ADAPTER *ioc)
+{
+	debugfs_remove_recursive(ioc->debugfs_root);
+}
+
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 04a40afe60e3..08fc4b381056 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -9928,6 +9928,7 @@ static void scsih_remove(struct pci_dev *pdev)
 				&ioc->ioc_pg1_copy);
 	/* release all the volumes */
 	_scsih_ir_shutdown(ioc);
+	mpt3sas_destroy_debugfs(ioc);
 	sas_remove_host(shost);
 	list_for_each_entry_safe(raid_device, next, &ioc->raid_device_list,
 	    list) {
@@ -10763,8 +10764,8 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		}
 	}
 	/* register EEDP capabilities with SCSI layer */
-	if (prot_mask > 0)
-		scsi_host_set_prot(shost, prot_mask);
+	if (prot_mask >= 0)
+		scsi_host_set_prot(shost, (prot_mask & 0x07));
 	else
 		scsi_host_set_prot(shost, SHOST_DIF_TYPE1_PROTECTION
 				   | SHOST_DIF_TYPE2_PROTECTION
@@ -10814,6 +10815,7 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	}
 
 	scsi_scan_host(shost);
+	mpt3sas_setup_debugfs(ioc);
 	return 0;
 out_add_shost_fail:
 	mpt3sas_base_detach(ioc);
@@ -11220,6 +11222,7 @@ scsih_init(void)
 	tm_sas_control_cb_idx = mpt3sas_base_register_callback_handler(
 	    _scsih_sas_control_complete);
 
+	mpt3sas_init_debugfs();
 	return 0;
 }
 
@@ -11251,6 +11254,7 @@ scsih_exit(void)
 	if (hbas_to_enumerate != 2)
 		raid_class_release(mpt2sas_raid_template);
 	sas_release_transport(mpt3sas_transport_template);
+	mpt3sas_exit_debugfs();
 }
 
 /**
diff --git a/drivers/scsi/mvme147.c b/drivers/scsi/mvme147.c
index ca96d6d9c350..869b8b058a43 100644
--- a/drivers/scsi/mvme147.c
+++ b/drivers/scsi/mvme147.c
@@ -8,7 +8,6 @@
 #include <linux/module.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/mvme147hw.h>
 #include <asm/irq.h>
 
diff --git a/drivers/scsi/mvsas/mv_init.c b/drivers/scsi/mvsas/mv_init.c
index 7af9173c4925..5973eed94938 100644
--- a/drivers/scsi/mvsas/mv_init.c
+++ b/drivers/scsi/mvsas/mv_init.c
@@ -25,7 +25,7 @@ static const struct mvs_chip_info mvs_chips[] = {
 	[chip_1320] =	{ 2, 4, 0x800, 17, 64, 8,  9, &mvs_94xx_dispatch, },
 };
 
-struct device_attribute *mvst_host_attrs[];
+static struct device_attribute *mvst_host_attrs[];
 
 #define SOC_SAS_NUM 2
 
@@ -759,8 +759,6 @@ static DEVICE_ATTR(interrupt_coalescing,
 			 mvs_show_interrupt_coalescing,
 			 mvs_store_interrupt_coalescing);
 
-/* task handler */
-struct task_struct *mvs_th;
 static int __init mvs_init(void)
 {
 	int rc;
@@ -785,7 +783,7 @@ static void __exit mvs_exit(void)
 	sas_release_transport(mvs_stt);
 }
 
-struct device_attribute *mvst_host_attrs[] = {
+static struct device_attribute *mvst_host_attrs[] = {
 	&dev_attr_driver_version,
 	&dev_attr_interrupt_coalescing,
 	NULL,
diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
index 7eb88fe1eb0b..aa9ae2ae8579 100644
--- a/drivers/scsi/pmcraid.c
+++ b/drivers/scsi/pmcraid.c
@@ -4652,7 +4652,7 @@ static int pmcraid_allocate_control_blocks(struct pmcraid_instance *pinstance)
 
 	for (i = 0; i < PMCRAID_MAX_CMD; i++) {
 		pinstance->cmd_list[i]->ioa_cb =
-			dma_pool_alloc(
+			dma_pool_zalloc(
 				pinstance->control_pool,
 				GFP_KERNEL,
 				&(pinstance->cmd_list[i]->ioa_cb_bus_addr));
@@ -4661,8 +4661,6 @@ static int pmcraid_allocate_control_blocks(struct pmcraid_instance *pinstance)
 			pmcraid_release_control_blocks(pinstance, i);
 			return -ENOMEM;
 		}
-		memset(pinstance->cmd_list[i]->ioa_cb, 0,
-			sizeof(struct pmcraid_control_block));
 	}
 	return 0;
 }
diff --git a/drivers/scsi/qedf/Kconfig b/drivers/scsi/qedf/Kconfig
index 7cd993be4e57..eb81a1b0374a 100644
--- a/drivers/scsi/qedf/Kconfig
+++ b/drivers/scsi/qedf/Kconfig
@@ -7,6 +7,6 @@ config QEDF
         depends on LIBFCOE
 	select QED_LL2
 	select QED_FCOE
-	---help---
+	help
 	This driver supports FCoE offload for the QLogic FastLinQ
 	41000 Series Converged Network Adapters.
diff --git a/drivers/scsi/qedf/qedf.h b/drivers/scsi/qedf/qedf.h
index f3f399fe10c8..e163be8af965 100644
--- a/drivers/scsi/qedf/qedf.h
+++ b/drivers/scsi/qedf/qedf.h
@@ -355,6 +355,7 @@ struct qedf_ctx {
 #define QEDF_GRCDUMP_CAPTURE		4
 #define QEDF_IN_RECOVERY		5
 #define QEDF_DBG_STOP_IO		6
+#define QEDF_PROBING			8
 	unsigned long flags; /* Miscellaneous state flags */
 	int fipvlan_retries;
 	u8 num_queues;
@@ -387,7 +388,9 @@ struct qedf_ctx {
 #define QEDF_IO_WORK_MIN		64
 	mempool_t *io_mempool;
 	struct workqueue_struct *dpc_wq;
+	struct delayed_work recovery_work;
 	struct delayed_work grcdump_work;
+	struct delayed_work stag_work;
 
 	u32 slow_sge_ios;
 	u32 fast_sge_ios;
@@ -403,6 +406,7 @@ struct qedf_ctx {
 
 	u32 flogi_cnt;
 	u32 flogi_failed;
+	u32 flogi_pending;
 
 	/* Used for fc statistics */
 	struct mutex stats_mutex;
@@ -468,7 +472,7 @@ extern uint qedf_dump_frames;
 extern uint qedf_io_tracing;
 extern uint qedf_stop_io_on_error;
 extern uint qedf_link_down_tmo;
-#define QEDF_RETRY_DELAY_MAX		20 /* 2 seconds */
+#define QEDF_RETRY_DELAY_MAX		600 /* 60 seconds */
 extern bool qedf_retry_delay;
 extern uint qedf_debug;
 
diff --git a/drivers/scsi/qedf/qedf_els.c b/drivers/scsi/qedf/qedf_els.c
index 87e169dcebdb..542ba9454257 100644
--- a/drivers/scsi/qedf/qedf_els.c
+++ b/drivers/scsi/qedf/qedf_els.c
@@ -388,14 +388,10 @@ void qedf_restart_rport(struct qedf_rport *fcport)
 		mutex_lock(&lport->disc.disc_mutex);
 		/* Recreate the rport and log back in */
 		rdata = fc_rport_create(lport, port_id);
-		if (rdata) {
-			mutex_unlock(&lport->disc.disc_mutex);
+		mutex_unlock(&lport->disc.disc_mutex);
+		if (rdata)
 			fc_rport_login(rdata);
-			fcport->rdata = rdata;
-		} else {
-			mutex_unlock(&lport->disc.disc_mutex);
-			fcport->rdata = NULL;
-		}
+		fcport->rdata = rdata;
 	}
 	clear_bit(QEDF_RPORT_IN_RESET, &fcport->flags);
 }
diff --git a/drivers/scsi/qedf/qedf_fip.c b/drivers/scsi/qedf/qedf_fip.c
index bb82f0875eca..ad6a56ce72a8 100644
--- a/drivers/scsi/qedf/qedf_fip.c
+++ b/drivers/scsi/qedf/qedf_fip.c
@@ -20,7 +20,7 @@ void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf)
 #define MY_FIP_ALL_FCF_MACS        ((__u8[6]) { 1, 0x10, 0x18, 1, 0, 2 })
 	static u8 my_fcoe_all_fcfs[ETH_ALEN] = MY_FIP_ALL_FCF_MACS;
 	unsigned long flags = 0;
-	int rc = -1;
+	int rc;
 
 	skb = dev_alloc_skb(sizeof(struct fip_vlan));
 	if (!skb) {
diff --git a/drivers/scsi/qedf/qedf_io.c b/drivers/scsi/qedf/qedf_io.c
index e749a2dcaad7..0f6a15c1a04b 100644
--- a/drivers/scsi/qedf/qedf_io.c
+++ b/drivers/scsi/qedf/qedf_io.c
@@ -1021,14 +1021,18 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 	atomic_inc(&fcport->ios_to_queue);
 
 	if (fcport->retry_delay_timestamp) {
+		/* Take fcport->rport_lock for resetting the delay_timestamp */
+		spin_lock_irqsave(&fcport->rport_lock, flags);
 		if (time_after(jiffies, fcport->retry_delay_timestamp)) {
 			fcport->retry_delay_timestamp = 0;
 		} else {
+			spin_unlock_irqrestore(&fcport->rport_lock, flags);
 			/* If retry_delay timer is active, flow off the ML */
 			rc = SCSI_MLQUEUE_TARGET_BUSY;
 			atomic_dec(&fcport->ios_to_queue);
 			goto exit_qcmd;
 		}
+		spin_unlock_irqrestore(&fcport->rport_lock, flags);
 	}
 
 	io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
@@ -1134,6 +1138,8 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 	int refcount;
 	u16 scope, qualifier = 0;
 	u8 fw_residual_flag = 0;
+	unsigned long flags = 0;
+	u16 chk_scope = 0;
 
 	if (!io_req)
 		return;
@@ -1267,16 +1273,8 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 				/* Lower 14 bits */
 				qualifier = fcp_rsp->retry_delay_timer & 0x3FFF;
 
-				if (qedf_retry_delay &&
-				    scope > 0 && qualifier > 0 &&
-				    qualifier <= 0x3FEF) {
-					/* Check we don't go over the max */
-					if (qualifier > QEDF_RETRY_DELAY_MAX)
-						qualifier =
-						    QEDF_RETRY_DELAY_MAX;
-					fcport->retry_delay_timestamp =
-					    jiffies + (qualifier * HZ / 10);
-				}
+				if (qedf_retry_delay)
+					chk_scope = 1;
 				/* Record stats */
 				if (io_req->cdb_status ==
 				    SAM_STAT_TASK_SET_FULL)
@@ -1287,6 +1285,36 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 		}
 		if (io_req->fcp_resid)
 			scsi_set_resid(sc_cmd, io_req->fcp_resid);
+
+		if (chk_scope == 1) {
+			if ((scope == 1 || scope == 2) &&
+			    (qualifier > 0 && qualifier <= 0x3FEF)) {
+				/* Check we don't go over the max */
+				if (qualifier > QEDF_RETRY_DELAY_MAX) {
+					qualifier = QEDF_RETRY_DELAY_MAX;
+					QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+						  "qualifier = %d\n",
+						  (fcp_rsp->retry_delay_timer &
+						  0x3FFF));
+				}
+				QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+					  "Scope = %d and qualifier = %d",
+					  scope, qualifier);
+				/*  Take fcport->rport_lock to
+				 *  update the retry_delay_timestamp
+				 */
+				spin_lock_irqsave(&fcport->rport_lock, flags);
+				fcport->retry_delay_timestamp =
+					jiffies + (qualifier * HZ / 10);
+				spin_unlock_irqrestore(&fcport->rport_lock,
+						       flags);
+
+			} else {
+				QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+					  "combination of scope = %d and qualifier = %d is not handled in qedf.\n",
+					  scope, qualifier);
+			}
+		}
 		break;
 	default:
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "fcp_status=%d.\n",
diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
index 5b19f5175c5c..36b1ca2dadbb 100644
--- a/drivers/scsi/qedf/qedf_main.c
+++ b/drivers/scsi/qedf/qedf_main.c
@@ -28,6 +28,8 @@ const struct qed_fcoe_ops *qed_ops;
 static int qedf_probe(struct pci_dev *pdev, const struct pci_device_id *id);
 static void qedf_remove(struct pci_dev *pdev);
 static void qedf_shutdown(struct pci_dev *pdev);
+static void qedf_schedule_recovery_handler(void *dev);
+static void qedf_recovery_handler(struct work_struct *work);
 
 /*
  * Driver module parameters.
@@ -282,6 +284,7 @@ static void qedf_flogi_resp(struct fc_seq *seq, struct fc_frame *fp,
 	else if (fc_frame_payload_op(fp) == ELS_LS_ACC) {
 		/* Set the source MAC we will use for FCoE traffic */
 		qedf_set_data_src_addr(qedf, fp);
+		qedf->flogi_pending = 0;
 	}
 
 	/* Complete flogi_compl so we can proceed to sending ADISCs */
@@ -307,6 +310,11 @@ static struct fc_seq *qedf_elsct_send(struct fc_lport *lport, u32 did,
 	 */
 	if (resp == fc_lport_flogi_resp) {
 		qedf->flogi_cnt++;
+		if (qedf->flogi_pending >= QEDF_FLOGI_RETRY_CNT) {
+			schedule_delayed_work(&qedf->stag_work, 2);
+			return NULL;
+		}
+		qedf->flogi_pending++;
 		return fc_elsct_send(lport, did, fp, op, qedf_flogi_resp,
 		    arg, timeout);
 	}
@@ -503,6 +511,32 @@ static void qedf_update_link_speed(struct qedf_ctx *qedf,
 	fc_host_supported_speeds(lport->host) = lport->link_supported_speeds;
 }
 
+static void qedf_bw_update(void *dev)
+{
+	struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
+	struct qed_link_output link;
+
+	/* Get the latest status of the link */
+	qed_ops->common->get_link(qedf->cdev, &link);
+
+	if (test_bit(QEDF_UNLOADING, &qedf->flags)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Ignore link update, driver getting unload.\n");
+		return;
+	}
+
+	if (link.link_up) {
+		if (atomic_read(&qedf->link_state) == QEDF_LINK_UP)
+			qedf_update_link_speed(qedf, &link);
+		else
+			QEDF_ERR(&qedf->dbg_ctx,
+				 "Ignore bw update, link is down.\n");
+
+	} else {
+		QEDF_ERR(&qedf->dbg_ctx, "link_up is not set.\n");
+	}
+}
+
 static void qedf_link_update(void *dev, struct qed_link_output *link)
 {
 	struct qedf_ctx *qedf = (struct qedf_ctx *)dev;
@@ -629,6 +663,8 @@ static u32 qedf_get_login_failures(void *cookie)
 static struct qed_fcoe_cb_ops qedf_cb_ops = {
 	{
 		.link_update = qedf_link_update,
+		.bw_update = qedf_bw_update,
+		.schedule_recovery_handler = qedf_schedule_recovery_handler,
 		.dcbx_aen = qedf_dcbx_handler,
 		.get_generic_tlv_data = qedf_get_generic_tlv_data,
 		.get_protocol_tlv_data = qedf_get_protocol_tlv_data,
@@ -850,6 +886,7 @@ void qedf_ctx_soft_reset(struct fc_lport *lport)
 
 	qedf = lport_priv(lport);
 
+	qedf->flogi_pending = 0;
 	/* For host reset, essentially do a soft link up/down */
 	atomic_set(&qedf->link_state, QEDF_LINK_DOWN);
 	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
@@ -3153,7 +3190,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 {
 	int rc = -EINVAL;
 	struct fc_lport *lport;
-	struct qedf_ctx *qedf;
+	struct qedf_ctx *qedf = NULL;
 	struct Scsi_Host *host;
 	bool is_vf = false;
 	struct qed_ll2_params params;
@@ -3183,6 +3220,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 
 		/* Initialize qedf_ctx */
 		qedf = lport_priv(lport);
+		set_bit(QEDF_PROBING, &qedf->flags);
 		qedf->lport = lport;
 		qedf->ctlr.lp = lport;
 		qedf->pdev = pdev;
@@ -3197,6 +3235,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 		init_completion(&qedf->fipvlan_compl);
 		mutex_init(&qedf->stats_mutex);
 		mutex_init(&qedf->flush_mutex);
+		qedf->flogi_pending = 0;
 
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_INFO,
 		   "QLogic FastLinQ FCoE Module qedf %s, "
@@ -3206,9 +3245,12 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	} else {
 		/* Init pointers during recovery */
 		qedf = pci_get_drvdata(pdev);
+		set_bit(QEDF_PROBING, &qedf->flags);
 		lport = qedf->lport;
 	}
 
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC, "Probe started.\n");
+
 	host = lport->host;
 
 	/* Allocate mempool for qedf_io_work structs */
@@ -3227,6 +3269,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	INIT_DELAYED_WORK(&qedf->link_update, qedf_handle_link_update);
 	INIT_DELAYED_WORK(&qedf->link_recovery, qedf_link_recovery);
 	INIT_DELAYED_WORK(&qedf->grcdump_work, qedf_wq_grcdump);
+	INIT_DELAYED_WORK(&qedf->stag_work, qedf_stag_change_work);
 	qedf->fipvlan_retries = qedf_fipvlan_retries;
 	/* Set a default prio in case DCBX doesn't converge */
 	if (qedf_default_prio > -1) {
@@ -3281,6 +3324,13 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	}
 	qed_ops->common->update_pf_params(qedf->cdev, &qedf->pf_params);
 
+	/* Learn information crucial for qedf to progress */
+	rc = qed_ops->fill_dev_info(qedf->cdev, &qedf->dev_info);
+	if (rc) {
+		QEDF_ERR(&qedf->dbg_ctx, "Failed to fill dev info.\n");
+		goto err2;
+	}
+
 	/* Record BDQ producer doorbell addresses */
 	qedf->bdq_primary_prod = qedf->dev_info.primary_dbq_rq_addr;
 	qedf->bdq_secondary_prod = qedf->dev_info.secondary_bdq_rq_addr;
@@ -3466,6 +3516,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 		    qedf->lport->host->host_no);
 		qedf->dpc_wq = create_workqueue(host_buf);
 	}
+	INIT_DELAYED_WORK(&qedf->recovery_work, qedf_recovery_handler);
 
 	/*
 	 * GRC dump and sysfs parameters are not reaped during the recovery
@@ -3513,6 +3564,10 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	else
 		fc_fabric_login(lport);
 
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC, "Probe done.\n");
+
+	clear_bit(QEDF_PROBING, &qedf->flags);
+
 	/* All good */
 	return 0;
 
@@ -3538,6 +3593,11 @@ err2:
 err1:
 	scsi_host_put(lport->host);
 err0:
+	if (qedf) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC, "Probe done.\n");
+
+		clear_bit(QEDF_PROBING, &qedf->flags);
+	}
 	return rc;
 }
 
@@ -3687,11 +3747,25 @@ void qedf_get_protocol_tlv_data(void *dev, void *data)
 {
 	struct qedf_ctx *qedf = dev;
 	struct qed_mfw_tlv_fcoe *fcoe = data;
-	struct fc_lport *lport = qedf->lport;
-	struct Scsi_Host *host = lport->host;
-	struct fc_host_attrs *fc_host = shost_to_fc_host(host);
+	struct fc_lport *lport;
+	struct Scsi_Host *host;
+	struct fc_host_attrs *fc_host;
 	struct fc_host_statistics *hst;
 
+	if (!qedf) {
+		QEDF_ERR(NULL, "qedf is null.\n");
+		return;
+	}
+
+	if (test_bit(QEDF_PROBING, &qedf->flags)) {
+		QEDF_ERR(&qedf->dbg_ctx, "Function is still probing.\n");
+		return;
+	}
+
+	lport = qedf->lport;
+	host = lport->host;
+	fc_host = shost_to_fc_host(host);
+
 	/* Force a refresh of the fc_host stats including offload stats */
 	hst = qedf_fc_get_host_stats(host);
 
@@ -3762,11 +3836,64 @@ void qedf_get_protocol_tlv_data(void *dev, void *data)
 	fcoe->scsi_tsk_full = qedf->task_set_fulls;
 }
 
+/* Deferred work function to perform soft context reset on STAG change */
+void qedf_stag_change_work(struct work_struct *work)
+{
+	struct qedf_ctx *qedf =
+	    container_of(work, struct qedf_ctx, stag_work.work);
+
+	if (!qedf) {
+		QEDF_ERR(&qedf->dbg_ctx, "qedf is NULL");
+		return;
+	}
+	QEDF_ERR(&qedf->dbg_ctx, "Performing software context reset.\n");
+	qedf_ctx_soft_reset(qedf->lport);
+}
+
 static void qedf_shutdown(struct pci_dev *pdev)
 {
 	__qedf_remove(pdev, QEDF_MODE_NORMAL);
 }
 
+/*
+ * Recovery handler code
+ */
+static void qedf_schedule_recovery_handler(void *dev)
+{
+	struct qedf_ctx *qedf = dev;
+
+	QEDF_ERR(&qedf->dbg_ctx, "Recovery handler scheduled.\n");
+	schedule_delayed_work(&qedf->recovery_work, 0);
+}
+
+static void qedf_recovery_handler(struct work_struct *work)
+{
+	struct qedf_ctx *qedf =
+	    container_of(work, struct qedf_ctx, recovery_work.work);
+
+	if (test_and_set_bit(QEDF_IN_RECOVERY, &qedf->flags))
+		return;
+
+	/*
+	 * Call common_ops->recovery_prolog to allow the MFW to quiesce
+	 * any PCI transactions.
+	 */
+	qed_ops->common->recovery_prolog(qedf->cdev);
+
+	QEDF_ERR(&qedf->dbg_ctx, "Recovery work start.\n");
+	__qedf_remove(qedf->pdev, QEDF_MODE_RECOVERY);
+	/*
+	 * Reset link and dcbx to down state since we will not get a link down
+	 * event from the MFW but calling __qedf_remove will essentially be a
+	 * link down event.
+	 */
+	atomic_set(&qedf->link_state, QEDF_LINK_DOWN);
+	atomic_set(&qedf->dcbx, QEDF_DCBX_PENDING);
+	__qedf_probe(qedf->pdev, QEDF_MODE_RECOVERY);
+	clear_bit(QEDF_IN_RECOVERY, &qedf->flags);
+	QEDF_ERR(&qedf->dbg_ctx, "Recovery work complete.\n");
+}
+
 /* Generic TLV data callback */
 void qedf_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
 {
diff --git a/drivers/scsi/qedi/Kconfig b/drivers/scsi/qedi/Kconfig
index 7ab07f3b453f..2091d883a584 100644
--- a/drivers/scsi/qedi/Kconfig
+++ b/drivers/scsi/qedi/Kconfig
@@ -8,6 +8,6 @@ config QEDI
 	select QED_OOO
 	select QED_ISCSI
 	select ISCSI_BOOT_SYSFS
-	---help---
+	help
 	This driver supports iSCSI offload for the QLogic FastLinQ
 	41000 Series Converged Network Adapters.
diff --git a/drivers/scsi/qedi/qedi_iscsi.c b/drivers/scsi/qedi/qedi_iscsi.c
index 1f4a5fb00a05..425e665ec08b 100644
--- a/drivers/scsi/qedi/qedi_iscsi.c
+++ b/drivers/scsi/qedi/qedi_iscsi.c
@@ -836,6 +836,11 @@ qedi_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
 		return ERR_PTR(ret);
 	}
 
+	if (atomic_read(&qedi->link_state) != QEDI_LINK_UP) {
+		QEDI_WARN(&qedi->dbg_ctx, "qedi link down\n");
+		return ERR_PTR(-ENXIO);
+	}
+
 	ep = iscsi_create_endpoint(sizeof(struct qedi_endpoint));
 	if (!ep) {
 		QEDI_ERR(&qedi->dbg_ctx, "endpoint create fail\n");
@@ -870,12 +875,6 @@ qedi_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
 		QEDI_ERR(&qedi->dbg_ctx, "Invalid endpoint\n");
 	}
 
-	if (atomic_read(&qedi->link_state) != QEDI_LINK_UP) {
-		QEDI_WARN(&qedi->dbg_ctx, "qedi link down\n");
-		ret = -ENXIO;
-		goto ep_conn_exit;
-	}
-
 	ret = qedi_alloc_sq(qedi, qedi_ep);
 	if (ret)
 		goto ep_conn_exit;
@@ -1001,7 +1000,8 @@ static void qedi_ep_disconnect(struct iscsi_endpoint *ep)
 	if (qedi_ep->state == EP_STATE_OFLDCONN_START)
 		goto ep_exit_recover;
 
-	flush_work(&qedi_ep->offload_work);
+	if (qedi_ep->state != EP_STATE_OFLDCONN_NONE)
+		flush_work(&qedi_ep->offload_work);
 
 	if (qedi_ep->conn) {
 		qedi_conn = qedi_ep->conn;
@@ -1065,6 +1065,9 @@ static void qedi_ep_disconnect(struct iscsi_endpoint *ep)
 		break;
 	}
 
+	if (!abrt_conn)
+		wait_delay += qedi->pf_params.iscsi_pf_params.two_msl_timer;
+
 	qedi_ep->state = EP_STATE_DISCONN_START;
 	ret = qedi_ops->destroy_conn(qedi->cdev, qedi_ep->handle, abrt_conn);
 	if (ret) {
@@ -1218,6 +1221,10 @@ static int qedi_set_path(struct Scsi_Host *shost, struct iscsi_path *path_data)
 	}
 
 	iscsi_cid = (u32)path_data->handle;
+	if (iscsi_cid >= qedi->max_active_conns) {
+		ret = -EINVAL;
+		goto set_path_exit;
+	}
 	qedi_ep = qedi->ep_tbl[iscsi_cid];
 	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
 		  "iscsi_cid=0x%x, qedi_ep=%p\n", iscsi_cid, qedi_ep);
diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c
index b995b19865ca..81a307695cc9 100644
--- a/drivers/scsi/qedi/qedi_main.c
+++ b/drivers/scsi/qedi/qedi_main.c
@@ -28,6 +28,10 @@
 #include "qedi_gbl.h"
 #include "qedi_iscsi.h"
 
+static uint qedi_qed_debug;
+module_param(qedi_qed_debug, uint, 0644);
+MODULE_PARM_DESC(qedi_qed_debug, " QED debug level 0 (default)");
+
 static uint qedi_fw_debug;
 module_param(qedi_fw_debug, uint, 0644);
 MODULE_PARM_DESC(qedi_fw_debug, " Firmware debug level 0(default) to 3");
@@ -41,7 +45,7 @@ module_param(qedi_io_tracing, uint, 0644);
 MODULE_PARM_DESC(qedi_io_tracing,
 		 " Enable logging of SCSI requests/completions into trace buffer. (default off).");
 
-uint qedi_ll2_buf_size = 0x400;
+static uint qedi_ll2_buf_size = 0x400;
 module_param(qedi_ll2_buf_size, uint, 0644);
 MODULE_PARM_DESC(qedi_ll2_buf_size,
 		 "parameter to set ping packet size, default - 0x400, Jumbo packets - 0x2400.");
@@ -658,8 +662,6 @@ exit_setup_shost:
 static int qedi_ll2_rx(void *cookie, struct sk_buff *skb, u32 arg1, u32 arg2)
 {
 	struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;
-	struct qedi_uio_dev *udev;
-	struct qedi_uio_ctrl *uctrl;
 	struct skb_work_list *work;
 	struct ethhdr *eh;
 
@@ -698,9 +700,6 @@ static int qedi_ll2_rx(void *cookie, struct sk_buff *skb, u32 arg1, u32 arg2)
 		  "Allowed frame ethertype [0x%x] len [0x%x].\n",
 		  eh->h_proto, skb->len);
 
-	udev = qedi->udev;
-	uctrl = udev->uctrl;
-
 	work = kzalloc(sizeof(*work), GFP_ATOMIC);
 	if (!work) {
 		QEDI_WARN(&qedi->dbg_ctx,
@@ -921,7 +920,7 @@ static void qedi_get_boot_tgt_info(struct nvm_iscsi_block *block,
 	ipv6_en = !!(block->generic.ctrl_flags &
 		     NVM_ISCSI_CFG_GEN_IPV6_ENABLED);
 
-	snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s\n",
+	snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s",
 		 block->target[index].target_name.byte);
 
 	tgt->ipv6_en = ipv6_en;
@@ -1302,13 +1301,13 @@ process_again:
 			  "process already running\n");
 	}
 
-	if (qedi_fp_has_work(fp) == 0)
+	if (!qedi_fp_has_work(fp))
 		qed_sb_update_sb_idx(fp->sb_info);
 
 	/* Check for more work */
 	rmb();
 
-	if (qedi_fp_has_work(fp) == 0)
+	if (!qedi_fp_has_work(fp))
 		qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
 	else
 		goto process_again;
@@ -1360,7 +1359,7 @@ static int qedi_request_msix_irq(struct qedi_ctx *qedi)
 	u16 idx;
 
 	cpu = cpumask_first(cpu_online_mask);
-	for (i = 0; i < MIN_NUM_CPUS_MSIX(qedi); i++) {
+	for (i = 0; i < qedi->int_info.msix_cnt; i++) {
 		idx = i * qedi->dev_info.common.num_hwfns +
 			  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
 
@@ -2422,7 +2421,6 @@ static int __qedi_probe(struct pci_dev *pdev, int mode)
 {
 	struct qedi_ctx *qedi;
 	struct qed_ll2_params params;
-	u32 dp_module = 0;
 	u8 dp_level = 0;
 	bool is_vf = false;
 	char host_buf[16];
@@ -2445,7 +2443,7 @@ static int __qedi_probe(struct pci_dev *pdev, int mode)
 
 	memset(&qed_params, 0, sizeof(qed_params));
 	qed_params.protocol = QED_PROTOCOL_ISCSI;
-	qed_params.dp_module = dp_module;
+	qed_params.dp_module = qedi_qed_debug;
 	qed_params.dp_level = dp_level;
 	qed_params.is_vf = is_vf;
 	qedi->cdev = qedi_ops->common->probe(pdev, &qed_params);
diff --git a/drivers/scsi/qla1280.c b/drivers/scsi/qla1280.c
index 3337cd341d21..441a45349349 100644
--- a/drivers/scsi/qla1280.c
+++ b/drivers/scsi/qla1280.c
@@ -526,7 +526,7 @@ static struct pci_device_id qla1280_pci_tbl[] = {
 };
 MODULE_DEVICE_TABLE(pci, qla1280_pci_tbl);
 
-DEFINE_MUTEX(qla1280_firmware_mutex);
+static DEFINE_MUTEX(qla1280_firmware_mutex);
 
 struct qla_fw {
 	char *fwname;
@@ -535,7 +535,7 @@ struct qla_fw {
 
 #define QL_NUM_FW_IMAGES 3
 
-struct qla_fw qla1280_fw_tbl[QL_NUM_FW_IMAGES] = {
+static struct qla_fw qla1280_fw_tbl[QL_NUM_FW_IMAGES] = {
 	{"qlogic/1040.bin",  NULL},	/* image 0 */
 	{"qlogic/1280.bin",  NULL},	/* image 1 */
 	{"qlogic/12160.bin", NULL},	/* image 2 */
diff --git a/drivers/scsi/qla2xxx/Kconfig b/drivers/scsi/qla2xxx/Kconfig
index 764501838e21..802c373fd6d9 100644
--- a/drivers/scsi/qla2xxx/Kconfig
+++ b/drivers/scsi/qla2xxx/Kconfig
@@ -6,7 +6,7 @@ config SCSI_QLA_FC
 	depends on NVME_FC || !NVME_FC
 	select FW_LOADER
 	select BTREE
-	---help---
+	help
 	This qla2xxx driver supports all QLogic Fibre Channel
 	PCI and PCIe host adapters.
 
@@ -37,14 +37,14 @@ config TCM_QLA2XXX
 	depends on LIBFC
 	select BTREE
 	default n
-	---help---
+	help
 	Say Y here to enable the TCM_QLA2XXX fabric module for QLogic 24xx+ series target mode HBAs
 
 if TCM_QLA2XXX
 config TCM_QLA2XXX_DEBUG
 	bool "TCM_QLA2XXX fabric module DEBUG mode for QLogic 24xx+ series target mode HBAs"
 	default n
-	---help---
+	help
 	Say Y here to enable the TCM_QLA2XXX fabric module DEBUG for QLogic 24xx+ series target mode HBAs
 	This will include code to enable the SCSI command jammer
 endif
diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
index 2c9e5ac24692..5d93ccc73153 100644
--- a/drivers/scsi/qla2xxx/qla_attr.c
+++ b/drivers/scsi/qla2xxx/qla_attr.c
@@ -26,7 +26,8 @@ qla2x00_sysfs_read_fw_dump(struct file *filp, struct kobject *kobj,
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
 
-	if (!(ha->fw_dump_reading || ha->mctp_dump_reading))
+	if (!(ha->fw_dump_reading || ha->mctp_dump_reading ||
+	      ha->mpi_fw_dump_reading))
 		return 0;
 
 	mutex_lock(&ha->optrom_mutex);
@@ -42,6 +43,10 @@ qla2x00_sysfs_read_fw_dump(struct file *filp, struct kobject *kobj,
 	} else if (ha->mctp_dumped && ha->mctp_dump_reading) {
 		rval = memory_read_from_buffer(buf, count, &off, ha->mctp_dump,
 		    MCTP_DUMP_SIZE);
+	} else if (ha->mpi_fw_dumped && ha->mpi_fw_dump_reading) {
+		rval = memory_read_from_buffer(buf, count, &off,
+					       ha->mpi_fw_dump,
+					       ha->mpi_fw_dump_len);
 	} else if (ha->fw_dump_reading) {
 		rval = memory_read_from_buffer(buf, count, &off, ha->fw_dump,
 					ha->fw_dump_len);
@@ -79,7 +84,7 @@ qla2x00_sysfs_write_fw_dump(struct file *filp, struct kobject *kobj,
 			qla82xx_md_prep(vha);
 		}
 		ha->fw_dump_reading = 0;
-		ha->fw_dumped = 0;
+		ha->fw_dumped = false;
 		break;
 	case 1:
 		if (ha->fw_dumped && !ha->fw_dump_reading) {
@@ -103,7 +108,6 @@ qla2x00_sysfs_write_fw_dump(struct file *filp, struct kobject *kobj,
 			qla82xx_set_reset_owner(vha);
 			qla8044_idc_unlock(ha);
 		} else {
-			ha->fw_dump_mpi = 1;
 			qla2x00_system_error(vha);
 		}
 		break;
@@ -137,6 +141,22 @@ qla2x00_sysfs_write_fw_dump(struct file *filp, struct kobject *kobj,
 			    vha->host_no);
 		}
 		break;
+	case 8:
+		if (!ha->mpi_fw_dump_reading)
+			break;
+		ql_log(ql_log_info, vha, 0x70e7,
+		       "MPI firmware dump cleared on (%ld).\n", vha->host_no);
+		ha->mpi_fw_dump_reading = 0;
+		ha->mpi_fw_dumped = 0;
+		break;
+	case 9:
+		if (ha->mpi_fw_dumped && !ha->mpi_fw_dump_reading) {
+			ha->mpi_fw_dump_reading = 1;
+			ql_log(ql_log_info, vha, 0x70e8,
+			       "Raw MPI firmware dump ready for read on (%ld).\n",
+			       vha->host_no);
+		}
+		break;
 	}
 	return count;
 }
@@ -207,10 +227,9 @@ qla2x00_sysfs_write_nvram(struct file *filp, struct kobject *kobj,
 
 	/* Checksum NVRAM. */
 	if (IS_FWI2_CAPABLE(ha)) {
-		uint32_t *iter;
+		__le32 *iter = (__force __le32 *)buf;
 		uint32_t chksum;
 
-		iter = (uint32_t *)buf;
 		chksum = 0;
 		for (cnt = 0; cnt < ((count >> 2) - 1); cnt++, iter++)
 			chksum += le32_to_cpu(*iter);
@@ -706,7 +725,8 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 		scsi_unblock_requests(vha->host);
 		break;
 	case 0x2025d:
-		if (!IS_QLA81XX(ha) && !IS_QLA83XX(ha))
+		if (!IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
+		    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 			return -EPERM;
 
 		ql_log(ql_log_info, vha, 0x706f,
@@ -724,6 +744,8 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 			qla83xx_idc_audit(vha, IDC_AUDIT_TIMESTAMP);
 			qla83xx_idc_unlock(vha, 0);
 			break;
+		} else if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+			qla27xx_reset_mpi(vha);
 		} else {
 			/* Make sure FC side is not in reset */
 			WARN_ON_ONCE(qla2x00_wait_for_hba_online(vha) !=
@@ -737,6 +759,7 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 			scsi_unblock_requests(vha->host);
 			break;
 		}
+		break;
 	case 0x2025e:
 		if (!IS_P3P_TYPE(ha) || vha != base_vha) {
 			ql_log(ql_log_info, vha, 0x7071,
@@ -1898,9 +1921,8 @@ static char *mode_to_str[] = {
 };
 
 #define NEED_EXCH_OFFLOAD(_exchg) ((_exchg) > FW_DEF_EXCHANGES_CNT)
-static int qla_set_ini_mode(scsi_qla_host_t *vha, int op)
+static void qla_set_ini_mode(scsi_qla_host_t *vha, int op)
 {
-	int rc = 0;
 	enum {
 		NO_ACTION,
 		MODE_CHANGE_ACCEPT,
@@ -2173,8 +2195,6 @@ static int qla_set_ini_mode(scsi_qla_host_t *vha, int op)
 		    vha->ql2xexchoffld, vha->u_ql2xexchoffld);
 		break;
 	}
-
-	return rc;
 }
 
 static ssize_t
diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
index 97b51c477972..88c0338a2ec7 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.c
+++ b/drivers/scsi/qla2xxx/qla_bsg.c
@@ -490,7 +490,7 @@ qla2x00_process_ct(struct bsg_job *bsg_job)
 			>> 24;
 	switch (loop_id) {
 	case 0xFC:
-		loop_id = cpu_to_le16(NPH_SNS);
+		loop_id = NPH_SNS;
 		break;
 	case 0xFA:
 		loop_id = vha->mgmt_svr_loop_id;
@@ -691,7 +691,7 @@ qla81xx_set_loopback_mode(scsi_qla_host_t *vha, uint16_t *config,
 		 * dump and reset the chip.
 		 */
 		if (ret) {
-			ha->isp_ops->fw_dump(vha, 0);
+			qla2xxx_dump_fw(vha);
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 		}
 		rval = -EINVAL;
@@ -896,7 +896,7 @@ qla2x00_process_loopback(struct bsg_job *bsg_job)
 					 * doesn't work take FCoE dump and then
 					 * reset the chip.
 					 */
-					ha->isp_ops->fw_dump(vha, 0);
+					qla2xxx_dump_fw(vha);
 					set_bit(ISP_ABORT_NEEDED,
 					    &vha->dpc_flags);
 				}
@@ -2042,7 +2042,7 @@ qlafx00_mgmt_cmd(struct bsg_job *bsg_job)
 
 	/* Initialize all required  fields of fcport */
 	fcport->vha = vha;
-	fcport->loop_id = piocb_rqst->dataword;
+	fcport->loop_id = le32_to_cpu(piocb_rqst->dataword);
 
 	sp->type = SRB_FXIOCB_BCMD;
 	sp->name = "bsg_fx_mgmt";
diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index bf1e98f11990..19005710f7f6 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -115,7 +115,7 @@ qla27xx_dump_mpi_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 {
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 	dma_addr_t dump_dma = ha->gid_list_dma;
-	uint32_t *chunk = (void *)ha->gid_list;
+	uint32_t *chunk = (uint32_t *)ha->gid_list;
 	uint32_t dwords = qla2x00_gid_list_size(ha) / 4;
 	uint32_t stat;
 	ulong i, j, timer = 6000000;
@@ -126,26 +126,26 @@ qla27xx_dump_mpi_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 		if (i + dwords > ram_dwords)
 			dwords = ram_dwords - i;
 
-		WRT_REG_WORD(&reg->mailbox0, MBC_LOAD_DUMP_MPI_RAM);
-		WRT_REG_WORD(&reg->mailbox1, LSW(addr));
-		WRT_REG_WORD(&reg->mailbox8, MSW(addr));
+		wrt_reg_word(&reg->mailbox0, MBC_LOAD_DUMP_MPI_RAM);
+		wrt_reg_word(&reg->mailbox1, LSW(addr));
+		wrt_reg_word(&reg->mailbox8, MSW(addr));
 
-		WRT_REG_WORD(&reg->mailbox2, MSW(LSD(dump_dma)));
-		WRT_REG_WORD(&reg->mailbox3, LSW(LSD(dump_dma)));
-		WRT_REG_WORD(&reg->mailbox6, MSW(MSD(dump_dma)));
-		WRT_REG_WORD(&reg->mailbox7, LSW(MSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox2, MSW(LSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox3, LSW(LSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox6, MSW(MSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox7, LSW(MSD(dump_dma)));
 
-		WRT_REG_WORD(&reg->mailbox4, MSW(dwords));
-		WRT_REG_WORD(&reg->mailbox5, LSW(dwords));
+		wrt_reg_word(&reg->mailbox4, MSW(dwords));
+		wrt_reg_word(&reg->mailbox5, LSW(dwords));
 
-		WRT_REG_WORD(&reg->mailbox9, 0);
-		WRT_REG_DWORD(&reg->hccr, HCCRX_SET_HOST_INT);
+		wrt_reg_word(&reg->mailbox9, 0);
+		wrt_reg_dword(&reg->hccr, HCCRX_SET_HOST_INT);
 
 		ha->flags.mbox_int = 0;
 		while (timer--) {
 			udelay(5);
 
-			stat = RD_REG_DWORD(&reg->host_status);
+			stat = rd_reg_dword(&reg->host_status);
 			/* Check for pending interrupts. */
 			if (!(stat & HSRX_RISC_INT))
 				continue;
@@ -155,15 +155,15 @@ qla27xx_dump_mpi_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 			    stat != 0x10 && stat != 0x11) {
 
 				/* Clear this intr; it wasn't a mailbox intr */
-				WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
-				RD_REG_DWORD(&reg->hccr);
+				wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
+				rd_reg_dword(&reg->hccr);
 				continue;
 			}
 
 			set_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
-			rval = RD_REG_WORD(&reg->mailbox0) & MBS_MASK;
-			WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
-			RD_REG_DWORD(&reg->hccr);
+			rval = rd_reg_word(&reg->mailbox0) & MBS_MASK;
+			wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
+			rd_reg_dword(&reg->hccr);
 			break;
 		}
 		ha->flags.mbox_int = 1;
@@ -189,13 +189,13 @@ qla27xx_dump_mpi_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 }
 
 int
-qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
-    uint32_t ram_dwords, void **nxt)
+qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, __be32 *ram,
+		 uint32_t ram_dwords, void **nxt)
 {
 	int rval = QLA_FUNCTION_FAILED;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 	dma_addr_t dump_dma = ha->gid_list_dma;
-	uint32_t *chunk = (void *)ha->gid_list;
+	uint32_t *chunk = (uint32_t *)ha->gid_list;
 	uint32_t dwords = qla2x00_gid_list_size(ha) / 4;
 	uint32_t stat;
 	ulong i, j, timer = 6000000;
@@ -206,23 +206,23 @@ qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 		if (i + dwords > ram_dwords)
 			dwords = ram_dwords - i;
 
-		WRT_REG_WORD(&reg->mailbox0, MBC_DUMP_RISC_RAM_EXTENDED);
-		WRT_REG_WORD(&reg->mailbox1, LSW(addr));
-		WRT_REG_WORD(&reg->mailbox8, MSW(addr));
+		wrt_reg_word(&reg->mailbox0, MBC_DUMP_RISC_RAM_EXTENDED);
+		wrt_reg_word(&reg->mailbox1, LSW(addr));
+		wrt_reg_word(&reg->mailbox8, MSW(addr));
 
-		WRT_REG_WORD(&reg->mailbox2, MSW(LSD(dump_dma)));
-		WRT_REG_WORD(&reg->mailbox3, LSW(LSD(dump_dma)));
-		WRT_REG_WORD(&reg->mailbox6, MSW(MSD(dump_dma)));
-		WRT_REG_WORD(&reg->mailbox7, LSW(MSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox2, MSW(LSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox3, LSW(LSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox6, MSW(MSD(dump_dma)));
+		wrt_reg_word(&reg->mailbox7, LSW(MSD(dump_dma)));
 
-		WRT_REG_WORD(&reg->mailbox4, MSW(dwords));
-		WRT_REG_WORD(&reg->mailbox5, LSW(dwords));
-		WRT_REG_DWORD(&reg->hccr, HCCRX_SET_HOST_INT);
+		wrt_reg_word(&reg->mailbox4, MSW(dwords));
+		wrt_reg_word(&reg->mailbox5, LSW(dwords));
+		wrt_reg_dword(&reg->hccr, HCCRX_SET_HOST_INT);
 
 		ha->flags.mbox_int = 0;
 		while (timer--) {
 			udelay(5);
-			stat = RD_REG_DWORD(&reg->host_status);
+			stat = rd_reg_dword(&reg->host_status);
 
 			/* Check for pending interrupts. */
 			if (!(stat & HSRX_RISC_INT))
@@ -231,15 +231,15 @@ qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 			stat &= 0xff;
 			if (stat != 0x1 && stat != 0x2 &&
 			    stat != 0x10 && stat != 0x11) {
-				WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
-				RD_REG_DWORD(&reg->hccr);
+				wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
+				rd_reg_dword(&reg->hccr);
 				continue;
 			}
 
 			set_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
-			rval = RD_REG_WORD(&reg->mailbox0) & MBS_MASK;
-			WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
-			RD_REG_DWORD(&reg->hccr);
+			rval = rd_reg_word(&reg->mailbox0) & MBS_MASK;
+			wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
+			rd_reg_dword(&reg->hccr);
 			break;
 		}
 		ha->flags.mbox_int = 1;
@@ -254,9 +254,9 @@ qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 			return rval;
 		}
 		for (j = 0; j < dwords; j++) {
-			ram[i + j] =
-			    (IS_QLA27XX(ha) || IS_QLA28XX(ha)) ?
-			    chunk[j] : swab32(chunk[j]);
+			ram[i + j] = (__force __be32)
+				((IS_QLA27XX(ha) || IS_QLA28XX(ha)) ?
+				 chunk[j] : swab32(chunk[j]));
 		}
 	}
 
@@ -265,8 +265,8 @@ qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 }
 
 static int
-qla24xx_dump_memory(struct qla_hw_data *ha, uint32_t *code_ram,
-    uint32_t cram_size, void **nxt)
+qla24xx_dump_memory(struct qla_hw_data *ha, __be32 *code_ram,
+		    uint32_t cram_size, void **nxt)
 {
 	int rval;
 
@@ -286,16 +286,16 @@ qla24xx_dump_memory(struct qla_hw_data *ha, uint32_t *code_ram,
 	return rval;
 }
 
-static uint32_t *
+static __be32 *
 qla24xx_read_window(struct device_reg_24xx __iomem *reg, uint32_t iobase,
-    uint32_t count, uint32_t *buf)
+		    uint32_t count, __be32 *buf)
 {
-	uint32_t __iomem *dmp_reg;
+	__le32 __iomem *dmp_reg;
 
-	WRT_REG_DWORD(&reg->iobase_addr, iobase);
+	wrt_reg_dword(&reg->iobase_addr, iobase);
 	dmp_reg = &reg->iobase_window;
 	for ( ; count--; dmp_reg++)
-		*buf++ = htonl(RD_REG_DWORD(dmp_reg));
+		*buf++ = htonl(rd_reg_dword(dmp_reg));
 
 	return buf;
 }
@@ -303,11 +303,11 @@ qla24xx_read_window(struct device_reg_24xx __iomem *reg, uint32_t iobase,
 void
 qla24xx_pause_risc(struct device_reg_24xx __iomem *reg, struct qla_hw_data *ha)
 {
-	WRT_REG_DWORD(&reg->hccr, HCCRX_SET_RISC_PAUSE);
+	wrt_reg_dword(&reg->hccr, HCCRX_SET_RISC_PAUSE);
 
 	/* 100 usec delay is sufficient enough for hardware to pause RISC */
 	udelay(100);
-	if (RD_REG_DWORD(&reg->host_status) & HSRX_RISC_PAUSED)
+	if (rd_reg_dword(&reg->host_status) & HSRX_RISC_PAUSED)
 		set_bit(RISC_PAUSE_CMPL, &ha->fw_dump_cap_flags);
 }
 
@@ -324,17 +324,17 @@ qla24xx_soft_reset(struct qla_hw_data *ha)
 	 * Driver can proceed with the reset sequence after waiting
 	 * for a timeout period.
 	 */
-	WRT_REG_DWORD(&reg->ctrl_status, CSRX_DMA_SHUTDOWN|MWB_4096_BYTES);
+	wrt_reg_dword(&reg->ctrl_status, CSRX_DMA_SHUTDOWN|MWB_4096_BYTES);
 	for (cnt = 0; cnt < 30000; cnt++) {
-		if ((RD_REG_DWORD(&reg->ctrl_status) & CSRX_DMA_ACTIVE) == 0)
+		if ((rd_reg_dword(&reg->ctrl_status) & CSRX_DMA_ACTIVE) == 0)
 			break;
 
 		udelay(10);
 	}
-	if (!(RD_REG_DWORD(&reg->ctrl_status) & CSRX_DMA_ACTIVE))
+	if (!(rd_reg_dword(&reg->ctrl_status) & CSRX_DMA_ACTIVE))
 		set_bit(DMA_SHUTDOWN_CMPL, &ha->fw_dump_cap_flags);
 
-	WRT_REG_DWORD(&reg->ctrl_status,
+	wrt_reg_dword(&reg->ctrl_status,
 	    CSRX_ISP_SOFT_RESET|CSRX_DMA_SHUTDOWN|MWB_4096_BYTES);
 	pci_read_config_word(ha->pdev, PCI_COMMAND, &wd);
 
@@ -342,19 +342,19 @@ qla24xx_soft_reset(struct qla_hw_data *ha)
 
 	/* Wait for soft-reset to complete. */
 	for (cnt = 0; cnt < 30000; cnt++) {
-		if ((RD_REG_DWORD(&reg->ctrl_status) &
+		if ((rd_reg_dword(&reg->ctrl_status) &
 		    CSRX_ISP_SOFT_RESET) == 0)
 			break;
 
 		udelay(10);
 	}
-	if (!(RD_REG_DWORD(&reg->ctrl_status) & CSRX_ISP_SOFT_RESET))
+	if (!(rd_reg_dword(&reg->ctrl_status) & CSRX_ISP_SOFT_RESET))
 		set_bit(ISP_RESET_CMPL, &ha->fw_dump_cap_flags);
 
-	WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_RESET);
-	RD_REG_DWORD(&reg->hccr);             /* PCI Posting. */
+	wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_RESET);
+	rd_reg_dword(&reg->hccr);             /* PCI Posting. */
 
-	for (cnt = 10000; RD_REG_WORD(&reg->mailbox0) != 0 &&
+	for (cnt = 10000; rd_reg_word(&reg->mailbox0) != 0 &&
 	    rval == QLA_SUCCESS; cnt--) {
 		if (cnt)
 			udelay(10);
@@ -368,7 +368,7 @@ qla24xx_soft_reset(struct qla_hw_data *ha)
 }
 
 static int
-qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
+qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, __be16 *ram,
     uint32_t ram_words, void **nxt)
 {
 	int rval;
@@ -376,7 +376,7 @@ qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
 	uint16_t mb0;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 	dma_addr_t dump_dma = ha->gid_list_dma;
-	uint16_t *dump = (uint16_t *)ha->gid_list;
+	__le16 *dump = (__force __le16 *)ha->gid_list;
 
 	rval = QLA_SUCCESS;
 	mb0 = 0;
@@ -399,11 +399,11 @@ qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
 		WRT_MAILBOX_REG(ha, reg, 7, LSW(MSD(dump_dma)));
 
 		WRT_MAILBOX_REG(ha, reg, 4, words);
-		WRT_REG_WORD(&reg->hccr, HCCR_SET_HOST_INT);
+		wrt_reg_word(&reg->hccr, HCCR_SET_HOST_INT);
 
 		for (timer = 6000000; timer; timer--) {
 			/* Check for pending interrupts. */
-			stat = RD_REG_DWORD(&reg->u.isp2300.host_status);
+			stat = rd_reg_dword(&reg->u.isp2300.host_status);
 			if (stat & HSR_RISC_INT) {
 				stat &= 0xff;
 
@@ -414,10 +414,10 @@ qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
 					mb0 = RD_MAILBOX_REG(ha, reg, 0);
 
 					/* Release mailbox registers. */
-					WRT_REG_WORD(&reg->semaphore, 0);
-					WRT_REG_WORD(&reg->hccr,
+					wrt_reg_word(&reg->semaphore, 0);
+					wrt_reg_word(&reg->hccr,
 					    HCCR_CLR_RISC_INT);
-					RD_REG_WORD(&reg->hccr);
+					rd_reg_word(&reg->hccr);
 					break;
 				} else if (stat == 0x10 || stat == 0x11) {
 					set_bit(MBX_INTERRUPT,
@@ -425,15 +425,15 @@ qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
 
 					mb0 = RD_MAILBOX_REG(ha, reg, 0);
 
-					WRT_REG_WORD(&reg->hccr,
+					wrt_reg_word(&reg->hccr,
 					    HCCR_CLR_RISC_INT);
-					RD_REG_WORD(&reg->hccr);
+					rd_reg_word(&reg->hccr);
 					break;
 				}
 
 				/* clear this intr; it wasn't a mailbox intr */
-				WRT_REG_WORD(&reg->hccr, HCCR_CLR_RISC_INT);
-				RD_REG_WORD(&reg->hccr);
+				wrt_reg_word(&reg->hccr, HCCR_CLR_RISC_INT);
+				rd_reg_word(&reg->hccr);
 			}
 			udelay(5);
 		}
@@ -441,7 +441,8 @@ qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
 		if (test_and_clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags)) {
 			rval = mb0 & MBS_MASK;
 			for (idx = 0; idx < words; idx++)
-				ram[cnt + idx] = swab16(dump[idx]);
+				ram[cnt + idx] =
+					cpu_to_be16(le16_to_cpu(dump[idx]));
 		} else {
 			rval = QLA_FUNCTION_FAILED;
 		}
@@ -453,12 +454,12 @@ qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
 
 static inline void
 qla2xxx_read_window(struct device_reg_2xxx __iomem *reg, uint32_t count,
-    uint16_t *buf)
+		    __be16 *buf)
 {
-	uint16_t __iomem *dmp_reg = &reg->u.isp2300.fb_cmd;
+	__le16 __iomem *dmp_reg = &reg->u.isp2300.fb_cmd;
 
 	for ( ; count--; dmp_reg++)
-		*buf++ = htons(RD_REG_WORD(dmp_reg));
+		*buf++ = htons(rd_reg_word(dmp_reg));
 }
 
 static inline void *
@@ -472,10 +473,10 @@ qla24xx_copy_eft(struct qla_hw_data *ha, void *ptr)
 }
 
 static inline void *
-qla25xx_copy_fce(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
+qla25xx_copy_fce(struct qla_hw_data *ha, void *ptr, __be32 **last_chain)
 {
 	uint32_t cnt;
-	uint32_t *iter_reg;
+	__be32 *iter_reg;
 	struct qla2xxx_fce_chain *fcec = ptr;
 
 	if (!ha->fce)
@@ -499,7 +500,7 @@ qla25xx_copy_fce(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
 }
 
 static inline void *
-qla25xx_copy_exlogin(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
+qla25xx_copy_exlogin(struct qla_hw_data *ha, void *ptr, __be32 **last_chain)
 {
 	struct qla2xxx_offld_chain *c = ptr;
 
@@ -517,11 +518,11 @@ qla25xx_copy_exlogin(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
 	ptr += sizeof(struct qla2xxx_offld_chain);
 	memcpy(ptr, ha->exlogin_buf, ha->exlogin_size);
 
-	return (char *)ptr + cpu_to_be32(c->size);
+	return (char *)ptr + be32_to_cpu(c->size);
 }
 
 static inline void *
-qla81xx_copy_exchoffld(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
+qla81xx_copy_exchoffld(struct qla_hw_data *ha, void *ptr, __be32 **last_chain)
 {
 	struct qla2xxx_offld_chain *c = ptr;
 
@@ -539,12 +540,12 @@ qla81xx_copy_exchoffld(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
 	ptr += sizeof(struct qla2xxx_offld_chain);
 	memcpy(ptr, ha->exchoffld_buf, ha->exchoffld_size);
 
-	return (char *)ptr + cpu_to_be32(c->size);
+	return (char *)ptr + be32_to_cpu(c->size);
 }
 
 static inline void *
 qla2xxx_copy_atioqueues(struct qla_hw_data *ha, void *ptr,
-	uint32_t **last_chain)
+			__be32 **last_chain)
 {
 	struct qla2xxx_mqueue_chain *q;
 	struct qla2xxx_mqueue_header *qh;
@@ -591,7 +592,7 @@ qla2xxx_copy_atioqueues(struct qla_hw_data *ha, void *ptr,
 }
 
 static inline void *
-qla25xx_copy_mqueues(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
+qla25xx_copy_mqueues(struct qla_hw_data *ha, void *ptr, __be32 **last_chain)
 {
 	struct qla2xxx_mqueue_chain *q;
 	struct qla2xxx_mqueue_header *qh;
@@ -662,7 +663,7 @@ qla25xx_copy_mqueues(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
 }
 
 static inline void *
-qla25xx_copy_mq(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
+qla25xx_copy_mq(struct qla_hw_data *ha, void *ptr, __be32 **last_chain)
 {
 	uint32_t cnt, que_idx;
 	uint8_t que_cnt;
@@ -685,13 +686,13 @@ qla25xx_copy_mq(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
 		reg = ISP_QUE_REG(ha, cnt);
 		que_idx = cnt * 4;
 		mq->qregs[que_idx] =
-		    htonl(RD_REG_DWORD(&reg->isp25mq.req_q_in));
+		    htonl(rd_reg_dword(&reg->isp25mq.req_q_in));
 		mq->qregs[que_idx+1] =
-		    htonl(RD_REG_DWORD(&reg->isp25mq.req_q_out));
+		    htonl(rd_reg_dword(&reg->isp25mq.req_q_out));
 		mq->qregs[que_idx+2] =
-		    htonl(RD_REG_DWORD(&reg->isp25mq.rsp_q_in));
+		    htonl(rd_reg_dword(&reg->isp25mq.rsp_q_in));
 		mq->qregs[que_idx+3] =
-		    htonl(RD_REG_DWORD(&reg->isp25mq.rsp_q_out));
+		    htonl(rd_reg_dword(&reg->isp25mq.rsp_q_out));
 	}
 
 	return ptr + sizeof(struct qla2xxx_mq_chain);
@@ -706,45 +707,47 @@ qla2xxx_dump_post_process(scsi_qla_host_t *vha, int rval)
 		ql_log(ql_log_warn, vha, 0xd000,
 		    "Failed to dump firmware (%x), dump status flags (0x%lx).\n",
 		    rval, ha->fw_dump_cap_flags);
-		ha->fw_dumped = 0;
+		ha->fw_dumped = false;
 	} else {
 		ql_log(ql_log_info, vha, 0xd001,
 		    "Firmware dump saved to temp buffer (%ld/%p), dump status flags (0x%lx).\n",
 		    vha->host_no, ha->fw_dump, ha->fw_dump_cap_flags);
-		ha->fw_dumped = 1;
+		ha->fw_dumped = true;
 		qla2x00_post_uevent_work(vha, QLA_UEVENT_CODE_FW_DUMP);
 	}
 }
 
+void qla2xxx_dump_fw(scsi_qla_host_t *vha)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&vha->hw->hardware_lock, flags);
+	vha->hw->isp_ops->fw_dump(vha);
+	spin_unlock_irqrestore(&vha->hw->hardware_lock, flags);
+}
+
 /**
  * qla2300_fw_dump() - Dumps binary data from the 2300 firmware.
  * @vha: HA context
- * @hardware_locked: Called with the hardware_lock
  */
 void
-qla2300_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla2300_fw_dump(scsi_qla_host_t *vha)
 {
 	int		rval;
 	uint32_t	cnt;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
-	uint16_t __iomem *dmp_reg;
-	unsigned long	flags;
+	__le16 __iomem *dmp_reg;
 	struct qla2300_fw_dump	*fw;
 	void		*nxt;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 
-	flags = 0;
-
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_lock_irqsave(&ha->hardware_lock, flags);
-#endif
+	lockdep_assert_held(&ha->hardware_lock);
 
 	if (!ha->fw_dump) {
 		ql_log(ql_log_warn, vha, 0xd002,
 		    "No buffer available for dump.\n");
-		goto qla2300_fw_dump_failed;
+		return;
 	}
 
 	if (ha->fw_dumped) {
@@ -752,19 +755,19 @@ qla2300_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		    "Firmware has been previously dumped (%p) "
 		    "-- ignoring request.\n",
 		    ha->fw_dump);
-		goto qla2300_fw_dump_failed;
+		return;
 	}
 	fw = &ha->fw_dump->isp.isp23;
 	qla2xxx_prep_dump(ha, ha->fw_dump);
 
 	rval = QLA_SUCCESS;
-	fw->hccr = htons(RD_REG_WORD(&reg->hccr));
+	fw->hccr = htons(rd_reg_word(&reg->hccr));
 
 	/* Pause RISC. */
-	WRT_REG_WORD(&reg->hccr, HCCR_PAUSE_RISC);
+	wrt_reg_word(&reg->hccr, HCCR_PAUSE_RISC);
 	if (IS_QLA2300(ha)) {
 		for (cnt = 30000;
-		    (RD_REG_WORD(&reg->hccr) & HCCR_RISC_PAUSE) == 0 &&
+		    (rd_reg_word(&reg->hccr) & HCCR_RISC_PAUSE) == 0 &&
 			rval == QLA_SUCCESS; cnt--) {
 			if (cnt)
 				udelay(100);
@@ -772,74 +775,74 @@ qla2300_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 				rval = QLA_FUNCTION_TIMEOUT;
 		}
 	} else {
-		RD_REG_WORD(&reg->hccr);		/* PCI Posting. */
+		rd_reg_word(&reg->hccr);		/* PCI Posting. */
 		udelay(10);
 	}
 
 	if (rval == QLA_SUCCESS) {
 		dmp_reg = &reg->flash_address;
-		for (cnt = 0; cnt < sizeof(fw->pbiu_reg) / 2; cnt++, dmp_reg++)
-			fw->pbiu_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+		for (cnt = 0; cnt < ARRAY_SIZE(fw->pbiu_reg); cnt++, dmp_reg++)
+			fw->pbiu_reg[cnt] = htons(rd_reg_word(dmp_reg));
 
 		dmp_reg = &reg->u.isp2300.req_q_in;
-		for (cnt = 0; cnt < sizeof(fw->risc_host_reg) / 2;
+		for (cnt = 0; cnt < ARRAY_SIZE(fw->risc_host_reg);
 		    cnt++, dmp_reg++)
-			fw->risc_host_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+			fw->risc_host_reg[cnt] = htons(rd_reg_word(dmp_reg));
 
 		dmp_reg = &reg->u.isp2300.mailbox0;
-		for (cnt = 0; cnt < sizeof(fw->mailbox_reg) / 2;
+		for (cnt = 0; cnt < ARRAY_SIZE(fw->mailbox_reg);
 		    cnt++, dmp_reg++)
-			fw->mailbox_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+			fw->mailbox_reg[cnt] = htons(rd_reg_word(dmp_reg));
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x40);
+		wrt_reg_word(&reg->ctrl_status, 0x40);
 		qla2xxx_read_window(reg, 32, fw->resp_dma_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x50);
+		wrt_reg_word(&reg->ctrl_status, 0x50);
 		qla2xxx_read_window(reg, 48, fw->dma_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x00);
+		wrt_reg_word(&reg->ctrl_status, 0x00);
 		dmp_reg = &reg->risc_hw;
-		for (cnt = 0; cnt < sizeof(fw->risc_hdw_reg) / 2;
+		for (cnt = 0; cnt < ARRAY_SIZE(fw->risc_hdw_reg);
 		    cnt++, dmp_reg++)
-			fw->risc_hdw_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+			fw->risc_hdw_reg[cnt] = htons(rd_reg_word(dmp_reg));
 
-		WRT_REG_WORD(&reg->pcr, 0x2000);
+		wrt_reg_word(&reg->pcr, 0x2000);
 		qla2xxx_read_window(reg, 16, fw->risc_gp0_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2200);
+		wrt_reg_word(&reg->pcr, 0x2200);
 		qla2xxx_read_window(reg, 16, fw->risc_gp1_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2400);
+		wrt_reg_word(&reg->pcr, 0x2400);
 		qla2xxx_read_window(reg, 16, fw->risc_gp2_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2600);
+		wrt_reg_word(&reg->pcr, 0x2600);
 		qla2xxx_read_window(reg, 16, fw->risc_gp3_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2800);
+		wrt_reg_word(&reg->pcr, 0x2800);
 		qla2xxx_read_window(reg, 16, fw->risc_gp4_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2A00);
+		wrt_reg_word(&reg->pcr, 0x2A00);
 		qla2xxx_read_window(reg, 16, fw->risc_gp5_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2C00);
+		wrt_reg_word(&reg->pcr, 0x2C00);
 		qla2xxx_read_window(reg, 16, fw->risc_gp6_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2E00);
+		wrt_reg_word(&reg->pcr, 0x2E00);
 		qla2xxx_read_window(reg, 16, fw->risc_gp7_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x10);
+		wrt_reg_word(&reg->ctrl_status, 0x10);
 		qla2xxx_read_window(reg, 64, fw->frame_buf_hdw_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x20);
+		wrt_reg_word(&reg->ctrl_status, 0x20);
 		qla2xxx_read_window(reg, 64, fw->fpm_b0_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x30);
+		wrt_reg_word(&reg->ctrl_status, 0x30);
 		qla2xxx_read_window(reg, 64, fw->fpm_b1_reg);
 
 		/* Reset RISC. */
-		WRT_REG_WORD(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
+		wrt_reg_word(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
 		for (cnt = 0; cnt < 30000; cnt++) {
-			if ((RD_REG_WORD(&reg->ctrl_status) &
+			if ((rd_reg_word(&reg->ctrl_status) &
 			    CSR_ISP_SOFT_RESET) == 0)
 				break;
 
@@ -860,12 +863,12 @@ qla2300_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	/* Get RISC SRAM. */
 	if (rval == QLA_SUCCESS)
 		rval = qla2xxx_dump_ram(ha, 0x800, fw->risc_ram,
-		    sizeof(fw->risc_ram) / 2, &nxt);
+					ARRAY_SIZE(fw->risc_ram), &nxt);
 
 	/* Get stack SRAM. */
 	if (rval == QLA_SUCCESS)
 		rval = qla2xxx_dump_ram(ha, 0x10000, fw->stack_ram,
-		    sizeof(fw->stack_ram) / 2, &nxt);
+					ARRAY_SIZE(fw->stack_ram), &nxt);
 
 	/* Get data SRAM. */
 	if (rval == QLA_SUCCESS)
@@ -876,48 +879,31 @@ qla2300_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		qla2xxx_copy_queues(ha, nxt);
 
 	qla2xxx_dump_post_process(base_vha, rval);
-
-qla2300_fw_dump_failed:
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
-#else
-	;
-#endif
 }
 
 /**
  * qla2100_fw_dump() - Dumps binary data from the 2100/2200 firmware.
  * @vha: HA context
- * @hardware_locked: Called with the hardware_lock
  */
 void
-qla2100_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla2100_fw_dump(scsi_qla_host_t *vha)
 {
 	int		rval;
 	uint32_t	cnt, timer;
-	uint16_t	risc_address;
-	uint16_t	mb0, mb2;
+	uint16_t	risc_address = 0;
+	uint16_t	mb0 = 0, mb2 = 0;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
-	uint16_t __iomem *dmp_reg;
-	unsigned long	flags;
+	__le16 __iomem *dmp_reg;
 	struct qla2100_fw_dump	*fw;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 
-	risc_address = 0;
-	mb0 = mb2 = 0;
-	flags = 0;
-
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_lock_irqsave(&ha->hardware_lock, flags);
-#endif
+	lockdep_assert_held(&ha->hardware_lock);
 
 	if (!ha->fw_dump) {
 		ql_log(ql_log_warn, vha, 0xd004,
 		    "No buffer available for dump.\n");
-		goto qla2100_fw_dump_failed;
+		return;
 	}
 
 	if (ha->fw_dumped) {
@@ -925,17 +911,17 @@ qla2100_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		    "Firmware has been previously dumped (%p) "
 		    "-- ignoring request.\n",
 		    ha->fw_dump);
-		goto qla2100_fw_dump_failed;
+		return;
 	}
 	fw = &ha->fw_dump->isp.isp21;
 	qla2xxx_prep_dump(ha, ha->fw_dump);
 
 	rval = QLA_SUCCESS;
-	fw->hccr = htons(RD_REG_WORD(&reg->hccr));
+	fw->hccr = htons(rd_reg_word(&reg->hccr));
 
 	/* Pause RISC. */
-	WRT_REG_WORD(&reg->hccr, HCCR_PAUSE_RISC);
-	for (cnt = 30000; (RD_REG_WORD(&reg->hccr) & HCCR_RISC_PAUSE) == 0 &&
+	wrt_reg_word(&reg->hccr, HCCR_PAUSE_RISC);
+	for (cnt = 30000; (rd_reg_word(&reg->hccr) & HCCR_RISC_PAUSE) == 0 &&
 	    rval == QLA_SUCCESS; cnt--) {
 		if (cnt)
 			udelay(100);
@@ -944,61 +930,61 @@ qla2100_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	}
 	if (rval == QLA_SUCCESS) {
 		dmp_reg = &reg->flash_address;
-		for (cnt = 0; cnt < sizeof(fw->pbiu_reg) / 2; cnt++, dmp_reg++)
-			fw->pbiu_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+		for (cnt = 0; cnt < ARRAY_SIZE(fw->pbiu_reg); cnt++, dmp_reg++)
+			fw->pbiu_reg[cnt] = htons(rd_reg_word(dmp_reg));
 
 		dmp_reg = &reg->u.isp2100.mailbox0;
 		for (cnt = 0; cnt < ha->mbx_count; cnt++, dmp_reg++) {
 			if (cnt == 8)
 				dmp_reg = &reg->u_end.isp2200.mailbox8;
 
-			fw->mailbox_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+			fw->mailbox_reg[cnt] = htons(rd_reg_word(dmp_reg));
 		}
 
 		dmp_reg = &reg->u.isp2100.unused_2[0];
-		for (cnt = 0; cnt < sizeof(fw->dma_reg) / 2; cnt++, dmp_reg++)
-			fw->dma_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+		for (cnt = 0; cnt < ARRAY_SIZE(fw->dma_reg); cnt++, dmp_reg++)
+			fw->dma_reg[cnt] = htons(rd_reg_word(dmp_reg));
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x00);
+		wrt_reg_word(&reg->ctrl_status, 0x00);
 		dmp_reg = &reg->risc_hw;
-		for (cnt = 0; cnt < sizeof(fw->risc_hdw_reg) / 2; cnt++, dmp_reg++)
-			fw->risc_hdw_reg[cnt] = htons(RD_REG_WORD(dmp_reg));
+		for (cnt = 0; cnt < ARRAY_SIZE(fw->risc_hdw_reg); cnt++, dmp_reg++)
+			fw->risc_hdw_reg[cnt] = htons(rd_reg_word(dmp_reg));
 
-		WRT_REG_WORD(&reg->pcr, 0x2000);
+		wrt_reg_word(&reg->pcr, 0x2000);
 		qla2xxx_read_window(reg, 16, fw->risc_gp0_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2100);
+		wrt_reg_word(&reg->pcr, 0x2100);
 		qla2xxx_read_window(reg, 16, fw->risc_gp1_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2200);
+		wrt_reg_word(&reg->pcr, 0x2200);
 		qla2xxx_read_window(reg, 16, fw->risc_gp2_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2300);
+		wrt_reg_word(&reg->pcr, 0x2300);
 		qla2xxx_read_window(reg, 16, fw->risc_gp3_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2400);
+		wrt_reg_word(&reg->pcr, 0x2400);
 		qla2xxx_read_window(reg, 16, fw->risc_gp4_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2500);
+		wrt_reg_word(&reg->pcr, 0x2500);
 		qla2xxx_read_window(reg, 16, fw->risc_gp5_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2600);
+		wrt_reg_word(&reg->pcr, 0x2600);
 		qla2xxx_read_window(reg, 16, fw->risc_gp6_reg);
 
-		WRT_REG_WORD(&reg->pcr, 0x2700);
+		wrt_reg_word(&reg->pcr, 0x2700);
 		qla2xxx_read_window(reg, 16, fw->risc_gp7_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x10);
+		wrt_reg_word(&reg->ctrl_status, 0x10);
 		qla2xxx_read_window(reg, 16, fw->frame_buf_hdw_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x20);
+		wrt_reg_word(&reg->ctrl_status, 0x20);
 		qla2xxx_read_window(reg, 64, fw->fpm_b0_reg);
 
-		WRT_REG_WORD(&reg->ctrl_status, 0x30);
+		wrt_reg_word(&reg->ctrl_status, 0x30);
 		qla2xxx_read_window(reg, 64, fw->fpm_b1_reg);
 
 		/* Reset the ISP. */
-		WRT_REG_WORD(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
+		wrt_reg_word(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
 	}
 
 	for (cnt = 30000; RD_MAILBOX_REG(ha, reg, 0) != 0 &&
@@ -1011,11 +997,11 @@ qla2100_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 
 	/* Pause RISC. */
 	if (rval == QLA_SUCCESS && (IS_QLA2200(ha) || (IS_QLA2100(ha) &&
-	    (RD_REG_WORD(&reg->mctr) & (BIT_1 | BIT_0)) != 0))) {
+	    (rd_reg_word(&reg->mctr) & (BIT_1 | BIT_0)) != 0))) {
 
-		WRT_REG_WORD(&reg->hccr, HCCR_PAUSE_RISC);
+		wrt_reg_word(&reg->hccr, HCCR_PAUSE_RISC);
 		for (cnt = 30000;
-		    (RD_REG_WORD(&reg->hccr) & HCCR_RISC_PAUSE) == 0 &&
+		    (rd_reg_word(&reg->hccr) & HCCR_RISC_PAUSE) == 0 &&
 		    rval == QLA_SUCCESS; cnt--) {
 			if (cnt)
 				udelay(100);
@@ -1025,13 +1011,13 @@ qla2100_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		if (rval == QLA_SUCCESS) {
 			/* Set memory configuration and timing. */
 			if (IS_QLA2100(ha))
-				WRT_REG_WORD(&reg->mctr, 0xf1);
+				wrt_reg_word(&reg->mctr, 0xf1);
 			else
-				WRT_REG_WORD(&reg->mctr, 0xf2);
-			RD_REG_WORD(&reg->mctr);	/* PCI Posting. */
+				wrt_reg_word(&reg->mctr, 0xf2);
+			rd_reg_word(&reg->mctr);	/* PCI Posting. */
 
 			/* Release RISC. */
-			WRT_REG_WORD(&reg->hccr, HCCR_RELEASE_RISC);
+			wrt_reg_word(&reg->hccr, HCCR_RELEASE_RISC);
 		}
 	}
 
@@ -1041,29 +1027,29 @@ qla2100_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
  		WRT_MAILBOX_REG(ha, reg, 0, MBC_READ_RAM_WORD);
 		clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
 	}
-	for (cnt = 0; cnt < sizeof(fw->risc_ram) / 2 && rval == QLA_SUCCESS;
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->risc_ram) && rval == QLA_SUCCESS;
 	    cnt++, risc_address++) {
  		WRT_MAILBOX_REG(ha, reg, 1, risc_address);
-		WRT_REG_WORD(&reg->hccr, HCCR_SET_HOST_INT);
+		wrt_reg_word(&reg->hccr, HCCR_SET_HOST_INT);
 
 		for (timer = 6000000; timer != 0; timer--) {
 			/* Check for pending interrupts. */
-			if (RD_REG_WORD(&reg->istatus) & ISR_RISC_INT) {
-				if (RD_REG_WORD(&reg->semaphore) & BIT_0) {
+			if (rd_reg_word(&reg->istatus) & ISR_RISC_INT) {
+				if (rd_reg_word(&reg->semaphore) & BIT_0) {
 					set_bit(MBX_INTERRUPT,
 					    &ha->mbx_cmd_flags);
 
 					mb0 = RD_MAILBOX_REG(ha, reg, 0);
 					mb2 = RD_MAILBOX_REG(ha, reg, 2);
 
-					WRT_REG_WORD(&reg->semaphore, 0);
-					WRT_REG_WORD(&reg->hccr,
+					wrt_reg_word(&reg->semaphore, 0);
+					wrt_reg_word(&reg->hccr,
 					    HCCR_CLR_RISC_INT);
-					RD_REG_WORD(&reg->hccr);
+					rd_reg_word(&reg->hccr);
 					break;
 				}
-				WRT_REG_WORD(&reg->hccr, HCCR_CLR_RISC_INT);
-				RD_REG_WORD(&reg->hccr);
+				wrt_reg_word(&reg->hccr, HCCR_CLR_RISC_INT);
+				rd_reg_word(&reg->hccr);
 			}
 			udelay(5);
 		}
@@ -1080,48 +1066,35 @@ qla2100_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		qla2xxx_copy_queues(ha, &fw->risc_ram[cnt]);
 
 	qla2xxx_dump_post_process(base_vha, rval);
-
-qla2100_fw_dump_failed:
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
-#else
-	;
-#endif
 }
 
 void
-qla24xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla24xx_fw_dump(scsi_qla_host_t *vha)
 {
 	int		rval;
 	uint32_t	cnt;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
-	uint32_t __iomem *dmp_reg;
-	uint32_t	*iter_reg;
-	uint16_t __iomem *mbx_reg;
-	unsigned long	flags;
+	__le32 __iomem *dmp_reg;
+	__be32		*iter_reg;
+	__le16 __iomem *mbx_reg;
 	struct qla24xx_fw_dump *fw;
 	void		*nxt;
 	void		*nxt_chain;
-	uint32_t	*last_chain = NULL;
+	__be32		*last_chain = NULL;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 
+	lockdep_assert_held(&ha->hardware_lock);
+
 	if (IS_P3P_TYPE(ha))
 		return;
 
-	flags = 0;
 	ha->fw_dump_cap_flags = 0;
 
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_lock_irqsave(&ha->hardware_lock, flags);
-#endif
-
 	if (!ha->fw_dump) {
 		ql_log(ql_log_warn, vha, 0xd006,
 		    "No buffer available for dump.\n");
-		goto qla24xx_fw_dump_failed;
+		return;
 	}
 
 	if (ha->fw_dumped) {
@@ -1129,13 +1102,13 @@ qla24xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		    "Firmware has been previously dumped (%p) "
 		    "-- ignoring request.\n",
 		    ha->fw_dump);
-		goto qla24xx_fw_dump_failed;
+		return;
 	}
 	QLA_FW_STOPPED(ha);
 	fw = &ha->fw_dump->isp.isp24;
 	qla2xxx_prep_dump(ha, ha->fw_dump);
 
-	fw->host_status = htonl(RD_REG_DWORD(&reg->host_status));
+	fw->host_status = htonl(rd_reg_dword(&reg->host_status));
 
 	/*
 	 * Pause RISC. No need to track timeout, as resetting the chip
@@ -1145,41 +1118,41 @@ qla24xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 
 	/* Host interface registers. */
 	dmp_reg = &reg->flash_addr;
-	for (cnt = 0; cnt < sizeof(fw->host_reg) / 4; cnt++, dmp_reg++)
-		fw->host_reg[cnt] = htonl(RD_REG_DWORD(dmp_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->host_reg); cnt++, dmp_reg++)
+		fw->host_reg[cnt] = htonl(rd_reg_dword(dmp_reg));
 
 	/* Disable interrupts. */
-	WRT_REG_DWORD(&reg->ictrl, 0);
-	RD_REG_DWORD(&reg->ictrl);
+	wrt_reg_dword(&reg->ictrl, 0);
+	rd_reg_dword(&reg->ictrl);
 
 	/* Shadow registers. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0F70);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0000000);
-	fw->shadow_reg[0] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_addr, 0x0F70);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_select, 0xB0000000);
+	fw->shadow_reg[0] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0100000);
-	fw->shadow_reg[1] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0100000);
+	fw->shadow_reg[1] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0200000);
-	fw->shadow_reg[2] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0200000);
+	fw->shadow_reg[2] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0300000);
-	fw->shadow_reg[3] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0300000);
+	fw->shadow_reg[3] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0400000);
-	fw->shadow_reg[4] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0400000);
+	fw->shadow_reg[4] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0500000);
-	fw->shadow_reg[5] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0500000);
+	fw->shadow_reg[5] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0600000);
-	fw->shadow_reg[6] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0600000);
+	fw->shadow_reg[6] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
 	/* Mailbox registers. */
 	mbx_reg = &reg->mailbox0;
-	for (cnt = 0; cnt < sizeof(fw->mailbox_reg) / 2; cnt++, mbx_reg++)
-		fw->mailbox_reg[cnt] = htons(RD_REG_WORD(mbx_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->mailbox_reg); cnt++, mbx_reg++)
+		fw->mailbox_reg[cnt] = htons(rd_reg_word(mbx_reg));
 
 	/* Transfer sequence registers. */
 	iter_reg = fw->xseq_gp_reg;
@@ -1218,19 +1191,19 @@ qla24xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	iter_reg = qla24xx_read_window(reg, 0x7200, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->resp0_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7300, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->req1_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7400, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	/* Transmit DMA registers. */
 	iter_reg = fw->xmt0_dma_reg;
@@ -1339,44 +1312,31 @@ qla24xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 
 qla24xx_fw_dump_failed_0:
 	qla2xxx_dump_post_process(base_vha, rval);
-
-qla24xx_fw_dump_failed:
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
-#else
-	;
-#endif
 }
 
 void
-qla25xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla25xx_fw_dump(scsi_qla_host_t *vha)
 {
 	int		rval;
 	uint32_t	cnt;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
-	uint32_t __iomem *dmp_reg;
-	uint32_t	*iter_reg;
-	uint16_t __iomem *mbx_reg;
-	unsigned long	flags;
+	__le32 __iomem *dmp_reg;
+	__be32		*iter_reg;
+	__le16 __iomem *mbx_reg;
 	struct qla25xx_fw_dump *fw;
 	void		*nxt, *nxt_chain;
-	uint32_t	*last_chain = NULL;
+	__be32		*last_chain = NULL;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 
-	flags = 0;
-	ha->fw_dump_cap_flags = 0;
+	lockdep_assert_held(&ha->hardware_lock);
 
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_lock_irqsave(&ha->hardware_lock, flags);
-#endif
+	ha->fw_dump_cap_flags = 0;
 
 	if (!ha->fw_dump) {
 		ql_log(ql_log_warn, vha, 0xd008,
 		    "No buffer available for dump.\n");
-		goto qla25xx_fw_dump_failed;
+		return;
 	}
 
 	if (ha->fw_dumped) {
@@ -1384,14 +1344,14 @@ qla25xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		    "Firmware has been previously dumped (%p) "
 		    "-- ignoring request.\n",
 		    ha->fw_dump);
-		goto qla25xx_fw_dump_failed;
+		return;
 	}
 	QLA_FW_STOPPED(ha);
 	fw = &ha->fw_dump->isp.isp25;
 	qla2xxx_prep_dump(ha, ha->fw_dump);
 	ha->fw_dump->version = htonl(2);
 
-	fw->host_status = htonl(RD_REG_DWORD(&reg->host_status));
+	fw->host_status = htonl(rd_reg_dword(&reg->host_status));
 
 	/*
 	 * Pause RISC. No need to track timeout, as resetting the chip
@@ -1405,73 +1365,73 @@ qla25xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	qla24xx_read_window(reg, 0x7010, 16, iter_reg);
 
 	/* PCIe registers. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x7C00);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_window, 0x01);
+	wrt_reg_dword(&reg->iobase_addr, 0x7C00);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_window, 0x01);
 	dmp_reg = &reg->iobase_c4;
-	fw->pcie_regs[0] = htonl(RD_REG_DWORD(dmp_reg));
+	fw->pcie_regs[0] = htonl(rd_reg_dword(dmp_reg));
 	dmp_reg++;
-	fw->pcie_regs[1] = htonl(RD_REG_DWORD(dmp_reg));
+	fw->pcie_regs[1] = htonl(rd_reg_dword(dmp_reg));
 	dmp_reg++;
-	fw->pcie_regs[2] = htonl(RD_REG_DWORD(dmp_reg));
-	fw->pcie_regs[3] = htonl(RD_REG_DWORD(&reg->iobase_window));
+	fw->pcie_regs[2] = htonl(rd_reg_dword(dmp_reg));
+	fw->pcie_regs[3] = htonl(rd_reg_dword(&reg->iobase_window));
 
-	WRT_REG_DWORD(&reg->iobase_window, 0x00);
-	RD_REG_DWORD(&reg->iobase_window);
+	wrt_reg_dword(&reg->iobase_window, 0x00);
+	rd_reg_dword(&reg->iobase_window);
 
 	/* Host interface registers. */
 	dmp_reg = &reg->flash_addr;
-	for (cnt = 0; cnt < sizeof(fw->host_reg) / 4; cnt++, dmp_reg++)
-		fw->host_reg[cnt] = htonl(RD_REG_DWORD(dmp_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->host_reg); cnt++, dmp_reg++)
+		fw->host_reg[cnt] = htonl(rd_reg_dword(dmp_reg));
 
 	/* Disable interrupts. */
-	WRT_REG_DWORD(&reg->ictrl, 0);
-	RD_REG_DWORD(&reg->ictrl);
+	wrt_reg_dword(&reg->ictrl, 0);
+	rd_reg_dword(&reg->ictrl);
 
 	/* Shadow registers. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0F70);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0000000);
-	fw->shadow_reg[0] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_addr, 0x0F70);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_select, 0xB0000000);
+	fw->shadow_reg[0] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0100000);
-	fw->shadow_reg[1] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0100000);
+	fw->shadow_reg[1] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0200000);
-	fw->shadow_reg[2] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0200000);
+	fw->shadow_reg[2] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0300000);
-	fw->shadow_reg[3] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0300000);
+	fw->shadow_reg[3] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0400000);
-	fw->shadow_reg[4] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0400000);
+	fw->shadow_reg[4] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0500000);
-	fw->shadow_reg[5] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0500000);
+	fw->shadow_reg[5] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0600000);
-	fw->shadow_reg[6] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0600000);
+	fw->shadow_reg[6] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0700000);
-	fw->shadow_reg[7] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0700000);
+	fw->shadow_reg[7] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0800000);
-	fw->shadow_reg[8] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0800000);
+	fw->shadow_reg[8] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0900000);
-	fw->shadow_reg[9] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0900000);
+	fw->shadow_reg[9] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0A00000);
-	fw->shadow_reg[10] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0A00000);
+	fw->shadow_reg[10] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
 	/* RISC I/O register. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0010);
-	fw->risc_io_reg = htonl(RD_REG_DWORD(&reg->iobase_window));
+	wrt_reg_dword(&reg->iobase_addr, 0x0010);
+	fw->risc_io_reg = htonl(rd_reg_dword(&reg->iobase_window));
 
 	/* Mailbox registers. */
 	mbx_reg = &reg->mailbox0;
-	for (cnt = 0; cnt < sizeof(fw->mailbox_reg) / 2; cnt++, mbx_reg++)
-		fw->mailbox_reg[cnt] = htons(RD_REG_WORD(mbx_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->mailbox_reg); cnt++, mbx_reg++)
+		fw->mailbox_reg[cnt] = htons(rd_reg_word(mbx_reg));
 
 	/* Transfer sequence registers. */
 	iter_reg = fw->xseq_gp_reg;
@@ -1535,19 +1495,19 @@ qla25xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	iter_reg = qla24xx_read_window(reg, 0x7200, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->resp0_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7300, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->req1_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7400, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	/* Transmit DMA registers. */
 	iter_reg = fw->xmt0_dma_reg;
@@ -1665,44 +1625,31 @@ qla25xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 
 qla25xx_fw_dump_failed_0:
 	qla2xxx_dump_post_process(base_vha, rval);
-
-qla25xx_fw_dump_failed:
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
-#else
-	;
-#endif
 }
 
 void
-qla81xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla81xx_fw_dump(scsi_qla_host_t *vha)
 {
 	int		rval;
 	uint32_t	cnt;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
-	uint32_t __iomem *dmp_reg;
-	uint32_t	*iter_reg;
-	uint16_t __iomem *mbx_reg;
-	unsigned long	flags;
+	__le32 __iomem *dmp_reg;
+	__be32		*iter_reg;
+	__le16 __iomem *mbx_reg;
 	struct qla81xx_fw_dump *fw;
 	void		*nxt, *nxt_chain;
-	uint32_t	*last_chain = NULL;
+	__be32		*last_chain = NULL;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 
-	flags = 0;
-	ha->fw_dump_cap_flags = 0;
+	lockdep_assert_held(&ha->hardware_lock);
 
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_lock_irqsave(&ha->hardware_lock, flags);
-#endif
+	ha->fw_dump_cap_flags = 0;
 
 	if (!ha->fw_dump) {
 		ql_log(ql_log_warn, vha, 0xd00a,
 		    "No buffer available for dump.\n");
-		goto qla81xx_fw_dump_failed;
+		return;
 	}
 
 	if (ha->fw_dumped) {
@@ -1710,12 +1657,12 @@ qla81xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 		    "Firmware has been previously dumped (%p) "
 		    "-- ignoring request.\n",
 		    ha->fw_dump);
-		goto qla81xx_fw_dump_failed;
+		return;
 	}
 	fw = &ha->fw_dump->isp.isp81;
 	qla2xxx_prep_dump(ha, ha->fw_dump);
 
-	fw->host_status = htonl(RD_REG_DWORD(&reg->host_status));
+	fw->host_status = htonl(rd_reg_dword(&reg->host_status));
 
 	/*
 	 * Pause RISC. No need to track timeout, as resetting the chip
@@ -1729,73 +1676,73 @@ qla81xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	qla24xx_read_window(reg, 0x7010, 16, iter_reg);
 
 	/* PCIe registers. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x7C00);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_window, 0x01);
+	wrt_reg_dword(&reg->iobase_addr, 0x7C00);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_window, 0x01);
 	dmp_reg = &reg->iobase_c4;
-	fw->pcie_regs[0] = htonl(RD_REG_DWORD(dmp_reg));
+	fw->pcie_regs[0] = htonl(rd_reg_dword(dmp_reg));
 	dmp_reg++;
-	fw->pcie_regs[1] = htonl(RD_REG_DWORD(dmp_reg));
+	fw->pcie_regs[1] = htonl(rd_reg_dword(dmp_reg));
 	dmp_reg++;
-	fw->pcie_regs[2] = htonl(RD_REG_DWORD(dmp_reg));
-	fw->pcie_regs[3] = htonl(RD_REG_DWORD(&reg->iobase_window));
+	fw->pcie_regs[2] = htonl(rd_reg_dword(dmp_reg));
+	fw->pcie_regs[3] = htonl(rd_reg_dword(&reg->iobase_window));
 
-	WRT_REG_DWORD(&reg->iobase_window, 0x00);
-	RD_REG_DWORD(&reg->iobase_window);
+	wrt_reg_dword(&reg->iobase_window, 0x00);
+	rd_reg_dword(&reg->iobase_window);
 
 	/* Host interface registers. */
 	dmp_reg = &reg->flash_addr;
-	for (cnt = 0; cnt < sizeof(fw->host_reg) / 4; cnt++, dmp_reg++)
-		fw->host_reg[cnt] = htonl(RD_REG_DWORD(dmp_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->host_reg); cnt++, dmp_reg++)
+		fw->host_reg[cnt] = htonl(rd_reg_dword(dmp_reg));
 
 	/* Disable interrupts. */
-	WRT_REG_DWORD(&reg->ictrl, 0);
-	RD_REG_DWORD(&reg->ictrl);
+	wrt_reg_dword(&reg->ictrl, 0);
+	rd_reg_dword(&reg->ictrl);
 
 	/* Shadow registers. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0F70);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0000000);
-	fw->shadow_reg[0] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_addr, 0x0F70);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_select, 0xB0000000);
+	fw->shadow_reg[0] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0100000);
-	fw->shadow_reg[1] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0100000);
+	fw->shadow_reg[1] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0200000);
-	fw->shadow_reg[2] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0200000);
+	fw->shadow_reg[2] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0300000);
-	fw->shadow_reg[3] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0300000);
+	fw->shadow_reg[3] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0400000);
-	fw->shadow_reg[4] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0400000);
+	fw->shadow_reg[4] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0500000);
-	fw->shadow_reg[5] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0500000);
+	fw->shadow_reg[5] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0600000);
-	fw->shadow_reg[6] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0600000);
+	fw->shadow_reg[6] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0700000);
-	fw->shadow_reg[7] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0700000);
+	fw->shadow_reg[7] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0800000);
-	fw->shadow_reg[8] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0800000);
+	fw->shadow_reg[8] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0900000);
-	fw->shadow_reg[9] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0900000);
+	fw->shadow_reg[9] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0A00000);
-	fw->shadow_reg[10] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0A00000);
+	fw->shadow_reg[10] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
 	/* RISC I/O register. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0010);
-	fw->risc_io_reg = htonl(RD_REG_DWORD(&reg->iobase_window));
+	wrt_reg_dword(&reg->iobase_addr, 0x0010);
+	fw->risc_io_reg = htonl(rd_reg_dword(&reg->iobase_window));
 
 	/* Mailbox registers. */
 	mbx_reg = &reg->mailbox0;
-	for (cnt = 0; cnt < sizeof(fw->mailbox_reg) / 2; cnt++, mbx_reg++)
-		fw->mailbox_reg[cnt] = htons(RD_REG_WORD(mbx_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->mailbox_reg); cnt++, mbx_reg++)
+		fw->mailbox_reg[cnt] = htons(rd_reg_word(mbx_reg));
 
 	/* Transfer sequence registers. */
 	iter_reg = fw->xseq_gp_reg;
@@ -1859,19 +1806,19 @@ qla81xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	iter_reg = qla24xx_read_window(reg, 0x7200, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->resp0_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7300, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->req1_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7400, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	/* Transmit DMA registers. */
 	iter_reg = fw->xmt0_dma_reg;
@@ -1993,57 +1940,44 @@ qla81xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 
 qla81xx_fw_dump_failed_0:
 	qla2xxx_dump_post_process(base_vha, rval);
-
-qla81xx_fw_dump_failed:
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
-#else
-	;
-#endif
 }
 
 void
-qla83xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla83xx_fw_dump(scsi_qla_host_t *vha)
 {
 	int		rval;
 	uint32_t	cnt;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
-	uint32_t __iomem *dmp_reg;
-	uint32_t	*iter_reg;
-	uint16_t __iomem *mbx_reg;
-	unsigned long	flags;
+	__le32 __iomem *dmp_reg;
+	__be32		*iter_reg;
+	__le16 __iomem *mbx_reg;
 	struct qla83xx_fw_dump *fw;
 	void		*nxt, *nxt_chain;
-	uint32_t	*last_chain = NULL;
+	__be32		*last_chain = NULL;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 
-	flags = 0;
-	ha->fw_dump_cap_flags = 0;
+	lockdep_assert_held(&ha->hardware_lock);
 
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_lock_irqsave(&ha->hardware_lock, flags);
-#endif
+	ha->fw_dump_cap_flags = 0;
 
 	if (!ha->fw_dump) {
 		ql_log(ql_log_warn, vha, 0xd00c,
 		    "No buffer available for dump!!!\n");
-		goto qla83xx_fw_dump_failed;
+		return;
 	}
 
 	if (ha->fw_dumped) {
 		ql_log(ql_log_warn, vha, 0xd00d,
 		    "Firmware has been previously dumped (%p) -- ignoring "
 		    "request...\n", ha->fw_dump);
-		goto qla83xx_fw_dump_failed;
+		return;
 	}
 	QLA_FW_STOPPED(ha);
 	fw = &ha->fw_dump->isp.isp83;
 	qla2xxx_prep_dump(ha, ha->fw_dump);
 
-	fw->host_status = htonl(RD_REG_DWORD(&reg->host_status));
+	fw->host_status = htonl(rd_reg_dword(&reg->host_status));
 
 	/*
 	 * Pause RISC. No need to track timeout, as resetting the chip
@@ -2051,24 +1985,24 @@ qla83xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	 */
 	qla24xx_pause_risc(reg, ha);
 
-	WRT_REG_DWORD(&reg->iobase_addr, 0x6000);
+	wrt_reg_dword(&reg->iobase_addr, 0x6000);
 	dmp_reg = &reg->iobase_window;
-	RD_REG_DWORD(dmp_reg);
-	WRT_REG_DWORD(dmp_reg, 0);
+	rd_reg_dword(dmp_reg);
+	wrt_reg_dword(dmp_reg, 0);
 
 	dmp_reg = &reg->unused_4_1[0];
-	RD_REG_DWORD(dmp_reg);
-	WRT_REG_DWORD(dmp_reg, 0);
+	rd_reg_dword(dmp_reg);
+	wrt_reg_dword(dmp_reg, 0);
 
-	WRT_REG_DWORD(&reg->iobase_addr, 0x6010);
+	wrt_reg_dword(&reg->iobase_addr, 0x6010);
 	dmp_reg = &reg->unused_4_1[2];
-	RD_REG_DWORD(dmp_reg);
-	WRT_REG_DWORD(dmp_reg, 0);
+	rd_reg_dword(dmp_reg);
+	wrt_reg_dword(dmp_reg, 0);
 
 	/* select PCR and disable ecc checking and correction */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0F70);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_select, 0x60000000);	/* write to F0h = PCR */
+	wrt_reg_dword(&reg->iobase_addr, 0x0F70);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_select, 0x60000000);	/* write to F0h = PCR */
 
 	/* Host/Risc registers. */
 	iter_reg = fw->host_risc_reg;
@@ -2077,73 +2011,73 @@ qla83xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	qla24xx_read_window(reg, 0x7040, 16, iter_reg);
 
 	/* PCIe registers. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x7C00);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_window, 0x01);
+	wrt_reg_dword(&reg->iobase_addr, 0x7C00);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_window, 0x01);
 	dmp_reg = &reg->iobase_c4;
-	fw->pcie_regs[0] = htonl(RD_REG_DWORD(dmp_reg));
+	fw->pcie_regs[0] = htonl(rd_reg_dword(dmp_reg));
 	dmp_reg++;
-	fw->pcie_regs[1] = htonl(RD_REG_DWORD(dmp_reg));
+	fw->pcie_regs[1] = htonl(rd_reg_dword(dmp_reg));
 	dmp_reg++;
-	fw->pcie_regs[2] = htonl(RD_REG_DWORD(dmp_reg));
-	fw->pcie_regs[3] = htonl(RD_REG_DWORD(&reg->iobase_window));
+	fw->pcie_regs[2] = htonl(rd_reg_dword(dmp_reg));
+	fw->pcie_regs[3] = htonl(rd_reg_dword(&reg->iobase_window));
 
-	WRT_REG_DWORD(&reg->iobase_window, 0x00);
-	RD_REG_DWORD(&reg->iobase_window);
+	wrt_reg_dword(&reg->iobase_window, 0x00);
+	rd_reg_dword(&reg->iobase_window);
 
 	/* Host interface registers. */
 	dmp_reg = &reg->flash_addr;
-	for (cnt = 0; cnt < sizeof(fw->host_reg) / 4; cnt++, dmp_reg++)
-		fw->host_reg[cnt] = htonl(RD_REG_DWORD(dmp_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->host_reg); cnt++, dmp_reg++)
+		fw->host_reg[cnt] = htonl(rd_reg_dword(dmp_reg));
 
 	/* Disable interrupts. */
-	WRT_REG_DWORD(&reg->ictrl, 0);
-	RD_REG_DWORD(&reg->ictrl);
+	wrt_reg_dword(&reg->ictrl, 0);
+	rd_reg_dword(&reg->ictrl);
 
 	/* Shadow registers. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0F70);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0000000);
-	fw->shadow_reg[0] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_addr, 0x0F70);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_select, 0xB0000000);
+	fw->shadow_reg[0] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0100000);
-	fw->shadow_reg[1] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0100000);
+	fw->shadow_reg[1] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0200000);
-	fw->shadow_reg[2] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0200000);
+	fw->shadow_reg[2] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0300000);
-	fw->shadow_reg[3] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0300000);
+	fw->shadow_reg[3] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0400000);
-	fw->shadow_reg[4] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0400000);
+	fw->shadow_reg[4] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0500000);
-	fw->shadow_reg[5] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0500000);
+	fw->shadow_reg[5] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0600000);
-	fw->shadow_reg[6] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0600000);
+	fw->shadow_reg[6] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0700000);
-	fw->shadow_reg[7] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0700000);
+	fw->shadow_reg[7] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0800000);
-	fw->shadow_reg[8] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0800000);
+	fw->shadow_reg[8] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0900000);
-	fw->shadow_reg[9] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0900000);
+	fw->shadow_reg[9] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
-	WRT_REG_DWORD(&reg->iobase_select, 0xB0A00000);
-	fw->shadow_reg[10] = htonl(RD_REG_DWORD(&reg->iobase_sdata));
+	wrt_reg_dword(&reg->iobase_select, 0xB0A00000);
+	fw->shadow_reg[10] = htonl(rd_reg_dword(&reg->iobase_sdata));
 
 	/* RISC I/O register. */
-	WRT_REG_DWORD(&reg->iobase_addr, 0x0010);
-	fw->risc_io_reg = htonl(RD_REG_DWORD(&reg->iobase_window));
+	wrt_reg_dword(&reg->iobase_addr, 0x0010);
+	fw->risc_io_reg = htonl(rd_reg_dword(&reg->iobase_window));
 
 	/* Mailbox registers. */
 	mbx_reg = &reg->mailbox0;
-	for (cnt = 0; cnt < sizeof(fw->mailbox_reg) / 2; cnt++, mbx_reg++)
-		fw->mailbox_reg[cnt] = htons(RD_REG_WORD(mbx_reg));
+	for (cnt = 0; cnt < ARRAY_SIZE(fw->mailbox_reg); cnt++, mbx_reg++)
+		fw->mailbox_reg[cnt] = htons(rd_reg_word(mbx_reg));
 
 	/* Transfer sequence registers. */
 	iter_reg = fw->xseq_gp_reg;
@@ -2239,19 +2173,19 @@ qla83xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 	iter_reg = qla24xx_read_window(reg, 0x7200, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->resp0_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7300, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	iter_reg = fw->req1_dma_reg;
 	iter_reg = qla24xx_read_window(reg, 0x7400, 8, iter_reg);
 	dmp_reg = &reg->iobase_q;
 	for (cnt = 0; cnt < 7; cnt++, dmp_reg++)
-		*iter_reg++ = htonl(RD_REG_DWORD(dmp_reg));
+		*iter_reg++ = htonl(rd_reg_dword(dmp_reg));
 
 	/* Transmit DMA registers. */
 	iter_reg = fw->xmt0_dma_reg;
@@ -2457,16 +2391,16 @@ qla83xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
 
 		ql_log(ql_log_warn, vha, 0xd00f, "try a bigger hammer!!!\n");
 
-		WRT_REG_DWORD(&reg->hccr, HCCRX_SET_RISC_RESET);
-		RD_REG_DWORD(&reg->hccr);
+		wrt_reg_dword(&reg->hccr, HCCRX_SET_RISC_RESET);
+		rd_reg_dword(&reg->hccr);
 
-		WRT_REG_DWORD(&reg->hccr, HCCRX_REL_RISC_PAUSE);
-		RD_REG_DWORD(&reg->hccr);
+		wrt_reg_dword(&reg->hccr, HCCRX_REL_RISC_PAUSE);
+		rd_reg_dword(&reg->hccr);
 
-		WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_RESET);
-		RD_REG_DWORD(&reg->hccr);
+		wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_RESET);
+		rd_reg_dword(&reg->hccr);
 
-		for (cnt = 30000; cnt && (RD_REG_WORD(&reg->mailbox0)); cnt--)
+		for (cnt = 30000; cnt && (rd_reg_word(&reg->mailbox0)); cnt--)
 			udelay(5);
 
 		if (!cnt) {
@@ -2507,14 +2441,6 @@ copy_queue:
 
 qla83xx_fw_dump_failed_0:
 	qla2xxx_dump_post_process(base_vha, rval);
-
-qla83xx_fw_dump_failed:
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_unlock_irqrestore(&ha->hardware_lock, flags);
-#else
-	;
-#endif
 }
 
 /****************************************************************************/
@@ -2735,7 +2661,7 @@ ql_dump_regs(uint level, scsi_qla_host_t *vha, uint id)
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 	struct device_reg_24xx __iomem *reg24 = &ha->iobase->isp24;
 	struct device_reg_82xx __iomem *reg82 = &ha->iobase->isp82;
-	uint16_t __iomem *mbx_reg;
+	__le16 __iomem *mbx_reg;
 
 	if (!ql_mask_match(level))
 		return;
@@ -2750,7 +2676,7 @@ ql_dump_regs(uint level, scsi_qla_host_t *vha, uint id)
 	ql_dbg(level, vha, id, "Mailbox registers:\n");
 	for (i = 0; i < 6; i++, mbx_reg++)
 		ql_dbg(level, vha, id,
-		    "mbox[%d] %#04x\n", i, RD_REG_WORD(mbx_reg));
+		    "mbox[%d] %#04x\n", i, rd_reg_word(mbx_reg));
 }
 
 
diff --git a/drivers/scsi/qla2xxx/qla_dbg.h b/drivers/scsi/qla2xxx/qla_dbg.h
index 433e95502808..54ed020e6f75 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.h
+++ b/drivers/scsi/qla2xxx/qla_dbg.h
@@ -12,205 +12,205 @@
  */
 
 struct qla2300_fw_dump {
-	uint16_t hccr;
-	uint16_t pbiu_reg[8];
-	uint16_t risc_host_reg[8];
-	uint16_t mailbox_reg[32];
-	uint16_t resp_dma_reg[32];
-	uint16_t dma_reg[48];
-	uint16_t risc_hdw_reg[16];
-	uint16_t risc_gp0_reg[16];
-	uint16_t risc_gp1_reg[16];
-	uint16_t risc_gp2_reg[16];
-	uint16_t risc_gp3_reg[16];
-	uint16_t risc_gp4_reg[16];
-	uint16_t risc_gp5_reg[16];
-	uint16_t risc_gp6_reg[16];
-	uint16_t risc_gp7_reg[16];
-	uint16_t frame_buf_hdw_reg[64];
-	uint16_t fpm_b0_reg[64];
-	uint16_t fpm_b1_reg[64];
-	uint16_t risc_ram[0xf800];
-	uint16_t stack_ram[0x1000];
-	uint16_t data_ram[1];
+	__be16 hccr;
+	__be16 pbiu_reg[8];
+	__be16 risc_host_reg[8];
+	__be16 mailbox_reg[32];
+	__be16 resp_dma_reg[32];
+	__be16 dma_reg[48];
+	__be16 risc_hdw_reg[16];
+	__be16 risc_gp0_reg[16];
+	__be16 risc_gp1_reg[16];
+	__be16 risc_gp2_reg[16];
+	__be16 risc_gp3_reg[16];
+	__be16 risc_gp4_reg[16];
+	__be16 risc_gp5_reg[16];
+	__be16 risc_gp6_reg[16];
+	__be16 risc_gp7_reg[16];
+	__be16 frame_buf_hdw_reg[64];
+	__be16 fpm_b0_reg[64];
+	__be16 fpm_b1_reg[64];
+	__be16 risc_ram[0xf800];
+	__be16 stack_ram[0x1000];
+	__be16 data_ram[1];
 };
 
 struct qla2100_fw_dump {
-	uint16_t hccr;
-	uint16_t pbiu_reg[8];
-	uint16_t mailbox_reg[32];
-	uint16_t dma_reg[48];
-	uint16_t risc_hdw_reg[16];
-	uint16_t risc_gp0_reg[16];
-	uint16_t risc_gp1_reg[16];
-	uint16_t risc_gp2_reg[16];
-	uint16_t risc_gp3_reg[16];
-	uint16_t risc_gp4_reg[16];
-	uint16_t risc_gp5_reg[16];
-	uint16_t risc_gp6_reg[16];
-	uint16_t risc_gp7_reg[16];
-	uint16_t frame_buf_hdw_reg[16];
-	uint16_t fpm_b0_reg[64];
-	uint16_t fpm_b1_reg[64];
-	uint16_t risc_ram[0xf000];
+	__be16 hccr;
+	__be16 pbiu_reg[8];
+	__be16 mailbox_reg[32];
+	__be16 dma_reg[48];
+	__be16 risc_hdw_reg[16];
+	__be16 risc_gp0_reg[16];
+	__be16 risc_gp1_reg[16];
+	__be16 risc_gp2_reg[16];
+	__be16 risc_gp3_reg[16];
+	__be16 risc_gp4_reg[16];
+	__be16 risc_gp5_reg[16];
+	__be16 risc_gp6_reg[16];
+	__be16 risc_gp7_reg[16];
+	__be16 frame_buf_hdw_reg[16];
+	__be16 fpm_b0_reg[64];
+	__be16 fpm_b1_reg[64];
+	__be16 risc_ram[0xf000];
 };
 
 struct qla24xx_fw_dump {
-	uint32_t host_status;
-	uint32_t host_reg[32];
-	uint32_t shadow_reg[7];
-	uint16_t mailbox_reg[32];
-	uint32_t xseq_gp_reg[128];
-	uint32_t xseq_0_reg[16];
-	uint32_t xseq_1_reg[16];
-	uint32_t rseq_gp_reg[128];
-	uint32_t rseq_0_reg[16];
-	uint32_t rseq_1_reg[16];
-	uint32_t rseq_2_reg[16];
-	uint32_t cmd_dma_reg[16];
-	uint32_t req0_dma_reg[15];
-	uint32_t resp0_dma_reg[15];
-	uint32_t req1_dma_reg[15];
-	uint32_t xmt0_dma_reg[32];
-	uint32_t xmt1_dma_reg[32];
-	uint32_t xmt2_dma_reg[32];
-	uint32_t xmt3_dma_reg[32];
-	uint32_t xmt4_dma_reg[32];
-	uint32_t xmt_data_dma_reg[16];
-	uint32_t rcvt0_data_dma_reg[32];
-	uint32_t rcvt1_data_dma_reg[32];
-	uint32_t risc_gp_reg[128];
-	uint32_t lmc_reg[112];
-	uint32_t fpm_hdw_reg[192];
-	uint32_t fb_hdw_reg[176];
-	uint32_t code_ram[0x2000];
-	uint32_t ext_mem[1];
+	__be32	host_status;
+	__be32	host_reg[32];
+	__be32	shadow_reg[7];
+	__be16	mailbox_reg[32];
+	__be32	xseq_gp_reg[128];
+	__be32	xseq_0_reg[16];
+	__be32	xseq_1_reg[16];
+	__be32	rseq_gp_reg[128];
+	__be32	rseq_0_reg[16];
+	__be32	rseq_1_reg[16];
+	__be32	rseq_2_reg[16];
+	__be32	cmd_dma_reg[16];
+	__be32	req0_dma_reg[15];
+	__be32	resp0_dma_reg[15];
+	__be32	req1_dma_reg[15];
+	__be32	xmt0_dma_reg[32];
+	__be32	xmt1_dma_reg[32];
+	__be32	xmt2_dma_reg[32];
+	__be32	xmt3_dma_reg[32];
+	__be32	xmt4_dma_reg[32];
+	__be32	xmt_data_dma_reg[16];
+	__be32	rcvt0_data_dma_reg[32];
+	__be32	rcvt1_data_dma_reg[32];
+	__be32	risc_gp_reg[128];
+	__be32	lmc_reg[112];
+	__be32	fpm_hdw_reg[192];
+	__be32	fb_hdw_reg[176];
+	__be32	code_ram[0x2000];
+	__be32	ext_mem[1];
 };
 
 struct qla25xx_fw_dump {
-	uint32_t host_status;
-	uint32_t host_risc_reg[32];
-	uint32_t pcie_regs[4];
-	uint32_t host_reg[32];
-	uint32_t shadow_reg[11];
-	uint32_t risc_io_reg;
-	uint16_t mailbox_reg[32];
-	uint32_t xseq_gp_reg[128];
-	uint32_t xseq_0_reg[48];
-	uint32_t xseq_1_reg[16];
-	uint32_t rseq_gp_reg[128];
-	uint32_t rseq_0_reg[32];
-	uint32_t rseq_1_reg[16];
-	uint32_t rseq_2_reg[16];
-	uint32_t aseq_gp_reg[128];
-	uint32_t aseq_0_reg[32];
-	uint32_t aseq_1_reg[16];
-	uint32_t aseq_2_reg[16];
-	uint32_t cmd_dma_reg[16];
-	uint32_t req0_dma_reg[15];
-	uint32_t resp0_dma_reg[15];
-	uint32_t req1_dma_reg[15];
-	uint32_t xmt0_dma_reg[32];
-	uint32_t xmt1_dma_reg[32];
-	uint32_t xmt2_dma_reg[32];
-	uint32_t xmt3_dma_reg[32];
-	uint32_t xmt4_dma_reg[32];
-	uint32_t xmt_data_dma_reg[16];
-	uint32_t rcvt0_data_dma_reg[32];
-	uint32_t rcvt1_data_dma_reg[32];
-	uint32_t risc_gp_reg[128];
-	uint32_t lmc_reg[128];
-	uint32_t fpm_hdw_reg[192];
-	uint32_t fb_hdw_reg[192];
-	uint32_t code_ram[0x2000];
-	uint32_t ext_mem[1];
+	__be32	host_status;
+	__be32	host_risc_reg[32];
+	__be32	pcie_regs[4];
+	__be32	host_reg[32];
+	__be32	shadow_reg[11];
+	__be32	risc_io_reg;
+	__be16	mailbox_reg[32];
+	__be32	xseq_gp_reg[128];
+	__be32	xseq_0_reg[48];
+	__be32	xseq_1_reg[16];
+	__be32	rseq_gp_reg[128];
+	__be32	rseq_0_reg[32];
+	__be32	rseq_1_reg[16];
+	__be32	rseq_2_reg[16];
+	__be32	aseq_gp_reg[128];
+	__be32	aseq_0_reg[32];
+	__be32	aseq_1_reg[16];
+	__be32	aseq_2_reg[16];
+	__be32	cmd_dma_reg[16];
+	__be32	req0_dma_reg[15];
+	__be32	resp0_dma_reg[15];
+	__be32	req1_dma_reg[15];
+	__be32	xmt0_dma_reg[32];
+	__be32	xmt1_dma_reg[32];
+	__be32	xmt2_dma_reg[32];
+	__be32	xmt3_dma_reg[32];
+	__be32	xmt4_dma_reg[32];
+	__be32	xmt_data_dma_reg[16];
+	__be32	rcvt0_data_dma_reg[32];
+	__be32	rcvt1_data_dma_reg[32];
+	__be32	risc_gp_reg[128];
+	__be32	lmc_reg[128];
+	__be32	fpm_hdw_reg[192];
+	__be32	fb_hdw_reg[192];
+	__be32	code_ram[0x2000];
+	__be32	ext_mem[1];
 };
 
 struct qla81xx_fw_dump {
-	uint32_t host_status;
-	uint32_t host_risc_reg[32];
-	uint32_t pcie_regs[4];
-	uint32_t host_reg[32];
-	uint32_t shadow_reg[11];
-	uint32_t risc_io_reg;
-	uint16_t mailbox_reg[32];
-	uint32_t xseq_gp_reg[128];
-	uint32_t xseq_0_reg[48];
-	uint32_t xseq_1_reg[16];
-	uint32_t rseq_gp_reg[128];
-	uint32_t rseq_0_reg[32];
-	uint32_t rseq_1_reg[16];
-	uint32_t rseq_2_reg[16];
-	uint32_t aseq_gp_reg[128];
-	uint32_t aseq_0_reg[32];
-	uint32_t aseq_1_reg[16];
-	uint32_t aseq_2_reg[16];
-	uint32_t cmd_dma_reg[16];
-	uint32_t req0_dma_reg[15];
-	uint32_t resp0_dma_reg[15];
-	uint32_t req1_dma_reg[15];
-	uint32_t xmt0_dma_reg[32];
-	uint32_t xmt1_dma_reg[32];
-	uint32_t xmt2_dma_reg[32];
-	uint32_t xmt3_dma_reg[32];
-	uint32_t xmt4_dma_reg[32];
-	uint32_t xmt_data_dma_reg[16];
-	uint32_t rcvt0_data_dma_reg[32];
-	uint32_t rcvt1_data_dma_reg[32];
-	uint32_t risc_gp_reg[128];
-	uint32_t lmc_reg[128];
-	uint32_t fpm_hdw_reg[224];
-	uint32_t fb_hdw_reg[208];
-	uint32_t code_ram[0x2000];
-	uint32_t ext_mem[1];
+	__be32	host_status;
+	__be32	host_risc_reg[32];
+	__be32	pcie_regs[4];
+	__be32	host_reg[32];
+	__be32	shadow_reg[11];
+	__be32	risc_io_reg;
+	__be16	mailbox_reg[32];
+	__be32	xseq_gp_reg[128];
+	__be32	xseq_0_reg[48];
+	__be32	xseq_1_reg[16];
+	__be32	rseq_gp_reg[128];
+	__be32	rseq_0_reg[32];
+	__be32	rseq_1_reg[16];
+	__be32	rseq_2_reg[16];
+	__be32	aseq_gp_reg[128];
+	__be32	aseq_0_reg[32];
+	__be32	aseq_1_reg[16];
+	__be32	aseq_2_reg[16];
+	__be32	cmd_dma_reg[16];
+	__be32	req0_dma_reg[15];
+	__be32	resp0_dma_reg[15];
+	__be32	req1_dma_reg[15];
+	__be32	xmt0_dma_reg[32];
+	__be32	xmt1_dma_reg[32];
+	__be32	xmt2_dma_reg[32];
+	__be32	xmt3_dma_reg[32];
+	__be32	xmt4_dma_reg[32];
+	__be32	xmt_data_dma_reg[16];
+	__be32	rcvt0_data_dma_reg[32];
+	__be32	rcvt1_data_dma_reg[32];
+	__be32	risc_gp_reg[128];
+	__be32	lmc_reg[128];
+	__be32	fpm_hdw_reg[224];
+	__be32	fb_hdw_reg[208];
+	__be32	code_ram[0x2000];
+	__be32	ext_mem[1];
 };
 
 struct qla83xx_fw_dump {
-	uint32_t host_status;
-	uint32_t host_risc_reg[48];
-	uint32_t pcie_regs[4];
-	uint32_t host_reg[32];
-	uint32_t shadow_reg[11];
-	uint32_t risc_io_reg;
-	uint16_t mailbox_reg[32];
-	uint32_t xseq_gp_reg[256];
-	uint32_t xseq_0_reg[48];
-	uint32_t xseq_1_reg[16];
-	uint32_t xseq_2_reg[16];
-	uint32_t rseq_gp_reg[256];
-	uint32_t rseq_0_reg[32];
-	uint32_t rseq_1_reg[16];
-	uint32_t rseq_2_reg[16];
-	uint32_t rseq_3_reg[16];
-	uint32_t aseq_gp_reg[256];
-	uint32_t aseq_0_reg[32];
-	uint32_t aseq_1_reg[16];
-	uint32_t aseq_2_reg[16];
-	uint32_t aseq_3_reg[16];
-	uint32_t cmd_dma_reg[64];
-	uint32_t req0_dma_reg[15];
-	uint32_t resp0_dma_reg[15];
-	uint32_t req1_dma_reg[15];
-	uint32_t xmt0_dma_reg[32];
-	uint32_t xmt1_dma_reg[32];
-	uint32_t xmt2_dma_reg[32];
-	uint32_t xmt3_dma_reg[32];
-	uint32_t xmt4_dma_reg[32];
-	uint32_t xmt_data_dma_reg[16];
-	uint32_t rcvt0_data_dma_reg[32];
-	uint32_t rcvt1_data_dma_reg[32];
-	uint32_t risc_gp_reg[128];
-	uint32_t lmc_reg[128];
-	uint32_t fpm_hdw_reg[256];
-	uint32_t rq0_array_reg[256];
-	uint32_t rq1_array_reg[256];
-	uint32_t rp0_array_reg[256];
-	uint32_t rp1_array_reg[256];
-	uint32_t queue_control_reg[16];
-	uint32_t fb_hdw_reg[432];
-	uint32_t at0_array_reg[128];
-	uint32_t code_ram[0x2400];
-	uint32_t ext_mem[1];
+	__be32	host_status;
+	__be32	host_risc_reg[48];
+	__be32	pcie_regs[4];
+	__be32	host_reg[32];
+	__be32	shadow_reg[11];
+	__be32	risc_io_reg;
+	__be16	mailbox_reg[32];
+	__be32	xseq_gp_reg[256];
+	__be32	xseq_0_reg[48];
+	__be32	xseq_1_reg[16];
+	__be32	xseq_2_reg[16];
+	__be32	rseq_gp_reg[256];
+	__be32	rseq_0_reg[32];
+	__be32	rseq_1_reg[16];
+	__be32	rseq_2_reg[16];
+	__be32	rseq_3_reg[16];
+	__be32	aseq_gp_reg[256];
+	__be32	aseq_0_reg[32];
+	__be32	aseq_1_reg[16];
+	__be32	aseq_2_reg[16];
+	__be32	aseq_3_reg[16];
+	__be32	cmd_dma_reg[64];
+	__be32	req0_dma_reg[15];
+	__be32	resp0_dma_reg[15];
+	__be32	req1_dma_reg[15];
+	__be32	xmt0_dma_reg[32];
+	__be32	xmt1_dma_reg[32];
+	__be32	xmt2_dma_reg[32];
+	__be32	xmt3_dma_reg[32];
+	__be32	xmt4_dma_reg[32];
+	__be32	xmt_data_dma_reg[16];
+	__be32	rcvt0_data_dma_reg[32];
+	__be32	rcvt1_data_dma_reg[32];
+	__be32	risc_gp_reg[128];
+	__be32	lmc_reg[128];
+	__be32	fpm_hdw_reg[256];
+	__be32	rq0_array_reg[256];
+	__be32	rq1_array_reg[256];
+	__be32	rp0_array_reg[256];
+	__be32	rp1_array_reg[256];
+	__be32	queue_control_reg[16];
+	__be32	fb_hdw_reg[432];
+	__be32	at0_array_reg[128];
+	__be32	code_ram[0x2400];
+	__be32	ext_mem[1];
 };
 
 #define EFT_NUM_BUFFERS		4
@@ -223,44 +223,45 @@ struct qla83xx_fw_dump {
 #define fce_calc_size(b)	((FCE_BYTES_PER_BUFFER) * (b))
 
 struct qla2xxx_fce_chain {
-	uint32_t type;
-	uint32_t chain_size;
+	__be32	type;
+	__be32	chain_size;
 
-	uint32_t size;
-	uint32_t addr_l;
-	uint32_t addr_h;
-	uint32_t eregs[8];
+	__be32	size;
+	__be32	addr_l;
+	__be32	addr_h;
+	__be32	eregs[8];
 };
 
 /* used by exchange off load and extended login offload */
 struct qla2xxx_offld_chain {
-	uint32_t type;
-	uint32_t chain_size;
+	__be32	type;
+	__be32	chain_size;
 
-	uint32_t size;
-	u64	 addr;
+	__be32	size;
+	__be32	reserved;
+	__be64	addr;
 };
 
 struct qla2xxx_mq_chain {
-	uint32_t type;
-	uint32_t chain_size;
+	__be32	type;
+	__be32	chain_size;
 
-	uint32_t count;
-	uint32_t qregs[4 * QLA_MQ_SIZE];
+	__be32	count;
+	__be32	qregs[4 * QLA_MQ_SIZE];
 };
 
 struct qla2xxx_mqueue_header {
-	uint32_t queue;
+	__be32	queue;
 #define TYPE_REQUEST_QUEUE	0x1
 #define TYPE_RESPONSE_QUEUE	0x2
 #define TYPE_ATIO_QUEUE		0x3
-	uint32_t number;
-	uint32_t size;
+	__be32	number;
+	__be32	size;
 };
 
 struct qla2xxx_mqueue_chain {
-	uint32_t type;
-	uint32_t chain_size;
+	__be32	type;
+	__be32	chain_size;
 };
 
 #define DUMP_CHAIN_VARIANT	0x80000000
@@ -273,28 +274,28 @@ struct qla2xxx_mqueue_chain {
 
 struct qla2xxx_fw_dump {
 	uint8_t signature[4];
-	uint32_t version;
+	__be32	version;
 
-	uint32_t fw_major_version;
-	uint32_t fw_minor_version;
-	uint32_t fw_subminor_version;
-	uint32_t fw_attributes;
+	__be32 fw_major_version;
+	__be32 fw_minor_version;
+	__be32 fw_subminor_version;
+	__be32 fw_attributes;
 
-	uint32_t vendor;
-	uint32_t device;
-	uint32_t subsystem_vendor;
-	uint32_t subsystem_device;
+	__be32 vendor;
+	__be32 device;
+	__be32 subsystem_vendor;
+	__be32 subsystem_device;
 
-	uint32_t fixed_size;
-	uint32_t mem_size;
-	uint32_t req_q_size;
-	uint32_t rsp_q_size;
+	__be32	fixed_size;
+	__be32	mem_size;
+	__be32	req_q_size;
+	__be32	rsp_q_size;
 
-	uint32_t eft_size;
-	uint32_t eft_addr_l;
-	uint32_t eft_addr_h;
+	__be32	eft_size;
+	__be32	eft_addr_l;
+	__be32	eft_addr_h;
 
-	uint32_t header_size;
+	__be32	header_size;
 
 	union {
 		struct qla2100_fw_dump isp21;
@@ -369,7 +370,7 @@ ql_log_qp(uint32_t, struct qla_qpair *, int32_t, const char *fmt, ...);
 
 extern int qla27xx_dump_mpi_ram(struct qla_hw_data *, uint32_t, uint32_t *,
 	uint32_t, void **);
-extern int qla24xx_dump_ram(struct qla_hw_data *, uint32_t, uint32_t *,
+extern int qla24xx_dump_ram(struct qla_hw_data *, uint32_t, __be32 *,
 	uint32_t, void **);
 extern void qla24xx_pause_risc(struct device_reg_24xx __iomem *,
 	struct qla_hw_data *);
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 47c7a56438b5..42dbf90d4651 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -128,15 +128,50 @@ static inline uint32_t make_handle(uint16_t x, uint16_t y)
  * I/O register
 */
 
-#define RD_REG_BYTE(addr)		readb(addr)
-#define RD_REG_WORD(addr)		readw(addr)
-#define RD_REG_DWORD(addr)		readl(addr)
-#define RD_REG_BYTE_RELAXED(addr)	readb_relaxed(addr)
-#define RD_REG_WORD_RELAXED(addr)	readw_relaxed(addr)
-#define RD_REG_DWORD_RELAXED(addr)	readl_relaxed(addr)
-#define WRT_REG_BYTE(addr, data)	writeb(data, addr)
-#define WRT_REG_WORD(addr, data)	writew(data, addr)
-#define WRT_REG_DWORD(addr, data)	writel(data, addr)
+static inline u8 rd_reg_byte(const volatile u8 __iomem *addr)
+{
+	return readb(addr);
+}
+
+static inline u16 rd_reg_word(const volatile __le16 __iomem *addr)
+{
+	return readw(addr);
+}
+
+static inline u32 rd_reg_dword(const volatile __le32 __iomem *addr)
+{
+	return readl(addr);
+}
+
+static inline u8 rd_reg_byte_relaxed(const volatile u8 __iomem *addr)
+{
+	return readb_relaxed(addr);
+}
+
+static inline u16 rd_reg_word_relaxed(const volatile __le16 __iomem *addr)
+{
+	return readw_relaxed(addr);
+}
+
+static inline u32 rd_reg_dword_relaxed(const volatile __le32 __iomem *addr)
+{
+	return readl_relaxed(addr);
+}
+
+static inline void wrt_reg_byte(volatile u8 __iomem *addr, u8 data)
+{
+	return writeb(data, addr);
+}
+
+static inline void wrt_reg_word(volatile __le16 __iomem *addr, u16 data)
+{
+	return writew(data, addr);
+}
+
+static inline void wrt_reg_dword(volatile __le32 __iomem *addr, u32 data)
+{
+	return writel(data, addr);
+}
 
 /*
  * ISP83XX specific remote register addresses
@@ -469,7 +504,7 @@ struct srb_iocb {
 			u32 rx_size;
 			dma_addr_t els_plogi_pyld_dma;
 			dma_addr_t els_resp_pyld_dma;
-			uint32_t	fw_status[3];
+			__le32	fw_status[3];
 			__le16	comp_status;
 			__le16	len;
 		} els_plogi;
@@ -520,8 +555,8 @@ struct srb_iocb {
 #define MAX_IOCB_MB_REG 28
 #define SIZEOF_IOCB_MB_REG (MAX_IOCB_MB_REG * sizeof(uint16_t))
 		struct {
-			__le16 in_mb[MAX_IOCB_MB_REG];	/* from FW */
-			__le16 out_mb[MAX_IOCB_MB_REG];	/* to FW */
+			u16 in_mb[MAX_IOCB_MB_REG];	/* from FW */
+			u16 out_mb[MAX_IOCB_MB_REG];	/* to FW */
 			void *out, *in;
 			dma_addr_t out_dma, in_dma;
 			struct completion comp;
@@ -532,7 +567,7 @@ struct srb_iocb {
 		} nack;
 		struct {
 			__le16 comp_status;
-			uint16_t rsp_pyld_len;
+			__le16 rsp_pyld_len;
 			uint8_t	aen_op;
 			void *desc;
 
@@ -663,23 +698,23 @@ struct msg_echo_lb {
  * ISP I/O Register Set structure definitions.
  */
 struct device_reg_2xxx {
-	uint16_t flash_address; 	/* Flash BIOS address */
-	uint16_t flash_data;		/* Flash BIOS data */
-	uint16_t unused_1[1];		/* Gap */
-	uint16_t ctrl_status;		/* Control/Status */
+	__le16	flash_address; 	/* Flash BIOS address */
+	__le16	flash_data;		/* Flash BIOS data */
+	__le16	unused_1[1];		/* Gap */
+	__le16	ctrl_status;		/* Control/Status */
 #define CSR_FLASH_64K_BANK	BIT_3	/* Flash upper 64K bank select */
 #define CSR_FLASH_ENABLE	BIT_1	/* Flash BIOS Read/Write enable */
 #define CSR_ISP_SOFT_RESET	BIT_0	/* ISP soft reset */
 
-	uint16_t ictrl;			/* Interrupt control */
+	__le16	ictrl;			/* Interrupt control */
 #define ICR_EN_INT		BIT_15	/* ISP enable interrupts. */
 #define ICR_EN_RISC		BIT_3	/* ISP enable RISC interrupts. */
 
-	uint16_t istatus;		/* Interrupt status */
+	__le16	istatus;		/* Interrupt status */
 #define ISR_RISC_INT		BIT_3	/* RISC interrupt */
 
-	uint16_t semaphore;		/* Semaphore */
-	uint16_t nvram;			/* NVRAM register. */
+	__le16	semaphore;		/* Semaphore */
+	__le16	nvram;			/* NVRAM register. */
 #define NVR_DESELECT		0
 #define NVR_BUSY		BIT_15
 #define NVR_WRT_ENABLE		BIT_14	/* Write enable */
@@ -693,80 +728,80 @@ struct device_reg_2xxx {
 
 	union {
 		struct {
-			uint16_t mailbox0;
-			uint16_t mailbox1;
-			uint16_t mailbox2;
-			uint16_t mailbox3;
-			uint16_t mailbox4;
-			uint16_t mailbox5;
-			uint16_t mailbox6;
-			uint16_t mailbox7;
-			uint16_t unused_2[59];	/* Gap */
+			__le16	mailbox0;
+			__le16	mailbox1;
+			__le16	mailbox2;
+			__le16	mailbox3;
+			__le16	mailbox4;
+			__le16	mailbox5;
+			__le16	mailbox6;
+			__le16	mailbox7;
+			__le16	unused_2[59];	/* Gap */
 		} __attribute__((packed)) isp2100;
 		struct {
 						/* Request Queue */
-			uint16_t req_q_in;	/*  In-Pointer */
-			uint16_t req_q_out;	/*  Out-Pointer */
+			__le16	req_q_in;	/*  In-Pointer */
+			__le16	req_q_out;	/*  Out-Pointer */
 						/* Response Queue */
-			uint16_t rsp_q_in;	/*  In-Pointer */
-			uint16_t rsp_q_out;	/*  Out-Pointer */
+			__le16	rsp_q_in;	/*  In-Pointer */
+			__le16	rsp_q_out;	/*  Out-Pointer */
 
 						/* RISC to Host Status */
-			uint32_t host_status;
+			__le32	host_status;
 #define HSR_RISC_INT		BIT_15	/* RISC interrupt */
 #define HSR_RISC_PAUSED		BIT_8	/* RISC Paused */
 
 					/* Host to Host Semaphore */
-			uint16_t host_semaphore;
-			uint16_t unused_3[17];	/* Gap */
-			uint16_t mailbox0;
-			uint16_t mailbox1;
-			uint16_t mailbox2;
-			uint16_t mailbox3;
-			uint16_t mailbox4;
-			uint16_t mailbox5;
-			uint16_t mailbox6;
-			uint16_t mailbox7;
-			uint16_t mailbox8;
-			uint16_t mailbox9;
-			uint16_t mailbox10;
-			uint16_t mailbox11;
-			uint16_t mailbox12;
-			uint16_t mailbox13;
-			uint16_t mailbox14;
-			uint16_t mailbox15;
-			uint16_t mailbox16;
-			uint16_t mailbox17;
-			uint16_t mailbox18;
-			uint16_t mailbox19;
-			uint16_t mailbox20;
-			uint16_t mailbox21;
-			uint16_t mailbox22;
-			uint16_t mailbox23;
-			uint16_t mailbox24;
-			uint16_t mailbox25;
-			uint16_t mailbox26;
-			uint16_t mailbox27;
-			uint16_t mailbox28;
-			uint16_t mailbox29;
-			uint16_t mailbox30;
-			uint16_t mailbox31;
-			uint16_t fb_cmd;
-			uint16_t unused_4[10];	/* Gap */
+			__le16	host_semaphore;
+			__le16	unused_3[17];	/* Gap */
+			__le16	mailbox0;
+			__le16	mailbox1;
+			__le16	mailbox2;
+			__le16	mailbox3;
+			__le16	mailbox4;
+			__le16	mailbox5;
+			__le16	mailbox6;
+			__le16	mailbox7;
+			__le16	mailbox8;
+			__le16	mailbox9;
+			__le16	mailbox10;
+			__le16	mailbox11;
+			__le16	mailbox12;
+			__le16	mailbox13;
+			__le16	mailbox14;
+			__le16	mailbox15;
+			__le16	mailbox16;
+			__le16	mailbox17;
+			__le16	mailbox18;
+			__le16	mailbox19;
+			__le16	mailbox20;
+			__le16	mailbox21;
+			__le16	mailbox22;
+			__le16	mailbox23;
+			__le16	mailbox24;
+			__le16	mailbox25;
+			__le16	mailbox26;
+			__le16	mailbox27;
+			__le16	mailbox28;
+			__le16	mailbox29;
+			__le16	mailbox30;
+			__le16	mailbox31;
+			__le16	fb_cmd;
+			__le16	unused_4[10];	/* Gap */
 		} __attribute__((packed)) isp2300;
 	} u;
 
-	uint16_t fpm_diag_config;
-	uint16_t unused_5[0x4];		/* Gap */
-	uint16_t risc_hw;
-	uint16_t unused_5_1;		/* Gap */
-	uint16_t pcr;			/* Processor Control Register. */
-	uint16_t unused_6[0x5];		/* Gap */
-	uint16_t mctr;			/* Memory Configuration and Timing. */
-	uint16_t unused_7[0x3];		/* Gap */
-	uint16_t fb_cmd_2100;		/* Unused on 23XX */
-	uint16_t unused_8[0x3];		/* Gap */
-	uint16_t hccr;			/* Host command & control register. */
+	__le16	fpm_diag_config;
+	__le16	unused_5[0x4];		/* Gap */
+	__le16	risc_hw;
+	__le16	unused_5_1;		/* Gap */
+	__le16	pcr;			/* Processor Control Register. */
+	__le16	unused_6[0x5];		/* Gap */
+	__le16	mctr;			/* Memory Configuration and Timing. */
+	__le16	unused_7[0x3];		/* Gap */
+	__le16	fb_cmd_2100;		/* Unused on 23XX */
+	__le16	unused_8[0x3];		/* Gap */
+	__le16	hccr;			/* Host command & control register. */
 #define HCCR_HOST_INT		BIT_7	/* Host interrupt bit */
 #define HCCR_RISC_PAUSE		BIT_5	/* Pause mode bit */
 					/* HCCR commands */
@@ -779,9 +814,9 @@ struct device_reg_2xxx {
 #define	HCCR_DISABLE_PARITY_PAUSE 0x4001 /* Disable parity error RISC pause. */
 #define HCCR_ENABLE_PARITY	0xA000	/* Enable PARITY interrupt */
 
-	uint16_t unused_9[5];		/* Gap */
-	uint16_t gpiod;			/* GPIO Data register. */
-	uint16_t gpioe;			/* GPIO Enable register. */
+	__le16	unused_9[5];		/* Gap */
+	__le16	gpiod;			/* GPIO Data register. */
+	__le16	gpioe;			/* GPIO Enable register. */
 #define GPIO_LED_MASK			0x00C0
 #define GPIO_LED_GREEN_OFF_AMBER_OFF	0x0000
 #define GPIO_LED_GREEN_ON_AMBER_OFF	0x0040
@@ -793,95 +828,95 @@ struct device_reg_2xxx {
 
 	union {
 		struct {
-			uint16_t unused_10[8];	/* Gap */
-			uint16_t mailbox8;
-			uint16_t mailbox9;
-			uint16_t mailbox10;
-			uint16_t mailbox11;
-			uint16_t mailbox12;
-			uint16_t mailbox13;
-			uint16_t mailbox14;
-			uint16_t mailbox15;
-			uint16_t mailbox16;
-			uint16_t mailbox17;
-			uint16_t mailbox18;
-			uint16_t mailbox19;
-			uint16_t mailbox20;
-			uint16_t mailbox21;
-			uint16_t mailbox22;
-			uint16_t mailbox23;	/* Also probe reg. */
+			__le16	unused_10[8];	/* Gap */
+			__le16	mailbox8;
+			__le16	mailbox9;
+			__le16	mailbox10;
+			__le16	mailbox11;
+			__le16	mailbox12;
+			__le16	mailbox13;
+			__le16	mailbox14;
+			__le16	mailbox15;
+			__le16	mailbox16;
+			__le16	mailbox17;
+			__le16	mailbox18;
+			__le16	mailbox19;
+			__le16	mailbox20;
+			__le16	mailbox21;
+			__le16	mailbox22;
+			__le16	mailbox23;	/* Also probe reg. */
 		} __attribute__((packed)) isp2200;
 	} u_end;
 };
 
 struct device_reg_25xxmq {
-	uint32_t req_q_in;
-	uint32_t req_q_out;
-	uint32_t rsp_q_in;
-	uint32_t rsp_q_out;
-	uint32_t atio_q_in;
-	uint32_t atio_q_out;
+	__le32	req_q_in;
+	__le32	req_q_out;
+	__le32	rsp_q_in;
+	__le32	rsp_q_out;
+	__le32	atio_q_in;
+	__le32	atio_q_out;
 };
 
 
 struct device_reg_fx00 {
-	uint32_t mailbox0;		/* 00 */
-	uint32_t mailbox1;		/* 04 */
-	uint32_t mailbox2;		/* 08 */
-	uint32_t mailbox3;		/* 0C */
-	uint32_t mailbox4;		/* 10 */
-	uint32_t mailbox5;		/* 14 */
-	uint32_t mailbox6;		/* 18 */
-	uint32_t mailbox7;		/* 1C */
-	uint32_t mailbox8;		/* 20 */
-	uint32_t mailbox9;		/* 24 */
-	uint32_t mailbox10;		/* 28 */
-	uint32_t mailbox11;
-	uint32_t mailbox12;
-	uint32_t mailbox13;
-	uint32_t mailbox14;
-	uint32_t mailbox15;
-	uint32_t mailbox16;
-	uint32_t mailbox17;
-	uint32_t mailbox18;
-	uint32_t mailbox19;
-	uint32_t mailbox20;
-	uint32_t mailbox21;
-	uint32_t mailbox22;
-	uint32_t mailbox23;
-	uint32_t mailbox24;
-	uint32_t mailbox25;
-	uint32_t mailbox26;
-	uint32_t mailbox27;
-	uint32_t mailbox28;
-	uint32_t mailbox29;
-	uint32_t mailbox30;
-	uint32_t mailbox31;
-	uint32_t aenmailbox0;
-	uint32_t aenmailbox1;
-	uint32_t aenmailbox2;
-	uint32_t aenmailbox3;
-	uint32_t aenmailbox4;
-	uint32_t aenmailbox5;
-	uint32_t aenmailbox6;
-	uint32_t aenmailbox7;
+	__le32	mailbox0;		/* 00 */
+	__le32	mailbox1;		/* 04 */
+	__le32	mailbox2;		/* 08 */
+	__le32	mailbox3;		/* 0C */
+	__le32	mailbox4;		/* 10 */
+	__le32	mailbox5;		/* 14 */
+	__le32	mailbox6;		/* 18 */
+	__le32	mailbox7;		/* 1C */
+	__le32	mailbox8;		/* 20 */
+	__le32	mailbox9;		/* 24 */
+	__le32	mailbox10;		/* 28 */
+	__le32	mailbox11;
+	__le32	mailbox12;
+	__le32	mailbox13;
+	__le32	mailbox14;
+	__le32	mailbox15;
+	__le32	mailbox16;
+	__le32	mailbox17;
+	__le32	mailbox18;
+	__le32	mailbox19;
+	__le32	mailbox20;
+	__le32	mailbox21;
+	__le32	mailbox22;
+	__le32	mailbox23;
+	__le32	mailbox24;
+	__le32	mailbox25;
+	__le32	mailbox26;
+	__le32	mailbox27;
+	__le32	mailbox28;
+	__le32	mailbox29;
+	__le32	mailbox30;
+	__le32	mailbox31;
+	__le32	aenmailbox0;
+	__le32	aenmailbox1;
+	__le32	aenmailbox2;
+	__le32	aenmailbox3;
+	__le32	aenmailbox4;
+	__le32	aenmailbox5;
+	__le32	aenmailbox6;
+	__le32	aenmailbox7;
 	/* Request Queue. */
-	uint32_t req_q_in;		/* A0 - Request Queue In-Pointer */
-	uint32_t req_q_out;		/* A4 - Request Queue Out-Pointer */
+	__le32	req_q_in;		/* A0 - Request Queue In-Pointer */
+	__le32	req_q_out;		/* A4 - Request Queue Out-Pointer */
 	/* Response Queue. */
-	uint32_t rsp_q_in;		/* A8 - Response Queue In-Pointer */
-	uint32_t rsp_q_out;		/* AC - Response Queue Out-Pointer */
+	__le32	rsp_q_in;		/* A8 - Response Queue In-Pointer */
+	__le32	rsp_q_out;		/* AC - Response Queue Out-Pointer */
 	/* Init values shadowed on FW Up Event */
-	uint32_t initval0;		/* B0 */
-	uint32_t initval1;		/* B4 */
-	uint32_t initval2;		/* B8 */
-	uint32_t initval3;		/* BC */
-	uint32_t initval4;		/* C0 */
-	uint32_t initval5;		/* C4 */
-	uint32_t initval6;		/* C8 */
-	uint32_t initval7;		/* CC */
-	uint32_t fwheartbeat;		/* D0 */
-	uint32_t pseudoaen;		/* D4 */
+	__le32	initval0;		/* B0 */
+	__le32	initval1;		/* B4 */
+	__le32	initval2;		/* B8 */
+	__le32	initval3;		/* BC */
+	__le32	initval4;		/* C0 */
+	__le32	initval5;		/* C4 */
+	__le32	initval6;		/* C8 */
+	__le32	initval7;		/* CC */
+	__le32	fwheartbeat;		/* D0 */
+	__le32	pseudoaen;		/* D4 */
 };
 
 
@@ -921,18 +956,18 @@ typedef union {
 	  &(reg)->u_end.isp2200.mailbox8 + (num) - 8) : \
 	 &(reg)->u.isp2300.mailbox0 + (num))
 #define RD_MAILBOX_REG(ha, reg, num) \
-	RD_REG_WORD(MAILBOX_REG(ha, reg, num))
+	rd_reg_word(MAILBOX_REG(ha, reg, num))
 #define WRT_MAILBOX_REG(ha, reg, num, data) \
-	WRT_REG_WORD(MAILBOX_REG(ha, reg, num), data)
+	wrt_reg_word(MAILBOX_REG(ha, reg, num), data)
 
 #define FB_CMD_REG(ha, reg) \
 	(IS_QLA2100(ha) || IS_QLA2200(ha) ? \
 	 &(reg)->fb_cmd_2100 : \
 	 &(reg)->u.isp2300.fb_cmd)
 #define RD_FB_CMD_REG(ha, reg) \
-	RD_REG_WORD(FB_CMD_REG(ha, reg))
+	rd_reg_word(FB_CMD_REG(ha, reg))
 #define WRT_FB_CMD_REG(ha, reg, data) \
-	WRT_REG_WORD(FB_CMD_REG(ha, reg), data)
+	wrt_reg_word(FB_CMD_REG(ha, reg), data)
 
 typedef struct {
 	uint32_t	out_mb;		/* outbound from driver */
@@ -1316,7 +1351,7 @@ typedef struct {
 	uint8_t port_id[4];
 	uint8_t node_name[WWN_SIZE];
 	uint8_t port_name[WWN_SIZE];
-	uint16_t execution_throttle;
+	__le16	execution_throttle;
 	uint16_t execution_count;
 	uint8_t reset_count;
 	uint8_t reserved_2;
@@ -1402,9 +1437,9 @@ typedef struct {
 	 */
 	uint8_t  firmware_options[2];
 
-	uint16_t frame_payload_size;
-	uint16_t max_iocb_allocation;
-	uint16_t execution_throttle;
+	__le16	frame_payload_size;
+	__le16	max_iocb_allocation;
+	__le16	execution_throttle;
 	uint8_t  retry_count;
 	uint8_t	 retry_delay;			/* unused */
 	uint8_t	 port_name[WWN_SIZE];		/* Big endian. */
@@ -1413,17 +1448,17 @@ typedef struct {
 	uint8_t	 login_timeout;
 	uint8_t	 node_name[WWN_SIZE];		/* Big endian. */
 
-	uint16_t request_q_outpointer;
-	uint16_t response_q_inpointer;
-	uint16_t request_q_length;
-	uint16_t response_q_length;
-	__le64   request_q_address __packed;
-	__le64   response_q_address __packed;
+	__le16	request_q_outpointer;
+	__le16	response_q_inpointer;
+	__le16	request_q_length;
+	__le16	response_q_length;
+	__le64  request_q_address __packed;
+	__le64  response_q_address __packed;
 
-	uint16_t lun_enables;
+	__le16	lun_enables;
 	uint8_t  command_resource_count;
 	uint8_t  immediate_notify_resource_count;
-	uint16_t timeout;
+	__le16	timeout;
 	uint8_t  reserved_2[2];
 
 	/*
@@ -1571,8 +1606,8 @@ typedef struct {
 	uint8_t	 firmware_options[2];
 
 	uint16_t frame_payload_size;
-	uint16_t max_iocb_allocation;
-	uint16_t execution_throttle;
+	__le16	max_iocb_allocation;
+	__le16	execution_throttle;
 	uint8_t	 retry_count;
 	uint8_t	 retry_delay;			/* unused */
 	uint8_t	 port_name[WWN_SIZE];		/* Big endian. */
@@ -1696,7 +1731,7 @@ typedef struct {
 	uint8_t reset_delay;
 	uint8_t port_down_retry_count;
 	uint8_t boot_id_number;
-	uint16_t max_luns_per_target;
+	__le16	max_luns_per_target;
 	uint8_t fcode_boot_port_name[WWN_SIZE];
 	uint8_t alternate_port_name[WWN_SIZE];
 	uint8_t alternate_node_name[WWN_SIZE];
@@ -1802,7 +1837,7 @@ struct atio {
 };
 
 typedef union {
-	uint16_t extended;
+	__le16	extended;
 	struct {
 		uint8_t reserved;
 		uint8_t standard;
@@ -1828,18 +1863,18 @@ typedef struct {
 	uint8_t entry_status;		/* Entry Status. */
 	uint32_t handle;		/* System handle. */
 	target_id_t target;		/* SCSI ID */
-	uint16_t lun;			/* SCSI LUN */
-	uint16_t control_flags;		/* Control flags. */
+	__le16	lun;			/* SCSI LUN */
+	__le16	control_flags;		/* Control flags. */
 #define CF_WRITE	BIT_6
 #define CF_READ		BIT_5
 #define CF_SIMPLE_TAG	BIT_3
 #define CF_ORDERED_TAG	BIT_2
 #define CF_HEAD_TAG	BIT_1
 	uint16_t reserved_1;
-	uint16_t timeout;		/* Command timeout. */
-	uint16_t dseg_count;		/* Data segment count. */
+	__le16	timeout;		/* Command timeout. */
+	__le16	dseg_count;		/* Data segment count. */
 	uint8_t scsi_cdb[MAX_CMDSZ]; 	/* SCSI command words. */
-	uint32_t byte_count;		/* Total byte count. */
+	__le32	byte_count;		/* Total byte count. */
 	union {
 		struct dsd32 dsd32[3];
 		struct dsd64 dsd64[2];
@@ -1857,11 +1892,11 @@ typedef struct {
 	uint8_t entry_status;		/* Entry Status. */
 	uint32_t handle;		/* System handle. */
 	target_id_t target;		/* SCSI ID */
-	uint16_t lun;			/* SCSI LUN */
-	uint16_t control_flags;		/* Control flags. */
+	__le16	lun;			/* SCSI LUN */
+	__le16	control_flags;		/* Control flags. */
 	uint16_t reserved_1;
-	uint16_t timeout;		/* Command timeout. */
-	uint16_t dseg_count;		/* Data segment count. */
+	__le16	timeout;		/* Command timeout. */
+	__le16	dseg_count;		/* Data segment count. */
 	uint8_t scsi_cdb[MAX_CMDSZ];	/* SCSI command words. */
 	uint32_t byte_count;		/* Total byte count. */
 	struct dsd64 dsd[2];
@@ -1923,7 +1958,7 @@ struct crc_context {
 	__le16 guard_seed;		/* Initial Guard Seed */
 	__le16 prot_opts;		/* Requested Data Protection Mode */
 	__le16 blk_size;		/* Data size in bytes */
-	uint16_t runt_blk_guard;	/* Guard value for runt block (tape
+	__le16	runt_blk_guard;	/* Guard value for runt block (tape
 					 * only) */
 	__le32 byte_count;		/* Total byte count/ total data
 					 * transfer count */
@@ -1976,13 +2011,13 @@ typedef struct {
 	uint8_t sys_define;		/* System defined. */
 	uint8_t entry_status;		/* Entry Status. */
 	uint32_t handle;		/* System handle. */
-	uint16_t scsi_status;		/* SCSI status. */
-	uint16_t comp_status;		/* Completion status. */
-	uint16_t state_flags;		/* State flags. */
-	uint16_t status_flags;		/* Status flags. */
-	uint16_t rsp_info_len;		/* Response Info Length. */
-	uint16_t req_sense_length;	/* Request sense data length. */
-	uint32_t residual_length;	/* Residual transfer length. */
+	__le16	scsi_status;		/* SCSI status. */
+	__le16	comp_status;		/* Completion status. */
+	__le16	state_flags;		/* State flags. */
+	__le16	status_flags;		/* Status flags. */
+	__le16	rsp_info_len;		/* Response Info Length. */
+	__le16	req_sense_length;	/* Request sense data length. */
+	__le32	residual_length;	/* Residual transfer length. */
 	uint8_t rsp_info[8];		/* FCP response information. */
 	uint8_t req_sense_data[32];	/* Request sense data. */
 } sts_entry_t;
@@ -2114,8 +2149,8 @@ typedef struct {
 					/* clear port changed, */
 					/* use sequence number. */
 	uint8_t reserved_1;
-	uint16_t sequence_number;	/* Sequence number of event */
-	uint16_t lun;			/* SCSI LUN */
+	__le16	sequence_number;	/* Sequence number of event */
+	__le16	lun;			/* SCSI LUN */
 	uint8_t reserved_2[48];
 } mrk_entry_t;
 
@@ -2130,19 +2165,19 @@ typedef struct {
 	uint8_t entry_status;		/* Entry Status. */
 	uint32_t handle1;		/* System handle. */
 	target_id_t loop_id;
-	uint16_t status;
-	uint16_t control_flags;		/* Control flags. */
+	__le16	status;
+	__le16	control_flags;		/* Control flags. */
 	uint16_t reserved2;
-	uint16_t timeout;
-	uint16_t cmd_dsd_count;
-	uint16_t total_dsd_count;
+	__le16	timeout;
+	__le16	cmd_dsd_count;
+	__le16	total_dsd_count;
 	uint8_t type;
 	uint8_t r_ctl;
-	uint16_t rx_id;
+	__le16	rx_id;
 	uint16_t reserved3;
 	uint32_t handle2;
-	uint32_t rsp_bytecount;
-	uint32_t req_bytecount;
+	__le32	rsp_bytecount;
+	__le32	req_bytecount;
 	struct dsd64 req_dsd;
 	struct dsd64 rsp_dsd;
 } ms_iocb_entry_t;
@@ -2170,20 +2205,20 @@ struct mbx_entry {
 	uint32_t handle;
 	target_id_t loop_id;
 
-	uint16_t status;
-	uint16_t state_flags;
-	uint16_t status_flags;
+	__le16	status;
+	__le16	state_flags;
+	__le16	status_flags;
 
 	uint32_t sys_define2[2];
 
-	uint16_t mb0;
-	uint16_t mb1;
-	uint16_t mb2;
-	uint16_t mb3;
-	uint16_t mb6;
-	uint16_t mb7;
-	uint16_t mb9;
-	uint16_t mb10;
+	__le16	mb0;
+	__le16	mb1;
+	__le16	mb2;
+	__le16	mb3;
+	__le16	mb6;
+	__le16	mb7;
+	__le16	mb9;
+	__le16	mb10;
 	uint32_t reserved_2[2];
 	uint8_t node_name[WWN_SIZE];
 	uint8_t port_name[WWN_SIZE];
@@ -2205,52 +2240,52 @@ struct imm_ntfy_from_isp {
 	uint8_t	 entry_status;		    /* Entry Status. */
 	union {
 		struct {
-			uint32_t sys_define_2; /* System defined. */
+			__le32	sys_define_2; /* System defined. */
 			target_id_t target;
-			uint16_t lun;
+			__le16	lun;
 			uint8_t  target_id;
 			uint8_t  reserved_1;
-			uint16_t status_modifier;
-			uint16_t status;
-			uint16_t task_flags;
-			uint16_t seq_id;
-			uint16_t srr_rx_id;
-			uint32_t srr_rel_offs;
-			uint16_t srr_ui;
+			__le16	status_modifier;
+			__le16	status;
+			__le16	task_flags;
+			__le16	seq_id;
+			__le16	srr_rx_id;
+			__le32	srr_rel_offs;
+			__le16	srr_ui;
 #define SRR_IU_DATA_IN	0x1
 #define SRR_IU_DATA_OUT	0x5
 #define SRR_IU_STATUS	0x7
-			uint16_t srr_ox_id;
+			__le16	srr_ox_id;
 			uint8_t reserved_2[28];
 		} isp2x;
 		struct {
 			uint32_t reserved;
-			uint16_t nport_handle;
+			__le16	nport_handle;
 			uint16_t reserved_2;
-			uint16_t flags;
+			__le16	flags;
 #define NOTIFY24XX_FLAGS_GLOBAL_TPRLO   BIT_1
 #define NOTIFY24XX_FLAGS_PUREX_IOCB     BIT_0
-			uint16_t srr_rx_id;
-			uint16_t status;
+			__le16	srr_rx_id;
+			__le16	status;
 			uint8_t  status_subcode;
 			uint8_t  fw_handle;
-			uint32_t exchange_address;
-			uint32_t srr_rel_offs;
-			uint16_t srr_ui;
-			uint16_t srr_ox_id;
+			__le32	exchange_address;
+			__le32	srr_rel_offs;
+			__le16	srr_ui;
+			__le16	srr_ox_id;
 			union {
 				struct {
 					uint8_t node_name[8];
 				} plogi; /* PLOGI/ADISC/PDISC */
 				struct {
 					/* PRLI word 3 bit 0-15 */
-					uint16_t wd3_lo;
+					__le16	wd3_lo;
 					uint8_t resv0[6];
 				} prli;
 				struct {
 					uint8_t port_id[3];
 					uint8_t resv1;
-					uint16_t nport_handle;
+					__le16	nport_handle;
 					uint16_t resv2;
 				} req_els;
 			} u;
@@ -2263,7 +2298,7 @@ struct imm_ntfy_from_isp {
 		} isp24;
 	} u;
 	uint16_t reserved_7;
-	uint16_t ox_id;
+	__le16	ox_id;
 } __packed;
 #endif
 
@@ -2653,8 +2688,8 @@ static const char * const port_dstate_str[] = {
 #define FDMI_HBA_VENDOR_IDENTIFIER		0xe0
 
 struct ct_fdmi_hba_attr {
-	uint16_t type;
-	uint16_t len;
+	__be16	type;
+	__be16	len;
 	union {
 		uint8_t node_name[WWN_SIZE];
 		uint8_t manufacturer[64];
@@ -2666,11 +2701,11 @@ struct ct_fdmi_hba_attr {
 		uint8_t orom_version[16];
 		uint8_t fw_version[32];
 		uint8_t os_version[128];
-		uint32_t max_ct_len;
+		__be32	 max_ct_len;
 
 		uint8_t sym_name[256];
-		uint32_t vendor_specific_info;
-		uint32_t num_ports;
+		__be32	 vendor_specific_info;
+		__be32	 num_ports;
 		uint8_t fabric_name[WWN_SIZE];
 		uint8_t bios_name[32];
 		uint8_t vendor_identifier[8];
@@ -2678,12 +2713,12 @@ struct ct_fdmi_hba_attr {
 };
 
 struct ct_fdmi1_hba_attributes {
-	uint32_t count;
+	__be32	count;
 	struct ct_fdmi_hba_attr entry[FDMI1_HBA_ATTR_COUNT];
 };
 
 struct ct_fdmi2_hba_attributes {
-	uint32_t count;
+	__be32	count;
 	struct ct_fdmi_hba_attr entry[FDMI2_HBA_ATTR_COUNT];
 };
 
@@ -2735,44 +2770,44 @@ struct ct_fdmi2_hba_attributes {
 #define FC_CLASS_2_3	0x0C
 
 struct ct_fdmi_port_attr {
-	uint16_t type;
-	uint16_t len;
+	__be16	type;
+	__be16	len;
 	union {
 		uint8_t fc4_types[32];
-		uint32_t sup_speed;
-		uint32_t cur_speed;
-		uint32_t max_frame_size;
+		__be32	sup_speed;
+		__be32	cur_speed;
+		__be32	max_frame_size;
 		uint8_t os_dev_name[32];
 		uint8_t host_name[256];
 
 		uint8_t node_name[WWN_SIZE];
 		uint8_t port_name[WWN_SIZE];
 		uint8_t port_sym_name[128];
-		uint32_t port_type;
-		uint32_t port_supported_cos;
+		__be32	port_type;
+		__be32	port_supported_cos;
 		uint8_t fabric_name[WWN_SIZE];
 		uint8_t port_fc4_type[32];
-		uint32_t port_state;
-		uint32_t num_ports;
-		uint32_t port_id;
+		__be32	 port_state;
+		__be32	 num_ports;
+		__be32	 port_id;
 
 		uint8_t smartsan_service[24];
 		uint8_t smartsan_guid[16];
 		uint8_t smartsan_version[24];
 		uint8_t smartsan_prod_name[16];
-		uint32_t smartsan_port_info;
-		uint32_t smartsan_qos_support;
-		uint32_t smartsan_security_support;
+		__be32	 smartsan_port_info;
+		__be32	 smartsan_qos_support;
+		__be32	 smartsan_security_support;
 	} a;
 };
 
 struct ct_fdmi1_port_attributes {
-	uint32_t count;
+	__be32	 count;
 	struct ct_fdmi_port_attr entry[FDMI1_PORT_ATTR_COUNT];
 };
 
 struct ct_fdmi2_port_attributes {
-	uint32_t count;
+	__be32	count;
 	struct ct_fdmi_port_attr entry[FDMI2_PORT_ATTR_COUNT];
 };
 
@@ -2826,8 +2861,8 @@ struct ct_cmd_hdr {
 /* CT command request */
 struct ct_sns_req {
 	struct ct_cmd_hdr header;
-	uint16_t command;
-	uint16_t max_rsp_size;
+	__be16	command;
+	__be16	max_rsp_size;
 	uint8_t fragment_id;
 	uint8_t reserved[3];
 
@@ -2884,7 +2919,7 @@ struct ct_sns_req {
 
 		struct {
 			uint8_t hba_identifier[8];
-			uint32_t entry_count;
+			__be32	entry_count;
 			uint8_t port_name[8];
 			struct ct_fdmi2_hba_attributes attrs;
 		} rhba;
@@ -2939,7 +2974,7 @@ struct ct_sns_req {
 /* CT command response header */
 struct ct_rsp_hdr {
 	struct ct_cmd_hdr header;
-	uint16_t response;
+	__be16	response;
 	uint16_t residual;
 	uint8_t fragment_id;
 	uint8_t reason_code;
@@ -3025,8 +3060,8 @@ struct ct_sns_rsp {
 		} gfpn_id;
 
 		struct {
-			uint16_t speeds;
-			uint16_t speed;
+			__be16	speeds;
+			__be16	speed;
 		} gpsc;
 
 #define GFF_FCP_SCSI_OFFSET	7
@@ -3116,13 +3151,13 @@ struct fab_scan {
 struct sns_cmd_pkt {
 	union {
 		struct {
-			uint16_t buffer_length;
-			uint16_t reserved_1;
-			__le64	 buffer_address __packed;
-			uint16_t subcommand_length;
-			uint16_t reserved_2;
-			uint16_t subcommand;
-			uint16_t size;
+			__le16	buffer_length;
+			__le16	reserved_1;
+			__le64	buffer_address __packed;
+			__le16	subcommand_length;
+			__le16	reserved_2;
+			__le16	subcommand;
+			__le16	size;
 			uint32_t reserved_3;
 			uint8_t param[36];
 		} cmd;
@@ -3148,7 +3183,7 @@ struct gid_list_info {
 	uint8_t	area;
 	uint8_t	domain;
 	uint8_t	loop_id_2100;	/* ISP2100/ISP2200 -- 4 bytes. */
-	uint16_t loop_id;	/* ISP23XX         -- 6 bytes. */
+	__le16	loop_id;	/* ISP23XX         -- 6 bytes. */
 	uint16_t reserved_1;	/* ISP24XX         -- 8 bytes. */
 };
 
@@ -3222,7 +3257,8 @@ struct isp_operations {
 	int (*write_nvram)(struct scsi_qla_host *, void *, uint32_t,
 		uint32_t);
 
-	void (*fw_dump) (struct scsi_qla_host *, int);
+	void (*fw_dump)(struct scsi_qla_host *vha);
+	void (*mpi_fw_dump)(struct scsi_qla_host *, int);
 
 	int (*beacon_on) (struct scsi_qla_host *);
 	int (*beacon_off) (struct scsi_qla_host *);
@@ -3456,8 +3492,8 @@ struct rsp_que {
 	dma_addr_t  dma;
 	response_t *ring;
 	response_t *ring_ptr;
-	uint32_t __iomem *rsp_q_in;	/* FWI2-capable only. */
-	uint32_t __iomem *rsp_q_out;
+	__le32	__iomem *rsp_q_in;	/* FWI2-capable only. */
+	__le32	__iomem *rsp_q_out;
 	uint16_t  ring_index;
 	uint16_t  out_ptr;
 	uint16_t  *in_ptr;		/* queue shadow in index */
@@ -3483,8 +3519,8 @@ struct req_que {
 	dma_addr_t  dma;
 	request_t *ring;
 	request_t *ring_ptr;
-	uint32_t __iomem *req_q_in;	/* FWI2-capable only. */
-	uint32_t __iomem *req_q_out;
+	__le32	__iomem *req_q_in;	/* FWI2-capable only. */
+	__le32	__iomem *req_q_out;
 	uint16_t  ring_index;
 	uint16_t  in_ptr;
 	uint16_t  *out_ptr;		/* queue shadow out index */
@@ -3552,7 +3588,7 @@ struct qla_qpair {
 	struct list_head hints_list;
 	uint16_t cpuid;
 	uint16_t retry_term_cnt;
-	uint32_t retry_term_exchg_addr;
+	__le32	retry_term_exchg_addr;
 	uint64_t retry_term_jiff;
 	struct qla_tgt_counters tgt_counters;
 };
@@ -3579,98 +3615,98 @@ struct rdp_req_payload {
 
 struct rdp_rsp_payload {
 	struct {
-		uint32_t cmd;
-		uint32_t len;
+		__be32	cmd;
+		__be32	len;
 	} hdr;
 
 	/* LS Request Info descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
-		uint32_t req_payload_word_0;
+		__be32	desc_tag;
+		__be32	desc_len;
+		__be32	req_payload_word_0;
 	} ls_req_info_desc;
 
 	/* LS Request Info descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
-		uint32_t req_payload_word_0;
+		__be32	desc_tag;
+		__be32	desc_len;
+		__be32	req_payload_word_0;
 	} ls_req_info_desc2;
 
 	/* SFP diagnostic param descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
-		uint16_t temperature;
-		uint16_t vcc;
-		uint16_t tx_bias;
-		uint16_t tx_power;
-		uint16_t rx_power;
-		uint16_t sfp_flags;
+		__be32	desc_tag;
+		__be32	desc_len;
+		__be16	temperature;
+		__be16	vcc;
+		__be16	tx_bias;
+		__be16	tx_power;
+		__be16	rx_power;
+		__be16	sfp_flags;
 	} sfp_diag_desc;
 
 	/* Port Speed Descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
-		uint16_t speed_capab;
-		uint16_t operating_speed;
+		__be32	desc_tag;
+		__be32	desc_len;
+		__be16	speed_capab;
+		__be16	operating_speed;
 	} port_speed_desc;
 
 	/* Link Error Status Descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
-		uint32_t link_fail_cnt;
-		uint32_t loss_sync_cnt;
-		uint32_t loss_sig_cnt;
-		uint32_t prim_seq_err_cnt;
-		uint32_t inval_xmit_word_cnt;
-		uint32_t inval_crc_cnt;
+		__be32	desc_tag;
+		__be32	desc_len;
+		__be32	link_fail_cnt;
+		__be32	loss_sync_cnt;
+		__be32	loss_sig_cnt;
+		__be32	prim_seq_err_cnt;
+		__be32	inval_xmit_word_cnt;
+		__be32	inval_crc_cnt;
 		uint8_t  pn_port_phy_type;
 		uint8_t  reserved[3];
 	} ls_err_desc;
 
 	/* Port name description with diag param */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
+		__be32	desc_tag;
+		__be32	desc_len;
 		uint8_t WWNN[WWN_SIZE];
 		uint8_t WWPN[WWN_SIZE];
 	} port_name_diag_desc;
 
 	/* Port Name desc for Direct attached Fx_Port or Nx_Port */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
+		__be32	desc_tag;
+		__be32	desc_len;
 		uint8_t WWNN[WWN_SIZE];
 		uint8_t WWPN[WWN_SIZE];
 	} port_name_direct_desc;
 
 	/* Buffer Credit descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
-		uint32_t fcport_b2b;
-		uint32_t attached_fcport_b2b;
-		uint32_t fcport_rtt;
+		__be32	desc_tag;
+		__be32	desc_len;
+		__be32	fcport_b2b;
+		__be32	attached_fcport_b2b;
+		__be32	fcport_rtt;
 	} buffer_credit_desc;
 
 	/* Optical Element Data Descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
-		uint16_t high_alarm;
-		uint16_t low_alarm;
-		uint16_t high_warn;
-		uint16_t low_warn;
-		uint32_t element_flags;
+		__be32	desc_tag;
+		__be32	desc_len;
+		__be16	high_alarm;
+		__be16	low_alarm;
+		__be16	high_warn;
+		__be16	low_warn;
+		__be32	element_flags;
 	} optical_elmt_desc[5];
 
 	/* Optical Product Data Descriptor */
 	struct {
-		uint32_t desc_tag;
-		uint32_t desc_len;
+		__be32	desc_tag;
+		__be32	desc_len;
 		uint8_t  vendor_name[16];
 		uint8_t  part_number[16];
 		uint8_t  serial_number[16];
@@ -3708,17 +3744,17 @@ struct qlt_hw_data {
 	struct atio *atio_ring_ptr;	/* Current address. */
 	uint16_t atio_ring_index; /* Current index. */
 	uint16_t atio_q_length;
-	uint32_t __iomem *atio_q_in;
-	uint32_t __iomem *atio_q_out;
+	__le32 __iomem *atio_q_in;
+	__le32 __iomem *atio_q_out;
 
 	struct qla_tgt_func_tmpl *tgt_ops;
 	struct qla_tgt_vp_map *tgt_vp_map;
 
 	int saved_set;
-	uint16_t saved_exchange_count;
-	uint32_t saved_firmware_options_1;
-	uint32_t saved_firmware_options_2;
-	uint32_t saved_firmware_options_3;
+	__le16	saved_exchange_count;
+	__le32	saved_firmware_options_1;
+	__le32	saved_firmware_options_2;
+	__le32	saved_firmware_options_3;
 	uint8_t saved_firmware_options[2];
 	uint8_t saved_add_firmware_options[2];
 
@@ -3748,6 +3784,11 @@ struct qlt_hw_data {
 
 #define LEAK_EXCHG_THRESH_HOLD_PERCENT 75	/* 75 percent */
 
+struct qla_hw_data_stat {
+	u32 num_fw_dump;
+	u32 num_mpi_reset;
+};
+
 /*
  * Qlogic host adapter specific data structure.
 */
@@ -4212,7 +4253,7 @@ struct qla_hw_data {
 
 	uint16_t	fw_options[16];         /* slots: 1,2,3,10,11 */
 	uint8_t		fw_seriallink_options[4];
-	uint16_t	fw_seriallink_options24[4];
+	__le16		fw_seriallink_options24[4];
 
 	uint8_t		serdes_version[3];
 	uint8_t		mpi_version[3];
@@ -4230,7 +4271,6 @@ struct qla_hw_data {
 	uint32_t	fw_dump_len;
 	u32		fw_dump_alloc_len;
 	bool		fw_dumped;
-	bool		fw_dump_mpi;
 	unsigned long	fw_dump_cap_flags;
 #define RISC_PAUSE_CMPL		0
 #define DMA_SHUTDOWN_CMPL	1
@@ -4241,6 +4281,10 @@ struct qla_hw_data {
 #define ISP_MBX_RDY		6
 #define ISP_SOFT_RESET_CMPL	7
 	int		fw_dump_reading;
+	void		*mpi_fw_dump;
+	u32		mpi_fw_dump_len;
+	unsigned int	mpi_fw_dump_reading:1;
+	unsigned int	mpi_fw_dumped:1;
 	int		prev_minidump_failed;
 	dma_addr_t	eft_dma;
 	void		*eft;
@@ -4392,7 +4436,7 @@ struct qla_hw_data {
 #define NUM_DSD_CHAIN 4096
 
 	uint8_t fw_type;
-	__le32 file_prd_off;	/* File firmware product offset */
+	uint32_t file_prd_off;	/* File firmware product offset */
 
 	uint32_t	md_template_size;
 	void		*md_tmplt_hdr;
@@ -4454,6 +4498,8 @@ struct qla_hw_data {
 	uint16_t last_zio_threshold;
 
 #define DEFAULT_ZIO_THRESHOLD 5
+
+	struct qla_hw_data_stat stat;
 };
 
 struct active_regions {
@@ -4698,13 +4744,13 @@ typedef struct scsi_qla_host {
 
 struct qla27xx_image_status {
 	uint8_t image_status_mask;
-	uint16_t generation;
+	__le16	generation;
 	uint8_t ver_major;
 	uint8_t ver_minor;
 	uint8_t bitmap;		/* 28xx only */
 	uint8_t reserved[2];
-	uint32_t checksum;
-	uint32_t signature;
+	__le32	checksum;
+	__le32	signature;
 } __packed;
 
 /* 28xx aux image status bimap values */
diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h
index f9bad5bd7198..d1e12a29c3f7 100644
--- a/drivers/scsi/qla2xxx/qla_fw.h
+++ b/drivers/scsi/qla2xxx/qla_fw.h
@@ -134,28 +134,28 @@ struct vp_database_24xx {
 struct nvram_24xx {
 	/* NVRAM header. */
 	uint8_t id[4];
-	uint16_t nvram_version;
+	__le16	nvram_version;
 	uint16_t reserved_0;
 
 	/* Firmware Initialization Control Block. */
-	uint16_t version;
+	__le16	version;
 	uint16_t reserved_1;
-	__le16 frame_payload_size;
-	uint16_t execution_throttle;
-	uint16_t exchange_count;
-	uint16_t hard_address;
+	__le16	frame_payload_size;
+	__le16	execution_throttle;
+	__le16	exchange_count;
+	__le16	hard_address;
 
 	uint8_t port_name[WWN_SIZE];
 	uint8_t node_name[WWN_SIZE];
 
-	uint16_t login_retry_count;
-	uint16_t link_down_on_nos;
-	uint16_t interrupt_delay_timer;
-	uint16_t login_timeout;
+	__le16	login_retry_count;
+	__le16	link_down_on_nos;
+	__le16	interrupt_delay_timer;
+	__le16	login_timeout;
 
-	uint32_t firmware_options_1;
-	uint32_t firmware_options_2;
-	uint32_t firmware_options_3;
+	__le32	firmware_options_1;
+	__le32	firmware_options_2;
+	__le32	firmware_options_3;
 
 	/* Offset 56. */
 
@@ -178,7 +178,7 @@ struct nvram_24xx {
 	 * BIT 11-13 = Output Emphasis 4G
 	 * BIT 14-15 = Reserved
 	 */
-	uint16_t seriallink_options[4];
+	__le16	seriallink_options[4];
 
 	uint16_t reserved_2[16];
 
@@ -218,25 +218,25 @@ struct nvram_24xx {
 	 *
 	 * BIT 16-31 =
 	 */
-	uint32_t host_p;
+	__le32	host_p;
 
 	uint8_t alternate_port_name[WWN_SIZE];
 	uint8_t alternate_node_name[WWN_SIZE];
 
 	uint8_t boot_port_name[WWN_SIZE];
-	uint16_t boot_lun_number;
+	__le16	boot_lun_number;
 	uint16_t reserved_8;
 
 	uint8_t alt1_boot_port_name[WWN_SIZE];
-	uint16_t alt1_boot_lun_number;
+	__le16	alt1_boot_lun_number;
 	uint16_t reserved_9;
 
 	uint8_t alt2_boot_port_name[WWN_SIZE];
-	uint16_t alt2_boot_lun_number;
+	__le16	alt2_boot_lun_number;
 	uint16_t reserved_10;
 
 	uint8_t alt3_boot_port_name[WWN_SIZE];
-	uint16_t alt3_boot_lun_number;
+	__le16	alt3_boot_lun_number;
 	uint16_t reserved_11;
 
 	/*
@@ -249,23 +249,23 @@ struct nvram_24xx {
 	 * BIT 6 = Reserved
 	 * BIT 7-31 =
 	 */
-	uint32_t efi_parameters;
+	__le32	efi_parameters;
 
 	uint8_t reset_delay;
 	uint8_t reserved_12;
 	uint16_t reserved_13;
 
-	uint16_t boot_id_number;
+	__le16	boot_id_number;
 	uint16_t reserved_14;
 
-	uint16_t max_luns_per_target;
+	__le16	max_luns_per_target;
 	uint16_t reserved_15;
 
-	uint16_t port_down_retry_count;
-	uint16_t link_down_timeout;
+	__le16	port_down_retry_count;
+	__le16	link_down_timeout;
 
 	/* FCode parameters. */
-	uint16_t fcode_parameter;
+	__le16	fcode_parameter;
 
 	uint16_t reserved_16[3];
 
@@ -275,13 +275,13 @@ struct nvram_24xx {
 	uint8_t prev_drv_ver_minor;
 	uint8_t prev_drv_ver_subminor;
 
-	uint16_t prev_bios_ver_major;
-	uint16_t prev_bios_ver_minor;
+	__le16	prev_bios_ver_major;
+	__le16	prev_bios_ver_minor;
 
-	uint16_t prev_efi_ver_major;
-	uint16_t prev_efi_ver_minor;
+	__le16	prev_efi_ver_major;
+	__le16	prev_efi_ver_minor;
 
-	uint16_t prev_fw_ver_major;
+	__le16	prev_fw_ver_major;
 	uint8_t prev_fw_ver_minor;
 	uint8_t prev_fw_ver_subminor;
 
@@ -309,7 +309,7 @@ struct nvram_24xx {
 	uint16_t subsystem_vendor_id;
 	uint16_t subsystem_device_id;
 
-	uint32_t checksum;
+	__le32	checksum;
 };
 
 /*
@@ -318,46 +318,46 @@ struct nvram_24xx {
  */
 #define	ICB_VERSION 1
 struct init_cb_24xx {
-	uint16_t version;
+	__le16	version;
 	uint16_t reserved_1;
 
-	uint16_t frame_payload_size;
-	uint16_t execution_throttle;
-	uint16_t exchange_count;
+	__le16	frame_payload_size;
+	__le16	execution_throttle;
+	__le16	exchange_count;
 
-	uint16_t hard_address;
+	__le16	hard_address;
 
 	uint8_t port_name[WWN_SIZE];		/* Big endian. */
 	uint8_t node_name[WWN_SIZE];		/* Big endian. */
 
-	uint16_t response_q_inpointer;
-	uint16_t request_q_outpointer;
+	__le16	response_q_inpointer;
+	__le16	request_q_outpointer;
 
-	uint16_t login_retry_count;
+	__le16	login_retry_count;
 
-	uint16_t prio_request_q_outpointer;
+	__le16	prio_request_q_outpointer;
 
-	uint16_t response_q_length;
-	uint16_t request_q_length;
+	__le16	response_q_length;
+	__le16	request_q_length;
 
-	uint16_t link_down_on_nos;		/* Milliseconds. */
+	__le16	link_down_on_nos;		/* Milliseconds. */
 
-	uint16_t prio_request_q_length;
+	__le16	prio_request_q_length;
 
 	__le64	 request_q_address __packed;
 	__le64	 response_q_address __packed;
 	__le64	 prio_request_q_address __packed;
 
-	uint16_t msix;
-	uint16_t msix_atio;
+	__le16	msix;
+	__le16	msix_atio;
 	uint8_t reserved_2[4];
 
-	uint16_t atio_q_inpointer;
-	uint16_t atio_q_length;
-	__le64	 atio_q_address __packed;
+	__le16	atio_q_inpointer;
+	__le16	atio_q_length;
+	__le64	atio_q_address __packed;
 
-	uint16_t interrupt_delay_timer;		/* 100us increments. */
-	uint16_t login_timeout;
+	__le16	interrupt_delay_timer;		/* 100us increments. */
+	__le16	login_timeout;
 
 	/*
 	 * BIT 0  = Enable Hard Loop Id
@@ -378,7 +378,7 @@ struct init_cb_24xx {
 	 * BIT 14 = Node Name Option
 	 * BIT 15-31 = Reserved
 	 */
-	uint32_t firmware_options_1;
+	__le32	firmware_options_1;
 
 	/*
 	 * BIT 0  = Operation Mode bit 0
@@ -399,7 +399,7 @@ struct init_cb_24xx {
 	 * BIT 14 = Enable Target PRLI Control
 	 * BIT 15-31 = Reserved
 	 */
-	uint32_t firmware_options_2;
+	__le32	firmware_options_2;
 
 	/*
 	 * BIT 0  = Reserved
@@ -425,9 +425,9 @@ struct init_cb_24xx {
 	 * BIT 30 = Enable request queue 0 out index shadowing
 	 * BIT 31 = Reserved
 	 */
-	uint32_t firmware_options_3;
-	uint16_t qos;
-	uint16_t rid;
+	__le32	firmware_options_3;
+	__le16	 qos;
+	__le16	 rid;
 	uint8_t  reserved_3[20];
 };
 
@@ -443,27 +443,27 @@ struct cmd_bidir {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t nport_handle;		/* N_PORT hanlde. */
+	__le16	nport_handle;		/* N_PORT handle. */
 
-	uint16_t timeout;		/* Commnad timeout. */
+	__le16	timeout;		/* Command timeout. */
 
-	uint16_t wr_dseg_count;		/* Write Data segment count. */
-	uint16_t rd_dseg_count;		/* Read Data segment count. */
+	__le16	wr_dseg_count;		/* Write Data segment count. */
+	__le16	rd_dseg_count;		/* Read Data segment count. */
 
 	struct scsi_lun lun;		/* FCP LUN (BE). */
 
-	uint16_t control_flags;		/* Control flags. */
+	__le16	control_flags;		/* Control flags. */
 #define BD_WRAP_BACK			BIT_3
 #define BD_READ_DATA			BIT_1
 #define BD_WRITE_DATA			BIT_0
 
-	uint16_t fcp_cmnd_dseg_len;		/* Data segment length. */
+	__le16	fcp_cmnd_dseg_len;		/* Data segment length. */
 	__le64	 fcp_cmnd_dseg_address __packed;/* Data segment address. */
 
 	uint16_t reserved[2];			/* Reserved */
 
-	uint32_t rd_byte_count;			/* Total Byte count Read. */
-	uint32_t wr_byte_count;			/* Total Byte count write. */
+	__le32	rd_byte_count;			/* Total Byte count Read. */
+	__le32	wr_byte_count;			/* Total Byte count write. */
 
 	uint8_t port_id[3];			/* PortID of destination port.*/
 	uint8_t vp_index;
@@ -480,28 +480,28 @@ struct cmd_type_6 {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
-	uint16_t timeout;		/* Command timeout. */
+	__le16	nport_handle;		/* N_PORT handle. */
+	__le16	timeout;		/* Command timeout. */
 
-	uint16_t dseg_count;		/* Data segment count. */
+	__le16	dseg_count;		/* Data segment count. */
 
-	uint16_t fcp_rsp_dsd_len;	/* FCP_RSP DSD length. */
+	__le16	fcp_rsp_dsd_len;	/* FCP_RSP DSD length. */
 
 	struct scsi_lun lun;		/* FCP LUN (BE). */
 
-	uint16_t control_flags;		/* Control flags. */
+	__le16	control_flags;		/* Control flags. */
 #define CF_DIF_SEG_DESCR_ENABLE		BIT_3
 #define CF_DATA_SEG_DESCR_ENABLE	BIT_2
 #define CF_READ_DATA			BIT_1
 #define CF_WRITE_DATA			BIT_0
 
-	uint16_t fcp_cmnd_dseg_len;	/* Data segment length. */
+	__le16	fcp_cmnd_dseg_len;	/* Data segment length. */
 					/* Data segment address. */
 	__le64	 fcp_cmnd_dseg_address __packed;
 					/* Data segment address. */
 	__le64	 fcp_rsp_dseg_address __packed;
 
-	uint32_t byte_count;		/* Total byte count. */
+	__le32	byte_count;		/* Total byte count. */
 
 	uint8_t port_id[3];		/* PortID of destination port. */
 	uint8_t vp_index;
@@ -518,16 +518,16 @@ struct cmd_type_7 {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
-	uint16_t timeout;		/* Command timeout. */
+	__le16	nport_handle;		/* N_PORT handle. */
+	__le16	timeout;		/* Command timeout. */
 #define FW_MAX_TIMEOUT		0x1999
 
-	uint16_t dseg_count;		/* Data segment count. */
+	__le16	dseg_count;		/* Data segment count. */
 	uint16_t reserved_1;
 
 	struct scsi_lun lun;		/* FCP LUN (BE). */
 
-	uint16_t task_mgmt_flags;	/* Task management flags. */
+	__le16	task_mgmt_flags;	/* Task management flags. */
 #define TMF_CLEAR_ACA		BIT_14
 #define TMF_TARGET_RESET	BIT_13
 #define TMF_LUN_RESET		BIT_12
@@ -547,7 +547,7 @@ struct cmd_type_7 {
 	uint8_t crn;
 
 	uint8_t fcp_cdb[MAX_CMDSZ]; 	/* SCSI command words. */
-	uint32_t byte_count;		/* Total byte count. */
+	__le32	byte_count;		/* Total byte count. */
 
 	uint8_t port_id[3];		/* PortID of destination port. */
 	uint8_t vp_index;
@@ -565,29 +565,29 @@ struct cmd_type_crc_2 {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
-	uint16_t timeout;		/* Command timeout. */
+	__le16	nport_handle;		/* N_PORT handle. */
+	__le16	timeout;		/* Command timeout. */
 
-	uint16_t dseg_count;		/* Data segment count. */
+	__le16	dseg_count;		/* Data segment count. */
 
-	uint16_t fcp_rsp_dseg_len;	/* FCP_RSP DSD length. */
+	__le16	fcp_rsp_dseg_len;	/* FCP_RSP DSD length. */
 
 	struct scsi_lun lun;		/* FCP LUN (BE). */
 
-	uint16_t control_flags;		/* Control flags. */
+	__le16	control_flags;		/* Control flags. */
 
-	uint16_t fcp_cmnd_dseg_len;	/* Data segment length. */
+	__le16	fcp_cmnd_dseg_len;	/* Data segment length. */
 	__le64	 fcp_cmnd_dseg_address __packed;
 					/* Data segment address. */
 	__le64	 fcp_rsp_dseg_address __packed;
 
-	uint32_t byte_count;		/* Total byte count. */
+	__le32	byte_count;		/* Total byte count. */
 
 	uint8_t port_id[3];		/* PortID of destination port. */
 	uint8_t vp_index;
 
 	__le64	 crc_context_address __packed;	/* Data segment address. */
-	uint16_t crc_context_len;		/* Data segment length. */
+	__le16	crc_context_len;		/* Data segment length. */
 	uint16_t reserved_1;			/* MUST be set to 0. */
 };
 
@@ -604,32 +604,32 @@ struct sts_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t comp_status;		/* Completion status. */
-	uint16_t ox_id;			/* OX_ID used by the firmware. */
+	__le16	comp_status;		/* Completion status. */
+	__le16	ox_id;			/* OX_ID used by the firmware. */
 
-	uint32_t residual_len;		/* FW calc residual transfer length. */
+	__le32	residual_len;		/* FW calc residual transfer length. */
 
 	union {
 		uint16_t reserved_1;
-		uint16_t nvme_rsp_pyld_len;
+		__le16	nvme_rsp_pyld_len;
 	};
 
-	uint16_t state_flags;		/* State flags. */
+	__le16	state_flags;		/* State flags. */
 #define SF_TRANSFERRED_DATA	BIT_11
 #define SF_NVME_ERSP            BIT_6
 #define SF_FCP_RSP_DMA		BIT_0
 
-	uint16_t retry_delay;
-	uint16_t scsi_status;		/* SCSI status. */
+	__le16	retry_delay;
+	__le16	scsi_status;		/* SCSI status. */
 #define SS_CONFIRMATION_REQ		BIT_12
 
-	uint32_t rsp_residual_count;	/* FCP RSP residual count. */
+	__le32	rsp_residual_count;	/* FCP RSP residual count. */
 
-	uint32_t sense_len;		/* FCP SENSE length. */
+	__le32	sense_len;		/* FCP SENSE length. */
 
 	union {
 		struct {
-			uint32_t rsp_data_len;	/* FCP response data length  */
+			__le32	rsp_data_len;	/* FCP response data length  */
 			uint8_t data[28];	/* FCP rsp/sense information */
 		};
 		struct nvme_fc_ersp_iu nvme_ersp;
@@ -672,7 +672,7 @@ struct mrk_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
+	__le16	nport_handle;		/* N_PORT handle. */
 
 	uint8_t modifier;		/* Modifier (7-0). */
 #define MK_SYNC_ID_LUN	0		/* Synchronize ID/LUN */
@@ -701,24 +701,24 @@ struct ct_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t comp_status;		/* Completion status. */
+	__le16	comp_status;		/* Completion status. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
+	__le16	nport_handle;		/* N_PORT handle. */
 
-	uint16_t cmd_dsd_count;
+	__le16	cmd_dsd_count;
 
 	uint8_t vp_index;
 	uint8_t reserved_1;
 
-	uint16_t timeout;		/* Command timeout. */
+	__le16	timeout;		/* Command timeout. */
 	uint16_t reserved_2;
 
-	uint16_t rsp_dsd_count;
+	__le16	rsp_dsd_count;
 
 	uint8_t reserved_3[10];
 
-	uint32_t rsp_byte_count;
-	uint32_t cmd_byte_count;
+	__le32	rsp_byte_count;
+	__le32	cmd_byte_count;
 
 	struct dsd64 dsd[2];
 };
@@ -733,17 +733,17 @@ struct purex_entry_24xx {
 	uint8_t sys_define;		/* System defined. */
 	uint8_t entry_status;		/* Entry Status. */
 
-	uint16_t reserved1;
+	__le16	reserved1;
 	uint8_t vp_idx;
 	uint8_t reserved2;
 
-	uint16_t status_flags;
-	uint16_t nport_handle;
+	__le16	status_flags;
+	__le16	nport_handle;
 
-	uint16_t frame_size;
-	uint16_t trunc_frame_size;
+	__le16	frame_size;
+	__le16	trunc_frame_size;
 
-	uint32_t rx_xchg_addr;
+	__le32	rx_xchg_addr;
 
 	uint8_t d_id[3];
 	uint8_t r_ctl;
@@ -754,13 +754,13 @@ struct purex_entry_24xx {
 	uint8_t f_ctl[3];
 	uint8_t type;
 
-	uint16_t seq_cnt;
+	__le16	seq_cnt;
 	uint8_t df_ctl;
 	uint8_t seq_id;
 
-	uint16_t rx_id;
-	uint16_t ox_id;
-	uint32_t param;
+	__le16	rx_id;
+	__le16	ox_id;
+	__le32	param;
 
 	uint8_t els_frame_payload[20];
 };
@@ -777,18 +777,18 @@ struct els_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t comp_status;		/* response only */
-	uint16_t nport_handle;
+	__le16	comp_status;		/* response only */
+	__le16	nport_handle;
 
-	uint16_t tx_dsd_count;
+	__le16	tx_dsd_count;
 
 	uint8_t vp_index;
 	uint8_t sof_type;
 #define EST_SOFI3		(1 << 4)
 #define EST_SOFI2		(3 << 4)
 
-	uint32_t rx_xchg_address;	/* Receive exchange address. */
-	uint16_t rx_dsd_count;
+	__le32	rx_xchg_address;	/* Receive exchange address. */
+	__le16	rx_dsd_count;
 
 	uint8_t opcode;
 	uint8_t reserved_2;
@@ -796,7 +796,7 @@ struct els_entry_24xx {
 	uint8_t d_id[3];
 	uint8_t s_id[3];
 
-	uint16_t control_flags;		/* Control flags. */
+	__le16	control_flags;		/* Control flags. */
 #define ECF_PAYLOAD_DESCR_MASK	(BIT_15|BIT_14|BIT_13)
 #define EPD_ELS_COMMAND		(0 << 13)
 #define EPD_ELS_ACC		(1 << 13)
@@ -817,10 +817,10 @@ struct els_entry_24xx {
 			__le32	 rx_len;		/* DSD 1 length. */
 		};
 		struct {
-			uint32_t total_byte_count;
-			uint32_t error_subcode_1;
-			uint32_t error_subcode_2;
-			uint32_t error_subcode_3;
+			__le32	total_byte_count;
+			__le32	error_subcode_1;
+			__le32	error_subcode_2;
+			__le32	error_subcode_3;
 		};
 	};
 };
@@ -831,19 +831,19 @@ struct els_sts_entry_24xx {
 	uint8_t sys_define;		/* System Defined. */
 	uint8_t entry_status;		/* Entry Status. */
 
-	uint32_t handle;		/* System handle. */
+	__le32	handle;		/* System handle. */
 
-	uint16_t comp_status;
+	__le16	comp_status;
 
-	uint16_t nport_handle;		/* N_PORT handle. */
+	__le16	nport_handle;		/* N_PORT handle. */
 
-	uint16_t reserved_1;
+	__le16	reserved_1;
 
 	uint8_t vp_index;
 	uint8_t sof_type;
 
-	uint32_t rx_xchg_address;	/* Receive exchange address. */
-	uint16_t reserved_2;
+	__le32	rx_xchg_address;	/* Receive exchange address. */
+	__le16	reserved_2;
 
 	uint8_t opcode;
 	uint8_t reserved_3;
@@ -851,13 +851,13 @@ struct els_sts_entry_24xx {
 	uint8_t d_id[3];
 	uint8_t s_id[3];
 
-	uint16_t control_flags;		/* Control flags. */
-	uint32_t total_byte_count;
-	uint32_t error_subcode_1;
-	uint32_t error_subcode_2;
-	uint32_t error_subcode_3;
+	__le16	control_flags;		/* Control flags. */
+	__le32	total_byte_count;
+	__le32	error_subcode_1;
+	__le32	error_subcode_2;
+	__le32	error_subcode_3;
 
-	uint32_t reserved_4[4];
+	__le32	reserved_4[4];
 };
 /*
  * ISP queue - Mailbox Command entry structure definition.
@@ -884,12 +884,12 @@ struct logio_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t comp_status;		/* Completion status. */
+	__le16	comp_status;		/* Completion status. */
 #define CS_LOGIO_ERROR		0x31	/* Login/Logout IOCB error. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
+	__le16	nport_handle;		/* N_PORT handle. */
 
-	uint16_t control_flags;		/* Control flags. */
+	__le16	control_flags;		/* Control flags. */
 					/* Modifiers. */
 #define LCF_INCLUDE_SNS		BIT_10	/* Include SNS (FFFFFC) during LOGO. */
 #define LCF_FCP2_OVERRIDE	BIT_9	/* Set/Reset word 3 of PRLI. */
@@ -918,7 +918,7 @@ struct logio_entry_24xx {
 
 	uint8_t rsp_size;		/* Response size in 32bit words. */
 
-	uint32_t io_parameter[11];	/* General I/O parameters. */
+	__le32	io_parameter[11];	/* General I/O parameters. */
 #define LSC_SCODE_NOLINK	0x01
 #define LSC_SCODE_NOIOCB	0x02
 #define LSC_SCODE_NOXCB		0x03
@@ -946,17 +946,17 @@ struct tsk_mgmt_entry {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
+	__le16	nport_handle;		/* N_PORT handle. */
 
 	uint16_t reserved_1;
 
-	uint16_t delay;			/* Activity delay in seconds. */
+	__le16	delay;			/* Activity delay in seconds. */
 
-	uint16_t timeout;		/* Command timeout. */
+	__le16	timeout;		/* Command timeout. */
 
 	struct scsi_lun lun;		/* FCP LUN (BE). */
 
-	uint32_t control_flags;		/* Control Flags. */
+	__le32	control_flags;		/* Control Flags. */
 #define TCF_NOTMCMD_TO_TARGET	BIT_31
 #define TCF_LUN_RESET		BIT_4
 #define TCF_ABORT_TASK_SET	BIT_3
@@ -981,15 +981,15 @@ struct abort_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t nport_handle;		/* N_PORT handle. */
+	__le16	nport_handle;		/* N_PORT handle. */
 					/* or Completion status. */
 
-	uint16_t options;		/* Options. */
+	__le16	options;		/* Options. */
 #define AOF_NO_ABTS		BIT_0	/* Do not send any ABTS. */
 
 	uint32_t handle_to_abort;	/* System handle to abort. */
 
-	uint16_t req_que_no;
+	__le16	req_que_no;
 	uint8_t reserved_1[30];
 
 	uint8_t port_id[3];		/* PortID of destination port. */
@@ -1006,16 +1006,16 @@ struct abts_entry_24xx {
 	uint8_t handle_count;
 	uint8_t entry_status;
 
-	uint32_t handle;		/* type 0x55 only */
+	__le32	handle;		/* type 0x55 only */
 
-	uint16_t comp_status;		/* type 0x55 only */
-	uint16_t nport_handle;		/* type 0x54 only */
+	__le16	comp_status;		/* type 0x55 only */
+	__le16	nport_handle;		/* type 0x54 only */
 
-	uint16_t control_flags;		/* type 0x55 only */
+	__le16	control_flags;		/* type 0x55 only */
 	uint8_t vp_idx;
 	uint8_t sof_type;		/* sof_type is upper nibble */
 
-	uint32_t rx_xch_addr;
+	__le32	rx_xch_addr;
 
 	uint8_t d_id[3];
 	uint8_t r_ctl;
@@ -1026,30 +1026,30 @@ struct abts_entry_24xx {
 	uint8_t f_ctl[3];
 	uint8_t type;
 
-	uint16_t seq_cnt;
+	__le16	seq_cnt;
 	uint8_t df_ctl;
 	uint8_t seq_id;
 
-	uint16_t rx_id;
-	uint16_t ox_id;
+	__le16	rx_id;
+	__le16	ox_id;
 
-	uint32_t param;
+	__le32	param;
 
 	union {
 		struct {
-			uint32_t subcode3;
-			uint32_t rsvd;
-			uint32_t subcode1;
-			uint32_t subcode2;
+			__le32	subcode3;
+			__le32	rsvd;
+			__le32	subcode1;
+			__le32	subcode2;
 		} error;
 		struct {
-			uint16_t rsrvd1;
+			__le16	rsrvd1;
 			uint8_t last_seq_id;
 			uint8_t seq_id_valid;
-			uint16_t aborted_rx_id;
-			uint16_t aborted_ox_id;
-			uint16_t high_seq_cnt;
-			uint16_t low_seq_cnt;
+			__le16	aborted_rx_id;
+			__le16	aborted_ox_id;
+			__le16	high_seq_cnt;
+			__le16	low_seq_cnt;
 		} ba_acc;
 		struct {
 			uint8_t vendor_unique;
@@ -1058,7 +1058,7 @@ struct abts_entry_24xx {
 		} ba_rjt;
 	} payload;
 
-	uint32_t rx_xch_addr_to_abort;
+	__le32	rx_xch_addr_to_abort;
 } __packed;
 
 /* ABTS payload explanation values */
@@ -1087,7 +1087,7 @@ struct abts_entry_24xx {
  * ISP I/O Register Set structure definitions.
  */
 struct device_reg_24xx {
-	uint32_t flash_addr;		/* Flash/NVRAM BIOS address. */
+	__le32	flash_addr;		/* Flash/NVRAM BIOS address. */
 #define FARX_DATA_FLAG	BIT_31
 #define FARX_ACCESS_FLASH_CONF	0x7FFD0000
 #define FARX_ACCESS_FLASH_DATA	0x7FF00000
@@ -1138,9 +1138,9 @@ struct device_reg_24xx {
 #define HW_EVENT_NVRAM_CHKSUM_ERR	0xF023
 #define HW_EVENT_FLASH_FW_ERR	0xF024
 
-	uint32_t flash_data;		/* Flash/NVRAM BIOS data. */
+	__le32	flash_data;		/* Flash/NVRAM BIOS data. */
 
-	uint32_t ctrl_status;		/* Control/Status. */
+	__le32	ctrl_status;		/* Control/Status. */
 #define CSRX_FLASH_ACCESS_ERROR	BIT_18	/* Flash/NVRAM Access Error. */
 #define CSRX_DMA_ACTIVE		BIT_17	/* DMA Active status. */
 #define CSRX_DMA_SHUTDOWN	BIT_16	/* DMA Shutdown control status. */
@@ -1166,35 +1166,35 @@ struct device_reg_24xx {
 #define CSRX_FLASH_ENABLE	BIT_1	/* Flash BIOS Read/Write enable. */
 #define CSRX_ISP_SOFT_RESET	BIT_0	/* ISP soft reset. */
 
-	uint32_t ictrl;			/* Interrupt control. */
+	__le32	ictrl;			/* Interrupt control. */
 #define ICRX_EN_RISC_INT	BIT_3	/* Enable RISC interrupts on PCI. */
 
-	uint32_t istatus;		/* Interrupt status. */
+	__le32	istatus;		/* Interrupt status. */
 #define ISRX_RISC_INT		BIT_3	/* RISC interrupt. */
 
-	uint32_t unused_1[2];		/* Gap. */
+	__le32	unused_1[2];		/* Gap. */
 
 					/* Request Queue. */
-	uint32_t req_q_in;		/*  In-Pointer. */
-	uint32_t req_q_out;		/*  Out-Pointer. */
+	__le32	req_q_in;		/*  In-Pointer. */
+	__le32	req_q_out;		/*  Out-Pointer. */
 					/* Response Queue. */
-	uint32_t rsp_q_in;		/*  In-Pointer. */
-	uint32_t rsp_q_out;		/*  Out-Pointer. */
+	__le32	rsp_q_in;		/*  In-Pointer. */
+	__le32	rsp_q_out;		/*  Out-Pointer. */
 					/* Priority Request Queue. */
-	uint32_t preq_q_in;		/*  In-Pointer. */
-	uint32_t preq_q_out;		/*  Out-Pointer. */
+	__le32	preq_q_in;		/*  In-Pointer. */
+	__le32	preq_q_out;		/*  Out-Pointer. */
 
-	uint32_t unused_2[2];		/* Gap. */
+	__le32	unused_2[2];		/* Gap. */
 
 					/* ATIO Queue. */
-	uint32_t atio_q_in;		/*  In-Pointer. */
-	uint32_t atio_q_out;		/*  Out-Pointer. */
+	__le32	atio_q_in;		/*  In-Pointer. */
+	__le32	atio_q_out;		/*  Out-Pointer. */
 
-	uint32_t host_status;
+	__le32	host_status;
 #define HSRX_RISC_INT		BIT_15	/* RISC to Host interrupt. */
 #define HSRX_RISC_PAUSED	BIT_8	/* RISC Paused. */
 
-	uint32_t hccr;			/* Host command & control register. */
+	__le32	hccr;			/* Host command & control register. */
 					/* HCCR statuses. */
 #define HCCRX_HOST_INT		BIT_6	/* Host to RISC interrupt bit. */
 #define HCCRX_RISC_RESET	BIT_5	/* RISC Reset mode bit. */
@@ -1216,7 +1216,7 @@ struct device_reg_24xx {
 					/* Clear RISC to PCI interrupt. */
 #define HCCRX_CLR_RISC_INT	0xA0000000
 
-	uint32_t gpiod;			/* GPIO Data register. */
+	__le32	gpiod;			/* GPIO Data register. */
 
 					/* LED update mask. */
 #define GPDX_LED_UPDATE_MASK	(BIT_20|BIT_19|BIT_18)
@@ -1235,7 +1235,7 @@ struct device_reg_24xx {
 					/* Data in/out. */
 #define GPDX_DATA_INOUT		(BIT_1|BIT_0)
 
-	uint32_t gpioe;			/* GPIO Enable register. */
+	__le32	gpioe;			/* GPIO Enable register. */
 					/* Enable update mask. */
 #define GPEX_ENABLE_UPDATE_MASK	(BIT_17|BIT_16)
 					/* Enable update mask. */
@@ -1243,56 +1243,56 @@ struct device_reg_24xx {
 					/* Enable. */
 #define GPEX_ENABLE		(BIT_1|BIT_0)
 
-	uint32_t iobase_addr;		/* I/O Bus Base Address register. */
-
-	uint32_t unused_3[10];		/* Gap. */
-
-	uint16_t mailbox0;
-	uint16_t mailbox1;
-	uint16_t mailbox2;
-	uint16_t mailbox3;
-	uint16_t mailbox4;
-	uint16_t mailbox5;
-	uint16_t mailbox6;
-	uint16_t mailbox7;
-	uint16_t mailbox8;
-	uint16_t mailbox9;
-	uint16_t mailbox10;
-	uint16_t mailbox11;
-	uint16_t mailbox12;
-	uint16_t mailbox13;
-	uint16_t mailbox14;
-	uint16_t mailbox15;
-	uint16_t mailbox16;
-	uint16_t mailbox17;
-	uint16_t mailbox18;
-	uint16_t mailbox19;
-	uint16_t mailbox20;
-	uint16_t mailbox21;
-	uint16_t mailbox22;
-	uint16_t mailbox23;
-	uint16_t mailbox24;
-	uint16_t mailbox25;
-	uint16_t mailbox26;
-	uint16_t mailbox27;
-	uint16_t mailbox28;
-	uint16_t mailbox29;
-	uint16_t mailbox30;
-	uint16_t mailbox31;
-
-	uint32_t iobase_window;
-	uint32_t iobase_c4;
-	uint32_t iobase_c8;
-	uint32_t unused_4_1[6];		/* Gap. */
-	uint32_t iobase_q;
-	uint32_t unused_5[2];		/* Gap. */
-	uint32_t iobase_select;
-	uint32_t unused_6[2];		/* Gap. */
-	uint32_t iobase_sdata;
+	__le32	iobase_addr;		/* I/O Bus Base Address register. */
+
+	__le32	unused_3[10];		/* Gap. */
+
+	__le16	mailbox0;
+	__le16	mailbox1;
+	__le16	mailbox2;
+	__le16	mailbox3;
+	__le16	mailbox4;
+	__le16	mailbox5;
+	__le16	mailbox6;
+	__le16	mailbox7;
+	__le16	mailbox8;
+	__le16	mailbox9;
+	__le16	mailbox10;
+	__le16	mailbox11;
+	__le16	mailbox12;
+	__le16	mailbox13;
+	__le16	mailbox14;
+	__le16	mailbox15;
+	__le16	mailbox16;
+	__le16	mailbox17;
+	__le16	mailbox18;
+	__le16	mailbox19;
+	__le16	mailbox20;
+	__le16	mailbox21;
+	__le16	mailbox22;
+	__le16	mailbox23;
+	__le16	mailbox24;
+	__le16	mailbox25;
+	__le16	mailbox26;
+	__le16	mailbox27;
+	__le16	mailbox28;
+	__le16	mailbox29;
+	__le16	mailbox30;
+	__le16	mailbox31;
+
+	__le32	iobase_window;
+	__le32	iobase_c4;
+	__le32	iobase_c8;
+	__le32	unused_4_1[6];		/* Gap. */
+	__le32	iobase_q;
+	__le32	unused_5[2];		/* Gap. */
+	__le32	iobase_select;
+	__le32	unused_6[2];		/* Gap. */
+	__le32	iobase_sdata;
 };
 /* RISC-RISC semaphore register PCI offet */
 #define RISC_REGISTER_BASE_OFFSET	0x7010
-#define RISC_REGISTER_WINDOW_OFFET	0x6
+#define RISC_REGISTER_WINDOW_OFFSET	0x6
 
 /* RISC-RISC semaphore/flag register (risc address 0x7016) */
 
@@ -1354,8 +1354,8 @@ struct mid_conf_entry_24xx {
 struct mid_init_cb_24xx {
 	struct init_cb_24xx init_cb;
 
-	uint16_t count;
-	uint16_t options;
+	__le16	count;
+	__le16	options;
 
 	struct mid_conf_entry_24xx entries[MAX_MULTI_ID_FABRIC];
 };
@@ -1389,27 +1389,27 @@ struct vp_ctrl_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t vp_idx_failed;
+	__le16	vp_idx_failed;
 
-	uint16_t comp_status;		/* Completion status. */
+	__le16	comp_status;		/* Completion status. */
 #define CS_VCE_IOCB_ERROR       0x01    /* Error processing IOCB */
 #define CS_VCE_ACQ_ID_ERROR	0x02	/* Error while acquireing ID. */
 #define CS_VCE_BUSY		0x05	/* Firmware not ready to accept cmd. */
 
-	uint16_t command;
+	__le16	command;
 #define VCE_COMMAND_ENABLE_VPS	0x00	/* Enable VPs. */
 #define VCE_COMMAND_DISABLE_VPS	0x08	/* Disable VPs. */
 #define VCE_COMMAND_DISABLE_VPS_REINIT	0x09 /* Disable VPs and reinit link. */
 #define VCE_COMMAND_DISABLE_VPS_LOGO	0x0a /* Disable VPs and LOGO ports. */
 #define VCE_COMMAND_DISABLE_VPS_LOGO_ALL        0x0b /* Disable VPs and LOGO ports. */
 
-	uint16_t vp_count;
+	__le16	vp_count;
 
 	uint8_t vp_idx_map[16];
-	uint16_t flags;
-	uint16_t id;
+	__le16	flags;
+	__le16	id;
 	uint16_t reserved_4;
-	uint16_t hopct;
+	__le16	hopct;
 	uint8_t reserved_5[24];
 };
 
@@ -1425,12 +1425,12 @@ struct vp_config_entry_24xx {
 
 	uint32_t handle;		/* System handle. */
 
-	uint16_t flags;
+	__le16	flags;
 #define CS_VF_BIND_VPORTS_TO_VF         BIT_0
 #define CS_VF_SET_QOS_OF_VPORTS         BIT_1
 #define CS_VF_SET_HOPS_OF_VPORTS        BIT_2
 
-	uint16_t comp_status;		/* Completion status. */
+	__le16	comp_status;		/* Completion status. */
 #define CS_VCT_STS_ERROR	0x01	/* Specified VPs were not disabled. */
 #define CS_VCT_CNT_ERROR	0x02	/* Invalid VP count. */
 #define CS_VCT_ERROR		0x03	/* Unknown error. */
@@ -1457,9 +1457,9 @@ struct vp_config_entry_24xx {
 	uint16_t reserved_vp2;
 	uint8_t port_name_idx2[WWN_SIZE];
 	uint8_t node_name_idx2[WWN_SIZE];
-	uint16_t id;
+	__le16	id;
 	uint16_t reserved_4;
-	uint16_t hopct;
+	__le16	hopct;
 	uint8_t reserved_5[2];
 };
 
@@ -1486,7 +1486,7 @@ struct vp_rpt_id_entry_24xx {
 	uint8_t entry_count;		/* Entry count. */
 	uint8_t sys_define;		/* System defined. */
 	uint8_t entry_status;		/* Entry Status. */
-	uint32_t resv1;
+	__le32 resv1;
 	uint8_t vp_acquired;
 	uint8_t vp_setup;
 	uint8_t vp_idx;		/* Format 0=reserved */
@@ -1550,15 +1550,15 @@ struct vf_evfp_entry_24xx {
         uint8_t entry_status;           /* Entry Status. */
 
         uint32_t handle;                /* System handle. */
-        uint16_t comp_status;           /* Completion status. */
-        uint16_t timeout;               /* timeout */
-        uint16_t adim_tagging_mode;
+        __le16	comp_status;           /* Completion status. */
+        __le16	timeout;               /* timeout */
+        __le16	adim_tagging_mode;
 
-        uint16_t vfport_id;
+        __le16	vfport_id;
         uint32_t exch_addr;
 
-        uint16_t nport_handle;          /* N_PORT handle. */
-        uint16_t control_flags;
+        __le16	nport_handle;          /* N_PORT handle. */
+        __le16	control_flags;
         uint32_t io_parameter_0;
         uint32_t io_parameter_1;
 	__le64	 tx_address __packed;	/* Data segment 0 address. */
@@ -1573,13 +1573,13 @@ struct vf_evfp_entry_24xx {
 
 struct qla_fdt_layout {
 	uint8_t sig[4];
-	uint16_t version;
-	uint16_t len;
-	uint16_t checksum;
+	__le16	version;
+	__le16	len;
+	__le16	checksum;
 	uint8_t unused1[2];
 	uint8_t model[16];
-	uint16_t man_id;
-	uint16_t id;
+	__le16	man_id;
+	__le16	id;
 	uint8_t flags;
 	uint8_t erase_cmd;
 	uint8_t alt_erase_cmd;
@@ -1588,15 +1588,15 @@ struct qla_fdt_layout {
 	uint8_t wrt_sts_reg_cmd;
 	uint8_t unprotect_sec_cmd;
 	uint8_t read_man_id_cmd;
-	uint32_t block_size;
-	uint32_t alt_block_size;
-	uint32_t flash_size;
-	uint32_t wrt_enable_data;
+	__le32 block_size;
+	__le32 alt_block_size;
+	__le32 flash_size;
+	__le32 wrt_enable_data;
 	uint8_t read_id_addr_len;
 	uint8_t wrt_disable_bits;
 	uint8_t read_dev_id_len;
 	uint8_t chip_erase_cmd;
-	uint16_t read_timeout;
+	__le16	read_timeout;
 	uint8_t protect_sec_cmd;
 	uint8_t unused2[65];
 };
@@ -1605,11 +1605,11 @@ struct qla_fdt_layout {
 
 struct qla_flt_location {
 	uint8_t sig[4];
-	uint16_t start_lo;
-	uint16_t start_hi;
+	__le16	start_lo;
+	__le16	start_hi;
 	uint8_t version;
 	uint8_t unused[5];
-	uint16_t checksum;
+	__le16	checksum;
 };
 
 #define FLT_REG_FW		0x01
@@ -1664,19 +1664,19 @@ struct qla_flt_location {
 #define FLT_REG_PEP_SEC_28XX		0xF1
 
 struct qla_flt_region {
-	uint16_t code;
+	__le16	code;
 	uint8_t attribute;
 	uint8_t reserved;
-	uint32_t size;
-	uint32_t start;
-	uint32_t end;
+	__le32 size;
+	__le32 start;
+	__le32 end;
 };
 
 struct qla_flt_header {
-	uint16_t version;
-	uint16_t length;
-	uint16_t checksum;
-	uint16_t unused;
+	__le16	version;
+	__le16	length;
+	__le16	checksum;
+	__le16	unused;
 	struct qla_flt_region region[0];
 };
 
@@ -1688,18 +1688,18 @@ struct qla_flt_header {
 
 struct qla_npiv_header {
 	uint8_t sig[2];
-	uint16_t version;
-	uint16_t entries;
-	uint16_t unused[4];
-	uint16_t checksum;
+	__le16	version;
+	__le16	entries;
+	__le16	unused[4];
+	__le16	checksum;
 };
 
 struct qla_npiv_entry {
-	uint16_t flags;
-	uint16_t vf_id;
+	__le16	flags;
+	__le16	vf_id;
 	uint8_t q_qos;
 	uint8_t f_qos;
-	uint16_t unused1;
+	__le16	unused1;
 	uint8_t port_name[WWN_SIZE];
 	uint8_t node_name[WWN_SIZE];
 };
@@ -1729,7 +1729,7 @@ struct verify_chip_entry_84xx {
 
 	uint32_t handle;
 
-	uint16_t options;
+	__le16	options;
 #define VCO_DONT_UPDATE_FW	BIT_0
 #define VCO_FORCE_UPDATE	BIT_1
 #define VCO_DONT_RESET_UPDATE	BIT_2
@@ -1737,18 +1737,18 @@ struct verify_chip_entry_84xx {
 #define VCO_END_OF_DATA		BIT_14
 #define VCO_ENABLE_DSD		BIT_15
 
-	uint16_t reserved_1;
+	__le16	reserved_1;
 
-	uint16_t data_seg_cnt;
-	uint16_t reserved_2[3];
+	__le16	data_seg_cnt;
+	__le16	reserved_2[3];
 
-	uint32_t fw_ver;
-	uint32_t exchange_address;
+	__le32	fw_ver;
+	__le32	exchange_address;
 
-	uint32_t reserved_3[3];
-	uint32_t fw_size;
-	uint32_t fw_seq_size;
-	uint32_t relative_offset;
+	__le32 reserved_3[3];
+	__le32	fw_size;
+	__le32	fw_seq_size;
+	__le32	relative_offset;
 
 	struct dsd64 dsd;
 };
@@ -1761,22 +1761,22 @@ struct verify_chip_rsp_84xx {
 
 	uint32_t handle;
 
-	uint16_t comp_status;
+	__le16	comp_status;
 #define CS_VCS_CHIP_FAILURE	0x3
 #define CS_VCS_BAD_EXCHANGE	0x8
 #define CS_VCS_SEQ_COMPLETEi	0x40
 
-	uint16_t failure_code;
+	__le16	failure_code;
 #define VFC_CHECKSUM_ERROR	0x1
 #define VFC_INVALID_LEN		0x2
 #define VFC_ALREADY_IN_PROGRESS	0x8
 
-	uint16_t reserved_1[4];
+	__le16	reserved_1[4];
 
-	uint32_t fw_ver;
-	uint32_t exchange_address;
+	__le32	fw_ver;
+	__le32	exchange_address;
 
-	uint32_t reserved_2[6];
+	__le32 reserved_2[6];
 };
 
 #define ACCESS_CHIP_IOCB_TYPE	0x2B
@@ -1788,24 +1788,24 @@ struct access_chip_84xx {
 
 	uint32_t handle;
 
-	uint16_t options;
+	__le16	options;
 #define ACO_DUMP_MEMORY		0x0
 #define ACO_LOAD_MEMORY		0x1
 #define ACO_CHANGE_CONFIG_PARAM	0x2
 #define ACO_REQUEST_INFO	0x3
 
-	uint16_t reserved1;
+	__le16	reserved1;
 
-	uint16_t dseg_count;
-	uint16_t reserved2[3];
+	__le16	dseg_count;
+	__le16	reserved2[3];
 
-	uint32_t parameter1;
-	uint32_t parameter2;
-	uint32_t parameter3;
+	__le32	parameter1;
+	__le32	parameter2;
+	__le32	parameter3;
 
-	uint32_t reserved3[3];
-	uint32_t total_byte_cnt;
-	uint32_t reserved4;
+	__le32	reserved3[3];
+	__le32	total_byte_cnt;
+	__le32	reserved4;
 
 	struct dsd64 dsd;
 };
@@ -1818,11 +1818,11 @@ struct access_chip_rsp_84xx {
 
 	uint32_t handle;
 
-	uint16_t comp_status;
-	uint16_t failure_code;
-	uint32_t residual_count;
+	__le16	comp_status;
+	__le16	failure_code;
+	__le32	residual_count;
 
-	uint32_t reserved[12];
+	__le32	reserved[12];
 };
 
 /* 81XX Support **************************************************************/
@@ -1877,52 +1877,52 @@ struct access_chip_rsp_84xx {
 struct nvram_81xx {
 	/* NVRAM header. */
 	uint8_t id[4];
-	uint16_t nvram_version;
-	uint16_t reserved_0;
+	__le16	nvram_version;
+	__le16	reserved_0;
 
 	/* Firmware Initialization Control Block. */
-	uint16_t version;
-	uint16_t reserved_1;
-	uint16_t frame_payload_size;
-	uint16_t execution_throttle;
-	uint16_t exchange_count;
-	uint16_t reserved_2;
+	__le16	version;
+	__le16	reserved_1;
+	__le16	frame_payload_size;
+	__le16	execution_throttle;
+	__le16	exchange_count;
+	__le16	reserved_2;
 
 	uint8_t port_name[WWN_SIZE];
 	uint8_t node_name[WWN_SIZE];
 
-	uint16_t login_retry_count;
-	uint16_t reserved_3;
-	uint16_t interrupt_delay_timer;
-	uint16_t login_timeout;
+	__le16	login_retry_count;
+	__le16	reserved_3;
+	__le16	interrupt_delay_timer;
+	__le16	login_timeout;
 
-	uint32_t firmware_options_1;
-	uint32_t firmware_options_2;
-	uint32_t firmware_options_3;
+	__le32	firmware_options_1;
+	__le32	firmware_options_2;
+	__le32	firmware_options_3;
 
-	uint16_t reserved_4[4];
+	__le16	reserved_4[4];
 
 	/* Offset 64. */
 	uint8_t enode_mac[6];
-	uint16_t reserved_5[5];
+	__le16	reserved_5[5];
 
 	/* Offset 80. */
-	uint16_t reserved_6[24];
+	__le16	reserved_6[24];
 
 	/* Offset 128. */
-	uint16_t ex_version;
+	__le16	ex_version;
 	uint8_t prio_fcf_matching_flags;
 	uint8_t reserved_6_1[3];
-	uint16_t pri_fcf_vlan_id;
+	__le16	pri_fcf_vlan_id;
 	uint8_t pri_fcf_fabric_name[8];
-	uint16_t reserved_6_2[7];
+	__le16	reserved_6_2[7];
 	uint8_t spma_mac_addr[6];
-	uint16_t reserved_6_3[14];
+	__le16	reserved_6_3[14];
 
 	/* Offset 192. */
 	uint8_t min_supported_speed;
 	uint8_t reserved_7_0;
-	uint16_t reserved_7[31];
+	__le16	reserved_7[31];
 
 	/*
 	 * BIT 0  = Enable spinup delay
@@ -1955,26 +1955,26 @@ struct nvram_81xx {
 	 * BIT 25 = Temp WWPN
 	 * BIT 26-31 =
 	 */
-	uint32_t host_p;
+	__le32	host_p;
 
 	uint8_t alternate_port_name[WWN_SIZE];
 	uint8_t alternate_node_name[WWN_SIZE];
 
 	uint8_t boot_port_name[WWN_SIZE];
-	uint16_t boot_lun_number;
-	uint16_t reserved_8;
+	__le16	boot_lun_number;
+	__le16	reserved_8;
 
 	uint8_t alt1_boot_port_name[WWN_SIZE];
-	uint16_t alt1_boot_lun_number;
-	uint16_t reserved_9;
+	__le16	alt1_boot_lun_number;
+	__le16	reserved_9;
 
 	uint8_t alt2_boot_port_name[WWN_SIZE];
-	uint16_t alt2_boot_lun_number;
-	uint16_t reserved_10;
+	__le16	alt2_boot_lun_number;
+	__le16	reserved_10;
 
 	uint8_t alt3_boot_port_name[WWN_SIZE];
-	uint16_t alt3_boot_lun_number;
-	uint16_t reserved_11;
+	__le16	alt3_boot_lun_number;
+	__le16	reserved_11;
 
 	/*
 	 * BIT 0 = Selective Login
@@ -1986,35 +1986,35 @@ struct nvram_81xx {
 	 * BIT 6 = Reserved
 	 * BIT 7-31 =
 	 */
-	uint32_t efi_parameters;
+	__le32	efi_parameters;
 
 	uint8_t reset_delay;
 	uint8_t reserved_12;
-	uint16_t reserved_13;
+	__le16	reserved_13;
 
-	uint16_t boot_id_number;
-	uint16_t reserved_14;
+	__le16	boot_id_number;
+	__le16	reserved_14;
 
-	uint16_t max_luns_per_target;
-	uint16_t reserved_15;
+	__le16	max_luns_per_target;
+	__le16	reserved_15;
 
-	uint16_t port_down_retry_count;
-	uint16_t link_down_timeout;
+	__le16	port_down_retry_count;
+	__le16	link_down_timeout;
 
 	/* FCode parameters. */
-	uint16_t fcode_parameter;
+	__le16	fcode_parameter;
 
-	uint16_t reserved_16[3];
+	__le16	reserved_16[3];
 
 	/* Offset 352. */
 	uint8_t reserved_17[4];
-	uint16_t reserved_18[5];
+	__le16	reserved_18[5];
 	uint8_t reserved_19[2];
-	uint16_t reserved_20[8];
+	__le16	reserved_20[8];
 
 	/* Offset 384. */
 	uint8_t reserved_21[16];
-	uint16_t reserved_22[3];
+	__le16	reserved_22[3];
 
 	/* Offset 406 (0x196) Enhanced Features
 	 * BIT 0    = Extended BB credits for LR
@@ -2027,20 +2027,20 @@ struct nvram_81xx {
 	uint16_t reserved_24[4];
 
 	/* Offset 416. */
-	uint16_t reserved_25[32];
+	__le16	reserved_25[32];
 
 	/* Offset 480. */
 	uint8_t model_name[16];
 
 	/* Offset 496. */
-	uint16_t feature_mask_l;
-	uint16_t feature_mask_h;
-	uint16_t reserved_26[2];
+	__le16	feature_mask_l;
+	__le16	feature_mask_h;
+	__le16	reserved_26[2];
 
-	uint16_t subsystem_vendor_id;
-	uint16_t subsystem_device_id;
+	__le16	subsystem_vendor_id;
+	__le16	subsystem_device_id;
 
-	uint32_t checksum;
+	__le32	checksum;
 };
 
 /*
@@ -2049,31 +2049,31 @@ struct nvram_81xx {
  */
 #define	ICB_VERSION 1
 struct init_cb_81xx {
-	uint16_t version;
-	uint16_t reserved_1;
+	__le16	version;
+	__le16	reserved_1;
 
-	uint16_t frame_payload_size;
-	uint16_t execution_throttle;
-	uint16_t exchange_count;
+	__le16	frame_payload_size;
+	__le16	execution_throttle;
+	__le16	exchange_count;
 
-	uint16_t reserved_2;
+	__le16	reserved_2;
 
 	uint8_t port_name[WWN_SIZE];		/* Big endian. */
 	uint8_t node_name[WWN_SIZE];		/* Big endian. */
 
-	uint16_t response_q_inpointer;
-	uint16_t request_q_outpointer;
+	__le16	response_q_inpointer;
+	__le16	request_q_outpointer;
 
-	uint16_t login_retry_count;
+	__le16	login_retry_count;
 
-	uint16_t prio_request_q_outpointer;
+	__le16	prio_request_q_outpointer;
 
-	uint16_t response_q_length;
-	uint16_t request_q_length;
+	__le16	response_q_length;
+	__le16	request_q_length;
 
-	uint16_t reserved_3;
+	__le16	reserved_3;
 
-	uint16_t prio_request_q_length;
+	__le16	prio_request_q_length;
 
 	__le64	 request_q_address __packed;
 	__le64	 response_q_address __packed;
@@ -2081,12 +2081,12 @@ struct init_cb_81xx {
 
 	uint8_t reserved_4[8];
 
-	uint16_t atio_q_inpointer;
-	uint16_t atio_q_length;
+	__le16	atio_q_inpointer;
+	__le16	atio_q_length;
 	__le64	 atio_q_address __packed;
 
-	uint16_t interrupt_delay_timer;		/* 100us increments. */
-	uint16_t login_timeout;
+	__le16	interrupt_delay_timer;		/* 100us increments. */
+	__le16	login_timeout;
 
 	/*
 	 * BIT 0-3 = Reserved
@@ -2099,7 +2099,7 @@ struct init_cb_81xx {
 	 * BIT 14 = Node Name Option
 	 * BIT 15-31 = Reserved
 	 */
-	uint32_t firmware_options_1;
+	__le32	firmware_options_1;
 
 	/*
 	 * BIT 0  = Operation Mode bit 0
@@ -2117,7 +2117,7 @@ struct init_cb_81xx {
 	 * BIT 14 = Enable Target PRLI Control
 	 * BIT 15-31 = Reserved
 	 */
-	uint32_t firmware_options_2;
+	__le32	firmware_options_2;
 
 	/*
 	 * BIT 0-3 = Reserved
@@ -2138,7 +2138,7 @@ struct init_cb_81xx {
 	 * BIT 28 = SPMA selection bit 1
 	 * BIT 30-31 = Reserved
 	 */
-	uint32_t firmware_options_3;
+	__le32	firmware_options_3;
 
 	uint8_t  reserved_5[8];
 
@@ -2216,9 +2216,9 @@ struct qla_fcp_prio_cfg {
 #define FCP_PRIO_ATTR_ENABLE    0x1
 #define FCP_PRIO_ATTR_PERSIST   0x2
 	uint8_t  reserved;      /* Reserved for future use          */
-#define FCP_PRIO_CFG_HDR_SIZE   0x10
-	struct qla_fcp_prio_entry entry[1];     /* fcp priority entries  */
-#define FCP_PRIO_CFG_ENTRY_SIZE 0x20
+#define FCP_PRIO_CFG_HDR_SIZE   offsetof(struct qla_fcp_prio_cfg, entry)
+	struct qla_fcp_prio_entry entry[1023]; /* fcp priority entries  */
+	uint8_t  reserved2[16];
 };
 
 #define FCP_PRIO_CFG_SIZE       (32*1024) /* fcp prio data per port*/
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 1b93f5b4d77d..061f91b521b3 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -173,6 +173,7 @@ extern int ql2xenablemsix;
 extern int qla2xuseresexchforels;
 extern int ql2xexlogins;
 extern int ql2xdifbundlinginternalbuffers;
+extern int ql2xfulldump_on_mpifail;
 
 extern int qla2x00_loop_reset(scsi_qla_host_t *);
 extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int);
@@ -636,15 +637,17 @@ extern int qla24xx_read_fcp_prio_cfg(scsi_qla_host_t *);
 /*
  * Global Function Prototypes in qla_dbg.c source file.
  */
-extern void qla2100_fw_dump(scsi_qla_host_t *, int);
-extern void qla2300_fw_dump(scsi_qla_host_t *, int);
-extern void qla24xx_fw_dump(scsi_qla_host_t *, int);
-extern void qla25xx_fw_dump(scsi_qla_host_t *, int);
-extern void qla81xx_fw_dump(scsi_qla_host_t *, int);
-extern void qla82xx_fw_dump(scsi_qla_host_t *, int);
-extern void qla8044_fw_dump(scsi_qla_host_t *, int);
-
-extern void qla27xx_fwdump(scsi_qla_host_t *, int);
+void qla2xxx_dump_fw(scsi_qla_host_t *vha);
+void qla2100_fw_dump(scsi_qla_host_t *vha);
+void qla2300_fw_dump(scsi_qla_host_t *vha);
+void qla24xx_fw_dump(scsi_qla_host_t *vha);
+void qla25xx_fw_dump(scsi_qla_host_t *vha);
+void qla81xx_fw_dump(scsi_qla_host_t *vha);
+void qla82xx_fw_dump(scsi_qla_host_t *vha);
+void qla8044_fw_dump(scsi_qla_host_t *vha);
+
+void qla27xx_fwdump(scsi_qla_host_t *vha);
+extern void qla27xx_mpi_fwdump(scsi_qla_host_t *, int);
 extern ulong qla27xx_fwdt_calculate_dump_size(struct scsi_qla_host *, void *);
 extern int qla27xx_fwdt_template_valid(void *);
 extern ulong qla27xx_fwdt_template_size(void *);
@@ -769,7 +772,7 @@ extern int qlafx00_fw_ready(scsi_qla_host_t *);
 extern int qlafx00_configure_devices(scsi_qla_host_t *);
 extern int qlafx00_reset_initialize(scsi_qla_host_t *);
 extern int qlafx00_fx_disc(scsi_qla_host_t *, fc_port_t *, uint16_t);
-extern int qlafx00_process_aen(struct scsi_qla_host *, struct qla_work_evt *);
+extern void qlafx00_process_aen(struct scsi_qla_host *, struct qla_work_evt *);
 extern int qlafx00_post_aenfx_work(struct scsi_qla_host *,  uint32_t,
 				   uint32_t *, int);
 extern uint32_t qlafx00_fw_state_show(struct device *,
@@ -871,7 +874,7 @@ extern int qla2x00_get_idma_speed(scsi_qla_host_t *, uint16_t,
 	uint16_t *, uint16_t *);
 
 /* 83xx related functions */
-extern void qla83xx_fw_dump(scsi_qla_host_t *, int);
+void qla83xx_fw_dump(scsi_qla_host_t *vha);
 
 /* Minidump related functions */
 extern int qla82xx_md_get_template_size(scsi_qla_host_t *);
@@ -933,5 +936,6 @@ extern void qla24xx_process_purex_list(struct purex_list *);
 
 /* nvme.c */
 void qla_nvme_unregister_remote_port(struct fc_port *fcport);
+void qla27xx_reset_mpi(scsi_qla_host_t *vha);
 void qla_handle_els_plogi_done(scsi_qla_host_t *vha, struct event_arg *ea);
 #endif /* _QLA_GBL_H */
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index caa6b840e459..4576d3ae9937 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -120,7 +120,7 @@ static void qla24xx_abort_iocb_timeout(void *data)
 	if (sp->cmd_sp)
 		sp->cmd_sp->done(sp->cmd_sp, QLA_OS_TIMER_EXPIRED);
 
-	abt->u.abt.comp_status = CS_TIMEOUT;
+	abt->u.abt.comp_status = cpu_to_le16(CS_TIMEOUT);
 	sp->done(sp, QLA_OS_TIMER_EXPIRED);
 }
 
@@ -992,7 +992,7 @@ static void qla24xx_async_gnl_sp_done(srb_t *sp, int res)
 
 		ql_dbg(ql_dbg_disc, vha, 0x20e8,
 		    "%s %8phC %02x:%02x:%02x CLS %x/%x lid %x \n",
-		    __func__, (void *)&wwn, e->port_id[2], e->port_id[1],
+		    __func__, &wwn, e->port_id[2], e->port_id[1],
 		    e->port_id[0], e->current_login_state, e->last_login_state,
 		    (loop_id & 0x7fff));
 	}
@@ -1343,7 +1343,7 @@ int qla24xx_async_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
 	mb[9] = vha->vp_idx;
 	mb[10] = opt;
 
-	mbx->u.mbx.in = (void *)pd;
+	mbx->u.mbx.in = pd;
 	mbx->u.mbx.in_dma = pd_dma;
 
 	sp->done = qla24xx_async_gpdb_sp_done;
@@ -1791,7 +1791,7 @@ qla2x00_tmf_iocb_timeout(void *data)
 			}
 		}
 		spin_unlock_irqrestore(sp->qpair->qp_lock_ptr, flags);
-		tmf->u.tmf.comp_status = CS_TIMEOUT;
+		tmf->u.tmf.comp_status = cpu_to_le16(CS_TIMEOUT);
 		tmf->u.tmf.data = QLA_FUNCTION_FAILED;
 		complete(&tmf->u.tmf.comp);
 	}
@@ -2219,7 +2219,7 @@ qla2x00_initialize_adapter(scsi_qla_host_t *vha)
 
 	/* Check for secure flash support */
 	if (IS_QLA28XX(ha)) {
-		if (RD_REG_DWORD(&reg->mailbox12) & BIT_0)
+		if (rd_reg_word(&reg->mailbox12) & BIT_0)
 			ha->flags.secure_adapter = 1;
 		ql_log(ql_log_info, vha, 0xffff, "Secure Adapter: %s\n",
 		    (ha->flags.secure_adapter) ? "Yes" : "No");
@@ -2357,7 +2357,7 @@ qla2100_pci_config(scsi_qla_host_t *vha)
 
 	/* Get PCI bus information. */
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	ha->pci_attr = RD_REG_WORD(&reg->ctrl_status);
+	ha->pci_attr = rd_reg_word(&reg->ctrl_status);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	return QLA_SUCCESS;
@@ -2399,17 +2399,17 @@ qla2300_pci_config(scsi_qla_host_t *vha)
 		spin_lock_irqsave(&ha->hardware_lock, flags);
 
 		/* Pause RISC. */
-		WRT_REG_WORD(&reg->hccr, HCCR_PAUSE_RISC);
+		wrt_reg_word(&reg->hccr, HCCR_PAUSE_RISC);
 		for (cnt = 0; cnt < 30000; cnt++) {
-			if ((RD_REG_WORD(&reg->hccr) & HCCR_RISC_PAUSE) != 0)
+			if ((rd_reg_word(&reg->hccr) & HCCR_RISC_PAUSE) != 0)
 				break;
 
 			udelay(10);
 		}
 
 		/* Select FPM registers. */
-		WRT_REG_WORD(&reg->ctrl_status, 0x20);
-		RD_REG_WORD(&reg->ctrl_status);
+		wrt_reg_word(&reg->ctrl_status, 0x20);
+		rd_reg_word(&reg->ctrl_status);
 
 		/* Get the fb rev level */
 		ha->fb_rev = RD_FB_CMD_REG(ha, reg);
@@ -2418,13 +2418,13 @@ qla2300_pci_config(scsi_qla_host_t *vha)
 			pci_clear_mwi(ha->pdev);
 
 		/* Deselect FPM registers. */
-		WRT_REG_WORD(&reg->ctrl_status, 0x0);
-		RD_REG_WORD(&reg->ctrl_status);
+		wrt_reg_word(&reg->ctrl_status, 0x0);
+		rd_reg_word(&reg->ctrl_status);
 
 		/* Release RISC module. */
-		WRT_REG_WORD(&reg->hccr, HCCR_RELEASE_RISC);
+		wrt_reg_word(&reg->hccr, HCCR_RELEASE_RISC);
 		for (cnt = 0; cnt < 30000; cnt++) {
-			if ((RD_REG_WORD(&reg->hccr) & HCCR_RISC_PAUSE) == 0)
+			if ((rd_reg_word(&reg->hccr) & HCCR_RISC_PAUSE) == 0)
 				break;
 
 			udelay(10);
@@ -2439,7 +2439,7 @@ qla2300_pci_config(scsi_qla_host_t *vha)
 
 	/* Get PCI bus information. */
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	ha->pci_attr = RD_REG_WORD(&reg->ctrl_status);
+	ha->pci_attr = rd_reg_word(&reg->ctrl_status);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	return QLA_SUCCESS;
@@ -2483,7 +2483,7 @@ qla24xx_pci_config(scsi_qla_host_t *vha)
 
 	/* Get PCI bus information. */
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	ha->pci_attr = RD_REG_DWORD(&reg->ctrl_status);
+	ha->pci_attr = rd_reg_dword(&reg->ctrl_status);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	return QLA_SUCCESS;
@@ -2587,36 +2587,36 @@ qla2x00_reset_chip(scsi_qla_host_t *vha)
 
 	if (!IS_QLA2100(ha)) {
 		/* Pause RISC. */
-		WRT_REG_WORD(&reg->hccr, HCCR_PAUSE_RISC);
+		wrt_reg_word(&reg->hccr, HCCR_PAUSE_RISC);
 		if (IS_QLA2200(ha) || IS_QLA2300(ha)) {
 			for (cnt = 0; cnt < 30000; cnt++) {
-				if ((RD_REG_WORD(&reg->hccr) &
+				if ((rd_reg_word(&reg->hccr) &
 				    HCCR_RISC_PAUSE) != 0)
 					break;
 				udelay(100);
 			}
 		} else {
-			RD_REG_WORD(&reg->hccr);	/* PCI Posting. */
+			rd_reg_word(&reg->hccr);	/* PCI Posting. */
 			udelay(10);
 		}
 
 		/* Select FPM registers. */
-		WRT_REG_WORD(&reg->ctrl_status, 0x20);
-		RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, 0x20);
+		rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
 
 		/* FPM Soft Reset. */
-		WRT_REG_WORD(&reg->fpm_diag_config, 0x100);
-		RD_REG_WORD(&reg->fpm_diag_config);	/* PCI Posting. */
+		wrt_reg_word(&reg->fpm_diag_config, 0x100);
+		rd_reg_word(&reg->fpm_diag_config);	/* PCI Posting. */
 
 		/* Toggle Fpm Reset. */
 		if (!IS_QLA2200(ha)) {
-			WRT_REG_WORD(&reg->fpm_diag_config, 0x0);
-			RD_REG_WORD(&reg->fpm_diag_config); /* PCI Posting. */
+			wrt_reg_word(&reg->fpm_diag_config, 0x0);
+			rd_reg_word(&reg->fpm_diag_config); /* PCI Posting. */
 		}
 
 		/* Select frame buffer registers. */
-		WRT_REG_WORD(&reg->ctrl_status, 0x10);
-		RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, 0x10);
+		rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
 
 		/* Reset frame buffer FIFOs. */
 		if (IS_QLA2200(ha)) {
@@ -2634,23 +2634,23 @@ qla2x00_reset_chip(scsi_qla_host_t *vha)
 		}
 
 		/* Select RISC module registers. */
-		WRT_REG_WORD(&reg->ctrl_status, 0);
-		RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, 0);
+		rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
 
 		/* Reset RISC processor. */
-		WRT_REG_WORD(&reg->hccr, HCCR_RESET_RISC);
-		RD_REG_WORD(&reg->hccr);		/* PCI Posting. */
+		wrt_reg_word(&reg->hccr, HCCR_RESET_RISC);
+		rd_reg_word(&reg->hccr);		/* PCI Posting. */
 
 		/* Release RISC processor. */
-		WRT_REG_WORD(&reg->hccr, HCCR_RELEASE_RISC);
-		RD_REG_WORD(&reg->hccr);		/* PCI Posting. */
+		wrt_reg_word(&reg->hccr, HCCR_RELEASE_RISC);
+		rd_reg_word(&reg->hccr);		/* PCI Posting. */
 	}
 
-	WRT_REG_WORD(&reg->hccr, HCCR_CLR_RISC_INT);
-	WRT_REG_WORD(&reg->hccr, HCCR_CLR_HOST_INT);
+	wrt_reg_word(&reg->hccr, HCCR_CLR_RISC_INT);
+	wrt_reg_word(&reg->hccr, HCCR_CLR_HOST_INT);
 
 	/* Reset ISP chip. */
-	WRT_REG_WORD(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
+	wrt_reg_word(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
 
 	/* Wait for RISC to recover from reset. */
 	if (IS_QLA2100(ha) || IS_QLA2200(ha) || IS_QLA2300(ha)) {
@@ -2661,7 +2661,7 @@ qla2x00_reset_chip(scsi_qla_host_t *vha)
 		 */
 		udelay(20);
 		for (cnt = 30000; cnt; cnt--) {
-			if ((RD_REG_WORD(&reg->ctrl_status) &
+			if ((rd_reg_word(&reg->ctrl_status) &
 			    CSR_ISP_SOFT_RESET) == 0)
 				break;
 			udelay(100);
@@ -2670,13 +2670,13 @@ qla2x00_reset_chip(scsi_qla_host_t *vha)
 		udelay(10);
 
 	/* Reset RISC processor. */
-	WRT_REG_WORD(&reg->hccr, HCCR_RESET_RISC);
+	wrt_reg_word(&reg->hccr, HCCR_RESET_RISC);
 
-	WRT_REG_WORD(&reg->semaphore, 0);
+	wrt_reg_word(&reg->semaphore, 0);
 
 	/* Release RISC processor. */
-	WRT_REG_WORD(&reg->hccr, HCCR_RELEASE_RISC);
-	RD_REG_WORD(&reg->hccr);			/* PCI Posting. */
+	wrt_reg_word(&reg->hccr, HCCR_RELEASE_RISC);
+	rd_reg_word(&reg->hccr);			/* PCI Posting. */
 
 	if (IS_QLA2100(ha) || IS_QLA2200(ha) || IS_QLA2300(ha)) {
 		for (cnt = 0; cnt < 30000; cnt++) {
@@ -2694,8 +2694,8 @@ qla2x00_reset_chip(scsi_qla_host_t *vha)
 
 	/* Disable RISC pause on FPM parity error. */
 	if (!IS_QLA2100(ha)) {
-		WRT_REG_WORD(&reg->hccr, HCCR_DISABLE_PARITY_PAUSE);
-		RD_REG_WORD(&reg->hccr);		/* PCI Posting. */
+		wrt_reg_word(&reg->hccr, HCCR_DISABLE_PARITY_PAUSE);
+		rd_reg_word(&reg->hccr);		/* PCI Posting. */
 	}
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -2740,32 +2740,32 @@ qla24xx_reset_risc(scsi_qla_host_t *vha)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
 	/* Reset RISC. */
-	WRT_REG_DWORD(&reg->ctrl_status, CSRX_DMA_SHUTDOWN|MWB_4096_BYTES);
+	wrt_reg_dword(&reg->ctrl_status, CSRX_DMA_SHUTDOWN|MWB_4096_BYTES);
 	for (cnt = 0; cnt < 30000; cnt++) {
-		if ((RD_REG_DWORD(&reg->ctrl_status) & CSRX_DMA_ACTIVE) == 0)
+		if ((rd_reg_dword(&reg->ctrl_status) & CSRX_DMA_ACTIVE) == 0)
 			break;
 
 		udelay(10);
 	}
 
-	if (!(RD_REG_DWORD(&reg->ctrl_status) & CSRX_DMA_ACTIVE))
+	if (!(rd_reg_dword(&reg->ctrl_status) & CSRX_DMA_ACTIVE))
 		set_bit(DMA_SHUTDOWN_CMPL, &ha->fw_dump_cap_flags);
 
 	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x017e,
 	    "HCCR: 0x%x, Control Status %x, DMA active status:0x%x\n",
-	    RD_REG_DWORD(&reg->hccr),
-	    RD_REG_DWORD(&reg->ctrl_status),
-	    (RD_REG_DWORD(&reg->ctrl_status) & CSRX_DMA_ACTIVE));
+	    rd_reg_dword(&reg->hccr),
+	    rd_reg_dword(&reg->ctrl_status),
+	    (rd_reg_dword(&reg->ctrl_status) & CSRX_DMA_ACTIVE));
 
-	WRT_REG_DWORD(&reg->ctrl_status,
+	wrt_reg_dword(&reg->ctrl_status,
 	    CSRX_ISP_SOFT_RESET|CSRX_DMA_SHUTDOWN|MWB_4096_BYTES);
 	pci_read_config_word(ha->pdev, PCI_COMMAND, &wd);
 
 	udelay(100);
 
 	/* Wait for firmware to complete NVRAM accesses. */
-	RD_REG_WORD(&reg->mailbox0);
-	for (cnt = 10000; RD_REG_WORD(&reg->mailbox0) != 0 &&
+	rd_reg_word(&reg->mailbox0);
+	for (cnt = 10000; rd_reg_word(&reg->mailbox0) != 0 &&
 	    rval == QLA_SUCCESS; cnt--) {
 		barrier();
 		if (cnt)
@@ -2779,26 +2779,26 @@ qla24xx_reset_risc(scsi_qla_host_t *vha)
 
 	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x017f,
 	    "HCCR: 0x%x, MailBox0 Status 0x%x\n",
-	    RD_REG_DWORD(&reg->hccr),
-	    RD_REG_DWORD(&reg->mailbox0));
+	    rd_reg_dword(&reg->hccr),
+	    rd_reg_word(&reg->mailbox0));
 
 	/* Wait for soft-reset to complete. */
-	RD_REG_DWORD(&reg->ctrl_status);
+	rd_reg_dword(&reg->ctrl_status);
 	for (cnt = 0; cnt < 60; cnt++) {
 		barrier();
-		if ((RD_REG_DWORD(&reg->ctrl_status) &
+		if ((rd_reg_dword(&reg->ctrl_status) &
 		    CSRX_ISP_SOFT_RESET) == 0)
 			break;
 
 		udelay(5);
 	}
-	if (!(RD_REG_DWORD(&reg->ctrl_status) & CSRX_ISP_SOFT_RESET))
+	if (!(rd_reg_dword(&reg->ctrl_status) & CSRX_ISP_SOFT_RESET))
 		set_bit(ISP_SOFT_RESET_CMPL, &ha->fw_dump_cap_flags);
 
 	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x015d,
 	    "HCCR: 0x%x, Soft Reset status: 0x%x\n",
-	    RD_REG_DWORD(&reg->hccr),
-	    RD_REG_DWORD(&reg->ctrl_status));
+	    rd_reg_dword(&reg->hccr),
+	    rd_reg_dword(&reg->ctrl_status));
 
 	/* If required, do an MPI FW reset now */
 	if (test_and_clear_bit(MPI_RESET_NEEDED, &vha->dpc_flags)) {
@@ -2817,17 +2817,17 @@ qla24xx_reset_risc(scsi_qla_host_t *vha)
 		}
 	}
 
-	WRT_REG_DWORD(&reg->hccr, HCCRX_SET_RISC_RESET);
-	RD_REG_DWORD(&reg->hccr);
+	wrt_reg_dword(&reg->hccr, HCCRX_SET_RISC_RESET);
+	rd_reg_dword(&reg->hccr);
 
-	WRT_REG_DWORD(&reg->hccr, HCCRX_REL_RISC_PAUSE);
-	RD_REG_DWORD(&reg->hccr);
+	wrt_reg_dword(&reg->hccr, HCCRX_REL_RISC_PAUSE);
+	rd_reg_dword(&reg->hccr);
 
-	WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_RESET);
-	RD_REG_DWORD(&reg->hccr);
+	wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_RESET);
+	rd_reg_dword(&reg->hccr);
 
-	RD_REG_WORD(&reg->mailbox0);
-	for (cnt = 60; RD_REG_WORD(&reg->mailbox0) != 0 &&
+	rd_reg_word(&reg->mailbox0);
+	for (cnt = 60; rd_reg_word(&reg->mailbox0) != 0 &&
 	    rval == QLA_SUCCESS; cnt--) {
 		barrier();
 		if (cnt)
@@ -2840,8 +2840,8 @@ qla24xx_reset_risc(scsi_qla_host_t *vha)
 
 	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x015e,
 	    "Host Risc 0x%x, mailbox0 0x%x\n",
-	    RD_REG_DWORD(&reg->hccr),
-	     RD_REG_WORD(&reg->mailbox0));
+	    rd_reg_dword(&reg->hccr),
+	     rd_reg_word(&reg->mailbox0));
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
@@ -2860,9 +2860,8 @@ qla25xx_read_risc_sema_reg(scsi_qla_host_t *vha, uint32_t *data)
 {
 	struct device_reg_24xx __iomem *reg = &vha->hw->iobase->isp24;
 
-	WRT_REG_DWORD(&reg->iobase_addr, RISC_REGISTER_BASE_OFFSET);
-	*data = RD_REG_DWORD(&reg->iobase_window + RISC_REGISTER_WINDOW_OFFET);
-
+	wrt_reg_dword(&reg->iobase_addr, RISC_REGISTER_BASE_OFFSET);
+	*data = rd_reg_dword(&reg->iobase_window + RISC_REGISTER_WINDOW_OFFSET);
 }
 
 static void
@@ -2870,8 +2869,8 @@ qla25xx_write_risc_sema_reg(scsi_qla_host_t *vha, uint32_t data)
 {
 	struct device_reg_24xx __iomem *reg = &vha->hw->iobase->isp24;
 
-	WRT_REG_DWORD(&reg->iobase_addr, RISC_REGISTER_BASE_OFFSET);
-	WRT_REG_DWORD(&reg->iobase_window + RISC_REGISTER_WINDOW_OFFET, data);
+	wrt_reg_dword(&reg->iobase_addr, RISC_REGISTER_BASE_OFFSET);
+	wrt_reg_dword(&reg->iobase_window + RISC_REGISTER_WINDOW_OFFSET, data);
 }
 
 static void
@@ -2887,7 +2886,7 @@ qla25xx_manipulate_risc_semaphore(scsi_qla_host_t *vha)
 	    vha->hw->pdev->subsystem_device != 0x0240)
 		return;
 
-	WRT_REG_DWORD(&vha->hw->iobase->isp24.hccr, HCCRX_SET_RISC_PAUSE);
+	wrt_reg_dword(&vha->hw->iobase->isp24.hccr, HCCRX_SET_RISC_PAUSE);
 	udelay(100);
 
 attempt:
@@ -2989,7 +2988,7 @@ qla2x00_chip_diag(scsi_qla_host_t *vha)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
 	/* Reset ISP chip. */
-	WRT_REG_WORD(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
+	wrt_reg_word(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
 
 	/*
 	 * We need to have a delay here since the card will not respond while
@@ -2999,7 +2998,7 @@ qla2x00_chip_diag(scsi_qla_host_t *vha)
 	data = qla2x00_debounce_register(&reg->ctrl_status);
 	for (cnt = 6000000 ; cnt && (data & CSR_ISP_SOFT_RESET); cnt--) {
 		udelay(5);
-		data = RD_REG_WORD(&reg->ctrl_status);
+		data = rd_reg_word(&reg->ctrl_status);
 		barrier();
 	}
 
@@ -3010,8 +3009,8 @@ qla2x00_chip_diag(scsi_qla_host_t *vha)
 	    "Reset register cleared by chip reset.\n");
 
 	/* Reset RISC processor. */
-	WRT_REG_WORD(&reg->hccr, HCCR_RESET_RISC);
-	WRT_REG_WORD(&reg->hccr, HCCR_RELEASE_RISC);
+	wrt_reg_word(&reg->hccr, HCCR_RESET_RISC);
+	wrt_reg_word(&reg->hccr, HCCR_RELEASE_RISC);
 
 	/* Workaround for QLA2312 PCI parity error */
 	if (IS_QLA2100(ha) || IS_QLA2200(ha) || IS_QLA2300(ha)) {
@@ -3339,6 +3338,8 @@ qla2x00_alloc_fw_dump(scsi_qla_host_t *vha)
 				    dump_size / 1024);
 
 				if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+					ha->mpi_fw_dump = (char *)fw_dump +
+						ha->fwdt[1].dump_size;
 					mutex_unlock(&ha->optrom_mutex);
 					return;
 				}
@@ -3650,8 +3651,8 @@ qla2x00_setup_chip(scsi_qla_host_t *vha)
 	if (!IS_FWI2_CAPABLE(ha) && !IS_QLA2100(ha) && !IS_QLA2200(ha)) {
 		/* Disable SRAM, Instruction RAM and GP RAM parity.  */
 		spin_lock_irqsave(&ha->hardware_lock, flags);
-		WRT_REG_WORD(&reg->hccr, (HCCR_ENABLE_PARITY + 0x0));
-		RD_REG_WORD(&reg->hccr);
+		wrt_reg_word(&reg->hccr, (HCCR_ENABLE_PARITY + 0x0));
+		rd_reg_word(&reg->hccr);
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	}
 
@@ -3758,11 +3759,11 @@ enable_82xx_npiv:
 		spin_lock_irqsave(&ha->hardware_lock, flags);
 		if (IS_QLA2300(ha))
 			/* SRAM parity */
-			WRT_REG_WORD(&reg->hccr, HCCR_ENABLE_PARITY + 0x1);
+			wrt_reg_word(&reg->hccr, HCCR_ENABLE_PARITY + 0x1);
 		else
 			/* SRAM, Instruction RAM and GP RAM parity */
-			WRT_REG_WORD(&reg->hccr, HCCR_ENABLE_PARITY + 0x7);
-		RD_REG_WORD(&reg->hccr);
+			wrt_reg_word(&reg->hccr, HCCR_ENABLE_PARITY + 0x7);
+		rd_reg_word(&reg->hccr);
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	}
 
@@ -4006,11 +4007,11 @@ qla2x00_config_rings(struct scsi_qla_host *vha)
 	put_unaligned_le64(req->dma, &ha->init_cb->request_q_address);
 	put_unaligned_le64(rsp->dma, &ha->init_cb->response_q_address);
 
-	WRT_REG_WORD(ISP_REQ_Q_IN(ha, reg), 0);
-	WRT_REG_WORD(ISP_REQ_Q_OUT(ha, reg), 0);
-	WRT_REG_WORD(ISP_RSP_Q_IN(ha, reg), 0);
-	WRT_REG_WORD(ISP_RSP_Q_OUT(ha, reg), 0);
-	RD_REG_WORD(ISP_RSP_Q_OUT(ha, reg));		/* PCI Posting. */
+	wrt_reg_word(ISP_REQ_Q_IN(ha, reg), 0);
+	wrt_reg_word(ISP_REQ_Q_OUT(ha, reg), 0);
+	wrt_reg_word(ISP_RSP_Q_IN(ha, reg), 0);
+	wrt_reg_word(ISP_RSP_Q_OUT(ha, reg), 0);
+	rd_reg_word(ISP_RSP_Q_OUT(ha, reg));		/* PCI Posting. */
 }
 
 void
@@ -4072,15 +4073,15 @@ qla24xx_config_rings(struct scsi_qla_host *vha)
 		}
 		icb->firmware_options_2 |= cpu_to_le32(BIT_23);
 
-		WRT_REG_DWORD(&reg->isp25mq.req_q_in, 0);
-		WRT_REG_DWORD(&reg->isp25mq.req_q_out, 0);
-		WRT_REG_DWORD(&reg->isp25mq.rsp_q_in, 0);
-		WRT_REG_DWORD(&reg->isp25mq.rsp_q_out, 0);
+		wrt_reg_dword(&reg->isp25mq.req_q_in, 0);
+		wrt_reg_dword(&reg->isp25mq.req_q_out, 0);
+		wrt_reg_dword(&reg->isp25mq.rsp_q_in, 0);
+		wrt_reg_dword(&reg->isp25mq.rsp_q_out, 0);
 	} else {
-		WRT_REG_DWORD(&reg->isp24.req_q_in, 0);
-		WRT_REG_DWORD(&reg->isp24.req_q_out, 0);
-		WRT_REG_DWORD(&reg->isp24.rsp_q_in, 0);
-		WRT_REG_DWORD(&reg->isp24.rsp_q_out, 0);
+		wrt_reg_dword(&reg->isp24.req_q_in, 0);
+		wrt_reg_dword(&reg->isp24.req_q_out, 0);
+		wrt_reg_dword(&reg->isp24.rsp_q_in, 0);
+		wrt_reg_dword(&reg->isp24.rsp_q_out, 0);
 	}
 
 	qlt_24xx_config_rings(vha);
@@ -4090,11 +4091,11 @@ qla24xx_config_rings(struct scsi_qla_host *vha)
 		ql_dbg(ql_dbg_init, vha, 0x00fd,
 		    "Speed set by user : %s Gbps \n",
 		    qla2x00_get_link_speed_str(ha, ha->set_data_rate));
-		icb->firmware_options_3 = (ha->set_data_rate << 13);
+		icb->firmware_options_3 = cpu_to_le32(ha->set_data_rate << 13);
 	}
 
 	/* PCI posting */
-	RD_REG_DWORD(&ioreg->hccr);
+	rd_reg_word(&ioreg->hccr);
 }
 
 /**
@@ -4125,7 +4126,7 @@ qla2x00_init_rings(scsi_qla_host_t *vha)
 		req = ha->req_q_map[que];
 		if (!req || !test_bit(que, ha->req_qid_map))
 			continue;
-		req->out_ptr = (void *)(req->ring + req->length);
+		req->out_ptr = (uint16_t *)(req->ring + req->length);
 		*req->out_ptr = 0;
 		for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++)
 			req->outstanding_cmds[cnt] = NULL;
@@ -4142,7 +4143,7 @@ qla2x00_init_rings(scsi_qla_host_t *vha)
 		rsp = ha->rsp_q_map[que];
 		if (!rsp || !test_bit(que, ha->rsp_qid_map))
 			continue;
-		rsp->in_ptr = (void *)(rsp->ring + rsp->length);
+		rsp->in_ptr = (uint16_t *)(rsp->ring + rsp->length);
 		*rsp->in_ptr = 0;
 		/* Initialize response queue entries */
 		if (IS_QLAFX00(ha))
@@ -4181,12 +4182,14 @@ qla2x00_init_rings(scsi_qla_host_t *vha)
 		mid_init_cb->init_cb.execution_throttle =
 		    cpu_to_le16(ha->cur_fw_xcb_count);
 		ha->flags.dport_enabled =
-		    (mid_init_cb->init_cb.firmware_options_1 & BIT_7) != 0;
+			(le32_to_cpu(mid_init_cb->init_cb.firmware_options_1) &
+			 BIT_7) != 0;
 		ql_dbg(ql_dbg_init, vha, 0x0191, "DPORT Support: %s.\n",
 		    (ha->flags.dport_enabled) ? "enabled" : "disabled");
 		/* FA-WWPN Status */
 		ha->flags.fawwpn_enabled =
-		    (mid_init_cb->init_cb.firmware_options_1 & BIT_6) != 0;
+			(le32_to_cpu(mid_init_cb->init_cb.firmware_options_1) &
+			 BIT_6) != 0;
 		ql_dbg(ql_dbg_init, vha, 0x00bc, "FA-WWPN Support: %s.\n",
 		    (ha->flags.fawwpn_enabled) ? "enabled" : "disabled");
 	}
@@ -4565,7 +4568,7 @@ qla2x00_nvram_config(scsi_qla_host_t *vha)
 	ha->nvram_size = sizeof(*nv);
 	ha->nvram_base = 0;
 	if (!IS_QLA2100(ha) && !IS_QLA2200(ha) && !IS_QLA2300(ha))
-		if ((RD_REG_WORD(&reg->ctrl_status) >> 14) == 1)
+		if ((rd_reg_word(&reg->ctrl_status) >> 14) == 1)
 			ha->nvram_base = 0x80;
 
 	/* Get NVRAM data and calculate checksum. */
@@ -5079,6 +5082,54 @@ qla2x00_configure_loop(scsi_qla_host_t *vha)
 	return (rval);
 }
 
+static int qla2x00_configure_n2n_loop(scsi_qla_host_t *vha)
+{
+	struct qla_hw_data *ha = vha->hw;
+	unsigned long flags;
+	fc_port_t *fcport;
+	int rval;
+
+	if (test_and_clear_bit(N2N_LOGIN_NEEDED, &vha->dpc_flags)) {
+		/* borrowing */
+		u32 *bp, sz;
+
+		memset(ha->init_cb, 0, ha->init_cb_size);
+		sz = min_t(int, sizeof(struct els_plogi_payload),
+			   ha->init_cb_size);
+		rval = qla24xx_get_port_login_templ(vha, ha->init_cb_dma,
+						    ha->init_cb, sz);
+		if (rval == QLA_SUCCESS) {
+			__be32 *q = &ha->plogi_els_payld.data[0];
+
+			bp = (uint32_t *)ha->init_cb;
+			cpu_to_be32_array(q, bp, sz / 4);
+			memcpy(bp, q, sizeof(ha->plogi_els_payld.data));
+		} else {
+			ql_dbg(ql_dbg_init, vha, 0x00d1,
+			       "PLOGI ELS param read fail.\n");
+			goto skip_login;
+		}
+	}
+
+	list_for_each_entry(fcport, &vha->vp_fcports, list) {
+		if (fcport->n2n_flag) {
+			qla24xx_fcport_handle_login(vha, fcport);
+			return QLA_SUCCESS;
+		}
+	}
+
+skip_login:
+	spin_lock_irqsave(&vha->work_lock, flags);
+	vha->scan.scan_retry++;
+	spin_unlock_irqrestore(&vha->work_lock, flags);
+
+	if (vha->scan.scan_retry < MAX_SCAN_RETRIES) {
+		set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags);
+		set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
+	}
+	return QLA_FUNCTION_FAILED;
+}
+
 /*
  * qla2x00_configure_local_loop
  *	Updates Fibre Channel Device Database with local loop devices.
@@ -5096,7 +5147,6 @@ qla2x00_configure_local_loop(scsi_qla_host_t *vha)
 	int		found_devs;
 	int		found;
 	fc_port_t	*fcport, *new_fcport;
-
 	uint16_t	index;
 	uint16_t	entries;
 	struct gid_list_info *gid;
@@ -5106,47 +5156,8 @@ qla2x00_configure_local_loop(scsi_qla_host_t *vha)
 	unsigned long flags;
 
 	/* Inititae N2N login. */
-	if (N2N_TOPO(ha)) {
-		if (test_and_clear_bit(N2N_LOGIN_NEEDED, &vha->dpc_flags)) {
-			/* borrowing */
-			u32 *bp, sz;
-
-			memset(ha->init_cb, 0, ha->init_cb_size);
-			sz = min_t(int, sizeof(struct els_plogi_payload),
-			    ha->init_cb_size);
-			rval = qla24xx_get_port_login_templ(vha,
-			    ha->init_cb_dma, (void *)ha->init_cb, sz);
-			if (rval == QLA_SUCCESS) {
-				__be32 *q = &ha->plogi_els_payld.data[0];
-
-				bp = (uint32_t *)ha->init_cb;
-				cpu_to_be32_array(q, bp, sz / 4);
-
-				memcpy(bp, q, sizeof(ha->plogi_els_payld.data));
-			} else {
-				ql_dbg(ql_dbg_init, vha, 0x00d1,
-				    "PLOGI ELS param read fail.\n");
-				goto skip_login;
-			}
-		}
-
-		list_for_each_entry(fcport, &vha->vp_fcports, list) {
-			if (fcport->n2n_flag) {
-				qla24xx_fcport_handle_login(vha, fcport);
-				return QLA_SUCCESS;
-			}
-		}
-skip_login:
-		spin_lock_irqsave(&vha->work_lock, flags);
-		vha->scan.scan_retry++;
-		spin_unlock_irqrestore(&vha->work_lock, flags);
-
-		if (vha->scan.scan_retry < MAX_SCAN_RETRIES) {
-			set_bit(LOCAL_LOOP_UPDATE, &vha->dpc_flags);
-			set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
-		}
-		return QLA_FUNCTION_FAILED;
-	}
+	if (N2N_TOPO(ha))
+		return qla2x00_configure_n2n_loop(vha);
 
 	found_devs = 0;
 	new_fcport = NULL;
@@ -7078,10 +7089,10 @@ qla2x00_reset_adapter(scsi_qla_host_t *vha)
 	ha->isp_ops->disable_intrs(ha);
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	WRT_REG_WORD(&reg->hccr, HCCR_RESET_RISC);
-	RD_REG_WORD(&reg->hccr);			/* PCI Posting. */
-	WRT_REG_WORD(&reg->hccr, HCCR_RELEASE_RISC);
-	RD_REG_WORD(&reg->hccr);			/* PCI Posting. */
+	wrt_reg_word(&reg->hccr, HCCR_RESET_RISC);
+	rd_reg_word(&reg->hccr);			/* PCI Posting. */
+	wrt_reg_word(&reg->hccr, HCCR_RELEASE_RISC);
+	rd_reg_word(&reg->hccr);			/* PCI Posting. */
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	return QLA_SUCCESS;
@@ -7102,10 +7113,10 @@ qla24xx_reset_adapter(scsi_qla_host_t *vha)
 	ha->isp_ops->disable_intrs(ha);
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	WRT_REG_DWORD(&reg->hccr, HCCRX_SET_RISC_RESET);
-	RD_REG_DWORD(&reg->hccr);
-	WRT_REG_DWORD(&reg->hccr, HCCRX_REL_RISC_PAUSE);
-	RD_REG_DWORD(&reg->hccr);
+	wrt_reg_dword(&reg->hccr, HCCRX_SET_RISC_RESET);
+	rd_reg_dword(&reg->hccr);
+	wrt_reg_dword(&reg->hccr, HCCRX_REL_RISC_PAUSE);
+	rd_reg_dword(&reg->hccr);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	if (IS_NOPOLLING_TYPE(ha))
@@ -7143,7 +7154,7 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 	int   rval;
 	struct init_cb_24xx *icb;
 	struct nvram_24xx *nv;
-	uint32_t *dptr;
+	__le32 *dptr;
 	uint8_t  *dptr1, *dptr2;
 	uint32_t chksum;
 	uint16_t cnt;
@@ -7171,7 +7182,7 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 	    ha->nvram_base - FA_NVRAM_FUNC0_ADDR, FA_NVRAM_VPD_SIZE * 4);
 
 	/* Get NVRAM data into cache and calculate checksum. */
-	dptr = (uint32_t *)nv;
+	dptr = (__force __le32 *)nv;
 	ha->isp_ops->read_nvram(vha, dptr, ha->nvram_base, ha->nvram_size);
 	for (cnt = 0, chksum = 0; cnt < ha->nvram_size >> 2; cnt++, dptr++)
 		chksum += le32_to_cpu(*dptr);
@@ -7199,7 +7210,7 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 		memset(nv, 0, ha->nvram_size);
 		nv->nvram_version = cpu_to_le16(ICB_VERSION);
 		nv->version = cpu_to_le16(ICB_VERSION);
-		nv->frame_payload_size = 2048;
+		nv->frame_payload_size = cpu_to_le16(2048);
 		nv->execution_throttle = cpu_to_le16(0xFFFF);
 		nv->exchange_count = cpu_to_le16(0);
 		nv->hard_address = cpu_to_le16(124);
@@ -7367,7 +7378,7 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 		ha->login_retry_count = ql2xloginretrycount;
 
 	/* N2N: driver will initiate Login instead of FW */
-	icb->firmware_options_3 |= BIT_8;
+	icb->firmware_options_3 |= cpu_to_le32(BIT_8);
 
 	/* Enable ZIO. */
 	if (!vha->flags.init_done) {
@@ -7435,7 +7446,7 @@ qla27xx_check_image_status_signature(struct qla27xx_image_status *image_status)
 static ulong
 qla27xx_image_status_checksum(struct qla27xx_image_status *image_status)
 {
-	uint32_t *p = (void *)image_status;
+	__le32 *p = (__force __le32 *)image_status;
 	uint n = sizeof(*image_status) / sizeof(*p);
 	uint32_t sum = 0;
 
@@ -7498,7 +7509,7 @@ qla28xx_get_aux_images(
 		goto check_sec_image;
 	}
 
-	qla24xx_read_flash_data(vha, (void *)&pri_aux_image_status,
+	qla24xx_read_flash_data(vha, (uint32_t *)&pri_aux_image_status,
 	    ha->flt_region_aux_img_status_pri,
 	    sizeof(pri_aux_image_status) >> 2);
 	qla27xx_print_image(vha, "Primary aux image", &pri_aux_image_status);
@@ -7531,7 +7542,7 @@ check_sec_image:
 		goto check_valid_image;
 	}
 
-	qla24xx_read_flash_data(vha, (void *)&sec_aux_image_status,
+	qla24xx_read_flash_data(vha, (uint32_t *)&sec_aux_image_status,
 	    ha->flt_region_aux_img_status_sec,
 	    sizeof(sec_aux_image_status) >> 2);
 	qla27xx_print_image(vha, "Secondary aux image", &sec_aux_image_status);
@@ -7596,7 +7607,7 @@ qla27xx_get_active_image(struct scsi_qla_host *vha,
 		goto check_sec_image;
 	}
 
-	if (qla24xx_read_flash_data(vha, (void *)(&pri_image_status),
+	if (qla24xx_read_flash_data(vha, (uint32_t *)&pri_image_status,
 	    ha->flt_region_img_status_pri, sizeof(pri_image_status) >> 2) !=
 	    QLA_SUCCESS) {
 		WARN_ON_ONCE(true);
@@ -7703,7 +7714,7 @@ qla24xx_load_risc_flash(scsi_qla_host_t *vha, uint32_t *srisc_addr,
 	ql_dbg(ql_dbg_init, vha, 0x008b,
 	    "FW: Loading firmware from flash (%x).\n", faddr);
 
-	dcode = (void *)req->ring;
+	dcode = (uint32_t *)req->ring;
 	qla24xx_read_flash_data(vha, dcode, faddr, 8);
 	if (qla24xx_risc_firmware_invalid(dcode)) {
 		ql_log(ql_log_fatal, vha, 0x008c,
@@ -7716,18 +7727,18 @@ qla24xx_load_risc_flash(scsi_qla_host_t *vha, uint32_t *srisc_addr,
 		return QLA_FUNCTION_FAILED;
 	}
 
-	dcode = (void *)req->ring;
+	dcode = (uint32_t *)req->ring;
 	*srisc_addr = 0;
 	segments = FA_RISC_CODE_SEGMENTS;
 	for (j = 0; j < segments; j++) {
 		ql_dbg(ql_dbg_init, vha, 0x008d,
 		    "-> Loading segment %u...\n", j);
 		qla24xx_read_flash_data(vha, dcode, faddr, 10);
-		risc_addr = be32_to_cpu(dcode[2]);
-		risc_size = be32_to_cpu(dcode[3]);
+		risc_addr = be32_to_cpu((__force __be32)dcode[2]);
+		risc_size = be32_to_cpu((__force __be32)dcode[3]);
 		if (!*srisc_addr) {
 			*srisc_addr = risc_addr;
-			risc_attr = be32_to_cpu(dcode[9]);
+			risc_attr = be32_to_cpu((__force __be32)dcode[9]);
 		}
 
 		dlen = ha->fw_transfer_size >> 2;
@@ -7767,9 +7778,9 @@ qla24xx_load_risc_flash(scsi_qla_host_t *vha, uint32_t *srisc_addr,
 		fwdt->template = NULL;
 		fwdt->length = 0;
 
-		dcode = (void *)req->ring;
+		dcode = (uint32_t *)req->ring;
 		qla24xx_read_flash_data(vha, dcode, faddr, 7);
-		risc_size = be32_to_cpu(dcode[2]);
+		risc_size = be32_to_cpu((__force __be32)dcode[2]);
 		ql_dbg(ql_dbg_init, vha, 0x0161,
 		    "-> fwdt%u template array at %#x (%#x dwords)\n",
 		    j, faddr, risc_size);
@@ -7838,7 +7849,8 @@ qla2x00_load_risc(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 {
 	int	rval;
 	int	i, fragment;
-	uint16_t *wcode, *fwcode;
+	uint16_t *wcode;
+	__be16	 *fwcode;
 	uint32_t risc_addr, risc_size, fwclen, wlen, *seg;
 	struct fw_blob *blob;
 	struct qla_hw_data *ha = vha->hw;
@@ -7858,7 +7870,7 @@ qla2x00_load_risc(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 
 	wcode = (uint16_t *)req->ring;
 	*srisc_addr = 0;
-	fwcode = (uint16_t *)blob->fw->data;
+	fwcode = (__force __be16 *)blob->fw->data;
 	fwclen = 0;
 
 	/* Validate firmware image by checking version. */
@@ -7906,7 +7918,7 @@ qla2x00_load_risc(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 			    "words 0x%x.\n", risc_addr, wlen);
 
 			for (i = 0; i < wlen; i++)
-				wcode[i] = swab16(fwcode[i]);
+				wcode[i] = swab16((__force u32)fwcode[i]);
 
 			rval = qla2x00_load_ram(vha, req->dma, risc_addr,
 			    wlen);
@@ -7943,7 +7955,7 @@ qla24xx_load_risc_blob(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 	ulong i;
 	uint j;
 	struct fw_blob *blob;
-	uint32_t *fwcode;
+	__be32 *fwcode;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
 	struct fwdt *fwdt = ha->fwdt;
@@ -7959,8 +7971,8 @@ qla24xx_load_risc_blob(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 		return QLA_FUNCTION_FAILED;
 	}
 
-	fwcode = (void *)blob->fw->data;
-	dcode = fwcode;
+	fwcode = (__force __be32 *)blob->fw->data;
+	dcode = (__force uint32_t *)fwcode;
 	if (qla24xx_risc_firmware_invalid(dcode)) {
 		ql_log(ql_log_fatal, vha, 0x0093,
 		    "Unable to verify integrity of firmware image (%zd).\n",
@@ -7971,7 +7983,7 @@ qla24xx_load_risc_blob(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 		return QLA_FUNCTION_FAILED;
 	}
 
-	dcode = (void *)req->ring;
+	dcode = (uint32_t *)req->ring;
 	*srisc_addr = 0;
 	segments = FA_RISC_CODE_SEGMENTS;
 	for (j = 0; j < segments; j++) {
@@ -7997,7 +8009,7 @@ qla24xx_load_risc_blob(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 			    dlen);
 
 			for (i = 0; i < dlen; i++)
-				dcode[i] = swab32(fwcode[i]);
+				dcode[i] = swab32((__force u32)fwcode[i]);
 
 			rval = qla2x00_load_ram(vha, req->dma, risc_addr, dlen);
 			if (rval) {
@@ -8051,7 +8063,7 @@ qla24xx_load_risc_blob(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 
 		dcode = fwdt->template;
 		for (i = 0; i < risc_size; i++)
-			dcode[i] = fwcode[i];
+			dcode[i] = (__force u32)fwcode[i];
 
 		if (!qla27xx_fwdt_template_valid(dcode)) {
 			ql_log(ql_log_warn, vha, 0x0175,
@@ -8322,7 +8334,7 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	int   rval;
 	struct init_cb_81xx *icb;
 	struct nvram_81xx *nv;
-	uint32_t *dptr;
+	__le32 *dptr;
 	uint8_t  *dptr1, *dptr2;
 	uint32_t chksum;
 	uint16_t cnt;
@@ -8369,7 +8381,7 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	    "primary" : "secondary");
 	ha->isp_ops->read_optrom(vha, ha->nvram, faddr << 2, ha->nvram_size);
 
-	dptr = (uint32_t *)nv;
+	dptr = (__force __le32 *)nv;
 	for (cnt = 0, chksum = 0; cnt < ha->nvram_size >> 2; cnt++, dptr++)
 		chksum += le32_to_cpu(*dptr);
 
@@ -8396,7 +8408,7 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 		memset(nv, 0, ha->nvram_size);
 		nv->nvram_version = cpu_to_le16(ICB_VERSION);
 		nv->version = cpu_to_le16(ICB_VERSION);
-		nv->frame_payload_size = 2048;
+		nv->frame_payload_size = cpu_to_le16(2048);
 		nv->execution_throttle = cpu_to_le16(0xFFFF);
 		nv->exchange_count = cpu_to_le16(0);
 		nv->port_name[0] = 0x21;
@@ -8440,7 +8452,7 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	}
 
 	if (IS_T10_PI_CAPABLE(ha))
-		nv->frame_payload_size &= ~7;
+		nv->frame_payload_size &= cpu_to_le16(~7);
 
 	qlt_81xx_config_nvram_stage1(vha, nv);
 
@@ -8603,10 +8615,10 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	}
 
 	 /* enable RIDA Format2 */
-	icb->firmware_options_3 |= BIT_0;
+	icb->firmware_options_3 |= cpu_to_le32(BIT_0);
 
 	/* N2N: driver will initiate Login instead of FW */
-	icb->firmware_options_3 |= BIT_8;
+	icb->firmware_options_3 |= cpu_to_le32(BIT_8);
 
 	/* Determine NVMe/FCP priority for target ports */
 	ha->fc4_type_priority = qla2xxx_get_fc4_priority(vha);
diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h
index 364b3db8b2dc..1fb6ccac07cc 100644
--- a/drivers/scsi/qla2xxx/qla_inline.h
+++ b/drivers/scsi/qla2xxx/qla_inline.h
@@ -40,16 +40,16 @@ qla24xx_calc_iocbs(scsi_qla_host_t *vha, uint16_t dsds)
  *      register value.
  */
 static __inline__ uint16_t
-qla2x00_debounce_register(volatile uint16_t __iomem *addr)
+qla2x00_debounce_register(volatile __le16 __iomem *addr)
 {
 	volatile uint16_t first;
 	volatile uint16_t second;
 
 	do {
-		first = RD_REG_WORD(addr);
+		first = rd_reg_word(addr);
 		barrier();
 		cpu_relax();
-		second = RD_REG_WORD(addr);
+		second = rd_reg_word(addr);
 	} while (first != second);
 
 	return (first);
@@ -329,7 +329,7 @@ qla_83xx_start_iocbs(struct qla_qpair *qpair)
 	} else
 		req->ring_ptr++;
 
-	WRT_REG_DWORD(req->req_q_in, req->ring_index);
+	wrt_reg_dword(req->req_q_in, req->ring_index);
 }
 
 static inline int
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index 182bd68c79ac..8865c35d3421 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -376,7 +376,7 @@ qla2x00_start_scsi(srb_t *sp)
 	/* Calculate the number of request entries needed. */
 	req_cnt = ha->isp_ops->calc_req_entries(tot_dsds);
 	if (req->cnt < (req_cnt + 2)) {
-		cnt = RD_REG_WORD_RELAXED(ISP_REQ_Q_OUT(ha, reg));
+		cnt = rd_reg_word_relaxed(ISP_REQ_Q_OUT(ha, reg));
 		if (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
 		else
@@ -428,8 +428,8 @@ qla2x00_start_scsi(srb_t *sp)
 	sp->flags |= SRB_DMA_VALID;
 
 	/* Set chip new ring index. */
-	WRT_REG_WORD(ISP_REQ_Q_IN(ha, reg), req->ring_index);
-	RD_REG_WORD_RELAXED(ISP_REQ_Q_IN(ha, reg));	/* PCI Posting. */
+	wrt_reg_word(ISP_REQ_Q_IN(ha, reg), req->ring_index);
+	rd_reg_word_relaxed(ISP_REQ_Q_IN(ha, reg));	/* PCI Posting. */
 
 	/* Manage unprocessed RIO/ZIO commands in response queue. */
 	if (vha->flags.process_response_queue &&
@@ -472,21 +472,21 @@ qla2x00_start_iocbs(struct scsi_qla_host *vha, struct req_que *req)
 
 		/* Set chip new ring index. */
 		if (ha->mqenable || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
-			WRT_REG_DWORD(req->req_q_in, req->ring_index);
+			wrt_reg_dword(req->req_q_in, req->ring_index);
 		} else if (IS_QLA83XX(ha)) {
-			WRT_REG_DWORD(req->req_q_in, req->ring_index);
-			RD_REG_DWORD_RELAXED(&ha->iobase->isp24.hccr);
+			wrt_reg_dword(req->req_q_in, req->ring_index);
+			rd_reg_dword_relaxed(&ha->iobase->isp24.hccr);
 		} else if (IS_QLAFX00(ha)) {
-			WRT_REG_DWORD(&reg->ispfx00.req_q_in, req->ring_index);
-			RD_REG_DWORD_RELAXED(&reg->ispfx00.req_q_in);
+			wrt_reg_dword(&reg->ispfx00.req_q_in, req->ring_index);
+			rd_reg_dword_relaxed(&reg->ispfx00.req_q_in);
 			QLAFX00_SET_HST_INTR(ha, ha->rqstq_intr_code);
 		} else if (IS_FWI2_CAPABLE(ha)) {
-			WRT_REG_DWORD(&reg->isp24.req_q_in, req->ring_index);
-			RD_REG_DWORD_RELAXED(&reg->isp24.req_q_in);
+			wrt_reg_dword(&reg->isp24.req_q_in, req->ring_index);
+			rd_reg_dword_relaxed(&reg->isp24.req_q_in);
 		} else {
-			WRT_REG_WORD(ISP_REQ_Q_IN(ha, &reg->isp),
+			wrt_reg_word(ISP_REQ_Q_IN(ha, &reg->isp),
 				req->ring_index);
-			RD_REG_WORD_RELAXED(ISP_REQ_Q_IN(ha, &reg->isp));
+			rd_reg_word_relaxed(ISP_REQ_Q_IN(ha, &reg->isp));
 		}
 	}
 }
@@ -661,7 +661,7 @@ qla24xx_build_scsi_type_6_iocbs(srb_t *sp, struct cmd_type_6 *cmd_pkt,
 	cur_dsd->address = 0;
 	cur_dsd->length = 0;
 	cur_dsd++;
-	cmd_pkt->control_flags |= CF_DATA_SEG_DESCR_ENABLE;
+	cmd_pkt->control_flags |= cpu_to_le16(CF_DATA_SEG_DESCR_ENABLE);
 	return 0;
 }
 
@@ -755,8 +755,8 @@ qla24xx_build_scsi_iocbs(srb_t *sp, struct cmd_type_7 *cmd_pkt,
 }
 
 struct fw_dif_context {
-	uint32_t ref_tag;
-	uint16_t app_tag;
+	__le32	ref_tag;
+	__le16	app_tag;
 	uint8_t ref_tag_mask[4];	/* Validation/Replacement Mask*/
 	uint8_t app_tag_mask[2];	/* Validation/Replacement Mask*/
 };
@@ -1389,7 +1389,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
     uint16_t tot_dsds, uint16_t tot_prot_dsds, uint16_t fw_prot_opts)
 {
 	struct dsd64		*cur_dsd;
-	uint32_t		*fcp_dl;
+	__be32			*fcp_dl;
 	scsi_qla_host_t		*vha;
 	struct scsi_cmnd	*cmd;
 	uint32_t		total_bytes = 0;
@@ -1456,7 +1456,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	    &crc_ctx_pkt->ref_tag, tot_prot_dsds);
 
 	put_unaligned_le64(crc_ctx_dma, &cmd_pkt->crc_context_address);
-	cmd_pkt->crc_context_len = CRC_CONTEXT_LEN_FW;
+	cmd_pkt->crc_context_len = cpu_to_le16(CRC_CONTEXT_LEN_FW);
 
 	/* Determine SCSI command length -- align to 4 byte boundary */
 	if (cmd->cmd_len > 16) {
@@ -1545,7 +1545,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	crc_ctx_pkt->guard_seed = cpu_to_le16(0);
 	/* Fibre channel byte count */
 	cmd_pkt->byte_count = cpu_to_le32(total_bytes);
-	fcp_dl = (uint32_t *)(crc_ctx_pkt->fcp_cmnd.cdb + 16 +
+	fcp_dl = (__be32 *)(crc_ctx_pkt->fcp_cmnd.cdb + 16 +
 	    additional_fcpcdb_len);
 	*fcp_dl = htonl(total_bytes);
 
@@ -1637,7 +1637,7 @@ qla24xx_start_scsi(srb_t *sp)
 	req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
 	if (req->cnt < (req_cnt + 2)) {
 		cnt = IS_SHADOW_REG_CAPABLE(ha) ? *req->out_ptr :
-		    RD_REG_DWORD_RELAXED(req->req_q_out);
+		    rd_reg_dword_relaxed(req->req_q_out);
 		if (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
 		else
@@ -1698,7 +1698,7 @@ qla24xx_start_scsi(srb_t *sp)
 	sp->flags |= SRB_DMA_VALID;
 
 	/* Set chip new ring index. */
-	WRT_REG_DWORD(req->req_q_in, req->ring_index);
+	wrt_reg_dword(req->req_q_in, req->ring_index);
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	return QLA_SUCCESS;
@@ -1822,7 +1822,7 @@ qla24xx_dif_start_scsi(srb_t *sp)
 	tot_dsds += nseg;
 	if (req->cnt < (req_cnt + 2)) {
 		cnt = IS_SHADOW_REG_CAPABLE(ha) ? *req->out_ptr :
-		    RD_REG_DWORD_RELAXED(req->req_q_out);
+		    rd_reg_dword_relaxed(req->req_q_out);
 		if (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
 		else
@@ -1881,7 +1881,7 @@ qla24xx_dif_start_scsi(srb_t *sp)
 		req->ring_ptr++;
 
 	/* Set chip new ring index. */
-	WRT_REG_DWORD(req->req_q_in, req->ring_index);
+	wrt_reg_dword(req->req_q_in, req->ring_index);
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
@@ -1957,7 +1957,7 @@ qla2xxx_start_scsi_mq(srb_t *sp)
 	req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
 	if (req->cnt < (req_cnt + 2)) {
 		cnt = IS_SHADOW_REG_CAPABLE(ha) ? *req->out_ptr :
-		    RD_REG_DWORD_RELAXED(req->req_q_out);
+		    rd_reg_dword_relaxed(req->req_q_out);
 		if (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
 		else
@@ -2018,7 +2018,7 @@ qla2xxx_start_scsi_mq(srb_t *sp)
 	sp->flags |= SRB_DMA_VALID;
 
 	/* Set chip new ring index. */
-	WRT_REG_DWORD(req->req_q_in, req->ring_index);
+	wrt_reg_dword(req->req_q_in, req->ring_index);
 
 	spin_unlock_irqrestore(&qpair->qp_lock, flags);
 	return QLA_SUCCESS;
@@ -2157,7 +2157,7 @@ qla2xxx_dif_start_scsi_mq(srb_t *sp)
 	tot_dsds += nseg;
 	if (req->cnt < (req_cnt + 2)) {
 		cnt = IS_SHADOW_REG_CAPABLE(ha) ? *req->out_ptr :
-		    RD_REG_DWORD_RELAXED(req->req_q_out);
+		    rd_reg_dword_relaxed(req->req_q_out);
 		if (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
 		else
@@ -2214,7 +2214,7 @@ qla2xxx_dif_start_scsi_mq(srb_t *sp)
 		req->ring_ptr++;
 
 	/* Set chip new ring index. */
-	WRT_REG_DWORD(req->req_q_in, req->ring_index);
+	wrt_reg_dword(req->req_q_in, req->ring_index);
 
 	/* Manage unprocessed RIO/ZIO commands in response queue. */
 	if (vha->flags.process_response_queue &&
@@ -2266,13 +2266,13 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp)
 			cnt = *req->out_ptr;
 		else if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
 		    IS_QLA28XX(ha))
-			cnt = RD_REG_DWORD(&reg->isp25mq.req_q_out);
+			cnt = rd_reg_dword(&reg->isp25mq.req_q_out);
 		else if (IS_P3P_TYPE(ha))
-			cnt = RD_REG_DWORD(&reg->isp82.req_q_out);
+			cnt = rd_reg_dword(reg->isp82.req_q_out);
 		else if (IS_FWI2_CAPABLE(ha))
-			cnt = RD_REG_DWORD(&reg->isp24.req_q_out);
+			cnt = rd_reg_dword(&reg->isp24.req_q_out);
 		else if (IS_QLAFX00(ha))
-			cnt = RD_REG_DWORD(&reg->ispfx00.req_q_out);
+			cnt = rd_reg_dword(&reg->ispfx00.req_q_out);
 		else
 			cnt = qla2x00_debounce_register(
 			    ISP_REQ_Q_OUT(ha, &reg->isp));
@@ -2305,8 +2305,8 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp)
 	pkt = req->ring_ptr;
 	memset(pkt, 0, REQUEST_ENTRY_SIZE);
 	if (IS_QLAFX00(ha)) {
-		WRT_REG_BYTE((void __iomem *)&pkt->entry_count, req_cnt);
-		WRT_REG_WORD((void __iomem *)&pkt->handle, handle);
+		wrt_reg_byte((void __iomem *)&pkt->entry_count, req_cnt);
+		wrt_reg_word((void __iomem *)&pkt->handle, handle);
 	} else {
 		pkt->entry_count = req_cnt;
 		pkt->handle = handle;
@@ -2344,9 +2344,10 @@ qla24xx_prli_iocb(srb_t *sp, struct logio_entry_24xx *logio)
 	logio->entry_type = LOGINOUT_PORT_IOCB_TYPE;
 	logio->control_flags = cpu_to_le16(LCF_COMMAND_PRLI);
 	if (lio->u.logio.flags & SRB_LOGIN_NVME_PRLI) {
-		logio->control_flags |= LCF_NVME_PRLI;
+		logio->control_flags |= cpu_to_le16(LCF_NVME_PRLI);
 		if (sp->vha->flags.nvme_first_burst)
-			logio->io_parameter[0] = NVME_PRLI_SP_FIRST_BURST;
+			logio->io_parameter[0] =
+				cpu_to_le32(NVME_PRLI_SP_FIRST_BURST);
 	}
 
 	logio->nport_handle = cpu_to_le16(sp->fcport->loop_id);
@@ -2680,7 +2681,7 @@ qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 	els_iocb->entry_status = 0;
 	els_iocb->handle = sp->handle;
 	els_iocb->nport_handle = cpu_to_le16(sp->fcport->loop_id);
-	els_iocb->tx_dsd_count = 1;
+	els_iocb->tx_dsd_count = cpu_to_le16(1);
 	els_iocb->vp_index = vha->vp_idx;
 	els_iocb->sof_type = EST_SOFI3;
 	els_iocb->rx_dsd_count = 0;
@@ -2700,7 +2701,7 @@ qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 			cpu_to_le32(sizeof(struct els_plogi_payload));
 		put_unaligned_le64(elsio->u.els_plogi.els_plogi_pyld_dma,
 				   &els_iocb->tx_address);
-		els_iocb->rx_dsd_count = 1;
+		els_iocb->rx_dsd_count = cpu_to_le16(1);
 		els_iocb->rx_byte_count = els_iocb->rx_len =
 			cpu_to_le32(sizeof(struct els_plogi_payload));
 		put_unaligned_le64(elsio->u.els_plogi.els_resp_pyld_dma,
@@ -2712,7 +2713,7 @@ qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 		    (uint8_t *)els_iocb,
 		    sizeof(*els_iocb));
 	} else {
-		els_iocb->control_flags = 1 << 13;
+		els_iocb->control_flags = cpu_to_le16(1 << 13);
 		els_iocb->tx_byte_count =
 			cpu_to_le32(sizeof(struct els_logo_payload));
 		put_unaligned_le64(elsio->u.els_logo.els_logo_pyld_dma,
@@ -2787,7 +2788,7 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
 	struct qla_work_evt *e;
 	struct fc_port *conflict_fcport;
 	port_id_t cid;	/* conflict Nport id */
-	u32 *fw_status = sp->u.iocb_cmd.u.els_plogi.fw_status;
+	const __le32 *fw_status = sp->u.iocb_cmd.u.els_plogi.fw_status;
 	u16 lid;
 
 	ql_dbg(ql_dbg_disc, vha, 0x3072,
@@ -2800,7 +2801,7 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
 	if (sp->flags & SRB_WAKEUP_ON_COMP)
 		complete(&lio->u.els_plogi.comp);
 	else {
-		switch (fw_status[0]) {
+		switch (le32_to_cpu(fw_status[0])) {
 		case CS_DATA_UNDERRUN:
 		case CS_COMPLETE:
 			memset(&ea, 0, sizeof(ea));
@@ -2810,9 +2811,9 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
 			break;
 
 		case CS_IOCB_ERROR:
-			switch (fw_status[1]) {
+			switch (le32_to_cpu(fw_status[1])) {
 			case LSC_SCODE_PORTID_USED:
-				lid = fw_status[2] & 0xffff;
+				lid = le32_to_cpu(fw_status[2]) & 0xffff;
 				qlt_find_sess_invalidate_other(vha,
 				    wwn_to_u64(fcport->port_name),
 				    fcport->d_id, lid, &conflict_fcport);
@@ -2846,9 +2847,11 @@ static void qla2x00_els_dcmd2_sp_done(srb_t *sp, int res)
 				break;
 
 			case LSC_SCODE_NPORT_USED:
-				cid.b.domain = (fw_status[2] >> 16) & 0xff;
-				cid.b.area   = (fw_status[2] >>  8) & 0xff;
-				cid.b.al_pa  = fw_status[2] & 0xff;
+				cid.b.domain = (le32_to_cpu(fw_status[2]) >> 16)
+					& 0xff;
+				cid.b.area   = (le32_to_cpu(fw_status[2]) >>  8)
+					& 0xff;
+				cid.b.al_pa  = le32_to_cpu(fw_status[2]) & 0xff;
 				cid.b.rsvd_1 = 0;
 
 				ql_dbg(ql_dbg_disc, vha, 0x20ec,
@@ -3022,7 +3025,7 @@ qla24xx_els_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
         els_iocb->sys_define = 0;
         els_iocb->entry_status = 0;
         els_iocb->handle = sp->handle;
-        els_iocb->nport_handle = cpu_to_le16(sp->fcport->loop_id);
+	els_iocb->nport_handle = cpu_to_le16(sp->fcport->loop_id);
 	els_iocb->tx_dsd_count = cpu_to_le16(bsg_job->request_payload.sg_cnt);
 	els_iocb->vp_index = sp->vha->vp_idx;
         els_iocb->sof_type = EST_SOFI3;
@@ -3216,7 +3219,7 @@ qla82xx_start_scsi(srb_t *sp)
 	uint16_t	tot_dsds;
 	struct device_reg_82xx __iomem *reg;
 	uint32_t dbval;
-	uint32_t *fcp_dl;
+	__be32 *fcp_dl;
 	uint8_t additional_cdb_len;
 	struct ct6_dsd *ctx;
 	struct scsi_qla_host *vha = sp->vha;
@@ -3310,7 +3313,7 @@ sufficient_dsds:
 		req_cnt = 1;
 
 		if (req->cnt < (req_cnt + 2)) {
-			cnt = (uint16_t)RD_REG_DWORD_RELAXED(
+			cnt = (uint16_t)rd_reg_dword_relaxed(
 				&reg->req_q_out[0]);
 			if (req->ring_index < cnt)
 				req->cnt = cnt - req->ring_index;
@@ -3398,7 +3401,7 @@ sufficient_dsds:
 
 		memcpy(ctx->fcp_cmnd->cdb, cmd->cmnd, cmd->cmd_len);
 
-		fcp_dl = (uint32_t *)(ctx->fcp_cmnd->cdb + 16 +
+		fcp_dl = (__be32 *)(ctx->fcp_cmnd->cdb + 16 +
 		    additional_cdb_len);
 		*fcp_dl = htonl((uint32_t)scsi_bufflen(cmd));
 
@@ -3419,7 +3422,7 @@ sufficient_dsds:
 
 		req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
 		if (req->cnt < (req_cnt + 2)) {
-			cnt = (uint16_t)RD_REG_DWORD_RELAXED(
+			cnt = (uint16_t)rd_reg_dword_relaxed(
 			    &reg->req_q_out[0]);
 			if (req->ring_index < cnt)
 				req->cnt = cnt - req->ring_index;
@@ -3495,10 +3498,10 @@ sufficient_dsds:
 	if (ql2xdbwr)
 		qla82xx_wr_32(ha, (uintptr_t __force)ha->nxdb_wr_ptr, dbval);
 	else {
-		WRT_REG_DWORD(ha->nxdb_wr_ptr, dbval);
+		wrt_reg_dword(ha->nxdb_wr_ptr, dbval);
 		wmb();
-		while (RD_REG_DWORD(ha->nxdb_rd_ptr) != dbval) {
-			WRT_REG_DWORD(ha->nxdb_wr_ptr, dbval);
+		while (rd_reg_dword(ha->nxdb_rd_ptr) != dbval) {
+			wrt_reg_dword(ha->nxdb_wr_ptr, dbval);
 			wmb();
 		}
 	}
@@ -3536,7 +3539,7 @@ qla24xx_abort_iocb(srb_t *sp, struct abort_entry_24xx *abt_iocb)
 	memset(abt_iocb, 0, sizeof(struct abort_entry_24xx));
 	abt_iocb->entry_type = ABORT_IOCB_TYPE;
 	abt_iocb->entry_count = 1;
-	abt_iocb->handle = cpu_to_le32(make_handle(req->id, sp->handle));
+	abt_iocb->handle = make_handle(req->id, sp->handle);
 	if (sp->fcport) {
 		abt_iocb->nport_handle = cpu_to_le16(sp->fcport->loop_id);
 		abt_iocb->port_id[0] = sp->fcport->d_id.b.al_pa;
@@ -3544,10 +3547,10 @@ qla24xx_abort_iocb(srb_t *sp, struct abort_entry_24xx *abt_iocb)
 		abt_iocb->port_id[2] = sp->fcport->d_id.b.domain;
 	}
 	abt_iocb->handle_to_abort =
-	    cpu_to_le32(make_handle(aio->u.abt.req_que_no,
-				    aio->u.abt.cmd_hndl));
+		make_handle(le16_to_cpu(aio->u.abt.req_que_no),
+			    aio->u.abt.cmd_hndl);
 	abt_iocb->vp_index = vha->vp_idx;
-	abt_iocb->req_que_no = cpu_to_le16(aio->u.abt.req_que_no);
+	abt_iocb->req_que_no = aio->u.abt.req_que_no;
 	/* Send the command to the firmware */
 	wmb();
 }
@@ -3562,7 +3565,7 @@ qla2x00_mb_iocb(srb_t *sp, struct mbx_24xx_entry *mbx)
 	sz = min(ARRAY_SIZE(mbx->mb), ARRAY_SIZE(sp->u.iocb_cmd.u.mbx.out_mb));
 
 	for (i = 0; i < sz; i++)
-		mbx->mb[i] = cpu_to_le16(sp->u.iocb_cmd.u.mbx.out_mb[i]);
+		mbx->mb[i] = sp->u.iocb_cmd.u.mbx.out_mb[i];
 }
 
 static void
@@ -3586,7 +3589,7 @@ static void qla2x00_send_notify_ack_iocb(srb_t *sp,
 	nack->u.isp24.nport_handle = ntfy->u.isp24.nport_handle;
 	if (le16_to_cpu(ntfy->u.isp24.status) == IMM_NTFY_ELS) {
 		nack->u.isp24.flags = ntfy->u.isp24.flags &
-			cpu_to_le32(NOTIFY24XX_FLAGS_PUREX_IOCB);
+			cpu_to_le16(NOTIFY24XX_FLAGS_PUREX_IOCB);
 	}
 	nack->u.isp24.srr_rx_id = ntfy->u.isp24.srr_rx_id;
 	nack->u.isp24.status = ntfy->u.isp24.status;
@@ -3604,32 +3607,29 @@ static void qla2x00_send_notify_ack_iocb(srb_t *sp,
 /*
  * Build NVME LS request
  */
-static int
+static void
 qla_nvme_ls(srb_t *sp, struct pt_ls4_request *cmd_pkt)
 {
 	struct srb_iocb *nvme;
-	int     rval = QLA_SUCCESS;
 
 	nvme = &sp->u.iocb_cmd;
 	cmd_pkt->entry_type = PT_LS4_REQUEST;
 	cmd_pkt->entry_count = 1;
-	cmd_pkt->control_flags = CF_LS4_ORIGINATOR << CF_LS4_SHIFT;
+	cmd_pkt->control_flags = cpu_to_le16(CF_LS4_ORIGINATOR << CF_LS4_SHIFT);
 
 	cmd_pkt->timeout = cpu_to_le16(nvme->u.nvme.timeout_sec);
 	cmd_pkt->nport_handle = cpu_to_le16(sp->fcport->loop_id);
 	cmd_pkt->vp_index = sp->fcport->vha->vp_idx;
 
-	cmd_pkt->tx_dseg_count = 1;
-	cmd_pkt->tx_byte_count = nvme->u.nvme.cmd_len;
-	cmd_pkt->dsd[0].length = nvme->u.nvme.cmd_len;
+	cmd_pkt->tx_dseg_count = cpu_to_le16(1);
+	cmd_pkt->tx_byte_count = cpu_to_le32(nvme->u.nvme.cmd_len);
+	cmd_pkt->dsd[0].length = cpu_to_le32(nvme->u.nvme.cmd_len);
 	put_unaligned_le64(nvme->u.nvme.cmd_dma, &cmd_pkt->dsd[0].address);
 
-	cmd_pkt->rx_dseg_count = 1;
-	cmd_pkt->rx_byte_count = nvme->u.nvme.rsp_len;
-	cmd_pkt->dsd[1].length  = nvme->u.nvme.rsp_len;
+	cmd_pkt->rx_dseg_count = cpu_to_le16(1);
+	cmd_pkt->rx_byte_count = cpu_to_le32(nvme->u.nvme.rsp_len);
+	cmd_pkt->dsd[1].length = cpu_to_le32(nvme->u.nvme.rsp_len);
 	put_unaligned_le64(nvme->u.nvme.rsp_dma, &cmd_pkt->dsd[1].address);
-
-	return rval;
 }
 
 static void
@@ -3894,7 +3894,7 @@ qla2x00_start_bidir(srb_t *sp, struct scsi_qla_host *vha, uint32_t tot_dsds)
 	/* Check for room on request queue. */
 	if (req->cnt < req_cnt + 2) {
 		cnt = IS_SHADOW_REG_CAPABLE(ha) ? *req->out_ptr :
-		    RD_REG_DWORD_RELAXED(req->req_q_out);
+		    rd_reg_dword_relaxed(req->req_q_out);
 		if  (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
 		else
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 8a78d395bbc8..cf0800546740 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -89,9 +89,9 @@ qla24xx_process_abts(struct scsi_qla_host *vha, void *pkt)
 	/* terminate exchange */
 	rsp_els->entry_type = ELS_IOCB_TYPE;
 	rsp_els->entry_count = 1;
-	rsp_els->nport_handle = ~0;
+	rsp_els->nport_handle = cpu_to_le16(~0);
 	rsp_els->rx_xchg_address = abts->rx_xch_addr_to_abort;
-	rsp_els->control_flags = EPD_RX_XCHG;
+	rsp_els->control_flags = cpu_to_le16(EPD_RX_XCHG);
 	ql_dbg(ql_dbg_init, vha, 0x0283,
 	    "Sending ELS Response to terminate exchange %#x...\n",
 	    abts->rx_xch_addr_to_abort);
@@ -141,7 +141,7 @@ qla24xx_process_abts(struct scsi_qla_host *vha, void *pkt)
 	abts_rsp->ox_id = abts->ox_id;
 	abts_rsp->payload.ba_acc.aborted_rx_id = abts->rx_id;
 	abts_rsp->payload.ba_acc.aborted_ox_id = abts->ox_id;
-	abts_rsp->payload.ba_acc.high_seq_cnt = ~0;
+	abts_rsp->payload.ba_acc.high_seq_cnt = cpu_to_le16(~0);
 	abts_rsp->rx_xch_addr_to_abort = abts->rx_xch_addr_to_abort;
 	ql_dbg(ql_dbg_init, vha, 0x028b,
 	    "Sending BA ACC response to ABTS %#x...\n",
@@ -204,7 +204,7 @@ qla2100_intr_handler(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
 	for (iter = 50; iter--; ) {
-		hccr = RD_REG_WORD(&reg->hccr);
+		hccr = rd_reg_word(&reg->hccr);
 		if (qla2x00_check_reg16_for_disconnect(vha, hccr))
 			break;
 		if (hccr & HCCR_RISC_PAUSE) {
@@ -216,18 +216,18 @@ qla2100_intr_handler(int irq, void *dev_id)
 			 * bit to be cleared.  Schedule a big hammer to get
 			 * out of the RISC PAUSED state.
 			 */
-			WRT_REG_WORD(&reg->hccr, HCCR_RESET_RISC);
-			RD_REG_WORD(&reg->hccr);
+			wrt_reg_word(&reg->hccr, HCCR_RESET_RISC);
+			rd_reg_word(&reg->hccr);
 
-			ha->isp_ops->fw_dump(vha, 1);
+			ha->isp_ops->fw_dump(vha);
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 			break;
-		} else if ((RD_REG_WORD(&reg->istatus) & ISR_RISC_INT) == 0)
+		} else if ((rd_reg_word(&reg->istatus) & ISR_RISC_INT) == 0)
 			break;
 
-		if (RD_REG_WORD(&reg->semaphore) & BIT_0) {
-			WRT_REG_WORD(&reg->hccr, HCCR_CLR_RISC_INT);
-			RD_REG_WORD(&reg->hccr);
+		if (rd_reg_word(&reg->semaphore) & BIT_0) {
+			wrt_reg_word(&reg->hccr, HCCR_CLR_RISC_INT);
+			rd_reg_word(&reg->hccr);
 
 			/* Get mailbox data. */
 			mb[0] = RD_MAILBOX_REG(ha, reg, 0);
@@ -246,13 +246,13 @@ qla2100_intr_handler(int irq, void *dev_id)
 				    mb[0]);
 			}
 			/* Release mailbox registers. */
-			WRT_REG_WORD(&reg->semaphore, 0);
-			RD_REG_WORD(&reg->semaphore);
+			wrt_reg_word(&reg->semaphore, 0);
+			rd_reg_word(&reg->semaphore);
 		} else {
 			qla2x00_process_response_queue(rsp);
 
-			WRT_REG_WORD(&reg->hccr, HCCR_CLR_RISC_INT);
-			RD_REG_WORD(&reg->hccr);
+			wrt_reg_word(&reg->hccr, HCCR_CLR_RISC_INT);
+			rd_reg_word(&reg->hccr);
 		}
 	}
 	qla2x00_handle_mbx_completion(ha, status);
@@ -324,14 +324,14 @@ qla2300_intr_handler(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
 	for (iter = 50; iter--; ) {
-		stat = RD_REG_DWORD(&reg->u.isp2300.host_status);
+		stat = rd_reg_dword(&reg->u.isp2300.host_status);
 		if (qla2x00_check_reg32_for_disconnect(vha, stat))
 			break;
 		if (stat & HSR_RISC_PAUSED) {
 			if (unlikely(pci_channel_offline(ha->pdev)))
 				break;
 
-			hccr = RD_REG_WORD(&reg->hccr);
+			hccr = rd_reg_word(&reg->hccr);
 
 			if (hccr & (BIT_15 | BIT_13 | BIT_11 | BIT_8))
 				ql_log(ql_log_warn, vha, 0x5026,
@@ -347,10 +347,10 @@ qla2300_intr_handler(int irq, void *dev_id)
 			 * interrupt bit to be cleared.  Schedule a big
 			 * hammer to get out of the RISC PAUSED state.
 			 */
-			WRT_REG_WORD(&reg->hccr, HCCR_RESET_RISC);
-			RD_REG_WORD(&reg->hccr);
+			wrt_reg_word(&reg->hccr, HCCR_RESET_RISC);
+			rd_reg_word(&reg->hccr);
 
-			ha->isp_ops->fw_dump(vha, 1);
+			ha->isp_ops->fw_dump(vha);
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 			break;
 		} else if ((stat & HSR_RISC_INT) == 0)
@@ -365,7 +365,7 @@ qla2300_intr_handler(int irq, void *dev_id)
 			status |= MBX_INTERRUPT;
 
 			/* Release mailbox registers. */
-			WRT_REG_WORD(&reg->semaphore, 0);
+			wrt_reg_word(&reg->semaphore, 0);
 			break;
 		case 0x12:
 			mb[0] = MSW(stat);
@@ -393,8 +393,8 @@ qla2300_intr_handler(int irq, void *dev_id)
 			    "Unrecognized interrupt type (%d).\n", stat & 0xff);
 			break;
 		}
-		WRT_REG_WORD(&reg->hccr, HCCR_CLR_RISC_INT);
-		RD_REG_WORD_RELAXED(&reg->hccr);
+		wrt_reg_word(&reg->hccr, HCCR_CLR_RISC_INT);
+		rd_reg_word_relaxed(&reg->hccr);
 	}
 	qla2x00_handle_mbx_completion(ha, status);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -412,7 +412,7 @@ qla2x00_mbx_completion(scsi_qla_host_t *vha, uint16_t mb0)
 {
 	uint16_t	cnt;
 	uint32_t	mboxes;
-	uint16_t __iomem *wptr;
+	__le16 __iomem *wptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
@@ -428,15 +428,15 @@ qla2x00_mbx_completion(scsi_qla_host_t *vha, uint16_t mb0)
 	ha->flags.mbox_int = 1;
 	ha->mailbox_out[0] = mb0;
 	mboxes >>= 1;
-	wptr = (uint16_t __iomem *)MAILBOX_REG(ha, reg, 1);
+	wptr = MAILBOX_REG(ha, reg, 1);
 
 	for (cnt = 1; cnt < ha->mbx_count; cnt++) {
 		if (IS_QLA2200(ha) && cnt == 8)
-			wptr = (uint16_t __iomem *)MAILBOX_REG(ha, reg, 8);
+			wptr = MAILBOX_REG(ha, reg, 8);
 		if ((cnt == 4 || cnt == 5) && (mboxes & BIT_0))
 			ha->mailbox_out[cnt] = qla2x00_debounce_register(wptr);
 		else if (mboxes & BIT_0)
-			ha->mailbox_out[cnt] = RD_REG_WORD(wptr);
+			ha->mailbox_out[cnt] = rd_reg_word(wptr);
 
 		wptr++;
 		mboxes >>= 1;
@@ -451,19 +451,19 @@ qla81xx_idc_event(scsi_qla_host_t *vha, uint16_t aen, uint16_t descr)
 	int rval;
 	struct device_reg_24xx __iomem *reg24 = &vha->hw->iobase->isp24;
 	struct device_reg_82xx __iomem *reg82 = &vha->hw->iobase->isp82;
-	uint16_t __iomem *wptr;
+	__le16 __iomem *wptr;
 	uint16_t cnt, timeout, mb[QLA_IDC_ACK_REGS];
 
 	/* Seed data -- mailbox1 -> mailbox7. */
 	if (IS_QLA81XX(vha->hw) || IS_QLA83XX(vha->hw))
-		wptr = (uint16_t __iomem *)&reg24->mailbox1;
+		wptr = &reg24->mailbox1;
 	else if (IS_QLA8044(vha->hw))
-		wptr = (uint16_t __iomem *)&reg82->mailbox_out[1];
+		wptr = &reg82->mailbox_out[1];
 	else
 		return;
 
 	for (cnt = 0; cnt < QLA_IDC_ACK_REGS; cnt++, wptr++)
-		mb[cnt] = RD_REG_WORD(wptr);
+		mb[cnt] = rd_reg_word(wptr);
 
 	ql_dbg(ql_dbg_async, vha, 0x5021,
 	    "Inter-Driver Communication %s -- "
@@ -756,6 +756,39 @@ qla2x00_find_fcport_by_nportid(scsi_qla_host_t *vha, port_id_t *id,
 	return NULL;
 }
 
+/* Shall be called only on supported adapters. */
+static void
+qla27xx_handle_8200_aen(scsi_qla_host_t *vha, uint16_t *mb)
+{
+	struct qla_hw_data *ha = vha->hw;
+	bool reset_isp_needed = 0;
+
+	ql_log(ql_log_warn, vha, 0x02f0,
+	       "MPI Heartbeat stop. MPI reset is%s needed. "
+	       "MB0[%xh] MB1[%xh] MB2[%xh] MB3[%xh]\n",
+	       mb[0] & BIT_8 ? "" : " not",
+	       mb[0], mb[1], mb[2], mb[3]);
+
+	if ((mb[1] & BIT_8) == 0)
+		return;
+
+	ql_log(ql_log_warn, vha, 0x02f1,
+	       "MPI Heartbeat stop. FW dump needed\n");
+
+	if (ql2xfulldump_on_mpifail) {
+		ha->isp_ops->fw_dump(vha);
+		reset_isp_needed = 1;
+	}
+
+	ha->isp_ops->mpi_fw_dump(vha, 1);
+
+	if (reset_isp_needed) {
+		vha->hw->flags.fw_init_done = 0;
+		set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
+		qla2xxx_wake_dpc(vha);
+	}
+}
+
 /**
  * qla2x00_async_event() - Process aynchronous events.
  * @vha: SCSI driver HA context
@@ -785,7 +818,7 @@ qla2x00_async_event(scsi_qla_host_t *vha, struct rsp_que *rsp, uint16_t *mb)
 		goto skip_rio;
 	switch (mb[0]) {
 	case MBA_SCSI_COMPLETION:
-		handles[0] = le32_to_cpu((uint32_t)((mb[2] << 16) | mb[1]));
+		handles[0] = make_handle(mb[2], mb[1]);
 		handle_cnt = 1;
 		break;
 	case MBA_CMPLT_1_16BIT:
@@ -824,10 +857,9 @@ qla2x00_async_event(scsi_qla_host_t *vha, struct rsp_que *rsp, uint16_t *mb)
 		mb[0] = MBA_SCSI_COMPLETION;
 		break;
 	case MBA_CMPLT_2_32BIT:
-		handles[0] = le32_to_cpu((uint32_t)((mb[2] << 16) | mb[1]));
-		handles[1] = le32_to_cpu(
-		    ((uint32_t)(RD_MAILBOX_REG(ha, reg, 7) << 16)) |
-		    RD_MAILBOX_REG(ha, reg, 6));
+		handles[0] = make_handle(mb[2], mb[1]);
+		handles[1] = make_handle(RD_MAILBOX_REG(ha, reg, 7),
+					 RD_MAILBOX_REG(ha, reg, 6));
 		handle_cnt = 2;
 		mb[0] = MBA_SCSI_COMPLETION;
 		break;
@@ -858,10 +890,10 @@ skip_rio:
 		    IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			u16 m[4];
 
-			m[0] = RD_REG_WORD(&reg24->mailbox4);
-			m[1] = RD_REG_WORD(&reg24->mailbox5);
-			m[2] = RD_REG_WORD(&reg24->mailbox6);
-			mbx = m[3] = RD_REG_WORD(&reg24->mailbox7);
+			m[0] = rd_reg_word(&reg24->mailbox4);
+			m[1] = rd_reg_word(&reg24->mailbox5);
+			m[2] = rd_reg_word(&reg24->mailbox6);
+			mbx = m[3] = rd_reg_word(&reg24->mailbox7);
 
 			ql_log(ql_log_warn, vha, 0x5003,
 			    "ISP System Error - mbx1=%xh mbx2=%xh mbx3=%xh mbx4=%xh mbx5=%xh mbx6=%xh mbx7=%xh.\n",
@@ -871,10 +903,10 @@ skip_rio:
 			    "ISP System Error - mbx1=%xh mbx2=%xh mbx3=%xh.\n ",
 			    mb[1], mb[2], mb[3]);
 
-		ha->fw_dump_mpi =
-		    (IS_QLA27XX(ha) || IS_QLA28XX(ha)) &&
-		    RD_REG_WORD(&reg24->mailbox7) & BIT_8;
-		ha->isp_ops->fw_dump(vha, 1);
+		if ((IS_QLA27XX(ha) || IS_QLA28XX(ha)) &&
+		    rd_reg_word(&reg24->mailbox7) & BIT_8)
+			ha->isp_ops->mpi_fw_dump(vha, 1);
+		ha->isp_ops->fw_dump(vha);
 		ha->flags.fw_init_done = 0;
 		QLA_FW_STOPPED(ha);
 
@@ -979,8 +1011,8 @@ skip_rio:
 		ha->current_topology = 0;
 
 		mbx = (IS_QLA81XX(ha) || IS_QLA8031(ha))
-			? RD_REG_WORD(&reg24->mailbox4) : 0;
-		mbx = (IS_P3P_TYPE(ha)) ? RD_REG_WORD(&reg82->mailbox_out[4])
+			? rd_reg_word(&reg24->mailbox4) : 0;
+		mbx = (IS_P3P_TYPE(ha)) ? rd_reg_word(&reg82->mailbox_out[4])
 			: mbx;
 		ql_log(ql_log_info, vha, 0x500b,
 		    "LOOP DOWN detected (%x %x %x %x).\n",
@@ -1347,7 +1379,7 @@ global_port_update:
 		break;
 	case MBA_IDC_NOTIFY:
 		if (IS_QLA8031(vha->hw) || IS_QLA8044(ha)) {
-			mb[4] = RD_REG_WORD(&reg24->mailbox4);
+			mb[4] = rd_reg_word(&reg24->mailbox4);
 			if (((mb[2] & 0x7fff) == MBC_PORT_RESET ||
 			    (mb[2] & 0x7fff) == MBC_SET_PORT_CONFIG) &&
 			    (mb[4] & INTERNAL_LOOPBACK_MASK) != 0) {
@@ -1374,25 +1406,12 @@ global_port_update:
 
 	case MBA_IDC_AEN:
 		if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
-			ha->flags.fw_init_done = 0;
-			ql_log(ql_log_warn, vha, 0xffff,
-			    "MPI Heartbeat stop. Chip reset needed. MB0[%xh] MB1[%xh] MB2[%xh] MB3[%xh]\n",
-			    mb[0], mb[1], mb[2], mb[3]);
-
-			if ((mb[1] & BIT_8) ||
-			    (mb[2] & BIT_8)) {
-				ql_log(ql_log_warn, vha, 0xd013,
-				    "MPI Heartbeat stop. FW dump needed\n");
-				ha->fw_dump_mpi = 1;
-				ha->isp_ops->fw_dump(vha, 1);
-			}
-			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
-			qla2xxx_wake_dpc(vha);
+			qla27xx_handle_8200_aen(vha, mb);
 		} else if (IS_QLA83XX(ha)) {
-			mb[4] = RD_REG_WORD(&reg24->mailbox4);
-			mb[5] = RD_REG_WORD(&reg24->mailbox5);
-			mb[6] = RD_REG_WORD(&reg24->mailbox6);
-			mb[7] = RD_REG_WORD(&reg24->mailbox7);
+			mb[4] = rd_reg_word(&reg24->mailbox4);
+			mb[5] = rd_reg_word(&reg24->mailbox5);
+			mb[6] = rd_reg_word(&reg24->mailbox6);
+			mb[7] = rd_reg_word(&reg24->mailbox7);
 			qla83xx_handle_8200_aen(vha, mb);
 		} else {
 			ql_dbg(ql_dbg_async, vha, 0x5052,
@@ -1646,7 +1665,7 @@ qla24xx_mbx_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 	sz = min(ARRAY_SIZE(pkt->mb), ARRAY_SIZE(sp->u.iocb_cmd.u.mbx.in_mb));
 
 	for (i = 0; i < sz; i++)
-		si->u.mbx.in_mb[i] = le16_to_cpu(pkt->mb[i]);
+		si->u.mbx.in_mb[i] = pkt->mb[i];
 
 	res = (si->u.mbx.in_mb[0] & MBS_MASK);
 
@@ -1747,6 +1766,7 @@ static void
 qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
     struct sts_entry_24xx *pkt, int iocb_type)
 {
+	struct els_sts_entry_24xx *ese = (struct els_sts_entry_24xx *)pkt;
 	const char func[] = "ELS_CT_IOCB";
 	const char *type;
 	srb_t *sp;
@@ -1796,23 +1816,22 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	}
 
 	comp_status = fw_status[0] = le16_to_cpu(pkt->comp_status);
-	fw_status[1] = le16_to_cpu(((struct els_sts_entry_24xx *)pkt)->error_subcode_1);
-	fw_status[2] = le16_to_cpu(((struct els_sts_entry_24xx *)pkt)->error_subcode_2);
+	fw_status[1] = le32_to_cpu(ese->error_subcode_1);
+	fw_status[2] = le32_to_cpu(ese->error_subcode_2);
 
 	if (iocb_type == ELS_IOCB_TYPE) {
 		els = &sp->u.iocb_cmd;
-		els->u.els_plogi.fw_status[0] = fw_status[0];
-		els->u.els_plogi.fw_status[1] = fw_status[1];
-		els->u.els_plogi.fw_status[2] = fw_status[2];
-		els->u.els_plogi.comp_status = fw_status[0];
+		els->u.els_plogi.fw_status[0] = cpu_to_le32(fw_status[0]);
+		els->u.els_plogi.fw_status[1] = cpu_to_le32(fw_status[1]);
+		els->u.els_plogi.fw_status[2] = cpu_to_le32(fw_status[2]);
+		els->u.els_plogi.comp_status = cpu_to_le16(fw_status[0]);
 		if (comp_status == CS_COMPLETE) {
 			res =  DID_OK << 16;
 		} else {
 			if (comp_status == CS_DATA_UNDERRUN) {
 				res =  DID_OK << 16;
-				els->u.els_plogi.len =
-				le16_to_cpu(((struct els_sts_entry_24xx *)
-					pkt)->total_byte_count);
+				els->u.els_plogi.len = cpu_to_le16(le32_to_cpu(
+					ese->total_byte_count));
 			} else {
 				els->u.els_plogi.len = 0;
 				res = DID_ERROR << 16;
@@ -1821,8 +1840,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 		ql_dbg(ql_dbg_user, vha, 0x503f,
 		    "ELS IOCB Done -%s error hdl=%x comp_status=0x%x error subcode 1=0x%x error subcode 2=0x%x total_byte=0x%x\n",
 		    type, sp->handle, comp_status, fw_status[1], fw_status[2],
-		    le16_to_cpu(((struct els_sts_entry_24xx *)
-			pkt)->total_byte_count));
+		    le32_to_cpu(ese->total_byte_count));
 		goto els_ct_done;
 	}
 
@@ -1838,23 +1856,20 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 		if (comp_status == CS_DATA_UNDERRUN) {
 			res = DID_OK << 16;
 			bsg_reply->reply_payload_rcv_len =
-			    le16_to_cpu(((struct els_sts_entry_24xx *)pkt)->total_byte_count);
+				le32_to_cpu(ese->total_byte_count);
 
 			ql_dbg(ql_dbg_user, vha, 0x503f,
 			    "ELS-CT pass-through-%s error hdl=%x comp_status-status=0x%x "
 			    "error subcode 1=0x%x error subcode 2=0x%x total_byte = 0x%x.\n",
 			    type, sp->handle, comp_status, fw_status[1], fw_status[2],
-			    le16_to_cpu(((struct els_sts_entry_24xx *)
-				pkt)->total_byte_count));
+			    le32_to_cpu(ese->total_byte_count));
 		} else {
 			ql_dbg(ql_dbg_user, vha, 0x5040,
 			    "ELS-CT pass-through-%s error hdl=%x comp_status-status=0x%x "
 			    "error subcode 1=0x%x error subcode 2=0x%x.\n",
 			    type, sp->handle, comp_status,
-			    le16_to_cpu(((struct els_sts_entry_24xx *)
-				pkt)->error_subcode_1),
-			    le16_to_cpu(((struct els_sts_entry_24xx *)
-				    pkt)->error_subcode_2));
+			    le32_to_cpu(ese->error_subcode_1),
+			    le32_to_cpu(ese->error_subcode_2));
 			res = DID_ERROR << 16;
 			bsg_reply->reply_payload_rcv_len = 0;
 		}
@@ -2062,7 +2077,7 @@ static void qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 	uint16_t        state_flags;
 	struct nvmefc_fcp_req *fd;
 	uint16_t        ret = QLA_SUCCESS;
-	uint16_t	comp_status = le16_to_cpu(sts->comp_status);
+	__le16		comp_status = sts->comp_status;
 	int		logit = 0;
 
 	iocb = &sp->u.iocb_cmd;
@@ -2093,7 +2108,7 @@ static void qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 	} else if ((state_flags & (SF_FCP_RSP_DMA | SF_NVME_ERSP)) ==
 			(SF_FCP_RSP_DMA | SF_NVME_ERSP)) {
 		/* Response already DMA'd to fd->rspaddr. */
-		iocb->u.nvme.rsp_pyld_len = le16_to_cpu(sts->nvme_rsp_pyld_len);
+		iocb->u.nvme.rsp_pyld_len = sts->nvme_rsp_pyld_len;
 	} else if ((state_flags & SF_FCP_RSP_DMA)) {
 		/*
 		 * Non-zero value in first 12 bytes of NVMe_RSP IU, treat this
@@ -2110,8 +2125,8 @@ static void qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 
 		inbuf = (uint32_t *)&sts->nvme_ersp_data;
 		outbuf = (uint32_t *)fd->rspaddr;
-		iocb->u.nvme.rsp_pyld_len = le16_to_cpu(sts->nvme_rsp_pyld_len);
-		if (unlikely(iocb->u.nvme.rsp_pyld_len >
+		iocb->u.nvme.rsp_pyld_len = sts->nvme_rsp_pyld_len;
+		if (unlikely(le16_to_cpu(iocb->u.nvme.rsp_pyld_len) >
 		    sizeof(struct nvme_fc_ersp_iu))) {
 			if (ql_mask_match(ql_dbg_io)) {
 				WARN_ONCE(1, "Unexpected response payload length %u.\n",
@@ -2121,9 +2136,9 @@ static void qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 				    iocb->u.nvme.rsp_pyld_len);
 			}
 			iocb->u.nvme.rsp_pyld_len =
-			    sizeof(struct nvme_fc_ersp_iu);
+				cpu_to_le16(sizeof(struct nvme_fc_ersp_iu));
 		}
-		iter = iocb->u.nvme.rsp_pyld_len >> 2;
+		iter = le16_to_cpu(iocb->u.nvme.rsp_pyld_len) >> 2;
 		for (; iter; iter--)
 			*outbuf++ = swab32(*inbuf++);
 	}
@@ -2138,7 +2153,7 @@ static void qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 				"Dropped frame(s) detected (sent/rcvd=%u/%u).\n",
 				tgt_xfer_len, fd->transferred_length);
 			logit = 1;
-		} else if (comp_status == CS_DATA_UNDERRUN) {
+		} else if (le16_to_cpu(comp_status) == CS_DATA_UNDERRUN) {
 			/*
 			 * Do not log if this is just an underflow and there
 			 * is no data loss.
@@ -2158,7 +2173,7 @@ static void qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 	 * If transport error then Failure (HBA rejects request)
 	 * otherwise transport will handle.
 	 */
-	switch (comp_status) {
+	switch (le16_to_cpu(comp_status)) {
 	case CS_COMPLETE:
 		break;
 
@@ -2300,7 +2315,7 @@ qla2x00_process_response_queue(struct rsp_que *rsp)
 	}
 
 	/* Adjust ring index */
-	WRT_REG_WORD(ISP_RSP_Q_OUT(ha, reg), rsp->ring_index);
+	wrt_reg_word(ISP_RSP_Q_OUT(ha, reg), rsp->ring_index);
 }
 
 static inline void
@@ -2391,9 +2406,9 @@ qla2x00_handle_dif_error(srb_t *sp, struct sts_entry_24xx *sts24)
 	 * For type     3: ref & app tag is all 'f's
 	 * For type 0,1,2: app tag is all 'f's
 	 */
-	if ((a_app_tag == T10_PI_APP_ESCAPE) &&
-	    ((scsi_get_prot_type(cmd) != SCSI_PROT_DIF_TYPE3) ||
-	     (a_ref_tag == T10_PI_REF_ESCAPE))) {
+	if (a_app_tag == be16_to_cpu(T10_PI_APP_ESCAPE) &&
+	    (scsi_get_prot_type(cmd) != SCSI_PROT_DIF_TYPE3 ||
+	     a_ref_tag == be32_to_cpu(T10_PI_REF_ESCAPE))) {
 		uint32_t blocks_done, resid;
 		sector_t lba_s = scsi_get_lba(cmd);
 
@@ -2751,6 +2766,8 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 	sense_len = par_sense_len = rsp_info_len = resid_len =
 	    fw_resid_len = 0;
 	if (IS_FWI2_CAPABLE(ha)) {
+		u16 sts24_retry_delay = le16_to_cpu(sts24->retry_delay);
+
 		if (scsi_status & SS_SENSE_LEN_VALID)
 			sense_len = le32_to_cpu(sts24->sense_len);
 		if (scsi_status & SS_RESPONSE_INFO_LEN_VALID)
@@ -2765,11 +2782,11 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 		ox_id = le16_to_cpu(sts24->ox_id);
 		par_sense_len = sizeof(sts24->data);
 		/* Valid values of the retry delay timer are 0x1-0xffef */
-		if (sts24->retry_delay > 0 && sts24->retry_delay < 0xfff1) {
-			retry_delay = sts24->retry_delay & 0x3fff;
+		if (sts24_retry_delay > 0 && sts24_retry_delay < 0xfff1) {
+			retry_delay = sts24_retry_delay & 0x3fff;
 			ql_dbg(ql_dbg_io, sp->vha, 0x3033,
 			    "%s: scope=%#x retry_delay=%#x\n", __func__,
-			    sts24->retry_delay >> 14, retry_delay);
+			    sts24_retry_delay >> 14, retry_delay);
 		}
 	} else {
 		if (scsi_status & SS_SENSE_LEN_VALID)
@@ -3143,7 +3160,7 @@ qla24xx_mbx_completion(scsi_qla_host_t *vha, uint16_t mb0)
 {
 	uint16_t	cnt;
 	uint32_t	mboxes;
-	uint16_t __iomem *wptr;
+	__le16 __iomem *wptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 
@@ -3159,11 +3176,11 @@ qla24xx_mbx_completion(scsi_qla_host_t *vha, uint16_t mb0)
 	ha->flags.mbox_int = 1;
 	ha->mailbox_out[0] = mb0;
 	mboxes >>= 1;
-	wptr = (uint16_t __iomem *)&reg->mailbox1;
+	wptr = &reg->mailbox1;
 
 	for (cnt = 1; cnt < ha->mbx_count; cnt++) {
 		if (mboxes & BIT_0)
-			ha->mailbox_out[cnt] = RD_REG_WORD(wptr);
+			ha->mailbox_out[cnt] = rd_reg_word(wptr);
 
 		mboxes >>= 1;
 		wptr++;
@@ -3183,7 +3200,7 @@ qla24xx_abort_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 		return;
 
 	abt = &sp->u.iocb_cmd;
-	abt->u.abt.comp_status = le16_to_cpu(pkt->nport_handle);
+	abt->u.abt.comp_status = pkt->nport_handle;
 	sp->done(sp, 0);
 }
 
@@ -3340,9 +3357,9 @@ process_err:
 	if (IS_P3P_TYPE(ha)) {
 		struct device_reg_82xx __iomem *reg = &ha->iobase->isp82;
 
-		WRT_REG_DWORD(&reg->rsp_q_out[0], rsp->ring_index);
+		wrt_reg_dword(&reg->rsp_q_out[0], rsp->ring_index);
 	} else {
-		WRT_REG_DWORD(rsp->rsp_q_out, rsp->ring_index);
+		wrt_reg_dword(rsp->rsp_q_out, rsp->ring_index);
 	}
 }
 
@@ -3359,13 +3376,13 @@ qla2xxx_check_risc_status(scsi_qla_host_t *vha)
 		return;
 
 	rval = QLA_SUCCESS;
-	WRT_REG_DWORD(&reg->iobase_addr, 0x7C00);
-	RD_REG_DWORD(&reg->iobase_addr);
-	WRT_REG_DWORD(&reg->iobase_window, 0x0001);
-	for (cnt = 10000; (RD_REG_DWORD(&reg->iobase_window) & BIT_0) == 0 &&
+	wrt_reg_dword(&reg->iobase_addr, 0x7C00);
+	rd_reg_dword(&reg->iobase_addr);
+	wrt_reg_dword(&reg->iobase_window, 0x0001);
+	for (cnt = 10000; (rd_reg_dword(&reg->iobase_window) & BIT_0) == 0 &&
 	    rval == QLA_SUCCESS; cnt--) {
 		if (cnt) {
-			WRT_REG_DWORD(&reg->iobase_window, 0x0001);
+			wrt_reg_dword(&reg->iobase_window, 0x0001);
 			udelay(10);
 		} else
 			rval = QLA_FUNCTION_TIMEOUT;
@@ -3374,11 +3391,11 @@ qla2xxx_check_risc_status(scsi_qla_host_t *vha)
 		goto next_test;
 
 	rval = QLA_SUCCESS;
-	WRT_REG_DWORD(&reg->iobase_window, 0x0003);
-	for (cnt = 100; (RD_REG_DWORD(&reg->iobase_window) & BIT_0) == 0 &&
+	wrt_reg_dword(&reg->iobase_window, 0x0003);
+	for (cnt = 100; (rd_reg_dword(&reg->iobase_window) & BIT_0) == 0 &&
 	    rval == QLA_SUCCESS; cnt--) {
 		if (cnt) {
-			WRT_REG_DWORD(&reg->iobase_window, 0x0003);
+			wrt_reg_dword(&reg->iobase_window, 0x0003);
 			udelay(10);
 		} else
 			rval = QLA_FUNCTION_TIMEOUT;
@@ -3387,13 +3404,13 @@ qla2xxx_check_risc_status(scsi_qla_host_t *vha)
 		goto done;
 
 next_test:
-	if (RD_REG_DWORD(&reg->iobase_c8) & BIT_3)
+	if (rd_reg_dword(&reg->iobase_c8) & BIT_3)
 		ql_log(ql_log_info, vha, 0x504c,
 		    "Additional code -- 0x55AA.\n");
 
 done:
-	WRT_REG_DWORD(&reg->iobase_window, 0x0000);
-	RD_REG_DWORD(&reg->iobase_window);
+	wrt_reg_dword(&reg->iobase_window, 0x0000);
+	rd_reg_dword(&reg->iobase_window);
 }
 
 /**
@@ -3437,14 +3454,14 @@ qla24xx_intr_handler(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
 	for (iter = 50; iter--; ) {
-		stat = RD_REG_DWORD(&reg->host_status);
+		stat = rd_reg_dword(&reg->host_status);
 		if (qla2x00_check_reg32_for_disconnect(vha, stat))
 			break;
 		if (stat & HSRX_RISC_PAUSED) {
 			if (unlikely(pci_channel_offline(ha->pdev)))
 				break;
 
-			hccr = RD_REG_DWORD(&reg->hccr);
+			hccr = rd_reg_dword(&reg->hccr);
 
 			ql_log(ql_log_warn, vha, 0x504b,
 			    "RISC paused -- HCCR=%x, Dumping firmware.\n",
@@ -3452,7 +3469,7 @@ qla24xx_intr_handler(int irq, void *dev_id)
 
 			qla2xxx_check_risc_status(vha);
 
-			ha->isp_ops->fw_dump(vha, 1);
+			ha->isp_ops->fw_dump(vha);
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 			break;
 		} else if ((stat & HSRX_RISC_INT) == 0)
@@ -3469,9 +3486,9 @@ qla24xx_intr_handler(int irq, void *dev_id)
 			break;
 		case INTR_ASYNC_EVENT:
 			mb[0] = MSW(stat);
-			mb[1] = RD_REG_WORD(&reg->mailbox1);
-			mb[2] = RD_REG_WORD(&reg->mailbox2);
-			mb[3] = RD_REG_WORD(&reg->mailbox3);
+			mb[1] = rd_reg_word(&reg->mailbox1);
+			mb[2] = rd_reg_word(&reg->mailbox2);
+			mb[3] = rd_reg_word(&reg->mailbox3);
 			qla2x00_async_event(vha, rsp, mb);
 			break;
 		case INTR_RSP_QUE_UPDATE:
@@ -3491,8 +3508,8 @@ qla24xx_intr_handler(int irq, void *dev_id)
 			    "Unrecognized interrupt type (%d).\n", stat * 0xff);
 			break;
 		}
-		WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
-		RD_REG_DWORD_RELAXED(&reg->hccr);
+		wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
+		rd_reg_dword_relaxed(&reg->hccr);
 		if (unlikely(IS_QLA83XX(ha) && (ha->pdev->revision == 1)))
 			ndelay(3500);
 	}
@@ -3531,8 +3548,8 @@ qla24xx_msix_rsp_q(int irq, void *dev_id)
 	vha = pci_get_drvdata(ha->pdev);
 	qla24xx_process_response_queue(vha, rsp);
 	if (!ha->flags.disable_msix_handshake) {
-		WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
-		RD_REG_DWORD_RELAXED(&reg->hccr);
+		wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
+		rd_reg_dword_relaxed(&reg->hccr);
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
@@ -3566,14 +3583,14 @@ qla24xx_msix_default(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
 	do {
-		stat = RD_REG_DWORD(&reg->host_status);
+		stat = rd_reg_dword(&reg->host_status);
 		if (qla2x00_check_reg32_for_disconnect(vha, stat))
 			break;
 		if (stat & HSRX_RISC_PAUSED) {
 			if (unlikely(pci_channel_offline(ha->pdev)))
 				break;
 
-			hccr = RD_REG_DWORD(&reg->hccr);
+			hccr = rd_reg_dword(&reg->hccr);
 
 			ql_log(ql_log_info, vha, 0x5050,
 			    "RISC paused -- HCCR=%x, Dumping firmware.\n",
@@ -3581,7 +3598,7 @@ qla24xx_msix_default(int irq, void *dev_id)
 
 			qla2xxx_check_risc_status(vha);
 
-			ha->isp_ops->fw_dump(vha, 1);
+			ha->isp_ops->fw_dump(vha);
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 			break;
 		} else if ((stat & HSRX_RISC_INT) == 0)
@@ -3598,9 +3615,9 @@ qla24xx_msix_default(int irq, void *dev_id)
 			break;
 		case INTR_ASYNC_EVENT:
 			mb[0] = MSW(stat);
-			mb[1] = RD_REG_WORD(&reg->mailbox1);
-			mb[2] = RD_REG_WORD(&reg->mailbox2);
-			mb[3] = RD_REG_WORD(&reg->mailbox3);
+			mb[1] = rd_reg_word(&reg->mailbox1);
+			mb[2] = rd_reg_word(&reg->mailbox2);
+			mb[3] = rd_reg_word(&reg->mailbox3);
 			qla2x00_async_event(vha, rsp, mb);
 			break;
 		case INTR_RSP_QUE_UPDATE:
@@ -3620,7 +3637,7 @@ qla24xx_msix_default(int irq, void *dev_id)
 			    "Unrecognized interrupt type (%d).\n", stat & 0xff);
 			break;
 		}
-		WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
+		wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
 	} while (0);
 	qla2x00_handle_mbx_completion(ha, status);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -3671,7 +3688,7 @@ qla2xxx_msix_rsp_q_hs(int irq, void *dev_id)
 
 	reg = &ha->iobase->isp24;
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
+	wrt_reg_dword(&reg->hccr, HCCRX_CLR_RISC_INT);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	queue_work(ha->wq, &qpair->q_work);
@@ -3932,7 +3949,7 @@ clear_risc_ints:
 		goto fail;
 
 	spin_lock_irq(&ha->hardware_lock);
-	WRT_REG_WORD(&reg->isp.semaphore, 0);
+	wrt_reg_word(&reg->isp.semaphore, 0);
 	spin_unlock_irq(&ha->hardware_lock);
 
 fail:
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index d6c991bd1bde..df31ee0d59b2 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -106,7 +106,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 	uint8_t		io_lock_on;
 	uint16_t	command = 0;
 	uint16_t	*iptr;
-	uint16_t __iomem *optr;
+	__le16 __iomem  *optr;
 	uint32_t	cnt;
 	uint32_t	mboxes;
 	unsigned long	wait_time;
@@ -208,11 +208,11 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 
 	/* Load mailbox registers. */
 	if (IS_P3P_TYPE(ha))
-		optr = (uint16_t __iomem *)&reg->isp82.mailbox_in[0];
+		optr = &reg->isp82.mailbox_in[0];
 	else if (IS_FWI2_CAPABLE(ha) && !(IS_P3P_TYPE(ha)))
-		optr = (uint16_t __iomem *)&reg->isp24.mailbox0;
+		optr = &reg->isp24.mailbox0;
 	else
-		optr = (uint16_t __iomem *)MAILBOX_REG(ha, &reg->isp, 0);
+		optr = MAILBOX_REG(ha, &reg->isp, 0);
 
 	iptr = mcp->mb;
 	command = mcp->mb[0];
@@ -222,12 +222,11 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 	    "Mailbox registers (OUT):\n");
 	for (cnt = 0; cnt < ha->mbx_count; cnt++) {
 		if (IS_QLA2200(ha) && cnt == 8)
-			optr =
-			    (uint16_t __iomem *)MAILBOX_REG(ha, &reg->isp, 8);
+			optr = MAILBOX_REG(ha, &reg->isp, 8);
 		if (mboxes & BIT_0) {
 			ql_dbg(ql_dbg_mbx, vha, 0x1112,
 			    "mbox[%d]<-0x%04x\n", cnt, *iptr);
-			WRT_REG_WORD(optr, *iptr);
+			wrt_reg_word(optr, *iptr);
 		}
 
 		mboxes >>= 1;
@@ -253,11 +252,11 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 		set_bit(MBX_INTR_WAIT, &ha->mbx_cmd_flags);
 
 		if (IS_P3P_TYPE(ha))
-			WRT_REG_DWORD(&reg->isp82.hint, HINT_MBX_INT_PENDING);
+			wrt_reg_dword(&reg->isp82.hint, HINT_MBX_INT_PENDING);
 		else if (IS_FWI2_CAPABLE(ha))
-			WRT_REG_DWORD(&reg->isp24.hccr, HCCRX_SET_HOST_INT);
+			wrt_reg_dword(&reg->isp24.hccr, HCCRX_SET_HOST_INT);
 		else
-			WRT_REG_WORD(&reg->isp.hccr, HCCR_SET_HOST_INT);
+			wrt_reg_word(&reg->isp.hccr, HCCR_SET_HOST_INT);
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 		wait_time = jiffies;
@@ -300,7 +299,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 		    "Cmd=%x Polling Mode.\n", command);
 
 		if (IS_P3P_TYPE(ha)) {
-			if (RD_REG_DWORD(&reg->isp82.hint) &
+			if (rd_reg_dword(&reg->isp82.hint) &
 				HINT_MBX_INT_PENDING) {
 				ha->flags.mbox_busy = 0;
 				spin_unlock_irqrestore(&ha->hardware_lock,
@@ -311,11 +310,11 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 				rval = QLA_FUNCTION_TIMEOUT;
 				goto premature_exit;
 			}
-			WRT_REG_DWORD(&reg->isp82.hint, HINT_MBX_INT_PENDING);
+			wrt_reg_dword(&reg->isp82.hint, HINT_MBX_INT_PENDING);
 		} else if (IS_FWI2_CAPABLE(ha))
-			WRT_REG_DWORD(&reg->isp24.hccr, HCCRX_SET_HOST_INT);
+			wrt_reg_dword(&reg->isp24.hccr, HCCRX_SET_HOST_INT);
 		else
-			WRT_REG_WORD(&reg->isp.hccr, HCCR_SET_HOST_INT);
+			wrt_reg_word(&reg->isp.hccr, HCCR_SET_HOST_INT);
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 		wait_time = jiffies + mcp->tov * HZ; /* wait at most tov secs */
@@ -413,14 +412,14 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 		uint16_t        w;
 
 		if (IS_FWI2_CAPABLE(ha)) {
-			mb[0] = RD_REG_WORD(&reg->isp24.mailbox0);
-			mb[1] = RD_REG_WORD(&reg->isp24.mailbox1);
-			mb[2] = RD_REG_WORD(&reg->isp24.mailbox2);
-			mb[3] = RD_REG_WORD(&reg->isp24.mailbox3);
-			mb[7] = RD_REG_WORD(&reg->isp24.mailbox7);
-			ictrl = RD_REG_DWORD(&reg->isp24.ictrl);
-			host_status = RD_REG_DWORD(&reg->isp24.host_status);
-			hccr = RD_REG_DWORD(&reg->isp24.hccr);
+			mb[0] = rd_reg_word(&reg->isp24.mailbox0);
+			mb[1] = rd_reg_word(&reg->isp24.mailbox1);
+			mb[2] = rd_reg_word(&reg->isp24.mailbox2);
+			mb[3] = rd_reg_word(&reg->isp24.mailbox3);
+			mb[7] = rd_reg_word(&reg->isp24.mailbox7);
+			ictrl = rd_reg_dword(&reg->isp24.ictrl);
+			host_status = rd_reg_dword(&reg->isp24.host_status);
+			hccr = rd_reg_dword(&reg->isp24.hccr);
 
 			ql_log(ql_log_warn, vha, 0xd04c,
 			    "MBX Command timeout for cmd %x, iocontrol=%x jiffies=%lx "
@@ -430,7 +429,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 
 		} else {
 			mb[0] = RD_MAILBOX_REG(ha, &reg->isp, 0);
-			ictrl = RD_REG_WORD(&reg->isp.ictrl);
+			ictrl = rd_reg_word(&reg->isp.ictrl);
 			ql_dbg(ql_dbg_mbx + ql_dbg_buffer, vha, 0x1119,
 			    "MBX Command timeout for cmd %x, iocontrol=%x jiffies=%lx "
 			    "mb[0]=0x%x\n", command, ictrl, jiffies, mb[0]);
@@ -462,7 +461,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 			 * a dump
 			 */
 			if (mcp->mb[0] != MBC_GEN_SYSTEM_ERROR)
-				ha->isp_ops->fw_dump(vha, 0);
+				qla2xxx_dump_fw(vha);
 			rval = QLA_FUNCTION_TIMEOUT;
 		 }
 	}
@@ -573,15 +572,15 @@ mbx_done:
 		if (IS_FWI2_CAPABLE(ha) && !(IS_P3P_TYPE(ha))) {
 			ql_dbg(ql_dbg_mbx, vha, 0x1198,
 			    "host_status=%#x intr_ctrl=%#x intr_status=%#x\n",
-			    RD_REG_DWORD(&reg->isp24.host_status),
-			    RD_REG_DWORD(&reg->isp24.ictrl),
-			    RD_REG_DWORD(&reg->isp24.istatus));
+			    rd_reg_dword(&reg->isp24.host_status),
+			    rd_reg_dword(&reg->isp24.ictrl),
+			    rd_reg_dword(&reg->isp24.istatus));
 		} else {
 			ql_dbg(ql_dbg_mbx, vha, 0x1206,
 			    "ctrl_status=%#x ictrl=%#x istatus=%#x\n",
-			    RD_REG_WORD(&reg->isp.ctrl_status),
-			    RD_REG_WORD(&reg->isp.ictrl),
-			    RD_REG_WORD(&reg->isp.istatus));
+			    rd_reg_word(&reg->isp.ctrl_status),
+			    rd_reg_word(&reg->isp.ictrl),
+			    rd_reg_word(&reg->isp.istatus));
 		}
 	} else {
 		ql_dbg(ql_dbg_mbx, base_vha, 0x1021, "Done %s.\n", __func__);
@@ -3038,7 +3037,7 @@ qla2x00_get_link_status(scsi_qla_host_t *vha, uint16_t loop_id,
 	int rval;
 	mbx_cmd_t mc;
 	mbx_cmd_t *mcp = &mc;
-	uint32_t *iter = (void *)stats;
+	uint32_t *iter = (uint32_t *)stats;
 	ushort dwords = offsetof(typeof(*stats), link_up_cnt)/sizeof(*iter);
 	struct qla_hw_data *ha = vha->hw;
 
@@ -3097,7 +3096,7 @@ qla24xx_get_isp_stats(scsi_qla_host_t *vha, struct link_statistics *stats,
 	int rval;
 	mbx_cmd_t mc;
 	mbx_cmd_t *mcp = &mc;
-	uint32_t *iter = (void *)stats;
+	uint32_t *iter = (uint32_t *)stats;
 	ushort dwords = sizeof(*stats)/sizeof(*iter);
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1088,
@@ -3110,8 +3109,8 @@ qla24xx_get_isp_stats(scsi_qla_host_t *vha, struct link_statistics *stats,
 	mc.mb[6] = MSW(MSD(stats_dma));
 	mc.mb[7] = LSW(MSD(stats_dma));
 	mc.mb[8] = dwords;
-	mc.mb[9] = cpu_to_le16(vha->vp_idx);
-	mc.mb[10] = cpu_to_le16(options);
+	mc.mb[9] = vha->vp_idx;
+	mc.mb[10] = options;
 
 	rval = qla24xx_send_mb_cmd(vha, &mc);
 
@@ -3204,7 +3203,7 @@ qla24xx_abort_command(srb_t *sp)
 		ql_dbg(ql_dbg_mbx, vha, 0x1090,
 		    "Failed to complete IOCB -- completion status (%x).\n",
 		    le16_to_cpu(abt->nport_handle));
-		if (abt->nport_handle == CS_IOCB_ERROR)
+		if (abt->nport_handle == cpu_to_le16(CS_IOCB_ERROR))
 			rval = QLA_FUNCTION_PARAMETER_ERROR;
 		else
 			rval = QLA_FUNCTION_FAILED;
@@ -4427,9 +4426,9 @@ qla25xx_init_req_que(struct scsi_qla_host *vha, struct req_que *req)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	if (!(req->options & BIT_0)) {
-		WRT_REG_DWORD(req->req_q_in, 0);
+		wrt_reg_dword(req->req_q_in, 0);
 		if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
-			WRT_REG_DWORD(req->req_q_out, 0);
+			wrt_reg_dword(req->req_q_out, 0);
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
@@ -4498,9 +4497,9 @@ qla25xx_init_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	if (!(rsp->options & BIT_0)) {
-		WRT_REG_DWORD(rsp->rsp_q_out, 0);
+		wrt_reg_dword(rsp->rsp_q_out, 0);
 		if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
-			WRT_REG_DWORD(rsp->rsp_q_in, 0);
+			wrt_reg_dword(rsp->rsp_q_in, 0);
 	}
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -4727,7 +4726,7 @@ qla82xx_set_driver_version(scsi_qla_host_t *vha, char *version)
 	mbx_cmd_t *mcp = &mc;
 	int i;
 	int len;
-	uint16_t *str;
+	__le16 *str;
 	struct qla_hw_data *ha = vha->hw;
 
 	if (!IS_P3P_TYPE(ha))
@@ -4736,14 +4735,14 @@ qla82xx_set_driver_version(scsi_qla_host_t *vha, char *version)
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x117b,
 	    "Entered %s.\n", __func__);
 
-	str = (void *)version;
+	str = (__force __le16 *)version;
 	len = strlen(version);
 
 	mcp->mb[0] = MBC_SET_RNID_PARAMS;
 	mcp->mb[1] = RNID_TYPE_SET_VERSION << 8;
 	mcp->out_mb = MBX_1|MBX_0;
 	for (i = 4; i < 16 && len; i++, str++, len -= 2) {
-		mcp->mb[i] = cpu_to_le16p(str);
+		mcp->mb[i] = le16_to_cpup(str);
 		mcp->out_mb |= 1<<i;
 	}
 	for (; i < 16; i++) {
@@ -4861,7 +4860,7 @@ qla24xx_get_port_login_templ(scsi_qla_host_t *vha, dma_addr_t buf_dma,
 		    "Done %s.\n", __func__);
 		bp = (uint32_t *) buf;
 		for (i = 0; i < (bufsiz-4)/4; i++, bp++)
-			*bp = le32_to_cpu(*bp);
+			*bp = le32_to_cpu((__force __le32)*bp);
 	}
 
 	return rval;
@@ -5411,18 +5410,18 @@ qla81xx_write_mpi_register(scsi_qla_host_t *vha, uint16_t *mb)
 	clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
 
 	/* Write the MBC data to the registers */
-	WRT_REG_WORD(&reg->mailbox0, MBC_WRITE_MPI_REGISTER);
-	WRT_REG_WORD(&reg->mailbox1, mb[0]);
-	WRT_REG_WORD(&reg->mailbox2, mb[1]);
-	WRT_REG_WORD(&reg->mailbox3, mb[2]);
-	WRT_REG_WORD(&reg->mailbox4, mb[3]);
+	wrt_reg_word(&reg->mailbox0, MBC_WRITE_MPI_REGISTER);
+	wrt_reg_word(&reg->mailbox1, mb[0]);
+	wrt_reg_word(&reg->mailbox2, mb[1]);
+	wrt_reg_word(&reg->mailbox3, mb[2]);
+	wrt_reg_word(&reg->mailbox4, mb[3]);
 
-	WRT_REG_DWORD(&reg->hccr, HCCRX_SET_HOST_INT);
+	wrt_reg_dword(&reg->hccr, HCCRX_SET_HOST_INT);
 
 	/* Poll for MBC interrupt */
 	for (timer = 6000000; timer; timer--) {
 		/* Check for pending interrupts. */
-		stat = RD_REG_DWORD(&reg->host_status);
+		stat = rd_reg_dword(&reg->host_status);
 		if (stat & HSRX_RISC_INT) {
 			stat &= 0xff;
 
@@ -5430,10 +5429,10 @@ qla81xx_write_mpi_register(scsi_qla_host_t *vha, uint16_t *mb)
 			    stat == 0x10 || stat == 0x11) {
 				set_bit(MBX_INTERRUPT,
 				    &ha->mbx_cmd_flags);
-				mb0 = RD_REG_WORD(&reg->mailbox0);
-				WRT_REG_DWORD(&reg->hccr,
+				mb0 = rd_reg_word(&reg->mailbox0);
+				wrt_reg_dword(&reg->hccr,
 				    HCCRX_CLR_RISC_INT);
-				RD_REG_DWORD(&reg->hccr);
+				rd_reg_dword(&reg->hccr);
 				break;
 			}
 		}
@@ -6211,7 +6210,7 @@ qla83xx_restart_nic_firmware(scsi_qla_host_t *vha)
 		ql_dbg(ql_dbg_mbx, vha, 0x1144,
 		    "Failed=%x mb[0]=%x mb[1]=%x.\n",
 		    rval, mcp->mb[0], mcp->mb[1]);
-		ha->isp_ops->fw_dump(vha, 0);
+		qla2xxx_dump_fw(vha);
 	} else {
 		ql_dbg(ql_dbg_mbx, vha, 0x1145, "Done %s.\n", __func__);
 	}
@@ -6256,7 +6255,7 @@ qla83xx_access_control(scsi_qla_host_t *vha, uint16_t options,
 		    "Failed=%x mb[0]=%x mb[1]=%x mb[2]=%x mb[3]=%x mb[4]=%x.\n",
 		    rval, mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3],
 		    mcp->mb[4]);
-		ha->isp_ops->fw_dump(vha, 0);
+		qla2xxx_dump_fw(vha);
 	} else {
 		if (subcode & BIT_5)
 			*sector_size = mcp->mb[1];
@@ -6470,13 +6469,13 @@ int qla24xx_gpdb_wait(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
 
 	memset(&mc, 0, sizeof(mc));
 	mc.mb[0] = MBC_GET_PORT_DATABASE;
-	mc.mb[1] = cpu_to_le16(fcport->loop_id);
+	mc.mb[1] = fcport->loop_id;
 	mc.mb[2] = MSW(pd_dma);
 	mc.mb[3] = LSW(pd_dma);
 	mc.mb[6] = MSW(MSD(pd_dma));
 	mc.mb[7] = LSW(MSD(pd_dma));
-	mc.mb[9] = cpu_to_le16(vha->vp_idx);
-	mc.mb[10] = cpu_to_le16((uint16_t)opt);
+	mc.mb[9] = vha->vp_idx;
+	mc.mb[10] = opt;
 
 	rval = qla24xx_send_mb_cmd(vha, &mc);
 	if (rval != QLA_SUCCESS) {
@@ -6587,7 +6586,7 @@ int qla24xx_gidlist_wait(struct scsi_qla_host *vha,
 	mc.mb[6] = MSW(MSD(id_list_dma));
 	mc.mb[7] = LSW(MSD(id_list_dma));
 	mc.mb[8] = 0;
-	mc.mb[9] = cpu_to_le16(vha->vp_idx);
+	mc.mb[9] = vha->vp_idx;
 
 	rval = qla24xx_send_mb_cmd(vha, &mc);
 	if (rval != QLA_SUCCESS) {
@@ -6613,8 +6612,8 @@ int qla27xx_set_zio_threshold(scsi_qla_host_t *vha, uint16_t value)
 
 	memset(mcp->mb, 0 , sizeof(mcp->mb));
 	mcp->mb[0] = MBC_GET_SET_ZIO_THRESHOLD;
-	mcp->mb[1] = cpu_to_le16(1);
-	mcp->mb[2] = cpu_to_le16(value);
+	mcp->mb[1] = 1;
+	mcp->mb[2] = value;
 	mcp->out_mb = MBX_2 | MBX_1 | MBX_0;
 	mcp->in_mb = MBX_2 | MBX_0;
 	mcp->tov = MBX_TOV_SECONDS;
@@ -6639,7 +6638,7 @@ int qla27xx_get_zio_threshold(scsi_qla_host_t *vha, uint16_t *value)
 
 	memset(mcp->mb, 0, sizeof(mcp->mb));
 	mcp->mb[0] = MBC_GET_SET_ZIO_THRESHOLD;
-	mcp->mb[1] = cpu_to_le16(0);
+	mcp->mb[1] = 0;
 	mcp->out_mb = MBX_1 | MBX_0;
 	mcp->in_mb = MBX_2 | MBX_0;
 	mcp->tov = MBX_TOV_SECONDS;
diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c
index d82e92da529a..15efe2f04b86 100644
--- a/drivers/scsi/qla2xxx/qla_mid.c
+++ b/drivers/scsi/qla2xxx/qla_mid.c
@@ -770,7 +770,7 @@ qla25xx_create_req_que(struct qla_hw_data *ha, uint16_t options,
 	req->req_q_in = &reg->isp25mq.req_q_in;
 	req->req_q_out = &reg->isp25mq.req_q_out;
 	req->max_q_depth = ha->req_q_map[0]->max_q_depth;
-	req->out_ptr = (void *)(req->ring + req->length);
+	req->out_ptr = (uint16_t *)(req->ring + req->length);
 	mutex_unlock(&ha->mq_lock);
 	ql_dbg(ql_dbg_multiq, base_vha, 0xc004,
 	    "ring_ptr=%p ring_index=%d, "
@@ -884,7 +884,7 @@ qla25xx_create_rsp_que(struct qla_hw_data *ha, uint16_t options,
 	reg = ISP_QUE_REG(ha, que_id);
 	rsp->rsp_q_in = &reg->isp25mq.rsp_q_in;
 	rsp->rsp_q_out = &reg->isp25mq.rsp_q_out;
-	rsp->in_ptr = (void *)(rsp->ring + rsp->length);
+	rsp->in_ptr = (uint16_t *)(rsp->ring + rsp->length);
 	mutex_unlock(&ha->mq_lock);
 	ql_dbg(ql_dbg_multiq, base_vha, 0xc00b,
 	    "options=%x id=%d rsp_q_in=%p rsp_q_out=%p\n",
diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c
index df99911b8bb9..a8fe4f725fa0 100644
--- a/drivers/scsi/qla2xxx/qla_mr.c
+++ b/drivers/scsi/qla2xxx/qla_mr.c
@@ -46,7 +46,7 @@ qlafx00_mailbox_command(scsi_qla_host_t *vha, struct mbx_cmd_32 *mcp)
 	uint8_t		io_lock_on;
 	uint16_t	command = 0;
 	uint32_t	*iptr;
-	uint32_t __iomem *optr;
+	__le32 __iomem *optr;
 	uint32_t	cnt;
 	uint32_t	mboxes;
 	unsigned long	wait_time;
@@ -109,7 +109,7 @@ qlafx00_mailbox_command(scsi_qla_host_t *vha, struct mbx_cmd_32 *mcp)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
 	/* Load mailbox registers. */
-	optr = (uint32_t __iomem *)&reg->ispfx00.mailbox0;
+	optr = &reg->ispfx00.mailbox0;
 
 	iptr = mcp->mb;
 	command = mcp->mb[0];
@@ -117,7 +117,7 @@ qlafx00_mailbox_command(scsi_qla_host_t *vha, struct mbx_cmd_32 *mcp)
 
 	for (cnt = 0; cnt < ha->mbx_count; cnt++) {
 		if (mboxes & BIT_0)
-			WRT_REG_DWORD(optr, *iptr);
+			wrt_reg_dword(optr, *iptr);
 
 		mboxes >>= 1;
 		optr++;
@@ -676,14 +676,14 @@ qlafx00_config_rings(struct scsi_qla_host *vha)
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_fx00 __iomem *reg = &ha->iobase->ispfx00;
 
-	WRT_REG_DWORD(&reg->req_q_in, 0);
-	WRT_REG_DWORD(&reg->req_q_out, 0);
+	wrt_reg_dword(&reg->req_q_in, 0);
+	wrt_reg_dword(&reg->req_q_out, 0);
 
-	WRT_REG_DWORD(&reg->rsp_q_in, 0);
-	WRT_REG_DWORD(&reg->rsp_q_out, 0);
+	wrt_reg_dword(&reg->rsp_q_in, 0);
+	wrt_reg_dword(&reg->rsp_q_out, 0);
 
 	/* PCI posting */
-	RD_REG_DWORD(&reg->rsp_q_out);
+	rd_reg_dword(&reg->rsp_q_out);
 }
 
 char *
@@ -912,9 +912,9 @@ qlafx00_init_fw_ready(scsi_qla_host_t *vha)
 	/* 30 seconds wait - Adjust if required */
 	wait_time = 30;
 
-	pseudo_aen = RD_REG_DWORD(&reg->pseudoaen);
+	pseudo_aen = rd_reg_dword(&reg->pseudoaen);
 	if (pseudo_aen == 1) {
-		aenmbx7 = RD_REG_DWORD(&reg->initval7);
+		aenmbx7 = rd_reg_dword(&reg->initval7);
 		ha->mbx_intr_code = MSW(aenmbx7);
 		ha->rqstq_intr_code = LSW(aenmbx7);
 		rval = qlafx00_driver_shutdown(vha, 10);
@@ -925,7 +925,7 @@ qlafx00_init_fw_ready(scsi_qla_host_t *vha)
 	/* wait time before firmware ready */
 	wtime = jiffies + (wait_time * HZ);
 	do {
-		aenmbx = RD_REG_DWORD(&reg->aenmailbox0);
+		aenmbx = rd_reg_dword(&reg->aenmailbox0);
 		barrier();
 		ql_dbg(ql_dbg_mbx, vha, 0x0133,
 		    "aenmbx: 0x%x\n", aenmbx);
@@ -944,15 +944,15 @@ qlafx00_init_fw_ready(scsi_qla_host_t *vha)
 
 		case MBA_FW_RESTART_CMPLT:
 			/* Set the mbx and rqstq intr code */
-			aenmbx7 = RD_REG_DWORD(&reg->aenmailbox7);
+			aenmbx7 = rd_reg_dword(&reg->aenmailbox7);
 			ha->mbx_intr_code = MSW(aenmbx7);
 			ha->rqstq_intr_code = LSW(aenmbx7);
-			ha->req_que_off = RD_REG_DWORD(&reg->aenmailbox1);
-			ha->rsp_que_off = RD_REG_DWORD(&reg->aenmailbox3);
-			ha->req_que_len = RD_REG_DWORD(&reg->aenmailbox5);
-			ha->rsp_que_len = RD_REG_DWORD(&reg->aenmailbox6);
-			WRT_REG_DWORD(&reg->aenmailbox0, 0);
-			RD_REG_DWORD_RELAXED(&reg->aenmailbox0);
+			ha->req_que_off = rd_reg_dword(&reg->aenmailbox1);
+			ha->rsp_que_off = rd_reg_dword(&reg->aenmailbox3);
+			ha->req_que_len = rd_reg_dword(&reg->aenmailbox5);
+			ha->rsp_que_len = rd_reg_dword(&reg->aenmailbox6);
+			wrt_reg_dword(&reg->aenmailbox0, 0);
+			rd_reg_dword_relaxed(&reg->aenmailbox0);
 			ql_dbg(ql_dbg_init, vha, 0x0134,
 			    "f/w returned mbx_intr_code: 0x%x, "
 			    "rqstq_intr_code: 0x%x\n",
@@ -982,13 +982,13 @@ qlafx00_init_fw_ready(scsi_qla_host_t *vha)
 			 * 3. issue Get FW State Mbox cmd to determine fw state
 			 * Set the mbx and rqstq intr code from Shadow Regs
 			 */
-			aenmbx7 = RD_REG_DWORD(&reg->initval7);
+			aenmbx7 = rd_reg_dword(&reg->initval7);
 			ha->mbx_intr_code = MSW(aenmbx7);
 			ha->rqstq_intr_code = LSW(aenmbx7);
-			ha->req_que_off = RD_REG_DWORD(&reg->initval1);
-			ha->rsp_que_off = RD_REG_DWORD(&reg->initval3);
-			ha->req_que_len = RD_REG_DWORD(&reg->initval5);
-			ha->rsp_que_len = RD_REG_DWORD(&reg->initval6);
+			ha->req_que_off = rd_reg_dword(&reg->initval1);
+			ha->rsp_que_off = rd_reg_dword(&reg->initval3);
+			ha->req_que_len = rd_reg_dword(&reg->initval5);
+			ha->rsp_que_len = rd_reg_dword(&reg->initval6);
 			ql_dbg(ql_dbg_init, vha, 0x0135,
 			    "f/w returned mbx_intr_code: 0x%x, "
 			    "rqstq_intr_code: 0x%x\n",
@@ -1034,7 +1034,7 @@ qlafx00_init_fw_ready(scsi_qla_host_t *vha)
 			if (time_after_eq(jiffies, wtime)) {
 				ql_dbg(ql_dbg_init, vha, 0x0137,
 				    "Init f/w failed: aen[7]: 0x%x\n",
-				    RD_REG_DWORD(&reg->aenmailbox7));
+				    rd_reg_dword(&reg->aenmailbox7));
 				rval = QLA_FUNCTION_FAILED;
 				done = true;
 				break;
@@ -1428,7 +1428,7 @@ qlafx00_init_response_q_entries(struct rsp_que *rsp)
 	pkt = rsp->ring_ptr;
 	for (cnt = 0; cnt < rsp->length; cnt++) {
 		pkt->signature = RESPONSE_PROCESSED;
-		WRT_REG_DWORD((void __force __iomem *)&pkt->signature,
+		wrt_reg_dword((void __force __iomem *)&pkt->signature,
 		    RESPONSE_PROCESSED);
 		pkt++;
 	}
@@ -1444,13 +1444,13 @@ qlafx00_rescan_isp(scsi_qla_host_t *vha)
 
 	qla2x00_request_irqs(ha, ha->rsp_q_map[0]);
 
-	aenmbx7 = RD_REG_DWORD(&reg->aenmailbox7);
+	aenmbx7 = rd_reg_dword(&reg->aenmailbox7);
 	ha->mbx_intr_code = MSW(aenmbx7);
 	ha->rqstq_intr_code = LSW(aenmbx7);
-	ha->req_que_off = RD_REG_DWORD(&reg->aenmailbox1);
-	ha->rsp_que_off = RD_REG_DWORD(&reg->aenmailbox3);
-	ha->req_que_len = RD_REG_DWORD(&reg->aenmailbox5);
-	ha->rsp_que_len = RD_REG_DWORD(&reg->aenmailbox6);
+	ha->req_que_off = rd_reg_dword(&reg->aenmailbox1);
+	ha->rsp_que_off = rd_reg_dword(&reg->aenmailbox3);
+	ha->req_que_len = rd_reg_dword(&reg->aenmailbox5);
+	ha->rsp_que_len = rd_reg_dword(&reg->aenmailbox6);
 
 	ql_dbg(ql_dbg_disc, vha, 0x2094,
 	    "fw returned mbx_intr_code: 0x%x, rqstq_intr_code: 0x%x "
@@ -1495,7 +1495,7 @@ qlafx00_timer_routine(scsi_qla_host_t *vha)
 		    (!test_bit(UNLOADING, &vha->dpc_flags)) &&
 		    (!test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags)) &&
 		    (ha->mr.fw_hbt_en)) {
-			fw_heart_beat = RD_REG_DWORD(&reg->fwheartbeat);
+			fw_heart_beat = rd_reg_dword(&reg->fwheartbeat);
 			if (fw_heart_beat != ha->mr.old_fw_hbt_cnt) {
 				ha->mr.old_fw_hbt_cnt = fw_heart_beat;
 				ha->mr.fw_hbt_miss_cnt = 0;
@@ -1515,7 +1515,7 @@ qlafx00_timer_routine(scsi_qla_host_t *vha)
 
 	if (test_bit(FX00_RESET_RECOVERY, &vha->dpc_flags)) {
 		/* Reset recovery to be performed in timer routine */
-		aenmbx0 = RD_REG_DWORD(&reg->aenmailbox0);
+		aenmbx0 = rd_reg_dword(&reg->aenmailbox0);
 		if (ha->mr.fw_reset_timer_exp) {
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 			qla2xxx_wake_dpc(vha);
@@ -1710,10 +1710,9 @@ qlafx00_tgt_detach(struct scsi_qla_host *vha, int tgt_id)
 	return;
 }
 
-int
+void
 qlafx00_process_aen(struct scsi_qla_host *vha, struct qla_work_evt *evt)
 {
-	int rval = 0;
 	uint32_t aen_code, aen_data;
 
 	aen_code = FCH_EVT_VENDOR_UNIQUE;
@@ -1764,8 +1763,6 @@ qlafx00_process_aen(struct scsi_qla_host *vha, struct qla_work_evt *evt)
 
 	fc_host_post_event(vha->host, fc_get_event_number(),
 	    aen_code, aen_data);
-
-	return rval;
 }
 
 static void
@@ -2721,7 +2718,7 @@ qlafx00_process_response_queue(struct scsi_qla_host *vha,
 	uint16_t lreq_q_in = 0;
 	uint16_t lreq_q_out = 0;
 
-	lreq_q_in = RD_REG_DWORD(rsp->rsp_q_in);
+	lreq_q_in = rd_reg_dword(rsp->rsp_q_in);
 	lreq_q_out = rsp->ring_index;
 
 	while (lreq_q_in != lreq_q_out) {
@@ -2783,7 +2780,7 @@ qlafx00_process_response_queue(struct scsi_qla_host *vha,
 	}
 
 	/* Adjust ring index */
-	WRT_REG_DWORD(rsp->rsp_q_out, rsp->ring_index);
+	wrt_reg_dword(rsp->rsp_q_out, rsp->ring_index);
 }
 
 /**
@@ -2814,9 +2811,9 @@ qlafx00_async_event(scsi_qla_host_t *vha)
 		break;
 
 	case QLAFX00_MBA_PORT_UPDATE:		/* Port database update */
-		ha->aenmb[1] = RD_REG_DWORD(&reg->aenmailbox1);
-		ha->aenmb[2] = RD_REG_DWORD(&reg->aenmailbox2);
-		ha->aenmb[3] = RD_REG_DWORD(&reg->aenmailbox3);
+		ha->aenmb[1] = rd_reg_dword(&reg->aenmailbox1);
+		ha->aenmb[2] = rd_reg_dword(&reg->aenmailbox2);
+		ha->aenmb[3] = rd_reg_dword(&reg->aenmailbox3);
 		ql_dbg(ql_dbg_async, vha, 0x5077,
 		    "Asynchronous port Update received "
 		    "aenmb[0]: %x, aenmb[1]: %x, aenmb[2]: %x, aenmb[3]: %x\n",
@@ -2846,13 +2843,13 @@ qlafx00_async_event(scsi_qla_host_t *vha)
 		break;
 
 	default:
-		ha->aenmb[1] = RD_REG_WORD(&reg->aenmailbox1);
-		ha->aenmb[2] = RD_REG_WORD(&reg->aenmailbox2);
-		ha->aenmb[3] = RD_REG_WORD(&reg->aenmailbox3);
-		ha->aenmb[4] = RD_REG_WORD(&reg->aenmailbox4);
-		ha->aenmb[5] = RD_REG_WORD(&reg->aenmailbox5);
-		ha->aenmb[6] = RD_REG_WORD(&reg->aenmailbox6);
-		ha->aenmb[7] = RD_REG_WORD(&reg->aenmailbox7);
+		ha->aenmb[1] = rd_reg_dword(&reg->aenmailbox1);
+		ha->aenmb[2] = rd_reg_dword(&reg->aenmailbox2);
+		ha->aenmb[3] = rd_reg_dword(&reg->aenmailbox3);
+		ha->aenmb[4] = rd_reg_dword(&reg->aenmailbox4);
+		ha->aenmb[5] = rd_reg_dword(&reg->aenmailbox5);
+		ha->aenmb[6] = rd_reg_dword(&reg->aenmailbox6);
+		ha->aenmb[7] = rd_reg_dword(&reg->aenmailbox7);
 		ql_dbg(ql_dbg_async, vha, 0x5078,
 		    "AEN:%04x %04x %04x %04x :%04x %04x %04x %04x\n",
 		    ha->aenmb[0], ha->aenmb[1], ha->aenmb[2], ha->aenmb[3],
@@ -2872,7 +2869,7 @@ static void
 qlafx00_mbx_completion(scsi_qla_host_t *vha, uint32_t mb0)
 {
 	uint16_t	cnt;
-	uint32_t __iomem *wptr;
+	__le32 __iomem *wptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_fx00 __iomem *reg = &ha->iobase->ispfx00;
 
@@ -2882,10 +2879,10 @@ qlafx00_mbx_completion(scsi_qla_host_t *vha, uint32_t mb0)
 	/* Load return mailbox registers. */
 	ha->flags.mbox_int = 1;
 	ha->mailbox_out32[0] = mb0;
-	wptr = (uint32_t __iomem *)&reg->mailbox17;
+	wptr = &reg->mailbox17;
 
 	for (cnt = 1; cnt < ha->mbx_count; cnt++) {
-		ha->mailbox_out32[cnt] = RD_REG_DWORD(wptr);
+		ha->mailbox_out32[cnt] = rd_reg_dword(wptr);
 		wptr++;
 	}
 }
@@ -2939,13 +2936,13 @@ qlafx00_intr_handler(int irq, void *dev_id)
 			break;
 
 		if (stat & QLAFX00_INTR_MB_CMPLT) {
-			mb[0] = RD_REG_WORD(&reg->mailbox16);
+			mb[0] = rd_reg_dword(&reg->mailbox16);
 			qlafx00_mbx_completion(vha, mb[0]);
 			status |= MBX_INTERRUPT;
 			clr_intr |= QLAFX00_INTR_MB_CMPLT;
 		}
 		if (intr_stat & QLAFX00_INTR_ASYNC_CMPLT) {
-			ha->aenmb[0] = RD_REG_WORD(&reg->aenmailbox0);
+			ha->aenmb[0] = rd_reg_dword(&reg->aenmailbox0);
 			qlafx00_async_event(vha);
 			clr_intr |= QLAFX00_INTR_ASYNC_CMPLT;
 		}
@@ -3113,7 +3110,7 @@ qlafx00_start_scsi(srb_t *sp)
 	tot_dsds = nseg;
 	req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
 	if (req->cnt < (req_cnt + 2)) {
-		cnt = RD_REG_DWORD_RELAXED(req->req_q_out);
+		cnt = rd_reg_dword_relaxed(req->req_q_out);
 
 		if (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
@@ -3178,7 +3175,7 @@ qlafx00_start_scsi(srb_t *sp)
 	sp->flags |= SRB_DMA_VALID;
 
 	/* Set chip new ring index. */
-	WRT_REG_DWORD(req->req_q_in, req->ring_index);
+	wrt_reg_dword(req->req_q_in, req->ring_index);
 	QLAFX00_SET_HST_INTR(ha, ha->rqstq_intr_code);
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -3205,7 +3202,7 @@ qlafx00_tm_iocb(srb_t *sp, struct tsk_mgmt_entry_fx00 *ptm_iocb)
 	memset(&tm_iocb, 0, sizeof(struct tsk_mgmt_entry_fx00));
 	tm_iocb.entry_type = TSK_MGMT_IOCB_TYPE_FX00;
 	tm_iocb.entry_count = 1;
-	tm_iocb.handle = cpu_to_le32(make_handle(req->id, sp->handle));
+	tm_iocb.handle = make_handle(req->id, sp->handle);
 	tm_iocb.reserved_0 = 0;
 	tm_iocb.tgt_id = cpu_to_le16(sp->fcport->tgt_id);
 	tm_iocb.control_flags = cpu_to_le32(fxio->u.tmf.flags);
@@ -3215,7 +3212,7 @@ qlafx00_tm_iocb(srb_t *sp, struct tsk_mgmt_entry_fx00 *ptm_iocb)
 		    sizeof(struct scsi_lun));
 	}
 
-	memcpy((void *)ptm_iocb, &tm_iocb,
+	memcpy(ptm_iocb, &tm_iocb,
 	    sizeof(struct tsk_mgmt_entry_fx00));
 	wmb();
 }
@@ -3231,13 +3228,12 @@ qlafx00_abort_iocb(srb_t *sp, struct abort_iocb_entry_fx00 *pabt_iocb)
 	memset(&abt_iocb, 0, sizeof(struct abort_iocb_entry_fx00));
 	abt_iocb.entry_type = ABORT_IOCB_TYPE_FX00;
 	abt_iocb.entry_count = 1;
-	abt_iocb.handle = cpu_to_le32(make_handle(req->id, sp->handle));
-	abt_iocb.abort_handle =
-	    cpu_to_le32(make_handle(req->id, fxio->u.abt.cmd_hndl));
+	abt_iocb.handle = make_handle(req->id, sp->handle);
+	abt_iocb.abort_handle = make_handle(req->id, fxio->u.abt.cmd_hndl);
 	abt_iocb.tgt_id_sts = cpu_to_le16(sp->fcport->tgt_id);
 	abt_iocb.req_que_no = cpu_to_le16(req->id);
 
-	memcpy((void *)pabt_iocb, &abt_iocb,
+	memcpy(pabt_iocb, &abt_iocb,
 	    sizeof(struct abort_iocb_entry_fx00));
 	wmb();
 }
@@ -3254,7 +3250,7 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 
 	memset(&fx_iocb, 0, sizeof(struct fxdisc_entry_fx00));
 	fx_iocb.entry_type = FX00_IOCB_TYPE;
-	fx_iocb.handle = cpu_to_le32(sp->handle);
+	fx_iocb.handle = sp->handle;
 	fx_iocb.entry_count = entry_cnt;
 
 	if (sp->type == SRB_FXIOCB_DCMD) {
diff --git a/drivers/scsi/qla2xxx/qla_mr.h b/drivers/scsi/qla2xxx/qla_mr.h
index 4567f0c42486..762250891a8f 100644
--- a/drivers/scsi/qla2xxx/qla_mr.h
+++ b/drivers/scsi/qla2xxx/qla_mr.h
@@ -96,7 +96,7 @@ struct tsk_mgmt_entry_fx00 {
 	uint8_t sys_define;
 	uint8_t entry_status;		/* Entry Status. */
 
-	__le32 handle;		/* System handle. */
+	uint32_t handle;		/* System handle. */
 
 	uint32_t reserved_0;
 
@@ -121,13 +121,13 @@ struct abort_iocb_entry_fx00 {
 	uint8_t sys_define;		/* System defined. */
 	uint8_t entry_status;		/* Entry Status. */
 
-	__le32 handle;		/* System handle. */
+	uint32_t handle;		/* System handle. */
 	__le32 reserved_0;
 
 	__le16 tgt_id_sts;		/* Completion status. */
 	__le16 options;
 
-	__le32 abort_handle;		/* System handle. */
+	uint32_t abort_handle;		/* System handle. */
 	__le32 reserved_2;
 
 	__le16 req_que_no;
@@ -166,7 +166,7 @@ struct fxdisc_entry_fx00 {
 	uint8_t sys_define;		/* System Defined. */
 	uint8_t entry_status;		/* Entry Status. */
 
-	__le32 handle;		/* System handle. */
+	uint32_t handle;		/* System handle. */
 	__le32 reserved_0;		/* System handle. */
 
 	__le16 func_num;
@@ -359,47 +359,47 @@ struct config_info_data {
 #define CONTINUE_A64_TYPE_FX00	0x03	/* Continuation entry. */
 
 #define QLAFX00_SET_HST_INTR(ha, value) \
-	WRT_REG_DWORD((ha)->cregbase + QLAFX00_HST_TO_HBA_REG, \
+	wrt_reg_dword((ha)->cregbase + QLAFX00_HST_TO_HBA_REG, \
 	value)
 
 #define QLAFX00_CLR_HST_INTR(ha, value) \
-	WRT_REG_DWORD((ha)->cregbase + QLAFX00_HBA_TO_HOST_REG, \
+	wrt_reg_dword((ha)->cregbase + QLAFX00_HBA_TO_HOST_REG, \
 	~value)
 
 #define QLAFX00_RD_INTR_REG(ha) \
-	RD_REG_DWORD((ha)->cregbase + QLAFX00_HBA_TO_HOST_REG)
+	rd_reg_dword((ha)->cregbase + QLAFX00_HBA_TO_HOST_REG)
 
 #define QLAFX00_CLR_INTR_REG(ha, value) \
-	WRT_REG_DWORD((ha)->cregbase + QLAFX00_HBA_TO_HOST_REG, \
+	wrt_reg_dword((ha)->cregbase + QLAFX00_HBA_TO_HOST_REG, \
 	~value)
 
 #define QLAFX00_SET_HBA_SOC_REG(ha, off, val)\
-	WRT_REG_DWORD((ha)->cregbase + off, val)
+	wrt_reg_dword((ha)->cregbase + off, val)
 
 #define QLAFX00_GET_HBA_SOC_REG(ha, off)\
-	RD_REG_DWORD((ha)->cregbase + off)
+	rd_reg_dword((ha)->cregbase + off)
 
 #define QLAFX00_HBA_RST_REG(ha, val)\
-	WRT_REG_DWORD((ha)->cregbase + QLAFX00_HST_RST_REG, val)
+	wrt_reg_dword((ha)->cregbase + QLAFX00_HST_RST_REG, val)
 
 #define QLAFX00_RD_ICNTRL_REG(ha) \
-	RD_REG_DWORD((ha)->cregbase + QLAFX00_HBA_ICNTRL_REG)
+	rd_reg_dword((ha)->cregbase + QLAFX00_HBA_ICNTRL_REG)
 
 #define QLAFX00_ENABLE_ICNTRL_REG(ha) \
-	WRT_REG_DWORD((ha)->cregbase + QLAFX00_HBA_ICNTRL_REG, \
+	wrt_reg_dword((ha)->cregbase + QLAFX00_HBA_ICNTRL_REG, \
 	(QLAFX00_GET_HBA_SOC_REG(ha, QLAFX00_HBA_ICNTRL_REG) | \
 	 QLAFX00_ICR_ENB_MASK))
 
 #define QLAFX00_DISABLE_ICNTRL_REG(ha) \
-	WRT_REG_DWORD((ha)->cregbase + QLAFX00_HBA_ICNTRL_REG, \
+	wrt_reg_dword((ha)->cregbase + QLAFX00_HBA_ICNTRL_REG, \
 	(QLAFX00_GET_HBA_SOC_REG(ha, QLAFX00_HBA_ICNTRL_REG) & \
 	 QLAFX00_ICR_DIS_MASK))
 
 #define QLAFX00_RD_REG(ha, off) \
-	RD_REG_DWORD((ha)->cregbase + off)
+	rd_reg_dword((ha)->cregbase + off)
 
 #define QLAFX00_WR_REG(ha, off, val) \
-	WRT_REG_DWORD((ha)->cregbase + off, val)
+	wrt_reg_dword((ha)->cregbase + off, val)
 
 struct qla_mt_iocb_rqst_fx00 {
 	__le32 reserved_0;
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index 4886d247df6f..d66d47a0f958 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -138,7 +138,7 @@ static void qla_nvme_release_fcp_cmd_kref(struct kref *kref)
 	priv->sp = NULL;
 	sp->priv = NULL;
 	if (priv->comp_status == QLA_SUCCESS) {
-		fd->rcv_rsplen = nvme->u.nvme.rsp_pyld_len;
+		fd->rcv_rsplen = le16_to_cpu(nvme->u.nvme.rsp_pyld_len);
 	} else {
 		fd->rcv_rsplen = 0;
 		fd->transferred_length = 0;
@@ -295,7 +295,7 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
 	sp->name = "nvme_ls";
 	sp->done = qla_nvme_sp_ls_done;
 	sp->put_fn = qla_nvme_release_ls_cmd_kref;
-	sp->priv = (void *)priv;
+	sp->priv = priv;
 	priv->sp = sp;
 	kref_init(&sp->cmd_kref);
 	spin_lock_init(&priv->cmd_lock);
@@ -384,7 +384,7 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 	req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
 	if (req->cnt < (req_cnt + 2)) {
 		cnt = IS_SHADOW_REG_CAPABLE(ha) ? *req->out_ptr :
-		    RD_REG_DWORD_RELAXED(req->req_q_out);
+		    rd_reg_dword_relaxed(req->req_q_out);
 
 		if (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
@@ -426,11 +426,11 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 
 	/* No data transfer how do we check buffer len == 0?? */
 	if (fd->io_dir == NVMEFC_FCP_READ) {
-		cmd_pkt->control_flags = CF_READ_DATA;
+		cmd_pkt->control_flags = cpu_to_le16(CF_READ_DATA);
 		vha->qla_stats.input_bytes += fd->payload_length;
 		vha->qla_stats.input_requests++;
 	} else if (fd->io_dir == NVMEFC_FCP_WRITE) {
-		cmd_pkt->control_flags = CF_WRITE_DATA;
+		cmd_pkt->control_flags = cpu_to_le16(CF_WRITE_DATA);
 		if ((vha->flags.nvme_first_burst) &&
 		    (sp->fcport->nvme_prli_service_param &
 			NVME_PRLI_SP_FIRST_BURST)) {
@@ -438,7 +438,7 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 			    sp->fcport->nvme_first_burst_size) ||
 				(sp->fcport->nvme_first_burst_size == 0))
 				cmd_pkt->control_flags |=
-				    CF_NVME_FIRST_BURST_ENABLE;
+					cpu_to_le16(CF_NVME_FIRST_BURST_ENABLE);
 		}
 		vha->qla_stats.output_bytes += fd->payload_length;
 		vha->qla_stats.output_requests++;
@@ -514,7 +514,7 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 	}
 
 	/* Set chip new ring index. */
-	WRT_REG_DWORD(req->req_q_in, req->ring_index);
+	wrt_reg_dword(req->req_q_in, req->ring_index);
 
 queuing_error:
 	spin_unlock_irqrestore(&qpair->qp_lock, flags);
@@ -560,7 +560,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 	init_waitqueue_head(&sp->nvme_ls_waitq);
 	kref_init(&sp->cmd_kref);
 	spin_lock_init(&priv->cmd_lock);
-	sp->priv = (void *)priv;
+	sp->priv = priv;
 	priv->sp = sp;
 	sp->type = SRB_NVME_CMD;
 	sp->name = "nvme_cmd";
diff --git a/drivers/scsi/qla2xxx/qla_nvme.h b/drivers/scsi/qla2xxx/qla_nvme.h
index ef912902d4e5..fbb844226630 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.h
+++ b/drivers/scsi/qla2xxx/qla_nvme.h
@@ -48,26 +48,26 @@ struct cmd_nvme {
 	uint8_t entry_status;           /* Entry Status. */
 
 	uint32_t handle;                /* System handle. */
-	uint16_t nport_handle;          /* N_PORT handle. */
-	uint16_t timeout;               /* Command timeout. */
+	__le16	nport_handle;		/* N_PORT handle. */
+	__le16	timeout;		/* Command timeout. */
 
-	uint16_t dseg_count;            /* Data segment count. */
-	uint16_t nvme_rsp_dsd_len;      /* NVMe RSP DSD length */
+	__le16	dseg_count;		/* Data segment count. */
+	__le16	nvme_rsp_dsd_len;	/* NVMe RSP DSD length */
 
 	uint64_t rsvd;
 
-	uint16_t control_flags;         /* Control Flags */
+	__le16	control_flags;		/* Control Flags */
 #define CF_NVME_FIRST_BURST_ENABLE	BIT_11
 #define CF_DIF_SEG_DESCR_ENABLE         BIT_3
 #define CF_DATA_SEG_DESCR_ENABLE        BIT_2
 #define CF_READ_DATA                    BIT_1
 #define CF_WRITE_DATA                   BIT_0
 
-	uint16_t nvme_cmnd_dseg_len;             /* Data segment length. */
+	__le16	nvme_cmnd_dseg_len;             /* Data segment length. */
 	__le64	 nvme_cmnd_dseg_address __packed;/* Data segment address. */
 	__le64	 nvme_rsp_dseg_address __packed; /* Data segment address. */
 
-	uint32_t byte_count;            /* Total byte count. */
+	__le32	byte_count;		/* Total byte count. */
 
 	uint8_t port_id[3];             /* PortID of destination port. */
 	uint8_t vp_index;
@@ -82,24 +82,24 @@ struct pt_ls4_request {
 	uint8_t sys_define;
 	uint8_t entry_status;
 	uint32_t handle;
-	uint16_t status;
-	uint16_t nport_handle;
-	uint16_t tx_dseg_count;
+	__le16	status;
+	__le16	nport_handle;
+	__le16	tx_dseg_count;
 	uint8_t  vp_index;
 	uint8_t  rsvd;
-	uint16_t timeout;
-	uint16_t control_flags;
+	__le16	timeout;
+	__le16	control_flags;
 #define CF_LS4_SHIFT		13
 #define CF_LS4_ORIGINATOR	0
 #define CF_LS4_RESPONDER	1
 #define CF_LS4_RESPONDER_TERM	2
 
-	uint16_t rx_dseg_count;
-	uint16_t rsvd2;
-	uint32_t exchange_address;
-	uint32_t rsvd3;
-	uint32_t rx_byte_count;
-	uint32_t tx_byte_count;
+	__le16	rx_dseg_count;
+	__le16	rsvd2;
+	__le32	exchange_address;
+	__le32	rsvd3;
+	__le32	rx_byte_count;
+	__le32	tx_byte_count;
 	struct dsd64 dsd[2];
 };
 
@@ -107,32 +107,32 @@ struct pt_ls4_request {
 struct pt_ls4_rx_unsol {
 	uint8_t entry_type;
 	uint8_t entry_count;
-	uint16_t rsvd0;
-	uint16_t rsvd1;
+	__le16	rsvd0;
+	__le16	rsvd1;
 	uint8_t vp_index;
 	uint8_t rsvd2;
-	uint16_t rsvd3;
-	uint16_t nport_handle;
-	uint16_t frame_size;
-	uint16_t rsvd4;
-	uint32_t exchange_address;
+	__le16	rsvd3;
+	__le16	nport_handle;
+	__le16	frame_size;
+	__le16	rsvd4;
+	__le32	exchange_address;
 	uint8_t d_id[3];
 	uint8_t r_ctl;
 	be_id_t s_id;
 	uint8_t cs_ctl;
 	uint8_t f_ctl[3];
 	uint8_t type;
-	uint16_t seq_cnt;
+	__le16	seq_cnt;
 	uint8_t df_ctl;
 	uint8_t seq_id;
-	uint16_t rx_id;
-	uint16_t ox_id;
-	uint32_t param;
-	uint32_t desc0;
+	__le16	rx_id;
+	__le16	ox_id;
+	__le32	param;
+	__le32	desc0;
 #define PT_LS4_PAYLOAD_OFFSET 0x2c
 #define PT_LS4_FIRST_PACKET_LEN 20
-	uint32_t desc_len;
-	uint32_t payload[3];
+	__le32	desc_len;
+	__le32	payload[3];
 };
 
 /*
diff --git a/drivers/scsi/qla2xxx/qla_nx.c b/drivers/scsi/qla2xxx/qla_nx.c
index 185c5f34d4c1..0baf55b7e88f 100644
--- a/drivers/scsi/qla2xxx/qla_nx.c
+++ b/drivers/scsi/qla2xxx/qla_nx.c
@@ -370,7 +370,7 @@ qla82xx_pci_set_crbwindow_2M(struct qla_hw_data *ha, ulong off_in,
 	/* Read back value to make sure write has gone through before trying
 	 * to use it.
 	 */
-	win_read = RD_REG_DWORD(CRB_WINDOW_2M + ha->nx_pcibase);
+	win_read = rd_reg_dword(CRB_WINDOW_2M + ha->nx_pcibase);
 	if (win_read != ha->crb_win) {
 		ql_dbg(ql_dbg_p3p, vha, 0xb000,
 		    "%s: Written crbwin (0x%x) "
@@ -380,47 +380,6 @@ qla82xx_pci_set_crbwindow_2M(struct qla_hw_data *ha, ulong off_in,
 	*off_out = (off_in & MASK(16)) + CRB_INDIRECT_2M + ha->nx_pcibase;
 }
 
-static inline unsigned long
-qla82xx_pci_set_crbwindow(struct qla_hw_data *ha, u64 off)
-{
-	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
-	/* See if we are currently pointing to the region we want to use next */
-	if ((off >= QLA82XX_CRB_PCIX_HOST) && (off < QLA82XX_CRB_DDR_NET)) {
-		/* No need to change window. PCIX and PCIEregs are in both
-		 * regs are in both windows.
-		 */
-		return off;
-	}
-
-	if ((off >= QLA82XX_CRB_PCIX_HOST) && (off < QLA82XX_CRB_PCIX_HOST2)) {
-		/* We are in first CRB window */
-		if (ha->curr_window != 0)
-			WARN_ON(1);
-		return off;
-	}
-
-	if ((off > QLA82XX_CRB_PCIX_HOST2) && (off < QLA82XX_CRB_MAX)) {
-		/* We are in second CRB window */
-		off = off - QLA82XX_CRB_PCIX_HOST2 + QLA82XX_CRB_PCIX_HOST;
-
-		if (ha->curr_window != 1)
-			return off;
-
-		/* We are in the QM or direct access
-		 * register region - do nothing
-		 */
-		if ((off >= QLA82XX_PCI_DIRECT_CRB) &&
-			(off < QLA82XX_PCI_CAMQM_MAX))
-			return off;
-	}
-	/* strange address given */
-	ql_dbg(ql_dbg_p3p, vha, 0xb001,
-	    "%s: Warning: unm_nic_pci_set_crbwindow "
-	    "called with an unknown address(%llx).\n",
-	    QLA2XXX_DRIVER_NAME, off);
-	return off;
-}
-
 static int
 qla82xx_pci_get_crb_addr_2M(struct qla_hw_data *ha, ulong off_in,
 			    void __iomem **off_out)
@@ -520,7 +479,7 @@ qla82xx_rd_32(struct qla_hw_data *ha, ulong off_in)
 		qla82xx_crb_win_lock(ha);
 		qla82xx_pci_set_crbwindow_2M(ha, off_in, &off);
 	}
-	data = RD_REG_DWORD(off);
+	data = rd_reg_dword(off);
 
 	if (rv == 1) {
 		qla82xx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM7_UNLOCK));
@@ -937,17 +896,17 @@ qla82xx_md_rw_32(struct qla_hw_data *ha, uint32_t off, u32 data, uint8_t flag)
 {
 	uint32_t  off_value, rval = 0;
 
-	WRT_REG_DWORD(CRB_WINDOW_2M + ha->nx_pcibase, off & 0xFFFF0000);
+	wrt_reg_dword(CRB_WINDOW_2M + ha->nx_pcibase, off & 0xFFFF0000);
 
 	/* Read back value to make sure write has gone through */
-	RD_REG_DWORD(CRB_WINDOW_2M + ha->nx_pcibase);
+	rd_reg_dword(CRB_WINDOW_2M + ha->nx_pcibase);
 	off_value  = (off & 0x0000FFFF);
 
 	if (flag)
-		WRT_REG_DWORD(off_value + CRB_INDIRECT_2M + ha->nx_pcibase,
+		wrt_reg_dword(off_value + CRB_INDIRECT_2M + ha->nx_pcibase,
 			      data);
 	else
-		rval = RD_REG_DWORD(off_value + CRB_INDIRECT_2M +
+		rval = rd_reg_dword(off_value + CRB_INDIRECT_2M +
 				    ha->nx_pcibase);
 
 	return rval;
@@ -1561,14 +1520,14 @@ qla82xx_get_table_desc(const u8 *unirom, int section)
 	uint32_t i;
 	struct qla82xx_uri_table_desc *directory =
 		(struct qla82xx_uri_table_desc *)&unirom[0];
-	__le32 offset;
-	__le32 tab_type;
-	__le32 entries = cpu_to_le32(directory->num_entries);
+	uint32_t offset;
+	uint32_t tab_type;
+	uint32_t entries = le32_to_cpu(directory->num_entries);
 
 	for (i = 0; i < entries; i++) {
-		offset = cpu_to_le32(directory->findex) +
-		    (i * cpu_to_le32(directory->entry_size));
-		tab_type = cpu_to_le32(*((u32 *)&unirom[offset] + 8));
+		offset = le32_to_cpu(directory->findex) +
+		    (i * le32_to_cpu(directory->entry_size));
+		tab_type = get_unaligned_le32((u32 *)&unirom[offset] + 8);
 
 		if (tab_type == section)
 			return (struct qla82xx_uri_table_desc *)&unirom[offset];
@@ -1582,16 +1541,17 @@ qla82xx_get_data_desc(struct qla_hw_data *ha,
 	u32 section, u32 idx_offset)
 {
 	const u8 *unirom = ha->hablob->fw->data;
-	int idx = cpu_to_le32(*((int *)&unirom[ha->file_prd_off] + idx_offset));
+	int idx = get_unaligned_le32((u32 *)&unirom[ha->file_prd_off] +
+				     idx_offset);
 	struct qla82xx_uri_table_desc *tab_desc = NULL;
-	__le32 offset;
+	uint32_t offset;
 
 	tab_desc = qla82xx_get_table_desc(unirom, section);
 	if (!tab_desc)
 		return NULL;
 
-	offset = cpu_to_le32(tab_desc->findex) +
-	    (cpu_to_le32(tab_desc->entry_size) * idx);
+	offset = le32_to_cpu(tab_desc->findex) +
+	    (le32_to_cpu(tab_desc->entry_size) * idx);
 
 	return (struct qla82xx_uri_data_desc *)&unirom[offset];
 }
@@ -1606,7 +1566,7 @@ qla82xx_get_bootld_offset(struct qla_hw_data *ha)
 		uri_desc = qla82xx_get_data_desc(ha,
 		    QLA82XX_URI_DIR_SECT_BOOTLD, QLA82XX_URI_BOOTLD_IDX_OFF);
 		if (uri_desc)
-			offset = cpu_to_le32(uri_desc->findex);
+			offset = le32_to_cpu(uri_desc->findex);
 	}
 
 	return (u8 *)&ha->hablob->fw->data[offset];
@@ -1620,7 +1580,7 @@ static u32 qla82xx_get_fw_size(struct qla_hw_data *ha)
 		uri_desc =  qla82xx_get_data_desc(ha, QLA82XX_URI_DIR_SECT_FW,
 		    QLA82XX_URI_FIRMWARE_IDX_OFF);
 		if (uri_desc)
-			return cpu_to_le32(uri_desc->size);
+			return le32_to_cpu(uri_desc->size);
 	}
 
 	return get_unaligned_le32(&ha->hablob->fw->data[FW_SIZE_OFFSET]);
@@ -1636,7 +1596,7 @@ qla82xx_get_fw_offs(struct qla_hw_data *ha)
 		uri_desc = qla82xx_get_data_desc(ha, QLA82XX_URI_DIR_SECT_FW,
 			QLA82XX_URI_FIRMWARE_IDX_OFF);
 		if (uri_desc)
-			offset = cpu_to_le32(uri_desc->findex);
+			offset = le32_to_cpu(uri_desc->findex);
 	}
 
 	return (u8 *)&ha->hablob->fw->data[offset];
@@ -1790,9 +1750,9 @@ void qla82xx_config_rings(struct scsi_qla_host *vha)
 	put_unaligned_le64(req->dma, &icb->request_q_address);
 	put_unaligned_le64(rsp->dma, &icb->response_q_address);
 
-	WRT_REG_DWORD(&reg->req_q_out[0], 0);
-	WRT_REG_DWORD(&reg->rsp_q_in[0], 0);
-	WRT_REG_DWORD(&reg->rsp_q_out[0], 0);
+	wrt_reg_dword(&reg->req_q_out[0], 0);
+	wrt_reg_dword(&reg->rsp_q_in[0], 0);
+	wrt_reg_dword(&reg->rsp_q_out[0], 0);
 }
 
 static int
@@ -1847,8 +1807,8 @@ qla82xx_set_product_offset(struct qla_hw_data *ha)
 	struct qla82xx_uri_table_desc *ptab_desc = NULL;
 	const uint8_t *unirom = ha->hablob->fw->data;
 	uint32_t i;
-	__le32 entries;
-	__le32 flags, file_chiprev, offset;
+	uint32_t entries;
+	uint32_t flags, file_chiprev, offset;
 	uint8_t chiprev = ha->chip_revision;
 	/* Hardcoding mn_present flag for P3P */
 	int mn_present = 0;
@@ -1859,14 +1819,14 @@ qla82xx_set_product_offset(struct qla_hw_data *ha)
 	if (!ptab_desc)
 		return -1;
 
-	entries = cpu_to_le32(ptab_desc->num_entries);
+	entries = le32_to_cpu(ptab_desc->num_entries);
 
 	for (i = 0; i < entries; i++) {
-		offset = cpu_to_le32(ptab_desc->findex) +
-			(i * cpu_to_le32(ptab_desc->entry_size));
-		flags = cpu_to_le32(*((int *)&unirom[offset] +
+		offset = le32_to_cpu(ptab_desc->findex) +
+			(i * le32_to_cpu(ptab_desc->entry_size));
+		flags = le32_to_cpu(*((__le32 *)&unirom[offset] +
 			QLA82XX_URI_FLAGS_OFF));
-		file_chiprev = cpu_to_le32(*((int *)&unirom[offset] +
+		file_chiprev = le32_to_cpu(*((__le32 *)&unirom[offset] +
 			QLA82XX_URI_CHIP_REV_OFF));
 
 		flagbit = mn_present ? 1 : 2;
@@ -1996,18 +1956,18 @@ void
 qla82xx_mbx_completion(scsi_qla_host_t *vha, uint16_t mb0)
 {
 	uint16_t	cnt;
-	uint16_t __iomem *wptr;
+	__le16 __iomem *wptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_82xx __iomem *reg = &ha->iobase->isp82;
 
-	wptr = (uint16_t __iomem *)&reg->mailbox_out[1];
+	wptr = &reg->mailbox_out[1];
 
 	/* Load return mailbox registers. */
 	ha->flags.mbox_int = 1;
 	ha->mailbox_out[0] = mb0;
 
 	for (cnt = 1; cnt < ha->mbx_count; cnt++) {
-		ha->mailbox_out[cnt] = RD_REG_WORD(wptr);
+		ha->mailbox_out[cnt] = rd_reg_word(wptr);
 		wptr++;
 	}
 
@@ -2069,8 +2029,8 @@ qla82xx_intr_handler(int irq, void *dev_id)
 	vha = pci_get_drvdata(ha->pdev);
 	for (iter = 1; iter--; ) {
 
-		if (RD_REG_DWORD(&reg->host_int)) {
-			stat = RD_REG_DWORD(&reg->host_status);
+		if (rd_reg_dword(&reg->host_int)) {
+			stat = rd_reg_dword(&reg->host_status);
 
 			switch (stat & 0xff) {
 			case 0x1:
@@ -2082,9 +2042,9 @@ qla82xx_intr_handler(int irq, void *dev_id)
 				break;
 			case 0x12:
 				mb[0] = MSW(stat);
-				mb[1] = RD_REG_WORD(&reg->mailbox_out[1]);
-				mb[2] = RD_REG_WORD(&reg->mailbox_out[2]);
-				mb[3] = RD_REG_WORD(&reg->mailbox_out[3]);
+				mb[1] = rd_reg_word(&reg->mailbox_out[1]);
+				mb[2] = rd_reg_word(&reg->mailbox_out[2]);
+				mb[3] = rd_reg_word(&reg->mailbox_out[3]);
 				qla2x00_async_event(vha, rsp, mb);
 				break;
 			case 0x13:
@@ -2097,7 +2057,7 @@ qla82xx_intr_handler(int irq, void *dev_id)
 				break;
 			}
 		}
-		WRT_REG_DWORD(&reg->host_int, 0);
+		wrt_reg_dword(&reg->host_int, 0);
 	}
 
 	qla2x00_handle_mbx_completion(ha, status);
@@ -2135,11 +2095,11 @@ qla82xx_msix_default(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
 	do {
-		host_int = RD_REG_DWORD(&reg->host_int);
+		host_int = rd_reg_dword(&reg->host_int);
 		if (qla2x00_check_reg32_for_disconnect(vha, host_int))
 			break;
 		if (host_int) {
-			stat = RD_REG_DWORD(&reg->host_status);
+			stat = rd_reg_dword(&reg->host_status);
 
 			switch (stat & 0xff) {
 			case 0x1:
@@ -2151,9 +2111,9 @@ qla82xx_msix_default(int irq, void *dev_id)
 				break;
 			case 0x12:
 				mb[0] = MSW(stat);
-				mb[1] = RD_REG_WORD(&reg->mailbox_out[1]);
-				mb[2] = RD_REG_WORD(&reg->mailbox_out[2]);
-				mb[3] = RD_REG_WORD(&reg->mailbox_out[3]);
+				mb[1] = rd_reg_word(&reg->mailbox_out[1]);
+				mb[2] = rd_reg_word(&reg->mailbox_out[2]);
+				mb[3] = rd_reg_word(&reg->mailbox_out[3]);
 				qla2x00_async_event(vha, rsp, mb);
 				break;
 			case 0x13:
@@ -2166,7 +2126,7 @@ qla82xx_msix_default(int irq, void *dev_id)
 				break;
 			}
 		}
-		WRT_REG_DWORD(&reg->host_int, 0);
+		wrt_reg_dword(&reg->host_int, 0);
 	} while (0);
 
 	qla2x00_handle_mbx_completion(ha, status);
@@ -2196,11 +2156,11 @@ qla82xx_msix_rsp_q(int irq, void *dev_id)
 	reg = &ha->iobase->isp82;
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
-	host_int = RD_REG_DWORD(&reg->host_int);
+	host_int = rd_reg_dword(&reg->host_int);
 	if (qla2x00_check_reg32_for_disconnect(vha, host_int))
 		goto out;
 	qla24xx_process_response_queue(vha, rsp);
-	WRT_REG_DWORD(&reg->host_int, 0);
+	wrt_reg_dword(&reg->host_int, 0);
 out:
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	return IRQ_HANDLED;
@@ -2231,11 +2191,11 @@ qla82xx_poll(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
 
-	host_int = RD_REG_DWORD(&reg->host_int);
+	host_int = rd_reg_dword(&reg->host_int);
 	if (qla2x00_check_reg32_for_disconnect(vha, host_int))
 		goto out;
 	if (host_int) {
-		stat = RD_REG_DWORD(&reg->host_status);
+		stat = rd_reg_dword(&reg->host_status);
 		switch (stat & 0xff) {
 		case 0x1:
 		case 0x2:
@@ -2246,9 +2206,9 @@ qla82xx_poll(int irq, void *dev_id)
 			break;
 		case 0x12:
 			mb[0] = MSW(stat);
-			mb[1] = RD_REG_WORD(&reg->mailbox_out[1]);
-			mb[2] = RD_REG_WORD(&reg->mailbox_out[2]);
-			mb[3] = RD_REG_WORD(&reg->mailbox_out[3]);
+			mb[1] = rd_reg_word(&reg->mailbox_out[1]);
+			mb[2] = rd_reg_word(&reg->mailbox_out[2]);
+			mb[3] = rd_reg_word(&reg->mailbox_out[3]);
 			qla2x00_async_event(vha, rsp, mb);
 			break;
 		case 0x13:
@@ -2260,7 +2220,7 @@ qla82xx_poll(int irq, void *dev_id)
 			    stat * 0xff);
 			break;
 		}
-		WRT_REG_DWORD(&reg->host_int, 0);
+		wrt_reg_dword(&reg->host_int, 0);
 	}
 out:
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -2549,8 +2509,8 @@ qla82xx_start_firmware(scsi_qla_host_t *vha)
 	return qla82xx_check_rcvpeg_state(ha);
 }
 
-static uint32_t *
-qla82xx_read_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
+static __le32 *
+qla82xx_read_flash_data(scsi_qla_host_t *vha, __le32 *dwptr, uint32_t faddr,
 	uint32_t length)
 {
 	uint32_t i;
@@ -2675,13 +2635,13 @@ qla82xx_read_optrom_data(struct scsi_qla_host *vha, void *buf,
 	uint32_t offset, uint32_t length)
 {
 	scsi_block_requests(vha->host);
-	qla82xx_read_flash_data(vha, (uint32_t *)buf, offset, length);
+	qla82xx_read_flash_data(vha, buf, offset, length);
 	scsi_unblock_requests(vha->host);
 	return buf;
 }
 
 static int
-qla82xx_write_flash_data(struct scsi_qla_host *vha, uint32_t *dwptr,
+qla82xx_write_flash_data(struct scsi_qla_host *vha, __le32 *dwptr,
 	uint32_t faddr, uint32_t dwords)
 {
 	int ret;
@@ -2758,7 +2718,7 @@ qla82xx_write_flash_data(struct scsi_qla_host *vha, uint32_t *dwptr,
 		}
 
 		ret = qla82xx_write_flash_dword(ha, faddr,
-		    cpu_to_le32(*dwptr));
+						le32_to_cpu(*dwptr));
 		if (ret) {
 			ql_dbg(ql_dbg_p3p, vha, 0xb020,
 			    "Unable to program flash address=%x data=%x.\n",
@@ -2818,10 +2778,10 @@ qla82xx_start_iocbs(scsi_qla_host_t *vha)
 	if (ql2xdbwr)
 		qla82xx_wr_32(ha, (unsigned long)ha->nxdb_wr_ptr, dbval);
 	else {
-		WRT_REG_DWORD(ha->nxdb_wr_ptr, dbval);
+		wrt_reg_dword(ha->nxdb_wr_ptr, dbval);
 		wmb();
-		while (RD_REG_DWORD(ha->nxdb_rd_ptr) != dbval) {
-			WRT_REG_DWORD(ha->nxdb_wr_ptr, dbval);
+		while (rd_reg_dword(ha->nxdb_rd_ptr) != dbval) {
+			wrt_reg_dword(ha->nxdb_wr_ptr, dbval);
 			wmb();
 		}
 	}
@@ -3724,7 +3684,7 @@ qla82xx_chip_reset_cleanup(scsi_qla_host_t *vha)
 /* Minidump related functions */
 static int
 qla82xx_minidump_process_control(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	struct qla82xx_md_entry_crb *crb_entry;
@@ -3841,12 +3801,12 @@ qla82xx_minidump_process_control(scsi_qla_host_t *vha,
 
 static void
 qla82xx_minidump_process_rdocm(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t r_addr, r_stride, loop_cnt, i, r_value;
 	struct qla82xx_md_entry_rdocm *ocm_hdr;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	ocm_hdr = (struct qla82xx_md_entry_rdocm *)entry_hdr;
 	r_addr = ocm_hdr->read_addr;
@@ -3854,7 +3814,7 @@ qla82xx_minidump_process_rdocm(scsi_qla_host_t *vha,
 	loop_cnt = ocm_hdr->op_count;
 
 	for (i = 0; i < loop_cnt; i++) {
-		r_value = RD_REG_DWORD(r_addr + ha->nx_pcibase);
+		r_value = rd_reg_dword(r_addr + ha->nx_pcibase);
 		*data_ptr++ = cpu_to_le32(r_value);
 		r_addr += r_stride;
 	}
@@ -3863,12 +3823,12 @@ qla82xx_minidump_process_rdocm(scsi_qla_host_t *vha,
 
 static void
 qla82xx_minidump_process_rdmux(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t r_addr, s_stride, s_addr, s_value, loop_cnt, i, r_value;
 	struct qla82xx_md_entry_mux *mux_hdr;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	mux_hdr = (struct qla82xx_md_entry_mux *)entry_hdr;
 	r_addr = mux_hdr->read_addr;
@@ -3889,12 +3849,12 @@ qla82xx_minidump_process_rdmux(scsi_qla_host_t *vha,
 
 static void
 qla82xx_minidump_process_rdcrb(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t r_addr, r_stride, loop_cnt, i, r_value;
 	struct qla82xx_md_entry_crb *crb_hdr;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	crb_hdr = (struct qla82xx_md_entry_crb *)entry_hdr;
 	r_addr = crb_hdr->addr;
@@ -3912,7 +3872,7 @@ qla82xx_minidump_process_rdcrb(scsi_qla_host_t *vha,
 
 static int
 qla82xx_minidump_process_l2tag(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t addr, r_addr, c_addr, t_r_addr;
@@ -3921,7 +3881,7 @@ qla82xx_minidump_process_l2tag(scsi_qla_host_t *vha,
 	uint32_t c_value_w, c_value_r;
 	struct qla82xx_md_entry_cache *cache_hdr;
 	int rval = QLA_FUNCTION_FAILED;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	cache_hdr = (struct qla82xx_md_entry_cache *)entry_hdr;
 	loop_count = cache_hdr->op_count;
@@ -3971,14 +3931,14 @@ qla82xx_minidump_process_l2tag(scsi_qla_host_t *vha,
 
 static void
 qla82xx_minidump_process_l1cache(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t addr, r_addr, c_addr, t_r_addr;
 	uint32_t i, k, loop_count, t_value, r_cnt, r_value;
 	uint32_t c_value_w;
 	struct qla82xx_md_entry_cache *cache_hdr;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	cache_hdr = (struct qla82xx_md_entry_cache *)entry_hdr;
 	loop_count = cache_hdr->op_count;
@@ -4006,14 +3966,14 @@ qla82xx_minidump_process_l1cache(scsi_qla_host_t *vha,
 
 static void
 qla82xx_minidump_process_queue(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t s_addr, r_addr;
 	uint32_t r_stride, r_value, r_cnt, qid = 0;
 	uint32_t i, k, loop_cnt;
 	struct qla82xx_md_entry_queue *q_hdr;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	q_hdr = (struct qla82xx_md_entry_queue *)entry_hdr;
 	s_addr = q_hdr->select_addr;
@@ -4036,13 +3996,13 @@ qla82xx_minidump_process_queue(scsi_qla_host_t *vha,
 
 static void
 qla82xx_minidump_process_rdrom(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t r_addr, r_value;
 	uint32_t i, loop_cnt;
 	struct qla82xx_md_entry_rdrom *rom_hdr;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	rom_hdr = (struct qla82xx_md_entry_rdrom *)entry_hdr;
 	r_addr = rom_hdr->read_addr;
@@ -4062,7 +4022,7 @@ qla82xx_minidump_process_rdrom(scsi_qla_host_t *vha,
 
 static int
 qla82xx_minidump_process_rdmem(scsi_qla_host_t *vha,
-	qla82xx_md_entry_hdr_t *entry_hdr, uint32_t **d_ptr)
+	qla82xx_md_entry_hdr_t *entry_hdr, __le32 **d_ptr)
 {
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t r_addr, r_value, r_data;
@@ -4070,7 +4030,7 @@ qla82xx_minidump_process_rdmem(scsi_qla_host_t *vha,
 	struct qla82xx_md_entry_rdmem *m_hdr;
 	unsigned long flags;
 	int rval = QLA_FUNCTION_FAILED;
-	uint32_t *data_ptr = *d_ptr;
+	__le32 *data_ptr = *d_ptr;
 
 	m_hdr = (struct qla82xx_md_entry_rdmem *)entry_hdr;
 	r_addr = m_hdr->read_addr;
@@ -4163,12 +4123,12 @@ qla82xx_md_collect(scsi_qla_host_t *vha)
 	int no_entry_hdr = 0;
 	qla82xx_md_entry_hdr_t *entry_hdr;
 	struct qla82xx_md_template_hdr *tmplt_hdr;
-	uint32_t *data_ptr;
+	__le32 *data_ptr;
 	uint32_t total_data_size = 0, f_capture_mask, data_collected = 0;
 	int i = 0, rval = QLA_FUNCTION_FAILED;
 
 	tmplt_hdr = (struct qla82xx_md_template_hdr *)ha->md_tmplt_hdr;
-	data_ptr = (uint32_t *)ha->md_dump;
+	data_ptr = ha->md_dump;
 
 	if (ha->fw_dumped) {
 		ql_log(ql_log_warn, vha, 0xb037,
@@ -4177,7 +4137,7 @@ qla82xx_md_collect(scsi_qla_host_t *vha)
 		goto md_failed;
 	}
 
-	ha->fw_dumped = 0;
+	ha->fw_dumped = false;
 
 	if (!ha->md_tmplt_hdr || !ha->md_dump) {
 		ql_log(ql_log_warn, vha, 0xb038,
@@ -4357,7 +4317,7 @@ skip_nxt_entry:
 	ql_log(ql_log_info, vha, 0xb044,
 	    "Firmware dump saved to temp buffer (%ld/%p %ld/%p).\n",
 	    vha->host_no, ha->md_tmplt_hdr, vha->host_no, ha->md_dump);
-	ha->fw_dumped = 1;
+	ha->fw_dumped = true;
 	qla2x00_post_uevent_work(vha, QLA_UEVENT_CODE_FW_DUMP);
 
 md_failed:
@@ -4514,7 +4474,7 @@ exit:
 }
 
 void
-qla82xx_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla82xx_fw_dump(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
 
diff --git a/drivers/scsi/qla2xxx/qla_nx.h b/drivers/scsi/qla2xxx/qla_nx.h
index 230abee10598..93344a05910a 100644
--- a/drivers/scsi/qla2xxx/qla_nx.h
+++ b/drivers/scsi/qla2xxx/qla_nx.h
@@ -800,16 +800,16 @@ struct qla82xx_legacy_intr_set {
 #define QLA82XX_URI_FIRMWARE_IDX_OFF	29
 
 struct qla82xx_uri_table_desc{
-	uint32_t	findex;
-	uint32_t	num_entries;
-	uint32_t	entry_size;
-	uint32_t	reserved[5];
+	__le32	findex;
+	__le32	num_entries;
+	__le32	entry_size;
+	__le32	reserved[5];
 };
 
 struct qla82xx_uri_data_desc{
-	uint32_t	findex;
-	uint32_t	size;
-	uint32_t	reserved[5];
+	__le32	findex;
+	__le32	size;
+	__le32	reserved[5];
 };
 
 /* UNIFIED ROMIMAGE END */
@@ -829,22 +829,22 @@ struct qla82xx_uri_data_desc{
  * ISP 8021 I/O Register Set structure definitions.
  */
 struct device_reg_82xx {
-	uint32_t req_q_out[64];		/* Request Queue out-Pointer (64 * 4) */
-	uint32_t rsp_q_in[64];		/* Response Queue In-Pointer. */
-	uint32_t rsp_q_out[64];		/* Response Queue Out-Pointer. */
+	__le32	req_q_out[64];		/* Request Queue out-Pointer (64 * 4) */
+	__le32	rsp_q_in[64];		/* Response Queue In-Pointer. */
+	__le32	rsp_q_out[64];		/* Response Queue Out-Pointer. */
 
-	uint16_t mailbox_in[32];	/* Mail box In registers */
-	uint16_t unused_1[32];
-	uint32_t hint;			/* Host interrupt register */
+	__le16	mailbox_in[32];		/* Mailbox In registers */
+	__le16	unused_1[32];
+	__le32	hint;			/* Host interrupt register */
 #define	HINT_MBX_INT_PENDING	BIT_0
-	uint16_t unused_2[62];
-	uint16_t mailbox_out[32];	/* Mail box Out registers */
-	uint32_t unused_3[48];
+	__le16	unused_2[62];
+	__le16	mailbox_out[32];	/* Mailbox Out registers */
+	__le32	unused_3[48];
 
-	uint32_t host_status;		/* host status */
+	__le32	host_status;		/* host status */
 #define HSRX_RISC_INT		BIT_15	/* RISC to Host interrupt. */
 #define HSRX_RISC_PAUSED	BIT_8	/* RISC Paused. */
-	uint32_t host_int;		/* Interrupt status. */
+	__le32	host_int;		/* Interrupt status. */
 #define ISRX_NX_RISC_INT	BIT_0	/* RISC interrupt. */
 };
 
diff --git a/drivers/scsi/qla2xxx/qla_nx2.c b/drivers/scsi/qla2xxx/qla_nx2.c
index c056f466f1f4..50e57603ce3d 100644
--- a/drivers/scsi/qla2xxx/qla_nx2.c
+++ b/drivers/scsi/qla2xxx/qla_nx2.c
@@ -1441,7 +1441,7 @@ qla8044_device_bootstrap(struct scsi_qla_host *vha)
 	if (idc_ctrl & GRACEFUL_RESET_BIT1) {
 		qla8044_wr_reg(ha, QLA8044_IDC_DRV_CTRL,
 		    (idc_ctrl & ~GRACEFUL_RESET_BIT1));
-		ha->fw_dumped = 0;
+		ha->fw_dumped = false;
 	}
 
 dev_ready:
@@ -2965,7 +2965,7 @@ qla8044_minidump_pex_dma_read(struct scsi_qla_host *vha,
 
 		/* Prepare: Write pex-dma descriptor to MS memory. */
 		rval = qla8044_ms_mem_write_128b(vha,
-		    m_hdr->desc_card_addr, (void *)&dma_desc,
+		    m_hdr->desc_card_addr, (uint32_t *)&dma_desc,
 		    (sizeof(struct qla8044_pex_dma_descriptor)/16));
 		if (rval) {
 			ql_log(ql_log_warn, vha, 0xb14a,
@@ -2987,7 +2987,7 @@ qla8044_minidump_pex_dma_read(struct scsi_qla_host *vha,
 		read_size += chunk_size;
 	}
 
-	*d_ptr = (void *)data_ptr;
+	*d_ptr = (uint32_t *)data_ptr;
 
 error_exit:
 	if (rdmem_buffer)
@@ -3249,7 +3249,7 @@ qla8044_collect_md_data(struct scsi_qla_host *vha)
 		goto md_failed;
 	}
 
-	ha->fw_dumped = 0;
+	ha->fw_dumped = false;
 
 	if (!ha->md_tmplt_hdr || !ha->md_dump) {
 		ql_log(ql_log_warn, vha, 0xb10e,
@@ -3470,7 +3470,7 @@ skip_nxt_entry:
 	ql_log(ql_log_info, vha, 0xb110,
 	    "Firmware dump saved to temp buffer (%ld/%p %ld/%p).\n",
 	    vha->host_no, ha->md_tmplt_hdr, vha->host_no, ha->md_dump);
-	ha->fw_dumped = 1;
+	ha->fw_dumped = true;
 	qla2x00_post_uevent_work(vha, QLA_UEVENT_CODE_FW_DUMP);
 
 
@@ -3487,7 +3487,7 @@ qla8044_get_minidump(struct scsi_qla_host *vha)
 	struct qla_hw_data *ha = vha->hw;
 
 	if (!qla8044_collect_md_data(vha)) {
-		ha->fw_dumped = 1;
+		ha->fw_dumped = true;
 		ha->prev_minidump_failed = 0;
 	} else {
 		ql_log(ql_log_fatal, vha, 0xb0db,
@@ -3946,8 +3946,8 @@ qla8044_intr_handler(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	for (iter = 1; iter--; ) {
 
-		if (RD_REG_DWORD(&reg->host_int)) {
-			stat = RD_REG_DWORD(&reg->host_status);
+		if (rd_reg_dword(&reg->host_int)) {
+			stat = rd_reg_dword(&reg->host_status);
 			if ((stat & HSRX_RISC_INT) == 0)
 				break;
 
@@ -3961,9 +3961,9 @@ qla8044_intr_handler(int irq, void *dev_id)
 				break;
 			case 0x12:
 				mb[0] = MSW(stat);
-				mb[1] = RD_REG_WORD(&reg->mailbox_out[1]);
-				mb[2] = RD_REG_WORD(&reg->mailbox_out[2]);
-				mb[3] = RD_REG_WORD(&reg->mailbox_out[3]);
+				mb[1] = rd_reg_word(&reg->mailbox_out[1]);
+				mb[2] = rd_reg_word(&reg->mailbox_out[2]);
+				mb[3] = rd_reg_word(&reg->mailbox_out[3]);
 				qla2x00_async_event(vha, rsp, mb);
 				break;
 			case 0x13:
@@ -3976,7 +3976,7 @@ qla8044_intr_handler(int irq, void *dev_id)
 				break;
 			}
 		}
-		WRT_REG_DWORD(&reg->host_int, 0);
+		wrt_reg_dword(&reg->host_int, 0);
 	}
 
 	qla2x00_handle_mbx_completion(ha, status);
@@ -4070,7 +4070,7 @@ exit_isp_reset:
 }
 
 void
-qla8044_fw_dump(scsi_qla_host_t *vha, int hardware_locked)
+qla8044_fw_dump(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
 
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 1d9a4866f9a7..e92fad99338c 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -35,6 +35,11 @@ static int apidev_major;
  */
 struct kmem_cache *srb_cachep;
 
+int ql2xfulldump_on_mpifail;
+module_param(ql2xfulldump_on_mpifail, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(ql2xfulldump_on_mpifail,
+		 "Set this to take full dump on MPI hang.");
+
 /*
  * CT6 CTX allocation cache
  */
@@ -1216,9 +1221,9 @@ uint32_t qla2x00_isp_reg_stat(struct qla_hw_data *ha)
 	struct device_reg_82xx __iomem *reg82 = &ha->iobase->isp82;
 
 	if (IS_P3P_TYPE(ha))
-		return ((RD_REG_DWORD(&reg82->host_int)) == ISP_REG_DISCONNECT);
+		return ((rd_reg_dword(&reg82->host_int)) == ISP_REG_DISCONNECT);
 	else
-		return ((RD_REG_DWORD(&reg->host_status)) ==
+		return ((rd_reg_dword(&reg->host_status)) ==
 			ISP_REG_DISCONNECT);
 }
 
@@ -1902,8 +1907,8 @@ qla2x00_enable_intrs(struct qla_hw_data *ha)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	ha->interrupts_on = 1;
 	/* enable risc and host interrupts */
-	WRT_REG_WORD(&reg->ictrl, ICR_EN_INT | ICR_EN_RISC);
-	RD_REG_WORD(&reg->ictrl);
+	wrt_reg_word(&reg->ictrl, ICR_EN_INT | ICR_EN_RISC);
+	rd_reg_word(&reg->ictrl);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 }
@@ -1917,8 +1922,8 @@ qla2x00_disable_intrs(struct qla_hw_data *ha)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	ha->interrupts_on = 0;
 	/* disable risc and host interrupts */
-	WRT_REG_WORD(&reg->ictrl, 0);
-	RD_REG_WORD(&reg->ictrl);
+	wrt_reg_word(&reg->ictrl, 0);
+	rd_reg_word(&reg->ictrl);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 }
 
@@ -1930,8 +1935,8 @@ qla24xx_enable_intrs(struct qla_hw_data *ha)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	ha->interrupts_on = 1;
-	WRT_REG_DWORD(&reg->ictrl, ICRX_EN_RISC_INT);
-	RD_REG_DWORD(&reg->ictrl);
+	wrt_reg_dword(&reg->ictrl, ICRX_EN_RISC_INT);
+	rd_reg_dword(&reg->ictrl);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 }
 
@@ -1945,8 +1950,8 @@ qla24xx_disable_intrs(struct qla_hw_data *ha)
 		return;
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	ha->interrupts_on = 0;
-	WRT_REG_DWORD(&reg->ictrl, 0);
-	RD_REG_DWORD(&reg->ictrl);
+	wrt_reg_dword(&reg->ictrl, 0);
+	rd_reg_dword(&reg->ictrl);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 }
 
@@ -2518,6 +2523,7 @@ static struct isp_operations qla27xx_isp_ops = {
 	.read_nvram		= NULL,
 	.write_nvram		= NULL,
 	.fw_dump		= qla27xx_fwdump,
+	.mpi_fw_dump		= qla27xx_mpi_fwdump,
 	.beacon_on		= qla24xx_beacon_on,
 	.beacon_off		= qla24xx_beacon_off,
 	.beacon_blink		= qla83xx_beacon_blink,
@@ -4614,7 +4620,7 @@ qla2x00_free_fw_dump(struct qla_hw_data *ha)
 	ha->flags.fce_enabled = 0;
 	ha->eft = NULL;
 	ha->eft_dma = 0;
-	ha->fw_dumped = 0;
+	ha->fw_dumped = false;
 	ha->fw_dump_cap_flags = 0;
 	ha->fw_dump_reading = 0;
 	ha->fw_dump = NULL;
@@ -5758,7 +5764,8 @@ qla25xx_rdp_rsp_reduce_size(struct scsi_qla_host *vha,
 	if (!pdb) {
 		ql_dbg(ql_dbg_init, vha, 0x0181,
 		    "%s: Failed allocate pdb\n", __func__);
-	} else if (qla24xx_get_port_database(vha, purex->nport_handle, pdb)) {
+	} else if (qla24xx_get_port_database(vha,
+				le16_to_cpu(purex->nport_handle), pdb)) {
 		ql_dbg(ql_dbg_init, vha, 0x0181,
 		    "%s: Failed get pdb sid=%x\n", __func__, sid);
 	} else if (pdb->current_login_state != PDS_PLOGI_COMPLETE &&
@@ -5910,7 +5917,7 @@ void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
 	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x0181,
 	    "-------- ELS REQ -------\n");
 	ql_dump_buffer(ql_dbg_init + ql_dbg_verbose, vha, 0x0182,
-	    (void *)purex, sizeof(*purex));
+	    purex, sizeof(*purex));
 
 	if (qla25xx_rdp_rsp_reduce_size(vha, purex)) {
 		rsp_payload_length =
@@ -5952,7 +5959,7 @@ void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
 	rsp_els->entry_status = 0;
 	rsp_els->handle = 0;
 	rsp_els->nport_handle = purex->nport_handle;
-	rsp_els->tx_dsd_count = 1;
+	rsp_els->tx_dsd_count = cpu_to_le16(1);
 	rsp_els->vp_index = purex->vp_idx;
 	rsp_els->sof_type = EST_SOFI3;
 	rsp_els->rx_xchg_address = purex->rx_xchg_addr;
@@ -5963,7 +5970,7 @@ void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
 	rsp_els->d_id[1] = purex->s_id[1];
 	rsp_els->d_id[2] = purex->s_id[2];
 
-	rsp_els->control_flags = EPD_ELS_ACC;
+	rsp_els->control_flags = cpu_to_le16(EPD_ELS_ACC);
 	rsp_els->rx_byte_count = 0;
 	rsp_els->tx_byte_count = cpu_to_le32(rsp_payload_length);
 
@@ -5975,8 +5982,8 @@ void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
 
 	/* Prepare Response Payload */
 	rsp_payload->hdr.cmd = cpu_to_be32(0x2 << 24); /* LS_ACC */
-	rsp_payload->hdr.len = cpu_to_be32(
-	    rsp_els->tx_byte_count - sizeof(rsp_payload->hdr));
+	rsp_payload->hdr.len = cpu_to_be32(le32_to_cpu(rsp_els->tx_byte_count) -
+					   sizeof(rsp_payload->hdr));
 
 	/* Link service Request Info Descriptor */
 	rsp_payload->ls_req_info_desc.desc_tag = cpu_to_be32(0x1);
@@ -6026,7 +6033,7 @@ void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
 		memset(sfp, 0, SFP_RTDI_LEN);
 		rval = qla2x00_read_sfp(vha, sfp_dma, sfp, 0xa2, 0x60, 10, 0);
 		if (!rval) {
-			uint16_t *trx = (void *)sfp; /* already be16 */
+			__be16 *trx = (__force __be16 *)sfp; /* already be16 */
 			rsp_payload->sfp_diag_desc.temperature = trx[0];
 			rsp_payload->sfp_diag_desc.vcc = trx[1];
 			rsp_payload->sfp_diag_desc.tx_bias = trx[2];
@@ -6053,17 +6060,17 @@ void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
 		rval = qla24xx_get_isp_stats(vha, stat, stat_dma, 0);
 		if (!rval) {
 			rsp_payload->ls_err_desc.link_fail_cnt =
-			    cpu_to_be32(stat->link_fail_cnt);
+			    cpu_to_be32(le32_to_cpu(stat->link_fail_cnt));
 			rsp_payload->ls_err_desc.loss_sync_cnt =
-			    cpu_to_be32(stat->loss_sync_cnt);
+			    cpu_to_be32(le32_to_cpu(stat->loss_sync_cnt));
 			rsp_payload->ls_err_desc.loss_sig_cnt =
-			    cpu_to_be32(stat->loss_sig_cnt);
+			    cpu_to_be32(le32_to_cpu(stat->loss_sig_cnt));
 			rsp_payload->ls_err_desc.prim_seq_err_cnt =
-			    cpu_to_be32(stat->prim_seq_err_cnt);
+			    cpu_to_be32(le32_to_cpu(stat->prim_seq_err_cnt));
 			rsp_payload->ls_err_desc.inval_xmit_word_cnt =
-			    cpu_to_be32(stat->inval_xmit_word_cnt);
+			    cpu_to_be32(le32_to_cpu(stat->inval_xmit_word_cnt));
 			rsp_payload->ls_err_desc.inval_crc_cnt =
-			    cpu_to_be32(stat->inval_crc_cnt);
+			    cpu_to_be32(le32_to_cpu(stat->inval_crc_cnt));
 			rsp_payload->ls_err_desc.pn_port_phy_type |= BIT_6;
 		}
 	}
@@ -6135,7 +6142,7 @@ void qla24xx_process_purex_rdp(struct scsi_qla_host *vha, void *pkt)
 		memset(sfp, 0, SFP_RTDI_LEN);
 		rval = qla2x00_read_sfp(vha, sfp_dma, sfp, 0xa2, 0, 64, 0);
 		if (!rval) {
-			uint16_t *trx = (void *)sfp; /* already be16 */
+			__be16 *trx = (__force __be16 *)sfp; /* already be16 */
 
 			/* Optical Element Descriptor, Temperature */
 			rsp_payload->optical_elmt_desc[0].high_alarm = trx[0];
@@ -6261,11 +6268,11 @@ send:
 	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x0184,
 	    "-------- ELS RSP -------\n");
 	ql_dump_buffer(ql_dbg_init + ql_dbg_verbose, vha, 0x0185,
-	    (void *)rsp_els, sizeof(*rsp_els));
+	    rsp_els, sizeof(*rsp_els));
 	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x0186,
 	    "-------- ELS RSP PAYLOAD -------\n");
 	ql_dump_buffer(ql_dbg_init + ql_dbg_verbose, vha, 0x0187,
-	    (void *)rsp_payload, rsp_payload_length);
+	    rsp_payload, rsp_payload_length);
 
 	rval = qla2x00_issue_iocb(vha, rsp_els, rsp_els_dma, 0);
 
@@ -6871,6 +6878,7 @@ qla2x00_do_dpc(void *data)
 
 			if (do_reset && !(test_and_set_bit(ABORT_ISP_ACTIVE,
 			    &base_vha->dpc_flags))) {
+				base_vha->flags.online = 1;
 				ql_dbg(ql_dbg_dpc, base_vha, 0x4007,
 				    "ISP abort scheduled.\n");
 				if (ha->isp_ops->abort_isp(base_vha)) {
@@ -7550,15 +7558,15 @@ qla2xxx_pci_mmio_enabled(struct pci_dev *pdev)
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	if (IS_QLA2100(ha) || IS_QLA2200(ha)){
-		stat = RD_REG_DWORD(&reg->hccr);
+		stat = rd_reg_word(&reg->hccr);
 		if (stat & HCCR_RISC_PAUSE)
 			risc_paused = 1;
 	} else if (IS_QLA23XX(ha)) {
-		stat = RD_REG_DWORD(&reg->u.isp2300.host_status);
+		stat = rd_reg_dword(&reg->u.isp2300.host_status);
 		if (stat & HSR_RISC_PAUSED)
 			risc_paused = 1;
 	} else if (IS_FWI2_CAPABLE(ha)) {
-		stat = RD_REG_DWORD(&reg24->host_status);
+		stat = rd_reg_dword(&reg24->host_status);
 		if (stat & HSRX_RISC_PAUSED)
 			risc_paused = 1;
 	}
@@ -7567,7 +7575,7 @@ qla2xxx_pci_mmio_enabled(struct pci_dev *pdev)
 	if (risc_paused) {
 		ql_log(ql_log_info, base_vha, 0x9003,
 		    "RISC paused -- mmio_enabled, Dumping firmware.\n");
-		ha->isp_ops->fw_dump(base_vha, 0);
+		qla2xxx_dump_fw(base_vha);
 
 		return PCI_ERS_RESULT_NEED_RESET;
 	} else
@@ -7814,13 +7822,19 @@ qla2x00_module_init(void)
 {
 	int ret = 0;
 
+	BUILD_BUG_ON(sizeof(cmd_a64_entry_t) != 64);
 	BUILD_BUG_ON(sizeof(cmd_entry_t) != 64);
 	BUILD_BUG_ON(sizeof(cont_a64_entry_t) != 64);
 	BUILD_BUG_ON(sizeof(cont_entry_t) != 64);
 	BUILD_BUG_ON(sizeof(init_cb_t) != 96);
+	BUILD_BUG_ON(sizeof(mrk_entry_t) != 64);
 	BUILD_BUG_ON(sizeof(ms_iocb_entry_t) != 64);
 	BUILD_BUG_ON(sizeof(request_t) != 64);
+	BUILD_BUG_ON(sizeof(struct abort_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct abort_iocb_entry_fx00) != 64);
+	BUILD_BUG_ON(sizeof(struct abts_entry_24xx) != 64);
 	BUILD_BUG_ON(sizeof(struct access_chip_84xx) != 64);
+	BUILD_BUG_ON(sizeof(struct access_chip_rsp_84xx) != 64);
 	BUILD_BUG_ON(sizeof(struct cmd_bidir) != 64);
 	BUILD_BUG_ON(sizeof(struct cmd_nvme) != 64);
 	BUILD_BUG_ON(sizeof(struct cmd_type_6) != 64);
@@ -7828,17 +7842,70 @@ qla2x00_module_init(void)
 	BUILD_BUG_ON(sizeof(struct cmd_type_7_fx00) != 64);
 	BUILD_BUG_ON(sizeof(struct cmd_type_crc_2) != 64);
 	BUILD_BUG_ON(sizeof(struct ct_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct ct_fdmi1_hba_attributes) != 2344);
+	BUILD_BUG_ON(sizeof(struct ct_fdmi2_hba_attributes) != 4424);
+	BUILD_BUG_ON(sizeof(struct ct_fdmi2_port_attributes) != 4164);
+	BUILD_BUG_ON(sizeof(struct ct_fdmi_hba_attr) != 260);
+	BUILD_BUG_ON(sizeof(struct ct_fdmi_port_attr) != 260);
+	BUILD_BUG_ON(sizeof(struct ct_rsp_hdr) != 16);
 	BUILD_BUG_ON(sizeof(struct ctio_crc2_to_fw) != 64);
+	BUILD_BUG_ON(sizeof(struct device_reg_24xx) != 256);
+	BUILD_BUG_ON(sizeof(struct device_reg_25xxmq) != 24);
+	BUILD_BUG_ON(sizeof(struct device_reg_2xxx) != 256);
+	BUILD_BUG_ON(sizeof(struct device_reg_82xx) != 1288);
+	BUILD_BUG_ON(sizeof(struct device_reg_fx00) != 216);
 	BUILD_BUG_ON(sizeof(struct els_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct els_sts_entry_24xx) != 64);
 	BUILD_BUG_ON(sizeof(struct fxdisc_entry_fx00) != 64);
+	BUILD_BUG_ON(sizeof(struct imm_ntfy_from_isp) != 64);
 	BUILD_BUG_ON(sizeof(struct init_cb_24xx) != 128);
 	BUILD_BUG_ON(sizeof(struct init_cb_81xx) != 128);
+	BUILD_BUG_ON(sizeof(struct logio_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct mbx_entry) != 64);
+	BUILD_BUG_ON(sizeof(struct mid_init_cb_24xx) != 5252);
+	BUILD_BUG_ON(sizeof(struct mrk_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct nvram_24xx) != 512);
+	BUILD_BUG_ON(sizeof(struct nvram_81xx) != 512);
 	BUILD_BUG_ON(sizeof(struct pt_ls4_request) != 64);
+	BUILD_BUG_ON(sizeof(struct pt_ls4_rx_unsol) != 64);
+	BUILD_BUG_ON(sizeof(struct purex_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct qla2100_fw_dump) != 123634);
+	BUILD_BUG_ON(sizeof(struct qla2300_fw_dump) != 136100);
+	BUILD_BUG_ON(sizeof(struct qla24xx_fw_dump) != 37976);
+	BUILD_BUG_ON(sizeof(struct qla25xx_fw_dump) != 39228);
+	BUILD_BUG_ON(sizeof(struct qla2xxx_fce_chain) != 52);
+	BUILD_BUG_ON(sizeof(struct qla2xxx_fw_dump) != 136172);
+	BUILD_BUG_ON(sizeof(struct qla2xxx_mq_chain) != 524);
+	BUILD_BUG_ON(sizeof(struct qla2xxx_mqueue_chain) != 8);
+	BUILD_BUG_ON(sizeof(struct qla2xxx_mqueue_header) != 12);
+	BUILD_BUG_ON(sizeof(struct qla2xxx_offld_chain) != 24);
+	BUILD_BUG_ON(sizeof(struct qla81xx_fw_dump) != 39420);
+	BUILD_BUG_ON(sizeof(struct qla82xx_uri_data_desc) != 28);
+	BUILD_BUG_ON(sizeof(struct qla82xx_uri_table_desc) != 32);
+	BUILD_BUG_ON(sizeof(struct qla83xx_fw_dump) != 51196);
+	BUILD_BUG_ON(sizeof(struct qla_fcp_prio_cfg) != FCP_PRIO_CFG_SIZE);
+	BUILD_BUG_ON(sizeof(struct qla_fdt_layout) != 128);
+	BUILD_BUG_ON(sizeof(struct qla_flt_header) != 8);
+	BUILD_BUG_ON(sizeof(struct qla_flt_region) != 16);
+	BUILD_BUG_ON(sizeof(struct qla_npiv_entry) != 24);
+	BUILD_BUG_ON(sizeof(struct qla_npiv_header) != 16);
+	BUILD_BUG_ON(sizeof(struct rdp_rsp_payload) != 336);
 	BUILD_BUG_ON(sizeof(struct sns_cmd_pkt) != 2064);
+	BUILD_BUG_ON(sizeof(struct sts_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct tsk_mgmt_entry) != 64);
+	BUILD_BUG_ON(sizeof(struct tsk_mgmt_entry_fx00) != 64);
 	BUILD_BUG_ON(sizeof(struct verify_chip_entry_84xx) != 64);
+	BUILD_BUG_ON(sizeof(struct verify_chip_rsp_84xx) != 52);
 	BUILD_BUG_ON(sizeof(struct vf_evfp_entry_24xx) != 56);
-	BUILD_BUG_ON(sizeof(struct qla_flt_region) != 16);
-	BUILD_BUG_ON(sizeof(struct qla_flt_header) != 8);
+	BUILD_BUG_ON(sizeof(struct vp_config_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct vp_ctrl_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct vp_rpt_id_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(sts21_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(sts22_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(sts_cont_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(sts_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(sw_info_t) != 32);
+	BUILD_BUG_ON(sizeof(target_id_t) != 2);
 
 	/* Allocate cache for SRBs. */
 	srb_cachep = kmem_cache_create("qla2xxx_srbs", sizeof(srb_t), 0,
diff --git a/drivers/scsi/qla2xxx/qla_sup.c b/drivers/scsi/qla2xxx/qla_sup.c
index 3da79ee1d88e..e161c05d7d82 100644
--- a/drivers/scsi/qla2xxx/qla_sup.c
+++ b/drivers/scsi/qla2xxx/qla_sup.c
@@ -26,24 +26,24 @@ qla2x00_lock_nvram_access(struct qla_hw_data *ha)
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
 	if (!IS_QLA2100(ha) && !IS_QLA2200(ha) && !IS_QLA2300(ha)) {
-		data = RD_REG_WORD(&reg->nvram);
+		data = rd_reg_word(&reg->nvram);
 		while (data & NVR_BUSY) {
 			udelay(100);
-			data = RD_REG_WORD(&reg->nvram);
+			data = rd_reg_word(&reg->nvram);
 		}
 
 		/* Lock resource */
-		WRT_REG_WORD(&reg->u.isp2300.host_semaphore, 0x1);
-		RD_REG_WORD(&reg->u.isp2300.host_semaphore);
+		wrt_reg_word(&reg->u.isp2300.host_semaphore, 0x1);
+		rd_reg_word(&reg->u.isp2300.host_semaphore);
 		udelay(5);
-		data = RD_REG_WORD(&reg->u.isp2300.host_semaphore);
+		data = rd_reg_word(&reg->u.isp2300.host_semaphore);
 		while ((data & BIT_0) == 0) {
 			/* Lock failed */
 			udelay(100);
-			WRT_REG_WORD(&reg->u.isp2300.host_semaphore, 0x1);
-			RD_REG_WORD(&reg->u.isp2300.host_semaphore);
+			wrt_reg_word(&reg->u.isp2300.host_semaphore, 0x1);
+			rd_reg_word(&reg->u.isp2300.host_semaphore);
 			udelay(5);
-			data = RD_REG_WORD(&reg->u.isp2300.host_semaphore);
+			data = rd_reg_word(&reg->u.isp2300.host_semaphore);
 		}
 	}
 }
@@ -58,8 +58,8 @@ qla2x00_unlock_nvram_access(struct qla_hw_data *ha)
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
 	if (!IS_QLA2100(ha) && !IS_QLA2200(ha) && !IS_QLA2300(ha)) {
-		WRT_REG_WORD(&reg->u.isp2300.host_semaphore, 0);
-		RD_REG_WORD(&reg->u.isp2300.host_semaphore);
+		wrt_reg_word(&reg->u.isp2300.host_semaphore, 0);
+		rd_reg_word(&reg->u.isp2300.host_semaphore);
 	}
 }
 
@@ -73,15 +73,15 @@ qla2x00_nv_write(struct qla_hw_data *ha, uint16_t data)
 {
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
-	WRT_REG_WORD(&reg->nvram, data | NVR_SELECT | NVR_WRT_ENABLE);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, data | NVR_SELECT | NVR_WRT_ENABLE);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	NVRAM_DELAY();
-	WRT_REG_WORD(&reg->nvram, data | NVR_SELECT | NVR_CLOCK |
+	wrt_reg_word(&reg->nvram, data | NVR_SELECT | NVR_CLOCK |
 	    NVR_WRT_ENABLE);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	NVRAM_DELAY();
-	WRT_REG_WORD(&reg->nvram, data | NVR_SELECT | NVR_WRT_ENABLE);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, data | NVR_SELECT | NVR_WRT_ENABLE);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	NVRAM_DELAY();
 }
 
@@ -120,21 +120,21 @@ qla2x00_nvram_request(struct qla_hw_data *ha, uint32_t nv_cmd)
 
 	/* Read data from NVRAM. */
 	for (cnt = 0; cnt < 16; cnt++) {
-		WRT_REG_WORD(&reg->nvram, NVR_SELECT | NVR_CLOCK);
-		RD_REG_WORD(&reg->nvram);	/* PCI Posting. */
+		wrt_reg_word(&reg->nvram, NVR_SELECT | NVR_CLOCK);
+		rd_reg_word(&reg->nvram);	/* PCI Posting. */
 		NVRAM_DELAY();
 		data <<= 1;
-		reg_data = RD_REG_WORD(&reg->nvram);
+		reg_data = rd_reg_word(&reg->nvram);
 		if (reg_data & NVR_DATA_IN)
 			data |= BIT_0;
-		WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-		RD_REG_WORD(&reg->nvram);	/* PCI Posting. */
+		wrt_reg_word(&reg->nvram, NVR_SELECT);
+		rd_reg_word(&reg->nvram);	/* PCI Posting. */
 		NVRAM_DELAY();
 	}
 
 	/* Deselect chip. */
-	WRT_REG_WORD(&reg->nvram, NVR_DESELECT);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, NVR_DESELECT);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	NVRAM_DELAY();
 
 	return data;
@@ -171,8 +171,8 @@ qla2x00_nv_deselect(struct qla_hw_data *ha)
 {
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
-	WRT_REG_WORD(&reg->nvram, NVR_DESELECT);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, NVR_DESELECT);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	NVRAM_DELAY();
 }
 
@@ -183,7 +183,7 @@ qla2x00_nv_deselect(struct qla_hw_data *ha)
  * @data: word to program
  */
 static void
-qla2x00_write_nvram_word(struct qla_hw_data *ha, uint32_t addr, uint16_t data)
+qla2x00_write_nvram_word(struct qla_hw_data *ha, uint32_t addr, __le16 data)
 {
 	int count;
 	uint16_t word;
@@ -202,7 +202,7 @@ qla2x00_write_nvram_word(struct qla_hw_data *ha, uint32_t addr, uint16_t data)
 
 	/* Write data */
 	nv_cmd = (addr << 16) | NV_WRITE_OP;
-	nv_cmd |= data;
+	nv_cmd |= (__force u16)data;
 	nv_cmd <<= 5;
 	for (count = 0; count < 27; count++) {
 		if (nv_cmd & BIT_31)
@@ -216,8 +216,8 @@ qla2x00_write_nvram_word(struct qla_hw_data *ha, uint32_t addr, uint16_t data)
 	qla2x00_nv_deselect(ha);
 
 	/* Wait for NVRAM to become ready */
-	WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, NVR_SELECT);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	wait_cnt = NVR_WAIT_CNT;
 	do {
 		if (!--wait_cnt) {
@@ -226,7 +226,7 @@ qla2x00_write_nvram_word(struct qla_hw_data *ha, uint32_t addr, uint16_t data)
 			break;
 		}
 		NVRAM_DELAY();
-		word = RD_REG_WORD(&reg->nvram);
+		word = rd_reg_word(&reg->nvram);
 	} while ((word & NVR_DATA_IN) == 0);
 
 	qla2x00_nv_deselect(ha);
@@ -241,7 +241,7 @@ qla2x00_write_nvram_word(struct qla_hw_data *ha, uint32_t addr, uint16_t data)
 
 static int
 qla2x00_write_nvram_word_tmo(struct qla_hw_data *ha, uint32_t addr,
-	uint16_t data, uint32_t tmo)
+			     __le16 data, uint32_t tmo)
 {
 	int ret, count;
 	uint16_t word;
@@ -261,7 +261,7 @@ qla2x00_write_nvram_word_tmo(struct qla_hw_data *ha, uint32_t addr,
 
 	/* Write data */
 	nv_cmd = (addr << 16) | NV_WRITE_OP;
-	nv_cmd |= data;
+	nv_cmd |= (__force u16)data;
 	nv_cmd <<= 5;
 	for (count = 0; count < 27; count++) {
 		if (nv_cmd & BIT_31)
@@ -275,11 +275,11 @@ qla2x00_write_nvram_word_tmo(struct qla_hw_data *ha, uint32_t addr,
 	qla2x00_nv_deselect(ha);
 
 	/* Wait for NVRAM to become ready */
-	WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, NVR_SELECT);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	do {
 		NVRAM_DELAY();
-		word = RD_REG_WORD(&reg->nvram);
+		word = rd_reg_word(&reg->nvram);
 		if (!--tmo) {
 			ret = QLA_FUNCTION_FAILED;
 			break;
@@ -308,7 +308,7 @@ qla2x00_clear_nvram_protection(struct qla_hw_data *ha)
 	int ret, stat;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 	uint32_t word, wait_cnt;
-	uint16_t wprot, wprot_old;
+	__le16 wprot, wprot_old;
 	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
 
 	/* Clear NVRAM write protection. */
@@ -318,7 +318,7 @@ qla2x00_clear_nvram_protection(struct qla_hw_data *ha)
 	stat = qla2x00_write_nvram_word_tmo(ha, ha->nvram_base,
 					    cpu_to_le16(0x1234), 100000);
 	wprot = cpu_to_le16(qla2x00_get_nvram_word(ha, ha->nvram_base));
-	if (stat != QLA_SUCCESS || wprot != 0x1234) {
+	if (stat != QLA_SUCCESS || wprot != cpu_to_le16(0x1234)) {
 		/* Write enable. */
 		qla2x00_nv_write(ha, NVR_DATA_OUT);
 		qla2x00_nv_write(ha, 0);
@@ -347,8 +347,8 @@ qla2x00_clear_nvram_protection(struct qla_hw_data *ha)
 		qla2x00_nv_deselect(ha);
 
 		/* Wait for NVRAM to become ready. */
-		WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-		RD_REG_WORD(&reg->nvram);	/* PCI Posting. */
+		wrt_reg_word(&reg->nvram, NVR_SELECT);
+		rd_reg_word(&reg->nvram);	/* PCI Posting. */
 		wait_cnt = NVR_WAIT_CNT;
 		do {
 			if (!--wait_cnt) {
@@ -357,7 +357,7 @@ qla2x00_clear_nvram_protection(struct qla_hw_data *ha)
 				break;
 			}
 			NVRAM_DELAY();
-			word = RD_REG_WORD(&reg->nvram);
+			word = rd_reg_word(&reg->nvram);
 		} while ((word & NVR_DATA_IN) == 0);
 
 		if (wait_cnt)
@@ -407,8 +407,8 @@ qla2x00_set_nvram_protection(struct qla_hw_data *ha, int stat)
 	qla2x00_nv_deselect(ha);
 
 	/* Wait for NVRAM to become ready. */
-	WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, NVR_SELECT);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	wait_cnt = NVR_WAIT_CNT;
 	do {
 		if (!--wait_cnt) {
@@ -417,7 +417,7 @@ qla2x00_set_nvram_protection(struct qla_hw_data *ha, int stat)
 			break;
 		}
 		NVRAM_DELAY();
-		word = RD_REG_WORD(&reg->nvram);
+		word = rd_reg_word(&reg->nvram);
 	} while ((word & NVR_DATA_IN) == 0);
 }
 
@@ -456,11 +456,11 @@ qla24xx_read_flash_dword(struct qla_hw_data *ha, uint32_t addr, uint32_t *data)
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 	ulong cnt = 30000;
 
-	WRT_REG_DWORD(&reg->flash_addr, addr & ~FARX_DATA_FLAG);
+	wrt_reg_dword(&reg->flash_addr, addr & ~FARX_DATA_FLAG);
 
 	while (cnt--) {
-		if (RD_REG_DWORD(&reg->flash_addr) & FARX_DATA_FLAG) {
-			*data = RD_REG_DWORD(&reg->flash_data);
+		if (rd_reg_dword(&reg->flash_addr) & FARX_DATA_FLAG) {
+			*data = rd_reg_dword(&reg->flash_data);
 			return QLA_SUCCESS;
 		}
 		udelay(10);
@@ -499,11 +499,11 @@ qla24xx_write_flash_dword(struct qla_hw_data *ha, uint32_t addr, uint32_t data)
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 	ulong cnt = 500000;
 
-	WRT_REG_DWORD(&reg->flash_data, data);
-	WRT_REG_DWORD(&reg->flash_addr, addr | FARX_DATA_FLAG);
+	wrt_reg_dword(&reg->flash_data, data);
+	wrt_reg_dword(&reg->flash_addr, addr | FARX_DATA_FLAG);
 
 	while (cnt--) {
-		if (!(RD_REG_DWORD(&reg->flash_addr) & FARX_DATA_FLAG))
+		if (!(rd_reg_dword(&reg->flash_addr) & FARX_DATA_FLAG))
 			return QLA_SUCCESS;
 		udelay(10);
 		cond_resched();
@@ -549,11 +549,12 @@ qla2xxx_find_flt_start(scsi_qla_host_t *vha, uint32_t *start)
 {
 	const char *loc, *locations[] = { "DEF", "PCI" };
 	uint32_t pcihdr, pcids;
-	uint16_t cnt, chksum, *wptr;
+	uint16_t cnt, chksum;
+	__le16 *wptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
 	struct qla_flt_location *fltl = (void *)req->ring;
-	uint32_t *dcode = (void *)req->ring;
+	uint32_t *dcode = (uint32_t *)req->ring;
 	uint8_t *buf = (void *)req->ring, *bcode,  last_image;
 
 	/*
@@ -610,7 +611,7 @@ qla2xxx_find_flt_start(scsi_qla_host_t *vha, uint32_t *start)
 	if (memcmp(fltl->sig, "QFLT", 4))
 		goto end;
 
-	wptr = (void *)req->ring;
+	wptr = (__force __le16 *)req->ring;
 	cnt = sizeof(*fltl) / sizeof(*wptr);
 	for (chksum = 0; cnt--; wptr++)
 		chksum += le16_to_cpu(*wptr);
@@ -671,7 +672,8 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 	uint32_t def = IS_QLA81XX(ha) ? 2 : IS_QLA25XX(ha) ? 1 : 0;
 	struct qla_flt_header *flt = ha->flt;
 	struct qla_flt_region *region = &flt->region[0];
-	uint16_t *wptr, cnt, chksum;
+	__le16 *wptr;
+	uint16_t cnt, chksum;
 	uint32_t start;
 
 	/* Assign FCP prio region since older adapters may not have FLT, or
@@ -681,8 +683,8 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 	    fcp_prio_cfg0[def] : fcp_prio_cfg1[def];
 
 	ha->flt_region_flt = flt_addr;
-	wptr = (uint16_t *)ha->flt;
-	ha->isp_ops->read_optrom(vha, (void *)flt, flt_addr << 2,
+	wptr = (__force __le16 *)ha->flt;
+	ha->isp_ops->read_optrom(vha, flt, flt_addr << 2,
 	    (sizeof(struct qla_flt_header) + FLT_REGIONS_SIZE));
 
 	if (le16_to_cpu(*wptr) == 0xffff)
@@ -949,7 +951,7 @@ qla2xxx_get_fdt_info(scsi_qla_host_t *vha)
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
 	uint16_t cnt, chksum;
-	uint16_t *wptr = (void *)req->ring;
+	__le16 *wptr = (__force __le16 *)req->ring;
 	struct qla_fdt_layout *fdt = (struct qla_fdt_layout *)req->ring;
 	uint8_t	man_id, flash_id;
 	uint16_t mid = 0, fid = 0;
@@ -1042,14 +1044,14 @@ static void
 qla2xxx_get_idc_param(scsi_qla_host_t *vha)
 {
 #define QLA82XX_IDC_PARAM_ADDR       0x003e885c
-	uint32_t *wptr;
+	__le32 *wptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
 
 	if (!(IS_P3P_TYPE(ha)))
 		return;
 
-	wptr = (uint32_t *)req->ring;
+	wptr = (__force __le32 *)req->ring;
 	ha->isp_ops->read_optrom(vha, req->ring, QLA82XX_IDC_PARAM_ADDR, 8);
 
 	if (*wptr == cpu_to_le32(0xffffffff)) {
@@ -1095,7 +1097,7 @@ qla2xxx_flash_npiv_conf(scsi_qla_host_t *vha)
 {
 #define NPIV_CONFIG_SIZE	(16*1024)
 	void *data;
-	uint16_t *wptr;
+	__le16 *wptr;
 	uint16_t cnt, chksum;
 	int i;
 	struct qla_npiv_header hdr;
@@ -1197,9 +1199,9 @@ qla24xx_unprotect_flash(scsi_qla_host_t *vha)
 		return qla81xx_fac_do_write_enable(vha, 1);
 
 	/* Enable flash write. */
-	WRT_REG_DWORD(&reg->ctrl_status,
-	    RD_REG_DWORD(&reg->ctrl_status) | CSRX_FLASH_ENABLE);
-	RD_REG_DWORD(&reg->ctrl_status);	/* PCI Posting. */
+	wrt_reg_dword(&reg->ctrl_status,
+	    rd_reg_dword(&reg->ctrl_status) | CSRX_FLASH_ENABLE);
+	rd_reg_dword(&reg->ctrl_status);	/* PCI Posting. */
 
 	if (!ha->fdt_wrt_disable)
 		goto done;
@@ -1240,8 +1242,8 @@ qla24xx_protect_flash(scsi_qla_host_t *vha)
 
 skip_wrt_protect:
 	/* Disable flash write. */
-	WRT_REG_DWORD(&reg->ctrl_status,
-	    RD_REG_DWORD(&reg->ctrl_status) & ~CSRX_FLASH_ENABLE);
+	wrt_reg_dword(&reg->ctrl_status,
+	    rd_reg_dword(&reg->ctrl_status) & ~CSRX_FLASH_ENABLE);
 
 	return QLA_SUCCESS;
 }
@@ -1265,7 +1267,7 @@ qla24xx_erase_sector(scsi_qla_host_t *vha, uint32_t fdata)
 }
 
 static int
-qla24xx_write_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
+qla24xx_write_flash_data(scsi_qla_host_t *vha, __le32 *dwptr, uint32_t faddr,
     uint32_t dwords)
 {
 	int ret;
@@ -1352,7 +1354,7 @@ next:
 
 		/* Slow write */
 		ret = qla24xx_write_flash_dword(ha,
-		    flash_data_addr(ha, faddr), cpu_to_le32(*dwptr));
+		    flash_data_addr(ha, faddr), le32_to_cpu(*dwptr));
 		if (ret) {
 			ql_dbg(ql_dbg_user, vha, 0x7006,
 			    "Failed slopw write %x (%x)\n", faddr, *dwptr);
@@ -1379,11 +1381,11 @@ qla2x00_read_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
     uint32_t bytes)
 {
 	uint32_t i;
-	uint16_t *wptr;
+	__le16 *wptr;
 	struct qla_hw_data *ha = vha->hw;
 
 	/* Word reads to NVRAM via registers. */
-	wptr = (uint16_t *)buf;
+	wptr = buf;
 	qla2x00_lock_nvram_access(ha);
 	for (i = 0; i < bytes >> 1; i++, naddr++)
 		wptr[i] = cpu_to_le16(qla2x00_get_nvram_word(ha,
@@ -1456,7 +1458,7 @@ qla24xx_write_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
 {
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
-	uint32_t *dwptr = buf;
+	__le32 *dwptr = buf;
 	uint32_t i;
 	int ret;
 
@@ -1466,9 +1468,9 @@ qla24xx_write_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
 		return ret;
 
 	/* Enable flash write. */
-	WRT_REG_DWORD(&reg->ctrl_status,
-	    RD_REG_DWORD(&reg->ctrl_status) | CSRX_FLASH_ENABLE);
-	RD_REG_DWORD(&reg->ctrl_status);	/* PCI Posting. */
+	wrt_reg_dword(&reg->ctrl_status,
+	    rd_reg_dword(&reg->ctrl_status) | CSRX_FLASH_ENABLE);
+	rd_reg_dword(&reg->ctrl_status);	/* PCI Posting. */
 
 	/* Disable NVRAM write-protection. */
 	qla24xx_write_flash_dword(ha, nvram_conf_addr(ha, 0x101), 0);
@@ -1478,7 +1480,7 @@ qla24xx_write_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
 	naddr = nvram_data_addr(ha, naddr);
 	bytes >>= 2;
 	for (i = 0; i < bytes; i++, naddr++, dwptr++) {
-		if (qla24xx_write_flash_dword(ha, naddr, cpu_to_le32(*dwptr))) {
+		if (qla24xx_write_flash_dword(ha, naddr, le32_to_cpu(*dwptr))) {
 			ql_dbg(ql_dbg_user, vha, 0x709a,
 			    "Unable to program nvram address=%x data=%x.\n",
 			    naddr, *dwptr);
@@ -1490,9 +1492,9 @@ qla24xx_write_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
 	qla24xx_write_flash_dword(ha, nvram_conf_addr(ha, 0x101), 0x8c);
 
 	/* Disable flash write. */
-	WRT_REG_DWORD(&reg->ctrl_status,
-	    RD_REG_DWORD(&reg->ctrl_status) & ~CSRX_FLASH_ENABLE);
-	RD_REG_DWORD(&reg->ctrl_status);	/* PCI Posting. */
+	wrt_reg_dword(&reg->ctrl_status,
+	    rd_reg_dword(&reg->ctrl_status) & ~CSRX_FLASH_ENABLE);
+	rd_reg_dword(&reg->ctrl_status);	/* PCI Posting. */
 
 	return ret;
 }
@@ -1588,8 +1590,8 @@ qla2x00_beacon_blink(struct scsi_qla_host *vha)
 		gpio_enable = RD_REG_WORD_PIO(PIO_REG(ha, gpioe));
 		gpio_data = RD_REG_WORD_PIO(PIO_REG(ha, gpiod));
 	} else {
-		gpio_enable = RD_REG_WORD(&reg->gpioe);
-		gpio_data = RD_REG_WORD(&reg->gpiod);
+		gpio_enable = rd_reg_word(&reg->gpioe);
+		gpio_data = rd_reg_word(&reg->gpiod);
 	}
 
 	/* Set the modified gpio_enable values */
@@ -1598,8 +1600,8 @@ qla2x00_beacon_blink(struct scsi_qla_host *vha)
 	if (ha->pio_address) {
 		WRT_REG_WORD_PIO(PIO_REG(ha, gpioe), gpio_enable);
 	} else {
-		WRT_REG_WORD(&reg->gpioe, gpio_enable);
-		RD_REG_WORD(&reg->gpioe);
+		wrt_reg_word(&reg->gpioe, gpio_enable);
+		rd_reg_word(&reg->gpioe);
 	}
 
 	qla2x00_flip_colors(ha, &led_color);
@@ -1614,8 +1616,8 @@ qla2x00_beacon_blink(struct scsi_qla_host *vha)
 	if (ha->pio_address) {
 		WRT_REG_WORD_PIO(PIO_REG(ha, gpiod), gpio_data);
 	} else {
-		WRT_REG_WORD(&reg->gpiod, gpio_data);
-		RD_REG_WORD(&reg->gpiod);
+		wrt_reg_word(&reg->gpiod, gpio_data);
+		rd_reg_word(&reg->gpiod);
 	}
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -1645,8 +1647,8 @@ qla2x00_beacon_on(struct scsi_qla_host *vha)
 		gpio_enable = RD_REG_WORD_PIO(PIO_REG(ha, gpioe));
 		gpio_data = RD_REG_WORD_PIO(PIO_REG(ha, gpiod));
 	} else {
-		gpio_enable = RD_REG_WORD(&reg->gpioe);
-		gpio_data = RD_REG_WORD(&reg->gpiod);
+		gpio_enable = rd_reg_word(&reg->gpioe);
+		gpio_data = rd_reg_word(&reg->gpiod);
 	}
 	gpio_enable |= GPIO_LED_MASK;
 
@@ -1654,8 +1656,8 @@ qla2x00_beacon_on(struct scsi_qla_host *vha)
 	if (ha->pio_address) {
 		WRT_REG_WORD_PIO(PIO_REG(ha, gpioe), gpio_enable);
 	} else {
-		WRT_REG_WORD(&reg->gpioe, gpio_enable);
-		RD_REG_WORD(&reg->gpioe);
+		wrt_reg_word(&reg->gpioe, gpio_enable);
+		rd_reg_word(&reg->gpioe);
 	}
 
 	/* Clear out previously set LED colour. */
@@ -1663,8 +1665,8 @@ qla2x00_beacon_on(struct scsi_qla_host *vha)
 	if (ha->pio_address) {
 		WRT_REG_WORD_PIO(PIO_REG(ha, gpiod), gpio_data);
 	} else {
-		WRT_REG_WORD(&reg->gpiod, gpio_data);
-		RD_REG_WORD(&reg->gpiod);
+		wrt_reg_word(&reg->gpiod, gpio_data);
+		rd_reg_word(&reg->gpiod);
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
@@ -1731,13 +1733,13 @@ qla24xx_beacon_blink(struct scsi_qla_host *vha)
 
 	/* Save the Original GPIOD. */
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	gpio_data = RD_REG_DWORD(&reg->gpiod);
+	gpio_data = rd_reg_dword(&reg->gpiod);
 
 	/* Enable the gpio_data reg for update. */
 	gpio_data |= GPDX_LED_UPDATE_MASK;
 
-	WRT_REG_DWORD(&reg->gpiod, gpio_data);
-	gpio_data = RD_REG_DWORD(&reg->gpiod);
+	wrt_reg_dword(&reg->gpiod, gpio_data);
+	gpio_data = rd_reg_dword(&reg->gpiod);
 
 	/* Set the color bits. */
 	qla24xx_flip_colors(ha, &led_color);
@@ -1749,8 +1751,8 @@ qla24xx_beacon_blink(struct scsi_qla_host *vha)
 	gpio_data |= led_color;
 
 	/* Set the modified gpio_data values. */
-	WRT_REG_DWORD(&reg->gpiod, gpio_data);
-	gpio_data = RD_REG_DWORD(&reg->gpiod);
+	wrt_reg_dword(&reg->gpiod, gpio_data);
+	gpio_data = rd_reg_dword(&reg->gpiod);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 }
 
@@ -1881,12 +1883,12 @@ qla24xx_beacon_on(struct scsi_qla_host *vha)
 			goto skip_gpio;
 
 		spin_lock_irqsave(&ha->hardware_lock, flags);
-		gpio_data = RD_REG_DWORD(&reg->gpiod);
+		gpio_data = rd_reg_dword(&reg->gpiod);
 
 		/* Enable the gpio_data reg for update. */
 		gpio_data |= GPDX_LED_UPDATE_MASK;
-		WRT_REG_DWORD(&reg->gpiod, gpio_data);
-		RD_REG_DWORD(&reg->gpiod);
+		wrt_reg_dword(&reg->gpiod, gpio_data);
+		rd_reg_dword(&reg->gpiod);
 
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	}
@@ -1929,12 +1931,12 @@ qla24xx_beacon_off(struct scsi_qla_host *vha)
 
 	/* Give control back to firmware. */
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	gpio_data = RD_REG_DWORD(&reg->gpiod);
+	gpio_data = rd_reg_dword(&reg->gpiod);
 
 	/* Disable the gpio_data reg for update. */
 	gpio_data &= ~GPDX_LED_UPDATE_MASK;
-	WRT_REG_DWORD(&reg->gpiod, gpio_data);
-	RD_REG_DWORD(&reg->gpiod);
+	wrt_reg_dword(&reg->gpiod, gpio_data);
+	rd_reg_dword(&reg->gpiod);
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 set_fw_options:
@@ -1970,10 +1972,10 @@ qla2x00_flash_enable(struct qla_hw_data *ha)
 	uint16_t data;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
-	data = RD_REG_WORD(&reg->ctrl_status);
+	data = rd_reg_word(&reg->ctrl_status);
 	data |= CSR_FLASH_ENABLE;
-	WRT_REG_WORD(&reg->ctrl_status, data);
-	RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
+	wrt_reg_word(&reg->ctrl_status, data);
+	rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
 }
 
 /**
@@ -1986,10 +1988,10 @@ qla2x00_flash_disable(struct qla_hw_data *ha)
 	uint16_t data;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
-	data = RD_REG_WORD(&reg->ctrl_status);
+	data = rd_reg_word(&reg->ctrl_status);
 	data &= ~(CSR_FLASH_ENABLE);
-	WRT_REG_WORD(&reg->ctrl_status, data);
-	RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
+	wrt_reg_word(&reg->ctrl_status, data);
+	rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
 }
 
 /**
@@ -2008,7 +2010,7 @@ qla2x00_read_flash_byte(struct qla_hw_data *ha, uint32_t addr)
 	uint16_t bank_select;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
-	bank_select = RD_REG_WORD(&reg->ctrl_status);
+	bank_select = rd_reg_word(&reg->ctrl_status);
 
 	if (IS_QLA2322(ha) || IS_QLA6322(ha)) {
 		/* Specify 64K address range: */
@@ -2016,11 +2018,11 @@ qla2x00_read_flash_byte(struct qla_hw_data *ha, uint32_t addr)
 		bank_select &= ~0xf8;
 		bank_select |= addr >> 12 & 0xf0;
 		bank_select |= CSR_FLASH_64K_BANK;
-		WRT_REG_WORD(&reg->ctrl_status, bank_select);
-		RD_REG_WORD(&reg->ctrl_status);	/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, bank_select);
+		rd_reg_word(&reg->ctrl_status);	/* PCI Posting. */
 
-		WRT_REG_WORD(&reg->flash_address, (uint16_t)addr);
-		data = RD_REG_WORD(&reg->flash_data);
+		wrt_reg_word(&reg->flash_address, (uint16_t)addr);
+		data = rd_reg_word(&reg->flash_data);
 
 		return (uint8_t)data;
 	}
@@ -2028,13 +2030,13 @@ qla2x00_read_flash_byte(struct qla_hw_data *ha, uint32_t addr)
 	/* Setup bit 16 of flash address. */
 	if ((addr & BIT_16) && ((bank_select & CSR_FLASH_64K_BANK) == 0)) {
 		bank_select |= CSR_FLASH_64K_BANK;
-		WRT_REG_WORD(&reg->ctrl_status, bank_select);
-		RD_REG_WORD(&reg->ctrl_status);	/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, bank_select);
+		rd_reg_word(&reg->ctrl_status);	/* PCI Posting. */
 	} else if (((addr & BIT_16) == 0) &&
 	    (bank_select & CSR_FLASH_64K_BANK)) {
 		bank_select &= ~(CSR_FLASH_64K_BANK);
-		WRT_REG_WORD(&reg->ctrl_status, bank_select);
-		RD_REG_WORD(&reg->ctrl_status);	/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, bank_select);
+		rd_reg_word(&reg->ctrl_status);	/* PCI Posting. */
 	}
 
 	/* Always perform IO mapped accesses to the FLASH registers. */
@@ -2049,7 +2051,7 @@ qla2x00_read_flash_byte(struct qla_hw_data *ha, uint32_t addr)
 			data2 = RD_REG_WORD_PIO(PIO_REG(ha, flash_data));
 		} while (data != data2);
 	} else {
-		WRT_REG_WORD(&reg->flash_address, (uint16_t)addr);
+		wrt_reg_word(&reg->flash_address, (uint16_t)addr);
 		data = qla2x00_debounce_register(&reg->flash_data);
 	}
 
@@ -2068,20 +2070,20 @@ qla2x00_write_flash_byte(struct qla_hw_data *ha, uint32_t addr, uint8_t data)
 	uint16_t bank_select;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 
-	bank_select = RD_REG_WORD(&reg->ctrl_status);
+	bank_select = rd_reg_word(&reg->ctrl_status);
 	if (IS_QLA2322(ha) || IS_QLA6322(ha)) {
 		/* Specify 64K address range: */
 		/*  clear out Module Select and Flash Address bits [19:16]. */
 		bank_select &= ~0xf8;
 		bank_select |= addr >> 12 & 0xf0;
 		bank_select |= CSR_FLASH_64K_BANK;
-		WRT_REG_WORD(&reg->ctrl_status, bank_select);
-		RD_REG_WORD(&reg->ctrl_status);	/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, bank_select);
+		rd_reg_word(&reg->ctrl_status);	/* PCI Posting. */
 
-		WRT_REG_WORD(&reg->flash_address, (uint16_t)addr);
-		RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
-		WRT_REG_WORD(&reg->flash_data, (uint16_t)data);
-		RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
+		wrt_reg_word(&reg->flash_address, (uint16_t)addr);
+		rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
+		wrt_reg_word(&reg->flash_data, (uint16_t)data);
+		rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
 
 		return;
 	}
@@ -2089,13 +2091,13 @@ qla2x00_write_flash_byte(struct qla_hw_data *ha, uint32_t addr, uint8_t data)
 	/* Setup bit 16 of flash address. */
 	if ((addr & BIT_16) && ((bank_select & CSR_FLASH_64K_BANK) == 0)) {
 		bank_select |= CSR_FLASH_64K_BANK;
-		WRT_REG_WORD(&reg->ctrl_status, bank_select);
-		RD_REG_WORD(&reg->ctrl_status);	/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, bank_select);
+		rd_reg_word(&reg->ctrl_status);	/* PCI Posting. */
 	} else if (((addr & BIT_16) == 0) &&
 	    (bank_select & CSR_FLASH_64K_BANK)) {
 		bank_select &= ~(CSR_FLASH_64K_BANK);
-		WRT_REG_WORD(&reg->ctrl_status, bank_select);
-		RD_REG_WORD(&reg->ctrl_status);	/* PCI Posting. */
+		wrt_reg_word(&reg->ctrl_status, bank_select);
+		rd_reg_word(&reg->ctrl_status);	/* PCI Posting. */
 	}
 
 	/* Always perform IO mapped accesses to the FLASH registers. */
@@ -2103,10 +2105,10 @@ qla2x00_write_flash_byte(struct qla_hw_data *ha, uint32_t addr, uint8_t data)
 		WRT_REG_WORD_PIO(PIO_REG(ha, flash_address), (uint16_t)addr);
 		WRT_REG_WORD_PIO(PIO_REG(ha, flash_data), (uint16_t)data);
 	} else {
-		WRT_REG_WORD(&reg->flash_address, (uint16_t)addr);
-		RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
-		WRT_REG_WORD(&reg->flash_data, (uint16_t)data);
-		RD_REG_WORD(&reg->ctrl_status);		/* PCI Posting. */
+		wrt_reg_word(&reg->flash_address, (uint16_t)addr);
+		rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
+		wrt_reg_word(&reg->flash_data, (uint16_t)data);
+		rd_reg_word(&reg->ctrl_status);		/* PCI Posting. */
 	}
 }
 
@@ -2289,12 +2291,12 @@ qla2x00_read_flash_data(struct qla_hw_data *ha, uint8_t *tmp_buf,
 
 	midpoint = length / 2;
 
-	WRT_REG_WORD(&reg->nvram, 0);
-	RD_REG_WORD(&reg->nvram);
+	wrt_reg_word(&reg->nvram, 0);
+	rd_reg_word(&reg->nvram);
 	for (ilength = 0; ilength < length; saddr++, ilength++, tmp_buf++) {
 		if (ilength == midpoint) {
-			WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-			RD_REG_WORD(&reg->nvram);
+			wrt_reg_word(&reg->nvram, NVR_SELECT);
+			rd_reg_word(&reg->nvram);
 		}
 		data = qla2x00_read_flash_byte(ha, saddr);
 		if (saddr % 100)
@@ -2319,11 +2321,11 @@ qla2x00_suspend_hba(struct scsi_qla_host *vha)
 
 	/* Pause RISC. */
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	WRT_REG_WORD(&reg->hccr, HCCR_PAUSE_RISC);
-	RD_REG_WORD(&reg->hccr);
+	wrt_reg_word(&reg->hccr, HCCR_PAUSE_RISC);
+	rd_reg_word(&reg->hccr);
 	if (IS_QLA2100(ha) || IS_QLA2200(ha) || IS_QLA2300(ha)) {
 		for (cnt = 0; cnt < 30000; cnt++) {
-			if ((RD_REG_WORD(&reg->hccr) & HCCR_RISC_PAUSE) != 0)
+			if ((rd_reg_word(&reg->hccr) & HCCR_RISC_PAUSE) != 0)
 				break;
 			udelay(100);
 		}
@@ -2362,12 +2364,12 @@ qla2x00_read_optrom_data(struct scsi_qla_host *vha, void *buf,
 	midpoint = ha->optrom_size / 2;
 
 	qla2x00_flash_enable(ha);
-	WRT_REG_WORD(&reg->nvram, 0);
-	RD_REG_WORD(&reg->nvram);		/* PCI Posting. */
+	wrt_reg_word(&reg->nvram, 0);
+	rd_reg_word(&reg->nvram);		/* PCI Posting. */
 	for (addr = offset, data = buf; addr < length; addr++, data++) {
 		if (addr == midpoint) {
-			WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-			RD_REG_WORD(&reg->nvram);	/* PCI Posting. */
+			wrt_reg_word(&reg->nvram, NVR_SELECT);
+			rd_reg_word(&reg->nvram);	/* PCI Posting. */
 		}
 
 		*data = qla2x00_read_flash_byte(ha, addr);
@@ -2399,7 +2401,7 @@ qla2x00_write_optrom_data(struct scsi_qla_host *vha, void *buf,
 	sec_number = 0;
 
 	/* Reset ISP chip. */
-	WRT_REG_WORD(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
+	wrt_reg_word(&reg->ctrl_status, CSR_ISP_SOFT_RESET);
 	pci_read_config_word(ha->pdev, PCI_COMMAND, &wd);
 
 	/* Go with write. */
@@ -2548,8 +2550,8 @@ update_flash:
 						}
 					}
 				} else if (addr == ha->optrom_size / 2) {
-					WRT_REG_WORD(&reg->nvram, NVR_SELECT);
-					RD_REG_WORD(&reg->nvram);
+					wrt_reg_word(&reg->nvram, NVR_SELECT);
+					rd_reg_word(&reg->nvram);
 				}
 
 				if (flash_id == 0xda && man_id == 0xc1) {
@@ -2610,7 +2612,7 @@ qla24xx_read_optrom_data(struct scsi_qla_host *vha, void *buf,
 	set_bit(MBX_UPDATE_FLASH_ACTIVE, &ha->mbx_cmd_flags);
 
 	/* Go with read. */
-	qla24xx_read_flash_data(vha, (void *)buf, offset >> 2, length >> 2);
+	qla24xx_read_flash_data(vha, buf, offset >> 2, length >> 2);
 
 	/* Resume HBA. */
 	clear_bit(MBX_UPDATE_FLASH_ACTIVE, &ha->mbx_cmd_flags);
@@ -2662,7 +2664,7 @@ qla28xx_get_flash_region(struct scsi_qla_host *vha, uint32_t start,
 
 	cnt = le16_to_cpu(flt->length) / sizeof(struct qla_flt_region);
 	for (; cnt; cnt--, flt_reg++) {
-		if (flt_reg->start == start) {
+		if (le32_to_cpu(flt_reg->start) == start) {
 			memcpy((uint8_t *)region, flt_reg,
 			    sizeof(struct qla_flt_region));
 			rval = QLA_SUCCESS;
@@ -2691,7 +2693,7 @@ qla28xx_write_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
 	struct qla_flt_region region;
 	bool reset_to_rom = false;
 	uint32_t risc_size, risc_attr = 0;
-	uint32_t *fw_array = NULL;
+	__be32 *fw_array = NULL;
 
 	/* Retrieve region info - must be a start address passed in */
 	rval = qla28xx_get_flash_region(vha, offset, &region);
@@ -2722,12 +2724,12 @@ qla28xx_write_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
 		ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
 		    "Region %x is secure\n", region.code);
 
-		switch (region.code) {
+		switch (le16_to_cpu(region.code)) {
 		case FLT_REG_FW:
 		case FLT_REG_FW_SEC_27XX:
 		case FLT_REG_MPI_PRI_28XX:
 		case FLT_REG_MPI_SEC_28XX:
-			fw_array = dwptr;
+			fw_array = (__force __be32 *)dwptr;
 
 			/* 1st fw array */
 			risc_size = be32_to_cpu(fw_array[3]);
@@ -2761,7 +2763,7 @@ qla28xx_write_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
 
 		case FLT_REG_PEP_PRI_28XX:
 		case FLT_REG_PEP_SEC_28XX:
-			fw_array = dwptr;
+			fw_array = (__force __be32 *)dwptr;
 
 			/* 1st fw array */
 			risc_size = be32_to_cpu(fw_array[3]);
@@ -2892,7 +2894,8 @@ qla28xx_write_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
 		if (region.attribute && buf_size_without_sfub) {
 			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
 			    "Sending Secure Flash MB Cmd\n");
-			rval = qla28xx_secure_flash_update(vha, 0, region.code,
+			rval = qla28xx_secure_flash_update(vha, 0,
+				le16_to_cpu(region.code),
 				buf_size_without_sfub, sfub_dma,
 				sizeof(struct secure_flash_update_block) >> 2);
 			if (rval != QLA_SUCCESS) {
@@ -2981,11 +2984,11 @@ qla24xx_write_optrom_data(struct scsi_qla_host *vha, void *buf,
 
 	/* Go with write. */
 	if (IS_QLA28XX(ha))
-		rval = qla28xx_write_flash_data(vha, (uint32_t *)buf,
-		    offset >> 2, length >> 2);
+		rval = qla28xx_write_flash_data(vha, buf, offset >> 2,
+						length >> 2);
 	else
-		rval = qla24xx_write_flash_data(vha, (uint32_t *)buf,
-		    offset >> 2, length >> 2);
+		rval = qla24xx_write_flash_data(vha, buf, offset >> 2,
+						length >> 2);
 
 	clear_bit(MBX_UPDATE_FLASH_ACTIVE, &ha->mbx_cmd_flags);
 	scsi_unblock_requests(vha->host);
@@ -3513,7 +3516,8 @@ qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 		ql_dump_buffer(ql_dbg_init, vha, 0x005f, dcode, 32);
 	} else {
 		for (i = 0; i < 4; i++)
-			ha->fw_revision[i] = be32_to_cpu(dcode[4+i]);
+			ha->fw_revision[i] =
+				be32_to_cpu((__force __be32)dcode[4+i]);
 		ql_dbg(ql_dbg_init, vha, 0x0060,
 		    "Firmware revision (flash) %u.%u.%u (%x).\n",
 		    ha->fw_revision[0], ha->fw_revision[1],
@@ -3528,7 +3532,7 @@ qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 
 	memset(ha->gold_fw_version, 0, sizeof(ha->gold_fw_version));
 	faddr = ha->flt_region_gold_fw;
-	qla24xx_read_flash_data(vha, (void *)dcode, ha->flt_region_gold_fw, 8);
+	qla24xx_read_flash_data(vha, dcode, ha->flt_region_gold_fw, 8);
 	if (qla24xx_risc_firmware_invalid(dcode)) {
 		ql_log(ql_log_warn, vha, 0x0056,
 		    "Unrecognized golden fw at %#x.\n", faddr);
@@ -3537,7 +3541,8 @@ qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 	}
 
 	for (i = 0; i < 4; i++)
-		ha->gold_fw_version[i] = be32_to_cpu(dcode[4+i]);
+		ha->gold_fw_version[i] =
+			be32_to_cpu((__force __be32)dcode[4+i]);
 
 	return ret;
 }
@@ -3617,7 +3622,7 @@ qla24xx_read_fcp_prio_cfg(scsi_qla_host_t *vha)
 
 	/* read remaining FCP CMD config data from flash */
 	fcp_prio_addr += (FCP_PRIO_CFG_HDR_SIZE >> 2);
-	len = ha->fcp_prio_cfg->num_entries * FCP_PRIO_CFG_ENTRY_SIZE;
+	len = ha->fcp_prio_cfg->num_entries * sizeof(struct qla_fcp_prio_entry);
 	max_len = FCP_PRIO_CFG_SIZE - FCP_PRIO_CFG_HDR_SIZE;
 
 	ha->isp_ops->read_optrom(vha, &ha->fcp_prio_cfg->entry[0],
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index 622e7337affc..fbb80a043b4f 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -378,7 +378,7 @@ static bool qlt_24xx_atio_pkt_all_vps(struct scsi_qla_host *vha,
 		qlt_issue_marker(vha, ha_locked);
 
 		if ((entry->u.isp24.vp_index != 0xFF) &&
-		    (entry->u.isp24.nport_handle != 0xFFFF)) {
+		    (entry->u.isp24.nport_handle != cpu_to_le16(0xFFFF))) {
 			host = qlt_find_host_by_vp_idx(vha,
 			    entry->u.isp24.vp_index);
 			if (unlikely(!host)) {
@@ -1697,7 +1697,7 @@ static void qlt_send_notify_ack(struct qla_qpair *qpair,
 	nack->u.isp24.nport_handle = ntfy->u.isp24.nport_handle;
 	if (le16_to_cpu(ntfy->u.isp24.status) == IMM_NTFY_ELS) {
 		nack->u.isp24.flags = ntfy->u.isp24.flags &
-			cpu_to_le32(NOTIFY24XX_FLAGS_PUREX_IOCB);
+			cpu_to_le16(NOTIFY24XX_FLAGS_PUREX_IOCB);
 	}
 	nack->u.isp24.srr_rx_id = ntfy->u.isp24.srr_rx_id;
 	nack->u.isp24.status = ntfy->u.isp24.status;
@@ -1725,7 +1725,8 @@ static int qlt_build_abts_resp_iocb(struct qla_tgt_mgmt_cmd *mcmd)
 	struct scsi_qla_host *vha = mcmd->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct abts_resp_to_24xx *resp;
-	uint32_t f_ctl, h;
+	__le32 f_ctl;
+	uint32_t h;
 	uint8_t *p;
 	int rc;
 	struct abts_recv_from_24xx *abts = &mcmd->orig_iocb.abts;
@@ -1782,7 +1783,7 @@ static int qlt_build_abts_resp_iocb(struct qla_tgt_mgmt_cmd *mcmd)
 		resp->fcp_hdr_le.r_ctl = R_CTL_BASIC_LINK_SERV | R_CTL_B_ACC;
 		resp->payload.ba_acct.seq_id_valid = SEQ_ID_INVALID;
 		resp->payload.ba_acct.low_seq_cnt = 0x0000;
-		resp->payload.ba_acct.high_seq_cnt = 0xFFFF;
+		resp->payload.ba_acct.high_seq_cnt = cpu_to_le16(0xFFFF);
 		resp->payload.ba_acct.ox_id = abts->fcp_hdr_le.ox_id;
 		resp->payload.ba_acct.rx_id = abts->fcp_hdr_le.rx_id;
 	} else {
@@ -1814,7 +1815,7 @@ static void qlt_24xx_send_abts_resp(struct qla_qpair *qpair,
 	struct scsi_qla_host *vha = qpair->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct abts_resp_to_24xx *resp;
-	uint32_t f_ctl;
+	__le32 f_ctl;
 	uint8_t *p;
 
 	ql_dbg(ql_dbg_tgt, vha, 0xe006,
@@ -1857,7 +1858,7 @@ static void qlt_24xx_send_abts_resp(struct qla_qpair *qpair,
 		resp->fcp_hdr_le.r_ctl = R_CTL_BASIC_LINK_SERV | R_CTL_B_ACC;
 		resp->payload.ba_acct.seq_id_valid = SEQ_ID_INVALID;
 		resp->payload.ba_acct.low_seq_cnt = 0x0000;
-		resp->payload.ba_acct.high_seq_cnt = 0xFFFF;
+		resp->payload.ba_acct.high_seq_cnt = cpu_to_le16(0xFFFF);
 		resp->payload.ba_acct.ox_id = abts->fcp_hdr_le.ox_id;
 		resp->payload.ba_acct.rx_id = abts->fcp_hdr_le.rx_id;
 	} else {
@@ -2030,7 +2031,7 @@ static void qlt_do_tmr_work(struct work_struct *work)
 
 	switch (mcmd->tmr_func) {
 	case QLA_TGT_ABTS:
-		tag = mcmd->orig_iocb.abts.exchange_addr_to_abort;
+		tag = le32_to_cpu(mcmd->orig_iocb.abts.exchange_addr_to_abort);
 		break;
 	default:
 		tag = 0;
@@ -2110,7 +2111,7 @@ static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 		struct qla_tgt_cmd *abort_cmd;
 
 		abort_cmd = ha->tgt.tgt_ops->find_cmd_by_tag(sess,
-		    abts->exchange_addr_to_abort);
+				le32_to_cpu(abts->exchange_addr_to_abort));
 		if (abort_cmd && abort_cmd->qpair) {
 			mcmd->qpair = abort_cmd->qpair;
 			mcmd->se_cmd.cpuid = abort_cmd->se_cmd.cpuid;
@@ -2133,7 +2134,7 @@ static void qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 {
 	struct qla_hw_data *ha = vha->hw;
 	struct fc_port *sess;
-	uint32_t tag = abts->exchange_addr_to_abort;
+	uint32_t tag = le32_to_cpu(abts->exchange_addr_to_abort);
 	be_id_t s_id;
 	int rc;
 	unsigned long flags;
@@ -2223,7 +2224,7 @@ static void qlt_24xx_send_task_mgmt_ctio(struct qla_qpair *qpair,
 	ctio->entry_type = CTIO_TYPE7;
 	ctio->entry_count = 1;
 	ctio->handle = QLA_TGT_SKIP_HANDLE | CTIO_COMPLETION_HANDLE_MARK;
-	ctio->nport_handle = mcmd->sess->loop_id;
+	ctio->nport_handle = cpu_to_le16(mcmd->sess->loop_id);
 	ctio->timeout = cpu_to_le16(QLA_TGT_TIMEOUT);
 	ctio->vp_index = ha->vp_idx;
 	ctio->initiator_id = be_id_to_le(atio->u.isp24.fcp_hdr.s_id);
@@ -2280,7 +2281,7 @@ void qlt_send_resp_ctio(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 	ctio->entry_type = CTIO_TYPE7;
 	ctio->entry_count = 1;
 	ctio->handle = QLA_TGT_SKIP_HANDLE;
-	ctio->nport_handle = cmd->sess->loop_id;
+	ctio->nport_handle = cpu_to_le16(cmd->sess->loop_id);
 	ctio->timeout = cpu_to_le16(QLA_TGT_TIMEOUT);
 	ctio->vp_index = vha->vp_idx;
 	ctio->initiator_id = be_id_to_le(atio->u.isp24.fcp_hdr.s_id);
@@ -2484,7 +2485,7 @@ static int qlt_check_reserve_free_req(struct qla_qpair *qpair,
 
 	if (req->cnt < (req_cnt + 2)) {
 		cnt = (uint16_t)(qpair->use_shadow_reg ? *req->out_ptr :
-		    RD_REG_DWORD_RELAXED(req->req_q_out));
+		    rd_reg_dword_relaxed(req->req_q_out));
 
 		if  (req->ring_index < cnt)
 			req->cnt = cnt - req->ring_index;
@@ -2840,10 +2841,14 @@ skip_explict_conf:
 		    cpu_to_le16(SS_SENSE_LEN_VALID);
 		ctio->u.status1.sense_length =
 		    cpu_to_le16(prm->sense_buffer_len);
-		for (i = 0; i < prm->sense_buffer_len/4; i++)
-			((uint32_t *)ctio->u.status1.sense_data)[i] =
-				cpu_to_be32(((uint32_t *)prm->sense_buffer)[i]);
+		for (i = 0; i < prm->sense_buffer_len/4; i++) {
+			uint32_t v;
 
+			v = get_unaligned_be32(
+					&((uint32_t *)prm->sense_buffer)[i]);
+			put_unaligned_le32(v,
+				&((uint32_t *)ctio->u.status1.sense_data)[i]);
+		}
 		qlt_print_dif_err(prm);
 
 	} else {
@@ -3114,7 +3119,7 @@ qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm)
 	else if (cmd->dma_data_direction == DMA_FROM_DEVICE)
 		pkt->flags = cpu_to_le16(CTIO7_FLAGS_DATA_OUT);
 
-	pkt->dseg_count = prm->tot_dsds;
+	pkt->dseg_count = cpu_to_le16(prm->tot_dsds);
 	/* Fibre channel byte count */
 	pkt->transfer_length = cpu_to_le32(transfer_length);
 
@@ -3136,7 +3141,7 @@ qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm)
 	qla_tgt_set_dif_tags(cmd, crc_ctx_pkt, &fw_prot_opts);
 
 	put_unaligned_le64(crc_ctx_dma, &pkt->crc_context_address);
-	pkt->crc_context_len = CRC_CONTEXT_LEN_FW;
+	pkt->crc_context_len = cpu_to_le16(CRC_CONTEXT_LEN_FW);
 
 	if (!bundling) {
 		cur_dsd = &crc_ctx_pkt->u.nobundling.data_dsd[0];
@@ -3573,7 +3578,7 @@ static int __qlt_send_term_imm_notif(struct scsi_qla_host *vha,
 	nack->u.isp24.nport_handle = ntfy->u.isp24.nport_handle;
 	if (le16_to_cpu(ntfy->u.isp24.status) == IMM_NTFY_ELS) {
 		nack->u.isp24.flags = ntfy->u.isp24.flags &
-			__constant_cpu_to_le32(NOTIFY24XX_FLAGS_PUREX_IOCB);
+			cpu_to_le16(NOTIFY24XX_FLAGS_PUREX_IOCB);
 	}
 
 	/* terminate */
@@ -3647,7 +3652,7 @@ static int __qlt_send_term_exchange(struct qla_qpair *qpair,
 
 	ctio24 = (struct ctio7_to_24xx *)pkt;
 	ctio24->entry_type = CTIO_TYPE7;
-	ctio24->nport_handle = CTIO7_NHANDLE_UNRECOGNIZED;
+	ctio24->nport_handle = cpu_to_le16(CTIO7_NHANDLE_UNRECOGNIZED);
 	ctio24->timeout = cpu_to_le16(QLA_TGT_TIMEOUT);
 	ctio24->vp_index = vha->vp_idx;
 	ctio24->initiator_id = be_id_to_le(atio->u.isp24.fcp_hdr.s_id);
@@ -3885,7 +3890,7 @@ static void *qlt_ctio_to_cmd(struct scsi_qla_host *vha,
 			return NULL;
 		}
 
-		cmd = (void *) req->outstanding_cmds[h];
+		cmd = req->outstanding_cmds[h];
 		if (unlikely(cmd == NULL)) {
 			ql_dbg(ql_dbg_async, vha, 0xe053,
 			    "qla_target(%d): Suspicious: unable to find the command with handle %x req->id %d rsp->id %d\n",
@@ -4110,7 +4115,7 @@ static void __qlt_do_work(struct qla_tgt_cmd *cmd)
 
 	spin_lock_init(&cmd->cmd_lock);
 	cdb = &atio->u.isp24.fcp_cmnd.cdb[0];
-	cmd->se_cmd.tag = atio->u.isp24.exchange_addr;
+	cmd->se_cmd.tag = le32_to_cpu(atio->u.isp24.exchange_addr);
 
 	if (atio->u.isp24.fcp_cmnd.rddata &&
 	    atio->u.isp24.fcp_cmnd.wrdata) {
@@ -5302,7 +5307,7 @@ static int __qlt_send_busy(struct qla_qpair *qpair,
 
 	ctio24 = (struct ctio7_to_24xx *)pkt;
 	ctio24->entry_type = CTIO_TYPE7;
-	ctio24->nport_handle = sess->loop_id;
+	ctio24->nport_handle = cpu_to_le16(sess->loop_id);
 	ctio24->timeout = cpu_to_le16(QLA_TGT_TIMEOUT);
 	ctio24->vp_index = vha->vp_idx;
 	ctio24->initiator_id = be_id_to_le(atio->u.isp24.fcp_hdr.s_id);
@@ -5315,13 +5320,14 @@ static int __qlt_send_busy(struct qla_qpair *qpair,
 	 * CTIO from fw w/o se_cmd doesn't provide enough info to retry it,
 	 * if the explicit conformation is used.
 	 */
-	ctio24->u.status1.ox_id = swab16(atio->u.isp24.fcp_hdr.ox_id);
+	ctio24->u.status1.ox_id =
+		cpu_to_le16(be16_to_cpu(atio->u.isp24.fcp_hdr.ox_id));
 	ctio24->u.status1.scsi_status = cpu_to_le16(status);
 
-	ctio24->u.status1.residual = get_datalen_for_atio(atio);
+	ctio24->u.status1.residual = cpu_to_le32(get_datalen_for_atio(atio));
 
 	if (ctio24->u.status1.residual != 0)
-		ctio24->u.status1.scsi_status |= SS_RESIDUAL_UNDER;
+		ctio24->u.status1.scsi_status |= cpu_to_le16(SS_RESIDUAL_UNDER);
 
 	/* Memory Barrier */
 	wmb();
@@ -5550,7 +5556,7 @@ static void qlt_24xx_atio_pkt(struct scsi_qla_host *vha,
 	switch (atio->u.raw.entry_type) {
 	case ATIO_TYPE7:
 		if (unlikely(atio->u.isp24.exchange_addr ==
-		    ATIO_EXCHANGE_ADDRESS_UNKNOWN)) {
+			     cpu_to_le32(ATIO_EXCHANGE_ADDRESS_UNKNOWN))) {
 			ql_dbg(ql_dbg_io, vha, 0x3065,
 			    "qla_target(%d): ATIO_TYPE7 "
 			    "received with UNKNOWN exchange address, "
@@ -5670,9 +5676,9 @@ static int qlt_chk_unresolv_exchg(struct scsi_qla_host *vha,
 			    vha, 0xffff, (uint8_t *)entry, sizeof(*entry));
 
 			if (qpair == ha->base_qpair)
-				ha->isp_ops->fw_dump(vha, 1);
+				ha->isp_ops->fw_dump(vha);
 			else
-				ha->isp_ops->fw_dump(vha, 0);
+				qla2xxx_dump_fw(vha);
 
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 			qla2xxx_wake_dpc(vha);
@@ -5713,8 +5719,8 @@ static void qlt_handle_abts_completion(struct scsi_qla_host *vha,
 	    entry->compl_status);
 
 	if (le16_to_cpu(entry->compl_status) != ABTS_RESP_COMPL_SUCCESS) {
-		if ((entry->error_subcode1 == 0x1E) &&
-		    (entry->error_subcode2 == 0)) {
+		if (le32_to_cpu(entry->error_subcode1) == 0x1E &&
+		    le32_to_cpu(entry->error_subcode2) == 0) {
 			if (qlt_chk_unresolv_exchg(vha, rsp->qpair, entry)) {
 				ha->tgt.tgt_ops->free_mcmd(mcmd);
 				return;
@@ -5928,11 +5934,10 @@ void qlt_async_event(uint16_t code, struct scsi_qla_host *vha,
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf03b,
 		    "qla_target(%d): Async LOOP_UP occurred "
 		    "(m[0]=%x, m[1]=%x, m[2]=%x, m[3]=%x)", vha->vp_idx,
-		    le16_to_cpu(mailbox[0]), le16_to_cpu(mailbox[1]),
-		    le16_to_cpu(mailbox[2]), le16_to_cpu(mailbox[3]));
+		    mailbox[0], mailbox[1], mailbox[2], mailbox[3]);
 		if (tgt->link_reinit_iocb_pending) {
 			qlt_send_notify_ack(ha->base_qpair,
-			    (void *)&tgt->link_reinit_iocb,
+			    &tgt->link_reinit_iocb,
 			    0, 0, 0, 0, 0, 0);
 			tgt->link_reinit_iocb_pending = 0;
 		}
@@ -5946,18 +5951,16 @@ void qlt_async_event(uint16_t code, struct scsi_qla_host *vha,
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf03c,
 		    "qla_target(%d): Async event %#x occurred "
 		    "(m[0]=%x, m[1]=%x, m[2]=%x, m[3]=%x)", vha->vp_idx, code,
-		    le16_to_cpu(mailbox[0]), le16_to_cpu(mailbox[1]),
-		    le16_to_cpu(mailbox[2]), le16_to_cpu(mailbox[3]));
+		    mailbox[0], mailbox[1], mailbox[2], mailbox[3]);
 		break;
 
 	case MBA_REJECTED_FCP_CMD:
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf017,
 		    "qla_target(%d): Async event LS_REJECT occurred (m[0]=%x, m[1]=%x, m[2]=%x, m[3]=%x)",
 		    vha->vp_idx,
-		    le16_to_cpu(mailbox[0]), le16_to_cpu(mailbox[1]),
-		    le16_to_cpu(mailbox[2]), le16_to_cpu(mailbox[3]));
+		    mailbox[0], mailbox[1], mailbox[2], mailbox[3]);
 
-		if (le16_to_cpu(mailbox[3]) == 1) {
+		if (mailbox[3] == 1) {
 			/* exchange starvation. */
 			vha->hw->exch_starvation++;
 			if (vha->hw->exch_starvation > 5) {
@@ -5981,10 +5984,9 @@ void qlt_async_event(uint16_t code, struct scsi_qla_host *vha,
 		    "qla_target(%d): Port update async event %#x "
 		    "occurred: updating the ports database (m[0]=%x, m[1]=%x, "
 		    "m[2]=%x, m[3]=%x)", vha->vp_idx, code,
-		    le16_to_cpu(mailbox[0]), le16_to_cpu(mailbox[1]),
-		    le16_to_cpu(mailbox[2]), le16_to_cpu(mailbox[3]));
+		    mailbox[0], mailbox[1], mailbox[2], mailbox[3]);
 
-		login_code = le16_to_cpu(mailbox[2]);
+		login_code = mailbox[2];
 		if (login_code == 0x4) {
 			ql_dbg(ql_dbg_tgt_mgt, vha, 0xf03e,
 			    "Async MB 2: Got PLOGI Complete\n");
@@ -6661,9 +6663,14 @@ static void qlt_disable_vha(struct scsi_qla_host *vha)
 
 	set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 	qla2xxx_wake_dpc(vha);
+
+	/*
+	 * We are expecting the offline state.
+	 * QLA_FUNCTION_FAILED means that adapter is offline.
+	 */
 	if (qla2x00_wait_for_hba_online(vha) != QLA_SUCCESS)
 		ql_dbg(ql_dbg_tgt, vha, 0xe081,
-		       "qla2x00_wait_for_hba_online() failed\n");
+		       "adapter is offline\n");
 }
 
 /*
@@ -6729,7 +6736,7 @@ qlt_init_atio_q_entries(struct scsi_qla_host *vha)
 		return;
 
 	for (cnt = 0; cnt < ha->tgt.atio_q_length; cnt++) {
-		pkt->u.raw.signature = ATIO_PROCESSED;
+		pkt->u.raw.signature = cpu_to_le32(ATIO_PROCESSED);
 		pkt++;
 	}
 
@@ -6764,7 +6771,7 @@ qlt_24xx_process_atio_queue(struct scsi_qla_host *vha, uint8_t ha_locked)
 			    "corrupted fcp frame SID[%3phN] OXID[%04x] EXCG[%x] %64phN\n",
 			    &pkt->u.isp24.fcp_hdr.s_id,
 			    be16_to_cpu(pkt->u.isp24.fcp_hdr.ox_id),
-			    le32_to_cpu(pkt->u.isp24.exchange_addr), pkt);
+			    pkt->u.isp24.exchange_addr, pkt);
 
 			adjust_corrupted_atio(pkt);
 			qlt_send_term_exchange(ha->base_qpair, NULL, pkt,
@@ -6782,14 +6789,14 @@ qlt_24xx_process_atio_queue(struct scsi_qla_host *vha, uint8_t ha_locked)
 			} else
 				ha->tgt.atio_ring_ptr++;
 
-			pkt->u.raw.signature = ATIO_PROCESSED;
+			pkt->u.raw.signature = cpu_to_le32(ATIO_PROCESSED);
 			pkt = (struct atio_from_isp *)ha->tgt.atio_ring_ptr;
 		}
 		wmb();
 	}
 
 	/* Adjust ring index */
-	WRT_REG_DWORD(ISP_ATIO_Q_OUT(vha), ha->tgt.atio_ring_index);
+	wrt_reg_dword(ISP_ATIO_Q_OUT(vha), ha->tgt.atio_ring_index);
 }
 
 void
@@ -6802,19 +6809,19 @@ qlt_24xx_config_rings(struct scsi_qla_host *vha)
 	if (!QLA_TGT_MODE_ENABLED())
 		return;
 
-	WRT_REG_DWORD(ISP_ATIO_Q_IN(vha), 0);
-	WRT_REG_DWORD(ISP_ATIO_Q_OUT(vha), 0);
-	RD_REG_DWORD(ISP_ATIO_Q_OUT(vha));
+	wrt_reg_dword(ISP_ATIO_Q_IN(vha), 0);
+	wrt_reg_dword(ISP_ATIO_Q_OUT(vha), 0);
+	rd_reg_dword(ISP_ATIO_Q_OUT(vha));
 
 	if (ha->flags.msix_enabled) {
 		if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			if (IS_QLA2071(ha)) {
 				/* 4 ports Baker: Enable Interrupt Handshake */
 				icb->msix_atio = 0;
-				icb->firmware_options_2 |= BIT_26;
+				icb->firmware_options_2 |= cpu_to_le32(BIT_26);
 			} else {
 				icb->msix_atio = cpu_to_le16(msix->entry);
-				icb->firmware_options_2 &= ~BIT_26;
+				icb->firmware_options_2 &= cpu_to_le32(~BIT_26);
 			}
 			ql_dbg(ql_dbg_init, vha, 0xf072,
 			    "Registering ICB vector 0x%x for atio que.\n",
@@ -6824,7 +6831,7 @@ qlt_24xx_config_rings(struct scsi_qla_host *vha)
 		/* INTx|MSI */
 		if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			icb->msix_atio = 0;
-			icb->firmware_options_2 |= BIT_26;
+			icb->firmware_options_2 |= cpu_to_le32(BIT_26);
 			ql_dbg(ql_dbg_init, vha, 0xf072,
 			    "%s: Use INTx for ATIOQ.\n", __func__);
 		}
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index 3cf8590feeac..010f12523b2a 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -135,37 +135,37 @@ struct nack_to_isp {
 	uint8_t	 entry_status;		    /* Entry Status. */
 	union {
 		struct {
-			uint32_t sys_define_2; /* System defined. */
+			__le32	sys_define_2; /* System defined. */
 			target_id_t target;
 			uint8_t	 target_id;
 			uint8_t	 reserved_1;
-			uint16_t flags;
-			uint16_t resp_code;
-			uint16_t status;
-			uint16_t task_flags;
-			uint16_t seq_id;
-			uint16_t srr_rx_id;
-			uint32_t srr_rel_offs;
-			uint16_t srr_ui;
-			uint16_t srr_flags;
-			uint16_t srr_reject_code;
+			__le16	flags;
+			__le16	resp_code;
+			__le16	status;
+			__le16	task_flags;
+			__le16	seq_id;
+			__le16	srr_rx_id;
+			__le32	srr_rel_offs;
+			__le16	srr_ui;
+			__le16	srr_flags;
+			__le16	srr_reject_code;
 			uint8_t  srr_reject_vendor_uniq;
 			uint8_t  srr_reject_code_expl;
 			uint8_t  reserved_2[24];
 		} isp2x;
 		struct {
 			uint32_t handle;
-			uint16_t nport_handle;
+			__le16	nport_handle;
 			uint16_t reserved_1;
-			uint16_t flags;
-			uint16_t srr_rx_id;
-			uint16_t status;
+			__le16	flags;
+			__le16	srr_rx_id;
+			__le16	status;
 			uint8_t  status_subcode;
 			uint8_t  fw_handle;
-			uint32_t exchange_address;
-			uint32_t srr_rel_offs;
-			uint16_t srr_ui;
-			uint16_t srr_flags;
+			__le32	exchange_address;
+			__le32	srr_rel_offs;
+			__le16	srr_ui;
+			__le16	srr_flags;
 			uint8_t  reserved_4[19];
 			uint8_t  vp_index;
 			uint8_t  srr_reject_vendor_uniq;
@@ -175,7 +175,7 @@ struct nack_to_isp {
 		} isp24;
 	} u;
 	uint8_t  reserved[2];
-	uint16_t ox_id;
+	__le16	ox_id;
 } __packed;
 #define NOTIFY_ACK_FLAGS_TERMINATE	BIT_3
 #define NOTIFY_ACK_SRR_FLAGS_ACCEPT	0
@@ -206,16 +206,16 @@ struct ctio_to_2xxx {
 	uint8_t	 entry_status;		/* Entry Status. */
 	uint32_t handle;		/* System defined handle */
 	target_id_t target;
-	uint16_t rx_id;
-	uint16_t flags;
-	uint16_t status;
-	uint16_t timeout;		/* 0 = 30 seconds, 0xFFFF = disable */
-	uint16_t dseg_count;		/* Data segment count. */
-	uint32_t relative_offset;
-	uint32_t residual;
-	uint16_t reserved_1[3];
-	uint16_t scsi_status;
-	uint32_t transfer_length;
+	__le16	rx_id;
+	__le16	flags;
+	__le16	status;
+	__le16	timeout;		/* 0 = 30 seconds, 0xFFFF = disable */
+	__le16	dseg_count;		/* Data segment count. */
+	__le32	relative_offset;
+	__le32	residual;
+	__le16	reserved_1[3];
+	__le16	scsi_status;
+	__le32	transfer_length;
 	struct dsd32 dsd[3];
 } __packed;
 #define ATIO_PATH_INVALID       0x07
@@ -257,7 +257,7 @@ struct fcp_hdr {
 	uint16_t seq_cnt;
 	__be16   ox_id;
 	uint16_t rx_id;
-	uint32_t parameter;
+	__le32	parameter;
 } __packed;
 
 struct fcp_hdr_le {
@@ -267,12 +267,12 @@ struct fcp_hdr_le {
 	uint8_t  cs_ctl;
 	uint8_t  f_ctl[3];
 	uint8_t  type;
-	uint16_t seq_cnt;
+	__le16	seq_cnt;
 	uint8_t  df_ctl;
 	uint8_t  seq_id;
-	uint16_t rx_id;
-	uint16_t ox_id;
-	uint32_t parameter;
+	__le16	rx_id;
+	__le16	ox_id;
+	__le32	parameter;
 } __packed;
 
 #define F_CTL_EXCH_CONTEXT_RESP	BIT_23
@@ -306,7 +306,7 @@ struct atio7_fcp_cmnd {
 	 * BUILD_BUG_ON in qlt_init().
 	 */
 	uint8_t  add_cdb[4];
-	/* uint32_t data_length; */
+	/* __le32	data_length; */
 } __packed;
 
 /*
@@ -316,31 +316,31 @@ struct atio7_fcp_cmnd {
 struct atio_from_isp {
 	union {
 		struct {
-			uint16_t entry_hdr;
+			__le16	entry_hdr;
 			uint8_t  sys_define;   /* System defined. */
 			uint8_t  entry_status; /* Entry Status.   */
-			uint32_t sys_define_2; /* System defined. */
+			__le32	sys_define_2; /* System defined. */
 			target_id_t target;
-			uint16_t rx_id;
-			uint16_t flags;
-			uint16_t status;
+			__le16	rx_id;
+			__le16	flags;
+			__le16	status;
 			uint8_t  command_ref;
 			uint8_t  task_codes;
 			uint8_t  task_flags;
 			uint8_t  execution_codes;
 			uint8_t  cdb[MAX_CMDSZ];
-			uint32_t data_length;
-			uint16_t lun;
+			__le32	data_length;
+			__le16	lun;
 			uint8_t  initiator_port_name[WWN_SIZE]; /* on qla23xx */
-			uint16_t reserved_32[6];
-			uint16_t ox_id;
+			__le16	reserved_32[6];
+			__le16	ox_id;
 		} isp2x;
 		struct {
-			uint16_t entry_hdr;
+			__le16	entry_hdr;
 			uint8_t  fcp_cmnd_len_low;
 			uint8_t  fcp_cmnd_len_high:4;
 			uint8_t  attr:4;
-			uint32_t exchange_addr;
+			__le32	exchange_addr;
 #define ATIO_EXCHANGE_ADDRESS_UNKNOWN	0xFFFFFFFF
 			struct fcp_hdr fcp_hdr;
 			struct atio7_fcp_cmnd fcp_cmnd;
@@ -352,7 +352,7 @@ struct atio_from_isp {
 #define FCP_CMD_LENGTH_MASK 0x0fff
 #define FCP_CMD_LENGTH_MIN  0x38
 			uint8_t  data[56];
-			uint32_t signature;
+			__le32	signature;
 #define ATIO_PROCESSED 0xDEADDEAD		/* Signature */
 		} raw;
 	} u;
@@ -395,36 +395,36 @@ struct ctio7_to_24xx {
 	uint8_t	 sys_define;		    /* System defined. */
 	uint8_t	 entry_status;		    /* Entry Status. */
 	uint32_t handle;		    /* System defined handle */
-	uint16_t nport_handle;
+	__le16	nport_handle;
 #define CTIO7_NHANDLE_UNRECOGNIZED	0xFFFF
-	uint16_t timeout;
-	uint16_t dseg_count;		    /* Data segment count. */
+	__le16	timeout;
+	__le16	dseg_count;		    /* Data segment count. */
 	uint8_t  vp_index;
 	uint8_t  add_flags;
 	le_id_t  initiator_id;
 	uint8_t  reserved;
-	uint32_t exchange_addr;
+	__le32	exchange_addr;
 	union {
 		struct {
-			uint16_t reserved1;
+			__le16	reserved1;
 			__le16 flags;
-			uint32_t residual;
+			__le32	residual;
 			__le16 ox_id;
-			uint16_t scsi_status;
-			uint32_t relative_offset;
-			uint32_t reserved2;
-			uint32_t transfer_length;
-			uint32_t reserved3;
+			__le16	scsi_status;
+			__le32	relative_offset;
+			__le32	reserved2;
+			__le32	transfer_length;
+			__le32	reserved3;
 			struct dsd64 dsd;
 		} status0;
 		struct {
-			uint16_t sense_length;
+			__le16	sense_length;
 			__le16 flags;
-			uint32_t residual;
+			__le32	residual;
 			__le16 ox_id;
-			uint16_t scsi_status;
-			uint16_t response_len;
-			uint16_t reserved;
+			__le16	scsi_status;
+			__le16	response_len;
+			__le16	reserved;
 			uint8_t sense_data[24];
 		} status1;
 	} u;
@@ -440,18 +440,18 @@ struct ctio7_from_24xx {
 	uint8_t	 sys_define;		    /* System defined. */
 	uint8_t	 entry_status;		    /* Entry Status. */
 	uint32_t handle;		    /* System defined handle */
-	uint16_t status;
-	uint16_t timeout;
-	uint16_t dseg_count;		    /* Data segment count. */
+	__le16	status;
+	__le16	timeout;
+	__le16	dseg_count;		    /* Data segment count. */
 	uint8_t  vp_index;
 	uint8_t  reserved1[5];
-	uint32_t exchange_address;
-	uint16_t reserved2;
-	uint16_t flags;
-	uint32_t residual;
-	uint16_t ox_id;
-	uint16_t reserved3;
-	uint32_t relative_offset;
+	__le32	exchange_address;
+	__le16	reserved2;
+	__le16	flags;
+	__le32	residual;
+	__le16	ox_id;
+	__le16	reserved3;
+	__le32	relative_offset;
 	uint8_t  reserved4[24];
 } __packed;
 
@@ -489,29 +489,29 @@ struct ctio_crc2_to_fw {
 	uint8_t entry_status;		/* Entry Status. */
 
 	uint32_t handle;		/* System handle. */
-	uint16_t nport_handle;		/* N_PORT handle. */
+	__le16	nport_handle;		/* N_PORT handle. */
 	__le16 timeout;		/* Command timeout. */
 
-	uint16_t dseg_count;		/* Data segment count. */
+	__le16	dseg_count;		/* Data segment count. */
 	uint8_t  vp_index;
 	uint8_t  add_flags;		/* additional flags */
 #define CTIO_CRC2_AF_DIF_DSD_ENA BIT_3
 
 	le_id_t  initiator_id;		/* initiator ID */
 	uint8_t  reserved1;
-	uint32_t exchange_addr;		/* rcv exchange address */
-	uint16_t reserved2;
+	__le32	exchange_addr;		/* rcv exchange address */
+	__le16	reserved2;
 	__le16 flags;			/* refer to CTIO7 flags values */
-	uint32_t residual;
+	__le32	residual;
 	__le16 ox_id;
-	uint16_t scsi_status;
+	__le16	scsi_status;
 	__le32 relative_offset;
-	uint32_t reserved5;
+	__le32	reserved5;
 	__le32 transfer_length;		/* total fc transfer length */
-	uint32_t reserved6;
+	__le32	reserved6;
 	__le64	 crc_context_address __packed; /* Data segment address. */
-	uint16_t crc_context_len;	/* Data segment length. */
-	uint16_t reserved_1;		/* MUST be set to 0. */
+	__le16	crc_context_len;	/* Data segment length. */
+	__le16	reserved_1;		/* MUST be set to 0. */
 };
 
 /* CTIO Type CRC_x Status IOCB */
@@ -522,20 +522,20 @@ struct ctio_crc_from_fw {
 	uint8_t entry_status;		/* Entry Status. */
 
 	uint32_t handle;		/* System handle. */
-	uint16_t status;
-	uint16_t timeout;		/* Command timeout. */
-	uint16_t dseg_count;		/* Data segment count. */
-	uint32_t reserved1;
-	uint16_t state_flags;
+	__le16	status;
+	__le16	timeout;		/* Command timeout. */
+	__le16	dseg_count;		/* Data segment count. */
+	__le32	reserved1;
+	__le16	state_flags;
 #define CTIO_CRC_SF_DIF_CHOPPED BIT_4
 
-	uint32_t exchange_address;	/* rcv exchange address */
-	uint16_t reserved2;
-	uint16_t flags;
-	uint32_t resid_xfer_length;
-	uint16_t ox_id;
+	__le32	exchange_address;	/* rcv exchange address */
+	__le16	reserved2;
+	__le16	flags;
+	__le32	resid_xfer_length;
+	__le16	ox_id;
 	uint8_t  reserved3[12];
-	uint16_t runt_guard;		/* reported runt blk guard */
+	__le16	runt_guard;		/* reported runt blk guard */
 	uint8_t  actual_dif[8];
 	uint8_t  expected_dif[8];
 } __packed;
@@ -558,29 +558,29 @@ struct abts_recv_from_24xx {
 	uint8_t	 sys_define;		    /* System defined. */
 	uint8_t	 entry_status;		    /* Entry Status. */
 	uint8_t  reserved_1[6];
-	uint16_t nport_handle;
+	__le16	nport_handle;
 	uint8_t  reserved_2[2];
 	uint8_t  vp_index;
 	uint8_t  reserved_3:4;
 	uint8_t  sof_type:4;
-	uint32_t exchange_address;
+	__le32	exchange_address;
 	struct fcp_hdr_le fcp_hdr_le;
 	uint8_t  reserved_4[16];
-	uint32_t exchange_addr_to_abort;
+	__le32	exchange_addr_to_abort;
 } __packed;
 
 #define ABTS_PARAM_ABORT_SEQ		BIT_0
 
 struct ba_acc_le {
-	uint16_t reserved;
+	__le16	reserved;
 	uint8_t  seq_id_last;
 	uint8_t  seq_id_valid;
 #define SEQ_ID_VALID	0x80
 #define SEQ_ID_INVALID	0x00
-	uint16_t rx_id;
-	uint16_t ox_id;
-	uint16_t high_seq_cnt;
-	uint16_t low_seq_cnt;
+	__le16	rx_id;
+	__le16	ox_id;
+	__le16	high_seq_cnt;
+	__le16	low_seq_cnt;
 } __packed;
 
 struct ba_rjt_le {
@@ -604,21 +604,21 @@ struct abts_resp_to_24xx {
 	uint8_t	 sys_define;		    /* System defined. */
 	uint8_t	 entry_status;		    /* Entry Status. */
 	uint32_t handle;
-	uint16_t reserved_1;
-	uint16_t nport_handle;
-	uint16_t control_flags;
+	__le16	reserved_1;
+	__le16	nport_handle;
+	__le16	control_flags;
 #define ABTS_CONTR_FLG_TERM_EXCHG	BIT_0
 	uint8_t  vp_index;
 	uint8_t  reserved_3:4;
 	uint8_t  sof_type:4;
-	uint32_t exchange_address;
+	__le32	exchange_address;
 	struct fcp_hdr_le fcp_hdr_le;
 	union {
 		struct ba_acc_le ba_acct;
 		struct ba_rjt_le ba_rjt;
 	} __packed payload;
-	uint32_t reserved_4;
-	uint32_t exchange_addr_to_abort;
+	__le32	reserved_4;
+	__le32	exchange_addr_to_abort;
 } __packed;
 
 /*
@@ -634,21 +634,21 @@ struct abts_resp_from_24xx_fw {
 	uint8_t	 sys_define;		    /* System defined. */
 	uint8_t	 entry_status;		    /* Entry Status. */
 	uint32_t handle;
-	uint16_t compl_status;
+	__le16	compl_status;
 #define ABTS_RESP_COMPL_SUCCESS		0
 #define ABTS_RESP_COMPL_SUBCODE_ERROR	0x31
-	uint16_t nport_handle;
-	uint16_t reserved_1;
+	__le16	nport_handle;
+	__le16	reserved_1;
 	uint8_t  reserved_2;
 	uint8_t  reserved_3:4;
 	uint8_t  sof_type:4;
-	uint32_t exchange_address;
+	__le32	exchange_address;
 	struct fcp_hdr_le fcp_hdr_le;
 	uint8_t reserved_4[8];
-	uint32_t error_subcode1;
+	__le32	error_subcode1;
 #define ABTS_RESP_SUBCODE_ERR_ABORTED_EXCH_NOT_TERM	0x1E
-	uint32_t error_subcode2;
-	uint32_t exchange_addr_to_abort;
+	__le32	error_subcode2;
+	__le32	exchange_addr_to_abort;
 } __packed;
 
 /********************************************************************\
diff --git a/drivers/scsi/qla2xxx/qla_tmpl.c b/drivers/scsi/qla2xxx/qla_tmpl.c
index 6aeb1c3fb7a8..8dc82cfd38b2 100644
--- a/drivers/scsi/qla2xxx/qla_tmpl.c
+++ b/drivers/scsi/qla2xxx/qla_tmpl.c
@@ -12,6 +12,33 @@
 #define IOBASE(vha)	IOBAR(ISPREG(vha))
 #define INVALID_ENTRY ((struct qla27xx_fwdt_entry *)0xffffffffffffffffUL)
 
+/* hardware_lock assumed held. */
+static void
+qla27xx_write_remote_reg(struct scsi_qla_host *vha,
+			 u32 addr, u32 data)
+{
+	struct device_reg_24xx __iomem *reg = &vha->hw->iobase->isp24;
+
+	ql_dbg(ql_dbg_misc, vha, 0xd300,
+	       "%s: addr/data = %xh/%xh\n", __func__, addr, data);
+
+	wrt_reg_dword(&reg->iobase_addr, 0x40);
+	wrt_reg_dword(&reg->iobase_c4, data);
+	wrt_reg_dword(&reg->iobase_window, addr);
+}
+
+void
+qla27xx_reset_mpi(scsi_qla_host_t *vha)
+{
+	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd301,
+	       "Entered %s.\n", __func__);
+
+	qla27xx_write_remote_reg(vha, 0x104050, 0x40004);
+	qla27xx_write_remote_reg(vha, 0x10405c, 0x4);
+
+	vha->hw->stat.num_mpi_reset++;
+}
+
 static inline void
 qla27xx_insert16(uint16_t value, void *buf, ulong *len)
 {
@@ -48,7 +75,7 @@ qla27xx_read8(void __iomem *window, void *buf, ulong *len)
 	uint8_t value = ~0;
 
 	if (buf) {
-		value = RD_REG_BYTE(window);
+		value = rd_reg_byte(window);
 	}
 	qla27xx_insert32(value, buf, len);
 }
@@ -59,7 +86,7 @@ qla27xx_read16(void __iomem *window, void *buf, ulong *len)
 	uint16_t value = ~0;
 
 	if (buf) {
-		value = RD_REG_WORD(window);
+		value = rd_reg_word(window);
 	}
 	qla27xx_insert32(value, buf, len);
 }
@@ -70,7 +97,7 @@ qla27xx_read32(void __iomem *window, void *buf, ulong *len)
 	uint32_t value = ~0;
 
 	if (buf) {
-		value = RD_REG_DWORD(window);
+		value = rd_reg_dword(window);
 	}
 	qla27xx_insert32(value, buf, len);
 }
@@ -99,7 +126,7 @@ qla27xx_write_reg(__iomem struct device_reg_24xx *reg,
 	if (buf) {
 		void __iomem *window = (void __iomem *)reg + offset;
 
-		WRT_REG_DWORD(window, data);
+		wrt_reg_dword(window, data);
 	}
 }
 
@@ -892,9 +919,9 @@ static void
 qla27xx_firmware_info(struct scsi_qla_host *vha,
     struct qla27xx_fwdt_template *tmp)
 {
-	tmp->firmware_version[0] = vha->hw->fw_major_version;
-	tmp->firmware_version[1] = vha->hw->fw_minor_version;
-	tmp->firmware_version[2] = vha->hw->fw_subminor_version;
+	tmp->firmware_version[0] = cpu_to_le32(vha->hw->fw_major_version);
+	tmp->firmware_version[1] = cpu_to_le32(vha->hw->fw_minor_version);
+	tmp->firmware_version[2] = cpu_to_le32(vha->hw->fw_subminor_version);
 	tmp->firmware_version[3] = cpu_to_le32(
 		vha->hw->fw_attributes_h << 16 | vha->hw->fw_attributes);
 	tmp->firmware_version[4] = cpu_to_le32(
@@ -998,14 +1025,65 @@ qla27xx_fwdt_template_valid(void *p)
 }
 
 void
-qla27xx_fwdump(scsi_qla_host_t *vha, int hardware_locked)
+qla27xx_mpi_fwdump(scsi_qla_host_t *vha, int hardware_locked)
 {
 	ulong flags = 0;
+	bool need_mpi_reset = true;
 
 #ifndef __CHECKER__
 	if (!hardware_locked)
 		spin_lock_irqsave(&vha->hw->hardware_lock, flags);
 #endif
+	if (!vha->hw->mpi_fw_dump) {
+		ql_log(ql_log_warn, vha, 0x02f3, "-> mpi_fwdump no buffer\n");
+	} else if (vha->hw->mpi_fw_dumped) {
+		ql_log(ql_log_warn, vha, 0x02f4,
+		       "-> MPI firmware already dumped (%p) -- ignoring request\n",
+		       vha->hw->mpi_fw_dump);
+	} else {
+		struct fwdt *fwdt = &vha->hw->fwdt[1];
+		ulong len;
+		void *buf = vha->hw->mpi_fw_dump;
+
+		ql_log(ql_log_warn, vha, 0x02f5, "-> fwdt1 running...\n");
+		if (!fwdt->template) {
+			ql_log(ql_log_warn, vha, 0x02f6,
+			       "-> fwdt1 no template\n");
+			goto bailout;
+		}
+		len = qla27xx_execute_fwdt_template(vha, fwdt->template, buf);
+		if (len == 0) {
+			goto bailout;
+		} else if (len != fwdt->dump_size) {
+			ql_log(ql_log_warn, vha, 0x02f7,
+			       "-> fwdt1 fwdump residual=%+ld\n",
+			       fwdt->dump_size - len);
+		} else {
+			need_mpi_reset = false;
+		}
+
+		vha->hw->mpi_fw_dump_len = len;
+		vha->hw->mpi_fw_dumped = 1;
+
+		ql_log(ql_log_warn, vha, 0x02f8,
+		       "-> MPI firmware dump saved to buffer (%lu/%p)\n",
+		       vha->host_no, vha->hw->mpi_fw_dump);
+		qla2x00_post_uevent_work(vha, QLA_UEVENT_CODE_FW_DUMP);
+	}
+
+bailout:
+	if (need_mpi_reset)
+		qla27xx_reset_mpi(vha);
+#ifndef __CHECKER__
+	if (!hardware_locked)
+		spin_unlock_irqrestore(&vha->hw->hardware_lock, flags);
+#endif
+}
+
+void
+qla27xx_fwdump(scsi_qla_host_t *vha)
+{
+	lockdep_assert_held(&vha->hw->hardware_lock);
 
 	if (!vha->hw->fw_dump) {
 		ql_log(ql_log_warn, vha, 0xd01e, "-> fwdump no buffer\n");
@@ -1015,42 +1093,30 @@ qla27xx_fwdump(scsi_qla_host_t *vha, int hardware_locked)
 		    vha->hw->fw_dump);
 	} else {
 		struct fwdt *fwdt = vha->hw->fwdt;
-		uint j;
 		ulong len;
 		void *buf = vha->hw->fw_dump;
-		uint count = vha->hw->fw_dump_mpi ? 2 : 1;
-
-		for (j = 0; j < count; j++, fwdt++, buf += len) {
-			ql_log(ql_log_warn, vha, 0xd011,
-			    "-> fwdt%u running...\n", j);
-			if (!fwdt->template) {
-				ql_log(ql_log_warn, vha, 0xd012,
-				    "-> fwdt%u no template\n", j);
-				break;
-			}
-			len = qla27xx_execute_fwdt_template(vha,
-			    fwdt->template, buf);
-			if (len == 0) {
-				goto bailout;
-			} else if (len != fwdt->dump_size) {
-				ql_log(ql_log_warn, vha, 0xd013,
-				    "-> fwdt%u fwdump residual=%+ld\n",
-				    j, fwdt->dump_size - len);
-			}
+
+		ql_log(ql_log_warn, vha, 0xd011, "-> fwdt0 running...\n");
+		if (!fwdt->template) {
+			ql_log(ql_log_warn, vha, 0xd012,
+			       "-> fwdt0 no template\n");
+			return;
 		}
-		vha->hw->fw_dump_len = buf - (void *)vha->hw->fw_dump;
-		vha->hw->fw_dumped = 1;
+		len = qla27xx_execute_fwdt_template(vha, fwdt->template, buf);
+		if (len == 0) {
+			return;
+		} else if (len != fwdt->dump_size) {
+			ql_log(ql_log_warn, vha, 0xd013,
+			       "-> fwdt0 fwdump residual=%+ld\n",
+				fwdt->dump_size - len);
+		}
+
+		vha->hw->fw_dump_len = len;
+		vha->hw->fw_dumped = true;
 
 		ql_log(ql_log_warn, vha, 0xd015,
 		    "-> Firmware dump saved to buffer (%lu/%p) <%lx>\n",
 		    vha->host_no, vha->hw->fw_dump, vha->hw->fw_dump_cap_flags);
 		qla2x00_post_uevent_work(vha, QLA_UEVENT_CODE_FW_DUMP);
 	}
-
-bailout:
-	vha->hw->fw_dump_mpi = 0;
-#ifndef __CHECKER__
-	if (!hardware_locked)
-		spin_unlock_irqrestore(&vha->hw->hardware_lock, flags);
-#endif
 }
diff --git a/drivers/scsi/qla2xxx/qla_tmpl.h b/drivers/scsi/qla2xxx/qla_tmpl.h
index bba8dc90acfb..89280b3477aa 100644
--- a/drivers/scsi/qla2xxx/qla_tmpl.h
+++ b/drivers/scsi/qla2xxx/qla_tmpl.h
@@ -27,7 +27,7 @@ struct __packed qla27xx_fwdt_template {
 	uint32_t saved_state[16];
 
 	uint32_t reserved_3[8];
-	uint32_t firmware_version[5];
+	__le32 firmware_version[5];
 };
 
 #define TEMPLATE_TYPE_FWDUMP		99
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index 1f0a185b2a95..68183a96a417 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -949,6 +949,7 @@ static ssize_t tcm_qla2xxx_tpg_enable_store(struct config_item *item,
 
 		atomic_set(&tpg->lport_tpg_enabled, 0);
 		qlt_stop_phase1(vha->vha_tgt.qla_tgt);
+		qlt_stop_phase2(vha->vha_tgt.qla_tgt);
 	}
 
 	return count;
@@ -1111,6 +1112,7 @@ static ssize_t tcm_qla2xxx_npiv_tpg_enable_store(struct config_item *item,
 
 		atomic_set(&tpg->lport_tpg_enabled, 0);
 		qlt_stop_phase1(vha->vha_tgt.qla_tgt);
+		qlt_stop_phase2(vha->vha_tgt.qla_tgt);
 	}
 
 	return count;
@@ -1958,6 +1960,20 @@ static int __init tcm_qla2xxx_init(void)
 {
 	int ret;
 
+	BUILD_BUG_ON(sizeof(struct abts_recv_from_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct abts_resp_from_24xx_fw) != 64);
+	BUILD_BUG_ON(sizeof(struct atio7_fcp_cmnd) != 32);
+	BUILD_BUG_ON(sizeof(struct atio_from_isp) != 64);
+	BUILD_BUG_ON(sizeof(struct ba_acc_le) != 12);
+	BUILD_BUG_ON(sizeof(struct ba_rjt_le) != 4);
+	BUILD_BUG_ON(sizeof(struct ctio7_from_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct ctio7_to_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct ctio_crc2_to_fw) != 64);
+	BUILD_BUG_ON(sizeof(struct ctio_crc_from_fw) != 64);
+	BUILD_BUG_ON(sizeof(struct ctio_to_2xxx) != 64);
+	BUILD_BUG_ON(sizeof(struct fcp_hdr_le) != 24);
+	BUILD_BUG_ON(sizeof(struct nack_to_isp) != 64);
+
 	ret = tcm_qla2xxx_register_configfs();
 	if (ret < 0)
 		return ret;
diff --git a/drivers/scsi/qla4xxx/Kconfig b/drivers/scsi/qla4xxx/Kconfig
index 4bdf31b1407a..2fa249db6656 100644
--- a/drivers/scsi/qla4xxx/Kconfig
+++ b/drivers/scsi/qla4xxx/Kconfig
@@ -4,6 +4,6 @@ config SCSI_QLA_ISCSI
 	depends on PCI && SCSI && NET
 	select SCSI_ISCSI_ATTRS
 	select ISCSI_BOOT_SYSFS
-	---help---
+	help
 	This driver supports the QLogic 40xx (ISP4XXX), 8022 (ISP82XX)
 	and 8032 (ISP83XX) iSCSI host adapter family.
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 5504ab11decc..5dc697ce8b5d 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -966,7 +966,7 @@ static int qla4xxx_set_chap_entry(struct Scsi_Host *shost, void *data, int len)
 				   "%s: No such sysfs attribute\n", __func__);
 			rc = -ENOSYS;
 			goto exit_set_chap;
-		};
+		}
 	}
 
 	if (chap_rec.chap_type == CHAP_TYPE_IN)
diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c
index d539beef3ce8..3790e8b70bba 100644
--- a/drivers/scsi/qlogicpti.c
+++ b/drivers/scsi/qlogicpti.c
@@ -30,6 +30,7 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/firmware.h>
+#include <linux/pgtable.h>
 
 #include <asm/byteorder.h>
 
@@ -37,7 +38,6 @@
 
 #include <asm/dma.h>
 #include <asm/ptrace.h>
-#include <asm/pgtable.h>
 #include <asm/oplib.h>
 #include <asm/io.h>
 #include <asm/irq.h>
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 4c6c448dc2df..843cccb38cb7 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -7,7 +7,7 @@
  *  anything out of the ordinary is seen.
  * ^^^^^^^^^^^^^^^^^^^^^^^ Original ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  *
- * Copyright (C) 2001 - 2018 Douglas Gilbert
+ * Copyright (C) 2001 - 2020 Douglas Gilbert
  *
  *  For documentation see http://sg.danny.cz/sg/sdebug26.html
  */
@@ -39,6 +39,9 @@
 #include <linux/uuid.h>
 #include <linux/t10-pi.h>
 #include <linux/msdos_partition.h>
+#include <linux/random.h>
+#include <linux/xarray.h>
+#include <linux/prefetch.h>
 
 #include <net/checksum.h>
 
@@ -57,8 +60,8 @@
 #include "scsi_logging.h"
 
 /* make sure inq_product_rev string corresponds to this version */
-#define SDEBUG_VERSION "0188"	/* format to fit INQUIRY revision field */
-static const char *sdebug_version_date = "20190125";
+#define SDEBUG_VERSION "0189"	/* format to fit INQUIRY revision field */
+static const char *sdebug_version_date = "20200421";
 
 #define MY_NAME "scsi_debug"
 
@@ -91,6 +94,11 @@ static const char *sdebug_version_date = "20190125";
 #define MICROCODE_CHANGED_ASCQ 0x1	/* with TARGET_CHANGED_ASC */
 #define MICROCODE_CHANGED_WO_RESET_ASCQ 0x16
 #define WRITE_ERROR_ASC 0xc
+#define UNALIGNED_WRITE_ASCQ 0x4
+#define WRITE_BOUNDARY_ASCQ 0x5
+#define READ_INVDATA_ASCQ 0x6
+#define READ_BOUNDARY_ASCQ 0x7
+#define INSUFF_ZONE_ASCQ 0xe
 
 /* Additional Sense Code Qualifier (ASCQ) */
 #define ACK_NAK_TO 0x3
@@ -105,9 +113,12 @@ static const char *sdebug_version_date = "20190125";
 #define DEF_ATO 1
 #define DEF_CDB_LEN 10
 #define DEF_JDELAY   1		/* if > 0 unit is a jiffy */
+#define DEF_DEV_SIZE_PRE_INIT   0
 #define DEF_DEV_SIZE_MB   8
+#define DEF_ZBC_DEV_SIZE_MB   128
 #define DEF_DIF 0
 #define DEF_DIX 0
+#define DEF_PER_HOST_STORE false
 #define DEF_D_SENSE   0
 #define DEF_EVERY_NTH   0
 #define DEF_FAKE_RW	0
@@ -126,6 +137,7 @@ static const char *sdebug_version_date = "20190125";
 #define DEF_PHYSBLK_EXP 0
 #define DEF_OPT_XFERLEN_EXP 0
 #define DEF_PTYPE   TYPE_DISK
+#define DEF_RANDOM false
 #define DEF_REMOVABLE false
 #define DEF_SCSI_LEVEL   7    /* INQUIRY, byte2 [6->SPC-4; 7->SPC-5] */
 #define DEF_SECTOR_SIZE 512
@@ -142,6 +154,11 @@ static const char *sdebug_version_date = "20190125";
 #define DEF_UUID_CTL 0
 #define JDELAY_OVERRIDDEN -9999
 
+/* Default parameters for ZBC drives */
+#define DEF_ZBC_ZONE_SIZE_MB	128
+#define DEF_ZBC_MAX_OPEN_ZONES	8
+#define DEF_ZBC_NR_CONV_ZONES	1
+
 #define SDEBUG_LUN_0_VAL 0
 
 /* bit mask values for sdebug_opts */
@@ -219,21 +236,23 @@ static const char *sdebug_version_date = "20190125";
 #define SDEBUG_CANQUEUE  (SDEBUG_CANQUEUE_WORDS * BITS_PER_LONG)
 #define DEF_CMD_PER_LUN  255
 
-#define F_D_IN			1
-#define F_D_OUT			2
+/* UA - Unit Attention; SA - Service Action; SSU - Start Stop Unit */
+#define F_D_IN			1	/* Data-in command (e.g. READ) */
+#define F_D_OUT			2	/* Data-out command (e.g. WRITE) */
 #define F_D_OUT_MAYBE		4	/* WRITE SAME, NDOB bit */
 #define F_D_UNKN		8
-#define F_RL_WLUN_OK		0x10
-#define F_SKIP_UA		0x20
-#define F_DELAY_OVERR		0x40
-#define F_SA_LOW		0x80	/* cdb byte 1, bits 4 to 0 */
-#define F_SA_HIGH		0x100	/* as used by variable length cdbs */
-#define F_INV_OP		0x200
-#define F_FAKE_RW		0x400
-#define F_M_ACCESS		0x800	/* media access */
-#define F_SSU_DELAY		0x1000
-#define F_SYNC_DELAY		0x2000
-
+#define F_RL_WLUN_OK		0x10	/* allowed with REPORT LUNS W-LUN */
+#define F_SKIP_UA		0x20	/* bypass UAs (e.g. INQUIRY command) */
+#define F_DELAY_OVERR		0x40	/* for commands like INQUIRY */
+#define F_SA_LOW		0x80	/* SA is in cdb byte 1, bits 4 to 0 */
+#define F_SA_HIGH		0x100	/* SA is in cdb bytes 8 and 9 */
+#define F_INV_OP		0x200	/* invalid opcode (not supported) */
+#define F_FAKE_RW		0x400	/* bypass resp_*() when fake_rw set */
+#define F_M_ACCESS		0x800	/* media access, reacts to SSU state */
+#define F_SSU_DELAY		0x1000	/* SSU command delay (long-ish) */
+#define F_SYNC_DELAY		0x2000	/* SYNCHRONIZE CACHE delay */
+
+/* Useful combinations of the above flags */
 #define FF_RESPOND (F_RL_WLUN_OK | F_SKIP_UA | F_DELAY_OVERR)
 #define FF_MEDIA_IO (F_M_ACCESS | F_FAKE_RW)
 #define FF_SA (F_SA_HIGH | F_SA_LOW)
@@ -243,6 +262,35 @@ static const char *sdebug_version_date = "20190125";
 
 #define SDEBUG_MAX_CMD_LEN 32
 
+#define SDEB_XA_NOT_IN_USE XA_MARK_1
+
+/* Zone types (zbcr05 table 25) */
+enum sdebug_z_type {
+	ZBC_ZONE_TYPE_CNV	= 0x1,
+	ZBC_ZONE_TYPE_SWR	= 0x2,
+	ZBC_ZONE_TYPE_SWP	= 0x3,
+};
+
+/* enumeration names taken from table 26, zbcr05 */
+enum sdebug_z_cond {
+	ZBC_NOT_WRITE_POINTER	= 0x0,
+	ZC1_EMPTY		= 0x1,
+	ZC2_IMPLICIT_OPEN	= 0x2,
+	ZC3_EXPLICIT_OPEN	= 0x3,
+	ZC4_CLOSED		= 0x4,
+	ZC6_READ_ONLY		= 0xd,
+	ZC5_FULL		= 0xe,
+	ZC7_OFFLINE		= 0xf,
+};
+
+struct sdeb_zone_state {	/* ZBC: per zone state */
+	enum sdebug_z_type z_type;
+	enum sdebug_z_cond z_cond;
+	bool z_non_seq_resource;
+	unsigned int z_size;
+	sector_t z_start;
+	sector_t z_wp;
+};
 
 struct sdebug_dev_info {
 	struct list_head dev_list;
@@ -255,15 +303,36 @@ struct sdebug_dev_info {
 	atomic_t num_in_q;
 	atomic_t stopped;
 	bool used;
+
+	/* For ZBC devices */
+	enum blk_zoned_model zmodel;
+	unsigned int zsize;
+	unsigned int zsize_shift;
+	unsigned int nr_zones;
+	unsigned int nr_conv_zones;
+	unsigned int nr_imp_open;
+	unsigned int nr_exp_open;
+	unsigned int nr_closed;
+	unsigned int max_open;
+	struct sdeb_zone_state *zstate;
 };
 
 struct sdebug_host_info {
 	struct list_head host_list;
+	int si_idx;	/* sdeb_store_info (per host) xarray index */
 	struct Scsi_Host *shost;
 	struct device dev;
 	struct list_head dev_info_list;
 };
 
+/* There is an xarray of pointers to this struct's objects, one per host */
+struct sdeb_store_info {
+	rwlock_t macc_lck;	/* for atomic media access on this store */
+	u8 *storep;		/* user data storage (ram) */
+	struct t10_pi_tuple *dif_storep; /* protection info */
+	void *map_storep;	/* provisioning map */
+};
+
 #define to_sdebug_host(d)	\
 	container_of(d, struct sdebug_host_info, dev)
 
@@ -339,7 +408,7 @@ enum sdeb_opcode_index {
 	SDEB_I_SERV_ACT_OUT_16 = 13,	/* add ...SERV_ACT_OUT_12 if needed */
 	SDEB_I_MAINT_IN = 14,
 	SDEB_I_MAINT_OUT = 15,
-	SDEB_I_VERIFY = 16,		/* 10 only */
+	SDEB_I_VERIFY = 16,		/* VERIFY(10), VERIFY(16) */
 	SDEB_I_VARIABLE_LEN = 17,	/* READ(32), WRITE(32), WR_SCAT(32) */
 	SDEB_I_RESERVE = 18,		/* 6, 10 */
 	SDEB_I_RELEASE = 19,		/* 6, 10 */
@@ -352,7 +421,10 @@ enum sdeb_opcode_index {
 	SDEB_I_WRITE_SAME = 26,		/* 10, 16 */
 	SDEB_I_SYNC_CACHE = 27,		/* 10, 16 */
 	SDEB_I_COMP_WRITE = 28,
-	SDEB_I_LAST_ELEMENT = 29,	/* keep this last (previous + 1) */
+	SDEB_I_PRE_FETCH = 29,		/* 10, 16 */
+	SDEB_I_ZONE_OUT = 30,		/* 0x94+SA; includes no data xfer */
+	SDEB_I_ZONE_IN = 31,		/* 0x95+SA; all have data-in */
+	SDEB_I_LAST_ELEM_P1 = 32,	/* keep this last (previous + 1) */
 };
 
 
@@ -368,7 +440,7 @@ static const unsigned char opcode_ind_arr[256] = {
 /* 0x20; 0x20->0x3f: 10 byte cdbs */
 	0, 0, 0, 0, 0, SDEB_I_READ_CAPACITY, 0, 0,
 	SDEB_I_READ, 0, SDEB_I_WRITE, 0, 0, 0, 0, SDEB_I_VERIFY,
-	0, 0, 0, 0, 0, SDEB_I_SYNC_CACHE, 0, 0,
+	0, 0, 0, 0, SDEB_I_PRE_FETCH, SDEB_I_SYNC_CACHE, 0, 0,
 	0, 0, 0, SDEB_I_WRITE_BUFFER, 0, 0, 0, 0,
 /* 0x40; 0x40->0x5f: 10 byte cdbs */
 	0, SDEB_I_WRITE_SAME, SDEB_I_UNMAP, 0, 0, 0, 0, 0,
@@ -382,8 +454,10 @@ static const unsigned char opcode_ind_arr[256] = {
 	0, SDEB_I_VARIABLE_LEN,
 /* 0x80; 0x80->0x9f: 16 byte cdbs */
 	0, 0, 0, 0, 0, SDEB_I_ATA_PT, 0, 0,
-	SDEB_I_READ, SDEB_I_COMP_WRITE, SDEB_I_WRITE, 0, 0, 0, 0, 0,
-	0, SDEB_I_SYNC_CACHE, 0, SDEB_I_WRITE_SAME, 0, 0, 0, 0,
+	SDEB_I_READ, SDEB_I_COMP_WRITE, SDEB_I_WRITE, 0,
+	0, 0, 0, SDEB_I_VERIFY,
+	SDEB_I_PRE_FETCH, SDEB_I_SYNC_CACHE, 0, SDEB_I_WRITE_SAME,
+	SDEB_I_ZONE_OUT, SDEB_I_ZONE_IN, 0, 0,
 	0, 0, 0, 0, 0, 0, SDEB_I_SERV_ACT_IN_16, SDEB_I_SERV_ACT_OUT_16,
 /* 0xa0; 0xa0->0xbf: 12 byte cdbs */
 	SDEB_I_REPORT_LUNS, SDEB_I_ATA_PT, 0, SDEB_I_MAINT_IN,
@@ -424,11 +498,25 @@ static int resp_report_tgtpgs(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_unmap(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_rsup_opcodes(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_rsup_tmfs(struct scsi_cmnd *, struct sdebug_dev_info *);
+static int resp_verify(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_write_same_10(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_write_same_16(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_comp_write(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_write_buffer(struct scsi_cmnd *, struct sdebug_dev_info *);
 static int resp_sync_cache(struct scsi_cmnd *, struct sdebug_dev_info *);
+static int resp_pre_fetch(struct scsi_cmnd *, struct sdebug_dev_info *);
+static int resp_report_zones(struct scsi_cmnd *, struct sdebug_dev_info *);
+static int resp_open_zone(struct scsi_cmnd *, struct sdebug_dev_info *);
+static int resp_close_zone(struct scsi_cmnd *, struct sdebug_dev_info *);
+static int resp_finish_zone(struct scsi_cmnd *, struct sdebug_dev_info *);
+static int resp_rwp_zone(struct scsi_cmnd *, struct sdebug_dev_info *);
+
+static int sdebug_do_add_host(bool mk_new_store);
+static int sdebug_add_host_helper(int per_host_idx);
+static void sdebug_do_remove_host(bool the_end);
+static int sdebug_add_store(void);
+static void sdebug_erase_store(int idx, struct sdeb_store_info *sip);
+static void sdebug_erase_all_stores(bool apart_from_first);
 
 /*
  * The following are overflow arrays for cdbs that "hit" the same index in
@@ -468,6 +556,12 @@ static const struct opcode_info_t write_iarr[] = {
 		   0xbf, 0xc7, 0, 0, 0, 0} },
 };
 
+static const struct opcode_info_t verify_iarr[] = {
+	{0, 0x2f, 0, F_D_OUT_MAYBE | FF_MEDIA_IO, resp_verify,/* VERIFY(10) */
+	    NULL, {10,  0xf7, 0xff, 0xff, 0xff, 0xff, 0xbf, 0xff, 0xff, 0xc7,
+		   0, 0, 0, 0, 0, 0} },
+};
+
 static const struct opcode_info_t sa_in_16_iarr[] = {
 	{0, 0x9e, 0x12, F_SA_LOW | F_D_IN, resp_get_lba_status, NULL,
 	    {16,  0x12, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -514,11 +608,35 @@ static const struct opcode_info_t sync_cache_iarr[] = {
 	     0xff, 0xff, 0xff, 0xff, 0x3f, 0xc7} },	/* SYNC_CACHE (16) */
 };
 
+static const struct opcode_info_t pre_fetch_iarr[] = {
+	{0, 0x90, 0, F_SYNC_DELAY | FF_MEDIA_IO, resp_pre_fetch, NULL,
+	    {16,  0x2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	     0xff, 0xff, 0xff, 0xff, 0x3f, 0xc7} },	/* PRE-FETCH (16) */
+};
+
+static const struct opcode_info_t zone_out_iarr[] = {	/* ZONE OUT(16) */
+	{0, 0x94, 0x1, F_SA_LOW | F_M_ACCESS, resp_close_zone, NULL,
+	    {16, 0x1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	     0xff, 0, 0, 0xff, 0xff, 0x1, 0xc7} },	/* CLOSE ZONE */
+	{0, 0x94, 0x2, F_SA_LOW | F_M_ACCESS, resp_finish_zone, NULL,
+	    {16, 0x2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	     0xff, 0, 0, 0xff, 0xff, 0x1, 0xc7} },	/* FINISH ZONE */
+	{0, 0x94, 0x4, F_SA_LOW | F_M_ACCESS, resp_rwp_zone, NULL,
+	    {16, 0x4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	     0xff, 0, 0, 0xff, 0xff, 0x1, 0xc7} },  /* RESET WRITE POINTER */
+};
+
+static const struct opcode_info_t zone_in_iarr[] = {	/* ZONE IN(16) */
+	{0, 0x95, 0x6, F_SA_LOW | F_D_IN | F_M_ACCESS, NULL, NULL,
+	    {16, 0x6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	     0xff, 0xff, 0xff, 0xff, 0x3f, 0xc7} }, /* REPORT ZONES */
+};
+
 
 /* This array is accessed via SDEB_I_* values. Make sure all are mapped,
  * plus the terminating elements for logic that scans this table such as
  * REPORT SUPPORTED OPERATION CODES. */
-static const struct opcode_info_t opcode_info_arr[SDEB_I_LAST_ELEMENT + 1] = {
+static const struct opcode_info_t opcode_info_arr[SDEB_I_LAST_ELEM_P1 + 1] = {
 /* 0 */
 	{0, 0, 0, F_INV_OP | FF_RESPOND, NULL, NULL,	/* unknown opcodes */
 	    {0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} },
@@ -568,9 +686,10 @@ static const struct opcode_info_t opcode_info_arr[SDEB_I_LAST_ELEMENT + 1] = {
 /* 15 */
 	{0, 0, 0, F_INV_OP | FF_RESPOND, NULL, NULL, /* MAINT OUT */
 	    {0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} },
-	{0, 0x2f, 0, F_D_OUT_MAYBE | FF_MEDIA_IO, NULL, NULL, /* VERIFY(10) */
-	    {10,  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc7,
-	     0, 0, 0, 0, 0, 0} },
+	{ARRAY_SIZE(verify_iarr), 0x8f, 0,
+	    F_D_OUT_MAYBE | FF_MEDIA_IO, resp_verify,	/* VERIFY(16) */
+	    verify_iarr, {16,  0xf6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+			  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xc7} },
 	{ARRAY_SIZE(vl_iarr), 0x7f, 0x9, F_SA_HIGH | F_D_IN | FF_MEDIA_IO,
 	    resp_read_dt0, vl_iarr,	/* VARIABLE LENGTH, READ(32) */
 	    {32,  0xc7, 0, 0, 0, 0, 0x3f, 0x18, 0x0, 0x9, 0xfe, 0, 0xff, 0xff,
@@ -609,17 +728,31 @@ static const struct opcode_info_t opcode_info_arr[SDEB_I_LAST_ELEMENT + 1] = {
 	{0, 0x89, 0, F_D_OUT | FF_MEDIA_IO, resp_comp_write, NULL,
 	    {16,  0xf8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0,
 	     0, 0xff, 0x3f, 0xc7} },		/* COMPARE AND WRITE */
-
-/* 29 */
+	{ARRAY_SIZE(pre_fetch_iarr), 0x34, 0, F_SYNC_DELAY | FF_MEDIA_IO,
+	    resp_pre_fetch, pre_fetch_iarr,
+	    {10,  0x2, 0xff, 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xc7, 0, 0,
+	     0, 0, 0, 0} },			/* PRE-FETCH (10) */
+
+/* 30 */
+	{ARRAY_SIZE(zone_out_iarr), 0x94, 0x3, F_SA_LOW | F_M_ACCESS,
+	    resp_open_zone, zone_out_iarr, /* ZONE_OUT(16), OPEN ZONE) */
+		{16,  0x3 /* SA */, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		 0xff, 0xff, 0x0, 0x0, 0xff, 0xff, 0x1, 0xc7} },
+	{ARRAY_SIZE(zone_in_iarr), 0x95, 0x0, F_SA_LOW | F_M_ACCESS,
+	    resp_report_zones, zone_in_iarr, /* ZONE_IN(16), REPORT ZONES) */
+		{16,  0x0 /* SA */, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+		 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbf, 0xc7} },
+/* sentinel */
 	{0xff, 0, 0, 0, NULL, NULL,		/* terminating element */
 	    {0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} },
 };
 
-static int sdebug_add_host = DEF_NUM_HOST;
+static int sdebug_num_hosts;
+static int sdebug_add_host = DEF_NUM_HOST;  /* in sysfs this is relative */
 static int sdebug_ato = DEF_ATO;
 static int sdebug_cdb_len = DEF_CDB_LEN;
 static int sdebug_jdelay = DEF_JDELAY;	/* if > 0 then unit is jiffies */
-static int sdebug_dev_size_mb = DEF_DEV_SIZE_MB;
+static int sdebug_dev_size_mb = DEF_DEV_SIZE_PRE_INIT;
 static int sdebug_dif = DEF_DIF;
 static int sdebug_dix = DEF_DIX;
 static int sdebug_dsense = DEF_D_SENSE;
@@ -656,6 +789,8 @@ static unsigned int sdebug_unmap_max_blocks = DEF_UNMAP_MAX_BLOCKS;
 static unsigned int sdebug_unmap_max_desc = DEF_UNMAP_MAX_DESC;
 static unsigned int sdebug_write_same_length = DEF_WRITESAME_LENGTH;
 static int sdebug_uuid_ctl = DEF_UUID_CTL;
+static bool sdebug_random = DEF_RANDOM;
+static bool sdebug_per_host_store = DEF_PER_HOST_STORE;
 static bool sdebug_removable = DEF_REMOVABLE;
 static bool sdebug_clustering;
 static bool sdebug_host_lock = DEF_HOST_LOCK;
@@ -666,6 +801,9 @@ static bool have_dif_prot;
 static bool write_since_sync;
 static bool sdebug_statistics = DEF_STATISTICS;
 static bool sdebug_wp;
+/* Following enum: 0: no zbc, def; 1: host aware; 2: host managed */
+static enum blk_zoned_model sdeb_zbc_model = BLK_ZONED_NONE;
+static char *sdeb_zbc_model_s;
 
 static unsigned int sdebug_store_sectors;
 static sector_t sdebug_capacity;	/* in sectors */
@@ -679,9 +817,11 @@ static int sdebug_sectors_per;		/* sectors per cylinder */
 static LIST_HEAD(sdebug_host_list);
 static DEFINE_SPINLOCK(sdebug_host_list_lock);
 
-static unsigned char *fake_storep;	/* ramdisk storage */
-static struct t10_pi_tuple *dif_storep;	/* protection info */
-static void *map_storep;		/* provisioning map */
+static struct xarray per_store_arr;
+static struct xarray *per_store_ap = &per_store_arr;
+static int sdeb_first_idx = -1;		/* invalid index ==> none created */
+static int sdeb_most_recent_idx = -1;
+static DEFINE_RWLOCK(sdeb_fake_rw_lck);	/* need a RW lock when fake_rw=1 */
 
 static unsigned long map_size;
 static int num_aborts;
@@ -693,10 +833,19 @@ static int dix_writes;
 static int dix_reads;
 static int dif_errors;
 
+/* ZBC global data */
+static bool sdeb_zbc_in_use;	/* true for host-aware and host-managed disks */
+static int sdeb_zbc_zone_size_mb;
+static int sdeb_zbc_max_open = DEF_ZBC_MAX_OPEN_ZONES;
+static int sdeb_zbc_nr_conv = DEF_ZBC_NR_CONV_ZONES;
+
 static int submit_queues = DEF_SUBMIT_QUEUES;  /* > 1 for multi-queue (mq) */
 static struct sdebug_queue *sdebug_q_arr;  /* ptr to array of submit queues */
 
 static DEFINE_RWLOCK(atomic_rw);
+static DEFINE_RWLOCK(atomic_rw2);
+
+static rwlock_t *ramdisk_lck_a[2];
 
 static char sdebug_proc_name[] = MY_NAME;
 static const char *my_name = MY_NAME;
@@ -717,6 +866,8 @@ static const int illegal_condition_result =
 static const int device_qfull_result =
 	(DID_OK << 16) | (COMMAND_COMPLETE << 8) | SAM_STAT_TASK_SET_FULL;
 
+static const int condition_met_result = SAM_STAT_CONDITION_MET;
+
 
 /* Only do the extra work involved in logical block provisioning if one or
  * more of the lbpu, lbpws or lbpws10 parameters are given and we are doing
@@ -728,18 +879,25 @@ static inline bool scsi_debug_lbp(void)
 		(sdebug_lbpu || sdebug_lbpws || sdebug_lbpws10);
 }
 
-static void *lba2fake_store(unsigned long long lba)
+static void *lba2fake_store(struct sdeb_store_info *sip,
+			    unsigned long long lba)
 {
-	lba = do_div(lba, sdebug_store_sectors);
+	struct sdeb_store_info *lsip = sip;
 
-	return fake_storep + lba * sdebug_sector_size;
+	lba = do_div(lba, sdebug_store_sectors);
+	if (!sip || !sip->storep) {
+		WARN_ON_ONCE(true);
+		lsip = xa_load(per_store_ap, 0);  /* should never be NULL */
+	}
+	return lsip->storep + lba * sdebug_sector_size;
 }
 
-static struct t10_pi_tuple *dif_store(sector_t sector)
+static struct t10_pi_tuple *dif_store(struct sdeb_store_info *sip,
+				      sector_t sector)
 {
 	sector = sector_div(sector, sdebug_store_sectors);
 
-	return dif_storep + sector;
+	return sip->dif_storep + sector;
 }
 
 static void sdebug_max_tgts_luns(void)
@@ -1041,7 +1199,7 @@ static int p_fill_from_dev_buffer(struct scsi_cmnd *scp, const void *arr,
 		 __func__, off_dst, scsi_bufflen(scp), act_len,
 		 scsi_get_resid(scp));
 	n = scsi_bufflen(scp) - (off_dst + act_len);
-	scsi_set_resid(scp, min(scsi_get_resid(scp), n));
+	scsi_set_resid(scp, min_t(int, scsi_get_resid(scp), n));
 	return 0;
 }
 
@@ -1354,13 +1512,15 @@ static int inquiry_vpd_b0(unsigned char *arr)
 }
 
 /* Block device characteristics VPD page (SBC-3) */
-static int inquiry_vpd_b1(unsigned char *arr)
+static int inquiry_vpd_b1(struct sdebug_dev_info *devip, unsigned char *arr)
 {
 	memset(arr, 0, 0x3c);
 	arr[0] = 0;
 	arr[1] = 1;	/* non rotating medium (e.g. solid state) */
 	arr[2] = 0;
 	arr[3] = 5;	/* less than 1.8" */
+	if (devip->zmodel == BLK_ZONED_HA)
+		arr[4] = 1 << 4;	/* zoned field = 01b */
 
 	return 0x3c;
 }
@@ -1384,6 +1544,26 @@ static int inquiry_vpd_b2(unsigned char *arr)
 	return 0x4;
 }
 
+/* Zoned block device characteristics VPD page (ZBC mandatory) */
+static int inquiry_vpd_b6(struct sdebug_dev_info *devip, unsigned char *arr)
+{
+	memset(arr, 0, 0x3c);
+	arr[0] = 0x1; /* set URSWRZ (unrestricted read in seq. wr req zone) */
+	/*
+	 * Set Optimal number of open sequential write preferred zones and
+	 * Optimal number of non-sequentially written sequential write
+	 * preferred zones fields to 'not reported' (0xffffffff). Leave other
+	 * fields set to zero, apart from Max. number of open swrz_s field.
+	 */
+	put_unaligned_be32(0xffffffff, &arr[4]);
+	put_unaligned_be32(0xffffffff, &arr[8]);
+	if (sdeb_zbc_model == BLK_ZONED_HM && devip->max_open)
+		put_unaligned_be32(devip->max_open, &arr[12]);
+	else
+		put_unaligned_be32(0xffffffff, &arr[12]);
+	return 0x3c;
+}
+
 #define SDEBUG_LONG_INQ_SZ 96
 #define SDEBUG_MAX_INQ_ARR_SZ 584
 
@@ -1393,13 +1573,15 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	unsigned char *arr;
 	unsigned char *cmd = scp->cmnd;
 	int alloc_len, n, ret;
-	bool have_wlun, is_disk;
+	bool have_wlun, is_disk, is_zbc, is_disk_zbc;
 
 	alloc_len = get_unaligned_be16(cmd + 3);
 	arr = kzalloc(SDEBUG_MAX_INQ_ARR_SZ, GFP_ATOMIC);
 	if (! arr)
 		return DID_REQUEUE << 16;
 	is_disk = (sdebug_ptype == TYPE_DISK);
+	is_zbc = (devip->zmodel != BLK_ZONED_NONE);
+	is_disk_zbc = (is_disk || is_zbc);
 	have_wlun = scsi_is_wlun(scp->device->lun);
 	if (have_wlun)
 		pq_pdt = TYPE_WLUN;	/* present, wlun */
@@ -1437,11 +1619,14 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 			arr[n++] = 0x86;  /* extended inquiry */
 			arr[n++] = 0x87;  /* mode page policy */
 			arr[n++] = 0x88;  /* SCSI ports */
-			if (is_disk) {	  /* SBC only */
+			if (is_disk_zbc) {	  /* SBC or ZBC */
 				arr[n++] = 0x89;  /* ATA information */
 				arr[n++] = 0xb0;  /* Block limits */
 				arr[n++] = 0xb1;  /* Block characteristics */
-				arr[n++] = 0xb2;  /* Logical Block Prov */
+				if (is_disk)
+					arr[n++] = 0xb2;  /* LB Provisioning */
+				if (is_zbc)
+					arr[n++] = 0xb6;  /* ZB dev. char. */
 			}
 			arr[3] = n - 4;	  /* number of supported VPD pages */
 		} else if (0x80 == cmd[2]) { /* unit serial number */
@@ -1480,19 +1665,22 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		} else if (0x88 == cmd[2]) { /* SCSI Ports */
 			arr[1] = cmd[2];	/*sanity */
 			arr[3] = inquiry_vpd_88(&arr[4], target_dev_id);
-		} else if (is_disk && 0x89 == cmd[2]) { /* ATA information */
+		} else if (is_disk_zbc && 0x89 == cmd[2]) { /* ATA info */
 			arr[1] = cmd[2];        /*sanity */
 			n = inquiry_vpd_89(&arr[4]);
 			put_unaligned_be16(n, arr + 2);
-		} else if (is_disk && 0xb0 == cmd[2]) { /* Block limits */
+		} else if (is_disk_zbc && 0xb0 == cmd[2]) { /* Block limits */
 			arr[1] = cmd[2];        /*sanity */
 			arr[3] = inquiry_vpd_b0(&arr[4]);
-		} else if (is_disk && 0xb1 == cmd[2]) { /* Block char. */
+		} else if (is_disk_zbc && 0xb1 == cmd[2]) { /* Block char. */
 			arr[1] = cmd[2];        /*sanity */
-			arr[3] = inquiry_vpd_b1(&arr[4]);
+			arr[3] = inquiry_vpd_b1(devip, &arr[4]);
 		} else if (is_disk && 0xb2 == cmd[2]) { /* LB Prov. */
 			arr[1] = cmd[2];        /*sanity */
 			arr[3] = inquiry_vpd_b2(&arr[4]);
+		} else if (is_zbc && cmd[2] == 0xb6) { /* ZB dev. charact. */
+			arr[1] = cmd[2];        /*sanity */
+			arr[3] = inquiry_vpd_b6(devip, &arr[4]);
 		} else {
 			mk_sense_invalid_fld(scp, SDEB_IN_CDB, 2, -1);
 			kfree(arr);
@@ -1530,10 +1718,13 @@ static int resp_inquiry(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	} else if (sdebug_ptype == TYPE_TAPE) {	/* SSC-4 rev 3 */
 		put_unaligned_be16(0x525, arr + n);
 		n += 2;
+	} else if (is_zbc) {	/* ZBC BSR INCITS 536 revision 05 */
+		put_unaligned_be16(0x624, arr + n);
+		n += 2;
 	}
 	put_unaligned_be16(0x2100, arr + n);	/* SPL-4 no version claimed */
 	ret = fill_from_dev_buffer(scp, arr,
-			    min(alloc_len, SDEBUG_LONG_INQ_SZ));
+			    min_t(int, alloc_len, SDEBUG_LONG_INQ_SZ));
 	kfree(arr);
 	return ret;
 }
@@ -1688,7 +1879,7 @@ static int resp_readcap16(struct scsi_cmnd *scp,
 	}
 
 	return fill_from_dev_buffer(scp, arr,
-				    min(alloc_len, SDEBUG_READCAP16_ARR_SZ));
+			    min_t(int, alloc_len, SDEBUG_READCAP16_ARR_SZ));
 }
 
 #define SDEBUG_MAX_TGTPGS_ARR_SZ 1412
@@ -1762,9 +1953,9 @@ static int resp_report_tgtpgs(struct scsi_cmnd *scp,
 	 * - The constructed command length
 	 * - The maximum array size
 	 */
-	rlen = min(alen,n);
+	rlen = min_t(int, alen, n);
 	ret = fill_from_dev_buffer(scp, arr,
-				   min(rlen, SDEBUG_MAX_TGTPGS_ARR_SZ));
+			   min_t(int, rlen, SDEBUG_MAX_TGTPGS_ARR_SZ));
 	kfree(arr);
 	return ret;
 }
@@ -2119,7 +2310,7 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
 	unsigned char *ap;
 	unsigned char arr[SDEBUG_MAX_MSENSE_SZ];
 	unsigned char *cmd = scp->cmnd;
-	bool dbd, llbaa, msense_6, is_disk, bad_pcode;
+	bool dbd, llbaa, msense_6, is_disk, is_zbc, bad_pcode;
 
 	dbd = !!(cmd[1] & 0x8);		/* disable block descriptors */
 	pcontrol = (cmd[2] & 0xc0) >> 6;
@@ -2128,7 +2319,8 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
 	msense_6 = (MODE_SENSE == cmd[0]);
 	llbaa = msense_6 ? false : !!(cmd[1] & 0x10);
 	is_disk = (sdebug_ptype == TYPE_DISK);
-	if (is_disk && !dbd)
+	is_zbc = (devip->zmodel != BLK_ZONED_NONE);
+	if ((is_disk || is_zbc) && !dbd)
 		bd_len = llbaa ? 16 : 8;
 	else
 		bd_len = 0;
@@ -2140,8 +2332,8 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
 	}
 	target_dev_id = ((devip->sdbg_host->shost->host_no + 1) * 2000) +
 			(devip->target * 1000) - 3;
-	/* for disks set DPOFUA bit and clear write protect (WP) bit */
-	if (is_disk) {
+	/* for disks+zbc set DPOFUA bit and clear write protect (WP) bit */
+	if (is_disk || is_zbc) {
 		dev_spec = 0x10;	/* =0x90 if WP=1 implies read-only */
 		if (sdebug_wp)
 			dev_spec |= 0x80;
@@ -2201,7 +2393,7 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
 			bad_pcode = true;
 		break;
 	case 0x8:	/* Caching page, direct access */
-		if (is_disk) {
+		if (is_disk || is_zbc) {
 			len = resp_caching_pg(ap, pcontrol, target);
 			offset += len;
 		} else
@@ -2239,6 +2431,9 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
 						      target);
 				len += resp_caching_pg(ap + len, pcontrol,
 						       target);
+			} else if (is_zbc) {
+				len += resp_caching_pg(ap + len, pcontrol,
+						       target);
 			}
 			len += resp_ctrl_m_pg(ap + len, pcontrol, target);
 			len += resp_sas_sf_m_pg(ap + len, pcontrol, target);
@@ -2266,7 +2461,7 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
 		arr[0] = offset - 1;
 	else
 		put_unaligned_be16((offset - 2), arr + 0);
-	return fill_from_dev_buffer(scp, arr, min(alloc_len, offset));
+	return fill_from_dev_buffer(scp, arr, min_t(int, alloc_len, offset));
 }
 
 #define SDEBUG_MAX_MSELECT_SZ 512
@@ -2451,14 +2646,217 @@ static int resp_log_sense(struct scsi_cmnd *scp,
 		mk_sense_invalid_fld(scp, SDEB_IN_CDB, 3, -1);
 		return check_condition_result;
 	}
-	len = min(get_unaligned_be16(arr + 2) + 4, alloc_len);
+	len = min_t(int, get_unaligned_be16(arr + 2) + 4, alloc_len);
 	return fill_from_dev_buffer(scp, arr,
-		    min(len, SDEBUG_MAX_INQ_ARR_SZ));
+		    min_t(int, len, SDEBUG_MAX_INQ_ARR_SZ));
+}
+
+static inline bool sdebug_dev_is_zoned(struct sdebug_dev_info *devip)
+{
+	return devip->nr_zones != 0;
+}
+
+static struct sdeb_zone_state *zbc_zone(struct sdebug_dev_info *devip,
+					unsigned long long lba)
+{
+	return &devip->zstate[lba >> devip->zsize_shift];
+}
+
+static inline bool zbc_zone_is_conv(struct sdeb_zone_state *zsp)
+{
+	return zsp->z_type == ZBC_ZONE_TYPE_CNV;
+}
+
+static void zbc_close_zone(struct sdebug_dev_info *devip,
+			   struct sdeb_zone_state *zsp)
+{
+	enum sdebug_z_cond zc;
+
+	if (zbc_zone_is_conv(zsp))
+		return;
+
+	zc = zsp->z_cond;
+	if (!(zc == ZC2_IMPLICIT_OPEN || zc == ZC3_EXPLICIT_OPEN))
+		return;
+
+	if (zc == ZC2_IMPLICIT_OPEN)
+		devip->nr_imp_open--;
+	else
+		devip->nr_exp_open--;
+
+	if (zsp->z_wp == zsp->z_start) {
+		zsp->z_cond = ZC1_EMPTY;
+	} else {
+		zsp->z_cond = ZC4_CLOSED;
+		devip->nr_closed++;
+	}
+}
+
+static void zbc_close_imp_open_zone(struct sdebug_dev_info *devip)
+{
+	struct sdeb_zone_state *zsp = &devip->zstate[0];
+	unsigned int i;
+
+	for (i = 0; i < devip->nr_zones; i++, zsp++) {
+		if (zsp->z_cond == ZC2_IMPLICIT_OPEN) {
+			zbc_close_zone(devip, zsp);
+			return;
+		}
+	}
+}
+
+static void zbc_open_zone(struct sdebug_dev_info *devip,
+			  struct sdeb_zone_state *zsp, bool explicit)
+{
+	enum sdebug_z_cond zc;
+
+	if (zbc_zone_is_conv(zsp))
+		return;
+
+	zc = zsp->z_cond;
+	if ((explicit && zc == ZC3_EXPLICIT_OPEN) ||
+	    (!explicit && zc == ZC2_IMPLICIT_OPEN))
+		return;
+
+	/* Close an implicit open zone if necessary */
+	if (explicit && zsp->z_cond == ZC2_IMPLICIT_OPEN)
+		zbc_close_zone(devip, zsp);
+	else if (devip->max_open &&
+		 devip->nr_imp_open + devip->nr_exp_open >= devip->max_open)
+		zbc_close_imp_open_zone(devip);
+
+	if (zsp->z_cond == ZC4_CLOSED)
+		devip->nr_closed--;
+	if (explicit) {
+		zsp->z_cond = ZC3_EXPLICIT_OPEN;
+		devip->nr_exp_open++;
+	} else {
+		zsp->z_cond = ZC2_IMPLICIT_OPEN;
+		devip->nr_imp_open++;
+	}
+}
+
+static void zbc_inc_wp(struct sdebug_dev_info *devip,
+		       unsigned long long lba, unsigned int num)
+{
+	struct sdeb_zone_state *zsp = zbc_zone(devip, lba);
+	unsigned long long n, end, zend = zsp->z_start + zsp->z_size;
+
+	if (zbc_zone_is_conv(zsp))
+		return;
+
+	if (zsp->z_type == ZBC_ZONE_TYPE_SWR) {
+		zsp->z_wp += num;
+		if (zsp->z_wp >= zend)
+			zsp->z_cond = ZC5_FULL;
+		return;
+	}
+
+	while (num) {
+		if (lba != zsp->z_wp)
+			zsp->z_non_seq_resource = true;
+
+		end = lba + num;
+		if (end >= zend) {
+			n = zend - lba;
+			zsp->z_wp = zend;
+		} else if (end > zsp->z_wp) {
+			n = num;
+			zsp->z_wp = end;
+		} else {
+			n = num;
+		}
+		if (zsp->z_wp >= zend)
+			zsp->z_cond = ZC5_FULL;
+
+		num -= n;
+		lba += n;
+		if (num) {
+			zsp++;
+			zend = zsp->z_start + zsp->z_size;
+		}
+	}
 }
 
-static inline int check_device_access_params(struct scsi_cmnd *scp,
-	unsigned long long lba, unsigned int num, bool write)
+static int check_zbc_access_params(struct scsi_cmnd *scp,
+			unsigned long long lba, unsigned int num, bool write)
 {
+	struct scsi_device *sdp = scp->device;
+	struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata;
+	struct sdeb_zone_state *zsp = zbc_zone(devip, lba);
+	struct sdeb_zone_state *zsp_end = zbc_zone(devip, lba + num - 1);
+
+	if (!write) {
+		if (devip->zmodel == BLK_ZONED_HA)
+			return 0;
+		/* For host-managed, reads cannot cross zone types boundaries */
+		if (zsp_end != zsp &&
+		    zbc_zone_is_conv(zsp) &&
+		    !zbc_zone_is_conv(zsp_end)) {
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LBA_OUT_OF_RANGE,
+					READ_INVDATA_ASCQ);
+			return check_condition_result;
+		}
+		return 0;
+	}
+
+	/* No restrictions for writes within conventional zones */
+	if (zbc_zone_is_conv(zsp)) {
+		if (!zbc_zone_is_conv(zsp_end)) {
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LBA_OUT_OF_RANGE,
+					WRITE_BOUNDARY_ASCQ);
+			return check_condition_result;
+		}
+		return 0;
+	}
+
+	if (zsp->z_type == ZBC_ZONE_TYPE_SWR) {
+		/* Writes cannot cross sequential zone boundaries */
+		if (zsp_end != zsp) {
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LBA_OUT_OF_RANGE,
+					WRITE_BOUNDARY_ASCQ);
+			return check_condition_result;
+		}
+		/* Cannot write full zones */
+		if (zsp->z_cond == ZC5_FULL) {
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					INVALID_FIELD_IN_CDB, 0);
+			return check_condition_result;
+		}
+		/* Writes must be aligned to the zone WP */
+		if (lba != zsp->z_wp) {
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LBA_OUT_OF_RANGE,
+					UNALIGNED_WRITE_ASCQ);
+			return check_condition_result;
+		}
+	}
+
+	/* Handle implicit open of closed and empty zones */
+	if (zsp->z_cond == ZC1_EMPTY || zsp->z_cond == ZC4_CLOSED) {
+		if (devip->max_open &&
+		    devip->nr_exp_open >= devip->max_open) {
+			mk_sense_buffer(scp, DATA_PROTECT,
+					INSUFF_RES_ASC,
+					INSUFF_ZONE_ASCQ);
+			return check_condition_result;
+		}
+		zbc_open_zone(devip, zsp, false);
+	}
+
+	return 0;
+}
+
+static inline int check_device_access_params
+			(struct scsi_cmnd *scp, unsigned long long lba,
+			 unsigned int num, bool write)
+{
+	struct scsi_device *sdp = scp->device;
+	struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata;
+
 	if (lba + num > sdebug_capacity) {
 		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
 		return check_condition_result;
@@ -2473,17 +2871,37 @@ static inline int check_device_access_params(struct scsi_cmnd *scp,
 		mk_sense_buffer(scp, DATA_PROTECT, WRITE_PROTECTED, 0x2);
 		return check_condition_result;
 	}
+	if (sdebug_dev_is_zoned(devip))
+		return check_zbc_access_params(scp, lba, num, write);
+
 	return 0;
 }
 
+/*
+ * Note: if BUG_ON() fires it usually indicates a problem with the parser
+ * tables. Perhaps a missing F_FAKE_RW or FF_MEDIA_IO flag. Response functions
+ * that access any of the "stores" in struct sdeb_store_info should call this
+ * function with bug_if_fake_rw set to true.
+ */
+static inline struct sdeb_store_info *devip2sip(struct sdebug_dev_info *devip,
+						bool bug_if_fake_rw)
+{
+	if (sdebug_fake_rw) {
+		BUG_ON(bug_if_fake_rw);	/* See note above */
+		return NULL;
+	}
+	return xa_load(per_store_ap, devip->sdbg_host->si_idx);
+}
+
 /* Returns number of bytes copied or -1 if error. */
-static int do_device_access(struct scsi_cmnd *scmd, u32 sg_skip, u64 lba,
-			    u32 num, bool do_write)
+static int do_device_access(struct sdeb_store_info *sip, struct scsi_cmnd *scp,
+			    u32 sg_skip, u64 lba, u32 num, bool do_write)
 {
 	int ret;
 	u64 block, rest = 0;
-	struct scsi_data_buffer *sdb = &scmd->sdb;
 	enum dma_data_direction dir;
+	struct scsi_data_buffer *sdb = &scp->sdb;
+	u8 *fsp;
 
 	if (do_write) {
 		dir = DMA_TO_DEVICE;
@@ -2492,24 +2910,25 @@ static int do_device_access(struct scsi_cmnd *scmd, u32 sg_skip, u64 lba,
 		dir = DMA_FROM_DEVICE;
 	}
 
-	if (!sdb->length)
+	if (!sdb->length || !sip)
 		return 0;
-	if (scmd->sc_data_direction != dir)
+	if (scp->sc_data_direction != dir)
 		return -1;
+	fsp = sip->storep;
 
 	block = do_div(lba, sdebug_store_sectors);
 	if (block + num > sdebug_store_sectors)
 		rest = block + num - sdebug_store_sectors;
 
 	ret = sg_copy_buffer(sdb->table.sgl, sdb->table.nents,
-		   fake_storep + (block * sdebug_sector_size),
+		   fsp + (block * sdebug_sector_size),
 		   (num - rest) * sdebug_sector_size, sg_skip, do_write);
 	if (ret != (num - rest) * sdebug_sector_size)
 		return ret;
 
 	if (rest) {
 		ret += sg_copy_buffer(sdb->table.sgl, sdb->table.nents,
-			    fake_storep, rest * sdebug_sector_size,
+			    fsp, rest * sdebug_sector_size,
 			    sg_skip + ((num - rest) * sdebug_sector_size),
 			    do_write);
 	}
@@ -2517,34 +2936,49 @@ static int do_device_access(struct scsi_cmnd *scmd, u32 sg_skip, u64 lba,
 	return ret;
 }
 
-/* If lba2fake_store(lba,num) compares equal to arr(num), then copy top half of
- * arr into lba2fake_store(lba,num) and return true. If comparison fails then
+/* Returns number of bytes copied or -1 if error. */
+static int do_dout_fetch(struct scsi_cmnd *scp, u32 num, u8 *doutp)
+{
+	struct scsi_data_buffer *sdb = &scp->sdb;
+
+	if (!sdb->length)
+		return 0;
+	if (scp->sc_data_direction != DMA_TO_DEVICE)
+		return -1;
+	return sg_copy_buffer(sdb->table.sgl, sdb->table.nents, doutp,
+			      num * sdebug_sector_size, 0, true);
+}
+
+/* If sip->storep+lba compares equal to arr(num), then copy top half of
+ * arr into sip->storep+lba and return true. If comparison fails then
  * return false. */
-static bool comp_write_worker(u64 lba, u32 num, const u8 *arr)
+static bool comp_write_worker(struct sdeb_store_info *sip, u64 lba, u32 num,
+			      const u8 *arr, bool compare_only)
 {
 	bool res;
 	u64 block, rest = 0;
 	u32 store_blks = sdebug_store_sectors;
 	u32 lb_size = sdebug_sector_size;
+	u8 *fsp = sip->storep;
 
 	block = do_div(lba, store_blks);
 	if (block + num > store_blks)
 		rest = block + num - store_blks;
 
-	res = !memcmp(fake_storep + (block * lb_size), arr,
-		      (num - rest) * lb_size);
+	res = !memcmp(fsp + (block * lb_size), arr, (num - rest) * lb_size);
 	if (!res)
 		return res;
 	if (rest)
-		res = memcmp(fake_storep, arr + ((num - rest) * lb_size),
+		res = memcmp(fsp, arr + ((num - rest) * lb_size),
 			     rest * lb_size);
 	if (!res)
 		return res;
+	if (compare_only)
+		return true;
 	arr += num * lb_size;
-	memcpy(fake_storep + (block * lb_size), arr, (num - rest) * lb_size);
+	memcpy(fsp + (block * lb_size), arr, (num - rest) * lb_size);
 	if (rest)
-		memcpy(fake_storep, arr + ((num - rest) * lb_size),
-		       rest * lb_size);
+		memcpy(fsp, arr + ((num - rest) * lb_size), rest * lb_size);
 	return res;
 }
 
@@ -2587,24 +3021,27 @@ static int dif_verify(struct t10_pi_tuple *sdt, const void *data,
 	return 0;
 }
 
-static void dif_copy_prot(struct scsi_cmnd *SCpnt, sector_t sector,
+static void dif_copy_prot(struct scsi_cmnd *scp, sector_t sector,
 			  unsigned int sectors, bool read)
 {
 	size_t resid;
 	void *paddr;
+	struct sdeb_store_info *sip = devip2sip((struct sdebug_dev_info *)
+						scp->device->hostdata, true);
+	struct t10_pi_tuple *dif_storep = sip->dif_storep;
 	const void *dif_store_end = dif_storep + sdebug_store_sectors;
 	struct sg_mapping_iter miter;
 
 	/* Bytes of protection data to copy into sgl */
 	resid = sectors * sizeof(*dif_storep);
 
-	sg_miter_start(&miter, scsi_prot_sglist(SCpnt),
-			scsi_prot_sg_count(SCpnt), SG_MITER_ATOMIC |
-			(read ? SG_MITER_TO_SG : SG_MITER_FROM_SG));
+	sg_miter_start(&miter, scsi_prot_sglist(scp),
+		       scsi_prot_sg_count(scp), SG_MITER_ATOMIC |
+		       (read ? SG_MITER_TO_SG : SG_MITER_FROM_SG));
 
 	while (sg_miter_next(&miter) && resid > 0) {
-		size_t len = min(miter.length, resid);
-		void *start = dif_store(sector);
+		size_t len = min_t(size_t, miter.length, resid);
+		void *start = dif_store(sip, sector);
 		size_t rest = 0;
 
 		if (dif_store_end < start + len)
@@ -2630,30 +3067,33 @@ static void dif_copy_prot(struct scsi_cmnd *SCpnt, sector_t sector,
 	sg_miter_stop(&miter);
 }
 
-static int prot_verify_read(struct scsi_cmnd *SCpnt, sector_t start_sec,
+static int prot_verify_read(struct scsi_cmnd *scp, sector_t start_sec,
 			    unsigned int sectors, u32 ei_lba)
 {
 	unsigned int i;
-	struct t10_pi_tuple *sdt;
 	sector_t sector;
+	struct sdeb_store_info *sip = devip2sip((struct sdebug_dev_info *)
+						scp->device->hostdata, true);
+	struct t10_pi_tuple *sdt;
 
 	for (i = 0; i < sectors; i++, ei_lba++) {
 		int ret;
 
 		sector = start_sec + i;
-		sdt = dif_store(sector);
+		sdt = dif_store(sip, sector);
 
 		if (sdt->app_tag == cpu_to_be16(0xffff))
 			continue;
 
-		ret = dif_verify(sdt, lba2fake_store(sector), sector, ei_lba);
+		ret = dif_verify(sdt, lba2fake_store(sip, sector), sector,
+				 ei_lba);
 		if (ret) {
 			dif_errors++;
 			return ret;
 		}
 	}
 
-	dif_copy_prot(SCpnt, start_sec, sectors, true);
+	dif_copy_prot(scp, start_sec, sectors, true);
 	dix_reads++;
 
 	return 0;
@@ -2661,14 +3101,15 @@ static int prot_verify_read(struct scsi_cmnd *SCpnt, sector_t start_sec,
 
 static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 {
-	u8 *cmd = scp->cmnd;
-	struct sdebug_queued_cmd *sqcp;
-	u64 lba;
+	bool check_prot;
 	u32 num;
 	u32 ei_lba;
-	unsigned long iflags;
 	int ret;
-	bool check_prot;
+	u64 lba;
+	struct sdeb_store_info *sip = devip2sip(devip, true);
+	rwlock_t *macc_lckp = sip ? &sip->macc_lck : &sdeb_fake_rw_lck;
+	u8 *cmd = scp->cmnd;
+	struct sdebug_queued_cmd *sqcp;
 
 	switch (cmd[0]) {
 	case READ_16:
@@ -2750,21 +3191,21 @@ static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		return check_condition_result;
 	}
 
-	read_lock_irqsave(&atomic_rw, iflags);
+	read_lock(macc_lckp);
 
 	/* DIX + T10 DIF */
 	if (unlikely(sdebug_dix && scsi_prot_sg_count(scp))) {
 		int prot_ret = prot_verify_read(scp, lba, num, ei_lba);
 
 		if (prot_ret) {
-			read_unlock_irqrestore(&atomic_rw, iflags);
+			read_unlock(macc_lckp);
 			mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, prot_ret);
 			return illegal_condition_result;
 		}
 	}
 
-	ret = do_device_access(scp, 0, lba, num, false);
-	read_unlock_irqrestore(&atomic_rw, iflags);
+	ret = do_device_access(sip, scp, 0, lba, num, false);
+	read_unlock(macc_lckp);
 	if (unlikely(ret == -1))
 		return DID_ERROR << 16;
 
@@ -2902,7 +3343,8 @@ static sector_t map_index_to_lba(unsigned long index)
 	return lba;
 }
 
-static unsigned int map_state(sector_t lba, unsigned int *num)
+static unsigned int map_state(struct sdeb_store_info *sip, sector_t lba,
+			      unsigned int *num)
 {
 	sector_t end;
 	unsigned int mapped;
@@ -2910,19 +3352,20 @@ static unsigned int map_state(sector_t lba, unsigned int *num)
 	unsigned long next;
 
 	index = lba_to_map_index(lba);
-	mapped = test_bit(index, map_storep);
+	mapped = test_bit(index, sip->map_storep);
 
 	if (mapped)
-		next = find_next_zero_bit(map_storep, map_size, index);
+		next = find_next_zero_bit(sip->map_storep, map_size, index);
 	else
-		next = find_next_bit(map_storep, map_size, index);
+		next = find_next_bit(sip->map_storep, map_size, index);
 
 	end = min_t(sector_t, sdebug_store_sectors,  map_index_to_lba(next));
 	*num = end - lba;
 	return mapped;
 }
 
-static void map_region(sector_t lba, unsigned int len)
+static void map_region(struct sdeb_store_info *sip, sector_t lba,
+		       unsigned int len)
 {
 	sector_t end = lba + len;
 
@@ -2930,15 +3373,17 @@ static void map_region(sector_t lba, unsigned int len)
 		unsigned long index = lba_to_map_index(lba);
 
 		if (index < map_size)
-			set_bit(index, map_storep);
+			set_bit(index, sip->map_storep);
 
 		lba = map_index_to_lba(index + 1);
 	}
 }
 
-static void unmap_region(sector_t lba, unsigned int len)
+static void unmap_region(struct sdeb_store_info *sip, sector_t lba,
+			 unsigned int len)
 {
 	sector_t end = lba + len;
+	u8 *fsp = sip->storep;
 
 	while (lba < end) {
 		unsigned long index = lba_to_map_index(lba);
@@ -2946,17 +3391,16 @@ static void unmap_region(sector_t lba, unsigned int len)
 		if (lba == map_index_to_lba(index) &&
 		    lba + sdebug_unmap_granularity <= end &&
 		    index < map_size) {
-			clear_bit(index, map_storep);
+			clear_bit(index, sip->map_storep);
 			if (sdebug_lbprz) {  /* for LBPRZ=2 return 0xff_s */
-				memset(fake_storep +
-				       lba * sdebug_sector_size,
+				memset(fsp + lba * sdebug_sector_size,
 				       (sdebug_lbprz & 1) ? 0 : 0xff,
 				       sdebug_sector_size *
 				       sdebug_unmap_granularity);
 			}
-			if (dif_storep) {
-				memset(dif_storep + lba, 0xff,
-				       sizeof(*dif_storep) *
+			if (sip->dif_storep) {
+				memset(sip->dif_storep + lba, 0xff,
+				       sizeof(*sip->dif_storep) *
 				       sdebug_unmap_granularity);
 			}
 		}
@@ -2966,13 +3410,14 @@ static void unmap_region(sector_t lba, unsigned int len)
 
 static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 {
-	u8 *cmd = scp->cmnd;
-	u64 lba;
+	bool check_prot;
 	u32 num;
 	u32 ei_lba;
-	unsigned long iflags;
 	int ret;
-	bool check_prot;
+	u64 lba;
+	struct sdeb_store_info *sip = devip2sip(devip, true);
+	rwlock_t *macc_lckp = &sip->macc_lck;
+	u8 *cmd = scp->cmnd;
 
 	switch (cmd[0]) {
 	case WRITE_16:
@@ -3025,26 +3470,32 @@ static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 			sdev_printk(KERN_ERR, scp->device, "Unprotected WR "
 				    "to DIF device\n");
 	}
+
+	write_lock(macc_lckp);
 	ret = check_device_access_params(scp, lba, num, true);
-	if (ret)
+	if (ret) {
+		write_unlock(macc_lckp);
 		return ret;
-	write_lock_irqsave(&atomic_rw, iflags);
+	}
 
 	/* DIX + T10 DIF */
 	if (unlikely(sdebug_dix && scsi_prot_sg_count(scp))) {
 		int prot_ret = prot_verify_write(scp, lba, num, ei_lba);
 
 		if (prot_ret) {
-			write_unlock_irqrestore(&atomic_rw, iflags);
+			write_unlock(macc_lckp);
 			mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, prot_ret);
 			return illegal_condition_result;
 		}
 	}
 
-	ret = do_device_access(scp, 0, lba, num, true);
+	ret = do_device_access(sip, scp, 0, lba, num, true);
 	if (unlikely(scsi_debug_lbp()))
-		map_region(lba, num);
-	write_unlock_irqrestore(&atomic_rw, iflags);
+		map_region(sip, lba, num);
+	/* If ZBC zone then bump its write pointer */
+	if (sdebug_dev_is_zoned(devip))
+		zbc_inc_wp(devip, lba, num);
+	write_unlock(macc_lckp);
 	if (unlikely(-1 == ret))
 		return DID_ERROR << 16;
 	else if (unlikely(sdebug_verbose &&
@@ -3085,13 +3536,14 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 	u8 *cmd = scp->cmnd;
 	u8 *lrdp = NULL;
 	u8 *up;
+	struct sdeb_store_info *sip = devip2sip(devip, true);
+	rwlock_t *macc_lckp = &sip->macc_lck;
 	u8 wrprotect;
 	u16 lbdof, num_lrd, k;
 	u32 num, num_by, bt_len, lbdof_blen, sg_off, cum_lb;
 	u32 lb_size = sdebug_sector_size;
 	u32 ei_lba;
 	u64 lba;
-	unsigned long iflags;
 	int ret, res;
 	bool is_16;
 	static const u32 lrd_size = 32; /* + parameter list header size */
@@ -3153,7 +3605,7 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 		goto err_out;
 	}
 
-	write_lock_irqsave(&atomic_rw, iflags);
+	write_lock(macc_lckp);
 	sg_off = lbdof_blen;
 	/* Spec says Buffer xfer Length field in number of LBs in dout */
 	cum_lb = 0;
@@ -3196,9 +3648,12 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 			}
 		}
 
-		ret = do_device_access(scp, sg_off, lba, num, true);
+		ret = do_device_access(sip, scp, sg_off, lba, num, true);
+		/* If ZBC zone then bump its write pointer */
+		if (sdebug_dev_is_zoned(devip))
+			zbc_inc_wp(devip, lba, num);
 		if (unlikely(scsi_debug_lbp()))
-			map_region(lba, num);
+			map_region(sip, lba, num);
 		if (unlikely(-1 == ret)) {
 			ret = DID_ERROR << 16;
 			goto err_out_unlock;
@@ -3236,7 +3691,7 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 	}
 	ret = 0;
 err_out_unlock:
-	write_unlock_irqrestore(&atomic_rw, iflags);
+	write_unlock(macc_lckp);
 err_out:
 	kfree(lrdp);
 	return ret;
@@ -3245,27 +3700,35 @@ err_out:
 static int resp_write_same(struct scsi_cmnd *scp, u64 lba, u32 num,
 			   u32 ei_lba, bool unmap, bool ndob)
 {
-	int ret;
-	unsigned long iflags;
+	struct scsi_device *sdp = scp->device;
+	struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata;
 	unsigned long long i;
-	u32 lb_size = sdebug_sector_size;
 	u64 block, lbaa;
+	u32 lb_size = sdebug_sector_size;
+	int ret;
+	struct sdeb_store_info *sip = devip2sip((struct sdebug_dev_info *)
+						scp->device->hostdata, true);
+	rwlock_t *macc_lckp = &sip->macc_lck;
 	u8 *fs1p;
+	u8 *fsp;
+
+	write_lock(macc_lckp);
 
 	ret = check_device_access_params(scp, lba, num, true);
-	if (ret)
+	if (ret) {
+		write_unlock(macc_lckp);
 		return ret;
-
-	write_lock_irqsave(&atomic_rw, iflags);
+	}
 
 	if (unmap && scsi_debug_lbp()) {
-		unmap_region(lba, num);
+		unmap_region(sip, lba, num);
 		goto out;
 	}
 	lbaa = lba;
 	block = do_div(lbaa, sdebug_store_sectors);
 	/* if ndob then zero 1 logical block, else fetch 1 logical block */
-	fs1p = fake_storep + (block * lb_size);
+	fsp = sip->storep;
+	fs1p = fsp + (block * lb_size);
 	if (ndob) {
 		memset(fs1p, 0, lb_size);
 		ret = 0;
@@ -3273,7 +3736,7 @@ static int resp_write_same(struct scsi_cmnd *scp, u64 lba, u32 num,
 		ret = fetch_to_dev_buffer(scp, fs1p, lb_size);
 
 	if (-1 == ret) {
-		write_unlock_irqrestore(&atomic_rw, iflags);
+		write_unlock(&sip->macc_lck);
 		return DID_ERROR << 16;
 	} else if (sdebug_verbose && !ndob && (ret < lb_size))
 		sdev_printk(KERN_INFO, scp->device,
@@ -3284,12 +3747,15 @@ static int resp_write_same(struct scsi_cmnd *scp, u64 lba, u32 num,
 	for (i = 1 ; i < num ; i++) {
 		lbaa = lba + i;
 		block = do_div(lbaa, sdebug_store_sectors);
-		memmove(fake_storep + (block * lb_size), fs1p, lb_size);
+		memmove(fsp + (block * lb_size), fs1p, lb_size);
 	}
 	if (scsi_debug_lbp())
-		map_region(lba, num);
+		map_region(sip, lba, num);
+	/* If ZBC zone then bump its write pointer */
+	if (sdebug_dev_is_zoned(devip))
+		zbc_inc_wp(devip, lba, num);
 out:
-	write_unlock_irqrestore(&atomic_rw, iflags);
+	write_unlock(macc_lckp);
 
 	return 0;
 }
@@ -3401,12 +3867,12 @@ static int resp_comp_write(struct scsi_cmnd *scp,
 {
 	u8 *cmd = scp->cmnd;
 	u8 *arr;
-	u8 *fake_storep_hold;
+	struct sdeb_store_info *sip = devip2sip(devip, true);
+	rwlock_t *macc_lckp = &sip->macc_lck;
 	u64 lba;
 	u32 dnum;
 	u32 lb_size = sdebug_sector_size;
 	u8 num;
-	unsigned long iflags;
 	int ret;
 	int retval = 0;
 
@@ -3435,14 +3901,9 @@ static int resp_comp_write(struct scsi_cmnd *scp,
 		return check_condition_result;
 	}
 
-	write_lock_irqsave(&atomic_rw, iflags);
+	write_lock(macc_lckp);
 
-	/* trick do_device_access() to fetch both compare and write buffers
-	 * from data-in into arr. Safe (atomic) since write_lock held. */
-	fake_storep_hold = fake_storep;
-	fake_storep = arr;
-	ret = do_device_access(scp, 0, 0, dnum, true);
-	fake_storep = fake_storep_hold;
+	ret = do_dout_fetch(scp, dnum, arr);
 	if (ret == -1) {
 		retval = DID_ERROR << 16;
 		goto cleanup;
@@ -3450,15 +3911,15 @@ static int resp_comp_write(struct scsi_cmnd *scp,
 		sdev_printk(KERN_INFO, scp->device, "%s: compare_write: cdb "
 			    "indicated=%u, IO sent=%d bytes\n", my_name,
 			    dnum * lb_size, ret);
-	if (!comp_write_worker(lba, num, arr)) {
+	if (!comp_write_worker(sip, lba, num, arr, false)) {
 		mk_sense_buffer(scp, MISCOMPARE, MISCOMPARE_VERIFY_ASC, 0);
 		retval = check_condition_result;
 		goto cleanup;
 	}
 	if (scsi_debug_lbp())
-		map_region(lba, num);
+		map_region(sip, lba, num);
 cleanup:
-	write_unlock_irqrestore(&atomic_rw, iflags);
+	write_unlock(macc_lckp);
 	kfree(arr);
 	return retval;
 }
@@ -3473,10 +3934,10 @@ static int resp_unmap(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 {
 	unsigned char *buf;
 	struct unmap_block_desc *desc;
+	struct sdeb_store_info *sip = devip2sip(devip, true);
+	rwlock_t *macc_lckp = &sip->macc_lck;
 	unsigned int i, payload_len, descriptors;
 	int ret;
-	unsigned long iflags;
-
 
 	if (!scsi_debug_lbp())
 		return 0;	/* fib and say its done */
@@ -3503,7 +3964,7 @@ static int resp_unmap(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 
 	desc = (void *)&buf[8];
 
-	write_lock_irqsave(&atomic_rw, iflags);
+	write_lock(macc_lckp);
 
 	for (i = 0 ; i < descriptors ; i++) {
 		unsigned long long lba = get_unaligned_be64(&desc[i].lba);
@@ -3513,13 +3974,13 @@ static int resp_unmap(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		if (ret)
 			goto out;
 
-		unmap_region(lba, num);
+		unmap_region(sip, lba, num);
 	}
 
 	ret = 0;
 
 out:
-	write_unlock_irqrestore(&atomic_rw, iflags);
+	write_unlock(macc_lckp);
 	kfree(buf);
 
 	return ret;
@@ -3533,8 +3994,8 @@ static int resp_get_lba_status(struct scsi_cmnd *scp,
 	u8 *cmd = scp->cmnd;
 	u64 lba;
 	u32 alloc_len, mapped, num;
-	u8 arr[SDEBUG_GET_LBA_STATUS_LEN];
 	int ret;
+	u8 arr[SDEBUG_GET_LBA_STATUS_LEN];
 
 	lba = get_unaligned_be64(cmd + 2);
 	alloc_len = get_unaligned_be32(cmd + 10);
@@ -3546,9 +4007,11 @@ static int resp_get_lba_status(struct scsi_cmnd *scp,
 	if (ret)
 		return ret;
 
-	if (scsi_debug_lbp())
-		mapped = map_state(lba, &num);
-	else {
+	if (scsi_debug_lbp()) {
+		struct sdeb_store_info *sip = devip2sip(devip, true);
+
+		mapped = map_state(sip, lba, &num);
+	} else {
 		mapped = 1;
 		/* following just in case virtual_gb changed */
 		sdebug_capacity = get_sdebug_capacity();
@@ -3593,6 +4056,56 @@ static int resp_sync_cache(struct scsi_cmnd *scp,
 	return res;
 }
 
+/*
+ * Assuming the LBA+num_blocks is not out-of-range, this function will return
+ * CONDITION MET if the specified blocks will/have fitted in the cache, and
+ * a GOOD status otherwise. Model a disk with a big cache and yield
+ * CONDITION MET. Actually tries to bring range in main memory into the
+ * cache associated with the CPU(s).
+ */
+static int resp_pre_fetch(struct scsi_cmnd *scp,
+			  struct sdebug_dev_info *devip)
+{
+	int res = 0;
+	u64 lba;
+	u64 block, rest = 0;
+	u32 nblks;
+	u8 *cmd = scp->cmnd;
+	struct sdeb_store_info *sip = devip2sip(devip, true);
+	rwlock_t *macc_lckp = &sip->macc_lck;
+	u8 *fsp = sip->storep;
+
+	if (cmd[0] == PRE_FETCH) {	/* 10 byte cdb */
+		lba = get_unaligned_be32(cmd + 2);
+		nblks = get_unaligned_be16(cmd + 7);
+	} else {			/* PRE-FETCH(16) */
+		lba = get_unaligned_be64(cmd + 2);
+		nblks = get_unaligned_be32(cmd + 10);
+	}
+	if (lba + nblks > sdebug_capacity) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		return check_condition_result;
+	}
+	if (!fsp)
+		goto fini;
+	/* PRE-FETCH spec says nothing about LBP or PI so skip them */
+	block = do_div(lba, sdebug_store_sectors);
+	if (block + nblks > sdebug_store_sectors)
+		rest = block + nblks - sdebug_store_sectors;
+
+	/* Try to bring the PRE-FETCH range into CPU's cache */
+	read_lock(macc_lckp);
+	prefetch_range(fsp + (sdebug_sector_size * block),
+		       (nblks - rest) * sdebug_sector_size);
+	if (rest)
+		prefetch_range(fsp, rest * sdebug_sector_size);
+	read_unlock(macc_lckp);
+fini:
+	if (cmd[1] & 0x2)
+		res = SDEG_RES_IMMED_MASK;
+	return res | condition_met_result;
+}
+
 #define RL_BUCKET_ELEMS 8
 
 /* Even though each pseudo target has a REPORT LUNS "well known logical unit"
@@ -3694,6 +4207,504 @@ static int resp_report_luns(struct scsi_cmnd *scp,
 	return res;
 }
 
+static int resp_verify(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+{
+	bool is_bytchk3 = false;
+	u8 bytchk;
+	int ret, j;
+	u32 vnum, a_num, off;
+	const u32 lb_size = sdebug_sector_size;
+	u64 lba;
+	u8 *arr;
+	u8 *cmd = scp->cmnd;
+	struct sdeb_store_info *sip = devip2sip(devip, true);
+	rwlock_t *macc_lckp = &sip->macc_lck;
+
+	bytchk = (cmd[1] >> 1) & 0x3;
+	if (bytchk == 0) {
+		return 0;	/* always claim internal verify okay */
+	} else if (bytchk == 2) {
+		mk_sense_invalid_fld(scp, SDEB_IN_CDB, 2, 2);
+		return check_condition_result;
+	} else if (bytchk == 3) {
+		is_bytchk3 = true;	/* 1 block sent, compared repeatedly */
+	}
+	switch (cmd[0]) {
+	case VERIFY_16:
+		lba = get_unaligned_be64(cmd + 2);
+		vnum = get_unaligned_be32(cmd + 10);
+		break;
+	case VERIFY:		/* is VERIFY(10) */
+		lba = get_unaligned_be32(cmd + 2);
+		vnum = get_unaligned_be16(cmd + 7);
+		break;
+	default:
+		mk_sense_invalid_opcode(scp);
+		return check_condition_result;
+	}
+	a_num = is_bytchk3 ? 1 : vnum;
+	/* Treat following check like one for read (i.e. no write) access */
+	ret = check_device_access_params(scp, lba, a_num, false);
+	if (ret)
+		return ret;
+
+	arr = kcalloc(lb_size, vnum, GFP_ATOMIC);
+	if (!arr) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
+				INSUFF_RES_ASCQ);
+		return check_condition_result;
+	}
+	/* Not changing store, so only need read access */
+	read_lock(macc_lckp);
+
+	ret = do_dout_fetch(scp, a_num, arr);
+	if (ret == -1) {
+		ret = DID_ERROR << 16;
+		goto cleanup;
+	} else if (sdebug_verbose && (ret < (a_num * lb_size))) {
+		sdev_printk(KERN_INFO, scp->device,
+			    "%s: %s: cdb indicated=%u, IO sent=%d bytes\n",
+			    my_name, __func__, a_num * lb_size, ret);
+	}
+	if (is_bytchk3) {
+		for (j = 1, off = lb_size; j < vnum; ++j, off += lb_size)
+			memcpy(arr + off, arr, lb_size);
+	}
+	ret = 0;
+	if (!comp_write_worker(sip, lba, vnum, arr, true)) {
+		mk_sense_buffer(scp, MISCOMPARE, MISCOMPARE_VERIFY_ASC, 0);
+		ret = check_condition_result;
+		goto cleanup;
+	}
+cleanup:
+	read_unlock(macc_lckp);
+	kfree(arr);
+	return ret;
+}
+
+#define RZONES_DESC_HD 64
+
+/* Report zones depending on start LBA nad reporting options */
+static int resp_report_zones(struct scsi_cmnd *scp,
+			     struct sdebug_dev_info *devip)
+{
+	unsigned int i, max_zones, rep_max_zones, nrz = 0;
+	int ret = 0;
+	u32 alloc_len, rep_opts, rep_len;
+	bool partial;
+	u64 lba, zs_lba;
+	u8 *arr = NULL, *desc;
+	u8 *cmd = scp->cmnd;
+	struct sdeb_zone_state *zsp;
+	struct sdeb_store_info *sip = devip2sip(devip, false);
+	rwlock_t *macc_lckp = sip ? &sip->macc_lck : &sdeb_fake_rw_lck;
+
+	if (!sdebug_dev_is_zoned(devip)) {
+		mk_sense_invalid_opcode(scp);
+		return check_condition_result;
+	}
+	zs_lba = get_unaligned_be64(cmd + 2);
+	alloc_len = get_unaligned_be32(cmd + 10);
+	rep_opts = cmd[14] & 0x3f;
+	partial = cmd[14] & 0x80;
+
+	if (zs_lba >= sdebug_capacity) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		return check_condition_result;
+	}
+
+	max_zones = devip->nr_zones - (zs_lba >> devip->zsize_shift);
+	rep_max_zones = min((alloc_len - 64) >> ilog2(RZONES_DESC_HD),
+			    max_zones);
+
+	arr = kcalloc(RZONES_DESC_HD, alloc_len, GFP_ATOMIC);
+	if (!arr) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
+				INSUFF_RES_ASCQ);
+		return check_condition_result;
+	}
+
+	read_lock(macc_lckp);
+
+	desc = arr + 64;
+	for (i = 0; i < max_zones; i++) {
+		lba = zs_lba + devip->zsize * i;
+		if (lba > sdebug_capacity)
+			break;
+		zsp = zbc_zone(devip, lba);
+		switch (rep_opts) {
+		case 0x00:
+			/* All zones */
+			break;
+		case 0x01:
+			/* Empty zones */
+			if (zsp->z_cond != ZC1_EMPTY)
+				continue;
+			break;
+		case 0x02:
+			/* Implicit open zones */
+			if (zsp->z_cond != ZC2_IMPLICIT_OPEN)
+				continue;
+			break;
+		case 0x03:
+			/* Explicit open zones */
+			if (zsp->z_cond != ZC3_EXPLICIT_OPEN)
+				continue;
+			break;
+		case 0x04:
+			/* Closed zones */
+			if (zsp->z_cond != ZC4_CLOSED)
+				continue;
+			break;
+		case 0x05:
+			/* Full zones */
+			if (zsp->z_cond != ZC5_FULL)
+				continue;
+			break;
+		case 0x06:
+		case 0x07:
+		case 0x10:
+			/*
+			 * Read-only, offline, reset WP recommended are
+			 * not emulated: no zones to report;
+			 */
+			continue;
+		case 0x11:
+			/* non-seq-resource set */
+			if (!zsp->z_non_seq_resource)
+				continue;
+			break;
+		case 0x3f:
+			/* Not write pointer (conventional) zones */
+			if (!zbc_zone_is_conv(zsp))
+				continue;
+			break;
+		default:
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					INVALID_FIELD_IN_CDB, 0);
+			ret = check_condition_result;
+			goto fini;
+		}
+
+		if (nrz < rep_max_zones) {
+			/* Fill zone descriptor */
+			desc[0] = zsp->z_type;
+			desc[1] = zsp->z_cond << 4;
+			if (zsp->z_non_seq_resource)
+				desc[1] |= 1 << 1;
+			put_unaligned_be64((u64)zsp->z_size, desc + 8);
+			put_unaligned_be64((u64)zsp->z_start, desc + 16);
+			put_unaligned_be64((u64)zsp->z_wp, desc + 24);
+			desc += 64;
+		}
+
+		if (partial && nrz >= rep_max_zones)
+			break;
+
+		nrz++;
+	}
+
+	/* Report header */
+	put_unaligned_be32(nrz * RZONES_DESC_HD, arr + 0);
+	put_unaligned_be64(sdebug_capacity - 1, arr + 8);
+
+	rep_len = (unsigned long)desc - (unsigned long)arr;
+	ret = fill_from_dev_buffer(scp, arr, min_t(int, alloc_len, rep_len));
+
+fini:
+	read_unlock(macc_lckp);
+	kfree(arr);
+	return ret;
+}
+
+/* Logic transplanted from tcmu-runner, file_zbc.c */
+static void zbc_open_all(struct sdebug_dev_info *devip)
+{
+	struct sdeb_zone_state *zsp = &devip->zstate[0];
+	unsigned int i;
+
+	for (i = 0; i < devip->nr_zones; i++, zsp++) {
+		if (zsp->z_cond == ZC4_CLOSED)
+			zbc_open_zone(devip, &devip->zstate[i], true);
+	}
+}
+
+static int resp_open_zone(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+{
+	int res = 0;
+	u64 z_id;
+	enum sdebug_z_cond zc;
+	u8 *cmd = scp->cmnd;
+	struct sdeb_zone_state *zsp;
+	bool all = cmd[14] & 0x01;
+	struct sdeb_store_info *sip = devip2sip(devip, false);
+	rwlock_t *macc_lckp = sip ? &sip->macc_lck : &sdeb_fake_rw_lck;
+
+	if (!sdebug_dev_is_zoned(devip)) {
+		mk_sense_invalid_opcode(scp);
+		return check_condition_result;
+	}
+
+	write_lock(macc_lckp);
+
+	if (all) {
+		/* Check if all closed zones can be open */
+		if (devip->max_open &&
+		    devip->nr_exp_open + devip->nr_closed > devip->max_open) {
+			mk_sense_buffer(scp, DATA_PROTECT, INSUFF_RES_ASC,
+					INSUFF_ZONE_ASCQ);
+			res = check_condition_result;
+			goto fini;
+		}
+		/* Open all closed zones */
+		zbc_open_all(devip);
+		goto fini;
+	}
+
+	/* Open the specified zone */
+	z_id = get_unaligned_be64(cmd + 2);
+	if (z_id >= sdebug_capacity) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zsp = zbc_zone(devip, z_id);
+	if (z_id != zsp->z_start) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+	if (zbc_zone_is_conv(zsp)) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zc = zsp->z_cond;
+	if (zc == ZC3_EXPLICIT_OPEN || zc == ZC5_FULL)
+		goto fini;
+
+	if (devip->max_open && devip->nr_exp_open >= devip->max_open) {
+		mk_sense_buffer(scp, DATA_PROTECT, INSUFF_RES_ASC,
+				INSUFF_ZONE_ASCQ);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	if (zc == ZC2_IMPLICIT_OPEN)
+		zbc_close_zone(devip, zsp);
+	zbc_open_zone(devip, zsp, true);
+fini:
+	write_unlock(macc_lckp);
+	return res;
+}
+
+static void zbc_close_all(struct sdebug_dev_info *devip)
+{
+	unsigned int i;
+
+	for (i = 0; i < devip->nr_zones; i++)
+		zbc_close_zone(devip, &devip->zstate[i]);
+}
+
+static int resp_close_zone(struct scsi_cmnd *scp,
+			   struct sdebug_dev_info *devip)
+{
+	int res = 0;
+	u64 z_id;
+	u8 *cmd = scp->cmnd;
+	struct sdeb_zone_state *zsp;
+	bool all = cmd[14] & 0x01;
+	struct sdeb_store_info *sip = devip2sip(devip, false);
+	rwlock_t *macc_lckp = sip ? &sip->macc_lck : &sdeb_fake_rw_lck;
+
+	if (!sdebug_dev_is_zoned(devip)) {
+		mk_sense_invalid_opcode(scp);
+		return check_condition_result;
+	}
+
+	write_lock(macc_lckp);
+
+	if (all) {
+		zbc_close_all(devip);
+		goto fini;
+	}
+
+	/* Close specified zone */
+	z_id = get_unaligned_be64(cmd + 2);
+	if (z_id >= sdebug_capacity) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zsp = zbc_zone(devip, z_id);
+	if (z_id != zsp->z_start) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+	if (zbc_zone_is_conv(zsp)) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zbc_close_zone(devip, zsp);
+fini:
+	write_unlock(macc_lckp);
+	return res;
+}
+
+static void zbc_finish_zone(struct sdebug_dev_info *devip,
+			    struct sdeb_zone_state *zsp, bool empty)
+{
+	enum sdebug_z_cond zc = zsp->z_cond;
+
+	if (zc == ZC4_CLOSED || zc == ZC2_IMPLICIT_OPEN ||
+	    zc == ZC3_EXPLICIT_OPEN || (empty && zc == ZC1_EMPTY)) {
+		if (zc == ZC2_IMPLICIT_OPEN || zc == ZC3_EXPLICIT_OPEN)
+			zbc_close_zone(devip, zsp);
+		if (zsp->z_cond == ZC4_CLOSED)
+			devip->nr_closed--;
+		zsp->z_wp = zsp->z_start + zsp->z_size;
+		zsp->z_cond = ZC5_FULL;
+	}
+}
+
+static void zbc_finish_all(struct sdebug_dev_info *devip)
+{
+	unsigned int i;
+
+	for (i = 0; i < devip->nr_zones; i++)
+		zbc_finish_zone(devip, &devip->zstate[i], false);
+}
+
+static int resp_finish_zone(struct scsi_cmnd *scp,
+			    struct sdebug_dev_info *devip)
+{
+	struct sdeb_zone_state *zsp;
+	int res = 0;
+	u64 z_id;
+	u8 *cmd = scp->cmnd;
+	bool all = cmd[14] & 0x01;
+	struct sdeb_store_info *sip = devip2sip(devip, false);
+	rwlock_t *macc_lckp = sip ? &sip->macc_lck : &sdeb_fake_rw_lck;
+
+	if (!sdebug_dev_is_zoned(devip)) {
+		mk_sense_invalid_opcode(scp);
+		return check_condition_result;
+	}
+
+	write_lock(macc_lckp);
+
+	if (all) {
+		zbc_finish_all(devip);
+		goto fini;
+	}
+
+	/* Finish the specified zone */
+	z_id = get_unaligned_be64(cmd + 2);
+	if (z_id >= sdebug_capacity) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zsp = zbc_zone(devip, z_id);
+	if (z_id != zsp->z_start) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+	if (zbc_zone_is_conv(zsp)) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zbc_finish_zone(devip, zsp, true);
+fini:
+	write_unlock(macc_lckp);
+	return res;
+}
+
+static void zbc_rwp_zone(struct sdebug_dev_info *devip,
+			 struct sdeb_zone_state *zsp)
+{
+	enum sdebug_z_cond zc;
+
+	if (zbc_zone_is_conv(zsp))
+		return;
+
+	zc = zsp->z_cond;
+	if (zc == ZC2_IMPLICIT_OPEN || zc == ZC3_EXPLICIT_OPEN)
+		zbc_close_zone(devip, zsp);
+
+	if (zsp->z_cond == ZC4_CLOSED)
+		devip->nr_closed--;
+
+	zsp->z_non_seq_resource = false;
+	zsp->z_wp = zsp->z_start;
+	zsp->z_cond = ZC1_EMPTY;
+}
+
+static void zbc_rwp_all(struct sdebug_dev_info *devip)
+{
+	unsigned int i;
+
+	for (i = 0; i < devip->nr_zones; i++)
+		zbc_rwp_zone(devip, &devip->zstate[i]);
+}
+
+static int resp_rwp_zone(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
+{
+	struct sdeb_zone_state *zsp;
+	int res = 0;
+	u64 z_id;
+	u8 *cmd = scp->cmnd;
+	bool all = cmd[14] & 0x01;
+	struct sdeb_store_info *sip = devip2sip(devip, false);
+	rwlock_t *macc_lckp = sip ? &sip->macc_lck : &sdeb_fake_rw_lck;
+
+	if (!sdebug_dev_is_zoned(devip)) {
+		mk_sense_invalid_opcode(scp);
+		return check_condition_result;
+	}
+
+	write_lock(macc_lckp);
+
+	if (all) {
+		zbc_rwp_all(devip);
+		goto fini;
+	}
+
+	z_id = get_unaligned_be64(cmd + 2);
+	if (z_id >= sdebug_capacity) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zsp = zbc_zone(devip, z_id);
+	if (z_id != zsp->z_start) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+	if (zbc_zone_is_conv(zsp)) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		res = check_condition_result;
+		goto fini;
+	}
+
+	zbc_rwp_zone(devip, zsp);
+fini:
+	write_unlock(macc_lckp);
+	return res;
+}
+
 static struct sdebug_queue *get_queue(struct scsi_cmnd *cmnd)
 {
 	u32 tag = blk_mq_unique_tag(cmnd->request);
@@ -3799,6 +4810,92 @@ static void sdebug_q_cmd_wq_complete(struct work_struct *work)
 static bool got_shared_uuid;
 static uuid_t shared_uuid;
 
+static int sdebug_device_create_zones(struct sdebug_dev_info *devip)
+{
+	struct sdeb_zone_state *zsp;
+	sector_t capacity = get_sdebug_capacity();
+	sector_t zstart = 0;
+	unsigned int i;
+
+	/*
+	 * Set the zone size: if sdeb_zbc_zone_size_mb is not set, figure out
+	 * a zone size allowing for at least 4 zones on the device. Otherwise,
+	 * use the specified zone size checking that at least 2 zones can be
+	 * created for the device.
+	 */
+	if (!sdeb_zbc_zone_size_mb) {
+		devip->zsize = (DEF_ZBC_ZONE_SIZE_MB * SZ_1M)
+			>> ilog2(sdebug_sector_size);
+		while (capacity < devip->zsize << 2 && devip->zsize >= 2)
+			devip->zsize >>= 1;
+		if (devip->zsize < 2) {
+			pr_err("Device capacity too small\n");
+			return -EINVAL;
+		}
+	} else {
+		if (!is_power_of_2(sdeb_zbc_zone_size_mb)) {
+			pr_err("Zone size is not a power of 2\n");
+			return -EINVAL;
+		}
+		devip->zsize = (sdeb_zbc_zone_size_mb * SZ_1M)
+			>> ilog2(sdebug_sector_size);
+		if (devip->zsize >= capacity) {
+			pr_err("Zone size too large for device capacity\n");
+			return -EINVAL;
+		}
+	}
+
+	devip->zsize_shift = ilog2(devip->zsize);
+	devip->nr_zones = (capacity + devip->zsize - 1) >> devip->zsize_shift;
+
+	if (sdeb_zbc_nr_conv >= devip->nr_zones) {
+		pr_err("Number of conventional zones too large\n");
+		return -EINVAL;
+	}
+	devip->nr_conv_zones = sdeb_zbc_nr_conv;
+
+	if (devip->zmodel == BLK_ZONED_HM) {
+		/* zbc_max_open_zones can be 0, meaning "not reported" */
+		if (sdeb_zbc_max_open >= devip->nr_zones - 1)
+			devip->max_open = (devip->nr_zones - 1) / 2;
+		else
+			devip->max_open = sdeb_zbc_max_open;
+	}
+
+	devip->zstate = kcalloc(devip->nr_zones,
+				sizeof(struct sdeb_zone_state), GFP_KERNEL);
+	if (!devip->zstate)
+		return -ENOMEM;
+
+	for (i = 0; i < devip->nr_zones; i++) {
+		zsp = &devip->zstate[i];
+
+		zsp->z_start = zstart;
+
+		if (i < devip->nr_conv_zones) {
+			zsp->z_type = ZBC_ZONE_TYPE_CNV;
+			zsp->z_cond = ZBC_NOT_WRITE_POINTER;
+			zsp->z_wp = (sector_t)-1;
+		} else {
+			if (devip->zmodel == BLK_ZONED_HM)
+				zsp->z_type = ZBC_ZONE_TYPE_SWR;
+			else
+				zsp->z_type = ZBC_ZONE_TYPE_SWP;
+			zsp->z_cond = ZC1_EMPTY;
+			zsp->z_wp = zsp->z_start;
+		}
+
+		if (zsp->z_start + devip->zsize < capacity)
+			zsp->z_size = devip->zsize;
+		else
+			zsp->z_size = capacity - zsp->z_start;
+
+		zstart += zsp->z_size;
+	}
+
+	return 0;
+}
+
 static struct sdebug_dev_info *sdebug_device_create(
 			struct sdebug_host_info *sdbg_host, gfp_t flags)
 {
@@ -3818,6 +4915,16 @@ static struct sdebug_dev_info *sdebug_device_create(
 			}
 		}
 		devip->sdbg_host = sdbg_host;
+		if (sdeb_zbc_in_use) {
+			devip->zmodel = sdeb_zbc_model;
+			if (sdebug_device_create_zones(devip)) {
+				kfree(devip);
+				return NULL;
+			}
+		} else {
+			devip->zmodel = BLK_ZONED_NONE;
+		}
+		devip->sdbg_host = sdbg_host;
 		list_add_tail(&devip->dev_list, &sdbg_host->dev_info_list);
 	}
 	return devip;
@@ -4144,8 +5251,7 @@ static int scsi_debug_host_reset(struct scsi_cmnd *SCpnt)
 	return SUCCESS;
 }
 
-static void __init sdebug_build_parts(unsigned char *ramp,
-				      unsigned long store_size)
+static void sdebug_build_parts(unsigned char *ramp, unsigned long store_size)
 {
 	struct msdos_partition *pp;
 	int starts[SDEBUG_MAX_PARTS + 2];
@@ -4247,6 +5353,8 @@ static void setup_inject(struct sdebug_queue *sqp,
 	sqcp->inj_cmd_abort = !!(SDEBUG_OPT_CMD_ABORT & sdebug_opts);
 }
 
+#define INCLUSIVE_TIMING_MAX_NS 1000000		/* 1 millisecond */
+
 /* Complete the processing of the thread that queued a SCSI command to this
  * driver. It either completes the command by calling cmnd_done() or
  * schedules a hr timer or work queue then returns 0. Returns
@@ -4258,8 +5366,10 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 				    struct sdebug_dev_info *),
 			 int delta_jiff, int ndelay)
 {
-	unsigned long iflags;
+	bool new_sd_dp;
 	int k, num_in_q, qdepth, inject;
+	unsigned long iflags;
+	u64 ns_from_boot = 0;
 	struct sdebug_queue *sqp;
 	struct sdebug_queued_cmd *sqcp;
 	struct scsi_device *sdp;
@@ -4275,7 +5385,6 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 	if (delta_jiff == 0)
 		goto respond_in_thread;
 
-	/* schedule the response at a later time if resources permit */
 	sqp = get_queue(cmnd);
 	spin_lock_irqsave(&sqp->qc_lock, iflags);
 	if (unlikely(atomic_read(&sqp->blocked))) {
@@ -4334,13 +5443,17 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 		sd_dp = kzalloc(sizeof(*sd_dp), GFP_ATOMIC);
 		if (sd_dp == NULL)
 			return SCSI_MLQUEUE_HOST_BUSY;
+		new_sd_dp = true;
+	} else {
+		new_sd_dp = false;
 	}
 
+	if (ndelay > 0 && ndelay < INCLUSIVE_TIMING_MAX_NS)
+		ns_from_boot = ktime_get_boottime_ns();
+
+	/* one of the resp_*() response functions is called here */
 	cmnd->result = pfp != NULL ? pfp(cmnd, devip) : 0;
 	if (cmnd->result & SDEG_RES_IMMED_MASK) {
-		/*
-		 * This is the F_DELAY_OVERR case. No delay.
-		 */
 		cmnd->result &= ~SDEG_RES_IMMED_MASK;
 		delta_jiff = ndelay = 0;
 	}
@@ -4355,9 +5468,37 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 		ktime_t kt;
 
 		if (delta_jiff > 0) {
-			kt = ns_to_ktime((u64)delta_jiff * (NSEC_PER_SEC / HZ));
-		} else
-			kt = ndelay;
+			u64 ns = jiffies_to_nsecs(delta_jiff);
+
+			if (sdebug_random && ns < U32_MAX) {
+				ns = prandom_u32_max((u32)ns);
+			} else if (sdebug_random) {
+				ns >>= 12;	/* scale to 4 usec precision */
+				if (ns < U32_MAX)	/* over 4 hours max */
+					ns = prandom_u32_max((u32)ns);
+				ns <<= 12;
+			}
+			kt = ns_to_ktime(ns);
+		} else {	/* ndelay has a 4.2 second max */
+			kt = sdebug_random ? prandom_u32_max((u32)ndelay) :
+					     (u32)ndelay;
+			if (ndelay < INCLUSIVE_TIMING_MAX_NS) {
+				u64 d = ktime_get_boottime_ns() - ns_from_boot;
+
+				if (kt <= d) {	/* elapsed duration >= kt */
+					sqcp->a_cmnd = NULL;
+					atomic_dec(&devip->num_in_q);
+					clear_bit(k, sqp->in_use_bm);
+					if (new_sd_dp)
+						kfree(sd_dp);
+					/* call scsi_done() from this thread */
+					cmnd->scsi_done(cmnd);
+					return 0;
+				}
+				/* otherwise reduce kt by elapsed time */
+				kt -= d;
+			}
+		}
 		if (!sd_dp->init_hrt) {
 			sd_dp->init_hrt = true;
 			sqcp->sd_dp = sd_dp;
@@ -4370,6 +5511,7 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 		if (sdebug_statistics)
 			sd_dp->issuing_cpu = raw_smp_processor_id();
 		sd_dp->defer_t = SDEB_DEFER_HRT;
+		/* schedule the invocation of scsi_done() for a later time */
 		hrtimer_start(&sd_dp->hrt, kt, HRTIMER_MODE_REL_PINNED);
 	} else {	/* jdelay < 0, use work queue */
 		if (!sd_dp->init_wq) {
@@ -4427,31 +5569,36 @@ module_param_named(every_nth, sdebug_every_nth, int, S_IRUGO | S_IWUSR);
 module_param_named(fake_rw, sdebug_fake_rw, int, S_IRUGO | S_IWUSR);
 module_param_named(guard, sdebug_guard, uint, S_IRUGO);
 module_param_named(host_lock, sdebug_host_lock, bool, S_IRUGO | S_IWUSR);
-module_param_string(inq_vendor, sdebug_inq_vendor_id,
-		    sizeof(sdebug_inq_vendor_id), S_IRUGO|S_IWUSR);
 module_param_string(inq_product, sdebug_inq_product_id,
-		    sizeof(sdebug_inq_product_id), S_IRUGO|S_IWUSR);
+		    sizeof(sdebug_inq_product_id), S_IRUGO | S_IWUSR);
 module_param_string(inq_rev, sdebug_inq_product_rev,
-		    sizeof(sdebug_inq_product_rev), S_IRUGO|S_IWUSR);
+		    sizeof(sdebug_inq_product_rev), S_IRUGO | S_IWUSR);
+module_param_string(inq_vendor, sdebug_inq_vendor_id,
+		    sizeof(sdebug_inq_vendor_id), S_IRUGO | S_IWUSR);
+module_param_named(lbprz, sdebug_lbprz, int, S_IRUGO);
 module_param_named(lbpu, sdebug_lbpu, int, S_IRUGO);
 module_param_named(lbpws, sdebug_lbpws, int, S_IRUGO);
 module_param_named(lbpws10, sdebug_lbpws10, int, S_IRUGO);
-module_param_named(lbprz, sdebug_lbprz, int, S_IRUGO);
 module_param_named(lowest_aligned, sdebug_lowest_aligned, int, S_IRUGO);
 module_param_named(max_luns, sdebug_max_luns, int, S_IRUGO | S_IWUSR);
 module_param_named(max_queue, sdebug_max_queue, int, S_IRUGO | S_IWUSR);
-module_param_named(medium_error_start, sdebug_medium_error_start, int, S_IRUGO | S_IWUSR);
-module_param_named(medium_error_count, sdebug_medium_error_count, int, S_IRUGO | S_IWUSR);
+module_param_named(medium_error_count, sdebug_medium_error_count, int,
+		   S_IRUGO | S_IWUSR);
+module_param_named(medium_error_start, sdebug_medium_error_start, int,
+		   S_IRUGO | S_IWUSR);
 module_param_named(ndelay, sdebug_ndelay, int, S_IRUGO | S_IWUSR);
 module_param_named(no_lun_0, sdebug_no_lun_0, int, S_IRUGO | S_IWUSR);
 module_param_named(no_uld, sdebug_no_uld, int, S_IRUGO);
 module_param_named(num_parts, sdebug_num_parts, int, S_IRUGO);
 module_param_named(num_tgts, sdebug_num_tgts, int, S_IRUGO | S_IWUSR);
 module_param_named(opt_blks, sdebug_opt_blks, int, S_IRUGO);
+module_param_named(opt_xferlen_exp, sdebug_opt_xferlen_exp, int, S_IRUGO);
 module_param_named(opts, sdebug_opts, int, S_IRUGO | S_IWUSR);
+module_param_named(per_host_store, sdebug_per_host_store, bool,
+		   S_IRUGO | S_IWUSR);
 module_param_named(physblk_exp, sdebug_physblk_exp, int, S_IRUGO);
-module_param_named(opt_xferlen_exp, sdebug_opt_xferlen_exp, int, S_IRUGO);
 module_param_named(ptype, sdebug_ptype, int, S_IRUGO | S_IWUSR);
+module_param_named(random, sdebug_random, bool, S_IRUGO | S_IWUSR);
 module_param_named(removable, sdebug_removable, bool, S_IRUGO | S_IWUSR);
 module_param_named(scsi_level, sdebug_scsi_level, int, S_IRUGO);
 module_param_named(sector_size, sdebug_sector_size, int, S_IRUGO);
@@ -4462,20 +5609,24 @@ module_param_named(unmap_alignment, sdebug_unmap_alignment, int, S_IRUGO);
 module_param_named(unmap_granularity, sdebug_unmap_granularity, int, S_IRUGO);
 module_param_named(unmap_max_blocks, sdebug_unmap_max_blocks, int, S_IRUGO);
 module_param_named(unmap_max_desc, sdebug_unmap_max_desc, int, S_IRUGO);
-module_param_named(virtual_gb, sdebug_virtual_gb, int, S_IRUGO | S_IWUSR);
 module_param_named(uuid_ctl, sdebug_uuid_ctl, int, S_IRUGO);
+module_param_named(virtual_gb, sdebug_virtual_gb, int, S_IRUGO | S_IWUSR);
 module_param_named(vpd_use_hostno, sdebug_vpd_use_hostno, int,
 		   S_IRUGO | S_IWUSR);
 module_param_named(wp, sdebug_wp, bool, S_IRUGO | S_IWUSR);
 module_param_named(write_same_length, sdebug_write_same_length, int,
 		   S_IRUGO | S_IWUSR);
+module_param_named(zbc, sdeb_zbc_model_s, charp, S_IRUGO);
+module_param_named(zone_max_open, sdeb_zbc_max_open, int, S_IRUGO);
+module_param_named(zone_nr_conv, sdeb_zbc_nr_conv, int, S_IRUGO);
+module_param_named(zone_size_mb, sdeb_zbc_zone_size_mb, int, S_IRUGO);
 
 MODULE_AUTHOR("Eric Youngdale + Douglas Gilbert");
 MODULE_DESCRIPTION("SCSI debug adapter driver");
 MODULE_LICENSE("GPL");
 MODULE_VERSION(SDEBUG_VERSION);
 
-MODULE_PARM_DESC(add_host, "0..127 hosts allowed(def=1)");
+MODULE_PARM_DESC(add_host, "add n hosts, in sysfs if negative remove host(s) (def=1)");
 MODULE_PARM_DESC(ato, "application tag ownership: 0=disk 1=host (def=1)");
 MODULE_PARM_DESC(cdb_len, "suggest CDB lengths to drivers (def=10)");
 MODULE_PARM_DESC(clustering, "when set enables larger transfers (def=0)");
@@ -4488,30 +5639,32 @@ MODULE_PARM_DESC(every_nth, "timeout every nth command(def=0)");
 MODULE_PARM_DESC(fake_rw, "fake reads/writes instead of copying (def=0)");
 MODULE_PARM_DESC(guard, "protection checksum: 0=crc, 1=ip (def=0)");
 MODULE_PARM_DESC(host_lock, "host_lock is ignored (def=0)");
-MODULE_PARM_DESC(inq_vendor, "SCSI INQUIRY vendor string (def=\"Linux\")");
 MODULE_PARM_DESC(inq_product, "SCSI INQUIRY product string (def=\"scsi_debug\")");
 MODULE_PARM_DESC(inq_rev, "SCSI INQUIRY revision string (def=\""
 		 SDEBUG_VERSION "\")");
+MODULE_PARM_DESC(inq_vendor, "SCSI INQUIRY vendor string (def=\"Linux\")");
+MODULE_PARM_DESC(lbprz,
+		 "on read unmapped LBs return 0 when 1 (def), return 0xff when 2");
 MODULE_PARM_DESC(lbpu, "enable LBP, support UNMAP command (def=0)");
 MODULE_PARM_DESC(lbpws, "enable LBP, support WRITE SAME(16) with UNMAP bit (def=0)");
 MODULE_PARM_DESC(lbpws10, "enable LBP, support WRITE SAME(10) with UNMAP bit (def=0)");
-MODULE_PARM_DESC(lbprz,
-	"on read unmapped LBs return 0 when 1 (def), return 0xff when 2");
 MODULE_PARM_DESC(lowest_aligned, "lowest aligned lba (def=0)");
 MODULE_PARM_DESC(max_luns, "number of LUNs per target to simulate(def=1)");
 MODULE_PARM_DESC(max_queue, "max number of queued commands (1 to max(def))");
-MODULE_PARM_DESC(medium_error_start, "starting sector number to return MEDIUM error");
 MODULE_PARM_DESC(medium_error_count, "count of sectors to return follow on MEDIUM error");
+MODULE_PARM_DESC(medium_error_start, "starting sector number to return MEDIUM error");
 MODULE_PARM_DESC(ndelay, "response delay in nanoseconds (def=0 -> ignore)");
 MODULE_PARM_DESC(no_lun_0, "no LU number 0 (def=0 -> have lun 0)");
 MODULE_PARM_DESC(no_uld, "stop ULD (e.g. sd driver) attaching (def=0))");
 MODULE_PARM_DESC(num_parts, "number of partitions(def=0)");
 MODULE_PARM_DESC(num_tgts, "number of targets per host to simulate(def=1)");
 MODULE_PARM_DESC(opt_blks, "optimal transfer length in blocks (def=1024)");
+MODULE_PARM_DESC(opt_xferlen_exp, "optimal transfer length granularity exponent (def=physblk_exp)");
 MODULE_PARM_DESC(opts, "1->noise, 2->medium_err, 4->timeout, 8->recovered_err... (def=0)");
+MODULE_PARM_DESC(per_host_store, "If set, next positive add_host will get new store (def=0)");
 MODULE_PARM_DESC(physblk_exp, "physical block exponent (def=0)");
-MODULE_PARM_DESC(opt_xferlen_exp, "optimal transfer length granularity exponent (def=physblk_exp)");
 MODULE_PARM_DESC(ptype, "SCSI peripheral type(def=0[disk])");
+MODULE_PARM_DESC(random, "If set, uniformly randomize command duration between 0 and delay_in_ns");
 MODULE_PARM_DESC(removable, "claim to have removable media (def=0)");
 MODULE_PARM_DESC(scsi_level, "SCSI level to simulate(def=7[SPC-5])");
 MODULE_PARM_DESC(sector_size, "logical block size in bytes (def=512)");
@@ -4528,6 +5681,10 @@ MODULE_PARM_DESC(virtual_gb, "virtual gigabyte (GiB) size (def=0 -> use dev_size
 MODULE_PARM_DESC(vpd_use_hostno, "0 -> dev ids ignore hostno (def=1 -> unique dev ids)");
 MODULE_PARM_DESC(wp, "Write Protect (def=0)");
 MODULE_PARM_DESC(write_same_length, "Maximum blocks per WRITE SAME cmd (def=0xffff)");
+MODULE_PARM_DESC(zbc, "'none' [0]; 'aware' [1]; 'managed' [2] (def=0). Can have 'host-' prefix");
+MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones; [0] for no limit (def=auto)");
+MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones (def=1)");
+MODULE_PARM_DESC(zone_size_mb, "Zone size in MiB (def=auto)");
 
 #define SDEBUG_INFO_LEN 256
 static char sdebug_info[SDEBUG_INFO_LEN];
@@ -4576,6 +5733,7 @@ static int scsi_debug_show_info(struct seq_file *m, struct Scsi_Host *host)
 {
 	int f, j, l;
 	struct sdebug_queue *sqp;
+	struct sdebug_host_info *sdhp;
 
 	seq_printf(m, "scsi_debug adapter driver, version %s [%s]\n",
 		   SDEBUG_VERSION, sdebug_version_date);
@@ -4611,6 +5769,34 @@ static int scsi_debug_show_info(struct seq_file *m, struct Scsi_Host *host)
 				   "first,last bits", f, l);
 		}
 	}
+
+	seq_printf(m, "this host_no=%d\n", host->host_no);
+	if (!xa_empty(per_store_ap)) {
+		bool niu;
+		int idx;
+		unsigned long l_idx;
+		struct sdeb_store_info *sip;
+
+		seq_puts(m, "\nhost list:\n");
+		j = 0;
+		list_for_each_entry(sdhp, &sdebug_host_list, host_list) {
+			idx = sdhp->si_idx;
+			seq_printf(m, "  %d: host_no=%d, si_idx=%d\n", j,
+				   sdhp->shost->host_no, idx);
+			++j;
+		}
+		seq_printf(m, "\nper_store array [most_recent_idx=%d]:\n",
+			   sdeb_most_recent_idx);
+		j = 0;
+		xa_for_each(per_store_ap, l_idx, sip) {
+			niu = xa_get_mark(per_store_ap, l_idx,
+					  SDEB_XA_NOT_IN_USE);
+			idx = (int)l_idx;
+			seq_printf(m, "  %d: idx=%d%s\n", j, idx,
+				   (niu ? "  not_in_use" : ""));
+			++j;
+		}
+	}
 	return 0;
 }
 
@@ -4734,7 +5920,13 @@ static ssize_t ptype_store(struct device_driver *ddp, const char *buf,
 {
 	int n;
 
+	/* Cannot change from or to TYPE_ZBC with sysfs */
+	if (sdebug_ptype == TYPE_ZBC)
+		return -EINVAL;
+
 	if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n >= 0)) {
+		if (n == TYPE_ZBC)
+			return -EINVAL;
 		sdebug_ptype = n;
 		return count;
 	}
@@ -4766,25 +5958,41 @@ static ssize_t fake_rw_show(struct device_driver *ddp, char *buf)
 static ssize_t fake_rw_store(struct device_driver *ddp, const char *buf,
 			     size_t count)
 {
-	int n;
+	int n, idx;
 
 	if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n >= 0)) {
+		bool want_store = (n == 0);
+		struct sdebug_host_info *sdhp;
+
 		n = (n > 0);
 		sdebug_fake_rw = (sdebug_fake_rw > 0);
-		if (sdebug_fake_rw != n) {
-			if ((0 == n) && (NULL == fake_storep)) {
-				unsigned long sz =
-					(unsigned long)sdebug_dev_size_mb *
-					1048576;
-
-				fake_storep = vzalloc(sz);
-				if (NULL == fake_storep) {
-					pr_err("out of memory, 9\n");
-					return -ENOMEM;
+		if (sdebug_fake_rw == n)
+			return count;	/* not transitioning so do nothing */
+
+		if (want_store) {	/* 1 --> 0 transition, set up store */
+			if (sdeb_first_idx < 0) {
+				idx = sdebug_add_store();
+				if (idx < 0)
+					return idx;
+			} else {
+				idx = sdeb_first_idx;
+				xa_clear_mark(per_store_ap, idx,
+					      SDEB_XA_NOT_IN_USE);
+			}
+			/* make all hosts use same store */
+			list_for_each_entry(sdhp, &sdebug_host_list,
+					    host_list) {
+				if (sdhp->si_idx != idx) {
+					xa_set_mark(per_store_ap, sdhp->si_idx,
+						    SDEB_XA_NOT_IN_USE);
+					sdhp->si_idx = idx;
 				}
 			}
-			sdebug_fake_rw = n;
+			sdeb_most_recent_idx = idx;
+		} else {	/* 0 --> 1 transition is trigger for shrink */
+			sdebug_erase_all_stores(true /* apart from first */);
 		}
+		sdebug_fake_rw = n;
 		return count;
 	}
 	return -EINVAL;
@@ -4832,6 +6040,24 @@ static ssize_t dev_size_mb_show(struct device_driver *ddp, char *buf)
 }
 static DRIVER_ATTR_RO(dev_size_mb);
 
+static ssize_t per_host_store_show(struct device_driver *ddp, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_per_host_store);
+}
+
+static ssize_t per_host_store_store(struct device_driver *ddp, const char *buf,
+				    size_t count)
+{
+	bool v;
+
+	if (kstrtobool(buf, &v))
+		return -EINVAL;
+
+	sdebug_per_host_store = v;
+	return count;
+}
+static DRIVER_ATTR_RW(per_host_store);
+
 static ssize_t num_parts_show(struct device_driver *ddp, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_num_parts);
@@ -4957,6 +6183,10 @@ static ssize_t virtual_gb_store(struct device_driver *ddp, const char *buf,
 	int n;
 	bool changed;
 
+	/* Ignore capacity change for ZBC drives for now */
+	if (sdeb_zbc_in_use)
+		return -ENOTSUPP;
+
 	if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n >= 0)) {
 		changed = (sdebug_virtual_gb != n);
 		sdebug_virtual_gb = n;
@@ -4984,26 +6214,42 @@ static DRIVER_ATTR_RW(virtual_gb);
 
 static ssize_t add_host_show(struct device_driver *ddp, char *buf)
 {
-	return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_add_host);
+	/* absolute number of hosts currently active is what is shown */
+	return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_num_hosts);
 }
 
-static int sdebug_add_adapter(void);
-static void sdebug_remove_adapter(void);
-
 static ssize_t add_host_store(struct device_driver *ddp, const char *buf,
 			      size_t count)
 {
+	bool found;
+	unsigned long idx;
+	struct sdeb_store_info *sip;
+	bool want_phs = (sdebug_fake_rw == 0) && sdebug_per_host_store;
 	int delta_hosts;
 
 	if (sscanf(buf, "%d", &delta_hosts) != 1)
 		return -EINVAL;
 	if (delta_hosts > 0) {
 		do {
-			sdebug_add_adapter();
+			found = false;
+			if (want_phs) {
+				xa_for_each_marked(per_store_ap, idx, sip,
+						   SDEB_XA_NOT_IN_USE) {
+					sdeb_most_recent_idx = (int)idx;
+					found = true;
+					break;
+				}
+				if (found)	/* re-use case */
+					sdebug_add_host_helper((int)idx);
+				else
+					sdebug_do_add_host(true);
+			} else {
+				sdebug_do_add_host(false);
+			}
 		} while (--delta_hosts);
 	} else if (delta_hosts < 0) {
 		do {
-			sdebug_remove_adapter();
+			sdebug_do_remove_host(false);
 		} while (++delta_hosts);
 	}
 	return count;
@@ -5087,14 +6333,19 @@ static DRIVER_ATTR_RO(ato);
 
 static ssize_t map_show(struct device_driver *ddp, char *buf)
 {
-	ssize_t count;
+	ssize_t count = 0;
 
 	if (!scsi_debug_lbp())
 		return scnprintf(buf, PAGE_SIZE, "0-%u\n",
 				 sdebug_store_sectors);
 
-	count = scnprintf(buf, PAGE_SIZE - 1, "%*pbl",
-			  (int)map_size, map_storep);
+	if (sdebug_fake_rw == 0 && !xa_empty(per_store_ap)) {
+		struct sdeb_store_info *sip = xa_load(per_store_ap, 0);
+
+		if (sip)
+			count = scnprintf(buf, PAGE_SIZE - 1, "%*pbl",
+					  (int)map_size, sip->map_storep);
+	}
 	buf[count++] = '\n';
 	buf[count] = '\0';
 
@@ -5102,6 +6353,24 @@ static ssize_t map_show(struct device_driver *ddp, char *buf)
 }
 static DRIVER_ATTR_RO(map);
 
+static ssize_t random_show(struct device_driver *ddp, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_random);
+}
+
+static ssize_t random_store(struct device_driver *ddp, const char *buf,
+			    size_t count)
+{
+	bool v;
+
+	if (kstrtobool(buf, &v))
+		return -EINVAL;
+
+	sdebug_random = v;
+	return count;
+}
+static DRIVER_ATTR_RW(random);
+
 static ssize_t removable_show(struct device_driver *ddp, char *buf)
 {
 	return scnprintf(buf, PAGE_SIZE, "%d\n", sdebug_removable ? 1 : 0);
@@ -5178,12 +6447,51 @@ static ssize_t cdb_len_store(struct device_driver *ddp, const char *buf,
 }
 static DRIVER_ATTR_RW(cdb_len);
 
+static const char * const zbc_model_strs_a[] = {
+	[BLK_ZONED_NONE] = "none",
+	[BLK_ZONED_HA]   = "host-aware",
+	[BLK_ZONED_HM]   = "host-managed",
+};
+
+static const char * const zbc_model_strs_b[] = {
+	[BLK_ZONED_NONE] = "no",
+	[BLK_ZONED_HA]   = "aware",
+	[BLK_ZONED_HM]   = "managed",
+};
+
+static const char * const zbc_model_strs_c[] = {
+	[BLK_ZONED_NONE] = "0",
+	[BLK_ZONED_HA]   = "1",
+	[BLK_ZONED_HM]   = "2",
+};
+
+static int sdeb_zbc_model_str(const char *cp)
+{
+	int res = sysfs_match_string(zbc_model_strs_a, cp);
+
+	if (res < 0) {
+		res = sysfs_match_string(zbc_model_strs_b, cp);
+		if (res < 0) {
+			res = sysfs_match_string(zbc_model_strs_c, cp);
+			if (res < 0)
+				return -EINVAL;
+		}
+	}
+	return res;
+}
+
+static ssize_t zbc_show(struct device_driver *ddp, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%s\n",
+			 zbc_model_strs_a[sdeb_zbc_model]);
+}
+static DRIVER_ATTR_RO(zbc);
 
 /* Note: The following array creates attribute files in the
    /sys/bus/pseudo/drivers/scsi_debug directory. The advantage of these
    files (over those found in the /sys/module/scsi_debug/parameters
    directory) is that auxiliary actions can be triggered when an attribute
-   is changed. For example see: sdebug_add_host_store() above.
+   is changed. For example see: add_host_store() above.
  */
 
 static struct attribute *sdebug_drv_attrs[] = {
@@ -5203,6 +6511,7 @@ static struct attribute *sdebug_drv_attrs[] = {
 	&driver_attr_scsi_level.attr,
 	&driver_attr_virtual_gb.attr,
 	&driver_attr_add_host.attr,
+	&driver_attr_per_host_store.attr,
 	&driver_attr_vpd_use_hostno.attr,
 	&driver_attr_sector_size.attr,
 	&driver_attr_statistics.attr,
@@ -5212,12 +6521,14 @@ static struct attribute *sdebug_drv_attrs[] = {
 	&driver_attr_guard.attr,
 	&driver_attr_ato.attr,
 	&driver_attr_map.attr,
+	&driver_attr_random.attr,
 	&driver_attr_removable.attr,
 	&driver_attr_host_lock.attr,
 	&driver_attr_ndelay.attr,
 	&driver_attr_strict.attr,
 	&driver_attr_uuid_ctl.attr,
 	&driver_attr_cdb_len.attr,
+	&driver_attr_zbc.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(sdebug_drv);
@@ -5226,11 +6537,13 @@ static struct device *pseudo_primary;
 
 static int __init scsi_debug_init(void)
 {
+	bool want_store = (sdebug_fake_rw == 0);
 	unsigned long sz;
-	int host_to_add;
-	int k;
-	int ret;
+	int k, ret, hosts_to_add;
+	int idx = -1;
 
+	ramdisk_lck_a[0] = &atomic_rw;
+	ramdisk_lck_a[1] = &atomic_rw2;
 	atomic_set(&retired_max_queue, 0);
 
 	if (sdebug_ndelay >= 1000 * 1000 * 1000) {
@@ -5304,6 +6617,40 @@ static int __init scsi_debug_init(void)
 	for (k = 0; k < submit_queues; ++k)
 		spin_lock_init(&sdebug_q_arr[k].qc_lock);
 
+	/*
+	 * check for host managed zoned block device specified with
+	 * ptype=0x14 or zbc=XXX.
+	 */
+	if (sdebug_ptype == TYPE_ZBC) {
+		sdeb_zbc_model = BLK_ZONED_HM;
+	} else if (sdeb_zbc_model_s && *sdeb_zbc_model_s) {
+		k = sdeb_zbc_model_str(sdeb_zbc_model_s);
+		if (k < 0) {
+			ret = k;
+			goto free_vm;
+		}
+		sdeb_zbc_model = k;
+		switch (sdeb_zbc_model) {
+		case BLK_ZONED_NONE:
+		case BLK_ZONED_HA:
+			sdebug_ptype = TYPE_DISK;
+			break;
+		case BLK_ZONED_HM:
+			sdebug_ptype = TYPE_ZBC;
+			break;
+		default:
+			pr_err("Invalid ZBC model\n");
+			return -EINVAL;
+		}
+	}
+	if (sdeb_zbc_model != BLK_ZONED_NONE) {
+		sdeb_zbc_in_use = true;
+		if (sdebug_dev_size_mb == DEF_DEV_SIZE_PRE_INIT)
+			sdebug_dev_size_mb = DEF_ZBC_DEV_SIZE_MB;
+	}
+
+	if (sdebug_dev_size_mb == DEF_DEV_SIZE_PRE_INIT)
+		sdebug_dev_size_mb = DEF_DEV_SIZE_MB;
 	if (sdebug_dev_size_mb < 1)
 		sdebug_dev_size_mb = 1;  /* force minimum 1 MB ramdisk */
 	sz = (unsigned long)sdebug_dev_size_mb * 1048576;
@@ -5326,36 +6673,6 @@ static int __init scsi_debug_init(void)
 		sdebug_cylinders_per = (unsigned long)sdebug_capacity /
 			       (sdebug_sectors_per * sdebug_heads);
 	}
-
-	if (sdebug_fake_rw == 0) {
-		fake_storep = vzalloc(sz);
-		if (NULL == fake_storep) {
-			pr_err("out of memory, 1\n");
-			ret = -ENOMEM;
-			goto free_q_arr;
-		}
-		if (sdebug_num_parts > 0)
-			sdebug_build_parts(fake_storep, sz);
-	}
-
-	if (sdebug_dix) {
-		int dif_size;
-
-		dif_size = sdebug_store_sectors * sizeof(struct t10_pi_tuple);
-		dif_storep = vmalloc(dif_size);
-
-		pr_err("dif_storep %u bytes @ %p\n", dif_size, dif_storep);
-
-		if (dif_storep == NULL) {
-			pr_err("out of mem. (DIX)\n");
-			ret = -ENOMEM;
-			goto free_vm;
-		}
-
-		memset(dif_storep, 0xff, dif_size);
-	}
-
-	/* Logical Block Provisioning */
 	if (scsi_debug_lbp()) {
 		sdebug_unmap_max_blocks =
 			clamp(sdebug_unmap_max_blocks, 0U, 0xffffffffU);
@@ -5371,26 +6688,16 @@ static int __init scsi_debug_init(void)
 		    sdebug_unmap_alignment) {
 			pr_err("ERR: unmap_granularity <= unmap_alignment\n");
 			ret = -EINVAL;
-			goto free_vm;
+			goto free_q_arr;
 		}
-
-		map_size = lba_to_map_index(sdebug_store_sectors - 1) + 1;
-		map_storep = vmalloc(array_size(sizeof(long),
-						BITS_TO_LONGS(map_size)));
-
-		pr_info("%lu provisioning blocks\n", map_size);
-
-		if (map_storep == NULL) {
-			pr_err("out of mem. (MAP)\n");
-			ret = -ENOMEM;
-			goto free_vm;
+	}
+	xa_init_flags(per_store_ap, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
+	if (want_store) {
+		idx = sdebug_add_store();
+		if (idx < 0) {
+			ret = idx;
+			goto free_q_arr;
 		}
-
-		bitmap_zero(map_storep, map_size);
-
-		/* Map first 1KB for partition table */
-		if (sdebug_num_parts)
-			map_region(0, 2);
 	}
 
 	pseudo_primary = root_device_register("pseudo_0");
@@ -5410,18 +6717,28 @@ static int __init scsi_debug_init(void)
 		goto bus_unreg;
 	}
 
-	host_to_add = sdebug_add_host;
+	hosts_to_add = sdebug_add_host;
 	sdebug_add_host = 0;
 
-	for (k = 0; k < host_to_add; k++) {
-		if (sdebug_add_adapter()) {
-			pr_err("sdebug_add_adapter failed k=%d\n", k);
-			break;
+	for (k = 0; k < hosts_to_add; k++) {
+		if (want_store && k == 0) {
+			ret = sdebug_add_host_helper(idx);
+			if (ret < 0) {
+				pr_err("add_host_helper k=%d, error=%d\n",
+				       k, -ret);
+				break;
+			}
+		} else {
+			ret = sdebug_do_add_host(want_store &&
+						 sdebug_per_host_store);
+			if (ret < 0) {
+				pr_err("add_host k=%d error=%d\n", k, -ret);
+				break;
+			}
 		}
 	}
-
 	if (sdebug_verbose)
-		pr_info("built %d host(s)\n", sdebug_add_host);
+		pr_info("built %d host(s)\n", sdebug_num_hosts);
 
 	return 0;
 
@@ -5430,9 +6747,7 @@ bus_unreg:
 dev_unreg:
 	root_device_unregister(pseudo_primary);
 free_vm:
-	vfree(map_storep);
-	vfree(dif_storep);
-	vfree(fake_storep);
+	sdebug_erase_store(idx, NULL);
 free_q_arr:
 	kfree(sdebug_q_arr);
 	return ret;
@@ -5440,20 +6755,18 @@ free_q_arr:
 
 static void __exit scsi_debug_exit(void)
 {
-	int k = sdebug_add_host;
+	int k = sdebug_num_hosts;
 
 	stop_all_queued();
 	for (; k; k--)
-		sdebug_remove_adapter();
+		sdebug_do_remove_host(true);
 	free_all_queued();
 	driver_unregister(&sdebug_driverfs_driver);
 	bus_unregister(&pseudo_lld_bus);
 	root_device_unregister(pseudo_primary);
 
-	vfree(map_storep);
-	vfree(dif_storep);
-	vfree(fake_storep);
-	kfree(sdebug_q_arr);
+	sdebug_erase_all_stores(false);
+	xa_destroy(per_store_ap);
 }
 
 device_initcall(scsi_debug_init);
@@ -5467,29 +6780,146 @@ static void sdebug_release_adapter(struct device *dev)
 	kfree(sdbg_host);
 }
 
-static int sdebug_add_adapter(void)
+/* idx must be valid, if sip is NULL then it will be obtained using idx */
+static void sdebug_erase_store(int idx, struct sdeb_store_info *sip)
 {
-	int k, devs_per_host;
-	int error = 0;
+	if (idx < 0)
+		return;
+	if (!sip) {
+		if (xa_empty(per_store_ap))
+			return;
+		sip = xa_load(per_store_ap, idx);
+		if (!sip)
+			return;
+	}
+	vfree(sip->map_storep);
+	vfree(sip->dif_storep);
+	vfree(sip->storep);
+	xa_erase(per_store_ap, idx);
+	kfree(sip);
+}
+
+/* Assume apart_from_first==false only in shutdown case. */
+static void sdebug_erase_all_stores(bool apart_from_first)
+{
+	unsigned long idx;
+	struct sdeb_store_info *sip = NULL;
+
+	xa_for_each(per_store_ap, idx, sip) {
+		if (apart_from_first)
+			apart_from_first = false;
+		else
+			sdebug_erase_store(idx, sip);
+	}
+	if (apart_from_first)
+		sdeb_most_recent_idx = sdeb_first_idx;
+}
+
+/*
+ * Returns store xarray new element index (idx) if >=0 else negated errno.
+ * Limit the number of stores to 65536.
+ */
+static int sdebug_add_store(void)
+{
+	int res;
+	u32 n_idx;
+	unsigned long iflags;
+	unsigned long sz = (unsigned long)sdebug_dev_size_mb * 1048576;
+	struct sdeb_store_info *sip = NULL;
+	struct xa_limit xal = { .max = 1 << 16, .min = 0 };
+
+	sip = kzalloc(sizeof(*sip), GFP_KERNEL);
+	if (!sip)
+		return -ENOMEM;
+
+	xa_lock_irqsave(per_store_ap, iflags);
+	res = __xa_alloc(per_store_ap, &n_idx, sip, xal, GFP_ATOMIC);
+	if (unlikely(res < 0)) {
+		xa_unlock_irqrestore(per_store_ap, iflags);
+		kfree(sip);
+		pr_warn("%s: xa_alloc() errno=%d\n", __func__, -res);
+		return res;
+	}
+	sdeb_most_recent_idx = n_idx;
+	if (sdeb_first_idx < 0)
+		sdeb_first_idx = n_idx;
+	xa_unlock_irqrestore(per_store_ap, iflags);
+
+	res = -ENOMEM;
+	sip->storep = vzalloc(sz);
+	if (!sip->storep) {
+		pr_err("user data oom\n");
+		goto err;
+	}
+	if (sdebug_num_parts > 0)
+		sdebug_build_parts(sip->storep, sz);
+
+	/* DIF/DIX: what T10 calls Protection Information (PI) */
+	if (sdebug_dix) {
+		int dif_size;
+
+		dif_size = sdebug_store_sectors * sizeof(struct t10_pi_tuple);
+		sip->dif_storep = vmalloc(dif_size);
+
+		pr_info("dif_storep %u bytes @ %pK\n", dif_size,
+			sip->dif_storep);
+
+		if (!sip->dif_storep) {
+			pr_err("DIX oom\n");
+			goto err;
+		}
+		memset(sip->dif_storep, 0xff, dif_size);
+	}
+	/* Logical Block Provisioning */
+	if (scsi_debug_lbp()) {
+		map_size = lba_to_map_index(sdebug_store_sectors - 1) + 1;
+		sip->map_storep = vmalloc(array_size(sizeof(long),
+						     BITS_TO_LONGS(map_size)));
+
+		pr_info("%lu provisioning blocks\n", map_size);
+
+		if (!sip->map_storep) {
+			pr_err("LBP map oom\n");
+			goto err;
+		}
+
+		bitmap_zero(sip->map_storep, map_size);
+
+		/* Map first 1KB for partition table */
+		if (sdebug_num_parts)
+			map_region(sip, 0, 2);
+	}
+
+	rwlock_init(&sip->macc_lck);
+	return (int)n_idx;
+err:
+	sdebug_erase_store((int)n_idx, sip);
+	pr_warn("%s: failed, errno=%d\n", __func__, -res);
+	return res;
+}
+
+static int sdebug_add_host_helper(int per_host_idx)
+{
+	int k, devs_per_host, idx;
+	int error = -ENOMEM;
 	struct sdebug_host_info *sdbg_host;
 	struct sdebug_dev_info *sdbg_devinfo, *tmp;
 
 	sdbg_host = kzalloc(sizeof(*sdbg_host), GFP_KERNEL);
-	if (sdbg_host == NULL) {
-		pr_err("out of memory at line %d\n", __LINE__);
+	if (!sdbg_host)
 		return -ENOMEM;
-	}
+	idx = (per_host_idx < 0) ? sdeb_first_idx : per_host_idx;
+	if (xa_get_mark(per_store_ap, idx, SDEB_XA_NOT_IN_USE))
+		xa_clear_mark(per_store_ap, idx, SDEB_XA_NOT_IN_USE);
+	sdbg_host->si_idx = idx;
 
 	INIT_LIST_HEAD(&sdbg_host->dev_info_list);
 
 	devs_per_host = sdebug_num_tgts * sdebug_max_luns;
 	for (k = 0; k < devs_per_host; k++) {
 		sdbg_devinfo = sdebug_device_create(sdbg_host, GFP_KERNEL);
-		if (!sdbg_devinfo) {
-			pr_err("out of memory at line %d\n", __LINE__);
-			error = -ENOMEM;
+		if (!sdbg_devinfo)
 			goto clean;
-		}
 	}
 
 	spin_lock(&sdebug_host_list_lock);
@@ -5499,44 +6929,77 @@ static int sdebug_add_adapter(void)
 	sdbg_host->dev.bus = &pseudo_lld_bus;
 	sdbg_host->dev.parent = pseudo_primary;
 	sdbg_host->dev.release = &sdebug_release_adapter;
-	dev_set_name(&sdbg_host->dev, "adapter%d", sdebug_add_host);
+	dev_set_name(&sdbg_host->dev, "adapter%d", sdebug_num_hosts);
 
 	error = device_register(&sdbg_host->dev);
-
 	if (error)
 		goto clean;
 
-	++sdebug_add_host;
-	return error;
+	++sdebug_num_hosts;
+	return 0;
 
 clean:
 	list_for_each_entry_safe(sdbg_devinfo, tmp, &sdbg_host->dev_info_list,
 				 dev_list) {
 		list_del(&sdbg_devinfo->dev_list);
+		kfree(sdbg_devinfo->zstate);
 		kfree(sdbg_devinfo);
 	}
-
 	kfree(sdbg_host);
+	pr_warn("%s: failed, errno=%d\n", __func__, -error);
 	return error;
 }
 
-static void sdebug_remove_adapter(void)
+static int sdebug_do_add_host(bool mk_new_store)
 {
+	int ph_idx = sdeb_most_recent_idx;
+
+	if (mk_new_store) {
+		ph_idx = sdebug_add_store();
+		if (ph_idx < 0)
+			return ph_idx;
+	}
+	return sdebug_add_host_helper(ph_idx);
+}
+
+static void sdebug_do_remove_host(bool the_end)
+{
+	int idx = -1;
 	struct sdebug_host_info *sdbg_host = NULL;
+	struct sdebug_host_info *sdbg_host2;
 
 	spin_lock(&sdebug_host_list_lock);
 	if (!list_empty(&sdebug_host_list)) {
 		sdbg_host = list_entry(sdebug_host_list.prev,
 				       struct sdebug_host_info, host_list);
-		list_del(&sdbg_host->host_list);
+		idx = sdbg_host->si_idx;
+	}
+	if (!the_end && idx >= 0) {
+		bool unique = true;
+
+		list_for_each_entry(sdbg_host2, &sdebug_host_list, host_list) {
+			if (sdbg_host2 == sdbg_host)
+				continue;
+			if (idx == sdbg_host2->si_idx) {
+				unique = false;
+				break;
+			}
+		}
+		if (unique) {
+			xa_set_mark(per_store_ap, idx, SDEB_XA_NOT_IN_USE);
+			if (idx == sdeb_most_recent_idx)
+				--sdeb_most_recent_idx;
+		}
 	}
+	if (sdbg_host)
+		list_del(&sdbg_host->host_list);
 	spin_unlock(&sdebug_host_list_lock);
 
 	if (!sdbg_host)
 		return;
 
 	device_unregister(&sdbg_host->dev);
-	--sdebug_add_host;
+	--sdebug_num_hosts;
 }
 
 static int sdebug_change_qdepth(struct scsi_device *sdev, int qdepth)
@@ -5595,6 +7058,7 @@ static int scsi_debug_queuecommand(struct Scsi_Host *shost,
 	const struct opcode_info_t *oip;
 	const struct opcode_info_t *r_oip;
 	struct sdebug_dev_info *devip;
+
 	u8 *cmd = scp->cmnd;
 	int (*r_pfp)(struct scsi_cmnd *, struct sdebug_dev_info *);
 	int (*pfp)(struct scsi_cmnd *, struct sdebug_dev_info *) = NULL;
@@ -5724,7 +7188,7 @@ static int scsi_debug_queuecommand(struct Scsi_Host *shost,
 		pfp = r_pfp;    /* if leaf function ptr NULL, try the root's */
 
 fini:
-	if (F_DELAY_OVERR & flags)
+	if (F_DELAY_OVERR & flags)	/* cmds like INQUIRY respond asap */
 		return schedule_resp(scp, devip, errsts, pfp, 0, 0);
 	else if ((flags & F_LONG_DELAY) && (sdebug_jdelay > 0 ||
 					    sdebug_ndelay > 10000)) {
@@ -5866,8 +7330,9 @@ static int sdebug_driver_probe(struct device *dev)
 		pr_err("scsi_add_host failed\n");
 		error = -ENODEV;
 		scsi_host_put(hpnt);
-	} else
+	} else {
 		scsi_scan_host(hpnt);
+	}
 
 	return error;
 }
@@ -5889,6 +7354,7 @@ static int sdebug_driver_remove(struct device *dev)
 	list_for_each_entry_safe(sdbg_devinfo, tmp, &sdbg_host->dev_info_list,
 				 dev_list) {
 		list_del(&sdbg_devinfo->dev_list);
+		kfree(sdbg_devinfo->zstate);
 		kfree(sdbg_devinfo);
 	}
 
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 978be1602f71..927b1e641842 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -1412,6 +1412,7 @@ static int scsi_eh_stu(struct Scsi_Host *shost,
 				sdev_printk(KERN_INFO, sdev,
 					    "%s: skip START_UNIT, past eh deadline\n",
 					    current->comm));
+			scsi_device_put(sdev);
 			break;
 		}
 		stu_scmd = NULL;
@@ -1478,6 +1479,7 @@ static int scsi_eh_bus_device_reset(struct Scsi_Host *shost,
 				sdev_printk(KERN_INFO, sdev,
 					    "%s: skip BDR, past eh deadline\n",
 					     current->comm));
+			scsi_device_put(sdev);
 			break;
 		}
 		bdr_scmd = NULL;
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index df4905df5cd4..0ba7a65e7c8d 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -202,24 +202,17 @@ static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)
 	blk_mq_requeue_request(cmd->request, true);
 }
 
-/*
- * Function:    scsi_queue_insert()
- *
- * Purpose:     Insert a command in the midlevel queue.
- *
- * Arguments:   cmd    - command that we are adding to queue.
- *              reason - why we are inserting command to queue.
- *
- * Lock status: Assumed that lock is not held upon entry.
+/**
+ * scsi_queue_insert - Reinsert a command in the queue.
+ * @cmd:    command that we are adding to queue.
+ * @reason: why we are inserting command to queue.
  *
- * Returns:     Nothing.
+ * We do this for one of two cases. Either the host is busy and it cannot accept
+ * any more commands for the time being, or the device returned QUEUE_FULL and
+ * can accept no more commands.
  *
- * Notes:       We do this for one of two cases.  Either the host is busy
- *              and it cannot accept any more commands for the time being,
- *              or the device returned QUEUE_FULL and can accept no more
- *              commands.
- * Notes:       This could be called either from an interrupt context or a
- *              normal process context.
+ * Context: This could be called either from an interrupt context or a normal
+ * process context.
  */
 void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
 {
@@ -301,16 +294,12 @@ int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
 }
 EXPORT_SYMBOL(__scsi_execute);
 
-/*
- * Function:    scsi_init_cmd_errh()
- *
- * Purpose:     Initialize cmd fields related to error handling.
- *
- * Arguments:   cmd	- command that is ready to be queued.
+/**
+ * scsi_init_cmd_errh - Initialize cmd fields related to error handling.
+ * @cmd:  command that is ready to be queued.
  *
- * Notes:       This function has the job of initializing a number of
- *              fields related to error handling.   Typically this will
- *              be called once for each command, as required.
+ * This function has the job of initializing a number of fields related to error
+ * handling. Typically this will be called once for each command, as required.
  */
 static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
 {
@@ -496,17 +485,11 @@ static void scsi_starved_list_run(struct Scsi_Host *shost)
 	spin_unlock_irqrestore(shost->host_lock, flags);
 }
 
-/*
- * Function:   scsi_run_queue()
- *
- * Purpose:    Select a proper request queue to serve next
- *
- * Arguments:  q       - last request's queue
- *
- * Returns:     Nothing
+/**
+ * scsi_run_queue - Select a proper request queue to serve next.
+ * @q:  last request's queue
  *
- * Notes:      The previous command was completely finished, start
- *             a new one if possible.
+ * The previous command was completely finished, start a new one if possible.
  */
 static void scsi_run_queue(struct request_queue *q)
 {
@@ -548,7 +531,7 @@ static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
 	}
 }
 
-static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd)
+static void scsi_free_sgtables(struct scsi_cmnd *cmd)
 {
 	if (cmd->sdb.table.nents)
 		sg_free_table_chained(&cmd->sdb.table,
@@ -560,7 +543,7 @@ static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd)
 
 static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
 {
-	scsi_mq_free_sgtables(cmd);
+	scsi_free_sgtables(cmd);
 	scsi_uninit_cmd(cmd);
 }
 
@@ -896,34 +879,27 @@ static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result,
 	return result;
 }
 
-/*
- * Function:    scsi_io_completion()
- *
- * Purpose:     Completion processing for block device I/O requests.
- *
- * Arguments:   cmd   - command that is finished.
- *
- * Lock status: Assumed that no lock is held upon entry.
- *
- * Returns:     Nothing
- *
- * Notes:       We will finish off the specified number of sectors.  If we
- *		are done, the command block will be released and the queue
- *		function will be goosed.  If we are not done then we have to
- *		figure out what to do next:
- *
- *		a) We can call scsi_requeue_command().  The request
- *		   will be unprepared and put back on the queue.  Then
- *		   a new command will be created for it.  This should
- *		   be used if we made forward progress, or if we want
- *		   to switch from READ(10) to READ(6) for example.
- *
- *		b) We can call __scsi_queue_insert().  The request will
- *		   be put back on the queue and retried using the same
- *		   command as before, possibly after a delay.
- *
- *		c) We can call scsi_end_request() with blk_stat other than
- *		   BLK_STS_OK, to fail the remainder of the request.
+/**
+ * scsi_io_completion - Completion processing for SCSI commands.
+ * @cmd:	command that is finished.
+ * @good_bytes:	number of processed bytes.
+ *
+ * We will finish off the specified number of sectors. If we are done, the
+ * command block will be released and the queue function will be goosed. If we
+ * are not done then we have to figure out what to do next:
+ *
+ *   a) We can call scsi_io_completion_reprep().  The request will be
+ *	unprepared and put back on the queue.  Then a new command will
+ *	be created for it.  This should be used if we made forward
+ *	progress, or if we want to switch from READ(10) to READ(6) for
+ *	example.
+ *
+ *   b) We can call scsi_io_completion_action().  The request will be
+ *	put back on the queue and retried using the same command as
+ *	before, possibly after a delay.
+ *
+ *   c) We can call scsi_end_request() with blk_stat other than
+ *	BLK_STS_OK, to fail the remainder of the request.
  */
 void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 {
@@ -951,8 +927,7 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 		blk_rq_sectors(req), good_bytes));
 
 	/*
-	 * Next deal with any sectors which we were able to correctly
-	 * handle. Failed, zero length commands always need to drop down
+	 * Failed, zero length commands always need to drop down
 	 * to retry code. Fast path should return in this block.
 	 */
 	if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) {
@@ -986,16 +961,14 @@ static inline bool scsi_cmd_needs_dma_drain(struct scsi_device *sdev,
 	       sdev->host->hostt->dma_need_drain(rq);
 }
 
-/*
- * Function:    scsi_init_io()
- *
- * Purpose:     SCSI I/O initialize function.
- *
- * Arguments:   cmd   - Command descriptor we wish to initialize
+/**
+ * scsi_init_io - SCSI I/O initialization function.
+ * @cmd:  command descriptor we wish to initialize
  *
- * Returns:     BLK_STS_OK on success
- *		BLK_STS_RESOURCE if the failure is retryable
- *		BLK_STS_IOERR if the failure is fatal
+ * Returns:
+ * * BLK_STS_OK       - on success
+ * * BLK_STS_RESOURCE - if the failure is retryable
+ * * BLK_STS_IOERR    - if the failure is fatal
  */
 blk_status_t scsi_init_io(struct scsi_cmnd *cmd)
 {
@@ -1086,7 +1059,7 @@ blk_status_t scsi_init_io(struct scsi_cmnd *cmd)
 
 	return BLK_STS_OK;
 out_free_sgtables:
-	scsi_mq_free_sgtables(cmd);
+	scsi_free_sgtables(cmd);
 	return ret;
 }
 EXPORT_SYMBOL(scsi_init_io);
@@ -1217,6 +1190,7 @@ static blk_status_t scsi_setup_cmnd(struct scsi_device *sdev,
 		struct request *req)
 {
 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
+	blk_status_t ret;
 
 	if (!blk_rq_bytes(req))
 		cmd->sc_data_direction = DMA_NONE;
@@ -1226,9 +1200,14 @@ static blk_status_t scsi_setup_cmnd(struct scsi_device *sdev,
 		cmd->sc_data_direction = DMA_FROM_DEVICE;
 
 	if (blk_rq_is_scsi(req))
-		return scsi_setup_scsi_cmnd(sdev, req);
+		ret = scsi_setup_scsi_cmnd(sdev, req);
 	else
-		return scsi_setup_fs_cmnd(sdev, req);
+		ret = scsi_setup_fs_cmnd(sdev, req);
+
+	if (ret != BLK_STS_OK)
+		scsi_free_sgtables(cmd);
+
+	return ret;
 }
 
 static blk_status_t
@@ -1893,6 +1872,7 @@ struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev)
 int scsi_mq_setup_tags(struct Scsi_Host *shost)
 {
 	unsigned int cmd_size, sgl_size;
+	struct blk_mq_tag_set *tag_set = &shost->tag_set;
 
 	sgl_size = max_t(unsigned int, sizeof(struct scatterlist),
 				scsi_mq_inline_sgl_size(shost));
@@ -1901,21 +1881,21 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost)
 		cmd_size += sizeof(struct scsi_data_buffer) +
 			sizeof(struct scatterlist) * SCSI_INLINE_PROT_SG_CNT;
 
-	memset(&shost->tag_set, 0, sizeof(shost->tag_set));
+	memset(tag_set, 0, sizeof(*tag_set));
 	if (shost->hostt->commit_rqs)
-		shost->tag_set.ops = &scsi_mq_ops;
+		tag_set->ops = &scsi_mq_ops;
 	else
-		shost->tag_set.ops = &scsi_mq_ops_no_commit;
-	shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1;
-	shost->tag_set.queue_depth = shost->can_queue;
-	shost->tag_set.cmd_size = cmd_size;
-	shost->tag_set.numa_node = NUMA_NO_NODE;
-	shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
-	shost->tag_set.flags |=
+		tag_set->ops = &scsi_mq_ops_no_commit;
+	tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1;
+	tag_set->queue_depth = shost->can_queue;
+	tag_set->cmd_size = cmd_size;
+	tag_set->numa_node = NUMA_NO_NODE;
+	tag_set->flags = BLK_MQ_F_SHOULD_MERGE;
+	tag_set->flags |=
 		BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy);
-	shost->tag_set.driver_data = shost;
+	tag_set->driver_data = shost;
 
-	return blk_mq_alloc_tag_set(&shost->tag_set);
+	return blk_mq_alloc_tag_set(tag_set);
 }
 
 void scsi_mq_destroy_tags(struct Scsi_Host *shost)
@@ -1944,21 +1924,13 @@ struct scsi_device *scsi_device_from_queue(struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(scsi_device_from_queue);
 
-/*
- * Function:    scsi_block_requests()
- *
- * Purpose:     Utility function used by low-level drivers to prevent further
- *		commands from being queued to the device.
- *
- * Arguments:   shost       - Host in question
- *
- * Returns:     Nothing
- *
- * Lock status: No locks are assumed held.
+/**
+ * scsi_block_requests - Utility function used by low-level drivers to prevent
+ * further commands from being queued to the device.
+ * @shost:  host in question
  *
- * Notes:       There is no timer nor any other means by which the requests
- *		get unblocked other than the low-level driver calling
- *		scsi_unblock_requests().
+ * There is no timer nor any other means by which the requests get unblocked
+ * other than the low-level driver calling scsi_unblock_requests().
  */
 void scsi_block_requests(struct Scsi_Host *shost)
 {
@@ -1966,25 +1938,15 @@ void scsi_block_requests(struct Scsi_Host *shost)
 }
 EXPORT_SYMBOL(scsi_block_requests);
 
-/*
- * Function:    scsi_unblock_requests()
- *
- * Purpose:     Utility function used by low-level drivers to allow further
- *		commands from being queued to the device.
- *
- * Arguments:   shost       - Host in question
- *
- * Returns:     Nothing
- *
- * Lock status: No locks are assumed held.
- *
- * Notes:       There is no timer nor any other means by which the requests
- *		get unblocked other than the low-level driver calling
- *		scsi_unblock_requests().
- *
- *		This is done as an API function so that changes to the
- *		internals of the scsi mid-layer won't require wholesale
- *		changes to drivers that use this feature.
+/**
+ * scsi_unblock_requests - Utility function used by low-level drivers to allow
+ * further commands to be queued to the device.
+ * @shost:  host in question
+ *
+ * There is no timer nor any other means by which the requests get unblocked
+ * other than the low-level driver calling scsi_unblock_requests(). This is done
+ * as an API function so that changes to the internals of the scsi mid-layer
+ * won't require wholesale changes to drivers that use this feature.
  */
 void scsi_unblock_requests(struct Scsi_Host *shost)
 {
@@ -2865,11 +2827,27 @@ scsi_host_block(struct Scsi_Host *shost)
 	struct scsi_device *sdev;
 	int ret = 0;
 
+	/*
+	 * Call scsi_internal_device_block_nowait so we can avoid
+	 * calling synchronize_rcu() for each LUN.
+	 */
 	shost_for_each_device(sdev, shost) {
-		ret = scsi_internal_device_block(sdev);
+		mutex_lock(&sdev->state_mutex);
+		ret = scsi_internal_device_block_nowait(sdev);
+		mutex_unlock(&sdev->state_mutex);
 		if (ret)
 			break;
 	}
+
+	/*
+	 * SCSI never enables blk-mq's BLK_MQ_F_BLOCKING flag so
+	 * calling synchronize_rcu() once is enough.
+	 */
+	WARN_ON_ONCE(shost->tag_set.flags & BLK_MQ_F_BLOCKING);
+
+	if (!ret)
+		synchronize_rcu();
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(scsi_host_block);
@@ -2882,8 +2860,10 @@ scsi_host_unblock(struct Scsi_Host *shost, int new_state)
 
 	shost_for_each_device(sdev, shost) {
 		ret = scsi_internal_device_unblock(sdev, new_state);
-		if (ret)
+		if (ret) {
+			scsi_device_put(sdev);
 			break;
+		}
 	}
 	return ret;
 }
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index b2a803c51288..f4cc08eb47ba 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -1616,6 +1616,12 @@ static DECLARE_TRANSPORT_CLASS(iscsi_connection_class,
 static struct sock *nls;
 static DEFINE_MUTEX(rx_queue_mutex);
 
+/*
+ * conn_mutex protects the {start,bind,stop,destroy}_conn from racing
+ * against the kernel stop_connection recovery mechanism
+ */
+static DEFINE_MUTEX(conn_mutex);
+
 static LIST_HEAD(sesslist);
 static LIST_HEAD(sessdestroylist);
 static DEFINE_SPINLOCK(sesslock);
@@ -2445,6 +2451,32 @@ int iscsi_offload_mesg(struct Scsi_Host *shost,
 }
 EXPORT_SYMBOL_GPL(iscsi_offload_mesg);
 
+/*
+ * This can be called without the rx_queue_mutex, if invoked by the kernel
+ * stop work. But, in that case, it is guaranteed not to race with
+ * iscsi_destroy by conn_mutex.
+ */
+static void iscsi_if_stop_conn(struct iscsi_cls_conn *conn, int flag)
+{
+	/*
+	 * It is important that this path doesn't rely on
+	 * rx_queue_mutex, otherwise, a thread doing allocation on a
+	 * start_session/start_connection could sleep waiting on a
+	 * writeback to a failed iscsi device, that cannot be recovered
+	 * because the lock is held.  If we don't hold it here, the
+	 * kernel stop_conn_work_fn has a chance to stop the broken
+	 * session and resolve the allocation.
+	 *
+	 * Still, the user invoked .stop_conn() needs to be serialized
+	 * with stop_conn_work_fn by a private mutex.  Not pretty, but
+	 * it works.
+	 */
+	mutex_lock(&conn_mutex);
+	conn->transport->stop_conn(conn, flag);
+	mutex_unlock(&conn_mutex);
+
+}
+
 static void stop_conn_work_fn(struct work_struct *work)
 {
 	struct iscsi_cls_conn *conn, *tmp;
@@ -2463,30 +2495,17 @@ static void stop_conn_work_fn(struct work_struct *work)
 		uint32_t sid = iscsi_conn_get_sid(conn);
 		struct iscsi_cls_session *session;
 
-		mutex_lock(&rx_queue_mutex);
-
 		session = iscsi_session_lookup(sid);
 		if (session) {
 			if (system_state != SYSTEM_RUNNING) {
 				session->recovery_tmo = 0;
-				conn->transport->stop_conn(conn,
-							   STOP_CONN_TERM);
+				iscsi_if_stop_conn(conn, STOP_CONN_TERM);
 			} else {
-				conn->transport->stop_conn(conn,
-							   STOP_CONN_RECOVER);
+				iscsi_if_stop_conn(conn, STOP_CONN_RECOVER);
 			}
 		}
 
 		list_del_init(&conn->conn_list_err);
-
-		mutex_unlock(&rx_queue_mutex);
-
-		/* we don't want to hold rx_queue_mutex for too long,
-		 * for instance if many conns failed at the same time,
-		 * since this stall other iscsi maintenance operations.
-		 * Give other users a chance to proceed.
-		 */
-		cond_resched();
 	}
 }
 
@@ -2846,8 +2865,11 @@ iscsi_if_destroy_conn(struct iscsi_transport *transport, struct iscsi_uevent *ev
 	spin_unlock_irqrestore(&connlock, flags);
 
 	ISCSI_DBG_TRANS_CONN(conn, "Destroying transport conn\n");
+
+	mutex_lock(&conn_mutex);
 	if (transport->destroy_conn)
 		transport->destroy_conn(conn);
+	mutex_unlock(&conn_mutex);
 
 	return 0;
 }
@@ -3689,9 +3711,12 @@ iscsi_if_recv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, uint32_t *group)
 			break;
 		}
 
+		mutex_lock(&conn_mutex);
 		ev->r.retcode =	transport->bind_conn(session, conn,
 						ev->u.b_conn.transport_eph,
 						ev->u.b_conn.is_leading);
+		mutex_unlock(&conn_mutex);
+
 		if (ev->r.retcode || !transport->ep_connect)
 			break;
 
@@ -3713,9 +3738,11 @@ iscsi_if_recv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, uint32_t *group)
 	case ISCSI_UEVENT_START_CONN:
 		conn = iscsi_conn_lookup(ev->u.start_conn.sid, ev->u.start_conn.cid);
 		if (conn) {
+			mutex_lock(&conn_mutex);
 			ev->r.retcode = transport->start_conn(conn);
 			if (!ev->r.retcode)
 				conn->state = ISCSI_CONN_UP;
+			mutex_unlock(&conn_mutex);
 		}
 		else
 			err = -EINVAL;
@@ -3723,17 +3750,20 @@ iscsi_if_recv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, uint32_t *group)
 	case ISCSI_UEVENT_STOP_CONN:
 		conn = iscsi_conn_lookup(ev->u.stop_conn.sid, ev->u.stop_conn.cid);
 		if (conn)
-			transport->stop_conn(conn, ev->u.stop_conn.flag);
+			iscsi_if_stop_conn(conn, ev->u.stop_conn.flag);
 		else
 			err = -EINVAL;
 		break;
 	case ISCSI_UEVENT_SEND_PDU:
 		conn = iscsi_conn_lookup(ev->u.send_pdu.sid, ev->u.send_pdu.cid);
-		if (conn)
+		if (conn) {
+			mutex_lock(&conn_mutex);
 			ev->r.retcode =	transport->send_pdu(conn,
 				(struct iscsi_hdr*)((char*)ev + sizeof(*ev)),
 				(char*)ev + sizeof(*ev) + ev->u.send_pdu.hdr_size,
 				ev->u.send_pdu.data_size);
+			mutex_unlock(&conn_mutex);
+		}
 		else
 			err = -EINVAL;
 		break;
@@ -4728,7 +4758,9 @@ static __init int iscsi_transport_init(void)
 		goto unregister_flashnode_bus;
 	}
 
-	iscsi_eh_timer_workq = create_singlethread_workqueue("iscsi_eh");
+	iscsi_eh_timer_workq = alloc_workqueue("%s",
+			WQ_SYSFS | __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_UNBOUND,
+			2, "iscsi_eh");
 	if (!iscsi_eh_timer_workq) {
 		err = -ENOMEM;
 		goto release_nls;
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 7b0383e42b4c..d90fefffe31b 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -528,6 +528,21 @@ max_write_same_blocks_store(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(max_write_same_blocks);
 
+static ssize_t
+zoned_cap_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct scsi_disk *sdkp = to_scsi_disk(dev);
+
+	if (sdkp->device->type == TYPE_ZBC)
+		return sprintf(buf, "host-managed\n");
+	if (sdkp->zoned == 1)
+		return sprintf(buf, "host-aware\n");
+	if (sdkp->zoned == 2)
+		return sprintf(buf, "drive-managed\n");
+	return sprintf(buf, "none\n");
+}
+static DEVICE_ATTR_RO(zoned_cap);
+
 static struct attribute *sd_disk_attrs[] = {
 	&dev_attr_cache_type.attr,
 	&dev_attr_FUA.attr,
@@ -541,6 +556,7 @@ static struct attribute *sd_disk_attrs[] = {
 	&dev_attr_zeroing_mode.attr,
 	&dev_attr_max_write_same_blocks.attr,
 	&dev_attr_max_medium_access_timeouts.attr,
+	&dev_attr_zoned_cap.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(sd_disk);
@@ -2962,6 +2978,9 @@ static void sd_read_block_characteristics(struct scsi_disk *sdkp)
 			 * with partitions as regular block devices.
 			 */
 			q->limits.zoned = BLK_ZONED_NONE;
+			if (sdkp->zoned == 2 && sdkp->first_scan)
+				sd_printk(KERN_NOTICE, sdkp,
+					  "Drive-managed SMR disk\n");
 		}
 	}
 	if (blk_queue_is_zoned(q) && sdkp->first_scan)
diff --git a/drivers/scsi/sgiwd93.c b/drivers/scsi/sgiwd93.c
index 713bce998b0e..3bdf0deb8f15 100644
--- a/drivers/scsi/sgiwd93.c
+++ b/drivers/scsi/sgiwd93.c
@@ -187,7 +187,7 @@ static inline void init_hpc_chain(struct ip22_hostdata *hdata)
 		hcp++;
 		dma += sizeof(struct hpc_chunk);
 		start += sizeof(struct hpc_chunk);
-	};
+	}
 	hcp--;
 	hcp->desc.pnext = hdata->dma;
 }
diff --git a/drivers/scsi/smartpqi/Kconfig b/drivers/scsi/smartpqi/Kconfig
index d3311c014863..8eec241f074b 100644
--- a/drivers/scsi/smartpqi/Kconfig
+++ b/drivers/scsi/smartpqi/Kconfig
@@ -42,7 +42,7 @@ config SCSI_SMARTPQI
 	depends on PCI && SCSI && !S390
 	select SCSI_SAS_ATTRS
 	select RAID_ATTRS
-	---help---
+	help
 	This driver supports Microsemi PQI controllers.
 
 	<http://www.microsemi.com>
diff --git a/drivers/scsi/sni_53c710.c b/drivers/scsi/sni_53c710.c
index f8397978f8ab..03d43f016397 100644
--- a/drivers/scsi/sni_53c710.c
+++ b/drivers/scsi/sni_53c710.c
@@ -28,7 +28,6 @@
 #include <linux/platform_device.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <asm/delay.h>
 
diff --git a/drivers/scsi/snic/snic.h b/drivers/scsi/snic/snic.h
index de0ab5fc8474..f4c666285bba 100644
--- a/drivers/scsi/snic/snic.h
+++ b/drivers/scsi/snic/snic.h
@@ -399,7 +399,7 @@ void snic_handle_link_event(struct snic *);
 void snic_handle_link(struct work_struct *);
 
 int snic_queue_exch_ver_req(struct snic *);
-int snic_io_exch_ver_cmpl_handler(struct snic *, struct snic_fw_req *);
+void snic_io_exch_ver_cmpl_handler(struct snic *, struct snic_fw_req *);
 
 int snic_queue_wq_desc(struct snic *, void *os_buf, u16 len);
 
diff --git a/drivers/scsi/snic/snic_ctl.c b/drivers/scsi/snic/snic_ctl.c
index 449b03f3bbd3..4cd86115cfb2 100644
--- a/drivers/scsi/snic/snic_ctl.c
+++ b/drivers/scsi/snic/snic_ctl.c
@@ -151,7 +151,7 @@ error:
 /*
  * snic_io_exch_ver_cmpl_handler
  */
-int
+void
 snic_io_exch_ver_cmpl_handler(struct snic *snic, struct snic_fw_req *fwreq)
 {
 	struct snic_req_info *rqi = NULL;
@@ -160,7 +160,6 @@ snic_io_exch_ver_cmpl_handler(struct snic *snic, struct snic_fw_req *fwreq)
 	u32 cmnd_id, hid, max_sgs;
 	ulong ctx = 0;
 	unsigned long flags;
-	int ret = 0;
 
 	SNIC_HOST_INFO(snic->shost, "Exch Ver Compl Received.\n");
 	snic_io_hdr_dec(&fwreq->hdr, &typ, &hdr_stat, &cmnd_id, &hid, &ctx);
@@ -224,8 +223,6 @@ exch_cmpl_end:
 	snic_release_untagged_req(snic, rqi);
 
 	SNIC_HOST_INFO(snic->shost, "Exch_cmpl Done, hdr_stat %d.\n", hdr_stat);
-
-	return ret;
 } /* end of snic_io_exch_ver_cmpl_handler */
 
 /*
diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
index f9b589d60a46..0c4aa4665a2f 100644
--- a/drivers/scsi/sr.c
+++ b/drivers/scsi/sr.c
@@ -51,6 +51,8 @@
 #include <linux/pm_runtime.h>
 #include <linux/uaccess.h>
 
+#include <asm/unaligned.h>
+
 #include <scsi/scsi.h>
 #include <scsi/scsi_dbg.h>
 #include <scsi/scsi_device.h>
@@ -344,10 +346,8 @@ static int sr_done(struct scsi_cmnd *SCpnt)
 		case ILLEGAL_REQUEST:
 			if (!(SCpnt->sense_buffer[0] & 0x90))
 				break;
-			error_sector = (SCpnt->sense_buffer[3] << 24) |
-				(SCpnt->sense_buffer[4] << 16) |
-				(SCpnt->sense_buffer[5] << 8) |
-				SCpnt->sense_buffer[6];
+			error_sector =
+				get_unaligned_be32(&SCpnt->sense_buffer[3]);
 			if (SCpnt->request->bio != NULL)
 				block_sectors =
 					bio_sectors(SCpnt->request->bio);
@@ -495,13 +495,9 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt)
 		SCpnt->sdb.length = this_count * s_size;
 	}
 
-	SCpnt->cmnd[2] = (unsigned char) (block >> 24) & 0xff;
-	SCpnt->cmnd[3] = (unsigned char) (block >> 16) & 0xff;
-	SCpnt->cmnd[4] = (unsigned char) (block >> 8) & 0xff;
-	SCpnt->cmnd[5] = (unsigned char) block & 0xff;
+	put_unaligned_be32(block, &SCpnt->cmnd[2]);
 	SCpnt->cmnd[6] = SCpnt->cmnd[9] = 0;
-	SCpnt->cmnd[7] = (unsigned char) (this_count >> 8) & 0xff;
-	SCpnt->cmnd[8] = (unsigned char) this_count & 0xff;
+	put_unaligned_be16(this_count, &SCpnt->cmnd[7]);
 
 	/*
 	 * We shouldn't disconnect in the middle of a sector, so with a dumb
@@ -796,7 +792,7 @@ static int sr_probe(struct device *dev)
 	disk->queue = sdev->request_queue;
 
 	if (register_cdrom(disk, &cd->cdi))
-		goto fail_put;
+		goto fail_minor;
 
 	/*
 	 * Initialize block layer runtime PM stuffs before the
@@ -814,8 +810,13 @@ static int sr_probe(struct device *dev)
 
 	return 0;
 
+fail_minor:
+	spin_lock(&sr_index_lock);
+	clear_bit(minor, sr_index_bits);
+	spin_unlock(&sr_index_lock);
 fail_put:
 	put_disk(disk);
+	mutex_destroy(&cd->lock);
 fail_free:
 	kfree(cd);
 fail:
@@ -853,8 +854,7 @@ static void get_sectorsize(struct scsi_cd *cd)
 	} else {
 		long last_written;
 
-		cd->capacity = 1 + ((buffer[0] << 24) | (buffer[1] << 16) |
-				    (buffer[2] << 8) | buffer[3]);
+		cd->capacity = 1 + get_unaligned_be32(&buffer[0]);
 		/*
 		 * READ_CAPACITY doesn't return the correct size on
 		 * certain UDF media.  If last_written is larger, use
@@ -865,8 +865,7 @@ static void get_sectorsize(struct scsi_cd *cd)
 		if (!cdrom_get_last_written(&cd->cdi, &last_written))
 			cd->capacity = max_t(long, cd->capacity, last_written);
 
-		sector_size = (buffer[4] << 24) |
-		    (buffer[5] << 16) | (buffer[6] << 8) | buffer[7];
+		sector_size = get_unaligned_be32(&buffer[4]);
 		switch (sector_size) {
 			/*
 			 * HP 4020i CD-Recorder reports 2340 byte sectors
@@ -954,13 +953,13 @@ static void get_capabilities(struct scsi_cd *cd)
 	}
 
 	n = data.header_length + data.block_descriptor_length;
-	cd->cdi.speed = ((buffer[n + 8] << 8) + buffer[n + 9]) / 176;
+	cd->cdi.speed = get_unaligned_be16(&buffer[n + 8]) / 176;
 	cd->readcd_known = 1;
 	cd->readcd_cdda = buffer[n + 5] & 0x01;
 	/* print some capability bits */
 	sr_printk(KERN_INFO, cd,
 		  "scsi3-mmc drive: %dx/%dx %s%s%s%s%s%s\n",
-		  ((buffer[n + 14] << 8) + buffer[n + 15]) / 176,
+		  get_unaligned_be16(&buffer[n + 14]) / 176,
 		  cd->cdi.speed,
 		  buffer[n + 3] & 0x01 ? "writer " : "", /* CD Writer */
 		  buffer[n + 3] & 0x20 ? "dvd-ram " : "",
diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
index c5f9b348b438..87fbc0ea350b 100644
--- a/drivers/scsi/st.c
+++ b/drivers/scsi/st.c
@@ -1457,7 +1457,6 @@ static int st_flush(struct file *filp, fl_owner_t id)
    accessing this tape. */
 static int st_release(struct inode *inode, struct file *filp)
 {
-	int result = 0;
 	struct scsi_tape *STp = filp->private_data;
 
 	if (STp->door_locked == ST_LOCKED_AUTO)
@@ -1470,9 +1469,9 @@ static int st_release(struct inode *inode, struct file *filp)
 	scsi_autopm_put_device(STp->device);
 	scsi_tape_put(STp);
 
-	return result;
+	return 0;
 }
-
+
 /* The checks common to both reading and writing */
 static ssize_t rw_checks(struct scsi_tape *STp, struct file *filp, size_t count)
 {
@@ -4922,7 +4921,7 @@ static int sgl_map_user_pages(struct st_buffer *STbp,
 	unsigned long end = (uaddr + count + PAGE_SIZE - 1) >> PAGE_SHIFT;
 	unsigned long start = uaddr >> PAGE_SHIFT;
 	const int nr_pages = end - start;
-	int res, i, j;
+	int res, i;
 	struct page **pages;
 	struct rq_map_data *mdata = &STbp->map_data;
 
@@ -4944,7 +4943,7 @@ static int sgl_map_user_pages(struct st_buffer *STbp,
 
         /* Try to fault in all of the necessary pages */
         /* rw==READ means read from drive, write into memory area */
-	res = get_user_pages_fast(uaddr, nr_pages, rw == READ ? FOLL_WRITE : 0,
+	res = pin_user_pages_fast(uaddr, nr_pages, rw == READ ? FOLL_WRITE : 0,
 				  pages);
 
 	/* Errors and no page mapped should return here */
@@ -4964,8 +4963,7 @@ static int sgl_map_user_pages(struct st_buffer *STbp,
 	return nr_pages;
  out_unmap:
 	if (res > 0) {
-		for (j=0; j < res; j++)
-			put_page(pages[j]);
+		unpin_user_pages(pages, res);
 		res = 0;
 	}
 	kfree(pages);
@@ -4977,18 +4975,9 @@ static int sgl_map_user_pages(struct st_buffer *STbp,
 static int sgl_unmap_user_pages(struct st_buffer *STbp,
 				const unsigned int nr_pages, int dirtied)
 {
-	int i;
+	/* FIXME: cache flush missing for rw==READ */
+	unpin_user_pages_dirty_lock(STbp->mapped_pages, nr_pages, dirtied);
 
-	for (i=0; i < nr_pages; i++) {
-		struct page *page = STbp->mapped_pages[i];
-
-		if (dirtied)
-			SetPageDirty(page);
-		/* FIXME: cache flush missing for rw==READ
-		 * FIXME: call the correct reference counting function
-		 */
-		put_page(page);
-	}
 	kfree(STbp->mapped_pages);
 	STbp->mapped_pages = NULL;
 
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index 072ed8728657..2d90cddd8ac2 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -2035,9 +2035,6 @@ static int storvsc_suspend(struct hv_device *hv_dev)
 
 	vmbus_close(hv_dev->channel);
 
-	memset(stor_device->stor_chns, 0,
-	       num_possible_cpus() * sizeof(void *));
-
 	kfree(stor_device->stor_chns);
 	stor_device->stor_chns = NULL;
 
diff --git a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig
index e2005aeddc2d..d35378be89e8 100644
--- a/drivers/scsi/ufs/Kconfig
+++ b/drivers/scsi/ufs/Kconfig
@@ -38,7 +38,7 @@ config SCSI_UFSHCD
 	select PM_DEVFREQ
 	select DEVFREQ_GOV_SIMPLE_ONDEMAND
 	select NLS
-	---help---
+	help
 	This selects the support for UFS devices in Linux, say Y and make
 	  sure that you know the name of your UFS host adapter (the card
 	  inside your computer that "speaks" the UFS protocol, also
@@ -53,7 +53,7 @@ config SCSI_UFSHCD
 config SCSI_UFSHCD_PCI
 	tristate "PCI bus based UFS Controller support"
 	depends on SCSI_UFSHCD && PCI
-	---help---
+	help
 	This selects the PCI UFS Host Controller Interface. Select this if
 	you have UFS Host Controller with PCI Interface.
 
@@ -64,7 +64,7 @@ config SCSI_UFSHCD_PCI
 config SCSI_UFS_DWC_TC_PCI
 	tristate "DesignWare pci support using a G210 Test Chip"
 	depends on SCSI_UFSHCD_PCI
-	---help---
+	help
 	  Synopsys Test Chip is a PHY for prototyping purposes.
 
 	  If unsure, say N.
@@ -72,7 +72,7 @@ config SCSI_UFS_DWC_TC_PCI
 config SCSI_UFSHCD_PLATFORM
 	tristate "Platform bus based UFS Controller support"
 	depends on SCSI_UFSHCD
-	---help---
+	help
 	This selects the UFS host controller support. Select this if
 	you have an UFS controller on Platform bus.
 
@@ -91,7 +91,7 @@ config SCSI_UFS_CDNS_PLATFORM
 config SCSI_UFS_DWC_TC_PLATFORM
 	tristate "DesignWare platform support using a G210 Test Chip"
 	depends on SCSI_UFSHCD_PLATFORM
-	---help---
+	help
 	  Synopsys Test Chip is a PHY for prototyping purposes.
 
 	  If unsure, say N.
@@ -126,7 +126,7 @@ config SCSI_UFS_MEDIATEK
 config SCSI_UFS_HISI
 	tristate "Hisilicon specific hooks to UFS controller platform driver"
 	depends on (ARCH_HISI || COMPILE_TEST) && SCSI_UFSHCD_PLATFORM
-	---help---
+	help
 	  This selects the Hisilicon specific additions to UFSHCD platform driver.
 
 	  Select this if you have UFS controller on Hisilicon chipset.
diff --git a/drivers/scsi/ufs/ti-j721e-ufs.c b/drivers/scsi/ufs/ti-j721e-ufs.c
index 5216d228cdd9..46bb905b4d6a 100644
--- a/drivers/scsi/ufs/ti-j721e-ufs.c
+++ b/drivers/scsi/ufs/ti-j721e-ufs.c
@@ -32,14 +32,14 @@ static int ti_j721e_ufs_probe(struct platform_device *pdev)
 	ret = pm_runtime_get_sync(dev);
 	if (ret < 0) {
 		pm_runtime_put_noidle(dev);
-		return ret;
+		goto disable_pm;
 	}
 
 	/* Select MPHY refclk frequency */
 	clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(clk)) {
 		dev_err(dev, "Cannot claim MPHY clock.\n");
-		return PTR_ERR(clk);
+		goto clk_err;
 	}
 	clk_rate = clk_get_rate(clk);
 	if (clk_rate == 26000000)
@@ -54,16 +54,23 @@ static int ti_j721e_ufs_probe(struct platform_device *pdev)
 				   dev);
 	if (ret) {
 		dev_err(dev, "failed to populate child nodes %d\n", ret);
-		pm_runtime_put_sync(dev);
+		goto clk_err;
 	}
 
 	return ret;
+
+clk_err:
+	pm_runtime_put_sync(dev);
+disable_pm:
+	pm_runtime_disable(dev);
+	return ret;
 }
 
 static int ti_j721e_ufs_remove(struct platform_device *pdev)
 {
 	of_platform_depopulate(&pdev->dev);
 	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
 
 	return 0;
 }
diff --git a/drivers/scsi/ufs/ufs-mediatek.c b/drivers/scsi/ufs/ufs-mediatek.c
index 673c16596fb2..d56ce8d97d4e 100644
--- a/drivers/scsi/ufs/ufs-mediatek.c
+++ b/drivers/scsi/ufs/ufs-mediatek.c
@@ -30,6 +30,12 @@
 #define ufs_mtk_device_reset_ctrl(high, res) \
 	ufs_mtk_smc(UFS_MTK_SIP_DEVICE_RESET, high, res)
 
+static struct ufs_dev_fix ufs_mtk_dev_fixups[] = {
+	UFS_FIX(UFS_VENDOR_SKHYNIX, "H9HQ21AFAMZDAR",
+		UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES),
+	END_FIX
+};
+
 static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable)
 {
 	u32 tmp;
@@ -73,9 +79,9 @@ static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
 
 	if (status == PRE_CHANGE) {
 		if (host->unipro_lpm)
-			hba->hba_enable_delay_us = 0;
+			hba->vps->hba_enable_delay_us = 0;
 		else
-			hba->hba_enable_delay_us = 600;
+			hba->vps->hba_enable_delay_us = 600;
 	}
 
 	return 0;
@@ -263,6 +269,10 @@ static int ufs_mtk_init(struct ufs_hba *hba)
 	/* Enable clock-gating */
 	hba->caps |= UFSHCD_CAP_CLK_GATING;
 
+	/* Enable WriteBooster */
+	hba->caps |= UFSHCD_CAP_WB_EN;
+	hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80);
+
 	/*
 	 * ufshcd_vops_init() is invoked after
 	 * ufshcd_setup_clock(true) in ufshcd_hba_init() thus
@@ -555,10 +565,8 @@ static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
 	struct ufs_dev_info *dev_info = &hba->dev_info;
 	u16 mid = dev_info->wmanufacturerid;
 
-	if (mid == UFS_VENDOR_SAMSUNG) {
-		hba->dev_quirks &= ~UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE;
+	if (mid == UFS_VENDOR_SAMSUNG)
 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 6);
-	}
 
 	/*
 	 * Decide waiting time before gating reference clock and
@@ -575,6 +583,17 @@ static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
 	return 0;
 }
 
+static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
+{
+	struct ufs_dev_info *dev_info = &hba->dev_info;
+	u16 mid = dev_info->wmanufacturerid;
+
+	ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups);
+
+	if (mid == UFS_VENDOR_SAMSUNG)
+		hba->dev_quirks &= ~UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE;
+}
+
 /**
  * struct ufs_hba_mtk_vops - UFS MTK specific variant operations
  *
@@ -589,6 +608,7 @@ static struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
 	.link_startup_notify = ufs_mtk_link_startup_notify,
 	.pwr_change_notify   = ufs_mtk_pwr_change_notify,
 	.apply_dev_quirks    = ufs_mtk_apply_dev_quirks,
+	.fixup_dev_quirks    = ufs_mtk_fixup_dev_quirks,
 	.suspend             = ufs_mtk_suspend,
 	.resume              = ufs_mtk_resume,
 	.dbg_register_dump   = ufs_mtk_dbg_register_dump,
diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c
index 19aa5c44e0da..2e6ddb5cdfc2 100644
--- a/drivers/scsi/ufs/ufs-qcom.c
+++ b/drivers/scsi/ufs/ufs-qcom.c
@@ -572,7 +572,6 @@ static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 {
 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);
 	struct phy *phy = host->generic_phy;
-	int ret = 0;
 
 	if (ufs_qcom_is_link_off(hba)) {
 		/*
@@ -587,7 +586,7 @@ static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 		ufs_qcom_disable_lane_clks(host);
 	}
 
-	return ret;
+	return 0;
 }
 
 static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
@@ -1071,6 +1070,7 @@ static void ufs_qcom_set_caps(struct ufs_hba *hba)
 	hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
 	hba->caps |= UFSHCD_CAP_CLK_SCALING;
 	hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
+	hba->caps |= UFSHCD_CAP_WB_EN;
 
 	if (host->hw_ver.major >= 0x2) {
 		host->caps = UFS_QCOM_CAP_QUNIPRO |
@@ -1658,11 +1658,11 @@ static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba)
 
 	/* sleep a bit intermittently as we are dumping too much data */
 	ufs_qcom_print_hw_debug_reg_all(hba, NULL, ufs_qcom_dump_regs_wrapper);
-	usleep_range(1000, 1100);
+	udelay(1000);
 	ufs_qcom_testbus_read(hba);
-	usleep_range(1000, 1100);
+	udelay(1000);
 	ufs_qcom_print_unipro_testbus(hba);
-	usleep_range(1000, 1100);
+	udelay(1000);
 }
 
 /**
diff --git a/drivers/scsi/ufs/ufs-sysfs.c b/drivers/scsi/ufs/ufs-sysfs.c
index 92a63eebdca9..2d71d232a69d 100644
--- a/drivers/scsi/ufs/ufs-sysfs.c
+++ b/drivers/scsi/ufs/ufs-sysfs.c
@@ -276,6 +276,10 @@ UFS_DEVICE_DESC_PARAM(device_version, _DEV_VER, 2);
 UFS_DEVICE_DESC_PARAM(number_of_secure_wpa, _NUM_SEC_WPA, 1);
 UFS_DEVICE_DESC_PARAM(psa_max_data_size, _PSA_MAX_DATA, 4);
 UFS_DEVICE_DESC_PARAM(psa_state_timeout, _PSA_TMT, 1);
+UFS_DEVICE_DESC_PARAM(ext_feature_sup, _EXT_UFS_FEATURE_SUP, 4);
+UFS_DEVICE_DESC_PARAM(wb_presv_us_en, _WB_PRESRV_USRSPC_EN, 1);
+UFS_DEVICE_DESC_PARAM(wb_type, _WB_TYPE, 1);
+UFS_DEVICE_DESC_PARAM(wb_shared_alloc_units, _WB_SHARED_ALLOC_UNITS, 4);
 
 static struct attribute *ufs_sysfs_device_descriptor[] = {
 	&dev_attr_device_type.attr,
@@ -304,6 +308,10 @@ static struct attribute *ufs_sysfs_device_descriptor[] = {
 	&dev_attr_number_of_secure_wpa.attr,
 	&dev_attr_psa_max_data_size.attr,
 	&dev_attr_psa_state_timeout.attr,
+	&dev_attr_ext_feature_sup.attr,
+	&dev_attr_wb_presv_us_en.attr,
+	&dev_attr_wb_type.attr,
+	&dev_attr_wb_shared_alloc_units.attr,
 	NULL,
 };
 
@@ -373,6 +381,12 @@ UFS_GEOMETRY_DESC_PARAM(enh4_memory_max_alloc_units,
 	_ENM4_MAX_NUM_UNITS, 4);
 UFS_GEOMETRY_DESC_PARAM(enh4_memory_capacity_adjustment_factor,
 	_ENM4_CAP_ADJ_FCTR, 2);
+UFS_GEOMETRY_DESC_PARAM(wb_max_alloc_units, _WB_MAX_ALLOC_UNITS, 4);
+UFS_GEOMETRY_DESC_PARAM(wb_max_wb_luns, _WB_MAX_WB_LUNS, 1);
+UFS_GEOMETRY_DESC_PARAM(wb_buff_cap_adj, _WB_BUFF_CAP_ADJ, 1);
+UFS_GEOMETRY_DESC_PARAM(wb_sup_red_type, _WB_SUP_RED_TYPE, 1);
+UFS_GEOMETRY_DESC_PARAM(wb_sup_wb_type, _WB_SUP_WB_TYPE, 1);
+
 
 static struct attribute *ufs_sysfs_geometry_descriptor[] = {
 	&dev_attr_raw_device_capacity.attr,
@@ -404,6 +418,11 @@ static struct attribute *ufs_sysfs_geometry_descriptor[] = {
 	&dev_attr_enh3_memory_capacity_adjustment_factor.attr,
 	&dev_attr_enh4_memory_max_alloc_units.attr,
 	&dev_attr_enh4_memory_capacity_adjustment_factor.attr,
+	&dev_attr_wb_max_alloc_units.attr,
+	&dev_attr_wb_max_wb_luns.attr,
+	&dev_attr_wb_buff_cap_adj.attr,
+	&dev_attr_wb_sup_red_type.attr,
+	&dev_attr_wb_sup_wb_type.attr,
 	NULL,
 };
 
@@ -603,20 +622,29 @@ static const struct attribute_group ufs_sysfs_string_descriptors_group = {
 	.attrs = ufs_sysfs_string_descriptors,
 };
 
+static inline bool ufshcd_is_wb_flags(enum flag_idn idn)
+{
+	return ((idn >= QUERY_FLAG_IDN_WB_EN) &&
+		(idn <= QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8));
+}
+
 #define UFS_FLAG(_name, _uname)						\
 static ssize_t _name##_show(struct device *dev,				\
 	struct device_attribute *attr, char *buf)			\
 {									\
 	bool flag;							\
+	u8 index = 0;							\
 	int ret;							\
 	struct ufs_hba *hba = dev_get_drvdata(dev);			\
+	if (ufshcd_is_wb_flags(QUERY_FLAG_IDN##_uname))			\
+		index = ufshcd_wb_get_query_index(hba);			\
 	pm_runtime_get_sync(hba->dev);					\
 	ret = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,	\
-		QUERY_FLAG_IDN##_uname, &flag);				\
+		QUERY_FLAG_IDN##_uname, index, &flag);			\
 	pm_runtime_put_sync(hba->dev);					\
 	if (ret)							\
 		return -EINVAL;						\
-	return sprintf(buf, "%s\n", flag ? "true" : "false");		\
+	return sprintf(buf, "%s\n", flag ? "true" : "false"); \
 }									\
 static DEVICE_ATTR_RO(_name)
 
@@ -628,6 +656,9 @@ UFS_FLAG(life_span_mode_enable, _LIFE_SPAN_MODE_ENABLE);
 UFS_FLAG(phy_resource_removal, _FPHYRESOURCEREMOVAL);
 UFS_FLAG(busy_rtc, _BUSY_RTC);
 UFS_FLAG(disable_fw_update, _PERMANENTLY_DISABLE_FW_UPDATE);
+UFS_FLAG(wb_enable, _WB_EN);
+UFS_FLAG(wb_flush_en, _WB_BUFF_FLUSH_EN);
+UFS_FLAG(wb_flush_during_h8, _WB_BUFF_FLUSH_DURING_HIBERN8);
 
 static struct attribute *ufs_sysfs_device_flags[] = {
 	&dev_attr_device_init.attr,
@@ -638,6 +669,9 @@ static struct attribute *ufs_sysfs_device_flags[] = {
 	&dev_attr_phy_resource_removal.attr,
 	&dev_attr_busy_rtc.attr,
 	&dev_attr_disable_fw_update.attr,
+	&dev_attr_wb_enable.attr,
+	&dev_attr_wb_flush_en.attr,
+	&dev_attr_wb_flush_during_h8.attr,
 	NULL,
 };
 
@@ -646,6 +680,12 @@ static const struct attribute_group ufs_sysfs_flags_group = {
 	.attrs = ufs_sysfs_device_flags,
 };
 
+static inline bool ufshcd_is_wb_attrs(enum attr_idn idn)
+{
+	return ((idn >= QUERY_ATTR_IDN_WB_FLUSH_STATUS) &&
+		(idn <= QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE));
+}
+
 #define UFS_ATTRIBUTE(_name, _uname)					\
 static ssize_t _name##_show(struct device *dev,				\
 	struct device_attribute *attr, char *buf)			\
@@ -653,9 +693,12 @@ static ssize_t _name##_show(struct device *dev,				\
 	struct ufs_hba *hba = dev_get_drvdata(dev);			\
 	u32 value;							\
 	int ret;							\
+	u8 index = 0;							\
+	if (ufshcd_is_wb_attrs(QUERY_ATTR_IDN##_uname))			\
+		index = ufshcd_wb_get_query_index(hba);			\
 	pm_runtime_get_sync(hba->dev);					\
 	ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,	\
-		QUERY_ATTR_IDN##_uname, 0, 0, &value);			\
+		QUERY_ATTR_IDN##_uname, index, 0, &value);		\
 	pm_runtime_put_sync(hba->dev);					\
 	if (ret)							\
 		return -EINVAL;						\
@@ -679,6 +722,11 @@ UFS_ATTRIBUTE(exception_event_status, _EE_STATUS);
 UFS_ATTRIBUTE(ffu_status, _FFU_STATUS);
 UFS_ATTRIBUTE(psa_state, _PSA_STATE);
 UFS_ATTRIBUTE(psa_data_size, _PSA_DATA_SIZE);
+UFS_ATTRIBUTE(wb_flush_status, _WB_FLUSH_STATUS);
+UFS_ATTRIBUTE(wb_avail_buf, _AVAIL_WB_BUFF_SIZE);
+UFS_ATTRIBUTE(wb_life_time_est, _WB_BUFF_LIFE_TIME_EST);
+UFS_ATTRIBUTE(wb_cur_buf, _CURR_WB_BUFF_SIZE);
+
 
 static struct attribute *ufs_sysfs_attributes[] = {
 	&dev_attr_boot_lun_enabled.attr,
@@ -697,6 +745,10 @@ static struct attribute *ufs_sysfs_attributes[] = {
 	&dev_attr_ffu_status.attr,
 	&dev_attr_psa_state.attr,
 	&dev_attr_psa_data_size.attr,
+	&dev_attr_wb_flush_status.attr,
+	&dev_attr_wb_avail_buf.attr,
+	&dev_attr_wb_life_time_est.attr,
+	&dev_attr_wb_cur_buf.attr,
 	NULL,
 };
 
@@ -748,6 +800,8 @@ UFS_UNIT_DESC_PARAM(provisioning_type, _PROVISIONING_TYPE, 1);
 UFS_UNIT_DESC_PARAM(physical_memory_resourse_count, _PHY_MEM_RSRC_CNT, 8);
 UFS_UNIT_DESC_PARAM(context_capabilities, _CTX_CAPABILITIES, 2);
 UFS_UNIT_DESC_PARAM(large_unit_granularity, _LARGE_UNIT_SIZE_M1, 1);
+UFS_UNIT_DESC_PARAM(wb_buf_alloc_units, _WB_BUF_ALLOC_UNITS, 4);
+
 
 static struct attribute *ufs_sysfs_unit_descriptor[] = {
 	&dev_attr_boot_lun_id.attr,
@@ -763,6 +817,7 @@ static struct attribute *ufs_sysfs_unit_descriptor[] = {
 	&dev_attr_physical_memory_resourse_count.attr,
 	&dev_attr_context_capabilities.attr,
 	&dev_attr_large_unit_granularity.attr,
+	&dev_attr_wb_buf_alloc_units.attr,
 	NULL,
 };
 
diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h
index 990cb48e2403..c70845d41449 100644
--- a/drivers/scsi/ufs/ufs.h
+++ b/drivers/scsi/ufs/ufs.h
@@ -64,6 +64,9 @@
 #define UFS_MAX_LUNS		(SCSI_W_LUN_BASE + UFS_UPIU_MAX_UNIT_NUM_ID)
 #define UFS_UPIU_WLUN_ID	(1 << 7)
 
+/* WriteBooster buffer is available only for the logical unit from 0 to 7 */
+#define UFS_UPIU_MAX_WB_LUN_ID	8
+
 /* Well known logical unit id in LUN field of UPIU */
 enum {
 	UFS_UPIU_REPORT_LUNS_WLUN	= 0x81,
@@ -140,6 +143,9 @@ enum flag_idn {
 	QUERY_FLAG_IDN_BUSY_RTC				= 0x09,
 	QUERY_FLAG_IDN_RESERVED3			= 0x0A,
 	QUERY_FLAG_IDN_PERMANENTLY_DISABLE_FW_UPDATE	= 0x0B,
+	QUERY_FLAG_IDN_WB_EN                            = 0x0E,
+	QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN                 = 0x0F,
+	QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8     = 0x10,
 };
 
 /* Attribute idn for Query requests */
@@ -168,6 +174,10 @@ enum attr_idn {
 	QUERY_ATTR_IDN_PSA_STATE		= 0x15,
 	QUERY_ATTR_IDN_PSA_DATA_SIZE		= 0x16,
 	QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME	= 0x17,
+	QUERY_ATTR_IDN_WB_FLUSH_STATUS	        = 0x1C,
+	QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE       = 0x1D,
+	QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST    = 0x1E,
+	QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE        = 0x1F,
 };
 
 /* Descriptor idn for Query requests */
@@ -191,9 +201,9 @@ enum desc_header_offset {
 };
 
 enum ufs_desc_def_size {
-	QUERY_DESC_DEVICE_DEF_SIZE		= 0x40,
+	QUERY_DESC_DEVICE_DEF_SIZE		= 0x59,
 	QUERY_DESC_CONFIGURATION_DEF_SIZE	= 0x90,
-	QUERY_DESC_UNIT_DEF_SIZE		= 0x23,
+	QUERY_DESC_UNIT_DEF_SIZE		= 0x2D,
 	QUERY_DESC_INTERCONNECT_DEF_SIZE	= 0x06,
 	QUERY_DESC_GEOMETRY_DEF_SIZE		= 0x48,
 	QUERY_DESC_POWER_DEF_SIZE		= 0x62,
@@ -219,6 +229,7 @@ enum unit_desc_param {
 	UNIT_DESC_PARAM_PHY_MEM_RSRC_CNT	= 0x18,
 	UNIT_DESC_PARAM_CTX_CAPABILITIES	= 0x20,
 	UNIT_DESC_PARAM_LARGE_UNIT_SIZE_M1	= 0x22,
+	UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS	= 0x29,
 };
 
 /* Device descriptor parameters offsets in bytes*/
@@ -258,6 +269,10 @@ enum device_desc_param {
 	DEVICE_DESC_PARAM_PSA_MAX_DATA		= 0x25,
 	DEVICE_DESC_PARAM_PSA_TMT		= 0x29,
 	DEVICE_DESC_PARAM_PRDCT_REV		= 0x2A,
+	DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP	= 0x4F,
+	DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN	= 0x53,
+	DEVICE_DESC_PARAM_WB_TYPE		= 0x54,
+	DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS = 0x55,
 };
 
 /* Interconnect descriptor parameters offsets in bytes*/
@@ -302,6 +317,11 @@ enum geometry_desc_param {
 	GEOMETRY_DESC_PARAM_ENM4_MAX_NUM_UNITS	= 0x3E,
 	GEOMETRY_DESC_PARAM_ENM4_CAP_ADJ_FCTR	= 0x42,
 	GEOMETRY_DESC_PARAM_OPT_LOG_BLK_SIZE	= 0x44,
+	GEOMETRY_DESC_PARAM_WB_MAX_ALLOC_UNITS	= 0x4F,
+	GEOMETRY_DESC_PARAM_WB_MAX_WB_LUNS	= 0x53,
+	GEOMETRY_DESC_PARAM_WB_BUFF_CAP_ADJ	= 0x54,
+	GEOMETRY_DESC_PARAM_WB_SUP_RED_TYPE	= 0x55,
+	GEOMETRY_DESC_PARAM_WB_SUP_WB_TYPE	= 0x56,
 };
 
 /* Health descriptor parameters offsets in bytes*/
@@ -313,6 +333,12 @@ enum health_desc_param {
 	HEALTH_DESC_PARAM_LIFE_TIME_EST_B	= 0x4,
 };
 
+/* WriteBooster buffer mode */
+enum {
+	WB_BUF_MODE_LU_DEDICATED	= 0x0,
+	WB_BUF_MODE_SHARED		= 0x1,
+};
+
 /*
  * Logical Unit Write Protect
  * 00h: LU not write protected
@@ -333,6 +359,11 @@ enum {
 	UFSHCD_AMP		= 3,
 };
 
+/* Possible values for dExtendedUFSFeaturesSupport */
+enum {
+	UFS_DEV_WRITE_BOOSTER_SUP	= BIT(8),
+};
+
 #define POWER_DESC_MAX_SIZE			0x62
 #define POWER_DESC_MAX_ACTV_ICC_LVLS		16
 
@@ -447,6 +478,8 @@ enum ufs_dev_pwr_mode {
 	UFS_POWERDOWN_PWR_MODE	= 3,
 };
 
+#define UFS_WB_BUF_REMAIN_PERCENT(val) ((val) / 10)
+
 /**
  * struct utp_cmd_rsp - Response UPIU structure
  * @residual_transfer_count: Residual transfer count DW-3
@@ -532,11 +565,17 @@ struct ufs_dev_info {
 	bool is_lu_power_on_wp;
 	/* Maximum number of general LU supported by the UFS device */
 	u8 max_lu_supported;
+	u8 wb_dedicated_lu;
 	u16 wmanufacturerid;
 	/*UFS device Product Name */
 	u8 *model;
 	u16 wspecversion;
 	u32 clk_gating_wait_us;
+	u32 d_ext_ufs_feature_sup;
+	u8 b_wb_buffer_type;
+	u32 d_wb_alloc_units;
+	bool b_rpm_dev_flush_capable;
+	u8 b_presrv_uspc_en;
 };
 
 /**
diff --git a/drivers/scsi/ufs/ufs_quirks.h b/drivers/scsi/ufs/ufs_quirks.h
index df7a1e6805a3..e3175a63c676 100644
--- a/drivers/scsi/ufs/ufs_quirks.h
+++ b/drivers/scsi/ufs/ufs_quirks.h
@@ -101,4 +101,11 @@ struct ufs_dev_fix {
  */
 #define UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME	(1 << 9)
 
+/*
+ * Some pre-3.1 UFS devices can support extended features by upgrading
+ * the firmware. Enable this quirk to make UFS core driver probe and enable
+ * supported features on such devices.
+ */
+#define UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES (1 << 10)
+
 #endif /* UFS_QUIRKS_H_ */
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index 698e8d20b4ba..ad4fc829cbb2 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -48,6 +48,8 @@
 #include "unipro.h"
 #include "ufs-sysfs.h"
 #include "ufs_bsg.h"
+#include <asm/unaligned.h>
+#include <linux/blkdev.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/ufs.h>
@@ -92,6 +94,9 @@
 /* default delay of autosuspend: 2000 ms */
 #define RPM_AUTOSUSPEND_DELAY_MS 2000
 
+/* Default delay of RPM device flush delayed work */
+#define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
+
 /* Default value of wait time before gating device ref clock */
 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
 
@@ -251,6 +256,12 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 			     struct ufs_pa_layer_attr *pwr_mode);
+static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba);
+static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba);
+static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable);
+static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
+static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
+
 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
 {
 	return tag >= 0 && tag < hba->nutrs;
@@ -272,6 +283,25 @@ static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 	}
 }
 
+static inline void ufshcd_wb_config(struct ufs_hba *hba)
+{
+	int ret;
+
+	if (!ufshcd_is_wb_allowed(hba))
+		return;
+
+	ret = ufshcd_wb_ctrl(hba, true);
+	if (ret)
+		dev_err(hba->dev, "%s: Enable WB failed: %d\n", __func__, ret);
+	else
+		dev_info(hba->dev, "%s: Write Booster Configured\n", __func__);
+	ret = ufshcd_wb_toggle_flush_during_h8(hba, true);
+	if (ret)
+		dev_err(hba->dev, "%s: En WB flush during H8: failed: %d\n",
+			__func__, ret);
+	ufshcd_wb_toggle_flush(hba, true);
+}
+
 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
 {
 	if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
@@ -535,21 +565,21 @@ void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
 }
 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
 
-/*
+/**
  * ufshcd_wait_for_register - wait for register value to change
- * @hba - per-adapter interface
- * @reg - mmio register offset
- * @mask - mask to apply to read register value
- * @val - wait condition
- * @interval_us - polling interval in microsecs
- * @timeout_ms - timeout in millisecs
- * @can_sleep - perform sleep or just spin
+ * @hba: per-adapter interface
+ * @reg: mmio register offset
+ * @mask: mask to apply to the read register value
+ * @val: value to wait for
+ * @interval_us: polling interval in microseconds
+ * @timeout_ms: timeout in milliseconds
  *
- * Returns -ETIMEDOUT on error, zero on success
+ * Return:
+ * -ETIMEDOUT on error, zero on success.
  */
 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 				u32 val, unsigned long interval_us,
-				unsigned long timeout_ms, bool can_sleep)
+				unsigned long timeout_ms)
 {
 	int err = 0;
 	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
@@ -558,10 +588,7 @@ int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 	val = val & mask;
 
 	while ((ufshcd_readl(hba, reg) & mask) != val) {
-		if (can_sleep)
-			usleep_range(interval_us, interval_us + 50);
-		else
-			udelay(interval_us);
+		usleep_range(interval_us, interval_us + 50);
 		if (time_after(jiffies, timeout)) {
 			if ((ufshcd_readl(hba, reg) & mask) != val)
 				err = -ETIMEDOUT;
@@ -1150,10 +1177,17 @@ static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
 	/* scale up the gear after scaling up clocks */
 	if (scale_up) {
 		ret = ufshcd_scale_gear(hba, true);
-		if (ret)
+		if (ret) {
 			ufshcd_scale_clks(hba, false);
+			goto out_unprepare;
+		}
 	}
 
+	/* Enable Write Booster if we have scaled up else disable it */
+	up_write(&hba->clk_scaling_lock);
+	ufshcd_wb_ctrl(hba, scale_up);
+	down_write(&hba->clk_scaling_lock);
+
 out_unprepare:
 	ufshcd_clock_scaling_unprepare(hba);
 out:
@@ -1319,23 +1353,6 @@ start_window:
 	return 0;
 }
 
-static struct devfreq_dev_profile ufs_devfreq_profile = {
-	.polling_ms	= 100,
-	.target		= ufshcd_devfreq_target,
-	.get_dev_status	= ufshcd_devfreq_get_dev_status,
-};
-
-#if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND)
-static struct devfreq_simple_ondemand_data ufs_ondemand_data = {
-	.upthreshold = 70,
-	.downdifferential = 5,
-};
-
-static void *gov_data = &ufs_ondemand_data;
-#else
-static void *gov_data; /* NULL */
-#endif
-
 static int ufshcd_devfreq_init(struct ufs_hba *hba)
 {
 	struct list_head *clk_list = &hba->clk_list_head;
@@ -1351,12 +1368,12 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba)
 	dev_pm_opp_add(hba->dev, clki->min_freq, 0);
 	dev_pm_opp_add(hba->dev, clki->max_freq, 0);
 
-	ufshcd_vops_config_scaling_param(hba, &ufs_devfreq_profile,
-					 gov_data);
+	ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
+					 &hba->vps->ondemand_data);
 	devfreq = devfreq_add_device(hba->dev,
-			&ufs_devfreq_profile,
+			&hba->vps->devfreq_profile,
 			DEVFREQ_GOV_SIMPLE_ONDEMAND,
-			gov_data);
+			&hba->vps->ondemand_data);
 	if (IS_ERR(devfreq)) {
 		ret = PTR_ERR(devfreq);
 		dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
@@ -2560,7 +2577,7 @@ ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
 	 */
 	err = ufshcd_wait_for_register(hba,
 			REG_UTP_TRANSFER_REQ_DOOR_BELL,
-			mask, ~mask, 1000, 1000, true);
+			mask, ~mask, 1000, 1000);
 
 	return err;
 }
@@ -2747,13 +2764,13 @@ static inline void ufshcd_init_query(struct ufs_hba *hba,
 }
 
 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
-	enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
+	enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
 {
 	int ret;
 	int retries;
 
 	for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
-		ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
+		ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
 		if (ret)
 			dev_dbg(hba->dev,
 				"%s: failed with error %d, retries %d\n",
@@ -2774,16 +2791,17 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba,
  * @hba: per-adapter instance
  * @opcode: flag query to perform
  * @idn: flag idn to access
+ * @index: flag index to access
  * @flag_res: the flag value after the query request completes
  *
  * Returns 0 for success, non-zero in case of failure
  */
 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
-			enum flag_idn idn, bool *flag_res)
+			enum flag_idn idn, u8 index, bool *flag_res)
 {
 	struct ufs_query_req *request = NULL;
 	struct ufs_query_res *response = NULL;
-	int err, index = 0, selector = 0;
+	int err, selector = 0;
 	int timeout = QUERY_REQ_TIMEOUT;
 
 	BUG_ON(!hba);
@@ -3223,7 +3241,7 @@ static inline int ufshcd_read_desc(struct ufs_hba *hba,
 struct uc_string_id {
 	u8 len;
 	u8 type;
-	wchar_t uc[0];
+	wchar_t uc[];
 } __packed;
 
 /* replace non-printable or non-ASCII characters with spaces */
@@ -4137,10 +4155,10 @@ static int ufshcd_complete_dev_init(struct ufs_hba *hba)
 {
 	int i;
 	int err;
-	bool flag_res = 1;
+	bool flag_res = true;
 
 	err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
-		QUERY_FLAG_IDN_FDEVICEINIT, NULL);
+		QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
 	if (err) {
 		dev_err(hba->dev,
 			"%s setting fDeviceInit flag failed with error %d\n",
@@ -4151,7 +4169,7 @@ static int ufshcd_complete_dev_init(struct ufs_hba *hba)
 	/* poll for max. 1000 iterations for fDeviceInit flag to clear */
 	for (i = 0; i < 1000 && !err && flag_res; i++)
 		err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
-			QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
+			QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
 
 	if (err)
 		dev_err(hba->dev,
@@ -4229,16 +4247,23 @@ EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
 /**
  * ufshcd_hba_stop - Send controller to reset state
  * @hba: per adapter instance
- * @can_sleep: perform sleep or just spin
  */
-static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
+static inline void ufshcd_hba_stop(struct ufs_hba *hba)
 {
+	unsigned long flags;
 	int err;
 
+	/*
+	 * Obtain the host lock to prevent that the controller is disabled
+	 * while the UFS interrupt handler is active on another CPU.
+	 */
+	spin_lock_irqsave(hba->host->host_lock, flags);
 	ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
+	spin_unlock_irqrestore(hba->host->host_lock, flags);
+
 	err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
 					CONTROLLER_ENABLE, CONTROLLER_DISABLE,
-					10, 1, can_sleep);
+					10, 1);
 	if (err)
 		dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
 }
@@ -4259,7 +4284,7 @@ int ufshcd_hba_enable(struct ufs_hba *hba)
 
 	if (!ufshcd_is_hba_active(hba))
 		/* change controller state to "reset state" */
-		ufshcd_hba_stop(hba, true);
+		ufshcd_hba_stop(hba);
 
 	/* UniPro link is disabled at this point */
 	ufshcd_set_link_off(hba);
@@ -4279,7 +4304,7 @@ int ufshcd_hba_enable(struct ufs_hba *hba)
 	 * instruction might be read back.
 	 * This delay can be changed based on the controller.
 	 */
-	ufshcd_delay_us(hba->hba_enable_delay_us, 100);
+	ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
 
 	/* wait for the host controller to complete initialization */
 	retry = 50;
@@ -4966,7 +4991,7 @@ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
 		goto out;
 
 	err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
-			QUERY_FLAG_IDN_BKOPS_EN, NULL);
+			QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
 	if (err) {
 		dev_err(hba->dev, "%s: failed to enable bkops %d\n",
 				__func__, err);
@@ -5016,7 +5041,7 @@ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
 	}
 
 	err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
-			QUERY_FLAG_IDN_BKOPS_EN, NULL);
+			QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
 	if (err) {
 		dev_err(hba->dev, "%s: failed to disable bkops %d\n",
 				__func__, err);
@@ -5051,6 +5076,7 @@ static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
 		hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
 		ufshcd_disable_auto_bkops(hba);
 	}
+	hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
 	hba->is_urgent_bkops_lvl_checked = false;
 }
 
@@ -5098,7 +5124,6 @@ static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
 		err = ufshcd_enable_auto_bkops(hba);
 	else
 		err = ufshcd_disable_auto_bkops(hba);
-	hba->urgent_bkops_lvl = curr_status;
 out:
 	return err;
 }
@@ -5161,6 +5186,190 @@ out:
 				__func__, err);
 }
 
+static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable)
+{
+	int ret;
+	u8 index;
+	enum query_opcode opcode;
+
+	if (!ufshcd_is_wb_allowed(hba))
+		return 0;
+
+	if (!(enable ^ hba->wb_enabled))
+		return 0;
+	if (enable)
+		opcode = UPIU_QUERY_OPCODE_SET_FLAG;
+	else
+		opcode = UPIU_QUERY_OPCODE_CLEAR_FLAG;
+
+	index = ufshcd_wb_get_query_index(hba);
+	ret = ufshcd_query_flag_retry(hba, opcode,
+				      QUERY_FLAG_IDN_WB_EN, index, NULL);
+	if (ret) {
+		dev_err(hba->dev, "%s write booster %s failed %d\n",
+			__func__, enable ? "enable" : "disable", ret);
+		return ret;
+	}
+
+	hba->wb_enabled = enable;
+	dev_dbg(hba->dev, "%s write booster %s %d\n",
+			__func__, enable ? "enable" : "disable", ret);
+
+	return ret;
+}
+
+static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
+{
+	int val;
+	u8 index;
+
+	if (set)
+		val =  UPIU_QUERY_OPCODE_SET_FLAG;
+	else
+		val = UPIU_QUERY_OPCODE_CLEAR_FLAG;
+
+	index = ufshcd_wb_get_query_index(hba);
+	return ufshcd_query_flag_retry(hba, val,
+				QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8,
+				index, NULL);
+}
+
+static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
+{
+	if (enable)
+		ufshcd_wb_buf_flush_enable(hba);
+	else
+		ufshcd_wb_buf_flush_disable(hba);
+
+}
+
+static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba)
+{
+	int ret;
+	u8 index;
+
+	if (!ufshcd_is_wb_allowed(hba) || hba->wb_buf_flush_enabled)
+		return 0;
+
+	index = ufshcd_wb_get_query_index(hba);
+	ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
+				      QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
+				      index, NULL);
+	if (ret)
+		dev_err(hba->dev, "%s WB - buf flush enable failed %d\n",
+			__func__, ret);
+	else
+		hba->wb_buf_flush_enabled = true;
+
+	dev_dbg(hba->dev, "WB - Flush enabled: %d\n", ret);
+	return ret;
+}
+
+static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba)
+{
+	int ret;
+	u8 index;
+
+	if (!ufshcd_is_wb_allowed(hba) || !hba->wb_buf_flush_enabled)
+		return 0;
+
+	index = ufshcd_wb_get_query_index(hba);
+	ret = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
+				      QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN,
+				      index, NULL);
+	if (ret) {
+		dev_warn(hba->dev, "%s: WB - buf flush disable failed %d\n",
+			 __func__, ret);
+	} else {
+		hba->wb_buf_flush_enabled = false;
+		dev_dbg(hba->dev, "WB - Flush disabled: %d\n", ret);
+	}
+
+	return ret;
+}
+
+static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
+						u32 avail_buf)
+{
+	u32 cur_buf;
+	int ret;
+	u8 index;
+
+	index = ufshcd_wb_get_query_index(hba);
+	ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
+					      QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
+					      index, 0, &cur_buf);
+	if (ret) {
+		dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
+			__func__, ret);
+		return false;
+	}
+
+	if (!cur_buf) {
+		dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
+			 cur_buf);
+		return false;
+	}
+	/* Let it continue to flush when available buffer exceeds threshold */
+	if (avail_buf < hba->vps->wb_flush_threshold)
+		return true;
+
+	return false;
+}
+
+static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
+{
+	int ret;
+	u32 avail_buf;
+	u8 index;
+
+	if (!ufshcd_is_wb_allowed(hba))
+		return false;
+	/*
+	 * The ufs device needs the vcc to be ON to flush.
+	 * With user-space reduction enabled, it's enough to enable flush
+	 * by checking only the available buffer. The threshold
+	 * defined here is > 90% full.
+	 * With user-space preserved enabled, the current-buffer
+	 * should be checked too because the wb buffer size can reduce
+	 * when disk tends to be full. This info is provided by current
+	 * buffer (dCurrentWriteBoosterBufferSize). There's no point in
+	 * keeping vcc on when current buffer is empty.
+	 */
+	index = ufshcd_wb_get_query_index(hba);
+	ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
+				      QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
+				      index, 0, &avail_buf);
+	if (ret) {
+		dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
+			 __func__, ret);
+		return false;
+	}
+
+	if (!hba->dev_info.b_presrv_uspc_en) {
+		if (avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10))
+			return true;
+		return false;
+	}
+
+	return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
+}
+
+static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
+{
+	struct ufs_hba *hba = container_of(to_delayed_work(work),
+					   struct ufs_hba,
+					   rpm_dev_flush_recheck_work);
+	/*
+	 * To prevent unnecessary VCC power drain after device finishes
+	 * WriteBooster buffer flush or Auto BKOPs, force runtime resume
+	 * after a certain delay to recheck the threshold by next runtime
+	 * suspend.
+	 */
+	pm_runtime_get_sync(hba->dev);
+	pm_runtime_put_sync(hba->dev);
+}
+
 /**
  * ufshcd_exception_event_handler - handle exceptions raised by device
  * @work: pointer to work data
@@ -5723,7 +5932,7 @@ static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
 	/* poll for max. 1 sec to clear door bell register by h/w */
 	err = ufshcd_wait_for_register(hba,
 			REG_UTP_TASK_REQ_DOOR_BELL,
-			mask, 0, 1000, 1000, true);
+			mask, 0, 1000, 1000);
 out:
 	return err;
 }
@@ -6299,8 +6508,9 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
 	 * Stop the host controller and complete the requests
 	 * cleared by h/w
 	 */
+	ufshcd_hba_stop(hba);
+
 	spin_lock_irqsave(hba->host->host_lock, flags);
-	ufshcd_hba_stop(hba, false);
 	hba->silence_err_logs = true;
 	ufshcd_complete_requests(hba);
 	hba->silence_err_logs = false;
@@ -6603,6 +6813,93 @@ out:
 	return ret;
 }
 
+static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
+{
+	u8 lun;
+	u32 d_lu_wb_buf_alloc;
+
+	if (!ufshcd_is_wb_allowed(hba))
+		return;
+
+	if (hba->desc_size.dev_desc < DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
+		goto wb_disabled;
+
+	hba->dev_info.d_ext_ufs_feature_sup =
+		get_unaligned_be32(desc_buf +
+				   DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
+
+	if (!(hba->dev_info.d_ext_ufs_feature_sup & UFS_DEV_WRITE_BOOSTER_SUP))
+		goto wb_disabled;
+
+	/*
+	 * WB may be supported but not configured while provisioning.
+	 * The spec says, in dedicated wb buffer mode,
+	 * a max of 1 lun would have wb buffer configured.
+	 * Now only shared buffer mode is supported.
+	 */
+	hba->dev_info.b_wb_buffer_type =
+		desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
+
+	hba->dev_info.b_presrv_uspc_en =
+		desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
+
+	if (hba->dev_info.b_wb_buffer_type == WB_BUF_MODE_SHARED) {
+		hba->dev_info.d_wb_alloc_units =
+		get_unaligned_be32(desc_buf +
+				   DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS);
+		if (!hba->dev_info.d_wb_alloc_units)
+			goto wb_disabled;
+	} else {
+		for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
+			d_lu_wb_buf_alloc = 0;
+			ufshcd_read_unit_desc_param(hba,
+					lun,
+					UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
+					(u8 *)&d_lu_wb_buf_alloc,
+					sizeof(d_lu_wb_buf_alloc));
+			if (d_lu_wb_buf_alloc) {
+				hba->dev_info.wb_dedicated_lu = lun;
+				break;
+			}
+		}
+
+		if (!d_lu_wb_buf_alloc)
+			goto wb_disabled;
+	}
+	return;
+
+wb_disabled:
+	hba->caps &= ~UFSHCD_CAP_WB_EN;
+}
+
+void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups)
+{
+	struct ufs_dev_fix *f;
+	struct ufs_dev_info *dev_info = &hba->dev_info;
+
+	if (!fixups)
+		return;
+
+	for (f = fixups; f->quirk; f++) {
+		if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
+		     f->wmanufacturerid == UFS_ANY_VENDOR) &&
+		     ((dev_info->model &&
+		       STR_PRFX_EQUAL(f->model, dev_info->model)) ||
+		      !strcmp(f->model, UFS_ANY_MODEL)))
+			hba->dev_quirks |= f->quirk;
+	}
+}
+EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
+
+static void ufs_fixup_device_setup(struct ufs_hba *hba)
+{
+	/* fix by general quirk table */
+	ufshcd_fixup_dev_quirks(hba, ufs_fixups);
+
+	/* allow vendors to fix quirks */
+	ufshcd_vops_fixup_dev_quirks(hba);
+}
+
 static int ufs_get_device_desc(struct ufs_hba *hba)
 {
 	int err;
@@ -6639,6 +6936,7 @@ static int ufs_get_device_desc(struct ufs_hba *hba)
 				      desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
 
 	model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
+
 	err = ufshcd_read_string_desc(hba, model_index,
 				      &dev_info->model, SD_ASCII_STD);
 	if (err < 0) {
@@ -6647,6 +6945,17 @@ static int ufs_get_device_desc(struct ufs_hba *hba)
 		goto out;
 	}
 
+	ufs_fixup_device_setup(hba);
+
+	/*
+	 * Probe WB only for UFS-3.1 devices or UFS devices with quirk
+	 * UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES enabled
+	 */
+	if (dev_info->wspecversion >= 0x310 ||
+	    dev_info->wspecversion == 0x220 ||
+	    (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))
+		ufshcd_wb_probe(hba, desc_buf);
+
 	/*
 	 * ufshcd_read_string_desc returns size of the string
 	 * reset the error value
@@ -6666,21 +6975,6 @@ static void ufs_put_device_desc(struct ufs_hba *hba)
 	dev_info->model = NULL;
 }
 
-static void ufs_fixup_device_setup(struct ufs_hba *hba)
-{
-	struct ufs_dev_fix *f;
-	struct ufs_dev_info *dev_info = &hba->dev_info;
-
-	for (f = ufs_fixups; f->quirk; f++) {
-		if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
-		     f->wmanufacturerid == UFS_ANY_VENDOR) &&
-		     ((dev_info->model &&
-		       STR_PRFX_EQUAL(f->model, dev_info->model)) ||
-		      !strcmp(f->model, UFS_ANY_MODEL)))
-			hba->dev_quirks |= f->quirk;
-	}
-}
-
 /**
  * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
  * @hba: per-adapter instance
@@ -6996,9 +7290,6 @@ static int ufshcd_device_params_init(struct ufs_hba *hba)
 	bool flag;
 	int ret;
 
-	/* Clear any previous UFS device information */
-	memset(&hba->dev_info, 0, sizeof(hba->dev_info));
-
 	/* Init check for device descriptor sizes */
 	ufshcd_init_desc_sizes(hba);
 
@@ -7017,10 +7308,8 @@ static int ufshcd_device_params_init(struct ufs_hba *hba)
 
 	ufshcd_get_ref_clk_gating_wait(hba);
 
-	ufs_fixup_device_setup(hba);
-
 	if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
-			QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
+			QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
 		hba->dev_info.f_power_on_wp_en = flag;
 
 	/* Probe maximum power mode co-supported by both UFS host and device */
@@ -7084,10 +7373,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
 	if (ret)
 		goto out;
 
-	/* set the default level for urgent bkops */
-	hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
-	hba->is_urgent_bkops_lvl_checked = false;
-
 	/* Debug counters initialization */
 	ufshcd_clear_dbg_ufs_stats(hba);
 
@@ -7149,6 +7434,7 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
 	/* set the state as operational after switching to desired gear */
 	hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
 
+	ufshcd_wb_config(hba);
 	/* Enable Auto-Hibernate if configured */
 	ufshcd_auto_hibern8_enable(hba);
 
@@ -7195,6 +7481,16 @@ static const struct attribute_group *ufshcd_driver_groups[] = {
 	NULL,
 };
 
+static struct ufs_hba_variant_params ufs_hba_vps = {
+	.hba_enable_delay_us		= 1000,
+	.wb_flush_threshold		= UFS_WB_BUF_REMAIN_PERCENT(40),
+	.devfreq_profile.polling_ms	= 100,
+	.devfreq_profile.target		= ufshcd_devfreq_target,
+	.devfreq_profile.get_dev_status	= ufshcd_devfreq_get_dev_status,
+	.ondemand_data.upthreshold	= 70,
+	.ondemand_data.downdifferential	= 5,
+};
+
 static struct scsi_host_template ufshcd_driver_template = {
 	.module			= THIS_MODULE,
 	.name			= UFSHCD,
@@ -7774,7 +8070,7 @@ static int ufshcd_link_state_transition(struct ufs_hba *hba,
 		 * Change controller state to "reset state" which
 		 * should also put the link in off/reset state
 		 */
-		ufshcd_hba_stop(hba, true);
+		ufshcd_hba_stop(hba);
 		/*
 		 * TODO: Check if we need any delay to make sure that
 		 * controller is reset
@@ -7809,6 +8105,9 @@ static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
 	 *
 	 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
 	 * in low power state which would save some power.
+	 *
+	 * If Write Booster is enabled and the device needs to flush the WB
+	 * buffer OR if bkops status is urgent for WB, keep Vcc on.
 	 */
 	if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
 	    !hba->dev_info.is_lu_power_on_wp) {
@@ -7938,16 +8237,31 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 			/* make sure that auto bkops is disabled */
 			ufshcd_disable_auto_bkops(hba);
 		}
-	}
+		/*
+		 * If device needs to do BKOP or WB buffer flush during
+		 * Hibern8, keep device power mode as "active power mode"
+		 * and VCC supply.
+		 */
+		hba->dev_info.b_rpm_dev_flush_capable =
+			hba->auto_bkops_enabled ||
+			(((req_link_state == UIC_LINK_HIBERN8_STATE) ||
+			((req_link_state == UIC_LINK_ACTIVE_STATE) &&
+			ufshcd_is_auto_hibern8_enabled(hba))) &&
+			ufshcd_wb_need_flush(hba));
+	}
+
+	if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
+		if ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
+		    !ufshcd_is_runtime_pm(pm_op)) {
+			/* ensure that bkops is disabled */
+			ufshcd_disable_auto_bkops(hba);
+		}
 
-	if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
-	     ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
-	       !ufshcd_is_runtime_pm(pm_op))) {
-		/* ensure that bkops is disabled */
-		ufshcd_disable_auto_bkops(hba);
-		ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
-		if (ret)
-			goto enable_gating;
+		if (!hba->dev_info.b_rpm_dev_flush_capable) {
+			ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
+			if (ret)
+				goto enable_gating;
+		}
 	}
 
 	flush_work(&hba->eeh_work);
@@ -8000,9 +8314,16 @@ enable_gating:
 	if (hba->clk_scaling.is_allowed)
 		ufshcd_resume_clkscaling(hba);
 	hba->clk_gating.is_suspended = false;
+	hba->dev_info.b_rpm_dev_flush_capable = false;
 	ufshcd_release(hba);
 out:
+	if (hba->dev_info.b_rpm_dev_flush_capable) {
+		schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
+			msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
+	}
+
 	hba->pm_op_in_progress = 0;
+
 	if (ret)
 		ufshcd_update_reg_hist(&hba->ufs_stats.suspend_err, (u32)ret);
 	return ret;
@@ -8055,9 +8376,13 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 		else
 			goto vendor_suspend;
 	} else if (ufshcd_is_link_off(hba)) {
-		ret = ufshcd_host_reset_and_restore(hba);
 		/*
-		 * ufshcd_host_reset_and_restore() should have already
+		 * A full initialization of the host and the device is
+		 * required since the link was put to off during suspend.
+		 */
+		ret = ufshcd_reset_and_restore(hba);
+		/*
+		 * ufshcd_reset_and_restore() should have already
 		 * set the link state as active
 		 */
 		if (ret || !ufshcd_is_link_active(hba))
@@ -8087,6 +8412,11 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 	/* Enable Auto-Hibernate if configured */
 	ufshcd_auto_hibern8_enable(hba);
 
+	if (hba->dev_info.b_rpm_dev_flush_capable) {
+		hba->dev_info.b_rpm_dev_flush_capable = false;
+		cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
+	}
+
 	/* Schedule clock gating in case of no access to UFS device yet */
 	ufshcd_release(hba);
 
@@ -8313,7 +8643,7 @@ void ufshcd_remove(struct ufs_hba *hba)
 	scsi_remove_host(hba->host);
 	/* disable interrupts */
 	ufshcd_disable_intr(hba, hba->intr_mask);
-	ufshcd_hba_stop(hba, true);
+	ufshcd_hba_stop(hba);
 
 	ufshcd_exit_clk_scaling(hba);
 	ufshcd_exit_clk_gating(hba);
@@ -8422,7 +8752,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
 
 	hba->mmio_base = mmio_base;
 	hba->irq = irq;
-	hba->hba_enable_delay_us = 1000;
+	hba->vps = &ufs_hba_vps;
 
 	err = ufshcd_hba_init(hba);
 	if (err)
@@ -8560,6 +8890,9 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
 						UFS_SLEEP_PWR_MODE,
 						UIC_LINK_HIBERN8_STATE);
 
+	INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
+			  ufshcd_rpm_dev_flush_recheck_work);
+
 	/* Set the default auto-hiberate idle timer value to 150 ms */
 	if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
 		hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
index 6ffc08ad85f6..bf97d616e597 100644
--- a/drivers/scsi/ufs/ufshcd.h
+++ b/drivers/scsi/ufs/ufshcd.h
@@ -69,6 +69,7 @@
 #include <scsi/scsi_eh.h>
 
 #include "ufs.h"
+#include "ufs_quirks.h"
 #include "ufshci.h"
 
 #define UFSHCD "ufshcd"
@@ -336,6 +337,7 @@ struct ufs_hba_variant_ops {
 	void    (*hibern8_notify)(struct ufs_hba *, enum uic_cmd_dme,
 					enum ufs_notify_change_status);
 	int	(*apply_dev_quirks)(struct ufs_hba *hba);
+	void	(*fixup_dev_quirks)(struct ufs_hba *hba);
 	int     (*suspend)(struct ufs_hba *, enum ufs_pm_op);
 	int     (*resume)(struct ufs_hba *, enum ufs_pm_op);
 	void	(*dbg_register_dump)(struct ufs_hba *hba);
@@ -555,6 +557,20 @@ enum ufshcd_caps {
 	 * for userspace to control the power management.
 	 */
 	UFSHCD_CAP_RPM_AUTOSUSPEND			= 1 << 6,
+
+	/*
+	 * This capability allows the host controller driver to turn-on
+	 * WriteBooster, if the underlying device supports it and is
+	 * provisioned to be used. This would increase the write performance.
+	 */
+	UFSHCD_CAP_WB_EN				= 1 << 7,
+};
+
+struct ufs_hba_variant_params {
+	struct devfreq_dev_profile devfreq_profile;
+	struct devfreq_simple_ondemand_data ondemand_data;
+	u16 hba_enable_delay_us;
+	u32 wb_flush_threshold;
 };
 
 /**
@@ -654,6 +670,7 @@ struct ufs_hba {
 	int nutmrs;
 	u32 ufs_version;
 	const struct ufs_hba_variant_ops *vops;
+	struct ufs_hba_variant_params *vps;
 	void *priv;
 	unsigned int irq;
 	bool is_irq_enabled;
@@ -675,7 +692,6 @@ struct ufs_hba {
 	u32 eh_flags;
 	u32 intr_mask;
 	u16 ee_ctrl_mask;
-	u16 hba_enable_delay_us;
 	bool is_powered;
 
 	/* Work Queues */
@@ -727,6 +743,9 @@ struct ufs_hba {
 
 	struct device		bsg_dev;
 	struct request_queue	*bsg_queue;
+	bool wb_buf_flush_enabled;
+	bool wb_enabled;
+	struct delayed_work rpm_dev_flush_recheck_work;
 };
 
 /* Returns true if clocks can be gated. Otherwise false */
@@ -775,6 +794,11 @@ static inline bool ufshcd_is_auto_hibern8_enabled(struct ufs_hba *hba)
 	return FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, hba->ahit) ? true : false;
 }
 
+static inline bool ufshcd_is_wb_allowed(struct ufs_hba *hba)
+{
+	return hba->caps & UFSHCD_CAP_WB_EN;
+}
+
 #define ufshcd_writel(hba, val, reg)	\
 	writel((val), (hba)->mmio_base + (reg))
 #define ufshcd_readl(hba, reg)	\
@@ -808,7 +832,7 @@ int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
 void ufshcd_delay_us(unsigned long us, unsigned long tolerance);
 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 				u32 val, unsigned long interval_us,
-				unsigned long timeout_ms, bool can_sleep);
+				unsigned long timeout_ms);
 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk);
 void ufshcd_update_reg_hist(struct ufs_err_reg_hist *reg_hist,
 			    u32 reg);
@@ -845,6 +869,13 @@ static inline bool ufshcd_keep_autobkops_enabled_except_suspend(
 	return hba->caps & UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND;
 }
 
+static inline u8 ufshcd_wb_get_query_index(struct ufs_hba *hba)
+{
+	if (hba->dev_info.b_wb_buffer_type == WB_BUF_MODE_LU_DEDICATED)
+		return hba->dev_info.wb_dedicated_lu;
+	return 0;
+}
+
 extern int ufshcd_runtime_suspend(struct ufs_hba *hba);
 extern int ufshcd_runtime_resume(struct ufs_hba *hba);
 extern int ufshcd_runtime_idle(struct ufs_hba *hba);
@@ -932,11 +963,11 @@ int ufshcd_read_desc_param(struct ufs_hba *hba,
 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
 		      enum attr_idn idn, u8 index, u8 selector, u32 *attr_val);
 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
-	enum flag_idn idn, bool *flag_res);
+	enum flag_idn idn, u8 index, bool *flag_res);
 
 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba);
 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit);
-
+void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups);
 #define SD_ASCII_STD true
 #define SD_RAW false
 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
@@ -1071,6 +1102,12 @@ static inline int ufshcd_vops_apply_dev_quirks(struct ufs_hba *hba)
 	return 0;
 }
 
+static inline void ufshcd_vops_fixup_dev_quirks(struct ufs_hba *hba)
+{
+	if (hba->vops && hba->vops->fixup_dev_quirks)
+		hba->vops->fixup_dev_quirks(hba);
+}
+
 static inline int ufshcd_vops_suspend(struct ufs_hba *hba, enum ufs_pm_op op)
 {
 	if (hba->vops && hba->vops->suspend)
diff --git a/drivers/scsi/vmw_pvscsi.c b/drivers/scsi/vmw_pvscsi.c
index c3f010df641e..8dbb4db6831a 100644
--- a/drivers/scsi/vmw_pvscsi.c
+++ b/drivers/scsi/vmw_pvscsi.c
@@ -908,7 +908,7 @@ static int pvscsi_host_reset(struct scsi_cmnd *cmd)
 	use_msg = adapter->use_msg;
 
 	if (use_msg) {
-		adapter->use_msg = 0;
+		adapter->use_msg = false;
 		spin_unlock_irqrestore(&adapter->hw_lock, flags);
 
 		/*
diff --git a/drivers/scsi/zorro_esp.c b/drivers/scsi/zorro_esp.c
index c6727bcbc2e3..928c8adf5cb3 100644
--- a/drivers/scsi/zorro_esp.c
+++ b/drivers/scsi/zorro_esp.c
@@ -34,9 +34,9 @@
 #include <linux/delay.h>
 #include <linux/zorro.h>
 #include <linux/slab.h>
+#include <linux/pgtable.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/cacheflush.h>
 #include <asm/amigahw.h>
 #include <asm/amigaints.h>
diff --git a/drivers/sfi/Kconfig b/drivers/sfi/Kconfig
index 1878d378869a..3d0b64db214e 100644
--- a/drivers/sfi/Kconfig
+++ b/drivers/sfi/Kconfig
@@ -5,7 +5,7 @@
 
 menuconfig SFI
 	bool "SFI (Simple Firmware Interface) Support"
-	---help---
+	help
 	The Simple Firmware Interface (SFI) provides a lightweight method
 	for platform firmware to pass information to the operating system
 	via static tables in memory.  Kernel SFI support is required to
diff --git a/drivers/slimbus/core.c b/drivers/slimbus/core.c
index 526e3215d8fe..ae1e248a8fb8 100644
--- a/drivers/slimbus/core.c
+++ b/drivers/slimbus/core.c
@@ -162,9 +162,8 @@ static int slim_add_device(struct slim_controller *ctrl,
 	sbdev->ctrl = ctrl;
 	INIT_LIST_HEAD(&sbdev->stream_list);
 	spin_lock_init(&sbdev->stream_list_lock);
-
-	if (node)
-		sbdev->dev.of_node = of_node_get(node);
+	sbdev->dev.of_node = of_node_get(node);
+	sbdev->dev.fwnode = of_fwnode_handle(node);
 
 	dev_set_name(&sbdev->dev, "%x:%x:%x:%x",
 				  sbdev->e_addr.manf_id,
@@ -283,6 +282,7 @@ EXPORT_SYMBOL_GPL(slim_register_controller);
 /* slim_remove_device: Remove the effect of slim_add_device() */
 static void slim_remove_device(struct slim_device *sbdev)
 {
+	of_node_put(sbdev->dev.of_node);
 	device_unregister(&sbdev->dev);
 }
 
diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c
index fc2575fef51b..743ee7b4e63f 100644
--- a/drivers/slimbus/qcom-ngd-ctrl.c
+++ b/drivers/slimbus/qcom-ngd-ctrl.c
@@ -1361,12 +1361,10 @@ static int of_qcom_slim_ngd_register(struct device *parent,
 		ngd->pdev->driver_override = QCOM_SLIM_NGD_DRV_NAME;
 		ngd->pdev->dev.of_node = node;
 		ctrl->ngd = ngd;
-		platform_set_drvdata(ngd->pdev, ctrl);
 
 		platform_device_add(ngd->pdev);
 		ngd->base = ctrl->base + ngd->id * data->offset +
 					(ngd->id - 1) * data->size;
-		ctrl->ngd = ngd;
 
 		return 0;
 	}
@@ -1376,12 +1374,13 @@ static int of_qcom_slim_ngd_register(struct device *parent,
 
 static int qcom_slim_ngd_probe(struct platform_device *pdev)
 {
-	struct qcom_slim_ngd_ctrl *ctrl = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
+	struct qcom_slim_ngd_ctrl *ctrl = dev_get_drvdata(dev->parent);
 	int ret;
 
 	ctrl->ctrl.dev = dev;
 
+	platform_set_drvdata(pdev, ctrl);
 	pm_runtime_use_autosuspend(dev);
 	pm_runtime_set_autosuspend_delay(dev, QCOM_SLIM_NGD_AUTOSUSPEND);
 	pm_runtime_set_suspended(dev);
diff --git a/drivers/soc/amlogic/meson-ee-pwrc.c b/drivers/soc/amlogic/meson-ee-pwrc.c
index 3f0261d53ad9..43665b77aa9e 100644
--- a/drivers/soc/amlogic/meson-ee-pwrc.c
+++ b/drivers/soc/amlogic/meson-ee-pwrc.c
@@ -14,13 +14,23 @@
 #include <linux/reset-controller.h>
 #include <linux/reset.h>
 #include <linux/clk.h>
+#include <dt-bindings/power/meson8-power.h>
 #include <dt-bindings/power/meson-g12a-power.h>
+#include <dt-bindings/power/meson-gxbb-power.h>
 #include <dt-bindings/power/meson-sm1-power.h>
 
 /* AO Offsets */
 
-#define AO_RTI_GEN_PWR_SLEEP0		(0x3a << 2)
-#define AO_RTI_GEN_PWR_ISO0		(0x3b << 2)
+#define GX_AO_RTI_GEN_PWR_SLEEP0	(0x3a << 2)
+#define GX_AO_RTI_GEN_PWR_ISO0		(0x3b << 2)
+
+/*
+ * Meson8/Meson8b/Meson8m2 only expose the power management registers of the
+ * AO-bus as syscon. 0x3a from GX translates to 0x02, 0x3b translates to 0x03
+ * and so on.
+ */
+#define MESON8_AO_RTI_GEN_PWR_SLEEP0	(0x02 << 2)
+#define MESON8_AO_RTI_GEN_PWR_ISO0	(0x03 << 2)
 
 /* HHI Offsets */
 
@@ -66,18 +76,25 @@ struct meson_ee_pwrc_domain_data {
 
 /* TOP Power Domains */
 
-static struct meson_ee_pwrc_top_domain g12a_pwrc_vpu = {
-	.sleep_reg = AO_RTI_GEN_PWR_SLEEP0,
+static struct meson_ee_pwrc_top_domain gx_pwrc_vpu = {
+	.sleep_reg = GX_AO_RTI_GEN_PWR_SLEEP0,
+	.sleep_mask = BIT(8),
+	.iso_reg = GX_AO_RTI_GEN_PWR_SLEEP0,
+	.iso_mask = BIT(9),
+};
+
+static struct meson_ee_pwrc_top_domain meson8_pwrc_vpu = {
+	.sleep_reg = MESON8_AO_RTI_GEN_PWR_SLEEP0,
 	.sleep_mask = BIT(8),
-	.iso_reg = AO_RTI_GEN_PWR_SLEEP0,
+	.iso_reg = MESON8_AO_RTI_GEN_PWR_SLEEP0,
 	.iso_mask = BIT(9),
 };
 
 #define SM1_EE_PD(__bit)					\
 	{							\
-		.sleep_reg = AO_RTI_GEN_PWR_SLEEP0, 		\
+		.sleep_reg = GX_AO_RTI_GEN_PWR_SLEEP0, 		\
 		.sleep_mask = BIT(__bit), 			\
-		.iso_reg = AO_RTI_GEN_PWR_ISO0, 		\
+		.iso_reg = GX_AO_RTI_GEN_PWR_ISO0, 		\
 		.iso_mask = BIT(__bit), 			\
 	}
 
@@ -124,10 +141,26 @@ static struct meson_ee_pwrc_mem_domain g12a_pwrc_mem_vpu[] = {
 	VPU_HHI_MEMPD(HHI_MEM_PD_REG0),
 };
 
-static struct meson_ee_pwrc_mem_domain g12a_pwrc_mem_eth[] = {
+static struct meson_ee_pwrc_mem_domain gxbb_pwrc_mem_vpu[] = {
+	VPU_MEMPD(HHI_VPU_MEM_PD_REG0),
+	VPU_MEMPD(HHI_VPU_MEM_PD_REG1),
+	VPU_HHI_MEMPD(HHI_MEM_PD_REG0),
+};
+
+static struct meson_ee_pwrc_mem_domain meson_pwrc_mem_eth[] = {
 	{ HHI_MEM_PD_REG0, GENMASK(3, 2) },
 };
 
+static struct meson_ee_pwrc_mem_domain meson8_pwrc_audio_dsp_mem[] = {
+	{ HHI_MEM_PD_REG0, GENMASK(1, 0) },
+};
+
+static struct meson_ee_pwrc_mem_domain meson8_pwrc_mem_vpu[] = {
+	VPU_MEMPD(HHI_VPU_MEM_PD_REG0),
+	VPU_MEMPD(HHI_VPU_MEM_PD_REG1),
+	VPU_HHI_MEMPD(HHI_MEM_PD_REG0),
+};
+
 static struct meson_ee_pwrc_mem_domain sm1_pwrc_mem_vpu[] = {
 	VPU_MEMPD(HHI_VPU_MEM_PD_REG0),
 	VPU_MEMPD(HHI_VPU_MEM_PD_REG1),
@@ -199,9 +232,35 @@ static struct meson_ee_pwrc_mem_domain sm1_pwrc_mem_audio[] = {
 static bool pwrc_ee_get_power(struct meson_ee_pwrc_domain *pwrc_domain);
 
 static struct meson_ee_pwrc_domain_desc g12a_pwrc_domains[] = {
-	[PWRC_G12A_VPU_ID]  = VPU_PD("VPU", &g12a_pwrc_vpu, g12a_pwrc_mem_vpu,
+	[PWRC_G12A_VPU_ID]  = VPU_PD("VPU", &gx_pwrc_vpu, g12a_pwrc_mem_vpu,
 				     pwrc_ee_get_power, 11, 2),
-	[PWRC_G12A_ETH_ID] = MEM_PD("ETH", g12a_pwrc_mem_eth),
+	[PWRC_G12A_ETH_ID] = MEM_PD("ETH", meson_pwrc_mem_eth),
+};
+
+static struct meson_ee_pwrc_domain_desc gxbb_pwrc_domains[] = {
+	[PWRC_GXBB_VPU_ID]  = VPU_PD("VPU", &gx_pwrc_vpu, gxbb_pwrc_mem_vpu,
+				     pwrc_ee_get_power, 12, 2),
+	[PWRC_GXBB_ETHERNET_MEM_ID] = MEM_PD("ETH", meson_pwrc_mem_eth),
+};
+
+static struct meson_ee_pwrc_domain_desc meson8_pwrc_domains[] = {
+	[PWRC_MESON8_VPU_ID]  = VPU_PD("VPU", &meson8_pwrc_vpu,
+				       meson8_pwrc_mem_vpu, pwrc_ee_get_power,
+				       0, 1),
+	[PWRC_MESON8_ETHERNET_MEM_ID] = MEM_PD("ETHERNET_MEM",
+					       meson_pwrc_mem_eth),
+	[PWRC_MESON8_AUDIO_DSP_MEM_ID] = MEM_PD("AUDIO_DSP_MEM",
+						meson8_pwrc_audio_dsp_mem),
+};
+
+static struct meson_ee_pwrc_domain_desc meson8b_pwrc_domains[] = {
+	[PWRC_MESON8_VPU_ID]  = VPU_PD("VPU", &meson8_pwrc_vpu,
+				       meson8_pwrc_mem_vpu, pwrc_ee_get_power,
+				       11, 1),
+	[PWRC_MESON8_ETHERNET_MEM_ID] = MEM_PD("ETHERNET_MEM",
+					       meson_pwrc_mem_eth),
+	[PWRC_MESON8_AUDIO_DSP_MEM_ID] = MEM_PD("AUDIO_DSP_MEM",
+						meson8_pwrc_audio_dsp_mem),
 };
 
 static struct meson_ee_pwrc_domain_desc sm1_pwrc_domains[] = {
@@ -216,7 +275,7 @@ static struct meson_ee_pwrc_domain_desc sm1_pwrc_domains[] = {
 	[PWRC_SM1_GE2D_ID] = TOP_PD("GE2D", &sm1_pwrc_ge2d, sm1_pwrc_mem_ge2d,
 				    pwrc_ee_get_power),
 	[PWRC_SM1_AUDIO_ID] = MEM_PD("AUDIO", sm1_pwrc_mem_audio),
-	[PWRC_SM1_ETH_ID] = MEM_PD("ETH", g12a_pwrc_mem_eth),
+	[PWRC_SM1_ETH_ID] = MEM_PD("ETH", meson_pwrc_mem_eth),
 };
 
 struct meson_ee_pwrc_domain {
@@ -470,6 +529,21 @@ static struct meson_ee_pwrc_domain_data meson_ee_g12a_pwrc_data = {
 	.domains = g12a_pwrc_domains,
 };
 
+static struct meson_ee_pwrc_domain_data meson_ee_gxbb_pwrc_data = {
+	.count = ARRAY_SIZE(gxbb_pwrc_domains),
+	.domains = gxbb_pwrc_domains,
+};
+
+static struct meson_ee_pwrc_domain_data meson_ee_m8_pwrc_data = {
+	.count = ARRAY_SIZE(meson8_pwrc_domains),
+	.domains = meson8_pwrc_domains,
+};
+
+static struct meson_ee_pwrc_domain_data meson_ee_m8b_pwrc_data = {
+	.count = ARRAY_SIZE(meson8b_pwrc_domains),
+	.domains = meson8b_pwrc_domains,
+};
+
 static struct meson_ee_pwrc_domain_data meson_ee_sm1_pwrc_data = {
 	.count = ARRAY_SIZE(sm1_pwrc_domains),
 	.domains = sm1_pwrc_domains,
@@ -477,6 +551,22 @@ static struct meson_ee_pwrc_domain_data meson_ee_sm1_pwrc_data = {
 
 static const struct of_device_id meson_ee_pwrc_match_table[] = {
 	{
+		.compatible = "amlogic,meson8-pwrc",
+		.data = &meson_ee_m8_pwrc_data,
+	},
+	{
+		.compatible = "amlogic,meson8b-pwrc",
+		.data = &meson_ee_m8b_pwrc_data,
+	},
+	{
+		.compatible = "amlogic,meson8m2-pwrc",
+		.data = &meson_ee_m8b_pwrc_data,
+	},
+	{
+		.compatible = "amlogic,meson-gxbb-pwrc",
+		.data = &meson_ee_gxbb_pwrc_data,
+	},
+	{
 		.compatible = "amlogic,meson-g12a-pwrc",
 		.data = &meson_ee_g12a_pwrc_data,
 	},
diff --git a/drivers/soc/aspeed/Kconfig b/drivers/soc/aspeed/Kconfig
index 323e177aa74d..c95fa30f1a76 100644
--- a/drivers/soc/aspeed/Kconfig
+++ b/drivers/soc/aspeed/Kconfig
@@ -8,7 +8,7 @@ config SOC_ASPEED
 config ASPEED_LPC_CTRL
 	depends on SOC_ASPEED && REGMAP && MFD_SYSCON
 	tristate "Aspeed ast2400/2500 HOST LPC to BMC bridge control"
-	---help---
+	help
 	  Control Aspeed ast2400/2500 HOST LPC to BMC mappings through
 	  ioctl()s, the driver also provides a read/write interface to a BMC ram
 	  region where the host LPC read/write region can be buffered.
diff --git a/drivers/soc/fsl/dpio/dpio-service.c b/drivers/soc/fsl/dpio/dpio-service.c
index bcdcd3e7d7f1..7351f3030550 100644
--- a/drivers/soc/fsl/dpio/dpio-service.c
+++ b/drivers/soc/fsl/dpio/dpio-service.c
@@ -58,7 +58,7 @@ static inline struct dpaa2_io *service_select_by_cpu(struct dpaa2_io *d,
 	 * If cpu == -1, choose the current cpu, with no guarantees about
 	 * potentially being migrated away.
 	 */
-	if (unlikely(cpu < 0))
+	if (cpu < 0)
 		cpu = smp_processor_id();
 
 	/* If a specific cpu was requested, pick it up immediately */
@@ -70,6 +70,10 @@ static inline struct dpaa2_io *service_select(struct dpaa2_io *d)
 	if (d)
 		return d;
 
+	d = service_select_by_cpu(d, -1);
+	if (d)
+		return d;
+
 	spin_lock(&dpio_list_lock);
 	d = list_entry(dpio_list.next, struct dpaa2_io, node);
 	list_del(&d->node);
diff --git a/drivers/soc/fsl/dpio/qbman-portal.c b/drivers/soc/fsl/dpio/qbman-portal.c
index 23a1377971f4..0ab85bfb116f 100644
--- a/drivers/soc/fsl/dpio/qbman-portal.c
+++ b/drivers/soc/fsl/dpio/qbman-portal.c
@@ -572,18 +572,6 @@ void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid,
 #define EQAR_VB(eqar)      ((eqar) & 0x80)
 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
 
-static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p,
-						   u8 idx)
-{
-	if (idx < 16)
-		qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4,
-				     QMAN_RT_MODE);
-	else
-		qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT2 +
-				     (idx - 16) * 4,
-				     QMAN_RT_MODE);
-}
-
 #define QB_RT_BIT ((u32)0x100)
 /**
  * qbman_swp_enqueue_direct() - Issue an enqueue command
diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c
index 1e164e03410a..9888a7061873 100644
--- a/drivers/soc/fsl/qbman/qman.c
+++ b/drivers/soc/fsl/qbman/qman.c
@@ -449,11 +449,6 @@ static inline int qm_eqcr_init(struct qm_portal *portal,
 	return 0;
 }
 
-static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal)
-{
-	return (qm_in(portal, QM_REG_CFG) >> 28) & 0x7;
-}
-
 static inline void qm_eqcr_finish(struct qm_portal *portal)
 {
 	struct qm_eqcr *eqcr = &portal->eqcr;
diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c
index 447146861c2c..2df20d6f85fa 100644
--- a/drivers/soc/fsl/qe/qe.c
+++ b/drivers/soc/fsl/qe/qe.c
@@ -448,7 +448,7 @@ int qe_upload_firmware(const struct qe_firmware *firmware)
 	unsigned int i;
 	unsigned int j;
 	u32 crc;
-	size_t calc_size = sizeof(struct qe_firmware);
+	size_t calc_size;
 	size_t length;
 	const struct qe_header *hdr;
 
@@ -480,7 +480,7 @@ int qe_upload_firmware(const struct qe_firmware *firmware)
 	}
 
 	/* Validate the length and check if there's a CRC */
-	calc_size += (firmware->count - 1) * sizeof(struct qe_microcode);
+	calc_size = struct_size(firmware, microcode, firmware->count);
 
 	for (i = 0; i < firmware->count; i++)
 		/*
diff --git a/drivers/soc/fsl/qe/ucc.c b/drivers/soc/fsl/qe/ucc.c
index d6c93970df4d..cac0fb7693a0 100644
--- a/drivers/soc/fsl/qe/ucc.c
+++ b/drivers/soc/fsl/qe/ucc.c
@@ -519,7 +519,7 @@ int ucc_set_tdm_rxtx_clk(u32 tdm_num, enum qe_clock clock,
 	int clock_bits;
 	u32 shift;
 	struct qe_mux __iomem *qe_mux_reg;
-	 __be32 __iomem *cmxs1cr;
+	__be32 __iomem *cmxs1cr;
 
 	qe_mux_reg = &qe_immr->qmx;
 
diff --git a/drivers/soc/imx/Makefile b/drivers/soc/imx/Makefile
index 103e2c93c342..446143241fe7 100644
--- a/drivers/soc/imx/Makefile
+++ b/drivers/soc/imx/Makefile
@@ -1,4 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
+ifeq ($(CONFIG_ARM),y)
+obj-$(CONFIG_ARCH_MXC) += soc-imx.o
+endif
 obj-$(CONFIG_HAVE_IMX_GPC) += gpc.o
 obj-$(CONFIG_IMX_GPCV2_PM_DOMAINS) += gpcv2.o
 obj-$(CONFIG_SOC_IMX8M) += soc-imx8m.o
diff --git a/drivers/soc/imx/soc-imx.c b/drivers/soc/imx/soc-imx.c
new file mode 100644
index 000000000000..fec3d672b606
--- /dev/null
+++ b/drivers/soc/imx/soc-imx.c
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 NXP
+ */
+
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/sys_soc.h>
+
+#include <soc/imx/cpu.h>
+#include <soc/imx/revision.h>
+
+#define OCOTP_UID_H	0x420
+#define OCOTP_UID_L	0x410
+
+#define OCOTP_ULP_UID_1		0x4b0
+#define OCOTP_ULP_UID_2		0x4c0
+#define OCOTP_ULP_UID_3		0x4d0
+#define OCOTP_ULP_UID_4		0x4e0
+
+static int __init imx_soc_device_init(void)
+{
+	struct soc_device_attribute *soc_dev_attr;
+	const char *ocotp_compat = NULL;
+	struct soc_device *soc_dev;
+	struct device_node *root;
+	struct regmap *ocotp = NULL;
+	const char *soc_id;
+	u64 soc_uid = 0;
+	u32 val;
+	int ret;
+
+	soc_dev_attr = kzalloc(sizeof(*soc_dev_attr), GFP_KERNEL);
+	if (!soc_dev_attr)
+		return -ENOMEM;
+
+	soc_dev_attr->family = "Freescale i.MX";
+
+	root = of_find_node_by_path("/");
+	ret = of_property_read_string(root, "model", &soc_dev_attr->machine);
+	of_node_put(root);
+	if (ret)
+		goto free_soc;
+
+	switch (__mxc_cpu_type) {
+	case MXC_CPU_MX1:
+		soc_id = "i.MX1";
+		break;
+	case MXC_CPU_MX21:
+		soc_id = "i.MX21";
+		break;
+	case MXC_CPU_MX25:
+		soc_id = "i.MX25";
+		break;
+	case MXC_CPU_MX27:
+		soc_id = "i.MX27";
+		break;
+	case MXC_CPU_MX31:
+		soc_id = "i.MX31";
+		break;
+	case MXC_CPU_MX35:
+		soc_id = "i.MX35";
+		break;
+	case MXC_CPU_MX51:
+		soc_id = "i.MX51";
+		break;
+	case MXC_CPU_MX53:
+		soc_id = "i.MX53";
+		break;
+	case MXC_CPU_IMX6SL:
+		ocotp_compat = "fsl,imx6sl-ocotp";
+		soc_id = "i.MX6SL";
+		break;
+	case MXC_CPU_IMX6DL:
+		ocotp_compat = "fsl,imx6q-ocotp";
+		soc_id = "i.MX6DL";
+		break;
+	case MXC_CPU_IMX6SX:
+		ocotp_compat = "fsl,imx6sx-ocotp";
+		soc_id = "i.MX6SX";
+		break;
+	case MXC_CPU_IMX6Q:
+		ocotp_compat = "fsl,imx6q-ocotp";
+		soc_id = "i.MX6Q";
+		break;
+	case MXC_CPU_IMX6UL:
+		ocotp_compat = "fsl,imx6ul-ocotp";
+		soc_id = "i.MX6UL";
+		break;
+	case MXC_CPU_IMX6ULL:
+		ocotp_compat = "fsl,imx6ull-ocotp";
+		soc_id = "i.MX6ULL";
+		break;
+	case MXC_CPU_IMX6ULZ:
+		ocotp_compat = "fsl,imx6ull-ocotp";
+		soc_id = "i.MX6ULZ";
+		break;
+	case MXC_CPU_IMX6SLL:
+		ocotp_compat = "fsl,imx6sll-ocotp";
+		soc_id = "i.MX6SLL";
+		break;
+	case MXC_CPU_IMX7D:
+		ocotp_compat = "fsl,imx7d-ocotp";
+		soc_id = "i.MX7D";
+		break;
+	case MXC_CPU_IMX7ULP:
+		ocotp_compat = "fsl,imx7ulp-ocotp";
+		soc_id = "i.MX7ULP";
+		break;
+	case MXC_CPU_VF500:
+		ocotp_compat = "fsl,vf610-ocotp";
+		soc_id = "VF500";
+		break;
+	case MXC_CPU_VF510:
+		ocotp_compat = "fsl,vf610-ocotp";
+		soc_id = "VF510";
+		break;
+	case MXC_CPU_VF600:
+		ocotp_compat = "fsl,vf610-ocotp";
+		soc_id = "VF600";
+		break;
+	case MXC_CPU_VF610:
+		ocotp_compat = "fsl,vf610-ocotp";
+		soc_id = "VF610";
+		break;
+	default:
+		soc_id = "Unknown";
+	}
+	soc_dev_attr->soc_id = soc_id;
+
+	if (ocotp_compat) {
+		ocotp = syscon_regmap_lookup_by_compatible(ocotp_compat);
+		if (IS_ERR(ocotp))
+			pr_err("%s: failed to find %s regmap!\n", __func__, ocotp_compat);
+	}
+
+	if (!IS_ERR_OR_NULL(ocotp)) {
+		if (__mxc_cpu_type == MXC_CPU_IMX7ULP) {
+			regmap_read(ocotp, OCOTP_ULP_UID_4, &val);
+			soc_uid = val & 0xffff;
+			regmap_read(ocotp, OCOTP_ULP_UID_3, &val);
+			soc_uid <<= 16;
+			soc_uid |= val & 0xffff;
+			regmap_read(ocotp, OCOTP_ULP_UID_2, &val);
+			soc_uid <<= 16;
+			soc_uid |= val & 0xffff;
+			regmap_read(ocotp, OCOTP_ULP_UID_1, &val);
+			soc_uid <<= 16;
+			soc_uid |= val & 0xffff;
+		} else {
+			regmap_read(ocotp, OCOTP_UID_H, &val);
+			soc_uid = val;
+			regmap_read(ocotp, OCOTP_UID_L, &val);
+			soc_uid <<= 32;
+			soc_uid |= val;
+		}
+	}
+
+	soc_dev_attr->revision = kasprintf(GFP_KERNEL, "%d.%d",
+					   (imx_get_soc_revision() >> 4) & 0xf,
+					   imx_get_soc_revision() & 0xf);
+	if (!soc_dev_attr->revision) {
+		ret = -ENOMEM;
+		goto free_soc;
+	}
+
+	soc_dev_attr->serial_number = kasprintf(GFP_KERNEL, "%016llX", soc_uid);
+	if (!soc_dev_attr->serial_number) {
+		ret = -ENOMEM;
+		goto free_rev;
+	}
+
+	soc_dev = soc_device_register(soc_dev_attr);
+	if (IS_ERR(soc_dev)) {
+		ret = PTR_ERR(soc_dev);
+		goto free_serial_number;
+	}
+
+	return 0;
+
+free_serial_number:
+	kfree(soc_dev_attr->serial_number);
+free_rev:
+	kfree(soc_dev_attr->revision);
+free_soc:
+	kfree(soc_dev_attr);
+	return ret;
+}
+device_initcall(imx_soc_device_init);
diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c
index 719e1f189ebf..7b0759adb47d 100644
--- a/drivers/soc/imx/soc-imx8m.c
+++ b/drivers/soc/imx/soc-imx8m.c
@@ -53,11 +53,11 @@ static u32 __init imx8mq_soc_revision(void)
 	struct device_node *np;
 	void __iomem *ocotp_base;
 	u32 magic;
-	u32 rev = 0;
+	u32 rev;
 
 	np = of_find_compatible_node(NULL, NULL, "fsl,imx8mq-ocotp");
 	if (!np)
-		goto out;
+		return 0;
 
 	ocotp_base = of_iomap(np, 0);
 	WARN_ON(!ocotp_base);
@@ -78,9 +78,8 @@ static u32 __init imx8mq_soc_revision(void)
 	soc_uid |= readl_relaxed(ocotp_base + OCOTP_UID_LOW);
 
 	iounmap(ocotp_base);
-
-out:
 	of_node_put(np);
+
 	return rev;
 }
 
diff --git a/drivers/soc/kendryte/k210-sysctl.c b/drivers/soc/kendryte/k210-sysctl.c
index 4608fbca20e1..707019223dd8 100644
--- a/drivers/soc/kendryte/k210-sysctl.c
+++ b/drivers/soc/kendryte/k210-sysctl.c
@@ -246,3 +246,15 @@ static void __init k210_soc_early_init(const void *fdt)
 	iounmap(regs);
 }
 SOC_EARLY_INIT_DECLARE(generic_k210, "kendryte,k210", k210_soc_early_init);
+
+#ifdef CONFIG_SOC_KENDRYTE_K210_DTB_BUILTIN
+/*
+ * Generic entry for the default k210.dtb embedded DTB for boards with:
+ *   - Vendor ID: 0x4B5
+ *   - Arch ID: 0xE59889E6A5A04149 (= "Canaan AI" in UTF-8 encoded Chinese)
+ *   - Impl ID:	0x4D41495832303030 (= "MAIX2000")
+ * These values are reported by the SiPEED MAXDUINO, SiPEED MAIX GO and
+ * SiPEED Dan dock boards.
+ */
+SOC_BUILTIN_DTB_DECLARE(k210, 0x4B5, 0xE59889E6A5A04149, 0x4D41495832303030);
+#endif
diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig
index 2114b563478c..59a56cd790ec 100644
--- a/drivers/soc/mediatek/Kconfig
+++ b/drivers/soc/mediatek/Kconfig
@@ -44,4 +44,11 @@ config MTK_SCPSYS
 	  Say yes here to add support for the MediaTek SCPSYS power domain
 	  driver.
 
+config MTK_MMSYS
+	bool "MediaTek MMSYS Support"
+	default ARCH_MEDIATEK
+	help
+	  Say yes here to add support for the MediaTek Multimedia
+	  Subsystem (MMSYS).
+
 endmenu
diff --git a/drivers/soc/mediatek/Makefile b/drivers/soc/mediatek/Makefile
index b01733074ad6..01f9f873634a 100644
--- a/drivers/soc/mediatek/Makefile
+++ b/drivers/soc/mediatek/Makefile
@@ -3,3 +3,4 @@ obj-$(CONFIG_MTK_CMDQ) += mtk-cmdq-helper.o
 obj-$(CONFIG_MTK_INFRACFG) += mtk-infracfg.o
 obj-$(CONFIG_MTK_PMIC_WRAP) += mtk-pmic-wrap.o
 obj-$(CONFIG_MTK_SCPSYS) += mtk-scpsys.o
+obj-$(CONFIG_MTK_MMSYS) += mtk-mmsys.o
diff --git a/drivers/soc/mediatek/mtk-mmsys.c b/drivers/soc/mediatek/mtk-mmsys.c
new file mode 100644
index 000000000000..a55f25511173
--- /dev/null
+++ b/drivers/soc/mediatek/mtk-mmsys.c
@@ -0,0 +1,378 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: James Liao <jamesjj.liao@mediatek.com>
+ */
+
+#include <linux/device.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/soc/mediatek/mtk-mmsys.h>
+
+#include "../../gpu/drm/mediatek/mtk_drm_ddp.h"
+#include "../../gpu/drm/mediatek/mtk_drm_ddp_comp.h"
+
+#define DISP_REG_CONFIG_DISP_OVL0_MOUT_EN	0x040
+#define DISP_REG_CONFIG_DISP_OVL1_MOUT_EN	0x044
+#define DISP_REG_CONFIG_DISP_OD_MOUT_EN		0x048
+#define DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN	0x04c
+#define DISP_REG_CONFIG_DISP_UFOE_MOUT_EN	0x050
+#define DISP_REG_CONFIG_DISP_COLOR0_SEL_IN	0x084
+#define DISP_REG_CONFIG_DISP_COLOR1_SEL_IN	0x088
+#define DISP_REG_CONFIG_DSIE_SEL_IN		0x0a4
+#define DISP_REG_CONFIG_DSIO_SEL_IN		0x0a8
+#define DISP_REG_CONFIG_DPI_SEL_IN		0x0ac
+#define DISP_REG_CONFIG_DISP_RDMA2_SOUT		0x0b8
+#define DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN	0x0c4
+#define DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN	0x0c8
+#define DISP_REG_CONFIG_MMSYS_CG_CON0		0x100
+
+#define DISP_REG_CONFIG_DISP_OVL_MOUT_EN	0x030
+#define DISP_REG_CONFIG_OUT_SEL			0x04c
+#define DISP_REG_CONFIG_DSI_SEL			0x050
+#define DISP_REG_CONFIG_DPI_SEL			0x064
+
+#define OVL0_MOUT_EN_COLOR0			0x1
+#define OD_MOUT_EN_RDMA0			0x1
+#define OD1_MOUT_EN_RDMA1			BIT(16)
+#define UFOE_MOUT_EN_DSI0			0x1
+#define COLOR0_SEL_IN_OVL0			0x1
+#define OVL1_MOUT_EN_COLOR1			0x1
+#define GAMMA_MOUT_EN_RDMA1			0x1
+#define RDMA0_SOUT_DPI0				0x2
+#define RDMA0_SOUT_DPI1				0x3
+#define RDMA0_SOUT_DSI1				0x1
+#define RDMA0_SOUT_DSI2				0x4
+#define RDMA0_SOUT_DSI3				0x5
+#define RDMA1_SOUT_DPI0				0x2
+#define RDMA1_SOUT_DPI1				0x3
+#define RDMA1_SOUT_DSI1				0x1
+#define RDMA1_SOUT_DSI2				0x4
+#define RDMA1_SOUT_DSI3				0x5
+#define RDMA2_SOUT_DPI0				0x2
+#define RDMA2_SOUT_DPI1				0x3
+#define RDMA2_SOUT_DSI1				0x1
+#define RDMA2_SOUT_DSI2				0x4
+#define RDMA2_SOUT_DSI3				0x5
+#define DPI0_SEL_IN_RDMA1			0x1
+#define DPI0_SEL_IN_RDMA2			0x3
+#define DPI1_SEL_IN_RDMA1			(0x1 << 8)
+#define DPI1_SEL_IN_RDMA2			(0x3 << 8)
+#define DSI0_SEL_IN_RDMA1			0x1
+#define DSI0_SEL_IN_RDMA2			0x4
+#define DSI1_SEL_IN_RDMA1			0x1
+#define DSI1_SEL_IN_RDMA2			0x4
+#define DSI2_SEL_IN_RDMA1			(0x1 << 16)
+#define DSI2_SEL_IN_RDMA2			(0x4 << 16)
+#define DSI3_SEL_IN_RDMA1			(0x1 << 16)
+#define DSI3_SEL_IN_RDMA2			(0x4 << 16)
+#define COLOR1_SEL_IN_OVL1			0x1
+
+#define OVL_MOUT_EN_RDMA			0x1
+#define BLS_TO_DSI_RDMA1_TO_DPI1		0x8
+#define BLS_TO_DPI_RDMA1_TO_DSI			0x2
+#define DSI_SEL_IN_BLS				0x0
+#define DPI_SEL_IN_BLS				0x0
+#define DSI_SEL_IN_RDMA				0x1
+
+struct mtk_mmsys_driver_data {
+	const char *clk_driver;
+};
+
+static const struct mtk_mmsys_driver_data mt2701_mmsys_driver_data = {
+	.clk_driver = "clk-mt2701-mm",
+};
+
+static const struct mtk_mmsys_driver_data mt2712_mmsys_driver_data = {
+	.clk_driver = "clk-mt2712-mm",
+};
+
+static const struct mtk_mmsys_driver_data mt6779_mmsys_driver_data = {
+	.clk_driver = "clk-mt6779-mm",
+};
+
+static const struct mtk_mmsys_driver_data mt6797_mmsys_driver_data = {
+	.clk_driver = "clk-mt6797-mm",
+};
+
+static const struct mtk_mmsys_driver_data mt8173_mmsys_driver_data = {
+	.clk_driver = "clk-mt8173-mm",
+};
+
+static const struct mtk_mmsys_driver_data mt8183_mmsys_driver_data = {
+	.clk_driver = "clk-mt8183-mm",
+};
+
+static unsigned int mtk_mmsys_ddp_mout_en(enum mtk_ddp_comp_id cur,
+					  enum mtk_ddp_comp_id next,
+					  unsigned int *addr)
+{
+	unsigned int value;
+
+	if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_COLOR0) {
+		*addr = DISP_REG_CONFIG_DISP_OVL0_MOUT_EN;
+		value = OVL0_MOUT_EN_COLOR0;
+	} else if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_RDMA0) {
+		*addr = DISP_REG_CONFIG_DISP_OVL_MOUT_EN;
+		value = OVL_MOUT_EN_RDMA;
+	} else if (cur == DDP_COMPONENT_OD0 && next == DDP_COMPONENT_RDMA0) {
+		*addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN;
+		value = OD_MOUT_EN_RDMA0;
+	} else if (cur == DDP_COMPONENT_UFOE && next == DDP_COMPONENT_DSI0) {
+		*addr = DISP_REG_CONFIG_DISP_UFOE_MOUT_EN;
+		value = UFOE_MOUT_EN_DSI0;
+	} else if (cur == DDP_COMPONENT_OVL1 && next == DDP_COMPONENT_COLOR1) {
+		*addr = DISP_REG_CONFIG_DISP_OVL1_MOUT_EN;
+		value = OVL1_MOUT_EN_COLOR1;
+	} else if (cur == DDP_COMPONENT_GAMMA && next == DDP_COMPONENT_RDMA1) {
+		*addr = DISP_REG_CONFIG_DISP_GAMMA_MOUT_EN;
+		value = GAMMA_MOUT_EN_RDMA1;
+	} else if (cur == DDP_COMPONENT_OD1 && next == DDP_COMPONENT_RDMA1) {
+		*addr = DISP_REG_CONFIG_DISP_OD_MOUT_EN;
+		value = OD1_MOUT_EN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DPI0;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DPI1;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DSI1;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DSI2;
+	} else if (cur == DDP_COMPONENT_RDMA0 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA0_SOUT_EN;
+		value = RDMA0_SOUT_DSI3;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DSI1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DSI2;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DSI3;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DPI0;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA1_SOUT_EN;
+		value = RDMA1_SOUT_DPI1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DPI0;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DPI1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DSI1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DSI2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DISP_RDMA2_SOUT;
+		value = RDMA2_SOUT_DSI3;
+	} else {
+		value = 0;
+	}
+
+	return value;
+}
+
+static unsigned int mtk_mmsys_ddp_sel_in(enum mtk_ddp_comp_id cur,
+					 enum mtk_ddp_comp_id next,
+					 unsigned int *addr)
+{
+	unsigned int value;
+
+	if (cur == DDP_COMPONENT_OVL0 && next == DDP_COMPONENT_COLOR0) {
+		*addr = DISP_REG_CONFIG_DISP_COLOR0_SEL_IN;
+		value = COLOR0_SEL_IN_OVL0;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
+		value = DPI0_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
+		value = DPI1_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI0) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI0_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
+		value = DSI1_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI2_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA1 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
+		value = DSI3_SEL_IN_RDMA1;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI0) {
+		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
+		value = DPI0_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DPI1) {
+		*addr = DISP_REG_CONFIG_DPI_SEL_IN;
+		value = DPI1_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI0) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI0_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI1) {
+		*addr = DISP_REG_CONFIG_DSIO_SEL_IN;
+		value = DSI1_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI2) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI2_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_RDMA2 && next == DDP_COMPONENT_DSI3) {
+		*addr = DISP_REG_CONFIG_DSIE_SEL_IN;
+		value = DSI3_SEL_IN_RDMA2;
+	} else if (cur == DDP_COMPONENT_OVL1 && next == DDP_COMPONENT_COLOR1) {
+		*addr = DISP_REG_CONFIG_DISP_COLOR1_SEL_IN;
+		value = COLOR1_SEL_IN_OVL1;
+	} else if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DSI0) {
+		*addr = DISP_REG_CONFIG_DSI_SEL;
+		value = DSI_SEL_IN_BLS;
+	} else {
+		value = 0;
+	}
+
+	return value;
+}
+
+static void mtk_mmsys_ddp_sout_sel(void __iomem *config_regs,
+				   enum mtk_ddp_comp_id cur,
+				   enum mtk_ddp_comp_id next)
+{
+	if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DSI0) {
+		writel_relaxed(BLS_TO_DSI_RDMA1_TO_DPI1,
+			       config_regs + DISP_REG_CONFIG_OUT_SEL);
+	} else if (cur == DDP_COMPONENT_BLS && next == DDP_COMPONENT_DPI0) {
+		writel_relaxed(BLS_TO_DPI_RDMA1_TO_DSI,
+			       config_regs + DISP_REG_CONFIG_OUT_SEL);
+		writel_relaxed(DSI_SEL_IN_RDMA,
+			       config_regs + DISP_REG_CONFIG_DSI_SEL);
+		writel_relaxed(DPI_SEL_IN_BLS,
+			       config_regs + DISP_REG_CONFIG_DPI_SEL);
+	}
+}
+
+void mtk_mmsys_ddp_connect(struct device *dev,
+			   enum mtk_ddp_comp_id cur,
+			   enum mtk_ddp_comp_id next)
+{
+	void __iomem *config_regs = dev_get_drvdata(dev);
+	unsigned int addr, value, reg;
+
+	value = mtk_mmsys_ddp_mout_en(cur, next, &addr);
+	if (value) {
+		reg = readl_relaxed(config_regs + addr) | value;
+		writel_relaxed(reg, config_regs + addr);
+	}
+
+	mtk_mmsys_ddp_sout_sel(config_regs, cur, next);
+
+	value = mtk_mmsys_ddp_sel_in(cur, next, &addr);
+	if (value) {
+		reg = readl_relaxed(config_regs + addr) | value;
+		writel_relaxed(reg, config_regs + addr);
+	}
+}
+EXPORT_SYMBOL_GPL(mtk_mmsys_ddp_connect);
+
+void mtk_mmsys_ddp_disconnect(struct device *dev,
+			      enum mtk_ddp_comp_id cur,
+			      enum mtk_ddp_comp_id next)
+{
+	void __iomem *config_regs = dev_get_drvdata(dev);
+	unsigned int addr, value, reg;
+
+	value = mtk_mmsys_ddp_mout_en(cur, next, &addr);
+	if (value) {
+		reg = readl_relaxed(config_regs + addr) & ~value;
+		writel_relaxed(reg, config_regs + addr);
+	}
+
+	value = mtk_mmsys_ddp_sel_in(cur, next, &addr);
+	if (value) {
+		reg = readl_relaxed(config_regs + addr) & ~value;
+		writel_relaxed(reg, config_regs + addr);
+	}
+}
+EXPORT_SYMBOL_GPL(mtk_mmsys_ddp_disconnect);
+
+static int mtk_mmsys_probe(struct platform_device *pdev)
+{
+	const struct mtk_mmsys_driver_data *data;
+	struct device *dev = &pdev->dev;
+	struct platform_device *clks;
+	struct platform_device *drm;
+	void __iomem *config_regs;
+	struct resource *mem;
+	int ret;
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	config_regs = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(config_regs)) {
+		ret = PTR_ERR(config_regs);
+		dev_err(dev, "Failed to ioremap mmsys-config resource: %d\n",
+			ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, config_regs);
+
+	data = of_device_get_match_data(&pdev->dev);
+
+	clks = platform_device_register_data(&pdev->dev, data->clk_driver,
+					     PLATFORM_DEVID_AUTO, NULL, 0);
+	if (IS_ERR(clks))
+		return PTR_ERR(clks);
+
+	drm = platform_device_register_data(&pdev->dev, "mediatek-drm",
+					    PLATFORM_DEVID_AUTO, NULL, 0);
+	if (IS_ERR(drm)) {
+		platform_device_unregister(clks);
+		return PTR_ERR(drm);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id of_match_mtk_mmsys[] = {
+	{
+		.compatible = "mediatek,mt2701-mmsys",
+		.data = &mt2701_mmsys_driver_data,
+	},
+	{
+		.compatible = "mediatek,mt2712-mmsys",
+		.data = &mt2712_mmsys_driver_data,
+	},
+	{
+		.compatible = "mediatek,mt6779-mmsys",
+		.data = &mt6779_mmsys_driver_data,
+	},
+	{
+		.compatible = "mediatek,mt6797-mmsys",
+		.data = &mt6797_mmsys_driver_data,
+	},
+	{
+		.compatible = "mediatek,mt8173-mmsys",
+		.data = &mt8173_mmsys_driver_data,
+	},
+	{
+		.compatible = "mediatek,mt8183-mmsys",
+		.data = &mt8183_mmsys_driver_data,
+	},
+	{ }
+};
+
+static struct platform_driver mtk_mmsys_drv = {
+	.driver = {
+		.name = "mtk-mmsys",
+		.of_match_table = of_match_mtk_mmsys,
+	},
+	.probe = mtk_mmsys_probe,
+};
+
+builtin_platform_driver(mtk_mmsys_drv);
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index 285baa7e474e..07bb261a63d2 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -35,15 +35,6 @@ config QCOM_GENI_SE
 	  driver is also used to manage the common aspects of multiple Serial
 	  Engines present in the QUP.
 
-config QCOM_GLINK_SSR
-	tristate "Qualcomm Glink SSR driver"
-	depends on RPMSG
-	depends on QCOM_RPROC_COMMON
-	help
-	  Say y here to enable GLINK SSR support. The GLINK SSR driver
-	  implements the SSR protocol for notifying the remote processor about
-	  neighboring subsystems going up or down.
-
 config QCOM_GSBI
 	tristate "QCOM General Serial Bus Interface"
 	depends on ARCH_QCOM || COMPILE_TEST
@@ -107,7 +98,7 @@ config QCOM_RPMH
 	  help apply the aggregated state on the resource.
 
 config QCOM_RPMHPD
-	bool "Qualcomm RPMh Power domain driver"
+	tristate "Qualcomm RPMh Power domain driver"
 	depends on QCOM_RPMH && QCOM_COMMAND_DB
 	help
 	  QCOM RPMh Power domain driver to support power-domains with
@@ -116,8 +107,8 @@ config QCOM_RPMHPD
 	  for the voltage rail.
 
 config QCOM_RPMPD
-	bool "Qualcomm RPM Power domain driver"
-	depends on QCOM_SMD_RPM=y
+	tristate "Qualcomm RPM Power domain driver"
+	depends on QCOM_SMD_RPM
 	help
 	  QCOM RPM Power domain driver to support power-domains with
 	  performance states. The driver communicates a performance state
diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile
index 92cc4232d72c..7d7e2ecbdce6 100644
--- a/drivers/soc/qcom/Makefile
+++ b/drivers/soc/qcom/Makefile
@@ -3,7 +3,6 @@ CFLAGS_rpmh-rsc.o := -I$(src)
 obj-$(CONFIG_QCOM_AOSS_QMP) +=	qcom_aoss.o
 obj-$(CONFIG_QCOM_GENI_SE) +=	qcom-geni-se.o
 obj-$(CONFIG_QCOM_COMMAND_DB) += cmd-db.o
-obj-$(CONFIG_QCOM_GLINK_SSR) +=	glink_ssr.o
 obj-$(CONFIG_QCOM_GSBI)	+=	qcom_gsbi.o
 obj-$(CONFIG_QCOM_MDT_LOADER)	+= mdt_loader.o
 obj-$(CONFIG_QCOM_OCMEM)	+= ocmem.o
diff --git a/drivers/soc/qcom/cmd-db.c b/drivers/soc/qcom/cmd-db.c
index f6c3d17b05c7..fc5610603b17 100644
--- a/drivers/soc/qcom/cmd-db.c
+++ b/drivers/soc/qcom/cmd-db.c
@@ -1,12 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. */
 
+#include <linux/debugfs.h>
 #include <linux/kernel.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/of_platform.h>
 #include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
+#include <linux/seq_file.h>
 #include <linux/types.h>
 
 #include <soc/qcom/cmd-db.h>
@@ -236,6 +237,77 @@ enum cmd_db_hw_type cmd_db_read_slave_id(const char *id)
 }
 EXPORT_SYMBOL(cmd_db_read_slave_id);
 
+#ifdef CONFIG_DEBUG_FS
+static int cmd_db_debugfs_dump(struct seq_file *seq, void *p)
+{
+	int i, j;
+	const struct rsc_hdr *rsc;
+	const struct entry_header *ent;
+	const char *name;
+	u16 len, version;
+	u8 major, minor;
+
+	seq_puts(seq, "Command DB DUMP\n");
+
+	for (i = 0; i < MAX_SLV_ID; i++) {
+		rsc = &cmd_db_header->header[i];
+		if (!rsc->slv_id)
+			break;
+
+		switch (le16_to_cpu(rsc->slv_id)) {
+		case CMD_DB_HW_ARC:
+			name = "ARC";
+			break;
+		case CMD_DB_HW_VRM:
+			name = "VRM";
+			break;
+		case CMD_DB_HW_BCM:
+			name = "BCM";
+			break;
+		default:
+			name = "Unknown";
+			break;
+		}
+
+		version = le16_to_cpu(rsc->version);
+		major = version >> 8;
+		minor = version;
+
+		seq_printf(seq, "Slave %s (v%u.%u)\n", name, major, minor);
+		seq_puts(seq, "-------------------------\n");
+
+		ent = rsc_to_entry_header(rsc);
+		for (j = 0; j < le16_to_cpu(rsc->cnt); j++, ent++) {
+			seq_printf(seq, "0x%05x: %*pEp", le32_to_cpu(ent->addr),
+				   (int)sizeof(ent->id), ent->id);
+
+			len = le16_to_cpu(ent->len);
+			if (len) {
+				seq_printf(seq, " [%*ph]",
+					   len, rsc_offset(rsc, ent));
+			}
+			seq_putc(seq, '\n');
+		}
+	}
+
+	return 0;
+}
+
+static int open_cmd_db_debugfs(struct inode *inode, struct file *file)
+{
+	return single_open(file, cmd_db_debugfs_dump, inode->i_private);
+}
+#endif
+
+static const struct file_operations cmd_db_debugfs_ops = {
+#ifdef CONFIG_DEBUG_FS
+	.open = open_cmd_db_debugfs,
+#endif
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
 static int cmd_db_dev_probe(struct platform_device *pdev)
 {
 	struct reserved_mem *rmem;
@@ -259,12 +331,14 @@ static int cmd_db_dev_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	debugfs_create_file("cmd-db", 0400, NULL, NULL, &cmd_db_debugfs_ops);
+
 	return 0;
 }
 
 static const struct of_device_id cmd_db_match_table[] = {
 	{ .compatible = "qcom,cmd-db" },
-	{ },
+	{ }
 };
 
 static struct platform_driver cmd_db_dev_driver = {
diff --git a/drivers/soc/qcom/pdr_interface.c b/drivers/soc/qcom/pdr_interface.c
index 17ad3b8698e1..bdcf16f88a97 100644
--- a/drivers/soc/qcom/pdr_interface.c
+++ b/drivers/soc/qcom/pdr_interface.c
@@ -155,10 +155,6 @@ static int pdr_register_listener(struct pdr_handle *pdr,
 		return ret;
 	}
 
-	if ((int)resp.curr_state < INT_MIN || (int)resp.curr_state > INT_MAX)
-		pr_err("PDR: %s notification state invalid: 0x%x\n",
-		       pds->service_path, resp.curr_state);
-
 	pds->state = resp.curr_state;
 
 	return 0;
diff --git a/drivers/soc/qcom/qcom_aoss.c b/drivers/soc/qcom/qcom_aoss.c
index f43a2e07ee83..ed2c687c16b3 100644
--- a/drivers/soc/qcom/qcom_aoss.c
+++ b/drivers/soc/qcom/qcom_aoss.c
@@ -599,6 +599,7 @@ static const struct of_device_id qmp_dt_match[] = {
 	{ .compatible = "qcom,sc7180-aoss-qmp", },
 	{ .compatible = "qcom,sdm845-aoss-qmp", },
 	{ .compatible = "qcom,sm8150-aoss-qmp", },
+	{ .compatible = "qcom,sm8250-aoss-qmp", },
 	{}
 };
 MODULE_DEVICE_TABLE(of, qmp_dt_match);
diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h
index 6eec32b97f83..ef60e790a750 100644
--- a/drivers/soc/qcom/rpmh-internal.h
+++ b/drivers/soc/qcom/rpmh-internal.h
@@ -22,16 +22,23 @@ struct rsc_drv;
  * struct tcs_group: group of Trigger Command Sets (TCS) to send state requests
  * to the controller
  *
- * @drv:       the controller
- * @type:      type of the TCS in this group - active, sleep, wake
- * @mask:      mask of the TCSes relative to all the TCSes in the RSC
- * @offset:    start of the TCS group relative to the TCSes in the RSC
- * @num_tcs:   number of TCSes in this type
- * @ncpt:      number of commands in each TCS
- * @lock:      lock for synchronizing this TCS writes
- * @req:       requests that are sent from the TCS
- * @cmd_cache: flattened cache of cmds in sleep/wake TCS
- * @slots:     indicates which of @cmd_addr are occupied
+ * @drv:       The controller.
+ * @type:      Type of the TCS in this group - active, sleep, wake.
+ * @mask:      Mask of the TCSes relative to all the TCSes in the RSC.
+ * @offset:    Start of the TCS group relative to the TCSes in the RSC.
+ * @num_tcs:   Number of TCSes in this type.
+ * @ncpt:      Number of commands in each TCS.
+ * @req:       Requests that are sent from the TCS; only used for ACTIVE_ONLY
+ *             transfers (could be on a wake/sleep TCS if we are borrowing for
+ *             an ACTIVE_ONLY transfer).
+ *             Start: grab drv->lock, set req, set tcs_in_use, drop drv->lock,
+ *                    trigger
+ *             End: get irq, access req,
+ *                  grab drv->lock, clear tcs_in_use, drop drv->lock
+ * @slots:     Indicates which of @cmd_addr are occupied; only used for
+ *             SLEEP / WAKE TCSs.  Things are tightly packed in the
+ *             case that (ncpt < MAX_CMDS_PER_TCS).  That is if ncpt = 2 and
+ *             MAX_CMDS_PER_TCS = 16 then bit[2] = the first bit in 2nd TCS.
  */
 struct tcs_group {
 	struct rsc_drv *drv;
@@ -40,9 +47,7 @@ struct tcs_group {
 	u32 offset;
 	int num_tcs;
 	int ncpt;
-	spinlock_t lock;
 	const struct tcs_request *req[MAX_TCS_PER_TYPE];
-	u32 *cmd_cache;
 	DECLARE_BITMAP(slots, MAX_TCS_SLOTS);
 };
 
@@ -84,20 +89,32 @@ struct rpmh_ctrlr {
  * struct rsc_drv: the Direct Resource Voter (DRV) of the
  * Resource State Coordinator controller (RSC)
  *
- * @name:       controller identifier
- * @tcs_base:   start address of the TCS registers in this controller
- * @id:         instance id in the controller (Direct Resource Voter)
- * @num_tcs:    number of TCSes in this DRV
- * @tcs:        TCS groups
- * @tcs_in_use: s/w state of the TCS
- * @lock:       synchronize state of the controller
- * @client:     handle to the DRV's client.
+ * @name:               Controller identifier.
+ * @tcs_base:           Start address of the TCS registers in this controller.
+ * @id:                 Instance id in the controller (Direct Resource Voter).
+ * @num_tcs:            Number of TCSes in this DRV.
+ * @rsc_pm:             CPU PM notifier for controller.
+ *                      Used when solver mode is not present.
+ * @cpus_in_pm:         Number of CPUs not in idle power collapse.
+ *                      Used when solver mode is not present.
+ * @tcs:                TCS groups.
+ * @tcs_in_use:         S/W state of the TCS; only set for ACTIVE_ONLY
+ *                      transfers, but might show a sleep/wake TCS in use if
+ *                      it was borrowed for an active_only transfer.  You
+ *                      must hold the lock in this struct (AKA drv->lock) in
+ *                      order to update this.
+ * @lock:               Synchronize state of the controller.  If RPMH's cache
+ *                      lock will also be held, the order is: drv->lock then
+ *                      cache_lock.
+ * @client:             Handle to the DRV's client.
  */
 struct rsc_drv {
 	const char *name;
 	void __iomem *tcs_base;
 	int id;
 	int num_tcs;
+	struct notifier_block rsc_pm;
+	atomic_t cpus_in_pm;
 	struct tcs_group tcs[TCS_TYPE_NR];
 	DECLARE_BITMAP(tcs_in_use, MAX_TCS_NR);
 	spinlock_t lock;
@@ -107,7 +124,7 @@ struct rsc_drv {
 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg);
 int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv,
 			     const struct tcs_request *msg);
-int rpmh_rsc_invalidate(struct rsc_drv *drv);
+void rpmh_rsc_invalidate(struct rsc_drv *drv);
 
 void rpmh_tx_done(const struct tcs_request *msg, int r);
 int rpmh_flush(struct rpmh_ctrlr *ctrlr);
diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c
index b71822131f59..076fd27f3081 100644
--- a/drivers/soc/qcom/rpmh-rsc.c
+++ b/drivers/soc/qcom/rpmh-rsc.c
@@ -6,9 +6,11 @@
 #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
 
 #include <linux/atomic.h>
+#include <linux/cpu_pm.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/of.h>
@@ -30,21 +32,41 @@
 #define RSC_DRV_TCS_OFFSET		672
 #define RSC_DRV_CMD_OFFSET		20
 
-/* DRV Configuration Information Register */
+/* DRV HW Solver Configuration Information Register */
+#define DRV_SOLVER_CONFIG		0x04
+#define DRV_HW_SOLVER_MASK		1
+#define DRV_HW_SOLVER_SHIFT		24
+
+/* DRV TCS Configuration Information Register */
 #define DRV_PRNT_CHLD_CONFIG		0x0C
 #define DRV_NUM_TCS_MASK		0x3F
 #define DRV_NUM_TCS_SHIFT		6
 #define DRV_NCPT_MASK			0x1F
 #define DRV_NCPT_SHIFT			27
 
-/* Register offsets */
+/* Offsets for common TCS Registers, one bit per TCS */
 #define RSC_DRV_IRQ_ENABLE		0x00
 #define RSC_DRV_IRQ_STATUS		0x04
-#define RSC_DRV_IRQ_CLEAR		0x08
-#define RSC_DRV_CMD_WAIT_FOR_CMPL	0x10
+#define RSC_DRV_IRQ_CLEAR		0x08	/* w/o; write 1 to clear */
+
+/*
+ * Offsets for per TCS Registers.
+ *
+ * TCSes start at 0x10 from tcs_base and are stored one after another.
+ * Multiply tcs_id by RSC_DRV_TCS_OFFSET to find a given TCS and add one
+ * of the below to find a register.
+ */
+#define RSC_DRV_CMD_WAIT_FOR_CMPL	0x10	/* 1 bit per command */
 #define RSC_DRV_CONTROL			0x14
-#define RSC_DRV_STATUS			0x18
-#define RSC_DRV_CMD_ENABLE		0x1C
+#define RSC_DRV_STATUS			0x18	/* zero if tcs is busy */
+#define RSC_DRV_CMD_ENABLE		0x1C	/* 1 bit per command */
+
+/*
+ * Offsets for per command in a TCS.
+ *
+ * Commands (up to 16) start at 0x30 in a TCS; multiply command index
+ * by RSC_DRV_CMD_OFFSET and add one of the below to find a register.
+ */
 #define RSC_DRV_CMD_MSGID		0x30
 #define RSC_DRV_CMD_ADDR		0x34
 #define RSC_DRV_CMD_DATA		0x38
@@ -61,94 +83,179 @@
 #define CMD_STATUS_ISSUED		BIT(8)
 #define CMD_STATUS_COMPL		BIT(16)
 
-static u32 read_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
+/*
+ * Here's a high level overview of how all the registers in RPMH work
+ * together:
+ *
+ * - The main rpmh-rsc address is the base of a register space that can
+ *   be used to find overall configuration of the hardware
+ *   (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
+ *   space are all the TCS blocks. The offset of the TCS blocks is
+ *   specified in the device tree by "qcom,tcs-offset" and used to
+ *   compute tcs_base.
+ * - TCS blocks come one after another. Type, count, and order are
+ *   specified by the device tree as "qcom,tcs-config".
+ * - Each TCS block has some registers, then space for up to 16 commands.
+ *   Note that though address space is reserved for 16 commands, fewer
+ *   might be present. See ncpt (num cmds per TCS).
+ *
+ * Here's a picture:
+ *
+ *  +---------------------------------------------------+
+ *  |RSC                                                |
+ *  | ctrl                                              |
+ *  |                                                   |
+ *  | Drvs:                                             |
+ *  | +-----------------------------------------------+ |
+ *  | |DRV0                                           | |
+ *  | | ctrl/config                                   | |
+ *  | | IRQ                                           | |
+ *  | |                                               | |
+ *  | | TCSes:                                        | |
+ *  | | +------------------------------------------+  | |
+ *  | | |TCS0  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
+ *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
+ *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
+ *  | | +------------------------------------------+  | |
+ *  | | +------------------------------------------+  | |
+ *  | | |TCS1  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
+ *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
+ *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
+ *  | | +------------------------------------------+  | |
+ *  | | +------------------------------------------+  | |
+ *  | | |TCS2  |  |  |  |  |  |  |  |  |  |  |  |  |  | |
+ *  | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15|  | |
+ *  | | |      |  |  |  |  |  |  |  |  |  |  |  |  |  | |
+ *  | | +------------------------------------------+  | |
+ *  | |                    ......                     | |
+ *  | +-----------------------------------------------+ |
+ *  | +-----------------------------------------------+ |
+ *  | |DRV1                                           | |
+ *  | | (same as DRV0)                                | |
+ *  | +-----------------------------------------------+ |
+ *  |                      ......                       |
+ *  +---------------------------------------------------+
+ */
+
+static inline void __iomem *
+tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
 {
-	return readl_relaxed(drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id +
-			     RSC_DRV_CMD_OFFSET * cmd_id);
+	return drv->tcs_base + RSC_DRV_TCS_OFFSET * tcs_id + reg;
 }
 
-static void write_tcs_cmd(struct rsc_drv *drv, int reg, int tcs_id, int cmd_id,
-			  u32 data)
+static inline void __iomem *
+tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
 {
-	writel_relaxed(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id +
-		       RSC_DRV_CMD_OFFSET * cmd_id);
+	return tcs_reg_addr(drv, reg, tcs_id) + RSC_DRV_CMD_OFFSET * cmd_id;
 }
 
-static void write_tcs_reg(struct rsc_drv *drv, int reg, int tcs_id, u32 data)
+static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
+			int cmd_id)
 {
-	writel_relaxed(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id);
+	return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
 }
 
-static void write_tcs_reg_sync(struct rsc_drv *drv, int reg, int tcs_id,
-			       u32 data)
+static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
 {
-	writel(data, drv->tcs_base + reg + RSC_DRV_TCS_OFFSET * tcs_id);
-	for (;;) {
-		if (data == readl(drv->tcs_base + reg +
-				  RSC_DRV_TCS_OFFSET * tcs_id))
-			break;
-		udelay(1);
-	}
+	return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
 }
 
-static bool tcs_is_free(struct rsc_drv *drv, int tcs_id)
+static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
+			  int cmd_id, u32 data)
 {
-	return !test_bit(tcs_id, drv->tcs_in_use) &&
-	       read_tcs_reg(drv, RSC_DRV_STATUS, tcs_id, 0);
+	writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
 }
 
-static struct tcs_group *get_tcs_of_type(struct rsc_drv *drv, int type)
+static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
+			  u32 data)
 {
-	return &drv->tcs[type];
+	writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
 }
 
-static int tcs_invalidate(struct rsc_drv *drv, int type)
+static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
+			       u32 data)
 {
-	int m;
-	struct tcs_group *tcs;
+	u32 new_data;
 
-	tcs = get_tcs_of_type(drv, type);
+	writel(data, tcs_reg_addr(drv, reg, tcs_id));
+	if (readl_poll_timeout_atomic(tcs_reg_addr(drv, reg, tcs_id), new_data,
+				      new_data == data, 1, USEC_PER_SEC))
+		pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
+		       data, tcs_id, reg);
+}
 
-	spin_lock(&tcs->lock);
-	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS)) {
-		spin_unlock(&tcs->lock);
-		return 0;
-	}
+/**
+ * tcs_is_free() - Return if a TCS is totally free.
+ * @drv:    The RSC controller.
+ * @tcs_id: The global ID of this TCS.
+ *
+ * Returns true if nobody has claimed this TCS (by setting tcs_in_use).
+ *
+ * Context: Must be called with the drv->lock held.
+ *
+ * Return: true if the given TCS is free.
+ */
+static bool tcs_is_free(struct rsc_drv *drv, int tcs_id)
+{
+	return !test_bit(tcs_id, drv->tcs_in_use);
+}
+
+/**
+ * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
+ * @drv:  The RSC controller.
+ * @type: SLEEP_TCS or WAKE_TCS
+ *
+ * This will clear the "slots" variable of the given tcs_group and also
+ * tell the hardware to forget about all entries.
+ *
+ * The caller must ensure that no other RPMH actions are happening when this
+ * function is called, since otherwise the device may immediately become
+ * used again even before this function exits.
+ */
+static void tcs_invalidate(struct rsc_drv *drv, int type)
+{
+	int m;
+	struct tcs_group *tcs = &drv->tcs[type];
+
+	/* Caller ensures nobody else is running so no lock */
+	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))
+		return;
 
 	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) {
-		if (!tcs_is_free(drv, m)) {
-			spin_unlock(&tcs->lock);
-			return -EAGAIN;
-		}
 		write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, m, 0);
 		write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, m, 0);
 	}
 	bitmap_zero(tcs->slots, MAX_TCS_SLOTS);
-	spin_unlock(&tcs->lock);
-
-	return 0;
 }
 
 /**
- * rpmh_rsc_invalidate - Invalidate sleep and wake TCSes
+ * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
+ * @drv: The RSC controller.
  *
- * @drv: the RSC controller
+ * The caller must ensure that no other RPMH actions are happening when this
+ * function is called, since otherwise the device may immediately become
+ * used again even before this function exits.
  */
-int rpmh_rsc_invalidate(struct rsc_drv *drv)
+void rpmh_rsc_invalidate(struct rsc_drv *drv)
 {
-	int ret;
-
-	ret = tcs_invalidate(drv, SLEEP_TCS);
-	if (!ret)
-		ret = tcs_invalidate(drv, WAKE_TCS);
-
-	return ret;
+	tcs_invalidate(drv, SLEEP_TCS);
+	tcs_invalidate(drv, WAKE_TCS);
 }
 
+/**
+ * get_tcs_for_msg() - Get the tcs_group used to send the given message.
+ * @drv: The RSC controller.
+ * @msg: The message we want to send.
+ *
+ * This is normally pretty straightforward except if we are trying to send
+ * an ACTIVE_ONLY message but don't have any active_only TCSes.
+ *
+ * Return: A pointer to a tcs_group or an ERR_PTR.
+ */
 static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
 					 const struct tcs_request *msg)
 {
-	int type, ret;
+	int type;
 	struct tcs_group *tcs;
 
 	switch (msg->state) {
@@ -168,24 +275,33 @@ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
 	/*
 	 * If we are making an active request on a RSC that does not have a
 	 * dedicated TCS for active state use, then re-purpose a wake TCS to
-	 * send active votes.
-	 * NOTE: The driver must be aware that this RSC does not have a
-	 * dedicated AMC, and therefore would invalidate the sleep and wake
-	 * TCSes before making an active state request.
+	 * send active votes. This is safe because we ensure any active-only
+	 * transfers have finished before we use it (maybe by running from
+	 * the last CPU in PM code).
 	 */
-	tcs = get_tcs_of_type(drv, type);
-	if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) {
-		tcs = get_tcs_of_type(drv, WAKE_TCS);
-		if (tcs->num_tcs) {
-			ret = rpmh_rsc_invalidate(drv);
-			if (ret)
-				return ERR_PTR(ret);
-		}
-	}
+	tcs = &drv->tcs[type];
+	if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
+		tcs = &drv->tcs[WAKE_TCS];
 
 	return tcs;
 }
 
+/**
+ * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
+ * @drv:    The RSC controller.
+ * @tcs_id: The global ID of this TCS.
+ *
+ * For ACTIVE_ONLY transfers we want to call back into the client when the
+ * transfer finishes. To do this we need the "request" that the client
+ * originally provided us. This function grabs the request that we stashed
+ * when we started the transfer.
+ *
+ * This only makes sense for ACTIVE_ONLY transfers since those are the only
+ * ones we track sending (the only ones we enable interrupts for and the only
+ * ones we call back to the client for).
+ *
+ * Return: The stashed request.
+ */
 static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
 						  int tcs_id)
 {
@@ -202,7 +318,76 @@ static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
 }
 
 /**
- * tcs_tx_done: TX Done interrupt handler
+ * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
+ * @drv:     The controller.
+ * @tcs_id:  The global ID of this TCS.
+ * @trigger: If true then untrigger/retrigger. If false then just untrigger.
+ *
+ * In the normal case we only ever call with "trigger=true" to start a
+ * transfer. That will un-trigger/disable the TCS from the last transfer
+ * then trigger/enable for this transfer.
+ *
+ * If we borrowed a wake TCS for an active-only transfer we'll also call
+ * this function with "trigger=false" to just do the un-trigger/disable
+ * before using the TCS for wake purposes again.
+ *
+ * Note that the AP is only in charge of triggering active-only transfers.
+ * The AP never triggers sleep/wake values using this function.
+ */
+static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
+{
+	u32 enable;
+
+	/*
+	 * HW req: Clear the DRV_CONTROL and enable TCS again
+	 * While clearing ensure that the AMC mode trigger is cleared
+	 * and then the mode enable is cleared.
+	 */
+	enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id);
+	enable &= ~TCS_AMC_MODE_TRIGGER;
+	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
+	enable &= ~TCS_AMC_MODE_ENABLE;
+	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
+
+	if (trigger) {
+		/* Enable the AMC mode on the TCS and then trigger the TCS */
+		enable = TCS_AMC_MODE_ENABLE;
+		write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
+		enable |= TCS_AMC_MODE_TRIGGER;
+		write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
+	}
+}
+
+/**
+ * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
+ * @drv:     The controller.
+ * @tcs_id:  The global ID of this TCS.
+ * @enable:  If true then enable; if false then disable
+ *
+ * We only ever call this when we borrow a wake TCS for an active-only
+ * transfer. For active-only TCSes interrupts are always left enabled.
+ */
+static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
+{
+	u32 data;
+
+	data = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_ENABLE);
+	if (enable)
+		data |= BIT(tcs_id);
+	else
+		data &= ~BIT(tcs_id);
+	writel_relaxed(data, drv->tcs_base + RSC_DRV_IRQ_ENABLE);
+}
+
+/**
+ * tcs_tx_done() - TX Done interrupt handler.
+ * @irq: The IRQ number (ignored).
+ * @p:   Pointer to "struct rsc_drv".
+ *
+ * Called for ACTIVE_ONLY transfers (those are the only ones we enable the
+ * IRQ for) when a transfer is done.
+ *
+ * Return: IRQ_HANDLED
  */
 static irqreturn_t tcs_tx_done(int irq, void *p)
 {
@@ -212,7 +397,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p)
 	const struct tcs_request *req;
 	struct tcs_cmd *cmd;
 
-	irq_status = read_tcs_reg(drv, RSC_DRV_IRQ_STATUS, 0, 0);
+	irq_status = readl_relaxed(drv->tcs_base + RSC_DRV_IRQ_STATUS);
 
 	for_each_set_bit(i, &irq_status, BITS_PER_LONG) {
 		req = get_req_from_tcs(drv, i);
@@ -226,7 +411,7 @@ static irqreturn_t tcs_tx_done(int irq, void *p)
 			u32 sts;
 
 			cmd = &req->cmds[j];
-			sts = read_tcs_reg(drv, RSC_DRV_CMD_STATUS, i, j);
+			sts = read_tcs_cmd(drv, RSC_DRV_CMD_STATUS, i, j);
 			if (!(sts & CMD_STATUS_ISSUED) ||
 			   ((req->wait_for_compl || cmd->wait) &&
 			   !(sts & CMD_STATUS_COMPL))) {
@@ -237,13 +422,28 @@ static irqreturn_t tcs_tx_done(int irq, void *p)
 		}
 
 		trace_rpmh_tx_done(drv, i, req, err);
+
+		/*
+		 * If wake tcs was re-purposed for sending active
+		 * votes, clear AMC trigger & enable modes and
+		 * disable interrupt for this TCS
+		 */
+		if (!drv->tcs[ACTIVE_TCS].num_tcs)
+			__tcs_set_trigger(drv, i, false);
 skip:
 		/* Reclaim the TCS */
 		write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, i, 0);
 		write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, i, 0);
-		write_tcs_reg(drv, RSC_DRV_IRQ_CLEAR, 0, BIT(i));
+		writel_relaxed(BIT(i), drv->tcs_base + RSC_DRV_IRQ_CLEAR);
 		spin_lock(&drv->lock);
 		clear_bit(i, drv->tcs_in_use);
+		/*
+		 * Disable interrupt for WAKE TCS to avoid being
+		 * spammed with interrupts coming when the solver
+		 * sends its wake votes.
+		 */
+		if (!drv->tcs[ACTIVE_TCS].num_tcs)
+			enable_tcs_irq(drv, i, false);
 		spin_unlock(&drv->lock);
 		if (req)
 			rpmh_tx_done(req, err);
@@ -252,6 +452,16 @@ skip:
 	return IRQ_HANDLED;
 }
 
+/**
+ * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
+ * @drv:    The controller.
+ * @tcs_id: The global ID of this TCS.
+ * @cmd_id: The index within the TCS to start writing.
+ * @msg:    The message we want to send, which will contain several addr/data
+ *          pairs to program (but few enough that they all fit in one TCS).
+ *
+ * This is used for all types of transfers (active, sleep, and wake).
+ */
 static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
 			       const struct tcs_request *msg)
 {
@@ -265,7 +475,7 @@ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
 	cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
 	cmd_msgid |= CMD_MSGID_WRITE;
 
-	cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0);
+	cmd_complete = read_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id);
 
 	for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
 		cmd = &msg->cmds[i];
@@ -281,32 +491,30 @@ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
 	}
 
 	write_tcs_reg(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, cmd_complete);
-	cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);
+	cmd_enable |= read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);
 	write_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, cmd_enable);
 }
 
-static void __tcs_trigger(struct rsc_drv *drv, int tcs_id)
-{
-	u32 enable;
-
-	/*
-	 * HW req: Clear the DRV_CONTROL and enable TCS again
-	 * While clearing ensure that the AMC mode trigger is cleared
-	 * and then the mode enable is cleared.
-	 */
-	enable = read_tcs_reg(drv, RSC_DRV_CONTROL, tcs_id, 0);
-	enable &= ~TCS_AMC_MODE_TRIGGER;
-	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
-	enable &= ~TCS_AMC_MODE_ENABLE;
-	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
-
-	/* Enable the AMC mode on the TCS and then trigger the TCS */
-	enable = TCS_AMC_MODE_ENABLE;
-	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
-	enable |= TCS_AMC_MODE_TRIGGER;
-	write_tcs_reg_sync(drv, RSC_DRV_CONTROL, tcs_id, enable);
-}
-
+/**
+ * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
+ * @drv: The controller.
+ * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
+ * @msg: The message we want to send, which will contain several addr/data
+ *       pairs to program (but few enough that they all fit in one TCS).
+ *
+ * This will walk through the TCSes in the group and check if any of them
+ * appear to be sending to addresses referenced in the message. If it finds
+ * one it'll return -EBUSY.
+ *
+ * Only for use for active-only transfers.
+ *
+ * Must be called with the drv->lock held since that protects tcs_in_use.
+ *
+ * Return: 0 if nothing in flight or -EBUSY if we should try again later.
+ *         The caller must re-enable interrupts between tries since that's
+ *         the only way tcs_is_free() will ever return true and the only way
+ *         RSC_DRV_CMD_ENABLE will ever be cleared.
+ */
 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
 				  const struct tcs_request *msg)
 {
@@ -319,10 +527,10 @@ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
 		if (tcs_is_free(drv, tcs_id))
 			continue;
 
-		curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);
+		curr_enabled = read_tcs_reg(drv, RSC_DRV_CMD_ENABLE, tcs_id);
 
 		for_each_set_bit(j, &curr_enabled, MAX_CMDS_PER_TCS) {
-			addr = read_tcs_reg(drv, RSC_DRV_CMD_ADDR, tcs_id, j);
+			addr = read_tcs_cmd(drv, RSC_DRV_CMD_ADDR, tcs_id, j);
 			for (k = 0; k < msg->num_cmds; k++) {
 				if (addr == msg->cmds[k].addr)
 					return -EBUSY;
@@ -333,6 +541,15 @@ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
 	return 0;
 }
 
+/**
+ * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
+ * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
+ *       we borrowed it because there are zero active-only ones).
+ *
+ * Must be called with the drv->lock held since that protects tcs_in_use.
+ *
+ * Return: The first tcs that's free.
+ */
 static int find_free_tcs(struct tcs_group *tcs)
 {
 	int i;
@@ -345,6 +562,20 @@ static int find_free_tcs(struct tcs_group *tcs)
 	return -EBUSY;
 }
 
+/**
+ * tcs_write() - Store messages into a TCS right now, or return -EBUSY.
+ * @drv: The controller.
+ * @msg: The data to be sent.
+ *
+ * Grabs a TCS for ACTIVE_ONLY transfers and writes the messages to it.
+ *
+ * If there are no free TCSes for ACTIVE_ONLY transfers or if a command for
+ * the same address is already transferring returns -EBUSY which means the
+ * client should retry shortly.
+ *
+ * Return: 0 on success, -EBUSY if client should retry, or an error.
+ *         Client should have interrupts enabled for a bit before retrying.
+ */
 static int tcs_write(struct rsc_drv *drv, const struct tcs_request *msg)
 {
 	struct tcs_group *tcs;
@@ -356,57 +587,77 @@ static int tcs_write(struct rsc_drv *drv, const struct tcs_request *msg)
 	if (IS_ERR(tcs))
 		return PTR_ERR(tcs);
 
-	spin_lock_irqsave(&tcs->lock, flags);
-	spin_lock(&drv->lock);
+	spin_lock_irqsave(&drv->lock, flags);
 	/*
 	 * The h/w does not like if we send a request to the same address,
 	 * when one is already in-flight or being processed.
 	 */
 	ret = check_for_req_inflight(drv, tcs, msg);
-	if (ret) {
-		spin_unlock(&drv->lock);
-		goto done_write;
-	}
+	if (ret)
+		goto unlock;
 
-	tcs_id = find_free_tcs(tcs);
-	if (tcs_id < 0) {
-		ret = tcs_id;
-		spin_unlock(&drv->lock);
-		goto done_write;
-	}
+	ret = find_free_tcs(tcs);
+	if (ret < 0)
+		goto unlock;
+	tcs_id = ret;
 
 	tcs->req[tcs_id - tcs->offset] = msg;
 	set_bit(tcs_id, drv->tcs_in_use);
-	spin_unlock(&drv->lock);
+	if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {
+		/*
+		 * Clear previously programmed WAKE commands in selected
+		 * repurposed TCS to avoid triggering them. tcs->slots will be
+		 * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate()
+		 */
+		write_tcs_reg_sync(drv, RSC_DRV_CMD_ENABLE, tcs_id, 0);
+		write_tcs_reg_sync(drv, RSC_DRV_CMD_WAIT_FOR_CMPL, tcs_id, 0);
+		enable_tcs_irq(drv, tcs_id, true);
+	}
+	spin_unlock_irqrestore(&drv->lock, flags);
 
+	/*
+	 * These two can be done after the lock is released because:
+	 * - We marked "tcs_in_use" under lock.
+	 * - Once "tcs_in_use" has been marked nobody else could be writing
+	 *   to these registers until the interrupt goes off.
+	 * - The interrupt can't go off until we trigger w/ the last line
+	 *   of __tcs_set_trigger() below.
+	 */
 	__tcs_buffer_write(drv, tcs_id, 0, msg);
-	__tcs_trigger(drv, tcs_id);
+	__tcs_set_trigger(drv, tcs_id, true);
 
-done_write:
-	spin_unlock_irqrestore(&tcs->lock, flags);
+	return 0;
+unlock:
+	spin_unlock_irqrestore(&drv->lock, flags);
 	return ret;
 }
 
 /**
- * rpmh_rsc_send_data: Validate the incoming message and write to the
- * appropriate TCS block.
+ * rpmh_rsc_send_data() - Write / trigger active-only message.
+ * @drv: The controller.
+ * @msg: The data to be sent.
  *
- * @drv: the controller
- * @msg: the data to be sent
+ * NOTES:
+ * - This is only used for "ACTIVE_ONLY" since the limitations of this
+ *   function don't make sense for sleep/wake cases.
+ * - To do the transfer, we will grab a whole TCS for ourselves--we don't
+ *   try to share. If there are none available we'll wait indefinitely
+ *   for a free one.
+ * - This function will not wait for the commands to be finished, only for
+ *   data to be programmed into the RPMh. See rpmh_tx_done() which will
+ *   be called when the transfer is fully complete.
+ * - This function must be called with interrupts enabled. If the hardware
+ *   is busy doing someone else's transfer we need that transfer to fully
+ *   finish so that we can have the hardware, and to fully finish it needs
+ *   the interrupt handler to run. If the interrupts is set to run on the
+ *   active CPU this can never happen if interrupts are disabled.
  *
  * Return: 0 on success, -EINVAL on error.
- * Note: This call blocks until a valid data is written to the TCS.
  */
 int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
 {
 	int ret;
 
-	if (!msg || !msg->cmds || !msg->num_cmds ||
-	    msg->num_cmds > MAX_RPMH_PAYLOAD) {
-		WARN_ON(1);
-		return -EINVAL;
-	}
-
 	do {
 		ret = tcs_write(drv, msg);
 		if (ret == -EBUSY) {
@@ -419,43 +670,28 @@ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg)
 	return ret;
 }
 
-static int find_match(const struct tcs_group *tcs, const struct tcs_cmd *cmd,
-		      int len)
-{
-	int i, j;
-
-	/* Check for already cached commands */
-	for_each_set_bit(i, tcs->slots, MAX_TCS_SLOTS) {
-		if (tcs->cmd_cache[i] != cmd[0].addr)
-			continue;
-		if (i + len >= tcs->num_tcs * tcs->ncpt)
-			goto seq_err;
-		for (j = 0; j < len; j++) {
-			if (tcs->cmd_cache[i + j] != cmd[j].addr)
-				goto seq_err;
-		}
-		return i;
-	}
-
-	return -ENODATA;
-
-seq_err:
-	WARN(1, "Message does not match previous sequence.\n");
-	return -EINVAL;
-}
-
+/**
+ * find_slots() - Find a place to write the given message.
+ * @tcs:    The tcs group to search.
+ * @msg:    The message we want to find room for.
+ * @tcs_id: If we return 0 from the function, we return the global ID of the
+ *          TCS to write to here.
+ * @cmd_id: If we return 0 from the function, we return the index of
+ *          the command array of the returned TCS where the client should
+ *          start writing the message.
+ *
+ * Only for use on sleep/wake TCSes since those are the only ones we maintain
+ * tcs->slots for.
+ *
+ * Return: -ENOMEM if there was no room, else 0.
+ */
 static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
 		      int *tcs_id, int *cmd_id)
 {
 	int slot, offset;
 	int i = 0;
 
-	/* Find if we already have the msg in our TCS */
-	slot = find_match(tcs, msg->cmds, msg->num_cmds);
-	if (slot >= 0)
-		goto copy_data;
-
-	/* Do over, until we can fit the full payload in a TCS */
+	/* Do over, until we can fit the full payload in a single TCS */
 	do {
 		slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,
 						  i, msg->num_cmds, 0);
@@ -464,11 +700,7 @@ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
 		i += tcs->ncpt;
 	} while (slot + msg->num_cmds - 1 >= i);
 
-copy_data:
 	bitmap_set(tcs->slots, slot, msg->num_cmds);
-	/* Copy the addresses of the resources over to the slots */
-	for (i = 0; i < msg->num_cmds; i++)
-		tcs->cmd_cache[slot + i] = msg->cmds[i].addr;
 
 	offset = slot / tcs->ncpt;
 	*tcs_id = offset + tcs->offset;
@@ -477,52 +709,157 @@ copy_data:
 	return 0;
 }
 
-static int tcs_ctrl_write(struct rsc_drv *drv, const struct tcs_request *msg)
+/**
+ * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
+ * @drv: The controller.
+ * @msg: The data to be written to the controller.
+ *
+ * This should only be called for for sleep/wake state, never active-only
+ * state.
+ *
+ * The caller must ensure that no other RPMH actions are happening and the
+ * controller is idle when this function is called since it runs lockless.
+ *
+ * Return: 0 if no error; else -error.
+ */
+int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
 {
 	struct tcs_group *tcs;
 	int tcs_id = 0, cmd_id = 0;
-	unsigned long flags;
 	int ret;
 
 	tcs = get_tcs_for_msg(drv, msg);
 	if (IS_ERR(tcs))
 		return PTR_ERR(tcs);
 
-	spin_lock_irqsave(&tcs->lock, flags);
 	/* find the TCS id and the command in the TCS to write to */
 	ret = find_slots(tcs, msg, &tcs_id, &cmd_id);
 	if (!ret)
 		__tcs_buffer_write(drv, tcs_id, cmd_id, msg);
-	spin_unlock_irqrestore(&tcs->lock, flags);
 
 	return ret;
 }
 
 /**
- * rpmh_rsc_write_ctrl_data: Write request to the controller
+ * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
+ * @drv: The controller
+ *
+ * Checks if any of the AMCs are busy in handling ACTIVE sets.
+ * This is called from the last cpu powering down before flushing
+ * SLEEP and WAKE sets. If AMCs are busy, controller can not enter
+ * power collapse, so deny from the last cpu's pm notification.
  *
- * @drv: the controller
- * @msg: the data to be written to the controller
+ * Context: Must be called with the drv->lock held.
  *
- * There is no response returned for writing the request to the controller.
+ * Return:
+ * * False		- AMCs are idle
+ * * True		- AMCs are busy
  */
-int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg)
+static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
 {
-	if (!msg || !msg->cmds || !msg->num_cmds ||
-	    msg->num_cmds > MAX_RPMH_PAYLOAD) {
-		pr_err("Payload error\n");
-		return -EINVAL;
+	int m;
+	struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];
+
+	/*
+	 * If we made an active request on a RSC that does not have a
+	 * dedicated TCS for active state use, then re-purposed wake TCSes
+	 * should be checked for not busy, because we used wake TCSes for
+	 * active requests in this case.
+	 */
+	if (!tcs->num_tcs)
+		tcs = &drv->tcs[WAKE_TCS];
+
+	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) {
+		if (!tcs_is_free(drv, m))
+			return true;
 	}
 
-	/* Data sent to this API will not be sent immediately */
-	if (msg->state == RPMH_ACTIVE_ONLY_STATE)
-		return -EINVAL;
+	return false;
+}
+
+/**
+ * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
+ * @nfb:    Pointer to the notifier block in struct rsc_drv.
+ * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
+ * @v:      Unused
+ *
+ * This function is given to cpu_pm_register_notifier so we can be informed
+ * about when CPUs go down. When all CPUs go down we know no more active
+ * transfers will be started so we write sleep/wake sets. This function gets
+ * called from cpuidle code paths and also at system suspend time.
+ *
+ * If its last CPU going down and AMCs are not busy then writes cached sleep
+ * and wake messages to TCSes. The firmware then takes care of triggering
+ * them when entering deepest low power modes.
+ *
+ * Return: See cpu_pm_register_notifier()
+ */
+static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
+				    unsigned long action, void *v)
+{
+	struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
+	int ret = NOTIFY_OK;
+	int cpus_in_pm;
 
-	return tcs_ctrl_write(drv, msg);
+	switch (action) {
+	case CPU_PM_ENTER:
+		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
+		/*
+		 * NOTE: comments for num_online_cpus() point out that it's
+		 * only a snapshot so we need to be careful. It should be OK
+		 * for us to use, though.  It's important for us not to miss
+		 * if we're the last CPU going down so it would only be a
+		 * problem if a CPU went offline right after we did the check
+		 * AND that CPU was not idle AND that CPU was the last non-idle
+		 * CPU. That can't happen. CPUs would have to come out of idle
+		 * before the CPU could go offline.
+		 */
+		if (cpus_in_pm < num_online_cpus())
+			return NOTIFY_OK;
+		break;
+	case CPU_PM_ENTER_FAILED:
+	case CPU_PM_EXIT:
+		atomic_dec(&drv->cpus_in_pm);
+		return NOTIFY_OK;
+	default:
+		return NOTIFY_DONE;
+	}
+
+	/*
+	 * It's likely we're on the last CPU. Grab the drv->lock and write
+	 * out the sleep/wake commands to RPMH hardware. Grabbing the lock
+	 * means that if we race with another CPU coming up we are still
+	 * guaranteed to be safe. If another CPU came up just after we checked
+	 * and has grabbed the lock or started an active transfer then we'll
+	 * notice we're busy and abort. If another CPU comes up after we start
+	 * flushing it will be blocked from starting an active transfer until
+	 * we're done flushing. If another CPU starts an active transfer after
+	 * we release the lock we're still OK because we're no longer the last
+	 * CPU.
+	 */
+	if (spin_trylock(&drv->lock)) {
+		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))
+			ret = NOTIFY_BAD;
+		spin_unlock(&drv->lock);
+	} else {
+		/* Another CPU must be up */
+		return NOTIFY_OK;
+	}
+
+	if (ret == NOTIFY_BAD) {
+		/* Double-check if we're here because someone else is up */
+		if (cpus_in_pm < num_online_cpus())
+			ret = NOTIFY_OK;
+		else
+			/* We won't be called w/ CPU_PM_ENTER_FAILED */
+			atomic_dec(&drv->cpus_in_pm);
+	}
+
+	return ret;
 }
 
 static int rpmh_probe_tcs_config(struct platform_device *pdev,
-				 struct rsc_drv *drv)
+				 struct rsc_drv *drv, void __iomem *base)
 {
 	struct tcs_type_config {
 		u32 type;
@@ -532,15 +869,6 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev,
 	u32 config, max_tcs, ncpt, offset;
 	int i, ret, n, st = 0;
 	struct tcs_group *tcs;
-	struct resource *res;
-	void __iomem *base;
-	char drv_id[10] = {0};
-
-	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id);
-	base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(base))
-		return PTR_ERR(base);
 
 	ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);
 	if (ret)
@@ -584,7 +912,6 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev,
 		tcs->type = tcs_cfg[i].type;
 		tcs->num_tcs = tcs_cfg[i].n;
 		tcs->ncpt = ncpt;
-		spin_lock_init(&tcs->lock);
 
 		if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
 			continue;
@@ -596,19 +923,6 @@ static int rpmh_probe_tcs_config(struct platform_device *pdev,
 		tcs->mask = ((1 << tcs->num_tcs) - 1) << st;
 		tcs->offset = st;
 		st += tcs->num_tcs;
-
-		/*
-		 * Allocate memory to cache sleep and wake requests to
-		 * avoid reading TCS register memory.
-		 */
-		if (tcs->type == ACTIVE_TCS)
-			continue;
-
-		tcs->cmd_cache = devm_kcalloc(&pdev->dev,
-					      tcs->num_tcs * ncpt, sizeof(u32),
-					      GFP_KERNEL);
-		if (!tcs->cmd_cache)
-			return -ENOMEM;
 	}
 
 	drv->num_tcs = st;
@@ -620,7 +934,11 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 {
 	struct device_node *dn = pdev->dev.of_node;
 	struct rsc_drv *drv;
+	struct resource *res;
+	char drv_id[10] = {0};
 	int ret, irq;
+	u32 solver_config;
+	void __iomem *base;
 
 	/*
 	 * Even though RPMh doesn't directly use cmd-db, all of its children
@@ -646,7 +964,13 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 	if (!drv->name)
 		drv->name = dev_name(&pdev->dev);
 
-	ret = rpmh_probe_tcs_config(pdev, drv);
+	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, drv_id);
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	ret = rpmh_probe_tcs_config(pdev, drv, base);
 	if (ret)
 		return ret;
 
@@ -663,8 +987,22 @@ static int rpmh_rsc_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	/*
+	 * CPU PM notification are not required for controllers that support
+	 * 'HW solver' mode where they can be in autonomous mode executing low
+	 * power mode to power down.
+	 */
+	solver_config = readl_relaxed(base + DRV_SOLVER_CONFIG);
+	solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
+	solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
+	if (!solver_config) {
+		drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
+		cpu_pm_register_notifier(&drv->rsc_pm);
+	}
+
 	/* Enable the active TCS to send requests immediately */
-	write_tcs_reg(drv, RSC_DRV_IRQ_ENABLE, 0, drv->tcs[ACTIVE_TCS].mask);
+	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,
+		       drv->tcs_base + RSC_DRV_IRQ_ENABLE);
 
 	spin_lock_init(&drv->client.cache_lock);
 	INIT_LIST_HEAD(&drv->client.cache);
diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c
index eb0ded059d2e..f2b5b46ccd1f 100644
--- a/drivers/soc/qcom/rpmh.c
+++ b/drivers/soc/qcom/rpmh.c
@@ -9,6 +9,7 @@
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
+#include <linux/lockdep.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -119,6 +120,7 @@ static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
 {
 	struct cache_req *req;
 	unsigned long flags;
+	u32 old_sleep_val, old_wake_val;
 
 	spin_lock_irqsave(&ctrlr->cache_lock, flags);
 	req = __find_req(ctrlr, cmd->addr);
@@ -133,26 +135,27 @@ static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
 
 	req->addr = cmd->addr;
 	req->sleep_val = req->wake_val = UINT_MAX;
-	INIT_LIST_HEAD(&req->list);
 	list_add_tail(&req->list, &ctrlr->cache);
 
 existing:
+	old_sleep_val = req->sleep_val;
+	old_wake_val = req->wake_val;
+
 	switch (state) {
 	case RPMH_ACTIVE_ONLY_STATE:
-		if (req->sleep_val != UINT_MAX)
-			req->wake_val = cmd->data;
-		break;
 	case RPMH_WAKE_ONLY_STATE:
 		req->wake_val = cmd->data;
 		break;
 	case RPMH_SLEEP_STATE:
 		req->sleep_val = cmd->data;
 		break;
-	default:
-		break;
 	}
 
-	ctrlr->dirty = true;
+	ctrlr->dirty |= (req->sleep_val != old_sleep_val ||
+			 req->wake_val != old_wake_val) &&
+			 req->sleep_val != UINT_MAX &&
+			 req->wake_val != UINT_MAX;
+
 unlock:
 	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
 
@@ -287,6 +290,7 @@ static void cache_batch(struct rpmh_ctrlr *ctrlr, struct batch_cache_req *req)
 
 	spin_lock_irqsave(&ctrlr->cache_lock, flags);
 	list_add_tail(&req->list, &ctrlr->batch_cache);
+	ctrlr->dirty = true;
 	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
 }
 
@@ -294,12 +298,10 @@ static int flush_batch(struct rpmh_ctrlr *ctrlr)
 {
 	struct batch_cache_req *req;
 	const struct rpmh_request *rpm_msg;
-	unsigned long flags;
 	int ret = 0;
 	int i;
 
 	/* Send Sleep/Wake requests to the controller, expect no response */
-	spin_lock_irqsave(&ctrlr->cache_lock, flags);
 	list_for_each_entry(req, &ctrlr->batch_cache, list) {
 		for (i = 0; i < req->count; i++) {
 			rpm_msg = req->rpm_msgs + i;
@@ -309,23 +311,10 @@ static int flush_batch(struct rpmh_ctrlr *ctrlr)
 				break;
 		}
 	}
-	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
 
 	return ret;
 }
 
-static void invalidate_batch(struct rpmh_ctrlr *ctrlr)
-{
-	struct batch_cache_req *req, *tmp;
-	unsigned long flags;
-
-	spin_lock_irqsave(&ctrlr->cache_lock, flags);
-	list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
-		kfree(req);
-	INIT_LIST_HEAD(&ctrlr->batch_cache);
-	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
-}
-
 /**
  * rpmh_write_batch: Write multiple sets of RPMH commands and wait for the
  * batch to finish.
@@ -442,36 +431,42 @@ static int send_single(struct rpmh_ctrlr *ctrlr, enum rpmh_state state,
 }
 
 /**
- * rpmh_flush: Flushes the buffered active and sleep sets to TCS
- *
- * @ctrlr: controller making request to flush cached data
+ * rpmh_flush() - Flushes the buffered sleep and wake sets to TCSes
  *
- * Return: -EBUSY if the controller is busy, probably waiting on a response
- * to a RPMH request sent earlier.
+ * @ctrlr: Controller making request to flush cached data
  *
- * This function is always called from the sleep code from the last CPU
- * that is powering down the entire system. Since no other RPMH API would be
- * executing at this time, it is safe to run lockless.
+ * Return:
+ * * 0          - Success
+ * * Error code - Otherwise
  */
 int rpmh_flush(struct rpmh_ctrlr *ctrlr)
 {
 	struct cache_req *p;
-	int ret;
+	int ret = 0;
+
+	lockdep_assert_irqs_disabled();
+
+	/*
+	 * Currently rpmh_flush() is only called when we think we're running
+	 * on the last processor.  If the lock is busy it means another
+	 * processor is up and it's better to abort than spin.
+	 */
+	if (!spin_trylock(&ctrlr->cache_lock))
+		return -EBUSY;
 
 	if (!ctrlr->dirty) {
 		pr_debug("Skipping flush, TCS has latest data.\n");
-		return 0;
+		goto exit;
 	}
 
+	/* Invalidate the TCSes first to avoid stale data */
+	rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
+
 	/* First flush the cached batch requests */
 	ret = flush_batch(ctrlr);
 	if (ret)
-		return ret;
+		goto exit;
 
-	/*
-	 * Nobody else should be calling this function other than system PM,
-	 * hence we can run without locks.
-	 */
 	list_for_each_entry(p, &ctrlr->cache, list) {
 		if (!is_req_valid(p)) {
 			pr_debug("%s: skipping RPMH req: a:%#x s:%#x w:%#x",
@@ -481,38 +476,40 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr)
 		ret = send_single(ctrlr, RPMH_SLEEP_STATE, p->addr,
 				  p->sleep_val);
 		if (ret)
-			return ret;
+			goto exit;
 		ret = send_single(ctrlr, RPMH_WAKE_ONLY_STATE, p->addr,
 				  p->wake_val);
 		if (ret)
-			return ret;
+			goto exit;
 	}
 
 	ctrlr->dirty = false;
 
-	return 0;
+exit:
+	spin_unlock(&ctrlr->cache_lock);
+	return ret;
 }
 
 /**
- * rpmh_invalidate: Invalidate all sleep and active sets
- * sets.
+ * rpmh_invalidate: Invalidate sleep and wake sets in batch_cache
  *
  * @dev: The device making the request
  *
- * Invalidate the sleep and active values in the TCS blocks.
+ * Invalidate the sleep and wake values in batch_cache.
  */
 int rpmh_invalidate(const struct device *dev)
 {
 	struct rpmh_ctrlr *ctrlr = get_rpmh_ctrlr(dev);
-	int ret;
+	struct batch_cache_req *req, *tmp;
+	unsigned long flags;
 
-	invalidate_batch(ctrlr);
+	spin_lock_irqsave(&ctrlr->cache_lock, flags);
+	list_for_each_entry_safe(req, tmp, &ctrlr->batch_cache, list)
+		kfree(req);
+	INIT_LIST_HEAD(&ctrlr->batch_cache);
 	ctrlr->dirty = true;
+	spin_unlock_irqrestore(&ctrlr->cache_lock, flags);
 
-	do {
-		ret = rpmh_rsc_invalidate(ctrlr_to_drv(ctrlr));
-	} while (ret == -EAGAIN);
-
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL(rpmh_invalidate);
diff --git a/drivers/soc/qcom/rpmhpd.c b/drivers/soc/qcom/rpmhpd.c
index 4d264d0672c4..e72426221a69 100644
--- a/drivers/soc/qcom/rpmhpd.c
+++ b/drivers/soc/qcom/rpmhpd.c
@@ -4,6 +4,7 @@
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/pm_domain.h>
 #include <linux/slab.h>
@@ -166,6 +167,24 @@ static const struct rpmhpd_desc sm8150_desc = {
 	.num_pds = ARRAY_SIZE(sm8150_rpmhpds),
 };
 
+static struct rpmhpd *sm8250_rpmhpds[] = {
+	[SM8250_CX] = &sdm845_cx,
+	[SM8250_CX_AO] = &sdm845_cx_ao,
+	[SM8250_EBI] = &sdm845_ebi,
+	[SM8250_GFX] = &sdm845_gfx,
+	[SM8250_LCX] = &sdm845_lcx,
+	[SM8250_LMX] = &sdm845_lmx,
+	[SM8250_MMCX] = &sm8150_mmcx,
+	[SM8250_MMCX_AO] = &sm8150_mmcx_ao,
+	[SM8250_MX] = &sdm845_mx,
+	[SM8250_MX_AO] = &sdm845_mx_ao,
+};
+
+static const struct rpmhpd_desc sm8250_desc = {
+	.rpmhpds = sm8250_rpmhpds,
+	.num_pds = ARRAY_SIZE(sm8250_rpmhpds),
+};
+
 /* SC7180 RPMH powerdomains */
 static struct rpmhpd *sc7180_rpmhpds[] = {
 	[SC7180_CX] = &sdm845_cx,
@@ -187,8 +206,10 @@ static const struct of_device_id rpmhpd_match_table[] = {
 	{ .compatible = "qcom,sc7180-rpmhpd", .data = &sc7180_desc },
 	{ .compatible = "qcom,sdm845-rpmhpd", .data = &sdm845_desc },
 	{ .compatible = "qcom,sm8150-rpmhpd", .data = &sm8150_desc },
+	{ .compatible = "qcom,sm8250-rpmhpd", .data = &sm8250_desc },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, rpmhpd_match_table);
 
 static int rpmhpd_send_corner(struct rpmhpd *pd, int state,
 			      unsigned int corner, bool sync)
@@ -460,3 +481,6 @@ static int __init rpmhpd_init(void)
 	return platform_driver_register(&rpmhpd_driver);
 }
 core_initcall(rpmhpd_init);
+
+MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Power Domain Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/qcom/rpmpd.c b/drivers/soc/qcom/rpmpd.c
index 2b1834c5609a..f2168e4259b2 100644
--- a/drivers/soc/qcom/rpmpd.c
+++ b/drivers/soc/qcom/rpmpd.c
@@ -4,6 +4,7 @@
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/pm_domain.h>
 #include <linux/of.h>
@@ -226,6 +227,7 @@ static const struct of_device_id rpmpd_match_table[] = {
 	{ .compatible = "qcom,qcs404-rpmpd", .data = &qcs404_desc },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, rpmpd_match_table);
 
 static int rpmpd_send_enable(struct rpmpd *pd, bool enable)
 {
@@ -422,3 +424,6 @@ static int __init rpmpd_init(void)
 	return platform_driver_register(&rpmpd_driver);
 }
 core_initcall(rpmpd_init);
+
+MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPM Power Domain Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/qcom/smp2p.c b/drivers/soc/qcom/smp2p.c
index c7300d54e444..07183d731d74 100644
--- a/drivers/soc/qcom/smp2p.c
+++ b/drivers/soc/qcom/smp2p.c
@@ -474,10 +474,8 @@ static int qcom_smp2p_probe(struct platform_device *pdev)
 		goto report_read_failure;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "unable to acquire smp2p interrupt\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	smp2p->mbox_client.dev = &pdev->dev;
 	smp2p->mbox_client.knows_txdone = true;
diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
index ebb49aee179b..5983c6ffb078 100644
--- a/drivers/soc/qcom/socinfo.c
+++ b/drivers/soc/qcom/socinfo.c
@@ -188,6 +188,10 @@ static const struct soc_id soc_id[] = {
 	{ 216, "MSM8674PRO" },
 	{ 217, "MSM8974-AA" },
 	{ 218, "MSM8974-AB" },
+	{ 233, "MSM8936" },
+	{ 239, "MSM8939" },
+	{ 240, "APQ8036" },
+	{ 241, "APQ8039" },
 	{ 246, "MSM8996" },
 	{ 247, "APQ8016" },
 	{ 248, "MSM8216" },
@@ -430,6 +434,8 @@ static int qcom_socinfo_probe(struct platform_device *pdev)
 	qs->attr.family = "Snapdragon";
 	qs->attr.machine = socinfo_machine(&pdev->dev,
 					   le32_to_cpu(info->id));
+	qs->attr.soc_id = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%u",
+					 le32_to_cpu(info->id));
 	qs->attr.revision = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%u.%u",
 					   SOCINFO_MAJOR(le32_to_cpu(info->ver)),
 					   SOCINFO_MINOR(le32_to_cpu(info->ver)));
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 1982c7fb45fa..53cd8d2d0cd2 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -83,6 +83,13 @@ config ARCH_R8A7740
 	select ARM_ERRATA_754322
 	select RENESAS_INTC_IRQPIN
 
+config ARCH_R8A7742
+	bool "RZ/G1H (R8A77420)"
+	select ARCH_RCAR_GEN2
+	select ARM_ERRATA_798181 if SMP
+	select ARM_ERRATA_814220
+	select SYSC_R8A7742
+
 config ARCH_R8A7743
 	bool "RZ/G1M (R8A77430)"
 	select ARCH_RCAR_GEN2
@@ -261,6 +268,10 @@ config ARCH_R8A77995
 endif # ARM64
 
 # SoC
+config SYSC_R8A7742
+	bool "RZ/G1H System Controller support" if COMPILE_TEST
+	select SYSC_RCAR
+
 config SYSC_R8A7743
 	bool "RZ/G1M System Controller support" if COMPILE_TEST
 	select SYSC_RCAR
diff --git a/drivers/soc/renesas/Makefile b/drivers/soc/renesas/Makefile
index e595c3c3bd10..08296d78e2ad 100644
--- a/drivers/soc/renesas/Makefile
+++ b/drivers/soc/renesas/Makefile
@@ -3,6 +3,7 @@
 obj-$(CONFIG_SOC_RENESAS)	+= renesas-soc.o
 
 # SoC
+obj-$(CONFIG_SYSC_R8A7742)	+= r8a7742-sysc.o
 obj-$(CONFIG_SYSC_R8A7743)	+= r8a7743-sysc.o
 obj-$(CONFIG_SYSC_R8A7745)	+= r8a7745-sysc.o
 obj-$(CONFIG_SYSC_R8A77470)	+= r8a77470-sysc.o
diff --git a/drivers/soc/renesas/r8a7742-sysc.c b/drivers/soc/renesas/r8a7742-sysc.c
new file mode 100644
index 000000000000..219a675f83f4
--- /dev/null
+++ b/drivers/soc/renesas/r8a7742-sysc.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Renesas RZ/G1H System Controller
+ *
+ * Copyright (C) 2020 Renesas Electronics Corp.
+ */
+
+#include <linux/kernel.h>
+
+#include <dt-bindings/power/r8a7742-sysc.h>
+
+#include "rcar-sysc.h"
+
+static const struct rcar_sysc_area r8a7742_areas[] __initconst = {
+	{ "always-on",	    0, 0, R8A7742_PD_ALWAYS_ON,	-1, PD_ALWAYS_ON },
+	{ "ca15-scu",	0x180, 0, R8A7742_PD_CA15_SCU,	R8A7742_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca15-cpu0",	 0x40, 0, R8A7742_PD_CA15_CPU0,	R8A7742_PD_CA15_SCU,
+	  PD_CPU_NOCR },
+	{ "ca15-cpu1",	 0x40, 1, R8A7742_PD_CA15_CPU1,	R8A7742_PD_CA15_SCU,
+	  PD_CPU_NOCR },
+	{ "ca15-cpu2",	 0x40, 2, R8A7742_PD_CA15_CPU2,	R8A7742_PD_CA15_SCU,
+	  PD_CPU_NOCR },
+	{ "ca15-cpu3",	 0x40, 3, R8A7742_PD_CA15_CPU3,	R8A7742_PD_CA15_SCU,
+	  PD_CPU_NOCR },
+	{ "ca7-scu",	0x100, 0, R8A7742_PD_CA7_SCU,	R8A7742_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca7-cpu0",	0x1c0, 0, R8A7742_PD_CA7_CPU0,	R8A7742_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "ca7-cpu1",	0x1c0, 1, R8A7742_PD_CA7_CPU1,	R8A7742_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "ca7-cpu2",	0x1c0, 2, R8A7742_PD_CA7_CPU2,	R8A7742_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "ca7-cpu3",	0x1c0, 3, R8A7742_PD_CA7_CPU3,	R8A7742_PD_CA7_SCU,
+	  PD_CPU_NOCR },
+	{ "rgx",	 0xc0, 0, R8A7742_PD_RGX,	R8A7742_PD_ALWAYS_ON },
+};
+
+const struct rcar_sysc_info r8a7742_sysc_info __initconst = {
+	.areas = r8a7742_areas,
+	.num_areas = ARRAY_SIZE(r8a7742_areas),
+};
diff --git a/drivers/soc/renesas/rcar-rst.c b/drivers/soc/renesas/rcar-rst.c
index 2af2e0dd83fe..a2b2b1768768 100644
--- a/drivers/soc/renesas/rcar-rst.c
+++ b/drivers/soc/renesas/rcar-rst.c
@@ -39,6 +39,7 @@ static const struct rst_config rcar_rst_gen3 __initconst = {
 
 static const struct of_device_id rcar_rst_matches[] __initconst = {
 	/* RZ/G1 is handled like R-Car Gen2 */
+	{ .compatible = "renesas,r8a7742-rst", .data = &rcar_rst_gen2 },
 	{ .compatible = "renesas,r8a7743-rst", .data = &rcar_rst_gen2 },
 	{ .compatible = "renesas,r8a7744-rst", .data = &rcar_rst_gen2 },
 	{ .compatible = "renesas,r8a7745-rst", .data = &rcar_rst_gen2 },
diff --git a/drivers/soc/renesas/rcar-sysc.c b/drivers/soc/renesas/rcar-sysc.c
index f0b291e02b8a..04ea87a188f1 100644
--- a/drivers/soc/renesas/rcar-sysc.c
+++ b/drivers/soc/renesas/rcar-sysc.c
@@ -273,6 +273,9 @@ finalize:
 }
 
 static const struct of_device_id rcar_sysc_matches[] __initconst = {
+#ifdef CONFIG_SYSC_R8A7742
+	{ .compatible = "renesas,r8a7742-sysc", .data = &r8a7742_sysc_info },
+#endif
 #ifdef CONFIG_SYSC_R8A7743
 	{ .compatible = "renesas,r8a7743-sysc", .data = &r8a7743_sysc_info },
 	/* RZ/G1N is identical to RZ/G2M w.r.t. power domains. */
diff --git a/drivers/soc/renesas/rcar-sysc.h b/drivers/soc/renesas/rcar-sysc.h
index 0fc3b119930a..e417f26fe155 100644
--- a/drivers/soc/renesas/rcar-sysc.h
+++ b/drivers/soc/renesas/rcar-sysc.h
@@ -49,6 +49,7 @@ struct rcar_sysc_info {
 	u32 extmask_val;	/* SYSCEXTMASK register mask value */
 };
 
+extern const struct rcar_sysc_info r8a7742_sysc_info;
 extern const struct rcar_sysc_info r8a7743_sysc_info;
 extern const struct rcar_sysc_info r8a7745_sysc_info;
 extern const struct rcar_sysc_info r8a77470_sysc_info;
diff --git a/drivers/soc/sifive/sifive_l2_cache.c b/drivers/soc/sifive/sifive_l2_cache.c
index a5069394cd61..44d7e1951da3 100644
--- a/drivers/soc/sifive/sifive_l2_cache.c
+++ b/drivers/soc/sifive/sifive_l2_cache.c
@@ -9,6 +9,8 @@
 #include <linux/interrupt.h>
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
+#include <linux/device.h>
+#include <asm/cacheinfo.h>
 #include <soc/sifive/sifive_l2_cache.h>
 
 #define SIFIVE_L2_DIRECCFIX_LOW 0x100
@@ -31,6 +33,7 @@
 
 static void __iomem *l2_base;
 static int g_irq[SIFIVE_L2_MAX_ECCINTR];
+static struct riscv_cacheinfo_ops l2_cache_ops;
 
 enum {
 	DIR_CORR = 0,
@@ -48,7 +51,7 @@ static ssize_t l2_write(struct file *file, const char __user *data,
 
 	if (kstrtouint_from_user(data, count, 0, &val))
 		return -EINVAL;
-	if ((val >= 0 && val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
+	if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
 		writel(val, l2_base + SIFIVE_L2_ECCINJECTERR);
 	else
 		return -EINVAL;
@@ -107,6 +110,38 @@ int unregister_sifive_l2_error_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier);
 
+static int l2_largest_wayenabled(void)
+{
+	return readl(l2_base + SIFIVE_L2_WAYENABLE) & 0xFF;
+}
+
+static ssize_t number_of_ways_enabled_show(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	return sprintf(buf, "%u\n", l2_largest_wayenabled());
+}
+
+static DEVICE_ATTR_RO(number_of_ways_enabled);
+
+static struct attribute *priv_attrs[] = {
+	&dev_attr_number_of_ways_enabled.attr,
+	NULL,
+};
+
+static const struct attribute_group priv_attr_group = {
+	.attrs = priv_attrs,
+};
+
+static const struct attribute_group *l2_get_priv_group(struct cacheinfo *this_leaf)
+{
+	/* We want to use private group for L2 cache only */
+	if (this_leaf->level == 2)
+		return &priv_attr_group;
+	else
+		return NULL;
+}
+
 static irqreturn_t l2_int_handler(int irq, void *device)
 {
 	unsigned int add_h, add_l;
@@ -170,6 +205,9 @@ static int __init sifive_l2_init(void)
 
 	l2_config_read();
 
+	l2_cache_ops.get_priv_group = l2_get_priv_group;
+	riscv_set_cacheinfo_ops(&l2_cache_ops);
+
 #ifdef CONFIG_DEBUG_FS
 	setup_sifive_debug();
 #endif
diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig
index 3693532949b8..6bc603d0b9d9 100644
--- a/drivers/soc/tegra/Kconfig
+++ b/drivers/soc/tegra/Kconfig
@@ -133,6 +133,7 @@ config SOC_TEGRA_FLOWCTRL
 
 config SOC_TEGRA_PMC
 	bool
+	select GENERIC_PINCONF
 
 config SOC_TEGRA_POWERGATE_BPMP
 	def_bool y
diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c
index 802717b9f6a3..d1f8dd0289e6 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra.c
@@ -300,6 +300,59 @@ static void tegra_enable_fuse_clk(void __iomem *base)
 	writel(reg, base + 0x14);
 }
 
+static ssize_t major_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	return sprintf(buf, "%d\n", tegra_get_major_rev());
+}
+
+static DEVICE_ATTR_RO(major);
+
+static ssize_t minor_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	return sprintf(buf, "%d\n", tegra_get_minor_rev());
+}
+
+static DEVICE_ATTR_RO(minor);
+
+static struct attribute *tegra_soc_attr[] = {
+	&dev_attr_major.attr,
+	&dev_attr_minor.attr,
+	NULL,
+};
+
+const struct attribute_group tegra_soc_attr_group = {
+	.attrs = tegra_soc_attr,
+};
+
+#ifdef CONFIG_ARCH_TEGRA_194_SOC
+static ssize_t platform_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	/*
+	 * Displays the value in the 'pre_si_platform' field of the HIDREV
+	 * register for Tegra194 devices. A value of 0 indicates that the
+	 * platform type is silicon and all other non-zero values indicate
+	 * the type of simulation platform is being used.
+	 */
+	return sprintf(buf, "%d\n", (tegra_read_chipid() >> 20) & 0xf);
+}
+
+static DEVICE_ATTR_RO(platform);
+
+static struct attribute *tegra194_soc_attr[] = {
+	&dev_attr_major.attr,
+	&dev_attr_minor.attr,
+	&dev_attr_platform.attr,
+	NULL,
+};
+
+const struct attribute_group tegra194_soc_attr_group = {
+	.attrs = tegra194_soc_attr,
+};
+#endif
+
 struct device * __init tegra_soc_device_register(void)
 {
 	struct soc_device_attribute *attr;
@@ -310,8 +363,10 @@ struct device * __init tegra_soc_device_register(void)
 		return NULL;
 
 	attr->family = kasprintf(GFP_KERNEL, "Tegra");
-	attr->revision = kasprintf(GFP_KERNEL, "%d", tegra_sku_info.revision);
+	attr->revision = kasprintf(GFP_KERNEL, "%s",
+		tegra_revision_name[tegra_sku_info.revision]);
 	attr->soc_id = kasprintf(GFP_KERNEL, "%u", tegra_get_chip_id());
+	attr->custom_attr_group = fuse->soc->soc_attr_group;
 
 	dev = soc_device_register(attr);
 	if (IS_ERR(dev)) {
diff --git a/drivers/soc/tegra/fuse/fuse-tegra20.c b/drivers/soc/tegra/fuse/fuse-tegra20.c
index d4aef9c4a94c..16aaa28573ac 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra20.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra20.c
@@ -164,4 +164,5 @@ const struct tegra_fuse_soc tegra20_fuse_soc = {
 	.speedo_init = tegra20_init_speedo_data,
 	.probe = tegra20_fuse_probe,
 	.info = &tegra20_fuse_info,
+	.soc_attr_group = &tegra_soc_attr_group,
 };
diff --git a/drivers/soc/tegra/fuse/fuse-tegra30.c b/drivers/soc/tegra/fuse/fuse-tegra30.c
index e6037f900fb7..85accef41fa1 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra30.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra30.c
@@ -111,6 +111,7 @@ const struct tegra_fuse_soc tegra30_fuse_soc = {
 	.init = tegra30_fuse_init,
 	.speedo_init = tegra30_init_speedo_data,
 	.info = &tegra30_fuse_info,
+	.soc_attr_group = &tegra_soc_attr_group,
 };
 #endif
 
@@ -125,6 +126,7 @@ const struct tegra_fuse_soc tegra114_fuse_soc = {
 	.init = tegra30_fuse_init,
 	.speedo_init = tegra114_init_speedo_data,
 	.info = &tegra114_fuse_info,
+	.soc_attr_group = &tegra_soc_attr_group,
 };
 #endif
 
@@ -205,6 +207,7 @@ const struct tegra_fuse_soc tegra124_fuse_soc = {
 	.info = &tegra124_fuse_info,
 	.lookups = tegra124_fuse_lookups,
 	.num_lookups = ARRAY_SIZE(tegra124_fuse_lookups),
+	.soc_attr_group = &tegra_soc_attr_group,
 };
 #endif
 
@@ -290,6 +293,7 @@ const struct tegra_fuse_soc tegra210_fuse_soc = {
 	.info = &tegra210_fuse_info,
 	.lookups = tegra210_fuse_lookups,
 	.num_lookups = ARRAY_SIZE(tegra210_fuse_lookups),
+	.soc_attr_group = &tegra_soc_attr_group,
 };
 #endif
 
@@ -319,6 +323,7 @@ const struct tegra_fuse_soc tegra186_fuse_soc = {
 	.info = &tegra186_fuse_info,
 	.lookups = tegra186_fuse_lookups,
 	.num_lookups = ARRAY_SIZE(tegra186_fuse_lookups),
+	.soc_attr_group = &tegra_soc_attr_group,
 };
 #endif
 
@@ -348,5 +353,6 @@ const struct tegra_fuse_soc tegra194_fuse_soc = {
 	.info = &tegra194_fuse_info,
 	.lookups = tegra194_fuse_lookups,
 	.num_lookups = ARRAY_SIZE(tegra194_fuse_lookups),
+	.soc_attr_group = &tegra194_soc_attr_group,
 };
 #endif
diff --git a/drivers/soc/tegra/fuse/fuse.h b/drivers/soc/tegra/fuse/fuse.h
index 94a059e577a1..9d4fc315a007 100644
--- a/drivers/soc/tegra/fuse/fuse.h
+++ b/drivers/soc/tegra/fuse/fuse.h
@@ -32,6 +32,8 @@ struct tegra_fuse_soc {
 
 	const struct nvmem_cell_lookup *lookups;
 	unsigned int num_lookups;
+
+	const struct attribute_group *soc_attr_group;
 };
 
 struct tegra_fuse {
@@ -64,6 +66,11 @@ void tegra_init_apbmisc(void);
 bool __init tegra_fuse_read_spare(unsigned int spare);
 u32 __init tegra_fuse_read_early(unsigned int offset);
 
+u8 tegra_get_major_rev(void);
+u8 tegra_get_minor_rev(void);
+
+extern const struct attribute_group tegra_soc_attr_group;
+
 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 void tegra20_init_speedo_data(struct tegra_sku_info *sku_info);
 #endif
@@ -110,6 +117,7 @@ extern const struct tegra_fuse_soc tegra186_fuse_soc;
 
 #ifdef CONFIG_ARCH_TEGRA_194_SOC
 extern const struct tegra_fuse_soc tegra194_fuse_soc;
+extern const struct attribute_group tegra194_soc_attr_group;
 #endif
 
 #endif
diff --git a/drivers/soc/tegra/fuse/tegra-apbmisc.c b/drivers/soc/tegra/fuse/tegra-apbmisc.c
index 089d9340564b..3cdd69d1bd4d 100644
--- a/drivers/soc/tegra/fuse/tegra-apbmisc.c
+++ b/drivers/soc/tegra/fuse/tegra-apbmisc.c
@@ -37,6 +37,16 @@ u8 tegra_get_chip_id(void)
 	return (tegra_read_chipid() >> 8) & 0xff;
 }
 
+u8 tegra_get_major_rev(void)
+{
+	return (tegra_read_chipid() >> 4) & 0xf;
+}
+
+u8 tegra_get_minor_rev(void)
+{
+	return (tegra_read_chipid() >> 16) & 0xf;
+}
+
 u32 tegra_read_straps(void)
 {
 	WARN(!chipid, "Tegra ABP MISC not yet available\n");
@@ -65,36 +75,32 @@ static const struct of_device_id apbmisc_match[] __initconst = {
 
 void __init tegra_init_revision(void)
 {
-	u32 id, chip_id, minor_rev;
-	int rev;
+	u8 chip_id, minor_rev;
 
-	id = tegra_read_chipid();
-	chip_id = (id >> 8) & 0xff;
-	minor_rev = (id >> 16) & 0xf;
+	chip_id = tegra_get_chip_id();
+	minor_rev = tegra_get_minor_rev();
 
 	switch (minor_rev) {
 	case 1:
-		rev = TEGRA_REVISION_A01;
+		tegra_sku_info.revision = TEGRA_REVISION_A01;
 		break;
 	case 2:
-		rev = TEGRA_REVISION_A02;
+		tegra_sku_info.revision = TEGRA_REVISION_A02;
 		break;
 	case 3:
 		if (chip_id == TEGRA20 && (tegra_fuse_read_spare(18) ||
 					   tegra_fuse_read_spare(19)))
-			rev = TEGRA_REVISION_A03p;
+			tegra_sku_info.revision = TEGRA_REVISION_A03p;
 		else
-			rev = TEGRA_REVISION_A03;
+			tegra_sku_info.revision = TEGRA_REVISION_A03;
 		break;
 	case 4:
-		rev = TEGRA_REVISION_A04;
+		tegra_sku_info.revision = TEGRA_REVISION_A04;
 		break;
 	default:
-		rev = TEGRA_REVISION_UNKNOWN;
+		tegra_sku_info.revision = TEGRA_REVISION_UNKNOWN;
 	}
 
-	tegra_sku_info.revision = rev;
-
 	tegra_sku_info.sku_id = tegra_fuse_read_early(FUSE_SKU_INFO);
 }
 
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index 1c533a969f54..42cf37a0556b 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -3063,6 +3063,7 @@ static const struct pinctrl_pin_desc tegra210_pin_descs[] = {
 
 static const struct tegra_wake_event tegra210_wake_events[] = {
 	TEGRA_WAKE_IRQ("rtc", 16, 2),
+	TEGRA_WAKE_IRQ("pmu", 51, 86),
 };
 
 static const struct tegra_pmc_soc tegra210_pmc_soc = {
@@ -3193,6 +3194,7 @@ static void tegra186_pmc_setup_irq_polarity(struct tegra_pmc *pmc,
 }
 
 static const struct tegra_wake_event tegra186_wake_events[] = {
+	TEGRA_WAKE_IRQ("pmu", 24, 209),
 	TEGRA_WAKE_GPIO("power", 29, 1, TEGRA186_AON_GPIO(FF, 0)),
 	TEGRA_WAKE_IRQ("rtc", 73, 10),
 };
@@ -3325,6 +3327,7 @@ static const char * const tegra194_reset_sources[] = {
 };
 
 static const struct tegra_wake_event tegra194_wake_events[] = {
+	TEGRA_WAKE_IRQ("pmu", 24, 209),
 	TEGRA_WAKE_GPIO("power", 29, 1, TEGRA194_AON_GPIO(EE, 4)),
 	TEGRA_WAKE_IRQ("rtc", 73, 10),
 };
diff --git a/drivers/soc/ti/Kconfig b/drivers/soc/ti/Kconfig
index 4486e055794c..e192fb788836 100644
--- a/drivers/soc/ti/Kconfig
+++ b/drivers/soc/ti/Kconfig
@@ -91,6 +91,16 @@ config TI_K3_RINGACC
 	  and a consumer. There is one RINGACC module per NAVSS on TI AM65x SoCs
 	  If unsure, say N.
 
+config TI_K3_SOCINFO
+	bool
+	depends on ARCH_K3 || COMPILE_TEST
+	select SOC_BUS
+	select MFD_SYSCON
+	help
+	  Include support for the SoC bus socinfo for the TI K3 Multicore SoC
+	  platforms to provide information about the SoC family and
+	  variant to user space.
+
 endif # SOC_TI
 
 config TI_SCI_INTA_MSI_DOMAIN
diff --git a/drivers/soc/ti/Makefile b/drivers/soc/ti/Makefile
index bec827937a5f..1110e5c98685 100644
--- a/drivers/soc/ti/Makefile
+++ b/drivers/soc/ti/Makefile
@@ -11,3 +11,4 @@ obj-$(CONFIG_WKUP_M3_IPC)		+= wkup_m3_ipc.o
 obj-$(CONFIG_TI_SCI_PM_DOMAINS)		+= ti_sci_pm_domains.o
 obj-$(CONFIG_TI_SCI_INTA_MSI_DOMAIN)	+= ti_sci_inta_msi.o
 obj-$(CONFIG_TI_K3_RINGACC)		+= k3-ringacc.o
+obj-$(CONFIG_TI_K3_SOCINFO)		+= k3-socinfo.o
diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c
new file mode 100644
index 000000000000..af0ba5288e58
--- /dev/null
+++ b/drivers/soc/ti/k3-socinfo.c
@@ -0,0 +1,152 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI K3 SoC info driver
+ *
+ * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/regmap.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/sys_soc.h>
+
+#define CTRLMMR_WKUP_JTAGID_REG		0
+/*
+ * Bits:
+ *  31-28 VARIANT	Device variant
+ *  27-12 PARTNO	Part number
+ *  11-1  MFG		Indicates TI as manufacturer (0x17)
+ *  1			Always 1
+ */
+#define CTRLMMR_WKUP_JTAGID_VARIANT_SHIFT	(28)
+#define CTRLMMR_WKUP_JTAGID_VARIANT_MASK	GENMASK(31, 28)
+
+#define CTRLMMR_WKUP_JTAGID_PARTNO_SHIFT	(12)
+#define CTRLMMR_WKUP_JTAGID_PARTNO_MASK		GENMASK(27, 12)
+
+#define CTRLMMR_WKUP_JTAGID_MFG_SHIFT		(1)
+#define CTRLMMR_WKUP_JTAGID_MFG_MASK		GENMASK(11, 1)
+
+#define CTRLMMR_WKUP_JTAGID_MFG_TI		0x17
+
+static const struct k3_soc_id {
+	unsigned int id;
+	const char *family_name;
+} k3_soc_ids[] = {
+	{ 0xBB5A, "AM65X" },
+	{ 0xBB64, "J721E" },
+};
+
+static int
+k3_chipinfo_partno_to_names(unsigned int partno,
+			    struct soc_device_attribute *soc_dev_attr)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(k3_soc_ids); i++)
+		if (partno == k3_soc_ids[i].id) {
+			soc_dev_attr->family = k3_soc_ids[i].family_name;
+			return 0;
+		}
+
+	return -EINVAL;
+}
+
+static int k3_chipinfo_probe(struct platform_device *pdev)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct soc_device_attribute *soc_dev_attr;
+	struct device *dev = &pdev->dev;
+	struct soc_device *soc_dev;
+	struct regmap *regmap;
+	u32 partno_id;
+	u32 variant;
+	u32 jtag_id;
+	u32 mfg;
+	int ret;
+
+	regmap = device_node_to_regmap(node);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	ret = regmap_read(regmap, CTRLMMR_WKUP_JTAGID_REG, &jtag_id);
+	if (ret < 0)
+		return ret;
+
+	mfg = (jtag_id & CTRLMMR_WKUP_JTAGID_MFG_MASK) >>
+	       CTRLMMR_WKUP_JTAGID_MFG_SHIFT;
+
+	if (mfg != CTRLMMR_WKUP_JTAGID_MFG_TI) {
+		dev_err(dev, "Invalid MFG SoC\n");
+		return -ENODEV;
+	}
+
+	variant = (jtag_id & CTRLMMR_WKUP_JTAGID_VARIANT_MASK) >>
+		  CTRLMMR_WKUP_JTAGID_VARIANT_SHIFT;
+	variant++;
+
+	partno_id = (jtag_id & CTRLMMR_WKUP_JTAGID_PARTNO_MASK) >>
+		 CTRLMMR_WKUP_JTAGID_PARTNO_SHIFT;
+
+	soc_dev_attr = kzalloc(sizeof(*soc_dev_attr), GFP_KERNEL);
+	if (!soc_dev_attr)
+		return -ENOMEM;
+
+	soc_dev_attr->revision = kasprintf(GFP_KERNEL, "SR%x.0", variant);
+	if (!soc_dev_attr->revision) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = k3_chipinfo_partno_to_names(partno_id, soc_dev_attr);
+	if (ret) {
+		dev_err(dev, "Unknown SoC JTAGID[0x%08X]\n", jtag_id);
+		ret = -ENODEV;
+		goto err_free_rev;
+	}
+
+	node = of_find_node_by_path("/");
+	of_property_read_string(node, "model", &soc_dev_attr->machine);
+	of_node_put(node);
+
+	soc_dev = soc_device_register(soc_dev_attr);
+	if (IS_ERR(soc_dev)) {
+		ret = PTR_ERR(soc_dev);
+		goto err_free_rev;
+	}
+
+	dev_info(dev, "Family:%s rev:%s JTAGID[0x%08x] Detected\n",
+		 soc_dev_attr->family,
+		 soc_dev_attr->revision, jtag_id);
+
+	return 0;
+
+err_free_rev:
+	kfree(soc_dev_attr->revision);
+err:
+	kfree(soc_dev_attr);
+	return ret;
+}
+
+static const struct of_device_id k3_chipinfo_of_match[] = {
+	{ .compatible = "ti,am654-chipid", },
+	{ /* sentinel */ },
+};
+
+static struct platform_driver k3_chipinfo_driver = {
+	.driver = {
+		.name = "k3-chipinfo",
+		.of_match_table = k3_chipinfo_of_match,
+	},
+	.probe = k3_chipinfo_probe,
+};
+
+static int __init k3_chipinfo_init(void)
+{
+	return platform_driver_register(&k3_chipinfo_driver);
+}
+subsys_initcall(k3_chipinfo_init);
diff --git a/drivers/soc/ti/knav_qmss.h b/drivers/soc/ti/knav_qmss.h
index 038aec352df7..a01eda720bf6 100644
--- a/drivers/soc/ti/knav_qmss.h
+++ b/drivers/soc/ti/knav_qmss.h
@@ -67,7 +67,7 @@ struct knav_reg_config {
 	u32		link_ram_size0;
 	u32		link_ram_base1;
 	u32		__pad2[2];
-	u32		starvation[0];
+	u32		starvation[];
 };
 
 struct knav_reg_region {
diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c
index 37f3db6c041c..aa071d96ef36 100644
--- a/drivers/soc/ti/knav_qmss_queue.c
+++ b/drivers/soc/ti/knav_qmss_queue.c
@@ -409,7 +409,7 @@ static int knav_gp_close_queue(struct knav_range_info *range,
 	return 0;
 }
 
-struct knav_range_ops knav_gp_range_ops = {
+static struct knav_range_ops knav_gp_range_ops = {
 	.set_notify	= knav_gp_set_notify,
 	.open_queue	= knav_gp_open_queue,
 	.close_queue	= knav_gp_close_queue,
diff --git a/drivers/soc/xilinx/zynqmp_pm_domains.c b/drivers/soc/xilinx/zynqmp_pm_domains.c
index 23d90cb12ba9..226d343f0a6a 100644
--- a/drivers/soc/xilinx/zynqmp_pm_domains.c
+++ b/drivers/soc/xilinx/zynqmp_pm_domains.c
@@ -23,8 +23,6 @@
 /* Flag stating if PM nodes mapped to the PM domain has been requested */
 #define ZYNQMP_PM_DOMAIN_REQUESTED	BIT(0)
 
-static const struct zynqmp_eemi_ops *eemi_ops;
-
 static int min_capability;
 
 /**
@@ -76,11 +74,8 @@ static int zynqmp_gpd_power_on(struct generic_pm_domain *domain)
 	int ret;
 	struct zynqmp_pm_domain *pd;
 
-	if (!eemi_ops->set_requirement)
-		return -ENXIO;
-
 	pd = container_of(domain, struct zynqmp_pm_domain, gpd);
-	ret = eemi_ops->set_requirement(pd->node_id,
+	ret = zynqmp_pm_set_requirement(pd->node_id,
 					ZYNQMP_PM_CAPABILITY_ACCESS,
 					ZYNQMP_PM_MAX_QOS,
 					ZYNQMP_PM_REQUEST_ACK_BLOCKING);
@@ -111,9 +106,6 @@ static int zynqmp_gpd_power_off(struct generic_pm_domain *domain)
 	u32 capabilities = min_capability;
 	bool may_wakeup;
 
-	if (!eemi_ops->set_requirement)
-		return -ENXIO;
-
 	pd = container_of(domain, struct zynqmp_pm_domain, gpd);
 
 	/* If domain is already released there is nothing to be done */
@@ -134,7 +126,7 @@ static int zynqmp_gpd_power_off(struct generic_pm_domain *domain)
 		}
 	}
 
-	ret = eemi_ops->set_requirement(pd->node_id, capabilities, 0,
+	ret = zynqmp_pm_set_requirement(pd->node_id, capabilities, 0,
 					ZYNQMP_PM_REQUEST_ACK_NO);
 	/**
 	 * If powering down of any node inside this domain fails,
@@ -163,16 +155,13 @@ static int zynqmp_gpd_attach_dev(struct generic_pm_domain *domain,
 	int ret;
 	struct zynqmp_pm_domain *pd;
 
-	if (!eemi_ops->request_node)
-		return -ENXIO;
-
 	pd = container_of(domain, struct zynqmp_pm_domain, gpd);
 
 	/* If this is not the first device to attach there is nothing to do */
 	if (domain->device_count)
 		return 0;
 
-	ret = eemi_ops->request_node(pd->node_id, 0, 0,
+	ret = zynqmp_pm_request_node(pd->node_id, 0, 0,
 				     ZYNQMP_PM_REQUEST_ACK_BLOCKING);
 	/* If requesting a node fails print and return the error */
 	if (ret) {
@@ -199,16 +188,13 @@ static void zynqmp_gpd_detach_dev(struct generic_pm_domain *domain,
 	int ret;
 	struct zynqmp_pm_domain *pd;
 
-	if (!eemi_ops->release_node)
-		return;
-
 	pd = container_of(domain, struct zynqmp_pm_domain, gpd);
 
 	/* If this is not the last device to detach there is nothing to do */
 	if (domain->device_count)
 		return;
 
-	ret = eemi_ops->release_node(pd->node_id);
+	ret = zynqmp_pm_release_node(pd->node_id);
 	/* If releasing a node fails print the error and return */
 	if (ret) {
 		pr_err("%s() %s release failed for node %d: %d\n",
@@ -266,10 +252,6 @@ static int zynqmp_gpd_probe(struct platform_device *pdev)
 	struct zynqmp_pm_domain *pd;
 	struct device *dev = &pdev->dev;
 
-	eemi_ops = zynqmp_pm_get_eemi_ops();
-	if (IS_ERR(eemi_ops))
-		return PTR_ERR(eemi_ops);
-
 	pd = devm_kcalloc(dev, ZYNQMP_NUM_DOMAINS, sizeof(*pd), GFP_KERNEL);
 	if (!pd)
 		return -ENOMEM;
diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c
index 09227895d216..31ff49fcd078 100644
--- a/drivers/soc/xilinx/zynqmp_power.c
+++ b/drivers/soc/xilinx/zynqmp_power.c
@@ -30,7 +30,6 @@ struct zynqmp_pm_work_struct {
 
 static struct zynqmp_pm_work_struct *zynqmp_pm_init_suspend_work;
 static struct mbox_chan *rx_chan;
-static const struct zynqmp_eemi_ops *eemi_ops;
 
 enum pm_suspend_mode {
 	PM_SUSPEND_MODE_FIRST = 0,
@@ -155,9 +154,6 @@ static ssize_t suspend_mode_store(struct device *dev,
 {
 	int md, ret = -EINVAL;
 
-	if (!eemi_ops->set_suspend_mode)
-		return ret;
-
 	for (md = PM_SUSPEND_MODE_FIRST; md < ARRAY_SIZE(suspend_modes); md++)
 		if (suspend_modes[md] &&
 		    sysfs_streq(suspend_modes[md], buf)) {
@@ -166,7 +162,7 @@ static ssize_t suspend_mode_store(struct device *dev,
 		}
 
 	if (!ret && md != suspend_mode) {
-		ret = eemi_ops->set_suspend_mode(md);
+		ret = zynqmp_pm_set_suspend_mode(md);
 		if (likely(!ret))
 			suspend_mode = md;
 	}
@@ -182,15 +178,8 @@ static int zynqmp_pm_probe(struct platform_device *pdev)
 	u32 pm_api_version;
 	struct mbox_client *client;
 
-	eemi_ops = zynqmp_pm_get_eemi_ops();
-	if (IS_ERR(eemi_ops))
-		return PTR_ERR(eemi_ops);
-
-	if (!eemi_ops->get_api_version || !eemi_ops->init_finalize)
-		return -ENXIO;
-
-	eemi_ops->init_finalize();
-	eemi_ops->get_api_version(&pm_api_version);
+	zynqmp_pm_init_finalize();
+	zynqmp_pm_get_api_version(&pm_api_version);
 
 	/* Check PM API version number */
 	if (pm_api_version < ZYNQMP_PM_VERSION)
diff --git a/drivers/soundwire/Makefile b/drivers/soundwire/Makefile
index e2cdff990e9f..b5871612613b 100644
--- a/drivers/soundwire/Makefile
+++ b/drivers/soundwire/Makefile
@@ -4,7 +4,8 @@
 #
 
 #Bus Objs
-soundwire-bus-objs := bus_type.o bus.o slave.o mipi_disco.o stream.o
+soundwire-bus-objs := bus_type.o bus.o master.o slave.o mipi_disco.o stream.o  \
+			sysfs_slave.o sysfs_slave_dpn.o
 obj-$(CONFIG_SOUNDWIRE) += soundwire-bus.o
 
 ifdef CONFIG_DEBUG_FS
@@ -16,12 +17,9 @@ soundwire-cadence-objs := cadence_master.o
 obj-$(CONFIG_SOUNDWIRE_CADENCE) += soundwire-cadence.o
 
 #Intel driver
-soundwire-intel-objs :=	intel.o
+soundwire-intel-objs :=	intel.o intel_init.o
 obj-$(CONFIG_SOUNDWIRE_INTEL) += soundwire-intel.o
 
-soundwire-intel-init-objs := intel_init.o
-obj-$(CONFIG_SOUNDWIRE_INTEL) += soundwire-intel-init.o
-
 #Qualcomm driver
 soundwire-qcom-objs :=	qcom.o
 obj-$(CONFIG_SOUNDWIRE_QCOM) += soundwire-qcom.o
diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c
index 488c3c9e4947..24ba77226376 100644
--- a/drivers/soundwire/bus.c
+++ b/drivers/soundwire/bus.c
@@ -8,24 +8,54 @@
 #include <linux/soundwire/sdw_registers.h>
 #include <linux/soundwire/sdw.h>
 #include "bus.h"
+#include "sysfs_local.h"
+
+static DEFINE_IDA(sdw_ida);
+
+static int sdw_get_id(struct sdw_bus *bus)
+{
+	int rc = ida_alloc(&sdw_ida, GFP_KERNEL);
+
+	if (rc < 0)
+		return rc;
+
+	bus->id = rc;
+	return 0;
+}
 
 /**
- * sdw_add_bus_master() - add a bus Master instance
+ * sdw_bus_master_add() - add a bus Master instance
  * @bus: bus instance
+ * @parent: parent device
+ * @fwnode: firmware node handle
  *
  * Initializes the bus instance, read properties and create child
  * devices.
  */
-int sdw_add_bus_master(struct sdw_bus *bus)
+int sdw_bus_master_add(struct sdw_bus *bus, struct device *parent,
+		       struct fwnode_handle *fwnode)
 {
 	struct sdw_master_prop *prop = NULL;
 	int ret;
 
-	if (!bus->dev) {
-		pr_err("SoundWire bus has no device\n");
+	if (!parent) {
+		pr_err("SoundWire parent device is not set\n");
 		return -ENODEV;
 	}
 
+	ret = sdw_get_id(bus);
+	if (ret) {
+		dev_err(parent, "Failed to get bus id\n");
+		return ret;
+	}
+
+	ret = sdw_master_device_add(bus, parent, fwnode);
+	if (ret) {
+		dev_err(parent, "Failed to add master device at link %d\n",
+			bus->link_id);
+		return ret;
+	}
+
 	if (!bus->ops) {
 		dev_err(bus->dev, "SoundWire Bus ops are not set\n");
 		return -EINVAL;
@@ -107,7 +137,7 @@ int sdw_add_bus_master(struct sdw_bus *bus)
 
 	return 0;
 }
-EXPORT_SYMBOL(sdw_add_bus_master);
+EXPORT_SYMBOL(sdw_bus_master_add);
 
 static int sdw_delete_slave(struct device *dev, void *data)
 {
@@ -131,18 +161,20 @@ static int sdw_delete_slave(struct device *dev, void *data)
 }
 
 /**
- * sdw_delete_bus_master() - delete the bus master instance
+ * sdw_bus_master_delete() - delete the bus master instance
  * @bus: bus to be deleted
  *
  * Remove the instance, delete the child devices.
  */
-void sdw_delete_bus_master(struct sdw_bus *bus)
+void sdw_bus_master_delete(struct sdw_bus *bus)
 {
 	device_for_each_child(bus->dev, NULL, sdw_delete_slave);
+	sdw_master_device_del(bus);
 
 	sdw_bus_debugfs_exit(bus);
+	ida_free(&sdw_ida, bus->id);
 }
-EXPORT_SYMBOL(sdw_delete_bus_master);
+EXPORT_SYMBOL(sdw_bus_master_delete);
 
 /*
  * SDW IO Calls
@@ -284,9 +316,10 @@ int sdw_fill_msg(struct sdw_msg *msg, struct sdw_slave *slave,
 	msg->flags = flags;
 	msg->buf = buf;
 
-	if (addr < SDW_REG_NO_PAGE) { /* no paging area */
+	if (addr < SDW_REG_NO_PAGE) /* no paging area */
 		return 0;
-	} else if (addr >= SDW_REG_MAX) { /* illegal addr */
+
+	if (addr >= SDW_REG_MAX) { /* illegal addr */
 		pr_err("SDW: Invalid address %x passed\n", addr);
 		return -EINVAL;
 	}
@@ -306,7 +339,9 @@ int sdw_fill_msg(struct sdw_msg *msg, struct sdw_slave *slave,
 	if (!slave) {
 		pr_err("SDW: No slave for paging addr\n");
 		return -EINVAL;
-	} else if (!slave->prop.paging_support) {
+	}
+
+	if (!slave->prop.paging_support) {
 		dev_err(&slave->dev,
 			"address %x needs paging but no support\n", addr);
 		return -EINVAL;
@@ -375,8 +410,8 @@ sdw_bread_no_pm(struct sdw_bus *bus, u16 dev_num, u32 addr)
 	ret = sdw_transfer(bus, &msg);
 	if (ret < 0)
 		return ret;
-	else
-		return buf;
+
+	return buf;
 }
 
 static int
@@ -471,8 +506,8 @@ int sdw_read(struct sdw_slave *slave, u32 addr)
 	ret = sdw_nread(slave, addr, 1, &buf);
 	if (ret < 0)
 		return ret;
-	else
-		return buf;
+
+	return buf;
 }
 EXPORT_SYMBOL(sdw_read);
 
@@ -563,9 +598,9 @@ static int sdw_assign_device_num(struct sdw_slave *slave)
 	}
 
 	if (!new_device)
-		dev_info(slave->bus->dev,
-			 "Slave already registered, reusing dev_num:%d\n",
-			 slave->dev_num);
+		dev_dbg(slave->bus->dev,
+			"Slave already registered, reusing dev_num:%d\n",
+			slave->dev_num);
 
 	/* Clear the slave->dev_num to transfer message on device 0 */
 	dev_num = slave->dev_num;
diff --git a/drivers/soundwire/bus.h b/drivers/soundwire/bus.h
index 204204a26db8..82484f741168 100644
--- a/drivers/soundwire/bus.h
+++ b/drivers/soundwire/bus.h
@@ -19,6 +19,9 @@ static inline int sdw_acpi_find_slaves(struct sdw_bus *bus)
 int sdw_of_find_slaves(struct sdw_bus *bus);
 void sdw_extract_slave_id(struct sdw_bus *bus,
 			  u64 addr, struct sdw_slave_id *id);
+int sdw_master_device_add(struct sdw_bus *bus, struct device *parent,
+			  struct fwnode_handle *fwnode);
+int sdw_master_device_del(struct sdw_bus *bus);
 
 #ifdef CONFIG_DEBUG_FS
 void sdw_bus_debugfs_init(struct sdw_bus *bus);
@@ -172,5 +175,6 @@ sdw_update(struct sdw_slave *slave, u32 addr, u8 mask, u8 val)
 #define SDW_UNATTACH_REQUEST_MASTER_RESET	BIT(0)
 
 void sdw_clear_slave_status(struct sdw_bus *bus, u32 request);
+int sdw_slave_modalias(const struct sdw_slave *slave, char *buf, size_t size);
 
 #endif /* __SDW_BUS_H */
diff --git a/drivers/soundwire/bus_type.c b/drivers/soundwire/bus_type.c
index 17f096dd6806..de9a671802b8 100644
--- a/drivers/soundwire/bus_type.c
+++ b/drivers/soundwire/bus_type.c
@@ -7,6 +7,7 @@
 #include <linux/soundwire/sdw.h>
 #include <linux/soundwire/sdw_type.h>
 #include "bus.h"
+#include "sysfs_local.h"
 
 /**
  * sdw_get_device_id - find the matching SoundWire device id
@@ -33,10 +34,17 @@ sdw_get_device_id(struct sdw_slave *slave, struct sdw_driver *drv)
 
 static int sdw_bus_match(struct device *dev, struct device_driver *ddrv)
 {
-	struct sdw_slave *slave = dev_to_sdw_dev(dev);
-	struct sdw_driver *drv = drv_to_sdw_driver(ddrv);
+	struct sdw_slave *slave;
+	struct sdw_driver *drv;
+	int ret = 0;
+
+	if (is_sdw_slave(dev)) {
+		slave = dev_to_sdw_dev(dev);
+		drv = drv_to_sdw_driver(ddrv);
 
-	return !!sdw_get_device_id(slave, drv);
+		ret = !!sdw_get_device_id(slave, drv);
+	}
+	return ret;
 }
 
 int sdw_slave_modalias(const struct sdw_slave *slave, char *buf, size_t size)
@@ -47,7 +55,7 @@ int sdw_slave_modalias(const struct sdw_slave *slave, char *buf, size_t size)
 			slave->id.mfg_id, slave->id.part_id);
 }
 
-static int sdw_uevent(struct device *dev, struct kobj_uevent_env *env)
+int sdw_slave_uevent(struct device *dev, struct kobj_uevent_env *env)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	char modalias[32];
@@ -63,7 +71,6 @@ static int sdw_uevent(struct device *dev, struct kobj_uevent_env *env)
 struct bus_type sdw_bus_type = {
 	.name = "soundwire",
 	.match = sdw_bus_match,
-	.uevent = sdw_uevent,
 };
 EXPORT_SYMBOL_GPL(sdw_bus_type);
 
@@ -98,6 +105,11 @@ static int sdw_drv_probe(struct device *dev)
 	if (slave->ops && slave->ops->read_prop)
 		slave->ops->read_prop(slave);
 
+	/* init the sysfs as we have properties now */
+	ret = sdw_slave_sysfs_init(slave);
+	if (ret < 0)
+		dev_warn(dev, "Slave sysfs init failed:%d\n", ret);
+
 	/*
 	 * Check for valid clk_stop_timeout, use DisCo worst case value of
 	 * 300ms
diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c
index ecd357d1c63d..9ea87538b9ef 100644
--- a/drivers/soundwire/cadence_master.c
+++ b/drivers/soundwire/cadence_master.c
@@ -407,7 +407,9 @@ cdns_fill_msg_resp(struct sdw_cdns *cdns,
 	if (nack) {
 		dev_err_ratelimited(cdns->dev, "Msg NACKed for Slave %d\n", msg->dev_num);
 		return SDW_CMD_FAIL;
-	} else if (no_ack) {
+	}
+
+	if (no_ack) {
 		dev_dbg_ratelimited(cdns->dev, "Msg ignored for Slave %d\n", msg->dev_num);
 		return SDW_CMD_IGNORED;
 	}
@@ -520,7 +522,9 @@ cdns_program_scp_addr(struct sdw_cdns *cdns, struct sdw_msg *msg)
 		dev_err_ratelimited(cdns->dev,
 				    "SCP_addrpage NACKed for Slave %d\n", msg->dev_num);
 		return SDW_CMD_FAIL;
-	} else if (no_ack) {
+	}
+
+	if (no_ack) {
 		dev_dbg_ratelimited(cdns->dev,
 				    "SCP_addrpage ignored for Slave %d\n", msg->dev_num);
 		return SDW_CMD_IGNORED;
diff --git a/drivers/soundwire/debugfs.c b/drivers/soundwire/debugfs.c
index fb1140e82b86..b6cad0d59b7b 100644
--- a/drivers/soundwire/debugfs.c
+++ b/drivers/soundwire/debugfs.c
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0-only
 // Copyright(c) 2017-2019 Intel Corporation.
 
 #include <linux/device.h>
diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c
index 3c83e76c6bf9..4cfdd074e310 100644
--- a/drivers/soundwire/intel.c
+++ b/drivers/soundwire/intel.c
@@ -669,11 +669,11 @@ static int sdw_stream_setup(struct snd_pcm_substream *substream,
 
 	/* Set stream pointer on all CODEC DAIs */
 	for (i = 0; i < rtd->num_codecs; i++) {
-		ret = snd_soc_dai_set_sdw_stream(rtd->codec_dais[i], sdw_stream,
+		ret = snd_soc_dai_set_sdw_stream(asoc_rtd_to_codec(rtd, i), sdw_stream,
 						 substream->stream);
 		if (ret < 0) {
 			dev_err(dai->dev, "failed to set stream pointer on codec dai %s",
-				rtd->codec_dais[i]->name);
+				asoc_rtd_to_codec(rtd, i)->name);
 			goto release_stream;
 		}
 	}
@@ -1099,7 +1099,6 @@ static int intel_probe(struct platform_device *pdev)
 	sdw->cdns.registers = sdw->link_res->registers;
 	sdw->cdns.instance = sdw->instance;
 	sdw->cdns.msg_count = 0;
-	sdw->cdns.bus.dev = &pdev->dev;
 	sdw->cdns.bus.link_id = pdev->id;
 
 	sdw_cdns_probe(&sdw->cdns);
@@ -1110,9 +1109,9 @@ static int intel_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, sdw);
 
-	ret = sdw_add_bus_master(&sdw->cdns.bus);
+	ret = sdw_bus_master_add(&sdw->cdns.bus, &pdev->dev, pdev->dev.fwnode);
 	if (ret) {
-		dev_err(&pdev->dev, "sdw_add_bus_master fail: %d\n", ret);
+		dev_err(&pdev->dev, "sdw_bus_master_add fail: %d\n", ret);
 		return ret;
 	}
 
@@ -1173,7 +1172,7 @@ err_interrupt:
 	sdw_cdns_enable_interrupt(&sdw->cdns, false);
 	free_irq(sdw->link_res->irq, sdw);
 err_init:
-	sdw_delete_bus_master(&sdw->cdns.bus);
+	sdw_bus_master_delete(&sdw->cdns.bus);
 	return ret;
 }
 
@@ -1189,7 +1188,7 @@ static int intel_remove(struct platform_device *pdev)
 		free_irq(sdw->link_res->irq, sdw);
 		snd_soc_unregister_component(sdw->cdns.dev);
 	}
-	sdw_delete_bus_master(&sdw->cdns.bus);
+	sdw_bus_master_delete(&sdw->cdns.bus);
 
 	return 0;
 }
diff --git a/drivers/soundwire/intel_init.c b/drivers/soundwire/intel_init.c
index 4b769409f6f8..d5d42795a48f 100644
--- a/drivers/soundwire/intel_init.c
+++ b/drivers/soundwire/intel_init.c
@@ -86,7 +86,9 @@ static struct sdw_intel_ctx
 		dev_err(&adev->dev, "Link count %d exceeds max %d\n",
 			count, SDW_MAX_LINKS);
 		return NULL;
-	} else if (!count) {
+	}
+
+	if (!count) {
 		dev_warn(&adev->dev, "No SoundWire links detected\n");
 		return NULL;
 	}
diff --git a/drivers/soundwire/master.c b/drivers/soundwire/master.c
new file mode 100644
index 000000000000..5f0b2189defe
--- /dev/null
+++ b/drivers/soundwire/master.c
@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright(c) 2019-2020 Intel Corporation.
+
+#include <linux/device.h>
+#include <linux/acpi.h>
+#include <linux/pm_runtime.h>
+#include <linux/soundwire/sdw.h>
+#include <linux/soundwire/sdw_type.h>
+#include "bus.h"
+
+/*
+ * The sysfs for properties reflects the MIPI description as given
+ * in the MIPI DisCo spec
+ *
+ * Base file is:
+ *	sdw-master-N
+ *      |---- revision
+ *      |---- clk_stop_modes
+ *      |---- max_clk_freq
+ *      |---- clk_freq
+ *      |---- clk_gears
+ *      |---- default_row
+ *      |---- default_col
+ *      |---- dynamic_shape
+ *      |---- err_threshold
+ */
+
+#define sdw_master_attr(field, format_string)				\
+static ssize_t field##_show(struct device *dev,				\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct sdw_master_device *md = dev_to_sdw_master_device(dev);	\
+	return sprintf(buf, format_string, md->bus->prop.field);	\
+}									\
+static DEVICE_ATTR_RO(field)
+
+sdw_master_attr(revision, "0x%x\n");
+sdw_master_attr(clk_stop_modes, "0x%x\n");
+sdw_master_attr(max_clk_freq, "%d\n");
+sdw_master_attr(default_row, "%d\n");
+sdw_master_attr(default_col, "%d\n");
+sdw_master_attr(default_frame_rate, "%d\n");
+sdw_master_attr(dynamic_frame, "%d\n");
+sdw_master_attr(err_threshold, "%d\n");
+
+static ssize_t clock_frequencies_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct sdw_master_device *md = dev_to_sdw_master_device(dev);
+	ssize_t size = 0;
+	int i;
+
+	for (i = 0; i < md->bus->prop.num_clk_freq; i++)
+		size += sprintf(buf + size, "%8d ",
+				md->bus->prop.clk_freq[i]);
+	size += sprintf(buf + size, "\n");
+
+	return size;
+}
+static DEVICE_ATTR_RO(clock_frequencies);
+
+static ssize_t clock_gears_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct sdw_master_device *md = dev_to_sdw_master_device(dev);
+	ssize_t size = 0;
+	int i;
+
+	for (i = 0; i < md->bus->prop.num_clk_gears; i++)
+		size += sprintf(buf + size, "%8d ",
+				md->bus->prop.clk_gears[i]);
+	size += sprintf(buf + size, "\n");
+
+	return size;
+}
+static DEVICE_ATTR_RO(clock_gears);
+
+static struct attribute *master_node_attrs[] = {
+	&dev_attr_revision.attr,
+	&dev_attr_clk_stop_modes.attr,
+	&dev_attr_max_clk_freq.attr,
+	&dev_attr_default_row.attr,
+	&dev_attr_default_col.attr,
+	&dev_attr_default_frame_rate.attr,
+	&dev_attr_dynamic_frame.attr,
+	&dev_attr_err_threshold.attr,
+	&dev_attr_clock_frequencies.attr,
+	&dev_attr_clock_gears.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(master_node);
+
+static void sdw_master_device_release(struct device *dev)
+{
+	struct sdw_master_device *md = dev_to_sdw_master_device(dev);
+
+	kfree(md);
+}
+
+static const struct dev_pm_ops master_dev_pm = {
+	SET_RUNTIME_PM_OPS(pm_generic_runtime_suspend,
+			   pm_generic_runtime_resume, NULL)
+};
+
+struct device_type sdw_master_type = {
+	.name =		"soundwire_master",
+	.release =	sdw_master_device_release,
+	.pm = &master_dev_pm,
+};
+
+/**
+ * sdw_master_device_add() - create a Linux Master Device representation.
+ * @bus: SDW bus instance
+ * @parent: parent device
+ * @fwnode: firmware node handle
+ */
+int sdw_master_device_add(struct sdw_bus *bus, struct device *parent,
+			  struct fwnode_handle *fwnode)
+{
+	struct sdw_master_device *md;
+	int ret;
+
+	if (!parent)
+		return -EINVAL;
+
+	md = kzalloc(sizeof(*md), GFP_KERNEL);
+	if (!md)
+		return -ENOMEM;
+
+	md->dev.bus = &sdw_bus_type;
+	md->dev.type = &sdw_master_type;
+	md->dev.parent = parent;
+	md->dev.groups = master_node_groups;
+	md->dev.of_node = parent->of_node;
+	md->dev.fwnode = fwnode;
+	md->dev.dma_mask = parent->dma_mask;
+
+	dev_set_name(&md->dev, "sdw-master-%d", bus->id);
+
+	ret = device_register(&md->dev);
+	if (ret) {
+		dev_err(parent, "Failed to add master: ret %d\n", ret);
+		/*
+		 * On err, don't free but drop ref as this will be freed
+		 * when release method is invoked.
+		 */
+		put_device(&md->dev);
+		goto device_register_err;
+	}
+
+	/* add shortcuts to improve code readability/compactness */
+	md->bus = bus;
+	bus->dev = &md->dev;
+	bus->md = md;
+
+device_register_err:
+	return ret;
+}
+
+/**
+ * sdw_master_device_del() - delete a Linux Master Device representation.
+ * @bus: bus handle
+ *
+ * This function is the dual of sdw_master_device_add()
+ */
+int sdw_master_device_del(struct sdw_bus *bus)
+{
+	device_unregister(bus->dev);
+
+	return 0;
+}
diff --git a/drivers/soundwire/mipi_disco.c b/drivers/soundwire/mipi_disco.c
index 844e6b22974f..4ae62b452b8c 100644
--- a/drivers/soundwire/mipi_disco.c
+++ b/drivers/soundwire/mipi_disco.c
@@ -231,16 +231,17 @@ static int sdw_slave_read_dpn(struct sdw_slave *slave,
 
 		nval = fwnode_property_count_u32(node, "mipi-sdw-channel-number-list");
 		if (nval > 0) {
-			dpn[i].num_ch = nval;
-			dpn[i].ch = devm_kcalloc(&slave->dev, dpn[i].num_ch,
-						 sizeof(*dpn[i].ch),
+			dpn[i].num_channels = nval;
+			dpn[i].channels = devm_kcalloc(&slave->dev,
+						       dpn[i].num_channels,
+						       sizeof(*dpn[i].channels),
 						 GFP_KERNEL);
-			if (!dpn[i].ch)
+			if (!dpn[i].channels)
 				return -ENOMEM;
 
 			fwnode_property_read_u32_array(node,
 					"mipi-sdw-channel-number-list",
-					dpn[i].ch, dpn[i].num_ch);
+					dpn[i].channels, dpn[i].num_channels);
 		}
 
 		nval = fwnode_property_count_u32(node, "mipi-sdw-channel-combination-list");
diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c
index d6c9ad231873..a1c2a44a3b4d 100644
--- a/drivers/soundwire/qcom.c
+++ b/drivers/soundwire/qcom.c
@@ -765,12 +765,16 @@ static int qcom_swrm_probe(struct platform_device *pdev)
 	}
 
 	ctrl->irq = of_irq_get(dev->of_node, 0);
-	if (ctrl->irq < 0)
-		return ctrl->irq;
+	if (ctrl->irq < 0) {
+		ret = ctrl->irq;
+		goto err_init;
+	}
 
 	ctrl->hclk = devm_clk_get(dev, "iface");
-	if (IS_ERR(ctrl->hclk))
-		return PTR_ERR(ctrl->hclk);
+	if (IS_ERR(ctrl->hclk)) {
+		ret = PTR_ERR(ctrl->hclk);
+		goto err_init;
+	}
 
 	clk_prepare_enable(ctrl->hclk);
 
@@ -780,14 +784,13 @@ static int qcom_swrm_probe(struct platform_device *pdev)
 	mutex_init(&ctrl->port_lock);
 	INIT_WORK(&ctrl->slave_work, qcom_swrm_slave_wq);
 
-	ctrl->bus.dev = dev;
 	ctrl->bus.ops = &qcom_swrm_ops;
 	ctrl->bus.port_ops = &qcom_swrm_port_ops;
 	ctrl->bus.compute_params = &qcom_swrm_compute_params;
 
 	ret = qcom_swrm_get_port_config(ctrl);
 	if (ret)
-		return ret;
+		goto err_clk;
 
 	params = &ctrl->bus.params;
 	params->max_dr_freq = DEFAULT_CLK_FREQ;
@@ -810,32 +813,37 @@ static int qcom_swrm_probe(struct platform_device *pdev)
 
 	ret = devm_request_threaded_irq(dev, ctrl->irq, NULL,
 					qcom_swrm_irq_handler,
-					IRQF_TRIGGER_RISING,
+					IRQF_TRIGGER_RISING |
+					IRQF_ONESHOT,
 					"soundwire", ctrl);
 	if (ret) {
 		dev_err(dev, "Failed to request soundwire irq\n");
-		goto err;
+		goto err_clk;
 	}
 
-	ret = sdw_add_bus_master(&ctrl->bus);
+	ret = sdw_bus_master_add(&ctrl->bus, dev, dev->fwnode);
 	if (ret) {
 		dev_err(dev, "Failed to register Soundwire controller (%d)\n",
 			ret);
-		goto err;
+		goto err_clk;
 	}
 
 	qcom_swrm_init(ctrl);
 	ret = qcom_swrm_register_dais(ctrl);
 	if (ret)
-		goto err;
+		goto err_master_add;
 
 	dev_info(dev, "Qualcomm Soundwire controller v%x.%x.%x Registered\n",
 		 (ctrl->version >> 24) & 0xff, (ctrl->version >> 16) & 0xff,
 		 ctrl->version & 0xffff);
 
 	return 0;
-err:
+
+err_master_add:
+	sdw_bus_master_delete(&ctrl->bus);
+err_clk:
 	clk_disable_unprepare(ctrl->hclk);
+err_init:
 	return ret;
 }
 
@@ -843,7 +851,7 @@ static int qcom_swrm_remove(struct platform_device *pdev)
 {
 	struct qcom_swrm_ctrl *ctrl = dev_get_drvdata(&pdev->dev);
 
-	sdw_delete_bus_master(&ctrl->bus);
+	sdw_bus_master_delete(&ctrl->bus);
 	clk_disable_unprepare(ctrl->hclk);
 
 	return 0;
diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c
index aace57fae7f8..0839445ee07b 100644
--- a/drivers/soundwire/slave.c
+++ b/drivers/soundwire/slave.c
@@ -14,6 +14,12 @@ static void sdw_slave_release(struct device *dev)
 	kfree(slave);
 }
 
+struct device_type sdw_slave_type = {
+	.name =		"sdw_slave",
+	.release =	sdw_slave_release,
+	.uevent =	sdw_slave_uevent,
+};
+
 static int sdw_slave_add(struct sdw_bus *bus,
 			 struct sdw_slave_id *id, struct fwnode_handle *fwnode)
 {
@@ -41,9 +47,9 @@ static int sdw_slave_add(struct sdw_bus *bus,
 			     id->class_id, id->unique_id);
 	}
 
-	slave->dev.release = sdw_slave_release;
 	slave->dev.bus = &sdw_bus_type;
 	slave->dev.of_node = of_node_get(to_of_node(fwnode));
+	slave->dev.type = &sdw_slave_type;
 	slave->bus = bus;
 	slave->status = SDW_SLAVE_UNATTACHED;
 	init_completion(&slave->enumeration_complete);
@@ -68,6 +74,8 @@ static int sdw_slave_add(struct sdw_bus *bus,
 		list_del(&slave->node);
 		mutex_unlock(&bus->bus_lock);
 		put_device(&slave->dev);
+
+		return ret;
 	}
 	sdw_slave_debugfs_init(slave);
 
diff --git a/drivers/soundwire/sysfs_local.h b/drivers/soundwire/sysfs_local.h
new file mode 100644
index 000000000000..ff60adee3c41
--- /dev/null
+++ b/drivers/soundwire/sysfs_local.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2015-2020 Intel Corporation. */
+
+#ifndef __SDW_SYSFS_LOCAL_H
+#define __SDW_SYSFS_LOCAL_H
+
+/*
+ * SDW sysfs APIs -
+ */
+
+int sdw_slave_sysfs_init(struct sdw_slave *slave);
+int sdw_slave_sysfs_dpn_init(struct sdw_slave *slave);
+
+#endif /* __SDW_SYSFS_LOCAL_H */
diff --git a/drivers/soundwire/sysfs_slave.c b/drivers/soundwire/sysfs_slave.c
new file mode 100644
index 000000000000..f510071b0add
--- /dev/null
+++ b/drivers/soundwire/sysfs_slave.c
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright(c) 2015-2020 Intel Corporation.
+
+#include <linux/device.h>
+#include <linux/mod_devicetable.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+#include <linux/soundwire/sdw.h>
+#include <linux/soundwire/sdw_type.h>
+#include "bus.h"
+#include "sysfs_local.h"
+
+/*
+ * Slave sysfs
+ */
+
+/*
+ * The sysfs for Slave reflects the MIPI description as given
+ * in the MIPI DisCo spec
+ *
+ * Base file is device
+ *	|---- modalias
+ *	|---- dev-properties
+ *		|---- mipi_revision
+ *		|---- wake_capable
+ *		|---- test_mode_capable
+ *		|---- clk_stop_mode1
+ *		|---- simple_clk_stop_capable
+ *		|---- clk_stop_timeout
+ *		|---- ch_prep_timeout
+ *		|---- reset_behave
+ *		|---- high_PHY_capable
+ *		|---- paging_support
+ *		|---- bank_delay_support
+ *		|---- p15_behave
+ *		|---- master_count
+ *		|---- source_ports
+ *		|---- sink_ports
+ *	|---- dp0
+ *		|---- max_word
+ *		|---- min_word
+ *		|---- words
+ *		|---- BRA_flow_controlled
+ *		|---- simple_ch_prep_sm
+ *		|---- imp_def_interrupts
+ *	|---- dpN_<sink/src>
+ *		|---- max_word
+ *		|---- min_word
+ *		|---- words
+ *		|---- type
+ *		|---- max_grouping
+ *		|---- simple_ch_prep_sm
+ *		|---- ch_prep_timeout
+ *		|---- imp_def_interrupts
+ *		|---- min_ch
+ *		|---- max_ch
+ *		|---- channels
+ *		|---- ch_combinations
+ *		|---- max_async_buffer
+ *		|---- block_pack_mode
+ *		|---- port_encoding
+ *
+ */
+
+#define sdw_slave_attr(field, format_string)			\
+static ssize_t field##_show(struct device *dev,			\
+			    struct device_attribute *attr,	\
+			    char *buf)				\
+{								\
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);		\
+	return sprintf(buf, format_string, slave->prop.field);	\
+}								\
+static DEVICE_ATTR_RO(field)
+
+sdw_slave_attr(mipi_revision, "0x%x\n");
+sdw_slave_attr(wake_capable, "%d\n");
+sdw_slave_attr(test_mode_capable, "%d\n");
+sdw_slave_attr(clk_stop_mode1, "%d\n");
+sdw_slave_attr(simple_clk_stop_capable, "%d\n");
+sdw_slave_attr(clk_stop_timeout, "%d\n");
+sdw_slave_attr(ch_prep_timeout, "%d\n");
+sdw_slave_attr(reset_behave, "%d\n");
+sdw_slave_attr(high_PHY_capable, "%d\n");
+sdw_slave_attr(paging_support, "%d\n");
+sdw_slave_attr(bank_delay_support, "%d\n");
+sdw_slave_attr(p15_behave, "%d\n");
+sdw_slave_attr(master_count, "%d\n");
+sdw_slave_attr(source_ports, "0x%x\n");
+sdw_slave_attr(sink_ports, "0x%x\n");
+
+static ssize_t modalias_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+
+	return sdw_slave_modalias(slave, buf, 256);
+}
+static DEVICE_ATTR_RO(modalias);
+
+static struct attribute *slave_attrs[] = {
+	&dev_attr_modalias.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(slave);
+
+static struct attribute *slave_dev_attrs[] = {
+	&dev_attr_mipi_revision.attr,
+	&dev_attr_wake_capable.attr,
+	&dev_attr_test_mode_capable.attr,
+	&dev_attr_clk_stop_mode1.attr,
+	&dev_attr_simple_clk_stop_capable.attr,
+	&dev_attr_clk_stop_timeout.attr,
+	&dev_attr_ch_prep_timeout.attr,
+	&dev_attr_reset_behave.attr,
+	&dev_attr_high_PHY_capable.attr,
+	&dev_attr_paging_support.attr,
+	&dev_attr_bank_delay_support.attr,
+	&dev_attr_p15_behave.attr,
+	&dev_attr_master_count.attr,
+	&dev_attr_source_ports.attr,
+	&dev_attr_sink_ports.attr,
+	NULL,
+};
+
+/*
+ * we don't use ATTRIBUTES_GROUP here since we want to add a subdirectory
+ * for device-level properties
+ */
+static struct attribute_group sdw_slave_dev_attr_group = {
+	.attrs	= slave_dev_attrs,
+	.name = "dev-properties",
+};
+
+/*
+ * DP0 sysfs
+ */
+
+#define sdw_dp0_attr(field, format_string)				\
+static ssize_t field##_show(struct device *dev,				\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);			\
+	return sprintf(buf, format_string, slave->prop.dp0_prop->field);\
+}									\
+static DEVICE_ATTR_RO(field)
+
+sdw_dp0_attr(max_word, "%d\n");
+sdw_dp0_attr(min_word, "%d\n");
+sdw_dp0_attr(BRA_flow_controlled, "%d\n");
+sdw_dp0_attr(simple_ch_prep_sm, "%d\n");
+sdw_dp0_attr(imp_def_interrupts, "0x%x\n");
+
+static ssize_t words_show(struct device *dev,
+			  struct device_attribute *attr, char *buf)
+{
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+	ssize_t size = 0;
+	int i;
+
+	for (i = 0; i < slave->prop.dp0_prop->num_words; i++)
+		size += sprintf(buf + size, "%d ",
+				slave->prop.dp0_prop->words[i]);
+	size += sprintf(buf + size, "\n");
+
+	return size;
+}
+static DEVICE_ATTR_RO(words);
+
+static struct attribute *dp0_attrs[] = {
+	&dev_attr_max_word.attr,
+	&dev_attr_min_word.attr,
+	&dev_attr_words.attr,
+	&dev_attr_BRA_flow_controlled.attr,
+	&dev_attr_simple_ch_prep_sm.attr,
+	&dev_attr_imp_def_interrupts.attr,
+	NULL,
+};
+
+/*
+ * we don't use ATTRIBUTES_GROUP here since we want to add a subdirectory
+ * for dp0-level properties
+ */
+static const struct attribute_group dp0_group = {
+	.attrs = dp0_attrs,
+	.name = "dp0",
+};
+
+int sdw_slave_sysfs_init(struct sdw_slave *slave)
+{
+	int ret;
+
+	ret = devm_device_add_groups(&slave->dev, slave_groups);
+	if (ret < 0)
+		return ret;
+
+	ret = devm_device_add_group(&slave->dev, &sdw_slave_dev_attr_group);
+	if (ret < 0)
+		return ret;
+
+	if (slave->prop.dp0_prop) {
+		ret = devm_device_add_group(&slave->dev, &dp0_group);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (slave->prop.source_ports || slave->prop.sink_ports) {
+		ret = sdw_slave_sysfs_dpn_init(slave);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
diff --git a/drivers/soundwire/sysfs_slave_dpn.c b/drivers/soundwire/sysfs_slave_dpn.c
new file mode 100644
index 000000000000..05a721ea9830
--- /dev/null
+++ b/drivers/soundwire/sysfs_slave_dpn.c
@@ -0,0 +1,300 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright(c) 2015-2020 Intel Corporation.
+
+#include <linux/device.h>
+#include <linux/mod_devicetable.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+#include <linux/soundwire/sdw.h>
+#include <linux/soundwire/sdw_type.h>
+#include "bus.h"
+#include "sysfs_local.h"
+
+struct dpn_attribute {
+	struct device_attribute	dev_attr;
+	int N;
+	int dir;
+	const char *format_string;
+};
+
+/*
+ * Since we can't use ARRAY_SIZE, hard-code number of dpN attributes.
+ * This needs to be updated when adding new attributes - an error will be
+ * flagged on a mismatch.
+ */
+#define SDW_DPN_ATTRIBUTES 15
+
+#define sdw_dpn_attribute_alloc(field)					\
+static int field##_attribute_alloc(struct device *dev,			\
+				struct attribute **res,			\
+				int N, int dir,				\
+				const char *format_string)		\
+{									\
+	struct dpn_attribute *dpn_attr;					\
+									\
+	dpn_attr = devm_kzalloc(dev, sizeof(*dpn_attr), GFP_KERNEL);	\
+	if (!dpn_attr)							\
+		return -ENOMEM;						\
+	dpn_attr->N = N;						\
+	dpn_attr->dir = dir;						\
+	dpn_attr->format_string = format_string;			\
+	dpn_attr->dev_attr.attr.name = __stringify(field);		\
+	dpn_attr->dev_attr.attr.mode = 0444;				\
+	dpn_attr->dev_attr.show = field##_show;				\
+									\
+	*res = &dpn_attr->dev_attr.attr;				\
+									\
+	return 0;							\
+}
+
+#define sdw_dpn_attr(field)						\
+									\
+static ssize_t field##_dpn_show(struct sdw_slave *slave,		\
+				int N,					\
+				int dir,				\
+				const char *format_string,		\
+				char *buf)				\
+{									\
+	struct sdw_dpn_prop *dpn;					\
+	unsigned long mask;						\
+	int bit;							\
+	int i;								\
+									\
+	if (dir) {							\
+		dpn = slave->prop.src_dpn_prop;				\
+		mask = slave->prop.source_ports;			\
+	} else {							\
+		dpn = slave->prop.sink_dpn_prop;			\
+		mask = slave->prop.sink_ports;				\
+	}								\
+									\
+	i = 0;								\
+	for_each_set_bit(bit, &mask, 32) {				\
+		if (bit == N) {						\
+			return sprintf(buf, format_string,		\
+				       dpn[i].field);			\
+		}							\
+		i++;							\
+	}								\
+	return -EINVAL;							\
+}									\
+									\
+static ssize_t field##_show(struct device *dev,				\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);			\
+	struct dpn_attribute *dpn_attr =				\
+		container_of(attr, struct dpn_attribute, dev_attr);	\
+									\
+	return field##_dpn_show(slave,					\
+				dpn_attr->N, dpn_attr->dir,		\
+				dpn_attr->format_string,		\
+				buf);					\
+}									\
+sdw_dpn_attribute_alloc(field)
+
+sdw_dpn_attr(imp_def_interrupts);
+sdw_dpn_attr(max_word);
+sdw_dpn_attr(min_word);
+sdw_dpn_attr(type);
+sdw_dpn_attr(max_grouping);
+sdw_dpn_attr(simple_ch_prep_sm);
+sdw_dpn_attr(ch_prep_timeout);
+sdw_dpn_attr(max_ch);
+sdw_dpn_attr(min_ch);
+sdw_dpn_attr(max_async_buffer);
+sdw_dpn_attr(block_pack_mode);
+sdw_dpn_attr(port_encoding);
+
+#define sdw_dpn_array_attr(field)					\
+									\
+static ssize_t field##_dpn_show(struct sdw_slave *slave,		\
+				int N,					\
+				int dir,				\
+				const char *format_string,		\
+				char *buf)				\
+{									\
+	struct sdw_dpn_prop *dpn;					\
+	unsigned long mask;						\
+	ssize_t size = 0;						\
+	int bit;							\
+	int i;								\
+	int j;								\
+									\
+	if (dir) {							\
+		dpn = slave->prop.src_dpn_prop;				\
+		mask = slave->prop.source_ports;			\
+	} else {							\
+		dpn = slave->prop.sink_dpn_prop;			\
+		mask = slave->prop.sink_ports;				\
+	}								\
+									\
+	i = 0;								\
+	for_each_set_bit(bit, &mask, 32) {				\
+		if (bit == N) {						\
+			for (j = 0; j < dpn[i].num_##field; j++)	\
+				size += sprintf(buf + size,		\
+						format_string,		\
+						dpn[i].field[j]);	\
+			size += sprintf(buf + size, "\n");		\
+			return size;					\
+		}							\
+		i++;							\
+	}								\
+	return -EINVAL;							\
+}									\
+static ssize_t field##_show(struct device *dev,				\
+			    struct device_attribute *attr,		\
+			    char *buf)					\
+{									\
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);			\
+	struct dpn_attribute *dpn_attr =				\
+		container_of(attr, struct dpn_attribute, dev_attr);	\
+									\
+	return field##_dpn_show(slave,					\
+				dpn_attr->N, dpn_attr->dir,		\
+				dpn_attr->format_string,		\
+				buf);					\
+}									\
+sdw_dpn_attribute_alloc(field)
+
+sdw_dpn_array_attr(words);
+sdw_dpn_array_attr(ch_combinations);
+sdw_dpn_array_attr(channels);
+
+static int add_all_attributes(struct device *dev, int N, int dir)
+{
+	struct attribute **dpn_attrs;
+	struct attribute_group *dpn_group;
+	int i = 0;
+	int ret;
+
+	/* allocate attributes, last one is NULL */
+	dpn_attrs = devm_kcalloc(dev, SDW_DPN_ATTRIBUTES + 1,
+				 sizeof(struct attribute *),
+				 GFP_KERNEL);
+	if (!dpn_attrs)
+		return -ENOMEM;
+
+	ret = max_word_attribute_alloc(dev, &dpn_attrs[i++],
+				       N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = min_word_attribute_alloc(dev, &dpn_attrs[i++],
+				       N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = words_attribute_alloc(dev, &dpn_attrs[i++],
+				    N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = type_attribute_alloc(dev, &dpn_attrs[i++],
+				   N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = max_grouping_attribute_alloc(dev, &dpn_attrs[i++],
+					   N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = simple_ch_prep_sm_attribute_alloc(dev, &dpn_attrs[i++],
+						N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = ch_prep_timeout_attribute_alloc(dev, &dpn_attrs[i++],
+					      N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = imp_def_interrupts_attribute_alloc(dev, &dpn_attrs[i++],
+						 N, dir, "0x%x\n");
+	if (ret < 0)
+		return ret;
+
+	ret = min_ch_attribute_alloc(dev, &dpn_attrs[i++],
+				     N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = max_ch_attribute_alloc(dev, &dpn_attrs[i++],
+				     N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = channels_attribute_alloc(dev, &dpn_attrs[i++],
+				       N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = ch_combinations_attribute_alloc(dev, &dpn_attrs[i++],
+					      N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = max_async_buffer_attribute_alloc(dev, &dpn_attrs[i++],
+					       N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = block_pack_mode_attribute_alloc(dev, &dpn_attrs[i++],
+					      N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	ret = port_encoding_attribute_alloc(dev, &dpn_attrs[i++],
+					    N, dir, "%d\n");
+	if (ret < 0)
+		return ret;
+
+	/* paranoia check for editing mistakes */
+	if (i != SDW_DPN_ATTRIBUTES) {
+		dev_err(dev, "mismatch in attributes, allocated %d got %d\n",
+			SDW_DPN_ATTRIBUTES, i);
+		return -EINVAL;
+	}
+
+	dpn_group = devm_kzalloc(dev, sizeof(*dpn_group), GFP_KERNEL);
+	if (!dpn_group)
+		return -ENOMEM;
+
+	dpn_group->attrs = dpn_attrs;
+	dpn_group->name = devm_kasprintf(dev, GFP_KERNEL, "dp%d_%s",
+					 N, dir ? "src" : "sink");
+	if (!dpn_group->name)
+		return -ENOMEM;
+
+	ret = devm_device_add_group(dev, dpn_group);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+int sdw_slave_sysfs_dpn_init(struct sdw_slave *slave)
+{
+	unsigned long mask;
+	int ret;
+	int i;
+
+	mask = slave->prop.source_ports;
+	for_each_set_bit(i, &mask, 32) {
+		ret = add_all_attributes(&slave->dev, i, 1);
+		if (ret < 0)
+			return ret;
+	}
+
+	mask = slave->prop.sink_ports;
+	for_each_set_bit(i, &mask, 32) {
+		ret = add_all_attributes(&slave->dev, i, 0);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c
index 7412a3042a8d..811c97a7c858 100644
--- a/drivers/spi/spi-zynqmp-gqspi.c
+++ b/drivers/spi/spi-zynqmp-gqspi.c
@@ -135,7 +135,6 @@
 
 #define SPI_AUTOSUSPEND_TIMEOUT		3000
 enum mode_type {GQSPI_MODE_IO, GQSPI_MODE_DMA};
-static const struct zynqmp_eemi_ops *eemi_ops;
 
 /**
  * struct zynqmp_qspi - Defines qspi driver instance
@@ -1015,10 +1014,6 @@ static int zynqmp_qspi_probe(struct platform_device *pdev)
 	struct zynqmp_qspi *xqspi;
 	struct device *dev = &pdev->dev;
 
-	eemi_ops = zynqmp_pm_get_eemi_ops();
-	if (IS_ERR(eemi_ops))
-		return PTR_ERR(eemi_ops);
-
 	master = spi_alloc_master(&pdev->dev, sizeof(*xqspi));
 	if (!master)
 		return -ENOMEM;
diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig
index a9939ff9490e..4ec5528f89fa 100644
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 menuconfig STAGING
 	bool "Staging drivers"
-	---help---
+	help
 	  This option allows you to select a number of drivers that are
 	  not of the "normal" Linux kernel quality level.  These drivers
 	  are placed here in order to get a wider audience to make use of
diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c
index 8044510d8ec6..c05a214191da 100644
--- a/drivers/staging/android/ashmem.c
+++ b/drivers/staging/android/ashmem.c
@@ -555,7 +555,7 @@ static int set_name(struct ashmem_area *asma, void __user *name)
 
 	/*
 	 * Holding the ashmem_mutex while doing a copy_from_user might cause
-	 * an data abort which would try to access mmap_sem. If another
+	 * an data abort which would try to access mmap_lock. If another
 	 * thread has invoked ashmem_mmap then it will be holding the
 	 * semaphore and will be waiting for ashmem_mutex, there by leading to
 	 * deadlock. We'll release the mutex and take the name to a local
@@ -586,7 +586,7 @@ static int get_name(struct ashmem_area *asma, void __user *name)
 	 * Have a local variable to which we'll copy the content
 	 * from asma with the lock held. Later we can copy this to the user
 	 * space safely without holding any locks. So even if we proceed to
-	 * wait for mmap_sem, it won't lead to deadlock.
+	 * wait for mmap_lock, it won't lead to deadlock.
 	 */
 	char local_name[ASHMEM_NAME_LEN];
 
diff --git a/drivers/staging/android/ion/ion_page_pool.c b/drivers/staging/android/ion/ion_page_pool.c
index f85ec5b16b65..0198b886d906 100644
--- a/drivers/staging/android/ion/ion_page_pool.c
+++ b/drivers/staging/android/ion/ion_page_pool.c
@@ -37,7 +37,7 @@ static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page)
 	}
 
 	mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
-							1 << pool->order);
+			    1 << pool->order);
 	mutex_unlock(&pool->mutex);
 }
 
@@ -57,7 +57,7 @@ static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high)
 
 	list_del(&page->lru);
 	mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
-							-(1 << pool->order));
+			    -(1 << pool->order));
 	return page;
 }
 
diff --git a/drivers/staging/axis-fifo/axis-fifo.c b/drivers/staging/axis-fifo/axis-fifo.c
index 5801067e7c1b..2bb1c2e9cb57 100644
--- a/drivers/staging/axis-fifo/axis-fifo.c
+++ b/drivers/staging/axis-fifo/axis-fifo.c
@@ -383,8 +383,9 @@ static ssize_t axis_fifo_read(struct file *f, char __user *buf,
 		mutex_lock(&fifo->read_lock);
 		ret = wait_event_interruptible_timeout(fifo->read_queue,
 			ioread32(fifo->base_addr + XLLF_RDFO_OFFSET),
-			(read_timeout >= 0) ? msecs_to_jiffies(read_timeout) :
-				MAX_SCHEDULE_TIMEOUT);
+				 (read_timeout >= 0) ?
+				  msecs_to_jiffies(read_timeout) :
+				  MAX_SCHEDULE_TIMEOUT);
 
 		if (ret <= 0) {
 			if (ret == 0) {
@@ -525,9 +526,10 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf,
 		mutex_lock(&fifo->write_lock);
 		ret = wait_event_interruptible_timeout(fifo->write_queue,
 			ioread32(fifo->base_addr + XLLF_TDFV_OFFSET)
-				>= words_to_write,
-			(write_timeout >= 0) ? msecs_to_jiffies(write_timeout) :
-				MAX_SCHEDULE_TIMEOUT);
+				 >= words_to_write,
+				 (write_timeout >= 0) ?
+				  msecs_to_jiffies(write_timeout) :
+				  MAX_SCHEDULE_TIMEOUT);
 
 		if (ret <= 0) {
 			if (ret == 0) {
diff --git a/drivers/staging/comedi/comedi_fops.c b/drivers/staging/comedi/comedi_fops.c
index a56c8f74a27b..e85a99b68f31 100644
--- a/drivers/staging/comedi/comedi_fops.c
+++ b/drivers/staging/comedi/comedi_fops.c
@@ -2325,7 +2325,7 @@ static int comedi_mmap(struct file *file, struct vm_area_struct *vma)
 	int retval = 0;
 
 	/*
-	 * 'trylock' avoids circular dependency with current->mm->mmap_sem
+	 * 'trylock' avoids circular dependency with current->mm->mmap_lock
 	 * and down-reading &dev->attach_lock should normally succeed without
 	 * contention unless the device is in the process of being attached
 	 * or detached.
diff --git a/drivers/staging/comedi/comedi_internal.h b/drivers/staging/comedi/comedi_internal.h
index 7c8f18f55122..9b3631a654c8 100644
--- a/drivers/staging/comedi/comedi_internal.h
+++ b/drivers/staging/comedi/comedi_internal.h
@@ -32,8 +32,8 @@ void comedi_buf_map_get(struct comedi_buf_map *bm);
 int comedi_buf_map_put(struct comedi_buf_map *bm);
 int comedi_buf_map_access(struct comedi_buf_map *bm, unsigned long offset,
 			  void *buf, int len, int write);
-struct comedi_buf_map *comedi_buf_map_from_subdev_get(
-		struct comedi_subdevice *s);
+struct comedi_buf_map *
+comedi_buf_map_from_subdev_get(struct comedi_subdevice *s);
 unsigned int comedi_buf_write_n_available(struct comedi_subdevice *s);
 unsigned int comedi_buf_write_n_allocated(struct comedi_subdevice *s);
 void comedi_device_cancel_all(struct comedi_device *dev);
diff --git a/drivers/staging/comedi/drivers/ni_pcimio.c b/drivers/staging/comedi/drivers/ni_pcimio.c
index 7c82d5f9778f..c1d70eec24ab 100644
--- a/drivers/staging/comedi/drivers/ni_pcimio.c
+++ b/drivers/staging/comedi/drivers/ni_pcimio.c
@@ -1214,7 +1214,7 @@ static void m_series_init_eeprom_buffer(struct comedi_device *dev)
 	struct ni_private *devpriv = dev->private;
 	struct mite *mite = devpriv->mite;
 	resource_size_t daq_phys_addr;
-	static const int Start_Cal_EEPROM = 0x400;
+	static const int start_cal_eeprom = 0x400;
 	static const unsigned int window_size = 10;
 	unsigned int old_iodwbsr_bits;
 	unsigned int old_iodwbsr1_bits;
@@ -1234,7 +1234,7 @@ static void m_series_init_eeprom_buffer(struct comedi_device *dev)
 	writel(0xf, mite->mmio + 0x30);
 
 	for (i = 0; i < M_SERIES_EEPROM_SIZE; ++i)
-		devpriv->eeprom_buffer[i] = ni_readb(dev, Start_Cal_EEPROM + i);
+		devpriv->eeprom_buffer[i] = ni_readb(dev, start_cal_eeprom + i);
 
 	writel(old_iodwbsr1_bits, mite->mmio + MITE_IODWBSR_1);
 	writel(old_iodwbsr_bits, mite->mmio + MITE_IODWBSR);
diff --git a/drivers/staging/fbtft/fb_st7789v.c b/drivers/staging/fbtft/fb_st7789v.c
index 3c3f387936e8..3a280cc1892c 100644
--- a/drivers/staging/fbtft/fb_st7789v.c
+++ b/drivers/staging/fbtft/fb_st7789v.c
@@ -20,6 +20,12 @@
 	"70 2C 2E 15 10 09 48 33 53 0B 19 18 20 25\n" \
 	"70 2C 2E 15 10 09 48 33 53 0B 19 18 20 25"
 
+#define HSD20_IPS_GAMMA \
+	"D0 05 0A 09 08 05 2E 44 45 0F 17 16 2B 33\n" \
+	"D0 05 0A 09 08 05 2E 43 45 0F 16 16 2B 33"
+
+#define HSD20_IPS 1
+
 /**
  * enum st7789v_command - ST7789V display controller commands
  *
@@ -82,14 +88,20 @@ static int init_display(struct fbtft_par *par)
 
 	/* set pixel format to RGB-565 */
 	write_reg(par, MIPI_DCS_SET_PIXEL_FORMAT, MIPI_DCS_PIXEL_FMT_16BIT);
+	if (HSD20_IPS)
+		write_reg(par, PORCTRL, 0x05, 0x05, 0x00, 0x33, 0x33);
 
-	write_reg(par, PORCTRL, 0x08, 0x08, 0x00, 0x22, 0x22);
+	else
+		write_reg(par, PORCTRL, 0x08, 0x08, 0x00, 0x22, 0x22);
 
 	/*
 	 * VGH = 13.26V
 	 * VGL = -10.43V
 	 */
-	write_reg(par, GCTRL, 0x35);
+	if (HSD20_IPS)
+		write_reg(par, GCTRL, 0x75);
+	else
+		write_reg(par, GCTRL, 0x35);
 
 	/*
 	 * VDV and VRH register values come from command write
@@ -101,13 +113,19 @@ static int init_display(struct fbtft_par *par)
 	 * VAP =  4.1V + (VCOM + VCOM offset + 0.5 * VDV)
 	 * VAN = -4.1V + (VCOM + VCOM offset + 0.5 * VDV)
 	 */
-	write_reg(par, VRHS, 0x0B);
+	if (HSD20_IPS)
+		write_reg(par, VRHS, 0x13);
+	else
+		write_reg(par, VRHS, 0x0B);
 
 	/* VDV = 0V */
 	write_reg(par, VDVS, 0x20);
 
 	/* VCOM = 0.9V */
-	write_reg(par, VCOMS, 0x20);
+	if (HSD20_IPS)
+		write_reg(par, VCOMS, 0x22);
+	else
+		write_reg(par, VCOMS, 0x20);
 
 	/* VCOM offset = 0V */
 	write_reg(par, VCMOFSET, 0x20);
@@ -120,6 +138,10 @@ static int init_display(struct fbtft_par *par)
 	write_reg(par, PWCTRL1, 0xA4, 0xA1);
 
 	write_reg(par, MIPI_DCS_SET_DISPLAY_ON);
+
+	if (HSD20_IPS)
+		write_reg(par, MIPI_DCS_ENTER_INVERT_MODE);
+
 	return 0;
 }
 
@@ -234,7 +256,7 @@ static struct fbtft_display display = {
 	.height = 320,
 	.gamma_num = 2,
 	.gamma_len = 14,
-	.gamma = DEFAULT_GAMMA,
+	.gamma = HSD20_IPS_GAMMA,
 	.fbtftops = {
 		.init_display = init_display,
 		.set_var = set_var,
diff --git a/drivers/staging/fsl-dpaa2/ethsw/ethsw.c b/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
index 676d1ad1b50d..546ad376df99 100644
--- a/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
+++ b/drivers/staging/fsl-dpaa2/ethsw/ethsw.c
@@ -1094,7 +1094,8 @@ static int swdev_port_obj_del(struct net_device *netdev,
 
 static int
 ethsw_switchdev_port_attr_set_event(struct net_device *netdev,
-		struct switchdev_notifier_port_attr_info *port_attr_info)
+				    struct switchdev_notifier_port_attr_info
+				    *port_attr_info)
 {
 	int err;
 
@@ -1277,7 +1278,8 @@ err_addr_alloc:
 
 static int
 ethsw_switchdev_port_obj_event(unsigned long event, struct net_device *netdev,
-			struct switchdev_notifier_port_obj_info *port_obj_info)
+			       struct switchdev_notifier_port_obj_info
+			       *port_obj_info)
 {
 	int err = -EOPNOTSUPP;
 
diff --git a/drivers/staging/gasket/gasket_page_table.c b/drivers/staging/gasket/gasket_page_table.c
index f6d715787da8..f3dbe0fe2a67 100644
--- a/drivers/staging/gasket/gasket_page_table.c
+++ b/drivers/staging/gasket/gasket_page_table.c
@@ -898,7 +898,7 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl,
  *
  * Note that memory for second level page tables is allocated as needed, but
  * that memory is only freed on the final close	of the device file, when the
- * page tables are repartitioned, or the the device is removed.  If there is an
+ * page tables are repartitioned, or the device is removed.  If there is an
  * error or if the full range of slots is not available, any memory
  * allocated for second level page tables remains allocated until final close,
  * repartition, or device removal.
diff --git a/drivers/staging/gasket/gasket_sysfs.c b/drivers/staging/gasket/gasket_sysfs.c
index 5f0e089573a2..af26bc9f184a 100644
--- a/drivers/staging/gasket/gasket_sysfs.c
+++ b/drivers/staging/gasket/gasket_sysfs.c
@@ -339,6 +339,7 @@ void gasket_sysfs_put_attr(struct device *device,
 
 	dev_err(device, "Unable to put unknown attribute: %s\n",
 		attr->attr.attr.name);
+	put_mapping(mapping);
 }
 EXPORT_SYMBOL(gasket_sysfs_put_attr);
 
@@ -372,6 +373,7 @@ ssize_t gasket_sysfs_register_store(struct device *device,
 	gasket_dev = mapping->gasket_dev;
 	if (!gasket_dev) {
 		dev_err(device, "Device driver may have been removed\n");
+		put_mapping(mapping);
 		return 0;
 	}
 
diff --git a/drivers/staging/gdm724x/gdm_lte.c b/drivers/staging/gdm724x/gdm_lte.c
index 354727f0a1fc..eb309190f5be 100644
--- a/drivers/staging/gdm724x/gdm_lte.c
+++ b/drivers/staging/gdm724x/gdm_lte.c
@@ -172,7 +172,7 @@ static int gdm_lte_emulate_arp(struct sk_buff *skb_in, u32 nic_type)
 
 static __sum16 icmp6_checksum(struct ipv6hdr *ipv6, u16 *ptr, int len)
 {
-	unsigned short *w = ptr;
+	unsigned short *w;
 	__wsum sum = 0;
 	int i;
 	u16 pa;
diff --git a/drivers/staging/greybus/Kconfig b/drivers/staging/greybus/Kconfig
index d4777f5a8b90..9389e7a922fa 100644
--- a/drivers/staging/greybus/Kconfig
+++ b/drivers/staging/greybus/Kconfig
@@ -4,7 +4,7 @@ if GREYBUS
 config GREYBUS_AUDIO
 	tristate "Greybus Audio Class driver"
 	depends on SOUND
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Audio Class specification.
 
@@ -13,7 +13,7 @@ config GREYBUS_AUDIO
 
 config GREYBUS_BOOTROM
 	tristate "Greybus Bootrom Class driver"
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Bootrom Class specification.
 
@@ -23,7 +23,7 @@ config GREYBUS_BOOTROM
 config GREYBUS_CAMERA
 	tristate "Greybus Camera Class driver"
 	depends on MEDIA_SUPPORT && LEDS_CLASS_FLASH && BROKEN
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Camera Class specification.
 
@@ -33,7 +33,7 @@ config GREYBUS_CAMERA
 config GREYBUS_FIRMWARE
 	tristate "Greybus Firmware Download Class driver"
 	depends on SPI
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Firmware Download Class specification.
 
@@ -43,7 +43,7 @@ config GREYBUS_FIRMWARE
 config GREYBUS_HID
 	tristate "Greybus HID Class driver"
 	depends on HID && INPUT
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus HID Class specification.
 
@@ -53,7 +53,7 @@ config GREYBUS_HID
 config GREYBUS_LIGHT
 	tristate "Greybus LED Class driver"
 	depends on LEDS_CLASS
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus LED Class specification.
 
@@ -62,7 +62,7 @@ config GREYBUS_LIGHT
 
 config GREYBUS_LOG
 	tristate "Greybus Debug Log Class driver"
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Debug Log Class specification.
 
@@ -71,7 +71,7 @@ config GREYBUS_LOG
 
 config GREYBUS_LOOPBACK
 	tristate "Greybus Loopback Class driver"
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Debug Log Class specification.
 
@@ -81,7 +81,7 @@ config GREYBUS_LOOPBACK
 config GREYBUS_POWER
 	tristate "Greybus Powersupply Class driver"
 	depends on POWER_SUPPLY
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Powersupply Class specification.
 
@@ -90,7 +90,7 @@ config GREYBUS_POWER
 
 config GREYBUS_RAW
 	tristate "Greybus Raw Class driver"
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Raw Class specification.
 
@@ -99,7 +99,7 @@ config GREYBUS_RAW
 
 config GREYBUS_VIBRATOR
 	tristate "Greybus Vibrator Motor Class driver"
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus Vibrator Motor Class specification.
 
@@ -108,7 +108,7 @@ config GREYBUS_VIBRATOR
 
 menuconfig GREYBUS_BRIDGED_PHY
 	tristate "Greybus Bridged PHY Class drivers"
-	---help---
+	help
 	  Select this option to pick from a variety of Greybus Bridged
 	  PHY class drivers.  These drivers emulate a number of
 	  different "traditional" busses by tunneling them over Greybus.
@@ -123,7 +123,7 @@ config GREYBUS_GPIO
 	tristate "Greybus GPIO Bridged PHY driver"
 	depends on GPIOLIB
 	select GPIOLIB_IRQCHIP
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus GPIO Bridged PHY Class specification.
 
@@ -133,7 +133,7 @@ config GREYBUS_GPIO
 config GREYBUS_I2C
 	tristate "Greybus I2C Bridged PHY driver"
 	depends on I2C
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus I2C Bridged PHY Class specification.
 
@@ -143,7 +143,7 @@ config GREYBUS_I2C
 config GREYBUS_PWM
 	tristate "Greybus PWM Bridged PHY driver"
 	depends on PWM
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus PWM Bridged PHY Class specification.
 
@@ -153,7 +153,7 @@ config GREYBUS_PWM
 config GREYBUS_SDIO
 	tristate "Greybus SDIO Bridged PHY driver"
 	depends on MMC
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus SDIO Bridged PHY Class specification.
 
@@ -163,7 +163,7 @@ config GREYBUS_SDIO
 config GREYBUS_SPI
 	tristate "Greybus SPI Bridged PHY driver"
 	depends on SPI
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus SPI Bridged PHY Class specification.
 
@@ -173,7 +173,7 @@ config GREYBUS_SPI
 config GREYBUS_UART
 	tristate "Greybus UART Bridged PHY driver"
 	depends on TTY
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus UART Bridged PHY Class specification.
 
@@ -183,7 +183,7 @@ config GREYBUS_UART
 config GREYBUS_USB
 	tristate "Greybus USB Host Bridged PHY driver"
 	depends on USB
-	---help---
+	help
 	  Select this option if you have a device that follows the
 	  Greybus USB Host Bridged PHY Class specification.
 
@@ -195,7 +195,7 @@ endif	# GREYBUS_BRIDGED_PHY
 config GREYBUS_ARCHE
 	tristate "Greybus Arche Platform driver"
 	depends on USB_HSIC_USB3613 || COMPILE_TEST
-	---help---
+	help
 	  Select this option if you have an Arche device.
 
 	  To compile this code as a module, chose M here: the module
diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c
index 04bfd9110502..ed706f39e87a 100644
--- a/drivers/staging/greybus/hid.c
+++ b/drivers/staging/greybus/hid.c
@@ -290,9 +290,8 @@ static int gb_hid_parse(struct hid_device *hid)
 	}
 
 	rdesc = kzalloc(rsize, GFP_KERNEL);
-	if (!rdesc) {
+	if (!rdesc)
 		return -ENOMEM;
-	}
 
 	ret = gb_hid_get_report_desc(ghid, rdesc);
 	if (ret) {
diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c
index d6ba25f21d80..d2672b65c3f4 100644
--- a/drivers/staging/greybus/light.c
+++ b/drivers/staging/greybus/light.c
@@ -1026,7 +1026,8 @@ static int gb_lights_light_config(struct gb_lights *glights, u8 id)
 
 	light->channels_count = conf.channel_count;
 	light->name = kstrndup(conf.name, NAMES_MAX, GFP_KERNEL);
-
+	if (!light->name)
+		return -ENOMEM;
 	light->channels = kcalloc(light->channels_count,
 				  sizeof(struct gb_channel), GFP_KERNEL);
 	if (!light->channels)
diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c
index 583d9708a191..2471448ba42a 100644
--- a/drivers/staging/greybus/loopback.c
+++ b/drivers/staging/greybus/loopback.c
@@ -135,7 +135,7 @@ static ssize_t name##_##field##_show(struct device *dev,	\
 			    char *buf)					\
 {									\
 	struct gb_loopback *gb = dev_get_drvdata(dev);			\
-	/* Report 0 for min and max if no transfer successed */		\
+	/* Report 0 for min and max if no transfer succeeded */		\
 	if (!gb->requests_completed)					\
 		return sprintf(buf, "0\n");				\
 	return sprintf(buf, "%" #type "\n", gb->name.field);		\
diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c
index 4ffb334cd5cd..607378bfebb7 100644
--- a/drivers/staging/greybus/uart.c
+++ b/drivers/staging/greybus/uart.c
@@ -40,14 +40,6 @@
 #define GB_UART_FIRMWARE_CREDITS	4096
 #define GB_UART_CREDIT_WAIT_TIMEOUT_MSEC	10000
 
-struct gb_tty_line_coding {
-	__le32	rate;
-	__u8	format;
-	__u8	parity;
-	__u8	data_bits;
-	__u8	flow_control;
-};
-
 struct gb_tty {
 	struct gbphy_device *gbphy_dev;
 	struct tty_port port;
@@ -66,7 +58,7 @@ struct gb_tty {
 	struct mutex mutex;
 	u8 ctrlin;	/* input control lines */
 	u8 ctrlout;	/* output control lines */
-	struct gb_tty_line_coding line_coding;
+	struct gb_uart_set_line_coding_request line_coding;
 	struct work_struct tx_work;
 	struct kfifo write_fifo;
 	bool close_pending;
@@ -288,12 +280,9 @@ static void  gb_uart_tx_write_work(struct work_struct *work)
 
 static int send_line_coding(struct gb_tty *tty)
 {
-	struct gb_uart_set_line_coding_request request;
-
-	memcpy(&request, &tty->line_coding,
-	       sizeof(tty->line_coding));
 	return gb_operation_sync(tty->connection, GB_UART_TYPE_SET_LINE_CODING,
-				 &request, sizeof(request), NULL, 0);
+				 &tty->line_coding, sizeof(tty->line_coding),
+				 NULL, 0);
 }
 
 static int send_control(struct gb_tty *gb_tty, u8 control)
@@ -493,9 +482,9 @@ static int gb_tty_break_ctl(struct tty_struct *tty, int state)
 static void gb_tty_set_termios(struct tty_struct *tty,
 			       struct ktermios *termios_old)
 {
+	struct gb_uart_set_line_coding_request newline;
 	struct gb_tty *gb_tty = tty->driver_data;
 	struct ktermios *termios = &tty->termios;
-	struct gb_tty_line_coding newline;
 	u8 newctrl = gb_tty->ctrlout;
 
 	newline.rate = cpu_to_le32(tty_get_baud_rate(tty));
diff --git a/drivers/staging/iio/Documentation/overview.txt b/drivers/staging/iio/Documentation/overview.txt
index 43f92b06bc3e..ebdc64f451d7 100644
--- a/drivers/staging/iio/Documentation/overview.txt
+++ b/drivers/staging/iio/Documentation/overview.txt
@@ -34,7 +34,7 @@ turned on or off (if possible) via sysfs interfaces.
 fifo / ring buffers on the sensor chip.  These greatly reduce the load
 on the host CPU by buffering relatively large numbers of data samples
 based on an internal sampling clock. Examples include VTI SCA3000
-series and Analog Device ADXL345 accelerometers.  Each buffer supports
+series and Analog Devices ADXL345 accelerometers.  Each buffer supports
 polling to establish when data is available.
 
 * Trigger and software buffer support. In many data analysis
diff --git a/drivers/staging/iio/impedance-analyzer/ad5933.c b/drivers/staging/iio/impedance-analyzer/ad5933.c
index af0bcf95ee8a..c468355b0848 100644
--- a/drivers/staging/iio/impedance-analyzer/ad5933.c
+++ b/drivers/staging/iio/impedance-analyzer/ad5933.c
@@ -602,11 +602,12 @@ static const struct iio_buffer_setup_ops ad5933_ring_setup_ops = {
 	.postdisable = ad5933_ring_postdisable,
 };
 
-static int ad5933_register_ring_funcs_and_init(struct iio_dev *indio_dev)
+static int ad5933_register_ring_funcs_and_init(struct device *dev,
+					       struct iio_dev *indio_dev)
 {
 	struct iio_buffer *buffer;
 
-	buffer = iio_kfifo_allocate();
+	buffer = devm_iio_kfifo_allocate(dev);
 	if (!buffer)
 		return -ENOMEM;
 
@@ -676,6 +677,20 @@ static void ad5933_work(struct work_struct *work)
 	}
 }
 
+static void ad5933_reg_disable(void *data)
+{
+	struct ad5933_state *st = data;
+
+	regulator_disable(st->reg);
+}
+
+static void ad5933_clk_disable(void *data)
+{
+	struct ad5933_state *st = data;
+
+	clk_disable_unprepare(st->mclk);
+}
+
 static int ad5933_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
@@ -703,23 +718,32 @@ static int ad5933_probe(struct i2c_client *client,
 		dev_err(&client->dev, "Failed to enable specified VDD supply\n");
 		return ret;
 	}
-	ret = regulator_get_voltage(st->reg);
 
+	ret = devm_add_action_or_reset(&client->dev, ad5933_reg_disable, st);
+	if (ret)
+		return ret;
+
+	ret = regulator_get_voltage(st->reg);
 	if (ret < 0)
-		goto error_disable_reg;
+		return ret;
 
 	st->vref_mv = ret / 1000;
 
 	st->mclk = devm_clk_get(&client->dev, "mclk");
-	if (IS_ERR(st->mclk) && PTR_ERR(st->mclk) != -ENOENT) {
-		ret = PTR_ERR(st->mclk);
-		goto error_disable_reg;
-	}
+	if (IS_ERR(st->mclk) && PTR_ERR(st->mclk) != -ENOENT)
+		return PTR_ERR(st->mclk);
 
 	if (!IS_ERR(st->mclk)) {
 		ret = clk_prepare_enable(st->mclk);
 		if (ret < 0)
-			goto error_disable_reg;
+			return ret;
+
+		ret = devm_add_action_or_reset(&client->dev,
+					       ad5933_clk_disable,
+					       st);
+		if (ret)
+			return ret;
+
 		ext_clk_hz = clk_get_rate(st->mclk);
 	}
 
@@ -742,41 +766,15 @@ static int ad5933_probe(struct i2c_client *client,
 	indio_dev->channels = ad5933_channels;
 	indio_dev->num_channels = ARRAY_SIZE(ad5933_channels);
 
-	ret = ad5933_register_ring_funcs_and_init(indio_dev);
+	ret = ad5933_register_ring_funcs_and_init(&client->dev, indio_dev);
 	if (ret)
-		goto error_disable_mclk;
+		return ret;
 
 	ret = ad5933_setup(st);
 	if (ret)
-		goto error_unreg_ring;
-
-	ret = iio_device_register(indio_dev);
-	if (ret)
-		goto error_unreg_ring;
-
-	return 0;
-
-error_unreg_ring:
-	iio_kfifo_free(indio_dev->buffer);
-error_disable_mclk:
-	clk_disable_unprepare(st->mclk);
-error_disable_reg:
-	regulator_disable(st->reg);
-
-	return ret;
-}
-
-static int ad5933_remove(struct i2c_client *client)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(client);
-	struct ad5933_state *st = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
-	iio_kfifo_free(indio_dev->buffer);
-	regulator_disable(st->reg);
-	clk_disable_unprepare(st->mclk);
+		return ret;
 
-	return 0;
+	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
 static const struct i2c_device_id ad5933_id[] = {
@@ -801,7 +799,6 @@ static struct i2c_driver ad5933_driver = {
 		.of_match_table = ad5933_of_match,
 	},
 	.probe = ad5933_probe,
-	.remove = ad5933_remove,
 	.id_table = ad5933_id,
 };
 module_i2c_driver(ad5933_driver);
diff --git a/drivers/staging/kpc2000/kpc_dma/fileops.c b/drivers/staging/kpc2000/kpc_dma/fileops.c
index 7caabdd77bbf..89753463e926 100644
--- a/drivers/staging/kpc2000/kpc_dma/fileops.c
+++ b/drivers/staging/kpc2000/kpc_dma/fileops.c
@@ -75,9 +75,9 @@ static int kpc_dma_transfer(struct dev_private_data *priv,
 	}
 
 	// Lock the user buffer pages in memory, and hold on to the page pointers (for the sglist)
-	down_read(&current->mm->mmap_sem);      /*  get memory map semaphore */
+	mmap_read_lock(current->mm);      /*  get memory map semaphore */
 	rv = get_user_pages(iov_base, acd->page_count, FOLL_TOUCH | FOLL_WRITE | FOLL_GET, acd->user_pages, NULL);
-	up_read(&current->mm->mmap_sem);        /*  release the semaphore */
+	mmap_read_unlock(current->mm);        /*  release the semaphore */
 	if (rv != acd->page_count) {
 		dev_err(&priv->ldev->pldev->dev, "Couldn't get_user_pages (%ld)\n", rv);
 		goto err_get_user_pages;
diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig
index 053f485eb994..4bb1eca6f597 100644
--- a/drivers/staging/media/Kconfig
+++ b/drivers/staging/media/Kconfig
@@ -38,6 +38,8 @@ source "drivers/staging/media/sunxi/Kconfig"
 
 source "drivers/staging/media/tegra-vde/Kconfig"
 
+source "drivers/staging/media/tegra-video/Kconfig"
+
 source "drivers/staging/media/ipu3/Kconfig"
 
 source "drivers/staging/media/soc_camera/Kconfig"
diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile
index e01f13a1b4a2..71a47b61836d 100644
--- a/drivers/staging/media/Makefile
+++ b/drivers/staging/media/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_VIDEO_MESON_VDEC)	+= meson/vdec/
 obj-$(CONFIG_VIDEO_OMAP4)	+= omap4iss/
 obj-$(CONFIG_VIDEO_ROCKCHIP_VDEC)	+= rkvdec/
 obj-$(CONFIG_VIDEO_SUNXI)	+= sunxi/
+obj-$(CONFIG_VIDEO_TEGRA)	+= tegra-video/
 obj-$(CONFIG_TEGRA_VDE)		+= tegra-vde/
 obj-$(CONFIG_VIDEO_HANTRO)	+= hantro/
 obj-$(CONFIG_VIDEO_IPU3_IMGU)	+= ipu3/
diff --git a/drivers/staging/media/atomisp/Kconfig b/drivers/staging/media/atomisp/Kconfig
index c4f3049b0706..fea06cb0eb48 100644
--- a/drivers/staging/media/atomisp/Kconfig
+++ b/drivers/staging/media/atomisp/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 menuconfig INTEL_ATOMISP
 	bool "Enable support to Intel Atom ISP camera drivers"
 	depends on X86 && EFI && PCI && ACPI
@@ -11,9 +12,10 @@ menuconfig INTEL_ATOMISP
 config VIDEO_ATOMISP
 	tristate "Intel Atom Image Signal Processor Driver"
 	depends on VIDEO_V4L2 && INTEL_ATOMISP
+	depends on PMIC_OPREGION
 	select IOSF_MBI
 	select VIDEOBUF_VMALLOC
-	---help---
+	help
 	  Say Y here if your platform supports Intel Atom SoC
 	  camera imaging subsystem.
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/staging/media/atomisp/Makefile b/drivers/staging/media/atomisp/Makefile
index eedecd49bbf4..9dc8072799e3 100644
--- a/drivers/staging/media/atomisp/Makefile
+++ b/drivers/staging/media/atomisp/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Makefile for camera drivers.
 #
@@ -36,7 +37,6 @@ atomisp-objs += \
 	pci/sh_css_param_dvs.o \
 	pci/sh_css_param_shading.o \
 	pci/sh_css_params.o \
-	pci/sh_css_pipe.o \
 	pci/sh_css_properties.o \
 	pci/sh_css_shading.o \
 	pci/sh_css_sp.o \
@@ -53,10 +53,7 @@ atomisp-objs += \
 	pci/hmm/hmm_dynamic_pool.o \
 	pci/hmm/hmm.o \
 	pci/hmm/hmm_reserved_pool.o \
-	pci/hmm/hmm_vm.o \
-	pci/hrt/hive_isp_css_mm_hrt.o \
 	pci/ia_css_device_access.o \
-	pci/ia_css_memory_access.o \
 	pci/isp/kernels/aa/aa_2/ia_css_aa2.host.o \
 	pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.o \
 	pci/isp/kernels/anr/anr_2/ia_css_anr2.host.o \
@@ -117,7 +114,6 @@ atomisp-objs += \
 	pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.o \
 	pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.o \
 	pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.o \
-	pci/memory_realloc.o \
 	pci/mmu/isp_mmu.o \
 	pci/mmu/sh_mmu_mrfld.o \
 	pci/runtime/binary/src/binary.o \
@@ -328,11 +324,6 @@ INCLUDES_cht += \
 #	-I$(atomisp)/pci/css_2401_system/hrt/ \
 #	-I$(atomisp)/pci/css_2401_system/hive_isp_css_2401_system_generated/ \
 
-
-ifeq ($(CONFIG_ION),y)
-INCLUDES += -I$(srctree)/drivers/staging/android/ion
-endif
-
 DEFINES := -DHRT_HW -DHRT_ISP_CSS_CUSTOM_HOST -DHRT_USE_VIR_ADDRS -D__HOST__
 #DEFINES += -DUSE_DYNAMIC_BIN
 #DEFINES += -DISP_POWER_GATING
@@ -352,12 +343,3 @@ DEFINES += -DISP2400 -DSYSTEM_hive_isp_css_2400_system
 endif
 
 ccflags-y += $(INCLUDES) $(DEFINES) -fno-common
-
-# HACK! While this driver is in bad shape, don't enable several warnings
-#       that would be otherwise enabled with W=1
-ccflags-y += $(call cc-disable-warning, implicit-fallthrough)
-ccflags-y += $(call cc-disable-warning, missing-prototypes)
-ccflags-y += $(call cc-disable-warning, missing-declarations)
-ccflags-y += $(call cc-disable-warning, suggest-attribute=format)
-ccflags-y += $(call cc-disable-warning, unused-const-variable)
-ccflags-y += $(call cc-disable-warning, unused-but-set-variable)
diff --git a/drivers/staging/media/atomisp/TODO b/drivers/staging/media/atomisp/TODO
index 52683a704223..6987bb2d32cf 100644
--- a/drivers/staging/media/atomisp/TODO
+++ b/drivers/staging/media/atomisp/TODO
@@ -1,71 +1,127 @@
-1. A single AtomISP driver needs to be implemented to support both
-   Baytrail (BYT and Cherrytail (CHT) platforms at the same time.
+NOTE:
+=====
+
+While the driver probes the hardware and reports itself as a
+V4L2 driver, there are still some issues preventing it to
+stream (at least it doesn't with the standard V4L2 applications.
+Didn't test yet with some custom-made app for this driver).
+Solving the related bugs and issues preventing it to work is
+needed (items 6 and 7 from the list below).
+
+TODO
+====
+
+1. The atomisp doesn't rely at the usual i2c stuff to discover the
+   sensors. Instead, it calls a function from atomisp_gmin_platform.c.
+   There are some hacks added there for it to wait for sensors to be
+   probed (with a timeout of 2 seconds or so).
+   This should be converted to the usual way, using V4L2 async subdev
+   framework to wait for cameras to be probed;
+
+2. Use ACPI _DSM table - DONE!
+
+3. Switch the driver to use pm_runtime stuff. Right now, it probes the
+   existing PMIC code and sensors call it directly.
+
+4. There's a problem at the sensor drivers: when trying to set a video
+   format, the atomisp main driver calls the sensor drivers with the
+   sensor turned off. This causes them to fail.
+
+   The only exception is the atomisp-ov2880, which has a hack inside it
+   to turn it on when VIDIOC_S_FMT is called.
+
+   The right fix seems to power on the sensor when a video device is
+   opened (or at the first VIDIOC_ ioctl - except for VIDIOC_QUERYCAP),
+   powering it down at close() syscall.
+
+   Such kind of control would need to be done inside the atomisp driver,
+   not at the sensors code.
+
+5. There are several issues related to memory management, causing
+   crashes. The atomisp splits the memory management on three separate
+   regions:
+
+	- dynamic pool;
+	- reserved pool;
+	- generic pool
+
+   The code implementing it is at:
+
+	drivers/staging/media/atomisp/pci/hmm/
+
+   It also has a separate code for managing DMA buffers at:
+
+	drivers/staging/media/atomisp/pci/mmu/
+
+   The code there is really dirty, ugly and probably wrong. I fixed
+   one bug there already, but the best would be to just trash it and use
+   something else. Maybe the code from the newer intel driver could
+   serve as a model:
+
+	drivers/staging/media/ipu3/ipu3-mmu.c
+
+   But converting it to use something like that is painful and may
+   cause some breakages.
+
+6. There is some issues at the frame receive logic, causing the
+   DQBUF ioctls to fail.
+
+7. A single AtomISP driver needs to be implemented to support both
+   Baytrail (BYT) and Cherrytail (CHT) platforms at the same time.
    The current driver is a mechanical and hand combined merge of the
    two using several runtime macros, plus some ifdef ISP2401 to select the
    CHT version. Yet, there are some ISP-specific headers that change the
    driver's behavior during compile time.
 
-2. The file structure needs to get tidied up to resemble a normal Linux
+8. The file structure needs to get tidied up to resemble a normal Linux
    driver.
 
-3. Lots of the midlayer glue. unused code and abstraction needs removing.
+9. Lots of the midlayer glue. unused code and abstraction needs removing.
 
-3. The sensor drivers read MIPI settings from EFI variables or default to the
-   settings hard-coded in the platform data file for different platforms.
-   It should be possible to improve it, by adding support for _DSM tables.
+10. The AtomISP driver includes some special IOCTLS (ATOMISP_IOC_XXXX_XXXX)
+    and controls that require some cleanup. Some of those code may have
+    been removed during the cleanups. They could be needed in order to
+    properly support 3A algorithms
 
-4. The sensor drivers use PMIC and the regulator framework API. In the ideal
-   world it would be using ACPI but that's not how the existing devices work.
+    Such IOCTL interface needs more documentation. The better would
+    be to use something close to the interface used by the IPU3 IMGU driver.
 
-5. The AtomISP driver includes some special IOCTLS (ATOMISP_IOC_XXXX_XXXX)
-   and controls that require some cleanup.
+11. The ISP code has some dependencies of the exact FW version.
+    The version defined in pci/sh_css_firmware.c:
 
-6. Correct Coding Style. Please don't send coding style patches for this
-   driver until the other work is done.
+    BYT (isp2400): "irci_stable_candrpv_0415_20150521_0458"
 
-7. The ISP code has some dependencies of the exact FW version.
-   The version defined in pci/sh_css_firmware.c:
-   BYT:
-	static const char *isp2400_release_version = STR(irci_stable_candrpv_0415_20150521_0458);
+    CHT (isp2401): "irci_ecr - master_20150911_0724"
 
-   CHT:
-	static const char *isp2401_release_version = STR(irci_ecr - master_20150911_0724);
+    Those versions don't seem to be available anymore. On the tests we've
+    done so far, this version also seems to work for CHT:
 
-   Those versions don't seem to be available anymore. On the tests we've
-   done so far, this version also seems to work for isp2401:
+		"irci_stable_candrpv_0415_20150521_0458"
 
-		irci_stable_candrpv_0415_20150521_0458
+    Which can be obtainable from Yocto Atom ISP respository.
 
-   At some point we may need to round up a few driver versions and see if
-   there are any specific things that can be done to fold in support for
-   multiple firmware versions.
+    but this was not thoroughly tested.
 
-8. Switch to V4L2 async API to set up sensor, lens and flash devices.
-   Control those devices using V4L2 sub-device API without custom
-   extensions.
+    At some point we may need to round up a few driver versions and see if
+    there are any specific things that can be done to fold in support for
+    multiple firmware versions.
 
-9. Switch to standard V4L2 sub-device API for sensor and lens. In
-   particular, the user space API needs to support V4L2 controls as
-   defined in the V4L2 spec and references to atomisp must be removed from
-   these drivers.
+12. Switch to standard V4L2 sub-device API for sensor and lens. In
+    particular, the user space API needs to support V4L2 controls as
+    defined in the V4L2 spec and references to atomisp must be removed from
+    these drivers.
 
-10. Use LED flash API for flash LED drivers such as LM3554 (which already
+13. Use LED flash API for flash LED drivers such as LM3554 (which already
     has a LED class driver).
 
-11. Switch from videobuf1 to videobuf2. Videobuf1 is being removed!
-
-12. There are some memory management code that seems to be
-    forked from Kernel 3.10 inside hmm/ directory. Get rid of it,
-    making the driver to use a more standard memory management module.
+14. Switch from videobuf1 to videobuf2. Videobuf1 is being removed!
 
-13. While the driver probes the hardware and reports itself as a
-    V4L2 driver, there are still some issues preventing it to
-    stream (at least it doesn't with the standard V4L2 applications.
-    Didn't test yet with some custom-made app for this driver).
-    Solving the related bugs and issues preventing it to work is
-    needed.
+15. Correct Coding Style. Please refrain sending coding style patches
+    for this driver until the other work is done, as there will be a lot
+    of code churn until this driver becomes functional again.
 
-Limitations:
+Limitations
+===========
 
 1. To test the patches, you also need the ISP firmware
 
@@ -76,14 +132,16 @@ Limitations:
    device but can also be extracted from the upgrade kit if you've managed
    to lose them somehow.
 
-2. Without a 3A libary the capture behaviour is not very good. To take a good
-   picture, you need tune ISP parameters by IOCTL functions or use a 3A libary
+2. Without a 3A library the capture behaviour is not very good. To take a good
+   picture, you need tune ISP parameters by IOCTL functions or use a 3A library
    such as libxcam.
 
 3. The driver is intended to drive the PCI exposed versions of the device.
    It will not detect those devices enumerated via ACPI as a field of the
    i915 GPU driver.
 
+   There are some patches adding i915 GPU support floating at the Yocto's
+   Aero repository (so far, untested upstream).
+
 4. The driver supports only v2 of the IPU/Camera. It will not work with the
    versions of the hardware in other SoCs.
-
diff --git a/drivers/staging/media/atomisp/i2c/Kconfig b/drivers/staging/media/atomisp/i2c/Kconfig
index f7f7177b9b37..7c7f0fc090b3 100644
--- a/drivers/staging/media/atomisp/i2c/Kconfig
+++ b/drivers/staging/media/atomisp/i2c/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Kconfig for sensor drivers
 #
@@ -8,7 +9,7 @@ config VIDEO_ATOMISP_OV2722
        tristate "OVT ov2722 sensor support"
 	depends on ACPI
        depends on I2C && VIDEO_V4L2
-       ---help---
+	help
 	 This is a Video4Linux2 sensor-level driver for the OVT
 	 OV2722 raw camera.
 
@@ -20,7 +21,7 @@ config VIDEO_ATOMISP_GC2235
        tristate "Galaxy gc2235 sensor support"
 	depends on ACPI
        depends on I2C && VIDEO_V4L2
-       ---help---
+	help
 	 This is a Video4Linux2 sensor-level driver for the OVT
 	 GC2235 raw camera.
 
@@ -31,7 +32,7 @@ config VIDEO_ATOMISP_GC2235
 config VIDEO_ATOMISP_MSRLIST_HELPER
        tristate "Helper library to load, parse and apply large register lists."
        depends on I2C
-       ---help---
+	help
 	 This is a helper library to be used from a sensor driver to load, parse
 	 and apply large register lists.
 
@@ -42,7 +43,7 @@ config VIDEO_ATOMISP_MT9M114
        tristate "Aptina mt9m114 sensor support"
 	depends on ACPI
        depends on I2C && VIDEO_V4L2
-       ---help---
+	help
 	 This is a Video4Linux2 sensor-level driver for the Micron
 	 mt9m114 1.3 Mpixel camera.
 
@@ -54,15 +55,15 @@ config VIDEO_ATOMISP_GC0310
 	tristate "GC0310 sensor support"
 	depends on ACPI
 	depends on I2C && VIDEO_V4L2
-	---help---
+	help
 	  This is a Video4Linux2 sensor-level driver for the Galaxycore
 	  GC0310 0.3MP sensor.
-	 
+
 config VIDEO_ATOMISP_OV2680
        tristate "Omnivision OV2680 sensor support"
 	depends on ACPI
        depends on I2C && VIDEO_V4L2
-       ---help---
+	help
 	 This is a Video4Linux2 sensor-level driver for the Omnivision
 	 OV2680 raw camera.
 
@@ -78,7 +79,7 @@ config VIDEO_ATOMISP_LM3554
        tristate "LM3554 flash light driver"
 	depends on ACPI
        depends on VIDEO_V4L2 && I2C
-       ---help---
+	help
 	 This is a Video4Linux2 sub-dev driver for the LM3554
 	 flash light driver.
 
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
index ad1bd7d6a02b..2b71de722ec3 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for GalaxyCore GC0310 VGA camera sensor.
  *
@@ -1309,18 +1310,6 @@ static int gc0310_probe(struct i2c_client *client)
 	int ret;
 	void *pdata;
 	unsigned int i;
-	acpi_handle handle;
-	struct acpi_device *adev;
-
-	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev)) {
-		dev_err(&client->dev, "Error could not get ACPI device\n");
-		return -ENODEV;
-	}
-	pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n",
-		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
-	// FIXME: may need to release resources allocated by acpi_bus_get_device()
-
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
index a12dd0e858bc..78147ffb6099 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for GalaxyCore GC2235 2M camera sensor.
  *
@@ -1051,17 +1052,6 @@ static int gc2235_probe(struct i2c_client *client)
 	void *gcpdev;
 	int ret;
 	unsigned int i;
-	acpi_handle handle;
-	struct acpi_device *adev;
-
-	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev)) {
-		dev_err(&client->dev, "Error could not get ACPI device\n");
-		return -ENODEV;
-	}
-	pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n",
-		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
-	// FIXME: may need to release resources allocated by acpi_bus_get_device()
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c b/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c
index 33ab884f7352..b93c80471f22 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
  *
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
index a899145265ff..809010af7855 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * LED flash driver for LM3554
  *
@@ -850,17 +851,6 @@ static int lm3554_probe(struct i2c_client *client)
 	struct lm3554 *flash;
 	unsigned int i;
 	int ret;
-	acpi_handle handle;
-	struct acpi_device *adev;
-
-	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev)) {
-		dev_err(&client->dev, "Error could not get ACPI device\n");
-		return -ENODEV;
-	}
-	pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n",
-		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
-	// FIXME: may need to release resources allocated by acpi_bus_get_device()
 
 	flash = kzalloc(sizeof(*flash), GFP_KERNEL);
 	if (!flash)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
index ac61b391e3f9..0d60918a9b19 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for mt9m114 Camera Sensor.
  *
@@ -1816,17 +1817,6 @@ static int mt9m114_probe(struct i2c_client *client)
 	int ret = 0;
 	unsigned int i;
 	void *pdata;
-	acpi_handle handle;
-	struct acpi_device *adev;
-
-	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev)) {
-		dev_err(&client->dev, "Error could not get ACPI device\n");
-		return -ENODEV;
-	}
-	pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n",
-		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
-	// FIXME: may need to release resources allocated by acpi_bus_get_device()
 
 	/* Setup sensor configuration structure */
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
index 1b60f6a9c0e0..90d125ba080f 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for OmniVision OV2680 1080p HD camera sensor.
  *
@@ -703,7 +704,7 @@ static int power_ctrl(struct v4l2_subdev *sd, bool flag)
 	if (!dev || !dev->platform_data)
 		return -ENODEV;
 
-	dev_dbg(&client->dev, "%s: %s", __func__, flag? "on" : "off");
+	dev_dbg(&client->dev, "%s: %s", __func__, flag ? "on" : "off");
 
 	if (flag) {
 		ret |= dev->platform_data->v1p8_ctrl(sd, 1);
@@ -1243,17 +1244,6 @@ static int ov2680_probe(struct i2c_client *client)
 	int ret;
 	void *pdata;
 	unsigned int i;
-	acpi_handle handle;
-	struct acpi_device *adev;
-
-	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev)) {
-		dev_err(&client->dev, "Error could not get ACPI device\n");
-		return -ENODEV;
-	}
-	dev_info(&client->dev, "%s: ACPI detected it on bus ID=%s, HID=%s\n",
-		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
-	// FIXME: may need to release resources allocated by acpi_bus_get_device()
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
index 718d10f89d5a..eecefcd734d0 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for OmniVision OV2722 1080p HD camera sensor.
  *
@@ -1214,17 +1215,6 @@ static int ov2722_probe(struct i2c_client *client)
 	struct ov2722_device *dev;
 	void *ovpdev;
 	int ret;
-	acpi_handle handle;
-	struct acpi_device *adev;
-
-	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev)) {
-		dev_err(&client->dev, "Error could not get ACPI device\n");
-		return -ENODEV;
-	}
-	pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n",
-		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
-	// FIXME: may need to release resources allocated by acpi_bus_get_device()
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
diff --git a/drivers/staging/media/atomisp/i2c/gc0310.h b/drivers/staging/media/atomisp/i2c/gc0310.h
index 12f746e7828b..2fe3de115083 100644
--- a/drivers/staging/media/atomisp/i2c/gc0310.h
+++ b/drivers/staging/media/atomisp/i2c/gc0310.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for GalaxyCore GC0310 VGA camera sensor.
  *
diff --git a/drivers/staging/media/atomisp/i2c/gc2235.h b/drivers/staging/media/atomisp/i2c/gc2235.h
index bb104de61af9..68252b8f516d 100644
--- a/drivers/staging/media/atomisp/i2c/gc2235.h
+++ b/drivers/staging/media/atomisp/i2c/gc2235.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for GalaxyCore GC2235 2M camera sensor.
  *
diff --git a/drivers/staging/media/atomisp/i2c/mt9m114.h b/drivers/staging/media/atomisp/i2c/mt9m114.h
index 172cec0bb398..787bbf59e895 100644
--- a/drivers/staging/media/atomisp/i2c/mt9m114.h
+++ b/drivers/staging/media/atomisp/i2c/mt9m114.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for mt9m114 Camera Sensor.
  *
diff --git a/drivers/staging/media/atomisp/i2c/ov2680.h b/drivers/staging/media/atomisp/i2c/ov2680.h
index 034e1032f6c0..49920245e064 100644
--- a/drivers/staging/media/atomisp/i2c/ov2680.h
+++ b/drivers/staging/media/atomisp/i2c/ov2680.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for OmniVision OV2680 5M camera sensor.
  *
@@ -456,6 +457,7 @@ static struct ov2680_reg const ov2680_656x496_30fps[] = {
 	// {0x5090, 0x0c},
 	{}
 };
+
 /*
 * 800x600 30fps  VBlanking 1lane 10Bit (binning)
 */
@@ -500,6 +502,7 @@ static struct ov2680_reg const ov2680_720x592_30fps[] = {
 	{0x5081, 0x41},
 	{}
 };
+
 /*
 * 800x600 30fps  VBlanking 1lane 10Bit (binning)
 */
diff --git a/drivers/staging/media/atomisp/i2c/ov2722.h b/drivers/staging/media/atomisp/i2c/ov2722.h
index 1110d723968e..7b0debb6c53d 100644
--- a/drivers/staging/media/atomisp/i2c/ov2722.h
+++ b/drivers/staging/media/atomisp/i2c/ov2722.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for OmniVision OV2722 1080p HD camera sensor.
  *
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/Kconfig b/drivers/staging/media/atomisp/i2c/ov5693/Kconfig
index 3f527f2047a7..c8d09f416c35 100644
--- a/drivers/staging/media/atomisp/i2c/ov5693/Kconfig
+++ b/drivers/staging/media/atomisp/i2c/ov5693/Kconfig
@@ -1,8 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
 config VIDEO_ATOMISP_OV5693
        tristate "Omnivision ov5693 sensor support"
 	depends on ACPI
        depends on I2C && VIDEO_V4L2
-       ---help---
+	help
 	 This is a Video4Linux2 sensor-level driver for the Micron
 	 ov5693 5 Mpixel camera.
 
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h b/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h
index c97ab24c5d2b..f1362cd69f6e 100644
--- a/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h
+++ b/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for AD5823 VCM.
  *
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
index 2be0ef14d53e..97ab10bc45ca 100644
--- a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
+++ b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for OmniVision OV5693 1080p HD camera sensor.
  *
@@ -1087,7 +1088,7 @@ static int ov5693_s_ctrl(struct v4l2_ctrl *ctrl)
 	case V4L2_CID_VCM_SLEW:
 		ret = ov5693_t_vcm_slew(&dev->sd, ctrl->val);
 		break;
-	case V4L2_CID_VCM_TIMEING:
+	case V4L2_CID_VCM_TIMING:
 		ret = ov5693_t_vcm_timing(&dev->sd, ctrl->val);
 		break;
 	default:
@@ -1230,7 +1231,7 @@ static const struct v4l2_ctrl_config ov5693_controls[] = {
 	},
 	{
 		.ops = &ctrl_ops,
-		.id = V4L2_CID_VCM_TIMEING,
+		.id = V4L2_CID_VCM_TIMING,
 		.type = V4L2_CTRL_TYPE_INTEGER,
 		.name = "vcm step time",
 		.min = 0,
@@ -1901,17 +1902,6 @@ static int ov5693_probe(struct i2c_client *client)
 	int ret = 0;
 	void *pdata;
 	unsigned int i;
-	acpi_handle handle;
-	struct acpi_device *adev;
-
-	handle = ACPI_HANDLE(&client->dev);
-	if (!handle || acpi_bus_get_device(handle, &adev)) {
-		dev_err(&client->dev, "Error could not get ACPI device\n");
-		return -ENODEV;
-	}
-	pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n",
-		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
-	// FIXME: may need to release resources allocated by acpi_bus_get_device()
 
 	/*
 	 * Firmware workaround: Some modules use a "secondary default"
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h b/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h
index 0189907969c6..79df07bd69b6 100644
--- a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h
+++ b/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for OmniVision OV5693 5M camera sensor.
  *
diff --git a/drivers/staging/media/atomisp/include/hmm/hmm.h b/drivers/staging/media/atomisp/include/hmm/hmm.h
index 254a71442451..b48bdf5c274c 100644
--- a/drivers/staging/media/atomisp/include/hmm/hmm.h
+++ b/drivers/staging/media/atomisp/include/hmm/hmm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -28,8 +29,8 @@
 #include "hmm/hmm_pool.h"
 #include "ia_css_types.h"
 
-#define HMM_CACHED true
-#define HMM_UNCACHED false
+#define mmgr_NULL              ((ia_css_ptr)0)
+#define mmgr_EXCEPTION         ((ia_css_ptr) - 1)
 
 int hmm_pool_register(unsigned int pool_size, enum hmm_pool_type pool_type);
 void hmm_pool_unregister(enum hmm_pool_type pool_type);
@@ -38,7 +39,8 @@ int hmm_init(void);
 void hmm_cleanup(void);
 
 ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
-		     int from_highmem, const void __user *userptr, bool cached);
+		     int from_highmem, const void __user *userptr,
+		     const uint16_t attrs);
 void hmm_free(ia_css_ptr ptr);
 int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes);
 int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes);
diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_bo.h b/drivers/staging/media/atomisp/include/hmm/hmm_bo.h
index f847d1de860e..8c78a5d87b65 100644
--- a/drivers/staging/media/atomisp/include/hmm/hmm_bo.h
+++ b/drivers/staging/media/atomisp/include/hmm/hmm_bo.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -77,9 +78,6 @@ enum hmm_bo_type {
 	HMM_BO_PRIVATE,
 	HMM_BO_SHARE,
 	HMM_BO_USER,
-#ifdef CONFIG_ION
-	HMM_BO_ION,
-#endif
 	HMM_BO_LAST,
 };
 
@@ -111,9 +109,6 @@ struct hmm_bo_device {
 
 	/* list lock is used to protect the entire_bo_list */
 	spinlock_t	list_lock;
-#ifdef CONFIG_ION
-	struct ion_client	*iclient;
-#endif
 	int flag;
 
 	/* linked list for entire buffer object */
@@ -136,15 +131,14 @@ struct hmm_buffer_object {
 	struct list_head	list;
 	struct kref	kref;
 
+	struct page **pages;
+
 	/* mutex protecting this BO */
 	struct mutex		mutex;
 	enum hmm_bo_type	type;
 	struct hmm_page_object	*page_obj;	/* physical pages */
 	int		from_highmem;
 	int		mmap_count;
-#ifdef CONFIG_ION
-	struct ion_handle	*ihandle;
-#endif
 	int		status;
 	int		mem_type;
 	void		*vmap_addr; /* kernel virtual address by vmap */
diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_common.h b/drivers/staging/media/atomisp/include/hmm/hmm_common.h
index 00885203fb14..7152e9b52ba4 100644
--- a/drivers/staging/media/atomisp/include/hmm/hmm_common.h
+++ b/drivers/staging/media/atomisp/include/hmm/hmm_common.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_pool.h b/drivers/staging/media/atomisp/include/hmm/hmm_pool.h
index 8caf00502d74..3fef57de973c 100644
--- a/drivers/staging/media/atomisp/include/hmm/hmm_pool.h
+++ b/drivers/staging/media/atomisp/include/hmm/hmm_pool.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_vm.h b/drivers/staging/media/atomisp/include/hmm/hmm_vm.h
deleted file mode 100644
index 93ac5e445137..000000000000
--- a/drivers/staging/media/atomisp/include/hmm/hmm_vm.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef	__HMM_VM_H__
-#define	__HMM_VM_H__
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mutex.h>
-#include <linux/list.h>
-
-struct hmm_vm {
-	unsigned int start;
-	unsigned int pgnr;
-	unsigned int size;
-	struct list_head vm_node_list;
-	spinlock_t lock;
-	struct kmem_cache *cache;
-};
-
-struct hmm_vm_node {
-	struct list_head list;
-	unsigned int start;
-	unsigned int pgnr;
-	unsigned int size;
-	struct hmm_vm *vm;
-};
-
-#define	ISP_VM_START	0x0
-#define	ISP_VM_SIZE	(0x7FFFFFFF)	/* 2G address space */
-#define	ISP_PTR_NULL	NULL
-
-int hmm_vm_init(struct hmm_vm *vm, unsigned int start,
-		unsigned int size);
-
-void hmm_vm_clean(struct hmm_vm *vm);
-
-struct hmm_vm_node *hmm_vm_alloc_node(struct hmm_vm *vm,
-				      unsigned int pgnr);
-
-void hmm_vm_free_node(struct hmm_vm_node *node);
-
-struct hmm_vm_node *hmm_vm_find_node_start(struct hmm_vm *vm,
-	unsigned int addr);
-
-struct hmm_vm_node *hmm_vm_find_node_in_range(struct hmm_vm *vm,
-	unsigned int addr);
-
-#endif
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp.h b/drivers/staging/media/atomisp/include/linux/atomisp.h
index e9670749bae0..22c4103b0385 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -69,9 +70,6 @@
 #define V4L2_MBUS_FMT_CUSTOM_M10MO_RAW	0x800b
 #endif
 
-/* FIXME: for now, let's use a boolean to identify the type of atomisp chipset */
-extern bool atomisp_hw_is_isp2401;
-
 /* Configuration used by Bayer noise reduction and YCC noise reduction */
 struct atomisp_nr_config {
 	/* [gain] Strength of noise reduction for Bayer NR (Used by Bayer NR) */
@@ -509,7 +507,7 @@ struct atomisp_parameters {
 	struct atomisp_shading_table *shading_table;
 	struct atomisp_morph_table   *morph_table;
 	struct atomisp_dvs_coefficients *dvs_coefs; /* DVS 1.0 coefficients */
-	struct atomisp_dvs2_coefficients *dvs2_coefs; /* DVS 2.0 coefficients */
+	struct atomisp_dis_coefficients *dvs2_coefs; /* DVS 2.0 coefficients */
 	struct atomisp_capture_config   *capture_config;
 	struct atomisp_anr_thres   *anr_thres;
 
@@ -917,6 +915,8 @@ struct atomisp_acc_map {
 
 #define ATOMISP_MAP_FLAG_NOFLUSH	0x0001	/* Do not flush cache */
 #define ATOMISP_MAP_FLAG_CACHED		0x0002	/* Enable cache */
+#define ATOMISP_MAP_FLAG_CONTIGUOUS	0x0004
+#define ATOMISP_MAP_FLAG_CLEARED	0x0008
 
 struct atomisp_acc_state {
 	__u32 flags;			/* Flags, see list below */
@@ -1272,7 +1272,7 @@ struct atomisp_sensor_ae_bracketing_lut {
 /* VCM slew control */
 #define V4L2_CID_VCM_SLEW                  (V4L2_CID_CAMERA_LASTP1 + 11)
 /* VCM step time */
-#define V4L2_CID_VCM_TIMEING               (V4L2_CID_CAMERA_LASTP1 + 12)
+#define V4L2_CID_VCM_TIMING                (V4L2_CID_CAMERA_LASTP1 + 12)
 
 /* Query Focus Status */
 #define V4L2_CID_FOCUS_STATUS              (V4L2_CID_CAMERA_LASTP1 + 14)
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
index 8700ffd32bdb..58e0ea5355a3 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel MID SoC Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
index 9cf46325a696..873344a02ccf 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -17,6 +18,9 @@
 #ifndef ATOMISP_PLATFORM_H_
 #define ATOMISP_PLATFORM_H_
 
+#include <asm/intel-family.h>
+#include <asm/processor.h>
+
 #include <linux/i2c.h>
 #include <linux/sfi.h>
 #include <media/v4l2-subdev.h>
@@ -237,11 +241,19 @@ const struct atomisp_camera_caps *atomisp_get_default_camera_caps(void);
 /* API from old platform_camera.h, new CPUID implementation */
 #define __IS_SOC(x) (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && \
 		     boot_cpu_data.x86 == 6 &&                       \
-		     boot_cpu_data.x86_model == x)
+		     boot_cpu_data.x86_model == (x))
+#define __IS_SOCS(x,y) (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && \
+		        boot_cpu_data.x86 == 6 &&                       \
+		        (boot_cpu_data.x86_model == (x) || \
+		         boot_cpu_data.x86_model == (y)))
+
+#define IS_MFLD	__IS_SOC(INTEL_FAM6_ATOM_SALTWELL_MID)
+#define IS_BYT	__IS_SOC(INTEL_FAM6_ATOM_SILVERMONT)
+#define IS_CHT	__IS_SOC(INTEL_FAM6_ATOM_AIRMONT)
+#define IS_MOFD	__IS_SOC(INTEL_FAM6_ATOM_AIRMONT_MID)
 
-#define IS_MFLD	__IS_SOC(0x27)
-#define IS_BYT	__IS_SOC(0x37)
-#define IS_CHT	__IS_SOC(0x4C)
-#define IS_MOFD	__IS_SOC(0x5A)
+/* Both CHT and MOFD come with ISP2401 */
+#define IS_ISP2401 __IS_SOCS(INTEL_FAM6_ATOM_AIRMONT, \
+			     INTEL_FAM6_ATOM_AIRMONT_MID)
 
 #endif /* ATOMISP_PLATFORM_H_ */
diff --git a/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h b/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h
index 1b5111ad9415..abc8fa809bce 100644
--- a/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h
+++ b/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2013 Intel Corporation. All Rights Reserved.
  *
diff --git a/drivers/staging/media/atomisp/include/media/lm3554.h b/drivers/staging/media/atomisp/include/media/lm3554.h
index 03a916ade531..812ce74f0635 100644
--- a/drivers/staging/media/atomisp/include/media/lm3554.h
+++ b/drivers/staging/media/atomisp/include/media/lm3554.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * include/media/lm3554.h
  *
diff --git a/drivers/staging/media/atomisp/include/mmu/isp_mmu.h b/drivers/staging/media/atomisp/include/mmu/isp_mmu.h
index d9662c515236..268560954792 100644
--- a/drivers/staging/media/atomisp/include/mmu/isp_mmu.h
+++ b/drivers/staging/media/atomisp/include/mmu/isp_mmu.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h b/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h
index 662e98f41da2..84fe7a368c14 100644
--- a/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h
+++ b/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Merrifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp-regs.h b/drivers/staging/media/atomisp/pci/atomisp-regs.h
index cc489a331a7c..de34ee28e390 100644
--- a/drivers/staging/media/atomisp/pci/atomisp-regs.h
+++ b/drivers/staging/media/atomisp/pci/atomisp-regs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_acc.c b/drivers/staging/media/atomisp/pci/atomisp_acc.c
index 8d575eb0a73f..76861396ba86 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_acc.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_acc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Clovertrail PNW Camera Imaging ISP subsystem.
  *
@@ -23,24 +24,24 @@
 #include <linux/init.h>
 #include <media/v4l2-event.h>
 
+#include "hmm.h"
+
 #include "atomisp_acc.h"
 #include "atomisp_internal.h"
 #include "atomisp_compat.h"
 #include "atomisp_cmd.h"
 
-#include "hrt/hive_isp_css_mm_hrt.h"
-#include "memory_access/memory_access.h"
 #include "ia_css.h"
 
 static const struct {
 	unsigned int flag;
-	enum atomisp_css_pipe_id pipe_id;
+	enum ia_css_pipe_id pipe_id;
 } acc_flag_to_pipe[] = {
-	{ ATOMISP_ACC_FW_LOAD_FL_PREVIEW, CSS_PIPE_ID_PREVIEW },
-	{ ATOMISP_ACC_FW_LOAD_FL_COPY, CSS_PIPE_ID_COPY },
-	{ ATOMISP_ACC_FW_LOAD_FL_VIDEO, CSS_PIPE_ID_VIDEO },
-	{ ATOMISP_ACC_FW_LOAD_FL_CAPTURE, CSS_PIPE_ID_CAPTURE },
-	{ ATOMISP_ACC_FW_LOAD_FL_ACC, CSS_PIPE_ID_ACC }
+	{ ATOMISP_ACC_FW_LOAD_FL_PREVIEW, IA_CSS_PIPE_ID_PREVIEW },
+	{ ATOMISP_ACC_FW_LOAD_FL_COPY, IA_CSS_PIPE_ID_COPY },
+	{ ATOMISP_ACC_FW_LOAD_FL_VIDEO, IA_CSS_PIPE_ID_VIDEO },
+	{ ATOMISP_ACC_FW_LOAD_FL_CAPTURE, IA_CSS_PIPE_ID_CAPTURE },
+	{ ATOMISP_ACC_FW_LOAD_FL_ACC, IA_CSS_PIPE_ID_ACC }
 };
 
 /*
@@ -353,16 +354,23 @@ int atomisp_acc_map(struct atomisp_sub_device *asd, struct atomisp_acc_map *map)
 		}
 
 		pgnr = DIV_ROUND_UP(map->length, PAGE_SIZE);
-		cssptr = hrt_isp_css_mm_alloc_user_ptr(map->length,
-						       map->user_ptr,
-						       pgnr, HRT_USR_PTR,
-						       (map->flags & ATOMISP_MAP_FLAG_CACHED));
+		if (pgnr < ((PAGE_ALIGN(map->length)) >> PAGE_SHIFT)) {
+			dev_err(atomisp_dev,
+				"user space memory size is less than the expected size..\n");
+			return -ENOMEM;
+		} else if (pgnr > ((PAGE_ALIGN(map->length)) >> PAGE_SHIFT)) {
+			dev_err(atomisp_dev,
+				"user space memory size is large than the expected size..\n");
+			return -ENOMEM;
+		}
+
+		cssptr = hmm_alloc(map->length, HMM_BO_USER, 0, map->user_ptr,
+				   map->flags & ATOMISP_MAP_FLAG_CACHED);
+
 	} else {
 		/* Allocate private buffer. */
-		if (map->flags & ATOMISP_MAP_FLAG_CACHED)
-			cssptr = hrt_isp_css_mm_calloc_cached(map->length);
-		else
-			cssptr = hrt_isp_css_mm_calloc(map->length);
+		cssptr = hmm_alloc(map->length, HMM_BO_PRIVATE, 0, NULL,
+				   map->flags & ATOMISP_MAP_FLAG_CACHED);
 	}
 
 	if (!cssptr)
@@ -552,7 +560,7 @@ int atomisp_acc_set_state(struct atomisp_sub_device *asd,
 	struct atomisp_acc_fw *acc_fw;
 	bool enable = (arg->flags & ATOMISP_STATE_FLAG_ENABLE) != 0;
 	struct ia_css_pipe *pipe;
-	enum ia_css_err r;
+	int r;
 	int i;
 
 	if (!asd->acc.extension_mode)
@@ -574,7 +582,7 @@ int atomisp_acc_set_state(struct atomisp_sub_device *asd,
 			       pipes[acc_flag_to_pipe[i].pipe_id];
 			r = ia_css_pipe_set_qos_ext_state(pipe, acc_fw->handle,
 							  enable);
-			if (r != IA_CSS_SUCCESS)
+			if (r)
 				return -EBADRQC;
 		}
 	}
diff --git a/drivers/staging/media/atomisp/pci/atomisp_acc.h b/drivers/staging/media/atomisp/pci/atomisp_acc.h
index ba14181962f8..48d94232229b 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_acc.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_acc.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Clovertrail PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
index 5be690f876c1..7b936e5a5f03 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -16,6 +17,7 @@
  *
  *
  */
+#include <linux/errno.h>
 #include <linux/firmware.h>
 #include <linux/pci.h>
 #include <linux/interrupt.h>
@@ -44,7 +46,7 @@
 #include "atomisp_subdev.h"
 #include "atomisp_dfs_tables.h"
 
-#include "hrt/hive_isp_css_mm_hrt.h"
+#include <hmm/hmm.h>
 
 #include "sh_css_hrt.h"
 #include "sh_css_defs.h"
@@ -57,7 +59,7 @@
 
 #include "ia_css_types.h"
 #include "ia_css_stream.h"
-#include "error_support.h"
+#include "ia_css_debug.h"
 #include "bits.h"
 
 /* We should never need to run the flash for more than 2 frames.
@@ -241,8 +243,12 @@ int atomisp_freq_scaling(struct atomisp_device *isp,
 	}
 
 	fps = atomisp_get_sensor_fps(asd);
-	if (fps == 0)
-		return -EINVAL;
+	if (fps == 0) {
+		dev_info(isp->dev,
+			 "Sensor didn't report FPS. Using DFS max mode.\n");
+		new_freq = dfs->highest_freq;
+		goto done;
+	}
 
 	curr_rules.width = asd->fmt[asd->capture_pad].fmt.width;
 	curr_rules.height = asd->fmt[asd->capture_pad].fmt.height;
@@ -445,29 +451,29 @@ static void print_csi_rx_errors(enum mipi_port_id port,
 	atomisp_css_rx_get_irq_info(port, &infos);
 
 	dev_err(isp->dev, "CSI Receiver port %d errors:\n", port);
-	if (infos & CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
+	if (infos & IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
 		dev_err(isp->dev, "  buffer overrun");
-	if (infos & CSS_RX_IRQ_INFO_ERR_SOT)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_SOT)
 		dev_err(isp->dev, "  start-of-transmission error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_SOT_SYNC)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_SOT_SYNC)
 		dev_err(isp->dev, "  start-of-transmission sync error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_CONTROL)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_CONTROL)
 		dev_err(isp->dev, "  control error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE)
 		dev_err(isp->dev, "  2 or more ECC errors");
-	if (infos & CSS_RX_IRQ_INFO_ERR_CRC)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_CRC)
 		dev_err(isp->dev, "  CRC mismatch");
-	if (infos & CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID)
 		dev_err(isp->dev, "  unknown error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_FRAME_SYNC)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_SYNC)
 		dev_err(isp->dev, "  frame sync error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_FRAME_DATA)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_DATA)
 		dev_err(isp->dev, "  frame data error");
-	if (infos & CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT)
 		dev_err(isp->dev, "  data timeout");
-	if (infos & CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC)
 		dev_err(isp->dev, "  unknown escape command entry");
-	if (infos & CSS_RX_IRQ_INFO_ERR_LINE_SYNC)
+	if (infos & IA_CSS_RX_IRQ_INFO_ERR_LINE_SYNC)
 		dev_err(isp->dev, "  line sync error");
 }
 
@@ -526,8 +532,6 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 		return IRQ_NONE;
 	}
 
-	dev_dbg(isp->dev, "irq:0x%x\n", irq_infos);
-
 	clear_irq_reg(isp);
 
 	if (!atomisp_streaming_count(isp) && !atomisp_is_acc_enabled(isp))
@@ -542,7 +546,7 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 		 * Current SOF only support one stream, so the SOF only valid
 		 * either solely one stream is running
 		 */
-		if (irq_infos & CSS_IRQ_INFO_CSS_RECEIVER_SOF) {
+		if (irq_infos & IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF) {
 			atomic_inc(&asd->sof_count);
 			atomisp_sof_event(asd);
 
@@ -559,16 +563,20 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 				atomic_set(&asd->sequence_temp,
 					   atomic_read(&asd->sof_count));
 		}
-		if (irq_infos & CSS_IRQ_INFO_EVENTS_READY)
+		if (irq_infos & IA_CSS_IRQ_INFO_EVENTS_READY)
 			atomic_set(&asd->sequence,
 				   atomic_read(&asd->sequence_temp));
 	}
 
-	if (irq_infos & CSS_IRQ_INFO_CSS_RECEIVER_SOF)
-		irq_infos &= ~CSS_IRQ_INFO_CSS_RECEIVER_SOF;
+	if (irq_infos & IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF) {
+		dev_dbg_ratelimited(isp->dev,
+				    "irq:0x%x (SOF)\n",
+				    irq_infos);
+		irq_infos &= ~IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF;
+	}
 
-	if ((irq_infos & CSS_IRQ_INFO_INPUT_SYSTEM_ERROR) ||
-	    (irq_infos & CSS_IRQ_INFO_IF_ERROR)) {
+	if ((irq_infos & IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR) ||
+	    (irq_infos & IA_CSS_IRQ_INFO_IF_ERROR)) {
 		/* handle mipi receiver error */
 		u32 rx_infos;
 		enum mipi_port_id port;
@@ -583,18 +591,20 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 
 	if (irq_infos & IA_CSS_IRQ_INFO_ISYS_EVENTS_READY) {
 		while (ia_css_dequeue_isys_event(&eof_event.event) ==
-		       IA_CSS_SUCCESS) {
+		       0) {
 			/* EOF Event does not have the css_pipe returned */
 			asd = __get_asd_from_port(isp, eof_event.event.port);
 			if (!asd) {
-				dev_err(isp->dev, "%s:no subdev.event:%d",  __func__,
-					eof_event.event.type);
+				dev_err(isp->dev, "%s: ISYS event, but no subdev.event:%d",
+					__func__, eof_event.event.type);
 				continue;
 			}
 
 			atomisp_eof_event(asd, eof_event.event.exp_id);
-			dev_dbg(isp->dev, "%s EOF exp_id %d, asd %d\n",
-				__func__, eof_event.event.exp_id, asd->index);
+			dev_dbg_ratelimited(isp->dev,
+					    "%s ISYS event: EOF exp_id %d, asd %d\n",
+					    __func__, eof_event.event.exp_id,
+					    asd->index);
 		}
 
 		irq_infos &= ~IA_CSS_IRQ_INFO_ISYS_EVENTS_READY;
@@ -604,11 +614,17 @@ irqreturn_t atomisp_isr(int irq, void *dev)
 
 	spin_unlock_irqrestore(&isp->lock, flags);
 
+	dev_dbg_ratelimited(isp->dev, "irq:0x%x (unhandled)\n", irq_infos);
+
 	return IRQ_WAKE_THREAD;
 
 out_nowake:
 	spin_unlock_irqrestore(&isp->lock, flags);
 
+	if (irq_infos)
+		dev_dbg_ratelimited(isp->dev, "irq:0x%x (ignored, as not streaming anymore)\n",
+				    irq_infos);
+
 	return IRQ_HANDLED;
 }
 
@@ -649,6 +665,7 @@ bool atomisp_buffers_queued_pipe(struct atomisp_video_pipe *pipe)
 void dump_sp_dmem(struct atomisp_device *isp, unsigned int addr,
 		  unsigned int size)
 {
+	u32 __iomem *io_virt_addr;
 	unsigned int data = 0;
 	unsigned int size32 = DIV_ROUND_UP(size, sizeof(u32));
 
@@ -661,20 +678,20 @@ void dump_sp_dmem(struct atomisp_device *isp, unsigned int addr,
 		return;
 	}
 	addr += SP_DMEM_BASE;
+	io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	do {
-		data = _hrt_master_port_uload_32(addr);
-
+		data = *io_virt_addr;
 		dev_dbg(isp->dev, "%s, \t [0x%x]:0x%x\n", __func__, addr, data);
-		addr += sizeof(unsigned int);
+		io_virt_addr += sizeof(u32);
 		size32 -= 1;
 	} while (size32 > 0);
 }
 
 static struct videobuf_buffer *atomisp_css_frame_to_vbuf(
-    struct atomisp_video_pipe *pipe, struct atomisp_css_frame *frame)
+    struct atomisp_video_pipe *pipe, struct ia_css_frame *frame)
 {
 	struct videobuf_vmalloc_memory *vm_mem;
-	struct atomisp_css_frame *handle;
+	struct ia_css_frame *handle;
 	int i;
 
 	for (i = 0; pipe->capq.bufs[i]; i++) {
@@ -756,12 +773,12 @@ static void atomisp_recover_params_queue(struct atomisp_video_pipe *pipe)
 static struct atomisp_video_pipe *__atomisp_get_pipe(
     struct atomisp_sub_device *asd,
     enum atomisp_input_stream_id stream_id,
-    enum atomisp_css_pipe_id css_pipe_id,
-    enum atomisp_css_buffer_type buf_type)
+    enum ia_css_pipe_id css_pipe_id,
+    enum ia_css_buffer_type buf_type)
 {
 	struct atomisp_device *isp = asd->isp;
 
-	if (css_pipe_id == CSS_PIPE_ID_COPY &&
+	if (css_pipe_id == IA_CSS_PIPE_ID_COPY &&
 	    isp->inputs[asd->input_curr].camera_caps->
 	    sensor[asd->sensor_curr].stream_num > 1) {
 		switch (stream_id) {
@@ -792,7 +809,7 @@ static struct atomisp_video_pipe *__atomisp_get_pipe(
 		 * buffering.
 		 */
 		return &asd->video_out_video_capture;
-	} else if (css_pipe_id == CSS_PIPE_ID_YUVPP) {
+	} else if (css_pipe_id == IA_CSS_PIPE_ID_YUVPP) {
 		/*
 		 * to SOC camera, yuvpp pipe is run for capture/video/SDV/ZSL.
 		 */
@@ -800,11 +817,11 @@ static struct atomisp_video_pipe *__atomisp_get_pipe(
 			if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 				/* SDV case */
 				switch (buf_type) {
-				case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
+				case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
 					return &asd->video_out_video_capture;
-				case CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
+				case IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
 					return &asd->video_out_preview;
-				case CSS_BUFFER_TYPE_OUTPUT_FRAME:
+				case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME:
 					return &asd->video_out_capture;
 				default:
 					return &asd->video_out_vf;
@@ -812,15 +829,15 @@ static struct atomisp_video_pipe *__atomisp_get_pipe(
 			} else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
 				/* ZSL case */
 				switch (buf_type) {
-				case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
+				case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
 					return &asd->video_out_preview;
-				case CSS_BUFFER_TYPE_OUTPUT_FRAME:
+				case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME:
 					return &asd->video_out_capture;
 				default:
 					return &asd->video_out_vf;
 				}
 			}
-		} else if (buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME) {
+		} else if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) {
 			switch (asd->run_mode->val) {
 			case ATOMISP_RUN_MODE_VIDEO:
 				return &asd->video_out_video_capture;
@@ -829,7 +846,7 @@ static struct atomisp_video_pipe *__atomisp_get_pipe(
 			default:
 				return &asd->video_out_capture;
 			}
-		} else if (buf_type == CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) {
+		} else if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) {
 			if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
 				return &asd->video_out_preview;
 			else
@@ -837,20 +854,20 @@ static struct atomisp_video_pipe *__atomisp_get_pipe(
 		}
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 		/* For online video or SDV video pipe. */
-		if (css_pipe_id == CSS_PIPE_ID_VIDEO ||
-		    css_pipe_id == CSS_PIPE_ID_COPY) {
-			if (buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
+		if (css_pipe_id == IA_CSS_PIPE_ID_VIDEO ||
+		    css_pipe_id == IA_CSS_PIPE_ID_COPY) {
+			if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)
 				return &asd->video_out_video_capture;
 			return &asd->video_out_preview;
 		}
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
 		/* For online preview or ZSL preview pipe. */
-		if (css_pipe_id == CSS_PIPE_ID_PREVIEW ||
-		    css_pipe_id == CSS_PIPE_ID_COPY)
+		if (css_pipe_id == IA_CSS_PIPE_ID_PREVIEW ||
+		    css_pipe_id == IA_CSS_PIPE_ID_COPY)
 			return &asd->video_out_preview;
 	}
 	/* For capture pipe. */
-	if (buf_type == CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
+	if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
 		return &asd->video_out_vf;
 	return &asd->video_out_capture;
 }
@@ -868,8 +885,8 @@ atomisp_get_metadata_type(struct atomisp_sub_device *asd,
 }
 
 void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
-		      enum atomisp_css_buffer_type buf_type,
-		      enum atomisp_css_pipe_id css_pipe_id,
+		      enum ia_css_buffer_type buf_type,
+		      enum ia_css_pipe_id css_pipe_id,
 		      bool q_buffers, enum atomisp_input_stream_id stream_id)
 {
 	struct videobuf_buffer *vb = NULL;
@@ -878,7 +895,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 	bool requeue = false;
 	int err;
 	unsigned long irqflags;
-	struct atomisp_css_frame *frame = NULL;
+	struct ia_css_frame *frame = NULL;
 	struct atomisp_s3a_buf *s3a_buf = NULL, *_s3a_buf_tmp;
 	struct atomisp_dis_buf *dis_buf = NULL, *_dis_buf_tmp;
 	struct atomisp_metadata_buf *md_buf = NULL, *_md_buf_tmp;
@@ -888,14 +905,14 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 	bool reset_wdt_timer = false;
 
 	if (
-	    buf_type != CSS_BUFFER_TYPE_METADATA &&
-	    buf_type != CSS_BUFFER_TYPE_3A_STATISTICS &&
-	    buf_type != CSS_BUFFER_TYPE_DIS_STATISTICS &&
-	    buf_type != CSS_BUFFER_TYPE_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME &&
-	    buf_type != CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) {
+	    buf_type != IA_CSS_BUFFER_TYPE_METADATA &&
+	    buf_type != IA_CSS_BUFFER_TYPE_3A_STATISTICS &&
+	    buf_type != IA_CSS_BUFFER_TYPE_DIS_STATISTICS &&
+	    buf_type != IA_CSS_BUFFER_TYPE_OUTPUT_FRAME &&
+	    buf_type != IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME &&
+	    buf_type != IA_CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME &&
+	    buf_type != IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME &&
+	    buf_type != IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) {
 		dev_err(isp->dev, "%s, unsupported buffer type: %d\n",
 			__func__, buf_type);
 		return;
@@ -919,7 +936,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 	}
 
 	switch (buf_type) {
-	case CSS_BUFFER_TYPE_3A_STATISTICS:
+	case IA_CSS_BUFFER_TYPE_3A_STATISTICS:
 		list_for_each_entry_safe(s3a_buf, _s3a_buf_tmp,
 					 &asd->s3a_stats_in_css, list) {
 			if (s3a_buf->s3a_data ==
@@ -936,7 +953,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		dev_dbg(isp->dev, "%s: s3a stat with exp_id %d is ready\n",
 			__func__, s3a_buf->s3a_data->exp_id);
 		break;
-	case CSS_BUFFER_TYPE_METADATA:
+	case IA_CSS_BUFFER_TYPE_METADATA:
 		if (error)
 			break;
 
@@ -956,7 +973,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		dev_dbg(isp->dev, "%s: metadata with exp_id %d is ready\n",
 			__func__, md_buf->metadata->exp_id);
 		break;
-	case CSS_BUFFER_TYPE_DIS_STATISTICS:
+	case IA_CSS_BUFFER_TYPE_DIS_STATISTICS:
 		list_for_each_entry_safe(dis_buf, _dis_buf_tmp,
 					 &asd->dis_stats_in_css, list) {
 			if (dis_buf->dis_data ==
@@ -975,9 +992,9 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		dev_dbg(isp->dev, "%s: dis stat with exp_id %d is ready\n",
 			__func__, dis_buf->dis_data->exp_id);
 		break;
-	case CSS_BUFFER_TYPE_VF_OUTPUT_FRAME:
-	case CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
-		if (atomisp_hw_is_isp2401)
+	case IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME:
+	case IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
+		if (IS_ISP2401)
 			reset_wdt_timer = true;
 
 		pipe->buffers_in_css--;
@@ -993,7 +1010,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		 * YUVPP doesn't set postview exp_id correctlly in SDV mode.
 		 * This is a WORKAROUND to set exp_id. see HSDES-1503911606.
 		 */
-		if (IS_BYT && buf_type == CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME &&
+		if (IS_BYT && buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME &&
 		    asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd))
 			frame->exp_id = (asd->postview_exp_id++) %
 					(ATOMISP_MAX_EXP_ID + 1);
@@ -1002,11 +1019,11 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 			__func__, frame->exp_id);
 		if (asd->params.flash_state == ATOMISP_FLASH_ONGOING) {
 			if (frame->flash_state
-			    == CSS_FRAME_FLASH_STATE_PARTIAL)
+			    == IA_CSS_FRAME_FLASH_STATE_PARTIAL)
 				dev_dbg(isp->dev, "%s thumb partially flashed\n",
 					__func__);
 			else if (frame->flash_state
-				 == CSS_FRAME_FLASH_STATE_FULL)
+				 == IA_CSS_FRAME_FLASH_STATE_FULL)
 				dev_dbg(isp->dev, "%s thumb completely flashed\n",
 					__func__);
 			else
@@ -1026,18 +1043,18 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 
 			asd->pending_capture_request--;
 
-			if (atomisp_hw_is_isp2401)
+			if (IS_ISP2401)
 				asd->re_trigger_capture = false;
 
 			dev_dbg(isp->dev, "Trigger capture again for new buffer. err=%d\n",
 				err);
-		} else if (atomisp_hw_is_isp2401) {
+		} else if (IS_ISP2401) {
 			asd->re_trigger_capture = true;
 		}
 		break;
-	case CSS_BUFFER_TYPE_OUTPUT_FRAME:
-	case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
-		if (atomisp_hw_is_isp2401)
+	case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME:
+	case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
+		if (IS_ISP2401)
 			reset_wdt_timer = true;
 
 		pipe->buffers_in_css--;
@@ -1054,7 +1071,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		 * YUVPP doesn't set preview exp_id correctlly in ZSL mode.
 		 * This is a WORKAROUND to set exp_id. see HSDES-1503911606.
 		 */
-		if (IS_BYT && buf_type == CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME &&
+		if (IS_BYT && buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME &&
 		    asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd))
 			frame->exp_id = (asd->preview_exp_id++) %
 					(ATOMISP_MAX_EXP_ID + 1);
@@ -1082,13 +1099,13 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		ctrl.id = V4L2_CID_FLASH_MODE;
 		if (asd->params.flash_state == ATOMISP_FLASH_ONGOING) {
 			if (frame->flash_state
-			    == CSS_FRAME_FLASH_STATE_PARTIAL) {
+			    == IA_CSS_FRAME_FLASH_STATE_PARTIAL) {
 				asd->frame_status[vb->i] =
 				    ATOMISP_FRAME_STATUS_FLASH_PARTIAL;
 				dev_dbg(isp->dev, "%s partially flashed\n",
 					__func__);
 			} else if (frame->flash_state
-				   == CSS_FRAME_FLASH_STATE_FULL) {
+				   == IA_CSS_FRAME_FLASH_STATE_FULL) {
 				asd->frame_status[vb->i] =
 				    ATOMISP_FRAME_STATUS_FLASH_EXPOSED;
 				asd->params.num_flash_frames--;
@@ -1128,11 +1145,11 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		asd->params.last_frame_status = asd->frame_status[vb->i];
 
 		if (asd->continuous_mode->val) {
-			if (css_pipe_id == CSS_PIPE_ID_PREVIEW ||
-			    css_pipe_id == CSS_PIPE_ID_VIDEO) {
+			if (css_pipe_id == IA_CSS_PIPE_ID_PREVIEW ||
+			    css_pipe_id == IA_CSS_PIPE_ID_VIDEO) {
 				asd->latest_preview_exp_id = frame->exp_id;
 			} else if (css_pipe_id ==
-				   CSS_PIPE_ID_CAPTURE) {
+				   IA_CSS_PIPE_ID_CAPTURE) {
 				if (asd->run_mode->val ==
 				    ATOMISP_RUN_MODE_VIDEO)
 					dev_dbg(isp->dev, "SDV capture raw buffer id: %u\n",
@@ -1148,8 +1165,8 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		 * in preview/video pipe, each buffer will
 		 * be locked automatically, so record it here.
 		 */
-		if (((css_pipe_id == CSS_PIPE_ID_PREVIEW) ||
-		     (css_pipe_id == CSS_PIPE_ID_VIDEO)) &&
+		if (((css_pipe_id == IA_CSS_PIPE_ID_PREVIEW) ||
+		     (css_pipe_id == IA_CSS_PIPE_ID_VIDEO)) &&
 		    asd->enable_raw_buffer_lock->val &&
 		    asd->continuous_mode->val) {
 			atomisp_set_raw_buffer_bitmap(asd, frame->exp_id);
@@ -1160,8 +1177,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 			atomisp_apply_css_parameters(asd,
 						     &asd->params.css_param);
 			if (asd->params.css_param.update_flag.dz_config)
-				atomisp_css_set_dz_config(asd,
-							  &asd->params.css_param.dz_config);
+				asd->params.config.dz_config = &asd->params.css_param.dz_config;
 			/* New global dvs 6axis config should be blocked
 			 * here if there's a buffer with per-frame parameters
 			 * pending in CSS frame buffer queue.
@@ -1208,7 +1224,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 		 */
 		wake_up(&vb->done);
 	}
-	if (atomisp_hw_is_isp2401)
+	if (IS_ISP2401)
 		atomic_set(&pipe->wdt_count, 0);
 
 	/*
@@ -1228,7 +1244,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 	if (!error && q_buffers)
 		atomisp_qbuffers_to_css(asd);
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		/* If there are no buffers queued then
 		* delete wdt timer. */
 		if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED)
@@ -1252,9 +1268,15 @@ void atomisp_delayed_init_work(struct work_struct *work)
 	 */
 	if (!ATOMISP_USE_YUVPP(asd)) {
 		struct v4l2_event event = {0};
+		struct ia_css_stream *stream;
+
+		stream = asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream;
+
+
+		if (ia_css_alloc_continuous_frame_remain(stream))
+			return;
 
-		atomisp_css_allocate_continuous_frames(false, asd);
-		atomisp_css_update_continuous_frames(asd);
+		ia_css_update_continuous_frames(stream);
 
 		event.type = V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE;
 		v4l2_event_queue(asd->subdev.devnode, &event);
@@ -1267,14 +1289,14 @@ void atomisp_delayed_init_work(struct work_struct *work)
 
 static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 {
-	enum atomisp_css_pipe_id css_pipe_id;
+	enum ia_css_pipe_id css_pipe_id;
 	bool stream_restart[MAX_STREAM_NUM] = {0};
 	bool depth_mode = false;
 	int i, ret, depth_cnt = 0;
 
 	if (!isp->sw_contex.file_input)
 		atomisp_css_irq_enable(isp,
-				       CSS_IRQ_INFO_CSS_RECEIVER_SOF, false);
+				       IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF, false);
 
 	BUG_ON(isp->num_of_streams > MAX_STREAM_NUM);
 
@@ -1292,7 +1314,7 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 		* By calling acc_done() for all loaded fw_handles,
 		* HAL will be unblocked.
 		*/
-		acc_pipe = asd->stream_env[i].pipes[CSS_PIPE_ID_ACC];
+		acc_pipe = asd->stream_env[i].pipes[IA_CSS_PIPE_ID_ACC];
 		if (acc_pipe) {
 			acc_pipeline = ia_css_pipe_get_pipeline(acc_pipe);
 			if (acc_pipeline) {
@@ -1373,7 +1395,7 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 
 		if (isp->inputs[asd->input_curr].type != FILE_INPUT)
 			atomisp_css_input_set_mode(asd,
-						   CSS_INPUT_MODE_SENSOR);
+						   IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
 
 		css_pipe_id = atomisp_get_css_pipe_id(asd);
 		if (atomisp_css_start(asd, css_pipe_id, true))
@@ -1386,14 +1408,14 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout)
 	}
 
 	if (!isp->sw_contex.file_input) {
-		atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF,
+		atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF,
 				       atomisp_css_valid_sof(isp));
 
 		if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_AUTO, true) < 0)
-			dev_dbg(isp->dev, "dfs failed!\n");
+			dev_dbg(isp->dev, "DFS auto failed while recovering!\n");
 	} else {
 		if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, true) < 0)
-			dev_dbg(isp->dev, "dfs failed!\n");
+			dev_dbg(isp->dev, "DFS max failed while recovering!\n");
 	}
 
 	for (i = 0; i < isp->num_of_streams; i++) {
@@ -1458,7 +1480,7 @@ void atomisp_wdt_work(struct work_struct *work)
 		return;
 	}
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		dev_err(isp->dev, "timeout %d of %d\n",
 			atomic_read(&isp->wdt_count) + 1,
 			ATOMISP_ISP_MAX_TIMEOUT_COUNT);
@@ -1494,11 +1516,8 @@ void atomisp_wdt_work(struct work_struct *work)
 	}
 
 	if (css_recover) {
-		unsigned int old_dbglevel = dbg_level;
-
-		atomisp_css_debug_dump_sp_sw_debug_info();
-		atomisp_css_debug_dump_debug_info(__func__);
-		dbg_level = old_dbglevel;
+		ia_css_debug_dump_sp_sw_debug_info();
+		ia_css_debug_dump_debug_info(__func__);
 		for (i = 0; i < isp->num_of_streams; i++) {
 			struct atomisp_sub_device *asd = &isp->asd[i];
 
@@ -1530,15 +1549,15 @@ void atomisp_wdt_work(struct work_struct *work)
 			dev_err(isp->dev,
 				"%s, s3a buffers in css preview pipe:%d\n",
 				__func__,
-				asd->s3a_bufs_in_css[CSS_PIPE_ID_PREVIEW]);
+				asd->s3a_bufs_in_css[IA_CSS_PIPE_ID_PREVIEW]);
 			dev_err(isp->dev,
 				"%s, s3a buffers in css capture pipe:%d\n",
 				__func__,
-				asd->s3a_bufs_in_css[CSS_PIPE_ID_CAPTURE]);
+				asd->s3a_bufs_in_css[IA_CSS_PIPE_ID_CAPTURE]);
 			dev_err(isp->dev,
 				"%s, s3a buffers in css video pipe:%d\n",
 				__func__,
-				asd->s3a_bufs_in_css[CSS_PIPE_ID_VIDEO]);
+				asd->s3a_bufs_in_css[IA_CSS_PIPE_ID_VIDEO]);
 			dev_err(isp->dev,
 				"%s, dis buffers in css: %d\n",
 				__func__, asd->dis_bufs_in_css);
@@ -1547,19 +1566,19 @@ void atomisp_wdt_work(struct work_struct *work)
 				__func__,
 				asd->metadata_bufs_in_css
 				[ATOMISP_INPUT_STREAM_GENERAL]
-				[CSS_PIPE_ID_PREVIEW]);
+				[IA_CSS_PIPE_ID_PREVIEW]);
 			dev_err(isp->dev,
 				"%s, metadata buffers in css capture pipe:%d\n",
 				__func__,
 				asd->metadata_bufs_in_css
 				[ATOMISP_INPUT_STREAM_GENERAL]
-				[CSS_PIPE_ID_CAPTURE]);
+				[IA_CSS_PIPE_ID_CAPTURE]);
 			dev_err(isp->dev,
 				"%s, metadata buffers in css video pipe:%d\n",
 				__func__,
 				asd->metadata_bufs_in_css
 				[ATOMISP_INPUT_STREAM_GENERAL]
-				[CSS_PIPE_ID_VIDEO]);
+				[IA_CSS_PIPE_ID_VIDEO]);
 			if (asd->enable_raw_buffer_lock->val) {
 				unsigned int j;
 
@@ -1584,11 +1603,11 @@ void atomisp_wdt_work(struct work_struct *work)
 				atomisp_flush_bufs_and_wakeup(asd);
 				complete(&asd->init_done);
 			}
-			if (atomisp_hw_is_isp2401)
+			if (IS_ISP2401)
 				atomisp_wdt_stop(asd, false);
 		}
 
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			atomic_set(&isp->wdt_count, 0);
 		} else {
 			isp->isp_fatal_error = true;
@@ -1600,7 +1619,7 @@ void atomisp_wdt_work(struct work_struct *work)
 	}
 
 	__atomisp_css_recover(isp, true);
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		for (i = 0; i < isp->num_of_streams; i++) {
 			struct atomisp_sub_device *asd = &isp->asd[i];
 
@@ -1657,7 +1676,7 @@ void atomisp_wdt(struct timer_list *t)
 	struct atomisp_sub_device *asd;
 	struct atomisp_device *isp;
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		asd = from_timer(asd, t, wdt);
 		isp = asd->isp;
 	} else {
@@ -1716,7 +1735,7 @@ void atomisp_wdt_refresh_pipe(struct atomisp_video_pipe *pipe,
 
 void atomisp_wdt_refresh(struct atomisp_sub_device *asd, unsigned int delay)
 {
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		unsigned long next;
 
 		if (delay != ATOMISP_WDT_KEEP_CURRENT_DELAY)
@@ -1779,7 +1798,7 @@ void atomisp_wdt_stop(struct atomisp_sub_device *asd, bool sync)
 {
 	dev_dbg(asd->isp->dev, "WDT stop:\n");
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		if (sync) {
 			del_timer_sync(&asd->wdt);
 			cancel_work_sync(&asd->isp->wdt_work);
@@ -1816,7 +1835,8 @@ void atomisp_setup_flash(struct atomisp_sub_device *asd)
 			return;
 		}
 
-		atomisp_css_request_flash(asd);
+		ia_css_stream_request_flash(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream);
+
 		asd->params.flash_state = ATOMISP_FLASH_ONGOING;
 	} else {
 		asd->params.flash_state = ATOMISP_FLASH_IDLE;
@@ -1901,7 +1921,7 @@ out:
  */
 
 int atomisp_get_frame_pgnr(struct atomisp_device *isp,
-			   const struct atomisp_css_frame *frame, u32 *p_pgnr)
+			   const struct ia_css_frame *frame, u32 *p_pgnr)
 {
 	if (!frame) {
 		dev_err(isp->dev, "%s: NULL frame pointer ERROR.\n", __func__);
@@ -1915,39 +1935,39 @@ int atomisp_get_frame_pgnr(struct atomisp_device *isp,
 /*
  * Get internal fmt according to V4L2 fmt
  */
-static enum atomisp_css_frame_format
+static enum ia_css_frame_format
 v4l2_fmt_to_sh_fmt(u32 fmt) {
 	switch (fmt)
 	{
 	case V4L2_PIX_FMT_YUV420:
-				return CSS_FRAME_FORMAT_YUV420;
+				return IA_CSS_FRAME_FORMAT_YUV420;
 	case V4L2_PIX_FMT_YVU420:
-		return CSS_FRAME_FORMAT_YV12;
+		return IA_CSS_FRAME_FORMAT_YV12;
 	case V4L2_PIX_FMT_YUV422P:
-		return CSS_FRAME_FORMAT_YUV422;
+		return IA_CSS_FRAME_FORMAT_YUV422;
 	case V4L2_PIX_FMT_YUV444:
-		return CSS_FRAME_FORMAT_YUV444;
+		return IA_CSS_FRAME_FORMAT_YUV444;
 	case V4L2_PIX_FMT_NV12:
-		return CSS_FRAME_FORMAT_NV12;
+		return IA_CSS_FRAME_FORMAT_NV12;
 	case V4L2_PIX_FMT_NV21:
-		return CSS_FRAME_FORMAT_NV21;
+		return IA_CSS_FRAME_FORMAT_NV21;
 	case V4L2_PIX_FMT_NV16:
-		return CSS_FRAME_FORMAT_NV16;
+		return IA_CSS_FRAME_FORMAT_NV16;
 	case V4L2_PIX_FMT_NV61:
-		return CSS_FRAME_FORMAT_NV61;
+		return IA_CSS_FRAME_FORMAT_NV61;
 	case V4L2_PIX_FMT_UYVY:
-		return CSS_FRAME_FORMAT_UYVY;
+		return IA_CSS_FRAME_FORMAT_UYVY;
 	case V4L2_PIX_FMT_YUYV:
-		return CSS_FRAME_FORMAT_YUYV;
+		return IA_CSS_FRAME_FORMAT_YUYV;
 	case V4L2_PIX_FMT_RGB24:
-		return CSS_FRAME_FORMAT_PLANAR_RGB888;
+		return IA_CSS_FRAME_FORMAT_PLANAR_RGB888;
 	case V4L2_PIX_FMT_RGB32:
-		return CSS_FRAME_FORMAT_RGBA888;
+		return IA_CSS_FRAME_FORMAT_RGBA888;
 	case V4L2_PIX_FMT_RGB565:
-		return CSS_FRAME_FORMAT_RGB565;
+		return IA_CSS_FRAME_FORMAT_RGB565;
 	case V4L2_PIX_FMT_JPEG:
 	case V4L2_PIX_FMT_CUSTOM_M10MO_RAW:
-		return CSS_FRAME_FORMAT_BINARY_8;
+		return IA_CSS_FRAME_FORMAT_BINARY_8;
 	case V4L2_PIX_FMT_SBGGR16:
 	case V4L2_PIX_FMT_SBGGR10:
 	case V4L2_PIX_FMT_SGBRG10:
@@ -1961,7 +1981,7 @@ v4l2_fmt_to_sh_fmt(u32 fmt) {
 	case V4L2_PIX_FMT_SGBRG8:
 	case V4L2_PIX_FMT_SGRBG8:
 	case V4L2_PIX_FMT_SRGGB8:
-		return CSS_FRAME_FORMAT_RAW;
+		return IA_CSS_FRAME_FORMAT_RAW;
 	default:
 		return -EINVAL;
 	}
@@ -1972,28 +1992,28 @@ v4l2_fmt_to_sh_fmt(u32 fmt) {
  */
 static int raw_output_format_match_input(u32 input, u32 output)
 {
-	if ((input == CSS_FORMAT_RAW_12) &&
+	if ((input == ATOMISP_INPUT_FORMAT_RAW_12) &&
 	    ((output == V4L2_PIX_FMT_SRGGB12) ||
 	     (output == V4L2_PIX_FMT_SGRBG12) ||
 	     (output == V4L2_PIX_FMT_SBGGR12) ||
 	     (output == V4L2_PIX_FMT_SGBRG12)))
 		return 0;
 
-	if ((input == CSS_FORMAT_RAW_10) &&
+	if ((input == ATOMISP_INPUT_FORMAT_RAW_10) &&
 	    ((output == V4L2_PIX_FMT_SRGGB10) ||
 	     (output == V4L2_PIX_FMT_SGRBG10) ||
 	     (output == V4L2_PIX_FMT_SBGGR10) ||
 	     (output == V4L2_PIX_FMT_SGBRG10)))
 		return 0;
 
-	if ((input == CSS_FORMAT_RAW_8) &&
+	if ((input == ATOMISP_INPUT_FORMAT_RAW_8) &&
 	    ((output == V4L2_PIX_FMT_SRGGB8) ||
 	     (output == V4L2_PIX_FMT_SGRBG8) ||
 	     (output == V4L2_PIX_FMT_SBGGR8) ||
 	     (output == V4L2_PIX_FMT_SGBRG8)))
 		return 0;
 
-	if ((input == CSS_FORMAT_RAW_16) && (output == V4L2_PIX_FMT_SBGGR16))
+	if ((input == ATOMISP_INPUT_FORMAT_RAW_16) && (output == V4L2_PIX_FMT_SBGGR16))
 		return 0;
 
 	return -EINVAL;
@@ -2055,13 +2075,13 @@ bool atomisp_is_mbuscode_raw(uint32_t code)
 static void atomisp_update_capture_mode(struct atomisp_sub_device *asd)
 {
 	if (asd->params.gdc_cac_en)
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_ADVANCED);
+		atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_ADVANCED);
 	else if (asd->params.low_light)
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_LOW_LIGHT);
-	else if (asd->video_out_capture.sh_fmt == CSS_FRAME_FORMAT_RAW)
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_RAW);
+		atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_LOW_LIGHT);
+	else if (asd->video_out_capture.sh_fmt == IA_CSS_FRAME_FORMAT_RAW)
+		atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_RAW);
 	else
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_PRIMARY);
+		atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_PRIMARY);
 }
 
 /* ISP2401 */
@@ -2100,10 +2120,9 @@ int atomisp_gdc_cac(struct atomisp_sub_device *asd, int flag,
 
 	asd->params.gdc_cac_en = !!*value;
 	if (asd->params.gdc_cac_en) {
-		atomisp_css_set_morph_table(asd,
-					    asd->params.css_param.morph_table);
+		asd->params.config.morph_table = asd->params.css_param.morph_table;
 	} else {
-		atomisp_css_set_morph_table(asd, NULL);
+		asd->params.config.morph_table = NULL;
 	}
 	asd->params.css_update_params_needed = true;
 	atomisp_update_capture_mode(asd);
@@ -2156,8 +2175,8 @@ int atomisp_nr(struct atomisp_sub_device *asd, int flag,
 	} else {
 		/* Set nr config to isp parameters */
 		memcpy(&asd->params.css_param.nr_config, arg,
-		       sizeof(struct atomisp_css_nr_config));
-		atomisp_css_set_nr_config(asd, &asd->params.css_param.nr_config);
+		       sizeof(struct ia_css_nr_config));
+		asd->params.config.nr_config = &asd->params.css_param.nr_config;
 		asd->params.css_update_params_needed = true;
 	}
 	return 0;
@@ -2177,8 +2196,8 @@ int atomisp_tnr(struct atomisp_sub_device *asd, int flag,
 	} else {
 		/* Set tnr config to isp parameters */
 		memcpy(&asd->params.css_param.tnr_config, config,
-		       sizeof(struct atomisp_css_tnr_config));
-		atomisp_css_set_tnr_config(asd, &asd->params.css_param.tnr_config);
+		       sizeof(struct ia_css_tnr_config));
+		asd->params.config.tnr_config = &asd->params.css_param.tnr_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -2198,8 +2217,8 @@ int atomisp_black_level(struct atomisp_sub_device *asd, int flag,
 	} else {
 		/* Set ob config to isp parameters */
 		memcpy(&asd->params.css_param.ob_config, config,
-		       sizeof(struct atomisp_css_ob_config));
-		atomisp_css_set_ob_config(asd, &asd->params.css_param.ob_config);
+		       sizeof(struct ia_css_ob_config));
+		asd->params.config.ob_config = &asd->params.css_param.ob_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -2220,7 +2239,7 @@ int atomisp_ee(struct atomisp_sub_device *asd, int flag,
 		/* Set ee config to isp parameters */
 		memcpy(&asd->params.css_param.ee_config, config,
 		       sizeof(asd->params.css_param.ee_config));
-		atomisp_css_set_ee_config(asd, &asd->params.css_param.ee_config);
+		asd->params.config.ee_config = &asd->params.css_param.ee_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -2241,7 +2260,7 @@ int atomisp_gamma(struct atomisp_sub_device *asd, int flag,
 		/* Set gamma table to isp parameters */
 		memcpy(&asd->params.css_param.gamma_table, config,
 		       sizeof(asd->params.css_param.gamma_table));
-		atomisp_css_set_gamma_table(asd, &asd->params.css_param.gamma_table);
+		asd->params.config.gamma_table = &asd->params.css_param.gamma_table;
 	}
 
 	return 0;
@@ -2281,7 +2300,7 @@ int atomisp_gamma_correction(struct atomisp_sub_device *asd, int flag,
 		/* Set gamma correction params to isp parameters */
 		memcpy(&asd->params.css_param.gc_config, config,
 		       sizeof(asd->params.css_param.gc_config));
-		atomisp_css_set_gc_config(asd, &asd->params.css_param.gc_config);
+		asd->params.config.gc_config = &asd->params.css_param.gc_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -2302,7 +2321,7 @@ int atomisp_formats(struct atomisp_sub_device *asd, int flag,
 		/* Set narrow gamma flag to isp parameters */
 		memcpy(&asd->params.css_param.formats_config, config,
 		       sizeof(asd->params.css_param.formats_config));
-		atomisp_css_set_formats_config(asd, &asd->params.css_param.formats_config);
+		asd->params.config.formats_config = &asd->params.css_param.formats_config;
 	}
 
 	return 0;
@@ -2313,13 +2332,13 @@ void atomisp_free_internal_buffers(struct atomisp_sub_device *asd)
 	atomisp_free_css_parameters(&asd->params.css_param);
 
 	if (asd->raw_output_frame) {
-		atomisp_css_frame_free(asd->raw_output_frame);
+		ia_css_frame_free(asd->raw_output_frame);
 		asd->raw_output_frame = NULL;
 	}
 }
 
 static void atomisp_update_grid_info(struct atomisp_sub_device *asd,
-				     enum atomisp_css_pipe_id pipe_id,
+				     enum ia_css_pipe_id pipe_id,
 				     int source_pad)
 {
 	struct atomisp_device *isp = asd->isp;
@@ -2371,7 +2390,7 @@ static void atomisp_curr_user_grid_info(struct atomisp_sub_device *asd,
 					struct atomisp_grid_info *info)
 {
 	memcpy(info, &asd->params.curr_grid_info.s3a_grid,
-	       sizeof(struct atomisp_css_3a_grid_info));
+	       sizeof(struct ia_css_3a_grid_info));
 }
 
 int atomisp_compare_grid(struct atomisp_sub_device *asd,
@@ -2395,14 +2414,14 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag,
 
 	if (flag == 0) {
 		/* Get gdc table from current setup */
-		struct atomisp_css_morph_table tab = {0};
+		struct ia_css_morph_table tab = {0};
 
 		atomisp_css_get_morph_table(asd, &tab);
 
 		config->width = tab.width;
 		config->height = tab.height;
 
-		for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) {
+		for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
 			ret = copy_to_user(config->coordinates_x[i],
 					   tab.coordinates_x[i], tab.height *
 					   tab.width * sizeof(*tab.coordinates_x[i]));
@@ -2421,7 +2440,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag,
 			}
 		}
 	} else {
-		struct atomisp_css_morph_table *tab =
+		struct ia_css_morph_table *tab =
 			    asd->params.css_param.morph_table;
 
 		/* free first if we have one */
@@ -2439,7 +2458,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag,
 			return -EINVAL;
 		}
 
-		for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) {
+		for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
 			ret = copy_from_user(tab->coordinates_x[i],
 					     config->coordinates_x[i],
 					     config->height * config->width *
@@ -2465,7 +2484,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag,
 		}
 		asd->params.css_param.morph_table = tab;
 		if (asd->params.gdc_cac_en)
-			atomisp_css_set_morph_table(asd, tab);
+			asd->params.config.morph_table = tab;
 	}
 
 	return 0;
@@ -2474,7 +2493,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag,
 int atomisp_macc_table(struct atomisp_sub_device *asd, int flag,
 		       struct atomisp_macc_config *config)
 {
-	struct atomisp_css_macc_table *macc_table;
+	struct ia_css_macc_table *macc_table;
 
 	switch (config->color_effect) {
 	case V4L2_COLORFX_NONE:
@@ -2502,12 +2521,12 @@ int atomisp_macc_table(struct atomisp_sub_device *asd, int flag,
 	if (flag == 0) {
 		/* Get macc table from current setup */
 		memcpy(&config->table, macc_table,
-		       sizeof(struct atomisp_css_macc_table));
+		       sizeof(struct ia_css_macc_table));
 	} else {
 		memcpy(macc_table, &config->table,
-		       sizeof(struct atomisp_css_macc_table));
+		       sizeof(struct ia_css_macc_table));
 		if (config->color_effect == asd->params.color_effect)
-			atomisp_css_set_macc_table(asd, macc_table);
+			asd->params.config.macc_table = macc_table;
 	}
 
 	return 0;
@@ -2567,7 +2586,7 @@ int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd,
 	}
 
 	pipe_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		   .pipe_configs[CSS_PIPE_ID_VIDEO];
+		   .pipe_configs[IA_CSS_PIPE_ID_VIDEO];
 	stream_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
 		     .stream_config;
 	input_config = &stream_cfg->input_config;
@@ -2951,7 +2970,7 @@ int atomisp_get_metadata_by_type(struct atomisp_sub_device *asd, int flag,
  */
 int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
 				       struct ia_css_dz_config   *dz_config,
-				       enum atomisp_css_pipe_id css_pipe_id)
+				       enum ia_css_pipe_id css_pipe_id)
 
 {
 	struct atomisp_stream_env *stream_env =
@@ -3000,7 +3019,7 @@ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
 	 * map real crop region base on above calculating base max crop region.
 	 */
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		dz_config->zoom_region.origin.x = dz_config->zoom_region.origin.x
 						  * eff_res.width
 						  / asd->sensor_array_res.width;
@@ -3144,87 +3163,85 @@ void atomisp_apply_css_parameters(
     struct atomisp_css_params *css_param)
 {
 	if (css_param->update_flag.wb_config)
-		atomisp_css_set_wb_config(asd, &css_param->wb_config);
+		asd->params.config.wb_config = &css_param->wb_config;
 
 	if (css_param->update_flag.ob_config)
-		atomisp_css_set_ob_config(asd, &css_param->ob_config);
+		asd->params.config.ob_config = &css_param->ob_config;
 
 	if (css_param->update_flag.dp_config)
-		atomisp_css_set_dp_config(asd, &css_param->dp_config);
+		asd->params.config.dp_config = &css_param->dp_config;
 
 	if (css_param->update_flag.nr_config)
-		atomisp_css_set_nr_config(asd, &css_param->nr_config);
+		asd->params.config.nr_config = &css_param->nr_config;
 
 	if (css_param->update_flag.ee_config)
-		atomisp_css_set_ee_config(asd, &css_param->ee_config);
+		asd->params.config.ee_config = &css_param->ee_config;
 
 	if (css_param->update_flag.tnr_config)
-		atomisp_css_set_tnr_config(asd, &css_param->tnr_config);
+		asd->params.config.tnr_config = &css_param->tnr_config;
 
 	if (css_param->update_flag.a3a_config)
-		atomisp_css_set_3a_config(asd, &css_param->s3a_config);
+		asd->params.config.s3a_config = &css_param->s3a_config;
 
 	if (css_param->update_flag.ctc_config)
-		atomisp_css_set_ctc_config(asd, &css_param->ctc_config);
+		asd->params.config.ctc_config = &css_param->ctc_config;
 
 	if (css_param->update_flag.cnr_config)
-		atomisp_css_set_cnr_config(asd, &css_param->cnr_config);
+		asd->params.config.cnr_config = &css_param->cnr_config;
 
 	if (css_param->update_flag.ecd_config)
-		atomisp_css_set_ecd_config(asd, &css_param->ecd_config);
+		asd->params.config.ecd_config = &css_param->ecd_config;
 
 	if (css_param->update_flag.ynr_config)
-		atomisp_css_set_ynr_config(asd, &css_param->ynr_config);
+		asd->params.config.ynr_config = &css_param->ynr_config;
 
 	if (css_param->update_flag.fc_config)
-		atomisp_css_set_fc_config(asd, &css_param->fc_config);
+		asd->params.config.fc_config = &css_param->fc_config;
 
 	if (css_param->update_flag.macc_config)
-		atomisp_css_set_macc_config(asd, &css_param->macc_config);
+		asd->params.config.macc_config = &css_param->macc_config;
 
 	if (css_param->update_flag.aa_config)
-		atomisp_css_set_aa_config(asd, &css_param->aa_config);
+		asd->params.config.aa_config = &css_param->aa_config;
 
 	if (css_param->update_flag.anr_config)
-		atomisp_css_set_anr_config(asd, &css_param->anr_config);
+		asd->params.config.anr_config = &css_param->anr_config;
 
 	if (css_param->update_flag.xnr_config)
-		atomisp_css_set_xnr_config(asd, &css_param->xnr_config);
+		asd->params.config.xnr_config = &css_param->xnr_config;
 
 	if (css_param->update_flag.yuv2rgb_cc_config)
-		atomisp_css_set_yuv2rgb_cc_config(asd,
-						  &css_param->yuv2rgb_cc_config);
+		asd->params.config.yuv2rgb_cc_config = &css_param->yuv2rgb_cc_config;
 
 	if (css_param->update_flag.rgb2yuv_cc_config)
-		atomisp_css_set_rgb2yuv_cc_config(asd,
-						  &css_param->rgb2yuv_cc_config);
+		asd->params.config.rgb2yuv_cc_config = &css_param->rgb2yuv_cc_config;
 
 	if (css_param->update_flag.macc_table)
-		atomisp_css_set_macc_table(asd, &css_param->macc_table);
+		asd->params.config.macc_table = &css_param->macc_table;
 
 	if (css_param->update_flag.xnr_table)
-		atomisp_css_set_xnr_table(asd, &css_param->xnr_table);
+		asd->params.config.xnr_table = &css_param->xnr_table;
 
 	if (css_param->update_flag.r_gamma_table)
-		atomisp_css_set_r_gamma_table(asd, &css_param->r_gamma_table);
+		asd->params.config.r_gamma_table = &css_param->r_gamma_table;
 
 	if (css_param->update_flag.g_gamma_table)
-		atomisp_css_set_g_gamma_table(asd, &css_param->g_gamma_table);
+		asd->params.config.g_gamma_table = &css_param->g_gamma_table;
 
 	if (css_param->update_flag.b_gamma_table)
-		atomisp_css_set_b_gamma_table(asd, &css_param->b_gamma_table);
+		asd->params.config.b_gamma_table = &css_param->b_gamma_table;
 
 	if (css_param->update_flag.anr_thres)
 		atomisp_css_set_anr_thres(asd, &css_param->anr_thres);
 
 	if (css_param->update_flag.shading_table)
-		atomisp_css_set_shading_table(asd, css_param->shading_table);
+		asd->params.config.shading_table = css_param->shading_table;
 
 	if (css_param->update_flag.morph_table && asd->params.gdc_cac_en)
-		atomisp_css_set_morph_table(asd, css_param->morph_table);
+		asd->params.config.morph_table = css_param->morph_table;
 
 	if (css_param->update_flag.dvs2_coefs) {
-		struct atomisp_css_dvs_grid_info *dvs_grid_info =
+		struct ia_css_dvs_grid_info *dvs_grid_info =
 		    atomisp_css_get_dvs_grid_info(
 			&asd->params.curr_grid_info);
 
@@ -3271,7 +3288,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 
 	if (arg->wb_config && (from_user || !cur_config->wb_config)) {
 		if (copy_from_compatible(&css_param->wb_config, arg->wb_config,
-					 sizeof(struct atomisp_css_wb_config),
+					 sizeof(struct ia_css_wb_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.wb_config =
@@ -3280,7 +3297,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 
 	if (arg->ob_config && (from_user || !cur_config->ob_config)) {
 		if (copy_from_compatible(&css_param->ob_config, arg->ob_config,
-					 sizeof(struct atomisp_css_ob_config),
+					 sizeof(struct ia_css_ob_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.ob_config =
@@ -3289,7 +3306,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 
 	if (arg->dp_config && (from_user || !cur_config->dp_config)) {
 		if (copy_from_compatible(&css_param->dp_config, arg->dp_config,
-					 sizeof(struct atomisp_css_dp_config),
+					 sizeof(struct ia_css_dp_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.dp_config =
@@ -3300,7 +3317,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 		if (arg->dz_config && (from_user || !cur_config->dz_config)) {
 			if (copy_from_compatible(&css_param->dz_config,
 						 arg->dz_config,
-						 sizeof(struct atomisp_css_dz_config),
+						 sizeof(struct ia_css_dz_config),
 						 from_user))
 				return -EFAULT;
 			if (!atomisp_check_zoom_region(asd,
@@ -3316,7 +3333,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 
 	if (arg->nr_config && (from_user || !cur_config->nr_config)) {
 		if (copy_from_compatible(&css_param->nr_config, arg->nr_config,
-					 sizeof(struct atomisp_css_nr_config),
+					 sizeof(struct ia_css_nr_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.nr_config =
@@ -3325,7 +3342,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 
 	if (arg->ee_config && (from_user || !cur_config->ee_config)) {
 		if (copy_from_compatible(&css_param->ee_config, arg->ee_config,
-					 sizeof(struct atomisp_css_ee_config),
+					 sizeof(struct ia_css_ee_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.ee_config =
@@ -3335,7 +3352,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->tnr_config && (from_user || !cur_config->tnr_config)) {
 		if (copy_from_compatible(&css_param->tnr_config,
 					 arg->tnr_config,
-					 sizeof(struct atomisp_css_tnr_config),
+					 sizeof(struct ia_css_tnr_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.tnr_config =
@@ -3346,7 +3363,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->a3a_config && (from_user || !cur_config->a3a_config)) {
 		if (copy_from_compatible(&css_param->s3a_config,
 					 arg->a3a_config,
-					 sizeof(struct atomisp_css_3a_config),
+					 sizeof(struct ia_css_3a_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.a3a_config =
@@ -3356,7 +3373,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->ctc_config && (from_user || !cur_config->ctc_config)) {
 		if (copy_from_compatible(&css_param->ctc_config,
 					 arg->ctc_config,
-					 sizeof(struct atomisp_css_ctc_config),
+					 sizeof(struct ia_css_ctc_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.ctc_config =
@@ -3367,7 +3384,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->cnr_config && (from_user || !cur_config->cnr_config)) {
 		if (copy_from_compatible(&css_param->cnr_config,
 					 arg->cnr_config,
-					 sizeof(struct atomisp_css_cnr_config),
+					 sizeof(struct ia_css_cnr_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.cnr_config =
@@ -3378,7 +3395,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->ecd_config && (from_user || !cur_config->ecd_config)) {
 		if (copy_from_compatible(&css_param->ecd_config,
 					 arg->ecd_config,
-					 sizeof(struct atomisp_css_ecd_config),
+					 sizeof(struct ia_css_ecd_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.ecd_config =
@@ -3389,7 +3406,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->ynr_config && (from_user || !cur_config->ynr_config)) {
 		if (copy_from_compatible(&css_param->ynr_config,
 					 arg->ynr_config,
-					 sizeof(struct atomisp_css_ynr_config),
+					 sizeof(struct ia_css_ynr_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.ynr_config =
@@ -3400,7 +3417,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->fc_config && (from_user || !cur_config->fc_config)) {
 		if (copy_from_compatible(&css_param->fc_config,
 					 arg->fc_config,
-					 sizeof(struct atomisp_css_fc_config),
+					 sizeof(struct ia_css_fc_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.fc_config =
@@ -3410,7 +3427,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->macc_config && (from_user || !cur_config->macc_config)) {
 		if (copy_from_compatible(&css_param->macc_config,
 					 arg->macc_config,
-					 sizeof(struct atomisp_css_macc_config),
+					 sizeof(struct ia_css_macc_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.macc_config =
@@ -3420,7 +3437,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 
 	if (arg->aa_config && (from_user || !cur_config->aa_config)) {
 		if (copy_from_compatible(&css_param->aa_config, arg->aa_config,
-					 sizeof(struct atomisp_css_aa_config),
+					 sizeof(struct ia_css_aa_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.aa_config =
@@ -3430,7 +3447,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->anr_config && (from_user || !cur_config->anr_config)) {
 		if (copy_from_compatible(&css_param->anr_config,
 					 arg->anr_config,
-					 sizeof(struct atomisp_css_anr_config),
+					 sizeof(struct ia_css_anr_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.anr_config =
@@ -3441,7 +3458,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->xnr_config && (from_user || !cur_config->xnr_config)) {
 		if (copy_from_compatible(&css_param->xnr_config,
 					 arg->xnr_config,
-					 sizeof(struct atomisp_css_xnr_config),
+					 sizeof(struct ia_css_xnr_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.xnr_config =
@@ -3453,7 +3470,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	    (from_user || !cur_config->yuv2rgb_cc_config)) {
 		if (copy_from_compatible(&css_param->yuv2rgb_cc_config,
 					 arg->yuv2rgb_cc_config,
-					 sizeof(struct atomisp_css_cc_config),
+					 sizeof(struct ia_css_cc_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.yuv2rgb_cc_config =
@@ -3465,7 +3482,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	    (from_user || !cur_config->rgb2yuv_cc_config)) {
 		if (copy_from_compatible(&css_param->rgb2yuv_cc_config,
 					 arg->rgb2yuv_cc_config,
-					 sizeof(struct atomisp_css_cc_config),
+					 sizeof(struct ia_css_cc_config),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.rgb2yuv_cc_config =
@@ -3476,7 +3493,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->macc_table && (from_user || !cur_config->macc_table)) {
 		if (copy_from_compatible(&css_param->macc_table,
 					 arg->macc_table,
-					 sizeof(struct atomisp_css_macc_table),
+					 sizeof(struct ia_css_macc_table),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.macc_table =
@@ -3487,7 +3504,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->xnr_table && (from_user || !cur_config->xnr_table)) {
 		if (copy_from_compatible(&css_param->xnr_table,
 					 arg->xnr_table,
-					 sizeof(struct atomisp_css_xnr_table),
+					 sizeof(struct ia_css_xnr_table),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.xnr_table =
@@ -3497,7 +3514,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->r_gamma_table && (from_user || !cur_config->r_gamma_table)) {
 		if (copy_from_compatible(&css_param->r_gamma_table,
 					 arg->r_gamma_table,
-					 sizeof(struct atomisp_css_rgb_gamma_table),
+					 sizeof(struct ia_css_rgb_gamma_table),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.r_gamma_table =
@@ -3508,7 +3525,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->g_gamma_table && (from_user || !cur_config->g_gamma_table)) {
 		if (copy_from_compatible(&css_param->g_gamma_table,
 					 arg->g_gamma_table,
-					 sizeof(struct atomisp_css_rgb_gamma_table),
+					 sizeof(struct ia_css_rgb_gamma_table),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.g_gamma_table =
@@ -3519,7 +3536,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 	if (arg->b_gamma_table && (from_user || !cur_config->b_gamma_table)) {
 		if (copy_from_compatible(&css_param->b_gamma_table,
 					 arg->b_gamma_table,
-					 sizeof(struct atomisp_css_rgb_gamma_table),
+					 sizeof(struct ia_css_rgb_gamma_table),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.b_gamma_table =
@@ -3529,7 +3546,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 
 	if (arg->anr_thres && (from_user || !cur_config->anr_thres)) {
 		if (copy_from_compatible(&css_param->anr_thres, arg->anr_thres,
-					 sizeof(struct atomisp_css_anr_thres),
+					 sizeof(struct ia_css_anr_thres),
 					 from_user))
 			return -EFAULT;
 		css_param->update_flag.anr_thres =
@@ -3559,8 +3576,8 @@ int atomisp_cp_lsc_table(struct atomisp_sub_device *asd,
 {
 	unsigned int i;
 	unsigned int len_table;
-	struct atomisp_css_shading_table *shading_table;
-	struct atomisp_css_shading_table *old_table;
+	struct ia_css_shading_table *shading_table;
+	struct ia_css_shading_table *old_table;
 	struct atomisp_shading_table *st, dest_st;
 
 	if (!source_st)
@@ -3572,7 +3589,7 @@ int atomisp_cp_lsc_table(struct atomisp_sub_device *asd,
 	if (!from_user && css_param->update_flag.shading_table)
 		return 0;
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		if (copy_from_compatible(&dest_st, source_st,
 					sizeof(struct atomisp_shading_table),
 					from_user)) {
@@ -3605,7 +3622,7 @@ int atomisp_cp_lsc_table(struct atomisp_sub_device *asd,
 	}
 
 	/* Shading table size per color */
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		if (st->width > ISP2400_SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR ||
 		    st->height > ISP2400_SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR) {
 			dev_err(asd->isp->dev, "shading table w/h validate failed!");
@@ -3677,7 +3694,7 @@ int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd,
 			      struct atomisp_css_params *css_param,
 			      bool from_user)
 {
-	struct atomisp_css_dvs_grid_info *cur =
+	struct ia_css_dvs_grid_info *cur =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 	int dvs_hor_coef_bytes, dvs_ver_coef_bytes;
 	struct ia_css_dvs2_coefficients dvs2_coefs;
@@ -3688,7 +3705,7 @@ int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd,
 	if (!from_user && css_param->update_flag.dvs2_coefs)
 		return 0;
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		if (sizeof(*cur) != sizeof(coefs->grid) ||
 		    memcmp(&coefs->grid, cur, sizeof(coefs->grid))) {
 			dev_err(asd->isp->dev, "dvs grid mis-match!\n");
@@ -3796,7 +3813,7 @@ int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd,
 	}
 
 	css_param->update_flag.dvs2_coefs =
-	    (struct atomisp_dvs2_coefficients *)css_param->dvs2_coeff;
+	    (struct atomisp_dis_coefficients *)css_param->dvs2_coeff;
 	return 0;
 }
 
@@ -3805,11 +3822,11 @@ int atomisp_cp_dvs_6axis_config(struct atomisp_sub_device *asd,
 				struct atomisp_css_params *css_param,
 				bool from_user)
 {
-	struct atomisp_css_dvs_6axis_config *dvs_6axis_config;
-	struct atomisp_css_dvs_6axis_config *old_6axis_config;
+	struct ia_css_dvs_6axis_config *dvs_6axis_config;
+	struct ia_css_dvs_6axis_config *old_6axis_config;
 	struct ia_css_stream *stream =
 		    asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
+	struct ia_css_dvs_grid_info *dvs_grid_info =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 	int ret = -EFAULT;
 
@@ -3831,8 +3848,8 @@ int atomisp_cp_dvs_6axis_config(struct atomisp_sub_device *asd,
 	old_6axis_config = css_param->dvs_6axis;
 	dvs_6axis_config = old_6axis_config;
 
-	if (atomisp_hw_is_isp2401) {
-		struct atomisp_css_dvs_6axis_config t_6axis_config;
+	if (IS_ISP2401) {
+		struct ia_css_dvs_6axis_config t_6axis_config;
 
 		if (copy_from_compatible(&t_6axis_config, source_6axis_config,
 					sizeof(struct atomisp_dvs_6axis_config),
@@ -3955,8 +3972,8 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd,
 {
 	int ret = -EFAULT;
 	unsigned int i;
-	struct atomisp_css_morph_table *morph_table;
-	struct atomisp_css_morph_table *old_morph_table;
+	struct ia_css_morph_table *morph_table;
+	struct ia_css_morph_table *old_morph_table;
 
 	if (!source_morph_table)
 		return 0;
@@ -3966,8 +3983,8 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd,
 
 	old_morph_table = css_param->morph_table;
 
-	if (atomisp_hw_is_isp2401) {
-		struct atomisp_css_morph_table mtbl;
+	if (IS_ISP2401) {
+		struct ia_css_morph_table mtbl;
 
 		if (copy_from_compatible(&mtbl, source_morph_table,
 				sizeof(struct atomisp_morph_table),
@@ -3982,7 +3999,7 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd,
 		if (!morph_table)
 			return -ENOMEM;
 
-		for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) {
+		for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
 			if (copy_from_compatible(morph_table->coordinates_x[i],
 						(__force void *)source_morph_table->coordinates_x[i],
 						mtbl.height * mtbl.width *
@@ -4004,7 +4021,7 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd,
 		if (!morph_table)
 			return -ENOMEM;
 
-		for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) {
+		for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
 			if (copy_from_compatible(morph_table->coordinates_x[i],
 						(__force void *)source_morph_table->coordinates_x[i],
 						source_morph_table->height * source_morph_table->width *
@@ -4156,7 +4173,7 @@ void atomisp_handle_parameter_and_buffer(struct atomisp_video_pipe *pipe)
 
 	atomisp_qbuffers_to_css(asd);
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		if (!atomisp_is_wdt_running(asd) && atomisp_buffers_queued(asd))
 			atomisp_wdt_start(asd);
 	} else {
@@ -4192,7 +4209,7 @@ int atomisp_set_parameters(struct video_device *vdev,
 		__func__, arg->per_frame_setting, asd->index,
 		arg->isp_config_id, vdev->name);
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		if (atomisp_is_vf_pipe(pipe) && arg->per_frame_setting) {
 			dev_err(asd->isp->dev, "%s: vf pipe not support per_frame_setting",
 				__func__);
@@ -4264,21 +4281,16 @@ apply_parameter_failed:
 int atomisp_param(struct atomisp_sub_device *asd, int flag,
 		  struct atomisp_parm *config)
 {
-	struct atomisp_device *isp = asd->isp;
 	struct ia_css_pipe_config *vp_cfg =
 		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
 		    pipe_configs[IA_CSS_PIPE_ID_VIDEO];
 
 	/* Read parameter for 3A binary info */
 	if (flag == 0) {
-		struct atomisp_css_dvs_grid_info *dvs_grid_info =
+		struct ia_css_dvs_grid_info *dvs_grid_info =
 		    atomisp_css_get_dvs_grid_info(
 			&asd->params.curr_grid_info);
 
-		if (!&config->info) {
-			dev_err(isp->dev, "ERROR: NULL pointer in grid_info\n");
-			return -EINVAL;
-		}
 		atomisp_curr_user_grid_info(asd, &config->info);
 
 		/* We always return the resolution and stride even if there is
@@ -4295,7 +4307,7 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag,
 		if (dvs_grid_info)
 			memcpy(&config->dvs_grid,
 			       dvs_grid_info,
-			       sizeof(struct atomisp_css_dvs_grid_info));
+			       sizeof(struct ia_css_dvs_grid_info));
 
 		if (asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO) {
 			config->dvs_envelop.width = 0;
@@ -4330,23 +4342,23 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag,
 	}
 
 	memcpy(&asd->params.css_param.wb_config, &config->wb_config,
-	       sizeof(struct atomisp_css_wb_config));
+	       sizeof(struct ia_css_wb_config));
 	memcpy(&asd->params.css_param.ob_config, &config->ob_config,
-	       sizeof(struct atomisp_css_ob_config));
+	       sizeof(struct ia_css_ob_config));
 	memcpy(&asd->params.css_param.dp_config, &config->dp_config,
-	       sizeof(struct atomisp_css_dp_config));
+	       sizeof(struct ia_css_dp_config));
 	memcpy(&asd->params.css_param.de_config, &config->de_config,
-	       sizeof(struct atomisp_css_de_config));
+	       sizeof(struct ia_css_de_config));
 	memcpy(&asd->params.css_param.dz_config, &config->dz_config,
-	       sizeof(struct atomisp_css_dz_config));
+	       sizeof(struct ia_css_dz_config));
 	memcpy(&asd->params.css_param.ce_config, &config->ce_config,
-	       sizeof(struct atomisp_css_ce_config));
+	       sizeof(struct ia_css_ce_config));
 	memcpy(&asd->params.css_param.nr_config, &config->nr_config,
-	       sizeof(struct atomisp_css_nr_config));
+	       sizeof(struct ia_css_nr_config));
 	memcpy(&asd->params.css_param.ee_config, &config->ee_config,
-	       sizeof(struct atomisp_css_ee_config));
+	       sizeof(struct ia_css_ee_config));
 	memcpy(&asd->params.css_param.tnr_config, &config->tnr_config,
-	       sizeof(struct atomisp_css_tnr_config));
+	       sizeof(struct ia_css_tnr_config));
 
 	if (asd->params.color_effect == V4L2_COLORFX_NEGATIVE) {
 		asd->params.css_param.cc_config.matrix[3] = -config->cc_config.matrix[3];
@@ -4360,19 +4372,19 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag,
 	if (asd->params.color_effect != V4L2_COLORFX_SEPIA &&
 	    asd->params.color_effect != V4L2_COLORFX_BW) {
 		memcpy(&asd->params.css_param.cc_config, &config->cc_config,
-		       sizeof(struct atomisp_css_cc_config));
-		atomisp_css_set_cc_config(asd, &asd->params.css_param.cc_config);
-	}
-
-	atomisp_css_set_wb_config(asd, &asd->params.css_param.wb_config);
-	atomisp_css_set_ob_config(asd, &asd->params.css_param.ob_config);
-	atomisp_css_set_de_config(asd, &asd->params.css_param.de_config);
-	atomisp_css_set_dz_config(asd, &asd->params.css_param.dz_config);
-	atomisp_css_set_ce_config(asd, &asd->params.css_param.ce_config);
-	atomisp_css_set_dp_config(asd, &asd->params.css_param.dp_config);
-	atomisp_css_set_nr_config(asd, &asd->params.css_param.nr_config);
-	atomisp_css_set_ee_config(asd, &asd->params.css_param.ee_config);
-	atomisp_css_set_tnr_config(asd, &asd->params.css_param.tnr_config);
+		       sizeof(struct ia_css_cc_config));
+		asd->params.config.cc_config = &asd->params.css_param.cc_config;
+	}
+
+	asd->params.config.wb_config = &asd->params.css_param.wb_config;
+	asd->params.config.ob_config = &asd->params.css_param.ob_config;
+	asd->params.config.de_config = &asd->params.css_param.de_config;
+	asd->params.config.dz_config = &asd->params.css_param.dz_config;
+	asd->params.config.ce_config = &asd->params.css_param.ce_config;
+	asd->params.config.dp_config = &asd->params.css_param.dp_config;
+	asd->params.config.nr_config = &asd->params.css_param.nr_config;
+	asd->params.config.ee_config = &asd->params.css_param.ee_config;
+	asd->params.config.tnr_config = &asd->params.css_param.tnr_config;
 	asd->params.css_update_params_needed = true;
 
 	return 0;
@@ -4384,9 +4396,9 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag,
 int atomisp_color_effect(struct atomisp_sub_device *asd, int flag,
 			 __s32 *effect)
 {
-	struct atomisp_css_cc_config *cc_config = NULL;
-	struct atomisp_css_macc_table *macc_table = NULL;
-	struct atomisp_css_ctc_table *ctc_table = NULL;
+	struct ia_css_cc_config *cc_config = NULL;
+	struct ia_css_macc_table *macc_table = NULL;
+	struct ia_css_ctc_table *ctc_table = NULL;
 	int ret = 0;
 	struct v4l2_control control;
 	struct atomisp_device *isp = asd->isp;
@@ -4461,9 +4473,9 @@ int atomisp_color_effect(struct atomisp_sub_device *asd, int flag,
 	atomisp_update_capture_mode(asd);
 
 	if (cc_config)
-		atomisp_css_set_cc_config(asd, cc_config);
+		asd->params.config.cc_config = cc_config;
 	if (macc_table)
-		atomisp_css_set_macc_table(asd, macc_table);
+		asd->params.config.macc_table = macc_table;
 	if (ctc_table)
 		atomisp_css_set_ctc_table(asd, ctc_table);
 	asd->params.color_effect = (u32)*effect;
@@ -4500,7 +4512,7 @@ int atomisp_bad_pixel_param(struct atomisp_sub_device *asd, int flag,
 		/* Set bad pixel to isp parameters */
 		memcpy(&asd->params.css_param.dp_config, config,
 		       sizeof(asd->params.css_param.dp_config));
-		atomisp_css_set_dp_config(asd, &asd->params.css_param.dp_config);
+		asd->params.config.dp_config = &asd->params.css_param.dp_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -4543,34 +4555,34 @@ int atomisp_fixed_pattern(struct atomisp_sub_device *asd, int flag,
 
 static unsigned int
 atomisp_bytesperline_to_padded_width(unsigned int bytesperline,
-				     enum atomisp_css_frame_format format)
+				     enum ia_css_frame_format format)
 {
 	switch (format) {
-	case CSS_FRAME_FORMAT_UYVY:
-	case CSS_FRAME_FORMAT_YUYV:
-	case CSS_FRAME_FORMAT_RAW:
-	case CSS_FRAME_FORMAT_RGB565:
+	case IA_CSS_FRAME_FORMAT_UYVY:
+	case IA_CSS_FRAME_FORMAT_YUYV:
+	case IA_CSS_FRAME_FORMAT_RAW:
+	case IA_CSS_FRAME_FORMAT_RGB565:
 		return bytesperline / 2;
-	case CSS_FRAME_FORMAT_RGBA888:
+	case IA_CSS_FRAME_FORMAT_RGBA888:
 		return bytesperline / 4;
 	/* The following cases could be removed, but we leave them
 	   in to document the formats that are included. */
-	case CSS_FRAME_FORMAT_NV11:
-	case CSS_FRAME_FORMAT_NV12:
-	case CSS_FRAME_FORMAT_NV16:
-	case CSS_FRAME_FORMAT_NV21:
-	case CSS_FRAME_FORMAT_NV61:
-	case CSS_FRAME_FORMAT_YV12:
-	case CSS_FRAME_FORMAT_YV16:
-	case CSS_FRAME_FORMAT_YUV420:
-	case CSS_FRAME_FORMAT_YUV420_16:
-	case CSS_FRAME_FORMAT_YUV422:
-	case CSS_FRAME_FORMAT_YUV422_16:
-	case CSS_FRAME_FORMAT_YUV444:
-	case CSS_FRAME_FORMAT_YUV_LINE:
-	case CSS_FRAME_FORMAT_PLANAR_RGB888:
-	case CSS_FRAME_FORMAT_QPLANE6:
-	case CSS_FRAME_FORMAT_BINARY_8:
+	case IA_CSS_FRAME_FORMAT_NV11:
+	case IA_CSS_FRAME_FORMAT_NV12:
+	case IA_CSS_FRAME_FORMAT_NV16:
+	case IA_CSS_FRAME_FORMAT_NV21:
+	case IA_CSS_FRAME_FORMAT_NV61:
+	case IA_CSS_FRAME_FORMAT_YV12:
+	case IA_CSS_FRAME_FORMAT_YV16:
+	case IA_CSS_FRAME_FORMAT_YUV420:
+	case IA_CSS_FRAME_FORMAT_YUV420_16:
+	case IA_CSS_FRAME_FORMAT_YUV422:
+	case IA_CSS_FRAME_FORMAT_YUV422_16:
+	case IA_CSS_FRAME_FORMAT_YUV444:
+	case IA_CSS_FRAME_FORMAT_YUV_LINE:
+	case IA_CSS_FRAME_FORMAT_PLANAR_RGB888:
+	case IA_CSS_FRAME_FORMAT_QPLANE6:
+	case IA_CSS_FRAME_FORMAT_BINARY_8:
 	default:
 		return bytesperline;
 	}
@@ -4578,11 +4590,11 @@ atomisp_bytesperline_to_padded_width(unsigned int bytesperline,
 
 static int
 atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg,
-				      struct atomisp_css_frame **result)
+				      struct ia_css_frame **result)
 {
-	struct atomisp_css_frame *res = NULL;
+	struct ia_css_frame *res = NULL;
 	unsigned int padded_width;
-	enum atomisp_css_frame_format sh_format;
+	enum ia_css_frame_format sh_format;
 	char *tmp_buf = NULL;
 	int ret = 0;
 
@@ -4590,10 +4602,10 @@ atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg,
 	padded_width = atomisp_bytesperline_to_padded_width(
 			   arg->fmt.bytesperline, sh_format);
 
-	/* Note: the padded width on an atomisp_css_frame is in elements, not in
+	/* Note: the padded width on an ia_css_frame is in elements, not in
 	   bytes. The RAW frame we use here should always be a 16bit RAW
 	   frame. This is why we bytesperline/2 is equal to the padded with */
-	if (atomisp_css_frame_allocate(&res, arg->fmt.width, arg->fmt.height,
+	if (ia_css_frame_allocate(&res, arg->fmt.width, arg->fmt.height,
 				       sh_format, padded_width, 0)) {
 		ret = -ENOMEM;
 		goto err;
@@ -4617,7 +4629,7 @@ atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg,
 
 err:
 	if (ret && res)
-		atomisp_css_frame_free(res);
+		ia_css_frame_free(res);
 	if (tmp_buf)
 		vfree(tmp_buf);
 	if (ret == 0)
@@ -4631,7 +4643,7 @@ err:
 int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd,
 				struct v4l2_framebuffer *arg)
 {
-	struct atomisp_css_frame *raw_black_frame = NULL;
+	struct ia_css_frame *raw_black_frame = NULL;
 	int ret;
 
 	if (!arg)
@@ -4640,10 +4652,12 @@ int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd,
 	ret = atomisp_v4l2_framebuffer_to_css_frame(arg, &raw_black_frame);
 	if (ret)
 		return ret;
-	if (atomisp_css_set_black_frame(asd, raw_black_frame))
-		ret = -ENOMEM;
 
-	atomisp_css_frame_free(raw_black_frame);
+	if (sh_css_set_black_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
+				   raw_black_frame) != 0)
+		return -ENOMEM;
+
+	ia_css_frame_free(raw_black_frame);
 	return ret;
 }
 
@@ -4661,10 +4675,10 @@ int atomisp_false_color(struct atomisp_sub_device *asd, int flag,
 
 	/* Set nr config to isp parameters */
 	if (*value) {
-		atomisp_css_set_default_de_config(asd);
+		asd->params.config.de_config = NULL;
 	} else {
 		asd->params.css_param.de_config.pixelnoise = 0;
-		atomisp_css_set_de_config(asd, &asd->params.css_param.de_config);
+		asd->params.config.de_config = &asd->params.css_param.de_config;
 	}
 	asd->params.css_update_params_needed = true;
 	asd->params.false_color = *value;
@@ -4685,7 +4699,7 @@ int atomisp_false_color_param(struct atomisp_sub_device *asd, int flag,
 		/* Set false color to isp parameters */
 		memcpy(&asd->params.css_param.de_config, config,
 		       sizeof(asd->params.css_param.de_config));
-		atomisp_css_set_de_config(asd, &asd->params.css_param.de_config);
+		asd->params.config.de_config = &asd->params.css_param.de_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -4706,7 +4720,7 @@ int atomisp_white_balance_param(struct atomisp_sub_device *asd, int flag,
 		/* Set white balance to isp parameters */
 		memcpy(&asd->params.css_param.wb_config, config,
 		       sizeof(asd->params.css_param.wb_config));
-		atomisp_css_set_wb_config(asd, &asd->params.css_param.wb_config);
+		asd->params.config.wb_config = &asd->params.css_param.wb_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -4728,7 +4742,7 @@ int atomisp_3a_config_param(struct atomisp_sub_device *asd, int flag,
 		/* Set white balance to isp parameters */
 		memcpy(&asd->params.css_param.s3a_config, config,
 		       sizeof(asd->params.css_param.s3a_config));
-		atomisp_css_set_3a_config(asd, &asd->params.css_param.s3a_config);
+		asd->params.config.s3a_config = &asd->params.css_param.s3a_config;
 		asd->params.css_update_params_needed = true;
 	}
 
@@ -4959,13 +4973,14 @@ enum mipi_port_id __get_mipi_port(struct atomisp_device *isp,
 {
 	switch (port) {
 	case ATOMISP_CAMERA_PORT_PRIMARY:
-				return MIPI_PORT0_ID;
+		return MIPI_PORT0_ID;
 	case ATOMISP_CAMERA_PORT_SECONDARY:
 		return MIPI_PORT1_ID;
 	case ATOMISP_CAMERA_PORT_TERTIARY:
-		if (MIPI_PORT1_ID + 1 != N_MIPI_PORT_ID)
+		if (MIPI_PORT1_ID + 1 != N_MIPI_PORT_ID) {
 			return MIPI_PORT1_ID + 1;
-	/* go through down for else case */
+		}
+	/* fall through */
 	default:
 		dev_err(isp->dev, "unsupported port: %d\n", port);
 		return MIPI_PORT0_ID;
@@ -5018,7 +5033,7 @@ static inline int atomisp_set_sensor_mipi_to_isp(
 			 mipi_info->input_format);
 		if (!fc)
 			return -EINVAL;
-		input_format = fc->css_stream_fmt;
+		input_format = fc->atomisp_in_fmt;
 	} else {
 		struct v4l2_mbus_framefmt *sink;
 
@@ -5028,7 +5043,7 @@ static inline int atomisp_set_sensor_mipi_to_isp(
 		fc = atomisp_find_in_fmt_conv(sink->code);
 		if (!fc)
 			return -EINVAL;
-		input_format = fc->css_stream_fmt;
+		input_format = fc->atomisp_in_fmt;
 		bayer_order = fc->bayer_order;
 	}
 
@@ -5039,7 +5054,7 @@ static inline int atomisp_set_sensor_mipi_to_isp(
 		 mipi_info->metadata_format);
 	if (!fc)
 		return -EINVAL;
-	input_format = fc->css_stream_fmt;
+	input_format = fc->atomisp_in_fmt;
 	atomisp_css_input_configure_port(asd,
 					 __get_mipi_port(asd->isp, mipi_info->port),
 					 mipi_info->num_lanes,
@@ -5060,8 +5075,8 @@ static int __enable_continuous_mode(struct atomisp_sub_device *asd,
 		enable, asd->continuous_raw_buffer_size->val,
 		!asd->continuous_viewfinder->val);
 
-	if (!atomisp_hw_is_isp2401)
-		atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_PRIMARY);
+	if (!IS_ISP2401)
+		atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_PRIMARY);
 	else
 		atomisp_update_capture_mode(asd);
 
@@ -5085,7 +5100,7 @@ static int __enable_continuous_mode(struct atomisp_sub_device *asd,
 	}
 
 	if (isp->inputs[asd->input_curr].type != FILE_INPUT)
-		atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_SENSOR);
+		atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
 
 	return atomisp_update_run_mode(asd);
 }
@@ -5099,13 +5114,13 @@ static int configure_pp_input_nop(struct atomisp_sub_device *asd,
 static int configure_output_nop(struct atomisp_sub_device *asd,
 				unsigned int width, unsigned int height,
 				unsigned int min_width,
-				enum atomisp_css_frame_format sh_fmt)
+				enum ia_css_frame_format sh_fmt)
 {
 	return 0;
 }
 
 static int get_frame_info_nop(struct atomisp_sub_device *asd,
-			      struct atomisp_css_frame_info *finfo)
+			      struct ia_css_frame_info *finfo)
 {
 	return 0;
 }
@@ -5164,8 +5179,8 @@ static int css_input_resolution_changed(struct atomisp_sub_device *asd,
 }
 
 static int atomisp_set_fmt_to_isp(struct video_device *vdev,
-				  struct atomisp_css_frame_info *output_info,
-				  struct atomisp_css_frame_info *raw_output_info,
+				  struct ia_css_frame_info *output_info,
+				  struct ia_css_frame_info *raw_output_info,
 				  struct v4l2_pix_format *pix,
 				  unsigned int source_pad)
 {
@@ -5174,22 +5189,22 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 	struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd;
 	const struct atomisp_format_bridge *format;
 	struct v4l2_rect *isp_sink_crop;
-	enum atomisp_css_pipe_id pipe_id;
+	enum ia_css_pipe_id pipe_id;
 	struct v4l2_subdev_fh fh;
 	int (*configure_output)(struct atomisp_sub_device *asd,
 				unsigned int width, unsigned int height,
 				unsigned int min_width,
-				enum atomisp_css_frame_format sh_fmt) =
+				enum ia_css_frame_format sh_fmt) =
 				    configure_output_nop;
 	int (*get_frame_info)(struct atomisp_sub_device *asd,
-			      struct atomisp_css_frame_info *finfo) =
+			      struct ia_css_frame_info *finfo) =
 				  get_frame_info_nop;
 	int (*configure_pp_input)(struct atomisp_sub_device *asd,
 				  unsigned int width, unsigned int height) =
 				      configure_pp_input_nop;
 	u16 stream_index = atomisp_source_pad_to_stream_id(asd, source_pad);
 	const struct atomisp_in_fmt_conv *fc;
-	int ret;
+	int ret, i;
 
 	v4l2_fh_init(&fh.vfh, vdev);
 
@@ -5221,8 +5236,8 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 							 ATOMISP_SUBDEV_PAD_SINK)->code);
 		if (!fc)
 			return -EINVAL;
-		if (format->sh_fmt == CSS_FRAME_FORMAT_RAW &&
-		    raw_output_format_match_input(fc->css_stream_fmt,
+		if (format->sh_fmt == IA_CSS_FRAME_FORMAT_RAW &&
+		    raw_output_format_match_input(fc->atomisp_in_fmt,
 						  pix->pixelformat))
 			return -EINVAL;
 	}
@@ -5255,7 +5270,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 		atomisp_subdev_set_ffmt(&asd->subdev, fh.pad,
 					V4L2_SUBDEV_FORMAT_ACTIVE,
 					ATOMISP_SUBDEV_PAD_SOURCE_VF, &vf_ffmt);
-		asd->video_out_vf.sh_fmt = CSS_FRAME_FORMAT_NV12;
+		asd->video_out_vf.sh_fmt = IA_CSS_FRAME_FORMAT_NV12;
 
 		if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
 			atomisp_css_video_configure_viewfinder(asd,
@@ -5285,25 +5300,26 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 			return -EINVAL;
 	}
 
-	atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_SENSOR);
-	atomisp_css_disable_vf_pp(asd,
-				  asd->vfpp->val != ATOMISP_VFPP_ENABLE);
+	atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_BUFFERED_SENSOR);
+
+	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
+		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].pipe_extra_configs[i].disable_vf_pp = asd->vfpp->val != ATOMISP_VFPP_ENABLE;
 
 	/* ISP2401 new input system need to use copy pipe */
 	if (asd->copy_mode) {
-		pipe_id = CSS_PIPE_ID_COPY;
+		pipe_id = IA_CSS_PIPE_ID_COPY;
 		atomisp_css_capture_enable_online(asd, stream_index, false);
 	} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
 		/* video same in continuouscapture and online modes */
 		configure_output = atomisp_css_video_configure_output;
 		get_frame_info = atomisp_css_video_get_output_frame_info;
-		pipe_id = CSS_PIPE_ID_VIDEO;
+		pipe_id = IA_CSS_PIPE_ID_VIDEO;
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 		if (!asd->continuous_mode->val) {
 			configure_output = atomisp_css_video_configure_output;
 			get_frame_info =
 			    atomisp_css_video_get_output_frame_info;
-			pipe_id = CSS_PIPE_ID_VIDEO;
+			pipe_id = IA_CSS_PIPE_ID_VIDEO;
 		} else {
 			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW ||
 			    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) {
@@ -5313,7 +5329,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 				    atomisp_css_video_get_output_frame_info;
 				configure_pp_input =
 				    atomisp_css_video_configure_pp_input;
-				pipe_id = CSS_PIPE_ID_VIDEO;
+				pipe_id = IA_CSS_PIPE_ID_VIDEO;
 			} else {
 				configure_output =
 				    atomisp_css_capture_configure_output;
@@ -5321,7 +5337,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 				    atomisp_css_capture_get_output_frame_info;
 				configure_pp_input =
 				    atomisp_css_capture_configure_pp_input;
-				pipe_id = CSS_PIPE_ID_CAPTURE;
+				pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 
 				atomisp_update_capture_mode(asd);
 				atomisp_css_capture_enable_online(asd, stream_index, false);
@@ -5331,7 +5347,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 		configure_output = atomisp_css_preview_configure_output;
 		get_frame_info = atomisp_css_preview_get_output_frame_info;
 		configure_pp_input = atomisp_css_preview_configure_pp_input;
-		pipe_id = CSS_PIPE_ID_PREVIEW;
+		pipe_id = IA_CSS_PIPE_ID_PREVIEW;
 	} else {
 		/* CSS doesn't support low light mode on SOC cameras, so disable
 		 * it. FIXME: if this is done elsewhere, it gives corrupted
@@ -5340,8 +5356,8 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 		if (isp->inputs[asd->input_curr].type == SOC_CAMERA)
 			asd->params.low_light = false;
 
-		if (format->sh_fmt == CSS_FRAME_FORMAT_RAW) {
-			atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_RAW);
+		if (format->sh_fmt == IA_CSS_FRAME_FORMAT_RAW) {
+			atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_RAW);
 			atomisp_css_enable_dz(asd, false);
 		} else {
 			atomisp_update_capture_mode(asd);
@@ -5356,7 +5372,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 		configure_output = atomisp_css_capture_configure_output;
 		get_frame_info = atomisp_css_capture_get_output_frame_info;
 		configure_pp_input = atomisp_css_capture_configure_pp_input;
-		pipe_id = CSS_PIPE_ID_CAPTURE;
+		pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 
 		if (!asd->params.online_process &&
 		    !asd->continuous_mode->val) {
@@ -5377,7 +5393,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 	 * to SOC camera, use yuvpp pipe.
 	 */
 	if (ATOMISP_USE_YUVPP(asd))
-		pipe_id = CSS_PIPE_ID_YUVPP;
+		pipe_id = IA_CSS_PIPE_ID_YUVPP;
 
 	if (asd->copy_mode)
 		ret = atomisp_css_copy_configure_output(asd, stream_index,
@@ -5432,12 +5448,12 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev,
 	atomisp_update_grid_info(asd, pipe_id, source_pad);
 
 	/* Free the raw_dump buffer first */
-	atomisp_css_frame_free(asd->raw_output_frame);
+	ia_css_frame_free(asd->raw_output_frame);
 	asd->raw_output_frame = NULL;
 
 	if (!asd->continuous_mode->val &&
 	    !asd->params.online_process && !isp->sw_contex.file_input &&
-	    atomisp_css_frame_allocate_from_info(&asd->raw_output_frame,
+	    ia_css_frame_allocate_from_info(&asd->raw_output_frame,
 		    raw_output_info))
 		return -ENOMEM;
 
@@ -5593,24 +5609,19 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 	struct atomisp_sub_device *asd = pipe->asd;
 	const struct atomisp_format_bridge *format_bridge;
 	const struct atomisp_format_bridge *snr_format_bridge;
-	struct atomisp_css_frame_info output_info, raw_output_info;
+	struct ia_css_frame_info output_info, raw_output_info;
 	struct v4l2_format snr_fmt = *f;
 	struct v4l2_format backup_fmt = *f, s_fmt = *f;
 	unsigned int dvs_env_w = 0, dvs_env_h = 0;
 	unsigned int padding_w = pad_w, padding_h = pad_h;
 	bool res_overflow = false, crop_needs_override = false;
-	struct v4l2_mbus_framefmt isp_sink_fmt;
+	struct v4l2_mbus_framefmt *isp_sink_fmt;
 	struct v4l2_mbus_framefmt isp_source_fmt = {0};
 	struct v4l2_rect isp_sink_crop;
 	u16 source_pad = atomisp_subdev_source_pad(vdev);
 	struct v4l2_subdev_fh fh;
 	int ret;
 
-	dev_dbg(isp->dev,
-		"setting resolution %ux%u on pad %u for asd%d, bytesperline %u\n",
-		f->fmt.pix.width, f->fmt.pix.height, source_pad,
-		asd->index, f->fmt.pix.bytesperline);
-
 	if (source_pad >= ATOMISP_SUBDEV_PADS_NUM)
 		return -EINVAL;
 
@@ -5619,6 +5630,11 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 		return -EBUSY;
 	}
 
+	dev_dbg(isp->dev,
+		"setting resolution %ux%u on pad %u for asd%d, bytesperline %u\n",
+		f->fmt.pix.width, f->fmt.pix.height, source_pad,
+		asd->index, f->fmt.pix.bytesperline);
+
 	v4l2_fh_init(&fh.vfh, vdev);
 
 	format_bridge = atomisp_get_format_bridge(f->fmt.pix.pixelformat);
@@ -5765,22 +5781,26 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 
 	/* get sensor resolution and format */
 	ret = atomisp_try_fmt(vdev, &snr_fmt, &res_overflow);
-	if (ret)
+	if (ret) {
+		dev_warn(isp->dev, "Try format failed with error %d\n", ret);
 		return ret;
+	}
 	f->fmt.pix.width = snr_fmt.fmt.pix.width;
 	f->fmt.pix.height = snr_fmt.fmt.pix.height;
 
 	snr_format_bridge =
 	    atomisp_get_format_bridge(snr_fmt.fmt.pix.pixelformat);
-	if (!snr_format_bridge)
+	if (!snr_format_bridge) {
+		dev_warn(isp->dev, "Can't find bridge format\n");
 		return -EINVAL;
+	}
 
 	atomisp_subdev_get_ffmt(&asd->subdev, NULL,
 				V4L2_SUBDEV_FORMAT_ACTIVE,
 				ATOMISP_SUBDEV_PAD_SINK)->code =
 				    snr_format_bridge->mbus_code;
 
-	isp_sink_fmt = *atomisp_subdev_get_ffmt(&asd->subdev, NULL,
+	isp_sink_fmt = atomisp_subdev_get_ffmt(&asd->subdev, NULL,
 						V4L2_SUBDEV_FORMAT_ACTIVE,
 						ATOMISP_SUBDEV_PAD_SINK);
 
@@ -5838,9 +5858,9 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 	 * capture pipe and usually has lower resolution than capture pipe.
 	 */
 	if (!asd->continuous_mode->val ||
-	    isp_sink_fmt.width < (f->fmt.pix.width + padding_w + dvs_env_w) ||
-	    isp_sink_fmt.height < (f->fmt.pix.height + padding_h +
-				   dvs_env_h)) {
+	    isp_sink_fmt->width < (f->fmt.pix.width + padding_w + dvs_env_w) ||
+	    isp_sink_fmt->height < (f->fmt.pix.height + padding_h +
+				    dvs_env_h)) {
 		/*
 		 * For jpeg or custom raw format the sensor will return constant
 		 * width and height. Because we already had quried try_mbus_fmt,
@@ -5859,8 +5879,11 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 		ret = atomisp_set_fmt_to_snr(vdev, &s_fmt,
 					     f->fmt.pix.pixelformat, padding_w,
 					     padding_h, dvs_env_w, dvs_env_h);
-		if (ret)
+		if (ret) {
+			dev_warn(isp->dev,
+				 "Set format to sensor failed with %d\n", ret);
 			return -EINVAL;
+		}
 
 		atomisp_csi_lane_config(isp);
 		crop_needs_override = true;
@@ -5943,7 +5966,7 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 		 * which appears to be related by a hardware
 		 * performance limitation.  It's unclear why this
 		 * particular code triggers the issue. */
-		if (!atomisp_hw_is_isp2401 || crop_needs_override) {
+		if (!IS_ISP2401 || crop_needs_override) {
 			if (isp_sink_crop.width * main_compose.height >
 			    isp_sink_crop.height * main_compose.width) {
 				sink_crop.height = isp_sink_crop.height;
@@ -5977,8 +6000,10 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 set_fmt_to_isp:
 	ret = atomisp_set_fmt_to_isp(vdev, &output_info, &raw_output_info,
 				     &f->fmt.pix, source_pad);
-	if (ret)
+	if (ret) {
+		dev_warn(isp->dev, "Can't set format on ISP. Error %d\n", ret);
 		return -EINVAL;
+	}
 done:
 	pipe->pix.width = f->fmt.pix.width;
 	pipe->pix.height = f->fmt.pix.height;
@@ -5995,6 +6020,9 @@ done:
 		pipe->pix.sizeimage =
 		    PAGE_ALIGN(f->fmt.pix.height * pipe->pix.bytesperline);
 	}
+	dev_dbg(isp->dev, "%s: image size: %d, %d bytes per line\n",
+		__func__, pipe->pix.sizeimage, pipe->pix.bytesperline);
+
 	if (f->fmt.pix.field == V4L2_FIELD_ANY)
 		f->fmt.pix.field = V4L2_FIELD_NONE;
 	pipe->pix.field = f->fmt.pix.field;
@@ -6045,7 +6073,7 @@ int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f)
 	}
 
 	pipe->pix = f->fmt.pix;
-	atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_FIFO);
+	atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_FIFO);
 	atomisp_css_input_configure_port(asd,
 					 __get_mipi_port(isp, ATOMISP_CAMERA_PORT_PRIMARY), 2, 0xffff4,
 					 0, 0, 0, 0);
@@ -6062,8 +6090,8 @@ int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f)
 int atomisp_set_shading_table(struct atomisp_sub_device *asd,
 			      struct atomisp_shading_table *user_shading_table)
 {
-	struct atomisp_css_shading_table *shading_table;
-	struct atomisp_css_shading_table *free_table;
+	struct ia_css_shading_table *shading_table;
+	struct ia_css_shading_table *free_table;
 	unsigned int len_table;
 	int i;
 	int ret = 0;
@@ -6072,7 +6100,7 @@ int atomisp_set_shading_table(struct atomisp_sub_device *asd,
 		return -EINVAL;
 
 	if (!user_shading_table->enable) {
-		atomisp_css_set_shading_table(asd, NULL);
+		asd->params.config.shading_table = NULL;
 		asd->params.sc_en = false;
 		return 0;
 	}
@@ -6084,7 +6112,7 @@ int atomisp_set_shading_table(struct atomisp_sub_device *asd,
 	}
 
 	/* Shading table size per color */
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		if (user_shading_table->width > ISP2400_SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR ||
 		    user_shading_table->height > ISP2400_SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR)
 			return -EINVAL;
@@ -6117,7 +6145,7 @@ int atomisp_set_shading_table(struct atomisp_sub_device *asd,
 
 	free_table = asd->params.css_param.shading_table;
 	asd->params.css_param.shading_table = shading_table;
-	atomisp_css_set_shading_table(asd, shading_table);
+	asd->params.config.shading_table = shading_table;
 	asd->params.sc_en = true;
 
 out:
@@ -6564,31 +6592,31 @@ static int atomisp_get_pipe_id(struct atomisp_video_pipe *pipe)
 	struct atomisp_sub_device *asd = pipe->asd;
 
 	if (ATOMISP_USE_YUVPP(asd))
-		return CSS_PIPE_ID_YUVPP;
+		return IA_CSS_PIPE_ID_YUVPP;
 	else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER)
-		return CSS_PIPE_ID_VIDEO;
+		return IA_CSS_PIPE_ID_VIDEO;
 	else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT)
-		return CSS_PIPE_ID_CAPTURE;
+		return IA_CSS_PIPE_ID_CAPTURE;
 	else if (pipe == &asd->video_out_video_capture)
-		return CSS_PIPE_ID_VIDEO;
+		return IA_CSS_PIPE_ID_VIDEO;
 	else if (pipe == &asd->video_out_vf)
-		return CSS_PIPE_ID_CAPTURE;
+		return IA_CSS_PIPE_ID_CAPTURE;
 	else if (pipe == &asd->video_out_preview) {
 		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			return CSS_PIPE_ID_VIDEO;
+			return IA_CSS_PIPE_ID_VIDEO;
 		else
-			return CSS_PIPE_ID_PREVIEW;
+			return IA_CSS_PIPE_ID_PREVIEW;
 	} else if (pipe == &asd->video_out_capture) {
 		if (asd->copy_mode)
 			return IA_CSS_PIPE_ID_COPY;
 		else
-			return CSS_PIPE_ID_CAPTURE;
+			return IA_CSS_PIPE_ID_CAPTURE;
 	}
 
 	/* fail through */
 	dev_warn(asd->isp->dev, "%s failed to find proper pipe\n",
 		 __func__);
-	return CSS_PIPE_ID_CAPTURE;
+	return IA_CSS_PIPE_ID_CAPTURE;
 }
 
 int atomisp_get_invalid_frame_num(struct video_device *vdev,
@@ -6596,7 +6624,7 @@ int atomisp_get_invalid_frame_num(struct video_device *vdev,
 {
 	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
 	struct atomisp_sub_device *asd = pipe->asd;
-	enum atomisp_css_pipe_id pipe_id;
+	enum ia_css_pipe_id pipe_id;
 	struct ia_css_pipe_info p_info;
 	int ret;
 
@@ -6618,7 +6646,7 @@ int atomisp_get_invalid_frame_num(struct video_device *vdev,
 	ret = ia_css_pipe_get_info(
 		  asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
 		  .pipes[pipe_id], &p_info);
-	if (ret == IA_CSS_SUCCESS) {
+	if (!ret) {
 		*invalid_frame_num = p_info.num_invalid_frames;
 		return 0;
 	} else {
diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.h b/drivers/staging/media/atomisp/pci/atomisp_cmd.h
index b5af9da3b0fb..0bde995f1a8d 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -32,7 +33,7 @@
 #include "ia_css.h"
 
 struct atomisp_device;
-struct atomisp_css_frame;
+struct ia_css_frame;
 
 #define MSI_ENABLE_BIT		16
 #define INTR_DISABLE_BIT	10
@@ -64,16 +65,6 @@ bool atomisp_buffers_queued(struct atomisp_sub_device *asd);
 /* ISP2401 */
 bool atomisp_buffers_queued_pipe(struct atomisp_video_pipe *pipe);
 
-/* TODO:should be here instead of atomisp_helper.h
-extern void __iomem *atomisp_io_base;
-
-static inline void __iomem *atomisp_get_io_virt_addr(unsigned int address)
-{
-	void __iomem *ret = atomisp_io_base + (address & 0x003FFFFF);
-	return ret;
-}
-*/
-
 /*
  * Interrupt functions
  */
@@ -88,7 +79,7 @@ const struct atomisp_format_bridge *get_atomisp_format_bridge_from_mbus(
     u32 mbus_code);
 bool atomisp_is_mbuscode_raw(uint32_t code);
 int atomisp_get_frame_pgnr(struct atomisp_device *isp,
-			   const struct atomisp_css_frame *frame, u32 *p_pgnr);
+			   const struct ia_css_frame *frame, u32 *p_pgnr);
 void atomisp_delayed_init_work(struct work_struct *work);
 
 /*
@@ -301,8 +292,8 @@ int atomisp_set_array_res(struct atomisp_sub_device *asd,
  * Function to calculate real zoom region for every pipe
  */
 int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
-				       struct atomisp_css_dz_config   *dz_config,
-				       enum atomisp_css_pipe_id css_pipe_id);
+				       struct ia_css_dz_config   *dz_config,
+				       enum ia_css_pipe_id css_pipe_id);
 
 int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd,
 				      struct atomisp_parameters *arg,
@@ -372,8 +363,8 @@ int atomisp_freq_scaling(struct atomisp_device *vdev,
 			 bool force);
 
 void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
-		      enum atomisp_css_buffer_type buf_type,
-		      enum atomisp_css_pipe_id css_pipe_id,
+		      enum ia_css_buffer_type buf_type,
+		      enum ia_css_pipe_id css_pipe_id,
 		      bool q_buffers, enum atomisp_input_stream_id stream_id);
 
 void atomisp_css_flush(struct atomisp_device *isp);
diff --git a/drivers/staging/media/atomisp/pci/atomisp_common.h b/drivers/staging/media/atomisp/pci/atomisp_common.h
index 65c9caf72b7b..b29874f2bc0f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_common.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_common.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -43,15 +44,11 @@ extern int pad_h;
 /* ISP2401 */
 #define ATOMISP_CSS_ISP_PIPE_VERSION_2_7    1
 
-#define IS_ISP2401(isp)							\
-	(((isp)->media_dev.hw_revision & ATOMISP_HW_REVISION_MASK)	\
-	 >= (ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT))
-
 struct atomisp_format_bridge {
 	unsigned int pixelformat;
 	unsigned int depth;
 	u32 mbus_code;
-	enum atomisp_css_frame_format sh_fmt;
+	enum ia_css_frame_format sh_fmt;
 	unsigned char description[32];	/* the same as struct v4l2_fmtdesc */
 	bool planar;
 };
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat.h b/drivers/staging/media/atomisp/pci/atomisp_compat.h
index 205c530e8090..b2ed83c2f337 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Clovertrail PNW Camera Imaging ISP subsystem.
  *
@@ -23,116 +24,20 @@
 #include "../../include/linux/atomisp.h"
 #include <media/videobuf-vmalloc.h>
 
-#define CSS_RX_IRQ_INFO_BUFFER_OVERRUN \
-	CSS_ID(CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
-#define CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE \
-	CSS_ID(CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE)
-#define CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE \
-	CSS_ID(CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE)
-#define CSS_RX_IRQ_INFO_ECC_CORRECTED \
-	CSS_ID(CSS_RX_IRQ_INFO_ECC_CORRECTED)
-#define CSS_RX_IRQ_INFO_ERR_SOT \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_SOT)
-#define CSS_RX_IRQ_INFO_ERR_SOT_SYNC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_SOT_SYNC)
-#define CSS_RX_IRQ_INFO_ERR_CONTROL \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_CONTROL)
-#define CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE)
-#define CSS_RX_IRQ_INFO_ERR_CRC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_CRC)
-#define CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID)
-#define CSS_RX_IRQ_INFO_ERR_FRAME_SYNC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_FRAME_SYNC)
-#define CSS_RX_IRQ_INFO_ERR_FRAME_DATA \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_FRAME_DATA)
-#define CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT)
-#define CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC)
-#define CSS_RX_IRQ_INFO_ERR_LINE_SYNC \
-	CSS_ID(CSS_RX_IRQ_INFO_ERR_LINE_SYNC)
-#define CSS_RX_IRQ_INFO_INIT_TIMEOUT \
-	CSS_ID(CSS_RX_IRQ_INFO_INIT_TIMEOUT)
-
-#define CSS_IRQ_INFO_CSS_RECEIVER_SOF	CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_SOF)
-#define CSS_IRQ_INFO_CSS_RECEIVER_EOF	CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_EOF)
-#define CSS_IRQ_INFO_CSS_RECEIVER_FIFO_OVERFLOW \
-	CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_FIFO_OVERFLOW)
-#define CSS_EVENT_OUTPUT_FRAME_DONE	CSS_EVENT(OUTPUT_FRAME_DONE)
-#define CSS_EVENT_SEC_OUTPUT_FRAME_DONE	CSS_EVENT(SECOND_OUTPUT_FRAME_DONE)
-#define CSS_EVENT_VF_OUTPUT_FRAME_DONE	CSS_EVENT(VF_OUTPUT_FRAME_DONE)
-#define CSS_EVENT_SEC_VF_OUTPUT_FRAME_DONE	CSS_EVENT(SECOND_VF_OUTPUT_FRAME_DONE)
-#define CSS_EVENT_3A_STATISTICS_DONE	CSS_EVENT(3A_STATISTICS_DONE)
-#define CSS_EVENT_DIS_STATISTICS_DONE	CSS_EVENT(DIS_STATISTICS_DONE)
-#define CSS_EVENT_PIPELINE_DONE		CSS_EVENT(PIPELINE_DONE)
-#define CSS_EVENT_METADATA_DONE		CSS_EVENT(METADATA_DONE)
-#define CSS_EVENT_ACC_STAGE_COMPLETE	CSS_EVENT(ACC_STAGE_COMPLETE)
-#define CSS_EVENT_TIMER			CSS_EVENT(TIMER)
-
-#define CSS_BUFFER_TYPE_METADATA	CSS_ID(CSS_BUFFER_TYPE_METADATA)
-#define CSS_BUFFER_TYPE_3A_STATISTICS	CSS_ID(CSS_BUFFER_TYPE_3A_STATISTICS)
-#define CSS_BUFFER_TYPE_DIS_STATISTICS	CSS_ID(CSS_BUFFER_TYPE_DIS_STATISTICS)
-#define CSS_BUFFER_TYPE_INPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_INPUT_FRAME)
-#define CSS_BUFFER_TYPE_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_VF_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME	CSS_ID(CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME)
-#define CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME \
-	CSS_ID(CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME)
-
-#define CSS_FORMAT_RAW_8	CSS_FORMAT(RAW_8)
-#define CSS_FORMAT_RAW_10	CSS_FORMAT(RAW_10)
-#define CSS_FORMAT_RAW_12	CSS_FORMAT(RAW_12)
-#define CSS_FORMAT_RAW_16	CSS_FORMAT(RAW_16)
-
-#define CSS_CAPTURE_MODE_RAW		CSS_ID(CSS_CAPTURE_MODE_RAW)
-#define CSS_CAPTURE_MODE_BAYER		CSS_ID(CSS_CAPTURE_MODE_BAYER)
-#define CSS_CAPTURE_MODE_PRIMARY	CSS_ID(CSS_CAPTURE_MODE_PRIMARY)
-#define CSS_CAPTURE_MODE_ADVANCED	CSS_ID(CSS_CAPTURE_MODE_ADVANCED)
-#define CSS_CAPTURE_MODE_LOW_LIGHT	CSS_ID(CSS_CAPTURE_MODE_LOW_LIGHT)
-
-#define CSS_MORPH_TABLE_NUM_PLANES	CSS_ID(CSS_MORPH_TABLE_NUM_PLANES)
-
-#define CSS_FRAME_FORMAT_NV11		CSS_ID(CSS_FRAME_FORMAT_NV11)
-#define CSS_FRAME_FORMAT_NV12		CSS_ID(CSS_FRAME_FORMAT_NV12)
-#define CSS_FRAME_FORMAT_NV16		CSS_ID(CSS_FRAME_FORMAT_NV16)
-#define CSS_FRAME_FORMAT_NV21		CSS_ID(CSS_FRAME_FORMAT_NV21)
-#define CSS_FRAME_FORMAT_NV61		CSS_ID(CSS_FRAME_FORMAT_NV61)
-#define CSS_FRAME_FORMAT_YV12		CSS_ID(CSS_FRAME_FORMAT_YV12)
-#define CSS_FRAME_FORMAT_YV16		CSS_ID(CSS_FRAME_FORMAT_YV16)
-#define CSS_FRAME_FORMAT_YUV420		CSS_ID(CSS_FRAME_FORMAT_YUV420)
-#define CSS_FRAME_FORMAT_YUV420_16	CSS_ID(CSS_FRAME_FORMAT_YUV420_16)
-#define CSS_FRAME_FORMAT_YUV422		CSS_ID(CSS_FRAME_FORMAT_YUV422)
-#define CSS_FRAME_FORMAT_YUV422_16	CSS_ID(CSS_FRAME_FORMAT_YUV422_16)
-#define CSS_FRAME_FORMAT_UYVY		CSS_ID(CSS_FRAME_FORMAT_UYVY)
-#define CSS_FRAME_FORMAT_YUYV		CSS_ID(CSS_FRAME_FORMAT_YUYV)
-#define CSS_FRAME_FORMAT_YUV444		CSS_ID(CSS_FRAME_FORMAT_YUV444)
-#define CSS_FRAME_FORMAT_YUV_LINE	CSS_ID(CSS_FRAME_FORMAT_YUV_LINE)
-#define CSS_FRAME_FORMAT_RAW		CSS_ID(CSS_FRAME_FORMAT_RAW)
-#define CSS_FRAME_FORMAT_RGB565		CSS_ID(CSS_FRAME_FORMAT_RGB565)
-#define CSS_FRAME_FORMAT_PLANAR_RGB888	CSS_ID(CSS_FRAME_FORMAT_PLANAR_RGB888)
-#define CSS_FRAME_FORMAT_RGBA888	CSS_ID(CSS_FRAME_FORMAT_RGBA888)
-#define CSS_FRAME_FORMAT_QPLANE6	CSS_ID(CSS_FRAME_FORMAT_QPLANE6)
-#define CSS_FRAME_FORMAT_BINARY_8	CSS_ID(CSS_FRAME_FORMAT_BINARY_8)
-
 struct atomisp_device;
 struct atomisp_sub_device;
 struct video_device;
 enum atomisp_input_stream_id;
 
+extern void __iomem *atomisp_io_base;
+
 struct atomisp_metadata_buf {
 	struct ia_css_metadata *metadata;
 	void *md_vptr;
 	struct list_head list;
 };
 
-void atomisp_css_debug_dump_sp_sw_debug_info(void);
-void atomisp_css_debug_dump_debug_info(const char *context);
-void atomisp_css_debug_set_dtrace_level(const unsigned int trace_level);
-
-void atomisp_store_uint32(hrt_address addr, uint32_t data);
+void atomisp_css2_hw_store_32(hrt_address addr, uint32_t data);
 void atomisp_load_uint32(hrt_address addr, uint32_t *data);
 
 int atomisp_css_init(struct atomisp_device *isp);
@@ -155,35 +60,35 @@ void atomisp_css_rx_clear_irq_info(enum mipi_port_id port,
 				   unsigned int infos);
 
 int atomisp_css_irq_enable(struct atomisp_device *isp,
-			   enum atomisp_css_irq_info info, bool enable);
+			   enum ia_css_irq_info info, bool enable);
 
 int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd,
 				  struct videobuf_vmalloc_memory *vm_mem,
 				  enum atomisp_input_stream_id stream_id,
-				  enum atomisp_css_buffer_type css_buf_type,
-				  enum atomisp_css_pipe_id css_pipe_id);
+				  enum ia_css_buffer_type css_buf_type,
+				  enum ia_css_pipe_id css_pipe_id);
 
 int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd,
 				struct atomisp_s3a_buf *s3a_buf,
 				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_pipe_id css_pipe_id);
+				enum ia_css_pipe_id css_pipe_id);
 
 int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd,
 				     struct atomisp_metadata_buf *metadata_buf,
 				     enum atomisp_input_stream_id stream_id,
-				     enum atomisp_css_pipe_id css_pipe_id);
+				     enum ia_css_pipe_id css_pipe_id);
 
 int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd,
 				struct atomisp_dis_buf *dis_buf,
 				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_pipe_id css_pipe_id);
+				enum ia_css_pipe_id css_pipe_id);
 
-void atomisp_css_mmu_invalidate_cache(void);
+void ia_css_mmu_invalidate_cache(void);
 
-void atomisp_css_mmu_invalidate_tlb(void);
+void ia_css_mmu_invalidate_cache(void);
 
 int atomisp_css_start(struct atomisp_sub_device *asd,
-		      enum atomisp_css_pipe_id pipe_id, bool in_reset);
+		      enum ia_css_pipe_id pipe_id, bool in_reset);
 
 void atomisp_css_update_isp_params(struct atomisp_sub_device *asd);
 void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd,
@@ -191,14 +96,14 @@ void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd,
 
 int atomisp_css_queue_buffer(struct atomisp_sub_device *asd,
 			     enum atomisp_input_stream_id stream_id,
-			     enum atomisp_css_pipe_id pipe_id,
-			     enum atomisp_css_buffer_type buf_type,
+			     enum ia_css_pipe_id pipe_id,
+			     enum ia_css_buffer_type buf_type,
 			     struct atomisp_css_buffer *isp_css_buffer);
 
 int atomisp_css_dequeue_buffer(struct atomisp_sub_device *asd,
 			       enum atomisp_input_stream_id stream_id,
-			       enum atomisp_css_pipe_id pipe_id,
-			       enum atomisp_css_buffer_type buf_type,
+			       enum ia_css_pipe_id pipe_id,
+			       enum ia_css_buffer_type buf_type,
 			       struct atomisp_css_buffer *isp_css_buffer);
 
 int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device *asd,
@@ -217,7 +122,7 @@ void atomisp_css_free_metadata_buffer(struct atomisp_metadata_buf
 				      *metadata_buf);
 
 int atomisp_css_get_grid_info(struct atomisp_sub_device *asd,
-			      enum atomisp_css_pipe_id pipe_id,
+			      enum ia_css_pipe_id pipe_id,
 			      int source_pad);
 
 int atomisp_alloc_3a_output_buf(struct atomisp_sub_device *asd);
@@ -287,7 +192,7 @@ void atomisp_css_input_set_binning_factor(struct atomisp_sub_device *asd,
 
 void atomisp_css_input_set_bayer_order(struct atomisp_sub_device *asd,
 				       enum atomisp_input_stream_id stream_id,
-				       enum atomisp_css_bayer_order bayer_order);
+				       enum ia_css_bayer_order bayer_order);
 
 void atomisp_css_input_set_format(struct atomisp_sub_device *asd,
 				  enum atomisp_input_stream_id stream_id,
@@ -312,10 +217,10 @@ void atomisp_css_enable_raw_binning(struct atomisp_sub_device *asd,
 void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable);
 
 void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd,
-				  enum atomisp_css_capture_mode mode);
+				  enum ia_css_capture_mode mode);
 
 void atomisp_css_input_set_mode(struct atomisp_sub_device *asd,
-				enum atomisp_css_input_mode mode);
+				enum ia_css_input_mode mode);
 
 void atomisp_css_capture_enable_online(struct atomisp_sub_device *asd,
 				       unsigned short stream_index, bool enable);
@@ -341,130 +246,103 @@ int atomisp_css_input_configure_port(struct atomisp_sub_device *asd,
 				     unsigned int metadata_width,
 				     unsigned int metadata_height);
 
-int atomisp_css_frame_allocate(struct atomisp_css_frame **frame,
-			       unsigned int width, unsigned int height,
-			       enum atomisp_css_frame_format format,
-			       unsigned int padded_width,
-			       unsigned int raw_bit_depth);
-
-int atomisp_css_frame_allocate_from_info(struct atomisp_css_frame **frame,
-	const struct atomisp_css_frame_info *info);
-
-void atomisp_css_frame_free(struct atomisp_css_frame *frame);
-
-int atomisp_css_frame_map(struct atomisp_css_frame **frame,
-			  const struct atomisp_css_frame_info *info,
-			  const void __user *data, uint16_t attribute,
-			  void *context);
-
-int atomisp_css_set_black_frame(struct atomisp_sub_device *asd,
-				const struct atomisp_css_frame *raw_black_frame);
-
-int atomisp_css_allocate_continuous_frames(bool init_time,
-	struct atomisp_sub_device *asd);
-
-void atomisp_css_update_continuous_frames(struct atomisp_sub_device *asd);
-
 void atomisp_create_pipes_stream(struct atomisp_sub_device *asd);
 void atomisp_destroy_pipes_stream_force(struct atomisp_sub_device *asd);
 
 int atomisp_css_stop(struct atomisp_sub_device *asd,
-		     enum atomisp_css_pipe_id pipe_id, bool in_reset);
+		     enum ia_css_pipe_id pipe_id, bool in_reset);
 
 int atomisp_css_continuous_set_num_raw_frames(
     struct atomisp_sub_device *asd,
     int num_frames);
 
-void atomisp_css_disable_vf_pp(struct atomisp_sub_device *asd,
-			       bool disable);
-
 int atomisp_css_copy_configure_output(struct atomisp_sub_device *asd,
 				      unsigned int stream_index,
 				      unsigned int width, unsigned int height,
 				      unsigned int padded_width,
-				      enum atomisp_css_frame_format format);
+				      enum ia_css_frame_format format);
 
 int atomisp_css_yuvpp_configure_output(struct atomisp_sub_device *asd,
 				       unsigned int stream_index,
 				       unsigned int width, unsigned int height,
 				       unsigned int padded_width,
-				       enum atomisp_css_frame_format format);
+				       enum ia_css_frame_format format);
 
 int atomisp_css_yuvpp_configure_viewfinder(
     struct atomisp_sub_device *asd,
     unsigned int stream_index,
     unsigned int width, unsigned int height,
     unsigned int min_width,
-    enum atomisp_css_frame_format format);
+    enum ia_css_frame_format format);
 
 int atomisp_css_yuvpp_get_output_frame_info(
     struct atomisp_sub_device *asd,
     unsigned int stream_index,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_yuvpp_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
     unsigned int stream_index,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd,
 	unsigned int width, unsigned int height,
 	unsigned int min_width,
-	enum atomisp_css_frame_format format);
+	enum ia_css_frame_format format);
 
 int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd,
 	unsigned int width, unsigned int height,
 	unsigned int min_width,
-	enum atomisp_css_frame_format format);
+	enum ia_css_frame_format format);
 
 int atomisp_css_video_configure_output(struct atomisp_sub_device *asd,
 				       unsigned int width, unsigned int height,
 				       unsigned int min_width,
-				       enum atomisp_css_frame_format format);
+				       enum ia_css_frame_format format);
 
 int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
 			       u16 source_pad,
-			       struct atomisp_css_frame_info *frame_info);
+			       struct ia_css_frame_info *frame_info);
 
 int atomisp_css_video_configure_viewfinder(struct atomisp_sub_device *asd,
 	unsigned int width, unsigned int height,
 	unsigned int min_width,
-	enum atomisp_css_frame_format format);
+	enum ia_css_frame_format format);
 
 int atomisp_css_capture_configure_viewfinder(
     struct atomisp_sub_device *asd,
     unsigned int width, unsigned int height,
     unsigned int min_width,
-    enum atomisp_css_frame_format format);
+    enum ia_css_frame_format format);
 
 int atomisp_css_video_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_capture_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_copy_get_output_frame_info(
     struct atomisp_sub_device *asd,
     unsigned int stream_index,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_capture_get_output_raw_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_preview_get_output_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_capture_get_output_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_video_get_output_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info);
+    struct ia_css_frame_info *info);
 
 int atomisp_css_preview_configure_pp_input(
     struct atomisp_sub_device *asd,
@@ -486,60 +364,8 @@ int atomisp_css_exp_id_unlock(struct atomisp_sub_device *asd, int exp_id);
 int atomisp_css_capture_enable_xnr(struct atomisp_sub_device *asd,
 				   bool enable);
 
-void atomisp_css_send_input_frame(struct atomisp_sub_device *asd,
-				  unsigned short *data, unsigned int width,
-				  unsigned int height);
-
-bool atomisp_css_isp_has_started(void);
-
-void atomisp_css_request_flash(struct atomisp_sub_device *asd);
-
-void atomisp_css_set_wb_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_wb_config *wb_config);
-
-void atomisp_css_set_ob_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ob_config *ob_config);
-
-void atomisp_css_set_dp_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_dp_config *dp_config);
-
-void atomisp_css_set_de_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_de_config *de_config);
-
-void atomisp_css_set_dz_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_dz_config *dz_config);
-
-void atomisp_css_set_default_de_config(struct atomisp_sub_device *asd);
-
-void atomisp_css_set_ce_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ce_config *ce_config);
-
-void atomisp_css_set_nr_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_nr_config *nr_config);
-
-void atomisp_css_set_ee_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ee_config *ee_config);
-
-void atomisp_css_set_tnr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_tnr_config *tnr_config);
-
-void atomisp_css_set_cc_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_cc_config *cc_config);
-
-void atomisp_css_set_macc_table(struct atomisp_sub_device *asd,
-				struct atomisp_css_macc_table *macc_table);
-
-void atomisp_css_set_gamma_table(struct atomisp_sub_device *asd,
-				 struct atomisp_css_gamma_table *gamma_table);
-
 void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ctc_table *ctc_table);
-
-void atomisp_css_set_gc_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_gc_config *gc_config);
-
-void atomisp_css_set_3a_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_3a_config *s3a_config);
+			       struct ia_css_ctc_table *ctc_table);
 
 void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd,
 				      struct atomisp_dis_vector *vector);
@@ -590,29 +416,29 @@ int atomisp_css_get_formats_config(struct atomisp_sub_device *asd,
 				   struct atomisp_formats_config *formats_config);
 
 void atomisp_css_set_formats_config(struct atomisp_sub_device *asd,
-				    struct atomisp_css_formats_config *formats_config);
+				    struct ia_css_formats_config *formats_config);
 
 int atomisp_css_get_zoom_factor(struct atomisp_sub_device *asd,
 				unsigned int *zoom);
 
-struct atomisp_css_shading_table *atomisp_css_shading_table_alloc(
+struct ia_css_shading_table *atomisp_css_shading_table_alloc(
     unsigned int width, unsigned int height);
 
 void atomisp_css_set_shading_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_shading_table *table);
+				   struct ia_css_shading_table *table);
 
-void atomisp_css_shading_table_free(struct atomisp_css_shading_table *table);
+void atomisp_css_shading_table_free(struct ia_css_shading_table *table);
 
-struct atomisp_css_morph_table *atomisp_css_morph_table_allocate(
+struct ia_css_morph_table *atomisp_css_morph_table_allocate(
     unsigned int width, unsigned int height);
 
 void atomisp_css_set_morph_table(struct atomisp_sub_device *asd,
-				 struct atomisp_css_morph_table *table);
+				 struct ia_css_morph_table *table);
 
 void atomisp_css_get_morph_table(struct atomisp_sub_device *asd,
-				 struct atomisp_css_morph_table *table);
+				 struct ia_css_morph_table *table);
 
-void atomisp_css_morph_table_free(struct atomisp_css_morph_table *table);
+void atomisp_css_morph_table_free(struct ia_css_morph_table *table);
 
 void atomisp_css_set_cont_prev_start_time(struct atomisp_device *isp,
 	unsigned int overlap);
@@ -631,20 +457,20 @@ int atomisp_css_stop_acc_pipe(struct atomisp_sub_device *asd);
 void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd);
 
 int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd,
-				   struct atomisp_css_fw_info *fw,
-				   enum atomisp_css_pipe_id pipe_id,
+				   struct ia_css_fw_info *fw,
+				   enum ia_css_pipe_id pipe_id,
 				   unsigned int type);
 
 void atomisp_css_unload_acc_extension(struct atomisp_sub_device *asd,
-				      struct atomisp_css_fw_info *fw,
-				      enum atomisp_css_pipe_id pipe_id);
+				      struct ia_css_fw_info *fw,
+				      enum ia_css_pipe_id pipe_id);
 
 int atomisp_css_wait_acc_finish(struct atomisp_sub_device *asd);
 
 void atomisp_css_acc_done(struct atomisp_sub_device *asd);
 
 int atomisp_css_load_acc_binary(struct atomisp_sub_device *asd,
-				struct atomisp_css_fw_info *fw,
+				struct ia_css_fw_info *fw,
 				unsigned int index);
 
 void atomisp_css_unload_acc_binary(struct atomisp_sub_device *asd);
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
index 209bc9954a53..c1e282a974d0 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Clovertrail PNW Camera Imaging ISP subsystem.
  *
@@ -32,8 +33,6 @@
 #include "atomisp_ioctl.h"
 #include "atomisp_acc.h"
 
-#include "hrt/hive_isp_css_mm_hrt.h"
-
 #include <asm/intel-mid.h>
 
 #include "ia_css_debug.h"
@@ -68,82 +67,68 @@ struct bayer_ds_factor {
 	unsigned int denominator;
 };
 
-void atomisp_css_debug_dump_sp_sw_debug_info(void)
-{
-	ia_css_debug_dump_sp_sw_debug_info();
-}
-
-void atomisp_css_debug_dump_debug_info(const char *context)
-{
-	ia_css_debug_dump_debug_info(context);
-}
-
-void atomisp_css_debug_set_dtrace_level(const unsigned int trace_level)
-{
-	ia_css_debug_set_dtrace_level(trace_level);
-}
-
-unsigned int atomisp_css_debug_get_dtrace_level(void)
-{
-	return ia_css_debug_trace_level;
-}
-
 static void atomisp_css2_hw_store_8(hrt_address addr, uint8_t data)
 {
+	s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	_hrt_master_port_store_8(addr, data);
+	*io_virt_addr = data;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 }
 
 static void atomisp_css2_hw_store_16(hrt_address addr, uint16_t data)
 {
+	s16 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	_hrt_master_port_store_16(addr, data);
+	*io_virt_addr = data;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 }
 
-static void atomisp_css2_hw_store_32(hrt_address addr, uint32_t data)
+void atomisp_css2_hw_store_32(hrt_address addr, uint32_t data)
 {
+	s32 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	_hrt_master_port_store_32(addr, data);
+	*io_virt_addr = data;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 }
 
 static uint8_t atomisp_css2_hw_load_8(hrt_address addr)
 {
+	s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 	u8 ret;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	ret = _hrt_master_port_load_8(addr);
+	ret = *io_virt_addr;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 	return ret;
 }
 
 static uint16_t atomisp_css2_hw_load_16(hrt_address addr)
 {
+	s16 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 	u16 ret;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	ret = _hrt_master_port_load_16(addr);
+	ret = *io_virt_addr;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 	return ret;
 }
 
 static uint32_t atomisp_css2_hw_load_32(hrt_address addr)
 {
+	s32 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 	u32 ret;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	ret = _hrt_master_port_load_32(addr);
+	ret = *io_virt_addr;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 	return ret;
 }
@@ -151,27 +136,25 @@ static uint32_t atomisp_css2_hw_load_32(hrt_address addr)
 static void atomisp_css2_hw_store(hrt_address addr,
 				  const void *from, uint32_t n)
 {
+	s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 	unsigned int i;
-	unsigned int _to = (unsigned int)addr;
-	const char *_from = (const char *)from;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	for (i = 0; i < n; i++, _to++, _from++)
-		_hrt_master_port_store_8(_to, *_from);
+	for (i = 0; i < n; i++, io_virt_addr++, from++)
+		*io_virt_addr = *(s8 *)from;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 }
 
 static void atomisp_css2_hw_load(hrt_address addr, void *to, uint32_t n)
 {
+	s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF);
 	unsigned long flags;
 	unsigned int i;
-	char *_to = (char *)to;
-	unsigned int _from = (unsigned int)addr;
 
 	spin_lock_irqsave(&mmio_lock, flags);
-	for (i = 0; i < n; i++, _to++, _from++)
-		*_to = _hrt_master_port_load_8(_from);
+	for (i = 0; i < n; i++, to++, io_virt_addr++)
+		*(s8 *)to = *io_virt_addr;
 	spin_unlock_irqrestore(&mmio_lock, flags);
 }
 
@@ -193,11 +176,6 @@ static int atomisp_css2_err_print(const char *fmt, va_list args)
 	return 0;
 }
 
-void atomisp_store_uint32(hrt_address addr, uint32_t data)
-{
-	atomisp_css2_hw_store_32(addr, data);
-}
-
 void atomisp_load_uint32(hrt_address addr, uint32_t *data)
 {
 	*data = atomisp_css2_hw_load_32(addr);
@@ -215,16 +193,6 @@ static int hmm_get_mmu_base_addr(unsigned int *mmu_base_addr)
 	return 0;
 }
 
-static void atomisp_isp_parameters_clean_up(
-    struct atomisp_css_isp_config *config)
-{
-	/*
-	 * Set NULL to configs pointer to avoid they are set into isp again when
-	 * some configs are changed and need to be updated later.
-	 */
-	memset(config, 0, sizeof(*config));
-}
-
 static void __dump_pipe_config(struct atomisp_sub_device *asd,
 			       struct atomisp_stream_env *stream_env,
 			       unsigned int pipe_id)
@@ -474,7 +442,7 @@ static int __destroy_stream(struct atomisp_sub_device *asd,
 	}
 
 	if (stream_env->stream_state == CSS_STREAM_STARTED
-	    && ia_css_stream_stop(stream_env->stream) != IA_CSS_SUCCESS) {
+	    && ia_css_stream_stop(stream_env->stream) != 0) {
 		dev_err(isp->dev, "stop stream failed.\n");
 		return -EINVAL;
 	}
@@ -496,7 +464,7 @@ static int __destroy_stream(struct atomisp_sub_device *asd,
 
 	stream_env->stream_state = CSS_STREAM_STOPPED;
 
-	if (ia_css_stream_destroy(stream_env->stream) != IA_CSS_SUCCESS) {
+	if (ia_css_stream_destroy(stream_env->stream)) {
 		dev_err(isp->dev, "destroy stream failed.\n");
 		return -EINVAL;
 	}
@@ -540,10 +508,10 @@ static int __create_stream(struct atomisp_sub_device *asd,
 
 	__dump_stream_config(asd, stream_env);
 	if (ia_css_stream_create(&stream_env->stream_config,
-				 pipe_index, multi_pipes, &stream_env->stream) != IA_CSS_SUCCESS)
+				 pipe_index, multi_pipes, &stream_env->stream) != 0)
 		return -EINVAL;
 	if (ia_css_stream_get_info(stream_env->stream,
-				   &stream_env->stream_info) != IA_CSS_SUCCESS) {
+				   &stream_env->stream_info) != 0) {
 		ia_css_stream_destroy(stream_env->stream);
 		stream_env->stream = NULL;
 		return -EINVAL;
@@ -583,7 +551,7 @@ static int __destroy_stream_pipes(struct atomisp_sub_device *asd,
 		    !(force || stream_env->update_pipe[i]))
 			continue;
 		if (ia_css_pipe_destroy(stream_env->pipes[i])
-		    != IA_CSS_SUCCESS) {
+		    != 0) {
 			dev_err(isp->dev,
 				"destroy pipe[%d]failed.cannot recover.\n", i);
 			ret = -EINVAL;
@@ -645,10 +613,10 @@ static void __apply_additional_pipe_config(
 		/* enable capture pp/dz manually or digital zoom would
 		 * fail*/
 		if (stream_env->pipe_configs[pipe_id].
-		    default_capture_config.mode == CSS_CAPTURE_MODE_RAW)
+		    default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW)
 			stream_env->pipe_configs[pipe_id].enable_dz = false;
 
-		if (atomisp_hw_is_isp2401) {
+		if (IS_ISP2401) {
 			/* the isp default to use ISP2.2 and the camera hal will
 			* control whether use isp2.7 */
 			if (asd->select_isp_version->val == ATOMISP_CSS_ISP_PIPE_VERSION_2_7)
@@ -699,7 +667,7 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd,
 	if (!asd)
 		return false;
 
-	if (pipe_id == CSS_PIPE_ID_ACC || pipe_id == CSS_PIPE_ID_YUVPP)
+	if (pipe_id == IA_CSS_PIPE_ID_ACC || pipe_id == IA_CSS_PIPE_ID_YUVPP)
 		return true;
 
 	if (asd->vfpp) {
@@ -726,22 +694,23 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd,
 	case ATOMISP_RUN_MODE_STILL_CAPTURE:
 		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE)
 			return true;
-		else
-			return false;
+
+		return false;
 	case ATOMISP_RUN_MODE_PREVIEW:
 		if (!asd->continuous_mode->val) {
 			if (pipe_id == IA_CSS_PIPE_ID_PREVIEW)
 				return true;
-			else
-				return false;
+
+			return false;
 		}
-	/* fall through to ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE */
+	/* fall-through */
 	case ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE:
 		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE ||
 		    pipe_id == IA_CSS_PIPE_ID_PREVIEW)
 			return true;
-		else
-			return false;
+
+		return false;
+	/* fall-through */
 	case ATOMISP_RUN_MODE_VIDEO:
 		if (!asd->continuous_mode->val) {
 			if (pipe_id == IA_CSS_PIPE_ID_VIDEO ||
@@ -750,13 +719,13 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd,
 			else
 				return false;
 		}
-	/* fall through to ATOMISP_RUN_MODE_SDV */
+	/* fall through  */
 	case ATOMISP_RUN_MODE_SDV:
 		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE ||
 		    pipe_id == IA_CSS_PIPE_ID_VIDEO)
 			return true;
-		else
-			return false;
+
+		return false;
 	}
 
 	return false;
@@ -768,16 +737,16 @@ static int __create_pipe(struct atomisp_sub_device *asd,
 {
 	struct atomisp_device *isp = asd->isp;
 	struct ia_css_pipe_extra_config extra_config;
-	enum ia_css_err ret;
+	int ret;
 
 	if (pipe_id >= IA_CSS_PIPE_ID_NUM)
 		return -EINVAL;
 
-	if (pipe_id != CSS_PIPE_ID_ACC &&
+	if (pipe_id != IA_CSS_PIPE_ID_ACC &&
 	    !stream_env->pipe_configs[pipe_id].output_info[0].res.width)
 		return 0;
 
-	if (pipe_id == CSS_PIPE_ID_ACC &&
+	if (pipe_id == IA_CSS_PIPE_ID_ACC &&
 	    !stream_env->pipe_configs[pipe_id].acc_extension)
 		return 0;
 
@@ -798,20 +767,20 @@ static int __create_pipe(struct atomisp_sub_device *asd,
 			  &stream_env->pipe_configs[pipe_id],
 			  &stream_env->pipe_extra_configs[pipe_id],
 			  &stream_env->pipes[pipe_id]);
-	if (ret != IA_CSS_SUCCESS)
+	if (ret)
 		dev_err(isp->dev, "create pipe[%d] error.\n", pipe_id);
 	return ret;
 }
 
 static int __create_pipes(struct atomisp_sub_device *asd)
 {
-	enum ia_css_err ret;
+	int ret;
 	int i, j;
 
 	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
 		for (j = 0; j < IA_CSS_PIPE_ID_NUM; j++) {
 			ret = __create_pipe(asd, &asd->stream_env[i], j);
-			if (ret != IA_CSS_SUCCESS)
+			if (ret)
 				break;
 		}
 		if (j < IA_CSS_PIPE_ID_NUM)
@@ -842,20 +811,20 @@ int atomisp_css_update_stream(struct atomisp_sub_device *asd)
 	int ret;
 	struct atomisp_device *isp = asd->isp;
 
-	if (__destroy_streams(asd, true) != IA_CSS_SUCCESS)
+	if (__destroy_streams(asd, true))
 		dev_warn(isp->dev, "destroy stream failed.\n");
 
-	if (__destroy_pipes(asd, true) != IA_CSS_SUCCESS)
+	if (__destroy_pipes(asd, true))
 		dev_warn(isp->dev, "destroy pipe failed.\n");
 
 	ret = __create_pipes(asd);
-	if (ret != IA_CSS_SUCCESS) {
+	if (ret) {
 		dev_err(isp->dev, "create pipe failed %d.\n", ret);
 		return -EIO;
 	}
 
 	ret = __create_streams(asd);
-	if (ret != IA_CSS_SUCCESS) {
+	if (ret) {
 		dev_warn(isp->dev, "create stream failed %d.\n", ret);
 		__destroy_pipes(asd, true);
 		return -EIO;
@@ -868,7 +837,7 @@ int atomisp_css_init(struct atomisp_device *isp)
 {
 	unsigned int mmu_base_addr;
 	int ret;
-	enum ia_css_err err;
+	int err;
 
 	ret = hmm_get_mmu_base_addr(&mmu_base_addr);
 	if (ret)
@@ -877,7 +846,7 @@ int atomisp_css_init(struct atomisp_device *isp)
 	/* Init ISP */
 	err = ia_css_init(isp->dev, &isp->css_env.isp_css_env, NULL,
 			  (uint32_t)mmu_base_addr, IA_CSS_IRQ_TYPE_PULSE);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		dev_err(isp->dev, "css init failed --- bad firmware?\n");
 		return -EINVAL;
 	}
@@ -907,17 +876,9 @@ static inline int __set_css_print_env(struct atomisp_device *isp, int opt)
 	return ret;
 }
 
-int atomisp_css_check_firmware_version(struct atomisp_device *isp)
-{
-	if (!sh_css_check_firmware_version(isp->dev, (void *)isp->firmware->data)) {
-		return -EINVAL;
-	}
-	return 0;
-}
-
 int atomisp_css_load_firmware(struct atomisp_device *isp)
 {
-	enum ia_css_err err;
+	int err;
 
 	/* set css env */
 	isp->css_env.isp_css_fw.data = (void *)isp->firmware->data;
@@ -946,7 +907,7 @@ int atomisp_css_load_firmware(struct atomisp_device *isp)
 	/* load isp fw into ISP memory */
 	err = ia_css_load_firmware(isp->dev, &isp->css_env.isp_css_env,
 				   &isp->css_env.isp_css_fw);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		dev_err(isp->dev, "css load fw failed.\n");
 		return -EINVAL;
 	}
@@ -954,11 +915,6 @@ int atomisp_css_load_firmware(struct atomisp_device *isp)
 	return 0;
 }
 
-void atomisp_css_unload_firmware(struct atomisp_device *isp)
-{
-	ia_css_unload_firmware();
-}
-
 void atomisp_css_uninit(struct atomisp_device *isp)
 {
 	struct atomisp_sub_device *asd;
@@ -966,7 +922,7 @@ void atomisp_css_uninit(struct atomisp_device *isp)
 
 	for (i = 0; i < isp->num_of_streams; i++) {
 		asd = &isp->asd[i];
-		atomisp_isp_parameters_clean_up(&asd->params.config);
+		memset(&asd->params.config, 0, sizeof(asd->params.config));
 		asd->params.css_update_params_needed = false;
 	}
 
@@ -1009,7 +965,7 @@ int atomisp_css_irq_translate(struct atomisp_device *isp,
 	int err;
 
 	err = ia_css_irq_translate(infos);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		dev_warn(isp->dev,
 			 "%s:failed to translate irq (err = %d,infos = %d)\n",
 			 __func__, err, *infos);
@@ -1038,13 +994,15 @@ void atomisp_css_rx_clear_irq_info(enum mipi_port_id port,
 }
 
 int atomisp_css_irq_enable(struct atomisp_device *isp,
-			   enum atomisp_css_irq_info info, bool enable)
+			   enum ia_css_irq_info info, bool enable)
 {
-	dev_dbg(isp->dev, "%s: css irq info 0x%08x: %s.\n",
+	dev_dbg(isp->dev, "%s: css irq info 0x%08x: %s (%d).\n",
 		__func__, info,
-		enable ? "enable" : "disable");
-	if (ia_css_irq_enable(info, enable) != IA_CSS_SUCCESS) {
-		dev_warn(isp->dev, "%s:Invalid irq info.\n", __func__);
+		enable ? "enable" : "disable", enable);
+	if (ia_css_irq_enable(info, enable)) {
+		dev_warn(isp->dev, "%s:Invalid irq info: 0x%08x when %s.\n",
+			 __func__, info,
+			 enable ? "enabling" : "disabling");
 		return -EINVAL;
 	}
 
@@ -1072,19 +1030,19 @@ void atomisp_css_init_struct(struct atomisp_sub_device *asd)
 int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd,
 				  struct videobuf_vmalloc_memory *vm_mem,
 				  enum atomisp_input_stream_id stream_id,
-				  enum atomisp_css_buffer_type css_buf_type,
-				  enum atomisp_css_pipe_id css_pipe_id)
+				  enum ia_css_buffer_type css_buf_type,
+				  enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
 	struct ia_css_buffer css_buf = {0};
-	enum ia_css_err err;
+	int err;
 
 	css_buf.type = css_buf_type;
 	css_buf.data.frame = vm_mem->vaddr;
 
 	err = ia_css_pipe_enqueue_buffer(
 		  stream_env->pipes[css_pipe_id], &css_buf);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return -EINVAL;
 
 	return 0;
@@ -1093,7 +1051,7 @@ int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd,
 int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd,
 				     struct atomisp_metadata_buf *metadata_buf,
 				     enum atomisp_input_stream_id stream_id,
-				     enum atomisp_css_pipe_id css_pipe_id)
+				     enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
 	struct ia_css_buffer buffer = {0};
@@ -1113,7 +1071,7 @@ int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd,
 int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd,
 				struct atomisp_s3a_buf *s3a_buf,
 				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_pipe_id css_pipe_id)
+				enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
 	struct ia_css_buffer buffer = {0};
@@ -1134,7 +1092,7 @@ int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd,
 int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd,
 				struct atomisp_dis_buf *dis_buf,
 				enum atomisp_input_stream_id stream_id,
-				enum atomisp_css_pipe_id css_pipe_id)
+				enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id];
 	struct ia_css_buffer buffer = {0};
@@ -1152,18 +1110,8 @@ int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd,
 	return 0;
 }
 
-void atomisp_css_mmu_invalidate_cache(void)
-{
-	ia_css_mmu_invalidate_cache();
-}
-
-void atomisp_css_mmu_invalidate_tlb(void)
-{
-	ia_css_mmu_invalidate_cache();
-}
-
 int atomisp_css_start(struct atomisp_sub_device *asd,
-		      enum atomisp_css_pipe_id pipe_id, bool in_reset)
+		      enum ia_css_pipe_id pipe_id, bool in_reset)
 {
 	struct atomisp_device *isp = asd->isp;
 	bool sp_is_started = false;
@@ -1222,7 +1170,7 @@ int atomisp_css_start(struct atomisp_sub_device *asd,
 	} else {
 		if (!sh_css_hrt_system_is_idle())
 			dev_err(isp->dev, "CSS HW not idle before starting SP\n");
-		if (ia_css_start_sp() != IA_CSS_SUCCESS) {
+		if (ia_css_start_sp()) {
 			dev_err(isp->dev, "start sp error.\n");
 			ret = -EINVAL;
 			goto start_err;
@@ -1234,7 +1182,7 @@ int atomisp_css_start(struct atomisp_sub_device *asd,
 	for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) {
 		if (asd->stream_env[i].stream) {
 			if (ia_css_stream_start(asd->stream_env[i]
-						.stream) != IA_CSS_SUCCESS) {
+						.stream) != 0) {
 				dev_err(isp->dev, "stream[%d] start error.\n", i);
 				ret = -EINVAL;
 				goto start_err;
@@ -1285,13 +1233,13 @@ void atomisp_css_update_isp_params(struct atomisp_sub_device *asd)
 	ia_css_stream_set_isp_config(
 	    asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
 	    &asd->params.config);
-	atomisp_isp_parameters_clean_up(&asd->params.config);
+	memset(&asd->params.config, 0, sizeof(asd->params.config));
 }
 
 void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd,
 	struct ia_css_pipe *pipe)
 {
-	enum ia_css_err ret;
+	int ret;
 
 	if (!pipe) {
 		atomisp_css_update_isp_params(asd);
@@ -1306,22 +1254,22 @@ void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd,
 	ret = ia_css_stream_set_isp_config_on_pipe(
 		  asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
 		  &asd->params.config, pipe);
-	if (ret != IA_CSS_SUCCESS)
+	if (ret)
 		dev_warn(asd->isp->dev, "%s: ia_css_stream_set_isp_config_on_pipe failed %d\n",
 			 __func__, ret);
-	atomisp_isp_parameters_clean_up(&asd->params.config);
+	memset(&asd->params.config, 0, sizeof(asd->params.config));
 }
 
 int atomisp_css_queue_buffer(struct atomisp_sub_device *asd,
 			     enum atomisp_input_stream_id stream_id,
-			     enum atomisp_css_pipe_id pipe_id,
-			     enum atomisp_css_buffer_type buf_type,
+			     enum ia_css_pipe_id pipe_id,
+			     enum ia_css_buffer_type buf_type,
 			     struct atomisp_css_buffer *isp_css_buffer)
 {
 	if (ia_css_pipe_enqueue_buffer(
 		asd->stream_env[stream_id].pipes[pipe_id],
 		&isp_css_buffer->css_buffer)
-	    != IA_CSS_SUCCESS)
+	    != 0)
 		return -EINVAL;
 
 	return 0;
@@ -1329,17 +1277,17 @@ int atomisp_css_queue_buffer(struct atomisp_sub_device *asd,
 
 int atomisp_css_dequeue_buffer(struct atomisp_sub_device *asd,
 			       enum atomisp_input_stream_id stream_id,
-			       enum atomisp_css_pipe_id pipe_id,
-			       enum atomisp_css_buffer_type buf_type,
+			       enum ia_css_pipe_id pipe_id,
+			       enum ia_css_buffer_type buf_type,
 			       struct atomisp_css_buffer *isp_css_buffer)
 {
 	struct atomisp_device *isp = asd->isp;
-	enum ia_css_err err;
+	int err;
 
 	err = ia_css_pipe_dequeue_buffer(
 		  asd->stream_env[stream_id].pipes[pipe_id],
 		  &isp_css_buffer->css_buffer);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		dev_err(isp->dev,
 			"ia_css_pipe_dequeue_buffer failed: 0x%x\n", err);
 		return -EINVAL;
@@ -1355,7 +1303,7 @@ int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device   *asd,
 				      struct atomisp_metadata_buf *md_buf)
 {
 	struct atomisp_device *isp = asd->isp;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
+	struct ia_css_dvs_grid_info *dvs_grid_info =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 
 	if (s3a_buf && asd->params.curr_grid_info.s3a_grid.enable) {
@@ -1442,7 +1390,7 @@ void atomisp_css_free_stat_buffers(struct atomisp_sub_device *asd)
 	struct atomisp_s3a_buf *s3a_buf, *_s3a_buf;
 	struct atomisp_dis_buf *dis_buf, *_dis_buf;
 	struct atomisp_metadata_buf *md_buf, *_md_buf;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
+	struct ia_css_dvs_grid_info *dvs_grid_info =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 	unsigned int i;
 
@@ -1524,7 +1472,7 @@ void atomisp_css_free_stat_buffers(struct atomisp_sub_device *asd)
 }
 
 int atomisp_css_get_grid_info(struct atomisp_sub_device *asd,
-			      enum atomisp_css_pipe_id pipe_id,
+			      enum ia_css_pipe_id pipe_id,
 			      int source_pad)
 {
 	struct ia_css_pipe_info p_info;
@@ -1539,7 +1487,7 @@ int atomisp_css_get_grid_info(struct atomisp_sub_device *asd,
 
 	if (ia_css_pipe_get_info(
 		asd->stream_env[stream_index].pipes[pipe_id],
-		&p_info) != IA_CSS_SUCCESS) {
+		&p_info) != 0) {
 		dev_err(isp->dev, "ia_css_pipe_get_info failed\n");
 		return -EINVAL;
 	}
@@ -1553,7 +1501,7 @@ int atomisp_css_get_grid_info(struct atomisp_sub_device *asd,
 	 * Currently would have one css pipe that need it
 	 */
 	if (asd->params.curr_grid_info.s3a_grid.enable) {
-		if (asd->params.s3a_enabled_pipe != CSS_PIPE_ID_NUM)
+		if (asd->params.s3a_enabled_pipe != IA_CSS_PIPE_ID_NUM)
 			dev_dbg(isp->dev, "css pipe %d enabled s3a grid replaced by: %d.\n",
 				asd->params.s3a_enabled_pipe, pipe_id);
 		asd->params.s3a_enabled_pipe = pipe_id;
@@ -1603,7 +1551,7 @@ int atomisp_alloc_3a_output_buf(struct atomisp_sub_device *asd)
 
 int atomisp_alloc_dis_coef_buf(struct atomisp_sub_device *asd)
 {
-	struct atomisp_css_dvs_grid_info *dvs_grid =
+	struct ia_css_dvs_grid_info *dvs_grid =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 
 	if (!dvs_grid)
@@ -1693,7 +1641,7 @@ void atomisp_css_get_dis_statistics(struct atomisp_sub_device *asd,
 
 int atomisp_css_dequeue_event(struct atomisp_css_event *current_event)
 {
-	if (ia_css_dequeue_event(&current_event->event) != IA_CSS_SUCCESS)
+	if (ia_css_dequeue_event(&current_event->event))
 		return -EINVAL;
 
 	return 0;
@@ -1753,7 +1701,7 @@ void atomisp_css_input_set_binning_factor(struct atomisp_sub_device *asd,
 
 void atomisp_css_input_set_bayer_order(struct atomisp_sub_device *asd,
 				       enum atomisp_input_stream_id stream_id,
-				       enum atomisp_css_bayer_order bayer_order)
+				       enum ia_css_bayer_order bayer_order)
 {
 	struct ia_css_stream_config *s_config =
 		    &asd->stream_env[stream_id].stream_config;
@@ -1959,7 +1907,7 @@ void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable)
 }
 
 void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd,
-				  enum atomisp_css_capture_mode mode)
+				  enum ia_css_capture_mode mode)
 {
 	struct atomisp_stream_env *stream_env =
 		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
@@ -1974,7 +1922,7 @@ void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd,
 }
 
 void atomisp_css_input_set_mode(struct atomisp_sub_device *asd,
-				enum atomisp_css_input_mode mode)
+				enum ia_css_input_mode mode)
 {
 	int i;
 	struct atomisp_device *isp = asd->isp;
@@ -2017,7 +1965,7 @@ void atomisp_css_input_set_mode(struct atomisp_sub_device *asd,
 			s_config->input_config.format,
 			true,
 			0x13000,
-			&size_mem_words) != IA_CSS_SUCCESS) {
+			&size_mem_words) != 0) {
 			if (intel_mid_identify_cpu() ==
 			    INTEL_MID_CPU_CHIP_TANGIER)
 				size_mem_words = CSS_MIPI_FRAME_BUFFER_SIZE_2;
@@ -2157,74 +2105,8 @@ int atomisp_css_input_configure_port(
 	return 0;
 }
 
-int atomisp_css_frame_allocate(struct atomisp_css_frame **frame,
-			       unsigned int width, unsigned int height,
-			       enum atomisp_css_frame_format format,
-			       unsigned int padded_width,
-			       unsigned int raw_bit_depth)
-{
-	if (ia_css_frame_allocate(frame, width, height, format,
-				  padded_width, raw_bit_depth) != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-int atomisp_css_frame_allocate_from_info(struct atomisp_css_frame **frame,
-	const struct atomisp_css_frame_info *info)
-{
-	if (ia_css_frame_allocate_from_info(frame, info) != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-void atomisp_css_frame_free(struct atomisp_css_frame *frame)
-{
-	ia_css_frame_free(frame);
-}
-
-int atomisp_css_frame_map(struct atomisp_css_frame **frame,
-			  const struct atomisp_css_frame_info *info,
-			  const void __user *data, uint16_t attribute,
-			  void *context)
-{
-	if (ia_css_frame_map(frame, info, data, attribute, context)
-	    != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-int atomisp_css_set_black_frame(struct atomisp_sub_device *asd,
-				const struct atomisp_css_frame *raw_black_frame)
-{
-	if (sh_css_set_black_frame(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-		raw_black_frame) != IA_CSS_SUCCESS)
-		return -ENOMEM;
-
-	return 0;
-}
-
-int atomisp_css_allocate_continuous_frames(bool init_time,
-	struct atomisp_sub_device *asd)
-{
-	if (ia_css_alloc_continuous_frame_remain(
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream)
-	    != IA_CSS_SUCCESS)
-		return -EINVAL;
-	return 0;
-}
-
-void atomisp_css_update_continuous_frames(struct atomisp_sub_device *asd)
-{
-	ia_css_update_continuous_frames(
-	    asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream);
-}
-
 int atomisp_css_stop(struct atomisp_sub_device *asd,
-		     enum atomisp_css_pipe_id pipe_id, bool in_reset)
+		     enum ia_css_pipe_id pipe_id, bool in_reset)
 {
 	struct atomisp_device *isp = asd->isp;
 	struct atomisp_s3a_buf *s3a_buf;
@@ -2264,7 +2146,7 @@ int atomisp_css_stop(struct atomisp_sub_device *asd,
 			ia_css_stream_config_defaults(
 			    &stream_env->stream_config);
 		}
-		atomisp_isp_parameters_clean_up(&asd->params.config);
+		memset(&asd->params.config, 0, sizeof(asd->params.config));
 		asd->params.css_update_params_needed = false;
 	}
 
@@ -2345,16 +2227,6 @@ int atomisp_css_continuous_set_num_raw_frames(
 	return 0;
 }
 
-void atomisp_css_disable_vf_pp(struct atomisp_sub_device *asd,
-			       bool disable)
-{
-	int i;
-
-	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++)
-		asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
-		.pipe_extra_configs[i].disable_vf_pp = !!disable;
-}
-
 static enum ia_css_pipe_mode __pipe_id_to_pipe_mode(
     struct atomisp_sub_device *asd,
     enum ia_css_pipe_id pipe_id)
@@ -2749,7 +2621,7 @@ done:
 static void __configure_vf_output(struct atomisp_sub_device *asd,
 				  unsigned int width, unsigned int height,
 				  unsigned int min_width,
-				  enum atomisp_css_frame_format format,
+				  enum ia_css_frame_format format,
 				  enum ia_css_pipe_id pipe_id)
 {
 	struct atomisp_device *isp = asd->isp;
@@ -2772,7 +2644,7 @@ static void __configure_vf_output(struct atomisp_sub_device *asd,
 static void __configure_video_vf_output(struct atomisp_sub_device *asd,
 					unsigned int width, unsigned int height,
 					unsigned int min_width,
-					enum atomisp_css_frame_format format,
+					enum ia_css_frame_format format,
 					enum ia_css_pipe_id pipe_id)
 {
 	struct atomisp_device *isp = asd->isp;
@@ -2807,12 +2679,12 @@ static void __configure_video_vf_output(struct atomisp_sub_device *asd,
 
 static int __get_frame_info(struct atomisp_sub_device *asd,
 			    unsigned int stream_index,
-			    struct atomisp_css_frame_info *info,
+			    struct ia_css_frame_info *info,
 			    enum frame_info_type type,
 			    enum ia_css_pipe_id pipe_id)
 {
 	struct atomisp_device *isp = asd->isp;
-	enum ia_css_err ret;
+	int ret;
 	struct ia_css_pipe_info p_info;
 
 	/* FIXME! No need to destroy/recreate all streams */
@@ -2831,7 +2703,7 @@ static int __get_frame_info(struct atomisp_sub_device *asd,
 	ret = ia_css_pipe_get_info(
 		  asd->stream_env[stream_index]
 		  .pipes[pipe_id], &p_info);
-	if (ret == IA_CSS_SUCCESS) {
+	if (!ret) {
 		switch (type) {
 		case ATOMISP_CSS_VF_FRAME:
 			*info = p_info.vf_output_info[0];
@@ -2882,16 +2754,18 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd,
 		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO
 		    || asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER)
 			return IA_CSS_PIPE_ID_VIDEO;
-		else
-			return IA_CSS_PIPE_ID_CAPTURE;
+
+		return IA_CSS_PIPE_ID_CAPTURE;
 	case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE:
 		if (asd->copy_mode)
 			return IA_CSS_PIPE_ID_COPY;
+
 		return IA_CSS_PIPE_ID_CAPTURE;
 	case ATOMISP_SUBDEV_PAD_SOURCE_VF:
-		if (!atomisp_is_mbuscode_raw(
-			asd->fmt[asd->capture_pad].fmt.code))
+		if (!atomisp_is_mbuscode_raw(asd->fmt[asd->capture_pad].fmt.code)) {
 			return IA_CSS_PIPE_ID_CAPTURE;
+		}
+		/* fall through */
 	case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW:
 		if (asd->yuvpp_mode)
 			return IA_CSS_PIPE_ID_YUVPP;
@@ -2899,8 +2773,8 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd,
 			return IA_CSS_PIPE_ID_COPY;
 		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
 			return IA_CSS_PIPE_ID_VIDEO;
-		else
-			return IA_CSS_PIPE_ID_PREVIEW;
+
+		return IA_CSS_PIPE_ID_PREVIEW;
 	}
 	dev_warn(isp->dev,
 		 "invalid source pad:%d, return default preview pipe index.\n",
@@ -2910,7 +2784,7 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd,
 
 int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
 			       u16 source_pad,
-			       struct atomisp_css_frame_info *frame_info)
+			       struct ia_css_frame_info *frame_info)
 {
 	struct ia_css_pipe_info info;
 	int pipe_index = atomisp_get_pipe_index(asd, source_pad);
@@ -2925,7 +2799,7 @@ int atomisp_get_css_frame_info(struct atomisp_sub_device *asd,
 			       atomisp_source_pad_to_stream_id(asd, source_pad);
 	}
 
-	if (IA_CSS_SUCCESS != ia_css_pipe_get_info(asd->stream_env[stream_index]
+	if (0 != ia_css_pipe_get_info(asd->stream_env[stream_index]
 		.pipes[pipe_index], &info)) {
 		dev_err(isp->dev, "ia_css_pipe_get_info FAILED");
 		return -EINVAL;
@@ -2978,11 +2852,11 @@ int atomisp_css_copy_configure_output(struct atomisp_sub_device *asd,
 				      unsigned int stream_index,
 				      unsigned int width, unsigned int height,
 				      unsigned int padded_width,
-				      enum atomisp_css_frame_format format)
+				      enum ia_css_frame_format format)
 {
 	asd->stream_env[stream_index].pipe_configs[IA_CSS_PIPE_ID_COPY].
 	default_capture_config.mode =
-	    CSS_CAPTURE_MODE_RAW;
+	    IA_CSS_CAPTURE_MODE_RAW;
 
 	__configure_output(asd, stream_index, width, height, padded_width,
 			   format, IA_CSS_PIPE_ID_COPY);
@@ -2993,11 +2867,11 @@ int atomisp_css_yuvpp_configure_output(struct atomisp_sub_device *asd,
 				       unsigned int stream_index,
 				       unsigned int width, unsigned int height,
 				       unsigned int padded_width,
-				       enum atomisp_css_frame_format format)
+				       enum ia_css_frame_format format)
 {
 	asd->stream_env[stream_index].pipe_configs[IA_CSS_PIPE_ID_YUVPP].
 	default_capture_config.mode =
-	    CSS_CAPTURE_MODE_RAW;
+	    IA_CSS_CAPTURE_MODE_RAW;
 
 	__configure_output(asd, stream_index, width, height, padded_width,
 			   format, IA_CSS_PIPE_ID_YUVPP);
@@ -3009,7 +2883,7 @@ int atomisp_css_yuvpp_configure_viewfinder(
     unsigned int stream_index,
     unsigned int width, unsigned int height,
     unsigned int min_width,
-    enum atomisp_css_frame_format format)
+    enum ia_css_frame_format format)
 {
 	struct atomisp_stream_env *stream_env =
 		    &asd->stream_env[stream_index];
@@ -3030,7 +2904,7 @@ int atomisp_css_yuvpp_configure_viewfinder(
 int atomisp_css_yuvpp_get_output_frame_info(
     struct atomisp_sub_device *asd,
     unsigned int stream_index,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	return __get_frame_info(asd, stream_index, info,
 				ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_YUVPP);
@@ -3039,7 +2913,7 @@ int atomisp_css_yuvpp_get_output_frame_info(
 int atomisp_css_yuvpp_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
     unsigned int stream_index,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	return __get_frame_info(asd, stream_index, info,
 				ATOMISP_CSS_VF_FRAME, IA_CSS_PIPE_ID_YUVPP);
@@ -3048,7 +2922,7 @@ int atomisp_css_yuvpp_get_viewfinder_frame_info(
 int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd,
 	unsigned int width, unsigned int height,
 	unsigned int min_width,
-	enum atomisp_css_frame_format format)
+	enum ia_css_frame_format format)
 {
 	/*
 	 * to SOC camera, use yuvpp pipe.
@@ -3066,7 +2940,7 @@ int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd,
 int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd,
 	unsigned int width, unsigned int height,
 	unsigned int min_width,
-	enum atomisp_css_frame_format format)
+	enum ia_css_frame_format format)
 {
 	enum ia_css_pipe_id pipe_id;
 
@@ -3086,7 +2960,7 @@ int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd,
 int atomisp_css_video_configure_output(struct atomisp_sub_device *asd,
 				       unsigned int width, unsigned int height,
 				       unsigned int min_width,
-				       enum atomisp_css_frame_format format)
+				       enum ia_css_frame_format format)
 {
 	/*
 	 * to SOC camera, use yuvpp pipe.
@@ -3105,7 +2979,7 @@ int atomisp_css_video_configure_viewfinder(
     struct atomisp_sub_device *asd,
     unsigned int width, unsigned int height,
     unsigned int min_width,
-    enum atomisp_css_frame_format format)
+    enum ia_css_frame_format format)
 {
 	/*
 	 * to SOC camera, video will use yuvpp pipe.
@@ -3123,7 +2997,7 @@ int atomisp_css_capture_configure_viewfinder(
     struct atomisp_sub_device *asd,
     unsigned int width, unsigned int height,
     unsigned int min_width,
-    enum atomisp_css_frame_format format)
+    enum ia_css_frame_format format)
 {
 	enum ia_css_pipe_id pipe_id;
 
@@ -3142,7 +3016,7 @@ int atomisp_css_capture_configure_viewfinder(
 
 int atomisp_css_video_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	enum ia_css_pipe_id pipe_id;
 	enum frame_info_type frame_type = ATOMISP_CSS_VF_FRAME;
@@ -3161,7 +3035,7 @@ int atomisp_css_video_get_viewfinder_frame_info(
 
 int atomisp_css_capture_get_viewfinder_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	enum ia_css_pipe_id pipe_id;
 
@@ -3176,7 +3050,7 @@ int atomisp_css_capture_get_viewfinder_frame_info(
 
 int atomisp_css_capture_get_output_raw_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	if (ATOMISP_USE_YUVPP(asd))
 		return 0;
@@ -3188,7 +3062,7 @@ int atomisp_css_capture_get_output_raw_frame_info(
 int atomisp_css_copy_get_output_frame_info(
     struct atomisp_sub_device *asd,
     unsigned int stream_index,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	return __get_frame_info(asd, stream_index, info,
 				ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_COPY);
@@ -3196,7 +3070,7 @@ int atomisp_css_copy_get_output_frame_info(
 
 int atomisp_css_preview_get_output_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	enum ia_css_pipe_id pipe_id;
 	enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME;
@@ -3215,7 +3089,7 @@ int atomisp_css_preview_get_output_frame_info(
 
 int atomisp_css_capture_get_output_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	enum ia_css_pipe_id pipe_id;
 
@@ -3230,7 +3104,7 @@ int atomisp_css_capture_get_output_frame_info(
 
 int atomisp_css_video_get_output_frame_info(
     struct atomisp_sub_device *asd,
-    struct atomisp_css_frame_info *info)
+    struct ia_css_frame_info *info)
 {
 	enum ia_css_pipe_id pipe_id;
 	enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME;
@@ -3297,7 +3171,7 @@ int atomisp_css_video_configure_pp_input(
 int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd,
 	int num_captures, unsigned int skip, int offset)
 {
-	enum ia_css_err ret;
+	int ret;
 
 	dev_dbg(asd->isp->dev, "%s num_capture:%d skip:%d offset:%d\n",
 		__func__, num_captures, skip, offset);
@@ -3305,7 +3179,7 @@ int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd,
 	ret = ia_css_stream_capture(
 		  asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
 		  num_captures, skip, offset);
-	if (ret != IA_CSS_SUCCESS)
+	if (ret)
 		return -EINVAL;
 
 	return 0;
@@ -3313,15 +3187,15 @@ int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd,
 
 int atomisp_css_exp_id_capture(struct atomisp_sub_device *asd, int exp_id)
 {
-	enum ia_css_err ret;
+	int ret;
 
 	ret = ia_css_stream_capture_frame(
 		  asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
 		  exp_id);
-	if (ret == IA_CSS_ERR_QUEUE_IS_FULL) {
+	if (ret == -ENOBUFS) {
 		/* capture cmd queue is full */
 		return -EBUSY;
-	} else if (ret != IA_CSS_SUCCESS) {
+	} else if (ret) {
 		return -EIO;
 	}
 
@@ -3330,14 +3204,14 @@ int atomisp_css_exp_id_capture(struct atomisp_sub_device *asd, int exp_id)
 
 int atomisp_css_exp_id_unlock(struct atomisp_sub_device *asd, int exp_id)
 {
-	enum ia_css_err ret;
+	int ret;
 
 	ret = ia_css_unlock_raw_frame(
 		  asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
 		  exp_id);
-	if (ret == IA_CSS_ERR_QUEUE_IS_FULL)
+	if (ret == -ENOBUFS)
 		return -EAGAIN;
-	else if (ret != IA_CSS_SUCCESS)
+	else if (ret)
 		return -EIO;
 
 	return 0;
@@ -3356,201 +3230,8 @@ int atomisp_css_capture_enable_xnr(struct atomisp_sub_device *asd,
 	return 0;
 }
 
-void atomisp_css_send_input_frame(struct atomisp_sub_device *asd,
-				  unsigned short *data, unsigned int width,
-				  unsigned int height)
-{
-	ia_css_stream_send_input_frame(
-	    asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-	    data, width, height);
-}
-
-bool atomisp_css_isp_has_started(void)
-{
-	return ia_css_isp_has_started();
-}
-
-void atomisp_css_request_flash(struct atomisp_sub_device *asd)
-{
-	ia_css_stream_request_flash(
-	    asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream);
-}
-
-void atomisp_css_set_wb_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_wb_config *wb_config)
-{
-	asd->params.config.wb_config = wb_config;
-}
-
-void atomisp_css_set_ob_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ob_config *ob_config)
-{
-	asd->params.config.ob_config = ob_config;
-}
-
-void atomisp_css_set_dp_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_dp_config *dp_config)
-{
-	asd->params.config.dp_config = dp_config;
-}
-
-void atomisp_css_set_de_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_de_config *de_config)
-{
-	asd->params.config.de_config = de_config;
-}
-
-void atomisp_css_set_dz_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_dz_config *dz_config)
-{
-	asd->params.config.dz_config = dz_config;
-}
-
-void atomisp_css_set_default_de_config(struct atomisp_sub_device *asd)
-{
-	asd->params.config.de_config = NULL;
-}
-
-void atomisp_css_set_ce_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ce_config *ce_config)
-{
-	asd->params.config.ce_config = ce_config;
-}
-
-void atomisp_css_set_nr_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_nr_config *nr_config)
-{
-	asd->params.config.nr_config = nr_config;
-}
-
-void atomisp_css_set_ee_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ee_config *ee_config)
-{
-	asd->params.config.ee_config = ee_config;
-}
-
-void atomisp_css_set_tnr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_tnr_config *tnr_config)
-{
-	asd->params.config.tnr_config = tnr_config;
-}
-
-void atomisp_css_set_cc_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_cc_config *cc_config)
-{
-	asd->params.config.cc_config = cc_config;
-}
-
-void atomisp_css_set_macc_table(struct atomisp_sub_device *asd,
-				struct atomisp_css_macc_table *macc_table)
-{
-	asd->params.config.macc_table = macc_table;
-}
-
-void atomisp_css_set_macc_config(struct atomisp_sub_device *asd,
-				 struct atomisp_css_macc_config *macc_config)
-{
-	asd->params.config.macc_config = macc_config;
-}
-
-void atomisp_css_set_ecd_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_ecd_config *ecd_config)
-{
-	asd->params.config.ecd_config = ecd_config;
-}
-
-void atomisp_css_set_ynr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_ynr_config *ynr_config)
-{
-	asd->params.config.ynr_config = ynr_config;
-}
-
-void atomisp_css_set_fc_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_fc_config *fc_config)
-{
-	asd->params.config.fc_config = fc_config;
-}
-
-void atomisp_css_set_ctc_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_ctc_config *ctc_config)
-{
-	asd->params.config.ctc_config = ctc_config;
-}
-
-void atomisp_css_set_cnr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_cnr_config *cnr_config)
-{
-	asd->params.config.cnr_config = cnr_config;
-}
-
-void atomisp_css_set_aa_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_aa_config *aa_config)
-{
-	asd->params.config.aa_config = aa_config;
-}
-
-void atomisp_css_set_baa_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_baa_config *baa_config)
-{
-	asd->params.config.baa_config = baa_config;
-}
-
-void atomisp_css_set_anr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_anr_config *anr_config)
-{
-	asd->params.config.anr_config = anr_config;
-}
-
-void atomisp_css_set_xnr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_xnr_config *xnr_config)
-{
-	asd->params.config.xnr_config = xnr_config;
-}
-
-void atomisp_css_set_yuv2rgb_cc_config(struct atomisp_sub_device *asd,
-				       struct atomisp_css_cc_config *yuv2rgb_cc_config)
-{
-	asd->params.config.yuv2rgb_cc_config = yuv2rgb_cc_config;
-}
-
-void atomisp_css_set_rgb2yuv_cc_config(struct atomisp_sub_device *asd,
-				       struct atomisp_css_cc_config *rgb2yuv_cc_config)
-{
-	asd->params.config.rgb2yuv_cc_config = rgb2yuv_cc_config;
-}
-
-void atomisp_css_set_xnr_table(struct atomisp_sub_device *asd,
-			       struct atomisp_css_xnr_table *xnr_table)
-{
-	asd->params.config.xnr_table = xnr_table;
-}
-
-void atomisp_css_set_r_gamma_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_rgb_gamma_table *r_gamma_table)
-{
-	asd->params.config.r_gamma_table = r_gamma_table;
-}
-
-void atomisp_css_set_g_gamma_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_rgb_gamma_table *g_gamma_table)
-{
-	asd->params.config.g_gamma_table = g_gamma_table;
-}
-
-void atomisp_css_set_b_gamma_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_rgb_gamma_table *b_gamma_table)
-{
-	asd->params.config.b_gamma_table = b_gamma_table;
-}
-
-void atomisp_css_set_gamma_table(struct atomisp_sub_device *asd,
-				 struct atomisp_css_gamma_table *gamma_table)
-{
-	asd->params.config.gamma_table = gamma_table;
-}
-
 void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd,
-			       struct atomisp_css_ctc_table *ctc_table)
+			       struct ia_css_ctc_table *ctc_table)
 {
 	int i;
 	u16 *vamem_ptr = ctc_table->data.vamem_1;
@@ -3577,29 +3258,17 @@ void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd,
 }
 
 void atomisp_css_set_anr_thres(struct atomisp_sub_device *asd,
-			       struct atomisp_css_anr_thres *anr_thres)
+			       struct ia_css_anr_thres *anr_thres)
 {
 	asd->params.config.anr_thres = anr_thres;
 }
 
 void atomisp_css_set_dvs_6axis(struct atomisp_sub_device *asd,
-			       struct atomisp_css_dvs_6axis *dvs_6axis)
+			       struct ia_css_dvs_6axis_config *dvs_6axis)
 {
 	asd->params.config.dvs_6axis_config = dvs_6axis;
 }
 
-void atomisp_css_set_gc_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_gc_config *gc_config)
-{
-	asd->params.config.gc_config = gc_config;
-}
-
-void atomisp_css_set_3a_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_3a_config *s3a_config)
-{
-	asd->params.config.s3a_config = s3a_config;
-}
-
 void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd,
 				      struct atomisp_dis_vector *vector)
 {
@@ -3615,7 +3284,7 @@ void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd,
 static int atomisp_compare_dvs_grid(struct atomisp_sub_device *asd,
 				    struct atomisp_dvs_grid_info *atomgrid)
 {
-	struct atomisp_css_dvs_grid_info *cur =
+	struct ia_css_dvs_grid_info *cur =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 
 	if (!cur) {
@@ -3696,8 +3365,8 @@ int atomisp_css_set_dis_coefs(struct atomisp_sub_device *asd,
 		return -EFAULT;
 
 	asd->params.css_param.update_flag.dvs2_coefs =
-	    (struct atomisp_dvs2_coefficients *)
-	    asd->params.css_param.dvs2_coeff;
+		(struct atomisp_dis_coefficients *)
+		asd->params.css_param.dvs2_coeff;
 	/* FIXME! */
 	/*	asd->params.dis_proj_data_valid = false; */
 	asd->params.css_update_params_needed = true;
@@ -3727,7 +3396,7 @@ void atomisp_css_set_zoom_factor(struct atomisp_sub_device *asd,
 }
 
 void atomisp_css_set_formats_config(struct atomisp_sub_device *asd,
-				    struct atomisp_css_formats_config *formats_config)
+				    struct ia_css_formats_config *formats_config)
 {
 	asd->params.config.formats_config = formats_config;
 }
@@ -3735,7 +3404,7 @@ void atomisp_css_set_formats_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_wb_config(struct atomisp_sub_device *asd,
 			      struct atomisp_wb_config *config)
 {
-	struct atomisp_css_wb_config wb_config;
+	struct ia_css_wb_config wb_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3744,7 +3413,7 @@ int atomisp_css_get_wb_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&wb_config, 0, sizeof(struct atomisp_css_wb_config));
+	memset(&wb_config, 0, sizeof(struct ia_css_wb_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.wb_config = &wb_config;
 	ia_css_stream_get_isp_config(
@@ -3758,7 +3427,7 @@ int atomisp_css_get_wb_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_ob_config(struct atomisp_sub_device *asd,
 			      struct atomisp_ob_config *config)
 {
-	struct atomisp_css_ob_config ob_config;
+	struct ia_css_ob_config ob_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3767,7 +3436,7 @@ int atomisp_css_get_ob_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&ob_config, 0, sizeof(struct atomisp_css_ob_config));
+	memset(&ob_config, 0, sizeof(struct ia_css_ob_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.ob_config = &ob_config;
 	ia_css_stream_get_isp_config(
@@ -3781,7 +3450,7 @@ int atomisp_css_get_ob_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_dp_config(struct atomisp_sub_device *asd,
 			      struct atomisp_dp_config *config)
 {
-	struct atomisp_css_dp_config dp_config;
+	struct ia_css_dp_config dp_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3790,7 +3459,7 @@ int atomisp_css_get_dp_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&dp_config, 0, sizeof(struct atomisp_css_dp_config));
+	memset(&dp_config, 0, sizeof(struct ia_css_dp_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.dp_config = &dp_config;
 	ia_css_stream_get_isp_config(
@@ -3804,7 +3473,7 @@ int atomisp_css_get_dp_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_de_config(struct atomisp_sub_device *asd,
 			      struct atomisp_de_config *config)
 {
-	struct atomisp_css_de_config de_config;
+	struct ia_css_de_config de_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3813,7 +3482,7 @@ int atomisp_css_get_de_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&de_config, 0, sizeof(struct atomisp_css_de_config));
+	memset(&de_config, 0, sizeof(struct ia_css_de_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.de_config = &de_config;
 	ia_css_stream_get_isp_config(
@@ -3827,7 +3496,7 @@ int atomisp_css_get_de_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_nr_config(struct atomisp_sub_device *asd,
 			      struct atomisp_nr_config *config)
 {
-	struct atomisp_css_nr_config nr_config;
+	struct ia_css_nr_config nr_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3836,7 +3505,7 @@ int atomisp_css_get_nr_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&nr_config, 0, sizeof(struct atomisp_css_nr_config));
+	memset(&nr_config, 0, sizeof(struct ia_css_nr_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 
 	isp_config.nr_config = &nr_config;
@@ -3851,7 +3520,7 @@ int atomisp_css_get_nr_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_ee_config(struct atomisp_sub_device *asd,
 			      struct atomisp_ee_config *config)
 {
-	struct atomisp_css_ee_config ee_config;
+	struct ia_css_ee_config ee_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3860,7 +3529,7 @@ int atomisp_css_get_ee_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&ee_config, 0, sizeof(struct atomisp_css_ee_config));
+	memset(&ee_config, 0, sizeof(struct ia_css_ee_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.ee_config = &ee_config;
 	ia_css_stream_get_isp_config(
@@ -3874,7 +3543,7 @@ int atomisp_css_get_ee_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd,
 			       struct atomisp_tnr_config *config)
 {
-	struct atomisp_css_tnr_config tnr_config;
+	struct ia_css_tnr_config tnr_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3883,7 +3552,7 @@ int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&tnr_config, 0, sizeof(struct atomisp_css_tnr_config));
+	memset(&tnr_config, 0, sizeof(struct ia_css_tnr_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.tnr_config = &tnr_config;
 	ia_css_stream_get_isp_config(
@@ -3897,7 +3566,7 @@ int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd,
 			      struct atomisp_ctc_table *config)
 {
-	struct atomisp_css_ctc_table *tab;
+	struct ia_css_ctc_table *tab;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3907,7 +3576,7 @@ int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd,
 		return -EINVAL;
 	}
 
-	tab = vzalloc(sizeof(struct atomisp_css_ctc_table));
+	tab = vzalloc(sizeof(struct ia_css_ctc_table));
 	if (!tab)
 		return -ENOMEM;
 
@@ -3925,7 +3594,7 @@ int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd,
 int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd,
 				struct atomisp_gamma_table *config)
 {
-	struct atomisp_css_gamma_table *tab;
+	struct ia_css_gamma_table *tab;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3935,7 +3604,7 @@ int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd,
 		return -EINVAL;
 	}
 
-	tab = vzalloc(sizeof(struct atomisp_css_gamma_table));
+	tab = vzalloc(sizeof(struct ia_css_gamma_table));
 	if (!tab)
 		return -ENOMEM;
 
@@ -3953,7 +3622,7 @@ int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd,
 int atomisp_css_get_gc_config(struct atomisp_sub_device *asd,
 			      struct atomisp_gc_config *config)
 {
-	struct atomisp_css_gc_config gc_config;
+	struct ia_css_gc_config gc_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3962,7 +3631,7 @@ int atomisp_css_get_gc_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&gc_config, 0, sizeof(struct atomisp_css_gc_config));
+	memset(&gc_config, 0, sizeof(struct ia_css_gc_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.gc_config = &gc_config;
 	ia_css_stream_get_isp_config(
@@ -3977,7 +3646,7 @@ int atomisp_css_get_gc_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_3a_config(struct atomisp_sub_device *asd,
 			      struct atomisp_3a_config *config)
 {
-	struct atomisp_css_3a_config s3a_config;
+	struct ia_css_3a_config s3a_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -3986,7 +3655,7 @@ int atomisp_css_get_3a_config(struct atomisp_sub_device *asd,
 			__func__);
 		return -EINVAL;
 	}
-	memset(&s3a_config, 0, sizeof(struct atomisp_css_3a_config));
+	memset(&s3a_config, 0, sizeof(struct ia_css_3a_config));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.s3a_config = &s3a_config;
 	ia_css_stream_get_isp_config(
@@ -4001,7 +3670,7 @@ int atomisp_css_get_3a_config(struct atomisp_sub_device *asd,
 int atomisp_css_get_formats_config(struct atomisp_sub_device *asd,
 				   struct atomisp_formats_config *config)
 {
-	struct atomisp_css_formats_config formats_config;
+	struct ia_css_formats_config formats_config;
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
 
@@ -4139,37 +3808,37 @@ int atomisp_css_get_dis_stat(struct atomisp_sub_device *asd,
 	return 0;
 }
 
-struct atomisp_css_shading_table *atomisp_css_shading_table_alloc(
+struct ia_css_shading_table *atomisp_css_shading_table_alloc(
     unsigned int width, unsigned int height)
 {
 	return ia_css_shading_table_alloc(width, height);
 }
 
 void atomisp_css_set_shading_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_shading_table *table)
+				   struct ia_css_shading_table *table)
 {
 	asd->params.config.shading_table = table;
 }
 
-void atomisp_css_shading_table_free(struct atomisp_css_shading_table *table)
+void atomisp_css_shading_table_free(struct ia_css_shading_table *table)
 {
 	ia_css_shading_table_free(table);
 }
 
-struct atomisp_css_morph_table *atomisp_css_morph_table_allocate(
+struct ia_css_morph_table *atomisp_css_morph_table_allocate(
     unsigned int width, unsigned int height)
 {
 	return ia_css_morph_table_allocate(width, height);
 }
 
 void atomisp_css_set_morph_table(struct atomisp_sub_device *asd,
-				 struct atomisp_css_morph_table *table)
+				 struct ia_css_morph_table *table)
 {
 	asd->params.config.morph_table = table;
 }
 
 void atomisp_css_get_morph_table(struct atomisp_sub_device *asd,
-				 struct atomisp_css_morph_table *table)
+				 struct ia_css_morph_table *table)
 {
 	struct ia_css_isp_config isp_config;
 	struct atomisp_device *isp = asd->isp;
@@ -4179,7 +3848,7 @@ void atomisp_css_get_morph_table(struct atomisp_sub_device *asd,
 			"%s called after streamoff, skipping.\n", __func__);
 		return;
 	}
-	memset(table, 0, sizeof(struct atomisp_css_morph_table));
+	memset(table, 0, sizeof(struct ia_css_morph_table));
 	memset(&isp_config, 0, sizeof(struct ia_css_isp_config));
 	isp_config.morph_table = table;
 	ia_css_stream_get_isp_config(
@@ -4187,7 +3856,7 @@ void atomisp_css_get_morph_table(struct atomisp_sub_device *asd,
 	    &isp_config);
 }
 
-void atomisp_css_morph_table_free(struct atomisp_css_morph_table *table)
+void atomisp_css_morph_table_free(struct ia_css_morph_table *table)
 {
 	ia_css_morph_table_free(table);
 }
@@ -4215,8 +3884,8 @@ int atomisp_css_wait_acc_finish(struct atomisp_sub_device *asd)
 	if (wait_for_completion_interruptible_timeout(&asd->acc.acc_done,
 		ATOMISP_ISP_TIMEOUT_DURATION) == 0) {
 		dev_err(isp->dev, "<%s: completion timeout\n", __func__);
-		atomisp_css_debug_dump_sp_sw_debug_info();
-		atomisp_css_debug_dump_debug_info(__func__);
+		ia_css_debug_dump_sp_sw_debug_info();
+		ia_css_debug_dump_debug_info(__func__);
 		ret = -EIO;
 	}
 	rt_mutex_lock(&isp->mutex);
@@ -4244,11 +3913,11 @@ int atomisp_css_set_acc_parameters(struct atomisp_acc_fw *acc_fw)
 
 /* Load acc binary extension */
 int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd,
-				   struct atomisp_css_fw_info *fw,
-				   enum atomisp_css_pipe_id pipe_id,
+				   struct ia_css_fw_info *fw,
+				   enum ia_css_pipe_id pipe_id,
 				   unsigned int type)
 {
-	struct atomisp_css_fw_info **hd;
+	struct ia_css_fw_info **hd;
 
 	fw->next = NULL;
 	hd = &(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
@@ -4264,10 +3933,10 @@ int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd,
 
 /* Unload acc binary extension */
 void atomisp_css_unload_acc_extension(struct atomisp_sub_device *asd,
-				      struct atomisp_css_fw_info *fw,
-				      enum atomisp_css_pipe_id pipe_id)
+				      struct ia_css_fw_info *fw,
+				      enum ia_css_pipe_id pipe_id)
 {
-	struct atomisp_css_fw_info **hd;
+	struct ia_css_fw_info **hd;
 
 	hd = &(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]
 	       .pipe_configs[pipe_id].acc_extension);
@@ -4294,21 +3963,21 @@ int atomisp_css_create_acc_pipe(struct atomisp_sub_device *asd)
 	if (stream_env->acc_stream) {
 		if (stream_env->acc_stream_state == CSS_STREAM_STARTED) {
 			if (ia_css_stream_stop(stream_env->acc_stream)
-			    != IA_CSS_SUCCESS) {
+			    != 0) {
 				dev_err(isp->dev, "stop acc_stream failed.\n");
 				return -EBUSY;
 			}
 		}
 
 		if (ia_css_stream_destroy(stream_env->acc_stream)
-		    != IA_CSS_SUCCESS) {
+		    != 0) {
 			dev_err(isp->dev, "destroy acc_stream failed.\n");
 			return -EBUSY;
 		}
 		stream_env->acc_stream = NULL;
 	}
 
-	pipe_config = &stream_env->pipe_configs[CSS_PIPE_ID_ACC];
+	pipe_config = &stream_env->pipe_configs[IA_CSS_PIPE_ID_ACC];
 	ia_css_pipe_config_defaults(pipe_config);
 	asd->acc.acc_stages = kzalloc(MAX_ACC_STAGES *
 				      sizeof(void *), GFP_KERNEL);
@@ -4335,7 +4004,7 @@ int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd)
 		    &stream_env->pipe_configs[IA_CSS_PIPE_ID_ACC];
 
 	if (ia_css_pipe_create(pipe_config,
-			       &stream_env->pipes[IA_CSS_PIPE_ID_ACC]) != IA_CSS_SUCCESS) {
+			       &stream_env->pipes[IA_CSS_PIPE_ID_ACC]) != 0) {
 		dev_err(isp->dev, "%s: ia_css_pipe_create failed\n",
 			__func__);
 		return -EBADE;
@@ -4345,7 +4014,7 @@ int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd)
 	       sizeof(struct ia_css_stream_config));
 	if (ia_css_stream_create(&stream_env->acc_stream_config, 1,
 				 &stream_env->pipes[IA_CSS_PIPE_ID_ACC],
-				 &stream_env->acc_stream) != IA_CSS_SUCCESS) {
+				 &stream_env->acc_stream) != 0) {
 		dev_err(isp->dev, "%s: create acc_stream error.\n", __func__);
 		return -EINVAL;
 	}
@@ -4356,13 +4025,13 @@ int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd)
 
 	atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false);
 
-	if (ia_css_start_sp() != IA_CSS_SUCCESS) {
+	if (ia_css_start_sp()) {
 		dev_err(isp->dev, "start sp error.\n");
 		return -EIO;
 	}
 
 	if (ia_css_stream_start(stream_env->acc_stream)
-	    != IA_CSS_SUCCESS) {
+	    != 0) {
 		dev_err(isp->dev, "acc_stream start error.\n");
 		return -EIO;
 	}
@@ -4388,7 +4057,7 @@ void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd)
 		    &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL];
 	if (stream_env->acc_stream) {
 		if (ia_css_stream_destroy(stream_env->acc_stream)
-		    != IA_CSS_SUCCESS)
+		    != 0)
 			dev_warn(asd->isp->dev,
 				 "destroy acc_stream failed.\n");
 		stream_env->acc_stream = NULL;
@@ -4396,7 +4065,7 @@ void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd)
 
 	if (stream_env->pipes[IA_CSS_PIPE_ID_ACC]) {
 		if (ia_css_pipe_destroy(stream_env->pipes[IA_CSS_PIPE_ID_ACC])
-		    != IA_CSS_SUCCESS)
+		    != 0)
 			dev_warn(asd->isp->dev,
 				 "destroy ACC pipe failed.\n");
 		stream_env->pipes[IA_CSS_PIPE_ID_ACC] = NULL;
@@ -4420,7 +4089,7 @@ void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd)
 }
 
 int atomisp_css_load_acc_binary(struct atomisp_sub_device *asd,
-				struct atomisp_css_fw_info *fw,
+				struct ia_css_fw_info *fw,
 				unsigned int index)
 {
 	struct ia_css_pipe_config *pipe_config =
@@ -4494,7 +4163,7 @@ int atomisp_css_isr_thread(struct atomisp_device *isp,
 			for (i = 0; i < isp->num_of_streams; i++)
 				atomisp_wdt_stop(&isp->asd[i], 0);
 
-			if (!atomisp_hw_is_isp2401)
+			if (!IS_ISP2401)
 				atomisp_wdt(&isp->asd[0].wdt);
 			else
 				queue_work(isp->wdt_work_queue, &isp->wdt_work);
@@ -4510,7 +4179,7 @@ int atomisp_css_isr_thread(struct atomisp_device *isp,
 		asd = __get_atomisp_subdev(current_event.event.pipe,
 					   isp, &stream_id);
 		if (!asd) {
-			if (current_event.event.type == CSS_EVENT_TIMER)
+			if (current_event.event.type == IA_CSS_EVENT_TYPE_TIMER)
 				dev_dbg(isp->dev,
 					"event: Timer event.");
 			else
@@ -4522,63 +4191,72 @@ int atomisp_css_isr_thread(struct atomisp_device *isp,
 
 		atomisp_css_temp_pipe_to_pipe_id(asd, &current_event);
 		switch (current_event.event.type) {
-		case CSS_EVENT_OUTPUT_FRAME_DONE:
+		case IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE:
+			dev_dbg(isp->dev, "event: Output frame done");
 			frame_done_found[asd->index] = true;
-			atomisp_buf_done(asd, 0, CSS_BUFFER_TYPE_OUTPUT_FRAME,
+			atomisp_buf_done(asd, 0, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
 
-			if (!atomisp_hw_is_isp2401)
+			if (!IS_ISP2401)
 				reset_wdt_timer[asd->index] = true; /* ISP running */
 
 			break;
-		case CSS_EVENT_SEC_OUTPUT_FRAME_DONE:
+		case IA_CSS_EVENT_TYPE_SECOND_OUTPUT_FRAME_DONE:
+			dev_dbg(isp->dev, "event: Second output frame done");
 			frame_done_found[asd->index] = true;
-			atomisp_buf_done(asd, 0, CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME,
+			atomisp_buf_done(asd, 0, IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
 
-			if (!atomisp_hw_is_isp2401)
+			if (!IS_ISP2401)
 				reset_wdt_timer[asd->index] = true; /* ISP running */
 
 			break;
-		case CSS_EVENT_3A_STATISTICS_DONE:
+		case IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE:
+			dev_dbg(isp->dev, "event: 3A stats frame done");
 			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_3A_STATISTICS,
+					 IA_CSS_BUFFER_TYPE_3A_STATISTICS,
 					 current_event.pipe,
 					 false, stream_id);
 			break;
-		case CSS_EVENT_METADATA_DONE:
+		case IA_CSS_EVENT_TYPE_METADATA_DONE:
+			dev_dbg(isp->dev, "event: metadata frame done");
 			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_METADATA,
+					 IA_CSS_BUFFER_TYPE_METADATA,
 					 current_event.pipe,
 					 false, stream_id);
 			break;
-		case CSS_EVENT_VF_OUTPUT_FRAME_DONE:
+		case IA_CSS_EVENT_TYPE_VF_OUTPUT_FRAME_DONE:
+			dev_dbg(isp->dev, "event: VF output frame done");
 			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_VF_OUTPUT_FRAME,
+					 IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
 
-			if (!atomisp_hw_is_isp2401)
+			if (!IS_ISP2401)
 				reset_wdt_timer[asd->index] = true; /* ISP running */
 
 			break;
-		case CSS_EVENT_SEC_VF_OUTPUT_FRAME_DONE:
+		case IA_CSS_EVENT_TYPE_SECOND_VF_OUTPUT_FRAME_DONE:
+			dev_dbg(isp->dev, "event: second VF output frame done");
 			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME,
+					 IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME,
 					 current_event.pipe, true, stream_id);
-			if (!atomisp_hw_is_isp2401)
+			if (!IS_ISP2401)
 				reset_wdt_timer[asd->index] = true; /* ISP running */
 
 			break;
-		case CSS_EVENT_DIS_STATISTICS_DONE:
+		case IA_CSS_EVENT_TYPE_DIS_STATISTICS_DONE:
+			dev_dbg(isp->dev, "event: dis stats frame done");
 			atomisp_buf_done(asd, 0,
-					 CSS_BUFFER_TYPE_DIS_STATISTICS,
+					 IA_CSS_BUFFER_TYPE_DIS_STATISTICS,
 					 current_event.pipe,
 					 false, stream_id);
 			break;
-		case CSS_EVENT_PIPELINE_DONE:
+		case IA_CSS_EVENT_TYPE_PIPELINE_DONE:
+			dev_dbg(isp->dev, "event: pipeline done");
 			css_pipe_done[asd->index] = true;
 			break;
-		case CSS_EVENT_ACC_STAGE_COMPLETE:
+		case IA_CSS_EVENT_TYPE_ACC_STAGE_COMPLETE:
+			dev_dbg(isp->dev, "event: acc stage done");
 			atomisp_acc_done(asd, current_event.event.fw_handle);
 			break;
 		default:
@@ -4588,7 +4266,7 @@ int atomisp_css_isr_thread(struct atomisp_device *isp,
 		}
 	}
 
-	if (atomisp_hw_is_isp2401)
+	if (IS_ISP2401)
 		return 0;
 
 	/* ISP2400: If there are no buffers queued then delete wdt timer. */
@@ -4618,8 +4296,13 @@ bool atomisp_css_valid_sof(struct atomisp_device *isp)
 		struct atomisp_sub_device *asd = &isp->asd[i];
 		/* Loop for each css vc stream */
 		for (j = 0; j < ATOMISP_INPUT_STREAM_NUM; j++) {
-			if (asd->stream_env[j].stream &&
-			    asd->stream_env[j].stream_config.mode ==
+			if (!asd->stream_env[j].stream)
+				continue;
+
+			dev_dbg(isp->dev,
+				"stream #%d: mode: %d\n", j,
+				asd->stream_env[j].stream_config.mode);
+			if (asd->stream_env[j].stream_config.mode ==
 			    IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
 				return false;
 		}
@@ -4640,6 +4323,20 @@ int atomisp_css_dump_sp_raw_copy_linecount(bool reduced)
 	return 0;
 }
 
+static const char * const fw_type_name[] = {
+	[ia_css_sp_firmware]		= "SP",
+	[ia_css_isp_firmware]		= "ISP",
+	[ia_css_bootloader_firmware]	= "BootLoader",
+	[ia_css_acc_firmware]		= "accel",
+};
+
+static const char * const fw_acc_type_name[] = {
+	[IA_CSS_ACC_NONE] =		"Normal",
+	[IA_CSS_ACC_OUTPUT] =		"Accel stage on output",
+	[IA_CSS_ACC_VIEWFINDER] =	"Accel stage on viewfinder",
+	[IA_CSS_ACC_STANDALONE] =	"Stand-alone acceleration",
+};
+
 int atomisp_css_dump_blob_infor(void)
 {
 	struct ia_css_blob_descr *bd = sh_css_blob_info;
@@ -4650,9 +4347,28 @@ int atomisp_css_dump_blob_infor(void)
 	if (!bd)
 		return -EPERM;
 
-	for (i = 1; i < sh_css_num_binaries; i++)
-		dev_dbg(atomisp_dev, "Num%d binary id is %d, name is %s\n", i,
-			bd[i - 1].header.info.isp.sp.id, bd[i - 1].name);
+	/*
+	 * The sh_css_load_firmware function discard the initial
+	 * "SPS" binaries
+	 */
+	for (i = 0; i < sh_css_num_binaries - NUM_OF_SPS; i++) {
+		switch (bd[i].header.type) {
+		case ia_css_isp_firmware:
+			dev_dbg(atomisp_dev,
+				"Num%2d type %s (%s), binary id is %2d, name is %s\n",
+				i + NUM_OF_SPS,
+				fw_type_name[bd[i].header.type],
+				fw_acc_type_name[bd[i].header.info.isp.type],
+				bd[i].header.info.isp.sp.id,
+				bd[i].name);
+			break;
+		default:
+			dev_dbg(atomisp_dev,
+				"Num%2d type %s, name is %s\n",
+				i + NUM_OF_SPS, fw_type_name[bd[i].header.type],
+				bd[i].name);
+		}
+	}
 
 	return 0;
 }
@@ -4664,7 +4380,7 @@ void atomisp_css_set_isp_config_id(struct atomisp_sub_device *asd,
 }
 
 void atomisp_css_set_isp_config_applied_frame(struct atomisp_sub_device *asd,
-	struct atomisp_css_frame *output_frame)
+	struct ia_css_frame *output_frame)
 {
 	asd->params.config.output_frame = output_frame;
 }
@@ -4692,8 +4408,8 @@ void atomisp_en_dz_capt_pipe(struct atomisp_sub_device *asd, bool enable)
 	    enable);
 }
 
-struct atomisp_css_dvs_grid_info *atomisp_css_get_dvs_grid_info(
-    struct atomisp_css_grid_info *grid_info)
+struct ia_css_dvs_grid_info *atomisp_css_get_dvs_grid_info(
+    struct ia_css_grid_info *grid_info)
 {
 	if (!grid_info)
 		return NULL;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h
index 7abd1ff35652..8376aec18e3e 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Clovertrail PNW Camera Imaging ISP subsystem.
  *
@@ -30,97 +31,6 @@
 #define ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES_LOCK_EN     4
 #define ATOMISP_CSS2_NUM_DVS_FRAME_DELAY     2
 
-#define atomisp_css_pipe_id ia_css_pipe_id
-#define atomisp_css_pipeline	ia_css_pipe
-#define atomisp_css_buffer_type ia_css_buffer_type
-#define atomisp_css_dis_data ia_css_isp_dvs_statistics
-#define atomisp_css_irq_info  ia_css_irq_info
-#define atomisp_css_isp_config ia_css_isp_config
-#define atomisp_css_bayer_order ia_css_bayer_order
-#define atomisp_css_capture_mode ia_css_capture_mode
-#define atomisp_css_input_mode ia_css_input_mode
-#define atomisp_css_frame ia_css_frame
-#define atomisp_css_frame_format ia_css_frame_format
-#define atomisp_css_frame_info ia_css_frame_info
-#define atomisp_css_dp_config	ia_css_dp_config
-#define atomisp_css_wb_config	ia_css_wb_config
-#define atomisp_css_cc_config	ia_css_cc_config
-#define atomisp_css_nr_config	ia_css_nr_config
-#define atomisp_css_ee_config	ia_css_ee_config
-#define atomisp_css_ob_config	ia_css_ob_config
-#define atomisp_css_de_config	ia_css_de_config
-#define atomisp_css_dz_config	ia_css_dz_config
-#define atomisp_css_ce_config	ia_css_ce_config
-#define atomisp_css_gc_config	ia_css_gc_config
-#define atomisp_css_tnr_config	ia_css_tnr_config
-#define atomisp_css_cnr_config	ia_css_cnr_config
-#define atomisp_css_ctc_config	ia_css_ctc_config
-#define atomisp_css_3a_config	ia_css_3a_config
-#define atomisp_css_ecd_config	ia_css_ecd_config
-#define atomisp_css_ynr_config	ia_css_ynr_config
-#define atomisp_css_fc_config	ia_css_fc_config
-#define atomisp_css_aa_config	ia_css_aa_config
-#define atomisp_css_baa_config	ia_css_aa_config
-#define atomisp_css_anr_config	ia_css_anr_config
-#define atomisp_css_xnr_config	ia_css_xnr_config
-#define atomisp_css_macc_config	ia_css_macc_config
-#define atomisp_css_gamma_table	ia_css_gamma_table
-#define atomisp_css_ctc_table	ia_css_ctc_table
-#define atomisp_css_macc_table	ia_css_macc_table
-#define atomisp_css_xnr_table	ia_css_xnr_table
-#define atomisp_css_rgb_gamma_table	ia_css_rgb_gamma_table
-#define atomisp_css_anr_thres	ia_css_anr_thres
-#define atomisp_css_dvs_6axis	ia_css_dvs_6axis_config
-#define atomisp_css_grid_info	ia_css_grid_info
-#define atomisp_css_3a_grid_info	ia_css_3a_grid_info
-#define atomisp_css_dvs_grid_info	ia_css_dvs_grid_info
-#define atomisp_css_shading_table	ia_css_shading_table
-#define atomisp_css_morph_table	ia_css_morph_table
-#define atomisp_css_dvs_6axis_config	ia_css_dvs_6axis_config
-#define atomisp_css_fw_info	ia_css_fw_info
-#define atomisp_css_formats_config	ia_css_formats_config
-
-#define CSS_PIPE_ID_PREVIEW	IA_CSS_PIPE_ID_PREVIEW
-#define CSS_PIPE_ID_COPY	IA_CSS_PIPE_ID_COPY
-#define CSS_PIPE_ID_VIDEO	IA_CSS_PIPE_ID_VIDEO
-#define CSS_PIPE_ID_CAPTURE	IA_CSS_PIPE_ID_CAPTURE
-#define CSS_PIPE_ID_ACC		IA_CSS_PIPE_ID_ACC
-#define CSS_PIPE_ID_YUVPP	IA_CSS_PIPE_ID_YUVPP
-#define CSS_PIPE_ID_NUM		IA_CSS_PIPE_ID_NUM
-
-#define CSS_INPUT_MODE_SENSOR	IA_CSS_INPUT_MODE_BUFFERED_SENSOR
-#define CSS_INPUT_MODE_FIFO	IA_CSS_INPUT_MODE_FIFO
-#define CSS_INPUT_MODE_TPG	IA_CSS_INPUT_MODE_TPG
-#define CSS_INPUT_MODE_PRBS	IA_CSS_INPUT_MODE_PRBS
-#define CSS_INPUT_MODE_MEMORY	IA_CSS_INPUT_MODE_MEMORY
-
-#define CSS_IRQ_INFO_CSS_RECEIVER_ERROR	IA_CSS_IRQ_INFO_CSS_RECEIVER_ERROR
-#define CSS_IRQ_INFO_EVENTS_READY	IA_CSS_IRQ_INFO_EVENTS_READY
-#define CSS_IRQ_INFO_INPUT_SYSTEM_ERROR \
-	IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR
-#define CSS_IRQ_INFO_IF_ERROR	IA_CSS_IRQ_INFO_IF_ERROR
-
-#define CSS_BUFFER_TYPE_NUM	IA_CSS_BUFFER_TYPE_NUM
-
-#define CSS_FRAME_FLASH_STATE_NONE	IA_CSS_FRAME_FLASH_STATE_NONE
-#define CSS_FRAME_FLASH_STATE_PARTIAL	IA_CSS_FRAME_FLASH_STATE_PARTIAL
-#define CSS_FRAME_FLASH_STATE_FULL	IA_CSS_FRAME_FLASH_STATE_FULL
-
-#define CSS_BAYER_ORDER_GRBG	IA_CSS_BAYER_ORDER_GRBG
-#define CSS_BAYER_ORDER_RGGB	IA_CSS_BAYER_ORDER_RGGB
-#define CSS_BAYER_ORDER_BGGR	IA_CSS_BAYER_ORDER_BGGR
-#define CSS_BAYER_ORDER_GBRG	IA_CSS_BAYER_ORDER_GBRG
-
-/*
- * Hide IA_ naming difference in otherwise common CSS macros.
- */
-#define CSS_ID(val)	(IA_ ## val)
-#define CSS_EVENT(val)	(IA_CSS_EVENT_TYPE_ ## val)
-#define CSS_FORMAT(val)	(ATOMISP_INPUT_FORMAT_ ## val)
-
-#define CSS_EVENT_PORT_EOF	CSS_EVENT(PORT_EOF)
-#define CSS_EVENT_FRAME_TAGGED	CSS_EVENT(FRAME_TAGGED)
-
 #define CSS_MIPI_FRAME_BUFFER_SIZE_1	0x60000
 #define CSS_MIPI_FRAME_BUFFER_SIZE_2	0x80000
 
@@ -181,7 +91,7 @@ struct atomisp_s3a_buf {
 };
 
 struct atomisp_dis_buf {
-	struct atomisp_css_dis_data *dis_data;
+	struct ia_css_isp_dvs_statistics *dis_data;
 	struct ia_css_isp_dvs_statistics_map *dvs_map;
 	struct list_head list;
 };
@@ -191,71 +101,53 @@ struct atomisp_css_buffer {
 };
 
 struct atomisp_css_event {
-	enum atomisp_css_pipe_id pipe;
+	enum ia_css_pipe_id pipe;
 	struct ia_css_event event;
 };
 
 void atomisp_css_set_macc_config(struct atomisp_sub_device *asd,
-				 struct atomisp_css_macc_config *macc_config);
+				 struct ia_css_macc_config *macc_config);
 
 void atomisp_css_set_ecd_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_ecd_config *ecd_config);
+				struct ia_css_ecd_config *ecd_config);
 
 void atomisp_css_set_ynr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_ynr_config *ynr_config);
+				struct ia_css_ynr_config *ynr_config);
 
 void atomisp_css_set_fc_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_fc_config *fc_config);
+			       struct ia_css_fc_config *fc_config);
 
 void atomisp_css_set_aa_config(struct atomisp_sub_device *asd,
-			       struct atomisp_css_aa_config *aa_config);
+			       struct ia_css_aa_config *aa_config);
 
 void atomisp_css_set_baa_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_baa_config *baa_config);
+				struct ia_css_aa_config *baa_config);
 
 void atomisp_css_set_anr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_anr_config *anr_config);
+				struct ia_css_anr_config *anr_config);
 
 void atomisp_css_set_xnr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_xnr_config *xnr_config);
+				struct ia_css_xnr_config *xnr_config);
 
 void atomisp_css_set_cnr_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_cnr_config *cnr_config);
+				struct ia_css_cnr_config *cnr_config);
 
 void atomisp_css_set_ctc_config(struct atomisp_sub_device *asd,
-				struct atomisp_css_ctc_config *ctc_config);
+				struct ia_css_ctc_config *ctc_config);
 
 void atomisp_css_set_yuv2rgb_cc_config(struct atomisp_sub_device *asd,
-				       struct atomisp_css_cc_config *yuv2rgb_cc_config);
+				       struct ia_css_cc_config *yuv2rgb_cc_config);
 
 void atomisp_css_set_rgb2yuv_cc_config(struct atomisp_sub_device *asd,
-				       struct atomisp_css_cc_config *rgb2yuv_cc_config);
-
-void atomisp_css_set_xnr_table(struct atomisp_sub_device *asd,
-			       struct atomisp_css_xnr_table *xnr_table);
-
-void atomisp_css_set_r_gamma_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_rgb_gamma_table *r_gamma_table);
-
-void atomisp_css_set_g_gamma_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_rgb_gamma_table *g_gamma_table);
-
-void atomisp_css_set_b_gamma_table(struct atomisp_sub_device *asd,
-				   struct atomisp_css_rgb_gamma_table *b_gamma_table);
+				       struct ia_css_cc_config *rgb2yuv_cc_config);
 
 void atomisp_css_set_anr_thres(struct atomisp_sub_device *asd,
-			       struct atomisp_css_anr_thres *anr_thres);
-
-int atomisp_css_check_firmware_version(struct atomisp_device *isp);
+			       struct ia_css_anr_thres *anr_thres);
 
 int atomisp_css_load_firmware(struct atomisp_device *isp);
 
-void atomisp_css_unload_firmware(struct atomisp_device *isp);
-
 void atomisp_css_set_dvs_6axis(struct atomisp_sub_device *asd,
-			       struct atomisp_css_dvs_6axis *dvs_6axis);
-
-unsigned int atomisp_css_debug_get_dtrace_level(void);
+			       struct ia_css_dvs_6axis_config *dvs_6axis);
 
 int atomisp_css_debug_dump_isp_binary(void);
 
@@ -267,11 +159,11 @@ void atomisp_css_set_isp_config_id(struct atomisp_sub_device *asd,
 				   uint32_t isp_config_id);
 
 void atomisp_css_set_isp_config_applied_frame(struct atomisp_sub_device *asd,
-	struct atomisp_css_frame *output_frame);
+	struct ia_css_frame *output_frame);
 
 int atomisp_get_css_dbgfunc(void);
 
 int atomisp_set_css_dbgfunc(struct atomisp_device *isp, int opt);
-struct atomisp_css_dvs_grid_info *atomisp_css_get_dvs_grid_info(
-    struct atomisp_css_grid_info *grid_info);
+struct ia_css_dvs_grid_info *atomisp_css_get_dvs_grid_info(
+    struct ia_css_grid_info *grid_info);
 #endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c
index 3079043f1fac..fa5918270614 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  *
@@ -24,66 +25,61 @@
 #include "atomisp_ioctl.h"
 #include "atomisp_compat_ioctl32.h"
 
-static int get_atomisp_histogram32(struct atomisp_histogram *kp,
+/* Macro borrowed from v4l2-compat-ioctl32.c */
+/* Use the same argument order as copy_in_user */
+#define assign_in_user(to, from)				\
+({								\
+	typeof(*from) __assign_tmp;				\
+								\
+	get_user(__assign_tmp, from) || put_user(__assign_tmp, to);	\
+})
+
+
+static int get_atomisp_histogram32(struct atomisp_histogram __user *kp,
 				   struct atomisp_histogram32 __user *up)
 {
 	compat_uptr_t tmp;
 
 	if (!access_ok(up, sizeof(struct atomisp_histogram32)) ||
-	    get_user(kp->num_elements, &up->num_elements) ||
-	    get_user(tmp, &up->data))
+	    assign_in_user(&kp->num_elements, &up->num_elements) ||
+	    get_user(tmp, &up->data) ||
+	    put_user(compat_ptr(tmp), &kp->data))
 		return -EFAULT;
 
-	kp->data = compat_ptr(tmp);
 	return 0;
 }
 
-static int put_atomisp_histogram32(struct atomisp_histogram *kp,
+static int put_atomisp_histogram32(struct atomisp_histogram __user *kp,
 				   struct atomisp_histogram32 __user *up)
 {
-	compat_uptr_t tmp = (compat_uptr_t)((uintptr_t)kp->data);
+	void __user *tmp;
 
 	if (!access_ok(up, sizeof(struct atomisp_histogram32)) ||
-	    put_user(kp->num_elements, &up->num_elements) ||
-	    put_user(tmp, &up->data))
+	    assign_in_user(&up->num_elements, &kp->num_elements) ||
+	    get_user(tmp, &kp->data) ||
+	    put_user(ptr_to_compat(tmp), &up->data))
 		return -EFAULT;
 
 	return 0;
 }
 
-static inline int get_v4l2_pix_format(struct v4l2_pix_format *kp,
-				      struct v4l2_pix_format __user *up)
-{
-	if (copy_from_user(kp, up, sizeof(struct v4l2_pix_format)))
-		return -EFAULT;
-	return 0;
-}
-
-static inline int put_v4l2_pix_format(struct v4l2_pix_format *kp,
-				      struct v4l2_pix_format __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_pix_format)))
-		return -EFAULT;
-	return 0;
-}
-
-static int get_v4l2_framebuffer32(struct v4l2_framebuffer *kp,
-				  struct v4l2_framebuffer32 __user *up)
+static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp,
+					struct v4l2_framebuffer32 __user *up)
 {
 	compat_uptr_t tmp;
 
 	if (!access_ok(up, sizeof(struct v4l2_framebuffer32)) ||
 	    get_user(tmp, &up->base) ||
-	    get_user(kp->capability, &up->capability) ||
-	    get_user(kp->flags, &up->flags))
+	    put_user(compat_ptr(tmp), &kp->base) ||
+	    assign_in_user(&kp->capability, &up->capability) ||
+	    assign_in_user(&kp->flags, &up->flags) ||
+	    copy_in_user(&kp->fmt, &up->fmt, sizeof(kp->fmt)))
 		return -EFAULT;
 
-	kp->base = (void __force *)compat_ptr(tmp);
-	get_v4l2_pix_format((struct v4l2_pix_format *)&kp->fmt, &up->fmt);
 	return 0;
 }
 
-static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp,
+static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics __user *kp,
 					struct atomisp_dis_statistics32 __user *up)
 {
 	compat_uptr_t hor_prod_odd_real;
@@ -96,67 +92,99 @@ static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp,
 	compat_uptr_t ver_prod_even_imag;
 
 	if (!access_ok(up, sizeof(struct atomisp_dis_statistics32)) ||
-	    copy_from_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) ||
-	    get_user(hor_prod_odd_real, &up->dvs2_stat.hor_prod.odd_real) ||
-	    get_user(hor_prod_odd_imag, &up->dvs2_stat.hor_prod.odd_imag) ||
-	    get_user(hor_prod_even_real, &up->dvs2_stat.hor_prod.even_real) ||
-	    get_user(hor_prod_even_imag, &up->dvs2_stat.hor_prod.even_imag) ||
-	    get_user(ver_prod_odd_real, &up->dvs2_stat.ver_prod.odd_real) ||
-	    get_user(ver_prod_odd_imag, &up->dvs2_stat.ver_prod.odd_imag) ||
-	    get_user(ver_prod_even_real, &up->dvs2_stat.ver_prod.even_real) ||
-	    get_user(ver_prod_even_imag, &up->dvs2_stat.ver_prod.even_imag) ||
-	    get_user(kp->exp_id, &up->exp_id))
+	    copy_in_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) ||
+	    get_user(hor_prod_odd_real,
+		     &up->dvs2_stat.hor_prod.odd_real) ||
+	    get_user(hor_prod_odd_imag,
+		     &up->dvs2_stat.hor_prod.odd_imag) ||
+	    get_user(hor_prod_even_real,
+		     &up->dvs2_stat.hor_prod.even_real) ||
+	    get_user(hor_prod_even_imag,
+		     &up->dvs2_stat.hor_prod.even_imag) ||
+	    get_user(ver_prod_odd_real,
+		     &up->dvs2_stat.ver_prod.odd_real) ||
+	    get_user(ver_prod_odd_imag,
+		     &up->dvs2_stat.ver_prod.odd_imag) ||
+	    get_user(ver_prod_even_real,
+		     &up->dvs2_stat.ver_prod.even_real) ||
+	    get_user(ver_prod_even_imag,
+		     &up->dvs2_stat.ver_prod.even_imag) ||
+	    assign_in_user(&kp->exp_id, &up->exp_id) ||
+	    put_user(compat_ptr(hor_prod_odd_real),
+		     &kp->dvs2_stat.hor_prod.odd_real) ||
+	    put_user(compat_ptr(hor_prod_odd_imag),
+		     &kp->dvs2_stat.hor_prod.odd_imag) ||
+	    put_user(compat_ptr(hor_prod_even_real),
+		     &kp->dvs2_stat.hor_prod.even_real) ||
+	    put_user(compat_ptr(hor_prod_even_imag),
+		     &kp->dvs2_stat.hor_prod.even_imag) ||
+	    put_user(compat_ptr(ver_prod_odd_real),
+		     &kp->dvs2_stat.ver_prod.odd_real) ||
+	    put_user(compat_ptr(ver_prod_odd_imag),
+		     &kp->dvs2_stat.ver_prod.odd_imag) ||
+	    put_user(compat_ptr(ver_prod_even_real),
+		     &kp->dvs2_stat.ver_prod.even_real) ||
+	    put_user(compat_ptr(ver_prod_even_imag),
+		     &kp->dvs2_stat.ver_prod.even_imag))
 		return -EFAULT;
 
-	kp->dvs2_stat.hor_prod.odd_real = compat_ptr(hor_prod_odd_real);
-	kp->dvs2_stat.hor_prod.odd_imag = compat_ptr(hor_prod_odd_imag);
-	kp->dvs2_stat.hor_prod.even_real = compat_ptr(hor_prod_even_real);
-	kp->dvs2_stat.hor_prod.even_imag = compat_ptr(hor_prod_even_imag);
-	kp->dvs2_stat.ver_prod.odd_real = compat_ptr(ver_prod_odd_real);
-	kp->dvs2_stat.ver_prod.odd_imag = compat_ptr(ver_prod_odd_imag);
-	kp->dvs2_stat.ver_prod.even_real = compat_ptr(ver_prod_even_real);
-	kp->dvs2_stat.ver_prod.even_imag = compat_ptr(ver_prod_even_imag);
 	return 0;
 }
 
-static int put_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp,
+static int put_atomisp_dis_statistics32(struct atomisp_dis_statistics __user *kp,
 					struct atomisp_dis_statistics32 __user *up)
 {
-	compat_uptr_t hor_prod_odd_real =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.odd_real);
-	compat_uptr_t hor_prod_odd_imag =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.odd_imag);
-	compat_uptr_t hor_prod_even_real =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.even_real);
-	compat_uptr_t hor_prod_even_imag =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.even_imag);
-	compat_uptr_t ver_prod_odd_real =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.odd_real);
-	compat_uptr_t ver_prod_odd_imag =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.odd_imag);
-	compat_uptr_t ver_prod_even_real =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.even_real);
-	compat_uptr_t ver_prod_even_imag =
-	    (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.even_imag);
-
-	if (!access_ok(up, sizeof(struct atomisp_dis_statistics32)) ||
-	    copy_to_user(up, kp, sizeof(struct atomisp_dvs_grid_info)) ||
-	    put_user(hor_prod_odd_real, &up->dvs2_stat.hor_prod.odd_real) ||
-	    put_user(hor_prod_odd_imag, &up->dvs2_stat.hor_prod.odd_imag) ||
-	    put_user(hor_prod_even_real, &up->dvs2_stat.hor_prod.even_real) ||
-	    put_user(hor_prod_even_imag, &up->dvs2_stat.hor_prod.even_imag) ||
-	    put_user(ver_prod_odd_real, &up->dvs2_stat.ver_prod.odd_real) ||
-	    put_user(ver_prod_odd_imag, &up->dvs2_stat.ver_prod.odd_imag) ||
-	    put_user(ver_prod_even_real, &up->dvs2_stat.ver_prod.even_real) ||
-	    put_user(ver_prod_even_imag, &up->dvs2_stat.ver_prod.even_imag) ||
-	    put_user(kp->exp_id, &up->exp_id))
+	void __user *hor_prod_odd_real;
+	void __user *hor_prod_odd_imag;
+	void __user *hor_prod_even_real;
+	void __user *hor_prod_even_imag;
+	void __user *ver_prod_odd_real;
+	void __user *ver_prod_odd_imag;
+	void __user *ver_prod_even_real;
+	void __user *ver_prod_even_imag;
+
+	if (!!access_ok(up, sizeof(struct atomisp_dis_statistics32)) ||
+	    copy_in_user(up, kp, sizeof(struct atomisp_dvs_grid_info)) ||
+	    get_user(hor_prod_odd_real,
+		     &kp->dvs2_stat.hor_prod.odd_real) ||
+	    get_user(hor_prod_odd_imag,
+		     &kp->dvs2_stat.hor_prod.odd_imag) ||
+	    get_user(hor_prod_even_real,
+		     &kp->dvs2_stat.hor_prod.even_real) ||
+	    get_user(hor_prod_even_imag,
+		     &kp->dvs2_stat.hor_prod.even_imag) ||
+	    get_user(ver_prod_odd_real,
+		     &kp->dvs2_stat.ver_prod.odd_real) ||
+	    get_user(ver_prod_odd_imag,
+		     &kp->dvs2_stat.ver_prod.odd_imag) ||
+	    get_user(ver_prod_even_real,
+		     &kp->dvs2_stat.ver_prod.even_real) ||
+	    get_user(ver_prod_even_imag,
+		     &kp->dvs2_stat.ver_prod.even_imag) ||
+	    put_user(ptr_to_compat(hor_prod_odd_real),
+		     &up->dvs2_stat.hor_prod.odd_real) ||
+	    put_user(ptr_to_compat(hor_prod_odd_imag),
+		     &up->dvs2_stat.hor_prod.odd_imag) ||
+	    put_user(ptr_to_compat(hor_prod_even_real),
+		     &up->dvs2_stat.hor_prod.even_real) ||
+	    put_user(ptr_to_compat(hor_prod_even_imag),
+		     &up->dvs2_stat.hor_prod.even_imag) ||
+	    put_user(ptr_to_compat(ver_prod_odd_real),
+		     &up->dvs2_stat.ver_prod.odd_real) ||
+	    put_user(ptr_to_compat(ver_prod_odd_imag),
+		     &up->dvs2_stat.ver_prod.odd_imag) ||
+	    put_user(ptr_to_compat(ver_prod_even_real),
+		     &up->dvs2_stat.ver_prod.even_real) ||
+	    put_user(ptr_to_compat(ver_prod_even_imag),
+		     &up->dvs2_stat.ver_prod.even_imag) ||
+	    assign_in_user(&up->exp_id, &kp->exp_id))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients *kp,
-	struct atomisp_dis_coefficients32 __user *up)
+static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients __user *kp,
+					  struct atomisp_dis_coefficients32 __user *up)
 {
 	compat_uptr_t hor_coefs_odd_real;
 	compat_uptr_t hor_coefs_odd_imag;
@@ -168,7 +196,7 @@ static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients *kp,
 	compat_uptr_t ver_coefs_even_imag;
 
 	if (!access_ok(up, sizeof(struct atomisp_dis_coefficients32)) ||
-	    copy_from_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) ||
+	    copy_in_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) ||
 	    get_user(hor_coefs_odd_real, &up->hor_coefs.odd_real) ||
 	    get_user(hor_coefs_odd_imag, &up->hor_coefs.odd_imag) ||
 	    get_user(hor_coefs_even_real, &up->hor_coefs.even_real) ||
@@ -176,22 +204,30 @@ static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients *kp,
 	    get_user(ver_coefs_odd_real, &up->ver_coefs.odd_real) ||
 	    get_user(ver_coefs_odd_imag, &up->ver_coefs.odd_imag) ||
 	    get_user(ver_coefs_even_real, &up->ver_coefs.even_real) ||
-	    get_user(ver_coefs_even_imag, &up->ver_coefs.even_imag))
+	    get_user(ver_coefs_even_imag, &up->ver_coefs.even_imag) ||
+	    put_user(compat_ptr(hor_coefs_odd_real),
+		     &kp->hor_coefs.odd_real) ||
+	    put_user(compat_ptr(hor_coefs_odd_imag),
+		     &kp->hor_coefs.odd_imag) ||
+	    put_user(compat_ptr(hor_coefs_even_real),
+		     &kp->hor_coefs.even_real) ||
+	    put_user(compat_ptr(hor_coefs_even_imag),
+		     &kp->hor_coefs.even_imag) ||
+	    put_user(compat_ptr(ver_coefs_odd_real),
+		     &kp->ver_coefs.odd_real) ||
+	    put_user(compat_ptr(ver_coefs_odd_imag),
+		     &kp->ver_coefs.odd_imag) ||
+	    put_user(compat_ptr(ver_coefs_even_real),
+		     &kp->ver_coefs.even_real) ||
+	    put_user(compat_ptr(ver_coefs_even_imag),
+		     &kp->ver_coefs.even_imag))
 		return -EFAULT;
 
-	kp->hor_coefs.odd_real = compat_ptr(hor_coefs_odd_real);
-	kp->hor_coefs.odd_imag = compat_ptr(hor_coefs_odd_imag);
-	kp->hor_coefs.even_real = compat_ptr(hor_coefs_even_real);
-	kp->hor_coefs.even_imag = compat_ptr(hor_coefs_even_imag);
-	kp->ver_coefs.odd_real = compat_ptr(ver_coefs_odd_real);
-	kp->ver_coefs.odd_imag = compat_ptr(ver_coefs_odd_imag);
-	kp->ver_coefs.even_real = compat_ptr(ver_coefs_even_real);
-	kp->ver_coefs.even_imag = compat_ptr(ver_coefs_even_imag);
 	return 0;
 }
 
-static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config *kp,
-	struct atomisp_dvs_6axis_config32 __user *up)
+static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config __user *kp,
+					  struct atomisp_dvs_6axis_config32 __user *up)
 {
 	compat_uptr_t xcoords_y;
 	compat_uptr_t ycoords_y;
@@ -199,62 +235,63 @@ static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config *kp,
 	compat_uptr_t ycoords_uv;
 
 	if (!access_ok(up, sizeof(struct atomisp_dvs_6axis_config32)) ||
-	    get_user(kp->exp_id, &up->exp_id) ||
-	    get_user(kp->width_y, &up->width_y) ||
-	    get_user(kp->height_y, &up->height_y) ||
-	    get_user(kp->width_uv, &up->width_uv) ||
-	    get_user(kp->height_uv, &up->height_uv) ||
+	    assign_in_user(&kp->exp_id, &up->exp_id) ||
+	    assign_in_user(&kp->width_y, &up->width_y) ||
+	    assign_in_user(&kp->height_y, &up->height_y) ||
+	    assign_in_user(&kp->width_uv, &up->width_uv) ||
+	    assign_in_user(&kp->height_uv, &up->height_uv) ||
 	    get_user(xcoords_y, &up->xcoords_y) ||
 	    get_user(ycoords_y, &up->ycoords_y) ||
 	    get_user(xcoords_uv, &up->xcoords_uv) ||
-	    get_user(ycoords_uv, &up->ycoords_uv))
+	    get_user(ycoords_uv, &up->ycoords_uv) ||
+	    put_user(compat_ptr(xcoords_y), &kp->xcoords_y) ||
+	    put_user(compat_ptr(ycoords_y), &kp->ycoords_y) ||
+	    put_user(compat_ptr(xcoords_uv), &kp->xcoords_uv) ||
+	    put_user(compat_ptr(ycoords_uv), &kp->ycoords_uv))
 		return -EFAULT;
 
-	kp->xcoords_y = (void __force *)compat_ptr(xcoords_y);
-	kp->ycoords_y = (void __force *)compat_ptr(ycoords_y);
-	kp->xcoords_uv = (void __force *)compat_ptr(xcoords_uv);
-	kp->ycoords_uv = (void __force *)compat_ptr(ycoords_uv);
 	return 0;
 }
 
-static int get_atomisp_3a_statistics32(struct atomisp_3a_statistics *kp,
+static int get_atomisp_3a_statistics32(struct atomisp_3a_statistics __user *kp,
 				       struct atomisp_3a_statistics32 __user *up)
 {
 	compat_uptr_t data;
 	compat_uptr_t rgby_data;
 
 	if (!access_ok(up, sizeof(struct atomisp_3a_statistics32)) ||
-	    copy_from_user(kp, up, sizeof(struct atomisp_grid_info)) ||
+	    copy_in_user(kp, up, sizeof(struct atomisp_grid_info)) ||
 	    get_user(rgby_data, &up->rgby_data) ||
+	    put_user(compat_ptr(rgby_data), &kp->rgby_data) ||
 	    get_user(data, &up->data) ||
-	    get_user(kp->exp_id, &up->exp_id) ||
-	    get_user(kp->isp_config_id, &up->isp_config_id))
+	    put_user(compat_ptr(data), &kp->data) ||
+	    assign_in_user(&kp->exp_id, &up->exp_id) ||
+	    assign_in_user(&kp->isp_config_id, &up->isp_config_id))
 		return -EFAULT;
 
-	kp->data = compat_ptr(data);
-	kp->rgby_data = compat_ptr(rgby_data);
-
 	return 0;
 }
 
-static int put_atomisp_3a_statistics32(struct atomisp_3a_statistics *kp,
+static int put_atomisp_3a_statistics32(struct atomisp_3a_statistics __user *kp,
 				       struct atomisp_3a_statistics32 __user *up)
 {
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-	compat_uptr_t rgby_data = (compat_uptr_t)((uintptr_t)kp->rgby_data);
+	void __user *data;
+	void __user *rgby_data;
 
 	if (!access_ok(up, sizeof(struct atomisp_3a_statistics32)) ||
 	    copy_to_user(up, kp, sizeof(struct atomisp_grid_info)) ||
-	    put_user(rgby_data, &up->rgby_data) ||
-	    put_user(data, &up->data) ||
-	    put_user(kp->exp_id, &up->exp_id) ||
-	    put_user(kp->isp_config_id, &up->isp_config_id))
+	    get_user(rgby_data, &kp->rgby_data) ||
+	    put_user(ptr_to_compat(rgby_data), &up->rgby_data) ||
+	    get_user(data, &kp->data) ||
+	    put_user(ptr_to_compat(data), &up->data) ||
+	    assign_in_user(&up->exp_id, &kp->exp_id) ||
+	    assign_in_user(&up->isp_config_id, &kp->isp_config_id))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_metadata_stat32(struct atomisp_metadata *kp,
+static int get_atomisp_metadata_stat32(struct atomisp_metadata __user *kp,
 				       struct atomisp_metadata32 __user *up)
 {
 	compat_uptr_t data;
@@ -262,553 +299,524 @@ static int get_atomisp_metadata_stat32(struct atomisp_metadata *kp,
 
 	if (!access_ok(up, sizeof(struct atomisp_metadata32)) ||
 	    get_user(data, &up->data) ||
-	    get_user(kp->width, &up->width) ||
-	    get_user(kp->height, &up->height) ||
-	    get_user(kp->stride, &up->stride) ||
-	    get_user(kp->exp_id, &up->exp_id) ||
-	    get_user(effective_width, &up->effective_width))
+	    put_user(compat_ptr(data), &kp->data) ||
+	    assign_in_user(&kp->width, &up->width) ||
+	    assign_in_user(&kp->height, &up->height) ||
+	    assign_in_user(&kp->stride, &up->stride) ||
+	    assign_in_user(&kp->exp_id, &up->exp_id) ||
+	    get_user(effective_width, &up->effective_width) ||
+	    put_user(compat_ptr(effective_width), &kp->effective_width))
 		return -EFAULT;
 
-	kp->data = compat_ptr(data);
-	kp->effective_width = (void __force *)compat_ptr(effective_width);
 	return 0;
 }
 
-static int put_atomisp_metadata_stat32(struct atomisp_metadata *kp,
-				       struct atomisp_metadata32 __user *up)
+static int put_atomisp_metadata_stat32(struct atomisp_metadata __user *kp,
+				struct atomisp_metadata32 __user *up)
 {
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-	compat_uptr_t effective_width =
-	    (compat_uptr_t)((uintptr_t)kp->effective_width);
+	void __user *data;
+	void __user *effective_width;
+
 	if (!access_ok(up, sizeof(struct atomisp_metadata32)) ||
-	    put_user(data, &up->data) ||
-	    put_user(kp->width, &up->width) ||
-	    put_user(kp->height, &up->height) ||
-	    put_user(kp->stride, &up->stride) ||
-	    put_user(kp->exp_id, &up->exp_id) ||
-	    put_user(effective_width, &up->effective_width))
+	    get_user(data, &kp->data) ||
+	    put_user(ptr_to_compat(data), &up->data) ||
+	    assign_in_user(&up->width, &kp->width) ||
+	    assign_in_user(&up->height, &kp->height) ||
+	    assign_in_user(&up->stride, &kp->stride) ||
+	    assign_in_user(&up->exp_id, &kp->exp_id) ||
+	    get_user(effective_width, &kp->effective_width) ||
+	    put_user(ptr_to_compat(effective_width), &up->effective_width))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int put_atomisp_metadata_by_type_stat32(
-    struct atomisp_metadata_with_type *kp,
-    struct atomisp_metadata_with_type32 __user *up)
+static int
+put_atomisp_metadata_by_type_stat32(struct atomisp_metadata_with_type __user *kp,
+				    struct atomisp_metadata_with_type32 __user *up)
 {
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
-	compat_uptr_t effective_width =
-	    (compat_uptr_t)((uintptr_t)kp->effective_width);
+	void __user *data;
+	void __user *effective_width;
+
 	if (!access_ok(up, sizeof(struct atomisp_metadata_with_type32)) ||
-	    put_user(data, &up->data) ||
-	    put_user(kp->width, &up->width) ||
-	    put_user(kp->height, &up->height) ||
-	    put_user(kp->stride, &up->stride) ||
-	    put_user(kp->exp_id, &up->exp_id) ||
-	    put_user(effective_width, &up->effective_width) ||
-	    put_user(kp->type, &up->type))
+	    get_user(data, &kp->data) ||
+	    put_user(ptr_to_compat(data), &up->data) ||
+	    assign_in_user(&up->width, &kp->width) ||
+	    assign_in_user(&up->height, &kp->height) ||
+	    assign_in_user(&up->stride, &kp->stride) ||
+	    assign_in_user(&up->exp_id, &kp->exp_id) ||
+	    get_user(effective_width, &kp->effective_width) ||
+	    put_user(ptr_to_compat(effective_width),
+		     &up->effective_width) ||
+	    assign_in_user(&up->type, &kp->type))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_metadata_by_type_stat32(
-    struct atomisp_metadata_with_type *kp,
-    struct atomisp_metadata_with_type32 __user *up)
+static int
+get_atomisp_metadata_by_type_stat32(struct atomisp_metadata_with_type __user *kp,
+				    struct atomisp_metadata_with_type32 __user *up)
 {
 	compat_uptr_t data;
 	compat_uptr_t effective_width;
 
 	if (!access_ok(up, sizeof(struct atomisp_metadata_with_type32)) ||
 	    get_user(data, &up->data) ||
-	    get_user(kp->width, &up->width) ||
-	    get_user(kp->height, &up->height) ||
-	    get_user(kp->stride, &up->stride) ||
-	    get_user(kp->exp_id, &up->exp_id) ||
+	    put_user(compat_ptr(data), &kp->data) ||
+	    assign_in_user(&kp->width, &up->width) ||
+	    assign_in_user(&kp->height, &up->height) ||
+	    assign_in_user(&kp->stride, &up->stride) ||
+	    assign_in_user(&kp->exp_id, &up->exp_id) ||
 	    get_user(effective_width, &up->effective_width) ||
-	    get_user(kp->type, &up->type))
+	    put_user(compat_ptr(effective_width), &kp->effective_width) ||
+	    assign_in_user(&kp->type, &up->type))
 		return -EFAULT;
 
-	kp->data = compat_ptr(data);
-	kp->effective_width = (void __force *)compat_ptr(effective_width);
 	return 0;
 }
 
-static int get_atomisp_morph_table32(struct atomisp_morph_table *kp,
-				     struct atomisp_morph_table32 __user *up)
+static int
+get_atomisp_morph_table32(struct atomisp_morph_table __user *kp,
+			  struct atomisp_morph_table32 __user *up)
 {
 	unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES;
 
 	if (!access_ok(up, sizeof(struct atomisp_morph_table32)) ||
-	    get_user(kp->enabled, &up->enabled) ||
-	    get_user(kp->width, &up->width) ||
-	    get_user(kp->height, &up->height))
-		return -EFAULT;
+		assign_in_user(&kp->enabled, &up->enabled) ||
+		assign_in_user(&kp->width, &up->width) ||
+		assign_in_user(&kp->height, &up->height))
+			return -EFAULT;
 
 	while (n-- > 0) {
-		uintptr_t *coord_kp = (uintptr_t *)&kp->coordinates_x[n];
-
-		if (get_user((*coord_kp), &up->coordinates_x[n]))
-			return -EFAULT;
+		compat_uptr_t coord_kp;
 
-		coord_kp = (uintptr_t *)&kp->coordinates_y[n];
-		if (get_user((*coord_kp), &up->coordinates_y[n]))
+		if (get_user(coord_kp, &up->coordinates_x[n]) ||
+		    put_user(compat_ptr(coord_kp), &kp->coordinates_x[n]) ||
+		    get_user(coord_kp, &up->coordinates_y[n]) ||
+		    put_user(compat_ptr(coord_kp), &kp->coordinates_y[n]))
 			return -EFAULT;
 	}
 	return 0;
 }
 
-static int put_atomisp_morph_table32(struct atomisp_morph_table *kp,
+static int put_atomisp_morph_table32(struct atomisp_morph_table __user *kp,
 				     struct atomisp_morph_table32 __user *up)
 {
 	unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES;
 
 	if (!access_ok(up, sizeof(struct atomisp_morph_table32)) ||
-	    put_user(kp->enabled, &up->enabled) ||
-	    put_user(kp->width, &up->width) ||
-	    put_user(kp->height, &up->height))
-		return -EFAULT;
+		assign_in_user(&up->enabled, &kp->enabled) ||
+		assign_in_user(&up->width, &kp->width) ||
+		assign_in_user(&up->height, &kp->height))
+			return -EFAULT;
 
 	while (n-- > 0) {
-		uintptr_t *coord_kp = (uintptr_t *)&kp->coordinates_x[n];
+		void __user *coord_kp;
 
-		if (put_user((*coord_kp), &up->coordinates_x[n]))
-			return -EFAULT;
-
-		coord_kp = (uintptr_t *)&kp->coordinates_y[n];
-		if (put_user((*coord_kp), &up->coordinates_y[n]))
+		if (get_user(coord_kp, &kp->coordinates_x[n]) ||
+		    put_user(ptr_to_compat(coord_kp), &up->coordinates_x[n]) ||
+		    get_user(coord_kp, &kp->coordinates_y[n]) ||
+		    put_user(ptr_to_compat(coord_kp), &up->coordinates_y[n]))
 			return -EFAULT;
 	}
 	return 0;
 }
 
-static int get_atomisp_overlay32(struct atomisp_overlay *kp,
+static int get_atomisp_overlay32(struct atomisp_overlay __user *kp,
 				 struct atomisp_overlay32 __user *up)
 {
 	compat_uptr_t frame;
 
 	if (!access_ok(up, sizeof(struct atomisp_overlay32)) ||
 	    get_user(frame, &up->frame) ||
-	    get_user(kp->bg_y, &up->bg_y) ||
-	    get_user(kp->bg_u, &up->bg_u) ||
-	    get_user(kp->bg_v, &up->bg_v) ||
-	    get_user(kp->blend_input_perc_y, &up->blend_input_perc_y) ||
-	    get_user(kp->blend_input_perc_u, &up->blend_input_perc_u) ||
-	    get_user(kp->blend_input_perc_v, &up->blend_input_perc_v) ||
-	    get_user(kp->blend_overlay_perc_y, &up->blend_overlay_perc_y) ||
-	    get_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) ||
-	    get_user(kp->blend_overlay_perc_v, &up->blend_overlay_perc_v) ||
-	    get_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) ||
-	    get_user(kp->overlay_start_x, &up->overlay_start_y))
+	    put_user(compat_ptr(frame), &kp->frame) ||
+	    assign_in_user(&kp->bg_y, &up->bg_y) ||
+	    assign_in_user(&kp->bg_u, &up->bg_u) ||
+	    assign_in_user(&kp->bg_v, &up->bg_v) ||
+	    assign_in_user(&kp->blend_input_perc_y,
+			   &up->blend_input_perc_y) ||
+	    assign_in_user(&kp->blend_input_perc_u,
+			   &up->blend_input_perc_u) ||
+	    assign_in_user(&kp->blend_input_perc_v,
+			   &up->blend_input_perc_v) ||
+	    assign_in_user(&kp->blend_overlay_perc_y,
+			   &up->blend_overlay_perc_y) ||
+	    assign_in_user(&kp->blend_overlay_perc_u,
+			   &up->blend_overlay_perc_u) ||
+	    assign_in_user(&kp->blend_overlay_perc_v,
+			   &up->blend_overlay_perc_v) ||
+	    assign_in_user(&kp->overlay_start_x, &up->overlay_start_x) ||
+	    assign_in_user(&kp->overlay_start_y, &up->overlay_start_y))
 		return -EFAULT;
 
-	kp->frame = (void __force *)compat_ptr(frame);
 	return 0;
 }
 
-static int put_atomisp_overlay32(struct atomisp_overlay *kp,
+static int put_atomisp_overlay32(struct atomisp_overlay __user *kp,
 				 struct atomisp_overlay32 __user *up)
 {
-	compat_uptr_t frame = (compat_uptr_t)((uintptr_t)kp->frame);
+	void __user *frame;
 
 	if (!access_ok(up, sizeof(struct atomisp_overlay32)) ||
-	    put_user(frame, &up->frame) ||
-	    put_user(kp->bg_y, &up->bg_y) ||
-	    put_user(kp->bg_u, &up->bg_u) ||
-	    put_user(kp->bg_v, &up->bg_v) ||
-	    put_user(kp->blend_input_perc_y, &up->blend_input_perc_y) ||
-	    put_user(kp->blend_input_perc_u, &up->blend_input_perc_u) ||
-	    put_user(kp->blend_input_perc_v, &up->blend_input_perc_v) ||
-	    put_user(kp->blend_overlay_perc_y, &up->blend_overlay_perc_y) ||
-	    put_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) ||
-	    put_user(kp->blend_overlay_perc_v, &up->blend_overlay_perc_v) ||
-	    put_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) ||
-	    put_user(kp->overlay_start_x, &up->overlay_start_y))
+	    get_user(frame, &kp->frame) ||
+	    put_user(ptr_to_compat(frame), &up->frame) ||
+	    assign_in_user(&up->bg_y, &kp->bg_y) ||
+	    assign_in_user(&up->bg_u, &kp->bg_u) ||
+	    assign_in_user(&up->bg_v, &kp->bg_v) ||
+	    assign_in_user(&up->blend_input_perc_y,
+			   &kp->blend_input_perc_y) ||
+	    assign_in_user(&up->blend_input_perc_u,
+			   &kp->blend_input_perc_u) ||
+	    assign_in_user(&up->blend_input_perc_v,
+			   &kp->blend_input_perc_v) ||
+	    assign_in_user(&up->blend_overlay_perc_y,
+			   &kp->blend_overlay_perc_y) ||
+	    assign_in_user(&up->blend_overlay_perc_u,
+			   &kp->blend_overlay_perc_u) ||
+	    assign_in_user(&up->blend_overlay_perc_v,
+			   &kp->blend_overlay_perc_v) ||
+	    assign_in_user(&up->overlay_start_x, &kp->overlay_start_x) ||
+	    assign_in_user(&up->overlay_start_y, &kp->overlay_start_y))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_calibration_group32(
-    struct atomisp_calibration_group *kp,
-    struct atomisp_calibration_group32 __user *up)
+static int
+get_atomisp_calibration_group32(struct atomisp_calibration_group __user *kp,
+				struct atomisp_calibration_group32 __user *up)
 {
 	compat_uptr_t calb_grp_values;
 
 	if (!access_ok(up, sizeof(struct atomisp_calibration_group32)) ||
-	    get_user(kp->size, &up->size) ||
-	    get_user(kp->type, &up->type) ||
-	    get_user(calb_grp_values, &up->calb_grp_values))
+	    assign_in_user(&kp->size, &up->size) ||
+	    assign_in_user(&kp->type, &up->type) ||
+	    get_user(calb_grp_values, &up->calb_grp_values) ||
+	    put_user(compat_ptr(calb_grp_values), &kp->calb_grp_values))
 		return -EFAULT;
 
-	kp->calb_grp_values = (void __force *)compat_ptr(calb_grp_values);
 	return 0;
 }
 
-static int put_atomisp_calibration_group32(
-    struct atomisp_calibration_group *kp,
-    struct atomisp_calibration_group32 __user *up)
+static int
+put_atomisp_calibration_group32(struct atomisp_calibration_group __user *kp,
+				struct atomisp_calibration_group32 __user *up)
 {
-	compat_uptr_t calb_grp_values =
-	    (compat_uptr_t)((uintptr_t)kp->calb_grp_values);
+	void __user *calb_grp_values;
 
 	if (!access_ok(up, sizeof(struct atomisp_calibration_group32)) ||
-	    put_user(kp->size, &up->size) ||
-	    put_user(kp->type, &up->type) ||
-	    put_user(calb_grp_values, &up->calb_grp_values))
+	    assign_in_user(&up->size, &kp->size) ||
+	    assign_in_user(&up->type, &kp->type) ||
+	    get_user(calb_grp_values, &kp->calb_grp_values) ||
+	    put_user(ptr_to_compat(calb_grp_values), &up->calb_grp_values))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_acc_fw_load32(struct atomisp_acc_fw_load *kp,
+static int get_atomisp_acc_fw_load32(struct atomisp_acc_fw_load __user *kp,
 				     struct atomisp_acc_fw_load32 __user *up)
 {
 	compat_uptr_t data;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_fw_load32)) ||
-	    get_user(kp->size, &up->size) ||
-	    get_user(kp->fw_handle, &up->fw_handle) ||
-	    get_user(data, &up->data))
+	    assign_in_user(&kp->size, &up->size) ||
+	    assign_in_user(&kp->fw_handle, &up->fw_handle) ||
+	    get_user(data, &up->data) ||
+	    put_user(compat_ptr(data), &kp->data))
 		return -EFAULT;
 
-	kp->data = compat_ptr(data);
 	return 0;
 }
 
-static int put_atomisp_acc_fw_load32(struct atomisp_acc_fw_load *kp,
+static int put_atomisp_acc_fw_load32(struct atomisp_acc_fw_load __user *kp,
 				     struct atomisp_acc_fw_load32 __user *up)
 {
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
+	void __user *data;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_fw_load32)) ||
-	    put_user(kp->size, &up->size) ||
-	    put_user(kp->fw_handle, &up->fw_handle) ||
-	    put_user(data, &up->data))
+	    assign_in_user(&up->size, &kp->size) ||
+	    assign_in_user(&up->fw_handle, &kp->fw_handle) ||
+	    get_user(data, &kp->data) ||
+	    put_user(ptr_to_compat(data), &up->data))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg *kp,
+static int get_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg __user *kp,
 				    struct atomisp_acc_fw_arg32 __user *up)
 {
 	compat_uptr_t value;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_fw_arg32)) ||
-	    get_user(kp->fw_handle, &up->fw_handle) ||
-	    get_user(kp->index, &up->index) ||
+	    assign_in_user(&kp->fw_handle, &up->fw_handle) ||
+	    assign_in_user(&kp->index, &up->index) ||
 	    get_user(value, &up->value) ||
-	    get_user(kp->size, &up->size))
+	    put_user(compat_ptr(value), &kp->value) ||
+	    assign_in_user(&kp->size, &up->size))
 		return -EFAULT;
 
-	kp->value = compat_ptr(value);
 	return 0;
 }
 
-static int put_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg *kp,
+static int put_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg __user *kp,
 				    struct atomisp_acc_fw_arg32 __user *up)
 {
-	compat_uptr_t value = (compat_uptr_t)((uintptr_t)kp->value);
+	void __user *value;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_fw_arg32)) ||
-	    put_user(kp->fw_handle, &up->fw_handle) ||
-	    put_user(kp->index, &up->index) ||
-	    put_user(value, &up->value) ||
-	    put_user(kp->size, &up->size))
+	    assign_in_user(&up->fw_handle, &kp->fw_handle) ||
+	    assign_in_user(&up->index, &kp->index) ||
+	    get_user(value, &kp->value) ||
+	    put_user(ptr_to_compat(value), &up->value) ||
+	    assign_in_user(&up->size, &kp->size))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_v4l2_private_int_data32(struct v4l2_private_int_data *kp,
+static int get_v4l2_private_int_data32(struct v4l2_private_int_data __user *kp,
 				       struct v4l2_private_int_data32 __user *up)
 {
 	compat_uptr_t data;
 
 	if (!access_ok(up, sizeof(struct v4l2_private_int_data32)) ||
-	    get_user(kp->size, &up->size) ||
+	    assign_in_user(&kp->size, &up->size) ||
 	    get_user(data, &up->data) ||
-	    get_user(kp->reserved[0], &up->reserved[0]) ||
-	    get_user(kp->reserved[1], &up->reserved[1]))
+	    put_user(compat_ptr(data), &kp->data) ||
+	    assign_in_user(&kp->reserved[0], &up->reserved[0]) ||
+	    assign_in_user(&kp->reserved[1], &up->reserved[1]))
 		return -EFAULT;
 
-	kp->data = compat_ptr(data);
 	return 0;
 }
 
-static int put_v4l2_private_int_data32(struct v4l2_private_int_data *kp,
+static int put_v4l2_private_int_data32(struct v4l2_private_int_data __user *kp,
 				       struct v4l2_private_int_data32 __user *up)
 {
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
+	void __user *data;
 
 	if (!access_ok(up, sizeof(struct v4l2_private_int_data32)) ||
-	    put_user(kp->size, &up->size) ||
-	    put_user(data, &up->data) ||
-	    put_user(kp->reserved[0], &up->reserved[0]) ||
-	    put_user(kp->reserved[1], &up->reserved[1]))
+	    assign_in_user(&up->size, &kp->size) ||
+	    get_user(data, &kp->data) ||
+	    put_user(ptr_to_compat(data), &up->data) ||
+	    assign_in_user(&up->reserved[0], &kp->reserved[0]) ||
+	    assign_in_user(&up->reserved[1], &kp->reserved[1]))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_shading_table32(struct atomisp_shading_table *kp,
+static int get_atomisp_shading_table32(struct atomisp_shading_table __user *kp,
 				       struct atomisp_shading_table32 __user *up)
 {
 	unsigned int n = ATOMISP_NUM_SC_COLORS;
 
 	if (!access_ok(up, sizeof(struct atomisp_shading_table32)) ||
-	    get_user(kp->enable, &up->enable) ||
-	    get_user(kp->sensor_width, &up->sensor_width) ||
-	    get_user(kp->sensor_height, &up->sensor_height) ||
-	    get_user(kp->width, &up->width) ||
-	    get_user(kp->height, &up->height) ||
-	    get_user(kp->fraction_bits, &up->fraction_bits))
+	    assign_in_user(&kp->enable, &up->enable) ||
+	    assign_in_user(&kp->sensor_width, &up->sensor_width) ||
+	    assign_in_user(&kp->sensor_height, &up->sensor_height) ||
+	    assign_in_user(&kp->width, &up->width) ||
+	    assign_in_user(&kp->height, &up->height) ||
+	    assign_in_user(&kp->fraction_bits, &up->fraction_bits))
 		return -EFAULT;
 
 	while (n-- > 0) {
-		uintptr_t *data_p = (uintptr_t *)&kp->data[n];
+		compat_uptr_t tmp;
 
-		if (get_user((*data_p), &up->data[n]))
+		if (get_user(tmp, &up->data[n]) ||
+		    put_user(compat_ptr(tmp), &kp->data[n]))
 			return -EFAULT;
 	}
 	return 0;
 }
 
-static int get_atomisp_acc_map32(struct atomisp_acc_map *kp,
+static int get_atomisp_acc_map32(struct atomisp_acc_map __user *kp,
 				 struct atomisp_acc_map32 __user *up)
 {
 	compat_uptr_t user_ptr;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_map32)) ||
-	    get_user(kp->flags, &up->flags) ||
-	    get_user(kp->length, &up->length) ||
+	    assign_in_user(&kp->flags, &up->flags) ||
+	    assign_in_user(&kp->length, &up->length) ||
 	    get_user(user_ptr, &up->user_ptr) ||
-	    get_user(kp->css_ptr, &up->css_ptr) ||
-	    get_user(kp->reserved[0], &up->reserved[0]) ||
-	    get_user(kp->reserved[1], &up->reserved[1]) ||
-	    get_user(kp->reserved[2], &up->reserved[2]) ||
-	    get_user(kp->reserved[3], &up->reserved[3]))
+	    put_user(compat_ptr(user_ptr), &kp->user_ptr) ||
+	    assign_in_user(&kp->css_ptr, &up->css_ptr) ||
+	    assign_in_user(&kp->reserved[0], &up->reserved[0]) ||
+	    assign_in_user(&kp->reserved[1], &up->reserved[1]) ||
+	    assign_in_user(&kp->reserved[2], &up->reserved[2]) ||
+	    assign_in_user(&kp->reserved[3], &up->reserved[3]))
 		return -EFAULT;
 
-	kp->user_ptr = compat_ptr(user_ptr);
 	return 0;
 }
 
-static int put_atomisp_acc_map32(struct atomisp_acc_map *kp,
+static int put_atomisp_acc_map32(struct atomisp_acc_map __user *kp,
 				 struct atomisp_acc_map32 __user *up)
 {
-	compat_uptr_t user_ptr = (compat_uptr_t)((uintptr_t)kp->user_ptr);
+	void __user *user_ptr;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_map32)) ||
-	    put_user(kp->flags, &up->flags) ||
-	    put_user(kp->length, &up->length) ||
-	    put_user(user_ptr, &up->user_ptr) ||
-	    put_user(kp->css_ptr, &up->css_ptr) ||
-	    put_user(kp->reserved[0], &up->reserved[0]) ||
-	    put_user(kp->reserved[1], &up->reserved[1]) ||
-	    put_user(kp->reserved[2], &up->reserved[2]) ||
-	    put_user(kp->reserved[3], &up->reserved[3]))
+	    assign_in_user(&up->flags, &kp->flags) ||
+	    assign_in_user(&up->length, &kp->length) ||
+	    get_user(user_ptr, &kp->user_ptr) ||
+	    put_user(ptr_to_compat(user_ptr), &up->user_ptr) ||
+	    assign_in_user(&up->css_ptr, &kp->css_ptr) ||
+	    assign_in_user(&up->reserved[0], &kp->reserved[0]) ||
+	    assign_in_user(&up->reserved[1], &kp->reserved[1]) ||
+	    assign_in_user(&up->reserved[2], &kp->reserved[2]) ||
+	    assign_in_user(&up->reserved[3], &kp->reserved[3]))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg *kp,
-	struct atomisp_acc_s_mapped_arg32 __user *up)
+static int
+get_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg __user *kp,
+			       struct atomisp_acc_s_mapped_arg32 __user *up)
 {
 	if (!access_ok(up, sizeof(struct atomisp_acc_s_mapped_arg32)) ||
-	    get_user(kp->fw_handle, &up->fw_handle) ||
-	    get_user(kp->memory, &up->memory) ||
-	    get_user(kp->length, &up->length) ||
-	    get_user(kp->css_ptr, &up->css_ptr))
+	    assign_in_user(&kp->fw_handle, &up->fw_handle) ||
+	    assign_in_user(&kp->memory, &up->memory) ||
+	    assign_in_user(&kp->length, &up->length) ||
+	    assign_in_user(&kp->css_ptr, &up->css_ptr))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int put_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg *kp,
-	struct atomisp_acc_s_mapped_arg32 __user *up)
+static int
+put_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg __user *kp,
+			       struct atomisp_acc_s_mapped_arg32 __user *up)
 {
 	if (!access_ok(up, sizeof(struct atomisp_acc_s_mapped_arg32)) ||
-	    put_user(kp->fw_handle, &up->fw_handle) ||
-	    put_user(kp->memory, &up->memory) ||
-	    put_user(kp->length, &up->length) ||
-	    put_user(kp->css_ptr, &up->css_ptr))
+	    assign_in_user(&up->fw_handle, &kp->fw_handle) ||
+	    assign_in_user(&up->memory, &kp->memory) ||
+	    assign_in_user(&up->length, &kp->length) ||
+	    assign_in_user(&up->css_ptr, &kp->css_ptr))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_parameters32(struct atomisp_parameters *kp,
+static int get_atomisp_parameters32(struct atomisp_parameters __user *kp,
 				    struct atomisp_parameters32 __user *up)
 {
 	int n = offsetof(struct atomisp_parameters32, output_frame) /
 		sizeof(compat_uptr_t);
-	unsigned int size, offset = 0;
-	void  __user *user_ptr;
-	unsigned int stp, mtp, dcp, dscp = 0;
+	compat_uptr_t stp, mtp, dcp, dscp;
+	struct {
+		struct atomisp_shading_table shading_table;
+		struct atomisp_morph_table morph_table;
+		struct atomisp_dis_coefficients dvs2_coefs;
+		struct atomisp_dvs_6axis_config dvs_6axis_config;
+	} __user *karg = (void *)(kp + 1);
 
 	if (!access_ok(up, sizeof(struct atomisp_parameters32)))
 		return -EFAULT;
 
 	while (n >= 0) {
-		compat_uptr_t __user *src = ((compat_uptr_t __user *)up) + n;
-		uintptr_t *dst = ((uintptr_t *)kp) + n;
+		compat_uptr_t *src = (compat_uptr_t *)up + n;
+		void * __user *dst = (void * __user *)kp + n;
+		compat_uptr_t tmp;
 
-		if (get_user((*dst), src))
+		if (get_user(tmp, src) || put_user(compat_ptr(tmp), dst))
 			return -EFAULT;
 		n--;
 	}
-	if (get_user(kp->isp_config_id, &up->isp_config_id) ||
-	    get_user(kp->per_frame_setting, &up->per_frame_setting) ||
+
+	if (assign_in_user(&kp->isp_config_id, &up->isp_config_id) ||
+	    assign_in_user(&kp->per_frame_setting, &up->per_frame_setting) ||
 	    get_user(stp, &up->shading_table) ||
 	    get_user(mtp, &up->morph_table) ||
 	    get_user(dcp, &up->dvs2_coefs) ||
 	    get_user(dscp, &up->dvs_6axis_config))
 		return -EFAULT;
 
-	{
-		union {
-			struct atomisp_shading_table shading_table;
-			struct atomisp_morph_table   morph_table;
-			struct atomisp_dis_coefficients dvs2_coefs;
-			struct atomisp_dvs_6axis_config dvs_6axis_config;
-		} karg;
-
-		size = sizeof(struct atomisp_shading_table) +
-		       sizeof(struct atomisp_morph_table) +
-		       sizeof(struct atomisp_dis_coefficients) +
-		       sizeof(struct atomisp_dvs_6axis_config);
-		user_ptr = compat_alloc_user_space(size);
-
-		/* handle shading table */
-		if (stp != 0) {
-			if (get_atomisp_shading_table32(&karg.shading_table,
-							(struct atomisp_shading_table32 __user *)
-							(uintptr_t)stp))
-				return -EFAULT;
-
-			kp->shading_table = (void __force *)user_ptr + offset;
-			offset = sizeof(struct atomisp_shading_table);
-			if (!kp->shading_table)
-				return -EFAULT;
-
-			if (copy_to_user((void __user *)kp->shading_table,
-					 &karg.shading_table,
-					 sizeof(struct atomisp_shading_table)))
-				return -EFAULT;
-		}
-
-		/* handle morph table */
-		if (mtp != 0) {
-			if (get_atomisp_morph_table32(&karg.morph_table,
-						      (struct atomisp_morph_table32 __user *)
-						      (uintptr_t)mtp))
-				return -EFAULT;
-
-			kp->morph_table = (void __force *)user_ptr + offset;
-			offset += sizeof(struct atomisp_morph_table);
-			if (!kp->morph_table)
-				return -EFAULT;
-
-			if (copy_to_user((void __user *)kp->morph_table,
-					 &karg.morph_table,
-					 sizeof(struct atomisp_morph_table)))
-				return -EFAULT;
-		}
-
-		/* handle dvs2 coefficients */
-		if (dcp != 0) {
-			if (get_atomisp_dis_coefficients32(&karg.dvs2_coefs,
-							   (struct atomisp_dis_coefficients32 __user *)
-							   (uintptr_t)dcp))
-				return -EFAULT;
-
-			kp->dvs2_coefs = (void __force *)user_ptr + offset;
-			offset += sizeof(struct atomisp_dis_coefficients);
-			if (!kp->dvs2_coefs)
-				return -EFAULT;
-
-			if (copy_to_user((void __user *)kp->dvs2_coefs,
-					 &karg.dvs2_coefs,
-					 sizeof(struct atomisp_dis_coefficients)))
-				return -EFAULT;
-		}
-		/* handle dvs 6axis configuration */
-		if (dscp != 0) {
-			if (get_atomisp_dvs_6axis_config32(&karg.dvs_6axis_config,
-							   (struct atomisp_dvs_6axis_config32 __user *)
-							   (uintptr_t)dscp))
-				return -EFAULT;
-
-			kp->dvs_6axis_config = (void __force *)user_ptr + offset;
-			offset += sizeof(struct atomisp_dvs_6axis_config);
-			if (!kp->dvs_6axis_config)
-				return -EFAULT;
-
-			if (copy_to_user((void __user *)kp->dvs_6axis_config,
-					 &karg.dvs_6axis_config,
-					 sizeof(struct atomisp_dvs_6axis_config)))
-				return -EFAULT;
-		}
-	}
+	/* handle shading table */
+	if (stp && (get_atomisp_shading_table32(&karg->shading_table,
+						compat_ptr(stp)) ||
+		    put_user(&karg->shading_table, &kp->shading_table)))
+		return -EFAULT;
+
+	/* handle morph table */
+	if (mtp && (get_atomisp_morph_table32(&karg->morph_table,
+					      compat_ptr(mtp)) ||
+		    put_user(&karg->morph_table, &kp->morph_table)))
+		return -EFAULT;
+
+	/* handle dvs2 coefficients */
+	if (dcp && (get_atomisp_dis_coefficients32(&karg->dvs2_coefs,
+						   compat_ptr(dcp)) ||
+		    put_user(&karg->dvs2_coefs, &kp->dvs2_coefs)))
+		return -EFAULT;
+
+	/* handle dvs 6axis configuration */
+	if (dscp &&
+	    (get_atomisp_dvs_6axis_config32(&karg->dvs_6axis_config,
+					    compat_ptr(dscp)) ||
+	     put_user(&karg->dvs_6axis_config, &kp->dvs_6axis_config)))
+		return -EFAULT;
+
 	return 0;
 }
 
-static int get_atomisp_acc_fw_load_to_pipe32(
-    struct atomisp_acc_fw_load_to_pipe *kp,
-    struct atomisp_acc_fw_load_to_pipe32 __user *up)
+static int
+get_atomisp_acc_fw_load_to_pipe32(struct atomisp_acc_fw_load_to_pipe __user *kp,
+				  struct atomisp_acc_fw_load_to_pipe32 __user *up)
 {
 	compat_uptr_t data;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_fw_load_to_pipe32)) ||
-	    get_user(kp->flags, &up->flags) ||
-	    get_user(kp->fw_handle, &up->fw_handle) ||
-	    get_user(kp->size, &up->size) ||
-	    get_user(kp->type, &up->type) ||
-	    get_user(kp->reserved[0], &up->reserved[0]) ||
-	    get_user(kp->reserved[1], &up->reserved[1]) ||
-	    get_user(kp->reserved[2], &up->reserved[2]) ||
-	    get_user(data, &up->data))
+	    assign_in_user(&kp->flags, &up->flags) ||
+	    assign_in_user(&kp->fw_handle, &up->fw_handle) ||
+	    assign_in_user(&kp->size, &up->size) ||
+	    assign_in_user(&kp->type, &up->type) ||
+	    assign_in_user(&kp->reserved[0], &up->reserved[0]) ||
+	    assign_in_user(&kp->reserved[1], &up->reserved[1]) ||
+	    assign_in_user(&kp->reserved[2], &up->reserved[2]) ||
+	    get_user(data, &up->data) ||
+	    put_user(compat_ptr(data), &kp->data))
 		return -EFAULT;
 
-	kp->data = compat_ptr(data);
 	return 0;
 }
 
-static int put_atomisp_acc_fw_load_to_pipe32(
-    struct atomisp_acc_fw_load_to_pipe *kp,
-    struct atomisp_acc_fw_load_to_pipe32 __user *up)
+static int
+put_atomisp_acc_fw_load_to_pipe32(struct atomisp_acc_fw_load_to_pipe __user *kp,
+				  struct atomisp_acc_fw_load_to_pipe32 __user *up)
 {
-	compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data);
+	void __user *data;
 
 	if (!access_ok(up, sizeof(struct atomisp_acc_fw_load_to_pipe32)) ||
-	    put_user(kp->flags, &up->flags) ||
-	    put_user(kp->fw_handle, &up->fw_handle) ||
-	    put_user(kp->size, &up->size) ||
-	    put_user(kp->type, &up->type) ||
-	    put_user(kp->reserved[0], &up->reserved[0]) ||
-	    put_user(kp->reserved[1], &up->reserved[1]) ||
-	    put_user(kp->reserved[2], &up->reserved[2]) ||
-	    put_user(data, &up->data))
+	    assign_in_user(&up->flags, &kp->flags) ||
+	    assign_in_user(&up->fw_handle, &kp->fw_handle) ||
+	    assign_in_user(&up->size, &kp->size) ||
+	    assign_in_user(&up->type, &kp->type) ||
+	    assign_in_user(&up->reserved[0], &kp->reserved[0]) ||
+	    assign_in_user(&up->reserved[1], &kp->reserved[1]) ||
+	    assign_in_user(&up->reserved[2], &kp->reserved[2]) ||
+	    get_user(data, &kp->data) ||
+	    put_user(ptr_to_compat(data), &up->data))
 		return -EFAULT;
 
 	return 0;
 }
 
-static int get_atomisp_sensor_ae_bracketing_lut(
-    struct atomisp_sensor_ae_bracketing_lut *kp,
-    struct atomisp_sensor_ae_bracketing_lut32 __user *up)
+static int
+get_atomisp_sensor_ae_bracketing_lut(struct atomisp_sensor_ae_bracketing_lut __user *kp,
+				     struct atomisp_sensor_ae_bracketing_lut32 __user *up)
 {
 	compat_uptr_t lut;
 
 	if (!access_ok(up, sizeof(struct atomisp_sensor_ae_bracketing_lut32)) ||
-	    get_user(kp->lut_size, &up->lut_size) ||
-	    get_user(lut, &up->lut))
+	    assign_in_user(&kp->lut_size, &up->lut_size) ||
+	    get_user(lut, &up->lut) ||
+	    put_user(compat_ptr(lut), &kp->lut))
 		return -EFAULT;
 
-	kp->lut = (void __force *)compat_ptr(lut);
 	return 0;
 }
 
@@ -846,11 +854,19 @@ static long atomisp_do_compat_ioctl(struct file *file,
 		struct atomisp_metadata md;
 		struct atomisp_metadata_with_type md_with_type;
 		struct atomisp_sensor_ae_bracketing_lut lut;
-	} karg;
-	mm_segment_t old_fs;
+	} __user *karg;
 	void __user *up = compat_ptr(arg);
 	long err = -ENOIOCTLCMD;
 
+	karg = compat_alloc_user_space(
+		sizeof(*karg) + (cmd == ATOMISP_IOC_S_PARAMETERS32 ?
+				 sizeof(struct atomisp_shading_table) +
+				 sizeof(struct atomisp_morph_table) +
+				 sizeof(struct atomisp_dis_coefficients) +
+				 sizeof(struct atomisp_dvs_6axis_config) : 0));
+	if (!karg)
+		return -ENOMEM;
+
 	/* First, convert the command. */
 	switch (cmd) {
 	case ATOMISP_IOC_G_HISTOGRAM32:
@@ -936,130 +952,127 @@ static long atomisp_do_compat_ioctl(struct file *file,
 	switch (cmd) {
 	case ATOMISP_IOC_G_HISTOGRAM:
 	case ATOMISP_IOC_S_HISTOGRAM:
-		err = get_atomisp_histogram32(&karg.his, up);
+		err = get_atomisp_histogram32(&karg->his, up);
 		break;
 	case ATOMISP_IOC_G_DIS_STAT:
-		err = get_atomisp_dis_statistics32(&karg.dis_s, up);
+		err = get_atomisp_dis_statistics32(&karg->dis_s, up);
 		break;
 	case ATOMISP_IOC_S_DIS_COEFS:
-		err = get_atomisp_dis_coefficients32(&karg.dis_c, up);
+		err = get_atomisp_dis_coefficients32(&karg->dis_c, up);
 		break;
 	case ATOMISP_IOC_S_DIS_VECTOR:
-		err = get_atomisp_dvs_6axis_config32(&karg.dvs_c, up);
+		err = get_atomisp_dvs_6axis_config32(&karg->dvs_c, up);
 		break;
 	case ATOMISP_IOC_G_3A_STAT:
-		err = get_atomisp_3a_statistics32(&karg.s3a_s, up);
+		err = get_atomisp_3a_statistics32(&karg->s3a_s, up);
 		break;
 	case ATOMISP_IOC_G_ISP_GDC_TAB:
 	case ATOMISP_IOC_S_ISP_GDC_TAB:
-		err = get_atomisp_morph_table32(&karg.mor_t, up);
+		err = get_atomisp_morph_table32(&karg->mor_t, up);
 		break;
 	case ATOMISP_IOC_S_ISP_FPN_TABLE:
-		err = get_v4l2_framebuffer32(&karg.v4l2_buf, up);
+		err = get_v4l2_framebuffer32(&karg->v4l2_buf, up);
 		break;
 	case ATOMISP_IOC_G_ISP_OVERLAY:
 	case ATOMISP_IOC_S_ISP_OVERLAY:
-		err = get_atomisp_overlay32(&karg.overlay, up);
+		err = get_atomisp_overlay32(&karg->overlay, up);
 		break;
 	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP:
-		err = get_atomisp_calibration_group32(&karg.cal_grp, up);
+		err = get_atomisp_calibration_group32(&karg->cal_grp, up);
 		break;
 	case ATOMISP_IOC_ACC_LOAD:
-		err = get_atomisp_acc_fw_load32(&karg.acc_fw_load, up);
+		err = get_atomisp_acc_fw_load32(&karg->acc_fw_load, up);
 		break;
 	case ATOMISP_IOC_ACC_S_ARG:
 	case ATOMISP_IOC_ACC_DESTAB:
-		err = get_atomisp_acc_fw_arg32(&karg.acc_fw_arg, up);
+		err = get_atomisp_acc_fw_arg32(&karg->acc_fw_arg, up);
 		break;
 	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
 	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA:
-		err = get_v4l2_private_int_data32(&karg.v4l2_pri_data, up);
+		err = get_v4l2_private_int_data32(&karg->v4l2_pri_data, up);
 		break;
 	case ATOMISP_IOC_S_ISP_SHD_TAB:
-		err = get_atomisp_shading_table32(&karg.shd_tbl, up);
+		err = get_atomisp_shading_table32(&karg->shd_tbl, up);
 		break;
 	case ATOMISP_IOC_ACC_MAP:
 	case ATOMISP_IOC_ACC_UNMAP:
-		err = get_atomisp_acc_map32(&karg.acc_map, up);
+		err = get_atomisp_acc_map32(&karg->acc_map, up);
 		break;
 	case ATOMISP_IOC_ACC_S_MAPPED_ARG:
-		err = get_atomisp_acc_s_mapped_arg32(&karg.acc_map_arg, up);
+		err = get_atomisp_acc_s_mapped_arg32(&karg->acc_map_arg, up);
 		break;
 	case ATOMISP_IOC_S_PARAMETERS:
-		err = get_atomisp_parameters32(&karg.param, up);
+		err = get_atomisp_parameters32(&karg->param, up);
 		break;
 	case ATOMISP_IOC_ACC_LOAD_TO_PIPE:
-		err = get_atomisp_acc_fw_load_to_pipe32(&karg.acc_fw_to_pipe,
+		err = get_atomisp_acc_fw_load_to_pipe32(&karg->acc_fw_to_pipe,
 							up);
 		break;
 	case ATOMISP_IOC_G_METADATA:
-		err = get_atomisp_metadata_stat32(&karg.md, up);
+		err = get_atomisp_metadata_stat32(&karg->md, up);
 		break;
 	case ATOMISP_IOC_G_METADATA_BY_TYPE:
-		err = get_atomisp_metadata_by_type_stat32(&karg.md_with_type,
-			up);
+		err = get_atomisp_metadata_by_type_stat32(&karg->md_with_type,
+							  up);
 		break;
 	case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT:
-		err = get_atomisp_sensor_ae_bracketing_lut(&karg.lut, up);
+		err = get_atomisp_sensor_ae_bracketing_lut(&karg->lut, up);
 		break;
 	}
 	if (err)
 		return err;
 
-	old_fs = get_fs();
-	set_fs(KERNEL_DS);
-	err = native_ioctl(file, cmd, (unsigned long)&karg);
-	set_fs(old_fs);
+	err = native_ioctl(file, cmd, (unsigned long)karg);
 	if (err)
 		return err;
 
 	switch (cmd) {
 	case ATOMISP_IOC_G_HISTOGRAM:
-		err = put_atomisp_histogram32(&karg.his, up);
+		err = put_atomisp_histogram32(&karg->his, up);
 		break;
 	case ATOMISP_IOC_G_DIS_STAT:
-		err = put_atomisp_dis_statistics32(&karg.dis_s, up);
+		err = put_atomisp_dis_statistics32(&karg->dis_s, up);
 		break;
 	case ATOMISP_IOC_G_3A_STAT:
-		err = put_atomisp_3a_statistics32(&karg.s3a_s, up);
+		err = put_atomisp_3a_statistics32(&karg->s3a_s, up);
 		break;
 	case ATOMISP_IOC_G_ISP_GDC_TAB:
-		err = put_atomisp_morph_table32(&karg.mor_t, up);
+		err = put_atomisp_morph_table32(&karg->mor_t, up);
 		break;
 	case ATOMISP_IOC_G_ISP_OVERLAY:
-		err = put_atomisp_overlay32(&karg.overlay, up);
+		err = put_atomisp_overlay32(&karg->overlay, up);
 		break;
 	case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP:
-		err = put_atomisp_calibration_group32(&karg.cal_grp, up);
+		err = put_atomisp_calibration_group32(&karg->cal_grp, up);
 		break;
 	case ATOMISP_IOC_ACC_LOAD:
-		err = put_atomisp_acc_fw_load32(&karg.acc_fw_load, up);
+		err = put_atomisp_acc_fw_load32(&karg->acc_fw_load, up);
 		break;
 	case ATOMISP_IOC_ACC_S_ARG:
 	case ATOMISP_IOC_ACC_DESTAB:
-		err = put_atomisp_acc_fw_arg32(&karg.acc_fw_arg, up);
+		err = put_atomisp_acc_fw_arg32(&karg->acc_fw_arg, up);
 		break;
 	case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA:
 	case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA:
-		err = put_v4l2_private_int_data32(&karg.v4l2_pri_data, up);
+		err = put_v4l2_private_int_data32(&karg->v4l2_pri_data, up);
 		break;
 	case ATOMISP_IOC_ACC_MAP:
 	case ATOMISP_IOC_ACC_UNMAP:
-		err = put_atomisp_acc_map32(&karg.acc_map, up);
+		err = put_atomisp_acc_map32(&karg->acc_map, up);
 		break;
 	case ATOMISP_IOC_ACC_S_MAPPED_ARG:
-		err = put_atomisp_acc_s_mapped_arg32(&karg.acc_map_arg, up);
+		err = put_atomisp_acc_s_mapped_arg32(&karg->acc_map_arg, up);
 		break;
 	case ATOMISP_IOC_ACC_LOAD_TO_PIPE:
-		err = put_atomisp_acc_fw_load_to_pipe32(&karg.acc_fw_to_pipe,
+		err = put_atomisp_acc_fw_load_to_pipe32(&karg->acc_fw_to_pipe,
 							up);
 		break;
 	case ATOMISP_IOC_G_METADATA:
-		err = put_atomisp_metadata_stat32(&karg.md, up);
+		err = put_atomisp_metadata_stat32(&karg->md, up);
 		break;
 	case ATOMISP_IOC_G_METADATA_BY_TYPE:
-		err = put_atomisp_metadata_by_type_stat32(&karg.md_with_type,
-			up);
+		err = put_atomisp_metadata_by_type_stat32(&karg->md_with_type,
+							  up);
 		break;
 	}
 
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h
index 7e59ccb88a2e..86d3fbe01378 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2.c b/drivers/staging/media/atomisp/pci/atomisp_csi2.c
index a2638863206e..060b8765ae96 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_csi2.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_csi2.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -384,9 +385,9 @@ static void atomisp_csi2_configure_isp2401(struct atomisp_sub_device *asd)
 	for (n = 0; n < csi2_port_lanes[port] + 1; n++) {
 		hrt_address base = csi2_port_base[port] + csi2_lane_base[n];
 
-		atomisp_store_uint32(base + CSI2_REG_RX_CSI_DLY_CNT_TERMEN,
+		atomisp_css2_hw_store_32(base + CSI2_REG_RX_CSI_DLY_CNT_TERMEN,
 				     n == 0 ? clk_termen : dat_termen);
-		atomisp_store_uint32(base + CSI2_REG_RX_CSI_DLY_CNT_SETTLE,
+		atomisp_css2_hw_store_32(base + CSI2_REG_RX_CSI_DLY_CNT_SETTLE,
 				     n == 0 ? clk_settle : dat_settle);
 	}
 }
diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2.h b/drivers/staging/media/atomisp/pci/atomisp_csi2.h
index 739c26f0807a..59261e8f1a1a 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_csi2.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_csi2.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h b/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h
index 9680f211d424..8f1cc3fca13a 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_drvfs.c b/drivers/staging/media/atomisp/pci/atomisp_drvfs.c
index 4a6ea021ddcc..fe0e2bfde27f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_drvfs.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_drvfs.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for atomisp driver sysfs interface
  *
@@ -24,6 +25,7 @@
 #include "atomisp_ioctl.h"
 #include "atomisp_drvfs.h"
 #include "hmm/hmm.h"
+#include "ia_css_debug.h"
 
 /*
  * _iunit_debug:
@@ -94,7 +96,7 @@ opt_err:
 
 static ssize_t iunit_dbglvl_show(struct device_driver *drv, char *buf)
 {
-	iunit_debug.dbglvl = atomisp_css_debug_get_dtrace_level();
+	iunit_debug.dbglvl = dbg_level;
 	return sprintf(buf, "dtrace level:%u\n", iunit_debug.dbglvl);
 }
 
@@ -106,7 +108,7 @@ static ssize_t iunit_dbglvl_store(struct device_driver *drv, const char *buf,
 	    || iunit_debug.dbglvl > 9) {
 		return -ERANGE;
 	}
-	atomisp_css_debug_set_dtrace_level(iunit_debug.dbglvl);
+	ia_css_debug_set_dtrace_level(iunit_debug.dbglvl);
 
 	return size;
 }
diff --git a/drivers/staging/media/atomisp/pci/atomisp_drvfs.h b/drivers/staging/media/atomisp/pci/atomisp_drvfs.h
index 7c99240d107a..4911037231fb 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_drvfs.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_drvfs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for atomisp driver sysfs interface.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_file.c b/drivers/staging/media/atomisp/pci/atomisp_file.c
index 4ab0390316cf..e568ca99c45a 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_file.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_file.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -50,11 +51,12 @@ static void file_work(struct work_struct *work)
 						V4L2_SUBDEV_FORMAT_ACTIVE,
 						ATOMISP_SUBDEV_PAD_SINK);
 
-	while (!atomisp_css_isp_has_started())
+	while (!ia_css_isp_has_started())
 		usleep_range(1000, 1500);
 
-	atomisp_css_send_input_frame(asd, buf, isp_sink_fmt.width,
-				     isp_sink_fmt.height);
+	ia_css_stream_send_input_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
+				       buf, isp_sink_fmt.width,
+				       isp_sink_fmt.height);
 	dev_dbg(isp->dev, "<%s: streaming done\n", __func__);
 }
 
@@ -217,7 +219,7 @@ int atomisp_file_input_init(struct atomisp_device *isp)
 
 	v4l2_subdev_init(sd, &file_input_ops);
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	strcpy(sd->name, "file_input_subdev");
+	strscpy(sd->name, "file_input_subdev", sizeof(sd->name));
 	v4l2_set_subdevdata(sd, file_dev);
 
 	pads[0].flags = MEDIA_PAD_FL_SINK;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_file.h b/drivers/staging/media/atomisp/pci/atomisp_file.h
index e38f8bc389f1..f166a2aefff1 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_file.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_file.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp_fops.c
index 667d6f7d1d5e..453bb6913550 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_fops.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_fops.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -34,11 +35,8 @@
 #include "atomisp-regs.h"
 #include "hmm/hmm.h"
 
-#include "hrt/hive_isp_css_mm_hrt.h"
-
 #include "type_support.h"
 #include "device_access/device_access.h"
-#include "memory_access/memory_access.h"
 
 #include "atomisp_acc.h"
 
@@ -88,7 +86,7 @@ static int atomisp_buf_prepare(struct videobuf_queue *vq,
 
 static int atomisp_q_one_metadata_buffer(struct atomisp_sub_device *asd,
 	enum atomisp_input_stream_id stream_id,
-	enum atomisp_css_pipe_id css_pipe_id)
+	enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_metadata_buf *metadata_buf;
 	enum atomisp_metadata_type md_type =
@@ -128,7 +126,7 @@ static int atomisp_q_one_metadata_buffer(struct atomisp_sub_device *asd,
 
 static int atomisp_q_one_s3a_buffer(struct atomisp_sub_device *asd,
 				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_css_pipe_id css_pipe_id)
+				    enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_s3a_buf *s3a_buf;
 	struct list_head *s3a_list;
@@ -170,7 +168,7 @@ static int atomisp_q_one_s3a_buffer(struct atomisp_sub_device *asd,
 
 static int atomisp_q_one_dis_buffer(struct atomisp_sub_device *asd,
 				    enum atomisp_input_stream_id stream_id,
-				    enum atomisp_css_pipe_id css_pipe_id)
+				    enum ia_css_pipe_id css_pipe_id)
 {
 	struct atomisp_dis_buf *dis_buf;
 	unsigned long irqflags;
@@ -213,12 +211,12 @@ static int atomisp_q_one_dis_buffer(struct atomisp_sub_device *asd,
 int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd,
 				   struct atomisp_video_pipe *pipe,
 				   enum atomisp_input_stream_id stream_id,
-				   enum atomisp_css_buffer_type css_buf_type,
-				   enum atomisp_css_pipe_id css_pipe_id)
+				   enum ia_css_buffer_type css_buf_type,
+				   enum ia_css_pipe_id css_pipe_id)
 {
 	struct videobuf_vmalloc_memory *vm_mem;
 	struct atomisp_css_params_with_list *param;
-	struct atomisp_css_dvs_grid_info *dvs_grid =
+	struct ia_css_dvs_grid_info *dvs_grid =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 	unsigned long irqflags;
 	int err = 0;
@@ -255,14 +253,13 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd,
 				err = atomisp_calculate_real_zoom_region(asd,
 					&param->params.dz_config, css_pipe_id);
 				if (!err)
-					atomisp_css_set_dz_config(asd,
-								  &param->params.dz_config);
+					asd->params.config.dz_config = &param->params.dz_config;
 			}
 			atomisp_css_set_isp_config_applied_frame(asd,
 				vm_mem->vaddr);
 			atomisp_css_update_isp_params_on_pipe(asd,
 							      asd->stream_env[stream_id].pipes[css_pipe_id]);
-			asd->params.dvs_6axis = (struct atomisp_css_dvs_6axis *)
+			asd->params.dvs_6axis = (struct ia_css_dvs_6axis_config *)
 						param->params.dvs_6axis;
 
 			/*
@@ -302,19 +299,19 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd,
 		/* enqueue 3A/DIS/metadata buffers */
 		if (asd->params.curr_grid_info.s3a_grid.enable &&
 		    css_pipe_id == asd->params.s3a_enabled_pipe &&
-		    css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
+		    css_buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)
 			atomisp_q_one_s3a_buffer(asd, stream_id,
 						 css_pipe_id);
 
 		if (asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream_info.
 		    metadata_info.size &&
-		    css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
+		    css_buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)
 			atomisp_q_one_metadata_buffer(asd, stream_id,
 						      css_pipe_id);
 
 		if (dvs_grid && dvs_grid->enable &&
-		    css_pipe_id == CSS_PIPE_ID_VIDEO &&
-		    css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME)
+		    css_pipe_id == IA_CSS_PIPE_ID_VIDEO &&
+		    css_buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)
 			atomisp_q_one_dis_buffer(asd, stream_id,
 						 css_pipe_id);
 	}
@@ -323,7 +320,7 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd,
 }
 
 static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd,
-				    enum atomisp_css_pipe_id pipe_id,
+				    enum ia_css_pipe_id pipe_id,
 				    uint16_t source_pad)
 {
 	if (ATOMISP_USE_YUVPP(asd)) {
@@ -331,24 +328,24 @@ static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd,
 		if (asd->continuous_mode->val &&
 		    asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
 			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE)
-				return CSS_BUFFER_TYPE_OUTPUT_FRAME;
+				return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
 			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				return CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
+				return IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
 			else
-				return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
+				return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
 		}
 
 		/*when run SDV case*/
 		if (asd->continuous_mode->val &&
 		    asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 			if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE)
-				return CSS_BUFFER_TYPE_OUTPUT_FRAME;
+				return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
 			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW)
-				return CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME;
+				return IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME;
 			else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO)
-				return CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
+				return IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME;
 			else
-				return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
+				return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
 		}
 
 		/*other case: default setting*/
@@ -356,27 +353,27 @@ static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd,
 		    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO ||
 		    (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
 		     asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO))
-			return CSS_BUFFER_TYPE_OUTPUT_FRAME;
+			return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
 		else
-			return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
+			return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
 	}
 
-	if (pipe_id == CSS_PIPE_ID_COPY ||
+	if (pipe_id == IA_CSS_PIPE_ID_COPY ||
 	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE ||
 	    source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO ||
 	    (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW &&
 	     asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO))
-		return CSS_BUFFER_TYPE_OUTPUT_FRAME;
+		return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME;
 	else
-		return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
+		return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME;
 }
 
 static int atomisp_qbuffers_to_css_for_all_pipes(struct atomisp_sub_device *asd)
 {
-	enum atomisp_css_buffer_type buf_type;
-	enum atomisp_css_pipe_id css_capture_pipe_id = CSS_PIPE_ID_COPY;
-	enum atomisp_css_pipe_id css_preview_pipe_id = CSS_PIPE_ID_COPY;
-	enum atomisp_css_pipe_id css_video_pipe_id = CSS_PIPE_ID_COPY;
+	enum ia_css_buffer_type buf_type;
+	enum ia_css_pipe_id css_capture_pipe_id = IA_CSS_PIPE_ID_COPY;
+	enum ia_css_pipe_id css_preview_pipe_id = IA_CSS_PIPE_ID_COPY;
+	enum ia_css_pipe_id css_video_pipe_id = IA_CSS_PIPE_ID_COPY;
 	enum atomisp_input_stream_id input_stream_id;
 	struct atomisp_video_pipe *capture_pipe;
 	struct atomisp_video_pipe *preview_pipe;
@@ -413,10 +410,10 @@ static int atomisp_qbuffers_to_css_for_all_pipes(struct atomisp_sub_device *asd)
 /* queue all available buffers to css */
 int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 {
-	enum atomisp_css_buffer_type buf_type;
-	enum atomisp_css_pipe_id css_capture_pipe_id = CSS_PIPE_ID_NUM;
-	enum atomisp_css_pipe_id css_preview_pipe_id = CSS_PIPE_ID_NUM;
-	enum atomisp_css_pipe_id css_video_pipe_id = CSS_PIPE_ID_NUM;
+	enum ia_css_buffer_type buf_type;
+	enum ia_css_pipe_id css_capture_pipe_id = IA_CSS_PIPE_ID_NUM;
+	enum ia_css_pipe_id css_preview_pipe_id = IA_CSS_PIPE_ID_NUM;
+	enum ia_css_pipe_id css_video_pipe_id = IA_CSS_PIPE_ID_NUM;
 	enum atomisp_input_stream_id input_stream_id;
 	struct atomisp_video_pipe *capture_pipe = NULL;
 	struct atomisp_video_pipe *vf_pipe = NULL;
@@ -432,43 +429,43 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 
 	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) {
 		video_pipe = &asd->video_out_video_capture;
-		css_video_pipe_id = CSS_PIPE_ID_VIDEO;
+		css_video_pipe_id = IA_CSS_PIPE_ID_VIDEO;
 	} else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) {
 		preview_pipe = &asd->video_out_capture;
-		css_preview_pipe_id = CSS_PIPE_ID_CAPTURE;
+		css_preview_pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) {
 		if (asd->continuous_mode->val) {
 			capture_pipe = &asd->video_out_capture;
 			vf_pipe = &asd->video_out_vf;
-			css_capture_pipe_id = CSS_PIPE_ID_CAPTURE;
+			css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 		}
 		video_pipe = &asd->video_out_video_capture;
 		preview_pipe = &asd->video_out_preview;
-		css_video_pipe_id = CSS_PIPE_ID_VIDEO;
-		css_preview_pipe_id = CSS_PIPE_ID_VIDEO;
+		css_video_pipe_id = IA_CSS_PIPE_ID_VIDEO;
+		css_preview_pipe_id = IA_CSS_PIPE_ID_VIDEO;
 	} else if (asd->continuous_mode->val) {
 		capture_pipe = &asd->video_out_capture;
 		vf_pipe = &asd->video_out_vf;
 		preview_pipe = &asd->video_out_preview;
 
-		css_preview_pipe_id = CSS_PIPE_ID_PREVIEW;
-		css_capture_pipe_id = CSS_PIPE_ID_CAPTURE;
+		css_preview_pipe_id = IA_CSS_PIPE_ID_PREVIEW;
+		css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 	} else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) {
 		preview_pipe = &asd->video_out_preview;
-		css_preview_pipe_id = CSS_PIPE_ID_PREVIEW;
+		css_preview_pipe_id = IA_CSS_PIPE_ID_PREVIEW;
 	} else {
 		/* ATOMISP_RUN_MODE_STILL_CAPTURE */
 		capture_pipe = &asd->video_out_capture;
 		if (!raw_mode)
 			vf_pipe = &asd->video_out_vf;
-		css_capture_pipe_id = CSS_PIPE_ID_CAPTURE;
+		css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE;
 	}
 
 #ifdef ISP2401_NEW_INPUT_SYSTEM
 	if (asd->copy_mode) {
-		css_capture_pipe_id = CSS_PIPE_ID_COPY;
-		css_preview_pipe_id = CSS_PIPE_ID_COPY;
-		css_video_pipe_id = CSS_PIPE_ID_COPY;
+		css_capture_pipe_id = IA_CSS_PIPE_ID_COPY;
+		css_preview_pipe_id = IA_CSS_PIPE_ID_COPY;
+		css_video_pipe_id = IA_CSS_PIPE_ID_COPY;
 	}
 #endif
 
@@ -476,9 +473,9 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		capture_pipe = &asd->video_out_capture;
 		video_pipe   = &asd->video_out_video_capture;
 		preview_pipe = &asd->video_out_preview;
-		css_capture_pipe_id = CSS_PIPE_ID_COPY;
-		css_video_pipe_id   = CSS_PIPE_ID_YUVPP;
-		css_preview_pipe_id = CSS_PIPE_ID_YUVPP;
+		css_capture_pipe_id = IA_CSS_PIPE_ID_COPY;
+		css_video_pipe_id   = IA_CSS_PIPE_ID_YUVPP;
+		css_preview_pipe_id = IA_CSS_PIPE_ID_YUVPP;
 	}
 
 	if (capture_pipe) {
@@ -491,7 +488,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		 * use yuvpp pipe for SOC camera.
 		 */
 		if (ATOMISP_USE_YUVPP(asd))
-			css_capture_pipe_id = CSS_PIPE_ID_YUVPP;
+			css_capture_pipe_id = IA_CSS_PIPE_ID_YUVPP;
 
 		atomisp_q_video_buffers_to_css(asd, capture_pipe,
 					       input_stream_id,
@@ -511,7 +508,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		 * use yuvpp pipe for SOC camera.
 		 */
 		if (ATOMISP_USE_YUVPP(asd))
-			css_capture_pipe_id = CSS_PIPE_ID_YUVPP;
+			css_capture_pipe_id = IA_CSS_PIPE_ID_YUVPP;
 		atomisp_q_video_buffers_to_css(asd, vf_pipe,
 					       input_stream_id,
 					       buf_type, css_capture_pipe_id);
@@ -521,10 +518,10 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		buf_type = atomisp_get_css_buf_type(
 			       asd, css_preview_pipe_id,
 			       atomisp_subdev_source_pad(&preview_pipe->vdev));
-		if (ATOMISP_SOC_CAMERA(asd) && css_preview_pipe_id == CSS_PIPE_ID_YUVPP)
+		if (ATOMISP_SOC_CAMERA(asd) && css_preview_pipe_id == IA_CSS_PIPE_ID_YUVPP)
 			input_stream_id = ATOMISP_INPUT_STREAM_GENERAL;
 		/* else for ext isp use case */
-		else if (css_preview_pipe_id == CSS_PIPE_ID_YUVPP)
+		else if (css_preview_pipe_id == IA_CSS_PIPE_ID_YUVPP)
 			input_stream_id = ATOMISP_INPUT_STREAM_VIDEO;
 		else if (asd->stream_env[ATOMISP_INPUT_STREAM_PREVIEW].stream)
 			input_stream_id = ATOMISP_INPUT_STREAM_PREVIEW;
@@ -535,7 +532,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		 * use yuvpp pipe for SOC camera.
 		 */
 		if (ATOMISP_USE_YUVPP(asd))
-			css_preview_pipe_id = CSS_PIPE_ID_YUVPP;
+			css_preview_pipe_id = IA_CSS_PIPE_ID_YUVPP;
 
 		atomisp_q_video_buffers_to_css(asd, preview_pipe,
 					       input_stream_id,
@@ -555,7 +552,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd)
 		 * use yuvpp pipe for SOC camera.
 		 */
 		if (ATOMISP_USE_YUVPP(asd))
-			css_video_pipe_id = CSS_PIPE_ID_YUVPP;
+			css_video_pipe_id = IA_CSS_PIPE_ID_YUVPP;
 
 		atomisp_q_video_buffers_to_css(asd, video_pipe,
 					       input_stream_id,
@@ -717,7 +714,7 @@ static void atomisp_subdev_init_struct(struct atomisp_sub_device *asd)
 	asd->params.online_process = 1;
 	asd->params.yuv_ds_en = 0;
 	/* s3a grid not enabled for any pipe */
-	asd->params.s3a_enabled_pipe = CSS_PIPE_ID_NUM;
+	asd->params.s3a_enabled_pipe = IA_CSS_PIPE_ID_NUM;
 
 	asd->params.offline_parm.num_captures = 1;
 	asd->params.offline_parm.skip_frames = 0;
@@ -1000,7 +997,7 @@ subdev_uninit:
 	atomisp_css_uninit(isp);
 
 	if (defer_fw_load) {
-		atomisp_css_unload_firmware(isp);
+		ia_css_unload_firmware();
 		isp->css_env.isp_css_fw.data = NULL;
 		isp->css_env.isp_css_fw.bytes = 0;
 	}
@@ -1054,7 +1051,7 @@ static int do_isp_mm_remap(struct atomisp_device *isp,
 }
 
 static int frame_mmap(struct atomisp_device *isp,
-		      const struct atomisp_css_frame *frame, struct vm_area_struct *vma)
+		      const struct ia_css_frame *frame, struct vm_area_struct *vma)
 {
 	ia_css_ptr isp_virt;
 	u32 host_virt;
@@ -1127,7 +1124,7 @@ int atomisp_videobuf_mmap_mapper(struct videobuf_queue *q,
  * There is also padding on the right (padded_width - width).
  */
 static int remove_pad_from_frame(struct atomisp_device *isp,
-				 struct atomisp_css_frame *in_frame, __u32 width, __u32 height)
+				 struct ia_css_frame *in_frame, __u32 width, __u32 height)
 {
 	unsigned int i;
 	unsigned short *buffer;
@@ -1164,7 +1161,7 @@ static int atomisp_mmap(struct file *file, struct vm_area_struct *vma)
 	struct atomisp_device *isp = video_get_drvdata(vdev);
 	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
 	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_css_frame *raw_virt_addr;
+	struct ia_css_frame *raw_virt_addr;
 	u32 start = vma->vm_start;
 	u32 end = vma->vm_end;
 	u32 size = end - start;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.h b/drivers/staging/media/atomisp/pci/atomisp_fops.h
index e05e8f3a4442..3f1e442ba782 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_fops.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_fops.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -24,8 +25,8 @@
 int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd,
 				   struct atomisp_video_pipe *pipe,
 				   enum atomisp_input_stream_id stream_id,
-				   enum atomisp_css_buffer_type css_buf_type,
-				   enum atomisp_css_pipe_id css_pipe_id);
+				   enum ia_css_buffer_type css_buf_type,
+				   enum ia_css_pipe_id css_pipe_id);
 
 unsigned int atomisp_dev_users(struct atomisp_device *isp);
 unsigned int atomisp_sub_dev_users(struct atomisp_sub_device *asd);
diff --git a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
index b096b7d30463..1af9da8acf4c 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 #include <linux/module.h>
 #include <linux/i2c.h>
 #include <linux/dmi.h>
@@ -327,15 +328,6 @@ static struct gmin_cfg_var i8880_vars[] = {
 	{},
 };
 
-static struct gmin_cfg_var asus_vars[] = {
-	{"OVTI2680:00_CsiPort", "1"},
-	{"OVTI2680:00_CsiLanes", "1"},
-	{"OVTI2680:00_CsiFmt", "15"},
-	{"OVTI2680:00_CsiBayer", "0"},
-	{"OVTI2680:00_CamClk", "1"},
-	{},
-};
-
 static const struct dmi_system_id gmin_vars[] = {
 	{
 		.ident = "BYT-T FFD8",
@@ -373,13 +365,6 @@ static const struct dmi_system_id gmin_vars[] = {
 		},
 		.driver_data = i8880_vars,
 	},
-	{
-		.ident = "T101HA",
-		.matches = {
-			DMI_MATCH(DMI_BOARD_NAME, "T101HA"),
-		},
-		.driver_data = asus_vars,
-	},
 	{}
 };
 
@@ -387,6 +372,10 @@ static const struct dmi_system_id gmin_vars[] = {
 				       0xa9, 0x71, 0xe8, 0x77,	   \
 				       0x75, 0x60, 0x68, 0xf7)
 
+static const guid_t atomisp_dsm_guid = GUID_INIT(0xdc2f6c4f, 0x045b, 0x4f1d,
+						 0x97, 0xb9, 0x88, 0x2a,
+						 0x68, 0x60, 0xa4, 0xbe);
+
 #define CFG_VAR_NAME_MAX 64
 
 #define GMIN_PMC_CLK_NAME 14 /* "pmc_plt_clk_[0..5]" */
@@ -454,15 +443,28 @@ static int gmin_i2c_write(struct device *dev, u16 i2c_addr, u8 reg,
 
 static struct gmin_subdev *gmin_subdev_add(struct v4l2_subdev *subdev)
 {
-	int i, ret;
-	struct device *dev;
 	struct i2c_client *power = NULL, *client = v4l2_get_subdevdata(subdev);
+	struct acpi_device *adev;
+	acpi_handle handle;
+	struct device *dev;
+	int i, ret;
 
 	if (!client)
 		return NULL;
 
 	dev = &client->dev;
 
+	handle = ACPI_HANDLE(dev);
+
+	// FIXME: may need to release resources allocated by acpi_bus_get_device()
+	if (!handle || acpi_bus_get_device(handle, &adev)) {
+		dev_err(dev, "Error could not get ACPI device\n");
+		return NULL;
+	}
+
+	dev_info(&client->dev, "%s: ACPI detected it on bus ID=%s, HID=%s\n",
+		__func__, acpi_device_bid(adev), acpi_device_hid(adev));
+
 	if (!pmic_id) {
 		if (gmin_i2c_dev_exists(dev, PMIC_ACPI_TI, &power))
 			pmic_id = PMIC_TI;
@@ -479,7 +481,6 @@ static struct gmin_subdev *gmin_subdev_add(struct v4l2_subdev *subdev)
 	if (i >= MAX_SUBDEVS)
 		return NULL;
 
-
 	if (power) {
 		gmin_subdevs[i].pwm_i2c_addr = power->addr;
 		dev_info(dev,
@@ -616,6 +617,7 @@ static int axp_regulator_set(struct device *dev, struct gmin_subdev *gs,
 static int axp_v1p8_on(struct device *dev, struct gmin_subdev *gs)
 {
 	int ret;
+
 	ret = axp_regulator_set(dev, gs, gs->eldo2_sel_reg, gs->eldo2_1p8v,
 				ELDO_CTRL_REG, gs->eldo2_ctrl_shift, true);
 	if (ret)
@@ -640,6 +642,7 @@ static int axp_v1p8_on(struct device *dev, struct gmin_subdev *gs)
 static int axp_v1p8_off(struct device *dev, struct gmin_subdev *gs)
 {
 	int ret;
+
 	ret = axp_regulator_set(dev, gs, gs->eldo1_sel_reg, gs->eldo1_1p8v,
 				ELDO_CTRL_REG, gs->eldo1_ctrl_shift, false);
 	if (ret)
@@ -650,7 +653,6 @@ static int axp_v1p8_off(struct device *dev, struct gmin_subdev *gs)
 	return ret;
 }
 
-
 static int gmin_gpio0_ctrl(struct v4l2_subdev *subdev, int on)
 {
 	struct gmin_subdev *gs = find_gmin_subdev(subdev);
@@ -753,7 +755,6 @@ static int gmin_v1p8_ctrl(struct v4l2_subdev *subdev, int on)
 		dev_err(subdev->dev, "Couldn't set power mode for v1p2\n");
 	}
 
-
 	return -EINVAL;
 }
 
@@ -921,7 +922,8 @@ int atomisp_gmin_register_vcm_control(struct camera_vcm_control *vcmCtrl)
 }
 EXPORT_SYMBOL_GPL(atomisp_gmin_register_vcm_control);
 
-static int gmin_get_hardcoded_var(struct gmin_cfg_var *varlist,
+static int gmin_get_hardcoded_var(struct device *dev,
+				  struct gmin_cfg_var *varlist,
 				  const char *var8, char *out, size_t *out_len)
 {
 	struct gmin_cfg_var *gv;
@@ -932,11 +934,13 @@ static int gmin_get_hardcoded_var(struct gmin_cfg_var *varlist,
 		if (strcmp(var8, gv->name))
 			continue;
 
+		dev_info(dev, "Found DMI entry for '%s'\n", var8);
+
 		vl = strlen(gv->val);
 		if (vl > *out_len - 1)
 			return -ENOSPC;
 
-		strcpy(out, gv->val);
+		strscpy(out, gv->val, *out_len);
 		*out_len = vl;
 		return 0;
 	}
@@ -944,6 +948,75 @@ static int gmin_get_hardcoded_var(struct gmin_cfg_var *varlist,
 	return -EINVAL;
 }
 
+
+static int gmin_get_config_dsm_var(struct device *dev,
+				   const char *var,
+				   char *out, size_t *out_len)
+{
+	acpi_handle handle = ACPI_HANDLE(dev);
+	union acpi_object *obj, *cur = NULL;
+	int i;
+
+	obj = acpi_evaluate_dsm(handle, &atomisp_dsm_guid, 0, 0, NULL);
+	if (!obj) {
+		dev_info_once(dev, "Didn't find ACPI _DSM table.\n");
+		return -EINVAL;
+	}
+
+#if 0 /* Just for debugging purposes */
+	for (i = 0; i < obj->package.count; i++) {
+		union acpi_object *cur = &obj->package.elements[i];
+
+		if (cur->type == ACPI_TYPE_INTEGER)
+			dev_info(dev, "object #%d, type %d, value: %lld\n",
+				 i, cur->type, cur->integer.value);
+		else if (cur->type == ACPI_TYPE_STRING)
+			dev_info(dev, "object #%d, type %d, string: %s\n",
+				 i, cur->type, cur->string.pointer);
+		else
+			dev_info(dev, "object #%d, type %d\n",
+				 i, cur->type);
+	}
+#endif
+
+	/* Seek for the desired var */
+	for (i = 0; i < obj->package.count - 1; i += 2) {
+		if (obj->package.elements[i].type == ACPI_TYPE_STRING &&
+		    !strcmp(obj->package.elements[i].string.pointer, var)) {
+			/* Next element should be the required value */
+			cur = &obj->package.elements[i + 1];
+			break;
+		}
+	}
+
+	if (!cur) {
+		dev_info(dev, "didn't found _DSM entry for '%s'\n", var);
+		ACPI_FREE(obj);
+		return -EINVAL;
+	}
+
+	/*
+	 * While it could be possible to have an ACPI_TYPE_INTEGER,
+	 * and read the value from cur->integer.value, the table
+	 * seen so far uses the string type. So, produce a warning
+	 * if it founds something different than string, letting it
+	 * to fall back to the old code.
+	 */
+	if (cur && cur->type != ACPI_TYPE_STRING) {
+		dev_info(dev, "found non-string _DSM entry for '%s'\n", var);
+		ACPI_FREE(obj);
+		return -EINVAL;
+	}
+
+	dev_info(dev, "found _DSM entry for '%s': %s\n", var,
+		 cur->string.pointer);
+	strscpy(out, cur->string.pointer, *out_len);
+	*out_len = strlen(cur->string.pointer);
+
+	ACPI_FREE(obj);
+	return 0;
+}
+
 /* Retrieves a device-specific configuration variable.  The dev
  * argument should be a device with an ACPI companion, as all
  * configuration is based on firmware ID.
@@ -953,12 +1026,21 @@ static int gmin_get_config_var(struct device *maindev,
 			       const char *var,
 			       char *out, size_t *out_len)
 {
-	char var8[CFG_VAR_NAME_MAX];
 	efi_char16_t var16[CFG_VAR_NAME_MAX];
-	struct efivar_entry *ev;
 	const struct dmi_system_id *id;
-	int i, ret;
 	struct device *dev = maindev;
+	char var8[CFG_VAR_NAME_MAX];
+	struct efivar_entry *ev;
+	int i, ret;
+
+	/* For sensors, try first to use the _DSM table */
+	if (!is_gmin) {
+		ret = gmin_get_config_dsm_var(maindev, var, out, out_len);
+		if (!ret)
+			return 0;
+	}
+
+	/* Fall-back to other approaches */
 
 	if (!is_gmin && ACPI_COMPANION(dev))
 		dev = &ACPI_COMPANION(dev)->dev;
@@ -977,9 +1059,10 @@ static int gmin_get_config_var(struct device *maindev,
 	 */
 	id = dmi_first_match(gmin_vars);
 	if (id) {
-		dev_info(maindev, "Found DMI entry for '%s'\n", var8);
-		return gmin_get_hardcoded_var(id->driver_data, var8, out,
-					      out_len);
+		ret = gmin_get_hardcoded_var(maindev, id->driver_data, var8,
+					     out, out_len);
+		if (!ret)
+			return 0;
 	}
 
 	/* Our variable names are ASCII by construction, but EFI names
@@ -1009,9 +1092,9 @@ static int gmin_get_config_var(struct device *maindev,
 		*out_len = ev->var.DataSize;
 		dev_info(maindev, "found EFI entry for '%s'\n", var8);
 	} else if (is_gmin) {
-		dev_warn(maindev, "Failed to find gmin variable %s\n", var8);
+		dev_info(maindev, "Failed to find EFI gmin variable %s\n", var8);
 	} else {
-		dev_warn(maindev, "Failed to find variable %s\n", var8);
+		dev_info(maindev, "Failed to find EFI variable %s\n", var8);
 	}
 
 	kfree(ev);
@@ -1030,6 +1113,8 @@ int gmin_get_var_int(struct device *dev, bool is_gmin, const char *var, int def)
 	if (!ret) {
 		val[len] = 0;
 		ret = kstrtol(val, 0, &result);
+	} else {
+		dev_info(dev, "%s: using default (%d)\n", var, def);
 	}
 
 	return ret ? def : result;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_helper.h b/drivers/staging/media/atomisp/pci/atomisp_helper.h
deleted file mode 100644
index 56035063f81d..000000000000
--- a/drivers/staging/media/atomisp/pci/atomisp_helper.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef _atomisp_helper_h_
-#define _atomisp_helper_h_
-extern void __iomem *atomisp_io_base;
-
-static inline void __iomem *atomisp_get_io_virt_addr(unsigned int address)
-{
-	void __iomem *ret = atomisp_io_base + (address & 0x003FFFFF);
-	return ret;
-}
-#endif
diff --git a/drivers/staging/media/atomisp/pci/atomisp_internal.h b/drivers/staging/media/atomisp/pci/atomisp_internal.h
index 26539f3ffb9b..ff3becd41110 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_internal.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_internal.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
index a5e71e5b714e..9404a678fa6f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -39,8 +40,6 @@
 #include "device_access.h"
 #include "irq.h"
 
-#include "hrt/hive_isp_css_mm_hrt.h"
-
 static const char *DRIVER = "atomisp";	/* max size 15 */
 static const char *CARD = "ATOM ISP";	/* max size 31 */
 
@@ -341,163 +340,163 @@ const struct atomisp_format_bridge atomisp_output_fmts[] = {
 		.pixelformat = V4L2_PIX_FMT_YUV420,
 		.depth = 12,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV420,
-		.sh_fmt = CSS_FRAME_FORMAT_YUV420,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_YUV420,
 		.description = "YUV420, planar",
 		.planar = true
 	}, {
 		.pixelformat = V4L2_PIX_FMT_YVU420,
 		.depth = 12,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YVU420,
-		.sh_fmt = CSS_FRAME_FORMAT_YV12,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_YV12,
 		.description = "YVU420, planar",
 		.planar = true
 	}, {
 		.pixelformat = V4L2_PIX_FMT_YUV422P,
 		.depth = 16,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV422P,
-		.sh_fmt = CSS_FRAME_FORMAT_YUV422,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_YUV422,
 		.description = "YUV422, planar",
 		.planar = true
 	}, {
 		.pixelformat = V4L2_PIX_FMT_YUV444,
 		.depth = 24,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV444,
-		.sh_fmt = CSS_FRAME_FORMAT_YUV444,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_YUV444,
 		.description = "YUV444"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_NV12,
 		.depth = 12,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_NV12,
-		.sh_fmt = CSS_FRAME_FORMAT_NV12,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_NV12,
 		.description = "NV12, Y-plane, CbCr interleaved",
 		.planar = true
 	}, {
 		.pixelformat = V4L2_PIX_FMT_NV21,
 		.depth = 12,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_NV21,
-		.sh_fmt = CSS_FRAME_FORMAT_NV21,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_NV21,
 		.description = "NV21, Y-plane, CbCr interleaved",
 		.planar = true
 	}, {
 		.pixelformat = V4L2_PIX_FMT_NV16,
 		.depth = 16,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_NV16,
-		.sh_fmt = CSS_FRAME_FORMAT_NV16,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_NV16,
 		.description = "NV16, Y-plane, CbCr interleaved",
 		.planar = true
 	}, {
 		.pixelformat = V4L2_PIX_FMT_YUYV,
 		.depth = 16,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_YUYV,
-		.sh_fmt = CSS_FRAME_FORMAT_YUYV,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_YUYV,
 		.description = "YUYV, interleaved"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_UYVY,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_UYVY8_1X16,
-		.sh_fmt = CSS_FRAME_FORMAT_UYVY,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_UYVY,
 		.description = "UYVY, interleaved"
 	}, { /* This one is for parallel sensors! DO NOT USE! */
 		.pixelformat = V4L2_PIX_FMT_UYVY,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_UYVY8_2X8,
-		.sh_fmt = CSS_FRAME_FORMAT_UYVY,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_UYVY,
 		.description = "UYVY, interleaved"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SBGGR16,
 		.depth = 16,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_SBGGR16,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 16"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SBGGR8,
 		.depth = 8,
 		.mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 8"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SGBRG8,
 		.depth = 8,
 		.mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 8"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SGRBG8,
 		.depth = 8,
 		.mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 8"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SRGGB8,
 		.depth = 8,
 		.mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 8"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SBGGR10,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 10"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SGBRG10,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 10"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SGRBG10,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 10"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SRGGB10,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 10"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SBGGR12,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 12"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SGBRG12,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 12"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SGRBG12,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 12"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_SRGGB12,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12,
-		.sh_fmt = CSS_FRAME_FORMAT_RAW,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RAW,
 		.description = "Bayer 12"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_RGB32,
 		.depth = 32,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_RGB32,
-		.sh_fmt = CSS_FRAME_FORMAT_RGBA888,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RGBA888,
 		.description = "32 RGB 8-8-8-8"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_RGB565,
 		.depth = 16,
 		.mbus_code = MEDIA_BUS_FMT_BGR565_2X8_LE,
-		.sh_fmt = CSS_FRAME_FORMAT_RGB565,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_RGB565,
 		.description = "16 RGB 5-6-5"
 	}, {
 		.pixelformat = V4L2_PIX_FMT_JPEG,
 		.depth = 8,
 		.mbus_code = MEDIA_BUS_FMT_JPEG_1X8,
-		.sh_fmt = CSS_FRAME_FORMAT_BINARY_8,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_BINARY_8,
 		.description = "JPEG"
 	},
 #if 0
@@ -506,7 +505,7 @@ const struct atomisp_format_bridge atomisp_output_fmts[] = {
 		.pixelformat = V4L2_PIX_FMT_CUSTOM_M10MO_RAW,
 		.depth = 8,
 		.mbus_code = V4L2_MBUS_FMT_CUSTOM_M10MO_RAW,
-		.sh_fmt = CSS_FRAME_FORMAT_BINARY_8,
+		.sh_fmt = IA_CSS_FRAME_FORMAT_BINARY_8,
 		.description = "Custom RAW for M10MO"
 	},
 #endif
@@ -548,8 +547,8 @@ static int atomisp_querycap(struct file *file, void *fh,
 	struct video_device *vdev = video_devdata(file);
 	struct atomisp_device *isp = video_get_drvdata(vdev);
 
-	strscpy(cap->driver, DRIVER, sizeof(cap->driver) - 1);
-	strscpy(cap->card, CARD, sizeof(cap->card) - 1);
+	strscpy(cap->driver, DRIVER, sizeof(cap->driver));
+	strscpy(cap->card, CARD, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s",
 		 pci_name(isp->pdev));
 
@@ -574,8 +573,8 @@ static int atomisp_enum_input(struct file *file, void *fh,
 		return -EINVAL;
 
 	memset(input, 0, sizeof(struct v4l2_input));
-	strncpy(input->name, isp->inputs[index].camera->name,
-		sizeof(input->name) - 1);
+	strscpy(input->name, isp->inputs[index].camera->name,
+		sizeof(input->name));
 
 	/*
 	 * HACK: append actuator's name to sensor's
@@ -583,7 +582,7 @@ static int atomisp_enum_input(struct file *file, void *fh,
 	 * ioctl is the only way to enum inputs + possible external actuators
 	 * for 3A tuning purpose.
 	 */
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		motor = isp->inputs[index].motor;
 	else
 		motor = isp->motor;
@@ -594,8 +593,8 @@ static int atomisp_enum_input(struct file *file, void *fh,
 
 		if (max_size > 1) {
 			input->name[cur_len] = '+';
-			strncpy(&input->name[cur_len + 1],
-				motor->name, max_size - 1);
+			strscpy(&input->name[cur_len + 1],
+				motor->name, max_size);
 		}
 	}
 
@@ -733,7 +732,7 @@ static int atomisp_s_input(struct file *file, void *fh, unsigned int input)
 		goto error;
 	}
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		motor = isp->inputs[input].motor;
 	} else {
 		motor = isp->motor;
@@ -789,7 +788,7 @@ static int atomisp_enum_fmt_cap(struct file *file, void *fh,
 		 * Is the atomisp-supported format is valid for the
 		 * sensor (configuration)? If not, skip it.
 		 */
-		if (format->sh_fmt == CSS_FRAME_FORMAT_RAW
+		if (format->sh_fmt == IA_CSS_FRAME_FORMAT_RAW
 		    && format->mbus_code != code.code)
 			continue;
 
@@ -893,7 +892,7 @@ void atomisp_videobuf_free_buf(struct videobuf_buffer *vb)
 
 	vm_mem = vb->priv;
 	if (vm_mem && vm_mem->vaddr) {
-		atomisp_css_frame_free(vm_mem->vaddr);
+		ia_css_frame_free(vm_mem->vaddr);
 		vm_mem->vaddr = NULL;
 	}
 }
@@ -920,7 +919,7 @@ int atomisp_alloc_css_stat_bufs(struct atomisp_sub_device *asd,
 	struct atomisp_dis_buf *dis_buf = NULL, *_dis_buf;
 	struct atomisp_metadata_buf *md_buf = NULL, *_md_buf;
 	int count;
-	struct atomisp_css_dvs_grid_info *dvs_grid_info =
+	struct ia_css_dvs_grid_info *dvs_grid_info =
 	    atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info);
 	unsigned int i;
 
@@ -1024,8 +1023,8 @@ int __atomisp_reqbufs(struct file *file, void *fh,
 	struct video_device *vdev = video_devdata(file);
 	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
 	struct atomisp_sub_device *asd = pipe->asd;
-	struct atomisp_css_frame_info frame_info;
-	struct atomisp_css_frame *frame;
+	struct ia_css_frame_info frame_info;
+	struct ia_css_frame *frame;
 	struct videobuf_vmalloc_memory *vm_mem;
 	u16 source_pad = atomisp_subdev_source_pad(vdev);
 	u16 stream_id = atomisp_source_pad_to_stream_id(asd, source_pad);
@@ -1054,7 +1053,7 @@ int __atomisp_reqbufs(struct file *file, void *fh,
 	atomisp_alloc_css_stat_bufs(asd, stream_id);
 
 	/*
-	 * for user pointer type, buffers are not really allcated here,
+	 * for user pointer type, buffers are not really allocated here,
 	 * buffers are setup in QBUF operation through v4l2_buffer structure
 	 */
 	if (req->memory == V4L2_MEMORY_USERPTR)
@@ -1069,7 +1068,7 @@ int __atomisp_reqbufs(struct file *file, void *fh,
 	 * memory management function
 	 */
 	for (i = 0; i < req->count; i++) {
-		if (atomisp_css_frame_allocate_from_info(&frame, &frame_info))
+		if (ia_css_frame_allocate_from_info(&frame, &frame_info))
 			goto error;
 		vm_mem = pipe->capq.bufs[i]->priv;
 		vm_mem->vaddr = frame;
@@ -1080,11 +1079,11 @@ int __atomisp_reqbufs(struct file *file, void *fh,
 error:
 	while (i--) {
 		vm_mem = pipe->capq.bufs[i]->priv;
-		atomisp_css_frame_free(vm_mem->vaddr);
+		ia_css_frame_free(vm_mem->vaddr);
 	}
 
 	if (asd->vf_frame)
-		atomisp_css_frame_free(asd->vf_frame);
+		ia_css_frame_free(asd->vf_frame);
 
 	return -ENOMEM;
 }
@@ -1152,8 +1151,8 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 	struct atomisp_sub_device *asd = pipe->asd;
 	struct videobuf_buffer *vb;
 	struct videobuf_vmalloc_memory *vm_mem;
-	struct atomisp_css_frame_info frame_info;
-	struct atomisp_css_frame *handle = NULL;
+	struct ia_css_frame_info frame_info;
+	struct ia_css_frame *handle = NULL;
 	u32 length;
 	u32 pgnr;
 	int ret = 0;
@@ -1183,8 +1182,6 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 	 * address and reprograme out page table properly
 	 */
 	if (buf->memory == V4L2_MEMORY_USERPTR) {
-		struct hrt_userbuffer_attr attributes;
-
 		vb = pipe->capq.bufs[buf->index];
 		vm_mem = vb->priv;
 		if (!vm_mem) {
@@ -1204,45 +1201,9 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 			goto error;
 		}
 
-		attributes.pgnr = pgnr;
-		attributes.type = HRT_USR_PTR;
-#ifdef CONFIG_ION
-		if (!atomisp_hw_is_isp2401) {
-			if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_ION)
-					attributes.type = HRT_USR_ION;
-		} else {
-			if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_ION) {
-				attributes.type = HRT_USR_ION;
-				if (asd->ion_dev_fd->val !=  ION_FD_UNSET) {
-					dev_dbg(isp->dev, "ION buffer queued, share_fd=%lddev_fd=%d.\n",
-						buf->m.userptr, asd->ion_dev_fd->val);
-					/*
-					* Make sure the shared fd we just got
-					* from user space isn't larger than
-					* the space we have for it.
-					*/
-					if ((buf->m.userptr &
-					    (ATOMISP_ION_DEVICE_FD_MASK)) != 0) {
-						dev_err(isp->dev,
-							"Error: v4l2 buffer fd:0X%0lX > 0XFFFF.\n",
-							buf->m.userptr);
-						ret = -EINVAL;
-						goto error;
-					}
-					buf->m.userptr |= asd->ion_dev_fd->val <<
-							ATOMISP_ION_DEVICE_FD_OFFSET;
-				} else {
-					dev_err(isp->dev, "v4l2 buffer type is ION, \
-							but no dev fd set from userspace.\n");
-					ret = -EINVAL;
-					goto error;
-				}
-			}
-		}
-#endif
-		ret = atomisp_css_frame_map(&handle, &frame_info,
+		ret = ia_css_frame_map(&handle, &frame_info,
 					    (void __user *)buf->m.userptr,
-					    0, &attributes);
+					    0, pgnr);
 		if (ret) {
 			dev_err(isp->dev, "Failed to map user buffer\n");
 			goto error;
@@ -1250,7 +1211,7 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 
 		if (vm_mem->vaddr) {
 			mutex_lock(&pipe->capq.vb_lock);
-			atomisp_css_frame_free(vm_mem->vaddr);
+			ia_css_frame_free(vm_mem->vaddr);
 			vm_mem->vaddr = NULL;
 			vb->state = VIDEOBUF_NEEDS_INIT;
 			mutex_unlock(&pipe->capq.vb_lock);
@@ -1265,6 +1226,10 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 		buf->flags |= V4L2_BUF_FLAG_MAPPED;
 		buf->flags |= V4L2_BUF_FLAG_QUEUED;
 		buf->flags &= ~V4L2_BUF_FLAG_DONE;
+
+		/*
+		 * For mmap, frames were allocated at request buffers
+		 */
 	}
 
 done:
@@ -1299,7 +1264,7 @@ done:
 		} else {
 			atomisp_qbuffers_to_css(asd);
 
-			if (!atomisp_hw_is_isp2401) {
+			if (!IS_ISP2401) {
 				if (!atomisp_is_wdt_running(asd) && atomisp_buffers_queued(asd))
 					atomisp_wdt_start(asd);
 			} else {
@@ -1323,7 +1288,7 @@ done:
 	    pipe->capq.streaming &&
 	    !asd->enable_raw_buffer_lock->val &&
 	    asd->params.offline_parm.num_captures == 1) {
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			asd->pending_capture_request++;
 			dev_dbg(isp->dev, "Add one pending capture request.\n");
 		} else {
@@ -1401,7 +1366,7 @@ static int __get_frame_exp_id(struct atomisp_video_pipe *pipe,
 			      struct v4l2_buffer *buf)
 {
 	struct videobuf_vmalloc_memory *vm_mem;
-	struct atomisp_css_frame *handle;
+	struct ia_css_frame *handle;
 	int i;
 
 	for (i = 0; pipe->capq.bufs[i]; i++) {
@@ -1443,7 +1408,8 @@ static int atomisp_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 
 	ret = videobuf_dqbuf(&pipe->capq, buf, file->f_flags & O_NONBLOCK);
 	if (ret) {
-		dev_dbg(isp->dev, "<%s: %d\n", __func__, ret);
+		if (ret != -EAGAIN)
+			dev_dbg(isp->dev, "<%s: %d\n", __func__, ret);
 		return ret;
 	}
 	rt_mutex_lock(&isp->mutex);
@@ -1469,16 +1435,16 @@ static int atomisp_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 	return 0;
 }
 
-enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd)
+enum ia_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd)
 {
 	if (ATOMISP_USE_YUVPP(asd))
-		return CSS_PIPE_ID_YUVPP;
+		return IA_CSS_PIPE_ID_YUVPP;
 
 	if (asd->continuous_mode->val) {
 		if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO)
-			return CSS_PIPE_ID_VIDEO;
+			return IA_CSS_PIPE_ID_VIDEO;
 		else
-			return CSS_PIPE_ID_PREVIEW;
+			return IA_CSS_PIPE_ID_PREVIEW;
 	}
 
 	/*
@@ -1486,7 +1452,7 @@ enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd)
 	 * scaling but it has one frame delay due to CSS internal buffering.
 	 */
 	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER)
-		return CSS_PIPE_ID_VIDEO;
+		return IA_CSS_PIPE_ID_VIDEO;
 
 	/*
 	 * Disable vf_pp and run CSS in still capture mode. In this mode
@@ -1494,17 +1460,17 @@ enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd)
 	 * is not available.
 	 */
 	if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT)
-		return CSS_PIPE_ID_CAPTURE;
+		return IA_CSS_PIPE_ID_CAPTURE;
 
 	switch (asd->run_mode->val) {
 	case ATOMISP_RUN_MODE_PREVIEW:
-		return CSS_PIPE_ID_PREVIEW;
+		return IA_CSS_PIPE_ID_PREVIEW;
 	case ATOMISP_RUN_MODE_VIDEO:
-		return CSS_PIPE_ID_VIDEO;
+		return IA_CSS_PIPE_ID_VIDEO;
 	case ATOMISP_RUN_MODE_STILL_CAPTURE:
 	/* fall through */
 	default:
-		return CSS_PIPE_ID_CAPTURE;
+		return IA_CSS_PIPE_ID_CAPTURE;
 	}
 }
 
@@ -1598,7 +1564,7 @@ int atomisp_stream_on_master_slave_sensor(struct atomisp_device *isp,
 
 /* FIXME! ISP2400 */
 static void __wdt_on_master_slave_sensor(struct atomisp_device *isp,
-				         unsigned int wdt_duration)
+					 unsigned int wdt_duration)
 {
 	if (atomisp_buffers_queued(&isp->asd[0]))
 		atomisp_wdt_refresh(&isp->asd[0], wdt_duration);
@@ -1654,9 +1620,9 @@ static void atomisp_dma_burst_len_cfg(struct atomisp_sub_device *asd)
 				       ATOMISP_SUBDEV_PAD_SINK);
 
 	if (sink->width * sink->height >= 4096 * 3072)
-		atomisp_store_uint32(DMA_BURST_SIZE_REG, 0x7F);
+		atomisp_css2_hw_store_32(DMA_BURST_SIZE_REG, 0x7F);
 	else
-		atomisp_store_uint32(DMA_BURST_SIZE_REG, 0x00);
+		atomisp_css2_hw_store_32(DMA_BURST_SIZE_REG, 0x00);
 }
 
 /*
@@ -1669,7 +1635,7 @@ static int atomisp_streamon(struct file *file, void *fh,
 	struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev);
 	struct atomisp_sub_device *asd = pipe->asd;
 	struct atomisp_device *isp = video_get_drvdata(vdev);
-	enum atomisp_css_pipe_id css_pipe_id;
+	enum ia_css_pipe_id css_pipe_id;
 	unsigned int sensor_start_stream;
 	unsigned int wdt_duration = ATOMISP_ISP_TIMEOUT_DURATION;
 	int ret = 0;
@@ -1721,7 +1687,7 @@ static int atomisp_streamon(struct file *file, void *fh,
 
 	/* Reset pending capture request count. */
 	asd->pending_capture_request = 0;
-	if (atomisp_hw_is_isp2401)
+	if (IS_ISP2401)
 		asd->re_trigger_capture = false;
 
 	if ((atomisp_subdev_streaming_count(asd) > sensor_start_stream) &&
@@ -1793,8 +1759,7 @@ static int atomisp_streamon(struct file *file, void *fh,
 	if (asd->params.css_update_params_needed) {
 		atomisp_apply_css_parameters(asd, &asd->params.css_param);
 		if (asd->params.css_param.update_flag.dz_config)
-			atomisp_css_set_dz_config(asd,
-						  &asd->params.css_param.dz_config);
+			asd->params.config.dz_config = &asd->params.css_param.dz_config;
 		atomisp_css_update_isp_params(asd);
 		asd->params.css_update_params_needed = false;
 		memset(&asd->params.css_param.update_flag, 0,
@@ -1835,7 +1800,7 @@ start_sensor:
 	}
 
 	if (!isp->sw_contex.file_input) {
-		atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF,
+		atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF,
 				       atomisp_css_valid_sof(isp));
 		atomisp_csi2_configure(asd);
 		/*
@@ -1845,15 +1810,15 @@ start_sensor:
 		if (atomisp_streaming_count(isp) > 1) {
 			if (atomisp_freq_scaling(isp,
 						 ATOMISP_DFS_MODE_MAX, false) < 0)
-				dev_dbg(isp->dev, "dfs failed!\n");
+				dev_dbg(isp->dev, "DFS max mode failed!\n");
 		} else {
 			if (atomisp_freq_scaling(isp,
 						 ATOMISP_DFS_MODE_AUTO, false) < 0)
-				dev_dbg(isp->dev, "dfs failed!\n");
+				dev_dbg(isp->dev, "DFS auto mode failed!\n");
 		}
 	} else {
 		if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false) < 0)
-			dev_dbg(isp->dev, "dfs failed!\n");
+			dev_dbg(isp->dev, "DFS max mode failed!\n");
 	}
 
 	if (asd->depth_mode->val && atomisp_streaming_count(isp) ==
@@ -1863,7 +1828,7 @@ start_sensor:
 			dev_err(isp->dev, "master slave sensor stream on failed!\n");
 			goto out;
 		}
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			__wdt_on_master_slave_sensor(isp, wdt_duration);
 		} else {
 			__wdt_on_master_slave_sensor_pipe(pipe, wdt_duration, true);
@@ -1871,7 +1836,7 @@ start_sensor:
 		goto start_delay_wq;
 	} else if (asd->depth_mode->val && (atomisp_streaming_count(isp) <
 					    ATOMISP_DEPTH_SENSOR_STREAMON_COUNT)) {
-		if (atomisp_hw_is_isp2401)
+		if (IS_ISP2401)
 			__wdt_on_master_slave_sensor_pipe(pipe, wdt_duration, false);
 		goto start_delay_wq;
 	}
@@ -1893,7 +1858,7 @@ start_sensor:
 		goto out;
 	}
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		if (atomisp_buffers_queued(asd))
 			atomisp_wdt_refresh(asd, wdt_duration);
 	} else {
@@ -1933,7 +1898,7 @@ int __atomisp_streamoff(struct file *file, void *fh, enum v4l2_buf_type type)
 	struct atomisp_video_pipe *preview_pipe = NULL;
 	struct atomisp_video_pipe *video_pipe = NULL;
 	struct videobuf_buffer *vb, *_vb;
-	enum atomisp_css_pipe_id css_pipe_id;
+	enum ia_css_pipe_id css_pipe_id;
 	int ret;
 	unsigned long flags;
 	bool first_streamoff = false;
@@ -2039,7 +2004,7 @@ int __atomisp_streamoff(struct file *file, void *fh, enum v4l2_buf_type type)
 	atomisp_clear_css_buffer_counters(asd);
 
 	if (!isp->sw_contex.file_input)
-		atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF,
+		atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF,
 				       false);
 
 	if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED) {
@@ -2352,7 +2317,7 @@ static int atomisp_queryctl(struct file *file, void *fh,
 	case V4L2_CID_FOCUS_ABSOLUTE:
 	case V4L2_CID_FOCUS_RELATIVE:
 	case V4L2_CID_FOCUS_STATUS:
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			return v4l2_queryctrl(isp->inputs[asd->input_curr].camera->
 					    ctrl_handler, qc);
 		}
@@ -2393,7 +2358,7 @@ static int atomisp_camera_g_ext_ctrls(struct file *file, void *fh,
 	int i;
 	int ret = 0;
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		motor = isp->inputs[asd->input_curr].motor;
 	else
 		motor = isp->motor;
@@ -2502,8 +2467,7 @@ static int atomisp_camera_s_ext_ctrls(struct file *file, void *fh,
 	int i;
 	int ret = 0;
 
-
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		motor = isp->inputs[asd->input_curr].motor;
 	else
 		motor = isp->motor;
@@ -2519,7 +2483,7 @@ static int atomisp_camera_s_ext_ctrls(struct file *file, void *fh,
 		case V4L2_CID_EXPOSURE_METERING:
 		case V4L2_CID_IRIS_ABSOLUTE:
 		case V4L2_CID_FNUMBER_ABSOLUTE:
-		case V4L2_CID_VCM_TIMEING:
+		case V4L2_CID_VCM_TIMING:
 		case V4L2_CID_VCM_SLEW:
 		case V4L2_CID_3A_LOCK:
 		case V4L2_CID_TEST_PATTERN:
@@ -2732,7 +2696,7 @@ static long atomisp_vidioc_default(struct file *file, void *fh,
 	else
 		asd = atomisp_to_video_pipe(vdev)->asd;
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		motor = isp->inputs[asd->input_curr].motor;
 	else
 		motor = isp->motor;
@@ -2757,7 +2721,7 @@ static long atomisp_vidioc_default(struct file *file, void *fh,
 	}
 	switch (cmd) {
 	case ATOMISP_IOC_S_SENSOR_RUNMODE:
-		if (atomisp_hw_is_isp2401)
+		if (IS_ISP2401)
 			err = atomisp_set_sensor_runmode(asd, arg);
 		else
 			err = -EINVAL;
@@ -2932,7 +2896,7 @@ static long atomisp_vidioc_default(struct file *file, void *fh,
 				       core, ioctl, cmd, arg);
 		break;
 	case ATOMISP_IOC_G_UPDATE_EXPOSURE:
-		if (atomisp_hw_is_isp2401)
+		if (IS_ISP2401)
 			err = v4l2_subdev_call(isp->inputs[asd->input_curr].camera,
 					       core, ioctl, cmd, arg);
 		else
diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.h b/drivers/staging/media/atomisp/pci/atomisp_ioctl.h
index 5f3f2ec2539b..412bfcf33c0f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -43,7 +44,7 @@ int __atomisp_reqbufs(struct file *file, void *fh,
 int atomisp_reqbufs(struct file *file, void *fh,
 		    struct v4l2_requestbuffers *req);
 
-enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device
+enum ia_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device
 	*asd);
 
 void atomisp_videobuf_free_buf(struct videobuf_buffer *vb);
diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
index 46590129cbe3..6ba817f15655 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -30,29 +31,31 @@
 #include "atomisp_internal.h"
 
 const struct atomisp_in_fmt_conv atomisp_in_fmt_conv[] = {
-	{ MEDIA_BUS_FMT_SBGGR8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SGBRG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SGRBG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SRGGB8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_8 },
-	{ MEDIA_BUS_FMT_SBGGR10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SGBRG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SGRBG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SRGGB10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_10 },
-	{ MEDIA_BUS_FMT_SBGGR12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_SGBRG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_SGRBG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_SRGGB12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_12 },
-	{ MEDIA_BUS_FMT_UYVY8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0, ATOMISP_INPUT_FORMAT_YUV422_8 },
-	{ MEDIA_BUS_FMT_YUYV8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0, ATOMISP_INPUT_FORMAT_YUV422_8 },
-	{ MEDIA_BUS_FMT_JPEG_1X8, 8, 8, CSS_FRAME_FORMAT_BINARY_8, 0, ATOMISP_INPUT_FORMAT_BINARY_8 },
-	{ V4L2_MBUS_FMT_CUSTOM_NV12, 12, 12, CSS_FRAME_FORMAT_NV12, 0, CSS_FRAME_FORMAT_NV12 },
-	{ V4L2_MBUS_FMT_CUSTOM_NV21, 12, 12, CSS_FRAME_FORMAT_NV21, 0, CSS_FRAME_FORMAT_NV21 },
-	{ V4L2_MBUS_FMT_CUSTOM_YUV420, 12, 12, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, 0, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY },
+	{ MEDIA_BUS_FMT_SBGGR8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_BGGR },
+	{ MEDIA_BUS_FMT_SGBRG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_GBRG },
+	{ MEDIA_BUS_FMT_SGRBG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_GRBG },
+	{ MEDIA_BUS_FMT_SRGGB8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_RGGB },
+	{ MEDIA_BUS_FMT_SBGGR10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_BGGR },
+	{ MEDIA_BUS_FMT_SGBRG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_GBRG },
+	{ MEDIA_BUS_FMT_SGRBG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_GRBG },
+	{ MEDIA_BUS_FMT_SRGGB10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_RGGB },
+	{ MEDIA_BUS_FMT_SBGGR12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_BGGR },
+	{ MEDIA_BUS_FMT_SGBRG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_GBRG },
+	{ MEDIA_BUS_FMT_SGRBG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_GRBG },
+	{ MEDIA_BUS_FMT_SRGGB12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_RGGB },
+	{ MEDIA_BUS_FMT_UYVY8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0 },
+	{ MEDIA_BUS_FMT_YUYV8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0 },
+#if 0 // disabled due to clang warnings
+	{ MEDIA_BUS_FMT_JPEG_1X8, 8, 8, IA_CSS_FRAME_FORMAT_BINARY_8, 0 },
+	{ V4L2_MBUS_FMT_CUSTOM_NV12, 12, 12, IA_CSS_FRAME_FORMAT_NV12, 0 },
+	{ V4L2_MBUS_FMT_CUSTOM_NV21, 12, 12, IA_CSS_FRAME_FORMAT_NV21, 0 },
+#endif
+	{ V4L2_MBUS_FMT_CUSTOM_YUV420, 12, 12, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, 0 },
 #if 0
-	{ V4L2_MBUS_FMT_CUSTOM_M10MO_RAW, 8, 8, CSS_FRAME_FORMAT_BINARY_8, 0, ATOMISP_INPUT_FORMAT_BINARY_8 },
+	{ V4L2_MBUS_FMT_CUSTOM_M10MO_RAW, 8, 8, IA_CSS_FRAME_FORMAT_BINARY_8, 0 },
 #endif
 	/* no valid V4L2 MBUS code for metadata format, so leave it 0. */
-	{ 0, 0, 0, ATOMISP_INPUT_FORMAT_EMBEDDED, 0, ATOMISP_INPUT_FORMAT_EMBEDDED },
+	{ 0, 0, 0, ATOMISP_INPUT_FORMAT_EMBEDDED, 0 },
 	{}
 };
 
@@ -639,7 +642,7 @@ void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd,
 			atomisp_css_input_set_bayer_order(isp_sd, stream_id,
 							  fc->bayer_order);
 			atomisp_css_input_set_format(isp_sd, stream_id,
-						     fc->css_stream_fmt);
+						     fc->atomisp_in_fmt);
 			atomisp_css_set_default_isys_config(isp_sd, stream_id,
 							    ffmt);
 		}
@@ -1219,15 +1222,10 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
 	    v4l2_ctrl_new_custom(&asd->ctrl_handler,
 				 &ctrl_disable_dz,
 				 NULL);
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		asd->select_isp_version = v4l2_ctrl_new_custom(&asd->ctrl_handler,
 							       &ctrl_select_isp_version,
 							       NULL);
-#if 0 /* #ifdef CONFIG_ION */
-		asd->ion_dev_fd = v4l2_ctrl_new_custom(&asd->ctrl_handler,
-						       &ctrl_ion_dev_fd,
-						       NULL);
-#endif
 	}
 
 	/* Make controls visible on subdev as well. */
@@ -1348,8 +1346,7 @@ int atomisp_subdev_register_entities(struct atomisp_sub_device *asd,
 	 * Should any of those use V4L2_CAP_META_OUTPUT? Probably yes.
 	 */
 
-	device_caps = V4L2_CAP_IO_MC |
-		      V4L2_CAP_VIDEO_CAPTURE |
+	device_caps = V4L2_CAP_VIDEO_CAPTURE |
 		      V4L2_CAP_STREAMING;
 
 	/* Register the subdev and video node. */
diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.h b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
index b0d561224beb..330a77eed8aa 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -59,8 +60,7 @@ struct atomisp_in_fmt_conv {
 	u8 bpp; /* bits per pixel */
 	u8 depth; /* uncompressed */
 	enum atomisp_input_format atomisp_in_fmt;
-	enum atomisp_css_bayer_order bayer_order;
-	enum atomisp_input_format css_stream_fmt;
+	enum ia_css_bayer_order bayer_order;
 };
 
 struct atomisp_sub_device;
@@ -215,8 +215,8 @@ struct atomisp_subdev_params {
 	unsigned int histogram_elenum;
 
 	/* Current grid info */
-	struct atomisp_css_grid_info curr_grid_info;
-	enum atomisp_css_pipe_id s3a_enabled_pipe;
+	struct ia_css_grid_info curr_grid_info;
+	enum ia_css_pipe_id s3a_enabled_pipe;
 
 	int s3a_output_bytes;
 
@@ -225,7 +225,7 @@ struct atomisp_subdev_params {
 	struct ia_css_dz_config   dz_config;  /** Digital Zoom */
 	struct ia_css_capture_config   capture_config;
 
-	struct atomisp_css_isp_config config;
+	struct ia_css_isp_config config;
 
 	/* current configurations */
 	struct atomisp_css_params css_param;
@@ -240,7 +240,7 @@ struct atomisp_subdev_params {
 	u32 metadata_width_size;
 
 	struct ia_css_dvs2_statistics *dvs_stat;
-	struct atomisp_css_dvs_6axis *dvs_6axis;
+	struct ia_css_dvs_6axis_config *dvs_6axis;
 	u32 exp_id;
 	int  dvs_hor_coef_bytes;
 	int  dvs_ver_coef_bytes;
@@ -265,7 +265,7 @@ struct atomisp_css_params_with_list {
 };
 
 struct atomisp_acc_fw {
-	struct atomisp_css_fw_info *fw;
+	struct ia_css_fw_info *fw;
 	unsigned int handle;
 	unsigned int flags;
 	unsigned int type;
@@ -323,7 +323,7 @@ struct atomisp_sub_device {
 	struct {
 		struct list_head fw;
 		struct list_head memory_maps;
-		struct atomisp_css_pipeline *pipeline;
+		struct ia_css_pipe *pipeline;
 		bool extension_mode;
 		struct ida ida;
 		struct completion acc_done;
@@ -335,11 +335,11 @@ struct atomisp_sub_device {
 	struct atomisp_stream_env stream_env[ATOMISP_INPUT_STREAM_NUM];
 
 	struct v4l2_pix_format dvs_envelop;
-	unsigned int s3a_bufs_in_css[CSS_PIPE_ID_NUM];
+	unsigned int s3a_bufs_in_css[IA_CSS_PIPE_ID_NUM];
 	unsigned int dis_bufs_in_css;
 
 	unsigned int metadata_bufs_in_css
-	[ATOMISP_INPUT_STREAM_NUM][CSS_PIPE_ID_NUM];
+	[ATOMISP_INPUT_STREAM_NUM][IA_CSS_PIPE_ID_NUM];
 	/* The list of free and available metadata buffers for CSS */
 	struct list_head metadata[ATOMISP_METADATA_TYPE_NUM];
 	/* The list of metadata buffers which have been en-queued to CSS */
@@ -358,8 +358,8 @@ struct atomisp_sub_device {
 	struct list_head dis_stats_in_css;
 	spinlock_t dis_stats_lock;
 
-	struct atomisp_css_frame *vf_frame; /* TODO: needed? */
-	struct atomisp_css_frame *raw_output_frame;
+	struct ia_css_frame *vf_frame; /* TODO: needed? */
+	struct ia_css_frame *raw_output_frame;
 	enum atomisp_frame_status frame_status[VIDEO_MAX_FRAME];
 
 	/* This field specifies which camera (v4l2 input) is selected. */
diff --git a/drivers/staging/media/atomisp/pci/atomisp_tables.h b/drivers/staging/media/atomisp/pci/atomisp_tables.h
index 22eac8a25dba..e718a3f661f9 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_tables.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_tables.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -22,25 +23,25 @@
 #include "sh_css_params.h"
 
 /*Sepia image effect table*/
-static struct atomisp_css_cc_config sepia_cc_config = {
+static struct ia_css_cc_config sepia_cc_config = {
 	.fraction_bits  = 8,
 	.matrix	 = {141, 18, 68, -40, -5, -19, 35, 4, 16},
 };
 
 /*Negative image effect table*/
-static struct atomisp_css_cc_config nega_cc_config = {
+static struct ia_css_cc_config nega_cc_config = {
 	.fraction_bits  = 8,
 	.matrix	 = {255, 29, 120, 0, 374, 342, 0, 672, -301},
 };
 
 /*Mono image effect table*/
-static struct atomisp_css_cc_config mono_cc_config = {
+static struct ia_css_cc_config mono_cc_config = {
 	.fraction_bits  = 8,
 	.matrix	 = {255, 29, 120, 0, 0, 0, 0, 0, 0},
 };
 
 /*Skin whiten image effect table*/
-static struct atomisp_css_macc_table skin_low_macc_table = {
+static struct ia_css_macc_table skin_low_macc_table = {
 	.data = {
 		8192, 0, 0, 8192,
 		8192, 0, 0, 8192,
@@ -61,7 +62,7 @@ static struct atomisp_css_macc_table skin_low_macc_table = {
 	}
 };
 
-static struct atomisp_css_macc_table skin_medium_macc_table = {
+static struct ia_css_macc_table skin_medium_macc_table = {
 	.data = {
 		8192, 0, 0, 8192,
 		8192, 0, 0, 8192,
@@ -82,7 +83,7 @@ static struct atomisp_css_macc_table skin_medium_macc_table = {
 	}
 };
 
-static struct atomisp_css_macc_table skin_high_macc_table = {
+static struct ia_css_macc_table skin_high_macc_table = {
 	.data = {
 		8192, 0, 0, 8192,
 		8192, 0, 0, 8192,
@@ -104,7 +105,7 @@ static struct atomisp_css_macc_table skin_high_macc_table = {
 };
 
 /*Blue enhencement image effect table*/
-static struct atomisp_css_macc_table blue_macc_table = {
+static struct ia_css_macc_table blue_macc_table = {
 	.data = {
 		9728, -3072, 0, 8192,
 		8192, 0, 0, 8192,
@@ -126,7 +127,7 @@ static struct atomisp_css_macc_table blue_macc_table = {
 };
 
 /*Green enhencement image effect table*/
-static struct atomisp_css_macc_table green_macc_table = {
+static struct ia_css_macc_table green_macc_table = {
 	.data = {
 		8192, 0, 0, 8192,
 		8192, 0, 0, 8192,
@@ -147,7 +148,7 @@ static struct atomisp_css_macc_table green_macc_table = {
 	}
 };
 
-static struct atomisp_css_ctc_table vivid_ctc_table = {
+static struct ia_css_ctc_table vivid_ctc_table = {
 	.data.vamem_2 = {
 		0,  384,  837,  957, 1011, 1062, 1083, 1080,
 		1078, 1077, 1053, 1039, 1012,  992,  969,  951,
diff --git a/drivers/staging/media/atomisp/pci/atomisp_tpg.c b/drivers/staging/media/atomisp/pci/atomisp_tpg.c
index 97176b54d1ec..1def80bab180 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_tpg.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_tpg.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -147,7 +148,7 @@ int atomisp_tpg_init(struct atomisp_device *isp)
 	tpg->isp = isp;
 	v4l2_subdev_init(sd, &tpg_ops);
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	strcpy(sd->name, "tpg_subdev");
+	strscpy(sd->name, "tpg_subdev", sizeof(sd->name));
 	v4l2_set_subdevdata(sd, tpg);
 
 	pads[0].flags = MEDIA_PAD_FL_SINK;
diff --git a/drivers/staging/media/atomisp/pci/atomisp_tpg.h b/drivers/staging/media/atomisp/pci/atomisp_tpg.h
index cf492d757773..4176e076f63d 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_tpg.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_tpg.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/atomisp_trace_event.h b/drivers/staging/media/atomisp/pci/atomisp_trace_event.h
index 4d7a6794ee66..538d45e008b5 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_trace_event.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_trace_event.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support Camera Imaging tracer core.
  *
@@ -43,7 +44,7 @@ TRACE_EVENT(camera_meminfo,
 	    ),
 
 	    TP_fast_assign(
-		strlcpy(__entry->name, name, 24);
+		strscpy(__entry->name, name, 24);
 		__entry->uptr_size = uptr_size;
 		__entry->counter = counter;
 		__entry->sys_size = sys_size;
@@ -73,8 +74,8 @@ TRACE_EVENT(camera_debug,
 	    ),
 
 	    TP_fast_assign(
-		strlcpy(__entry->name, name, 24);
-		strlcpy(__entry->info, info, 24);
+		strscpy(__entry->name, name, 24);
+		strscpy(__entry->info, info, 24);
 		__entry->line = line;
 	    ),
 
diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
index 98cff13d9f93..d36809a0182c 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -42,7 +43,7 @@
 #include "hmm/hmm.h"
 #include "atomisp_trace_event.h"
 
-#include "hrt/hive_isp_css_mm_hrt.h"
+#include "sh_css_firmware.h"
 
 #include "device_access.h"
 
@@ -58,21 +59,21 @@ module_param(skip_fwload, uint, 0644);
 MODULE_PARM_DESC(skip_fwload, "Skip atomisp firmware load");
 
 /* set reserved memory pool size in page */
-static unsigned int repool_pgnr;
+static unsigned int repool_pgnr = 32768;
 module_param(repool_pgnr, uint, 0644);
 MODULE_PARM_DESC(repool_pgnr,
-		 "Set the reserved memory pool size in page (default:0)");
+		 "Set the reserved memory pool size in page (default:32768)");
 
 /* set dynamic memory pool size in page */
 unsigned int dypool_pgnr = UINT_MAX;
 module_param(dypool_pgnr, uint, 0644);
 MODULE_PARM_DESC(dypool_pgnr,
-		 "Set the dynamic memory pool size in page (default:0)");
+		 "Set the dynamic memory pool size in page (default: unlimited)");
 
-bool dypool_enable;
+bool dypool_enable = true;
 module_param(dypool_enable, bool, 0644);
 MODULE_PARM_DESC(dypool_enable,
-		 "dynamic memory pool enable/disable (default:disable)");
+		 "dynamic memory pool enable/disable (default:enabled)");
 
 /* memory optimization: deferred firmware loading */
 bool defer_fw_load;
@@ -83,7 +84,7 @@ MODULE_PARM_DESC(defer_fw_load,
 /* cross componnet debug message flag */
 int dbg_level;
 module_param(dbg_level, int, 0644);
-MODULE_PARM_DESC(dbg_level, "debug message on/off (default:off)");
+MODULE_PARM_DESC(dbg_level, "debug message level (default:0)");
 
 /* log function switch */
 int dbg_func = 2;
@@ -95,6 +96,10 @@ int mipicsi_flag;
 module_param(mipicsi_flag, int, 0644);
 MODULE_PARM_DESC(mipicsi_flag, "mipi csi compression predictor algorithm");
 
+static char firmware_name[256];
+module_param_string(firmware_name, firmware_name, sizeof(firmware_name), 0);
+MODULE_PARM_DESC(firmware_name, "Firmware file name. Allows overriding the default firmware name.");
+
 /*set to 16x16 since this is the amount of lines and pixels the sensor
 exports extra. If these are kept at the 10x8 that they were on, in yuv
 downscaling modes incorrect resolutions where requested to the sensor
@@ -119,11 +124,6 @@ MODULE_PARM_DESC(pad_h, "extra data for ISP processing");
  * be to replace this to something stored inside atomisp allocated
  * structures.
  */
-bool atomisp_hw_is_isp2401;
-
-/* Types of atomisp hardware */
-#define HW_IS_ISP2400 0
-#define HW_IS_ISP2401 1
 
 struct device *atomisp_dev;
 
@@ -349,52 +349,6 @@ static const struct atomisp_dfs_config dfs_config_byt = {
 	.dfs_table_size = ARRAY_SIZE(dfs_rules_byt),
 };
 
-static const struct atomisp_freq_scaling_rule dfs_rules_byt_cr[] = {
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_VIDEO,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_PREVIEW,
-	},
-	{
-		.width = ISP_FREQ_RULE_ANY,
-		.height = ISP_FREQ_RULE_ANY,
-		.fps = ISP_FREQ_RULE_ANY,
-		.isp_freq = ISP_FREQ_320MHZ,
-		.run_mode = ATOMISP_RUN_MODE_SDV,
-	},
-};
-
-static const struct atomisp_dfs_config dfs_config_byt_cr = {
-	.lowest_freq = ISP_FREQ_200MHZ,
-	.max_freq_at_vmin = ISP_FREQ_320MHZ,
-	.highest_freq = ISP_FREQ_320MHZ,
-	.dfs_table = dfs_rules_byt_cr,
-	.dfs_table_size = ARRAY_SIZE(dfs_rules_byt_cr),
-};
-
 static const struct atomisp_freq_scaling_rule dfs_rules_cht[] = {
 	{
 		.width = ISP_FREQ_RULE_ANY,
@@ -659,7 +613,7 @@ static int __maybe_unused atomisp_restore_iunit_reg(struct atomisp_device *isp)
 	 * which has bugs(like sighting:4567697 and 4567699) and
 	 * will be removed in B0
 	 */
-	atomisp_store_uint32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1);
+	atomisp_css2_hw_store_32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1);
 	return 0;
 }
 
@@ -689,7 +643,7 @@ static int atomisp_mrfld_pre_power_down(struct atomisp_device *isp)
 	if (!(irq & (1 << INTR_IIR)))
 		goto done;
 
-	atomisp_store_uint32(MRFLD_INTR_CLEAR_REG, 0xFFFFFFFF);
+	atomisp_css2_hw_store_32(MRFLD_INTR_CLEAR_REG, 0xFFFFFFFF);
 	atomisp_load_uint32(MRFLD_INTR_STATUS_REG, &irq);
 	if (irq != 0) {
 		dev_err(isp->dev,
@@ -704,7 +658,7 @@ static int atomisp_mrfld_pre_power_down(struct atomisp_device *isp)
 
 		pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq);
 		if (!(irq & (1 << INTR_IIR))) {
-			atomisp_store_uint32(MRFLD_INTR_ENABLE_REG, 0x0);
+			atomisp_css2_hw_store_32(MRFLD_INTR_ENABLE_REG, 0x0);
 			goto done;
 		}
 		dev_err(isp->dev,
@@ -1084,15 +1038,15 @@ static int atomisp_subdev_probe(struct atomisp_device *isp)
 	/* FIXME: should return -EPROBE_DEFER if not all subdevs were probed */
 	for (count = 0; count < SUBDEV_WAIT_TIMEOUT_MAX_COUNT; count++) {
 		int camera_count = 0;
+
 		for (subdevs = pdata->subdevs; subdevs->type; ++subdevs) {
 			if (subdevs->type == RAW_CAMERA ||
 			    subdevs->type == SOC_CAMERA)
-				camera_count ++;
+				camera_count++;
 		}
 		if (camera_count)
 			break;
 		msleep(SUBDEV_WAIT_TIMEOUT);
-		count++;
 	}
 	/* Wait more time to give more time for subdev init code to finish */
 	msleep(5 * SUBDEV_WAIT_TIMEOUT);
@@ -1143,9 +1097,9 @@ static int atomisp_subdev_probe(struct atomisp_device *isp)
 
 		switch (subdevs->type) {
 		case RAW_CAMERA:
-			raw_index = isp->input_cnt;
 			dev_dbg(isp->dev, "raw_index: %d\n", raw_index);
-			/* pass-though */
+			raw_index = isp->input_cnt;
+			/* fall through */
 		case SOC_CAMERA:
 			dev_dbg(isp->dev, "SOC_INDEX: %d\n", isp->input_cnt);
 			if (isp->input_cnt >= ATOM_ISP_MAX_INPUTS) {
@@ -1250,7 +1204,7 @@ static int atomisp_register_entities(struct atomisp_device *isp)
 
 	isp->media_dev.dev = isp->dev;
 
-	strlcpy(isp->media_dev.model, "Intel Atom ISP",
+	strscpy(isp->media_dev.model, "Intel Atom ISP",
 		sizeof(isp->media_dev.model));
 
 	media_device_init(&isp->media_dev);
@@ -1447,19 +1401,23 @@ atomisp_load_firmware(struct atomisp_device *isp)
 	if (skip_fwload)
 		return NULL;
 
-	if ((isp->media_dev.hw_revision  >> ATOMISP_HW_REVISION_SHIFT)
-	     == ATOMISP_HW_REVISION_ISP2401)
-		fw_path = "shisp_2401a0_v21.bin";
-
-	if (isp->media_dev.hw_revision ==
-	    ((ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT)
-	     | ATOMISP_HW_STEPPING_A0))
-		fw_path = "shisp_2401a0_legacy_v21.bin";
-
-	if (isp->media_dev.hw_revision ==
-	    ((ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT)
-	     | ATOMISP_HW_STEPPING_B0))
-		fw_path = "shisp_2400b0_v21.bin";
+	if (firmware_name[0] != '\0') {
+		fw_path = firmware_name;
+	} else {
+		if ((isp->media_dev.hw_revision  >> ATOMISP_HW_REVISION_SHIFT)
+		    == ATOMISP_HW_REVISION_ISP2401)
+			fw_path = "shisp_2401a0_v21.bin";
+
+		if (isp->media_dev.hw_revision ==
+		    ((ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT)
+		    | ATOMISP_HW_STEPPING_A0))
+			fw_path = "shisp_2401a0_legacy_v21.bin";
+
+		if (isp->media_dev.hw_revision ==
+		    ((ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT)
+		    | ATOMISP_HW_STEPPING_B0))
+			fw_path = "shisp_2400b0_v21.bin";
+	}
 
 	if (!fw_path) {
 		dev_err(isp->dev, "Unsupported hw_revision 0x%x\n",
@@ -1494,21 +1452,17 @@ static bool is_valid_device(struct pci_dev *dev,
 	switch (id->device & ATOMISP_PCI_DEVICE_SOC_MASK) {
 	case ATOMISP_PCI_DEVICE_SOC_MRFLD:
 		a0_max_id = ATOMISP_PCI_REV_MRFLD_A0_MAX;
-		atomisp_hw_is_isp2401 = false;
 		name = "Merrifield";
 		break;
 	case ATOMISP_PCI_DEVICE_SOC_BYT:
 		a0_max_id = ATOMISP_PCI_REV_BYT_A0_MAX;
-		atomisp_hw_is_isp2401 = false;
 		name = "Baytrail";
 		break;
 	case ATOMISP_PCI_DEVICE_SOC_ANN:
 		name = "Anniedale";
-		atomisp_hw_is_isp2401 = true;
 		break;
 	case ATOMISP_PCI_DEVICE_SOC_CHT:
 		name = "Cherrytrail";
-		atomisp_hw_is_isp2401 = true;
 		break;
 	default:
 		dev_err(&dev->dev, "%s: unknown device ID %x04:%x04\n",
@@ -1528,13 +1482,13 @@ static bool is_valid_device(struct pci_dev *dev,
 	 */
 
 #if defined(ISP2400)
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		dev_err(&dev->dev, "Support for %s (ISP2401) was disabled at compile time\n",
 			name);
 		return false;
 	}
 #else
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		dev_err(&dev->dev, "Support for %s (ISP2400) was disabled at compile time\n",
 			name);
 		return false;
@@ -1543,7 +1497,7 @@ static bool is_valid_device(struct pci_dev *dev,
 
 	dev_info(&dev->dev, "Detected %s version %d (ISP240%c) on %s\n",
 		name, dev->revision,
-		atomisp_hw_is_isp2401 ? '1' : '0',
+		IS_ISP2401 ? '1' : '0',
 		product);
 
 	return true;
@@ -1564,7 +1518,8 @@ static int init_atomisp_wdts(struct atomisp_device *isp)
 
 	for (i = 0; i < isp->num_of_streams; i++) {
 		struct atomisp_sub_device *asd = &isp->asd[i];
-		if (!atomisp_hw_is_isp2401)
+
+		if (!IS_ISP2401)
 			timer_setup(&asd->wdt, atomisp_wdt, 0);
 		else {
 			timer_setup(&asd->video_out_capture.wdt,
@@ -1670,20 +1625,29 @@ static int atomisp_pci_probe(struct pci_dev *dev,
 		    (ATOMISP_HW_REVISION_ISP2400
 		     << ATOMISP_HW_REVISION_SHIFT) |
 		    ATOMISP_HW_STEPPING_B0;
-#ifdef FIXME
-		if (INTEL_MID_BOARD(3, TABLET, BYT, BLK, PRO, CRV2) ||
-		    INTEL_MID_BOARD(3, TABLET, BYT, BLK, ENG, CRV2)) {
-			isp->dfs = &dfs_config_byt_cr;
-			isp->hpll_freq = HPLL_FREQ_2000MHZ;
-		} else
-#endif
-		{
-			isp->dfs = &dfs_config_byt;
-			isp->hpll_freq = HPLL_FREQ_1600MHZ;
-		}
-		/* HPLL frequency is known to be device-specific, but we don't
+
+		/*
+		 * Note: some Intel-based tablets with Android use a different
+		 * DFS table. Based on the comments at the Yocto Aero meta
+		 * version of this driver (at the ssid.h header), they're
+		 * identified via a "spid" var:
+		 *
+		 *	androidboot.spid=vend:cust:manu:plat:prod:hard
+		 *
+		 * As we don't have this upstream, nor we know enough details
+		 * to use a DMI or PCI match table, the old code was just
+		 * removed, but let's keep a note here as a reminder that,
+		 * for certain devices, we may need to limit the max DFS
+		 * frequency to be below certain values, adjusting the
+		 * resolution accordingly.
+		 */
+		isp->dfs = &dfs_config_byt;
+
+		/*
+		 * HPLL frequency is known to be device-specific, but we don't
 		 * have specs yet for exactly how it varies.  Default to
-		 * BYT-CR but let provisioning set it via EFI variable */
+		 * BYT-CR but let provisioning set it via EFI variable
+		 */
 		isp->hpll_freq = gmin_get_var_int(&dev->dev, false, "HpllFreq",
 						  HPLL_FREQ_2000MHZ);
 
@@ -1735,7 +1699,7 @@ static int atomisp_pci_probe(struct pci_dev *dev,
 		default:
 			isp->hpll_freq = HPLL_FREQ_1600MHZ;
 			dev_warn(isp->dev,
-				 "read HPLL from cck failed.default 1600MHz.\n");
+				 "read HPLL from cck failed. Default to 1600 MHz.\n");
 		}
 		break;
 	default:
@@ -1758,7 +1722,8 @@ static int atomisp_pci_probe(struct pci_dev *dev,
 			goto load_fw_fail;
 		}
 
-		err = atomisp_css_check_firmware_version(isp);
+		err = sh_css_check_firmware_version(isp->dev,
+						    isp->firmware->data);
 		if (err) {
 			dev_dbg(&dev->dev, "Firmware version check failed\n");
 			goto fw_validation_fail;
@@ -1787,7 +1752,7 @@ static int atomisp_pci_probe(struct pci_dev *dev,
 	 * bugs(like sighting:4567697 and 4567699) and will be removed
 	 * in B0
 	 */
-	atomisp_store_uint32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1);
+	atomisp_css2_hw_store_32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1);
 
 	if ((id->device & ATOMISP_PCI_DEVICE_SOC_MASK) ==
 	    ATOMISP_PCI_DEVICE_SOC_MRFLD) {
@@ -1938,7 +1903,7 @@ static void atomisp_pci_remove(struct pci_dev *dev)
 
 	atomisp_acc_cleanup(isp);
 
-	atomisp_css_unload_firmware(isp);
+	ia_css_unload_firmware();
 	hmm_cleanup();
 
 	pm_runtime_forbid(&dev->dev);
diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.h b/drivers/staging/media/atomisp/pci/atomisp_v4l2.h
index 87881fa6a698..81bb356b8172 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
index 789a2e68cab8..0579deac5535 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h
index 09b049b3bd15..6fa6da859158 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
index 47c488cec8ad..1071813a284c 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c b/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c
index 78e98268e188..d9f7c143794d 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h b/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h
index 8cf3b0e0cc39..78cf0cbfb3be 100644
--- a/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h
+++ b/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -16,17 +17,18 @@
 #define _IA_CSS_REFCOUNT_H_
 
 #include <type_support.h>
-#include <system_types.h>
+#include <system_local.h>
 #include <ia_css_err.h>
+#include <ia_css_types.h>
 
-typedef void (*clear_func)(hrt_vaddress ptr);
+typedef void (*clear_func)(ia_css_ptr ptr);
 
 /*! \brief Function for initializing refcount list
  *
  * \param[in]	size		Size of the refcount list.
  * \return				ia_css_err
  */
-enum ia_css_err ia_css_refcount_init(uint32_t size);
+int ia_css_refcount_init(uint32_t size);
 
 /*! \brief Function for de-initializing refcount list
  *
@@ -38,9 +40,9 @@ void ia_css_refcount_uninit(void);
  *
  * \param[in]	id		ID of the object.
  * \param[in]	ptr		Data of the object (ptr).
- * \return				hrt_vaddress (saved address)
+ * \return				ia_css_ptr (saved address)
  */
-hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr);
+ia_css_ptr ia_css_refcount_increment(s32 id, ia_css_ptr ptr);
 
 /*! \brief Function for decrease reference by 1.
  *
@@ -50,7 +52,7 @@ hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr);
  *	- true, if it is successful.
  *	- false, otherwise.
  */
-bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr);
+bool ia_css_refcount_decrement(s32 id, ia_css_ptr ptr);
 
 /*! \brief Function to check if reference count is 1.
  *
@@ -59,7 +61,7 @@ bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr);
  *	- true, if it is successful.
  *	- false, otherwise.
  */
-bool ia_css_refcount_is_single(hrt_vaddress ptr);
+bool ia_css_refcount_is_single(ia_css_ptr ptr);
 
 /*! \brief Function to clear reference list objects.
  *
@@ -78,6 +80,6 @@ void ia_css_refcount_clear(s32 id,
  *      - true, if valid
  *      - false, if invalid
  */
-bool ia_css_refcount_is_valid(hrt_vaddress ptr);
+bool ia_css_refcount_is_valid(ia_css_ptr ptr);
 
 #endif /* _IA_CSS_REFCOUNT_H_ */
diff --git a/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c b/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c
index e39cc2132953..cf02737cf8d4 100644
--- a/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c
+++ b/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,8 +13,9 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include "ia_css_refcount.h"
-#include "memory_access/memory_access.h"
 #include "sh_css_defs.h"
 
 #include "platform_support.h"
@@ -23,10 +25,10 @@
 #include "ia_css_debug.h"
 
 /* TODO: enable for other memory aswell
-	 now only for hrt_vaddress */
+	 now only for ia_css_ptr */
 struct ia_css_refcount_entry {
 	u32 count;
-	hrt_vaddress data;
+	ia_css_ptr data;
 	s32 id;
 };
 
@@ -37,7 +39,7 @@ struct ia_css_refcount_list {
 
 static struct ia_css_refcount_list myrefcount;
 
-static struct ia_css_refcount_entry *refcount_find_entry(hrt_vaddress ptr,
+static struct ia_css_refcount_entry *refcount_find_entry(ia_css_ptr ptr,
 	bool firstfree)
 {
 	u32 i;
@@ -46,7 +48,7 @@ static struct ia_css_refcount_entry *refcount_find_entry(hrt_vaddress ptr,
 		return NULL;
 	if (!myrefcount.items) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				    "refcount_find_entry(): Ref count not initiliazed!\n");
+				    "refcount_find_entry(): Ref count not initialized!\n");
 		return NULL;
 	}
 
@@ -65,25 +67,25 @@ static struct ia_css_refcount_entry *refcount_find_entry(hrt_vaddress ptr,
 	return NULL;
 }
 
-enum ia_css_err ia_css_refcount_init(uint32_t size)
+int ia_css_refcount_init(uint32_t size)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	if (size == 0) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_refcount_init(): Size of 0 for Ref count init!\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	if (myrefcount.items) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_refcount_init(): Ref count is already initialized\n");
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
 	myrefcount.items =
-	    sh_css_malloc(sizeof(struct ia_css_refcount_entry) * size);
+	    kvmalloc(sizeof(struct ia_css_refcount_entry) * size, GFP_KERNEL);
 	if (!myrefcount.items)
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	if (err == IA_CSS_SUCCESS) {
+		err = -ENOMEM;
+	if (!err) {
 		memset(myrefcount.items, 0,
 		       sizeof(struct ia_css_refcount_entry) * size);
 		myrefcount.size = size;
@@ -114,14 +116,14 @@ void ia_css_refcount_uninit(void)
 			entry->id = 0;
 		}
 	}
-	sh_css_free(myrefcount.items);
+	kvfree(myrefcount.items);
 	myrefcount.items = NULL;
 	myrefcount.size = 0;
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_refcount_uninit() leave\n");
 }
 
-hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr)
+ia_css_ptr ia_css_refcount_increment(s32 id, ia_css_ptr ptr)
 {
 	struct ia_css_refcount_entry *entry;
 
@@ -158,7 +160,7 @@ hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr)
 	return ptr;
 }
 
-bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr)
+bool ia_css_refcount_decrement(s32 id, ia_css_ptr ptr)
 {
 	struct ia_css_refcount_entry *entry;
 
@@ -201,7 +203,7 @@ bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr)
 	return false;
 }
 
-bool ia_css_refcount_is_single(hrt_vaddress ptr)
+bool ia_css_refcount_is_single(ia_css_ptr ptr)
 {
 	struct ia_css_refcount_entry *entry;
 
@@ -262,7 +264,7 @@ void ia_css_refcount_clear(s32 id, clear_func clear_func_ptr)
 			    count);
 }
 
-bool ia_css_refcount_is_valid(hrt_vaddress ptr)
+bool ia_css_refcount_is_valid(ia_css_ptr ptr)
 {
 	struct ia_css_refcount_entry *entry;
 
diff --git a/drivers/staging/media/atomisp/pci/bits.h b/drivers/staging/media/atomisp/pci/bits.h
index c6d2a5cba213..9fab02ebddc5 100644
--- a/drivers/staging/media/atomisp/pci/bits.h
+++ b/drivers/staging/media/atomisp/pci/bits.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h
index 551e8d7c5003..965cfda50707 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -59,10 +60,10 @@ void ia_css_pipe_get_vfpp_binarydesc(
  *		(= The numerator member in the sh_css_bds_factor structure.)
  * @param[out] bds_factor_denominator: The denominator of the bayer downscaling factor.
  *		(= The denominator member in the sh_css_bds_factor structure.)
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err sh_css_bds_factor_get_numerator_denominator(
+int sh_css_bds_factor_get_numerator_denominator(
     unsigned int bds_factor,
     unsigned int *bds_factor_numerator,
     unsigned int *bds_factor_denominator);
@@ -75,10 +76,10 @@ enum ia_css_err sh_css_bds_factor_get_numerator_denominator(
  * @param[in/out] bds_out_info
  * @param[in/out] out_info
  * @param[in/out] vf_info
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_pipe_get_preview_binarydesc(
+int ia_css_pipe_get_preview_binarydesc(
     struct ia_css_pipe *const pipe,
     struct ia_css_binary_descr *preview_descr,
     struct ia_css_frame_info *in_info,
@@ -93,10 +94,10 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc(
  * @param[in/out] in_info
  * @param[in/out] bds_out_info
  * @param[in/out] vf_info
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_pipe_get_video_binarydesc(
+int ia_css_pipe_get_video_binarydesc(
     struct ia_css_pipe *const pipe,
     struct ia_css_binary_descr *video_descr,
     struct ia_css_frame_info *in_info,
@@ -287,9 +288,9 @@ void ia_css_pipe_get_ldc_binarydesc(
  * @param[in] input_res
  * @param[in] output_res
  * @param[in/out] bds_factor
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  */
-enum ia_css_err binarydesc_calculate_bds_factor(
+int binarydesc_calculate_bds_factor(
     struct ia_css_resolution input_res,
     struct ia_css_resolution output_res,
     unsigned int *bds_factor);
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h
index e58c9190310d..40c8145a0797 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h
index ad60210abe95..c23d1bd915a3 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c
index c6b07d65ce3e..f20c9b02fbe0 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -141,7 +142,7 @@ static struct sh_css_bds_factor bds_factors_list[] = {
 	{8, 1, SH_CSS_BDS_FACTOR_8_00}
 };
 
-enum ia_css_err sh_css_bds_factor_get_numerator_denominator(
+int sh_css_bds_factor_get_numerator_denominator(
     unsigned int bds_factor,
     unsigned int *bds_factor_numerator,
     unsigned int *bds_factor_denominator)
@@ -153,16 +154,16 @@ enum ia_css_err sh_css_bds_factor_get_numerator_denominator(
 		if (bds_factors_list[i].bds_factor == bds_factor) {
 			*bds_factor_numerator = bds_factors_list[i].numerator;
 			*bds_factor_denominator = bds_factors_list[i].denominator;
-			return IA_CSS_SUCCESS;
+			return 0;
 		}
 	}
 
 	/* Throw an error since bds_factor cannot be found
 	in bds_factors_list */
-	return IA_CSS_ERR_INVALID_ARGUMENTS;
+	return -EINVAL;
 }
 
-enum ia_css_err binarydesc_calculate_bds_factor(
+int binarydesc_calculate_bds_factor(
     struct ia_css_resolution input_res,
     struct ia_css_resolution output_res,
     unsigned int *bds_factor)
@@ -195,15 +196,15 @@ enum ia_css_err binarydesc_calculate_bds_factor(
 
 		if (cond) {
 			*bds_factor = bds_factors_list[i].bds_factor;
-			return IA_CSS_SUCCESS;
+			return 0;
 		}
 	}
 
 	/* Throw an error since a suitable bds_factor cannot be found */
-	return IA_CSS_ERR_INVALID_ARGUMENTS;
+	return -EINVAL;
 }
 
-enum ia_css_err ia_css_pipe_get_preview_binarydesc(
+int ia_css_pipe_get_preview_binarydesc(
     struct ia_css_pipe *const pipe,
     struct ia_css_binary_descr *preview_descr,
     struct ia_css_frame_info *in_info,
@@ -211,7 +212,7 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc(
     struct ia_css_frame_info *out_info,
     struct ia_css_frame_info *vf_info)
 {
-	enum ia_css_err err;
+	int err;
 	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
 	int mode = IA_CSS_BINARY_MODE_PREVIEW;
 	unsigned int i;
@@ -264,7 +265,7 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc(
 			    binarydesc_calculate_bds_factor(in_info->res,
 							    bds_out_info->res,
 							    &preview_descr->required_bds_factor);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				return err;
 		} else {
 			bds_out_info->res.width = in_info->res.width / 2;
@@ -318,11 +319,11 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc(
 	preview_descr->enable_dpc = pipe->config.enable_dpc;
 
 	preview_descr->isp_pipe_version = pipe->config.isp_pipe_version;
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
-enum ia_css_err ia_css_pipe_get_video_binarydesc(
+int ia_css_pipe_get_video_binarydesc(
     struct ia_css_pipe *const pipe,
     struct ia_css_binary_descr *video_descr,
     struct ia_css_frame_info *in_info,
@@ -334,7 +335,7 @@ enum ia_css_err ia_css_pipe_get_video_binarydesc(
 	int mode = IA_CSS_BINARY_MODE_VIDEO;
 	unsigned int i;
 	struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool stream_dz_config = false;
 
 	/* vf_info can be NULL */
@@ -407,7 +408,7 @@ enum ia_css_err ia_css_pipe_get_video_binarydesc(
 				    binarydesc_calculate_bds_factor(
 					in_info->res, bds_out_info->res,
 					&video_descr->required_bds_factor);
-				if (err != IA_CSS_SUCCESS)
+				if (err)
 					return err;
 			} else {
 				bds_out_info->res.width =
@@ -607,7 +608,7 @@ void ia_css_pipe_get_primary_binarydesc(
 		 * since it has better performance. */
 		if (pipe_version == IA_CSS_PIPE_VERSION_2_6_1)
 			prim_descr->striped = false;
-		else if (!atomisp_hw_is_isp2401) {
+		else if (!IS_ISP2401) {
 			prim_descr->striped = prim_descr->continuous &&
 					      (!pipe->stream->stop_copy_preview || !pipe->stream->disable_cont_vf);
 		} else {
@@ -848,7 +849,7 @@ void ia_css_pipe_get_ldc_binarydesc(
 	assert(out_info);
 	IA_CSS_ENTER_PRIVATE("");
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		*in_info = *out_info;
 	} else {
 		if (pipe->out_yuv_ds_input_info.res.width)
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c
index 43f63cc20f49..82a24aabe8ce 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c
index cc0631550724..03d9d168fcc9 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h b/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h
index 75333166ed9b..59df44d696a0 100644
--- a/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h
+++ b/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,8 +16,9 @@
 #ifndef __IA_CSS_UTIL_H__
 #define __IA_CSS_UTIL_H__
 
+#include <linux/errno.h>
+
 #include <ia_css_err.h>
-#include <error_support.h>
 #include <type_support.h>
 #include <ia_css_frame_public.h>
 #include <ia_css_stream_public.h>
@@ -28,26 +30,26 @@
  * @return	"ia_css_err" error code
  *
  */
-enum ia_css_err ia_css_convert_errno(
+int ia_css_convert_errno(
     int in_err);
 
 /* @brief check vf frame info.
  *
  * @param[in] info
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_util_check_vf_info(
+int ia_css_util_check_vf_info(
     const struct ia_css_frame_info *const info);
 
 /* @brief check input configuration.
  *
  * @param[in] stream_config
  * @param[in] must_be_raw
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_util_check_input(
+int ia_css_util_check_input(
     const struct ia_css_stream_config *const stream_config,
     bool must_be_raw,
     bool must_be_yuv);
@@ -56,10 +58,10 @@ enum ia_css_err ia_css_util_check_input(
  *
  * @param[in] out_info
  * @param[in] vf_info
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_util_check_vf_out_info(
+int ia_css_util_check_vf_out_info(
     const struct ia_css_frame_info *const out_info,
     const struct ia_css_frame_info *const vf_info);
 
@@ -67,10 +69,10 @@ enum ia_css_err ia_css_util_check_vf_out_info(
  *
  * @param[in] width
  * @param[in] height
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_util_check_res(
+int ia_css_util_check_res(
     unsigned int width,
     unsigned int height);
 
diff --git a/drivers/staging/media/atomisp/pci/camera/util/src/util.c b/drivers/staging/media/atomisp/pci/camera/util/src/util.c
index 217fe9cb54ff..40a71e37cc4e 100644
--- a/drivers/staging/media/atomisp/pci/camera/util/src/util.c
+++ b/drivers/staging/media/atomisp/pci/camera/util/src/util.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -20,35 +21,6 @@
 /* for ia_css_binary_max_vf_width() */
 #include "ia_css_binary.h"
 
-enum ia_css_err ia_css_convert_errno(
-    int in_err)
-{
-	enum ia_css_err out_err;
-
-	switch (in_err) {
-	case 0:
-		out_err = IA_CSS_SUCCESS;
-		break;
-	case EINVAL:
-		out_err = IA_CSS_ERR_INVALID_ARGUMENTS;
-		break;
-	case ENODATA:
-		out_err = IA_CSS_ERR_QUEUE_IS_EMPTY;
-		break;
-	case ENOSYS:
-	case ENOTSUP:
-		out_err = IA_CSS_ERR_INTERNAL_ERROR;
-		break;
-	case ENOBUFS:
-		out_err = IA_CSS_ERR_QUEUE_IS_FULL;
-		break;
-	default:
-		out_err = IA_CSS_ERR_INTERNAL_ERROR;
-		break;
-	}
-	return out_err;
-}
-
 /* MW: Table look-up ??? */
 unsigned int ia_css_util_input_format_bpp(
     enum atomisp_input_format format,
@@ -113,49 +85,49 @@ unsigned int ia_css_util_input_format_bpp(
 	return rval;
 }
 
-enum ia_css_err ia_css_util_check_vf_info(
+int ia_css_util_check_vf_info(
     const struct ia_css_frame_info *const info)
 {
-	enum ia_css_err err;
+	int err;
 	unsigned int max_vf_width;
 
 	assert(info);
 	err = ia_css_frame_check_info(info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	max_vf_width = ia_css_binary_max_vf_width();
 	if (max_vf_width != 0 && info->res.width > max_vf_width * 2)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	return IA_CSS_SUCCESS;
+		return -EINVAL;
+	return 0;
 }
 
-enum ia_css_err ia_css_util_check_vf_out_info(
+int ia_css_util_check_vf_out_info(
     const struct ia_css_frame_info *const out_info,
     const struct ia_css_frame_info *const vf_info)
 {
-	enum ia_css_err err;
+	int err;
 
 	assert(out_info);
 	assert(vf_info);
 
 	err = ia_css_frame_check_info(out_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	err = ia_css_util_check_vf_info(vf_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err ia_css_util_check_res(unsigned int width, unsigned int height)
+int ia_css_util_check_res(unsigned int width, unsigned int height)
 {
 	/* height can be odd number for jpeg/embedded data from ISYS2401 */
 	if (((width  == 0)   ||
 	     (height == 0)   ||
 	     IS_ODD(width))) {
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /* ISP2401 */
@@ -200,7 +172,7 @@ bool ia_css_util_is_input_format_yuv(enum atomisp_input_format format)
 	       format == ATOMISP_INPUT_FORMAT_YUV422_16;
 }
 
-enum ia_css_err ia_css_util_check_input(
+int ia_css_util_check_input(
     const struct ia_css_stream_config *const stream_config,
     bool must_be_raw,
     bool must_be_yuv)
@@ -208,18 +180,18 @@ enum ia_css_err ia_css_util_check_input(
 	assert(stream_config);
 
 	if (!stream_config)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if (stream_config->input_config.effective_res.width == 0 ||
 	    stream_config->input_config.effective_res.height == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	if (must_be_raw &&
 	    !ia_css_util_is_input_format_raw(stream_config->input_config.format))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if (must_be_yuv &&
 	    !ia_css_util_is_input_format_yuv(stream_config->input_config.format))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/cell_params.h b/drivers/staging/media/atomisp/pci/cell_params.h
index 0eabc59ff5af..3c21a18990ba 100644
--- a/drivers/staging/media/atomisp/pci/cell_params.h
+++ b/drivers/staging/media/atomisp/pci/cell_params.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c
index 3ef556a64825..1a021ae841fe 100644
--- a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c
+++ b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c
index 2b90a7075b9b..b786247b322b 100644
--- a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c
+++ b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c
index 42e0344c677d..a6bc2e9eddea 100644
--- a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c
+++ b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -210,7 +211,7 @@ ia_css_initialize_ynr_state(
 
 /* Code generated by genparam/genstate.c:gen_state_init_table() */
 
-void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(
+void (*ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(
     const struct ia_css_binary *binary) = {
 	ia_css_initialize_aa_state,
 	ia_css_initialize_cnr_state,
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h
index 4de5bb81bd23..3aabd0248e4f 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c
index 29d85407cac4..1a021ae841fe 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -272,7 +273,6 @@ ia_css_configure_output(
 			    "ia_css_configure_output() leave:\n");
 }
 
-
 /* Code generated by genparam/genconfig.c:gen_configure_function() */
 
 void
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c
index 68297296885e..d9c672d8904e 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c
index c54787f3fc24..514ffe0303cb 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -210,7 +211,7 @@ ia_css_initialize_ynr_state(
 
 /* Code generated by genparam/genstate.c:gen_state_init_table() */
 
-void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(
+void (*ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(
     const struct ia_css_binary *binary) = {
 	ia_css_initialize_aa_state,
 	ia_css_initialize_cnr_state,
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c
index 50080565d0d6..8e661091f7d9 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h
index a86de89b2cfc..6489ee644a4a 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h
index 3fa3c3a487ab..ece45d80eb2a 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -289,12 +290,12 @@ static inline void csi_rx_be_ctrl_dump_state(
 	 * lut.
 	 */
 	for (i = 0; i < N_SHORT_PACKET_LUT_ENTRIES[ID]; i++) {
-		ia_css_print("CSI RX BE STATE Controller ID %d Short packat entry %d shart packet lut id 0x%x\n",
+		ia_css_print("CSI RX BE STATE Controller ID %d Short packet entry %d short packet lut id 0x%x\n",
 			     ID, i,
 			     state->short_packet_lut_entry[i]);
 	}
 	for (i = 0; i < N_LONG_PACKET_LUT_ENTRIES[ID]; i++) {
-		ia_css_print("CSI RX BE STATE Controller ID %d Long packat entry %d Long packet lut id 0x%x\n",
+		ia_css_print("CSI RX BE STATE Controller ID %d Long packet entry %d long packet lut id 0x%x\n",
 			     ID, i,
 			     state->long_packet_lut_entry[i]);
 	}
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c
index 8b06b2410d1d..58fec54a914d 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h
index ea40284623d1..4952b42d8191 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h
index a0800a5df68a..a58e8477da6e 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c
index 36c026cbd7cc..5809dbb6e5aa 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h
index 5c694a26386e..878933261a43 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h
index a1a222372ed3..eb35b7bcead4 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c
index 567c926bd47f..99576af4713c 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h
index 4fd05b29dfdb..e3d6d5e1634e 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h
index c519e6f06462..91ef000d76dc 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c
index 67570138ba24..b7d893aea88d 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h
index 1449c19abc86..4fbbcc2338d3 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
index e5aae5c022eb..4a5646a229b8 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h
index 24f4da9aef40..efaa4da8d36d 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h
index 65ea23604479..4faa519219ee 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -97,9 +98,9 @@ STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_dump_state(
     pixelgen_ctrl_state_t *state)
 {
 	ia_css_print("Pixel Generator ID %d Enable  0x%x\n", ID, state->com_enable);
-	ia_css_print("Pixel Generator ID %d PRBS reset vlue 0 0x%x\n", ID,
+	ia_css_print("Pixel Generator ID %d PRBS reset value 0 0x%x\n", ID,
 		     state->prbs_rstval0);
-	ia_css_print("Pixel Generator ID %d PRBS reset vlue 1 0x%x\n", ID,
+	ia_css_print("Pixel Generator ID %d PRBS reset value 1 0x%x\n", ID,
 		     state->prbs_rstval1);
 	ia_css_print("Pixel Generator ID %d SYNC SID 0x%x\n", ID, state->syng_sid);
 	ia_css_print("Pixel Generator ID %d syng free run 0x%x\n", ID,
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h
index ce53ba4837ea..ae471dd51737 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h
index 5975b094a9d0..374466e6b7bf 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h
index 84fe95c16404..ac8be2d49227 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h
index 45f20b524368..6fae1c262446 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h
index a8d0dbd7f6d7..d0e5b54d1afc 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h
index a3940d246890..e17783f96b23 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h
index dc8d091c6769..1b9f03d57659 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h
index 2ca4d5210a38..f423f34134d3 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -46,6 +47,7 @@ struct isys2401_dma_port_cfg_s {
 	u32 cropping;
 	u32 width;
 };
+
 /* end of DMA Port */
 
 /************************************************
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h
index 41d051db3987..156b4c95277e 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h
index bcb46b293b6a..0611047eabbc 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h
index cde599c5d0d2..75722ef572d0 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h b/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h
index 99d292164efc..d2c39f9600bd 100644
--- a/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h b/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h
index f4b2b41b6d94..180ff7cd9ff5 100644
--- a/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h
+++ b/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_trace.h b/drivers/staging/media/atomisp/pci/css_trace.h
index 32520c21c324..d2ce50fcfb7a 100644
--- a/drivers/staging/media/atomisp/pci/css_trace.h
+++ b/drivers/staging/media/atomisp/pci/css_trace.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -171,7 +172,6 @@ enum TRACE_DUMP_FORMAT {
 #define TRACE_ISP_DATA_SIZE   (TRACE_ISP_SIZE - TRACE_ISP_HEADER_SIZE)
 #define TRACE_ISP_MAX_POINTS  (TRACE_ISP_DATA_SIZE / TRACE_ISP_ITEM_SIZE)
 
-
 /* common majors */
 /* SP0 */
 #define MAJOR_MAIN              1
diff --git a/drivers/staging/media/atomisp/pci/defs.h b/drivers/staging/media/atomisp/pci/defs.h
index 47505f41790c..785e7a670a00 100644
--- a/drivers/staging/media/atomisp/pci/defs.h
+++ b/drivers/staging/media/atomisp/pci/defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/dma_v2_defs.h b/drivers/staging/media/atomisp/pci/dma_v2_defs.h
index 8741b8347dd4..27299e3a185d 100644
--- a/drivers/staging/media/atomisp/pci/dma_v2_defs.h
+++ b/drivers/staging/media/atomisp/pci/dma_v2_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/gdc_v2_defs.h b/drivers/staging/media/atomisp/pci/gdc_v2_defs.h
index 3cc627aa6b09..804df8179e36 100644
--- a/drivers/staging/media/atomisp/pci/gdc_v2_defs.h
+++ b/drivers/staging/media/atomisp/pci/gdc_v2_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/gp_timer_defs.h b/drivers/staging/media/atomisp/pci/gp_timer_defs.h
index ffd7b38fce9d..9bc04e5b4292 100644
--- a/drivers/staging/media/atomisp/pci/gp_timer_defs.h
+++ b/drivers/staging/media/atomisp/pci/gp_timer_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/gpio_block_defs.h b/drivers/staging/media/atomisp/pci/gpio_block_defs.h
index 96286a141b00..e1bd638d344a 100644
--- a/drivers/staging/media/atomisp/pci/gpio_block_defs.h
+++ b/drivers/staging/media/atomisp/pci/gpio_block_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_2401_irq_types_hrt.h b/drivers/staging/media/atomisp/pci/hive_isp_css_2401_irq_types_hrt.h
deleted file mode 100644
index 0760b95818f6..000000000000
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_2401_irq_types_hrt.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_
-#define _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_
-
-/*
- * These are the indices of each interrupt in the interrupt
- * controller's registers. these can be used as the irq_id
- * argument to the hrt functions irq_controller.h.
- *
- * The definitions are taken from <system>_defs.h
- */
-typedef enum hrt_isp_css_irq {
-	hrt_isp_css_irq_gpio_pin_0           = HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_1           = HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_2           = HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_3           = HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_4           = HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_5           = HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_6           = HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_7           = HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_8           = HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_9           = HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_10          = HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID,
-	hrt_isp_css_irq_gpio_pin_11          = HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID,
-	hrt_isp_css_irq_sp                   = HIVE_GP_DEV_IRQ_SP_BIT_ID,
-	hrt_isp_css_irq_isp                  = HIVE_GP_DEV_IRQ_ISP_BIT_ID,
-	hrt_isp_css_irq_isys                 = HIVE_GP_DEV_IRQ_ISYS_BIT_ID,
-	hrt_isp_css_irq_isel                 = HIVE_GP_DEV_IRQ_ISEL_BIT_ID,
-	hrt_isp_css_irq_ifmt                 = HIVE_GP_DEV_IRQ_IFMT_BIT_ID,
-	hrt_isp_css_irq_sp_stream_mon        = HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID,
-	hrt_isp_css_irq_isp_stream_mon       = HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID,
-	hrt_isp_css_irq_mod_stream_mon       = HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID,
-	hrt_isp_css_irq_is2401               = HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID,
-	hrt_isp_css_irq_isp_bamem_error      = HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID,
-	hrt_isp_css_irq_isp_dmem_error       = HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID,
-	hrt_isp_css_irq_sp_icache_mem_error  = HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID,
-	hrt_isp_css_irq_sp_dmem_error        = HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID,
-	hrt_isp_css_irq_mmu_cache_mem_error  = HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID,
-	hrt_isp_css_irq_gp_timer_0           = HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID,
-	hrt_isp_css_irq_gp_timer_1           = HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID,
-	hrt_isp_css_irq_sw_pin_0             = HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID,
-	hrt_isp_css_irq_sw_pin_1             = HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID,
-	hrt_isp_css_irq_dma                  = HIVE_GP_DEV_IRQ_DMA_BIT_ID,
-	hrt_isp_css_irq_sp_stream_mon_b      = HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID,
-	/* this must (obviously) be the last on in the enum */
-	hrt_isp_css_irq_num_irqs
-} hrt_isp_css_irq_t;
-
-typedef enum hrt_isp_css_irq_status {
-	hrt_isp_css_irq_status_error,
-	hrt_isp_css_irq_status_more_irqs,
-	hrt_isp_css_irq_status_success
-} hrt_isp_css_irq_status_t;
-
-#endif /* _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_ */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h
index 7580cf5c9624..b6538beca18a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h
index 85d509f5b923..135034c7245a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h
index 4df7a405cdcf..a50635b717e3 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h
index f43bf0ad2468..d941c82d530b 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h
index f3ce9e9f1ad4..599d993b832e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h
index 1c1b0667a53b..c8f416515e2c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h
index ee636ad6c5b3..163003f2c759 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h
index a82ca2a8cada..b5f017482f89 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h
index e4b9daa2d062..746b07097681 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c
index d911aec24185..a502ba9f8c7f 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2016, Intel Corporation.
@@ -14,12 +15,12 @@
 
 #include "debug.h"
 
+#include "hmm.h"
+
 #ifndef __INLINE_DEBUG__
 #include "debug_private.h"
 #endif /* __INLINE_DEBUG__ */
 
-#include "memory_access.h"
-
 #define __INLINE_SP__
 #include "sp.h"
 
@@ -27,7 +28,7 @@
 
 /* The address of the remote copy */
 hrt_address	debug_buffer_address = (hrt_address) - 1;
-hrt_vaddress	debug_buffer_ddr_address = (hrt_vaddress)-1;
+ia_css_ptr	debug_buffer_ddr_address = (ia_css_ptr)-1;
 /* The local copy */
 static debug_data_t		debug_data;
 debug_data_t		*debug_data_ptr = &debug_data;
@@ -40,7 +41,7 @@ void debug_buffer_init(const hrt_address addr)
 	debug_data.tail = 0;
 }
 
-void debug_buffer_ddr_init(const hrt_vaddress addr)
+void debug_buffer_ddr_init(const ia_css_ptr addr)
 {
 	debug_buf_mode_t mode = DEBUG_BUFFER_MODE_LINEAR;
 	u32 enable = 1;
@@ -48,13 +49,13 @@ void debug_buffer_ddr_init(const hrt_vaddress addr)
 	u32 tail = 0;
 	/* set the ddr queue */
 	debug_buffer_ddr_address = addr;
-	mmgr_store(addr + DEBUG_DATA_BUF_MODE_DDR_ADDR,
+	hmm_store(addr + DEBUG_DATA_BUF_MODE_DDR_ADDR,
 		   &mode, sizeof(debug_buf_mode_t));
-	mmgr_store(addr + DEBUG_DATA_HEAD_DDR_ADDR,
+	hmm_store(addr + DEBUG_DATA_HEAD_DDR_ADDR,
 		   &head, sizeof(uint32_t));
-	mmgr_store(addr + DEBUG_DATA_TAIL_DDR_ADDR,
+	hmm_store(addr + DEBUG_DATA_TAIL_DDR_ADDR,
 		   &tail, sizeof(uint32_t));
-	mmgr_store(addr + DEBUG_DATA_ENABLE_DDR_ADDR,
+	hmm_store(addr + DEBUG_DATA_ENABLE_DDR_ADDR,
 		   &enable, sizeof(uint32_t));
 
 	/* set the local copy */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h
index 4c95eda694f7..536a4dcf0f62 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h
index 8447e33d1c04..3fea43a2125e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
@@ -22,8 +23,6 @@
 #define __INLINE_ISP__
 #include "isp.h"
 
-#include "memory_access.h"
-
 #include "assert_support.h"
 
 STORAGE_CLASS_DEBUG_C bool is_debug_buffer_empty(void)
@@ -101,22 +100,22 @@ STORAGE_CLASS_DEBUG_C void debug_synch_queue_ddr(void)
 {
 	u32	remote_tail;
 
-	mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_TAIL_DDR_ADDR, &remote_tail,
+	hmm_load(debug_buffer_ddr_address + DEBUG_DATA_TAIL_DDR_ADDR, &remote_tail,
 		  sizeof(uint32_t));
 	/* We could move the remote head after the upload, but we would have to limit the upload w.r.t. the local head. This is easier */
 	if (remote_tail > debug_data_ptr->tail) {
 		size_t	delta = remote_tail - debug_data_ptr->tail;
 
-		mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR +
+		hmm_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR +
 			  debug_data_ptr->tail * sizeof(uint32_t),
 			  (void *)&debug_data_ptr->buf[debug_data_ptr->tail], delta * sizeof(uint32_t));
 	} else if (remote_tail < debug_data_ptr->tail) {
 		size_t	delta = DEBUG_BUF_SIZE - debug_data_ptr->tail;
 
-		mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR +
+		hmm_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR +
 			  debug_data_ptr->tail * sizeof(uint32_t),
 			  (void *)&debug_data_ptr->buf[debug_data_ptr->tail], delta * sizeof(uint32_t));
-		mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR,
+		hmm_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR,
 			  (void *)&debug_data_ptr->buf[0],
 			  remote_tail * sizeof(uint32_t));
 	} /* else we are up to date */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c
index 87df1da1164e..f85950c471c7 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2016, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h
index d7db964d5cec..7e4cc75733cd 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h
index ebb75da72e18..1f62bc2f176f 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c
index 777670948d6f..62d4809e33dd 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h
index 39a9dd697096..25d3823026e8 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h
index 3b6cc27ecb28..f59d45cc78b7 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c
index 82f7c43bcb0a..01698064bbe1 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h
index a557ff8a416f..dfdca944a40b 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h
index abaef8672ae2..10d9c076c140 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c
index 65c5296163dd..8ed1cffc5384 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h
index 0c6de867e012..4b2b3282c1b2 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h
index f7dec75adf78..73051112f354 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c
index 5f20ac0b492e..a80e547d47b3 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h
index 113d5ed32d42..320ed35269ea 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h
index cdc1b12a9e8a..f11a19f21d10 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c
index 4a856f1f14bf..2a58dba3c87b 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h
index 4d5961c78c16..efede25587fd 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h
index 705be5e5cc70..3e1b36105bb6 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h
index f4652b79734d..14013733f826 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h
index 56b442040ad9..cc60bed71ddb 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c
index e48f180c9507..be102d5cec87 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h
index 499f55f07253..a3ee274bdf19 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h
index 270d04cc9d09..80d81983bd06 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c
index 0c90c5ed659b..bec9c7238a78 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h
index ee2c8372421c..94fff77584f7 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h
index bdca709219a4..e2bc952e6694 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c
index 2114cf4f3fda..fc000af042dc 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-015, Intel Corporation.
@@ -855,9 +856,9 @@ input_system_error_t input_system_configuration_reset(void)
 
 	input_system_network_rst(INPUT_SYSTEM0_ID);
 
-	gp_device_rst(INPUT_SYSTEM0_ID);
+	gp_device_rst(GP_DEVICE0_ID);
 
-	input_switch_rst(INPUT_SYSTEM0_ID);
+	input_switch_rst(GP_DEVICE0_ID);
 
 	//target_rst();
 
@@ -873,7 +874,7 @@ input_system_error_t input_system_configuration_reset(void)
 
 	for (i = 0; i < N_CSI_PORTS; i++) {
 		config.csi_buffer_flags[i]	 = INPUT_SYSTEM_CFG_FLAG_RESET;
-		config.multicast[i]		 = INPUT_SYSTEM_CFG_FLAG_RESET;
+		config.multicast[i]		 = INPUT_SYSTEM_DISCARD_ALL;
 	}
 
 	config.source_type_flags				 = INPUT_SYSTEM_CFG_FLAG_RESET;
@@ -1323,10 +1324,10 @@ static input_system_error_t configuration_to_registers(void)
 	} // end of switch (source_type)
 
 	// Set input selector.
-	input_selector_cfg_for_sensor(INPUT_SYSTEM0_ID);
+	input_selector_cfg_for_sensor(GP_DEVICE0_ID);
 
 	// Set input switch.
-	input_switch_cfg(INPUT_SYSTEM0_ID, &config.input_switch_cfg);
+	input_switch_cfg(GP_DEVICE0_ID, &config.input_switch_cfg);
 
 	// Set input formatters.
 	// AM: IF are set dynamically.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c
index fdc99cc6eae4..80b5fd0dc9f6 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
@@ -20,17 +21,14 @@
 #endif
 #include "gp_device.h"	/* _REG_GP_IRQ_REQUEST_ADDR */
 
-#include "platform_support.h"			/* hrt_sleep() */
-
 static inline void irq_wait_for_write_complete(
     const irq_ID_t		ID);
 
 static inline bool any_irq_channel_enabled(
     const irq_ID_t				ID);
 
-static inline irq_ID_t virq_get_irq_id(
-    const virq_id_t		irq_ID,
-    unsigned int		*channel_ID);
+static inline irq_ID_t virq_get_irq_id(const enum virq_id irq_ID,
+				       unsigned int *channel_ID);
 
 #ifndef __INLINE_IRQ__
 #include "irq_private.h"
@@ -51,7 +49,7 @@ static unsigned short IRQ_N_ID_OFFSET[N_IRQ_ID + 1] = {
 	IRQ_END_OFFSET
 };
 
-static virq_id_t IRQ_NESTING_ID[N_IRQ_ID] = {
+static enum virq_id IRQ_NESTING_ID[N_IRQ_ID] = {
 	N_virq_id,
 	virq_ifmt,
 	virq_isys,
@@ -227,9 +225,8 @@ void irq_raise(
 	return;
 }
 
-void irq_controller_get_state(
-    const irq_ID_t				ID,
-    irq_controller_state_t		*state)
+void irq_controller_get_state(const irq_ID_t ID,
+			      struct irq_controller_state *state)
 {
 	assert(ID < N_IRQ_ID);
 	assert(state);
@@ -256,7 +253,7 @@ bool any_virq_signal(void)
 }
 
 void cnd_virq_enable_channel(
-    const virq_id_t				irq_ID,
+    const enum virq_id				irq_ID,
     const bool					en)
 {
 	irq_ID_t		i;
@@ -296,8 +293,8 @@ void virq_clear_all(void)
 	return;
 }
 
-enum hrt_isp_css_irq_status virq_get_channel_signals(
-    virq_info_t					*irq_info)
+enum hrt_isp_css_irq_status
+virq_get_channel_signals(struct virq_info *irq_info)
 {
 	enum hrt_isp_css_irq_status irq_status = hrt_isp_css_irq_status_error;
 	irq_ID_t ID;
@@ -326,8 +323,7 @@ enum hrt_isp_css_irq_status virq_get_channel_signals(
 	return irq_status;
 }
 
-void virq_clear_info(
-    virq_info_t					*irq_info)
+void virq_clear_info(struct virq_info *irq_info)
 {
 	irq_ID_t ID;
 
@@ -340,7 +336,7 @@ void virq_clear_info(
 }
 
 enum hrt_isp_css_irq_status virq_get_channel_id(
-    virq_id_t					*irq_id)
+    enum virq_id					*irq_id)
 {
 	unsigned int irq_status = irq_reg_load(IRQ0_ID,
 					       _HRT_IRQ_CONTROLLER_STATUS_REG_IDX);
@@ -367,7 +363,7 @@ enum hrt_isp_css_irq_status virq_get_channel_id(
 
 	/* Check whether we have an IRQ on one of the nested devices */
 	for (ID = N_IRQ_ID - 1 ; ID > (irq_ID_t)0; ID--) {
-		if (IRQ_NESTING_ID[ID] == (virq_id_t)idx) {
+		if (IRQ_NESTING_ID[ID] == (enum virq_id)idx) {
 			break;
 		}
 	}
@@ -404,7 +400,7 @@ enum hrt_isp_css_irq_status virq_get_channel_id(
 
 	idx += IRQ_N_ID_OFFSET[ID];
 	if (irq_id)
-		*irq_id = (virq_id_t)idx;
+		*irq_id = (enum virq_id)idx;
 
 	return status;
 }
@@ -432,7 +428,7 @@ static inline bool any_irq_channel_enabled(
 }
 
 static inline irq_ID_t virq_get_irq_id(
-    const virq_id_t		irq_ID,
+    const enum virq_id		irq_ID,
     unsigned int		*channel_ID)
 {
 	irq_ID_t ID;
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h
index 86028fde2a94..6a25345ae88e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
@@ -42,10 +43,7 @@
 #define	IRQ2_ID_N_CHANNEL	HIVE_ISYS_IRQ_NUM_BITS
 #define	IRQ3_ID_N_CHANNEL	HIVE_ISEL_IRQ_NUM_IRQS
 
-typedef struct virq_info_s					virq_info_t;
-typedef struct irq_controller_state_s		irq_controller_state_t;
-
-typedef enum {
+enum virq_id {
 	virq_gpio_pin_0            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID,
 	virq_gpio_pin_1            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID,
 	virq_gpio_pin_2            = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID,
@@ -66,13 +64,7 @@ typedef enum {
 	virq_sp_stream_mon         = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID,
 	virq_isp_stream_mon        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID,
 	virq_mod_stream_mon        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID,
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM)
 	virq_isp_pmem_error        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID,
-#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-	virq_isys_2401             = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID,
-#else
-#error "irq_local.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
 	virq_isp_bamem_error       = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID,
 	virq_isp_dmem_error        = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID,
 	virq_sp_icache_mem_error   = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID,
@@ -117,13 +109,13 @@ typedef enum {
 	virq_isel_eol              = IRQ3_ID_OFFSET + HIVE_ISEL_IRQ_SYNC_GEN_EOL_BIT_ID,
 
 	N_virq_id                  = IRQ_END_OFFSET
-} virq_id_t;
+};
 
-struct virq_info_s {
+struct virq_info {
 	hrt_data		irq_status_reg[N_IRQ_ID];
 };
 
-struct irq_controller_state_s {
+struct irq_controller_state {
 	unsigned int	irq_edge;
 	unsigned int	irq_mask;
 	unsigned int	irq_status;
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h
index 8a947aefd851..e98663ef0fcd 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c
index 7de7d08f4757..4ad5e2db8a89 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
@@ -12,6 +13,8 @@
  * more details.
  */
 
+#include <linux/delay.h>
+
 #include <system_global.h>
 #include "isp.h"
 
@@ -20,7 +23,6 @@
 #endif /* __INLINE_ISP__ */
 
 #include "assert_support.h"
-#include "platform_support.h"			/* hrt_sleep() */
 
 void cnd_isp_irq_enable(
     const isp_ID_t		ID,
@@ -124,5 +126,5 @@ void isp_wake(isp_ID_t ID)
 {
 	assert(ID < N_ISP_ID);
 	isp_ctrl_setbit(ID, ISP_SC_REG, ISP_START_BIT);
-	hrt_sleep();
+	udelay(1);
 }
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h
index b04da7f1f98c..eceeb5d160ad 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h
index a6ab10711255..2f9aeb3bd9d4 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c
index a17b32b6d414..eb02835aa98a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h
index 7c3ad157189f..913150504f0f 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c
index f084b316e373..aae18465b6ae 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h
index 0e477b497c98..2956c7023b33 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h
index e3e24fac126e..05e6b438d2c8 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c
index aaea74389443..bc9e7f10f1ab 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h
index e570813af28d..f58ee6afcff9 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h
index 3c137badbd43..c19eeafed3a3 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h
index c4e99afe0d29..c68ed984ca48 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c
index 0c6830ae7344..6620f091442f 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2016, Intel Corporation.
@@ -20,7 +21,6 @@
 #include "ia_css_device_access.h"
 #endif
 #include "assert_support.h"
-#include "platform_support.h"			/* hrt_sleep() */
 
 typedef unsigned long long hive_uedge;
 typedef hive_uedge *hive_wide;
@@ -154,7 +154,7 @@ static void load_vector(
 		hive_sim_wide_unpack(data, &elem, ISP_VEC_ELEMBITS, i);
 		to[i] = elem;
 	}
-	hrt_sleep(); /* Spend at least 1 cycles per vector */
+	udelay(1); /* Spend at least 1 cycles per vector */
 }
 
 static void store_vector(
@@ -179,7 +179,7 @@ static void store_vector(
 	//hrt_mem_store (ISP, VMEM, (unsigned)to, &v, siz); /* This will overwrite the next vector as well */
 	hrt_master_port_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size);
 #endif
-	hrt_sleep(); /* Spend at least 1 cycles per vector */
+	udelay(1); /* Spend at least 1 cycles per vector */
 }
 
 void isp_vmem_load(
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h
index a42cce42f29d..d0ba59cedc92 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h
index f48d1281b5a7..39cf1316b404 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h
index 163521c53d4b..605cf02e520c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -19,7 +20,7 @@
 #define IS_INPUT_SWITCH_VERSION2
 
 #include <type_support.h>
-#include <system_types.h>
+#include <system_local.h>
 #include "if_defs.h"
 #include "str2mem_defs.h"
 #include "input_switch_2400_defs.h"
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h
index 64554d80dc0b..4a1dea6dfd40 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,22 +16,14 @@
 #ifndef __IRQ_GLOBAL_H_INCLUDED__
 #define __IRQ_GLOBAL_H_INCLUDED__
 
-#include <system_types.h>
+#include <system_local.h>
 
 #define IS_IRQ_VERSION_2
 #define IS_IRQ_MAP_VERSION_2
 
 /* We cannot include the (hrt host ID) file defining the "CSS_RECEIVER" property without side effects */
 #ifndef HAS_NO_RX
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM)
-/*#define CSS_RECEIVER testbench_isp_inp_sys_csi_receiver*/
-#include "hive_isp_css_irq_types_hrt.h"	/* enum	hrt_isp_css_irq */
-#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-/*#define CSS_RECEIVER testbench_isp_is_2400_inp_sys_csi_receiver*/
-#include "hive_isp_css_2401_irq_types_hrt.h"	/* enum	hrt_isp_css_irq */
-#else
-#error "irq_global.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
+#include "irq_types_hrt.h"
 #endif
 
 /* The IRQ is not mapped uniformly on its related interfaces */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h
index 1a8547d58435..5c6891c9b451 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,19 +16,9 @@
 #ifndef __ISP_GLOBAL_H_INCLUDED__
 #define __ISP_GLOBAL_H_INCLUDED__
 
-#include <system_types.h>
+#include <system_local.h>
 
-#if defined(HAS_ISP_2401_MAMOIADA)
-#define IS_ISP_2401_MAMOIADA
-
-#include "isp2401_mamoiada_params.h"
-#elif defined(HAS_ISP_2400_MAMOIADA)
-#define IS_ISP_2400_MAMOIADA
-
-#include "isp2400_mamoiada_params.h"
-#else
-#error "isp_global_h: ISP_2400_MAMOIDA must be one of {2400, 2401 }"
-#endif
+#include "mamoiada_params.h"
 
 #define ISP_PMEM_WIDTH_LOG2		ISP_LOG2_PMEM_WIDTH
 #define ISP_PMEM_SIZE			ISP_PMEM_DEPTH
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h
index 83ca418c8ff2..8738fed6afdf 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h
index 6ec4e590e3b4..b8338f9b5c0c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,19 +16,9 @@
 #ifndef __SP_GLOBAL_H_INCLUDED__
 #define __SP_GLOBAL_H_INCLUDED__
 
-#include <system_types.h>
+#include <system_local.h>
 
-#if defined(HAS_SP_2401)
-#define IS_SP_2401
-/* 2401 uses 2400 */
 #include <scalar_processor_2400_params.h>
-#elif defined(HAS_SP_2400)
-#define IS_SP_2400
-
-#include <scalar_processor_2400_params.h>
-#else
-#error "sp_global.h: SP_2400 must be one of {2400, 2401 }"
-#endif
 
 #define SP_PMEM_WIDTH_LOG2		SP_PMEM_LOG_WIDTH_BITS
 #define SP_PMEM_SIZE			SP_PMEM_DEPTH
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h
index f185859e3084..3f2915a78031 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h
index 92b783fed82c..0d290e815767 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h
index 7867cd137f3f..537b074211da 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h b/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h
index 52676f3610ba..e9cf2743868c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h
index 4cb7e4c952c5..7382c0bbf7cb 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h
index 76856db58626..29f14e900580 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h
index badb15705710..4602885d50e8 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h
index ba11b956eb1a..0f8195ba8d1a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h
index be031d41de7c..492f9e26cfff 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
@@ -39,7 +40,7 @@ more details.
  * User provided file that defines the system address types:
  *	- hrt_address	a type that can hold the (sub)system address range
  */
-#include "system_types.h"
+#include "system_local.h"
 /*
  * We cannot assume that the global system address size is the size of
  * a pointer because a (say) 64-bit host can be simulated in a 32-bit
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h
index b6c464a530ea..2f5ebfcd7e8b 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/error_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/error_support.h
deleted file mode 100644
index 4f0d259bf7ed..000000000000
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/error_support.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ERROR_SUPPORT_H_INCLUDED__
-#define __ERROR_SUPPORT_H_INCLUDED__
-
-#include <linux/errno.h>
-/*
- * Put here everything __KERNEL__ specific not covered in
- * "errno.h"
- */
-#define ENOTSUP 252
-
-#define verifexit(cond, error_tag)  \
-do {                               \
-	if (!(cond)) {              \
-		goto EXIT;         \
-	}                          \
-} while (0)
-
-#define verifjmpexit(cond)         \
-do {                               \
-	if (!(cond)) {              \
-		goto EXIT;         \
-	}                          \
-} while (0)
-
-#endif /* __ERROR_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h
index 8bfe348772f4..0a085abd3ade 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h
index 1743caa006d0..19a1bdd9171a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h
index 4f8d7fbc8e7f..4ed57fb4530e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h
index 665557bae7a1..d122bdeae7e7 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h
index cd26c9d16a35..0b7e92b963d0 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h
index ad79c03e59f4..6f16ca77cf75 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h
index f87fd6b2ba23..898facd7b17a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h
index f7cd4d7b96e5..e6f695691407 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h
index 79a8446658ee..ee861ddb8e92 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -16,7 +17,8 @@
 #define __DEBUG_PUBLIC_H_INCLUDED__
 
 #include <type_support.h>
-#include "system_types.h"
+#include <ia_css_types.h>
+#include "system_local.h"
 
 /*! brief
  *
@@ -42,7 +44,7 @@ typedef struct debug_data_ddr_s	debug_data_ddr_t;
 
 extern debug_data_t				*debug_data_ptr;
 extern hrt_address				debug_buffer_address;
-extern hrt_vaddress				debug_buffer_ddr_address;
+extern ia_css_ptr				debug_buffer_ddr_address;
 
 /*! Check the empty state of the local debug data buffer
 
@@ -86,7 +88,7 @@ void debug_buffer_init(
  \return none
  */
 void debug_buffer_ddr_init(
-    const hrt_vaddress		addr);
+    const ia_css_ptr		addr);
 
 /*! Set the (remote) operating mode of the debug buffer
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h
index 385b978b703b..a23cbc9a2129 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef __DMA_PUBLIC_H_INCLUDED__
 #define __DMA_PUBLIC_H_INCLUDED__
 
-#include "system_types.h"
+#include "system_local.h"
 
 typedef struct dma_state_s		dma_state_t;
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h
index a84b74b3bc1e..22f1875f038e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -16,7 +17,7 @@
 #define __EVENT_FIFO_PUBLIC_H
 
 #include <type_support.h>
-#include "system_types.h"
+#include "system_local.h"
 
 /*! Blocking read from an event source EVENT[ID]
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h
index e451d6f2a70d..7c1c3d2f24c6 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef __FIFO_MONITOR_PUBLIC_H_INCLUDED__
 #define __FIFO_MONITOR_PUBLIC_H_INCLUDED__
 
-#include "system_types.h"
+#include "system_local.h"
 
 typedef struct fifo_channel_state_s		fifo_channel_state_t;
 typedef struct fifo_switch_state_s		fifo_switch_state_t;
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h
index fc6f42e76fbe..385b79254455 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h
index 7cc0799d49ed..f017742d9ac4 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef __GP_DEVICE_PUBLIC_H_INCLUDED__
 #define __GP_DEVICE_PUBLIC_H_INCLUDED__
 
-#include "system_types.h"
+#include "system_local.h"
 
 typedef struct gp_device_state_s		gp_device_state_t;
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h
index 2ddb8c40a5b2..13baf7236375 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef __GP_TIMER_PUBLIC_H_INCLUDED__
 #define __GP_TIMER_PUBLIC_H_INCLUDED__
 
-#include "system_types.h"
+#include "system_local.h"
 
 /*! initialize mentioned timer
 param ID		timer_id
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h
index d21aab3a179d..13df9b57a5fb 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef __GPIO_PUBLIC_H_INCLUDED__
 #define __GPIO_PUBLIC_H_INCLUDED__
 
-#include "system_types.h"
+#include "system_local.h"
 
 /*! Write to a control register of GPIO[ID]
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h
index 567fbc1d35e7..8d271fb84209 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h
index 6b17a6b651b7..053803d2cae3 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h
index e5758cb8bedd..81dc58640d83 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -16,7 +17,7 @@
 #define __INPUT_FORMATTER_PUBLIC_H_INCLUDED__
 
 #include <type_support.h>
-#include "system_types.h"
+#include "system_local.h"
 
 /*! Reset INPUT_FORMATTER[ID]
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h
index dfe2aa9ff257..d335e7b0a76e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -16,7 +17,7 @@
 #define __IRQ_PUBLIC_H_INCLUDED__
 
 #include <type_support.h>
-#include "system_types.h"
+#include "system_local.h"
 
 /*! Read the control registers of IRQ[ID]
 
@@ -25,9 +26,8 @@
 
  \return none, state = IRQ[ID].state
  */
-void irq_controller_get_state(
-    const irq_ID_t				ID,
-    irq_controller_state_t		*state);
+void irq_controller_get_state(const irq_ID_t ID,
+			      struct irq_controller_state *state);
 
 /*! Write to a control register of IRQ[ID]
 
@@ -136,7 +136,7 @@ bool any_virq_signal(void);
  \return none, VIRQ.channel[irq_ID].enable = en
  */
 void cnd_virq_enable_channel(
-    const virq_id_t				irq_ID,
+    const enum virq_id				irq_ID,
     const bool					en);
 
 /*! Clear the state of all IRQ channels of the virtual super IRQ
@@ -151,8 +151,7 @@ void virq_clear_all(void);
 
  \return none
  */
-void virq_clear_info(
-    virq_info_t					*irq_info);
+void virq_clear_info(struct virq_info *irq_info);
 
 /*! Return the ID of a signalling IRQ channel of the virtual super IRQ
 
@@ -165,7 +164,7 @@ void virq_clear_info(
  \return state(IRQ[...])
  */
 enum hrt_isp_css_irq_status virq_get_channel_id(
-    virq_id_t					*irq_id);
+    enum virq_id					*irq_id);
 
 /*! Return the IDs of all signaling IRQ channels of the virtual super IRQ
 
@@ -178,7 +177,7 @@ enum hrt_isp_css_irq_status virq_get_channel_id(
 
  \return (error(state(IRQ[...]))
  */
-enum hrt_isp_css_irq_status virq_get_channel_signals(
-    virq_info_t					*irq_info);
+enum hrt_isp_css_irq_status
+virq_get_channel_signals(struct virq_info *irq_info);
 
 #endif /* __IRQ_PUBLIC_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h
index 0da2937b900e..a8ff75c639e5 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -16,7 +17,7 @@
 #define __ISP_PUBLIC_H_INCLUDED__
 
 #include <type_support.h>
-#include "system_types.h"
+#include "system_local.h"
 
 /*! Enable or disable the program complete irq signal of ISP[ID]
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h
index 734634aedadf..23a158b81b13 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -17,7 +18,7 @@
 
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
 
-#include "system_types.h"
+#include "system_local.h"
 #include "type_support.h"
 
 STORAGE_CLASS_ISYS2401_DMA_H void isys2401_dma_reg_store(
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h
index d561783e47a3..b9befdd2508e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h
index 2d1859f2c656..509f75fe025c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h
index 87a8d512713e..73bcc424e472 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h
index 278f9cd85a00..b8c7bbb71b01 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef __MMU_PUBLIC_H_INCLUDED__
 #define __MMU_PUBLIC_H_INCLUDED__
 
-#include "system_types.h"
+#include "system_local.h"
 #include "device_access.h"
 #include "assert_support.h"
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h
index ba67a276ce55..ded4dce06d09 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h
index b8db5469b592..b0b7f2e27854 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -16,7 +17,7 @@
 #define __SP_PUBLIC_H_INCLUDED__
 
 #include <type_support.h>
-#include "system_types.h"
+#include "system_local.h"
 
 typedef struct sp_state_s		sp_state_t;
 typedef struct sp_stall_s		sp_stall_t;
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h
index afd5a59489cc..b18b4a4e13ac 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h
index 5f9277adb2ab..563a2833d1d9 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef __TIMED_CTRL_PUBLIC_H_INCLUDED__
 #define __TIMED_CTRL_PUBLIC_H_INCLUDED__
 
-#include "system_types.h"
+#include "system_local.h"
 
 /*! Write to a control register of TIMED_CTRL[ID]
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h
index 577b9b8449e8..823e3857e83c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h
index e9801c0fe147..c510d6a08017 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h
index f9cf7b586045..218341041811 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h
index 377996e0536d..daeb919b5384 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h
index 33ab8a85909e..0d951fbf42e9 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h
index 133dd9014fef..3a83a85111a2 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h
index 749610b8a831..cb64e62c5569 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h
index dbdd17115018..6a759142eda8 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h
index d3f64cfd0b7d..d854124f4f97 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h
index e2ebeb14e7c2..b0f09ffb4f18 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h
index 0b9519c8961c..a444ec14ff9d 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_access/memory_access.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_access/memory_access.h
deleted file mode 100644
index dc63ff0c9c6a..000000000000
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_access/memory_access.h
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015-2017, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __MEMORY_ACCESS_H_INCLUDED__
-#define __MEMORY_ACCESS_H_INCLUDED__
-
-/*!
- * \brief
- * Define the public interface for virtual memory
- * access functions. Access types are limited to
- * those defined in <stdint.h>
- *
- * The address representation is private to the system
- * and represented as "hrt_vaddress" rather than a
- * pointer, as the memory allocation cannot be accessed
- * by dereferencing but reaquires load and store access
- * functions
- *
- * The page table selection or virtual memory context;
- * The page table base index; Is implicit. This page
- * table base index must be set by the implementation
- * of the access function
- *
- * "store" is a transfer to the system
- * "load" is a transfer from the system
- *
- * Allocation properties can be specified by setting
- * attributes (see below) in case of multiple physical
- * memories the memory ID is encoded on the attribute
- *
- * Allocations in the same physical memory, but in a
- * different (set of) page tables can be shared through
- * a page table information mapping function
- */
-
-#include <type_support.h>
-#include "platform_support.h"	/* for __func__ */
-
-/*
- * User provided file that defines the (sub)system address types:
- *	- hrt_vaddress	a type that can hold the (sub)system virtual address range
- */
-#include "system_types.h"
-
-/*
- * The MMU base address is a physical address, thus the same type is used
- * as for the device base address
- */
-#include "device_access.h"
-
-#include "hmm/hmm.h"
-
-/*!
- * \brief
- * Bit masks for specialised allocation functions
- * the default is "uncached", "not contiguous",
- * "not page aligned" and "not cleared"
- *
- * Forcing alignment (usually) returns a pointer
- * at an alignment boundary that is offset from
- * the allocated pointer. Without storing this
- * pointer/offset, we cannot free it. The memory
- * manager is responsible for the bookkeeping, e.g.
- * the allocation function creates a sentinel
- * within the allocation referencable from the
- * returned pointer/address.
- */
-#define MMGR_ATTRIBUTE_MASK		0x000f
-#define MMGR_ATTRIBUTE_CACHED		0x0001
-#define MMGR_ATTRIBUTE_CONTIGUOUS	0x0002
-#define MMGR_ATTRIBUTE_PAGEALIGN	0x0004
-#define MMGR_ATTRIBUTE_CLEARED		0x0008
-#define MMGR_ATTRIBUTE_UNUSED		0xfff0
-
-/* #define MMGR_ATTRIBUTE_DEFAULT	(MMGR_ATTRIBUTE_CACHED) */
-#define MMGR_ATTRIBUTE_DEFAULT	0
-
-extern const hrt_vaddress	mmgr_NULL;
-extern const hrt_vaddress	mmgr_EXCEPTION;
-
-/*! Return the address of an allocation in memory
-
- \param	size[in]		Size in bytes of the allocation
- \param	caller_func[in]		Caller function name
- \param	caller_line[in]		Caller function line number
-
- \return vaddress
- */
-hrt_vaddress mmgr_malloc(const size_t size);
-
-/*! Return the address of a zero initialised allocation in memory
-
- \param	N[in]			Horizontal dimension of array
- \param	size[in]		Vertical dimension of array  Total size is N*size
-
- \return vaddress
- */
-hrt_vaddress mmgr_calloc(const size_t N, const size_t size);
-
-/*! Return the address of an allocation in memory
-
- \param	size[in]		Size in bytes of the allocation
- \param	attribute[in]		Bit vector specifying the properties
-				of the allocation including zero initialisation
-
- \return vaddress
- */
-
-hrt_vaddress mmgr_alloc_attr(const size_t size, const uint16_t attribute);
-
-/*! Return the address of a mapped existing allocation in memory
-
- \param	ptr[in]			Pointer to an allocation in a different
-				virtual memory page table, but the same
-				physical memory
- \param size[in]		Size of the memory of the pointer
- \param	attribute[in]		Bit vector specifying the properties
-				of the allocation
- \param context			Pointer of a context provided by
-				client/driver for additional parameters
-				needed by the implementation
- \Note
-	This interface is tentative, limited to the desired function
-	the actual interface may require furhter parameters
-
- \return vaddress
- */
-hrt_vaddress mmgr_mmap(
-    const void __user *ptr,
-    const size_t size,
-    u16 attribute,
-    void *context);
-
-/*! Zero initialise an allocation in memory
-
- \param	vaddr[in]		Address of an allocation
- \param	size[in]		Size in bytes of the area to be cleared
-
- \return none
- */
-void mmgr_clear(hrt_vaddress vaddr, const size_t	size);
-
-/*! Read an array of bytes from a virtual memory address
-
- \param	vaddr[in]		Address of an allocation
- \param	data[out]		pointer to the destination array
- \param	size[in]		number of bytes to read
-
- \return none
- */
-void mmgr_load(const hrt_vaddress vaddr, void *data, const size_t size);
-
-/*! Write an array of bytes to device registers or memory in the device
-
- \param	vaddr[in]		Address of an allocation
- \param	data[in]		pointer to the source array
- \param	size[in]		number of bytes to write
-
- \return none
- */
-void mmgr_store(const hrt_vaddress vaddr, const void *data, const size_t size);
-
-#endif /* __MEMORY_ACCESS_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_realloc.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_realloc.h
deleted file mode 100644
index 546b93ca1186..000000000000
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_realloc.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#ifndef __MEMORY_REALLOC_H_INCLUDED__
-#define __MEMORY_REALLOC_H_INCLUDED__
-
-/*!
- * \brief
- * Define the internal reallocation of private css memory
- *
- */
-
-#include <type_support.h>
-/*
- * User provided file that defines the (sub)system address types:
- *	- hrt_vaddress	a type that can hold the (sub)system virtual address range
- */
-#include "system_types.h"
-#include "ia_css_err.h"
-
-bool reallocate_buffer(
-    hrt_vaddress *curr_buf,
-    size_t *curr_size,
-    size_t needed_size,
-    bool force,
-    enum ia_css_err *err);
-
-#endif /*__MEMORY_REALLOC_H_INCLUDED__*/
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h
index 38db1ecef3c8..393452d7a3d6 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h
index 1a36cb493fd8..b6f6eda4c55e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h
index 74335fdeff7d..e34cd3c58dd7 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h
index 525c34882fd7..0cdef4a5e8b1 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -24,9 +25,6 @@
 #include <linux/kernel.h>
 #include <linux/string.h>
 
-/* For definition of hrt_sleep() */
-#include "hive_isp_css_custom_host_hrt.h"
-
 #define UINT16_MAX USHRT_MAX
 #define UINT32_MAX UINT_MAX
 #define UCHAR_MAX  (255)
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h
index f5fcf6b1d667..a1f7a5839560 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h
index 1bcadd838161..e6978750a362 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h
index 129446600067..9be45b679386 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h
index 194cd64a7da8..a7d00c7bb8bc 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/string_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/string_support.h
deleted file mode 100644
index 84efbbe78650..000000000000
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/string_support.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#ifndef __STRING_SUPPORT_H_INCLUDED__
-#define __STRING_SUPPORT_H_INCLUDED__
-#include <platform_support.h>
-#include <type_support.h>
-
-/*
- * For all non microsoft cases, we need the following functions
- */
-
-/* @brief Copy from src_buf to dest_buf.
- *
- * @param[out] dest_buf. Destination buffer to copy to
- * @param[in]  dest_size. The size of the destination buffer in bytes
- * @param[in]  src_buf. The source buffer
- * @param[in]  src_size. The size of the source buffer in bytes
- * @return     0 on success, error code on failure
- * @return     EINVAL on Invalid arguments
- * @return     ERANGE on Destination size too small
- */
-static inline int memcpy_s(
-    void *dest_buf,
-    size_t dest_size,
-    const void *src_buf,
-    size_t src_size)
-{
-	if ((!src_buf) || (!dest_buf)) {
-		/* Invalid arguments*/
-		return EINVAL;
-	}
-
-	if ((dest_size < src_size) || (src_size == 0)) {
-		/* Destination too small*/
-		return ERANGE;
-	}
-
-	memcpy(dest_buf, src_buf, src_size);
-	return 0;
-}
-
-/* @brief Get the length of the string, excluding the null terminator
- *
- * @param[in]  src_str. The source string
- * @param[in]  max_len. Look only for max_len bytes in the string
- * @return     Return the string length excluding null character
- * @return     Return max_len if no null character in the first max_len bytes
- * @return     Returns 0 if src_str is NULL
- */
-static size_t strnlen_s(
-    const char *src_str,
-    size_t max_len)
-{
-	size_t ix;
-
-	if (!src_str) {
-		/* Invalid arguments*/
-		return 0;
-	}
-
-	for (ix = 0; ix < max_len && src_str[ix] != '\0'; ix++)
-		;
-
-	/* On Error, it will return src_size == max_len*/
-	return ix;
-}
-
-/* @brief Copy string from src_str to dest_str
- *
- * @param[out] dest_str. Destination buffer to copy to
- * @param[in]  dest_size. The size of the destination buffer in bytes
- * @param[in]  src_str. The source buffer
- * @param[in]  src_size. The size of the source buffer in bytes
- * @return     Returns 0 on success
- * @return     Returns EINVAL on invalid arguments
- * @return     Returns ERANGE on destination size too small
- */
-static inline int strncpy_s(
-    char *dest_str,
-    size_t dest_size,
-    const char *src_str,
-    size_t src_size)
-{
-	size_t len;
-
-	if (!dest_str) {
-		/* Invalid arguments*/
-		return EINVAL;
-	}
-
-	if ((!src_str) || (dest_size == 0)) {
-		/* Invalid arguments*/
-		dest_str[0] = '\0';
-		return EINVAL;
-	}
-
-	len = strnlen_s(src_str, src_size);
-
-	if (len >= dest_size) {
-		/* Destination too small*/
-		dest_str[0] = '\0';
-		return ERANGE;
-	}
-
-	/* dest_str is big enough for the len */
-	strncpy(dest_str, src_str, len);
-	dest_str[len] = '\0';
-	return 0;
-}
-
-/* @brief Copy string from src_str to dest_str
- *
- * @param[out] dest_str. Destination buffer to copy to
- * @param[in]  dest_size. The size of the destination buffer in bytes
- * @param[in]  src_str. The source buffer
- * @return     Returns 0 on success
- * @return     Returns EINVAL on invalid arguments
- * @return     Returns ERANGE on destination size too small
- */
-static inline int strcpy_s(
-    char *dest_str,
-    size_t dest_size,
-    const char *src_str)
-{
-	size_t len;
-
-	if (!dest_str) {
-		/* Invalid arguments*/
-		return EINVAL;
-	}
-
-	if ((!src_str) || (dest_size == 0)) {
-		/* Invalid arguments*/
-		dest_str[0] = '\0';
-		return EINVAL;
-	}
-
-	len = strnlen_s(src_str, dest_size);
-
-	if (len >= dest_size) {
-		/* Destination too small*/
-		dest_str[0] = '\0';
-		return ERANGE;
-	}
-
-	/* dest_str is big enough for the len */
-	strncpy(dest_str, src_str, len);
-	dest_str[len] = '\0';
-	return 0;
-}
-
-
-#endif /* __STRING_SUPPORT_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/system_types.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/system_types.h
deleted file mode 100644
index 764fda8dd214..000000000000
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/system_types.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-#ifndef __SYSTEM_TYPES_H_INCLUDED__
-#define __SYSTEM_TYPES_H_INCLUDED__
-
-/**
-* @file
-* Platform specific types.
-*/
-
-#include "system_local.h"
-
-#endif /* __SYSTEM_TYPES_H_INCLUDED__ */
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h
index 1f0a5d948316..98d7e922aed9 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h
index 403abcb828bf..65b2871fb4ea 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h
index bc77537fa73a..b996ee54d4a5 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h
index 9918ca398138..3ea6758aa798 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h
index 873e01e6d054..da479b370192 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h
index 9f4060319b4b..31121a22d13d 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h
index 2b396955cdad..be6162dfbc66 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c
index a7089ee7462a..8931539a4c01 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h
index 01a8977c189e..921e50a4554a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h
index 0570a95ec5bf..b14f09adef07 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h
index ce0d99418538..6ae453782515 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h
index 549c0d2b7970..b256ea19c0eb 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h
index 9db8766b3a7b..af5a47ace32a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h b/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h
index a22b771f61f2..301dd923950c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/hive_types.h b/drivers/staging/media/atomisp/pci/hive_types.h
index 9715893c8a36..addda9b81d7b 100644
--- a/drivers/staging/media/atomisp/pci/hive_types.h
+++ b/drivers/staging/media/atomisp/pci/hive_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -78,9 +79,6 @@ typedef hive_uint32 hrt_address;
 #error adddres width not supported
 #endif
 
-/* The SP side representation of an HMM virtual address */
-typedef hive_uint32 hrt_vaddress;
-
 /* use 64 bit addresses in simulation, where possible */
 typedef hive_uint64  hive_sim_address;
 
diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm.c b/drivers/staging/media/atomisp/pci/hmm/hmm.c
index cd70307ffd57..42fef1779862 100644
--- a/drivers/staging/media/atomisp/pci/hmm/hmm.c
+++ b/drivers/staging/media/atomisp/pci/hmm/hmm.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -193,7 +194,7 @@ int hmm_init(void)
 	 * at the beginning, to avoid hmm_alloc return 0 in the
 	 * further allocation.
 	 */
-	dummy_ptr = hmm_alloc(1, HMM_BO_PRIVATE, 0, NULL, HMM_UNCACHED);
+	dummy_ptr = hmm_alloc(1, HMM_BO_PRIVATE, 0, NULL, 0);
 
 	if (!ret) {
 		ret = sysfs_create_group(&atomisp_dev->kobj,
@@ -208,6 +209,8 @@ int hmm_init(void)
 
 void hmm_cleanup(void)
 {
+	if (!dummy_ptr)
+		return;
 	sysfs_remove_group(&atomisp_dev->kobj, atomisp_attribute_group);
 
 	/* free dummy memory first */
@@ -219,12 +222,16 @@ void hmm_cleanup(void)
 }
 
 ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
-		     int from_highmem, const void __user *userptr, bool cached)
+		     int from_highmem, const void __user *userptr,
+		     const uint16_t attrs)
 {
 	unsigned int pgnr;
 	struct hmm_buffer_object *bo;
+	bool cached = attrs & ATOMISP_MAP_FLAG_CACHED;
 	int ret;
 
+	WARN_ON(attrs & ATOMISP_MAP_FLAG_CONTIGUOUS);
+
 	/*
 	 * Check if we are initialized. In the ideal world we wouldn't need
 	 * this but we can tackle it once the driver is a lot cleaner
@@ -249,7 +256,7 @@ ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
 		goto alloc_page_err;
 	}
 
-	/* Combind the virtual address and pages togather */
+	/* Combine the virtual address and pages together */
 	ret = hmm_bo_bind(bo);
 	if (ret) {
 		dev_err(atomisp_dev, "hmm_bo_bind failed.\n");
@@ -258,6 +265,13 @@ ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
 
 	hmm_mem_stat.tol_cnt += pgnr;
 
+	if (attrs & ATOMISP_MAP_FLAG_CLEARED)
+		hmm_set(bo->start, 0, bytes);
+
+	    dev_dbg(atomisp_dev,
+	    "%s: pages: 0x%08x (%ld bytes), type: %d from highmem %d, user ptr %p, cached %d\n",
+	    __func__, bo->start, bytes, type, from_highmem, userptr, cached);
+
 	return bo->start;
 
 bind_err:
@@ -272,6 +286,8 @@ void hmm_free(ia_css_ptr virt)
 {
 	struct hmm_buffer_object *bo;
 
+	dev_dbg(atomisp_dev, "%s: free 0x%08x\n", __func__, virt);
+
 	WARN_ON(!virt);
 
 	bo = hmm_bo_device_search_start(&bo_device, (unsigned int)virt);
@@ -396,9 +412,14 @@ static int load_and_flush(ia_css_ptr virt, void *data, unsigned int bytes)
 /* Read function in ISP memory management */
 int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes)
 {
+	if (!virt) {
+		dev_warn(atomisp_dev,
+			"hmm_store: address is NULL\n");
+		return -EINVAL;
+	}
 	if (!data) {
 		dev_err(atomisp_dev,
-			"hmm_load NULL argument\n");
+			"hmm_store: data is a NULL argument\n");
 		return -EINVAL;
 	}
 	return load_and_flush(virt, data, bytes);
@@ -418,6 +439,17 @@ int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes)
 	char *src, *des;
 	int ret;
 
+	if (!virt) {
+		dev_warn(atomisp_dev,
+			"hmm_store: address is NULL\n");
+		return -EINVAL;
+	}
+	if (!data) {
+		dev_err(atomisp_dev,
+			"hmm_store: data is a NULL argument\n");
+		return -EINVAL;
+	}
+
 	bo = hmm_bo_device_search_in_range(&bo_device, virt);
 	ret = hmm_check_bo(bo, virt);
 	if (ret)
diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c
index 492b76c29490..4fb9bfdd2f4c 100644
--- a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c
+++ b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
@@ -659,6 +660,8 @@ static void free_private_bo_pages(struct hmm_buffer_object *bo,
 				break;
 			}
 
+			/* fall through */
+
 		/*
 		 * if dynamic memory pool doesn't exist, need to free
 		 * pages to system directly.
@@ -854,109 +857,20 @@ static void free_private_pages(struct hmm_buffer_object *bo,
 	kfree(bo->page_obj);
 }
 
-/*
- * Hacked from kernel function __get_user_pages in mm/memory.c
- *
- * Handle buffers allocated by other kernel space driver and mmaped into user
- * space, function Ignore the VM_PFNMAP and VM_IO flag in VMA structure
- *
- * Get physical pages from user space virtual address and update into page list
- */
-static int __get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
-			      unsigned long start, int nr_pages,
-			      unsigned int gup_flags, struct page **pages,
-			      struct vm_area_struct **vmas)
-{
-	int i, ret;
-	unsigned long vm_flags;
-
-	if (nr_pages <= 0)
-		return 0;
-
-	VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
-
-	/*
-	 * Require read or write permissions.
-	 * If FOLL_FORCE is set, we only require the "MAY" flags.
-	 */
-	vm_flags  = (gup_flags & FOLL_WRITE) ?
-		    (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
-	vm_flags &= (gup_flags & FOLL_FORCE) ?
-		    (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
-	i = 0;
-
-	do {
-		struct vm_area_struct *vma;
-
-		vma = find_vma(mm, start);
-		if (!vma) {
-			dev_err(atomisp_dev, "find_vma failed\n");
-			return i ? : -EFAULT;
-		}
-
-		if (is_vm_hugetlb_page(vma)) {
-			/*
-			i = follow_hugetlb_page(mm, vma, pages, vmas,
-					&start, &nr_pages, i, gup_flags);
-			*/
-			continue;
-		}
-
-		do {
-			struct page *page;
-			unsigned long pfn;
-
-			/*
-			 * If we have a pending SIGKILL, don't keep faulting
-			 * pages and potentially allocating memory.
-			 */
-			if (unlikely(fatal_signal_pending(current))) {
-				dev_err(atomisp_dev,
-					"fatal_signal_pending in %s\n",
-					__func__);
-				return i ? i : -ERESTARTSYS;
-			}
-
-			ret = follow_pfn(vma, start, &pfn);
-			if (ret) {
-				dev_err(atomisp_dev, "follow_pfn() failed\n");
-				return i ? : -EFAULT;
-			}
-
-			page = pfn_to_page(pfn);
-			if (IS_ERR(page))
-				return i ? i : PTR_ERR(page);
-			if (pages) {
-				pages[i] = page;
-				get_page(page);
-				flush_anon_page(vma, page, start);
-				flush_dcache_page(page);
-			}
-			if (vmas)
-				vmas[i] = vma;
-			i++;
-			start += PAGE_SIZE;
-			nr_pages--;
-		} while (nr_pages && start < vma->vm_end);
-	} while (nr_pages);
-
-	return i;
-}
-
-static int get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
-			    unsigned long start, int nr_pages, int write, int force,
-			    struct page **pages, struct vm_area_struct **vmas)
+static void free_user_pages(struct hmm_buffer_object *bo)
 {
-	int flags = FOLL_TOUCH;
+	int i;
 
-	if (pages)
-		flags |= FOLL_GET;
-	if (write)
-		flags |= FOLL_WRITE;
-	if (force)
-		flags |= FOLL_FORCE;
+	hmm_mem_stat.usr_size -= bo->pgnr;
 
-	return __get_pfnmap_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
+	if (bo->mem_type == HMM_BO_MEM_TYPE_PFN) {
+		unpin_user_pages(bo->pages, bo->pgnr);
+	} else {
+		for (i = 0; i < bo->pgnr; i++)
+			put_page(bo->pages[i]);
+	}
+	kfree(bo->pages);
+	kfree(bo->page_obj);
 }
 
 /*
@@ -982,9 +896,9 @@ static int alloc_user_pages(struct hmm_buffer_object *bo,
 	}
 
 	mutex_unlock(&bo->mutex);
-	down_read(&current->mm->mmap_sem);
+	mmap_read_lock(current->mm);
 	vma = find_vma(current->mm, (unsigned long)userptr);
-	up_read(&current->mm->mmap_sem);
+	mmap_read_unlock(current->mm);
 	if (!vma) {
 		dev_err(atomisp_dev, "find_vma failed\n");
 		kfree(bo->page_obj);
@@ -997,11 +911,15 @@ static int alloc_user_pages(struct hmm_buffer_object *bo,
 	 * Handle frame buffer allocated in other kerenl space driver
 	 * and map to user space
 	 */
+
+	userptr = untagged_addr(userptr);
+
+	bo->pages = pages;
+
 	if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
-		page_nr = get_pfnmap_pages(current, current->mm,
-					   (unsigned long)userptr,
-					   (int)(bo->pgnr), 1, 0,
-					   pages, NULL);
+		page_nr = pin_user_pages((unsigned long)userptr, bo->pgnr,
+					 FOLL_LONGTERM | FOLL_WRITE,
+					 pages, NULL);
 		bo->mem_type = HMM_BO_MEM_TYPE_PFN;
 	} else {
 		/*Handle frame buffer allocated in user space*/
@@ -1012,6 +930,13 @@ static int alloc_user_pages(struct hmm_buffer_object *bo,
 		bo->mem_type = HMM_BO_MEM_TYPE_USER;
 	}
 
+	dev_dbg(atomisp_dev, "%s: %d %s pages were allocated as 0x%08x\n",
+		__func__,
+		bo->pgnr,
+		bo->mem_type == HMM_BO_MEM_TYPE_USER ? "user" : "pfn", page_nr);
+
+	hmm_mem_stat.usr_size += bo->pgnr;
+
 	/* can be written by caller, not forced */
 	if (page_nr != bo->pgnr) {
 		dev_err(atomisp_dev,
@@ -1024,29 +949,14 @@ static int alloc_user_pages(struct hmm_buffer_object *bo,
 		bo->page_obj[i].page = pages[i];
 		bo->page_obj[i].type = HMM_PAGE_TYPE_GENERAL;
 	}
-	hmm_mem_stat.usr_size += bo->pgnr;
-	kfree(pages);
 
 	return 0;
 
 out_of_mem:
-	for (i = 0; i < page_nr; i++)
-		put_page(pages[i]);
-	kfree(pages);
-	kfree(bo->page_obj);
 
-	return -ENOMEM;
-}
+	free_user_pages(bo);
 
-static void free_user_pages(struct hmm_buffer_object *bo)
-{
-	int i;
-
-	for (i = 0; i < bo->pgnr; i++)
-		put_page(bo->page_obj[i].page);
-	hmm_mem_stat.usr_size -= bo->pgnr;
-
-	kfree(bo->page_obj);
+	return -ENOMEM;
 }
 
 /*
diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c b/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c
index 1a87af68a924..eaf97e5f3b68 100644
--- a/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c
+++ b/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c b/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c
index d739ed914d65..57525fece921 100644
--- a/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c
+++ b/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_vm.c b/drivers/staging/media/atomisp/pci/hmm/hmm_vm.c
deleted file mode 100644
index 976a2cb51354..000000000000
--- a/drivers/staging/media/atomisp/pci/hmm/hmm_vm.c
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-/*
- * This file contains function for ISP virtual address management in ISP driver
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include <asm/page.h>
-
-#include "atomisp_internal.h"
-#include "mmu/isp_mmu.h"
-#include "hmm/hmm_vm.h"
-#include "hmm/hmm_common.h"
-
-static unsigned int vm_node_end(unsigned int start, unsigned int pgnr)
-{
-	return start + pgnr_to_size(pgnr);
-}
-
-static int addr_in_vm_node(unsigned int addr,
-			   struct hmm_vm_node *node)
-{
-	return (addr >= node->start) && (addr < (node->start + node->size));
-}
-
-int hmm_vm_init(struct hmm_vm *vm, unsigned int start,
-		unsigned int size)
-{
-	if (!vm)
-		return -1;
-
-	vm->start = start;
-	vm->pgnr = size_to_pgnr_ceil(size);
-	vm->size = pgnr_to_size(vm->pgnr);
-
-	INIT_LIST_HEAD(&vm->vm_node_list);
-	spin_lock_init(&vm->lock);
-	vm->cache = kmem_cache_create("atomisp_vm", sizeof(struct hmm_vm_node),
-				      0, 0, NULL);
-
-	return vm->cache ? 0 : -ENOMEM;
-}
-
-void hmm_vm_clean(struct hmm_vm *vm)
-{
-	struct hmm_vm_node *node, *tmp;
-	struct list_head new_head;
-
-	if (!vm)
-		return;
-
-	spin_lock(&vm->lock);
-	list_replace_init(&vm->vm_node_list, &new_head);
-	spin_unlock(&vm->lock);
-
-	list_for_each_entry_safe(node, tmp, &new_head, list) {
-		list_del(&node->list);
-		kmem_cache_free(vm->cache, node);
-	}
-
-	kmem_cache_destroy(vm->cache);
-}
-
-static struct hmm_vm_node *alloc_hmm_vm_node(unsigned int pgnr,
-	struct hmm_vm *vm)
-{
-	struct hmm_vm_node *node;
-
-	node = kmem_cache_alloc(vm->cache, GFP_KERNEL);
-	if (!node)
-		return NULL;
-
-	INIT_LIST_HEAD(&node->list);
-	node->pgnr = pgnr;
-	node->size = pgnr_to_size(pgnr);
-	node->vm = vm;
-
-	return node;
-}
-
-struct hmm_vm_node *hmm_vm_alloc_node(struct hmm_vm *vm, unsigned int pgnr)
-{
-	struct list_head *head;
-	struct hmm_vm_node *node, *cur, *next;
-	unsigned int vm_start, vm_end;
-	unsigned int addr;
-	unsigned int size;
-
-	if (!vm)
-		return NULL;
-
-	vm_start = vm->start;
-	vm_end = vm_node_end(vm->start, vm->pgnr);
-	size = pgnr_to_size(pgnr);
-
-	addr = vm_start;
-	head = &vm->vm_node_list;
-
-	node = alloc_hmm_vm_node(pgnr, vm);
-	if (!node) {
-		dev_err(atomisp_dev, "no memory to allocate hmm vm node.\n");
-		return NULL;
-	}
-
-	spin_lock(&vm->lock);
-	/*
-	 * if list is empty, the loop code will not be executed.
-	 */
-	list_for_each_entry(cur, head, list) {
-		/* Add gap between vm areas as helper to not hide overflow */
-		addr = PAGE_ALIGN(vm_node_end(cur->start, cur->pgnr) + 1);
-
-		if (list_is_last(&cur->list, head)) {
-			if (addr + size > vm_end) {
-				/* vm area does not have space anymore */
-				spin_unlock(&vm->lock);
-				kmem_cache_free(vm->cache, node);
-				dev_err(atomisp_dev,
-					"no enough virtual address space.\n");
-				return NULL;
-			}
-
-			/* We still have vm space to add new node to tail */
-			break;
-		}
-
-		next = list_entry(cur->list.next, struct hmm_vm_node, list);
-		if ((next->start - addr) > size)
-			break;
-	}
-	node->start = addr;
-	node->vm = vm;
-	list_add(&node->list, &cur->list);
-	spin_unlock(&vm->lock);
-
-	return node;
-}
-
-void hmm_vm_free_node(struct hmm_vm_node *node)
-{
-	struct hmm_vm *vm;
-
-	if (!node)
-		return;
-
-	vm = node->vm;
-
-	spin_lock(&vm->lock);
-	list_del(&node->list);
-	spin_unlock(&vm->lock);
-
-	kmem_cache_free(vm->cache, node);
-}
-
-struct hmm_vm_node *hmm_vm_find_node_start(struct hmm_vm *vm, unsigned int addr)
-{
-	struct hmm_vm_node *node;
-
-	if (!vm)
-		return NULL;
-
-	spin_lock(&vm->lock);
-
-	list_for_each_entry(node, &vm->vm_node_list, list) {
-		if (node->start == addr) {
-			spin_unlock(&vm->lock);
-			return node;
-		}
-	}
-
-	spin_unlock(&vm->lock);
-	return NULL;
-}
-
-struct hmm_vm_node *hmm_vm_find_node_in_range(struct hmm_vm *vm,
-	unsigned int addr)
-{
-	struct hmm_vm_node *node;
-
-	if (!vm)
-		return NULL;
-
-	spin_lock(&vm->lock);
-
-	list_for_each_entry(node, &vm->vm_node_list, list) {
-		if (addr_in_vm_node(addr, node)) {
-			spin_unlock(&vm->lock);
-			return node;
-		}
-	}
-
-	spin_unlock(&vm->lock);
-	return NULL;
-}
diff --git a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_custom_host_hrt.h b/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_custom_host_hrt.h
deleted file mode 100644
index c6893d712d61..000000000000
--- a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_custom_host_hrt.h
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-#ifndef _hive_isp_css_custom_host_hrt_h_
-#define _hive_isp_css_custom_host_hrt_h_
-
-#include <linux/delay.h>
-#include "atomisp_helper.h"
-
-/*
- * _hrt_master_port_store/load/uload -macros using __force attributed
- * cast to intentional dereferencing __iomem attributed (noderef)
- * pointer from atomisp_get_io_virt_addr
- */
-#define _hrt_master_port_store_8(a, d) \
-	(*((s8 __force *)atomisp_get_io_virt_addr(a)) = (d))
-
-#define _hrt_master_port_store_16(a, d) \
-	(*((s16 __force *)atomisp_get_io_virt_addr(a)) = (d))
-
-#define _hrt_master_port_store_32(a, d) \
-	(*((s32 __force *)atomisp_get_io_virt_addr(a)) = (d))
-
-#define _hrt_master_port_load_8(a) \
-	(*(s8 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_load_16(a) \
-	(*(s16 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_load_32(a) \
-	(*(s32 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_uload_8(a) \
-	(*(u8 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_uload_16(a) \
-	(*(u16 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_uload_32(a) \
-	(*(u32 __force *)atomisp_get_io_virt_addr(a))
-
-#define _hrt_master_port_store_8_volatile(a, d)  _hrt_master_port_store_8(a, d)
-#define _hrt_master_port_store_16_volatile(a, d) _hrt_master_port_store_16(a, d)
-#define _hrt_master_port_store_32_volatile(a, d) _hrt_master_port_store_32(a, d)
-
-#define _hrt_master_port_load_8_volatile(a)      _hrt_master_port_load_8(a)
-#define _hrt_master_port_load_16_volatile(a)     _hrt_master_port_load_16(a)
-#define _hrt_master_port_load_32_volatile(a)     _hrt_master_port_load_32(a)
-
-#define _hrt_master_port_uload_8_volatile(a)     _hrt_master_port_uload_8(a)
-#define _hrt_master_port_uload_16_volatile(a)    _hrt_master_port_uload_16(a)
-#define _hrt_master_port_uload_32_volatile(a)    _hrt_master_port_uload_32(a)
-
-static inline void hrt_sleep(void)
-{
-	udelay(1);
-}
-
-static inline u32 _hrt_mem_store(u32 to, const void *from, size_t n)
-{
-	unsigned int i;
-	u32 _to = to;
-	const char *_from = (const char *)from;
-
-	for (i = 0; i < n; i++, _to++, _from++)
-		_hrt_master_port_store_8(_to, *_from);
-	return _to;
-}
-
-static inline void *_hrt_mem_load(u32 from, void *to, size_t n)
-{
-	unsigned int i;
-	char *_to = (char *)to;
-	u32 _from = from;
-
-	for (i = 0; i < n; i++, _to++, _from++)
-		*_to = _hrt_master_port_load_8(_from);
-	return _to;
-}
-
-static inline u32 _hrt_mem_set(u32 to, int c, size_t n)
-{
-	unsigned int i;
-	u32 _to = to;
-
-	for (i = 0; i < n; i++, _to++)
-		_hrt_master_port_store_8(_to, c);
-	return _to;
-}
-
-#endif /* _hive_isp_css_custom_host_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.c b/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.c
deleted file mode 100644
index 236f27b50386..000000000000
--- a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Support for Medifield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#include "atomisp_internal.h"
-
-#include "hive_isp_css_mm_hrt.h"
-#include "hmm/hmm.h"
-
-#define __page_align(size)	(((size) + (PAGE_SIZE - 1)) & (~(PAGE_SIZE - 1)))
-
-static void __user *my_userptr;
-static unsigned int my_num_pages;
-static enum hrt_userptr_type my_usr_type;
-
-void hrt_isp_css_mm_set_user_ptr(void __user *userptr,
-				 unsigned int num_pages,
-				 enum hrt_userptr_type type)
-{
-	my_userptr = userptr;
-	my_num_pages = num_pages;
-	my_usr_type = type;
-}
-
-static ia_css_ptr __hrt_isp_css_mm_alloc(size_t bytes,
-	const void __user *userptr,
-	unsigned int num_pages,
-	enum hrt_userptr_type type,
-	bool cached)
-{
-#ifdef CONFIG_ION
-	if (type == HRT_USR_ION)
-		return hmm_alloc(bytes, HMM_BO_ION, 0,
-				 userptr, cached);
-
-#endif
-	if (type == HRT_USR_PTR) {
-		if (!userptr)
-			return hmm_alloc(bytes, HMM_BO_PRIVATE, 0,
-					 NULL, cached);
-		else {
-			if (num_pages < ((__page_align(bytes)) >> PAGE_SHIFT))
-				dev_err(atomisp_dev,
-					"user space memory size is less than the expected size..\n");
-			else if (num_pages > ((__page_align(bytes))
-					      >> PAGE_SHIFT))
-				dev_err(atomisp_dev,
-					"user space memory size is large than the expected size..\n");
-
-			return hmm_alloc(bytes, HMM_BO_USER, 0,
-					 userptr, cached);
-		}
-	} else {
-		dev_err(atomisp_dev, "user ptr type is incorrect.\n");
-		return 0;
-	}
-}
-
-ia_css_ptr hrt_isp_css_mm_alloc(size_t bytes)
-{
-	return __hrt_isp_css_mm_alloc(bytes, my_userptr,
-				      my_num_pages, my_usr_type, false);
-}
-
-ia_css_ptr hrt_isp_css_mm_alloc_user_ptr(size_t bytes,
-	const void __user *userptr,
-	unsigned int num_pages,
-	enum hrt_userptr_type type,
-	bool cached)
-{
-	return __hrt_isp_css_mm_alloc(bytes, userptr, num_pages,
-				      type, cached);
-}
-
-ia_css_ptr hrt_isp_css_mm_alloc_cached(size_t bytes)
-{
-	if (!my_userptr)
-		return hmm_alloc(bytes, HMM_BO_PRIVATE, 0, NULL,
-				 HMM_CACHED);
-	else {
-		if (my_num_pages < ((__page_align(bytes)) >> PAGE_SHIFT))
-			dev_err(atomisp_dev,
-				"user space memory size is less than the expected size..\n");
-		else if (my_num_pages > ((__page_align(bytes)) >> PAGE_SHIFT))
-			dev_err(atomisp_dev,
-				"user space memory size is large than the expected size..\n");
-
-		return hmm_alloc(bytes, HMM_BO_USER, 0,
-				 my_userptr, HMM_CACHED);
-	}
-}
-
-ia_css_ptr hrt_isp_css_mm_calloc(size_t bytes)
-{
-	ia_css_ptr ptr = hrt_isp_css_mm_alloc(bytes);
-
-	if (ptr)
-		hmm_set(ptr, 0, bytes);
-	return ptr;
-}
-
-ia_css_ptr hrt_isp_css_mm_calloc_cached(size_t bytes)
-{
-	ia_css_ptr ptr = hrt_isp_css_mm_alloc_cached(bytes);
-
-	if (ptr)
-		hmm_set(ptr, 0, bytes);
-	return ptr;
-}
diff --git a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.h b/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.h
deleted file mode 100644
index 818ecf90b1f5..000000000000
--- a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Support for Medfield PNW Camera Imaging ISP subsystem.
- *
- * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
- *
- * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- *
- */
-
-#ifndef _hive_isp_css_mm_hrt_h_
-#define _hive_isp_css_mm_hrt_h_
-
-#include <hmm/hmm.h>
-#include <hrt/hive_isp_css_custom_host_hrt.h>
-
-#define HRT_BUF_FLAG_CACHED BIT(0)
-
-enum hrt_userptr_type {
-	HRT_USR_PTR = 0,
-#ifdef CONFIG_ION
-	HRT_USR_ION,
-#endif
-};
-
-struct hrt_userbuffer_attr {
-	enum hrt_userptr_type	type;
-	unsigned int		pgnr;
-};
-
-void hrt_isp_css_mm_set_user_ptr(void __user *userptr,
-				 unsigned int num_pages, enum hrt_userptr_type);
-
-/* Allocate memory, returns a virtual address */
-ia_css_ptr hrt_isp_css_mm_alloc(size_t bytes);
-ia_css_ptr hrt_isp_css_mm_alloc_user_ptr(size_t bytes,
-	const void __user *userptr,
-	unsigned int num_pages,
-	enum hrt_userptr_type,
-	bool cached);
-ia_css_ptr hrt_isp_css_mm_alloc_cached(size_t bytes);
-
-/* allocate memory and initialize with zeros,
-   returns a virtual address */
-ia_css_ptr hrt_isp_css_mm_calloc(size_t bytes);
-ia_css_ptr hrt_isp_css_mm_calloc_cached(size_t bytes);
-
-#endif /* _hive_isp_css_mm_hrt_h_ */
diff --git a/drivers/staging/media/atomisp/pci/ia_css.h b/drivers/staging/media/atomisp/pci/ia_css.h
index e44df6916d90..d83e1ae5b0b3 100644
--- a/drivers/staging/media/atomisp/pci/ia_css.h
+++ b/drivers/staging/media/atomisp/pci/ia_css.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_3a.h b/drivers/staging/media/atomisp/pci/ia_css_3a.h
index a79941a2e0f2..70cfc915cc56 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_3a.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_3a.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -106,7 +107,7 @@ struct ia_css_isp_3a_statistics_map {
  * used.
  * Always use this function, never copy the buffer directly.
  */
-enum ia_css_err
+int
 ia_css_get_3a_statistics(struct ia_css_3a_statistics           *host_stats,
 			 const struct ia_css_isp_3a_statistics *isp_stats);
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_acc_types.h b/drivers/staging/media/atomisp/pci/ia_css_acc_types.h
index d281846eeba5..36583ab12e3f 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_acc_types.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_acc_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -19,7 +20,7 @@
  * This file contains types used for acceleration
  */
 
-#include <system_types.h>	/* HAS_IRQ_MAP_VERSION_# */
+#include <system_local.h>	/* HAS_IRQ_MAP_VERSION_# */
 #include <type_support.h>
 #include <platform_support.h>
 #include <debug_global.h>
@@ -62,8 +63,8 @@ enum ia_css_cell_type {
  */
 enum ia_css_fw_type {
 	ia_css_sp_firmware,		/** Firmware for the SP */
-	ia_css_isp_firmware,	/** Firmware for the ISP */
-	ia_css_bootloader_firmware, /** Firmware for the BootLoader */
+	ia_css_isp_firmware,		/** Firmware for the ISP */
+	ia_css_bootloader_firmware,	/** Firmware for the BootLoader */
 	ia_css_acc_firmware		/** Firmware for accelrations */
 };
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_buffer.h b/drivers/staging/media/atomisp/pci/ia_css_buffer.h
index 38e1f4791029..b1e8357b94b5 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_buffer.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_buffer.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_control.h b/drivers/staging/media/atomisp/pci/ia_css_control.h
index 248040b3ec07..88f031a63ba2 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_control.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_control.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -37,15 +38,15 @@
  *                              of the L1 page table. This is a physical
  *                              address or index.
  * @param[in]	irq_type	The type of interrupt to be used (edge or level)
- * @return				Returns IA_CSS_ERR_INTERNAL_ERROR in case of any
- *				errors and IA_CSS_SUCCESS otherwise.
+ * @return				Returns -EINVAL in case of any
+ *				errors and 0 otherwise.
  *
  * This function initializes the API which includes allocating and initializing
  * internal data structures. This also interprets the firmware package. All
  * contents of this firmware package are copied into local data structures, so
  * the fw pointer could be freed after this function completes.
  */
-enum ia_css_err ia_css_init(struct device           *dev,
+int ia_css_init(struct device           *dev,
 			    const struct ia_css_env *env,
 			    const struct ia_css_fw  *fw,
 			    u32                     l1_base,
@@ -77,7 +78,7 @@ ia_css_uninit(void);
  * This function should only be called when the SP is not running, calling it
  * when the SP is running will result in an error value being returned. }
  */
-enum ia_css_err
+int
 ia_css_enable_isys_event_queue(bool enable);
 
 /* @brief Test whether the ISP has started.
@@ -109,23 +110,23 @@ ia_css_sp_has_terminated(void);
 
 /* @brief start SP hardware
  *
- * @return			IA_CSS_SUCCESS or error code upon error.
+ * @return			0 or error code upon error.
  *
  * It will boot the SP hardware and start multi-threading infrastructure.
  * All threads will be started and blocked by semaphore. This function should
  * be called before any ia_css_stream_start().
  */
-enum ia_css_err
+int
 ia_css_start_sp(void);
 
 /* @brief stop SP hardware
  *
- * @return			IA_CSS_SUCCESS or error code upon error.
+ * @return			0 or error code upon error.
  *
  * This function will terminate all threads and shut down SP. It should be
  * called after all ia_css_stream_stop().
  */
-enum ia_css_err
+int
 ia_css_stop_sp(void);
 
 #endif /* __IA_CSS_CONTROL_H */
diff --git a/drivers/staging/media/atomisp/pci/ia_css_device_access.c b/drivers/staging/media/atomisp/pci/ia_css_device_access.c
index 6ad8687cf08b..9cd2d3caa5c9 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_device_access.c
+++ b/drivers/staging/media/atomisp/pci/ia_css_device_access.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -14,7 +15,7 @@
 
 #include "ia_css_device_access.h"
 #include <type_support.h>   /* for uint*, size_t */
-#include <system_types.h>   /* for hrt_address */
+#include <system_local.h>   /* for hrt_address */
 #include <ia_css_env.h>     /* for ia_css_hw_access_env */
 #include <assert_support.h> /* for assert */
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_device_access.h b/drivers/staging/media/atomisp/pci/ia_css_device_access.h
index b2bf7d540b62..07d611fdd19f 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_device_access.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_device_access.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -20,7 +21,7 @@
  */
 
 #include <type_support.h> /* for uint*, size_t */
-#include <system_types.h> /* for hrt_address */
+#include <system_local.h> /* for hrt_address */
 #include <ia_css_env.h>   /* for ia_css_hw_access_env */
 
 void
diff --git a/drivers/staging/media/atomisp/pci/ia_css_dvs.h b/drivers/staging/media/atomisp/pci/ia_css_dvs.h
index e647f73c3bd6..3367dfd64050 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_dvs.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_dvs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -94,7 +95,7 @@ union ia_css_dvs_statistics_host {
  * advised to map the ISP memory into a host-side pointer and use
  * the ia_css_translate_dvs_statistics() function instead.
  */
-enum ia_css_err
+int
 ia_css_get_dvs_statistics(struct ia_css_dvs_statistics *host_stats,
 			  const struct ia_css_isp_dvs_statistics *isp_stats);
 
@@ -128,7 +129,7 @@ ia_css_translate_dvs_statistics(
  * advised to map the ISP memory into a host-side pointer and use
  * the ia_css_translate_dvs2_statistics() function instead.
  */
-enum ia_css_err
+int
 ia_css_get_dvs2_statistics(struct ia_css_dvs2_statistics *host_stats,
 			   const struct ia_css_isp_dvs_statistics *isp_stats);
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_env.h b/drivers/staging/media/atomisp/pci/ia_css_env.h
index 8b0218ee658d..8debf334c15c 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_env.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_env.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_err.h b/drivers/staging/media/atomisp/pci/ia_css_err.h
index 375952a7782e..98401a4a171d 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_err.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_err.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -20,27 +21,6 @@
  * functions in the CSS-API.
  */
 
-/* Errors, these values are used as the return value for most
- *  functions in this API.
- */
-enum ia_css_err {
-	IA_CSS_SUCCESS,
-	IA_CSS_ERR_INTERNAL_ERROR,
-	IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY,
-	IA_CSS_ERR_INVALID_ARGUMENTS,
-	IA_CSS_ERR_SYSTEM_NOT_IDLE,
-	IA_CSS_ERR_MODE_HAS_NO_VIEWFINDER,
-	IA_CSS_ERR_QUEUE_IS_FULL,
-	IA_CSS_ERR_QUEUE_IS_EMPTY,
-	IA_CSS_ERR_RESOURCE_NOT_AVAILABLE,
-	IA_CSS_ERR_RESOURCE_LIST_TO_SMALL,
-	IA_CSS_ERR_RESOURCE_ITEMS_STILL_ALLOCATED,
-	IA_CSS_ERR_RESOURCE_EXHAUSTED,
-	IA_CSS_ERR_RESOURCE_ALREADY_ALLOCATED,
-	IA_CSS_ERR_VERSION_MISMATCH,
-	IA_CSS_ERR_NOT_SUPPORTED
-};
-
 /* FW warnings. This enum contains a value for each warning that
  * the SP FW could indicate potential performance issue
  */
diff --git a/drivers/staging/media/atomisp/pci/ia_css_event_public.h b/drivers/staging/media/atomisp/pci/ia_css_event_public.h
index 5c0470fa4a74..08ea801dd5ac 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_event_public.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_event_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -143,9 +144,9 @@ struct ia_css_event {
  *
  * @param[out]	event   Pointer to the event struct which will be filled by
  *                      this function if an event is available.
- * @return		IA_CSS_ERR_QUEUE_IS_EMPTY if no events are
+ * @return		-ENODATA if no events are
  *			available or
- *			IA_CSS_SUCCESS otherwise.
+ *			0 otherwise.
  *
  * This function dequeues an event from the PSYS event queue. The queue is
  * between the Host CPU and the CSS system. This function can be
@@ -153,31 +154,31 @@ struct ia_css_event {
  * was available and can be used in a polling-like situation where the NO_EVENT
  * return value is used to determine whether an event was available or not.
  */
-enum ia_css_err
+int
 ia_css_dequeue_psys_event(struct ia_css_event *event);
 
 /* @brief Dequeue an event from the CSS system.
  *
  * @param[out]	event   Pointer to the event struct which will be filled by
  *                      this function if an event is available.
- * @return		IA_CSS_ERR_QUEUE_IS_EMPTY if no events are
+ * @return		-ENODATA if no events are
  *			available or
- *			IA_CSS_SUCCESS otherwise.
+ *			0 otherwise.
  *
  * deprecated{Use ia_css_dequeue_psys_event instead}.
  * Unless the isys event queue is explicitly enabled, this function will
  * dequeue both isys (EOF) and psys events (all others).
  */
-enum ia_css_err
+int
 ia_css_dequeue_event(struct ia_css_event *event);
 
 /* @brief Dequeue an ISYS event from the CSS system.
  *
  * @param[out]	event   Pointer to the event struct which will be filled by
  *                      this function if an event is available.
- * @return		IA_CSS_ERR_QUEUE_IS_EMPTY if no events are
+ * @return		-ENODATA if no events are
  *			available or
- *			IA_CSS_SUCCESS otherwise.
+ *			0 otherwise.
  *
  * This function dequeues an event from the ISYS event queue. The queue is
  * between host and the CSS system.
@@ -190,7 +191,7 @@ ia_css_dequeue_event(struct ia_css_event *event);
  * incurring additional latency due to locks being held by other CSS API
  * functions.
  */
-enum ia_css_err
+int
 ia_css_dequeue_isys_event(struct ia_css_event *event);
 
 #endif /* __IA_CSS_EVENT_PUBLIC_H */
diff --git a/drivers/staging/media/atomisp/pci/ia_css_firmware.h b/drivers/staging/media/atomisp/pci/ia_css_firmware.h
index 50817162703b..edab72256494 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_firmware.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_firmware.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -36,8 +37,8 @@ struct ia_css_fw {
  *				printing.
  * @param[in]	fw		Firmware package containing the firmware for all
  *				predefined ISP binaries.
- * @return			Returns IA_CSS_ERR_INTERNAL_ERROR in case of any
- *				errors and IA_CSS_SUCCESS otherwise.
+ * @return			Returns -EINVAL in case of any
+ *				errors and 0 otherwise.
  *
  * This function interprets the firmware package. All
  * contents of this firmware package are copied into local data structures, so
@@ -47,7 +48,7 @@ struct ia_css_fw {
  * speeds up ia_css_init (ia_css_init is called each time a stream is created but the
  * firmware only needs to be loaded once).
  */
-enum ia_css_err
+int
 ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,
 		     const struct ia_css_fw  *fw);
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_frac.h b/drivers/staging/media/atomisp/pci/ia_css_frac.h
index 59720370cb8e..661af9225b19 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_frac.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_frac.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_frame_format.h b/drivers/staging/media/atomisp/pci/ia_css_frame_format.h
index 2f177edc36ac..093e23a9b079 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_frame_format.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_frame_format.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_frame_public.h b/drivers/staging/media/atomisp/pci/ia_css_frame_public.h
index 69e9143e5418..96c86f0dc81c 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_frame_public.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_frame_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -121,8 +122,7 @@ struct ia_css_frame_info {
 	struct ia_css_crop_info crop_info;
 };
 
-#define IA_CSS_BINARY_DEFAULT_FRAME_INFO \
-(struct ia_css_frame_info) { \
+#define IA_CSS_BINARY_DEFAULT_FRAME_INFO { \
 	.format			= IA_CSS_FRAME_FORMAT_NUM,  \
 	.raw_bayer_order	= IA_CSS_BAYER_ORDER_NUM, \
 }
@@ -184,8 +184,7 @@ struct ia_css_frame {
 		       info.format */
 };
 
-#define DEFAULT_FRAME \
-(struct ia_css_frame) { \
+#define DEFAULT_FRAME { \
 	.info			= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
 	.dynamic_queue_id	= SH_CSS_INVALID_QUEUE_ID, \
 	.buf_type		= IA_CSS_BUFFER_TYPE_INVALID, \
@@ -214,7 +213,7 @@ void ia_css_frame_zero(struct ia_css_frame *frame);
  * Allocate a CSS frame structure. The memory for the frame data will be
  * allocated in the CSS address space.
  */
-enum ia_css_err
+int
 ia_css_frame_allocate(struct ia_css_frame **frame,
 		      unsigned int width,
 		      unsigned int height,
@@ -232,7 +231,7 @@ ia_css_frame_allocate(struct ia_css_frame **frame,
  * This is a convenience function, implemented on top of
  * ia_css_frame_allocate().
  */
-enum ia_css_err
+int
 ia_css_frame_allocate_from_info(struct ia_css_frame **frame,
 				const struct ia_css_frame_info *info);
 /* @brief Free a CSS frame structure.
@@ -260,7 +259,7 @@ ia_css_frame_free(struct ia_css_frame *frame);
  * physically contiguous memory.
  * Deprecated.
  */
-enum ia_css_err
+int
 ia_css_frame_allocate_contiguous(struct ia_css_frame **frame,
 				 unsigned int width,
 				 unsigned int height,
@@ -280,7 +279,7 @@ ia_css_frame_allocate_contiguous(struct ia_css_frame **frame,
  * Only for FPGA display driver which needs physically contiguous memory.
  * Deprecated.
  */
-enum ia_css_err
+int
 ia_css_frame_allocate_contiguous_from_info(struct ia_css_frame **frame,
 	const struct ia_css_frame_info *info);
 
@@ -293,7 +292,7 @@ ia_css_frame_allocate_contiguous_from_info(struct ia_css_frame **frame,
  * Allocate an empty CSS frame with no data buffer using the parameters
  * in the frame info.
  */
-enum ia_css_err
+int
 ia_css_frame_create_from_info(struct ia_css_frame **frame,
 			      const struct ia_css_frame_info *info);
 
@@ -310,7 +309,7 @@ ia_css_frame_create_from_info(struct ia_css_frame **frame,
  * free the mapped_data buffer. However if ia_css_frame_free() is called and
  * the frame had a valid data buffer, it would be freed along with the frame.
  */
-enum ia_css_err
+int
 ia_css_frame_set_data(struct ia_css_frame *frame,
 		      const ia_css_ptr   mapped_data,
 		      size_t data_size_bytes);
@@ -331,12 +330,12 @@ ia_css_frame_set_data(struct ia_css_frame *frame,
  * ia_css_frame_allocate() does, but instead of allocating the memory, it will
  * map the pre-allocated memory into the CSS address space.
  */
-enum ia_css_err
+int
 ia_css_frame_map(struct ia_css_frame **frame,
 		 const struct ia_css_frame_info *info,
 		 const void __user *data,
 		 u16 attribute,
-		 void *context);
+		 unsigned int pgnr);
 
 /* @brief Unmap a CSS frame structure.
  *
diff --git a/drivers/staging/media/atomisp/pci/ia_css_host_data.h b/drivers/staging/media/atomisp/pci/ia_css_host_data.h
index bc82e97d24cb..f54cc504f5d4 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_host_data.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_host_data.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_input_port.h b/drivers/staging/media/atomisp/pci/ia_css_input_port.h
index ad9ca5449369..9772b6928239 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_input_port.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_input_port.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_irq.h b/drivers/staging/media/atomisp/pci/ia_css_irq.h
index 7716373553e0..3b81a39cfe97 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_irq.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_irq.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -55,11 +56,8 @@ enum ia_css_irq_info {
 	/** the css receiver received the end of frame */
 	IA_CSS_IRQ_INFO_CSS_RECEIVER_SOL              = 1 << 4,
 	/** the css receiver received the start of line */
-	IA_CSS_IRQ_INFO_PSYS_EVENTS_READY             = 1 << 5,
+	IA_CSS_IRQ_INFO_EVENTS_READY                  = 1 << 5,
 	/** One or more events are available in the PSYS event queue */
-	IA_CSS_IRQ_INFO_EVENTS_READY = IA_CSS_IRQ_INFO_PSYS_EVENTS_READY,
-	/** deprecated{obsolete version of IA_CSS_IRQ_INFO_PSYS_EVENTS_READY,
-	 * same functionality.} */
 	IA_CSS_IRQ_INFO_CSS_RECEIVER_EOL              = 1 << 6,
 	/** the css receiver received the end of line */
 	IA_CSS_IRQ_INFO_CSS_RECEIVER_SIDEBAND_CHANGED = 1 << 7,
@@ -144,14 +142,14 @@ struct ia_css_irq {
  * @return		If an error is encountered during the interrupt info
  *			and no interrupt could be translated successfully, this
  *			will return IA_CSS_INTERNAL_ERROR. Otherwise
- *			IA_CSS_SUCCESS.
+ *			0.
  *
  * This function is expected to be executed after an interrupt has been sent
  * to the IA from the CSS. This function returns information about the interrupt
  * which is needed by the IA code to properly handle the interrupt. This
  * information includes the image pipe, buffer type etc.
  */
-enum ia_css_err
+int
 ia_css_irq_translate(unsigned int *info);
 
 /* @brief Get CSI receiver error info.
@@ -227,9 +225,9 @@ ia_css_rx_port_clear_irq_info(enum mipi_port_id port, unsigned int irq_bits);
  * @return		Returns IA_CSS_INTERNAL_ERROR if this interrupt
  *			type cannot be enabled/disabled which is true for
  *			CSS internal interrupts. Otherwise returns
- *			IA_CSS_SUCCESS.
+ *			0.
  */
-enum ia_css_err
+int
 ia_css_irq_enable(enum ia_css_irq_info type, bool enable);
 
 #endif /* __IA_CSS_IRQ_H */
diff --git a/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h b/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h
index 6545efd24cbe..1abb2fd6a913 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_isp_params.h b/drivers/staging/media/atomisp/pci/ia_css_isp_params.h
index b8b3c48492ae..6e3082b39ed6 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_isp_params.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_isp_params.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_isp_states.h b/drivers/staging/media/atomisp/pci/ia_css_isp_states.h
index cc9cdcd0e2be..9f6c342a1705 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_isp_states.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_isp_states.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -65,7 +66,7 @@ struct ia_css_state_memory_offsets {
 #include "ia_css_binary.h"   /* struct ia_css_binary */
 /* Code generated by genparam/genstate.c:gen_state_init_table() */
 
-extern void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(
+extern void (*ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])(
     const struct ia_css_binary *binary);
 
 #endif /* IA_CSS_INCLUDE_STATE */
diff --git a/drivers/staging/media/atomisp/pci/ia_css_memory_access.c b/drivers/staging/media/atomisp/pci/ia_css_memory_access.c
deleted file mode 100644
index 8d1356047448..000000000000
--- a/drivers/staging/media/atomisp/pci/ia_css_memory_access.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015-2017, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-#include <type_support.h>
-#include <system_types.h>
-#include <assert_support.h>
-#include <memory_access.h>
-#include <ia_css_env.h>
-#include <hrt/hive_isp_css_mm_hrt.h>
-
-const hrt_vaddress mmgr_NULL = (hrt_vaddress)0;
-const hrt_vaddress mmgr_EXCEPTION = (hrt_vaddress)-1;
-
-hrt_vaddress
-mmgr_malloc(const size_t size)
-{
-	return mmgr_alloc_attr(size, 0);
-}
-
-hrt_vaddress mmgr_alloc_attr(const size_t size, const uint16_t attrs)
-{
-	u16 masked_attrs = attrs & MMGR_ATTRIBUTE_MASK;
-
-	WARN_ON(attrs & MMGR_ATTRIBUTE_CONTIGUOUS);
-
-	if (masked_attrs & MMGR_ATTRIBUTE_CLEARED) {
-		if (masked_attrs & MMGR_ATTRIBUTE_CACHED)
-			return (ia_css_ptr) hrt_isp_css_mm_calloc_cached(size);
-		else
-			return (ia_css_ptr) hrt_isp_css_mm_calloc(size);
-	} else {
-		if (masked_attrs & MMGR_ATTRIBUTE_CACHED)
-			return (ia_css_ptr) hrt_isp_css_mm_alloc_cached(size);
-		else
-			return (ia_css_ptr) hrt_isp_css_mm_alloc(size);
-	}
-}
-
-hrt_vaddress
-mmgr_calloc(const size_t N, const size_t size)
-{
-	return mmgr_alloc_attr(size * N, MMGR_ATTRIBUTE_CLEARED);
-}
-
-void mmgr_clear(hrt_vaddress vaddr, const size_t size)
-{
-	if (vaddr)
-		hmm_set(vaddr, 0, size);
-}
-
-void mmgr_load(const hrt_vaddress vaddr, void *data, const size_t size)
-{
-	if (vaddr && data)
-		hmm_load(vaddr, data, size);
-}
-
-void
-mmgr_store(const hrt_vaddress vaddr, const void *data, const size_t size)
-{
-	if (vaddr && data)
-		hmm_store(vaddr, data, size);
-}
-
-hrt_vaddress
-mmgr_mmap(const void __user *ptr, const size_t size,
-	  u16 attribute, void *context)
-{
-	struct hrt_userbuffer_attr *userbuffer_attr = context;
-
-	return hrt_isp_css_mm_alloc_user_ptr(
-		   size, ptr, userbuffer_attr->pgnr,
-		   userbuffer_attr->type,
-		   attribute & HRT_BUF_FLAG_CACHED);
-}
diff --git a/drivers/staging/media/atomisp/pci/ia_css_metadata.h b/drivers/staging/media/atomisp/pci/ia_css_metadata.h
index 0212d71b3355..9eb1b76a3b2a 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_metadata.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_metadata.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_mipi.h b/drivers/staging/media/atomisp/pci/ia_css_mipi.h
index c02138ee2511..56a2fca8117f 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_mipi.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_mipi.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -37,7 +38,7 @@
  *
  * Specifies a CSS MIPI frame buffer: size in memory words (32B).
  */
-enum ia_css_err
+int
 ia_css_mipi_frame_specify(const unsigned int	size_mem_words,
 			  const bool contiguous);
 
@@ -54,7 +55,7 @@ ia_css_mipi_frame_specify(const unsigned int	size_mem_words,
  *
  *
  */
-enum ia_css_err
+int
 ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
 				       const unsigned int	size_mem_words);
 #endif
@@ -71,7 +72,7 @@ ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
  *
  * Calculate the size of a mipi frame, based on the resolution and format.
  */
-enum ia_css_err
+int
 ia_css_mipi_frame_calculate_size(const unsigned int width,
 				 const unsigned int height,
 				 const enum atomisp_input_format format,
diff --git a/drivers/staging/media/atomisp/pci/ia_css_mmu.h b/drivers/staging/media/atomisp/pci/ia_css_mmu.h
index 13c21056bfbf..8f04d196c646 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_mmu.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_mmu.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h b/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h
index 1021e4f380a5..dc6542aa64f2 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_morph.h b/drivers/staging/media/atomisp/pci/ia_css_morph.h
index de409638d009..9c4b41b94256 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_morph.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_morph.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_pipe.h b/drivers/staging/media/atomisp/pci/ia_css_pipe.h
index 91653952f1a7..bb0abf9bffb1 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_pipe.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_pipe.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -31,7 +32,7 @@ struct ia_css_preview_settings {
 	struct ia_css_binary vf_pp_binary;
 
 	/* 2401 only for these two - do we in fact use them for anything real */
-	struct ia_css_frame *delay_frames[MAX_NUM_DELAY_FRAMES];
+	struct ia_css_frame *delay_frames[MAX_NUM_VIDEO_DELAY_FRAMES];
 	struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES];
 
 	struct ia_css_pipe *copy_pipe;
@@ -39,8 +40,7 @@ struct ia_css_preview_settings {
 	struct ia_css_pipe *acc_pipe;
 };
 
-#define IA_CSS_DEFAULT_PREVIEW_SETTINGS \
-(struct ia_css_preview_settings) { \
+#define IA_CSS_DEFAULT_PREVIEW_SETTINGS { \
 	.copy_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
 	.preview_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
 	.vf_pp_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
@@ -64,8 +64,7 @@ struct ia_css_capture_settings {
 	unsigned int num_yuv_scaler;
 };
 
-#define IA_CSS_DEFAULT_CAPTURE_SETTINGS \
-(struct ia_css_capture_settings) { \
+#define IA_CSS_DEFAULT_CAPTURE_SETTINGS { \
 	.copy_binary		= IA_CSS_BINARY_DEFAULT_SETTINGS, \
 	.primary_binary		= {IA_CSS_BINARY_DEFAULT_SETTINGS}, \
 	.pre_isp_binary		= IA_CSS_BINARY_DEFAULT_SETTINGS, \
@@ -90,8 +89,7 @@ struct ia_css_video_settings {
 	unsigned int num_yuv_scaler;
 };
 
-#define IA_CSS_DEFAULT_VIDEO_SETTINGS \
-(struct ia_css_video_settings) { \
+#define IA_CSS_DEFAULT_VIDEO_SETTINGS { \
 	.copy_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
 	.video_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
 	.vf_pp_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
@@ -107,8 +105,7 @@ struct ia_css_yuvpp_settings {
 	unsigned int num_output;
 };
 
-#define IA_CSS_DEFAULT_YUVPP_SETTINGS \
-(struct ia_css_yuvpp_settings) { \
+#define IA_CSS_DEFAULT_YUVPP_SETTINGS { \
 	.copy_binary	= IA_CSS_BINARY_DEFAULT_SETTINGS, \
 }
 
@@ -146,7 +143,7 @@ struct ia_css_pipe {
 		struct ia_css_capture_settings capture;
 		struct ia_css_yuvpp_settings yuvpp;
 	} pipe_settings;
-	hrt_vaddress scaler_pp_lut;
+	ia_css_ptr scaler_pp_lut;
 	struct osys_object *osys_obj;
 
 	/* This number is unique per pipe each instance of css. This number is
@@ -156,8 +153,7 @@ struct ia_css_pipe {
 	unsigned int pipe_num;
 };
 
-#define IA_CSS_DEFAULT_PIPE \
-(struct ia_css_pipe) { \
+#define IA_CSS_DEFAULT_PIPE { \
 	.config			= DEFAULT_PIPE_CONFIG, \
 	.info			= DEFAULT_PIPE_INFO, \
 	.mode			= IA_CSS_PIPE_ID_ACC, /* (pipe_id) */ \
@@ -181,7 +177,7 @@ struct ia_css_pipe {
 
 void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map);
 
-enum ia_css_err
+int
 sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 			       struct ia_css_isp_parameters *params,
 			       bool commit, struct ia_css_pipe *pipe);
diff --git a/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h b/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h
index e782978a5ce7..4affd21f9e3f 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -148,8 +149,7 @@ struct ia_css_pipe_config {
 /**
  * Default settings for newly created pipe configurations.
  */
-#define DEFAULT_PIPE_CONFIG \
-(struct ia_css_pipe_config) { \
+#define DEFAULT_PIPE_CONFIG { \
 	.mode			= IA_CSS_PIPE_MODE_PREVIEW, \
 	.isp_pipe_version	= 1, \
 	.output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
@@ -202,8 +202,7 @@ struct ia_css_pipe_info {
 /**
  * Defaults for ia_css_pipe_info structs.
  */
-#define DEFAULT_PIPE_INFO \
-(struct ia_css_pipe_info) { \
+#define DEFAULT_PIPE_INFO {\
 	.output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
 	.vf_output_info		= {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \
 	.raw_output_info	= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
@@ -249,44 +248,44 @@ void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config);
 /* @brief Create a pipe
  * @param[in]	config The pipe configuration.
  * @param[out]	pipe The pipe.
- * @return	IA_CSS_SUCCESS or the error code.
+ * @return	0 or the error code.
  *
  * This function will create a pipe with the given
  * configuration.
  */
-enum ia_css_err
+int
 ia_css_pipe_create(const struct ia_css_pipe_config *config,
 		   struct ia_css_pipe **pipe);
 
 /* @brief Destroy a pipe
  * @param[in]	pipe The pipe.
- * @return	IA_CSS_SUCCESS or the error code.
+ * @return	0 or the error code.
  *
  * This function will destroy a given pipe.
  */
-enum ia_css_err
+int
 ia_css_pipe_destroy(struct ia_css_pipe *pipe);
 
 /* @brief Provides information about a pipe
  * @param[in]	pipe The pipe.
  * @param[out]	pipe_info The pipe information.
- * @return	IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS.
+ * @return	0 or -EINVAL.
  *
  * This function will provide information about a given pipe.
  */
-enum ia_css_err
+int
 ia_css_pipe_get_info(const struct ia_css_pipe *pipe,
 		     struct ia_css_pipe_info *pipe_info);
 
 /* @brief Configure a pipe with filter coefficients.
  * @param[in]	pipe	The pipe.
  * @param[in]	config	The pointer to ISP configuration.
- * @return		IA_CSS_SUCCESS or error code upon error.
+ * @return		0 or error code upon error.
  *
  * This function configures the filter coefficients for an image
  * pipe.
  */
-enum ia_css_err
+int
 ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe,
 			   struct ia_css_isp_config *config);
 
@@ -304,7 +303,7 @@ ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe,
 			at the same moment in time. There is no control over
 			the order of these events. Once an IRQ has been raised
 			all remembered events are reset.
- * @return		IA_CSS_SUCCESS.
+ * @return		0.
  *
  Controls when the Event generator in the CSS raises an IRQ to the Host.
  The main purpose of this function is to reduce the amount of interrupts
@@ -362,7 +361,7 @@ ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe,
  All other events (3A, VF output, pipeline done) will not raise an interrupt
  to the Host. These events are not lost but always stored in the event queue.
  */
-enum ia_css_err
+int
 ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe,
 			 unsigned int or_mask,
 			 unsigned int and_mask);
@@ -374,7 +373,7 @@ ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe,
 		of enum ia_css_event_irq_mask_type. Pointer may be NULL.
  * @param[out]	and_mask Current and_mask.The bits in this mask are a binary or
 		of enum ia_css_event_irq_mask_type. Pointer may be NULL.
- * @return	IA_CSS_SUCCESS.
+ * @return	0.
  *
  Reads the current event IRQ mask from the CSS. Reading returns the actual
  values as used by the SP and not any mirrored values stored at the Host.\n
@@ -383,7 +382,7 @@ Precondition:\n
 SP must be running.\n
 
 */
-enum ia_css_err
+int
 ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
 			  unsigned int *or_mask,
 			  unsigned int *and_mask);
@@ -396,7 +395,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
  *			structure. Only the data pointer within it will
  *			be passed into the internal queues.
  * @return		IA_CSS_INTERNAL_ERROR in case of unexpected errors,
- *			IA_CSS_SUCCESS otherwise.
+ *			0 otherwise.
  *
  * This function adds a buffer (which has a certain buffer type) to the queue
  * for this type. This queue is owned by the image pipe. After this function
@@ -406,7 +405,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
  * host code via an interrupt. Buffers will be consumed in the same order they
  * get queued, but may be returned to the host out of order.
  */
-enum ia_css_err
+int
 ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 			   const struct ia_css_buffer *buffer);
 
@@ -418,7 +417,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
  *			 The resulting buffer pointer is written into the dta
  *			 field.
  * @return		 IA_CSS_ERR_NO_BUFFER if the queue is empty or
- *			 IA_CSS_SUCCESS otherwise.
+ *			 0 otherwise.
  *
  * This function dequeues a buffer from a buffer queue. The queue is indicated
  * by the buffer type argument. This function can be called after an interrupt
@@ -426,7 +425,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
  * be used in a polling-like situation where the NO_BUFFER return value is used
  * to determine whether a buffer was available or not.
  */
-enum ia_css_err
+int
 ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 			   struct ia_css_buffer *buffer);
 
@@ -437,9 +436,9 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
  * @param[in] enable       Enable Flag (1 to enable ; 0 to disable)
  *
  * @return
- * IA_CSS_SUCCESS			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
- * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE	: Inactive QOS Pipe
+ * 0			: Success
+ * -EINVAL		: Invalid Parameters
+ * -EBUSY	: Inactive QOS Pipe
  *					(No active stream with this pipe)
  *
  * This function will request state change (enable or disable) for the Extension
@@ -452,7 +451,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
  *	4. State change cannot be guaranteed immediately OR on frame boundary
  *
  */
-enum ia_css_err
+int
 ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe,
 			      u32 fw_handle,
 			      bool  enable);
@@ -464,9 +463,9 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe,
  * @param[out] *enable     Enable Flag
  *
  * @return
- * IA_CSS_SUCCESS			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
- * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE	: Inactive QOS Pipe
+ * 0			: Success
+ * -EINVAL		: Invalid Parameters
+ * -EBUSY	: Inactive QOS Pipe
  *					(No active stream with this pipe)
  *
  * This function will query the state of the Extension stage (firmware handle)
@@ -478,7 +477,7 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe,
  *	3. Initial(Default) state of QOS Extensions is Disabled.
  *
  */
-enum ia_css_err
+int
 ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe,
 			      u32 fw_handle,
 			      bool *enable);
@@ -491,18 +490,18 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe,
  * @param[in] isp_seg	Parameter memory descriptors for ISP segments.
  *
  * @return
- * IA_CSS_SUCCESS			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
- * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE	: Inactive QOS Pipe
+ * 0			: Success
+ * -EINVAL		: Invalid Parameters
+ * -EBUSY	: Inactive QOS Pipe
  *					(No active stream with this pipe)
  *
  * \deprecated{This interface is used to temporarily support a late-developed,
  * specific use-case on a specific IPU2 platform. It will not be supported or
  * maintained on IPU3 or further.}
  */
-enum ia_css_err
+int
 ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe,
-				      uint32_t fw_handle,
+				      u32 fw_handle,
 				      struct ia_css_isp_param_css_segments *css_seg,
 				      struct ia_css_isp_param_isp_segments *isp_seg);
 
@@ -521,8 +520,8 @@ ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe,
  * @param[in]  lut         Look up tabel
  *
  * @return
- * IA_CSS_SUCCESS			: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS		: Invalid Parameters
+ * 0			: Success
+ * -EINVAL		: Invalid Parameters
  *
  * Note:
  * 1) Note that both GDC's are programmed with the same table.
@@ -531,7 +530,7 @@ ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe,
  * 3) This function must be called before stream start
  *
  */
-enum ia_css_err
+int
 ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe,
 			       const void *lut);
 /* @brief Checking of DVS statistics ability
@@ -550,9 +549,9 @@ bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info);
  * @param[in]  format      Format to set
  *
  * @return
- * IA_CSS_SUCCESS		: Success
- * IA_CSS_ERR_INVALID_ARGUMENTS	: Invalid Parameters
- * IA_CSS_ERR_INTERNAL_ERROR	: Pipe misses binary info
+ * 0		: Success
+ * -EINVAL	: Invalid Parameters
+ * -EINVAL	: Pipe misses binary info
  *
  * Note:
  * 1) This is an optional function to override the formats set in the pipe.
@@ -561,7 +560,7 @@ bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info);
  * 4) If this function is used, it MUST be called after ia_css_pipe_create.
  * 5) If this function is used, this function MUST be called before ia_css_stream_start.
  */
-enum ia_css_err
+int
 ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe,
 				  int output_pin,
 				  enum ia_css_frame_format format);
diff --git a/drivers/staging/media/atomisp/pci/ia_css_prbs.h b/drivers/staging/media/atomisp/pci/ia_css_prbs.h
index 037fc4f77c77..53bbf1dce3bf 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_prbs.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_prbs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_properties.h b/drivers/staging/media/atomisp/pci/ia_css_properties.h
index 9a167306611c..2cd014f7ae29 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_properties.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_properties.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_shading.h b/drivers/staging/media/atomisp/pci/ia_css_shading.h
index 588f53d32b72..de7ae5cabf7d 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_shading.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_shading.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_stream.h b/drivers/staging/media/atomisp/pci/ia_css_stream.h
index 5690fe832f41..e3e7a8a03b04 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_stream.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_stream.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -102,7 +103,7 @@ ia_css_get_isp_dvs2_coefficients(struct ia_css_stream *stream,
 				 short *ver_coefs_even_real,
 				 short *ver_coefs_even_imag);
 
-enum ia_css_err
+int
 ia_css_stream_isp_parameters_init(struct ia_css_stream *stream);
 
 void
diff --git a/drivers/staging/media/atomisp/pci/ia_css_stream_format.h b/drivers/staging/media/atomisp/pci/ia_css_stream_format.h
index 4cd29833584f..aac22d8581a0 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_stream_format.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_stream_format.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_stream_public.h b/drivers/staging/media/atomisp/pci/ia_css_stream_public.h
index fe11c8bf3cdc..83846e417ae5 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_stream_public.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_stream_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -171,11 +172,11 @@ void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config);
 * @param[in]	num_pipes The number of pipes to incorporate in the stream.
 * @param[in]	pipes The pipes.
 * @param[out]	stream The stream.
-* @return	IA_CSS_SUCCESS or the error code.
+* @return	0 or the error code.
 *
 * This function will create a stream with a given configuration and given pipes.
 */
-enum ia_css_err
+int
 ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 		     int num_pipes,
 		     struct ia_css_pipe *pipes[],
@@ -183,37 +184,37 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 
 /* @brief Destroys a stream
  * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS or the error code.
+ * @return	0 or the error code.
  *
  * This function will destroy a given stream.
  */
-enum ia_css_err
+int
 ia_css_stream_destroy(struct ia_css_stream *stream);
 
 /* @brief Provides information about a stream
  * @param[in]	stream The stream.
  * @param[out]	stream_info The information about the stream.
- * @return	IA_CSS_SUCCESS or the error code.
+ * @return	0 or the error code.
  *
  * This function will destroy a given stream.
  */
-enum ia_css_err
+int
 ia_css_stream_get_info(const struct ia_css_stream *stream,
 		       struct ia_css_stream_info *stream_info);
 
 /* @brief load (rebuild) a stream that was unloaded.
  * @param[in]	stream The stream
- * @return		IA_CSS_SUCCESS or the error code
+ * @return		0 or the error code
  *
  * Rebuild a stream, including allocating structs, setting configuration and
  * building the required pipes.
  */
-enum ia_css_err
+int
 ia_css_stream_load(struct ia_css_stream *stream);
 
 /* @brief Starts the stream.
  * @param[in]	stream The stream.
- * @return IA_CSS_SUCCESS or the error code.
+ * @return 0 or the error code.
  *
  * The dynamic data in
  * the buffers are not used and need to be queued with a separate call
@@ -221,17 +222,17 @@ ia_css_stream_load(struct ia_css_stream *stream);
  * NOTE: this function will only send start event to corresponding
  * thread and will not start SP any more.
  */
-enum ia_css_err
+int
 ia_css_stream_start(struct ia_css_stream *stream);
 
 /* @brief Stop the stream.
  * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS or the error code.
+ * @return	0 or the error code.
  *
  * NOTE: this function will send stop event to pipes belong to this
  * stream but will not terminate threads.
  */
-enum ia_css_err
+int
 ia_css_stream_stop(struct ia_css_stream *stream);
 
 /* @brief Check if a stream has stopped
@@ -245,11 +246,11 @@ ia_css_stream_has_stopped(struct ia_css_stream *stream);
 
 /* @brief	destroy a stream according to the stream seed previosly saved in the seed array.
  * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS (no other errors are generated now)
+ * @return	0 (no other errors are generated now)
  *
  * Destroy the stream and all the pipes related to it.
  */
-enum ia_css_err
+int
 ia_css_stream_unload(struct ia_css_stream *stream);
 
 /* @brief Returns stream format
@@ -278,19 +279,19 @@ ia_css_stream_get_two_pixels_per_clock(const struct ia_css_stream *stream);
  *
  * This function will Set the output frame stride (at the last pipe)
  */
-enum ia_css_err
+int
 ia_css_stream_set_output_padded_width(struct ia_css_stream *stream,
 				      unsigned int output_padded_width);
 
 /* @brief Return max number of continuous RAW frames.
  * @param[in]	stream The stream.
  * @param[out]	buffer_depth The maximum number of continuous RAW frames.
- * @return	IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS
+ * @return	0 or -EINVAL
  *
  * This function will return the maximum number of continuous RAW frames
  * the system can support.
  */
-enum ia_css_err
+int
 ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream,
 				   int *buffer_depth);
 
@@ -298,22 +299,22 @@ ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream,
  *
  * @param[in]	stream The stream.
  * @param[in]	buffer_depth	Number of frames to set.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  * Set the number of continuous frames to use during continuous modes.
  */
-enum ia_css_err
+int
 ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth);
 
 /* @brief Get number of continuous RAW frames to use.
  * @param[in]	stream The stream.
  * @param[out]	buffer_depth The number of frames to use
- * @return	IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS
+ * @return	0 or -EINVAL
  *
  * Get the currently set number of continuous frames
  * to use during continuous modes.
  */
-enum ia_css_err
+int
 ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth);
 
 /* ===== CAPTURE ===== */
@@ -338,13 +339,13 @@ ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth);
  *				with this offset. This allows the user to
  *				process RAW frames that were captured in the
  *				past or future.
- * @return			IA_CSS_SUCCESS or error code upon error.
+ * @return			0 or error code upon error.
  *
  *  For example, to capture the current frame plus the 2 previous
  *  frames and 2 subsequent frames, you would call
  *  ia_css_stream_capture(5, 0, -2).
  */
-enum ia_css_err
+int
 ia_css_stream_capture(struct ia_css_stream *stream,
 		      int num_captures,
 		      unsigned int skip,
@@ -355,12 +356,12 @@ ia_css_stream_capture(struct ia_css_stream *stream,
  * @param[in]	stream The stream.
  * @param[in]	exp_id	The exposure id of the raw frame to tag.
  *
- * @return			IA_CSS_SUCCESS or error code upon error.
+ * @return			0 or error code upon error.
  *
  * This function allows the user to tag a raw frame based on the exposure id
  * found in the viewfinder frames' frame info.
  */
-enum ia_css_err
+int
 ia_css_stream_capture_frame(struct ia_css_stream *stream,
 			    unsigned int exp_id);
 
@@ -492,7 +493,7 @@ ia_css_stream_request_flash(struct ia_css_stream *stream);
  * @param[in]	config	The set of filter coefficients.
  * @param[in]   pipe Pipe to be updated when set isp config, NULL means to
  *		   update all pipes in the stream.
- * @return		IA_CSS_SUCCESS or error code upon error.
+ * @return		0 or error code upon error.
  *
  * This function configures the filter coefficients for an image
  * stream. For image pipes that do not execute any ISP filters, this
@@ -501,7 +502,7 @@ ia_css_stream_request_flash(struct ia_css_stream *stream);
  * in fact this is the expected behavior most of the time. Proper
  * resource locking and double buffering is in place to allow for this.
  */
-enum ia_css_err
+int
 ia_css_stream_set_isp_config_on_pipe(struct ia_css_stream *stream,
 				     const struct ia_css_isp_config *config,
 				     struct ia_css_pipe *pipe);
@@ -511,7 +512,7 @@ ia_css_stream_set_isp_config_on_pipe(struct ia_css_stream *stream,
  *				   ia_css_pipe_set_isp_config()}
  * @param[in]	stream	The stream.
  * @param[in]	config	The set of filter coefficients.
- * @return		IA_CSS_SUCCESS or error code upon error.
+ * @return		0 or error code upon error.
  *
  * This function configures the filter coefficients for an image
  * stream. For image pipes that do not execute any ISP filters, this
@@ -521,7 +522,7 @@ ia_css_stream_set_isp_config_on_pipe(struct ia_css_stream *stream,
  * in fact this is the expected behaviour most of the time. Proper
  * resource locking and double buffering is in place to allow for this.
  */
-enum ia_css_err
+int
 ia_css_stream_set_isp_config(
     struct ia_css_stream *stream,
     const struct ia_css_isp_config *config);
@@ -537,37 +538,37 @@ ia_css_stream_get_isp_config(const struct ia_css_stream *stream,
 
 /* @brief allocate continuous raw frames for continuous capture
  * @param[in]	stream The stream.
- * @return IA_CSS_SUCCESS or error code.
+ * @return 0 or error code.
  *
  *  because this allocation takes a long time (around 120ms per frame),
  *  we separate the allocation part and update part to let driver call
  *  this function without locking. This function is the allocation part
  *  and next one is update part
  */
-enum ia_css_err
+int
 ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream);
 
 /* @brief allocate continuous raw frames for continuous capture
  * @param[in]	stream The stream.
- * @return	IA_CSS_SUCCESS or error code.
+ * @return	0 or error code.
  *
  *  because this allocation takes a long time (around 120ms per frame),
  *  we separate the allocation part and update part to let driver call
  *  this function without locking. This function is the update part
  */
-enum ia_css_err
+int
 ia_css_update_continuous_frames(struct ia_css_stream *stream);
 
 /* @brief ia_css_unlock_raw_frame . unlock a raw frame (HALv3 Support)
  * @param[in]	stream The stream.
  * @param[in]   exp_id exposure id that uniquely identifies the locked Raw Frame Buffer
- * @return      ia_css_err IA_CSS_SUCCESS or error code
+ * @return      ia_css_err 0 or error code
  *
  * As part of HALv3 Feature requirement, SP locks raw buffer until the Application
  * releases its reference to a raw buffer (which are managed by SP), this function allows
  * application to explicitly unlock that buffer in SP.
  */
-enum ia_css_err
+int
 ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id);
 
 /* @brief ia_css_en_dz_capt_pipe . Enable/Disable digital zoom for capture pipe
diff --git a/drivers/staging/media/atomisp/pci/ia_css_timer.h b/drivers/staging/media/atomisp/pci/ia_css_timer.h
index a37cfa60ad35..c78fbda907a1 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_timer.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_timer.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
@@ -58,10 +59,10 @@ struct ia_css_time_meas {
 /* @brief API to fetch timer count directly
 *
 * @param curr_ts [out] measured count value
-* @return IA_CSS_SUCCESS if success
+* @return 0 if success
 *
 */
-enum ia_css_err
+int
 ia_css_timer_get_current_tick(
     struct ia_css_clock_tick *curr_ts);
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_tpg.h b/drivers/staging/media/atomisp/pci/ia_css_tpg.h
index 79c4e1b3b48f..8c744bedb0a6 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_tpg.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_tpg.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/ia_css_types.h b/drivers/staging/media/atomisp/pci/ia_css_types.h
index d3584756e34e..6e34d401f9df 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_types.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
@@ -306,7 +307,6 @@ struct ia_css_shading_info {
 			u32 bayer_scale_ver_ratio_in;
 			u32 bayer_scale_ver_ratio_out;
 
-
 			/** Vertical ratio of bayer scaling
 			between input height and output height, for the scaling
 			which should be done before shading correction.
@@ -392,8 +392,7 @@ struct ia_css_grid_info {
 };
 
 /* defaults for ia_css_grid_info structs */
-#define DEFAULT_GRID_INFO \
-(struct ia_css_grid_info) { \
+#define DEFAULT_GRID_INFO { \
 	.dvs_grid	= DEFAULT_DVS_GRID_INFO, \
 	.vamem_type	= IA_CSS_VAMEM_TYPE_1 \
 }
@@ -485,8 +484,7 @@ struct ia_css_capture_config {
 };
 
 /* default settings for ia_css_capture_config structs */
-#define DEFAULT_CAPTURE_CONFIG \
-(struct ia_css_capture_config) { \
+#define DEFAULT_CAPTURE_CONFIG { \
 	.mode	= IA_CSS_CAPTURE_MODE_PRIMARY, \
 }
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_version.h b/drivers/staging/media/atomisp/pci/ia_css_version.h
index 1e88901e0b82..cf1d010baceb 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_version.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_version.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -34,7 +35,7 @@
  * This function generates and returns the version string. If FW is loaded, it
  * attaches the FW version.
  */
-enum ia_css_err
+int
 ia_css_get_version(char *version, int max_size);
 
 #endif /* __IA_CSS_VERSION_H */
diff --git a/drivers/staging/media/atomisp/pci/ia_css_version_data.h b/drivers/staging/media/atomisp/pci/ia_css_version_data.h
index f630fa5d55cc..428d78e1616f 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_version_data.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_version_data.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/if_defs.h b/drivers/staging/media/atomisp/pci/if_defs.h
index 7d39e45796ae..e21efa749368 100644
--- a/drivers/staging/media/atomisp/pci/if_defs.h
+++ b/drivers/staging/media/atomisp/pci/if_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h b/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h
index 176456da961f..594fc36a01c7 100644
--- a/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h
+++ b/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/input_selector_defs.h b/drivers/staging/media/atomisp/pci/input_selector_defs.h
index 1dd8ea3cd6d4..61882e4cb813 100644
--- a/drivers/staging/media/atomisp/pci/input_selector_defs.h
+++ b/drivers/staging/media/atomisp/pci/input_selector_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h b/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h
index 2d5baae30522..8ea1d7991d38 100644
--- a/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h
+++ b/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h b/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h
index fcfa8c4971be..c801ddd71192 100644
--- a/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h
+++ b/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/input_system_defs.h b/drivers/staging/media/atomisp/pci/input_system_defs.h
index ae62163034a6..0c6a74b1891f 100644
--- a/drivers/staging/media/atomisp/pci/input_system_defs.h
+++ b/drivers/staging/media/atomisp/pci/input_system_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/input_system_global.h b/drivers/staging/media/atomisp/pci/input_system_global.h
index e75c2f29042d..5ac580ce64ed 100644
--- a/drivers/staging/media/atomisp/pci/input_system_global.h
+++ b/drivers/staging/media/atomisp/pci/input_system_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  *    (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/drivers/staging/media/atomisp/pci/input_system_local.h b/drivers/staging/media/atomisp/pci/input_system_local.h
index 8533a1e017e4..b33aa2838290 100644
--- a/drivers/staging/media/atomisp/pci/input_system_local.h
+++ b/drivers/staging/media/atomisp/pci/input_system_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  *    (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/drivers/staging/media/atomisp/pci/input_system_private.h b/drivers/staging/media/atomisp/pci/input_system_private.h
index 69c63a79a30c..889f204e77d5 100644
--- a/drivers/staging/media/atomisp/pci/input_system_private.h
+++ b/drivers/staging/media/atomisp/pci/input_system_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  *    (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/drivers/staging/media/atomisp/pci/input_system_public.h b/drivers/staging/media/atomisp/pci/input_system_public.h
index 17682c86bceb..3f5167fd6643 100644
--- a/drivers/staging/media/atomisp/pci/input_system_public.h
+++ b/drivers/staging/media/atomisp/pci/input_system_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  *    (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/drivers/staging/media/atomisp/pci/irq_controller_defs.h b/drivers/staging/media/atomisp/pci/irq_controller_defs.h
index efb3d7e135bd..e49e61e17ee7 100644
--- a/drivers/staging/media/atomisp/pci/irq_controller_defs.h
+++ b/drivers/staging/media/atomisp/pci/irq_controller_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h b/drivers/staging/media/atomisp/pci/irq_types_hrt.h
index 5c636effba48..4212bb01c8d8 100644
--- a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h
+++ b/drivers/staging/media/atomisp/pci/irq_types_hrt.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c
index 9cdfe50b2835..daf2f25c1ed6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h
index 71587d85ff2d..3abc125debd0 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h
index 3c699bae2f55..4f8bb4de4edc 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h
index cc6a444ac716..900ba8f5e30c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
index c190483dc2b3..3f079c954c1f 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
index 3855f54765e3..a4720c4a948a 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h
index 6bf834cb47d9..37dcb013b76d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h
index d3fa0193ae07..be3534e46c15 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c
index feee073b5099..9cdefedc6312 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h
index e99108682f5d..2b3ab01c279d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h
index 47a0fb08cfcc..4b83b8100160 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -21,7 +22,7 @@
 /* Advanced Noise Reduction (ANR) thresholds */
 
 struct ia_css_isp_anr2_params {
-	VMEM_ARRAY(data, ANR_PARAM_SIZE *ISP_VEC_NELEMS);
+	VMEM_ARRAY(data, ANR_PARAM_SIZE * ISP_VEC_NELEMS);
 };
 
 #endif /* __IA_CSS_ANR2_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c
index 070e90e3e2b5..649283bd44f2 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h
index 534119e064c1..9e383e030ac4 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h
index 200df3829fc7..e12aae819dce 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c
index 6c7aa51ec079..82aa69b74677 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -14,7 +15,6 @@
 
 #if !defined(HAS_NO_HMEM)
 
-#include "memory_access.h"
 #include "ia_css_types.h"
 #include "sh_css_internal.h"
 #include "assert_support.h"
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h
index ccd83169fe22..736b6e3f9512 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h
index 692a855ba012..05d5c43e6b16 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h
index 8b2a53a26b75..4c0e92f13d6c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c
index 6888a7363710..45e37dc4f1e3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -42,7 +43,7 @@ static const s32 div_lut_intercepts[BNLM_DIV_LUT_SIZE] = {
  */
 static inline void
 bnlm_lut_encode(struct bnlm_lut *lut, const int32_t *lut_thr,
-		const int32_t *lut_val, const uint32_t lut_size)
+		const s32 *lut_val, const uint32_t lut_size)
 {
 	u32 blk, i;
 	const u32 block_size = 16;
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h
index a57933bfb974..3632bf27cc21 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h
index c7d5cadf5fd4..30672db269df 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h
index 8dd1b1766c64..407b5a3b0fcd 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c
index a5e20596539d..c42fcb1d9100 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h
index a021733dcdf7..f6ab5d2bb218 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h
index 698fdc0b13fa..087723795476 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h
index ee9569891747..5f3dfa59f950 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c
index 5efb0ce7f323..457a004e194d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h
index 4c29b47b8177..7fc2a728a6c2 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h
index 52f21ce8f4d2..4f6469315386 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c
index c50afa6bf8a6..0eb40517e08c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h
index 87250ca5842c..4d046b730f06 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h
index c1af207cbf9a..971ab87af2c5 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c
index 610871d213bb..495dc1f33ca6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h
index d322359feedf..38f848137eda 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h
index 0d2fb2897720..3709aa4d3652 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h
index 35fc2e77eb3d..d0a25616727c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c
index e64e26089a4d..ff452e2cc23a 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h
index c136d5e03511..520623e27349 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h
index 3a6ede394bdc..fcbec189eff8 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
index 79a626fe3a29..34152d6d09be 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c
index 6e29b7eeb3ed..5d34f3256a43 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h
index 6f42abdec9bb..615cb6771884 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h
index 587d0c62c936..56daa1d96747 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c
index c6a3bd4fbf80..38912062edd4 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h
index 2e451a872d2a..21a259d33256 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h
index 35835929d252..7416e74dd782 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h
index 5c166be6c5e8..aaaae5e2abb9 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c
index ea81e1d3e445..284c17970e55 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h
index 347ccd864577..6b0256a73e52 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h
index 53e270df2db7..3809ef73e490 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h
index d49203d322bd..160f19bdfca0 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c
index e80f42ab0e6a..149adbc57730 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h
index f3c40a49f7c0..8c17e7b921b5 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h
index 95cf34ef4ed2..c18cfc930db6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
index b247dc6bec6a..e3d3f1253422 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h
index 3733aee24dcd..eb10c3884020 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h
index 224bdb199942..94844da665e5 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h
index 9d5dadf70f1a..f9f329a58737 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c
index 26311b0a23f9..82f2adbbfac3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h
index e4ad676361dd..57d1d08e1bc8 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h
index 6e541a0ebaa9..7e2fa192a0fe 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c
index adb146c03a73..f13b79586963 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,8 +13,9 @@
  * more details.
  */
 
+#include <linux/string.h> /* for memcpy() */
+
 #include <type_support.h>
-#include <string_support.h> /* memcpy */
 #include "system_global.h"
 #include "vamem.h"
 #include "ia_css_types.h"
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h
index a350dec8b4ad..33e8a05455a3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h
index f6f5ec28827f..b2d42f3c1f4d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c
index 8e4218cb70cd..25e3f0822fb8 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h
index baae1d9809da..cb91062029cf 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h
index c85a57e194cc..2070ce040470 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h
index a4b446904570..daac1275c129 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c
index ade23d53f6bb..f90da39296ec 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h
index f3749e514505..294f619a3b15 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h
index 868dfaaf78c7..4c9d5c630cb0 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h
index 24700d256bfd..372cd9d2b803 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c
index 9fb37447831c..eff428c67c86 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h
index 009541fafda0..e5f5a2716010 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h
index 8567a620696a..e0e7f2d48237 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h
index e96f83e5d47c..c1666ebf1d3b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c
index 4dfad4ace20b..f6fe064bdda4 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h
index a31ef0e5047b..f6e019a65208 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h
index 6df06fb249aa..1ccceadbb7bf 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h
index f78451be8d6a..f742a8dc1d67 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
index d2c3e8edf626..b8b71791466f 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,6 +13,8 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include "ia_css_frame_public.h"
 #define IA_CSS_INCLUDE_CONFIGURATIONS
 #include "ia_css_isp_configs.h"
@@ -22,7 +25,6 @@
 #include "sh_css_params.h"
 #include "ia_css_binary.h"
 #include "ia_css_debug.h"
-#include "memory_access.h"
 #include "assert_support.h"
 
 #include "ia_css_dvs.host.h"
@@ -270,12 +272,12 @@ convert_allocate_dvs_6axis_config(
 	return me;
 }
 
-enum ia_css_err
+int
 store_dvs_6axis_config(
     const struct ia_css_dvs_6axis_config *dvs_6axis_config,
     const struct ia_css_binary *binary,
     const struct ia_css_frame_info *dvs_in_frame_info,
-    hrt_vaddress ddr_addr_y) {
+    ia_css_ptr ddr_addr_y) {
 	struct ia_css_host_data *me;
 
 	assert(dvs_6axis_config);
@@ -288,8 +290,8 @@ store_dvs_6axis_config(
 
 	if (!me)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+		return -ENOMEM;
 	}
 
 	ia_css_params_store_ia_css_host_data(
@@ -297,5 +299,5 @@ store_dvs_6axis_config(
 	    me);
 	ia_css_host_data_free(me);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h
index d711170cf7cc..f9bc17ee0f86 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -50,11 +51,11 @@ convert_allocate_dvs_6axis_config(
     const struct ia_css_binary *binary,
     const struct ia_css_frame_info *dvs_in_frame_info);
 
-enum ia_css_err
+int
 store_dvs_6axis_config(
     const struct ia_css_dvs_6axis_config *dvs_6axis_config,
     const struct ia_css_binary *binary,
     const struct ia_css_frame_info *dvs_in_frame_info,
-    hrt_vaddress ddr_addr_y);
+    ia_css_ptr ddr_addr_y);
 
 #endif /* __IA_CSS_DVS_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h
index f8842dae943b..2963bb10b129 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h
index a1a14d93ef29..e99ff0ce8bab 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c
index 03e1998b0464..bfea78171f7c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h
index 05f817125d3c..f1ad07e78b3a 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h
index 880454d4dcf5..6fb3b38f49e7 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h
index b8fdb7a51a12..836e348c184f 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c
index 0b96b9618ab6..bae1ca2cd505 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h
index 0aac424d9d54..540423d85e9d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h
index 8f36af1a5ae6..5275a1dadefa 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h
index 7cfebaf05dc2..16b6a3ddcd08 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h
index adfd4b37171c..6cd635f3ee27 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h
index 6485834704da..3a55d4c698e6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c
index 7b55dfea359a..47b5c7956fbd 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h
index 02e85570bd1c..12187d213d90 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h
index f103ddd882fd..b21415743705 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h
index 95552a0e3c45..14dc5e183184 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c
index 1a489c93eb97..7f3f87920dc7 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h
index 2fb2927b07f1..c2dc1574a3da 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h
index beeba6c9be6a..a81233add437 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c
index 15cf0575aac5..f48f876777dc 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,8 +13,9 @@
  * more details.
  */
 
+#include <linux/string.h> /* for memcpy() */
+
 #include <type_support.h>
-#include <string_support.h> /* memcpy */
 #include "system_global.h"
 #include "vamem.h"
 #include "ia_css_types.h"
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h
index 9686623d9cdd..ee6fa07b3511 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h
index c896c138b569..ccd3d91a24d3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c
index 29a1e013a9aa..76209b7c14cb 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h
index ca7d54576471..eabf78737bfc 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h
index 458c72a45eef..af456e75e476 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c
index d2fe0052fb00..7eadb31268eb 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,8 +13,9 @@
  * more details.
  */
 
+#include <linux/string.h> /* for memcpy() */
+
 #include <type_support.h>
-#include <string_support.h> /* memcpy */
 #include "system_global.h"
 #include "vamem.h"
 #include "ia_css_types.h"
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h
index 8686e6e3586c..13049fbfab84 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h
index 30780394ed7f..ae16409d84a5 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c
index 643b7d9095e6..698550cc2fcc 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */
 /* Release Version: irci_ecr-master_20150911_0724 */
 /*
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h
index ecc8bea3542b..04599ab590cd 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */
 /* Release Version: irci_ecr-master_20150911_0724 */
 /*
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h
index 47651cdf94b7..97a89fd3cfda 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */
 /* Release Version: irci_ecr-master_20150911_0724 */
 /*
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h
index 7c2f8f213bef..1b4090880201 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */
 /* Release Version: irci_ecr-master_20150911_0724 */
 /*
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c
index bf71a7f661e6..ea8055148fb3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h
index f9db75a089af..635ccb1b27d0 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h
index 77cfed002e14..283ace8385b0 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h
index 59b58f30af11..d06d25c9b81c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
index 22aedcc4470f..5e0e4cd5bfba 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
index e49bd95f77da..0801481c4b49 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c
index ba490c5fc18e..f8bd207b28e1 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
index 556e53e05607..e7cfd380e108 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
index 1cc2aff57ef3..9b7537d508ad 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
index 990299a0d2c7..137a2a05c65b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c
index 49c1b3e3370d..6d8a35a73750 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -37,7 +38,7 @@ ia_css_iterator_config(
 	ia_css_resolution_to_sp_resolution(&to->dvs_envelope,  from->dvs_envelope);
 }
 
-enum ia_css_err
+int
 ia_css_iterator_configure(
     const struct ia_css_binary *binary,
     const struct ia_css_frame_info *in_info) {
@@ -76,5 +77,5 @@ ia_css_iterator_configure(
 
 	ia_css_configure_iterator(binary, &config);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h
index c5e8d58e0fe1..1419fa9a07f0 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -26,7 +27,7 @@ ia_css_iterator_config(
     const struct ia_css_iterator_configuration *from,
     unsigned int size);
 
-enum ia_css_err
+int
 ia_css_iterator_configure(
     const struct ia_css_binary *binary,
     const struct ia_css_frame_info *in_info);
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h
index d308126e41d3..e062f8d06128 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c
index 7a6abe0c5b7d..562662ab8a44 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h
index ae9ede2b685a..6c1189e1d263 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h
index 497ad89ab728..66b8fb259218 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -25,7 +26,7 @@ struct sh_css_isp_macc1_5_params {
 };
 
 struct sh_css_isp_macc1_5_vmem_params {
-	VMEM_ARRAY(data, IA_CSS_MACC_NUM_COEFS *ISP_NWAY);
+	VMEM_ARRAY(data, IA_CSS_MACC_NUM_COEFS * ISP_NWAY);
 };
 
 #endif /* __IA_CSS_MACC1_5_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c
index c094f3df10aa..d205d64e0b94 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h
index 10a50aa82be8..d451efbaa184 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h
index 9aa352cbcffc..5492af0dfa9f 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c
index 0b1d1bf5e8a0..f2d3832a0039 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h
index 0e13e9cb0547..912db92540e6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h
index 3b4e440c3c30..71665204e4dd 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c
index f9a430da54b8..946b074e8288 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h
index 96d62c9912b8..35099cb79d6a 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h
index 093302f08bca..172a518cb935 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c
index 102dc6feb6d1..69283b631da7 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h
index 42b5143ef78f..3987abcae85c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h
index d432e2e39df6..06c39fdfc9e3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c
index f7403ce16c99..c4ffff630b64 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h
index 936f6a08a174..26c2e43202dc 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h
index c728f8791ef4..c3c9fc3f9064 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h
index 0ccc09f6eb0f..51e4c35cf659 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c
index 6367d94275fb..12191cd36d5c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h
index d767c5856880..dfcac0c64040 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h
index f5c3e14a1a8a..991aa3c4051b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h
index 317b24e240d8..b74296517b03 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c
index df4cb9c362a4..c8e074f42353 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h
index 3d8f61c225cf..1f5a2242640e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h
index 3a63eee58cb6..df125674bb35 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h
index 3248bc3fd6c3..e5f9c05d2120 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c
index 3de108b56005..1603fd44ece3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h
index ad6d7ca783e4..8d940959f40a 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h
index 87898d2df2de..9ae290450865 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h
index b7ecd8f40c1c..549f1a36bb7e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
index 1a85f20770c1..1c6f6792d57b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h
index 36a4079aa24a..346928435a8b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h
index a1a314272a77..c4b5f719a336 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h
index 7838f59a2986..1ccaa3c48407 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c
index 2045b974ec8a..29c707ecf9f3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -14,7 +15,6 @@
 
 #if !defined(HAS_NO_HMEM)
 
-#include "memory_access.h"
 #include "ia_css_types.h"
 #include "sh_css_internal.h"
 #include "sh_css_frac.h"
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h
index d4df1dc540a0..4c2b3de7281e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c
index c3f43fd327d4..061558fbe329 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -29,10 +30,15 @@ ia_css_ref_config(
 {
 	unsigned int elems_a = ISP_VEC_NELEMS, i;
 
-	(void)size;
-	ia_css_dma_configure_from_info(&to->port_b, &from->ref_frames[0]->info);
-	to->width_a_over_b = elems_a / to->port_b.elems;
-	to->dvs_frame_delay = from->dvs_frame_delay;
+	if (from->ref_frames[0]) {
+		ia_css_dma_configure_from_info(&to->port_b, &from->ref_frames[0]->info);
+		to->width_a_over_b = elems_a / to->port_b.elems;
+		to->dvs_frame_delay = from->dvs_frame_delay;
+	} else {
+		to->width_a_over_b = 1;
+		to->dvs_frame_delay = 0;
+		to->port_b.elems = elems_a;
+	}
 	for (i = 0; i < MAX_NUM_VIDEO_DELAY_FRAMES; i++) {
 		if (from->ref_frames[i]) {
 			to->ref_frame_addr_y[i] = from->ref_frames[i]->data +
@@ -52,7 +58,7 @@ ia_css_ref_config(
 void
 ia_css_ref_configure(
     const struct ia_css_binary     *binary,
-    const struct ia_css_frame **ref_frames,
+    const struct ia_css_frame * const *ref_frames,
     const uint32_t dvs_frame_delay)
 {
 	struct ia_css_ref_configuration config;
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h
index 4f48a8cfc604..3ce590b436a1 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -31,7 +32,7 @@ ia_css_ref_config(
 void
 ia_css_ref_configure(
     const struct ia_css_binary     *binary,
-    const struct ia_css_frame **ref_frames,
+    const struct ia_css_frame * const *ref_frames,
     const uint32_t dvs_frame_delay);
 
 void
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h
index 0a0498c17fba..c727e27a8e14 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -28,8 +29,8 @@ struct ia_css_ref_configuration {
 struct sh_css_isp_ref_isp_config {
 	u32 width_a_over_b;
 	struct dma_port_config port_b;
-	hrt_vaddress ref_frame_addr_y[MAX_NUM_VIDEO_DELAY_FRAMES];
-	hrt_vaddress ref_frame_addr_c[MAX_NUM_VIDEO_DELAY_FRAMES];
+	ia_css_ptr ref_frame_addr_y[MAX_NUM_VIDEO_DELAY_FRAMES];
+	ia_css_ptr ref_frame_addr_c[MAX_NUM_VIDEO_DELAY_FRAMES];
 	u32 dvs_frame_delay;
 };
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h
index 1d30ccc2c638..d4f7a66763eb 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h
index 156d6cd8cf3a..07d040bcf281 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
index d093565d9eb8..ba52c80df4a5 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h
index 13d19dab1f1d..f9926e297568 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h
index 041101767ff2..9cb75b220678 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
index 5a5b277ca0eb..f369e9b95ca8 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c
index 43954ed6d106..f3fb4b9b3c82 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h
index efbe40b399dd..f1eb568f23d4 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h
index 38a625821987..fab11d3350fd 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h
index 41f3ee7158ff..aae534521b7b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h
index c03936fb0550..7b661e49b4d0 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -76,12 +77,12 @@ struct sh_css_isp_sdis_vert_proj_tbl {
 
 struct sh_css_isp_sdis_hori_coef_tbl {
 	VMEM_ARRAY(tbl[ISP_DVS_NUM_COEF_TYPES],
-		   ISP_MAX_SDIS_HOR_COEF_NUM_VECS *ISP_NWAY);
+		   ISP_MAX_SDIS_HOR_COEF_NUM_VECS * ISP_NWAY);
 };
 
 struct sh_css_isp_sdis_vert_coef_tbl {
 	VMEM_ARRAY(tbl[ISP_DVS_NUM_COEF_TYPES],
-		   ISP_MAX_SDIS_VER_COEF_NUM_VECS *ISP_NWAY);
+		   ISP_MAX_SDIS_VER_COEF_NUM_VECS * ISP_NWAY);
 };
 
 #endif /* defined(__ISP) || defined (MK_FIRMWARE) */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h
index e257841bba67..c2ec30b4abd4 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -196,12 +197,11 @@ struct ia_css_dvs_stat_grid_info {
 
 /* DVS statistics generated by accelerator default grid info
  */
-#define DEFAULT_DVS_GRID_INFO \
-(union ia_css_dvs_grid_u) { \
-	.dvs_stat_grid_info = (struct ia_css_dvs_stat_grid_info) { \
+#define DEFAULT_DVS_GRID_INFO { \
+	.dvs_stat_grid_info = { \
 		.fe_roi_cfg = { \
-			[1] = (struct dvs_stat_public_dvs_level_fe_roi_cfg) { \
-				.x_start = 4 \
+			[1] = { \
+			    .x_start = 4 \
 			} \
 		} \
 	} \
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
index 418481e016f7..3e72dabe7ef6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,7 +13,8 @@
  * more details.
  */
 
-#include "memory_access.h"
+#include "hmm.h"
+
 #include "assert_support.h"
 #include "ia_css_debug.h"
 #include "ia_css_sdis_types.h"
@@ -234,12 +236,12 @@ void ia_css_sdis_clear_coefficients(
 	dvs_coefs->ver_coefs = NULL;
 }
 
-enum ia_css_err
+int
 ia_css_get_dvs_statistics(
     struct ia_css_dvs_statistics	       *host_stats,
     const struct ia_css_isp_dvs_statistics *isp_stats) {
 	struct ia_css_isp_dvs_statistics_map *map;
-	enum ia_css_err ret = IA_CSS_SUCCESS;
+	int ret = 0;
 
 	IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats);
 
@@ -249,13 +251,13 @@ ia_css_get_dvs_statistics(
 	map = ia_css_isp_dvs_statistics_map_allocate(isp_stats, NULL);
 	if (map)
 	{
-		mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
+		hmm_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
 		ia_css_translate_dvs_statistics(host_stats, map);
 		ia_css_isp_dvs_statistics_map_free(map);
 	} else
 	{
 		IA_CSS_ERROR("out of memory");
-		ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		ret = -ENOMEM;
 	}
 
 	IA_CSS_LEAVE_ERR(ret);
@@ -317,7 +319,7 @@ ia_css_isp_dvs_statistics_allocate(
 	if (!grid->enable)
 		return NULL;
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
@@ -329,7 +331,7 @@ ia_css_isp_dvs_statistics_allocate(
 			    HIVE_ISP_DDR_WORD_BYTES);
 
 	me->size = hor_size + ver_size;
-	me->data_ptr = mmgr_malloc(me->size);
+	me->data_ptr = hmm_alloc(me->size, HMM_BO_PRIVATE, 0, NULL, 0);
 	if (me->data_ptr == mmgr_NULL)
 		goto err;
 	me->hor_size = hor_size;
@@ -358,7 +360,7 @@ ia_css_isp_dvs_statistics_map_allocate(
 	 * so we use a local char * instead. */
 	char *base_ptr;
 
-	me = sh_css_malloc(sizeof(*me));
+	me = kvmalloc(sizeof(*me), GFP_KERNEL);
 	if (!me) {
 		IA_CSS_LOG("cannot allocate memory");
 		goto err;
@@ -368,7 +370,7 @@ ia_css_isp_dvs_statistics_map_allocate(
 	me->data_allocated = !data_ptr;
 
 	if (!me->data_ptr) {
-		me->data_ptr = sh_css_malloc(isp_stats->size);
+		me->data_ptr = kvmalloc(isp_stats->size, GFP_KERNEL);
 		if (!me->data_ptr) {
 			IA_CSS_LOG("cannot allocate memory");
 			goto err;
@@ -385,7 +387,7 @@ ia_css_isp_dvs_statistics_map_allocate(
 	return me;
 err:
 	if (me)
-		sh_css_free(me);
+		kvfree(me);
 	return NULL;
 }
 
@@ -394,8 +396,8 @@ ia_css_isp_dvs_statistics_map_free(struct ia_css_isp_dvs_statistics_map *me)
 {
 	if (me) {
 		if (me->data_allocated)
-			sh_css_free(me->data_ptr);
-		sh_css_free(me);
+			kvfree(me->data_ptr);
+		kvfree(me);
 	}
 }
 
@@ -404,7 +406,7 @@ ia_css_isp_dvs_statistics_free(struct ia_css_isp_dvs_statistics *me)
 {
 	if (me) {
 		hmm_free(me->data_ptr);
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h
index b1b0cb8ea175..0d0ed96e08fe 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -53,7 +54,7 @@ void ia_css_get_isp_dis_coefficients(
     short *horizontal_coefficients,
     short *vertical_coefficients);
 
-enum ia_css_err
+int
 ia_css_get_dvs_statistics(
     struct ia_css_dvs_statistics	       *host_stats,
     const struct ia_css_isp_dvs_statistics *isp_stats);
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h
index 5542fa5555b4..a8f2b8afcfd6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
index 20fa7d924d58..7922198f6784 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,8 +13,9 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include <assert_support.h>
-#include "memory_access.h"
 #include "ia_css_debug.h"
 #include "ia_css_sdis2.host.h"
 
@@ -174,12 +176,12 @@ void ia_css_sdis2_clear_coefficients(
 	dvs2_coefs->ver_coefs.even_imag = NULL;
 }
 
-enum ia_css_err
+int
 ia_css_get_dvs2_statistics(
     struct ia_css_dvs2_statistics          *host_stats,
     const struct ia_css_isp_dvs_statistics *isp_stats) {
 	struct ia_css_isp_dvs_statistics_map *map;
-	enum ia_css_err ret = IA_CSS_SUCCESS;
+	int ret = 0;
 
 	IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats);
 
@@ -189,13 +191,13 @@ ia_css_get_dvs2_statistics(
 	map = ia_css_isp_dvs_statistics_map_allocate(isp_stats, NULL);
 	if (map)
 	{
-		mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
+		hmm_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
 		ia_css_translate_dvs2_statistics(host_stats, map);
 		ia_css_isp_dvs_statistics_map_free(map);
 	} else
 	{
 		IA_CSS_ERROR("out of memory");
-		ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		ret = -ENOMEM;
 	}
 
 	IA_CSS_LEAVE_ERR(ret);
@@ -284,7 +286,7 @@ ia_css_isp_dvs2_statistics_allocate(
 	if (!grid->enable)
 		return NULL;
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
@@ -295,7 +297,7 @@ ia_css_isp_dvs2_statistics_allocate(
 	       * grid->aligned_height * IA_CSS_DVS2_NUM_COEF_TYPES;
 
 	me->size = 2 * size;
-	me->data_ptr = mmgr_malloc(me->size);
+	me->data_ptr = hmm_alloc(me->size, HMM_BO_PRIVATE, 0, NULL, 0);
 	if (me->data_ptr == mmgr_NULL)
 		goto err;
 	me->hor_proj = me->data_ptr;
@@ -317,7 +319,7 @@ ia_css_isp_dvs2_statistics_free(struct ia_css_isp_dvs_statistics *me)
 {
 	if (me) {
 		hmm_free(me->data_ptr);
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h
index a966a6bcb692..e0e6b9c338ad 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -62,7 +63,7 @@ void ia_css_get_isp_dvs2_coefficients(
 void ia_css_sdis2_clear_coefficients(
     struct ia_css_dvs2_coefficients *dvs2_coefs);
 
-enum ia_css_err
+int
 ia_css_get_dvs2_statistics(
     struct ia_css_dvs2_statistics	       *host_stats,
     const struct ia_css_isp_dvs_statistics *isp_stats);
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h
index e8ae135bfd6a..d75b72e9551b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c
index 69921c27bfae..fef8c5457cda 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h
index bc6e1653e354..7e44d78c5d5d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h
index a93891448cde..e904f7122142 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h
index e4263afee7da..0f69f9128f0e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
index 349f0800bbe6..4b53fddfcd2d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c
index ecbd3042951a..ac80e6c6e67e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -95,7 +96,7 @@ ia_css_tnr_config(
 void
 ia_css_tnr_configure(
     const struct ia_css_binary     *binary,
-    const struct ia_css_frame **frames)
+    const struct ia_css_frame * const *frames)
 {
 	struct ia_css_tnr_configuration config;
 	unsigned int i;
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h
index 3dbf962089d0..90d6e6b44a8d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -47,7 +48,7 @@ ia_css_tnr_config(
 void
 ia_css_tnr_configure(
     const struct ia_css_binary     *binary,
-    const struct ia_css_frame **frames);
+    const struct ia_css_frame * const *frames);
 
 void
 ia_css_init_tnr_state(
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h
index 1973766d8e41..60a2542cf685 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -34,7 +35,7 @@ struct sh_css_isp_tnr_isp_config {
 	u32 width_a_over_b;
 	u32 frame_height;
 	struct dma_port_config port_b;
-	hrt_vaddress tnr_frame_addr[NUM_TNR_FRAMES];
+	ia_css_ptr tnr_frame_addr[NUM_TNR_FRAMES];
 };
 
 #endif /* __IA_CSS_TNR_PARAM_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h
index 901aa1e298e0..d57238423947 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h
index 98b0daeeab39..92dbe13895c6 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h
index 26b7b5bc9391..784b5c4facd2 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
index be274d680caf..358cb7d2cd4c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -49,7 +50,7 @@ ia_css_vf_config(
  * to the requested viewfinder resolution on the upper side. The output cannot
  * be smaller than the requested viewfinder resolution.
  */
-enum ia_css_err
+int
 sh_css_vf_downscale_log2(
     const struct ia_css_frame_info *out_info,
     const struct ia_css_frame_info *vf_info,
@@ -58,12 +59,12 @@ sh_css_vf_downscale_log2(
 	unsigned int out_width;
 
 	if ((!out_info) | (!vf_info))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	out_width = out_info->res.width;
 
 	if (out_width == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/* downscale until width smaller than the viewfinder width. We don't
 	* test for the height since the vmem buffers only put restrictions on
@@ -79,26 +80,26 @@ sh_css_vf_downscale_log2(
 		ds_log2--;
 	/* TODO: use actual max input resolution of vf_pp binary */
 	if ((out_info->res.width >> ds_log2) >= 2 * ia_css_binary_max_vf_width())
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	*downscale_log2 = ds_log2;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 configure_kernel(
     const struct ia_css_binary_info *info,
     const struct ia_css_frame_info *out_info,
     const struct ia_css_frame_info *vf_info,
     unsigned int *downscale_log2,
     struct ia_css_vf_configuration *config) {
-	enum ia_css_err err;
+	int err;
 	unsigned int vf_log_ds = 0;
 
 	/* First compute value */
 	if (vf_info)
 	{
 		err = sh_css_vf_downscale_log2(out_info, vf_info, &vf_log_ds);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 	vf_log_ds = min(vf_log_ds, info->vf_dec.max_log_downscale);
@@ -106,7 +107,7 @@ configure_kernel(
 
 	/* Then store it in isp config section */
 	config->vf_downscale_bits = vf_log_ds;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -117,13 +118,13 @@ configure_dma(
 	config->info = vf_info;
 }
 
-enum ia_css_err
+int
 ia_css_vf_configure(
     const struct ia_css_binary *binary,
     const struct ia_css_frame_info *out_info,
     struct ia_css_frame_info *vf_info,
     unsigned int *downscale_log2) {
-	enum ia_css_err err;
+	int err;
 	struct ia_css_vf_configuration config;
 	const struct ia_css_binary_info *info = &binary->info->sp;
 
@@ -134,5 +135,5 @@ ia_css_vf_configure(
 		vf_info->raw_bit_depth = info->dma.vfdec_bits_per_pixel;
 	ia_css_configure_vf(binary, &config);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h
index 9cc594f9a840..0e8de034a00e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -25,7 +26,7 @@
  * to the requested viewfinder resolution on the upper side. The output cannot
  * be smaller than the requested viewfinder resolution.
  */
-enum ia_css_err
+int
 sh_css_vf_downscale_log2(
     const struct ia_css_frame_info *out_info,
     const struct ia_css_frame_info *vf_info,
@@ -37,7 +38,7 @@ ia_css_vf_config(
     const struct ia_css_vf_configuration *from,
     unsigned int size);
 
-enum ia_css_err
+int
 ia_css_vf_configure(
     const struct ia_css_binary *binary,
     const struct ia_css_frame_info *out_info,
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h
index 171a98508a88..487ddf163324 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h
index a4d39e2e9d8e..24fbb61d349c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c
index d07c500eb542..01d1a2d361c3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h
index 545dea39c2e0..ffd75c8a64fb 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h
index dcf548da55cc..51b2ba8efc18 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h
index 59cbd71ef332..20ae73c0ef8c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c
index e04c604ba612..1cd59660857f 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h
index 31833b78739f..686101c0b6a9 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h
index 72a5c5fd10e7..7ebf139f3618 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c
index 78653b2666a4..5566f3c16aac 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,8 +13,9 @@
  * more details.
  */
 
+#include <linux/string.h> /* for memcpy() */
+
 #include <type_support.h>
-#include <string_support.h> /* memcpy */
 #include "system_global.h"
 #include "vamem.h"
 #include "ia_css_types.h"
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h
index 130086713a7f..2f4ab8ad402b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h
index 22189c936f64..9a4d2e470524 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c
index a9db6366d20b..9c9d9b9a453e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -126,7 +127,7 @@ compute_blending(int strength)
 	 * exactly as s0.11 fixed point, but -1.0 can.
 	 */
 	isp_strength = -(((strength * isp_scale) + offset) / host_scale);
-	return MAX(MIN(isp_strength, 0), -XNR_BLENDING_SCALE_FACTOR);
+	return MAX(MIN(isp_strength, 0), -isp_scale);
 }
 
 void
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h
index 959533ec29c6..6b57990b72da 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h
index 7d108669e19a..c728db7ce917 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
index 6963bef3c07d..4447ba31ad69 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c
index a1d0e915636d..048ffbc90b8e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h
index 20165093a298..049706e1ffa9 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h
index 8f104bcb4d0f..8d9069ec28a1 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h
index 1a62e1dbfc6f..da1ba21a0726 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c
index 9a3cd59c4507..08e9d72c143f 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h
index 38204f8c5735..eaf253d59c4d 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h
index 7479bce598d5..96c80b3af426 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h
index 36e4bb61b38c..ab77f4e85319 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h b/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h
index 5774c905d8e1..f86cf9bf13a5 100644
--- a/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h
+++ b/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h
index fc392c7fb18b..11e439d838ae 100644
--- a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h
+++ b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h
index 6bdf8451e7d4..ae273c826808 100644
--- a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h
index 759141c9310a..b4142bdde51b 100644
--- a/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h
+++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h
index 3c0e2efb08ae..33ebf89ca053 100644
--- a/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h
+++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
@@ -20,15 +21,15 @@
 #include "input_system_global.h"
 
 #include "input_system_defs.h"		/* HIVE_ISYS_GPREG_MULTICAST_A_IDX,... */
-#include "css_receiver_2400_defs.h"	/* _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX, _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX,... */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM)
-#include "isp_capture_defs.h"
-#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-/* Same name, but keep the distinction,it is a different device */
+
+/*
+ * _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX,
+ * _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX,...
+ */
+#include "css_receiver_2400_defs.h"
+
 #include "isp_capture_defs.h"
-#else
-#error "input_system_local.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
+
 #include "isp_acquisition_defs.h"
 #include "input_system_ctrl_defs.h"
 
diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h
index 0ce9cbc0063e..9c39ca2da923 100644
--- a/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h
+++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h
index d0de27abb95a..689e451f1ce2 100644
--- a/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h
+++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2400_support.h b/drivers/staging/media/atomisp/pci/isp2400_support.h
index e9106d1e6a63..06d04853d4ee 100644
--- a/drivers/staging/media/atomisp/pci/isp2400_support.h
+++ b/drivers/staging/media/atomisp/pci/isp2400_support.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2400_system_global.h b/drivers/staging/media/atomisp/pci/isp2400_system_global.h
index 06fce25f2034..d87ddf1d2fe9 100644
--- a/drivers/staging/media/atomisp/pci/isp2400_system_global.h
+++ b/drivers/staging/media/atomisp/pci/isp2400_system_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -46,27 +47,6 @@
  * N.B. the 3 input formatters are of 2 different classess
  */
 
-/*
- * Since this file is visible everywhere and the system definition
- * macros are not, detect the separate definitions for {host, SP, ISP}
- *
- * The 2401 system has the nice property that it uses a vanilla 2400 SP
- * so the SP will believe it is a 2400 system rather than 2401...
- */
-//#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) || defined(__scalar_processor_2401)
-#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada)
-#define IS_ISP_2401_MAMOIADA_SYSTEM
-#define HAS_ISP_2401_MAMOIADA
-#define HAS_SP_2400
-//#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) || defined(__scalar_processor_2400)
-#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada)
-#define IS_ISP_2400_MAMOIADA_SYSTEM
-#define HAS_ISP_2400_MAMOIADA
-#define HAS_SP_2400
-#else
-#error "system_global.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
-
 #define USE_INPUT_SYSTEM_VERSION_2
 
 #define HAS_MMU_VERSION_2
@@ -129,21 +109,11 @@ typedef enum {
 	N_SP_ID
 } sp_ID_t;
 
-#if defined(IS_ISP_2401_MAMOIADA_SYSTEM)
 typedef enum {
 	MMU0_ID = 0,
 	MMU1_ID,
 	N_MMU_ID
 } mmu_ID_t;
-#elif defined(IS_ISP_2400_MAMOIADA_SYSTEM)
-typedef enum {
-	MMU0_ID = 0,
-	MMU1_ID,
-	N_MMU_ID
-} mmu_ID_t;
-#else
-#error "system_global.h: SYSTEM must be one of {2400, 2401}"
-#endif
 
 typedef enum {
 	DMA0_ID = 0,
diff --git a/drivers/staging/media/atomisp/pci/isp2400_system_local.h b/drivers/staging/media/atomisp/pci/isp2400_system_local.h
index ee38059d6ceb..675b8e5bdcc1 100644
--- a/drivers/staging/media/atomisp/pci/isp2400_system_local.h
+++ b/drivers/staging/media/atomisp/pci/isp2400_system_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
@@ -19,8 +20,6 @@
 #ifndef HRT_USE_VIR_ADDRS
 #define HRT_USE_VIR_ADDRS
 #endif
-/* This interface is deprecated */
-/*#include "hive_isp_css_custom_host_hrt.h"*/
 #endif
 
 #include "system_global.h"
@@ -81,7 +80,6 @@ static const hrt_address SP_PMEM_BASE[N_SP_ID] = {
 };
 
 /* MMU */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM)
 /*
  * MMU0_ID: The data MMU
  * MMU1_ID: The icache MMU
@@ -90,9 +88,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = {
 	(hrt_address)0x0000000000070000ULL,
 	(hrt_address)0x00000000000A0000ULL
 };
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
 
 /* DMA */
 static const hrt_address DMA_BASE[N_DMA_ID] = {
@@ -106,6 +101,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = {
 	(hrt_address)0x000000000008C000ULL,
 	(hrt_address)0x0000000000090200ULL
 };
+
 /*
 	(hrt_address)0x0000000000000500ULL};
  */
@@ -161,6 +157,7 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
 static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
 	(hrt_address)0x0000000000080000ULL
 };
+
 /*	(hrt_address)0x0000000000081000ULL, */ /* capture A */
 /*	(hrt_address)0x0000000000082000ULL, */ /* capture B */
 /*	(hrt_address)0x0000000000083000ULL, */ /* capture C */
@@ -222,7 +219,6 @@ static const hrt_address SP_PMEM_BASE[N_SP_ID] = {
 };
 
 /* MMU */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM)
 /*
  * MMU0_ID: The data MMU
  * MMU1_ID: The icache MMU
@@ -231,9 +227,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = {
 	(hrt_address)0x00070000UL,
 	(hrt_address)0x000A0000UL
 };
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
 
 /* DMA */
 static const hrt_address DMA_BASE[N_DMA_ID] = {
@@ -247,6 +240,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = {
 	(hrt_address)0x0008C000UL,
 	(hrt_address)0x00090200UL
 };
+
 /*
 	(hrt_address)0x00000500UL};
  */
@@ -297,12 +291,14 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
 	(hrt_address)0x00030200UL,
 	(hrt_address)0x00030400UL
 };
+
 /*	(hrt_address)0x00030600UL, */ /* memcpy() */
 
 /* INPUT_SYSTEM */
 static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
 	(hrt_address)0x00080000UL
 };
+
 /*	(hrt_address)0x00081000UL, */ /* capture A */
 /*	(hrt_address)0x00082000UL, */ /* capture B */
 /*	(hrt_address)0x00083000UL, */ /* capture C */
diff --git a/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h b/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h
index 9c882fe134f4..5070e651f7c4 100644
--- a/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h
+++ b/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h b/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h
index f199423e28da..f52a8ca5f86b 100644
--- a/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h
+++ b/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h b/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h
index 3f60f59ae51f..f3ca5d1bcb01 100644
--- a/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h
+++ b/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp2401_mamoiada_params.h b/drivers/staging/media/atomisp/pci/isp2401_mamoiada_params.h
deleted file mode 100644
index 42f821473826..000000000000
--- a/drivers/staging/media/atomisp/pci/isp2401_mamoiada_params.h
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* Version */
-#define RTL_VERSION
-
-/* Cell name  */
-#define ISP_CELL_TYPE                          isp2401_mamoiada
-#define ISP_VMEM                               simd_vmem
-#define _HRT_ISP_VMEM                          isp2401_mamoiada_simd_vmem
-
-/* instruction pipeline depth */
-#define ISP_BRANCHDELAY                        5
-
-/* bus */
-#define ISP_BUS_WIDTH                          32
-#define ISP_BUS_ADDR_WIDTH                     32
-#define ISP_BUS_BURST_SIZE                     1
-
-/* data-path */
-#define ISP_SCALAR_WIDTH                       32
-#define ISP_SLICE_NELEMS                       4
-#define ISP_VEC_NELEMS                         64
-#define ISP_VEC_ELEMBITS                       14
-#define ISP_VEC_ELEM8BITS                      16
-#define ISP_CLONE_DATAPATH_IS_16               1
-
-/* memories */
-#define ISP_DMEM_DEPTH                         4096
-#define ISP_DMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_DEPTH                         3072
-#define ISP_VMEM_BSEL_DOWNSAMPLE               8
-#define ISP_VMEM_ELEMBITS                      14
-#define ISP_VMEM_ELEM_PRECISION                14
-#define ISP_PMEM_DEPTH                         2048
-#define ISP_PMEM_WIDTH                         640
-#define ISP_VAMEM_ADDRESS_BITS                 12
-#define ISP_VAMEM_ELEMBITS                     12
-#define ISP_VAMEM_DEPTH                        2048
-#define ISP_VAMEM_ALIGNMENT                    2
-#define ISP_VA_ADDRESS_WIDTH                   896
-#define ISP_VEC_VALSU_LATENCY                  ISP_VEC_NELEMS
-#define ISP_HIST_ADDRESS_BITS                  12
-#define ISP_HIST_ALIGNMENT                     4
-#define ISP_HIST_COMP_IN_PREC                  12
-#define ISP_HIST_DEPTH                         1024
-#define ISP_HIST_WIDTH                         24
-#define ISP_HIST_COMPONENTS                    4
-
-/* program counter */
-#define ISP_PC_WIDTH                           13
-
-/* Template switches */
-#define ISP_SHIELD_INPUT_DMEM                  0
-#define ISP_SHIELD_OUTPUT_DMEM                 1
-#define ISP_SHIELD_INPUT_VMEM                  0
-#define ISP_SHIELD_OUTPUT_VMEM                 0
-#define ISP_SHIELD_INPUT_PMEM                  1
-#define ISP_SHIELD_OUTPUT_PMEM                 1
-#define ISP_SHIELD_INPUT_HIST                  1
-#define ISP_SHIELD_OUTPUT_HIST                 1
-/* When LUT is select the shielding is always on */
-#define ISP_SHIELD_INPUT_VAMEM                 1
-#define ISP_SHIELD_OUTPUT_VAMEM                1
-
-#define ISP_HAS_IRQ                            1
-#define ISP_HAS_SOFT_RESET                     1
-#define ISP_HAS_VEC_DIV                        0
-#define ISP_HAS_VFU_W_2O                       1
-#define ISP_HAS_DEINT3                         1
-#define ISP_HAS_LUT                            1
-#define ISP_HAS_HIST                           1
-#define ISP_HAS_VALSU                          1
-#define ISP_HAS_3rdVALSU                       1
-#define ISP_VRF1_HAS_2P                        1
-
-#define ISP_SRU_GUARDING                       1
-#define ISP_VLSU_GUARDING                      1
-
-#define ISP_VRF_RAM			     1
-#define ISP_SRF_RAM			     1
-
-#define ISP_SPLIT_VMUL_VADD_IS                 0
-#define ISP_RFSPLIT_FPGA                       0
-
-/* RSN or Bus pipelining */
-#define ISP_RSN_PIPE                           1
-#define ISP_VSF_BUS_PIPE                       0
-
-/* extra slave port to vmem */
-#define ISP_IF_VMEM                            0
-#define ISP_GDC_VMEM                           0
-
-/* Streaming ports */
-#define ISP_IF                                 1
-#define ISP_IF_B                               1
-#define ISP_GDC                                1
-#define ISP_SCL                                1
-#define ISP_GPFIFO                             1
-#define ISP_SP                                 1
-
-/* Removing Issue Slot(s) */
-#define ISP_HAS_NOT_SIMD_IS2                   0
-#define ISP_HAS_NOT_SIMD_IS3                   0
-#define ISP_HAS_NOT_SIMD_IS4                   0
-#define ISP_HAS_NOT_SIMD_IS4_VADD              0
-#define ISP_HAS_NOT_SIMD_IS5                   0
-#define ISP_HAS_NOT_SIMD_IS6                   0
-#define ISP_HAS_NOT_SIMD_IS7                   0
-#define ISP_HAS_NOT_SIMD_IS8                   0
-
-/* ICache  */
-#define ISP_ICACHE                             1
-#define ISP_ICACHE_ONLY                        0
-#define ISP_ICACHE_PREFETCH                    1
-#define ISP_ICACHE_INDEX_BITS                  8
-#define ISP_ICACHE_SET_BITS                    5
-#define ISP_ICACHE_BLOCKS_PER_SET_BITS         1
-
-/* Experimental Flags */
-#define ISP_EXP_1                              0
-#define ISP_EXP_2                              0
-#define ISP_EXP_3                              0
-#define ISP_EXP_4                              0
-#define ISP_EXP_5                              0
-#define ISP_EXP_6                              0
-
-/* Derived values */
-#define ISP_LOG2_PMEM_WIDTH                    10
-#define ISP_VEC_WIDTH                          896
-#define ISP_SLICE_WIDTH                        56
-#define ISP_VMEM_WIDTH                         896
-#define ISP_VMEM_ALIGN                         128
-#define ISP_SIMDLSU                            1
-#define ISP_LSU_IMM_BITS                       12
-
-/* convenient shortcuts for software*/
-#define ISP_NWAY                               ISP_VEC_NELEMS
-#define NBITS                                  ISP_VEC_ELEMBITS
-
-#define _isp_ceil_div(a, b)                     (((a) + (b) - 1) / (b))
-
-#define ISP_VEC_ALIGN                          ISP_VMEM_ALIGN
-
-/* HRT specific vector support */
-#define isp2401_mamoiada_vector_alignment         ISP_VEC_ALIGN
-#define isp2401_mamoiada_vector_elem_bits         ISP_VMEM_ELEMBITS
-#define isp2401_mamoiada_vector_elem_precision    ISP_VMEM_ELEM_PRECISION
-#define isp2401_mamoiada_vector_num_elems         ISP_VEC_NELEMS
-
-/* register file sizes */
-#define ISP_RF0_SIZE        64
-#define ISP_RF1_SIZE        16
-#define ISP_RF2_SIZE        64
-#define ISP_RF3_SIZE        4
-#define ISP_RF4_SIZE        64
-#define ISP_RF5_SIZE        16
-#define ISP_RF6_SIZE        16
-#define ISP_RF7_SIZE        16
-#define ISP_RF8_SIZE        16
-#define ISP_RF9_SIZE        16
-#define ISP_RF10_SIZE       16
-#define ISP_RF11_SIZE       16
-#define ISP_VRF1_SIZE       32
-#define ISP_VRF2_SIZE       32
-#define ISP_VRF3_SIZE       32
-#define ISP_VRF4_SIZE       32
-#define ISP_VRF5_SIZE       32
-#define ISP_VRF6_SIZE       32
-#define ISP_VRF7_SIZE       32
-#define ISP_VRF8_SIZE       32
-#define ISP_SRF1_SIZE       4
-#define ISP_SRF2_SIZE       64
-#define ISP_SRF3_SIZE       64
-#define ISP_SRF4_SIZE       32
-#define ISP_SRF5_SIZE       64
-#define ISP_FRF0_SIZE       16
-#define ISP_FRF1_SIZE       4
-#define ISP_FRF2_SIZE       16
-#define ISP_FRF3_SIZE       4
-#define ISP_FRF4_SIZE       4
-#define ISP_FRF5_SIZE       8
-#define ISP_FRF6_SIZE       4
-/* register file read latency */
-#define ISP_VRF1_READ_LAT       1
-#define ISP_VRF2_READ_LAT       1
-#define ISP_VRF3_READ_LAT       1
-#define ISP_VRF4_READ_LAT       1
-#define ISP_VRF5_READ_LAT       1
-#define ISP_VRF6_READ_LAT       1
-#define ISP_VRF7_READ_LAT       1
-#define ISP_VRF8_READ_LAT       1
-#define ISP_SRF1_READ_LAT       1
-#define ISP_SRF2_READ_LAT       1
-#define ISP_SRF3_READ_LAT       1
-#define ISP_SRF4_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-#define ISP_SRF5_READ_LAT       1
-/* immediate sizes */
-#define ISP_IS1_IMM_BITS        14
-#define ISP_IS2_IMM_BITS        13
-#define ISP_IS3_IMM_BITS        14
-#define ISP_IS4_IMM_BITS        14
-#define ISP_IS5_IMM_BITS        9
-#define ISP_IS6_IMM_BITS        16
-#define ISP_IS7_IMM_BITS        9
-#define ISP_IS8_IMM_BITS        16
-#define ISP_IS9_IMM_BITS        11
-/* fifo depths */
-#define ISP_IF_FIFO_DEPTH         0
-#define ISP_IF_B_FIFO_DEPTH       0
-#define ISP_DMA_FIFO_DEPTH        0
-#define ISP_OF_FIFO_DEPTH         0
-#define ISP_GDC_FIFO_DEPTH        0
-#define ISP_SCL_FIFO_DEPTH        0
-#define ISP_GPFIFO_FIFO_DEPTH     0
-#define ISP_SP_FIFO_DEPTH         0
diff --git a/drivers/staging/media/atomisp/pci/isp2401_system_global.h b/drivers/staging/media/atomisp/pci/isp2401_system_global.h
index 213b6ee52208..8bb2a956f983 100644
--- a/drivers/staging/media/atomisp/pci/isp2401_system_global.h
+++ b/drivers/staging/media/atomisp/pci/isp2401_system_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -52,27 +53,6 @@
 
 #define USE_INPUT_SYSTEM_VERSION_2401
 
-/*
- * Since this file is visible everywhere and the system definition
- * macros are not, detect the separate definitions for {host, SP, ISP}
- *
- * The 2401 system has the nice property that it uses a vanilla 2400 SP
- * so the SP will believe it is a 2400 system rather than 2401...
- */
-/* #if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) || defined(__scalar_processor_2401) */
-#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada)
-#define IS_ISP_2401_MAMOIADA_SYSTEM
-#define HAS_ISP_2401_MAMOIADA
-#define HAS_SP_2400
-/* #elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) || defined(__scalar_processor_2400)*/
-#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada)
-#define IS_ISP_2400_MAMOIADA_SYSTEM
-#define HAS_ISP_2400_MAMOIADA
-#define HAS_SP_2400
-#else
-#error "system_global.h: 2400_SYSTEM must be one of {2400, 2401 }"
-#endif
-
 #define HAS_MMU_VERSION_2
 #define HAS_DMA_VERSION_2
 #define HAS_GDC_VERSION_2
@@ -143,21 +123,11 @@ typedef enum {
 	N_SP_ID
 } sp_ID_t;
 
-#if defined(IS_ISP_2401_MAMOIADA_SYSTEM)
-typedef enum {
-	MMU0_ID = 0,
-	MMU1_ID,
-	N_MMU_ID
-} mmu_ID_t;
-#elif defined(IS_ISP_2400_MAMOIADA_SYSTEM)
 typedef enum {
 	MMU0_ID = 0,
 	MMU1_ID,
 	N_MMU_ID
 } mmu_ID_t;
-#else
-#error "system_global.h: SYSTEM must be one of {2400, 2401}"
-#endif
 
 typedef enum {
 	DMA0_ID = 0,
diff --git a/drivers/staging/media/atomisp/pci/isp2401_system_local.h b/drivers/staging/media/atomisp/pci/isp2401_system_local.h
index 4bd95b818494..b09f8faadb13 100644
--- a/drivers/staging/media/atomisp/pci/isp2401_system_local.h
+++ b/drivers/staging/media/atomisp/pci/isp2401_system_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -19,8 +20,6 @@
 #ifndef HRT_USE_VIR_ADDRS
 #define HRT_USE_VIR_ADDRS
 #endif
-/* This interface is deprecated */
-/*#include "hive_isp_css_custom_host_hrt.h"*/
 #endif
 
 #include "system_global.h"
@@ -75,7 +74,6 @@ static const hrt_address SP_DMEM_BASE[N_SP_ID] = {
 };
 
 /* MMU */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM)
 /*
  * MMU0_ID: The data MMU
  * MMU1_ID: The icache MMU
@@ -84,9 +82,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = {
 	0x0000000000070000ULL,
 	0x00000000000A0000ULL
 };
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
 
 /* DMA */
 static const hrt_address DMA_BASE[N_DMA_ID] = {
@@ -104,6 +99,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = {
 	0x000000000008C000ULL,
 	0x0000000000090200ULL
 };
+
 /*
 	0x0000000000000500ULL};
  */
@@ -160,6 +156,7 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
 static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
 	0x0000000000080000ULL
 };
+
 /*	0x0000000000081000ULL, */ /* capture A */
 /*	0x0000000000082000ULL, */ /* capture B */
 /*	0x0000000000083000ULL, */ /* capture C */
@@ -258,7 +255,6 @@ static const hrt_address SP_DMEM_BASE[N_SP_ID] = {
 };
 
 /* MMU */
-#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM)
 /*
  * MMU0_ID: The data MMU
  * MMU1_ID: The icache MMU
@@ -267,9 +263,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = {
 	0x00070000UL,
 	0x000A0000UL
 };
-#else
-#error "system_local.h: SYSTEM must be one of {2400, 2401 }"
-#endif
 
 /* DMA */
 static const hrt_address DMA_BASE[N_DMA_ID] = {
@@ -287,6 +280,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = {
 	0x0008C000UL,
 	0x00090200UL
 };
+
 /*
 	0x00000500UL};
  */
@@ -336,12 +330,14 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = {
 	0x00030200UL,
 	0x00030400UL
 };
+
 /*	0x00030600UL, */ /* memcpy() */
 
 /* INPUT_SYSTEM */
 static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = {
 	0x00080000UL
 };
+
 /*	0x00081000UL, */ /* capture A */
 /*	0x00082000UL, */ /* capture B */
 /*	0x00083000UL, */ /* capture C */
diff --git a/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h b/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h
index 5bdc16c71e82..8e4bddc3790d 100644
--- a/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h
+++ b/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/isp_capture_defs.h b/drivers/staging/media/atomisp/pci/isp_capture_defs.h
index 5ab796e5a53f..14cbb6390d3e 100644
--- a/drivers/staging/media/atomisp/pci/isp_capture_defs.h
+++ b/drivers/staging/media/atomisp/pci/isp_capture_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/isp2400_mamoiada_params.h b/drivers/staging/media/atomisp/pci/mamoiada_params.h
index edc4d4ff1846..e18e5f3576df 100644
--- a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/isp2400_mamoiada_params.h
+++ b/drivers/staging/media/atomisp/pci/mamoiada_params.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,11 +16,6 @@
 /* Version */
 #define RTL_VERSION
 
-/* Cell name  */
-#define ISP_CELL_TYPE                          isp2400_mamoiada
-#define ISP_VMEM                               simd_vmem
-#define _HRT_ISP_VMEM                          isp2400_mamoiada_simd_vmem
-
 /* instruction pipeline depth */
 #define ISP_BRANCHDELAY                        5
 
@@ -153,12 +149,6 @@
 
 #define ISP_VEC_ALIGN                          ISP_VMEM_ALIGN
 
-/* HRT specific vector support */
-#define isp2400_mamoiada_vector_alignment         ISP_VEC_ALIGN
-#define isp2400_mamoiada_vector_elem_bits         ISP_VMEM_ELEMBITS
-#define isp2400_mamoiada_vector_elem_precision    ISP_VMEM_ELEM_PRECISION
-#define isp2400_mamoiada_vector_num_elems         ISP_VEC_NELEMS
-
 /* register file sizes */
 #define ISP_RF0_SIZE        64
 #define ISP_RF1_SIZE        16
@@ -172,14 +162,7 @@
 #define ISP_RF9_SIZE        16
 #define ISP_RF10_SIZE       16
 #define ISP_RF11_SIZE       16
-#define ISP_VRF1_SIZE       24
-#define ISP_VRF2_SIZE       24
-#define ISP_VRF3_SIZE       24
-#define ISP_VRF4_SIZE       24
-#define ISP_VRF5_SIZE       24
-#define ISP_VRF6_SIZE       24
-#define ISP_VRF7_SIZE       24
-#define ISP_VRF8_SIZE       24
+
 #define ISP_SRF1_SIZE       4
 #define ISP_SRF2_SIZE       64
 #define ISP_SRF3_SIZE       64
diff --git a/drivers/staging/media/atomisp/pci/memory_realloc.c b/drivers/staging/media/atomisp/pci/memory_realloc.c
deleted file mode 100644
index e640d5daf502..000000000000
--- a/drivers/staging/media/atomisp/pci/memory_realloc.c
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
-#include "memory_realloc.h"
-#include "ia_css_debug.h"
-#include "ia_css_refcount.h"
-#include "memory_access.h"
-
-static bool realloc_isp_css_mm_buf(
-    hrt_vaddress *curr_buf,
-    size_t *curr_size,
-    size_t needed_size,
-    bool force,
-    enum ia_css_err *err,
-    uint16_t mmgr_attribute);
-
-bool reallocate_buffer(
-    hrt_vaddress *curr_buf,
-    size_t *curr_size,
-    size_t needed_size,
-    bool force,
-    enum ia_css_err *err)
-{
-	bool ret;
-	u16	mmgr_attribute = MMGR_ATTRIBUTE_DEFAULT;
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	ret = realloc_isp_css_mm_buf(curr_buf,
-				     curr_size, needed_size, force, err, mmgr_attribute);
-
-	IA_CSS_LEAVE_PRIVATE("ret=%d", ret);
-	return ret;
-}
-
-static bool realloc_isp_css_mm_buf(
-    hrt_vaddress *curr_buf,
-    size_t *curr_size,
-    size_t needed_size,
-    bool force,
-    enum ia_css_err *err,
-    uint16_t mmgr_attribute)
-{
-	s32 id;
-
-	*err = IA_CSS_SUCCESS;
-	/* Possible optimization: add a function sh_css_isp_css_mm_realloc()
-	 * and implement on top of hmm. */
-
-	IA_CSS_ENTER_PRIVATE("void");
-
-	if (ia_css_refcount_is_single(*curr_buf) && !force &&
-	    *curr_size >= needed_size) {
-		IA_CSS_LEAVE_PRIVATE("false");
-		return false;
-	}
-
-	id = IA_CSS_REFCOUNT_PARAM_BUFFER;
-	ia_css_refcount_decrement(id, *curr_buf);
-	*curr_buf = ia_css_refcount_increment(id, mmgr_alloc_attr(needed_size,
-					      mmgr_attribute));
-
-	if (!*curr_buf) {
-		*err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-		*curr_size = 0;
-	} else {
-		*curr_size = needed_size;
-	}
-	IA_CSS_LEAVE_PRIVATE("true");
-	return true;
-}
diff --git a/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c b/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c
index 8930fd629dc3..72287de75a63 100644
--- a/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c
+++ b/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Medifield PNW Camera Imaging ISP subsystem.
  *
diff --git a/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c b/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c
index 031d7fa00510..0fbb361f5661 100644
--- a/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c
+++ b/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Merrifield PNW Camera Imaging ISP subsystem.
  *
@@ -19,7 +20,6 @@
 #include "type_support.h"
 #include "mmu/isp_mmu.h"
 #include "mmu/sh_mmu_mrfld.h"
-#include "memory_access/memory_access.h"
 #include "atomisp_compat.h"
 
 #define MERR_VALID_PTE_MASK	0x80000000
@@ -63,7 +63,7 @@ static unsigned int sh_get_pd_base(struct isp_mmu *mmu,
  */
 static void sh_tlb_flush(struct isp_mmu *mmu)
 {
-	atomisp_css_mmu_invalidate_cache();
+	ia_css_mmu_invalidate_cache();
 }
 
 struct isp_mmu_client sh_mmu_mrfld = {
diff --git a/drivers/staging/media/atomisp/pci/mmu_defs.h b/drivers/staging/media/atomisp/pci/mmu_defs.h
index c038f39ffd25..03cfb5833027 100644
--- a/drivers/staging/media/atomisp/pci/mmu_defs.h
+++ b/drivers/staging/media/atomisp/pci/mmu_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h b/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h
index 26a3fc4d48e8..b44099dbdacd 100644
--- a/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h
+++ b/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
@@ -143,8 +144,7 @@ struct ia_css_binary {
 	struct ia_css_isp_param_css_segments  css_params;
 };
 
-#define IA_CSS_BINARY_DEFAULT_SETTINGS \
-(struct ia_css_binary) { \
+#define IA_CSS_BINARY_DEFAULT_SETTINGS { \
 	.input_format		= ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, \
 	.in_frame_info		= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
 	.internal_frame_info	= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
@@ -152,13 +152,13 @@ struct ia_css_binary {
 	.vf_frame_info		= IA_CSS_BINARY_DEFAULT_FRAME_INFO, \
 }
 
-enum ia_css_err
+int
 ia_css_binary_init_infos(void);
 
-enum ia_css_err
+int
 ia_css_binary_uninit(void);
 
-enum ia_css_err
+int
 ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 			bool online,
 			bool two_ppc,
@@ -172,7 +172,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 			int stream_config_left_padding,
 			bool accelerator);
 
-enum ia_css_err
+int
 ia_css_binary_find(struct ia_css_binary_descr *descr,
 		   struct ia_css_binary *binary);
 
@@ -188,10 +188,10 @@ ia_css_binary_find(struct ia_css_binary_descr *descr,
  *		the shading table directly required from ISP.
  * @param[out] pipe_config: The pipe configuration.
  *		The shading information related to ISP (but, not necessary as API) is stored in the pipe_config.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err
+int
 ia_css_binary_get_shading_info(const struct ia_css_binary *binary,
 			       enum ia_css_shading_correction_type type,
 			       unsigned int required_bds_factor,
@@ -199,7 +199,7 @@ ia_css_binary_get_shading_info(const struct ia_css_binary *binary,
 			       struct ia_css_shading_info *shading_info,
 			       struct ia_css_pipe_config *pipe_config);
 
-enum ia_css_err
+int
 ia_css_binary_3a_grid_info(const struct ia_css_binary *binary,
 			   struct ia_css_grid_info *info,
 			   struct ia_css_pipe *pipe);
diff --git a/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c b/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c
index 2a23b7c6aeeb..9813014c3fd3 100644
--- a/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c
+++ b/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -14,6 +15,9 @@
 
 #include <math_support.h>
 #include <gdc_device.h>	/* HR_GDC_N */
+
+#include "hmm.h"
+
 #include "isp.h"	/* ISP_VEC_NELEMS */
 
 #include "ia_css_binary.h"
@@ -26,6 +30,8 @@
 #include "sh_css_defs.h"
 #include "sh_css_legacy.h"
 
+#include "atomisp_internal.h"
+
 #include "vf/vf_1.0/ia_css_vf.host.h"
 #include "sc/sc_1.0/ia_css_sc.host.h"
 #include "sdis/sdis_1.0/ia_css_sdis.host.h"
@@ -33,8 +39,6 @@
 
 #include "camera/pipe/interface/ia_css_pipe_binarydesc.h"
 
-#include "memory_access.h"
-
 #include "assert_support.h"
 
 #define IMPLIES(a, b)           (!(a) || (b))   /* A => B */
@@ -132,7 +136,7 @@ struct sh_css_binary_sc_requirements {
 };
 
 /* Get the requirements for the shading correction. */
-static enum ia_css_err
+static int
 #ifndef ISP2401
 ia_css_binary_compute_shading_table_bayer_origin(
     const struct ia_css_binary *binary,				/* [in] */
@@ -147,7 +151,7 @@ sh_css_binary_get_sc_requirements(
     struct sh_css_binary_sc_requirements *scr)		/* [out] */
 #endif
 {
-	enum ia_css_err err;
+	int err;
 
 #ifndef ISP2401
 	/* Numerator and denominator of the fixed bayer downscaling factor.
@@ -196,7 +200,7 @@ sh_css_binary_get_sc_requirements(
 	/* Get the numerator and denominator of bayer downscaling factor. */
 	err = sh_css_bds_factor_get_numerator_denominator
 	(required_bds_factor, &bds_num, &bds_den);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 #else
 	/* Flags corresponding to NEED_BDS_FACTOR_2_00/NEED_BDS_FACTOR_1_50/NEED_BDS_FACTOR_1_25 macros
 	 * defined in isp kernels. */
@@ -226,7 +230,7 @@ sh_css_binary_get_sc_requirements(
 
 	/* Get the numerator and denominator of the required bayer downscaling factor. */
 	err = sh_css_bds_factor_get_numerator_denominator(required_bds_factor, &bds_num, &bds_den);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 #endif
@@ -479,7 +483,7 @@ return err;
 }
 
 /* Get the shading information of Shading Correction Type 1. */
-static enum ia_css_err
+static int
 ia_css_binary_get_shading_info_type_1(const struct ia_css_binary
 				      *binary,	/* [in] */
 				      unsigned int required_bds_factor,			/* [in] */
@@ -491,7 +495,7 @@ ia_css_binary_get_shading_info_type_1(const struct ia_css_binary
 				      struct ia_css_pipe_config *pipe_config)			/* [out] */
 #endif
 {
-	enum ia_css_err err;
+	int err;
 #ifndef ISP2401
 	struct sh_css_shading_table_bayer_origin_compute_results res;
 #else
@@ -545,12 +549,12 @@ ia_css_binary_get_shading_info_type_1(const struct ia_css_binary
 	    required_bds_factor,
 	    stream_config,
 	    &res);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 #else
 	*shading_info = DEFAULT_SHADING_INFO_TYPE_1;
 
 	err = sh_css_binary_get_sc_requirements(binary, required_bds_factor, stream_config, &scr);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 #endif
@@ -639,8 +643,8 @@ IA_CSS_LOG("adjust_width_bqs=%d, adjust_height_bqs=%d", adjust_width_bqs, adjust
 
 if (adjust_width_bqs > tbl_width_bqs || adjust_height_bqs > tbl_height_bqs)
 {
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-	return IA_CSS_ERR_INTERNAL_ERROR;
+	IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+	return -EINVAL;
 }
 
 /* Origin of the internal frame on the shading table. */
@@ -698,7 +702,7 @@ IA_CSS_LEAVE_ERR_PRIVATE(err);
 return err;
 }
 
-enum ia_css_err
+int
 ia_css_binary_get_shading_info(const struct ia_css_binary *binary,			/* [in] */
 			       enum ia_css_shading_correction_type type,		/* [in] */
 			       unsigned int required_bds_factor,			/* [in] */
@@ -706,7 +710,7 @@ ia_css_binary_get_shading_info(const struct ia_css_binary *binary,			/* [in] */
 			       struct ia_css_shading_info *shading_info,		/* [out] */
 			       struct ia_css_pipe_config *pipe_config)			/* [out] */
 {
-	enum ia_css_err err;
+	int err;
 
 	assert(binary);
 	assert(shading_info);
@@ -726,7 +730,7 @@ ia_css_binary_get_shading_info(const struct ia_css_binary *binary,			/* [in] */
 	/* Other function calls can be added here when other shading correction types will be added in the future. */
 
 	else
-		err = IA_CSS_ERR_NOT_SUPPORTED;
+		err = -ENOTSUPP;
 
 	IA_CSS_LEAVE_ERR_PRIVATE(err);
 	return err;
@@ -784,12 +788,12 @@ ia_css_binary_dvs_stat_grid_info(
 	return;
 }
 
-enum ia_css_err
+int
 ia_css_binary_3a_grid_info(const struct ia_css_binary *binary,
 			   struct ia_css_grid_info *info,
 			   struct ia_css_pipe *pipe) {
 	struct ia_css_3a_grid_info *s3a_info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("binary=%p, info=%p, pipe=%p",
 			     binary, info, pipe);
@@ -857,18 +861,6 @@ binary_supports_output_format(const struct ia_css_binary_xinfo *info,
 	return false;
 }
 
-#ifdef ISP2401
-static bool
-binary_supports_input_format(const struct ia_css_binary_xinfo *info,
-			     enum atomisp_input_format format)
-{
-	assert(info);
-	(void)format;
-
-	return true;
-}
-#endif
-
 static bool
 binary_supports_vf_format(const struct ia_css_binary_xinfo *info,
 			  enum ia_css_frame_format format)
@@ -892,52 +884,52 @@ supports_bds_factor(u32 supported_factors,
 	return ((supported_factors & PACK_BDS_FACTOR(bds_factor)) != 0);
 }
 
-static enum ia_css_err
+static int
 binary_init_info(struct ia_css_binary_xinfo *info, unsigned int i,
 		 bool *binary_found) {
 	const unsigned char *blob = sh_css_blob_info[i].blob;
 	unsigned int size = sh_css_blob_info[i].header.blob.size;
 
 	if ((!info) || (!binary_found))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	*info = sh_css_blob_info[i].header.info.isp;
 	*binary_found = blob;
 	info->blob_index = i;
 	/* we don't have this binary, skip it */
 	if (!size)
-		return IA_CSS_SUCCESS;
+		return 0;
 
 	info->xmem_addr = sh_css_load_blob(blob, size);
 	if (!info->xmem_addr)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	return IA_CSS_SUCCESS;
+		return -ENOMEM;
+	return 0;
 }
 
 /* When binaries are put at the beginning, they will only
  * be selected if no other primary matches.
  */
-enum ia_css_err
+int
 ia_css_binary_init_infos(void) {
 	unsigned int i;
 	unsigned int num_of_isp_binaries = sh_css_num_binaries - NUM_OF_SPS - NUM_OF_BLS;
 
 	if (num_of_isp_binaries == 0)
-		return IA_CSS_SUCCESS;
+		return 0;
 
-	all_binaries = sh_css_malloc(num_of_isp_binaries *
-				     sizeof(*all_binaries));
+	all_binaries = kvmalloc(num_of_isp_binaries * sizeof(*all_binaries),
+				GFP_KERNEL);
 	if (!all_binaries)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	for (i = 0; i < num_of_isp_binaries; i++)
 	{
-		enum ia_css_err ret;
+		int ret;
 		struct ia_css_binary_xinfo *binary = &all_binaries[i];
 		bool binary_found;
 
 		ret = binary_init_info(binary, i, &binary_found);
-		if (ret != IA_CSS_SUCCESS)
+		if (ret)
 			return ret;
 		if (!binary_found)
 			continue;
@@ -947,10 +939,10 @@ ia_css_binary_init_infos(void) {
 		binary->blob = &sh_css_blob_info[i];
 		binary->mem_offsets = sh_css_blob_info[i].mem_offsets;
 	}
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
+int
 ia_css_binary_uninit(void) {
 	unsigned int i;
 	struct ia_css_binary_xinfo *b;
@@ -964,8 +956,8 @@ ia_css_binary_uninit(void) {
 		}
 		binary_infos[i] = NULL;
 	}
-	sh_css_free(all_binaries);
-	return IA_CSS_SUCCESS;
+	kvfree(all_binaries);
+	return 0;
 }
 
 /* @brief Compute decimation factor for 3A statistics and shading correction.
@@ -1083,7 +1075,7 @@ binary_in_frame_padded_width(int in_frame_width,
 	return rval;
 }
 
-enum ia_css_err
+int
 ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 			bool online,
 			bool two_ppc,
@@ -1114,7 +1106,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 
 	bool need_scaling = false;
 	struct ia_css_resolution binary_dvs_env, internal_res;
-	enum ia_css_err err;
+	int err;
 	unsigned int i;
 	const struct ia_css_frame_info *bin_out_info = NULL;
 
@@ -1128,7 +1120,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 		err = ia_css_isp_param_allocate_isp_parameters(
 		    &binary->mem_params, &binary->css_params,
 		    &info->mem_initializers);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			return err;
 		}
 	}
@@ -1231,7 +1223,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 	{
 		err = ia_css_vf_configure(binary, bin_out_info,
 					  (struct ia_css_frame_info *)vf_info, &vf_log_ds);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			if (!accelerator) {
 				ia_css_isp_param_destroy_isp_parameters(
 				    &binary->mem_params,
@@ -1252,7 +1244,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 
 		binary->vf_frame_info.format = vf_info->format;
 		if (!bin_out_info)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 		vf_out_vecs = __ISP_VF_OUTPUT_WIDTH_VECS(bin_out_info->padded_width,
 			      vf_log_ds);
 		vf_out_width = _ISP_VF_OUTPUT_WIDTH(vf_out_vecs);
@@ -1350,7 +1342,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 
 	if (info->enable.sc)
 	{
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			binary->sctbl_width_per_color = _ISP2400_SCTBL_WIDTH_PER_COLOR(sc_3a_dis_padded_width, s3a_log_deci);
 			binary->sctbl_aligned_width_per_color = ISP2400_SH_CSS_MAX_SCTBL_ALIGNED_WIDTH_PER_COLOR;
 			binary->sctbl_height = _ISP2400_SCTBL_HEIGHT(sc_3a_dis_height, s3a_log_deci);
@@ -1366,7 +1358,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 		binary->sctbl_width_per_color         = 0;
 		binary->sctbl_aligned_width_per_color = 0;
 		binary->sctbl_height                  = 0;
-		if (atomisp_hw_is_isp2401) {
+		if (IS_ISP2401) {
 			binary->sctbl_legacy_width_per_color  = 0;
 			binary->sctbl_legacy_height	      = 0;
 		}
@@ -1382,12 +1374,11 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 	else
 		binary->left_padding = 0;
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
-ia_css_binary_find(struct ia_css_binary_descr *descr,
-		   struct ia_css_binary *binary) {
+static int __ia_css_binary_find(struct ia_css_binary_descr *descr,
+				struct ia_css_binary *binary) {
 	int mode;
 	bool online;
 	bool two_ppc;
@@ -1413,7 +1404,7 @@ ia_css_binary_find(struct ia_css_binary_descr *descr,
 #ifdef ISP2401
 	bool enable_luma_only;
 #endif
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
+	int err = -EINVAL;
 	bool continuous;
 	unsigned int isp_pipe_version;
 	struct ia_css_resolution dvs_env, internal_res;
@@ -1441,7 +1432,7 @@ ia_css_binary_find(struct ia_css_binary_descr *descr,
 			req_bin_out_info = req_out_info[i];
 	}
 	if (!req_bin_out_info)
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 #ifndef ISP2401
 	req_vf_info = descr->vf_info;
 #else
@@ -1699,15 +1690,6 @@ ia_css_binary_find(struct ia_css_binary_descr *descr,
 					    binary_supports_output_format(xcandidate, req_bin_out_info->format));
 			continue;
 		}
-#ifdef ISP2401
-		if (!binary_supports_input_format(xcandidate, descr->stream_format)) {
-			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-					    "ia_css_binary_find() [%d] continue: !%d\n",
-					    __LINE__,
-					    binary_supports_input_format(xcandidate, req_in_info->format));
-			continue;
-		}
-#endif
 		if (xcandidate->num_output_pins > 1 &&
 		    /* in case we have a second output pin, */
 		    req_vf_info                   && /* and we need vf output. */
@@ -1799,7 +1781,7 @@ ia_css_binary_find(struct ia_css_binary_descr *descr,
 					      descr->stream_config_left_padding,
 					      false);
 
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			break;
 		binary_init_metrics(&binary->metrics, &binary->info->sp);
 		break;
@@ -1812,9 +1794,32 @@ ia_css_binary_find(struct ia_css_binary_descr *descr,
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_binary_find() leave: return_err=%d\n", err);
 
+	if (!err && xcandidate)
+		dev_dbg(atomisp_dev,
+			"Using binary %s (id %d), type %d, mode %d, continuous %s\n",
+			xcandidate->blob->name,
+			xcandidate->sp.id,
+			xcandidate->type,
+			xcandidate->sp.pipeline.mode,
+			xcandidate->sp.enable.continuous ? "true" : "false");
+
+
 	return err;
 }
 
+int ia_css_binary_find(struct ia_css_binary_descr *descr,
+		       struct ia_css_binary *binary)
+{
+	int ret = __ia_css_binary_find(descr, binary);
+
+	if (unlikely(ret)) {
+		dev_dbg(atomisp_dev, "Seeking for binary failed at:");
+		dump_stack();
+	}
+
+	return ret;
+}
+
 unsigned
 ia_css_binary_max_vf_width(void)
 {
diff --git a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h
index 78e433fa3466..a461b0ed03f1 100644
--- a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h
+++ b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -77,10 +78,10 @@ void ia_css_bufq_init(void);
  *
  * @param queue_id[in]		Index of the queue in the specified thread
  * @param item[in]		Object to enqueue.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
 */
-enum ia_css_err ia_css_bufq_enqueue_buffer(
+int ia_css_bufq_enqueue_buffer(
     int thread_index,
     int queue_id,
     uint32_t item);
@@ -91,10 +92,10 @@ enum ia_css_err ia_css_bufq_enqueue_buffer(
  * @param queue_id[in]		Specifies  the index of the queue in the list where
  *				the item has to be read.
  * @paramitem [out]		Object to be dequeued into this item.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
 */
-enum  ia_css_err ia_css_bufq_dequeue_buffer(
+int ia_css_bufq_dequeue_buffer(
     int queue_id,
     uint32_t *item);
 
@@ -105,10 +106,10 @@ enum  ia_css_err ia_css_bufq_dequeue_buffer(
  * @param[in]	evt_payload_0	The event payload.
  * @param[in]	evt_payload_1	The event payload.
  * @param[in]	evt_payload_2	The event payload.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
 */
-enum ia_css_err ia_css_bufq_enqueue_psys_event(
+int ia_css_bufq_enqueue_psys_event(
     u8 evt_id,
     u8 evt_payload_0,
     u8 evt_payload_1,
@@ -119,10 +120,10 @@ enum ia_css_err ia_css_bufq_enqueue_psys_event(
  * @brief   Dequeue an item from  SP to host communication event queue.
  *
  * @param item	Object to be dequeued into this item.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
 */
-enum  ia_css_err ia_css_bufq_dequeue_psys_event(
+int ia_css_bufq_dequeue_psys_event(
     u8 item[BUFQ_EVENT_SIZE]
 
 );
@@ -131,10 +132,10 @@ enum  ia_css_err ia_css_bufq_dequeue_psys_event(
  * @brief  Enqueue an event item into host to SP EOF event queue.
  *
  * @param[in]	evt_id		      The event ID.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_bufq_enqueue_isys_event(
+int ia_css_bufq_enqueue_isys_event(
     uint8_t evt_id);
 
 /**
@@ -142,29 +143,29 @@ enum ia_css_err ia_css_bufq_enqueue_isys_event(
 
  *
  * @param item	Object to be dequeued into this item.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
  */
-enum  ia_css_err ia_css_bufq_dequeue_isys_event(
+int ia_css_bufq_dequeue_isys_event(
     u8 item[BUFQ_EVENT_SIZE]);
 
 /**
 * @brief   Enqueue a tagger command item into tagger command queue..
  *
  * @param item	Object to be enqueue.
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
 */
-enum ia_css_err ia_css_bufq_enqueue_tag_cmd(
+int ia_css_bufq_enqueue_tag_cmd(
     uint32_t item);
 
 /**
 * @brief  Uninitializes bufq module.
  *
- * @return	IA_CSS_SUCCESS or error code upon error.
+ * @return	0 or error code upon error.
  *
 */
-enum ia_css_err ia_css_bufq_deinit(void);
+int ia_css_bufq_deinit(void);
 
 /**
 * @brief  Dump queue states
diff --git a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h
index 508209711edc..cddf5882b76a 100644
--- a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h
+++ b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c b/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c
index 7e01df257150..38e85735293b 100644
--- a/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c
+++ b/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -22,7 +23,6 @@
 #include "ia_css_debug.h"		/* ia_css_debug_dtrace*/
 #include "sh_css_internal.h"		/* sh_css_queue_type */
 #include "sp_local.h"			/* sp_address_of */
-#include "ia_css_util.h"		/* ia_css_convert_errno()*/
 #include "sh_css_firmware.h"		/* sh_css_sp_fw*/
 
 #define BUFQ_DUMP_FILE_NAME_PREFIX_SIZE 256
@@ -329,19 +329,18 @@ void ia_css_bufq_init(void)
 	IA_CSS_LEAVE_PRIVATE("");
 }
 
-enum ia_css_err ia_css_bufq_enqueue_buffer(
+int ia_css_bufq_enqueue_buffer(
     int thread_index,
     int queue_id,
     uint32_t item)
 {
-	enum ia_css_err return_err = IA_CSS_SUCCESS;
 	ia_css_queue_t *q;
 	int error;
 
 	IA_CSS_ENTER_PRIVATE("queue_id=%d", queue_id);
 	if ((thread_index >= SH_CSS_MAX_SP_THREADS) || (thread_index < 0) ||
 	    (queue_id == SH_CSS_INVALID_QUEUE_ID))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/* Get the queue for communication */
 	q = bufq_get_qhandle(sh_css_host2sp_buffer_queue,
@@ -349,22 +348,20 @@ enum ia_css_err ia_css_bufq_enqueue_buffer(
 			     thread_index);
 	if (q) {
 		error = ia_css_queue_enqueue(q, item);
-		return_err = ia_css_convert_errno(error);
 	} else {
 		IA_CSS_ERROR("queue is not initialized");
-		return_err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		error = -EBUSY;
 	}
 
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
+	IA_CSS_LEAVE_ERR_PRIVATE(error);
+	return error;
 }
 
-enum ia_css_err ia_css_bufq_dequeue_buffer(
+int ia_css_bufq_dequeue_buffer(
     int queue_id,
     uint32_t *item)
 {
-	enum ia_css_err return_err;
-	int error = 0;
+	int error;
 	ia_css_queue_t *q;
 
 	IA_CSS_ENTER_PRIVATE("queue_id=%d", queue_id);
@@ -372,52 +369,49 @@ enum ia_css_err ia_css_bufq_dequeue_buffer(
 	    (queue_id <= SH_CSS_INVALID_QUEUE_ID) ||
 	    (queue_id >= SH_CSS_MAX_NUM_QUEUES)
 	   )
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	q = bufq_get_qhandle(sh_css_sp2host_buffer_queue,
 			     queue_id,
 			     -1);
 	if (q) {
 		error = ia_css_queue_dequeue(q, item);
-		return_err = ia_css_convert_errno(error);
 	} else {
 		IA_CSS_ERROR("queue is not initialized");
-		return_err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		error = -EBUSY;
 	}
 
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
+	IA_CSS_LEAVE_ERR_PRIVATE(error);
+	return error;
 }
 
-enum ia_css_err ia_css_bufq_enqueue_psys_event(
+int ia_css_bufq_enqueue_psys_event(
     u8 evt_id,
     u8 evt_payload_0,
     u8 evt_payload_1,
     uint8_t evt_payload_2)
 {
-	enum ia_css_err return_err;
-	int error = 0;
+
+    int error = 0;
 	ia_css_queue_t *q;
 
 	IA_CSS_ENTER_PRIVATE("evt_id=%d", evt_id);
 	q = bufq_get_qhandle(sh_css_host2sp_psys_event_queue, -1, -1);
 	if (!q) {
 		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 
 	error = ia_css_eventq_send(q,
 				   evt_id, evt_payload_0, evt_payload_1, evt_payload_2);
 
-	return_err = ia_css_convert_errno(error);
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
+	IA_CSS_LEAVE_ERR_PRIVATE(error);
+	return error;
 }
 
-enum  ia_css_err ia_css_bufq_dequeue_psys_event(
+int ia_css_bufq_dequeue_psys_event(
     u8 item[BUFQ_EVENT_SIZE])
 {
-	enum ia_css_err;
 	int error = 0;
 	ia_css_queue_t *q;
 
@@ -425,23 +419,22 @@ enum  ia_css_err ia_css_bufq_dequeue_psys_event(
 	 * by some test apps. Enablign logging here floods the log
 	 * files which may cause timeouts. */
 	if (!item)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	q = bufq_get_qhandle(sh_css_sp2host_psys_event_queue, -1, -1);
 	if (!q) {
 		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 	error = ia_css_eventq_recv(q, item);
 
-	return ia_css_convert_errno(error);
+	return error;
 }
 
-enum  ia_css_err ia_css_bufq_dequeue_isys_event(
+int ia_css_bufq_dequeue_isys_event(
     u8 item[BUFQ_EVENT_SIZE])
 {
 #if !defined(HAS_NO_INPUT_SYSTEM)
-	enum ia_css_err;
 	int error = 0;
 	ia_css_queue_t *q;
 
@@ -449,25 +442,24 @@ enum  ia_css_err ia_css_bufq_dequeue_isys_event(
 	 * by some test apps. Enablign logging here floods the log
 	 * files which may cause timeouts. */
 	if (!item)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	q = bufq_get_qhandle(sh_css_sp2host_isys_event_queue, -1, -1);
 	if (!q) {
 		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 	error = ia_css_eventq_recv(q, item);
-	return ia_css_convert_errno(error);
+	return error;
 #else
 	(void)item;
-	return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+	return -EBUSY;
 #endif
 }
 
-enum ia_css_err ia_css_bufq_enqueue_isys_event(uint8_t evt_id)
+int ia_css_bufq_enqueue_isys_event(uint8_t evt_id)
 {
 #if !defined(HAS_NO_INPUT_SYSTEM)
-	enum ia_css_err return_err;
 	int error = 0;
 	ia_css_queue_t *q;
 
@@ -475,47 +467,45 @@ enum ia_css_err ia_css_bufq_enqueue_isys_event(uint8_t evt_id)
 	q = bufq_get_qhandle(sh_css_host2sp_isys_event_queue, -1, -1);
 	if (!q) {
 		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 
 	error = ia_css_eventq_send(q, evt_id, 0, 0, 0);
-	return_err = ia_css_convert_errno(error);
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
+
+	IA_CSS_LEAVE_ERR_PRIVATE(error);
+	return error;
 #else
 	(void)evt_id;
-	return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+	return -EBUSY;
 #endif
 }
 
-enum ia_css_err ia_css_bufq_enqueue_tag_cmd(
+int ia_css_bufq_enqueue_tag_cmd(
     uint32_t item)
 {
 #if !defined(HAS_NO_INPUT_SYSTEM)
-	enum ia_css_err return_err;
-	int error = 0;
+	int error;
 	ia_css_queue_t *q;
 
 	IA_CSS_ENTER_PRIVATE("item=%d", item);
 	q = bufq_get_qhandle(sh_css_host2sp_tag_cmd_queue, -1, -1);
 	if (!q) {
 		IA_CSS_ERROR("queue is not initialized");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 	error = ia_css_queue_enqueue(q, item);
 
-	return_err = ia_css_convert_errno(error);
-	IA_CSS_LEAVE_ERR_PRIVATE(return_err);
-	return return_err;
+	IA_CSS_LEAVE_ERR_PRIVATE(error);
+	return error;
 #else
 	(void)item;
-	return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+	return -EBUSY;
 #endif
 }
 
-enum ia_css_err ia_css_bufq_deinit(void)
+int ia_css_bufq_deinit(void)
 {
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void bufq_dump_queue_info(const char *prefix, ia_css_queue_t *qhandle)
diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h
index 61d612ec3a05..e04d2485ea75 100644
--- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h
+++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -44,8 +45,9 @@
 #define IA_CSS_DEBUG_PARAM   8
 /*! Level for tracing info messages */
 #define IA_CSS_DEBUG_INFO    9
+
 /* Global variable which controls the verbosity levels of the debug tracing */
-extern unsigned int ia_css_debug_trace_level;
+extern int dbg_level;
 
 /*! @brief Enum defining the different isp parameters to dump.
  *  Values can be combined to dump a combination of sets.
@@ -90,7 +92,7 @@ enum ia_css_debug_enable_param_dump {
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \
 		"%s(): leave: " fmt "\n", __func__, ##__VA_ARGS__)
 
-/* Shorthand for returning an enum ia_css_err return value */
+/* Shorthand for returning an int return value */
 #define IA_CSS_LEAVE_ERR(__err) \
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \
 		"%s() %d: leave: return_err=%d\n", __func__, __LINE__, __err)
@@ -111,7 +113,7 @@ enum ia_css_debug_enable_param_dump {
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \
 		"%s(): leave: " fmt "\n", __func__, ##__VA_ARGS__)
 
-/* Shorthand for returning an enum ia_css_err return value */
+/* Shorthand for returning an int return value */
 #define IA_CSS_LEAVE_ERR_PRIVATE(__err) \
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \
 		"%s() %d: leave: return_err=%d\n", __func__, __LINE__, __err)
@@ -130,7 +132,7 @@ enum ia_css_debug_enable_param_dump {
 static inline void
 ia_css_debug_vdtrace(unsigned int level, const char *fmt, va_list args)
 {
-	if (ia_css_debug_trace_level >= level)
+	if (dbg_level >= level)
 		sh_css_vprint(fmt, args);
 }
 
diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h
index 27136381857f..8ec487ad4298 100644
--- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h
+++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h
index e9964bb421d6..538918cfb2fc 100644
--- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h
+++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c b/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c
index 6fadc20104bf..2bca27a04b02 100644
--- a/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c
+++ b/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -13,7 +14,6 @@
  */
 
 #include "debug.h"
-#include "memory_access.h"
 
 #ifndef __INLINE_INPUT_SYSTEM__
 #define __INLINE_INPUT_SYSTEM__
@@ -31,6 +31,8 @@
 #define __INLINE_STREAM2MMIO__
 #endif
 
+#include <linux/string.h> /* for strscpy() */
+
 #include "ia_css_debug.h"
 #include "ia_css_debug_pipe.h"
 #include "ia_css_irq.h"
@@ -47,7 +49,6 @@
 #include "system_local.h"
 #include "assert_support.h"
 #include "print_support.h"
-#include "string_support.h"
 
 #include "fifo_monitor.h"
 
@@ -103,9 +104,6 @@
 #include "gc/gc_2/ia_css_gc2.host.h"
 #include "ynr/ynr_2/ia_css_ynr2.host.h"
 
-/* Global variable to store the dtrace verbosity level */
-unsigned int ia_css_debug_trace_level = IA_CSS_DEBUG_WARNING;
-
 #define DPG_START "ia_css_debug_pipe_graph_dump_start "
 #define DPG_END   " ia_css_debug_pipe_graph_dump_end\n"
 
@@ -244,13 +242,13 @@ void ia_css_debug_dump_sp_stack_info(void)
 
 void ia_css_debug_set_dtrace_level(const unsigned int trace_level)
 {
-	ia_css_debug_trace_level = trace_level;
+	dbg_level = trace_level;
 	return;
 }
 
 unsigned int ia_css_debug_get_dtrace_level(void)
 {
-	return ia_css_debug_trace_level;
+	return dbg_level;
 }
 
 static const char *debug_stream_format2str(const enum atomisp_input_format
@@ -463,7 +461,6 @@ void ia_css_debug_dump_isp_state(void)
 #endif
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[2] dma_FIFO stalled",
 				    stall.fifo2);
-#if defined(HAS_ISP_2400_MAMOIADA) || defined(HAS_ISP_2401_MAMOIADA) || defined(IS_ISP_2500_SYSTEM)
 
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[3] gdc0_FIFO stalled",
 				    stall.fifo3);
@@ -475,9 +472,6 @@ void ia_css_debug_dump_isp_state(void)
 #endif
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[6] sp_FIFO stalled",
 				    stall.fifo6);
-#else
-#error "ia_css_debug: ISP cell must be one of {2400_MAMOIADA,, 2401_MAMOIADA, 2500_SKYCAM}"
-#endif
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
 				    "status & control stalled",
 				    stall.stat_ctrl);
@@ -489,14 +483,12 @@ void ia_css_debug_dump_isp_state(void)
 				    stall.vamem1);
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vamem2 stalled",
 				    stall.vamem2);
-#if defined(HAS_ISP_2400_MAMOIADA) || defined(HAS_ISP_2401_MAMOIADA)
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vamem3 stalled",
 				    stall.vamem3);
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "hmem stalled",
 				    stall.hmem);
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "pmem stalled",
 				    stall.pmem);
-#endif
 	}
 	return;
 }
@@ -509,7 +501,6 @@ void ia_css_debug_dump_sp_state(void)
 	sp_get_state(SP0_ID, &state, &stall);
 	debug_print_sp_state(&state, "SP");
 	if (state.is_stalling) {
-#if defined(HAS_SP_2400) || defined(IS_ISP_2500_SYSTEM)
 #if !defined(HAS_NO_INPUT_SYSTEM)
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "isys_FIFO stalled",
 				    stall.fifo0);
@@ -540,9 +531,6 @@ void ia_css_debug_dump_sp_state(void)
 #endif
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "irq FIFO stalled",
 				    stall.fifoa);
-#else
-#error "ia_css_debug: SP cell must be one of {SP2400, SP2500}"
-#endif
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n", "dmem stalled",
 				    stall.dmem);
 		ia_css_debug_dtrace(2, "\t%-32s: %d\n",
@@ -2231,7 +2219,7 @@ void ia_css_debug_dump_debug_info(const char *context)
 	ia_css_debug_dump_isys_state();
 
 	{
-		irq_controller_state_t state;
+		struct irq_controller_state state;
 
 		irq_controller_get_state(IRQ2_ID, &state);
 
@@ -2492,8 +2480,8 @@ void ia_css_debug_dump_perf_counters(void)
 
 void sh_css_init_ddr_debug_queue(void)
 {
-	hrt_vaddress ddr_debug_queue_addr =
-			mmgr_malloc(sizeof(debug_data_ddr_t));
+	ia_css_ptr ddr_debug_queue_addr =
+			hmm_alloc(sizeof(debug_data_ddr_t), HMM_BO_PRIVATE, 0, NULL, 0);
 	const struct ia_css_fw_info *fw;
 	unsigned int HIVE_ADDR_debug_buffer_ddr_address;
 
@@ -2665,7 +2653,7 @@ void sh_css_dump_pipe_stripe_info(void)
 
 static void
 ia_css_debug_pipe_graph_dump_frame(
-    struct ia_css_frame *frame,
+    const struct ia_css_frame *frame,
     enum ia_css_pipe_id id,
     char const *blob_name,
     char const *frame_name,
@@ -2782,8 +2770,9 @@ ia_css_debug_pipe_graph_dump_stage(
 				 stage->binary->info->blob->name, stage->stage_num);
 	} else if (stage->firmware) {
 		bin_type = "firmware";
-		strncpy_s(blob_name, sizeof(blob_name),
-			  IA_CSS_EXT_ISP_PROG_NAME(stage->firmware), sizeof(blob_name));
+
+		strscpy(blob_name, IA_CSS_EXT_ISP_PROG_NAME(stage->firmware),
+			sizeof(blob_name));
 	}
 
 	/* Guard in case of binaries that don't have any binary_info */
@@ -2849,10 +2838,8 @@ ia_css_debug_pipe_graph_dump_stage(
 				while (ei[p] != ',')
 					p--;
 				/* Last comma found, copy till that comma */
-				strncpy_s(enable_info1,
-					  sizeof(enable_info1),
-					  ei, p);
-				enable_info1[p] = '\0';
+				strscpy(enable_info1, ei,
+                                        p > sizeof(enable_info1) ? sizeof(enable_info1) : p);
 
 				ei += p + 1;
 				l = strlen(ei);
@@ -2862,10 +2849,10 @@ ia_css_debug_pipe_graph_dump_stage(
 					/* we cannot use ei as argument because
 					 * it is not guaranteed dword aligned
 					 */
-					strncpy_s(enable_info2,
-						  sizeof(enable_info2),
-						  ei, l);
-					enable_info2[l] = '\0';
+
+					strscpy(enable_info2, ei,
+						l > sizeof(enable_info2) ? sizeof(enable_info2) : l);
+
 					snprintf(enable_info, sizeof(enable_info), "%s\\n%s",
 						 enable_info1, enable_info2);
 
@@ -2874,10 +2861,10 @@ ia_css_debug_pipe_graph_dump_stage(
 					p = ENABLE_LINE_MAX_LENGTH;
 					while (ei[p] != ',')
 						p--;
-					strncpy_s(enable_info2,
-						  sizeof(enable_info2),
-						  ei, p);
-					enable_info2[p] = '\0';
+
+					strscpy(enable_info2, ei,
+						p > sizeof(enable_info2) ? sizeof(enable_info2) : p);
+
 					ei += p + 1;
 					l = strlen(ei);
 
@@ -2886,9 +2873,8 @@ ia_css_debug_pipe_graph_dump_stage(
 						/* we cannot use ei as argument because
 						* it is not guaranteed dword aligned
 						*/
-						strcpy_s(enable_info3,
-							 sizeof(enable_info3), ei);
-						enable_info3[l] = '\0';
+						strscpy(enable_info3, ei,
+							sizeof(enable_info3));
 						snprintf(enable_info, sizeof(enable_info),
 							 "%s\\n%s\\n%s",
 							 enable_info1, enable_info2,
@@ -2898,13 +2884,11 @@ ia_css_debug_pipe_graph_dump_stage(
 						p = ENABLE_LINE_MAX_LENGTH;
 						while (ei[p] != ',')
 							p--;
-						strncpy_s(enable_info3,
-							  sizeof(enable_info3),
-							  ei, p);
-						enable_info3[p] = '\0';
+						strscpy(enable_info3, ei,
+							p > sizeof(enable_info3) ? sizeof(enable_info3) : p);
 						ei += p + 1;
-						strcpy_s(enable_info3,
-							 sizeof(enable_info3), ei);
+						strscpy(enable_info3, ei,
+							sizeof(enable_info3));
 						snprintf(enable_info, sizeof(enable_info),
 							 "%s\\n%s\\n%s",
 							 enable_info1, enable_info2,
@@ -3100,7 +3084,7 @@ ia_css_debug_dump_pipe_config(
 				     "capt_pp_in_res");
 	ia_css_debug_dump_resolution(&config->vf_pp_in_res, "vf_pp_in_res");
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		ia_css_debug_dump_resolution(&config->output_system_in_res,
 					    "output_system_in_res");
 	}
@@ -3328,7 +3312,7 @@ static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id)
 		return;
 	}
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		tmp = ia_css_device_load_uint32(start_addr);
 		point_num = (tmp >> 16) & 0xFFFF;
 
@@ -3362,7 +3346,7 @@ static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id)
 		if ((limit == (-1)) && (trace_read_buf[i] == 0))
 			limit = i;
 	}
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "Status:\n");
 		for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++)
 			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
@@ -3395,7 +3379,7 @@ static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id)
 	for (i = 0; i < point_num; i++) {
 		j = (limit + i) % point_num;
 		if (trace_read_buf[j]) {
-			if (!atomisp_hw_is_isp2401) {
+			if (!IS_ISP2401) {
 				TRACE_DUMP_FORMAT dump_format = FIELD_FORMAT_UNPACK(trace_read_buf[j]);
 			} else {
 				tid_val = FIELD_TID_UNPACK(trace_read_buf[j]);
diff --git a/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h b/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h
index 1fcd0fadcac8..ebbd90b14bff 100644
--- a/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h
+++ b/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/event/src/event.c b/drivers/staging/media/atomisp/pci/runtime/event/src/event.c
index c4578470ad8c..e702297b0a76 100644
--- a/drivers/staging/media/atomisp/pci/runtime/event/src/event.c
+++ b/drivers/staging/media/atomisp/pci/runtime/event/src/event.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -29,10 +30,7 @@
 
 /*#include "sp.h"*/	/* host2sp_enqueue_frame_data() */
 
-#include "memory_access.h"
-
 #include "assert_support.h"
-#include "platform_support.h"	/* hrt_sleep() */
 
 #include "ia_css_queue.h"	/* host_sp_enqueue_XXX */
 #include "ia_css_event.h"	/* ia_css_event_encode */
diff --git a/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h b/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h
index 8602398ede52..fd001ae3522d 100644
--- a/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h
+++ b/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -23,8 +24,8 @@
  * @param[in]	eventq_handle	eventq_handle.
  * @param[in]	payload		The event payload.
  * @return	0		- Successfully dequeue.
- * @return	EINVAL		- Invalid argument.
- * @return	ENODATA		- Queue is empty.
+ * @return	-EINVAL		- Invalid argument.
+ * @return	-ENODATA		- Queue is empty.
  */
 int ia_css_eventq_recv(
     ia_css_queue_t *eventq_handle,
@@ -41,8 +42,8 @@ int ia_css_eventq_recv(
  * @param[in]	evt_payload_1	The event payload.
  * @param[in]	evt_payload_2	The event payload.
  * @return	0		- Successfully enqueue.
- * @return	EINVAL		- Invalid argument.
- * @return	ENOBUFS		- Queue is full.
+ * @return	-EINVAL		- Invalid argument.
+ * @return	-ENOBUFS		- Queue is full.
  */
 int ia_css_eventq_send(
     ia_css_queue_t *eventq_handle,
diff --git a/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c b/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c
index 0460f102d36f..df75cef46a51 100644
--- a/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c
+++ b/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -19,8 +20,6 @@
 #include "ia_css_event.h"	/* ia_css_event_encode()
 				ia_css_event_decode()
 				*/
-#include "platform_support.h" /* hrt_sleep() */
-
 int ia_css_eventq_recv(
     ia_css_queue_t *eventq_handle,
     uint8_t *payload)
@@ -50,7 +49,7 @@ int ia_css_eventq_send(
 {
 	u8 tmp[4];
 	u32 sw_event;
-	int error = ENOSYS;
+	int error = -ENOSYS;
 
 	/*
 	 * Encode the queue type, the thread ID and
@@ -65,13 +64,13 @@ int ia_css_eventq_send(
 	/* queue the software event (busy-waiting) */
 	for ( ; ; ) {
 		error = ia_css_queue_enqueue(eventq_handle, sw_event);
-		if (error != ENOBUFS) {
+		if (error != -ENOBUFS) {
 			/* We were able to successfully send the event
 			   or had a real failure. return the status*/
 			break;
 		}
 		/* Wait for the queue to be not full and try again*/
-		hrt_sleep();
+		udelay(1);
 	}
 	return error;
 }
diff --git a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h
index 613fa33ab930..31f01e0f58aa 100644
--- a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h
+++ b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -80,7 +81,7 @@ bool ia_css_frame_info_is_same_resolution(
  * @param[in]	info       The frame attributes to be initialized
  * @return	The error code.
  */
-enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info);
+int ia_css_frame_check_info(const struct ia_css_frame_info *info);
 
 /*********************************************************************
 ****	Frame APIs
@@ -92,7 +93,7 @@ enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info);
  * @param[in]	frame           The frame attributes to be initialized
  * @return	The error code.
  */
-enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame);
+int ia_css_frame_init_planes(struct ia_css_frame *frame);
 
 /* @brief Free an array of frames
  *
@@ -114,7 +115,7 @@ void ia_css_frame_free_multiple(unsigned int num_frames,
  * Allocate a frame using the given size in bytes.
  * The frame structure is partially null initialized.
  */
-enum ia_css_err ia_css_frame_allocate_with_buffer_size(
+int ia_css_frame_allocate_with_buffer_size(
     struct ia_css_frame **frame,
     const unsigned int size_bytes,
     const bool contiguous);
@@ -153,9 +154,9 @@ void ia_css_dma_configure_from_info(
  * @param[in]	in_res		Resolution of input image
  * @param[in]	out_res		Resolution of output image
  * @param[out]	crop_res	Crop resolution of input image
- * @return	Returns IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS on error
+ * @return	Returns 0 or -EINVAL on error
  */
-enum ia_css_err
+int
 ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res,
 				  const struct ia_css_resolution *out_res,
 				  struct ia_css_resolution *crop_res);
diff --git a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h
index 8861d07193bd..ce6110efbfc9 100644
--- a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h
+++ b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -18,7 +19,7 @@
 #include "type_support.h"
 #include "platform_support.h"
 #include "runtime/bufq/interface/ia_css_bufq_comm.h"
-#include <system_types.h>	 /* hrt_vaddress */
+#include <system_local.h>	 /* ia_css_ptr */
 
 /*
  * These structs are derived from structs defined in ia_css_types.h
@@ -83,7 +84,7 @@ struct ia_css_frame_sp_info {
 
 struct ia_css_buffer_sp {
 	union {
-		hrt_vaddress xmem_addr;
+		ia_css_ptr xmem_addr;
 		enum sh_css_queue_id queue_id;
 	} buf_src;
 	enum ia_css_buffer_type buf_type;
diff --git a/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c b/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c
index fcd8b06034f2..10c4907187d9 100644
--- a/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c
+++ b/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,13 +13,15 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include "ia_css_frame.h"
 #include <math_support.h>
 #include "assert_support.h"
 #include "ia_css_debug.h"
 #include "isp.h"
 #include "sh_css_internal.h"
-#include "memory_access.h"
+#include "atomisp_internal.h"
 
 #define NV12_TILEY_TILE_WIDTH  128
 #define NV12_TILEY_TILE_HEIGHT  32
@@ -67,9 +70,9 @@ static void frame_init_rgb_planes(struct ia_css_frame *frame,
 
 static void frame_init_qplane6_planes(struct ia_css_frame *frame);
 
-static enum ia_css_err frame_allocate_buffer_data(struct ia_css_frame *frame);
+static int frame_allocate_buffer_data(struct ia_css_frame *frame);
 
-static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame,
+static int frame_allocate_with_data(struct ia_css_frame **frame,
 	unsigned int width,
 	unsigned int height,
 	enum ia_css_frame_format format,
@@ -96,16 +99,16 @@ ia_css_elems_bytes_from_info(
 void ia_css_frame_zero(struct ia_css_frame *frame)
 {
 	assert(frame);
-	mmgr_clear(frame->data, frame->data_bytes);
+	hmm_set(frame->data, 0, frame->data_bytes);
 }
 
-enum ia_css_err ia_css_frame_allocate_from_info(struct ia_css_frame **frame,
+int ia_css_frame_allocate_from_info(struct ia_css_frame **frame,
 	const struct ia_css_frame_info *info)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	if (!frame || !info)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_frame_allocate_from_info() enter:\n");
 	err =
@@ -117,17 +120,17 @@ enum ia_css_err ia_css_frame_allocate_from_info(struct ia_css_frame **frame,
 	return err;
 }
 
-enum ia_css_err ia_css_frame_allocate(struct ia_css_frame **frame,
+int ia_css_frame_allocate(struct ia_css_frame **frame,
 				      unsigned int width,
 				      unsigned int height,
 				      enum ia_css_frame_format format,
 				      unsigned int padded_width,
 				      unsigned int raw_bit_depth)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	if (!frame || width == 0 || height == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_frame_allocate() enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n",
@@ -136,7 +139,7 @@ enum ia_css_err ia_css_frame_allocate(struct ia_css_frame **frame,
 	err = frame_allocate_with_data(frame, width, height, format,
 				       padded_width, raw_bit_depth, false);
 
-	if ((*frame) && err == IA_CSS_SUCCESS)
+	if ((*frame) && err == 0)
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_frame_allocate() leave: frame=%p, data(DDR address)=0x%x\n", *frame,
 				    (*frame)->data);
@@ -148,13 +151,13 @@ enum ia_css_err ia_css_frame_allocate(struct ia_css_frame **frame,
 	return err;
 }
 
-enum ia_css_err ia_css_frame_map(struct ia_css_frame **frame,
+int ia_css_frame_map(struct ia_css_frame **frame,
 				 const struct ia_css_frame_info *info,
 				 const void __user *data,
 				 u16 attribute,
-				 void *context)
+				 unsigned int pgnr)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame *me;
 
 	assert(frame);
@@ -162,20 +165,32 @@ enum ia_css_err ia_css_frame_map(struct ia_css_frame **frame,
 	/* Create the frame structure */
 	err = ia_css_frame_create_from_info(&me, info);
 
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
-	if (err == IA_CSS_SUCCESS) {
-		/* use mmgr_mmap to map */
-		me->data = (ia_css_ptr) mmgr_mmap(data,
-						  me->data_bytes,
-						  attribute, context);
+	if (!err) {
+		if (pgnr < ((PAGE_ALIGN(me->data_bytes)) >> PAGE_SHIFT)) {
+			dev_err(atomisp_dev,
+				"user space memory size is less than the expected size..\n");
+			err = -ENOMEM;
+			goto error;
+		} else if (pgnr > ((PAGE_ALIGN(me->data_bytes)) >> PAGE_SHIFT)) {
+			dev_err(atomisp_dev,
+				"user space memory size is large than the expected size..\n");
+			err = -ENOMEM;
+			goto error;
+		}
+
+		me->data = hmm_alloc(me->data_bytes, HMM_BO_USER, 0, data,
+				     attribute & ATOMISP_MAP_FLAG_CACHED);
+
 		if (me->data == mmgr_NULL)
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
+			err = -EINVAL;
 	}
 
-	if (err != IA_CSS_SUCCESS) {
-		sh_css_free(me);
+error:
+	if (err) {
+		kvfree(me);
 		me = NULL;
 	}
 
@@ -184,10 +199,10 @@ enum ia_css_err ia_css_frame_map(struct ia_css_frame **frame,
 	return err;
 }
 
-enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame,
+int ia_css_frame_create_from_info(struct ia_css_frame **frame,
 	const struct ia_css_frame_info *info)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame *me;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
@@ -195,7 +210,7 @@ enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame,
 	if (!frame || !info) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_frame_create_from_info() leave: invalid arguments\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	me = frame_create(info->res.width,
@@ -208,13 +223,13 @@ enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame,
 	if (!me) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_frame_create_from_info() leave: frame create failed\n");
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 	}
 
 	err = ia_css_frame_init_planes(me);
 
-	if (err != IA_CSS_SUCCESS) {
-		sh_css_free(me);
+	if (err) {
+		kvfree(me);
 		me = NULL;
 	}
 
@@ -226,18 +241,18 @@ enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame,
 	return err;
 }
 
-enum ia_css_err ia_css_frame_set_data(struct ia_css_frame *frame,
+int ia_css_frame_set_data(struct ia_css_frame *frame,
 				      const ia_css_ptr mapped_data,
 				      size_t data_bytes)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_frame_set_data() enter:\n");
 	if (!frame) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_frame_set_data() leave: NULL frame\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	/* If we are setting a valid data.
@@ -247,7 +262,7 @@ enum ia_css_err ia_css_frame_set_data(struct ia_css_frame *frame,
 	if ((mapped_data != mmgr_NULL) && (frame->data_bytes > data_bytes)) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_frame_set_data() leave: invalid arguments\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	frame->data = mapped_data;
@@ -257,18 +272,17 @@ enum ia_css_err ia_css_frame_set_data(struct ia_css_frame *frame,
 	return err;
 }
 
-enum ia_css_err ia_css_frame_allocate_contiguous(struct ia_css_frame **frame,
+int ia_css_frame_allocate_contiguous(struct ia_css_frame **frame,
 	unsigned int width,
 	unsigned int height,
 	enum ia_css_frame_format format,
 	unsigned int padded_width,
 	unsigned int raw_bit_depth)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			    "ia_css_frame_allocate_contiguous() "
-			    "enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n",
+			    "ia_css_frame_allocate_contiguous() enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n",
 			    width, height, format, padded_width, raw_bit_depth);
 
 	err = frame_allocate_with_data(frame, width, height, format,
@@ -281,11 +295,11 @@ enum ia_css_err ia_css_frame_allocate_contiguous(struct ia_css_frame **frame,
 	return err;
 }
 
-enum ia_css_err ia_css_frame_allocate_contiguous_from_info(
+int ia_css_frame_allocate_contiguous_from_info(
     struct ia_css_frame **frame,
     const struct ia_css_frame_info *info)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(frame);
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
@@ -307,7 +321,7 @@ void ia_css_frame_free(struct ia_css_frame *frame)
 
 	if (frame) {
 		hmm_free(frame->data);
-		sh_css_free(frame);
+		kvfree(frame);
 	}
 
 	IA_CSS_LEAVE_PRIVATE("void");
@@ -317,15 +331,15 @@ void ia_css_frame_free(struct ia_css_frame *frame)
 **	Module public functions
 **************************************************************************/
 
-enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info)
+int ia_css_frame_check_info(const struct ia_css_frame_info *info)
 {
 	assert(info);
 	if (info->res.width == 0 || info->res.height == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	return IA_CSS_SUCCESS;
+		return -EINVAL;
+	return 0;
 }
 
-enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame)
+int ia_css_frame_init_planes(struct ia_css_frame *frame)
 {
 	assert(frame);
 
@@ -430,9 +444,9 @@ enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame)
 		frame->planes.binary.size = 0;
 		break;
 	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 void ia_css_frame_info_set_width(struct ia_css_frame_info *info,
@@ -517,27 +531,27 @@ void ia_css_frame_free_multiple(unsigned int num_frames,
 	}
 }
 
-enum ia_css_err ia_css_frame_allocate_with_buffer_size(
+int ia_css_frame_allocate_with_buffer_size(
     struct ia_css_frame **frame,
     const unsigned int buffer_size_bytes,
     const bool contiguous)
 {
 	/* AM: Body coppied from frame_allocate_with_data(). */
-	enum ia_css_err err;
+	int err;
 	struct ia_css_frame *me = frame_create(0, 0,
 					       IA_CSS_FRAME_FORMAT_NUM,/* Not valid format yet */
 					       0, 0, contiguous, false);
 
 	if (!me)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	/* Get the data size */
 	me->data_bytes = buffer_size_bytes;
 
 	err = frame_allocate_buffer_data(me);
 
-	if (err != IA_CSS_SUCCESS) {
-		sh_css_free(me);
+	if (err) {
+		kvfree(me);
 		me = NULL;
 	}
 
@@ -782,22 +796,22 @@ static void frame_init_qplane6_planes(struct ia_css_frame *frame)
 	return;
 }
 
-static enum ia_css_err frame_allocate_buffer_data(struct ia_css_frame *frame)
+static int frame_allocate_buffer_data(struct ia_css_frame *frame)
 {
 #ifdef ISP2401
 	IA_CSS_ENTER_LEAVE_PRIVATE("frame->data_bytes=%d\n", frame->data_bytes);
 #endif
-	frame->data = mmgr_alloc_attr(frame->data_bytes,
-				      frame->contiguous ?
-				      MMGR_ATTRIBUTE_CONTIGUOUS :
-				      MMGR_ATTRIBUTE_DEFAULT);
+	frame->data = hmm_alloc(frame->data_bytes,
+				HMM_BO_PRIVATE, 0, NULL,
+				frame->contiguous ?
+				ATOMISP_MAP_FLAG_CONTIGUOUS : 0);
 
 	if (frame->data == mmgr_NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	return IA_CSS_SUCCESS;
+		return -ENOMEM;
+	return 0;
 }
 
-static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame,
+static int frame_allocate_with_data(struct ia_css_frame **frame,
 	unsigned int width,
 	unsigned int height,
 	enum ia_css_frame_format format,
@@ -805,7 +819,7 @@ static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame,
 	unsigned int raw_bit_depth,
 	bool contiguous)
 {
-	enum ia_css_err err;
+	int err;
 	struct ia_css_frame *me = frame_create(width,
 					       height,
 					       format,
@@ -815,15 +829,15 @@ static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame,
 					       true);
 
 	if (!me)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	err = ia_css_frame_init_planes(me);
 
-	if (err == IA_CSS_SUCCESS)
+	if (!err)
 		err = frame_allocate_buffer_data(me);
 
-	if (err != IA_CSS_SUCCESS) {
-		sh_css_free(me);
+	if (err) {
+		kvfree(me);
 #ifndef ISP2401
 		return err;
 #else
@@ -844,7 +858,7 @@ static struct ia_css_frame *frame_create(unsigned int width,
 	bool contiguous,
 	bool valid)
 {
-	struct ia_css_frame *me = sh_css_malloc(sizeof(*me));
+	struct ia_css_frame *me = kvmalloc(sizeof(*me), GFP_KERNEL);
 
 	if (!me)
 		return NULL;
@@ -919,7 +933,7 @@ void ia_css_resolution_to_sp_resolution(
 }
 
 /* ISP2401 */
-enum ia_css_err
+int
 ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res,
 				  const struct ia_css_resolution *out_res,
 				  struct ia_css_resolution *crop_res) {
@@ -927,7 +941,7 @@ ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res,
 	u32 in_ratio, out_ratio;
 
 	if ((!in_res) || (!out_res) || (!crop_res))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	IA_CSS_ENTER_PRIVATE("in(%ux%u) -> out(%ux%u)", in_res->width,
 			     in_res->height, out_res->width, out_res->height);
@@ -936,11 +950,11 @@ ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res,
 	    || (in_res->height == 0)
 	    || (out_res->width == 0)
 	    || (out_res->height == 0))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if ((out_res->width > in_res->width) ||
 	    (out_res->height > in_res->height))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/* If aspect ratio (width/height) of out_res is higher than the aspect
 	 * ratio of the in_res, then we crop vertically, otherwise we crop
@@ -985,5 +999,5 @@ ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res,
 
 	IA_CSS_LEAVE_PRIVATE("in(%ux%u) -> out(%ux%u)", crop_res->width,
 			     crop_res->height, out_res->width, out_res->height);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h b/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h
index d4b0b2361176..2c440feec3ce 100644
--- a/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h
+++ b/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -27,7 +28,7 @@ unsigned int ia_css_ifmtr_lines_needed_for_bayer_order(
 unsigned int ia_css_ifmtr_columns_needed_for_bayer_order(
     const struct ia_css_stream_config *config);
 
-enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
+int ia_css_ifmtr_configure(struct ia_css_stream_config *config,
 				       struct ia_css_binary *binary);
 
 #endif /* __IA_CSS_IFMTR_H__ */
diff --git a/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c b/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c
index 7a18eae8c638..89cded6b6e2b 100644
--- a/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c
+++ b/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -28,12 +29,12 @@
 /************************************************************
  * Static functions declarations
  ************************************************************/
-static enum ia_css_err ifmtr_start_column(
+static int ifmtr_start_column(
     const struct ia_css_stream_config *config,
     unsigned int bin_in,
     unsigned int *start_column);
 
-static enum ia_css_err ifmtr_input_start_line(
+static int ifmtr_input_start_line(
     const struct ia_css_stream_config *config,
     unsigned int bin_in,
     unsigned int *start_line);
@@ -71,7 +72,7 @@ unsigned int ia_css_ifmtr_columns_needed_for_bayer_order(
 	return 0;
 }
 
-enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
+int ia_css_ifmtr_configure(struct ia_css_stream_config *config,
 				       struct ia_css_binary *binary)
 {
 	unsigned int start_line, start_column = 0,
@@ -97,7 +98,7 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
 				 left_padding = 0;
 	input_formatter_cfg_t if_a_config, if_b_config;
 	enum atomisp_input_format input_format;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	u8 if_config_index;
 
 	/* Determine which input formatter config set is targeted. */
@@ -142,10 +143,10 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
 	 * columns.
 	 */
 	err = ifmtr_input_start_line(config, cropped_height, &start_line);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	err = ifmtr_start_column(config, cropped_width, &start_column);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	if (config->left_padding == -1)
@@ -358,7 +359,7 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
 		break;
 	}
 	if (width_a == 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if (two_ppc)
 		left_padding /= 2;
@@ -459,7 +460,7 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config,
 		}
 	}
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 bool ifmtr_set_if_blocking_mode_reset = true;
@@ -496,7 +497,7 @@ static void ifmtr_set_if_blocking_mode(
 	return;
 }
 
-static enum ia_css_err ifmtr_start_column(
+static int ifmtr_start_column(
     const struct ia_css_stream_config *config,
     unsigned int bin_in,
     unsigned int *start_column)
@@ -505,7 +506,7 @@ static enum ia_css_err ifmtr_start_column(
 		     for_bayer = ia_css_ifmtr_columns_needed_for_bayer_order(config);
 
 	if (bin_in + 2 * for_bayer > in)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/* On the hardware, we want to use the middle of the input, so we
 	 * divide the start column by 2. */
@@ -519,10 +520,10 @@ static enum ia_css_err ifmtr_start_column(
 	 */
 	start += for_bayer;
 	*start_column = start;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err ifmtr_input_start_line(
+static int ifmtr_input_start_line(
     const struct ia_css_stream_config *config,
     unsigned int bin_in,
     unsigned int *start_line)
@@ -531,7 +532,7 @@ static enum ia_css_err ifmtr_input_start_line(
 		     for_bayer = ia_css_ifmtr_lines_needed_for_bayer_order(config);
 
 	if (bin_in + 2 * for_bayer > in)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/* On the hardware, we want to use the middle of the input, so we
 	 * divide the start line by 2. On the simulator, we cannot handle extra
@@ -546,7 +547,7 @@ static enum ia_css_err ifmtr_input_start_line(
 	/* now we add the one line (if needed) to correct for the bayer order */
 	start += for_bayer;
 	*start_line = start;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 #endif
diff --git a/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h b/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h
index d2dd231b6296..7950c5c36693 100644
--- a/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h
+++ b/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c b/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c
index e5a339fb52f2..38712530f566 100644
--- a/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c
+++ b/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -103,7 +104,7 @@ static inline void
 _sh_css_fifo_snd(unsigned int token)
 {
 	while (!can_event_send_token(STR2MIPI_EVENT_ID))
-		hrt_sleep();
+		udelay(1);
 	event_send_token(STR2MIPI_EVENT_ID, token);
 	return;
 }
diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h
index 2769183a8956..0ea5d6fdc88b 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -31,7 +32,7 @@ ia_css_isp_param_set_css_mem_init(
     struct ia_css_isp_param_css_segments *mem_init,
     enum ia_css_param_class pclass,
     enum ia_css_isp_memories mem,
-    hrt_vaddress address, size_t size);
+    ia_css_ptr address, size_t size);
 
 void
 ia_css_isp_param_set_isp_mem_init(
@@ -67,7 +68,7 @@ ia_css_init_memory_interface(
     const struct ia_css_isp_param_css_segments *css_params);
 
 /* Allocate memory parameters */
-enum ia_css_err
+int
 ia_css_isp_param_allocate_isp_parameters(
     struct ia_css_isp_param_host_segments *mem_params,
     struct ia_css_isp_param_css_segments *css_params,
@@ -88,7 +89,7 @@ ia_css_isp_param_load_fw_params(
     bool init);
 
 /* Copy host parameter images to ddr */
-enum ia_css_err
+int
 ia_css_isp_param_copy_isp_mem_if_to_ddr(
     struct ia_css_isp_param_css_segments *ddr,
     const struct ia_css_isp_param_host_segments *host,
diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h
index 5d23b2f57719..8cdeae98bda8 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c b/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c
index 443e412d05ad..e861777385a0 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -12,7 +13,8 @@
  * more details.
  */
 
-#include "memory_access.h"
+#include "hmm.h"
+
 #include "ia_css_pipeline.h"
 #include "ia_css_isp_param.h"
 
@@ -34,7 +36,7 @@ ia_css_isp_param_set_css_mem_init(
     struct ia_css_isp_param_css_segments *mem_init,
     enum ia_css_param_class pclass,
     enum ia_css_isp_memories mem,
-    hrt_vaddress address, size_t size)
+    ia_css_ptr address, size_t size)
 {
 	mem_init->params[pclass][mem].address = address;
 	mem_init->params[pclass][mem].size = (uint32_t)size;
@@ -100,12 +102,12 @@ ia_css_init_memory_interface(
 	}
 }
 
-enum ia_css_err
+int
 ia_css_isp_param_allocate_isp_parameters(
     struct ia_css_isp_param_host_segments *mem_params,
     struct ia_css_isp_param_css_segments *css_params,
     const struct ia_css_isp_param_isp_segments *mem_initializers) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int mem, pclass;
 
 	pclass = IA_CSS_PARAM_CLASS_PARAM;
@@ -121,15 +123,17 @@ ia_css_isp_param_allocate_isp_parameters(
 			css_params->params[pclass][mem].size = size;
 			css_params->params[pclass][mem].address = 0x0;
 			if (size) {
-				mem_params->params[pclass][mem].address = sh_css_calloc(1, size);
+				mem_params->params[pclass][mem].address = kvcalloc(1,
+										   size,
+										   GFP_KERNEL);
 				if (!mem_params->params[pclass][mem].address) {
-					err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+					err = -ENOMEM;
 					goto cleanup;
 				}
 				if (pclass != IA_CSS_PARAM_CLASS_PARAM) {
-					css_params->params[pclass][mem].address = mmgr_malloc(size);
+					css_params->params[pclass][mem].address = hmm_alloc(size, HMM_BO_PRIVATE, 0, NULL, 0);
 					if (!css_params->params[pclass][mem].address) {
-						err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+						err = -ENOMEM;
 						goto cleanup;
 					}
 				}
@@ -152,7 +156,7 @@ ia_css_isp_param_destroy_isp_parameters(
 	for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) {
 		for (pclass = 0; pclass < IA_CSS_NUM_PARAM_CLASSES; pclass++) {
 			if (mem_params->params[pclass][mem].address)
-				sh_css_free(mem_params->params[pclass][mem].address);
+				kvfree(mem_params->params[pclass][mem].address);
 			if (css_params->params[pclass][mem].address)
 				hmm_free(css_params->params[pclass][mem].address);
 			mem_params->params[pclass][mem].address = NULL;
@@ -177,7 +181,7 @@ ia_css_isp_param_load_fw_params(
 	}
 }
 
-enum ia_css_err
+int
 ia_css_isp_param_copy_isp_mem_if_to_ddr(
     struct ia_css_isp_param_css_segments *ddr,
     const struct ia_css_isp_param_host_segments *host,
@@ -187,16 +191,16 @@ ia_css_isp_param_copy_isp_mem_if_to_ddr(
 	for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++)
 	{
 		size_t       size	  = host->params[pclass][mem].size;
-		hrt_vaddress ddr_mem_ptr  = ddr->params[pclass][mem].address;
+		ia_css_ptr ddr_mem_ptr  = ddr->params[pclass][mem].address;
 		char	    *host_mem_ptr = host->params[pclass][mem].address;
 
 		if (size != ddr->params[pclass][mem].size)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 		if (!size)
 			continue;
-		mmgr_store(ddr_mem_ptr, host_mem_ptr, size);
+		hmm_store(ddr_mem_ptr, host_mem_ptr, size);
 	}
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 void
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h
index e2aca35452c0..f975429b8705 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -48,10 +49,10 @@ enum mipi_port_id ia_css_isys_port_to_mipi_port(
  * @param[in]	port		CSI port
  * @param[in]	isys_stream_id	Stream handle generated with ia_css_isys_generate_stream_id()
  *				Must be lower than SH_CSS_MAX_ISYS_CHANNEL_NODES
- * @return			IA_CSS_SUCCESS if successful, IA_CSS_ERR_INTERNAL_ERROR if
+ * @return			0 if successful, -EINVAL if
  *				there is already a stream registered with the same handle
  */
-enum ia_css_err ia_css_isys_csi_rx_register_stream(
+int ia_css_isys_csi_rx_register_stream(
     enum mipi_port_id port,
     uint32_t isys_stream_id);
 
@@ -63,14 +64,14 @@ enum ia_css_err ia_css_isys_csi_rx_register_stream(
  * @param[in]	port		CSI port
  * @param[in]	isys_stream_id	Stream handle generated with ia_css_isys_generate_stream_id()
  *				Must be lower than SH_CSS_MAX_ISYS_CHANNEL_NODES
- * @return			IA_CSS_SUCCESS if successful, IA_CSS_ERR_INTERNAL_ERROR if
+ * @return			0 if successful, -EINVAL if
  *				there is no stream registered with that handle
  */
-enum ia_css_err ia_css_isys_csi_rx_unregister_stream(
+int ia_css_isys_csi_rx_unregister_stream(
     enum mipi_port_id port,
     uint32_t isys_stream_id);
 
-enum ia_css_err ia_css_isys_convert_compressed_format(
+int ia_css_isys_convert_compressed_format(
     struct ia_css_csi2_compression *comp,
     struct input_system_cfg_s *cfg);
 unsigned int ia_css_csi2_calculate_input_system_alignment(
@@ -107,7 +108,7 @@ unsigned int ia_css_isys_rx_translate_irq_infos(unsigned int bits);
  * This is normally done by the sensor, but when using the input fifo, this
  * format type must be sumitted correctly by the application.
  */
-enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format(
+int ia_css_isys_convert_stream_format_to_mipi_format(
     enum atomisp_input_format input_format,
     mipi_predictor_t compression,
     unsigned int *fmt_type);
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h
index 6ad7a0cd5146..6f1a86c81d7c 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c
index 8f2ce2c057eb..5a44d8f6c196 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -126,11 +127,11 @@ void ia_css_isys_csi_rx_lut_rmgr_release(
 	}
 }
 
-enum ia_css_err ia_css_isys_csi_rx_register_stream(
+int ia_css_isys_csi_rx_register_stream(
     enum mipi_port_id port,
     uint32_t isys_stream_id)
 {
-	enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR;
+	int retval = -EINVAL;
 
 	if ((port < N_INPUT_SYSTEM_CSI_PORT) &&
 	    (isys_stream_id < SH_CSS_MAX_ISYS_CHANNEL_NODES)) {
@@ -140,17 +141,17 @@ enum ia_css_err ia_css_isys_csi_rx_register_stream(
 		if (bitop_getbit(pipe_io_status->active[port], isys_stream_id) == 0) {
 			bitop_setbit(pipe_io_status->active[port], isys_stream_id);
 			pipe_io_status->running[port] = 0;
-			retval = IA_CSS_SUCCESS;
+			retval = 0;
 		}
 	}
 	return retval;
 }
 
-enum ia_css_err ia_css_isys_csi_rx_unregister_stream(
+int ia_css_isys_csi_rx_unregister_stream(
     enum mipi_port_id port,
     uint32_t isys_stream_id)
 {
-	enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR;
+	int retval = -EINVAL;
 
 	if ((port < N_INPUT_SYSTEM_CSI_PORT) &&
 	    (isys_stream_id < SH_CSS_MAX_ISYS_CHANNEL_NODES)) {
@@ -159,7 +160,7 @@ enum ia_css_err ia_css_isys_csi_rx_unregister_stream(
 		pipe_io_status = ia_css_pipeline_get_pipe_io_status();
 		if (bitop_getbit(pipe_io_status->active[port], isys_stream_id) == 1) {
 			bitop_clearbit(pipe_io_status->active[port], isys_stream_id);
-			retval = IA_CSS_SUCCESS;
+			retval = 0;
 		}
 	}
 	return retval;
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h
index 79d7c4b29bf4..11f730dc1c08 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c
index 9055ed387673..9710493c47ac 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h
index 7155e2c6e05c..7c754ec7224a 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c
index 930fa7a0ff53..68baec78b1c4 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h
index e3d07ac390fc..88c3d5581999 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c
index b923233ec5b0..de442f1fa6ba 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c
index 53355a55d05d..bc4a2ff3c0fc 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h
index b55cf02c8bce..78a4c867fb1b 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c
index 43665ddff8ea..4f0dcdfa13be 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -216,7 +217,7 @@ void ia_css_isys_rx_clear_irq_info(enum mipi_port_id port,
 }
 #endif /* #if !defined(USE_INPUT_SYSTEM_VERSION_2401) */
 
-enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format(
+int ia_css_isys_convert_stream_format_to_mipi_format(
     enum atomisp_input_format input_format,
     mipi_predictor_t compression,
     unsigned int *fmt_type)
@@ -254,9 +255,9 @@ enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format(
 			*fmt_type = 16;
 			break;
 		default:
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 		}
-		return IA_CSS_SUCCESS;
+		return 0;
 	}
 	/*
 	 * This mapping comes from the Arasan CSS function spec
@@ -356,9 +357,9 @@ enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format(
 	case ATOMISP_INPUT_FORMAT_YUV420_16:
 	case ATOMISP_INPUT_FORMAT_YUV422_16:
 	default:
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 #if defined(USE_INPUT_SYSTEM_VERSION_2401)
@@ -379,11 +380,11 @@ static mipi_predictor_t sh_css_csi2_compression_type_2_mipi_predictor(
 	return predictor;
 }
 
-enum ia_css_err ia_css_isys_convert_compressed_format(
+int ia_css_isys_convert_compressed_format(
     struct ia_css_csi2_compression *comp,
     struct input_system_cfg_s *cfg)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(comp);
 	assert(cfg);
@@ -414,7 +415,7 @@ enum ia_css_err ia_css_isys_convert_compressed_format(
 				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_10_8_10;
 				break;
 			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
+				err = -EINVAL;
 			}
 		} else if (comp->uncompressed_bits_per_pixel ==
 			   UNCOMPRESSED_BITS_PER_PIXEL_12) {
@@ -429,10 +430,10 @@ enum ia_css_err ia_css_isys_convert_compressed_format(
 				cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_12_8_12;
 				break;
 			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
+				err = -EINVAL;
 			}
 		} else
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
+			err = -EINVAL;
 		cfg->csi_port_attr.comp_predictor =
 		    sh_css_csi2_compression_type_2_mipi_predictor(comp->type);
 		cfg->csi_port_attr.comp_enable = true;
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c
index 9a795a21d3e6..b3c6831cb9e3 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,6 +13,8 @@
  * more details.
  */
 
+#include <linux/string.h> /* for memcpy() */
+
 #include "system_global.h"
 
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
@@ -19,7 +22,6 @@
 #include "ia_css_isys.h"
 #include "ia_css_debug.h"
 #include "math_support.h"
-#include "string_support.h"
 #include "virtual_isys.h"
 #include "isp.h"
 #include "sh_css_defs.h"
@@ -649,14 +651,8 @@ static bool calculate_tpg_cfg(
     input_system_cfg_t		*isys_cfg,
     pixelgen_tpg_cfg_t		*cfg)
 {
-	(void)channel;
-	(void)input_port;
+	memcpy(cfg, &isys_cfg->tpg_port_attr, sizeof(pixelgen_tpg_cfg_t));
 
-	memcpy_s(
-	    (void *)cfg,
-	    sizeof(pixelgen_tpg_cfg_t),
-	    (void *)(&isys_cfg->tpg_port_attr),
-	    sizeof(pixelgen_tpg_cfg_t));
 	return true;
 }
 
@@ -666,14 +662,8 @@ static bool calculate_prbs_cfg(
     input_system_cfg_t		*isys_cfg,
     pixelgen_prbs_cfg_t		*cfg)
 {
-	(void)channel;
-	(void)input_port;
+	memcpy(cfg, &isys_cfg->prbs_port_attr, sizeof(pixelgen_prbs_cfg_t));
 
-	memcpy_s(
-	    (void *)cfg,
-	    sizeof(pixelgen_prbs_cfg_t),
-	    (void *)(&isys_cfg->prbs_port_attr),
-	    sizeof(pixelgen_prbs_cfg_t));
 	return true;
 }
 
@@ -691,12 +681,10 @@ static bool calculate_be_cfg(
     bool				metadata,
     csi_rx_backend_cfg_t		*cfg)
 {
-	memcpy_s(
-	    (void *)(&cfg->lut_entry),
-	    sizeof(csi_rx_backend_lut_entry_t),
-	    metadata ? (void *)(&input_port->metadata.backend_lut_entry) :
-	    (void *)(&input_port->csi_rx.backend_lut_entry),
-	    sizeof(csi_rx_backend_lut_entry_t));
+	memcpy(&cfg->lut_entry,
+	      metadata ? &input_port->metadata.backend_lut_entry :
+			 &input_port->csi_rx.backend_lut_entry,
+	      sizeof(csi_rx_backend_lut_entry_t));
 
 	cfg->csi_mipi_cfg.virtual_channel = isys_cfg->csi_port_attr.ch_id;
 	if (metadata) {
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h
index b675907791ad..fbdbca0cfcc8 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h
index 6a41efee5635..18a7d18e197e 100644
--- a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h
+++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -56,8 +57,7 @@ struct ia_css_pipeline {
 	u32 pipe_qos_config;
 };
 
-#define DEFAULT_PIPELINE \
-(struct ia_css_pipeline) { \
+#define DEFAULT_PIPELINE { \
 	.pipe_id		= IA_CSS_PIPE_ID_PREVIEW, \
 	.in_frame		= DEFAULT_FRAME, \
 	.out_frame		= {DEFAULT_FRAME}, \
@@ -94,13 +94,13 @@ void ia_css_pipeline_init(void);
  * @param[out] pipeline  structure to be initialized with defaults
  * @param[in] pipe_id
  * @param[in] pipe_num Number that uniquely identifies a pipeline.
- * @return                     IA_CSS_SUCCESS or error code upon error.
+ * @return                     0 or error code upon error.
  *
  * Initializes the pipeline structure with a set of default values.
  * This API is expected to be used when a pipeline structure is allocated
  * externally and needs sane defaults
  */
-enum ia_css_err ia_css_pipeline_create(
+int ia_css_pipeline_create(
     struct ia_css_pipeline *pipeline,
     enum ia_css_pipe_id pipe_id,
     unsigned int pipe_num,
@@ -127,10 +127,10 @@ void ia_css_pipeline_start(enum ia_css_pipe_id pipe_id,
 /* @brief Request to stop a pipeline
  *
  * @param[in] pipeline
- * @return                     IA_CSS_SUCCESS or error code upon error.
+ * @return                     0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline);
+int ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline);
 
 /* @brief Check whether pipeline has stopped
  *
@@ -153,13 +153,13 @@ void ia_css_pipeline_clean(struct ia_css_pipeline *pipeline);
  * @param     pipeline               Pointer to the pipeline to be added to.
  * @param[in] stage_desc       The description of the stage
  * @param[out] stage            The successor of the stage.
- * @return                     IA_CSS_SUCCESS or error code upon error.
+ * @return                     0 or error code upon error.
  *
  * Add a new stage to a non-NULL pipeline.
  * The stage consists of an ISP binary or firmware and input and output
  * arguments.
 */
-enum ia_css_err ia_css_pipeline_create_and_add_stage(
+int ia_css_pipeline_create_and_add_stage(
     struct ia_css_pipeline *pipeline,
     struct ia_css_pipeline_stage_desc *stage_desc,
     struct ia_css_pipeline_stage **stage);
@@ -177,10 +177,10 @@ void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline,
 /* @brief gets a stage from the pipeline
  *
  * @param[in] pipeline
- * @return                     IA_CSS_SUCCESS or error code upon error.
+ * @return                     0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
+int ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
 	int mode,
 	struct ia_css_pipeline_stage **stage);
 
@@ -190,10 +190,10 @@ enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
  * @param[in] fw_handle
  * @param[out] stage Pointer to Stage
  *
- * @return   IA_CSS_SUCCESS or error code upon error.
+ * @return   0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline
+int ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline
 	*pipeline,
 	u32 fw_handle,
 	struct ia_css_pipeline_stage **stage);
@@ -204,10 +204,10 @@ enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline
  * @param[in] stage_num
  * @param[out] fw_handle
  *
- * @return   IA_CSS_SUCCESS or error code upon error.
+ * @return   0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline
+int ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline
 	*pipeline,
 	u32 stage_num,
 	uint32_t *fw_handle);
@@ -215,10 +215,10 @@ enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline
 /* @brief gets the output stage from the pipeline
  *
  * @param[in] pipeline
- * @return                     IA_CSS_SUCCESS or error code upon error.
+ * @return                     0 or error code upon error.
  *
  */
-enum ia_css_err ia_css_pipeline_get_output_stage(
+int ia_css_pipeline_get_output_stage(
     struct ia_css_pipeline *pipeline,
     int mode,
     struct ia_css_pipeline_stage **stage);
diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h
index b96a5b146096..cc44f03c3b34 100644
--- a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h
+++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c b/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c
index 8b9982de8deb..4b8e85bc2122 100644
--- a/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c
+++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -12,11 +13,12 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include "ia_css_debug.h"
 #include "sw_event_global.h"		/* encode_sw_event */
 #include "sp.h"			/* cnd_sp_irq_enable() */
 #include "assert_support.h"
-#include "memory_access.h"
 #include "sh_css_sp.h"
 #include "ia_css_pipeline.h"
 #include "ia_css_isp_param.h"
@@ -45,7 +47,7 @@ static void pipeline_init_defaults(
     unsigned int dvs_frame_delay);
 
 static void pipeline_stage_destroy(struct ia_css_pipeline_stage *stage);
-static enum ia_css_err pipeline_stage_create(
+static int pipeline_stage_create(
     struct ia_css_pipeline_stage_desc *stage_desc,
     struct ia_css_pipeline_stage **new_stage);
 static void ia_css_pipeline_set_zoom_stage(struct ia_css_pipeline *pipeline);
@@ -60,7 +62,7 @@ void ia_css_pipeline_init(void)
 	pipeline_init_sp_thread_map();
 }
 
-enum ia_css_err ia_css_pipeline_create(
+int ia_css_pipeline_create(
     struct ia_css_pipeline *pipeline,
     enum ia_css_pipe_id pipe_id,
     unsigned int pipe_num,
@@ -70,14 +72,14 @@ enum ia_css_err ia_css_pipeline_create(
 	IA_CSS_ENTER_PRIVATE("pipeline = %p, pipe_id = %d, pipe_num = %d, dvs_frame_delay = %d",
 			     pipeline, pipe_id, pipe_num, dvs_frame_delay);
 	if (!pipeline) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	pipeline_init_defaults(pipeline, pipe_id, pipe_num, dvs_frame_delay);
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
 void ia_css_pipeline_map(unsigned int pipe_num, bool map)
@@ -195,15 +197,15 @@ void ia_css_pipeline_dump_thread_map_info(void)
 	}
 }
 
-enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline)
+int ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int thread_id;
 
 	assert(pipeline);
 
 	if (!pipeline)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_pipeline_request_stop() enter: pipeline=%p\n",
@@ -218,7 +220,7 @@ enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline)
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_pipeline_request_stop() leaving\n");
 		/* queues are invalid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 	ia_css_bufq_enqueue_psys_event(IA_CSS_PSYS_SW_EVENT_STOP_STREAM,
 				       (uint8_t)thread_id,
@@ -263,19 +265,19 @@ void ia_css_pipeline_clean(struct ia_css_pipeline *pipeline)
  * @param       pipeline      Pointer to the pipeline to be added to.
  * @param[in]   stage_desc    The description of the stage
  * @param[out]	stage         The successor of the stage.
- * @return      IA_CSS_SUCCESS or error code upon error.
+ * @return      0 or error code upon error.
  *
  * Add a new stage to a non-NULL pipeline.
  * The stage consists of an ISP binary or firmware and input and
  * output arguments.
 */
-enum ia_css_err ia_css_pipeline_create_and_add_stage(
+int ia_css_pipeline_create_and_add_stage(
     struct ia_css_pipeline *pipeline,
     struct ia_css_pipeline_stage_desc *stage_desc,
     struct ia_css_pipeline_stage **stage)
 {
 	struct ia_css_pipeline_stage *last, *new_stage = NULL;
-	enum ia_css_err err;
+	int err;
 
 	/* other arguments can be NULL */
 	assert(pipeline);
@@ -289,7 +291,7 @@ enum ia_css_err ia_css_pipeline_create_and_add_stage(
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_pipeline_create_and_add_stage() done: Invalid args\n");
 
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
 
 	/* Find the last stage */
@@ -308,12 +310,12 @@ enum ia_css_err ia_css_pipeline_create_and_add_stage(
 			stage_desc->in_frame = last->args.out_frame[0];
 
 		if (!stage_desc->in_frame)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 	}
 
 	/* Create the new stage */
 	err = pipeline_stage_create(stage_desc, &new_stage);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_pipeline_create_and_add_stage() done: stage_create_failed\n");
 		return err;
@@ -330,7 +332,7 @@ enum ia_css_err ia_css_pipeline_create_and_add_stage(
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_pipeline_create_and_add_stage() done:\n");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline,
@@ -350,7 +352,7 @@ void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline,
 	ia_css_pipeline_configure_inout_port(pipeline, continuous);
 }
 
-enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
+int ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
 	int mode,
 	struct ia_css_pipeline_stage **stage)
 {
@@ -363,13 +365,13 @@ enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline,
 	for (s = pipeline->stages; s; s = s->next) {
 		if (s->mode == mode) {
 			*stage = s;
-			return IA_CSS_SUCCESS;
+			return 0;
 		}
 	}
-	return IA_CSS_ERR_INTERNAL_ERROR;
+	return -EINVAL;
 }
 
-enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline
+int ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline
 	*pipeline,
 	u32 fw_handle,
 	struct ia_css_pipeline_stage **stage)
@@ -382,13 +384,13 @@ enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline
 	for (s = pipeline->stages; s; s = s->next) {
 		if ((s->firmware) && (s->firmware->handle == fw_handle)) {
 			*stage = s;
-			return IA_CSS_SUCCESS;
+			return 0;
 		}
 	}
-	return IA_CSS_ERR_INTERNAL_ERROR;
+	return -EINVAL;
 }
 
-enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline
+int ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline
 	*pipeline,
 	u32 stage_num,
 	uint32_t *fw_handle)
@@ -397,18 +399,18 @@ enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s()\n", __func__);
 	if ((!pipeline) || (!fw_handle))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	for (s = pipeline->stages; s; s = s->next) {
 		if ((s->stage_num == stage_num) && (s->firmware)) {
 			*fw_handle = s->firmware->handle;
-			return IA_CSS_SUCCESS;
+			return 0;
 		}
 	}
-	return IA_CSS_ERR_INTERNAL_ERROR;
+	return -EINVAL;
 }
 
-enum ia_css_err ia_css_pipeline_get_output_stage(
+int ia_css_pipeline_get_output_stage(
     struct ia_css_pipeline *pipeline,
     int mode,
     struct ia_css_pipeline_stage **stage)
@@ -428,7 +430,7 @@ enum ia_css_err ia_css_pipeline_get_output_stage(
 			*stage = s;
 	}
 	if (*stage)
-		return IA_CSS_SUCCESS;
+		return 0;
 	/* If no firmware, find binary in pipe */
 	return ia_css_pipeline_get_stage(pipeline, mode, stage);
 }
@@ -511,7 +513,7 @@ static void pipeline_stage_destroy(struct ia_css_pipeline_stage *stage)
 		ia_css_frame_free(stage->args.out_vf_frame);
 		stage->args.out_vf_frame = NULL;
 	}
-	sh_css_free(stage);
+	kvfree(stage);
 }
 
 static void pipeline_init_sp_thread_map(void)
@@ -566,11 +568,11 @@ static void pipeline_unmap_num_to_sp_thread(unsigned int pipe_num)
 	pipeline_sp_thread_list[thread_id] = PIPELINE_SP_THREAD_EMPTY_TOKEN;
 }
 
-static enum ia_css_err pipeline_stage_create(
+static int pipeline_stage_create(
     struct ia_css_pipeline_stage_desc *stage_desc,
     struct ia_css_pipeline_stage **new_stage)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipeline_stage *stage = NULL;
 	struct ia_css_binary *binary;
 	struct ia_css_frame *vf_frame;
@@ -581,7 +583,7 @@ static enum ia_css_err pipeline_stage_create(
 	/* Verify input parameters*/
 	if (!(stage_desc->in_frame) && !(stage_desc->firmware)
 	    && (stage_desc->binary) && !(stage_desc->binary->online)) {
-		err = IA_CSS_ERR_INTERNAL_ERROR;
+		err = -EINVAL;
 		goto ERR;
 	}
 
@@ -592,12 +594,11 @@ static enum ia_css_err pipeline_stage_create(
 		out_frame[i] = stage_desc->out_frame[i];
 	}
 
-	stage = sh_css_malloc(sizeof(*stage));
+	stage = kvzalloc(sizeof(*stage), GFP_KERNEL);
 	if (!stage) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
-	memset(stage, 0, sizeof(*stage));
 
 	if (firmware) {
 		stage->binary = NULL;
@@ -627,7 +628,7 @@ static enum ia_css_err pipeline_stage_create(
 		    && (binary->out_frame_info[i].res.width)) {
 			err = ia_css_frame_allocate_from_info(&out_frame[i],
 							      &binary->out_frame_info[i]);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 			stage->out_frame_allocated[i] = true;
 		}
@@ -641,7 +642,7 @@ static enum ia_css_err pipeline_stage_create(
 		   ) {
 			err = ia_css_frame_allocate_from_info(&vf_frame,
 							      &binary->vf_frame_info);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 			stage->vf_frame_allocated = true;
 		}
@@ -664,6 +665,8 @@ ERR:
 	return err;
 }
 
+static const struct ia_css_frame ia_css_default_frame = DEFAULT_FRAME;
+
 static void pipeline_init_defaults(
     struct ia_css_pipeline *pipeline,
     enum ia_css_pipe_id pipe_id,
@@ -676,10 +679,15 @@ static void pipeline_init_defaults(
 	pipeline->stages = NULL;
 	pipeline->stop_requested = false;
 	pipeline->current_stage = NULL;
-	pipeline->in_frame = DEFAULT_FRAME;
+
+	memcpy(&pipeline->in_frame, &ia_css_default_frame,
+	       sizeof(ia_css_default_frame));
+
 	for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) {
-		pipeline->out_frame[i] = DEFAULT_FRAME;
-		pipeline->vf_frame[i] = DEFAULT_FRAME;
+		memcpy(&pipeline->out_frame[i], &ia_css_default_frame,
+		       sizeof(ia_css_default_frame));
+		memcpy(&pipeline->vf_frame[i], &ia_css_default_frame,
+		       sizeof(ia_css_default_frame));
 	}
 	pipeline->num_execs = -1;
 	pipeline->acquire_isp_each_stage = true;
@@ -690,30 +698,30 @@ static void pipeline_init_defaults(
 static void ia_css_pipeline_set_zoom_stage(struct ia_css_pipeline *pipeline)
 {
 	struct ia_css_pipeline_stage *stage = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(pipeline);
 	if (pipeline->pipe_id == IA_CSS_PIPE_ID_PREVIEW) {
 		/* in preview pipeline, vf_pp stage should do zoom */
 		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_VF_PP, &stage);
-		if (err == IA_CSS_SUCCESS)
+		if (!err)
 			stage->enable_zoom = true;
 	} else if (pipeline->pipe_id == IA_CSS_PIPE_ID_CAPTURE) {
 		/* in capture pipeline, capture_pp stage should do zoom */
 		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP,
 						&stage);
-		if (err == IA_CSS_SUCCESS)
+		if (!err)
 			stage->enable_zoom = true;
 	} else if (pipeline->pipe_id == IA_CSS_PIPE_ID_VIDEO) {
 		/* in video pipeline, video stage should do zoom */
 		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_VIDEO, &stage);
-		if (err == IA_CSS_SUCCESS)
+		if (!err)
 			stage->enable_zoom = true;
 	} else if (pipeline->pipe_id == IA_CSS_PIPE_ID_YUVPP) {
 		/* in yuvpp pipeline, first yuv_scaler stage should do zoom */
 		err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP,
 						&stage);
-		if (err == IA_CSS_SUCCESS)
+		if (!err)
 			stage->enable_zoom = true;
 	}
 }
diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h
index 6daeb060daf9..08112be4633f 100644
--- a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h
+++ b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -40,7 +41,7 @@ typedef struct ia_css_queue ia_css_queue_t;
  * @param[out] qhandle. Handle to queue instance for use with API
  * @param[in]  desc.   Descriptor with queue properties filled-in
  * @return     0      - Successful init of local queue instance.
- * @return     EINVAL - Invalid argument.
+ * @return     -EINVAL - Invalid argument.
  *
  */
 int ia_css_queue_local_init(
@@ -52,7 +53,7 @@ int ia_css_queue_local_init(
  * @param[out] qhandle. Handle to queue instance for use with API
  * @param[in]  desc.   Descriptor with queue properties filled-in
  * @return     0      - Successful init of remote queue instance.
- * @return     EINVAL - Invalid argument.
+ * @return     -EINVAL - Invalid argument.
  */
 int ia_css_queue_remote_init(
     ia_css_queue_t *qhandle,
@@ -72,8 +73,8 @@ int ia_css_queue_uninit(
  * @param[in]  qhandle. Handle to queue instance
  * @param[in]  item.    Object to be enqueued.
  * @return     0       - Successful enqueue.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOBUFS - Queue is full.
+ * @return     -EINVAL  - Invalid argument.
+ * @return     -ENOBUFS - Queue is full.
  *
  */
 int ia_css_queue_enqueue(
@@ -86,8 +87,8 @@ int ia_css_queue_enqueue(
  * @param[out] item.    Object to be dequeued into this item.
 
  * @return     0       - Successful dequeue.
- * @return     EINVAL  - Invalid argument.
- * @return     ENODATA - Queue is empty.
+ * @return     -EINVAL  - Invalid argument.
+ * @return     -ENODATA - Queue is empty.
  *
  */
 int ia_css_queue_dequeue(
@@ -99,8 +100,8 @@ int ia_css_queue_dequeue(
  * @param[in]  qhandle.  Handle to queue instance
  * @param[in]  is_empty  True if empty, False if not.
  * @return     0       - Successful access state.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOSYS  - Function not implemented.
+ * @return     -EINVAL  - Invalid argument.
+ * @return     -ENOSYS  - Function not implemented.
  *
  */
 int ia_css_queue_is_empty(
@@ -112,8 +113,8 @@ int ia_css_queue_is_empty(
  * @param[in]  qhandle.  Handle to queue instance
  * @param[in]  is_full   True if Full, False if not.
  * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOSYS  - Function not implemented.
+ * @return     -EINVAL  - Invalid argument.
+ * @return     -ENOSYS  - Function not implemented.
  *
  */
 int ia_css_queue_is_full(
@@ -125,7 +126,7 @@ int ia_css_queue_is_full(
  * @param[in]  qhandle.  Handle to queue instance
  * @param[in]  size      Number of available elements in the queue
  * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
+ * @return     -EINVAL  - Invalid argument.
  *
  */
 int ia_css_queue_get_used_space(
@@ -137,7 +138,7 @@ int ia_css_queue_get_used_space(
  * @param[in]  qhandle.  Handle to queue instance
  * @param[in]  size      Number of free elements in the queue
  * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
+ * @return     -EINVAL  - Invalid argument.
  *
  */
 int ia_css_queue_get_free_space(
@@ -151,7 +152,7 @@ int ia_css_queue_get_free_space(
  *			 starting from head of queue
  * @param[in]  element   Value of element returned
  * @return     0       - Successfully access state.
- * @return     EINVAL  - Invalid argument.
+ * @return     -EINVAL  - Invalid argument.
  *
  */
 int ia_css_queue_peek(
@@ -164,8 +165,8 @@ int ia_css_queue_peek(
  * @param[in]  qhandle. Handle to queue instance
  * @param[out] size     Size value to be returned here.
  * @return     0       - Successful get size.
- * @return     EINVAL  - Invalid argument.
- * @return     ENOSYS  - Function not implemented.
+ * @return     -EINVAL  - Invalid argument.
+ * @return     -ENOSYS  - Function not implemented.
  *
  */
 int ia_css_queue_get_size(
diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h
index 87fa4288d9a6..1379ae8f8c01 100644
--- a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h
+++ b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c
index dd79c6f180af..aea6c66a3cee 100644
--- a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c
+++ b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -28,7 +29,7 @@ int ia_css_queue_local_init(
 	if (NULL == qhandle || NULL == desc
 	    || NULL == desc->cb_elems || NULL == desc->cb_desc) {
 		/* Invalid parameters, return error*/
-		return EINVAL;
+		return -EINVAL;
 	}
 
 	/* Mark the queue as Local */
@@ -48,7 +49,7 @@ int ia_css_queue_remote_init(
 {
 	if (NULL == qhandle || NULL == desc) {
 		/* Invalid parameters, return error*/
-		return EINVAL;
+		return -EINVAL;
 	}
 
 	/* Mark the queue as remote*/
@@ -72,7 +73,7 @@ int ia_css_queue_uninit(
     ia_css_queue_t *qhandle)
 {
 	if (!qhandle)
-		return EINVAL;
+		return -EINVAL;
 
 	/* Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -90,7 +91,7 @@ int ia_css_queue_enqueue(
 	int error = 0;
 
 	if (!qhandle)
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -99,7 +100,7 @@ int ia_css_queue_enqueue(
 		 */
 		if (ia_css_circbuf_is_full(&qhandle->desc.cb_local)) {
 			/* Cannot push the element. Return*/
-			return ENOBUFS;
+			return -ENOBUFS;
 		}
 
 		/* Push the element*/
@@ -117,7 +118,7 @@ int ia_css_queue_enqueue(
 
 		/* b. Operate on the queue */
 		if (ia_css_circbuf_desc_is_full(&cb_desc))
-			return ENOBUFS;
+			return -ENOBUFS;
 
 		cb_elem.val = item;
 
@@ -149,7 +150,7 @@ int ia_css_queue_dequeue(
 	int error = 0;
 
 	if (!qhandle || NULL == item)
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -158,7 +159,7 @@ int ia_css_queue_dequeue(
 		 */
 		if (ia_css_circbuf_is_empty(&qhandle->desc.cb_local)) {
 			/* Nothing to pop. Return empty queue*/
-			return ENODATA;
+			return -ENODATA;
 		}
 
 		*item = ia_css_circbuf_pop(&qhandle->desc.cb_local);
@@ -176,7 +177,7 @@ int ia_css_queue_dequeue(
 
 		/* b. Operate on the queue */
 		if (ia_css_circbuf_desc_is_empty(&cb_desc))
-			return ENODATA;
+			return -ENODATA;
 
 		error = ia_css_queue_item_load(qhandle, cb_desc.start, &cb_elem);
 		if (error != 0)
@@ -206,7 +207,7 @@ int ia_css_queue_is_full(
 	int error = 0;
 
 	if ((!qhandle) || (!is_full))
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -230,7 +231,7 @@ int ia_css_queue_is_full(
 		return 0;
 	}
 
-	return EINVAL;
+	return -EINVAL;
 }
 
 int ia_css_queue_get_free_space(
@@ -240,7 +241,7 @@ int ia_css_queue_get_free_space(
 	int error = 0;
 
 	if ((!qhandle) || (!size))
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -264,7 +265,7 @@ int ia_css_queue_get_free_space(
 		return 0;
 	}
 
-	return EINVAL;
+	return -EINVAL;
 }
 
 int ia_css_queue_get_used_space(
@@ -274,7 +275,7 @@ int ia_css_queue_get_used_space(
 	int error = 0;
 
 	if ((!qhandle) || (!size))
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -298,7 +299,7 @@ int ia_css_queue_get_used_space(
 		return 0;
 	}
 
-	return EINVAL;
+	return -EINVAL;
 }
 
 int ia_css_queue_peek(
@@ -310,7 +311,7 @@ int ia_css_queue_peek(
 	int error = 0;
 
 	if ((!qhandle) || (!element))
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -320,7 +321,7 @@ int ia_css_queue_peek(
 		/* Check if offset is valid */
 		num_elems = ia_css_circbuf_get_num_elems(&qhandle->desc.cb_local);
 		if (offset > num_elems)
-			return EINVAL;
+			return -EINVAL;
 
 		*element = ia_css_circbuf_peek_from_start(&qhandle->desc.cb_local, (int)offset);
 		return 0;
@@ -339,7 +340,7 @@ int ia_css_queue_peek(
 		/* Check if offset is valid */
 		num_elems = ia_css_circbuf_desc_get_num_elems(&cb_desc);
 		if (offset > num_elems)
-			return EINVAL;
+			return -EINVAL;
 
 		offset = OP_std_modadd(cb_desc.start, offset, cb_desc.size);
 		error = ia_css_queue_item_load(qhandle, (uint8_t)offset, &cb_elem);
@@ -350,7 +351,7 @@ int ia_css_queue_peek(
 		return 0;
 	}
 
-	return EINVAL;
+	return -EINVAL;
 }
 
 int ia_css_queue_is_empty(
@@ -360,7 +361,7 @@ int ia_css_queue_is_empty(
 	int error = 0;
 
 	if ((!qhandle) || (!is_empty))
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
@@ -384,7 +385,7 @@ int ia_css_queue_is_empty(
 		return 0;
 	}
 
-	return EINVAL;
+	return -EINVAL;
 }
 
 int ia_css_queue_get_size(
@@ -394,7 +395,7 @@ int ia_css_queue_get_size(
 	int error = 0;
 
 	if ((!qhandle) || (!size))
-		return EINVAL;
+		return -EINVAL;
 
 	/* 1. Load the required queue object */
 	if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) {
diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c
index 1e8d3eb82eab..fdca743c4ab7 100644
--- a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c
+++ b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -12,11 +13,12 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include "type_support.h"
 #include "queue_access.h"
 #include "ia_css_circbuf.h"
 #include "sp.h"
-#include "memory_access.h"
 #include "assert_support.h"
 
 int ia_css_queue_load(
@@ -25,7 +27,7 @@ int ia_css_queue_load(
     uint32_t ignore_desc_flags)
 {
 	if (!rdesc || !cb_desc)
-		return EINVAL;
+		return -EINVAL;
 
 	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
 		assert(ignore_desc_flags <= QUEUE_IGNORE_DESC_FLAGS_MAX);
@@ -63,12 +65,12 @@ int ia_css_queue_load(
 
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
 		/* doing DMA transfer of entire structure */
-		mmgr_load(rdesc->desc.remote.cb_desc_addr,
+		hmm_load(rdesc->desc.remote.cb_desc_addr,
 			  (void *)cb_desc,
 			  sizeof(ia_css_circbuf_desc_t));
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
 		/* Not supported yet */
-		return ENOTSUP;
+		return -ENOTSUPP;
 	}
 
 	return 0;
@@ -80,7 +82,7 @@ int ia_css_queue_store(
     uint32_t ignore_desc_flags)
 {
 	if (!rdesc || !cb_desc)
-		return EINVAL;
+		return -EINVAL;
 
 	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
 		assert(ignore_desc_flags <= QUEUE_IGNORE_DESC_FLAGS_MAX);
@@ -110,12 +112,12 @@ int ia_css_queue_store(
 					    cb_desc->step);
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
 		/* doing DMA transfer of entire structure */
-		mmgr_store(rdesc->desc.remote.cb_desc_addr,
+		hmm_store(rdesc->desc.remote.cb_desc_addr,
 			   (void *)cb_desc,
 			   sizeof(ia_css_circbuf_desc_t));
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
 		/* Not supported yet */
-		return ENOTSUP;
+		return -ENOTSUPP;
 	}
 
 	return 0;
@@ -127,7 +129,7 @@ int ia_css_queue_item_load(
     ia_css_circbuf_elem_t *item)
 {
 	if (!rdesc || !item)
-		return EINVAL;
+		return -EINVAL;
 
 	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
 		sp_dmem_load(rdesc->proc_id,
@@ -136,13 +138,13 @@ int ia_css_queue_item_load(
 			     item,
 			     sizeof(ia_css_circbuf_elem_t));
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
-		mmgr_load(rdesc->desc.remote.cb_elems_addr
+		hmm_load(rdesc->desc.remote.cb_elems_addr
 			  + position * sizeof(ia_css_circbuf_elem_t),
 			  (void *)item,
 			  sizeof(ia_css_circbuf_elem_t));
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
 		/* Not supported yet */
-		return ENOTSUP;
+		return -ENOTSUPP;
 	}
 
 	return 0;
@@ -154,7 +156,7 @@ int ia_css_queue_item_store(
     ia_css_circbuf_elem_t *item)
 {
 	if (!rdesc || !item)
-		return EINVAL;
+		return -EINVAL;
 
 	if (rdesc->location == IA_CSS_QUEUE_LOC_SP) {
 		sp_dmem_store(rdesc->proc_id,
@@ -163,13 +165,13 @@ int ia_css_queue_item_store(
 			      item,
 			      sizeof(ia_css_circbuf_elem_t));
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) {
-		mmgr_store(rdesc->desc.remote.cb_elems_addr
+		hmm_store(rdesc->desc.remote.cb_elems_addr
 			   + position * sizeof(ia_css_circbuf_elem_t),
 			   (void *)item,
 			   sizeof(ia_css_circbuf_elem_t));
 	} else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) {
 		/* Not supported yet */
-		return ENOTSUP;
+		return -ENOTSUPP;
 	}
 
 	return 0;
diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h
index 884c55a754d1..d5107adccef9 100644
--- a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h
+++ b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -15,10 +16,11 @@
 #ifndef __QUEUE_ACCESS_H
 #define __QUEUE_ACCESS_H
 
+#include <linux/errno.h>
+
 #include <type_support.h>
 #include <ia_css_queue_comm.h>
 #include <ia_css_circbuf.h>
-#include <error_support.h>
 
 #define QUEUE_IGNORE_START_FLAG	0x0001
 #define QUEUE_IGNORE_END_FLAG	0x0002
diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h
index 47a80ae8dbf3..9cd3d92b34c9 100644
--- a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h
+++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -28,7 +29,7 @@
 /**
  * @brief Initialize resource manager (host/common)
  */
-enum ia_css_err ia_css_rmgr_init(void);
+int ia_css_rmgr_init(void);
 
 /**
  * @brief Uninitialize resource manager (host/common)
diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
index 0660b65f2e34..ac969afc8bb4 100644
--- a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
+++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -17,13 +18,14 @@
 
 #include "ia_css_rmgr.h"
 #include <type_support.h>
-#include <system_types.h>
+#include <ia_css_types.h>
+#include <system_local.h>
 
 /**
  * @brief Data structure for the resource handle (host, vbuf)
  */
 struct ia_css_rmgr_vbuf_handle {
-	hrt_vaddress vptr;
+	ia_css_ptr vptr;
 	u8 count;
 	u32 size;
 };
@@ -51,7 +53,7 @@ extern struct ia_css_rmgr_vbuf_pool *hmm_buffer_pool;
  *
  * @param pool	The pointer to the pool
  */
-STORAGE_CLASS_RMGR_H enum ia_css_err ia_css_rmgr_init_vbuf(
+STORAGE_CLASS_RMGR_H int ia_css_rmgr_init_vbuf(
     struct ia_css_rmgr_vbuf_pool *pool);
 
 /**
diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c
index 23ae19ee65ca..c94a428aadde 100644
--- a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c
+++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -14,16 +15,16 @@
 
 #include "ia_css_rmgr.h"
 
-enum ia_css_err ia_css_rmgr_init(void)
+int ia_css_rmgr_init(void)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	err = ia_css_rmgr_init_vbuf(vbuf_ref);
-	if (err == IA_CSS_SUCCESS)
+	if (!err)
 		err = ia_css_rmgr_init_vbuf(vbuf_write);
-	if (err == IA_CSS_SUCCESS)
+	if (!err)
 		err = ia_css_rmgr_init_vbuf(hmm_buffer_pool);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		ia_css_rmgr_uninit();
 	return err;
 }
diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c
index 2c204dceb491..1ea74296fc8d 100644
--- a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c
+++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010-2015, Intel Corporation.
@@ -12,12 +13,12 @@
  * more details.
  */
 
+#include "hmm.h"
 #include "ia_css_rmgr.h"
 
 #include <type_support.h>
 #include <assert_support.h>
 #include <platform_support.h> /* memset */
-#include <memory_access.h>    /* mmmgr_malloc, mhmm_free */
 #include <ia_css_debug.h>
 
 /*
@@ -137,26 +138,26 @@ void ia_css_rmgr_refcount_release_vbuf(struct ia_css_rmgr_vbuf_handle **handle)
  *
  * @param pool	The pointer to the pool
  */
-enum ia_css_err ia_css_rmgr_init_vbuf(struct ia_css_rmgr_vbuf_pool *pool)
+int ia_css_rmgr_init_vbuf(struct ia_css_rmgr_vbuf_pool *pool)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	size_t bytes_needed;
 
 	rmgr_refcount_init_vbuf();
 	assert(pool);
 	if (!pool)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	/* initialize the recycle pool if used */
 	if (pool->recycle && pool->size) {
 		/* allocate memory for storing the handles */
 		bytes_needed =
 		    sizeof(void *) *
 		    pool->size;
-		pool->handles = sh_css_malloc(bytes_needed);
+		pool->handles = kvmalloc(bytes_needed, GFP_KERNEL);
 		if (pool->handles)
 			memset(pool->handles, 0, bytes_needed);
 		else
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 	} else {
 		/* just in case, set the size to 0 */
 		pool->size = 0;
@@ -196,7 +197,7 @@ void ia_css_rmgr_uninit_vbuf(struct ia_css_rmgr_vbuf_pool *pool)
 			}
 		}
 		/* now free the pool handles list */
-		sh_css_free(pool->handles);
+		kvfree(pool->handles);
 		pool->handles = NULL;
 	}
 }
@@ -297,7 +298,7 @@ void ia_css_rmgr_acq_vbuf(struct ia_css_rmgr_vbuf_pool *pool,
 			}
 			if ((*handle)->vptr == 0x0) {
 				/* we need to allocate */
-				(*handle)->vptr = mmgr_malloc((*handle)->size);
+				(*handle)->vptr = hmm_alloc((*handle)->size, HMM_BO_PRIVATE, 0, NULL, 0);
 			} else {
 				/* we popped a buffer */
 				return;
diff --git a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h
index 543ca8968418..efe6c4a82caf 100644
--- a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h
+++ b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -34,11 +35,11 @@ typedef struct {
 } ia_css_spctrl_cfg;
 
 /* Get the code addr in DDR of SP */
-hrt_vaddress get_sp_code_addr(sp_ID_t  sp_id);
+ia_css_ptr get_sp_code_addr(sp_ID_t  sp_id);
 
 /* ! Load firmware on to specfied SP
 */
-enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id,
+int ia_css_spctrl_load_fw(sp_ID_t sp_id,
 				      ia_css_spctrl_cfg *spctrl_cfg);
 
 /* ISP2401 */
@@ -47,15 +48,15 @@ void sh_css_spctrl_reload_fw(sp_ID_t sp_id);
 
 /*!  Unload/release any memory allocated to hold the firmware
 */
-enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id);
+int ia_css_spctrl_unload_fw(sp_ID_t sp_id);
 
 /*! Intilaize dmem_cfg in SP dmem  and  start SP program
 */
-enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id);
+int ia_css_spctrl_start(sp_ID_t sp_id);
 
 /*! stop spctrl
 */
-enum ia_css_err ia_css_spctrl_stop(sp_ID_t sp_id);
+int ia_css_spctrl_stop(sp_ID_t sp_id);
 
 /*! Query the state of SP
 */
diff --git a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h
index ca37c4ab7544..78e0f3096f60 100644
--- a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h
+++ b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c b/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c
index db39fa273251..753a99703f1e 100644
--- a/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c
+++ b/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -12,11 +13,12 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include "ia_css_types.h"
 #define __INLINE_SP__
 #include "sp.h"
 
-#include "memory_access.h"
 #include "assert_support.h"
 #include "ia_css_spctrl.h"
 #include "ia_css_debug.h"
@@ -26,7 +28,7 @@ struct spctrl_context_info {
 	u32        spctrl_config_dmem_addr; /* location of dmem_cfg  in SP dmem */
 	u32        spctrl_state_dmem_addr;
 	unsigned int    sp_entry;           /* entry function ptr on SP */
-	hrt_vaddress    code_addr;          /* sp firmware location in host mem-DDR*/
+	ia_css_ptr    code_addr;          /* sp firmware location in host mem-DDR*/
 	u32        code_size;
 	char           *program_name;       /* used in case of PLATFORM_SIM */
 };
@@ -35,14 +37,14 @@ static struct spctrl_context_info spctrl_cofig_info[N_SP_ID];
 static bool spctrl_loaded[N_SP_ID] = {0};
 
 /* Load firmware */
-enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id,
+int ia_css_spctrl_load_fw(sp_ID_t sp_id,
 				      ia_css_spctrl_cfg *spctrl_cfg)
 {
-	hrt_vaddress code_addr = mmgr_NULL;
+	ia_css_ptr code_addr = mmgr_NULL;
 	struct ia_css_sp_init_dmem_cfg *init_dmem_cfg;
 
 	if ((sp_id >= N_SP_ID) || (!spctrl_cfg))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	spctrl_cofig_info[sp_id].code_addr = mmgr_NULL;
 
@@ -63,17 +65,17 @@ enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id,
 	 * Data used to be stored separately, because of access alignment constraints,
 	 * fix the FW generation instead
 	 */
-	code_addr = mmgr_malloc(spctrl_cfg->code_size);
+	code_addr = hmm_alloc(spctrl_cfg->code_size, HMM_BO_PRIVATE, 0, NULL, 0);
 	if (code_addr == mmgr_NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	mmgr_store(code_addr, spctrl_cfg->code, spctrl_cfg->code_size);
+		return -ENOMEM;
+	hmm_store(code_addr, spctrl_cfg->code, spctrl_cfg->code_size);
 
-	if (sizeof(hrt_vaddress) > sizeof(hrt_data)) {
+	if (sizeof(ia_css_ptr) > sizeof(hrt_data)) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
-				    "size of hrt_vaddress can not be greater than hrt_data\n");
+				    "size of ia_css_ptr can not be greater than hrt_data\n");
 		hmm_free(code_addr);
 		code_addr = mmgr_NULL;
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
 
 	init_dmem_cfg->ddr_data_addr  = code_addr + spctrl_cfg->ddr_data_offset;
@@ -82,7 +84,7 @@ enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id,
 				    "DDR address pointer is not properly aligned for DMA transfer\n");
 		hmm_free(code_addr);
 		code_addr = mmgr_NULL;
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
 
 	spctrl_cofig_info[sp_id].sp_entry = spctrl_cfg->sp_entry;
@@ -96,7 +98,7 @@ enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id,
 		      (hrt_data)spctrl_cofig_info[sp_id].code_addr);
 	sp_ctrl_setbit(sp_id, SP_ICACHE_INV_REG, SP_ICACHE_INV_BIT);
 	spctrl_loaded[sp_id] = true;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /* ISP2401 */
@@ -112,15 +114,15 @@ void sh_css_spctrl_reload_fw(sp_ID_t sp_id)
 	spctrl_loaded[sp_id] = true;
 }
 
-hrt_vaddress get_sp_code_addr(sp_ID_t  sp_id)
+ia_css_ptr get_sp_code_addr(sp_ID_t  sp_id)
 {
 	return spctrl_cofig_info[sp_id].code_addr;
 }
 
-enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id)
+int ia_css_spctrl_unload_fw(sp_ID_t sp_id)
 {
 	if ((sp_id >= N_SP_ID) || ((sp_id < N_SP_ID) && (!spctrl_loaded[sp_id])))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/*  freeup the resource */
 	if (spctrl_cofig_info[sp_id].code_addr) {
@@ -128,14 +130,14 @@ enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id)
 		spctrl_cofig_info[sp_id].code_addr = mmgr_NULL;
 	}
 	spctrl_loaded[sp_id] = false;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /* Initialize dmem_cfg in SP dmem  and  start SP program*/
-enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id)
+int ia_css_spctrl_start(sp_ID_t sp_id)
 {
 	if ((sp_id >= N_SP_ID) || ((sp_id < N_SP_ID) && (!spctrl_loaded[sp_id])))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/* Set descr in the SP to initialize the SP DMEM */
 	/*
@@ -154,7 +156,7 @@ enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id)
 		      (hrt_data)spctrl_cofig_info[sp_id].sp_entry);
 	sp_ctrl_setbit(sp_id, SP_SC_REG, SP_RUN_BIT);
 	sp_ctrl_setbit(sp_id, SP_SC_REG, SP_START_BIT);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /* Query the state of SP1 */
diff --git a/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h b/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h
index 899294646494..49801fbc1924 100644
--- a/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h
+++ b/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2010 - 2015, Intel Corporation.
@@ -23,11 +24,7 @@
  *
  * Should be one less than NUM_CONTINUOUS_FRAMES in sh_css_internal.h
  */
-#if defined(HAS_SP_2400)
 #define MAX_CB_ELEMS_FOR_TAGGER 14
-#else
-#define MAX_CB_ELEMS_FOR_TAGGER 9
-#endif
 
 /**
  * @brief Data structure for the tagger buffer element.
diff --git a/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c b/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c
index 57dddd74d668..679ef8242574 100644
--- a/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c
+++ b/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -18,14 +19,14 @@
 #include "gp_timer.h" /*gp_timer_read()*/
 #include "assert_support.h"
 
-enum ia_css_err
+int
 ia_css_timer_get_current_tick(
     struct ia_css_clock_tick *curr_ts) {
 	assert(curr_ts);
 	if (!curr_ts)
 	{
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	curr_ts->ticks = (clock_value_t)gp_timer_read(GP_TIMER_SEL);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h b/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h
index 9b6c2893d950..7e7188797b0a 100644
--- a/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h
+++ b/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css.c b/drivers/staging/media/atomisp/pci/sh_css.c
index d77432254a2c..6676537f0e97 100644
--- a/drivers/staging/media/atomisp/pci/sh_css.c
+++ b/drivers/staging/media/atomisp/pci/sh_css.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -17,6 +18,8 @@
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 
+#include "hmm.h"
+
 #include "ia_css.h"
 #include "sh_css_hrt.h"		/* only for file 2 MIPI */
 #include "ia_css_buffer.h"
@@ -52,7 +55,6 @@
 #include "ia_css_isys.h"
 #endif
 
-#include "memory_access.h"
 #include "tag.h"
 #include "assert_support.h"
 #include "math_support.h"
@@ -74,7 +76,6 @@
 #define __INLINE_GPIO__
 #include "gpio.h"
 #include "timed_ctrl.h"
-#include "platform_support.h" /* hrt_sleep(), inline */
 #include "ia_css_inputfifo.h"
 #define WITH_PC_MONITORING  0
 
@@ -181,10 +182,10 @@ static bool fw_explicitly_loaded;
  * Local prototypes
  */
 
-static enum ia_css_err
+static int
 allocate_delay_frames(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 sh_css_pipe_start(struct ia_css_stream *stream);
 
 /* ISP 2401 */
@@ -195,7 +196,7 @@ sh_css_pipe_start(struct ia_css_stream *stream);
  * @param[in] stream	Point to the target "ia_css_stream" instance.
  *
  * @return
- * - IA_CSS_SUCCESS, if the "stop" requests have been successfully sent out.
+ * - 0, if the "stop" requests have been successfully sent out.
  * - CSS error code, otherwise.
  *
  *
@@ -205,7 +206,7 @@ sh_css_pipe_start(struct ia_css_stream *stream);
  * return without waiting for all "ia_css_pipe" instatnces
  * being stopped.
  */
-static enum ia_css_err
+static int
 sh_css_pipes_stop(struct ia_css_stream *stream);
 
 /*
@@ -224,15 +225,15 @@ static bool
 sh_css_pipes_have_stopped(struct ia_css_stream *stream);
 
 /* ISP 2401 */
-static enum ia_css_err
+static int
 ia_css_pipe_check_format(struct ia_css_pipe *pipe,
 			 enum ia_css_frame_format format);
 
 /* ISP 2401 */
-static enum ia_css_err
+static int
 check_pipe_resolutions(const struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 ia_css_pipe_load_extension(struct ia_css_pipe *pipe,
 			   struct ia_css_fw_info *firmware);
 
@@ -245,7 +246,7 @@ ia_css_reset_defaults(struct sh_css *css);
 static void
 sh_css_init_host_sp_control_vars(void);
 
-static enum ia_css_err set_num_primary_stages(unsigned int *num,
+static int set_num_primary_stages(unsigned int *num,
 	enum ia_css_pipe_version version);
 
 static bool
@@ -254,7 +255,7 @@ need_capture_pp(const struct ia_css_pipe *pipe);
 static bool
 need_yuv_scaler_stage(const struct ia_css_pipe *pipe);
 
-static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output(
+static int ia_css_pipe_create_cas_scaler_desc_single_output(
     struct ia_css_frame_info *cas_scaler_in_info,
     struct ia_css_frame_info *cas_scaler_out_info,
     struct ia_css_frame_info *cas_scaler_vf_info,
@@ -269,91 +270,91 @@ need_downscaling(const struct ia_css_resolution in_res,
 
 static bool need_capt_ldc(const struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 sh_css_pipe_load_binaries(struct ia_css_pipe *pipe);
 
 static
-enum ia_css_err sh_css_pipe_get_viewfinder_frame_info(
+int sh_css_pipe_get_viewfinder_frame_info(
     struct ia_css_pipe *pipe,
     struct ia_css_frame_info *info,
     unsigned int idx);
 
-static enum ia_css_err
+static int
 sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe,
 				  struct ia_css_frame_info *info,
 				  unsigned int idx);
 
-static enum ia_css_err
+static int
 capture_start(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 video_start(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 preview_start(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 yuvpp_start(struct ia_css_pipe *pipe);
 
 static bool copy_on_sp(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 init_vf_frameinfo_defaults(struct ia_css_pipe *pipe,
 			   struct ia_css_frame *vf_frame, unsigned int idx);
 
-static enum ia_css_err
+static int
 init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe,
 				  struct ia_css_frame *frame, enum ia_css_frame_format format);
 
-static enum ia_css_err
+static int
 init_out_frameinfo_defaults(struct ia_css_pipe *pipe,
 			    struct ia_css_frame *out_frame, unsigned int idx);
 
-static enum ia_css_err
+static int
 sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline,
 			      const void *acc_fw);
 
-static enum ia_css_err
+static int
 alloc_continuous_frames(
     struct ia_css_pipe *pipe, bool init_time);
 
 static void
 pipe_global_init(void);
 
-static enum ia_css_err
+static int
 pipe_generate_pipe_num(const struct ia_css_pipe *pipe,
 		       unsigned int *pipe_number);
 
 static void
 pipe_release_pipe_num(unsigned int pipe_num);
 
-static enum ia_css_err
+static int
 create_host_pipeline_structure(struct ia_css_stream *stream);
 
-static enum ia_css_err
+static int
 create_host_pipeline(struct ia_css_stream *stream);
 
-static enum ia_css_err
+static int
 create_host_preview_pipeline(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 create_host_video_pipeline(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 create_host_copy_pipeline(struct ia_css_pipe *pipe,
 			  unsigned int max_input_width,
 			  struct ia_css_frame *out_frame);
 
-static enum ia_css_err
+static int
 create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 create_host_capture_pipeline(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 create_host_yuvpp_pipeline(struct ia_css_pipe *pipe);
 
-static enum ia_css_err
+static int
 create_host_acc_pipeline(struct ia_css_pipe *pipe);
 
 static unsigned int
@@ -383,7 +384,7 @@ static struct sh_css_hmm_buffer_record
 				  hrt_address kernel_ptr);
 
 static struct sh_css_hmm_buffer_record
-*sh_css_hmm_buffer_record_validate(hrt_vaddress ddr_buffer_addr,
+*sh_css_hmm_buffer_record_validate(ia_css_ptr ddr_buffer_addr,
 				   enum ia_css_buffer_type type);
 
 void
@@ -392,7 +393,7 @@ ia_css_get_acc_configs(
     struct ia_css_isp_config *config);
 
 #if CONFIG_ON_FRAME_ENQUEUE()
-static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info
+static int set_config_on_frame_enqueue(struct ia_css_frame_info
 	*info, struct frame_data_wrapper *frame);
 #endif
 
@@ -403,7 +404,7 @@ static unsigned int get_crop_columns_for_bayer_order(const struct
 	ia_css_stream_config *config);
 static void get_pipe_extra_pixel(struct ia_css_pipe *pipe,
 				 unsigned int *extra_row, unsigned int *extra_column);
-static enum ia_css_err
+static int
 aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
 		       struct ia_css_pipe *pipes[],
 		       bool *do_crop_status);
@@ -411,7 +412,7 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
 static bool
 aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe);
 
-static enum ia_css_err
+static int
 aspect_ratio_crop(struct ia_css_pipe *curr_pipe,
 		  struct ia_css_resolution *effective_res);
 #endif
@@ -458,7 +459,7 @@ static enum ia_css_frame_format yuv422_copy_formats[] = {
 /* Verify whether the selected output format is can be produced
  * by the copy binary given the stream format.
  * */
-static enum ia_css_err
+static int
 verify_copy_out_frame_format(struct ia_css_pipe *pipe) {
 	enum ia_css_frame_format out_fmt = pipe->output_info[0].format;
 	unsigned int i, found = 0;
@@ -514,8 +515,8 @@ verify_copy_out_frame_format(struct ia_css_pipe *pipe) {
 		break;
 	}
 	if (!found)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	return IA_CSS_SUCCESS;
+		return -EINVAL;
+	return 0;
 }
 
 unsigned int
@@ -533,12 +534,12 @@ ia_css_stream_input_format_bits_per_pixel(struct ia_css_stream *stream)
 #define GP_ISEL_TPG_MODE 0x90058
 
 #if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-static enum ia_css_err
+static int
 sh_css_config_input_network(struct ia_css_stream *stream) {
 	unsigned int fmt_type;
 	struct ia_css_pipe *pipe = stream->last_pipe;
 	struct ia_css_binary *binary = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(stream);
 	assert(pipe);
@@ -553,7 +554,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 	    stream->config.input_config.format,
 	    stream->csi_rx_config.comp,
 	    &fmt_type);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	sh_css_sp_program_input_circuit(fmt_type,
 					stream->config.channel_id,
@@ -564,7 +565,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 	{
 		err = ia_css_ifmtr_configure(&stream->config,
 					     binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -582,7 +583,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 		vblank_cycles = vblank_lines * (width + hblank_cycles);
 		sh_css_sp_configure_sync_gen(width, height, hblank_cycles,
 					     vblank_cycles);
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG) {
 				/* TODO: move define to proper file in tools */
 				ia_css_device_store_uint32(GP_ISEL_TPG_MODE, 0);
@@ -591,7 +592,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 	}
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 			    "sh_css_config_input_network() leave:\n");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 #elif !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
 static unsigned int csi2_protocol_calculate_max_subpixels_per_line(
@@ -879,14 +880,14 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr(
 
 		break;
 	case IA_CSS_INPUT_MODE_BUFFERED_SENSOR: {
-		enum ia_css_err err;
+		int err;
 		unsigned int fmt_type;
 
 		err = ia_css_isys_convert_stream_format_to_mipi_format(
 			  stream_cfg->isys_config[isys_stream_idx].format,
 			  MIPI_PREDICTOR_NONE,
 			  &fmt_type);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			rc = false;
 
 		isys_stream_descr->csi_port_attr.active_lanes =
@@ -899,7 +900,7 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr(
 		err |= ia_css_isys_convert_compressed_format(
 			   &stream_cfg->source.port.compression,
 			   isys_stream_descr);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			rc = false;
 
 		/* metadata */
@@ -909,7 +910,7 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr(
 				  stream_cfg->metadata_config.data_type,
 				  MIPI_PREDICTOR_NONE,
 				  &fmt_type);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				rc = false;
 			isys_stream_descr->metadata.fmt_type = fmt_type;
 			isys_stream_descr->metadata.bits_per_pixel =
@@ -1050,7 +1051,7 @@ static bool sh_css_translate_binary_info_to_input_system_output_port_attr(
 	return true;
 }
 
-static enum ia_css_err
+static int
 sh_css_config_input_network(struct ia_css_stream *stream) {
 	bool					rc;
 	ia_css_isys_descr_t			isys_stream_descr;
@@ -1095,7 +1096,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 
 	assert(pipe);
 	if (!pipe)
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 
 	if (pipe->pipeline.stages)
 		if (pipe->pipeline.stages->binary)
@@ -1113,7 +1114,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 	/* get the SP thread id */
 	rc = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &sp_thread_id);
 	if (!rc)
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	/* get the target input terminal */
 	sp_pipeline_input_terminal = &sh_css_sp_group.pipe_io[sp_thread_id].input;
 
@@ -1140,7 +1141,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 		}
 
 		if (!rc)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 
 		isys_stream_id = ia_css_isys_generate_stream_id(sp_thread_id, i);
 
@@ -1150,7 +1151,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 			  &sp_pipeline_input_terminal->context.virtual_input_system_stream[i],
 			  isys_stream_id);
 		if (!rc)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 
 		/* calculate the configuration of the virtual Input System (2401) */
 		rc = ia_css_isys_stream_calculate_cfg(
@@ -1160,14 +1161,14 @@ sh_css_config_input_network(struct ia_css_stream *stream) {
 		if (!rc) {
 			ia_css_isys_stream_destroy(
 			    &sp_pipeline_input_terminal->context.virtual_input_system_stream[i]);
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 		}
 	}
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 			    "sh_css_config_input_network() leave:\n");
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static inline struct ia_css_pipe *stream_get_last_pipe(
@@ -1225,11 +1226,11 @@ static inline struct ia_css_pipe *stream_get_target_pipe(
 	return target_pipe;
 }
 
-static enum ia_css_err stream_csi_rx_helper(
+static int stream_csi_rx_helper(
     struct ia_css_stream *stream,
-    enum ia_css_err (*func)(enum mipi_port_id, uint32_t))
+    int (*func)(enum mipi_port_id, uint32_t))
 {
-	enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR;
+	int retval = -EINVAL;
 	u32 sp_thread_id, stream_id;
 	bool rc;
 	struct ia_css_pipe *target_pipe = NULL;
@@ -1258,20 +1259,20 @@ static enum ia_css_err stream_csi_rx_helper(
 			retval = func(stream->config.source.port.port, isys_stream_id);
 		}
 		stream_id++;
-	} while ((retval == IA_CSS_SUCCESS) &&
+	} while ((retval == 0) &&
 		 (stream_id < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH));
 
 exit:
 	return retval;
 }
 
-static inline enum ia_css_err stream_register_with_csi_rx(
+static inline int stream_register_with_csi_rx(
     struct ia_css_stream *stream)
 {
 	return stream_csi_rx_helper(stream, ia_css_isys_csi_rx_register_stream);
 }
 
-static inline enum ia_css_err stream_unregister_with_csi_rx(
+static inline int stream_unregister_with_csi_rx(
     struct ia_css_stream *stream)
 {
 	return stream_csi_rx_helper(stream, ia_css_isys_csi_rx_unregister_stream);
@@ -1324,7 +1325,7 @@ static void print_pc_histogram(void)
 		sh_css_print(" pc_histogram for binary %d\n", metrics->id);
 		print_pc_histo("  ISP", &metrics->isp_histogram);
 		print_pc_histo("  SP",   &metrics->sp_histogram);
-		sh_css_print("print_pc_histogram() done for binay->id = %d, done.\n",
+		sh_css_print("print_pc_histogram() done for binary->id = %d, done.\n",
 			     metrics->id);
 	}
 
@@ -1404,7 +1405,7 @@ start_binary(struct ia_css_pipe *pipe,
 }
 
 /* start the copy function on the SP */
-static enum ia_css_err
+static int
 start_copy_on_sp(struct ia_css_pipe *pipe,
 		 struct ia_css_frame *out_frame) {
 	(void)out_frame;
@@ -1412,7 +1413,7 @@ start_copy_on_sp(struct ia_css_pipe *pipe,
 	assert(pipe->stream);
 
 	if ((!pipe) || (!pipe->stream))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 #if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
 	if (pipe->stream->reconfigure_css_rx)
@@ -1420,7 +1421,7 @@ start_copy_on_sp(struct ia_css_pipe *pipe,
 #endif
 
 	if (pipe->stream->config.input_config.format != ATOMISP_INPUT_FORMAT_BINARY_8)
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	sh_css_sp_start_binary_copy(ia_css_pipe_get_pipe_num(pipe), out_frame, pipe->stream->config.pixels_per_clock == 2);
 
 #if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
@@ -1432,7 +1433,7 @@ start_copy_on_sp(struct ia_css_pipe *pipe,
 	}
 #endif
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 void sh_css_binary_args_reset(struct sh_css_binary_args *args)
@@ -1469,7 +1470,7 @@ static void start_pipe(
 
 	assert(me); /* all callers are in this file and call with non null argument */
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		coord = &me->config.internal_frame_origin_bqs_on_sctbl;
 		params = me->stream->isp_params_configs;
 	}
@@ -1544,16 +1545,16 @@ enable_interrupts(enum ia_css_irq_type irq_type)
 
 	/* Enable SW interrupt 0, this is used to signal ISYS events */
 	cnd_virq_enable_channel(
-	    (virq_id_t)(IRQ_SW_CHANNEL0_ID + IRQ_SW_CHANNEL_OFFSET),
+	    (enum virq_id)(IRQ_SW_CHANNEL0_ID + IRQ_SW_CHANNEL_OFFSET),
 	    true);
 	/* Enable SW interrupt 1, this is used to signal PSYS events */
 	cnd_virq_enable_channel(
-	    (virq_id_t)(IRQ_SW_CHANNEL1_ID + IRQ_SW_CHANNEL_OFFSET),
+	    (enum virq_id)(IRQ_SW_CHANNEL1_ID + IRQ_SW_CHANNEL_OFFSET),
 	    true);
 #if !defined(HAS_IRQ_MAP_VERSION_2)
 	/* IRQ_SW_CHANNEL2_ID does not exist on 240x systems */
 	cnd_virq_enable_channel(
-	    (virq_id_t)(IRQ_SW_CHANNEL2_ID + IRQ_SW_CHANNEL_OFFSET),
+	    (enum virq_id)(IRQ_SW_CHANNEL2_ID + IRQ_SW_CHANNEL_OFFSET),
 	    true);
 	virq_clear_all();
 #endif
@@ -1624,15 +1625,15 @@ ia_css_reset_defaults(struct sh_css *css)
 	*css = default_css;
 }
 
-enum ia_css_err
+int
 ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,
 		     const struct ia_css_fw  *fw) {
-	enum ia_css_err err;
+	int err;
 
 	if (!env)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	if (!fw)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_load_firmware() enter\n");
 
@@ -1645,10 +1646,10 @@ ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,
 
 	ia_css_unload_firmware(); /* in case we are called twice */
 	err = sh_css_load_firmware(dev, fw->data, fw->bytes);
-	if (err == IA_CSS_SUCCESS)
+	if (!err)
 	{
 		err = ia_css_binary_init_infos();
-		if (err == IA_CSS_SUCCESS)
+		if (!err)
 			fw_explicitly_loaded = true;
 	}
 
@@ -1656,12 +1657,12 @@ ia_css_load_firmware(struct device *dev, const struct ia_css_env *env,
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_init(struct device *dev, const struct ia_css_env *env,
 	    const struct ia_css_fw  *fw,
 	    u32                 mmu_l1_base,
 	    enum ia_css_irq_type     irq_type) {
-	enum ia_css_err err;
+	int err;
 	ia_css_spctrl_cfg spctrl_cfg;
 
 	void (*flush_func)(struct ia_css_acc_fw *fw);
@@ -1704,9 +1705,9 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 	COMPILATION_ERROR_IF(sizeof(struct ia_css_sp_init_dmem_cfg)		!= SIZE_OF_IA_CSS_SP_INIT_DMEM_CFG_STRUCT);
 
 	if (!fw && !fw_explicitly_loaded)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	if (!env)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	sh_css_printf = env->print_env.debug_print;
 
@@ -1734,7 +1735,7 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 	my_css.flush     = flush_func;
 
 	err = ia_css_rmgr_init();
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR(err);
 		return err;
@@ -1771,13 +1772,13 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 	gpio_reg_store(GPIO0_ID, _gpio_block_reg_do_0, 0);
 
 	err = ia_css_refcount_init(REFCOUNT_SIZE);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
 	err = sh_css_params_init();
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR(err);
 		return err;
@@ -1786,12 +1787,12 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 	{
 		ia_css_unload_firmware(); /* in case we already had firmware loaded */
 		err = sh_css_load_firmware(dev, fw->data, fw->bytes);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR(err);
 			return err;
 		}
 		err = ia_css_binary_init_infos();
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR(err);
 			return err;
 		}
@@ -1801,10 +1802,10 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 #endif
 	}
 	if (!sh_css_setup_spctrl_config(&sh_css_sp_fw, SP_PROG_NAME, &spctrl_cfg))
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 
 	err = ia_css_spctrl_load_fw(SP0_ID, &spctrl_cfg);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR(err);
 		return err;
@@ -1821,8 +1822,8 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 #endif
 	if (!sh_css_hrt_system_is_idle())
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_SYSTEM_NOT_IDLE);
-		return IA_CSS_ERR_SYSTEM_NOT_IDLE;
+		IA_CSS_LEAVE_ERR(-EBUSY);
+		return -EBUSY;
 	}
 	/* can be called here, queuing works, but:
 	   - when sp is started later, it will wipe queued items
@@ -1844,7 +1845,7 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 			       ISP_DMA_MAX_BURST_LENGTH);
 
 	if (ia_css_isys_init() != INPUT_SYSTEM_ERR_NO_ERROR)
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 #endif
 
 	sh_css_params_map_and_store_default_gdc_lut();
@@ -1853,49 +1854,12 @@ ia_css_init(struct device *dev, const struct ia_css_env *env,
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_enable_isys_event_queue(bool enable) {
 	if (sh_css_sp_is_running())
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	sh_css_sp_enable_isys_event_queue(enable);
-	return IA_CSS_SUCCESS;
-}
-
-void *sh_css_malloc(size_t size)
-{
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_malloc() enter: size=%zu\n",
-			    size);
-	/* FIXME: This first test can probably go away */
-	if (size == 0)
-		return NULL;
-	if (size > PAGE_SIZE)
-		return vmalloc(size);
-	return kmalloc(size, GFP_KERNEL);
-}
-
-void *sh_css_calloc(size_t N, size_t size)
-{
-	void *p;
-
-	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
-			    "sh_css_calloc() enter: N=%zu, size=%zu\n", N, size);
-
-	/* FIXME: this test can probably go away */
-	if (size > 0) {
-		p = sh_css_malloc(N * size);
-		if (p)
-			memset(p, 0, size);
-		return p;
-	}
-	return NULL;
-}
-
-void sh_css_free(void *ptr)
-{
-	if (is_vmalloc_addr(ptr))
-		vfree(ptr);
-	else
-		kfree(ptr);
+	return 0;
 }
 
 /* For Acceleration API: Flush FW (shared buffer pointer) arguments */
@@ -1911,13 +1875,13 @@ sh_css_flush(struct ia_css_acc_fw *fw)
  * pipelines are ready to be converted to sp pipelines. Be careful if you are
  * doing it from stream_create since we could run out of sp threads due to
  * allocation on inactive pipelines. */
-static enum ia_css_err
+static int
 map_sp_threads(struct ia_css_stream *stream, bool map) {
 	struct ia_css_pipe *main_pipe = NULL;
 	struct ia_css_pipe *copy_pipe = NULL;
 	struct ia_css_pipe *capture_pipe = NULL;
 	struct ia_css_pipe *acc_pipe = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	enum ia_css_pipe_id pipe_id;
 
 	assert(stream);
@@ -1926,8 +1890,8 @@ map_sp_threads(struct ia_css_stream *stream, bool map) {
 
 	if (!stream)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	main_pipe = stream->last_pipe;
@@ -1984,13 +1948,13 @@ map_sp_threads(struct ia_css_stream *stream, bool map) {
 
 /* creates a host pipeline skeleton for all pipes in a stream. Called during
  * stream_create. */
-static enum ia_css_err
+static int
 create_host_pipeline_structure(struct ia_css_stream *stream) {
 	struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL;
 	struct ia_css_pipe *acc_pipe = NULL;
 	enum ia_css_pipe_id pipe_id;
 	struct ia_css_pipe *main_pipe = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int copy_pipe_delay = 0,
 	capture_pipe_delay = 0;
 
@@ -1999,16 +1963,16 @@ create_host_pipeline_structure(struct ia_css_stream *stream) {
 
 	if (!stream)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	main_pipe	= stream->last_pipe;
 	assert(main_pipe);
 	if (!main_pipe)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	pipe_id	= main_pipe->mode;
@@ -2050,10 +2014,10 @@ create_host_pipeline_structure(struct ia_css_stream *stream) {
 		break;
 
 	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	}
 
-	if ((err == IA_CSS_SUCCESS) && copy_pipe)
+	if (!(err) && copy_pipe)
 	{
 		err = ia_css_pipeline_create(&copy_pipe->pipeline,
 					     copy_pipe->mode,
@@ -2061,7 +2025,7 @@ create_host_pipeline_structure(struct ia_css_stream *stream) {
 					     copy_pipe_delay);
 	}
 
-	if ((err == IA_CSS_SUCCESS) && capture_pipe)
+	if (!(err) && capture_pipe)
 	{
 		err = ia_css_pipeline_create(&capture_pipe->pipeline,
 					     capture_pipe->mode,
@@ -2069,7 +2033,7 @@ create_host_pipeline_structure(struct ia_css_stream *stream) {
 					     capture_pipe_delay);
 	}
 
-	if ((err == IA_CSS_SUCCESS) && acc_pipe)
+	if (!(err) && acc_pipe)
 	{
 		err = ia_css_pipeline_create(&acc_pipe->pipeline, acc_pipe->mode,
 					     acc_pipe->pipe_num, main_pipe->dvs_frame_delay);
@@ -2080,7 +2044,7 @@ create_host_pipeline_structure(struct ia_css_stream *stream) {
 	{
 		int i;
 
-		for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err; i++) {
+		for (i = 1; i < stream->num_pipes && 0 == err; i++) {
 			main_pipe = stream->pipes[i];
 			err = ia_css_pipeline_create(&main_pipe->pipeline,
 						     main_pipe->mode,
@@ -2095,20 +2059,20 @@ create_host_pipeline_structure(struct ia_css_stream *stream) {
 
 /* creates a host pipeline for all pipes in a stream. Called during
  * stream_start. */
-static enum ia_css_err
+static int
 create_host_pipeline(struct ia_css_stream *stream) {
 	struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL;
 	struct ia_css_pipe *acc_pipe = NULL;
 	enum ia_css_pipe_id pipe_id;
 	struct ia_css_pipe *main_pipe = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int max_input_width = 0;
 
 	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
 	if (!stream)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	main_pipe	= stream->last_pipe;
@@ -2129,7 +2093,7 @@ create_host_pipeline(struct ia_css_stream *stream) {
 		    (pipe_id == IA_CSS_PIPE_ID_PREVIEW &&
 		     stream->config.mode != IA_CSS_INPUT_MODE_MEMORY)) {
 			err = alloc_continuous_frames(main_pipe, true);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 		}
 	}
@@ -2139,7 +2103,7 @@ create_host_pipeline(struct ia_css_stream *stream) {
 	if (pipe_id != IA_CSS_PIPE_ID_ACC)
 	{
 		err = allocate_mipi_frames(main_pipe, &stream->info);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 #elif defined(USE_INPUT_SYSTEM_VERSION_2401)
@@ -2147,7 +2111,7 @@ create_host_pipeline(struct ia_css_stream *stream) {
 	    (main_pipe->config.mode != IA_CSS_PIPE_MODE_COPY))
 	{
 		err = allocate_mipi_frames(main_pipe, &stream->info);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 #endif
@@ -2162,7 +2126,7 @@ create_host_pipeline(struct ia_css_stream *stream) {
 		    main_pipe->pipe_settings.preview.preview_binary.info->sp.input.max_width;
 
 		err = create_host_preview_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 
 		break;
@@ -2174,7 +2138,7 @@ create_host_pipeline(struct ia_css_stream *stream) {
 		    main_pipe->pipe_settings.video.video_binary.info->sp.input.max_width;
 
 		err = create_host_video_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 
 		break;
@@ -2186,42 +2150,42 @@ create_host_pipeline(struct ia_css_stream *stream) {
 
 	case IA_CSS_PIPE_ID_YUVPP:
 		err = create_host_yuvpp_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 
 		break;
 
 	case IA_CSS_PIPE_ID_ACC:
 		err = create_host_acc_pipeline(main_pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 
 		break;
 	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	}
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 	if (copy_pipe)
 	{
 		err = create_host_copy_pipeline(copy_pipe, max_input_width,
 						main_pipe->continuous_frames[0]);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
 	if (capture_pipe)
 	{
 		err = create_host_capture_pipeline(capture_pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
 	if (acc_pipe)
 	{
 		err = create_host_acc_pipeline(acc_pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
@@ -2230,7 +2194,7 @@ create_host_pipeline(struct ia_css_stream *stream) {
 	{
 		int i;
 
-		for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err; i++) {
+		for (i = 1; i < stream->num_pipes && 0 == err; i++) {
 			switch (stream->pipes[i]->mode) {
 			case IA_CSS_PIPE_ID_PREVIEW:
 				err = create_host_preview_pipeline(stream->pipes[i]);
@@ -2248,9 +2212,9 @@ create_host_pipeline(struct ia_css_stream *stream) {
 				err = create_host_acc_pipeline(stream->pipes[i]);
 				break;
 			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
+				err = -EINVAL;
 			}
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 		}
 	}
@@ -2260,25 +2224,32 @@ ERR:
 	return err;
 }
 
-static enum ia_css_err
+static const struct ia_css_pipe default_pipe = IA_CSS_DEFAULT_PIPE;
+static const struct ia_css_preview_settings preview = IA_CSS_DEFAULT_PREVIEW_SETTINGS;
+static const struct ia_css_capture_settings capture = IA_CSS_DEFAULT_CAPTURE_SETTINGS;
+static const struct ia_css_video_settings video = IA_CSS_DEFAULT_VIDEO_SETTINGS;
+static const struct ia_css_yuvpp_settings yuvpp = IA_CSS_DEFAULT_YUVPP_SETTINGS;
+
+static int
 init_pipe_defaults(enum ia_css_pipe_mode mode,
 		   struct ia_css_pipe *pipe,
 		   bool copy_pipe) {
+
 	if (!pipe)
 	{
 		IA_CSS_ERROR("NULL pipe parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	/* Initialize pipe to pre-defined defaults */
-	*pipe = IA_CSS_DEFAULT_PIPE;
+	memcpy(pipe, &default_pipe, sizeof(default_pipe));
 
 	/* TODO: JB should not be needed, but temporary backward reference */
 	switch (mode)
 	{
 	case IA_CSS_PIPE_MODE_PREVIEW:
 		pipe->mode = IA_CSS_PIPE_ID_PREVIEW;
-		pipe->pipe_settings.preview = IA_CSS_DEFAULT_PREVIEW_SETTINGS;
+		memcpy(&pipe->pipe_settings.preview, &preview, sizeof(preview));
 		break;
 	case IA_CSS_PIPE_MODE_CAPTURE:
 		if (copy_pipe) {
@@ -2286,11 +2257,11 @@ init_pipe_defaults(enum ia_css_pipe_mode mode,
 		} else {
 			pipe->mode = IA_CSS_PIPE_ID_CAPTURE;
 		}
-		pipe->pipe_settings.capture = IA_CSS_DEFAULT_CAPTURE_SETTINGS;
+		memcpy(&pipe->pipe_settings.capture, &capture, sizeof(capture));
 		break;
 	case IA_CSS_PIPE_MODE_VIDEO:
 		pipe->mode = IA_CSS_PIPE_ID_VIDEO;
-		pipe->pipe_settings.video = IA_CSS_DEFAULT_VIDEO_SETTINGS;
+		memcpy(&pipe->pipe_settings.video, &video, sizeof(video));
 		break;
 	case IA_CSS_PIPE_MODE_ACC:
 		pipe->mode = IA_CSS_PIPE_ID_ACC;
@@ -2300,13 +2271,13 @@ init_pipe_defaults(enum ia_css_pipe_mode mode,
 		break;
 	case IA_CSS_PIPE_MODE_YUVPP:
 		pipe->mode = IA_CSS_PIPE_ID_YUVPP;
-		pipe->pipe_settings.yuvpp = IA_CSS_DEFAULT_YUVPP_SETTINGS;
+		memcpy(&pipe->pipe_settings.yuvpp, &yuvpp, sizeof(yuvpp));
 		break;
 	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -2320,7 +2291,7 @@ pipe_global_init(void)
 	}
 }
 
-static enum ia_css_err
+static int
 pipe_generate_pipe_num(const struct ia_css_pipe *pipe,
 		       unsigned int *pipe_number) {
 	const u8 INVALID_PIPE_NUM = (uint8_t)~(0);
@@ -2330,7 +2301,7 @@ pipe_generate_pipe_num(const struct ia_css_pipe *pipe,
 	if (!pipe)
 	{
 		IA_CSS_ERROR("NULL pipe parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	/* Assign a new pipe_num .... search for empty place */
@@ -2347,7 +2318,7 @@ pipe_generate_pipe_num(const struct ia_css_pipe *pipe,
 	{
 		/* Max number of pipes already allocated */
 		IA_CSS_ERROR("Max number of pipes already created");
-		return IA_CSS_ERR_RESOURCE_EXHAUSTED;
+		return -ENOSPC;
 	}
 
 	my_css.pipe_counter++;
@@ -2355,7 +2326,7 @@ pipe_generate_pipe_num(const struct ia_css_pipe *pipe,
 	IA_CSS_LOG("pipe_num (%d)", pipe_num);
 
 	*pipe_number = pipe_num;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -2367,39 +2338,39 @@ pipe_release_pipe_num(unsigned int pipe_num)
 			    "pipe_release_pipe_num (%d)\n", pipe_num);
 }
 
-static enum ia_css_err
+static int
 create_pipe(enum ia_css_pipe_mode mode,
 	    struct ia_css_pipe **pipe,
 	    bool copy_pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipe *me;
 
 	if (!pipe)
 	{
 		IA_CSS_ERROR("NULL pipe parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	me = kmalloc(sizeof(*me), GFP_KERNEL);
 	if (!me)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	err = init_pipe_defaults(mode, me, copy_pipe);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		kfree(me);
 		return err;
 	}
 
 	err = pipe_generate_pipe_num(me, &me->pipe_num);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		kfree(me);
 		return err;
 	}
 
 	*pipe = me;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 struct ia_css_pipe *
@@ -2438,23 +2409,23 @@ static void sh_css_pipe_free_acc_binaries(
 	}
 }
 
-enum ia_css_err
+int
 ia_css_pipe_destroy(struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER("pipe = %p", pipe);
 
 	if (!pipe)
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (pipe->stream)
 	{
 		IA_CSS_LOG("ia_css_stream_destroy not called!");
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	switch (pipe->config.mode)
@@ -2574,10 +2545,10 @@ ia_css_uninit(void)
 }
 
 #if defined(HAS_IRQ_MAP_VERSION_2)
-enum ia_css_err ia_css_irq_translate(
+int ia_css_irq_translate(
     unsigned int *irq_infos)
 {
-	virq_id_t	irq;
+	enum virq_id	irq;
 	enum hrt_isp_css_irq_status status = hrt_isp_css_irq_status_more_irqs;
 	unsigned int infos = 0;
 
@@ -2589,7 +2560,7 @@ enum ia_css_err ia_css_irq_translate(
 	while (status == hrt_isp_css_irq_status_more_irqs) {
 		status = virq_get_channel_id(&irq);
 		if (status == hrt_isp_css_irq_status_error)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
 
 #if WITH_PC_MONITORING
 		sh_css_print("PC_MONITORING: %s() irq = %d, sh_binary_running set to 0\n",
@@ -2643,14 +2614,14 @@ enum ia_css_err ia_css_irq_translate(
 			    "ia_css_irq_translate() leave: irq_infos=%u\n",
 			    infos);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err ia_css_irq_enable(
+int ia_css_irq_enable(
     enum ia_css_irq_info info,
     bool enable)
 {
-	virq_id_t	irq = N_virq_id;
+	enum virq_id	irq = N_virq_id;
 
 	IA_CSS_ENTER("info=%d, enable=%d", info, enable);
 
@@ -2674,7 +2645,7 @@ enum ia_css_err ia_css_irq_enable(
 	case IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR:
 	case IA_CSS_IRQ_INFO_IF_ERROR:
 		/* Just ignore those unused IRQs without printing errors */
-		return IA_CSS_SUCCESS;
+		return 0;
 #endif
 	case IA_CSS_IRQ_INFO_DMA_ERROR:
 		irq = virq_dma;
@@ -2686,14 +2657,14 @@ enum ia_css_err ia_css_irq_enable(
 		irq = virq_sw_pin_1;
 		break;
 	default:
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	cnd_virq_enable_channel(irq, enable);
 
-	IA_CSS_LEAVE_ERR(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR(0);
+	return 0;
 }
 
 #else
@@ -2715,14 +2686,14 @@ sh_css_get_sw_interrupt_value(unsigned int irq)
 
 /* configure and load the copy binary, the next binary is used to
    determine whether the copy binary needs to do left padding. */
-static enum ia_css_err load_copy_binary(
+static int load_copy_binary(
     struct ia_css_pipe *pipe,
     struct ia_css_binary *copy_binary,
     struct ia_css_binary *next_binary)
 {
 	struct ia_css_frame_info copy_out_info, copy_in_info, copy_vf_info;
 	unsigned int left_padding;
-	enum ia_css_err err;
+	int err;
 	struct ia_css_binary_descr copy_descr;
 
 	/* next_binary can be NULL */
@@ -2745,16 +2716,16 @@ static enum ia_css_err load_copy_binary(
 					&copy_in_info, &copy_out_info,
 					(next_binary) ? NULL : NULL/*TODO: &copy_vf_info*/);
 	err = ia_css_binary_find(&copy_descr, copy_binary);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	copy_binary->left_padding = left_padding;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 alloc_continuous_frames(
     struct ia_css_pipe *pipe, bool init_time) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame_info ref_info;
 	enum ia_css_pipe_id pipe_id;
 	bool continuous;
@@ -2766,8 +2737,8 @@ alloc_continuous_frames(
 
 	if ((!pipe) || (!pipe->stream))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	pipe_id = pipe->mode;
@@ -2794,8 +2765,8 @@ alloc_continuous_frames(
 	} else
 	{
 		/* should not happen */
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 #if defined(USE_INPUT_SYSTEM_VERSION_2401)
@@ -2834,8 +2805,8 @@ alloc_continuous_frames(
 	} else
 	{
 		/* should not happen */
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (init_time)
@@ -2860,7 +2831,7 @@ alloc_continuous_frames(
 			err = ia_css_frame_allocate_from_info(
 				  &pipe->continuous_frames[i],
 				  &ref_info);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -2869,18 +2840,18 @@ alloc_continuous_frames(
 						       &pipe->stream->info.metadata_info);
 		}
 	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
-enum ia_css_err
+int
 ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream) {
 	if (!stream)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	return alloc_continuous_frames(stream->continuous_pipe, false);
 }
 
-static enum ia_css_err
+static int
 load_preview_binaries(struct ia_css_pipe *pipe) {
 	struct ia_css_frame_info prev_in_info,
 		prev_bds_out_info,
@@ -2888,7 +2859,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 		prev_vf_info;
 	struct ia_css_binary_descr preview_descr;
 	bool online;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool continuous, need_vf_pp = false;
 	bool need_isp_copy_binary = false;
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
@@ -2910,13 +2881,13 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 #endif
 
 	if (mycs->preview_binary.info)
-		return IA_CSS_SUCCESS;
+		return 0;
 
 	err = ia_css_util_check_input(&pipe->stream->config, false, false);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	err = ia_css_frame_check_info(pipe_out_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	/* Note: the current selection of vf_pp binary and
@@ -2960,13 +2931,13 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 	    &prev_bds_out_info,
 	    &prev_out_info,
 	    &prev_vf_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	err = ia_css_binary_find(&preview_descr, &mycs->preview_binary);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		/* The delay latency determines the number of invalid frames after
 		* a stream is started. */
 		pipe->num_invalid_frames = pipe->dvs_frame_delay;
@@ -3004,11 +2975,11 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 		    &prev_bds_out_info,
 		    &prev_out_info,
 		    &prev_vf_info);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 		err = ia_css_binary_find(&preview_descr,
 					 &mycs->preview_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -3022,7 +2993,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 						pipe_out_info);
 		err = ia_css_binary_find(&vf_pp_descr,
 					 &mycs->vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -3037,7 +3008,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 	 * where the driver chooses for memory based input frames. In these cases, a copy binary (which typical
 	 * copies sensor data to DDR) does not have much use.
 	 */
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		need_isp_copy_binary = !online && !continuous;
 	else
 		need_isp_copy_binary = !online && !continuous && !(pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY);
@@ -3049,7 +3020,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 		err = load_copy_binary(pipe,
 				       &mycs->copy_binary,
 				       &mycs->preview_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -3059,7 +3030,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) {
 		pipe->shading_table = NULL;
 	}
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -3068,21 +3039,21 @@ ia_css_binary_unload(struct ia_css_binary *binary)
 	ia_css_binary_destroy_isp_parameters(binary);
 }
 
-static enum ia_css_err
+static int
 unload_preview_binaries(struct ia_css_pipe *pipe) {
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 
 	if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	ia_css_binary_unload(&pipe->pipe_settings.preview.copy_binary);
 	ia_css_binary_unload(&pipe->pipe_settings.preview.preview_binary);
 	ia_css_binary_unload(&pipe->pipe_settings.preview.vf_pp_binary);
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
 static const struct ia_css_fw_info *last_output_firmware(
@@ -3101,7 +3072,7 @@ static const struct ia_css_fw_info *last_output_firmware(
 	return last_fw;
 }
 
-static enum ia_css_err add_firmwares(
+static int add_firmwares(
     struct ia_css_pipeline *me,
     struct ia_css_binary *binary,
     const struct ia_css_fw_info *fw,
@@ -3113,7 +3084,7 @@ static enum ia_css_err add_firmwares(
     struct ia_css_pipeline_stage **my_stage,
     struct ia_css_pipeline_stage **vf_stage)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipeline_stage *extra_stage = NULL;
 	struct ia_css_pipeline_stage_desc stage_desc;
 
@@ -3140,7 +3111,7 @@ static enum ia_css_err add_firmwares(
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc,
 			&extra_stage);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 		if (fw->info.isp.sp.enable.output != 0)
 			in_frame = extra_stage->args.out_frame[0];
@@ -3153,7 +3124,7 @@ static enum ia_css_err add_firmwares(
 	return err;
 }
 
-static enum ia_css_err add_vf_pp_stage(
+static int add_vf_pp_stage(
     struct ia_css_pipe *pipe,
     struct ia_css_frame *in_frame,
     struct ia_css_frame *out_frame,
@@ -3162,20 +3133,20 @@ static enum ia_css_err add_vf_pp_stage(
 {
 	struct ia_css_pipeline *me = NULL;
 	const struct ia_css_fw_info *last_fw = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
 	struct ia_css_pipeline_stage_desc stage_desc;
 
 	/* out_frame can be NULL ??? */
 
 	if (!pipe)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	if (!in_frame)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	if (!vf_pp_binary)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	if (!vf_pp_stage)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	ia_css_pipe_util_create_output_frames(out_frames);
 	me = &pipe->pipeline;
@@ -3197,7 +3168,7 @@ static enum ia_css_err add_vf_pp_stage(
 							   out_frames, in_frame, NULL);
 		}
 		err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, vf_pp_stage);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 		in_frame = (*vf_pp_stage)->args.out_frame[0];
 	}
@@ -3208,7 +3179,7 @@ static enum ia_css_err add_vf_pp_stage(
 	return err;
 }
 
-static enum ia_css_err add_yuv_scaler_stage(
+static int add_yuv_scaler_stage(
     struct ia_css_pipe *pipe,
     struct ia_css_pipeline *me,
     struct ia_css_frame *in_frame,
@@ -3218,7 +3189,7 @@ static enum ia_css_err add_yuv_scaler_stage(
     struct ia_css_pipeline_stage **pre_vf_pp_stage)
 {
 	const struct ia_css_fw_info *last_fw;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame *vf_frame = NULL;
 	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
 	struct ia_css_pipeline_stage_desc stage_desc;
@@ -3250,7 +3221,7 @@ static enum ia_css_err add_yuv_scaler_stage(
 	err = ia_css_pipeline_create_and_add_stage(me,
 		&stage_desc,
 		pre_vf_pp_stage);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	in_frame = (*pre_vf_pp_stage)->args.out_frame[0];
 
@@ -3267,7 +3238,7 @@ static enum ia_css_err add_yuv_scaler_stage(
 	return err;
 }
 
-static enum ia_css_err add_capture_pp_stage(
+static int add_capture_pp_stage(
     struct ia_css_pipe *pipe,
     struct ia_css_pipeline *me,
     struct ia_css_frame *in_frame,
@@ -3276,7 +3247,7 @@ static enum ia_css_err add_capture_pp_stage(
     struct ia_css_pipeline_stage **capture_pp_stage)
 {
 	const struct ia_css_fw_info *last_fw = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame *vf_frame = NULL;
 	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
 	struct ia_css_pipeline_stage_desc stage_desc;
@@ -3296,7 +3267,7 @@ static enum ia_css_err add_capture_pp_stage(
 	last_fw = last_output_firmware(pipe->output_stage);
 	err = ia_css_frame_allocate_from_info(&vf_frame,
 					      &capture_pp_binary->vf_frame_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	if (last_fw)	{
 		ia_css_pipe_util_set_output_frames(out_frames, 0, NULL);
@@ -3310,7 +3281,7 @@ static enum ia_css_err add_capture_pp_stage(
 	err = ia_css_pipeline_create_and_add_stage(me,
 		&stage_desc,
 		capture_pp_stage);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	err = add_firmwares(me, capture_pp_binary, pipe->output_stage, last_fw,
 			    IA_CSS_BINARY_MODE_CAPTURE_PP,
@@ -3346,10 +3317,10 @@ static void sh_css_setup_queues(void)
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_setup_queues() leave:\n");
 }
 
-static enum ia_css_err
+static int
 init_vf_frameinfo_defaults(struct ia_css_pipe *pipe,
 			   struct ia_css_frame *vf_frame, unsigned int idx) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int thread_id;
 	enum sh_css_queue_id queue_id;
 
@@ -3511,11 +3482,11 @@ ia_css_get_crop_offsets(
 }
 #endif
 
-static enum ia_css_err
+static int
 init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe,
 				  struct ia_css_frame *frame, enum ia_css_frame_format format) {
 	struct ia_css_frame *in_frame;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int thread_id;
 	enum sh_css_queue_id queue_id;
 
@@ -3552,10 +3523,10 @@ init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe,
 	return err;
 }
 
-static enum ia_css_err
+static int
 init_out_frameinfo_defaults(struct ia_css_pipe *pipe,
 			    struct ia_css_frame *out_frame, unsigned int idx) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int thread_id;
 	enum sh_css_queue_id queue_id;
 
@@ -3574,7 +3545,7 @@ init_out_frameinfo_defaults(struct ia_css_pipe *pipe,
 }
 
 /* Create stages for video pipe */
-static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
+static int create_host_video_pipeline(struct ia_css_pipe *pipe)
 {
 	struct ia_css_pipeline_stage_desc stage_desc;
 	struct ia_css_binary *copy_binary, *video_binary,
@@ -3588,7 +3559,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 	struct ia_css_frame *out_frame;
 	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
 	struct ia_css_frame *vf_frame = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool need_copy   = false;
 	bool need_vf_pp  = false;
 	bool need_yuv_pp = false;
@@ -3600,8 +3571,8 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	ia_css_pipe_util_create_output_frames(out_frames);
 	out_frame = &pipe->out_frame_struct;
@@ -3629,20 +3600,20 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 		in_frame = &pipe->in_frame_struct;
 		err = init_in_frameinfo_memory_defaults(pipe, in_frame,
 							IA_CSS_FRAME_FORMAT_RAW);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
 	out_frame->data = 0;
 	err = init_out_frameinfo_defaults(pipe, out_frame, 0);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
 		vf_frame = &pipe->vf_frame_struct;
 		vf_frame->data = 0;
 		err = init_vf_frameinfo_defaults(pipe, vf_frame, 0);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
@@ -3666,7 +3637,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc,
 			&copy_stage);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 		in_frame = me->stages->args.out_frame[0];
 	} else if (pipe->stream->config.continuous) {
@@ -3695,7 +3666,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 	err = ia_css_pipeline_create_and_add_stage(me,
 		&stage_desc,
 		&video_stage);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 	/* If we use copy iso video, the input must be yuv iso raw */
@@ -3711,16 +3682,13 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 		in_frame = video_stage->args.out_vf_frame;
 		err = add_vf_pp_stage(pipe, in_frame, vf_frame, vf_pp_binary,
 				      &vf_pp_stage);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 	if (video_stage) {
 		int frm;
-#ifndef ISP2401
-		for (frm = 0; frm < NUM_TNR_FRAMES; frm++) {
-#else
+
 		for (frm = 0; frm < NUM_TNR_FRAMES; frm++) {
-#endif
 			video_stage->args.tnr_frames[frm] =
 			    pipe->pipe_settings.video.tnr_frames[frm];
 		}
@@ -3742,7 +3710,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 			/* In/Out Frame mapping to support output frame extension.*/
 			out = video_stage->args.out_frame[0];
 			err = ia_css_frame_allocate_from_info(&in, &pipe->output_info[0]);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 			video_stage->args.out_frame[0] = in;
 		}
@@ -3751,7 +3719,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 				    last_output_firmware(pipe->output_stage),
 				    IA_CSS_BINARY_MODE_VIDEO,
 				    in, out, NULL, &video_stage, NULL);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
@@ -3770,7 +3738,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe)
 						   &yuv_scaler_binary[i],
 						   &yuv_scaler_stage);
 
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -3789,17 +3757,17 @@ ERR:
 	return err;
 }
 
-static enum ia_css_err
+static int
 create_host_acc_pipeline(struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	const struct ia_css_fw_info *fw;
 	unsigned int i;
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if ((!pipe) || (!pipe->stream))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	pipe->pipeline.num_execs = pipe->config.acc_num_execs;
@@ -3811,7 +3779,7 @@ create_host_acc_pipeline(struct ia_css_pipe *pipe) {
 	for (i = 0; fw; fw = fw->next)
 	{
 		err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
@@ -3820,7 +3788,7 @@ create_host_acc_pipeline(struct ia_css_pipe *pipe) {
 		struct ia_css_fw_info *fw = pipe->config.acc_stages[i];
 
 		err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
@@ -3832,7 +3800,7 @@ ERR:
 }
 
 /* Create stages for preview */
-static enum ia_css_err
+static int
 create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 	struct ia_css_pipeline_stage *copy_stage = NULL;
 	struct ia_css_pipeline_stage *preview_stage = NULL;
@@ -3841,7 +3809,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 	struct ia_css_pipeline *me = NULL;
 	struct ia_css_binary *copy_binary, *preview_binary, *vf_pp_binary = NULL;
 	struct ia_css_frame *in_frame = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame *out_frame;
 	struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
 	bool need_in_frameinfo_memory = false;
@@ -3855,8 +3823,8 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	ia_css_pipe_util_create_output_frames(out_frames);
@@ -3886,7 +3854,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 	{
 		err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame,
 							IA_CSS_FRAME_FORMAT_RAW);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 
 		in_frame = &me->in_frame;
@@ -3896,7 +3864,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 	}
 
 	err = init_out_frameinfo_defaults(pipe, &me->out_frame[0], 0);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 	out_frame = &me->out_frame[0];
 
@@ -3913,7 +3881,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc,
 			&copy_stage);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 		in_frame = me->stages->args.out_frame[0];
 #ifndef ISP2401
@@ -3949,7 +3917,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 	err = ia_css_pipeline_create_and_add_stage(me,
 		&stage_desc,
 		&preview_stage);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 	/* If we use copy iso preview, the input must be yuv iso raw */
 	preview_stage->args.copy_vf =
@@ -3969,7 +3937,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) {
 			in_frame = preview_stage->args.out_frame[0];
 		err = add_vf_pp_stage(pipe, in_frame, out_frame, vf_pp_binary,
 				      &vf_pp_stage);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	}
 
@@ -4001,11 +3969,11 @@ static void send_raw_frames(struct ia_css_pipe *pipe)
 	return;
 }
 
-static enum ia_css_err
+static int
 preview_start(struct ia_css_pipe *pipe) {
 	struct ia_css_pipeline *me;
 	struct ia_css_binary *copy_binary, *preview_binary, *vf_pp_binary = NULL;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipe *copy_pipe, *capture_pipe;
 	struct ia_css_pipe *acc_pipe;
 	enum sh_css_pipe_config_override copy_ovrd;
@@ -4016,8 +3984,8 @@ preview_start(struct ia_css_pipe *pipe) {
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	me = &pipe->pipeline;
@@ -4038,8 +4006,8 @@ preview_start(struct ia_css_pipe *pipe) {
 #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
 	/* multi stream video needs mipi buffers */
 	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS) {
-	    	IA_CSS_LEAVE_ERR_PRIVATE(err);
+	if (err) {
+		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
 #endif
@@ -4059,7 +4027,7 @@ preview_start(struct ia_css_pipe *pipe) {
 		}
 	}
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		coord = &pipe->config.internal_frame_origin_bqs_on_sctbl;
 		params = pipe->stream->isp_params_configs;
 	}
@@ -4137,10 +4105,10 @@ preview_start(struct ia_css_pipe *pipe) {
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 			   const struct ia_css_buffer *buffer) {
-	enum ia_css_err return_err = IA_CSS_SUCCESS;
+	int return_err = 0;
 	unsigned int thread_id;
 	enum sh_css_queue_id queue_id;
 	struct ia_css_pipeline *pipeline;
@@ -4156,8 +4124,8 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 
 	if ((!pipe) || (!buffer))
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	buf_type = buffer->type;
@@ -4177,7 +4145,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 			}
 		}
 		if (!found_pipe)
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			return -EINVAL;
 	}
 	if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)
 	{
@@ -4192,7 +4160,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 			}
 		}
 		if (!found_pipe)
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			return -EINVAL;
 	}
 #endif
 	pipe_id = pipe->mode;
@@ -4205,36 +4173,36 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 	    (buf_type >= IA_CSS_NUM_DYNAMIC_BUFFER_TYPE) ||
 	    (pipe_id >= IA_CSS_PIPE_ID_NUM))
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
 	if (!ret_err)
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id);
 	if (!ret_err)
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES))
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (!sh_css_sp_is_running())
 	{
 		IA_CSS_LOG("SP is not running!");
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
+		IA_CSS_LEAVE_ERR(-EBUSY);
 		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 
 	pipeline = &pipe->pipeline;
@@ -4251,24 +4219,24 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 	if (buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS)
 	{
 		if (!buffer->data.stats_3a) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			IA_CSS_LEAVE_ERR(-EINVAL);
+			return -EINVAL;
 		}
 		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_3a);
 		ddr_buffer.payload.s3a = *buffer->data.stats_3a;
 	} else if (buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS)
 	{
 		if (!buffer->data.stats_dvs) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			IA_CSS_LEAVE_ERR(-EINVAL);
+			return -EINVAL;
 		}
 		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_dvs);
 		ddr_buffer.payload.dis = *buffer->data.stats_dvs;
 	} else if (buf_type == IA_CSS_BUFFER_TYPE_METADATA)
 	{
 		if (!buffer->data.metadata) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			IA_CSS_LEAVE_ERR(-EINVAL);
+			return -EINVAL;
 		}
 		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.metadata);
 		ddr_buffer.payload.metadata = *buffer->data.metadata;
@@ -4279,8 +4247,8 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 		   || (buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME))
 	{
 		if (!buffer->data.frame) {
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			IA_CSS_LEAVE_ERR(-EINVAL);
+			return -EINVAL;
 		}
 		ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.frame);
 		ddr_buffer.payload.frame.frame_data = buffer->data.frame->data;
@@ -4294,7 +4262,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 		return_err = set_config_on_frame_enqueue(
 				 &buffer->data.frame->info,
 				 &ddr_buffer.payload.frame);
-		if (return_err != IA_CSS_SUCCESS) {
+		if (return_err) {
 			IA_CSS_LEAVE_ERR(return_err);
 			return return_err;
 		}
@@ -4314,11 +4282,11 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 
 	if ((!h_vbuf) || (h_vbuf->vptr == 0x0))
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
-	mmgr_store(h_vbuf->vptr,
+	hmm_store(h_vbuf->vptr,
 		   (void *)(&ddr_buffer),
 		   sizeof(struct sh_css_hmm_buffer));
 	if ((buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS)
@@ -4328,8 +4296,8 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 		if (!pipeline) {
 			ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &h_vbuf);
 			IA_CSS_LOG("pipeline is empty!");
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			IA_CSS_LEAVE_ERR(-EINVAL);
+			return -EINVAL;
 		}
 
 		for (stage = pipeline->stages; stage; stage = stage->next) {
@@ -4353,7 +4321,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 							queue_id,
 							(uint32_t)h_vbuf->vptr);
 #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-		if ((return_err == IA_CSS_SUCCESS) &&
+		if (!(return_err) &&
 		    (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)) {
 			IA_CSS_LOG("pfp: enqueued OF %d to q %d thread %d",
 				   ddr_buffer.payload.frame.frame_data,
@@ -4362,14 +4330,14 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 #endif
 	}
 
-	if (return_err == IA_CSS_SUCCESS)
+	if (!return_err)
 	{
 		if (sh_css_hmm_buffer_record_acquire(
 			h_vbuf, buf_type,
 			HOST_ADDRESS(ddr_buffer.kernel_ptr))) {
 			IA_CSS_LOG("send vbuf=%p", h_vbuf);
 		} else {
-			return_err = IA_CSS_ERR_INTERNAL_ERROR;
+			return_err = -EINVAL;
 			IA_CSS_ERROR("hmm_buffer_record[]: no available slots\n");
 		}
 	}
@@ -4378,13 +4346,13 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 	 * Tell the SP which queues are not empty,
 	 * by sending the software event.
 	 */
-	if (return_err == IA_CSS_SUCCESS)
+	if (!return_err)
 	{
 		if (!sh_css_sp_is_running()) {
 			/* SP is not running. The queues are not valid */
 			IA_CSS_LOG("SP is not running!");
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-			return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+			IA_CSS_LEAVE_ERR(-EBUSY);
+			return -EBUSY;
 		}
 		return_err = ia_css_bufq_enqueue_psys_event(
 				 IA_CSS_PSYS_SW_EVENT_BUFFER_ENQUEUED,
@@ -4405,12 +4373,12 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe,
 /*
  * TODO: Free up the hmm memory space.
 	 */
-enum ia_css_err
+int
 ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 			   struct ia_css_buffer *buffer) {
-	enum ia_css_err return_err;
+	int return_err;
 	enum sh_css_queue_id queue_id;
-	hrt_vaddress ddr_buffer_addr = (hrt_vaddress)0;
+	ia_css_ptr ddr_buffer_addr = (ia_css_ptr)0;
 	struct sh_css_hmm_buffer ddr_buffer;
 	enum ia_css_buffer_type buf_type;
 	enum ia_css_pipe_id pipe_id;
@@ -4422,8 +4390,8 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 
 	if ((!pipe) || (!buffer))
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	pipe_id = pipe->mode;
@@ -4437,35 +4405,35 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 	ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
 	if (!ret_err)
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id);
 	if (!ret_err)
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES))
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (!sh_css_sp_is_running())
 	{
 		IA_CSS_LOG("SP is not running!");
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
+		IA_CSS_LEAVE_ERR(-EBUSY);
 		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 
 	return_err = ia_css_bufq_dequeue_buffer(queue_id,
 						(uint32_t *)&ddr_buffer_addr);
 
-	if (return_err == IA_CSS_SUCCESS)
+	if (!return_err)
 	{
 		struct ia_css_frame *frame;
 		struct sh_css_hmm_buffer_record *hmm_buffer_record = NULL;
@@ -4477,7 +4445,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 		    ddr_buffer_addr, buf_type);
 		if (hmm_buffer_record) {
 			/* valid hmm_buffer_record found. Save the kernel_ptr
-			 * for validation after performing mmgr_load.  The
+			 * for validation after performing hmm_load.  The
 			 * vbuf handle and buffer_record can be released.
 			 */
 			kernel_ptr = hmm_buffer_record->kernel_ptr;
@@ -4486,11 +4454,11 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 		} else {
 			IA_CSS_ERROR("hmm_buffer_record not found (0x%x) buf_type(%d)",
 				     ddr_buffer_addr, buf_type);
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			IA_CSS_LEAVE_ERR(-EINVAL);
+			return -EINVAL;
 		}
 
-		mmgr_load(ddr_buffer_addr,
+		hmm_load(ddr_buffer_addr,
 			  &ddr_buffer,
 			  sizeof(struct sh_css_hmm_buffer));
 
@@ -4503,8 +4471,8 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 			IA_CSS_ERROR("expected: (0x%llx)", (u64)kernel_ptr);
 			IA_CSS_ERROR("actual: (0x%llx)", (u64)HOST_ADDRESS(ddr_buffer.kernel_ptr));
 			IA_CSS_ERROR("buf_type: %d\n", buf_type);
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			IA_CSS_LEAVE_ERR(-EINVAL);
+			return -EINVAL;
 		}
 
 		if (ddr_buffer.kernel_ptr != 0) {
@@ -4529,7 +4497,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 					 * for 2401 it is done in ia_css_stream_destroy call
 					 */
 					return_err = free_mipi_frames(pipe);
-					if (return_err != IA_CSS_SUCCESS) {
+					if (return_err) {
 						IA_CSS_LOG("free_mipi_frames() failed");
 						IA_CSS_LEAVE_ERR(return_err);
 						return return_err;
@@ -4537,6 +4505,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 #endif
 					pipe->stop_requested = false;
 				}
+				/* fall through */
 			case IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME:
 			case IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME:
 				frame = (struct ia_css_frame *)HOST_ADDRESS(ddr_buffer.kernel_ptr);
@@ -4597,7 +4566,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 				buffer->data.metadata->exp_id = ddr_buffer.payload.metadata.exp_id;
 				break;
 			default:
-				return_err = IA_CSS_ERR_INTERNAL_ERROR;
+				return_err = -EINVAL;
 				break;
 			}
 		}
@@ -4607,13 +4576,13 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe,
 	 * Tell the SP which queues are not full,
 	 * by sending the software event.
 	 */
-	if (return_err == IA_CSS_SUCCESS)
+	if (!return_err)
 	{
 		if (!sh_css_sp_is_running()) {
 			IA_CSS_LOG("SP is not running!");
-			IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
+			IA_CSS_LEAVE_ERR(-EBUSY);
 			/* SP is not running. The queues are not valid */
-			return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+			return -EBUSY;
 		}
 		ia_css_bufq_enqueue_psys_event(
 		    IA_CSS_PSYS_SW_EVENT_BUFFER_DEQUEUED,
@@ -4656,16 +4625,16 @@ static enum ia_css_event_type convert_event_sp_to_host_domain[] = {
 	0,					/* error if sp passes  SH_CSS_SP_EVENT_NR_OF_TYPES as a valid event. */
 };
 
-enum ia_css_err
+int
 ia_css_dequeue_event(struct ia_css_event *event) {
 	return ia_css_dequeue_psys_event(event);
 }
 
-enum ia_css_err
+int
 ia_css_dequeue_psys_event(struct ia_css_event *event) {
 	enum ia_css_pipe_id pipe_id = 0;
 	u8 payload[4] = {0, 0, 0, 0};
-	enum ia_css_err ret_err;
+	int ret_err;
 
 	/*TODO:
 	 * a) use generic decoding function , same as the one used by sp.
@@ -4675,17 +4644,17 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) {
 	 * to avoid flooding the logs when the host application
 	 * uses polling. */
 	if (!event)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if (!sh_css_sp_is_running())
 	{
 		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 
 	/* dequeue the event (if any) from the psys event queue */
 	ret_err = ia_css_bufq_dequeue_psys_event(payload);
-	if (ret_err != IA_CSS_SUCCESS)
+	if (ret_err)
 		return ret_err;
 
 	IA_CSS_LOG("event dequeued from psys event queue");
@@ -4717,7 +4686,7 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) {
 		event->timer_code = payload[2];
 		payload[0] = payload[1] = payload[2] = payload[3] = 0;
 		ret_err = ia_css_bufq_dequeue_psys_event(payload);
-		if (ret_err != IA_CSS_SUCCESS) {
+		if (ret_err) {
 			/* no 2nd event ??? an error */
 			/* Putting IA_CSS_ERROR is resulting in failures in
 			 * Merrifield smoke testing  */
@@ -4771,7 +4740,7 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) {
 		pipe_id = (enum ia_css_pipe_id)payload[2];
 		/* Check to see if pipe still exists */
 		if (!event->pipe)
-			return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+			return -EBUSY;
 
 		if (event->type == IA_CSS_EVENT_TYPE_FRAME_TAGGED) {
 			/* find the capture pipe that goes with this */
@@ -4796,7 +4765,7 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) {
 				      &event->pipe->pipeline,
 				      stage_num,
 				      &event->fw_handle);
-			if (ret_err != IA_CSS_SUCCESS) {
+			if (ret_err) {
 				IA_CSS_ERROR("Invalid stage num received for ACC event. stage_num:%u",
 					     stage_num);
 				return ret_err;
@@ -4809,28 +4778,28 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) {
 	else
 		IA_CSS_LEAVE("event_id=%d", event->type);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
+int
 ia_css_dequeue_isys_event(struct ia_css_event *event) {
 	u8 payload[4] = {0, 0, 0, 0};
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	/* We skip the IA_CSS_ENTER logging call
 	 * to avoid flooding the logs when the host application
 	 * uses polling. */
 	if (!event)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if (!sh_css_sp_is_running())
 	{
 		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 
 	err = ia_css_bufq_dequeue_isys_event(payload);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	IA_CSS_LOG("event dequeued from isys event queue");
@@ -4859,9 +4828,9 @@ acc_start(struct ia_css_pipe *pipe)
 		   pipe->stream->config.mode);
 }
 
-static enum ia_css_err
+static int
 sh_css_pipe_start(struct ia_css_stream *stream) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	struct ia_css_pipe *pipe;
 	enum ia_css_pipe_id pipe_id;
@@ -4871,14 +4840,14 @@ sh_css_pipe_start(struct ia_css_stream *stream) {
 
 	if (!stream)
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 	pipe = stream->last_pipe;
 	if (!pipe)
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	pipe_id = pipe->mode;
@@ -4910,14 +4879,14 @@ sh_css_pipe_start(struct ia_css_stream *stream) {
 		acc_start(pipe);
 		break;
 	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	}
 	/* DH regular multi pipe - not continuous mode: start the next pipes too */
 	if (!stream->config.continuous)
 	{
 		int i;
 
-		for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err ; i++) {
+		for (i = 1; i < stream->num_pipes && 0 == err ; i++) {
 			switch (stream->pipes[i]->mode) {
 			case IA_CSS_PIPE_ID_PREVIEW:
 				stream->pipes[i]->stop_requested = false;
@@ -4940,11 +4909,11 @@ sh_css_pipe_start(struct ia_css_stream *stream) {
 				acc_start(stream->pipes[i]);
 				break;
 			default:
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
+				err = -EINVAL;
 			}
 		}
 	}
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -4960,7 +4929,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) {
 		sh_css_invalidate_params(stream);
 		err = sh_css_param_update_isp_params(pipe,
 						     stream->isp_params_configs, true, NULL);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -4972,9 +4941,9 @@ sh_css_pipe_start(struct ia_css_stream *stream) {
 
 	if (!sh_css_sp_is_running())
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
+		IA_CSS_LEAVE_ERR_PRIVATE(-EBUSY);
 		/* SP is not running. The queues are not valid */
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 	ia_css_bufq_enqueue_psys_event(IA_CSS_PSYS_SW_EVENT_START_STREAM,
 				       (uint8_t)thread_id, 0, 0);
@@ -5005,8 +4974,8 @@ sh_css_pipe_start(struct ia_css_stream *stream) {
 			copy_pipe = pipe->pipe_settings.video.copy_pipe;
 
 		if (!copy_pipe) {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+			return -EINVAL;
 		}
 		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(copy_pipe),
 						 &thread_id);
@@ -5025,8 +4994,8 @@ sh_css_pipe_start(struct ia_css_stream *stream) {
 			capture_pipe = pipe->pipe_settings.video.capture_pipe;
 
 		if (!capture_pipe) {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+			return -EINVAL;
 		}
 		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(capture_pipe),
 						 &thread_id);
@@ -5087,39 +5056,39 @@ sh_css_continuous_is_enabled(uint8_t pipe_num)
 }
 
 /* ISP2400 */
-enum ia_css_err
+int
 ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream,
 				   int *buffer_depth) {
 	if (!buffer_depth)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_max_buffer_depth() enter: void\n");
 	(void)stream;
 	*buffer_depth = NUM_CONTINUOUS_FRAMES;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
+int
 ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth) {
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_set_buffer_depth() enter: num_frames=%d\n", buffer_depth);
 	(void)stream;
 	if (buffer_depth > NUM_CONTINUOUS_FRAMES || buffer_depth < 1)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	/* ok, value allowed */
 	stream->config.target_num_cont_raw_buf = buffer_depth;
 	/* TODO: check what to regarding initialization */
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /* ISP2401 */
-enum ia_css_err
+int
 ia_css_stream_get_buffer_depth(struct ia_css_stream *stream,
 			       int *buffer_depth) {
 	if (!buffer_depth)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_buffer_depth() enter: void\n");
 	(void)stream;
 	*buffer_depth = stream->config.target_num_cont_raw_buf;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /*
@@ -5129,10 +5098,10 @@ ia_css_stream_get_buffer_depth(struct ia_css_stream *stream,
  * Refer to "Local prototypes" for more info.
  */
 /* ISP2401 */
-static enum ia_css_err
+static int
 sh_css_pipes_stop(struct ia_css_stream *stream)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipe *main_pipe;
 	enum ia_css_pipe_id main_pipe_id;
 	int i;
@@ -5141,7 +5110,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream)
 	if (!stream)
 	{
 		IA_CSS_LOG("stream does NOT exist!");
-		err = IA_CSS_ERR_INTERNAL_ERROR;
+		err = -EINVAL;
 		goto ERR;
 	}
 
@@ -5150,7 +5119,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream)
 	if (!main_pipe)
 	{
 		IA_CSS_LOG("main_pipe does NOT exist!");
-		err = IA_CSS_ERR_INTERNAL_ERROR;
+		err = -EINVAL;
 		goto ERR;
 	}
 
@@ -5183,7 +5152,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream)
 	 * the CSS driver should capture these error code and
 	 * handle it in the driver exception handling mechanism.
 	 */
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		goto ERR;
 	}
 	}
@@ -5211,7 +5180,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream)
 		assert(copy_pipe);
 		if (!copy_pipe) {
 			IA_CSS_LOG("Copy Pipe does NOT exist!");
-			err = IA_CSS_ERR_INTERNAL_ERROR;
+			err = -EINVAL;
 			goto ERR;
 		}
 
@@ -5325,7 +5294,7 @@ sh_css_get_mipi_sizes_for_check(const unsigned int port, const unsigned int idx)
 }
 #endif
 
-static enum ia_css_err sh_css_pipe_configure_output(
+static int sh_css_pipe_configure_output(
     struct ia_css_pipe *pipe,
     unsigned int width,
     unsigned int height,
@@ -5333,17 +5302,17 @@ static enum ia_css_err sh_css_pipe_configure_output(
     enum ia_css_frame_format format,
     unsigned int idx)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
-	IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, paddaed width = %d, format = %d, idx = %d",
+	IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, padded width = %d, format = %d, idx = %d",
 			     pipe, width, height, padded_width, format, idx);
 	if (!pipe) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	err = ia_css_util_check_res(width, height);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
@@ -5357,16 +5326,16 @@ static enum ia_css_err sh_css_pipe_configure_output(
 		    format,
 		    padded_width);
 	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
-static enum ia_css_err
+static int
 sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe,
 			     struct ia_css_shading_info *shading_info,
 			     struct ia_css_pipe_config *pipe_config)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_binary *binary = NULL;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
@@ -5390,17 +5359,17 @@ sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe,
 		/* When the pipe does not have a binary which has the shading
 		 * correction, this function does not need to fill the shading
 		 * information. It is not a error case, and then
-		 * this function should return IA_CSS_SUCCESS.
+		 * this function should return 0.
 		 */
 		memset(shading_info, 0, sizeof(*shading_info));
 	}
 	return err;
 }
 
-static enum ia_css_err
+static int
 sh_css_pipe_get_grid_info(struct ia_css_pipe *pipe,
 			  struct ia_css_grid_info *info) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_binary *binary = NULL;
 
 	assert(pipe);
@@ -5413,7 +5382,7 @@ sh_css_pipe_get_grid_info(struct ia_css_pipe *pipe,
 	if (binary)
 	{
 		err = ia_css_binary_3a_grid_info(binary, info, pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 	} else
 		memset(&info->s3a_grid, 0, sizeof(info->s3a_grid));
@@ -5457,7 +5426,7 @@ ERR :
  * @brief Check if a format is supported by the pipe.
  *
  */
-static enum ia_css_err
+static int
 ia_css_pipe_check_format(struct ia_css_pipe *pipe,
 			 enum ia_css_frame_format format) {
 	const enum ia_css_frame_format *supported_formats;
@@ -5470,8 +5439,8 @@ ia_css_pipe_check_format(struct ia_css_pipe *pipe,
 	if (NULL == pipe || NULL == pipe->pipe_settings.video.video_binary.info)
 	{
 		IA_CSS_ERROR("Pipe or binary info is not set");
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	supported_formats = pipe->pipe_settings.video.video_binary.info->output_formats;
@@ -5487,21 +5456,21 @@ ia_css_pipe_check_format(struct ia_css_pipe *pipe,
 	if (!found)
 	{
 		IA_CSS_ERROR("Requested format is not supported by binary");
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	} else
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
+		IA_CSS_LEAVE_ERR_PRIVATE(0);
+		return 0;
 	}
 }
 
-static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
+static int load_video_binaries(struct ia_css_pipe *pipe)
 {
 	struct ia_css_frame_info video_in_info, tnr_info,
 		       *video_vf_info, video_bds_out_info, *pipe_out_info, *pipe_vf_out_info;
 	bool online;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool continuous = pipe->stream->config.continuous;
 	unsigned int i;
 	unsigned int num_output_pins;
@@ -5520,7 +5489,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 	 * All are always reset at the same time anyway.
 	 */
 	if (mycs->video_binary.info)
-		return IA_CSS_SUCCESS;
+		return 0;
 
 	online = pipe->stream->config.online;
 	pipe_out_info = &pipe->output_info[0];
@@ -5534,19 +5503,19 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 	 * This is checked in the binary_find(), so no need to check it here
 	 */
 	err = ia_css_util_check_input(&pipe->stream->config, false, false);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	/* cannot have online video and input_mode memory */
 	if (online && pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
 		err = ia_css_util_check_vf_out_info(pipe_out_info,
 						    pipe_vf_out_info);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	} else {
 		err = ia_css_frame_check_info(pipe_out_info);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -5581,19 +5550,19 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 			  pipe_out_info,
 			  NULL,
 			  &cas_scaler_descr);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 		mycs->num_yuv_scaler = cas_scaler_descr.num_stage;
 		mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage *
 						  sizeof(struct ia_css_binary), GFP_KERNEL);
 		if (!mycs->yuv_scaler_binary) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			return err;
 		}
 		mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage
 						* sizeof(bool), GFP_KERNEL);
 		if (!mycs->is_output_stage) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			return err;
 		}
 		for (i = 0; i < cas_scaler_descr.num_stage; i++) {
@@ -5607,7 +5576,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 							     &cas_scaler_descr.vf_info[i]);
 			err = ia_css_binary_find(&yuv_scaler_descr,
 						 &mycs->yuv_scaler_binary[i]);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				kfree(mycs->is_output_stage);
 				mycs->is_output_stage = NULL;
 				return err;
@@ -5624,7 +5593,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 						       &video_descr, &video_in_info, &video_bds_out_info, &video_bin_out_info,
 						       video_vf_info,
 						       pipe->stream->config.left_padding);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 
 		/* In the case where video_vf_info is not NULL, this allows
@@ -5635,7 +5604,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 		err = ia_css_binary_find(&video_descr,
 					 &mycs->video_binary);
 
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			if (video_vf_info) {
 				/* This will do another video binary lookup later for YUV_LINE format*/
 				need_vf_pp = true;
@@ -5677,7 +5646,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 			/* restore original vf_info format */
 			ia_css_frame_info_set_format(video_vf_info,
 						     vf_info_format);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				return err;
 		}
 	}
@@ -5712,7 +5681,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 		err = load_copy_binary(pipe,
 				       &mycs->copy_binary,
 				       &mycs->video_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 #else
@@ -5739,14 +5708,14 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 
 		err = ia_css_binary_find(&vf_pp_descr,
 					 &mycs->vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 #endif
 
 	err = allocate_delay_frames(pipe);
 
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	if (mycs->video_binary.info->sp.enable.block_output) {
@@ -5755,7 +5724,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 
 		tnr_info = mycs->video_binary.out_frame_info[0];
 
-		if (atomisp_hw_is_isp2401) {
+		if (IS_ISP2401) {
 			/* Select resolution for TNR. If
 			* output_system_in_resolution(GDC_out_resolution) is
 			* being used, then select that as it will also be in resolution for
@@ -5794,14 +5763,14 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe)
 		err = ia_css_frame_allocate_from_info(
 			  &mycs->tnr_frames[i],
 			  &tnr_info);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 	IA_CSS_LEAVE_PRIVATE("");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 unload_video_binaries(struct ia_css_pipe *pipe) {
 	unsigned int i;
 
@@ -5809,8 +5778,8 @@ unload_video_binaries(struct ia_css_pipe *pipe) {
 
 	if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	ia_css_binary_unload(&pipe->pipe_settings.video.copy_binary);
 	ia_css_binary_unload(&pipe->pipe_settings.video.video_binary);
@@ -5824,14 +5793,14 @@ unload_video_binaries(struct ia_css_pipe *pipe) {
 	kfree(pipe->pipe_settings.video.yuv_scaler_binary);
 	pipe->pipe_settings.video.yuv_scaler_binary = NULL;
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
-static enum ia_css_err video_start(struct ia_css_pipe *pipe)
+static int video_start(struct ia_css_pipe *pipe)
 {
 	struct ia_css_binary *copy_binary;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipe *copy_pipe, *capture_pipe;
 	enum sh_css_pipe_config_override copy_ovrd;
 	enum ia_css_input_mode video_pipe_input_mode;
@@ -5841,8 +5810,8 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe)
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	video_pipe_input_mode = pipe->stream->config.mode;
@@ -5858,7 +5827,7 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe)
 
 #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
 	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 #endif
 
@@ -5876,7 +5845,7 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe)
 		}
 	}
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		coord = &pipe->config.internal_frame_origin_bqs_on_sctbl;
 		params = pipe->stream->isp_params_configs;
 	}
@@ -5932,7 +5901,7 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe)
 }
 
 static
-enum ia_css_err sh_css_pipe_get_viewfinder_frame_info(
+int sh_css_pipe_get_viewfinder_frame_info(
     struct ia_css_pipe *pipe,
     struct ia_css_frame_info *info,
     unsigned int idx)
@@ -5947,7 +5916,7 @@ enum ia_css_err sh_css_pipe_get_viewfinder_frame_info(
 	if (pipe->mode == IA_CSS_PIPE_ID_CAPTURE &&
 	    (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW ||
 	     pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER))
-		return IA_CSS_ERR_MODE_HAS_NO_VIEWFINDER;
+		return -EINVAL;
 	/* offline video does not generate viewfinder output */
 	*info = pipe->vf_output_info[idx];
 
@@ -5960,27 +5929,27 @@ enum ia_css_err sh_css_pipe_get_viewfinder_frame_info(
 			    info->padded_width, info->format,
 			    info->raw_bit_depth, info->raw_bayer_order);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 sh_css_pipe_configure_viewfinder(struct ia_css_pipe *pipe, unsigned int width,
 				 unsigned int height, unsigned int min_width,
 				 enum ia_css_frame_format format,
 				 unsigned int idx) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, min_width = %d, format = %d, idx = %d\n",
 			     pipe, width, height, min_width, format, idx);
 
 	if (!pipe)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	err = ia_css_util_check_res(width, height);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -5992,13 +5961,13 @@ sh_css_pipe_configure_viewfinder(struct ia_css_pipe *pipe, unsigned int width,
 		ia_css_frame_info_init(&pipe->vf_output_info[idx], width, height,
 				       format, min_width);
 	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
-static enum ia_css_err load_copy_binaries(struct ia_css_pipe *pipe)
+static int load_copy_binaries(struct ia_css_pipe *pipe)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(pipe);
 	IA_CSS_ENTER_PRIVATE("");
@@ -6006,14 +5975,14 @@ static enum ia_css_err load_copy_binaries(struct ia_css_pipe *pipe)
 	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE ||
 	       pipe->mode == IA_CSS_PIPE_ID_COPY);
 	if (pipe->pipe_settings.capture.copy_binary.info)
-		return IA_CSS_SUCCESS;
+		return 0;
 
 	err = ia_css_frame_check_info(&pipe->output_info[0]);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 	err = verify_copy_out_frame_format(pipe);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 	err = load_copy_binary(pipe,
@@ -6034,7 +6003,7 @@ static bool need_capture_pp(
 	assert(pipe);
 	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE);
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		/* ldc and capture_pp are not supported in the same pipeline */
 		if (need_capt_ldc(pipe) == true)
 			return false;
@@ -6071,13 +6040,13 @@ static bool need_capt_ldc(
 	return (pipe->extra_config.enable_dvs_6axis) ? true : false;
 }
 
-static enum ia_css_err set_num_primary_stages(unsigned int *num,
+static int set_num_primary_stages(unsigned int *num,
 	enum ia_css_pipe_version version)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	if (!num)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	switch (version) {
 	case IA_CSS_PIPE_VERSION_2_6_1:
@@ -6088,14 +6057,14 @@ static enum ia_css_err set_num_primary_stages(unsigned int *num,
 		*num = NUM_PRIMARY_STAGES;
 		break;
 	default:
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		break;
 	}
 
 	return err;
 }
 
-static enum ia_css_err load_primary_binaries(
+static int load_primary_binaries(
     struct ia_css_pipe *pipe)
 {
 	bool online = false;
@@ -6113,10 +6082,11 @@ static enum ia_css_err load_primary_binaries(
 		       *vf_pp_in_info, *pipe_out_info,
 		       *pipe_vf_out_info, *capt_pp_in_info,
 		       capt_ldc_out_info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_capture_settings *mycs;
 	unsigned int i;
 	bool need_extra_yuv_scaler = false;
+	struct ia_css_binary_descr prim_descr[MAX_NUM_PRIMARY_STAGES];
 
 	IA_CSS_ENTER_PRIVATE("");
 	assert(pipe);
@@ -6136,24 +6106,24 @@ static enum ia_css_err load_primary_binaries(
 	pipe_vf_out_info = &pipe->vf_output_info[0];
 
 	if (mycs->primary_binary[0].info)
-		return IA_CSS_SUCCESS;
+		return 0;
 
 	err = set_num_primary_stages(&mycs->num_primary_stage,
 				     pipe->config.isp_pipe_version);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
 
 	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
 		err = ia_css_util_check_vf_out_info(pipe_out_info, pipe_vf_out_info);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
 	} else {
 		err = ia_css_frame_check_info(pipe_out_info);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -6187,18 +6157,8 @@ static enum ia_css_err load_primary_binaries(
 	capt_pp_out_info.res.height /= MAX_PREFERRED_YUV_DS_PER_STEP;
 	ia_css_frame_info_set_width(&capt_pp_out_info, capt_pp_out_info.res.width, 0);
 
-	/*
-	 * WARNING: The #if def flag has been added below as a
-	 * temporary solution to solve the problem of enabling the
-	 * view finder in a single binary in a capture flow. The
-	 * vf-pp stage has been removed for Skycam in the solution
-	 * provided. The vf-pp stage should be re-introduced when
-	 * required. This should not be considered as a clean solution.
-	 * Proper investigation should be done to come up with the clean
-	 * solution.
-	 * */
 	need_extra_yuv_scaler = need_downscaling(capt_pp_out_info.res,
-				pipe_out_info->res);
+						 pipe_out_info->res);
 
 	if (need_extra_yuv_scaler) {
 		struct ia_css_cas_binary_descr cas_scaler_descr = { };
@@ -6208,7 +6168,7 @@ static enum ia_css_err load_primary_binaries(
 			  pipe_out_info,
 			  NULL,
 			  &cas_scaler_descr);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -6216,14 +6176,14 @@ static enum ia_css_err load_primary_binaries(
 		mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage *
 						  sizeof(struct ia_css_binary), GFP_KERNEL);
 		if (!mycs->yuv_scaler_binary) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
 		mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage *
 						sizeof(bool), GFP_KERNEL);
 		if (!mycs->is_output_stage) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -6238,7 +6198,7 @@ static enum ia_css_err load_primary_binaries(
 							     &cas_scaler_descr.vf_info[i]);
 			err = ia_css_binary_find(&yuv_scaler_descr,
 						 &mycs->yuv_scaler_binary[i]);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -6251,29 +6211,30 @@ static enum ia_css_err load_primary_binaries(
 
 	/* TODO Do we disable ldc for skycam */
 	need_ldc = need_capt_ldc(pipe);
-
-	if (atomisp_hw_is_isp2401 && need_ldc) {
+	if (IS_ISP2401 && need_ldc) {
 		/* ldc and capt_pp are not supported in the same pipeline */
 		struct ia_css_binary_descr capt_ldc_descr;
 
 		ia_css_pipe_get_ldc_binarydesc(pipe,
-					    &capt_ldc_descr, &prim_out_info,
-					    &capt_pp_out_info);
+					       &capt_ldc_descr, &prim_out_info,
+					       &capt_pp_out_info);
 
 		err = ia_css_binary_find(&capt_ldc_descr,
-					&mycs->capture_ldc_binary);
-		if (err != IA_CSS_SUCCESS) {
+					 &mycs->capture_ldc_binary);
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
 		need_pp = 0;
 		need_ldc = 0;
 	}
+
+	/* we build up the pipeline starting at the end */
+	/* Capture post-processing */
 	if (need_pp) {
 		struct ia_css_binary_descr capture_pp_descr;
-		struct ia_css_binary_descr prim_descr[MAX_NUM_PRIMARY_STAGES];
 
-		if (!atomisp_hw_is_isp2401)
+		if (!IS_ISP2401)
 			capt_pp_in_info = need_ldc ? &capt_ldc_out_info : &prim_out_info;
 		else
 			capt_pp_in_info = &prim_out_info;
@@ -6283,7 +6244,7 @@ static enum ia_css_err load_primary_binaries(
 							&capt_pp_out_info, &vf_info);
 		err = ia_css_binary_find(&capture_pp_descr,
 					    &mycs->capture_pp_binary);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -6297,92 +6258,92 @@ static enum ia_css_err load_primary_binaries(
 
 			err = ia_css_binary_find(&capt_ldc_descr,
 						    &mycs->capture_ldc_binary);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
-		} else {
-			prim_out_info = *pipe_out_info;
 		}
+	} else {
+		prim_out_info = *pipe_out_info;
+	}
 
-		/* Primary */
-		for (i = 0; i < mycs->num_primary_stage; i++) {
-			struct ia_css_frame_info *local_vf_info = NULL;
+	/* Primary */
+	for (i = 0; i < mycs->num_primary_stage; i++) {
+		struct ia_css_frame_info *local_vf_info = NULL;
 
-			if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] &&
-			    (i == mycs->num_primary_stage - 1))
-				local_vf_info = &vf_info;
-			ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], &prim_in_info,
-							    &prim_out_info, local_vf_info, i);
-			err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
+		if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] &&
+		    (i == mycs->num_primary_stage - 1))
+			local_vf_info = &vf_info;
+		ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], &prim_in_info,
+						    &prim_out_info, local_vf_info, i);
+		err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]);
+		if (err) {
+			IA_CSS_LEAVE_ERR_PRIVATE(err);
+			return err;
 		}
+	}
 
-		/* Viewfinder post-processing */
-		if (need_pp)
-			vf_pp_in_info = &mycs->capture_pp_binary.vf_frame_info;
-		else
-			vf_pp_in_info = &mycs->primary_binary[mycs->num_primary_stage - 1].vf_frame_info;
+	/* Viewfinder post-processing */
+	if (need_pp)
+		vf_pp_in_info = &mycs->capture_pp_binary.vf_frame_info;
+	else
+		vf_pp_in_info = &mycs->primary_binary[mycs->num_primary_stage - 1].vf_frame_info;
 
-		/*
-		 * WARNING: The #if def flag has been added below as a
-		 * temporary solution to solve the problem of enabling the
-		 * view finder in a single binary in a capture flow. The
-		 * vf-pp stage has been removed for Skycam in the solution
-		 * provided. The vf-pp stage should be re-introduced when
-		 * required. Thisshould not be considered as a clean solution.
-		 * Proper  * investigation should be done to come up with the clean
-		 * solution.
-		 * */
-		if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
-			struct ia_css_binary_descr vf_pp_descr;
-
-			ia_css_pipe_get_vfpp_binarydesc(pipe,
-							&vf_pp_descr, vf_pp_in_info, pipe_vf_out_info);
-			err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
-		}
-		err = allocate_delay_frames(pipe);
+	/*
+	    * WARNING: The #if def flag has been added below as a
+	    * temporary solution to solve the problem of enabling the
+	    * view finder in a single binary in a capture flow. The
+	    * vf-pp stage has been removed for Skycam in the solution
+	    * provided. The vf-pp stage should be re-introduced when
+	    * required. Thisshould not be considered as a clean solution.
+	    * Proper  * investigation should be done to come up with the clean
+	    * solution.
+	    * */
+	if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) {
+		struct ia_css_binary_descr vf_pp_descr;
 
-		if (err != IA_CSS_SUCCESS)
+		ia_css_pipe_get_vfpp_binarydesc(pipe,
+						&vf_pp_descr, vf_pp_in_info, pipe_vf_out_info);
+		err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary);
+		if (err) {
+			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
+		}
+	}
+	err = allocate_delay_frames(pipe);
+
+	if (err)
+		return err;
 
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
-		/* When the input system is 2401, only the Direct Sensor Mode
-		 * Offline Capture uses the ISP copy binary.
-		 */
-		need_isp_copy_binary = !online && sensor;
+	/* When the input system is 2401, only the Direct Sensor Mode
+	    * Offline Capture uses the ISP copy binary.
+	    */
+	need_isp_copy_binary = !online && sensor;
 #else
-		need_isp_copy_binary = !online && !continuous && !memory;
+	need_isp_copy_binary = !online && !continuous && !memory;
 #endif
 
-		/* ISP Copy */
-		if (need_isp_copy_binary) {
-			err = load_copy_binary(pipe,
-					       &mycs->copy_binary,
-					       &mycs->primary_binary[0]);
-			if (err != IA_CSS_SUCCESS) {
-				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				return err;
-			}
+	/* ISP Copy */
+	if (need_isp_copy_binary) {
+		err = load_copy_binary(pipe,
+					&mycs->copy_binary,
+					&mycs->primary_binary[0]);
+		if (err) {
+			IA_CSS_LEAVE_ERR_PRIVATE(err);
+			return err;
 		}
 	}
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 allocate_delay_frames(struct ia_css_pipe *pipe) {
 	unsigned int num_delay_frames = 0, i = 0;
 	unsigned int dvs_frame_delay = 0;
 	struct ia_css_frame_info ref_info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	enum ia_css_pipe_id mode = IA_CSS_PIPE_ID_VIDEO;
 	struct ia_css_frame **delay_frames = NULL;
 
@@ -6391,7 +6352,7 @@ allocate_delay_frames(struct ia_css_pipe *pipe) {
 	if (!pipe)
 	{
 		IA_CSS_ERROR("Invalid args - pipe %p", pipe);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	mode = pipe->mode;
@@ -6447,7 +6408,7 @@ allocate_delay_frames(struct ia_css_pipe *pipe) {
 	}
 	break;
 	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	ref_info.raw_bit_depth = SH_CSS_REF_BIT_DEPTH;
@@ -6456,14 +6417,14 @@ allocate_delay_frames(struct ia_css_pipe *pipe) {
 	for (i = 0; i < num_delay_frames; i++)
 	{
 		err = ia_css_frame_allocate_from_info(&delay_frames[i],	&ref_info);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 	IA_CSS_LEAVE_PRIVATE("");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err load_advanced_binaries(
+static int load_advanced_binaries(
     struct ia_css_pipe *pipe) {
 	struct ia_css_frame_info pre_in_info, gdc_in_info,
 			post_in_info, post_out_info,
@@ -6471,7 +6432,7 @@ static enum ia_css_err load_advanced_binaries(
 			*pipe_vf_out_info;
 	bool need_pp;
 	bool need_isp_copy = true;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("");
 
@@ -6479,13 +6440,13 @@ static enum ia_css_err load_advanced_binaries(
 	assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE ||
 		pipe->mode == IA_CSS_PIPE_ID_COPY);
 	if (pipe->pipe_settings.capture.pre_isp_binary.info)
-		return IA_CSS_SUCCESS;
+		return 0;
 	pipe_out_info = &pipe->output_info[0];
 	pipe_vf_out_info = &pipe->vf_output_info[0];
 
 	vf_info = *pipe_vf_out_info;
 	err = ia_css_util_check_vf_out_info(pipe_out_info, &vf_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	need_pp = need_capture_pp(pipe);
 
@@ -6501,7 +6462,7 @@ static enum ia_css_err load_advanced_binaries(
 							&capture_pp_descr, &post_out_info, pipe_out_info, &vf_info);
 		err = ia_css_binary_find(&capture_pp_descr,
 					    &pipe->pipe_settings.capture.capture_pp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	} else {
 		post_out_info = *pipe_out_info;
@@ -6515,7 +6476,7 @@ static enum ia_css_err load_advanced_binaries(
 						    &post_gdc_descr, &post_in_info, &post_out_info, &vf_info);
 		err = ia_css_binary_find(&post_gdc_descr,
 					    &pipe->pipe_settings.capture.post_isp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -6527,7 +6488,7 @@ static enum ia_css_err load_advanced_binaries(
 						&pipe->pipe_settings.capture.post_isp_binary.in_frame_info);
 		err = ia_css_binary_find(&gdc_descr,
 					    &pipe->pipe_settings.capture.anr_gdc_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 	pipe->pipe_settings.capture.anr_gdc_binary.left_padding =
@@ -6541,7 +6502,7 @@ static enum ia_css_err load_advanced_binaries(
 						    &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info);
 		err = ia_css_binary_find(&pre_gdc_descr,
 					    &pipe->pipe_settings.capture.pre_isp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 	pipe->pipe_settings.capture.pre_isp_binary.left_padding =
@@ -6563,7 +6524,7 @@ static enum ia_css_err load_advanced_binaries(
 						&vf_pp_descr, vf_pp_in_info, pipe_vf_out_info);
 		err = ia_css_binary_find(&vf_pp_descr,
 					    &pipe->pipe_settings.capture.vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -6580,10 +6541,10 @@ static enum ia_css_err load_advanced_binaries(
 	return err;
 }
 
-static enum ia_css_err load_bayer_isp_binaries(
+static int load_bayer_isp_binaries(
     struct ia_css_pipe *pipe) {
 	struct ia_css_frame_info pre_isp_in_info, *pipe_out_info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_binary_descr pre_de_descr;
 
 	IA_CSS_ENTER_PRIVATE("");
@@ -6593,10 +6554,10 @@ static enum ia_css_err load_bayer_isp_binaries(
 	pipe_out_info = &pipe->output_info[0];
 
 	if (pipe->pipe_settings.capture.pre_isp_binary.info)
-		return IA_CSS_SUCCESS;
+		return 0;
 
 	err = ia_css_frame_check_info(pipe_out_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	ia_css_pipe_get_pre_de_binarydesc(pipe, &pre_de_descr,
@@ -6609,7 +6570,7 @@ static enum ia_css_err load_bayer_isp_binaries(
 	return err;
 }
 
-static enum ia_css_err load_low_light_binaries(
+static int load_low_light_binaries(
     struct ia_css_pipe *pipe) {
 	struct ia_css_frame_info pre_in_info, anr_in_info,
 			post_in_info, post_out_info,
@@ -6617,7 +6578,7 @@ static enum ia_css_err load_low_light_binaries(
 			*vf_pp_in_info;
 	bool need_pp;
 	bool need_isp_copy = true;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("");
 	assert(pipe);
@@ -6625,14 +6586,14 @@ static enum ia_css_err load_low_light_binaries(
 		pipe->mode == IA_CSS_PIPE_ID_COPY);
 
 	if (pipe->pipe_settings.capture.pre_isp_binary.info)
-		return IA_CSS_SUCCESS;
+		return 0;
 	pipe_vf_out_info = &pipe->vf_output_info[0];
 	pipe_out_info = &pipe->output_info[0];
 
 	vf_info = *pipe_vf_out_info;
 	err = ia_css_util_check_vf_out_info(pipe_out_info,
 					    &vf_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	need_pp = need_capture_pp(pipe);
 
@@ -6648,7 +6609,7 @@ static enum ia_css_err load_low_light_binaries(
 							&capture_pp_descr, &post_out_info, pipe_out_info, &vf_info);
 		err = ia_css_binary_find(&capture_pp_descr,
 					    &pipe->pipe_settings.capture.capture_pp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	} else {
 		post_out_info = *pipe_out_info;
@@ -6662,7 +6623,7 @@ static enum ia_css_err load_low_light_binaries(
 						    &post_anr_descr, &post_in_info, &post_out_info, &vf_info);
 		err = ia_css_binary_find(&post_anr_descr,
 					    &pipe->pipe_settings.capture.post_isp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -6674,7 +6635,7 @@ static enum ia_css_err load_low_light_binaries(
 						&pipe->pipe_settings.capture.post_isp_binary.in_frame_info);
 		err = ia_css_binary_find(&anr_descr,
 					    &pipe->pipe_settings.capture.anr_gdc_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 	pipe->pipe_settings.capture.anr_gdc_binary.left_padding =
@@ -6688,7 +6649,7 @@ static enum ia_css_err load_low_light_binaries(
 						    &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info);
 		err = ia_css_binary_find(&pre_anr_descr,
 					    &pipe->pipe_settings.capture.pre_isp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 	pipe->pipe_settings.capture.pre_isp_binary.left_padding =
@@ -6710,7 +6671,7 @@ static enum ia_css_err load_low_light_binaries(
 						&vf_pp_descr, vf_pp_in_info, pipe_vf_out_info);
 		err = ia_css_binary_find(&vf_pp_descr,
 					    &pipe->pipe_settings.capture.vf_pp_binary);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 
@@ -6727,7 +6688,8 @@ static enum ia_css_err load_low_light_binaries(
 	return err;
 }
 
-static bool copy_on_sp(struct ia_css_pipe *pipe) {
+static bool copy_on_sp(struct ia_css_pipe *pipe)
+{
 	bool rval;
 
 	assert(pipe);
@@ -6746,9 +6708,9 @@ static bool copy_on_sp(struct ia_css_pipe *pipe) {
 	return rval;
 }
 
-static enum ia_css_err load_capture_binaries(
+static int load_capture_binaries(
     struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool must_be_raw;
 
 	IA_CSS_ENTER_PRIVATE("");
@@ -6757,8 +6719,8 @@ static enum ia_css_err load_capture_binaries(
 		pipe->mode == IA_CSS_PIPE_ID_COPY);
 
 	if (pipe->pipe_settings.capture.primary_binary[0].info) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
+		IA_CSS_LEAVE_ERR_PRIVATE(0);
+		return 0;
 	}
 
 	/* in primary, advanced,low light or bayer,
@@ -6768,7 +6730,7 @@ static enum ia_css_err load_capture_binaries(
 	    pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER ||
 	    pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT;
 	err = ia_css_util_check_input(&pipe->stream->config, must_be_raw, false);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
@@ -6780,15 +6742,15 @@ static enum ia_css_err load_capture_binaries(
 		    1,
 		    IA_CSS_FRAME_FORMAT_BINARY_8,
 		    0);
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
+		IA_CSS_LEAVE_ERR_PRIVATE(0);
+		return 0;
 	}
 
 	switch (pipe->config.default_capture_config.mode) {
 	case IA_CSS_CAPTURE_MODE_RAW:
 		err = load_copy_binaries(pipe);
 #if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
-		if (err == IA_CSS_SUCCESS)
+		if (!err)
 			pipe->pipe_settings.capture.copy_binary.online = pipe->stream->config.online;
 #endif
 		break;
@@ -6805,7 +6767,7 @@ static enum ia_css_err load_capture_binaries(
 		err = load_low_light_binaries(pipe);
 		break;
 	}
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
@@ -6814,7 +6776,7 @@ static enum ia_css_err load_capture_binaries(
 	return err;
 }
 
-static enum ia_css_err
+static int
 unload_capture_binaries(struct ia_css_pipe *pipe) {
 	unsigned int i;
 
@@ -6822,8 +6784,8 @@ unload_capture_binaries(struct ia_css_pipe *pipe) {
 
 	if ((!pipe) || ((pipe->mode != IA_CSS_PIPE_ID_CAPTURE) && (pipe->mode != IA_CSS_PIPE_ID_COPY)))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	ia_css_binary_unload(&pipe->pipe_settings.capture.copy_binary);
 	for (i = 0; i < MAX_NUM_PRIMARY_STAGES; i++)
@@ -6843,8 +6805,8 @@ unload_capture_binaries(struct ia_css_pipe *pipe) {
 	kfree(pipe->pipe_settings.capture.yuv_scaler_binary);
 	pipe->pipe_settings.capture.yuv_scaler_binary = NULL;
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
 static bool
@@ -6895,14 +6857,14 @@ need_yuv_scaler_stage(const struct ia_css_pipe *pipe) {
 /* TODO: it is temporarily created from ia_css_pipe_create_cas_scaler_desc */
 /* which has some hard-coded knowledge which prevents reuse of the function. */
 /* Later, merge this with ia_css_pipe_create_cas_scaler_desc */
-static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output(
+static int ia_css_pipe_create_cas_scaler_desc_single_output(
     struct ia_css_frame_info *cas_scaler_in_info,
     struct ia_css_frame_info *cas_scaler_out_info,
     struct ia_css_frame_info *cas_scaler_vf_info,
     struct ia_css_cas_binary_descr *descr) {
 	unsigned int i;
 	unsigned int hor_ds_factor = 0, ver_ds_factor = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_frame_info tmp_in_info;
 
 	unsigned int max_scale_factor_per_stage = MAX_PREFERRED_YUV_DS_PER_STEP;
@@ -6932,30 +6894,30 @@ static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output(
 	descr->in_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info),
 				    GFP_KERNEL);
 	if (!descr->in_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->internal_out_info = kmalloc(descr->num_stage * sizeof(
 						struct ia_css_frame_info), GFP_KERNEL);
 	if (!descr->internal_out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info),
 				    GFP_KERNEL);
 	if (!descr->out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info),
 				    GFP_KERNEL);
 	if (!descr->vf_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL);
 	if (!descr->is_output_stage) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 
@@ -7015,7 +6977,7 @@ ERR:
 }
 
 /* FIXME: merge most of this and single output version */
-static enum ia_css_err ia_css_pipe_create_cas_scaler_desc(
+static int ia_css_pipe_create_cas_scaler_desc(
     struct ia_css_pipe *pipe,
     struct ia_css_cas_binary_descr *descr) {
 	struct ia_css_frame_info in_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO;
@@ -7027,7 +6989,7 @@ static enum ia_css_err ia_css_pipe_create_cas_scaler_desc(
 		    ver_scale_factor[IA_CSS_PIPE_MAX_OUTPUT_STAGE],
 		    scale_factor = 0;
 	unsigned int num_stages = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	unsigned int max_scale_factor_per_stage = MAX_PREFERRED_YUV_DS_PER_STEP;
 
@@ -7076,30 +7038,30 @@ static enum ia_css_err ia_css_pipe_create_cas_scaler_desc(
 	descr->in_info = kmalloc_array(descr->num_stage,
 					sizeof(struct ia_css_frame_info), GFP_KERNEL);
 	if (!descr->in_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->internal_out_info = kmalloc(descr->num_stage * sizeof(
 						struct ia_css_frame_info), GFP_KERNEL);
 	if (!descr->internal_out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info),
 				    GFP_KERNEL);
 	if (!descr->out_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info),
 				    GFP_KERNEL);
 	if (!descr->vf_info) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 	descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL);
 	if (!descr->is_output_stage) {
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 
@@ -7190,9 +7152,9 @@ static void ia_css_pipe_destroy_cas_scaler_desc(struct ia_css_cas_binary_descr
 			    "ia_css_pipe_destroy_cas_scaler_desc() leave\n");
 }
 
-static enum ia_css_err
+static int
 load_yuvpp_binaries(struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool need_scaler = false;
 	struct ia_css_frame_info *vf_pp_in_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
 	struct ia_css_yuvpp_settings *mycs;
@@ -7211,7 +7173,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) {
 
 	/* Set both must_be_raw and must_be_yuv to false then yuvpp can take rgb inputs */
 	err = ia_css_util_check_input(&pipe->stream->config, false, false);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 	mycs = &pipe->pipe_settings.yuvpp;
@@ -7221,7 +7183,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) {
 		if (pipe->vf_output_info[i].res.width != 0) {
 			err = ia_css_util_check_vf_out_info(&pipe->output_info[i],
 							    &pipe->vf_output_info[i]);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 		}
 		vf_pp_in_info[i] = NULL;
@@ -7237,20 +7199,20 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) {
 
 		err = ia_css_pipe_create_cas_scaler_desc(pipe,
 			&cas_scaler_descr);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 		mycs->num_output = cas_scaler_descr.num_output_stage;
 		mycs->num_yuv_scaler = cas_scaler_descr.num_stage;
 		mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage *
 						    sizeof(struct ia_css_binary), GFP_KERNEL);
 		if (!mycs->yuv_scaler_binary) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			goto ERR;
 		}
 		mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage *
 						sizeof(bool), GFP_KERNEL);
 		if (!mycs->is_output_stage) {
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			goto ERR;
 		}
 		for (i = 0; i < cas_scaler_descr.num_stage; i++) {
@@ -7262,7 +7224,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) {
 								&cas_scaler_descr.vf_info[i]);
 			err = ia_css_binary_find(&yuv_scaler_descr,
 						    &mycs->yuv_scaler_binary[i]);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 		}
 		ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr);
@@ -7309,7 +7271,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) {
 					&mycs->copy_binary,
 					next_binary);
 
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 
 		/*
@@ -7358,7 +7320,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) {
 					GFP_KERNEL);
 	if (!mycs->vf_pp_binary)
 	{
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		goto ERR;
 	}
 
@@ -7371,13 +7333,13 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) {
 				ia_css_pipe_get_vfpp_binarydesc(pipe,
 								&vf_pp_descr, vf_pp_in_info[i], &pipe->vf_output_info[i]);
 				err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary[i]);
-				if (err != IA_CSS_SUCCESS)
+				if (err)
 					goto ERR;
 			}
 		}
 	}
 
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 ERR:
@@ -7390,7 +7352,7 @@ ERR:
 	return err;
 }
 
-static enum ia_css_err
+static int
 unload_yuvpp_binaries(struct ia_css_pipe *pipe) {
 	unsigned int i;
 
@@ -7398,8 +7360,8 @@ unload_yuvpp_binaries(struct ia_css_pipe *pipe) {
 
 	if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	ia_css_binary_unload(&pipe->pipe_settings.yuvpp.copy_binary);
 	for (i = 0; i < pipe->pipe_settings.yuvpp.num_yuv_scaler; i++)
@@ -7417,20 +7379,21 @@ unload_yuvpp_binaries(struct ia_css_pipe *pipe) {
 	kfree(pipe->pipe_settings.yuvpp.vf_pp_binary);
 	pipe->pipe_settings.yuvpp.vf_pp_binary = NULL;
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
-static enum ia_css_err yuvpp_start(struct ia_css_pipe *pipe) {
+static int yuvpp_start(struct ia_css_pipe *pipe)
+{
 	struct ia_css_binary *copy_binary;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	enum sh_css_pipe_config_override copy_ovrd;
 	enum ia_css_input_mode yuvpp_pipe_input_mode;
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	yuvpp_pipe_input_mode = pipe->stream->config.mode;
@@ -7443,7 +7406,7 @@ static enum ia_css_err yuvpp_start(struct ia_css_pipe *pipe) {
 
 #if !defined(HAS_NO_INPUT_SYSTEM) && (defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401))
 	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
@@ -7462,22 +7425,22 @@ static enum ia_css_err yuvpp_start(struct ia_css_pipe *pipe) {
 	return err;
 }
 
-static enum ia_css_err
+static int
 sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 
 	if (!pipe)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	/* PIPE_MODE_COPY has no binaries, but has output frames to outside*/
 	if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-		return IA_CSS_SUCCESS;
+		IA_CSS_LEAVE_ERR_PRIVATE(0);
+		return 0;
 	}
 
 	switch (pipe->mode)
@@ -7501,9 +7464,9 @@ sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe) {
 	return err;
 }
 
-static enum ia_css_err
+static int
 sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(pipe);
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_pipe_load_binaries() enter:\n");
@@ -7529,24 +7492,24 @@ sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) {
 	case IA_CSS_PIPE_ID_ACC:
 		break;
 	default:
-		err = IA_CSS_ERR_INTERNAL_ERROR;
+		err = -EINVAL;
 		break;
 	}
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
-		if (sh_css_pipe_unload_binaries(pipe) != IA_CSS_SUCCESS) {
+		if (sh_css_pipe_unload_binaries(pipe)) {
 			/* currently css does not support multiple error returns in a single function,
-			    * using IA_CSS_ERR_INTERNAL_ERROR in this case */
-			err = IA_CSS_ERR_INTERNAL_ERROR;
+			    * using -EINVAL in this case */
+			err = -EINVAL;
 		}
 	}
 	return err;
 }
 
-static enum ia_css_err
+static int
 create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 	struct ia_css_pipeline *me;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipeline_stage *vf_pp_stage = NULL,
 		*copy_stage = NULL,
 		*yuv_scaler_stage = NULL;
@@ -7573,8 +7536,8 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	me = &pipe->pipeline;
 	ia_css_pipeline_clean(me);
@@ -7652,7 +7615,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 							&me->in_frame,
 							in_frame_format);
 
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -7668,7 +7631,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 		assert(i < IA_CSS_PIPE_MAX_OUTPUT_STAGE);
 		if (pipe->output_info[i].res.width != 0) {
 			err = init_out_frameinfo_defaults(pipe, &me->out_frame[i], i);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -7678,7 +7641,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 		/* Construct vf_frame info (only in case we have VF) */
 		if (pipe->vf_output_info[i].res.width != 0) {
 			err = init_vf_frameinfo_defaults(pipe, &me->vf_frame[i], i);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -7715,7 +7678,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 			&stage_desc,
 			&copy_stage);
 
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -7751,7 +7714,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 						    &yuv_scaler_binary[i],
 						    &yuv_scaler_stage);
 
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -7763,7 +7726,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 					err = add_vf_pp_stage(pipe, in_frame, tmp_vf_frame, &vf_pp_binary[j],
 								&vf_pp_stage);
 
-					if (err != IA_CSS_SUCCESS) {
+					if (err) {
 						IA_CSS_LEAVE_ERR_PRIVATE(err);
 						return err;
 					}
@@ -7778,7 +7741,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 			err = add_vf_pp_stage(pipe, in_frame, vf_frame[0], &vf_pp_binary[0],
 						&vf_pp_stage);
 		}
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -7786,17 +7749,17 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) {
 
 	ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous);
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 create_host_copy_pipeline(struct ia_css_pipe *pipe,
 			    unsigned int max_input_width,
 			    struct ia_css_frame *out_frame) {
 	struct ia_css_pipeline *me;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipeline_stage_desc stage_desc;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
@@ -7843,10 +7806,10 @@ create_host_copy_pipeline(struct ia_css_pipe *pipe,
 	return err;
 }
 
-static enum ia_css_err
+static int
 create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) {
 	struct ia_css_pipeline *me = &pipe->pipeline;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_pipeline_stage_desc stage_desc;
 	struct ia_css_frame *out_frame = &me->out_frame[0];
 	struct ia_css_pipeline_stage *out_stage = NULL;
@@ -7860,7 +7823,7 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) {
 
 	/* Construct out_frame info */
 	err = sh_css_pipe_get_output_frame_info(pipe, &out_frame->info, 0);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	out_frame->contiguous = false;
 	out_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE;
@@ -7876,7 +7839,7 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) {
 					    IA_CSS_PIPELINE_ISYS_COPY, max_input_width);
 	err = ia_css_pipeline_create_and_add_stage(me,
 		&stage_desc, &out_stage);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	ia_css_pipeline_finalize_stages(me, pipe->stream->config.continuous);
@@ -7887,10 +7850,10 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) {
 	return err;
 }
 
-static enum ia_css_err
+static int
 create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 	struct ia_css_pipeline *me;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	enum ia_css_capture_mode mode;
 	struct ia_css_pipeline_stage *current_stage = NULL;
 	struct ia_css_pipeline_stage *yuv_scaler_stage = NULL;
@@ -7956,7 +7919,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 	{
 		err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame,
 							IA_CSS_FRAME_FORMAT_RAW);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -7968,7 +7931,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 	}
 
 	err = init_out_frameinfo_defaults(pipe, &me->out_frame[0], 0);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -7994,8 +7957,8 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 	num_primary_stage = pipe->pipe_settings.capture.num_primary_stage;
 	if ((num_primary_stage == 0) && (mode == IA_CSS_CAPTURE_MODE_PRIMARY))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	for (i = 0; i < num_primary_stage; i++)
 	{
@@ -8043,7 +8006,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc,
 			&current_stage);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8086,7 +8049,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 			err = ia_css_pipeline_create_and_add_stage(me,
 				&stage_desc,
 				&current_stage);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -8105,7 +8068,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 						    out_frames, in_frame, NULL);
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc, NULL);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8114,7 +8077,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 						    out_frames, NULL, NULL);
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc, NULL);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8131,7 +8094,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc, &current_stage);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8143,7 +8106,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 		err = ia_css_pipeline_create_and_add_stage(me,
 			&stage_desc,
 			NULL);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8185,7 +8148,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 #endif
 					    capture_pp_binary,
 					    &current_stage);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8206,7 +8169,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 						    NULL,
 						    &yuv_scaler_binary[i],
 						    &yuv_scaler_stage);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -8229,7 +8192,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 		in_frame = current_stage->args.out_vf_frame;
 		err = add_vf_pp_stage(pipe, in_frame, vf_frame, vf_pp_binary,
 					&current_stage);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8239,12 +8202,12 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) {
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 			    "create_host_regular_capture_pipeline() leave:\n");
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 create_host_capture_pipeline(struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 
@@ -8252,7 +8215,7 @@ create_host_capture_pipeline(struct ia_css_pipe *pipe) {
 		err = create_host_isyscopy_capture_pipeline(pipe);
 	else
 		err = create_host_regular_capture_pipeline(pipe);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -8263,17 +8226,17 @@ create_host_capture_pipeline(struct ia_css_pipe *pipe) {
 	return err;
 }
 
-static enum ia_css_err capture_start(
+static int capture_start(
     struct ia_css_pipe *pipe) {
 	struct ia_css_pipeline *me;
 
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	enum sh_css_pipe_config_override copy_ovrd;
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p", pipe);
 	if (!pipe) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	me = &pipe->pipeline;
@@ -8291,14 +8254,14 @@ static enum ia_css_err capture_start(
 #if defined(USE_INPUT_SYSTEM_VERSION_2)
 	/* old isys: need to send_mipi_frames() in all pipe modes */
 	err = send_mipi_frames(pipe);
-	if (err != IA_CSS_SUCCESS) {
+	if (err) {
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
 #elif defined(USE_INPUT_SYSTEM_VERSION_2401)
 	if (pipe->config.mode != IA_CSS_PIPE_MODE_COPY) {
 		err = send_mipi_frames(pipe);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -8332,7 +8295,7 @@ static enum ia_css_err capture_start(
 	return err;
 }
 
-static enum ia_css_err
+static int
 sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe,
 				    struct ia_css_frame_info *info,
 				    unsigned int idx) {
@@ -8361,7 +8324,7 @@ sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe,
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 			    "sh_css_pipe_get_output_frame_info() leave:\n");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 #if !defined(HAS_NO_INPUT_SYSTEM)
@@ -8448,12 +8411,13 @@ remove_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) {
 	return; /* removing single and multiple firmware is handled in acc_unload_extension() */
 }
 
-static enum ia_css_err upload_isp_code(struct ia_css_fw_info *firmware) {
-	hrt_vaddress binary;
+static int upload_isp_code(struct ia_css_fw_info *firmware)
+{
+	ia_css_ptr binary;
 
 	if (!firmware) {
 		IA_CSS_ERROR("NULL input parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	binary = firmware->info.isp.xmem_addr;
 
@@ -8473,27 +8437,27 @@ static enum ia_css_err upload_isp_code(struct ia_css_fw_info *firmware) {
 	}
 
 	if (!binary)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-	return IA_CSS_SUCCESS;
+		return -ENOMEM;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 acc_load_extension(struct ia_css_fw_info *firmware) {
-	enum ia_css_err err;
+	int err;
 	struct ia_css_fw_info *hd = firmware;
 
 	while (hd)
 	{
 		err = upload_isp_code(hd);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 		hd = hd->next;
 	}
 
 	if (!firmware)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	firmware->loaded = true;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -8519,36 +8483,23 @@ acc_unload_extension(struct ia_css_fw_info *firmware) {
 }
 
 /* Load firmware for extension */
-static enum ia_css_err
+static int
 ia_css_pipe_load_extension(struct ia_css_pipe *pipe,
 			    struct ia_css_fw_info *firmware) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("fw = %p pipe = %p", firmware, pipe);
 
 	if ((!firmware) || (!pipe))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (firmware->info.isp.type == IA_CSS_ACC_OUTPUT)
-	{
-		if (&pipe->output_stage)
-			append_firmware(&pipe->output_stage, firmware);
-		else {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-	} else if (firmware->info.isp.type == IA_CSS_ACC_VIEWFINDER)
-	{
-		if (&pipe->vf_stage)
-			append_firmware(&pipe->vf_stage, firmware);
-		else {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		}
-	}
+		append_firmware(&pipe->output_stage, firmware);
+	else if (firmware->info.isp.type == IA_CSS_ACC_VIEWFINDER)
+		append_firmware(&pipe->vf_stage, firmware);
 	err = acc_load_extension(firmware);
 
 	IA_CSS_LEAVE_ERR_PRIVATE(err);
@@ -8596,12 +8547,12 @@ ia_css_pipeline_uses_params(struct ia_css_pipeline *me) {
 	return false;
 }
 
-static enum ia_css_err
+static int
 sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline,
 				const void *acc_fw) {
 	struct ia_css_fw_info *fw = (struct ia_css_fw_info *)acc_fw;
 	/* In QoS case, load_extension already called, so skipping */
-	enum ia_css_err	err = IA_CSS_SUCCESS;
+	int	err = 0;
 
 	if (fw->loaded == false)
 		err = acc_load_extension(fw);
@@ -8610,7 +8561,7 @@ sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline,
 			    "sh_css_pipeline_add_acc_stage() enter: pipeline=%p, acc_fw=%p\n",
 			    pipeline, acc_fw);
 
-	if (err == IA_CSS_SUCCESS)
+	if (!err)
 	{
 		struct ia_css_pipeline_stage_desc stage_desc;
 
@@ -8629,25 +8580,25 @@ sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline,
     * @brief Tag a specific frame in continuous capture.
     * Refer to "sh_css_internal.h" for details.
     */
-enum ia_css_err ia_css_stream_capture_frame(struct ia_css_stream *stream,
+int ia_css_stream_capture_frame(struct ia_css_stream *stream,
 	unsigned int exp_id) {
 	struct sh_css_tag_descr tag_descr;
 	u32 encoded_tag_descr;
-	enum ia_css_err err;
+	int err;
 
 	assert(stream);
 	IA_CSS_ENTER("exp_id=%d", exp_id);
 
 	/* Only continuous streams have a tagger */
 	if (exp_id == 0 || !stream->config.continuous) {
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (!sh_css_sp_is_running()) {
 		/* SP is not running. The queues are not valid */
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		IA_CSS_LEAVE_ERR(-EBUSY);
+		return -EBUSY;
 	}
 
 	/* Create the tag descriptor from the parameters */
@@ -8668,17 +8619,17 @@ enum ia_css_err ia_css_stream_capture_frame(struct ia_css_stream *stream,
     * @brief Configure the continuous capture.
     * Refer to "sh_css_internal.h" for details.
     */
-enum ia_css_err ia_css_stream_capture(
+int ia_css_stream_capture(
     struct ia_css_stream *stream,
     int num_captures,
     unsigned int skip,
     int offset) {
 	struct sh_css_tag_descr tag_descr;
 	unsigned int encoded_tag_descr;
-	enum ia_css_err return_err;
+	int return_err;
 
 	if (!stream)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 			    "ia_css_stream_capture() enter: num_captures=%d, skip=%d, offset=%d\n",
@@ -8688,8 +8639,8 @@ enum ia_css_err ia_css_stream_capture(
 	if (num_captures < SH_CSS_MINIMUM_TAG_ID) {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_stream_capture() leave: return_err=%d\n",
-				    IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+				    -EINVAL);
+		return -EINVAL;
 	}
 
 	/* Create the tag descriptor from the parameters */
@@ -8702,7 +8653,7 @@ enum ia_css_err ia_css_stream_capture(
 		/* SP is not running. The queues are not valid */
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 				    "ia_css_stream_capture() leaving:queues unavailable\n");
-		return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		return -EBUSY;
 	}
 
 	/* Enqueue the encoded tag to the host2sp queue.
@@ -8718,7 +8669,8 @@ enum ia_css_err ia_css_stream_capture(
 	return return_err;
 }
 
-void ia_css_stream_request_flash(struct ia_css_stream *stream) {
+void ia_css_stream_request_flash(struct ia_css_stream *stream)
+{
 	(void)stream;
 
 	assert(stream);
@@ -8817,11 +8769,16 @@ sh_css_init_host_sp_control_vars(void) {
 }
 
 /*
-    * create the internal structures and fill in the configuration data
-    */
-void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config) {
+ * create the internal structures and fill in the configuration data
+ */
+
+static const struct
+ia_css_pipe_config ia_css_pipe_default_config = DEFAULT_PIPE_CONFIG;
+
+void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config)
+{
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_config_defaults()\n");
-	*pipe_config = DEFAULT_PIPE_CONFIG;
+	memcpy(pipe_config, &ia_css_pipe_default_config, sizeof(*pipe_config));
 }
 
 void
@@ -8841,7 +8798,8 @@ ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config
 	extra_config->enable_fractional_ds = false;
 }
 
-void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config) {
+void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config)
+{
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_config_defaults()\n");
 	assert(stream_config);
 	memset(stream_config, 0, sizeof(*stream_config));
@@ -8854,14 +8812,14 @@ void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config) {
 	stream_config->source.port.rxcount = 0x04040404;
 }
 
-static enum ia_css_err
+static int
 ia_css_acc_pipe_create(struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	if (!pipe)
 	{
 		IA_CSS_ERROR("NULL input parameter");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	/* There is not meaning for num_execs = 0 semantically. Run atleast once. */
@@ -8876,30 +8834,31 @@ ia_css_acc_pipe_create(struct ia_css_pipe *pipe) {
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_pipe_create(const struct ia_css_pipe_config *config,
 		    struct ia_css_pipe **pipe) {
 #ifndef ISP2401
 	if (!config)
 #else
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("config = %p, pipe = %p", config, pipe);
 
 	if (!config)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
 #endif
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 #ifndef ISP2401
 	if (!pipe)
 #else
 }
+
 if (!pipe)
 {
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
+	IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
 #endif
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 #ifndef ISP2401
 	return ia_css_pipe_create_extra(config, NULL, pipe);
 #else
@@ -8907,7 +8866,7 @@ if (!pipe)
 
 err = ia_css_pipe_create_extra(config, NULL, pipe);
 
-if (err == IA_CSS_SUCCESS)
+if (err == 0)
 {
 	IA_CSS_LOG("pipe created successfully = %p", *pipe);
 }
@@ -8918,11 +8877,11 @@ return err;
 #endif
 }
 
-enum ia_css_err
+int
 ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 			    const struct ia_css_pipe_extra_config *extra_config,
 			    struct ia_css_pipe **pipe) {
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
+	int err = -EINVAL;
 	struct ia_css_pipe *internal_pipe = NULL;
 	unsigned int i;
 
@@ -8931,21 +8890,21 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 	/* do not allow to create more than the maximum limit */
 	if (my_css.pipe_counter >= IA_CSS_PIPELINE_NUM_MAX)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_EXHAUSTED);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-ENOSPC);
+		return -EINVAL;
 	}
 
 	if ((!pipe) || (!config))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	ia_css_debug_dump_pipe_config(config);
 	ia_css_debug_dump_pipe_extra_config(extra_config);
 
 	err = create_pipe(config->mode, &internal_pipe, false);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -8968,8 +8927,8 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 		    internal_pipe->config.num_acc_stages ==
 		    0) { /* if no acc binary and no standalone stage */
 			*pipe = NULL;
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-			return IA_CSS_SUCCESS;
+			IA_CSS_LEAVE_ERR_PRIVATE(0);
+			return 0;
 		}
 		return ia_css_acc_pipe_create(internal_pipe);
 	}
@@ -9044,9 +9003,9 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 				    internal_pipe->config.output_info[i].padded_width,
 				    internal_pipe->config.output_info[i].format,
 				    i);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				sh_css_free(internal_pipe);
+				kvfree(internal_pipe);
 				internal_pipe = NULL;
 				return err;
 			}
@@ -9063,9 +9022,9 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 				    internal_pipe->config.vf_output_info[i].padded_width,
 				    internal_pipe->config.vf_output_info[i].format,
 				    i);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
-				sh_css_free(internal_pipe);
+				kvfree(internal_pipe);
 				internal_pipe = NULL;
 				return err;
 			}
@@ -9075,9 +9034,9 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 	{
 		err = ia_css_pipe_load_extension(internal_pipe,
 						    internal_pipe->config.acc_extension);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
-			sh_css_free(internal_pipe);
+			kvfree(internal_pipe);
 			return err;
 		}
 	}
@@ -9086,11 +9045,11 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 
 	/* all went well, return the pipe */
 	*pipe = internal_pipe;
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
-enum ia_css_err
+int
 ia_css_pipe_get_info(const struct ia_css_pipe *pipe,
 			struct ia_css_pipe_info *pipe_info) {
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
@@ -9100,21 +9059,22 @@ ia_css_pipe_get_info(const struct ia_css_pipe *pipe,
 	{
 		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
 				    "ia_css_pipe_get_info: pipe_info cannot be NULL\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	if (!pipe || !pipe->stream)
 	{
 		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
 				    "ia_css_pipe_get_info: ia_css_stream_create needs to be called before ia_css_[stream/pipe]_get_info\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	/* we succeeded return the info */
 	*pipe_info = pipe->info;
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_get_info() leave\n");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info) {
+bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info)
+{
 	unsigned int i;
 
 	if (pipe_info) {
@@ -9127,38 +9087,38 @@ bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info) {
 	return false;
 }
 
-enum ia_css_err
+int
 ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe,
 				    int pin_index,
 				    enum ia_css_frame_format new_format) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("pipe = %p, pin_index = %d, new_formats = %d", pipe, pin_index, new_format);
 
 	if (!pipe)
 	{
 		IA_CSS_ERROR("pipe is not set");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
 	if (0 != pin_index && 1 != pin_index)
 	{
 		IA_CSS_ERROR("pin index is not valid");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
 	if (new_format != IA_CSS_FRAME_FORMAT_NV12_TILEY)
 	{
 		IA_CSS_ERROR("new format is not valid");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	} else
 	{
 		err = ia_css_pipe_check_format(pipe, new_format);
-		if (err == IA_CSS_SUCCESS) {
+		if (!err) {
 			if (pin_index == 0) {
 				pipe->output_info[0].format = new_format;
 			} else {
@@ -9172,7 +9132,7 @@ ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe,
 
 #if defined(USE_INPUT_SYSTEM_VERSION_2)
 /* Configuration of INPUT_SYSTEM_VERSION_2401 is done on SP */
-static enum ia_css_err
+static int
 ia_css_stream_configure_rx(struct ia_css_stream *stream) {
 	struct ia_css_input_port *config;
 
@@ -9189,10 +9149,10 @@ ia_css_stream_configure_rx(struct ia_css_stream *stream) {
 	else if (config->num_lanes == 4)
 		stream->csi_rx_config.mode = MONO_4L_1L_0L;
 	else if (config->num_lanes != 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if (config->port > MIPI_PORT2_ID)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	stream->csi_rx_config.port =
 	ia_css_isys_port_to_mipi_port(config->port);
 	stream->csi_rx_config.timeout    = config->timeout;
@@ -9205,11 +9165,11 @@ ia_css_stream_configure_rx(struct ia_css_stream *stream) {
 	{
 		/* not implemented yet, requires extension of the rx_cfg_t
 		    * struct */
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	stream->csi_rx_config.is_two_ppc = (stream->config.pixels_per_clock == 2);
 	stream->reconfigure_css_rx = true;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 #endif
 
@@ -9232,18 +9192,18 @@ find_pipe(struct ia_css_pipe *pipes[],
 	return NULL;
 }
 
-static enum ia_css_err
+static int
 ia_css_acc_stream_create(struct ia_css_stream *stream) {
 	int i;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(stream);
 	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
 
 	if (!stream)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 
 	for (i = 0;  i < stream->num_pipes; i++)
@@ -9252,8 +9212,8 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) {
 
 		assert(pipe);
 		if (!pipe) {
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+			return -EINVAL;
 		}
 
 		pipe->stream = stream;
@@ -9261,7 +9221,7 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) {
 
 	/* Map SP threads before doing anything. */
 	err = map_sp_threads(stream, true);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -9276,7 +9236,7 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) {
 	}
 
 	err = create_host_pipeline_structure(stream);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -9284,42 +9244,43 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) {
 
 	stream->started = false;
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 metadata_info_init(const struct ia_css_metadata_config *mdc,
 		    struct ia_css_metadata_info *md) {
 	/* Either both width and height should be set or neither */
 	if ((mdc->resolution.height > 0) ^ (mdc->resolution.width > 0))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	md->resolution = mdc->resolution;
 	/* We round up the stride to a multiple of the width
 	    * of the port going to DDR, this is a HW requirements (DMA). */
 	md->stride = CEIL_MUL(mdc->resolution.width, HIVE_ISP_DDR_WORD_BYTES);
 	md->size = mdc->resolution.height * md->stride;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /* ISP2401 */
-static enum ia_css_err check_pipe_resolutions(const struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+static int check_pipe_resolutions(const struct ia_css_pipe *pipe)
+{
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("");
 
 	if (!pipe || !pipe->stream) {
 		IA_CSS_ERROR("null arguments");
-		err = IA_CSS_ERR_INTERNAL_ERROR;
+		err = -EINVAL;
 		goto EXIT;
 	}
 
 	if (ia_css_util_check_res(pipe->config.input_effective_res.width,
-				    pipe->config.input_effective_res.height) != IA_CSS_SUCCESS) {
+				    pipe->config.input_effective_res.height) != 0) {
 		IA_CSS_ERROR("effective resolution not supported");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		goto EXIT;
 	}
 	if (!ia_css_util_resolution_is_zero(
@@ -9327,18 +9288,18 @@ static enum ia_css_err check_pipe_resolutions(const struct ia_css_pipe *pipe) {
 		if (!ia_css_util_res_leq(pipe->config.input_effective_res,
 					    pipe->stream->config.input_config.input_res)) {
 			IA_CSS_ERROR("effective resolution is larger than input resolution");
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
+			err = -EINVAL;
 			goto EXIT;
 		}
 	}
 	if (!ia_css_util_resolution_is_even(pipe->config.output_info[0].res)) {
 		IA_CSS_ERROR("output resolution must be even");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		goto EXIT;
 	}
 	if (!ia_css_util_resolution_is_even(pipe->config.vf_output_info[0].res)) {
 		IA_CSS_ERROR("VF resolution must be even");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		goto EXIT;
 	}
 EXIT:
@@ -9346,7 +9307,7 @@ EXIT:
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 			int num_pipes,
 			struct ia_css_pipe *pipes[],
@@ -9356,7 +9317,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	bool spcopyonly;
 	bool sensor_binning_changed;
 	int i, j;
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
+	int err = -EINVAL;
 	struct ia_css_metadata_info md_info;
 	struct ia_css_resolution effective_res;
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
@@ -9371,7 +9332,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	    !stream ||
 	    !pipes)
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
@@ -9381,7 +9342,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	if (stream_config->input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8 &&
 	    stream_config->metadata_config.resolution.height > 0)
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
@@ -9391,7 +9352,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	if (stream_config->online && stream_config->pack_raw_pixels)
 	{
 		IA_CSS_LOG("online and pack raw is invalid on input system 2401");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
@@ -9409,7 +9370,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 			unsigned int port = (unsigned int)stream_config->source.port.port;
 
 			if (port >= N_MIPI_PORT_ID) {
-				err = IA_CSS_ERR_INVALID_ARGUMENTS;
+				err = -EINVAL;
 				IA_CSS_LEAVE_ERR(err);
 				return err;
 			}
@@ -9422,7 +9383,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 						    "ia_css_stream_create() exit: error, need to set mipi frame size.\n");
 				assert(stream_config->mipi_buffer_config.size_mem_words != 0);
-				err = IA_CSS_ERR_INTERNAL_ERROR;
+				err = -EINVAL;
 				IA_CSS_LEAVE_ERR(err);
 				return err;
 			}
@@ -9437,7 +9398,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 				ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,
 						    "ia_css_stream_create() exit: error, need to set number of mipi frames.\n");
 				assert(stream_config->mipi_buffer_config.nof_mipi_buffers != 0);
-				err = IA_CSS_ERR_INTERNAL_ERROR;
+				err = -EINVAL;
 				IA_CSS_LEAVE_ERR(err);
 				return err;
 			}
@@ -9446,22 +9407,21 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 
 	/* Currently we only supported metadata up to a certain size. */
 	err = metadata_info_init(&stream_config->metadata_config, &md_info);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
 
 	/* allocate the stream instance */
-	curr_stream = kmalloc(sizeof(struct ia_css_stream), GFP_KERNEL);
+	curr_stream = kzalloc(sizeof(struct ia_css_stream), GFP_KERNEL);
 	if (!curr_stream)
 	{
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
 	/* default all to 0 */
-	memset(curr_stream, 0, sizeof(struct ia_css_stream));
 	curr_stream->info.metadata_info = md_info;
 
 	/* allocate pipes */
@@ -9472,7 +9432,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 		curr_stream->num_pipes = 0;
 		kfree(curr_stream);
 		curr_stream = NULL;
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
@@ -9553,7 +9513,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	err = aspect_ratio_crop_init(curr_stream,
 					pipes,
 					&aspect_ratio_crop_enabled);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR(err);
 		return err;
@@ -9579,7 +9539,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 				struct ia_css_resolution crop_res;
 
 				err = aspect_ratio_crop(curr_pipe, &crop_res);
-				if (err == IA_CSS_SUCCESS) {
+				if (!err) {
 					effective_res = crop_res;
 				} else {
 					/* in case of error fallback to default
@@ -9595,12 +9555,12 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 			    effective_res.height);
 	}
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		for (i = 0; i < num_pipes; i++) {
 			if (pipes[i]->config.mode != IA_CSS_PIPE_MODE_ACC &&
 			    pipes[i]->config.mode != IA_CSS_PIPE_MODE_COPY) {
 				err = check_pipe_resolutions(pipes[i]);
-				if (err != IA_CSS_SUCCESS) {
+				if (err) {
 					goto ERR;
 				}
 			}
@@ -9608,7 +9568,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	}
 
 	err = ia_css_stream_isp_parameters_init(curr_stream);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 	IA_CSS_LOG("isp_params_configs: %p", curr_stream->isp_params_configs);
 
@@ -9649,7 +9609,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 			curr_stream->cont_capt = true;
 			curr_stream->disable_cont_vf = curr_stream->config.disable_cont_viewfinder;
 
-			if (!atomisp_hw_is_isp2401)
+			if (!IS_ISP2401)
 				curr_stream->stop_copy_preview = my_css.stop_copy_preview;
 		}
 
@@ -9667,19 +9627,19 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 			capture_pipe = find_pipe(pipes, num_pipes,
 						    IA_CSS_PIPE_MODE_CAPTURE, false);
 			if (!capture_pipe) {
-				err = IA_CSS_ERR_INTERNAL_ERROR;
+				err = -EINVAL;
 				goto ERR;
 			}
 		}
 		/* We do not support preview and video pipe at the same time */
 		if (preview_pipe && video_pipe) {
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
+			err = -EINVAL;
 			goto ERR;
 		}
 
 		if (preview_pipe && !preview_pipe->pipe_settings.preview.copy_pipe) {
 			err = create_pipe(IA_CSS_PIPE_MODE_CAPTURE, &copy_pipe, true);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 			ia_css_pipe_config_defaults(&copy_pipe->config);
 			preview_pipe->pipe_settings.preview.copy_pipe = copy_pipe;
@@ -9690,7 +9650,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 		}
 		if (video_pipe && !video_pipe->pipe_settings.video.copy_pipe) {
 			err = create_pipe(IA_CSS_PIPE_MODE_CAPTURE, &copy_pipe, true);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 			ia_css_pipe_config_defaults(&copy_pipe->config);
 			video_pipe->pipe_settings.video.copy_pipe = copy_pipe;
@@ -9709,14 +9669,14 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 		/* set current stream */
 		curr_pipe->stream = curr_stream;
 
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			/* take over effective info */
 
 			effective_res = curr_pipe->config.input_effective_res;
 			err = ia_css_util_check_res(
 				effective_res.width,
 				effective_res.height);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 		}
 		/* sensor binning per pipe */
@@ -9732,7 +9692,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 		curr_pipe = pipes[i];
 
 		err = sh_css_pipe_load_binaries(curr_pipe);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto ERR;
 
 		/* handle each pipe */
@@ -9740,31 +9700,31 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 		for (j = 0; j < IA_CSS_PIPE_MAX_OUTPUT_STAGE; j++) {
 			err = sh_css_pipe_get_output_frame_info(curr_pipe,
 								&pipe_info->output_info[j], j);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 		}
 
-		if (atomisp_hw_is_isp2401)
+		if (IS_ISP2401)
 			pipe_info->output_system_in_res_info = curr_pipe->config.output_system_in_res;
 
 		if (!spcopyonly) {
-			if (!atomisp_hw_is_isp2401)
+			if (!IS_ISP2401)
 				err = sh_css_pipe_get_shading_info(curr_pipe,
 								    &pipe_info->shading_info, NULL);
 			else
 				err = sh_css_pipe_get_shading_info(curr_pipe,
 								    &pipe_info->shading_info, &curr_pipe->config);
 
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 			err = sh_css_pipe_get_grid_info(curr_pipe,
 							&pipe_info->grid_info);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				goto ERR;
 			for (j = 0; j < IA_CSS_PIPE_MAX_OUTPUT_STAGE; j++) {
 				sh_css_pipe_get_viewfinder_frame_info(curr_pipe,
 									&pipe_info->vf_output_info[j], j);
-				if (err != IA_CSS_SUCCESS)
+				if (err)
 					goto ERR;
 			}
 		}
@@ -9776,7 +9736,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 
 	/* Map SP threads before doing anything. */
 	err = map_sp_threads(curr_stream, true);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LOG("map_sp_threads: return_err=%d", err);
 		goto ERR;
@@ -9790,7 +9750,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 
 	/* Create host side pipeline objects without stages */
 	err = create_host_pipeline_structure(curr_stream);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LOG("create_host_pipeline_structure: return_err=%d", err);
 		goto ERR;
@@ -9800,7 +9760,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	*stream = curr_stream;
 
 ERR:
-	if (err == IA_CSS_SUCCESS) {
+	if (!err) {
 		/* working mode: enter into the seed list */
 		if (my_css_save.mode == sh_css_mode_working) {
 			for (i = 0; i < MAX_ACTIVE_STREAMS; i++) {
@@ -9828,15 +9788,15 @@ ERR:
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_stream_destroy(struct ia_css_stream *stream) {
 	int i;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("stream = %p", stream);
 	if (!stream)
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
@@ -9859,7 +9819,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) {
 				/* get the SP thread id */
 				if (ia_css_pipeline_get_sp_thread_id(
 					ia_css_pipe_get_pipe_num(entry), &sp_thread_id) != true)
-					return IA_CSS_ERR_INTERNAL_ERROR;
+					return -EINVAL;
 				/* get the target input terminal */
 				sp_pipeline_input_terminal =
 				&sh_css_sp_group.pipe_io[sp_thread_id].input;
@@ -9873,7 +9833,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) {
 			}
 		}
 		free_mpi = stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR;
-		if (atomisp_hw_is_isp2401) {
+		if (IS_ISP2401) {
 			free_mpi |= stream->config.mode == IA_CSS_INPUT_MODE_TPG;
 			free_mpi |= stream->config.mode == IA_CSS_INPUT_MODE_PRBS;
 		}
@@ -9900,7 +9860,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) {
 		}
 
 		err = map_sp_threads(stream, false);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -9952,7 +9912,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) {
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_stream_get_info(const struct ia_css_stream *stream,
 			struct ia_css_stream_info *stream_info) {
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_info: enter/exit\n");
@@ -9960,7 +9920,7 @@ ia_css_stream_get_info(const struct ia_css_stream *stream,
 	assert(stream_info);
 
 	*stream_info = stream->info;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 /*
@@ -9969,12 +9929,11 @@ ia_css_stream_get_info(const struct ia_css_stream *stream,
     * The data is taken from the css_save struct updated upon stream creation.
     * The stream handle is used to identify the correct entry in the css_save struct
     */
-enum ia_css_err
+int
 ia_css_stream_load(struct ia_css_stream *stream) {
-
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		int i;
-		enum ia_css_err err;
+		int err;
 
 		assert(stream);
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_load() enter,\n");
@@ -9985,7 +9944,7 @@ ia_css_stream_load(struct ia_css_stream *stream) {
 
 				for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) {
 					if ((err = ia_css_pipe_create(&my_css_save.stream_seeds[i].pipe_config[j],
-								    &my_css_save.stream_seeds[i].pipes[j])) != IA_CSS_SUCCESS) {
+								    &my_css_save.stream_seeds[i].pipes[j])) != 0) {
 						if (j) {
 							int k;
 
@@ -9999,7 +9958,7 @@ ia_css_stream_load(struct ia_css_stream *stream) {
 							my_css_save.stream_seeds[i].num_pipes,
 							my_css_save.stream_seeds[i].pipes,
 							&my_css_save.stream_seeds[i].stream);
-				if (err != IA_CSS_SUCCESS) {
+				if (err) {
 					ia_css_stream_destroy(stream);
 					for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++)
 						ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[j]);
@@ -10009,23 +9968,23 @@ ia_css_stream_load(struct ia_css_stream *stream) {
 			}
 		}
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_load() exit,\n");
-		return IA_CSS_SUCCESS;
+		return 0;
 	} else {
 		/* TODO remove function - DEPRECATED */
 		(void)stream;
-		return IA_CSS_ERR_NOT_SUPPORTED;
+		return -ENOTSUPP;
 	}
 }
 
-enum ia_css_err
+int
 ia_css_stream_start(struct ia_css_stream *stream) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER("stream = %p", stream);
 	if ((!stream) || (!stream->last_pipe))
 	{
-		IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR(-EINVAL);
+		return -EINVAL;
 	}
 	IA_CSS_LOG("starting %d", stream->last_pipe->mode);
 
@@ -10033,7 +9992,7 @@ ia_css_stream_start(struct ia_css_stream *stream) {
 
 	/* Create host side pipeline. */
 	err = create_host_pipeline(stream);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 	{
 		IA_CSS_LEAVE_ERR(err);
 		return err;
@@ -10065,7 +10024,7 @@ ia_css_stream_start(struct ia_css_stream *stream) {
 	if (stream->config.mode != IA_CSS_INPUT_MODE_MEMORY)
 	{
 		err = sh_css_config_input_network(stream);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			return err;
 	}
 #endif /* !HAS_NO_INPUT_SYSTEM */
@@ -10075,9 +10034,9 @@ ia_css_stream_start(struct ia_css_stream *stream) {
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_stream_stop(struct ia_css_stream *stream) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_stop() enter/exit\n");
 	assert(stream);
@@ -10098,13 +10057,13 @@ ia_css_stream_stop(struct ia_css_stream *stream) {
 	}
 #endif
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		err = ia_css_pipeline_request_stop(&stream->last_pipe->pipeline);
 	} else {
 		err = sh_css_pipes_stop(stream);
 	}
 
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	/* Ideally, unmapping should happen after pipeline_stop, but current
@@ -10120,7 +10079,7 @@ ia_css_stream_has_stopped(struct ia_css_stream *stream) {
 
 	assert(stream);
 
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		stopped = ia_css_pipeline_has_stopped(&stream->last_pipe->pipeline);
 	} else {
 		stopped = sh_css_pipes_have_stopped(stream);
@@ -10134,7 +10093,7 @@ ia_css_stream_has_stopped(struct ia_css_stream *stream) {
     * Destroy the stream and all the pipes related to it.
     * The stream handle is used to identify the correct entry in the css_save struct
     */
-enum ia_css_err
+int
 ia_css_stream_unload(struct ia_css_stream *stream) {
 	int i;
 
@@ -10159,10 +10118,10 @@ ia_css_stream_unload(struct ia_css_stream *stream) {
 			break;
 		}
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE,	"ia_css_stream_unload() exit,\n");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
+int
 ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe,
 			    enum ia_css_pipe_id *pipe_id) {
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_temp_pipe_to_pipe_id() enter/exit\n");
@@ -10171,7 +10130,7 @@ ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe,
 	else
 		*pipe_id = IA_CSS_PIPE_ID_COPY;
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 enum atomisp_input_format
@@ -10243,7 +10202,7 @@ ia_css_stream_get_3a_binary(const struct ia_css_stream *stream) {
 	return s3a_binary;
 }
 
-enum ia_css_err
+int
 ia_css_stream_set_output_padded_width(struct ia_css_stream *stream,
 					unsigned int output_padded_width) {
 	struct ia_css_pipe *pipe;
@@ -10258,7 +10217,7 @@ ia_css_stream_set_output_padded_width(struct ia_css_stream *stream,
 	pipe->config.output_info[IA_CSS_PIPE_OUTPUT_STAGE_0].padded_width = output_padded_width;
 	pipe->output_info[IA_CSS_PIPE_OUTPUT_STAGE_0].padded_width = output_padded_width;
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static struct ia_css_binary *
@@ -10406,10 +10365,10 @@ ia_css_pipe_get_isp_pipe_version(const struct ia_css_pipe *pipe) {
 
 #define SP_START_TIMEOUT_US 30000000
 
-enum ia_css_err
+int
 ia_css_start_sp(void) {
 	unsigned long timeout;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER("");
 	sh_css_sp_start_isp();
@@ -10419,12 +10378,12 @@ ia_css_start_sp(void) {
 	while ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_INITIALIZED) && timeout)
 	{
 		timeout--;
-		hrt_sleep();
+		udelay(1);
 	}
 	if (timeout == 0)
 	{
 		IA_CSS_ERROR("timeout during SP initialization");
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
 
 	/* Workaround, in order to run two streams in parallel. See TASK 4271*/
@@ -10449,16 +10408,16 @@ ia_css_start_sp(void) {
     */
 #define SP_SHUTDOWN_TIMEOUT_US 200000
 
-enum ia_css_err
+int
 ia_css_stop_sp(void) {
 	unsigned long timeout;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER("void");
 
 	if (!sh_css_sp_is_running())
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE("SP already stopped : return_err=%d", err);
 
 		/* Return an error - stop SP should not have been called by driver */
@@ -10466,7 +10425,7 @@ ia_css_stop_sp(void) {
 	}
 
 	/* For now, stop whole SP */
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		sh_css_write_host2sp_command(host2sp_cmd_terminate);
 	} else {
 		if (!sh_css_write_host2sp_command(host2sp_cmd_terminate))
@@ -10483,7 +10442,7 @@ ia_css_stop_sp(void) {
 	while (!ia_css_spctrl_is_idle(SP0_ID) && timeout)
 	{
 		timeout--;
-		hrt_sleep();
+		udelay(1);
 	}
 	if ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_TERMINATED))
 		IA_CSS_WARNING("SP has not terminated (SW)");
@@ -10497,7 +10456,7 @@ ia_css_stop_sp(void) {
 	while (!isp_ctrl_getbit(ISP0_ID, ISP_SC_REG, ISP_IDLE_BIT) && timeout)
 	{
 		timeout--;
-		hrt_sleep();
+		udelay(1);
 	}
 	if (timeout == 0)
 	{
@@ -10514,7 +10473,7 @@ ia_css_stop_sp(void) {
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_update_continuous_frames(struct ia_css_stream *stream) {
 	struct ia_css_pipe *pipe;
 	unsigned int i;
@@ -10528,7 +10487,7 @@ ia_css_update_continuous_frames(struct ia_css_stream *stream) {
 		ia_css_debug_dtrace(
 		    IA_CSS_DEBUG_TRACE,
 		    "sh_css_update_continuous_frames() leave: invalid stream, return_void\n");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	pipe = stream->continuous_pipe;
@@ -10545,10 +10504,11 @@ ia_css_update_continuous_frames(struct ia_css_stream *stream) {
 	    IA_CSS_DEBUG_TRACE,
 	    "sh_css_update_continuous_frames() leave: return_void\n");
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map) {
+void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map)
+{
 	unsigned int thread_id;
 	enum ia_css_pipe_id pipe_id;
 	unsigned int pipe_num;
@@ -10664,7 +10624,7 @@ void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map) {
 }
 
 #if CONFIG_ON_FRAME_ENQUEUE()
-static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info
+static int set_config_on_frame_enqueue(struct ia_css_frame_info
 	*info, struct frame_data_wrapper *frame) {
 	frame->config_on_frame_enqueue.padded_width = 0;
 
@@ -10676,7 +10636,7 @@ static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info
 		if (info->padded_width > info->res.width) {
 			frame->config_on_frame_enqueue.padded_width = info->padded_width;
 		} else if ((info->padded_width < info->res.width) && (info->padded_width > 0)) {
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			return -EINVAL;
 		}
 		/* nothing to do if width == padded width or padded width is zeroed (the same) */
 		break;
@@ -10684,13 +10644,13 @@ static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info
 		break;
 	}
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 #endif
 
-enum ia_css_err
+int
 ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) {
-	enum ia_css_err ret;
+	int ret;
 
 	IA_CSS_ENTER("");
 
@@ -10699,14 +10659,14 @@ ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) {
 	if (!stream || !stream->config.continuous)
 	{
 		IA_CSS_ERROR("invalid stream pointer");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	if (exp_id > IA_CSS_ISYS_MAX_EXPOSURE_ID ||
 	    exp_id < IA_CSS_ISYS_MIN_EXPOSURE_ID)
 	{
-		IA_CSS_ERROR("invalid expsure ID: %d\n", exp_id);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_ERROR("invalid exposure ID: %d\n", exp_id);
+		return -EINVAL;
 	}
 
 	/* Send the event. Since we verified that the exp_id is valid,
@@ -10721,12 +10681,12 @@ ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) {
 /* @brief	Set the state (Enable or Disable) of the Extension stage in the
     *		given pipe.
     */
-enum ia_css_err
+int
 ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle,
 				bool enable) {
 	unsigned int thread_id;
 	struct ia_css_pipeline_stage *stage;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER("");
 
@@ -10734,28 +10694,28 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle,
 	if (!pipe || !pipe->stream)
 	{
 		IA_CSS_ERROR("Invalid Pipe.");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	} else if (!(pipe->config.acc_extension))
 	{
 		IA_CSS_ERROR("Invalid Pipe(No Extension Firmware)");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	} else if (!sh_css_sp_is_running())
 	{
 		IA_CSS_ERROR("Leaving: queue unavailable.");
-		err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		err = -EBUSY;
 	} else
 	{
 		/* Query the threadid and stage_num for the Extension firmware*/
 		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
 		err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage);
-		if (err == IA_CSS_SUCCESS) {
+		if (!err) {
 			/* Set the Extension State;. TODO: Add check for stage firmware.type (QOS)*/
 			err = ia_css_bufq_enqueue_psys_event(
 			    (uint8_t)IA_CSS_PSYS_SW_EVENT_STAGE_ENABLE_DISABLE,
 			    (uint8_t)thread_id,
 			    (uint8_t)stage->stage_num,
 			    enable ? 1 : 0);
-			if (err == IA_CSS_SUCCESS) {
+			if (!err) {
 				if (enable)
 					SH_CSS_QOS_STAGE_ENABLE(&sh_css_sp_group.pipe[thread_id], stage->stage_num);
 				else
@@ -10770,12 +10730,12 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle,
 /*	@brief	Get the state (Enable or Disable) of the Extension stage in the
     *	given pipe.
     */
-enum ia_css_err
+int
 ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle,
 				bool *enable) {
 	struct ia_css_pipeline_stage *stage;
 	unsigned int thread_id;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER("");
 
@@ -10783,22 +10743,22 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle,
 	if (!pipe || !pipe->stream)
 	{
 		IA_CSS_ERROR("Invalid Pipe.");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	} else if (!(pipe->config.acc_extension))
 	{
 		IA_CSS_ERROR("Invalid Pipe (No Extension Firmware).");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	} else if (!sh_css_sp_is_running())
 	{
 		IA_CSS_ERROR("Leaving: queue unavailable.");
-		err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		err = -EBUSY;
 	} else
 	{
 		/* Query the threadid and stage_num corresponding to the Extension firmware*/
 		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
 		err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage);
 
-		if (err == IA_CSS_SUCCESS) {
+		if (!err) {
 			/* Get the Extension State */
 			*enable = (SH_CSS_QOS_STAGE_IS_ENABLED(&sh_css_sp_group.pipe[thread_id],
 								stage->stage_num)) ? true : false;
@@ -10809,9 +10769,9 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle,
 }
 
 /* ISP2401 */
-enum ia_css_err
+int
 ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe,
-					uint32_t fw_handle,
+					u32 fw_handle,
 					struct ia_css_isp_param_css_segments *css_seg,
 					struct ia_css_isp_param_isp_segments *isp_seg) {
 	unsigned int HIVE_ADDR_sp_group;
@@ -10821,7 +10781,7 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe,
 	const struct ia_css_fw_info *fw;
 	unsigned int thread_id;
 	struct ia_css_pipeline_stage *stage;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	int stage_num = 0;
 	enum ia_css_isp_memories mem;
 	bool enabled;
@@ -10834,28 +10794,28 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe,
 	if (!pipe || !pipe->stream)
 	{
 		IA_CSS_ERROR("Invalid Pipe.");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	} else if (!(pipe->config.acc_extension))
 	{
 		IA_CSS_ERROR("Invalid Pipe (No Extension Firmware).");
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	} else if (!sh_css_sp_is_running())
 	{
 		IA_CSS_ERROR("Leaving: queue unavailable.");
-		err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+		err = -EBUSY;
 	} else
 	{
 		/* Query the thread_id and stage_num corresponding to the Extension firmware */
 		ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id);
 		err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage);
-		if (err == IA_CSS_SUCCESS) {
+		if (!err) {
 			/* Get the Extension State */
 			enabled = (SH_CSS_QOS_STAGE_IS_ENABLED(&sh_css_sp_group.pipe[thread_id],
 								stage->stage_num)) ? true : false;
 			/* Update mapped arg only when extension stage is not enabled */
 			if (enabled) {
 				IA_CSS_ERROR("Leaving: cannot update when stage is enabled.");
-				err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+				err = -EBUSY;
 			} else {
 				stage_num = stage->stage_num;
 
@@ -10863,10 +10823,10 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe,
 				sp_dmem_load(SP0_ID,
 						(unsigned int)sp_address_of(sp_group),
 						&sp_group, sizeof(struct sh_css_sp_group));
-				mmgr_load(sp_group.pipe[thread_id].sp_stage_addr[stage_num],
+				hmm_load(sp_group.pipe[thread_id].sp_stage_addr[stage_num],
 					    &sp_stage, sizeof(struct sh_css_sp_stage));
 
-				mmgr_load(sp_stage.isp_stage_addr,
+				hmm_load(sp_stage.isp_stage_addr,
 					    &isp_stage, sizeof(struct sh_css_isp_stage));
 
 				for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++) {
@@ -10882,7 +10842,7 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe,
 						isp_seg->params[IA_CSS_PARAM_CLASS_PARAM][mem].size;
 				}
 
-				mmgr_store(sp_stage.isp_stage_addr,
+				hmm_store(sp_stage.isp_stage_addr,
 					    &isp_stage, sizeof(struct sh_css_isp_stage));
 			}
 		}
@@ -10892,11 +10852,11 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe,
 }
 
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
-static enum ia_css_err
+static int
 aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
 			struct ia_css_pipe *pipes[],
 			bool *do_crop_status) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	int i;
 	struct ia_css_pipe *curr_pipe;
 	u32 pipe_mask = 0;
@@ -10906,7 +10866,7 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
 	    (!pipes) ||
 	    (!do_crop_status))
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
@@ -10922,7 +10882,7 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
 	    (pipe_mask & (1 << IA_CSS_PIPE_MODE_VIDEO))) &&
 	    (pipe_mask & (1 << IA_CSS_PIPE_MODE_CAPTURE)) &&
 	    curr_stream->config.continuous);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static bool
@@ -10939,10 +10899,10 @@ aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe) {
 	return status;
 }
 
-static enum ia_css_err
+static int
 aspect_ratio_crop(struct ia_css_pipe *curr_pipe,
 		    struct ia_css_resolution *effective_res) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_resolution crop_res;
 	struct ia_css_resolution *in_res = NULL;
 	struct ia_css_resolution *out_res = NULL;
@@ -10953,7 +10913,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe,
 	if ((!curr_pipe) ||
 	    (!effective_res))
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
@@ -10962,7 +10922,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe,
 	    (curr_pipe->config.mode != IA_CSS_PIPE_MODE_VIDEO) &&
 	    (curr_pipe->config.mode != IA_CSS_PIPE_MODE_CAPTURE))
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR(err);
 		return err;
 	}
@@ -11009,7 +10969,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe,
 	}
 
 	err = ia_css_frame_find_crop_resolution(in_res, out_res, &crop_res);
-	if (err == IA_CSS_SUCCESS)
+	if (!err)
 	{
 		*effective_res = crop_res;
 	} else
@@ -11085,7 +11045,7 @@ static struct sh_css_hmm_buffer_record
 }
 
 static struct sh_css_hmm_buffer_record
-*sh_css_hmm_buffer_record_validate(hrt_vaddress ddr_buffer_addr,
+*sh_css_hmm_buffer_record_validate(ia_css_ptr ddr_buffer_addr,
 				    enum ia_css_buffer_type type) {
 	int i;
 	struct sh_css_hmm_buffer_record *buffer_record = NULL;
diff --git a/drivers/staging/media/atomisp/pci/sh_css_defs.h b/drivers/staging/media/atomisp/pci/sh_css_defs.h
index fcd5081edf82..92d80213860f 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_defs.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -177,7 +178,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191]
 #define SH_CSS_MORPH_TABLE_ELEMS_PER_DDR_WORD \
 	(HIVE_ISP_DDR_WORD_BYTES / SH_CSS_MORPH_TABLE_ELEM_BYTES)
 
-
 #define ISP2400_SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR   (SH_CSS_MAX_BQ_GRID_WIDTH + 1)
 #define ISP2400_SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR   (SH_CSS_MAX_BQ_GRID_HEIGHT + 1)
 
@@ -225,8 +225,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191]
 
 #define NUM_VIDEO_TNR_FRAMES		2
 
-#define MAX_NUM_DELAY_FRAMES	MAX(MAX_NUM_VIDEO_DELAY_FRAMES, NUM_PREVIEW_DVS_FRAMES)
-
 /* Note that this is the define used to configure all data structures common for all modes */
 /* It should be equal or bigger to the max number of DVS frames for all possible modes */
 /* Rules: these implement logic shared between the host code and ISP firmware.
@@ -277,7 +275,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191]
 	(ISP_BQ_GRID_HEIGHT(input_height, deci_factor_log2) + \
 	SH_CSS_SCTBL_CENTERING_MARGIN + SH_CSS_SCTBL_LAST_GRID_COUNT)
 
-
 /* ISP2401: Legacy API: Number of horizontal grids per color in the shading table. */
 #define _ISP_SCTBL_LEGACY_WIDTH_PER_COLOR(input_width, deci_factor_log2) \
 	(ISP_BQ_GRID_WIDTH(input_width, deci_factor_log2) + SH_CSS_SCTBL_LAST_GRID_COUNT)
@@ -286,7 +283,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191]
 #define _ISP_SCTBL_LEGACY_HEIGHT(input_height, deci_factor_log2) \
 	(ISP_BQ_GRID_HEIGHT(input_height, deci_factor_log2) + SH_CSS_SCTBL_LAST_GRID_COUNT)
 
-
 /* *****************************************************************
  * Statistics for 3A (Auto Focus, Auto White Balance, Auto Exposure)
  * *****************************************************************/
diff --git a/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h b/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h
index 23044aad654f..6f058f132300 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
 Support for Intel Camera Imaging ISP subsystem.
 Copyright (c) 2010 - 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_firmware.c b/drivers/staging/media/atomisp/pci/sh_css_firmware.c
index eb3c01574853..d4ab15b6d1ac 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_firmware.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_firmware.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,9 +13,12 @@
  * more details.
  */
 
+#include <linux/string.h> /* for memcpy() */
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 
+#include "hmm.h"
+
 #include <math_support.h>
 #include "platform_support.h"
 #include "sh_css_firmware.h"
@@ -24,9 +28,7 @@
 #include "sh_css_internal.h"
 #include "ia_css_isp_param.h"
 
-#include "memory_access.h"
 #include "assert_support.h"
-#include "string_support.h"
 
 #include "isp.h"				/* PMEM_WIDTH_LOG2 */
 
@@ -74,13 +76,13 @@ char *sh_css_get_fw_version(void)
  */
 
 /* Setup sp/sp1 binary */
-static enum ia_css_err
+static int
 setup_binary(struct ia_css_fw_info *fw, const char *fw_data,
 	     struct ia_css_fw_info *sh_css_fw, unsigned int binary_id) {
 	const char *blob_data;
 
 	if ((!fw) || (!fw_data))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	blob_data = fw_data + fw->blob.offset;
 
@@ -88,16 +90,16 @@ setup_binary(struct ia_css_fw_info *fw, const char *fw_data,
 
 	sh_css_fw->blob.code = vmalloc(fw->blob.size);
 	if (!sh_css_fw->blob.code)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	memcpy((void *)sh_css_fw->blob.code, blob_data, fw->blob.size);
 	sh_css_fw->blob.data = (char *)sh_css_fw->blob.code + fw->blob.data_source;
 	fw_minibuffer[binary_id].buffer = sh_css_fw->blob.code;
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
+int
 sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
 		      struct ia_css_blob_descr *bd,
 		      unsigned int index) {
@@ -105,7 +107,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
 	const unsigned char *blob;
 
 	if ((!fw) || (!bd))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	/* Special case: only one binary in fw */
 	if (!bi) bi = (const struct ia_css_fw_info *)fw;
@@ -117,11 +119,11 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
 	if (bi->blob.size != bi->blob.text_size + bi->blob.icache_size + bi->blob.data_size + bi->blob.padding_size)
 	{
 		/* sanity check, note the padding bytes added for section to DDR alignment */
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	if ((bi->blob.offset % (1UL << (ISP_PMEM_WIDTH_LOG2 - 3))) != 0)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	bd->blob = blob;
 	bd->header = *bi;
@@ -132,7 +134,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
 
 		namebuffer = kstrdup(name, GFP_KERNEL);
 		if (!namebuffer)
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			return -ENOMEM;
 		bd->name = fw_minibuffer[index].name = namebuffer;
 	} else
 	{
@@ -149,7 +151,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
 					 statestruct_size,
 					 GFP_KERNEL);
 		if (!parambuf)
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			return -ENOMEM;
 
 		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_PARAM].ptr = NULL;
 		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_CONFIG].ptr = NULL;
@@ -177,7 +179,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
 		bd->mem_offsets.array[IA_CSS_PARAM_CLASS_STATE].ptr = parambuf +
 		paramstruct_size + configstruct_size;
 	}
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 bool
@@ -187,7 +189,7 @@ sh_css_check_firmware_version(struct device *dev, const char *fw_data)
 
 	const char *release_version;
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		release_version = isp2400_release_version;
 	else
 		release_version = isp2401_release_version;
@@ -202,19 +204,33 @@ sh_css_check_firmware_version(struct device *dev, const char *fw_data)
 	}
 
 	/* For now, let's just accept a wrong version, even if wrong */
-	return true;
+	return 0;
 }
 
-enum ia_css_err
+static const char * const fw_type_name[] = {
+	[ia_css_sp_firmware]		= "SP",
+	[ia_css_isp_firmware]		= "ISP",
+	[ia_css_bootloader_firmware]	= "BootLoader",
+	[ia_css_acc_firmware]		= "accel",
+};
+
+static const char * const fw_acc_type_name[] = {
+	[IA_CSS_ACC_NONE] =		"Normal",
+	[IA_CSS_ACC_OUTPUT] =		"Accel for output",
+	[IA_CSS_ACC_VIEWFINDER] =	"Accel for viewfinder",
+	[IA_CSS_ACC_STANDALONE] =	"Stand-alone accel",
+};
+
+int
 sh_css_load_firmware(struct device *dev, const char *fw_data,
 		     unsigned int fw_size) {
 	unsigned int i;
 	struct ia_css_fw_info *binaries;
 	struct sh_css_fw_bi_file_h *file_header;
-	bool valid_firmware = false;
+	int ret;
 	const char *release_version;
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		release_version = isp2400_release_version;
 	else
 		release_version = isp2401_release_version;
@@ -223,21 +239,21 @@ sh_css_load_firmware(struct device *dev, const char *fw_data,
 	file_header = &firmware_header->file_header;
 	binaries = &firmware_header->binary_header;
 	strscpy(FW_rel_ver_name, file_header->version, min(sizeof(FW_rel_ver_name), sizeof(file_header->version)));
-	valid_firmware = sh_css_check_firmware_version(dev, fw_data);
-	if (!valid_firmware) {
+	ret = sh_css_check_firmware_version(dev, fw_data);
+	if (ret) {
 		IA_CSS_ERROR("CSS code version (%s) and firmware version (%s) mismatch!",
 			     file_header->version, release_version);
-		return IA_CSS_ERR_VERSION_MISMATCH;
+		return -EINVAL;
 	} else {
 		IA_CSS_LOG("successfully load firmware version %s", release_version);
 	}
 
 	/* some sanity checks */
 	if (!fw_data || fw_size < sizeof(struct sh_css_fw_bi_file_h))
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 
 	if (file_header->h_size != sizeof(struct sh_css_fw_bi_file_h))
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 
 	sh_css_num_binaries = file_header->binary_nr;
 	/* Only allocate memory for ISP blob info */
@@ -247,7 +263,7 @@ sh_css_load_firmware(struct device *dev, const char *fw_data,
 		    (sh_css_num_binaries - NUM_OF_SPS) *
 		    sizeof(*sh_css_blob_info), GFP_KERNEL);
 		if (!sh_css_blob_info)
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			return -ENOMEM;
 	} else {
 		sh_css_blob_info = NULL;
 	}
@@ -255,7 +271,7 @@ sh_css_load_firmware(struct device *dev, const char *fw_data,
 	fw_minibuffer = kcalloc(sh_css_num_binaries, sizeof(struct fw_param),
 				GFP_KERNEL);
 	if (!fw_minibuffer)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	for (i = 0; i < sh_css_num_binaries; i++)
 	{
@@ -265,36 +281,71 @@ sh_css_load_firmware(struct device *dev, const char *fw_data,
 		   cause issues for drivers
 		*/
 		static struct ia_css_blob_descr bd;
-		enum ia_css_err err;
+		int err;
 
 		err = sh_css_load_blob_info(fw_data, bi, &bd, i);
 
-		if (err != IA_CSS_SUCCESS)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+		if (err)
+			return -EINVAL;
 
 		if (bi->blob.offset + bi->blob.size > fw_size)
-			return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
+
+		switch (bd.header.type) {
+		case ia_css_isp_firmware:
+			if (bd.header.info.isp.type > IA_CSS_ACC_STANDALONE) {
+				dev_err(dev, "binary #%2d: invalid SP type\n",
+					i);
+				return -EINVAL;
+			}
+
+			dev_dbg(dev,
+				"binary #%-2d type %s (%s), binary id is %2d: %s\n",
+				i,
+				fw_type_name[bd.header.type],
+				fw_acc_type_name[bd.header.info.isp.type],
+				bd.header.info.isp.sp.id,
+				bd.name);
+			break;
+		case ia_css_sp_firmware:
+		case ia_css_bootloader_firmware:
+		case ia_css_acc_firmware:
+			dev_dbg(dev,
+				"binary #%-2d type %s: %s\n",
+				i, fw_type_name[bd.header.type],
+				bd.name);
+			break;
+		default:
+			if (bd.header.info.isp.type > IA_CSS_ACC_STANDALONE) {
+				dev_err(dev,
+					"binary #%2d: invalid firmware type\n",
+					i);
+				return -EINVAL;
+			}
+			break;
+		}
 
 		if (bi->type == ia_css_sp_firmware) {
 			if (i != SP_FIRMWARE)
-				return IA_CSS_ERR_INTERNAL_ERROR;
+				return -EINVAL;
 			err = setup_binary(bi, fw_data, &sh_css_sp_fw, i);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				return err;
+
 		} else {
 			/* All subsequent binaries (including bootloaders) (i>NUM_OF_SPS) are ISP firmware */
 			if (i < NUM_OF_SPS)
-				return IA_CSS_ERR_INTERNAL_ERROR;
+				return -EINVAL;
 
 			if (bi->type != ia_css_isp_firmware)
-				return IA_CSS_ERR_INTERNAL_ERROR;
+				return -EINVAL;
 			if (!sh_css_blob_info) /* cannot happen but KW does not see this */
-				return IA_CSS_ERR_INTERNAL_ERROR;
+				return -EINVAL;
 			sh_css_blob_info[i - NUM_OF_SPS] = bd;
 		}
 	}
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 void sh_css_unload_firmware(void)
@@ -319,15 +370,15 @@ void sh_css_unload_firmware(void)
 	sh_css_num_binaries = 0;
 }
 
-hrt_vaddress
+ia_css_ptr
 sh_css_load_blob(const unsigned char *blob, unsigned int size)
 {
-	hrt_vaddress target_addr = mmgr_malloc(size);
+	ia_css_ptr target_addr = hmm_alloc(size, HMM_BO_PRIVATE, 0, NULL, 0);
 	/* this will allocate memory aligned to a DDR word boundary which
 	   is required for the CSS DMA to read the instructions. */
 
 	assert(blob);
 	if (target_addr)
-		mmgr_store(target_addr, blob, size);
+		hmm_store(target_addr, blob, size);
 	return target_addr;
 }
diff --git a/drivers/staging/media/atomisp/pci/sh_css_firmware.h b/drivers/staging/media/atomisp/pci/sh_css_firmware.h
index f6253392a6c9..66cd38f08f71 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_firmware.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_firmware.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -15,7 +16,7 @@
 #ifndef _SH_CSS_FIRMWARE_H_
 #define _SH_CSS_FIRMWARE_H_
 
-#include <system_types.h>
+#include <system_local.h>
 
 #include <ia_css_err.h>
 #include <ia_css_acc_types.h>
@@ -37,18 +38,19 @@ extern unsigned int sh_css_num_binaries;
 char
 *sh_css_get_fw_version(void);
 
+struct device;
 bool
 sh_css_check_firmware_version(struct device *dev, const char *fw_data);
 
-enum ia_css_err
+int
 sh_css_load_firmware(struct device *dev, const char *fw_data,
 		     unsigned int fw_size);
 
 void sh_css_unload_firmware(void);
 
-hrt_vaddress sh_css_load_blob(const unsigned char *blob, unsigned int size);
+ia_css_ptr sh_css_load_blob(const unsigned char *blob, unsigned int size);
 
-enum ia_css_err
+int
 sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
 		      struct ia_css_blob_descr *bd, unsigned int i);
 
diff --git a/drivers/staging/media/atomisp/pci/sh_css_frac.h b/drivers/staging/media/atomisp/pci/sh_css_frac.h
index cd2d755ec523..8f08df5c88cc 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_frac.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_frac.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_host_data.c b/drivers/staging/media/atomisp/pci/sh_css_host_data.c
index 348183a221a8..39a9b9812682 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_host_data.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_host_data.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -24,7 +25,7 @@ struct ia_css_host_data *ia_css_host_data_allocate(size_t size)
 	if (!me)
 		return NULL;
 	me->size = (uint32_t)size;
-	me->address = sh_css_malloc(size);
+	me->address = kvmalloc(size, GFP_KERNEL);
 	if (!me->address) {
 		kfree(me);
 		return NULL;
@@ -35,7 +36,7 @@ struct ia_css_host_data *ia_css_host_data_allocate(size_t size)
 void ia_css_host_data_free(struct ia_css_host_data *me)
 {
 	if (me) {
-		sh_css_free(me->address);
+		kvfree(me->address);
 		me->address = NULL;
 		kfree(me);
 	}
diff --git a/drivers/staging/media/atomisp/pci/sh_css_hrt.c b/drivers/staging/media/atomisp/pci/sh_css_hrt.c
index 94b2de5b5ef4..06b502151af9 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_hrt.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_hrt.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -61,7 +62,7 @@ bool sh_css_hrt_system_is_idle(void)
 	return !not_idle;
 }
 
-enum ia_css_err sh_css_hrt_sp_wait(void)
+int sh_css_hrt_sp_wait(void)
 {
 #if defined(HAS_IRQ_MAP_VERSION_2)
 	irq_sw_channel_id_t	irq_id = IRQ_SW_CHANNEL0_ID;
@@ -78,8 +79,8 @@ enum ia_css_err sh_css_hrt_sp_wait(void)
 	       ((irq_reg_load(IRQ0_ID,
 			      _HRT_IRQ_CONTROLLER_STATUS_REG_IDX) &
 		 (1U << (irq_id + IRQ_SW_CHANNEL_OFFSET))) == 0)) {
-		hrt_sleep();
+		udelay(1);
 	}
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/sh_css_hrt.h b/drivers/staging/media/atomisp/pci/sh_css_hrt.h
index fd23ed1848ec..168bbd579798 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_hrt.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_hrt.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -27,7 +28,7 @@ void sh_css_hrt_sp_start_copy_frame(void);
 
 void sh_css_hrt_sp_start_isp(void);
 
-enum ia_css_err sh_css_hrt_sp_wait(void);
+int sh_css_hrt_sp_wait(void);
 
 bool sh_css_hrt_system_is_idle(void);
 
diff --git a/drivers/staging/media/atomisp/pci/sh_css_internal.h b/drivers/staging/media/atomisp/pci/sh_css_internal.h
index 5f271d9ae485..5c25a25dce92 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_internal.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_internal.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -75,11 +76,7 @@
 #define SH_CSS_REF_BIT_DEPTH 8
 
 /* keep next up to date with the definition for MAX_CB_ELEMS_FOR_TAGGER in tagger.sp.c */
-#if defined(HAS_SP_2400)
 #define NUM_CONTINUOUS_FRAMES	15
-#else
-#define NUM_CONTINUOUS_FRAMES	10
-#endif
 #define NUM_MIPI_FRAMES_PER_STREAM		2
 
 #define NUM_ONLINE_INIT_CONTINUOUS_FRAMES      2
@@ -209,28 +206,28 @@ enum sh_css_sp_event_type {
 };
 
 /* xmem address map allocation per pipeline, css pointers */
-/* Note that the struct below should only consist of hrt_vaddress-es
+/* Note that the struct below should only consist of ia_css_ptr-es
    Otherwise this will cause a fail in the function ref_sh_css_ddr_address_map
  */
 struct sh_css_ddr_address_map {
-	hrt_vaddress isp_param;
-	hrt_vaddress isp_mem_param[SH_CSS_MAX_STAGES][IA_CSS_NUM_MEMORIES];
-	hrt_vaddress macc_tbl;
-	hrt_vaddress fpn_tbl;
-	hrt_vaddress sc_tbl;
-	hrt_vaddress tetra_r_x;
-	hrt_vaddress tetra_r_y;
-	hrt_vaddress tetra_gr_x;
-	hrt_vaddress tetra_gr_y;
-	hrt_vaddress tetra_gb_x;
-	hrt_vaddress tetra_gb_y;
-	hrt_vaddress tetra_b_x;
-	hrt_vaddress tetra_b_y;
-	hrt_vaddress tetra_ratb_x;
-	hrt_vaddress tetra_ratb_y;
-	hrt_vaddress tetra_batr_x;
-	hrt_vaddress tetra_batr_y;
-	hrt_vaddress dvs_6axis_params_y;
+	ia_css_ptr isp_param;
+	ia_css_ptr isp_mem_param[SH_CSS_MAX_STAGES][IA_CSS_NUM_MEMORIES];
+	ia_css_ptr macc_tbl;
+	ia_css_ptr fpn_tbl;
+	ia_css_ptr sc_tbl;
+	ia_css_ptr tetra_r_x;
+	ia_css_ptr tetra_r_y;
+	ia_css_ptr tetra_gr_x;
+	ia_css_ptr tetra_gr_y;
+	ia_css_ptr tetra_gb_x;
+	ia_css_ptr tetra_gb_y;
+	ia_css_ptr tetra_b_x;
+	ia_css_ptr tetra_b_y;
+	ia_css_ptr tetra_ratb_x;
+	ia_css_ptr tetra_ratb_y;
+	ia_css_ptr tetra_batr_x;
+	ia_css_ptr tetra_batr_y;
+	ia_css_ptr dvs_6axis_params_y;
 };
 
 #define SIZE_OF_SH_CSS_DDR_ADDRESS_MAP_STRUCT					\
@@ -279,9 +276,9 @@ struct ia_css_isp_parameter_set_info {
    a binary. It depends on the binary which ones are used. */
 struct sh_css_binary_args {
 	struct ia_css_frame *in_frame;	     /* input frame */
-	struct ia_css_frame
+	const struct ia_css_frame
 		*delay_frames[MAX_NUM_VIDEO_DELAY_FRAMES];   /* reference input frame */
-	struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES];   /* tnr frames */
+	const struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES];   /* tnr frames */
 	struct ia_css_frame
 		*out_frame[IA_CSS_BINARY_MAX_OUTPUT_PORTS];      /* output frame */
 	struct ia_css_frame *out_vf_frame;   /* viewfinder output frame */
@@ -531,8 +528,8 @@ struct sh_css_sp_pipeline {
 	u32	port_id;	/* port_id for input system */
 	u32	num_stages;		/* the pipe config */
 	u32	running;	/* needed for pipe termination */
-	hrt_vaddress	sp_stage_addr[SH_CSS_MAX_STAGES];
-	hrt_vaddress	scaler_pp_lut; /* Early bound LUT */
+	ia_css_ptr	sp_stage_addr[SH_CSS_MAX_STAGES];
+	ia_css_ptr	scaler_pp_lut; /* Early bound LUT */
 	u32	dummy; /* stage ptr is only used on sp but lives in
 				  this struct; needs cleanup */
 	s32 num_execs; /* number of times to run if this is
@@ -544,7 +541,7 @@ struct sh_css_sp_pipeline {
 		u32        height;   /* Number of lines */
 		u32        stride;   /* Stride (in bytes) per line */
 		u32        size;     /* Total size (in bytes) */
-		hrt_vaddress    cont_buf; /* Address of continuous buffer */
+		ia_css_ptr    cont_buf; /* Address of continuous buffer */
 	} metadata;
 #endif
 #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
@@ -657,9 +654,9 @@ struct sh_css_sp_stage {
 	struct ia_css_frames_sp		frames;
 	struct ia_css_resolution	dvs_envelope;
 	struct sh_css_uds_info		uds;
-	hrt_vaddress			isp_stage_addr;
-	hrt_vaddress			xmem_bin_addr;
-	hrt_vaddress			xmem_map_addr;
+	ia_css_ptr			isp_stage_addr;
+	ia_css_ptr			xmem_bin_addr;
+	ia_css_ptr			xmem_map_addr;
 
 	u16		top_cropping;
 	u16		row_stripes_height;
@@ -692,7 +689,7 @@ struct sh_css_sp_group {
 /* Data in SP dmem that is set from the host every stage. */
 struct sh_css_sp_per_frame_data {
 	/* ddr address of sp_group and sp_stage */
-	hrt_vaddress			sp_group_addr;
+	ia_css_ptr			sp_group_addr;
 };
 
 #define SH_CSS_NUM_SDW_IRQS 3
@@ -728,25 +725,19 @@ struct sh_css_sp_output {
  * separate SP thread for this. */
 #define  IA_CSS_NUM_ELEMS_HOST2SP_ISYS_EVENT_QUEUE (2 * N_CSI_PORTS)
 
-#if defined(HAS_SP_2400)
 #define  IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE    13
 #define  IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE        19
 #define  IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE    26 /* holds events for all type of buffers, hence deeper */
-#else
-#define  IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE    6
-#define  IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE        6
-#define  IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE    6
-#endif
 
 struct sh_css_hmm_buffer {
 	union {
 		struct ia_css_isp_3a_statistics  s3a;
 		struct ia_css_isp_dvs_statistics dis;
-		hrt_vaddress skc_dvs_statistics;
-		hrt_vaddress lace_stat;
+		ia_css_ptr skc_dvs_statistics;
+		ia_css_ptr lace_stat;
 		struct ia_css_metadata	metadata;
 		struct frame_data_wrapper {
-			hrt_vaddress	frame_data;
+			ia_css_ptr	frame_data;
 			u32	flashed;
 			u32	exp_id;
 			u32	isp_parameters_id; /** Unique ID to track which config was
@@ -755,7 +746,7 @@ struct sh_css_hmm_buffer {
 			struct sh_css_config_on_frame_enqueue config_on_frame_enqueue;
 #endif
 		} frame;
-		hrt_vaddress ddr_ptrs;
+		ia_css_ptr ddr_ptrs;
 	} payload;
 	/*
 	 * kernel_ptr is present for host administration purposes only.
@@ -834,12 +825,12 @@ struct host_sp_communication {
 	 * TODO:
 	 *   Remove it when the Host and the SP is decoupled.
 	 */
-	hrt_vaddress host2sp_offline_frames[NUM_CONTINUOUS_FRAMES];
-	hrt_vaddress host2sp_offline_metadata[NUM_CONTINUOUS_FRAMES];
+	ia_css_ptr host2sp_offline_frames[NUM_CONTINUOUS_FRAMES];
+	ia_css_ptr host2sp_offline_metadata[NUM_CONTINUOUS_FRAMES];
 
 #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	hrt_vaddress host2sp_mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
-	hrt_vaddress host2sp_mipi_metadata[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
+	ia_css_ptr host2sp_mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
+	ia_css_ptr host2sp_mipi_metadata[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
 	u32 host2sp_num_mipi_frames[N_CSI_PORTS];
 #endif
 	u32 host2sp_cont_avail_num_raw_frames;
@@ -960,20 +951,14 @@ sh_css_vprint(const char *fmt, va_list args)
    issue with the firmware struct/union's.
    More permanent solution will be to refactor this include.
 */
-hrt_vaddress sh_css_params_ddr_address_map(void);
+ia_css_ptr sh_css_params_ddr_address_map(void);
 
-enum ia_css_err
+int
 sh_css_params_init(void);
 
 void
 sh_css_params_uninit(void);
 
-void *sh_css_malloc(size_t size);
-
-void *sh_css_calloc(size_t N, size_t size);
-
-void sh_css_free(void *ptr);
-
 /* For Acceleration API: Flush FW (shared buffer pointer) arguments */
 void sh_css_flush(struct ia_css_acc_fw *fw);
 
@@ -1010,13 +995,13 @@ sh_css_get_mipi_sizes_for_check(const unsigned int port,
 
 #endif
 
-hrt_vaddress
+ia_css_ptr
 sh_css_store_sp_group_to_ddr(void);
 
-hrt_vaddress
+ia_css_ptr
 sh_css_store_sp_stage_to_ddr(unsigned int pipe, unsigned int stage);
 
-hrt_vaddress
+ia_css_ptr
 sh_css_store_isp_stage_to_ddr(unsigned int pipe, unsigned int stage);
 
 void
diff --git a/drivers/staging/media/atomisp/pci/sh_css_legacy.h b/drivers/staging/media/atomisp/pci/sh_css_legacy.h
index 99ac690ba7aa..567c8d6dcc2c 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_legacy.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_legacy.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -45,7 +46,7 @@ struct ia_css_pipe_extra_config {
 	bool disable_vf_pp;
 };
 
-enum ia_css_err
+int
 ia_css_pipe_create_extra(const struct ia_css_pipe_config *config,
 			 const struct ia_css_pipe_extra_config *extra_config,
 			 struct ia_css_pipe **pipe);
@@ -54,12 +55,12 @@ void
 ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config
 				  *extra_config);
 
-enum ia_css_err
+int
 ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe,
 			    enum ia_css_pipe_id *pipe_id);
 
 /* DEPRECATED. FPN is not supported. */
-enum ia_css_err
+int
 sh_css_set_black_frame(struct ia_css_stream *stream,
 		       const struct ia_css_frame *raw_black_frame);
 
diff --git a/drivers/staging/media/atomisp/pci/sh_css_metadata.c b/drivers/staging/media/atomisp/pci/sh_css_metadata.c
index ebdf84d4a138..04a4b7da85e2 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_metadata.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_metadata.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_metrics.c b/drivers/staging/media/atomisp/pci/sh_css_metrics.c
index 17f6dd9afab4..9744bbebe1bc 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_metrics.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_metrics.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -67,13 +68,16 @@ make_histogram(struct sh_css_pc_histogram *histogram, unsigned int length)
 		return;
 	if (histogram->run)
 		return;
-	histogram->run = sh_css_malloc(length * sizeof(*histogram->run));
+	histogram->run = kvmalloc(length * sizeof(*histogram->run),
+				  GFP_KERNEL);
 	if (!histogram->run)
 		return;
-	histogram->stall = sh_css_malloc(length * sizeof(*histogram->stall));
+	histogram->stall = kvmalloc(length * sizeof(*histogram->stall),
+				    GFP_KERNEL);
 	if (!histogram->stall)
 		return;
-	histogram->msink = sh_css_malloc(length * sizeof(*histogram->msink));
+	histogram->msink = kvmalloc(length * sizeof(*histogram->msink),
+				    GFP_KERNEL);
 	if (!histogram->msink)
 		return;
 
diff --git a/drivers/staging/media/atomisp/pci/sh_css_metrics.h b/drivers/staging/media/atomisp/pci/sh_css_metrics.h
index f465d1545b8b..f4bcd08384e9 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_metrics.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_metrics.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_mipi.c b/drivers/staging/media/atomisp/pci/sh_css_mipi.c
index 35cbef5f9f71..2ef5dbd62a6d 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_mipi.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_mipi.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -34,10 +35,10 @@ static u32
 ref_count_mipi_allocation[N_CSI_PORTS]; /* Initialized in mipi_init */
 #endif
 
-enum ia_css_err
+int
 ia_css_mipi_frame_specify(const unsigned int size_mem_words,
 			  const bool contiguous) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	my_css.size_mem_words = size_mem_words;
 	(void)contiguous;
@@ -97,14 +98,14 @@ static bool ia_css_mipi_is_source_port_valid(struct ia_css_pipe *pipe,
  *  etc.).
  * Result is given in DDR mem words, 32B or 256 bits
  */
-enum ia_css_err
+int
 ia_css_mipi_frame_calculate_size(const unsigned int width,
 				 const unsigned int height,
 				 const enum atomisp_input_format format,
 				 const bool hasSOLandEOL,
 				 const unsigned int embedded_data_size_words,
 				 unsigned int *size_mem_words) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	unsigned int bits_per_pixel = 0;
 	unsigned int even_line_bytes = 0;
@@ -182,7 +183,7 @@ ia_css_mipi_frame_calculate_size(const unsigned int width,
 	case ATOMISP_INPUT_FORMAT_YUV422_16:		/* Not supported */
 	case ATOMISP_INPUT_FORMAT_RAW_16:		/* TODO: not specified in MIPI SPEC, check */
 	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	odd_line_bytes = (width_padded * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */
@@ -246,12 +247,12 @@ ia_css_mipi_frame_calculate_size(const unsigned int width,
 }
 
 #if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
-enum ia_css_err
+int
 ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
 				       const unsigned int	size_mem_words) {
 	u32 idx;
 
-	enum ia_css_err err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+	int err = -EBUSY;
 
 	OP___assert(port < N_CSI_PORTS);
 	OP___assert(size_mem_words != 0);
@@ -264,7 +265,7 @@ ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
 	if (idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT)
 	{
 		my_css.mipi_sizes_for_check[port][idx] = size_mem_words;
-		err = IA_CSS_SUCCESS;
+		err = 0;
 	}
 
 	return err;
@@ -282,12 +283,12 @@ mipi_init(void)
 #endif
 }
 
-enum ia_css_err
+int
 calculate_mipi_buff_size(
     struct ia_css_stream_config *stream_cfg,
     unsigned int *size_mem_words) {
 #if !defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
+	int err = -EINVAL;
 	(void)stream_cfg;
 	(void)size_mem_words;
 #else
@@ -310,7 +311,7 @@ calculate_mipi_buff_size(
 
 	unsigned int mem_words_per_buff_line = 0;
 	unsigned int mem_words_per_buff = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	/**
 	 * zhengjie.lu@intel.com
@@ -354,7 +355,7 @@ calculate_mipi_buff_size(
 	bits_per_pixel =
 	(format == ATOMISP_INPUT_FORMAT_RAW_10 && pack_raw_pixels) ? bits_per_pixel : 16;
 	if (bits_per_pixel == 0)
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 
 	odd_line_bytes = (width_padded * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */
 
@@ -390,7 +391,7 @@ calculate_mipi_buff_size(
 
 static bool buffers_needed(struct ia_css_pipe *pipe)
 {
-	if (!atomisp_hw_is_isp2401) {
+	if (!IS_ISP2401) {
 		if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
 			return false;
 		else
@@ -405,11 +406,11 @@ static bool buffers_needed(struct ia_css_pipe *pipe)
 	return true;
 }
 
-enum ia_css_err
+int
 allocate_mipi_frames(struct ia_css_pipe *pipe,
 		     struct ia_css_stream_info *info) {
 #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
+	int err = -EINVAL;
 	unsigned int port;
 	struct ia_css_frame_info mipi_intermediate_info;
 
@@ -423,7 +424,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "allocate_mipi_frames(%p) exit: pipe or stream is null.\n",
 				    pipe);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
@@ -432,7 +433,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "allocate_mipi_frames(%p) exit: no buffers needed for 2401 pipe mode.\n",
 				    pipe);
-		return IA_CSS_SUCCESS;
+		return 0;
 	}
 
 #endif
@@ -441,10 +442,10 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "allocate_mipi_frames(%p) exit: no buffers needed for pipe mode.\n",
 				    pipe);
-		return IA_CSS_SUCCESS; /* AM TODO: Check  */
+		return 0; /* AM TODO: Check  */
 	}
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		port = (unsigned int)pipe->stream->config.source.port.port;
 	else
 		err = ia_css_mipi_is_source_port_valid(pipe, &port);
@@ -455,7 +456,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "allocate_mipi_frames(%p) exit: error: port is not correct (port=%d).\n",
 				    pipe, port);
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
 
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
@@ -470,7 +471,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "allocate_mipi_frames(%p) exit: already allocated for this port (port=%d).\n",
 				    pipe, port);
-		return IA_CSS_SUCCESS;
+		return 0;
 	}
 #else
 	/* 2401 system allows multiple streams to use same physical port. This is not
@@ -484,7 +485,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "allocate_mipi_frames(%p) leave: nothing to do, already allocated for this port (port=%d).\n",
 				    pipe, port);
-		return IA_CSS_SUCCESS;
+		return 0;
 	}
 #endif
 
@@ -522,7 +523,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 					  &my_css.mipi_frames[port][i],
 					  my_css.mipi_frame_size[port] * HIVE_ISP_DDR_WORD_BYTES,
 					  false);
-				if (err != IA_CSS_SUCCESS) {
+				if (err) {
 					for (j = 0; j < i; j++) {
 						if (my_css.mipi_frames[port][j]) {
 							ia_css_frame_free(my_css.mipi_frames[port][j]);
@@ -562,14 +563,14 @@ allocate_mipi_frames(struct ia_css_pipe *pipe,
 #else
 	(void)pipe;
 	(void)info;
-	return IA_CSS_SUCCESS;
+	return 0;
 #endif
 }
 
-enum ia_css_err
+int
 free_mipi_frames(struct ia_css_pipe *pipe) {
 #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
+	int err = -EINVAL;
 	unsigned int port;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
@@ -583,7 +584,7 @@ free_mipi_frames(struct ia_css_pipe *pipe) {
 			ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 					    "free_mipi_frames(%p) exit: error: pipe or stream is null.\n",
 					    pipe);
-			return IA_CSS_ERR_INVALID_ARGUMENTS;
+			return -EINVAL;
 		}
 
 		if (!buffers_needed(pipe)) {
@@ -593,7 +594,7 @@ free_mipi_frames(struct ia_css_pipe *pipe) {
 			return err;
 		}
 
-		if (!atomisp_hw_is_isp2401)
+		if (!IS_ISP2401)
 			port = (unsigned int)pipe->stream->config.source.port.port;
 		else
 			err = ia_css_mipi_is_source_port_valid(pipe, &port);
@@ -677,13 +678,13 @@ free_mipi_frames(struct ia_css_pipe *pipe) {
 #else
 	(void)pipe;
 #endif
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
+int
 send_mipi_frames(struct ia_css_pipe *pipe) {
 #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-	enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR;
+	int err = -EINVAL;
 	unsigned int i;
 #ifndef ISP2401
 	unsigned int port;
@@ -698,18 +699,18 @@ send_mipi_frames(struct ia_css_pipe *pipe) {
 	if (!pipe || !pipe->stream)
 	{
 		IA_CSS_ERROR("pipe or stream is null");
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	/* multi stream video needs mipi buffers */
 	/* nothing to be done in other cases. */
 	if (!buffers_needed(pipe)) {
 		IA_CSS_LOG("nothing to be done for this mode");
-		return IA_CSS_SUCCESS;
+		return 0;
 		/* TODO: AM: maybe this should be returning an error. */
 	}
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		port = (unsigned int)pipe->stream->config.source.port.port;
 	else
 		err = ia_css_mipi_is_source_port_valid(pipe, &port);
@@ -749,9 +750,9 @@ send_mipi_frames(struct ia_css_pipe *pipe) {
 	    (uint8_t)port,
 	    (uint8_t)my_css.num_mipi_frames[port],
 	    0 /* not used */);
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
 #else
 	(void)pipe;
 #endif
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/sh_css_mipi.h b/drivers/staging/media/atomisp/pci/sh_css_mipi.h
index ab38589d6457..52f08a103883 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_mipi.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_mipi.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -22,13 +23,13 @@
 void
 mipi_init(void);
 
-enum ia_css_err
+int
 allocate_mipi_frames(struct ia_css_pipe *pipe, struct ia_css_stream_info *info);
 
-enum ia_css_err
+int
 free_mipi_frames(struct ia_css_pipe *pipe);
 
-enum ia_css_err
+int
 send_mipi_frames(struct ia_css_pipe *pipe);
 
 /**
@@ -41,7 +42,7 @@ send_mipi_frames(struct ia_css_pipe *pipe);
  *
  * @return
  */
-enum ia_css_err
+int
 calculate_mipi_buff_size(
     struct ia_css_stream_config *stream_cfg,
     unsigned int *size_mem_words);
diff --git a/drivers/staging/media/atomisp/pci/sh_css_mmu.c b/drivers/staging/media/atomisp/pci/sh_css_mmu.c
index 179b6f40be49..1da7459eaa25 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_mmu.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_mmu.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -43,7 +44,6 @@ ia_css_mmu_invalidate_cache(void)
 			    "ia_css_mmu_invalidate_cache() leave\n");
 }
 
-/* Deprecated, this is an HRT backend function (memory_access.h) */
 void
 sh_css_mmu_set_page_table_base_index(hrt_data base_index)
 {
diff --git a/drivers/staging/media/atomisp/pci/sh_css_morph.c b/drivers/staging/media/atomisp/pci/sh_css_morph.c
index 1f4fa25b1e79..edd1da941ccb 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_morph.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_morph.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
index 52e29161cb35..ff0082d02af3 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -18,7 +19,6 @@
 #include <ia_css_err.h>
 #include <ia_css_types.h>
 #include "ia_css_debug.h"
-#include "memory_access.h"
 
 static struct ia_css_dvs_6axis_config *
 alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res,
@@ -28,14 +28,14 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res,
 	unsigned int height_y = 0;
 	unsigned int width_uv = 0;
 	unsigned int height_uv = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_dvs_6axis_config  *dvs_config = NULL;
 
-	dvs_config = (struct ia_css_dvs_6axis_config *)sh_css_malloc(sizeof(
-			 struct ia_css_dvs_6axis_config));
+	dvs_config = kvmalloc(sizeof(struct ia_css_dvs_6axis_config),
+			      GFP_KERNEL);
 	if (!dvs_config)	{
 		IA_CSS_ERROR("out of memory");
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 	} else {
 		/*Initialize new struct with latest config settings*/
 		if (dvs_config_src) {
@@ -58,41 +58,41 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res,
 		}
 
 		/* Generate Y buffers  */
-		dvs_config->xcoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(
-					    uint32_t));
+		dvs_config->xcoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+						 GFP_KERNEL);
 		if (!dvs_config->xcoords_y) {
 			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			goto exit;
 		}
 
-		dvs_config->ycoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(
-					    uint32_t));
+		dvs_config->ycoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+						 GFP_KERNEL);
 		if (!dvs_config->ycoords_y) {
 			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			goto exit;
 		}
 
 		/* Generate UV buffers  */
 		IA_CSS_LOG("UV W %d H %d", width_uv, height_uv);
 
-		dvs_config->xcoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv *
-					 sizeof(uint32_t));
+		dvs_config->xcoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t),
+						  GFP_KERNEL);
 		if (!dvs_config->xcoords_uv) {
 			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 			goto exit;
 		}
 
-		dvs_config->ycoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv *
-					 sizeof(uint32_t));
+		dvs_config->ycoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t),
+						  GFP_KERNEL);
 		if (!dvs_config->ycoords_uv) {
 			IA_CSS_ERROR("out of memory");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 		}
 exit:
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			free_dvs_6axis_table(
 			    &dvs_config); /* we might have allocated some memory, release this */
 			dvs_config = NULL;
@@ -208,28 +208,28 @@ free_dvs_6axis_table(struct ia_css_dvs_6axis_config  **dvs_6axis_config)
 	if ((dvs_6axis_config) && (*dvs_6axis_config)) {
 		IA_CSS_ENTER_PRIVATE("dvs_6axis_config %p", (*dvs_6axis_config));
 		if ((*dvs_6axis_config)->xcoords_y) {
-			sh_css_free((*dvs_6axis_config)->xcoords_y);
+			kvfree((*dvs_6axis_config)->xcoords_y);
 			(*dvs_6axis_config)->xcoords_y = NULL;
 		}
 
 		if ((*dvs_6axis_config)->ycoords_y) {
-			sh_css_free((*dvs_6axis_config)->ycoords_y);
+			kvfree((*dvs_6axis_config)->ycoords_y);
 			(*dvs_6axis_config)->ycoords_y = NULL;
 		}
 
 		/* Free up UV buffers */
 		if ((*dvs_6axis_config)->xcoords_uv) {
-			sh_css_free((*dvs_6axis_config)->xcoords_uv);
+			kvfree((*dvs_6axis_config)->xcoords_uv);
 			(*dvs_6axis_config)->xcoords_uv = NULL;
 		}
 
 		if ((*dvs_6axis_config)->ycoords_uv) {
-			sh_css_free((*dvs_6axis_config)->ycoords_uv);
+			kvfree((*dvs_6axis_config)->ycoords_uv);
 			(*dvs_6axis_config)->ycoords_uv = NULL;
 		}
 
 		IA_CSS_LEAVE_PRIVATE("dvs_6axis_config %p", (*dvs_6axis_config));
-		sh_css_free(*dvs_6axis_config);
+		kvfree(*dvs_6axis_config);
 		*dvs_6axis_config = NULL;
 	}
 }
diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h
index b4cffbbdafde..7782f76b9f97 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_shading.c b/drivers/staging/media/atomisp/pci/sh_css_param_shading.c
index 4b648df2d073..046f34857891 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_param_shading.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_param_shading.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -360,12 +361,13 @@ ia_css_shading_table_alloc(
 	me->fraction_bits = 0;
 	for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) {
 		me->data[i] =
-		    sh_css_malloc(width * height * sizeof(*me->data[0]));
+		    kvmalloc(width * height * sizeof(*me->data[0]),
+			     GFP_KERNEL);
 		if (!me->data[i]) {
 			unsigned int j;
 
 			for (j = 0; j < i; j++) {
-				sh_css_free(me->data[j]);
+				kvfree(me->data[j]);
 				me->data[j] = NULL;
 			}
 			kfree(me);
@@ -392,7 +394,7 @@ ia_css_shading_table_free(struct ia_css_shading_table *table)
 
 	for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) {
 		if (table->data[i]) {
-			sh_css_free(table->data[i]);
+			kvfree(table->data[i]);
 			table->data[i] = NULL;
 		}
 	}
diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_shading.h b/drivers/staging/media/atomisp/pci/sh_css_param_shading.h
index 6e480d31c201..7cdfaaec0b1c 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_param_shading.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_param_shading.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_params.c b/drivers/staging/media/atomisp/pci/sh_css_params.c
index 2e719f7db89e..ba42be9b06eb 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_params.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_params.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -24,6 +25,7 @@
 #define IA_CSS_INCLUDE_PARAMETERS
 #define IA_CSS_INCLUDE_ACC_PARAMETERS
 
+#include "hmm.h"
 #include "sh_css_params.h"
 #include "ia_css_queue.h"
 #include "sw_event_global.h"		/* Event IDs */
@@ -45,10 +47,7 @@
 #include "sh_css_sp.h"
 #include "ia_css_pipeline.h"
 #include "ia_css_debug.h"
-#include "memory_access.h"
-#if 0   /* FIXME */
-#include "memory_realloc.h"
-#endif
+
 #include "ia_css_isp_param.h"
 #include "ia_css_isp_params.h"
 #include "ia_css_mipi.h"
@@ -126,17 +125,17 @@
 
 /* We keep a second copy of the ptr struct for the SP to access.
    Again, this would not be necessary on the chip. */
-static hrt_vaddress sp_ddr_ptrs;
+static ia_css_ptr sp_ddr_ptrs;
 
 /* sp group address on DDR */
-static hrt_vaddress xmem_sp_group_ptrs;
+static ia_css_ptr xmem_sp_group_ptrs;
 
-static hrt_vaddress xmem_sp_stage_ptrs[IA_CSS_PIPE_ID_NUM]
+static ia_css_ptr xmem_sp_stage_ptrs[IA_CSS_PIPE_ID_NUM]
 [SH_CSS_MAX_STAGES];
-static hrt_vaddress xmem_isp_stage_ptrs[IA_CSS_PIPE_ID_NUM]
+static ia_css_ptr xmem_isp_stage_ptrs[IA_CSS_PIPE_ID_NUM]
 [SH_CSS_MAX_STAGES];
 
-static hrt_vaddress default_gdc_lut;
+static ia_css_ptr default_gdc_lut;
 static int interleaved_lut_temp[4][HRT_GDC_N];
 
 /* END DO NOT MOVE INTO VIMALS_WORLD */
@@ -1229,20 +1228,20 @@ struct ia_css_isp_skc_dvs_statistics {
 	ia_css_ptr p_data;
 };
 
-static enum ia_css_err
+static int
 ref_sh_css_ddr_address_map(
     struct sh_css_ddr_address_map *map,
     struct sh_css_ddr_address_map *out);
 
-static enum ia_css_err
+static int
 write_ia_css_isp_parameter_set_info_to_ddr(
     struct ia_css_isp_parameter_set_info *me,
-    hrt_vaddress *out);
+    ia_css_ptr *out);
 
-static enum ia_css_err
-free_ia_css_isp_parameter_set_info(hrt_vaddress ptr);
+static int
+free_ia_css_isp_parameter_set_info(ia_css_ptr ptr);
 
-static enum ia_css_err
+static int
 sh_css_params_write_to_ddr_internal(
     struct ia_css_pipe *pipe,
     unsigned int pipe_id,
@@ -1251,7 +1250,7 @@ sh_css_params_write_to_ddr_internal(
     struct sh_css_ddr_address_map *ddr_map,
     struct sh_css_ddr_address_map_size *ddr_map_size);
 
-static enum ia_css_err
+static int
 sh_css_create_isp_params(struct ia_css_stream *stream,
 			 struct ia_css_isp_parameters **isp_params_out);
 
@@ -1261,27 +1260,27 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream,
 				   bool use_default_config,
 				   struct ia_css_pipe *pipe_in);
 
-static enum ia_css_err
+static int
 sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe,
 				   struct ia_css_isp_parameters *params,
 				   const struct ia_css_isp_config *config,
 				   struct ia_css_pipe *pipe_in);
 
-static enum ia_css_err
+static int
 sh_css_set_global_isp_config_on_pipe(
     struct ia_css_pipe *curr_pipe,
     const struct ia_css_isp_config *config,
     struct ia_css_pipe *pipe);
 
 #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-static enum ia_css_err
+static int
 sh_css_set_per_frame_isp_config_on_pipe(
     struct ia_css_stream *stream,
     const struct ia_css_isp_config *config,
     struct ia_css_pipe *pipe);
 #endif
 
-static enum ia_css_err
+static int
 sh_css_update_uds_and_crop_info_based_on_zoom_region(
     const struct ia_css_binary_info *info,
     const struct ia_css_frame_info *in_frame_info,
@@ -1294,7 +1293,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region(
     struct ia_css_resolution pipe_in_res,
     bool enable_zoom);
 
-hrt_vaddress
+ia_css_ptr
 sh_css_params_ddr_address_map(void)
 {
 	return sp_ddr_ptrs;
@@ -1351,8 +1350,8 @@ convert_allocate_fpntbl(struct ia_css_isp_parameters *params)
 	return me;
 }
 
-static enum ia_css_err
-store_fpntbl(struct ia_css_isp_parameters *params, hrt_vaddress ptr) {
+static int
+store_fpntbl(struct ia_css_isp_parameters *params, ia_css_ptr ptr) {
 	struct ia_css_host_data *isp_data;
 
 	assert(params);
@@ -1361,13 +1360,13 @@ store_fpntbl(struct ia_css_isp_parameters *params, hrt_vaddress ptr) {
 	isp_data = convert_allocate_fpntbl(params);
 	if (!isp_data)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+		return -ENOMEM;
 	}
 	ia_css_params_store_ia_css_host_data(ptr, isp_data);
 
 	ia_css_host_data_free(isp_data);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -1431,10 +1430,10 @@ ia_css_process_kernel(struct ia_css_stream *stream,
 	}
 }
 
-static enum ia_css_err
+static int
 sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe,
 			      bool *is_dp_10bpp) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	/* Currently we check if 10bpp DPC configuration is required based
 	 * on the use case,i.e. if BDS and DPC is both enabled. The more cleaner
 	 * design choice would be to expose the type of DPC (either 10bpp or 13bpp)
@@ -1444,8 +1443,8 @@ sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe,
 	 */
 	if ((!pipe) || (!is_dp_10bpp))
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR);
-		err = IA_CSS_ERR_INTERNAL_ERROR;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		err = -EINVAL;
 	} else
 	{
 		*is_dp_10bpp = false;
@@ -1457,7 +1456,7 @@ sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe,
 
 			if ((pipe->config.bayer_ds_out_res.width != 0) &&
 			    (pipe->config.bayer_ds_out_res.height != 0)) {
-				if (IA_CSS_SUCCESS == binarydesc_calculate_bds_factor(
+				if (0 == binarydesc_calculate_bds_factor(
 					pipe->config.input_effective_res,
 					pipe->config.bayer_ds_out_res,
 					&required_bds_factor)) {
@@ -1473,7 +1472,7 @@ sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe,
 	return err;
 }
 
-enum ia_css_err
+int
 sh_css_set_black_frame(struct ia_css_stream *stream,
 		       const struct ia_css_frame *raw_black_frame) {
 	struct ia_css_isp_parameters *params;
@@ -1481,7 +1480,7 @@ sh_css_set_black_frame(struct ia_css_stream *stream,
 	 * that it can use the DMA.
 	 */
 	unsigned int height, width, y, x, k, data;
-	hrt_vaddress ptr;
+	ia_css_ptr ptr;
 
 	assert(stream);
 	assert(raw_black_frame);
@@ -1498,16 +1497,17 @@ sh_css_set_black_frame(struct ia_css_stream *stream,
 	if (params->fpn_config.data &&
 	    (params->fpn_config.width != width || params->fpn_config.height != height))
 	{
-		sh_css_free(params->fpn_config.data);
+		kvfree(params->fpn_config.data);
 		params->fpn_config.data = NULL;
 	}
 	if (!params->fpn_config.data)
 	{
-		params->fpn_config.data = sh_css_malloc(height * width * sizeof(short));
+		params->fpn_config.data = kvmalloc(height * width *
+						   sizeof(short), GFP_KERNEL);
 		if (!params->fpn_config.data) {
 			IA_CSS_ERROR("out of memory");
-			IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+			return -ENOMEM;
 		}
 		params->fpn_config.width = width;
 		params->fpn_config.height = height;
@@ -1521,7 +1521,7 @@ sh_css_set_black_frame(struct ia_css_stream *stream,
 			int ofs = y * width + x;
 
 			for (k = 0; k < ISP_VEC_NELEMS; k += 2) {
-				mmgr_load(ptr, (void *)(&data), sizeof(int));
+				hmm_load(ptr, (void *)(&data), sizeof(int));
 				params->fpn_config.data[ofs + 2 * k] =
 				    (short)(data & 0xFFFF);
 				params->fpn_config.data[ofs + 2 * k + 2] =
@@ -1529,7 +1529,7 @@ sh_css_set_black_frame(struct ia_css_stream *stream,
 				ptr += sizeof(int);	/* byte system address */
 			}
 			for (k = 0; k < ISP_VEC_NELEMS; k += 2) {
-				mmgr_load(ptr, (void *)(&data), sizeof(int));
+				hmm_load(ptr, (void *)(&data), sizeof(int));
 				params->fpn_config.data[ofs + 2 * k + 1] =
 				    (short)(data & 0xFFFF);
 				params->fpn_config.data[ofs + 2 * k + 3] =
@@ -1545,9 +1545,9 @@ sh_css_set_black_frame(struct ia_css_stream *stream,
 	/* overwrite isp parameter */
 	ia_css_process_kernel(stream, params, ia_css_kernel_process_param[IA_CSS_FPN_ID]);
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 bool
@@ -1611,7 +1611,7 @@ sh_css_set_shading_table(struct ia_css_stream *stream,
 
 void
 ia_css_params_store_ia_css_host_data(
-    hrt_vaddress ddr_addr,
+    ia_css_ptr ddr_addr,
     struct ia_css_host_data *data)
 {
 	assert(data);
@@ -1620,7 +1620,7 @@ ia_css_params_store_ia_css_host_data(
 
 	IA_CSS_ENTER_PRIVATE("");
 
-	mmgr_store(ddr_addr,
+	hmm_store(ddr_addr,
 		   (void *)(data->address),
 		   (size_t)data->size);
 
@@ -1676,9 +1676,9 @@ ia_css_params_alloc_convert_sctbl(
 	return sctbl;
 }
 
-enum ia_css_err ia_css_params_store_sctbl(
+int ia_css_params_store_sctbl(
     const struct ia_css_pipeline_stage *stage,
-    hrt_vaddress sc_tbl,
+    ia_css_ptr sc_tbl,
     const struct ia_css_shading_table  *sc_config)
 {
 	struct ia_css_host_data *isp_sc_tbl;
@@ -1687,13 +1687,13 @@ enum ia_css_err ia_css_params_store_sctbl(
 
 	if (!sc_config) {
 		IA_CSS_LEAVE_PRIVATE("void");
-		return IA_CSS_SUCCESS;
+		return 0;
 	}
 
 	isp_sc_tbl = ia_css_params_alloc_convert_sctbl(stage, sc_config);
 	if (!isp_sc_tbl) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+		return -ENOMEM;
 	}
 	/* store the shading table to ddr */
 	ia_css_params_store_ia_css_host_data(sc_tbl, isp_sc_tbl);
@@ -1701,7 +1701,7 @@ enum ia_css_err ia_css_params_store_sctbl(
 
 	IA_CSS_LEAVE_PRIVATE("void");
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -1717,13 +1717,13 @@ sh_css_enable_pipeline(const struct ia_css_binary *binary)
 	IA_CSS_LEAVE_PRIVATE("void");
 }
 
-static enum ia_css_err
+static int
 ia_css_process_zoom_and_motion(
     struct ia_css_isp_parameters *params,
     const struct ia_css_pipeline_stage *first_stage) {
 	/* first_stage can be  NULL */
 	const struct ia_css_pipeline_stage *stage;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	struct ia_css_resolution pipe_in_res;
 
 	pipe_in_res.width = 0;
@@ -1799,7 +1799,7 @@ ia_css_process_zoom_and_motion(
 				  &params->uds[stage->stage_num].crop_pos,
 				  pipe_in_res,
 				  stage->enable_zoom);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				return err;
 		}
 	}
@@ -1912,16 +1912,16 @@ void ia_css_morph_table_free(
 
 	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
 		if (me->coordinates_x[i]) {
-			sh_css_free(me->coordinates_x[i]);
+			kvfree(me->coordinates_x[i]);
 			me->coordinates_x[i] = NULL;
 		}
 		if (me->coordinates_y[i]) {
-			sh_css_free(me->coordinates_y[i]);
+			kvfree(me->coordinates_y[i]);
 			me->coordinates_y[i] = NULL;
 		}
 	}
 
-	sh_css_free(me);
+	kvfree(me);
 	IA_CSS_LEAVE("void");
 }
 
@@ -1934,7 +1934,7 @@ struct ia_css_morph_table *ia_css_morph_table_allocate(
 
 	IA_CSS_ENTER("");
 
-	me = sh_css_malloc(sizeof(*me));
+	me = kvmalloc(sizeof(*me), GFP_KERNEL);
 	if (!me) {
 		IA_CSS_ERROR("out of memory");
 		return me;
@@ -1946,12 +1946,12 @@ struct ia_css_morph_table *ia_css_morph_table_allocate(
 	}
 
 	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-		me->coordinates_x[i] =
-		    sh_css_malloc(height * width *
-				  sizeof(*me->coordinates_x[i]));
-		me->coordinates_y[i] =
-		    sh_css_malloc(height * width *
-				  sizeof(*me->coordinates_y[i]));
+		me->coordinates_x[i] = kvmalloc(height * width *
+						sizeof(*me->coordinates_x[i]),
+						GFP_KERNEL);
+		me->coordinates_y[i] = kvmalloc(height * width *
+						sizeof(*me->coordinates_y[i]),
+						GFP_KERNEL);
 
 		if ((!me->coordinates_x[i]) ||
 		    (!me->coordinates_y[i])) {
@@ -1966,7 +1966,7 @@ struct ia_css_morph_table *ia_css_morph_table_allocate(
 	return me;
 }
 
-static enum ia_css_err sh_css_params_default_morph_table(
+static int sh_css_params_default_morph_table(
     struct ia_css_morph_table **table,
     const struct ia_css_binary *binary)
 {
@@ -1987,7 +1987,7 @@ static enum ia_css_err sh_css_params_default_morph_table(
 
 	tab = ia_css_morph_table_allocate(width, height);
 	if (!tab)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
 		short val_y = start_y[i];
@@ -2016,9 +2016,9 @@ static enum ia_css_err sh_css_params_default_morph_table(
 	}
 	*table = tab;
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -2065,11 +2065,11 @@ ia_css_isp_3a_statistics_map_free(struct ia_css_isp_3a_statistics_map *me)
 {
 	if (me) {
 		if (me->data_allocated) {
-			sh_css_free(me->data_ptr);
+			kvfree(me->data_ptr);
 			me->data_ptr = NULL;
 			me->data_allocated = false;
 		}
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
@@ -2083,7 +2083,7 @@ ia_css_isp_3a_statistics_map_allocate(
 	 * so we use a local char * instead. */
 	char *base_ptr;
 
-	me = sh_css_malloc(sizeof(*me));
+	me = kvmalloc(sizeof(*me), GFP_KERNEL);
 	if (!me) {
 		IA_CSS_LEAVE("cannot allocate memory");
 		goto err;
@@ -2092,7 +2092,7 @@ ia_css_isp_3a_statistics_map_allocate(
 	me->data_ptr = data_ptr;
 	me->data_allocated = !data_ptr;
 	if (!data_ptr) {
-		me->data_ptr = sh_css_malloc(isp_stats->size);
+		me->data_ptr = kvmalloc(isp_stats->size, GFP_KERNEL);
 		if (!me->data_ptr) {
 			IA_CSS_LEAVE("cannot allocate memory");
 			goto err;
@@ -2115,15 +2115,15 @@ ia_css_isp_3a_statistics_map_allocate(
 
 err:
 	if (me)
-		sh_css_free(me);
+		kvfree(me);
 	return NULL;
 }
 
-enum ia_css_err
+int
 ia_css_get_3a_statistics(struct ia_css_3a_statistics           *host_stats,
 			 const struct ia_css_isp_3a_statistics *isp_stats) {
 	struct ia_css_isp_3a_statistics_map *map;
-	enum ia_css_err ret = IA_CSS_SUCCESS;
+	int ret = 0;
 
 	IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats);
 
@@ -2133,13 +2133,13 @@ ia_css_get_3a_statistics(struct ia_css_3a_statistics           *host_stats,
 	map = ia_css_isp_3a_statistics_map_allocate(isp_stats, NULL);
 	if (map)
 	{
-		mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
+		hmm_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
 		ia_css_translate_3a_statistics(host_stats, map);
 		ia_css_isp_3a_statistics_map_free(map);
 	} else
 	{
 		IA_CSS_ERROR("out of memory");
-		ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		ret = -ENOMEM;
 	}
 
 	IA_CSS_LEAVE_ERR(ret);
@@ -2161,7 +2161,7 @@ ia_css_set_param_exceptions(const struct ia_css_pipe *pipe,
 	params->dp_config.b  = params->wb_config.b;
 	params->dp_config.gb = params->wb_config.gb;
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		assert(pipe);
 		assert(pipe->mode < IA_CSS_PIPE_ID_NUM);
 
@@ -2431,22 +2431,22 @@ sh_css_pipe_isp_config_get(struct ia_css_pipe *pipe)
 	return pipe->config.p_isp_config;
 }
 
-enum ia_css_err
+int
 ia_css_stream_set_isp_config(
     struct ia_css_stream *stream,
     const struct ia_css_isp_config *config) {
 	return ia_css_stream_set_isp_config_on_pipe(stream, config, NULL);
 }
 
-enum ia_css_err
+int
 ia_css_stream_set_isp_config_on_pipe(
     struct ia_css_stream *stream,
     const struct ia_css_isp_config *config,
     struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	if ((!stream) || (!config))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	IA_CSS_ENTER("stream=%p, config=%p, pipe=%p", stream, config, pipe);
 
@@ -2461,16 +2461,16 @@ ia_css_stream_set_isp_config_on_pipe(
 	return err;
 }
 
-enum ia_css_err
+int
 ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe,
 			   struct ia_css_isp_config *config) {
 	struct ia_css_pipe *pipe_in = pipe;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER("pipe=%p", pipe);
 
 	if ((!pipe) || (!pipe->stream))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "config=%p\n", config);
 
@@ -2484,14 +2484,14 @@ ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe,
 	return err;
 }
 
-static enum ia_css_err
+static int
 sh_css_set_global_isp_config_on_pipe(
     struct ia_css_pipe *curr_pipe,
     const struct ia_css_isp_config *config,
     struct ia_css_pipe *pipe) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum ia_css_err err1 = IA_CSS_SUCCESS;
-	enum ia_css_err err2 = IA_CSS_SUCCESS;
+	int err = 0;
+	int err1 = 0;
+	int err2 = 0;
 
 	IA_CSS_ENTER_PRIVATE("stream=%p, config=%p, pipe=%p", curr_pipe, config, pipe);
 
@@ -2506,24 +2506,24 @@ sh_css_set_global_isp_config_on_pipe(
 	 * but instead continue with updating the ISP params to enable testing of features
 	 * which are currently in TR phase. */
 
-	err = (err1 != IA_CSS_SUCCESS) ? err1 : ((err2 != IA_CSS_SUCCESS) ? err2 : err);
+	err = (err1 != 0) ? err1 : ((err2 != 0) ? err2 : err);
 
 	IA_CSS_LEAVE_ERR_PRIVATE(err);
 	return err;
 }
 
 #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
-static enum ia_css_err
+static int
 sh_css_set_per_frame_isp_config_on_pipe(
     struct ia_css_stream *stream,
     const struct ia_css_isp_config *config,
     struct ia_css_pipe *pipe) {
 	unsigned int i;
 	bool per_frame_config_created = false;
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	enum ia_css_err err1 = IA_CSS_SUCCESS;
-	enum ia_css_err err2 = IA_CSS_SUCCESS;
-	enum ia_css_err err3 = IA_CSS_SUCCESS;
+	int err = 0;
+	int err1 = 0;
+	int err2 = 0;
+	int err3 = 0;
 
 	struct sh_css_ddr_address_map *ddr_ptrs;
 	struct sh_css_ddr_address_map_size *ddr_ptrs_size;
@@ -2533,7 +2533,7 @@ sh_css_set_per_frame_isp_config_on_pipe(
 
 	if (!pipe)
 	{
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		goto exit;
 	}
 
@@ -2544,7 +2544,7 @@ sh_css_set_per_frame_isp_config_on_pipe(
 	{
 		err = sh_css_create_isp_params(stream,
 					       &stream->per_frame_isp_params_configs);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			goto exit;
 		per_frame_config_created = true;
 	}
@@ -2554,7 +2554,7 @@ sh_css_set_per_frame_isp_config_on_pipe(
 	/* update new ISP params object with the new config */
 	if (!sh_css_init_isp_params_from_global(stream, params, false, pipe))
 	{
-		err1 = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err1 = -EINVAL;
 	}
 
 	err2 = sh_css_init_isp_params_from_config(stream->pipes[0], params, config, pipe);
@@ -2578,21 +2578,21 @@ sh_css_set_per_frame_isp_config_on_pipe(
 	 * The CSS API must pass this error information to the caller, ie. the host.
 	 * We do not return this error immediately, but instead continue with updating the ISP params
 	 *  to enable testing of features which are currently in TR phase. */
-	err = (err1 != IA_CSS_SUCCESS) ? err1 :
-	      (err2 != IA_CSS_SUCCESS) ? err2 :
-	      (err3 != IA_CSS_SUCCESS) ? err3 : err;
+	err = (err1 != 0) ? err1 :
+	      (err2 != 0) ? err2 :
+	      (err3 != 0) ? err3 : err;
 exit:
 	IA_CSS_LEAVE_ERR_PRIVATE(err);
 	return err;
 }
 #endif
 
-static enum ia_css_err
+static int
 sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe,
 				   struct ia_css_isp_parameters *params,
 				   const struct ia_css_isp_config *config,
 				   struct ia_css_pipe *pipe_in) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool is_dp_10bpp = true;
 
 	assert(pipe);
@@ -2630,12 +2630,12 @@ sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe,
 	 * before (NORM+OBC) or after. The folllowing code to set the
 	 * DPC configuration should be updated when this interface is made
 	 * available */
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		sh_css_set_dp_config(pipe, params, config->dp_config);
 		ia_css_set_param_exceptions(pipe, params);
 	}
 
-	if (IA_CSS_SUCCESS ==
+	if (0 ==
 	    sh_css_select_dp_10bpp_config(pipe, &is_dp_10bpp))
 	{
 		/* return an error when both DPC and BDS is enabled by the
@@ -2643,15 +2643,15 @@ sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe,
 		/* we do not exit from this point immediately to allow internal
 		 * firmware feature testing. */
 		if (is_dp_10bpp) {
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
+			err = -EINVAL;
 		}
 	} else
 	{
-		err = IA_CSS_ERR_INTERNAL_ERROR;
+		err = -EINVAL;
 		goto exit;
 	}
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		ia_css_set_param_exceptions(pipe, params);
 
 exit:
@@ -2709,16 +2709,16 @@ ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe,
  * Deprecated: Implement mmgr_realloc()
  */
 static bool realloc_isp_css_mm_buf(
-    hrt_vaddress *curr_buf,
+    ia_css_ptr *curr_buf,
     size_t *curr_size,
     size_t needed_size,
     bool force,
-    enum ia_css_err *err,
+    int *err,
     uint16_t mmgr_attribute)
 {
 	s32 id;
 
-	*err = IA_CSS_SUCCESS;
+	*err = 0;
 	/* Possible optimization: add a function sh_css_isp_css_mm_realloc()
 	 * and implement on top of hmm. */
 
@@ -2736,11 +2736,13 @@ static bool realloc_isp_css_mm_buf(
 
 	id = IA_CSS_REFCOUNT_PARAM_BUFFER;
 	ia_css_refcount_decrement(id, *curr_buf);
-	*curr_buf = ia_css_refcount_increment(id, mmgr_alloc_attr(needed_size,
-					      mmgr_attribute));
+	*curr_buf = ia_css_refcount_increment(id, hmm_alloc(needed_size,
+							    HMM_BO_PRIVATE, 0,
+							    NULL,
+							    mmgr_attribute));
 
 	if (!*curr_buf) {
-		*err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		*err = -ENOMEM;
 		*curr_size = 0;
 	} else {
 		*curr_size = needed_size;
@@ -2750,19 +2752,18 @@ static bool realloc_isp_css_mm_buf(
 }
 
 static bool reallocate_buffer(
-    hrt_vaddress *curr_buf,
+    ia_css_ptr *curr_buf,
     size_t *curr_size,
     size_t needed_size,
     bool force,
-    enum ia_css_err *err)
+    int *err)
 {
 	bool ret;
-	u16	mmgr_attribute = MMGR_ATTRIBUTE_DEFAULT;
 
 	IA_CSS_ENTER_PRIVATE("void");
 
 	ret = realloc_isp_css_mm_buf(curr_buf,
-				     curr_size, needed_size, force, err, mmgr_attribute);
+				     curr_size, needed_size, force, err, 0);
 
 	IA_CSS_LEAVE_PRIVATE("ret=%d", ret);
 	return ret;
@@ -2781,7 +2782,7 @@ ia_css_isp_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid)
 	if (!grid->enable)
 		return NULL;
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
@@ -2803,9 +2804,9 @@ ia_css_isp_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid)
 	me->hmem_size = CEIL_MUL(me->hmem_size, HIVE_ISP_DDR_WORD_BYTES);
 
 	me->size = me->dmem_size + me->vmem_size * 2 + me->hmem_size;
-	me->data_ptr = mmgr_malloc(me->size);
+	me->data_ptr = hmm_alloc(me->size, HMM_BO_PRIVATE, 0, NULL, 0);
 	if (me->data_ptr == mmgr_NULL) {
-		sh_css_free(me);
+		kvfree(me);
 		me = NULL;
 		goto err;
 	}
@@ -2828,7 +2829,7 @@ ia_css_isp_3a_statistics_free(struct ia_css_isp_3a_statistics *me)
 {
 	if (me) {
 		hmm_free(me->data_ptr);
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
@@ -2847,13 +2848,13 @@ ia_css_metadata_allocate(const struct ia_css_metadata_info *metadata_info)
 	if (metadata_info->size == 0)
 		return NULL;
 
-	md = sh_css_malloc(sizeof(*md));
+	md = kvmalloc(sizeof(*md), GFP_KERNEL);
 	if (!md)
 		goto error;
 
 	md->info = *metadata_info;
 	md->exp_id = 0;
-	md->address = mmgr_malloc(metadata_info->size);
+	md->address = hmm_alloc(metadata_info->size, HMM_BO_PRIVATE, 0, NULL, 0);
 	if (md->address == mmgr_NULL)
 		goto error;
 
@@ -2877,7 +2878,7 @@ ia_css_metadata_free(struct ia_css_metadata *me)
 		 * and debugging. */
 		IA_CSS_ENTER("me=%p", me);
 		hmm_free(me->address);
-		sh_css_free(me);
+		kvfree(me);
 		IA_CSS_LEAVE("void");
 	}
 }
@@ -2897,9 +2898,9 @@ ia_css_metadata_free_multiple(unsigned int num_bufs,
 static unsigned int g_param_buffer_dequeue_count;
 static unsigned int g_param_buffer_enqueue_count;
 
-enum ia_css_err
+int
 ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int i;
 	struct sh_css_ddr_address_map *ddr_ptrs;
 	struct sh_css_ddr_address_map_size *ddr_ptrs_size;
@@ -2910,8 +2911,8 @@ ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) {
 
 	if (!stream)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS);
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+		return -EINVAL;
 	}
 	/* TMP: tracking of paramsets */
 	g_param_buffer_dequeue_count = 0;
@@ -2920,7 +2921,7 @@ ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) {
 	stream->per_frame_isp_params_configs = NULL;
 	err = sh_css_create_isp_params(stream,
 				       &stream->isp_params_configs);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		goto ERR;
 
 	params = stream->isp_params_configs;
@@ -2928,7 +2929,7 @@ ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) {
 	{
 		/* we do not return the error immediately to enable internal
 		 * firmware feature testing */
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 	}
 
 	ddr_ptrs = &params->ddr_ptrs;
@@ -2968,22 +2969,22 @@ ia_css_set_sdis2_config(
 	ia_css_set_sdis2_vertproj_config(params, dvs2_coefs);
 }
 
-static enum ia_css_err
+static int
 sh_css_create_isp_params(struct ia_css_stream *stream,
 			 struct ia_css_isp_parameters **isp_params_out) {
 	bool succ = true;
 	unsigned int i;
 	struct sh_css_ddr_address_map *ddr_ptrs;
 	struct sh_css_ddr_address_map_size *ddr_ptrs_size;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	size_t params_size;
 	struct ia_css_isp_parameters *params =
-	sh_css_malloc(sizeof(struct ia_css_isp_parameters));
+	kvmalloc(sizeof(struct ia_css_isp_parameters), GFP_KERNEL);
 
 	if (!params)
 	{
 		*isp_params_out = NULL;
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 		IA_CSS_ERROR("%s:%d error: cannot allocate memory", __FILE__, __LINE__);
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
@@ -3010,13 +3011,13 @@ sh_css_create_isp_params(struct ia_css_stream *stream,
 	ddr_ptrs_size->isp_param = params_size;
 	ddr_ptrs->isp_param =
 	ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_BUFFER,
-				  mmgr_malloc(params_size));
+				  hmm_alloc(params_size, HMM_BO_PRIVATE, 0, NULL, 0));
 	succ &= (ddr_ptrs->isp_param != mmgr_NULL);
 
 	ddr_ptrs_size->macc_tbl = sizeof(struct ia_css_macc_table);
 	ddr_ptrs->macc_tbl =
 	ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_BUFFER,
-				  mmgr_malloc(sizeof(struct ia_css_macc_table)));
+				  hmm_alloc(sizeof(struct ia_css_macc_table), HMM_BO_PRIVATE, 0, NULL, 0));
 	succ &= (ddr_ptrs->macc_tbl != mmgr_NULL);
 
 	*isp_params_out = params;
@@ -3068,12 +3069,12 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream,
 		ia_css_set_ob_config(params, &default_ob_config);
 		ia_css_set_dp_config(params, &default_dp_config);
 
-		if (!atomisp_hw_is_isp2401) {
+		if (!IS_ISP2401) {
 			ia_css_set_param_exceptions(pipe_in, params);
 		} else {
 			for (i = 0; i < stream->num_pipes; i++) {
 				if (sh_css_select_dp_10bpp_config(stream->pipes[i],
-								&is_dp_10bpp) == IA_CSS_SUCCESS) {
+								&is_dp_10bpp) == 0) {
 					/* set the return value as false if both DPC and
 					* BDS is enabled by the user. But we do not return
 					* the value immediately to enable internal firmware
@@ -3180,7 +3181,7 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream,
 		ia_css_set_formats_config(params, &stream_params->formats_config);
 
 		for (i = 0; i < stream->num_pipes; i++) {
-			if (IA_CSS_SUCCESS ==
+			if (0 ==
 			    sh_css_select_dp_10bpp_config(stream->pipes[i], &is_dp_10bpp)) {
 				/* set the return value as false if both DPC and
 				 * BDS is enabled by the user. But we do not return
@@ -3195,7 +3196,7 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream,
 				retval = false;
 				goto exit;
 			}
-			if (atomisp_hw_is_isp2401) {
+			if (IS_ISP2401) {
 				if (stream->pipes[i]->mode < IA_CSS_PIPE_ID_NUM) {
 					sh_css_set_dp_config(stream->pipes[i], params,
 							    &stream_params->pipe_dp_config[stream->pipes[i]->mode]);
@@ -3207,7 +3208,7 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream,
 			}
 		}
 
-		if (!atomisp_hw_is_isp2401)
+		if (!IS_ISP2401)
 			ia_css_set_param_exceptions(pipe_in, params);
 
 		params->fpn_config.data = stream_params->fpn_config.data;
@@ -3252,7 +3253,7 @@ exit:
 	return retval;
 }
 
-enum ia_css_err
+int
 sh_css_params_init(void) {
 	int i, p;
 
@@ -3267,18 +3268,20 @@ sh_css_params_init(void) {
 		for (i = 0; i < SH_CSS_MAX_STAGES; i++) {
 			xmem_sp_stage_ptrs[p][i] =
 			ia_css_refcount_increment(-1,
-						  mmgr_calloc(1,
-							      sizeof(struct sh_css_sp_stage)));
+						  hmm_alloc(sizeof(struct sh_css_sp_stage),
+							    HMM_BO_PRIVATE, 0, NULL,
+							    ATOMISP_MAP_FLAG_CLEARED));
 			xmem_isp_stage_ptrs[p][i] =
 			ia_css_refcount_increment(-1,
-						  mmgr_calloc(1,
-							      sizeof(struct sh_css_isp_stage)));
+						  hmm_alloc(sizeof(struct sh_css_sp_stage),
+							    HMM_BO_PRIVATE, 0, NULL,
+							    ATOMISP_MAP_FLAG_CLEARED));
 
 			if ((xmem_sp_stage_ptrs[p][i] == mmgr_NULL) ||
 			    (xmem_isp_stage_ptrs[p][i] == mmgr_NULL)) {
 				sh_css_params_uninit();
-				IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-				return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+				IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+				return -ENOMEM;
 			}
 		}
 	}
@@ -3288,21 +3291,25 @@ sh_css_params_init(void) {
 	ia_css_config_rgb_gamma_tables();
 	ia_css_config_xnr_table();
 
-	sp_ddr_ptrs = ia_css_refcount_increment(-1, mmgr_calloc(1,
-						CEIL_MUL(sizeof(struct sh_css_ddr_address_map),
-							HIVE_ISP_DDR_WORD_BYTES)));
-	xmem_sp_group_ptrs = ia_css_refcount_increment(-1, mmgr_calloc(1,
-			     sizeof(struct sh_css_sp_group)));
+	sp_ddr_ptrs = ia_css_refcount_increment(-1,
+						hmm_alloc(CEIL_MUL(sizeof(struct sh_css_ddr_address_map),
+								   HIVE_ISP_DDR_WORD_BYTES),
+							  HMM_BO_PRIVATE, 0, NULL,
+							  ATOMISP_MAP_FLAG_CLEARED));
+	xmem_sp_group_ptrs = ia_css_refcount_increment(-1,
+						       hmm_alloc(sizeof(struct sh_css_sp_group),
+								 HMM_BO_PRIVATE, 0, NULL,
+								 ATOMISP_MAP_FLAG_CLEARED));
 
 	if ((sp_ddr_ptrs == mmgr_NULL) ||
 	    (xmem_sp_group_ptrs == mmgr_NULL))
 	{
 		ia_css_uninit();
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+		return -ENOMEM;
 	}
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
 static void host_lut_store(const void *lut)
@@ -3314,27 +3321,27 @@ static void host_lut_store(const void *lut)
 }
 
 /* Note that allocation is in ipu address space. */
-inline hrt_vaddress sh_css_params_alloc_gdc_lut(void)
+inline ia_css_ptr sh_css_params_alloc_gdc_lut(void)
 {
-	return mmgr_malloc(sizeof(zoom_table));
+	return hmm_alloc(sizeof(zoom_table), HMM_BO_PRIVATE, 0, NULL, 0);
 }
 
-inline void sh_css_params_free_gdc_lut(hrt_vaddress addr)
+inline void sh_css_params_free_gdc_lut(ia_css_ptr addr)
 {
 	if (addr != mmgr_NULL)
 		hmm_free(addr);
 }
 
-enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe,
+int ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe,
 	const void *lut)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool stream_started = false;
 
 	IA_CSS_ENTER("pipe=%p lut=%p", pipe, lut);
 
 	if (!lut || !pipe) {
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE("err=%d", err);
 		return err;
 	}
@@ -3347,7 +3354,7 @@ enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe,
 		ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
 				    "unable to set scaler lut since stream has started\n");
 		stream_started = true;
-		err = IA_CSS_ERR_NOT_SUPPORTED;
+		err = -ENOTSUPP;
 	}
 
 	/* Free any existing tables. */
@@ -3355,19 +3362,19 @@ enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe,
 	pipe->scaler_pp_lut = mmgr_NULL;
 
 	if (!stream_started) {
-		if (!atomisp_hw_is_isp2401)
-			pipe->scaler_pp_lut = mmgr_malloc(sizeof(zoom_table));
+		if (!IS_ISP2401)
+			pipe->scaler_pp_lut = hmm_alloc(sizeof(zoom_table), HMM_BO_PRIVATE, 0, NULL, 0);
 		else
 			pipe->scaler_pp_lut = sh_css_params_alloc_gdc_lut();
 
 		if (pipe->scaler_pp_lut == mmgr_NULL) {
 			ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR,
 					    "unable to allocate scaler_pp_lut\n");
-			err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+			err = -ENOMEM;
 		} else {
 			gdc_lut_convert_to_isp_format((const int(*)[HRT_GDC_N])lut,
 						      interleaved_lut_temp);
-			mmgr_store(pipe->scaler_pp_lut,
+			hmm_store(pipe->scaler_pp_lut,
 				   (int *)interleaved_lut_temp,
 				   sizeof(zoom_table));
 		}
@@ -3378,7 +3385,7 @@ enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe,
 }
 
 /* if pipe is NULL, returns default lut addr. */
-hrt_vaddress sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe)
+ia_css_ptr sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe)
 {
 	assert(pipe);
 
@@ -3388,9 +3395,9 @@ hrt_vaddress sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe)
 		return sh_css_params_get_default_gdc_lut();
 }
 
-enum ia_css_err sh_css_params_map_and_store_default_gdc_lut(void)
+int sh_css_params_map_and_store_default_gdc_lut(void)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	IA_CSS_ENTER_PRIVATE("void");
 
@@ -3400,17 +3407,17 @@ enum ia_css_err sh_css_params_map_and_store_default_gdc_lut(void)
 
 	host_lut_store((void *)zoom_table);
 
-	if (!atomisp_hw_is_isp2401)
-		default_gdc_lut = mmgr_malloc(sizeof(zoom_table));
+	if (!IS_ISP2401)
+		default_gdc_lut = hmm_alloc(sizeof(zoom_table), HMM_BO_PRIVATE, 0, NULL, 0);
 	else
 		default_gdc_lut = sh_css_params_alloc_gdc_lut();
 
 	if (default_gdc_lut == mmgr_NULL)
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		return -ENOMEM;
 
 	gdc_lut_convert_to_isp_format((const int(*)[HRT_GDC_N])zoom_table,
 				      interleaved_lut_temp);
-	mmgr_store(default_gdc_lut, (int *)interleaved_lut_temp,
+	hmm_store(default_gdc_lut, (int *)interleaved_lut_temp,
 		   sizeof(zoom_table));
 
 	IA_CSS_LEAVE_PRIVATE("lut(%u) err=%d", default_gdc_lut, err);
@@ -3427,13 +3434,13 @@ void sh_css_params_free_default_gdc_lut(void)
 	IA_CSS_LEAVE_PRIVATE("void");
 }
 
-hrt_vaddress sh_css_params_get_default_gdc_lut(void)
+ia_css_ptr sh_css_params_get_default_gdc_lut(void)
 {
 	return default_gdc_lut;
 }
 
 static void free_param_set_callback(
-    hrt_vaddress ptr)
+    ia_css_ptr ptr)
 {
 	IA_CSS_ENTER_PRIVATE("void");
 
@@ -3443,7 +3450,7 @@ static void free_param_set_callback(
 }
 
 static void free_buffer_callback(
-    hrt_vaddress ptr)
+    ia_css_ptr ptr)
 {
 	IA_CSS_ENTER_PRIVATE("void");
 
@@ -3476,7 +3483,7 @@ static void free_map(struct sh_css_ddr_address_map *map)
 {
 	unsigned int i;
 
-	hrt_vaddress *addrs = (hrt_vaddress *)map;
+	ia_css_ptr *addrs = (ia_css_ptr *)map;
 
 	IA_CSS_ENTER_PRIVATE("void");
 
@@ -3521,7 +3528,7 @@ ia_css_stream_isp_parameters_uninit(struct ia_css_stream *stream)
 		free_map(&per_frame_params->ddr_ptrs);
 
 	if (params->fpn_config.data) {
-		sh_css_free(params->fpn_config.data);
+		kvfree(params->fpn_config.data);
 		params->fpn_config.data = NULL;
 	}
 
@@ -3537,9 +3544,9 @@ ia_css_stream_isp_parameters_uninit(struct ia_css_stream *stream)
 		}
 	}
 
-	sh_css_free(params);
+	kvfree(params);
 	if (per_frame_params)
-		sh_css_free(per_frame_params);
+		kvfree(per_frame_params);
 	stream->isp_params_configs = NULL;
 	stream->per_frame_isp_params_configs = NULL;
 
@@ -3603,7 +3610,7 @@ convert_allocate_morph_plane(
 	me = ia_css_host_data_allocate((size_t)isp_data_size);
 
 	if (!me) {
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
+		IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
 		return NULL;
 	}
 
@@ -3622,12 +3629,12 @@ convert_allocate_morph_plane(
 	return me;
 }
 
-static enum ia_css_err
+static int
 store_morph_plane(
     unsigned short *data,
     unsigned int width,
     unsigned int height,
-    hrt_vaddress dest,
+    ia_css_ptr dest,
     unsigned int aligned_width) {
 	struct ia_css_host_data *isp_data;
 
@@ -3636,18 +3643,18 @@ store_morph_plane(
 	isp_data = convert_allocate_morph_plane(data, width, height, aligned_width);
 	if (!isp_data)
 	{
-		IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-		return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+		return -ENOMEM;
 	}
 	ia_css_params_store_ia_css_host_data(dest, isp_data);
 
 	ia_css_host_data_free(isp_data);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void sh_css_update_isp_params_to_ddr(
     struct ia_css_isp_parameters *params,
-    hrt_vaddress ddr_ptr)
+    ia_css_ptr ddr_ptr)
 {
 	size_t size = sizeof(params->uds);
 
@@ -3655,13 +3662,13 @@ static void sh_css_update_isp_params_to_ddr(
 
 	assert(params);
 
-	mmgr_store(ddr_ptr, &params->uds, size);
+	hmm_store(ddr_ptr, &params->uds, size);
 	IA_CSS_LEAVE_PRIVATE("void");
 }
 
 static void sh_css_update_isp_mem_params_to_ddr(
     const struct ia_css_binary *binary,
-    hrt_vaddress ddr_mem_ptr,
+    ia_css_ptr ddr_mem_ptr,
     size_t size,
     enum ia_css_isp_memories mem)
 {
@@ -3671,7 +3678,7 @@ static void sh_css_update_isp_mem_params_to_ddr(
 
 	params = ia_css_isp_param_get_mem_init(&binary->mem_params,
 					       IA_CSS_PARAM_CLASS_PARAM, mem);
-	mmgr_store(ddr_mem_ptr, params->address, size);
+	hmm_store(ddr_mem_ptr, params->address, size);
 
 	IA_CSS_LEAVE_PRIVATE("void");
 }
@@ -3679,7 +3686,7 @@ static void sh_css_update_isp_mem_params_to_ddr(
 void ia_css_dequeue_param_buffers(/*unsigned int pipe_num*/ void)
 {
 	unsigned int i;
-	hrt_vaddress cpy;
+	ia_css_ptr cpy;
 	enum sh_css_queue_id param_queue_ids[3] = {	IA_CSS_PARAMETER_SET_QUEUE_ID,
 						    IA_CSS_PER_FRAME_PARAMETER_SET_QUEUE_ID,
 						    SH_CSS_INVALID_QUEUE_ID
@@ -3694,10 +3701,10 @@ void ia_css_dequeue_param_buffers(/*unsigned int pipe_num*/ void)
 	}
 
 	for (i = 0; SH_CSS_INVALID_QUEUE_ID != param_queue_ids[i]; i++) {
-		cpy = (hrt_vaddress)0;
+		cpy = (ia_css_ptr)0;
 		/* clean-up old copy */
 		while (ia_css_bufq_dequeue_buffer(param_queue_ids[i],
-						  (uint32_t *)&cpy) == IA_CSS_SUCCESS) {
+						  (uint32_t *)&cpy) == 0) {
 			/* TMP: keep track of dequeued param set count
 			 */
 			g_param_buffer_dequeue_count++;
@@ -3709,7 +3716,7 @@ void ia_css_dequeue_param_buffers(/*unsigned int pipe_num*/ void)
 
 			IA_CSS_LOG("dequeued param set %x from %d, release ref", cpy, 0);
 			free_ia_css_isp_parameter_set_info(cpy);
-			cpy = (hrt_vaddress)0;
+			cpy = (ia_css_ptr)0;
 		}
 	}
 
@@ -3751,13 +3758,13 @@ process_kernel_parameters(unsigned int pipe_id,
 	}
 }
 
-enum ia_css_err
+int
 sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 			       struct ia_css_isp_parameters *params,
 			       bool commit,
 			       struct ia_css_pipe *pipe_in) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
-	hrt_vaddress cpy;
+	int err = 0;
+	ia_css_ptr cpy;
 	int i;
 	unsigned int raw_bit_depth = 10;
 	unsigned int isp_pipe_version = SH_CSS_ISP_PIPE_VERSION_1;
@@ -3807,7 +3814,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 #endif
 		if (!sh_css_sp_is_running()) {
 			/* SP is not running. The queues are not valid */
-			err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+			err = -EBUSY;
 			break;
 		}
 		cur_map = &params->pipe_ddr_ptrs[pipeline->pipe_id];
@@ -3823,7 +3830,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 			/* the processing is a.o. resolution dependent */
 			err = ia_css_process_zoom_and_motion(params,
 							     pipeline->stages);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				return err;
 		}
 		/* check if to actually update the parameters for this pipe */
@@ -3854,14 +3861,14 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 				  cur_map,
 				  cur_map_size);
 
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				break;
 			for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) {
 				params->isp_mem_params_changed
 				[pipeline->pipe_id][stage->stage_num][mem] = false;
 			}
 		} /* for */
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			break;
 		/* update isp_params to pipe specific copies */
 		if (params->isp_params_changed) {
@@ -3870,7 +3877,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 					  cur_map_size->isp_param,
 					  true,
 					  &err);
-			if (err != IA_CSS_SUCCESS)
+			if (err)
 				break;
 			sh_css_update_isp_params_to_ddr(params, cur_map->isp_param);
 		}
@@ -3879,7 +3886,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 		err = ref_sh_css_ddr_address_map(
 			  cur_map,
 			  &isp_params_info.mem_map);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			break;
 
 		/* Update Parameters ID */
@@ -3891,14 +3898,14 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 
 		/* now write the copy to ddr */
 		err = write_ia_css_isp_parameter_set_info_to_ddr(&isp_params_info, &cpy);
-		if (err != IA_CSS_SUCCESS)
+		if (err)
 			break;
 
 		/* enqueue the set to sp */
 		IA_CSS_LOG("queue param set %x to %d", cpy, thread_id);
 
 		err = ia_css_bufq_enqueue_buffer(thread_id, queue_id, (uint32_t)cpy);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			free_ia_css_isp_parameter_set_info(cpy);
 #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS)
 			IA_CSS_LOG("pfp: FAILED to add config id %d for OF %d to q %d on thread %d",
@@ -3919,8 +3926,8 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 			 */
 			if (!sh_css_sp_is_running()) {
 				/* SP is not running. The queues are not valid */
-				IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE);
-				return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE;
+				IA_CSS_LEAVE_ERR_PRIVATE(-EBUSY);
+				return -EBUSY;
 			}
 			ia_css_bufq_enqueue_psys_event(
 			    IA_CSS_PSYS_SW_EVENT_BUFFER_ENQUEUED,
@@ -3957,7 +3964,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe,
 	return err;
 }
 
-static enum ia_css_err
+static int
 sh_css_params_write_to_ddr_internal(
     struct ia_css_pipe *pipe,
     unsigned int pipe_id,
@@ -3965,7 +3972,7 @@ sh_css_params_write_to_ddr_internal(
     const struct ia_css_pipeline_stage *stage,
     struct sh_css_ddr_address_map *ddr_map,
     struct sh_css_ddr_address_map_size *ddr_map_size) {
-	enum ia_css_err err;
+	int err;
 	const struct ia_css_binary *binary;
 
 	unsigned int stage_num;
@@ -3993,14 +4000,14 @@ sh_css_params_write_to_ddr_internal(
 						   (size_t)(FPNTBL_BYTES(binary)),
 						   params->config_changed[IA_CSS_FPN_ID],
 						   &err);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
 		if (params->config_changed[IA_CSS_FPN_ID] || buff_realloced) {
 			if (params->fpn_config.enabled) {
 				err = store_fpntbl(params, ddr_map->fpn_tbl);
-				if (err != IA_CSS_SUCCESS) {
+				if (err) {
 					IA_CSS_LEAVE_ERR_PRIVATE(err);
 					return err;
 				}
@@ -4013,7 +4020,7 @@ sh_css_params_write_to_ddr_internal(
 		u32 enable_conv;
 		size_t bytes;
 
-		if (!atomisp_hw_is_isp2401)
+		if (!IS_ISP2401)
 			bytes = ISP2400_SCTBL_BYTES(binary);
 		else
 			bytes = ISP2401_SCTBL_BYTES(binary);
@@ -4025,7 +4032,7 @@ sh_css_params_write_to_ddr_internal(
 						   bytes,
 						   params->sc_table_changed,
 						   &err);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -4036,7 +4043,7 @@ sh_css_params_write_to_ddr_internal(
 				if (params->sc_table) {
 					/* store the shading table to ddr */
 					err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_table);
-					if (err != IA_CSS_SUCCESS) {
+					if (err) {
 						IA_CSS_LEAVE_ERR_PRIVATE(err);
 						return err;
 					}
@@ -4054,13 +4061,13 @@ sh_css_params_write_to_ddr_internal(
 										binary->sctbl_width_per_color,
 										binary->sctbl_height);
 					if (!params->sc_config) {
-						IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-						return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+						IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+						return -ENOMEM;
 					}
 
 					/* store the shading table to ddr */
 					err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_config);
-					if (err != IA_CSS_SUCCESS) {
+					if (err) {
 						IA_CSS_LEAVE_ERR_PRIVATE(err);
 						return err;
 					}
@@ -4085,13 +4092,13 @@ sh_css_params_write_to_ddr_internal(
 				    &params->sc_config,
 				    binary, pipe->required_bds_factor);
 				if (!params->sc_config) {
-					IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-					return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+					IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+					return -ENOMEM;
 				}
 
 				/* store the shading table to ddr */
 				err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_config);
-				if (err != IA_CSS_SUCCESS) {
+				if (err) {
 					IA_CSS_LEAVE_ERR_PRIVATE(err);
 					return err;
 				}
@@ -4110,7 +4117,7 @@ sh_css_params_write_to_ddr_internal(
 	/* DPC configuration is made pipe specific to allow flexibility in positioning of the
 	 * DPC kernel. The code below sets the pipe specific configuration to
 	 * individual binaries. */
-	if (atomisp_hw_is_isp2401 &&
+	if (IS_ISP2401 &&
 	    params->pipe_dpc_config_changed[pipe_id] && binary->info->sp.enable.dpc)
 	{
 		unsigned int size   =
@@ -4171,11 +4178,11 @@ sh_css_params_write_to_ddr_internal(
 				  ddr_map_size->macc_tbl,
 				  true,
 				  &err);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
-		mmgr_store(ddr_map->macc_tbl,
+		hmm_store(ddr_map->macc_tbl,
 			   converted_macc_table.data,
 			   sizeof(converted_macc_table.data));
 	}
@@ -4192,7 +4199,7 @@ sh_css_params_write_to_ddr_internal(
 				     (size_t)((DVS_6AXIS_BYTES(binary) / 2) * 3),
 				     params->pipe_dvs_6axis_config_changed[pipe_id],
 				     &err);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -4213,7 +4220,7 @@ sh_css_params_write_to_ddr_internal(
 			if (!params->pipe_dvs_6axis_config[pipe_id]) {
 				struct ia_css_resolution dvs_offset = {0};
 
-				if (!atomisp_hw_is_isp2401) {
+				if (!IS_ISP2401) {
 					dvs_offset.width = (PIX_SHIFT_FILTER_RUN_IN_X + binary->dvs_envelope.width) / 2;
 				} else {
 					if (binary->dvs_envelope.width || binary->dvs_envelope.height) {
@@ -4225,8 +4232,8 @@ sh_css_params_write_to_ddr_internal(
 				params->pipe_dvs_6axis_config[pipe_id] =
 				    generate_dvs_6axis_table(&binary->out_frame_info[0].res, &dvs_offset);
 				if (!params->pipe_dvs_6axis_config[pipe_id]) {
-					IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY);
-					return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+					IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
+					return -ENOMEM;
 				}
 				params->pipe_dvs_6axis_config_changed[pipe_id] = true;
 
@@ -4242,13 +4249,17 @@ sh_css_params_write_to_ddr_internal(
 	if (binary->info->sp.enable.ca_gdc)
 	{
 		unsigned int i;
-		hrt_vaddress *virt_addr_tetra_x[
+		ia_css_ptr *virt_addr_tetra_x[
+
 		IA_CSS_MORPH_TABLE_NUM_PLANES];
 		size_t *virt_size_tetra_x[
+
 		IA_CSS_MORPH_TABLE_NUM_PLANES];
-		hrt_vaddress *virt_addr_tetra_y[
+		ia_css_ptr *virt_addr_tetra_y[
+
 		IA_CSS_MORPH_TABLE_NUM_PLANES];
 		size_t *virt_size_tetra_y[
+
 		IA_CSS_MORPH_TABLE_NUM_PLANES];
 
 		virt_addr_tetra_x[0] = &ddr_map->tetra_r_x;
@@ -4288,7 +4299,7 @@ sh_css_params_write_to_ddr_internal(
 					    (MORPH_PLANE_BYTES(binary)),
 					    params->morph_table_changed,
 					    &err);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -4299,7 +4310,7 @@ sh_css_params_write_to_ddr_internal(
 					    (MORPH_PLANE_BYTES(binary)),
 					    params->morph_table_changed,
 					    &err);
-			if (err != IA_CSS_SUCCESS) {
+			if (err) {
 				IA_CSS_LEAVE_ERR_PRIVATE(err);
 				return err;
 			}
@@ -4316,7 +4327,7 @@ sh_css_params_write_to_ddr_internal(
 			if (!table) {
 				err = sh_css_params_default_morph_table(&id_table,
 									binary);
-				if (err != IA_CSS_SUCCESS) {
+				if (err) {
 					IA_CSS_LEAVE_ERR_PRIVATE(err);
 					return err;
 				}
@@ -4354,7 +4365,7 @@ sh_css_params_write_to_ddr_internal(
 						size,
 						params->isp_mem_params_changed[pipe_id][stage_num][mem],
 						&err);
-		if (err != IA_CSS_SUCCESS) {
+		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
 		}
@@ -4365,8 +4376,8 @@ sh_css_params_write_to_ddr_internal(
 		}
 	}
 
-	IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS);
-	return IA_CSS_SUCCESS;
+	IA_CSS_LEAVE_ERR_PRIVATE(0);
+	return 0;
 }
 
 const struct ia_css_fpn_table *ia_css_get_fpn_table(struct ia_css_stream
@@ -4457,42 +4468,42 @@ struct ia_css_shading_table *ia_css_get_shading_table(struct ia_css_stream
 	return table;
 }
 
-hrt_vaddress sh_css_store_sp_group_to_ddr(void)
+ia_css_ptr sh_css_store_sp_group_to_ddr(void)
 {
 	IA_CSS_ENTER_LEAVE_PRIVATE("void");
-	mmgr_store(xmem_sp_group_ptrs,
+	hmm_store(xmem_sp_group_ptrs,
 		   &sh_css_sp_group,
 		   sizeof(struct sh_css_sp_group));
 	return xmem_sp_group_ptrs;
 }
 
-hrt_vaddress sh_css_store_sp_stage_to_ddr(
+ia_css_ptr sh_css_store_sp_stage_to_ddr(
     unsigned int pipe,
     unsigned int stage)
 {
 	IA_CSS_ENTER_LEAVE_PRIVATE("void");
-	mmgr_store(xmem_sp_stage_ptrs[pipe][stage],
+	hmm_store(xmem_sp_stage_ptrs[pipe][stage],
 		   &sh_css_sp_stage,
 		   sizeof(struct sh_css_sp_stage));
 	return xmem_sp_stage_ptrs[pipe][stage];
 }
 
-hrt_vaddress sh_css_store_isp_stage_to_ddr(
+ia_css_ptr sh_css_store_isp_stage_to_ddr(
     unsigned int pipe,
     unsigned int stage)
 {
 	IA_CSS_ENTER_LEAVE_PRIVATE("void");
-	mmgr_store(xmem_isp_stage_ptrs[pipe][stage],
+	hmm_store(xmem_isp_stage_ptrs[pipe][stage],
 		   &sh_css_isp_stage,
 		   sizeof(struct sh_css_isp_stage));
 	return xmem_isp_stage_ptrs[pipe][stage];
 }
 
-static enum ia_css_err ref_sh_css_ddr_address_map(
+static int ref_sh_css_ddr_address_map(
     struct sh_css_ddr_address_map *map,
     struct sh_css_ddr_address_map *out)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	unsigned int i;
 
 	/* we will use a union to copy things; overlaying an array
@@ -4501,7 +4512,7 @@ static enum ia_css_err ref_sh_css_ddr_address_map(
 	*/
 	union {
 		struct sh_css_ddr_address_map *map;
-		hrt_vaddress *addrs;
+		ia_css_ptr *addrs;
 	} in_addrs, to_addrs;
 
 	IA_CSS_ENTER_PRIVATE("void");
@@ -4512,7 +4523,7 @@ static enum ia_css_err ref_sh_css_ddr_address_map(
 	to_addrs.map = out;
 
 	assert(sizeof(struct sh_css_ddr_address_map_size) / sizeof(size_t) ==
-	       sizeof(struct sh_css_ddr_address_map) / sizeof(hrt_vaddress));
+	       sizeof(struct sh_css_ddr_address_map) / sizeof(ia_css_ptr));
 
 	/* copy map using size info */
 	for (i = 0; i < (sizeof(struct sh_css_ddr_address_map_size) /
@@ -4528,11 +4539,11 @@ static enum ia_css_err ref_sh_css_ddr_address_map(
 	return err;
 }
 
-static enum ia_css_err write_ia_css_isp_parameter_set_info_to_ddr(
+static int write_ia_css_isp_parameter_set_info_to_ddr(
     struct ia_css_isp_parameter_set_info *me,
-    hrt_vaddress *out)
+    ia_css_ptr *out)
 {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	bool succ;
 
 	IA_CSS_ENTER_PRIVATE("void");
@@ -4540,26 +4551,26 @@ static enum ia_css_err write_ia_css_isp_parameter_set_info_to_ddr(
 	assert(me);
 	assert(out);
 
-	*out = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_SET_POOL, mmgr_malloc(
-					     sizeof(struct ia_css_isp_parameter_set_info)));
+	*out = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_SET_POOL,
+					 hmm_alloc(sizeof(struct ia_css_isp_parameter_set_info), HMM_BO_PRIVATE, 0, NULL, 0));
 	succ = (*out != mmgr_NULL);
 	if (succ)
-		mmgr_store(*out,
+		hmm_store(*out,
 			   me, sizeof(struct ia_css_isp_parameter_set_info));
 	else
-		err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
+		err = -ENOMEM;
 
 	IA_CSS_LEAVE_ERR_PRIVATE(err);
 	return err;
 }
 
-static enum ia_css_err
+static int
 free_ia_css_isp_parameter_set_info(
-    hrt_vaddress ptr) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+    ia_css_ptr ptr) {
+	int err = 0;
 	struct ia_css_isp_parameter_set_info isp_params_info;
 	unsigned int i;
-	hrt_vaddress *addrs = (hrt_vaddress *) &isp_params_info.mem_map;
+	ia_css_ptr *addrs = (ia_css_ptr *)&isp_params_info.mem_map;
 
 	IA_CSS_ENTER_PRIVATE("ptr = %u", ptr);
 
@@ -4568,12 +4579,12 @@ free_ia_css_isp_parameter_set_info(
 	{
 		IA_CSS_ERROR("%s: IA_CSS_REFCOUNT_PARAM_SET_POOL(0x%x) invalid arg", __func__,
 			     ptr);
-		err = IA_CSS_ERR_INVALID_ARGUMENTS;
+		err = -EINVAL;
 		IA_CSS_LEAVE_ERR_PRIVATE(err);
 		return err;
 	}
 
-	mmgr_load(ptr, &isp_params_info.mem_map, sizeof(struct sh_css_ddr_address_map));
+	hmm_load(ptr, &isp_params_info.mem_map, sizeof(struct sh_css_ddr_address_map));
 	/* copy map using size info */
 	for (i = 0; i < (sizeof(struct sh_css_ddr_address_map_size) /
 			 sizeof(size_t)); i++)
@@ -4585,7 +4596,7 @@ free_ia_css_isp_parameter_set_info(
 		if (!ia_css_refcount_is_valid(addrs[i])) {
 			IA_CSS_ERROR("%s: IA_CSS_REFCOUNT_PARAM_BUFFER(0x%x) invalid arg", __func__,
 				     ptr);
-			err = IA_CSS_ERR_INVALID_ARGUMENTS;
+			err = -EINVAL;
 			continue;
 		}
 
@@ -4773,7 +4784,7 @@ sh_css_update_uds_and_crop_info(
 	IA_CSS_LEAVE_PRIVATE("void");
 }
 
-static enum ia_css_err
+static int
 sh_css_update_uds_and_crop_info_based_on_zoom_region(
     const struct ia_css_binary_info *info,
     const struct ia_css_frame_info *in_frame_info,
@@ -4786,7 +4797,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region(
     struct ia_css_resolution pipe_in_res,
     bool enable_zoom) {
 	unsigned int x0 = 0, y0 = 0, x1 = 0, y1 = 0;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	/* Note:
 	* Filter_Envelope = 0 for NND/LUT
 	* Filter_Envelope = 1 for BCI
@@ -4813,7 +4824,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region(
 	y1 = zoom->zoom_region.resolution.height + y0;
 
 	if ((x0 > x1) || (y0 > y1) || (x1 > pipe_in_res.width) || (y1 > pipe_in_res.height))
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	if (!enable_zoom)
 	{
@@ -4825,7 +4836,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region(
 	{
 		/* Zoom region is only supported by the UDS module on ISP
 		 * 2 and higher. It is not supported in video mode on ISP 1 */
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	} else
 	{
 		if (enable_zoom) {
@@ -4875,19 +4886,18 @@ ia_css_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid)
 
 	assert(grid);
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
 	me->grid = *grid;
 	grid_size = grid->width * grid->height;
-	me->data = sh_css_malloc(grid_size * sizeof(*me->data));
+	me->data = kvmalloc(grid_size * sizeof(*me->data), GFP_KERNEL);
 	if (!me->data)
 		goto err;
 #if !defined(HAS_NO_HMEM)
 	/* No weighted histogram, no structure, treat the histogram data as a byte dump in a byte array */
-	me->rgby_data = (struct ia_css_3a_rgby_output *)sh_css_malloc(sizeof_hmem(
-			    HMEM0_ID));
+	me->rgby_data = kvmalloc(sizeof_hmem(HMEM0_ID), GFP_KERNEL);
 #else
 	me->rgby_data = NULL;
 #endif
@@ -4905,10 +4915,10 @@ void
 ia_css_3a_statistics_free(struct ia_css_3a_statistics *me)
 {
 	if (me) {
-		sh_css_free(me->rgby_data);
-		sh_css_free(me->data);
+		kvfree(me->rgby_data);
+		kvfree(me->data);
 		memset(me, 0, sizeof(struct ia_css_3a_statistics));
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
@@ -4919,18 +4929,18 @@ ia_css_dvs_statistics_allocate(const struct ia_css_dvs_grid_info *grid)
 
 	assert(grid);
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
 	me->grid = *grid;
-	me->hor_proj = sh_css_malloc(grid->height * IA_CSS_DVS_NUM_COEF_TYPES *
-				     sizeof(*me->hor_proj));
+	me->hor_proj = kvmalloc(grid->height * IA_CSS_DVS_NUM_COEF_TYPES *
+				sizeof(*me->hor_proj), GFP_KERNEL);
 	if (!me->hor_proj)
 		goto err;
 
-	me->ver_proj = sh_css_malloc(grid->width * IA_CSS_DVS_NUM_COEF_TYPES *
-				     sizeof(*me->ver_proj));
+	me->ver_proj = kvmalloc(grid->width * IA_CSS_DVS_NUM_COEF_TYPES *
+				sizeof(*me->ver_proj), GFP_KERNEL);
 	if (!me->ver_proj)
 		goto err;
 
@@ -4944,10 +4954,10 @@ void
 ia_css_dvs_statistics_free(struct ia_css_dvs_statistics *me)
 {
 	if (me) {
-		sh_css_free(me->hor_proj);
-		sh_css_free(me->ver_proj);
+		kvfree(me->hor_proj);
+		kvfree(me->ver_proj);
 		memset(me, 0, sizeof(struct ia_css_dvs_statistics));
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
@@ -4958,21 +4968,21 @@ ia_css_dvs_coefficients_allocate(const struct ia_css_dvs_grid_info *grid)
 
 	assert(grid);
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
 	me->grid = *grid;
 
-	me->hor_coefs = sh_css_malloc(grid->num_hor_coefs *
-				      IA_CSS_DVS_NUM_COEF_TYPES *
-				      sizeof(*me->hor_coefs));
+	me->hor_coefs = kvmalloc(grid->num_hor_coefs *
+				 IA_CSS_DVS_NUM_COEF_TYPES *
+				 sizeof(*me->hor_coefs), GFP_KERNEL);
 	if (!me->hor_coefs)
 		goto err;
 
-	me->ver_coefs = sh_css_malloc(grid->num_ver_coefs *
-				      IA_CSS_DVS_NUM_COEF_TYPES *
-				      sizeof(*me->ver_coefs));
+	me->ver_coefs = kvmalloc(grid->num_ver_coefs *
+				 IA_CSS_DVS_NUM_COEF_TYPES *
+				 sizeof(*me->ver_coefs), GFP_KERNEL);
 	if (!me->ver_coefs)
 		goto err;
 
@@ -4986,10 +4996,10 @@ void
 ia_css_dvs_coefficients_free(struct ia_css_dvs_coefficients *me)
 {
 	if (me) {
-		sh_css_free(me->hor_coefs);
-		sh_css_free(me->ver_coefs);
+		kvfree(me->hor_coefs);
+		kvfree(me->ver_coefs);
 		memset(me, 0, sizeof(struct ia_css_dvs_coefficients));
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
@@ -5000,49 +5010,65 @@ ia_css_dvs2_statistics_allocate(const struct ia_css_dvs_grid_info *grid)
 
 	assert(grid);
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
 	me->grid = *grid;
 
-	me->hor_prod.odd_real = sh_css_malloc(grid->aligned_width *
-					      grid->aligned_height * sizeof(*me->hor_prod.odd_real));
+	me->hor_prod.odd_real = kvmalloc(grid->aligned_width *
+					 grid->aligned_height *
+					 sizeof(*me->hor_prod.odd_real),
+					 GFP_KERNEL);
 	if (!me->hor_prod.odd_real)
 		goto err;
 
-	me->hor_prod.odd_imag = sh_css_malloc(grid->aligned_width *
-					      grid->aligned_height * sizeof(*me->hor_prod.odd_imag));
+	me->hor_prod.odd_imag = kvmalloc(grid->aligned_width *
+					 grid->aligned_height *
+					 sizeof(*me->hor_prod.odd_imag),
+					 GFP_KERNEL);
 	if (!me->hor_prod.odd_imag)
 		goto err;
 
-	me->hor_prod.even_real = sh_css_malloc(grid->aligned_width *
-					       grid->aligned_height * sizeof(*me->hor_prod.even_real));
+	me->hor_prod.even_real = kvmalloc(grid->aligned_width *
+					  grid->aligned_height *
+					  sizeof(*me->hor_prod.even_real),
+					  GFP_KERNEL);
 	if (!me->hor_prod.even_real)
 		goto err;
 
-	me->hor_prod.even_imag = sh_css_malloc(grid->aligned_width *
-					       grid->aligned_height * sizeof(*me->hor_prod.even_imag));
+	me->hor_prod.even_imag = kvmalloc(grid->aligned_width *
+					  grid->aligned_height *
+					  sizeof(*me->hor_prod.even_imag),
+					  GFP_KERNEL);
 	if (!me->hor_prod.even_imag)
 		goto err;
 
-	me->ver_prod.odd_real = sh_css_malloc(grid->aligned_width *
-					      grid->aligned_height * sizeof(*me->ver_prod.odd_real));
+	me->ver_prod.odd_real = kvmalloc(grid->aligned_width *
+					 grid->aligned_height *
+					 sizeof(*me->ver_prod.odd_real),
+					 GFP_KERNEL);
 	if (!me->ver_prod.odd_real)
 		goto err;
 
-	me->ver_prod.odd_imag = sh_css_malloc(grid->aligned_width *
-					      grid->aligned_height * sizeof(*me->ver_prod.odd_imag));
+	me->ver_prod.odd_imag = kvmalloc(grid->aligned_width *
+					 grid->aligned_height *
+					 sizeof(*me->ver_prod.odd_imag),
+					 GFP_KERNEL);
 	if (!me->ver_prod.odd_imag)
 		goto err;
 
-	me->ver_prod.even_real = sh_css_malloc(grid->aligned_width *
-					       grid->aligned_height * sizeof(*me->ver_prod.even_real));
+	me->ver_prod.even_real = kvmalloc(grid->aligned_width *
+					  grid->aligned_height *
+					  sizeof(*me->ver_prod.even_real),
+					  GFP_KERNEL);
 	if (!me->ver_prod.even_real)
 		goto err;
 
-	me->ver_prod.even_imag = sh_css_malloc(grid->aligned_width *
-					       grid->aligned_height * sizeof(*me->ver_prod.even_imag));
+	me->ver_prod.even_imag = kvmalloc(grid->aligned_width *
+					  grid->aligned_height *
+					  sizeof(*me->ver_prod.even_imag),
+					  GFP_KERNEL);
 	if (!me->ver_prod.even_imag)
 		goto err;
 
@@ -5056,16 +5082,16 @@ void
 ia_css_dvs2_statistics_free(struct ia_css_dvs2_statistics *me)
 {
 	if (me) {
-		sh_css_free(me->hor_prod.odd_real);
-		sh_css_free(me->hor_prod.odd_imag);
-		sh_css_free(me->hor_prod.even_real);
-		sh_css_free(me->hor_prod.even_imag);
-		sh_css_free(me->ver_prod.odd_real);
-		sh_css_free(me->ver_prod.odd_imag);
-		sh_css_free(me->ver_prod.even_real);
-		sh_css_free(me->ver_prod.even_imag);
+		kvfree(me->hor_prod.odd_real);
+		kvfree(me->hor_prod.odd_imag);
+		kvfree(me->hor_prod.even_real);
+		kvfree(me->hor_prod.even_imag);
+		kvfree(me->ver_prod.odd_real);
+		kvfree(me->ver_prod.odd_imag);
+		kvfree(me->ver_prod.even_real);
+		kvfree(me->ver_prod.even_imag);
 		memset(me, 0, sizeof(struct ia_css_dvs2_statistics));
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
@@ -5076,49 +5102,57 @@ ia_css_dvs2_coefficients_allocate(const struct ia_css_dvs_grid_info *grid)
 
 	assert(grid);
 
-	me = sh_css_calloc(1, sizeof(*me));
+	me = kvcalloc(1, sizeof(*me), GFP_KERNEL);
 	if (!me)
 		goto err;
 
 	me->grid = *grid;
 
-	me->hor_coefs.odd_real = sh_css_malloc(grid->num_hor_coefs *
-					       sizeof(*me->hor_coefs.odd_real));
+	me->hor_coefs.odd_real = kvmalloc(grid->num_hor_coefs *
+					  sizeof(*me->hor_coefs.odd_real),
+					  GFP_KERNEL);
 	if (!me->hor_coefs.odd_real)
 		goto err;
 
-	me->hor_coefs.odd_imag = sh_css_malloc(grid->num_hor_coefs *
-					       sizeof(*me->hor_coefs.odd_imag));
+	me->hor_coefs.odd_imag = kvmalloc(grid->num_hor_coefs *
+					  sizeof(*me->hor_coefs.odd_imag),
+					  GFP_KERNEL);
 	if (!me->hor_coefs.odd_imag)
 		goto err;
 
-	me->hor_coefs.even_real = sh_css_malloc(grid->num_hor_coefs *
-						sizeof(*me->hor_coefs.even_real));
+	me->hor_coefs.even_real = kvmalloc(grid->num_hor_coefs *
+					   sizeof(*me->hor_coefs.even_real),
+					   GFP_KERNEL);
 	if (!me->hor_coefs.even_real)
 		goto err;
 
-	me->hor_coefs.even_imag = sh_css_malloc(grid->num_hor_coefs *
-						sizeof(*me->hor_coefs.even_imag));
+	me->hor_coefs.even_imag = kvmalloc(grid->num_hor_coefs *
+					   sizeof(*me->hor_coefs.even_imag),
+					   GFP_KERNEL);
 	if (!me->hor_coefs.even_imag)
 		goto err;
 
-	me->ver_coefs.odd_real = sh_css_malloc(grid->num_ver_coefs *
-					       sizeof(*me->ver_coefs.odd_real));
+	me->ver_coefs.odd_real = kvmalloc(grid->num_ver_coefs *
+					  sizeof(*me->ver_coefs.odd_real),
+					  GFP_KERNEL);
 	if (!me->ver_coefs.odd_real)
 		goto err;
 
-	me->ver_coefs.odd_imag = sh_css_malloc(grid->num_ver_coefs *
-					       sizeof(*me->ver_coefs.odd_imag));
+	me->ver_coefs.odd_imag = kvmalloc(grid->num_ver_coefs *
+					  sizeof(*me->ver_coefs.odd_imag),
+					  GFP_KERNEL);
 	if (!me->ver_coefs.odd_imag)
 		goto err;
 
-	me->ver_coefs.even_real = sh_css_malloc(grid->num_ver_coefs *
-						sizeof(*me->ver_coefs.even_real));
+	me->ver_coefs.even_real = kvmalloc(grid->num_ver_coefs *
+					   sizeof(*me->ver_coefs.even_real),
+					   GFP_KERNEL);
 	if (!me->ver_coefs.even_real)
 		goto err;
 
-	me->ver_coefs.even_imag = sh_css_malloc(grid->num_ver_coefs *
-						sizeof(*me->ver_coefs.even_imag));
+	me->ver_coefs.even_imag = kvmalloc(grid->num_ver_coefs *
+					   sizeof(*me->ver_coefs.even_imag),
+					   GFP_KERNEL);
 	if (!me->ver_coefs.even_imag)
 		goto err;
 
@@ -5132,16 +5166,16 @@ void
 ia_css_dvs2_coefficients_free(struct ia_css_dvs2_coefficients *me)
 {
 	if (me) {
-		sh_css_free(me->hor_coefs.odd_real);
-		sh_css_free(me->hor_coefs.odd_imag);
-		sh_css_free(me->hor_coefs.even_real);
-		sh_css_free(me->hor_coefs.even_imag);
-		sh_css_free(me->ver_coefs.odd_real);
-		sh_css_free(me->ver_coefs.odd_imag);
-		sh_css_free(me->ver_coefs.even_real);
-		sh_css_free(me->ver_coefs.even_imag);
+		kvfree(me->hor_coefs.odd_real);
+		kvfree(me->hor_coefs.odd_imag);
+		kvfree(me->hor_coefs.even_real);
+		kvfree(me->hor_coefs.even_imag);
+		kvfree(me->ver_coefs.odd_real);
+		kvfree(me->ver_coefs.odd_imag);
+		kvfree(me->ver_coefs.even_real);
+		kvfree(me->ver_coefs.even_imag);
 		memset(me, 0, sizeof(struct ia_css_dvs2_coefficients));
-		sh_css_free(me);
+		kvfree(me);
 	}
 }
 
@@ -5164,8 +5198,8 @@ ia_css_dvs2_6axis_config_allocate(const struct ia_css_stream *stream)
 		goto err;
 	}
 
-	dvs_config = (struct ia_css_dvs_6axis_config *)sh_css_calloc(1,
-		     sizeof(struct ia_css_dvs_6axis_config));
+	dvs_config = kvcalloc(1, sizeof(struct ia_css_dvs_6axis_config),
+			      GFP_KERNEL);
 	if (!dvs_config)
 		goto err;
 
@@ -5179,23 +5213,25 @@ ia_css_dvs2_6axis_config_allocate(const struct ia_css_stream *stream)
 				    params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO]->height_uv;
 	IA_CSS_LOG("table Y: W %d H %d", width_y, height_y);
 	IA_CSS_LOG("table UV: W %d H %d", width_uv, height_uv);
-	dvs_config->xcoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(
-				    uint32_t));
+	dvs_config->xcoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+					 GFP_KERNEL);
 	if (!dvs_config->xcoords_y)
 		goto err;
 
-	dvs_config->ycoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof(
-				    uint32_t));
+	dvs_config->ycoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+					 GFP_KERNEL);
 	if (!dvs_config->ycoords_y)
 		goto err;
 
-	dvs_config->xcoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv *
-				 sizeof(uint32_t));
+	dvs_config->xcoords_uv = kvmalloc(width_uv * height_uv *
+					  sizeof(uint32_t),
+					  GFP_KERNEL);
 	if (!dvs_config->xcoords_uv)
 		goto err;
 
-	dvs_config->ycoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv *
-				 sizeof(uint32_t));
+	dvs_config->ycoords_uv = kvmalloc(width_uv * height_uv *
+					  sizeof(uint32_t),
+					  GFP_KERNEL);
 	if (!dvs_config->ycoords_uv)
 		goto err;
 
@@ -5209,12 +5245,12 @@ void
 ia_css_dvs2_6axis_config_free(struct ia_css_dvs_6axis_config *dvs_6axis_config)
 {
 	if (dvs_6axis_config) {
-		sh_css_free(dvs_6axis_config->xcoords_y);
-		sh_css_free(dvs_6axis_config->ycoords_y);
-		sh_css_free(dvs_6axis_config->xcoords_uv);
-		sh_css_free(dvs_6axis_config->ycoords_uv);
+		kvfree(dvs_6axis_config->xcoords_y);
+		kvfree(dvs_6axis_config->ycoords_y);
+		kvfree(dvs_6axis_config->xcoords_uv);
+		kvfree(dvs_6axis_config->ycoords_uv);
 		memset(dvs_6axis_config, 0, sizeof(struct ia_css_dvs_6axis_config));
-		sh_css_free(dvs_6axis_config);
+		kvfree(dvs_6axis_config);
 	}
 }
 
@@ -5225,7 +5261,7 @@ ia_css_en_dz_capt_pipe(struct ia_css_stream *stream, bool enable)
 	struct ia_css_pipeline *pipeline;
 	struct ia_css_pipeline_stage *stage;
 	enum ia_css_pipe_id pipe_id;
-	enum ia_css_err err;
+	int err;
 	int i;
 
 	if (!stream)
@@ -5239,7 +5275,7 @@ ia_css_en_dz_capt_pipe(struct ia_css_stream *stream, bool enable)
 		if (pipe_id == IA_CSS_PIPE_ID_CAPTURE) {
 			err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP,
 							&stage);
-			if (err == IA_CSS_SUCCESS)
+			if (!err)
 				stage->enable_zoom = enable;
 			break;
 		}
diff --git a/drivers/staging/media/atomisp/pci/sh_css_params.h b/drivers/staging/media/atomisp/pci/sh_css_params.h
index 96d503967fd1..62a7b6ada237 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_params.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_params.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -150,13 +151,13 @@ struct ia_css_isp_parameters {
 
 void
 ia_css_params_store_ia_css_host_data(
-    hrt_vaddress ddr_addr,
+    ia_css_ptr ddr_addr,
     struct ia_css_host_data *data);
 
-enum ia_css_err
+int
 ia_css_params_store_sctbl(
     const struct ia_css_pipeline_stage *stage,
-    hrt_vaddress ddr_addr,
+    ia_css_ptr ddr_addr,
     const struct ia_css_shading_table *shading_table);
 
 struct ia_css_host_data *
@@ -168,21 +169,21 @@ struct ia_css_isp_config *
 sh_css_pipe_isp_config_get(struct ia_css_pipe *pipe);
 
 /* ipu address allocation/free for gdc lut */
-hrt_vaddress
+ia_css_ptr
 sh_css_params_alloc_gdc_lut(void);
 void
-sh_css_params_free_gdc_lut(hrt_vaddress addr);
+sh_css_params_free_gdc_lut(ia_css_ptr addr);
 
-enum ia_css_err
+int
 sh_css_params_map_and_store_default_gdc_lut(void);
 
 void
 sh_css_params_free_default_gdc_lut(void);
 
-hrt_vaddress
+ia_css_ptr
 sh_css_params_get_default_gdc_lut(void);
 
-hrt_vaddress
+ia_css_ptr
 sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe);
 
 #endif /* _SH_CSS_PARAMS_H_ */
diff --git a/drivers/staging/media/atomisp/pci/sh_css_params_internal.h b/drivers/staging/media/atomisp/pci/sh_css_params_internal.h
index baca24532f9f..8e5e6f273a95 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_params_internal.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_params_internal.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_pipe.c b/drivers/staging/media/atomisp/pci/sh_css_pipe.c
deleted file mode 100644
index 1f57ffad8921..000000000000
--- a/drivers/staging/media/atomisp/pci/sh_css_pipe.c
+++ /dev/null
@@ -1,16 +0,0 @@
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- */
-
-/* This file will contain the code to implement the functions declared in ia_css_pipe.h and ia_css_pipe_public.h
-   and associated helper functions */
diff --git a/drivers/staging/media/atomisp/pci/sh_css_properties.c b/drivers/staging/media/atomisp/pci/sh_css_properties.c
index 50f99c53c3d4..de588f9bd540 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_properties.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_properties.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_shading.c b/drivers/staging/media/atomisp/pci/sh_css_shading.c
index 2a2d0f4db44b..462caf9cb571 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_shading.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_shading.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_sp.c b/drivers/staging/media/atomisp/pci/sh_css_sp.c
index e574396ad0f4..a26680b1d0b0 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_sp.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_sp.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -12,6 +13,8 @@
  * more details.
  */
 
+#include "hmm.h"
+
 #include "sh_css_sp.h"
 
 #if !defined(HAS_NO_INPUT_FORMATTER)
@@ -43,10 +46,8 @@
 
 /*#include "sp.h"*/	/* host2sp_enqueue_frame_data() */
 
-#include "memory_access.h"
 
 #include "assert_support.h"
-#include "platform_support.h"	/* hrt_sleep() */
 
 #include "sw_event_global.h"			/* Event IDs.*/
 #include "ia_css_event.h"
@@ -75,14 +76,14 @@ static struct sh_css_sp_per_frame_data per_frame_data;
 /* TODO: add code that sets this bool to false */
 static bool sp_running;
 
-static enum ia_css_err
+static int
 set_output_frame_buffer(const struct ia_css_frame *frame,
 			unsigned int idx);
 
 static void
 sh_css_copy_buffer_attr_to_spbuffer(struct ia_css_buffer_sp *dest_buf,
 				    const enum sh_css_queue_id queue_id,
-				    const hrt_vaddress xmem_addr,
+				    const ia_css_ptr xmem_addr,
 				    const enum ia_css_buffer_type buf_type);
 
 static void
@@ -412,7 +413,7 @@ sh_css_sp_get_sw_interrupt_value(unsigned int irq)
 static void
 sh_css_copy_buffer_attr_to_spbuffer(struct ia_css_buffer_sp *dest_buf,
 				    const enum sh_css_queue_id queue_id,
-				    const hrt_vaddress xmem_addr,
+				    const ia_css_ptr xmem_addr,
 				    const enum ia_css_buffer_type buf_type)
 {
 	assert(buf_type < IA_CSS_NUM_BUFFER_TYPE);
@@ -535,10 +536,10 @@ sh_css_copy_frame_to_spframe(struct ia_css_frame_sp *sp_frame_out,
 	}
 }
 
-static enum ia_css_err
+static int
 set_input_frame_buffer(const struct ia_css_frame *frame) {
 	if (!frame)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	switch (frame->info.format)
 	{
@@ -558,18 +559,18 @@ set_input_frame_buffer(const struct ia_css_frame *frame) {
 	case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_10:
 		break;
 	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.in, frame);
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 set_output_frame_buffer(const struct ia_css_frame *frame,
 			unsigned int idx) {
 	if (!frame)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	switch (frame->info.format)
 	{
@@ -601,16 +602,16 @@ set_output_frame_buffer(const struct ia_css_frame *frame,
 	case IA_CSS_FRAME_FORMAT_BINARY_8:
 		break;
 	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 	sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.out[idx], frame);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 set_view_finder_buffer(const struct ia_css_frame *frame) {
 	if (!frame)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 
 	switch (frame->info.format)
 	{
@@ -630,11 +631,11 @@ set_view_finder_buffer(const struct ia_css_frame *frame) {
 	case IA_CSS_FRAME_FORMAT_YUV_LINE:
 		break;
 	default:
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
+		return -EINVAL;
 	}
 
 	sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.out_vf, frame);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 #if !defined(HAS_NO_INPUT_FORMATTER)
@@ -736,26 +737,26 @@ sh_css_sp_set_disable_continuous_viewfinder(bool flag)
 	sh_css_sp_group.config.disable_cont_vf = flag;
 }
 
-static enum ia_css_err
+static int
 sh_css_sp_write_frame_pointers(const struct sh_css_binary_args *args) {
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	int i;
 
 	assert(args);
 
 	if (args->in_frame)
 		err = set_input_frame_buffer(args->in_frame);
-	if (err == IA_CSS_SUCCESS && args->out_vf_frame)
+	if (!err && args->out_vf_frame)
 		err = set_view_finder_buffer(args->out_vf_frame);
 	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
 	{
-		if (err == IA_CSS_SUCCESS && args->out_frame[i])
+		if (!err && args->out_frame[i])
 			err = set_output_frame_buffer(args->out_frame[i], i);
 	}
 
 	/* we don't pass this error back to the upper layer, so we add a assert here
 	   because we actually hit the error here but it still works by accident... */
-	if (err != IA_CSS_SUCCESS) assert(false);
+	if (err) assert(false);
 	return err;
 }
 
@@ -790,23 +791,23 @@ sh_css_stage_write_binary_info(struct ia_css_binary_info *info)
 	sh_css_isp_stage.binary_info = *info;
 }
 
-static enum ia_css_err
+static int
 copy_isp_mem_if_to_ddr(struct ia_css_binary *binary) {
-	enum ia_css_err err;
+	int err;
 
 	err = ia_css_isp_param_copy_isp_mem_if_to_ddr(
 	    &binary->css_params,
 	    &binary->mem_params,
 	    IA_CSS_PARAM_CLASS_CONFIG);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 	err = ia_css_isp_param_copy_isp_mem_if_to_ddr(
 	    &binary->css_params,
 	    &binary->mem_params,
 	    IA_CSS_PARAM_CLASS_STATE);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static bool
@@ -816,7 +817,7 @@ is_sp_stage(struct ia_css_pipeline_stage *stage)
 	return stage->sp_func != IA_CSS_PIPELINE_NO_FUNC;
 }
 
-static enum ia_css_err
+static int
 configure_isp_from_args(
     const struct sh_css_sp_pipeline *pipeline,
     const struct ia_css_binary      *binary,
@@ -838,10 +839,21 @@ configure_isp_from_args(
 	ia_css_dvs_configure(binary, &args->out_frame[0]->info);
 	ia_css_output_configure(binary, &args->out_frame[0]->info);
 	ia_css_raw_configure(pipeline, binary, &args->in_frame->info, &binary->in_frame_info, two_ppc, deinterleaved);
-	ia_css_ref_configure(binary, (const struct ia_css_frame **)args->delay_frames, pipeline->dvs_frame_delay);
-	ia_css_tnr_configure(binary, (const struct ia_css_frame **)args->tnr_frames);
+
+	/*
+	 * FIXME: args->delay_frames can be NULL here
+	 *
+	 * Somehow, the driver at the Intel Atom Yocto tree doesn't seem to
+	 * suffer from the same issue.
+	 *
+	 * Anyway, the function below should now handle a NULL delay_frames
+	 * without crashing, but the pipeline should likely be built without
+	 * adding it at the first place (or there are a hidden bug somewhere)
+	 */
+	ia_css_ref_configure(binary, args->delay_frames, pipeline->dvs_frame_delay);
+	ia_css_tnr_configure(binary, args->tnr_frames);
 	ia_css_bayer_io_config(binary, args);
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 static void
@@ -880,7 +892,7 @@ initialize_stage_frames(struct ia_css_frames_sp *frames)
 #endif
 }
 
-static enum ia_css_err
+static int
 sh_css_sp_init_stage(struct ia_css_binary *binary,
 		     const char *binary_name,
 		     const struct ia_css_blob_info *blob_info,
@@ -893,7 +905,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary,
 		     bool two_ppc) {
 	const struct ia_css_binary_xinfo *xinfo;
 	const struct ia_css_binary_info  *info;
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 	int i;
 	struct ia_css_pipe *pipe = NULL;
 	unsigned int thread_id;
@@ -925,7 +937,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary,
 	if (!info)
 	{
 		sh_css_sp_group.pipe[thread_id].sp_stage_addr[stage] = mmgr_NULL;
-		return IA_CSS_SUCCESS;
+		return 0;
 	}
 
 #if defined(USE_INPUT_SYSTEM_VERSION_2401)
@@ -978,8 +990,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary,
 
 	/* Make sure binary name is smaller than allowed string size */
 	assert(strlen(binary_name) < SH_CSS_MAX_BINARY_NAME - 1);
-	strncpy(sh_css_isp_stage.binary_name, binary_name, SH_CSS_MAX_BINARY_NAME - 1);
-	sh_css_isp_stage.binary_name[SH_CSS_MAX_BINARY_NAME - 1] = 0;
+	strscpy(sh_css_isp_stage.binary_name, binary_name, SH_CSS_MAX_BINARY_NAME);
 	sh_css_isp_stage.mem_initializers = *isp_mem_if;
 
 	/*
@@ -1011,38 +1022,19 @@ sh_css_sp_init_stage(struct ia_css_binary *binary,
 	ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_METADATA, thread_id, &queue_id);
 	sh_css_copy_buffer_attr_to_spbuffer(&sh_css_sp_stage.frames.metadata_buf, queue_id, mmgr_EXCEPTION, IA_CSS_BUFFER_TYPE_METADATA);
 #endif
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 #ifdef USE_INPUT_SYSTEM_VERSION_2401
-#ifndef ISP2401
-	if (args->in_frame)
-	{
-		pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
-		if (!pipe)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		ia_css_get_crop_offsets(pipe, &args->in_frame->info);
-	} else if (&binary->in_frame_info)
-	{
+	if (stage == 0) {
 		pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
 		if (!pipe)
-			return IA_CSS_ERR_INTERNAL_ERROR;
-		ia_css_get_crop_offsets(pipe, &binary->in_frame_info);
-#else
-	if (stage == 0)
-	{
-		if (args->in_frame) {
-			pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
-			if (!pipe)
-				return IA_CSS_ERR_INTERNAL_ERROR;
+			return -EINVAL;
+
+		if (args->in_frame)
 			ia_css_get_crop_offsets(pipe, &args->in_frame->info);
-		} else if (&binary->in_frame_info) {
-			pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
-			if (!pipe)
-				return IA_CSS_ERR_INTERNAL_ERROR;
+		else
 			ia_css_get_crop_offsets(pipe, &binary->in_frame_info);
-		}
-#endif
 	}
 #else
 	(void)pipe; /*avoid build warning*/
@@ -1050,7 +1042,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary,
 
 	err = configure_isp_from_args(&sh_css_sp_group.pipe[thread_id],
 				      binary, args, two_ppc, sh_css_sp_stage.deinterleaved);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
 	initialize_isp_states(binary);
@@ -1072,13 +1064,13 @@ sh_css_sp_init_stage(struct ia_css_binary *binary,
 		<<= binary->vf_downscale_log2;
 	}
 	err = copy_isp_mem_if_to_ddr(binary);
-	if (err != IA_CSS_SUCCESS)
+	if (err)
 		return err;
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-static enum ia_css_err
+static int
 sp_init_stage(struct ia_css_pipeline_stage *stage,
 	      unsigned int pipe_num,
 	      bool xnr,
@@ -1110,7 +1102,7 @@ sp_init_stage(struct ia_css_pipeline_stage *stage,
 	*/
 	struct ia_css_isp_param_css_segments *mem_if = &isp_mem_if;
 
-	enum ia_css_err err = IA_CSS_SUCCESS;
+	int err = 0;
 
 	assert(stage);
 
@@ -1155,7 +1147,7 @@ sp_init_stage(struct ia_css_pipeline_stage *stage,
 		   These will be passed to sh_css_sp_init_stage
 		   and dereferenced there, so passing a NULL
 		   pointer is no good. return an error */
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	}
 
 	err = sh_css_sp_init_stage(binary,
@@ -1190,12 +1182,14 @@ sp_init_sp_stage(struct ia_css_pipeline_stage *stage,
 		break;
 	case IA_CSS_PIPELINE_BIN_COPY:
 		assert(false); /* TBI */
+		break;
 	case IA_CSS_PIPELINE_ISYS_COPY:
 		sh_css_sp_start_isys_copy(args->out_frame[0],
 					  pipe_num, stage->max_input_width, if_config_index);
 		break;
 	case IA_CSS_PIPELINE_NO_FUNC:
 		assert(false);
+		break;
 	}
 }
 
@@ -1347,7 +1341,7 @@ sh_css_sp_init_pipeline(struct ia_css_pipeline *me,
 	}
 #endif
 
-	if (atomisp_hw_is_isp2401) {
+	if (IS_ISP2401) {
 		/* For the shading correction type 1 (the legacy shading table conversion in css is not used),
 		* the parameters are passed to the isp for the shading table centering.
 		*/
@@ -1608,7 +1602,7 @@ sh_css_event_init_irq_mask(void)
 	}
 }
 
-enum ia_css_err
+int
 ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe,
 			 unsigned int or_mask,
 			 unsigned int and_mask) {
@@ -1636,7 +1630,7 @@ ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe,
 
 	pipe_num = ia_css_pipe_get_pipe_num(pipe);
 	if (pipe_num >= IA_CSS_PIPE_ID_NUM)
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	offset = (unsigned int)offsetof(struct host_sp_communication,
 					host2sp_event_irq_mask[pipe_num]);
 	assert(offset % HRT_BUS_BYTES == 0);
@@ -1644,10 +1638,10 @@ ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe,
 		      (unsigned int)sp_address_of(host_sp_com) + offset,
 		      &event_irq_mask, sizeof(event_irq_mask));
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
-enum ia_css_err
+int
 ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
 			  unsigned int *or_mask,
 			  unsigned int *and_mask) {
@@ -1665,7 +1659,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
 
 	pipe_num = ia_css_pipe_get_pipe_num(pipe);
 	if (pipe_num >= IA_CSS_PIPE_ID_NUM)
-		return IA_CSS_ERR_INTERNAL_ERROR;
+		return -EINVAL;
 	offset = (unsigned int)offsetof(struct host_sp_communication,
 					host2sp_event_irq_mask[pipe_num]);
 	assert(offset % HRT_BUS_BYTES == 0);
@@ -1679,7 +1673,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe,
 	if (and_mask)
 		*and_mask = event_irq_mask.and_mask;
 
-	return IA_CSS_SUCCESS;
+	return 0;
 }
 
 void
diff --git a/drivers/staging/media/atomisp/pci/sh_css_sp.h b/drivers/staging/media/atomisp/pci/sh_css_sp.h
index 7d4e13f1e038..153b005becda 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_sp.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_sp.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_stream.c b/drivers/staging/media/atomisp/pci/sh_css_stream.c
index 60bddbb3d4c6..a768ce90f51c 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_stream.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_stream.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_stream_format.c b/drivers/staging/media/atomisp/pci/sh_css_stream_format.c
index 548d4a3567b2..a798f0537050 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_stream_format.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_stream_format.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_stream_format.h b/drivers/staging/media/atomisp/pci/sh_css_stream_format.h
index 32ebd6e0f344..84b7942147ad 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_stream_format.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_stream_format.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_struct.h b/drivers/staging/media/atomisp/pci/sh_css_struct.h
index 81b9598ef8b7..bd260252317a 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_struct.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_struct.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -24,7 +25,7 @@
 */
 
 #include <type_support.h>
-#include <system_types.h>
+#include <system_local.h>
 #include "ia_css_pipeline.h"
 #include "ia_css_pipe_public.h"
 #include "ia_css_frame_public.h"
@@ -64,8 +65,9 @@ struct sh_css {
 	mipi_sizes_for_check[N_CSI_PORTS][IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT];
 	unsigned int                   mipi_frame_size[N_CSI_PORTS];
 #endif
-	hrt_vaddress                   sp_bin_addr;
+	ia_css_ptr                   sp_bin_addr;
 	hrt_data                       page_table_base_index;
+
 	unsigned int
 	size_mem_words; /* \deprecated{Use ia_css_mipi_buffer_config instead.}*/
 	enum ia_css_irq_type           irq_type;
diff --git a/drivers/staging/media/atomisp/pci/sh_css_uds.h b/drivers/staging/media/atomisp/pci/sh_css_uds.h
index d9bcae6007bf..8d9c5c92b692 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_uds.h
+++ b/drivers/staging/media/atomisp/pci/sh_css_uds.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/sh_css_version.c b/drivers/staging/media/atomisp/pci/sh_css_version.c
index eb986e15c7fa..fa6de61e4995 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_version.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_version.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
@@ -13,25 +14,26 @@
  */
 
 #include "../../include/linux/atomisp.h"
+#include "../../include/linux/atomisp_platform.h"
 #include "ia_css_version.h"
 #include "ia_css_version_data.h"
 #include "ia_css_err.h"
 #include "sh_css_firmware.h"
 
-enum ia_css_err
+int
 ia_css_get_version(char *version, int max_size) {
 	char *css_version;
 
-	if (!atomisp_hw_is_isp2401)
+	if (!IS_ISP2401)
 		css_version = ISP2400_CSS_VERSION_STRING;
 	else
 		css_version = ISP2401_CSS_VERSION_STRING;
 
 	if (max_size <= (int)strlen(css_version) + (int)strlen(sh_css_get_fw_version()) + 5)
-		return IA_CSS_ERR_INVALID_ARGUMENTS;
-	strcpy(version, css_version);
+		return -EINVAL;
+	strscpy(version, css_version, max_size);
 	strcat(version, "FW:");
 	strcat(version, sh_css_get_fw_version());
 	strcat(version, "; ");
-	return IA_CSS_SUCCESS;
+	return 0;
 }
diff --git a/drivers/staging/media/atomisp/pci/str2mem_defs.h b/drivers/staging/media/atomisp/pci/str2mem_defs.h
index 1cb62444cf68..e8cb456ac9c7 100644
--- a/drivers/staging/media/atomisp/pci/str2mem_defs.h
+++ b/drivers/staging/media/atomisp/pci/str2mem_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h b/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h
index 60143b8743a2..e0e3a6a66245 100644
--- a/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h
+++ b/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/system_global.h b/drivers/staging/media/atomisp/pci/system_global.h
index 7f833c15f3ce..16d0a2e9a4dc 100644
--- a/drivers/staging/media/atomisp/pci/system_global.h
+++ b/drivers/staging/media/atomisp/pci/system_global.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  *    (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/drivers/staging/media/atomisp/pci/system_local.h b/drivers/staging/media/atomisp/pci/system_local.h
index fbb5daadac9f..dfcc4c2b8f16 100644
--- a/drivers/staging/media/atomisp/pci/system_local.h
+++ b/drivers/staging/media/atomisp/pci/system_local.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  *    (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
diff --git a/drivers/staging/media/atomisp/pci/timed_controller_defs.h b/drivers/staging/media/atomisp/pci/timed_controller_defs.h
index 75451e090f4f..9ddad87702de 100644
--- a/drivers/staging/media/atomisp/pci/timed_controller_defs.h
+++ b/drivers/staging/media/atomisp/pci/timed_controller_defs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/atomisp/pci/version.h b/drivers/staging/media/atomisp/pci/version.h
index bbc4948baea9..a74a7bbfdb0e 100644
--- a/drivers/staging/media/atomisp/pci/version.h
+++ b/drivers/staging/media/atomisp/pci/version.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Support for Intel Camera Imaging ISP subsystem.
  * Copyright (c) 2015, Intel Corporation.
diff --git a/drivers/staging/media/rkvdec/rkvdec-h264.c b/drivers/staging/media/rkvdec/rkvdec-h264.c
index cd4980d06be7..7b66e2743a4f 100644
--- a/drivers/staging/media/rkvdec/rkvdec-h264.c
+++ b/drivers/staging/media/rkvdec/rkvdec-h264.c
@@ -18,11 +18,16 @@
 /* Size with u32 units. */
 #define RKV_CABAC_INIT_BUFFER_SIZE	(3680 + 128)
 #define RKV_RPS_SIZE			((128 + 128) / 4)
-#define RKV_SCALING_LIST_SIZE		(6 * 16 + 6 * 64 + 128)
 #define RKV_ERROR_INFO_SIZE		(256 * 144 * 4)
 
 #define RKVDEC_NUM_REFLIST		3
 
+struct rkvdec_h264_scaling_list {
+	u8 scaling_list_4x4[6][16];
+	u8 scaling_list_8x8[6][64];
+	u8 padding[128];
+};
+
 struct rkvdec_sps_pps_packet {
 	u32 info[8];
 };
@@ -86,7 +91,7 @@ struct rkvdec_ps_field {
 /* Data structure describing auxiliary buffer format. */
 struct rkvdec_h264_priv_tbl {
 	s8 cabac_table[4][464][2];
-	u8 scaling_list[RKV_SCALING_LIST_SIZE];
+	struct rkvdec_h264_scaling_list scaling_list;
 	u32 rps[RKV_RPS_SIZE];
 	struct rkvdec_sps_pps_packet param_set[256];
 	u8 err_info[RKV_ERROR_INFO_SIZE];
@@ -785,56 +790,25 @@ static void assemble_hw_rps(struct rkvdec_ctx *ctx,
 	}
 }
 
-/*
- * NOTE: The values in a scaling list are in zig-zag order, apply inverse
- * scanning process to get the values in matrix order.
- */
-static const u32 zig_zag_4x4[16] = {
-	0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15
-};
-
-static const u32 zig_zag_8x8[64] = {
-	0,  1,  8, 16,  9,  2,  3, 10, 17, 24, 32, 25, 18, 11,  4,  5,
-	12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13,  6,  7, 14, 21, 28,
-	35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51,
-	58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63
-};
-
-static void reorder_scaling_list(struct rkvdec_ctx *ctx,
-				 struct rkvdec_h264_run *run)
+static void assemble_hw_scaling_list(struct rkvdec_ctx *ctx,
+				     struct rkvdec_h264_run *run)
 {
 	const struct v4l2_ctrl_h264_scaling_matrix *scaling = run->scaling_matrix;
-	const size_t num_list_4x4 = ARRAY_SIZE(scaling->scaling_list_4x4);
-	const size_t list_len_4x4 = ARRAY_SIZE(scaling->scaling_list_4x4[0]);
-	const size_t num_list_8x8 = ARRAY_SIZE(scaling->scaling_list_8x8);
-	const size_t list_len_8x8 = ARRAY_SIZE(scaling->scaling_list_8x8[0]);
 	struct rkvdec_h264_ctx *h264_ctx = ctx->priv;
 	struct rkvdec_h264_priv_tbl *tbl = h264_ctx->priv_tbl.cpu;
-	u8 *dst = tbl->scaling_list;
-	const u8 *src;
-	int i, j;
-
-	BUILD_BUG_ON(ARRAY_SIZE(zig_zag_4x4) != list_len_4x4);
-	BUILD_BUG_ON(ARRAY_SIZE(zig_zag_8x8) != list_len_8x8);
-	BUILD_BUG_ON(ARRAY_SIZE(tbl->scaling_list) <
-		     num_list_4x4 * list_len_4x4 +
-		     num_list_8x8 * list_len_8x8);
-
-	src = &scaling->scaling_list_4x4[0][0];
-	for (i = 0; i < num_list_4x4; ++i) {
-		for (j = 0; j < list_len_4x4; ++j)
-			dst[zig_zag_4x4[j]] = src[j];
-		src += list_len_4x4;
-		dst += list_len_4x4;
-	}
 
-	src = &scaling->scaling_list_8x8[0][0];
-	for (i = 0; i < num_list_8x8; ++i) {
-		for (j = 0; j < list_len_8x8; ++j)
-			dst[zig_zag_8x8[j]] = src[j];
-		src += list_len_8x8;
-		dst += list_len_8x8;
-	}
+	BUILD_BUG_ON(sizeof(tbl->scaling_list.scaling_list_4x4) !=
+		     sizeof(scaling->scaling_list_4x4));
+	BUILD_BUG_ON(sizeof(tbl->scaling_list.scaling_list_8x8) !=
+		     sizeof(scaling->scaling_list_8x8));
+
+	memcpy(tbl->scaling_list.scaling_list_4x4,
+	       scaling->scaling_list_4x4,
+	       sizeof(scaling->scaling_list_4x4));
+
+	memcpy(tbl->scaling_list.scaling_list_8x8,
+	       scaling->scaling_list_8x8,
+	       sizeof(scaling->scaling_list_8x8));
 }
 
 /*
@@ -1126,7 +1100,7 @@ static int rkvdec_h264_run(struct rkvdec_ctx *ctx)
 	v4l2_h264_build_b_ref_lists(&reflist_builder, h264_ctx->reflists.b0,
 				    h264_ctx->reflists.b1);
 
-	reorder_scaling_list(ctx, &run);
+	assemble_hw_scaling_list(ctx, &run);
 	assemble_hw_pps(ctx, &run);
 	assemble_hw_rps(ctx, &run);
 	config_registers(ctx, &run);
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c
index 05a85517ff60..bc27f9430eeb 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/pm.h>
 
 #include <media/v4l2-device.h>
 #include <media/v4l2-ioctl.h>
@@ -551,12 +552,18 @@ static const struct of_device_id cedrus_dt_match[] = {
 };
 MODULE_DEVICE_TABLE(of, cedrus_dt_match);
 
+static const struct dev_pm_ops cedrus_dev_pm_ops = {
+	SET_RUNTIME_PM_OPS(cedrus_hw_suspend,
+			   cedrus_hw_resume, NULL)
+};
+
 static struct platform_driver cedrus_driver = {
 	.probe		= cedrus_probe,
 	.remove		= cedrus_remove,
 	.driver		= {
 		.name		= CEDRUS_NAME,
 		.of_match_table	= of_match_ptr(cedrus_dt_match),
+		.pm		= &cedrus_dev_pm_ops,
 	},
 };
 module_platform_driver(cedrus_driver);
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c
index 4a2fc33a1d79..58c48e4fdfe9 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c
@@ -74,6 +74,8 @@ void cedrus_device_run(void *priv)
 
 	v4l2_m2m_buf_copy_metadata(run.src, run.dst, true);
 
+	cedrus_dst_format_set(dev, &ctx->dst_fmt);
+
 	dev->dec_ops[ctx->current_codec]->setup(ctx, &run);
 
 	/* Complete request(s) controls if needed. */
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
index daf5f244f93b..1744e6fcc999 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
@@ -19,6 +19,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/clk.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/reset.h>
 #include <linux/soc/sunxi/sunxi_sram.h>
@@ -140,6 +141,64 @@ static irqreturn_t cedrus_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+int cedrus_hw_suspend(struct device *device)
+{
+	struct cedrus_dev *dev = dev_get_drvdata(device);
+
+	reset_control_assert(dev->rstc);
+
+	clk_disable_unprepare(dev->ram_clk);
+	clk_disable_unprepare(dev->mod_clk);
+	clk_disable_unprepare(dev->ahb_clk);
+
+	return 0;
+}
+
+int cedrus_hw_resume(struct device *device)
+{
+	struct cedrus_dev *dev = dev_get_drvdata(device);
+	int ret;
+
+	ret = clk_prepare_enable(dev->ahb_clk);
+	if (ret) {
+		dev_err(dev->dev, "Failed to enable AHB clock\n");
+
+		return ret;
+	}
+
+	ret = clk_prepare_enable(dev->mod_clk);
+	if (ret) {
+		dev_err(dev->dev, "Failed to enable MOD clock\n");
+
+		goto err_ahb_clk;
+	}
+
+	ret = clk_prepare_enable(dev->ram_clk);
+	if (ret) {
+		dev_err(dev->dev, "Failed to enable RAM clock\n");
+
+		goto err_mod_clk;
+	}
+
+	ret = reset_control_reset(dev->rstc);
+	if (ret) {
+		dev_err(dev->dev, "Failed to apply reset\n");
+
+		goto err_ram_clk;
+	}
+
+	return 0;
+
+err_ram_clk:
+	clk_disable_unprepare(dev->ram_clk);
+err_mod_clk:
+	clk_disable_unprepare(dev->mod_clk);
+err_ahb_clk:
+	clk_disable_unprepare(dev->ahb_clk);
+
+	return ret;
+}
+
 int cedrus_hw_probe(struct cedrus_dev *dev)
 {
 	const struct cedrus_variant *variant;
@@ -236,42 +295,17 @@ int cedrus_hw_probe(struct cedrus_dev *dev)
 		goto err_sram;
 	}
 
-	ret = clk_prepare_enable(dev->ahb_clk);
-	if (ret) {
-		dev_err(dev->dev, "Failed to enable AHB clock\n");
-
-		goto err_sram;
-	}
-
-	ret = clk_prepare_enable(dev->mod_clk);
-	if (ret) {
-		dev_err(dev->dev, "Failed to enable MOD clock\n");
-
-		goto err_ahb_clk;
-	}
-
-	ret = clk_prepare_enable(dev->ram_clk);
-	if (ret) {
-		dev_err(dev->dev, "Failed to enable RAM clock\n");
-
-		goto err_mod_clk;
-	}
-
-	ret = reset_control_reset(dev->rstc);
-	if (ret) {
-		dev_err(dev->dev, "Failed to apply reset\n");
-
-		goto err_ram_clk;
+	pm_runtime_enable(dev->dev);
+	if (!pm_runtime_enabled(dev->dev)) {
+		ret = cedrus_hw_resume(dev->dev);
+		if (ret)
+			goto err_pm;
 	}
 
 	return 0;
 
-err_ram_clk:
-	clk_disable_unprepare(dev->ram_clk);
-err_mod_clk:
-	clk_disable_unprepare(dev->mod_clk);
-err_ahb_clk:
-	clk_disable_unprepare(dev->ahb_clk);
+err_pm:
+	pm_runtime_disable(dev->dev);
 err_sram:
 	sunxi_sram_release(dev->dev);
 err_mem:
@@ -282,11 +316,9 @@ err_mem:
 
 void cedrus_hw_remove(struct cedrus_dev *dev)
 {
-	reset_control_assert(dev->rstc);
-
-	clk_disable_unprepare(dev->ram_clk);
-	clk_disable_unprepare(dev->mod_clk);
-	clk_disable_unprepare(dev->ahb_clk);
+	pm_runtime_disable(dev->dev);
+	if (!pm_runtime_status_suspended(dev->dev))
+		cedrus_hw_suspend(dev->dev);
 
 	sunxi_sram_release(dev->dev);
 
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_hw.h b/drivers/staging/media/sunxi/cedrus/cedrus_hw.h
index 604ff932fbf5..45f641f0bfa2 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_hw.h
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_hw.h
@@ -22,6 +22,9 @@ void cedrus_engine_disable(struct cedrus_dev *dev);
 void cedrus_dst_format_set(struct cedrus_dev *dev,
 			   struct v4l2_pix_format *fmt);
 
+int cedrus_hw_suspend(struct device *device);
+int cedrus_hw_resume(struct device *device);
+
 int cedrus_hw_probe(struct cedrus_dev *dev);
 void cedrus_hw_remove(struct cedrus_dev *dev);
 
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_video.c b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
index 15cf1f10221b..16d82309e7b6 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_video.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
@@ -13,6 +13,8 @@
  * Marek Szyprowski, <m.szyprowski@samsung.com>
  */
 
+#include <linux/pm_runtime.h>
+
 #include <media/videobuf2-dma-contig.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-ioctl.h>
@@ -273,7 +275,6 @@ static int cedrus_s_fmt_vid_cap(struct file *file, void *priv,
 				struct v4l2_format *f)
 {
 	struct cedrus_ctx *ctx = cedrus_file2ctx(file);
-	struct cedrus_dev *dev = ctx->dev;
 	struct vb2_queue *vq;
 	int ret;
 
@@ -287,8 +288,6 @@ static int cedrus_s_fmt_vid_cap(struct file *file, void *priv,
 
 	ctx->dst_fmt = f->fmt.pix;
 
-	cedrus_dst_format_set(dev, &ctx->dst_fmt);
-
 	return 0;
 }
 
@@ -453,12 +452,24 @@ static int cedrus_start_streaming(struct vb2_queue *vq, unsigned int count)
 		return -EINVAL;
 	}
 
-	if (V4L2_TYPE_IS_OUTPUT(vq->type) &&
-	    dev->dec_ops[ctx->current_codec]->start)
-		ret = dev->dec_ops[ctx->current_codec]->start(ctx);
+	if (V4L2_TYPE_IS_OUTPUT(vq->type)) {
+		ret = pm_runtime_get_sync(dev->dev);
+		if (ret < 0)
+			goto err_cleanup;
 
-	if (ret)
-		cedrus_queue_cleanup(vq, VB2_BUF_STATE_QUEUED);
+		if (dev->dec_ops[ctx->current_codec]->start) {
+			ret = dev->dec_ops[ctx->current_codec]->start(ctx);
+			if (ret)
+				goto err_pm;
+		}
+	}
+
+	return 0;
+
+err_pm:
+	pm_runtime_put(dev->dev);
+err_cleanup:
+	cedrus_queue_cleanup(vq, VB2_BUF_STATE_QUEUED);
 
 	return ret;
 }
@@ -468,9 +479,12 @@ static void cedrus_stop_streaming(struct vb2_queue *vq)
 	struct cedrus_ctx *ctx = vb2_get_drv_priv(vq);
 	struct cedrus_dev *dev = ctx->dev;
 
-	if (V4L2_TYPE_IS_OUTPUT(vq->type) &&
-	    dev->dec_ops[ctx->current_codec]->stop)
-		dev->dec_ops[ctx->current_codec]->stop(ctx);
+	if (V4L2_TYPE_IS_OUTPUT(vq->type)) {
+		if (dev->dec_ops[ctx->current_codec]->stop)
+			dev->dec_ops[ctx->current_codec]->stop(ctx);
+
+		pm_runtime_put(dev->dev);
+	}
 
 	cedrus_queue_cleanup(vq, VB2_BUF_STATE_ERROR);
 }
diff --git a/drivers/staging/media/tegra-video/Kconfig b/drivers/staging/media/tegra-video/Kconfig
new file mode 100644
index 000000000000..f6c61ec74386
--- /dev/null
+++ b/drivers/staging/media/tegra-video/Kconfig
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VIDEO_TEGRA
+	tristate "NVIDIA Tegra VI driver"
+	depends on TEGRA_HOST1X
+	depends on VIDEO_V4L2
+	select MEDIA_CONTROLLER
+	select VIDEOBUF2_DMA_CONTIG
+	help
+	  Choose this option if you have an NVIDIA Tegra SoC.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called tegra-video.
diff --git a/drivers/staging/media/tegra-video/Makefile b/drivers/staging/media/tegra-video/Makefile
new file mode 100644
index 000000000000..dfa2ef8f99ef
--- /dev/null
+++ b/drivers/staging/media/tegra-video/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+tegra-video-objs := \
+		video.o \
+		vi.o \
+		csi.o
+
+tegra-video-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210.o
+obj-$(CONFIG_VIDEO_TEGRA) += tegra-video.o
diff --git a/drivers/staging/media/tegra-video/TODO b/drivers/staging/media/tegra-video/TODO
new file mode 100644
index 000000000000..6ceb7549c218
--- /dev/null
+++ b/drivers/staging/media/tegra-video/TODO
@@ -0,0 +1,11 @@
+TODO list
+* Currently driver supports Tegra build-in TPG only with direct media links
+  from CSI to VI. Add kernel config CONFIG_VIDEO_TEGRA_TPG and update the
+  driver to do TPG Vs Sensor media links based on CONFIG_VIDEO_TEGRA_TPG.
+* Add real camera sensor capture support.
+* Add Tegra CSI MIPI pads calibration.
+* Add MIPI clock Settle time computation based on the data rate.
+* Add support for Ganged mode.
+* Add RAW10 packed video format support to Tegra210 video formats.
+* Add support for suspend and resume.
+* Make sure v4l2-compliance tests pass with all of the above implementations.
diff --git a/drivers/staging/media/tegra-video/csi.c b/drivers/staging/media/tegra-video/csi.c
new file mode 100644
index 000000000000..40ea195d141d
--- /dev/null
+++ b/drivers/staging/media/tegra-video/csi.c
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk/tegra.h>
+#include <linux/device.h>
+#include <linux/host1x.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include "csi.h"
+#include "video.h"
+
+static inline struct tegra_csi *
+host1x_client_to_csi(struct host1x_client *client)
+{
+	return container_of(client, struct tegra_csi, client);
+}
+
+static inline struct tegra_csi_channel *to_csi_chan(struct v4l2_subdev *subdev)
+{
+	return container_of(subdev, struct tegra_csi_channel, subdev);
+}
+
+/*
+ * CSI is a separate subdevice which has 6 source pads to generate
+ * test pattern. CSI subdevice pad ops are used only for TPG and
+ * allows below TPG formats.
+ */
+static const struct v4l2_mbus_framefmt tegra_csi_tpg_fmts[] = {
+	{
+		TEGRA_DEF_WIDTH,
+		TEGRA_DEF_HEIGHT,
+		MEDIA_BUS_FMT_SRGGB10_1X10,
+		V4L2_FIELD_NONE,
+		V4L2_COLORSPACE_SRGB
+	},
+	{
+		TEGRA_DEF_WIDTH,
+		TEGRA_DEF_HEIGHT,
+		MEDIA_BUS_FMT_RGB888_1X32_PADHI,
+		V4L2_FIELD_NONE,
+		V4L2_COLORSPACE_SRGB
+	},
+};
+
+static const struct v4l2_frmsize_discrete tegra_csi_tpg_sizes[] = {
+	{ 1280, 720 },
+	{ 1920, 1080 },
+	{ 3840, 2160 },
+};
+
+/*
+ * V4L2 Subdevice Pad Operations
+ */
+static int csi_enum_bus_code(struct v4l2_subdev *subdev,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index >= ARRAY_SIZE(tegra_csi_tpg_fmts))
+		return -EINVAL;
+
+	code->code = tegra_csi_tpg_fmts[code->index].code;
+
+	return 0;
+}
+
+static int csi_get_format(struct v4l2_subdev *subdev,
+			  struct v4l2_subdev_pad_config *cfg,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+
+	fmt->format = csi_chan->format;
+
+	return 0;
+}
+
+static int csi_get_frmrate_table_index(struct tegra_csi *csi, u32 code,
+				       u32 width, u32 height)
+{
+	const struct tpg_framerate *frmrate;
+	unsigned int i;
+
+	frmrate = csi->soc->tpg_frmrate_table;
+	for (i = 0; i < csi->soc->tpg_frmrate_table_size; i++) {
+		if (frmrate[i].code == code &&
+		    frmrate[i].frmsize.width == width &&
+		    frmrate[i].frmsize.height == height) {
+			return i;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static void csi_chan_update_blank_intervals(struct tegra_csi_channel *csi_chan,
+					    u32 code, u32 width, u32 height)
+{
+	struct tegra_csi *csi = csi_chan->csi;
+	const struct tpg_framerate *frmrate = csi->soc->tpg_frmrate_table;
+	int index;
+
+	index = csi_get_frmrate_table_index(csi_chan->csi, code,
+					    width, height);
+	if (index >= 0) {
+		csi_chan->h_blank = frmrate[index].h_blank;
+		csi_chan->v_blank = frmrate[index].v_blank;
+		csi_chan->framerate = frmrate[index].framerate;
+	}
+}
+
+static int csi_enum_framesizes(struct v4l2_subdev *subdev,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_frame_size_enum *fse)
+{
+	unsigned int i;
+
+	if (fse->index >= ARRAY_SIZE(tegra_csi_tpg_sizes))
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(tegra_csi_tpg_fmts); i++)
+		if (fse->code == tegra_csi_tpg_fmts[i].code)
+			break;
+
+	if (i == ARRAY_SIZE(tegra_csi_tpg_fmts))
+		return -EINVAL;
+
+	fse->min_width = tegra_csi_tpg_sizes[fse->index].width;
+	fse->max_width = tegra_csi_tpg_sizes[fse->index].width;
+	fse->min_height = tegra_csi_tpg_sizes[fse->index].height;
+	fse->max_height = tegra_csi_tpg_sizes[fse->index].height;
+
+	return 0;
+}
+
+static int csi_enum_frameintervals(struct v4l2_subdev *subdev,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_frame_interval_enum *fie)
+{
+	struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+	struct tegra_csi *csi = csi_chan->csi;
+	const struct tpg_framerate *frmrate = csi->soc->tpg_frmrate_table;
+	int index;
+
+	/* one framerate per format and resolution */
+	if (fie->index > 0)
+		return -EINVAL;
+
+	index = csi_get_frmrate_table_index(csi_chan->csi, fie->code,
+					    fie->width, fie->height);
+	if (index < 0)
+		return -EINVAL;
+
+	fie->interval.numerator = 1;
+	fie->interval.denominator = frmrate[index].framerate;
+
+	return 0;
+}
+
+static int csi_set_format(struct v4l2_subdev *subdev,
+			  struct v4l2_subdev_pad_config *cfg,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+	struct v4l2_mbus_framefmt *format = &fmt->format;
+	const struct v4l2_frmsize_discrete *sizes;
+	unsigned int i;
+
+	sizes = v4l2_find_nearest_size(tegra_csi_tpg_sizes,
+				       ARRAY_SIZE(tegra_csi_tpg_sizes),
+				       width, height,
+				       format->width, format->width);
+	format->width = sizes->width;
+	format->height = sizes->height;
+
+	for (i = 0; i < ARRAY_SIZE(tegra_csi_tpg_fmts); i++)
+		if (format->code == tegra_csi_tpg_fmts[i].code)
+			break;
+
+	if (i == ARRAY_SIZE(tegra_csi_tpg_fmts))
+		i = 0;
+
+	format->code = tegra_csi_tpg_fmts[i].code;
+	format->field = V4L2_FIELD_NONE;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)
+		return 0;
+
+	/* update blanking intervals from frame rate table and format */
+	csi_chan_update_blank_intervals(csi_chan, format->code,
+					format->width, format->height);
+	csi_chan->format = *format;
+
+	return 0;
+}
+
+/*
+ * V4L2 Subdevice Video Operations
+ */
+static int tegra_csi_g_frame_interval(struct v4l2_subdev *subdev,
+				      struct v4l2_subdev_frame_interval *vfi)
+{
+	struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+
+	vfi->interval.numerator = 1;
+	vfi->interval.denominator = csi_chan->framerate;
+
+	return 0;
+}
+
+static int tegra_csi_s_stream(struct v4l2_subdev *subdev, int enable)
+{
+	struct tegra_vi_channel *chan = v4l2_get_subdev_hostdata(subdev);
+	struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+	struct tegra_csi *csi = csi_chan->csi;
+	int ret = 0;
+
+	csi_chan->pg_mode = chan->pg_mode;
+	if (enable) {
+		ret = pm_runtime_get_sync(csi->dev);
+		if (ret < 0) {
+			dev_err(csi->dev,
+				"failed to get runtime PM: %d\n", ret);
+			pm_runtime_put_noidle(csi->dev);
+			return ret;
+		}
+
+		ret = csi->ops->csi_start_streaming(csi_chan);
+		if (ret < 0)
+			goto rpm_put;
+
+		return 0;
+	}
+
+	csi->ops->csi_stop_streaming(csi_chan);
+
+rpm_put:
+	pm_runtime_put(csi->dev);
+	return ret;
+}
+
+/*
+ * V4L2 Subdevice Operations
+ */
+static const struct v4l2_subdev_video_ops tegra_csi_video_ops = {
+	.s_stream = tegra_csi_s_stream,
+	.g_frame_interval = tegra_csi_g_frame_interval,
+	.s_frame_interval = tegra_csi_g_frame_interval,
+};
+
+static const struct v4l2_subdev_pad_ops tegra_csi_pad_ops = {
+	.enum_mbus_code		= csi_enum_bus_code,
+	.enum_frame_size	= csi_enum_framesizes,
+	.enum_frame_interval	= csi_enum_frameintervals,
+	.get_fmt		= csi_get_format,
+	.set_fmt		= csi_set_format,
+};
+
+static const struct v4l2_subdev_ops tegra_csi_ops = {
+	.video  = &tegra_csi_video_ops,
+	.pad    = &tegra_csi_pad_ops,
+};
+
+static int tegra_csi_tpg_channels_alloc(struct tegra_csi *csi)
+{
+	struct device_node *node = csi->dev->of_node;
+	unsigned int port_num;
+	struct tegra_csi_channel *chan;
+	unsigned int tpg_channels = csi->soc->csi_max_channels;
+
+	/* allocate CSI channel for each CSI x2 ports */
+	for (port_num = 0; port_num < tpg_channels; port_num++) {
+		chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+		if (!chan)
+			return -ENOMEM;
+
+		list_add_tail(&chan->list, &csi->csi_chans);
+		chan->csi = csi;
+		chan->csi_port_num = port_num;
+		chan->numlanes = 2;
+		chan->of_node = node;
+		chan->numpads = 1;
+		chan->pads[0].flags = MEDIA_PAD_FL_SOURCE;
+	}
+
+	return 0;
+}
+
+static int tegra_csi_channel_init(struct tegra_csi_channel *chan)
+{
+	struct tegra_csi *csi = chan->csi;
+	struct v4l2_subdev *subdev;
+	int ret;
+
+	/* initialize the default format */
+	chan->format.code = MEDIA_BUS_FMT_SRGGB10_1X10;
+	chan->format.field = V4L2_FIELD_NONE;
+	chan->format.colorspace = V4L2_COLORSPACE_SRGB;
+	chan->format.width = TEGRA_DEF_WIDTH;
+	chan->format.height = TEGRA_DEF_HEIGHT;
+	csi_chan_update_blank_intervals(chan, chan->format.code,
+					chan->format.width,
+					chan->format.height);
+	/* initialize V4L2 subdevice and media entity */
+	subdev = &chan->subdev;
+	v4l2_subdev_init(subdev, &tegra_csi_ops);
+	subdev->dev = csi->dev;
+	snprintf(subdev->name, V4L2_SUBDEV_NAME_SIZE, "%s-%d", "tpg",
+		 chan->csi_port_num);
+
+	v4l2_set_subdevdata(subdev, chan);
+	subdev->fwnode = of_fwnode_handle(chan->of_node);
+	subdev->entity.function = MEDIA_ENT_F_VID_IF_BRIDGE;
+
+	/* initialize media entity pads */
+	ret = media_entity_pads_init(&subdev->entity, chan->numpads,
+				     chan->pads);
+	if (ret < 0) {
+		dev_err(csi->dev,
+			"failed to initialize media entity: %d\n", ret);
+		subdev->dev = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
+void tegra_csi_error_recover(struct v4l2_subdev *sd)
+{
+	struct tegra_csi_channel *csi_chan = to_csi_chan(sd);
+	struct tegra_csi *csi = csi_chan->csi;
+
+	/* stop streaming during error recovery */
+	csi->ops->csi_stop_streaming(csi_chan);
+	csi->ops->csi_err_recover(csi_chan);
+	csi->ops->csi_start_streaming(csi_chan);
+}
+
+static int tegra_csi_channels_init(struct tegra_csi *csi)
+{
+	struct tegra_csi_channel *chan;
+	int ret;
+
+	list_for_each_entry(chan, &csi->csi_chans, list) {
+		ret = tegra_csi_channel_init(chan);
+		if (ret) {
+			dev_err(csi->dev,
+				"failed to initialize channel-%d: %d\n",
+				chan->csi_port_num, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void tegra_csi_channels_cleanup(struct tegra_csi *csi)
+{
+	struct v4l2_subdev *subdev;
+	struct tegra_csi_channel *chan, *tmp;
+
+	list_for_each_entry_safe(chan, tmp, &csi->csi_chans, list) {
+		subdev = &chan->subdev;
+		if (subdev->dev)
+			media_entity_cleanup(&subdev->entity);
+		list_del(&chan->list);
+		kfree(chan);
+	}
+}
+
+static int __maybe_unused csi_runtime_suspend(struct device *dev)
+{
+	struct tegra_csi *csi = dev_get_drvdata(dev);
+
+	clk_bulk_disable_unprepare(csi->soc->num_clks, csi->clks);
+
+	return 0;
+}
+
+static int __maybe_unused csi_runtime_resume(struct device *dev)
+{
+	struct tegra_csi *csi = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_bulk_prepare_enable(csi->soc->num_clks, csi->clks);
+	if (ret < 0) {
+		dev_err(csi->dev, "failed to enable clocks: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int tegra_csi_init(struct host1x_client *client)
+{
+	struct tegra_csi *csi = host1x_client_to_csi(client);
+	struct tegra_video_device *vid = dev_get_drvdata(client->host);
+	int ret;
+
+	INIT_LIST_HEAD(&csi->csi_chans);
+
+	ret = tegra_csi_tpg_channels_alloc(csi);
+	if (ret < 0) {
+		dev_err(csi->dev,
+			"failed to allocate tpg channels: %d\n", ret);
+		goto cleanup;
+	}
+
+	ret = tegra_csi_channels_init(csi);
+	if (ret < 0)
+		goto cleanup;
+
+	vid->csi = csi;
+
+	return 0;
+
+cleanup:
+	tegra_csi_channels_cleanup(csi);
+	return ret;
+}
+
+static int tegra_csi_exit(struct host1x_client *client)
+{
+	struct tegra_csi *csi = host1x_client_to_csi(client);
+
+	tegra_csi_channels_cleanup(csi);
+
+	return 0;
+}
+
+static const struct host1x_client_ops csi_client_ops = {
+	.init = tegra_csi_init,
+	.exit = tegra_csi_exit,
+};
+
+static int tegra_csi_probe(struct platform_device *pdev)
+{
+	struct tegra_csi *csi;
+	unsigned int i;
+	int ret;
+
+	csi = devm_kzalloc(&pdev->dev, sizeof(*csi), GFP_KERNEL);
+	if (!csi)
+		return -ENOMEM;
+
+	csi->iomem = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(csi->iomem))
+		return PTR_ERR(csi->iomem);
+
+	csi->soc = of_device_get_match_data(&pdev->dev);
+
+	csi->clks = devm_kcalloc(&pdev->dev, csi->soc->num_clks,
+				 sizeof(*csi->clks), GFP_KERNEL);
+	if (!csi->clks)
+		return -ENOMEM;
+
+	for (i = 0; i < csi->soc->num_clks; i++)
+		csi->clks[i].id = csi->soc->clk_names[i];
+
+	ret = devm_clk_bulk_get(&pdev->dev, csi->soc->num_clks, csi->clks);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to get the clocks: %d\n", ret);
+		return ret;
+	}
+
+	if (!pdev->dev.pm_domain) {
+		ret = -ENOENT;
+		dev_warn(&pdev->dev, "PM domain is not attached: %d\n", ret);
+		return ret;
+	}
+
+	csi->dev = &pdev->dev;
+	csi->ops = csi->soc->ops;
+	platform_set_drvdata(pdev, csi);
+	pm_runtime_enable(&pdev->dev);
+
+	/* initialize host1x interface */
+	INIT_LIST_HEAD(&csi->client.list);
+	csi->client.ops = &csi_client_ops;
+	csi->client.dev = &pdev->dev;
+
+	ret = host1x_client_register(&csi->client);
+	if (ret < 0) {
+		dev_err(&pdev->dev,
+			"failed to register host1x client: %d\n", ret);
+		goto rpm_disable;
+	}
+
+	return 0;
+
+rpm_disable:
+	pm_runtime_disable(&pdev->dev);
+	return ret;
+}
+
+static int tegra_csi_remove(struct platform_device *pdev)
+{
+	struct tegra_csi *csi = platform_get_drvdata(pdev);
+	int err;
+
+	err = host1x_client_unregister(&csi->client);
+	if (err < 0) {
+		dev_err(&pdev->dev,
+			"failed to unregister host1x client: %d\n", err);
+		return err;
+	}
+
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static const struct of_device_id tegra_csi_of_id_table[] = {
+#if defined(CONFIG_ARCH_TEGRA_210_SOC)
+	{ .compatible = "nvidia,tegra210-csi", .data = &tegra210_csi_soc },
+#endif
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tegra_csi_of_id_table);
+
+static const struct dev_pm_ops tegra_csi_pm_ops = {
+	SET_RUNTIME_PM_OPS(csi_runtime_suspend, csi_runtime_resume, NULL)
+};
+
+struct platform_driver tegra_csi_driver = {
+	.driver = {
+		.name		= "tegra-csi",
+		.of_match_table	= tegra_csi_of_id_table,
+		.pm		= &tegra_csi_pm_ops,
+	},
+	.probe			= tegra_csi_probe,
+	.remove			= tegra_csi_remove,
+};
diff --git a/drivers/staging/media/tegra-video/csi.h b/drivers/staging/media/tegra-video/csi.h
new file mode 100644
index 000000000000..93bd2a05797d
--- /dev/null
+++ b/drivers/staging/media/tegra-video/csi.h
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#ifndef __TEGRA_CSI_H__
+#define __TEGRA_CSI_H__
+
+#include <media/media-entity.h>
+#include <media/v4l2-subdev.h>
+
+/*
+ * Each CSI brick supports max of 4 lanes that can be used as either
+ * one x4 port using both CILA and CILB partitions of a CSI brick or can
+ * be used as two x2 ports with one x2 from CILA and the other x2 from
+ * CILB.
+ */
+#define CSI_PORTS_PER_BRICK	2
+
+/* each CSI channel can have one sink and one source pads */
+#define TEGRA_CSI_PADS_NUM	2
+
+enum tegra_csi_cil_port {
+	PORT_A = 0,
+	PORT_B,
+};
+
+enum tegra_csi_block {
+	CSI_CIL_AB = 0,
+	CSI_CIL_CD,
+	CSI_CIL_EF,
+};
+
+struct tegra_csi;
+
+/**
+ * struct tegra_csi_channel - Tegra CSI channel
+ *
+ * @list: list head for this entry
+ * @subdev: V4L2 subdevice associated with this channel
+ * @pads: media pads for the subdevice entity
+ * @numpads: number of pads.
+ * @csi: Tegra CSI device structure
+ * @of_node: csi device tree node
+ * @numlanes: number of lanes used per port/channel
+ * @csi_port_num: CSI channel port number
+ * @pg_mode: test pattern generator mode for channel
+ * @format: active format of the channel
+ * @framerate: active framerate for TPG
+ * @h_blank: horizontal blanking for TPG active format
+ * @v_blank: vertical blanking for TPG active format
+ */
+struct tegra_csi_channel {
+	struct list_head list;
+	struct v4l2_subdev subdev;
+	struct media_pad pads[TEGRA_CSI_PADS_NUM];
+	unsigned int numpads;
+	struct tegra_csi *csi;
+	struct device_node *of_node;
+	unsigned int numlanes;
+	u8 csi_port_num;
+	u8 pg_mode;
+	struct v4l2_mbus_framefmt format;
+	unsigned int framerate;
+	unsigned int h_blank;
+	unsigned int v_blank;
+};
+
+/**
+ * struct tpg_framerate - Tegra CSI TPG framerate configuration
+ *
+ * @frmsize: frame resolution
+ * @code: media bus format code
+ * @h_blank: horizontal blanking used for TPG
+ * @v_blank: vertical blanking interval used for TPG
+ * @framerate: framerate achieved with the corresponding blanking intervals,
+ *		format and resolution.
+ */
+struct tpg_framerate {
+	struct v4l2_frmsize_discrete frmsize;
+	u32 code;
+	unsigned int h_blank;
+	unsigned int v_blank;
+	unsigned int framerate;
+};
+
+/**
+ * struct tegra_csi_ops - Tegra CSI operations
+ *
+ * @csi_start_streaming: programs csi hardware to enable streaming.
+ * @csi_stop_streaming: programs csi hardware to disable streaming.
+ * @csi_err_recover: csi hardware block recovery in case of any capture errors
+ *		due to missing source stream or due to improper csi input from
+ *		the external source.
+ */
+struct tegra_csi_ops {
+	int (*csi_start_streaming)(struct tegra_csi_channel *csi_chan);
+	void (*csi_stop_streaming)(struct tegra_csi_channel *csi_chan);
+	void (*csi_err_recover)(struct tegra_csi_channel *csi_chan);
+};
+
+/**
+ * struct tegra_csi_soc - NVIDIA Tegra CSI SoC structure
+ *
+ * @ops: csi hardware operations
+ * @csi_max_channels: supported max streaming channels
+ * @clk_names: csi and cil clock names
+ * @num_clks: total clocks count
+ * @tpg_frmrate_table: csi tpg frame rate table with blanking intervals
+ * @tpg_frmrate_table_size: size of frame rate table
+ */
+struct tegra_csi_soc {
+	const struct tegra_csi_ops *ops;
+	unsigned int csi_max_channels;
+	const char * const *clk_names;
+	unsigned int num_clks;
+	const struct tpg_framerate *tpg_frmrate_table;
+	unsigned int tpg_frmrate_table_size;
+};
+
+/**
+ * struct tegra_csi - NVIDIA Tegra CSI device structure
+ *
+ * @dev: device struct
+ * @client: host1x_client struct
+ * @iomem: register base
+ * @clks: clock for CSI and CIL
+ * @soc: pointer to SoC data structure
+ * @ops: csi operations
+ * @channels: list head for CSI channels
+ */
+struct tegra_csi {
+	struct device *dev;
+	struct host1x_client client;
+	void __iomem *iomem;
+	struct clk_bulk_data *clks;
+	const struct tegra_csi_soc *soc;
+	const struct tegra_csi_ops *ops;
+	struct list_head csi_chans;
+};
+
+#if defined(CONFIG_ARCH_TEGRA_210_SOC)
+extern const struct tegra_csi_soc tegra210_csi_soc;
+#endif
+
+void tegra_csi_error_recover(struct v4l2_subdev *subdev);
+#endif
diff --git a/drivers/staging/media/tegra-video/tegra210.c b/drivers/staging/media/tegra-video/tegra210.c
new file mode 100644
index 000000000000..3baa4e314203
--- /dev/null
+++ b/drivers/staging/media/tegra-video/tegra210.c
@@ -0,0 +1,978 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
+ */
+
+/*
+ * This source file contains Tegra210 supported video formats,
+ * VI and CSI SoC specific data, operations and registers accessors.
+ */
+#include <linux/clk.h>
+#include <linux/clk/tegra.h>
+#include <linux/delay.h>
+#include <linux/host1x.h>
+#include <linux/kthread.h>
+
+#include "csi.h"
+#include "vi.h"
+
+#define TEGRA_VI_SYNCPT_WAIT_TIMEOUT			msecs_to_jiffies(200)
+
+/* Tegra210 VI registers */
+#define TEGRA_VI_CFG_VI_INCR_SYNCPT			0x000
+#define   VI_CFG_VI_INCR_SYNCPT_COND(x)			(((x) & 0xff) << 8)
+#define   VI_CSI_PP_FRAME_START(port)			(5 + (port) * 4)
+#define   VI_CSI_MW_ACK_DONE(port)			(7 + (port) * 4)
+#define TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL		0x004
+#define   VI_INCR_SYNCPT_NO_STALL			BIT(8)
+#define TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR		0x008
+#define TEGRA_VI_CFG_CG_CTRL				0x0b8
+#define   VI_CG_2ND_LEVEL_EN				0x1
+
+/* Tegra210 VI CSI registers */
+#define TEGRA_VI_CSI_SW_RESET				0x000
+#define TEGRA_VI_CSI_SINGLE_SHOT			0x004
+#define   SINGLE_SHOT_CAPTURE				0x1
+#define TEGRA_VI_CSI_IMAGE_DEF				0x00c
+#define   BYPASS_PXL_TRANSFORM_OFFSET			24
+#define   IMAGE_DEF_FORMAT_OFFSET			16
+#define   IMAGE_DEF_DEST_MEM				0x1
+#define TEGRA_VI_CSI_IMAGE_SIZE				0x018
+#define   IMAGE_SIZE_HEIGHT_OFFSET			16
+#define TEGRA_VI_CSI_IMAGE_SIZE_WC			0x01c
+#define TEGRA_VI_CSI_IMAGE_DT				0x020
+#define TEGRA_VI_CSI_SURFACE0_OFFSET_MSB		0x024
+#define TEGRA_VI_CSI_SURFACE0_OFFSET_LSB		0x028
+#define TEGRA_VI_CSI_SURFACE1_OFFSET_MSB		0x02c
+#define TEGRA_VI_CSI_SURFACE1_OFFSET_LSB		0x030
+#define TEGRA_VI_CSI_SURFACE2_OFFSET_MSB		0x034
+#define TEGRA_VI_CSI_SURFACE2_OFFSET_LSB		0x038
+#define TEGRA_VI_CSI_SURFACE0_STRIDE			0x054
+#define TEGRA_VI_CSI_SURFACE1_STRIDE			0x058
+#define TEGRA_VI_CSI_SURFACE2_STRIDE			0x05c
+#define TEGRA_VI_CSI_SURFACE_HEIGHT0			0x060
+#define TEGRA_VI_CSI_ERROR_STATUS			0x084
+
+/* Tegra210 CSI Pixel Parser registers: Starts from 0x838, offset 0x0 */
+#define TEGRA_CSI_INPUT_STREAM_CONTROL                  0x000
+#define   CSI_SKIP_PACKET_THRESHOLD_OFFSET		16
+#define TEGRA_CSI_PIXEL_STREAM_CONTROL0			0x004
+#define   CSI_PP_PACKET_HEADER_SENT			BIT(4)
+#define   CSI_PP_DATA_IDENTIFIER_ENABLE			BIT(5)
+#define   CSI_PP_WORD_COUNT_SELECT_HEADER		BIT(6)
+#define   CSI_PP_CRC_CHECK_ENABLE			BIT(7)
+#define   CSI_PP_WC_CHECK				BIT(8)
+#define   CSI_PP_OUTPUT_FORMAT_STORE			(0x3 << 16)
+#define   CSI_PPA_PAD_LINE_NOPAD			(0x2 << 24)
+#define   CSI_PP_HEADER_EC_DISABLE			(0x1 << 27)
+#define   CSI_PPA_PAD_FRAME_NOPAD			(0x2 << 28)
+#define TEGRA_CSI_PIXEL_STREAM_CONTROL1                 0x008
+#define   CSI_PP_TOP_FIELD_FRAME_OFFSET			0
+#define   CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET		4
+#define TEGRA_CSI_PIXEL_STREAM_GAP                      0x00c
+#define   PP_FRAME_MIN_GAP_OFFSET			16
+#define TEGRA_CSI_PIXEL_STREAM_PP_COMMAND               0x010
+#define   CSI_PP_ENABLE					0x1
+#define   CSI_PP_DISABLE				0x2
+#define   CSI_PP_RST					0x3
+#define   CSI_PP_SINGLE_SHOT_ENABLE			(0x1 << 2)
+#define   CSI_PP_START_MARKER_FRAME_MAX_OFFSET		12
+#define TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME           0x014
+#define TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK           0x018
+#define TEGRA_CSI_PIXEL_PARSER_STATUS                   0x01c
+
+/* Tegra210 CSI PHY registers */
+/* CSI_PHY_CIL_COMMAND_0 offset 0x0d0 from TEGRA_CSI_PIXEL_PARSER_0_BASE */
+#define TEGRA_CSI_PHY_CIL_COMMAND                       0x0d0
+#define   CSI_A_PHY_CIL_NOP				0x0
+#define   CSI_A_PHY_CIL_ENABLE				0x1
+#define   CSI_A_PHY_CIL_DISABLE				0x2
+#define   CSI_A_PHY_CIL_ENABLE_MASK			0x3
+#define   CSI_B_PHY_CIL_NOP				(0x0 << 8)
+#define   CSI_B_PHY_CIL_ENABLE				(0x1 << 8)
+#define   CSI_B_PHY_CIL_DISABLE				(0x2 << 8)
+#define   CSI_B_PHY_CIL_ENABLE_MASK			(0x3 << 8)
+
+#define TEGRA_CSI_CIL_PAD_CONFIG0                       0x000
+#define   BRICK_CLOCK_A_4X				(0x1 << 16)
+#define   BRICK_CLOCK_B_4X				(0x2 << 16)
+#define TEGRA_CSI_CIL_PAD_CONFIG1                       0x004
+#define TEGRA_CSI_CIL_PHY_CONTROL                       0x008
+#define TEGRA_CSI_CIL_INTERRUPT_MASK                    0x00c
+#define TEGRA_CSI_CIL_STATUS                            0x010
+#define TEGRA_CSI_CILX_STATUS                           0x014
+#define TEGRA_CSI_CIL_SW_SENSOR_RESET                   0x020
+
+#define TEGRA_CSI_PATTERN_GENERATOR_CTRL		0x000
+#define   PG_MODE_OFFSET				2
+#define   PG_ENABLE					0x1
+#define   PG_DISABLE					0x0
+#define TEGRA_CSI_PG_BLANK				0x004
+#define   PG_VBLANK_OFFSET				16
+#define TEGRA_CSI_PG_PHASE				0x008
+#define TEGRA_CSI_PG_RED_FREQ				0x00c
+#define   PG_RED_VERT_INIT_FREQ_OFFSET			16
+#define   PG_RED_HOR_INIT_FREQ_OFFSET			0
+#define TEGRA_CSI_PG_RED_FREQ_RATE			0x010
+#define TEGRA_CSI_PG_GREEN_FREQ				0x014
+#define   PG_GREEN_VERT_INIT_FREQ_OFFSET		16
+#define   PG_GREEN_HOR_INIT_FREQ_OFFSET			0
+#define TEGRA_CSI_PG_GREEN_FREQ_RATE			0x018
+#define TEGRA_CSI_PG_BLUE_FREQ				0x01c
+#define   PG_BLUE_VERT_INIT_FREQ_OFFSET			16
+#define   PG_BLUE_HOR_INIT_FREQ_OFFSET			0
+#define TEGRA_CSI_PG_BLUE_FREQ_RATE			0x020
+#define TEGRA_CSI_PG_AOHDR				0x024
+#define TEGRA_CSI_CSI_SW_STATUS_RESET			0x214
+#define TEGRA_CSI_CLKEN_OVERRIDE			0x218
+
+#define TEGRA210_CSI_PORT_OFFSET			0x34
+#define TEGRA210_CSI_CIL_OFFSET				0x0f4
+#define TEGRA210_CSI_TPG_OFFSET				0x18c
+
+#define CSI_PP_OFFSET(block)				((block) * 0x800)
+#define TEGRA210_VI_CSI_BASE(x)				(0x100 + (x) * 0x100)
+
+/* Tegra210 VI registers accessors */
+static void tegra_vi_write(struct tegra_vi_channel *chan, unsigned int addr,
+			   u32 val)
+{
+	writel_relaxed(val, chan->vi->iomem + addr);
+}
+
+static u32 tegra_vi_read(struct tegra_vi_channel *chan, unsigned int addr)
+{
+	return readl_relaxed(chan->vi->iomem + addr);
+}
+
+/* Tegra210 VI_CSI registers accessors */
+static void vi_csi_write(struct tegra_vi_channel *chan, unsigned int addr,
+			 u32 val)
+{
+	void __iomem *vi_csi_base;
+
+	vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(chan->portno);
+
+	writel_relaxed(val, vi_csi_base + addr);
+}
+
+static u32 vi_csi_read(struct tegra_vi_channel *chan, unsigned int addr)
+{
+	void __iomem *vi_csi_base;
+
+	vi_csi_base = chan->vi->iomem + TEGRA210_VI_CSI_BASE(chan->portno);
+
+	return readl_relaxed(vi_csi_base + addr);
+}
+
+/*
+ * Tegra210 VI channel capture operations
+ */
+static int tegra_channel_capture_setup(struct tegra_vi_channel *chan)
+{
+	u32 height = chan->format.height;
+	u32 width = chan->format.width;
+	u32 format = chan->fmtinfo->img_fmt;
+	u32 data_type = chan->fmtinfo->img_dt;
+	u32 word_count = (width * chan->fmtinfo->bit_width) / 8;
+
+	vi_csi_write(chan, TEGRA_VI_CSI_ERROR_STATUS, 0xffffffff);
+	vi_csi_write(chan, TEGRA_VI_CSI_IMAGE_DEF,
+		     ((chan->pg_mode ? 0 : 1) << BYPASS_PXL_TRANSFORM_OFFSET) |
+		     (format << IMAGE_DEF_FORMAT_OFFSET) |
+		     IMAGE_DEF_DEST_MEM);
+	vi_csi_write(chan, TEGRA_VI_CSI_IMAGE_DT, data_type);
+	vi_csi_write(chan, TEGRA_VI_CSI_IMAGE_SIZE_WC, word_count);
+	vi_csi_write(chan, TEGRA_VI_CSI_IMAGE_SIZE,
+		     (height << IMAGE_SIZE_HEIGHT_OFFSET) | width);
+	return 0;
+}
+
+static void tegra_channel_vi_soft_reset(struct tegra_vi_channel *chan)
+{
+	/* disable clock gating to enable continuous clock */
+	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, 0);
+	/*
+	 * Soft reset memory client interface, pixel format logic, sensor
+	 * control logic, and a shadow copy logic to bring VI to clean state.
+	 */
+	vi_csi_write(chan, TEGRA_VI_CSI_SW_RESET, 0xf);
+	usleep_range(100, 200);
+	vi_csi_write(chan, TEGRA_VI_CSI_SW_RESET, 0x0);
+
+	/* enable back VI clock gating */
+	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
+}
+
+static void tegra_channel_capture_error_recover(struct tegra_vi_channel *chan)
+{
+	struct v4l2_subdev *subdev;
+	u32 val;
+
+	/*
+	 * Recover VI and CSI hardware blocks in case of missing frame start
+	 * events due to source not streaming or noisy csi inputs from the
+	 * external source or many outstanding frame start or MW_ACK_DONE
+	 * events which can cause CSI and VI hardware hang.
+	 * This helps to have a clean capture for next frame.
+	 */
+	val = vi_csi_read(chan, TEGRA_VI_CSI_ERROR_STATUS);
+	dev_dbg(&chan->video.dev, "TEGRA_VI_CSI_ERROR_STATUS 0x%08x\n", val);
+	vi_csi_write(chan, TEGRA_VI_CSI_ERROR_STATUS, val);
+
+	val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
+	dev_dbg(&chan->video.dev,
+		"TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR 0x%08x\n", val);
+	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
+
+	/* recover VI by issuing software reset and re-setup for capture */
+	tegra_channel_vi_soft_reset(chan);
+	tegra_channel_capture_setup(chan);
+
+	/* recover CSI block */
+	subdev = tegra_channel_get_remote_subdev(chan);
+	tegra_csi_error_recover(subdev);
+}
+
+static struct tegra_channel_buffer *
+dequeue_buf_done(struct tegra_vi_channel *chan)
+{
+	struct tegra_channel_buffer *buf = NULL;
+
+	spin_lock(&chan->done_lock);
+	if (list_empty(&chan->done)) {
+		spin_unlock(&chan->done_lock);
+		return NULL;
+	}
+
+	buf = list_first_entry(&chan->done,
+			       struct tegra_channel_buffer, queue);
+	if (buf)
+		list_del_init(&buf->queue);
+	spin_unlock(&chan->done_lock);
+
+	return buf;
+}
+
+static void release_buffer(struct tegra_vi_channel *chan,
+			   struct tegra_channel_buffer *buf,
+			   enum vb2_buffer_state state)
+{
+	struct vb2_v4l2_buffer *vb = &buf->buf;
+
+	vb->sequence = chan->sequence++;
+	vb->field = V4L2_FIELD_NONE;
+	vb->vb2_buf.timestamp = ktime_get_ns();
+	vb2_buffer_done(&vb->vb2_buf, state);
+}
+
+static int tegra_channel_capture_frame(struct tegra_vi_channel *chan,
+				       struct tegra_channel_buffer *buf)
+{
+	u32 thresh, value, frame_start, mw_ack_done;
+	int bytes_per_line = chan->format.bytesperline;
+	int err;
+
+	/* program buffer address by using surface 0 */
+	vi_csi_write(chan, TEGRA_VI_CSI_SURFACE0_OFFSET_MSB,
+		     (u64)buf->addr >> 32);
+	vi_csi_write(chan, TEGRA_VI_CSI_SURFACE0_OFFSET_LSB, buf->addr);
+	vi_csi_write(chan, TEGRA_VI_CSI_SURFACE0_STRIDE, bytes_per_line);
+
+	/*
+	 * Tegra VI block interacts with host1x syncpt for synchronizing
+	 * programmed condition of capture state and hardware operation.
+	 * Frame start and Memory write acknowledge syncpts has their own
+	 * FIFO of depth 2.
+	 *
+	 * Syncpoint trigger conditions set through VI_INCR_SYNCPT register
+	 * are added to HW syncpt FIFO and when the HW triggers, syncpt
+	 * condition is removed from the FIFO and counter at syncpoint index
+	 * will be incremented by the hardware and software can wait for
+	 * counter to reach threshold to synchronize capturing frame with the
+	 * hardware capture events.
+	 */
+
+	/* increase channel syncpoint threshold for FRAME_START */
+	thresh = host1x_syncpt_incr_max(chan->frame_start_sp, 1);
+
+	/* Program FRAME_START trigger condition syncpt request */
+	frame_start = VI_CSI_PP_FRAME_START(chan->portno);
+	value = VI_CFG_VI_INCR_SYNCPT_COND(frame_start) |
+		host1x_syncpt_id(chan->frame_start_sp);
+	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
+
+	/* increase channel syncpoint threshold for MW_ACK_DONE */
+	buf->mw_ack_sp_thresh = host1x_syncpt_incr_max(chan->mw_ack_sp, 1);
+
+	/* Program MW_ACK_DONE trigger condition syncpt request */
+	mw_ack_done = VI_CSI_MW_ACK_DONE(chan->portno);
+	value = VI_CFG_VI_INCR_SYNCPT_COND(mw_ack_done) |
+		host1x_syncpt_id(chan->mw_ack_sp);
+	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT, value);
+
+	/* enable single shot capture */
+	vi_csi_write(chan, TEGRA_VI_CSI_SINGLE_SHOT, SINGLE_SHOT_CAPTURE);
+
+	/* wait for syncpt counter to reach frame start event threshold */
+	err = host1x_syncpt_wait(chan->frame_start_sp, thresh,
+				 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
+	if (err) {
+		dev_err_ratelimited(&chan->video.dev,
+				    "frame start syncpt timeout: %d\n", err);
+		/* increment syncpoint counter for timedout events */
+		host1x_syncpt_incr(chan->frame_start_sp);
+		spin_lock(&chan->sp_incr_lock);
+		host1x_syncpt_incr(chan->mw_ack_sp);
+		spin_unlock(&chan->sp_incr_lock);
+		/* clear errors and recover */
+		tegra_channel_capture_error_recover(chan);
+		release_buffer(chan, buf, VB2_BUF_STATE_ERROR);
+		return err;
+	}
+
+	/* move buffer to capture done queue */
+	spin_lock(&chan->done_lock);
+	list_add_tail(&buf->queue, &chan->done);
+	spin_unlock(&chan->done_lock);
+
+	/* wait up kthread for capture done */
+	wake_up_interruptible(&chan->done_wait);
+
+	return 0;
+}
+
+static void tegra_channel_capture_done(struct tegra_vi_channel *chan,
+				       struct tegra_channel_buffer *buf)
+{
+	enum vb2_buffer_state state = VB2_BUF_STATE_DONE;
+	u32 value;
+	int ret;
+
+	/* wait for syncpt counter to reach MW_ACK_DONE event threshold */
+	ret = host1x_syncpt_wait(chan->mw_ack_sp, buf->mw_ack_sp_thresh,
+				 TEGRA_VI_SYNCPT_WAIT_TIMEOUT, &value);
+	if (ret) {
+		dev_err_ratelimited(&chan->video.dev,
+				    "MW_ACK_DONE syncpt timeout: %d\n", ret);
+		state = VB2_BUF_STATE_ERROR;
+		/* increment syncpoint counter for timedout event */
+		spin_lock(&chan->sp_incr_lock);
+		host1x_syncpt_incr(chan->mw_ack_sp);
+		spin_unlock(&chan->sp_incr_lock);
+	}
+
+	release_buffer(chan, buf, state);
+}
+
+static int chan_capture_kthread_start(void *data)
+{
+	struct tegra_vi_channel *chan = data;
+	struct tegra_channel_buffer *buf;
+	int err = 0;
+
+	while (1) {
+		/*
+		 * Source is not streaming if error is non-zero.
+		 * So, do not dequeue buffers on error and let the thread sleep
+		 * till kthread stop signal is received.
+		 */
+		wait_event_interruptible(chan->start_wait,
+					 kthread_should_stop() ||
+					 (!list_empty(&chan->capture) &&
+					 !err));
+
+		if (kthread_should_stop())
+			break;
+
+		/* dequeue the buffer and start capture */
+		spin_lock(&chan->start_lock);
+		if (list_empty(&chan->capture)) {
+			spin_unlock(&chan->start_lock);
+			continue;
+		}
+
+		buf = list_first_entry(&chan->capture,
+				       struct tegra_channel_buffer, queue);
+		list_del_init(&buf->queue);
+		spin_unlock(&chan->start_lock);
+
+		err = tegra_channel_capture_frame(chan, buf);
+		if (err)
+			vb2_queue_error(&chan->queue);
+	}
+
+	return 0;
+}
+
+static int chan_capture_kthread_finish(void *data)
+{
+	struct tegra_vi_channel *chan = data;
+	struct tegra_channel_buffer *buf;
+
+	while (1) {
+		wait_event_interruptible(chan->done_wait,
+					 !list_empty(&chan->done) ||
+					 kthread_should_stop());
+
+		/* dequeue buffers and finish capture */
+		buf = dequeue_buf_done(chan);
+		while (buf) {
+			tegra_channel_capture_done(chan, buf);
+			buf = dequeue_buf_done(chan);
+		}
+
+		if (kthread_should_stop())
+			break;
+	}
+
+	return 0;
+}
+
+static int tegra210_vi_start_streaming(struct vb2_queue *vq, u32 count)
+{
+	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
+	struct media_pipeline *pipe = &chan->video.pipe;
+	u32 val;
+	int ret;
+
+	tegra_vi_write(chan, TEGRA_VI_CFG_CG_CTRL, VI_CG_2ND_LEVEL_EN);
+
+	/* clear errors */
+	val = vi_csi_read(chan, TEGRA_VI_CSI_ERROR_STATUS);
+	vi_csi_write(chan, TEGRA_VI_CSI_ERROR_STATUS, val);
+
+	val = tegra_vi_read(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR);
+	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_ERROR, val);
+
+	/*
+	 * Sync point FIFO full stalls the host interface.
+	 * Setting NO_STALL will drop INCR_SYNCPT methods when fifos are
+	 * full and the corresponding condition bits in INCR_SYNCPT_ERROR
+	 * register will be set.
+	 * This allows SW to process error recovery.
+	 */
+	tegra_vi_write(chan, TEGRA_VI_CFG_VI_INCR_SYNCPT_CNTRL,
+		       VI_INCR_SYNCPT_NO_STALL);
+
+	/* start the pipeline */
+	ret = media_pipeline_start(&chan->video.entity, pipe);
+	if (ret < 0)
+		goto error_pipeline_start;
+
+	tegra_channel_capture_setup(chan);
+	ret = tegra_channel_set_stream(chan, true);
+	if (ret < 0)
+		goto error_set_stream;
+
+	chan->sequence = 0;
+
+	/* start kthreads to capture data to buffer and return them */
+	chan->kthread_start_capture = kthread_run(chan_capture_kthread_start,
+						  chan, "%s:0",
+						  chan->video.name);
+	if (IS_ERR(chan->kthread_start_capture)) {
+		ret = PTR_ERR(chan->kthread_start_capture);
+		chan->kthread_start_capture = NULL;
+		dev_err(&chan->video.dev,
+			"failed to run capture start kthread: %d\n", ret);
+		goto error_kthread_start;
+	}
+
+	chan->kthread_finish_capture = kthread_run(chan_capture_kthread_finish,
+						   chan, "%s:1",
+						   chan->video.name);
+	if (IS_ERR(chan->kthread_finish_capture)) {
+		ret = PTR_ERR(chan->kthread_finish_capture);
+		chan->kthread_finish_capture = NULL;
+		dev_err(&chan->video.dev,
+			"failed to run capture finish kthread: %d\n", ret);
+		goto error_kthread_done;
+	}
+
+	return 0;
+
+error_kthread_done:
+	kthread_stop(chan->kthread_start_capture);
+error_kthread_start:
+	tegra_channel_set_stream(chan, false);
+error_set_stream:
+	media_pipeline_stop(&chan->video.entity);
+error_pipeline_start:
+	tegra_channel_release_buffers(chan, VB2_BUF_STATE_QUEUED);
+	return ret;
+}
+
+static void tegra210_vi_stop_streaming(struct vb2_queue *vq)
+{
+	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
+
+	if (chan->kthread_start_capture) {
+		kthread_stop(chan->kthread_start_capture);
+		chan->kthread_start_capture = NULL;
+	}
+
+	if (chan->kthread_finish_capture) {
+		kthread_stop(chan->kthread_finish_capture);
+		chan->kthread_finish_capture = NULL;
+	}
+
+	tegra_channel_release_buffers(chan, VB2_BUF_STATE_ERROR);
+	tegra_channel_set_stream(chan, false);
+	media_pipeline_stop(&chan->video.entity);
+}
+
+/*
+ * Tegra210 VI Pixel memory format enum.
+ * These format enum value gets programmed into corresponding Tegra VI
+ * channel register bits.
+ */
+enum tegra210_image_format {
+	TEGRA210_IMAGE_FORMAT_T_L8 = 16,
+
+	TEGRA210_IMAGE_FORMAT_T_R16_I = 32,
+	TEGRA210_IMAGE_FORMAT_T_B5G6R5,
+	TEGRA210_IMAGE_FORMAT_T_R5G6B5,
+	TEGRA210_IMAGE_FORMAT_T_A1B5G5R5,
+	TEGRA210_IMAGE_FORMAT_T_A1R5G5B5,
+	TEGRA210_IMAGE_FORMAT_T_B5G5R5A1,
+	TEGRA210_IMAGE_FORMAT_T_R5G5B5A1,
+	TEGRA210_IMAGE_FORMAT_T_A4B4G4R4,
+	TEGRA210_IMAGE_FORMAT_T_A4R4G4B4,
+	TEGRA210_IMAGE_FORMAT_T_B4G4R4A4,
+	TEGRA210_IMAGE_FORMAT_T_R4G4B4A4,
+
+	TEGRA210_IMAGE_FORMAT_T_A8B8G8R8 = 64,
+	TEGRA210_IMAGE_FORMAT_T_A8R8G8B8,
+	TEGRA210_IMAGE_FORMAT_T_B8G8R8A8,
+	TEGRA210_IMAGE_FORMAT_T_R8G8B8A8,
+	TEGRA210_IMAGE_FORMAT_T_A2B10G10R10,
+	TEGRA210_IMAGE_FORMAT_T_A2R10G10B10,
+	TEGRA210_IMAGE_FORMAT_T_B10G10R10A2,
+	TEGRA210_IMAGE_FORMAT_T_R10G10B10A2,
+
+	TEGRA210_IMAGE_FORMAT_T_A8Y8U8V8 = 193,
+	TEGRA210_IMAGE_FORMAT_T_V8U8Y8A8,
+
+	TEGRA210_IMAGE_FORMAT_T_A2Y10U10V10 = 197,
+	TEGRA210_IMAGE_FORMAT_T_V10U10Y10A2,
+	TEGRA210_IMAGE_FORMAT_T_Y8_U8__Y8_V8,
+	TEGRA210_IMAGE_FORMAT_T_Y8_V8__Y8_U8,
+	TEGRA210_IMAGE_FORMAT_T_U8_Y8__V8_Y8,
+	TEGRA210_IMAGE_FORMAT_T_V8_Y8__U8_Y8,
+
+	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N444 = 224,
+	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N444,
+	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N444,
+	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N422,
+	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N422,
+	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N422,
+	TEGRA210_IMAGE_FORMAT_T_Y8__U8__V8_N420,
+	TEGRA210_IMAGE_FORMAT_T_Y8__U8V8_N420,
+	TEGRA210_IMAGE_FORMAT_T_Y8__V8U8_N420,
+	TEGRA210_IMAGE_FORMAT_T_X2LC10LB10LA10,
+	TEGRA210_IMAGE_FORMAT_T_A2R6R6R6R6R6,
+};
+
+#define TEGRA210_VIDEO_FMT(DATA_TYPE, BIT_WIDTH, MBUS_CODE, BPP,	\
+			   FORMAT, FOURCC)				\
+{									\
+	TEGRA_IMAGE_DT_##DATA_TYPE,					\
+	BIT_WIDTH,							\
+	MEDIA_BUS_FMT_##MBUS_CODE,					\
+	BPP,								\
+	TEGRA210_IMAGE_FORMAT_##FORMAT,					\
+	V4L2_PIX_FMT_##FOURCC,						\
+}
+
+/* Tegra210 supported video formats */
+static const struct tegra_video_format tegra210_video_formats[] = {
+	/* RAW 8 */
+	TEGRA210_VIDEO_FMT(RAW8, 8, SRGGB8_1X8, 1, T_L8, SRGGB8),
+	TEGRA210_VIDEO_FMT(RAW8, 8, SGRBG8_1X8, 1, T_L8, SGRBG8),
+	TEGRA210_VIDEO_FMT(RAW8, 8, SGBRG8_1X8, 1, T_L8, SGBRG8),
+	TEGRA210_VIDEO_FMT(RAW8, 8, SBGGR8_1X8, 1, T_L8, SBGGR8),
+	/* RAW 10 */
+	TEGRA210_VIDEO_FMT(RAW10, 10, SRGGB10_1X10, 2, T_R16_I, SRGGB10),
+	TEGRA210_VIDEO_FMT(RAW10, 10, SGRBG10_1X10, 2, T_R16_I, SGRBG10),
+	TEGRA210_VIDEO_FMT(RAW10, 10, SGBRG10_1X10, 2, T_R16_I, SGBRG10),
+	TEGRA210_VIDEO_FMT(RAW10, 10, SBGGR10_1X10, 2, T_R16_I, SBGGR10),
+	/* RAW 12 */
+	TEGRA210_VIDEO_FMT(RAW12, 12, SRGGB12_1X12, 2, T_R16_I, SRGGB12),
+	TEGRA210_VIDEO_FMT(RAW12, 12, SGRBG12_1X12, 2, T_R16_I, SGRBG12),
+	TEGRA210_VIDEO_FMT(RAW12, 12, SGBRG12_1X12, 2, T_R16_I, SGBRG12),
+	TEGRA210_VIDEO_FMT(RAW12, 12, SBGGR12_1X12, 2, T_R16_I, SBGGR12),
+	/* RGB888 */
+	TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X24, 4, T_A8R8G8B8, RGB24),
+	TEGRA210_VIDEO_FMT(RGB888, 24, RGB888_1X32_PADHI, 4, T_A8B8G8R8,
+			   XBGR32),
+	/* YUV422 */
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 2, T_U8_Y8__V8_Y8, UYVY),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_1X16, 2, T_V8_Y8__U8_Y8, VYUY),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_1X16, 2, T_Y8_U8__Y8_V8, YUYV),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_1X16, 2, T_Y8_V8__Y8_U8, YVYU),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_1X16, 1, T_Y8__V8U8_N422, NV16),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, UYVY8_2X8, 2, T_U8_Y8__V8_Y8, UYVY),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, VYUY8_2X8, 2, T_V8_Y8__U8_Y8, VYUY),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, YUYV8_2X8, 2, T_Y8_U8__Y8_V8, YUYV),
+	TEGRA210_VIDEO_FMT(YUV422_8, 16, YVYU8_2X8, 2, T_Y8_V8__Y8_U8, YVYU),
+};
+
+/* Tegra210 VI operations */
+static const struct tegra_vi_ops tegra210_vi_ops = {
+	.vi_start_streaming = tegra210_vi_start_streaming,
+	.vi_stop_streaming = tegra210_vi_stop_streaming,
+};
+
+/* Tegra210 VI SoC data */
+const struct tegra_vi_soc tegra210_vi_soc = {
+	.video_formats = tegra210_video_formats,
+	.nformats = ARRAY_SIZE(tegra210_video_formats),
+	.ops = &tegra210_vi_ops,
+	.hw_revision = 3,
+	.vi_max_channels = 6,
+	.vi_max_clk_hz = 499200000,
+};
+
+/* Tegra210 CSI PHY registers accessors */
+static void csi_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
+		      u32 val)
+{
+	void __iomem *csi_pp_base;
+
+	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
+
+	writel_relaxed(val, csi_pp_base + addr);
+}
+
+/* Tegra210 CSI Pixel parser registers accessors */
+static void pp_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
+{
+	void __iomem *csi_pp_base;
+	unsigned int offset;
+
+	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
+	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
+
+	writel_relaxed(val, csi_pp_base + offset + addr);
+}
+
+static u32 pp_read(struct tegra_csi *csi, u8 portno, u32 addr)
+{
+	void __iomem *csi_pp_base;
+	unsigned int offset;
+
+	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
+	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
+
+	return readl_relaxed(csi_pp_base + offset + addr);
+}
+
+/* Tegra210 CSI CIL A/B port registers accessors */
+static void cil_write(struct tegra_csi *csi, u8 portno, u32 addr, u32 val)
+{
+	void __iomem *csi_cil_base;
+	unsigned int offset;
+
+	csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
+		       TEGRA210_CSI_CIL_OFFSET;
+	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
+
+	writel_relaxed(val, csi_cil_base + offset + addr);
+}
+
+static u32 cil_read(struct tegra_csi *csi, u8 portno, u32 addr)
+{
+	void __iomem *csi_cil_base;
+	unsigned int offset;
+
+	csi_cil_base = csi->iomem + CSI_PP_OFFSET(portno >> 1) +
+		       TEGRA210_CSI_CIL_OFFSET;
+	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET;
+
+	return readl_relaxed(csi_cil_base + offset + addr);
+}
+
+/* Tegra210 CSI Test pattern generator registers accessor */
+static void tpg_write(struct tegra_csi *csi, u8 portno, unsigned int addr,
+		      u32 val)
+{
+	void __iomem *csi_pp_base;
+	unsigned int offset;
+
+	csi_pp_base = csi->iomem + CSI_PP_OFFSET(portno >> 1);
+	offset = (portno % CSI_PORTS_PER_BRICK) * TEGRA210_CSI_PORT_OFFSET +
+		 TEGRA210_CSI_TPG_OFFSET;
+
+	writel_relaxed(val, csi_pp_base + offset + addr);
+}
+
+/*
+ * Tegra210 CSI operations
+ */
+static void tegra210_csi_error_recover(struct tegra_csi_channel *csi_chan)
+{
+	struct tegra_csi *csi = csi_chan->csi;
+	unsigned int portno = csi_chan->csi_port_num;
+	u32 val;
+
+	/*
+	 * Recover CSI hardware in case of capture errors by issuing
+	 * software reset to CSICIL sensor, pixel parser, and clear errors
+	 * to have clean capture on  next streaming.
+	 */
+	val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
+	dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
+
+	val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
+	dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
+
+	val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
+	dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
+
+	if (csi_chan->numlanes == 4) {
+		/* reset CSI CIL sensor */
+		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
+		cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
+		/*
+		 * SW_STATUS_RESET resets all status bits of PPA, PPB, CILA,
+		 * CILB status registers and debug counters.
+		 * So, SW_STATUS_RESET can be used only when CSI brick is in
+		 * x4 mode.
+		 */
+		csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x1);
+
+		/* sleep for 20 clock cycles to drain the FIFO */
+		usleep_range(10, 20);
+
+		cil_write(csi, portno + 1, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
+		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
+		csi_write(csi, portno, TEGRA_CSI_CSI_SW_STATUS_RESET, 0x0);
+	} else {
+		/* reset CSICIL sensor */
+		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x1);
+		usleep_range(10, 20);
+		cil_write(csi, portno, TEGRA_CSI_CIL_SW_SENSOR_RESET, 0x0);
+
+		/* clear the errors */
+		pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS,
+			 0xffffffff);
+		cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
+		cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
+	}
+}
+
+static int tegra210_csi_start_streaming(struct tegra_csi_channel *csi_chan)
+{
+	struct tegra_csi *csi = csi_chan->csi;
+	unsigned int portno = csi_chan->csi_port_num;
+	u32 val;
+
+	csi_write(csi, portno, TEGRA_CSI_CLKEN_OVERRIDE, 0);
+
+	/* clean up status */
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, 0xffffffff);
+	cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, 0xffffffff);
+	cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, 0xffffffff);
+	cil_write(csi, portno, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
+
+	/* CIL PHY registers setup */
+	cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
+	cil_write(csi, portno, TEGRA_CSI_CIL_PHY_CONTROL, 0xa);
+
+	/*
+	 * The CSI unit provides for connection of up to six cameras in
+	 * the system and is organized as three identical instances of
+	 * two MIPI support blocks, each with a separate 4-lane
+	 * interface that can be configured as a single camera with 4
+	 * lanes or as a dual camera with 2 lanes available for each
+	 * camera.
+	 */
+	if (csi_chan->numlanes == 4) {
+		cil_write(csi, portno + 1, TEGRA_CSI_CIL_STATUS, 0xffffffff);
+		cil_write(csi, portno + 1, TEGRA_CSI_CILX_STATUS, 0xffffffff);
+		cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
+
+		cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0,
+			  BRICK_CLOCK_A_4X);
+		cil_write(csi, portno + 1, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
+		cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
+		cil_write(csi, portno + 1, TEGRA_CSI_CIL_PHY_CONTROL, 0xa);
+		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
+			  CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_ENABLE);
+	} else {
+		val = ((portno & 1) == PORT_A) ?
+		      CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_NOP :
+		      CSI_B_PHY_CIL_ENABLE | CSI_A_PHY_CIL_NOP;
+		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
+	}
+
+	/* CSI pixel parser registers setup */
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
+		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
+		 CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_RST);
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_INTERRUPT_MASK, 0x0);
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL0,
+		 CSI_PP_PACKET_HEADER_SENT |
+		 CSI_PP_DATA_IDENTIFIER_ENABLE |
+		 CSI_PP_WORD_COUNT_SELECT_HEADER |
+		 CSI_PP_CRC_CHECK_ENABLE |  CSI_PP_WC_CHECK |
+		 CSI_PP_OUTPUT_FORMAT_STORE | CSI_PPA_PAD_LINE_NOPAD |
+		 CSI_PP_HEADER_EC_DISABLE | CSI_PPA_PAD_FRAME_NOPAD |
+		 (portno & 1));
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_CONTROL1,
+		 (0x1 << CSI_PP_TOP_FIELD_FRAME_OFFSET) |
+		 (0x1 << CSI_PP_TOP_FIELD_FRAME_MASK_OFFSET));
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_GAP,
+		 0x14 << PP_FRAME_MIN_GAP_OFFSET);
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_EXPECTED_FRAME, 0x0);
+	pp_write(csi, portno, TEGRA_CSI_INPUT_STREAM_CONTROL,
+		 (0x3f << CSI_SKIP_PACKET_THRESHOLD_OFFSET) |
+		 (csi_chan->numlanes - 1));
+
+	/* TPG setup */
+	if (csi_chan->pg_mode) {
+		tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
+			  ((csi_chan->pg_mode - 1) << PG_MODE_OFFSET) |
+			  PG_ENABLE);
+		tpg_write(csi, portno, TEGRA_CSI_PG_BLANK,
+			  csi_chan->v_blank << PG_VBLANK_OFFSET |
+			  csi_chan->h_blank);
+		tpg_write(csi, portno, TEGRA_CSI_PG_PHASE, 0x0);
+		tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ,
+			  (0x10 << PG_RED_VERT_INIT_FREQ_OFFSET) |
+			  (0x10 << PG_RED_HOR_INIT_FREQ_OFFSET));
+		tpg_write(csi, portno, TEGRA_CSI_PG_RED_FREQ_RATE, 0x0);
+		tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ,
+			  (0x10 << PG_GREEN_VERT_INIT_FREQ_OFFSET) |
+			  (0x10 << PG_GREEN_HOR_INIT_FREQ_OFFSET));
+		tpg_write(csi, portno, TEGRA_CSI_PG_GREEN_FREQ_RATE, 0x0);
+		tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ,
+			  (0x10 << PG_BLUE_VERT_INIT_FREQ_OFFSET) |
+			  (0x10 << PG_BLUE_HOR_INIT_FREQ_OFFSET));
+		tpg_write(csi, portno, TEGRA_CSI_PG_BLUE_FREQ_RATE, 0x0);
+	}
+
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
+		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
+		 CSI_PP_SINGLE_SHOT_ENABLE | CSI_PP_ENABLE);
+
+	return 0;
+}
+
+static void tegra210_csi_stop_streaming(struct tegra_csi_channel *csi_chan)
+{
+	struct tegra_csi *csi = csi_chan->csi;
+	unsigned int portno = csi_chan->csi_port_num;
+	u32 val;
+
+	val = pp_read(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS);
+
+	dev_dbg(csi->dev, "TEGRA_CSI_PIXEL_PARSER_STATUS 0x%08x\n", val);
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_PARSER_STATUS, val);
+
+	val = cil_read(csi, portno, TEGRA_CSI_CIL_STATUS);
+	dev_dbg(csi->dev, "TEGRA_CSI_CIL_STATUS 0x%08x\n", val);
+	cil_write(csi, portno, TEGRA_CSI_CIL_STATUS, val);
+
+	val = cil_read(csi, portno, TEGRA_CSI_CILX_STATUS);
+	dev_dbg(csi->dev, "TEGRA_CSI_CILX_STATUS 0x%08x\n", val);
+	cil_write(csi, portno, TEGRA_CSI_CILX_STATUS, val);
+
+	pp_write(csi, portno, TEGRA_CSI_PIXEL_STREAM_PP_COMMAND,
+		 (0xf << CSI_PP_START_MARKER_FRAME_MAX_OFFSET) |
+		 CSI_PP_DISABLE);
+
+	if (csi_chan->pg_mode) {
+		tpg_write(csi, portno, TEGRA_CSI_PATTERN_GENERATOR_CTRL,
+			  PG_DISABLE);
+		return;
+	}
+
+	if (csi_chan->numlanes == 4) {
+		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
+			  CSI_A_PHY_CIL_DISABLE |
+			  CSI_B_PHY_CIL_DISABLE);
+	} else {
+		val = ((portno & 1) == PORT_A) ?
+		      CSI_A_PHY_CIL_DISABLE | CSI_B_PHY_CIL_NOP :
+		      CSI_B_PHY_CIL_DISABLE | CSI_A_PHY_CIL_NOP;
+		csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND, val);
+	}
+}
+
+/*
+ * Tegra210 CSI TPG frame rate table with horizontal and vertical
+ * blanking intervals for corresponding format and resolution.
+ * Blanking intervals are tuned values from design team for max TPG
+ * clock rate.
+ */
+static const struct tpg_framerate tegra210_tpg_frmrate_table[] = {
+	{
+		.frmsize = { 1280, 720 },
+		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.framerate = 120,
+		.h_blank = 512,
+		.v_blank = 8,
+	},
+	{
+		.frmsize = { 1920, 1080 },
+		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.framerate = 60,
+		.h_blank = 512,
+		.v_blank = 8,
+	},
+	{
+		.frmsize = { 3840, 2160 },
+		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+		.framerate = 20,
+		.h_blank = 8,
+		.v_blank = 8,
+	},
+	{
+		.frmsize = { 1280, 720 },
+		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
+		.framerate = 60,
+		.h_blank = 512,
+		.v_blank = 8,
+	},
+	{
+		.frmsize = { 1920, 1080 },
+		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
+		.framerate = 30,
+		.h_blank = 512,
+		.v_blank = 8,
+	},
+	{
+		.frmsize = { 3840, 2160 },
+		.code = MEDIA_BUS_FMT_RGB888_1X32_PADHI,
+		.framerate = 8,
+		.h_blank = 8,
+		.v_blank = 8,
+	},
+};
+
+static const char * const tegra210_csi_cil_clks[] = {
+	"csi",
+	"cilab",
+	"cilcd",
+	"cile",
+	"csi_tpg",
+};
+
+/* Tegra210 CSI operations */
+static const struct tegra_csi_ops tegra210_csi_ops = {
+	.csi_start_streaming = tegra210_csi_start_streaming,
+	.csi_stop_streaming = tegra210_csi_stop_streaming,
+	.csi_err_recover = tegra210_csi_error_recover,
+};
+
+/* Tegra210 CSI SoC data */
+const struct tegra_csi_soc tegra210_csi_soc = {
+	.ops = &tegra210_csi_ops,
+	.csi_max_channels = 6,
+	.clk_names = tegra210_csi_cil_clks,
+	.num_clks = ARRAY_SIZE(tegra210_csi_cil_clks),
+	.tpg_frmrate_table = tegra210_tpg_frmrate_table,
+	.tpg_frmrate_table_size = ARRAY_SIZE(tegra210_tpg_frmrate_table),
+};
diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c
new file mode 100644
index 000000000000..1b5e660155f5
--- /dev/null
+++ b/drivers/staging/media/tegra-video/vi.c
@@ -0,0 +1,1074 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#include <linux/bitmap.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/host1x.h>
+#include <linux/lcm.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+
+#include <media/v4l2-event.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include <soc/tegra/pmc.h>
+
+#include "vi.h"
+#include "video.h"
+
+#define SURFACE_ALIGN_BYTES		64
+#define MAX_CID_CONTROLS		1
+
+static const struct tegra_video_format tegra_default_format = {
+	.img_dt = TEGRA_IMAGE_DT_RAW10,
+	.bit_width = 10,
+	.code = MEDIA_BUS_FMT_SRGGB10_1X10,
+	.bpp = 2,
+	.img_fmt = TEGRA_IMAGE_FORMAT_DEF,
+	.fourcc = V4L2_PIX_FMT_SRGGB10,
+};
+
+static inline struct tegra_vi *
+host1x_client_to_vi(struct host1x_client *client)
+{
+	return container_of(client, struct tegra_vi, client);
+}
+
+static inline struct tegra_channel_buffer *
+to_tegra_channel_buffer(struct vb2_v4l2_buffer *vb)
+{
+	return container_of(vb, struct tegra_channel_buffer, buf);
+}
+
+static int tegra_get_format_idx_by_code(struct tegra_vi *vi,
+					unsigned int code)
+{
+	unsigned int i;
+
+	for (i = 0; i < vi->soc->nformats; ++i) {
+		if (vi->soc->video_formats[i].code == code)
+			return i;
+	}
+
+	return -1;
+}
+
+static u32 tegra_get_format_fourcc_by_idx(struct tegra_vi *vi,
+					  unsigned int index)
+{
+	if (index >= vi->soc->nformats)
+		return -EINVAL;
+
+	return vi->soc->video_formats[index].fourcc;
+}
+
+static const struct tegra_video_format *
+tegra_get_format_by_fourcc(struct tegra_vi *vi, u32 fourcc)
+{
+	unsigned int i;
+
+	for (i = 0; i < vi->soc->nformats; ++i) {
+		if (vi->soc->video_formats[i].fourcc == fourcc)
+			return &vi->soc->video_formats[i];
+	}
+
+	return NULL;
+}
+
+/*
+ * videobuf2 queue operations
+ */
+static int tegra_channel_queue_setup(struct vb2_queue *vq,
+				     unsigned int *nbuffers,
+				     unsigned int *nplanes,
+				     unsigned int sizes[],
+				     struct device *alloc_devs[])
+{
+	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
+
+	if (*nplanes)
+		return sizes[0] < chan->format.sizeimage ? -EINVAL : 0;
+
+	*nplanes = 1;
+	sizes[0] = chan->format.sizeimage;
+	alloc_devs[0] = chan->vi->dev;
+
+	return 0;
+}
+
+static int tegra_channel_buffer_prepare(struct vb2_buffer *vb)
+{
+	struct tegra_vi_channel *chan = vb2_get_drv_priv(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct tegra_channel_buffer *buf = to_tegra_channel_buffer(vbuf);
+	unsigned long size = chan->format.sizeimage;
+
+	if (vb2_plane_size(vb, 0) < size) {
+		v4l2_err(chan->video.v4l2_dev,
+			 "buffer too small (%lu < %lu)\n",
+			 vb2_plane_size(vb, 0), size);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(vb, 0, size);
+	buf->chan = chan;
+	buf->addr = vb2_dma_contig_plane_dma_addr(vb, 0);
+
+	return 0;
+}
+
+static void tegra_channel_buffer_queue(struct vb2_buffer *vb)
+{
+	struct tegra_vi_channel *chan = vb2_get_drv_priv(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct tegra_channel_buffer *buf = to_tegra_channel_buffer(vbuf);
+
+	/* put buffer into the capture queue */
+	spin_lock(&chan->start_lock);
+	list_add_tail(&buf->queue, &chan->capture);
+	spin_unlock(&chan->start_lock);
+
+	/* wait up kthread for capture */
+	wake_up_interruptible(&chan->start_wait);
+}
+
+struct v4l2_subdev *
+tegra_channel_get_remote_subdev(struct tegra_vi_channel *chan)
+{
+	struct media_pad *pad;
+	struct v4l2_subdev *subdev;
+	struct media_entity *entity;
+
+	pad = media_entity_remote_pad(&chan->pad);
+	entity = pad->entity;
+	subdev = media_entity_to_v4l2_subdev(entity);
+
+	return subdev;
+}
+
+int tegra_channel_set_stream(struct tegra_vi_channel *chan, bool on)
+{
+	struct v4l2_subdev *subdev;
+	int ret;
+
+	/* stream CSI */
+	subdev = tegra_channel_get_remote_subdev(chan);
+	ret = v4l2_subdev_call(subdev, video, s_stream, on);
+	if (on && ret < 0 && ret != -ENOIOCTLCMD)
+		return ret;
+
+	return 0;
+}
+
+void tegra_channel_release_buffers(struct tegra_vi_channel *chan,
+				   enum vb2_buffer_state state)
+{
+	struct tegra_channel_buffer *buf, *nbuf;
+
+	spin_lock(&chan->start_lock);
+	list_for_each_entry_safe(buf, nbuf, &chan->capture, queue) {
+		vb2_buffer_done(&buf->buf.vb2_buf, state);
+		list_del(&buf->queue);
+	}
+	spin_unlock(&chan->start_lock);
+
+	spin_lock(&chan->done_lock);
+	list_for_each_entry_safe(buf, nbuf, &chan->done, queue) {
+		vb2_buffer_done(&buf->buf.vb2_buf, state);
+		list_del(&buf->queue);
+	}
+	spin_unlock(&chan->done_lock);
+}
+
+static int tegra_channel_start_streaming(struct vb2_queue *vq, u32 count)
+{
+	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
+	int ret;
+
+	ret = pm_runtime_get_sync(chan->vi->dev);
+	if (ret < 0) {
+		dev_err(chan->vi->dev, "failed to get runtime PM: %d\n", ret);
+		pm_runtime_put_noidle(chan->vi->dev);
+		return ret;
+	}
+
+	ret = chan->vi->ops->vi_start_streaming(vq, count);
+	if (ret < 0)
+		pm_runtime_put(chan->vi->dev);
+
+	return ret;
+}
+
+static void tegra_channel_stop_streaming(struct vb2_queue *vq)
+{
+	struct tegra_vi_channel *chan = vb2_get_drv_priv(vq);
+
+	chan->vi->ops->vi_stop_streaming(vq);
+	pm_runtime_put(chan->vi->dev);
+}
+
+static const struct vb2_ops tegra_channel_queue_qops = {
+	.queue_setup = tegra_channel_queue_setup,
+	.buf_prepare = tegra_channel_buffer_prepare,
+	.buf_queue = tegra_channel_buffer_queue,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
+	.start_streaming = tegra_channel_start_streaming,
+	.stop_streaming = tegra_channel_stop_streaming,
+};
+
+/*
+ * V4L2 ioctl operations
+ */
+static int tegra_channel_querycap(struct file *file, void *fh,
+				  struct v4l2_capability *cap)
+{
+	struct tegra_vi_channel *chan = video_drvdata(file);
+
+	strscpy(cap->driver, "tegra-video", sizeof(cap->driver));
+	strscpy(cap->card, chan->video.name, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 dev_name(chan->vi->dev));
+
+	return 0;
+}
+
+static int tegra_channel_g_parm(struct file *file, void *fh,
+				struct v4l2_streamparm *a)
+{
+	struct tegra_vi_channel *chan = video_drvdata(file);
+	struct v4l2_subdev *subdev;
+
+	subdev = tegra_channel_get_remote_subdev(chan);
+	return v4l2_g_parm_cap(&chan->video, subdev, a);
+}
+
+static int tegra_channel_s_parm(struct file *file, void *fh,
+				struct v4l2_streamparm *a)
+{
+	struct tegra_vi_channel *chan = video_drvdata(file);
+	struct v4l2_subdev *subdev;
+
+	subdev = tegra_channel_get_remote_subdev(chan);
+	return v4l2_s_parm_cap(&chan->video, subdev, a);
+}
+
+static int tegra_channel_enum_framesizes(struct file *file, void *fh,
+					 struct v4l2_frmsizeenum *sizes)
+{
+	int ret;
+	struct tegra_vi_channel *chan = video_drvdata(file);
+	struct v4l2_subdev *subdev;
+	const struct tegra_video_format *fmtinfo;
+	struct v4l2_subdev_frame_size_enum fse = {
+		.index = sizes->index,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+
+	fmtinfo = tegra_get_format_by_fourcc(chan->vi, sizes->pixel_format);
+	if (!fmtinfo)
+		return -EINVAL;
+
+	fse.code = fmtinfo->code;
+
+	subdev = tegra_channel_get_remote_subdev(chan);
+	ret = v4l2_subdev_call(subdev, pad, enum_frame_size, NULL, &fse);
+	if (ret)
+		return ret;
+
+	sizes->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+	sizes->discrete.width = fse.max_width;
+	sizes->discrete.height = fse.max_height;
+
+	return 0;
+}
+
+static int tegra_channel_enum_frameintervals(struct file *file, void *fh,
+					     struct v4l2_frmivalenum *ivals)
+{
+	int ret;
+	struct tegra_vi_channel *chan = video_drvdata(file);
+	struct v4l2_subdev *subdev;
+	const struct tegra_video_format *fmtinfo;
+	struct v4l2_subdev_frame_interval_enum fie = {
+		.index = ivals->index,
+		.width = ivals->width,
+		.height = ivals->height,
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+
+	fmtinfo = tegra_get_format_by_fourcc(chan->vi, ivals->pixel_format);
+	if (!fmtinfo)
+		return -EINVAL;
+
+	fie.code = fmtinfo->code;
+
+	subdev = tegra_channel_get_remote_subdev(chan);
+	ret = v4l2_subdev_call(subdev, pad, enum_frame_interval, NULL, &fie);
+	if (ret)
+		return ret;
+
+	ivals->type = V4L2_FRMIVAL_TYPE_DISCRETE;
+	ivals->discrete.numerator = fie.interval.numerator;
+	ivals->discrete.denominator = fie.interval.denominator;
+
+	return 0;
+}
+
+static int tegra_channel_enum_format(struct file *file, void *fh,
+				     struct v4l2_fmtdesc *f)
+{
+	struct tegra_vi_channel *chan = video_drvdata(file);
+	unsigned int index = 0, i;
+	unsigned long *fmts_bitmap = chan->tpg_fmts_bitmap;
+
+	if (f->index >= bitmap_weight(fmts_bitmap, MAX_FORMAT_NUM))
+		return -EINVAL;
+
+	for (i = 0; i < f->index + 1; i++, index++)
+		index = find_next_bit(fmts_bitmap, MAX_FORMAT_NUM, index);
+
+	f->pixelformat = tegra_get_format_fourcc_by_idx(chan->vi, index - 1);
+
+	return 0;
+}
+
+static int tegra_channel_get_format(struct file *file, void *fh,
+				    struct v4l2_format *format)
+{
+	struct tegra_vi_channel *chan = video_drvdata(file);
+
+	format->fmt.pix = chan->format;
+
+	return 0;
+}
+
+static void tegra_channel_fmt_align(struct tegra_vi_channel *chan,
+				    struct v4l2_pix_format *pix,
+				    unsigned int bpp)
+{
+	unsigned int align;
+	unsigned int min_width;
+	unsigned int max_width;
+	unsigned int width;
+	unsigned int min_bpl;
+	unsigned int max_bpl;
+	unsigned int bpl;
+
+	/*
+	 * The transfer alignment requirements are expressed in bytes. Compute
+	 * minimum and maximum values, clamp the requested width and convert
+	 * it back to pixels. Use bytesperline to adjust the width.
+	 */
+	align = lcm(SURFACE_ALIGN_BYTES, bpp);
+	min_width = roundup(TEGRA_MIN_WIDTH, align);
+	max_width = rounddown(TEGRA_MAX_WIDTH, align);
+	width = roundup(pix->width * bpp, align);
+
+	pix->width = clamp(width, min_width, max_width) / bpp;
+	pix->height = clamp(pix->height, TEGRA_MIN_HEIGHT, TEGRA_MAX_HEIGHT);
+
+	/* Clamp the requested bytes per line value. If the maximum bytes per
+	 * line value is zero, the module doesn't support user configurable
+	 * line sizes. Override the requested value with the minimum in that
+	 * case.
+	 */
+	min_bpl = pix->width * bpp;
+	max_bpl = rounddown(TEGRA_MAX_WIDTH, SURFACE_ALIGN_BYTES);
+	bpl = roundup(pix->bytesperline, SURFACE_ALIGN_BYTES);
+
+	pix->bytesperline = clamp(bpl, min_bpl, max_bpl);
+	pix->sizeimage = pix->bytesperline * pix->height;
+}
+
+static int __tegra_channel_try_format(struct tegra_vi_channel *chan,
+				      struct v4l2_pix_format *pix)
+{
+	const struct tegra_video_format *fmtinfo;
+	struct v4l2_subdev *subdev;
+	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev_pad_config *pad_cfg;
+
+	subdev = tegra_channel_get_remote_subdev(chan);
+	pad_cfg = v4l2_subdev_alloc_pad_config(subdev);
+	if (!pad_cfg)
+		return -ENOMEM;
+	/*
+	 * Retrieve the format information and if requested format isn't
+	 * supported, keep the current format.
+	 */
+	fmtinfo = tegra_get_format_by_fourcc(chan->vi, pix->pixelformat);
+	if (!fmtinfo) {
+		pix->pixelformat = chan->format.pixelformat;
+		pix->colorspace = chan->format.colorspace;
+		fmtinfo = tegra_get_format_by_fourcc(chan->vi,
+						     pix->pixelformat);
+	}
+
+	pix->field = V4L2_FIELD_NONE;
+	fmt.which = V4L2_SUBDEV_FORMAT_TRY;
+	fmt.pad = 0;
+	v4l2_fill_mbus_format(&fmt.format, pix, fmtinfo->code);
+	v4l2_subdev_call(subdev, pad, set_fmt, pad_cfg, &fmt);
+	v4l2_fill_pix_format(pix, &fmt.format);
+	tegra_channel_fmt_align(chan, pix, fmtinfo->bpp);
+
+	v4l2_subdev_free_pad_config(pad_cfg);
+
+	return 0;
+}
+
+static int tegra_channel_try_format(struct file *file, void *fh,
+				    struct v4l2_format *format)
+{
+	struct tegra_vi_channel *chan = video_drvdata(file);
+
+	return __tegra_channel_try_format(chan, &format->fmt.pix);
+}
+
+static int tegra_channel_set_format(struct file *file, void *fh,
+				    struct v4l2_format *format)
+{
+	struct tegra_vi_channel *chan = video_drvdata(file);
+	const struct tegra_video_format *fmtinfo;
+	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev *subdev;
+	struct v4l2_pix_format *pix = &format->fmt.pix;
+	int ret;
+
+	if (vb2_is_busy(&chan->queue))
+		return -EBUSY;
+
+	/* get supported format by try_fmt */
+	ret = __tegra_channel_try_format(chan, pix);
+	if (ret)
+		return ret;
+
+	fmtinfo = tegra_get_format_by_fourcc(chan->vi, pix->pixelformat);
+
+	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	fmt.pad = 0;
+	v4l2_fill_mbus_format(&fmt.format, pix, fmtinfo->code);
+	subdev = tegra_channel_get_remote_subdev(chan);
+	v4l2_subdev_call(subdev, pad, set_fmt, NULL, &fmt);
+	v4l2_fill_pix_format(pix, &fmt.format);
+	tegra_channel_fmt_align(chan, pix, fmtinfo->bpp);
+
+	chan->format = *pix;
+	chan->fmtinfo = fmtinfo;
+
+	return 0;
+}
+
+static int tegra_channel_enum_input(struct file *file, void *fh,
+				    struct v4l2_input *inp)
+{
+	/* currently driver supports internal TPG only */
+	if (inp->index)
+		return -EINVAL;
+
+	inp->type = V4L2_INPUT_TYPE_CAMERA;
+	strscpy(inp->name, "Tegra TPG", sizeof(inp->name));
+
+	return 0;
+}
+
+static int tegra_channel_g_input(struct file *file, void *priv,
+				 unsigned int *i)
+{
+	*i = 0;
+
+	return 0;
+}
+
+static int tegra_channel_s_input(struct file *file, void *priv,
+				 unsigned int input)
+{
+	if (input > 0)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops tegra_channel_ioctl_ops = {
+	.vidioc_querycap		= tegra_channel_querycap,
+	.vidioc_g_parm			= tegra_channel_g_parm,
+	.vidioc_s_parm			= tegra_channel_s_parm,
+	.vidioc_enum_framesizes		= tegra_channel_enum_framesizes,
+	.vidioc_enum_frameintervals	= tegra_channel_enum_frameintervals,
+	.vidioc_enum_fmt_vid_cap	= tegra_channel_enum_format,
+	.vidioc_g_fmt_vid_cap		= tegra_channel_get_format,
+	.vidioc_s_fmt_vid_cap		= tegra_channel_set_format,
+	.vidioc_try_fmt_vid_cap		= tegra_channel_try_format,
+	.vidioc_enum_input		= tegra_channel_enum_input,
+	.vidioc_g_input			= tegra_channel_g_input,
+	.vidioc_s_input			= tegra_channel_s_input,
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_expbuf			= vb2_ioctl_expbuf,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
+};
+
+/*
+ * V4L2 file operations
+ */
+static const struct v4l2_file_operations tegra_channel_fops = {
+	.owner		= THIS_MODULE,
+	.unlocked_ioctl	= video_ioctl2,
+	.open		= v4l2_fh_open,
+	.release	= vb2_fop_release,
+	.read		= vb2_fop_read,
+	.poll		= vb2_fop_poll,
+	.mmap		= vb2_fop_mmap,
+};
+
+/*
+ * V4L2 control operations
+ */
+static int vi_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct tegra_vi_channel *chan = container_of(ctrl->handler,
+						     struct tegra_vi_channel,
+						     ctrl_handler);
+
+	switch (ctrl->id) {
+	case V4L2_CID_TEST_PATTERN:
+		/* pattern change takes effect on next stream */
+		chan->pg_mode = ctrl->val + 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops vi_ctrl_ops = {
+	.s_ctrl	= vi_s_ctrl,
+};
+
+static const char *const vi_pattern_strings[] = {
+	"Black/White Direct Mode",
+	"Color Patch Mode",
+};
+
+static int tegra_channel_setup_ctrl_handler(struct tegra_vi_channel *chan)
+{
+	int ret;
+
+	/* add test pattern control handler to v4l2 device */
+	v4l2_ctrl_new_std_menu_items(&chan->ctrl_handler, &vi_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(vi_pattern_strings) - 1,
+				     0, 0, vi_pattern_strings);
+	if (chan->ctrl_handler.error) {
+		dev_err(chan->vi->dev, "failed to add TPG ctrl handler: %d\n",
+			chan->ctrl_handler.error);
+		v4l2_ctrl_handler_free(&chan->ctrl_handler);
+		return chan->ctrl_handler.error;
+	}
+
+	/* setup the controls */
+	ret = v4l2_ctrl_handler_setup(&chan->ctrl_handler);
+	if (ret < 0) {
+		dev_err(chan->vi->dev,
+			"failed to setup v4l2 ctrl handler: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/* VI only support 2 formats in TPG mode */
+static void vi_tpg_fmts_bitmap_init(struct tegra_vi_channel *chan)
+{
+	int index;
+
+	bitmap_zero(chan->tpg_fmts_bitmap, MAX_FORMAT_NUM);
+
+	index = tegra_get_format_idx_by_code(chan->vi,
+					     MEDIA_BUS_FMT_SRGGB10_1X10);
+	bitmap_set(chan->tpg_fmts_bitmap, index, 1);
+
+	index = tegra_get_format_idx_by_code(chan->vi,
+					     MEDIA_BUS_FMT_RGB888_1X32_PADHI);
+	bitmap_set(chan->tpg_fmts_bitmap, index, 1);
+}
+
+static void tegra_channel_cleanup(struct tegra_vi_channel *chan)
+{
+	v4l2_ctrl_handler_free(&chan->ctrl_handler);
+	media_entity_cleanup(&chan->video.entity);
+	host1x_syncpt_free(chan->mw_ack_sp);
+	host1x_syncpt_free(chan->frame_start_sp);
+	mutex_destroy(&chan->video_lock);
+}
+
+void tegra_channels_cleanup(struct tegra_vi *vi)
+{
+	struct tegra_vi_channel *chan, *tmp;
+
+	if (!vi)
+		return;
+
+	list_for_each_entry_safe(chan, tmp, &vi->vi_chans, list) {
+		tegra_channel_cleanup(chan);
+		list_del(&chan->list);
+		kfree(chan);
+	}
+}
+
+static int tegra_channel_init(struct tegra_vi_channel *chan)
+{
+	struct tegra_vi *vi = chan->vi;
+	struct tegra_video_device *vid = dev_get_drvdata(vi->client.host);
+	unsigned long flags = HOST1X_SYNCPT_CLIENT_MANAGED;
+	int ret;
+
+	mutex_init(&chan->video_lock);
+	INIT_LIST_HEAD(&chan->capture);
+	INIT_LIST_HEAD(&chan->done);
+	spin_lock_init(&chan->start_lock);
+	spin_lock_init(&chan->done_lock);
+	spin_lock_init(&chan->sp_incr_lock);
+	init_waitqueue_head(&chan->start_wait);
+	init_waitqueue_head(&chan->done_wait);
+
+	/* initialize the video format */
+	chan->fmtinfo = &tegra_default_format;
+	chan->format.pixelformat = chan->fmtinfo->fourcc;
+	chan->format.colorspace = V4L2_COLORSPACE_SRGB;
+	chan->format.field = V4L2_FIELD_NONE;
+	chan->format.width = TEGRA_DEF_WIDTH;
+	chan->format.height = TEGRA_DEF_HEIGHT;
+	chan->format.bytesperline = TEGRA_DEF_WIDTH * chan->fmtinfo->bpp;
+	chan->format.sizeimage = chan->format.bytesperline * TEGRA_DEF_HEIGHT;
+	tegra_channel_fmt_align(chan, &chan->format, chan->fmtinfo->bpp);
+
+	chan->frame_start_sp = host1x_syncpt_request(&vi->client, flags);
+	if (!chan->frame_start_sp) {
+		dev_err(vi->dev, "failed to request frame start syncpoint\n");
+		return -ENOMEM;
+	}
+
+	chan->mw_ack_sp = host1x_syncpt_request(&vi->client, flags);
+	if (!chan->mw_ack_sp) {
+		dev_err(vi->dev, "failed to request memory ack syncpoint\n");
+		ret = -ENOMEM;
+		goto free_fs_syncpt;
+	}
+
+	/* initialize the media entity */
+	chan->pad.flags = MEDIA_PAD_FL_SINK;
+	ret = media_entity_pads_init(&chan->video.entity, 1, &chan->pad);
+	if (ret < 0) {
+		dev_err(vi->dev,
+			"failed to initialize media entity: %d\n", ret);
+		goto free_mw_ack_syncpt;
+	}
+
+	ret = v4l2_ctrl_handler_init(&chan->ctrl_handler, MAX_CID_CONTROLS);
+	if (chan->ctrl_handler.error) {
+		dev_err(vi->dev,
+			"failed to initialize v4l2 ctrl handler: %d\n", ret);
+		goto cleanup_media;
+	}
+
+	/* initialize the video_device */
+	chan->video.fops = &tegra_channel_fops;
+	chan->video.v4l2_dev = &vid->v4l2_dev;
+	chan->video.release = video_device_release_empty;
+	chan->video.queue = &chan->queue;
+	snprintf(chan->video.name, sizeof(chan->video.name), "%s-%s-%u",
+		 dev_name(vi->dev), "output", chan->portno);
+	chan->video.vfl_type = VFL_TYPE_VIDEO;
+	chan->video.vfl_dir = VFL_DIR_RX;
+	chan->video.ioctl_ops = &tegra_channel_ioctl_ops;
+	chan->video.ctrl_handler = &chan->ctrl_handler;
+	chan->video.lock = &chan->video_lock;
+	chan->video.device_caps = V4L2_CAP_VIDEO_CAPTURE |
+				  V4L2_CAP_STREAMING |
+				  V4L2_CAP_READWRITE;
+	video_set_drvdata(&chan->video, chan);
+
+	chan->queue.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	chan->queue.io_modes = VB2_MMAP | VB2_DMABUF | VB2_READ;
+	chan->queue.lock = &chan->video_lock;
+	chan->queue.drv_priv = chan;
+	chan->queue.buf_struct_size = sizeof(struct tegra_channel_buffer);
+	chan->queue.ops = &tegra_channel_queue_qops;
+	chan->queue.mem_ops = &vb2_dma_contig_memops;
+	chan->queue.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	chan->queue.min_buffers_needed = 2;
+	chan->queue.dev = vi->dev;
+	ret = vb2_queue_init(&chan->queue);
+	if (ret < 0) {
+		dev_err(vi->dev, "failed to initialize vb2 queue: %d\n", ret);
+		goto free_v4l2_ctrl_hdl;
+	}
+
+	return 0;
+
+free_v4l2_ctrl_hdl:
+	v4l2_ctrl_handler_free(&chan->ctrl_handler);
+cleanup_media:
+	media_entity_cleanup(&chan->video.entity);
+free_mw_ack_syncpt:
+	host1x_syncpt_free(chan->mw_ack_sp);
+free_fs_syncpt:
+	host1x_syncpt_free(chan->frame_start_sp);
+	return ret;
+}
+
+static int tegra_vi_tpg_channels_alloc(struct tegra_vi *vi)
+{
+	struct tegra_vi_channel *chan;
+	unsigned int port_num;
+	unsigned int nchannels = vi->soc->vi_max_channels;
+
+	for (port_num = 0; port_num < nchannels; port_num++) {
+		/*
+		 * Do not use devm_kzalloc as memory is freed immediately
+		 * when device instance is unbound but application might still
+		 * be holding the device node open. Channel memory allocated
+		 * with kzalloc is freed during video device release callback.
+		 */
+		chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+		if (!chan)
+			return -ENOMEM;
+
+		chan->vi = vi;
+		chan->portno = port_num;
+		list_add_tail(&chan->list, &vi->vi_chans);
+	}
+
+	return 0;
+}
+
+static int tegra_vi_channels_init(struct tegra_vi *vi)
+{
+	struct tegra_vi_channel *chan;
+	int ret;
+
+	list_for_each_entry(chan, &vi->vi_chans, list) {
+		ret = tegra_channel_init(chan);
+		if (ret < 0) {
+			dev_err(vi->dev,
+				"failed to initialize channel-%d: %d\n",
+				chan->portno, ret);
+			goto cleanup;
+		}
+	}
+
+	return 0;
+
+cleanup:
+	list_for_each_entry_continue_reverse(chan, &vi->vi_chans, list)
+		tegra_channel_cleanup(chan);
+
+	return ret;
+}
+
+void tegra_v4l2_nodes_cleanup_tpg(struct tegra_video_device *vid)
+{
+	struct tegra_vi *vi = vid->vi;
+	struct tegra_csi *csi = vid->csi;
+	struct tegra_csi_channel *csi_chan;
+	struct tegra_vi_channel *chan;
+
+	list_for_each_entry(chan, &vi->vi_chans, list) {
+		video_unregister_device(&chan->video);
+		mutex_lock(&chan->video_lock);
+		vb2_queue_release(&chan->queue);
+		mutex_unlock(&chan->video_lock);
+	}
+
+	list_for_each_entry(csi_chan, &csi->csi_chans, list)
+		v4l2_device_unregister_subdev(&csi_chan->subdev);
+}
+
+int tegra_v4l2_nodes_setup_tpg(struct tegra_video_device *vid)
+{
+	struct tegra_vi *vi = vid->vi;
+	struct tegra_csi *csi = vid->csi;
+	struct tegra_vi_channel *vi_chan;
+	struct tegra_csi_channel *csi_chan;
+	u32 link_flags = MEDIA_LNK_FL_ENABLED;
+	int ret;
+
+	if (!vi || !csi)
+		return -ENODEV;
+
+	csi_chan = list_first_entry(&csi->csi_chans,
+				    struct tegra_csi_channel, list);
+
+	list_for_each_entry(vi_chan, &vi->vi_chans, list) {
+		struct media_entity *source = &csi_chan->subdev.entity;
+		struct media_entity *sink = &vi_chan->video.entity;
+		struct media_pad *source_pad = csi_chan->pads;
+		struct media_pad *sink_pad = &vi_chan->pad;
+
+		ret = v4l2_device_register_subdev(&vid->v4l2_dev,
+						  &csi_chan->subdev);
+		if (ret) {
+			dev_err(vi->dev,
+				"failed to register subdev: %d\n", ret);
+			goto cleanup;
+		}
+
+		ret = video_register_device(&vi_chan->video,
+					    VFL_TYPE_VIDEO, -1);
+		if (ret < 0) {
+			dev_err(vi->dev,
+				"failed to register video device: %d\n", ret);
+			goto cleanup;
+		}
+
+		dev_dbg(vi->dev, "creating %s:%u -> %s:%u link\n",
+			source->name, source_pad->index,
+			sink->name, sink_pad->index);
+
+		ret = media_create_pad_link(source, source_pad->index,
+					    sink, sink_pad->index,
+					    link_flags);
+		if (ret < 0) {
+			dev_err(vi->dev,
+				"failed to create %s:%u -> %s:%u link: %d\n",
+				source->name, source_pad->index,
+				sink->name, sink_pad->index, ret);
+			goto cleanup;
+		}
+
+		ret = tegra_channel_setup_ctrl_handler(vi_chan);
+		if (ret < 0)
+			goto cleanup;
+
+		v4l2_set_subdev_hostdata(&csi_chan->subdev, vi_chan);
+		vi_tpg_fmts_bitmap_init(vi_chan);
+		csi_chan = list_next_entry(csi_chan, list);
+	}
+
+	return 0;
+
+cleanup:
+	tegra_v4l2_nodes_cleanup_tpg(vid);
+	return ret;
+}
+
+static int __maybe_unused vi_runtime_resume(struct device *dev)
+{
+	struct tegra_vi *vi = dev_get_drvdata(dev);
+	int ret;
+
+	ret = regulator_enable(vi->vdd);
+	if (ret) {
+		dev_err(dev, "failed to enable VDD supply: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_set_rate(vi->clk, vi->soc->vi_max_clk_hz);
+	if (ret) {
+		dev_err(dev, "failed to set vi clock rate: %d\n", ret);
+		goto disable_vdd;
+	}
+
+	ret = clk_prepare_enable(vi->clk);
+	if (ret) {
+		dev_err(dev, "failed to enable vi clock: %d\n", ret);
+		goto disable_vdd;
+	}
+
+	return 0;
+
+disable_vdd:
+	regulator_disable(vi->vdd);
+	return ret;
+}
+
+static int __maybe_unused vi_runtime_suspend(struct device *dev)
+{
+	struct tegra_vi *vi = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(vi->clk);
+
+	regulator_disable(vi->vdd);
+
+	return 0;
+}
+
+static int tegra_vi_init(struct host1x_client *client)
+{
+	struct tegra_video_device *vid = dev_get_drvdata(client->host);
+	struct tegra_vi *vi = host1x_client_to_vi(client);
+	struct tegra_vi_channel *chan, *tmp;
+	int ret;
+
+	vid->media_dev.hw_revision = vi->soc->hw_revision;
+	snprintf(vid->media_dev.bus_info, sizeof(vid->media_dev.bus_info),
+		 "platform:%s", dev_name(vi->dev));
+
+	INIT_LIST_HEAD(&vi->vi_chans);
+
+	ret = tegra_vi_tpg_channels_alloc(vi);
+	if (ret < 0) {
+		dev_err(vi->dev, "failed to allocate tpg channels: %d\n", ret);
+		goto free_chans;
+	}
+
+	ret = tegra_vi_channels_init(vi);
+	if (ret < 0)
+		goto free_chans;
+
+	vid->vi = vi;
+
+	return 0;
+
+free_chans:
+	list_for_each_entry_safe(chan, tmp, &vi->vi_chans, list) {
+		list_del(&chan->list);
+		kfree(chan);
+	}
+
+	return ret;
+}
+
+static int tegra_vi_exit(struct host1x_client *client)
+{
+	/*
+	 * Do not cleanup the channels here as application might still be
+	 * holding video device nodes. Channels cleanup will happen during
+	 * v4l2_device release callback which gets called after all video
+	 * device nodes are released.
+	 */
+
+	return 0;
+}
+
+static const struct host1x_client_ops vi_client_ops = {
+	.init = tegra_vi_init,
+	.exit = tegra_vi_exit,
+};
+
+static int tegra_vi_probe(struct platform_device *pdev)
+{
+	struct tegra_vi *vi;
+	int ret;
+
+	vi = devm_kzalloc(&pdev->dev, sizeof(*vi), GFP_KERNEL);
+	if (!vi)
+		return -ENOMEM;
+
+	vi->iomem = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(vi->iomem))
+		return PTR_ERR(vi->iomem);
+
+	vi->soc = of_device_get_match_data(&pdev->dev);
+
+	vi->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(vi->clk)) {
+		ret = PTR_ERR(vi->clk);
+		dev_err(&pdev->dev, "failed to get vi clock: %d\n", ret);
+		return ret;
+	}
+
+	vi->vdd = devm_regulator_get(&pdev->dev, "avdd-dsi-csi");
+	if (IS_ERR(vi->vdd)) {
+		ret = PTR_ERR(vi->vdd);
+		dev_err(&pdev->dev, "failed to get VDD supply: %d\n", ret);
+		return ret;
+	}
+
+	if (!pdev->dev.pm_domain) {
+		ret = -ENOENT;
+		dev_warn(&pdev->dev, "PM domain is not attached: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_of_platform_populate(&pdev->dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev,
+			"failed to populate vi child device: %d\n", ret);
+		return ret;
+	}
+
+	vi->dev = &pdev->dev;
+	vi->ops = vi->soc->ops;
+	platform_set_drvdata(pdev, vi);
+	pm_runtime_enable(&pdev->dev);
+
+	/* initialize host1x interface */
+	INIT_LIST_HEAD(&vi->client.list);
+	vi->client.ops = &vi_client_ops;
+	vi->client.dev = &pdev->dev;
+
+	ret = host1x_client_register(&vi->client);
+	if (ret < 0) {
+		dev_err(&pdev->dev,
+			"failed to register host1x client: %d\n", ret);
+		goto rpm_disable;
+	}
+
+	return 0;
+
+rpm_disable:
+	pm_runtime_disable(&pdev->dev);
+	return ret;
+}
+
+static int tegra_vi_remove(struct platform_device *pdev)
+{
+	struct tegra_vi *vi = platform_get_drvdata(pdev);
+	int err;
+
+	err = host1x_client_unregister(&vi->client);
+	if (err < 0) {
+		dev_err(&pdev->dev,
+			"failed to unregister host1x client: %d\n", err);
+		return err;
+	}
+
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static const struct of_device_id tegra_vi_of_id_table[] = {
+#if defined(CONFIG_ARCH_TEGRA_210_SOC)
+	{ .compatible = "nvidia,tegra210-vi", .data = &tegra210_vi_soc },
+#endif
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tegra_vi_of_id_table);
+
+static const struct dev_pm_ops tegra_vi_pm_ops = {
+	SET_RUNTIME_PM_OPS(vi_runtime_suspend, vi_runtime_resume, NULL)
+};
+
+struct platform_driver tegra_vi_driver = {
+	.driver = {
+		.name = "tegra-vi",
+		.of_match_table = tegra_vi_of_id_table,
+		.pm = &tegra_vi_pm_ops,
+	},
+	.probe = tegra_vi_probe,
+	.remove = tegra_vi_remove,
+};
diff --git a/drivers/staging/media/tegra-video/vi.h b/drivers/staging/media/tegra-video/vi.h
new file mode 100644
index 000000000000..6272c9a61809
--- /dev/null
+++ b/drivers/staging/media/tegra-video/vi.h
@@ -0,0 +1,257 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#ifndef __TEGRA_VI_H__
+#define __TEGRA_VI_H__
+
+#include <linux/host1x.h>
+#include <linux/list.h>
+
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+#include <linux/wait.h>
+
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-subdev.h>
+#include <media/videobuf2-v4l2.h>
+
+#define TEGRA_MIN_WIDTH		32U
+#define TEGRA_MAX_WIDTH		32768U
+#define TEGRA_MIN_HEIGHT	32U
+#define TEGRA_MAX_HEIGHT	32768U
+
+#define TEGRA_DEF_WIDTH		1920
+#define TEGRA_DEF_HEIGHT	1080
+#define TEGRA_IMAGE_FORMAT_DEF	32
+
+#define MAX_FORMAT_NUM		64
+
+enum tegra_vi_pg_mode {
+	TEGRA_VI_PG_DISABLED = 0,
+	TEGRA_VI_PG_DIRECT,
+	TEGRA_VI_PG_PATCH,
+};
+
+/**
+ * struct tegra_vi_ops - Tegra VI operations
+ * @vi_start_streaming: starts media pipeline, subdevice streaming, sets up
+ *		VI for capture and runs capture start and capture finish
+ *		kthreads for capturing frames to buffer and returns them back.
+ * @vi_stop_streaming: stops media pipeline and subdevice streaming and returns
+ *		back any queued buffers.
+ */
+struct tegra_vi_ops {
+	int (*vi_start_streaming)(struct vb2_queue *vq, u32 count);
+	void (*vi_stop_streaming)(struct vb2_queue *vq);
+};
+
+/**
+ * struct tegra_vi_soc - NVIDIA Tegra Video Input SoC structure
+ *
+ * @video_formats: supported video formats
+ * @nformats: total video formats
+ * @ops: vi operations
+ * @hw_revision: VI hw_revision
+ * @vi_max_channels: supported max streaming channels
+ * @vi_max_clk_hz: VI clock max frequency
+ */
+struct tegra_vi_soc {
+	const struct tegra_video_format *video_formats;
+	const unsigned int nformats;
+	const struct tegra_vi_ops *ops;
+	u32 hw_revision;
+	unsigned int vi_max_channels;
+	unsigned int vi_max_clk_hz;
+};
+
+/**
+ * struct tegra_vi - NVIDIA Tegra Video Input device structure
+ *
+ * @dev: device struct
+ * @client: host1x_client struct
+ * @iomem: register base
+ * @clk: main clock for VI block
+ * @vdd: vdd regulator for VI hardware, normally it is avdd_dsi_csi
+ * @soc: pointer to SoC data structure
+ * @ops: vi operations
+ * @vi_chans: list head for VI channels
+ */
+struct tegra_vi {
+	struct device *dev;
+	struct host1x_client client;
+	void __iomem *iomem;
+	struct clk *clk;
+	struct regulator *vdd;
+	const struct tegra_vi_soc *soc;
+	const struct tegra_vi_ops *ops;
+	struct list_head vi_chans;
+};
+
+/**
+ * struct tegra_vi_channel - Tegra video channel
+ *
+ * @list: list head for this entry
+ * @video: V4L2 video device associated with the video channel
+ * @video_lock: protects the @format and @queue fields
+ * @pad: media pad for the video device entity
+ *
+ * @vi: Tegra video input device structure
+ * @frame_start_sp: host1x syncpoint pointer to synchronize programmed capture
+ *		start condition with hardware frame start events through host1x
+ *		syncpoint counters.
+ * @mw_ack_sp: host1x syncpoint pointer to synchronize programmed memory write
+ *		ack trigger condition with hardware memory write done at end of
+ *		frame through host1x syncpoint counters.
+ * @sp_incr_lock: protects cpu syncpoint increment.
+ *
+ * @kthread_start_capture: kthread to start capture of single frame when
+ *		vb buffer is available. This thread programs VI CSI hardware
+ *		for single frame capture and waits for frame start event from
+ *		the hardware. On receiving frame start event, it wakes up
+ *		kthread_finish_capture thread to wait for finishing frame data
+ *		write to the memory. In case of missing frame start event, this
+ *		thread returns buffer back to vb with VB2_BUF_STATE_ERROR.
+ * @start_wait: waitqueue for starting frame capture when buffer is available.
+ * @kthread_finish_capture: kthread to finish the buffer capture and return to.
+ *		This thread is woken up by kthread_start_capture on receiving
+ *		frame start event from the hardware and this thread waits for
+ *		MW_ACK_DONE event which indicates completion of writing frame
+ *		data to the memory. On receiving MW_ACK_DONE event, buffer is
+ *		returned back to vb with VB2_BUF_STATE_DONE and in case of
+ *		missing MW_ACK_DONE event, buffer is returned back to vb with
+ *		VB2_BUF_STATE_ERROR.
+ * @done_wait: waitqueue for finishing capture data writes to memory.
+ *
+ * @format: active V4L2 pixel format
+ * @fmtinfo: format information corresponding to the active @format
+ * @queue: vb2 buffers queue
+ * @sequence: V4L2 buffers sequence number
+ *
+ * @capture: list of queued buffers for capture
+ * @start_lock: protects the capture queued list
+ * @done: list of capture done queued buffers
+ * @done_lock: protects the capture done queue list
+ *
+ * @portno: VI channel port number
+ *
+ * @ctrl_handler: V4L2 control handler of this video channel
+ * @tpg_fmts_bitmap: a bitmap for supported TPG formats
+ * @pg_mode: test pattern generator mode (disabled/direct/patch)
+ */
+struct tegra_vi_channel {
+	struct list_head list;
+	struct video_device video;
+	/* protects the @format and @queue fields */
+	struct mutex video_lock;
+	struct media_pad pad;
+
+	struct tegra_vi *vi;
+	struct host1x_syncpt *frame_start_sp;
+	struct host1x_syncpt *mw_ack_sp;
+	/* protects the cpu syncpoint increment */
+	spinlock_t sp_incr_lock;
+
+	struct task_struct *kthread_start_capture;
+	wait_queue_head_t start_wait;
+	struct task_struct *kthread_finish_capture;
+	wait_queue_head_t done_wait;
+
+	struct v4l2_pix_format format;
+	const struct tegra_video_format *fmtinfo;
+	struct vb2_queue queue;
+	u32 sequence;
+
+	struct list_head capture;
+	/* protects the capture queued list */
+	spinlock_t start_lock;
+	struct list_head done;
+	/* protects the capture done queue list */
+	spinlock_t done_lock;
+
+	unsigned char portno;
+
+	struct v4l2_ctrl_handler ctrl_handler;
+	DECLARE_BITMAP(tpg_fmts_bitmap, MAX_FORMAT_NUM);
+	enum tegra_vi_pg_mode pg_mode;
+};
+
+/**
+ * struct tegra_channel_buffer - video channel buffer
+ *
+ * @buf: vb2 buffer base object
+ * @queue: buffer list entry in the channel queued buffers list
+ * @chan: channel that uses the buffer
+ * @addr: Tegra IOVA buffer address for VI output
+ * @mw_ack_sp_thresh: MW_ACK_DONE syncpoint threshold corresponding
+ *		      to the capture buffer.
+ */
+struct tegra_channel_buffer {
+	struct vb2_v4l2_buffer buf;
+	struct list_head queue;
+	struct tegra_vi_channel *chan;
+	dma_addr_t addr;
+	u32 mw_ack_sp_thresh;
+};
+
+/*
+ * VI channel input data type enum.
+ * These data type enum value gets programmed into corresponding Tegra VI
+ * channel register bits.
+ */
+enum tegra_image_dt {
+	TEGRA_IMAGE_DT_YUV420_8 = 24,
+	TEGRA_IMAGE_DT_YUV420_10,
+
+	TEGRA_IMAGE_DT_YUV420CSPS_8 = 28,
+	TEGRA_IMAGE_DT_YUV420CSPS_10,
+	TEGRA_IMAGE_DT_YUV422_8,
+	TEGRA_IMAGE_DT_YUV422_10,
+	TEGRA_IMAGE_DT_RGB444,
+	TEGRA_IMAGE_DT_RGB555,
+	TEGRA_IMAGE_DT_RGB565,
+	TEGRA_IMAGE_DT_RGB666,
+	TEGRA_IMAGE_DT_RGB888,
+
+	TEGRA_IMAGE_DT_RAW6 = 40,
+	TEGRA_IMAGE_DT_RAW7,
+	TEGRA_IMAGE_DT_RAW8,
+	TEGRA_IMAGE_DT_RAW10,
+	TEGRA_IMAGE_DT_RAW12,
+	TEGRA_IMAGE_DT_RAW14,
+};
+
+/**
+ * struct tegra_video_format - Tegra video format description
+ *
+ * @img_dt: image data type
+ * @bit_width: format width in bits per component
+ * @code: media bus format code
+ * @bpp: bytes per pixel (when stored in memory)
+ * @img_fmt: image format
+ * @fourcc: V4L2 pixel format FCC identifier
+ */
+struct tegra_video_format {
+	enum tegra_image_dt img_dt;
+	unsigned int bit_width;
+	unsigned int code;
+	unsigned int bpp;
+	u32 img_fmt;
+	u32 fourcc;
+};
+
+#if defined(CONFIG_ARCH_TEGRA_210_SOC)
+extern const struct tegra_vi_soc tegra210_vi_soc;
+#endif
+
+struct v4l2_subdev *
+tegra_channel_get_remote_subdev(struct tegra_vi_channel *chan);
+int tegra_channel_set_stream(struct tegra_vi_channel *chan, bool on);
+void tegra_channel_release_buffers(struct tegra_vi_channel *chan,
+				   enum vb2_buffer_state state);
+void tegra_channels_cleanup(struct tegra_vi *vi);
+#endif
diff --git a/drivers/staging/media/tegra-video/video.c b/drivers/staging/media/tegra-video/video.c
new file mode 100644
index 000000000000..30816aa41e81
--- /dev/null
+++ b/drivers/staging/media/tegra-video/video.c
@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#include <linux/host1x.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "video.h"
+
+static void tegra_v4l2_dev_release(struct v4l2_device *v4l2_dev)
+{
+	struct tegra_video_device *vid;
+
+	vid = container_of(v4l2_dev, struct tegra_video_device, v4l2_dev);
+
+	/* cleanup channels here as all video device nodes are released */
+	tegra_channels_cleanup(vid->vi);
+
+	v4l2_device_unregister(v4l2_dev);
+	media_device_unregister(&vid->media_dev);
+	media_device_cleanup(&vid->media_dev);
+	kfree(vid);
+}
+
+static int host1x_video_probe(struct host1x_device *dev)
+{
+	struct tegra_video_device *vid;
+	int ret;
+
+	vid = kzalloc(sizeof(*vid), GFP_KERNEL);
+	if (!vid)
+		return -ENOMEM;
+
+	dev_set_drvdata(&dev->dev, vid);
+
+	vid->media_dev.dev = &dev->dev;
+	strscpy(vid->media_dev.model, "NVIDIA Tegra Video Input Device",
+		sizeof(vid->media_dev.model));
+
+	media_device_init(&vid->media_dev);
+	ret = media_device_register(&vid->media_dev);
+	if (ret < 0) {
+		dev_err(&dev->dev,
+			"failed to register media device: %d\n", ret);
+		goto cleanup;
+	}
+
+	vid->v4l2_dev.mdev = &vid->media_dev;
+	vid->v4l2_dev.release = tegra_v4l2_dev_release;
+	ret = v4l2_device_register(&dev->dev, &vid->v4l2_dev);
+	if (ret < 0) {
+		dev_err(&dev->dev,
+			"V4L2 device registration failed: %d\n", ret);
+		goto unregister_media;
+	}
+
+	ret = host1x_device_init(dev);
+	if (ret < 0)
+		goto unregister_v4l2;
+
+	/*
+	 * Both vi and csi channels are available now.
+	 * Register v4l2 nodes and create media links for TPG.
+	 */
+	ret = tegra_v4l2_nodes_setup_tpg(vid);
+	if (ret < 0) {
+		dev_err(&dev->dev,
+			"failed to setup tpg graph: %d\n", ret);
+		goto device_exit;
+	}
+
+	return 0;
+
+device_exit:
+	host1x_device_exit(dev);
+	/* vi exit ops does not clean channels, so clean them here */
+	tegra_channels_cleanup(vid->vi);
+unregister_v4l2:
+	v4l2_device_unregister(&vid->v4l2_dev);
+unregister_media:
+	media_device_unregister(&vid->media_dev);
+cleanup:
+	media_device_cleanup(&vid->media_dev);
+	kfree(vid);
+	return ret;
+}
+
+static int host1x_video_remove(struct host1x_device *dev)
+{
+	struct tegra_video_device *vid = dev_get_drvdata(&dev->dev);
+
+	tegra_v4l2_nodes_cleanup_tpg(vid);
+
+	host1x_device_exit(dev);
+
+	/* This calls v4l2_dev release callback on last reference */
+	v4l2_device_put(&vid->v4l2_dev);
+
+	return 0;
+}
+
+static const struct of_device_id host1x_video_subdevs[] = {
+#if defined(CONFIG_ARCH_TEGRA_210_SOC)
+	{ .compatible = "nvidia,tegra210-csi", },
+	{ .compatible = "nvidia,tegra210-vi", },
+#endif
+	{ }
+};
+
+static struct host1x_driver host1x_video_driver = {
+	.driver = {
+		.name = "tegra-video",
+	},
+	.probe = host1x_video_probe,
+	.remove = host1x_video_remove,
+	.subdevs = host1x_video_subdevs,
+};
+
+static struct platform_driver * const drivers[] = {
+	&tegra_csi_driver,
+	&tegra_vi_driver,
+};
+
+static int __init host1x_video_init(void)
+{
+	int err;
+
+	err = host1x_driver_register(&host1x_video_driver);
+	if (err < 0)
+		return err;
+
+	err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
+	if (err < 0)
+		goto unregister_host1x;
+
+	return 0;
+
+unregister_host1x:
+	host1x_driver_unregister(&host1x_video_driver);
+	return err;
+}
+module_init(host1x_video_init);
+
+static void __exit host1x_video_exit(void)
+{
+	platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
+	host1x_driver_unregister(&host1x_video_driver);
+}
+module_exit(host1x_video_exit);
+
+MODULE_AUTHOR("Sowjanya Komatineni <skomatineni@nvidia.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra Host1x Video driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/media/tegra-video/video.h b/drivers/staging/media/tegra-video/video.h
new file mode 100644
index 000000000000..fadaf2189dc9
--- /dev/null
+++ b/drivers/staging/media/tegra-video/video.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#ifndef __TEGRA_VIDEO_H__
+#define __TEGRA_VIDEO_H__
+
+#include <linux/host1x.h>
+
+#include <media/media-device.h>
+#include <media/v4l2-device.h>
+
+#include "vi.h"
+#include "csi.h"
+
+struct tegra_video_device {
+	struct v4l2_device v4l2_dev;
+	struct media_device media_dev;
+	struct tegra_vi *vi;
+	struct tegra_csi *csi;
+};
+
+int tegra_v4l2_nodes_setup_tpg(struct tegra_video_device *vid);
+void tegra_v4l2_nodes_cleanup_tpg(struct tegra_video_device *vid);
+
+extern struct platform_driver tegra_vi_driver;
+extern struct platform_driver tegra_csi_driver;
+#endif
diff --git a/drivers/staging/most/cdev/Kconfig b/drivers/staging/most/cdev/Kconfig
index 330c95fe6550..dab99477858e 100644
--- a/drivers/staging/most/cdev/Kconfig
+++ b/drivers/staging/most/cdev/Kconfig
@@ -6,7 +6,7 @@
 config MOST_CDEV
 	tristate "Cdev"
 
-	---help---
+	help
 	  Say Y here if you want to commumicate via character devices.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/staging/most/dim2/Kconfig b/drivers/staging/most/dim2/Kconfig
index 22f6900187b5..c211f0d9139c 100644
--- a/drivers/staging/most/dim2/Kconfig
+++ b/drivers/staging/most/dim2/Kconfig
@@ -7,7 +7,7 @@ config MOST_DIM2
 	tristate "DIM2"
 	depends on HAS_IOMEM && OF
 
-	---help---
+	help
 	  Say Y here if you want to connect via MediaLB to network transceiver.
 	  This device driver is platform dependent and needs an additional
 	  platform driver to be installed. For more information contact
diff --git a/drivers/staging/most/usb/Kconfig b/drivers/staging/most/usb/Kconfig
index a86f1f63def4..75dc25c0e0e5 100644
--- a/drivers/staging/most/usb/Kconfig
+++ b/drivers/staging/most/usb/Kconfig
@@ -7,7 +7,7 @@ config MOST_USB
 	tristate "USB"
 	depends on USB && NET
 	help
-	  Say Y here if you want to connect via USB to network tranceiver.
+	  Say Y here if you want to connect via USB to network transceiver.
 	  This device driver depends on the networking AIM.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/staging/most/usb/usb.c b/drivers/staging/most/usb/usb.c
index e8c5a8c98375..2640c5b326a4 100644
--- a/drivers/staging/most/usb/usb.c
+++ b/drivers/staging/most/usb/usb.c
@@ -5,7 +5,6 @@
  * Copyright (C) 2013-2015 Microchip Technology Germany II GmbH & Co. KG
  */
 
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/module.h>
 #include <linux/fs.h>
 #include <linux/usb.h>
@@ -140,9 +139,10 @@ static void wq_netinfo(struct work_struct *wq_obj);
 static inline int drci_rd_reg(struct usb_device *dev, u16 reg, u16 *buf)
 {
 	int retval;
-	__le16 *dma_buf = kzalloc(sizeof(*dma_buf), GFP_KERNEL);
+	__le16 *dma_buf;
 	u8 req_type = USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE;
 
+	dma_buf = kzalloc(sizeof(*dma_buf), GFP_KERNEL);
 	if (!dma_buf)
 		return -ENOMEM;
 
@@ -153,7 +153,9 @@ static inline int drci_rd_reg(struct usb_device *dev, u16 reg, u16 *buf)
 	*buf = le16_to_cpu(*dma_buf);
 	kfree(dma_buf);
 
-	return retval;
+	if (retval < 0)
+		return retval;
+	return 0;
 }
 
 /**
@@ -184,16 +186,18 @@ static inline int start_sync_ep(struct usb_device *usb_dev, u16 ep)
 
 /**
  * get_stream_frame_size - calculate frame size of current configuration
+ * @dev: device structure
  * @cfg: channel configuration
  */
-static unsigned int get_stream_frame_size(struct most_channel_config *cfg)
+static unsigned int get_stream_frame_size(struct device *dev,
+					  struct most_channel_config *cfg)
 {
-	unsigned int frame_size = 0;
+	unsigned int frame_size;
 	unsigned int sub_size = cfg->subbuffer_size;
 
 	if (!sub_size) {
-		pr_warn("Misconfig: Subbuffer size zero.\n");
-		return frame_size;
+		dev_warn(dev, "Misconfig: Subbuffer size zero.\n");
+		return 0;
 	}
 	switch (cfg->data_type) {
 	case MOST_CH_ISOC:
@@ -201,7 +205,7 @@ static unsigned int get_stream_frame_size(struct most_channel_config *cfg)
 		break;
 	case MOST_CH_SYNC:
 		if (cfg->packets_per_xact == 0) {
-			pr_warn("Misconfig: Packets per XACT zero\n");
+			dev_warn(dev, "Misconfig: Packets per XACT zero\n");
 			frame_size = 0;
 		} else if (cfg->packets_per_xact == 0xFF) {
 			frame_size = (USB_MTU / sub_size) * sub_size;
@@ -210,7 +214,8 @@ static unsigned int get_stream_frame_size(struct most_channel_config *cfg)
 		}
 		break;
 	default:
-		pr_warn("Query frame size of non-streaming channel\n");
+		dev_warn(dev, "Query frame size of non-streaming channel\n");
+		frame_size = 0;
 		break;
 	}
 	return frame_size;
@@ -233,11 +238,7 @@ static int hdm_poison_channel(struct most_interface *iface, int channel)
 	unsigned long flags;
 	spinlock_t *lock; /* temp. lock */
 
-	if (unlikely(!iface)) {
-		dev_warn(&mdev->usb_device->dev, "Poison: Bad interface.\n");
-		return -EIO;
-	}
-	if (unlikely(channel < 0 || channel >= iface->num_channels)) {
+	if (channel < 0 || channel >= iface->num_channels) {
 		dev_warn(&mdev->usb_device->dev, "Channel ID out of range.\n");
 		return -ECHRNG;
 	}
@@ -274,17 +275,17 @@ static int hdm_poison_channel(struct most_interface *iface, int channel)
 static int hdm_add_padding(struct most_dev *mdev, int channel, struct mbo *mbo)
 {
 	struct most_channel_config *conf = &mdev->conf[channel];
-	unsigned int frame_size = get_stream_frame_size(conf);
+	unsigned int frame_size = get_stream_frame_size(&mdev->dev, conf);
 	unsigned int j, num_frames;
 
 	if (!frame_size)
-		return -EIO;
+		return -EINVAL;
 	num_frames = mbo->buffer_length / frame_size;
 
 	if (num_frames < 1) {
 		dev_err(&mdev->usb_device->dev,
 			"Missed minimal transfer unit.\n");
-		return -EIO;
+		return -EINVAL;
 	}
 
 	for (j = num_frames - 1; j > 0; j--)
@@ -308,11 +309,11 @@ static int hdm_remove_padding(struct most_dev *mdev, int channel,
 			      struct mbo *mbo)
 {
 	struct most_channel_config *const conf = &mdev->conf[channel];
-	unsigned int frame_size = get_stream_frame_size(conf);
+	unsigned int frame_size = get_stream_frame_size(&mdev->dev, conf);
 	unsigned int j, num_frames;
 
 	if (!frame_size)
-		return -EIO;
+		return -EINVAL;
 	num_frames = mbo->processed_length / USB_MTU;
 
 	for (j = 1; j < num_frames; j++)
@@ -386,103 +387,6 @@ static void hdm_write_completion(struct urb *urb)
  * padding bytes -if necessary- and calls the completion function.
  *
  * Context: interrupt!
- *
- * **************************************************************************
- *                   Error codes returned by in urb->status
- *                   or in iso_frame_desc[n].status (for ISO)
- * *************************************************************************
- *
- * USB device drivers may only test urb status values in completion handlers.
- * This is because otherwise there would be a race between HCDs updating
- * these values on one CPU, and device drivers testing them on another CPU.
- *
- * A transfer's actual_length may be positive even when an error has been
- * reported.  That's because transfers often involve several packets, so that
- * one or more packets could finish before an error stops further endpoint I/O.
- *
- * For isochronous URBs, the urb status value is non-zero only if the URB is
- * unlinked, the device is removed, the host controller is disabled or the total
- * transferred length is less than the requested length and the URB_SHORT_NOT_OK
- * flag is set.  Completion handlers for isochronous URBs should only see
- * urb->status set to zero, -ENOENT, -ECONNRESET, -ESHUTDOWN, or -EREMOTEIO.
- * Individual frame descriptor status fields may report more status codes.
- *
- *
- * 0			Transfer completed successfully
- *
- * -ENOENT		URB was synchronously unlinked by usb_unlink_urb
- *
- * -EINPROGRESS		URB still pending, no results yet
- *			(That is, if drivers see this it's a bug.)
- *
- * -EPROTO (*, **)	a) bitstuff error
- *			b) no response packet received within the
- *			   prescribed bus turn-around time
- *			c) unknown USB error
- *
- * -EILSEQ (*, **)	a) CRC mismatch
- *			b) no response packet received within the
- *			   prescribed bus turn-around time
- *			c) unknown USB error
- *
- *			Note that often the controller hardware does not
- *			distinguish among cases a), b), and c), so a
- *			driver cannot tell whether there was a protocol
- *			error, a failure to respond (often caused by
- *			device disconnect), or some other fault.
- *
- * -ETIME (**)		No response packet received within the prescribed
- *			bus turn-around time.  This error may instead be
- *			reported as -EPROTO or -EILSEQ.
- *
- * -ETIMEDOUT		Synchronous USB message functions use this code
- *			to indicate timeout expired before the transfer
- *			completed, and no other error was reported by HC.
- *
- * -EPIPE (**)		Endpoint stalled.  For non-control endpoints,
- *			reset this status with usb_clear_halt().
- *
- * -ECOMM		During an IN transfer, the host controller
- *			received data from an endpoint faster than it
- *			could be written to system memory
- *
- * -ENOSR		During an OUT transfer, the host controller
- *			could not retrieve data from system memory fast
- *			enough to keep up with the USB data rate
- *
- * -EOVERFLOW (*)	The amount of data returned by the endpoint was
- *			greater than either the max packet size of the
- *			endpoint or the remaining buffer size.  "Babble".
- *
- * -EREMOTEIO		The data read from the endpoint did not fill the
- *			specified buffer, and URB_SHORT_NOT_OK was set in
- *			urb->transfer_flags.
- *
- * -ENODEV		Device was removed.  Often preceded by a burst of
- *			other errors, since the hub driver doesn't detect
- *			device removal events immediately.
- *
- * -EXDEV		ISO transfer only partially completed
- *			(only set in iso_frame_desc[n].status, not urb->status)
- *
- * -EINVAL		ISO madness, if this happens: Log off and go home
- *
- * -ECONNRESET		URB was asynchronously unlinked by usb_unlink_urb
- *
- * -ESHUTDOWN		The device or host controller has been disabled due
- *			to some problem that could not be worked around,
- *			such as a physical disconnect.
- *
- *
- * (*) Error codes like -EPROTO, -EILSEQ and -EOVERFLOW normally indicate
- * hardware problems such as bad devices (including firmware) or cables.
- *
- * (**) This is also one of several codes that different kinds of host
- * controller use to indicate a transfer has failed because of device
- * disconnect.  In the interval before the hub driver starts disconnect
- * processing, devices may receive such fault reports for every request.
- *
- * See <https://www.kernel.org/doc/Documentation/driver-api/usb/error-codes.rst>
  */
 static void hdm_read_completion(struct urb *urb)
 {
@@ -552,36 +456,33 @@ static void hdm_read_completion(struct urb *urb)
 static int hdm_enqueue(struct most_interface *iface, int channel,
 		       struct mbo *mbo)
 {
-	struct most_dev *mdev;
+	struct most_dev *mdev = to_mdev(iface);
 	struct most_channel_config *conf;
 	int retval = 0;
 	struct urb *urb;
 	unsigned long length;
 	void *virt_address;
 
-	if (unlikely(!iface || !mbo))
-		return -EIO;
-	if (unlikely(iface->num_channels <= channel || channel < 0))
+	if (!mbo)
+		return -EINVAL;
+	if (iface->num_channels <= channel || channel < 0)
 		return -ECHRNG;
 
-	mdev = to_mdev(iface);
+	urb = usb_alloc_urb(NO_ISOCHRONOUS_URB, GFP_KERNEL);
+	if (!urb)
+		return -ENOMEM;
+
 	conf = &mdev->conf[channel];
 
 	mutex_lock(&mdev->io_mutex);
 	if (!mdev->usb_device) {
 		retval = -ENODEV;
-		goto unlock_io_mutex;
-	}
-
-	urb = usb_alloc_urb(NO_ISOCHRONOUS_URB, GFP_ATOMIC);
-	if (!urb) {
-		retval = -ENOMEM;
-		goto unlock_io_mutex;
+		goto err_free_urb;
 	}
 
 	if ((conf->direction & MOST_CH_TX) && mdev->padding_active[channel] &&
 	    hdm_add_padding(mdev, channel, mbo)) {
-		retval = -EIO;
+		retval = -EINVAL;
 		goto err_free_urb;
 	}
 
@@ -619,13 +520,13 @@ static int hdm_enqueue(struct most_interface *iface, int channel,
 			"URB submit failed with error %d.\n", retval);
 		goto err_unanchor_urb;
 	}
-	goto unlock_io_mutex;
+	mutex_unlock(&mdev->io_mutex);
+	return 0;
 
 err_unanchor_urb:
 	usb_unanchor_urb(urb);
 err_free_urb:
 	usb_free_urb(urb);
-unlock_io_mutex:
 	mutex_unlock(&mdev->io_mutex);
 	return retval;
 }
@@ -669,19 +570,20 @@ static int hdm_configure_channel(struct most_interface *iface, int channel,
 	struct most_dev *mdev = to_mdev(iface);
 	struct device *dev = &mdev->usb_device->dev;
 
-	mdev->is_channel_healthy[channel] = true;
-	mdev->clear_work[channel].channel = channel;
-	mdev->clear_work[channel].mdev = mdev;
-	INIT_WORK(&mdev->clear_work[channel].ws, wq_clear_halt);
-
-	if (unlikely(!iface || !conf)) {
-		dev_err(dev, "Bad interface or config pointer.\n");
+	if (!conf) {
+		dev_err(dev, "Bad config pointer.\n");
 		return -EINVAL;
 	}
-	if (unlikely(channel < 0 || channel >= iface->num_channels)) {
+	if (channel < 0 || channel >= iface->num_channels) {
 		dev_err(dev, "Channel ID out of range.\n");
 		return -EINVAL;
 	}
+
+	mdev->is_channel_healthy[channel] = true;
+	mdev->clear_work[channel].channel = channel;
+	mdev->clear_work[channel].mdev = mdev;
+	INIT_WORK(&mdev->clear_work[channel].ws, wq_clear_halt);
+
 	if (!conf->num_buffers || !conf->buffer_size) {
 		dev_err(dev, "Misconfig: buffer size or #buffers zero.\n");
 		return -EINVAL;
@@ -701,7 +603,7 @@ static int hdm_configure_channel(struct most_interface *iface, int channel,
 
 	mdev->padding_active[channel] = true;
 
-	frame_size = get_stream_frame_size(conf);
+	frame_size = get_stream_frame_size(&mdev->dev, conf);
 	if (frame_size == 0 || frame_size > USB_MTU) {
 		dev_warn(dev, "Misconfig: frame size wrong\n");
 		return -EINVAL;
@@ -745,10 +647,8 @@ static void hdm_request_netinfo(struct most_interface *iface, int channel,
 						   unsigned char,
 						   unsigned char *))
 {
-	struct most_dev *mdev;
+	struct most_dev *mdev = to_mdev(iface);
 
-	BUG_ON(!iface);
-	mdev = to_mdev(iface);
 	mdev->on_netinfo = on_netinfo;
 	if (!on_netinfo)
 		return;
@@ -787,22 +687,22 @@ static void wq_netinfo(struct work_struct *wq_obj)
 	u16 hi, mi, lo, link;
 	u8 hw_addr[6];
 
-	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_HI, &hi) < 0) {
+	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_HI, &hi)) {
 		dev_err(dev, "Vendor request 'hw_addr_hi' failed\n");
 		return;
 	}
 
-	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_MI, &mi) < 0) {
+	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_MI, &mi)) {
 		dev_err(dev, "Vendor request 'hw_addr_mid' failed\n");
 		return;
 	}
 
-	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_LO, &lo) < 0) {
+	if (drci_rd_reg(usb_device, DRCI_REG_HW_ADDR_LO, &lo)) {
 		dev_err(dev, "Vendor request 'hw_addr_low' failed\n");
 		return;
 	}
 
-	if (drci_rd_reg(usb_device, DRCI_REG_NI_STATE, &link) < 0) {
+	if (drci_rd_reg(usb_device, DRCI_REG_NI_STATE, &link)) {
 		dev_err(dev, "Vendor request 'link status' failed\n");
 		return;
 	}
@@ -830,6 +730,8 @@ static void wq_clear_halt(struct work_struct *wq_obj)
 	struct most_dev *mdev = clear_work->mdev;
 	unsigned int channel = clear_work->channel;
 	int pipe = clear_work->pipe;
+	int snd_pipe;
+	int peer;
 
 	mutex_lock(&mdev->io_mutex);
 	most_stop_enqueue(&mdev->iface, channel);
@@ -847,9 +749,12 @@ static void wq_clear_halt(struct work_struct *wq_obj)
 	 */
 	if (mdev->conf[channel].data_type == MOST_CH_ASYNC &&
 	    mdev->conf[channel].direction == MOST_CH_RX) {
-		int peer = 1 - channel;
-		int snd_pipe = usb_sndbulkpipe(mdev->usb_device,
-					       mdev->ep_address[peer]);
+		if (channel == 0)
+			peer = 1;
+		else
+			peer = 0;
+		snd_pipe = usb_sndbulkpipe(mdev->usb_device,
+					   mdev->ep_address[peer]);
 		usb_clear_halt(mdev->usb_device, snd_pipe);
 	}
 	mdev->is_channel_healthy[channel] = true;
@@ -904,12 +809,12 @@ static int get_stat_reg_addr(const struct regs *regs, int size,
 	int i;
 
 	for (i = 0; i < size; i++) {
-		if (!strcmp(name, regs[i].name)) {
+		if (sysfs_streq(name, regs[i].name)) {
 			*reg_addr = regs[i].reg;
 			return 0;
 		}
 	}
-	return -EFAULT;
+	return -EINVAL;
 }
 
 #define get_static_reg_addr(regs, name, reg_addr) \
@@ -924,14 +829,14 @@ static ssize_t value_show(struct device *dev, struct device_attribute *attr,
 	u16 reg_addr;
 	int err;
 
-	if (!strcmp(name, "arb_address"))
+	if (sysfs_streq(name, "arb_address"))
 		return snprintf(buf, PAGE_SIZE, "%04x\n", dci_obj->reg_addr);
 
-	if (!strcmp(name, "arb_value"))
+	if (sysfs_streq(name, "arb_value"))
 		reg_addr = dci_obj->reg_addr;
 	else if (get_static_reg_addr(ro_regs, name, &reg_addr) &&
 		 get_static_reg_addr(rw_regs, name, &reg_addr))
-		return -EFAULT;
+		return -EINVAL;
 
 	err = drci_rd_reg(dci_obj->usb_device, reg_addr, &val);
 	if (err < 0)
@@ -948,24 +853,25 @@ static ssize_t value_store(struct device *dev, struct device_attribute *attr,
 	const char *name = attr->attr.name;
 	struct most_dci_obj *dci_obj = to_dci_obj(dev);
 	struct usb_device *usb_dev = dci_obj->usb_device;
-	int err = kstrtou16(buf, 16, &val);
+	int err;
 
+	err = kstrtou16(buf, 16, &val);
 	if (err)
 		return err;
 
-	if (!strcmp(name, "arb_address")) {
+	if (sysfs_streq(name, "arb_address")) {
 		dci_obj->reg_addr = val;
 		return count;
 	}
 
-	if (!strcmp(name, "arb_value"))
+	if (sysfs_streq(name, "arb_value"))
 		err = drci_wr_reg(usb_dev, dci_obj->reg_addr, val);
-	else if (!strcmp(name, "sync_ep"))
+	else if (sysfs_streq(name, "sync_ep"))
 		err = start_sync_ep(usb_dev, val);
 	else if (!get_static_reg_addr(rw_regs, name, &reg_addr))
 		err = drci_wr_reg(usb_dev, reg_addr, val);
 	else
-		return -EFAULT;
+		return -EINVAL;
 
 	if (err < 0)
 		return err;
@@ -1008,19 +914,13 @@ static struct attribute *dci_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group dci_attr_group = {
-	.attrs = dci_attrs,
-};
-
-static const struct attribute_group *dci_attr_groups[] = {
-	&dci_attr_group,
-	NULL,
-};
+ATTRIBUTE_GROUPS(dci);
 
 static void release_dci(struct device *dev)
 {
 	struct most_dci_obj *dci = to_dci_obj(dev);
 
+	put_device(dev->parent);
 	kfree(dci);
 }
 
@@ -1048,18 +948,23 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 	struct usb_host_interface *usb_iface_desc = interface->cur_altsetting;
 	struct usb_device *usb_dev = interface_to_usbdev(interface);
 	struct device *dev = &usb_dev->dev;
-	struct most_dev *mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
+	struct most_dev *mdev;
 	unsigned int i;
 	unsigned int num_endpoints;
 	struct most_channel_capability *tmp_cap;
 	struct usb_endpoint_descriptor *ep_desc;
-	int ret = 0;
+	int ret = -ENOMEM;
 
+	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
 	if (!mdev)
-		goto err_out_of_memory;
+		return -ENOMEM;
 
 	usb_set_intfdata(interface, mdev);
 	num_endpoints = usb_iface_desc->desc.bNumEndpoints;
+	if (num_endpoints > MAX_NUM_ENDPOINTS) {
+		kfree(mdev);
+		return -EINVAL;
+	}
 	mutex_init(&mdev->io_mutex);
 	INIT_WORK(&mdev->poll_work_obj, wq_netinfo);
 	timer_setup(&mdev->link_stat_timer, link_stat_timer_handler, 0);
@@ -1134,17 +1039,17 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 		init_usb_anchor(&mdev->busy_urbs[i]);
 		spin_lock_init(&mdev->channel_lock[i]);
 	}
-	dev_notice(dev, "claimed gadget: Vendor=%4.4x ProdID=%4.4x Bus=%02x Device=%02x\n",
-		   le16_to_cpu(usb_dev->descriptor.idVendor),
-		   le16_to_cpu(usb_dev->descriptor.idProduct),
-		   usb_dev->bus->busnum,
-		   usb_dev->devnum);
-
-	dev_notice(dev, "device path: /sys/bus/usb/devices/%d-%s:%d.%d\n",
-		   usb_dev->bus->busnum,
-		   usb_dev->devpath,
-		   usb_dev->config->desc.bConfigurationValue,
-		   usb_iface_desc->desc.bInterfaceNumber);
+	dev_dbg(dev, "claimed gadget: Vendor=%4.4x ProdID=%4.4x Bus=%02x Device=%02x\n",
+		le16_to_cpu(usb_dev->descriptor.idVendor),
+		le16_to_cpu(usb_dev->descriptor.idProduct),
+		usb_dev->bus->busnum,
+		usb_dev->devnum);
+
+	dev_dbg(dev, "device path: /sys/bus/usb/devices/%d-%s:%d.%d\n",
+		usb_dev->bus->busnum,
+		usb_dev->devpath,
+		usb_dev->config->desc.bConfigurationValue,
+		usb_iface_desc->desc.bInterfaceNumber);
 
 	ret = most_register_interface(&mdev->iface);
 	if (ret)
@@ -1164,7 +1069,7 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id)
 
 		mdev->dci->dev.init_name = "dci";
 		mdev->dci->dev.parent = get_device(mdev->iface.dev);
-		mdev->dci->dev.groups = dci_attr_groups;
+		mdev->dci->dev.groups = dci_groups;
 		mdev->dci->dev.release = release_dci;
 		if (device_register(&mdev->dci->dev)) {
 			mutex_unlock(&mdev->io_mutex);
@@ -1188,11 +1093,6 @@ err_free_conf:
 	kfree(mdev->conf);
 err_free_mdev:
 	put_device(&mdev->dev);
-err_out_of_memory:
-	if (ret == 0 || ret == -ENOMEM) {
-		ret = -ENOMEM;
-		dev_err(dev, "out of memory\n");
-	}
 	return ret;
 }
 
@@ -1225,14 +1125,43 @@ static void hdm_disconnect(struct usb_interface *interface)
 	kfree(mdev->cap);
 	kfree(mdev->conf);
 	kfree(mdev->ep_address);
+	put_device(&mdev->dci->dev);
 	put_device(&mdev->dev);
 }
 
+static int hdm_suspend(struct usb_interface *interface, pm_message_t message)
+{
+	struct most_dev *mdev = usb_get_intfdata(interface);
+	int i;
+
+	mutex_lock(&mdev->io_mutex);
+	for (i = 0; i < mdev->iface.num_channels; i++) {
+		most_stop_enqueue(&mdev->iface, i);
+		usb_kill_anchored_urbs(&mdev->busy_urbs[i]);
+	}
+	mutex_unlock(&mdev->io_mutex);
+	return 0;
+}
+
+static int hdm_resume(struct usb_interface *interface)
+{
+	struct most_dev *mdev = usb_get_intfdata(interface);
+	int i;
+
+	mutex_lock(&mdev->io_mutex);
+	for (i = 0; i < mdev->iface.num_channels; i++)
+		most_resume_enqueue(&mdev->iface, i);
+	mutex_unlock(&mdev->io_mutex);
+	return 0;
+}
+
 static struct usb_driver hdm_usb = {
 	.name = "hdm_usb",
 	.id_table = usbid,
 	.probe = hdm_probe,
 	.disconnect = hdm_disconnect,
+	.resume = hdm_resume,
+	.suspend = hdm_suspend,
 };
 
 module_usb_driver(hdm_usb);
diff --git a/drivers/staging/mt7621-dts/mt7621.dtsi b/drivers/staging/mt7621-dts/mt7621.dtsi
index 9e5cf68731bb..82aa93634eda 100644
--- a/drivers/staging/mt7621-dts/mt7621.dtsi
+++ b/drivers/staging/mt7621-dts/mt7621.dtsi
@@ -523,11 +523,10 @@
 			0x01000000 0 0x00000000 0x1e160000 0 0x00010000 /* io space */
 		>;
 
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0xF0000 0 0 1>;
-		interrupt-map = <0x10000 0 0 1 &gic GIC_SHARED 4 IRQ_TYPE_LEVEL_HIGH>,
-				<0x20000 0 0 1 &gic GIC_SHARED 24 IRQ_TYPE_LEVEL_HIGH>,
-				<0x30000 0 0 1 &gic GIC_SHARED 25 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SHARED 4 IRQ_TYPE_LEVEL_HIGH
+				GIC_SHARED 24 IRQ_TYPE_LEVEL_HIGH
+				GIC_SHARED 25 IRQ_TYPE_LEVEL_HIGH>;
 
 		status = "disabled";
 
diff --git a/drivers/staging/mt7621-pci-phy/mediatek,mt7621-pci-phy.txt b/drivers/staging/mt7621-pci-phy/mediatek,mt7621-pci-phy.txt
deleted file mode 100644
index a369d715378b..000000000000
--- a/drivers/staging/mt7621-pci-phy/mediatek,mt7621-pci-phy.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-Mediatek Mt7621 PCIe PHY
-
-Required properties:
-- compatible: must be "mediatek,mt7621-pci-phy"
-- reg: base address and length of the PCIe PHY block
-- #phy-cells: must be <1> for pcie0_phy and for pcie1_phy.
-
-Example:
-	pcie0_phy: pcie-phy@1e149000 {
-		compatible = "mediatek,mt7621-pci-phy";
-		reg = <0x1e149000 0x0700>;
-		#phy-cells = <1>;
-	};
-
-	pcie1_phy: pcie-phy@1e14a000 {
-		compatible = "mediatek,mt7621-pci-phy";
-		reg = <0x1e14a000 0x0700>;
-		#phy-cells = <1>;
-	};
-
-	/* users of the PCIe phy */
-
-	pcie: pcie@1e140000 {
-		...
-		...
-		phys = <&pcie0_phy 0>, <&pcie0_phy 1>, <&pcie1_phy 0>;
-		phy-names = "pcie-phy0", "pcie-phy1", "pcie-phy2";
-	};
diff --git a/drivers/staging/mt7621-pci-phy/mediatek,mt7621-pci-phy.yaml b/drivers/staging/mt7621-pci-phy/mediatek,mt7621-pci-phy.yaml
new file mode 100644
index 000000000000..cf32bbc45b5d
--- /dev/null
+++ b/drivers/staging/mt7621-pci-phy/mediatek,mt7621-pci-phy.yaml
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: "http://devicetree.org/schemas/phy/mediatek,mt7621-pci-phy.yaml#"
+$schema: "http://devicetree.org/meta-schemas/core.yaml#"
+
+title: Mediatek Mt7621 PCIe PHY Device Tree Bindings
+
+maintainers:
+  - Sergio Paracuellos <sergio.paracuellos@gmail.com>
+
+properties:
+  compatible:
+    const: mediatek,mt7621-pci-phy
+
+  reg:
+    maxItems: 1
+
+  "#phy-cells":
+    const: 1
+    description: selects if the phy is dual-ported
+
+required:
+  - compatible
+  - reg
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    pcie0_phy: pcie-phy@1e149000 {
+      compatible = "mediatek,mt7621-pci-phy";
+      reg = <0x1e149000 0x0700>;
+      #phy-cells = <1>;
+    };
diff --git a/drivers/staging/mt7621-pci/pci-mt7621.c b/drivers/staging/mt7621-pci/pci-mt7621.c
index f58e3a51fc71..f961b353c22e 100644
--- a/drivers/staging/mt7621-pci/pci-mt7621.c
+++ b/drivers/staging/mt7621-pci/pci-mt7621.c
@@ -55,7 +55,7 @@
 #define RALINK_PCI_IOBASE		0x002C
 
 /* PCICFG virtual bridges */
-#define PCIE_P2P_MAX			3
+#define PCIE_P2P_CNT			3
 #define PCIE_P2P_BR_DEVNUM_SHIFT(p)	(16 + (p) * 4)
 #define PCIE_P2P_BR_DEVNUM0_SHIFT	PCIE_P2P_BR_DEVNUM_SHIFT(0)
 #define PCIE_P2P_BR_DEVNUM1_SHIFT	PCIE_P2P_BR_DEVNUM_SHIFT(1)
@@ -97,6 +97,7 @@
  * @pcie_rst: pointer to port reset control
  * @gpio_rst: gpio reset
  * @slot: port slot
+ * @irq: GIC irq
  * @enabled: indicates if port is enabled
  */
 struct mt7621_pcie_port {
@@ -107,6 +108,7 @@ struct mt7621_pcie_port {
 	struct reset_control *pcie_rst;
 	struct gpio_desc *gpio_rst;
 	u32 slot;
+	int irq;
 	bool enabled;
 };
 
@@ -120,6 +122,7 @@ struct mt7621_pcie_port {
  * @dev: Pointer to PCIe device
  * @io_map_base: virtual memory base address for io
  * @ports: pointer to PCIe port information
+ * @irq_map: irq mapping info according pcie link status
  * @resets_inverted: depends on chip revision
  * reset lines are inverted.
  */
@@ -135,6 +138,7 @@ struct mt7621_pcie {
 	} offset;
 	unsigned long io_map_base;
 	struct list_head ports;
+	int irq_map[PCIE_P2P_CNT];
 	bool resets_inverted;
 };
 
@@ -279,6 +283,16 @@ static void setup_cm_memory_region(struct mt7621_pcie *pcie)
 	}
 }
 
+static int mt7621_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
+{
+	struct mt7621_pcie *pcie = pdev->bus->sysdata;
+	struct device *dev = pcie->dev;
+	int irq = pcie->irq_map[slot];
+
+	dev_info(dev, "bus=%d slot=%d irq=%d\n", pdev->bus->number, slot, irq);
+	return irq;
+}
+
 static int mt7621_pci_parse_request_of_pci_ranges(struct mt7621_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
@@ -330,6 +344,7 @@ static int mt7621_pcie_parse_port(struct mt7621_pcie *pcie,
 {
 	struct mt7621_pcie_port *port;
 	struct device *dev = pcie->dev;
+	struct platform_device *pdev = to_platform_device(dev);
 	struct device_node *pnode = dev->of_node;
 	struct resource regs;
 	char name[10];
@@ -371,6 +386,12 @@ static int mt7621_pcie_parse_port(struct mt7621_pcie *pcie,
 	port->slot = slot;
 	port->pcie = pcie;
 
+	port->irq = platform_get_irq(pdev, slot);
+	if (port->irq < 0) {
+		dev_err(dev, "Failed to get IRQ for PCIe%d\n", slot);
+		return -ENXIO;
+	}
+
 	INIT_LIST_HEAD(&port->list);
 	list_add_tail(&port->list, &pcie->ports);
 
@@ -502,17 +523,25 @@ static void mt7621_pcie_init_ports(struct mt7621_pcie *pcie)
 
 	mt7621_pcie_reset_ep_deassert(pcie);
 
+	tmp = NULL;
 	list_for_each_entry(port, &pcie->ports, list) {
 		u32 slot = port->slot;
 
 		if (!mt7621_pcie_port_is_linkup(port)) {
 			dev_err(dev, "pcie%d no card, disable it (RST & CLK)\n",
 				slot);
-			if (slot != 1)
-				phy_power_off(port->phy);
 			mt7621_control_assert(port);
 			mt7621_pcie_port_clk_disable(port);
 			port->enabled = false;
+
+			if (slot == 0) {
+				tmp = port;
+				continue;
+			}
+
+			if (slot == 1 && tmp && !tmp->enabled)
+				phy_power_off(tmp->phy);
+
 		}
 	}
 }
@@ -576,14 +605,16 @@ static void mt7621_pcie_enable_ports(struct mt7621_pcie *pcie)
 static int mt7621_pcie_init_virtual_bridges(struct mt7621_pcie *pcie)
 {
 	u32 pcie_link_status = 0;
-	u32 n;
-	int i;
-	u32 p2p_br_devnum[PCIE_P2P_MAX];
+	u32 n = 0;
+	int i = 0;
+	u32 p2p_br_devnum[PCIE_P2P_CNT];
+	int irqs[PCIE_P2P_CNT];
 	struct mt7621_pcie_port *port;
 
 	list_for_each_entry(port, &pcie->ports, list) {
 		u32 slot = port->slot;
 
+		irqs[i++] = port->irq;
 		if (port->enabled)
 			pcie_link_status |= BIT(slot);
 	}
@@ -591,12 +622,16 @@ static int mt7621_pcie_init_virtual_bridges(struct mt7621_pcie *pcie)
 	if (pcie_link_status == 0)
 		return -1;
 
-	n = 0;
-	for (i = 0; i < PCIE_P2P_MAX; i++)
+	/*
+	 * Assign device numbers from zero to the enabled ports,
+	 * then assigning remaining device numbers to any disabled
+	 * ports.
+	 */
+	for (i = 0; i < PCIE_P2P_CNT; i++)
 		if (pcie_link_status & BIT(i))
 			p2p_br_devnum[i] = n++;
 
-	for (i = 0; i < PCIE_P2P_MAX; i++)
+	for (i = 0; i < PCIE_P2P_CNT; i++)
 		if ((pcie_link_status & BIT(i)) == 0)
 			p2p_br_devnum[i] = n++;
 
@@ -606,6 +641,15 @@ static int mt7621_pcie_init_virtual_bridges(struct mt7621_pcie *pcie)
 		 (p2p_br_devnum[1] << PCIE_P2P_BR_DEVNUM1_SHIFT) |
 		 (p2p_br_devnum[2] << PCIE_P2P_BR_DEVNUM2_SHIFT));
 
+	/* Assign IRQs */
+	n = 0;
+	for (i = 0; i < PCIE_P2P_CNT; i++)
+		if (pcie_link_status & BIT(i))
+			pcie->irq_map[n++] = irqs[i];
+
+	for (i = n; i < PCIE_P2P_CNT; i++)
+		pcie->irq_map[i] = -1;
+
 	return 0;
 }
 
@@ -630,7 +674,7 @@ static int mt7621_pcie_register_host(struct pci_host_bridge *host,
 	host->busnr = pcie->busn.start;
 	host->dev.parent = pcie->dev;
 	host->ops = &mt7621_pci_ops;
-	host->map_irq = of_irq_parse_and_map_pci;
+	host->map_irq = mt7621_map_irq;
 	host->swizzle_irq = pci_common_swizzle;
 	host->sysdata = pcie;
 
diff --git a/drivers/staging/mt7621-pinctrl/pinctrl-rt2880.c b/drivers/staging/mt7621-pinctrl/pinctrl-rt2880.c
index d0f06790d38f..caaf9e34f1ee 100644
--- a/drivers/staging/mt7621-pinctrl/pinctrl-rt2880.c
+++ b/drivers/staging/mt7621-pinctrl/pinctrl-rt2880.c
@@ -220,7 +220,7 @@ static int rt2880_pinmux_index(struct rt2880_priv *p)
 	/* allocate our function and group mapping index buffers */
 	f = p->func = devm_kcalloc(p->dev,
 				   p->func_count,
-				   sizeof(struct rt2880_pmx_func),
+				   sizeof(*p->func),
 				   GFP_KERNEL);
 	gpio_func.groups = devm_kcalloc(p->dev, p->group_count, sizeof(int),
 					GFP_KERNEL);
diff --git a/drivers/staging/pi433/pi433_if.c b/drivers/staging/pi433/pi433_if.c
index 313d22f6210f..c8d0c63fdd1d 100644
--- a/drivers/staging/pi433/pi433_if.c
+++ b/drivers/staging/pi433/pi433_if.c
@@ -1230,6 +1230,7 @@ static int pi433_probe(struct spi_device *spi)
 	device->cdev = cdev_alloc();
 	if (!device->cdev) {
 		dev_dbg(device->dev, "allocation of cdev failed");
+		retval = -ENOMEM;
 		goto cdev_failed;
 	}
 	device->cdev->owner = THIS_MODULE;
diff --git a/drivers/staging/qlge/qlge_dbg.c b/drivers/staging/qlge/qlge_dbg.c
index 1795533cbd3a..058889687907 100644
--- a/drivers/staging/qlge/qlge_dbg.c
+++ b/drivers/staging/qlge/qlge_dbg.c
@@ -1559,18 +1559,17 @@ void ql_dump_stat(struct ql_adapter *qdev)
 #ifdef QL_DEV_DUMP
 
 #define DUMP_QDEV_FIELD(qdev, type, field)		\
-	pr_err("qdev->%-24s = " type "\n", #field, (qdev)->(field))
+	pr_err("qdev->%-24s = " type "\n", #field, (qdev)->field)
 #define DUMP_QDEV_DMA_FIELD(qdev, field)		\
 	pr_err("qdev->%-24s = %llx\n", #field, (unsigned long long)qdev->field)
 #define DUMP_QDEV_ARRAY(qdev, type, array, index, field) \
 	pr_err("%s[%d].%s = " type "\n",		 \
-	       #array, index, #field, (qdev)->array[index].field);
+	       #array, index, #field, (qdev)->array[index].field)
 void ql_dump_qdev(struct ql_adapter *qdev)
 {
 	int i;
 
 	DUMP_QDEV_FIELD(qdev, "%lx", flags);
-	DUMP_QDEV_FIELD(qdev, "%p", vlgrp);
 	DUMP_QDEV_FIELD(qdev, "%p", pdev);
 	DUMP_QDEV_FIELD(qdev, "%p", ndev);
 	DUMP_QDEV_FIELD(qdev, "%d", chip_rev_id);
@@ -1758,8 +1757,6 @@ void ql_dump_rx_ring(struct rx_ring *rx_ring)
 	       rx_ring->lbq.prod_idx_db_reg);
 	pr_err("rx_ring->lbq.next_to_use = %d\n", rx_ring->lbq.next_to_use);
 	pr_err("rx_ring->lbq.next_to_clean = %d\n", rx_ring->lbq.next_to_clean);
-	pr_err("rx_ring->lbq_clean_idx = %d\n", rx_ring->lbq_clean_idx);
-	pr_err("rx_ring->lbq_free_cnt = %d\n", rx_ring->lbq_free_cnt);
 
 	pr_err("rx_ring->sbq.base = %p\n", rx_ring->sbq.base);
 	pr_err("rx_ring->sbq.base_dma = %llx\n",
diff --git a/drivers/staging/qlge/qlge_main.c b/drivers/staging/qlge/qlge_main.c
index c92820f07968..402edaeffe12 100644
--- a/drivers/staging/qlge/qlge_main.c
+++ b/drivers/staging/qlge/qlge_main.c
@@ -214,19 +214,20 @@ int ql_write_cfg(struct ql_adapter *qdev, void *ptr, int size, u32 bit,
 	u32 mask;
 	u32 value;
 
-	direction =
-	    (bit & (CFG_LRQ | CFG_LR | CFG_LCQ)) ? PCI_DMA_TODEVICE :
-	    PCI_DMA_FROMDEVICE;
+	if (bit & (CFG_LRQ | CFG_LR | CFG_LCQ))
+		direction = DMA_TO_DEVICE;
+	else
+		direction = DMA_FROM_DEVICE;
 
-	map = pci_map_single(qdev->pdev, ptr, size, direction);
-	if (pci_dma_mapping_error(qdev->pdev, map)) {
+	map = dma_map_single(&qdev->pdev->dev, ptr, size, direction);
+	if (dma_mapping_error(&qdev->pdev->dev, map)) {
 		netif_err(qdev, ifup, qdev->ndev, "Couldn't map DMA area.\n");
 		return -ENOMEM;
 	}
 
 	status = ql_sem_spinlock(qdev, SEM_ICB_MASK);
 	if (status)
-		return status;
+		goto lock_failed;
 
 	status = ql_wait_cfg(qdev, bit);
 	if (status) {
@@ -235,8 +236,8 @@ int ql_write_cfg(struct ql_adapter *qdev, void *ptr, int size, u32 bit,
 		goto exit;
 	}
 
-	ql_write32(qdev, ICB_L, (u32) map);
-	ql_write32(qdev, ICB_H, (u32) (map >> 32));
+	ql_write32(qdev, ICB_L, (u32)map);
+	ql_write32(qdev, ICB_H, (u32)(map >> 32));
 
 	mask = CFG_Q_MASK | (bit << 16);
 	value = bit | (q_id << CFG_Q_SHIFT);
@@ -248,7 +249,8 @@ int ql_write_cfg(struct ql_adapter *qdev, void *ptr, int size, u32 bit,
 	status = ql_wait_cfg(qdev, bit);
 exit:
 	ql_sem_unlock(qdev, SEM_ICB_MASK);	/* does flush too */
-	pci_unmap_single(qdev->pdev, map, size, direction);
+lock_failed:
+	dma_unmap_single(&qdev->pdev->dev, map, size, direction);
 	return status;
 }
 
@@ -261,52 +263,50 @@ int ql_get_mac_addr_reg(struct ql_adapter *qdev, u32 type, u16 index,
 
 	switch (type) {
 	case MAC_ADDR_TYPE_MULTI_MAC:
-	case MAC_ADDR_TYPE_CAM_MAC:
-		{
-			status =
-			    ql_wait_reg_rdy(qdev,
-					    MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
+	case MAC_ADDR_TYPE_CAM_MAC: {
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
+			break;
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   (offset++) | /* offset */
 				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
-				   MAC_ADDR_ADR | MAC_ADDR_RS | type); /* type */
-			status =
-			    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MR, 0);
-			if (status)
-				goto exit;
-			*value++ = ql_read32(qdev, MAC_ADDR_DATA);
-			status =
-			    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
+				   MAC_ADDR_ADR | MAC_ADDR_RS |
+				   type); /* type */
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MR, 0);
+		if (status)
+			break;
+		*value++ = ql_read32(qdev, MAC_ADDR_DATA);
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
+			break;
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   (offset++) | /* offset */
 				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
-				   MAC_ADDR_ADR | MAC_ADDR_RS | type); /* type */
-			status =
-			    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MR, 0);
+				   MAC_ADDR_ADR | MAC_ADDR_RS |
+				   type); /* type */
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MR, 0);
+		if (status)
+			break;
+		*value++ = ql_read32(qdev, MAC_ADDR_DATA);
+		if (type == MAC_ADDR_TYPE_CAM_MAC) {
+			status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX,
+						 MAC_ADDR_MW, 0);
 			if (status)
-				goto exit;
+				break;
+			ql_write32(qdev, MAC_ADDR_IDX,
+				   (offset++) | /* offset */
+					   (index
+					    << MAC_ADDR_IDX_SHIFT) | /* index */
+					   MAC_ADDR_ADR |
+					   MAC_ADDR_RS | type); /* type */
+			status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX,
+						 MAC_ADDR_MR, 0);
+			if (status)
+				break;
 			*value++ = ql_read32(qdev, MAC_ADDR_DATA);
-			if (type == MAC_ADDR_TYPE_CAM_MAC) {
-				status =
-				    ql_wait_reg_rdy(qdev,
-						    MAC_ADDR_IDX, MAC_ADDR_MW,
-						    0);
-				if (status)
-					goto exit;
-				ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
-					   (index << MAC_ADDR_IDX_SHIFT) | /* index */
-					   MAC_ADDR_ADR | MAC_ADDR_RS | type); /* type */
-				status =
-				    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX,
-						    MAC_ADDR_MR, 0);
-				if (status)
-					goto exit;
-				*value++ = ql_read32(qdev, MAC_ADDR_DATA);
-			}
-			break;
 		}
+		break;
+	}
 	case MAC_ADDR_TYPE_VLAN:
 	case MAC_ADDR_TYPE_MULTI_FLTR:
 	default:
@@ -314,7 +314,6 @@ int ql_get_mac_addr_reg(struct ql_adapter *qdev, u32 type, u16 index,
 			   "Address type %d not yet supported.\n", type);
 		status = -EPERM;
 	}
-exit:
 	return status;
 }
 
@@ -328,107 +327,93 @@ static int ql_set_mac_addr_reg(struct ql_adapter *qdev, u8 *addr, u32 type,
 	int status = 0;
 
 	switch (type) {
-	case MAC_ADDR_TYPE_MULTI_MAC:
-		{
-			u32 upper = (addr[0] << 8) | addr[1];
-			u32 lower = (addr[2] << 24) | (addr[3] << 16) |
-					(addr[4] << 8) | (addr[5]);
-
-			status =
-				ql_wait_reg_rdy(qdev,
-						MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, (offset++) |
-				(index << MAC_ADDR_IDX_SHIFT) |
-				type | MAC_ADDR_E);
-			ql_write32(qdev, MAC_ADDR_DATA, lower);
-			status =
-				ql_wait_reg_rdy(qdev,
-						MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, (offset++) |
-				(index << MAC_ADDR_IDX_SHIFT) |
-				type | MAC_ADDR_E);
+	case MAC_ADDR_TYPE_MULTI_MAC: {
+		u32 upper = (addr[0] << 8) | addr[1];
+		u32 lower = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
+			    (addr[5]);
 
-			ql_write32(qdev, MAC_ADDR_DATA, upper);
-			status =
-				ql_wait_reg_rdy(qdev,
-						MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
 			break;
-		}
-	case MAC_ADDR_TYPE_CAM_MAC:
-		{
-			u32 cam_output;
-			u32 upper = (addr[0] << 8) | addr[1];
-			u32 lower =
-			    (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   (offset++) | (index << MAC_ADDR_IDX_SHIFT) | type |
+				   MAC_ADDR_E);
+		ql_write32(qdev, MAC_ADDR_DATA, lower);
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
+			break;
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   (offset++) | (index << MAC_ADDR_IDX_SHIFT) | type |
+				   MAC_ADDR_E);
+
+		ql_write32(qdev, MAC_ADDR_DATA, upper);
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		break;
+	}
+	case MAC_ADDR_TYPE_CAM_MAC: {
+		u32 cam_output;
+		u32 upper = (addr[0] << 8) | addr[1];
+		u32 lower = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
 			    (addr[5]);
-			status =
-			    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
+			break;
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   (offset++) | /* offset */
 				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
-				   type);	/* type */
-			ql_write32(qdev, MAC_ADDR_DATA, lower);
-			status =
-			    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
+				   type); /* type */
+		ql_write32(qdev, MAC_ADDR_DATA, lower);
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
+			break;
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   (offset++) | /* offset */
 				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
-				   type);	/* type */
-			ql_write32(qdev, MAC_ADDR_DATA, upper);
-			status =
-			    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, (offset) |	/* offset */
-				   (index << MAC_ADDR_IDX_SHIFT) |	/* index */
-				   type);	/* type */
-			/* This field should also include the queue id
-			 * and possibly the function id.  Right now we hardcode
-			 * the route field to NIC core.
-			 */
-			cam_output = (CAM_OUT_ROUTE_NIC |
-				      (qdev->
-				       func << CAM_OUT_FUNC_SHIFT) |
-					(0 << CAM_OUT_CQ_ID_SHIFT));
-			if (qdev->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
-				cam_output |= CAM_OUT_RV;
-			/* route to NIC core */
-			ql_write32(qdev, MAC_ADDR_DATA, cam_output);
+				   type); /* type */
+		ql_write32(qdev, MAC_ADDR_DATA, upper);
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
 			break;
-		}
-	case MAC_ADDR_TYPE_VLAN:
-		{
-			u32 enable_bit = *((u32 *) &addr[0]);
-			/* For VLAN, the addr actually holds a bit that
-			 * either enables or disables the vlan id we are
-			 * addressing. It's either MAC_ADDR_E on or off.
-			 * That's bit-27 we're talking about.
-			 */
-			status =
-			    ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
-			if (status)
-				goto exit;
-			ql_write32(qdev, MAC_ADDR_IDX, offset |	/* offset */
-				   (index << MAC_ADDR_IDX_SHIFT) |	/* index */
-				   type |	/* type */
-				   enable_bit);	/* enable/disable */
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   (offset) | /* offset */
+				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
+				   type); /* type */
+		/* This field should also include the queue id
+		 * and possibly the function id.  Right now we hardcode
+		 * the route field to NIC core.
+		 */
+		cam_output = (CAM_OUT_ROUTE_NIC |
+			      (qdev->func << CAM_OUT_FUNC_SHIFT) |
+			      (0 << CAM_OUT_CQ_ID_SHIFT));
+		if (qdev->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
+			cam_output |= CAM_OUT_RV;
+		/* route to NIC core */
+		ql_write32(qdev, MAC_ADDR_DATA, cam_output);
+		break;
+	}
+	case MAC_ADDR_TYPE_VLAN: {
+		u32 enable_bit = *((u32 *)&addr[0]);
+		/* For VLAN, the addr actually holds a bit that
+		 * either enables or disables the vlan id we are
+		 * addressing. It's either MAC_ADDR_E on or off.
+		 * That's bit-27 we're talking about.
+		 */
+		status = ql_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
+		if (status)
 			break;
-		}
+		ql_write32(qdev, MAC_ADDR_IDX,
+			   offset | /* offset */
+				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
+				   type | /* type */
+				   enable_bit); /* enable/disable */
+		break;
+	}
 	case MAC_ADDR_TYPE_MULTI_FLTR:
 	default:
 		netif_crit(qdev, ifup, qdev->ndev,
 			   "Address type %d not yet supported.\n", type);
 		status = -EPERM;
 	}
-exit:
 	return status;
 }
 
@@ -455,7 +440,7 @@ static int ql_set_mac_addr(struct ql_adapter *qdev, int set)
 	status = ql_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
 	if (status)
 		return status;
-	status = ql_set_mac_addr_reg(qdev, (u8 *) addr,
+	status = ql_set_mac_addr_reg(qdev, (u8 *)addr,
 				     MAC_ADDR_TYPE_CAM_MAC,
 				     qdev->func * MAX_CQ);
 	ql_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
@@ -857,7 +842,7 @@ int ql_read_xgmac_reg64(struct ql_adapter *qdev, u32 reg, u64 *data)
 	if (status)
 		goto exit;
 
-	*data = (u64) lo | ((u64) hi << 32);
+	*data = (u64)lo | ((u64)hi << 32);
 
 exit:
 	return status;
@@ -983,14 +968,14 @@ static struct qlge_bq_desc *ql_get_curr_lchunk(struct ql_adapter *qdev,
 {
 	struct qlge_bq_desc *lbq_desc = qlge_get_curr_buf(&rx_ring->lbq);
 
-	pci_dma_sync_single_for_cpu(qdev->pdev, lbq_desc->dma_addr,
-				    qdev->lbq_buf_size, PCI_DMA_FROMDEVICE);
+	dma_sync_single_for_cpu(&qdev->pdev->dev, lbq_desc->dma_addr,
+				qdev->lbq_buf_size, DMA_FROM_DEVICE);
 
 	if ((lbq_desc->p.pg_chunk.offset + qdev->lbq_buf_size) ==
 	    ql_lbq_block_size(qdev)) {
 		/* last chunk of the master page */
-		pci_unmap_page(qdev->pdev, lbq_desc->dma_addr,
-			       ql_lbq_block_size(qdev), PCI_DMA_FROMDEVICE);
+		dma_unmap_page(&qdev->pdev->dev, lbq_desc->dma_addr,
+			       ql_lbq_block_size(qdev), DMA_FROM_DEVICE);
 	}
 
 	return lbq_desc;
@@ -1036,10 +1021,10 @@ static int qlge_refill_sb(struct rx_ring *rx_ring,
 		return -ENOMEM;
 	skb_reserve(skb, QLGE_SB_PAD);
 
-	sbq_desc->dma_addr = pci_map_single(qdev->pdev, skb->data,
+	sbq_desc->dma_addr = dma_map_single(&qdev->pdev->dev, skb->data,
 					    SMALL_BUF_MAP_SIZE,
-					    PCI_DMA_FROMDEVICE);
-	if (pci_dma_mapping_error(qdev->pdev, sbq_desc->dma_addr)) {
+					    DMA_FROM_DEVICE);
+	if (dma_mapping_error(&qdev->pdev->dev, sbq_desc->dma_addr)) {
 		netif_err(qdev, ifup, qdev->ndev, "PCI mapping failed.\n");
 		dev_kfree_skb_any(skb);
 		return -EIO;
@@ -1064,10 +1049,10 @@ static int qlge_refill_lb(struct rx_ring *rx_ring,
 		page = alloc_pages(gfp | __GFP_COMP, qdev->lbq_buf_order);
 		if (unlikely(!page))
 			return -ENOMEM;
-		dma_addr = pci_map_page(qdev->pdev, page, 0,
+		dma_addr = dma_map_page(&qdev->pdev->dev, page, 0,
 					ql_lbq_block_size(qdev),
-					PCI_DMA_FROMDEVICE);
-		if (pci_dma_mapping_error(qdev->pdev, dma_addr)) {
+					DMA_FROM_DEVICE);
+		if (dma_mapping_error(&qdev->pdev->dev, dma_addr)) {
 			__free_pages(page, qdev->lbq_buf_order);
 			netif_err(qdev, drv, qdev->ndev,
 				  "PCI mapping failed.\n");
@@ -1224,20 +1209,20 @@ static void ql_unmap_send(struct ql_adapter *qdev,
 					     qdev->ndev,
 					     "unmapping OAL area.\n");
 			}
-			pci_unmap_single(qdev->pdev,
+			dma_unmap_single(&qdev->pdev->dev,
 					 dma_unmap_addr(&tx_ring_desc->map[i],
 							mapaddr),
 					 dma_unmap_len(&tx_ring_desc->map[i],
 						       maplen),
-					 PCI_DMA_TODEVICE);
+					 DMA_TO_DEVICE);
 		} else {
 			netif_printk(qdev, tx_done, KERN_DEBUG, qdev->ndev,
 				     "unmapping frag %d.\n", i);
-			pci_unmap_page(qdev->pdev,
+			dma_unmap_page(&qdev->pdev->dev,
 				       dma_unmap_addr(&tx_ring_desc->map[i],
 						      mapaddr),
 				       dma_unmap_len(&tx_ring_desc->map[i],
-						     maplen), PCI_DMA_TODEVICE);
+						     maplen), DMA_TO_DEVICE);
 		}
 	}
 
@@ -1263,9 +1248,9 @@ static int ql_map_send(struct ql_adapter *qdev,
 	/*
 	 * Map the skb buffer first.
 	 */
-	map = pci_map_single(qdev->pdev, skb->data, len, PCI_DMA_TODEVICE);
+	map = dma_map_single(&qdev->pdev->dev, skb->data, len, DMA_TO_DEVICE);
 
-	err = pci_dma_mapping_error(qdev->pdev, map);
+	err = dma_mapping_error(&qdev->pdev->dev, map);
 	if (err) {
 		netif_err(qdev, tx_queued, qdev->ndev,
 			  "PCI mapping failed with error: %d\n", err);
@@ -1310,10 +1295,10 @@ static int ql_map_send(struct ql_adapter *qdev,
 			 *      etc...
 			 */
 			/* Tack on the OAL in the eighth segment of IOCB. */
-			map = pci_map_single(qdev->pdev, &tx_ring_desc->oal,
+			map = dma_map_single(&qdev->pdev->dev, &tx_ring_desc->oal,
 					     sizeof(struct oal),
-					     PCI_DMA_TODEVICE);
-			err = pci_dma_mapping_error(qdev->pdev, map);
+					     DMA_TO_DEVICE);
+			err = dma_mapping_error(&qdev->pdev->dev, map);
 			if (err) {
 				netif_err(qdev, tx_queued, qdev->ndev,
 					  "PCI mapping outbound address list with error: %d\n",
@@ -1584,8 +1569,8 @@ static void ql_process_mac_rx_skb(struct ql_adapter *qdev,
 	}
 	skb_reserve(new_skb, NET_IP_ALIGN);
 
-	pci_dma_sync_single_for_cpu(qdev->pdev, sbq_desc->dma_addr,
-				    SMALL_BUF_MAP_SIZE, PCI_DMA_FROMDEVICE);
+	dma_sync_single_for_cpu(&qdev->pdev->dev, sbq_desc->dma_addr,
+				SMALL_BUF_MAP_SIZE, DMA_FROM_DEVICE);
 
 	skb_put_data(new_skb, skb->data, length);
 
@@ -1647,7 +1632,7 @@ static void ql_process_mac_rx_skb(struct ql_adapter *qdev,
 		} else if ((ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_U) &&
 				(ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_V4)) {
 			/* Unfragmented ipv4 UDP frame. */
-			struct iphdr *iph = (struct iphdr *) skb->data;
+			struct iphdr *iph = (struct iphdr *)skb->data;
 
 			if (!(iph->frag_off &
 				htons(IP_MF|IP_OFFSET))) {
@@ -1707,8 +1692,8 @@ static struct sk_buff *ql_build_rx_skb(struct ql_adapter *qdev,
 		 * Headers fit nicely into a small buffer.
 		 */
 		sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
-		pci_unmap_single(qdev->pdev, sbq_desc->dma_addr,
-				 SMALL_BUF_MAP_SIZE, PCI_DMA_FROMDEVICE);
+		dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
+				 SMALL_BUF_MAP_SIZE, DMA_FROM_DEVICE);
 		skb = sbq_desc->p.skb;
 		ql_realign_skb(skb, hdr_len);
 		skb_put(skb, hdr_len);
@@ -1737,10 +1722,10 @@ static struct sk_buff *ql_build_rx_skb(struct ql_adapter *qdev,
 			 * buffer.
 			 */
 			sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
-			pci_dma_sync_single_for_cpu(qdev->pdev,
-						    sbq_desc->dma_addr,
-						    SMALL_BUF_MAP_SIZE,
-						    PCI_DMA_FROMDEVICE);
+			dma_sync_single_for_cpu(&qdev->pdev->dev,
+						sbq_desc->dma_addr,
+						SMALL_BUF_MAP_SIZE,
+						DMA_FROM_DEVICE);
 			skb_put_data(skb, sbq_desc->p.skb->data, length);
 		} else {
 			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
@@ -1750,9 +1735,9 @@ static struct sk_buff *ql_build_rx_skb(struct ql_adapter *qdev,
 			skb = sbq_desc->p.skb;
 			ql_realign_skb(skb, length);
 			skb_put(skb, length);
-			pci_unmap_single(qdev->pdev, sbq_desc->dma_addr,
+			dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
 					 SMALL_BUF_MAP_SIZE,
-					 PCI_DMA_FROMDEVICE);
+					 DMA_FROM_DEVICE);
 			sbq_desc->p.skb = NULL;
 		}
 	} else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) {
@@ -1787,9 +1772,9 @@ static struct sk_buff *ql_build_rx_skb(struct ql_adapter *qdev,
 					     "No skb available, drop the packet.\n");
 				return NULL;
 			}
-			pci_unmap_page(qdev->pdev, lbq_desc->dma_addr,
+			dma_unmap_page(&qdev->pdev->dev, lbq_desc->dma_addr,
 				       qdev->lbq_buf_size,
-				       PCI_DMA_FROMDEVICE);
+				       DMA_FROM_DEVICE);
 			skb_reserve(skb, NET_IP_ALIGN);
 			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
 				     "%d bytes of headers and data in large. Chain page to new skb and pull tail.\n",
@@ -1820,8 +1805,8 @@ static struct sk_buff *ql_build_rx_skb(struct ql_adapter *qdev,
 		int size, i = 0;
 
 		sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
-		pci_unmap_single(qdev->pdev, sbq_desc->dma_addr,
-				 SMALL_BUF_MAP_SIZE, PCI_DMA_FROMDEVICE);
+		dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
+				 SMALL_BUF_MAP_SIZE, DMA_FROM_DEVICE);
 		if (!(ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS)) {
 			/*
 			 * This is an non TCP/UDP IP frame, so
@@ -1936,7 +1921,7 @@ static void ql_process_mac_split_rx_intr(struct ql_adapter *qdev,
 		} else if ((ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_U) &&
 				(ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_V4)) {
 		/* Unfragmented ipv4 UDP frame. */
-			struct iphdr *iph = (struct iphdr *) skb->data;
+			struct iphdr *iph = (struct iphdr *)skb->data;
 
 			if (!(iph->frag_off &
 				htons(IP_MF|IP_OFFSET))) {
@@ -2317,7 +2302,7 @@ static int __qlge_vlan_rx_add_vid(struct ql_adapter *qdev, u16 vid)
 	u32 enable_bit = MAC_ADDR_E;
 	int err;
 
-	err = ql_set_mac_addr_reg(qdev, (u8 *) &enable_bit,
+	err = ql_set_mac_addr_reg(qdev, (u8 *)&enable_bit,
 				  MAC_ADDR_TYPE_VLAN, vid);
 	if (err)
 		netif_err(qdev, ifup, qdev->ndev,
@@ -2348,7 +2333,7 @@ static int __qlge_vlan_rx_kill_vid(struct ql_adapter *qdev, u16 vid)
 	u32 enable_bit = 0;
 	int err;
 
-	err = ql_set_mac_addr_reg(qdev, (u8 *) &enable_bit,
+	err = ql_set_mac_addr_reg(qdev, (u8 *)&enable_bit,
 				  MAC_ADDR_TYPE_VLAN, vid);
 	if (err)
 		netif_err(qdev, ifup, qdev->ndev,
@@ -2489,7 +2474,7 @@ static int ql_tso(struct sk_buff *skb, struct ob_mac_tso_iocb_req *mac_iocb_ptr)
 
 		mac_iocb_ptr->opcode = OPCODE_OB_MAC_TSO_IOCB;
 		mac_iocb_ptr->flags3 |= OB_MAC_TSO_IOCB_IC;
-		mac_iocb_ptr->frame_len = cpu_to_le32((u32) skb->len);
+		mac_iocb_ptr->frame_len = cpu_to_le32((u32)skb->len);
 		mac_iocb_ptr->total_hdrs_len =
 		    cpu_to_le16(skb_transport_offset(skb) + tcp_hdrlen(skb));
 		mac_iocb_ptr->net_trans_offset =
@@ -2527,7 +2512,7 @@ static void ql_hw_csum_setup(struct sk_buff *skb,
 	__sum16 *check;
 
 	mac_iocb_ptr->opcode = OPCODE_OB_MAC_TSO_IOCB;
-	mac_iocb_ptr->frame_len = cpu_to_le32((u32) skb->len);
+	mac_iocb_ptr->frame_len = cpu_to_le32((u32)skb->len);
 	mac_iocb_ptr->net_trans_offset =
 		cpu_to_le16(skb_network_offset(skb) |
 		skb_transport_offset(skb) << OB_MAC_TRANSPORT_HDR_SHIFT);
@@ -2558,7 +2543,7 @@ static netdev_tx_t qlge_send(struct sk_buff *skb, struct net_device *ndev)
 	struct ql_adapter *qdev = netdev_priv(ndev);
 	int tso;
 	struct tx_ring *tx_ring;
-	u32 tx_ring_idx = (u32) skb->queue_mapping;
+	u32 tx_ring_idx = (u32)skb->queue_mapping;
 
 	tx_ring = &qdev->tx_ring[tx_ring_idx];
 
@@ -2585,7 +2570,7 @@ static netdev_tx_t qlge_send(struct sk_buff *skb, struct net_device *ndev)
 	mac_iocb_ptr->txq_idx = tx_ring_idx;
 	tx_ring_desc->skb = skb;
 
-	mac_iocb_ptr->frame_len = cpu_to_le16((u16) skb->len);
+	mac_iocb_ptr->frame_len = cpu_to_le16((u16)skb->len);
 
 	if (skb_vlan_tag_present(skb)) {
 		netif_printk(qdev, tx_queued, KERN_DEBUG, qdev->ndev,
@@ -2636,17 +2621,17 @@ static netdev_tx_t qlge_send(struct sk_buff *skb, struct net_device *ndev)
 static void ql_free_shadow_space(struct ql_adapter *qdev)
 {
 	if (qdev->rx_ring_shadow_reg_area) {
-		pci_free_consistent(qdev->pdev,
-				    PAGE_SIZE,
-				    qdev->rx_ring_shadow_reg_area,
-				    qdev->rx_ring_shadow_reg_dma);
+		dma_free_coherent(&qdev->pdev->dev,
+				  PAGE_SIZE,
+				  qdev->rx_ring_shadow_reg_area,
+				  qdev->rx_ring_shadow_reg_dma);
 		qdev->rx_ring_shadow_reg_area = NULL;
 	}
 	if (qdev->tx_ring_shadow_reg_area) {
-		pci_free_consistent(qdev->pdev,
-				    PAGE_SIZE,
-				    qdev->tx_ring_shadow_reg_area,
-				    qdev->tx_ring_shadow_reg_dma);
+		dma_free_coherent(&qdev->pdev->dev,
+				  PAGE_SIZE,
+				  qdev->tx_ring_shadow_reg_area,
+				  qdev->tx_ring_shadow_reg_dma);
 		qdev->tx_ring_shadow_reg_area = NULL;
 	}
 }
@@ -2654,8 +2639,8 @@ static void ql_free_shadow_space(struct ql_adapter *qdev)
 static int ql_alloc_shadow_space(struct ql_adapter *qdev)
 {
 	qdev->rx_ring_shadow_reg_area =
-		pci_zalloc_consistent(qdev->pdev, PAGE_SIZE,
-				      &qdev->rx_ring_shadow_reg_dma);
+		dma_alloc_coherent(&qdev->pdev->dev, PAGE_SIZE,
+				   &qdev->rx_ring_shadow_reg_dma, GFP_ATOMIC);
 	if (!qdev->rx_ring_shadow_reg_area) {
 		netif_err(qdev, ifup, qdev->ndev,
 			  "Allocation of RX shadow space failed.\n");
@@ -2663,8 +2648,8 @@ static int ql_alloc_shadow_space(struct ql_adapter *qdev)
 	}
 
 	qdev->tx_ring_shadow_reg_area =
-		pci_zalloc_consistent(qdev->pdev, PAGE_SIZE,
-				      &qdev->tx_ring_shadow_reg_dma);
+		dma_alloc_coherent(&qdev->pdev->dev, PAGE_SIZE,
+				   &qdev->tx_ring_shadow_reg_dma, GFP_ATOMIC);
 	if (!qdev->tx_ring_shadow_reg_area) {
 		netif_err(qdev, ifup, qdev->ndev,
 			  "Allocation of TX shadow space failed.\n");
@@ -2673,10 +2658,10 @@ static int ql_alloc_shadow_space(struct ql_adapter *qdev)
 	return 0;
 
 err_wqp_sh_area:
-	pci_free_consistent(qdev->pdev,
-			    PAGE_SIZE,
-			    qdev->rx_ring_shadow_reg_area,
-			    qdev->rx_ring_shadow_reg_dma);
+	dma_free_coherent(&qdev->pdev->dev,
+			  PAGE_SIZE,
+			  qdev->rx_ring_shadow_reg_area,
+			  qdev->rx_ring_shadow_reg_dma);
 	return -ENOMEM;
 }
 
@@ -2702,8 +2687,8 @@ static void ql_free_tx_resources(struct ql_adapter *qdev,
 				 struct tx_ring *tx_ring)
 {
 	if (tx_ring->wq_base) {
-		pci_free_consistent(qdev->pdev, tx_ring->wq_size,
-				    tx_ring->wq_base, tx_ring->wq_base_dma);
+		dma_free_coherent(&qdev->pdev->dev, tx_ring->wq_size,
+				  tx_ring->wq_base, tx_ring->wq_base_dma);
 		tx_ring->wq_base = NULL;
 	}
 	kfree(tx_ring->q);
@@ -2714,8 +2699,8 @@ static int ql_alloc_tx_resources(struct ql_adapter *qdev,
 				 struct tx_ring *tx_ring)
 {
 	tx_ring->wq_base =
-	    pci_alloc_consistent(qdev->pdev, tx_ring->wq_size,
-				 &tx_ring->wq_base_dma);
+	    dma_alloc_coherent(&qdev->pdev->dev, tx_ring->wq_size,
+			       &tx_ring->wq_base_dma, GFP_ATOMIC);
 
 	if (!tx_ring->wq_base ||
 	    tx_ring->wq_base_dma & WQ_ADDR_ALIGN)
@@ -2729,8 +2714,8 @@ static int ql_alloc_tx_resources(struct ql_adapter *qdev,
 
 	return 0;
 err:
-	pci_free_consistent(qdev->pdev, tx_ring->wq_size,
-			    tx_ring->wq_base, tx_ring->wq_base_dma);
+	dma_free_coherent(&qdev->pdev->dev, tx_ring->wq_size,
+			  tx_ring->wq_base, tx_ring->wq_base_dma);
 	tx_ring->wq_base = NULL;
 pci_alloc_err:
 	netif_err(qdev, ifup, qdev->ndev, "tx_ring alloc failed.\n");
@@ -2748,17 +2733,17 @@ static void ql_free_lbq_buffers(struct ql_adapter *qdev, struct rx_ring *rx_ring
 			&lbq->queue[lbq->next_to_clean];
 
 		if (lbq_desc->p.pg_chunk.offset == last_offset)
-			pci_unmap_page(qdev->pdev, lbq_desc->dma_addr,
+			dma_unmap_page(&qdev->pdev->dev, lbq_desc->dma_addr,
 				       ql_lbq_block_size(qdev),
-				       PCI_DMA_FROMDEVICE);
+				       DMA_FROM_DEVICE);
 		put_page(lbq_desc->p.pg_chunk.page);
 
 		lbq->next_to_clean = QLGE_BQ_WRAP(lbq->next_to_clean + 1);
 	}
 
 	if (rx_ring->master_chunk.page) {
-		pci_unmap_page(qdev->pdev, rx_ring->chunk_dma_addr,
-			       ql_lbq_block_size(qdev), PCI_DMA_FROMDEVICE);
+		dma_unmap_page(&qdev->pdev->dev, rx_ring->chunk_dma_addr,
+			       ql_lbq_block_size(qdev), DMA_FROM_DEVICE);
 		put_page(rx_ring->master_chunk.page);
 		rx_ring->master_chunk.page = NULL;
 	}
@@ -2777,9 +2762,9 @@ static void ql_free_sbq_buffers(struct ql_adapter *qdev, struct rx_ring *rx_ring
 			return;
 		}
 		if (sbq_desc->p.skb) {
-			pci_unmap_single(qdev->pdev, sbq_desc->dma_addr,
+			dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
 					 SMALL_BUF_MAP_SIZE,
-					 PCI_DMA_FROMDEVICE);
+					 DMA_FROM_DEVICE);
 			dev_kfree_skb(sbq_desc->p.skb);
 			sbq_desc->p.skb = NULL;
 		}
@@ -2820,8 +2805,8 @@ static int qlge_init_bq(struct qlge_bq *bq)
 	__le64 *buf_ptr;
 	int i;
 
-	bq->base = pci_alloc_consistent(qdev->pdev, QLGE_BQ_SIZE,
-					&bq->base_dma);
+	bq->base = dma_alloc_coherent(&qdev->pdev->dev, QLGE_BQ_SIZE,
+				      &bq->base_dma, GFP_ATOMIC);
 	if (!bq->base) {
 		netif_err(qdev, ifup, qdev->ndev,
 			  "ring %u %s allocation failed.\n", rx_ring->cq_id,
@@ -2850,8 +2835,8 @@ static void ql_free_rx_resources(struct ql_adapter *qdev,
 {
 	/* Free the small buffer queue. */
 	if (rx_ring->sbq.base) {
-		pci_free_consistent(qdev->pdev, QLGE_BQ_SIZE,
-				    rx_ring->sbq.base, rx_ring->sbq.base_dma);
+		dma_free_coherent(&qdev->pdev->dev, QLGE_BQ_SIZE,
+				  rx_ring->sbq.base, rx_ring->sbq.base_dma);
 		rx_ring->sbq.base = NULL;
 	}
 
@@ -2861,8 +2846,8 @@ static void ql_free_rx_resources(struct ql_adapter *qdev,
 
 	/* Free the large buffer queue. */
 	if (rx_ring->lbq.base) {
-		pci_free_consistent(qdev->pdev, QLGE_BQ_SIZE,
-				    rx_ring->lbq.base, rx_ring->lbq.base_dma);
+		dma_free_coherent(&qdev->pdev->dev, QLGE_BQ_SIZE,
+				  rx_ring->lbq.base, rx_ring->lbq.base_dma);
 		rx_ring->lbq.base = NULL;
 	}
 
@@ -2872,9 +2857,9 @@ static void ql_free_rx_resources(struct ql_adapter *qdev,
 
 	/* Free the rx queue. */
 	if (rx_ring->cq_base) {
-		pci_free_consistent(qdev->pdev,
-				    rx_ring->cq_size,
-				    rx_ring->cq_base, rx_ring->cq_base_dma);
+		dma_free_coherent(&qdev->pdev->dev,
+				  rx_ring->cq_size,
+				  rx_ring->cq_base, rx_ring->cq_base_dma);
 		rx_ring->cq_base = NULL;
 	}
 }
@@ -2890,8 +2875,8 @@ static int ql_alloc_rx_resources(struct ql_adapter *qdev,
 	 * Allocate the completion queue for this rx_ring.
 	 */
 	rx_ring->cq_base =
-	    pci_alloc_consistent(qdev->pdev, rx_ring->cq_size,
-				 &rx_ring->cq_base_dma);
+	    dma_alloc_coherent(&qdev->pdev->dev, rx_ring->cq_size,
+			       &rx_ring->cq_base_dma, GFP_ATOMIC);
 
 	if (!rx_ring->cq_base) {
 		netif_err(qdev, ifup, qdev->ndev, "rx_ring alloc failed.\n");
@@ -3008,7 +2993,7 @@ static int ql_start_rx_ring(struct ql_adapter *qdev, struct rx_ring *rx_ring)
 	rx_ring->sbq.base_indirect_dma = shadow_reg_dma;
 
 	/* PCI doorbell mem area + 0x00 for consumer index register */
-	rx_ring->cnsmr_idx_db_reg = (u32 __iomem *) doorbell_area;
+	rx_ring->cnsmr_idx_db_reg = (u32 __iomem *)doorbell_area;
 	rx_ring->cnsmr_idx = 0;
 	rx_ring->curr_entry = rx_ring->cq_base;
 
@@ -3108,7 +3093,7 @@ static int ql_start_tx_ring(struct ql_adapter *qdev, struct tx_ring *tx_ring)
 	 * Assign doorbell registers for this tx_ring.
 	 */
 	/* TX PCI doorbell mem area for tx producer index */
-	tx_ring->prod_idx_db_reg = (u32 __iomem *) doorbell_area;
+	tx_ring->prod_idx_db_reg = (u32 __iomem *)doorbell_area;
 	tx_ring->prod_idx = 0;
 	/* TX PCI doorbell mem area + 0x04 */
 	tx_ring->valid_db_reg = doorbell_area + 0x04;
@@ -3131,7 +3116,7 @@ static int ql_start_tx_ring(struct ql_adapter *qdev, struct tx_ring *tx_ring)
 	ql_init_tx_ring(qdev, tx_ring);
 
 	err = ql_write_cfg(qdev, wqicb, sizeof(*wqicb), CFG_LRQ,
-			   (u16) tx_ring->wq_id);
+			   (u16)tx_ring->wq_id);
 	if (err) {
 		netif_err(qdev, ifup, qdev->ndev, "Failed to load tx_ring.\n");
 		return err;
@@ -3431,9 +3416,9 @@ static int ql_request_irq(struct ql_adapter *qdev)
 				     &qdev->rx_ring[0]);
 			status =
 			    request_irq(pdev->irq, qlge_isr,
-					test_bit(QL_MSI_ENABLED,
-						 &qdev->
-						 flags) ? 0 : IRQF_SHARED,
+					test_bit(QL_MSI_ENABLED, &qdev->flags)
+						? 0
+						: IRQF_SHARED,
 					intr_context->name, &qdev->rx_ring[0]);
 			if (status)
 				goto err_irq;
@@ -3463,7 +3448,7 @@ static int ql_start_rss(struct ql_adapter *qdev)
 	struct ricb *ricb = &qdev->ricb;
 	int status = 0;
 	int i;
-	u8 *hash_id = (u8 *) ricb->hash_cq_id;
+	u8 *hash_id = (u8 *)ricb->hash_cq_id;
 
 	memset((void *)ricb, 0, sizeof(*ricb));
 
@@ -4125,11 +4110,11 @@ static struct net_device_stats *qlge_get_stats(struct net_device
 	/* Get RX stats. */
 	pkts = mcast = dropped = errors = bytes = 0;
 	for (i = 0; i < qdev->rss_ring_count; i++, rx_ring++) {
-			pkts += rx_ring->rx_packets;
-			bytes += rx_ring->rx_bytes;
-			dropped += rx_ring->rx_dropped;
-			errors += rx_ring->rx_errors;
-			mcast += rx_ring->rx_multicast;
+		pkts += rx_ring->rx_packets;
+		bytes += rx_ring->rx_bytes;
+		dropped += rx_ring->rx_dropped;
+		errors += rx_ring->rx_errors;
+		mcast += rx_ring->rx_multicast;
 	}
 	ndev->stats.rx_packets = pkts;
 	ndev->stats.rx_bytes = bytes;
@@ -4140,9 +4125,9 @@ static struct net_device_stats *qlge_get_stats(struct net_device
 	/* Get TX stats. */
 	pkts = errors = bytes = 0;
 	for (i = 0; i < qdev->tx_ring_count; i++, tx_ring++) {
-			pkts += tx_ring->tx_packets;
-			bytes += tx_ring->tx_bytes;
-			errors += tx_ring->tx_errors;
+		pkts += tx_ring->tx_packets;
+		bytes += tx_ring->tx_bytes;
+		errors += tx_ring->tx_errors;
 	}
 	ndev->stats.tx_packets = pkts;
 	ndev->stats.tx_bytes = bytes;
@@ -4218,7 +4203,7 @@ static void qlge_set_multicast_list(struct net_device *ndev)
 			goto exit;
 		i = 0;
 		netdev_for_each_mc_addr(ha, ndev) {
-			if (ql_set_mac_addr_reg(qdev, (u8 *) ha->addr,
+			if (ql_set_mac_addr_reg(qdev, (u8 *)ha->addr,
 						MAC_ADDR_TYPE_MULTI_MAC, i)) {
 				netif_err(qdev, hw, qdev->ndev,
 					  "Failed to loadmulticast address.\n");
@@ -4255,7 +4240,7 @@ static int qlge_set_mac_address(struct net_device *ndev, void *p)
 	status = ql_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
 	if (status)
 		return status;
-	status = ql_set_mac_addr_reg(qdev, (u8 *) ndev->dev_addr,
+	status = ql_set_mac_addr_reg(qdev, (u8 *)ndev->dev_addr,
 				     MAC_ADDR_TYPE_CAM_MAC,
 				     qdev->func * MAX_CQ);
 	if (status)
@@ -4430,13 +4415,14 @@ static int ql_init_device(struct pci_dev *pdev, struct net_device *ndev,
 	}
 
 	pci_set_master(pdev);
-	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
+	if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
 		set_bit(QL_DMA64, &qdev->flags);
-		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+		err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
 	} else {
-		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
 		if (!err)
-		       err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+			err = dma_set_coherent_mask(&pdev->dev,
+						    DMA_BIT_MASK(32));
 	}
 
 	if (err) {
@@ -4448,8 +4434,7 @@ static int ql_init_device(struct pci_dev *pdev, struct net_device *ndev,
 	pdev->needs_freset = 1;
 	pci_save_state(pdev);
 	qdev->reg_base =
-	    ioremap(pci_resource_start(pdev, 1),
-			    pci_resource_len(pdev, 1));
+		ioremap(pci_resource_start(pdev, 1), pci_resource_len(pdev, 1));
 	if (!qdev->reg_base) {
 		dev_err(&pdev->dev, "Register mapping failed.\n");
 		err = -ENOMEM;
@@ -4458,8 +4443,7 @@ static int ql_init_device(struct pci_dev *pdev, struct net_device *ndev,
 
 	qdev->doorbell_area_size = pci_resource_len(pdev, 3);
 	qdev->doorbell_area =
-	    ioremap(pci_resource_start(pdev, 3),
-			    pci_resource_len(pdev, 3));
+		ioremap(pci_resource_start(pdev, 3), pci_resource_len(pdev, 3));
 	if (!qdev->doorbell_area) {
 		dev_err(&pdev->dev, "Doorbell register mapping failed.\n");
 		err = -ENOMEM;
diff --git a/drivers/staging/rtl8188eu/core/rtw_ap.c b/drivers/staging/rtl8188eu/core/rtw_ap.c
index 93283c7deec4..817793b9aff2 100644
--- a/drivers/staging/rtl8188eu/core/rtw_ap.c
+++ b/drivers/staging/rtl8188eu/core/rtw_ap.c
@@ -264,8 +264,10 @@ void expire_timeout_chk(struct adapter *padapter)
 			list_del_init(&psta->asoc_list);
 			pstapriv->asoc_list_cnt--;
 
-			DBG_88E("asoc expire %pM, state = 0x%x\n", (psta->hwaddr), psta->state);
-			updated = ap_free_sta(padapter, psta, true, WLAN_REASON_DEAUTH_LEAVING);
+			DBG_88E("asoc expire %pM, state = 0x%x\n",
+				(psta->hwaddr), psta->state);
+			updated = ap_free_sta(padapter, psta, true,
+					      WLAN_REASON_DEAUTH_LEAVING);
 		} else {
 			/* TODO: Aging mechanism to digest frames in sleep_q to avoid running out of xmitframe */
 			if (psta->sleepq_len > (NR_XMITFRAME / pstapriv->asoc_list_cnt) &&
@@ -294,16 +296,21 @@ void expire_timeout_chk(struct adapter *padapter)
 		for (i = 0; i < chk_alive_num; i++) {
 			int ret = _FAIL;
 
-			psta = rtw_get_stainfo_by_offset(pstapriv, chk_alive_list[i]);
+			psta = rtw_get_stainfo_by_offset(pstapriv,
+							 chk_alive_list[i]);
 
-			if (psta->state & WIFI_SLEEP_STATE)
-				ret = issue_nulldata(padapter, psta->hwaddr, 0, 1, 50);
-			else
-				ret = issue_nulldata(padapter, psta->hwaddr, 0, 3, 50);
+			if (psta->state & WIFI_SLEEP_STATE) {
+				ret = issue_nulldata(padapter, psta->hwaddr,
+						     0, 1, 50);
+			} else {
+				ret = issue_nulldata(padapter, psta->hwaddr,
+						     0, 3, 50);
+			}
 
 			psta->keep_alive_trycnt++;
 			if (ret == _SUCCESS) {
-				DBG_88E("asoc check, sta(%pM) is alive\n", (psta->hwaddr));
+				DBG_88E("asoc check, sta(%pM) is alive\n",
+					(psta->hwaddr));
 				psta->expire_to = pstapriv->expire_to;
 				psta->keep_alive_trycnt = 0;
 				continue;
@@ -315,11 +322,13 @@ void expire_timeout_chk(struct adapter *padapter)
 
 			psta->keep_alive_trycnt = 0;
 
-			DBG_88E("asoc expire %pM, state = 0x%x\n", (psta->hwaddr), psta->state);
+			DBG_88E("asoc expire %pM, state = 0x%x\n",
+				psta->hwaddr, psta->state);
 			spin_lock_bh(&pstapriv->asoc_list_lock);
 			list_del_init(&psta->asoc_list);
 			pstapriv->asoc_list_cnt--;
-			updated = ap_free_sta(padapter, psta, true, WLAN_REASON_DEAUTH_LEAVING);
+			updated = ap_free_sta(padapter, psta, true,
+					      WLAN_REASON_DEAUTH_LEAVING);
 			spin_unlock_bh(&pstapriv->asoc_list_lock);
 		}
 
@@ -431,7 +440,8 @@ static void update_bmc_sta(struct adapter *padapter)
 		supportRateNum = rtw_get_rateset_len((u8 *)&pcur_network->SupportedRates);
 		network_type = rtw_check_network_type((u8 *)&pcur_network->SupportedRates);
 
-		memcpy(psta->bssrateset, &pcur_network->SupportedRates, supportRateNum);
+		memcpy(psta->bssrateset, &pcur_network->SupportedRates,
+		       supportRateNum);
 		psta->bssratelen = supportRateNum;
 
 		/* b/g mode ra_bitmap */
@@ -445,7 +455,8 @@ static void update_bmc_sta(struct adapter *padapter)
 		tx_ra_bitmap = 0xf;
 
 		raid = networktype_to_raid(network_type);
-		init_rate = get_highest_rate_idx(tx_ra_bitmap & 0x0fffffff) & 0x3f;
+		init_rate = get_highest_rate_idx(tx_ra_bitmap & 0x0fffffff) &
+			    0x3f;
 
 		/* ap mode */
 		rtw_hal_set_odm_var(padapter, HAL_ODM_STA_INFO, psta, true);
@@ -456,7 +467,8 @@ static void update_bmc_sta(struct adapter *padapter)
 			arg = psta->mac_id & 0x1f;
 			arg |= BIT(7);
 			tx_ra_bitmap |= ((raid << 28) & 0xf0000000);
-			DBG_88E("%s, mask = 0x%x, arg = 0x%x\n", __func__, tx_ra_bitmap, arg);
+			DBG_88E("%s, mask = 0x%x, arg = 0x%x\n", __func__,
+				tx_ra_bitmap, arg);
 
 			/* bitmap[0:27] = tx_rate_bitmap */
 			/* bitmap[28:31]= Rate Adaptive id */
@@ -647,7 +659,8 @@ static void start_bss_network(struct adapter *padapter, u8 *pbuf)
 	rtw_hal_set_hwreg(padapter, HW_VAR_SEC_CFG, (u8 *)(&val8));
 
 	/* Beacon Control related register */
-	rtw_hal_set_hwreg(padapter, HW_VAR_BEACON_INTERVAL, (u8 *)(&bcn_interval));
+	rtw_hal_set_hwreg(padapter, HW_VAR_BEACON_INTERVAL,
+			  (u8 *)(&bcn_interval));
 
 	UpdateBrateTbl(padapter, pnetwork->SupportedRates);
 	rtw_hal_set_hwreg(padapter, HW_VAR_BASIC_RATE, pnetwork->SupportedRates);
@@ -657,7 +670,10 @@ static void start_bss_network(struct adapter *padapter, u8 *pbuf)
 		Switch_DM_Func(padapter, DYNAMIC_ALL_FUNC_ENABLE, true);
 	}
 	/* set channel, bwmode */
-	p = rtw_get_ie((pnetwork->ies + sizeof(struct ndis_802_11_fixed_ie)), _HT_ADD_INFO_IE_, &ie_len, (pnetwork->ie_length - sizeof(struct ndis_802_11_fixed_ie)));
+	p = rtw_get_ie(pnetwork->ies + sizeof(struct ndis_802_11_fixed_ie),
+		       _HT_ADD_INFO_IE_, &ie_len,
+		       pnetwork->ie_length -
+		       sizeof(struct ndis_802_11_fixed_ie));
 	if (p && ie_len) {
 		pht_info = (struct HT_info_element *)(p + 2);
 
@@ -682,7 +698,8 @@ static void start_bss_network(struct adapter *padapter, u8 *pbuf)
 	 */
 	set_channel_bwmode(padapter, cur_channel, cur_ch_offset, cur_bwmode);
 
-	DBG_88E("CH =%d, BW =%d, offset =%d\n", cur_channel, cur_bwmode, cur_ch_offset);
+	DBG_88E("CH =%d, BW =%d, offset =%d\n", cur_channel, cur_bwmode,
+		cur_ch_offset);
 
 	/*  */
 	pmlmeext->cur_channel = cur_channel;
@@ -771,17 +788,19 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	cap = get_unaligned_le16(ie);
 
 	/* SSID */
-	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _SSID_IE_, &ie_len, (pbss_network->ie_length - _BEACON_IE_OFFSET_));
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _SSID_IE_, &ie_len,
+		       pbss_network->ie_length - _BEACON_IE_OFFSET_);
 	if (p && ie_len > 0) {
 		memset(&pbss_network->ssid, 0, sizeof(struct ndis_802_11_ssid));
-		memcpy(pbss_network->ssid.ssid, (p + 2), ie_len);
+		memcpy(pbss_network->ssid.ssid, p + 2, ie_len);
 		pbss_network->ssid.ssid_length = ie_len;
 	}
 
 	/* channel */
 	channel = 0;
 	pbss_network->Configuration.Length = 0;
-	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _DSSET_IE_, &ie_len, (pbss_network->ie_length - _BEACON_IE_OFFSET_));
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _DSSET_IE_, &ie_len,
+		       (pbss_network->ie_length - _BEACON_IE_OFFSET_));
 	if (p && ie_len > 0)
 		channel = *(p + 2);
 
@@ -789,14 +808,16 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 
 	memset(supportRate, 0, NDIS_802_11_LENGTH_RATES_EX);
 	/*  get supported rates */
-	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _SUPPORTEDRATES_IE_, &ie_len, (pbss_network->ie_length - _BEACON_IE_OFFSET_));
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _SUPPORTEDRATES_IE_, &ie_len,
+		       pbss_network->ie_length - _BEACON_IE_OFFSET_);
 	if (p) {
 		memcpy(supportRate, p + 2, ie_len);
 		supportRateNum = ie_len;
 	}
 
 	/* get ext_supported rates */
-	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _EXT_SUPPORTEDRATES_IE_, &ie_len, pbss_network->ie_length - _BEACON_IE_OFFSET_);
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _EXT_SUPPORTEDRATES_IE_,
+		       &ie_len, pbss_network->ie_length - _BEACON_IE_OFFSET_);
 	if (p) {
 		memcpy(supportRate + supportRateNum, p + 2, ie_len);
 		supportRateNum += ie_len;
@@ -807,7 +828,8 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	rtw_set_supported_rate(pbss_network->SupportedRates, network_type);
 
 	/* parsing ERP_IE */
-	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _ERPINFO_IE_, &ie_len, (pbss_network->ie_length - _BEACON_IE_OFFSET_));
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _ERPINFO_IE_, &ie_len,
+		       pbss_network->ie_length - _BEACON_IE_OFFSET_);
 	if (p && ie_len > 0)
 		ERP_IE_handler(padapter, (struct ndis_802_11_var_ie *)p);
 
@@ -824,7 +846,8 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	pairwise_cipher = 0;
 	psecuritypriv->wpa2_group_cipher = _NO_PRIVACY_;
 	psecuritypriv->wpa2_pairwise_cipher = _NO_PRIVACY_;
-	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _RSN_IE_2_, &ie_len, (pbss_network->ie_length - _BEACON_IE_OFFSET_));
+	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _RSN_IE_2_, &ie_len,
+		       pbss_network->ie_length - _BEACON_IE_OFFSET_);
 	if (p && ie_len > 0) {
 		if (rtw_parse_wpa2_ie(p, ie_len + 2, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
 			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
@@ -844,7 +867,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	psecuritypriv->wpa_pairwise_cipher = _NO_PRIVACY_;
 	for (p = ie + _BEACON_IE_OFFSET_;; p += (ie_len + 2)) {
 		p = rtw_get_ie(p, _SSN_IE_1_, &ie_len,
-			       (pbss_network->ie_length - _BEACON_IE_OFFSET_ - (ie_len + 2)));
+			       pbss_network->ie_length - _BEACON_IE_OFFSET_ - (ie_len + 2));
 		if ((p) && (!memcmp(p + 2, OUI1, 4))) {
 			if (rtw_parse_wpa_ie(p, ie_len + 2, &group_cipher,
 					     &pairwise_cipher, NULL) == _SUCCESS) {
@@ -869,7 +892,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	if (pregistrypriv->wmm_enable) {
 		for (p = ie + _BEACON_IE_OFFSET_;; p += (ie_len + 2)) {
 			p = rtw_get_ie(p, _VENDOR_SPECIFIC_IE_, &ie_len,
-				       (pbss_network->ie_length - _BEACON_IE_OFFSET_ - (ie_len + 2)));
+				       pbss_network->ie_length - _BEACON_IE_OFFSET_ - (ie_len + 2));
 			if ((p) && !memcmp(p + 2, WMM_PARA_IE, 6)) {
 				pmlmepriv->qospriv.qos_option = 1;
 
@@ -892,7 +915,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	}
 	/* parsing HT_CAP_IE */
 	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_CAPABILITY_IE_, &ie_len,
-		       (pbss_network->ie_length - _BEACON_IE_OFFSET_));
+		       pbss_network->ie_length - _BEACON_IE_OFFSET_);
 	if (p && ie_len > 0) {
 		struct ieee80211_ht_cap *pht_cap = (struct ieee80211_ht_cap *)(p + 2);
 
@@ -916,7 +939,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 
 	/* parsing HT_INFO_IE */
 	p = rtw_get_ie(ie + _BEACON_IE_OFFSET_, _HT_ADD_INFO_IE_, &ie_len,
-		       (pbss_network->ie_length - _BEACON_IE_OFFSET_));
+		       pbss_network->ie_length - _BEACON_IE_OFFSET_);
 	if (p && ie_len > 0)
 		pHT_info_ie = p;
 	switch (network_type) {
@@ -1226,17 +1249,17 @@ void update_beacon(struct adapter *padapter, u8 ie_id, u8 *oui, u8 tx)
 }
 
 /*
-op_mode
-Set to 0 (HT pure) under the following conditions
-	- all STAs in the BSS are 20/40 MHz HT in 20/40 MHz BSS or
-	- all STAs in the BSS are 20 MHz HT in 20 MHz BSS
-Set to 1 (HT non-member protection) if there may be non-HT STAs
-	in both the primary and the secondary channel
-Set to 2 if only HT STAs are associated in BSS,
-	however and at least one 20 MHz HT STA is associated
-Set to 3 (HT mixed mode) when one or more non-HT STAs are associated
-	(currently non-GF HT station is considered as non-HT STA also)
-*/
+ * op_mode
+ * Set to 0 (HT pure) under the following conditions
+ *	- all STAs in the BSS are 20/40 MHz HT in 20/40 MHz BSS or
+ *	- all STAs in the BSS are 20 MHz HT in 20 MHz BSS
+ * Set to 1 (HT non-member protection) if there may be non-HT STAs
+ *	  in both the primary and the secondary channel
+ * Set to 2 if only HT STAs are associated in BSS,
+ *	  however and at least one 20 MHz HT STA is associated
+ * Set to 3 (HT mixed mode) when one or more non-HT STAs are associated
+ *	  (currently non-GF HT station is considered as non-HT STA also)
+ */
 static int rtw_ht_operation_update(struct adapter *padapter)
 {
 	u16 cur_op_mode, new_op_mode;
diff --git a/drivers/staging/rtl8188eu/core/rtw_efuse.c b/drivers/staging/rtl8188eu/core/rtw_efuse.c
index c525682d0edf..9bb3ec0cd62f 100644
--- a/drivers/staging/rtl8188eu/core/rtw_efuse.c
+++ b/drivers/staging/rtl8188eu/core/rtw_efuse.c
@@ -370,28 +370,27 @@ static u16 Efuse_GetCurrentSize(struct adapter *pAdapter)
 
 	while (efuse_OneByteRead(pAdapter, efuse_addr, &efuse_data) &&
 	       AVAILABLE_EFUSE_ADDR(efuse_addr)) {
-		if (efuse_data != 0xFF) {
-			if ((efuse_data & 0x1F) == 0x0F) {		/* extended header */
-				hoffset = efuse_data;
+		if (efuse_data == 0xFF)
+			break;
+		if ((efuse_data & 0x1F) == 0x0F) { /* extended header */
+			hoffset = efuse_data;
+			efuse_addr++;
+			efuse_OneByteRead(pAdapter, efuse_addr, &efuse_data);
+			if ((efuse_data & 0x0F) == 0x0F) {
 				efuse_addr++;
-				efuse_OneByteRead(pAdapter, efuse_addr, &efuse_data);
-				if ((efuse_data & 0x0F) == 0x0F) {
-					efuse_addr++;
-					continue;
-				} else {
-					hoffset = ((hoffset & 0xE0) >> 5) | ((efuse_data & 0xF0) >> 1);
-					hworden = efuse_data & 0x0F;
-				}
+				continue;
 			} else {
-				hoffset = (efuse_data >> 4) & 0x0F;
-				hworden =  efuse_data & 0x0F;
+				hoffset = ((hoffset & 0xE0) >> 5) |
+					  ((efuse_data & 0xF0) >> 1);
+				hworden = efuse_data & 0x0F;
 			}
-			word_cnts = Efuse_CalculateWordCnts(hworden);
-			/* read next header */
-			efuse_addr = efuse_addr + (word_cnts * 2) + 1;
 		} else {
-			break;
+			hoffset = (efuse_data >> 4) & 0x0F;
+			hworden =  efuse_data & 0x0F;
 		}
+		word_cnts = Efuse_CalculateWordCnts(hworden);
+		/* read next header */
+		efuse_addr = efuse_addr + (word_cnts * 2) + 1;
 	}
 
 	rtw_hal_set_hwreg(pAdapter, HW_VAR_EFUSE_BYTES, (u8 *)&efuse_addr);
diff --git a/drivers/staging/rtl8188eu/core/rtw_ieee80211.c b/drivers/staging/rtl8188eu/core/rtw_ieee80211.c
index e186982d5908..caf600eba03b 100644
--- a/drivers/staging/rtl8188eu/core/rtw_ieee80211.c
+++ b/drivers/staging/rtl8188eu/core/rtw_ieee80211.c
@@ -253,11 +253,11 @@ int rtw_generate_ie(struct registry_priv *pregistrypriv)
 	}
 
 	/* DS parameter set */
-	ie = rtw_set_ie(ie, _DSSET_IE_, 1, (u8 *)&(pdev_network->Configuration.DSConfig), &sz);
+	ie = rtw_set_ie(ie, _DSSET_IE_, 1, (u8 *)&pdev_network->Configuration.DSConfig, &sz);
 
 	/* IBSS Parameter Set */
 
-	ie = rtw_set_ie(ie, _IBSS_PARA_IE_, 2, (u8 *)&(pdev_network->Configuration.ATIMWindow), &sz);
+	ie = rtw_set_ie(ie, _IBSS_PARA_IE_, 2, (u8 *)&pdev_network->Configuration.ATIMWindow, &sz);
 
 	if (rateLen > 8)
 		ie = rtw_set_ie(ie, _EXT_SUPPORTEDRATES_IE_, (rateLen - 8), (pdev_network->SupportedRates + 8), &sz);
diff --git a/drivers/staging/rtl8188eu/core/rtw_led.c b/drivers/staging/rtl8188eu/core/rtw_led.c
index d1406cc99768..32dccae186ca 100644
--- a/drivers/staging/rtl8188eu/core/rtw_led.c
+++ b/drivers/staging/rtl8188eu/core/rtw_led.c
@@ -90,7 +90,6 @@ static void SwLedBlink1(struct LED_871x *pLed)
 {
 	struct adapter *padapter = pLed->padapter;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	u8 bStopBlinking = false;
 
 	/*  Change LED according to BlinkingLedState specified. */
 	if (pLed->BlinkingLedState == RTW_LED_ON) {
@@ -128,9 +127,7 @@ static void SwLedBlink1(struct LED_871x *pLed)
 		break;
 	case LED_BLINK_SCAN:
 		pLed->BlinkTimes--;
-		if (pLed->BlinkTimes == 0)
-			bStopBlinking = true;
-		if (bStopBlinking) {
+		if (pLed->BlinkTimes == 0) {
 			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
 				pLed->bLedLinkBlinkInProgress = true;
 				pLed->CurrLedState = LED_BLINK_NORMAL;
@@ -164,9 +161,7 @@ static void SwLedBlink1(struct LED_871x *pLed)
 		break;
 	case LED_BLINK_TXRX:
 		pLed->BlinkTimes--;
-		if (pLed->BlinkTimes == 0)
-			bStopBlinking = true;
-		if (bStopBlinking) {
+		if (pLed->BlinkTimes == 0) {
 			if (check_fwstate(pmlmepriv, _FW_LINKED)) {
 				pLed->bLedLinkBlinkInProgress = true;
 				pLed->CurrLedState = LED_BLINK_NORMAL;
@@ -188,7 +183,6 @@ static void SwLedBlink1(struct LED_871x *pLed)
 					  msecs_to_jiffies(LED_BLINK_NO_LINK_INTERVAL_ALPHA));
 				RT_TRACE(_module_rtl8712_led_c_, _drv_info_, ("CurrLedState %d\n", pLed->CurrLedState));
 			}
-			pLed->BlinkTimes = 0;
 			pLed->bLedBlinkInProgress = false;
 		} else {
 			if (pLed->bLedOn)
@@ -208,12 +202,7 @@ static void SwLedBlink1(struct LED_871x *pLed)
 			  msecs_to_jiffies(LED_BLINK_SCAN_INTERVAL_ALPHA));
 		break;
 	case LED_BLINK_WPS_STOP:	/* WPS success */
-		if (pLed->BlinkingLedState == RTW_LED_ON)
-			bStopBlinking = false;
-		else
-			bStopBlinking = true;
-
-		if (bStopBlinking) {
+		if (pLed->BlinkingLedState != RTW_LED_ON) {
 			pLed->bLedLinkBlinkInProgress = true;
 			pLed->CurrLedState = LED_BLINK_NORMAL;
 			if (pLed->bLedOn)
diff --git a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
index 04897cd48370..8d035f67ef61 100644
--- a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
@@ -1781,7 +1781,7 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter)
 
 			p = rtw_get_ie(pbss_network->ies + _FIXED_IE_LENGTH_, _HT_CAPABILITY_IE_, &len, pbss_network->ie_length - _FIXED_IE_LENGTH_);
 			if (!p || len == 0) { /* non-HT */
-				if ((pbss_network->Configuration.DSConfig <= 0) || (pbss_network->Configuration.DSConfig > 14))
+				if (pbss_network->Configuration.DSConfig <= 0)
 					continue;
 
 				ICS[0][pbss_network->Configuration.DSConfig] = 1;
@@ -1932,11 +1932,11 @@ static void site_survey(struct adapter *padapter)
 				if (pmlmeext->sitesurvey_res.ssid[i].ssid_length) {
 					/* todo: to issue two probe req??? */
 					issue_probereq(padapter,
-					&(pmlmeext->sitesurvey_res.ssid[i]),
+					&pmlmeext->sitesurvey_res.ssid[i],
 								NULL, false);
 					/* msleep(SURVEY_TO>>1); */
 					issue_probereq(padapter,
-					&(pmlmeext->sitesurvey_res.ssid[i]),
+					&pmlmeext->sitesurvey_res.ssid[i],
 								NULL, false);
 				}
 			}
diff --git a/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c b/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
index c4f58507dbfd..c000382c96d9 100644
--- a/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
+++ b/drivers/staging/rtl8188eu/core/rtw_pwrctrl.c
@@ -173,7 +173,7 @@ int ips_leave(struct adapter *padapter)
 
 		DBG_88E_LEVEL(_drv_info_, "nolinked power save leave\n");
 
-		if ((_WEP40_ == psecuritypriv->dot11PrivacyAlgrthm) || (_WEP104_ == psecuritypriv->dot11PrivacyAlgrthm)) {
+		if ((psecuritypriv->dot11PrivacyAlgrthm == _WEP40_) || (psecuritypriv->dot11PrivacyAlgrthm == _WEP104_)) {
 			DBG_88E("==>%s, channel(%d), processing(%x)\n", __func__, padapter->mlmeextpriv.cur_channel, pwrpriv->bips_processing);
 			set_channel_bwmode(padapter, padapter->mlmeextpriv.cur_channel, HAL_PRIME_CHNL_OFFSET_DONT_CARE, HT_CHANNEL_WIDTH_20);
 			for (keyid = 0; keyid < 4; keyid++) {
diff --git a/drivers/staging/rtl8188eu/core/rtw_recv.c b/drivers/staging/rtl8188eu/core/rtw_recv.c
index d4278361e002..a036ef104198 100644
--- a/drivers/staging/rtl8188eu/core/rtw_recv.c
+++ b/drivers/staging/rtl8188eu/core/rtw_recv.c
@@ -1525,21 +1525,14 @@ static int amsdu_to_msdu(struct adapter *padapter, struct recv_frame *prframe)
 
 		/* Allocate new skb for releasing to upper layer */
 		sub_skb = dev_alloc_skb(nSubframe_Length + 12);
-		if (sub_skb) {
-			skb_reserve(sub_skb, 12);
-			skb_put_data(sub_skb, pdata, nSubframe_Length);
-		} else {
-			sub_skb = skb_clone(prframe->pkt, GFP_ATOMIC);
-			if (sub_skb) {
-				sub_skb->data = pdata;
-				sub_skb->len = nSubframe_Length;
-				skb_set_tail_pointer(sub_skb, nSubframe_Length);
-			} else {
-				DBG_88E("skb_clone() Fail!!! , nr_subframes=%d\n", nr_subframes);
-				break;
-			}
+		if (!sub_skb) {
+			DBG_88E("dev_alloc_skb() Fail!!! , nr_subframes=%d\n", nr_subframes);
+			break;
 		}
 
+		skb_reserve(sub_skb, 12);
+		skb_put_data(sub_skb, pdata, nSubframe_Length);
+
 		subframes[nr_subframes++] = sub_skb;
 
 		if (nr_subframes >= MAX_SUBFRAME_COUNT) {
diff --git a/drivers/staging/rtl8188eu/hal/fw.c b/drivers/staging/rtl8188eu/hal/fw.c
index 486ee4bd4744..3d1d29e9f8e0 100644
--- a/drivers/staging/rtl8188eu/hal/fw.c
+++ b/drivers/staging/rtl8188eu/hal/fw.c
@@ -111,7 +111,7 @@ static int _rtl88e_fw_free_to_go(struct adapter *adapt)
 
 	do {
 		value32 = usb_read32(adapt, REG_MCUFWDL);
-		if (value32 & FWDL_ChkSum_rpt)
+		if (value32 & FWDL_CHKSUM_RPT)
 			break;
 	} while (counter++ < POLLING_READY_TIMEOUT_COUNT);
 
@@ -146,7 +146,7 @@ int rtl88eu_download_fw(struct adapter *adapt)
 	struct dvobj_priv *dvobj = adapter_to_dvobj(adapt);
 	struct device *device = dvobj_to_dev(dvobj);
 	const struct firmware *fw;
-	const char fw_name[] = "rtlwifi/rtl8188eufw.bin";
+	static const char fw_name[] = "rtlwifi/rtl8188eufw.bin";
 	struct rtl92c_firmware_header *pfwheader = NULL;
 	u8 *download_data, *fw_data;
 	size_t download_size;
@@ -192,7 +192,8 @@ int rtl88eu_download_fw(struct adapter *adapt)
 		rtl88e_firmware_selfreset(adapt);
 	}
 	_rtl88e_enable_fw_download(adapt, true);
-	usb_write8(adapt, REG_MCUFWDL, usb_read8(adapt, REG_MCUFWDL) | FWDL_ChkSum_rpt);
+	usb_write8(adapt, REG_MCUFWDL,
+		   usb_read8(adapt, REG_MCUFWDL) | FWDL_CHKSUM_RPT);
 	_rtl88e_write_fw(adapt, download_data, download_size);
 	_rtl88e_enable_fw_download(adapt, false);
 
diff --git a/drivers/staging/rtl8188eu/hal/odm.c b/drivers/staging/rtl8188eu/hal/odm.c
index 698377ea60ee..28974808839d 100644
--- a/drivers/staging/rtl8188eu/hal/odm.c
+++ b/drivers/staging/rtl8188eu/hal/odm.c
@@ -5,8 +5,6 @@
  *
  ******************************************************************************/
 
-/*  include files */
-
 #include "odm_precomp.h"
 #include "phy.h"
 
@@ -193,7 +191,7 @@ void ODM_DMWatchdog(struct odm_dm_struct *pDM_Odm)
 	odm_DIG(pDM_Odm);
 	odm_CCKPacketDetectionThresh(pDM_Odm);
 
-	if (*(pDM_Odm->pbPowerSaving))
+	if (*pDM_Odm->pbPowerSaving)
 		return;
 
 	odm_RefreshRateAdaptiveMask(pDM_Odm);
@@ -229,13 +227,13 @@ void odm_CommonInfoSelfUpdate(struct odm_dm_struct *pDM_Odm)
 	u8 i;
 	struct sta_info *pEntry;
 
-	if (*(pDM_Odm->pBandWidth) == ODM_BW40M) {
-		if (*(pDM_Odm->pSecChOffset) == 1)
-			pDM_Odm->ControlChannel = *(pDM_Odm->pChannel) - 2;
-		else if (*(pDM_Odm->pSecChOffset) == 2)
-			pDM_Odm->ControlChannel = *(pDM_Odm->pChannel) + 2;
+	if (*pDM_Odm->pBandWidth == ODM_BW40M) {
+		if (*pDM_Odm->pSecChOffset == 1)
+			pDM_Odm->ControlChannel = *pDM_Odm->pChannel - 2;
+		else if (*pDM_Odm->pSecChOffset == 2)
+			pDM_Odm->ControlChannel = *pDM_Odm->pChannel + 2;
 	} else {
-		pDM_Odm->ControlChannel = *(pDM_Odm->pChannel);
+		pDM_Odm->ControlChannel = *pDM_Odm->pChannel;
 	}
 
 	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++) {
@@ -270,16 +268,16 @@ void odm_CmnInfoInit_Debug(struct odm_dm_struct *pDM_Odm)
 void odm_CmnInfoHook_Debug(struct odm_dm_struct *pDM_Odm)
 {
 	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("odm_CmnInfoHook_Debug==>\n"));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pNumTxBytesUnicast=%llu\n", *(pDM_Odm->pNumTxBytesUnicast)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pNumRxBytesUnicast=%llu\n", *(pDM_Odm->pNumRxBytesUnicast)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pWirelessMode=0x%x\n", *(pDM_Odm->pWirelessMode)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pSecChOffset=%d\n", *(pDM_Odm->pSecChOffset)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pSecurity=%d\n", *(pDM_Odm->pSecurity)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pBandWidth=%d\n", *(pDM_Odm->pBandWidth)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pChannel=%d\n", *(pDM_Odm->pChannel)));
-
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pbScanInProcess=%d\n", *(pDM_Odm->pbScanInProcess)));
-	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pbPowerSaving=%d\n", *(pDM_Odm->pbPowerSaving)));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pNumTxBytesUnicast=%llu\n", *pDM_Odm->pNumTxBytesUnicast));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pNumRxBytesUnicast=%llu\n", *pDM_Odm->pNumRxBytesUnicast));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pWirelessMode=0x%x\n", *pDM_Odm->pWirelessMode));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pSecChOffset=%d\n", *pDM_Odm->pSecChOffset));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pSecurity=%d\n", *pDM_Odm->pSecurity));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pBandWidth=%d\n", *pDM_Odm->pBandWidth));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pChannel=%d\n", *pDM_Odm->pChannel));
+
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pbScanInProcess=%d\n", *pDM_Odm->pbScanInProcess));
+	ODM_RT_TRACE(pDM_Odm, ODM_COMP_COMMON, ODM_DBG_LOUD, ("pbPowerSaving=%d\n", *pDM_Odm->pbPowerSaving));
 }
 
 void odm_CmnInfoUpdate_Debug(struct odm_dm_struct *pDM_Odm)
@@ -348,7 +346,7 @@ void odm_DIG(struct odm_dm_struct *pDM_Odm)
 		return;
 	}
 
-	if (*(pDM_Odm->pbScanInProcess)) {
+	if (*pDM_Odm->pbScanInProcess) {
 		ODM_RT_TRACE(pDM_Odm, ODM_COMP_DIG, ODM_DBG_LOUD, ("odm_DIG() Return: In Scan Progress\n"));
 		return;
 	}
@@ -508,7 +506,7 @@ void odm_FalseAlarmCounterStatistics(struct odm_dm_struct *pDM_Odm)
 {
 	struct adapter *adapter = pDM_Odm->Adapter;
 	u32 ret_value;
-	struct false_alarm_stats *FalseAlmCnt = &(pDM_Odm->FalseAlmCnt);
+	struct false_alarm_stats *FalseAlmCnt = &pDM_Odm->FalseAlmCnt;
 
 	if (!(pDM_Odm->SupportAbility & ODM_BB_FA_CNT))
 		return;
@@ -581,7 +579,7 @@ void odm_FalseAlarmCounterStatistics(struct odm_dm_struct *pDM_Odm)
 void odm_CCKPacketDetectionThresh(struct odm_dm_struct *pDM_Odm)
 {
 	u8 CurCCK_CCAThres;
-	struct false_alarm_stats *FalseAlmCnt = &(pDM_Odm->FalseAlmCnt);
+	struct false_alarm_stats *FalseAlmCnt = &pDM_Odm->FalseAlmCnt;
 
 	if (!(pDM_Odm->SupportAbility & (ODM_BB_CCK_PD | ODM_BB_FA_CNT)))
 		return;
@@ -739,7 +737,7 @@ u32 ODM_Get_Rate_Bitmap(struct odm_dm_struct *pDM_Odm, u32 macid, u32 ra_mask, u
 		} else if (rssi_level == DM_RATR_STA_MIDDLE) {
 			rate_bitmap = 0x000ff000;
 		} else {
-			if (*(pDM_Odm->pBandWidth) == ODM_BW40M)
+			if (*pDM_Odm->pBandWidth == ODM_BW40M)
 				rate_bitmap = 0x000ff015;
 			else
 				rate_bitmap = 0x000ff005;
@@ -945,7 +943,7 @@ void odm_TXPowerTrackingInit(struct odm_dm_struct *pDM_Odm)
 {
 	pDM_Odm->RFCalibrateInfo.bTXPowerTracking = true;
 	pDM_Odm->RFCalibrateInfo.TXPowercount = 0;
-	if (*(pDM_Odm->mp_mode) != 1)
+	if (*pDM_Odm->mp_mode != 1)
 		pDM_Odm->RFCalibrateInfo.TxPowerTrackControl = true;
 	MSG_88E("pDM_Odm TxPowerTrackControl = %d\n", pDM_Odm->RFCalibrateInfo.TxPowerTrackControl);
 
@@ -1035,11 +1033,11 @@ void odm_EdcaTurboCheckCE(struct odm_dm_struct *pDM_Odm)
 	u64	cur_tx_bytes = 0;
 	u64	cur_rx_bytes = 0;
 	u8	bbtchange = false;
-	struct xmit_priv		*pxmitpriv = &(Adapter->xmitpriv);
-	struct recv_priv		*precvpriv = &(Adapter->recvpriv);
+	struct xmit_priv		*pxmitpriv = &Adapter->xmitpriv;
+	struct recv_priv		*precvpriv = &Adapter->recvpriv;
 	struct registry_priv	*pregpriv = &Adapter->registrypriv;
-	struct mlme_ext_priv	*pmlmeext = &(Adapter->mlmeextpriv);
-	struct mlme_ext_info	*pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_ext_priv	*pmlmeext = &Adapter->mlmeextpriv;
+	struct mlme_ext_info	*pmlmeinfo = &pmlmeext->mlmext_info;
 
 	if (pregpriv->wifi_spec == 1) /*  (pmlmeinfo->HT_enable == 0)) */
 		goto dm_CheckEdcaTurbo_EXIT;
diff --git a/drivers/staging/rtl8188eu/hal/odm_hwconfig.c b/drivers/staging/rtl8188eu/hal/odm_hwconfig.c
index a6f2731b076d..65a346ae3cb0 100644
--- a/drivers/staging/rtl8188eu/hal/odm_hwconfig.c
+++ b/drivers/staging/rtl8188eu/hal/odm_hwconfig.c
@@ -5,8 +5,6 @@
  *
  ******************************************************************************/
 
-/*  include files */
-
 #include "odm_precomp.h"
 
 #define READ_AND_CONFIG     READ_AND_CONFIG_MP
diff --git a/drivers/staging/rtl8188eu/hal/phy.c b/drivers/staging/rtl8188eu/hal/phy.c
index b9025815b682..920688fc9e9f 100644
--- a/drivers/staging/rtl8188eu/hal/phy.c
+++ b/drivers/staging/rtl8188eu/hal/phy.c
@@ -345,8 +345,8 @@ static void dm_txpwr_track_setpwr(struct odm_dm_struct *dm_odm)
 {
 	if (dm_odm->BbSwingFlagOfdm || dm_odm->BbSwingFlagCck) {
 		ODM_RT_TRACE(dm_odm, ODM_COMP_TX_PWR_TRACK, ODM_DBG_LOUD,
-			     ("dm_txpwr_track_setpwr CH=%d\n", *(dm_odm->pChannel)));
-		phy_set_tx_power_level(dm_odm->Adapter, *(dm_odm->pChannel));
+			     ("dm_txpwr_track_setpwr CH=%d\n", *dm_odm->pChannel));
+		phy_set_tx_power_level(dm_odm->Adapter, *dm_odm->pChannel);
 		dm_odm->BbSwingFlagOfdm = false;
 		dm_odm->BbSwingFlagCck = false;
 	}
@@ -786,7 +786,7 @@ static void pathb_fill_iqk(struct adapter *adapt, bool iqkok, s32 result[][8],
 	}
 }
 
-static void save_adda_registers(struct adapter *adapt, u32 *addareg,
+static void save_adda_registers(struct adapter *adapt, const u32 *addareg,
 				u32 *backup, u32 register_num)
 {
 	u32 i;
@@ -795,7 +795,7 @@ static void save_adda_registers(struct adapter *adapt, u32 *addareg,
 		backup[i] = phy_query_bb_reg(adapt, addareg[i], bMaskDWord);
 }
 
-static void save_mac_registers(struct adapter *adapt, u32 *mac_reg,
+static void save_mac_registers(struct adapter *adapt, const u32 *mac_reg,
 			       u32 *backup)
 {
 	u32 i;
@@ -806,7 +806,7 @@ static void save_mac_registers(struct adapter *adapt, u32 *mac_reg,
 	backup[i] = usb_read32(adapt, mac_reg[i]);
 }
 
-static void reload_adda_reg(struct adapter *adapt, u32 *adda_reg,
+static void reload_adda_reg(struct adapter *adapt, const u32 *adda_reg,
 			    u32 *backup, u32 regiester_num)
 {
 	u32 i;
@@ -815,8 +815,8 @@ static void reload_adda_reg(struct adapter *adapt, u32 *adda_reg,
 		phy_set_bb_reg(adapt, adda_reg[i], bMaskDWord, backup[i]);
 }
 
-static void reload_mac_registers(struct adapter *adapt,
-				 u32 *mac_reg, u32 *backup)
+static void reload_mac_registers(struct adapter *adapt, const u32 *mac_reg,
+				 u32 *backup)
 {
 	u32 i;
 
@@ -826,7 +826,7 @@ static void reload_mac_registers(struct adapter *adapt,
 	usb_write32(adapt, mac_reg[i], backup[i]);
 }
 
-static void path_adda_on(struct adapter *adapt, u32 *adda_reg,
+static void path_adda_on(struct adapter *adapt, const u32 *adda_reg,
 			 bool is_path_a_on, bool is2t)
 {
 	u32 path_on;
@@ -844,7 +844,8 @@ static void path_adda_on(struct adapter *adapt, u32 *adda_reg,
 		phy_set_bb_reg(adapt, adda_reg[i], bMaskDWord, path_on);
 }
 
-static void mac_setting_calibration(struct adapter *adapt, u32 *mac_reg, u32 *backup)
+static void mac_setting_calibration(struct adapter *adapt, const u32 *mac_reg,
+				    u32 *backup)
 {
 	u32 i = 0;
 
@@ -952,30 +953,31 @@ static void phy_iq_calibrate(struct adapter *adapt, s32 result[][8],
 	struct odm_dm_struct *dm_odm = &adapt->HalData->odmpriv;
 	u32 i;
 	u8 path_a_ok, path_b_ok;
-	u32 adda_reg[IQK_ADDA_REG_NUM] = {
-					  rFPGA0_XCD_SwitchControl, rBlue_Tooth,
-					  rRx_Wait_CCA, rTx_CCK_RFON,
-					  rTx_CCK_BBON, rTx_OFDM_RFON,
-					  rTx_OFDM_BBON, rTx_To_Rx,
-					  rTx_To_Tx, rRx_CCK,
-					  rRx_OFDM, rRx_Wait_RIFS,
-					  rRx_TO_Rx, rStandby,
-					  rSleep, rPMPD_ANAEN};
-
-	u32 iqk_mac_reg[IQK_MAC_REG_NUM] = {
-					    REG_TXPAUSE, REG_BCN_CTRL,
-					    REG_BCN_CTRL_1, REG_GPIO_MUXCFG};
-
+	static const u32 adda_reg[IQK_ADDA_REG_NUM] = {
+		rFPGA0_XCD_SwitchControl, rBlue_Tooth,
+		rRx_Wait_CCA, rTx_CCK_RFON,
+		rTx_CCK_BBON, rTx_OFDM_RFON,
+		rTx_OFDM_BBON, rTx_To_Rx,
+		rTx_To_Tx, rRx_CCK,
+		rRx_OFDM, rRx_Wait_RIFS,
+		rRx_TO_Rx, rStandby,
+		rSleep, rPMPD_ANAEN
+	};
+	static const u32 iqk_mac_reg[IQK_MAC_REG_NUM] = {
+		REG_TXPAUSE, REG_BCN_CTRL,
+		REG_BCN_CTRL_1, REG_GPIO_MUXCFG
+	};
 	/* since 92C & 92D have the different define in IQK_BB_REG */
-	u32 iqk_bb_reg_92c[IQK_BB_REG_NUM] = {
-					      rOFDM0_TRxPathEnable, rOFDM0_TRMuxPar,
-					      rFPGA0_XCD_RFInterfaceSW, rConfig_AntA, rConfig_AntB,
-					      rFPGA0_XAB_RFInterfaceSW, rFPGA0_XA_RFInterfaceOE,
-					      rFPGA0_XB_RFInterfaceOE, rFPGA0_RFMOD};
+	static const u32 iqk_bb_reg_92c[IQK_BB_REG_NUM] = {
+		rOFDM0_TRxPathEnable, rOFDM0_TRMuxPar,
+		rFPGA0_XCD_RFInterfaceSW, rConfig_AntA, rConfig_AntB,
+		rFPGA0_XAB_RFInterfaceSW, rFPGA0_XA_RFInterfaceOE,
+		rFPGA0_XB_RFInterfaceOE, rFPGA0_RFMOD
+	};
 
 	u32 retry_count = 9;
 
-	if (*(dm_odm->mp_mode) == 1)
+	if (*dm_odm->mp_mode == 1)
 		retry_count = 9;
 	else
 		retry_count = 2;
@@ -1320,7 +1322,7 @@ void rtl88eu_phy_lc_calibrate(struct adapter *adapt)
 	if (singletone || carrier_sup)
 		return;
 
-	while (*(dm_odm->pbScanInProcess) && timecount < timeout) {
+	while (*dm_odm->pbScanInProcess && timecount < timeout) {
 		mdelay(50);
 		timecount += 50;
 	}
diff --git a/drivers/staging/rtl8188eu/hal/rf.c b/drivers/staging/rtl8188eu/hal/rf.c
index 00a9f692bb06..6702f263c770 100644
--- a/drivers/staging/rtl8188eu/hal/rf.c
+++ b/drivers/staging/rtl8188eu/hal/rf.c
@@ -79,7 +79,7 @@ void rtl88eu_phy_rf6052_set_cck_txpower(struct adapter *adapt, u8 *powerlevel)
 		}
 	}
 	for (idx1 = RF_PATH_A; idx1 <= RF_PATH_B; idx1++) {
-		ptr = (u8 *)(&(tx_agc[idx1]));
+		ptr = (u8 *)(&tx_agc[idx1]);
 		for (idx2 = 0; idx2 < 4; idx2++) {
 			if (*ptr > RF6052_MAX_TX_PWR)
 				*ptr = RF6052_MAX_TX_PWR;
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c b/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c
index 371e746915dd..176716d3e903 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188e_cmd.c
@@ -256,7 +256,7 @@ static void ConstructBeacon(struct adapter *adapt, u8 *pframe, u32 *pLength)
 	pframe = rtw_set_ie(pframe, _SUPPORTEDRATES_IE_, min_t(u32, rate_len, 8), cur_network->SupportedRates, &pktlen);
 
 	/*  DS parameter set */
-	pframe = rtw_set_ie(pframe, _DSSET_IE_, 1, (unsigned char *)&(cur_network->Configuration.DSConfig), &pktlen);
+	pframe = rtw_set_ie(pframe, _DSSET_IE_, 1, (unsigned char *)&cur_network->Configuration.DSConfig, &pktlen);
 
 	if ((pmlmeinfo->state & 0x03) == WIFI_FW_ADHOC_STATE) {
 		u32 ATIMWindow;
diff --git a/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c b/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c
index 241f55b92808..1af919ff6d93 100644
--- a/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c
+++ b/drivers/staging/rtl8188eu/hal/rtl8188e_dm.c
@@ -29,9 +29,6 @@ static void dm_InitGPIOSetting(struct adapter *Adapter)
 	usb_write8(Adapter, REG_GPIO_MUXCFG, tmp1byte);
 }
 
-/*  */
-/*  functions */
-/*  */
 static void Init_ODM_ComInfo_88E(struct adapter *Adapter)
 {
 	struct hal_data_8188e *hal_data = Adapter->HalData;
diff --git a/drivers/staging/rtl8188eu/include/osdep_service.h b/drivers/staging/rtl8188eu/include/osdep_service.h
index c0114ad79788..0d3e4a6e7e85 100644
--- a/drivers/staging/rtl8188eu/include/osdep_service.h
+++ b/drivers/staging/rtl8188eu/include/osdep_service.h
@@ -50,7 +50,7 @@ struct	__queue	{
 
 static inline struct list_head *get_list_head(struct __queue *queue)
 {
-	return &(queue->queue);
+	return &queue->queue;
 }
 
 static inline int rtw_netif_queue_stopped(struct net_device *pnetdev)
diff --git a/drivers/staging/rtl8188eu/include/rtl8188e_spec.h b/drivers/staging/rtl8188eu/include/rtl8188e_spec.h
index dd943c831d91..be30c9434a29 100644
--- a/drivers/staging/rtl8188eu/include/rtl8188e_spec.h
+++ b/drivers/staging/rtl8188eu/include/rtl8188e_spec.h
@@ -817,7 +817,7 @@ So the following defines for 92C is not entire!!!!!!
 /* 2 MCUFWDL */
 #define MCUFWDL_EN			BIT(0)
 #define MCUFWDL_RDY			BIT(1)
-#define FWDL_ChkSum_rpt			BIT(2)
+#define FWDL_CHKSUM_RPT			BIT(2)
 #define MACINI_RDY			BIT(3)
 #define BBINI_RDY			BIT(4)
 #define RFINI_RDY			BIT(5)
diff --git a/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c b/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
index 9a89791720e0..d5968ef9f43d 100644
--- a/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
@@ -93,7 +93,7 @@ static char *translate_scan(struct adapter *padapter,
 			    struct wlan_network *pnetwork,
 			    char *start, char *stop)
 {
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct iw_event iwe;
 	u16 cap;
 	__le16 le_tmp;
@@ -417,7 +417,7 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 				ret = -EOPNOTSUPP;
 				goto exit;
 			}
-			memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), pwep->KeyMaterial, pwep->KeyLength);
+			memcpy(&psecuritypriv->dot11DefKey[wep_key_idx].skey[0], pwep->KeyMaterial, pwep->KeyLength);
 			psecuritypriv->dot11DefKeylen[wep_key_idx] = pwep->KeyLength;
 			rtw_set_key(padapter, psecuritypriv, wep_key_idx, 0);
 		}
@@ -444,8 +444,8 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 					memcpy(psta->dot118021x_UncstKey.skey,  param->u.crypt.key, min_t(u16, param->u.crypt.key_len, 16));
 
 					if (strcmp(param->u.crypt.alg, "TKIP") == 0) { /* set mic key */
-						memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
-						memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+						memcpy(psta->dot11tkiptxmickey.skey, &param->u.crypt.key[16], 8);
+						memcpy(psta->dot11tkiprxmickey.skey, &param->u.crypt.key[24], 8);
 						padapter->securitypriv.busetkipkey = false;
 					}
 
@@ -454,8 +454,8 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 					rtw_setstakey_cmd(padapter, (unsigned char *)psta, true);
 				} else { /* group key */
 					memcpy(padapter->securitypriv.dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, min_t(u16, param->u.crypt.key_len, 16 ));
-					memcpy(padapter->securitypriv.dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
-					memcpy(padapter->securitypriv.dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+					memcpy(padapter->securitypriv.dot118021XGrptxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[16], 8);
+					memcpy(padapter->securitypriv.dot118021XGrprxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[24], 8);
 					padapter->securitypriv.binstallGrpkey = true;
 					DBG_88E(" ~~~~set sta key:groupkey\n");
 
@@ -620,7 +620,7 @@ static int rtw_wx_get_name(struct net_device *dev,
 	u32 ht_ielen = 0;
 	char *p;
 	u8 ht_cap = false;
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct wlan_bssid_ex  *pcur_bss = &pmlmepriv->cur_network.network;
 	NDIS_802_11_RATES_EX *prates = NULL;
 
@@ -669,7 +669,7 @@ static int rtw_wx_get_freq(struct net_device *dev,
 			     union iwreq_data *wrqu, char *extra)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct wlan_bssid_ex  *pcur_bss = &pmlmepriv->cur_network.network;
 
 	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
@@ -738,7 +738,7 @@ static int rtw_wx_get_mode(struct net_device *dev, struct iw_request_info *a,
 			     union iwreq_data *wrqu, char *b)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 
 	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_, (" rtw_wx_get_mode\n"));
 
@@ -938,10 +938,10 @@ static int rtw_wx_set_wap(struct net_device *dev,
 	uint ret = 0;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
 	struct sockaddr *temp = (struct sockaddr *)awrq;
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct list_head *phead;
 	u8 *dst_bssid, *src_bssid;
-	struct __queue *queue	= &(pmlmepriv->scanned_queue);
+	struct __queue *queue	= &pmlmepriv->scanned_queue;
 	struct	wlan_network	*pnetwork = NULL;
 	enum ndis_802_11_auth_mode	authmode;
 
@@ -1002,7 +1002,7 @@ static int rtw_wx_get_wap(struct net_device *dev,
 			    union iwreq_data *wrqu, char *extra)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct wlan_bssid_ex  *pcur_bss = &pmlmepriv->cur_network.network;
 
 	wrqu->ap_addr.sa_family = ARPHRD_ETHER;
@@ -1188,8 +1188,8 @@ static int rtw_wx_get_scan(struct net_device *dev, struct iw_request_info *a,
 {
 	struct list_head *plist, *phead;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
-	struct __queue *queue	= &(pmlmepriv->scanned_queue);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
+	struct __queue *queue	= &pmlmepriv->scanned_queue;
 	struct	wlan_network	*pnetwork = NULL;
 	char *ev = extra;
 	char *stop = ev + wrqu->data.length;
@@ -1217,7 +1217,7 @@ static int rtw_wx_get_scan(struct net_device *dev, struct iw_request_info *a,
 			break;
 	}
 
-	spin_lock_bh(&(pmlmepriv->scanned_queue.lock));
+	spin_lock_bh(&pmlmepriv->scanned_queue.lock);
 
 	phead = get_list_head(queue);
 	plist = phead->next;
@@ -1358,7 +1358,7 @@ static int rtw_wx_get_essid(struct net_device *dev,
 {
 	u32 len;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct wlan_bssid_ex  *pcur_bss = &pmlmepriv->cur_network.network;
 
 	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_, ("rtw_wx_get_essid\n"));
@@ -1564,7 +1564,7 @@ static int rtw_wx_set_enc(struct net_device *dev,
 	struct ndis_802_11_wep	 wep;
 	enum ndis_802_11_auth_mode authmode;
 
-	struct iw_point *erq = &(wrqu->encoding);
+	struct iw_point *erq = &wrqu->encoding;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
 	struct pwrctrl_priv *pwrpriv = &padapter->pwrctrlpriv;
 
@@ -1675,8 +1675,8 @@ static int rtw_wx_get_enc(struct net_device *dev,
 {
 	uint key;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct iw_point *erq = &(wrqu->encoding);
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct iw_point *erq = &wrqu->encoding;
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 
 	if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
 		if (!check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
@@ -1759,7 +1759,7 @@ static int rtw_wx_set_auth(struct net_device *dev,
 			     union iwreq_data *wrqu, char *extra)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct iw_param *param = (struct iw_param *)&(wrqu->param);
+	struct iw_param *param = (struct iw_param *)&wrqu->param;
 	int ret = 0;
 
 	switch (param->flags & IW_AUTH_INDEX) {
@@ -2012,14 +2012,9 @@ static int wpa_supplicant_ioctl(struct net_device *dev, struct iw_point *p)
 	if (!p->pointer || p->length != sizeof(struct ieee_param))
 		return -EINVAL;
 
-	param = (struct ieee_param *)rtw_malloc(p->length);
-	if (!param)
-		return -ENOMEM;
-
-	if (copy_from_user(param, p->pointer, p->length)) {
-		kfree(param);
-		return -EFAULT;
-	}
+	param = memdup_user(p->pointer, p->length);
+	if (IS_ERR(param))
+		return PTR_ERR(param);
 
 	switch (param->cmd) {
 	case IEEE_CMD_SET_WPA_PARAM:
@@ -2093,7 +2088,7 @@ static int set_group_key(struct adapter *padapter, u8 *key, u8 alg, int keyid)
 	u8 keylen;
 	struct cmd_obj *pcmd;
 	struct setkey_parm *psetkeyparm;
-	struct cmd_priv	*pcmdpriv = &(padapter->cmdpriv);
+	struct cmd_priv	*pcmdpriv = &padapter->cmdpriv;
 	int res = _SUCCESS;
 
 	DBG_88E("%s\n", __func__);
@@ -2130,7 +2125,7 @@ static int set_group_key(struct adapter *padapter, u8 *key, u8 alg, int keyid)
 		keylen = 16;
 	}
 
-	memcpy(&(psetkeyparm->key[0]), key, keylen);
+	memcpy(&psetkeyparm->key[0], key, keylen);
 
 	pcmd->cmdcode = _SetKey_CMD_;
 	pcmd->parmbuf = (u8 *)psetkeyparm;
@@ -2173,7 +2168,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 	struct sta_info *psta = NULL, *pbcmc_sta = NULL;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
 	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv = &(padapter->securitypriv);
+	struct security_priv *psecuritypriv = &padapter->securitypriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 
 	DBG_88E("%s\n", __func__);
@@ -2245,7 +2240,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 			psecuritypriv->dot11PrivacyKeyIndex = wep_key_idx;
 
-			memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), pwep->KeyMaterial, pwep->KeyLength);
+			memcpy(&psecuritypriv->dot11DefKey[wep_key_idx].skey[0], pwep->KeyMaterial, pwep->KeyLength);
 
 			psecuritypriv->dot11DefKeylen[wep_key_idx] = pwep->KeyLength;
 
@@ -2256,7 +2251,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 			/* don't update "psecuritypriv->dot11PrivacyAlgrthm" and */
 			/* psecuritypriv->dot11PrivacyKeyIndex = keyid", but can rtw_set_key to cam */
 
-			memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), pwep->KeyMaterial, pwep->KeyLength);
+			memcpy(&psecuritypriv->dot11DefKey[wep_key_idx].skey[0], pwep->KeyMaterial, pwep->KeyLength);
 
 			psecuritypriv->dot11DefKeylen[wep_key_idx] = pwep->KeyLength;
 
@@ -2283,8 +2278,8 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 				memcpy(psecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,
 					    param->u.crypt.key, min_t(u16, param->u.crypt.key_len, 16));
 				/* set mic key */
-				memcpy(psecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
-				memcpy(psecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+				memcpy(psecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[16], 8);
+				memcpy(psecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[24], 8);
 
 				psecuritypriv->busetkipkey = true;
 			} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
@@ -2326,8 +2321,8 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 					psta->dot118021XPrivacy = _TKIP_;
 
 					/* set mic key */
-					memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
-					memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+					memcpy(psta->dot11tkiptxmickey.skey, &param->u.crypt.key[16], 8);
+					memcpy(psta->dot11tkiprxmickey.skey, &param->u.crypt.key[24], 8);
 
 					psecuritypriv->busetkipkey = true;
 				} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
@@ -2357,8 +2352,8 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 						    param->u.crypt.key, min_t(u16, param->u.crypt.key_len, 16));
 
 					/* set mic key */
-					memcpy(psecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
-					memcpy(psecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+					memcpy(psecuritypriv->dot118021XGrptxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[16], 8);
+					memcpy(psecuritypriv->dot118021XGrprxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[24], 8);
 
 					psecuritypriv->busetkipkey = true;
 				} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
@@ -2398,7 +2393,7 @@ static int rtw_set_beacon(struct net_device *dev, struct ieee_param *param, int
 {
 	int ret = 0;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	unsigned char *pbuf = param->u.bcn_ie.buf;
 
@@ -2436,7 +2431,7 @@ static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
 	int ret = 0;
 	struct sta_info *psta = NULL;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 
 	DBG_88E("rtw_add_sta(aid =%d) =%pM\n", param->u.add_sta.aid, (param->sta_addr));
@@ -2489,7 +2484,7 @@ static int rtw_del_sta(struct net_device *dev, struct ieee_param *param)
 {
 	struct sta_info *psta = NULL;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	int updated = 0;
 
@@ -2524,7 +2519,7 @@ static int rtw_ioctl_get_sta_data(struct net_device *dev, struct ieee_param *par
 	int ret = 0;
 	struct sta_info *psta = NULL;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	struct ieee_param_ex *param_ex = (struct ieee_param_ex *)param;
 	struct sta_data *psta_data = (struct sta_data *)param_ex->data;
@@ -2580,7 +2575,7 @@ static int rtw_get_sta_wpaie(struct net_device *dev, struct ieee_param *param)
 	int ret = 0;
 	struct sta_info *psta = NULL;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 
 	DBG_88E("rtw_get_sta_wpaie, sta_addr: %pM\n", (param->sta_addr));
@@ -2616,8 +2611,8 @@ static int rtw_set_wps_beacon(struct net_device *dev, struct ieee_param *param,
 {
 	unsigned char wps_oui[4] = {0x0, 0x50, 0xf2, 0x04};
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
-	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
 	int ie_len;
 
 	DBG_88E("%s, len =%d\n", __func__, len);
@@ -2651,7 +2646,7 @@ static int rtw_set_wps_beacon(struct net_device *dev, struct ieee_param *param,
 static int rtw_set_wps_probe_resp(struct net_device *dev, struct ieee_param *param, int len)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	int ie_len;
 
 	DBG_88E("%s, len =%d\n", __func__, len);
@@ -2680,7 +2675,7 @@ static int rtw_set_wps_probe_resp(struct net_device *dev, struct ieee_param *par
 static int rtw_set_wps_assoc_resp(struct net_device *dev, struct ieee_param *param, int len)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	int ie_len;
 
 	DBG_88E("%s, len =%d\n", __func__, len);
@@ -2710,9 +2705,9 @@ static int rtw_set_wps_assoc_resp(struct net_device *dev, struct ieee_param *par
 static int rtw_set_hidden_ssid(struct net_device *dev, struct ieee_param *param, int len)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
-	struct mlme_ext_priv	*pmlmeext = &(padapter->mlmeextpriv);
-	struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_ext_priv	*pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
 	u8 value;
 
@@ -2734,7 +2729,7 @@ static int rtw_set_hidden_ssid(struct net_device *dev, struct ieee_param *param,
 static int rtw_ioctl_acl_remove_sta(struct net_device *dev, struct ieee_param *param, int len)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return -EINVAL;
@@ -2748,7 +2743,7 @@ static int rtw_ioctl_acl_remove_sta(struct net_device *dev, struct ieee_param *p
 static int rtw_ioctl_acl_add_sta(struct net_device *dev, struct ieee_param *param, int len)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return -EINVAL;
@@ -2762,7 +2757,7 @@ static int rtw_ioctl_acl_add_sta(struct net_device *dev, struct ieee_param *para
 static int rtw_ioctl_set_macaddr_acl(struct net_device *dev, struct ieee_param *param, int len)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return -EINVAL;
@@ -2789,14 +2784,9 @@ static int rtw_hostapd_ioctl(struct net_device *dev, struct iw_point *p)
 	if (!p->pointer || p->length != sizeof(struct ieee_param))
 		return -EINVAL;
 
-	param = (struct ieee_param *)rtw_malloc(p->length);
-	if (!param)
-		return -ENOMEM;
-
-	if (copy_from_user(param, p->pointer, p->length)) {
-		kfree(param);
-		return -EFAULT;
-	}
+	param = memdup_user(p->pointer, p->length);
+	if (IS_ERR(param))
+		return PTR_ERR(param);
 
 	switch (param->cmd) {
 	case RTL871X_HOSTAPD_FLUSH:
@@ -2882,7 +2872,7 @@ static int rtw_wx_set_priv(struct net_device *dev,
 	/* added for wps2.0 @20110524 */
 	if (dwrq->flags == 0x8766 && len > 8) {
 		u32 cp_sz;
-		struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+		struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 		u8 *probereq_wpsie = ext;
 		int probereq_wpsie_len = len;
 		u8 wps_oui[4] = {0x0, 0x50, 0xf2, 0x04};
diff --git a/drivers/staging/rtl8188eu/os_dep/rtw_android.c b/drivers/staging/rtl8188eu/os_dep/rtw_android.c
index daf6db354982..bf86d03820ca 100644
--- a/drivers/staging/rtl8188eu/os_dep/rtw_android.c
+++ b/drivers/staging/rtl8188eu/os_dep/rtw_android.c
@@ -77,7 +77,7 @@ static int rtw_android_get_rssi(struct net_device *net, char *command,
 				int total_len)
 {
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(net);
-	struct	mlme_priv	*pmlmepriv = &(padapter->mlmepriv);
+	struct	mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct	wlan_network	*pcur_network = &pmlmepriv->cur_network;
 	int bytes_written = 0;
 
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
index d3664e508cbe..a7cd4de65b28 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c
@@ -81,8 +81,8 @@ static int _rtl92e_hard_start_xmit(struct sk_buff *skb, struct net_device *dev);
 static void _rtl92e_tx_cmd(struct net_device *dev, struct sk_buff *skb);
 static short _rtl92e_tx(struct net_device *dev, struct sk_buff *skb);
 static short _rtl92e_pci_initdescring(struct net_device *dev);
-static void _rtl92e_irq_tx_tasklet(struct r8192_priv *priv);
-static void _rtl92e_irq_rx_tasklet(struct r8192_priv *priv);
+static void _rtl92e_irq_tx_tasklet(unsigned long data);
+static void _rtl92e_irq_rx_tasklet(unsigned long data);
 static void _rtl92e_cancel_deferred_work(struct r8192_priv *priv);
 static int _rtl92e_up(struct net_device *dev, bool is_silent_reset);
 static int _rtl92e_try_up(struct net_device *dev);
@@ -516,8 +516,9 @@ static int _rtl92e_handle_assoc_response(struct net_device *dev,
 	return 0;
 }
 
-static void _rtl92e_prepare_beacon(struct r8192_priv *priv)
+static void _rtl92e_prepare_beacon(unsigned long data)
 {
+	struct r8192_priv *priv = (struct r8192_priv *)data;
 	struct net_device *dev = priv->rtllib->dev;
 	struct sk_buff *pskb = NULL, *pnewskb = NULL;
 	struct cb_desc *tcb_desc = NULL;
@@ -1007,14 +1008,11 @@ static void _rtl92e_init_priv_task(struct net_device *dev)
 			      (void *)rtl92e_hw_wakeup_wq, dev);
 	INIT_DELAYED_WORK_RSL(&priv->rtllib->hw_sleep_wq,
 			      (void *)rtl92e_hw_sleep_wq, dev);
-	tasklet_init(&priv->irq_rx_tasklet,
-		     (void(*)(unsigned long))_rtl92e_irq_rx_tasklet,
+	tasklet_init(&priv->irq_rx_tasklet, _rtl92e_irq_rx_tasklet,
 		     (unsigned long)priv);
-	tasklet_init(&priv->irq_tx_tasklet,
-		     (void(*)(unsigned long))_rtl92e_irq_tx_tasklet,
+	tasklet_init(&priv->irq_tx_tasklet, _rtl92e_irq_tx_tasklet,
 		     (unsigned long)priv);
-	tasklet_init(&priv->irq_prepare_beacon_tasklet,
-		     (void(*)(unsigned long))_rtl92e_prepare_beacon,
+	tasklet_init(&priv->irq_prepare_beacon_tasklet, _rtl92e_prepare_beacon,
 		     (unsigned long)priv);
 }
 
@@ -2113,13 +2111,17 @@ static void _rtl92e_tx_resume(struct net_device *dev)
 	}
 }
 
-static void _rtl92e_irq_tx_tasklet(struct r8192_priv *priv)
+static void _rtl92e_irq_tx_tasklet(unsigned long data)
 {
+	struct r8192_priv *priv = (struct r8192_priv *)data;
+
 	_rtl92e_tx_resume(priv->rtllib->dev);
 }
 
-static void _rtl92e_irq_rx_tasklet(struct r8192_priv *priv)
+static void _rtl92e_irq_rx_tasklet(unsigned long data)
 {
+	struct r8192_priv *priv = (struct r8192_priv *)data;
+
 	_rtl92e_rx_normal(priv->rtllib->dev);
 
 	rtl92e_writel(priv->rtllib->dev, INTA_MASK,
diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c
index 20e494186c9e..462835684e8b 100644
--- a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c
+++ b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c
@@ -456,7 +456,7 @@ static void _rtl92e_dm_bandwidth_autoswitch(struct net_device *dev)
 	if (priv->CurrentChannelBW == HT_CHANNEL_WIDTH_20 ||
 	   !priv->rtllib->bandwidth_auto_switch.bautoswitch_enable)
 		return;
-	if (priv->rtllib->bandwidth_auto_switch.bforced_tx20Mhz == false) {
+	if (!priv->rtllib->bandwidth_auto_switch.bforced_tx20Mhz) {
 		if (priv->undecorated_smoothed_pwdb <=
 		    priv->rtllib->bandwidth_auto_switch.threshold_40Mhzto20Mhz)
 			priv->rtllib->bandwidth_auto_switch.bforced_tx20Mhz = true;
@@ -1297,7 +1297,7 @@ static void _rtl92e_dm_dig_init(struct net_device *dev)
 static void _rtl92e_dm_ctrl_initgain_byrssi(struct net_device *dev)
 {
 
-	if (dm_digtable.dig_enable_flag == false)
+	if (!dm_digtable.dig_enable_flag)
 		return;
 
 	if (dm_digtable.dig_algorithm == DIG_ALGO_BY_FALSE_ALARM)
@@ -1332,7 +1332,7 @@ static void _rtl92e_dm_ctrl_initgain_byrssi_driver(struct net_device *dev)
 	u8 i;
 	static u8	fw_dig;
 
-	if (dm_digtable.dig_enable_flag == false)
+	if (!dm_digtable.dig_enable_flag)
 		return;
 
 	if (dm_digtable.dig_algorithm_switch)
@@ -1366,7 +1366,7 @@ static void _rtl92e_dm_ctrl_initgain_byrssi_false_alarm(struct net_device *dev)
 	static u32 reset_cnt;
 	u8 i;
 
-	if (dm_digtable.dig_enable_flag == false)
+	if (!dm_digtable.dig_enable_flag)
 		return;
 
 	if (dm_digtable.dig_algorithm_switch) {
@@ -1501,7 +1501,7 @@ static void _rtl92e_dm_initial_gain(struct net_device *dev)
 		reset_cnt = 0;
 	}
 
-	if (rtllib_act_scanning(priv->rtllib, true) == true) {
+	if (rtllib_act_scanning(priv->rtllib, true)) {
 		force_write = 1;
 		return;
 	}
@@ -2444,7 +2444,7 @@ static void _rtl92e_dm_init_dynamic_tx_power(struct net_device *dev)
 static void _rtl92e_dm_dynamic_tx_power(struct net_device *dev)
 {
 	struct r8192_priv *priv = rtllib_priv(dev);
-	unsigned int txhipower_threshhold = 0;
+	unsigned int txhipower_threshold = 0;
 	unsigned int txlowpower_threshold = 0;
 
 	if (priv->rtllib->bdynamic_txpower_enable != true) {
@@ -2454,10 +2454,10 @@ static void _rtl92e_dm_dynamic_tx_power(struct net_device *dev)
 	}
 	if ((priv->rtllib->pHTInfo->IOTPeer == HT_IOT_PEER_ATHEROS) &&
 	    (priv->rtllib->mode == IEEE_G)) {
-		txhipower_threshhold = TX_POWER_ATHEROAP_THRESH_HIGH;
+		txhipower_threshold = TX_POWER_ATHEROAP_THRESH_HIGH;
 		txlowpower_threshold = TX_POWER_ATHEROAP_THRESH_LOW;
 	} else {
-		txhipower_threshhold = TX_POWER_NEAR_FIELD_THRESH_HIGH;
+		txhipower_threshold = TX_POWER_NEAR_FIELD_THRESH_HIGH;
 		txlowpower_threshold = TX_POWER_NEAR_FIELD_THRESH_LOW;
 	}
 
@@ -2465,7 +2465,7 @@ static void _rtl92e_dm_dynamic_tx_power(struct net_device *dev)
 		 priv->undecorated_smoothed_pwdb);
 
 	if (priv->rtllib->state == RTLLIB_LINKED) {
-		if (priv->undecorated_smoothed_pwdb >= txhipower_threshhold) {
+		if (priv->undecorated_smoothed_pwdb >= txhipower_threshold) {
 			priv->bDynamicTxHighPower = true;
 			priv->bDynamicTxLowPower = false;
 		} else {
diff --git a/drivers/staging/rtl8192e/rtl819x_HTProc.c b/drivers/staging/rtl8192e/rtl819x_HTProc.c
index d83d72594312..8abc921ecb3e 100644
--- a/drivers/staging/rtl8192e/rtl819x_HTProc.c
+++ b/drivers/staging/rtl8192e/rtl819x_HTProc.c
@@ -371,7 +371,7 @@ void HTConstructInfoElement(struct rtllib_device *ieee, u8 *posHTInfo,
 	if ((ieee->iw_mode == IW_MODE_ADHOC) ||
 	    (ieee->iw_mode == IW_MODE_MASTER)) {
 		pHTInfoEle->ControlChl	= ieee->current_network.channel;
-		pHTInfoEle->ExtChlOffset = ((pHT->bRegBW40MHz == false) ?
+		pHTInfoEle->ExtChlOffset = ((!pHT->bRegBW40MHz) ?
 					    HT_EXTCHNL_OFFSET_NO_EXT :
 					    (ieee->current_network.channel <= 6)
 					    ? HT_EXTCHNL_OFFSET_UPPER :
@@ -526,7 +526,7 @@ void HTOnAssocRsp(struct rtllib_device *ieee)
 	static u8 EWC11NHTCap[] = {0x00, 0x90, 0x4c, 0x33};
 	static u8 EWC11NHTInfo[] = {0x00, 0x90, 0x4c, 0x34};
 
-	if (pHTInfo->bCurrentHTSupport == false) {
+	if (!pHTInfo->bCurrentHTSupport) {
 		netdev_warn(ieee->dev, "%s(): HT_DISABLE\n", __func__);
 		return;
 	}
@@ -873,7 +873,7 @@ void HTSetConnectBwMode(struct rtllib_device *ieee,
 {
 	struct rt_hi_throughput *pHTInfo = ieee->pHTInfo;
 
-	if (pHTInfo->bRegBW40MHz == false)
+	if (!pHTInfo->bRegBW40MHz)
 		return;
 
 	if (ieee->GetHalfNmodeSupportByAPsHandler(ieee->dev))
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
index e101f7b13c7e..195d963c4fbb 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
@@ -520,55 +520,68 @@ static bool AddReorderEntry(struct rx_ts_record *pTS, struct rx_reorder_entry *p
 	return true;
 }
 
-void ieee80211_indicate_packets(struct ieee80211_device *ieee, struct ieee80211_rxb **prxbIndicateArray, u8  index)
+static void indicate_packets(struct ieee80211_device *ieee,
+			     struct ieee80211_rxb *rxb)
 {
-	u8 i = 0, j = 0;
+	struct net_device_stats *stats = &ieee->stats;
+	struct net_device *dev = ieee->dev;
 	u16 ethertype;
-//	if(index > 1)
-//		IEEE80211_DEBUG(IEEE80211_DL_REORDER,"%s(): hahahahhhh, We indicate packet from reorder list, index is %u\n",__func__,index);
-	for (j = 0; j < index; j++) {
-//added by amy for reorder
-		struct ieee80211_rxb *prxb = prxbIndicateArray[j];
-		for (i = 0; i < prxb->nr_subframes; i++) {
-			struct sk_buff *sub_skb = prxb->subframes[i];
+	u8 i;
+
+	for (i = 0; i < rxb->nr_subframes; i++) {
+		struct sk_buff *sub_skb = rxb->subframes[i];
+
+		if (!sub_skb)
+			continue;
 
 		/* convert hdr + possible LLC headers into Ethernet header */
-			ethertype = (sub_skb->data[6] << 8) | sub_skb->data[7];
-			if (sub_skb->len >= 8 &&
-				((memcmp(sub_skb->data, rfc1042_header, SNAP_SIZE) == 0 &&
-				  ethertype != ETH_P_AARP && ethertype != ETH_P_IPX) ||
-				 memcmp(sub_skb->data, bridge_tunnel_header, SNAP_SIZE) == 0)) {
+		ethertype = (sub_skb->data[6] << 8) | sub_skb->data[7];
+		if (sub_skb->len >= 8 &&
+		    ((!memcmp(sub_skb->data, rfc1042_header, SNAP_SIZE) &&
+			ethertype != ETH_P_AARP &&
+			ethertype != ETH_P_IPX) ||
+		     !memcmp(sub_skb->data, bridge_tunnel_header, SNAP_SIZE))) {
 			/* remove RFC1042 or Bridge-Tunnel encapsulation and
 			 * replace EtherType */
-				skb_pull(sub_skb, SNAP_SIZE);
-				memcpy(skb_push(sub_skb, ETH_ALEN), prxb->src, ETH_ALEN);
-				memcpy(skb_push(sub_skb, ETH_ALEN), prxb->dst, ETH_ALEN);
-			} else {
+			skb_pull(sub_skb, SNAP_SIZE);
+		} else {
 			/* Leave Ethernet header part of hdr and full payload */
-				put_unaligned_be16(sub_skb->len, skb_push(sub_skb, 2));
-				memcpy(skb_push(sub_skb, ETH_ALEN), prxb->src, ETH_ALEN);
-				memcpy(skb_push(sub_skb, ETH_ALEN), prxb->dst, ETH_ALEN);
-			}
-			//stats->rx_packets++;
-			//stats->rx_bytes += sub_skb->len;
+			put_unaligned_be16(sub_skb->len, skb_push(sub_skb, 2));
+		}
+		memcpy(skb_push(sub_skb, ETH_ALEN), rxb->src, ETH_ALEN);
+		memcpy(skb_push(sub_skb, ETH_ALEN), rxb->dst, ETH_ALEN);
+
+		stats->rx_packets++;
+		stats->rx_bytes += sub_skb->len;
+		if (is_multicast_ether_addr(rxb->dst))
+			stats->multicast++;
 
 		/* Indicate the packets to upper layer */
-			if (sub_skb) {
-				sub_skb->protocol = eth_type_trans(sub_skb, ieee->dev);
-				memset(sub_skb->cb, 0, sizeof(sub_skb->cb));
-				sub_skb->dev = ieee->dev;
-				sub_skb->ip_summed = CHECKSUM_NONE; /* 802.11 crc not sufficient */
-				//skb->ip_summed = CHECKSUM_UNNECESSARY; /* 802.11 crc not sufficient */
-				ieee->last_rx_ps_time = jiffies;
-				netif_rx(sub_skb);
-			}
-		}
+		sub_skb->protocol = eth_type_trans(sub_skb, dev);
+		memset(sub_skb->cb, 0, sizeof(sub_skb->cb));
+		sub_skb->dev = dev;
+		/* 802.11 crc not sufficient */
+		sub_skb->ip_summed = CHECKSUM_NONE;
+		ieee->last_rx_ps_time = jiffies;
+		netif_rx(sub_skb);
+	}
+}
+
+void ieee80211_indicate_packets(struct ieee80211_device *ieee,
+				struct ieee80211_rxb **prxbIndicateArray,
+				u8 index)
+{
+	u8 i;
+
+	for (i = 0; i < index; i++) {
+		struct ieee80211_rxb *prxb = prxbIndicateArray[i];
+
+		indicate_packets(ieee, prxb);
 		kfree(prxb);
 		prxb = NULL;
 	}
 }
 
-
 static void RxReorderIndicatePacket(struct ieee80211_device *ieee,
 				    struct ieee80211_rxb *prxb,
 				    struct rx_ts_record *pTS, u16 SeqNum)
@@ -877,7 +890,6 @@ int ieee80211_rx(struct ieee80211_device *ieee, struct sk_buff *skb,
 	u16 fc, type, stype, sc;
 	struct net_device_stats *stats;
 	unsigned int frag;
-	u16 ethertype;
 	//added by amy for reorder
 	u8	TID = 0;
 	u16	SeqNum = 0;
@@ -1260,47 +1272,7 @@ int ieee80211_rx(struct ieee80211_device *ieee, struct sk_buff *skb,
 
 //added by amy for reorder
 	if (!ieee->pHTInfo->bCurRxReorderEnable || !pTS) {
-//added by amy for reorder
-		for (i = 0; i < rxb->nr_subframes; i++) {
-			struct sk_buff *sub_skb = rxb->subframes[i];
-
-			if (sub_skb) {
-				/* convert hdr + possible LLC headers into Ethernet header */
-				ethertype = (sub_skb->data[6] << 8) | sub_skb->data[7];
-				if (sub_skb->len >= 8 &&
-						((memcmp(sub_skb->data, rfc1042_header, SNAP_SIZE) == 0 &&
-						  ethertype != ETH_P_AARP && ethertype != ETH_P_IPX) ||
-						 memcmp(sub_skb->data, bridge_tunnel_header, SNAP_SIZE) == 0)) {
-					/* remove RFC1042 or Bridge-Tunnel encapsulation and
-					 * replace EtherType */
-					skb_pull(sub_skb, SNAP_SIZE);
-					memcpy(skb_push(sub_skb, ETH_ALEN), src, ETH_ALEN);
-					memcpy(skb_push(sub_skb, ETH_ALEN), dst, ETH_ALEN);
-				} else {
-					u16 len;
-					/* Leave Ethernet header part of hdr and full payload */
-					len = be16_to_cpu(htons(sub_skb->len));
-					memcpy(skb_push(sub_skb, 2), &len, 2);
-					memcpy(skb_push(sub_skb, ETH_ALEN), src, ETH_ALEN);
-					memcpy(skb_push(sub_skb, ETH_ALEN), dst, ETH_ALEN);
-				}
-
-				stats->rx_packets++;
-				stats->rx_bytes += sub_skb->len;
-				if (is_multicast_ether_addr(dst)) {
-					stats->multicast++;
-				}
-
-				/* Indicate the packets to upper layer */
-				sub_skb->protocol = eth_type_trans(sub_skb, dev);
-				memset(sub_skb->cb, 0, sizeof(sub_skb->cb));
-				sub_skb->dev = dev;
-				sub_skb->ip_summed = CHECKSUM_NONE; /* 802.11 crc not sufficient */
-				//skb->ip_summed = CHECKSUM_UNNECESSARY; /* 802.11 crc not sufficient */
-				ieee->last_rx_ps_time = jiffies;
-				netif_rx(sub_skb);
-			}
-		}
+		indicate_packets(ieee, rxb);
 		kfree(rxb);
 		rxb = NULL;
 
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c
index 0ee054d82832..63a561ab4a76 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c
@@ -372,9 +372,9 @@ ieee80211_query_HTCapShortGI(struct ieee80211_device *ieee, struct cb_desc *tcb_
 		return;
 	}
 
-	if ((pHTInfo->bCurBW40MHz == true) && pHTInfo->bCurShortGI40MHz)
+	if (pHTInfo->bCurBW40MHz && pHTInfo->bCurShortGI40MHz)
 		tcb_desc->bUseShortGI = true;
-	else if ((pHTInfo->bCurBW40MHz == false) && pHTInfo->bCurShortGI20MHz)
+	else if (!pHTInfo->bCurBW40MHz && pHTInfo->bCurShortGI20MHz)
 		tcb_desc->bUseShortGI = true;
 }
 
diff --git a/drivers/staging/rtl8712/rtl871x_xmit.c b/drivers/staging/rtl8712/rtl871x_xmit.c
index f0b85338b567..2f0d0ffa6fae 100644
--- a/drivers/staging/rtl8712/rtl871x_xmit.c
+++ b/drivers/staging/rtl8712/rtl871x_xmit.c
@@ -71,12 +71,13 @@ int _r8712_init_xmit_priv(struct xmit_priv *pxmitpriv,
 	_init_queue(&pxmitpriv->apsd_queue);
 	_init_queue(&pxmitpriv->free_xmit_queue);
 	/*
-	 * Please allocate memory with the sz = (struct xmit_frame) * NR_XMITFRAME,
+	 * Please allocate memory with sz = (struct xmit_frame) * NR_XMITFRAME,
 	 * and initialize free_xmit_frame below.
 	 * Please also apply  free_txobj to link_up all the xmit_frames...
 	 */
 	pxmitpriv->pallocated_frame_buf =
-		kmalloc(NR_XMITFRAME * sizeof(struct xmit_frame) + 4, GFP_ATOMIC);
+		kmalloc(NR_XMITFRAME * sizeof(struct xmit_frame) + 4,
+			GFP_ATOMIC);
 	if (!pxmitpriv->pallocated_frame_buf) {
 		pxmitpriv->pxmit_frame_buf = NULL;
 		return -ENOMEM;
@@ -126,8 +127,8 @@ int _r8712_init_xmit_priv(struct xmit_priv *pxmitpriv,
 	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf;
 	for (i = 0; i < NR_XMITBUFF; i++) {
 		INIT_LIST_HEAD(&pxmitbuf->list);
-		pxmitbuf->pallocated_buf = kmalloc(MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ,
-						   GFP_ATOMIC);
+		pxmitbuf->pallocated_buf =
+			kmalloc(MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ, GFP_ATOMIC);
 		if (!pxmitbuf->pallocated_buf)
 			return -ENOMEM;
 		pxmitbuf->pbuf = pxmitbuf->pallocated_buf + XMITBUF_ALIGN_SZ -
@@ -350,7 +351,7 @@ static int xmitframe_addmic(struct _adapter *padapter,
 	struct	sta_info *stainfo;
 	struct	qos_priv *pqospriv = &(padapter->mlmepriv.qospriv);
 	struct	pkt_attrib  *pattrib = &pxmitframe->attrib;
-	struct	security_priv *psecuritypriv = &padapter->securitypriv;
+	struct	security_priv *psecpriv = &padapter->securitypriv;
 	struct	xmit_priv *pxmitpriv = &padapter->xmitpriv;
 	u8 priority[4] = {0x0, 0x0, 0x0, 0x0};
 	bool bmcst = is_multicast_ether_addr(pattrib->ra);
@@ -368,15 +369,14 @@ static int xmitframe_addmic(struct _adapter *padapter,
 					   0x0, 0x0};
 			pframe = pxmitframe->buf_addr + TXDESC_OFFSET;
 			if (bmcst) {
-				if (!memcmp(psecuritypriv->XGrptxmickey
-				   [psecuritypriv->XGrpKeyid].skey,
+				if (!memcmp(psecpriv->XGrptxmickey
+				   [psecpriv->XGrpKeyid].skey,
 				   null_key, 16))
 					return -ENOMEM;
 				/*start to calculate the mic code*/
 				r8712_secmicsetkey(&micdata,
-					 psecuritypriv->
-					 XGrptxmickey[psecuritypriv->
-					XGrpKeyid].skey);
+					psecpriv->XGrptxmickey
+					[psecpriv->XGrpKeyid].skey);
 			} else {
 				if (!memcmp(&stainfo->tkiptxmickey.skey[0],
 					    null_key, 16))
@@ -416,7 +416,7 @@ static int xmitframe_addmic(struct _adapter *padapter,
 					length = pattrib->last_txcmdsz -
 						  pattrib->hdrlen -
 						  pattrib->iv_len -
-						  ((psecuritypriv->sw_encrypt)
+						  ((psecpriv->sw_encrypt)
 						  ? pattrib->icv_len : 0);
 					r8712_secmicappend(&micdata, payload,
 							   length);
@@ -424,7 +424,7 @@ static int xmitframe_addmic(struct _adapter *padapter,
 				} else {
 					length = pxmitpriv->frag_len -
 					    pattrib->hdrlen - pattrib->iv_len -
-					    ((psecuritypriv->sw_encrypt) ?
+					    ((psecpriv->sw_encrypt) ?
 					    pattrib->icv_len : 0);
 					r8712_secmicappend(&micdata, payload,
 							   length);
@@ -477,75 +477,72 @@ static int make_wlanhdr(struct _adapter *padapter, u8 *hdr,
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct qos_priv *pqospriv = &pmlmepriv->qospriv;
 	__le16 *fctrl = &pwlanhdr->frame_ctl;
+	u8 *bssid;
 
 	memset(hdr, 0, WLANHDR_OFFSET);
 	SetFrameSubType(fctrl, pattrib->subtype);
-	if (pattrib->subtype & WIFI_DATA_TYPE) {
-		if (check_fwstate(pmlmepriv,  WIFI_STATION_STATE)) {
-			/* to_ds = 1, fr_ds = 0; */
-			SetToDs(fctrl);
-			memcpy(pwlanhdr->addr1, get_bssid(pmlmepriv),
-				ETH_ALEN);
-			memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
-			memcpy(pwlanhdr->addr3, pattrib->dst, ETH_ALEN);
-		} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
-			/* to_ds = 0, fr_ds = 1; */
-			SetFrDs(fctrl);
-			memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
-			memcpy(pwlanhdr->addr2, get_bssid(pmlmepriv),
-				ETH_ALEN);
-			memcpy(pwlanhdr->addr3, pattrib->src, ETH_ALEN);
-		} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
-			   check_fwstate(pmlmepriv,
-					 WIFI_ADHOC_MASTER_STATE)) {
-			memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
-			memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
-			memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv),
-				ETH_ALEN);
-		} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
-			memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
-			memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
-			memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv),
-				ETH_ALEN);
-		} else {
-			return -EINVAL;
-		}
+	if (!(pattrib->subtype & WIFI_DATA_TYPE))
+		return 0;
 
-		if (pattrib->encrypt)
-			SetPrivacy(fctrl);
-		if (pqospriv->qos_option) {
-			qc = (unsigned short *)(hdr + pattrib->hdrlen - 2);
-			if (pattrib->priority)
-				SetPriority(qc, pattrib->priority);
-			SetAckpolicy(qc, pattrib->ack_policy);
-		}
-		/* TODO: fill HT Control Field */
-		/* Update Seq Num will be handled by f/w */
-		{
-			struct sta_info *psta;
-			bool bmcst = is_multicast_ether_addr(pattrib->ra);
-
-			if (pattrib->psta) {
-				psta = pattrib->psta;
-			} else {
-				if (bmcst)
-					psta = r8712_get_bcmc_stainfo(padapter);
-				else
-					psta =
-					 r8712_get_stainfo(&padapter->stapriv,
-					 pattrib->ra);
-			}
-			if (psta) {
-				psta->sta_xmitpriv.txseq_tid
-						  [pattrib->priority]++;
-				psta->sta_xmitpriv.txseq_tid[pattrib->priority]
-						   &= 0xFFF;
-				pattrib->seqnum = psta->sta_xmitpriv.
-						  txseq_tid[pattrib->priority];
-				SetSeqNum(hdr, pattrib->seqnum);
-			}
+	bssid = get_bssid(pmlmepriv);
+
+	if (check_fwstate(pmlmepriv,  WIFI_STATION_STATE)) {
+		/* to_ds = 1, fr_ds = 0; */
+		SetToDs(fctrl);
+		ether_addr_copy(pwlanhdr->addr1, bssid);
+		ether_addr_copy(pwlanhdr->addr2, pattrib->src);
+		ether_addr_copy(pwlanhdr->addr3, pattrib->dst);
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+		/* to_ds = 0, fr_ds = 1; */
+		SetFrDs(fctrl);
+		ether_addr_copy(pwlanhdr->addr1, pattrib->dst);
+		ether_addr_copy(pwlanhdr->addr2, bssid);
+		ether_addr_copy(pwlanhdr->addr3, pattrib->src);
+	} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
+		   check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
+		ether_addr_copy(pwlanhdr->addr1, pattrib->dst);
+		ether_addr_copy(pwlanhdr->addr2, pattrib->src);
+		ether_addr_copy(pwlanhdr->addr3, bssid);
+	} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
+		ether_addr_copy(pwlanhdr->addr1, pattrib->dst);
+		ether_addr_copy(pwlanhdr->addr2, pattrib->src);
+		ether_addr_copy(pwlanhdr->addr3, bssid);
+	} else {
+		return -EINVAL;
+	}
+
+	if (pattrib->encrypt)
+		SetPrivacy(fctrl);
+	if (pqospriv->qos_option) {
+		qc = (unsigned short *)(hdr + pattrib->hdrlen - 2);
+		if (pattrib->priority)
+			SetPriority(qc, pattrib->priority);
+		SetAckpolicy(qc, pattrib->ack_policy);
+	}
+	/* TODO: fill HT Control Field */
+	/* Update Seq Num will be handled by f/w */
+	{
+		struct sta_info *psta;
+		bool bmcst = is_multicast_ether_addr(pattrib->ra);
+
+		if (pattrib->psta)
+			psta = pattrib->psta;
+		else if (bmcst)
+			psta = r8712_get_bcmc_stainfo(padapter);
+		else
+			psta = r8712_get_stainfo(&padapter->stapriv,
+						 pattrib->ra);
+
+		if (psta) {
+			u16 *txtid = psta->sta_xmitpriv.txseq_tid;
+
+			txtid[pattrib->priority]++;
+			txtid[pattrib->priority] &= 0xFFF;
+			pattrib->seqnum = txtid[pattrib->priority];
+			SetSeqNum(hdr, pattrib->seqnum);
 		}
 	}
+
 	return 0;
 }
 
@@ -589,7 +586,7 @@ sint r8712_xmitframe_coalesce(struct _adapter *padapter, _pkt *pkt,
 	addr_t addr;
 	u8 *pframe, *mem_start, *ptxdesc;
 	struct sta_info		*psta;
-	struct security_priv	*psecuritypriv = &padapter->securitypriv;
+	struct security_priv	*psecpriv = &padapter->securitypriv;
 	struct mlme_priv	*pmlmepriv = &padapter->mlmepriv;
 	struct xmit_priv	*pxmitpriv = &padapter->xmitpriv;
 	struct pkt_attrib	*pattrib = &pxmitframe->attrib;
@@ -632,15 +629,13 @@ sint r8712_xmitframe_coalesce(struct _adapter *padapter, _pkt *pkt,
 				case _WEP40_:
 				case _WEP104_:
 					WEP_IV(pattrib->iv, psta->txpn,
-					       (u8)psecuritypriv->
-					       PrivacyKeyIndex);
+					       (u8)psecpriv->PrivacyKeyIndex);
 					break;
 				case _TKIP_:
 					if (bmcst)
 						TKIP_IV(pattrib->iv,
 						    psta->txpn,
-						    (u8)psecuritypriv->
-						    XGrpKeyid);
+						    (u8)psecpriv->XGrpKeyid);
 					else
 						TKIP_IV(pattrib->iv, psta->txpn,
 							0);
@@ -648,8 +643,7 @@ sint r8712_xmitframe_coalesce(struct _adapter *padapter, _pkt *pkt,
 				case _AES_:
 					if (bmcst)
 						AES_IV(pattrib->iv, psta->txpn,
-						    (u8)psecuritypriv->
-						    XGrpKeyid);
+						    (u8)psecpriv->XGrpKeyid);
 					else
 						AES_IV(pattrib->iv, psta->txpn,
 						       0);
diff --git a/drivers/staging/rtl8712/rtl871x_xmit.h b/drivers/staging/rtl8712/rtl871x_xmit.h
index f227828094bf..c0c0c781fe17 100644
--- a/drivers/staging/rtl8712/rtl871x_xmit.h
+++ b/drivers/staging/rtl8712/rtl871x_xmit.h
@@ -115,7 +115,7 @@ struct pkt_attrib {
 	u8	icv_len;
 	unsigned char iv[8];
 	unsigned char icv[8];
-	u8	dst[ETH_ALEN];
+	u8	dst[ETH_ALEN] __aligned(2);	/* for ether_addr_copy */
 	u8	src[ETH_ALEN];
 	u8	ta[ETH_ALEN];
 	u8	ra[ETH_ALEN];
diff --git a/drivers/staging/rtl8712/usb_halinit.c b/drivers/staging/rtl8712/usb_halinit.c
index 6cc4a704c3a0..313c569748e9 100644
--- a/drivers/staging/rtl8712/usb_halinit.c
+++ b/drivers/staging/rtl8712/usb_halinit.c
@@ -58,7 +58,7 @@ u8 r8712_usb_hal_bus_init(struct _adapter *adapter)
 		r8712_write8(adapter, SYS_ISO_CTRL + 1, val8);
 		val8 = r8712_read8(adapter, SYS_ISO_CTRL + 1);
 		val8 = val8 & 0xEF;
-		/* attatch AFE PLL to MACTOP/BB/PCIe Digital */
+		/* attach AFE PLL to MACTOP/BB/PCIe Digital */
 		r8712_write8(adapter, SYS_ISO_CTRL + 1, val8);
 		val8 = r8712_read8(adapter, AFE_XTAL_CTRL + 1);
 		val8 = val8 & 0xFB;
diff --git a/drivers/staging/rtl8712/wifi.h b/drivers/staging/rtl8712/wifi.h
index be731f1a2209..91b65731fcaa 100644
--- a/drivers/staging/rtl8712/wifi.h
+++ b/drivers/staging/rtl8712/wifi.h
@@ -440,7 +440,7 @@ static inline unsigned char *get_hdr_bssid(unsigned char *pframe)
 /* block-ack parameters */
 #define IEEE80211_ADDBA_PARAM_POLICY_MASK 0x0002
 #define IEEE80211_ADDBA_PARAM_TID_MASK 0x003C
-#define IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK 0xFFA0
+#define IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK 0xFFC0
 #define IEEE80211_DELBA_PARAM_TID_MASK 0xF000
 #define IEEE80211_DELBA_PARAM_INITIATOR_MASK 0x0800
 
@@ -532,13 +532,6 @@ struct ieee80211_ht_addt_info {
 #define IEEE80211_HT_IE_NON_GF_STA_PRSNT	0x0004
 #define IEEE80211_HT_IE_NON_HT_STA_PRSNT	0x0010
 
-/* block-ack parameters */
-#define IEEE80211_ADDBA_PARAM_POLICY_MASK 0x0002
-#define IEEE80211_ADDBA_PARAM_TID_MASK 0x003C
-#define IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK 0xFFA0
-#define IEEE80211_DELBA_PARAM_TID_MASK 0xF000
-#define IEEE80211_DELBA_PARAM_INITIATOR_MASK 0x0800
-
 /*
  * A-PMDU buffer sizes
  * According to IEEE802.11n spec size varies from 8K to 64K (in powers of 2)
diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c
index efb5135ad743..bd18d1803e27 100644
--- a/drivers/staging/rtl8723bs/core/rtw_cmd.c
+++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c
@@ -822,7 +822,7 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 
 	psecnetwork->IELength = 0;
 	/*  Added by Albert 2009/02/18 */
-	/*  If the the driver wants to use the bssid to create the connection. */
+	/*  If the driver wants to use the bssid to create the connection. */
 	/*  If not,  we have to copy the connecting AP's MAC address to it so that */
 	/*  the driver just has the bssid information for PMKIDList searching. */
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
index d7a58af76ea0..e65c5a870b46 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
@@ -1097,9 +1097,6 @@ inline void rtw_indicate_scan_done(struct adapter *padapter, bool aborted)
 	    (!adapter_to_pwrctl(padapter)->bInSuspend) &&
 	    (!check_fwstate(&padapter->mlmepriv,
 			    WIFI_ASOC_STATE|WIFI_UNDER_LINKING))) {
-		struct pwrctrl_priv *pwrpriv;
-
-		pwrpriv = adapter_to_pwrctl(padapter);
 		rtw_set_ips_deny(padapter, 0);
 		_set_timer(&padapter->mlmepriv.dynamic_chk_timer, 1);
 	}
@@ -2917,12 +2914,11 @@ void rtw_append_exented_cap(struct adapter *padapter, u8 *out_ie, uint *pout_len
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct ht_priv 	*phtpriv = &pmlmepriv->htpriv;
 	u8 cap_content[8] = {0};
-	u8 *pframe;
 
 	if (phtpriv->bss_coexist)
 		SET_EXT_CAPABILITY_ELE_BSS_COEXIST(cap_content, 1);
 
-	pframe = rtw_set_ie(out_ie + *pout_len, EID_EXTCapability, 8, cap_content, pout_len);
+	rtw_set_ie(out_ie + *pout_len, EID_EXTCapability, 8, cap_content, pout_len);
 }
 
 inline void rtw_set_to_roam(struct adapter *adapter, u8 to_roam)
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
index 8f9da1d49343..d6d7198dfe45 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
@@ -1084,7 +1084,7 @@ auth_fail:
 
 unsigned int OnAuthClient(struct adapter *padapter, union recv_frame *precv_frame)
 {
-	unsigned int	seq, len, status, algthm, offset;
+	unsigned int	seq, len, status, offset;
 	unsigned char *p;
 	unsigned int	go2asoc = 0;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
@@ -1103,7 +1103,6 @@ unsigned int OnAuthClient(struct adapter *padapter, union recv_frame *precv_fram
 
 	offset = (GetPrivacy(pframe)) ? 4 : 0;
 
-	algthm	= le16_to_cpu(*(__le16 *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset));
 	seq	= le16_to_cpu(*(__le16 *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset + 2));
 	status	= le16_to_cpu(*(__le16 *)((SIZE_PTR)pframe + WLAN_HDR_A3_LEN + offset + 4));
 
@@ -1170,7 +1169,7 @@ authclnt_fail:
 
 unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 {
-	u16 capab_info, listen_interval;
+	u16 capab_info;
 	struct rtw_ieee802_11_elems elems;
 	struct sta_info *pstat;
 	unsigned char 	reassoc, *p, *pos, *wpa_ie;
@@ -1216,8 +1215,6 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 
 	capab_info = RTW_GET_LE16(pframe + WLAN_HDR_A3_LEN);
 	/* capab_info = le16_to_cpu(*(unsigned short *)(pframe + WLAN_HDR_A3_LEN)); */
-	/* listen_interval = le16_to_cpu(*(unsigned short *)(pframe + WLAN_HDR_A3_LEN+2)); */
-	listen_interval = RTW_GET_LE16(pframe + WLAN_HDR_A3_LEN+2);
 
 	left = pkt_len - (sizeof(struct ieee80211_hdr_3addr) + ie_offset);
 	pos = pframe + (sizeof(struct ieee80211_hdr_3addr) + ie_offset);
diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c
index 5245098b9ecf..7e1da0e35812 100644
--- a/drivers/staging/rtl8723bs/core/rtw_recv.c
+++ b/drivers/staging/rtl8723bs/core/rtw_recv.c
@@ -10,14 +10,11 @@
 #include <rtw_debug.h>
 #include <linux/jiffies.h>
 #include <rtw_recv.h>
+#include <net/cfg80211.h>
 
 static u8 SNAP_ETH_TYPE_IPX[2] = {0x81, 0x37};
 static u8 SNAP_ETH_TYPE_APPLETALK_AARP[2] = {0x80, 0xf3};
 
-u8 rtw_rfc1042_header[] = { 0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00 };
-/* Bridge-Tunnel header (for EtherTypes ETH_P_AARP and ETH_P_IPX) */
-u8 rtw_bridge_tunnel_header[] = { 0xaa, 0xaa, 0x03, 0x00, 0x00, 0xf8 };
-
 static void rtw_signal_stat_timer_hdl(struct timer_list *t);
 
 void _rtw_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv)
@@ -1625,11 +1622,11 @@ sint wlanhdr_to_ethhdr(union recv_frame *precvframe)
 	psnap_type = ptr+pattrib->hdrlen + pattrib->iv_len+SNAP_SIZE;
 	/* convert hdr + possible LLC headers into Ethernet header */
 	/* eth_type = (psnap_type[0] << 8) | psnap_type[1]; */
-	if ((!memcmp(psnap, rtw_rfc1042_header, SNAP_SIZE) &&
+	if ((!memcmp(psnap, rfc1042_header, SNAP_SIZE) &&
 		(memcmp(psnap_type, SNAP_ETH_TYPE_IPX, 2)) &&
 		(memcmp(psnap_type, SNAP_ETH_TYPE_APPLETALK_AARP, 2))) ||
 		/* eth_type != ETH_P_AARP && eth_type != ETH_P_IPX) || */
-		 !memcmp(psnap, rtw_bridge_tunnel_header, SNAP_SIZE)) {
+		 !memcmp(psnap, bridge_tunnel_header, SNAP_SIZE)) {
 		/* remove RFC1042 or Bridge-Tunnel encapsulation and replace EtherType */
 		bsnaphdr = true;
 	} else
diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index 5ebf691bd743..0f95009a30b6 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -756,7 +756,7 @@ u32 rtw_tkip_decrypt(struct adapter *padapter, u8 *precvframe)
 				static u32 no_gkey_bc_cnt;
 				static u32 no_gkey_mc_cnt;
 
-				if (psecuritypriv->binstallGrpkey == false) {
+				if (!psecuritypriv->binstallGrpkey) {
 					res = _FAIL;
 
 					if (start == 0)
@@ -1837,7 +1837,7 @@ u32 rtw_aes_decrypt(struct adapter *padapter, u8 *precvframe)
 				static u32 no_gkey_bc_cnt;
 				static u32 no_gkey_mc_cnt;
 
-				if (psecuritypriv->binstallGrpkey == false) {
+				if (!psecuritypriv->binstallGrpkey) {
 					res = _FAIL;
 
 					if (start == 0)
@@ -2369,7 +2369,7 @@ u8 rtw_handle_tkip_countermeasure(struct adapter *adapter, const char *caller)
 	struct security_priv *securitypriv = &(adapter->securitypriv);
 	u8 status = _SUCCESS;
 
-	if (securitypriv->btkip_countermeasure == true) {
+	if (securitypriv->btkip_countermeasure) {
 		unsigned long passing_ms = jiffies_to_msecs(jiffies - securitypriv->btkip_countermeasure_time);
 		if (passing_ms > 60*1000) {
 			DBG_871X_LEVEL(_drv_always_, "%s("ADPT_FMT") countermeasure time:%lus > 60s\n",
diff --git a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
index 09d2ca30d653..e3f56c6cc882 100644
--- a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
+++ b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
@@ -553,7 +553,6 @@ u32 rtw_init_bcmc_stainfo(struct adapter *padapter)
 {
 
 	struct sta_info *psta;
-	struct tx_servq	*ptxservq;
 	u32 res = _SUCCESS;
 	NDIS_802_11_MAC_ADDRESS	bcast_addr = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
@@ -571,7 +570,6 @@ u32 rtw_init_bcmc_stainfo(struct adapter *padapter)
 	/*  default broadcast & multicast use macid 1 */
 	psta->mac_id = 1;
 
-	ptxservq = &(psta->sta_xmitpriv.be_q);
 exit:
 	return _SUCCESS;
 }
diff --git a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
index 110338dbe372..69bcd172b298 100644
--- a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
+++ b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
@@ -1271,13 +1271,13 @@ int rtw_check_bcn_info(struct adapter *Adapter, u8 *pframe, u32 packet_len)
 	unsigned char *pbuf;
 	u32 wpa_ielen = 0;
 	u8 *pbssid = GetAddr3Ptr(pframe);
-	u32 hidden_ssid = 0;
 	struct HT_info_element *pht_info = NULL;
 	struct rtw_ieee80211_ht_cap *pht_cap = NULL;
 	u32 bcn_channel;
 	unsigned short	ht_cap_info;
 	unsigned char ht_info_infos_0;
 	struct mlme_priv *pmlmepriv = &Adapter->mlmepriv;
+	int ssid_len;
 
 	if (is_client_associated_to_ap(Adapter) == false)
 		return true;
@@ -1370,21 +1370,15 @@ int rtw_check_bcn_info(struct adapter *Adapter, u8 *pframe, u32 packet_len)
 	}
 
 	/* checking SSID */
+	ssid_len = 0;
 	p = rtw_get_ie(bssid->IEs + _FIXED_IE_LENGTH_, _SSID_IE_, &len, bssid->IELength - _FIXED_IE_LENGTH_);
-	if (!p) {
-		DBG_871X("%s marc: cannot find SSID for survey event\n", __func__);
-		hidden_ssid = true;
-	} else {
-		hidden_ssid = false;
-	}
-
-	if ((NULL != p) && (false == hidden_ssid && (*(p + 1)))) {
-		memcpy(bssid->Ssid.Ssid, (p + 2), *(p + 1));
-		bssid->Ssid.SsidLength = *(p + 1);
-	} else {
-		bssid->Ssid.SsidLength = 0;
-		bssid->Ssid.Ssid[0] = '\0';
+	if (p) {
+		ssid_len = *(p + 1);
+		if (ssid_len > NDIS_802_11_LENGTH_SSID)
+			ssid_len = 0;
 	}
+	memcpy(bssid->Ssid.Ssid, (p + 2), ssid_len);
+	bssid->Ssid.SsidLength = ssid_len;
 
 	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_, ("%s bssid.Ssid.Ssid:%s bssid.Ssid.SsidLength:%d "
 				"cur_network->network.Ssid.Ssid:%s len:%d\n", __func__, bssid->Ssid.Ssid,
diff --git a/drivers/staging/rtl8723bs/hal/hal_btcoex.c b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
index d5793e4614bf..3705a60a0546 100644
--- a/drivers/staging/rtl8723bs/hal/hal_btcoex.c
+++ b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
@@ -12,49 +12,6 @@
 #include <Mp_Precomp.h>
 
 /* 		Global variables */
-static const char *const BtProfileString[] = {
-	"NONE",
-	"A2DP",
-	"PAN",
-	"HID",
-	"SCO",
-};
-
-static const char *const BtSpecString[] = {
-	"1.0b",
-	"1.1",
-	"1.2",
-	"2.0+EDR",
-	"2.1+EDR",
-	"3.0+HS",
-	"4.0",
-};
-
-static const char *const BtLinkRoleString[] = {
-	"Master",
-	"Slave",
-};
-
-static const char *const h2cStaString[] = {
-	"successful",
-	"h2c busy",
-	"rf off",
-	"fw not read",
-};
-
-static const char *const ioStaString[] = {
-	"success",
-	"can not IO",
-	"rf off",
-	"fw not read",
-	"wait io timeout",
-	"invalid len",
-	"idle Q empty",
-	"insert waitQ fail",
-	"unknown fail",
-	"wrong level",
-	"h2c stopped",
-};
 
 BTC_COEXIST GLBtCoexist;
 static u8 GLBtcWiFiInScanState;
@@ -450,7 +407,7 @@ static u8 halbtcoutsrc_Get(void *pBtcContext, u8 getType, void *pOutBuf)
 		break;
 
 	case BTC_GET_BL_WIFI_ENABLE_ENCRYPTION:
-		*pu8 = padapter->securitypriv.dot11PrivacyAlgrthm == 0 ? false : true;
+		*pu8 = padapter->securitypriv.dot11PrivacyAlgrthm != 0;
 		break;
 
 	case BTC_GET_BL_WIFI_UNDER_B_MODE:
diff --git a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
index 767e2a784f78..10250642d30a 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
@@ -1816,7 +1816,7 @@ static void phy_CrossReferenceHTAndVHTTxPowerLimit(struct adapter *padapter)
 	s8 tempPwrLmt = 0;
 
 	for (regulation = 0; regulation < MAX_REGULATION_NUM; ++regulation) {
-		for (bw = 0; bw < MAX_5G_BANDWITH_NUM; ++bw) {
+		for (bw = 0; bw < MAX_5G_BANDWIDTH_NUM; ++bw) {
 			for (channel = 0; channel < CHANNEL_MAX_NUMBER_5G; ++channel) {
 				for (rateSection = 0; rateSection < MAX_RATE_SECTION_NUM; ++rateSection) {
 					tempPwrLmt = pHalData->TxPwrLimit_5G[regulation][bw][rateSection][channel][ODM_RF_PATH_A];
@@ -1877,7 +1877,7 @@ void PHY_ConvertTxPowerLimitToPowerIndex(struct adapter *Adapter)
 	phy_CrossReferenceHTAndVHTTxPowerLimit(Adapter);
 
 	for (regulation = 0; regulation < MAX_REGULATION_NUM; ++regulation) {
-		for (bw = 0; bw < MAX_2_4G_BANDWITH_NUM; ++bw) {
+		for (bw = 0; bw < MAX_2_4G_BANDWIDTH_NUM; ++bw) {
 			for (channel = 0; channel < CHANNEL_MAX_NUMBER_2G; ++channel) {
 				for (rateSection = 0; rateSection < MAX_RATE_SECTION_NUM; ++rateSection) {
 					tempPwrLmt = pHalData->TxPwrLimit_2_4G[regulation][bw][rateSection][channel][ODM_RF_PATH_A];
@@ -1920,7 +1920,7 @@ void PHY_InitTxPowerLimit(struct adapter *Adapter)
 	/* DBG_871X("=====> PHY_InitTxPowerLimit()!\n"); */
 
 	for (i = 0; i < MAX_REGULATION_NUM; ++i) {
-		for (j = 0; j < MAX_2_4G_BANDWITH_NUM; ++j)
+		for (j = 0; j < MAX_2_4G_BANDWIDTH_NUM; ++j)
 			for (k = 0; k < MAX_RATE_SECTION_NUM; ++k)
 				for (m = 0; m < CHANNEL_MAX_NUMBER_2G; ++m)
 					for (l = 0; l < MAX_RF_PATH_NUM; ++l)
@@ -1928,7 +1928,7 @@ void PHY_InitTxPowerLimit(struct adapter *Adapter)
 	}
 
 	for (i = 0; i < MAX_REGULATION_NUM; ++i) {
-		for (j = 0; j < MAX_5G_BANDWITH_NUM; ++j)
+		for (j = 0; j < MAX_5G_BANDWIDTH_NUM; ++j)
 			for (k = 0; k < MAX_RATE_SECTION_NUM; ++k)
 				for (m = 0; m < CHANNEL_MAX_NUMBER_5G; ++m)
 					for (l = 0; l < MAX_RF_PATH_NUM; ++l)
diff --git a/drivers/staging/rtl8723bs/hal/odm.c b/drivers/staging/rtl8723bs/hal/odm.c
index aa6631ee4ea7..f2a9e95a1563 100644
--- a/drivers/staging/rtl8723bs/hal/odm.c
+++ b/drivers/staging/rtl8723bs/hal/odm.c
@@ -7,19 +7,6 @@
 
 #include "odm_precomp.h"
 
-static const u16 dB_Invert_Table[8][12] = {
-	{1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4},
-	{4, 5, 6, 6, 7, 8, 9, 10, 11, 13, 14, 16},
-	{18, 20, 22, 25, 28, 32, 35, 40, 45, 50, 56, 63},
-	{71, 79, 89, 100, 112, 126, 141, 158, 178, 200, 224, 251},
-	{282, 316, 355, 398, 447, 501, 562, 631, 708, 794, 891, 1000},
-	{1122, 1259, 1413, 1585, 1778, 1995, 2239, 2512, 2818, 3162, 3548, 3981},
-	{4467, 5012, 5623, 6310, 7079, 7943, 8913, 10000, 11220, 12589, 14125,
-	 15849},
-	{17783, 19953, 22387, 25119, 28184, 31623, 35481, 39811, 44668, 50119,
-	 56234, 65535}
- };
-
 /*  Global var */
 
 u32 OFDMSwingTable[OFDM_TABLE_SIZE] = {
diff --git a/drivers/staging/rtl8723bs/hal/odm.h b/drivers/staging/rtl8723bs/hal/odm.h
index b77d1fe33a28..16e8f66a3171 100644
--- a/drivers/staging/rtl8723bs/hal/odm.h
+++ b/drivers/staging/rtl8723bs/hal/odm.h
@@ -541,7 +541,7 @@ typedef enum tag_Operation_Mode_Definition {
 
 /*  ODM_CMNINFO_WM_MODE */
 typedef enum tag_Wireless_Mode_Definition {
-	ODM_WM_UNKNOW     = 0x0,
+	ODM_WM_UNKNOWN    = 0x0,
 	ODM_WM_B          = BIT0,
 	ODM_WM_G          = BIT1,
 	ODM_WM_A          = BIT2,
diff --git a/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h b/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h
index f2c0707aad4c..1c6c08000e27 100644
--- a/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h
+++ b/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h
@@ -31,8 +31,8 @@
 #define	ODM_REG_TX_ANT_CTRL_11N			0x80C
 #define	ODM_REG_BB_PWR_SAV5_11N		0x818
 #define	ODM_REG_CCK_RPT_FORMAT_11N		0x824
-#define	ODM_REG_RX_DEFUALT_A_11N		0x858
-#define	ODM_REG_RX_DEFUALT_B_11N		0x85A
+#define	ODM_REG_RX_DEFAULT_A_11N		0x858
+#define	ODM_REG_RX_DEFAULT_B_11N		0x85A
 #define	ODM_REG_BB_PWR_SAV3_11N		0x85C
 #define	ODM_REG_ANTSEL_CTRL_11N			0x860
 #define	ODM_REG_RX_ANT_CTRL_11N			0x864
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
index c3051ebaeb78..29c29e2e125b 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
@@ -311,39 +311,21 @@ static void rtl8723bs_recv_tasklet(unsigned long priv)
 				}
 
 				pkt_copy = rtw_skb_alloc(alloc_sz);
-
-				if (pkt_copy) {
-					pkt_copy->dev = padapter->pnetdev;
-					precvframe->u.hdr.pkt = pkt_copy;
-					skb_reserve(pkt_copy, 8 - ((SIZE_PTR)(pkt_copy->data) & 7));/* force pkt_copy->data at 8-byte alignment address */
-					skb_reserve(pkt_copy, shift_sz);/* force ip_hdr at 8-byte alignment address according to shift_sz. */
-					memcpy(pkt_copy->data, (ptr + rx_report_sz + pattrib->shift_sz), skb_len);
-					precvframe->u.hdr.rx_head = pkt_copy->head;
-					precvframe->u.hdr.rx_data = precvframe->u.hdr.rx_tail = pkt_copy->data;
-					precvframe->u.hdr.rx_end = skb_end_pointer(pkt_copy);
-				} else {
-					if ((pattrib->mfrag == 1) && (pattrib->frag_num == 0)) {
-						DBG_8192C("%s: alloc_skb fail, drop frag frame\n", __func__);
-						rtw_free_recvframe(precvframe, &precvpriv->free_recv_queue);
-						break;
-					}
-
-					precvframe->u.hdr.pkt = rtw_skb_clone(precvbuf->pskb);
-					if (precvframe->u.hdr.pkt) {
-						_pkt *pkt_clone = precvframe->u.hdr.pkt;
-
-						pkt_clone->data = ptr + rx_report_sz + pattrib->shift_sz;
-						skb_reset_tail_pointer(pkt_clone);
-						precvframe->u.hdr.rx_head = precvframe->u.hdr.rx_data = precvframe->u.hdr.rx_tail
-							= pkt_clone->data;
-						precvframe->u.hdr.rx_end = pkt_clone->data + skb_len;
-					} else {
-						DBG_8192C("%s: rtw_skb_clone fail\n", __func__);
-						rtw_free_recvframe(precvframe, &precvpriv->free_recv_queue);
-						break;
-					}
+				if (!pkt_copy) {
+					DBG_8192C("%s: alloc_skb fail, drop frame\n", __func__);
+					rtw_free_recvframe(precvframe, &precvpriv->free_recv_queue);
+					break;
 				}
 
+				pkt_copy->dev = padapter->pnetdev;
+				precvframe->u.hdr.pkt = pkt_copy;
+				skb_reserve(pkt_copy, 8 - ((SIZE_PTR)(pkt_copy->data) & 7));/* force pkt_copy->data at 8-byte alignment address */
+				skb_reserve(pkt_copy, shift_sz);/* force ip_hdr at 8-byte alignment address according to shift_sz. */
+				memcpy(pkt_copy->data, (ptr + rx_report_sz + pattrib->shift_sz), skb_len);
+				precvframe->u.hdr.rx_head = pkt_copy->head;
+				precvframe->u.hdr.rx_data = precvframe->u.hdr.rx_tail = pkt_copy->data;
+				precvframe->u.hdr.rx_end = skb_end_pointer(pkt_copy);
+
 				recvframe_put(precvframe, skb_len);
 				/* recvframe_pull(precvframe, drvinfo_sz + RXDESC_SIZE); */
 
diff --git a/drivers/staging/rtl8723bs/hal/sdio_halinit.c b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
index 7853af53051d..e42d8c18e1ae 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_halinit.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
@@ -544,13 +544,9 @@ static void _InitRetryFunction(struct adapter *padapter)
 
 static void HalRxAggr8723BSdio(struct adapter *padapter)
 {
-	struct registry_priv *pregistrypriv;
 	u8 valueDMATimeout;
 	u8 valueDMAPageCount;
 
-
-	pregistrypriv = &padapter->registrypriv;
-
 	valueDMATimeout = 0x06;
 	valueDMAPageCount = 0x06;
 
diff --git a/drivers/staging/rtl8723bs/include/hal_data.h b/drivers/staging/rtl8723bs/include/hal_data.h
index e5e667df6154..fa5d70016f05 100644
--- a/drivers/staging/rtl8723bs/include/hal_data.h
+++ b/drivers/staging/rtl8723bs/include/hal_data.h
@@ -56,9 +56,9 @@ enum RT_AMPDU_BURST {
 /*  Tx Power Limit Table Size */
 #define MAX_REGULATION_NUM			4
 #define MAX_RF_PATH_NUM_IN_POWER_LIMIT_TABLE	4
-#define MAX_2_4G_BANDWITH_NUM			4
+#define MAX_2_4G_BANDWIDTH_NUM			4
 #define MAX_RATE_SECTION_NUM			10
-#define MAX_5G_BANDWITH_NUM			4
+#define MAX_5G_BANDWIDTH_NUM			4
 
 #define MAX_BASE_NUM_IN_PHY_REG_PG_2_4G		10 /*   CCK:1, OFDM:1, HT:4, VHT:4 */
 #define MAX_BASE_NUM_IN_PHY_REG_PG_5G		9 /*  OFDM:1, HT:4, VHT:4 */
@@ -280,14 +280,14 @@ struct hal_com_data {
 
 	/*  Power Limit Table for 2.4G */
 	s8	TxPwrLimit_2_4G[MAX_REGULATION_NUM]
-						[MAX_2_4G_BANDWITH_NUM]
+						[MAX_2_4G_BANDWIDTH_NUM]
 	                                [MAX_RATE_SECTION_NUM]
 	                                [CHANNEL_MAX_NUMBER_2G]
 						[MAX_RF_PATH_NUM];
 
 	/*  Power Limit Table for 5G */
 	s8	TxPwrLimit_5G[MAX_REGULATION_NUM]
-						[MAX_5G_BANDWITH_NUM]
+						[MAX_5G_BANDWIDTH_NUM]
 						[MAX_RATE_SECTION_NUM]
 						[CHANNEL_MAX_NUMBER_5G]
 						[MAX_RF_PATH_NUM];
diff --git a/drivers/staging/rtl8723bs/include/rtw_recv.h b/drivers/staging/rtl8723bs/include/rtw_recv.h
index 98c3e92245b7..a851b818ef0e 100644
--- a/drivers/staging/rtl8723bs/include/rtw_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtw_recv.h
@@ -38,8 +38,6 @@
 #define RX_MAX_QUEUE				2
 
 #define MAX_SUBFRAME_COUNT	64
-extern u8 rtw_rfc1042_header[];
-extern u8 rtw_bridge_tunnel_header[];
 
 /* for Rx reordering buffer control */
 struct recv_reorder_ctrl
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
index cd31ad2b8a7b..2fb80b6eb51d 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
@@ -98,7 +98,7 @@ static struct ieee80211_channel rtw_2ghz_channels[] = {
 
 static void rtw_2g_channels_init(struct ieee80211_channel *channels)
 {
-	memcpy((void*)channels, (void*)rtw_2ghz_channels,
+	memcpy((void *)channels, (void *)rtw_2ghz_channels,
 		sizeof(struct ieee80211_channel)*RTW_2G_CHANNELS_NUM
 	);
 }
@@ -133,8 +133,8 @@ static struct ieee80211_supported_band *rtw_spt_band_alloc(
 	if (!spt_band)
 		goto exit;
 
-	spt_band->channels = (struct ieee80211_channel*)(((u8 *)spt_band)+sizeof(struct ieee80211_supported_band));
-	spt_band->bitrates = (struct ieee80211_rate*)(((u8 *)spt_band->channels)+sizeof(struct ieee80211_channel)*n_channels);
+	spt_band->channels = (struct ieee80211_channel *)(((u8 *)spt_band)+sizeof(struct ieee80211_supported_band));
+	spt_band->bitrates = (struct ieee80211_rate *)(((u8 *)spt_band->channels)+sizeof(struct ieee80211_channel)*n_channels);
 	spt_band->band = band;
 	spt_band->n_channels = n_channels;
 	spt_band->n_bitrates = n_bitrates;
@@ -152,22 +152,6 @@ exit:
 	return spt_band;
 }
 
-static void rtw_spt_band_free(struct ieee80211_supported_band *spt_band)
-{
-	u32 size = 0;
-
-	if (!spt_band)
-		return;
-
-	if (spt_band->band == NL80211_BAND_2GHZ)
-	{
-		size = sizeof(struct ieee80211_supported_band)
-			+ sizeof(struct ieee80211_channel)*RTW_2G_CHANNELS_NUM
-			+ sizeof(struct ieee80211_rate)*RTW_G_RATES_NUM;
-	}
-	kfree(spt_band);
-}
-
 static const struct ieee80211_txrx_stypes
 rtw_cfg80211_default_mgmt_stypes[NUM_NL80211_IFTYPES] = {
 	[NL80211_IFTYPE_ADHOC] = {
@@ -358,7 +342,7 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl
 	memcpy(pbuf, pnetwork->network.IEs, pnetwork->network.IELength);
 	len += pnetwork->network.IELength;
 
-	*((__le64*)pbuf) = cpu_to_le64(notify_timestamp);
+	*((__le64 *)pbuf) = cpu_to_le64(notify_timestamp);
 
 	bss = cfg80211_inform_bss_frame(wiphy, notify_channel, (struct ieee80211_mgmt *)buf,
 		len, notify_signal, GFP_ATOMIC);
@@ -1129,7 +1113,7 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev,
 	else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true)
 	{
 		if (mac_addr)
-			memcpy(param->sta_addr, (void*)mac_addr, ETH_ALEN);
+			memcpy(param->sta_addr, (void *)mac_addr, ETH_ALEN);
 
 		ret = rtw_cfg80211_ap_set_encryption(ndev, param, param_len);
 	}
@@ -2485,7 +2469,7 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc
 		 * for two MAC addresses
 		 */
 		skb_pull(skb, dot11_hdr_len + qos_len + snap_len - sizeof(src_mac_addr) * 2);
-		pdata = (unsigned char*)skb->data;
+		pdata = (unsigned char *)skb->data;
 		memcpy(pdata, dst_mac_addr, sizeof(dst_mac_addr));
 		memcpy(pdata + sizeof(dst_mac_addr), src_mac_addr, sizeof(src_mac_addr));
 
@@ -2540,7 +2524,7 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc
 
 		pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
 
-		memcpy(pframe, (void*)buf, len);
+		memcpy(pframe, (void *)buf, len);
 		pattrib->pktlen = len;
 
 		pwlanhdr = (struct ieee80211_hdr *)pframe;
@@ -3030,7 +3014,7 @@ static int _cfg80211_rtw_mgmt_tx(struct adapter *padapter, u8 tx_ch, const u8 *b
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
 
-	memcpy(pframe, (void*)buf, len);
+	memcpy(pframe, (void *)buf, len);
 	pattrib->pktlen = len;
 
 	pwlanhdr = (struct ieee80211_hdr *)pframe;
@@ -3463,7 +3447,7 @@ void rtw_wdev_free(struct wireless_dev *wdev)
 	if (!wdev)
 		return;
 
-	rtw_spt_band_free(wdev->wiphy->bands[NL80211_BAND_2GHZ]);
+	kfree(wdev->wiphy->bands[NL80211_BAND_2GHZ]);
 
 	wiphy_free(wdev->wiphy);
 
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
index 5059b874080e..902ac8169948 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
@@ -2561,14 +2561,16 @@ static int rtw_wps_start(struct net_device *dev,
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
 	struct iw_point *pdata = &wrqu->data;
 	u32   u32wps_start = 0;
-        unsigned int uintRet = 0;
 
 	if ((true == padapter->bDriverStopped) || (true == padapter->bSurpriseRemoved) || (NULL == pdata)) {
 		ret = -EINVAL;
 		goto exit;
 	}
 
-	uintRet = copy_from_user((void *)&u32wps_start, pdata->pointer, 4);
+	if (copy_from_user((void *)&u32wps_start, pdata->pointer, 4)) {
+		ret = -EFAULT;
+		goto exit;
+	}
 	if (u32wps_start == 0)
 		u32wps_start = *extra;
 
diff --git a/drivers/staging/rtl8723bs/os_dep/os_intfs.c b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
index d29f59bbb613..50a3c2c3a8d2 100644
--- a/drivers/staging/rtl8723bs/os_dep/os_intfs.c
+++ b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
@@ -1057,9 +1057,9 @@ static int pm_netdev_open(struct net_device *pnetdev, u8 bnormal)
 			status = _netdev_open(pnetdev);
 			mutex_unlock(&(adapter_to_dvobj(padapter)->hw_init_mutex));
 		}
-	}
-	else
+	} else {
 		status =  (_SUCCESS == ips_netdrv_open(padapter)) ? (0) : (-1);
+	}
 
 	return status;
 }
@@ -1192,8 +1192,7 @@ void rtw_dev_unload(struct adapter *padapter)
 		padapter->bup = false;
 
 		DBG_871X("<=== %s\n", __func__);
-	}
-	else {
+	} else {
 		RT_TRACE(_module_hci_intfs_c_, _drv_notice_, ("%s: bup ==false\n", __func__));
 		DBG_871X("%s: bup ==false\n", __func__);
 	}
@@ -1223,8 +1222,7 @@ static int rtw_suspend_free_assoc_resource(struct adapter *padapter)
 		rtw_disassoc_cmd(padapter, 0, false);
 		/* s2-2.  indicate disconnect to os */
 		rtw_indicate_disconnect(padapter);
-	}
-	else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 		rtw_sta_flush(padapter);
 	}
 
@@ -1270,9 +1268,8 @@ void rtw_suspend_wow(struct adapter *padapter)
 		padapter->bDriverStopped = false;	/* for 32k command */
 
 		/*  2. disable interrupt */
-		if (padapter->intf_stop) {
+		if (padapter->intf_stop)
 			padapter->intf_stop(padapter);
-		}
 
 		/*  2.1 clean interrupt */
 		if (padapter->HalFunc.clear_interrupt)
@@ -1448,14 +1445,13 @@ int rtw_suspend_common(struct adapter *padapter)
 
 	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true) {
 	#ifdef CONFIG_WOWLAN
-		if (check_fwstate(pmlmepriv, _FW_LINKED)) {
+		if (check_fwstate(pmlmepriv, _FW_LINKED))
 			pwrpriv->wowlan_mode = true;
-		} else if (pwrpriv->wowlan_pno_enable == true) {
+		else if (pwrpriv->wowlan_pno_enable == true)
 			pwrpriv->wowlan_mode |= pwrpriv->wowlan_pno_enable;
-		}
 
 		if (pwrpriv->wowlan_mode == true)
-		rtw_suspend_wow(padapter);
+			rtw_suspend_wow(padapter);
 		else
 			rtw_suspend_normal(padapter);
 
@@ -1522,9 +1518,8 @@ int rtw_resume_process_wow(struct adapter *padapter)
 
 		pwrpriv->bFwCurrentInPSMode = false;
 
-		if (padapter->intf_stop) {
+		if (padapter->intf_stop)
 			padapter->intf_stop(padapter);
-		}
 
 		if (padapter->HalFunc.clear_interrupt)
 			padapter->HalFunc.clear_interrupt(padapter);
@@ -1541,18 +1536,15 @@ int rtw_resume_process_wow(struct adapter *padapter)
 		padapter->HalFunc.SetHwRegHandler(padapter, HW_VAR_WOWLAN, (u8 *)&poidparam);
 
 		psta = rtw_get_stainfo(&padapter->stapriv, get_bssid(&padapter->mlmepriv));
-		if (psta) {
+		if (psta)
 			set_sta_rate(padapter, psta);
-		}
-
 
 		padapter->bDriverStopped = false;
 		DBG_871X("%s: wowmode resuming, DriverStopped:%d\n", __func__, padapter->bDriverStopped);
 		rtw_start_drv_threads(padapter);
 
-		if (padapter->intf_start) {
+		if (padapter->intf_start)
 			padapter->intf_start(padapter);
-		}
 
 		/*  start netif queue */
 		if (pnetdev) {
@@ -1656,9 +1648,8 @@ int rtw_resume_process_ap_wow(struct adapter *padapter)
 	DBG_871X("%s: wowmode resuming, DriverStopped:%d\n", __func__, padapter->bDriverStopped);
 	rtw_start_drv_threads(padapter);
 
-	if (padapter->intf_start) {
+	if (padapter->intf_start)
 		padapter->intf_start(padapter);
-	}
 
 	/*  start netif queue */
 	if (pnetdev) {
diff --git a/drivers/staging/rtl8723bs/os_dep/recv_linux.c b/drivers/staging/rtl8723bs/os_dep/recv_linux.c
index 60c35d92ba29..eb4d1c3008fe 100644
--- a/drivers/staging/rtl8723bs/os_dep/recv_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/recv_linux.c
@@ -9,6 +9,7 @@
 #include <drv_types.h>
 #include <rtw_debug.h>
 #include <linux/jiffies.h>
+#include <net/cfg80211.h>
 
 void rtw_os_free_recvframe(union recv_frame *precvframe)
 {
@@ -60,27 +61,20 @@ _pkt *rtw_os_alloc_msdu_pkt(union recv_frame *prframe, u16 nSubframe_Length, u8
 	pattrib = &prframe->u.hdr.attrib;
 
 	sub_skb = rtw_skb_alloc(nSubframe_Length + 12);
-	if (sub_skb) {
-		skb_reserve(sub_skb, 12);
-		skb_put_data(sub_skb, (pdata + ETH_HLEN), nSubframe_Length);
-	} else {
-		sub_skb = rtw_skb_clone(prframe->u.hdr.pkt);
-		if (sub_skb) {
-			sub_skb->data = pdata + ETH_HLEN;
-			sub_skb->len = nSubframe_Length;
-			skb_set_tail_pointer(sub_skb, nSubframe_Length);
-		} else {
-			DBG_871X("%s(): rtw_skb_clone() Fail!!!\n", __func__);
-			return NULL;
-		}
+	if (!sub_skb) {
+		DBG_871X("%s(): rtw_skb_alloc() Fail!!!\n", __func__);
+		return NULL;
 	}
 
+	skb_reserve(sub_skb, 12);
+	skb_put_data(sub_skb, (pdata + ETH_HLEN), nSubframe_Length);
+
 	eth_type = RTW_GET_BE16(&sub_skb->data[6]);
 
 	if (sub_skb->len >= 8 &&
-		((!memcmp(sub_skb->data, rtw_rfc1042_header, SNAP_SIZE) &&
+		((!memcmp(sub_skb->data, rfc1042_header, SNAP_SIZE) &&
 		  eth_type != ETH_P_AARP && eth_type != ETH_P_IPX) ||
-		 !memcmp(sub_skb->data, rtw_bridge_tunnel_header, SNAP_SIZE))) {
+		 !memcmp(sub_skb->data, bridge_tunnel_header, SNAP_SIZE))) {
 		/*
 		 * remove RFC1042 or Bridge-Tunnel encapsulation and replace
 		 * EtherType
@@ -230,7 +224,7 @@ static void rtw_os_ksocket_send(struct adapter *padapter, union recv_frame *prec
 
 		if (rx_pid == psta->pid) {
 			int i;
-			u16 len = *(u16*)(skb->data+ETH_HLEN+2);
+			u16 len = *(u16 *)(skb->data+ETH_HLEN+2);
 			DBG_871X("eth, RC: len = 0x%x\n", len);
 
 			for (i = 0; i < len; i++)
diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
index b093b5629171..5b1392deb0a7 100644
--- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
+++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
@@ -514,7 +514,7 @@ static void rtw_dev_remove(struct sdio_func *func)
 
 	rtw_unregister_netdevs(dvobj);
 
-	if (padapter->bSurpriseRemoved == false) {
+	if (!padapter->bSurpriseRemoved) {
 		int err;
 
 		/* test surprise remove */
@@ -554,12 +554,12 @@ static int rtw_sdio_suspend(struct device *dev)
 	struct adapter *padapter = psdpriv->if1;
 	struct debug_priv *pdbgpriv = &psdpriv->drv_dbg;
 
-	if (padapter->bDriverStopped == true) {
+	if (padapter->bDriverStopped) {
 		DBG_871X("%s bDriverStopped = %d\n", __func__, padapter->bDriverStopped);
 		return 0;
 	}
 
-	if (pwrpriv->bInSuspend == true) {
+	if (pwrpriv->bInSuspend) {
 		DBG_871X("%s bInSuspend = %d\n", __func__, pwrpriv->bInSuspend);
 		pdbgpriv->dbg_suspend_error_cnt++;
 		return 0;
@@ -574,7 +574,7 @@ static int rtw_resume_process(struct adapter *padapter)
 	struct dvobj_priv *psdpriv = padapter->dvobj;
 	struct debug_priv *pdbgpriv = &psdpriv->drv_dbg;
 
-	if (pwrpriv->bInSuspend == false) {
+	if (!pwrpriv->bInSuspend) {
 		pdbgpriv->dbg_resume_error_cnt++;
 		DBG_871X("%s bInSuspend = %d\n", __func__, pwrpriv->bInSuspend);
 		return -1;
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index 59568d18ce23..a1a82e59dfee 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -297,6 +297,62 @@ static int lynxfb_ops_pan_display(struct fb_var_screeninfo *var,
 	return hw_sm750_pan_display(crtc, var, info);
 }
 
+static inline void lynxfb_set_visual_mode(struct fb_info *info)
+{
+	switch (info->var.bits_per_pixel) {
+	case 8:
+		info->fix.visual = FB_VISUAL_PSEUDOCOLOR;
+		break;
+	case 16:
+	case 24:
+	case 32:
+		info->fix.visual = FB_VISUAL_TRUECOLOR;
+		break;
+	default:
+		break;
+	}
+}
+
+static inline int lynxfb_set_color_offsets(struct fb_info *info)
+{
+	lynxfb_set_visual_mode(info);
+
+	switch (info->var.bits_per_pixel) {
+	case 8:
+		info->var.red.offset = 0;
+		info->var.red.length = 8;
+		info->var.green.offset = 0;
+		info->var.green.length = 8;
+		info->var.blue.offset = 0;
+		info->var.blue.length = 8;
+		info->var.transp.length = 0;
+		info->var.transp.offset = 0;
+		break;
+	case 16:
+		info->var.red.offset = 11;
+		info->var.red.length = 5;
+		info->var.green.offset = 5;
+		info->var.green.length = 6;
+		info->var.blue.offset = 0;
+		info->var.blue.length = 5;
+		info->var.transp.length = 0;
+		info->var.transp.offset = 0;
+		break;
+	case 24:
+	case 32:
+		info->var.red.offset = 16;
+		info->var.red.length = 8;
+		info->var.green.offset = 8;
+		info->var.green.length = 8;
+		info->var.blue.offset = 0;
+		info->var.blue.length = 8;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
 static int lynxfb_ops_set_par(struct fb_info *info)
 {
 	struct lynxfb_par *par;
@@ -328,48 +384,13 @@ static int lynxfb_ops_set_par(struct fb_info *info)
 	 * and these data should be set before setcolreg routine
 	 */
 
-	switch (var->bits_per_pixel) {
-	case 8:
-		fix->visual = FB_VISUAL_PSEUDOCOLOR;
-		var->red.offset = 0;
-		var->red.length = 8;
-		var->green.offset = 0;
-		var->green.length = 8;
-		var->blue.offset = 0;
-		var->blue.length = 8;
-		var->transp.length = 0;
-		var->transp.offset = 0;
-		break;
-	case 16:
-		var->red.offset = 11;
-		var->red.length = 5;
-		var->green.offset = 5;
-		var->green.length = 6;
-		var->blue.offset = 0;
-		var->blue.length = 5;
-		var->transp.length = 0;
-		var->transp.offset = 0;
-		fix->visual = FB_VISUAL_TRUECOLOR;
-		break;
-	case 24:
-	case 32:
-		var->red.offset = 16;
-		var->red.length = 8;
-		var->green.offset = 8;
-		var->green.length = 8;
-		var->blue.offset = 0;
-		var->blue.length = 8;
-		fix->visual = FB_VISUAL_TRUECOLOR;
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
+	ret = lynxfb_set_color_offsets(info);
+
 	var->height = var->width = -1;
 	var->accel_flags = 0;/*FB_ACCELF_TEXT;*/
 
 	if (ret) {
-		pr_err("pixel bpp format not satisfied\n.");
+		pr_err("bpp %d not supported\n", var->bits_per_pixel);
 		return ret;
 	}
 	ret = hw_sm750_crtc_setMode(crtc, var, fix);
@@ -511,10 +532,12 @@ lynxfb_resume_err:
 static int lynxfb_ops_check_var(struct fb_var_screeninfo *var,
 				struct fb_info *info)
 {
+	int ret;
 	struct lynxfb_par *par;
 	struct lynxfb_crtc *crtc;
 	resource_size_t request;
 
+	ret = 0;
 	par = info->par;
 	crtc = &par->crtc;
 
@@ -523,43 +546,13 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var,
 		 var->yres,
 		 var->bits_per_pixel);
 
-	switch (var->bits_per_pixel) {
-	case 8:
-		info->fix.visual = FB_VISUAL_PSEUDOCOLOR;
-		var->red.offset = 0;
-		var->red.length = 8;
-		var->green.offset = 0;
-		var->green.length = 8;
-		var->blue.offset = 0;
-		var->blue.length = 8;
-		var->transp.length = 0;
-		var->transp.offset = 0;
-		break;
-	case 16:
-		var->red.offset = 11;
-		var->red.length = 5;
-		var->green.offset = 5;
-		var->green.length = 6;
-		var->blue.offset = 0;
-		var->blue.length = 5;
-		var->transp.length = 0;
-		var->transp.offset = 0;
-		info->fix.visual = FB_VISUAL_TRUECOLOR;
-		break;
-	case 24:
-	case 32:
-		var->red.offset = 16;
-		var->red.length = 8;
-		var->green.offset = 8;
-		var->green.length = 8;
-		var->blue.offset = 0;
-		var->blue.length = 8;
-		info->fix.visual = FB_VISUAL_TRUECOLOR;
-		break;
-	default:
+	ret = lynxfb_set_color_offsets(info);
+
+	if (ret) {
 		pr_err("bpp %d not supported\n", var->bits_per_pixel);
-		return -EINVAL;
+		return ret;
 	}
+
 	var->height = var->width = -1;
 	var->accel_flags = 0;/* FB_ACCELF_TEXT; */
 
@@ -709,7 +702,9 @@ static int sm750fb_set_drv(struct lynxfb_par *par)
 		} else {
 			output->paths = sm750_crt;
 			crtc->channel = sm750_primary;
-			/* not consider of padding stuffs for oScreen,need fix */
+			/* not consider of padding stuffs for oScreen,
+			 * need fix
+			 */
 			crtc->oScreen = sm750_dev->vidmem_size >> 1;
 			crtc->vScreen = sm750_dev->pvMem + crtc->oScreen;
 		}
@@ -893,15 +888,8 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	pr_info("fix->mmio_start = %lx\n", fix->mmio_start);
 	fix->mmio_len = sm750_dev->vidreg_size;
 	pr_info("fix->mmio_len = %x\n", fix->mmio_len);
-	switch (var->bits_per_pixel) {
-	case 8:
-		fix->visual = FB_VISUAL_PSEUDOCOLOR;
-		break;
-	case 16:
-	case 32:
-		fix->visual = FB_VISUAL_TRUECOLOR;
-		break;
-	}
+
+	lynxfb_set_visual_mode(info);
 
 	/* set var */
 	var->activate = FB_ACTIVATE_NOW;
diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h
index ce90adcb449d..19823c7277a4 100644
--- a/drivers/staging/sm750fb/sm750.h
+++ b/drivers/staging/sm750fb/sm750.h
@@ -59,16 +59,19 @@ struct lynx_accel {
 
 	int (*de_wait)(void);/* see if hardware ready to work */
 
-	int (*de_fillrect)(struct lynx_accel *, u32, u32, u32, u32,
-						u32, u32, u32, u32, u32);
+	int (*de_fillrect)(struct lynx_accel *,
+			   u32, u32, u32, u32,
+			   u32, u32, u32, u32, u32);
 
-	int (*de_copyarea)(struct lynx_accel *, u32, u32, u32, u32,
-						u32, u32, u32, u32,
-						u32, u32, u32, u32);
+	int (*de_copyarea)(struct lynx_accel *,
+			   u32, u32, u32, u32,
+			   u32, u32, u32, u32,
+			   u32, u32, u32, u32);
 
-	int (*de_imageblit)(struct lynx_accel *, const char *, u32, u32, u32, u32,
-							       u32, u32, u32, u32,
-							       u32, u32, u32, u32);
+	int (*de_imageblit)(struct lynx_accel *, const char *,
+			    u32, u32, u32, u32,
+			    u32, u32, u32, u32,
+			    u32, u32, u32, u32);
 
 };
 
@@ -163,7 +166,7 @@ struct lynxfb_output {
 	 */
 	void *priv;
 
-	int (*proc_setBLANK)(struct lynxfb_output*, int);
+	int (*proc_setBLANK)(struct lynxfb_output *output, int blank);
 };
 
 struct lynxfb_par {
diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c
index b8d60701f898..7136d751cff5 100644
--- a/drivers/staging/sm750fb/sm750_hw.c
+++ b/drivers/staging/sm750fb/sm750_hw.c
@@ -51,7 +51,7 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 
 	/* now map mmio and vidmem */
 	sm750_dev->pvReg = ioremap(sm750_dev->vidreg_start,
-					   sm750_dev->vidreg_size);
+				   sm750_dev->vidreg_size);
 	if (!sm750_dev->pvReg) {
 		pr_err("mmio failed\n");
 		ret = -EFAULT;
diff --git a/drivers/staging/speakup/speakup_decext.c b/drivers/staging/speakup/speakup_decext.c
index ddbb7e97d118..7408eb29cf38 100644
--- a/drivers/staging/speakup/speakup_decext.c
+++ b/drivers/staging/speakup/speakup_decext.c
@@ -43,6 +43,7 @@ static struct var_t vars[] = {
 	{ CAPS_STOP, .u.s = {"[:dv ap 100]" } },
 	{ RATE, .u.n = {"[:ra %d]", 7, 0, 9, 150, 25, NULL } },
 	{ PITCH, .u.n = {"[:dv ap %d]", 100, 0, 100, 0, 0, NULL } },
+	{ INFLECTION, .u.n = {"[:dv pr %d] ", 100, 0, 10000, 0, 0, NULL } },
 	{ VOL, .u.n = {"[:dv gv %d]", 13, 0, 16, 0, 5, NULL } },
 	{ PUNCT, .u.n = {"[:pu %c]", 0, 0, 2, 0, 0, "nsa" } },
 	{ VOICE, .u.n = {"[:n%c]", 0, 0, 9, 0, 0, "phfdburwkv" } },
@@ -59,6 +60,8 @@ static struct kobj_attribute caps_stop_attribute =
 	__ATTR(caps_stop, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
 	__ATTR(pitch, 0644, spk_var_show, spk_var_store);
+static struct kobj_attribute inflection_attribute =
+	__ATTR(inflection, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute punct_attribute =
 	__ATTR(punct, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
@@ -87,6 +90,7 @@ static struct attribute *synth_attrs[] = {
 	&caps_start_attribute.attr,
 	&caps_stop_attribute.attr,
 	&pitch_attribute.attr,
+	&inflection_attribute.attr,
 	&punct_attribute.attr,
 	&rate_attribute.attr,
 	&voice_attribute.attr,
diff --git a/drivers/staging/speakup/speakup_decpc.c b/drivers/staging/speakup/speakup_decpc.c
index 798c42dfa16c..96f24c848cc5 100644
--- a/drivers/staging/speakup/speakup_decpc.c
+++ b/drivers/staging/speakup/speakup_decpc.c
@@ -139,6 +139,7 @@ static struct var_t vars[] = {
 	{ CAPS_STOP, .u.s = {"[:dv ap 100]" } },
 	{ RATE, .u.n = {"[:ra %d]", 9, 0, 18, 150, 25, NULL } },
 	{ PITCH, .u.n = {"[:dv ap %d]", 80, 0, 100, 20, 0, NULL } },
+	{ INFLECTION, .u.n = {"[:dv pr %d] ", 100, 0, 10000, 0, 0, NULL } },
 	{ VOL, .u.n = {"[:vo se %d]", 5, 0, 9, 5, 10, NULL } },
 	{ PUNCT, .u.n = {"[:pu %c]", 0, 0, 2, 0, 0, "nsa" } },
 	{ VOICE, .u.n = {"[:n%c]", 0, 0, 9, 0, 0, "phfdburwkv" } },
@@ -155,6 +156,8 @@ static struct kobj_attribute caps_stop_attribute =
 	__ATTR(caps_stop, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
 	__ATTR(pitch, 0644, spk_var_show, spk_var_store);
+static struct kobj_attribute inflection_attribute =
+	__ATTR(inflection, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute punct_attribute =
 	__ATTR(punct, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
@@ -183,6 +186,7 @@ static struct attribute *synth_attrs[] = {
 	&caps_start_attribute.attr,
 	&caps_stop_attribute.attr,
 	&pitch_attribute.attr,
+	&inflection_attribute.attr,
 	&punct_attribute.attr,
 	&rate_attribute.attr,
 	&voice_attribute.attr,
diff --git a/drivers/staging/speakup/speakup_dectlk.c b/drivers/staging/speakup/speakup_dectlk.c
index dccb4ea29d37..780214b5ca16 100644
--- a/drivers/staging/speakup/speakup_dectlk.c
+++ b/drivers/staging/speakup/speakup_dectlk.c
@@ -44,7 +44,7 @@ static struct var_t vars[] = {
 	{ CAPS_START, .u.s = {"[:dv ap 160] " } },
 	{ CAPS_STOP, .u.s = {"[:dv ap 100 ] " } },
 	{ RATE, .u.n = {"[:ra %d] ", 180, 75, 650, 0, 0, NULL } },
-	{ PITCH, .u.n = {"[:dv ap %d] ", 122, 50, 350, 0, 0, NULL } },
+	{ INFLECTION, .u.n = {"[:dv pr %d] ", 100, 0, 10000, 0, 0, NULL } },
 	{ VOL, .u.n = {"[:dv g5 %d] ", 86, 60, 86, 0, 0, NULL } },
 	{ PUNCT, .u.n = {"[:pu %c] ", 0, 0, 2, 0, 0, "nsa" } },
 	{ VOICE, .u.n = {"[:n%c] ", 0, 0, 9, 0, 0, "phfdburwkv" } },
@@ -61,6 +61,8 @@ static struct kobj_attribute caps_stop_attribute =
 	__ATTR(caps_stop, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
 	__ATTR(pitch, 0644, spk_var_show, spk_var_store);
+static struct kobj_attribute inflection_attribute =
+	__ATTR(inflection, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute punct_attribute =
 	__ATTR(punct, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
@@ -89,6 +91,7 @@ static struct attribute *synth_attrs[] = {
 	&caps_start_attribute.attr,
 	&caps_stop_attribute.attr,
 	&pitch_attribute.attr,
+	&inflection_attribute.attr,
 	&punct_attribute.attr,
 	&rate_attribute.attr,
 	&voice_attribute.attr,
diff --git a/drivers/staging/speakup/speakup_dummy.c b/drivers/staging/speakup/speakup_dummy.c
index 7df1a84297f6..e393438af81b 100644
--- a/drivers/staging/speakup/speakup_dummy.c
+++ b/drivers/staging/speakup/speakup_dummy.c
@@ -24,6 +24,7 @@ static struct var_t vars[] = {
 	{ PAUSE, .u.s = {"PAUSE\n"} },
 	{ RATE, .u.n = {"RATE %d\n", 8, 1, 16, 0, 0, NULL } },
 	{ PITCH, .u.n = {"PITCH %d\n", 8, 0, 16, 0, 0, NULL } },
+	{ INFLECTION, .u.n = {"INFLECTION %d\n", 8, 0, 16, 0, 0, NULL } },
 	{ VOL, .u.n = {"VOL %d\n", 8, 0, 16, 0, 0, NULL } },
 	{ TONE, .u.n = {"TONE %d\n", 8, 0, 16, 0, 0, NULL } },
 	{ DIRECT, .u.n = {NULL, 0, 0, 1, 0, 0, NULL } },
@@ -39,6 +40,8 @@ static struct kobj_attribute caps_stop_attribute =
 	__ATTR(caps_stop, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
 	__ATTR(pitch, 0644, spk_var_show, spk_var_store);
+static struct kobj_attribute inflection_attribute =
+	__ATTR(inflection, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
 	__ATTR(rate, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute tone_attribute =
@@ -65,6 +68,7 @@ static struct attribute *synth_attrs[] = {
 	&caps_start_attribute.attr,
 	&caps_stop_attribute.attr,
 	&pitch_attribute.attr,
+	&inflection_attribute.attr,
 	&rate_attribute.attr,
 	&tone_attribute.attr,
 	&vol_attribute.attr,
diff --git a/drivers/staging/speakup/speakup_soft.c b/drivers/staging/speakup/speakup_soft.c
index f591ec095582..9a7029539f35 100644
--- a/drivers/staging/speakup/speakup_soft.c
+++ b/drivers/staging/speakup/speakup_soft.c
@@ -38,6 +38,7 @@ static struct var_t vars[] = {
 	{ PAUSE, .u.n = {"\x01P" } },
 	{ RATE, .u.n = {"\x01%ds", 2, 0, 9, 0, 0, NULL } },
 	{ PITCH, .u.n = {"\x01%dp", 5, 0, 9, 0, 0, NULL } },
+	{ INFLECTION, .u.n = {"\x01%dr", 5, 0, 9, 0, 0, NULL } },
 	{ VOL, .u.n = {"\x01%dv", 5, 0, 9, 0, 0, NULL } },
 	{ TONE, .u.n = {"\x01%dx", 1, 0, 2, 0, 0, NULL } },
 	{ PUNCT, .u.n = {"\x01%db", 0, 0, 2, 0, 0, NULL } },
@@ -57,6 +58,8 @@ static struct kobj_attribute freq_attribute =
 	__ATTR(freq, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute pitch_attribute =
 	__ATTR(pitch, 0644, spk_var_show, spk_var_store);
+static struct kobj_attribute inflection_attribute =
+	__ATTR(inflection, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute punct_attribute =
 	__ATTR(punct, 0644, spk_var_show, spk_var_store);
 static struct kobj_attribute rate_attribute =
@@ -96,6 +99,7 @@ static struct attribute *synth_attrs[] = {
 	&freq_attribute.attr,
 /*	&lang_attribute.attr, */
 	&pitch_attribute.attr,
+	&inflection_attribute.attr,
 	&punct_attribute.attr,
 	&rate_attribute.attr,
 	&tone_attribute.attr,
diff --git a/drivers/staging/speakup/spk_types.h b/drivers/staging/speakup/spk_types.h
index fc6a9416829c..d3272c6d199a 100644
--- a/drivers/staging/speakup/spk_types.h
+++ b/drivers/staging/speakup/spk_types.h
@@ -42,7 +42,8 @@ enum var_id_t {
 	SAY_CONTROL, SAY_WORD_CTL, NO_INTERRUPT, KEY_ECHO,
 	SPELL_DELAY, PUNC_LEVEL, READING_PUNC,
 	ATTRIB_BLEEP, BLEEPS,
-	RATE, PITCH, VOL, TONE, PUNCT, VOICE, FREQUENCY, LANG, DIRECT, PAUSE,
+	RATE, PITCH, INFLECTION, VOL, TONE, PUNCT, VOICE, FREQUENCY, LANG,
+	DIRECT, PAUSE,
 	CAPS_START, CAPS_STOP, CHARTAB,
 	MAXVARS
 };
diff --git a/drivers/staging/speakup/spkguide.txt b/drivers/staging/speakup/spkguide.txt
index c23549c54c3c..1e622cd34363 100644
--- a/drivers/staging/speakup/spkguide.txt
+++ b/drivers/staging/speakup/spkguide.txt
@@ -406,6 +406,7 @@ freq
 full_time
 jiffy_delta
 pitch
+inflection
 punct
 rate
 tone
@@ -518,9 +519,9 @@ All the entries in the Speakup sys system are readable, some are
 writable by root only, and some are writable by everyone.  Unless you
 know what you are doing, you should probably leave the ones that are
 writable by root only alone.  Most of the names are self explanatory.
-Vol for controlling volume, pitch for pitch, rate for controlling speaking
-rate, etc.  If you find one you aren't sure about, you can post a query
-on the Speakup list.
+Vol for controlling volume, pitch for pitch, inflection for pitch range, rate
+for controlling speaking rate, etc.  If you find one you aren't sure about, you
+can post a query on the Speakup list.
 
 6.  Changing Synthesizers
 
diff --git a/drivers/staging/speakup/sysfs-driver-speakup b/drivers/staging/speakup/sysfs-driver-speakup
index be3f5d6962e9..c6a32c434ce9 100644
--- a/drivers/staging/speakup/sysfs-driver-speakup
+++ b/drivers/staging/speakup/sysfs-driver-speakup
@@ -325,6 +325,12 @@ KernelVersion:	2.6
 Contact:	speakup@linux-speakup.org
 Description:	Gets or sets the pitch of the synthesizer. The range is 0-9.
 
+What:		/sys/accessibility/speakup/soft/inflection
+KernelVersion:	5.8
+Contact:	speakup@linux-speakup.org
+Description:	Gets or sets the inflection of the synthesizer, i.e. the pitch
+		range. The range is 0-9.
+
 What:		/sys/accessibility/speakup/soft/punct
 KernelVersion:	2.6
 Contact:	speakup@linux-speakup.org
diff --git a/drivers/staging/speakup/varhandlers.c b/drivers/staging/speakup/varhandlers.c
index 5741d1cb6227..d7f6bec7ff06 100644
--- a/drivers/staging/speakup/varhandlers.c
+++ b/drivers/staging/speakup/varhandlers.c
@@ -37,6 +37,7 @@ static struct st_var_header var_headers[] = {
 	{ "bell_pos", BELL_POS, VAR_NUM, &spk_bell_pos, NULL },
 	{ "rate", RATE, VAR_NUM, NULL, NULL },
 	{ "pitch", PITCH, VAR_NUM, NULL, NULL },
+	{ "inflection", INFLECTION, VAR_NUM, NULL, NULL },
 	{ "vol", VOL, VAR_NUM, NULL, NULL },
 	{ "tone", TONE, VAR_NUM, NULL, NULL },
 	{ "punct", PUNCT, VAR_NUM, NULL, NULL   },
diff --git a/drivers/staging/unisys/visorhba/visorhba_main.c b/drivers/staging/unisys/visorhba/visorhba_main.c
index dd979ee4dcf1..99c57ceeb357 100644
--- a/drivers/staging/unisys/visorhba/visorhba_main.c
+++ b/drivers/staging/unisys/visorhba/visorhba_main.c
@@ -292,7 +292,7 @@ static void cleanup_scsitaskmgmt_handles(struct idr *idrtable,
  * @tasktype: Type of taskmgmt command
  * @scsidev:  Scsidev that issued command
  *
- * Create a cmdrsp packet and send it to the Serivce Partition
+ * Create a cmdrsp packet and send it to the Service Partition
  * that will service this request.
  *
  * Return: Int representing whether command was queued successfully or not
diff --git a/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c b/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c
index 33485184a98a..f783b632141b 100644
--- a/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c
+++ b/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c
@@ -233,7 +233,7 @@ static int snd_bcm2835_pcm_prepare(struct snd_pcm_substream *substream)
 }
 
 static void snd_bcm2835_pcm_transfer(struct snd_pcm_substream *substream,
-	struct snd_pcm_indirect *rec, size_t bytes)
+				     struct snd_pcm_indirect *rec, size_t bytes)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	struct bcm2835_alsa_stream *alsa_stream = runtime->private_data;
@@ -346,7 +346,7 @@ int snd_bcm2835_new_pcm(struct bcm2835_chip *chip, const char *name,
 			&snd_bcm2835_playback_ops);
 
 	snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV,
-		chip->card->dev, 128 * 1024, 128 * 1024);
+				       chip->card->dev, 128 * 1024, 128 * 1024);
 
 	if (spdif)
 		chip->pcm_spdif = pcm;
diff --git a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
index 597acef35d0b..4f1adddb804f 100644
--- a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
+++ b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
@@ -369,8 +369,8 @@ static void buffer_cb(struct vchiq_mmal_instance *instance,
 
 	if (dev->capture.vc_start_timestamp != -1 && pts) {
 		ktime_t timestamp;
-		s64 runtime_us = pts -
-		    dev->capture.vc_start_timestamp;
+		s64 runtime_us = pts - dev->capture.vc_start_timestamp;
+
 		timestamp = ktime_add_us(dev->capture.kernel_start_ts,
 					 runtime_us);
 		v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
@@ -420,9 +420,8 @@ static int enable_camera(struct bm2835_mmal_dev *dev)
 			return -EINVAL;
 		}
 
-		ret = vchiq_mmal_component_enable(
-				dev->instance,
-				dev->component[COMP_CAMERA]);
+		ret = vchiq_mmal_component_enable(dev->instance,
+						  dev->component[COMP_CAMERA]);
 		if (ret < 0) {
 			v4l2_err(&dev->v4l2_dev,
 				 "Failed enabling camera, ret %d\n", ret);
@@ -451,10 +450,8 @@ static int disable_camera(struct bm2835_mmal_dev *dev)
 
 		v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
 			 "Disabling camera\n");
-		ret =
-		    vchiq_mmal_component_disable(
-				dev->instance,
-				dev->component[COMP_CAMERA]);
+		ret = vchiq_mmal_component_disable(dev->instance,
+						   dev->component[COMP_CAMERA]);
 		if (ret < 0) {
 			v4l2_err(&dev->v4l2_dev,
 				 "Failed disabling camera, ret %d\n", ret);
@@ -555,8 +552,8 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	/* enable the camera port */
 	dev->capture.port->cb_ctx = dev;
-	ret =
-	    vchiq_mmal_port_enable(dev->instance, dev->capture.port, buffer_cb);
+	ret = vchiq_mmal_port_enable(dev->instance, dev->capture.port,
+				     buffer_cb);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev,
 			 "Failed to enable capture port - error %d. Disabling camera port again\n",
@@ -668,7 +665,7 @@ static int set_overlay_params(struct bm2835_mmal_dev *dev,
 			MMAL_DISPLAY_SET_ALPHA |
 			MMAL_DISPLAY_SET_DEST_RECT |
 			MMAL_DISPLAY_SET_FULLSCREEN,
-		.layer = PREVIEW_LAYER,
+		.layer = 2,
 		.alpha = dev->overlay.global_alpha,
 		.fullscreen = 0,
 		.dest_rect = {
@@ -767,16 +764,14 @@ static int vidioc_overlay(struct file *file, void *f, unsigned int on)
 	    (!on && !dev->component[COMP_PREVIEW]->enabled))
 		return 0;	/* already in requested state */
 
-	src =
-	    &dev->component[COMP_CAMERA]->output[CAM_PORT_PREVIEW];
+	src = &dev->component[COMP_CAMERA]->output[CAM_PORT_PREVIEW];
 
 	if (!on) {
 		/* disconnect preview ports and disable component */
 		ret = vchiq_mmal_port_disable(dev->instance, src);
 		if (!ret)
-			ret =
-			    vchiq_mmal_port_connect_tunnel(dev->instance, src,
-							   NULL);
+			ret = vchiq_mmal_port_connect_tunnel(dev->instance, src,
+							     NULL);
 		if (ret >= 0)
 			ret = vchiq_mmal_component_disable(
 					dev->instance,
@@ -800,9 +795,8 @@ static int vidioc_overlay(struct file *file, void *f, unsigned int on)
 	if (enable_camera(dev) < 0)
 		return -EINVAL;
 
-	ret = vchiq_mmal_component_enable(
-			dev->instance,
-			dev->component[COMP_PREVIEW]);
+	ret = vchiq_mmal_component_enable(dev->instance,
+					  dev->component[COMP_PREVIEW]);
 	if (ret < 0)
 		return ret;
 
@@ -1001,6 +995,141 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
+
+static int mmal_setup_video_component(struct bm2835_mmal_dev *dev,
+				      struct v4l2_format *f)
+{
+	bool overlay_enabled = !!dev->component[COMP_PREVIEW]->enabled;
+	struct vchiq_mmal_port *preview_port;
+	int ret;
+
+	preview_port = &dev->component[COMP_CAMERA]->output[CAM_PORT_PREVIEW];
+
+	/* Preview and encode ports need to match on resolution */
+	if (overlay_enabled) {
+		/* Need to disable the overlay before we can update
+		 * the resolution
+		 */
+		ret = vchiq_mmal_port_disable(dev->instance, preview_port);
+		if (!ret) {
+			ret = vchiq_mmal_port_connect_tunnel(dev->instance,
+							     preview_port,
+							     NULL);
+		}
+	}
+	preview_port->es.video.width = f->fmt.pix.width;
+	preview_port->es.video.height = f->fmt.pix.height;
+	preview_port->es.video.crop.x = 0;
+	preview_port->es.video.crop.y = 0;
+	preview_port->es.video.crop.width = f->fmt.pix.width;
+	preview_port->es.video.crop.height = f->fmt.pix.height;
+	preview_port->es.video.frame_rate.num =
+				  dev->capture.timeperframe.denominator;
+	preview_port->es.video.frame_rate.den =
+				  dev->capture.timeperframe.numerator;
+	ret = vchiq_mmal_port_set_format(dev->instance, preview_port);
+
+	if (overlay_enabled) {
+		ret = vchiq_mmal_port_connect_tunnel(dev->instance,
+				preview_port,
+				&dev->component[COMP_PREVIEW]->input[0]);
+		if (ret)
+			return ret;
+
+		ret = vchiq_mmal_port_enable(dev->instance, preview_port, NULL);
+	}
+
+	return ret;
+}
+
+static int mmal_setup_encode_component(struct bm2835_mmal_dev *dev,
+				       struct v4l2_format *f,
+				       struct vchiq_mmal_port *port,
+				       struct vchiq_mmal_port *camera_port,
+				       struct vchiq_mmal_component *component)
+{
+	struct mmal_fmt *mfmt = get_format(f);
+	int ret;
+
+	v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
+		 "vid_cap - set up encode comp\n");
+
+	/* configure buffering */
+	camera_port->current_buffer.size = camera_port->recommended_buffer.size;
+	camera_port->current_buffer.num = camera_port->recommended_buffer.num;
+
+	ret = vchiq_mmal_port_connect_tunnel(dev->instance, camera_port,
+					     &component->input[0]);
+	if (ret) {
+		v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
+			 "%s failed to create connection\n", __func__);
+		/* ensure capture is not going to be tried */
+		dev->capture.port = NULL;
+		return ret;
+	}
+
+	port->es.video.width = f->fmt.pix.width;
+	port->es.video.height = f->fmt.pix.height;
+	port->es.video.crop.x = 0;
+	port->es.video.crop.y = 0;
+	port->es.video.crop.width = f->fmt.pix.width;
+	port->es.video.crop.height = f->fmt.pix.height;
+	port->es.video.frame_rate.num =
+		  dev->capture.timeperframe.denominator;
+	port->es.video.frame_rate.den =
+		  dev->capture.timeperframe.numerator;
+
+	port->format.encoding = mfmt->mmal;
+	port->format.encoding_variant = 0;
+	/* Set any encoding specific parameters */
+	switch (mfmt->mmal_component) {
+	case COMP_VIDEO_ENCODE:
+		port->format.bitrate = dev->capture.encode_bitrate;
+		break;
+	case COMP_IMAGE_ENCODE:
+		/* Could set EXIF parameters here */
+		break;
+	default:
+		break;
+	}
+
+	ret = vchiq_mmal_port_set_format(dev->instance, port);
+	if (ret) {
+		v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
+			 "%s failed to set format %dx%d fmt %08X\n",
+			 __func__,
+			 f->fmt.pix.width,
+			 f->fmt.pix.height,
+			 f->fmt.pix.pixelformat);
+		return ret;
+	}
+
+	ret = vchiq_mmal_component_enable(dev->instance, component);
+	if (ret) {
+		v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
+			 "%s Failed to enable encode components\n", __func__);
+		return ret;
+	}
+
+	/* configure buffering */
+	port->current_buffer.num = 1;
+	port->current_buffer.size = f->fmt.pix.sizeimage;
+	if (port->format.encoding == MMAL_ENCODING_JPEG) {
+		v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
+			 "JPG - buf size now %d was %d\n",
+			 f->fmt.pix.sizeimage,
+			 port->current_buffer.size);
+		port->current_buffer.size =
+		    (f->fmt.pix.sizeimage < (100 << 10)) ?
+		    (100 << 10) : f->fmt.pix.sizeimage;
+	}
+	v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
+		 "vid_cap - cur_buf.size set to %d\n", f->fmt.pix.sizeimage);
+	port->current_buffer.alignment = 0;
+
+	return 0;
+}
+
 static int mmal_setup_components(struct bm2835_mmal_dev *dev,
 				 struct v4l2_format *f)
 {
@@ -1075,8 +1204,7 @@ static int mmal_setup_components(struct bm2835_mmal_dev *dev,
 	}
 
 	remove_padding = mfmt->remove_padding;
-	vchiq_mmal_port_parameter_set(dev->instance,
-				      camera_port,
+	vchiq_mmal_port_parameter_set(dev->instance, camera_port,
 				      MMAL_PARAMETER_NO_IMAGE_PADDING,
 				      &remove_padding, sizeof(remove_padding));
 
@@ -1096,46 +1224,7 @@ static int mmal_setup_components(struct bm2835_mmal_dev *dev,
 	if (!ret &&
 	    camera_port ==
 	    &dev->component[COMP_CAMERA]->output[CAM_PORT_VIDEO]) {
-		bool overlay_enabled =
-		    !!dev->component[COMP_PREVIEW]->enabled;
-		struct vchiq_mmal_port *preview_port =
-		    &dev->component[COMP_CAMERA]->output[CAM_PORT_PREVIEW];
-		/* Preview and encode ports need to match on resolution */
-		if (overlay_enabled) {
-			/* Need to disable the overlay before we can update
-			 * the resolution
-			 */
-			ret =
-			    vchiq_mmal_port_disable(dev->instance,
-						    preview_port);
-			if (!ret)
-				ret =
-				    vchiq_mmal_port_connect_tunnel(
-						dev->instance,
-						preview_port,
-						NULL);
-		}
-		preview_port->es.video.width = f->fmt.pix.width;
-		preview_port->es.video.height = f->fmt.pix.height;
-		preview_port->es.video.crop.x = 0;
-		preview_port->es.video.crop.y = 0;
-		preview_port->es.video.crop.width = f->fmt.pix.width;
-		preview_port->es.video.crop.height = f->fmt.pix.height;
-		preview_port->es.video.frame_rate.num =
-					  dev->capture.timeperframe.denominator;
-		preview_port->es.video.frame_rate.den =
-					  dev->capture.timeperframe.numerator;
-		ret = vchiq_mmal_port_set_format(dev->instance, preview_port);
-		if (overlay_enabled) {
-			ret = vchiq_mmal_port_connect_tunnel(
-				dev->instance,
-				preview_port,
-				&dev->component[COMP_PREVIEW]->input[0]);
-			if (!ret)
-				ret = vchiq_mmal_port_enable(dev->instance,
-							     preview_port,
-							     NULL);
-		}
+		ret = mmal_setup_video_component(dev, f);
 	}
 
 	if (ret) {
@@ -1145,128 +1234,39 @@ static int mmal_setup_components(struct bm2835_mmal_dev *dev,
 			 f->fmt.pix.pixelformat);
 		/* ensure capture is not going to be tried */
 		dev->capture.port = NULL;
-	} else {
-		if (encode_component) {
-			v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
-				 "vid_cap - set up encode comp\n");
+		return ret;
+	}
 
-			/* configure buffering */
-			camera_port->current_buffer.size =
-			    camera_port->recommended_buffer.size;
-			camera_port->current_buffer.num =
-			    camera_port->recommended_buffer.num;
+	if (encode_component) {
+		ret = mmal_setup_encode_component(dev, f, port,
+						  camera_port,
+						  encode_component);
 
-			ret =
-			    vchiq_mmal_port_connect_tunnel(
-					dev->instance,
-					camera_port,
-					&encode_component->input[0]);
-			if (ret) {
-				v4l2_dbg(1, bcm2835_v4l2_debug,
-					 &dev->v4l2_dev,
-					 "%s failed to create connection\n",
-					 __func__);
-				/* ensure capture is not going to be tried */
-				dev->capture.port = NULL;
-			} else {
-				port->es.video.width = f->fmt.pix.width;
-				port->es.video.height = f->fmt.pix.height;
-				port->es.video.crop.x = 0;
-				port->es.video.crop.y = 0;
-				port->es.video.crop.width = f->fmt.pix.width;
-				port->es.video.crop.height = f->fmt.pix.height;
-				port->es.video.frame_rate.num =
-					  dev->capture.timeperframe.denominator;
-				port->es.video.frame_rate.den =
-					  dev->capture.timeperframe.numerator;
-
-				port->format.encoding = mfmt->mmal;
-				port->format.encoding_variant = 0;
-				/* Set any encoding specific parameters */
-				switch (mfmt->mmal_component) {
-				case COMP_VIDEO_ENCODE:
-					port->format.bitrate =
-					    dev->capture.encode_bitrate;
-					break;
-				case COMP_IMAGE_ENCODE:
-					/* Could set EXIF parameters here */
-					break;
-				default:
-					break;
-				}
-				ret = vchiq_mmal_port_set_format(dev->instance,
-								 port);
-				if (ret)
-					v4l2_dbg(1, bcm2835_v4l2_debug,
-						 &dev->v4l2_dev,
-						 "%s failed to set format %dx%d fmt %08X\n",
-						 __func__,
-						 f->fmt.pix.width,
-						 f->fmt.pix.height,
-						 f->fmt.pix.pixelformat
-						 );
-			}
+		if (ret)
+			return ret;
+	} else {
+		/* configure buffering */
+		camera_port->current_buffer.num = 1;
+		camera_port->current_buffer.size = f->fmt.pix.sizeimage;
+		camera_port->current_buffer.alignment = 0;
+	}
 
-			if (!ret) {
-				ret = vchiq_mmal_component_enable(
-						dev->instance,
-						encode_component);
-				if (ret) {
-					v4l2_dbg(1, bcm2835_v4l2_debug,
-						 &dev->v4l2_dev,
-						 "%s Failed to enable encode components\n",
-						 __func__);
-				}
-			}
-			if (!ret) {
-				/* configure buffering */
-				port->current_buffer.num = 1;
-				port->current_buffer.size =
-				    f->fmt.pix.sizeimage;
-				if (port->format.encoding ==
-				    MMAL_ENCODING_JPEG) {
-					v4l2_dbg(1, bcm2835_v4l2_debug,
-						 &dev->v4l2_dev,
-						 "JPG - buf size now %d was %d\n",
-						 f->fmt.pix.sizeimage,
-						 port->current_buffer.size);
-					port->current_buffer.size =
-					    (f->fmt.pix.sizeimage <
-					     (100 << 10)) ?
-					    (100 << 10) : f->fmt.pix.sizeimage;
-				}
-				v4l2_dbg(1, bcm2835_v4l2_debug,
-					 &dev->v4l2_dev,
-					 "vid_cap - cur_buf.size set to %d\n",
-					 f->fmt.pix.sizeimage);
-				port->current_buffer.alignment = 0;
-			}
-		} else {
-			/* configure buffering */
-			camera_port->current_buffer.num = 1;
-			camera_port->current_buffer.size = f->fmt.pix.sizeimage;
-			camera_port->current_buffer.alignment = 0;
-		}
+	dev->capture.fmt = mfmt;
+	dev->capture.stride = f->fmt.pix.bytesperline;
+	dev->capture.width = camera_port->es.video.crop.width;
+	dev->capture.height = camera_port->es.video.crop.height;
+	dev->capture.buffersize = port->current_buffer.size;
 
-		if (!ret) {
-			dev->capture.fmt = mfmt;
-			dev->capture.stride = f->fmt.pix.bytesperline;
-			dev->capture.width = camera_port->es.video.crop.width;
-			dev->capture.height = camera_port->es.video.crop.height;
-			dev->capture.buffersize = port->current_buffer.size;
-
-			/* select port for capture */
-			dev->capture.port = port;
-			dev->capture.camera_port = camera_port;
-			dev->capture.encode_component = encode_component;
-			v4l2_dbg(1, bcm2835_v4l2_debug,
-				 &dev->v4l2_dev,
-				"Set dev->capture.fmt %08X, %dx%d, stride %d, size %d",
-				port->format.encoding,
-				dev->capture.width, dev->capture.height,
-				dev->capture.stride, dev->capture.buffersize);
-		}
-	}
+	/* select port for capture */
+	dev->capture.port = port;
+	dev->capture.camera_port = camera_port;
+	dev->capture.encode_component = encode_component;
+	v4l2_dbg(1, bcm2835_v4l2_debug,
+		 &dev->v4l2_dev,
+		"Set dev->capture.fmt %08X, %dx%d, stride %d, size %d",
+		port->format.encoding,
+		dev->capture.width, dev->capture.height,
+		dev->capture.stride, dev->capture.buffersize);
 
 	/* todo: Need to convert the vchiq/mmal error into a v4l2 error. */
 	return ret;
@@ -1658,9 +1658,8 @@ static int mmal_init(struct bm2835_mmal_dev *dev)
 	dev->capture.enc_level = V4L2_MPEG_VIDEO_H264_LEVEL_4_0;
 
 	/* get the preview component ready */
-	ret = vchiq_mmal_component_init(
-			dev->instance, "ril.video_render",
-			&dev->component[COMP_PREVIEW]);
+	ret = vchiq_mmal_component_init(dev->instance, "ril.video_render",
+					&dev->component[COMP_PREVIEW]);
 	if (ret < 0)
 		goto unreg_camera;
 
@@ -1672,9 +1671,8 @@ static int mmal_init(struct bm2835_mmal_dev *dev)
 	}
 
 	/* get the image encoder component ready */
-	ret = vchiq_mmal_component_init(
-		dev->instance, "ril.image_encode",
-		&dev->component[COMP_IMAGE_ENCODE]);
+	ret = vchiq_mmal_component_init(dev->instance, "ril.image_encode",
+					&dev->component[COMP_IMAGE_ENCODE]);
 	if (ret < 0)
 		goto unreg_preview;
 
@@ -1734,15 +1732,13 @@ static int mmal_init(struct bm2835_mmal_dev *dev)
 
 unreg_vid_encoder:
 	pr_err("Cleanup: Destroy video encoder\n");
-	vchiq_mmal_component_finalise(
-		dev->instance,
-		dev->component[COMP_VIDEO_ENCODE]);
+	vchiq_mmal_component_finalise(dev->instance,
+				      dev->component[COMP_VIDEO_ENCODE]);
 
 unreg_image_encoder:
 	pr_err("Cleanup: Destroy image encoder\n");
-	vchiq_mmal_component_finalise(
-		dev->instance,
-		dev->component[COMP_IMAGE_ENCODE]);
+	vchiq_mmal_component_finalise(dev->instance,
+				      dev->component[COMP_IMAGE_ENCODE]);
 
 unreg_preview:
 	pr_err("Cleanup: Destroy video render\n");
@@ -1775,8 +1771,7 @@ static int bm2835_mmal_init_device(struct bm2835_mmal_dev *dev,
 	/* video device needs to be able to access instance data */
 	video_set_drvdata(vfd, dev);
 
-	ret = video_register_device(vfd,
-				    VFL_TYPE_VIDEO,
+	ret = video_register_device(vfd, VFL_TYPE_VIDEO,
 				    video_nr[dev->camera_num]);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.h b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.h
index b5fce38de038..75524adff0f5 100644
--- a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.h
+++ b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.h
@@ -30,79 +30,77 @@ enum {
 	CAM_PORT_COUNT
 };
 
-#define PREVIEW_LAYER      2
-
 extern int bcm2835_v4l2_debug;
 
 struct bm2835_mmal_dev {
 	/* v4l2 devices */
-	struct v4l2_device     v4l2_dev;
-	struct video_device    vdev;
-	struct mutex           mutex;
+	struct v4l2_device v4l2_dev;
+	struct video_device vdev;
+	struct mutex mutex;
 
 	/* controls */
-	struct v4l2_ctrl_handler  ctrl_handler;
-	struct v4l2_ctrl          *ctrls[V4L2_CTRL_COUNT];
-	enum v4l2_scene_mode	  scene_mode;
-	struct mmal_colourfx      colourfx;
-	int                       hflip;
-	int                       vflip;
-	int			  red_gain;
-	int			  blue_gain;
+	struct v4l2_ctrl_handler ctrl_handler;
+	struct v4l2_ctrl *ctrls[V4L2_CTRL_COUNT];
+	enum v4l2_scene_mode scene_mode;
+	struct mmal_colourfx colourfx;
+	int hflip;
+	int vflip;
+	int red_gain;
+	int blue_gain;
 	enum mmal_parameter_exposuremode exposure_mode_user;
 	enum v4l2_exposure_auto_type exposure_mode_v4l2_user;
 	/* active exposure mode may differ if selected via a scene mode */
 	enum mmal_parameter_exposuremode exposure_mode_active;
 	enum mmal_parameter_exposuremeteringmode metering_mode;
-	unsigned int		  manual_shutter_speed;
-	bool			  exp_auto_priority;
+	unsigned int manual_shutter_speed;
+	bool exp_auto_priority;
 	bool manual_iso_enabled;
 	u32 iso;
 
 	/* allocated mmal instance and components */
-	struct vchiq_mmal_instance   *instance;
-	struct vchiq_mmal_component  *component[COMP_COUNT];
+	struct vchiq_mmal_instance *instance;
+	struct vchiq_mmal_component *component[COMP_COUNT];
 	int camera_use_count;
 
 	struct v4l2_window overlay;
 
 	struct {
-		unsigned int     width;  /* width */
-		unsigned int     height;  /* height */
-		unsigned int     stride;  /* stride */
-		unsigned int     buffersize; /* buffer size with padding */
-		struct mmal_fmt  *fmt;
+		unsigned int width;  /* width */
+		unsigned int height;  /* height */
+		unsigned int stride;  /* stride */
+		unsigned int buffersize; /* buffer size with padding */
+		struct mmal_fmt *fmt;
 		struct v4l2_fract timeperframe;
 
 		/* H264 encode bitrate */
-		int         encode_bitrate;
+		int encode_bitrate;
 		/* H264 bitrate mode. CBR/VBR */
-		int         encode_bitrate_mode;
+		int encode_bitrate_mode;
 		/* H264 profile */
 		enum v4l2_mpeg_video_h264_profile enc_profile;
 		/* H264 level */
 		enum v4l2_mpeg_video_h264_level enc_level;
 		/* JPEG Q-factor */
-		int         q_factor;
+		int q_factor;
 
-		struct vb2_queue	vb_vidq;
+		struct vb2_queue vb_vidq;
 
 		/* VC start timestamp for streaming */
-		s64         vc_start_timestamp;
+		s64 vc_start_timestamp;
 		/* Kernel start timestamp for streaming */
 		ktime_t kernel_start_ts;
 		/* Sequence number of last buffer */
-		u32		sequence;
+		u32 sequence;
 
-		struct vchiq_mmal_port  *port; /* port being used for capture */
+		struct vchiq_mmal_port *port; /* port being used for capture */
 		/* camera port being used for capture */
-		struct vchiq_mmal_port  *camera_port;
+		struct vchiq_mmal_port *camera_port;
 		/* component being used for encode */
 		struct vchiq_mmal_component *encode_component;
 		/* number of frames remaining which driver should capture */
-		unsigned int  frame_count;
+		unsigned int frame_count;
 		/* last frame completion */
-		struct completion  frame_cmplt;
+		struct completion frame_cmplt;
 
 	} capture;
 
diff --git a/drivers/staging/vc04_services/bcm2835-camera/controls.c b/drivers/staging/vc04_services/bcm2835-camera/controls.c
index 5137fcf203d6..b096a12387f7 100644
--- a/drivers/staging/vc04_services/bcm2835-camera/controls.c
+++ b/drivers/staging/vc04_services/bcm2835-camera/controls.c
@@ -135,8 +135,8 @@ static const struct v4l2_to_mmal_effects_setting
 };
 
 struct v4l2_mmal_scene_config {
-	enum v4l2_scene_mode			v4l2_scene;
-	enum mmal_parameter_exposuremode	exposure_mode;
+	enum v4l2_scene_mode v4l2_scene;
+	enum mmal_parameter_exposuremode exposure_mode;
 	enum mmal_parameter_exposuremeteringmode metering_mode;
 };
 
@@ -377,11 +377,9 @@ static int ctrl_set_metering_mode(struct bm2835_mmal_dev *dev,
 		dev->metering_mode = MMAL_PARAM_EXPOSUREMETERINGMODE_SPOT;
 		break;
 
-	/* todo matrix weighting not added to Linux API till 3.9
-	 * case V4L2_EXPOSURE_METERING_MATRIX:
-	 *	dev->metering_mode = MMAL_PARAM_EXPOSUREMETERINGMODE_MATRIX;
-	 *	break;
-	 */
+	case V4L2_EXPOSURE_METERING_MATRIX:
+		dev->metering_mode = MMAL_PARAM_EXPOSUREMETERINGMODE_MATRIX;
+		break;
 	}
 
 	if (dev->scene_mode == V4L2_SCENE_MODE_NONE) {
@@ -516,42 +514,41 @@ static int ctrl_set_image_effect(struct bm2835_mmal_dev *dev,
 	struct mmal_parameter_imagefx_parameters imagefx;
 
 	for (i = 0; i < ARRAY_SIZE(v4l2_to_mmal_effects_values); i++) {
-		if (ctrl->val == v4l2_to_mmal_effects_values[i].v4l2_effect) {
-			imagefx.effect =
-				v4l2_to_mmal_effects_values[i].mmal_effect;
-			imagefx.num_effect_params =
-				v4l2_to_mmal_effects_values[i].num_effect_params;
-
-			if (imagefx.num_effect_params > MMAL_MAX_IMAGEFX_PARAMETERS)
-				imagefx.num_effect_params = MMAL_MAX_IMAGEFX_PARAMETERS;
-
-			for (j = 0; j < imagefx.num_effect_params; j++)
-				imagefx.effect_parameter[j] =
-					v4l2_to_mmal_effects_values[i].effect_params[j];
-
-			dev->colourfx.enable =
-				v4l2_to_mmal_effects_values[i].col_fx_enable;
-			if (!v4l2_to_mmal_effects_values[i].col_fx_fixed_cbcr) {
-				dev->colourfx.u =
-					v4l2_to_mmal_effects_values[i].u;
-				dev->colourfx.v =
-					v4l2_to_mmal_effects_values[i].v;
-			}
+		if (ctrl->val != v4l2_to_mmal_effects_values[i].v4l2_effect)
+			continue;
+
+		imagefx.effect =
+			v4l2_to_mmal_effects_values[i].mmal_effect;
+		imagefx.num_effect_params =
+			v4l2_to_mmal_effects_values[i].num_effect_params;
 
-			control = &dev->component[COMP_CAMERA]->control;
+		if (imagefx.num_effect_params > MMAL_MAX_IMAGEFX_PARAMETERS)
+			imagefx.num_effect_params = MMAL_MAX_IMAGEFX_PARAMETERS;
 
-			ret = vchiq_mmal_port_parameter_set(
-					dev->instance, control,
-					MMAL_PARAMETER_IMAGE_EFFECT_PARAMETERS,
-					&imagefx, sizeof(imagefx));
-			if (ret)
-				goto exit;
+		for (j = 0; j < imagefx.num_effect_params; j++)
+			imagefx.effect_parameter[j] =
+				v4l2_to_mmal_effects_values[i].effect_params[j];
 
-			ret = vchiq_mmal_port_parameter_set(
-					dev->instance, control,
-					MMAL_PARAMETER_COLOUR_EFFECT,
-					&dev->colourfx, sizeof(dev->colourfx));
+		dev->colourfx.enable =
+			v4l2_to_mmal_effects_values[i].col_fx_enable;
+		if (!v4l2_to_mmal_effects_values[i].col_fx_fixed_cbcr) {
+			dev->colourfx.u = v4l2_to_mmal_effects_values[i].u;
+			dev->colourfx.v = v4l2_to_mmal_effects_values[i].v;
 		}
+
+		control = &dev->component[COMP_CAMERA]->control;
+
+		ret = vchiq_mmal_port_parameter_set(
+				dev->instance, control,
+				MMAL_PARAMETER_IMAGE_EFFECT_PARAMETERS,
+				&imagefx, sizeof(imagefx));
+		if (ret)
+			goto exit;
+
+		ret = vchiq_mmal_port_parameter_set(
+				dev->instance, control,
+				MMAL_PARAMETER_COLOUR_EFFECT,
+				&dev->colourfx, sizeof(dev->colourfx));
 	}
 
 exit:
@@ -841,8 +838,7 @@ static int ctrl_set_scene_mode(struct bm2835_mmal_dev *dev,
 		enum mmal_parameter_exposuremeteringmode metering_mode;
 
 		for (i = 0; i < ARRAY_SIZE(scene_configs); i++) {
-			if (scene_configs[i].v4l2_scene ==
-				ctrl->val) {
+			if (scene_configs[i].v4l2_scene == ctrl->val) {
 				scene = &scene_configs[i];
 				break;
 			}
@@ -1045,8 +1041,8 @@ static const struct bm2835_mmal_v4l2_ctrl v4l2_ctrls[V4L2_CTRL_COUNT] = {
 	{
 		.id = V4L2_CID_EXPOSURE_METERING,
 		.type = MMAL_CONTROL_TYPE_STD_MENU,
-		.min = ~0x7,
-		.max = V4L2_EXPOSURE_METERING_SPOT,
+		.min = ~0xf,
+		.max = V4L2_EXPOSURE_METERING_MATRIX,
 		.def = V4L2_EXPOSURE_METERING_AVERAGE,
 		.step = 0,
 		.imenu = NULL,
@@ -1282,21 +1278,18 @@ int set_framerate_params(struct bm2835_mmal_dev *dev)
 	struct mmal_parameter_fps_range fps_range;
 	int ret;
 
+	fps_range.fps_high.num = dev->capture.timeperframe.denominator;
+	fps_range.fps_high.den = dev->capture.timeperframe.numerator;
+
 	if ((dev->exposure_mode_active != MMAL_PARAM_EXPOSUREMODE_OFF) &&
 	    (dev->exp_auto_priority)) {
-		/* Variable FPS. Define min FPS as 1fps.
-		 * Max as max defined FPS.
-		 */
+		/* Variable FPS. Define min FPS as 1fps. */
 		fps_range.fps_low.num = 1;
 		fps_range.fps_low.den = 1;
-		fps_range.fps_high.num = dev->capture.timeperframe.denominator;
-		fps_range.fps_high.den = dev->capture.timeperframe.numerator;
 	} else {
 		/* Fixed FPS - set min and max to be the same */
-		fps_range.fps_low.num = fps_range.fps_high.num =
-			dev->capture.timeperframe.denominator;
-		fps_range.fps_low.den = fps_range.fps_high.den =
-			dev->capture.timeperframe.numerator;
+		fps_range.fps_low.num = fps_range.fps_high.num;
+		fps_range.fps_low.den = fps_range.fps_high.den;
 	}
 
 	v4l2_dbg(1, bcm2835_v4l2_debug, &dev->v4l2_dev,
diff --git a/drivers/staging/vc04_services/bcm2835-camera/mmal-common.h b/drivers/staging/vc04_services/bcm2835-camera/mmal-common.h
index ff5398737b4a..ce88fac7c24b 100644
--- a/drivers/staging/vc04_services/bcm2835-camera/mmal-common.h
+++ b/drivers/staging/vc04_services/bcm2835-camera/mmal-common.h
@@ -26,13 +26,13 @@ struct mmal_msg_context;
 
 /* mapping between v4l and mmal video modes */
 struct mmal_fmt {
-	u32   fourcc;          /* v4l2 format id */
-	int   flags;           /* v4l2 flags field */
-	u32   mmal;
-	int   depth;
-	u32   mmal_component;  /* MMAL component index to be used to encode */
-	u32   ybbp;            /* depth of first Y plane for planar formats */
-	bool  remove_padding;  /* Does the GPU have to remove padding,
+	u32 fourcc;          /* v4l2 format id */
+	int flags;           /* v4l2 flags field */
+	u32 mmal;
+	int depth;
+	u32 mmal_component;  /* MMAL component index to be used to encode */
+	u32 ybbp;            /* depth of first Y plane for planar formats */
+	bool remove_padding;   /* Does the GPU have to remove padding,
 				* or can we do hide padding via bytesperline.
 				*/
 };
@@ -40,10 +40,10 @@ struct mmal_fmt {
 /* buffer for one video frame */
 struct mmal_buffer {
 	/* v4l buffer data -- must be first */
-	struct vb2_v4l2_buffer	vb;
+	struct vb2_v4l2_buffer vb;
 
 	/* list of buffers available */
-	struct list_head	list;
+	struct list_head list;
 
 	void *buffer; /* buffer pointer */
 	unsigned long buffer_size; /* size of allocated buffer */
diff --git a/drivers/staging/vc04_services/bcm2835-camera/mmal-parameters.h b/drivers/staging/vc04_services/bcm2835-camera/mmal-parameters.h
index 80a99128f5f3..f4ac5a6149ea 100644
--- a/drivers/staging/vc04_services/bcm2835-camera/mmal-parameters.h
+++ b/drivers/staging/vc04_services/bcm2835-camera/mmal-parameters.h
@@ -724,11 +724,11 @@ struct mmal_parameter_imagefx_parameters {
 #define MMAL_PARAMETER_CAMERA_INFO_MAX_STR_LEN 16
 
 struct mmal_parameter_camera_info_camera_t {
-	u32    port_id;
-	u32    max_width;
-	u32    max_height;
-	u32    lens_present;
-	u8     camera_name[MMAL_PARAMETER_CAMERA_INFO_MAX_STR_LEN];
+	u32 port_id;
+	u32 max_width;
+	u32 max_height;
+	u32 lens_present;
+	u8 camera_name[MMAL_PARAMETER_CAMERA_INFO_MAX_STR_LEN];
 };
 
 enum mmal_parameter_camera_info_flash_type_t {
@@ -744,8 +744,8 @@ struct mmal_parameter_camera_info_flash_t {
 };
 
 struct mmal_parameter_camera_info_t {
-	u32                            num_cameras;
-	u32                            num_flashes;
+	u32 num_cameras;
+	u32 num_flashes;
 	struct mmal_parameter_camera_info_camera_t
 				cameras[MMAL_PARAMETER_CAMERA_INFO_MAX_CAMERAS];
 	struct mmal_parameter_camera_info_flash_t
diff --git a/drivers/staging/vc04_services/interface/vchi/vchi.h b/drivers/staging/vc04_services/interface/vchi/vchi.h
index ff2b960d8cac..1a981e98e82b 100644
--- a/drivers/staging/vc04_services/interface/vchi/vchi.h
+++ b/drivers/staging/vc04_services/interface/vchi/vchi.h
@@ -60,32 +60,18 @@ struct vchi_service_handle;
  * (local / remote)
  *****************************************************************************/
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
 // Routine used to initialise the vchi on both local + remote connections
 extern int32_t vchi_initialise(struct vchi_instance_handle **instance_handle);
 
-extern int32_t vchi_exit(void);
-
 extern int32_t vchi_connect(struct vchi_instance_handle *instance_handle);
 
 //When this is called, ensure that all services have no data pending.
 //Bulk transfers can remain 'queued'
 extern int32_t vchi_disconnect(struct vchi_instance_handle *instance_handle);
 
-// helper functions
-extern void *vchi_allocate_buffer(struct vchi_service_handle *handle, uint32_t *length);
-extern void vchi_free_buffer(struct vchi_service_handle *handle, void *address);
-extern uint32_t vchi_current_time(struct vchi_instance_handle *instance_handle);
-
 /******************************************************************************
  * Global service API
  *****************************************************************************/
-// Routine to destroy a service
-extern int32_t vchi_service_destroy(const struct vchi_service_handle *handle);
-
 // Routine to open a named service
 extern int32_t vchi_service_open(struct vchi_instance_handle *instance_handle,
 				 struct service_creation *setup,
@@ -103,23 +89,12 @@ extern int32_t vchi_service_use(const struct vchi_service_handle *handle);
 // Routine to decrement ref count on a named service
 extern int32_t vchi_service_release(const struct vchi_service_handle *handle);
 
-// Routine to set a control option for a named service
-extern int32_t vchi_service_set_option(const struct vchi_service_handle *handle,
-				       enum vchi_service_option option,
-				       int value);
-
 /* Routine to send a message from kernel memory across a service */
 extern int
 vchi_queue_kernel_message(struct vchi_service_handle *handle,
 			  void *data,
 			  unsigned int size);
 
-/* Routine to send a message from user memory across a service */
-extern int
-vchi_queue_user_message(struct vchi_service_handle *handle,
-			void __user *data,
-			unsigned int size);
-
 // Routine to receive a msg from a service
 // Dequeue is equivalent to hold, copy into client buffer, release
 extern int32_t vchi_msg_dequeue(struct vchi_service_handle *handle,
@@ -149,54 +124,14 @@ extern int32_t vchi_msg_hold(struct vchi_service_handle *handle,
 			     enum vchi_flags flags,
 			     struct vchi_held_msg *message_descriptor);
 
-// Initialise an iterator to look through messages in place
-extern int32_t vchi_msg_look_ahead(struct vchi_service_handle *handle,
-				   struct vchi_msg_iter *iter,
-				   enum vchi_flags flags);
-
 /*******************************************************************************
  * Global service support API - operations on held messages
  * and message iterators
  ******************************************************************************/
 
-// Routine to get the address of a held message
-extern void *vchi_held_msg_ptr(const struct vchi_held_msg *message);
-
-// Routine to get the size of a held message
-extern int32_t vchi_held_msg_size(const struct vchi_held_msg *message);
-
-// Routine to get the transmit timestamp as written into the header by the peer
-extern uint32_t vchi_held_msg_tx_timestamp(const struct vchi_held_msg *message);
-
-// Routine to get the reception timestamp, written as we parsed the header
-extern uint32_t vchi_held_msg_rx_timestamp(const struct vchi_held_msg *message);
-
 // Routine to release a held message after it has been processed
 extern int32_t vchi_held_msg_release(struct vchi_held_msg *message);
 
-// Indicates whether the iterator has a next message.
-extern int32_t vchi_msg_iter_has_next(const struct vchi_msg_iter *iter);
-
-// Return the pointer and length for the next message and advance the iterator.
-extern int32_t vchi_msg_iter_next(struct vchi_msg_iter *iter,
-				  void **data,
-				  uint32_t *msg_size);
-
-// Remove the last message returned by vchi_msg_iter_next.
-// Can only be called once after each call to vchi_msg_iter_next.
-extern int32_t vchi_msg_iter_remove(struct vchi_msg_iter *iter);
-
-// Hold the last message returned by vchi_msg_iter_next.
-// Can only be called once after each call to vchi_msg_iter_next.
-extern int32_t vchi_msg_iter_hold(struct vchi_msg_iter *iter,
-				  struct vchi_held_msg *message);
-
-// Return information for the next message, and hold it, advancing the iterator.
-extern int32_t vchi_msg_iter_hold_next(struct vchi_msg_iter *iter,
-				       void **data,        // } may be NULL
-				       uint32_t *msg_size, // }
-				       struct vchi_held_msg *message);
-
 /******************************************************************************
  * Global bulk API
  *****************************************************************************/
@@ -208,13 +143,6 @@ extern int32_t vchi_bulk_queue_receive(struct vchi_service_handle *handle,
 				       enum vchi_flags flags,
 				       void *transfer_handle);
 
-// Prepare interface for a transfer from the other side into relocatable memory.
-int32_t vchi_bulk_queue_receive_reloc(const struct vchi_service_handle *handle,
-				      uint32_t offset,
-				      uint32_t data_size,
-				      const enum vchi_flags flags,
-				      void * const bulk_handle);
-
 // Routine to queue up data ready for transfer to the other (once they have signalled they are ready)
 extern int32_t vchi_bulk_queue_transmit(struct vchi_service_handle *handle,
 					const void *data_src,
@@ -226,15 +154,6 @@ extern int32_t vchi_bulk_queue_transmit(struct vchi_service_handle *handle,
  * Configuration plumbing
  *****************************************************************************/
 
-#ifdef __cplusplus
-}
-#endif
-
-extern int32_t vchi_bulk_queue_transmit_reloc(struct vchi_service_handle *handle,
-					      uint32_t offset,
-					      uint32_t data_size,
-					      enum vchi_flags flags,
-					      void *transfer_handle);
 #endif /* VCHI_H_ */
 
 /****************************** End of file **********************************/
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
index c18c6ca0b6c0..38a13e4618a8 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c
@@ -371,14 +371,15 @@ create_pagelist(char __user *buf, size_t count, unsigned short type)
 	pagelistinfo->scatterlist = scatterlist;
 	pagelistinfo->scatterlist_mapped = 0;
 
-	if (is_vmalloc_addr(buf)) {
+	if (is_vmalloc_addr((void __force *)buf)) {
 		unsigned long length = count;
 		unsigned int off = offset;
 
 		for (actual_pages = 0; actual_pages < num_pages;
 		     actual_pages++) {
-			struct page *pg = vmalloc_to_page(buf + (actual_pages *
-								 PAGE_SIZE));
+			struct page *pg =
+				vmalloc_to_page((void __force *)(buf +
+						 (actual_pages * PAGE_SIZE)));
 			size_t bytes = PAGE_SIZE - off;
 
 			if (!pg) {
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
index a1ea9777a444..28ea8c3a4cba 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_arm.c
@@ -1209,7 +1209,9 @@ vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 
 					/* The completion must point to the
 					** msgbuf. */
-					completion->header = msgbuf;
+					completion->header =
+						(struct vchiq_header __force *)
+						msgbuf;
 				}
 
 				if ((completion->reason ==
@@ -2353,7 +2355,7 @@ vchiq_use_internal(struct vchiq_state *state, struct vchiq_service *service,
 	enum vchiq_status ret = VCHIQ_SUCCESS;
 	char entity[16];
 	int *entity_uc;
-	int local_uc, local_entity_uc;
+	int local_uc;
 
 	if (!arm_state)
 		goto out;
@@ -2377,7 +2379,7 @@ vchiq_use_internal(struct vchiq_state *state, struct vchiq_service *service,
 
 	write_lock_bh(&arm_state->susp_res_lock);
 	local_uc = ++arm_state->videocore_use_count;
-	local_entity_uc = ++(*entity_uc);
+	++(*entity_uc);
 
 	vchiq_log_trace(vchiq_susp_log_level,
 		"%s %s count %d, state count %d",
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
index 1640906e3929..79b75efa6868 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_connected.c
@@ -14,12 +14,7 @@ static   VCHIQ_CONNECTED_CALLBACK_T g_deferred_callback[MAX_CALLBACKS];
 static   int                        g_once_init;
 static   struct mutex               g_connected_mutex;
 
-/****************************************************************************
-*
-* Function to initialize our lock.
-*
-***************************************************************************/
-
+/* Function to initialize our lock */
 static void connected_init(void)
 {
 	if (!g_once_init) {
@@ -28,15 +23,12 @@ static void connected_init(void)
 	}
 }
 
-/****************************************************************************
-*
-* This function is used to defer initialization until the vchiq stack is
-* initialized. If the stack is already initialized, then the callback will
-* be made immediately, otherwise it will be deferred until
-* vchiq_call_connected_callbacks is called.
-*
-***************************************************************************/
-
+/*
+ * This function is used to defer initialization until the vchiq stack is
+ * initialized. If the stack is already initialized, then the callback will
+ * be made immediately, otherwise it will be deferred until
+ * vchiq_call_connected_callbacks is called.
+ */
 void vchiq_add_connected_callback(VCHIQ_CONNECTED_CALLBACK_T callback)
 {
 	connected_init();
@@ -63,13 +55,10 @@ void vchiq_add_connected_callback(VCHIQ_CONNECTED_CALLBACK_T callback)
 	mutex_unlock(&g_connected_mutex);
 }
 
-/****************************************************************************
-*
-* This function is called by the vchiq stack once it has been connected to
-* the videocore and clients can start to use the stack.
-*
-***************************************************************************/
-
+/*
+ * This function is called by the vchiq stack once it has been connected to
+ * the videocore and clients can start to use the stack.
+ */
 void vchiq_call_connected_callbacks(void)
 {
 	int i;
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
index edcd97373809..ae9183db44ee 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c
@@ -372,6 +372,10 @@ make_service_callback(struct vchiq_service *service, enum vchiq_reason reason,
 			service->state->id, service->handle);
 		status = VCHIQ_SUCCESS;
 	}
+
+	if (reason != VCHIQ_MESSAGE_AVAILABLE)
+		vchiq_release_message(service->handle, header);
+
 	return status;
 }
 
@@ -1480,15 +1484,6 @@ parse_open(struct vchiq_state *state, struct vchiq_header *header)
 					: VCHIQ_SRVSTATE_OPEN);
 			}
 
-			service->remoteport = remoteport;
-			service->client_id = ((int *)header->data)[1];
-			if (make_service_callback(service, VCHIQ_SERVICE_OPENED,
-				NULL, NULL) == VCHIQ_RETRY) {
-				/* Bail out if not ready */
-				service->remoteport = VCHIQ_PORT_FREE;
-				goto bail_not_ready;
-			}
-
 			/* Success - the message has been dealt with */
 			unlock_service(service);
 			return 1;
@@ -3147,6 +3142,12 @@ error_exit:
 	return status;
 }
 
+enum vchiq_status vchiq_queue_kernel_message(unsigned int handle, void *context,
+				      size_t size)
+{
+	return vchiq_queue_message(handle, memcpy_copy_callback, context, size);
+}
+
 void
 vchiq_release_message(unsigned int handle,
 		      struct vchiq_header *header)
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
index cedd8e721aae..1fe6cd8b86c0 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h
@@ -587,6 +587,13 @@ lock_service(struct vchiq_service *service);
 extern void
 unlock_service(struct vchiq_service *service);
 
+extern enum vchiq_status
+vchiq_queue_message(unsigned int handle,
+		    ssize_t (*copy_callback)(void *context, void *dest,
+					     size_t offset, size_t maxsize),
+		    void *context,
+		    size_t size);
+
 /* The following functions are called from vchiq_core, and external
 ** implementations must be provided. */
 
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
index 39b77ea19210..b62fd6d6f1ac 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_if.h
@@ -105,12 +105,8 @@ extern enum vchiq_status vchiq_close_service(unsigned int service);
 extern enum vchiq_status vchiq_remove_service(unsigned int service);
 extern enum vchiq_status vchiq_use_service(unsigned int service);
 extern enum vchiq_status vchiq_release_service(unsigned int service);
-extern enum vchiq_status
-vchiq_queue_message(unsigned int handle,
-		    ssize_t (*copy_callback)(void *context, void *dest,
-					     size_t offset, size_t maxsize),
-		    void *context,
-		    size_t size);
+extern enum vchiq_status vchiq_queue_kernel_message(unsigned int handle,
+						    void *context, size_t size);
 extern void           vchiq_release_message(unsigned int service,
 	struct vchiq_header *header);
 extern enum vchiq_status vchiq_bulk_transmit(unsigned int service,
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
index efdd3b1c7d85..75d87b6992c4 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_shim.c
@@ -9,8 +9,6 @@
 
 #include "vchiq_util.h"
 
-#define vchiq_status_to_vchi(status) ((int32_t)status)
-
 struct shim_service {
 	unsigned int handle;
 
@@ -84,35 +82,15 @@ int32_t vchi_msg_remove(struct vchi_service_handle *handle)
 }
 EXPORT_SYMBOL(vchi_msg_remove);
 
-/***********************************************************
- * Name: vchi_msg_queue
- *
- * Arguments:  struct vchi_service_handle *handle,
- *             ssize_t (*copy_callback)(void *context, void *dest,
- *				        size_t offset, size_t maxsize),
- *	       void *context,
- *             uint32_t data_size
- *
- * Description: Thin wrapper to queue a message onto a connection
- *
- * Returns: int32_t - success == 0
- *
- ***********************************************************/
-static
-int32_t vchi_msg_queue(struct vchi_service_handle *handle,
-	ssize_t (*copy_callback)(void *context, void *dest,
-				 size_t offset, size_t maxsize),
-	void *context,
-	uint32_t data_size)
+int vchi_queue_kernel_message(struct vchi_service_handle *handle, void *data,
+			       unsigned int size)
 {
 	struct shim_service *service = (struct shim_service *)handle;
 	enum vchiq_status status;
 
 	while (1) {
-		status = vchiq_queue_message(service->handle,
-					     copy_callback,
-					     context,
-					     data_size);
+		status = vchiq_queue_kernel_message(service->handle, data,
+						    size);
 
 		/*
 		 * vchiq_queue_message() may return VCHIQ_RETRY, so we need to
@@ -125,65 +103,10 @@ int32_t vchi_msg_queue(struct vchi_service_handle *handle,
 		msleep(1);
 	}
 
-	return vchiq_status_to_vchi(status);
-}
-
-static ssize_t
-vchi_queue_kernel_message_callback(void *context,
-				   void *dest,
-				   size_t offset,
-				   size_t maxsize)
-{
-	memcpy(dest, context + offset, maxsize);
-	return maxsize;
-}
-
-int
-vchi_queue_kernel_message(struct vchi_service_handle *handle,
-			  void *data,
-			  unsigned int size)
-{
-	return vchi_msg_queue(handle,
-			      vchi_queue_kernel_message_callback,
-			      data,
-			      size);
+	return status;
 }
 EXPORT_SYMBOL(vchi_queue_kernel_message);
 
-struct vchi_queue_user_message_context {
-	void __user *data;
-};
-
-static ssize_t
-vchi_queue_user_message_callback(void *context,
-				 void *dest,
-				 size_t offset,
-				 size_t maxsize)
-{
-	struct vchi_queue_user_message_context *copycontext = context;
-
-	if (copy_from_user(dest, copycontext->data + offset, maxsize))
-		return -EFAULT;
-
-	return maxsize;
-}
-
-int
-vchi_queue_user_message(struct vchi_service_handle *handle,
-			void __user *data,
-			unsigned int size)
-{
-	struct vchi_queue_user_message_context copycontext = {
-		.data = data
-	};
-
-	return vchi_msg_queue(handle,
-			      vchi_queue_user_message_callback,
-			      &copycontext,
-			      size);
-}
-EXPORT_SYMBOL(vchi_queue_user_message);
-
 /***********************************************************
  * Name: vchi_bulk_queue_receive
  *
@@ -221,7 +144,7 @@ int32_t vchi_bulk_queue_receive(struct vchi_service_handle *handle, void *data_d
 		break;
 	default:
 		WARN(1, "unsupported message\n");
-		return vchiq_status_to_vchi(VCHIQ_ERROR);
+		return VCHIQ_ERROR;
 	}
 
 	while (1) {
@@ -238,7 +161,7 @@ int32_t vchi_bulk_queue_receive(struct vchi_service_handle *handle, void *data_d
 		msleep(1);
 	}
 
-	return vchiq_status_to_vchi(status);
+	return status;
 }
 EXPORT_SYMBOL(vchi_bulk_queue_receive);
 
@@ -282,7 +205,7 @@ int32_t vchi_bulk_queue_transmit(struct vchi_service_handle *handle,
 		break;
 	default:
 		WARN(1, "unsupported message\n");
-		return vchiq_status_to_vchi(VCHIQ_ERROR);
+		return VCHIQ_ERROR;
 	}
 
 	while (1) {
@@ -300,7 +223,7 @@ int32_t vchi_bulk_queue_transmit(struct vchi_service_handle *handle,
 		msleep(1);
 	}
 
-	return vchiq_status_to_vchi(status);
+	return status;
 }
 EXPORT_SYMBOL(vchi_bulk_queue_transmit);
 
@@ -447,7 +370,7 @@ int32_t vchi_initialise(struct vchi_instance_handle **instance_handle)
 
 	*instance_handle = (struct vchi_instance_handle *)instance;
 
-	return vchiq_status_to_vchi(status);
+	return status;
 }
 EXPORT_SYMBOL(vchi_initialise);
 
@@ -485,7 +408,7 @@ int32_t vchi_disconnect(struct vchi_instance_handle *instance_handle)
 {
 	struct vchiq_instance *instance = (struct vchiq_instance *)instance_handle;
 
-	return vchiq_status_to_vchi(vchiq_shutdown(instance));
+	return vchiq_shutdown(instance);
 }
 EXPORT_SYMBOL(vchi_disconnect);
 
@@ -521,7 +444,7 @@ static enum vchiq_status shim_callback(enum vchiq_reason reason,
 		service->callback(service->callback_param,
 				  VCHI_CALLBACK_MSG_AVAILABLE, NULL);
 
-		goto done;
+		break;
 
 	case VCHIQ_BULK_TRANSMIT_DONE:
 		service->callback(service->callback_param,
@@ -538,10 +461,6 @@ static enum vchiq_status shim_callback(enum vchiq_reason reason,
 				  VCHI_CALLBACK_SERVICE_CLOSED, NULL);
 		break;
 
-	case VCHIQ_SERVICE_OPENED:
-		/* No equivalent VCHI reason */
-		break;
-
 	case VCHIQ_BULK_TRANSMIT_ABORTED:
 		service->callback(service->callback_param,
 				  VCHI_CALLBACK_BULK_TRANSMIT_ABORTED,
@@ -560,8 +479,6 @@ static enum vchiq_status shim_callback(enum vchiq_reason reason,
 	}
 
 release:
-	vchiq_release_message(service->handle, header);
-done:
 	return VCHIQ_SUCCESS;
 }
 
@@ -636,62 +553,12 @@ int32_t vchi_service_close(const struct vchi_service_handle *handle)
 		if (status == VCHIQ_SUCCESS)
 			service_free(service);
 
-		ret = vchiq_status_to_vchi(status);
+		ret = status;
 	}
 	return ret;
 }
 EXPORT_SYMBOL(vchi_service_close);
 
-int32_t vchi_service_destroy(const struct vchi_service_handle *handle)
-{
-	int32_t ret = -1;
-	struct shim_service *service = (struct shim_service *)handle;
-
-	if (service) {
-		enum vchiq_status status = vchiq_remove_service(service->handle);
-
-		if (status == VCHIQ_SUCCESS) {
-			service_free(service);
-			service = NULL;
-		}
-
-		ret = vchiq_status_to_vchi(status);
-	}
-	return ret;
-}
-EXPORT_SYMBOL(vchi_service_destroy);
-
-int32_t vchi_service_set_option(const struct vchi_service_handle *handle,
-				enum vchi_service_option option,
-				int value)
-{
-	int32_t ret = -1;
-	struct shim_service *service = (struct shim_service *)handle;
-	enum vchiq_service_option vchiq_option;
-
-	switch (option) {
-	case VCHI_SERVICE_OPTION_TRACE:
-		vchiq_option = VCHIQ_SERVICE_OPTION_TRACE;
-		break;
-	case VCHI_SERVICE_OPTION_SYNCHRONOUS:
-		vchiq_option = VCHIQ_SERVICE_OPTION_SYNCHRONOUS;
-		break;
-	default:
-		service = NULL;
-		break;
-	}
-	if (service) {
-		enum vchiq_status status =
-			vchiq_set_service_option(service->handle,
-						vchiq_option,
-						value);
-
-		ret = vchiq_status_to_vchi(status);
-	}
-	return ret;
-}
-EXPORT_SYMBOL(vchi_service_set_option);
-
 int32_t vchi_get_peer_version(const struct vchi_service_handle *handle, short *peer_version)
 {
 	int32_t ret = -1;
@@ -701,7 +568,7 @@ int32_t vchi_get_peer_version(const struct vchi_service_handle *handle, short *p
 		enum vchiq_status status;
 
 		status = vchiq_get_peer_version(service->handle, peer_version);
-		ret = vchiq_status_to_vchi(status);
+		ret = status;
 	}
 	return ret;
 }
@@ -723,7 +590,7 @@ int32_t vchi_service_use(const struct vchi_service_handle *handle)
 
 	struct shim_service *service = (struct shim_service *)handle;
 	if (service)
-		ret = vchiq_status_to_vchi(vchiq_use_service(service->handle));
+		ret = vchiq_use_service(service->handle);
 	return ret;
 }
 EXPORT_SYMBOL(vchi_service_use);
@@ -744,8 +611,7 @@ int32_t vchi_service_release(const struct vchi_service_handle *handle)
 
 	struct shim_service *service = (struct shim_service *)handle;
 	if (service)
-		ret = vchiq_status_to_vchi(
-			vchiq_release_service(service->handle));
+		ret = vchiq_release_service(service->handle);
 	return ret;
 }
 EXPORT_SYMBOL(vchi_service_release);
diff --git a/drivers/staging/vt6655/Makefile b/drivers/staging/vt6655/Makefile
index a151f30fc46f..e70357ec0af8 100644
--- a/drivers/staging/vt6655/Makefile
+++ b/drivers/staging/vt6655/Makefile
@@ -1,7 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-# TODO: all of these should be removed
-ccflags-y := -DLINUX -D__KERNEL__ -D__NO_VERSION__
-ccflags-y += -DHOSTAP
 
 vt6655_stage-y +=	device_main.o \
 	card.o \
diff --git a/drivers/staging/vt6655/baseband.c b/drivers/staging/vt6655/baseband.c
index b4cdc0b7fee7..6b25d75d2501 100644
--- a/drivers/staging/vt6655/baseband.c
+++ b/drivers/staging/vt6655/baseband.c
@@ -12,12 +12,10 @@
  * Date: Aug.22, 2002
  *
  * Functions:
- *      BBuGetFrameTime        - Calculate data frame transmitting time
- *      BBvCalculateParameter   - Calculate PhyLength, PhyService and Phy Signal
- *                               parameter for baseband Tx
- *      BBbReadEmbedded         - Embedded read baseband register via MAC
- *      BBbWriteEmbedded        - Embedded write baseband register via MAC
- *      BBbVT3253Init          - VIA VT3253 baseband chip init code
+ *      bb_get_frame_time	 - Calculate data frame transmitting time
+ *      bb_read_embedded	 - Embedded read baseband register via MAC
+ *      bb_write_embedded	 - Embedded write baseband register via MAC
+ *      bb_vt3253_init		 - VIA VT3253 baseband chip init code
  *
  * Revision History:
  *      06-10-2003 Bryan YC Fan:  Re-write codes to support VT3253 spec.
@@ -1695,53 +1693,53 @@ static const unsigned short awcFrameTime[MAX_RATE] = {
  *
  * Parameters:
  *  In:
- *      byPreambleType  - Preamble Type
- *      byPktType        - PK_TYPE_11A, PK_TYPE_11B, PK_TYPE_11GB, PK_TYPE_11GA
- *      cbFrameLength   - Baseband Type
- *      wRate           - Tx Rate
+ *      by_preamble_type  - Preamble Type
+ *      by_pkt_type        - PK_TYPE_11A, PK_TYPE_11B, PK_TYPE_11GB, PK_TYPE_11GA
+ *      cb_frame_length   - Baseband Type
+ *      tx_rate           - Tx Rate
  *  Out:
  *
  * Return Value: FrameTime
  *
  */
-unsigned int BBuGetFrameTime(unsigned char byPreambleType,
-			     unsigned char byPktType,
-			     unsigned int cbFrameLength, unsigned short wRate)
+unsigned int bb_get_frame_time(unsigned char by_preamble_type,
+			       unsigned char by_pkt_type,
+			       unsigned int cb_frame_length,
+			       unsigned short tx_rate)
 {
-	unsigned int uFrameTime;
-	unsigned int uPreamble;
-	unsigned int uTmp;
-	unsigned int uRateIdx = (unsigned int)wRate;
-	unsigned int uRate = 0;
+	unsigned int frame_time;
+	unsigned int preamble;
+	unsigned int tmp;
+	unsigned int rate_idx = (unsigned int)tx_rate;
+	unsigned int rate = 0;
 
-	if (uRateIdx > RATE_54M)
+	if (rate_idx > RATE_54M)
 		return 0;
 
-	uRate = (unsigned int)awcFrameTime[uRateIdx];
+	rate = (unsigned int)awcFrameTime[rate_idx];
 
-	if (uRateIdx <= 3) {          /* CCK mode */
-		if (byPreambleType == 1) /* Short */
-			uPreamble = 96;
+	if (rate_idx <= 3) {		    /* CCK mode */
+		if (by_preamble_type == 1) /* Short */
+			preamble = 96;
 		else
-			uPreamble = 192;
-
-		uFrameTime = (cbFrameLength * 80) / uRate;  /* ????? */
-		uTmp = (uFrameTime * uRate) / 80;
-		if (cbFrameLength != uTmp)
-			uFrameTime++;
+			preamble = 192;
+		frame_time = (cb_frame_length * 80) / rate;  /* ????? */
+		tmp = (frame_time * rate) / 80;
+		if (cb_frame_length != tmp)
+			frame_time++;
 
-		return uPreamble + uFrameTime;
+		return preamble + frame_time;
 	}
-	uFrameTime = (cbFrameLength * 8 + 22) / uRate; /* ???????? */
-	uTmp = ((uFrameTime * uRate) - 22) / 8;
-	if (cbFrameLength != uTmp)
-		uFrameTime++;
+	frame_time = (cb_frame_length * 8 + 22) / rate; /* ???????? */
+	tmp = ((frame_time * rate) - 22) / 8;
+	if (cb_frame_length != tmp)
+		frame_time++;
 
-	uFrameTime = uFrameTime * 4;    /* ??????? */
-	if (byPktType != PK_TYPE_11A)
-		uFrameTime += 6;     /* ?????? */
+	frame_time = frame_time * 4;    /* ??????? */
+	if (by_pkt_type != PK_TYPE_11A)
+		frame_time += 6;     /* ?????? */
 
-	return 20 + uFrameTime; /* ?????? */
+	return 20 + frame_time; /* ?????? */
 }
 
 /*
@@ -1899,34 +1897,34 @@ void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
  * Parameters:
  *  In:
  *      iobase      - I/O base address
- *      byBBAddr    - address of register in Baseband
+ *      by_bb_addr  - address of register in Baseband
  *  Out:
- *      pbyData     - data read
+ *      pby_data    - data read
  *
  * Return Value: true if succeeded; false if failed.
  *
  */
-bool BBbReadEmbedded(struct vnt_private *priv,
-		     unsigned char byBBAddr, unsigned char *pbyData)
+bool bb_read_embedded(struct vnt_private *priv, unsigned char by_bb_addr,
+		      unsigned char *pby_data)
 {
 	void __iomem *iobase = priv->PortOffset;
 	unsigned short ww;
-	unsigned char byValue;
+	unsigned char by_value;
 
 	/* BB reg offset */
-	VNSvOutPortB(iobase + MAC_REG_BBREGADR, byBBAddr);
+	VNSvOutPortB(iobase + MAC_REG_BBREGADR, by_bb_addr);
 
 	/* turn on REGR */
 	MACvRegBitsOn(iobase, MAC_REG_BBREGCTL, BBREGCTL_REGR);
 	/* W_MAX_TIMEOUT is the timeout period */
 	for (ww = 0; ww < W_MAX_TIMEOUT; ww++) {
-		VNSvInPortB(iobase + MAC_REG_BBREGCTL, &byValue);
-		if (byValue & BBREGCTL_DONE)
+		VNSvInPortB(iobase + MAC_REG_BBREGCTL, &by_value);
+		if (by_value & BBREGCTL_DONE)
 			break;
 	}
 
 	/* get BB data */
-	VNSvInPortB(iobase + MAC_REG_BBREGDATA, pbyData);
+	VNSvInPortB(iobase + MAC_REG_BBREGDATA, pby_data);
 
 	if (ww == W_MAX_TIMEOUT) {
 		pr_debug(" DBG_PORT80(0x30)\n");
@@ -1941,32 +1939,32 @@ bool BBbReadEmbedded(struct vnt_private *priv,
  * Parameters:
  *  In:
  *      iobase      - I/O base address
- *      byBBAddr    - address of register in Baseband
- *      byData      - data to write
+ *      by_bb_addr  - address of register in Baseband
+ *      by_data     - data to write
  *  Out:
  *      none
  *
  * Return Value: true if succeeded; false if failed.
  *
  */
-bool BBbWriteEmbedded(struct vnt_private *priv,
-		      unsigned char byBBAddr, unsigned char byData)
+bool bb_write_embedded(struct vnt_private *priv, unsigned char by_bb_addr,
+		       unsigned char by_data)
 {
 	void __iomem *iobase = priv->PortOffset;
 	unsigned short ww;
-	unsigned char byValue;
+	unsigned char by_value;
 
 	/* BB reg offset */
-	VNSvOutPortB(iobase + MAC_REG_BBREGADR, byBBAddr);
+	VNSvOutPortB(iobase + MAC_REG_BBREGADR, by_bb_addr);
 	/* set BB data */
-	VNSvOutPortB(iobase + MAC_REG_BBREGDATA, byData);
+	VNSvOutPortB(iobase + MAC_REG_BBREGDATA, by_data);
 
 	/* turn on BBREGCTL_REGW */
 	MACvRegBitsOn(iobase, MAC_REG_BBREGCTL, BBREGCTL_REGW);
 	/* W_MAX_TIMEOUT is the timeout period */
 	for (ww = 0; ww < W_MAX_TIMEOUT; ww++) {
-		VNSvInPortB(iobase + MAC_REG_BBREGCTL, &byValue);
-		if (byValue & BBREGCTL_DONE)
+		VNSvInPortB(iobase + MAC_REG_BBREGCTL, &by_value);
+		if (by_value & BBREGCTL_DONE)
 			break;
 	}
 
@@ -1992,29 +1990,29 @@ bool BBbWriteEmbedded(struct vnt_private *priv,
  *
  */
 
-bool BBbVT3253Init(struct vnt_private *priv)
+bool bb_vt3253_init(struct vnt_private *priv)
 {
-	bool bResult = true;
+	bool result = true;
 	int        ii;
 	void __iomem *iobase = priv->PortOffset;
-	unsigned char byRFType = priv->byRFType;
-	unsigned char byLocalID = priv->byLocalID;
+	unsigned char by_rf_type = priv->byRFType;
+	unsigned char by_local_id = priv->byLocalID;
 
-	if (byRFType == RF_RFMD2959) {
-		if (byLocalID <= REV_ID_VT3253_A1) {
+	if (by_rf_type == RF_RFMD2959) {
+		if (by_local_id <= REV_ID_VT3253_A1) {
 			for (ii = 0; ii < CB_VT3253_INIT_FOR_RFMD; ii++)
-				bResult &= BBbWriteEmbedded(priv,
+				result &= bb_write_embedded(priv,
 					byVT3253InitTab_RFMD[ii][0],
 					byVT3253InitTab_RFMD[ii][1]);
 
 		} else {
 			for (ii = 0; ii < CB_VT3253B0_INIT_FOR_RFMD; ii++)
-				bResult &= BBbWriteEmbedded(priv,
+				result &= bb_write_embedded(priv,
 					byVT3253B0_RFMD[ii][0],
 					byVT3253B0_RFMD[ii][1]);
 
 			for (ii = 0; ii < CB_VT3253B0_AGC_FOR_RFMD2959; ii++)
-				bResult &= BBbWriteEmbedded(priv,
+				result &= bb_write_embedded(priv,
 					byVT3253B0_AGC4_RFMD2959[ii][0],
 					byVT3253B0_AGC4_RFMD2959[ii][1]);
 
@@ -2029,14 +2027,14 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		priv->ldBmThreshold[1] = -50;
 		priv->ldBmThreshold[2] = 0;
 		priv->ldBmThreshold[3] = 0;
-	} else if ((byRFType == RF_AIROHA) || (byRFType == RF_AL2230S)) {
+	} else if ((by_rf_type == RF_AIROHA) || (by_rf_type == RF_AL2230S)) {
 		for (ii = 0; ii < CB_VT3253B0_INIT_FOR_AIROHA2230; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AIROHA2230[ii][0],
 				byVT3253B0_AIROHA2230[ii][1]);
 
 		for (ii = 0; ii < CB_VT3253B0_AGC; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AGC[ii][0], byVT3253B0_AGC[ii][1]);
 
 		priv->abyBBVGA[0] = 0x1C;
@@ -2047,14 +2045,14 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		priv->ldBmThreshold[1] = -48;
 		priv->ldBmThreshold[2] = 0;
 		priv->ldBmThreshold[3] = 0;
-	} else if (byRFType == RF_UW2451) {
+	} else if (by_rf_type == RF_UW2451) {
 		for (ii = 0; ii < CB_VT3253B0_INIT_FOR_UW2451; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_UW2451[ii][0],
 				byVT3253B0_UW2451[ii][1]);
 
 		for (ii = 0; ii < CB_VT3253B0_AGC; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AGC[ii][0],
 				byVT3253B0_AGC[ii][1]);
 
@@ -2069,9 +2067,9 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		priv->ldBmThreshold[1] = -50;
 		priv->ldBmThreshold[2] = 0;
 		priv->ldBmThreshold[3] = 0;
-	} else if (byRFType == RF_UW2452) {
+	} else if (by_rf_type == RF_UW2452) {
 		for (ii = 0; ii < CB_VT3253B0_INIT_FOR_UW2451; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_UW2451[ii][0],
 				byVT3253B0_UW2451[ii][1]);
 
@@ -2080,7 +2078,7 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		 * 0x45->0x41(VC1/VC2 define, make the ANT_A, ANT_B inverted)
 		 */
 
-		/*bResult &= BBbWriteEmbedded(iobase,0x09,0x41);*/
+		/*bResult &= bb_write_embedded(iobase,0x09,0x41);*/
 
 		/* Init ANT B select,
 		 * RX Config CR10 = 0x28->0x2A,
@@ -2088,23 +2086,23 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		 * make the ANT_A, ANT_B inverted)
 		 */
 
-		/*bResult &= BBbWriteEmbedded(iobase,0x0a,0x28);*/
+		/*bResult &= bb_write_embedded(iobase,0x0a,0x28);*/
 		/* Select VC1/VC2, CR215 = 0x02->0x06 */
-		bResult &= BBbWriteEmbedded(priv, 0xd7, 0x06);
+		result &= bb_write_embedded(priv, 0xd7, 0x06);
 
 		/* {{RobertYu:20050125, request by Jack */
-		bResult &= BBbWriteEmbedded(priv, 0x90, 0x20);
-		bResult &= BBbWriteEmbedded(priv, 0x97, 0xeb);
+		result &= bb_write_embedded(priv, 0x90, 0x20);
+		result &= bb_write_embedded(priv, 0x97, 0xeb);
 		/* }} */
 
 		/* {{RobertYu:20050221, request by Jack */
-		bResult &= BBbWriteEmbedded(priv, 0xa6, 0x00);
-		bResult &= BBbWriteEmbedded(priv, 0xa8, 0x30);
+		result &= bb_write_embedded(priv, 0xa6, 0x00);
+		result &= bb_write_embedded(priv, 0xa8, 0x30);
 		/* }} */
-		bResult &= BBbWriteEmbedded(priv, 0xb0, 0x58);
+		result &= bb_write_embedded(priv, 0xb0, 0x58);
 
 		for (ii = 0; ii < CB_VT3253B0_AGC; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AGC[ii][0], byVT3253B0_AGC[ii][1]);
 
 		priv->abyBBVGA[0] = 0x14;
@@ -2117,14 +2115,14 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		priv->ldBmThreshold[3] = 0;
 		/* }} RobertYu */
 
-	} else if (byRFType == RF_VT3226) {
+	} else if (by_rf_type == RF_VT3226) {
 		for (ii = 0; ii < CB_VT3253B0_INIT_FOR_AIROHA2230; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AIROHA2230[ii][0],
 				byVT3253B0_AIROHA2230[ii][1]);
 
 		for (ii = 0; ii < CB_VT3253B0_AGC; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AGC[ii][0], byVT3253B0_AGC[ii][1]);
 
 		priv->abyBBVGA[0] = 0x1C;
@@ -2138,9 +2136,9 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		/* Fix VT3226 DFC system timing issue */
 		MACvSetRFLE_LatchBase(iobase);
 		/* {{ RobertYu: 20050104 */
-	} else if (byRFType == RF_AIROHA7230) {
+	} else if (by_rf_type == RF_AIROHA7230) {
 		for (ii = 0; ii < CB_VT3253B0_INIT_FOR_AIROHA2230; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AIROHA2230[ii][0],
 				byVT3253B0_AIROHA2230[ii][1]);
 
@@ -2148,17 +2146,17 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		/* Init ANT B select,TX Config CR09 = 0x61->0x45,
 		 * 0x45->0x41(VC1/VC2 define, make the ANT_A, ANT_B inverted)
 		 */
-		/*bResult &= BBbWriteEmbedded(iobase,0x09,0x41);*/
+		/* bResult &= bb_write_embedded(iobase,0x09,0x41);*/
 		/* Init ANT B select,RX Config CR10 = 0x28->0x2A,
 		 * 0x2A->0x28(VC1/VC2 define, make the ANT_A, ANT_B inverted)
 		 */
-		/*bResult &= BBbWriteEmbedded(iobase,0x0a,0x28);*/
+		/* bResult &= BBbWriteEmbedded(iobase,0x0a,0x28);*/
 		/* Select VC1/VC2, CR215 = 0x02->0x06 */
-		bResult &= BBbWriteEmbedded(priv, 0xd7, 0x06);
+		result &= bb_write_embedded(priv, 0xd7, 0x06);
 		/* }} */
 
 		for (ii = 0; ii < CB_VT3253B0_AGC; ii++)
-			bResult &= BBbWriteEmbedded(priv,
+			result &= bb_write_embedded(priv,
 				byVT3253B0_AGC[ii][0], byVT3253B0_AGC[ii][1]);
 
 		priv->abyBBVGA[0] = 0x1C;
@@ -2176,12 +2174,12 @@ bool BBbVT3253Init(struct vnt_private *priv)
 		priv->abyBBVGA[0] = 0x1C;
 	}
 
-	if (byLocalID > REV_ID_VT3253_A1) {
-		BBbWriteEmbedded(priv, 0x04, 0x7F);
-		BBbWriteEmbedded(priv, 0x0D, 0x01);
+	if (by_local_id > REV_ID_VT3253_A1) {
+		bb_write_embedded(priv, 0x04, 0x7F);
+		bb_write_embedded(priv, 0x0D, 0x01);
 	}
 
-	return bResult;
+	return result;
 }
 
 /*
@@ -2197,42 +2195,42 @@ bool BBbVT3253Init(struct vnt_private *priv)
  *
  */
 void
-BBvSetShortSlotTime(struct vnt_private *priv)
+bb_set_short_slot_time(struct vnt_private *priv)
 {
-	unsigned char byBBRxConf = 0;
-	unsigned char byBBVGA = 0;
+	unsigned char by_bb_rx_conf = 0;
+	unsigned char by_bb_vga = 0;
 
-	BBbReadEmbedded(priv, 0x0A, &byBBRxConf); /* CR10 */
+	bb_read_embedded(priv, 0x0A, &by_bb_rx_conf); /* CR10 */
 
 	if (priv->bShortSlotTime)
-		byBBRxConf &= 0xDF; /* 1101 1111 */
+		by_bb_rx_conf &= 0xDF; /* 1101 1111 */
 	else
-		byBBRxConf |= 0x20; /* 0010 0000 */
+		by_bb_rx_conf |= 0x20; /* 0010 0000 */
 
 	/* patch for 3253B0 Baseband with Cardbus module */
-	BBbReadEmbedded(priv, 0xE7, &byBBVGA);
-	if (byBBVGA == priv->abyBBVGA[0])
-		byBBRxConf |= 0x20; /* 0010 0000 */
+	bb_read_embedded(priv, 0xE7, &by_bb_vga);
+	if (by_bb_vga == priv->abyBBVGA[0])
+		by_bb_rx_conf |= 0x20; /* 0010 0000 */
 
-	BBbWriteEmbedded(priv, 0x0A, byBBRxConf); /* CR10 */
+	bb_write_embedded(priv, 0x0A, by_bb_rx_conf); /* CR10 */
 }
 
-void BBvSetVGAGainOffset(struct vnt_private *priv, unsigned char byData)
+void bb_set_vga_gain_offset(struct vnt_private *priv, unsigned char by_data)
 {
-	unsigned char byBBRxConf = 0;
+	unsigned char by_bb_rx_conf = 0;
 
-	BBbWriteEmbedded(priv, 0xE7, byData);
+	bb_write_embedded(priv, 0xE7, by_data);
 
-	BBbReadEmbedded(priv, 0x0A, &byBBRxConf); /* CR10 */
+	bb_read_embedded(priv, 0x0A, &by_bb_rx_conf); /* CR10 */
 	/* patch for 3253B0 Baseband with Cardbus module */
-	if (byData == priv->abyBBVGA[0])
-		byBBRxConf |= 0x20; /* 0010 0000 */
+	if (by_data == priv->abyBBVGA[0])
+		by_bb_rx_conf |= 0x20; /* 0010 0000 */
 	else if (priv->bShortSlotTime)
-		byBBRxConf &= 0xDF; /* 1101 1111 */
+		by_bb_rx_conf &= 0xDF; /* 1101 1111 */
 	else
-		byBBRxConf |= 0x20; /* 0010 0000 */
-	priv->byBBVGACurrent = byData;
-	BBbWriteEmbedded(priv, 0x0A, byBBRxConf); /* CR10 */
+		by_bb_rx_conf |= 0x20; /* 0010 0000 */
+	priv->byBBVGACurrent = by_data;
+	bb_write_embedded(priv, 0x0A, by_bb_rx_conf); /* CR10 */
 }
 
 /*
@@ -2248,12 +2246,12 @@ void BBvSetVGAGainOffset(struct vnt_private *priv, unsigned char byData)
  *
  */
 void
-BBvSoftwareReset(struct vnt_private *priv)
+bb_software_reset(struct vnt_private *priv)
 {
-	BBbWriteEmbedded(priv, 0x50, 0x40);
-	BBbWriteEmbedded(priv, 0x50, 0);
-	BBbWriteEmbedded(priv, 0x9C, 0x01);
-	BBbWriteEmbedded(priv, 0x9C, 0);
+	bb_write_embedded(priv, 0x50, 0x40);
+	bb_write_embedded(priv, 0x50, 0);
+	bb_write_embedded(priv, 0x9C, 0x01);
+	bb_write_embedded(priv, 0x9C, 0);
 }
 
 /*
@@ -2269,13 +2267,13 @@ BBvSoftwareReset(struct vnt_private *priv)
  *
  */
 void
-BBvPowerSaveModeON(struct vnt_private *priv)
+bb_power_save_mode_on(struct vnt_private *priv)
 {
-	unsigned char byOrgData;
+	unsigned char by_org_data;
 
-	BBbReadEmbedded(priv, 0x0D, &byOrgData);
-	byOrgData |= BIT(0);
-	BBbWriteEmbedded(priv, 0x0D, byOrgData);
+	bb_read_embedded(priv, 0x0D, &by_org_data);
+	by_org_data |= BIT(0);
+	bb_write_embedded(priv, 0x0D, by_org_data);
 }
 
 /*
@@ -2291,13 +2289,13 @@ BBvPowerSaveModeON(struct vnt_private *priv)
  *
  */
 void
-BBvPowerSaveModeOFF(struct vnt_private *priv)
+bb_power_save_mode_off(struct vnt_private *priv)
 {
-	unsigned char byOrgData;
+	unsigned char by_org_data;
 
-	BBbReadEmbedded(priv, 0x0D, &byOrgData);
-	byOrgData &= ~(BIT(0));
-	BBbWriteEmbedded(priv, 0x0D, byOrgData);
+	bb_read_embedded(priv, 0x0D, &by_org_data);
+	by_org_data &= ~(BIT(0));
+	bb_write_embedded(priv, 0x0D, by_org_data);
 }
 
 /*
@@ -2306,7 +2304,7 @@ BBvPowerSaveModeOFF(struct vnt_private *priv)
  * Parameters:
  *  In:
  *      priv          - Device Structure
- *      byAntennaMode    - Antenna Mode
+ *      by_antenna_mode    - Antenna Mode
  *  Out:
  *      none
  *
@@ -2315,22 +2313,22 @@ BBvPowerSaveModeOFF(struct vnt_private *priv)
  */
 
 void
-BBvSetTxAntennaMode(struct vnt_private *priv, unsigned char byAntennaMode)
+bb_set_tx_antenna_mode(struct vnt_private *priv, unsigned char by_antenna_mode)
 {
-	unsigned char byBBTxConf;
+	unsigned char by_bb_tx_conf;
 
-	BBbReadEmbedded(priv, 0x09, &byBBTxConf); /* CR09 */
-	if (byAntennaMode == ANT_DIVERSITY) {
+	bb_read_embedded(priv, 0x09, &by_bb_tx_conf); /* CR09 */
+	if (by_antenna_mode == ANT_DIVERSITY) {
 		/* bit 1 is diversity */
-		byBBTxConf |= 0x02;
-	} else if (byAntennaMode == ANT_A) {
+		by_bb_tx_conf |= 0x02;
+	} else if (by_antenna_mode == ANT_A) {
 		/* bit 2 is ANTSEL */
-		byBBTxConf &= 0xF9; /* 1111 1001 */
-	} else if (byAntennaMode == ANT_B) {
-		byBBTxConf &= 0xFD; /* 1111 1101 */
-		byBBTxConf |= 0x04;
+		by_bb_tx_conf &= 0xF9; /* 1111 1001 */
+	} else if (by_antenna_mode == ANT_B) {
+		by_bb_tx_conf &= 0xFD; /* 1111 1101 */
+		by_bb_tx_conf |= 0x04;
 	}
-	BBbWriteEmbedded(priv, 0x09, byBBTxConf); /* CR09 */
+	bb_write_embedded(priv, 0x09, by_bb_tx_conf); /* CR09 */
 }
 
 /*
@@ -2339,7 +2337,7 @@ BBvSetTxAntennaMode(struct vnt_private *priv, unsigned char byAntennaMode)
  * Parameters:
  *  In:
  *      priv          - Device Structure
- *      byAntennaMode    - Antenna Mode
+ *      by_antenna_mode   - Antenna Mode
  *  Out:
  *      none
  *
@@ -2348,25 +2346,25 @@ BBvSetTxAntennaMode(struct vnt_private *priv, unsigned char byAntennaMode)
  */
 
 void
-BBvSetRxAntennaMode(struct vnt_private *priv, unsigned char byAntennaMode)
+bb_set_rx_antenna_mode(struct vnt_private *priv, unsigned char by_antenna_mode)
 {
-	unsigned char byBBRxConf;
+	unsigned char by_bb_rx_conf;
 
-	BBbReadEmbedded(priv, 0x0A, &byBBRxConf); /* CR10 */
-	if (byAntennaMode == ANT_DIVERSITY) {
-		byBBRxConf |= 0x01;
+	bb_read_embedded(priv, 0x0A, &by_bb_rx_conf); /* CR10 */
+	if (by_antenna_mode == ANT_DIVERSITY) {
+		by_bb_rx_conf |= 0x01;
 
-	} else if (byAntennaMode == ANT_A) {
-		byBBRxConf &= 0xFC; /* 1111 1100 */
-	} else if (byAntennaMode == ANT_B) {
-		byBBRxConf &= 0xFE; /* 1111 1110 */
-		byBBRxConf |= 0x02;
+	} else if (by_antenna_mode == ANT_A) {
+		by_bb_rx_conf &= 0xFC; /* 1111 1100 */
+	} else if (by_antenna_mode == ANT_B) {
+		by_bb_rx_conf &= 0xFE; /* 1111 1110 */
+		by_bb_rx_conf |= 0x02;
 	}
-	BBbWriteEmbedded(priv, 0x0A, byBBRxConf); /* CR10 */
+	bb_write_embedded(priv, 0x0A, by_bb_rx_conf); /* CR10 */
 }
 
 /*
- * Description: BBvSetDeepSleep
+ * Description: bb_set_deep_sleep
  *
  * Parameters:
  *  In:
@@ -2378,15 +2376,9 @@ BBvSetRxAntennaMode(struct vnt_private *priv, unsigned char byAntennaMode)
  *
  */
 void
-BBvSetDeepSleep(struct vnt_private *priv, unsigned char byLocalID)
+bb_set_deep_sleep(struct vnt_private *priv, unsigned char by_local_id)
 {
-	BBbWriteEmbedded(priv, 0x0C, 0x17); /* CR12 */
-	BBbWriteEmbedded(priv, 0x0D, 0xB9); /* CR13 */
+	bb_write_embedded(priv, 0x0C, 0x17); /* CR12 */
+	bb_write_embedded(priv, 0x0D, 0xB9); /* CR13 */
 }
 
-void
-BBvExitDeepSleep(struct vnt_private *priv, unsigned char byLocalID)
-{
-	BBbWriteEmbedded(priv, 0x0C, 0x00); /* CR12 */
-	BBbWriteEmbedded(priv, 0x0D, 0x01); /* CR13 */
-}
diff --git a/drivers/staging/vt6655/baseband.h b/drivers/staging/vt6655/baseband.h
index 0cc2e07829c5..9354ce724446 100644
--- a/drivers/staging/vt6655/baseband.h
+++ b/drivers/staging/vt6655/baseband.h
@@ -46,30 +46,31 @@
 #define TOP_RATE_2M         0x00200000
 #define TOP_RATE_1M         0x00100000
 
-unsigned int BBuGetFrameTime(unsigned char byPreambleType,
-			     unsigned char byPktType,
-			     unsigned int cbFrameLength,
-			     unsigned short wRate);
+unsigned int bb_get_frame_time(unsigned char by_preamble_type,
+			       unsigned char by_pkt_type,
+			       unsigned int cb_frame_length,
+			       unsigned short w_rate);
 
 void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
 		       u16 tx_rate, u8 pkt_type, struct vnt_phy_field *phy);
 
-bool BBbReadEmbedded(struct vnt_private *priv, unsigned char byBBAddr,
-		     unsigned char *pbyData);
-bool BBbWriteEmbedded(struct vnt_private *priv, unsigned char byBBAddr,
-		      unsigned char byData);
+bool bb_read_embedded(struct vnt_private *priv, unsigned char by_bb_addr,
+		      unsigned char *pby_data);
+bool bb_write_embedded(struct vnt_private *priv, unsigned char by_bb_addr,
+		       unsigned char by_data);
 
-void BBvSetShortSlotTime(struct vnt_private *priv);
-void BBvSetVGAGainOffset(struct vnt_private *priv, unsigned char byData);
+void bb_set_short_slot_time(struct vnt_private *priv);
+void bb_set_vga_gain_offset(struct vnt_private *priv, unsigned char by_data);
 
 /* VT3253 Baseband */
-bool BBbVT3253Init(struct vnt_private *priv);
-void BBvSoftwareReset(struct vnt_private *priv);
-void BBvPowerSaveModeON(struct vnt_private *priv);
-void BBvPowerSaveModeOFF(struct vnt_private *priv);
-void BBvSetTxAntennaMode(struct vnt_private *priv, unsigned char byAntennaMode);
-void BBvSetRxAntennaMode(struct vnt_private *priv, unsigned char byAntennaMode);
-void BBvSetDeepSleep(struct vnt_private *priv, unsigned char byLocalID);
-void BBvExitDeepSleep(struct vnt_private *priv, unsigned char byLocalID);
+bool bb_vt3253_init(struct vnt_private *priv);
+void bb_software_reset(struct vnt_private *priv);
+void bb_power_save_mode_on(struct vnt_private *priv);
+void bb_power_save_mode_off(struct vnt_private *priv);
+void bb_set_tx_antenna_mode(struct vnt_private *priv,
+			    unsigned char by_antenna_mode);
+void bb_set_rx_antenna_mode(struct vnt_private *priv,
+			    unsigned char by_antenna_mode);
+void bb_set_deep_sleep(struct vnt_private *priv, unsigned char by_local_id);
 
 #endif /* __BASEBAND_H__ */
diff --git a/drivers/staging/vt6655/card.c b/drivers/staging/vt6655/card.c
index e65c9825ea5a..6148310c06d6 100644
--- a/drivers/staging/vt6655/card.c
+++ b/drivers/staging/vt6655/card.c
@@ -11,15 +11,12 @@
  *      CARDvUpdateBasicTopRate - Update BasicTopRate
  *      CARDbAddBasicRate - Add to BasicRateSet
  *      CARDbIsOFDMinBasicRate - Check if any OFDM rate is in BasicRateSet
- *      CARDvSetLoopbackMode - Set Loopback mode
- *      CARDbSoftwareReset - Sortware reset NIC
  *      CARDqGetTSFOffset - Calculate TSFOffset
  *      CARDbGetCurrentTSF - Read Current NIC TSF counter
  *      CARDqGetNextTBTT - Calculate Next Beacon TSF counter
  *      CARDvSetFirstNextTBTT - Set NIC Beacon time
  *      CARDvUpdateNextTBTT - Sync. NIC Beacon time
  *      CARDbRadioPowerOff - Turn Off NIC Radio Power
- *      CARDbRadioPowerOn - Turn On NIC Radio Power
  *
  * Revision History:
  *      06-10-2003 Bryan YC Fan:  Re-write codes to support VT3253 spec.
@@ -198,22 +195,22 @@ bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type)
 			priv->abyBBVGA[0] = 0x20;
 			priv->abyBBVGA[2] = 0x10;
 			priv->abyBBVGA[3] = 0x10;
-			BBbReadEmbedded(priv, 0xE7, &byData);
+			bb_read_embedded(priv, 0xE7, &byData);
 			if (byData == 0x1C)
-				BBbWriteEmbedded(priv, 0xE7, priv->abyBBVGA[0]);
+				bb_write_embedded(priv, 0xE7, priv->abyBBVGA[0]);
 
 		} else if (priv->byRFType == RF_UW2452) {
 			MACvSetBBType(priv->PortOffset, BB_TYPE_11A);
 			priv->abyBBVGA[0] = 0x18;
-			BBbReadEmbedded(priv, 0xE7, &byData);
+			bb_read_embedded(priv, 0xE7, &byData);
 			if (byData == 0x14) {
-				BBbWriteEmbedded(priv, 0xE7, priv->abyBBVGA[0]);
-				BBbWriteEmbedded(priv, 0xE1, 0x57);
+				bb_write_embedded(priv, 0xE7, priv->abyBBVGA[0]);
+				bb_write_embedded(priv, 0xE1, 0x57);
 			}
 		} else {
 			MACvSetBBType(priv->PortOffset, BB_TYPE_11A);
 		}
-		BBbWriteEmbedded(priv, 0x88, 0x03);
+		bb_write_embedded(priv, 0x88, 0x03);
 		bySlot = C_SLOT_SHORT;
 		bySIFS = C_SIFS_A;
 		byDIFS = C_SIFS_A + 2 * C_SLOT_SHORT;
@@ -224,19 +221,19 @@ bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type)
 			priv->abyBBVGA[0] = 0x1C;
 			priv->abyBBVGA[2] = 0x00;
 			priv->abyBBVGA[3] = 0x00;
-			BBbReadEmbedded(priv, 0xE7, &byData);
+			bb_read_embedded(priv, 0xE7, &byData);
 			if (byData == 0x20)
-				BBbWriteEmbedded(priv, 0xE7, priv->abyBBVGA[0]);
+				bb_write_embedded(priv, 0xE7, priv->abyBBVGA[0]);
 
 		} else if (priv->byRFType == RF_UW2452) {
 			priv->abyBBVGA[0] = 0x14;
-			BBbReadEmbedded(priv, 0xE7, &byData);
+			bb_read_embedded(priv, 0xE7, &byData);
 			if (byData == 0x18) {
-				BBbWriteEmbedded(priv, 0xE7, priv->abyBBVGA[0]);
-				BBbWriteEmbedded(priv, 0xE1, 0xD3);
+				bb_write_embedded(priv, 0xE7, priv->abyBBVGA[0]);
+				bb_write_embedded(priv, 0xE1, 0xD3);
 			}
 		}
-		BBbWriteEmbedded(priv, 0x88, 0x02);
+		bb_write_embedded(priv, 0x88, 0x02);
 		bySlot = C_SLOT_LONG;
 		bySIFS = C_SIFS_BG;
 		byDIFS = C_SIFS_BG + 2 * C_SLOT_LONG;
@@ -247,19 +244,19 @@ bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type)
 			priv->abyBBVGA[0] = 0x1C;
 			priv->abyBBVGA[2] = 0x00;
 			priv->abyBBVGA[3] = 0x00;
-			BBbReadEmbedded(priv, 0xE7, &byData);
+			bb_read_embedded(priv, 0xE7, &byData);
 			if (byData == 0x20)
-				BBbWriteEmbedded(priv, 0xE7, priv->abyBBVGA[0]);
+				bb_write_embedded(priv, 0xE7, priv->abyBBVGA[0]);
 
 		} else if (priv->byRFType == RF_UW2452) {
 			priv->abyBBVGA[0] = 0x14;
-			BBbReadEmbedded(priv, 0xE7, &byData);
+			bb_read_embedded(priv, 0xE7, &byData);
 			if (byData == 0x18) {
-				BBbWriteEmbedded(priv, 0xE7, priv->abyBBVGA[0]);
-				BBbWriteEmbedded(priv, 0xE1, 0xD3);
+				bb_write_embedded(priv, 0xE7, priv->abyBBVGA[0]);
+				bb_write_embedded(priv, 0xE1, 0xD3);
 			}
 		}
-		BBbWriteEmbedded(priv, 0x88, 0x08);
+		bb_write_embedded(priv, 0x88, 0x08);
 		bySIFS = C_SIFS_BG;
 
 		if (priv->bShortSlotTime) {
@@ -310,7 +307,7 @@ bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type)
 		priv->bySlot = bySlot;
 		VNSvOutPortB(priv->PortOffset + MAC_REG_SLOT, priv->bySlot);
 
-		BBvSetShortSlotTime(priv);
+		bb_set_short_slot_time(priv);
 	}
 	if (priv->byCWMaxMin != byCWMaxMin) {
 		priv->byCWMaxMin = byCWMaxMin;
@@ -431,7 +428,7 @@ void CARDbRadioPowerOff(struct vnt_private *priv)
 
 	MACvRegBitsOff(priv->PortOffset, MAC_REG_HOSTCR, HOSTCR_RXON);
 
-	BBvSetDeepSleep(priv, priv->byLocalID);
+	bb_set_deep_sleep(priv, priv->byLocalID);
 
 	priv->bRadioOff = true;
 	pr_debug("chester power off\n");
@@ -439,60 +436,6 @@ void CARDbRadioPowerOff(struct vnt_private *priv)
 		      LED_ACTSET);  /* LED issue */
 }
 
-/*
- * Description: Turn on Radio power
- *
- * Parameters:
- *  In:
- *      priv         - The adapter to be turned on
- *  Out:
- *      none
- *
- * Return Value: true if success; otherwise false
- */
-bool CARDbRadioPowerOn(struct vnt_private *priv)
-{
-	bool bResult = true;
-
-	pr_debug("chester power on\n");
-	if (priv->bRadioControlOff) {
-		if (priv->bHWRadioOff)
-			pr_debug("chester bHWRadioOff\n");
-		if (priv->bRadioControlOff)
-			pr_debug("chester bRadioControlOff\n");
-		return false; }
-
-	if (!priv->bRadioOff) {
-		pr_debug("chester pbRadioOff\n");
-		return true; }
-
-	BBvExitDeepSleep(priv, priv->byLocalID);
-
-	MACvRegBitsOn(priv->PortOffset, MAC_REG_HOSTCR, HOSTCR_RXON);
-
-	switch (priv->byRFType) {
-	case RF_RFMD2959:
-		MACvWordRegBitsOn(priv->PortOffset, MAC_REG_SOFTPWRCTL,
-				  SOFTPWRCTL_TXPEINV);
-		MACvWordRegBitsOff(priv->PortOffset, MAC_REG_SOFTPWRCTL,
-				   SOFTPWRCTL_SWPE1);
-		break;
-
-	case RF_AIROHA:
-	case RF_AL2230S:
-	case RF_AIROHA7230:
-		MACvWordRegBitsOn(priv->PortOffset, MAC_REG_SOFTPWRCTL,
-				  (SOFTPWRCTL_SWPE2 | SOFTPWRCTL_SWPE3));
-		break;
-	}
-
-	priv->bRadioOff = false;
-	pr_debug("chester power on\n");
-	MACvRegBitsOff(priv->PortOffset, MAC_REG_GPIOCTL0,
-		       LED_ACTSET); /* LED issue */
-	return bResult;
-}
-
 void CARDvSafeResetTx(struct vnt_private *priv)
 {
 	unsigned int uu;
@@ -816,54 +759,6 @@ unsigned char CARDbyGetPktType(struct vnt_private *priv)
 }
 
 /*
- * Description: Set NIC Loopback mode
- *
- * Parameters:
- *  In:
- *      priv         - The adapter to be set
- *      wLoopbackMode   - Loopback mode to be set
- *  Out:
- *      none
- *
- * Return Value: none
- */
-void CARDvSetLoopbackMode(struct vnt_private *priv,
-			  unsigned short wLoopbackMode)
-{
-	switch (wLoopbackMode) {
-	case CARD_LB_NONE:
-	case CARD_LB_MAC:
-	case CARD_LB_PHY:
-		break;
-	default:
-		break;
-	}
-	/* set MAC loopback */
-	MACvSetLoopbackMode(priv, LOBYTE(wLoopbackMode));
-	/* set Baseband loopback */
-}
-
-/*
- * Description: Software Reset NIC
- *
- * Parameters:
- *  In:
- *      priv         - The adapter to be reset
- *  Out:
- *      none
- *
- * Return Value: none
- */
-bool CARDbSoftwareReset(struct vnt_private *priv)
-{
-	/* reset MAC */
-	if (!MACbSafeSoftwareReset(priv))
-		return false;
-
-	return true;
-}
-
-/*
  * Description: Calculate TSF offset of two TSF input
  *              Get TSF Offset from RxBCN's TSF and local TSF
  *
diff --git a/drivers/staging/vt6655/card.h b/drivers/staging/vt6655/card.h
index 337266add6b2..568a2ddd6588 100644
--- a/drivers/staging/vt6655/card.h
+++ b/drivers/staging/vt6655/card.h
@@ -44,9 +44,6 @@ struct vnt_private;
 void CARDvSetRSPINF(struct vnt_private *priv, u8 bb_type);
 void CARDvUpdateBasicTopRate(struct vnt_private *priv);
 bool CARDbIsOFDMinBasicRate(struct vnt_private *priv);
-void CARDvSetLoopbackMode(struct vnt_private *priv,
-			  unsigned short wLoopbackMode);
-bool CARDbSoftwareReset(struct vnt_private *priv);
 void CARDvSetFirstNextTBTT(struct vnt_private *priv,
 			   unsigned short wBeaconInterval);
 void CARDvUpdateNextTBTT(struct vnt_private *priv, u64 qwTSF,
@@ -58,7 +55,6 @@ unsigned char CARDbyGetPktType(struct vnt_private *priv);
 void CARDvSafeResetTx(struct vnt_private *priv);
 void CARDvSafeResetRx(struct vnt_private *priv);
 void CARDbRadioPowerOff(struct vnt_private *priv);
-bool CARDbRadioPowerOn(struct vnt_private *priv);
 bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type);
 bool CARDbUpdateTSF(struct vnt_private *priv, unsigned char byRxRate,
 		    u64 qwBSSTimestamp);
diff --git a/drivers/staging/vt6655/channel.c b/drivers/staging/vt6655/channel.c
index dec6f0f23b88..62a85c1ca6c4 100644
--- a/drivers/staging/vt6655/channel.c
+++ b/drivers/staging/vt6655/channel.c
@@ -173,7 +173,7 @@ bool set_channel(struct vnt_private *priv, struct ieee80211_channel *ch)
 	    priv->byBBVGACurrent != priv->abyBBVGA[0]) {
 		priv->byBBVGACurrent = priv->abyBBVGA[0];
 
-		BBvSetVGAGainOffset(priv, priv->byBBVGACurrent);
+		bb_set_vga_gain_offset(priv, priv->byBBVGACurrent);
 	}
 
 	/* clear NAV */
@@ -195,7 +195,7 @@ bool set_channel(struct vnt_private *priv, struct ieee80211_channel *ch)
 	if (priv->bEnablePSMode)
 		RFvWriteWakeProgSyn(priv, priv->byRFType, ch->hw_value);
 
-	BBvSoftwareReset(priv);
+	bb_software_reset(priv);
 
 	if (priv->byLocalID > REV_ID_VT3253_B1) {
 		unsigned long flags;
diff --git a/drivers/staging/vt6655/device_main.c b/drivers/staging/vt6655/device_main.c
index 5c86cc60eb5c..41cbec4134b0 100644
--- a/drivers/staging/vt6655/device_main.c
+++ b/drivers/staging/vt6655/device_main.c
@@ -21,18 +21,17 @@
  *   device_alloc_rx_buf - rx buffer pre-allocated function
  *   device_free_rx_buf - free rx buffer function
  *   device_free_tx_buf - free tx buffer function
- *   device_init_rd0_ring- initial rd dma0 ring
- *   device_init_rd1_ring- initial rd dma1 ring
- *   device_init_td0_ring- initial tx dma0 ring buffer
- *   device_init_td1_ring- initial tx dma1 ring buffer
- *   device_init_registers- initial MAC & BBP & RF internal registers.
- *   device_init_rings- initial tx/rx ring buffer
- *   device_free_rings- free all allocated ring buffer
- *   device_tx_srv- tx interrupt service function
+ *   device_init_rd0_ring - initial rd dma0 ring
+ *   device_init_rd1_ring - initial rd dma1 ring
+ *   device_init_td0_ring - initial tx dma0 ring buffer
+ *   device_init_td1_ring - initial tx dma1 ring buffer
+ *   device_init_registers - initial MAC & BBP & RF internal registers.
+ *   device_init_rings - initial tx/rx ring buffer
+ *   device_free_rings - free all allocated ring buffer
+ *   device_tx_srv - tx interrupt service function
  *
  * Revision History:
  */
-#undef __NO_VERSION__
 
 #include <linux/file.h>
 #include "device.h"
@@ -202,7 +201,7 @@ static void device_init_registers(struct vnt_private *priv)
 	unsigned char byOFDMPwrdBm = 0;
 
 	MACbShutdown(priv);
-	BBvSoftwareReset(priv);
+	bb_software_reset(priv);
 
 	/* Do MACbSoftwareReset in MACvInitialize */
 	MACbSoftwareReset(priv);
@@ -279,8 +278,8 @@ static void device_init_registers(struct vnt_private *priv)
 	}
 
 	/* Set initial antenna mode */
-	BBvSetTxAntennaMode(priv, priv->byTxAntennaMode);
-	BBvSetRxAntennaMode(priv, priv->byRxAntennaMode);
+	bb_set_tx_antenna_mode(priv, priv->byTxAntennaMode);
+	bb_set_rx_antenna_mode(priv, priv->byRxAntennaMode);
 
 	/* zonetype initial */
 	priv->byOriginalZonetype = priv->abyEEPROM[EEP_OFS_ZONETYPE];
@@ -357,16 +356,16 @@ static void device_init_registers(struct vnt_private *priv)
 	VNSvOutPortB(priv->PortOffset + MAC_REG_TFTCTL, TFTCTL_TSFCNTREN);
 
 	/* initialize BBP registers */
-	BBbVT3253Init(priv);
+	bb_vt3253_init(priv);
 
 	if (priv->bUpdateBBVGA) {
 		priv->byBBVGACurrent = priv->abyBBVGA[0];
 		priv->byBBVGANew = priv->byBBVGACurrent;
-		BBvSetVGAGainOffset(priv, priv->abyBBVGA[0]);
+		bb_set_vga_gain_offset(priv, priv->abyBBVGA[0]);
 	}
 
-	BBvSetRxAntennaMode(priv, priv->byRxAntennaMode);
-	BBvSetTxAntennaMode(priv, priv->byTxAntennaMode);
+	bb_set_rx_antenna_mode(priv, priv->byRxAntennaMode);
+	bb_set_tx_antenna_mode(priv, priv->byTxAntennaMode);
 
 	/* Set BB and packet type at the same time. */
 	/* Set Short Slot Time, xIFS, and RSPINF. */
@@ -1001,7 +1000,7 @@ static void vnt_check_bb_vga(struct vnt_private *priv)
 
 	if (priv->uBBVGADiffCount == 1) {
 		/* first VGA diff gain */
-		BBvSetVGAGainOffset(priv, priv->byBBVGANew);
+		bb_set_vga_gain_offset(priv, priv->byBBVGANew);
 
 		dev_dbg(&priv->pcid->dev,
 			"First RSSI[%d] NewGain[%d] OldGain[%d] Count[%d]\n",
@@ -1017,7 +1016,7 @@ static void vnt_check_bb_vga(struct vnt_private *priv)
 			priv->byBBVGACurrent,
 			(int)priv->uBBVGADiffCount);
 
-		BBvSetVGAGainOffset(priv, priv->byBBVGANew);
+		bb_set_vga_gain_offset(priv, priv->byBBVGANew);
 	}
 }
 
@@ -1445,7 +1444,7 @@ static void vnt_bss_info_changed(struct ieee80211_hw *hw,
 			priv->bShortSlotTime = false;
 
 		CARDbSetPhyParameter(priv, priv->byBBType);
-		BBvSetVGAGainOffset(priv, priv->abyBBVGA[0]);
+		bb_set_vga_gain_offset(priv, priv->abyBBVGA[0]);
 	}
 
 	if (changed & BSS_CHANGED_TXPOWER)
diff --git a/drivers/staging/vt6655/rf.c b/drivers/staging/vt6655/rf.c
index d6ca6e5551a7..747d79265a7c 100644
--- a/drivers/staging/vt6655/rf.c
+++ b/drivers/staging/vt6655/rf.c
@@ -419,7 +419,7 @@ static bool s_bAL7230Init(struct vnt_private *priv)
 
 	MACvWordRegBitsOn(iobase, MAC_REG_SOFTPWRCTL, (SOFTPWRCTL_SWPECTI  |
 							 SOFTPWRCTL_TXPEINV));
-	BBvPowerSaveModeOFF(priv); /* RobertYu:20050106, have DC value for Calibration */
+	bb_power_save_mode_off(priv); /* RobertYu:20050106, have DC value for Calibration */
 
 	for (ii = 0; ii < CB_AL7230_INIT_SEQ; ii++)
 		ret &= IFRFbWriteEmbedded(priv, dwAL7230InitTable[ii]);
@@ -443,7 +443,7 @@ static bool s_bAL7230Init(struct vnt_private *priv)
 							 SOFTPWRCTL_SWPECTI  |
 							 SOFTPWRCTL_TXPEINV));
 
-	BBvPowerSaveModeON(priv); /* RobertYu:20050106 */
+	bb_power_save_mode_on(priv); /* RobertYu:20050106 */
 
 	/* PE1: TX_ON, PE2: RX_ON, PE3: PLLON */
 	/* 3-wire control for power saving mode */
diff --git a/drivers/staging/vt6655/rxtx.c b/drivers/staging/vt6655/rxtx.c
index 37fcc42ed000..cfab64d2b312 100644
--- a/drivers/staging/vt6655/rxtx.c
+++ b/drivers/staging/vt6655/rxtx.c
@@ -165,16 +165,21 @@ s_uGetTxRsvTime(
 {
 	unsigned int uDataTime, uAckTime;
 
-	uDataTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, cbFrameLength, wRate);
-	if (byPktType == PK_TYPE_11B) /* llb,CCK mode */
-		uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, (unsigned short)pDevice->byTopCCKBasicRate);
-	else /* 11g 2.4G OFDM mode & 11a 5G OFDM mode */
-		uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, (unsigned short)pDevice->byTopOFDMBasicRate);
-
-	if (bNeedAck)
-		return uDataTime + pDevice->uSIFS + uAckTime;
-	else
+	uDataTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, cbFrameLength, wRate);
+
+	if (!bNeedAck)
 		return uDataTime;
+
+	/*
+	 * CCK mode  - 11b
+	 * OFDM mode - 11g 2.4G & 11a 5G
+	 */
+	uAckTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14,
+				     byPktType == PK_TYPE_11B ?
+				     pDevice->byTopCCKBasicRate :
+				     pDevice->byTopOFDMBasicRate);
+
+	return uDataTime + pDevice->uSIFS + uAckTime;
 }
 
 static __le16 vnt_rxtx_rsvtime_le16(struct vnt_private *priv, u8 pkt_type,
@@ -195,24 +200,28 @@ s_uGetRTSCTSRsvTime(
 	unsigned short wCurrentRate
 )
 {
-	unsigned int uRrvTime, uRTSTime, uCTSTime, uAckTime, uDataTime;
-
-	uRrvTime = uRTSTime = uCTSTime = uAckTime = uDataTime = 0;
+	unsigned int uRrvTime = 0;
+	unsigned int uRTSTime = 0;
+	unsigned int uCTSTime = 0;
+	unsigned int uAckTime = 0;
+	unsigned int uDataTime = 0;
 
-	uDataTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, cbFrameLength, wCurrentRate);
+	uDataTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, cbFrameLength, wCurrentRate);
 	if (byRTSRsvType == 0) { /* RTSTxRrvTime_bb */
-		uRTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 20, pDevice->byTopCCKBasicRate);
-		uCTSTime = uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uRTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 20, pDevice->byTopCCKBasicRate);
+		uAckTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uCTSTime = uAckTime;
 	} else if (byRTSRsvType == 1) { /* RTSTxRrvTime_ba, only in 2.4GHZ */
-		uRTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 20, pDevice->byTopCCKBasicRate);
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
-		uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
+		uRTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 20, pDevice->byTopCCKBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uAckTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
 	} else if (byRTSRsvType == 2) { /* RTSTxRrvTime_aa */
-		uRTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 20, pDevice->byTopOFDMBasicRate);
-		uCTSTime = uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
+		uRTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 20, pDevice->byTopOFDMBasicRate);
+		uAckTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
+		uCTSTime = uAckTime;
 	} else if (byRTSRsvType == 3) { /* CTSTxRrvTime_ba, only in 2.4GHZ */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
-		uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uAckTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
 		uRrvTime = uCTSTime + uAckTime + uDataTime + 2 * pDevice->uSIFS;
 		return cpu_to_le16((u16)uRrvTime);
 	}
@@ -239,138 +248,83 @@ s_uGetDataDuration(
 )
 {
 	bool bLastFrag = false;
-	unsigned int uAckTime = 0, uNextPktTime = 0;
+	unsigned int uAckTime = 0, uNextPktTime = 0, len;
 
 	if (uFragIdx == (uMACfragNum - 1))
 		bLastFrag = true;
 
+	if (uFragIdx == (uMACfragNum - 2))
+		len = cbLastFragmentSize;
+	else
+		len = cbFrameLength;
+
 	switch (byDurType) {
 	case DATADUR_B:    /* DATADUR_B */
-		if (((uMACfragNum == 1)) || bLastFrag) {/* Non Frag or Last Frag */
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
-				return pDevice->uSIFS + uAckTime;
-			} else {
+		if (bNeedAck) {
+			uAckTime = bb_get_frame_time(pDevice->byPreambleType,
+						     byPktType, 14,
+						     pDevice->byTopCCKBasicRate);
+		}
+		/* Non Frag or Last Frag */
+		if ((uMACfragNum == 1) || bLastFrag) {
+			if (!bNeedAck)
 				return 0;
-			}
-		} else {/* First Frag or Mid Frag */
-			if (uFragIdx == (uMACfragNum - 2))
-				uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbLastFragmentSize, wRate, bNeedAck);
-			else
-				uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wRate, bNeedAck);
-
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
-				return pDevice->uSIFS + uAckTime + uNextPktTime;
-			} else {
-				return pDevice->uSIFS + uNextPktTime;
-			}
+		} else {
+			/* First Frag or Mid Frag */
+			uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType,
+						       len, wRate, bNeedAck);
 		}
-		break;
+
+		return pDevice->uSIFS + uAckTime + uNextPktTime;
 
 	case DATADUR_A:    /* DATADUR_A */
-		if (((uMACfragNum == 1)) || bLastFrag) {/* Non Frag or Last Frag */
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
-				return pDevice->uSIFS + uAckTime;
-			} else {
+		if (bNeedAck) {
+			uAckTime = bb_get_frame_time(pDevice->byPreambleType,
+						     byPktType, 14,
+						     pDevice->byTopOFDMBasicRate);
+		}
+		/* Non Frag or Last Frag */
+		if ((uMACfragNum == 1) || bLastFrag) {
+			if (!bNeedAck)
 				return 0;
-			}
-		} else {/* First Frag or Mid Frag */
-			if (uFragIdx == (uMACfragNum - 2))
-				uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbLastFragmentSize, wRate, bNeedAck);
-			else
-				uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wRate, bNeedAck);
-
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
-				return pDevice->uSIFS + uAckTime + uNextPktTime;
-			} else {
-				return pDevice->uSIFS + uNextPktTime;
-			}
+		} else {
+			/* First Frag or Mid Frag */
+			uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType,
+						       len, wRate, bNeedAck);
 		}
-		break;
+
+		return pDevice->uSIFS + uAckTime + uNextPktTime;
 
 	case DATADUR_A_F0:    /* DATADUR_A_F0 */
-		if (((uMACfragNum == 1)) || bLastFrag) {/* Non Frag or Last Frag */
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
-				return pDevice->uSIFS + uAckTime;
-			} else {
+	case DATADUR_A_F1:    /* DATADUR_A_F1 */
+		if (bNeedAck) {
+			uAckTime = bb_get_frame_time(pDevice->byPreambleType,
+						     byPktType, 14,
+						     pDevice->byTopOFDMBasicRate);
+		}
+		/* Non Frag or Last Frag */
+		if ((uMACfragNum == 1) || bLastFrag) {
+			if (!bNeedAck)
 				return 0;
-			}
-		} else { /* First Frag or Mid Frag */
-			if (byFBOption == AUTO_FB_0) {
-				if (wRate < RATE_18M)
-					wRate = RATE_18M;
-				else if (wRate > RATE_54M)
-					wRate = RATE_54M;
-
-				if (uFragIdx == (uMACfragNum - 2))
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbLastFragmentSize, wFB_Opt0[FB_RATE0][wRate-RATE_18M], bNeedAck);
-				else
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt0[FB_RATE0][wRate-RATE_18M], bNeedAck);
-
-			} else { /* (byFBOption == AUTO_FB_1) */
-				if (wRate < RATE_18M)
-					wRate = RATE_18M;
-				else if (wRate > RATE_54M)
-					wRate = RATE_54M;
-
-				if (uFragIdx == (uMACfragNum - 2))
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbLastFragmentSize, wFB_Opt1[FB_RATE0][wRate-RATE_18M], bNeedAck);
-				else
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt1[FB_RATE0][wRate-RATE_18M], bNeedAck);
-			}
+		} else {
+			/* First Frag or Mid Frag */
+			if (wRate < RATE_18M)
+				wRate = RATE_18M;
+			else if (wRate > RATE_54M)
+				wRate = RATE_54M;
 
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
-				return pDevice->uSIFS + uAckTime + uNextPktTime;
-			} else {
-				return pDevice->uSIFS + uNextPktTime;
-			}
-		}
-		break;
+			wRate -= RATE_18M;
 
-	case DATADUR_A_F1:    /* DATADUR_A_F1 */
-		if (((uMACfragNum == 1)) || bLastFrag) { /* Non Frag or Last Frag */
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
-				return pDevice->uSIFS + uAckTime;
-			} else {
-				return 0;
-			}
-		} else { /* First Frag or Mid Frag */
-			if (byFBOption == AUTO_FB_0) {
-				if (wRate < RATE_18M)
-					wRate = RATE_18M;
-				else if (wRate > RATE_54M)
-					wRate = RATE_54M;
-
-				if (uFragIdx == (uMACfragNum - 2))
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbLastFragmentSize, wFB_Opt0[FB_RATE1][wRate-RATE_18M], bNeedAck);
-				else
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt0[FB_RATE1][wRate-RATE_18M], bNeedAck);
-
-			} else { /* (byFBOption == AUTO_FB_1) */
-				if (wRate < RATE_18M)
-					wRate = RATE_18M;
-				else if (wRate > RATE_54M)
-					wRate = RATE_54M;
-
-				if (uFragIdx == (uMACfragNum - 2))
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbLastFragmentSize, wFB_Opt1[FB_RATE1][wRate-RATE_18M], bNeedAck);
-				else
-					uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt1[FB_RATE1][wRate-RATE_18M], bNeedAck);
-			}
-			if (bNeedAck) {
-				uAckTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
-				return pDevice->uSIFS + uAckTime + uNextPktTime;
-			} else {
-				return pDevice->uSIFS + uNextPktTime;
-			}
+			if (byFBOption == AUTO_FB_0)
+				wRate = wFB_Opt0[FB_RATE0][wRate];
+			else
+				wRate = wFB_Opt1[FB_RATE0][wRate];
+
+			uNextPktTime = s_uGetTxRsvTime(pDevice, byPktType,
+						       len, wRate, bNeedAck);
 		}
-		break;
+
+		return pDevice->uSIFS + uAckTime + uNextPktTime;
 
 	default:
 		break;
@@ -396,17 +350,17 @@ s_uGetRTSCTSDuration(
 
 	switch (byDurType) {
 	case RTSDUR_BB:    /* RTSDuration_bb */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
 		uDurTime = uCTSTime + 2 * pDevice->uSIFS + s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wRate, bNeedAck);
 		break;
 
 	case RTSDUR_BA:    /* RTSDuration_ba */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
 		uDurTime = uCTSTime + 2 * pDevice->uSIFS + s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wRate, bNeedAck);
 		break;
 
 	case RTSDUR_AA:    /* RTSDuration_aa */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
 		uDurTime = uCTSTime + 2 * pDevice->uSIFS + s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wRate, bNeedAck);
 		break;
 
@@ -415,7 +369,7 @@ s_uGetRTSCTSDuration(
 		break;
 
 	case RTSDUR_BA_F0: /* RTSDuration_ba_f0 */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
 		if ((byFBOption == AUTO_FB_0) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
 			uDurTime = uCTSTime + 2 * pDevice->uSIFS + s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt0[FB_RATE0][wRate-RATE_18M], bNeedAck);
 		else if ((byFBOption == AUTO_FB_1) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
@@ -424,7 +378,7 @@ s_uGetRTSCTSDuration(
 		break;
 
 	case RTSDUR_AA_F0: /* RTSDuration_aa_f0 */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
 		if ((byFBOption == AUTO_FB_0) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
 			uDurTime = uCTSTime + 2 * pDevice->uSIFS + s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt0[FB_RATE0][wRate-RATE_18M], bNeedAck);
 		else if ((byFBOption == AUTO_FB_1) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
@@ -433,7 +387,7 @@ s_uGetRTSCTSDuration(
 		break;
 
 	case RTSDUR_BA_F1: /* RTSDuration_ba_f1 */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopCCKBasicRate);
 		if ((byFBOption == AUTO_FB_0) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
 			uDurTime = uCTSTime + 2*pDevice->uSIFS + s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt0[FB_RATE1][wRate-RATE_18M], bNeedAck);
 		else if ((byFBOption == AUTO_FB_1) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
@@ -442,7 +396,7 @@ s_uGetRTSCTSDuration(
 		break;
 
 	case RTSDUR_AA_F1: /* RTSDuration_aa_f1 */
-		uCTSTime = BBuGetFrameTime(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
+		uCTSTime = bb_get_frame_time(pDevice->byPreambleType, byPktType, 14, pDevice->byTopOFDMBasicRate);
 		if ((byFBOption == AUTO_FB_0) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
 			uDurTime = uCTSTime + 2 * pDevice->uSIFS + s_uGetTxRsvTime(pDevice, byPktType, cbFrameLength, wFB_Opt0[FB_RATE1][wRate-RATE_18M], bNeedAck);
 		else if ((byFBOption == AUTO_FB_1) && (wRate >= RATE_18M) && (wRate <= RATE_54M))
@@ -1040,16 +994,14 @@ s_cbFillTxBufHead(struct vnt_private *pDevice, unsigned char byPktType,
 	bool bRTS = (bool)(fifo_ctl & FIFOCTL_RTS);
 	struct vnt_tx_desc *ptdCurr;
 	unsigned int cbHeaderLength = 0;
-	void *pvRrvTime;
-	struct vnt_mic_hdr *pMICHDR;
-	void *pvRTS;
-	void *pvCTS;
-	void *pvTxDataHd;
+	void *pvRrvTime = NULL;
+	struct vnt_mic_hdr *pMICHDR = NULL;
+	void *pvRTS = NULL;
+	void *pvCTS = NULL;
+	void *pvTxDataHd = NULL;
 	unsigned short wTxBufSize;   /* FFinfo size */
 	unsigned char byFBOption = AUTO_FB_NONE;
 
-	pvRrvTime = pMICHDR = pvRTS = pvCTS = pvTxDataHd = NULL;
-
 	cbFrameSize = skb->len + 4;
 
 	if (info->control.hw_key) {
diff --git a/drivers/staging/vt6656/Makefile b/drivers/staging/vt6656/Makefile
index 375f54e9f58b..f696a9d7a143 100644
--- a/drivers/staging/vt6656/Makefile
+++ b/drivers/staging/vt6656/Makefile
@@ -1,7 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-# TODO: all of these should be removed
-ccflags-y := -DLINUX -D__KERNEL__ -DEXPORT_SYMTAB -D__NO_VERSION__
-ccflags-y += -DHOSTAP
 
 vt6656_stage-y +=	main_usb.o \
 			card.o  \
@@ -13,7 +10,6 @@ vt6656_stage-y +=	main_usb.o \
 			key.o \
 			rf.o \
 			usbpipe.o \
-			channel.o \
-			firmware.o
+			channel.o
 
 obj-$(CONFIG_VT6656) +=	vt6656_stage.o
diff --git a/drivers/staging/vt6656/baseband.c b/drivers/staging/vt6656/baseband.c
index a19a563d8bcc..41ae779ec61f 100644
--- a/drivers/staging/vt6656/baseband.c
+++ b/drivers/staging/vt6656/baseband.c
@@ -23,13 +23,15 @@
  */
 
 #include <linux/bits.h>
+#include <linux/errno.h>
 #include <linux/kernel.h>
+#include "device.h"
 #include "mac.h"
 #include "baseband.h"
 #include "rf.h"
 #include "usbpipe.h"
 
-static u8 vnt_vt3184_agc[] = {
+static const u8 vnt_vt3184_agc[] = {
 	0x00, 0x00, 0x02, 0x02, 0x04, 0x04, 0x06, 0x06,
 	0x08, 0x08, 0x0a, 0x0a, 0x0c, 0x0c, 0x0e, 0x0e, /* 0x0f */
 	0x10, 0x10, 0x12, 0x12, 0x14, 0x14, 0x16, 0x16,
@@ -76,7 +78,7 @@ static u8 vnt_vt3184_al2230[] = {
 };
 
 /* {{RobertYu:20060515, new BB setting for VT3226D0 */
-static u8 vnt_vt3184_vt3226d0[] = {
+static const u8 vnt_vt3184_vt3226d0[] = {
 	0x31, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00,
 	0x70, 0x45, 0x2a, 0x76, 0x00, 0x00, 0x80, 0x00, /* 0x0f */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -111,198 +113,85 @@ static u8 vnt_vt3184_vt3226d0[] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /* 0xff */
 };
 
-static const u16 vnt_frame_time[MAX_RATE] = {
-	10, 20, 55, 110, 24, 36, 48, 72, 96, 144, 192, 216
+struct vnt_threshold {
+	u8 bb_pre_ed_rssi;
+	u8 cr_201;
+	u8 cr_206;
 };
 
-/*
- * Description: Calculate data frame transmitting time
- *
- * Parameters:
- *  In:
- *	preamble_type	- Preamble Type
- *	pkt_type	- PK_TYPE_11A, PK_TYPE_11B, PK_TYPE_11GB, PK_TYPE_11GA
- *	frame_length	- Baseband Type
- *	tx_rate		- Tx Rate
- *  Out:
- *
- * Return Value: FrameTime
- *
- */
-unsigned int vnt_get_frame_time(u8 preamble_type, u8 pkt_type,
-				unsigned int frame_length, u16 tx_rate)
-{
-	unsigned int frame_time;
-	unsigned int preamble;
-	unsigned int rate = 0;
-
-	if (tx_rate > RATE_54M)
-		return 0;
-
-	rate = (unsigned int)vnt_frame_time[tx_rate];
-
-	if (tx_rate <= 3) {
-		if (preamble_type == 1)
-			preamble = 96;
-		else
-			preamble = 192;
-
-		frame_time = DIV_ROUND_UP(frame_length * 80, rate);
-		return preamble + frame_time;
-	}
-
-	frame_time = DIV_ROUND_UP(frame_length * 8 + 22, rate);
-	frame_time = frame_time * 4;
-
-	if (pkt_type != PK_TYPE_11A)
-		frame_time += 6;
-	return 20 + frame_time;
-}
-
-/*
- * Description: Calculate Length, Service, and Signal fields of Phy for Tx
- *
- * Parameters:
- *  In:
- *      priv         - Device Structure
- *      frame_length   - Tx Frame Length
- *      tx_rate           - Tx Rate
- *  Out:
- *	struct vnt_phy_field *phy
- *		- pointer to Phy Length field
- *		- pointer to Phy Service field
- *		- pointer to Phy Signal field
- *
- * Return Value: none
- *
- */
-void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
-		       u16 tx_rate, u8 pkt_type, struct vnt_phy_field *phy)
-{
-	u32 bit_count;
-	u32 count = 0;
-	u32 tmp;
-	int ext_bit;
-	u8 preamble_type = priv->preamble_type;
-
-	bit_count = frame_length * 8;
-	ext_bit = false;
-
-	switch (tx_rate) {
-	case RATE_1M:
-		count = bit_count;
-
-		phy->signal = 0x00;
-
-		break;
-	case RATE_2M:
-		count = bit_count / 2;
-
-		if (preamble_type == 1)
-			phy->signal = 0x09;
-		else
-			phy->signal = 0x01;
-
-		break;
-	case RATE_5M:
-		count = DIV_ROUND_UP(bit_count * 10, 55);
-
-		if (preamble_type == 1)
-			phy->signal = 0x0a;
-		else
-			phy->signal = 0x02;
-
-		break;
-	case RATE_11M:
-		count = bit_count / 11;
-		tmp = count * 11;
-
-		if (tmp != bit_count) {
-			count++;
-
-			if ((bit_count - tmp) <= 3)
-				ext_bit = true;
-		}
-
-		if (preamble_type == 1)
-			phy->signal = 0x0b;
-		else
-			phy->signal = 0x03;
-
-		break;
-	case RATE_6M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x9b;
-		else
-			phy->signal = 0x8b;
-
-		break;
-	case RATE_9M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x9f;
-		else
-			phy->signal = 0x8f;
-
-		break;
-	case RATE_12M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x9a;
-		else
-			phy->signal = 0x8a;
-
-		break;
-	case RATE_18M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x9e;
-		else
-			phy->signal = 0x8e;
-
-		break;
-	case RATE_24M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x99;
-		else
-			phy->signal = 0x89;
-
-		break;
-	case RATE_36M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x9d;
-		else
-			phy->signal = 0x8d;
-
-		break;
-	case RATE_48M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x98;
-		else
-			phy->signal = 0x88;
+static const struct vnt_threshold al2230_vnt_threshold[] = {
+	{0, 0x00, 0x30},	/* Max sensitivity */
+	{68, 0x00, 0x36},
+	{67, 0x00, 0x43},
+	{66, 0x00, 0x51},
+	{65, 0x00, 0x62},
+	{64, 0x00, 0x79},
+	{63, 0x00, 0x93},
+	{62, 0x00, 0xb9},
+	{61, 0x00, 0xe3},
+	{60, 0x01, 0x18},
+	{59, 0x01, 0x54},
+	{58, 0x01, 0xa0},
+	{57, 0x02, 0x20},
+	{56, 0x02, 0xa0},
+	{55, 0x03, 0x00},
+	{53, 0x06, 0x00},
+	{51, 0x09, 0x00},
+	{49, 0x0e, 0x00},
+	{47, 0x15, 0x00},
+	{46, 0x1a, 0x00},
+	{45, 0xff, 0x00}
+};
 
-		break;
-	case RATE_54M:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x9c;
-		else
-			phy->signal = 0x8c;
-		break;
-	default:
-		if (pkt_type == PK_TYPE_11A)
-			phy->signal = 0x9c;
-		else
-			phy->signal = 0x8c;
-		break;
-	}
+static const struct vnt_threshold vt3226_vnt_threshold[] = {
+	{0, 0x00, 0x24},	/* Max sensitivity */
+	{68, 0x00, 0x2d},
+	{67, 0x00, 0x36},
+	{66, 0x00, 0x43},
+	{65, 0x00, 0x52},
+	{64, 0x00, 0x68},
+	{63, 0x00, 0x80},
+	{62, 0x00, 0x9c},
+	{61, 0x00, 0xc0},
+	{60, 0x00, 0xea},
+	{59, 0x01, 0x30},
+	{58, 0x01, 0x70},
+	{57, 0x01, 0xb0},
+	{56, 0x02, 0x30},
+	{55, 0x02, 0xc0},
+	{53, 0x04, 0x00},
+	{51, 0x07, 0x00},
+	{49, 0x0a, 0x00},
+	{47, 0x11, 0x00},
+	{45, 0x18, 0x00},
+	{43, 0x26, 0x00},
+	{42, 0x36, 0x00},
+	{41, 0xff, 0x00}
+};
 
-	if (pkt_type == PK_TYPE_11B) {
-		phy->service = 0x00;
-		if (ext_bit)
-			phy->service |= 0x80;
-		phy->len = cpu_to_le16((u16)count);
-	} else {
-		phy->service = 0x00;
-		phy->len = cpu_to_le16((u16)frame_length);
-	}
-}
+static const struct vnt_threshold vt3342_vnt_threshold[] = {
+	{0, 0x00, 0x38},	/* Max sensitivity */
+	{66, 0x00, 0x43},
+	{65, 0x00, 0x52},
+	{64, 0x00, 0x68},
+	{63, 0x00, 0x80},
+	{62, 0x00, 0x9c},
+	{61, 0x00, 0xc0},
+	{60, 0x00, 0xea},
+	{59, 0x01, 0x30},
+	{58, 0x01, 0x70},
+	{57, 0x01, 0xb0},
+	{56, 0x02, 0x30},
+	{55, 0x02, 0xc0},
+	{53, 0x04, 0x00},
+	{51, 0x07, 0x00},
+	{49, 0x0a, 0x00},
+	{47, 0x11, 0x00},
+	{45, 0x18, 0x00},
+	{43, 0x26, 0x00},
+	{42, 0x36, 0x00},
+	{41, 0xff, 0x00}
+};
 
 /*
  * Description: Set Antenna mode
@@ -352,9 +241,10 @@ int vnt_set_antenna_mode(struct vnt_private *priv, u8 antenna_mode)
 
 int vnt_vt3184_init(struct vnt_private *priv)
 {
-	int ret = 0;
+	int ret;
 	u16 length;
-	u8 *addr;
+	u8 *addr = NULL;
+	const u8 *c_addr;
 	u8 data;
 
 	ret = vnt_control_in(priv, MESSAGE_TYPE_READ, 0, MESSAGE_REQUEST_EEPROM,
@@ -366,23 +256,15 @@ int vnt_vt3184_init(struct vnt_private *priv)
 
 	dev_dbg(&priv->usb->dev, "RF Type %d\n", priv->rf_type);
 
-	if (priv->rf_type == RF_AL2230 ||
-	    priv->rf_type == RF_AL2230S) {
+	if ((priv->rf_type == RF_AL2230) ||
+	    (priv->rf_type == RF_AL2230S) ||
+	    (priv->rf_type == RF_AIROHA7230)) {
 		priv->bb_rx_conf = vnt_vt3184_al2230[10];
 		length = sizeof(vnt_vt3184_al2230);
 		addr = vnt_vt3184_al2230;
 
-		priv->bb_vga[0] = 0x1C;
-		priv->bb_vga[1] = 0x10;
-		priv->bb_vga[2] = 0x0;
-		priv->bb_vga[3] = 0x0;
-
-	} else if (priv->rf_type == RF_AIROHA7230) {
-		priv->bb_rx_conf = vnt_vt3184_al2230[10];
-		length = sizeof(vnt_vt3184_al2230);
-		addr = vnt_vt3184_al2230;
-
-		addr[0xd7] = 0x06;
+		if (priv->rf_type == RF_AIROHA7230)
+			addr[0xd7] = 0x06;
 
 		priv->bb_vga[0] = 0x1c;
 		priv->bb_vga[1] = 0x10;
@@ -390,25 +272,11 @@ int vnt_vt3184_init(struct vnt_private *priv)
 		priv->bb_vga[3] = 0x0;
 
 	} else if ((priv->rf_type == RF_VT3226) ||
-			(priv->rf_type == RF_VT3226D0)) {
+		   (priv->rf_type == RF_VT3226D0) ||
+		   (priv->rf_type == RF_VT3342A0)) {
 		priv->bb_rx_conf = vnt_vt3184_vt3226d0[10];
 		length = sizeof(vnt_vt3184_vt3226d0);
-		addr = vnt_vt3184_vt3226d0;
-
-		priv->bb_vga[0] = 0x20;
-		priv->bb_vga[1] = 0x10;
-		priv->bb_vga[2] = 0x0;
-		priv->bb_vga[3] = 0x0;
-
-		/* Fix VT3226 DFC system timing issue */
-		ret = vnt_mac_reg_bits_on(priv, MAC_REG_SOFTPWRCTL2,
-					  SOFTPWRCTL_RFLEOPT);
-		if (ret)
-			goto end;
-	} else if (priv->rf_type == RF_VT3342A0) {
-		priv->bb_rx_conf = vnt_vt3184_vt3226d0[10];
-		length = sizeof(vnt_vt3184_vt3226d0);
-		addr = vnt_vt3184_vt3226d0;
+		c_addr = vnt_vt3184_vt3226d0;
 
 		priv->bb_vga[0] = 0x20;
 		priv->bb_vga[1] = 0x10;
@@ -424,8 +292,11 @@ int vnt_vt3184_init(struct vnt_private *priv)
 		goto end;
 	}
 
+	if (addr)
+		c_addr = addr;
+
 	ret = vnt_control_out_blocks(priv, VNT_REG_BLOCK_SIZE,
-				     MESSAGE_REQUEST_BBREG, length, addr);
+				     MESSAGE_REQUEST_BBREG, length, c_addr);
 	if (ret)
 		goto end;
 
@@ -435,19 +306,13 @@ int vnt_vt3184_init(struct vnt_private *priv)
 	if (ret)
 		goto end;
 
-	if (priv->rf_type == RF_VT3226 ||
-	    priv->rf_type == RF_VT3342A0) {
-		ret = vnt_control_out_u8(priv, MESSAGE_REQUEST_MACREG,
-					 MAC_REG_ITRTMSET, 0x23);
-		if (ret)
-			goto end;
+	if ((priv->rf_type == RF_VT3226) ||
+	    (priv->rf_type == RF_VT3342A0) ||
+	    (priv->rf_type == RF_VT3226D0)) {
+		data = (priv->rf_type == RF_VT3226D0) ? 0x11 : 0x23;
 
-		ret = vnt_mac_reg_bits_on(priv, MAC_REG_PAPEDELAY, BIT(0));
-		if (ret)
-			goto end;
-	} else if (priv->rf_type == RF_VT3226D0) {
 		ret = vnt_control_out_u8(priv, MESSAGE_REQUEST_MACREG,
-					 MAC_REG_ITRTMSET, 0x11);
+					 MAC_REG_ITRTMSET, data);
 		if (ret)
 			goto end;
 
@@ -507,21 +372,22 @@ int vnt_set_short_slot_time(struct vnt_private *priv)
 
 	ret = vnt_control_in_u8(priv, MESSAGE_REQUEST_BBREG, 0xe7, &bb_vga);
 	if (ret)
-		goto end;
+		return ret;
 
 	if (bb_vga == priv->bb_vga[0])
 		priv->bb_rx_conf |= 0x20;
 
-	ret = vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x0a,
-				 priv->bb_rx_conf);
-
-end:
-	return ret;
+	return vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x0a,
+				  priv->bb_rx_conf);
 }
 
-void vnt_set_vga_gain_offset(struct vnt_private *priv, u8 data)
+int vnt_set_vga_gain_offset(struct vnt_private *priv, u8 data)
 {
-	vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0xE7, data);
+	int ret;
+
+	ret = vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0xE7, data);
+	if (ret)
+		return ret;
 
 	/* patch for 3253B0 Baseband with Cardbus module */
 	if (priv->short_slot_time)
@@ -529,7 +395,8 @@ void vnt_set_vga_gain_offset(struct vnt_private *priv, u8 data)
 	else
 		priv->bb_rx_conf |= 0x20; /* 0010 0000 */
 
-	vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x0a, priv->bb_rx_conf);
+	return vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x0a,
+				  priv->bb_rx_conf);
 }
 
 /*
@@ -570,268 +437,57 @@ int vnt_exit_deep_sleep(struct vnt_private *priv)
 	return vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x0d, 0x01);
 }
 
-void vnt_update_pre_ed_threshold(struct vnt_private *priv, int scanning)
+int vnt_update_pre_ed_threshold(struct vnt_private *priv, int scanning)
 {
-	u8 cr_201 = 0x0, cr_206 = 0x0;
-	u8 ed_inx = priv->bb_pre_ed_index;
+	const struct vnt_threshold *threshold = NULL;
+	u8 length;
+	u8 cr_201, cr_206;
+	u8 ed_inx;
+	int ret;
 
 	switch (priv->rf_type) {
 	case RF_AL2230:
 	case RF_AL2230S:
 	case RF_AIROHA7230:
-		if (scanning) { /* Max sensitivity */
-			ed_inx = 0;
-			cr_206 = 0x30;
-			break;
-		}
-
-		if (priv->bb_pre_ed_rssi <= 45) {
-			ed_inx = 20;
-			cr_201 = 0xff;
-		} else if (priv->bb_pre_ed_rssi <= 46) {
-			ed_inx = 19;
-			cr_201 = 0x1a;
-		} else if (priv->bb_pre_ed_rssi <= 47) {
-			ed_inx = 18;
-			cr_201 = 0x15;
-		} else if (priv->bb_pre_ed_rssi <= 49) {
-			ed_inx = 17;
-			cr_201 = 0xe;
-		} else if (priv->bb_pre_ed_rssi <= 51) {
-			ed_inx = 16;
-			cr_201 = 0x9;
-		} else if (priv->bb_pre_ed_rssi <= 53) {
-			ed_inx = 15;
-			cr_201 = 0x6;
-		} else if (priv->bb_pre_ed_rssi <= 55) {
-			ed_inx = 14;
-			cr_201 = 0x3;
-		} else if (priv->bb_pre_ed_rssi <= 56) {
-			ed_inx = 13;
-			cr_201 = 0x2;
-			cr_206 = 0xa0;
-		} else if (priv->bb_pre_ed_rssi <= 57) {
-			ed_inx = 12;
-			cr_201 = 0x2;
-			cr_206 = 0x20;
-		} else if (priv->bb_pre_ed_rssi <= 58) {
-			ed_inx = 11;
-			cr_201 = 0x1;
-			cr_206 = 0xa0;
-		} else if (priv->bb_pre_ed_rssi <= 59) {
-			ed_inx = 10;
-			cr_201 = 0x1;
-			cr_206 = 0x54;
-		} else if (priv->bb_pre_ed_rssi <= 60) {
-			ed_inx = 9;
-			cr_201 = 0x1;
-			cr_206 = 0x18;
-		} else if (priv->bb_pre_ed_rssi <= 61) {
-			ed_inx = 8;
-			cr_206 = 0xe3;
-		} else if (priv->bb_pre_ed_rssi <= 62) {
-			ed_inx = 7;
-			cr_206 = 0xb9;
-		} else if (priv->bb_pre_ed_rssi <= 63) {
-			ed_inx = 6;
-			cr_206 = 0x93;
-		} else if (priv->bb_pre_ed_rssi <= 64) {
-			ed_inx = 5;
-			cr_206 = 0x79;
-		} else if (priv->bb_pre_ed_rssi <= 65) {
-			ed_inx = 4;
-			cr_206 = 0x62;
-		} else if (priv->bb_pre_ed_rssi <= 66) {
-			ed_inx = 3;
-			cr_206 = 0x51;
-		} else if (priv->bb_pre_ed_rssi <= 67) {
-			ed_inx = 2;
-			cr_206 = 0x43;
-		} else if (priv->bb_pre_ed_rssi <= 68) {
-			ed_inx = 1;
-			cr_206 = 0x36;
-		} else {
-			ed_inx = 0;
-			cr_206 = 0x30;
-		}
+		threshold = al2230_vnt_threshold;
+		length = ARRAY_SIZE(al2230_vnt_threshold);
 		break;
 
 	case RF_VT3226:
 	case RF_VT3226D0:
-		if (scanning)	{ /* Max sensitivity */
-			ed_inx = 0;
-			cr_206 = 0x24;
-			break;
-		}
-
-		if (priv->bb_pre_ed_rssi <= 41) {
-			ed_inx = 22;
-			cr_201 = 0xff;
-		} else if (priv->bb_pre_ed_rssi <= 42) {
-			ed_inx = 21;
-			cr_201 = 0x36;
-		} else if (priv->bb_pre_ed_rssi <= 43) {
-			ed_inx = 20;
-			cr_201 = 0x26;
-		} else if (priv->bb_pre_ed_rssi <= 45) {
-			ed_inx = 19;
-			cr_201 = 0x18;
-		} else if (priv->bb_pre_ed_rssi <= 47) {
-			ed_inx = 18;
-			cr_201 = 0x11;
-		} else if (priv->bb_pre_ed_rssi <= 49) {
-			ed_inx = 17;
-			cr_201 = 0xa;
-		} else if (priv->bb_pre_ed_rssi <= 51) {
-			ed_inx = 16;
-			cr_201 = 0x7;
-		} else if (priv->bb_pre_ed_rssi <= 53) {
-			ed_inx = 15;
-			cr_201 = 0x4;
-		} else if (priv->bb_pre_ed_rssi <= 55) {
-			ed_inx = 14;
-			cr_201 = 0x2;
-			cr_206 = 0xc0;
-		} else if (priv->bb_pre_ed_rssi <= 56) {
-			ed_inx = 13;
-			cr_201 = 0x2;
-			cr_206 = 0x30;
-		} else if (priv->bb_pre_ed_rssi <= 57) {
-			ed_inx = 12;
-			cr_201 = 0x1;
-			cr_206 = 0xb0;
-		} else if (priv->bb_pre_ed_rssi <= 58) {
-			ed_inx = 11;
-			cr_201 = 0x1;
-			cr_206 = 0x70;
-		} else if (priv->bb_pre_ed_rssi <= 59) {
-			ed_inx = 10;
-			cr_201 = 0x1;
-			cr_206 = 0x30;
-		} else if (priv->bb_pre_ed_rssi <= 60) {
-			ed_inx = 9;
-			cr_206 = 0xea;
-		} else if (priv->bb_pre_ed_rssi <= 61) {
-			ed_inx = 8;
-			cr_206 = 0xc0;
-		} else if (priv->bb_pre_ed_rssi <= 62) {
-			ed_inx = 7;
-			cr_206 = 0x9c;
-		} else if (priv->bb_pre_ed_rssi <= 63) {
-			ed_inx = 6;
-			cr_206 = 0x80;
-		} else if (priv->bb_pre_ed_rssi <= 64) {
-			ed_inx = 5;
-			cr_206 = 0x68;
-		} else if (priv->bb_pre_ed_rssi <= 65) {
-			ed_inx = 4;
-			cr_206 = 0x52;
-		} else if (priv->bb_pre_ed_rssi <= 66) {
-			ed_inx = 3;
-			cr_206 = 0x43;
-		} else if (priv->bb_pre_ed_rssi <= 67) {
-			ed_inx = 2;
-			cr_206 = 0x36;
-		} else if (priv->bb_pre_ed_rssi <= 68) {
-			ed_inx = 1;
-			cr_206 = 0x2d;
-		} else {
-			ed_inx = 0;
-			cr_206 = 0x24;
-		}
+		threshold = vt3226_vnt_threshold;
+		length = ARRAY_SIZE(vt3226_vnt_threshold);
 		break;
 
 	case RF_VT3342A0:
-		if (scanning) { /* need Max sensitivity */
-			ed_inx = 0;
-			cr_206 = 0x38;
-			break;
-		}
-
-		if (priv->bb_pre_ed_rssi <= 41) {
-			ed_inx = 20;
-			cr_201 = 0xff;
-		} else if (priv->bb_pre_ed_rssi <= 42) {
-			ed_inx = 19;
-			cr_201 = 0x36;
-		} else if (priv->bb_pre_ed_rssi <= 43) {
-			ed_inx = 18;
-			cr_201 = 0x26;
-		} else if (priv->bb_pre_ed_rssi <= 45) {
-			ed_inx = 17;
-			cr_201 = 0x18;
-		} else if (priv->bb_pre_ed_rssi <= 47) {
-			ed_inx = 16;
-			cr_201 = 0x11;
-		} else if (priv->bb_pre_ed_rssi <= 49) {
-			ed_inx = 15;
-			cr_201 = 0xa;
-		} else if (priv->bb_pre_ed_rssi <= 51) {
-			ed_inx = 14;
-			cr_201 = 0x7;
-		} else if (priv->bb_pre_ed_rssi <= 53) {
-			ed_inx = 13;
-			cr_201 = 0x4;
-		} else if (priv->bb_pre_ed_rssi <= 55) {
-			ed_inx = 12;
-			cr_201 = 0x2;
-			cr_206 = 0xc0;
-		} else if (priv->bb_pre_ed_rssi <= 56) {
-			ed_inx = 11;
-			cr_201 = 0x2;
-			cr_206 = 0x30;
-		} else if (priv->bb_pre_ed_rssi <= 57) {
-			ed_inx = 10;
-			cr_201 = 0x1;
-			cr_206 = 0xb0;
-		} else if (priv->bb_pre_ed_rssi <= 58) {
-			ed_inx = 9;
-			cr_201 = 0x1;
-			cr_206 = 0x70;
-		} else if (priv->bb_pre_ed_rssi <= 59) {
-			ed_inx = 8;
-			cr_201 = 0x1;
-			cr_206 = 0x30;
-		} else if (priv->bb_pre_ed_rssi <= 60) {
-			ed_inx = 7;
-			cr_206 = 0xea;
-		} else if (priv->bb_pre_ed_rssi <= 61) {
-			ed_inx = 6;
-			cr_206 = 0xc0;
-		} else if (priv->bb_pre_ed_rssi <= 62) {
-			ed_inx = 5;
-			cr_206 = 0x9c;
-		} else if (priv->bb_pre_ed_rssi <= 63) {
-			ed_inx = 4;
-			cr_206 = 0x80;
-		} else if (priv->bb_pre_ed_rssi <= 64) {
-			ed_inx = 3;
-			cr_206 = 0x68;
-		} else if (priv->bb_pre_ed_rssi <= 65) {
-			ed_inx = 2;
-			cr_206 = 0x52;
-		} else if (priv->bb_pre_ed_rssi <= 66) {
-			ed_inx = 1;
-			cr_206 = 0x43;
-		} else {
-			ed_inx = 0;
-			cr_206 = 0x38;
-		}
+		threshold = vt3342_vnt_threshold;
+		length = ARRAY_SIZE(vt3342_vnt_threshold);
 		break;
 	}
 
+	if (!threshold)
+		return -EINVAL;
+
+	for (ed_inx = scanning ? 0 : length - 1; ed_inx > 0; ed_inx--) {
+		if (priv->bb_pre_ed_rssi <= threshold[ed_inx].bb_pre_ed_rssi)
+			break;
+	}
+
+	cr_201 = threshold[ed_inx].cr_201;
+	cr_206 = threshold[ed_inx].cr_206;
+
 	if (ed_inx == priv->bb_pre_ed_index && !scanning)
-		return;
+		return 0;
 
 	priv->bb_pre_ed_index = ed_inx;
 
 	dev_dbg(&priv->usb->dev, "%s bb_pre_ed_rssi %d\n",
 		__func__, priv->bb_pre_ed_rssi);
 
-	if (!cr_201 && !cr_206)
-		return;
+	ret = vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0xc9, cr_201);
+	if (ret)
+		return ret;
 
-	vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0xc9, cr_201);
-	vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0xce, cr_206);
+	return vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0xce, cr_206);
 }
 
diff --git a/drivers/staging/vt6656/baseband.h b/drivers/staging/vt6656/baseband.h
index dc42aa6ae1d9..12456ebc23ec 100644
--- a/drivers/staging/vt6656/baseband.h
+++ b/drivers/staging/vt6656/baseband.h
@@ -66,25 +66,12 @@
 #define TOP_RATE_2M         0x00200000
 #define TOP_RATE_1M         0x00100000
 
-/* Length, Service, and Signal fields of Phy for Tx */
-struct vnt_phy_field {
-	u8 signal;
-	u8 service;
-	__le16 len;
-} __packed;
-
-unsigned int vnt_get_frame_time(u8 preamble_type, u8 pkt_type,
-				unsigned int frame_length, u16 tx_rate);
-
-void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
-		       u16 tx_rate, u8 pkt_type, struct vnt_phy_field *phy);
-
 int vnt_set_short_slot_time(struct vnt_private *priv);
-void vnt_set_vga_gain_offset(struct vnt_private *priv, u8 data);
+int vnt_set_vga_gain_offset(struct vnt_private *priv, u8 data);
 int vnt_set_antenna_mode(struct vnt_private *priv, u8 antenna_mode);
 int vnt_vt3184_init(struct vnt_private *priv);
 int vnt_set_deep_sleep(struct vnt_private *priv);
 int vnt_exit_deep_sleep(struct vnt_private *priv);
-void vnt_update_pre_ed_threshold(struct vnt_private *priv, int scanning);
+int vnt_update_pre_ed_threshold(struct vnt_private *priv, int scanning);
 
 #endif /* __BASEBAND_H__ */
diff --git a/drivers/staging/vt6656/card.c b/drivers/staging/vt6656/card.c
index dc3ab10eb630..10f3dfda83b5 100644
--- a/drivers/staging/vt6656/card.c
+++ b/drivers/staging/vt6656/card.c
@@ -26,7 +26,8 @@
  *
  */
 
-#include <linux/bits.h>
+#include <linux/bitops.h>
+#include <linux/errno.h>
 #include "device.h"
 #include "card.h"
 #include "baseband.h"
@@ -45,20 +46,12 @@ static const u16 cw_rxbcntsf_off[MAX_RATE] = {
 	192, 96, 34, 17, 34, 23, 17, 11, 8, 5, 4, 3
 };
 
-/*
- * Description: Set NIC media channel
- *
- * Parameters:
- *  In:
- *      pDevice             - The adapter to be set
- *      connection_channel  - Channel to be set
- *  Out:
- *      none
- */
-void vnt_set_channel(struct vnt_private *priv, u32 connection_channel)
+int vnt_set_channel(struct vnt_private *priv, u32 connection_channel)
 {
+	int ret;
+
 	if (connection_channel > CB_MAX_CHANNEL || !connection_channel)
-		return;
+		return -EINVAL;
 
 	/* clear NAV */
 	vnt_mac_reg_bits_on(priv, MAC_REG_MACCR, MACCR_CLRNAV);
@@ -67,284 +60,73 @@ void vnt_set_channel(struct vnt_private *priv, u32 connection_channel)
 	vnt_mac_reg_bits_off(priv, MAC_REG_CHANNEL,
 			     (BIT(7) | BIT(5) | BIT(4)));
 
-	vnt_control_out(priv, MESSAGE_TYPE_SELECT_CHANNEL,
-			connection_channel, 0, 0, NULL);
-
-	vnt_control_out_u8(priv, MESSAGE_REQUEST_MACREG, MAC_REG_CHANNEL,
-			   (u8)(connection_channel | 0x80));
-}
-
-/*
- * Description: Get CCK mode basic rate
- *
- * Parameters:
- *  In:
- *      priv		- The adapter to be set
- *      rate_idx	- Receiving data rate
- *  Out:
- *      none
- *
- * Return Value: response Control frame rate
- *
- */
-static u16 vnt_get_cck_rate(struct vnt_private *priv, u16 rate_idx)
-{
-	u16 ui = rate_idx;
-
-	while (ui > RATE_1M) {
-		if (priv->basic_rates & (1 << ui))
-			return ui;
-		ui--;
-	}
+	ret = vnt_control_out(priv, MESSAGE_TYPE_SELECT_CHANNEL,
+			      connection_channel, 0, 0, NULL);
+	if (ret)
+		return ret;
 
-	return RATE_1M;
+	return vnt_control_out_u8(priv, MESSAGE_REQUEST_MACREG, MAC_REG_CHANNEL,
+				  (u8)(connection_channel | 0x80));
 }
 
-/*
- * Description: Get OFDM mode basic rate
- *
- * Parameters:
- *  In:
- *      priv		- The adapter to be set
- *      rate_idx	- Receiving data rate
- *  Out:
- *      none
- *
- * Return Value: response Control frame rate
- *
- */
-static u16 vnt_get_ofdm_rate(struct vnt_private *priv, u16 rate_idx)
-{
-	u16 ui = rate_idx;
-
-	dev_dbg(&priv->usb->dev, "%s basic rate: %d\n",
-		__func__,  priv->basic_rates);
-
-	if (!vnt_ofdm_min_rate(priv)) {
-		dev_dbg(&priv->usb->dev, "%s (NO OFDM) %d\n",
-			__func__, rate_idx);
-		if (rate_idx > RATE_24M)
-			rate_idx = RATE_24M;
-		return rate_idx;
-	}
-
-	while (ui > RATE_11M) {
-		if (priv->basic_rates & (1 << ui)) {
-			dev_dbg(&priv->usb->dev, "%s rate: %d\n",
-				__func__, ui);
-			return ui;
-		}
-		ui--;
-	}
-
-	dev_dbg(&priv->usb->dev, "%s basic rate: 24M\n", __func__);
+static const u8 vnt_rspinf_b_short_table[] = {
+	0x70, 0x00, 0x00, 0x00, 0x38, 0x00, 0x09, 0x00,
+	0x15, 0x00, 0x0a, 0x00, 0x0b, 0x00, 0x0b, 0x80
+};
 
-	return RATE_24M;
-}
+static const u8 vnt_rspinf_b_long_table[] = {
+	0x70, 0x00, 0x00, 0x00, 0x38, 0x00, 0x01, 0x00,
+	0x15, 0x00, 0x02, 0x00, 0x0b, 0x00, 0x03, 0x80
+};
 
-/*
- * Description: Calculate TxRate and RsvTime fields for RSPINF in OFDM mode.
- *
- * Parameters:
- * In:
- *	rate	- Tx Rate
- *	bb_type	- Tx Packet type
- * Out:
- *	tx_rate	- pointer to RSPINF TxRate field
- *	rsv_time- pointer to RSPINF RsvTime field
- *
- * Return Value: none
- *
- */
-static void vnt_calculate_ofdm_rate(u16 rate, u8 bb_type,
-				    u8 *tx_rate, u8 *rsv_time)
-{
-	switch (rate) {
-	case RATE_6M:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x9b;
-			*rsv_time = 24;
-		} else {
-			*tx_rate = 0x8b;
-			*rsv_time = 30;
-		}
-		break;
-	case RATE_9M:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x9f;
-			*rsv_time = 16;
-		} else {
-			*tx_rate = 0x8f;
-			*rsv_time = 22;
-		}
-		break;
-	case RATE_12M:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x9a;
-			*rsv_time = 12;
-		} else {
-			*tx_rate = 0x8a;
-			*rsv_time = 18;
-		}
-		break;
-	case RATE_18M:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x9e;
-			*rsv_time = 8;
-		} else {
-			*tx_rate = 0x8e;
-			*rsv_time = 14;
-		}
-		break;
-	case RATE_36M:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x9d;
-			*rsv_time = 4;
-		} else {
-			*tx_rate = 0x8d;
-			*rsv_time = 10;
-		}
-		break;
-	case RATE_48M:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x98;
-			*rsv_time = 4;
-		} else {
-			*tx_rate = 0x88;
-			*rsv_time = 10;
-		}
-		break;
-	case RATE_54M:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x9c;
-			*rsv_time = 4;
-		} else {
-			*tx_rate = 0x8c;
-			*rsv_time = 10;
-		}
-		break;
-	case RATE_24M:
-	default:
-		if (bb_type == BB_TYPE_11A) {
-			*tx_rate = 0x99;
-			*rsv_time = 8;
-		} else {
-			*tx_rate = 0x89;
-			*rsv_time = 14;
-		}
-		break;
-	}
-}
+static const u8 vnt_rspinf_a_table[] = {
+	0x9b, 0x18, 0x9f, 0x10, 0x9a, 0x0a, 0x9e, 0x08, 0x99,
+	0x08, 0x9d, 0x04, 0x98, 0x04, 0x9c, 0x04, 0x9c, 0x04
+};
 
-/*
- * Description: Set RSPINF
- *
- * Parameters:
- *  In:
- *      pDevice             - The adapter to be set
- *  Out:
- *      none
- *
- * Return Value: None.
- *
- */
+static const u8 vnt_rspinf_gb_table[] = {
+	0x8b, 0x1e, 0x8f, 0x16, 0x8a, 0x12, 0x8e, 0x0e, 0x89,
+	0x0e, 0x8d, 0x0a, 0x88, 0x0a, 0x8c, 0x0a, 0x8c, 0x0a
+};
 
-void vnt_set_rspinf(struct vnt_private *priv, u8 bb_type)
+int vnt_set_rspinf(struct vnt_private *priv, u8 bb_type)
 {
-	struct vnt_phy_field phy[4];
-	u8 tx_rate[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; /* For OFDM */
-	u8 rsv_time[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
-	u8 data[34];
-	int i;
-
-	/*RSPINF_b_1*/
-	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_1M),
-			  PK_TYPE_11B, &phy[0]);
-
-	/*RSPINF_b_2*/
-	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_2M),
-			  PK_TYPE_11B, &phy[1]);
-
-	/*RSPINF_b_5*/
-	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_5M),
-			  PK_TYPE_11B, &phy[2]);
-
-	/*RSPINF_b_11*/
-	vnt_get_phy_field(priv, 14, vnt_get_cck_rate(priv, RATE_11M),
-			  PK_TYPE_11B, &phy[3]);
-
-	/*RSPINF_a_6*/
-	vnt_calculate_ofdm_rate(RATE_6M, bb_type, &tx_rate[0], &rsv_time[0]);
-
-	/*RSPINF_a_9*/
-	vnt_calculate_ofdm_rate(RATE_9M, bb_type, &tx_rate[1], &rsv_time[1]);
+	const u8 *data;
+	u16 len;
+	int ret;
 
-	/*RSPINF_a_12*/
-	vnt_calculate_ofdm_rate(RATE_12M, bb_type, &tx_rate[2], &rsv_time[2]);
-
-	/*RSPINF_a_18*/
-	vnt_calculate_ofdm_rate(RATE_18M, bb_type, &tx_rate[3], &rsv_time[3]);
-
-	/*RSPINF_a_24*/
-	vnt_calculate_ofdm_rate(RATE_24M, bb_type, &tx_rate[4], &rsv_time[4]);
-
-	/*RSPINF_a_36*/
-	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_36M),
-				bb_type, &tx_rate[5], &rsv_time[5]);
-
-	/*RSPINF_a_48*/
-	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_48M),
-				bb_type, &tx_rate[6], &rsv_time[6]);
-
-	/*RSPINF_a_54*/
-	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_54M),
-				bb_type, &tx_rate[7], &rsv_time[7]);
-
-	/*RSPINF_a_72*/
-	vnt_calculate_ofdm_rate(vnt_get_ofdm_rate(priv, RATE_54M),
-				bb_type, &tx_rate[8], &rsv_time[8]);
-
-	put_unaligned(phy[0].len, (u16 *)&data[0]);
-	data[2] = phy[0].signal;
-	data[3] = phy[0].service;
-
-	put_unaligned(phy[1].len, (u16 *)&data[4]);
-	data[6] = phy[1].signal;
-	data[7] = phy[1].service;
-
-	put_unaligned(phy[2].len, (u16 *)&data[8]);
-	data[10] = phy[2].signal;
-	data[11] = phy[2].service;
+	if (priv->preamble_type) {
+		data = vnt_rspinf_b_short_table;
+		len = ARRAY_SIZE(vnt_rspinf_b_short_table);
+	} else {
+		data = vnt_rspinf_b_long_table;
+		len = ARRAY_SIZE(vnt_rspinf_b_long_table);
+	}
 
-	put_unaligned(phy[3].len, (u16 *)&data[12]);
-	data[14] = phy[3].signal;
-	data[15] = phy[3].service;
+	 /* RSPINF_b_1 to RSPINF_b_11 */
+	ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_RSPINF_B_1,
+			      MESSAGE_REQUEST_MACREG, len, data);
+	if (ret)
+		return ret;
 
-	for (i = 0; i < 9; i++) {
-		data[16 + i * 2] = tx_rate[i];
-		data[16 + i * 2 + 1] = rsv_time[i];
+	if (bb_type == BB_TYPE_11A) {
+		data = vnt_rspinf_a_table;
+		len = ARRAY_SIZE(vnt_rspinf_a_table);
+	} else {
+		data = vnt_rspinf_gb_table;
+		len = ARRAY_SIZE(vnt_rspinf_gb_table);
 	}
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_RSPINF_B_1,
-			MESSAGE_REQUEST_MACREG, 34, &data[0]);
+	/* RSPINF_a_6 to RSPINF_a_72 */
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_RSPINF_A_6,
+			       MESSAGE_REQUEST_MACREG, len, data);
 }
 
-/*
- * Description: Update IFS
- *
- * Parameters:
- *  In:
- *	priv - The adapter to be set
- * Out:
- *	none
- *
- * Return Value: None.
- *
- */
-void vnt_update_ifs(struct vnt_private *priv)
+int vnt_update_ifs(struct vnt_private *priv)
 {
 	u8 max_min = 0;
 	u8 data[4];
+	int ret;
 
 	if (priv->packet_type == PK_TYPE_11A) {
 		priv->slot = C_SLOT_SHORT;
@@ -367,89 +149,36 @@ void vnt_update_ifs(struct vnt_private *priv)
 
 	priv->eifs = C_EIFS;
 
-	switch (priv->rf_type) {
-	case RF_VT3226D0:
-		if (priv->bb_type != BB_TYPE_11B) {
-			priv->sifs -= 1;
-			priv->difs -= 1;
-			break;
-		}
-		/* fall through */
-	case RF_AIROHA7230:
-	case RF_AL2230:
-	case RF_AL2230S:
-		if (priv->bb_type != BB_TYPE_11B)
-			break;
-		/* fall through */
-	case RF_RFMD2959:
-	case RF_VT3226:
-	case RF_VT3342A0:
-		priv->sifs -= 3;
-		priv->difs -= 3;
-		break;
-	case RF_MAXIM2829:
-		if (priv->bb_type == BB_TYPE_11A) {
-			priv->sifs -= 5;
-			priv->difs -= 5;
-		} else {
-			priv->sifs -= 2;
-			priv->difs -= 2;
-		}
-
-		break;
-	}
-
 	data[0] = (u8)priv->sifs;
 	data[1] = (u8)priv->difs;
 	data[2] = (u8)priv->eifs;
 	data[3] = (u8)priv->slot;
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_SIFS,
-			MESSAGE_REQUEST_MACREG, 4, &data[0]);
+	ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_SIFS,
+			      MESSAGE_REQUEST_MACREG, 4, &data[0]);
+	if (ret)
+		return ret;
 
 	max_min |= 0xa0;
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_CWMAXMIN0,
-			MESSAGE_REQUEST_MACREG, 1, &max_min);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_CWMAXMIN0,
+			       MESSAGE_REQUEST_MACREG, 1, &max_min);
 }
 
 void vnt_update_top_rates(struct vnt_private *priv)
 {
-	u8 top_ofdm = RATE_24M, top_cck = RATE_1M;
-	u8 i;
-
-	/*Determines the highest basic rate.*/
-	for (i = RATE_54M; i >= RATE_6M; i--) {
-		if (priv->basic_rates & (u16)(1 << i)) {
-			top_ofdm = i;
-			break;
-		}
-	}
-
-	priv->top_ofdm_basic_rate = top_ofdm;
+	int pos;
 
-	for (i = RATE_11M;; i--) {
-		if (priv->basic_rates & (u16)(1 << i)) {
-			top_cck = i;
-			break;
-		}
-		if (i == RATE_1M)
-			break;
-	}
+	pos = fls(priv->basic_rates & GENMASK(RATE_54M, RATE_6M));
+	priv->top_ofdm_basic_rate = pos ? (pos - 1) : RATE_24M;
 
-	priv->top_cck_basic_rate = top_cck;
+	pos = fls(priv->basic_rates & GENMASK(RATE_11M, RATE_1M));
+	priv->top_cck_basic_rate = pos ? (pos - 1) : RATE_1M;
 }
 
-int vnt_ofdm_min_rate(struct vnt_private *priv)
+bool vnt_ofdm_min_rate(struct vnt_private *priv)
 {
-	int ii;
-
-	for (ii = RATE_54M; ii >= RATE_6M; ii--) {
-		if ((priv->basic_rates) & ((u16)BIT(ii)))
-			return true;
-	}
-
-	return false;
+	return priv->basic_rates & GENMASK(RATE_54M, RATE_6M) ? true : false;
 }
 
 u8 vnt_get_pkt_type(struct vnt_private *priv)
@@ -481,23 +210,8 @@ u64 vnt_get_tsf_offset(u8 rx_rate, u64 tsf1, u64 tsf2)
 	return tsf1 - tsf2 - (u64)cw_rxbcntsf_off[rx_rate % MAX_RATE];
 }
 
-/*
- * Description: Sync. TSF counter to BSS
- *              Get TSF offset and write to HW
- *
- * Parameters:
- *  In:
- *      priv		- The adapter to be sync.
- *      time_stamp	- Rx BCN's TSF
- *      local_tsf	- Local TSF
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
-		    u64 time_stamp, u64 local_tsf)
+int vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
+		   u64 time_stamp, u64 local_tsf)
 {
 	u64 tsf_offset = 0;
 	u8 data[8];
@@ -513,8 +227,8 @@ void vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
 	data[6] = (u8)(tsf_offset >> 48);
 	data[7] = (u8)(tsf_offset >> 56);
 
-	vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
-			MESSAGE_REQUEST_TSF, 0, 8, data);
+	return vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
+			       MESSAGE_REQUEST_TSF, 0, 8, data);
 }
 
 /*
@@ -589,21 +303,7 @@ u64 vnt_get_next_tbtt(u64 tsf, u16 beacon_interval)
 	return tsf;
 }
 
-/*
- * Description: Set NIC TSF counter for first Beacon time
- *              Get NEXTTBTT from adjusted TSF and Beacon Interval
- *
- * Parameters:
- *  In:
- *      dwIoBase        - IO Base
- *	beacon_interval - Beacon Interval
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval)
+int vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval)
 {
 	u64 next_tbtt = 0;
 	u8 data[8];
@@ -621,29 +321,15 @@ void vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval)
 	data[6] = (u8)(next_tbtt >> 48);
 	data[7] = (u8)(next_tbtt >> 56);
 
-	vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
-			MESSAGE_REQUEST_TBTT, 0, 8, data);
+	return vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
+			       MESSAGE_REQUEST_TBTT, 0, 8, data);
 }
 
-/*
- * Description: Sync NIC TSF counter for Beacon time
- *              Get NEXTTBTT and write to HW
- *
- * Parameters:
- *  In:
- *	priv		- The adapter to be set
- *      tsf		- Current TSF counter
- *      beacon_interval - Beacon Interval
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void vnt_update_next_tbtt(struct vnt_private *priv, u64 tsf,
-			  u16 beacon_interval)
+int vnt_update_next_tbtt(struct vnt_private *priv, u64 tsf,
+			 u16 beacon_interval)
 {
 	u8 data[8];
+	int ret;
 
 	tsf = vnt_get_next_tbtt(tsf, beacon_interval);
 
@@ -656,10 +342,13 @@ void vnt_update_next_tbtt(struct vnt_private *priv, u64 tsf,
 	data[6] = (u8)(tsf >> 48);
 	data[7] = (u8)(tsf >> 56);
 
-	vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
-			MESSAGE_REQUEST_TBTT, 0, 8, data);
+	ret = vnt_control_out(priv, MESSAGE_TYPE_SET_TSFTBTT,
+			      MESSAGE_REQUEST_TBTT, 0, 8, data);
+	if (ret)
+		return ret;
 
 	dev_dbg(&priv->usb->dev, "%s TBTT: %8llx\n", __func__, tsf);
+	return 0;
 }
 
 /*
@@ -723,9 +412,13 @@ int vnt_radio_power_on(struct vnt_private *priv)
 {
 	int ret = 0;
 
-	vnt_exit_deep_sleep(priv);
+	ret = vnt_exit_deep_sleep(priv);
+	if (ret)
+		return ret;
 
-	vnt_mac_reg_bits_on(priv, MAC_REG_HOSTCR, HOSTCR_RXON);
+	ret = vnt_mac_reg_bits_on(priv, MAC_REG_HOSTCR, HOSTCR_RXON);
+	if (ret)
+		return ret;
 
 	switch (priv->rf_type) {
 	case RF_AL2230:
@@ -734,56 +427,69 @@ int vnt_radio_power_on(struct vnt_private *priv)
 	case RF_VT3226:
 	case RF_VT3226D0:
 	case RF_VT3342A0:
-		vnt_mac_reg_bits_on(priv, MAC_REG_SOFTPWRCTL,
-				    (SOFTPWRCTL_SWPE2 | SOFTPWRCTL_SWPE3));
-		break;
+		ret = vnt_mac_reg_bits_on(priv, MAC_REG_SOFTPWRCTL,
+					  (SOFTPWRCTL_SWPE2 |
+					   SOFTPWRCTL_SWPE3));
+		if (ret)
+			return ret;
 	}
 
-	vnt_mac_reg_bits_off(priv, MAC_REG_GPIOCTL1, GPIO3_INTMD);
-
-	return ret;
+	return vnt_mac_reg_bits_off(priv, MAC_REG_GPIOCTL1, GPIO3_INTMD);
 }
 
-void vnt_set_bss_mode(struct vnt_private *priv)
+int vnt_set_bss_mode(struct vnt_private *priv)
 {
-	if (priv->rf_type == RF_AIROHA7230 && priv->bb_type == BB_TYPE_11A)
-		vnt_mac_set_bb_type(priv, BB_TYPE_11G);
-	else
-		vnt_mac_set_bb_type(priv, priv->bb_type);
+	int ret;
+	unsigned char type = priv->bb_type;
+	unsigned char data = 0;
+	unsigned char bb_vga_0 = 0x1c;
+	unsigned char bb_vga_2_3 = 0x00;
 
-	priv->packet_type = vnt_get_pkt_type(priv);
+	if (priv->rf_type == RF_AIROHA7230 && priv->bb_type == BB_TYPE_11A)
+		type = BB_TYPE_11G;
 
-	if (priv->bb_type == BB_TYPE_11A)
-		vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x88, 0x03);
-	else if (priv->bb_type == BB_TYPE_11B)
-		vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x88, 0x02);
-	else if (priv->bb_type == BB_TYPE_11G)
-		vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG, 0x88, 0x08);
+	ret = vnt_mac_set_bb_type(priv, type);
+	if (ret)
+		return ret;
 
-	vnt_update_ifs(priv);
-	vnt_set_rspinf(priv, (u8)priv->bb_type);
+	priv->packet_type = vnt_get_pkt_type(priv);
 
 	if (priv->bb_type == BB_TYPE_11A) {
-		if (priv->rf_type == RF_AIROHA7230) {
-			priv->bb_vga[0] = 0x20;
+		data = 0x03;
+		bb_vga_0 = 0x20;
+		bb_vga_2_3 = 0x10;
+	} else if (priv->bb_type == BB_TYPE_11B) {
+		data = 0x02;
+	} else if (priv->bb_type == BB_TYPE_11G) {
+		data = 0x08;
+	}
 
-			vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG,
-					   0xe7, priv->bb_vga[0]);
-		}
+	if (data) {
+		ret = vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG,
+					 0x88, data);
+		if (ret)
+			return ret;
+	}
 
-		priv->bb_vga[2] = 0x10;
-		priv->bb_vga[3] = 0x10;
-	} else {
-		if (priv->rf_type == RF_AIROHA7230) {
-			priv->bb_vga[0] = 0x1c;
+	ret = vnt_update_ifs(priv);
+	if (ret)
+		return ret;
 
-			vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG,
-					   0xe7, priv->bb_vga[0]);
-		}
+	ret = vnt_set_rspinf(priv, priv->bb_type);
+	if (ret)
+		return ret;
 
-		priv->bb_vga[2] = 0x0;
-		priv->bb_vga[3] = 0x0;
+	if (priv->rf_type == RF_AIROHA7230) {
+		priv->bb_vga[0] = bb_vga_0;
+
+		ret = vnt_control_out_u8(priv, MESSAGE_REQUEST_BBREG,
+					 0xe7, priv->bb_vga[0]);
+		if (ret)
+			return ret;
 	}
 
-	vnt_set_vga_gain_offset(priv, priv->bb_vga[0]);
+	priv->bb_vga[2] = bb_vga_2_3;
+	priv->bb_vga[3] = bb_vga_2_3;
+
+	return vnt_set_vga_gain_offset(priv, priv->bb_vga[0]);
 }
diff --git a/drivers/staging/vt6656/card.h b/drivers/staging/vt6656/card.h
index 75cd340c0cce..a524fdc60ae3 100644
--- a/drivers/staging/vt6656/card.h
+++ b/drivers/staging/vt6656/card.h
@@ -25,23 +25,23 @@
 
 struct vnt_private;
 
-void vnt_set_channel(struct vnt_private *priv, u32 connection_channel);
-void vnt_set_rspinf(struct vnt_private *priv, u8 bb_type);
-void vnt_update_ifs(struct vnt_private *priv);
+int vnt_set_channel(struct vnt_private *priv, u32 connection_channel);
+int vnt_set_rspinf(struct vnt_private *priv, u8 bb_type);
+int vnt_update_ifs(struct vnt_private *priv);
 void vnt_update_top_rates(struct vnt_private *priv);
-int vnt_ofdm_min_rate(struct vnt_private *priv);
-void vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
-		    u64 time_stamp, u64 local_tsf);
+bool vnt_ofdm_min_rate(struct vnt_private *priv);
+int vnt_adjust_tsf(struct vnt_private *priv, u8 rx_rate,
+		   u64 time_stamp, u64 local_tsf);
 bool vnt_get_current_tsf(struct vnt_private *priv, u64 *current_tsf);
 bool vnt_clear_current_tsf(struct vnt_private *priv);
-void vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval);
-void vnt_update_next_tbtt(struct vnt_private *priv, u64 tsf,
-			  u16 beacon_interval);
+int vnt_reset_next_tbtt(struct vnt_private *priv, u16 beacon_interval);
+int vnt_update_next_tbtt(struct vnt_private *priv, u64 tsf,
+			 u16 beacon_interval);
 u64 vnt_get_next_tbtt(u64 tsf, u16 beacon_interval);
 u64 vnt_get_tsf_offset(u8 rx_rate, u64 tsf1, u64 tsf2);
 int vnt_radio_power_off(struct vnt_private *priv);
 int vnt_radio_power_on(struct vnt_private *priv);
 u8 vnt_get_pkt_type(struct vnt_private *priv);
-void vnt_set_bss_mode(struct vnt_private *priv);
+int vnt_set_bss_mode(struct vnt_private *priv);
 
 #endif /* __CARD_H__ */
diff --git a/drivers/staging/vt6656/device.h b/drivers/staging/vt6656/device.h
index e6ee9411f080..947530fefe94 100644
--- a/drivers/staging/vt6656/device.h
+++ b/drivers/staging/vt6656/device.h
@@ -73,6 +73,10 @@
 
 #define DEVICE_VERSION			"mac80211"
 
+#define FIRMWARE_VERSION		0x133		/* version 1.51 */
+#define FIRMWARE_NAME			"vntwusb.fw"
+#define FIRMWARE_CHUNK_SIZE		0x400
+
 #define CONFIG_PATH			"/etc/vntconfiguration.dat"
 
 #define MAX_UINTS			8
@@ -202,8 +206,7 @@ struct vnt_rsp_card_init {
  * Enum of context types for SendPacket
  */
 enum {
-	CONTEXT_DATA_PACKET = 1,
-	CONTEXT_MGMT_PACKET,
+	CONTEXT_DATA_PACKET = 0,
 	CONTEXT_BEACON_PACKET
 };
 
@@ -234,18 +237,14 @@ struct vnt_rcb {
 struct vnt_usb_send_context {
 	void *priv;
 	struct sk_buff *skb;
-	struct urb *urb;
-	struct ieee80211_hdr *hdr;
-	unsigned int buf_len;
+	void *tx_buffer;
 	u32 frame_len;
 	u16 tx_hdr_size;
 	u16 tx_rate;
 	u8 type;
 	u8 pkt_no;
 	u8 pkt_type;
-	u8 need_ack;
 	bool in_use;
-	unsigned char data[MAX_TOTAL_SIZE_WITH_ALL_HEADERS];
 };
 
 /*
@@ -288,6 +287,7 @@ struct vnt_private {
 
 	/* Variables to track resources for the BULK Out Pipe */
 	struct vnt_usb_send_context *tx_context[CB_MAX_TX_DESC];
+	struct usb_anchor tx_submitted;
 	u32 num_tx_context;
 
 	/* Variables to track resources for the Interrupt In Pipe */
@@ -344,13 +344,9 @@ struct vnt_private {
 	u8 ofdm_pwr_tbl[14];
 	u8 ofdm_a_pwr_tbl[42];
 
-	u16 current_rate;
 	u16 tx_rate_fb0;
 	u16 tx_rate_fb1;
 
-	u8 short_retry_limit;
-	u8 long_retry_limit;
-
 	enum nl80211_iftype op_mode;
 
 	int short_slot_time;
@@ -383,8 +379,6 @@ struct vnt_private {
 	u8 bb_pre_ed_rssi;
 	u8 bb_pre_ed_index;
 
-	u16 wake_up_count;
-
 	/* command timer */
 	struct delayed_work run_command_work;
 
diff --git a/drivers/staging/vt6656/firmware.c b/drivers/staging/vt6656/firmware.c
deleted file mode 100644
index 70358d427211..000000000000
--- a/drivers/staging/vt6656/firmware.c
+++ /dev/null
@@ -1,106 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Copyright (c) 1996, 2003 VIA Networking Technologies, Inc.
- * All rights reserved.
- *
- * File: baseband.c
- *
- * Purpose: Implement functions to access baseband
- *
- * Author: Yiching Chen
- *
- * Date: May 20, 2004
- *
- * Functions:
- *
- * Revision History:
- *
- */
-
-#include <linux/compiler.h>
-#include "firmware.h"
-#include "usbpipe.h"
-
-#define FIRMWARE_VERSION	0x133		/* version 1.51 */
-#define FIRMWARE_NAME		"vntwusb.fw"
-
-#define FIRMWARE_CHUNK_SIZE	0x400
-
-int vnt_download_firmware(struct vnt_private *priv)
-{
-	struct device *dev = &priv->usb->dev;
-	const struct firmware *fw;
-	u16 length;
-	int ii;
-	int ret = 0;
-
-	dev_dbg(dev, "---->Download firmware\n");
-
-	ret = request_firmware(&fw, FIRMWARE_NAME, dev);
-	if (ret) {
-		dev_err(dev, "firmware file %s request failed (%d)\n",
-			FIRMWARE_NAME, ret);
-		goto end;
-	}
-
-	for (ii = 0; ii < fw->size; ii += FIRMWARE_CHUNK_SIZE) {
-		length = min_t(int, fw->size - ii, FIRMWARE_CHUNK_SIZE);
-
-		ret = vnt_control_out(priv, 0, 0x1200 + ii, 0x0000, length,
-				      fw->data + ii);
-		if (ret)
-			goto free_fw;
-
-		dev_dbg(dev, "Download firmware...%d %zu\n", ii, fw->size);
-	}
-
-free_fw:
-	release_firmware(fw);
-end:
-	return ret;
-}
-MODULE_FIRMWARE(FIRMWARE_NAME);
-
-int vnt_firmware_branch_to_sram(struct vnt_private *priv)
-{
-	dev_dbg(&priv->usb->dev, "---->Branch to Sram\n");
-
-	return vnt_control_out(priv, 1, 0x1200, 0x0000, 0, NULL);
-}
-
-int vnt_check_firmware_version(struct vnt_private *priv)
-{
-	int ret = 0;
-
-	ret = vnt_control_in(priv, MESSAGE_TYPE_READ, 0,
-			     MESSAGE_REQUEST_VERSION, 2,
-			     (u8 *)&priv->firmware_version);
-	if (ret) {
-		dev_dbg(&priv->usb->dev,
-			"Could not get firmware version: %d.\n", ret);
-		goto end;
-	}
-
-	dev_dbg(&priv->usb->dev, "Firmware Version [%04x]\n",
-		priv->firmware_version);
-
-	if (priv->firmware_version == 0xFFFF) {
-		dev_dbg(&priv->usb->dev, "In Loader.\n");
-		ret = -EINVAL;
-		goto end;
-	}
-
-	if (priv->firmware_version < FIRMWARE_VERSION) {
-		/* branch to loader for download new firmware */
-		ret = vnt_firmware_branch_to_sram(priv);
-		if (ret) {
-			dev_dbg(&priv->usb->dev,
-				"Could not branch to SRAM: %d.\n", ret);
-		} else {
-			ret = -EINVAL;
-		}
-	}
-
-end:
-	return ret;
-}
diff --git a/drivers/staging/vt6656/firmware.h b/drivers/staging/vt6656/firmware.h
deleted file mode 100644
index 161126faf396..000000000000
--- a/drivers/staging/vt6656/firmware.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Copyright (c) 1996, 2003 VIA Networking Technologies, Inc.
- * All rights reserved.
- *
- * File: firmware.h
- *
- * Purpose: Version and Release Information
- *
- * Author: Yiching Chen
- *
- * Date: May 20, 2004
- *
- */
-
-#ifndef __FIRMWARE_H__
-#define __FIRMWARE_H__
-
-#include "device.h"
-
-int vnt_download_firmware(struct vnt_private *priv);
-int vnt_firmware_branch_to_sram(struct vnt_private *priv);
-int vnt_check_firmware_version(struct vnt_private *priv);
-
-#endif /* __FIRMWARE_H__ */
diff --git a/drivers/staging/vt6656/key.c b/drivers/staging/vt6656/key.c
index ac3b188984d0..c66cb53cfc09 100644
--- a/drivers/staging/vt6656/key.c
+++ b/drivers/staging/vt6656/key.c
@@ -35,7 +35,7 @@ int vnt_key_init_table(struct vnt_private *priv)
 
 static int vnt_set_keymode(struct ieee80211_hw *hw, u8 *mac_addr,
 			   struct ieee80211_key_conf *key, u32 key_type,
-			   u32 mode, bool onfly_latch)
+			   u32 mode)
 {
 	struct vnt_private *priv = hw->priv;
 	u8 broadcast[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
@@ -68,17 +68,11 @@ static int vnt_set_keymode(struct ieee80211_hw *hw, u8 *mac_addr,
 		entry = MAX_KEY_TABLE - 1;
 		key->hw_key_idx = entry;
 		/* fall through */
-	case VNT_KEY_ALLGROUP:
-		key_mode |= VNT_KEY_ALLGROUP;
-		if (onfly_latch)
-			key_mode |= VNT_KEY_ONFLY_ALL;
-		/* fall through */
 	case VNT_KEY_GROUP_ADDRESS:
-		key_mode |= mode;
-		/* fall through */
+		key_mode = mode | (mode << 4);
+		break;
 	case VNT_KEY_GROUP:
-		key_mode |= (mode << 4);
-		key_mode |= VNT_KEY_GROUP;
+		key_mode = mode << 4;
 		break;
 	case  VNT_KEY_PAIRWISE:
 		key_mode |= mode;
@@ -88,8 +82,7 @@ static int vnt_set_keymode(struct ieee80211_hw *hw, u8 *mac_addr,
 		return -EINVAL;
 	}
 
-	if (onfly_latch)
-		key_mode |= VNT_KEY_ONFLY;
+	key_mode |= key_type;
 
 	if (mode == KEY_CTL_WEP) {
 		if (key->keylen == WLAN_KEY_LEN_WEP40)
@@ -98,9 +91,8 @@ static int vnt_set_keymode(struct ieee80211_hw *hw, u8 *mac_addr,
 			key->key[15] |= 0x80;
 	}
 
-	vnt_mac_set_keyentry(priv, key_mode, entry, key_inx, bssid, key->key);
-
-	return 0;
+	return vnt_mac_set_keyentry(priv, key_mode, entry,
+				    key_inx, bssid, key->key);
 }
 
 int vnt_set_keys(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
@@ -109,28 +101,21 @@ int vnt_set_keys(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
 	struct vnt_private *priv = hw->priv;
 	u8 *mac_addr = NULL;
 	u8 key_dec_mode = 0;
-	int ret = 0, u;
 
 	if (sta)
 		mac_addr = &sta->addr[0];
 
 	switch (key->cipher) {
-	case 0:
-		for (u = 0 ; u < MAX_KEY_TABLE; u++)
-			vnt_mac_disable_keyentry(priv, u);
-		return ret;
-
 	case WLAN_CIPHER_SUITE_WEP40:
 	case WLAN_CIPHER_SUITE_WEP104:
-		for (u = 0; u < MAX_KEY_TABLE; u++)
-			vnt_mac_disable_keyentry(priv, u);
-
 		vnt_set_keymode(hw, mac_addr, key, VNT_KEY_DEFAULTKEY,
-				KEY_CTL_WEP, true);
+				KEY_CTL_WEP);
 
 		key->flags |= IEEE80211_KEY_FLAG_GENERATE_IV;
 
-		return ret;
+		return vnt_set_keymode(hw, mac_addr, key, VNT_KEY_DEFAULTKEY,
+				       KEY_CTL_WEP);
+
 	case WLAN_CIPHER_SUITE_TKIP:
 		key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIC;
 		key->flags |= IEEE80211_KEY_FLAG_GENERATE_IV;
@@ -151,11 +136,9 @@ int vnt_set_keys(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
 	}
 
 	if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE)
-		vnt_set_keymode(hw, mac_addr, key, VNT_KEY_PAIRWISE,
-				key_dec_mode, true);
-	else
-		vnt_set_keymode(hw, mac_addr, key, VNT_KEY_GROUP_ADDRESS,
-				key_dec_mode, true);
+		return vnt_set_keymode(hw, mac_addr, key, VNT_KEY_PAIRWISE,
+				       key_dec_mode);
 
-	return 0;
+	return vnt_set_keymode(hw, mac_addr, key,
+				VNT_KEY_GROUP_ADDRESS, key_dec_mode);
 }
diff --git a/drivers/staging/vt6656/key.h b/drivers/staging/vt6656/key.h
index 918c07cf86cd..1f3449e66143 100644
--- a/drivers/staging/vt6656/key.h
+++ b/drivers/staging/vt6656/key.h
@@ -25,13 +25,14 @@
 #define KEY_CTL_TKIP        0x02
 #define KEY_CTL_CCMP        0x03
 
-#define VNT_KEY_DEFAULTKEY	0x1
-#define VNT_KEY_GROUP_ADDRESS	0x2
-#define VNT_KEY_ALLGROUP	0x4
-#define VNT_KEY_GROUP		0x40
-#define VNT_KEY_PAIRWISE	0x00
-#define VNT_KEY_ONFLY		0x8000
 #define VNT_KEY_ONFLY_ALL	0x4000
+#define VNT_KEY_ONFLY		0x8000
+#define VNT_KEY_ALLGROUP	0x04
+#define VNT_KEY_GROUP		0x40
+#define VNT_KEY_PAIRWISE	VNT_KEY_ONFLY
+#define VNT_KEY_GROUP_ADDRESS	(VNT_KEY_ALLGROUP | VNT_KEY_GROUP)
+#define VNT_KEY_DEFAULTKEY	(VNT_KEY_GROUP_ADDRESS | VNT_KEY_ONFLY |\
+				VNT_KEY_ONFLY_ALL)
 
 int vnt_key_init_table(struct vnt_private *priv);
 
diff --git a/drivers/staging/vt6656/mac.c b/drivers/staging/vt6656/mac.c
index 5cacf6e60e90..da7067c34643 100644
--- a/drivers/staging/vt6656/mac.c
+++ b/drivers/staging/vt6656/mac.c
@@ -22,90 +22,40 @@
 #include "mac.h"
 #include "usbpipe.h"
 
-/*
- * Description:
- *      Write MAC Multicast Address Mask
- *
- * Parameters:
- *  In:
- *	mc_filter (mac filter)
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void vnt_mac_set_filter(struct vnt_private *priv, u64 mc_filter)
+int vnt_mac_set_filter(struct vnt_private *priv, u64 mc_filter)
 {
 	__le64 le_mc = cpu_to_le64(mc_filter);
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_MAR0,
-			MESSAGE_REQUEST_MACREG, sizeof(le_mc), (u8 *)&le_mc);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_MAR0,
+			       MESSAGE_REQUEST_MACREG, sizeof(le_mc),
+			       (u8 *)&le_mc);
 }
 
-/*
- * Description:
- *      Shut Down MAC
- *
- * Parameters:
- *  In:
- *  Out:
- *      none
- *
- *
- */
-void vnt_mac_shutdown(struct vnt_private *priv)
+int vnt_mac_shutdown(struct vnt_private *priv)
 {
-	vnt_control_out(priv, MESSAGE_TYPE_MACSHUTDOWN, 0, 0, 0, NULL);
+	return vnt_control_out(priv, MESSAGE_TYPE_MACSHUTDOWN, 0, 0, 0, NULL);
 }
 
-void vnt_mac_set_bb_type(struct vnt_private *priv, u8 type)
+int vnt_mac_set_bb_type(struct vnt_private *priv, u8 type)
 {
 	u8 data[2];
 
 	data[0] = type;
 	data[1] = EnCFG_BBType_MASK;
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG0,
-			MESSAGE_REQUEST_MACREG,	ARRAY_SIZE(data), data);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG0,
+			       MESSAGE_REQUEST_MACREG,	ARRAY_SIZE(data),
+			       data);
 }
 
-/*
- * Description:
- *      Disable the Key Entry by MISCFIFO
- *
- * Parameters:
- *  In:
- *      dwIoBase        - Base Address for MAC
- *
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void vnt_mac_disable_keyentry(struct vnt_private *priv, u8 entry_idx)
+int vnt_mac_disable_keyentry(struct vnt_private *priv, u8 entry_idx)
 {
-	vnt_control_out(priv, MESSAGE_TYPE_CLRKEYENTRY, 0, 0,
-			sizeof(entry_idx), &entry_idx);
+	return vnt_control_out(priv, MESSAGE_TYPE_CLRKEYENTRY, 0, 0,
+			       sizeof(entry_idx), &entry_idx);
 }
 
-/*
- * Description:
- *      Set the Key by MISCFIFO
- *
- * Parameters:
- *  In:
- *      dwIoBase        - Base Address for MAC
- *
- *  Out:
- *      none
- *
- * Return Value: none
- *
- */
-void vnt_mac_set_keyentry(struct vnt_private *priv, u16 key_ctl, u32 entry_idx,
-			  u32 key_idx, u8 *addr, u8 *key)
+int vnt_mac_set_keyentry(struct vnt_private *priv, u16 key_ctl, u32 entry_idx,
+			 u32 key_idx, u8 *addr, u8 *key)
 {
 	struct vnt_mac_set_key set_key;
 	u16 offset;
@@ -124,9 +74,9 @@ void vnt_mac_set_keyentry(struct vnt_private *priv, u16 key_ctl, u32 entry_idx,
 	dev_dbg(&priv->usb->dev, "offset %d key ctl %d set key %24ph\n",
 		offset, key_ctl, (u8 *)&set_key);
 
-	vnt_control_out(priv, MESSAGE_TYPE_SETKEY, offset,
-			(u16)key_idx, sizeof(struct vnt_mac_set_key),
-			(u8 *)&set_key);
+	return vnt_control_out(priv, MESSAGE_TYPE_SETKEY, offset,
+			       (u16)key_idx, sizeof(struct vnt_mac_set_key),
+			       (u8 *)&set_key);
 }
 
 int vnt_mac_reg_bits_off(struct vnt_private *priv, u8 reg_ofs, u8 bits)
@@ -151,76 +101,76 @@ int vnt_mac_reg_bits_on(struct vnt_private *priv, u8 reg_ofs, u8 bits)
 			       MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
 }
 
-void vnt_mac_write_word(struct vnt_private *priv, u8 reg_ofs, u16 word)
+int vnt_mac_write_word(struct vnt_private *priv, u8 reg_ofs, u16 word)
 {
 	u8 data[2];
 
 	data[0] = (u8)(word & 0xff);
 	data[1] = (u8)(word >> 8);
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE, reg_ofs,
-			MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE, reg_ofs,
+			       MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
 }
 
-void vnt_mac_set_bssid_addr(struct vnt_private *priv, u8 *addr)
+int vnt_mac_set_bssid_addr(struct vnt_private *priv, u8 *addr)
 {
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_BSSID0,
-			MESSAGE_REQUEST_MACREG, ETH_ALEN, addr);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_BSSID0,
+			       MESSAGE_REQUEST_MACREG, ETH_ALEN, addr);
 }
 
-void vnt_mac_enable_protect_mode(struct vnt_private *priv)
+int vnt_mac_enable_protect_mode(struct vnt_private *priv)
 {
 	u8 data[2];
 
 	data[0] = EnCFG_ProtectMd;
 	data[1] = EnCFG_ProtectMd;
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG0,
-			MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG0,
+			       MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
 }
 
-void vnt_mac_disable_protect_mode(struct vnt_private *priv)
+int vnt_mac_disable_protect_mode(struct vnt_private *priv)
 {
 	u8 data[2];
 
 	data[0] = 0;
 	data[1] = EnCFG_ProtectMd;
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG0,
-			MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG0,
+			       MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
 }
 
-void vnt_mac_enable_barker_preamble_mode(struct vnt_private *priv)
+int vnt_mac_enable_barker_preamble_mode(struct vnt_private *priv)
 {
 	u8 data[2];
 
 	data[0] = EnCFG_BarkerPream;
 	data[1] = EnCFG_BarkerPream;
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG2,
-			MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG2,
+			       MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
 }
 
-void vnt_mac_disable_barker_preamble_mode(struct vnt_private *priv)
+int vnt_mac_disable_barker_preamble_mode(struct vnt_private *priv)
 {
 	u8 data[2];
 
 	data[0] = 0;
 	data[1] = EnCFG_BarkerPream;
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG2,
-			MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE_MASK, MAC_REG_ENCFG2,
+			       MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
 }
 
-void vnt_mac_set_beacon_interval(struct vnt_private *priv, u16 interval)
+int vnt_mac_set_beacon_interval(struct vnt_private *priv, u16 interval)
 {
 	u8 data[2];
 
 	data[0] = (u8)(interval & 0xff);
 	data[1] = (u8)(interval >> 8);
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_BI,
-			MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE, MAC_REG_BI,
+			       MESSAGE_REQUEST_MACREG, ARRAY_SIZE(data), data);
 }
 
 int vnt_mac_set_led(struct vnt_private *priv, u8 state, u8 led)
diff --git a/drivers/staging/vt6656/mac.h b/drivers/staging/vt6656/mac.h
index c532b27de37f..dae70b5c7634 100644
--- a/drivers/staging/vt6656/mac.h
+++ b/drivers/staging/vt6656/mac.h
@@ -177,7 +177,7 @@
 #define EnCFG_BBType_a		0x00
 #define EnCFG_BBType_b		BIT(0)
 #define EnCFG_BBType_g		BIT(1)
-#define EnCFG_BBType_MASK	(BIT(0) | BIT(1))
+#define EnCFG_BBType_MASK	(EnCFG_BBType_b | EnCFG_BBType_g)
 #define EnCFG_ProtectMd		BIT(5)
 
 /* Bits in the EnhanceCFG_1 register */
@@ -355,21 +355,21 @@ struct vnt_mac_set_key {
 	u8 key[WLAN_KEY_LEN_CCMP];
 } __packed;
 
-void vnt_mac_set_filter(struct vnt_private *priv, u64 mc_filter);
-void vnt_mac_shutdown(struct vnt_private *priv);
-void vnt_mac_set_bb_type(struct vnt_private *priv, u8 type);
-void vnt_mac_disable_keyentry(struct vnt_private *priv, u8 entry_idx);
-void vnt_mac_set_keyentry(struct vnt_private *priv, u16 key_ctl, u32 entry_idx,
-			  u32 key_idx, u8 *addr, u8 *key);
+int vnt_mac_set_filter(struct vnt_private *priv, u64 mc_filter);
+int vnt_mac_shutdown(struct vnt_private *priv);
+int vnt_mac_set_bb_type(struct vnt_private *priv, u8 type);
+int vnt_mac_disable_keyentry(struct vnt_private *priv, u8 entry_idx);
+int vnt_mac_set_keyentry(struct vnt_private *priv, u16 key_ctl, u32 entry_idx,
+			 u32 key_idx, u8 *addr, u8 *key);
 int vnt_mac_reg_bits_off(struct vnt_private *priv, u8 reg_ofs, u8 bits);
 int vnt_mac_reg_bits_on(struct vnt_private *priv, u8 reg_ofs, u8 bits);
-void vnt_mac_write_word(struct vnt_private *priv, u8 reg_ofs, u16 word);
-void vnt_mac_set_bssid_addr(struct vnt_private *priv, u8 *addr);
-void vnt_mac_enable_protect_mode(struct vnt_private *priv);
-void vnt_mac_disable_protect_mode(struct vnt_private *priv);
-void vnt_mac_enable_barker_preamble_mode(struct vnt_private *priv);
-void vnt_mac_disable_barker_preamble_mode(struct vnt_private *priv);
-void vnt_mac_set_beacon_interval(struct vnt_private *priv, u16 interval);
+int vnt_mac_write_word(struct vnt_private *priv, u8 reg_ofs, u16 word);
+int vnt_mac_set_bssid_addr(struct vnt_private *priv, u8 *addr);
+int vnt_mac_enable_protect_mode(struct vnt_private *priv);
+int vnt_mac_disable_protect_mode(struct vnt_private *priv);
+int vnt_mac_enable_barker_preamble_mode(struct vnt_private *priv);
+int vnt_mac_disable_barker_preamble_mode(struct vnt_private *priv);
+int vnt_mac_set_beacon_interval(struct vnt_private *priv, u16 interval);
 int vnt_mac_set_led(struct vnt_private *privpriv, u8 state, u8 led);
 
 #endif /* __MAC_H__ */
diff --git a/drivers/staging/vt6656/main_usb.c b/drivers/staging/vt6656/main_usb.c
index 5f78cad3b647..8bf851c53f4e 100644
--- a/drivers/staging/vt6656/main_usb.c
+++ b/drivers/staging/vt6656/main_usb.c
@@ -33,7 +33,6 @@
 #include "wcmd.h"
 #include "rxtx.h"
 #include "rf.h"
-#include "firmware.h"
 #include "usbpipe.h"
 #include "channel.h"
 
@@ -60,8 +59,6 @@ MODULE_PARM_DESC(tx_buffers, "Number of receive usb tx buffers");
 
 #define RTS_THRESH_DEF     2347
 #define FRAG_THRESH_DEF     2346
-#define SHORT_RETRY_DEF     8
-#define LONG_RETRY_DEF     4
 
 /* BasebandType[] baseband type selected
  * 0: indicate 802.11a type
@@ -94,15 +91,91 @@ static void vnt_set_options(struct vnt_private *priv)
 	else
 		priv->num_rcb = vnt_rx_buffers;
 
-	priv->short_retry_limit = SHORT_RETRY_DEF;
-	priv->long_retry_limit = LONG_RETRY_DEF;
 	priv->op_mode = NL80211_IFTYPE_UNSPECIFIED;
 	priv->bb_type = BBP_TYPE_DEF;
 	priv->packet_type = priv->bb_type;
-	priv->preamble_type = 0;
+	priv->preamble_type = PREAMBLE_LONG;
 	priv->exist_sw_net_addr = false;
 }
 
+static int vnt_download_firmware(struct vnt_private *priv)
+{
+	struct device *dev = &priv->usb->dev;
+	const struct firmware *fw;
+	u16 length;
+	int ii;
+	int ret = 0;
+
+	dev_dbg(dev, "---->Download firmware\n");
+
+	ret = request_firmware(&fw, FIRMWARE_NAME, dev);
+	if (ret) {
+		dev_err(dev, "firmware file %s request failed (%d)\n",
+			FIRMWARE_NAME, ret);
+		goto end;
+	}
+
+	for (ii = 0; ii < fw->size; ii += FIRMWARE_CHUNK_SIZE) {
+		length = min_t(int, fw->size - ii, FIRMWARE_CHUNK_SIZE);
+
+		ret = vnt_control_out(priv, 0, 0x1200 + ii, 0x0000, length,
+				      fw->data + ii);
+		if (ret)
+			goto free_fw;
+
+		dev_dbg(dev, "Download firmware...%d %zu\n", ii, fw->size);
+	}
+
+free_fw:
+	release_firmware(fw);
+end:
+	return ret;
+}
+
+static int vnt_firmware_branch_to_sram(struct vnt_private *priv)
+{
+	dev_dbg(&priv->usb->dev, "---->Branch to Sram\n");
+
+	return vnt_control_out(priv, 1, 0x1200, 0x0000, 0, NULL);
+}
+
+static int vnt_check_firmware_version(struct vnt_private *priv)
+{
+	int ret = 0;
+
+	ret = vnt_control_in(priv, MESSAGE_TYPE_READ, 0,
+			     MESSAGE_REQUEST_VERSION, 2,
+			     (u8 *)&priv->firmware_version);
+	if (ret) {
+		dev_dbg(&priv->usb->dev,
+			"Could not get firmware version: %d.\n", ret);
+		goto end;
+	}
+
+	dev_dbg(&priv->usb->dev, "Firmware Version [%04x]\n",
+		priv->firmware_version);
+
+	if (priv->firmware_version == 0xFFFF) {
+		dev_dbg(&priv->usb->dev, "In Loader.\n");
+		ret = -EINVAL;
+		goto end;
+	}
+
+	if (priv->firmware_version < FIRMWARE_VERSION) {
+		/* branch to loader for download new firmware */
+		ret = vnt_firmware_branch_to_sram(priv);
+		if (ret) {
+			dev_dbg(&priv->usb->dev,
+				"Could not branch to SRAM: %d.\n", ret);
+		} else {
+			ret = -EINVAL;
+		}
+	}
+
+end:
+	return ret;
+}
+
 /*
  * initialization of MAC & BBP registers
  */
@@ -146,8 +219,8 @@ static int vnt_init_registers(struct vnt_private *priv)
 	init_cmd->exist_sw_net_addr = priv->exist_sw_net_addr;
 	for (ii = 0; ii < ARRAY_SIZE(init_cmd->sw_net_addr); ii++)
 		init_cmd->sw_net_addr[ii] = priv->current_net_addr[ii];
-	init_cmd->short_retry_limit = priv->short_retry_limit;
-	init_cmd->long_retry_limit = priv->long_retry_limit;
+	init_cmd->short_retry_limit = priv->hw->wiphy->retry_short;
+	init_cmd->long_retry_limit = priv->hw->wiphy->retry_long;
 
 	/* issue card_init command to device */
 	ret = vnt_control_out(priv, MESSAGE_TYPE_CARDINIT, 0, 0,
@@ -324,19 +397,6 @@ static int vnt_init_registers(struct vnt_private *priv)
 	dev_dbg(&priv->usb->dev, "Network address = %pM\n",
 		priv->current_net_addr);
 
-	/*
-	 * set BB and packet type at the same time
-	 * set Short Slot Time, xIFS, and RSPINF
-	 */
-	if (priv->bb_type == BB_TYPE_11A)
-		priv->short_slot_time = true;
-	else
-		priv->short_slot_time = false;
-
-	ret = vnt_set_short_slot_time(priv);
-	if (ret)
-		goto end;
-
 	priv->radio_ctl = priv->eeprom[EEP_OFS_RADIOCTL];
 
 	if ((priv->radio_ctl & EEP_RADIOCTL_ENABLE) != 0) {
@@ -385,17 +445,13 @@ static void vnt_free_tx_bufs(struct vnt_private *priv)
 	struct vnt_usb_send_context *tx_context;
 	int ii;
 
+	usb_kill_anchored_urbs(&priv->tx_submitted);
+
 	for (ii = 0; ii < priv->num_tx_context; ii++) {
 		tx_context = priv->tx_context[ii];
 		if (!tx_context)
 			continue;
 
-		/* deallocate URBs */
-		if (tx_context->urb) {
-			usb_kill_urb(tx_context->urb);
-			usb_free_urb(tx_context->urb);
-		}
-
 		kfree(tx_context);
 	}
 }
@@ -436,6 +492,8 @@ static int vnt_alloc_bufs(struct vnt_private *priv)
 	struct vnt_rcb *rcb;
 	int ii;
 
+	init_usb_anchor(&priv->tx_submitted);
+
 	for (ii = 0; ii < priv->num_tx_context; ii++) {
 		tx_context = kmalloc(sizeof(*tx_context), GFP_KERNEL);
 		if (!tx_context) {
@@ -446,14 +504,6 @@ static int vnt_alloc_bufs(struct vnt_private *priv)
 		priv->tx_context[ii] = tx_context;
 		tx_context->priv = priv;
 		tx_context->pkt_no = ii;
-
-		/* allocate URBs */
-		tx_context->urb = usb_alloc_urb(0, GFP_KERNEL);
-		if (!tx_context->urb) {
-			ret = -ENOMEM;
-			goto free_tx;
-		}
-
 		tx_context->in_use = false;
 	}
 
@@ -683,15 +733,14 @@ static int vnt_config(struct ieee80211_hw *hw, u32 changed)
 			priv->bb_type = BB_TYPE_11G;
 	}
 
-	if (changed & IEEE80211_CONF_CHANGE_POWER) {
-		if (priv->bb_type == BB_TYPE_11B)
-			priv->current_rate = RATE_1M;
-		else
-			priv->current_rate = RATE_54M;
+	if (changed & IEEE80211_CONF_CHANGE_POWER)
+		vnt_rf_setpower(priv, conf->chandef.chan);
 
-		vnt_rf_setpower(priv, priv->current_rate,
-				conf->chandef.chan->hw_value);
-	}
+	if (conf->flags & (IEEE80211_CONF_OFFCHANNEL | IEEE80211_CONF_IDLE))
+		/* Set max sensitivity*/
+		vnt_update_pre_ed_threshold(priv, true);
+	else
+		vnt_update_pre_ed_threshold(priv, false);
 
 	return 0;
 }
@@ -718,10 +767,10 @@ static void vnt_bss_info_changed(struct ieee80211_hw *hw,
 	if (changed & BSS_CHANGED_ERP_PREAMBLE) {
 		if (conf->use_short_preamble) {
 			vnt_mac_enable_barker_preamble_mode(priv);
-			priv->preamble_type = true;
+			priv->preamble_type = PREAMBLE_SHORT;
 		} else {
 			vnt_mac_disable_barker_preamble_mode(priv);
-			priv->preamble_type = false;
+			priv->preamble_type = PREAMBLE_LONG;
 		}
 	}
 
@@ -740,16 +789,14 @@ static void vnt_bss_info_changed(struct ieee80211_hw *hw,
 
 		vnt_set_short_slot_time(priv);
 		vnt_set_vga_gain_offset(priv, priv->bb_vga[0]);
-		vnt_update_pre_ed_threshold(priv, false);
 	}
 
 	if (changed & (BSS_CHANGED_BASIC_RATES | BSS_CHANGED_ERP_PREAMBLE |
 		       BSS_CHANGED_ERP_SLOT))
 		vnt_set_bss_mode(priv);
 
-	if (changed & BSS_CHANGED_TXPOWER)
-		vnt_rf_setpower(priv, priv->current_rate,
-				conf->chandef.chan->hw_value);
+	if (changed & (BSS_CHANGED_TXPOWER | BSS_CHANGED_BANDWIDTH))
+		vnt_rf_setpower(priv, conf->chandef.chan);
 
 	if (changed & BSS_CHANGED_BEACON_ENABLED) {
 		dev_dbg(&priv->usb->dev,
@@ -767,10 +814,17 @@ static void vnt_bss_info_changed(struct ieee80211_hw *hw,
 	if (changed & (BSS_CHANGED_ASSOC | BSS_CHANGED_BEACON_INFO) &&
 	    priv->op_mode != NL80211_IFTYPE_AP) {
 		if (conf->assoc && conf->beacon_rate) {
+			u16 ps_beacon_int = conf->beacon_int;
+
+			if (conf->dtim_period)
+				ps_beacon_int *= conf->dtim_period;
+			else if (hw->conf.listen_interval)
+				ps_beacon_int *= hw->conf.listen_interval;
+
 			vnt_mac_reg_bits_on(priv, MAC_REG_TFTCTL,
 					    TFTCTL_TSFCNTREN);
 
-			vnt_mac_set_beacon_interval(priv, conf->beacon_int);
+			vnt_mac_set_beacon_interval(priv, ps_beacon_int);
 
 			vnt_reset_next_tbtt(priv, conf->beacon_int);
 
@@ -778,7 +832,7 @@ static void vnt_bss_info_changed(struct ieee80211_hw *hw,
 				       conf->sync_tsf, priv->current_tsf);
 
 			vnt_update_next_tbtt(priv,
-					     conf->sync_tsf, conf->beacon_int);
+					     conf->sync_tsf, ps_beacon_int);
 		} else {
 			vnt_clear_current_tsf(priv);
 
@@ -868,25 +922,6 @@ static int vnt_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 	return 0;
 }
 
-static void vnt_sw_scan_start(struct ieee80211_hw *hw,
-			      struct ieee80211_vif *vif,
-			      const u8 *addr)
-{
-	struct vnt_private *priv = hw->priv;
-
-	/* Set max sensitivity*/
-	vnt_update_pre_ed_threshold(priv, true);
-}
-
-static void vnt_sw_scan_complete(struct ieee80211_hw *hw,
-				 struct ieee80211_vif *vif)
-{
-	struct vnt_private *priv = hw->priv;
-
-	/* Return sensitivity to channel level*/
-	vnt_update_pre_ed_threshold(priv, false);
-}
-
 static int vnt_get_stats(struct ieee80211_hw *hw,
 			 struct ieee80211_low_level_stats *stats)
 {
@@ -932,8 +967,6 @@ static const struct ieee80211_ops vnt_mac_ops = {
 	.prepare_multicast	= vnt_prepare_multicast,
 	.configure_filter	= vnt_configure,
 	.set_key		= vnt_set_key,
-	.sw_scan_start		= vnt_sw_scan_start,
-	.sw_scan_complete	= vnt_sw_scan_complete,
 	.get_stats		= vnt_get_stats,
 	.get_tsf		= vnt_get_tsf,
 	.set_tsf		= vnt_set_tsf,
@@ -1010,6 +1043,8 @@ vt6656_probe(struct usb_interface *intf, const struct usb_device_id *id)
 	ieee80211_hw_set(priv->hw, SUPPORTS_PS);
 	ieee80211_hw_set(priv->hw, PS_NULLFUNC_STACK);
 
+	priv->hw->extra_tx_headroom =
+		sizeof(struct vnt_tx_buffer) + sizeof(struct vnt_tx_usb_header);
 	priv->hw->max_signal = 100;
 
 	SET_IEEE80211_DEV(priv->hw, &intf->dev);
@@ -1078,3 +1113,5 @@ static struct usb_driver vt6656_driver = {
 };
 
 module_usb_driver(vt6656_driver);
+
+MODULE_FIRMWARE(FIRMWARE_NAME);
diff --git a/drivers/staging/vt6656/power.c b/drivers/staging/vt6656/power.c
index 7a086c72d5a8..2f49c870272a 100644
--- a/drivers/staging/vt6656/power.c
+++ b/drivers/staging/vt6656/power.c
@@ -63,41 +63,29 @@ void vnt_enable_power_saving(struct vnt_private *priv, u16 listen_interval)
 	 */
 	vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_GO2DOZE);
 
-	if (listen_interval >= 2) {
-		/* clear always listen beacon */
-		vnt_mac_reg_bits_off(priv, MAC_REG_PSCTL, PSCTL_ALBCN);
-
-		/* first time set listen next beacon */
-		vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_LNBCN);
-	} else {
-		/* always listen beacon */
-		vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_ALBCN);
-	}
+	/* always listen beacon */
+	vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_ALBCN);
 
 	dev_dbg(&priv->usb->dev,  "PS:Power Saving Mode Enable...\n");
 }
 
-/*
- *
- * Routine Description:
- * Disable hw power saving functions
- *
- * Return Value:
- *    None.
- *
- */
-
-void vnt_disable_power_saving(struct vnt_private *priv)
+int vnt_disable_power_saving(struct vnt_private *priv)
 {
+	int ret;
+
 	/* disable power saving hw function */
-	vnt_control_out(priv, MESSAGE_TYPE_DISABLE_PS, 0,
-			0, 0, NULL);
+	ret = vnt_control_out(priv, MESSAGE_TYPE_DISABLE_PS, 0,
+			      0, 0, NULL);
+	if (ret)
+		return ret;
 
 	/* clear AutoSleep */
 	vnt_mac_reg_bits_off(priv, MAC_REG_PSCFG, PSCFG_AUTOSLEEP);
 
 	/* set always listen beacon */
 	vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_ALBCN);
+
+	return 0;
 }
 
 /*
diff --git a/drivers/staging/vt6656/power.h b/drivers/staging/vt6656/power.h
index 58755ae16e5a..160872026db3 100644
--- a/drivers/staging/vt6656/power.h
+++ b/drivers/staging/vt6656/power.h
@@ -18,7 +18,7 @@
 
 #define C_PWBT	1000 /* micro sec. power up before TBTT */
 
-void vnt_disable_power_saving(struct vnt_private *priv);
+int vnt_disable_power_saving(struct vnt_private *priv);
 void vnt_enable_power_saving(struct vnt_private *priv, u16 listen_interval);
 int vnt_next_tbtt_wakeup(struct vnt_private *priv);
 
diff --git a/drivers/staging/vt6656/rf.c b/drivers/staging/vt6656/rf.c
index 43237b7e1dbe..5b8da06e3916 100644
--- a/drivers/staging/vt6656/rf.c
+++ b/drivers/staging/vt6656/rf.c
@@ -21,22 +21,16 @@
  *
  */
 
+#include <linux/errno.h>
 #include "mac.h"
 #include "rf.h"
 #include "baseband.h"
 #include "usbpipe.h"
 
 #define CB_AL2230_INIT_SEQ    15
-#define AL2230_PWR_IDX_LEN    64
-
 #define CB_AL7230_INIT_SEQ    16
-#define AL7230_PWR_IDX_LEN    64
-
 #define CB_VT3226_INIT_SEQ    11
-#define VT3226_PWR_IDX_LEN    64
-
 #define CB_VT3342_INIT_SEQ    13
-#define VT3342_PWR_IDX_LEN    64
 
 static u8 al2230_init_table[CB_AL2230_INIT_SEQ][3] = {
 	{0x03, 0xf7, 0x90},
@@ -518,72 +512,45 @@ static u8 vt3342_channel_table1[CB_MAX_CHANNEL][3] = {
 	{0x03, 0x00, 0x04}
 };
 
-/* Power Table */
-static const u32 al2230_power_table[AL2230_PWR_IDX_LEN] = {
-	0x04040900,
-	0x04041900,
-	0x04042900,
-	0x04043900,
-	0x04044900,
-	0x04045900,
-	0x04046900,
-	0x04047900,
-	0x04048900,
-	0x04049900,
-	0x0404a900,
-	0x0404b900,
-	0x0404c900,
-	0x0404d900,
-	0x0404e900,
-	0x0404f900,
-	0x04050900,
-	0x04051900,
-	0x04052900,
-	0x04053900,
-	0x04054900,
-	0x04055900,
-	0x04056900,
-	0x04057900,
-	0x04058900,
-	0x04059900,
-	0x0405a900,
-	0x0405b900,
-	0x0405c900,
-	0x0405d900,
-	0x0405e900,
-	0x0405f900,
-	0x04060900,
-	0x04061900,
-	0x04062900,
-	0x04063900,
-	0x04064900,
-	0x04065900,
-	0x04066900,
-	0x04067900,
-	0x04068900,
-	0x04069900,
-	0x0406a900,
-	0x0406b900,
-	0x0406c900,
-	0x0406d900,
-	0x0406e900,
-	0x0406f900,
-	0x04070900,
-	0x04071900,
-	0x04072900,
-	0x04073900,
-	0x04074900,
-	0x04075900,
-	0x04076900,
-	0x04077900,
-	0x04078900,
-	0x04079900,
-	0x0407a900,
-	0x0407b900,
-	0x0407c900,
-	0x0407d900,
-	0x0407e900,
-	0x0407f900
+enum {
+	VNT_TABLE_INIT = 0,
+	VNT_TABLE_INIT_2 = 0,
+	VNT_TABLE_0 = 1,
+	VNT_TABLE_1 = 2,
+	VNT_TABLE_2 = 1
+};
+
+struct vnt_table_info {
+	u8 *addr;
+	int length;
+};
+
+static const struct vnt_table_info vnt_table_seq[][3] = {
+	{	/* RF_AL2230, RF_AL2230S init table, channel table 0 and 1 */
+		{&al2230_init_table[0][0], CB_AL2230_INIT_SEQ * 3},
+		{&al2230_channel_table0[0][0], CB_MAX_CHANNEL_24G * 3},
+		{&al2230_channel_table1[0][0], CB_MAX_CHANNEL_24G * 3}
+	}, {	/* RF_AIROHA7230 init table, channel table 0 and 1 */
+		{&al7230_init_table[0][0], CB_AL7230_INIT_SEQ * 3},
+		{&al7230_channel_table0[0][0], CB_MAX_CHANNEL * 3},
+		{&al7230_channel_table1[0][0], CB_MAX_CHANNEL * 3}
+	}, {	/* RF_VT3226 init table, channel table 0 and 1 */
+		{&vt3226_init_table[0][0], CB_VT3226_INIT_SEQ * 3},
+		{&vt3226_channel_table0[0][0], CB_MAX_CHANNEL_24G * 3},
+		{&vt3226_channel_table1[0][0], CB_MAX_CHANNEL_24G * 3}
+	}, {	/* RF_VT3226D0 init table, channel table 0 and 1 */
+		{&vt3226d0_init_table[0][0], CB_VT3226_INIT_SEQ * 3},
+		{&vt3226_channel_table0[0][0], CB_MAX_CHANNEL_24G * 3},
+		{&vt3226_channel_table1[0][0], CB_MAX_CHANNEL_24G * 3}
+	}, {	/* RF_VT3342A0 init table, channel table 0 and 1 */
+		{&vt3342a0_init_table[0][0], CB_VT3342_INIT_SEQ * 3},
+		{&vt3342_channel_table0[0][0], CB_MAX_CHANNEL * 3},
+		{&vt3342_channel_table1[0][0], CB_MAX_CHANNEL * 3}
+	}, {	/* RF_AIROHA7230 init table 2 and channel table 2 */
+		{&al7230_init_table_amode[0][0], CB_AL7230_INIT_SEQ * 3},
+		{&al7230_channel_table2[0][0], CB_MAX_CHANNEL * 3},
+		{NULL, 0}
+	}
 };
 
 /*
@@ -600,124 +567,90 @@ int vnt_rf_write_embedded(struct vnt_private *priv, u32 data)
 	reg_data[2] = (u8)(data >> 16);
 	reg_data[3] = (u8)(data >> 24);
 
-	vnt_control_out(priv, MESSAGE_TYPE_WRITE_IFRF,
-			0, 0, ARRAY_SIZE(reg_data), reg_data);
-
-	return true;
-}
-
-/* Set Tx power by rate and channel number */
-int vnt_rf_setpower(struct vnt_private *priv, u32 rate, u32 channel)
-{
-	u8 power = priv->cck_pwr;
-
-	if (channel == 0)
-		return -EINVAL;
-
-	switch (rate) {
-	case RATE_1M:
-	case RATE_2M:
-	case RATE_5M:
-	case RATE_11M:
-		channel--;
-
-		if (channel < sizeof(priv->cck_pwr_tbl))
-			power = priv->cck_pwr_tbl[channel];
-		break;
-	case RATE_6M:
-	case RATE_9M:
-	case RATE_12M:
-	case RATE_18M:
-	case RATE_24M:
-	case RATE_36M:
-	case RATE_48M:
-	case RATE_54M:
-		if (channel > CB_MAX_CHANNEL_24G)
-			power = priv->ofdm_a_pwr_tbl[channel - 15];
-		else
-			power = priv->ofdm_pwr_tbl[channel - 1];
-		break;
-	}
-
-	return vnt_rf_set_txpower(priv, power, rate);
+	return vnt_control_out(priv, MESSAGE_TYPE_WRITE_IFRF, 0, 0,
+			       ARRAY_SIZE(reg_data), reg_data);
 }
 
 static u8 vnt_rf_addpower(struct vnt_private *priv)
 {
+	int base;
 	s32 rssi = -priv->current_rssi;
 
 	if (!rssi)
 		return 7;
 
-	if (priv->rf_type == RF_VT3226D0) {
-		if (rssi < -70)
-			return 9;
-		else if (rssi < -65)
-			return 7;
-		else if (rssi < -60)
-			return 5;
-	} else {
-		if (rssi < -80)
-			return 9;
-		else if (rssi < -75)
-			return 7;
-		else if (rssi < -70)
-			return 5;
-	}
+	if (priv->rf_type == RF_VT3226D0)
+		base = -60;
+	else
+		base = -70;
+
+	if (rssi < base)
+		return ((rssi - base + 1) / -5) * 2 + 5;
 
 	return 0;
 }
 
 /* Set Tx power by power level and rate */
-int vnt_rf_set_txpower(struct vnt_private *priv, u8 power, u32 rate)
+static int vnt_rf_set_txpower(struct vnt_private *priv, u8 power,
+			      struct ieee80211_channel *ch)
 {
 	u32 power_setting = 0;
-	int ret = true;
+	int ret = 0;
 
 	power += vnt_rf_addpower(priv);
 	if (power > VNT_RF_MAX_POWER)
 		power = VNT_RF_MAX_POWER;
 
 	if (priv->power == power)
-		return true;
+		return 0;
 
 	priv->power = power;
 
 	switch (priv->rf_type) {
 	case RF_AL2230:
-		if (power >= AL2230_PWR_IDX_LEN)
-			return false;
+		power_setting = 0x0404090 | (power << 12);
 
-		ret &= vnt_rf_write_embedded(priv, al2230_power_table[power]);
+		ret = vnt_rf_write_embedded(priv, power_setting);
+		if (ret)
+			return ret;
 
-		if (rate <= RATE_11M)
-			ret &= vnt_rf_write_embedded(priv, 0x0001b400);
+		if (ch->flags & IEEE80211_CHAN_NO_OFDM)
+			ret = vnt_rf_write_embedded(priv, 0x0001b400);
 		else
-			ret &= vnt_rf_write_embedded(priv, 0x0005a400);
+			ret = vnt_rf_write_embedded(priv, 0x0005a400);
+
 		break;
 	case RF_AL2230S:
-		if (power >= AL2230_PWR_IDX_LEN)
-			return false;
+		power_setting = 0x0404090 | (power << 12);
 
-		ret &= vnt_rf_write_embedded(priv, al2230_power_table[power]);
+		ret = vnt_rf_write_embedded(priv, power_setting);
+		if (ret)
+			return ret;
 
-		if (rate <= RATE_11M) {
-			ret &= vnt_rf_write_embedded(priv, 0x040c1400);
-			ret &= vnt_rf_write_embedded(priv, 0x00299b00);
+		if (ch->flags & IEEE80211_CHAN_NO_OFDM) {
+			ret = vnt_rf_write_embedded(priv, 0x040c1400);
+			if (ret)
+				return ret;
+
+			ret = vnt_rf_write_embedded(priv, 0x00299b00);
 		} else {
-			ret &= vnt_rf_write_embedded(priv, 0x0005a400);
-			ret &= vnt_rf_write_embedded(priv, 0x00099b00);
+			ret = vnt_rf_write_embedded(priv, 0x0005a400);
+			if (ret)
+				return ret;
+
+			ret = vnt_rf_write_embedded(priv, 0x00099b00);
 		}
+
 		break;
 
 	case RF_AIROHA7230:
-		if (rate <= RATE_11M)
-			ret &= vnt_rf_write_embedded(priv, 0x111bb900);
+		if (ch->flags & IEEE80211_CHAN_NO_OFDM)
+			ret = vnt_rf_write_embedded(priv, 0x111bb900);
 		else
-			ret &= vnt_rf_write_embedded(priv, 0x221bb900);
+			ret = vnt_rf_write_embedded(priv, 0x221bb900);
 
-		if (power >= AL7230_PWR_IDX_LEN)
-			return false;
+		if (ret)
+			return ret;
 
 		/*
 		 * 0x080F1B00 for 3 wire control TxGain(D10)
@@ -725,61 +658,68 @@ int vnt_rf_set_txpower(struct vnt_private *priv, u8 power, u32 rate)
 		 */
 		power_setting = 0x080c0b00 | (power << 12);
 
-		ret &= vnt_rf_write_embedded(priv, power_setting);
-
+		ret = vnt_rf_write_embedded(priv, power_setting);
 		break;
 
 	case RF_VT3226:
-		if (power >= VT3226_PWR_IDX_LEN)
-			return false;
 		power_setting = ((0x3f - power) << 20) | (0x17 << 8);
 
-		ret &= vnt_rf_write_embedded(priv, power_setting);
-
+		ret = vnt_rf_write_embedded(priv, power_setting);
 		break;
 	case RF_VT3226D0:
-		if (power >= VT3226_PWR_IDX_LEN)
-			return false;
-
-		if (rate <= RATE_11M) {
-			u16 hw_value = priv->hw->conf.chandef.chan->hw_value;
+		if (ch->flags & IEEE80211_CHAN_NO_OFDM) {
+			u16 hw_value = ch->hw_value;
 
 			power_setting = ((0x3f - power) << 20) | (0xe07 << 8);
 
-			ret &= vnt_rf_write_embedded(priv, power_setting);
-			ret &= vnt_rf_write_embedded(priv, 0x03c6a200);
+			ret = vnt_rf_write_embedded(priv, power_setting);
+			if (ret)
+				return ret;
+
+			ret = vnt_rf_write_embedded(priv, 0x03c6a200);
+			if (ret)
+				return ret;
 
 			dev_dbg(&priv->usb->dev,
 				"%s 11b channel [%d]\n", __func__, hw_value);
 
 			hw_value--;
 
-			if (hw_value < ARRAY_SIZE(vt3226d0_lo_current_table))
-				ret &= vnt_rf_write_embedded(priv,
+			if (hw_value < ARRAY_SIZE(vt3226d0_lo_current_table)) {
+				ret = vnt_rf_write_embedded(priv,
 					vt3226d0_lo_current_table[hw_value]);
+				if (ret)
+					return ret;
+			}
 
-			ret &= vnt_rf_write_embedded(priv, 0x015C0800);
+			ret = vnt_rf_write_embedded(priv, 0x015C0800);
 		} else {
 			dev_dbg(&priv->usb->dev,
 				"@@@@ %s> 11G mode\n", __func__);
 
 			power_setting = ((0x3f - power) << 20) | (0x7 << 8);
 
-			ret &= vnt_rf_write_embedded(priv, power_setting);
-			ret &= vnt_rf_write_embedded(priv, 0x00C6A200);
-			ret &= vnt_rf_write_embedded(priv, 0x016BC600);
-			ret &= vnt_rf_write_embedded(priv, 0x00900800);
+			ret = vnt_rf_write_embedded(priv, power_setting);
+			if (ret)
+				return ret;
+
+			ret = vnt_rf_write_embedded(priv, 0x00C6A200);
+			if (ret)
+				return ret;
+
+			ret = vnt_rf_write_embedded(priv, 0x016BC600);
+			if (ret)
+				return ret;
+
+			ret = vnt_rf_write_embedded(priv, 0x00900800);
 		}
+
 		break;
 
 	case RF_VT3342A0:
-		if (power >= VT3342_PWR_IDX_LEN)
-			return false;
-
 		power_setting =  ((0x3f - power) << 20) | (0x27 << 8);
 
-		ret &= vnt_rf_write_embedded(priv, power_setting);
-
+		ret = vnt_rf_write_embedded(priv, power_setting);
 		break;
 	default:
 		break;
@@ -787,6 +727,36 @@ int vnt_rf_set_txpower(struct vnt_private *priv, u8 power, u32 rate)
 	return ret;
 }
 
+/* Set Tx power by channel number type */
+int vnt_rf_setpower(struct vnt_private *priv,
+		    struct ieee80211_channel *ch)
+{
+	u16 channel;
+	u8 power = priv->cck_pwr;
+
+	if (!ch)
+		return -EINVAL;
+
+	/* set channel number to array number */
+	channel = ch->hw_value - 1;
+
+	if (ch->flags & IEEE80211_CHAN_NO_OFDM) {
+		if (channel < ARRAY_SIZE(priv->cck_pwr_tbl))
+			power = priv->cck_pwr_tbl[channel];
+	} else if (ch->band == NL80211_BAND_5GHZ) {
+		/* remove 14 channels to array size */
+		channel -= 14;
+
+		if (channel < ARRAY_SIZE(priv->ofdm_a_pwr_tbl))
+			power = priv->ofdm_a_pwr_tbl[channel];
+	} else {
+		if (channel < ARRAY_SIZE(priv->ofdm_pwr_tbl))
+			power = priv->ofdm_pwr_tbl[channel];
+	}
+
+	return vnt_rf_set_txpower(priv, power, ch);
+}
+
 /* Convert rssi to dbm */
 void vnt_rf_rssi_to_dbm(struct vnt_private *priv, u8 rssi, long *dbm)
 {
@@ -813,140 +783,73 @@ void vnt_rf_rssi_to_dbm(struct vnt_private *priv, u8 rssi, long *dbm)
 
 int vnt_rf_table_download(struct vnt_private *priv)
 {
-	int ret = 0;
-	u16 length1 = 0, length2 = 0, length3 = 0;
-	u8 *addr1 = NULL, *addr2 = NULL, *addr3 = NULL;
-	u16 length, value;
-	u8 array[256];
+	int ret;
+	int idx = -1;
+	const struct vnt_table_info *table_seq;
 
 	switch (priv->rf_type) {
 	case RF_AL2230:
 	case RF_AL2230S:
-		length1 = CB_AL2230_INIT_SEQ * 3;
-		length2 = CB_MAX_CHANNEL_24G * 3;
-		length3 = CB_MAX_CHANNEL_24G * 3;
-		addr1 = &al2230_init_table[0][0];
-		addr2 = &al2230_channel_table0[0][0];
-		addr3 = &al2230_channel_table1[0][0];
+		idx = 0;
 		break;
 	case RF_AIROHA7230:
-		length1 = CB_AL7230_INIT_SEQ * 3;
-		length2 = CB_MAX_CHANNEL * 3;
-		length3 = CB_MAX_CHANNEL * 3;
-		addr1 = &al7230_init_table[0][0];
-		addr2 = &al7230_channel_table0[0][0];
-		addr3 = &al7230_channel_table1[0][0];
+		idx = 1;
 		break;
 	case RF_VT3226:
-		length1 = CB_VT3226_INIT_SEQ * 3;
-		length2 = CB_MAX_CHANNEL_24G * 3;
-		length3 = CB_MAX_CHANNEL_24G * 3;
-		addr1 = &vt3226_init_table[0][0];
-		addr2 = &vt3226_channel_table0[0][0];
-		addr3 = &vt3226_channel_table1[0][0];
+		idx = 2;
 		break;
 	case RF_VT3226D0:
-		length1 = CB_VT3226_INIT_SEQ * 3;
-		length2 = CB_MAX_CHANNEL_24G * 3;
-		length3 = CB_MAX_CHANNEL_24G * 3;
-		addr1 = &vt3226d0_init_table[0][0];
-		addr2 = &vt3226_channel_table0[0][0];
-		addr3 = &vt3226_channel_table1[0][0];
+		idx = 3;
 		break;
 	case RF_VT3342A0:
-		length1 = CB_VT3342_INIT_SEQ * 3;
-		length2 = CB_MAX_CHANNEL * 3;
-		length3 = CB_MAX_CHANNEL * 3;
-		addr1 = &vt3342a0_init_table[0][0];
-		addr2 = &vt3342_channel_table0[0][0];
-		addr3 = &vt3342_channel_table1[0][0];
+		idx = 4;
 		break;
 	}
 
-	/* Init Table */
-	memcpy(array, addr1, length1);
+	if (idx < 0)
+		return 0;
 
+	table_seq = &vnt_table_seq[idx][0];
+
+	/* Init Table */
 	ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, 0,
-			      MESSAGE_REQUEST_RF_INIT, length1, array);
+			      MESSAGE_REQUEST_RF_INIT,
+			      table_seq[VNT_TABLE_INIT].length,
+			      table_seq[VNT_TABLE_INIT].addr);
 	if (ret)
-		goto end;
+		return ret;
 
 	/* Channel Table 0 */
-	value = 0;
-	while (length2 > 0) {
-		if (length2 >= 64)
-			length = 64;
-		else
-			length = length2;
-
-		memcpy(array, addr2, length);
-
-		ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, value,
-				      MESSAGE_REQUEST_RF_CH0, length, array);
-		if (ret)
-			goto end;
-
-		length2 -= length;
-		value += length;
-		addr2 += length;
-	}
-
-	/* Channel table 1 */
-	value = 0;
-	while (length3 > 0) {
-		if (length3 >= 64)
-			length = 64;
-		else
-			length = length3;
-
-		memcpy(array, addr3, length);
+	ret = vnt_control_out_blocks(priv, VNT_REG_BLOCK_SIZE,
+				     MESSAGE_REQUEST_RF_CH0,
+				     table_seq[VNT_TABLE_0].length,
+				     table_seq[VNT_TABLE_0].addr);
+	if (ret)
+		return ret;
 
-		ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, value,
-				      MESSAGE_REQUEST_RF_CH1, length, array);
-		if (ret)
-			goto end;
-
-		length3 -= length;
-		value += length;
-		addr3 += length;
-	}
+	/* Channel Table 1 */
+	ret = vnt_control_out_blocks(priv, VNT_REG_BLOCK_SIZE,
+				     MESSAGE_REQUEST_RF_CH1,
+				     table_seq[VNT_TABLE_1].length,
+				     table_seq[VNT_TABLE_1].addr);
 
 	if (priv->rf_type == RF_AIROHA7230) {
-		length1 = CB_AL7230_INIT_SEQ * 3;
-		length2 = CB_MAX_CHANNEL * 3;
-		addr1 = &al7230_init_table_amode[0][0];
-		addr2 = &al7230_channel_table2[0][0];
-
-		memcpy(array, addr1, length1);
+		table_seq = &vnt_table_seq[5][0];
 
 		/* Init Table 2 */
 		ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, 0,
-				      MESSAGE_REQUEST_RF_INIT2, length1, array);
+				      MESSAGE_REQUEST_RF_INIT2,
+				      table_seq[VNT_TABLE_INIT_2].length,
+				      table_seq[VNT_TABLE_INIT_2].addr);
 		if (ret)
-			goto end;
+			return ret;
 
-		/* Channel Table 0 */
-		value = 0;
-		while (length2 > 0) {
-			if (length2 >= 64)
-				length = 64;
-			else
-				length = length2;
-
-			memcpy(array, addr2, length);
-
-			ret = vnt_control_out(priv, MESSAGE_TYPE_WRITE, value,
-					      MESSAGE_REQUEST_RF_CH2, length,
-					      array);
-			if (ret)
-				goto end;
-
-			length2 -= length;
-			value += length;
-			addr2 += length;
-		}
+		/* Channel Table 2 */
+		ret = vnt_control_out_blocks(priv, VNT_REG_BLOCK_SIZE,
+					     MESSAGE_REQUEST_RF_CH2,
+					     table_seq[VNT_TABLE_2].length,
+					     table_seq[VNT_TABLE_2].addr);
 	}
 
-end:
 	return ret;
 }
diff --git a/drivers/staging/vt6656/rf.h b/drivers/staging/vt6656/rf.h
index 7494546d71b8..493faaf4e2b5 100644
--- a/drivers/staging/vt6656/rf.h
+++ b/drivers/staging/vt6656/rf.h
@@ -41,8 +41,7 @@
 #define	VNT_RF_REG_LEN      0x17 /* 24 bit length */
 
 int vnt_rf_write_embedded(struct vnt_private *priv, u32 data);
-int vnt_rf_setpower(struct vnt_private *priv, u32 rate, u32 channel);
-int vnt_rf_set_txpower(struct vnt_private *priv, u8 power, u32 rate);
+int vnt_rf_setpower(struct vnt_private *priv, struct ieee80211_channel *ch);
 void vnt_rf_rssi_to_dbm(struct vnt_private *priv, u8 rssi, long *dbm);
 int vnt_rf_table_download(struct vnt_private *priv);
 
diff --git a/drivers/staging/vt6656/rxtx.c b/drivers/staging/vt6656/rxtx.c
index 9439d190f431..5dd6b4d2bf20 100644
--- a/drivers/staging/vt6656/rxtx.c
+++ b/drivers/staging/vt6656/rxtx.c
@@ -13,8 +13,6 @@
  *
  * Functions:
  *      vnt_generate_tx_parameter - Generate tx dma required parameter.
- *      vnt_get_rtscts_duration_le- get rtx/cts required duration
- *      vnt_get_rtscts_rsvtime_le- get rts/cts reserved time
  *      vnt_get_rsvtime- get frame reserved time
  *      vnt_fill_cts_head- fulfill CTS ctl header
  *
@@ -38,13 +36,24 @@ static const u16 vnt_time_stampoff[2][MAX_RATE] = {
 	{384, 192, 130, 113, 54, 43, 37, 31, 28, 25, 24, 23},
 };
 
-#define RTSDUR_BB       0
-#define RTSDUR_BA       1
-#define RTSDUR_AA       2
-#define CTSDUR_BA       3
 #define DATADUR_B       10
 #define DATADUR_A       11
 
+static const u8 vnt_phy_signal[] = {
+	0x00,	/* RATE_1M  */
+	0x01,	/* RATE_2M  */
+	0x02,	/* RATE_5M  */
+	0x03,	/* RATE_11M */
+	0x8b,	/* RATE_6M  */
+	0x8f,	/* RATE_9M  */
+	0x8a,	/* RATE_12M */
+	0x8e,	/* RATE_18M */
+	0x89,	/* RATE_24M */
+	0x8d,	/* RATE_36M */
+	0x88,	/* RATE_48M */
+	0x8c	/* RATE_54M */
+};
+
 static struct vnt_usb_send_context
 	*vnt_get_free_context(struct vnt_private *priv)
 {
@@ -60,11 +69,6 @@ static struct vnt_usb_send_context
 		context = priv->tx_context[ii];
 		if (!context->in_use) {
 			context->in_use = true;
-			memset(context->data, 0,
-			       MAX_TOTAL_SIZE_WITH_ALL_HEADERS);
-
-			context->hdr = NULL;
-
 			return context;
 		}
 	}
@@ -78,146 +82,105 @@ static struct vnt_usb_send_context
 	return NULL;
 }
 
-static __le16 vnt_time_stamp_off(struct vnt_private *priv, u16 rate)
+/* Get Length, Service, and Signal fields of Phy for Tx */
+static void vnt_get_phy_field(struct vnt_private *priv, u32 frame_length,
+			      u16 tx_rate, u8 pkt_type,
+			      struct vnt_phy_field *phy)
 {
-	return cpu_to_le16(vnt_time_stampoff[priv->preamble_type % 2]
-							[rate % MAX_RATE]);
-}
+	u32 bit_count;
+	u32 count = 0;
+	u32 tmp;
+	int ext_bit;
+	int i;
+	u8 mask = 0;
+	u8 preamble_type = priv->preamble_type;
+
+	bit_count = frame_length * 8;
+	ext_bit = false;
+
+	switch (tx_rate) {
+	case RATE_1M:
+		count = bit_count;
+		break;
+	case RATE_2M:
+		count = bit_count / 2;
+		break;
+	case RATE_5M:
+		count = DIV_ROUND_UP(bit_count * 10, 55);
+		break;
+	case RATE_11M:
+		count = bit_count / 11;
+		tmp = count * 11;
 
-static u32 vnt_get_rsvtime(struct vnt_private *priv, u8 pkt_type,
-			   u32 frame_length, u16 rate, int need_ack)
-{
-	u32 data_time, ack_time;
+		if (tmp != bit_count) {
+			count++;
 
-	data_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-				       frame_length, rate);
+			if ((bit_count - tmp) <= 3)
+				ext_bit = true;
+		}
+
+		break;
+	}
 
-	if (pkt_type == PK_TYPE_11B)
-		ack_time = vnt_get_frame_time(priv->preamble_type, pkt_type, 14,
-					      (u16)priv->top_cck_basic_rate);
-	else
-		ack_time = vnt_get_frame_time(priv->preamble_type, pkt_type, 14,
-					      (u16)priv->top_ofdm_basic_rate);
+	if (tx_rate > RATE_11M) {
+		if (pkt_type == PK_TYPE_11A)
+			mask = BIT(4);
+	} else if (tx_rate > RATE_1M) {
+		if (preamble_type == PREAMBLE_SHORT)
+			mask = BIT(3);
+	}
 
-	if (need_ack)
-		return data_time + priv->sifs + ack_time;
+	i = tx_rate > RATE_54M ? RATE_54M : tx_rate;
+	phy->signal = vnt_phy_signal[i] | mask;
+	phy->service = 0x00;
 
-	return data_time;
+	if (pkt_type == PK_TYPE_11B) {
+		if (ext_bit)
+			phy->service |= 0x80;
+		phy->len = cpu_to_le16((u16)count);
+	} else {
+		phy->len = cpu_to_le16((u16)frame_length);
+	}
 }
 
-static __le16 vnt_rxtx_rsvtime_le16(struct vnt_private *priv, u8 pkt_type,
-				    u32 frame_length, u16 rate, int need_ack)
+static __le16 vnt_time_stamp_off(struct vnt_private *priv, u16 rate)
 {
-	return cpu_to_le16((u16)vnt_get_rsvtime(priv, pkt_type,
-		frame_length, rate, need_ack));
+	return cpu_to_le16(vnt_time_stampoff[priv->preamble_type % 2]
+							[rate % MAX_RATE]);
 }
 
-static __le16 vnt_get_rtscts_rsvtime_le(struct vnt_private *priv, u8 rsv_type,
-					u8 pkt_type, u32 frame_length,
-					u16 current_rate)
+static __le16 vnt_rxtx_rsvtime_le16(struct vnt_usb_send_context *context)
 {
-	u32 rrv_time, rts_time, cts_time, ack_time, data_time;
-
-	rrv_time = 0;
-	rts_time = 0;
-	cts_time = 0;
-	ack_time = 0;
-
-	data_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-				       frame_length, current_rate);
-
-	if (rsv_type == 0) {
-		rts_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      20, priv->top_cck_basic_rate);
-		ack_time = vnt_get_frame_time(priv->preamble_type,
-					      pkt_type, 14,
-					      priv->top_cck_basic_rate);
-		cts_time = ack_time;
-
-	} else if (rsv_type == 1) {
-		rts_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      20, priv->top_cck_basic_rate);
-		cts_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      14, priv->top_cck_basic_rate);
-		ack_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      14, priv->top_ofdm_basic_rate);
-	} else if (rsv_type == 2) {
-		rts_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      20, priv->top_ofdm_basic_rate);
-		ack_time = vnt_get_frame_time(priv->preamble_type,
-					      pkt_type, 14,
-					      priv->top_ofdm_basic_rate);
-		cts_time = ack_time;
-
-	} else if (rsv_type == 3) {
-		cts_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      14, priv->top_cck_basic_rate);
-		ack_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      14, priv->top_ofdm_basic_rate);
-
-		rrv_time = cts_time + ack_time + data_time + 2 * priv->sifs;
-
-		return cpu_to_le16((u16)rrv_time);
-	}
-
-	rrv_time = rts_time + cts_time + ack_time + data_time + 3 * priv->sifs;
+	struct vnt_private *priv = context->priv;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(context->skb);
+	struct ieee80211_rate *rate = ieee80211_get_tx_rate(priv->hw, info);
 
-	return cpu_to_le16((u16)rrv_time);
+	return ieee80211_generic_frame_duration(priv->hw,
+						 info->control.vif, info->band,
+						 context->frame_len,
+						 rate);
 }
 
-static __le16 vnt_get_rtscts_duration_le(struct vnt_usb_send_context *context,
-					 u8 dur_type, u8 pkt_type, u16 rate)
+static __le16 vnt_get_rts_duration(struct vnt_usb_send_context *context)
 {
 	struct vnt_private *priv = context->priv;
-	u32 cts_time = 0, dur_time = 0;
-	u32 frame_length = context->frame_len;
-	u8 need_ack = context->need_ack;
-
-	switch (dur_type) {
-	case RTSDUR_BB:
-	case RTSDUR_BA:
-		cts_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      14, priv->top_cck_basic_rate);
-		dur_time = cts_time + 2 * priv->sifs +
-			vnt_get_rsvtime(priv, pkt_type,
-					frame_length, rate, need_ack);
-		break;
-
-	case RTSDUR_AA:
-		cts_time = vnt_get_frame_time(priv->preamble_type, pkt_type,
-					      14, priv->top_ofdm_basic_rate);
-		dur_time = cts_time + 2 * priv->sifs +
-			vnt_get_rsvtime(priv, pkt_type,
-					frame_length, rate, need_ack);
-		break;
-
-	case CTSDUR_BA:
-		dur_time = priv->sifs + vnt_get_rsvtime(priv,
-				pkt_type, frame_length, rate, need_ack);
-		break;
-
-	default:
-		break;
-	}
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(context->skb);
 
-	return cpu_to_le16((u16)dur_time);
+	return ieee80211_rts_duration(priv->hw, priv->vif,
+				      context->frame_len, info);
 }
 
-static u16 vnt_mac_hdr_pos(struct vnt_usb_send_context *tx_context,
-			   struct ieee80211_hdr *hdr)
+static __le16 vnt_get_cts_duration(struct vnt_usb_send_context *context)
 {
-	u8 *head = tx_context->data + offsetof(struct vnt_tx_buffer, fifo_head);
-	u8 *hdr_pos = (u8 *)hdr;
-
-	tx_context->hdr = hdr;
-	if (!tx_context->hdr)
-		return 0;
+	struct vnt_private *priv = context->priv;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(context->skb);
 
-	return (u16)(hdr_pos - head);
+	return ieee80211_ctstoself_duration(priv->hw, priv->vif,
+					    context->frame_len, info);
 }
 
-static u16 vnt_rxtx_datahead_g(struct vnt_usb_send_context *tx_context,
-			       struct vnt_tx_datahead_g *buf)
+static void vnt_rxtx_datahead_g(struct vnt_usb_send_context *tx_context,
+				struct vnt_tx_datahead_g *buf)
 {
 	struct vnt_private *priv = tx_context->priv;
 	struct ieee80211_hdr *hdr =
@@ -236,14 +199,10 @@ static u16 vnt_rxtx_datahead_g(struct vnt_usb_send_context *tx_context,
 	buf->time_stamp_off_a = vnt_time_stamp_off(priv, rate);
 	buf->time_stamp_off_b = vnt_time_stamp_off(priv,
 						   priv->top_cck_basic_rate);
-
-	tx_context->tx_hdr_size = vnt_mac_hdr_pos(tx_context, &buf->hdr);
-
-	return le16_to_cpu(buf->duration_a);
 }
 
-static u16 vnt_rxtx_datahead_ab(struct vnt_usb_send_context *tx_context,
-				struct vnt_tx_datahead_ab *buf)
+static void vnt_rxtx_datahead_ab(struct vnt_usb_send_context *tx_context,
+				 struct vnt_tx_datahead_ab *buf)
 {
 	struct vnt_private *priv = tx_context->priv;
 	struct ieee80211_hdr *hdr =
@@ -258,14 +217,10 @@ static u16 vnt_rxtx_datahead_ab(struct vnt_usb_send_context *tx_context,
 	/* Get Duration and TimeStampOff */
 	buf->duration = hdr->duration_id;
 	buf->time_stamp_off = vnt_time_stamp_off(priv, rate);
-
-	tx_context->tx_hdr_size = vnt_mac_hdr_pos(tx_context, &buf->hdr);
-
-	return le16_to_cpu(buf->duration);
 }
 
-static int vnt_fill_ieee80211_rts(struct vnt_usb_send_context *tx_context,
-				  struct ieee80211_rts *rts, __le16 duration)
+static void vnt_fill_ieee80211_rts(struct vnt_usb_send_context *tx_context,
+				   struct ieee80211_rts *rts, __le16 duration)
 {
 	struct ieee80211_hdr *hdr =
 				(struct ieee80211_hdr *)tx_context->skb->data;
@@ -276,72 +231,56 @@ static int vnt_fill_ieee80211_rts(struct vnt_usb_send_context *tx_context,
 
 	ether_addr_copy(rts->ra, hdr->addr1);
 	ether_addr_copy(rts->ta, hdr->addr2);
-
-	return 0;
 }
 
-static u16 vnt_rxtx_rts_g_head(struct vnt_usb_send_context *tx_context,
-			       struct vnt_rts_g *buf)
+static void vnt_rxtx_rts_g_head(struct vnt_usb_send_context *tx_context,
+				struct vnt_rts_g *buf)
 {
 	struct vnt_private *priv = tx_context->priv;
 	u16 rts_frame_len = 20;
-	u16 current_rate = tx_context->tx_rate;
 
 	vnt_get_phy_field(priv, rts_frame_len, priv->top_cck_basic_rate,
 			  PK_TYPE_11B, &buf->b);
 	vnt_get_phy_field(priv, rts_frame_len, priv->top_ofdm_basic_rate,
 			  tx_context->pkt_type, &buf->a);
 
-	buf->duration_bb = vnt_get_rtscts_duration_le(tx_context, RTSDUR_BB,
-						      PK_TYPE_11B,
-						      priv->top_cck_basic_rate);
-	buf->duration_aa = vnt_get_rtscts_duration_le(tx_context, RTSDUR_AA,
-						      tx_context->pkt_type,
-						      current_rate);
-	buf->duration_ba = vnt_get_rtscts_duration_le(tx_context, RTSDUR_BA,
-						      tx_context->pkt_type,
-						      current_rate);
+	buf->duration_bb = vnt_get_rts_duration(tx_context);
+	buf->duration_aa = buf->duration_bb;
+	buf->duration_ba = buf->duration_bb;
 
 	vnt_fill_ieee80211_rts(tx_context, &buf->data, buf->duration_aa);
 
-	return vnt_rxtx_datahead_g(tx_context, &buf->data_head);
+	vnt_rxtx_datahead_g(tx_context, &buf->data_head);
 }
 
-static u16 vnt_rxtx_rts_ab_head(struct vnt_usb_send_context *tx_context,
-				struct vnt_rts_ab *buf)
+static void vnt_rxtx_rts_ab_head(struct vnt_usb_send_context *tx_context,
+				 struct vnt_rts_ab *buf)
 {
 	struct vnt_private *priv = tx_context->priv;
-	u16 current_rate = tx_context->tx_rate;
 	u16 rts_frame_len = 20;
 
 	vnt_get_phy_field(priv, rts_frame_len, priv->top_ofdm_basic_rate,
 			  tx_context->pkt_type, &buf->ab);
 
-	buf->duration = vnt_get_rtscts_duration_le(tx_context, RTSDUR_AA,
-						   tx_context->pkt_type,
-						   current_rate);
+	buf->duration = vnt_get_rts_duration(tx_context);
 
 	vnt_fill_ieee80211_rts(tx_context, &buf->data, buf->duration);
 
-	return vnt_rxtx_datahead_ab(tx_context, &buf->data_head);
+	vnt_rxtx_datahead_ab(tx_context, &buf->data_head);
 }
 
-static u16 vnt_fill_cts_head(struct vnt_usb_send_context *tx_context,
-			     union vnt_tx_data_head *head)
+static void vnt_fill_cts_head(struct vnt_usb_send_context *tx_context,
+			      union vnt_tx_data_head *head)
 {
 	struct vnt_private *priv = tx_context->priv;
 	struct vnt_cts *buf = &head->cts_g;
 	u32 cts_frame_len = 14;
-	u16 current_rate = tx_context->tx_rate;
 
 	/* Get SignalField,ServiceField,Length */
 	vnt_get_phy_field(priv, cts_frame_len, priv->top_cck_basic_rate,
 			  PK_TYPE_11B, &buf->b);
 	/* Get CTSDuration_ba */
-	buf->duration_ba =
-		vnt_get_rtscts_duration_le(tx_context, CTSDUR_BA,
-					   tx_context->pkt_type,
-					   current_rate);
+	buf->duration_ba = vnt_get_cts_duration(tx_context);
 	/*Get CTS Frame body*/
 	buf->data.duration = buf->duration_ba;
 	buf->data.frame_control =
@@ -349,127 +288,19 @@ static u16 vnt_fill_cts_head(struct vnt_usb_send_context *tx_context,
 
 	ether_addr_copy(buf->data.ra, priv->current_net_addr);
 
-	return vnt_rxtx_datahead_g(tx_context, &buf->data_head);
+	vnt_rxtx_datahead_g(tx_context, &buf->data_head);
 }
 
-static u16 vnt_rxtx_rts(struct vnt_usb_send_context *tx_context,
-			union vnt_tx_head *tx_head, bool need_mic)
+/* returns true if mic_hdr is needed */
+static bool vnt_fill_txkey(struct vnt_tx_buffer *tx_buffer, struct sk_buff *skb)
 {
-	struct vnt_private *priv = tx_context->priv;
-	struct vnt_rrv_time_rts *buf = &tx_head->tx_rts.rts;
-	union vnt_tx_data_head *head = &tx_head->tx_rts.tx.head;
-	u32 frame_len = tx_context->frame_len;
-	u16 current_rate = tx_context->tx_rate;
-	u8 need_ack = tx_context->need_ack;
-
-	buf->rts_rrv_time_aa = vnt_get_rtscts_rsvtime_le(priv, 2,
-			tx_context->pkt_type, frame_len, current_rate);
-	buf->rts_rrv_time_ba = vnt_get_rtscts_rsvtime_le(priv, 1,
-			tx_context->pkt_type, frame_len, current_rate);
-	buf->rts_rrv_time_bb = vnt_get_rtscts_rsvtime_le(priv, 0,
-			tx_context->pkt_type, frame_len, current_rate);
-
-	buf->rrv_time_a = vnt_rxtx_rsvtime_le16(priv, tx_context->pkt_type,
-						frame_len, current_rate,
-						need_ack);
-	buf->rrv_time_b = vnt_rxtx_rsvtime_le16(priv, PK_TYPE_11B, frame_len,
-					priv->top_cck_basic_rate, need_ack);
-
-	if (need_mic)
-		head = &tx_head->tx_rts.tx.mic.head;
-
-	return vnt_rxtx_rts_g_head(tx_context, &head->rts_g);
-}
-
-static u16 vnt_rxtx_cts(struct vnt_usb_send_context *tx_context,
-			union vnt_tx_head *tx_head, bool need_mic)
-{
-	struct vnt_private *priv = tx_context->priv;
-	struct vnt_rrv_time_cts *buf = &tx_head->tx_cts.cts;
-	union vnt_tx_data_head *head = &tx_head->tx_cts.tx.head;
-	u32 frame_len = tx_context->frame_len;
-	u16 current_rate = tx_context->tx_rate;
-	u8 need_ack = tx_context->need_ack;
-
-	buf->rrv_time_a = vnt_rxtx_rsvtime_le16(priv, tx_context->pkt_type,
-					frame_len, current_rate, need_ack);
-	buf->rrv_time_b = vnt_rxtx_rsvtime_le16(priv, PK_TYPE_11B,
-				frame_len, priv->top_cck_basic_rate, need_ack);
-
-	buf->cts_rrv_time_ba = vnt_get_rtscts_rsvtime_le(priv, 3,
-			tx_context->pkt_type, frame_len, current_rate);
-
-	if (need_mic)
-		head = &tx_head->tx_cts.tx.mic.head;
-
-	return vnt_fill_cts_head(tx_context, head);
-}
-
-static u16 vnt_rxtx_ab(struct vnt_usb_send_context *tx_context,
-		       union vnt_tx_head *tx_head, bool need_rts, bool need_mic)
-{
-	struct vnt_private *priv = tx_context->priv;
-	struct vnt_rrv_time_ab *buf = &tx_head->tx_ab.ab;
-	union vnt_tx_data_head *head = &tx_head->tx_ab.tx.head;
-	u32 frame_len = tx_context->frame_len;
-	u16 current_rate = tx_context->tx_rate;
-	u8 need_ack = tx_context->need_ack;
-
-	buf->rrv_time = vnt_rxtx_rsvtime_le16(priv, tx_context->pkt_type,
-					      frame_len, current_rate, need_ack);
-
-	if (need_mic)
-		head = &tx_head->tx_ab.tx.mic.head;
-
-	if (need_rts) {
-		if (tx_context->pkt_type == PK_TYPE_11B)
-			buf->rts_rrv_time = vnt_get_rtscts_rsvtime_le(priv, 0,
-				tx_context->pkt_type, frame_len, current_rate);
-		else /* PK_TYPE_11A */
-			buf->rts_rrv_time = vnt_get_rtscts_rsvtime_le(priv, 2,
-				tx_context->pkt_type, frame_len, current_rate);
-
-		return vnt_rxtx_rts_ab_head(tx_context, &head->rts_ab);
-	}
-
-	return vnt_rxtx_datahead_ab(tx_context, &head->data_head_ab);
-}
-
-static u16 vnt_generate_tx_parameter(struct vnt_usb_send_context *tx_context,
-				     struct vnt_tx_buffer *tx_buffer,
-				     struct vnt_mic_hdr **mic_hdr, u32 need_mic,
-				     bool need_rts)
-{
-	if (tx_context->pkt_type == PK_TYPE_11GB ||
-	    tx_context->pkt_type == PK_TYPE_11GA) {
-		if (need_rts) {
-			if (need_mic)
-				*mic_hdr =
-					&tx_buffer->tx_head.tx_rts.tx.mic.hdr;
-
-			return vnt_rxtx_rts(tx_context, &tx_buffer->tx_head,
-					    need_mic);
-		}
-
-		if (need_mic)
-			*mic_hdr = &tx_buffer->tx_head.tx_cts.tx.mic.hdr;
-
-		return vnt_rxtx_cts(tx_context, &tx_buffer->tx_head, need_mic);
-	}
-
-	if (need_mic)
-		*mic_hdr = &tx_buffer->tx_head.tx_ab.tx.mic.hdr;
-
-	return vnt_rxtx_ab(tx_context, &tx_buffer->tx_head, need_rts, need_mic);
-}
-
-static void vnt_fill_txkey(struct vnt_usb_send_context *tx_context,
-			   u8 *key_buffer, struct ieee80211_key_conf *tx_key,
-			   struct sk_buff *skb, u16 payload_len,
-			   struct vnt_mic_hdr *mic_hdr)
-{
-	struct ieee80211_hdr *hdr = tx_context->hdr;
+	struct vnt_tx_fifo_head *fifo = &tx_buffer->fifo_head;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_key_conf *tx_key = info->control.hw_key;
+	struct vnt_mic_hdr *mic_hdr;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	u64 pn64;
+	u16 payload_len = skb->len;
 	u8 *iv = ((u8 *)hdr + ieee80211_get_hdrlen_from_skb(skb));
 
 	/* strip header and icv len from payload */
@@ -479,24 +310,31 @@ static void vnt_fill_txkey(struct vnt_usb_send_context *tx_context,
 	switch (tx_key->cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
 	case WLAN_CIPHER_SUITE_WEP104:
-		memcpy(key_buffer, iv, 3);
-		memcpy(key_buffer + 3, tx_key->key, tx_key->keylen);
+		memcpy(fifo->tx_key, iv, 3);
+		memcpy(fifo->tx_key + 3, tx_key->key, tx_key->keylen);
 
 		if (tx_key->keylen == WLAN_KEY_LEN_WEP40) {
-			memcpy(key_buffer + 8, iv, 3);
-			memcpy(key_buffer + 11,
+			memcpy(fifo->tx_key + 8, iv, 3);
+			memcpy(fifo->tx_key + 11,
 			       tx_key->key, WLAN_KEY_LEN_WEP40);
 		}
 
+		fifo->frag_ctl |= cpu_to_le16(FRAGCTL_LEGACY);
 		break;
 	case WLAN_CIPHER_SUITE_TKIP:
-		ieee80211_get_tkip_p2k(tx_key, skb, key_buffer);
+		ieee80211_get_tkip_p2k(tx_key, skb, fifo->tx_key);
 
+		fifo->frag_ctl |= cpu_to_le16(FRAGCTL_TKIP);
 		break;
 	case WLAN_CIPHER_SUITE_CCMP:
-
-		if (!mic_hdr)
-			return;
+		if (info->control.use_cts_prot) {
+			if (info->control.use_rts)
+				mic_hdr = &tx_buffer->tx_head.tx_rts.tx.mic.hdr;
+			else
+				mic_hdr = &tx_buffer->tx_head.tx_cts.tx.mic.hdr;
+		} else {
+			mic_hdr = &tx_buffer->tx_head.tx_ab.tx.mic.hdr;
+		}
 
 		mic_hdr->id = 0x59;
 		mic_hdr->payload_len = cpu_to_be16(payload_len);
@@ -527,12 +365,137 @@ static void vnt_fill_txkey(struct vnt_usb_send_context *tx_context,
 		if (ieee80211_has_a4(hdr->frame_control))
 			ether_addr_copy(mic_hdr->addr4, hdr->addr4);
 
-		memcpy(key_buffer, tx_key->key, WLAN_KEY_LEN_CCMP);
+		memcpy(fifo->tx_key, tx_key->key, WLAN_KEY_LEN_CCMP);
 
-		break;
+		fifo->frag_ctl |= cpu_to_le16(FRAGCTL_AES);
+		return true;
 	default:
 		break;
 	}
+
+	return false;
+}
+
+static void vnt_rxtx_rts(struct vnt_usb_send_context *tx_context)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_context->skb);
+	struct vnt_tx_buffer *tx_buffer = tx_context->tx_buffer;
+	union vnt_tx_head *tx_head = &tx_buffer->tx_head;
+	struct vnt_rrv_time_rts *buf = &tx_head->tx_rts.rts;
+	union vnt_tx_data_head *head = &tx_head->tx_rts.tx.head;
+
+	buf->rts_rrv_time_aa = vnt_get_rts_duration(tx_context);
+	buf->rts_rrv_time_ba = buf->rts_rrv_time_aa;
+	buf->rts_rrv_time_bb = buf->rts_rrv_time_aa;
+
+	buf->rrv_time_a = vnt_rxtx_rsvtime_le16(tx_context);
+	buf->rrv_time_b = buf->rrv_time_a;
+
+	if (info->control.hw_key) {
+		if (vnt_fill_txkey(tx_buffer, tx_context->skb))
+			head = &tx_head->tx_rts.tx.mic.head;
+	}
+
+	vnt_rxtx_rts_g_head(tx_context, &head->rts_g);
+}
+
+static void vnt_rxtx_cts(struct vnt_usb_send_context *tx_context)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_context->skb);
+	struct vnt_tx_buffer *tx_buffer = tx_context->tx_buffer;
+	union vnt_tx_head *tx_head = &tx_buffer->tx_head;
+	struct vnt_rrv_time_cts *buf = &tx_head->tx_cts.cts;
+	union vnt_tx_data_head *head = &tx_head->tx_cts.tx.head;
+
+	buf->rrv_time_a = vnt_rxtx_rsvtime_le16(tx_context);
+	buf->rrv_time_b = buf->rrv_time_a;
+
+	buf->cts_rrv_time_ba = vnt_get_cts_duration(tx_context);
+
+	if (info->control.hw_key) {
+		if (vnt_fill_txkey(tx_buffer, tx_context->skb))
+			head = &tx_head->tx_cts.tx.mic.head;
+	}
+
+	vnt_fill_cts_head(tx_context, head);
+}
+
+static void vnt_rxtx_ab(struct vnt_usb_send_context *tx_context)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_context->skb);
+	struct vnt_tx_buffer *tx_buffer = tx_context->tx_buffer;
+	union vnt_tx_head *tx_head = &tx_buffer->tx_head;
+	struct vnt_rrv_time_ab *buf = &tx_head->tx_ab.ab;
+	union vnt_tx_data_head *head = &tx_head->tx_ab.tx.head;
+
+	buf->rrv_time = vnt_rxtx_rsvtime_le16(tx_context);
+
+	if (info->control.hw_key) {
+		if (vnt_fill_txkey(tx_buffer, tx_context->skb))
+			head = &tx_head->tx_ab.tx.mic.head;
+	}
+
+	if (info->control.use_rts) {
+		buf->rts_rrv_time = vnt_get_rts_duration(tx_context);
+
+		vnt_rxtx_rts_ab_head(tx_context, &head->rts_ab);
+
+		return;
+	}
+
+	vnt_rxtx_datahead_ab(tx_context, &head->data_head_ab);
+}
+
+static void vnt_generate_tx_parameter(struct vnt_usb_send_context *tx_context)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_context->skb);
+
+	if (info->control.use_cts_prot) {
+		if (info->control.use_rts) {
+			vnt_rxtx_rts(tx_context);
+
+			return;
+		}
+
+		vnt_rxtx_cts(tx_context);
+
+		return;
+	}
+
+	vnt_rxtx_ab(tx_context);
+}
+
+static u16 vnt_get_hdr_size(struct ieee80211_tx_info *info)
+{
+	u16 size = sizeof(struct vnt_tx_datahead_ab);
+
+	if (info->control.use_cts_prot) {
+		if (info->control.use_rts)
+			size = sizeof(struct vnt_rts_g);
+		else
+			size = sizeof(struct vnt_cts);
+	} else if (info->control.use_rts) {
+		size = sizeof(struct vnt_rts_ab);
+	}
+
+	if (info->control.hw_key) {
+		if (info->control.hw_key->cipher == WLAN_CIPHER_SUITE_CCMP)
+			size += sizeof(struct vnt_mic_hdr);
+	}
+
+	/* Get rrv_time header */
+	if (info->control.use_cts_prot) {
+		if (info->control.use_rts)
+			size += sizeof(struct vnt_rrv_time_rts);
+		else
+			size += sizeof(struct vnt_rrv_time_cts);
+	} else {
+		size += sizeof(struct vnt_rrv_time_ab);
+	}
+
+	size += sizeof(struct vnt_tx_fifo_head);
+
+	return size;
 }
 
 int vnt_tx_packet(struct vnt_private *priv, struct sk_buff *skb)
@@ -540,30 +503,18 @@ int vnt_tx_packet(struct vnt_private *priv, struct sk_buff *skb)
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_tx_rate *tx_rate = &info->control.rates[0];
 	struct ieee80211_rate *rate;
-	struct ieee80211_key_conf *tx_key;
 	struct ieee80211_hdr *hdr;
-	struct vnt_mic_hdr *mic_hdr = NULL;
 	struct vnt_tx_buffer *tx_buffer;
 	struct vnt_tx_fifo_head *tx_buffer_head;
 	struct vnt_usb_send_context *tx_context;
 	unsigned long flags;
-	u16 tx_bytes, tx_header_size, tx_body_size, current_rate, duration_id;
 	u8 pkt_type;
-	bool need_rts = false;
-	bool need_mic = false;
 
 	hdr = (struct ieee80211_hdr *)(skb->data);
 
 	rate = ieee80211_get_tx_rate(priv->hw, info);
 
-	current_rate = rate->hw_value;
-	if (priv->current_rate != current_rate &&
-	    !(priv->hw->conf.flags & IEEE80211_CONF_OFFCHANNEL)) {
-		priv->current_rate = current_rate;
-		vnt_schedule_command(priv, WLAN_CMD_SETPOWER);
-	}
-
-	if (current_rate > RATE_11M) {
+	if (rate->hw_value > RATE_11M) {
 		if (info->band == NL80211_BAND_5GHZ) {
 			pkt_type = PK_TYPE_11A;
 		} else {
@@ -589,17 +540,23 @@ int vnt_tx_packet(struct vnt_private *priv, struct sk_buff *skb)
 		return -ENOMEM;
 	}
 
-	tx_context->skb = skb;
 	tx_context->pkt_type = pkt_type;
-	tx_context->need_ack = false;
 	tx_context->frame_len = skb->len + 4;
-	tx_context->tx_rate = current_rate;
+	tx_context->tx_rate =  rate->hw_value;
 
 	spin_unlock_irqrestore(&priv->lock, flags);
 
-	tx_buffer = (struct vnt_tx_buffer *)tx_context->data;
+	tx_context->skb = skb_clone(skb, GFP_ATOMIC);
+	if (!tx_context->skb) {
+		tx_context->in_use = false;
+		return -ENOMEM;
+	}
+
+	tx_buffer = skb_push(skb, vnt_get_hdr_size(info));
+	tx_context->tx_buffer = tx_buffer;
 	tx_buffer_head = &tx_buffer->fifo_head;
-	tx_body_size = skb->len;
+
+	tx_context->type = CONTEXT_DATA_PACKET;
 
 	/*Set fifo controls */
 	if (pkt_type == PK_TYPE_11A)
@@ -623,92 +580,43 @@ int vnt_tx_packet(struct vnt_private *priv, struct sk_buff *skb)
 			cpu_to_le16(DEFAULT_MSDU_LIFETIME_RES_64us);
 	}
 
-	if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) {
+	if (!(info->flags & IEEE80211_TX_CTL_NO_ACK))
 		tx_buffer_head->fifo_ctl |= cpu_to_le16(FIFOCTL_NEEDACK);
-		tx_context->need_ack = true;
-	}
 
 	if (ieee80211_has_retry(hdr->frame_control))
 		tx_buffer_head->fifo_ctl |= cpu_to_le16(FIFOCTL_LRETRY);
 
-	if (tx_rate->flags & IEEE80211_TX_RC_USE_SHORT_PREAMBLE)
-		priv->preamble_type = PREAMBLE_SHORT;
-	else
-		priv->preamble_type = PREAMBLE_LONG;
-
-	if (tx_rate->flags & IEEE80211_TX_RC_USE_RTS_CTS) {
-		need_rts = true;
+	if (info->control.use_rts)
 		tx_buffer_head->fifo_ctl |= cpu_to_le16(FIFOCTL_RTS);
-	}
 
 	if (ieee80211_has_a4(hdr->frame_control))
 		tx_buffer_head->fifo_ctl |= cpu_to_le16(FIFOCTL_LHEAD);
 
 	tx_buffer_head->frag_ctl =
-			cpu_to_le16(ieee80211_get_hdrlen_from_skb(skb) << 10);
+			cpu_to_le16(ieee80211_hdrlen(hdr->frame_control) << 10);
 
-	if (info->control.hw_key) {
-		tx_key = info->control.hw_key;
-		switch (info->control.hw_key->cipher) {
-		case WLAN_CIPHER_SUITE_WEP40:
-		case WLAN_CIPHER_SUITE_WEP104:
-			tx_buffer_head->frag_ctl |= cpu_to_le16(FRAGCTL_LEGACY);
-			break;
-		case WLAN_CIPHER_SUITE_TKIP:
-			tx_buffer_head->frag_ctl |= cpu_to_le16(FRAGCTL_TKIP);
-			break;
-		case WLAN_CIPHER_SUITE_CCMP:
-			tx_buffer_head->frag_ctl |= cpu_to_le16(FRAGCTL_AES);
-			need_mic = true;
-		default:
-			break;
-		}
-		tx_context->frame_len += tx_key->icv_len;
-	}
+	if (info->control.hw_key)
+		tx_context->frame_len += info->control.hw_key->icv_len;
 
-	tx_buffer_head->current_rate = cpu_to_le16(current_rate);
+	tx_buffer_head->current_rate = cpu_to_le16(rate->hw_value);
 
-	duration_id = vnt_generate_tx_parameter(tx_context, tx_buffer, &mic_hdr,
-						need_mic, need_rts);
-
-	tx_header_size = tx_context->tx_hdr_size;
-	if (!tx_header_size) {
-		tx_context->in_use = false;
-		return -ENOMEM;
-	}
+	vnt_generate_tx_parameter(tx_context);
 
 	tx_buffer_head->frag_ctl |= cpu_to_le16(FRAGCTL_NONFRAG);
 
-	tx_bytes = tx_header_size + tx_body_size;
-
-	memcpy(tx_context->hdr, skb->data, tx_body_size);
-
-	if (info->control.hw_key) {
-		tx_key = info->control.hw_key;
-		if (tx_key->keylen > 0)
-			vnt_fill_txkey(tx_context, tx_buffer_head->tx_key,
-				       tx_key, skb, tx_body_size, mic_hdr);
-	}
-
 	priv->seq_counter = (le16_to_cpu(hdr->seq_ctrl) &
 						IEEE80211_SCTL_SEQ) >> 4;
 
-	tx_buffer->tx_byte_count = cpu_to_le16(tx_bytes);
-	tx_buffer->pkt_no = tx_context->pkt_no;
-	tx_buffer->type = 0x00;
-
-	tx_bytes += 4;
-
-	tx_context->type = CONTEXT_DATA_PACKET;
-	tx_context->buf_len = tx_bytes;
-
 	spin_lock_irqsave(&priv->lock, flags);
 
-	if (vnt_tx_context(priv, tx_context)) {
+	if (vnt_tx_context(priv, tx_context, skb)) {
+		dev_kfree_skb(tx_context->skb);
 		spin_unlock_irqrestore(&priv->lock, flags);
 		return -EIO;
 	}
 
+	dev_kfree_skb(skb);
+
 	spin_unlock_irqrestore(&priv->lock, flags);
 
 	return 0;
@@ -716,14 +624,13 @@ int vnt_tx_packet(struct vnt_private *priv, struct sk_buff *skb)
 
 static int vnt_beacon_xmit(struct vnt_private *priv, struct sk_buff *skb)
 {
-	struct vnt_beacon_buffer *beacon_buffer;
 	struct vnt_tx_short_buf_head *short_head;
 	struct ieee80211_tx_info *info;
 	struct vnt_usb_send_context *context;
 	struct ieee80211_mgmt *mgmt_hdr;
 	unsigned long flags;
 	u32 frame_size = skb->len + 4;
-	u16 current_rate, count;
+	u16 current_rate;
 
 	spin_lock_irqsave(&priv->lock, flags);
 
@@ -738,8 +645,8 @@ static int vnt_beacon_xmit(struct vnt_private *priv, struct sk_buff *skb)
 
 	spin_unlock_irqrestore(&priv->lock, flags);
 
-	beacon_buffer = (struct vnt_beacon_buffer *)&context->data[0];
-	short_head = &beacon_buffer->short_head;
+	mgmt_hdr = (struct ieee80211_mgmt *)skb->data;
+	short_head = skb_push(skb, sizeof(*short_head));
 
 	if (priv->bb_type == BB_TYPE_11A) {
 		current_rate = RATE_6M;
@@ -764,10 +671,6 @@ static int vnt_beacon_xmit(struct vnt_private *priv, struct sk_buff *skb)
 			vnt_time_stamp_off(priv, current_rate);
 	}
 
-	/* Generate Beacon Header */
-	mgmt_hdr = &beacon_buffer->mgmt_hdr;
-	memcpy(mgmt_hdr, skb->data, skb->len);
-
 	/* Get Duration */
 	short_head->duration = mgmt_hdr->duration;
 
@@ -786,18 +689,11 @@ static int vnt_beacon_xmit(struct vnt_private *priv, struct sk_buff *skb)
 	if (priv->seq_counter > 0x0fff)
 		priv->seq_counter = 0;
 
-	count = sizeof(struct vnt_tx_short_buf_head) + skb->len;
-
-	beacon_buffer->tx_byte_count = cpu_to_le16(count);
-	beacon_buffer->pkt_no = context->pkt_no;
-	beacon_buffer->type = 0x01;
-
 	context->type = CONTEXT_BEACON_PACKET;
-	context->buf_len = count + 4; /* USB header */
 
 	spin_lock_irqsave(&priv->lock, flags);
 
-	if (vnt_tx_context(priv, context))
+	if (vnt_tx_context(priv, context, skb))
 		ieee80211_free_txskb(priv->hw, context->skb);
 
 	spin_unlock_irqrestore(&priv->lock, flags);
diff --git a/drivers/staging/vt6656/rxtx.h b/drivers/staging/vt6656/rxtx.h
index 0e6226af7d41..6ca2ca32d036 100644
--- a/drivers/staging/vt6656/rxtx.h
+++ b/drivers/staging/vt6656/rxtx.h
@@ -23,6 +23,13 @@
 #define DEFAULT_MGN_LIFETIME_RES_64us	125  /* 64us */
 #define DEFAULT_MSDU_LIFETIME_RES_64us  8000
 
+/* Length, Service, and Signal fields of Phy for Tx */
+struct vnt_phy_field {
+	u8 signal;
+	u8 service;
+	__le16 len;
+} __packed;
+
 /* MIC HDR data header */
 struct vnt_mic_hdr {
 	u8 id;
@@ -70,14 +77,12 @@ struct vnt_tx_datahead_g {
 	__le16 duration_a;
 	__le16 time_stamp_off_b;
 	__le16 time_stamp_off_a;
-	struct ieee80211_hdr hdr;
 } __packed;
 
 struct vnt_tx_datahead_ab {
 	struct vnt_phy_field ab;
 	__le16 duration;
 	__le16 time_stamp_off;
-	struct ieee80211_hdr hdr;
 } __packed;
 
 /* RTS buffer header */
@@ -155,9 +160,6 @@ struct vnt_tx_fifo_head {
 } __packed;
 
 struct vnt_tx_buffer {
-	u8 type;
-	u8 pkt_no;
-	__le16 tx_byte_count;
 	struct vnt_tx_fifo_head fifo_head;
 	union vnt_tx_head tx_head;
 } __packed;
@@ -170,14 +172,6 @@ struct vnt_tx_short_buf_head {
 	__le16 time_stamp_off;
 } __packed;
 
-struct vnt_beacon_buffer {
-	u8 type;
-	u8 pkt_no;
-	__le16 tx_byte_count;
-	struct vnt_tx_short_buf_head short_head;
-	struct ieee80211_mgmt mgmt_hdr;
-} __packed;
-
 int vnt_tx_packet(struct vnt_private *priv, struct sk_buff *skb);
 int vnt_beacon_make(struct vnt_private *priv, struct ieee80211_vif *vif);
 int vnt_beacon_enable(struct vnt_private *priv, struct ieee80211_vif *vif,
diff --git a/drivers/staging/vt6656/usbpipe.c b/drivers/staging/vt6656/usbpipe.c
index 91b62c3dff7b..82b774be6485 100644
--- a/drivers/staging/vt6656/usbpipe.c
+++ b/drivers/staging/vt6656/usbpipe.c
@@ -77,7 +77,7 @@ int vnt_control_out_u8(struct vnt_private *priv, u8 reg, u8 reg_off, u8 data)
 }
 
 int vnt_control_out_blocks(struct vnt_private *priv,
-			   u16 block, u8 reg, u16 length, u8 *data)
+			   u16 block, u8 reg, u16 length, const u8 *data)
 {
 	int ret = 0, i;
 
@@ -196,32 +196,18 @@ static void vnt_int_process_data(struct vnt_private *priv)
 	if (int_data->tsr3 & TSR_VALID)
 		vnt_int_report_rate(priv, int_data->pkt3, int_data->tsr3);
 
-	if (int_data->isr0 != 0) {
-		if (int_data->isr0 & ISR_BNTX &&
-		    priv->op_mode == NL80211_IFTYPE_AP)
-			vnt_schedule_command(priv, WLAN_CMD_BECON_SEND);
-
-		if (int_data->isr0 & ISR_TBTT &&
-		    priv->hw->conf.flags & IEEE80211_CONF_PS) {
-			if (!priv->wake_up_count)
-				priv->wake_up_count =
-					priv->hw->conf.listen_interval;
+	if (!int_data->isr0)
+		return;
 
-			if (priv->wake_up_count)
-				--priv->wake_up_count;
+	if (int_data->isr0 & ISR_BNTX && priv->op_mode == NL80211_IFTYPE_AP)
+		vnt_schedule_command(priv, WLAN_CMD_BECON_SEND);
 
-			/* Turn on wake up to listen next beacon */
-			if (priv->wake_up_count == 1)
-				vnt_schedule_command(priv,
-						     WLAN_CMD_TBTT_WAKEUP);
-		}
-		priv->current_tsf = le64_to_cpu(int_data->tsf);
+	priv->current_tsf = le64_to_cpu(int_data->tsf);
 
-		low_stats->dot11RTSSuccessCount += int_data->rts_success;
-		low_stats->dot11RTSFailureCount += int_data->rts_fail;
-		low_stats->dot11ACKFailureCount += int_data->ack_fail;
-		low_stats->dot11FCSErrorCount += int_data->fcs_err;
-	}
+	low_stats->dot11RTSSuccessCount += int_data->rts_success;
+	low_stats->dot11RTSFailureCount += int_data->rts_fail;
+	low_stats->dot11ACKFailureCount += int_data->ack_fail;
+	low_stats->dot11FCSErrorCount += int_data->fcs_err;
 }
 
 static void vnt_start_interrupt_urb_complete(struct urb *urb)
@@ -442,7 +428,8 @@ static void vnt_tx_context_complete(struct urb *urb)
 
 	switch (urb->status) {
 	case 0:
-		dev_dbg(&priv->usb->dev, "Write %d bytes\n", context->buf_len);
+		dev_dbg(&priv->usb->dev,
+			"Write %d bytes\n", urb->actual_length);
 		break;
 	case -ECONNRESET:
 	case -ENOENT:
@@ -467,30 +454,53 @@ static void vnt_tx_context_complete(struct urb *urb)
 }
 
 int vnt_tx_context(struct vnt_private *priv,
-		   struct vnt_usb_send_context *context)
+		   struct vnt_usb_send_context *context,
+		   struct sk_buff *skb)
 {
+	struct vnt_tx_usb_header *usb;
+	struct urb *urb;
 	int status;
-	struct urb *urb = context->urb;
+	u16 count = skb->len;
+
+	usb = skb_push(skb, sizeof(*usb));
+	usb->tx_byte_count = cpu_to_le16(count);
+	usb->pkt_no = context->pkt_no;
+	usb->type = context->type;
 
 	if (test_bit(DEVICE_FLAGS_DISCONNECTED, &priv->flags)) {
 		context->in_use = false;
 		return -ENODEV;
 	}
 
+	if (skb->len > MAX_TOTAL_SIZE_WITH_ALL_HEADERS) {
+		context->in_use = false;
+		return -E2BIG;
+	}
+
+	urb = usb_alloc_urb(0, GFP_ATOMIC);
+	if (!urb) {
+		context->in_use = false;
+		return -ENOMEM;
+	}
+
 	usb_fill_bulk_urb(urb,
 			  priv->usb,
 			  usb_sndbulkpipe(priv->usb, 3),
-			  context->data,
-			  context->buf_len,
+			  skb->data,
+			  skb->len,
 			  vnt_tx_context_complete,
 			  context);
 
+	usb_anchor_urb(urb, &priv->tx_submitted);
+
 	status = usb_submit_urb(urb, GFP_ATOMIC);
 	if (status) {
 		dev_dbg(&priv->usb->dev, "Submit Tx URB failed %d\n", status);
-
+		usb_unanchor_urb(urb);
 		context->in_use = false;
 	}
 
+	usb_free_urb(urb);
+
 	return status;
 }
diff --git a/drivers/staging/vt6656/usbpipe.h b/drivers/staging/vt6656/usbpipe.h
index 35697b58d748..52c2a928c9c1 100644
--- a/drivers/staging/vt6656/usbpipe.h
+++ b/drivers/staging/vt6656/usbpipe.h
@@ -41,6 +41,12 @@ struct vnt_interrupt_data {
 	u8 sw[2];
 } __packed;
 
+struct vnt_tx_usb_header {
+	u8 type;
+	u8 pkt_no;
+	__le16 tx_byte_count;
+} __packed;
+
 #define VNT_REG_BLOCK_SIZE	64
 
 int vnt_control_out(struct vnt_private *priv, u8 request, u16 value,
@@ -52,11 +58,12 @@ int vnt_control_out_u8(struct vnt_private *priv, u8 reg, u8 ref_off, u8 data);
 int vnt_control_in_u8(struct vnt_private *priv, u8 reg, u8 reg_off, u8 *data);
 
 int vnt_control_out_blocks(struct vnt_private *priv,
-			   u16 block, u8 reg, u16 len, u8 *data);
+			   u16 block, u8 reg, u16 len, const u8 *data);
 
 int vnt_start_interrupt_urb(struct vnt_private *priv);
 int vnt_submit_rx_urb(struct vnt_private *priv, struct vnt_rcb *rcb);
 int vnt_tx_context(struct vnt_private *priv,
-		   struct vnt_usb_send_context *context);
+		   struct vnt_usb_send_context *context,
+		   struct sk_buff *skb);
 
 #endif /* __USBPIPE_H__ */
diff --git a/drivers/staging/vt6656/wcmd.c b/drivers/staging/vt6656/wcmd.c
index 2c5250ca2801..0ccc87da394e 100644
--- a/drivers/staging/vt6656/wcmd.c
+++ b/drivers/staging/vt6656/wcmd.c
@@ -122,8 +122,7 @@ void vnt_run_command(struct work_struct *work)
 
 	case WLAN_CMD_SETPOWER_START:
 
-		vnt_rf_setpower(priv, priv->current_rate,
-				priv->hw->conf.chandef.chan->hw_value);
+		vnt_rf_setpower(priv, priv->hw->conf.chandef.chan);
 
 		break;
 
diff --git a/drivers/staging/wfx/Makefile b/drivers/staging/wfx/Makefile
index 0d9c1ed092f6..0e0cc982ceab 100644
--- a/drivers/staging/wfx/Makefile
+++ b/drivers/staging/wfx/Makefile
@@ -7,6 +7,7 @@ wfx-y := \
 	bh.o \
 	hwio.o \
 	fwio.o \
+	hif_tx_mib.o \
 	hif_tx.o \
 	hif_rx.o \
 	queue.o \
diff --git a/drivers/staging/wfx/TODO b/drivers/staging/wfx/TODO
index efcb7c6a5aa7..42bf36d43970 100644
--- a/drivers/staging/wfx/TODO
+++ b/drivers/staging/wfx/TODO
@@ -1,60 +1,25 @@
 This is a list of things that need to be done to get this driver out of the
 staging directory.
 
-  - All structures defined in hif_api_*.h are intended to sent/received to/from
-    hardware. All their members whould be declared __le32 or __le16.
-    See:
-       https://lore.kernel.org/lkml/20191111202852.GX26530@ZenIV.linux.org.uk
+  - The HIF API is not yet clean enough.
 
-  - Once previous item done, it will be possible to audit the driver with
-    `sparse'. It will probably find tons of problems with big endian
-    architectures.
-
-  - hif_api_*.h whave been imported from firmware code. Some of the structures
-    are never used in driver.
-
-  - Driver try to maintains power save status of the stations. However, this
-    work is already done by mac80211. sta_asleep_mask and pspoll_mask should be
-    dropped.
-
-  - wfx_tx_queues_get() should be reworked. It currently try compute itself the
-    QoS policy. However, firmware already do the job. Firmware would prefer to
-    have a few packets in each queue and be able to choose itself which queue to
-    use.
+  - The code that check the corectness of received message (in rx_helper()) can
+    be improved. See:
+       https://lore.kernel.org/driverdev-devel/2302785.6C7ODC2LYm@pc-42/
 
   - As suggested by Felix, rate control could be improved following this idea:
         https://lore.kernel.org/lkml/3099559.gv3Q75KnN1@pc-42/
 
-  - When driver is about to loose BSS, it forge its own Null Func request (see
-    wfx_cqm_bssloss_sm()). It should use mechanism provided by mac80211.
-
-  - AP is actually is setup after a call to wfx_bss_info_changed(). Yet,
-    ieee80211_ops provide callback start_ap().
-
-  - The current process for joining a network is incredibly complex. Should be
-    reworked.
-
-  - Monitoring mode is not implemented despite being mandatory by mac80211.
-
-  - "compatible" value are not correct. They should be "vendor,chip". See:
-       https://lore.kernel.org/driverdev-devel/5226570.CMH5hVlZcI@pc-42
-
-  - The "state" field from wfx_vif should be replaced by "vif->type".
-
-  - It seems that wfx_upload_keys() is useless.
-
-  - "event_queue" from wfx_vif seems overkill. These event are rare and they
-     probably could be handled in a simpler fashion.
-
   - Feature called "secure link" should be either developed (using kernel
     crypto API) or dropped.
 
+  - The device allows to filter multicast traffic. The code to support these
+    filters exists in the driver but it is disabled because it has never been
+    tested.
+
   - In wfx_cmd_send(), "async" allow to send command without waiting the reply.
     It may help in some situation, but it is not yet used. In add, it may cause
     some trouble:
       https://lore.kernel.org/driverdev-devel/alpine.DEB.2.21.1910041317381.2992@hadrien/
     So, fix it (by replacing the mutex with a semaphore) or drop it.
 
-  - Chip support P2P, but driver does not implement it.
-
-  - Chip support kind of Mesh, but driver does not implement it.
diff --git a/drivers/staging/wfx/bh.c b/drivers/staging/wfx/bh.c
index 9fcab00a3733..1cbaf8bb4fa3 100644
--- a/drivers/staging/wfx/bh.c
+++ b/drivers/staging/wfx/bh.c
@@ -70,7 +70,7 @@ static int rx_helper(struct wfx_dev *wdev, size_t read_len, int *is_cnf)
 	if (wfx_data_read(wdev, skb->data, alloc_len))
 		goto err;
 
-	piggyback = le16_to_cpup((u16 *)(skb->data + alloc_len - 2));
+	piggyback = le16_to_cpup((__le16 *)(skb->data + alloc_len - 2));
 	_trace_piggyback(piggyback, false);
 
 	hif = (struct hif_msg *)skb->data;
@@ -84,13 +84,12 @@ static int rx_helper(struct wfx_dev *wdev, size_t read_len, int *is_cnf)
 			// piggyback is probably correct.
 			return piggyback;
 		}
-		le16_to_cpus(&hif->len);
-		computed_len = round_up(hif->len - sizeof(hif->len), 16)
-			       + sizeof(struct hif_sl_msg)
-			       + sizeof(struct hif_sl_tag);
+		computed_len =
+			round_up(le16_to_cpu(hif->len) - sizeof(hif->len), 16) +
+			sizeof(struct hif_sl_msg) +
+			sizeof(struct hif_sl_tag);
 	} else {
-		le16_to_cpus(&hif->len);
-		computed_len = round_up(hif->len, 2);
+		computed_len = round_up(le16_to_cpu(hif->len), 2);
 	}
 	if (computed_len != read_len) {
 		dev_err(wdev->dev, "inconsistent message length: %zu != %zu\n",
@@ -103,13 +102,11 @@ static int rx_helper(struct wfx_dev *wdev, size_t read_len, int *is_cnf)
 	if (!(hif->id & HIF_ID_IS_INDICATION)) {
 		(*is_cnf)++;
 		if (hif->id == HIF_CNF_ID_MULTI_TRANSMIT)
-			release_count = le32_to_cpu(((struct hif_cnf_multi_transmit *)hif->body)->num_tx_confs);
+			release_count = ((struct hif_cnf_multi_transmit *)hif->body)->num_tx_confs;
 		else
 			release_count = 1;
 		WARN(wdev->hif.tx_buffers_used < release_count, "corrupted buffer counter");
 		wdev->hif.tx_buffers_used -= release_count;
-		if (!wdev->hif.tx_buffers_used)
-			wake_up(&wdev->hif.tx_buffers_empty);
 	}
 	_trace_hif_recv(hif, wdev->hif.tx_buffers_used);
 
@@ -120,9 +117,11 @@ static int rx_helper(struct wfx_dev *wdev, size_t read_len, int *is_cnf)
 		wdev->hif.rx_seqnum = (hif->seqnum + 1) % (HIF_COUNTER_MAX + 1);
 	}
 
-	skb_put(skb, hif->len);
+	skb_put(skb, le16_to_cpu(hif->len));
 	// wfx_handle_rx takes care on SKB livetime
 	wfx_handle_rx(wdev, skb);
+	if (!wdev->hif.tx_buffers_used)
+		wake_up(&wdev->hif.tx_buffers_empty);
 
 	return piggyback;
 
@@ -307,6 +306,35 @@ void wfx_bh_request_tx(struct wfx_dev *wdev)
 	queue_work(system_highpri_wq, &wdev->hif.bh);
 }
 
+/*
+ * If IRQ is not available, this function allow to manually poll the control
+ * register and simulate an IRQ ahen an event happened.
+ *
+ * Note that the device has a bug: If an IRQ raise while host read control
+ * register, the IRQ is lost. So, use this function carefully (only duing
+ * device initialisation).
+ */
+void wfx_bh_poll_irq(struct wfx_dev *wdev)
+{
+	ktime_t now, start;
+	u32 reg;
+
+	WARN(!wdev->poll_irq, "unexpected IRQ polling can mask IRQ");
+	start = ktime_get();
+	for (;;) {
+		control_reg_read(wdev, &reg);
+		now = ktime_get();
+		if (reg & 0xFFF)
+			break;
+		if (ktime_after(now, ktime_add_ms(start, 1000))) {
+			dev_err(wdev->dev, "time out while polling control register\n");
+			return;
+		}
+		udelay(200);
+	}
+	wfx_bh_request_rx(wdev);
+}
+
 void wfx_bh_register(struct wfx_dev *wdev)
 {
 	INIT_WORK(&wdev->hif.bh, bh_work);
diff --git a/drivers/staging/wfx/bh.h b/drivers/staging/wfx/bh.h
index 93ca98424e0b..4b73437869e1 100644
--- a/drivers/staging/wfx/bh.h
+++ b/drivers/staging/wfx/bh.h
@@ -28,5 +28,6 @@ void wfx_bh_register(struct wfx_dev *wdev);
 void wfx_bh_unregister(struct wfx_dev *wdev);
 void wfx_bh_request_rx(struct wfx_dev *wdev);
 void wfx_bh_request_tx(struct wfx_dev *wdev);
+void wfx_bh_poll_irq(struct wfx_dev *wdev);
 
 #endif /* WFX_BH_H */
diff --git a/drivers/staging/wfx/bus.h b/drivers/staging/wfx/bus.h
index 62d6ecabe4cb..0370b6c59863 100644
--- a/drivers/staging/wfx/bus.h
+++ b/drivers/staging/wfx/bus.h
@@ -25,6 +25,8 @@ struct hwbus_ops {
 			    void *dst, size_t count);
 	int (*copy_to_io)(void *bus_priv, unsigned int addr,
 			  const void *src, size_t count);
+	int (*irq_subscribe)(void *bus_priv);
+	int (*irq_unsubscribe)(void *bus_priv);
 	void (*lock)(void *bus_priv);
 	void (*unlock)(void *bus_priv);
 	size_t (*align_size)(void *bus_priv, size_t size);
diff --git a/drivers/staging/wfx/bus_sdio.c b/drivers/staging/wfx/bus_sdio.c
index dedc3ff58d3e..496bfc8bbacc 100644
--- a/drivers/staging/wfx/bus_sdio.c
+++ b/drivers/staging/wfx/bus_sdio.c
@@ -6,10 +6,12 @@
  * Copyright (c) 2010, ST-Ericsson
  */
 #include <linux/module.h>
+#include <linux/mmc/sdio.h>
 #include <linux/mmc/sdio_func.h>
 #include <linux/mmc/card.h>
 #include <linux/interrupt.h>
 #include <linux/of_irq.h>
+#include <linux/irq.h>
 
 #include "bus.h"
 #include "wfx.h"
@@ -91,53 +93,58 @@ static void wfx_sdio_irq_handler(struct sdio_func *func)
 {
 	struct wfx_sdio_priv *bus = sdio_get_drvdata(func);
 
-	if (bus->core)
-		wfx_bh_request_rx(bus->core);
-	else
-		WARN(!bus->core, "race condition in driver init/deinit");
+	wfx_bh_request_rx(bus->core);
 }
 
 static irqreturn_t wfx_sdio_irq_handler_ext(int irq, void *priv)
 {
 	struct wfx_sdio_priv *bus = priv;
 
-	if (!bus->core) {
-		WARN(!bus->core, "race condition in driver init/deinit");
-		return IRQ_NONE;
-	}
 	sdio_claim_host(bus->func);
 	wfx_bh_request_rx(bus->core);
 	sdio_release_host(bus->func);
 	return IRQ_HANDLED;
 }
 
-static int wfx_sdio_irq_subscribe(struct wfx_sdio_priv *bus)
+static int wfx_sdio_irq_subscribe(void *priv)
 {
+	struct wfx_sdio_priv *bus = priv;
+	u32 flags;
 	int ret;
+	u8 cccr;
 
-	if (bus->of_irq) {
-		ret = request_irq(bus->of_irq, wfx_sdio_irq_handler_ext,
-				  IRQF_TRIGGER_RISING, "wfx", bus);
-	} else {
+	if (!bus->of_irq) {
 		sdio_claim_host(bus->func);
 		ret = sdio_claim_irq(bus->func, wfx_sdio_irq_handler);
 		sdio_release_host(bus->func);
+		return ret;
 	}
-	return ret;
+
+	sdio_claim_host(bus->func);
+	cccr = sdio_f0_readb(bus->func, SDIO_CCCR_IENx, NULL);
+	cccr |= BIT(0);
+	cccr |= BIT(bus->func->num);
+	sdio_f0_writeb(bus->func, cccr, SDIO_CCCR_IENx, NULL);
+	sdio_release_host(bus->func);
+	flags = irq_get_trigger_type(bus->of_irq);
+	if (!flags)
+		flags = IRQF_TRIGGER_HIGH;
+	flags |= IRQF_ONESHOT;
+	return devm_request_threaded_irq(&bus->func->dev, bus->of_irq, NULL,
+					 wfx_sdio_irq_handler_ext, flags,
+					 "wfx", bus);
 }
 
-static int wfx_sdio_irq_unsubscribe(struct wfx_sdio_priv *bus)
+static int wfx_sdio_irq_unsubscribe(void *priv)
 {
+	struct wfx_sdio_priv *bus = priv;
 	int ret;
 
-	if (bus->of_irq) {
-		free_irq(bus->of_irq, bus);
-		ret = 0;
-	} else {
-		sdio_claim_host(bus->func);
-		ret = sdio_release_irq(bus->func);
-		sdio_release_host(bus->func);
-	}
+	if (bus->of_irq)
+		devm_free_irq(&bus->func->dev, bus->of_irq, bus);
+	sdio_claim_host(bus->func);
+	ret = sdio_release_irq(bus->func);
+	sdio_release_host(bus->func);
 	return ret;
 }
 
@@ -151,12 +158,20 @@ static size_t wfx_sdio_align_size(void *priv, size_t size)
 static const struct hwbus_ops wfx_sdio_hwbus_ops = {
 	.copy_from_io = wfx_sdio_copy_from_io,
 	.copy_to_io = wfx_sdio_copy_to_io,
+	.irq_subscribe = wfx_sdio_irq_subscribe,
+	.irq_unsubscribe = wfx_sdio_irq_unsubscribe,
 	.lock			= wfx_sdio_lock,
 	.unlock			= wfx_sdio_unlock,
 	.align_size		= wfx_sdio_align_size,
 };
 
-static const struct of_device_id wfx_sdio_of_match[];
+static const struct of_device_id wfx_sdio_of_match[] = {
+	{ .compatible = "silabs,wfx-sdio" },
+	{ .compatible = "silabs,wf200" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, wfx_sdio_of_match);
+
 static int wfx_sdio_probe(struct sdio_func *func,
 			  const struct sdio_device_id *id)
 {
@@ -165,7 +180,8 @@ static int wfx_sdio_probe(struct sdio_func *func,
 	int ret;
 
 	if (func->num != 1) {
-		dev_err(&func->dev, "SDIO function number is %d while it should always be 1 (unsupported chip?)\n", func->num);
+		dev_err(&func->dev, "SDIO function number is %d while it should always be 1 (unsupported chip?)\n",
+			func->num);
 		return -ENODEV;
 	}
 
@@ -207,18 +223,12 @@ static int wfx_sdio_probe(struct sdio_func *func,
 		goto err1;
 	}
 
-	ret = wfx_sdio_irq_subscribe(bus);
-	if (ret)
-		goto err1;
-
 	ret = wfx_probe(bus->core);
 	if (ret)
-		goto err2;
+		goto err1;
 
 	return 0;
 
-err2:
-	wfx_sdio_irq_unsubscribe(bus);
 err1:
 	sdio_claim_host(func);
 	sdio_disable_func(func);
@@ -232,7 +242,6 @@ static void wfx_sdio_remove(struct sdio_func *func)
 	struct wfx_sdio_priv *bus = sdio_get_drvdata(func);
 
 	wfx_release(bus->core);
-	wfx_sdio_irq_unsubscribe(bus);
 	sdio_claim_host(func);
 	sdio_disable_func(func);
 	sdio_release_host(func);
@@ -248,15 +257,6 @@ static const struct sdio_device_id wfx_sdio_ids[] = {
 };
 MODULE_DEVICE_TABLE(sdio, wfx_sdio_ids);
 
-#ifdef CONFIG_OF
-static const struct of_device_id wfx_sdio_of_match[] = {
-	{ .compatible = "silabs,wfx-sdio" },
-	{ .compatible = "silabs,wf200" },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, wfx_sdio_of_match);
-#endif
-
 struct sdio_driver wfx_sdio_driver = {
 	.name = "wfx-sdio",
 	.id_table = wfx_sdio_ids,
@@ -264,6 +264,6 @@ struct sdio_driver wfx_sdio_driver = {
 	.remove = wfx_sdio_remove,
 	.drv = {
 		.owner = THIS_MODULE,
-		.of_match_table = of_match_ptr(wfx_sdio_of_match),
+		.of_match_table = wfx_sdio_of_match,
 	}
 };
diff --git a/drivers/staging/wfx/bus_spi.c b/drivers/staging/wfx/bus_spi.c
index 61e99b09decb..e8da61fb096b 100644
--- a/drivers/staging/wfx/bus_spi.c
+++ b/drivers/staging/wfx/bus_spi.c
@@ -12,6 +12,7 @@
 #include <linux/gpio/consumer.h>
 #include <linux/spi/spi.h>
 #include <linux/interrupt.h>
+#include <linux/irq.h>
 #include <linux/of.h>
 
 #include "bus.h"
@@ -39,7 +40,6 @@ struct wfx_spi_priv {
 	struct spi_device *func;
 	struct wfx_dev *core;
 	struct gpio_desc *gpio_reset;
-	struct work_struct request_rx;
 	bool need_swab;
 };
 
@@ -140,25 +140,30 @@ static irqreturn_t wfx_spi_irq_handler(int irq, void *priv)
 {
 	struct wfx_spi_priv *bus = priv;
 
-	if (!bus->core) {
-		WARN(!bus->core, "race condition in driver init/deinit");
-		return IRQ_NONE;
-	}
-	queue_work(system_highpri_wq, &bus->request_rx);
+	wfx_bh_request_rx(bus->core);
 	return IRQ_HANDLED;
 }
 
-static void wfx_spi_request_rx(struct work_struct *work)
+static int wfx_spi_irq_subscribe(void *priv)
 {
-	struct wfx_spi_priv *bus =
-		container_of(work, struct wfx_spi_priv, request_rx);
-
-	wfx_bh_request_rx(bus->core);
+	struct wfx_spi_priv *bus = priv;
+	u32 flags;
+
+	flags = irq_get_trigger_type(bus->func->irq);
+	if (!flags)
+		flags = IRQF_TRIGGER_HIGH;
+	flags |= IRQF_ONESHOT;
+	return devm_request_threaded_irq(&bus->func->dev, bus->func->irq, NULL,
+					 wfx_spi_irq_handler, IRQF_ONESHOT,
+					 "wfx", bus);
 }
 
-static void wfx_flush_irq_work(void *w)
+static int wfx_spi_irq_unsubscribe(void *priv)
 {
-	flush_work(w);
+	struct wfx_spi_priv *bus = priv;
+
+	devm_free_irq(&bus->func->dev, bus->func->irq, bus);
+	return 0;
 }
 
 static size_t wfx_spi_align_size(void *priv, size_t size)
@@ -170,6 +175,8 @@ static size_t wfx_spi_align_size(void *priv, size_t size)
 static const struct hwbus_ops wfx_spi_hwbus_ops = {
 	.copy_from_io = wfx_spi_copy_from_io,
 	.copy_to_io = wfx_spi_copy_to_io,
+	.irq_subscribe = wfx_spi_irq_subscribe,
+	.irq_unsubscribe = wfx_spi_irq_unsubscribe,
 	.lock			= wfx_spi_lock,
 	.unlock			= wfx_spi_unlock,
 	.align_size		= wfx_spi_align_size,
@@ -216,22 +223,11 @@ static int wfx_spi_probe(struct spi_device *func)
 		usleep_range(2000, 2500);
 	}
 
-	INIT_WORK(&bus->request_rx, wfx_spi_request_rx);
 	bus->core = wfx_init_common(&func->dev, &wfx_spi_pdata,
 				    &wfx_spi_hwbus_ops, bus);
 	if (!bus->core)
 		return -EIO;
 
-	ret = devm_add_action_or_reset(&func->dev, wfx_flush_irq_work,
-				       &bus->request_rx);
-	if (ret)
-		return ret;
-
-	ret = devm_request_irq(&func->dev, func->irq, wfx_spi_irq_handler,
-			       IRQF_TRIGGER_RISING, "wfx", bus);
-	if (ret)
-		return ret;
-
 	return wfx_probe(bus->core);
 }
 
diff --git a/drivers/staging/wfx/data_rx.c b/drivers/staging/wfx/data_rx.c
index c5b83fedeb55..0e959ebc38b5 100644
--- a/drivers/staging/wfx/data_rx.c
+++ b/drivers/staging/wfx/data_rx.c
@@ -49,7 +49,7 @@ static int wfx_drop_encrypt_data(struct wfx_dev *wdev,
 	}
 
 	/* Firmware strips ICV in case of MIC failure. */
-	if (arg->status == HIF_STATUS_MICFAILURE)
+	if (arg->status == HIF_STATUS_RX_FAIL_MIC)
 		icv_len = 0;
 
 	if (skb->len < hdrlen + iv_len + icv_len) {
@@ -79,7 +79,7 @@ void wfx_rx_cb(struct wfx_vif *wvif,
 	     ieee80211_is_beacon(frame->frame_control)))
 		goto drop;
 
-	if (arg->status == HIF_STATUS_MICFAILURE)
+	if (arg->status == HIF_STATUS_RX_FAIL_MIC)
 		hdr->flag |= RX_FLAG_MMIC_ERROR;
 	else if (arg->status)
 		goto drop;
@@ -118,18 +118,6 @@ void wfx_rx_cb(struct wfx_vif *wvif,
 	    arg->rx_flags.match_uc_addr &&
 	    mgmt->u.action.category == WLAN_CATEGORY_BACK)
 		goto drop;
-	if (ieee80211_is_beacon(frame->frame_control) &&
-	    !arg->status && wvif->vif &&
-	    ether_addr_equal(ieee80211_get_SA(frame),
-			     wvif->vif->bss_conf.bssid)) {
-		/* Disable beacon filter once we're associated... */
-		if (wvif->disable_beacon_filter &&
-		    (wvif->vif->bss_conf.assoc ||
-		     wvif->vif->bss_conf.ibss_joined)) {
-			wvif->disable_beacon_filter = false;
-			schedule_work(&wvif->update_filtering_work);
-		}
-	}
 	ieee80211_rx_irqsafe(wvif->wdev->hw, skb);
 
 	return;
diff --git a/drivers/staging/wfx/data_rx.h b/drivers/staging/wfx/data_rx.h
index 61c28bfd2a37..125dbfc1f875 100644
--- a/drivers/staging/wfx/data_rx.h
+++ b/drivers/staging/wfx/data_rx.h
@@ -8,10 +8,9 @@
 #ifndef WFX_DATA_RX_H
 #define WFX_DATA_RX_H
 
-#include "hif_api_cmd.h"
-
 struct wfx_vif;
 struct sk_buff;
+struct hif_ind_rx;
 
 void wfx_rx_cb(struct wfx_vif *wvif,
 	       const struct hif_ind_rx *arg, struct sk_buff *skb);
diff --git a/drivers/staging/wfx/data_tx.c b/drivers/staging/wfx/data_tx.c
index 42183c70d4df..f042ef36b408 100644
--- a/drivers/staging/wfx/data_tx.c
+++ b/drivers/staging/wfx/data_tx.c
@@ -6,6 +6,7 @@
  * Copyright (c) 2010, ST-Ericsson
  */
 #include <net/mac80211.h>
+#include <linux/etherdevice.h>
 
 #include "data_tx.h"
 #include "wfx.h"
@@ -16,12 +17,11 @@
 #include "traces.h"
 #include "hif_tx_mib.h"
 
-#define WFX_INVALID_RATE_ID    15
-#define WFX_LINK_ID_GC_TIMEOUT ((unsigned long)(10 * HZ))
-
 static int wfx_get_hw_rate(struct wfx_dev *wdev,
 			   const struct ieee80211_tx_rate *rate)
 {
+	struct ieee80211_supported_band *band;
+
 	if (rate->idx < 0)
 		return -1;
 	if (rate->flags & IEEE80211_TX_RC_MCS) {
@@ -33,7 +33,8 @@ static int wfx_get_hw_rate(struct wfx_dev *wdev,
 	}
 	// WFx only support 2GHz, else band information should be retrieved
 	// from ieee80211_tx_info
-	return wdev->hw->wiphy->bands[NL80211_BAND_2GHZ]->bitrates[rate->idx].hw_value;
+	band = wdev->hw->wiphy->bands[NL80211_BAND_2GHZ];
+	return band->bitrates[rate->idx].hw_value;
 }
 
 /* TX policy cache implementation */
@@ -41,73 +42,13 @@ static int wfx_get_hw_rate(struct wfx_dev *wdev,
 static void wfx_tx_policy_build(struct wfx_vif *wvif, struct tx_policy *policy,
 				struct ieee80211_tx_rate *rates)
 {
-	int i;
-	size_t count;
 	struct wfx_dev *wdev = wvif->wdev;
+	int i, rateid;
+	u8 count;
 
 	WARN(rates[0].idx < 0, "invalid rate policy");
 	memset(policy, 0, sizeof(*policy));
-	for (i = 1; i < IEEE80211_TX_MAX_RATES; i++)
-		if (rates[i].idx < 0)
-			break;
-	count = i;
-
-	/* HACK!!! Device has problems (at least) switching from
-	 * 54Mbps CTS to 1Mbps. This switch takes enormous amount
-	 * of time (100-200 ms), leading to valuable throughput drop.
-	 * As a workaround, additional g-rates are injected to the
-	 * policy.
-	 */
-	if (count == 2 && !(rates[0].flags & IEEE80211_TX_RC_MCS) &&
-	    rates[0].idx > 4 && rates[0].count > 2 &&
-	    rates[1].idx < 2) {
-		int mid_rate = (rates[0].idx + 4) >> 1;
-
-		/* Decrease number of retries for the initial rate */
-		rates[0].count -= 2;
-
-		if (mid_rate != 4) {
-			/* Keep fallback rate at 1Mbps. */
-			rates[3] = rates[1];
-
-			/* Inject 1 transmission on lowest g-rate */
-			rates[2].idx = 4;
-			rates[2].count = 1;
-			rates[2].flags = rates[1].flags;
-
-			/* Inject 1 transmission on mid-rate */
-			rates[1].idx = mid_rate;
-			rates[1].count = 1;
-
-			/* Fallback to 1 Mbps is a really bad thing,
-			 * so let's try to increase probability of
-			 * successful transmission on the lowest g rate
-			 * even more
-			 */
-			if (rates[0].count >= 3) {
-				--rates[0].count;
-				++rates[2].count;
-			}
-
-			/* Adjust amount of rates defined */
-			count += 2;
-		} else {
-			/* Keep fallback rate at 1Mbps. */
-			rates[2] = rates[1];
-
-			/* Inject 2 transmissions on lowest g-rate */
-			rates[1].idx = 4;
-			rates[1].count = 2;
-
-			/* Adjust amount of rates defined */
-			count += 1;
-		}
-	}
-
 	for (i = 0; i < IEEE80211_TX_MAX_RATES; ++i) {
-		int rateid;
-		u8 count;
-
 		if (rates[i].idx < 0)
 			break;
 		WARN_ON(rates[i].count > 15);
@@ -158,8 +99,7 @@ static int wfx_tx_policy_release(struct tx_policy_cache *cache,
 }
 
 static int wfx_tx_policy_get(struct wfx_vif *wvif,
-			     struct ieee80211_tx_rate *rates,
-			     bool *renew)
+			     struct ieee80211_tx_rate *rates, bool *renew)
 {
 	int idx;
 	struct tx_policy_cache *cache = &wvif->tx_policy_cache;
@@ -171,7 +111,7 @@ static int wfx_tx_policy_get(struct wfx_vif *wvif,
 	if (list_empty(&cache->free)) {
 		WARN(1, "unable to get a valid Tx policy");
 		spin_unlock_bh(&cache->lock);
-		return WFX_INVALID_RATE_ID;
+		return HIF_TX_RETRY_POLICY_INVALID;
 	}
 	idx = wfx_tx_policy_find(cache, &wanted);
 	if (idx >= 0) {
@@ -189,10 +129,8 @@ static int wfx_tx_policy_get(struct wfx_vif *wvif,
 		idx = entry - cache->cache;
 	}
 	wfx_tx_policy_use(cache, &cache->cache[idx]);
-	if (list_empty(&cache->free)) {
-		/* Lock TX queues. */
-		wfx_tx_queues_lock(wvif->wdev);
-	}
+	if (list_empty(&cache->free))
+		ieee80211_stop_queues(wvif->wdev->hw);
 	spin_unlock_bh(&cache->lock);
 	return idx;
 }
@@ -202,15 +140,13 @@ static void wfx_tx_policy_put(struct wfx_vif *wvif, int idx)
 	int usage, locked;
 	struct tx_policy_cache *cache = &wvif->tx_policy_cache;
 
-	if (idx == WFX_INVALID_RATE_ID)
+	if (idx == HIF_TX_RETRY_POLICY_INVALID)
 		return;
 	spin_lock_bh(&cache->lock);
 	locked = list_empty(&cache->free);
 	usage = wfx_tx_policy_release(cache, &cache->cache[idx]);
-	if (locked && !usage) {
-		/* Unlock TX queues. */
-		wfx_tx_queues_unlock(wvif->wdev);
-	}
+	if (locked && !usage)
+		ieee80211_wake_queues(wvif->wdev->hw);
 	spin_unlock_bh(&cache->lock);
 }
 
@@ -218,15 +154,17 @@ static int wfx_tx_policy_upload(struct wfx_vif *wvif)
 {
 	struct tx_policy *policies = wvif->tx_policy_cache.cache;
 	u8 tmp_rates[12];
-	int i;
+	int i, is_used;
 
 	do {
 		spin_lock_bh(&wvif->tx_policy_cache.lock);
-		for (i = 0; i < HIF_MIB_NUM_TX_RATE_RETRY_POLICIES; ++i)
-			if (!policies[i].uploaded &&
-			    memzcmp(policies[i].rates, sizeof(policies[i].rates)))
+		for (i = 0; i < ARRAY_SIZE(wvif->tx_policy_cache.cache); ++i) {
+			is_used = memzcmp(policies[i].rates,
+					  sizeof(policies[i].rates));
+			if (!policies[i].uploaded && is_used)
 				break;
-		if (i < HIF_MIB_NUM_TX_RATE_RETRY_POLICIES) {
+		}
+		if (i < ARRAY_SIZE(wvif->tx_policy_cache.cache)) {
 			policies[i].uploaded = true;
 			memcpy(tmp_rates, policies[i].rates, sizeof(tmp_rates));
 			spin_unlock_bh(&wvif->tx_policy_cache.lock);
@@ -234,7 +172,7 @@ static int wfx_tx_policy_upload(struct wfx_vif *wvif)
 		} else {
 			spin_unlock_bh(&wvif->tx_policy_cache.lock);
 		}
-	} while (i < HIF_MIB_NUM_TX_RATE_RETRY_POLICIES);
+	} while (i < ARRAY_SIZE(wvif->tx_policy_cache.cache));
 	return 0;
 }
 
@@ -244,9 +182,7 @@ void wfx_tx_policy_upload_work(struct work_struct *work)
 		container_of(work, struct wfx_vif, tx_policy_upload_work);
 
 	wfx_tx_policy_upload(wvif);
-
 	wfx_tx_unlock(wvif->wdev);
-	wfx_tx_queues_unlock(wvif->wdev);
 }
 
 void wfx_tx_policy_init(struct wfx_vif *wvif)
@@ -260,7 +196,7 @@ void wfx_tx_policy_init(struct wfx_vif *wvif)
 	INIT_LIST_HEAD(&cache->used);
 	INIT_LIST_HEAD(&cache->free);
 
-	for (i = 0; i < HIF_MIB_NUM_TX_RATE_RETRY_POLICIES; ++i)
+	for (i = 0; i < ARRAY_SIZE(cache->cache); ++i)
 		list_add(&cache->cache[i].link, &cache->free);
 }
 
@@ -281,16 +217,11 @@ static void wfx_tx_manage_pm(struct wfx_vif *wvif, struct ieee80211_hdr *hdr,
 			     struct wfx_tx_priv *tx_priv,
 			     struct ieee80211_sta *sta)
 {
-	u32 mask = ~BIT(tx_priv->raw_link_id);
 	struct wfx_sta_priv *sta_priv;
 	int tid = ieee80211_get_tid(hdr);
 
-	spin_lock_bh(&wvif->ps_state_lock);
-	if (ieee80211_is_auth(hdr->frame_control))
-		wvif->sta_asleep_mask &= mask;
-	spin_unlock_bh(&wvif->ps_state_lock);
-
 	if (sta) {
+		tx_priv->has_sta = true;
 		sta_priv = (struct wfx_sta_priv *)&sta->drv_priv;
 		spin_lock_bh(&sta_priv->lock);
 		sta_priv->buffered[tid]++;
@@ -299,9 +230,8 @@ static void wfx_tx_manage_pm(struct wfx_vif *wvif, struct ieee80211_hdr *hdr,
 	}
 }
 
-static u8 wfx_tx_get_raw_link_id(struct wfx_vif *wvif,
-				 struct ieee80211_sta *sta,
-				 struct ieee80211_hdr *hdr)
+static u8 wfx_tx_get_link_id(struct wfx_vif *wvif, struct ieee80211_sta *sta,
+			     struct ieee80211_hdr *hdr)
 {
 	struct wfx_sta_priv *sta_priv =
 		sta ? (struct wfx_sta_priv *)&sta->drv_priv : NULL;
@@ -313,7 +243,7 @@ static u8 wfx_tx_get_raw_link_id(struct wfx_vif *wvif,
 		return 0;
 	if (is_multicast_ether_addr(da))
 		return 0;
-	return WFX_LINK_ID_NO_ASSOC;
+	return HIF_LINK_ID_NOT_ASSOCIATED;
 }
 
 static void wfx_tx_fixup_rates(struct ieee80211_tx_rate *rates)
@@ -358,7 +288,8 @@ static void wfx_tx_fixup_rates(struct ieee80211_tx_rate *rates)
 		if (rates[i].idx == -1) {
 			rates[i].idx = 0;
 			rates[i].count = 8; // == hw->max_rate_tries
-			rates[i].flags = rates[i - 1].flags & IEEE80211_TX_RC_MCS;
+			rates[i].flags = rates[i - 1].flags &
+					 IEEE80211_TX_RC_MCS;
 			break;
 		}
 	}
@@ -375,24 +306,19 @@ static u8 wfx_tx_get_rate_id(struct wfx_vif *wvif,
 
 	rate_id = wfx_tx_policy_get(wvif,
 				    tx_info->driver_rates, &tx_policy_renew);
-	if (rate_id == WFX_INVALID_RATE_ID)
+	if (rate_id == HIF_TX_RETRY_POLICY_INVALID)
 		dev_warn(wvif->wdev->dev, "unable to get a valid Tx policy");
 
 	if (tx_policy_renew) {
-		/* FIXME: It's not so optimal to stop TX queues every now and
-		 * then.  Better to reimplement task scheduling with a counter.
-		 */
 		wfx_tx_lock(wvif->wdev);
-		wfx_tx_queues_lock(wvif->wdev);
-		if (!schedule_work(&wvif->tx_policy_upload_work)) {
-			wfx_tx_queues_unlock(wvif->wdev);
+		if (!schedule_work(&wvif->tx_policy_upload_work))
 			wfx_tx_unlock(wvif->wdev);
-		}
 	}
 	return rate_id;
 }
 
-static struct hif_ht_tx_parameters wfx_tx_get_tx_parms(struct wfx_dev *wdev, struct ieee80211_tx_info *tx_info)
+static struct hif_ht_tx_parameters wfx_tx_get_tx_parms(struct wfx_dev *wdev,
+						       struct ieee80211_tx_info *tx_info)
 {
 	struct ieee80211_tx_rate *rate = &tx_info->driver_rates[0];
 	struct hif_ht_tx_parameters ret = { };
@@ -429,7 +355,7 @@ static int wfx_tx_inner(struct wfx_vif *wvif, struct ieee80211_sta *sta,
 	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_key_conf *hw_key = tx_info->control.hw_key;
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
-	int queue_id = tx_info->hw_queue;
+	int queue_id = skb_get_queue_mapping(skb);
 	size_t offset = (size_t)skb->data & 3;
 	int wmsg_len = sizeof(struct hif_msg) +
 			sizeof(struct hif_req_tx) + offset;
@@ -441,14 +367,8 @@ static int wfx_tx_inner(struct wfx_vif *wvif, struct ieee80211_sta *sta,
 	memset(tx_info->rate_driver_data, 0, sizeof(struct wfx_tx_priv));
 	// Fill tx_priv
 	tx_priv = (struct wfx_tx_priv *)tx_info->rate_driver_data;
-	tx_priv->raw_link_id = wfx_tx_get_raw_link_id(wvif, sta, hdr);
-	tx_priv->link_id = tx_priv->raw_link_id;
 	if (ieee80211_has_protected(hdr->frame_control))
 		tx_priv->hw_key = hw_key;
-	if (tx_info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM)
-		tx_priv->link_id = WFX_LINK_ID_AFTER_DTIM;
-	if (sta && (sta->uapsd_queues & BIT(queue_id)))
-		tx_priv->link_id = WFX_LINK_ID_UAPSD;
 
 	// Fill hif_msg
 	WARN(skb_headroom(skb) < wmsg_len, "not enough space in skb");
@@ -461,7 +381,8 @@ static int wfx_tx_inner(struct wfx_vif *wvif, struct ieee80211_sta *sta,
 	hif_msg->id = HIF_REQ_ID_TX;
 	hif_msg->interface = wvif->id;
 	if (skb->len > wvif->wdev->hw_caps.size_inp_ch_buf) {
-		dev_warn(wvif->wdev->dev, "requested frame size (%d) is larger than maximum supported (%d)\n",
+		dev_warn(wvif->wdev->dev,
+			 "requested frame size (%d) is larger than maximum supported (%d)\n",
 			 skb->len, wvif->wdev->hw_caps.size_inp_ch_buf);
 		skb_pull(skb, wmsg_len);
 		return -EIO;
@@ -472,13 +393,14 @@ static int wfx_tx_inner(struct wfx_vif *wvif, struct ieee80211_sta *sta,
 	// packet_id just need to be unique on device. 32bits are more than
 	// necessary for that task, so we tae advantage of it to add some extra
 	// data for debug.
-	req->packet_id = queue_id << 28 |
-			 IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl)) << 16 |
-			 (atomic_add_return(1, &wvif->wdev->packet_id) & 0xFFFF);
+	req->packet_id = atomic_add_return(1, &wvif->wdev->packet_id) & 0xFFFF;
+	req->packet_id |= IEEE80211_SEQ_TO_SN(le16_to_cpu(hdr->seq_ctrl)) << 16;
+	req->packet_id |= queue_id << 28;
+
 	req->data_flags.fc_offset = offset;
 	if (tx_info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM)
 		req->data_flags.after_dtim = 1;
-	req->queue_id.peer_sta_id = tx_priv->raw_link_id;
+	req->queue_id.peer_sta_id = wfx_tx_get_link_id(wvif, sta, hdr);
 	// Queue index are inverted between firmware and Linux
 	req->queue_id.queue_id = 3 - queue_id;
 	req->ht_tx_parameters = wfx_tx_get_tx_parms(wvif->wdev, tx_info);
@@ -486,7 +408,7 @@ static int wfx_tx_inner(struct wfx_vif *wvif, struct ieee80211_sta *sta,
 
 	// Auxiliary operations
 	wfx_tx_manage_pm(wvif, hdr, tx_priv, sta);
-	wfx_tx_queue_put(wvif->wdev, &wvif->wdev->tx_queue[queue_id], skb);
+	wfx_tx_queues_put(wvif->wdev, skb);
 	if (tx_info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM)
 		schedule_work(&wvif->update_tim_work);
 	wfx_bh_request_tx(wvif->wdev);
@@ -528,28 +450,59 @@ drop:
 	ieee80211_tx_status_irqsafe(wdev->hw, skb);
 }
 
-void wfx_tx_confirm_cb(struct wfx_vif *wvif, const struct hif_cnf_tx *arg)
+static struct ieee80211_hdr *wfx_skb_hdr80211(struct sk_buff *skb)
 {
-	int i;
-	int tx_count;
-	struct sk_buff *skb;
-	struct ieee80211_tx_rate *rate;
-	struct ieee80211_tx_info *tx_info;
-	const struct wfx_tx_priv *tx_priv;
+	struct hif_msg *hif = (struct hif_msg *)skb->data;
+	struct hif_req_tx *req = (struct hif_req_tx *)hif->body;
 
-	skb = wfx_pending_get(wvif->wdev, arg->packet_id);
-	if (!skb) {
-		dev_warn(wvif->wdev->dev,
-			 "received unknown packet_id (%#.8x) from chip\n",
-			 arg->packet_id);
-		return;
+	return (struct ieee80211_hdr *)(req->frame + req->data_flags.fc_offset);
+}
+
+static void wfx_tx_update_sta(struct wfx_vif *wvif, struct ieee80211_hdr *hdr)
+{
+	int tid = ieee80211_get_tid(hdr);
+	struct wfx_sta_priv *sta_priv;
+	struct ieee80211_sta *sta;
+
+	rcu_read_lock(); // protect sta
+	sta = ieee80211_find_sta(wvif->vif, hdr->addr1);
+	if (sta) {
+		sta_priv = (struct wfx_sta_priv *)&sta->drv_priv;
+		spin_lock_bh(&sta_priv->lock);
+		WARN(!sta_priv->buffered[tid], "inconsistent notification");
+		sta_priv->buffered[tid]--;
+		if (!sta_priv->buffered[tid])
+			ieee80211_sta_set_buffered(sta, tid, false);
+		spin_unlock_bh(&sta_priv->lock);
+	} else {
+		dev_dbg(wvif->wdev->dev, "%s: sta does not exist anymore\n",
+			__func__);
 	}
-	tx_info = IEEE80211_SKB_CB(skb);
-	tx_priv = wfx_skb_tx_priv(skb);
-	_trace_tx_stats(arg, skb,
-			wfx_pending_get_pkt_us_delay(wvif->wdev, skb));
+	rcu_read_unlock();
+}
+
+static void wfx_skb_dtor(struct wfx_vif *wvif, struct sk_buff *skb)
+{
+	struct hif_msg *hif = (struct hif_msg *)skb->data;
+	struct hif_req_tx *req = (struct hif_req_tx *)hif->body;
+	unsigned int offset = sizeof(struct hif_msg) +
+			      sizeof(struct hif_req_tx) +
+			      req->data_flags.fc_offset;
+
+	WARN_ON(!wvif);
+	wfx_tx_policy_put(wvif, req->tx_flags.retry_policy_index);
+	skb_pull(skb, offset);
+	ieee80211_tx_status_irqsafe(wvif->wdev->hw, skb);
+}
+
+static void wfx_tx_fill_rates(struct wfx_dev *wdev,
+			      struct ieee80211_tx_info *tx_info,
+			      const struct hif_cnf_tx *arg)
+{
+	struct ieee80211_tx_rate *rate;
+	int tx_count;
+	int i;
 
-	// You can touch to tx_priv, but don't touch to tx_info->status.
 	tx_count = arg->ack_failures;
 	if (!arg->status || arg->ack_failures)
 		tx_count += 1; // Also report success
@@ -558,16 +511,14 @@ void wfx_tx_confirm_cb(struct wfx_vif *wvif, const struct hif_cnf_tx *arg)
 		if (rate->idx < 0)
 			break;
 		if (tx_count < rate->count &&
-		    arg->status == HIF_STATUS_RETRY_EXCEEDED &&
+		    arg->status == HIF_STATUS_TX_FAIL_RETRIES &&
 		    arg->ack_failures)
-			dev_dbg(wvif->wdev->dev, "all retries were not consumed: %d != %d\n",
+			dev_dbg(wdev->dev, "all retries were not consumed: %d != %d\n",
 				rate->count, tx_count);
 		if (tx_count <= rate->count && tx_count &&
-		    arg->txed_rate != wfx_get_hw_rate(wvif->wdev, rate))
-			dev_dbg(wvif->wdev->dev,
-				"inconsistent tx_info rates: %d != %d\n",
-				arg->txed_rate,
-				wfx_get_hw_rate(wvif->wdev, rate));
+		    arg->txed_rate != wfx_get_hw_rate(wdev, rate))
+			dev_dbg(wdev->dev, "inconsistent tx_info rates: %d != %d\n",
+				arg->txed_rate, wfx_get_hw_rate(wdev, rate));
 		if (tx_count > rate->count) {
 			tx_count -= rate->count;
 		} else if (!tx_count) {
@@ -579,8 +530,30 @@ void wfx_tx_confirm_cb(struct wfx_vif *wvif, const struct hif_cnf_tx *arg)
 		}
 	}
 	if (tx_count)
-		dev_dbg(wvif->wdev->dev,
-			"%d more retries than expected\n", tx_count);
+		dev_dbg(wdev->dev, "%d more retries than expected\n", tx_count);
+}
+
+void wfx_tx_confirm_cb(struct wfx_vif *wvif, const struct hif_cnf_tx *arg)
+{
+	struct ieee80211_tx_info *tx_info;
+	const struct wfx_tx_priv *tx_priv;
+	struct sk_buff *skb;
+
+	skb = wfx_pending_get(wvif->wdev, arg->packet_id);
+	if (!skb) {
+		dev_warn(wvif->wdev->dev, "received unknown packet_id (%#.8x) from chip\n",
+			 arg->packet_id);
+		return;
+	}
+	tx_info = IEEE80211_SKB_CB(skb);
+	tx_priv = wfx_skb_tx_priv(skb);
+	_trace_tx_stats(arg, skb,
+			wfx_pending_get_pkt_us_delay(wvif->wdev, skb));
+
+	// You can touch to tx_priv, but don't touch to tx_info->status.
+	wfx_tx_fill_rates(wvif->wdev, tx_info, arg);
+	if (tx_priv->has_sta)
+		wfx_tx_update_sta(wvif, wfx_skb_hdr80211(skb));
 	skb_trim(skb, skb->len - wfx_tx_get_icv_len(tx_priv->hw_key));
 
 	// From now, you can touch to tx_info->status, but do not touch to
@@ -590,63 +563,64 @@ void wfx_tx_confirm_cb(struct wfx_vif *wvif, const struct hif_cnf_tx *arg)
 	memset(tx_info->pad, 0, sizeof(tx_info->pad));
 
 	if (!arg->status) {
-		if (wvif->bss_loss_state &&
-		    arg->packet_id == wvif->bss_loss_confirm_id)
-			wfx_cqm_bssloss_sm(wvif, 0, 1, 0);
 		tx_info->status.tx_time =
-		arg->media_delay - arg->tx_queue_delay;
+			le32_to_cpu(arg->media_delay) -
+			le32_to_cpu(arg->tx_queue_delay);
 		if (tx_info->flags & IEEE80211_TX_CTL_NO_ACK)
 			tx_info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
 		else
 			tx_info->flags |= IEEE80211_TX_STAT_ACK;
-	} else if (arg->status == HIF_REQUEUE) {
-		WARN(!arg->tx_result_flags.requeue, "incoherent status and result_flags");
+	} else if (arg->status == HIF_STATUS_TX_FAIL_REQUEUE) {
+		WARN(!arg->tx_result_flags.requeue,
+		     "incoherent status and result_flags");
 		if (tx_info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) {
 			wvif->after_dtim_tx_allowed = false; // DTIM period elapsed
 			schedule_work(&wvif->update_tim_work);
 		}
 		tx_info->flags |= IEEE80211_TX_STAT_TX_FILTERED;
-	} else {
-		if (wvif->bss_loss_state &&
-		    arg->packet_id == wvif->bss_loss_confirm_id)
-			wfx_cqm_bssloss_sm(wvif, 0, 0, 1);
 	}
-	wfx_pending_remove(wvif->wdev, skb);
+	wfx_skb_dtor(wvif, skb);
 }
 
-static void wfx_notify_buffered_tx(struct wfx_vif *wvif, struct sk_buff *skb)
+void wfx_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+	       u32 queues, bool drop)
 {
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
-	struct ieee80211_sta *sta;
-	struct wfx_sta_priv *sta_priv;
-	int tid = ieee80211_get_tid(hdr);
+	struct wfx_dev *wdev = hw->priv;
+	struct sk_buff_head dropped;
+	struct wfx_queue *queue;
+	struct wfx_vif *wvif;
+	struct hif_msg *hif;
+	struct sk_buff *skb;
+	int vif_id = -1;
+	int i;
 
-	rcu_read_lock(); // protect sta
-	sta = ieee80211_find_sta(wvif->vif, hdr->addr1);
-	if (sta) {
-		sta_priv = (struct wfx_sta_priv *)&sta->drv_priv;
-		spin_lock_bh(&sta_priv->lock);
-		WARN(!sta_priv->buffered[tid], "inconsistent notification");
-		sta_priv->buffered[tid]--;
-		if (!sta_priv->buffered[tid])
-			ieee80211_sta_set_buffered(sta, tid, false);
-		spin_unlock_bh(&sta_priv->lock);
+	if (vif)
+		vif_id = ((struct wfx_vif *)vif->drv_priv)->id;
+	skb_queue_head_init(&dropped);
+	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
+		if (!(BIT(i) & queues))
+			continue;
+		queue = &wdev->tx_queue[i];
+		if (drop)
+			wfx_tx_queue_drop(wdev, queue, vif_id, &dropped);
+		if (wdev->chip_frozen)
+			continue;
+		if (wait_event_timeout(wdev->tx_dequeue,
+				       wfx_tx_queue_empty(wdev, queue, vif_id),
+				       msecs_to_jiffies(1000)) <= 0)
+			dev_warn(wdev->dev,
+				 "frames queued while flushing tx queues?");
+	}
+	wfx_tx_flush(wdev);
+	if (wdev->chip_frozen)
+		wfx_pending_drop(wdev, &dropped);
+	while ((skb = skb_dequeue(&dropped)) != NULL) {
+		hif = (struct hif_msg *)skb->data;
+		wvif = wdev_to_wvif(wdev, hif->interface);
+		if (wfx_skb_tx_priv(skb)->has_sta)
+			wfx_tx_update_sta(wvif, wfx_skb_hdr80211(skb));
+		ieee80211_tx_info_clear_status(IEEE80211_SKB_CB(skb));
+		wfx_skb_dtor(wvif, skb);
 	}
-	rcu_read_unlock();
 }
 
-void wfx_skb_dtor(struct wfx_dev *wdev, struct sk_buff *skb)
-{
-	struct hif_msg *hif = (struct hif_msg *)skb->data;
-	struct hif_req_tx *req = (struct hif_req_tx *)hif->body;
-	struct wfx_vif *wvif = wdev_to_wvif(wdev, hif->interface);
-	unsigned int offset = sizeof(struct hif_req_tx) +
-				sizeof(struct hif_msg) +
-				req->data_flags.fc_offset;
-
-	WARN_ON(!wvif);
-	skb_pull(skb, offset);
-	wfx_notify_buffered_tx(wvif, skb);
-	wfx_tx_policy_put(wvif, req->tx_flags.retry_policy_index);
-	ieee80211_tx_status_irqsafe(wdev->hw, skb);
-}
diff --git a/drivers/staging/wfx/data_tx.h b/drivers/staging/wfx/data_tx.h
index c545dd75449b..54fff24508fb 100644
--- a/drivers/staging/wfx/data_tx.h
+++ b/drivers/staging/wfx/data_tx.h
@@ -26,7 +26,7 @@ struct tx_policy {
 };
 
 struct tx_policy_cache {
-	struct tx_policy cache[HIF_MIB_NUM_TX_RATE_RETRY_POLICIES];
+	struct tx_policy cache[HIF_TX_RETRY_POLICY_MAX];
 	// FIXME: use a trees and drop hash from tx_policy
 	struct list_head used;
 	struct list_head free;
@@ -36,8 +36,7 @@ struct tx_policy_cache {
 struct wfx_tx_priv {
 	ktime_t xmit_timestamp;
 	struct ieee80211_key_conf *hw_key;
-	u8 link_id;
-	u8 raw_link_id;
+	bool has_sta;
 } __packed;
 
 void wfx_tx_policy_init(struct wfx_vif *wvif);
@@ -46,7 +45,8 @@ void wfx_tx_policy_upload_work(struct work_struct *work);
 void wfx_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control,
 	    struct sk_buff *skb);
 void wfx_tx_confirm_cb(struct wfx_vif *wvif, const struct hif_cnf_tx *arg);
-void wfx_skb_dtor(struct wfx_dev *wdev, struct sk_buff *skb);
+void wfx_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+	       u32 queues, bool drop);
 
 static inline struct wfx_tx_priv *wfx_skb_tx_priv(struct sk_buff *skb)
 {
diff --git a/drivers/staging/wfx/debug.c b/drivers/staging/wfx/debug.c
index 1164aba118a1..10d649985696 100644
--- a/drivers/staging/wfx/debug.c
+++ b/drivers/staging/wfx/debug.c
@@ -61,19 +61,26 @@ const char *get_reg_name(unsigned long id)
 
 static int wfx_counters_show(struct seq_file *seq, void *v)
 {
-	int ret;
+	int ret, i;
 	struct wfx_dev *wdev = seq->private;
-	struct hif_mib_extended_count_table counters;
+	struct hif_mib_extended_count_table counters[3];
+
+	for (i = 0; i < ARRAY_SIZE(counters); i++) {
+		ret = hif_get_counters_table(wdev, i, counters + i);
+		if (ret < 0)
+			return ret;
+		if (ret > 0)
+			return -EIO;
+	}
 
-	ret = hif_get_counters_table(wdev, &counters);
-	if (ret < 0)
-		return ret;
-	if (ret > 0)
-		return -EIO;
+	seq_printf(seq, "%-24s %12s %12s %12s\n",
+		   "", "global", "iface 0", "iface 1");
 
 #define PUT_COUNTER(name) \
-	seq_printf(seq, "%24s %d\n", #name ":",\
-		   le32_to_cpu(counters.count_##name))
+	seq_printf(seq, "%-24s %12d %12d %12d\n", #name, \
+		   le32_to_cpu(counters[2].count_##name), \
+		   le32_to_cpu(counters[0].count_##name), \
+		   le32_to_cpu(counters[1].count_##name))
 
 	PUT_COUNTER(tx_packets);
 	PUT_COUNTER(tx_multicast_frames);
@@ -105,6 +112,12 @@ static int wfx_counters_show(struct seq_file *seq, void *v)
 
 #undef PUT_COUNTER
 
+	for (i = 0; i < ARRAY_SIZE(counters[0].reserved); i++)
+		seq_printf(seq, "reserved[%02d]%12s %12d %12d %12d\n", i, "",
+			   le32_to_cpu(counters[2].reserved[i]),
+			   le32_to_cpu(counters[0].reserved[i]),
+			   le32_to_cpu(counters[1].reserved[i]));
+
 	return 0;
 }
 DEFINE_SHOW_ATTRIBUTE(wfx_counters);
@@ -142,7 +155,7 @@ static int wfx_rx_stats_show(struct seq_file *seq, void *v)
 	mutex_lock(&wdev->rx_stats_lock);
 	seq_printf(seq, "Timestamp: %dus\n", st->date);
 	seq_printf(seq, "Low power clock: frequency %uHz, external %s\n",
-		   st->pwr_clk_freq,
+		   le32_to_cpu(st->pwr_clk_freq),
 		   st->is_ext_pwr_clk ? "yes" : "no");
 	seq_printf(seq,
 		   "Num. of frames: %d, PER (x10e4): %d, Throughput: %dKbps/s\n",
@@ -152,9 +165,12 @@ static int wfx_rx_stats_show(struct seq_file *seq, void *v)
 	for (i = 0; i < ARRAY_SIZE(channel_names); i++) {
 		if (channel_names[i])
 			seq_printf(seq, "%5s %8d %8d %8d %8d %8d\n",
-				   channel_names[i], st->nb_rx_by_rate[i],
-				   st->per[i], st->rssi[i] / 100,
-				   st->snr[i] / 100, st->cfo[i]);
+				   channel_names[i],
+				   le32_to_cpu(st->nb_rx_by_rate[i]),
+				   le16_to_cpu(st->per[i]),
+				   (s16)le16_to_cpu(st->rssi[i]) / 100,
+				   (s16)le16_to_cpu(st->snr[i]) / 100,
+				   (s16)le16_to_cpu(st->cfo[i]));
 	}
 	mutex_unlock(&wdev->rx_stats_lock);
 
@@ -162,6 +178,30 @@ static int wfx_rx_stats_show(struct seq_file *seq, void *v)
 }
 DEFINE_SHOW_ATTRIBUTE(wfx_rx_stats);
 
+static int wfx_tx_power_loop_show(struct seq_file *seq, void *v)
+{
+	struct wfx_dev *wdev = seq->private;
+	struct hif_tx_power_loop_info *st = &wdev->tx_power_loop_info;
+	int tmp;
+
+	mutex_lock(&wdev->tx_power_loop_info_lock);
+	tmp = le16_to_cpu(st->tx_gain_dig);
+	seq_printf(seq, "Tx gain digital: %d\n", tmp);
+	tmp = le16_to_cpu(st->tx_gain_pa);
+	seq_printf(seq, "Tx gain PA: %d\n", tmp);
+	tmp = (s16)le16_to_cpu(st->target_pout);
+	seq_printf(seq, "Target Pout: %d.%02d dBm\n", tmp / 4, (tmp % 4) * 25);
+	tmp = (s16)le16_to_cpu(st->p_estimation);
+	seq_printf(seq, "FEM Pout: %d.%02d dBm\n", tmp / 4, (tmp % 4) * 25);
+	tmp = le16_to_cpu(st->vpdet);
+	seq_printf(seq, "Vpdet: %d mV\n", tmp);
+	seq_printf(seq, "Measure index: %d\n", st->measurement_index);
+	mutex_unlock(&wdev->tx_power_loop_info_lock);
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(wfx_tx_power_loop);
+
 static ssize_t wfx_send_pds_write(struct file *file,
 				  const char __user *user_buf,
 				  size_t count, loff_t *ppos)
@@ -234,7 +274,7 @@ static ssize_t wfx_send_hif_msg_write(struct file *file,
 	request = memdup_user(user_buf, count);
 	if (IS_ERR(request))
 		return PTR_ERR(request);
-	if (request->len != count) {
+	if (le16_to_cpu(request->len) != count) {
 		kfree(request);
 		return -EINVAL;
 	}
@@ -301,6 +341,8 @@ int wfx_debug_init(struct wfx_dev *wdev)
 	d = debugfs_create_dir("wfx", wdev->hw->wiphy->debugfsdir);
 	debugfs_create_file("counters", 0444, d, wdev, &wfx_counters_fops);
 	debugfs_create_file("rx_stats", 0444, d, wdev, &wfx_rx_stats_fops);
+	debugfs_create_file("tx_power_loop", 0444, d, wdev,
+			    &wfx_tx_power_loop_fops);
 	debugfs_create_file("send_pds", 0200, d, wdev, &wfx_send_pds_fops);
 	debugfs_create_file("burn_slk_key", 0200, d, wdev,
 			    &wfx_burn_slk_key_fops);
diff --git a/drivers/staging/wfx/fwio.c b/drivers/staging/wfx/fwio.c
index 9d61082c1e6c..72bb3d2a9613 100644
--- a/drivers/staging/wfx/fwio.c
+++ b/drivers/staging/wfx/fwio.c
@@ -99,16 +99,16 @@ static int sram_write_dma_safe(struct wfx_dev *wdev, u32 addr, const u8 *buf,
 	return ret;
 }
 
-int get_firmware(struct wfx_dev *wdev, u32 keyset_chip,
-		 const struct firmware **fw, int *file_offset)
+static int get_firmware(struct wfx_dev *wdev, u32 keyset_chip,
+			const struct firmware **fw, int *file_offset)
 {
 	int keyset_file;
 	char filename[256];
 	const char *data;
 	int ret;
 
-	snprintf(filename, sizeof(filename), "%s_%02X.sec", wdev->pdata.file_fw,
-		 keyset_chip);
+	snprintf(filename, sizeof(filename), "%s_%02X.sec",
+		 wdev->pdata.file_fw, keyset_chip);
 	ret = firmware_request_nowarn(fw, filename, wdev->dev);
 	if (ret) {
 		dev_info(wdev->dev, "can't load %s, falling back to %s.sec\n",
@@ -325,8 +325,8 @@ static int init_gpr(struct wfx_dev *wdev)
 				     gpr_init[i].value);
 		if (ret < 0)
 			return ret;
-		dev_dbg(wdev->dev, "  index %02x: %08x\n", gpr_init[i].index,
-			gpr_init[i].value);
+		dev_dbg(wdev->dev, "  index %02x: %08x\n",
+			gpr_init[i].index, gpr_init[i].value);
 	}
 	return 0;
 }
@@ -360,7 +360,7 @@ int wfx_init_device(struct wfx_dev *wdev)
 	dev_dbg(wdev->dev, "initial config register value: %08x\n", reg);
 
 	hw_revision = FIELD_GET(CFG_DEVICE_ID_MAJOR, reg);
-	if (hw_revision == 0 || hw_revision > 2) {
+	if (hw_revision == 0) {
 		dev_err(wdev->dev, "bad hardware revision number: %d\n",
 			hw_revision);
 		return -ENODEV;
diff --git a/drivers/staging/wfx/hif_api_cmd.h b/drivers/staging/wfx/hif_api_cmd.h
index 071b71e2a107..21cde19cff75 100644
--- a/drivers/staging/wfx/hif_api_cmd.h
+++ b/drivers/staging/wfx/hif_api_cmd.h
@@ -10,56 +10,54 @@
 
 #include "hif_api_general.h"
 
-#define HIF_NUM_AC                             4
-
 #define HIF_API_SSID_SIZE                      API_SSID_SIZE
 
 enum hif_requests_ids {
-	HIF_REQ_ID_RESET                             = 0x0a,
-	HIF_REQ_ID_READ_MIB                          = 0x05,
-	HIF_REQ_ID_WRITE_MIB                         = 0x06,
-	HIF_REQ_ID_START_SCAN                        = 0x07,
-	HIF_REQ_ID_STOP_SCAN                         = 0x08,
-	HIF_REQ_ID_TX                                = 0x04,
-	HIF_REQ_ID_JOIN                              = 0x0b,
-	HIF_REQ_ID_SET_PM_MODE                       = 0x10,
-	HIF_REQ_ID_SET_BSS_PARAMS                    = 0x11,
-	HIF_REQ_ID_ADD_KEY                           = 0x0c,
-	HIF_REQ_ID_REMOVE_KEY                        = 0x0d,
-	HIF_REQ_ID_EDCA_QUEUE_PARAMS                 = 0x13,
-	HIF_REQ_ID_START                             = 0x17,
-	HIF_REQ_ID_BEACON_TRANSMIT                   = 0x18,
-	HIF_REQ_ID_UPDATE_IE                         = 0x1b,
-	HIF_REQ_ID_MAP_LINK                          = 0x1c,
+	HIF_REQ_ID_RESET                = 0x0a,
+	HIF_REQ_ID_READ_MIB             = 0x05,
+	HIF_REQ_ID_WRITE_MIB            = 0x06,
+	HIF_REQ_ID_START_SCAN           = 0x07,
+	HIF_REQ_ID_STOP_SCAN            = 0x08,
+	HIF_REQ_ID_TX                   = 0x04,
+	HIF_REQ_ID_JOIN                 = 0x0b,
+	HIF_REQ_ID_SET_PM_MODE          = 0x10,
+	HIF_REQ_ID_SET_BSS_PARAMS       = 0x11,
+	HIF_REQ_ID_ADD_KEY              = 0x0c,
+	HIF_REQ_ID_REMOVE_KEY           = 0x0d,
+	HIF_REQ_ID_EDCA_QUEUE_PARAMS    = 0x13,
+	HIF_REQ_ID_START                = 0x17,
+	HIF_REQ_ID_BEACON_TRANSMIT      = 0x18,
+	HIF_REQ_ID_UPDATE_IE            = 0x1b,
+	HIF_REQ_ID_MAP_LINK             = 0x1c,
 };
 
 enum hif_confirmations_ids {
-	HIF_CNF_ID_RESET                             = 0x0a,
-	HIF_CNF_ID_READ_MIB                          = 0x05,
-	HIF_CNF_ID_WRITE_MIB                         = 0x06,
-	HIF_CNF_ID_START_SCAN                        = 0x07,
-	HIF_CNF_ID_STOP_SCAN                         = 0x08,
-	HIF_CNF_ID_TX                                = 0x04,
-	HIF_CNF_ID_MULTI_TRANSMIT                    = 0x1e,
-	HIF_CNF_ID_JOIN                              = 0x0b,
-	HIF_CNF_ID_SET_PM_MODE                       = 0x10,
-	HIF_CNF_ID_SET_BSS_PARAMS                    = 0x11,
-	HIF_CNF_ID_ADD_KEY                           = 0x0c,
-	HIF_CNF_ID_REMOVE_KEY                        = 0x0d,
-	HIF_CNF_ID_EDCA_QUEUE_PARAMS                 = 0x13,
-	HIF_CNF_ID_START                             = 0x17,
-	HIF_CNF_ID_BEACON_TRANSMIT                   = 0x18,
-	HIF_CNF_ID_UPDATE_IE                         = 0x1b,
-	HIF_CNF_ID_MAP_LINK                          = 0x1c,
+	HIF_CNF_ID_RESET                = 0x0a,
+	HIF_CNF_ID_READ_MIB             = 0x05,
+	HIF_CNF_ID_WRITE_MIB            = 0x06,
+	HIF_CNF_ID_START_SCAN           = 0x07,
+	HIF_CNF_ID_STOP_SCAN            = 0x08,
+	HIF_CNF_ID_TX                   = 0x04,
+	HIF_CNF_ID_MULTI_TRANSMIT       = 0x1e,
+	HIF_CNF_ID_JOIN                 = 0x0b,
+	HIF_CNF_ID_SET_PM_MODE          = 0x10,
+	HIF_CNF_ID_SET_BSS_PARAMS       = 0x11,
+	HIF_CNF_ID_ADD_KEY              = 0x0c,
+	HIF_CNF_ID_REMOVE_KEY           = 0x0d,
+	HIF_CNF_ID_EDCA_QUEUE_PARAMS    = 0x13,
+	HIF_CNF_ID_START                = 0x17,
+	HIF_CNF_ID_BEACON_TRANSMIT      = 0x18,
+	HIF_CNF_ID_UPDATE_IE            = 0x1b,
+	HIF_CNF_ID_MAP_LINK             = 0x1c,
 };
 
 enum hif_indications_ids {
-	HIF_IND_ID_RX                                = 0x84,
-	HIF_IND_ID_SCAN_CMPL                         = 0x86,
-	HIF_IND_ID_JOIN_COMPLETE                     = 0x8f,
-	HIF_IND_ID_SET_PM_MODE_CMPL                  = 0x89,
-	HIF_IND_ID_SUSPEND_RESUME_TX                 = 0x8c,
-	HIF_IND_ID_EVENT                             = 0x85
+	HIF_IND_ID_RX                   = 0x84,
+	HIF_IND_ID_SCAN_CMPL            = 0x86,
+	HIF_IND_ID_JOIN_COMPLETE        = 0x8f,
+	HIF_IND_ID_SET_PM_MODE_CMPL     = 0x89,
+	HIF_IND_ID_SUSPEND_RESUME_TX    = 0x8c,
+	HIF_IND_ID_EVENT                = 0x85
 };
 
 union hif_commands_ids {
@@ -68,27 +66,11 @@ union hif_commands_ids {
 	enum hif_indications_ids indication;
 };
 
-enum hif_status {
-	HIF_STATUS_SUCCESS                         = 0x0,
-	HIF_STATUS_FAILURE                         = 0x1,
-	HIF_INVALID_PARAMETER                      = 0x2,
-	HIF_STATUS_WARNING                         = 0x3,
-	HIF_ERROR_UNSUPPORTED_MSG_ID               = 0x4,
-	HIF_STATUS_DECRYPTFAILURE                  = 0x10,
-	HIF_STATUS_MICFAILURE                      = 0x11,
-	HIF_STATUS_NO_KEY_FOUND                    = 0x12,
-	HIF_STATUS_RETRY_EXCEEDED                  = 0x13,
-	HIF_STATUS_TX_LIFETIME_EXCEEDED            = 0x14,
-	HIF_REQUEUE                                = 0x15,
-	HIF_STATUS_REFUSED                         = 0x16,
-	HIF_STATUS_BUSY                            = 0x17
-};
-
 struct hif_reset_flags {
-	u8    reset_stat:1;
-	u8    reset_all_int:1;
-	u8    reserved1:6;
-	u8    reserved2[3];
+	u8     reset_stat:1;
+	u8     reset_all_int:1;
+	u8     reserved1:6;
+	u8     reserved2[3];
 } __packed;
 
 struct hif_req_reset {
@@ -96,117 +78,101 @@ struct hif_req_reset {
 } __packed;
 
 struct hif_req_read_mib {
-	u16   mib_id;
-	u16   reserved;
+	__le16 mib_id;
+	__le16 reserved;
 } __packed;
 
 struct hif_cnf_read_mib {
-	u32   status;
-	u16   mib_id;
-	u16   length;
-	u8    mib_data[];
+	__le32 status;
+	__le16 mib_id;
+	__le16 length;
+	u8     mib_data[];
 } __packed;
 
 struct hif_req_write_mib {
-	u16   mib_id;
-	u16   length;
-	u8    mib_data[];
+	__le16 mib_id;
+	__le16 length;
+	u8     mib_data[];
 } __packed;
 
 struct hif_cnf_write_mib {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_ie_flags {
-	u8    beacon:1;
-	u8    probe_resp:1;
-	u8    probe_req:1;
-	u8    reserved1:5;
-	u8    reserved2;
+	u8     beacon:1;
+	u8     probe_resp:1;
+	u8     probe_req:1;
+	u8     reserved1:5;
+	u8     reserved2;
 } __packed;
 
 struct hif_ie_tlv {
-	u8    type;
-	u8    length;
-	u8    data[];
+	u8     type;
+	u8     length;
+	u8     data[];
 } __packed;
 
 struct hif_req_update_ie {
 	struct hif_ie_flags ie_flags;
-	u16   num_ies;
+	__le16 num_ies;
 	struct hif_ie_tlv ie[];
 } __packed;
 
 struct hif_cnf_update_ie {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_scan_type {
-	u8    type:1;
-	u8    mode:1;
-	u8    reserved:6;
+	u8     type:1;
+	u8     mode:1;
+	u8     reserved:6;
 } __packed;
 
 struct hif_scan_flags {
-	u8    fbg:1;
-	u8    reserved1:1;
-	u8    pre:1;
-	u8    reserved2:5;
+	u8     fbg:1;
+	u8     reserved1:1;
+	u8     pre:1;
+	u8     reserved2:5;
 } __packed;
 
 struct hif_auto_scan_param {
-	u16   interval;
-	u8    reserved;
+	__le16 interval;
+	u8     reserved;
 	s8     rssi_thr;
 } __packed;
 
 struct hif_ssid_def {
-	u32   ssid_length;
-	u8    ssid[HIF_API_SSID_SIZE];
+	__le32 ssid_length;
+	u8     ssid[HIF_API_SSID_SIZE];
 } __packed;
 
 #define HIF_API_MAX_NB_SSIDS                           2
 #define HIF_API_MAX_NB_CHANNELS                       14
 
-struct hif_req_start_scan {
-	u8    band;
-	struct hif_scan_type scan_type;
-	struct hif_scan_flags scan_flags;
-	u8    max_transmit_rate;
-	struct hif_auto_scan_param auto_scan_param;
-	u8    num_of_probe_requests;
-	u8    probe_delay;
-	u8    num_of_ssids;
-	u8    num_of_channels;
-	u32   min_channel_time;
-	u32   max_channel_time;
-	s32    tx_power_level;
-	u8    ssid_and_channel_lists[];
-} __packed;
-
 struct hif_req_start_scan_alt {
-	u8    band;
+	u8     band;
 	struct hif_scan_type scan_type;
 	struct hif_scan_flags scan_flags;
-	u8    max_transmit_rate;
+	u8     max_transmit_rate;
 	struct hif_auto_scan_param auto_scan_param;
-	u8    num_of_probe_requests;
-	u8    probe_delay;
-	u8    num_of_ssids;
-	u8    num_of_channels;
-	u32   min_channel_time;
-	u32   max_channel_time;
-	s32    tx_power_level;
+	u8     num_of_probe_requests;
+	u8     probe_delay;
+	u8     num_of_ssids;
+	u8     num_of_channels;
+	__le32 min_channel_time;
+	__le32 max_channel_time;
+	__le32 tx_power_level; // signed value
 	struct hif_ssid_def ssid_def[HIF_API_MAX_NB_SSIDS];
-	u8    channel_list[];
+	u8     channel_list[];
 } __packed;
 
 struct hif_cnf_start_scan {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_cnf_stop_scan {
-	u32   status;
+	__le32 status;
 } __packed;
 
 enum hif_pm_mode_status {
@@ -216,10 +182,10 @@ enum hif_pm_mode_status {
 };
 
 struct hif_ind_scan_cmpl {
-	u32   status;
-	u8    pm_mode;
-	u8    num_channels_completed;
-	u16   reserved;
+	__le32 status;
+	u8     pm_mode;
+	u8     num_channels_completed;
+	__le16 reserved;
 } __packed;
 
 enum hif_queue_id {
@@ -241,46 +207,48 @@ enum hif_stbc {
 };
 
 struct hif_queue {
-	u8    queue_id:2;
-	u8    peer_sta_id:4;
-	u8    reserved:2;
+	u8     queue_id:2;
+	u8     peer_sta_id:4;
+	u8     reserved:2;
 } __packed;
 
 struct hif_data_flags {
-	u8    more:1;
-	u8    fc_offset:3;
-	u8    after_dtim:1;
-	u8    reserved:3;
+	u8     more:1;
+	u8     fc_offset:3;
+	u8     after_dtim:1;
+	u8     reserved:3;
 } __packed;
 
 struct hif_tx_flags {
-	u8    start_exp:1;
-	u8    reserved:3;
-	u8    retry_policy_index:4;
+	u8     start_exp:1;
+	u8     reserved:3;
+	u8     retry_policy_index:4;
 } __packed;
 
 struct hif_ht_tx_parameters {
-	u8    frame_format:4;
-	u8    fec_coding:1;
-	u8    short_gi:1;
-	u8    reserved1:1;
-	u8    stbc:1;
-	u8    reserved2;
-	u8    aggregation:1;
-	u8    reserved3:7;
-	u8    reserved4;
+	u8     frame_format:4;
+	u8     fec_coding:1;
+	u8     short_gi:1;
+	u8     reserved1:1;
+	u8     stbc:1;
+	u8     reserved2;
+	u8     aggregation:1;
+	u8     reserved3:7;
+	u8     reserved4;
 } __packed;
 
 struct hif_req_tx {
-	u32   packet_id;
-	u8    max_tx_rate;
+	// packet_id is not interpreted by the device, so it is not necessary to
+	// declare it little endian
+	u32    packet_id;
+	u8     max_tx_rate;
 	struct hif_queue queue_id;
 	struct hif_data_flags data_flags;
 	struct hif_tx_flags tx_flags;
-	u32   reserved;
-	u32   expire_time;
+	__le32 reserved;
+	__le32 expire_time;
 	struct hif_ht_tx_parameters ht_tx_parameters;
-	u8    frame[];
+	u8     frame[];
 } __packed;
 
 enum hif_qos_ackplcy {
@@ -291,26 +259,29 @@ enum hif_qos_ackplcy {
 };
 
 struct hif_tx_result_flags {
-	u8    aggr:1;
-	u8    requeue:1;
-	u8    ack_policy:2;
-	u8    txop_limit:1;
-	u8    reserved1:3;
-	u8    reserved2;
+	u8     aggr:1;
+	u8     requeue:1;
+	u8     ack_policy:2;
+	u8     txop_limit:1;
+	u8     reserved1:3;
+	u8     reserved2;
 } __packed;
 
 struct hif_cnf_tx {
-	u32   status;
-	u32   packet_id;
-	u8    txed_rate;
-	u8    ack_failures;
+	__le32 status;
+	// packet_id is copied from struct hif_req_tx without been interpreted
+	// by the device, so it is not necessary to declare it little endian
+	u32    packet_id;
+	u8     txed_rate;
+	u8     ack_failures;
 	struct hif_tx_result_flags tx_result_flags;
-	u32   media_delay;
-	u32   tx_queue_delay;
+	__le32 media_delay;
+	__le32 tx_queue_delay;
 } __packed;
 
 struct hif_cnf_multi_transmit {
-	u32   num_tx_confs;
+	u8     num_tx_confs;
+	u8     reserved[3];
 	struct hif_cnf_tx   tx_conf_payload[];
 } __packed;
 
@@ -323,147 +294,150 @@ enum hif_ri_flags_encrypt {
 };
 
 struct hif_rx_flags {
-	u8    encryp:3;
-	u8    in_aggr:1;
-	u8    first_aggr:1;
-	u8    last_aggr:1;
-	u8    defrag:1;
-	u8    beacon:1;
-	u8    tim:1;
-	u8    bitmap:1;
-	u8    match_ssid:1;
-	u8    match_bssid:1;
-	u8    more:1;
-	u8    reserved1:1;
-	u8    ht:1;
-	u8    stbc:1;
-	u8    match_uc_addr:1;
-	u8    match_mc_addr:1;
-	u8    match_bc_addr:1;
-	u8    key_type:1;
-	u8    key_index:4;
-	u8    reserved2:1;
-	u8    peer_sta_id:4;
-	u8    reserved3:2;
-	u8    reserved4:1;
+	u8     encryp:3;
+	u8     in_aggr:1;
+	u8     first_aggr:1;
+	u8     last_aggr:1;
+	u8     defrag:1;
+	u8     beacon:1;
+	u8     tim:1;
+	u8     bitmap:1;
+	u8     match_ssid:1;
+	u8     match_bssid:1;
+	u8     more:1;
+	u8     reserved1:1;
+	u8     ht:1;
+	u8     stbc:1;
+	u8     match_uc_addr:1;
+	u8     match_mc_addr:1;
+	u8     match_bc_addr:1;
+	u8     key_type:1;
+	u8     key_index:4;
+	u8     reserved2:1;
+	u8     peer_sta_id:4;
+	u8     reserved3:2;
+	u8     reserved4:1;
 } __packed;
 
 struct hif_ind_rx {
-	u32   status;
-	u16   channel_number;
-	u8    rxed_rate;
-	u8    rcpi_rssi;
+	__le32 status;
+	u8     channel_number;
+	u8     reserved;
+	u8     rxed_rate;
+	u8     rcpi_rssi;
 	struct hif_rx_flags rx_flags;
-	u8    frame[];
+	u8     frame[];
 } __packed;
 
 
 struct hif_req_edca_queue_params {
-	u8    queue_id;
-	u8    reserved1;
-	u8    aifsn;
-	u8    reserved2;
-	u16   cw_min;
-	u16   cw_max;
-	u16   tx_op_limit;
-	u16   allowed_medium_time;
-	u32   reserved3;
+	u8     queue_id;
+	u8     reserved1;
+	u8     aifsn;
+	u8     reserved2;
+	__le16 cw_min;
+	__le16 cw_max;
+	__le16 tx_op_limit;
+	__le16 allowed_medium_time;
+	__le32 reserved3;
 } __packed;
 
 struct hif_cnf_edca_queue_params {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_join_flags {
-	u8    reserved1:2;
-	u8    force_no_beacon:1;
-	u8    force_with_ind:1;
-	u8    reserved2:4;
+	u8     reserved1:2;
+	u8     force_no_beacon:1;
+	u8     force_with_ind:1;
+	u8     reserved2:4;
 } __packed;
 
 struct hif_req_join {
-	u8    infrastructure_bss_mode:1;
-	u8    reserved1:7;
-	u8    band;
-	u16   channel_number;
-	u8    bssid[ETH_ALEN];
-	u16   atim_window;
-	u8    short_preamble:1;
-	u8    reserved2:7;
-	u8    probe_for_join;
-	u8    reserved3;
+	u8     infrastructure_bss_mode:1;
+	u8     reserved1:7;
+	u8     band;
+	u8     channel_number;
+	u8     reserved;
+	u8     bssid[ETH_ALEN];
+	__le16 atim_window;
+	u8     short_preamble:1;
+	u8     reserved2:7;
+	u8     probe_for_join;
+	u8     reserved3;
 	struct hif_join_flags join_flags;
-	u32   ssid_length;
-	u8    ssid[HIF_API_SSID_SIZE];
-	u32   beacon_interval;
-	u32   basic_rate_set;
+	__le32 ssid_length;
+	u8     ssid[HIF_API_SSID_SIZE];
+	__le32 beacon_interval;
+	__le32 basic_rate_set;
 } __packed;
 
 struct hif_cnf_join {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_ind_join_complete {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_bss_flags {
-	u8    lost_count_only:1;
-	u8    reserved:7;
+	u8     lost_count_only:1;
+	u8     reserved:7;
 } __packed;
 
 struct hif_req_set_bss_params {
 	struct hif_bss_flags bss_flags;
-	u8    beacon_lost_count;
-	u16   aid;
-	u32   operational_rate_set;
+	u8     beacon_lost_count;
+	__le16 aid;
+	__le32 operational_rate_set;
 } __packed;
 
 struct hif_cnf_set_bss_params {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_pm_mode {
-	u8    enter_psm:1;
-	u8    reserved:6;
-	u8    fast_psm:1;
+	u8     enter_psm:1;
+	u8     reserved:6;
+	u8     fast_psm:1;
 } __packed;
 
 struct hif_req_set_pm_mode {
 	struct hif_pm_mode pm_mode;
-	u8    fast_psm_idle_period;
-	u8    ap_psm_change_period;
-	u8    min_auto_ps_poll_period;
+	u8     fast_psm_idle_period;
+	u8     ap_psm_change_period;
+	u8     min_auto_ps_poll_period;
 } __packed;
 
 struct hif_cnf_set_pm_mode {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_ind_set_pm_mode_cmpl {
-	u32   status;
-	u8    pm_mode;
-	u8    reserved[3];
+	__le32 status;
+	u8     pm_mode;
+	u8     reserved[3];
 } __packed;
 
 
 struct hif_req_start {
-	u8    mode;
-	u8    band;
-	u16   channel_number;
-	u32   reserved1;
-	u32   beacon_interval;
-	u8    dtim_period;
-	u8    short_preamble:1;
-	u8    reserved2:7;
-	u8    reserved3;
-	u8    ssid_length;
-	u8    ssid[HIF_API_SSID_SIZE];
-	u32   basic_rate_set;
+	u8     mode;
+	u8     band;
+	u8     channel_number;
+	u8     reserved1;
+	__le32 reserved2;
+	__le32 beacon_interval;
+	u8     dtim_period;
+	u8     short_preamble:1;
+	u8     reserved3:7;
+	u8     reserved4;
+	u8     ssid_length;
+	u8     ssid[HIF_API_SSID_SIZE];
+	__le32 basic_rate_set;
 } __packed;
 
 struct hif_cnf_start {
-	u32   status;
+	__le32 status;
 } __packed;
 
 enum hif_beacon {
@@ -472,46 +446,49 @@ enum hif_beacon {
 };
 
 struct hif_req_beacon_transmit {
-	u8    enable_beaconing;
-	u8    reserved[3];
+	u8     enable_beaconing;
+	u8     reserved[3];
 } __packed;
 
 struct hif_cnf_beacon_transmit {
-	u32   status;
+	__le32 status;
 } __packed;
 
+#define HIF_LINK_ID_MAX            14
+#define HIF_LINK_ID_NOT_ASSOCIATED (HIF_LINK_ID_MAX + 1)
+
 enum hif_sta_map_direction {
 	HIF_STA_MAP                       = 0x0,
 	HIF_STA_UNMAP                     = 0x1
 };
 
 struct hif_map_link_flags {
-	u8    map_direction:1;
-	u8    mfpc:1;
-	u8    reserved:6;
+	u8     map_direction:1;
+	u8     mfpc:1;
+	u8     reserved:6;
 } __packed;
 
 struct hif_req_map_link {
-	u8    mac_addr[ETH_ALEN];
+	u8     mac_addr[ETH_ALEN];
 	struct hif_map_link_flags map_link_flags;
-	u8    peer_sta_id;
+	u8     peer_sta_id;
 } __packed;
 
 struct hif_cnf_map_link {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_suspend_resume_flags {
-	u8    resume:1;
-	u8    reserved1:2;
-	u8    bc_mc_only:1;
-	u8    reserved2:4;
-	u8    reserved3;
+	u8     resume:1;
+	u8     reserved1:2;
+	u8     bc_mc_only:1;
+	u8     reserved2:4;
+	u8     reserved3;
 } __packed;
 
 struct hif_ind_suspend_resume_tx {
 	struct hif_suspend_resume_flags suspend_resume_flags;
-	u16   peer_sta_set;
+	__le16 peer_sta_set;
 } __packed;
 
 
@@ -541,102 +518,102 @@ enum hif_key_type {
 };
 
 struct hif_wep_pairwise_key {
-	u8    peer_address[ETH_ALEN];
-	u8    reserved;
-	u8    key_length;
-	u8    key_data[HIF_API_WEP_KEY_DATA_SIZE];
+	u8     peer_address[ETH_ALEN];
+	u8     reserved;
+	u8     key_length;
+	u8     key_data[HIF_API_WEP_KEY_DATA_SIZE];
 } __packed;
 
 struct hif_wep_group_key {
-	u8    key_id;
-	u8    key_length;
-	u8    reserved[2];
-	u8    key_data[HIF_API_WEP_KEY_DATA_SIZE];
+	u8     key_id;
+	u8     key_length;
+	u8     reserved[2];
+	u8     key_data[HIF_API_WEP_KEY_DATA_SIZE];
 } __packed;
 
 struct hif_tkip_pairwise_key {
-	u8    peer_address[ETH_ALEN];
-	u8    reserved[2];
-	u8    tkip_key_data[HIF_API_TKIP_KEY_DATA_SIZE];
-	u8    rx_mic_key[HIF_API_RX_MIC_KEY_SIZE];
-	u8    tx_mic_key[HIF_API_TX_MIC_KEY_SIZE];
+	u8     peer_address[ETH_ALEN];
+	u8     reserved[2];
+	u8     tkip_key_data[HIF_API_TKIP_KEY_DATA_SIZE];
+	u8     rx_mic_key[HIF_API_RX_MIC_KEY_SIZE];
+	u8     tx_mic_key[HIF_API_TX_MIC_KEY_SIZE];
 } __packed;
 
 struct hif_tkip_group_key {
-	u8    tkip_key_data[HIF_API_TKIP_KEY_DATA_SIZE];
-	u8    rx_mic_key[HIF_API_RX_MIC_KEY_SIZE];
-	u8    key_id;
-	u8    reserved[3];
-	u8    rx_sequence_counter[HIF_API_RX_SEQUENCE_COUNTER_SIZE];
+	u8     tkip_key_data[HIF_API_TKIP_KEY_DATA_SIZE];
+	u8     rx_mic_key[HIF_API_RX_MIC_KEY_SIZE];
+	u8     key_id;
+	u8     reserved[3];
+	u8     rx_sequence_counter[HIF_API_RX_SEQUENCE_COUNTER_SIZE];
 } __packed;
 
 struct hif_aes_pairwise_key {
-	u8    peer_address[ETH_ALEN];
-	u8    reserved[2];
-	u8    aes_key_data[HIF_API_AES_KEY_DATA_SIZE];
+	u8     peer_address[ETH_ALEN];
+	u8     reserved[2];
+	u8     aes_key_data[HIF_API_AES_KEY_DATA_SIZE];
 } __packed;
 
 struct hif_aes_group_key {
-	u8    aes_key_data[HIF_API_AES_KEY_DATA_SIZE];
-	u8    key_id;
-	u8    reserved[3];
-	u8    rx_sequence_counter[HIF_API_RX_SEQUENCE_COUNTER_SIZE];
+	u8     aes_key_data[HIF_API_AES_KEY_DATA_SIZE];
+	u8     key_id;
+	u8     reserved[3];
+	u8     rx_sequence_counter[HIF_API_RX_SEQUENCE_COUNTER_SIZE];
 } __packed;
 
 struct hif_wapi_pairwise_key {
-	u8    peer_address[ETH_ALEN];
-	u8    key_id;
-	u8    reserved;
-	u8    wapi_key_data[HIF_API_WAPI_KEY_DATA_SIZE];
-	u8    mic_key_data[HIF_API_MIC_KEY_DATA_SIZE];
+	u8     peer_address[ETH_ALEN];
+	u8     key_id;
+	u8     reserved;
+	u8     wapi_key_data[HIF_API_WAPI_KEY_DATA_SIZE];
+	u8     mic_key_data[HIF_API_MIC_KEY_DATA_SIZE];
 } __packed;
 
 struct hif_wapi_group_key {
-	u8    wapi_key_data[HIF_API_WAPI_KEY_DATA_SIZE];
-	u8    mic_key_data[HIF_API_MIC_KEY_DATA_SIZE];
-	u8    key_id;
-	u8    reserved[3];
+	u8     wapi_key_data[HIF_API_WAPI_KEY_DATA_SIZE];
+	u8     mic_key_data[HIF_API_MIC_KEY_DATA_SIZE];
+	u8     key_id;
+	u8     reserved[3];
 } __packed;
 
 struct hif_igtk_group_key {
-	u8    igtk_key_data[HIF_API_IGTK_KEY_DATA_SIZE];
-	u8    key_id;
-	u8    reserved[3];
-	u8    ipn[HIF_API_IPN_SIZE];
+	u8     igtk_key_data[HIF_API_IGTK_KEY_DATA_SIZE];
+	u8     key_id;
+	u8     reserved[3];
+	u8     ipn[HIF_API_IPN_SIZE];
 } __packed;
 
 union hif_privacy_key_data {
-	struct hif_wep_pairwise_key                       wep_pairwise_key;
-	struct hif_wep_group_key                          wep_group_key;
-	struct hif_tkip_pairwise_key                      tkip_pairwise_key;
-	struct hif_tkip_group_key                         tkip_group_key;
-	struct hif_aes_pairwise_key                       aes_pairwise_key;
-	struct hif_aes_group_key                          aes_group_key;
-	struct hif_wapi_pairwise_key                      wapi_pairwise_key;
-	struct hif_wapi_group_key                         wapi_group_key;
-	struct hif_igtk_group_key                         igtk_group_key;
+	struct hif_wep_pairwise_key  wep_pairwise_key;
+	struct hif_wep_group_key     wep_group_key;
+	struct hif_tkip_pairwise_key tkip_pairwise_key;
+	struct hif_tkip_group_key    tkip_group_key;
+	struct hif_aes_pairwise_key  aes_pairwise_key;
+	struct hif_aes_group_key     aes_group_key;
+	struct hif_wapi_pairwise_key wapi_pairwise_key;
+	struct hif_wapi_group_key    wapi_group_key;
+	struct hif_igtk_group_key    igtk_group_key;
 };
 
 struct hif_req_add_key {
-	u8    type;
-	u8    entry_index;
-	u8    int_id:2;
-	u8    reserved1:6;
-	u8    reserved2;
+	u8     type;
+	u8     entry_index;
+	u8     int_id:2;
+	u8     reserved1:6;
+	u8     reserved2;
 	union hif_privacy_key_data key;
 } __packed;
 
 struct hif_cnf_add_key {
-	u32   status;
+	__le32 status;
 } __packed;
 
 struct hif_req_remove_key {
-	u8    entry_index;
-	u8    reserved[3];
+	u8     entry_index;
+	u8     reserved[3];
 } __packed;
 
 struct hif_cnf_remove_key {
-	u32   status;
+	__le32 status;
 } __packed;
 
 enum hif_event_ind {
@@ -656,13 +633,13 @@ enum hif_ps_mode_error {
 };
 
 union hif_event_data {
-	u8    rcpi_rssi;
-	u32   ps_mode_error;
-	u32   peer_sta_set;
+	u8     rcpi_rssi;
+	__le32 ps_mode_error;
+	__le32 peer_sta_set;
 };
 
 struct hif_ind_event {
-	u32   event_id;
+	__le32 event_id;
 	union hif_event_data event_data;
 } __packed;
 
diff --git a/drivers/staging/wfx/hif_api_general.h b/drivers/staging/wfx/hif_api_general.h
index a069c3a21b4d..dba18a7ae919 100644
--- a/drivers/staging/wfx/hif_api_general.h
+++ b/drivers/staging/wfx/hif_api_general.h
@@ -17,13 +17,13 @@
 #define __packed __attribute__((__packed__))
 #endif
 
-#define API_SSID_SIZE                       32
+#define API_SSID_SIZE             32
 
-#define HIF_ID_IS_INDICATION               0x80
-#define HIF_COUNTER_MAX                    7
+#define HIF_ID_IS_INDICATION      0x80
+#define HIF_COUNTER_MAX           7
 
 struct hif_msg {
-	u16    len;
+	__le16 len;
 	u8     id;
 	u8     reserved:1;
 	u8     interface:2;
@@ -33,239 +33,252 @@ struct hif_msg {
 } __packed;
 
 enum hif_general_requests_ids {
-	HIF_REQ_ID_CONFIGURATION                         = 0x09,
-	HIF_REQ_ID_CONTROL_GPIO                          = 0x26,
-	HIF_REQ_ID_SET_SL_MAC_KEY                        = 0x27,
-	HIF_REQ_ID_SL_EXCHANGE_PUB_KEYS                  = 0x28,
-	HIF_REQ_ID_SL_CONFIGURE                          = 0x29,
-	HIF_REQ_ID_PREVENT_ROLLBACK                      = 0x2a,
-	HIF_REQ_ID_PTA_SETTINGS                          = 0x2b,
-	HIF_REQ_ID_PTA_PRIORITY                          = 0x2c,
-	HIF_REQ_ID_PTA_STATE                             = 0x2d,
-	HIF_REQ_ID_SHUT_DOWN                             = 0x32,
+	HIF_REQ_ID_CONFIGURATION        = 0x09,
+	HIF_REQ_ID_CONTROL_GPIO         = 0x26,
+	HIF_REQ_ID_SET_SL_MAC_KEY       = 0x27,
+	HIF_REQ_ID_SL_EXCHANGE_PUB_KEYS = 0x28,
+	HIF_REQ_ID_SL_CONFIGURE         = 0x29,
+	HIF_REQ_ID_PREVENT_ROLLBACK     = 0x2a,
+	HIF_REQ_ID_PTA_SETTINGS         = 0x2b,
+	HIF_REQ_ID_PTA_PRIORITY         = 0x2c,
+	HIF_REQ_ID_PTA_STATE            = 0x2d,
+	HIF_REQ_ID_SHUT_DOWN            = 0x32,
 };
 
 enum hif_general_confirmations_ids {
-	HIF_CNF_ID_CONFIGURATION                         = 0x09,
-	HIF_CNF_ID_CONTROL_GPIO                          = 0x26,
-	HIF_CNF_ID_SET_SL_MAC_KEY                        = 0x27,
-	HIF_CNF_ID_SL_EXCHANGE_PUB_KEYS                  = 0x28,
-	HIF_CNF_ID_SL_CONFIGURE                          = 0x29,
-	HIF_CNF_ID_PREVENT_ROLLBACK                      = 0x2a,
-	HIF_CNF_ID_PTA_SETTINGS                          = 0x2b,
-	HIF_CNF_ID_PTA_PRIORITY                          = 0x2c,
-	HIF_CNF_ID_PTA_STATE                             = 0x2d,
-	HIF_CNF_ID_SHUT_DOWN                             = 0x32,
+	HIF_CNF_ID_CONFIGURATION        = 0x09,
+	HIF_CNF_ID_CONTROL_GPIO         = 0x26,
+	HIF_CNF_ID_SET_SL_MAC_KEY       = 0x27,
+	HIF_CNF_ID_SL_EXCHANGE_PUB_KEYS = 0x28,
+	HIF_CNF_ID_SL_CONFIGURE         = 0x29,
+	HIF_CNF_ID_PREVENT_ROLLBACK     = 0x2a,
+	HIF_CNF_ID_PTA_SETTINGS         = 0x2b,
+	HIF_CNF_ID_PTA_PRIORITY         = 0x2c,
+	HIF_CNF_ID_PTA_STATE            = 0x2d,
+	HIF_CNF_ID_SHUT_DOWN            = 0x32,
 };
 
 enum hif_general_indications_ids {
-	HIF_IND_ID_EXCEPTION                             = 0xe0,
-	HIF_IND_ID_STARTUP                               = 0xe1,
-	HIF_IND_ID_WAKEUP                                = 0xe2,
-	HIF_IND_ID_GENERIC                               = 0xe3,
-	HIF_IND_ID_ERROR                                 = 0xe4,
-	HIF_IND_ID_SL_EXCHANGE_PUB_KEYS                  = 0xe5
+	HIF_IND_ID_EXCEPTION            = 0xe0,
+	HIF_IND_ID_STARTUP              = 0xe1,
+	HIF_IND_ID_WAKEUP               = 0xe2,
+	HIF_IND_ID_GENERIC              = 0xe3,
+	HIF_IND_ID_ERROR                = 0xe4,
+	HIF_IND_ID_SL_EXCHANGE_PUB_KEYS = 0xe5
 };
 
-enum hif_hi_status {
-	HI_STATUS_SUCCESS                             = 0x0000,
-	HI_STATUS_FAILURE                             = 0x0001,
-	HI_INVALID_PARAMETER                          = 0x0002,
-	HI_STATUS_GPIO_WARNING                        = 0x0003,
-	HI_ERROR_UNSUPPORTED_MSG_ID                   = 0x0004,
-	SL_MAC_KEY_STATUS_SUCCESS                     = 0x005A,
-	SL_MAC_KEY_STATUS_FAILED_KEY_ALREADY_BURNED   = 0x006B,
-	SL_MAC_KEY_STATUS_FAILED_RAM_MODE_NOT_ALLOWED = 0x007C,
-	SL_MAC_KEY_STATUS_FAILED_UNKNOWN_MODE         = 0x008D,
-	SL_PUB_KEY_EXCHANGE_STATUS_SUCCESS            = 0x009E,
-	SL_PUB_KEY_EXCHANGE_STATUS_FAILED             = 0x00AF,
-	PREVENT_ROLLBACK_CNF_SUCCESS                  = 0x1234,
-	PREVENT_ROLLBACK_CNF_WRONG_MAGIC_WORD         = 0x1256
-};
+#define HIF_STATUS_SUCCESS                         (cpu_to_le32(0x0000))
+#define HIF_STATUS_FAIL                            (cpu_to_le32(0x0001))
+#define HIF_STATUS_INVALID_PARAMETER               (cpu_to_le32(0x0002))
+#define HIF_STATUS_WARNING                         (cpu_to_le32(0x0003))
+#define HIF_STATUS_UNKNOWN_REQUEST                 (cpu_to_le32(0x0004))
+#define HIF_STATUS_RX_FAIL_DECRYPT                 (cpu_to_le32(0x0010))
+#define HIF_STATUS_RX_FAIL_MIC                     (cpu_to_le32(0x0011))
+#define HIF_STATUS_RX_FAIL_NO_KEY                  (cpu_to_le32(0x0012))
+#define HIF_STATUS_TX_FAIL_RETRIES                 (cpu_to_le32(0x0013))
+#define HIF_STATUS_TX_FAIL_TIMEOUT                 (cpu_to_le32(0x0014))
+#define HIF_STATUS_TX_FAIL_REQUEUE                 (cpu_to_le32(0x0015))
+#define HIF_STATUS_REFUSED                         (cpu_to_le32(0x0016))
+#define HIF_STATUS_BUSY                            (cpu_to_le32(0x0017))
+#define HIF_STATUS_SLK_SET_KEY_SUCCESS             (cpu_to_le32(0x005A))
+#define HIF_STATUS_SLK_SET_KEY_ALREADY_BURNED      (cpu_to_le32(0x006B))
+#define HIF_STATUS_SLK_SET_KEY_DISALLOWED_MODE     (cpu_to_le32(0x007C))
+#define HIF_STATUS_SLK_SET_KEY_UNKNOWN_MODE        (cpu_to_le32(0x008D))
+#define HIF_STATUS_SLK_NEGO_SUCCESS                (cpu_to_le32(0x009E))
+#define HIF_STATUS_SLK_NEGO_FAILED                 (cpu_to_le32(0x00AF))
+#define HIF_STATUS_ROLLBACK_SUCCESS                (cpu_to_le32(0x1234))
+#define HIF_STATUS_ROLLBACK_FAIL                   (cpu_to_le32(0x1256))
 
 enum hif_api_rate_index {
-	API_RATE_INDEX_B_1MBPS                   = 0,
-	API_RATE_INDEX_B_2MBPS                   = 1,
-	API_RATE_INDEX_B_5P5MBPS                 = 2,
-	API_RATE_INDEX_B_11MBPS                  = 3,
-	API_RATE_INDEX_PBCC_22MBPS               = 4,
-	API_RATE_INDEX_PBCC_33MBPS               = 5,
-	API_RATE_INDEX_G_6MBPS                   = 6,
-	API_RATE_INDEX_G_9MBPS                   = 7,
-	API_RATE_INDEX_G_12MBPS                  = 8,
-	API_RATE_INDEX_G_18MBPS                  = 9,
-	API_RATE_INDEX_G_24MBPS                  = 10,
-	API_RATE_INDEX_G_36MBPS                  = 11,
-	API_RATE_INDEX_G_48MBPS                  = 12,
-	API_RATE_INDEX_G_54MBPS                  = 13,
-	API_RATE_INDEX_N_6P5MBPS                 = 14,
-	API_RATE_INDEX_N_13MBPS                  = 15,
-	API_RATE_INDEX_N_19P5MBPS                = 16,
-	API_RATE_INDEX_N_26MBPS                  = 17,
-	API_RATE_INDEX_N_39MBPS                  = 18,
-	API_RATE_INDEX_N_52MBPS                  = 19,
-	API_RATE_INDEX_N_58P5MBPS                = 20,
-	API_RATE_INDEX_N_65MBPS                  = 21,
-	API_RATE_NUM_ENTRIES                     = 22
+	API_RATE_INDEX_B_1MBPS     = 0,
+	API_RATE_INDEX_B_2MBPS     = 1,
+	API_RATE_INDEX_B_5P5MBPS   = 2,
+	API_RATE_INDEX_B_11MBPS    = 3,
+	API_RATE_INDEX_PBCC_22MBPS = 4,
+	API_RATE_INDEX_PBCC_33MBPS = 5,
+	API_RATE_INDEX_G_6MBPS     = 6,
+	API_RATE_INDEX_G_9MBPS     = 7,
+	API_RATE_INDEX_G_12MBPS    = 8,
+	API_RATE_INDEX_G_18MBPS    = 9,
+	API_RATE_INDEX_G_24MBPS    = 10,
+	API_RATE_INDEX_G_36MBPS    = 11,
+	API_RATE_INDEX_G_48MBPS    = 12,
+	API_RATE_INDEX_G_54MBPS    = 13,
+	API_RATE_INDEX_N_6P5MBPS   = 14,
+	API_RATE_INDEX_N_13MBPS    = 15,
+	API_RATE_INDEX_N_19P5MBPS  = 16,
+	API_RATE_INDEX_N_26MBPS    = 17,
+	API_RATE_INDEX_N_39MBPS    = 18,
+	API_RATE_INDEX_N_52MBPS    = 19,
+	API_RATE_INDEX_N_58P5MBPS  = 20,
+	API_RATE_INDEX_N_65MBPS    = 21,
+	API_RATE_NUM_ENTRIES       = 22
 };
 
 
 enum hif_fw_type {
-	HIF_FW_TYPE_ETF                             = 0x0,
-	HIF_FW_TYPE_WFM                             = 0x1,
-	HIF_FW_TYPE_WSM                             = 0x2
+	HIF_FW_TYPE_ETF  = 0x0,
+	HIF_FW_TYPE_WFM  = 0x1,
+	HIF_FW_TYPE_WSM  = 0x2
 };
 
 struct hif_capabilities {
-	u8    link_mode:2;
-	u8    reserved1:6;
-	u8    reserved2;
-	u8    reserved3;
-	u8    reserved4;
+	u8     link_mode:2;
+	u8     reserved1:6;
+	u8     reserved2;
+	u8     reserved3;
+	u8     reserved4;
 } __packed;
 
 struct hif_otp_regul_sel_mode_info {
-	u8    region_sel_mode:4;
-	u8    reserved:4;
+	u8     region_sel_mode:4;
+	u8     reserved:4;
 } __packed;
 
 struct hif_otp_phy_info {
-	u8    phy1_region:3;
-	u8    phy0_region:3;
-	u8    otp_phy_ver:2;
+	u8     phy1_region:3;
+	u8     phy0_region:3;
+	u8     otp_phy_ver:2;
 } __packed;
 
-#define API_OPN_SIZE                                    14
-#define API_UID_SIZE                                    8
-#define API_DISABLED_CHANNEL_LIST_SIZE                  2
-#define API_FIRMWARE_LABEL_SIZE                         128
-
 struct hif_ind_startup {
-	u32   status;
-	u16   hardware_id;
-	u8    opn[API_OPN_SIZE];
-	u8    uid[API_UID_SIZE];
-	u16   num_inp_ch_bufs;
-	u16   size_inp_ch_buf;
-	u8    num_links_ap;
-	u8    num_interfaces;
-	u8    mac_addr[2][ETH_ALEN];
-	u8    api_version_minor;
-	u8    api_version_major;
+	// As the others, this struct is interpreted as little endian by the
+	// device. However, this struct is also used by the driver. We prefer to
+	// declare it in native order and doing byte swap on reception.
+	__le32 status;
+	u16    hardware_id;
+	u8     opn[14];
+	u8     uid[8];
+	u16    num_inp_ch_bufs;
+	u16    size_inp_ch_buf;
+	u8     num_links_ap;
+	u8     num_interfaces;
+	u8     mac_addr[2][ETH_ALEN];
+	u8     api_version_minor;
+	u8     api_version_major;
 	struct hif_capabilities capabilities;
-	u8    firmware_build;
-	u8    firmware_minor;
-	u8    firmware_major;
-	u8    firmware_type;
-	u8    disabled_channel_list[API_DISABLED_CHANNEL_LIST_SIZE];
+	u8     firmware_build;
+	u8     firmware_minor;
+	u8     firmware_major;
+	u8     firmware_type;
+	u8     disabled_channel_list[2];
 	struct hif_otp_regul_sel_mode_info regul_sel_mode_info;
 	struct hif_otp_phy_info otp_phy_info;
-	u32   supported_rate_mask;
-	u8    firmware_label[API_FIRMWARE_LABEL_SIZE];
+	u32    supported_rate_mask;
+	u8     firmware_label[128];
 } __packed;
 
 struct hif_ind_wakeup {
 } __packed;
 
 struct hif_req_configuration {
-	u16   length;
-	u8    pds_data[];
+	__le16 length;
+	u8     pds_data[];
 } __packed;
 
 struct hif_cnf_configuration {
-	u32   status;
+	__le32 status;
 } __packed;
 
 enum hif_gpio_mode {
-	HIF_GPIO_MODE_D0                            = 0x0,
-	HIF_GPIO_MODE_D1                            = 0x1,
-	HIF_GPIO_MODE_OD0                           = 0x2,
-	HIF_GPIO_MODE_OD1                           = 0x3,
-	HIF_GPIO_MODE_TRISTATE                      = 0x4,
-	HIF_GPIO_MODE_TOGGLE                        = 0x5,
-	HIF_GPIO_MODE_READ                          = 0x6
+	HIF_GPIO_MODE_D0       = 0x0,
+	HIF_GPIO_MODE_D1       = 0x1,
+	HIF_GPIO_MODE_OD0      = 0x2,
+	HIF_GPIO_MODE_OD1      = 0x3,
+	HIF_GPIO_MODE_TRISTATE = 0x4,
+	HIF_GPIO_MODE_TOGGLE   = 0x5,
+	HIF_GPIO_MODE_READ     = 0x6
 };
 
 struct hif_req_control_gpio {
-	u8 gpio_label;
-	u8 gpio_mode;
+	u8     gpio_label;
+	u8     gpio_mode;
 } __packed;
 
-enum hif_gpio_error {
-	HIF_GPIO_ERROR_0                            = 0x0,
-	HIF_GPIO_ERROR_1                            = 0x1,
-	HIF_GPIO_ERROR_2                            = 0x2
-};
-
 struct hif_cnf_control_gpio {
-	u32 status;
-	u32 value;
+	__le32 status;
+	__le32 value;
 } __packed;
 
 enum hif_generic_indication_type {
-	HIF_GENERIC_INDICATION_TYPE_RAW               = 0x0,
-	HIF_GENERIC_INDICATION_TYPE_STRING            = 0x1,
-	HIF_GENERIC_INDICATION_TYPE_RX_STATS          = 0x2
+	HIF_GENERIC_INDICATION_TYPE_RAW                = 0x0,
+	HIF_GENERIC_INDICATION_TYPE_STRING             = 0x1,
+	HIF_GENERIC_INDICATION_TYPE_RX_STATS           = 0x2,
+	HIF_GENERIC_INDICATION_TYPE_TX_POWER_LOOP_INFO = 0x3,
 };
 
 struct hif_rx_stats {
-	u32   nb_rx_frame;
-	u32   nb_crc_frame;
-	u32   per_total;
-	u32   throughput;
-	u32   nb_rx_by_rate[API_RATE_NUM_ENTRIES];
-	u16   per[API_RATE_NUM_ENTRIES];
-	s16    snr[API_RATE_NUM_ENTRIES];
-	s16    rssi[API_RATE_NUM_ENTRIES];
-	s16    cfo[API_RATE_NUM_ENTRIES];
-	u32   date;
-	u32   pwr_clk_freq;
-	u8    is_ext_pwr_clk;
+	__le32 nb_rx_frame;
+	__le32 nb_crc_frame;
+	__le32 per_total;
+	__le32 throughput;
+	__le32 nb_rx_by_rate[API_RATE_NUM_ENTRIES];
+	__le16 per[API_RATE_NUM_ENTRIES];
+	__le16 snr[API_RATE_NUM_ENTRIES];  // signed value
+	__le16 rssi[API_RATE_NUM_ENTRIES]; // signed value
+	__le16 cfo[API_RATE_NUM_ENTRIES];  // signed value
+	__le32 date;
+	__le32 pwr_clk_freq;
+	u8     is_ext_pwr_clk;
 	s8     current_temp;
 } __packed;
 
+struct hif_tx_power_loop_info {
+	__le16 tx_gain_dig;
+	__le16 tx_gain_pa;
+	__le16 target_pout; // signed value
+	__le16 p_estimation; // signed value
+	__le16 vpdet;
+	u8     measurement_index;
+	u8     reserved;
+} __packed;
+
 union hif_indication_data {
-	struct hif_rx_stats                                   rx_stats;
-	u8                                       raw_data[1];
+	struct hif_rx_stats rx_stats;
+	struct hif_tx_power_loop_info tx_power_loop_info;
+	u8     raw_data[1];
 };
 
 struct hif_ind_generic {
-	u32 indication_type;
+	__le32 indication_type;
 	union hif_indication_data indication_data;
 } __packed;
 
-
-#define HIF_EXCEPTION_DATA_SIZE            124
-
-struct hif_ind_exception {
-	u8    data[HIF_EXCEPTION_DATA_SIZE];
-} __packed;
-
-
 enum hif_error {
-	HIF_ERROR_FIRMWARE_ROLLBACK             = 0x0,
-	HIF_ERROR_FIRMWARE_DEBUG_ENABLED        = 0x1,
-	HIF_ERROR_OUTDATED_SESSION_KEY          = 0x2,
-	HIF_ERROR_INVALID_SESSION_KEY           = 0x3,
-	HIF_ERROR_OOR_VOLTAGE                   = 0x4,
-	HIF_ERROR_PDS_VERSION                   = 0x5,
-	HIF_ERROR_OOR_TEMPERATURE               = 0x6,
-	HIF_ERROR_REQ_DURING_KEY_EXCHANGE       = 0x7,
-	HIF_ERROR_MULTI_TX_CNF_SECURELINK       = 0x8,
-	HIF_ERROR_SECURELINK_OVERFLOW           = 0x9,
-	HIF_ERROR_SECURELINK_DECRYPTION         = 0xa
+	HIF_ERROR_FIRMWARE_ROLLBACK           = 0x00,
+	HIF_ERROR_FIRMWARE_DEBUG_ENABLED      = 0x01,
+	HIF_ERROR_SLK_OUTDATED_SESSION_KEY    = 0x02,
+	HIF_ERROR_SLK_SESSION_KEY             = 0x03,
+	HIF_ERROR_OOR_VOLTAGE                 = 0x04,
+	HIF_ERROR_PDS_PAYLOAD                 = 0x05,
+	HIF_ERROR_OOR_TEMPERATURE             = 0x06,
+	HIF_ERROR_SLK_REQ_DURING_KEY_EXCHANGE = 0x07,
+	HIF_ERROR_SLK_MULTI_TX_UNSUPPORTED    = 0x08,
+	HIF_ERROR_SLK_OVERFLOW                = 0x09,
+	HIF_ERROR_SLK_DECRYPTION              = 0x0a,
+	HIF_ERROR_SLK_WRONG_ENCRYPTION_STATE  = 0x0b,
+	HIF_ERROR_HIF_BUS_FREQUENCY_TOO_LOW   = 0x0c,
+	HIF_ERROR_HIF_RX_DATA_TOO_LARGE       = 0x0e,
+	HIF_ERROR_HIF_TX_QUEUE_FULL           = 0x0d,
+	HIF_ERROR_HIF_BUS                     = 0x0f,
+	HIF_ERROR_PDS_TESTFEATURE             = 0x10,
 };
 
 struct hif_ind_error {
-	u32   type;
-	u8    data[];
+	__le32 type;
+	u8     data[];
+} __packed;
+
+struct hif_ind_exception {
+	__le32 type;
+	u8     data[];
 } __packed;
 
 enum hif_secure_link_state {
-	SEC_LINK_UNAVAILABLE                    = 0x0,
-	SEC_LINK_RESERVED                       = 0x1,
-	SEC_LINK_EVAL                           = 0x2,
-	SEC_LINK_ENFORCED                       = 0x3
+	SEC_LINK_UNAVAILABLE = 0x0,
+	SEC_LINK_RESERVED    = 0x1,
+	SEC_LINK_EVAL        = 0x2,
+	SEC_LINK_ENFORCED    = 0x3
 };
 
 enum hif_sl_encryption_type {
@@ -282,156 +295,70 @@ struct hif_sl_msg_hdr {
 
 struct hif_sl_msg {
 	struct hif_sl_msg_hdr hdr;
-	u16        len;
-	u8         payload[];
+	__le16 len;
+	u8     payload[];
 } __packed;
 
-#define AES_CCM_TAG_SIZE     16
+#define AES_CCM_TAG_SIZE          16
 
 struct hif_sl_tag {
-	u8 tag[16];
+	u8     tag[16];
 } __packed;
 
 enum hif_sl_mac_key_dest {
-	SL_MAC_KEY_DEST_OTP                        = 0x78,
-	SL_MAC_KEY_DEST_RAM                        = 0x87
+	SL_MAC_KEY_DEST_OTP = 0x78,
+	SL_MAC_KEY_DEST_RAM = 0x87
 };
 
-#define API_KEY_VALUE_SIZE      32
+#define API_KEY_VALUE_SIZE        32
 
 struct hif_req_set_sl_mac_key {
-	u8    otp_or_ram;
-	u8    key_value[API_KEY_VALUE_SIZE];
+	u8     otp_or_ram;
+	u8     key_value[API_KEY_VALUE_SIZE];
 } __packed;
 
 struct hif_cnf_set_sl_mac_key {
-	u32   status;
+	__le32 status;
 } __packed;
 
-#define API_HOST_PUB_KEY_SIZE                           32
-#define API_HOST_PUB_KEY_MAC_SIZE                       64
-
 enum hif_sl_session_key_alg {
-	HIF_SL_CURVE25519                                = 0x01,
-	HIF_SL_KDF                                       = 0x02
+	HIF_SL_CURVE25519 = 0x01,
+	HIF_SL_KDF        = 0x02
 };
 
+#define API_HOST_PUB_KEY_SIZE     32
+#define API_HOST_PUB_KEY_MAC_SIZE 64
+
 struct hif_req_sl_exchange_pub_keys {
-	u8    algorithm:2;
-	u8    reserved1:6;
-	u8    reserved2[3];
-	u8    host_pub_key[API_HOST_PUB_KEY_SIZE];
-	u8    host_pub_key_mac[API_HOST_PUB_KEY_MAC_SIZE];
+	u8     algorithm:2;
+	u8     reserved1:6;
+	u8     reserved2[3];
+	u8     host_pub_key[API_HOST_PUB_KEY_SIZE];
+	u8     host_pub_key_mac[API_HOST_PUB_KEY_MAC_SIZE];
 } __packed;
 
 struct hif_cnf_sl_exchange_pub_keys {
-	u32   status;
+	__le32 status;
 } __packed;
 
-#define API_NCP_PUB_KEY_SIZE                            32
-#define API_NCP_PUB_KEY_MAC_SIZE                        64
+#define API_NCP_PUB_KEY_SIZE      32
+#define API_NCP_PUB_KEY_MAC_SIZE  64
 
 struct hif_ind_sl_exchange_pub_keys {
-	u32   status;
-	u8    ncp_pub_key[API_NCP_PUB_KEY_SIZE];
-	u8    ncp_pub_key_mac[API_NCP_PUB_KEY_MAC_SIZE];
+	__le32 status;
+	u8     ncp_pub_key[API_NCP_PUB_KEY_SIZE];
+	u8     ncp_pub_key_mac[API_NCP_PUB_KEY_MAC_SIZE];
 } __packed;
 
-#define API_ENCR_BMP_SIZE        32
-
 struct hif_req_sl_configure {
-	u8    encr_bmp[API_ENCR_BMP_SIZE];
-	u8    disable_session_key_protection:1;
-	u8    reserved1:7;
-	u8    reserved2[3];
+	u8     encr_bmp[32];
+	u8     disable_session_key_protection:1;
+	u8     reserved1:7;
+	u8     reserved2[3];
 } __packed;
 
 struct hif_cnf_sl_configure {
-	u32 status;
-} __packed;
-
-struct hif_req_prevent_rollback {
-	u32   magic_word;
-} __packed;
-
-struct hif_cnf_prevent_rollback {
-	u32    status;
-} __packed;
-
-enum hif_pta_mode {
-	PTA_1W_WLAN_MASTER = 0,
-	PTA_1W_COEX_MASTER = 1,
-	PTA_2W             = 2,
-	PTA_3W             = 3,
-	PTA_4W             = 4
-};
-
-enum hif_signal_level {
-	SIGNAL_LOW  = 0,
-	SIGNAL_HIGH = 1
-};
-
-enum hif_coex_type {
-	COEX_TYPE_GENERIC = 0,
-	COEX_TYPE_BLE     = 1
-};
-
-enum hif_grant_state {
-	NO_GRANT = 0,
-	GRANT    = 1
-};
-
-struct hif_req_pta_settings {
-	u8 pta_mode;
-	u8 request_signal_active_level;
-	u8 priority_signal_active_level;
-	u8 freq_signal_active_level;
-	u8 grant_signal_active_level;
-	u8 coex_type;
-	u8 default_grant_state;
-	u8 simultaneous_rx_accesses;
-	u8 priority_sampling_time;
-	u8 tx_rx_sampling_time;
-	u8 freq_sampling_time;
-	u8 grant_valid_time;
-	u8 fem_control_time;
-	u8 first_slot_time;
-	u16 periodic_tx_rx_sampling_time;
-	u16 coex_quota;
-	u16 wlan_quota;
-} __packed;
-
-struct hif_cnf_pta_settings {
-	u32 status;
-} __packed;
-
-enum hif_pta_priority {
-	HIF_PTA_PRIORITY_COEX_MAXIMIZED = 0x00000562,
-	HIF_PTA_PRIORITY_COEX_HIGH      = 0x00000462,
-	HIF_PTA_PRIORITY_BALANCED       = 0x00001461,
-	HIF_PTA_PRIORITY_WLAN_HIGH      = 0x00001851,
-	HIF_PTA_PRIORITY_WLAN_MAXIMIZED = 0x00001A51
-};
-
-struct hif_req_pta_priority {
-	u32 priority;
-} __packed;
-
-struct hif_cnf_pta_priority {
-	u32 status;
-} __packed;
-
-enum hif_pta_state {
-	PTA_OFF = 0,
-	PTA_ON  = 1
-};
-
-struct hif_req_pta_state {
-	u32 pta_state;
-} __packed;
-
-struct hif_cnf_pta_state {
-	u32 status;
+	__le32 status;
 } __packed;
 
 #endif
diff --git a/drivers/staging/wfx/hif_api_mib.h b/drivers/staging/wfx/hif_api_mib.h
index 0c67cd4c1593..6f1434795fa8 100644
--- a/drivers/staging/wfx/hif_api_mib.h
+++ b/drivers/staging/wfx/hif_api_mib.h
@@ -10,175 +10,88 @@
 
 #include "hif_api_general.h"
 
-#define HIF_API_IPV4_ADDRESS_SIZE                       4
-#define HIF_API_IPV6_ADDRESS_SIZE                       16
+#define HIF_API_IPV4_ADDRESS_SIZE 4
+#define HIF_API_IPV6_ADDRESS_SIZE 16
 
 enum hif_mib_ids {
-	HIF_MIB_ID_GL_OPERATIONAL_POWER_MODE       = 0x2000,
-	HIF_MIB_ID_GL_BLOCK_ACK_INFO               = 0x2001,
-	HIF_MIB_ID_GL_SET_MULTI_MSG                = 0x2002,
-	HIF_MIB_ID_CCA_CONFIG                      = 0x2003,
-	HIF_MIB_ID_ETHERTYPE_DATAFRAME_CONDITION   = 0x2010,
-	HIF_MIB_ID_PORT_DATAFRAME_CONDITION        = 0x2011,
-	HIF_MIB_ID_MAGIC_DATAFRAME_CONDITION       = 0x2012,
-	HIF_MIB_ID_MAC_ADDR_DATAFRAME_CONDITION    = 0x2013,
-	HIF_MIB_ID_IPV4_ADDR_DATAFRAME_CONDITION   = 0x2014,
-	HIF_MIB_ID_IPV6_ADDR_DATAFRAME_CONDITION   = 0x2015,
-	HIF_MIB_ID_UC_MC_BC_DATAFRAME_CONDITION    = 0x2016,
-	HIF_MIB_ID_CONFIG_DATA_FILTER              = 0x2017,
-	HIF_MIB_ID_SET_DATA_FILTERING              = 0x2018,
-	HIF_MIB_ID_ARP_IP_ADDRESSES_TABLE          = 0x2019,
-	HIF_MIB_ID_NS_IP_ADDRESSES_TABLE           = 0x201A,
-	HIF_MIB_ID_RX_FILTER                       = 0x201B,
-	HIF_MIB_ID_BEACON_FILTER_TABLE             = 0x201C,
-	HIF_MIB_ID_BEACON_FILTER_ENABLE            = 0x201D,
-	HIF_MIB_ID_GRP_SEQ_COUNTER                 = 0x2030,
-	HIF_MIB_ID_TSF_COUNTER                     = 0x2031,
-	HIF_MIB_ID_STATISTICS_TABLE                = 0x2032,
-	HIF_MIB_ID_COUNTERS_TABLE                  = 0x2033,
-	HIF_MIB_ID_MAX_TX_POWER_LEVEL              = 0x2034,
-	HIF_MIB_ID_EXTENDED_COUNTERS_TABLE         = 0x2035,
-	HIF_MIB_ID_DOT11_MAC_ADDRESS               = 0x2040,
+	HIF_MIB_ID_GL_OPERATIONAL_POWER_MODE        = 0x2000,
+	HIF_MIB_ID_GL_BLOCK_ACK_INFO                = 0x2001,
+	HIF_MIB_ID_GL_SET_MULTI_MSG                 = 0x2002,
+	HIF_MIB_ID_CCA_CONFIG                       = 0x2003,
+	HIF_MIB_ID_ETHERTYPE_DATAFRAME_CONDITION    = 0x2010,
+	HIF_MIB_ID_PORT_DATAFRAME_CONDITION         = 0x2011,
+	HIF_MIB_ID_MAGIC_DATAFRAME_CONDITION        = 0x2012,
+	HIF_MIB_ID_MAC_ADDR_DATAFRAME_CONDITION     = 0x2013,
+	HIF_MIB_ID_IPV4_ADDR_DATAFRAME_CONDITION    = 0x2014,
+	HIF_MIB_ID_IPV6_ADDR_DATAFRAME_CONDITION    = 0x2015,
+	HIF_MIB_ID_UC_MC_BC_DATAFRAME_CONDITION     = 0x2016,
+	HIF_MIB_ID_CONFIG_DATA_FILTER               = 0x2017,
+	HIF_MIB_ID_SET_DATA_FILTERING               = 0x2018,
+	HIF_MIB_ID_ARP_IP_ADDRESSES_TABLE           = 0x2019,
+	HIF_MIB_ID_NS_IP_ADDRESSES_TABLE            = 0x201A,
+	HIF_MIB_ID_RX_FILTER                        = 0x201B,
+	HIF_MIB_ID_BEACON_FILTER_TABLE              = 0x201C,
+	HIF_MIB_ID_BEACON_FILTER_ENABLE             = 0x201D,
+	HIF_MIB_ID_GRP_SEQ_COUNTER                  = 0x2030,
+	HIF_MIB_ID_TSF_COUNTER                      = 0x2031,
+	HIF_MIB_ID_STATISTICS_TABLE                 = 0x2032,
+	HIF_MIB_ID_COUNTERS_TABLE                   = 0x2033,
+	HIF_MIB_ID_MAX_TX_POWER_LEVEL               = 0x2034,
+	HIF_MIB_ID_EXTENDED_COUNTERS_TABLE          = 0x2035,
+	HIF_MIB_ID_DOT11_MAC_ADDRESS                = 0x2040,
 	HIF_MIB_ID_DOT11_MAX_TRANSMIT_MSDU_LIFETIME = 0x2041,
-	HIF_MIB_ID_DOT11_MAX_RECEIVE_LIFETIME      = 0x2042,
-	HIF_MIB_ID_DOT11_WEP_DEFAULT_KEY_ID        = 0x2043,
-	HIF_MIB_ID_DOT11_RTS_THRESHOLD             = 0x2044,
-	HIF_MIB_ID_SLOT_TIME                       = 0x2045,
-	HIF_MIB_ID_CURRENT_TX_POWER_LEVEL          = 0x2046,
-	HIF_MIB_ID_NON_ERP_PROTECTION              = 0x2047,
-	HIF_MIB_ID_TEMPLATE_FRAME                  = 0x2048,
-	HIF_MIB_ID_BEACON_WAKEUP_PERIOD            = 0x2049,
-	HIF_MIB_ID_RCPI_RSSI_THRESHOLD             = 0x204A,
-	HIF_MIB_ID_BLOCK_ACK_POLICY                = 0x204B,
-	HIF_MIB_ID_OVERRIDE_INTERNAL_TX_RATE       = 0x204C,
-	HIF_MIB_ID_SET_ASSOCIATION_MODE            = 0x204D,
-	HIF_MIB_ID_SET_UAPSD_INFORMATION           = 0x204E,
-	HIF_MIB_ID_SET_TX_RATE_RETRY_POLICY        = 0x204F,
-	HIF_MIB_ID_PROTECTED_MGMT_POLICY           = 0x2050,
-	HIF_MIB_ID_SET_HT_PROTECTION               = 0x2051,
-	HIF_MIB_ID_KEEP_ALIVE_PERIOD               = 0x2052,
-	HIF_MIB_ID_ARP_KEEP_ALIVE_PERIOD           = 0x2053,
-	HIF_MIB_ID_INACTIVITY_TIMER                = 0x2054,
-	HIF_MIB_ID_INTERFACE_PROTECTION            = 0x2055,
-	HIF_MIB_ID_BEACON_STATS                    = 0x2056,
+	HIF_MIB_ID_DOT11_MAX_RECEIVE_LIFETIME       = 0x2042,
+	HIF_MIB_ID_DOT11_WEP_DEFAULT_KEY_ID         = 0x2043,
+	HIF_MIB_ID_DOT11_RTS_THRESHOLD              = 0x2044,
+	HIF_MIB_ID_SLOT_TIME                        = 0x2045,
+	HIF_MIB_ID_CURRENT_TX_POWER_LEVEL           = 0x2046,
+	HIF_MIB_ID_NON_ERP_PROTECTION               = 0x2047,
+	HIF_MIB_ID_TEMPLATE_FRAME                   = 0x2048,
+	HIF_MIB_ID_BEACON_WAKEUP_PERIOD             = 0x2049,
+	HIF_MIB_ID_RCPI_RSSI_THRESHOLD              = 0x204A,
+	HIF_MIB_ID_BLOCK_ACK_POLICY                 = 0x204B,
+	HIF_MIB_ID_OVERRIDE_INTERNAL_TX_RATE        = 0x204C,
+	HIF_MIB_ID_SET_ASSOCIATION_MODE             = 0x204D,
+	HIF_MIB_ID_SET_UAPSD_INFORMATION            = 0x204E,
+	HIF_MIB_ID_SET_TX_RATE_RETRY_POLICY         = 0x204F,
+	HIF_MIB_ID_PROTECTED_MGMT_POLICY            = 0x2050,
+	HIF_MIB_ID_SET_HT_PROTECTION                = 0x2051,
+	HIF_MIB_ID_KEEP_ALIVE_PERIOD                = 0x2052,
+	HIF_MIB_ID_ARP_KEEP_ALIVE_PERIOD            = 0x2053,
+	HIF_MIB_ID_INACTIVITY_TIMER                 = 0x2054,
+	HIF_MIB_ID_INTERFACE_PROTECTION             = 0x2055,
+	HIF_MIB_ID_BEACON_STATS                     = 0x2056,
 };
 
-#define HIF_OP_POWER_MODE_MASK                     0xf
-
 enum hif_op_power_mode {
-	HIF_OP_POWER_MODE_ACTIVE                   = 0x0,
-	HIF_OP_POWER_MODE_DOZE                     = 0x1,
-	HIF_OP_POWER_MODE_QUIESCENT                = 0x2
+	HIF_OP_POWER_MODE_ACTIVE    = 0x0,
+	HIF_OP_POWER_MODE_DOZE      = 0x1,
+	HIF_OP_POWER_MODE_QUIESCENT = 0x2
 };
 
 struct hif_mib_gl_operational_power_mode {
-	u8    power_mode:4;
-	u8    reserved1:3;
-	u8    wup_ind_activation:1;
-	u8    reserved2[3];
-} __packed;
-
-struct hif_mib_gl_block_ack_info {
-	u8    rx_buffer_size;
-	u8    rx_max_num_agreements;
-	u8    tx_buffer_size;
-	u8    tx_max_num_agreements;
+	u8     power_mode:4;
+	u8     reserved1:3;
+	u8     wup_ind_activation:1;
+	u8     reserved2[3];
 } __packed;
 
 struct hif_mib_gl_set_multi_msg {
-	u8    enable_multi_tx_conf:1;
-	u8    reserved1:7;
-	u8    reserved2[3];
-} __packed;
-
-enum hif_cca_thr_mode {
-	HIF_CCA_THR_MODE_RELATIVE = 0x0,
-	HIF_CCA_THR_MODE_ABSOLUTE = 0x1
-};
-
-struct hif_mib_gl_cca_config {
-	u8  cca_thr_mode;
-	u8  reserved[3];
-} __packed;
-
-#define MAX_NUMBER_DATA_FILTERS             0xA
-
-#define MAX_NUMBER_IPV4_ADDR_CONDITIONS     0x4
-#define MAX_NUMBER_IPV6_ADDR_CONDITIONS     0x4
-#define MAX_NUMBER_MAC_ADDR_CONDITIONS      0x4
-#define MAX_NUMBER_UC_MC_BC_CONDITIONS      0x4
-#define MAX_NUMBER_ETHER_TYPE_CONDITIONS    0x4
-#define MAX_NUMBER_PORT_CONDITIONS          0x4
-#define MAX_NUMBER_MAGIC_CONDITIONS         0x4
-#define MAX_NUMBER_ARP_CONDITIONS           0x2
-#define MAX_NUMBER_NS_CONDITIONS            0x2
-
-struct hif_mib_ethertype_data_frame_condition {
-	u8    condition_idx;
-	u8    reserved;
-	u16   ether_type;
-} __packed;
-
-enum hif_udp_tcp_protocol {
-	HIF_PROTOCOL_UDP                       = 0x0,
-	HIF_PROTOCOL_TCP                       = 0x1,
-	HIF_PROTOCOL_BOTH_UDP_TCP              = 0x2
-};
-
-enum hif_which_port {
-	HIF_PORT_DST                           = 0x0,
-	HIF_PORT_SRC                           = 0x1,
-	HIF_PORT_SRC_OR_DST                    = 0x2
-};
-
-struct hif_mib_ports_data_frame_condition {
-	u8    condition_idx;
-	u8    protocol;
-	u8    which_port;
-	u8    reserved1;
-	u16   port_number;
-	u8    reserved2[2];
-} __packed;
-
-#define HIF_API_MAGIC_PATTERN_SIZE                 32
-
-struct hif_mib_magic_data_frame_condition {
-	u8    condition_idx;
-	u8    offset;
-	u8    magic_pattern_length;
-	u8    reserved;
-	u8    magic_pattern[HIF_API_MAGIC_PATTERN_SIZE];
+	u8     enable_multi_tx_conf:1;
+	u8     reserved1:7;
+	u8     reserved2[3];
 } __packed;
 
 enum hif_mac_addr_type {
-	HIF_MAC_ADDR_A1                            = 0x0,
-	HIF_MAC_ADDR_A2                            = 0x1,
-	HIF_MAC_ADDR_A3                            = 0x2
+	HIF_MAC_ADDR_A1 = 0x0,
+	HIF_MAC_ADDR_A2 = 0x1,
+	HIF_MAC_ADDR_A3 = 0x2
 };
 
 struct hif_mib_mac_addr_data_frame_condition {
-	u8    condition_idx;
-	u8    address_type;
-	u8    mac_address[ETH_ALEN];
-} __packed;
-
-enum hif_ip_addr_mode {
-	HIF_IP_ADDR_SRC                            = 0x0,
-	HIF_IP_ADDR_DST                            = 0x1
-};
-
-struct hif_mib_ipv4_addr_data_frame_condition {
-	u8    condition_idx;
-	u8    address_mode;
-	u8    reserved[2];
-	u8    i_pv4_address[HIF_API_IPV4_ADDRESS_SIZE];
-} __packed;
-
-struct hif_mib_ipv6_addr_data_frame_condition {
-	u8    condition_idx;
-	u8    address_mode;
-	u8    reserved[2];
-	u8    i_pv6_address[HIF_API_IPV6_ADDRESS_SIZE];
+	u8     condition_idx;
+	u8     address_type;
+	u8     mac_address[ETH_ALEN];
 } __packed;
 
 #define HIF_FILTER_UNICAST   0x1
@@ -186,365 +99,289 @@ struct hif_mib_ipv6_addr_data_frame_condition {
 #define HIF_FILTER_BROADCAST 0x4
 
 struct hif_mib_uc_mc_bc_data_frame_condition {
-	u8    condition_idx;
-	u8    allowed_frames;
-	u8    reserved[2];
+	u8     condition_idx;
+	u8     allowed_frames;
+	u8     reserved[2];
 } __packed;
 
 struct hif_mib_config_data_filter {
-	u8    filter_idx;
-	u8    enable;
-	u8    reserved1[2];
-	u8    eth_type_cond;
-	u8    port_cond;
-	u8    magic_cond;
-	u8    mac_cond;
-	u8    ipv4_cond;
-	u8    ipv6_cond;
-	u8    uc_mc_bc_cond;
-	u8    reserved2;
+	u8     filter_idx;
+	u8     enable;
+	u8     reserved1[2];
+	u8     eth_type_cond;
+	u8     port_cond;
+	u8     magic_cond;
+	u8     mac_cond;
+	u8     ipv4_cond;
+	u8     ipv6_cond;
+	u8     uc_mc_bc_cond;
+	u8     reserved2;
 } __packed;
 
 struct hif_mib_set_data_filtering {
-	u8    invert_matching:1;
-	u8    reserved1:7;
-	u8    enable:1;
-	u8    reserved2:7;
-	u8    reserved3[2];
+	u8     invert_matching:1;
+	u8     reserved1:7;
+	u8     enable:1;
+	u8     reserved2:7;
+	u8     reserved3[2];
 } __packed;
 
 enum hif_arp_ns_frame_treatment {
-	HIF_ARP_NS_FILTERING_DISABLE                  = 0x0,
-	HIF_ARP_NS_FILTERING_ENABLE                   = 0x1,
-	HIF_ARP_NS_REPLY_ENABLE                       = 0x2
+	HIF_ARP_NS_FILTERING_DISABLE = 0x0,
+	HIF_ARP_NS_FILTERING_ENABLE  = 0x1,
+	HIF_ARP_NS_REPLY_ENABLE      = 0x2
 };
 
 struct hif_mib_arp_ip_addr_table {
-	u8    condition_idx;
-	u8    arp_enable;
-	u8    reserved[2];
-	u8    ipv4_address[HIF_API_IPV4_ADDRESS_SIZE];
-} __packed;
-
-struct hif_mib_ns_ip_addr_table {
-	u8    condition_idx;
-	u8    ns_enable;
-	u8    reserved[2];
-	u8    ipv6_address[HIF_API_IPV6_ADDRESS_SIZE];
+	u8     condition_idx;
+	u8     arp_enable;
+	u8     reserved[2];
+	u8     ipv4_address[HIF_API_IPV4_ADDRESS_SIZE];
 } __packed;
 
 struct hif_mib_rx_filter {
-	u8    reserved1:1;
-	u8    bssid_filter:1;
-	u8    reserved2:1;
-	u8    fwd_probe_req:1;
-	u8    keep_alive_filter:1;
-	u8    reserved3:3;
-	u8    reserved4[3];
+	u8     reserved1:1;
+	u8     bssid_filter:1;
+	u8     reserved2:1;
+	u8     fwd_probe_req:1;
+	u8     keep_alive_filter:1;
+	u8     reserved3:3;
+	u8     reserved4[3];
 } __packed;
 
-#define HIF_API_OUI_SIZE                                3
-#define HIF_API_MATCH_DATA_SIZE                         3
-
 struct hif_ie_table_entry {
-	u8    ie_id;
-	u8    has_changed:1;
-	u8    no_longer:1;
-	u8    has_appeared:1;
-	u8    reserved:1;
-	u8    num_match_data:4;
-	u8    oui[HIF_API_OUI_SIZE];
-	u8    match_data[HIF_API_MATCH_DATA_SIZE];
+	u8     ie_id;
+	u8     has_changed:1;
+	u8     no_longer:1;
+	u8     has_appeared:1;
+	u8     reserved:1;
+	u8     num_match_data:4;
+	u8     oui[3];
+	u8     match_data[3];
 } __packed;
 
 struct hif_mib_bcn_filter_table {
-	u32   num_of_info_elmts;
+	__le32 num_of_info_elmts;
 	struct hif_ie_table_entry ie_table[];
 } __packed;
 
 enum hif_beacon_filter {
-	HIF_BEACON_FILTER_DISABLE                  = 0x0,
-	HIF_BEACON_FILTER_ENABLE                   = 0x1,
-	HIF_BEACON_FILTER_AUTO_ERP                 = 0x2
+	HIF_BEACON_FILTER_DISABLE  = 0x0,
+	HIF_BEACON_FILTER_ENABLE   = 0x1,
+	HIF_BEACON_FILTER_AUTO_ERP = 0x2
 };
 
 struct hif_mib_bcn_filter_enable {
-	u32   enable;
-	u32   bcn_count;
-} __packed;
-
-struct hif_mib_group_seq_counter {
-	u32   bits4716;
-	u16   bits1500;
-	u16   reserved;
-} __packed;
-
-struct hif_mib_tsf_counter {
-	u32   tsf_counterlo;
-	u32   tsf_counterhi;
-} __packed;
-
-struct hif_mib_stats_table {
-	s16    latest_snr;
-	u8    latest_rcpi;
-	s8     latest_rssi;
+	__le32 enable;
+	__le32 bcn_count;
 } __packed;
 
 struct hif_mib_extended_count_table {
-	u32   count_plcp_errors;
-	u32   count_fcs_errors;
-	u32   count_tx_packets;
-	u32   count_rx_packets;
-	u32   count_rx_packet_errors;
-	u32   count_rx_decryption_failures;
-	u32   count_rx_mic_failures;
-	u32   count_rx_no_key_failures;
-	u32   count_tx_multicast_frames;
-	u32   count_tx_frames_success;
-	u32   count_tx_frame_failures;
-	u32   count_tx_frames_retried;
-	u32   count_tx_frames_multi_retried;
-	u32   count_rx_frame_duplicates;
-	u32   count_rts_success;
-	u32   count_rts_failures;
-	u32   count_ack_failures;
-	u32   count_rx_multicast_frames;
-	u32   count_rx_frames_success;
-	u32   count_rx_cmacicv_errors;
-	u32   count_rx_cmac_replays;
-	u32   count_rx_mgmt_ccmp_replays;
-	u32   count_rx_bipmic_errors;
-	u32   count_rx_beacon;
-	u32   count_miss_beacon;
-	u32   reserved[15];
+	__le32 count_plcp_errors;
+	__le32 count_fcs_errors;
+	__le32 count_tx_packets;
+	__le32 count_rx_packets;
+	__le32 count_rx_packet_errors;
+	__le32 count_rx_decryption_failures;
+	__le32 count_rx_mic_failures;
+	__le32 count_rx_no_key_failures;
+	__le32 count_tx_multicast_frames;
+	__le32 count_tx_frames_success;
+	__le32 count_tx_frame_failures;
+	__le32 count_tx_frames_retried;
+	__le32 count_tx_frames_multi_retried;
+	__le32 count_rx_frame_duplicates;
+	__le32 count_rts_success;
+	__le32 count_rts_failures;
+	__le32 count_ack_failures;
+	__le32 count_rx_multicast_frames;
+	__le32 count_rx_frames_success;
+	__le32 count_rx_cmacicv_errors;
+	__le32 count_rx_cmac_replays;
+	__le32 count_rx_mgmt_ccmp_replays;
+	__le32 count_rx_bipmic_errors;
+	__le32 count_rx_beacon;
+	__le32 count_miss_beacon;
+	__le32 reserved[15];
 } __packed;
 
 struct hif_mib_count_table {
-	u32   count_plcp_errors;
-	u32   count_fcs_errors;
-	u32   count_tx_packets;
-	u32   count_rx_packets;
-	u32   count_rx_packet_errors;
-	u32   count_rx_decryption_failures;
-	u32   count_rx_mic_failures;
-	u32   count_rx_no_key_failures;
-	u32   count_tx_multicast_frames;
-	u32   count_tx_frames_success;
-	u32   count_tx_frame_failures;
-	u32   count_tx_frames_retried;
-	u32   count_tx_frames_multi_retried;
-	u32   count_rx_frame_duplicates;
-	u32   count_rts_success;
-	u32   count_rts_failures;
-	u32   count_ack_failures;
-	u32   count_rx_multicast_frames;
-	u32   count_rx_frames_success;
-	u32   count_rx_cmacicv_errors;
-	u32   count_rx_cmac_replays;
-	u32   count_rx_mgmt_ccmp_replays;
-	u32   count_rx_bipmic_errors;
-} __packed;
-
-struct hif_mib_max_tx_power_level {
-	s32       max_tx_power_level_rf_port1;
-	s32       max_tx_power_level_rf_port2;
-} __packed;
-
-struct hif_mib_beacon_stats {
-	s32     latest_tbtt_diff;
-	u32    reserved[4];
+	__le32 count_plcp_errors;
+	__le32 count_fcs_errors;
+	__le32 count_tx_packets;
+	__le32 count_rx_packets;
+	__le32 count_rx_packet_errors;
+	__le32 count_rx_decryption_failures;
+	__le32 count_rx_mic_failures;
+	__le32 count_rx_no_key_failures;
+	__le32 count_tx_multicast_frames;
+	__le32 count_tx_frames_success;
+	__le32 count_tx_frame_failures;
+	__le32 count_tx_frames_retried;
+	__le32 count_tx_frames_multi_retried;
+	__le32 count_rx_frame_duplicates;
+	__le32 count_rts_success;
+	__le32 count_rts_failures;
+	__le32 count_ack_failures;
+	__le32 count_rx_multicast_frames;
+	__le32 count_rx_frames_success;
+	__le32 count_rx_cmacicv_errors;
+	__le32 count_rx_cmac_replays;
+	__le32 count_rx_mgmt_ccmp_replays;
+	__le32 count_rx_bipmic_errors;
 } __packed;
 
 struct hif_mib_mac_address {
-	u8    mac_addr[ETH_ALEN];
-	u16   reserved;
-} __packed;
-
-struct hif_mib_dot11_max_transmit_msdu_lifetime {
-	u32   max_life_time;
-} __packed;
-
-struct hif_mib_dot11_max_receive_lifetime {
-	u32   max_life_time;
+	u8     mac_addr[ETH_ALEN];
+	__le16 reserved;
 } __packed;
 
 struct hif_mib_wep_default_key_id {
-	u8    wep_default_key_id;
-	u8    reserved[3];
+	u8     wep_default_key_id;
+	u8     reserved[3];
 } __packed;
 
 struct hif_mib_dot11_rts_threshold {
-	u32   threshold;
+	__le32 threshold;
 } __packed;
 
 struct hif_mib_slot_time {
-	u32   slot_time;
+	__le32 slot_time;
 } __packed;
 
 struct hif_mib_current_tx_power_level {
-	s32   power_level;
+	__le32 power_level; // signed value
 } __packed;
 
 struct hif_mib_non_erp_protection {
-	u8   use_cts_to_self:1;
-	u8   reserved1:7;
-	u8   reserved2[3];
+	u8     use_cts_to_self:1;
+	u8     reserved1:7;
+	u8     reserved2[3];
 } __packed;
 
 enum hif_tmplt {
-	HIF_TMPLT_PRBREQ                           = 0x0,
-	HIF_TMPLT_BCN                              = 0x1,
-	HIF_TMPLT_NULL                             = 0x2,
-	HIF_TMPLT_QOSNUL                           = 0x3,
-	HIF_TMPLT_PSPOLL                           = 0x4,
-	HIF_TMPLT_PRBRES                           = 0x5,
-	HIF_TMPLT_ARP                              = 0x6,
-	HIF_TMPLT_NA                               = 0x7
+	HIF_TMPLT_PRBREQ = 0x0,
+	HIF_TMPLT_BCN    = 0x1,
+	HIF_TMPLT_NULL   = 0x2,
+	HIF_TMPLT_QOSNUL = 0x3,
+	HIF_TMPLT_PSPOLL = 0x4,
+	HIF_TMPLT_PRBRES = 0x5,
+	HIF_TMPLT_ARP    = 0x6,
+	HIF_TMPLT_NA     = 0x7
 };
 
-#define HIF_API_MAX_TEMPLATE_FRAME_SIZE                              700
+#define HIF_API_MAX_TEMPLATE_FRAME_SIZE 700
 
 struct hif_mib_template_frame {
-	u8    frame_type;
-	u8    init_rate:7;
-	u8    mode:1;
-	u16   frame_length;
-	u8    frame[HIF_API_MAX_TEMPLATE_FRAME_SIZE];
+	u8     frame_type;
+	u8     init_rate:7;
+	u8     mode:1;
+	__le16 frame_length;
+	u8     frame[];
 } __packed;
 
 struct hif_mib_beacon_wake_up_period {
-	u8    wakeup_period_min;
-	u8    receive_dtim:1;
-	u8    reserved1:7;
-	u8    wakeup_period_max;
-	u8    reserved2;
+	u8     wakeup_period_min;
+	u8     receive_dtim:1;
+	u8     reserved1:7;
+	u8     wakeup_period_max;
+	u8     reserved2;
 } __packed;
 
 struct hif_mib_rcpi_rssi_threshold {
-	u8    detection:1;
-	u8    rcpi_rssi:1;
-	u8    upperthresh:1;
-	u8    lowerthresh:1;
-	u8    reserved:4;
-	u8    lower_threshold;
-	u8    upper_threshold;
-	u8    rolling_average_count;
+	u8     detection:1;
+	u8     rcpi_rssi:1;
+	u8     upperthresh:1;
+	u8     lowerthresh:1;
+	u8     reserved:4;
+	u8     lower_threshold;
+	u8     upper_threshold;
+	u8     rolling_average_count;
 } __packed;
 
 #define DEFAULT_BA_MAX_RX_BUFFER_SIZE 16
 
 struct hif_mib_block_ack_policy {
-	u8    block_ack_tx_tid_policy;
-	u8    reserved1;
-	u8    block_ack_rx_tid_policy;
-	u8    block_ack_rx_max_buffer_size;
-} __packed;
-
-struct hif_mib_override_int_rate {
-	u8    internal_tx_rate;
-	u8    non_erp_internal_tx_rate;
-	u8    reserved[2];
+	u8     block_ack_tx_tid_policy;
+	u8     reserved1;
+	u8     block_ack_rx_tid_policy;
+	u8     block_ack_rx_max_buffer_size;
 } __packed;
 
 enum hif_mpdu_start_spacing {
-	HIF_MPDU_START_SPACING_NO_RESTRIC          = 0x0,
-	HIF_MPDU_START_SPACING_QUARTER             = 0x1,
-	HIF_MPDU_START_SPACING_HALF                = 0x2,
-	HIF_MPDU_START_SPACING_ONE                 = 0x3,
-	HIF_MPDU_START_SPACING_TWO                 = 0x4,
-	HIF_MPDU_START_SPACING_FOUR                = 0x5,
-	HIF_MPDU_START_SPACING_EIGHT               = 0x6,
-	HIF_MPDU_START_SPACING_SIXTEEN             = 0x7
+	HIF_MPDU_START_SPACING_NO_RESTRIC = 0x0,
+	HIF_MPDU_START_SPACING_QUARTER    = 0x1,
+	HIF_MPDU_START_SPACING_HALF       = 0x2,
+	HIF_MPDU_START_SPACING_ONE        = 0x3,
+	HIF_MPDU_START_SPACING_TWO        = 0x4,
+	HIF_MPDU_START_SPACING_FOUR       = 0x5,
+	HIF_MPDU_START_SPACING_EIGHT      = 0x6,
+	HIF_MPDU_START_SPACING_SIXTEEN    = 0x7
 };
 
 struct hif_mib_set_association_mode {
-	u8    preambtype_use:1;
-	u8    mode:1;
-	u8    rateset:1;
-	u8    spacing:1;
-	u8    reserved1:4;
-	u8    short_preamble:1;
-	u8    reserved2:7;
-	u8    greenfield:1;
-	u8    reserved3:7;
-	u8    mpdu_start_spacing;
-	u32   basic_rate_set;
+	u8     preambtype_use:1;
+	u8     mode:1;
+	u8     rateset:1;
+	u8     spacing:1;
+	u8     reserved1:4;
+	u8     short_preamble:1;
+	u8     reserved2:7;
+	u8     greenfield:1;
+	u8     reserved3:7;
+	u8     mpdu_start_spacing;
+	__le32 basic_rate_set;
 } __packed;
 
 struct hif_mib_set_uapsd_information {
-	u8    trig_bckgrnd:1;
-	u8    trig_be:1;
-	u8    trig_video:1;
-	u8    trig_voice:1;
-	u8    reserved1:4;
-	u8    deliv_bckgrnd:1;
-	u8    deliv_be:1;
-	u8    deliv_video:1;
-	u8    deliv_voice:1;
-	u8    reserved2:4;
-	u16   min_auto_trigger_interval;
-	u16   max_auto_trigger_interval;
-	u16   auto_trigger_step;
+	u8     trig_bckgrnd:1;
+	u8     trig_be:1;
+	u8     trig_video:1;
+	u8     trig_voice:1;
+	u8     reserved1:4;
+	u8     deliv_bckgrnd:1;
+	u8     deliv_be:1;
+	u8     deliv_video:1;
+	u8     deliv_voice:1;
+	u8     reserved2:4;
+	__le16 min_auto_trigger_interval;
+	__le16 max_auto_trigger_interval;
+	__le16 auto_trigger_step;
 } __packed;
 
 struct hif_mib_tx_rate_retry_policy {
-	u8    policy_index;
-	u8    short_retry_count;
-	u8    long_retry_count;
-	u8    first_rate_sel:2;
-	u8    terminate:1;
-	u8    count_init:1;
-	u8    reserved1:4;
-	u8    rate_recovery_count;
-	u8    reserved2[3];
-	u8    rates[12];
+	u8     policy_index;
+	u8     short_retry_count;
+	u8     long_retry_count;
+	u8     first_rate_sel:2;
+	u8     terminate:1;
+	u8     count_init:1;
+	u8     reserved1:4;
+	u8     rate_recovery_count;
+	u8     reserved2[3];
+	u8     rates[12];
 } __packed;
 
-#define HIF_MIB_NUM_TX_RATE_RETRY_POLICIES    15
+#define HIF_TX_RETRY_POLICY_MAX     15
+#define HIF_TX_RETRY_POLICY_INVALID HIF_TX_RETRY_POLICY_MAX
 
 struct hif_mib_set_tx_rate_retry_policy {
-	u8    num_tx_rate_policies;
-	u8    reserved[3];
+	u8     num_tx_rate_policies;
+	u8     reserved[3];
 	struct hif_mib_tx_rate_retry_policy tx_rate_retry_policy[];
 } __packed;
 
 struct hif_mib_protected_mgmt_policy {
-	u8   pmf_enable:1;
-	u8   unpmf_allowed:1;
-	u8   host_enc_auth_frames:1;
-	u8   reserved1:5;
-	u8   reserved2[3];
-} __packed;
-
-struct hif_mib_set_ht_protection {
-	u8   dual_cts_prot:1;
-	u8   reserved1:7;
-	u8   reserved2[3];
+	u8     pmf_enable:1;
+	u8     unpmf_allowed:1;
+	u8     host_enc_auth_frames:1;
+	u8     reserved1:5;
+	u8     reserved2[3];
 } __packed;
 
 struct hif_mib_keep_alive_period {
-	u16   keep_alive_period;
-	u8    reserved[2];
-} __packed;
-
-struct hif_mib_arp_keep_alive_period {
-	u16   arp_keep_alive_period;
-	u8    encr_type;
-	u8    reserved;
-	u8    sender_ipv4_address[HIF_API_IPV4_ADDRESS_SIZE];
-	u8    target_ipv4_address[HIF_API_IPV4_ADDRESS_SIZE];
-} __packed;
-
-struct hif_mib_inactivity_timer {
-	u8    min_active_time;
-	u8    max_active_time;
-	u16   reserved;
-} __packed;
-
-struct hif_mib_interface_protection {
-	u8   use_cts_prot:1;
-	u8   reserved1:7;
-	u8   reserved2[3];
+	__le16 keep_alive_period;
+	u8     reserved[2];
 } __packed;
 
 #endif
diff --git a/drivers/staging/wfx/hif_rx.c b/drivers/staging/wfx/hif_rx.c
index 33c22c5d629d..bb156033d1e1 100644
--- a/drivers/staging/wfx/hif_rx.c
+++ b/drivers/staging/wfx/hif_rx.c
@@ -22,9 +22,9 @@ static int hif_generic_confirm(struct wfx_dev *wdev,
 			       const struct hif_msg *hif, const void *buf)
 {
 	// All confirm messages start with status
-	int status = le32_to_cpu(*((__le32 *) buf));
+	int status = le32_to_cpup((__le32 *)buf);
 	int cmd = hif->id;
-	int len = hif->len - 4; // drop header
+	int len = le16_to_cpu(hif->len) - 4; // drop header
 
 	WARN(!mutex_is_locked(&wdev->hif_cmd.lock), "data locking error");
 
@@ -100,10 +100,10 @@ static int hif_startup_indication(struct wfx_dev *wdev,
 		return -EINVAL;
 	}
 	memcpy(&wdev->hw_caps, body, sizeof(struct hif_ind_startup));
-	le32_to_cpus(&wdev->hw_caps.status);
-	le16_to_cpus(&wdev->hw_caps.hardware_id);
-	le16_to_cpus(&wdev->hw_caps.num_inp_ch_bufs);
-	le16_to_cpus(&wdev->hw_caps.size_inp_ch_buf);
+	le16_to_cpus((__le16 *)&wdev->hw_caps.hardware_id);
+	le16_to_cpus((__le16 *)&wdev->hw_caps.num_inp_ch_bufs);
+	le16_to_cpus((__le16 *)&wdev->hw_caps.size_inp_ch_buf);
+	le32_to_cpus((__le32 *)&wdev->hw_caps.supported_rate_mask);
 
 	complete(&wdev->firmware_ready);
 	return 0;
@@ -127,7 +127,7 @@ static int hif_keys_indication(struct wfx_dev *wdev,
 	u8 pubkey[API_NCP_PUB_KEY_SIZE];
 
 	// SL_PUB_KEY_EXCHANGE_STATUS_SUCCESS is used by legacy secure link
-	if (body->status && body->status != SL_PUB_KEY_EXCHANGE_STATUS_SUCCESS)
+	if (body->status && body->status != HIF_STATUS_SLK_NEGO_SUCCESS)
 		dev_warn(wdev->dev, "secure link negociation error\n");
 	memcpy(pubkey, body->ncp_pub_key, sizeof(pubkey));
 	memreverse(pubkey, sizeof(pubkey));
@@ -158,26 +158,39 @@ static int hif_event_indication(struct wfx_dev *wdev,
 {
 	struct wfx_vif *wvif = wdev_to_wvif(wdev, hif->interface);
 	const struct hif_ind_event *body = buf;
-	struct wfx_hif_event *event;
-	int first;
+	int type = le32_to_cpu(body->event_id);
+	int cause;
 
-	WARN_ON(!wvif);
-	if (!wvif)
+	if (!wvif) {
+		dev_warn(wdev->dev, "received event for non-existent vif\n");
 		return 0;
+	}
 
-	event = kzalloc(sizeof(*event), GFP_KERNEL);
-	if (!event)
-		return -ENOMEM;
-
-	memcpy(&event->evt, body, sizeof(struct hif_ind_event));
-	spin_lock(&wvif->event_queue_lock);
-	first = list_empty(&wvif->event_queue);
-	list_add_tail(&event->link, &wvif->event_queue);
-	spin_unlock(&wvif->event_queue_lock);
-
-	if (first)
-		schedule_work(&wvif->event_handler_work);
-
+	switch (type) {
+	case HIF_EVENT_IND_RCPI_RSSI:
+		wfx_event_report_rssi(wvif, body->event_data.rcpi_rssi);
+		break;
+	case HIF_EVENT_IND_BSSLOST:
+		schedule_delayed_work(&wvif->beacon_loss_work, 0);
+		break;
+	case HIF_EVENT_IND_BSSREGAINED:
+		cancel_delayed_work(&wvif->beacon_loss_work);
+		dev_dbg(wdev->dev, "ignore BSSREGAINED indication\n");
+		break;
+	case HIF_EVENT_IND_PS_MODE_ERROR:
+		cause = le32_to_cpu(body->event_data.ps_mode_error);
+		dev_warn(wdev->dev, "error while processing power save request: %d\n",
+			 cause);
+		if (cause == HIF_PS_ERROR_AP_NOT_RESP_TO_POLL) {
+			wvif->bss_not_support_ps_poll = true;
+			schedule_work(&wvif->update_pm_work);
+		}
+		break;
+	default:
+		dev_warn(wdev->dev, "unhandled event indication: %.2x\n",
+			 type);
+		break;
+	}
 	return 0;
 }
 
@@ -224,53 +237,21 @@ static int hif_suspend_resume_indication(struct wfx_dev *wdev,
 	struct wfx_vif *wvif = wdev_to_wvif(wdev, hif->interface);
 	const struct hif_ind_suspend_resume_tx *body = buf;
 
-	WARN_ON(!wvif);
-	WARN(!body->suspend_resume_flags.bc_mc_only, "unsupported suspend/resume notification");
-	if (body->suspend_resume_flags.resume)
-		wfx_suspend_resume_mc(wvif, STA_NOTIFY_AWAKE);
-	else
-		wfx_suspend_resume_mc(wvif, STA_NOTIFY_SLEEP);
-
-	return 0;
-}
-
-static int hif_error_indication(struct wfx_dev *wdev,
-				const struct hif_msg *hif, const void *buf)
-{
-	const struct hif_ind_error *body = buf;
-	u8 *pRollback = (u8 *) body->data;
-	u32 *pStatus = (u32 *) body->data;
-
-	switch (body->type) {
-	case HIF_ERROR_FIRMWARE_ROLLBACK:
-		dev_err(wdev->dev,
-			"asynchronous error: firmware rollback error %d\n",
-			*pRollback);
-		break;
-	case HIF_ERROR_FIRMWARE_DEBUG_ENABLED:
-		dev_err(wdev->dev, "asynchronous error: firmware debug feature enabled\n");
-		break;
-	case HIF_ERROR_OUTDATED_SESSION_KEY:
-		dev_err(wdev->dev, "asynchronous error: secure link outdated key: %#.8x\n",
-			*pStatus);
-		break;
-	case HIF_ERROR_INVALID_SESSION_KEY:
-		dev_err(wdev->dev, "asynchronous error: invalid session key\n");
-		break;
-	case HIF_ERROR_OOR_VOLTAGE:
-		dev_err(wdev->dev, "asynchronous error: out-of-range overvoltage: %#.8x\n",
-			*pStatus);
-		break;
-	case HIF_ERROR_PDS_VERSION:
-		dev_err(wdev->dev,
-			"asynchronous error: wrong PDS payload or version: %#.8x\n",
-			*pStatus);
-		break;
-	default:
-		dev_err(wdev->dev, "asynchronous error: unknown (%d)\n",
-			body->type);
-		break;
+	if (body->suspend_resume_flags.bc_mc_only) {
+		WARN_ON(!wvif);
+		if (body->suspend_resume_flags.resume)
+			wfx_suspend_resume_mc(wvif, STA_NOTIFY_AWAKE);
+		else
+			wfx_suspend_resume_mc(wvif, STA_NOTIFY_SLEEP);
+	} else {
+		WARN(body->peer_sta_set, "misunderstood indication");
+		WARN(hif->interface != 2, "misunderstood indication");
+		if (body->suspend_resume_flags.resume)
+			wfx_suspend_hot_dev(wdev, STA_NOTIFY_AWAKE);
+		else
+			wfx_suspend_hot_dev(wdev, STA_NOTIFY_SLEEP);
 	}
+
 	return 0;
 }
 
@@ -278,13 +259,14 @@ static int hif_generic_indication(struct wfx_dev *wdev,
 				  const struct hif_msg *hif, const void *buf)
 {
 	const struct hif_ind_generic *body = buf;
+	int type = le32_to_cpu(body->indication_type);
 
-	switch (body->indication_type) {
+	switch (type) {
 	case HIF_GENERIC_INDICATION_TYPE_RAW:
 		return 0;
 	case HIF_GENERIC_INDICATION_TYPE_STRING:
 		dev_info(wdev->dev, "firmware says: %s\n",
-			 (char *) body->indication_data.raw_data);
+			 (char *)body->indication_data.raw_data);
 		return 0;
 	case HIF_GENERIC_INDICATION_TYPE_RX_STATS:
 		mutex_lock(&wdev->rx_stats_lock);
@@ -296,22 +278,103 @@ static int hif_generic_indication(struct wfx_dev *wdev,
 		       sizeof(wdev->rx_stats));
 		mutex_unlock(&wdev->rx_stats_lock);
 		return 0;
+	case HIF_GENERIC_INDICATION_TYPE_TX_POWER_LOOP_INFO:
+		mutex_lock(&wdev->tx_power_loop_info_lock);
+		memcpy(&wdev->tx_power_loop_info,
+		       &body->indication_data.tx_power_loop_info,
+		       sizeof(wdev->tx_power_loop_info));
+		mutex_unlock(&wdev->tx_power_loop_info_lock);
+		return 0;
 	default:
-		dev_err(wdev->dev,
-			"generic_indication: unknown indication type: %#.8x\n",
-			body->indication_type);
+		dev_err(wdev->dev, "generic_indication: unknown indication type: %#.8x\n",
+			type);
 		return -EIO;
 	}
 }
 
+static const struct {
+	int val;
+	const char *str;
+	bool has_param;
+} hif_errors[] = {
+	{ HIF_ERROR_FIRMWARE_ROLLBACK,
+		"rollback status" },
+	{ HIF_ERROR_FIRMWARE_DEBUG_ENABLED,
+		"debug feature enabled" },
+	{ HIF_ERROR_PDS_PAYLOAD,
+		"PDS version is not supported" },
+	{ HIF_ERROR_PDS_TESTFEATURE,
+		"PDS ask for an unknown test mode" },
+	{ HIF_ERROR_OOR_VOLTAGE,
+		"out-of-range power supply voltage", true },
+	{ HIF_ERROR_OOR_TEMPERATURE,
+		"out-of-range temperature", true },
+	{ HIF_ERROR_SLK_REQ_DURING_KEY_EXCHANGE,
+		"secure link does not expect request during key exchange" },
+	{ HIF_ERROR_SLK_SESSION_KEY,
+		"secure link session key is invalid" },
+	{ HIF_ERROR_SLK_OVERFLOW,
+		"secure link overflow" },
+	{ HIF_ERROR_SLK_WRONG_ENCRYPTION_STATE,
+		"secure link messages list does not match message encryption" },
+	{ HIF_ERROR_HIF_BUS_FREQUENCY_TOO_LOW,
+		"bus clock is too slow (<1kHz)" },
+	{ HIF_ERROR_HIF_RX_DATA_TOO_LARGE,
+		"HIF message too large" },
+	// Following errors only exists in old firmware versions:
+	{ HIF_ERROR_HIF_TX_QUEUE_FULL,
+		"HIF messages queue is full" },
+	{ HIF_ERROR_HIF_BUS,
+		"HIF bus" },
+	{ HIF_ERROR_SLK_MULTI_TX_UNSUPPORTED,
+		"secure link does not support multi-tx confirmations" },
+	{ HIF_ERROR_SLK_OUTDATED_SESSION_KEY,
+		"secure link session key is outdated" },
+	{ HIF_ERROR_SLK_DECRYPTION,
+		"secure link params (nonce or tag) mismatch" },
+};
+
+static int hif_error_indication(struct wfx_dev *wdev,
+				const struct hif_msg *hif, const void *buf)
+{
+	const struct hif_ind_error *body = buf;
+	int type = le32_to_cpu(body->type);
+	int param = (s8)body->data[0];
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(hif_errors); i++)
+		if (type == hif_errors[i].val)
+			break;
+	if (i < ARRAY_SIZE(hif_errors))
+		if (hif_errors[i].has_param)
+			dev_err(wdev->dev, "asynchronous error: %s: %d\n",
+				hif_errors[i].str, param);
+		else
+			dev_err(wdev->dev, "asynchronous error: %s\n",
+				hif_errors[i].str);
+	else
+		dev_err(wdev->dev, "asynchronous error: unknown: %08x\n", type);
+	print_hex_dump(KERN_INFO, "hif: ", DUMP_PREFIX_OFFSET,
+		       16, 1, hif, le16_to_cpu(hif->len), false);
+	wdev->chip_frozen = true;
+
+	return 0;
+};
+
 static int hif_exception_indication(struct wfx_dev *wdev,
 				    const struct hif_msg *hif, const void *buf)
 {
-	size_t len = hif->len - 4; // drop header
+	const struct hif_ind_exception *body = buf;
+	int type = le32_to_cpu(body->type);
 
-	dev_err(wdev->dev, "firmware exception\n");
-	print_hex_dump_bytes("Dump: ", DUMP_PREFIX_NONE, buf, len);
-	wdev->chip_frozen = 1;
+	if (type == 4)
+		dev_err(wdev->dev, "firmware assert %d\n",
+			le32_to_cpup((__le32 *)body->data));
+	else
+		dev_err(wdev->dev, "firmware exception\n");
+	print_hex_dump(KERN_INFO, "hif: ", DUMP_PREFIX_OFFSET,
+		       16, 1, hif, le16_to_cpu(hif->len), false);
+	wdev->chip_frozen = true;
 
 	return -1;
 }
diff --git a/drivers/staging/wfx/hif_tx.c b/drivers/staging/wfx/hif_tx.c
index 77bca43aca42..893b67f2f792 100644
--- a/drivers/staging/wfx/hif_tx.c
+++ b/drivers/staging/wfx/hif_tx.c
@@ -23,8 +23,8 @@ void wfx_init_hif_cmd(struct wfx_hif_cmd *hif_cmd)
 	mutex_init(&hif_cmd->key_renew_lock);
 }
 
-static void wfx_fill_header(struct hif_msg *hif, int if_id, unsigned int cmd,
-			    size_t size)
+static void wfx_fill_header(struct hif_msg *hif, int if_id,
+			    unsigned int cmd, size_t size)
 {
 	if (if_id == -1)
 		if_id = 2;
@@ -47,8 +47,8 @@ static void *wfx_alloc_hif(size_t body_len, struct hif_msg **hif)
 		return NULL;
 }
 
-int wfx_cmd_send(struct wfx_dev *wdev, struct hif_msg *request, void *reply,
-		 size_t reply_len, bool async)
+int wfx_cmd_send(struct wfx_dev *wdev, struct hif_msg *request,
+		 void *reply, size_t reply_len, bool async)
 {
 	const char *mib_name = "";
 	const char *mib_sep = "";
@@ -82,6 +82,9 @@ int wfx_cmd_send(struct wfx_dev *wdev, struct hif_msg *request, void *reply,
 	if (async)
 		return 0;
 
+	if (wdev->poll_irq)
+		wfx_bh_poll_irq(wdev);
+
 	ret = wait_for_completion_timeout(&wdev->hif_cmd.done, 1 * HZ);
 	if (!ret) {
 		dev_err(wdev->dev, "chip is abnormally long to answer\n");
@@ -91,7 +94,7 @@ int wfx_cmd_send(struct wfx_dev *wdev, struct hif_msg *request, void *reply,
 	if (!ret) {
 		dev_err(wdev->dev, "chip did not answer\n");
 		wfx_pending_dump_old_frames(wdev, 3000);
-		wdev->chip_frozen = 1;
+		wdev->chip_frozen = true;
 		reinit_completion(&wdev->hif_cmd.done);
 		ret = -ETIMEDOUT;
 	} else {
@@ -103,7 +106,7 @@ int wfx_cmd_send(struct wfx_dev *wdev, struct hif_msg *request, void *reply,
 
 	if (ret &&
 	    (cmd == HIF_REQ_ID_READ_MIB || cmd == HIF_REQ_ID_WRITE_MIB)) {
-		mib_name = get_mib_name(((u16 *) request)[2]);
+		mib_name = get_mib_name(((u16 *)request)[2]);
 		mib_sep = "/";
 	}
 	if (ret < 0)
@@ -128,7 +131,11 @@ int hif_shutdown(struct wfx_dev *wdev)
 	int ret;
 	struct hif_msg *hif;
 
+	if (wdev->chip_frozen)
+		return 0;
 	wfx_alloc_hif(0, &hif);
+	if (!hif)
+		return -ENOMEM;
 	wfx_fill_header(hif, -1, HIF_REQ_ID_SHUT_DOWN, 0);
 	ret = wfx_cmd_send(wdev, hif, NULL, 0, true);
 	// After this command, chip won't reply. Be sure to give enough time to
@@ -152,6 +159,8 @@ int hif_configuration(struct wfx_dev *wdev, const u8 *conf, size_t len)
 	struct hif_msg *hif;
 	struct hif_req_configuration *body = wfx_alloc_hif(buf_len, &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	body->length = cpu_to_le16(len);
 	memcpy(body->pds_data, conf, len);
 	wfx_fill_header(hif, -1, HIF_REQ_ID_CONFIGURATION, buf_len);
@@ -166,6 +175,8 @@ int hif_reset(struct wfx_vif *wvif, bool reset_stat)
 	struct hif_msg *hif;
 	struct hif_req_reset *body = wfx_alloc_hif(sizeof(*body), &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	body->reset_flags.reset_stat = reset_stat;
 	wfx_fill_header(hif, wvif->id, HIF_REQ_ID_RESET, sizeof(*body));
 	ret = wfx_cmd_send(wvif->wdev, hif, NULL, 0, false);
@@ -173,8 +184,8 @@ int hif_reset(struct wfx_vif *wvif, bool reset_stat)
 	return ret;
 }
 
-int hif_read_mib(struct wfx_dev *wdev, int vif_id, u16 mib_id, void *val,
-		 size_t val_len)
+int hif_read_mib(struct wfx_dev *wdev, int vif_id, u16 mib_id,
+		 void *val, size_t val_len)
 {
 	int ret;
 	struct hif_msg *hif;
@@ -182,36 +193,43 @@ int hif_read_mib(struct wfx_dev *wdev, int vif_id, u16 mib_id, void *val,
 	struct hif_req_read_mib *body = wfx_alloc_hif(sizeof(*body), &hif);
 	struct hif_cnf_read_mib *reply = kmalloc(buf_len, GFP_KERNEL);
 
+	if (!body || !reply) {
+		ret = -ENOMEM;
+		goto out;
+	}
 	body->mib_id = cpu_to_le16(mib_id);
 	wfx_fill_header(hif, vif_id, HIF_REQ_ID_READ_MIB, sizeof(*body));
 	ret = wfx_cmd_send(wdev, hif, reply, buf_len, false);
 
-	if (!ret && mib_id != reply->mib_id) {
-		dev_warn(wdev->dev,
-			 "%s: confirmation mismatch request\n", __func__);
+	if (!ret && mib_id != le16_to_cpu(reply->mib_id)) {
+		dev_warn(wdev->dev, "%s: confirmation mismatch request\n",
+			 __func__);
 		ret = -EIO;
 	}
 	if (ret == -ENOMEM)
-		dev_err(wdev->dev,
-			"buffer is too small to receive %s (%zu < %d)\n",
-			get_mib_name(mib_id), val_len, reply->length);
+		dev_err(wdev->dev, "buffer is too small to receive %s (%zu < %d)\n",
+			get_mib_name(mib_id), val_len,
+			le16_to_cpu(reply->length));
 	if (!ret)
-		memcpy(val, &reply->mib_data, reply->length);
+		memcpy(val, &reply->mib_data, le16_to_cpu(reply->length));
 	else
 		memset(val, 0xFF, val_len);
+out:
 	kfree(hif);
 	kfree(reply);
 	return ret;
 }
 
-int hif_write_mib(struct wfx_dev *wdev, int vif_id, u16 mib_id, void *val,
-		  size_t val_len)
+int hif_write_mib(struct wfx_dev *wdev, int vif_id, u16 mib_id,
+		  void *val, size_t val_len)
 {
 	int ret;
 	struct hif_msg *hif;
 	int buf_len = sizeof(struct hif_req_write_mib) + val_len;
 	struct hif_req_write_mib *body = wfx_alloc_hif(buf_len, &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	body->mib_id = cpu_to_le16(mib_id);
 	body->length = cpu_to_le16(val_len);
 	memcpy(&body->mib_data, val, val_len);
@@ -236,6 +254,8 @@ int hif_scan(struct wfx_vif *wvif, struct cfg80211_scan_request *req,
 
 	compiletime_assert(IEEE80211_MAX_SSID_LEN == HIF_API_SSID_SIZE,
 			   "API inconsistency");
+	if (!hif)
+		return -ENOMEM;
 	for (i = 0; i < req->n_ssids; i++) {
 		memcpy(body->ssid_def[i].ssid, req->ssids[i].ssid,
 		       IEEE80211_MAX_SSID_LEN);
@@ -268,7 +288,7 @@ int hif_scan(struct wfx_vif *wvif, struct cfg80211_scan_request *req,
 	tmo_chan_bg = le32_to_cpu(body->max_channel_time) * USEC_PER_TU;
 	tmo_chan_fg = 512 * USEC_PER_TU + body->probe_delay;
 	tmo_chan_fg *= body->num_of_probe_requests;
-	tmo = chan_num * max(tmo_chan_bg, tmo_chan_fg);
+	tmo = chan_num * max(tmo_chan_bg, tmo_chan_fg) + 512 * USEC_PER_TU;
 
 	wfx_fill_header(hif, wvif->id, HIF_REQ_ID_START_SCAN, buf_len);
 	ret = wfx_cmd_send(wvif->wdev, hif, NULL, 0, false);
@@ -283,6 +303,8 @@ int hif_stop_scan(struct wfx_vif *wvif)
 	// body associated to HIF_REQ_ID_STOP_SCAN is empty
 	wfx_alloc_hif(0, &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	wfx_fill_header(hif, wvif->id, HIF_REQ_ID_STOP_SCAN, 0);
 	ret = wfx_cmd_send(wvif->wdev, hif, NULL, 0, false);
 	kfree(hif);
@@ -296,19 +318,24 @@ int hif_join(struct wfx_vif *wvif, const struct ieee80211_bss_conf *conf,
 	struct hif_msg *hif;
 	struct hif_req_join *body = wfx_alloc_hif(sizeof(*body), &hif);
 
+	WARN_ON(!conf->beacon_int);
 	WARN_ON(!conf->basic_rates);
+	WARN_ON(sizeof(body->ssid) < ssidlen);
+	WARN(!conf->ibss_joined && !ssidlen, "joining an unknown BSS");
+	if (!hif)
+		return -ENOMEM;
 	body->infrastructure_bss_mode = !conf->ibss_joined;
 	body->short_preamble = conf->use_short_preamble;
 	if (channel && channel->flags & IEEE80211_CHAN_NO_IR)
 		body->probe_for_join = 0;
 	else
 		body->probe_for_join = 1;
-	body->channel_number = cpu_to_le16(channel->hw_value);
+	body->channel_number = channel->hw_value;
 	body->beacon_interval = cpu_to_le32(conf->beacon_int);
 	body->basic_rate_set =
 		cpu_to_le32(wfx_rate_mask_to_hw(wvif->wdev, conf->basic_rates));
 	memcpy(body->bssid, conf->bssid, sizeof(body->bssid));
-	if (!conf->ibss_joined && ssid) {
+	if (ssid) {
 		body->ssid_length = cpu_to_le32(ssidlen);
 		memcpy(body->ssid, ssid, ssidlen);
 	}
@@ -318,17 +345,17 @@ int hif_join(struct wfx_vif *wvif, const struct ieee80211_bss_conf *conf,
 	return ret;
 }
 
-int hif_set_bss_params(struct wfx_vif *wvif,
-		       const struct hif_req_set_bss_params *arg)
+int hif_set_bss_params(struct wfx_vif *wvif, int aid, int beacon_lost_count)
 {
 	int ret;
 	struct hif_msg *hif;
-	struct hif_req_set_bss_params *body = wfx_alloc_hif(sizeof(*body),
-							    &hif);
+	struct hif_req_set_bss_params *body =
+		wfx_alloc_hif(sizeof(*body), &hif);
 
-	memcpy(body, arg, sizeof(*body));
-	cpu_to_le16s(&body->aid);
-	cpu_to_le32s(&body->operational_rate_set);
+	if (!hif)
+		return -ENOMEM;
+	body->aid = cpu_to_le16(aid);
+	body->beacon_lost_count = beacon_lost_count;
 	wfx_fill_header(hif, wvif->id, HIF_REQ_ID_SET_BSS_PARAMS,
 			sizeof(*body));
 	ret = wfx_cmd_send(wvif->wdev, hif, NULL, 0, false);
@@ -343,6 +370,8 @@ int hif_add_key(struct wfx_dev *wdev, const struct hif_req_add_key *arg)
 	// FIXME: only send necessary bits
 	struct hif_req_add_key *body = wfx_alloc_hif(sizeof(*body), &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	// FIXME: swap bytes as necessary in body
 	memcpy(body, arg, sizeof(*body));
 	if (wfx_api_older_than(wdev, 1, 5))
@@ -363,6 +392,8 @@ int hif_remove_key(struct wfx_dev *wdev, int idx)
 	struct hif_msg *hif;
 	struct hif_req_remove_key *body = wfx_alloc_hif(sizeof(*body), &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	body->entry_index = idx;
 	wfx_fill_header(hif, -1, HIF_REQ_ID_REMOVE_KEY, sizeof(*body));
 	ret = wfx_cmd_send(wdev, hif, NULL, 0, false);
@@ -382,6 +413,8 @@ int hif_set_edca_queue_params(struct wfx_vif *wvif, u16 queue,
 		return -ENOMEM;
 
 	WARN_ON(arg->aifs > 255);
+	if (!hif)
+		return -ENOMEM;
 	body->aifsn = arg->aifs;
 	body->cw_min = cpu_to_le16(arg->cw_min);
 	body->cw_max = cpu_to_le16(arg->cw_max);
@@ -408,6 +441,8 @@ int hif_set_pm(struct wfx_vif *wvif, bool ps, int dynamic_ps_timeout)
 	if (!body)
 		return -ENOMEM;
 
+	if (!hif)
+		return -ENOMEM;
 	if (ps) {
 		body->pm_mode.enter_psm = 1;
 		// Firmware does not support more than 128ms
@@ -428,9 +463,12 @@ int hif_start(struct wfx_vif *wvif, const struct ieee80211_bss_conf *conf,
 	struct hif_msg *hif;
 	struct hif_req_start *body = wfx_alloc_hif(sizeof(*body), &hif);
 
+	WARN_ON(!conf->beacon_int);
+	if (!hif)
+		return -ENOMEM;
 	body->dtim_period = conf->dtim_period;
 	body->short_preamble = conf->use_short_preamble;
-	body->channel_number = cpu_to_le16(channel->hw_value);
+	body->channel_number = channel->hw_value;
 	body->beacon_interval = cpu_to_le32(conf->beacon_int);
 	body->basic_rate_set =
 		cpu_to_le32(wfx_rate_mask_to_hw(wvif->wdev, conf->basic_rates));
@@ -449,6 +487,8 @@ int hif_beacon_transmit(struct wfx_vif *wvif, bool enable)
 	struct hif_req_beacon_transmit *body = wfx_alloc_hif(sizeof(*body),
 							     &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	body->enable_beaconing = enable ? 1 : 0;
 	wfx_fill_header(hif, wvif->id, HIF_REQ_ID_BEACON_TRANSMIT,
 			sizeof(*body));
@@ -463,9 +503,11 @@ int hif_map_link(struct wfx_vif *wvif, u8 *mac_addr, int flags, int sta_id)
 	struct hif_msg *hif;
 	struct hif_req_map_link *body = wfx_alloc_hif(sizeof(*body), &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	if (mac_addr)
 		ether_addr_copy(body->mac_addr, mac_addr);
-	body->map_link_flags = *(struct hif_map_link_flags *) &flags;
+	body->map_link_flags = *(struct hif_map_link_flags *)&flags;
 	body->peer_sta_id = sta_id;
 	wfx_fill_header(hif, wvif->id, HIF_REQ_ID_MAP_LINK, sizeof(*body));
 	ret = wfx_cmd_send(wvif->wdev, hif, NULL, 0, false);
@@ -480,6 +522,8 @@ int hif_update_ie_beacon(struct wfx_vif *wvif, const u8 *ies, size_t ies_len)
 	int buf_len = sizeof(struct hif_req_update_ie) + ies_len;
 	struct hif_req_update_ie *body = wfx_alloc_hif(buf_len, &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	body->ie_flags.beacon = 1;
 	body->num_ies = cpu_to_le16(1);
 	memcpy(body->ie, ies, ies_len);
@@ -489,14 +533,16 @@ int hif_update_ie_beacon(struct wfx_vif *wvif, const u8 *ies, size_t ies_len)
 	return ret;
 }
 
-int hif_sl_send_pub_keys(struct wfx_dev *wdev, const uint8_t *pubkey,
-			 const uint8_t *pubkey_hmac)
+int hif_sl_send_pub_keys(struct wfx_dev *wdev,
+			 const u8 *pubkey, const u8 *pubkey_hmac)
 {
 	int ret;
 	struct hif_msg *hif;
 	struct hif_req_sl_exchange_pub_keys *body = wfx_alloc_hif(sizeof(*body),
 								  &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	body->algorithm = HIF_SL_CURVE25519;
 	memcpy(body->host_pub_key, pubkey, sizeof(body->host_pub_key));
 	memcpy(body->host_pub_key_mac, pubkey_hmac,
@@ -506,7 +552,7 @@ int hif_sl_send_pub_keys(struct wfx_dev *wdev, const uint8_t *pubkey,
 	ret = wfx_cmd_send(wdev, hif, NULL, 0, false);
 	kfree(hif);
 	// Compatibility with legacy secure link
-	if (ret == SL_PUB_KEY_EXCHANGE_STATUS_SUCCESS)
+	if (ret == le32_to_cpu(HIF_STATUS_SLK_NEGO_SUCCESS))
 		ret = 0;
 	return ret;
 }
@@ -517,6 +563,8 @@ int hif_sl_config(struct wfx_dev *wdev, const unsigned long *bitmap)
 	struct hif_msg *hif;
 	struct hif_req_sl_configure *body = wfx_alloc_hif(sizeof(*body), &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	memcpy(body->encr_bmp, bitmap, sizeof(body->encr_bmp));
 	wfx_fill_header(hif, -1, HIF_REQ_ID_SL_CONFIGURE, sizeof(*body));
 	ret = wfx_cmd_send(wdev, hif, NULL, 0, false);
@@ -524,21 +572,22 @@ int hif_sl_config(struct wfx_dev *wdev, const unsigned long *bitmap)
 	return ret;
 }
 
-int hif_sl_set_mac_key(struct wfx_dev *wdev, const u8 *slk_key,
-		       int destination)
+int hif_sl_set_mac_key(struct wfx_dev *wdev, const u8 *slk_key, int destination)
 {
 	int ret;
 	struct hif_msg *hif;
 	struct hif_req_set_sl_mac_key *body = wfx_alloc_hif(sizeof(*body),
 							    &hif);
 
+	if (!hif)
+		return -ENOMEM;
 	memcpy(body->key_value, slk_key, sizeof(body->key_value));
 	body->otp_or_ram = destination;
 	wfx_fill_header(hif, -1, HIF_REQ_ID_SET_SL_MAC_KEY, sizeof(*body));
 	ret = wfx_cmd_send(wdev, hif, NULL, 0, false);
 	kfree(hif);
 	// Compatibility with legacy secure link
-	if (ret == SL_MAC_KEY_STATUS_SUCCESS)
+	if (ret == le32_to_cpu(HIF_STATUS_SLK_SET_KEY_SUCCESS))
 		ret = 0;
 	return ret;
 }
diff --git a/drivers/staging/wfx/hif_tx.h b/drivers/staging/wfx/hif_tx.h
index f8520a14c14c..e9eca9330178 100644
--- a/drivers/staging/wfx/hif_tx.h
+++ b/drivers/staging/wfx/hif_tx.h
@@ -10,12 +10,11 @@
 #ifndef WFX_HIF_TX_H
 #define WFX_HIF_TX_H
 
-#include "hif_api_cmd.h"
-
 struct ieee80211_channel;
 struct ieee80211_bss_conf;
 struct ieee80211_tx_queue_params;
 struct cfg80211_scan_request;
+struct hif_req_add_key;
 struct wfx_dev;
 struct wfx_vif;
 
@@ -48,8 +47,7 @@ int hif_stop_scan(struct wfx_vif *wvif);
 int hif_join(struct wfx_vif *wvif, const struct ieee80211_bss_conf *conf,
 	     struct ieee80211_channel *channel, const u8 *ssid, int ssidlen);
 int hif_set_pm(struct wfx_vif *wvif, bool ps, int dynamic_ps_timeout);
-int hif_set_bss_params(struct wfx_vif *wvif,
-		       const struct hif_req_set_bss_params *arg);
+int hif_set_bss_params(struct wfx_vif *wvif, int aid, int beacon_lost_count);
 int hif_add_key(struct wfx_dev *wdev, const struct hif_req_add_key *arg);
 int hif_remove_key(struct wfx_dev *wdev, int idx);
 int hif_set_edca_queue_params(struct wfx_vif *wvif, u16 queue,
@@ -59,8 +57,8 @@ int hif_start(struct wfx_vif *wvif, const struct ieee80211_bss_conf *conf,
 int hif_beacon_transmit(struct wfx_vif *wvif, bool enable);
 int hif_map_link(struct wfx_vif *wvif, u8 *mac_addr, int flags, int sta_id);
 int hif_update_ie_beacon(struct wfx_vif *wvif, const u8 *ies, size_t ies_len);
-int hif_sl_set_mac_key(struct wfx_dev *wdev, const u8 *slk_key,
-		       int destination);
+int hif_sl_set_mac_key(struct wfx_dev *wdev,
+		       const u8 *slk_key, int destination);
 int hif_sl_config(struct wfx_dev *wdev, const unsigned long *bitmap);
 int hif_sl_send_pub_keys(struct wfx_dev *wdev,
 			 const u8 *pubkey, const u8 *pubkey_hmac);
diff --git a/drivers/staging/wfx/hif_tx_mib.c b/drivers/staging/wfx/hif_tx_mib.c
new file mode 100644
index 000000000000..1689cb42acc0
--- /dev/null
+++ b/drivers/staging/wfx/hif_tx_mib.c
@@ -0,0 +1,386 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Implementation of host-to-chip MIBs of WFxxx Split Mac (WSM) API.
+ *
+ * Copyright (c) 2017-2019, Silicon Laboratories, Inc.
+ * Copyright (c) 2010, ST-Ericsson
+ * Copyright (C) 2010, ST-Ericsson SA
+ */
+
+#include <linux/etherdevice.h>
+
+#include "wfx.h"
+#include "hif_tx.h"
+#include "hif_tx_mib.h"
+#include "hif_api_mib.h"
+
+int hif_set_output_power(struct wfx_vif *wvif, int val)
+{
+	struct hif_mib_current_tx_power_level arg = {
+		.power_level = cpu_to_le32(val * 10),
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_CURRENT_TX_POWER_LEVEL,
+			     &arg, sizeof(arg));
+}
+
+int hif_set_beacon_wakeup_period(struct wfx_vif *wvif,
+				 unsigned int dtim_interval,
+				 unsigned int listen_interval)
+{
+	struct hif_mib_beacon_wake_up_period val = {
+		.wakeup_period_min = dtim_interval,
+		.receive_dtim = 0,
+		.wakeup_period_max = listen_interval,
+	};
+
+	if (dtim_interval > 0xFF || listen_interval > 0xFFFF)
+		return -EINVAL;
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_BEACON_WAKEUP_PERIOD,
+			     &val, sizeof(val));
+}
+
+int hif_set_rcpi_rssi_threshold(struct wfx_vif *wvif,
+				int rssi_thold, int rssi_hyst)
+{
+	struct hif_mib_rcpi_rssi_threshold arg = {
+		.rolling_average_count = 8,
+		.detection = 1,
+	};
+
+	if (!rssi_thold && !rssi_hyst) {
+		arg.upperthresh = 1;
+		arg.lowerthresh = 1;
+	} else {
+		arg.upper_threshold = rssi_thold + rssi_hyst;
+		arg.upper_threshold = (arg.upper_threshold + 110) * 2;
+		arg.lower_threshold = rssi_thold;
+		arg.lower_threshold = (arg.lower_threshold + 110) * 2;
+	}
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_RCPI_RSSI_THRESHOLD, &arg, sizeof(arg));
+}
+
+int hif_get_counters_table(struct wfx_dev *wdev, int vif_id,
+			   struct hif_mib_extended_count_table *arg)
+{
+	if (wfx_api_older_than(wdev, 1, 3)) {
+		// extended_count_table is wider than count_table
+		memset(arg, 0xFF, sizeof(*arg));
+		return hif_read_mib(wdev, vif_id, HIF_MIB_ID_COUNTERS_TABLE,
+				    arg, sizeof(struct hif_mib_count_table));
+	} else {
+		return hif_read_mib(wdev, vif_id,
+				    HIF_MIB_ID_EXTENDED_COUNTERS_TABLE, arg,
+				sizeof(struct hif_mib_extended_count_table));
+	}
+}
+
+int hif_set_macaddr(struct wfx_vif *wvif, u8 *mac)
+{
+	struct hif_mib_mac_address msg = { };
+
+	if (mac)
+		ether_addr_copy(msg.mac_addr, mac);
+	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_DOT11_MAC_ADDRESS,
+			     &msg, sizeof(msg));
+}
+
+int hif_set_rx_filter(struct wfx_vif *wvif,
+		      bool filter_bssid, bool filter_prbreq)
+{
+	struct hif_mib_rx_filter val = { };
+
+	if (filter_bssid)
+		val.bssid_filter = 1;
+	if (!filter_prbreq)
+		val.fwd_probe_req = 1;
+	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_RX_FILTER,
+			     &val, sizeof(val));
+}
+
+int hif_set_beacon_filter_table(struct wfx_vif *wvif, int tbl_len,
+				const struct hif_ie_table_entry *tbl)
+{
+	int ret;
+	struct hif_mib_bcn_filter_table *val;
+	int buf_len = struct_size(val, ie_table, tbl_len);
+
+	val = kzalloc(buf_len, GFP_KERNEL);
+	if (!val)
+		return -ENOMEM;
+	val->num_of_info_elmts = cpu_to_le32(tbl_len);
+	memcpy(val->ie_table, tbl, tbl_len * sizeof(*tbl));
+	ret = hif_write_mib(wvif->wdev, wvif->id,
+			    HIF_MIB_ID_BEACON_FILTER_TABLE, val, buf_len);
+	kfree(val);
+	return ret;
+}
+
+int hif_beacon_filter_control(struct wfx_vif *wvif,
+			      int enable, int beacon_count)
+{
+	struct hif_mib_bcn_filter_enable arg = {
+		.enable = cpu_to_le32(enable),
+		.bcn_count = cpu_to_le32(beacon_count),
+	};
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_BEACON_FILTER_ENABLE,
+			     &arg, sizeof(arg));
+}
+
+int hif_set_operational_mode(struct wfx_dev *wdev, enum hif_op_power_mode mode)
+{
+	struct hif_mib_gl_operational_power_mode val = {
+		.power_mode = mode,
+		.wup_ind_activation = 1,
+	};
+
+	return hif_write_mib(wdev, -1, HIF_MIB_ID_GL_OPERATIONAL_POWER_MODE,
+			     &val, sizeof(val));
+}
+
+int hif_set_template_frame(struct wfx_vif *wvif, struct sk_buff *skb,
+			   u8 frame_type, int init_rate)
+{
+	struct hif_mib_template_frame *arg;
+
+	WARN(skb->len > HIF_API_MAX_TEMPLATE_FRAME_SIZE, "frame is too big");
+	skb_push(skb, 4);
+	arg = (struct hif_mib_template_frame *)skb->data;
+	skb_pull(skb, 4);
+	arg->init_rate = init_rate;
+	arg->frame_type = frame_type;
+	arg->frame_length = cpu_to_le16(skb->len);
+	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_TEMPLATE_FRAME,
+			     arg, sizeof(*arg) + skb->len);
+}
+
+int hif_set_mfp(struct wfx_vif *wvif, bool capable, bool required)
+{
+	struct hif_mib_protected_mgmt_policy val = { };
+
+	WARN(required && !capable, "incoherent arguments");
+	if (capable) {
+		val.pmf_enable = 1;
+		val.host_enc_auth_frames = 1;
+	}
+	if (!required)
+		val.unpmf_allowed = 1;
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_PROTECTED_MGMT_POLICY,
+			     &val, sizeof(val));
+}
+
+int hif_set_block_ack_policy(struct wfx_vif *wvif,
+			     u8 tx_tid_policy, u8 rx_tid_policy)
+{
+	struct hif_mib_block_ack_policy val = {
+		.block_ack_tx_tid_policy = tx_tid_policy,
+		.block_ack_rx_tid_policy = rx_tid_policy,
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_BLOCK_ACK_POLICY,
+			     &val, sizeof(val));
+}
+
+int hif_set_association_mode(struct wfx_vif *wvif,
+			     struct ieee80211_bss_conf *info)
+{
+	struct ieee80211_sta *sta = NULL;
+	struct hif_mib_set_association_mode val = {
+		.preambtype_use = 1,
+		.mode = 1,
+		.spacing = 1,
+		.short_preamble = info->use_short_preamble,
+	};
+
+	rcu_read_lock(); // protect sta
+	if (info->bssid && !info->ibss_joined)
+		sta = ieee80211_find_sta(wvif->vif, info->bssid);
+
+	// FIXME: it is strange to not retrieve all information from bss_info
+	if (sta && sta->ht_cap.ht_supported) {
+		val.mpdu_start_spacing = sta->ht_cap.ampdu_density;
+		if (!(info->ht_operation_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT))
+			val.greenfield = !!(sta->ht_cap.cap & IEEE80211_HT_CAP_GRN_FLD);
+	}
+	rcu_read_unlock();
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_SET_ASSOCIATION_MODE, &val, sizeof(val));
+}
+
+int hif_set_tx_rate_retry_policy(struct wfx_vif *wvif,
+				 int policy_index, u8 *rates)
+{
+	struct hif_mib_set_tx_rate_retry_policy *arg;
+	size_t size = struct_size(arg, tx_rate_retry_policy, 1);
+	int ret;
+
+	arg = kzalloc(size, GFP_KERNEL);
+	if (!arg)
+		return -ENOMEM;
+	arg->num_tx_rate_policies = 1;
+	arg->tx_rate_retry_policy[0].policy_index = policy_index;
+	arg->tx_rate_retry_policy[0].short_retry_count = 255;
+	arg->tx_rate_retry_policy[0].long_retry_count = 255;
+	arg->tx_rate_retry_policy[0].first_rate_sel = 1;
+	arg->tx_rate_retry_policy[0].terminate = 1;
+	arg->tx_rate_retry_policy[0].count_init = 1;
+	memcpy(&arg->tx_rate_retry_policy[0].rates, rates,
+	       sizeof(arg->tx_rate_retry_policy[0].rates));
+	ret = hif_write_mib(wvif->wdev, wvif->id,
+			    HIF_MIB_ID_SET_TX_RATE_RETRY_POLICY, arg, size);
+	kfree(arg);
+	return ret;
+}
+
+int hif_set_mac_addr_condition(struct wfx_vif *wvif,
+			       int idx, const u8 *mac_addr)
+{
+	struct hif_mib_mac_addr_data_frame_condition val = {
+		.condition_idx = idx,
+		.address_type = HIF_MAC_ADDR_A1,
+	};
+
+	ether_addr_copy(val.mac_address, mac_addr);
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_MAC_ADDR_DATAFRAME_CONDITION,
+			     &val, sizeof(val));
+}
+
+int hif_set_uc_mc_bc_condition(struct wfx_vif *wvif, int idx, u8 allowed_frames)
+{
+	struct hif_mib_uc_mc_bc_data_frame_condition val = {
+		.condition_idx = idx,
+		.allowed_frames = allowed_frames,
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_UC_MC_BC_DATAFRAME_CONDITION,
+			     &val, sizeof(val));
+}
+
+int hif_set_config_data_filter(struct wfx_vif *wvif, bool enable, int idx,
+			       int mac_filters, int frames_types_filters)
+{
+	struct hif_mib_config_data_filter val = {
+		.enable = enable,
+		.filter_idx = idx,
+		.mac_cond = mac_filters,
+		.uc_mc_bc_cond = frames_types_filters,
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_CONFIG_DATA_FILTER, &val, sizeof(val));
+}
+
+int hif_set_data_filtering(struct wfx_vif *wvif, bool enable, bool invert)
+{
+	struct hif_mib_set_data_filtering val = {
+		.enable = enable,
+		.invert_matching = invert,
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_SET_DATA_FILTERING, &val, sizeof(val));
+}
+
+int hif_keep_alive_period(struct wfx_vif *wvif, int period)
+{
+	struct hif_mib_keep_alive_period arg = {
+		.keep_alive_period = cpu_to_le16(period),
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_KEEP_ALIVE_PERIOD,
+			     &arg, sizeof(arg));
+};
+
+int hif_set_arp_ipv4_filter(struct wfx_vif *wvif, int idx, __be32 *addr)
+{
+	struct hif_mib_arp_ip_addr_table arg = {
+		.condition_idx = idx,
+		.arp_enable = HIF_ARP_NS_FILTERING_DISABLE,
+	};
+
+	if (addr) {
+		// Caution: type of addr is __be32
+		memcpy(arg.ipv4_address, addr, sizeof(arg.ipv4_address));
+		arg.arp_enable = HIF_ARP_NS_FILTERING_ENABLE;
+	}
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_ARP_IP_ADDRESSES_TABLE,
+			     &arg, sizeof(arg));
+}
+
+int hif_use_multi_tx_conf(struct wfx_dev *wdev, bool enable)
+{
+	struct hif_mib_gl_set_multi_msg arg = {
+		.enable_multi_tx_conf = enable,
+	};
+
+	return hif_write_mib(wdev, -1, HIF_MIB_ID_GL_SET_MULTI_MSG,
+			     &arg, sizeof(arg));
+}
+
+int hif_set_uapsd_info(struct wfx_vif *wvif, unsigned long val)
+{
+	struct hif_mib_set_uapsd_information arg = { };
+
+	if (val & BIT(IEEE80211_AC_VO))
+		arg.trig_voice = 1;
+	if (val & BIT(IEEE80211_AC_VI))
+		arg.trig_video = 1;
+	if (val & BIT(IEEE80211_AC_BE))
+		arg.trig_be = 1;
+	if (val & BIT(IEEE80211_AC_BK))
+		arg.trig_bckgrnd = 1;
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_SET_UAPSD_INFORMATION,
+			     &arg, sizeof(arg));
+}
+
+int hif_erp_use_protection(struct wfx_vif *wvif, bool enable)
+{
+	struct hif_mib_non_erp_protection arg = {
+		.use_cts_to_self = enable,
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_NON_ERP_PROTECTION, &arg, sizeof(arg));
+}
+
+int hif_slot_time(struct wfx_vif *wvif, int val)
+{
+	struct hif_mib_slot_time arg = {
+		.slot_time = cpu_to_le32(val),
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_SLOT_TIME,
+			     &arg, sizeof(arg));
+}
+
+int hif_wep_default_key_id(struct wfx_vif *wvif, int val)
+{
+	struct hif_mib_wep_default_key_id arg = {
+		.wep_default_key_id = val,
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_DOT11_WEP_DEFAULT_KEY_ID,
+			     &arg, sizeof(arg));
+}
+
+int hif_rts_threshold(struct wfx_vif *wvif, int val)
+{
+	struct hif_mib_dot11_rts_threshold arg = {
+		.threshold = cpu_to_le32(val >= 0 ? val : 0xFFFF),
+	};
+
+	return hif_write_mib(wvif->wdev, wvif->id,
+			     HIF_MIB_ID_DOT11_RTS_THRESHOLD, &arg, sizeof(arg));
+}
diff --git a/drivers/staging/wfx/hif_tx_mib.h b/drivers/staging/wfx/hif_tx_mib.h
index 26b1406f9f6c..86683de7de7c 100644
--- a/drivers/staging/wfx/hif_tx_mib.h
+++ b/drivers/staging/wfx/hif_tx_mib.h
@@ -9,398 +9,48 @@
 #ifndef WFX_HIF_TX_MIB_H
 #define WFX_HIF_TX_MIB_H
 
-#include <linux/etherdevice.h>
-
-#include "wfx.h"
-#include "hif_tx.h"
-#include "hif_api_mib.h"
-
-static inline int hif_set_output_power(struct wfx_vif *wvif, int val)
-{
-	struct hif_mib_current_tx_power_level arg = {
-		.power_level = cpu_to_le32(val * 10),
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_CURRENT_TX_POWER_LEVEL,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_set_beacon_wakeup_period(struct wfx_vif *wvif,
-					       unsigned int dtim_interval,
-					       unsigned int listen_interval)
-{
-	struct hif_mib_beacon_wake_up_period val = {
-		.wakeup_period_min = dtim_interval,
-		.receive_dtim = 0,
-		.wakeup_period_max = cpu_to_le16(listen_interval),
-	};
-
-	if (dtim_interval > 0xFF || listen_interval > 0xFFFF)
-		return -EINVAL;
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_BEACON_WAKEUP_PERIOD,
-			     &val, sizeof(val));
-}
-
-static inline int hif_set_rcpi_rssi_threshold(struct wfx_vif *wvif,
-					      int rssi_thold, int rssi_hyst)
-{
-	struct hif_mib_rcpi_rssi_threshold arg = {
-		.rolling_average_count = 8,
-		.detection = 1,
-	};
-
-	if (!rssi_thold && !rssi_hyst) {
-		arg.upperthresh = 1;
-		arg.lowerthresh = 1;
-	} else {
-		arg.upper_threshold = rssi_thold + rssi_hyst;
-		arg.upper_threshold = (arg.upper_threshold + 110) * 2;
-		arg.lower_threshold = rssi_thold;
-		arg.lower_threshold = (arg.lower_threshold + 110) * 2;
-	}
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_RCPI_RSSI_THRESHOLD, &arg, sizeof(arg));
-}
-
-static inline int hif_get_counters_table(struct wfx_dev *wdev,
-					 struct hif_mib_extended_count_table *arg)
-{
-	if (wfx_api_older_than(wdev, 1, 3)) {
-		// extended_count_table is wider than count_table
-		memset(arg, 0xFF, sizeof(*arg));
-		return hif_read_mib(wdev, 0, HIF_MIB_ID_COUNTERS_TABLE,
-				    arg, sizeof(struct hif_mib_count_table));
-	} else {
-		return hif_read_mib(wdev, 0,
-				    HIF_MIB_ID_EXTENDED_COUNTERS_TABLE, arg,
-				sizeof(struct hif_mib_extended_count_table));
-	}
-}
-
-static inline int hif_set_macaddr(struct wfx_vif *wvif, u8 *mac)
-{
-	struct hif_mib_mac_address msg = { };
-
-	if (mac)
-		ether_addr_copy(msg.mac_addr, mac);
-	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_DOT11_MAC_ADDRESS,
-			     &msg, sizeof(msg));
-}
-
-static inline int hif_set_rx_filter(struct wfx_vif *wvif, bool filter_bssid,
-				    bool fwd_probe_req)
-{
-	struct hif_mib_rx_filter val = { };
-
-	if (filter_bssid)
-		val.bssid_filter = 1;
-	if (fwd_probe_req)
-		val.fwd_probe_req = 1;
-	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_RX_FILTER,
-			     &val, sizeof(val));
-}
-
-static inline int hif_set_beacon_filter_table(struct wfx_vif *wvif,
-					      int tbl_len,
-					      struct hif_ie_table_entry *tbl)
-{
-	int ret;
-	struct hif_mib_bcn_filter_table *val;
-	int buf_len = struct_size(val, ie_table, tbl_len);
-
-	val = kzalloc(buf_len, GFP_KERNEL);
-	if (!val)
-		return -ENOMEM;
-	val->num_of_info_elmts = cpu_to_le32(tbl_len);
-	memcpy(val->ie_table, tbl, tbl_len * sizeof(*tbl));
-	ret = hif_write_mib(wvif->wdev, wvif->id,
-			    HIF_MIB_ID_BEACON_FILTER_TABLE, val, buf_len);
-	kfree(val);
-	return ret;
-}
-
-static inline int hif_beacon_filter_control(struct wfx_vif *wvif,
-					    int enable, int beacon_count)
-{
-	struct hif_mib_bcn_filter_enable arg = {
-		.enable = cpu_to_le32(enable),
-		.bcn_count = cpu_to_le32(beacon_count),
-	};
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_BEACON_FILTER_ENABLE,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_set_operational_mode(struct wfx_dev *wdev,
-					   enum hif_op_power_mode mode)
-{
-	struct hif_mib_gl_operational_power_mode val = {
-		.power_mode = mode,
-		.wup_ind_activation = 1,
-	};
-
-	return hif_write_mib(wdev, -1, HIF_MIB_ID_GL_OPERATIONAL_POWER_MODE,
-			     &val, sizeof(val));
-}
-
-static inline int hif_set_template_frame(struct wfx_vif *wvif,
-					 struct sk_buff *skb,
-					 u8 frame_type, int init_rate)
-{
-	struct hif_mib_template_frame *arg;
-
-	skb_push(skb, 4);
-	arg = (struct hif_mib_template_frame *)skb->data;
-	skb_pull(skb, 4);
-	arg->init_rate = init_rate;
-	arg->frame_type = frame_type;
-	arg->frame_length = cpu_to_le16(skb->len);
-	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_TEMPLATE_FRAME,
-			     arg, sizeof(*arg));
-}
-
-static inline int hif_set_mfp(struct wfx_vif *wvif, bool capable, bool required)
-{
-	struct hif_mib_protected_mgmt_policy val = { };
-
-	WARN(required && !capable, "incoherent arguments");
-	if (capable) {
-		val.pmf_enable = 1;
-		val.host_enc_auth_frames = 1;
-	}
-	if (!required)
-		val.unpmf_allowed = 1;
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_PROTECTED_MGMT_POLICY,
-			     &val, sizeof(val));
-}
-
-static inline int hif_set_block_ack_policy(struct wfx_vif *wvif,
-					   u8 tx_tid_policy, u8 rx_tid_policy)
-{
-	struct hif_mib_block_ack_policy val = {
-		.block_ack_tx_tid_policy = tx_tid_policy,
-		.block_ack_rx_tid_policy = rx_tid_policy,
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_BLOCK_ACK_POLICY,
-			     &val, sizeof(val));
-}
-
-static inline int hif_set_association_mode(struct wfx_vif *wvif,
-					   struct ieee80211_bss_conf *info)
-{
-	int basic_rates = wfx_rate_mask_to_hw(wvif->wdev, info->basic_rates);
-	struct ieee80211_sta *sta = NULL;
-	struct hif_mib_set_association_mode val = {
-		.preambtype_use = 1,
-		.mode = 1,
-		.rateset = 1,
-		.spacing = 1,
-		.short_preamble = info->use_short_preamble,
-		.basic_rate_set = cpu_to_le32(basic_rates)
-	};
-
-	rcu_read_lock(); // protect sta
-	if (info->bssid && !info->ibss_joined)
-		sta = ieee80211_find_sta(wvif->vif, info->bssid);
-
-	// FIXME: it is strange to not retrieve all information from bss_info
-	if (sta && sta->ht_cap.ht_supported) {
-		val.mpdu_start_spacing = sta->ht_cap.ampdu_density;
-		if (!(info->ht_operation_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT))
-			val.greenfield = !!(sta->ht_cap.cap & IEEE80211_HT_CAP_GRN_FLD);
-	}
-	rcu_read_unlock();
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_SET_ASSOCIATION_MODE, &val, sizeof(val));
-}
-
-static inline int hif_set_tx_rate_retry_policy(struct wfx_vif *wvif,
-					       int policy_index, uint8_t *rates)
-{
-	struct hif_mib_set_tx_rate_retry_policy *arg;
-	size_t size = struct_size(arg, tx_rate_retry_policy, 1);
-	int ret;
-
-	arg = kzalloc(size, GFP_KERNEL);
-	arg->num_tx_rate_policies = 1;
-	arg->tx_rate_retry_policy[0].policy_index = policy_index;
-	arg->tx_rate_retry_policy[0].short_retry_count = 255;
-	arg->tx_rate_retry_policy[0].long_retry_count = 255;
-	arg->tx_rate_retry_policy[0].first_rate_sel = 1;
-	arg->tx_rate_retry_policy[0].terminate = 1;
-	arg->tx_rate_retry_policy[0].count_init = 1;
-	memcpy(&arg->tx_rate_retry_policy[0].rates, rates,
-	       sizeof(arg->tx_rate_retry_policy[0].rates));
-	ret = hif_write_mib(wvif->wdev, wvif->id,
-			    HIF_MIB_ID_SET_TX_RATE_RETRY_POLICY, arg, size);
-	kfree(arg);
-	return ret;
-}
-
-static inline int hif_set_mac_addr_condition(struct wfx_vif *wvif,
-					     int idx, const u8 *mac_addr)
-{
-	struct hif_mib_mac_addr_data_frame_condition val = {
-		.condition_idx = idx,
-		.address_type = HIF_MAC_ADDR_A1,
-	};
-
-	ether_addr_copy(val.mac_address, mac_addr);
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_MAC_ADDR_DATAFRAME_CONDITION,
-			     &val, sizeof(val));
-}
-
-static inline int hif_set_uc_mc_bc_condition(struct wfx_vif *wvif,
-					     int idx, u8 allowed_frames)
-{
-	struct hif_mib_uc_mc_bc_data_frame_condition val = {
-		.condition_idx = idx,
-		.allowed_frames = allowed_frames,
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_UC_MC_BC_DATAFRAME_CONDITION,
-			     &val, sizeof(val));
-}
-
-static inline int hif_set_config_data_filter(struct wfx_vif *wvif, bool enable,
-					     int idx, int mac_filters,
-					     int frames_types_filters)
-{
-	struct hif_mib_config_data_filter val = {
-		.enable = enable,
-		.filter_idx = idx,
-		.mac_cond = mac_filters,
-		.uc_mc_bc_cond = frames_types_filters,
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_CONFIG_DATA_FILTER, &val, sizeof(val));
-}
-
-static inline int hif_set_data_filtering(struct wfx_vif *wvif,
-					 bool enable, bool invert)
-{
-	struct hif_mib_set_data_filtering val = {
-		.enable = enable,
-		.invert_matching = invert,
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_SET_DATA_FILTERING, &val, sizeof(val));
-}
-
-static inline int hif_keep_alive_period(struct wfx_vif *wvif, int period)
-{
-	struct hif_mib_keep_alive_period arg = {
-		.keep_alive_period = cpu_to_le16(period),
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_KEEP_ALIVE_PERIOD,
-			     &arg, sizeof(arg));
-};
-
-static inline int hif_set_arp_ipv4_filter(struct wfx_vif *wvif, int idx,
-					  __be32 *addr)
-{
-	struct hif_mib_arp_ip_addr_table arg = {
-		.condition_idx = idx,
-		.arp_enable = HIF_ARP_NS_FILTERING_DISABLE,
-	};
-
-	if (addr) {
-		// Caution: type of addr is __be32
-		memcpy(arg.ipv4_address, addr, sizeof(arg.ipv4_address));
-		arg.arp_enable = HIF_ARP_NS_FILTERING_ENABLE;
-	}
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_ARP_IP_ADDRESSES_TABLE,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_use_multi_tx_conf(struct wfx_dev *wdev, bool enable)
-{
-	struct hif_mib_gl_set_multi_msg arg = {
-		.enable_multi_tx_conf = enable,
-	};
-
-	return hif_write_mib(wdev, -1, HIF_MIB_ID_GL_SET_MULTI_MSG,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_set_uapsd_info(struct wfx_vif *wvif, unsigned long val)
-{
-	struct hif_mib_set_uapsd_information arg = { };
-
-	if (val & BIT(IEEE80211_AC_VO))
-		arg.trig_voice = 1;
-	if (val & BIT(IEEE80211_AC_VI))
-		arg.trig_video = 1;
-	if (val & BIT(IEEE80211_AC_BE))
-		arg.trig_be = 1;
-	if (val & BIT(IEEE80211_AC_BK))
-		arg.trig_bckgrnd = 1;
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_SET_UAPSD_INFORMATION,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_erp_use_protection(struct wfx_vif *wvif, bool enable)
-{
-	struct hif_mib_non_erp_protection arg = {
-		.use_cts_to_self = enable,
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_NON_ERP_PROTECTION, &arg, sizeof(arg));
-}
-
-static inline int hif_slot_time(struct wfx_vif *wvif, int val)
-{
-	struct hif_mib_slot_time arg = {
-		.slot_time = cpu_to_le32(val),
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_SLOT_TIME,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_dual_cts_protection(struct wfx_vif *wvif, bool enable)
-{
-	struct hif_mib_set_ht_protection arg = {
-		.dual_cts_prot = enable,
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id, HIF_MIB_ID_SET_HT_PROTECTION,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_wep_default_key_id(struct wfx_vif *wvif, int val)
-{
-	struct hif_mib_wep_default_key_id arg = {
-		.wep_default_key_id = val,
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_DOT11_WEP_DEFAULT_KEY_ID,
-			     &arg, sizeof(arg));
-}
-
-static inline int hif_rts_threshold(struct wfx_vif *wvif, int val)
-{
-	struct hif_mib_dot11_rts_threshold arg = {
-		.threshold = cpu_to_le32(val >= 0 ? val : 0xFFFF),
-	};
-
-	return hif_write_mib(wvif->wdev, wvif->id,
-			     HIF_MIB_ID_DOT11_RTS_THRESHOLD, &arg, sizeof(arg));
-}
+struct wfx_vif;
+struct sk_buff;
+
+int hif_set_output_power(struct wfx_vif *wvif, int val);
+int hif_set_beacon_wakeup_period(struct wfx_vif *wvif,
+				 unsigned int dtim_interval,
+				 unsigned int listen_interval);
+int hif_set_rcpi_rssi_threshold(struct wfx_vif *wvif,
+				int rssi_thold, int rssi_hyst);
+int hif_get_counters_table(struct wfx_dev *wdev, int vif_id,
+			   struct hif_mib_extended_count_table *arg);
+int hif_set_macaddr(struct wfx_vif *wvif, u8 *mac);
+int hif_set_rx_filter(struct wfx_vif *wvif,
+		      bool filter_bssid, bool fwd_probe_req);
+int hif_set_beacon_filter_table(struct wfx_vif *wvif, int tbl_len,
+				const struct hif_ie_table_entry *tbl);
+int hif_beacon_filter_control(struct wfx_vif *wvif,
+			      int enable, int beacon_count);
+int hif_set_operational_mode(struct wfx_dev *wdev, enum hif_op_power_mode mode);
+int hif_set_template_frame(struct wfx_vif *wvif, struct sk_buff *skb,
+			   u8 frame_type, int init_rate);
+int hif_set_mfp(struct wfx_vif *wvif, bool capable, bool required);
+int hif_set_block_ack_policy(struct wfx_vif *wvif,
+			     u8 tx_tid_policy, u8 rx_tid_policy);
+int hif_set_association_mode(struct wfx_vif *wvif,
+			     struct ieee80211_bss_conf *info);
+int hif_set_tx_rate_retry_policy(struct wfx_vif *wvif,
+				 int policy_index, u8 *rates);
+int hif_set_mac_addr_condition(struct wfx_vif *wvif,
+			       int idx, const u8 *mac_addr);
+int hif_set_uc_mc_bc_condition(struct wfx_vif *wvif,
+			       int idx, u8 allowed_frames);
+int hif_set_config_data_filter(struct wfx_vif *wvif, bool enable, int idx,
+			       int mac_filters, int frames_types_filters);
+int hif_set_data_filtering(struct wfx_vif *wvif, bool enable, bool invert);
+int hif_keep_alive_period(struct wfx_vif *wvif, int period);
+int hif_set_arp_ipv4_filter(struct wfx_vif *wvif, int idx, __be32 *addr);
+int hif_use_multi_tx_conf(struct wfx_dev *wdev, bool enable);
+int hif_set_uapsd_info(struct wfx_vif *wvif, unsigned long val);
+int hif_erp_use_protection(struct wfx_vif *wvif, bool enable);
+int hif_slot_time(struct wfx_vif *wvif, int val);
+int hif_wep_default_key_id(struct wfx_vif *wvif, int val);
+int hif_rts_threshold(struct wfx_vif *wvif, int val);
 
 #endif
diff --git a/drivers/staging/wfx/hwio.c b/drivers/staging/wfx/hwio.c
index d3a141d95a0e..777217cdf9a7 100644
--- a/drivers/staging/wfx/hwio.c
+++ b/drivers/staging/wfx/hwio.c
@@ -106,8 +106,8 @@ err:
 	return ret;
 }
 
-static int indirect_read(struct wfx_dev *wdev, int reg, u32 addr, void *buf,
-			 size_t len)
+static int indirect_read(struct wfx_dev *wdev, int reg, u32 addr,
+			 void *buf, size_t len)
 {
 	int ret;
 	int i;
@@ -195,8 +195,8 @@ static int indirect_write_locked(struct wfx_dev *wdev, int reg, u32 addr,
 	return ret;
 }
 
-static int indirect_read32_locked(struct wfx_dev *wdev, int reg, u32 addr,
-				  u32 *val)
+static int indirect_read32_locked(struct wfx_dev *wdev, int reg,
+				  u32 addr, u32 *val)
 {
 	int ret;
 	__le32 *tmp = kmalloc(sizeof(u32), GFP_KERNEL);
@@ -205,15 +205,15 @@ static int indirect_read32_locked(struct wfx_dev *wdev, int reg, u32 addr,
 		return -ENOMEM;
 	wdev->hwbus_ops->lock(wdev->hwbus_priv);
 	ret = indirect_read(wdev, reg, addr, tmp, sizeof(u32));
-	*val = cpu_to_le32(*tmp);
+	*val = le32_to_cpu(*tmp);
 	_trace_io_ind_read32(reg, addr, *val);
 	wdev->hwbus_ops->unlock(wdev->hwbus_priv);
 	kfree(tmp);
 	return ret;
 }
 
-static int indirect_write32_locked(struct wfx_dev *wdev, int reg, u32 addr,
-				   u32 val)
+static int indirect_write32_locked(struct wfx_dev *wdev, int reg,
+				   u32 addr, u32 val)
 {
 	int ret;
 	__le32 *tmp = kmalloc(sizeof(u32), GFP_KERNEL);
@@ -233,7 +233,7 @@ int wfx_data_read(struct wfx_dev *wdev, void *buf, size_t len)
 {
 	int ret;
 
-	WARN((long) buf & 3, "%s: unaligned buffer", __func__);
+	WARN((long)buf & 3, "%s: unaligned buffer", __func__);
 	wdev->hwbus_ops->lock(wdev->hwbus_priv);
 	ret = wdev->hwbus_ops->copy_from_io(wdev->hwbus_priv,
 					    WFX_REG_IN_OUT_QUEUE, buf, len);
@@ -249,7 +249,7 @@ int wfx_data_write(struct wfx_dev *wdev, const void *buf, size_t len)
 {
 	int ret;
 
-	WARN((long) buf & 3, "%s: unaligned buffer", __func__);
+	WARN((long)buf & 3, "%s: unaligned buffer", __func__);
 	wdev->hwbus_ops->lock(wdev->hwbus_priv);
 	ret = wdev->hwbus_ops->copy_to_io(wdev->hwbus_priv,
 					  WFX_REG_IN_OUT_QUEUE, buf, len);
diff --git a/drivers/staging/wfx/key.c b/drivers/staging/wfx/key.c
index 96adfa330604..5ee2ffc5f935 100644
--- a/drivers/staging/wfx/key.c
+++ b/drivers/staging/wfx/key.c
@@ -5,6 +5,7 @@
  * Copyright (c) 2017-2019, Silicon Laboratories, Inc.
  * Copyright (c) 2010, ST-Ericsson
  */
+#include <linux/etherdevice.h>
 #include <net/mac80211.h>
 
 #include "key.h"
@@ -20,14 +21,12 @@ static int wfx_alloc_key(struct wfx_dev *wdev)
 		return -1;
 
 	wdev->key_map |= BIT(idx);
-	wdev->keys[idx].entry_index = idx;
 	return idx;
 }
 
 static void wfx_free_key(struct wfx_dev *wdev, int idx)
 {
 	WARN(!(wdev->key_map & BIT(idx)), "inconsistent key allocation");
-	memset(&wdev->keys[idx], 0, sizeof(wdev->keys[idx]));
 	wdev->key_map &= ~BIT(idx);
 }
 
@@ -159,7 +158,7 @@ static int wfx_add_key(struct wfx_vif *wvif, struct ieee80211_sta *sta,
 		       struct ieee80211_key_conf *key)
 {
 	int ret;
-	struct hif_req_add_key *k;
+	struct hif_req_add_key k = { };
 	struct ieee80211_key_seq seq;
 	struct wfx_dev *wdev = wvif->wdev;
 	int idx = wfx_alloc_key(wvif->wdev);
@@ -169,44 +168,44 @@ static int wfx_add_key(struct wfx_vif *wvif, struct ieee80211_sta *sta,
 	ieee80211_get_key_rx_seq(key, 0, &seq);
 	if (idx < 0)
 		return -EINVAL;
-	k = &wdev->keys[idx];
-	k->int_id = wvif->id;
+	k.int_id = wvif->id;
+	k.entry_index = idx;
 	if (key->cipher == WLAN_CIPHER_SUITE_WEP40 ||
 	    key->cipher ==  WLAN_CIPHER_SUITE_WEP104) {
 		if (pairwise)
-			k->type = fill_wep_pair(&k->key.wep_pairwise_key, key,
-						sta->addr);
+			k.type = fill_wep_pair(&k.key.wep_pairwise_key, key,
+					       sta->addr);
 		else
-			k->type = fill_wep_group(&k->key.wep_group_key, key);
+			k.type = fill_wep_group(&k.key.wep_group_key, key);
 	} else if (key->cipher == WLAN_CIPHER_SUITE_TKIP) {
 		if (pairwise)
-			k->type = fill_tkip_pair(&k->key.tkip_pairwise_key, key,
-						 sta->addr);
+			k.type = fill_tkip_pair(&k.key.tkip_pairwise_key, key,
+						sta->addr);
 		else
-			k->type = fill_tkip_group(&k->key.tkip_group_key, key,
-						  &seq, wvif->vif->type);
+			k.type = fill_tkip_group(&k.key.tkip_group_key, key,
+						 &seq, wvif->vif->type);
 	} else if (key->cipher == WLAN_CIPHER_SUITE_CCMP) {
 		if (pairwise)
-			k->type = fill_ccmp_pair(&k->key.aes_pairwise_key, key,
-						 sta->addr);
+			k.type = fill_ccmp_pair(&k.key.aes_pairwise_key, key,
+						sta->addr);
 		else
-			k->type = fill_ccmp_group(&k->key.aes_group_key, key,
-						  &seq);
+			k.type = fill_ccmp_group(&k.key.aes_group_key, key,
+						 &seq);
 	} else if (key->cipher ==  WLAN_CIPHER_SUITE_SMS4) {
 		if (pairwise)
-			k->type = fill_sms4_pair(&k->key.wapi_pairwise_key, key,
-						 sta->addr);
+			k.type = fill_sms4_pair(&k.key.wapi_pairwise_key, key,
+						sta->addr);
 		else
-			k->type = fill_sms4_group(&k->key.wapi_group_key, key);
+			k.type = fill_sms4_group(&k.key.wapi_group_key, key);
 	} else if (key->cipher ==  WLAN_CIPHER_SUITE_AES_CMAC) {
-		k->type = fill_aes_cmac_group(&k->key.igtk_group_key, key,
-					      &seq);
+		k.type = fill_aes_cmac_group(&k.key.igtk_group_key, key,
+					     &seq);
 	} else {
 		dev_warn(wdev->dev, "unsupported key type %d\n", key->cipher);
 		wfx_free_key(wdev, idx);
 		return -EOPNOTSUPP;
 	}
-	ret = hif_add_key(wdev, k);
+	ret = hif_add_key(wdev, &k);
 	if (ret) {
 		wfx_free_key(wdev, idx);
 		return -EOPNOTSUPP;
@@ -229,7 +228,7 @@ int wfx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		struct ieee80211_key_conf *key)
 {
 	int ret = -EOPNOTSUPP;
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 
 	mutex_lock(&wvif->wdev->conf_mutex);
 	if (cmd == SET_KEY)
@@ -240,29 +239,3 @@ int wfx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 	return ret;
 }
 
-int wfx_upload_keys(struct wfx_vif *wvif)
-{
-	int i;
-	struct hif_req_add_key *key;
-	struct wfx_dev *wdev = wvif->wdev;
-
-	for (i = 0; i < ARRAY_SIZE(wdev->keys); i++) {
-		if (wdev->key_map & BIT(i)) {
-			key = &wdev->keys[i];
-			if (key->int_id == wvif->id)
-				hif_add_key(wdev, key);
-		}
-	}
-	return 0;
-}
-
-void wfx_wep_key_work(struct work_struct *work)
-{
-	struct wfx_vif *wvif = container_of(work, struct wfx_vif, wep_key_work);
-
-	wfx_tx_flush(wvif->wdev);
-	hif_wep_default_key_id(wvif, wvif->wep_default_key_id);
-	wfx_pending_requeue(wvif->wdev, wvif->wep_pending_skb);
-	wvif->wep_pending_skb = NULL;
-	wfx_tx_unlock(wvif->wdev);
-}
diff --git a/drivers/staging/wfx/key.h b/drivers/staging/wfx/key.h
index 9436ccdf4d3b..ff31fc9c565a 100644
--- a/drivers/staging/wfx/key.h
+++ b/drivers/staging/wfx/key.h
@@ -16,7 +16,5 @@ struct wfx_vif;
 int wfx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		struct ieee80211_vif *vif, struct ieee80211_sta *sta,
 		struct ieee80211_key_conf *key);
-int wfx_upload_keys(struct wfx_vif *wvif);
-void wfx_wep_key_work(struct work_struct *work);
 
 #endif /* WFX_STA_H */
diff --git a/drivers/staging/wfx/main.c b/drivers/staging/wfx/main.c
index 3c4c240229ad..6bd96f476388 100644
--- a/drivers/staging/wfx/main.c
+++ b/drivers/staging/wfx/main.c
@@ -28,6 +28,7 @@
 #include "bh.h"
 #include "sta.h"
 #include "key.h"
+#include "scan.h"
 #include "debug.h"
 #include "data_tx.h"
 #include "secure_link.h"
@@ -106,7 +107,7 @@ static const struct ieee80211_supported_band wfx_band_2ghz = {
 		.ampdu_density = IEEE80211_HT_MPDU_DENSITY_NONE,
 		.mcs = {
 			.rx_mask = { 0xFF }, // MCS0 to MCS7
-			.rx_highest = 65,
+			.rx_highest = cpu_to_le16(72),
 			.tx_params = IEEE80211_HT_MCS_TX_DEFINED,
 		},
 	},
@@ -133,15 +134,19 @@ static const struct ieee80211_ops wfx_ops = {
 	.remove_interface	= wfx_remove_interface,
 	.config                 = wfx_config,
 	.tx			= wfx_tx,
+	.join_ibss		= wfx_join_ibss,
+	.leave_ibss		= wfx_leave_ibss,
 	.conf_tx		= wfx_conf_tx,
 	.hw_scan		= wfx_hw_scan,
 	.cancel_hw_scan		= wfx_cancel_hw_scan,
+	.start_ap		= wfx_start_ap,
+	.stop_ap		= wfx_stop_ap,
 	.sta_add		= wfx_sta_add,
 	.sta_remove		= wfx_sta_remove,
-	.sta_notify		= wfx_sta_notify,
 	.set_tim		= wfx_set_tim,
 	.set_key		= wfx_set_key,
 	.set_rts_threshold	= wfx_set_rts_threshold,
+	.set_default_unicast_key = wfx_set_default_unicast_key,
 	.bss_info_changed	= wfx_bss_info_changed,
 	.prepare_multicast	= wfx_prepare_multicast,
 	.configure_filter	= wfx_configure_filter,
@@ -165,8 +170,8 @@ bool wfx_api_older_than(struct wfx_dev *wdev, int major, int minor)
 	return false;
 }
 
-struct gpio_desc *wfx_get_gpio(struct device *dev, int override,
-			       const char *label)
+struct gpio_desc *wfx_get_gpio(struct device *dev,
+			       int override, const char *label)
 {
 	struct gpio_desc *ret;
 	char label_buf[256];
@@ -187,18 +192,18 @@ struct gpio_desc *wfx_get_gpio(struct device *dev, int override,
 		if (!ret || PTR_ERR(ret) == -ENOENT)
 			dev_warn(dev, "gpio %s is not defined\n", label);
 		else
-			dev_warn(dev,
-				 "error while requesting gpio %s\n", label);
+			dev_warn(dev, "error while requesting gpio %s\n",
+				 label);
 		ret = NULL;
 	} else {
-		dev_dbg(dev,
-			"using gpio %d for %s\n", desc_to_gpio(ret), label);
+		dev_dbg(dev, "using gpio %d for %s\n",
+			desc_to_gpio(ret), label);
 	}
 	return ret;
 }
 
 /* NOTE: wfx_send_pds() destroy buf */
-int wfx_send_pds(struct wfx_dev *wdev, unsigned char *buf, size_t len)
+int wfx_send_pds(struct wfx_dev *wdev, u8 *buf, size_t len)
 {
 	int ret;
 	int start, brace_level, i;
@@ -224,16 +229,19 @@ int wfx_send_pds(struct wfx_dev *wdev, unsigned char *buf, size_t len)
 			buf[i] = '}';
 			ret = hif_configuration(wdev, buf + start,
 						i - start + 1);
-			if (ret == HIF_STATUS_FAILURE) {
-				dev_err(wdev->dev, "PDS bytes %d to %d: invalid data (unsupported options?)\n", start, i);
+			if (ret > 0) {
+				dev_err(wdev->dev, "PDS bytes %d to %d: invalid data (unsupported options?)\n",
+					start, i);
 				return -EINVAL;
 			}
 			if (ret == -ETIMEDOUT) {
-				dev_err(wdev->dev, "PDS bytes %d to %d: chip didn't reply (corrupted file?)\n", start, i);
+				dev_err(wdev->dev, "PDS bytes %d to %d: chip didn't reply (corrupted file?)\n",
+					start, i);
 				return ret;
 			}
 			if (ret) {
-				dev_err(wdev->dev, "PDS bytes %d to %d: chip returned an unknown error\n", start, i);
+				dev_err(wdev->dev, "PDS bytes %d to %d: chip returned an unknown error\n",
+					start, i);
 				return -EIO;
 			}
 			buf[i] = ',';
@@ -247,7 +255,7 @@ static int wfx_send_pdata_pds(struct wfx_dev *wdev)
 {
 	int ret = 0;
 	const struct firmware *pds;
-	unsigned char *tmp_buf;
+	u8 *tmp_buf;
 
 	ret = request_firmware(&pds, wdev->pdata.file_pds, wdev->dev);
 	if (ret) {
@@ -266,9 +274,9 @@ static void wfx_free_common(void *data)
 {
 	struct wfx_dev *wdev = data;
 
+	mutex_destroy(&wdev->tx_power_loop_info_lock);
 	mutex_destroy(&wdev->rx_stats_lock);
 	mutex_destroy(&wdev->conf_mutex);
-	wfx_tx_queues_deinit(wdev);
 	ieee80211_free_hw(wdev->hw);
 }
 
@@ -286,7 +294,6 @@ struct wfx_dev *wfx_init_common(struct device *dev,
 
 	SET_IEEE80211_DEV(hw, dev);
 
-	ieee80211_hw_set(hw, NEED_DTIM_BEFORE_ASSOC);
 	ieee80211_hw_set(hw, TX_AMPDU_SETUP_IN_HW);
 	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
 	ieee80211_hw_set(hw, CONNECTION_MONITOR);
@@ -315,7 +322,7 @@ struct wfx_dev *wfx_init_common(struct device *dev,
 	hw->wiphy->flags |= WIPHY_FLAG_AP_PROBE_RESP_OFFLOAD;
 	hw->wiphy->flags |= WIPHY_FLAG_AP_UAPSD;
 	hw->wiphy->flags &= ~WIPHY_FLAG_PS_ON_BY_DEFAULT;
-	hw->wiphy->max_ap_assoc_sta = WFX_MAX_STA_IN_AP_MODE;
+	hw->wiphy->max_ap_assoc_sta = HIF_LINK_ID_MAX;
 	hw->wiphy->max_scan_ssids = 2;
 	hw->wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN;
 	hw->wiphy->n_iface_combinations = ARRAY_SIZE(wfx_iface_combinations);
@@ -338,7 +345,10 @@ struct wfx_dev *wfx_init_common(struct device *dev,
 
 	mutex_init(&wdev->conf_mutex);
 	mutex_init(&wdev->rx_stats_lock);
+	mutex_init(&wdev->tx_power_loop_info_lock);
 	init_completion(&wdev->firmware_ready);
+	INIT_DELAYED_WORK(&wdev->cooling_timeout_work,
+			  wfx_cooling_timeout_work);
 	wfx_init_hif_cmd(&wdev->hif_cmd);
 	wfx_tx_queues_init(wdev);
 
@@ -359,23 +369,24 @@ int wfx_probe(struct wfx_dev *wdev)
 	// prevent bh() to touch it.
 	gpio_saved = wdev->pdata.gpio_wakeup;
 	wdev->pdata.gpio_wakeup = NULL;
+	wdev->poll_irq = true;
 
 	wfx_bh_register(wdev);
 
 	err = wfx_init_device(wdev);
 	if (err)
-		goto err1;
+		goto err0;
 
-	err = wait_for_completion_interruptible_timeout(&wdev->firmware_ready,
-							10 * HZ);
+	wfx_bh_poll_irq(wdev);
+	err = wait_for_completion_timeout(&wdev->firmware_ready, 1 * HZ);
 	if (err <= 0) {
 		if (err == 0) {
-			dev_err(wdev->dev, "timeout while waiting for startup indication. IRQ configuration error?\n");
+			dev_err(wdev->dev, "timeout while waiting for startup indication\n");
 			err = -ETIMEDOUT;
 		} else if (err == -ERESTARTSYS) {
 			dev_info(wdev->dev, "probe interrupted by user\n");
 		}
-		goto err1;
+		goto err0;
 	}
 
 	// FIXME: fill wiphy::hw_version
@@ -384,7 +395,7 @@ int wfx_probe(struct wfx_dev *wdev)
 		 wdev->hw_caps.firmware_build, wdev->hw_caps.firmware_label,
 		 wdev->hw_caps.api_version_major,
 		 wdev->hw_caps.api_version_minor,
-		 wdev->keyset, *((u32 *) &wdev->hw_caps.capabilities));
+		 wdev->keyset, *((u32 *)&wdev->hw_caps.capabilities));
 	snprintf(wdev->hw->wiphy->fw_version,
 		 sizeof(wdev->hw->wiphy->fw_version),
 		 "%d.%d.%d",
@@ -397,14 +408,14 @@ int wfx_probe(struct wfx_dev *wdev)
 			"unsupported firmware API version (expect 1 while firmware returns %d)\n",
 			wdev->hw_caps.api_version_major);
 		err = -ENOTSUPP;
-		goto err1;
+		goto err0;
 	}
 
 	err = wfx_sl_init(wdev);
 	if (err && wdev->hw_caps.capabilities.link_mode == SEC_LINK_ENFORCED) {
 		dev_err(wdev->dev,
 			"chip require secure_link, but can't negociate it\n");
-		goto err1;
+		goto err0;
 	}
 
 	if (wdev->hw_caps.regul_sel_mode_info.region_sel_mode) {
@@ -417,7 +428,16 @@ int wfx_probe(struct wfx_dev *wdev)
 		wdev->pdata.file_pds);
 	err = wfx_send_pdata_pds(wdev);
 	if (err < 0)
-		goto err1;
+		goto err0;
+
+	wdev->poll_irq = false;
+	err = wdev->hwbus_ops->irq_subscribe(wdev->hwbus_priv);
+	if (err)
+		goto err0;
+
+	err = hif_use_multi_tx_conf(wdev, true);
+	if (err)
+		dev_err(wdev->dev, "misconfigured IRQ?\n");
 
 	wdev->pdata.gpio_wakeup = gpio_saved;
 	if (wdev->pdata.gpio_wakeup) {
@@ -432,8 +452,6 @@ int wfx_probe(struct wfx_dev *wdev)
 		hif_set_operational_mode(wdev, HIF_OP_POWER_MODE_DOZE);
 	}
 
-	hif_use_multi_tx_conf(wdev, true);
-
 	for (i = 0; i < ARRAY_SIZE(wdev->addresses); i++) {
 		eth_zero_addr(wdev->addresses[i].addr);
 		macaddr = of_get_mac_address(wdev->dev->of_node);
@@ -466,8 +484,9 @@ int wfx_probe(struct wfx_dev *wdev)
 
 err2:
 	ieee80211_unregister_hw(wdev->hw);
-	ieee80211_free_hw(wdev->hw);
 err1:
+	wdev->hwbus_ops->irq_unsubscribe(wdev->hwbus_priv);
+err0:
 	wfx_bh_unregister(wdev);
 	return err;
 }
@@ -476,6 +495,7 @@ void wfx_release(struct wfx_dev *wdev)
 {
 	ieee80211_unregister_hw(wdev->hw);
 	hif_shutdown(wdev);
+	wdev->hwbus_ops->irq_unsubscribe(wdev->hwbus_priv);
 	wfx_bh_unregister(wdev);
 	wfx_sl_deinit(wdev);
 }
diff --git a/drivers/staging/wfx/main.h b/drivers/staging/wfx/main.h
index 9c9410072def..f832ce409fda 100644
--- a/drivers/staging/wfx/main.h
+++ b/drivers/staging/wfx/main.h
@@ -13,10 +13,10 @@
 #include <linux/device.h>
 #include <linux/gpio/consumer.h>
 
-#include "bus.h"
 #include "hif_api_general.h"
 
 struct wfx_dev;
+struct hwbus_ops;
 
 struct wfx_platform_data {
 	/* Keyset and ".sec" extention will appended to this string */
@@ -41,6 +41,6 @@ void wfx_release(struct wfx_dev *wdev);
 struct gpio_desc *wfx_get_gpio(struct device *dev, int override,
 			       const char *label);
 bool wfx_api_older_than(struct wfx_dev *wdev, int major, int minor);
-int wfx_send_pds(struct wfx_dev *wdev, unsigned char *buf, size_t len);
+int wfx_send_pds(struct wfx_dev *wdev, u8 *buf, size_t len);
 
 #endif
diff --git a/drivers/staging/wfx/queue.c b/drivers/staging/wfx/queue.c
index 39d9127ce4b9..3248ecefda56 100644
--- a/drivers/staging/wfx/queue.c
+++ b/drivers/staging/wfx/queue.c
@@ -35,6 +35,7 @@ void wfx_tx_flush(struct wfx_dev *wdev)
 	if (wdev->chip_frozen)
 		return;
 
+	wfx_tx_lock(wdev);
 	mutex_lock(&wdev->hif_cmd.lock);
 	ret = wait_event_timeout(wdev->hif.tx_buffers_empty,
 				 !wdev->hif.tx_buffers_used,
@@ -44,9 +45,10 @@ void wfx_tx_flush(struct wfx_dev *wdev)
 			 wdev->hif.tx_buffers_used);
 		wfx_pending_dump_old_frames(wdev, 3000);
 		// FIXME: drop pending frames here
-		wdev->chip_frozen = 1;
+		wdev->chip_frozen = true;
 	}
 	mutex_unlock(&wdev->hif_cmd.lock);
+	wfx_tx_unlock(wdev);
 }
 
 void wfx_tx_lock_flush(struct wfx_dev *wdev)
@@ -55,261 +57,142 @@ void wfx_tx_lock_flush(struct wfx_dev *wdev)
 	wfx_tx_flush(wdev);
 }
 
-void wfx_tx_queues_lock(struct wfx_dev *wdev)
+void wfx_tx_queues_init(struct wfx_dev *wdev)
 {
 	int i;
-	struct wfx_queue *queue;
 
+	skb_queue_head_init(&wdev->tx_pending);
+	init_waitqueue_head(&wdev->tx_dequeue);
 	for (i = 0; i < IEEE80211_NUM_ACS; ++i) {
-		queue = &wdev->tx_queue[i];
-		spin_lock_bh(&queue->queue.lock);
-		if (queue->tx_locked_cnt++ == 0)
-			ieee80211_stop_queue(wdev->hw, queue->queue_id);
-		spin_unlock_bh(&queue->queue.lock);
+		skb_queue_head_init(&wdev->tx_queue[i].normal);
+		skb_queue_head_init(&wdev->tx_queue[i].cab);
 	}
 }
 
-void wfx_tx_queues_unlock(struct wfx_dev *wdev)
+void wfx_tx_queues_check_empty(struct wfx_dev *wdev)
 {
 	int i;
-	struct wfx_queue *queue;
 
+	WARN_ON(!skb_queue_empty_lockless(&wdev->tx_pending));
 	for (i = 0; i < IEEE80211_NUM_ACS; ++i) {
-		queue = &wdev->tx_queue[i];
-		spin_lock_bh(&queue->queue.lock);
-		WARN(!queue->tx_locked_cnt, "queue already unlocked");
-		if (--queue->tx_locked_cnt == 0)
-			ieee80211_wake_queue(wdev->hw, queue->queue_id);
-		spin_unlock_bh(&queue->queue.lock);
+		WARN_ON(atomic_read(&wdev->tx_queue[i].pending_frames));
+		WARN_ON(!skb_queue_empty_lockless(&wdev->tx_queue[i].normal));
+		WARN_ON(!skb_queue_empty_lockless(&wdev->tx_queue[i].cab));
 	}
 }
 
-/* If successful, LOCKS the TX queue! */
-void wfx_tx_queues_wait_empty_vif(struct wfx_vif *wvif)
+static bool __wfx_tx_queue_empty(struct wfx_dev *wdev,
+				 struct sk_buff_head *skb_queue, int vif_id)
 {
-	int i;
-	bool done;
-	struct wfx_queue *queue;
-	struct sk_buff *item;
-	struct wfx_dev *wdev = wvif->wdev;
-	struct hif_msg *hif;
-
-	if (wvif->wdev->chip_frozen) {
-		wfx_tx_lock_flush(wdev);
-		wfx_tx_queues_clear(wdev);
-		return;
-	}
+	struct hif_msg *hif_msg;
+	struct sk_buff *skb;
 
-	do {
-		done = true;
-		wfx_tx_lock_flush(wdev);
-		for (i = 0; i < IEEE80211_NUM_ACS && done; ++i) {
-			queue = &wdev->tx_queue[i];
-			spin_lock_bh(&queue->queue.lock);
-			skb_queue_walk(&queue->queue, item) {
-				hif = (struct hif_msg *) item->data;
-				if (hif->interface == wvif->id)
-					done = false;
-			}
-			spin_unlock_bh(&queue->queue.lock);
-		}
-		if (!done) {
-			wfx_tx_unlock(wdev);
-			msleep(20);
+	spin_lock_bh(&skb_queue->lock);
+	skb_queue_walk(skb_queue, skb) {
+		hif_msg = (struct hif_msg *)skb->data;
+		if (vif_id < 0 || hif_msg->interface == vif_id) {
+			spin_unlock_bh(&skb_queue->lock);
+			return false;
 		}
-	} while (!done);
-}
-
-static void wfx_tx_queue_clear(struct wfx_dev *wdev, struct wfx_queue *queue,
-			       struct sk_buff_head *gc_list)
-{
-	int i;
-	struct sk_buff *item;
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
-
-	spin_lock_bh(&queue->queue.lock);
-	while ((item = __skb_dequeue(&queue->queue)) != NULL)
-		skb_queue_head(gc_list, item);
-	spin_lock_nested(&stats->pending.lock, 1);
-	for (i = 0; i < ARRAY_SIZE(stats->link_map_cache); ++i) {
-		stats->link_map_cache[i] -= queue->link_map_cache[i];
-		queue->link_map_cache[i] = 0;
-	}
-	spin_unlock(&stats->pending.lock);
-	spin_unlock_bh(&queue->queue.lock);
-}
-
-void wfx_tx_queues_clear(struct wfx_dev *wdev)
-{
-	int i;
-	struct sk_buff *item;
-	struct sk_buff_head gc_list;
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
-
-	skb_queue_head_init(&gc_list);
-	for (i = 0; i < IEEE80211_NUM_ACS; ++i)
-		wfx_tx_queue_clear(wdev, &wdev->tx_queue[i], &gc_list);
-	wake_up(&stats->wait_link_id_empty);
-	while ((item = skb_dequeue(&gc_list)) != NULL)
-		wfx_skb_dtor(wdev, item);
-}
-
-void wfx_tx_queues_init(struct wfx_dev *wdev)
-{
-	int i;
-
-	memset(&wdev->tx_queue_stats, 0, sizeof(wdev->tx_queue_stats));
-	memset(wdev->tx_queue, 0, sizeof(wdev->tx_queue));
-	skb_queue_head_init(&wdev->tx_queue_stats.pending);
-	init_waitqueue_head(&wdev->tx_queue_stats.wait_link_id_empty);
-
-	for (i = 0; i < IEEE80211_NUM_ACS; ++i) {
-		wdev->tx_queue[i].queue_id = i;
-		skb_queue_head_init(&wdev->tx_queue[i].queue);
 	}
+	spin_unlock_bh(&skb_queue->lock);
+	return true;
 }
 
-void wfx_tx_queues_deinit(struct wfx_dev *wdev)
+bool wfx_tx_queue_empty(struct wfx_dev *wdev,
+			struct wfx_queue *queue, int vif_id)
 {
-	WARN_ON(!skb_queue_empty(&wdev->tx_queue_stats.pending));
-	wfx_tx_queues_clear(wdev);
+	return __wfx_tx_queue_empty(wdev, &queue->normal, vif_id) &&
+	       __wfx_tx_queue_empty(wdev, &queue->cab, vif_id);
 }
 
-int wfx_tx_queue_get_num_queued(struct wfx_queue *queue, u32 link_id_map)
+static void __wfx_tx_queue_drop(struct wfx_dev *wdev,
+				struct sk_buff_head *skb_queue, int vif_id,
+				struct sk_buff_head *dropped)
 {
-	int ret, i;
-
-	if (!link_id_map)
-		return 0;
-
-	spin_lock_bh(&queue->queue.lock);
-	if (link_id_map == (u32)-1) {
-		ret = skb_queue_len(&queue->queue);
-	} else {
-		ret = 0;
-		for (i = 0; i < ARRAY_SIZE(queue->link_map_cache); i++)
-			if (link_id_map & BIT(i))
-				ret += queue->link_map_cache[i];
+	struct sk_buff *skb, *tmp;
+	struct hif_msg *hif_msg;
+
+	spin_lock_bh(&skb_queue->lock);
+	skb_queue_walk_safe(skb_queue, skb, tmp) {
+		hif_msg = (struct hif_msg *)skb->data;
+		if (vif_id < 0 || hif_msg->interface == vif_id) {
+			__skb_unlink(skb, skb_queue);
+			skb_queue_head(dropped, skb);
+		}
 	}
-	spin_unlock_bh(&queue->queue.lock);
-	return ret;
+	spin_unlock_bh(&skb_queue->lock);
 }
 
-void wfx_tx_queue_put(struct wfx_dev *wdev, struct wfx_queue *queue,
-		      struct sk_buff *skb)
+void wfx_tx_queue_drop(struct wfx_dev *wdev, struct wfx_queue *queue,
+		       int vif_id, struct sk_buff_head *dropped)
 {
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
-	struct wfx_tx_priv *tx_priv = wfx_skb_tx_priv(skb);
-
-	WARN(tx_priv->link_id >= ARRAY_SIZE(stats->link_map_cache), "invalid link-id value");
-	spin_lock_bh(&queue->queue.lock);
-	__skb_queue_tail(&queue->queue, skb);
-
-	++queue->link_map_cache[tx_priv->link_id];
-
-	spin_lock_nested(&stats->pending.lock, 1);
-	++stats->link_map_cache[tx_priv->link_id];
-	spin_unlock(&stats->pending.lock);
-	spin_unlock_bh(&queue->queue.lock);
-}
-
-static struct sk_buff *wfx_tx_queue_get(struct wfx_dev *wdev,
-					struct wfx_queue *queue,
-					u32 link_id_map)
-{
-	struct sk_buff *skb = NULL;
-	struct sk_buff *item;
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
-	struct wfx_tx_priv *tx_priv;
-	bool wakeup_stats = false;
-
-	spin_lock_bh(&queue->queue.lock);
-	skb_queue_walk(&queue->queue, item) {
-		tx_priv = wfx_skb_tx_priv(item);
-		if (link_id_map & BIT(tx_priv->link_id)) {
-			skb = item;
-			break;
-		}
-	}
-	if (skb) {
-		tx_priv = wfx_skb_tx_priv(skb);
-		tx_priv->xmit_timestamp = ktime_get();
-		__skb_unlink(skb, &queue->queue);
-		--queue->link_map_cache[tx_priv->link_id];
-
-		spin_lock_nested(&stats->pending.lock, 1);
-		__skb_queue_tail(&stats->pending, skb);
-		if (!--stats->link_map_cache[tx_priv->link_id])
-			wakeup_stats = true;
-		spin_unlock(&stats->pending.lock);
-	}
-	spin_unlock_bh(&queue->queue.lock);
-	if (wakeup_stats)
-		wake_up(&stats->wait_link_id_empty);
-	return skb;
+	__wfx_tx_queue_drop(wdev, &queue->cab, vif_id, dropped);
+	__wfx_tx_queue_drop(wdev, &queue->normal, vif_id, dropped);
+	wake_up(&wdev->tx_dequeue);
 }
 
-int wfx_pending_requeue(struct wfx_dev *wdev, struct sk_buff *skb)
+void wfx_tx_queues_put(struct wfx_dev *wdev, struct sk_buff *skb)
 {
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
-	struct wfx_tx_priv *tx_priv = wfx_skb_tx_priv(skb);
 	struct wfx_queue *queue = &wdev->tx_queue[skb_get_queue_mapping(skb)];
+	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
 
-	WARN_ON(skb_get_queue_mapping(skb) > 3);
-	spin_lock_bh(&queue->queue.lock);
-	++queue->link_map_cache[tx_priv->link_id];
-
-	spin_lock_nested(&stats->pending.lock, 1);
-	++stats->link_map_cache[tx_priv->link_id];
-	__skb_unlink(skb, &stats->pending);
-	spin_unlock(&stats->pending.lock);
-	__skb_queue_tail(&queue->queue, skb);
-	spin_unlock_bh(&queue->queue.lock);
-	return 0;
+	if (tx_info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM)
+		skb_queue_tail(&queue->cab, skb);
+	else
+		skb_queue_tail(&queue->normal, skb);
 }
 
-int wfx_pending_remove(struct wfx_dev *wdev, struct sk_buff *skb)
+void wfx_pending_drop(struct wfx_dev *wdev, struct sk_buff_head *dropped)
 {
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
-
-	spin_lock_bh(&stats->pending.lock);
-	__skb_unlink(skb, &stats->pending);
-	spin_unlock_bh(&stats->pending.lock);
-	wfx_skb_dtor(wdev, skb);
+	struct wfx_queue *queue;
+	struct sk_buff *skb;
 
-	return 0;
+	WARN(!wdev->chip_frozen, "%s should only be used to recover a frozen device",
+	     __func__);
+	while ((skb = skb_dequeue(&wdev->tx_pending)) != NULL) {
+		queue = &wdev->tx_queue[skb_get_queue_mapping(skb)];
+		WARN_ON(skb_get_queue_mapping(skb) > 3);
+		WARN_ON(!atomic_read(&queue->pending_frames));
+		atomic_dec(&queue->pending_frames);
+		skb_queue_head(dropped, skb);
+	}
 }
 
 struct sk_buff *wfx_pending_get(struct wfx_dev *wdev, u32 packet_id)
 {
-	struct sk_buff *skb;
+	struct wfx_queue *queue;
 	struct hif_req_tx *req;
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
+	struct sk_buff *skb;
 
-	spin_lock_bh(&stats->pending.lock);
-	skb_queue_walk(&stats->pending, skb) {
+	spin_lock_bh(&wdev->tx_pending.lock);
+	skb_queue_walk(&wdev->tx_pending, skb) {
 		req = wfx_skb_txreq(skb);
 		if (req->packet_id == packet_id) {
-			spin_unlock_bh(&stats->pending.lock);
+			spin_unlock_bh(&wdev->tx_pending.lock);
+			queue = &wdev->tx_queue[skb_get_queue_mapping(skb)];
+			WARN_ON(skb_get_queue_mapping(skb) > 3);
+			WARN_ON(!atomic_read(&queue->pending_frames));
+			atomic_dec(&queue->pending_frames);
+			skb_unlink(skb, &wdev->tx_pending);
 			return skb;
 		}
 	}
-	spin_unlock_bh(&stats->pending.lock);
+	spin_unlock_bh(&wdev->tx_pending.lock);
 	WARN(1, "cannot find packet in pending queue");
 	return NULL;
 }
 
 void wfx_pending_dump_old_frames(struct wfx_dev *wdev, unsigned int limit_ms)
 {
-	struct wfx_queue_stats *stats = &wdev->tx_queue_stats;
 	ktime_t now = ktime_get();
 	struct wfx_tx_priv *tx_priv;
 	struct hif_req_tx *req;
 	struct sk_buff *skb;
 	bool first = true;
 
-	spin_lock_bh(&stats->pending.lock);
-	skb_queue_walk(&stats->pending, skb) {
+	spin_lock_bh(&wdev->tx_pending.lock);
+	skb_queue_walk(&wdev->tx_pending, skb) {
 		tx_priv = wfx_skb_tx_priv(skb);
 		req = wfx_skb_txreq(skb);
 		if (ktime_after(now, ktime_add_ms(tx_priv->xmit_timestamp,
@@ -324,7 +207,7 @@ void wfx_pending_dump_old_frames(struct wfx_dev *wdev, unsigned int limit_ms)
 				 ktime_ms_delta(now, tx_priv->xmit_timestamp));
 		}
 	}
-	spin_unlock_bh(&stats->pending.lock);
+	spin_unlock_bh(&wdev->tx_pending.lock);
 }
 
 unsigned int wfx_pending_get_pkt_us_delay(struct wfx_dev *wdev,
@@ -336,138 +219,66 @@ unsigned int wfx_pending_get_pkt_us_delay(struct wfx_dev *wdev,
 	return ktime_us_delta(now, tx_priv->xmit_timestamp);
 }
 
-bool wfx_tx_queues_is_empty(struct wfx_dev *wdev)
+bool wfx_tx_queues_has_cab(struct wfx_vif *wvif)
 {
+	struct wfx_dev *wdev = wvif->wdev;
 	int i;
-	struct sk_buff_head *queue;
-	bool ret = true;
-
-	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
-		queue = &wdev->tx_queue[i].queue;
-		spin_lock_bh(&queue->lock);
-		if (!skb_queue_empty(queue))
-			ret = false;
-		spin_unlock_bh(&queue->lock);
-	}
-	return ret;
-}
-
-static bool hif_handle_tx_data(struct wfx_vif *wvif, struct sk_buff *skb,
-			       struct wfx_queue *queue)
-{
-	struct hif_req_tx *req = wfx_skb_txreq(skb);
-	struct ieee80211_key_conf *hw_key = wfx_skb_tx_priv(skb)->hw_key;
-	struct ieee80211_hdr *frame =
-		(struct ieee80211_hdr *)(req->frame + req->data_flags.fc_offset);
-
-	// FIXME: mac80211 is smart enough to handle BSS loss. Driver should not
-	// try to do anything about that.
-	if (ieee80211_is_nullfunc(frame->frame_control)) {
-		mutex_lock(&wvif->bss_loss_lock);
-		if (wvif->bss_loss_state) {
-			wvif->bss_loss_confirm_id = req->packet_id;
-			req->queue_id.queue_id = HIF_QUEUE_ID_VOICE;
-		}
-		mutex_unlock(&wvif->bss_loss_lock);
-	}
 
-	// FIXME: identify the exact scenario matched by this condition. Does it
-	// happen yet?
-	if (ieee80211_has_protected(frame->frame_control) &&
-	    hw_key && hw_key->keyidx != wvif->wep_default_key_id &&
-	    (hw_key->cipher == WLAN_CIPHER_SUITE_WEP40 ||
-	     hw_key->cipher == WLAN_CIPHER_SUITE_WEP104)) {
-		wfx_tx_lock(wvif->wdev);
-		WARN_ON(wvif->wep_pending_skb);
-		wvif->wep_default_key_id = hw_key->keyidx;
-		wvif->wep_pending_skb = skb;
-		if (!schedule_work(&wvif->wep_key_work))
-			wfx_tx_unlock(wvif->wdev);
-		return true;
-	} else {
+	if (wvif->vif->type != NL80211_IFTYPE_AP)
 		return false;
-	}
-}
-
-static int wfx_get_prio_queue(struct wfx_vif *wvif,
-				 u32 tx_allowed_mask, int *total)
-{
-	static const int urgent = BIT(WFX_LINK_ID_AFTER_DTIM) |
-		BIT(WFX_LINK_ID_UAPSD);
-	const struct ieee80211_tx_queue_params *edca;
-	unsigned int score, best = -1;
-	int winner = -1;
-	int i;
-
-	/* search for a winner using edca params */
-	for (i = 0; i < IEEE80211_NUM_ACS; ++i) {
-		int queued;
-
-		edca = &wvif->edca_params[i];
-		queued = wfx_tx_queue_get_num_queued(&wvif->wdev->tx_queue[i],
-				tx_allowed_mask);
-		if (!queued)
-			continue;
-		*total += queued;
-		score = ((edca->aifs + edca->cw_min) << 16) +
-			((edca->cw_max - edca->cw_min) *
-			 (get_random_int() & 0xFFFF));
-		if (score < best && (winner < 0 || i != 3)) {
-			best = score;
-			winner = i;
-		}
-	}
-
-	/* override winner if bursting */
-	if (winner >= 0 && wvif->wdev->tx_burst_idx >= 0 &&
-	    winner != wvif->wdev->tx_burst_idx &&
-	    !wfx_tx_queue_get_num_queued(&wvif->wdev->tx_queue[winner],
-					 tx_allowed_mask & urgent) &&
-	    wfx_tx_queue_get_num_queued(&wvif->wdev->tx_queue[wvif->wdev->tx_burst_idx], tx_allowed_mask))
-		winner = wvif->wdev->tx_burst_idx;
-
-	return winner;
-}
-
-static int wfx_tx_queue_mask_get(struct wfx_vif *wvif,
-				     struct wfx_queue **queue_p,
-				     u32 *tx_allowed_mask_p)
-{
-	int idx;
-	u32 tx_allowed_mask;
-	int total = 0;
-
-	/* Search for unicast traffic */
-	tx_allowed_mask = ~wvif->sta_asleep_mask;
-	tx_allowed_mask |= BIT(WFX_LINK_ID_UAPSD);
-	if (wvif->sta_asleep_mask)
-		tx_allowed_mask &= ~BIT(WFX_LINK_ID_AFTER_DTIM);
-	else
-		tx_allowed_mask |= BIT(WFX_LINK_ID_AFTER_DTIM);
-	idx = wfx_get_prio_queue(wvif, tx_allowed_mask, &total);
-	if (idx < 0)
-		return -ENOENT;
-
-	*queue_p = &wvif->wdev->tx_queue[idx];
-	*tx_allowed_mask_p = tx_allowed_mask;
-	return 0;
+	for (i = 0; i < IEEE80211_NUM_ACS; ++i)
+		// Note: since only AP can have mcast frames in queue and only
+		// one vif can be AP, all queued frames has same interface id
+		if (!skb_queue_empty_lockless(&wdev->tx_queue[i].cab))
+			return true;
+	return false;
 }
 
-struct hif_msg *wfx_tx_queues_get_after_dtim(struct wfx_vif *wvif)
+static struct sk_buff *wfx_tx_queues_get_skb(struct wfx_dev *wdev)
 {
-	struct wfx_dev *wdev = wvif->wdev;
-	struct ieee80211_tx_info *tx_info;
+	struct wfx_queue *sorted_queues[IEEE80211_NUM_ACS];
+	struct wfx_vif *wvif;
 	struct hif_msg *hif;
 	struct sk_buff *skb;
-	int i;
+	int i, j;
 
-	for (i = 0; i < IEEE80211_NUM_ACS; ++i) {
-		skb_queue_walk(&wdev->tx_queue[i].queue, skb) {
-			tx_info = IEEE80211_SKB_CB(skb);
+	// bubble sort
+	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
+		sorted_queues[i] = &wdev->tx_queue[i];
+		for (j = i; j > 0; j--)
+			if (atomic_read(&sorted_queues[j]->pending_frames) >
+			    atomic_read(&sorted_queues[j - 1]->pending_frames))
+				swap(sorted_queues[j - 1], sorted_queues[j]);
+	}
+	wvif = NULL;
+	while ((wvif = wvif_iterate(wdev, wvif)) != NULL) {
+		if (!wvif->after_dtim_tx_allowed)
+			continue;
+		for (i = 0; i < IEEE80211_NUM_ACS; i++) {
+			skb = skb_dequeue(&sorted_queues[i]->cab);
+			if (!skb)
+				continue;
+			// Note: since only AP can have mcast frames in queue
+			// and only one vif can be AP, all queued frames has
+			// same interface id
 			hif = (struct hif_msg *)skb->data;
-			if ((tx_info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) &&
-			    (hif->interface == wvif->id))
-				return (struct hif_msg *)skb->data;
+			WARN_ON(hif->interface != wvif->id);
+			WARN_ON(sorted_queues[i] !=
+				&wdev->tx_queue[skb_get_queue_mapping(skb)]);
+			atomic_inc(&sorted_queues[i]->pending_frames);
+			return skb;
+		}
+		// No more multicast to sent
+		wvif->after_dtim_tx_allowed = false;
+		schedule_work(&wvif->update_tim_work);
+	}
+	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
+		skb = skb_dequeue(&sorted_queues[i]->normal);
+		if (skb) {
+			WARN_ON(sorted_queues[i] !=
+				&wdev->tx_queue[skb_get_queue_mapping(skb)]);
+			atomic_inc(&sorted_queues[i]->pending_frames);
+			return skb;
 		}
 	}
 	return NULL;
@@ -475,90 +286,20 @@ struct hif_msg *wfx_tx_queues_get_after_dtim(struct wfx_vif *wvif)
 
 struct hif_msg *wfx_tx_queues_get(struct wfx_dev *wdev)
 {
+	struct wfx_tx_priv *tx_priv;
 	struct sk_buff *skb;
-	struct hif_msg *hif = NULL;
-	struct wfx_queue *queue = NULL;
-	struct wfx_queue *vif_queue = NULL;
-	u32 tx_allowed_mask = 0;
-	u32 vif_tx_allowed_mask = 0;
-	struct wfx_vif *wvif;
-	int not_found;
-	int burst;
-	int i;
 
 	if (atomic_read(&wdev->tx_lock))
 		return NULL;
 
-	wvif = NULL;
-	while ((wvif = wvif_iterate(wdev, wvif)) != NULL) {
-		if (wvif->after_dtim_tx_allowed) {
-			for (i = 0; i < IEEE80211_NUM_ACS; ++i) {
-				skb = wfx_tx_queue_get(wvif->wdev,
-						       &wdev->tx_queue[i],
-						       BIT(WFX_LINK_ID_AFTER_DTIM));
-				if (skb) {
-					hif = (struct hif_msg *)skb->data;
-					// Cannot happen since only one vif can
-					// be AP at time
-					WARN_ON(wvif->id != hif->interface);
-					return hif;
-				}
-			}
-			// No more multicast to sent
-			wvif->after_dtim_tx_allowed = false;
-			schedule_work(&wvif->update_tim_work);
-		}
-	}
-
 	for (;;) {
-		int ret = -ENOENT;
-		int queue_num;
-
-		wvif = NULL;
-		while ((wvif = wvif_iterate(wdev, wvif)) != NULL) {
-			spin_lock_bh(&wvif->ps_state_lock);
-
-			not_found = wfx_tx_queue_mask_get(wvif, &vif_queue,
-							  &vif_tx_allowed_mask);
-
-			spin_unlock_bh(&wvif->ps_state_lock);
-
-			if (!not_found) {
-				if (queue && queue != vif_queue)
-					dev_info(wdev->dev, "vifs disagree about queue priority\n");
-				tx_allowed_mask |= vif_tx_allowed_mask;
-				queue = vif_queue;
-				ret = 0;
-			}
-		}
-
-		if (ret)
-			return NULL;
-
-		queue_num = queue - wdev->tx_queue;
-
-		skb = wfx_tx_queue_get(wdev, queue, tx_allowed_mask);
+		skb = wfx_tx_queues_get_skb(wdev);
 		if (!skb)
-			continue;
-		hif = (struct hif_msg *)skb->data;
-		wvif = wdev_to_wvif(wdev, hif->interface);
-		WARN_ON(!wvif);
-
-		if (hif_handle_tx_data(wvif, skb, queue))
-			continue;  /* Handled by WSM */
-
-		/* allow bursting if txop is set */
-		if (wvif->edca_params[queue_num].txop)
-			burst = wfx_tx_queue_get_num_queued(queue, tx_allowed_mask) + 1;
-		else
-			burst = 1;
-
-		/* store index of bursting queue */
-		if (burst > 1)
-			wdev->tx_burst_idx = queue_num;
-		else
-			wdev->tx_burst_idx = -1;
-
-		return hif;
+			return NULL;
+		skb_queue_tail(&wdev->tx_pending, skb);
+		wake_up(&wdev->tx_dequeue);
+		tx_priv = wfx_skb_tx_priv(skb);
+		tx_priv->xmit_timestamp = ktime_get();
+		return (struct hif_msg *)skb->data;
 	}
 }
diff --git a/drivers/staging/wfx/queue.h b/drivers/staging/wfx/queue.h
index 90bb060d1204..0c3b7244498e 100644
--- a/drivers/staging/wfx/queue.h
+++ b/drivers/staging/wfx/queue.h
@@ -9,29 +9,15 @@
 #define WFX_QUEUE_H
 
 #include <linux/skbuff.h>
-
-#include "hif_api_cmd.h"
-
-#define WFX_MAX_STA_IN_AP_MODE    14
-#define WFX_LINK_ID_NO_ASSOC      15
-#define WFX_LINK_ID_AFTER_DTIM    (WFX_LINK_ID_NO_ASSOC + 1)
-#define WFX_LINK_ID_UAPSD         (WFX_LINK_ID_NO_ASSOC + 2)
-#define WFX_LINK_ID_MAX           (WFX_LINK_ID_NO_ASSOC + 3)
+#include <linux/atomic.h>
 
 struct wfx_dev;
 struct wfx_vif;
 
 struct wfx_queue {
-	struct sk_buff_head	queue;
-	int			tx_locked_cnt;
-	int			link_map_cache[WFX_LINK_ID_MAX];
-	u8			queue_id;
-};
-
-struct wfx_queue_stats {
-	int			link_map_cache[WFX_LINK_ID_MAX];
-	struct sk_buff_head	pending;
-	wait_queue_head_t	wait_link_id_empty;
+	struct sk_buff_head	normal;
+	struct sk_buff_head	cab; // Content After (DTIM) Beacon
+	atomic_t		pending_frames;
 };
 
 void wfx_tx_lock(struct wfx_dev *wdev);
@@ -40,22 +26,18 @@ void wfx_tx_flush(struct wfx_dev *wdev);
 void wfx_tx_lock_flush(struct wfx_dev *wdev);
 
 void wfx_tx_queues_init(struct wfx_dev *wdev);
-void wfx_tx_queues_deinit(struct wfx_dev *wdev);
-void wfx_tx_queues_lock(struct wfx_dev *wdev);
-void wfx_tx_queues_unlock(struct wfx_dev *wdev);
-void wfx_tx_queues_clear(struct wfx_dev *wdev);
-bool wfx_tx_queues_is_empty(struct wfx_dev *wdev);
-void wfx_tx_queues_wait_empty_vif(struct wfx_vif *wvif);
+void wfx_tx_queues_check_empty(struct wfx_dev *wdev);
+bool wfx_tx_queues_has_cab(struct wfx_vif *wvif);
+void wfx_tx_queues_put(struct wfx_dev *wdev, struct sk_buff *skb);
 struct hif_msg *wfx_tx_queues_get(struct wfx_dev *wdev);
-struct hif_msg *wfx_tx_queues_get_after_dtim(struct wfx_vif *wvif);
 
-void wfx_tx_queue_put(struct wfx_dev *wdev, struct wfx_queue *queue,
-		      struct sk_buff *skb);
-int wfx_tx_queue_get_num_queued(struct wfx_queue *queue, u32 link_id_map);
+bool wfx_tx_queue_empty(struct wfx_dev *wdev, struct wfx_queue *queue,
+			int vif_id);
+void wfx_tx_queue_drop(struct wfx_dev *wdev, struct wfx_queue *queue,
+		       int vif_id, struct sk_buff_head *dropped);
 
 struct sk_buff *wfx_pending_get(struct wfx_dev *wdev, u32 packet_id);
-int wfx_pending_remove(struct wfx_dev *wdev, struct sk_buff *skb);
-int wfx_pending_requeue(struct wfx_dev *wdev, struct sk_buff *skb);
+void wfx_pending_drop(struct wfx_dev *wdev, struct sk_buff_head *dropped);
 unsigned int wfx_pending_get_pkt_us_delay(struct wfx_dev *wdev,
 					  struct sk_buff *skb);
 void wfx_pending_dump_old_frames(struct wfx_dev *wdev, unsigned int limit_ms);
diff --git a/drivers/staging/wfx/scan.c b/drivers/staging/wfx/scan.c
index 9aa14331affd..57ea9997800b 100644
--- a/drivers/staging/wfx/scan.c
+++ b/drivers/staging/wfx/scan.c
@@ -88,18 +88,22 @@ void wfx_hw_scan_work(struct work_struct *work)
 	struct ieee80211_scan_request *hw_req = wvif->scan_req;
 	int chan_cur, ret;
 
-	mutex_lock(&wvif->scan_lock);
 	mutex_lock(&wvif->wdev->conf_mutex);
+	mutex_lock(&wvif->scan_lock);
+	if (wvif->join_in_progress) {
+		dev_info(wvif->wdev->dev, "%s: abort in-progress REQ_JOIN",
+			 __func__);
+		wfx_reset(wvif);
+	}
 	update_probe_tmpl(wvif, &hw_req->req);
-	wfx_fwd_probe_req(wvif, true);
 	chan_cur = 0;
 	do {
 		ret = send_scan_req(wvif, &hw_req->req, chan_cur);
 		if (ret > 0)
 			chan_cur += ret;
 	} while (ret > 0 && chan_cur < hw_req->req.n_channels);
-	mutex_unlock(&wvif->wdev->conf_mutex);
 	mutex_unlock(&wvif->scan_lock);
+	mutex_unlock(&wvif->wdev->conf_mutex);
 	__ieee80211_scan_completed_compat(wvif->wdev->hw, ret < 0);
 }
 
@@ -113,9 +117,6 @@ int wfx_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	if (vif->type == NL80211_IFTYPE_AP)
 		return -EOPNOTSUPP;
 
-	if (wvif->state == WFX_STATE_PRE_STA)
-		return -EBUSY;
-
 	wvif->scan_req = hw_req;
 	schedule_work(&wvif->scan_work);
 	return 0;
diff --git a/drivers/staging/wfx/sta.c b/drivers/staging/wfx/sta.c
index 9d430346a58b..12e8a5b638f1 100644
--- a/drivers/staging/wfx/sta.c
+++ b/drivers/staging/wfx/sta.c
@@ -5,6 +5,7 @@
  * Copyright (c) 2017-2019, Silicon Laboratories, Inc.
  * Copyright (c) 2010, ST-Ericsson
  */
+#include <linux/etherdevice.h>
 #include <net/mac80211.h>
 
 #include "sta.h"
@@ -37,117 +38,32 @@ u32 wfx_rate_mask_to_hw(struct wfx_dev *wdev, u32 rates)
 	return ret;
 }
 
-static void __wfx_free_event_queue(struct list_head *list)
+void wfx_cooling_timeout_work(struct work_struct *work)
 {
-	struct wfx_hif_event *event, *tmp;
+	struct wfx_dev *wdev = container_of(to_delayed_work(work),
+					    struct wfx_dev,
+					    cooling_timeout_work);
 
-	list_for_each_entry_safe(event, tmp, list, link) {
-		list_del(&event->link);
-		kfree(event);
-	}
-}
-
-static void wfx_free_event_queue(struct wfx_vif *wvif)
-{
-	LIST_HEAD(list);
-
-	spin_lock(&wvif->event_queue_lock);
-	list_splice_init(&wvif->event_queue, &list);
-	spin_unlock(&wvif->event_queue_lock);
-
-	__wfx_free_event_queue(&list);
+	wdev->chip_frozen = true;
+	wfx_tx_unlock(wdev);
 }
 
-void wfx_cqm_bssloss_sm(struct wfx_vif *wvif, int init, int good, int bad)
+void wfx_suspend_hot_dev(struct wfx_dev *wdev, enum sta_notify_cmd cmd)
 {
-	int tx = 0;
-
-	mutex_lock(&wvif->bss_loss_lock);
-	cancel_work_sync(&wvif->bss_params_work);
-
-	if (init) {
-		schedule_delayed_work(&wvif->bss_loss_work, HZ);
-		wvif->bss_loss_state = 0;
-
-		if (!atomic_read(&wvif->wdev->tx_lock))
-			tx = 1;
-	} else if (good) {
-		cancel_delayed_work_sync(&wvif->bss_loss_work);
-		wvif->bss_loss_state = 0;
-		schedule_work(&wvif->bss_params_work);
-	} else if (bad) {
-		/* FIXME Should we just keep going until we time out? */
-		if (wvif->bss_loss_state < 3)
-			tx = 1;
+	if (cmd == STA_NOTIFY_AWAKE) {
+		// Device recover normal temperature
+		if (cancel_delayed_work(&wdev->cooling_timeout_work))
+			wfx_tx_unlock(wdev);
 	} else {
-		cancel_delayed_work_sync(&wvif->bss_loss_work);
-		wvif->bss_loss_state = 0;
-	}
-
-	/* Spit out a NULL packet to our AP if necessary */
-	// FIXME: call ieee80211_beacon_loss/ieee80211_connection_loss instead
-	if (tx) {
-		struct sk_buff *skb;
-		struct ieee80211_hdr *hdr;
-		struct ieee80211_tx_control control = { };
-
-		wvif->bss_loss_state++;
-
-		skb = ieee80211_nullfunc_get(wvif->wdev->hw, wvif->vif, false);
-		if (!skb)
-			goto end;
-		hdr = (struct ieee80211_hdr *)skb->data;
-		memset(IEEE80211_SKB_CB(skb), 0,
-		       sizeof(*IEEE80211_SKB_CB(skb)));
-		IEEE80211_SKB_CB(skb)->control.vif = wvif->vif;
-		IEEE80211_SKB_CB(skb)->driver_rates[0].idx = 0;
-		IEEE80211_SKB_CB(skb)->driver_rates[0].count = 1;
-		IEEE80211_SKB_CB(skb)->driver_rates[1].idx = -1;
-		rcu_read_lock(); // protect control.sta
-		control.sta = ieee80211_find_sta(wvif->vif, hdr->addr1);
-		wfx_tx(wvif->wdev->hw, &control, skb);
-		rcu_read_unlock();
+		// Device is too hot
+		schedule_delayed_work(&wdev->cooling_timeout_work, 10 * HZ);
+		wfx_tx_lock(wdev);
 	}
-end:
-	mutex_unlock(&wvif->bss_loss_lock);
-}
-
-int wfx_fwd_probe_req(struct wfx_vif *wvif, bool enable)
-{
-	wvif->fwd_probe_req = enable;
-	return hif_set_rx_filter(wvif, wvif->filter_bssid,
-				 wvif->fwd_probe_req);
-}
-
-static int wfx_set_mcast_filter(struct wfx_vif *wvif,
-				    struct wfx_grp_addr_table *fp)
-{
-	int i;
-
-	// Temporary workaround for filters
-	return hif_set_data_filtering(wvif, false, true);
-
-	if (!fp->enable)
-		return hif_set_data_filtering(wvif, false, true);
-
-	for (i = 0; i < fp->num_addresses; i++)
-		hif_set_mac_addr_condition(wvif, i, fp->address_list[i]);
-	hif_set_uc_mc_bc_condition(wvif, 0,
-				   HIF_FILTER_UNICAST | HIF_FILTER_BROADCAST);
-	hif_set_config_data_filter(wvif, true, 0, BIT(1),
-				   BIT(fp->num_addresses) - 1);
-	hif_set_data_filtering(wvif, true, true);
-
-	return 0;
 }
 
-void wfx_update_filtering(struct wfx_vif *wvif)
+static void wfx_filter_beacon(struct wfx_vif *wvif, bool filter_beacon)
 {
-	int ret;
-	int bf_enable;
-	int bf_count;
-	int n_filter_ies;
-	struct hif_ie_table_entry filter_ies[] = {
+	const struct hif_ie_table_entry filter_ies[] = {
 		{
 			.ie_id        = WLAN_EID_VENDOR_SPECIFIC,
 			.has_changed  = 1,
@@ -167,40 +83,33 @@ void wfx_update_filtering(struct wfx_vif *wvif)
 		}
 	};
 
-	if (wvif->state == WFX_STATE_PASSIVE)
-		return;
-
-	if (wvif->disable_beacon_filter) {
-		bf_enable = 0;
-		bf_count = 1;
-		n_filter_ies = 0;
-	} else if (wvif->vif->type != NL80211_IFTYPE_STATION) {
-		bf_enable = HIF_BEACON_FILTER_ENABLE | HIF_BEACON_FILTER_AUTO_ERP;
-		bf_count = 0;
-		n_filter_ies = 2;
+	if (!filter_beacon) {
+		hif_beacon_filter_control(wvif, 0, 1);
 	} else {
-		bf_enable = HIF_BEACON_FILTER_ENABLE;
-		bf_count = 0;
-		n_filter_ies = 3;
+		hif_set_beacon_filter_table(wvif, 3, filter_ies);
+		hif_beacon_filter_control(wvif, HIF_BEACON_FILTER_ENABLE, 0);
 	}
-
-	ret = hif_set_rx_filter(wvif, wvif->filter_bssid, wvif->fwd_probe_req);
-	if (!ret)
-		ret = hif_set_beacon_filter_table(wvif, n_filter_ies, filter_ies);
-	if (!ret)
-		ret = hif_beacon_filter_control(wvif, bf_enable, bf_count);
-	if (!ret)
-		ret = wfx_set_mcast_filter(wvif, &wvif->mcast_filter);
-	if (ret)
-		dev_err(wvif->wdev->dev, "update filtering failed: %d\n", ret);
 }
 
-static void wfx_update_filtering_work(struct work_struct *work)
+static void wfx_filter_mcast(struct wfx_vif *wvif, bool filter_mcast)
 {
-	struct wfx_vif *wvif = container_of(work, struct wfx_vif,
-					    update_filtering_work);
+	int i;
 
-	wfx_update_filtering(wvif);
+	// Temporary workaround for filters
+	hif_set_data_filtering(wvif, false, true);
+	return;
+
+	if (!filter_mcast) {
+		hif_set_data_filtering(wvif, false, true);
+		return;
+	}
+	for (i = 0; i < wvif->filter_mcast_count; i++)
+		hif_set_mac_addr_condition(wvif, i, wvif->filter_mcast_addr[i]);
+	hif_set_uc_mc_bc_condition(wvif, 0,
+				   HIF_FILTER_UNICAST | HIF_FILTER_BROADCAST);
+	hif_set_config_data_filter(wvif, true, 0, BIT(1),
+				   BIT(wvif->filter_mcast_count) - 1);
+	hif_set_data_filtering(wvif, true, true);
 }
 
 u64 wfx_prepare_multicast(struct ieee80211_hw *hw,
@@ -213,72 +122,127 @@ u64 wfx_prepare_multicast(struct ieee80211_hw *hw,
 	int count = netdev_hw_addr_list_count(mc_list);
 
 	while ((wvif = wvif_iterate(wdev, wvif)) != NULL) {
-		memset(&wvif->mcast_filter, 0x00, sizeof(wvif->mcast_filter));
-		if (!count ||
-		    count > ARRAY_SIZE(wvif->mcast_filter.address_list))
+		if (count > ARRAY_SIZE(wvif->filter_mcast_addr)) {
+			wvif->filter_mcast_count = 0;
 			continue;
+		}
+		wvif->filter_mcast_count = count;
 
 		i = 0;
 		netdev_hw_addr_list_for_each(ha, mc_list) {
-			ether_addr_copy(wvif->mcast_filter.address_list[i],
-					ha->addr);
+			ether_addr_copy(wvif->filter_mcast_addr[i], ha->addr);
 			i++;
 		}
-		wvif->mcast_filter.enable = true;
-		wvif->mcast_filter.num_addresses = count;
 	}
 
 	return 0;
 }
 
-void wfx_configure_filter(struct ieee80211_hw *hw,
-			     unsigned int changed_flags,
-			     unsigned int *total_flags,
-			     u64 unused)
+void wfx_configure_filter(struct ieee80211_hw *hw, unsigned int changed_flags,
+			  unsigned int *total_flags, u64 unused)
 {
 	struct wfx_vif *wvif = NULL;
 	struct wfx_dev *wdev = hw->priv;
+	bool filter_bssid, filter_prbreq, filter_beacon, filter_mcast;
+
+	// Notes:
+	//   - Probe responses (FIF_BCN_PRBRESP_PROMISC) are never filtered
+	//   - PS-Poll (FIF_PSPOLL) are never filtered
+	//   - RTS, CTS and Ack (FIF_CONTROL) are always filtered
+	//   - Broken frames (FIF_FCSFAIL and FIF_PLCPFAIL) are always filtered
+	//   - Firmware does (yet) allow to forward unicast traffic sent to
+	//     other stations (aka. promiscuous mode)
+	*total_flags &= FIF_BCN_PRBRESP_PROMISC | FIF_ALLMULTI | FIF_OTHER_BSS |
+			FIF_PROBE_REQ | FIF_PSPOLL;
 
-	*total_flags &= FIF_OTHER_BSS | FIF_FCSFAIL | FIF_PROBE_REQ;
-
+	mutex_lock(&wdev->conf_mutex);
 	while ((wvif = wvif_iterate(wdev, wvif)) != NULL) {
 		mutex_lock(&wvif->scan_lock);
-		wvif->filter_bssid = (*total_flags &
-				      (FIF_OTHER_BSS | FIF_PROBE_REQ)) ? 0 : 1;
-		wvif->disable_beacon_filter = !(*total_flags & FIF_PROBE_REQ);
-		wfx_fwd_probe_req(wvif, true);
-		wfx_update_filtering(wvif);
+
+		// Note: FIF_BCN_PRBRESP_PROMISC covers probe response and
+		// beacons from other BSS
+		if (*total_flags & FIF_BCN_PRBRESP_PROMISC)
+			filter_beacon = false;
+		else
+			filter_beacon = true;
+		wfx_filter_beacon(wvif, filter_beacon);
+
+		if (*total_flags & FIF_ALLMULTI) {
+			filter_mcast = false;
+		} else if (!wvif->filter_mcast_count) {
+			dev_dbg(wdev->dev, "disabling unconfigured multicast filter");
+			filter_mcast = false;
+		} else {
+			filter_mcast = true;
+		}
+		wfx_filter_mcast(wvif, filter_mcast);
+
+		if (*total_flags & FIF_OTHER_BSS)
+			filter_bssid = false;
+		else
+			filter_bssid = true;
+
+		// In AP mode, chip can reply to probe request itself
+		if (*total_flags & FIF_PROBE_REQ &&
+		    wvif->vif->type == NL80211_IFTYPE_AP) {
+			dev_dbg(wdev->dev, "do not forward probe request in AP mode\n");
+			*total_flags &= ~FIF_PROBE_REQ;
+		}
+
+		if (*total_flags & FIF_PROBE_REQ)
+			filter_prbreq = false;
+		else
+			filter_prbreq = true;
+		hif_set_rx_filter(wvif, filter_bssid, filter_prbreq);
+
 		mutex_unlock(&wvif->scan_lock);
 	}
+	mutex_unlock(&wdev->conf_mutex);
 }
 
-static int wfx_update_pm(struct wfx_vif *wvif)
+int wfx_get_ps_timeout(struct wfx_vif *wvif, bool *enable_ps)
 {
-	struct ieee80211_conf *conf = &wvif->wdev->hw->conf;
-	bool ps = conf->flags & IEEE80211_CONF_PS;
-	int ps_timeout = conf->dynamic_ps_timeout;
 	struct ieee80211_channel *chan0 = NULL, *chan1 = NULL;
+	struct ieee80211_conf *conf = &wvif->wdev->hw->conf;
 
-	WARN_ON(conf->dynamic_ps_timeout < 0);
-	if (wvif->state != WFX_STATE_STA || !wvif->bss_params.aid)
-		return 0;
-	if (!ps)
-		ps_timeout = 0;
-	if (wvif->uapsd_mask)
-		ps_timeout = 0;
-
-	// Kernel disable powersave when an AP is in use. In contrary, it is
-	// absolutely necessary to enable legacy powersave for WF200 if channels
-	// are differents.
+	WARN(!wvif->vif->bss_conf.assoc && enable_ps,
+	     "enable_ps is reliable only if associated");
 	if (wdev_to_wvif(wvif->wdev, 0))
 		chan0 = wdev_to_wvif(wvif->wdev, 0)->vif->bss_conf.chandef.chan;
 	if (wdev_to_wvif(wvif->wdev, 1))
 		chan1 = wdev_to_wvif(wvif->wdev, 1)->vif->bss_conf.chandef.chan;
 	if (chan0 && chan1 && chan0->hw_value != chan1->hw_value &&
 	    wvif->vif->type != NL80211_IFTYPE_AP) {
-		ps = true;
-		ps_timeout = 0;
+		// It is necessary to enable powersave if channels
+		// are differents.
+		if (enable_ps)
+			*enable_ps = true;
+		if (wvif->bss_not_support_ps_poll)
+			return 30;
+		else
+			return 0;
 	}
+	if (enable_ps)
+		*enable_ps = wvif->vif->bss_conf.ps;
+	if (wvif->vif->bss_conf.assoc && wvif->vif->bss_conf.ps)
+		return conf->dynamic_ps_timeout;
+	else
+		return -1;
+}
+
+int wfx_update_pm(struct wfx_vif *wvif)
+{
+	int ps_timeout;
+	bool ps;
+
+	if (!wvif->vif->bss_conf.assoc)
+		return 0;
+	ps_timeout = wfx_get_ps_timeout(wvif, &ps);
+	if (!ps)
+		ps_timeout = 0;
+	WARN_ON(ps_timeout < 0);
+	if (wvif->uapsd_mask)
+		ps_timeout = 0;
 
 	if (!wait_for_completion_timeout(&wvif->set_pm_mode_complete,
 					 TU_TO_JIFFIES(512)))
@@ -287,18 +251,25 @@ static int wfx_update_pm(struct wfx_vif *wvif)
 	return hif_set_pm(wvif, ps, ps_timeout);
 }
 
+static void wfx_update_pm_work(struct work_struct *work)
+{
+	struct wfx_vif *wvif = container_of(work, struct wfx_vif,
+					    update_pm_work);
+
+	wfx_update_pm(wvif);
+}
+
 int wfx_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		   u16 queue, const struct ieee80211_tx_queue_params *params)
 {
 	struct wfx_dev *wdev = hw->priv;
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 	int old_uapsd = wvif->uapsd_mask;
 
 	WARN_ON(queue >= hw->queues);
 
 	mutex_lock(&wdev->conf_mutex);
 	assign_bit(queue, &wvif->uapsd_mask, params->uapsd);
-	memcpy(&wvif->edca_params[queue], params, sizeof(*params));
 	hif_set_edca_queue_params(wvif, queue, params);
 	if (wvif->vif->type == NL80211_IFTYPE_STATION &&
 	    old_uapsd != wvif->uapsd_mask) {
@@ -319,32 +290,9 @@ int wfx_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
 	return 0;
 }
 
-static int __wfx_flush(struct wfx_dev *wdev, bool drop)
-{
-	for (;;) {
-		if (drop)
-			wfx_tx_queues_clear(wdev);
-		if (wait_event_timeout(wdev->tx_queue_stats.wait_link_id_empty,
-				       wfx_tx_queues_is_empty(wdev),
-				       2 * HZ) <= 0)
-			return -ETIMEDOUT;
-		wfx_tx_flush(wdev);
-		if (wfx_tx_queues_is_empty(wdev))
-			return 0;
-		dev_warn(wdev->dev, "frames queued while flushing tx queues");
-	}
-}
-
-void wfx_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-		  u32 queues, bool drop)
-{
-	// FIXME: only flush requested vif and queues
-	__wfx_flush(hw->priv, drop);
-}
-
 /* WSM callbacks */
 
-static void wfx_event_report_rssi(struct wfx_vif *wvif, u8 raw_rcpi_rssi)
+void wfx_event_report_rssi(struct wfx_vif *wvif, u8 raw_rcpi_rssi)
 {
 	/* RSSI: signed Q8.0, RCPI: unsigned Q7.1
 	 * RSSI = RCPI / 2 - 110
@@ -360,100 +308,23 @@ static void wfx_event_report_rssi(struct wfx_vif *wvif, u8 raw_rcpi_rssi)
 	ieee80211_cqm_rssi_notify(wvif->vif, cqm_evt, rcpi_rssi, GFP_KERNEL);
 }
 
-static void wfx_event_handler_work(struct work_struct *work)
+static void wfx_beacon_loss_work(struct work_struct *work)
 {
-	struct wfx_vif *wvif =
-		container_of(work, struct wfx_vif, event_handler_work);
-	struct wfx_hif_event *event;
-
-	LIST_HEAD(list);
-
-	spin_lock(&wvif->event_queue_lock);
-	list_splice_init(&wvif->event_queue, &list);
-	spin_unlock(&wvif->event_queue_lock);
-
-	list_for_each_entry(event, &list, link) {
-		switch (event->evt.event_id) {
-		case HIF_EVENT_IND_BSSLOST:
-			cancel_work_sync(&wvif->unjoin_work);
-			mutex_lock(&wvif->scan_lock);
-			wfx_cqm_bssloss_sm(wvif, 1, 0, 0);
-			mutex_unlock(&wvif->scan_lock);
-			break;
-		case HIF_EVENT_IND_BSSREGAINED:
-			wfx_cqm_bssloss_sm(wvif, 0, 0, 0);
-			cancel_work_sync(&wvif->unjoin_work);
-			break;
-		case HIF_EVENT_IND_RCPI_RSSI:
-			wfx_event_report_rssi(wvif,
-					      event->evt.event_data.rcpi_rssi);
-			break;
-		case HIF_EVENT_IND_PS_MODE_ERROR:
-			dev_warn(wvif->wdev->dev,
-				 "error while processing power save request\n");
-			break;
-		default:
-			dev_warn(wvif->wdev->dev,
-				 "unhandled event indication: %.2x\n",
-				 event->evt.event_id);
-			break;
-		}
-	}
-	__wfx_free_event_queue(&list);
-}
-
-static void wfx_bss_loss_work(struct work_struct *work)
-{
-	struct wfx_vif *wvif = container_of(work, struct wfx_vif,
-					    bss_loss_work.work);
-
-	ieee80211_connection_loss(wvif->vif);
-}
-
-static void wfx_bss_params_work(struct work_struct *work)
-{
-	struct wfx_vif *wvif = container_of(work, struct wfx_vif,
-					    bss_params_work);
+	struct wfx_vif *wvif = container_of(to_delayed_work(work),
+					    struct wfx_vif, beacon_loss_work);
+	struct ieee80211_bss_conf *bss_conf = &wvif->vif->bss_conf;
 
-	mutex_lock(&wvif->wdev->conf_mutex);
-	wvif->bss_params.bss_flags.lost_count_only = 1;
-	hif_set_bss_params(wvif, &wvif->bss_params);
-	wvif->bss_params.bss_flags.lost_count_only = 0;
-	mutex_unlock(&wvif->wdev->conf_mutex);
+	ieee80211_beacon_loss(wvif->vif);
+	schedule_delayed_work(to_delayed_work(work),
+			      msecs_to_jiffies(bss_conf->beacon_int));
 }
 
-static void wfx_do_unjoin(struct wfx_vif *wvif)
+void wfx_set_default_unicast_key(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif, int idx)
 {
-	mutex_lock(&wvif->wdev->conf_mutex);
-
-	if (!wvif->state)
-		goto done;
-
-	if (wvif->state == WFX_STATE_AP)
-		goto done;
-
-	cancel_work_sync(&wvif->update_filtering_work);
-	wvif->state = WFX_STATE_PASSIVE;
-
-	/* Unjoin is a reset. */
-	wfx_tx_flush(wvif->wdev);
-	hif_keep_alive_period(wvif, 0);
-	hif_reset(wvif, false);
-	wfx_tx_policy_init(wvif);
-	hif_set_macaddr(wvif, wvif->vif->addr);
-	wfx_free_event_queue(wvif);
-	cancel_work_sync(&wvif->event_handler_work);
-	wfx_cqm_bssloss_sm(wvif, 0, 0, 0);
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 
-	/* Disable Block ACKs */
-	hif_set_block_ack_policy(wvif, 0, 0);
-
-	wvif->disable_beacon_filter = false;
-	wfx_update_filtering(wvif);
-	memset(&wvif->bss_params, 0, sizeof(wvif->bss_params));
-
-done:
-	mutex_unlock(&wvif->wdev->conf_mutex);
+	hif_wep_default_key_id(wvif, idx);
 }
 
 static void wfx_set_mfp(struct wfx_vif *wvif,
@@ -472,8 +343,7 @@ static void wfx_set_mfp(struct wfx_vif *wvif,
 
 	rcu_read_lock();
 	if (bss)
-		ptr = (const u16 *) ieee80211_bss_get_ie(bss,
-							      WLAN_EID_RSN);
+		ptr = (const u16 *)ieee80211_bss_get_ie(bss, WLAN_EID_RSN);
 
 	if (ptr) {
 		ptr += pairwise_cipher_suite_count_offset;
@@ -487,6 +357,24 @@ static void wfx_set_mfp(struct wfx_vif *wvif,
 	hif_set_mfp(wvif, mfpc, mfpr);
 }
 
+void wfx_reset(struct wfx_vif *wvif)
+{
+	struct wfx_dev *wdev = wvif->wdev;
+
+	wfx_tx_lock_flush(wdev);
+	hif_reset(wvif, false);
+	wfx_tx_policy_init(wvif);
+	if (wvif_count(wdev) <= 1)
+		hif_set_block_ack_policy(wvif, 0xFF, 0xFF);
+	wfx_tx_unlock(wdev);
+	wvif->join_in_progress = false;
+	wvif->bss_not_support_ps_poll = false;
+	cancel_delayed_work_sync(&wvif->beacon_loss_work);
+	wvif =  NULL;
+	while ((wvif = wvif_iterate(wdev, wvif)) != NULL)
+		wfx_update_pm(wvif);
+}
+
 static void wfx_do_join(struct wfx_vif *wvif)
 {
 	int ret;
@@ -498,9 +386,6 @@ static void wfx_do_join(struct wfx_vif *wvif)
 
 	wfx_tx_lock_flush(wvif->wdev);
 
-	if (wvif->state)
-		wfx_do_unjoin(wvif);
-
 	bss = cfg80211_get_bss(wvif->wdev->hw->wiphy, wvif->channel,
 			       conf->bssid, NULL, 0,
 			       IEEE80211_BSS_TYPE_ANY, IEEE80211_PRIVACY_ANY);
@@ -509,109 +394,72 @@ static void wfx_do_join(struct wfx_vif *wvif)
 		return;
 	}
 
-	mutex_lock(&wvif->wdev->conf_mutex);
-
-	/* Sanity check beacon interval */
-	if (!wvif->beacon_int)
-		wvif->beacon_int = 1;
-
 	rcu_read_lock(); // protect ssidie
-	if (!conf->ibss_joined)
+	if (bss)
 		ssidie = ieee80211_bss_get_ie(bss, WLAN_EID_SSID);
 	if (ssidie) {
 		ssidlen = ssidie[1];
-		memcpy(ssid, &ssidie[2], ssidie[1]);
+		if (ssidlen > IEEE80211_MAX_SSID_LEN)
+			ssidlen = IEEE80211_MAX_SSID_LEN;
+		memcpy(ssid, &ssidie[2], ssidlen);
 	}
 	rcu_read_unlock();
 
-	wfx_tx_flush(wvif->wdev);
-
-	if (wvif_count(wvif->wdev) <= 1)
-		hif_set_block_ack_policy(wvif, 0xFF, 0xFF);
-
 	wfx_set_mfp(wvif, bss);
+	cfg80211_put_bss(wvif->wdev->hw->wiphy, bss);
 
-	wvif->wdev->tx_burst_idx = -1;
+	wvif->join_in_progress = true;
 	ret = hif_join(wvif, conf, wvif->channel, ssid, ssidlen);
 	if (ret) {
 		ieee80211_connection_loss(wvif->vif);
-		wvif->join_complete_status = -1;
-		/* Tx lock still held, unjoin will clear it. */
-		if (!schedule_work(&wvif->unjoin_work))
-			wfx_tx_unlock(wvif->wdev);
+		wfx_reset(wvif);
 	} else {
-		wvif->join_complete_status = 0;
-		if (wvif->vif->type == NL80211_IFTYPE_ADHOC)
-			wvif->state = WFX_STATE_IBSS;
-		else
-			wvif->state = WFX_STATE_PRE_STA;
-		wfx_tx_unlock(wvif->wdev);
-
-		/* Upload keys */
-		wfx_upload_keys(wvif);
-
 		/* Due to beacon filtering it is possible that the
 		 * AP's beacon is not known for the mac80211 stack.
 		 * Disable filtering temporary to make sure the stack
 		 * receives at least one
 		 */
-		wvif->disable_beacon_filter = true;
+		wfx_filter_beacon(wvif, false);
 	}
-	wfx_update_filtering(wvif);
-
-	mutex_unlock(&wvif->wdev->conf_mutex);
-	if (bss)
-		cfg80211_put_bss(wvif->wdev->hw->wiphy, bss);
-}
-
-static void wfx_unjoin_work(struct work_struct *work)
-{
-	struct wfx_vif *wvif = container_of(work, struct wfx_vif, unjoin_work);
-
-	wfx_do_unjoin(wvif);
 	wfx_tx_unlock(wvif->wdev);
 }
 
 int wfx_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		struct ieee80211_sta *sta)
 {
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
-	struct wfx_sta_priv *sta_priv = (struct wfx_sta_priv *) &sta->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
+	struct wfx_sta_priv *sta_priv = (struct wfx_sta_priv *)&sta->drv_priv;
 
 	spin_lock_init(&sta_priv->lock);
 	sta_priv->vif_id = wvif->id;
 
-	// FIXME: in station mode, the current API interprets new link-id as a
-	// tdls peer.
-	if (vif->type == NL80211_IFTYPE_STATION)
+	// In station mode, the firmware interprets new link-id as a TDLS peer.
+	if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls)
 		return 0;
 	sta_priv->link_id = ffz(wvif->link_id_map);
 	wvif->link_id_map |= BIT(sta_priv->link_id);
 	WARN_ON(!sta_priv->link_id);
-	WARN_ON(sta_priv->link_id >= WFX_MAX_STA_IN_AP_MODE);
+	WARN_ON(sta_priv->link_id >= HIF_LINK_ID_MAX);
 	hif_map_link(wvif, sta->addr, 0, sta_priv->link_id);
 
-	spin_lock_bh(&wvif->ps_state_lock);
-	if ((sta->uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_MASK) ==
-					IEEE80211_WMM_IE_STA_QOSINFO_AC_MASK)
-		wvif->sta_asleep_mask |= BIT(sta_priv->link_id);
-	spin_unlock_bh(&wvif->ps_state_lock);
 	return 0;
 }
 
 int wfx_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		   struct ieee80211_sta *sta)
 {
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
-	struct wfx_sta_priv *sta_priv = (struct wfx_sta_priv *) &sta->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
+	struct wfx_sta_priv *sta_priv = (struct wfx_sta_priv *)&sta->drv_priv;
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(sta_priv->buffered); i++)
 		if (sta_priv->buffered[i])
-			dev_warn(wvif->wdev->dev, "release station while %d pending frame on queue %d",
-				 sta_priv->buffered[i], i);
-	// FIXME: see note in wfx_sta_add()
-	if (vif->type == NL80211_IFTYPE_STATION)
+			// Not an error if paired with trace in
+			// wfx_tx_update_sta()
+			dev_dbg(wvif->wdev->dev, "release station while %d pending frame on queue %d",
+				sta_priv->buffered[i], i);
+	// See note in wfx_sta_add()
+	if (!sta_priv->link_id)
 		return 0;
 	// FIXME add a mutex?
 	hif_map_link(wvif, sta->addr, 1, sta_priv->link_id);
@@ -619,50 +467,10 @@ int wfx_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	return 0;
 }
 
-static int wfx_start_ap(struct wfx_vif *wvif)
-{
-	int ret;
-
-	wvif->beacon_int = wvif->vif->bss_conf.beacon_int;
-	wvif->wdev->tx_burst_idx = -1;
-	ret = hif_start(wvif, &wvif->vif->bss_conf, wvif->channel);
-	if (ret)
-		return ret;
-	ret = wfx_upload_keys(wvif);
-	if (ret)
-		return ret;
-	if (wvif_count(wvif->wdev) <= 1)
-		hif_set_block_ack_policy(wvif, 0xFF, 0xFF);
-	wvif->state = WFX_STATE_AP;
-	wfx_update_filtering(wvif);
-	return 0;
-}
-
-static int wfx_update_beaconing(struct wfx_vif *wvif)
-{
-	if (wvif->vif->type != NL80211_IFTYPE_AP)
-		return 0;
-	if (wvif->state == WFX_STATE_AP &&
-	    wvif->beacon_int == wvif->vif->bss_conf.beacon_int)
-		return 0;
-	wfx_tx_lock_flush(wvif->wdev);
-	hif_reset(wvif, false);
-	wfx_tx_policy_init(wvif);
-	wvif->state = WFX_STATE_PASSIVE;
-	wfx_start_ap(wvif);
-	wfx_tx_unlock(wvif->wdev);
-	return 0;
-}
-
 static int wfx_upload_ap_templates(struct wfx_vif *wvif)
 {
 	struct sk_buff *skb;
 
-	if (wvif->vif->type == NL80211_IFTYPE_STATION ||
-	    wvif->vif->type == NL80211_IFTYPE_MONITOR ||
-	    wvif->vif->type == NL80211_IFTYPE_UNSPECIFIED)
-		return 0;
-
 	skb = ieee80211_beacon_get(wvif->wdev->hw, wvif->vif);
 	if (!skb)
 		return -ENOMEM;
@@ -679,52 +487,77 @@ static int wfx_upload_ap_templates(struct wfx_vif *wvif)
 	return 0;
 }
 
+int wfx_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
+	struct wfx_dev *wdev = wvif->wdev;
+	int ret;
+
+	wvif =  NULL;
+	while ((wvif = wvif_iterate(wdev, wvif)) != NULL)
+		wfx_update_pm(wvif);
+	wvif = (struct wfx_vif *)vif->drv_priv;
+	wfx_upload_ap_templates(wvif);
+	ret = hif_start(wvif, &vif->bss_conf, wvif->channel);
+	if (ret > 0)
+		return -EIO;
+	return ret;
+}
+
+void wfx_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
+
+	wfx_reset(wvif);
+}
+
 static void wfx_join_finalize(struct wfx_vif *wvif,
 			      struct ieee80211_bss_conf *info)
 {
-	struct ieee80211_sta *sta = NULL;
-
-	wvif->beacon_int = info->beacon_int;
-	rcu_read_lock(); // protect sta
-	if (info->bssid && !info->ibss_joined)
-		sta = ieee80211_find_sta(wvif->vif, info->bssid);
-	if (sta)
-		wvif->bss_params.operational_rate_set =
-			wfx_rate_mask_to_hw(wvif->wdev, sta->supp_rates[wvif->channel->band]);
-	else
-		wvif->bss_params.operational_rate_set = -1;
-	rcu_read_unlock();
-	if (sta &&
-	    info->ht_operation_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT)
-		hif_dual_cts_protection(wvif, true);
-	else
-		hif_dual_cts_protection(wvif, false);
+	wvif->join_in_progress = false;
+	hif_set_association_mode(wvif, info);
+	hif_keep_alive_period(wvif, 0);
+	// beacon_loss_count is defined to 7 in net/mac80211/mlme.c. Let's use
+	// the same value.
+	hif_set_bss_params(wvif, info->aid, 7);
+	hif_set_beacon_wakeup_period(wvif, 1, 1);
+	wfx_update_pm(wvif);
+}
 
-	wfx_cqm_bssloss_sm(wvif, 0, 0, 0);
-	cancel_work_sync(&wvif->unjoin_work);
+int wfx_join_ibss(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 
-	wvif->bss_params.beacon_lost_count = 20;
-	wvif->bss_params.aid = info->aid;
+	wfx_upload_ap_templates(wvif);
+	wfx_do_join(wvif);
+	return 0;
+}
 
-	hif_set_association_mode(wvif, info);
+void wfx_leave_ibss(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 
-	if (!info->ibss_joined) {
-		hif_keep_alive_period(wvif, 30 /* sec */);
-		hif_set_bss_params(wvif, &wvif->bss_params);
-		hif_set_beacon_wakeup_period(wvif, info->dtim_period,
-					     info->dtim_period);
-		wfx_update_pm(wvif);
+	wfx_reset(wvif);
+}
+
+static void wfx_enable_beacon(struct wfx_vif *wvif, bool enable)
+{
+	// Driver has Content After DTIM Beacon in queue. Driver is waiting for
+	// a signal from the firmware. Since we are going to stop to send
+	// beacons, this signal will never happens. See also
+	// wfx_suspend_resume_mc()
+	if (!enable && wfx_tx_queues_has_cab(wvif)) {
+		wvif->after_dtim_tx_allowed = true;
+		wfx_bh_request_tx(wvif->wdev);
 	}
+	hif_beacon_transmit(wvif, enable);
 }
 
-void wfx_bss_info_changed(struct ieee80211_hw *hw,
-			     struct ieee80211_vif *vif,
-			     struct ieee80211_bss_conf *info,
-			     u32 changed)
+void wfx_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			  struct ieee80211_bss_conf *info, u32 changed)
 {
 	struct wfx_dev *wdev = hw->priv;
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
-	bool do_join = false;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 	int i;
 
 	mutex_lock(&wdev->conf_mutex);
@@ -742,92 +575,52 @@ void wfx_bss_info_changed(struct ieee80211_hw *hw,
 		}
 	}
 
-	if (changed & BSS_CHANGED_BEACON ||
-	    changed & BSS_CHANGED_AP_PROBE_RESP ||
-	    changed & BSS_CHANGED_BSSID ||
-	    changed & BSS_CHANGED_SSID ||
-	    changed & BSS_CHANGED_IBSS) {
-		wvif->beacon_int = info->beacon_int;
-		wfx_update_beaconing(wvif);
-		wfx_upload_ap_templates(wvif);
-		wfx_fwd_probe_req(wvif, false);
+	if (changed & BSS_CHANGED_BASIC_RATES ||
+	    changed & BSS_CHANGED_BEACON_INT ||
+	    changed & BSS_CHANGED_BSSID) {
+		if (vif->type == NL80211_IFTYPE_STATION)
+			wfx_do_join(wvif);
 	}
 
-	if (changed & BSS_CHANGED_BEACON_ENABLED &&
-	    wvif->state != WFX_STATE_IBSS)
-		hif_beacon_transmit(wvif, info->enable_beacon);
+	if (changed & BSS_CHANGED_AP_PROBE_RESP ||
+	    changed & BSS_CHANGED_BEACON)
+		wfx_upload_ap_templates(wvif);
+
+	if (changed & BSS_CHANGED_BEACON_ENABLED)
+		wfx_enable_beacon(wvif, info->enable_beacon);
 
-	if (changed & BSS_CHANGED_BEACON_INFO)
+	if (changed & BSS_CHANGED_BEACON_INFO) {
+		if (vif->type != NL80211_IFTYPE_STATION)
+			dev_warn(wdev->dev, "%s: misunderstood change: BEACON_INFO\n",
+				 __func__);
 		hif_set_beacon_wakeup_period(wvif, info->dtim_period,
 					     info->dtim_period);
-
-	/* assoc/disassoc, or maybe AID changed */
-	if (changed & BSS_CHANGED_ASSOC) {
-		wfx_tx_lock_flush(wdev);
-		wvif->wep_default_key_id = -1;
-		wfx_tx_unlock(wdev);
+		// We temporary forwarded beacon for join process. It is now no
+		// more necessary.
+		wfx_filter_beacon(wvif, true);
 	}
 
-	if (changed & BSS_CHANGED_ASSOC && !info->assoc &&
-	    (wvif->state == WFX_STATE_STA || wvif->state == WFX_STATE_IBSS)) {
-		/* Shedule unjoin work */
-		wfx_tx_lock(wdev);
-		if (!schedule_work(&wvif->unjoin_work))
-			wfx_tx_unlock(wdev);
-	} else {
-		if (changed & BSS_CHANGED_BEACON_INT) {
-			if (info->ibss_joined)
-				do_join = true;
-			else if (wvif->state == WFX_STATE_AP)
-				wfx_update_beaconing(wvif);
-		}
-
-		if (changed & BSS_CHANGED_BSSID)
-			do_join = true;
-
-		if (changed & BSS_CHANGED_ASSOC ||
-		    changed & BSS_CHANGED_BSSID ||
-		    changed & BSS_CHANGED_IBSS ||
-		    changed & BSS_CHANGED_BASIC_RATES ||
-		    changed & BSS_CHANGED_HT) {
-			if (info->assoc) {
-				if (wvif->state < WFX_STATE_PRE_STA) {
-					ieee80211_connection_loss(vif);
-					mutex_unlock(&wdev->conf_mutex);
-					return;
-				} else if (wvif->state == WFX_STATE_PRE_STA) {
-					wvif->state = WFX_STATE_STA;
-				}
-			} else {
-				do_join = true;
-			}
-
-			if (info->assoc || info->ibss_joined)
-				wfx_join_finalize(wvif, info);
-			else
-				memset(&wvif->bss_params, 0,
-				       sizeof(wvif->bss_params));
-		}
+	if (changed & BSS_CHANGED_ASSOC) {
+		if (info->assoc || info->ibss_joined)
+			wfx_join_finalize(wvif, info);
+		else if (!info->assoc && vif->type == NL80211_IFTYPE_STATION)
+			wfx_reset(wvif);
+		else
+			dev_warn(wdev->dev, "%s: misunderstood change: ASSOC\n",
+				 __func__);
 	}
 
-	if (changed & BSS_CHANGED_ASSOC ||
-	    changed & BSS_CHANGED_ERP_CTS_PROT ||
-	    changed & BSS_CHANGED_ERP_PREAMBLE) {
-		u8 erp_ie[3] = { WLAN_EID_ERP_INFO, 1, 0 };
+	if (changed & BSS_CHANGED_KEEP_ALIVE)
+		hif_keep_alive_period(wvif, info->max_idle_period *
+					    USEC_PER_TU / USEC_PER_MSEC);
 
+	if (changed & BSS_CHANGED_ERP_CTS_PROT)
 		hif_erp_use_protection(wvif, info->use_cts_prot);
-		if (info->use_cts_prot)
-			erp_ie[2] |= WLAN_ERP_USE_PROTECTION;
-		if (info->use_short_preamble)
-			erp_ie[2] |= WLAN_ERP_BARKER_PREAMBLE;
-		if (wvif->vif->type != NL80211_IFTYPE_STATION)
-			hif_update_ie_beacon(wvif, erp_ie, sizeof(erp_ie));
-	}
 
-	if (changed & BSS_CHANGED_ASSOC || changed & BSS_CHANGED_ERP_SLOT)
+	if (changed & BSS_CHANGED_ERP_SLOT)
 		hif_slot_time(wvif, info->use_short_slot ? 9 : 20);
 
-	if (changed & BSS_CHANGED_ASSOC || changed & BSS_CHANGED_CQM)
+	if (changed & BSS_CHANGED_CQM)
 		hif_set_rcpi_rssi_threshold(wvif, info->cqm_rssi_thold,
 					    info->cqm_rssi_hyst);
 
@@ -838,31 +631,6 @@ void wfx_bss_info_changed(struct ieee80211_hw *hw,
 		wfx_update_pm(wvif);
 
 	mutex_unlock(&wdev->conf_mutex);
-
-	if (do_join)
-		wfx_do_join(wvif);
-}
-
-static void wfx_ps_notify_sta(struct wfx_vif *wvif,
-			      enum sta_notify_cmd notify_cmd, int link_id)
-{
-	spin_lock_bh(&wvif->ps_state_lock);
-	if (notify_cmd == STA_NOTIFY_SLEEP)
-		wvif->sta_asleep_mask |= BIT(link_id);
-	else // notify_cmd == STA_NOTIFY_AWAKE
-		wvif->sta_asleep_mask &= ~BIT(link_id);
-	spin_unlock_bh(&wvif->ps_state_lock);
-	if (notify_cmd == STA_NOTIFY_AWAKE)
-		wfx_bh_request_tx(wvif->wdev);
-}
-
-void wfx_sta_notify(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-		    enum sta_notify_cmd notify_cmd, struct ieee80211_sta *sta)
-{
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
-	struct wfx_sta_priv *sta_priv = (struct wfx_sta_priv *) &sta->drv_priv;
-
-	wfx_ps_notify_sta(wvif, notify_cmd, sta_priv->link_id);
 }
 
 static int wfx_update_tim(struct wfx_vif *wvif)
@@ -873,10 +641,8 @@ static int wfx_update_tim(struct wfx_vif *wvif)
 
 	skb = ieee80211_beacon_get_tim(wvif->wdev->hw, wvif->vif,
 				       &tim_offset, &tim_length);
-	if (!skb) {
-		__wfx_flush(wvif->wdev, true);
+	if (!skb)
 		return -ENOENT;
-	}
 	tim_ptr = skb->data + tim_offset;
 
 	if (tim_offset && tim_length >= 6) {
@@ -886,7 +652,7 @@ static int wfx_update_tim(struct wfx_vif *wvif)
 		tim_ptr[2] = 0;
 
 		/* Set/reset aid0 bit */
-		if (wfx_tx_queues_get_after_dtim(wvif))
+		if (wfx_tx_queues_has_cab(wvif))
 			tim_ptr[4] |= 1;
 		else
 			tim_ptr[4] &= ~1;
@@ -917,7 +683,9 @@ int wfx_set_tim(struct ieee80211_hw *hw, struct ieee80211_sta *sta, bool set)
 
 void wfx_suspend_resume_mc(struct wfx_vif *wvif, enum sta_notify_cmd notify_cmd)
 {
-	WARN(!wfx_tx_queues_get_after_dtim(wvif), "incorrect sequence");
+	if (notify_cmd != STA_NOTIFY_AWAKE)
+		return;
+	WARN(!wfx_tx_queues_has_cab(wvif), "incorrect sequence");
 	WARN(wvif->after_dtim_tx_allowed, "incorrect sequence");
 	wvif->after_dtim_tx_allowed = true;
 	wfx_bh_request_tx(wvif->wdev);
@@ -958,7 +726,7 @@ void wfx_change_chanctx(struct ieee80211_hw *hw,
 int wfx_assign_vif_chanctx(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			   struct ieee80211_chanctx_conf *conf)
 {
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 	struct ieee80211_channel *ch = conf->def.chan;
 
 	WARN(wvif->channel, "channel overwrite");
@@ -971,7 +739,7 @@ void wfx_unassign_vif_chanctx(struct ieee80211_hw *hw,
 			      struct ieee80211_vif *vif,
 			      struct ieee80211_chanctx_conf *conf)
 {
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 	struct ieee80211_channel *ch = conf->def.chan;
 
 	WARN(wvif->channel != ch, "channel mismatch");
@@ -987,7 +755,7 @@ int wfx_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 {
 	int i, ret = 0;
 	struct wfx_dev *wdev = hw->priv;
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 
 	vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER |
 			     IEEE80211_VIF_SUPPORTS_UAPSD |
@@ -1021,33 +789,18 @@ int wfx_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 	wvif->wdev = wdev;
 
 	wvif->link_id_map = 1; // link-id 0 is reserved for multicast
-	spin_lock_init(&wvif->ps_state_lock);
 	INIT_WORK(&wvif->update_tim_work, wfx_update_tim_work);
-
-	memset(&wvif->bss_params, 0, sizeof(wvif->bss_params));
-
-	mutex_init(&wvif->bss_loss_lock);
-	INIT_DELAYED_WORK(&wvif->bss_loss_work, wfx_bss_loss_work);
-
-	wvif->wep_default_key_id = -1;
-	INIT_WORK(&wvif->wep_key_work, wfx_wep_key_work);
-
-	spin_lock_init(&wvif->event_queue_lock);
-	INIT_LIST_HEAD(&wvif->event_queue);
-	INIT_WORK(&wvif->event_handler_work, wfx_event_handler_work);
+	INIT_DELAYED_WORK(&wvif->beacon_loss_work, wfx_beacon_loss_work);
 
 	init_completion(&wvif->set_pm_mode_complete);
 	complete(&wvif->set_pm_mode_complete);
-	INIT_WORK(&wvif->update_filtering_work, wfx_update_filtering_work);
-	INIT_WORK(&wvif->bss_params_work, wfx_bss_params_work);
-	INIT_WORK(&wvif->unjoin_work, wfx_unjoin_work);
+	INIT_WORK(&wvif->update_pm_work, wfx_update_pm_work);
 	INIT_WORK(&wvif->tx_policy_upload_work, wfx_tx_policy_upload_work);
 
 	mutex_init(&wvif->scan_lock);
 	init_completion(&wvif->scan_complete);
 	INIT_WORK(&wvif->scan_work, wfx_hw_scan_work);
 
-	INIT_WORK(&wvif->tx_policy_upload_work, wfx_tx_policy_upload_work);
 	mutex_unlock(&wdev->conf_mutex);
 
 	hif_set_macaddr(wvif, vif->addr);
@@ -1060,50 +813,25 @@ int wfx_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 			hif_set_block_ack_policy(wvif, 0xFF, 0xFF);
 		else
 			hif_set_block_ack_policy(wvif, 0x00, 0x00);
-		// Combo force powersave mode. We can re-enable it now
-		ret = wfx_update_pm(wvif);
 	}
 	return ret;
 }
 
-void wfx_remove_interface(struct ieee80211_hw *hw,
-			  struct ieee80211_vif *vif)
+void wfx_remove_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
 {
 	struct wfx_dev *wdev = hw->priv;
-	struct wfx_vif *wvif = (struct wfx_vif *) vif->drv_priv;
+	struct wfx_vif *wvif = (struct wfx_vif *)vif->drv_priv;
 
 	wait_for_completion_timeout(&wvif->set_pm_mode_complete, msecs_to_jiffies(300));
 
 	mutex_lock(&wdev->conf_mutex);
 	WARN(wvif->link_id_map != 1, "corrupted state");
-	switch (wvif->state) {
-	case WFX_STATE_PRE_STA:
-	case WFX_STATE_STA:
-	case WFX_STATE_IBSS:
-		wfx_tx_lock_flush(wdev);
-		if (!schedule_work(&wvif->unjoin_work))
-			wfx_tx_unlock(wdev);
-		break;
-	case WFX_STATE_AP:
-		wvif->sta_asleep_mask = 0;
-		/* reset.link_id = 0; */
-		hif_reset(wvif, false);
-		break;
-	default:
-		break;
-	}
-
-	wvif->state = WFX_STATE_PASSIVE;
-	wfx_tx_queues_wait_empty_vif(wvif);
-	wfx_tx_unlock(wdev);
 
-	/* FIXME: In add to reset MAC address, try to reset interface */
+	hif_reset(wvif, false);
 	hif_set_macaddr(wvif, NULL);
+	wfx_tx_policy_init(wvif);
 
-	wfx_cqm_bssloss_sm(wvif, 0, 0, 0);
-	cancel_work_sync(&wvif->unjoin_work);
-	wfx_free_event_queue(wvif);
-
+	cancel_delayed_work_sync(&wvif->beacon_loss_work);
 	wdev->vif[wvif->id] = NULL;
 	wvif->vif = NULL;
 
@@ -1115,8 +843,6 @@ void wfx_remove_interface(struct ieee80211_hw *hw,
 			hif_set_block_ack_policy(wvif, 0xFF, 0xFF);
 		else
 			hif_set_block_ack_policy(wvif, 0x00, 0x00);
-		// Combo force powersave mode. We can re-enable it now
-		wfx_update_pm(wvif);
 	}
 }
 
@@ -1129,10 +855,5 @@ void wfx_stop(struct ieee80211_hw *hw)
 {
 	struct wfx_dev *wdev = hw->priv;
 
-	wfx_tx_lock_flush(wdev);
-	mutex_lock(&wdev->conf_mutex);
-	wfx_tx_queues_clear(wdev);
-	mutex_unlock(&wdev->conf_mutex);
-	wfx_tx_unlock(wdev);
-	WARN(atomic_read(&wdev->tx_lock), "tx_lock is locked");
+	wfx_tx_queues_check_empty(wdev);
 }
diff --git a/drivers/staging/wfx/sta.h b/drivers/staging/wfx/sta.h
index cf99a8a74a81..8a20ad9ae017 100644
--- a/drivers/staging/wfx/sta.h
+++ b/drivers/staging/wfx/sta.h
@@ -10,34 +10,13 @@
 
 #include <net/mac80211.h>
 
-#include "hif_api_cmd.h"
-
 struct wfx_dev;
 struct wfx_vif;
 
-enum wfx_state {
-	WFX_STATE_PASSIVE = 0,
-	WFX_STATE_PRE_STA,
-	WFX_STATE_STA,
-	WFX_STATE_IBSS,
-	WFX_STATE_AP,
-};
-
-struct wfx_hif_event {
-	struct list_head link;
-	struct hif_ind_event evt;
-};
-
-struct wfx_grp_addr_table {
-	bool enable;
-	int num_addresses;
-	u8 address_list[8][ETH_ALEN];
-};
-
 struct wfx_sta_priv {
 	int link_id;
 	int vif_id;
-	u8 buffered[IEEE80211_NUM_TIDS];
+	int buffered[IEEE80211_NUM_TIDS];
 	// Ensure atomicity of "buffered" and calls to ieee80211_sta_set_buffered()
 	spinlock_t lock;
 };
@@ -47,6 +26,8 @@ int wfx_start(struct ieee80211_hw *hw);
 void wfx_stop(struct ieee80211_hw *hw);
 int wfx_config(struct ieee80211_hw *hw, u32 changed);
 int wfx_set_rts_threshold(struct ieee80211_hw *hw, u32 value);
+void wfx_set_default_unicast_key(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif, int idx);
 u64 wfx_prepare_multicast(struct ieee80211_hw *hw,
 			  struct netdev_hw_addr_list *mc_list);
 void wfx_configure_filter(struct ieee80211_hw *hw, unsigned int changed_flags,
@@ -54,8 +35,10 @@ void wfx_configure_filter(struct ieee80211_hw *hw, unsigned int changed_flags,
 
 int wfx_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif);
 void wfx_remove_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif);
-void wfx_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-	       u32 queues, bool drop);
+int wfx_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif);
+void wfx_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif);
+int wfx_join_ibss(struct ieee80211_hw *hw, struct ieee80211_vif *vif);
+void wfx_leave_ibss(struct ieee80211_hw *hw, struct ieee80211_vif *vif);
 int wfx_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		u16 queue, const struct ieee80211_tx_queue_params *params);
 void wfx_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
@@ -82,12 +65,13 @@ void wfx_unassign_vif_chanctx(struct ieee80211_hw *hw,
 			      struct ieee80211_chanctx_conf *conf);
 
 // WSM Callbacks
+void wfx_cooling_timeout_work(struct work_struct *work);
+void wfx_suspend_hot_dev(struct wfx_dev *wdev, enum sta_notify_cmd cmd);
 void wfx_suspend_resume_mc(struct wfx_vif *wvif, enum sta_notify_cmd cmd);
+void wfx_event_report_rssi(struct wfx_vif *wvif, u8 raw_rcpi_rssi);
 
 // Other Helpers
-void wfx_cqm_bssloss_sm(struct wfx_vif *wvif, int init, int good, int bad);
-void wfx_update_filtering(struct wfx_vif *wvif);
-int wfx_fwd_probe_req(struct wfx_vif *wvif, bool enable);
+void wfx_reset(struct wfx_vif *wvif);
 u32 wfx_rate_mask_to_hw(struct wfx_dev *wdev, u32 rates);
 
 #endif /* WFX_STA_H */
diff --git a/drivers/staging/wfx/traces.h b/drivers/staging/wfx/traces.h
index 30c6a13f0e22..0b6fbd518638 100644
--- a/drivers/staging/wfx/traces.h
+++ b/drivers/staging/wfx/traces.h
@@ -32,16 +32,16 @@
  *             xxx_name(XXX)       \
  *             ...
  *
- *   3. Instanciate that list_names:
+ *   3. Instantiate that list_names:
  *
  *          list_names
  *
- *   4. Redefine xxx_name() as a entry of array for __print_symbolic()
+ *   4. Redefine xxx_name() as an entry of array for __print_symbolic()
  *
  *          #undef xxx_name
  *          #define xxx_name(msg) { msg, #msg },
  *
- *   5. list_name can now nearlu be used with __print_symbolic() but,
+ *   5. list_name can now nearly be used with __print_symbolic() but,
  *      __print_symbolic() dislike last comma of list. So we define a new list
  *      with a dummy element:
  *
@@ -104,8 +104,10 @@ hif_msg_list_enum
 	hif_mib_name(ARP_KEEP_ALIVE_PERIOD)          \
 	hif_mib_name(BEACON_FILTER_ENABLE)           \
 	hif_mib_name(BEACON_FILTER_TABLE)            \
+	hif_mib_name(BEACON_STATS)                   \
 	hif_mib_name(BEACON_WAKEUP_PERIOD)           \
 	hif_mib_name(BLOCK_ACK_POLICY)               \
+	hif_mib_name(CCA_CONFIG)                     \
 	hif_mib_name(CONFIG_DATA_FILTER)             \
 	hif_mib_name(COUNTERS_TABLE)                 \
 	hif_mib_name(CURRENT_TX_POWER_LEVEL)         \
@@ -114,29 +116,32 @@ hif_msg_list_enum
 	hif_mib_name(DOT11_MAX_TRANSMIT_MSDU_LIFETIME) \
 	hif_mib_name(DOT11_RTS_THRESHOLD)            \
 	hif_mib_name(DOT11_WEP_DEFAULT_KEY_ID)       \
+	hif_mib_name(ETHERTYPE_DATAFRAME_CONDITION)  \
+	hif_mib_name(EXTENDED_COUNTERS_TABLE)        \
 	hif_mib_name(GL_BLOCK_ACK_INFO)              \
 	hif_mib_name(GL_OPERATIONAL_POWER_MODE)      \
 	hif_mib_name(GL_SET_MULTI_MSG)               \
+	hif_mib_name(GRP_SEQ_COUNTER)                \
 	hif_mib_name(INACTIVITY_TIMER)               \
 	hif_mib_name(INTERFACE_PROTECTION)           \
 	hif_mib_name(IPV4_ADDR_DATAFRAME_CONDITION)  \
 	hif_mib_name(IPV6_ADDR_DATAFRAME_CONDITION)  \
 	hif_mib_name(KEEP_ALIVE_PERIOD)              \
 	hif_mib_name(MAC_ADDR_DATAFRAME_CONDITION)   \
+	hif_mib_name(MAGIC_DATAFRAME_CONDITION)      \
+	hif_mib_name(MAX_TX_POWER_LEVEL)             \
 	hif_mib_name(NON_ERP_PROTECTION)             \
 	hif_mib_name(NS_IP_ADDRESSES_TABLE)          \
 	hif_mib_name(OVERRIDE_INTERNAL_TX_RATE)      \
+	hif_mib_name(PORT_DATAFRAME_CONDITION)       \
 	hif_mib_name(PROTECTED_MGMT_POLICY)          \
-	hif_mib_name(RX_FILTER)                      \
 	hif_mib_name(RCPI_RSSI_THRESHOLD)            \
+	hif_mib_name(RX_FILTER)                      \
 	hif_mib_name(SET_ASSOCIATION_MODE)           \
 	hif_mib_name(SET_DATA_FILTERING)             \
-	hif_mib_name(ETHERTYPE_DATAFRAME_CONDITION)  \
 	hif_mib_name(SET_HT_PROTECTION)              \
-	hif_mib_name(MAGIC_DATAFRAME_CONDITION)      \
 	hif_mib_name(SET_TX_RATE_RETRY_POLICY)       \
 	hif_mib_name(SET_UAPSD_INFORMATION)          \
-	hif_mib_name(PORT_DATAFRAME_CONDITION)       \
 	hif_mib_name(SLOT_TIME)                      \
 	hif_mib_name(STATISTICS_TABLE)               \
 	hif_mib_name(TEMPLATE_FRAME)                 \
@@ -169,7 +174,7 @@ DECLARE_EVENT_CLASS(hif_data,
 		int header_len;
 
 		__entry->tx_fill_level = tx_fill_level;
-		__entry->msg_len = hif->len;
+		__entry->msg_len = le16_to_cpu(hif->len);
 		__entry->msg_id = hif->id;
 		__entry->if_id = hif->interface;
 		if (is_recv)
@@ -179,7 +184,7 @@ DECLARE_EVENT_CLASS(hif_data,
 		if (!is_recv &&
 		    (__entry->msg_id == HIF_REQ_ID_READ_MIB ||
 		     __entry->msg_id == HIF_REQ_ID_WRITE_MIB)) {
-			__entry->mib = le16_to_cpup((u16 *) hif->body);
+			__entry->mib = le16_to_cpup((__le16 *)hif->body);
 			header_len = 4;
 		} else {
 			__entry->mib = -1;
@@ -193,8 +198,8 @@ DECLARE_EVENT_CLASS(hif_data,
 	TP_printk("%d:%d:%s_%s%s%s: %s%s (%d bytes)",
 		__entry->tx_fill_level,
 		__entry->if_id,
-		__print_symbolic(__entry->msg_id, hif_msg_list),
 		__entry->msg_type,
+		__print_symbolic(__entry->msg_id, hif_msg_list),
 		__entry->mib != -1 ? "/" : "",
 		__entry->mib != -1 ? __print_symbolic(__entry->mib, hif_mib_list) : "",
 		__print_hex(__entry->buf, __entry->buf_len),
@@ -382,8 +387,8 @@ TRACE_EVENT(tx_stats,
 		int i;
 
 		__entry->pkt_id = tx_cnf->packet_id;
-		__entry->delay_media = tx_cnf->media_delay;
-		__entry->delay_queue = tx_cnf->tx_queue_delay;
+		__entry->delay_media = le32_to_cpu(tx_cnf->media_delay);
+		__entry->delay_queue = le32_to_cpu(tx_cnf->tx_queue_delay);
 		__entry->delay_fw = delay;
 		__entry->ack_failures = tx_cnf->ack_failures;
 		if (!tx_cnf->status || __entry->ack_failures)
@@ -409,7 +414,7 @@ TRACE_EVENT(tx_stats,
 			__entry->flags |= 0x10;
 		if (tx_cnf->status)
 			__entry->flags |= 0x20;
-		if (tx_cnf->status == HIF_REQUEUE)
+		if (tx_cnf->status == HIF_STATUS_TX_FAIL_REQUEUE)
 			__entry->flags |= 0x40;
 	),
 	TP_printk("packet ID: %08x, rate policy: %s %d|%d %d|%d %d|%d %d|%d -> %d attempt, Delays media/queue/total: %4dus/%4dus/%4dus",
diff --git a/drivers/staging/wfx/wfx.h b/drivers/staging/wfx/wfx.h
index 8b85bb1abb9c..73e216733ce4 100644
--- a/drivers/staging/wfx/wfx.h
+++ b/drivers/staging/wfx/wfx.h
@@ -21,10 +21,7 @@
 #include "main.h"
 #include "queue.h"
 #include "secure_link.h"
-#include "sta.h"
-#include "scan.h"
 #include "hif_tx.h"
-#include "hif_api_general.h"
 
 #define USEC_PER_TXOP 32 // see struct ieee80211_tx_queue_params
 #define USEC_PER_TU 1024
@@ -45,21 +42,24 @@ struct wfx_dev {
 	struct hif_ind_startup	hw_caps;
 	struct wfx_hif		hif;
 	struct sl_context	sl;
-	int			chip_frozen;
+	struct delayed_work	cooling_timeout_work;
+	bool			poll_irq;
+	bool			chip_frozen;
 	struct mutex		conf_mutex;
 
 	struct wfx_hif_cmd	hif_cmd;
 	struct wfx_queue	tx_queue[4];
-	struct wfx_queue_stats	tx_queue_stats;
-	int			tx_burst_idx;
+	struct sk_buff_head	tx_pending;
+	wait_queue_head_t	tx_dequeue;
 	atomic_t		tx_lock;
 
 	atomic_t		packet_id;
 	u32			key_map;
-	struct hif_req_add_key	keys[MAX_KEY_ENTRIES];
 
 	struct hif_rx_stats	rx_stats;
 	struct mutex		rx_stats_lock;
+	struct hif_tx_power_loop_info tx_power_loop_info;
+	struct mutex		tx_power_loop_info_lock;
 };
 
 struct wfx_vif {
@@ -67,42 +67,23 @@ struct wfx_vif {
 	struct ieee80211_vif	*vif;
 	struct ieee80211_channel *channel;
 	int			id;
-	enum wfx_state		state;
-
-	int			bss_loss_state;
-	u32			bss_loss_confirm_id;
-	struct mutex		bss_loss_lock;
-	struct delayed_work	bss_loss_work;
 
 	u32			link_id_map;
 
 	bool			after_dtim_tx_allowed;
-	struct wfx_grp_addr_table mcast_filter;
+	bool			join_in_progress;
 
-	s8			wep_default_key_id;
-	struct sk_buff		*wep_pending_skb;
-	struct work_struct	wep_key_work;
+	struct delayed_work	beacon_loss_work;
 
 	struct tx_policy_cache	tx_policy_cache;
 	struct work_struct	tx_policy_upload_work;
 
-	u32			sta_asleep_mask;
-	spinlock_t		ps_state_lock;
 	struct work_struct	update_tim_work;
 
-	int			beacon_int;
-	bool			filter_bssid;
-	bool			fwd_probe_req;
-	bool			disable_beacon_filter;
-	struct work_struct	update_filtering_work;
+	int			filter_mcast_count;
+	u8			filter_mcast_addr[8][ETH_ALEN];
 
 	unsigned long		uapsd_mask;
-	struct ieee80211_tx_queue_params edca_params[IEEE80211_NUM_ACS];
-	struct hif_req_set_bss_params bss_params;
-	struct work_struct	bss_params_work;
-
-	int			join_complete_status;
-	struct work_struct	unjoin_work;
 
 	/* avoid some operations in parallel with scan */
 	struct mutex		scan_lock;
@@ -111,11 +92,9 @@ struct wfx_vif {
 	bool			scan_abort;
 	struct ieee80211_scan_request *scan_req;
 
+	bool			bss_not_support_ps_poll;
+	struct work_struct	update_pm_work;
 	struct completion	set_pm_mode_complete;
-
-	struct list_head	event_queue;
-	spinlock_t		event_queue_lock;
-	struct work_struct	event_handler_work;
 };
 
 static inline struct wfx_vif *wdev_to_wvif(struct wfx_dev *wdev, int vif_id)
diff --git a/drivers/staging/wilc1000/hif.c b/drivers/staging/wilc1000/hif.c
index 6c7de2f8d3f2..d025a3093015 100644
--- a/drivers/staging/wilc1000/hif.c
+++ b/drivers/staging/wilc1000/hif.c
@@ -11,6 +11,8 @@
 
 #define WILC_FALSE_FRMWR_CHANNEL		100
 
+#define WILC_SCAN_WID_LIST_SIZE		6
+
 struct wilc_rcvd_mac_info {
 	u8 status;
 };
@@ -151,7 +153,7 @@ int wilc_scan(struct wilc_vif *vif, u8 scan_source, u8 scan_type,
 	      void *user_arg, struct cfg80211_scan_request *request)
 {
 	int result = 0;
-	struct wid wid_list[5];
+	struct wid wid_list[WILC_SCAN_WID_LIST_SIZE];
 	u32 index = 0;
 	u32 i, scan_timeout;
 	u8 *buffer;
diff --git a/drivers/target/iscsi/cxgbit/Kconfig b/drivers/target/iscsi/cxgbit/Kconfig
index 8686dbd1d2e7..bdeefa75f29e 100644
--- a/drivers/target/iscsi/cxgbit/Kconfig
+++ b/drivers/target/iscsi/cxgbit/Kconfig
@@ -3,6 +3,6 @@ config ISCSI_TARGET_CXGB4
 	tristate "Chelsio iSCSI target offload driver"
 	depends on ISCSI_TARGET && CHELSIO_T4 && INET
 	select CHELSIO_LIB
-	---help---
+	help
 	To compile this driver as module, choose M here: the module
 	will be called cxgbit.
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index 59379d662626..c9689610e186 100644
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -1158,7 +1158,7 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 	transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
 			conn->sess->se_sess, be32_to_cpu(hdr->data_length),
 			cmd->data_direction, sam_task_attr,
-			cmd->sense_buffer + 2);
+			cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun));
 
 	pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
 		" ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
@@ -1167,23 +1167,26 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 
 	target_get_sess_cmd(&cmd->se_cmd, true);
 
-	cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd,
-						     scsilun_to_int(&hdr->lun));
-	if (cmd->sense_reason)
-		goto attach_cmd;
-
-	/* only used for printks or comparing with ->ref_task_tag */
-	cmd->se_cmd.tag = (__force u32)cmd->init_task_tag;
-	cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb);
+	cmd->sense_reason = target_cmd_init_cdb(&cmd->se_cmd, hdr->cdb);
 	if (cmd->sense_reason) {
 		if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) {
 			return iscsit_add_reject_cmd(cmd,
-					ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
+				ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
 		}
 
 		goto attach_cmd;
 	}
 
+	cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd);
+	if (cmd->sense_reason)
+		goto attach_cmd;
+
+	/* only used for printks or comparing with ->ref_task_tag */
+	cmd->se_cmd.tag = (__force u32)cmd->init_task_tag;
+	cmd->sense_reason = target_cmd_parse_cdb(&cmd->se_cmd);
+	if (cmd->sense_reason)
+		goto attach_cmd;
+
 	if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) {
 		return iscsit_add_reject_cmd(cmd,
 				ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
@@ -2000,7 +2003,8 @@ iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 
 	transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
 			      conn->sess->se_sess, 0, DMA_NONE,
-			      TCM_SIMPLE_TAG, cmd->sense_buffer + 2);
+			      TCM_SIMPLE_TAG, cmd->sense_buffer + 2,
+			      scsilun_to_int(&hdr->lun));
 
 	target_get_sess_cmd(&cmd->se_cmd, true);
 
@@ -2038,8 +2042,7 @@ iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 	 * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
 	 */
 	if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
-		ret = transport_lookup_tmr_lun(&cmd->se_cmd,
-					       scsilun_to_int(&hdr->lun));
+		ret = transport_lookup_tmr_lun(&cmd->se_cmd);
 		if (ret < 0) {
 			se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
 			goto attach;
diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c
index 89183b3b178f..45ba07c6ec27 100644
--- a/drivers/target/iscsi/iscsi_target_util.c
+++ b/drivers/target/iscsi/iscsi_target_util.c
@@ -1228,18 +1228,20 @@ void iscsit_print_session_params(struct iscsi_session *sess)
 	iscsi_dump_sess_ops(sess->sess_ops);
 }
 
-static int iscsit_do_rx_data(
+int rx_data(
 	struct iscsi_conn *conn,
-	struct iscsi_data_count *count)
+	struct kvec *iov,
+	int iov_count,
+	int data)
 {
-	int data = count->data_length, rx_loop = 0, total_rx = 0;
+	int rx_loop = 0, total_rx = 0;
 	struct msghdr msg;
 
 	if (!conn || !conn->sock || !conn->conn_ops)
 		return -1;
 
 	memset(&msg, 0, sizeof(struct msghdr));
-	iov_iter_kvec(&msg.msg_iter, READ, count->iov, count->iov_count, data);
+	iov_iter_kvec(&msg.msg_iter, READ, iov, iov_count, data);
 
 	while (msg_data_left(&msg)) {
 		rx_loop = sock_recvmsg(conn->sock, &msg, MSG_WAITALL);
@@ -1256,26 +1258,6 @@ static int iscsit_do_rx_data(
 	return total_rx;
 }
 
-int rx_data(
-	struct iscsi_conn *conn,
-	struct kvec *iov,
-	int iov_count,
-	int data)
-{
-	struct iscsi_data_count c;
-
-	if (!conn || !conn->sock || !conn->conn_ops)
-		return -1;
-
-	memset(&c, 0, sizeof(struct iscsi_data_count));
-	c.iov = iov;
-	c.iov_count = iov_count;
-	c.data_length = data;
-	c.type = ISCSI_RX_DATA;
-
-	return iscsit_do_rx_data(conn, &c);
-}
-
 int tx_data(
 	struct iscsi_conn *conn,
 	struct kvec *iov,
diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index 3305b47fdf53..16d5a4e117a2 100644
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -545,32 +545,15 @@ static int tcm_loop_write_pending(struct se_cmd *se_cmd)
 	return 0;
 }
 
-static int tcm_loop_queue_data_in(struct se_cmd *se_cmd)
+static int tcm_loop_queue_data_or_status(const char *func,
+		struct se_cmd *se_cmd, u8 scsi_status)
 {
 	struct tcm_loop_cmd *tl_cmd = container_of(se_cmd,
 				struct tcm_loop_cmd, tl_se_cmd);
 	struct scsi_cmnd *sc = tl_cmd->sc;
 
 	pr_debug("%s() called for scsi_cmnd: %p cdb: 0x%02x\n",
-		 __func__, sc, sc->cmnd[0]);
-
-	sc->result = SAM_STAT_GOOD;
-	set_host_byte(sc, DID_OK);
-	if ((se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) ||
-	    (se_cmd->se_cmd_flags & SCF_UNDERFLOW_BIT))
-		scsi_set_resid(sc, se_cmd->residual_count);
-	sc->scsi_done(sc);
-	return 0;
-}
-
-static int tcm_loop_queue_status(struct se_cmd *se_cmd)
-{
-	struct tcm_loop_cmd *tl_cmd = container_of(se_cmd,
-				struct tcm_loop_cmd, tl_se_cmd);
-	struct scsi_cmnd *sc = tl_cmd->sc;
-
-	pr_debug("%s() called for scsi_cmnd: %p cdb: 0x%02x\n",
-		 __func__, sc, sc->cmnd[0]);
+		 func, sc, sc->cmnd[0]);
 
 	if (se_cmd->sense_buffer &&
 	   ((se_cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
@@ -581,7 +564,7 @@ static int tcm_loop_queue_status(struct se_cmd *se_cmd)
 		sc->result = SAM_STAT_CHECK_CONDITION;
 		set_driver_byte(sc, DRIVER_SENSE);
 	} else
-		sc->result = se_cmd->scsi_status;
+		sc->result = scsi_status;
 
 	set_host_byte(sc, DID_OK);
 	if ((se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) ||
@@ -591,6 +574,17 @@ static int tcm_loop_queue_status(struct se_cmd *se_cmd)
 	return 0;
 }
 
+static int tcm_loop_queue_data_in(struct se_cmd *se_cmd)
+{
+	return tcm_loop_queue_data_or_status(__func__, se_cmd, SAM_STAT_GOOD);
+}
+
+static int tcm_loop_queue_status(struct se_cmd *se_cmd)
+{
+	return tcm_loop_queue_data_or_status(__func__,
+					     se_cmd, se_cmd->scsi_status);
+}
+
 static void tcm_loop_queue_tm_rsp(struct se_cmd *se_cmd)
 {
 	struct tcm_loop_cmd *tl_cmd = container_of(se_cmd,
diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c
index 385e4cf9cfa6..6b72afee2f8b 100644
--- a/drivers/target/target_core_alua.c
+++ b/drivers/target/target_core_alua.c
@@ -677,7 +677,7 @@ target_alua_state_check(struct se_cmd *cmd)
 
 	if (dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE)
 		return 0;
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA)
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA)
 		return 0;
 
 	/*
@@ -1090,7 +1090,7 @@ int core_alua_do_port_transition(
 	struct t10_alua_tg_pt_gp *tg_pt_gp;
 	int primary, valid_states, rc = 0;
 
-	if (l_dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA)
+	if (l_dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA)
 		return -ENODEV;
 
 	valid_states = l_tg_pt_gp->tg_pt_gp_alua_supported_states;
@@ -1920,7 +1920,7 @@ ssize_t core_alua_store_tg_pt_gp_info(
 	unsigned char buf[TG_PT_GROUP_NAME_BUF];
 	int move = 0;
 
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA ||
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA ||
 	    (dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE))
 		return -ENODEV;
 
@@ -2177,7 +2177,7 @@ ssize_t core_alua_store_offline_bit(
 	unsigned long tmp;
 	int ret;
 
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA ||
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA ||
 	    (dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE))
 		return -ENODEV;
 
@@ -2263,7 +2263,7 @@ ssize_t core_alua_store_secondary_write_metadata(
 
 int core_setup_alua(struct se_device *dev)
 {
-	if (!(dev->transport->transport_flags &
+	if (!(dev->transport_flags &
 	     TRANSPORT_FLAG_PASSTHROUGH_ALUA) &&
 	    !(dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE)) {
 		struct t10_alua_lu_gp_member *lu_gp_mem;
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index ff82b21fdcce..f04352285155 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -1099,21 +1099,73 @@ static ssize_t block_size_store(struct config_item *item,
 static ssize_t alua_support_show(struct config_item *item, char *page)
 {
 	struct se_dev_attrib *da = to_attrib(item);
-	u8 flags = da->da_dev->transport->transport_flags;
+	u8 flags = da->da_dev->transport_flags;
 
 	return snprintf(page, PAGE_SIZE, "%d\n",
 			flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA ? 0 : 1);
 }
 
+static ssize_t alua_support_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct se_dev_attrib *da = to_attrib(item);
+	struct se_device *dev = da->da_dev;
+	bool flag;
+	int ret;
+
+	if (!(dev->transport->transport_flags_changeable &
+	      TRANSPORT_FLAG_PASSTHROUGH_ALUA)) {
+		pr_err("dev[%p]: Unable to change SE Device alua_support:"
+			" alua_support has fixed value\n", dev);
+		return -EINVAL;
+	}
+
+	ret = strtobool(page, &flag);
+	if (ret < 0)
+		return ret;
+
+	if (flag)
+		dev->transport_flags &= ~TRANSPORT_FLAG_PASSTHROUGH_ALUA;
+	else
+		dev->transport_flags |= TRANSPORT_FLAG_PASSTHROUGH_ALUA;
+	return count;
+}
+
 static ssize_t pgr_support_show(struct config_item *item, char *page)
 {
 	struct se_dev_attrib *da = to_attrib(item);
-	u8 flags = da->da_dev->transport->transport_flags;
+	u8 flags = da->da_dev->transport_flags;
 
 	return snprintf(page, PAGE_SIZE, "%d\n",
 			flags & TRANSPORT_FLAG_PASSTHROUGH_PGR ? 0 : 1);
 }
 
+static ssize_t pgr_support_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct se_dev_attrib *da = to_attrib(item);
+	struct se_device *dev = da->da_dev;
+	bool flag;
+	int ret;
+
+	if (!(dev->transport->transport_flags_changeable &
+	      TRANSPORT_FLAG_PASSTHROUGH_PGR)) {
+		pr_err("dev[%p]: Unable to change SE Device pgr_support:"
+			" pgr_support has fixed value\n", dev);
+		return -EINVAL;
+	}
+
+	ret = strtobool(page, &flag);
+	if (ret < 0)
+		return ret;
+
+	if (flag)
+		dev->transport_flags &= ~TRANSPORT_FLAG_PASSTHROUGH_PGR;
+	else
+		dev->transport_flags |= TRANSPORT_FLAG_PASSTHROUGH_PGR;
+	return count;
+}
+
 CONFIGFS_ATTR(, emulate_model_alias);
 CONFIGFS_ATTR(, emulate_dpo);
 CONFIGFS_ATTR(, emulate_fua_write);
@@ -1146,8 +1198,8 @@ CONFIGFS_ATTR(, unmap_granularity);
 CONFIGFS_ATTR(, unmap_granularity_alignment);
 CONFIGFS_ATTR(, unmap_zeroes_data);
 CONFIGFS_ATTR(, max_write_same_len);
-CONFIGFS_ATTR_RO(, alua_support);
-CONFIGFS_ATTR_RO(, pgr_support);
+CONFIGFS_ATTR(, alua_support);
+CONFIGFS_ATTR(, pgr_support);
 
 /*
  * dev_attrib attributes for devices using the target core SBC/SPC
@@ -1203,12 +1255,24 @@ struct configfs_attribute *passthrough_attrib_attrs[] = {
 	&attr_hw_block_size,
 	&attr_hw_max_sectors,
 	&attr_hw_queue_depth,
+	&attr_emulate_pr,
 	&attr_alua_support,
 	&attr_pgr_support,
 	NULL,
 };
 EXPORT_SYMBOL(passthrough_attrib_attrs);
 
+/*
+ * pr related dev_attrib attributes for devices passing through CDBs,
+ * but allowing in core pr emulation.
+ */
+struct configfs_attribute *passthrough_pr_attrib_attrs[] = {
+	&attr_enforce_pr_isids,
+	&attr_force_pr_aptpl,
+	NULL,
+};
+EXPORT_SYMBOL(passthrough_pr_attrib_attrs);
+
 TB_CIT_SETUP_DRV(dev_attrib, NULL, NULL);
 TB_CIT_SETUP_DRV(dev_action, NULL, NULL);
 
@@ -1642,7 +1706,7 @@ static ssize_t target_pr_res_holder_show(struct config_item *item, char *page)
 	if (!dev->dev_attrib.emulate_pr)
 		return sprintf(page, "SPC_RESERVATIONS_DISABLED\n");
 
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
 		return sprintf(page, "Passthrough\n");
 
 	spin_lock(&dev->dev_reservation_lock);
@@ -1784,7 +1848,7 @@ static ssize_t target_pr_res_type_show(struct config_item *item, char *page)
 
 	if (!dev->dev_attrib.emulate_pr)
 		return sprintf(page, "SPC_RESERVATIONS_DISABLED\n");
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
 		return sprintf(page, "SPC_PASSTHROUGH\n");
 	if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS)
 		return sprintf(page, "SPC2_RESERVATIONS\n");
@@ -1798,7 +1862,7 @@ static ssize_t target_pr_res_aptpl_active_show(struct config_item *item,
 	struct se_device *dev = pr_to_dev(item);
 
 	if (!dev->dev_attrib.emulate_pr ||
-	    (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR))
+	    (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR))
 		return 0;
 
 	return sprintf(page, "APTPL Bit Status: %s\n",
@@ -1811,7 +1875,7 @@ static ssize_t target_pr_res_aptpl_metadata_show(struct config_item *item,
 	struct se_device *dev = pr_to_dev(item);
 
 	if (!dev->dev_attrib.emulate_pr ||
-	    (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR))
+	    (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR))
 		return 0;
 
 	return sprintf(page, "Ready to process PR APTPL metadata..\n");
@@ -1858,7 +1922,7 @@ static ssize_t target_pr_res_aptpl_metadata_store(struct config_item *item,
 	u8 type = 0;
 
 	if (!dev->dev_attrib.emulate_pr ||
-	    (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR))
+	    (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR))
 		return count;
 	if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS)
 		return count;
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index 4cee1138284b..405d82d44717 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -45,7 +45,7 @@ static struct se_hba *lun0_hba;
 struct se_device *g_lun0_dev;
 
 sense_reason_t
-transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
+transport_lookup_cmd_lun(struct se_cmd *se_cmd)
 {
 	struct se_lun *se_lun = NULL;
 	struct se_session *se_sess = se_cmd->se_sess;
@@ -54,7 +54,7 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
 	sense_reason_t ret = TCM_NO_SENSE;
 
 	rcu_read_lock();
-	deve = target_nacl_find_deve(nacl, unpacked_lun);
+	deve = target_nacl_find_deve(nacl, se_cmd->orig_fe_lun);
 	if (deve) {
 		atomic_long_inc(&deve->total_cmds);
 
@@ -74,7 +74,6 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
 
 		se_cmd->se_lun = se_lun;
 		se_cmd->pr_res_key = deve->pr_res_key;
-		se_cmd->orig_fe_lun = unpacked_lun;
 		se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
 		se_cmd->lun_ref_active = true;
 
@@ -83,7 +82,7 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
 			pr_err("TARGET_CORE[%s]: Detected WRITE_PROTECTED LUN"
 				" Access for 0x%08llx\n",
 				se_cmd->se_tfo->fabric_name,
-				unpacked_lun);
+				se_cmd->orig_fe_lun);
 			rcu_read_unlock();
 			ret = TCM_WRITE_PROTECTED;
 			goto ref_dev;
@@ -98,17 +97,17 @@ out_unlock:
 		 * REPORT_LUNS, et al to be returned when no active
 		 * MappedLUN=0 exists for this Initiator Port.
 		 */
-		if (unpacked_lun != 0) {
+		if (se_cmd->orig_fe_lun != 0) {
 			pr_err("TARGET_CORE[%s]: Detected NON_EXISTENT_LUN"
-				" Access for 0x%08llx\n",
+				" Access for 0x%08llx from %s\n",
 				se_cmd->se_tfo->fabric_name,
-				unpacked_lun);
+				se_cmd->orig_fe_lun,
+				nacl->initiatorname);
 			return TCM_NON_EXISTENT_LUN;
 		}
 
 		se_lun = se_sess->se_tpg->tpg_virt_lun0;
 		se_cmd->se_lun = se_sess->se_tpg->tpg_virt_lun0;
-		se_cmd->orig_fe_lun = 0;
 		se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
 
 		percpu_ref_get(&se_lun->lun_ref);
@@ -144,7 +143,7 @@ ref_dev:
 }
 EXPORT_SYMBOL(transport_lookup_cmd_lun);
 
-int transport_lookup_tmr_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
+int transport_lookup_tmr_lun(struct se_cmd *se_cmd)
 {
 	struct se_dev_entry *deve;
 	struct se_lun *se_lun = NULL;
@@ -154,7 +153,7 @@ int transport_lookup_tmr_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
 	unsigned long flags;
 
 	rcu_read_lock();
-	deve = target_nacl_find_deve(nacl, unpacked_lun);
+	deve = target_nacl_find_deve(nacl, se_cmd->orig_fe_lun);
 	if (deve) {
 		se_lun = rcu_dereference(deve->se_lun);
 
@@ -165,7 +164,6 @@ int transport_lookup_tmr_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
 
 		se_cmd->se_lun = se_lun;
 		se_cmd->pr_res_key = deve->pr_res_key;
-		se_cmd->orig_fe_lun = unpacked_lun;
 		se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
 		se_cmd->lun_ref_active = true;
 	}
@@ -174,9 +172,10 @@ out_unlock:
 
 	if (!se_lun) {
 		pr_debug("TARGET_CORE[%s]: Detected NON_EXISTENT_LUN"
-			" Access for 0x%08llx\n",
+			" Access for 0x%08llx for %s\n",
 			se_cmd->se_tfo->fabric_name,
-			unpacked_lun);
+			se_cmd->orig_fe_lun,
+			nacl->initiatorname);
 		return -ENODEV;
 	}
 	se_cmd->se_dev = rcu_dereference_raw(se_lun->lun_se_dev);
@@ -732,6 +731,7 @@ struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
 
 	dev->se_hba = hba;
 	dev->transport = hba->backend->ops;
+	dev->transport_flags = dev->transport->transport_flags_default;
 	dev->prot_length = sizeof(struct t10_pi_tuple);
 	dev->hba_index = hba->hba_index;
 
@@ -1100,7 +1100,7 @@ passthrough_parse_cdb(struct se_cmd *cmd,
 	 * emulate the response, since tcmu does not have the information
 	 * required to process these commands.
 	 */
-	if (!(dev->transport->transport_flags &
+	if (!(dev->transport_flags &
 	      TRANSPORT_FLAG_PASSTHROUGH_PGR)) {
 		if (cdb[0] == PERSISTENT_RESERVE_IN) {
 			cmd->execute_cmd = target_scsi3_emulate_pr_in;
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 5e931690e697..91e41cc55704 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -4086,7 +4086,7 @@ target_check_reservation(struct se_cmd *cmd)
 		return 0;
 	if (!dev->dev_attrib.emulate_pr)
 		return 0;
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_PGR)
 		return 0;
 
 	spin_lock(&dev->dev_reservation_lock);
diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c
index c9d92b3e777d..4e37fa9b409d 100644
--- a/drivers/target/target_core_pscsi.c
+++ b/drivers/target/target_core_pscsi.c
@@ -1070,9 +1070,9 @@ static void pscsi_req_done(struct request *req, blk_status_t status)
 static const struct target_backend_ops pscsi_ops = {
 	.name			= "pscsi",
 	.owner			= THIS_MODULE,
-	.transport_flags	= TRANSPORT_FLAG_PASSTHROUGH |
-				  TRANSPORT_FLAG_PASSTHROUGH_ALUA |
-				  TRANSPORT_FLAG_PASSTHROUGH_PGR,
+	.transport_flags_default = TRANSPORT_FLAG_PASSTHROUGH |
+				   TRANSPORT_FLAG_PASSTHROUGH_ALUA |
+				   TRANSPORT_FLAG_PASSTHROUGH_PGR,
 	.attach_hba		= pscsi_attach_hba,
 	.detach_hba		= pscsi_detach_hba,
 	.pmode_enable_hba	= pscsi_pmode_enable_hba,
diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c
index afbd492c76a9..89c84d472cd7 100644
--- a/drivers/target/target_core_tmr.c
+++ b/drivers/target/target_core_tmr.c
@@ -148,8 +148,8 @@ void core_tmr_abort_task(
 		 * code.
 		 */
 		if (!tmr->tmr_dev)
-			WARN_ON_ONCE(transport_lookup_tmr_lun(tmr->task_cmd,
-						se_cmd->orig_fe_lun) < 0);
+			WARN_ON_ONCE(transport_lookup_tmr_lun(tmr->task_cmd) <
+					0);
 
 		target_put_cmd_and_wait(se_cmd);
 
diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c
index d24e0a3ba3ff..62aa5fa63ac0 100644
--- a/drivers/target/target_core_tpg.c
+++ b/drivers/target/target_core_tpg.c
@@ -582,8 +582,7 @@ int core_tpg_add_lun(
 	if (ret)
 		goto out_kill_ref;
 
-	if (!(dev->transport->transport_flags &
-	     TRANSPORT_FLAG_PASSTHROUGH_ALUA) &&
+	if (!(dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH_ALUA) &&
 	    !(dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE))
 		target_attach_tg_pt_gp(lun, dev->t10_alua.default_tg_pt_gp);
 
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 264a822c0bfa..90ecdd706a01 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -1364,7 +1364,7 @@ void transport_init_se_cmd(
 	u32 data_length,
 	int data_direction,
 	int task_attr,
-	unsigned char *sense_buffer)
+	unsigned char *sense_buffer, u64 unpacked_lun)
 {
 	INIT_LIST_HEAD(&cmd->se_delayed_node);
 	INIT_LIST_HEAD(&cmd->se_qf_node);
@@ -1383,6 +1383,7 @@ void transport_init_se_cmd(
 	cmd->data_direction = data_direction;
 	cmd->sam_task_attr = task_attr;
 	cmd->sense_buffer = sense_buffer;
+	cmd->orig_fe_lun = unpacked_lun;
 
 	cmd->state_active = false;
 }
@@ -1397,7 +1398,7 @@ transport_check_alloc_task_attr(struct se_cmd *cmd)
 	 * Check if SAM Task Attribute emulation is enabled for this
 	 * struct se_device storage object
 	 */
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
 		return 0;
 
 	if (cmd->sam_task_attr == TCM_ACA_TAG) {
@@ -1410,11 +1411,11 @@ transport_check_alloc_task_attr(struct se_cmd *cmd)
 }
 
 sense_reason_t
-target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
+target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb)
 {
-	struct se_device *dev = cmd->se_dev;
 	sense_reason_t ret;
 
+	cmd->t_task_cdb = &cmd->__t_task_cdb[0];
 	/*
 	 * Ensure that the received CDB is less than the max (252 + 8) bytes
 	 * for VARIABLE_LENGTH_CMD
@@ -1423,7 +1424,8 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
 		pr_err("Received SCSI CDB with command_size: %d that"
 			" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
 			scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
-		return TCM_INVALID_CDB_FIELD;
+		ret = TCM_INVALID_CDB_FIELD;
+		goto err;
 	}
 	/*
 	 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
@@ -1438,16 +1440,34 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
 				" %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
 				scsi_command_size(cdb),
 				(unsigned long)sizeof(cmd->__t_task_cdb));
-			return TCM_OUT_OF_RESOURCES;
+			ret = TCM_OUT_OF_RESOURCES;
+			goto err;
 		}
-	} else
-		cmd->t_task_cdb = &cmd->__t_task_cdb[0];
+	}
 	/*
 	 * Copy the original CDB into cmd->
 	 */
 	memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
 
 	trace_target_sequencer_start(cmd);
+	return 0;
+
+err:
+	/*
+	 * Copy the CDB here to allow trace_target_cmd_complete() to
+	 * print the cdb to the trace buffers.
+	 */
+	memcpy(cmd->t_task_cdb, cdb, min(scsi_command_size(cdb),
+					 (unsigned int)TCM_MAX_COMMAND_SIZE));
+	return ret;
+}
+EXPORT_SYMBOL(target_cmd_init_cdb);
+
+sense_reason_t
+target_cmd_parse_cdb(struct se_cmd *cmd)
+{
+	struct se_device *dev = cmd->se_dev;
+	sense_reason_t ret;
 
 	ret = dev->transport->parse_cdb(cmd);
 	if (ret == TCM_UNSUPPORTED_SCSI_OPCODE)
@@ -1466,7 +1486,7 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
 	atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus);
 	return 0;
 }
-EXPORT_SYMBOL(target_setup_cmd_from_cdb);
+EXPORT_SYMBOL(target_cmd_parse_cdb);
 
 /*
  * Used by fabric module frontends to queue tasks directly.
@@ -1588,7 +1608,8 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess
 	 * target_core_fabric_ops->queue_status() callback
 	 */
 	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
-				data_length, data_dir, task_attr, sense);
+				data_length, data_dir, task_attr, sense,
+				unpacked_lun);
 
 	if (flags & TARGET_SCF_USE_CPUID)
 		se_cmd->se_cmd_flags |= SCF_USE_CPUID;
@@ -1611,17 +1632,25 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess
 	 */
 	if (flags & TARGET_SCF_BIDI_OP)
 		se_cmd->se_cmd_flags |= SCF_BIDI;
+
+	rc = target_cmd_init_cdb(se_cmd, cdb);
+	if (rc) {
+		transport_send_check_condition_and_sense(se_cmd, rc, 0);
+		target_put_sess_cmd(se_cmd);
+		return 0;
+	}
+
 	/*
 	 * Locate se_lun pointer and attach it to struct se_cmd
 	 */
-	rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
+	rc = transport_lookup_cmd_lun(se_cmd);
 	if (rc) {
 		transport_send_check_condition_and_sense(se_cmd, rc, 0);
 		target_put_sess_cmd(se_cmd);
 		return 0;
 	}
 
-	rc = target_setup_cmd_from_cdb(se_cmd, cdb);
+	rc = target_cmd_parse_cdb(se_cmd);
 	if (rc != 0) {
 		transport_generic_request_failure(se_cmd, rc);
 		return 0;
@@ -1782,7 +1811,7 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
 	BUG_ON(!se_tpg);
 
 	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
-			      0, DMA_NONE, TCM_SIMPLE_TAG, sense);
+			      0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun);
 	/*
 	 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
 	 * allocation failure.
@@ -1810,7 +1839,7 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
 			goto failure;
 	}
 
-	ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
+	ret = transport_lookup_tmr_lun(se_cmd);
 	if (ret)
 		goto failure;
 
@@ -2012,7 +2041,7 @@ static bool target_handle_task_attr(struct se_cmd *cmd)
 {
 	struct se_device *dev = cmd->se_dev;
 
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
 		return false;
 
 	cmd->se_cmd_flags |= SCF_TASK_ATTR_SET;
@@ -2126,7 +2155,7 @@ static void transport_complete_task_attr(struct se_cmd *cmd)
 {
 	struct se_device *dev = cmd->se_dev;
 
-	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
+	if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
 		return;
 
 	if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET))
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index f769bb1e3735..560bfec933bc 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -882,41 +882,24 @@ static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd,
 	return command_size;
 }
 
-static int tcmu_setup_cmd_timer(struct tcmu_cmd *tcmu_cmd, unsigned int tmo,
-				struct timer_list *timer)
+static void tcmu_setup_cmd_timer(struct tcmu_cmd *tcmu_cmd, unsigned int tmo,
+				 struct timer_list *timer)
 {
-	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
-	int cmd_id;
-
-	if (tcmu_cmd->cmd_id)
-		goto setup_timer;
-
-	cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 1, USHRT_MAX, GFP_NOWAIT);
-	if (cmd_id < 0) {
-		pr_err("tcmu: Could not allocate cmd id.\n");
-		return cmd_id;
-	}
-	tcmu_cmd->cmd_id = cmd_id;
-
-	pr_debug("allocated cmd %u for dev %s tmo %lu\n", tcmu_cmd->cmd_id,
-		 udev->name, tmo / MSEC_PER_SEC);
-
-setup_timer:
 	if (!tmo)
-		return 0;
+		return;
 
 	tcmu_cmd->deadline = round_jiffies_up(jiffies + msecs_to_jiffies(tmo));
 	if (!timer_pending(timer))
 		mod_timer(timer, tcmu_cmd->deadline);
 
-	return 0;
+	pr_debug("Timeout set up for cmd %p, dev = %s, tmo = %lu\n", tcmu_cmd,
+		 tcmu_cmd->tcmu_dev->name, tmo / MSEC_PER_SEC);
 }
 
 static int add_to_qfull_queue(struct tcmu_cmd *tcmu_cmd)
 {
 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
 	unsigned int tmo;
-	int ret;
 
 	/*
 	 * For backwards compat if qfull_time_out is not set use
@@ -931,13 +914,11 @@ static int add_to_qfull_queue(struct tcmu_cmd *tcmu_cmd)
 	else
 		tmo = TCMU_TIME_OUT;
 
-	ret = tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer);
-	if (ret)
-		return ret;
+	tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer);
 
 	list_add_tail(&tcmu_cmd->queue_entry, &udev->qfull_queue);
-	pr_debug("adding cmd %u on dev %s to ring space wait queue\n",
-		 tcmu_cmd->cmd_id, udev->name);
+	pr_debug("adding cmd %p on dev %s to ring space wait queue\n",
+		 tcmu_cmd, udev->name);
 	return 0;
 }
 
@@ -959,7 +940,7 @@ static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
 	struct tcmu_mailbox *mb;
 	struct tcmu_cmd_entry *entry;
 	struct iovec *iov;
-	int iov_cnt, ret;
+	int iov_cnt, cmd_id;
 	uint32_t cmd_head;
 	uint64_t cdb_off;
 	bool copy_to_data_area;
@@ -1026,7 +1007,7 @@ static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
 		entry->hdr.cmd_id = 0; /* not used for PAD */
 		entry->hdr.kflags = 0;
 		entry->hdr.uflags = 0;
-		tcmu_flush_dcache_range(entry, sizeof(*entry));
+		tcmu_flush_dcache_range(entry, sizeof(entry->hdr));
 
 		UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
 		tcmu_flush_dcache_range(mb, sizeof(*mb));
@@ -1060,14 +1041,21 @@ static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
 	}
 	entry->req.iov_bidi_cnt = iov_cnt;
 
-	ret = tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out,
-				   &udev->cmd_timer);
-	if (ret) {
-		tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
+	cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 1, USHRT_MAX, GFP_NOWAIT);
+	if (cmd_id < 0) {
+		pr_err("tcmu: Could not allocate cmd id.\n");
 
+		tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
 		*scsi_err = TCM_OUT_OF_RESOURCES;
 		return -1;
 	}
+	tcmu_cmd->cmd_id = cmd_id;
+
+	pr_debug("allocated cmd id %u for cmd %p dev %s\n", tcmu_cmd->cmd_id,
+		 tcmu_cmd, udev->name);
+
+	tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out, &udev->cmd_timer);
+
 	entry->hdr.cmd_id = tcmu_cmd->cmd_id;
 
 	/*
@@ -1084,7 +1072,7 @@ static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
 	cdb_off = CMDR_OFF + cmd_head + base_command_size;
 	memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb));
 	entry->req.cdb_off = cdb_off;
-	tcmu_flush_dcache_range(entry, sizeof(*entry));
+	tcmu_flush_dcache_range(entry, command_size);
 
 	UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);
 	tcmu_flush_dcache_range(mb, sizeof(*mb));
@@ -1279,50 +1267,39 @@ static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
 	return handled;
 }
 
-static int tcmu_check_expired_cmd(int id, void *p, void *data)
+static void tcmu_check_expired_ring_cmd(struct tcmu_cmd *cmd)
 {
-	struct tcmu_cmd *cmd = p;
-	struct tcmu_dev *udev = cmd->tcmu_dev;
-	u8 scsi_status;
 	struct se_cmd *se_cmd;
-	bool is_running;
-
-	if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags))
-		return 0;
 
 	if (!time_after(jiffies, cmd->deadline))
-		return 0;
+		return;
 
-	is_running = test_bit(TCMU_CMD_BIT_INFLIGHT, &cmd->flags);
+	set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
+	list_del_init(&cmd->queue_entry);
 	se_cmd = cmd->se_cmd;
+	cmd->se_cmd = NULL;
 
-	if (is_running) {
-		/*
-		 * If cmd_time_out is disabled but qfull is set deadline
-		 * will only reflect the qfull timeout. Ignore it.
-		 */
-		if (!udev->cmd_time_out)
-			return 0;
+	pr_debug("Timing out inflight cmd %u on dev %s.\n",
+		 cmd->cmd_id, cmd->tcmu_dev->name);
 
-		set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
-		/*
-		 * target_complete_cmd will translate this to LUN COMM FAILURE
-		 */
-		scsi_status = SAM_STAT_CHECK_CONDITION;
-		list_del_init(&cmd->queue_entry);
-		cmd->se_cmd = NULL;
-	} else {
-		list_del_init(&cmd->queue_entry);
-		idr_remove(&udev->commands, id);
-		tcmu_free_cmd(cmd);
-		scsi_status = SAM_STAT_TASK_SET_FULL;
-	}
+	target_complete_cmd(se_cmd, SAM_STAT_CHECK_CONDITION);
+}
 
-	pr_debug("Timing out cmd %u on dev %s that is %s.\n",
-		 id, udev->name, is_running ? "inflight" : "queued");
+static void tcmu_check_expired_queue_cmd(struct tcmu_cmd *cmd)
+{
+	struct se_cmd *se_cmd;
 
-	target_complete_cmd(se_cmd, scsi_status);
-	return 0;
+	if (!time_after(jiffies, cmd->deadline))
+		return;
+
+	pr_debug("Timing out queued cmd %p on dev %s.\n",
+		  cmd, cmd->tcmu_dev->name);
+
+	list_del_init(&cmd->queue_entry);
+	se_cmd = cmd->se_cmd;
+	tcmu_free_cmd(cmd);
+
+	target_complete_cmd(se_cmd, SAM_STAT_TASK_SET_FULL);
 }
 
 static void tcmu_device_timedout(struct tcmu_dev *udev)
@@ -1407,16 +1384,15 @@ static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name)
 	return &udev->se_dev;
 }
 
-static bool run_qfull_queue(struct tcmu_dev *udev, bool fail)
+static void run_qfull_queue(struct tcmu_dev *udev, bool fail)
 {
 	struct tcmu_cmd *tcmu_cmd, *tmp_cmd;
 	LIST_HEAD(cmds);
-	bool drained = true;
 	sense_reason_t scsi_ret;
 	int ret;
 
 	if (list_empty(&udev->qfull_queue))
-		return true;
+		return;
 
 	pr_debug("running %s's cmdr queue forcefail %d\n", udev->name, fail);
 
@@ -1425,11 +1401,10 @@ static bool run_qfull_queue(struct tcmu_dev *udev, bool fail)
 	list_for_each_entry_safe(tcmu_cmd, tmp_cmd, &cmds, queue_entry) {
 		list_del_init(&tcmu_cmd->queue_entry);
 
-	        pr_debug("removing cmd %u on dev %s from queue\n",
-		         tcmu_cmd->cmd_id, udev->name);
+		pr_debug("removing cmd %p on dev %s from queue\n",
+			 tcmu_cmd, udev->name);
 
 		if (fail) {
-			idr_remove(&udev->commands, tcmu_cmd->cmd_id);
 			/*
 			 * We were not able to even start the command, so
 			 * fail with busy to allow a retry in case runner
@@ -1444,10 +1419,8 @@ static bool run_qfull_queue(struct tcmu_dev *udev, bool fail)
 
 		ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
 		if (ret < 0) {
-		        pr_debug("cmd %u on dev %s failed with %u\n",
-			         tcmu_cmd->cmd_id, udev->name, scsi_ret);
-
-			idr_remove(&udev->commands, tcmu_cmd->cmd_id);
+			pr_debug("cmd %p on dev %s failed with %u\n",
+				 tcmu_cmd, udev->name, scsi_ret);
 			/*
 			 * Ignore scsi_ret for now. target_complete_cmd
 			 * drops it.
@@ -1462,13 +1435,11 @@ static bool run_qfull_queue(struct tcmu_dev *udev, bool fail)
 			 * the queue
 			 */
 			list_splice_tail(&cmds, &udev->qfull_queue);
-			drained = false;
 			break;
 		}
 	}
 
 	tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
-	return drained;
 }
 
 static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on)
@@ -1652,6 +1623,8 @@ static void tcmu_dev_kref_release(struct kref *kref)
 		if (tcmu_check_and_free_pending_cmd(cmd) != 0)
 			all_expired = false;
 	}
+	if (!list_empty(&udev->qfull_queue))
+		all_expired = false;
 	idr_destroy(&udev->commands);
 	WARN_ON(!all_expired);
 
@@ -2037,9 +2010,6 @@ static void tcmu_reset_ring(struct tcmu_dev *udev, u8 err_level)
 	mutex_lock(&udev->cmdr_lock);
 
 	idr_for_each_entry(&udev->commands, cmd, i) {
-		if (!test_bit(TCMU_CMD_BIT_INFLIGHT, &cmd->flags))
-			continue;
-
 		pr_debug("removing cmd %u on dev %s from ring (is expired %d)\n",
 			  cmd->cmd_id, udev->name,
 			  test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags));
@@ -2077,6 +2047,8 @@ static void tcmu_reset_ring(struct tcmu_dev *udev, u8 err_level)
 
 	del_timer(&udev->cmd_timer);
 
+	run_qfull_queue(udev, false);
+
 	mutex_unlock(&udev->cmdr_lock);
 }
 
@@ -2617,7 +2589,9 @@ static struct configfs_attribute *tcmu_action_attrs[] = {
 static struct target_backend_ops tcmu_ops = {
 	.name			= "user",
 	.owner			= THIS_MODULE,
-	.transport_flags	= TRANSPORT_FLAG_PASSTHROUGH,
+	.transport_flags_default = TRANSPORT_FLAG_PASSTHROUGH,
+	.transport_flags_changeable = TRANSPORT_FLAG_PASSTHROUGH_PGR |
+				      TRANSPORT_FLAG_PASSTHROUGH_ALUA,
 	.attach_hba		= tcmu_attach_hba,
 	.detach_hba		= tcmu_detach_hba,
 	.alloc_device		= tcmu_alloc_device,
@@ -2698,6 +2672,7 @@ static void find_free_blocks(void)
 static void check_timedout_devices(void)
 {
 	struct tcmu_dev *udev, *tmp_dev;
+	struct tcmu_cmd *cmd, *tmp_cmd;
 	LIST_HEAD(devs);
 
 	spin_lock_bh(&timed_out_udevs_lock);
@@ -2708,9 +2683,24 @@ static void check_timedout_devices(void)
 		spin_unlock_bh(&timed_out_udevs_lock);
 
 		mutex_lock(&udev->cmdr_lock);
-		idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL);
 
-		tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer);
+		/*
+		 * If cmd_time_out is disabled but qfull is set deadline
+		 * will only reflect the qfull timeout. Ignore it.
+		 */
+		if (udev->cmd_time_out) {
+			list_for_each_entry_safe(cmd, tmp_cmd,
+						 &udev->inflight_queue,
+						 queue_entry) {
+				tcmu_check_expired_ring_cmd(cmd);
+			}
+			tcmu_set_next_deadline(&udev->inflight_queue,
+					       &udev->cmd_timer);
+		}
+		list_for_each_entry_safe(cmd, tmp_cmd, &udev->qfull_queue,
+					 queue_entry) {
+			tcmu_check_expired_queue_cmd(cmd);
+		}
 		tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
 
 		mutex_unlock(&udev->cmdr_lock);
@@ -2753,12 +2743,12 @@ static int __init tcmu_module_init(void)
 		goto out_unreg_device;
 	}
 
-	for (i = 0; passthrough_attrib_attrs[i] != NULL; i++) {
+	for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
 		len += sizeof(struct configfs_attribute *);
-	}
-	for (i = 0; tcmu_attrib_attrs[i] != NULL; i++) {
+	for (i = 0; passthrough_pr_attrib_attrs[i] != NULL; i++)
+		len += sizeof(struct configfs_attribute *);
+	for (i = 0; tcmu_attrib_attrs[i] != NULL; i++)
 		len += sizeof(struct configfs_attribute *);
-	}
 	len += sizeof(struct configfs_attribute *);
 
 	tcmu_attrs = kzalloc(len, GFP_KERNEL);
@@ -2767,13 +2757,12 @@ static int __init tcmu_module_init(void)
 		goto out_unreg_genl;
 	}
 
-	for (i = 0; passthrough_attrib_attrs[i] != NULL; i++) {
+	for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
 		tcmu_attrs[i] = passthrough_attrib_attrs[i];
-	}
-	for (k = 0; tcmu_attrib_attrs[k] != NULL; k++) {
-		tcmu_attrs[i] = tcmu_attrib_attrs[k];
-		i++;
-	}
+	for (k = 0; passthrough_pr_attrib_attrs[k] != NULL; k++)
+		tcmu_attrs[i++] = passthrough_pr_attrib_attrs[k];
+	for (k = 0; tcmu_attrib_attrs[k] != NULL; k++)
+		tcmu_attrs[i++] = tcmu_attrib_attrs[k];
 	tcmu_ops.tb_dev_attrib_attrs = tcmu_attrs;
 
 	ret = transport_backend_register(&tcmu_ops);
diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c
index bd3ed6ce7571..0d00ccbeb050 100644
--- a/drivers/target/target_core_xcopy.c
+++ b/drivers/target/target_core_xcopy.c
@@ -526,8 +526,11 @@ static int target_xcopy_setup_pt_cmd(
 	}
 	cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
 
+	if (target_cmd_init_cdb(cmd, cdb))
+		return -EINVAL;
+
 	cmd->tag = 0;
-	if (target_setup_cmd_from_cdb(cmd, cdb))
+	if (target_cmd_parse_cdb(cmd))
 		return -EINVAL;
 
 	if (transport_generic_map_mem_to_cmd(cmd, xop->xop_data_sg,
@@ -585,7 +588,7 @@ static int target_xcopy_read_source(
 		(unsigned long long)src_lba, src_sectors, length);
 
 	transport_init_se_cmd(se_cmd, &xcopy_pt_tfo, &xcopy_pt_sess, length,
-			      DMA_FROM_DEVICE, 0, &xpt_cmd.sense_buffer[0]);
+			      DMA_FROM_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0);
 
 	rc = target_xcopy_setup_pt_cmd(&xpt_cmd, xop, src_dev, &cdb[0],
 				remote_port);
@@ -630,7 +633,7 @@ static int target_xcopy_write_destination(
 		(unsigned long long)dst_lba, dst_sectors, length);
 
 	transport_init_se_cmd(se_cmd, &xcopy_pt_tfo, &xcopy_pt_sess, length,
-			      DMA_TO_DEVICE, 0, &xpt_cmd.sense_buffer[0]);
+			      DMA_TO_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0);
 
 	rc = target_xcopy_setup_pt_cmd(&xpt_cmd, xop, dst_dev, &cdb[0],
 				remote_port);
diff --git a/drivers/tee/Kconfig b/drivers/tee/Kconfig
index 8da63f38e6bd..e99d840c2511 100644
--- a/drivers/tee/Kconfig
+++ b/drivers/tee/Kconfig
@@ -3,6 +3,8 @@
 config TEE
 	tristate "Trusted Execution Environment support"
 	depends on HAVE_ARM_SMCCC || COMPILE_TEST || CPU_SUP_AMD
+	select CRYPTO
+	select CRYPTO_SHA1
 	select DMA_SHARED_BUFFER
 	select GENERIC_ALLOCATOR
 	help
diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c
index cf2367ba08d6..20b6fd7383c5 100644
--- a/drivers/tee/optee/call.c
+++ b/drivers/tee/optee/call.c
@@ -233,9 +233,13 @@ int optee_open_session(struct tee_context *ctx,
 	msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT |
 				  OPTEE_MSG_ATTR_META;
 	memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid));
-	memcpy(&msg_arg->params[1].u.value, arg->uuid, sizeof(arg->clnt_uuid));
 	msg_arg->params[1].u.value.c = arg->clnt_login;
 
+	rc = tee_session_calc_client_uuid((uuid_t *)&msg_arg->params[1].u.value,
+					  arg->clnt_login, arg->clnt_uuid);
+	if (rc)
+		goto out;
+
 	rc = optee_to_msg_param(msg_arg->params + 2, arg->num_params, param);
 	if (rc)
 		goto out;
@@ -561,10 +565,10 @@ static int check_mem_type(unsigned long start, size_t num_pages)
 	if (virt_addr_valid(start))
 		return 0;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	rc = __check_mem_type(find_vma(mm, start),
 			      start + num_pages * PAGE_SIZE);
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 
 	return rc;
 }
diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
index 6aec502c495c..64637e09a095 100644
--- a/drivers/tee/tee_core.c
+++ b/drivers/tee/tee_core.c
@@ -6,18 +6,33 @@
 #define pr_fmt(fmt) "%s: " fmt, __func__
 
 #include <linux/cdev.h>
+#include <linux/cred.h>
 #include <linux/fs.h>
 #include <linux/idr.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/tee_drv.h>
 #include <linux/uaccess.h>
+#include <crypto/hash.h>
+#include <crypto/sha.h>
 #include "tee_private.h"
 
 #define TEE_NUM_DEVICES	32
 
 #define TEE_IOCTL_PARAM_SIZE(x) (sizeof(struct tee_param) * (x))
 
+#define TEE_UUID_NS_NAME_SIZE	128
+
+/*
+ * TEE Client UUID name space identifier (UUIDv4)
+ *
+ * Value here is random UUID that is allocated as name space identifier for
+ * forming Client UUID's for TEE environment using UUIDv5 scheme.
+ */
+static const uuid_t tee_client_uuid_ns = UUID_INIT(0x58ac9ca0, 0x2086, 0x4683,
+						   0xa1, 0xb8, 0xec, 0x4b,
+						   0xc0, 0x8e, 0x01, 0xb6);
+
 /*
  * Unprivileged devices in the lower half range and privileged devices in
  * the upper half range.
@@ -110,6 +125,143 @@ static int tee_release(struct inode *inode, struct file *filp)
 	return 0;
 }
 
+/**
+ * uuid_v5() - Calculate UUIDv5
+ * @uuid: Resulting UUID
+ * @ns: Name space ID for UUIDv5 function
+ * @name: Name for UUIDv5 function
+ * @size: Size of name
+ *
+ * UUIDv5 is specific in RFC 4122.
+ *
+ * This implements section (for SHA-1):
+ * 4.3.  Algorithm for Creating a Name-Based UUID
+ */
+static int uuid_v5(uuid_t *uuid, const uuid_t *ns, const void *name,
+		   size_t size)
+{
+	unsigned char hash[SHA1_DIGEST_SIZE];
+	struct crypto_shash *shash = NULL;
+	struct shash_desc *desc = NULL;
+	int rc;
+
+	shash = crypto_alloc_shash("sha1", 0, 0);
+	if (IS_ERR(shash)) {
+		rc = PTR_ERR(shash);
+		pr_err("shash(sha1) allocation failed\n");
+		return rc;
+	}
+
+	desc = kzalloc(sizeof(*desc) + crypto_shash_descsize(shash),
+		       GFP_KERNEL);
+	if (!desc) {
+		rc = -ENOMEM;
+		goto out_free_shash;
+	}
+
+	desc->tfm = shash;
+
+	rc = crypto_shash_init(desc);
+	if (rc < 0)
+		goto out_free_desc;
+
+	rc = crypto_shash_update(desc, (const u8 *)ns, sizeof(*ns));
+	if (rc < 0)
+		goto out_free_desc;
+
+	rc = crypto_shash_update(desc, (const u8 *)name, size);
+	if (rc < 0)
+		goto out_free_desc;
+
+	rc = crypto_shash_final(desc, hash);
+	if (rc < 0)
+		goto out_free_desc;
+
+	memcpy(uuid->b, hash, UUID_SIZE);
+
+	/* Tag for version 5 */
+	uuid->b[6] = (hash[6] & 0x0F) | 0x50;
+	uuid->b[8] = (hash[8] & 0x3F) | 0x80;
+
+out_free_desc:
+	kfree(desc);
+
+out_free_shash:
+	crypto_free_shash(shash);
+	return rc;
+}
+
+int tee_session_calc_client_uuid(uuid_t *uuid, u32 connection_method,
+				 const u8 connection_data[TEE_IOCTL_UUID_LEN])
+{
+	gid_t ns_grp = (gid_t)-1;
+	kgid_t grp = INVALID_GID;
+	char *name = NULL;
+	int name_len;
+	int rc;
+
+	if (connection_method == TEE_IOCTL_LOGIN_PUBLIC) {
+		/* Nil UUID to be passed to TEE environment */
+		uuid_copy(uuid, &uuid_null);
+		return 0;
+	}
+
+	/*
+	 * In Linux environment client UUID is based on UUIDv5.
+	 *
+	 * Determine client UUID with following semantics for 'name':
+	 *
+	 * For TEEC_LOGIN_USER:
+	 * uid=<uid>
+	 *
+	 * For TEEC_LOGIN_GROUP:
+	 * gid=<gid>
+	 *
+	 */
+
+	name = kzalloc(TEE_UUID_NS_NAME_SIZE, GFP_KERNEL);
+	if (!name)
+		return -ENOMEM;
+
+	switch (connection_method) {
+	case TEE_IOCTL_LOGIN_USER:
+		name_len = snprintf(name, TEE_UUID_NS_NAME_SIZE, "uid=%x",
+				    current_euid().val);
+		if (name_len >= TEE_UUID_NS_NAME_SIZE) {
+			rc = -E2BIG;
+			goto out_free_name;
+		}
+		break;
+
+	case TEE_IOCTL_LOGIN_GROUP:
+		memcpy(&ns_grp, connection_data, sizeof(gid_t));
+		grp = make_kgid(current_user_ns(), ns_grp);
+		if (!gid_valid(grp) || !in_egroup_p(grp)) {
+			rc = -EPERM;
+			goto out_free_name;
+		}
+
+		name_len = snprintf(name, TEE_UUID_NS_NAME_SIZE, "gid=%x",
+				    grp.val);
+		if (name_len >= TEE_UUID_NS_NAME_SIZE) {
+			rc = -E2BIG;
+			goto out_free_name;
+		}
+		break;
+
+	default:
+		rc = -EINVAL;
+		goto out_free_name;
+	}
+
+	rc = uuid_v5(uuid, &tee_client_uuid_ns, name, name_len);
+out_free_name:
+	kfree(name);
+
+	return rc;
+}
+EXPORT_SYMBOL_GPL(tee_session_calc_client_uuid);
+
 static int tee_ioctl_version(struct tee_context *ctx,
 			     struct tee_ioctl_version_data __user *uvers)
 {
@@ -333,6 +485,13 @@ static int tee_ioctl_open_session(struct tee_context *ctx,
 			goto out;
 	}
 
+	if (arg.clnt_login >= TEE_IOCTL_LOGIN_REE_KERNEL_MIN &&
+	    arg.clnt_login <= TEE_IOCTL_LOGIN_REE_KERNEL_MAX) {
+		pr_debug("login method not allowed for user-space client\n");
+		rc = -EPERM;
+		goto out;
+	}
+
 	rc = ctx->teedev->desc->ops->open_session(ctx, &arg, params);
 	if (rc)
 		goto out;
diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c
index bd679b72bd05..827ac3d0fea9 100644
--- a/drivers/tee/tee_shm.c
+++ b/drivers/tee/tee_shm.c
@@ -9,6 +9,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/tee_drv.h>
+#include <linux/uio.h>
 #include "tee_private.h"
 
 static void tee_shm_release(struct tee_shm *shm)
@@ -161,8 +162,7 @@ struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
 		}
 	}
 
-	if (ctx)
-		teedev_ctx_get(ctx);
+	teedev_ctx_get(ctx);
 
 	return shm;
 err_rem:
@@ -185,14 +185,15 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
 				 size_t length, u32 flags)
 {
 	struct tee_device *teedev = ctx->teedev;
-	const u32 req_flags = TEE_SHM_DMA_BUF | TEE_SHM_USER_MAPPED;
+	const u32 req_user_flags = TEE_SHM_DMA_BUF | TEE_SHM_USER_MAPPED;
+	const u32 req_kernel_flags = TEE_SHM_DMA_BUF | TEE_SHM_KERNEL_MAPPED;
 	struct tee_shm *shm;
 	void *ret;
 	int rc;
 	int num_pages;
 	unsigned long start;
 
-	if (flags != req_flags)
+	if (flags != req_user_flags && flags != req_kernel_flags)
 		return ERR_PTR(-ENOTSUPP);
 
 	if (!tee_device_get(teedev))
@@ -226,7 +227,27 @@ struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
 		goto err;
 	}
 
-	rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);
+	if (flags & TEE_SHM_USER_MAPPED) {
+		rc = get_user_pages_fast(start, num_pages, FOLL_WRITE,
+					 shm->pages);
+	} else {
+		struct kvec *kiov;
+		int i;
+
+		kiov = kcalloc(num_pages, sizeof(*kiov), GFP_KERNEL);
+		if (!kiov) {
+			ret = ERR_PTR(-ENOMEM);
+			goto err;
+		}
+
+		for (i = 0; i < num_pages; i++) {
+			kiov[i].iov_base = (void *)(start + i * PAGE_SIZE);
+			kiov[i].iov_len = PAGE_SIZE;
+		}
+
+		rc = get_kernel_pages(kiov, num_pages, 0, shm->pages);
+		kfree(kiov);
+	}
 	if (rc > 0)
 		shm->num_pages = rc;
 	if (rc != num_pages) {
diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig
index 91af271e9bb0..3eb2348e5242 100644
--- a/drivers/thermal/Kconfig
+++ b/drivers/thermal/Kconfig
@@ -273,6 +273,16 @@ config IMX8MM_THERMAL
 	  cpufreq is used as the cooling device to throttle CPUs when the passive
 	  trip is crossed.
 
+config K3_THERMAL
+	tristate "Texas Instruments K3 thermal support"
+	depends on ARCH_K3 || COMPILE_TEST
+	help
+	  If you say yes here you get thermal support for the Texas Instruments
+	  K3 SoC family. The current chip supported is:
+	  - AM654
+
+	  This includes temperature reading functionality.
+
 config MAX77620_THERMAL
 	tristate "Temperature sensor driver for Maxim MAX77620 PMIC"
 	depends on MFD_MAX77620
@@ -285,8 +295,8 @@ config MAX77620_THERMAL
 
 config QORIQ_THERMAL
 	tristate "QorIQ Thermal Monitoring Unit"
-	depends on THERMAL_OF
-	depends on HAS_IOMEM
+	depends on THERMAL_OF && HAS_IOMEM
+	depends on PPC_E500MC || SOC_LS1021A || ARCH_LAYERSCAPE || (ARCH_MXC && ARM64) || COMPILE_TEST
 	select REGMAP_MMIO
 	help
 	  Support for Thermal Monitoring Unit (TMU) found on QorIQ platforms.
diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile
index 8c8ed7b79915..0c8b84a09b9a 100644
--- a/drivers/thermal/Makefile
+++ b/drivers/thermal/Makefile
@@ -9,14 +9,14 @@ thermal_sys-y			+= thermal_core.o thermal_sysfs.o \
 
 # interface to/from other layers providing sensors
 thermal_sys-$(CONFIG_THERMAL_HWMON)		+= thermal_hwmon.o
-thermal_sys-$(CONFIG_THERMAL_OF)		+= of-thermal.o
+thermal_sys-$(CONFIG_THERMAL_OF)		+= thermal_of.o
 
 # governors
-thermal_sys-$(CONFIG_THERMAL_GOV_FAIR_SHARE)	+= fair_share.o
+thermal_sys-$(CONFIG_THERMAL_GOV_FAIR_SHARE)	+= gov_fair_share.o
 thermal_sys-$(CONFIG_THERMAL_GOV_BANG_BANG)	+= gov_bang_bang.o
-thermal_sys-$(CONFIG_THERMAL_GOV_STEP_WISE)	+= step_wise.o
-thermal_sys-$(CONFIG_THERMAL_GOV_USER_SPACE)	+= user_space.o
-thermal_sys-$(CONFIG_THERMAL_GOV_POWER_ALLOCATOR)	+= power_allocator.o
+thermal_sys-$(CONFIG_THERMAL_GOV_STEP_WISE)	+= gov_step_wise.o
+thermal_sys-$(CONFIG_THERMAL_GOV_USER_SPACE)	+= gov_user_space.o
+thermal_sys-$(CONFIG_THERMAL_GOV_POWER_ALLOCATOR)	+= gov_power_allocator.o
 
 # cpufreq cooling
 thermal_sys-$(CONFIG_CPU_FREQ_THERMAL)	+= cpufreq_cooling.o
@@ -28,6 +28,7 @@ thermal_sys-$(CONFIG_CLOCK_THERMAL)	+= clock_cooling.o
 # devfreq cooling
 thermal_sys-$(CONFIG_DEVFREQ_THERMAL) += devfreq_cooling.o
 
+obj-$(CONFIG_K3_THERMAL)	+= k3_bandgap.o
 # platform thermal drivers
 obj-y				+= broadcom/
 obj-$(CONFIG_THERMAL_MMIO)		+= thermal_mmio.o
diff --git a/drivers/thermal/clock_cooling.c b/drivers/thermal/clock_cooling.c
index 7cb3ae4b44ee..56cb1f46a428 100644
--- a/drivers/thermal/clock_cooling.c
+++ b/drivers/thermal/clock_cooling.c
@@ -12,15 +12,16 @@
  *  Copyright (C) 2012  Amit Daniel <amit.kachhap@linaro.org>
  */
 #include <linux/clk.h>
+#include <linux/clock_cooling.h>
 #include <linux/cpufreq.h>
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/idr.h>
 #include <linux/mutex.h>
 #include <linux/pm_opp.h>
 #include <linux/slab.h>
 #include <linux/thermal.h>
-#include <linux/clock_cooling.h>
 
 /**
  * struct clock_cooling_device - data for cooling device with clock
diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c
index e297e135c031..9e124020519f 100644
--- a/drivers/thermal/cpufreq_cooling.c
+++ b/drivers/thermal/cpufreq_cooling.c
@@ -10,17 +10,17 @@
  *		Viresh Kumar <viresh.kumar@linaro.org>
  *
  */
-#include <linux/module.h>
-#include <linux/thermal.h>
+#include <linux/cpu.h>
 #include <linux/cpufreq.h>
+#include <linux/cpu_cooling.h>
+#include <linux/energy_model.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/idr.h>
 #include <linux/pm_opp.h>
 #include <linux/pm_qos.h>
 #include <linux/slab.h>
-#include <linux/cpu.h>
-#include <linux/cpu_cooling.h>
-#include <linux/energy_model.h>
+#include <linux/thermal.h>
 
 #include <trace/events/thermal.h>
 
diff --git a/drivers/thermal/cpuidle_cooling.c b/drivers/thermal/cpuidle_cooling.c
index 0bb843246f59..78e3e8238116 100644
--- a/drivers/thermal/cpuidle_cooling.c
+++ b/drivers/thermal/cpuidle_cooling.c
@@ -5,11 +5,14 @@
  *  Author: Daniel Lezcano <daniel.lezcano@linaro.org>
  *
  */
+#define pr_fmt(fmt) "cpuidle cooling: " fmt
+
 #include <linux/cpu_cooling.h>
 #include <linux/cpuidle.h>
 #include <linux/err.h>
 #include <linux/idle_inject.h>
 #include <linux/idr.h>
+#include <linux/of_device.h>
 #include <linux/slab.h>
 #include <linux/thermal.h>
 
@@ -154,22 +157,25 @@ static struct thermal_cooling_device_ops cpuidle_cooling_ops = {
 };
 
 /**
- * cpuidle_of_cooling_register - Idle cooling device initialization function
+ * __cpuidle_cooling_register: register the cooling device
  * @drv: a cpuidle driver structure pointer
- * @np: a node pointer to a device tree cooling device node
+ * @np: a device node structure pointer used for the thermal binding
  *
- * This function is in charge of creating a cooling device per cpuidle
- * driver and register it to thermal framework.
+ * This function is in charge of allocating the cpuidle cooling device
+ * structure, the idle injection, initialize them and register the
+ * cooling device to the thermal framework.
  *
- * Return: zero on success, or negative value corresponding to the
- * error detected in the underlying subsystems.
+ * Return: zero on success, a negative value returned by one of the
+ * underlying subsystem in case of error
  */
-int cpuidle_of_cooling_register(struct device_node *np,
-				struct cpuidle_driver *drv)
+static int __cpuidle_cooling_register(struct device_node *np,
+				      struct cpuidle_driver *drv)
 {
 	struct idle_inject_device *ii_dev;
 	struct cpuidle_cooling_device *idle_cdev;
 	struct thermal_cooling_device *cdev;
+	unsigned int idle_duration_us = TICK_USEC;
+	unsigned int latency_us = UINT_MAX;
 	char dev_name[THERMAL_NAME_LENGTH];
 	int id, ret;
 
@@ -191,7 +197,11 @@ int cpuidle_of_cooling_register(struct device_node *np,
 		goto out_id;
 	}
 
-	idle_inject_set_duration(ii_dev, TICK_USEC, TICK_USEC);
+	of_property_read_u32(np, "duration-us", &idle_duration_us);
+	of_property_read_u32(np, "exit-latency-us", &latency_us);
+
+	idle_inject_set_duration(ii_dev, TICK_USEC, idle_duration_us);
+	idle_inject_set_latency(ii_dev, latency_us);
 
 	idle_cdev->ii_dev = ii_dev;
 
@@ -204,6 +214,9 @@ int cpuidle_of_cooling_register(struct device_node *np,
 		goto out_unregister;
 	}
 
+	pr_debug("%s: Idle injection set with idle duration=%u, latency=%u\n",
+		 dev_name, idle_duration_us, latency_us);
+
 	return 0;
 
 out_unregister:
@@ -221,12 +234,38 @@ out:
  * @drv: a cpuidle driver structure pointer
  *
  * This function is in charge of creating a cooling device per cpuidle
- * driver and register it to thermal framework.
+ * driver and register it to the thermal framework.
  *
  * Return: zero on success, or negative value corresponding to the
  * error detected in the underlying subsystems.
  */
-int cpuidle_cooling_register(struct cpuidle_driver *drv)
+void cpuidle_cooling_register(struct cpuidle_driver *drv)
 {
-	return cpuidle_of_cooling_register(NULL, drv);
+	struct device_node *cooling_node;
+	struct device_node *cpu_node;
+	int cpu, ret;
+
+	for_each_cpu(cpu, drv->cpumask) {
+
+		cpu_node = of_cpu_device_node_get(cpu);
+
+		cooling_node = of_get_child_by_name(cpu_node, "thermal-idle");
+
+		of_node_put(cpu_node);
+
+		if (!cooling_node) {
+			pr_debug("'thermal-idle' node not found for cpu%d\n", cpu);
+			continue;
+		}
+
+		ret = __cpuidle_cooling_register(cooling_node, drv);
+
+		of_node_put(cooling_node);
+
+		if (ret) {
+			pr_err("Failed to register the cpuidle cooling device" \
+			       "for cpu%d: %d\n", cpu, ret);
+			break;
+		}
+	}
 }
diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c
index a87d4fa031c8..f7f32e98331b 100644
--- a/drivers/thermal/devfreq_cooling.c
+++ b/drivers/thermal/devfreq_cooling.c
@@ -24,11 +24,13 @@
 #include <linux/idr.h>
 #include <linux/slab.h>
 #include <linux/pm_opp.h>
+#include <linux/pm_qos.h>
 #include <linux/thermal.h>
 
 #include <trace/events/thermal.h>
 
-#define SCALE_ERROR_MITIGATION 100
+#define HZ_PER_KHZ		1000
+#define SCALE_ERROR_MITIGATION	100
 
 static DEFINE_IDA(devfreq_ida);
 
@@ -54,6 +56,8 @@ static DEFINE_IDA(devfreq_ida);
  *		The 'res_util' range is from 100 to (power_table[state] * 100)
  *		for the corresponding 'state'.
  * @capped_state:	index to cooling state with in dynamic power budget
+ * @req_max_freq:	PM QoS request for limiting the maximum frequency
+ *			of the devfreq device.
  */
 struct devfreq_cooling_device {
 	int id;
@@ -66,49 +70,9 @@ struct devfreq_cooling_device {
 	struct devfreq_cooling_power *power_ops;
 	u32 res_util;
 	int capped_state;
+	struct dev_pm_qos_request req_max_freq;
 };
 
-/**
- * partition_enable_opps() - disable all opps above a given state
- * @dfc:	Pointer to devfreq we are operating on
- * @cdev_state:	cooling device state we're setting
- *
- * Go through the OPPs of the device, enabling all OPPs until
- * @cdev_state and disabling those frequencies above it.
- */
-static int partition_enable_opps(struct devfreq_cooling_device *dfc,
-				 unsigned long cdev_state)
-{
-	int i;
-	struct device *dev = dfc->devfreq->dev.parent;
-
-	for (i = 0; i < dfc->freq_table_size; i++) {
-		struct dev_pm_opp *opp;
-		int ret = 0;
-		unsigned int freq = dfc->freq_table[i];
-		bool want_enable = i >= cdev_state ? true : false;
-
-		opp = dev_pm_opp_find_freq_exact(dev, freq, !want_enable);
-
-		if (PTR_ERR(opp) == -ERANGE)
-			continue;
-		else if (IS_ERR(opp))
-			return PTR_ERR(opp);
-
-		dev_pm_opp_put(opp);
-
-		if (want_enable)
-			ret = dev_pm_opp_enable(dev, freq);
-		else
-			ret = dev_pm_opp_disable(dev, freq);
-
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
 static int devfreq_cooling_get_max_state(struct thermal_cooling_device *cdev,
 					 unsigned long *state)
 {
@@ -135,7 +99,7 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev,
 	struct devfreq_cooling_device *dfc = cdev->devdata;
 	struct devfreq *df = dfc->devfreq;
 	struct device *dev = df->dev.parent;
-	int ret;
+	unsigned long freq;
 
 	if (state == dfc->cooling_state)
 		return 0;
@@ -145,9 +109,10 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev,
 	if (state >= dfc->freq_table_size)
 		return -EINVAL;
 
-	ret = partition_enable_opps(dfc, state);
-	if (ret)
-		return ret;
+	freq = dfc->freq_table[state];
+
+	dev_pm_qos_update_request(&dfc->req_max_freq,
+				  DIV_ROUND_UP(freq, HZ_PER_KHZ));
 
 	dfc->cooling_state = state;
 
@@ -530,9 +495,15 @@ of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df,
 	if (err)
 		goto free_dfc;
 
-	err = ida_simple_get(&devfreq_ida, 0, 0, GFP_KERNEL);
+	err = dev_pm_qos_add_request(df->dev.parent, &dfc->req_max_freq,
+				     DEV_PM_QOS_MAX_FREQUENCY,
+				     PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE);
 	if (err < 0)
 		goto free_tables;
+
+	err = ida_simple_get(&devfreq_ida, 0, 0, GFP_KERNEL);
+	if (err < 0)
+		goto remove_qos_req;
 	dfc->id = err;
 
 	snprintf(dev_name, sizeof(dev_name), "thermal-devfreq-%d", dfc->id);
@@ -553,6 +524,10 @@ of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df,
 
 release_ida:
 	ida_simple_remove(&devfreq_ida, dfc->id);
+
+remove_qos_req:
+	dev_pm_qos_remove_request(&dfc->req_max_freq);
+
 free_tables:
 	kfree(dfc->power_table);
 	kfree(dfc->freq_table);
@@ -601,6 +576,7 @@ void devfreq_cooling_unregister(struct thermal_cooling_device *cdev)
 
 	thermal_cooling_device_unregister(dfc->cdev);
 	ida_simple_remove(&devfreq_ida, dfc->id);
+	dev_pm_qos_remove_request(&dfc->req_max_freq);
 	kfree(dfc->power_table);
 	kfree(dfc->freq_table);
 
diff --git a/drivers/thermal/fair_share.c b/drivers/thermal/gov_fair_share.c
index aaa07180ab48..aaa07180ab48 100644
--- a/drivers/thermal/fair_share.c
+++ b/drivers/thermal/gov_fair_share.c
diff --git a/drivers/thermal/power_allocator.c b/drivers/thermal/gov_power_allocator.c
index 44636475b2a3..44636475b2a3 100644
--- a/drivers/thermal/power_allocator.c
+++ b/drivers/thermal/gov_power_allocator.c
diff --git a/drivers/thermal/step_wise.c b/drivers/thermal/gov_step_wise.c
index 2ae7198d3067..2ae7198d3067 100644
--- a/drivers/thermal/step_wise.c
+++ b/drivers/thermal/gov_step_wise.c
diff --git a/drivers/thermal/user_space.c b/drivers/thermal/gov_user_space.c
index 293cffd9c8ad..82a7198bbe71 100644
--- a/drivers/thermal/user_space.c
+++ b/drivers/thermal/gov_user_space.c
@@ -10,8 +10,8 @@
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  */
 
-#include <linux/thermal.h>
 #include <linux/slab.h>
+#include <linux/thermal.h>
 
 #include "thermal_core.h"
 
diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c
index 0d60f8d7894f..e6061e26d4ac 100644
--- a/drivers/thermal/imx8mm_thermal.c
+++ b/drivers/thermal/imx8mm_thermal.c
@@ -54,7 +54,7 @@ struct imx8mm_tmu {
 	void __iomem *base;
 	struct clk *clk;
 	const struct thermal_soc_data *socdata;
-	struct tmu_sensor sensors[0];
+	struct tmu_sensor sensors[];
 };
 
 static int imx8mm_tmu_get_temp(void *data, int *temp)
diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c
index a8723b1eb8b0..b01d28eca7ee 100644
--- a/drivers/thermal/imx_sc_thermal.c
+++ b/drivers/thermal/imx_sc_thermal.c
@@ -3,9 +3,9 @@
  * Copyright 2018-2020 NXP.
  */
 
+#include <dt-bindings/firmware/imx/rsrc.h>
 #include <linux/err.h>
 #include <linux/firmware/imx/sci.h>
-#include <linux/firmware/imx/types.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -14,6 +14,7 @@
 #include <linux/thermal.h>
 
 #include "thermal_core.h"
+#include "thermal_hwmon.h"
 
 #define IMX_SC_MISC_FUNC_GET_TEMP	13
 
@@ -115,6 +116,9 @@ static int imx_sc_thermal_probe(struct platform_device *pdev)
 			ret = PTR_ERR(sensor->tzd);
 			break;
 		}
+
+		if (devm_thermal_add_hwmon_sysfs(sensor->tzd))
+			dev_warn(&pdev->dev, "failed to add hwmon sysfs attributes\n");
 	}
 
 	of_node_put(sensor_np);
diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
index ceef89c956bd..0b3a62655843 100644
--- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
+++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
@@ -13,6 +13,7 @@
 #include "acpi_thermal_rel.h"
 
 #define INT3400_THERMAL_TABLE_CHANGED 0x83
+#define INT3400_ODVP_CHANGED 0x88
 
 enum int3400_thermal_uuid {
 	INT3400_THERMAL_PASSIVE_1,
@@ -41,8 +42,11 @@ static char *int3400_thermal_uuids[INT3400_THERMAL_MAXIMUM_UUID] = {
 	"BE84BABF-C4D4-403D-B495-3128FD44dAC1",
 };
 
+struct odvp_attr;
+
 struct int3400_thermal_priv {
 	struct acpi_device *adev;
+	struct platform_device *pdev;
 	struct thermal_zone_device *thermal;
 	int mode;
 	int art_count;
@@ -52,6 +56,36 @@ struct int3400_thermal_priv {
 	u8 uuid_bitmap;
 	int rel_misc_dev_res;
 	int current_uuid_index;
+	char *data_vault;
+	int odvp_count;
+	int *odvp;
+	struct odvp_attr *odvp_attrs;
+};
+
+static int evaluate_odvp(struct int3400_thermal_priv *priv);
+
+struct odvp_attr {
+	int odvp;
+	struct int3400_thermal_priv *priv;
+	struct kobj_attribute attr;
+};
+
+static ssize_t data_vault_read(struct file *file, struct kobject *kobj,
+	     struct bin_attribute *attr, char *buf, loff_t off, size_t count)
+{
+	memcpy(buf, attr->private + off, count);
+	return count;
+}
+
+static BIN_ATTR_RO(data_vault, 0);
+
+static struct bin_attribute *data_attributes[] = {
+	&bin_attr_data_vault,
+	NULL,
+};
+
+static const struct attribute_group data_attribute_group = {
+	.bin_attrs = data_attributes,
 };
 
 static ssize_t available_uuids_show(struct device *dev,
@@ -62,6 +96,9 @@ static ssize_t available_uuids_show(struct device *dev,
 	int i;
 	int length = 0;
 
+	if (!priv->uuid_bitmap)
+		return sprintf(buf, "UNKNOWN\n");
+
 	for (i = 0; i < INT3400_THERMAL_MAXIMUM_UUID; i++) {
 		if (priv->uuid_bitmap & (1 << i))
 			if (PAGE_SIZE - length > 0)
@@ -79,11 +116,11 @@ static ssize_t current_uuid_show(struct device *dev,
 {
 	struct int3400_thermal_priv *priv = dev_get_drvdata(dev);
 
-	if (priv->uuid_bitmap & (1 << priv->current_uuid_index))
-		return sprintf(buf, "%s\n",
-			       int3400_thermal_uuids[priv->current_uuid_index]);
-	else
+	if (priv->current_uuid_index == -1)
 		return sprintf(buf, "INVALID\n");
+
+	return sprintf(buf, "%s\n",
+		       int3400_thermal_uuids[priv->current_uuid_index]);
 }
 
 static ssize_t current_uuid_store(struct device *dev,
@@ -94,9 +131,16 @@ static ssize_t current_uuid_store(struct device *dev,
 	int i;
 
 	for (i = 0; i < INT3400_THERMAL_MAXIMUM_UUID; ++i) {
-		if ((priv->uuid_bitmap & (1 << i)) &&
-		    !(strncmp(buf, int3400_thermal_uuids[i],
-			      sizeof(int3400_thermal_uuids[i]) - 1))) {
+		if (!strncmp(buf, int3400_thermal_uuids[i],
+			     sizeof(int3400_thermal_uuids[i]) - 1)) {
+			/*
+			 * If we have a list of supported UUIDs, make sure
+			 * this one is supported.
+			 */
+			if (priv->uuid_bitmap &&
+			    !(priv->uuid_bitmap & (1 << i)))
+				return -EINVAL;
+
 			priv->current_uuid_index = i;
 			return count;
 		}
@@ -191,9 +235,110 @@ static int int3400_thermal_run_osc(acpi_handle handle,
 		result = -EPERM;
 
 	kfree(context.ret.pointer);
+
 	return result;
 }
 
+static ssize_t odvp_show(struct kobject *kobj, struct kobj_attribute *attr,
+			 char *buf)
+{
+	struct odvp_attr *odvp_attr;
+
+	odvp_attr = container_of(attr, struct odvp_attr, attr);
+
+	return sprintf(buf, "%d\n", odvp_attr->priv->odvp[odvp_attr->odvp]);
+}
+
+static void cleanup_odvp(struct int3400_thermal_priv *priv)
+{
+	int i;
+
+	if (priv->odvp_attrs) {
+		for (i = 0; i < priv->odvp_count; i++) {
+			sysfs_remove_file(&priv->pdev->dev.kobj,
+					  &priv->odvp_attrs[i].attr.attr);
+			kfree(priv->odvp_attrs[i].attr.attr.name);
+		}
+		kfree(priv->odvp_attrs);
+	}
+	kfree(priv->odvp);
+	priv->odvp_count = 0;
+}
+
+static int evaluate_odvp(struct int3400_thermal_priv *priv)
+{
+	struct acpi_buffer odvp = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj = NULL;
+	acpi_status status;
+	int i, ret;
+
+	status = acpi_evaluate_object(priv->adev->handle, "ODVP", NULL, &odvp);
+	if (ACPI_FAILURE(status)) {
+		ret = -EINVAL;
+		goto out_err;
+	}
+
+	obj = odvp.pointer;
+	if (obj->type != ACPI_TYPE_PACKAGE) {
+		ret = -EINVAL;
+		goto out_err;
+	}
+
+	if (priv->odvp == NULL) {
+		priv->odvp_count = obj->package.count;
+		priv->odvp = kmalloc_array(priv->odvp_count, sizeof(int),
+				     GFP_KERNEL);
+		if (!priv->odvp) {
+			ret = -ENOMEM;
+			goto out_err;
+		}
+	}
+
+	if (priv->odvp_attrs == NULL) {
+		priv->odvp_attrs = kcalloc(priv->odvp_count,
+					   sizeof(struct odvp_attr),
+					   GFP_KERNEL);
+		if (!priv->odvp_attrs) {
+			ret = -ENOMEM;
+			goto out_err;
+		}
+		for (i = 0; i < priv->odvp_count; i++) {
+			struct odvp_attr *odvp = &priv->odvp_attrs[i];
+
+			sysfs_attr_init(&odvp->attr.attr);
+			odvp->priv = priv;
+			odvp->odvp = i;
+			odvp->attr.attr.name = kasprintf(GFP_KERNEL,
+							 "odvp%d", i);
+
+			if (!odvp->attr.attr.name) {
+				ret = -ENOMEM;
+				goto out_err;
+			}
+			odvp->attr.attr.mode = 0444;
+			odvp->attr.show = odvp_show;
+			odvp->attr.store = NULL;
+			ret = sysfs_create_file(&priv->pdev->dev.kobj,
+						&odvp->attr.attr);
+			if (ret)
+				goto out_err;
+		}
+	}
+
+	for (i = 0; i < obj->package.count; i++) {
+		if (obj->package.elements[i].type == ACPI_TYPE_INTEGER)
+			priv->odvp[i] = obj->package.elements[i].integer.value;
+	}
+
+	kfree(obj);
+	return 0;
+
+out_err:
+	cleanup_odvp(priv);
+	kfree(obj);
+	return ret;
+}
+
 static void int3400_notify(acpi_handle handle,
 			u32 event,
 			void *data)
@@ -217,6 +362,9 @@ static void int3400_notify(acpi_handle handle,
 		kobject_uevent_env(&priv->thermal->device.kobj, KOBJ_CHANGE,
 				thermal_prop);
 		break;
+	case INT3400_ODVP_CHANGED:
+		evaluate_odvp(priv);
+		break;
 	default:
 		/* Ignore unknown notification codes sent to INT3400 device */
 		break;
@@ -266,11 +414,16 @@ static int int3400_thermal_set_mode(struct thermal_zone_device *thermal,
 						 priv->current_uuid_index,
 						 enable);
 	}
+
+	evaluate_odvp(priv);
+
 	return result;
 }
 
 static struct thermal_zone_device_ops int3400_thermal_ops = {
 	.get_temp = int3400_thermal_get_temp,
+	.get_mode = int3400_thermal_get_mode,
+	.set_mode = int3400_thermal_set_mode,
 };
 
 static struct thermal_zone_params int3400_thermal_params = {
@@ -278,6 +431,32 @@ static struct thermal_zone_params int3400_thermal_params = {
 	.no_hwmon = true,
 };
 
+static void int3400_setup_gddv(struct int3400_thermal_priv *priv)
+{
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj;
+	acpi_status status;
+
+	status = acpi_evaluate_object(priv->adev->handle, "GDDV", NULL,
+				      &buffer);
+	if (ACPI_FAILURE(status) || !buffer.length)
+		return;
+
+	obj = buffer.pointer;
+	if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != 1
+	    || obj->package.elements[0].type != ACPI_TYPE_BUFFER) {
+		kfree(buffer.pointer);
+		return;
+	}
+
+	priv->data_vault = kmemdup(obj->package.elements[0].buffer.pointer,
+				   obj->package.elements[0].buffer.length,
+				   GFP_KERNEL);
+	bin_attr_data_vault.private = priv->data_vault;
+	bin_attr_data_vault.size = obj->package.elements[0].buffer.length;
+	kfree(buffer.pointer);
+}
+
 static int int3400_thermal_probe(struct platform_device *pdev)
 {
 	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
@@ -291,12 +470,17 @@ static int int3400_thermal_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
+	priv->pdev = pdev;
 	priv->adev = adev;
 
 	result = int3400_thermal_get_uuids(priv);
-	if (result)
+
+	/* Missing IDSP isn't fatal */
+	if (result && result != -ENODEV)
 		goto free_priv;
 
+	priv->current_uuid_index = -1;
+
 	result = acpi_parse_art(priv->adev->handle, &priv->art_count,
 				&priv->arts, true);
 	if (result)
@@ -309,8 +493,9 @@ static int int3400_thermal_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, priv);
 
-	int3400_thermal_ops.get_mode = int3400_thermal_get_mode;
-	int3400_thermal_ops.set_mode = int3400_thermal_set_mode;
+	int3400_setup_gddv(priv);
+
+	evaluate_odvp(priv);
 
 	priv->thermal = thermal_zone_device_register("INT3400 Thermal", 0, 0,
 						priv, &int3400_thermal_ops,
@@ -327,6 +512,13 @@ static int int3400_thermal_probe(struct platform_device *pdev)
 	if (result)
 		goto free_rel_misc;
 
+	if (priv->data_vault) {
+		result = sysfs_create_group(&pdev->dev.kobj,
+					    &data_attribute_group);
+		if (result)
+			goto free_uuid;
+	}
+
 	result = acpi_install_notify_handler(
 			priv->adev->handle, ACPI_DEVICE_NOTIFY, int3400_notify,
 			(void *)priv);
@@ -336,6 +528,12 @@ static int int3400_thermal_probe(struct platform_device *pdev)
 	return 0;
 
 free_sysfs:
+	cleanup_odvp(priv);
+	if (priv->data_vault) {
+		sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group);
+		kfree(priv->data_vault);
+	}
+free_uuid:
 	sysfs_remove_group(&pdev->dev.kobj, &uuid_attribute_group);
 free_rel_misc:
 	if (!priv->rel_misc_dev_res)
@@ -357,11 +555,16 @@ static int int3400_thermal_remove(struct platform_device *pdev)
 			priv->adev->handle, ACPI_DEVICE_NOTIFY,
 			int3400_notify);
 
+	cleanup_odvp(priv);
+
 	if (!priv->rel_misc_dev_res)
 		acpi_thermal_rel_misc_device_remove(priv->adev->handle);
 
+	if (priv->data_vault)
+		sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group);
 	sysfs_remove_group(&pdev->dev.kobj, &uuid_attribute_group);
 	thermal_zone_device_unregister(priv->thermal);
+	kfree(priv->data_vault);
 	kfree(priv->trts);
 	kfree(priv->arts);
 	kfree(priv);
diff --git a/drivers/thermal/k3_bandgap.c b/drivers/thermal/k3_bandgap.c
new file mode 100644
index 000000000000..35f41e8a0b75
--- /dev/null
+++ b/drivers/thermal/k3_bandgap.c
@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * TI Bandgap temperature sensor driver for K3 SoC Family
+ *
+ * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <linux/thermal.h>
+#include <linux/types.h>
+
+#define K3_VTM_DEVINFO_PWR0_OFFSET		0x4
+#define K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK	0xf0
+#define K3_VTM_TMPSENS0_CTRL_OFFSET	0x80
+#define K3_VTM_REGS_PER_TS			0x10
+#define K3_VTM_TS_STAT_DTEMP_MASK	0x3ff
+#define K3_VTM_TMPSENS_CTRL_CBIASSEL	BIT(0)
+#define K3_VTM_TMPSENS_CTRL_SOC		BIT(5)
+#define K3_VTM_TMPSENS_CTRL_CLRZ		BIT(6)
+#define K3_VTM_TMPSENS_CTRL_CLKON_REQ	BIT(7)
+
+#define K3_VTM_ADC_BEGIN_VAL		540
+#define K3_VTM_ADC_END_VAL		944
+
+static const int k3_adc_to_temp[] = {
+	-40000, -40000, -40000, -40000, -39800, -39400, -39000, -38600, -38200,
+	-37800, -37400, -37000, -36600, -36200, -35800, -35300, -34700, -34200,
+	-33800, -33400, -33000, -32600, -32200, -31800, -31400, -31000, -30600,
+	-30200, -29800, -29400, -29000, -28600, -28200, -27700, -27100, -26600,
+	-26200, -25800, -25400, -25000, -24600, -24200, -23800, -23400, -23000,
+	-22600, -22200, -21800, -21400, -21000, -20500, -19900, -19400, -19000,
+	-18600, -18200, -17800, -17400, -17000, -16600, -16200, -15800, -15400,
+	-15000, -14600, -14200, -13800, -13400, -13000, -12500, -11900, -11400,
+	-11000, -10600, -10200, -9800, -9400, -9000, -8600, -8200, -7800, -7400,
+	-7000, -6600, -6200, -5800, -5400, -5000, -4500, -3900, -3400, -3000,
+	-2600, -2200, -1800, -1400, -1000, -600, -200, 200, 600, 1000, 1400,
+	1800, 2200, 2600, 3000, 3400, 3900, 4500, 5000, 5400, 5800, 6200, 6600,
+	7000, 7400, 7800, 8200, 8600, 9000, 9400, 9800, 10200, 10600, 11000,
+	11400, 11800, 12200, 12700, 13300, 13800, 14200, 14600, 15000, 15400,
+	15800, 16200, 16600, 17000, 17400, 17800, 18200, 18600, 19000, 19400,
+	19800, 20200, 20600, 21000, 21400, 21900, 22500, 23000, 23400, 23800,
+	24200, 24600, 25000, 25400, 25800, 26200, 26600, 27000, 27400, 27800,
+	28200, 28600, 29000, 29400, 29800, 30200, 30600, 31000, 31400, 31900,
+	32500, 33000, 33400, 33800, 34200, 34600, 35000, 35400, 35800, 36200,
+	36600, 37000, 37400, 37800, 38200, 38600, 39000, 39400, 39800, 40200,
+	40600, 41000, 41400, 41800, 42200, 42600, 43100, 43700, 44200, 44600,
+	45000, 45400, 45800, 46200, 46600, 47000, 47400, 47800, 48200, 48600,
+	49000, 49400, 49800, 50200, 50600, 51000, 51400, 51800, 52200, 52600,
+	53000, 53400, 53800, 54200, 54600, 55000, 55400, 55900, 56500, 57000,
+	57400, 57800, 58200, 58600, 59000, 59400, 59800, 60200, 60600, 61000,
+	61400, 61800, 62200, 62600, 63000, 63400, 63800, 64200, 64600, 65000,
+	65400, 65800, 66200, 66600, 67000, 67400, 67800, 68200, 68600, 69000,
+	69400, 69800, 70200, 70600, 71000, 71500, 72100, 72600, 73000, 73400,
+	73800, 74200, 74600, 75000, 75400, 75800, 76200, 76600, 77000, 77400,
+	77800, 78200, 78600, 79000, 79400, 79800, 80200, 80600, 81000, 81400,
+	81800, 82200, 82600, 83000, 83400, 83800, 84200, 84600, 85000, 85400,
+	85800, 86200, 86600, 87000, 87400, 87800, 88200, 88600, 89000, 89400,
+	89800, 90200, 90600, 91000, 91400, 91800, 92200, 92600, 93000, 93400,
+	93800, 94200, 94600, 95000, 95400, 95800, 96200, 96600, 97000, 97500,
+	98100, 98600, 99000, 99400, 99800, 100200, 100600, 101000, 101400,
+	101800, 102200, 102600, 103000, 103400, 103800, 104200, 104600, 105000,
+	105400, 105800, 106200, 106600, 107000, 107400, 107800, 108200, 108600,
+	109000, 109400, 109800, 110200, 110600, 111000, 111400, 111800, 112200,
+	112600, 113000, 113400, 113800, 114200, 114600, 115000, 115400, 115800,
+	116200, 116600, 117000, 117400, 117800, 118200, 118600, 119000, 119400,
+	119800, 120200, 120600, 121000, 121400, 121800, 122200, 122600, 123000,
+	123400, 123800, 124200, 124600, 124900, 125000,
+};
+
+struct k3_bandgap {
+	void __iomem *base;
+	const struct k3_bandgap_data *conf;
+};
+
+/* common data structures */
+struct k3_thermal_data {
+	struct thermal_zone_device *tzd;
+	struct k3_bandgap *bgp;
+	int sensor_id;
+	u32 ctrl_offset;
+	u32 stat_offset;
+};
+
+static unsigned int vtm_get_best_value(unsigned int s0, unsigned int s1,
+				       unsigned int s2)
+{
+	int d01 = abs(s0 - s1);
+	int d02 = abs(s0 - s2);
+	int d12 = abs(s1 - s2);
+
+	if (d01 <= d02 && d01 <= d12)
+		return (s0 + s1) / 2;
+
+	if (d02 <= d01 && d02 <= d12)
+		return (s0 + s2) / 2;
+
+	return (s1 + s2) / 2;
+}
+
+static int k3_bgp_read_temp(struct k3_thermal_data *devdata,
+			    int *temp)
+{
+	struct k3_bandgap *bgp;
+	unsigned int dtemp, s0, s1, s2;
+
+	bgp = devdata->bgp;
+
+	/*
+	 * Errata is applicable for am654 pg 1.0 silicon. There
+	 * is a variation of the order for 8-10 degree centigrade.
+	 * Work around that by getting the average of two closest
+	 * readings out of three readings everytime we want to
+	 * report temperatures.
+	 *
+	 * Errata workaround.
+	 */
+	s0 = readl(bgp->base + devdata->stat_offset) &
+		K3_VTM_TS_STAT_DTEMP_MASK;
+	s1 = readl(bgp->base + devdata->stat_offset) &
+		K3_VTM_TS_STAT_DTEMP_MASK;
+	s2 = readl(bgp->base + devdata->stat_offset) &
+		K3_VTM_TS_STAT_DTEMP_MASK;
+	dtemp = vtm_get_best_value(s0, s1, s2);
+
+	if (dtemp < K3_VTM_ADC_BEGIN_VAL || dtemp > K3_VTM_ADC_END_VAL)
+		return -EINVAL;
+
+	*temp = k3_adc_to_temp[dtemp - K3_VTM_ADC_BEGIN_VAL];
+
+	return 0;
+}
+
+static int k3_thermal_get_temp(void *devdata, int *temp)
+{
+	struct k3_thermal_data *data = devdata;
+	int ret = 0;
+
+	ret = k3_bgp_read_temp(data, temp);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static const struct thermal_zone_of_device_ops k3_of_thermal_ops = {
+	.get_temp = k3_thermal_get_temp,
+};
+
+static const struct of_device_id of_k3_bandgap_match[];
+
+static int k3_bandgap_probe(struct platform_device *pdev)
+{
+	int ret = 0, cnt, val, id;
+	struct resource *res;
+	struct device *dev = &pdev->dev;
+	struct k3_bandgap *bgp;
+	struct k3_thermal_data *data;
+
+	if (ARRAY_SIZE(k3_adc_to_temp) != (K3_VTM_ADC_END_VAL + 1 -
+						K3_VTM_ADC_BEGIN_VAL))
+		return -EINVAL;
+
+	bgp = devm_kzalloc(&pdev->dev, sizeof(*bgp), GFP_KERNEL);
+	if (!bgp)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	bgp->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(bgp->base))
+		return PTR_ERR(bgp->base);
+
+	pm_runtime_enable(dev);
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		pm_runtime_disable(dev);
+		return ret;
+	}
+
+	/* Get the sensor count in the VTM */
+	val = readl(bgp->base + K3_VTM_DEVINFO_PWR0_OFFSET);
+	cnt = val & K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK;
+	cnt >>= __ffs(K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK);
+
+	data = devm_kcalloc(dev, cnt, sizeof(*data), GFP_KERNEL);
+	if (!data) {
+		ret = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Register the thermal sensors */
+	for (id = 0; id < cnt; id++) {
+		data[id].sensor_id = id;
+		data[id].bgp = bgp;
+		data[id].ctrl_offset = K3_VTM_TMPSENS0_CTRL_OFFSET +
+					id * K3_VTM_REGS_PER_TS;
+		data[id].stat_offset = data[id].ctrl_offset + 0x8;
+
+		val = readl(data[id].bgp->base + data[id].ctrl_offset);
+		val |= (K3_VTM_TMPSENS_CTRL_SOC |
+			K3_VTM_TMPSENS_CTRL_CLRZ |
+			K3_VTM_TMPSENS_CTRL_CLKON_REQ);
+		val &= ~K3_VTM_TMPSENS_CTRL_CBIASSEL;
+		writel(val, data[id].bgp->base + data[id].ctrl_offset);
+
+		data[id].tzd =
+		devm_thermal_zone_of_sensor_register(dev, id,
+						     &data[id],
+						     &k3_of_thermal_ops);
+		if (IS_ERR(data[id].tzd)) {
+			dev_err(dev, "thermal zone device is NULL\n");
+			ret = PTR_ERR(data[id].tzd);
+			goto err_alloc;
+		}
+	}
+
+	platform_set_drvdata(pdev, bgp);
+
+	return 0;
+
+err_alloc:
+	pm_runtime_put_sync(dev);
+	pm_runtime_disable(dev);
+
+	return ret;
+}
+
+static int k3_bandgap_remove(struct platform_device *pdev)
+{
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static const struct of_device_id of_k3_bandgap_match[] = {
+	{
+		.compatible = "ti,am654-vtm",
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, of_k3_bandgap_match);
+
+static struct platform_driver k3_bandgap_sensor_driver = {
+	.probe = k3_bandgap_probe,
+	.remove = k3_bandgap_remove,
+	.driver = {
+		.name = "k3-soc-thermal",
+		.of_match_table	= of_k3_bandgap_match,
+	},
+};
+
+module_platform_driver(k3_bandgap_sensor_driver);
+
+MODULE_DESCRIPTION("K3 bandgap temperature sensor driver");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("J Keerthy <j-keerthy@ti.com>");
diff --git a/drivers/thermal/qcom/Makefile b/drivers/thermal/qcom/Makefile
index 7c8dc6e36693..ec86eef7f6a6 100644
--- a/drivers/thermal/qcom/Makefile
+++ b/drivers/thermal/qcom/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-$(CONFIG_QCOM_TSENS)	+= qcom_tsens.o
 
-qcom_tsens-y			+= tsens.o tsens-common.o tsens-v0_1.o \
-				   tsens-8960.o tsens-v2.o tsens-v1.o
+qcom_tsens-y			+= tsens.o tsens-v2.o tsens-v1.o tsens-v0_1.o \
+				   tsens-8960.o
 obj-$(CONFIG_QCOM_SPMI_TEMP_ALARM)	+= qcom-spmi-temp-alarm.o
diff --git a/drivers/thermal/qcom/tsens-common.c b/drivers/thermal/qcom/tsens-common.c
deleted file mode 100644
index 172545366636..000000000000
--- a/drivers/thermal/qcom/tsens-common.c
+++ /dev/null
@@ -1,843 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (c) 2015, The Linux Foundation. All rights reserved.
- */
-
-#include <linux/debugfs.h>
-#include <linux/err.h>
-#include <linux/io.h>
-#include <linux/nvmem-consumer.h>
-#include <linux/of_address.h>
-#include <linux/of_platform.h>
-#include <linux/platform_device.h>
-#include <linux/regmap.h>
-#include "tsens.h"
-
-/**
- * struct tsens_irq_data - IRQ status and temperature violations
- * @up_viol:        upper threshold violated
- * @up_thresh:      upper threshold temperature value
- * @up_irq_mask:    mask register for upper threshold irqs
- * @up_irq_clear:   clear register for uppper threshold irqs
- * @low_viol:       lower threshold violated
- * @low_thresh:     lower threshold temperature value
- * @low_irq_mask:   mask register for lower threshold irqs
- * @low_irq_clear:  clear register for lower threshold irqs
- * @crit_viol:      critical threshold violated
- * @crit_thresh:    critical threshold temperature value
- * @crit_irq_mask:  mask register for critical threshold irqs
- * @crit_irq_clear: clear register for critical threshold irqs
- *
- * Structure containing data about temperature threshold settings and
- * irq status if they were violated.
- */
-struct tsens_irq_data {
-	u32 up_viol;
-	int up_thresh;
-	u32 up_irq_mask;
-	u32 up_irq_clear;
-	u32 low_viol;
-	int low_thresh;
-	u32 low_irq_mask;
-	u32 low_irq_clear;
-	u32 crit_viol;
-	u32 crit_thresh;
-	u32 crit_irq_mask;
-	u32 crit_irq_clear;
-};
-
-char *qfprom_read(struct device *dev, const char *cname)
-{
-	struct nvmem_cell *cell;
-	ssize_t data;
-	char *ret;
-
-	cell = nvmem_cell_get(dev, cname);
-	if (IS_ERR(cell))
-		return ERR_CAST(cell);
-
-	ret = nvmem_cell_read(cell, &data);
-	nvmem_cell_put(cell);
-
-	return ret;
-}
-
-/*
- * Use this function on devices where slope and offset calculations
- * depend on calibration data read from qfprom. On others the slope
- * and offset values are derived from tz->tzp->slope and tz->tzp->offset
- * resp.
- */
-void compute_intercept_slope(struct tsens_priv *priv, u32 *p1,
-			     u32 *p2, u32 mode)
-{
-	int i;
-	int num, den;
-
-	for (i = 0; i < priv->num_sensors; i++) {
-		dev_dbg(priv->dev,
-			"%s: sensor%d - data_point1:%#x data_point2:%#x\n",
-			__func__, i, p1[i], p2[i]);
-
-		priv->sensor[i].slope = SLOPE_DEFAULT;
-		if (mode == TWO_PT_CALIB) {
-			/*
-			 * slope (m) = adc_code2 - adc_code1 (y2 - y1)/
-			 *	temp_120_degc - temp_30_degc (x2 - x1)
-			 */
-			num = p2[i] - p1[i];
-			num *= SLOPE_FACTOR;
-			den = CAL_DEGC_PT2 - CAL_DEGC_PT1;
-			priv->sensor[i].slope = num / den;
-		}
-
-		priv->sensor[i].offset = (p1[i] * SLOPE_FACTOR) -
-				(CAL_DEGC_PT1 *
-				priv->sensor[i].slope);
-		dev_dbg(priv->dev, "%s: offset:%d\n", __func__, priv->sensor[i].offset);
-	}
-}
-
-static inline u32 degc_to_code(int degc, const struct tsens_sensor *s)
-{
-	u64 code = div_u64(((u64)degc * s->slope + s->offset), SLOPE_FACTOR);
-
-	pr_debug("%s: raw_code: 0x%llx, degc:%d\n", __func__, code, degc);
-	return clamp_val(code, THRESHOLD_MIN_ADC_CODE, THRESHOLD_MAX_ADC_CODE);
-}
-
-static inline int code_to_degc(u32 adc_code, const struct tsens_sensor *s)
-{
-	int degc, num, den;
-
-	num = (adc_code * SLOPE_FACTOR) - s->offset;
-	den = s->slope;
-
-	if (num > 0)
-		degc = num + (den / 2);
-	else if (num < 0)
-		degc = num - (den / 2);
-	else
-		degc = num;
-
-	degc /= den;
-
-	return degc;
-}
-
-/**
- * tsens_hw_to_mC - Return sign-extended temperature in mCelsius.
- * @s:     Pointer to sensor struct
- * @field: Index into regmap_field array pointing to temperature data
- *
- * This function handles temperature returned in ADC code or deciCelsius
- * depending on IP version.
- *
- * Return: Temperature in milliCelsius on success, a negative errno will
- * be returned in error cases
- */
-static int tsens_hw_to_mC(const struct tsens_sensor *s, int field)
-{
-	struct tsens_priv *priv = s->priv;
-	u32 resolution;
-	u32 temp = 0;
-	int ret;
-
-	resolution = priv->fields[LAST_TEMP_0].msb -
-		priv->fields[LAST_TEMP_0].lsb;
-
-	ret = regmap_field_read(priv->rf[field], &temp);
-	if (ret)
-		return ret;
-
-	/* Convert temperature from ADC code to milliCelsius */
-	if (priv->feat->adc)
-		return code_to_degc(temp, s) * 1000;
-
-	/* deciCelsius -> milliCelsius along with sign extension */
-	return sign_extend32(temp, resolution) * 100;
-}
-
-/**
- * tsens_mC_to_hw - Convert temperature to hardware register value
- * @s: Pointer to sensor struct
- * @temp: temperature in milliCelsius to be programmed to hardware
- *
- * This function outputs the value to be written to hardware in ADC code
- * or deciCelsius depending on IP version.
- *
- * Return: ADC code or temperature in deciCelsius.
- */
-static int tsens_mC_to_hw(const struct tsens_sensor *s, int temp)
-{
-	struct tsens_priv *priv = s->priv;
-
-	/* milliC to adc code */
-	if (priv->feat->adc)
-		return degc_to_code(temp / 1000, s);
-
-	/* milliC to deciC */
-	return temp / 100;
-}
-
-static inline enum tsens_ver tsens_version(struct tsens_priv *priv)
-{
-	return priv->feat->ver_major;
-}
-
-static void tsens_set_interrupt_v1(struct tsens_priv *priv, u32 hw_id,
-				   enum tsens_irq_type irq_type, bool enable)
-{
-	u32 index = 0;
-
-	switch (irq_type) {
-	case UPPER:
-		index = UP_INT_CLEAR_0 + hw_id;
-		break;
-	case LOWER:
-		index = LOW_INT_CLEAR_0 + hw_id;
-		break;
-	case CRITICAL:
-		/* No critical interrupts before v2 */
-		return;
-	}
-	regmap_field_write(priv->rf[index], enable ? 0 : 1);
-}
-
-static void tsens_set_interrupt_v2(struct tsens_priv *priv, u32 hw_id,
-				   enum tsens_irq_type irq_type, bool enable)
-{
-	u32 index_mask = 0, index_clear = 0;
-
-	/*
-	 * To enable the interrupt flag for a sensor:
-	 *    - clear the mask bit
-	 * To disable the interrupt flag for a sensor:
-	 *    - Mask further interrupts for this sensor
-	 *    - Write 1 followed by 0 to clear the interrupt
-	 */
-	switch (irq_type) {
-	case UPPER:
-		index_mask  = UP_INT_MASK_0 + hw_id;
-		index_clear = UP_INT_CLEAR_0 + hw_id;
-		break;
-	case LOWER:
-		index_mask  = LOW_INT_MASK_0 + hw_id;
-		index_clear = LOW_INT_CLEAR_0 + hw_id;
-		break;
-	case CRITICAL:
-		index_mask  = CRIT_INT_MASK_0 + hw_id;
-		index_clear = CRIT_INT_CLEAR_0 + hw_id;
-		break;
-	}
-
-	if (enable) {
-		regmap_field_write(priv->rf[index_mask], 0);
-	} else {
-		regmap_field_write(priv->rf[index_mask],  1);
-		regmap_field_write(priv->rf[index_clear], 1);
-		regmap_field_write(priv->rf[index_clear], 0);
-	}
-}
-
-/**
- * tsens_set_interrupt - Set state of an interrupt
- * @priv: Pointer to tsens controller private data
- * @hw_id: Hardware ID aka. sensor number
- * @irq_type: irq_type from enum tsens_irq_type
- * @enable: false = disable, true = enable
- *
- * Call IP-specific function to set state of an interrupt
- *
- * Return: void
- */
-static void tsens_set_interrupt(struct tsens_priv *priv, u32 hw_id,
-				enum tsens_irq_type irq_type, bool enable)
-{
-	dev_dbg(priv->dev, "[%u] %s: %s -> %s\n", hw_id, __func__,
-		irq_type ? ((irq_type == 1) ? "UP" : "CRITICAL") : "LOW",
-		enable ? "en" : "dis");
-	if (tsens_version(priv) > VER_1_X)
-		tsens_set_interrupt_v2(priv, hw_id, irq_type, enable);
-	else
-		tsens_set_interrupt_v1(priv, hw_id, irq_type, enable);
-}
-
-/**
- * tsens_threshold_violated - Check if a sensor temperature violated a preset threshold
- * @priv: Pointer to tsens controller private data
- * @hw_id: Hardware ID aka. sensor number
- * @d: Pointer to irq state data
- *
- * Return: 0 if threshold was not violated, 1 if it was violated and negative
- * errno in case of errors
- */
-static int tsens_threshold_violated(struct tsens_priv *priv, u32 hw_id,
-				    struct tsens_irq_data *d)
-{
-	int ret;
-
-	ret = regmap_field_read(priv->rf[UPPER_STATUS_0 + hw_id], &d->up_viol);
-	if (ret)
-		return ret;
-	ret = regmap_field_read(priv->rf[LOWER_STATUS_0 + hw_id], &d->low_viol);
-	if (ret)
-		return ret;
-
-	if (priv->feat->crit_int) {
-		ret = regmap_field_read(priv->rf[CRITICAL_STATUS_0 + hw_id],
-					&d->crit_viol);
-		if (ret)
-			return ret;
-	}
-
-	if (d->up_viol || d->low_viol || d->crit_viol)
-		return 1;
-
-	return 0;
-}
-
-static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id,
-				const struct tsens_sensor *s,
-				struct tsens_irq_data *d)
-{
-	int ret;
-
-	ret = regmap_field_read(priv->rf[UP_INT_CLEAR_0 + hw_id], &d->up_irq_clear);
-	if (ret)
-		return ret;
-	ret = regmap_field_read(priv->rf[LOW_INT_CLEAR_0 + hw_id], &d->low_irq_clear);
-	if (ret)
-		return ret;
-	if (tsens_version(priv) > VER_1_X) {
-		ret = regmap_field_read(priv->rf[UP_INT_MASK_0 + hw_id], &d->up_irq_mask);
-		if (ret)
-			return ret;
-		ret = regmap_field_read(priv->rf[LOW_INT_MASK_0 + hw_id], &d->low_irq_mask);
-		if (ret)
-			return ret;
-		ret = regmap_field_read(priv->rf[CRIT_INT_CLEAR_0 + hw_id],
-					&d->crit_irq_clear);
-		if (ret)
-			return ret;
-		ret = regmap_field_read(priv->rf[CRIT_INT_MASK_0 + hw_id],
-					&d->crit_irq_mask);
-		if (ret)
-			return ret;
-
-		d->crit_thresh = tsens_hw_to_mC(s, CRIT_THRESH_0 + hw_id);
-	} else {
-		/* No mask register on older TSENS */
-		d->up_irq_mask = 0;
-		d->low_irq_mask = 0;
-		d->crit_irq_clear = 0;
-		d->crit_irq_mask = 0;
-		d->crit_thresh = 0;
-	}
-
-	d->up_thresh  = tsens_hw_to_mC(s, UP_THRESH_0 + hw_id);
-	d->low_thresh = tsens_hw_to_mC(s, LOW_THRESH_0 + hw_id);
-
-	dev_dbg(priv->dev, "[%u] %s%s: status(%u|%u|%u) | clr(%u|%u|%u) | mask(%u|%u|%u)\n",
-		hw_id, __func__,
-		(d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "",
-		d->low_viol, d->up_viol, d->crit_viol,
-		d->low_irq_clear, d->up_irq_clear, d->crit_irq_clear,
-		d->low_irq_mask, d->up_irq_mask, d->crit_irq_mask);
-	dev_dbg(priv->dev, "[%u] %s%s: thresh: (%d:%d:%d)\n", hw_id, __func__,
-		(d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "",
-		d->low_thresh, d->up_thresh, d->crit_thresh);
-
-	return 0;
-}
-
-static inline u32 masked_irq(u32 hw_id, u32 mask, enum tsens_ver ver)
-{
-	if (ver > VER_1_X)
-		return mask & (1 << hw_id);
-
-	/* v1, v0.1 don't have a irq mask register */
-	return 0;
-}
-
-/**
- * tsens_critical_irq_thread() - Threaded handler for critical interrupts
- * @irq: irq number
- * @data: tsens controller private data
- *
- * Check FSM watchdog bark status and clear if needed.
- * Check all sensors to find ones that violated their critical threshold limits.
- * Clear and then re-enable the interrupt.
- *
- * The level-triggered interrupt might deassert if the temperature returned to
- * within the threshold limits by the time the handler got scheduled. We
- * consider the irq to have been handled in that case.
- *
- * Return: IRQ_HANDLED
- */
-irqreturn_t tsens_critical_irq_thread(int irq, void *data)
-{
-	struct tsens_priv *priv = data;
-	struct tsens_irq_data d;
-	int temp, ret, i;
-	u32 wdog_status, wdog_count;
-
-	if (priv->feat->has_watchdog) {
-		ret = regmap_field_read(priv->rf[WDOG_BARK_STATUS],
-					&wdog_status);
-		if (ret)
-			return ret;
-
-		if (wdog_status) {
-			/* Clear WDOG interrupt */
-			regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 1);
-			regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 0);
-			ret = regmap_field_read(priv->rf[WDOG_BARK_COUNT],
-						&wdog_count);
-			if (ret)
-				return ret;
-			if (wdog_count)
-				dev_dbg(priv->dev, "%s: watchdog count: %d\n",
-					__func__, wdog_count);
-
-			/* Fall through to handle critical interrupts if any */
-		}
-	}
-
-	for (i = 0; i < priv->num_sensors; i++) {
-		const struct tsens_sensor *s = &priv->sensor[i];
-		u32 hw_id = s->hw_id;
-
-		if (IS_ERR(s->tzd))
-			continue;
-		if (!tsens_threshold_violated(priv, hw_id, &d))
-			continue;
-		ret = get_temp_tsens_valid(s, &temp);
-		if (ret) {
-			dev_err(priv->dev, "[%u] %s: error reading sensor\n",
-				hw_id, __func__);
-			continue;
-		}
-
-		tsens_read_irq_state(priv, hw_id, s, &d);
-		if (d.crit_viol &&
-		    !masked_irq(hw_id, d.crit_irq_mask, tsens_version(priv))) {
-			/* Mask critical interrupts, unused on Linux */
-			tsens_set_interrupt(priv, hw_id, CRITICAL, false);
-		}
-	}
-
-	return IRQ_HANDLED;
-}
-
-/**
- * tsens_irq_thread - Threaded interrupt handler for uplow interrupts
- * @irq: irq number
- * @data: tsens controller private data
- *
- * Check all sensors to find ones that violated their threshold limits. If the
- * temperature is still outside the limits, call thermal_zone_device_update() to
- * update the thresholds, else re-enable the interrupts.
- *
- * The level-triggered interrupt might deassert if the temperature returned to
- * within the threshold limits by the time the handler got scheduled. We
- * consider the irq to have been handled in that case.
- *
- * Return: IRQ_HANDLED
- */
-irqreturn_t tsens_irq_thread(int irq, void *data)
-{
-	struct tsens_priv *priv = data;
-	struct tsens_irq_data d;
-	bool enable = true, disable = false;
-	unsigned long flags;
-	int temp, ret, i;
-
-	for (i = 0; i < priv->num_sensors; i++) {
-		bool trigger = false;
-		const struct tsens_sensor *s = &priv->sensor[i];
-		u32 hw_id = s->hw_id;
-
-		if (IS_ERR(s->tzd))
-			continue;
-		if (!tsens_threshold_violated(priv, hw_id, &d))
-			continue;
-		ret = get_temp_tsens_valid(s, &temp);
-		if (ret) {
-			dev_err(priv->dev, "[%u] %s: error reading sensor\n", hw_id, __func__);
-			continue;
-		}
-
-		spin_lock_irqsave(&priv->ul_lock, flags);
-
-		tsens_read_irq_state(priv, hw_id, s, &d);
-
-		if (d.up_viol &&
-		    !masked_irq(hw_id, d.up_irq_mask, tsens_version(priv))) {
-			tsens_set_interrupt(priv, hw_id, UPPER, disable);
-			if (d.up_thresh > temp) {
-				dev_dbg(priv->dev, "[%u] %s: re-arm upper\n",
-					hw_id, __func__);
-				tsens_set_interrupt(priv, hw_id, UPPER, enable);
-			} else {
-				trigger = true;
-				/* Keep irq masked */
-			}
-		} else if (d.low_viol &&
-			   !masked_irq(hw_id, d.low_irq_mask, tsens_version(priv))) {
-			tsens_set_interrupt(priv, hw_id, LOWER, disable);
-			if (d.low_thresh < temp) {
-				dev_dbg(priv->dev, "[%u] %s: re-arm low\n",
-					hw_id, __func__);
-				tsens_set_interrupt(priv, hw_id, LOWER, enable);
-			} else {
-				trigger = true;
-				/* Keep irq masked */
-			}
-		}
-
-		spin_unlock_irqrestore(&priv->ul_lock, flags);
-
-		if (trigger) {
-			dev_dbg(priv->dev, "[%u] %s: TZ update trigger (%d mC)\n",
-				hw_id, __func__, temp);
-			thermal_zone_device_update(s->tzd,
-						   THERMAL_EVENT_UNSPECIFIED);
-		} else {
-			dev_dbg(priv->dev, "[%u] %s: no violation:  %d\n",
-				hw_id, __func__, temp);
-		}
-	}
-
-	return IRQ_HANDLED;
-}
-
-int tsens_set_trips(void *_sensor, int low, int high)
-{
-	struct tsens_sensor *s = _sensor;
-	struct tsens_priv *priv = s->priv;
-	struct device *dev = priv->dev;
-	struct tsens_irq_data d;
-	unsigned long flags;
-	int high_val, low_val, cl_high, cl_low;
-	u32 hw_id = s->hw_id;
-
-	dev_dbg(dev, "[%u] %s: proposed thresholds: (%d:%d)\n",
-		hw_id, __func__, low, high);
-
-	cl_high = clamp_val(high, -40000, 120000);
-	cl_low  = clamp_val(low, -40000, 120000);
-
-	high_val = tsens_mC_to_hw(s, cl_high);
-	low_val  = tsens_mC_to_hw(s, cl_low);
-
-	spin_lock_irqsave(&priv->ul_lock, flags);
-
-	tsens_read_irq_state(priv, hw_id, s, &d);
-
-	/* Write the new thresholds and clear the status */
-	regmap_field_write(priv->rf[LOW_THRESH_0 + hw_id], low_val);
-	regmap_field_write(priv->rf[UP_THRESH_0 + hw_id], high_val);
-	tsens_set_interrupt(priv, hw_id, LOWER, true);
-	tsens_set_interrupt(priv, hw_id, UPPER, true);
-
-	spin_unlock_irqrestore(&priv->ul_lock, flags);
-
-	dev_dbg(dev, "[%u] %s: (%d:%d)->(%d:%d)\n",
-		hw_id, __func__, d.low_thresh, d.up_thresh, cl_low, cl_high);
-
-	return 0;
-}
-
-int tsens_enable_irq(struct tsens_priv *priv)
-{
-	int ret;
-	int val = tsens_version(priv) > VER_1_X ? 7 : 1;
-
-	ret = regmap_field_write(priv->rf[INT_EN], val);
-	if (ret < 0)
-		dev_err(priv->dev, "%s: failed to enable interrupts\n", __func__);
-
-	return ret;
-}
-
-void tsens_disable_irq(struct tsens_priv *priv)
-{
-	regmap_field_write(priv->rf[INT_EN], 0);
-}
-
-int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp)
-{
-	struct tsens_priv *priv = s->priv;
-	int hw_id = s->hw_id;
-	u32 temp_idx = LAST_TEMP_0 + hw_id;
-	u32 valid_idx = VALID_0 + hw_id;
-	u32 valid;
-	int ret;
-
-	ret = regmap_field_read(priv->rf[valid_idx], &valid);
-	if (ret)
-		return ret;
-	while (!valid) {
-		/* Valid bit is 0 for 6 AHB clock cycles.
-		 * At 19.2MHz, 1 AHB clock is ~60ns.
-		 * We should enter this loop very, very rarely.
-		 */
-		ndelay(400);
-		ret = regmap_field_read(priv->rf[valid_idx], &valid);
-		if (ret)
-			return ret;
-	}
-
-	/* Valid bit is set, OK to read the temperature */
-	*temp = tsens_hw_to_mC(s, temp_idx);
-
-	return 0;
-}
-
-int get_temp_common(const struct tsens_sensor *s, int *temp)
-{
-	struct tsens_priv *priv = s->priv;
-	int hw_id = s->hw_id;
-	int last_temp = 0, ret;
-
-	ret = regmap_field_read(priv->rf[LAST_TEMP_0 + hw_id], &last_temp);
-	if (ret)
-		return ret;
-
-	*temp = code_to_degc(last_temp, s) * 1000;
-
-	return 0;
-}
-
-#ifdef CONFIG_DEBUG_FS
-static int dbg_sensors_show(struct seq_file *s, void *data)
-{
-	struct platform_device *pdev = s->private;
-	struct tsens_priv *priv = platform_get_drvdata(pdev);
-	int i;
-
-	seq_printf(s, "max: %2d\nnum: %2d\n\n",
-		   priv->feat->max_sensors, priv->num_sensors);
-
-	seq_puts(s, "      id    slope   offset\n--------------------------\n");
-	for (i = 0;  i < priv->num_sensors; i++) {
-		seq_printf(s, "%8d %8d %8d\n", priv->sensor[i].hw_id,
-			   priv->sensor[i].slope, priv->sensor[i].offset);
-	}
-
-	return 0;
-}
-
-static int dbg_version_show(struct seq_file *s, void *data)
-{
-	struct platform_device *pdev = s->private;
-	struct tsens_priv *priv = platform_get_drvdata(pdev);
-	u32 maj_ver, min_ver, step_ver;
-	int ret;
-
-	if (tsens_version(priv) > VER_0_1) {
-		ret = regmap_field_read(priv->rf[VER_MAJOR], &maj_ver);
-		if (ret)
-			return ret;
-		ret = regmap_field_read(priv->rf[VER_MINOR], &min_ver);
-		if (ret)
-			return ret;
-		ret = regmap_field_read(priv->rf[VER_STEP], &step_ver);
-		if (ret)
-			return ret;
-		seq_printf(s, "%d.%d.%d\n", maj_ver, min_ver, step_ver);
-	} else {
-		seq_puts(s, "0.1.0\n");
-	}
-
-	return 0;
-}
-
-DEFINE_SHOW_ATTRIBUTE(dbg_version);
-DEFINE_SHOW_ATTRIBUTE(dbg_sensors);
-
-static void tsens_debug_init(struct platform_device *pdev)
-{
-	struct tsens_priv *priv = platform_get_drvdata(pdev);
-	struct dentry *root, *file;
-
-	root = debugfs_lookup("tsens", NULL);
-	if (!root)
-		priv->debug_root = debugfs_create_dir("tsens", NULL);
-	else
-		priv->debug_root = root;
-
-	file = debugfs_lookup("version", priv->debug_root);
-	if (!file)
-		debugfs_create_file("version", 0444, priv->debug_root,
-				    pdev, &dbg_version_fops);
-
-	/* A directory for each instance of the TSENS IP */
-	priv->debug = debugfs_create_dir(dev_name(&pdev->dev), priv->debug_root);
-	debugfs_create_file("sensors", 0444, priv->debug, pdev, &dbg_sensors_fops);
-}
-#else
-static inline void tsens_debug_init(struct platform_device *pdev) {}
-#endif
-
-static const struct regmap_config tsens_config = {
-	.name		= "tm",
-	.reg_bits	= 32,
-	.val_bits	= 32,
-	.reg_stride	= 4,
-};
-
-static const struct regmap_config tsens_srot_config = {
-	.name		= "srot",
-	.reg_bits	= 32,
-	.val_bits	= 32,
-	.reg_stride	= 4,
-};
-
-int __init init_common(struct tsens_priv *priv)
-{
-	void __iomem *tm_base, *srot_base;
-	struct device *dev = priv->dev;
-	u32 ver_minor;
-	struct resource *res;
-	u32 enabled;
-	int ret, i, j;
-	struct platform_device *op = of_find_device_by_node(priv->dev->of_node);
-
-	if (!op)
-		return -EINVAL;
-
-	if (op->num_resources > 1) {
-		/* DT with separate SROT and TM address space */
-		priv->tm_offset = 0;
-		res = platform_get_resource(op, IORESOURCE_MEM, 1);
-		srot_base = devm_ioremap_resource(dev, res);
-		if (IS_ERR(srot_base)) {
-			ret = PTR_ERR(srot_base);
-			goto err_put_device;
-		}
-
-		priv->srot_map = devm_regmap_init_mmio(dev, srot_base,
-							&tsens_srot_config);
-		if (IS_ERR(priv->srot_map)) {
-			ret = PTR_ERR(priv->srot_map);
-			goto err_put_device;
-		}
-	} else {
-		/* old DTs where SROT and TM were in a contiguous 2K block */
-		priv->tm_offset = 0x1000;
-	}
-
-	res = platform_get_resource(op, IORESOURCE_MEM, 0);
-	tm_base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(tm_base)) {
-		ret = PTR_ERR(tm_base);
-		goto err_put_device;
-	}
-
-	priv->tm_map = devm_regmap_init_mmio(dev, tm_base, &tsens_config);
-	if (IS_ERR(priv->tm_map)) {
-		ret = PTR_ERR(priv->tm_map);
-		goto err_put_device;
-	}
-
-	if (tsens_version(priv) > VER_0_1) {
-		for (i = VER_MAJOR; i <= VER_STEP; i++) {
-			priv->rf[i] = devm_regmap_field_alloc(dev, priv->srot_map,
-							      priv->fields[i]);
-			if (IS_ERR(priv->rf[i]))
-				return PTR_ERR(priv->rf[i]);
-		}
-		ret = regmap_field_read(priv->rf[VER_MINOR], &ver_minor);
-		if (ret)
-			goto err_put_device;
-	}
-
-	priv->rf[TSENS_EN] = devm_regmap_field_alloc(dev, priv->srot_map,
-						     priv->fields[TSENS_EN]);
-	if (IS_ERR(priv->rf[TSENS_EN])) {
-		ret = PTR_ERR(priv->rf[TSENS_EN]);
-		goto err_put_device;
-	}
-	ret = regmap_field_read(priv->rf[TSENS_EN], &enabled);
-	if (ret)
-		goto err_put_device;
-	if (!enabled) {
-		dev_err(dev, "%s: device not enabled\n", __func__);
-		ret = -ENODEV;
-		goto err_put_device;
-	}
-
-	priv->rf[SENSOR_EN] = devm_regmap_field_alloc(dev, priv->srot_map,
-						      priv->fields[SENSOR_EN]);
-	if (IS_ERR(priv->rf[SENSOR_EN])) {
-		ret = PTR_ERR(priv->rf[SENSOR_EN]);
-		goto err_put_device;
-	}
-	priv->rf[INT_EN] = devm_regmap_field_alloc(dev, priv->tm_map,
-						   priv->fields[INT_EN]);
-	if (IS_ERR(priv->rf[INT_EN])) {
-		ret = PTR_ERR(priv->rf[INT_EN]);
-		goto err_put_device;
-	}
-
-	/* This loop might need changes if enum regfield_ids is reordered */
-	for (j = LAST_TEMP_0; j <= UP_THRESH_15; j += 16) {
-		for (i = 0; i < priv->feat->max_sensors; i++) {
-			int idx = j + i;
-
-			priv->rf[idx] = devm_regmap_field_alloc(dev, priv->tm_map,
-								priv->fields[idx]);
-			if (IS_ERR(priv->rf[idx])) {
-				ret = PTR_ERR(priv->rf[idx]);
-				goto err_put_device;
-			}
-		}
-	}
-
-	if (priv->feat->crit_int) {
-		/* Loop might need changes if enum regfield_ids is reordered */
-		for (j = CRITICAL_STATUS_0; j <= CRIT_THRESH_15; j += 16) {
-			for (i = 0; i < priv->feat->max_sensors; i++) {
-				int idx = j + i;
-
-				priv->rf[idx] =
-					devm_regmap_field_alloc(dev,
-								priv->tm_map,
-								priv->fields[idx]);
-				if (IS_ERR(priv->rf[idx])) {
-					ret = PTR_ERR(priv->rf[idx]);
-					goto err_put_device;
-				}
-			}
-		}
-	}
-
-	if (tsens_version(priv) > VER_1_X &&  ver_minor > 2) {
-		/* Watchdog is present only on v2.3+ */
-		priv->feat->has_watchdog = 1;
-		for (i = WDOG_BARK_STATUS; i <= CC_MON_MASK; i++) {
-			priv->rf[i] = devm_regmap_field_alloc(dev, priv->tm_map,
-							      priv->fields[i]);
-			if (IS_ERR(priv->rf[i])) {
-				ret = PTR_ERR(priv->rf[i]);
-				goto err_put_device;
-			}
-		}
-		/*
-		 * Watchdog is already enabled, unmask the bark.
-		 * Disable cycle completion monitoring
-		 */
-		regmap_field_write(priv->rf[WDOG_BARK_MASK], 0);
-		regmap_field_write(priv->rf[CC_MON_MASK], 1);
-	}
-
-	spin_lock_init(&priv->ul_lock);
-	tsens_enable_irq(priv);
-	tsens_debug_init(op);
-
-err_put_device:
-	put_device(&op->dev);
-	return ret;
-}
diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c
index 2f77d235cf73..8d3e94d2a9ed 100644
--- a/drivers/thermal/qcom/tsens.c
+++ b/drivers/thermal/qcom/tsens.c
@@ -1,19 +1,857 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2019, 2020, Linaro Ltd.
  */
 
 #include <linux/debugfs.h>
 #include <linux/err.h>
+#include <linux/io.h>
 #include <linux/module.h>
+#include <linux/nvmem-consumer.h>
 #include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
+#include <linux/regmap.h>
 #include <linux/slab.h>
 #include <linux/thermal.h>
 #include "tsens.h"
 
+/**
+ * struct tsens_irq_data - IRQ status and temperature violations
+ * @up_viol:        upper threshold violated
+ * @up_thresh:      upper threshold temperature value
+ * @up_irq_mask:    mask register for upper threshold irqs
+ * @up_irq_clear:   clear register for uppper threshold irqs
+ * @low_viol:       lower threshold violated
+ * @low_thresh:     lower threshold temperature value
+ * @low_irq_mask:   mask register for lower threshold irqs
+ * @low_irq_clear:  clear register for lower threshold irqs
+ * @crit_viol:      critical threshold violated
+ * @crit_thresh:    critical threshold temperature value
+ * @crit_irq_mask:  mask register for critical threshold irqs
+ * @crit_irq_clear: clear register for critical threshold irqs
+ *
+ * Structure containing data about temperature threshold settings and
+ * irq status if they were violated.
+ */
+struct tsens_irq_data {
+	u32 up_viol;
+	int up_thresh;
+	u32 up_irq_mask;
+	u32 up_irq_clear;
+	u32 low_viol;
+	int low_thresh;
+	u32 low_irq_mask;
+	u32 low_irq_clear;
+	u32 crit_viol;
+	u32 crit_thresh;
+	u32 crit_irq_mask;
+	u32 crit_irq_clear;
+};
+
+char *qfprom_read(struct device *dev, const char *cname)
+{
+	struct nvmem_cell *cell;
+	ssize_t data;
+	char *ret;
+
+	cell = nvmem_cell_get(dev, cname);
+	if (IS_ERR(cell))
+		return ERR_CAST(cell);
+
+	ret = nvmem_cell_read(cell, &data);
+	nvmem_cell_put(cell);
+
+	return ret;
+}
+
+/*
+ * Use this function on devices where slope and offset calculations
+ * depend on calibration data read from qfprom. On others the slope
+ * and offset values are derived from tz->tzp->slope and tz->tzp->offset
+ * resp.
+ */
+void compute_intercept_slope(struct tsens_priv *priv, u32 *p1,
+			     u32 *p2, u32 mode)
+{
+	int i;
+	int num, den;
+
+	for (i = 0; i < priv->num_sensors; i++) {
+		dev_dbg(priv->dev,
+			"%s: sensor%d - data_point1:%#x data_point2:%#x\n",
+			__func__, i, p1[i], p2[i]);
+
+		priv->sensor[i].slope = SLOPE_DEFAULT;
+		if (mode == TWO_PT_CALIB) {
+			/*
+			 * slope (m) = adc_code2 - adc_code1 (y2 - y1)/
+			 *	temp_120_degc - temp_30_degc (x2 - x1)
+			 */
+			num = p2[i] - p1[i];
+			num *= SLOPE_FACTOR;
+			den = CAL_DEGC_PT2 - CAL_DEGC_PT1;
+			priv->sensor[i].slope = num / den;
+		}
+
+		priv->sensor[i].offset = (p1[i] * SLOPE_FACTOR) -
+				(CAL_DEGC_PT1 *
+				priv->sensor[i].slope);
+		dev_dbg(priv->dev, "%s: offset:%d\n", __func__,
+			priv->sensor[i].offset);
+	}
+}
+
+static inline u32 degc_to_code(int degc, const struct tsens_sensor *s)
+{
+	u64 code = div_u64(((u64)degc * s->slope + s->offset), SLOPE_FACTOR);
+
+	pr_debug("%s: raw_code: 0x%llx, degc:%d\n", __func__, code, degc);
+	return clamp_val(code, THRESHOLD_MIN_ADC_CODE, THRESHOLD_MAX_ADC_CODE);
+}
+
+static inline int code_to_degc(u32 adc_code, const struct tsens_sensor *s)
+{
+	int degc, num, den;
+
+	num = (adc_code * SLOPE_FACTOR) - s->offset;
+	den = s->slope;
+
+	if (num > 0)
+		degc = num + (den / 2);
+	else if (num < 0)
+		degc = num - (den / 2);
+	else
+		degc = num;
+
+	degc /= den;
+
+	return degc;
+}
+
+/**
+ * tsens_hw_to_mC - Return sign-extended temperature in mCelsius.
+ * @s:     Pointer to sensor struct
+ * @field: Index into regmap_field array pointing to temperature data
+ *
+ * This function handles temperature returned in ADC code or deciCelsius
+ * depending on IP version.
+ *
+ * Return: Temperature in milliCelsius on success, a negative errno will
+ * be returned in error cases
+ */
+static int tsens_hw_to_mC(const struct tsens_sensor *s, int field)
+{
+	struct tsens_priv *priv = s->priv;
+	u32 resolution;
+	u32 temp = 0;
+	int ret;
+
+	resolution = priv->fields[LAST_TEMP_0].msb -
+		priv->fields[LAST_TEMP_0].lsb;
+
+	ret = regmap_field_read(priv->rf[field], &temp);
+	if (ret)
+		return ret;
+
+	/* Convert temperature from ADC code to milliCelsius */
+	if (priv->feat->adc)
+		return code_to_degc(temp, s) * 1000;
+
+	/* deciCelsius -> milliCelsius along with sign extension */
+	return sign_extend32(temp, resolution) * 100;
+}
+
+/**
+ * tsens_mC_to_hw - Convert temperature to hardware register value
+ * @s: Pointer to sensor struct
+ * @temp: temperature in milliCelsius to be programmed to hardware
+ *
+ * This function outputs the value to be written to hardware in ADC code
+ * or deciCelsius depending on IP version.
+ *
+ * Return: ADC code or temperature in deciCelsius.
+ */
+static int tsens_mC_to_hw(const struct tsens_sensor *s, int temp)
+{
+	struct tsens_priv *priv = s->priv;
+
+	/* milliC to adc code */
+	if (priv->feat->adc)
+		return degc_to_code(temp / 1000, s);
+
+	/* milliC to deciC */
+	return temp / 100;
+}
+
+static inline enum tsens_ver tsens_version(struct tsens_priv *priv)
+{
+	return priv->feat->ver_major;
+}
+
+static void tsens_set_interrupt_v1(struct tsens_priv *priv, u32 hw_id,
+				   enum tsens_irq_type irq_type, bool enable)
+{
+	u32 index = 0;
+
+	switch (irq_type) {
+	case UPPER:
+		index = UP_INT_CLEAR_0 + hw_id;
+		break;
+	case LOWER:
+		index = LOW_INT_CLEAR_0 + hw_id;
+		break;
+	case CRITICAL:
+		/* No critical interrupts before v2 */
+		return;
+	}
+	regmap_field_write(priv->rf[index], enable ? 0 : 1);
+}
+
+static void tsens_set_interrupt_v2(struct tsens_priv *priv, u32 hw_id,
+				   enum tsens_irq_type irq_type, bool enable)
+{
+	u32 index_mask = 0, index_clear = 0;
+
+	/*
+	 * To enable the interrupt flag for a sensor:
+	 *    - clear the mask bit
+	 * To disable the interrupt flag for a sensor:
+	 *    - Mask further interrupts for this sensor
+	 *    - Write 1 followed by 0 to clear the interrupt
+	 */
+	switch (irq_type) {
+	case UPPER:
+		index_mask  = UP_INT_MASK_0 + hw_id;
+		index_clear = UP_INT_CLEAR_0 + hw_id;
+		break;
+	case LOWER:
+		index_mask  = LOW_INT_MASK_0 + hw_id;
+		index_clear = LOW_INT_CLEAR_0 + hw_id;
+		break;
+	case CRITICAL:
+		index_mask  = CRIT_INT_MASK_0 + hw_id;
+		index_clear = CRIT_INT_CLEAR_0 + hw_id;
+		break;
+	}
+
+	if (enable) {
+		regmap_field_write(priv->rf[index_mask], 0);
+	} else {
+		regmap_field_write(priv->rf[index_mask],  1);
+		regmap_field_write(priv->rf[index_clear], 1);
+		regmap_field_write(priv->rf[index_clear], 0);
+	}
+}
+
+/**
+ * tsens_set_interrupt - Set state of an interrupt
+ * @priv: Pointer to tsens controller private data
+ * @hw_id: Hardware ID aka. sensor number
+ * @irq_type: irq_type from enum tsens_irq_type
+ * @enable: false = disable, true = enable
+ *
+ * Call IP-specific function to set state of an interrupt
+ *
+ * Return: void
+ */
+static void tsens_set_interrupt(struct tsens_priv *priv, u32 hw_id,
+				enum tsens_irq_type irq_type, bool enable)
+{
+	dev_dbg(priv->dev, "[%u] %s: %s -> %s\n", hw_id, __func__,
+		irq_type ? ((irq_type == 1) ? "UP" : "CRITICAL") : "LOW",
+		enable ? "en" : "dis");
+	if (tsens_version(priv) > VER_1_X)
+		tsens_set_interrupt_v2(priv, hw_id, irq_type, enable);
+	else
+		tsens_set_interrupt_v1(priv, hw_id, irq_type, enable);
+}
+
+/**
+ * tsens_threshold_violated - Check if a sensor temperature violated a preset threshold
+ * @priv: Pointer to tsens controller private data
+ * @hw_id: Hardware ID aka. sensor number
+ * @d: Pointer to irq state data
+ *
+ * Return: 0 if threshold was not violated, 1 if it was violated and negative
+ * errno in case of errors
+ */
+static int tsens_threshold_violated(struct tsens_priv *priv, u32 hw_id,
+				    struct tsens_irq_data *d)
+{
+	int ret;
+
+	ret = regmap_field_read(priv->rf[UPPER_STATUS_0 + hw_id], &d->up_viol);
+	if (ret)
+		return ret;
+	ret = regmap_field_read(priv->rf[LOWER_STATUS_0 + hw_id], &d->low_viol);
+	if (ret)
+		return ret;
+
+	if (priv->feat->crit_int) {
+		ret = regmap_field_read(priv->rf[CRITICAL_STATUS_0 + hw_id],
+					&d->crit_viol);
+		if (ret)
+			return ret;
+	}
+
+	if (d->up_viol || d->low_viol || d->crit_viol)
+		return 1;
+
+	return 0;
+}
+
+static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id,
+				const struct tsens_sensor *s,
+				struct tsens_irq_data *d)
+{
+	int ret;
+
+	ret = regmap_field_read(priv->rf[UP_INT_CLEAR_0 + hw_id], &d->up_irq_clear);
+	if (ret)
+		return ret;
+	ret = regmap_field_read(priv->rf[LOW_INT_CLEAR_0 + hw_id], &d->low_irq_clear);
+	if (ret)
+		return ret;
+	if (tsens_version(priv) > VER_1_X) {
+		ret = regmap_field_read(priv->rf[UP_INT_MASK_0 + hw_id], &d->up_irq_mask);
+		if (ret)
+			return ret;
+		ret = regmap_field_read(priv->rf[LOW_INT_MASK_0 + hw_id], &d->low_irq_mask);
+		if (ret)
+			return ret;
+		ret = regmap_field_read(priv->rf[CRIT_INT_CLEAR_0 + hw_id],
+					&d->crit_irq_clear);
+		if (ret)
+			return ret;
+		ret = regmap_field_read(priv->rf[CRIT_INT_MASK_0 + hw_id],
+					&d->crit_irq_mask);
+		if (ret)
+			return ret;
+
+		d->crit_thresh = tsens_hw_to_mC(s, CRIT_THRESH_0 + hw_id);
+	} else {
+		/* No mask register on older TSENS */
+		d->up_irq_mask = 0;
+		d->low_irq_mask = 0;
+		d->crit_irq_clear = 0;
+		d->crit_irq_mask = 0;
+		d->crit_thresh = 0;
+	}
+
+	d->up_thresh  = tsens_hw_to_mC(s, UP_THRESH_0 + hw_id);
+	d->low_thresh = tsens_hw_to_mC(s, LOW_THRESH_0 + hw_id);
+
+	dev_dbg(priv->dev, "[%u] %s%s: status(%u|%u|%u) | clr(%u|%u|%u) | mask(%u|%u|%u)\n",
+		hw_id, __func__,
+		(d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "",
+		d->low_viol, d->up_viol, d->crit_viol,
+		d->low_irq_clear, d->up_irq_clear, d->crit_irq_clear,
+		d->low_irq_mask, d->up_irq_mask, d->crit_irq_mask);
+	dev_dbg(priv->dev, "[%u] %s%s: thresh: (%d:%d:%d)\n", hw_id, __func__,
+		(d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "",
+		d->low_thresh, d->up_thresh, d->crit_thresh);
+
+	return 0;
+}
+
+static inline u32 masked_irq(u32 hw_id, u32 mask, enum tsens_ver ver)
+{
+	if (ver > VER_1_X)
+		return mask & (1 << hw_id);
+
+	/* v1, v0.1 don't have a irq mask register */
+	return 0;
+}
+
+/**
+ * tsens_critical_irq_thread() - Threaded handler for critical interrupts
+ * @irq: irq number
+ * @data: tsens controller private data
+ *
+ * Check FSM watchdog bark status and clear if needed.
+ * Check all sensors to find ones that violated their critical threshold limits.
+ * Clear and then re-enable the interrupt.
+ *
+ * The level-triggered interrupt might deassert if the temperature returned to
+ * within the threshold limits by the time the handler got scheduled. We
+ * consider the irq to have been handled in that case.
+ *
+ * Return: IRQ_HANDLED
+ */
+irqreturn_t tsens_critical_irq_thread(int irq, void *data)
+{
+	struct tsens_priv *priv = data;
+	struct tsens_irq_data d;
+	int temp, ret, i;
+	u32 wdog_status, wdog_count;
+
+	if (priv->feat->has_watchdog) {
+		ret = regmap_field_read(priv->rf[WDOG_BARK_STATUS],
+					&wdog_status);
+		if (ret)
+			return ret;
+
+		if (wdog_status) {
+			/* Clear WDOG interrupt */
+			regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 1);
+			regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 0);
+			ret = regmap_field_read(priv->rf[WDOG_BARK_COUNT],
+						&wdog_count);
+			if (ret)
+				return ret;
+			if (wdog_count)
+				dev_dbg(priv->dev, "%s: watchdog count: %d\n",
+					__func__, wdog_count);
+
+			/* Fall through to handle critical interrupts if any */
+		}
+	}
+
+	for (i = 0; i < priv->num_sensors; i++) {
+		const struct tsens_sensor *s = &priv->sensor[i];
+		u32 hw_id = s->hw_id;
+
+		if (IS_ERR(s->tzd))
+			continue;
+		if (!tsens_threshold_violated(priv, hw_id, &d))
+			continue;
+		ret = get_temp_tsens_valid(s, &temp);
+		if (ret) {
+			dev_err(priv->dev, "[%u] %s: error reading sensor\n",
+				hw_id, __func__);
+			continue;
+		}
+
+		tsens_read_irq_state(priv, hw_id, s, &d);
+		if (d.crit_viol &&
+		    !masked_irq(hw_id, d.crit_irq_mask, tsens_version(priv))) {
+			/* Mask critical interrupts, unused on Linux */
+			tsens_set_interrupt(priv, hw_id, CRITICAL, false);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * tsens_irq_thread - Threaded interrupt handler for uplow interrupts
+ * @irq: irq number
+ * @data: tsens controller private data
+ *
+ * Check all sensors to find ones that violated their threshold limits. If the
+ * temperature is still outside the limits, call thermal_zone_device_update() to
+ * update the thresholds, else re-enable the interrupts.
+ *
+ * The level-triggered interrupt might deassert if the temperature returned to
+ * within the threshold limits by the time the handler got scheduled. We
+ * consider the irq to have been handled in that case.
+ *
+ * Return: IRQ_HANDLED
+ */
+irqreturn_t tsens_irq_thread(int irq, void *data)
+{
+	struct tsens_priv *priv = data;
+	struct tsens_irq_data d;
+	bool enable = true, disable = false;
+	unsigned long flags;
+	int temp, ret, i;
+
+	for (i = 0; i < priv->num_sensors; i++) {
+		bool trigger = false;
+		const struct tsens_sensor *s = &priv->sensor[i];
+		u32 hw_id = s->hw_id;
+
+		if (IS_ERR(s->tzd))
+			continue;
+		if (!tsens_threshold_violated(priv, hw_id, &d))
+			continue;
+		ret = get_temp_tsens_valid(s, &temp);
+		if (ret) {
+			dev_err(priv->dev, "[%u] %s: error reading sensor\n",
+				hw_id, __func__);
+			continue;
+		}
+
+		spin_lock_irqsave(&priv->ul_lock, flags);
+
+		tsens_read_irq_state(priv, hw_id, s, &d);
+
+		if (d.up_viol &&
+		    !masked_irq(hw_id, d.up_irq_mask, tsens_version(priv))) {
+			tsens_set_interrupt(priv, hw_id, UPPER, disable);
+			if (d.up_thresh > temp) {
+				dev_dbg(priv->dev, "[%u] %s: re-arm upper\n",
+					hw_id, __func__);
+				tsens_set_interrupt(priv, hw_id, UPPER, enable);
+			} else {
+				trigger = true;
+				/* Keep irq masked */
+			}
+		} else if (d.low_viol &&
+			   !masked_irq(hw_id, d.low_irq_mask, tsens_version(priv))) {
+			tsens_set_interrupt(priv, hw_id, LOWER, disable);
+			if (d.low_thresh < temp) {
+				dev_dbg(priv->dev, "[%u] %s: re-arm low\n",
+					hw_id, __func__);
+				tsens_set_interrupt(priv, hw_id, LOWER, enable);
+			} else {
+				trigger = true;
+				/* Keep irq masked */
+			}
+		}
+
+		spin_unlock_irqrestore(&priv->ul_lock, flags);
+
+		if (trigger) {
+			dev_dbg(priv->dev, "[%u] %s: TZ update trigger (%d mC)\n",
+				hw_id, __func__, temp);
+			thermal_zone_device_update(s->tzd,
+						   THERMAL_EVENT_UNSPECIFIED);
+		} else {
+			dev_dbg(priv->dev, "[%u] %s: no violation:  %d\n",
+				hw_id, __func__, temp);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+int tsens_set_trips(void *_sensor, int low, int high)
+{
+	struct tsens_sensor *s = _sensor;
+	struct tsens_priv *priv = s->priv;
+	struct device *dev = priv->dev;
+	struct tsens_irq_data d;
+	unsigned long flags;
+	int high_val, low_val, cl_high, cl_low;
+	u32 hw_id = s->hw_id;
+
+	dev_dbg(dev, "[%u] %s: proposed thresholds: (%d:%d)\n",
+		hw_id, __func__, low, high);
+
+	cl_high = clamp_val(high, -40000, 120000);
+	cl_low  = clamp_val(low, -40000, 120000);
+
+	high_val = tsens_mC_to_hw(s, cl_high);
+	low_val  = tsens_mC_to_hw(s, cl_low);
+
+	spin_lock_irqsave(&priv->ul_lock, flags);
+
+	tsens_read_irq_state(priv, hw_id, s, &d);
+
+	/* Write the new thresholds and clear the status */
+	regmap_field_write(priv->rf[LOW_THRESH_0 + hw_id], low_val);
+	regmap_field_write(priv->rf[UP_THRESH_0 + hw_id], high_val);
+	tsens_set_interrupt(priv, hw_id, LOWER, true);
+	tsens_set_interrupt(priv, hw_id, UPPER, true);
+
+	spin_unlock_irqrestore(&priv->ul_lock, flags);
+
+	dev_dbg(dev, "[%u] %s: (%d:%d)->(%d:%d)\n",
+		hw_id, __func__, d.low_thresh, d.up_thresh, cl_low, cl_high);
+
+	return 0;
+}
+
+int tsens_enable_irq(struct tsens_priv *priv)
+{
+	int ret;
+	int val = tsens_version(priv) > VER_1_X ? 7 : 1;
+
+	ret = regmap_field_write(priv->rf[INT_EN], val);
+	if (ret < 0)
+		dev_err(priv->dev, "%s: failed to enable interrupts\n",
+			__func__);
+
+	return ret;
+}
+
+void tsens_disable_irq(struct tsens_priv *priv)
+{
+	regmap_field_write(priv->rf[INT_EN], 0);
+}
+
+int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp)
+{
+	struct tsens_priv *priv = s->priv;
+	int hw_id = s->hw_id;
+	u32 temp_idx = LAST_TEMP_0 + hw_id;
+	u32 valid_idx = VALID_0 + hw_id;
+	u32 valid;
+	int ret;
+
+	ret = regmap_field_read(priv->rf[valid_idx], &valid);
+	if (ret)
+		return ret;
+	while (!valid) {
+		/* Valid bit is 0 for 6 AHB clock cycles.
+		 * At 19.2MHz, 1 AHB clock is ~60ns.
+		 * We should enter this loop very, very rarely.
+		 */
+		ndelay(400);
+		ret = regmap_field_read(priv->rf[valid_idx], &valid);
+		if (ret)
+			return ret;
+	}
+
+	/* Valid bit is set, OK to read the temperature */
+	*temp = tsens_hw_to_mC(s, temp_idx);
+
+	return 0;
+}
+
+int get_temp_common(const struct tsens_sensor *s, int *temp)
+{
+	struct tsens_priv *priv = s->priv;
+	int hw_id = s->hw_id;
+	int last_temp = 0, ret;
+
+	ret = regmap_field_read(priv->rf[LAST_TEMP_0 + hw_id], &last_temp);
+	if (ret)
+		return ret;
+
+	*temp = code_to_degc(last_temp, s) * 1000;
+
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int dbg_sensors_show(struct seq_file *s, void *data)
+{
+	struct platform_device *pdev = s->private;
+	struct tsens_priv *priv = platform_get_drvdata(pdev);
+	int i;
+
+	seq_printf(s, "max: %2d\nnum: %2d\n\n",
+		   priv->feat->max_sensors, priv->num_sensors);
+
+	seq_puts(s, "      id    slope   offset\n--------------------------\n");
+	for (i = 0;  i < priv->num_sensors; i++) {
+		seq_printf(s, "%8d %8d %8d\n", priv->sensor[i].hw_id,
+			   priv->sensor[i].slope, priv->sensor[i].offset);
+	}
+
+	return 0;
+}
+
+static int dbg_version_show(struct seq_file *s, void *data)
+{
+	struct platform_device *pdev = s->private;
+	struct tsens_priv *priv = platform_get_drvdata(pdev);
+	u32 maj_ver, min_ver, step_ver;
+	int ret;
+
+	if (tsens_version(priv) > VER_0_1) {
+		ret = regmap_field_read(priv->rf[VER_MAJOR], &maj_ver);
+		if (ret)
+			return ret;
+		ret = regmap_field_read(priv->rf[VER_MINOR], &min_ver);
+		if (ret)
+			return ret;
+		ret = regmap_field_read(priv->rf[VER_STEP], &step_ver);
+		if (ret)
+			return ret;
+		seq_printf(s, "%d.%d.%d\n", maj_ver, min_ver, step_ver);
+	} else {
+		seq_puts(s, "0.1.0\n");
+	}
+
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(dbg_version);
+DEFINE_SHOW_ATTRIBUTE(dbg_sensors);
+
+static void tsens_debug_init(struct platform_device *pdev)
+{
+	struct tsens_priv *priv = platform_get_drvdata(pdev);
+	struct dentry *root, *file;
+
+	root = debugfs_lookup("tsens", NULL);
+	if (!root)
+		priv->debug_root = debugfs_create_dir("tsens", NULL);
+	else
+		priv->debug_root = root;
+
+	file = debugfs_lookup("version", priv->debug_root);
+	if (!file)
+		debugfs_create_file("version", 0444, priv->debug_root,
+				    pdev, &dbg_version_fops);
+
+	/* A directory for each instance of the TSENS IP */
+	priv->debug = debugfs_create_dir(dev_name(&pdev->dev), priv->debug_root);
+	debugfs_create_file("sensors", 0444, priv->debug, pdev, &dbg_sensors_fops);
+}
+#else
+static inline void tsens_debug_init(struct platform_device *pdev) {}
+#endif
+
+static const struct regmap_config tsens_config = {
+	.name		= "tm",
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+};
+
+static const struct regmap_config tsens_srot_config = {
+	.name		= "srot",
+	.reg_bits	= 32,
+	.val_bits	= 32,
+	.reg_stride	= 4,
+};
+
+int __init init_common(struct tsens_priv *priv)
+{
+	void __iomem *tm_base, *srot_base;
+	struct device *dev = priv->dev;
+	u32 ver_minor;
+	struct resource *res;
+	u32 enabled;
+	int ret, i, j;
+	struct platform_device *op = of_find_device_by_node(priv->dev->of_node);
+
+	if (!op)
+		return -EINVAL;
+
+	if (op->num_resources > 1) {
+		/* DT with separate SROT and TM address space */
+		priv->tm_offset = 0;
+		res = platform_get_resource(op, IORESOURCE_MEM, 1);
+		srot_base = devm_ioremap_resource(dev, res);
+		if (IS_ERR(srot_base)) {
+			ret = PTR_ERR(srot_base);
+			goto err_put_device;
+		}
+
+		priv->srot_map = devm_regmap_init_mmio(dev, srot_base,
+						       &tsens_srot_config);
+		if (IS_ERR(priv->srot_map)) {
+			ret = PTR_ERR(priv->srot_map);
+			goto err_put_device;
+		}
+	} else {
+		/* old DTs where SROT and TM were in a contiguous 2K block */
+		priv->tm_offset = 0x1000;
+	}
+
+	res = platform_get_resource(op, IORESOURCE_MEM, 0);
+	tm_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(tm_base)) {
+		ret = PTR_ERR(tm_base);
+		goto err_put_device;
+	}
+
+	priv->tm_map = devm_regmap_init_mmio(dev, tm_base, &tsens_config);
+	if (IS_ERR(priv->tm_map)) {
+		ret = PTR_ERR(priv->tm_map);
+		goto err_put_device;
+	}
+
+	if (tsens_version(priv) > VER_0_1) {
+		for (i = VER_MAJOR; i <= VER_STEP; i++) {
+			priv->rf[i] = devm_regmap_field_alloc(dev, priv->srot_map,
+							      priv->fields[i]);
+			if (IS_ERR(priv->rf[i]))
+				return PTR_ERR(priv->rf[i]);
+		}
+		ret = regmap_field_read(priv->rf[VER_MINOR], &ver_minor);
+		if (ret)
+			goto err_put_device;
+	}
+
+	priv->rf[TSENS_EN] = devm_regmap_field_alloc(dev, priv->srot_map,
+						     priv->fields[TSENS_EN]);
+	if (IS_ERR(priv->rf[TSENS_EN])) {
+		ret = PTR_ERR(priv->rf[TSENS_EN]);
+		goto err_put_device;
+	}
+	ret = regmap_field_read(priv->rf[TSENS_EN], &enabled);
+	if (ret)
+		goto err_put_device;
+	if (!enabled) {
+		dev_err(dev, "%s: device not enabled\n", __func__);
+		ret = -ENODEV;
+		goto err_put_device;
+	}
+
+	priv->rf[SENSOR_EN] = devm_regmap_field_alloc(dev, priv->srot_map,
+						      priv->fields[SENSOR_EN]);
+	if (IS_ERR(priv->rf[SENSOR_EN])) {
+		ret = PTR_ERR(priv->rf[SENSOR_EN]);
+		goto err_put_device;
+	}
+	priv->rf[INT_EN] = devm_regmap_field_alloc(dev, priv->tm_map,
+						   priv->fields[INT_EN]);
+	if (IS_ERR(priv->rf[INT_EN])) {
+		ret = PTR_ERR(priv->rf[INT_EN]);
+		goto err_put_device;
+	}
+
+	/* This loop might need changes if enum regfield_ids is reordered */
+	for (j = LAST_TEMP_0; j <= UP_THRESH_15; j += 16) {
+		for (i = 0; i < priv->feat->max_sensors; i++) {
+			int idx = j + i;
+
+			priv->rf[idx] = devm_regmap_field_alloc(dev,
+								priv->tm_map,
+								priv->fields[idx]);
+			if (IS_ERR(priv->rf[idx])) {
+				ret = PTR_ERR(priv->rf[idx]);
+				goto err_put_device;
+			}
+		}
+	}
+
+	if (priv->feat->crit_int) {
+		/* Loop might need changes if enum regfield_ids is reordered */
+		for (j = CRITICAL_STATUS_0; j <= CRIT_THRESH_15; j += 16) {
+			for (i = 0; i < priv->feat->max_sensors; i++) {
+				int idx = j + i;
+
+				priv->rf[idx] =
+					devm_regmap_field_alloc(dev,
+								priv->tm_map,
+								priv->fields[idx]);
+				if (IS_ERR(priv->rf[idx])) {
+					ret = PTR_ERR(priv->rf[idx]);
+					goto err_put_device;
+				}
+			}
+		}
+	}
+
+	if (tsens_version(priv) > VER_1_X &&  ver_minor > 2) {
+		/* Watchdog is present only on v2.3+ */
+		priv->feat->has_watchdog = 1;
+		for (i = WDOG_BARK_STATUS; i <= CC_MON_MASK; i++) {
+			priv->rf[i] = devm_regmap_field_alloc(dev, priv->tm_map,
+							      priv->fields[i]);
+			if (IS_ERR(priv->rf[i])) {
+				ret = PTR_ERR(priv->rf[i]);
+				goto err_put_device;
+			}
+		}
+		/*
+		 * Watchdog is already enabled, unmask the bark.
+		 * Disable cycle completion monitoring
+		 */
+		regmap_field_write(priv->rf[WDOG_BARK_MASK], 0);
+		regmap_field_write(priv->rf[CC_MON_MASK], 1);
+	}
+
+	spin_lock_init(&priv->ul_lock);
+	tsens_enable_irq(priv);
+	tsens_debug_init(op);
+
+err_put_device:
+	put_device(&op->dev);
+	return ret;
+}
+
 static int tsens_get_temp(void *data, int *temp)
 {
 	struct tsens_sensor *s = data;
diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h
index 502acf0e6828..59d01162c66a 100644
--- a/drivers/thermal/qcom/tsens.h
+++ b/drivers/thermal/qcom/tsens.h
@@ -580,11 +580,6 @@ void compute_intercept_slope(struct tsens_priv *priv, u32 *pt1, u32 *pt2, u32 mo
 int init_common(struct tsens_priv *priv);
 int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp);
 int get_temp_common(const struct tsens_sensor *s, int *temp);
-int tsens_enable_irq(struct tsens_priv *priv);
-void tsens_disable_irq(struct tsens_priv *priv);
-int tsens_set_trips(void *_sensor, int low, int high);
-irqreturn_t tsens_irq_thread(int irq, void *data);
-irqreturn_t tsens_critical_irq_thread(int irq, void *data);
 
 /* TSENS target */
 extern struct tsens_plat_data data_8960;
diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c
index 028a6bbf75dc..73049f9bea25 100644
--- a/drivers/thermal/qoriq_thermal.c
+++ b/drivers/thermal/qoriq_thermal.c
@@ -11,6 +11,7 @@
 #include <linux/regmap.h>
 #include <linux/sizes.h>
 #include <linux/thermal.h>
+#include <linux/units.h>
 
 #include "thermal_core.h"
 #include "thermal_hwmon.h"
@@ -23,6 +24,7 @@
 #define TMTMIR_DEFAULT	0x0000000f
 #define TIER_DISABLE	0x0
 #define TEUMR0_V2		0x51009c00
+#define TMSARA_V2		0xe
 #define TMU_VER1		0x1
 #define TMU_VER2		0x2
 
@@ -50,6 +52,9 @@
 					    * Site Register
 					    */
 #define TRITSR_V	BIT(31)
+#define REGS_V2_TMSAR(n)	(0x304 + 16 * (n))	/* TMU monitoring
+						* site adjustment register
+						*/
 #define REGS_TTRnCR(n)	(0xf10 + 4 * (n)) /* Temperature Range n
 					   * Control Register
 					   */
@@ -85,12 +90,21 @@ static int tmu_get_temp(void *p, int *temp)
 	/*
 	 * REGS_TRITSR(id) has the following layout:
 	 *
+	 * For TMU Rev1:
 	 * 31  ... 7 6 5 4 3 2 1 0
 	 *  V          TEMP
 	 *
 	 * Where V bit signifies if the measurement is ready and is
 	 * within sensor range. TEMP is an 8 bit value representing
-	 * temperature in C.
+	 * temperature in Celsius.
+
+	 * For TMU Rev2:
+	 * 31  ... 8 7 6 5 4 3 2 1 0
+	 *  V          TEMP
+	 *
+	 * Where V bit signifies if the measurement is ready and is
+	 * within sensor range. TEMP is an 9 bit value representing
+	 * temperature in KelVin.
 	 */
 	if (regmap_read_poll_timeout(qdata->regmap,
 				     REGS_TRITSR(qsensor->id),
@@ -100,7 +114,10 @@ static int tmu_get_temp(void *p, int *temp)
 				     10 * USEC_PER_MSEC))
 		return -ENODATA;
 
-	*temp = (val & 0xff) * 1000;
+	if (qdata->ver == TMU_VER1)
+		*temp = (val & GENMASK(7, 0)) * MILLIDEGREE_PER_DEGREE;
+	else
+		*temp = kelvin_to_millicelsius(val & GENMASK(8, 0));
 
 	return 0;
 }
@@ -192,6 +209,8 @@ static int qoriq_tmu_calibration(struct device *dev,
 
 static void qoriq_tmu_init_device(struct qoriq_tmu_data *data)
 {
+	int i;
+
 	/* Disable interrupt, using polling instead */
 	regmap_write(data->regmap, REGS_TIER, TIER_DISABLE);
 
@@ -202,6 +221,8 @@ static void qoriq_tmu_init_device(struct qoriq_tmu_data *data)
 	} else {
 		regmap_write(data->regmap, REGS_V2_TMTMIR, TMTMIR_DEFAULT);
 		regmap_write(data->regmap, REGS_V2_TEUMR(0), TEUMR0_V2);
+		for (i = 0; i < SITES_MAX; i++)
+			regmap_write(data->regmap, REGS_V2_TMSAR(i), TMSARA_V2);
 	}
 
 	/* Disable monitoring */
@@ -212,6 +233,7 @@ static const struct regmap_range qoriq_yes_ranges[] = {
 	regmap_reg_range(REGS_TMR, REGS_TSCFGR),
 	regmap_reg_range(REGS_TTRnCR(0), REGS_TTRnCR(3)),
 	regmap_reg_range(REGS_V2_TEUMR(0), REGS_V2_TEUMR(2)),
+	regmap_reg_range(REGS_V2_TMSAR(0), REGS_V2_TMSAR(15)),
 	regmap_reg_range(REGS_IPBRR(0), REGS_IPBRR(1)),
 	/* Read only registers below */
 	regmap_reg_range(REGS_TRITSR(0), REGS_TRITSR(15)),
diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c
index e0c1f2409035..46aeb28b4e90 100644
--- a/drivers/thermal/rcar_thermal.c
+++ b/drivers/thermal/rcar_thermal.c
@@ -198,8 +198,8 @@ static void _rcar_thermal_bset(struct rcar_thermal_priv *priv, u32 reg,
 static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv)
 {
 	struct device *dev = rcar_priv_to_dev(priv);
-	int i;
-	u32 ctemp, old, new;
+	int old, new, ctemp = -EINVAL;
+	unsigned int i;
 
 	mutex_lock(&priv->lock);
 
@@ -209,7 +209,6 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv)
 	 */
 	rcar_thermal_bset(priv, THSCR, CPCTL, CPCTL);
 
-	ctemp = 0;
 	old = ~0;
 	for (i = 0; i < 128; i++) {
 		/*
@@ -227,7 +226,7 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv)
 		old = new;
 	}
 
-	if (!ctemp) {
+	if (ctemp < 0) {
 		dev_err(dev, "thermal sensor was broken\n");
 		goto err_out_unlock;
 	}
@@ -248,7 +247,7 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv)
 err_out_unlock:
 	mutex_unlock(&priv->lock);
 
-	return ctemp ? ctemp : -EINVAL;
+	return ctemp;
 }
 
 static int rcar_thermal_get_current_temp(struct rcar_thermal_priv *priv,
diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c
index 7c1a8bccdcba..15a71ecc916c 100644
--- a/drivers/thermal/rockchip_thermal.c
+++ b/drivers/thermal/rockchip_thermal.c
@@ -1241,10 +1241,8 @@ static int rockchip_thermal_probe(struct platform_device *pdev)
 		return -ENXIO;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(&pdev->dev, "no irq resource?\n");
+	if (irq < 0)
 		return -EINVAL;
-	}
 
 	thermal = devm_kzalloc(&pdev->dev, sizeof(struct rockchip_thermal_data),
 			       GFP_KERNEL);
diff --git a/drivers/thermal/st/st_thermal_memmap.c b/drivers/thermal/st/st_thermal_memmap.c
index a824b78dabf8..a0114452d11f 100644
--- a/drivers/thermal/st/st_thermal_memmap.c
+++ b/drivers/thermal/st/st_thermal_memmap.c
@@ -94,10 +94,8 @@ static int st_mmap_register_enable_irq(struct st_thermal_sensor *sensor)
 	int ret;
 
 	sensor->irq = platform_get_irq(pdev, 0);
-	if (sensor->irq < 0) {
-		dev_err(dev, "failed to register IRQ\n");
+	if (sensor->irq < 0)
 		return sensor->irq;
-	}
 
 	ret = devm_request_threaded_irq(dev, sensor->irq,
 					NULL, st_mmap_thermal_trip_handler,
diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c
index 9314e3df6a42..331e2b768df5 100644
--- a/drivers/thermal/st/stm_thermal.c
+++ b/drivers/thermal/st/stm_thermal.c
@@ -385,10 +385,8 @@ static int stm_register_irq(struct stm_thermal_sensor *sensor)
 	int ret;
 
 	sensor->irq = platform_get_irq(pdev, 0);
-	if (sensor->irq < 0) {
-		dev_err(dev, "%s: Unable to find IRQ\n", __func__);
+	if (sensor->irq < 0)
 		return sensor->irq;
-	}
 
 	ret = devm_request_threaded_irq(dev, sensor->irq,
 					NULL,
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 9a321dc548c8..b71196eaf90e 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -9,9 +9,9 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/module.h>
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/slab.h>
 #include <linux/kdev_t.h>
 #include <linux/idr.h>
@@ -27,10 +27,6 @@
 #include "thermal_core.h"
 #include "thermal_hwmon.h"
 
-MODULE_AUTHOR("Zhang Rui");
-MODULE_DESCRIPTION("Generic thermal management sysfs support");
-MODULE_LICENSE("GPL v2");
-
 static DEFINE_IDA(thermal_tz_ida);
 static DEFINE_IDA(thermal_cdev_ida);
 
@@ -447,12 +443,6 @@ static void update_temperature(struct thermal_zone_device *tz)
 	mutex_unlock(&tz->lock);
 
 	trace_thermal_temperature(tz);
-	if (tz->last_temperature == THERMAL_TEMP_INVALID)
-		dev_dbg(&tz->device, "last_temperature N/A, current_temperature=%d\n",
-			tz->temperature);
-	else
-		dev_dbg(&tz->device, "last_temperature=%d, current_temperature=%d\n",
-			tz->last_temperature, tz->temperature);
 }
 
 static void thermal_zone_device_init(struct thermal_zone_device *tz)
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index a9bf00e91d64..c95689586e19 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -12,6 +12,17 @@
 #include <linux/device.h>
 #include <linux/thermal.h>
 
+/* Default Thermal Governor */
+#if defined(CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE)
+#define DEFAULT_THERMAL_GOVERNOR       "step_wise"
+#elif defined(CONFIG_THERMAL_DEFAULT_GOV_FAIR_SHARE)
+#define DEFAULT_THERMAL_GOVERNOR       "fair_share"
+#elif defined(CONFIG_THERMAL_DEFAULT_GOV_USER_SPACE)
+#define DEFAULT_THERMAL_GOVERNOR       "user_space"
+#elif defined(CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR)
+#define DEFAULT_THERMAL_GOVERNOR       "power_allocator"
+#endif
+
 /* Initial state of a cooling device during binding */
 #define THERMAL_NO_TARGET -1UL
 
@@ -30,6 +41,44 @@ extern struct thermal_governor *__governor_thermal_table_end[];
 	     __governor < __governor_thermal_table_end;	\
 	     __governor++)
 
+struct thermal_attr {
+	struct device_attribute attr;
+	char name[THERMAL_NAME_LENGTH];
+};
+
+static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev)
+{
+	return cdev->ops->get_requested_power && cdev->ops->state2power &&
+		cdev->ops->power2state;
+}
+
+int power_actor_get_max_power(struct thermal_cooling_device *cdev,
+			      struct thermal_zone_device *tz, u32 *max_power);
+int power_actor_get_min_power(struct thermal_cooling_device *cdev,
+			      struct thermal_zone_device *tz, u32 *min_power);
+int power_actor_set_power(struct thermal_cooling_device *cdev,
+			  struct thermal_instance *ti, u32 power);
+/**
+ * struct thermal_trip - representation of a point in temperature domain
+ * @np: pointer to struct device_node that this trip point was created from
+ * @temperature: temperature value in miliCelsius
+ * @hysteresis: relative hysteresis in miliCelsius
+ * @type: trip point type
+ */
+struct thermal_trip {
+	struct device_node *np;
+	int temperature;
+	int hysteresis;
+	enum thermal_trip_type type;
+};
+
+int get_tz_trend(struct thermal_zone_device *tz, int trip);
+
+struct thermal_instance *
+get_thermal_instance(struct thermal_zone_device *tz,
+		     struct thermal_cooling_device *cdev,
+		     int trip);
+
 /*
  * This structure is used to describe the behavior of
  * a certain cooling device on a certain trip point
@@ -69,6 +118,9 @@ void thermal_zone_device_unbind_exception(struct thermal_zone_device *,
 int thermal_zone_device_set_policy(struct thermal_zone_device *, char *);
 int thermal_build_list_of_policies(char *buf);
 
+/* Helpers */
+void thermal_zone_set_trips(struct thermal_zone_device *tz);
+
 /* sysfs I/F */
 int thermal_zone_create_device_groups(struct thermal_zone_device *, int);
 void thermal_zone_destroy_device_groups(struct thermal_zone_device *);
diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c
index 2ba756af76b7..87b1256fa2f2 100644
--- a/drivers/thermal/thermal_helpers.c
+++ b/drivers/thermal/thermal_helpers.c
@@ -12,11 +12,12 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/sysfs.h>
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/sysfs.h>
 
 #include <trace/events/thermal.h>
 
@@ -113,6 +114,18 @@ exit:
 }
 EXPORT_SYMBOL_GPL(thermal_zone_get_temp);
 
+/**
+ * thermal_zone_set_trips - Computes the next trip points for the driver
+ * @tz: a pointer to a thermal zone device structure
+ *
+ * The function computes the next temperature boundaries by browsing
+ * the trip points. The result is the closer low and high trip points
+ * to the current temperature. These values are passed to the backend
+ * driver to let it set its own notification mechanism (usually an
+ * interrupt).
+ *
+ * It does not return a value
+ */
 void thermal_zone_set_trips(struct thermal_zone_device *tz)
 {
 	int low = -INT_MAX;
@@ -161,7 +174,6 @@ void thermal_zone_set_trips(struct thermal_zone_device *tz)
 exit:
 	mutex_unlock(&tz->lock);
 }
-EXPORT_SYMBOL_GPL(thermal_zone_set_trips);
 
 void thermal_cdev_update(struct thermal_cooling_device *cdev)
 {
diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c
index c8d2620f2e42..8b92e00ff236 100644
--- a/drivers/thermal/thermal_hwmon.c
+++ b/drivers/thermal/thermal_hwmon.c
@@ -10,10 +10,12 @@
  *  Copyright (C) 2013 Texas Instruments
  *  Copyright (C) 2013 Eduardo Valentin <eduardo.valentin@ti.com>
  */
+#include <linux/err.h>
+#include <linux/export.h>
 #include <linux/hwmon.h>
-#include <linux/thermal.h>
 #include <linux/slab.h>
-#include <linux/err.h>
+#include <linux/thermal.h>
+
 #include "thermal_hwmon.h"
 
 /* hwmon sys I/F */
diff --git a/drivers/thermal/of-thermal.c b/drivers/thermal/thermal_of.c
index 874a47d6923f..ddf88dbe7ba2 100644
--- a/drivers/thermal/of-thermal.c
+++ b/drivers/thermal/thermal_of.c
@@ -8,13 +8,13 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/thermal.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/of_device.h>
-#include <linux/of_platform.h>
 #include <linux/err.h>
 #include <linux/export.h>
+#include <linux/of_device.h>
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <linux/thermal.h>
+#include <linux/types.h>
 #include <linux/string.h>
 
 #include "thermal_core.h"
diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.c b/drivers/thermal/ti-soc-thermal/ti-bandgap.c
index 263b0420fbe4..ab19ceff6e2a 100644
--- a/drivers/thermal/ti-soc-thermal/ti-bandgap.c
+++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.c
@@ -772,10 +772,9 @@ static int ti_bandgap_talert_init(struct ti_bandgap *bgp,
 	int ret;
 
 	bgp->irq = platform_get_irq(pdev, 0);
-	if (bgp->irq < 0) {
-		dev_err(&pdev->dev, "get_irq failed\n");
+	if (bgp->irq < 0)
 		return bgp->irq;
-	}
+
 	ret = request_threaded_irq(bgp->irq, NULL,
 				   ti_bandgap_talert_irq_handler,
 				   IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
diff --git a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
index d3e959d01606..85776db4bf34 100644
--- a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
+++ b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
@@ -169,7 +169,7 @@ int ti_thermal_expose_sensor(struct ti_bandgap *bgp, int id,
 
 	data = ti_bandgap_get_sensor_data(bgp, id);
 
-	if (!data || IS_ERR(data))
+	if (!IS_ERR_OR_NULL(data))
 		data = ti_thermal_build_data(bgp, id);
 
 	if (!data)
@@ -196,7 +196,7 @@ int ti_thermal_remove_sensor(struct ti_bandgap *bgp, int id)
 
 	data = ti_bandgap_get_sensor_data(bgp, id);
 
-	if (data && data->ti_thermal) {
+	if (!IS_ERR_OR_NULL(data) && data->ti_thermal) {
 		if (data->our_zone)
 			thermal_zone_device_unregister(data->ti_thermal);
 	}
@@ -262,7 +262,7 @@ int ti_thermal_unregister_cpu_cooling(struct ti_bandgap *bgp, int id)
 
 	data = ti_bandgap_get_sensor_data(bgp, id);
 
-	if (data) {
+	if (!IS_ERR_OR_NULL(data)) {
 		cpufreq_cooling_unregister(data->cool_dev);
 		if (data->policy)
 			cpufreq_cpu_put(data->policy);
diff --git a/drivers/thunderbolt/Kconfig b/drivers/thunderbolt/Kconfig
index 1eb757e8df3b..f02010738bb6 100644
--- a/drivers/thunderbolt/Kconfig
+++ b/drivers/thunderbolt/Kconfig
@@ -2,7 +2,6 @@
 menuconfig USB4
 	tristate "Unified support for USB4 and Thunderbolt"
 	depends on PCI
-	depends on X86 || COMPILE_TEST
 	select APPLE_PROPERTIES if EFI_STUB && X86
 	select CRC32
 	select CRYPTO
diff --git a/drivers/thunderbolt/icm.c b/drivers/thunderbolt/icm.c
index fbbe32ca1e69..ffcc8c3459e5 100644
--- a/drivers/thunderbolt/icm.c
+++ b/drivers/thunderbolt/icm.c
@@ -1633,6 +1633,15 @@ static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
 		icm_veto_end(tb);
 }
 
+static bool icm_tgl_is_supported(struct tb *tb)
+{
+	/*
+	 * If the firmware is not running use software CM. This platform
+	 * should fully support both.
+	 */
+	return icm_firmware_running(tb->nhi);
+}
+
 static void icm_handle_notification(struct work_struct *work)
 {
 	struct icm_notification *n = container_of(work, typeof(*n), work);
@@ -2269,6 +2278,19 @@ struct tb *icm_probe(struct tb_nhi *nhi)
 		icm->rtd3_veto = icm_icl_rtd3_veto;
 		tb->cm_ops = &icm_icl_ops;
 		break;
+
+	case PCI_DEVICE_ID_INTEL_TGL_NHI0:
+	case PCI_DEVICE_ID_INTEL_TGL_NHI1:
+		icm->is_supported = icm_tgl_is_supported;
+		icm->driver_ready = icm_icl_driver_ready;
+		icm->set_uuid = icm_icl_set_uuid;
+		icm->device_connected = icm_icl_device_connected;
+		icm->device_disconnected = icm_tr_device_disconnected;
+		icm->xdomain_connected = icm_tr_xdomain_connected;
+		icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
+		icm->rtd3_veto = icm_icl_rtd3_veto;
+		tb->cm_ops = &icm_icl_ops;
+		break;
 	}
 
 	if (!icm->is_supported || !icm->is_supported(tb)) {
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index 1be491ecbb45..d299dc168147 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -1270,6 +1270,10 @@ static struct pci_device_id nhi_ids[] = {
 	  .driver_data = (kernel_ulong_t)&icl_nhi_ops },
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ICL_NHI1),
 	  .driver_data = (kernel_ulong_t)&icl_nhi_ops },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI0),
+	  .driver_data = (kernel_ulong_t)&icl_nhi_ops },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI1),
+	  .driver_data = (kernel_ulong_t)&icl_nhi_ops },
 
 	/* Any USB4 compliant host */
 	{ PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_USB4, ~0) },
@@ -1285,6 +1289,7 @@ static struct pci_driver nhi_driver = {
 	.id_table = nhi_ids,
 	.probe = nhi_probe,
 	.remove = nhi_remove,
+	.shutdown = nhi_remove,
 	.driver.pm = &nhi_pm_ops,
 };
 
diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h
index 5d276ee9b38e..80162e4b013f 100644
--- a/drivers/thunderbolt/nhi.h
+++ b/drivers/thunderbolt/nhi.h
@@ -73,6 +73,8 @@ extern const struct tb_nhi_ops icl_nhi_ops;
 #define PCI_DEVICE_ID_INTEL_TITAN_RIDGE_DD_BRIDGE	0x15ef
 #define PCI_DEVICE_ID_INTEL_ICL_NHI1			0x8a0d
 #define PCI_DEVICE_ID_INTEL_ICL_NHI0			0x8a17
+#define PCI_DEVICE_ID_INTEL_TGL_NHI0			0x9a1b
+#define PCI_DEVICE_ID_INTEL_TGL_NHI1			0x9a1d
 
 #define PCI_CLASS_SERIAL_USB_USB4			0x0c0340
 
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index a2ce99051c51..d7d60cd9226f 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -263,7 +263,7 @@ static void nvm_authenticate_start_dma_port(struct tb_switch *sw)
 	 * itself. To be on the safe side keep the root port in D0 during
 	 * the whole upgrade process.
 	 */
-	root_port = pci_find_pcie_root_port(sw->tb->nhi->pdev);
+	root_port = pcie_find_root_port(sw->tb->nhi->pdev);
 	if (root_port)
 		pm_runtime_get_noresume(&root_port->dev);
 }
@@ -272,7 +272,7 @@ static void nvm_authenticate_complete_dma_port(struct tb_switch *sw)
 {
 	struct pci_dev *root_port;
 
-	root_port = pci_find_pcie_root_port(sw->tb->nhi->pdev);
+	root_port = pcie_find_root_port(sw->tb->nhi->pdev);
 	if (root_port)
 		pm_runtime_put(&root_port->dev);
 }
@@ -348,12 +348,6 @@ out:
 	return ret;
 }
 
-static int tb_switch_nvm_no_read(void *priv, unsigned int offset, void *val,
-				 size_t bytes)
-{
-	return -EPERM;
-}
-
 static int tb_switch_nvm_write(void *priv, unsigned int offset, void *val,
 			       size_t bytes)
 {
@@ -399,7 +393,6 @@ static struct nvmem_device *register_nvmem(struct tb_switch *sw, int id,
 		config.read_only = true;
 	} else {
 		config.name = "nvm_non_active";
-		config.reg_read = tb_switch_nvm_no_read;
 		config.reg_write = tb_switch_nvm_write;
 		config.root_only = true;
 	}
diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig
index 2dff93d7a501..93fd984eb2f5 100644
--- a/drivers/tty/Kconfig
+++ b/drivers/tty/Kconfig
@@ -2,7 +2,7 @@
 config TTY
 	bool "Enable TTY" if EXPERT
 	default y
-	---help---
+	help
 	  Allows you to remove TTY support which can save space, and
 	  blocks features that require TTY from inclusion in the kernel.
 	  TTY is required for any text terminals or serial port
@@ -15,7 +15,7 @@ config VT
 	depends on !UML
 	select INPUT
 	default y
-	---help---
+	help
 	  If you say Y here, you will get support for terminal devices with
 	  display and keyboard devices. These are called "virtual" because you
 	  can run several virtual terminals (also called virtual consoles) on
@@ -46,7 +46,7 @@ config CONSOLE_TRANSLATIONS
 	depends on VT
 	default y
 	bool "Enable character translations in console" if EXPERT
-	---help---
+	help
 	  This enables support for font mapping and Unicode translation
 	  on virtual consoles.
 
@@ -54,7 +54,7 @@ config VT_CONSOLE
 	bool "Support for console on virtual terminal" if EXPERT
 	depends on VT
 	default y
-	---help---
+	help
 	  The system console is the device which receives all kernel messages
 	  and warnings and which allows logins in single user mode. If you
 	  answer Y here, a virtual terminal (the device used to interact with
@@ -84,7 +84,7 @@ config HW_CONSOLE
 config VT_HW_CONSOLE_BINDING
 	bool "Support for binding and unbinding console drivers"
 	depends on HW_CONSOLE
-	---help---
+	help
 	  The virtual terminal is the device that interacts with the physical
 	  terminal through console drivers. On these systems, at least one
 	  console driver is loaded. In other configurations, additional console
@@ -100,7 +100,7 @@ config VT_HW_CONSOLE_BINDING
 config UNIX98_PTYS
 	bool "Unix98 PTY support" if EXPERT
 	default y
-	---help---
+	help
 	  A pseudo terminal (PTY) is a software device consisting of two
 	  halves: a master and a slave. The slave device behaves identical to
 	  a physical terminal; the master device is used by a process to
@@ -123,7 +123,7 @@ config UNIX98_PTYS
 config LEGACY_PTYS
 	bool "Legacy (BSD) PTY support"
 	default y
-	---help---
+	help
 	  A pseudo terminal (PTY) is a software device consisting of two
 	  halves: a master and a slave. The slave device behaves identical to
 	  a physical terminal; the master device is used by a process to
@@ -142,7 +142,7 @@ config LEGACY_PTY_COUNT
 	depends on LEGACY_PTYS
 	range 0 256
 	default "256"
-	---help---
+	help
 	  The maximum number of legacy PTYs that can be used at any one time.
 	  The default is 256, and should be more than enough.  Embedded
 	  systems may want to reduce this to save memory.
@@ -178,7 +178,7 @@ source "drivers/tty/serial/Kconfig"
 config SERIAL_NONSTANDARD
 	bool "Non-standard serial port support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  Say Y here if you have any non-standard serial boards -- boards
 	  which aren't supported using the standard "dumb" serial driver.
 	  This includes intelligent serial boards such as Cyclades,
@@ -211,7 +211,7 @@ config CYCLADES
 	tristate "Cyclades async mux support"
 	depends on SERIAL_NONSTANDARD && (PCI || ISA)
 	select FW_LOADER
-	---help---
+	help
 	  This driver supports Cyclades Z and Y multiserial boards.
 	  You would need something like this to connect more than two modems to
 	  your Linux box, for instance in order to become a dial-in server.
diff --git a/drivers/tty/hvc/hvc_console.c b/drivers/tty/hvc/hvc_console.c
index 436cc51c92c3..cdcc64ea2554 100644
--- a/drivers/tty/hvc/hvc_console.c
+++ b/drivers/tty/hvc/hvc_console.c
@@ -371,15 +371,14 @@ static int hvc_open(struct tty_struct *tty, struct file * filp)
 	 * tty fields and return the kref reference.
 	 */
 	if (rc) {
-		tty_port_tty_set(&hp->port, NULL);
-		tty->driver_data = NULL;
-		tty_port_put(&hp->port);
 		printk(KERN_ERR "hvc_open: request_irq failed with rc %d.\n", rc);
-	} else
+	} else {
 		/* We are ready... raise DTR/RTS */
 		if (C_BAUD(tty))
 			if (hp->ops->dtr_rts)
 				hp->ops->dtr_rts(hp, 1);
+		tty_port_set_initialized(&hp->port, true);
+	}
 
 	/* Force wakeup of the polling thread */
 	hvc_kick();
@@ -389,22 +388,12 @@ static int hvc_open(struct tty_struct *tty, struct file * filp)
 
 static void hvc_close(struct tty_struct *tty, struct file * filp)
 {
-	struct hvc_struct *hp;
+	struct hvc_struct *hp = tty->driver_data;
 	unsigned long flags;
 
 	if (tty_hung_up_p(filp))
 		return;
 
-	/*
-	 * No driver_data means that this close was issued after a failed
-	 * hvc_open by the tty layer's release_dev() function and we can just
-	 * exit cleanly because the kref reference wasn't made.
-	 */
-	if (!tty->driver_data)
-		return;
-
-	hp = tty->driver_data;
-
 	spin_lock_irqsave(&hp->port.lock, flags);
 
 	if (--hp->port.count == 0) {
@@ -412,6 +401,9 @@ static void hvc_close(struct tty_struct *tty, struct file * filp)
 		/* We are done with the tty pointer now. */
 		tty_port_tty_set(&hp->port, NULL);
 
+		if (!tty_port_initialized(&hp->port))
+			return;
+
 		if (C_HUPCL(tty))
 			if (hp->ops->dtr_rts)
 				hp->ops->dtr_rts(hp, 0);
@@ -428,6 +420,7 @@ static void hvc_close(struct tty_struct *tty, struct file * filp)
 		 * waking periodically to check chars_in_buffer().
 		 */
 		tty_wait_until_sent(tty, HVC_CLOSE_WAIT);
+		tty_port_set_initialized(&hp->port, false);
 	} else {
 		if (hp->port.count < 0)
 			printk(KERN_ERR "hvc_close %X: oops, count is %d\n",
diff --git a/drivers/tty/hvc/hvcs.c b/drivers/tty/hvc/hvcs.c
index ee0604cd9c6b..55105ac38f89 100644
--- a/drivers/tty/hvc/hvcs.c
+++ b/drivers/tty/hvc/hvcs.c
@@ -196,8 +196,6 @@ module_param(hvcs_parm_num_devs, int, 0);
 
 static const char hvcs_driver_name[] = "hvcs";
 static const char hvcs_device_node[] = "hvcs";
-static const char hvcs_driver_string[]
-	= "IBM hvcs (Hypervisor Virtual Console Server) Driver";
 
 /* Status of partner info rescan triggered via sysfs. */
 static int hvcs_rescan_status;
diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c
index 9d00ff5ef961..3703987c4666 100644
--- a/drivers/tty/mxser.c
+++ b/drivers/tty/mxser.c
@@ -638,16 +638,15 @@ static int mxser_set_baud(struct tty_struct *tty, long newspd)
  * This routine is called to set the UART divisor registers to match
  * the specified baud rate for a serial port.
  */
-static int mxser_change_speed(struct tty_struct *tty)
+static void mxser_change_speed(struct tty_struct *tty)
 {
 	struct mxser_port *info = tty->driver_data;
 	unsigned cflag, cval, fcr;
-	int ret = 0;
 	unsigned char status;
 
 	cflag = tty->termios.c_cflag;
 	if (!info->ioaddr)
-		return ret;
+		return;
 
 	if (mxser_set_baud_method[tty->index] == 0)
 		mxser_set_baud(tty, tty_get_baud_rate(tty));
@@ -803,8 +802,6 @@ static int mxser_change_speed(struct tty_struct *tty)
 
 	outb(fcr, info->ioaddr + UART_FCR);	/* set fcr */
 	outb(cval, info->ioaddr + UART_LCR);
-
-	return ret;
 }
 
 static void mxser_check_modem_status(struct tty_struct *tty,
diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c
index d77ed82a4840..0a29a94ec438 100644
--- a/drivers/tty/n_gsm.c
+++ b/drivers/tty/n_gsm.c
@@ -504,18 +504,7 @@ static void gsm_print_packet(const char *hdr, int addr, int cr,
 	else
 		pr_cont("(F)");
 
-	if (dlen) {
-		int ct = 0;
-		while (dlen--) {
-			if (ct % 8 == 0) {
-				pr_cont("\n");
-				pr_debug("    ");
-			}
-			pr_cont("%02X ", *data++);
-			ct++;
-		}
-	}
-	pr_cont("\n");
+	print_hex_dump_bytes("", DUMP_PREFIX_NONE, data, dlen);
 }
 
 
@@ -673,11 +662,10 @@ static struct gsm_msg *gsm_data_alloc(struct gsm_mux *gsm, u8 addr, int len,
  *	FIXME: lock against link layer control transmissions
  */
 
-static void gsm_data_kick(struct gsm_mux *gsm)
+static void gsm_data_kick(struct gsm_mux *gsm, struct gsm_dlci *dlci)
 {
 	struct gsm_msg *msg, *nmsg;
 	int len;
-	int skip_sof = 0;
 
 	list_for_each_entry_safe(msg, nmsg, &gsm->tx_list, list) {
 		if (gsm->constipated && msg->addr)
@@ -699,18 +687,23 @@ static void gsm_data_kick(struct gsm_mux *gsm)
 			print_hex_dump_bytes("gsm_data_kick: ",
 					     DUMP_PREFIX_OFFSET,
 					     gsm->txframe, len);
-
-		if (gsm->output(gsm, gsm->txframe + skip_sof,
-						len - skip_sof) < 0)
+		if (gsm->output(gsm, gsm->txframe, len) < 0)
 			break;
 		/* FIXME: Can eliminate one SOF in many more cases */
 		gsm->tx_bytes -= msg->len;
-		/* For a burst of frames skip the extra SOF within the
-		   burst */
-		skip_sof = 1;
 
 		list_del(&msg->list);
 		kfree(msg);
+
+		if (dlci) {
+			tty_port_tty_wakeup(&dlci->port);
+		} else {
+			int i = 0;
+
+			for (i = 0; i < NUM_DLCI; i++)
+				if (gsm->dlci[i])
+					tty_port_tty_wakeup(&gsm->dlci[i]->port);
+		}
 	}
 }
 
@@ -762,7 +755,7 @@ static void __gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg)
 	/* Add to the actual output queue */
 	list_add_tail(&msg->list, &gsm->tx_list);
 	gsm->tx_bytes += msg->len;
-	gsm_data_kick(gsm);
+	gsm_data_kick(gsm, dlci);
 }
 
 /**
@@ -1223,7 +1216,7 @@ static void gsm_control_message(struct gsm_mux *gsm, unsigned int command,
 		gsm_control_reply(gsm, CMD_FCON, NULL, 0);
 		/* Kick the link in case it is idling */
 		spin_lock_irqsave(&gsm->tx_lock, flags);
-		gsm_data_kick(gsm);
+		gsm_data_kick(gsm, NULL);
 		spin_unlock_irqrestore(&gsm->tx_lock, flags);
 		break;
 	case CMD_FCOFF:
@@ -2545,7 +2538,7 @@ static void gsmld_write_wakeup(struct tty_struct *tty)
 	/* Queue poll */
 	clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags);
 	spin_lock_irqsave(&gsm->tx_lock, flags);
-	gsm_data_kick(gsm);
+	gsm_data_kick(gsm, NULL);
 	if (gsm->tx_bytes < TX_THRESH_LO) {
 		gsm_dlci_data_sweep(gsm);
 	}
diff --git a/drivers/tty/rocket.c b/drivers/tty/rocket.c
index e2138e7d5dc6..2540b2e4c8e8 100644
--- a/drivers/tty/rocket.c
+++ b/drivers/tty/rocket.c
@@ -1885,7 +1885,7 @@ static int sPCIInitController(CONTROLLER_T * CtlP, int CtlNum,
  */
 static __init int register_PCI(int i, struct pci_dev *dev)
 {
-	int num_aiops, aiop, max_num_aiops, num_chan, chan;
+	int num_aiops, aiop, max_num_aiops, chan;
 	unsigned int aiopio[MAX_AIOPS_PER_BOARD];
 	CONTROLLER_t *ctlp;
 
@@ -2157,8 +2157,7 @@ static __init int register_PCI(int i, struct pci_dev *dev)
 	/*  Reset the AIOPIC, init the serial ports */
 	for (aiop = 0; aiop < num_aiops; aiop++) {
 		sResetAiopByNum(ctlp, aiop);
-		num_chan = ports_per_aiop;
-		for (chan = 0; chan < num_chan; chan++)
+		for (chan = 0; chan < ports_per_aiop; chan++)
 			init_r_port(i, aiop, chan, dev);
 	}
 
@@ -2166,11 +2165,10 @@ static __init int register_PCI(int i, struct pci_dev *dev)
 	if ((rcktpt_type[i] == ROCKET_TYPE_MODEM) ||
 	    (rcktpt_type[i] == ROCKET_TYPE_MODEMII) ||
 	    (rcktpt_type[i] == ROCKET_TYPE_MODEMIII)) {
-		num_chan = ports_per_aiop;
-		for (chan = 0; chan < num_chan; chan++)
+		for (chan = 0; chan < ports_per_aiop; chan++)
 			sPCIModemReset(ctlp, chan, 1);
 		msleep(500);
-		for (chan = 0; chan < num_chan; chan++)
+		for (chan = 0; chan < ports_per_aiop; chan++)
 			sPCIModemReset(ctlp, chan, 0);
 		msleep(500);
 		rmSpeakerReset(ctlp, rocketModel[i].model);
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index 45d9117cab68..fc118f649887 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -1026,7 +1026,9 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 
 		if (up->port.dev) {
 			uart->port.dev = up->port.dev;
-			uart_get_rs485_mode(uart->port.dev, &uart->port.rs485);
+			ret = uart_get_rs485_mode(&uart->port);
+			if (ret)
+				goto err;
 		}
 
 		if (up->port.flags & UPF_FIXED_TYPE)
@@ -1040,7 +1042,7 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 			gpios = mctrl_gpio_init(&uart->port, 0);
 			if (IS_ERR(gpios)) {
 				ret = PTR_ERR(gpios);
-				goto out_unlock;
+				goto err;
 			} else {
 				uart->gpios = gpios;
 			}
@@ -1089,8 +1091,10 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 			serial8250_apply_quirks(uart);
 			ret = uart_add_one_port(&serial8250_reg,
 						&uart->port);
-			if (ret == 0)
-				ret = uart->port.line;
+			if (ret)
+				goto err;
+
+			ret = uart->port.line;
 		} else {
 			dev_info(uart->port.dev,
 				"skipping CIR port at 0x%lx / 0x%llx, IRQ %d\n",
@@ -1112,10 +1116,14 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 		}
 	}
 
-out_unlock:
 	mutex_unlock(&serial_mutex);
 
 	return ret;
+
+err:
+	uart->port.dev = NULL;
+	mutex_unlock(&serial_mutex);
+	return ret;
 }
 EXPORT_SYMBOL(serial8250_register_8250_port);
 
diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c
index 59449b6500cd..ddb6aeb76dc5 100644
--- a/drivers/tty/serial/8250/8250_exar.c
+++ b/drivers/tty/serial/8250/8250_exar.c
@@ -25,13 +25,13 @@
 
 #include "8250.h"
 
-#define PCI_DEVICE_ID_ACCES_COM_2S		0x1052
-#define PCI_DEVICE_ID_ACCES_COM_4S		0x105d
-#define PCI_DEVICE_ID_ACCES_COM_8S		0x106c
-#define PCI_DEVICE_ID_ACCES_COM232_8		0x10a8
-#define PCI_DEVICE_ID_ACCES_COM_2SM		0x10d2
-#define PCI_DEVICE_ID_ACCES_COM_4SM		0x10db
-#define PCI_DEVICE_ID_ACCES_COM_8SM		0x10ea
+#define PCI_DEVICE_ID_ACCESSIO_COM_2S		0x1052
+#define PCI_DEVICE_ID_ACCESSIO_COM_4S		0x105d
+#define PCI_DEVICE_ID_ACCESSIO_COM_8S		0x106c
+#define PCI_DEVICE_ID_ACCESSIO_COM232_8		0x10a8
+#define PCI_DEVICE_ID_ACCESSIO_COM_2SM		0x10d2
+#define PCI_DEVICE_ID_ACCESSIO_COM_4SM		0x10db
+#define PCI_DEVICE_ID_ACCESSIO_COM_8SM		0x10ea
 
 #define PCI_DEVICE_ID_COMMTECH_4224PCI335	0x0002
 #define PCI_DEVICE_ID_COMMTECH_4222PCI335	0x0004
@@ -755,9 +755,7 @@ static const struct exar8250_board pbn_exar_XR17V8358 = {
 		(kernel_ulong_t)&bd					\
 	}
 
-#define EXAR_DEVICE(vend, devid, bd) {					\
-	PCI_VDEVICE(vend, PCI_DEVICE_ID_##devid), (kernel_ulong_t)&bd	\
-	}
+#define EXAR_DEVICE(vend, devid, bd) { PCI_DEVICE_DATA(vend, devid, &bd) }
 
 #define IBM_DEVICE(devid, sdevid, bd) {			\
 	PCI_DEVICE_SUB(					\
@@ -769,14 +767,13 @@ static const struct exar8250_board pbn_exar_XR17V8358 = {
 	}
 
 static const struct pci_device_id exar_pci_tbl[] = {
-	EXAR_DEVICE(ACCESSIO, ACCES_COM_2S, acces_com_2x),
-	EXAR_DEVICE(ACCESSIO, ACCES_COM_4S, acces_com_4x),
-	EXAR_DEVICE(ACCESSIO, ACCES_COM_8S, acces_com_8x),
-	EXAR_DEVICE(ACCESSIO, ACCES_COM232_8, acces_com_8x),
-	EXAR_DEVICE(ACCESSIO, ACCES_COM_2SM, acces_com_2x),
-	EXAR_DEVICE(ACCESSIO, ACCES_COM_4SM, acces_com_4x),
-	EXAR_DEVICE(ACCESSIO, ACCES_COM_8SM, acces_com_8x),
-
+	EXAR_DEVICE(ACCESSIO, COM_2S, acces_com_2x),
+	EXAR_DEVICE(ACCESSIO, COM_4S, acces_com_4x),
+	EXAR_DEVICE(ACCESSIO, COM_8S, acces_com_8x),
+	EXAR_DEVICE(ACCESSIO, COM232_8, acces_com_8x),
+	EXAR_DEVICE(ACCESSIO, COM_2SM, acces_com_2x),
+	EXAR_DEVICE(ACCESSIO, COM_4SM, acces_com_4x),
+	EXAR_DEVICE(ACCESSIO, COM_8SM, acces_com_8x),
 
 	CONNECT_DEVICE(XR17C152, UART_2_232, pbn_connect),
 	CONNECT_DEVICE(XR17C154, UART_4_232, pbn_connect),
@@ -794,24 +791,24 @@ static const struct pci_device_id exar_pci_tbl[] = {
 	IBM_DEVICE(XR17C152, SATURN_SERIAL_ONE_PORT, pbn_exar_ibm_saturn),
 
 	/* Exar Corp. XR17C15[248] Dual/Quad/Octal UART */
-	EXAR_DEVICE(EXAR, EXAR_XR17C152, pbn_exar_XR17C15x),
-	EXAR_DEVICE(EXAR, EXAR_XR17C154, pbn_exar_XR17C15x),
-	EXAR_DEVICE(EXAR, EXAR_XR17C158, pbn_exar_XR17C15x),
+	EXAR_DEVICE(EXAR, XR17C152, pbn_exar_XR17C15x),
+	EXAR_DEVICE(EXAR, XR17C154, pbn_exar_XR17C15x),
+	EXAR_DEVICE(EXAR, XR17C158, pbn_exar_XR17C15x),
 
 	/* Exar Corp. XR17V[48]35[248] Dual/Quad/Octal/Hexa PCIe UARTs */
-	EXAR_DEVICE(EXAR, EXAR_XR17V352, pbn_exar_XR17V35x),
-	EXAR_DEVICE(EXAR, EXAR_XR17V354, pbn_exar_XR17V35x),
-	EXAR_DEVICE(EXAR, EXAR_XR17V358, pbn_exar_XR17V35x),
-	EXAR_DEVICE(EXAR, EXAR_XR17V4358, pbn_exar_XR17V4358),
-	EXAR_DEVICE(EXAR, EXAR_XR17V8358, pbn_exar_XR17V8358),
-	EXAR_DEVICE(COMMTECH, COMMTECH_4222PCIE, pbn_exar_XR17V35x),
-	EXAR_DEVICE(COMMTECH, COMMTECH_4224PCIE, pbn_exar_XR17V35x),
-	EXAR_DEVICE(COMMTECH, COMMTECH_4228PCIE, pbn_exar_XR17V35x),
-
-	EXAR_DEVICE(COMMTECH, COMMTECH_4222PCI335, pbn_fastcom335_2),
-	EXAR_DEVICE(COMMTECH, COMMTECH_4224PCI335, pbn_fastcom335_4),
-	EXAR_DEVICE(COMMTECH, COMMTECH_2324PCI335, pbn_fastcom335_4),
-	EXAR_DEVICE(COMMTECH, COMMTECH_2328PCI335, pbn_fastcom335_8),
+	EXAR_DEVICE(EXAR, XR17V352, pbn_exar_XR17V35x),
+	EXAR_DEVICE(EXAR, XR17V354, pbn_exar_XR17V35x),
+	EXAR_DEVICE(EXAR, XR17V358, pbn_exar_XR17V35x),
+	EXAR_DEVICE(EXAR, XR17V4358, pbn_exar_XR17V4358),
+	EXAR_DEVICE(EXAR, XR17V8358, pbn_exar_XR17V8358),
+	EXAR_DEVICE(COMMTECH, 4222PCIE, pbn_exar_XR17V35x),
+	EXAR_DEVICE(COMMTECH, 4224PCIE, pbn_exar_XR17V35x),
+	EXAR_DEVICE(COMMTECH, 4228PCIE, pbn_exar_XR17V35x),
+
+	EXAR_DEVICE(COMMTECH, 4222PCI335, pbn_fastcom335_2),
+	EXAR_DEVICE(COMMTECH, 4224PCI335, pbn_fastcom335_4),
+	EXAR_DEVICE(COMMTECH, 2324PCI335, pbn_fastcom335_4),
+	EXAR_DEVICE(COMMTECH, 2328PCI335, pbn_fastcom335_8),
 	{ 0, }
 };
 MODULE_DEVICE_TABLE(pci, exar_pci_tbl);
diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c
index 31c91c2f8c6e..d1d253c4b518 100644
--- a/drivers/tty/serial/8250/8250_fintek.c
+++ b/drivers/tty/serial/8250/8250_fintek.c
@@ -19,6 +19,7 @@
 #define CHIP_ID2  0x21
 #define CHIP_ID_F81865 0x0407
 #define CHIP_ID_F81866 0x1010
+#define CHIP_ID_F81966 0x0215
 #define CHIP_ID_F81216AD 0x1602
 #define CHIP_ID_F81216H 0x0501
 #define CHIP_ID_F81216 0x0802
@@ -62,9 +63,9 @@
 #define F81216_LDN_HIGH	0x4
 
 /*
- * F81866 registers
+ * F81866/966 registers
  *
- * The IRQ setting mode of F81866 is not the same with F81216 series.
+ * The IRQ setting mode of F81866/966 is not the same with F81216 series.
  *	Level/Low: IRQ_MODE0:0, IRQ_MODE1:0
  *	Edge/High: IRQ_MODE0:1, IRQ_MODE1:0
  *
@@ -155,6 +156,7 @@ static int fintek_8250_check_id(struct fintek_8250 *pdata)
 	switch (chip) {
 	case CHIP_ID_F81865:
 	case CHIP_ID_F81866:
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81216AD:
 	case CHIP_ID_F81216H:
 	case CHIP_ID_F81216:
@@ -171,6 +173,7 @@ static int fintek_8250_get_ldn_range(struct fintek_8250 *pdata, int *min,
 				     int *max)
 {
 	switch (pdata->pid) {
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81865:
 	case CHIP_ID_F81866:
 		*min = F81866_LDN_LOW;
@@ -248,6 +251,7 @@ static void fintek_8250_set_irq_mode(struct fintek_8250 *pdata, bool is_level)
 	sio_write_reg(pdata, LDN, pdata->index);
 
 	switch (pdata->pid) {
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81866:
 		sio_write_mask_reg(pdata, F81866_FIFO_CTRL, F81866_IRQ_MODE1,
 				   0);
@@ -274,6 +278,7 @@ static void fintek_8250_set_max_fifo(struct fintek_8250 *pdata)
 {
 	switch (pdata->pid) {
 	case CHIP_ID_F81216H: /* 128Bytes FIFO */
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81866:
 		sio_write_mask_reg(pdata, FIFO_CTRL,
 				   FIFO_MODE_MASK | RXFTHR_MODE_MASK,
@@ -291,6 +296,7 @@ static void fintek_8250_goto_highspeed(struct uart_8250_port *uart,
 	sio_write_reg(pdata, LDN, pdata->index);
 
 	switch (pdata->pid) {
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81866: /* set uart clock for high speed serial mode */
 		sio_write_mask_reg(pdata, F81866_UART_CLK,
 			F81866_UART_CLK_MASK,
@@ -327,6 +333,7 @@ static void fintek_8250_set_termios(struct uart_port *port,
 	case CHIP_ID_F81216H:
 		reg = RS485;
 		break;
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81866:
 		reg = F81866_UART_CLK;
 		break;
@@ -373,6 +380,7 @@ static void fintek_8250_set_termios_handler(struct uart_8250_port *uart)
 
 	switch (pdata->pid) {
 	case CHIP_ID_F81216H:
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81866:
 		uart->port.set_termios = fintek_8250_set_termios;
 		break;
@@ -443,6 +451,7 @@ static void fintek_8250_set_rs485_handler(struct uart_8250_port *uart)
 	switch (pdata->pid) {
 	case CHIP_ID_F81216AD:
 	case CHIP_ID_F81216H:
+	case CHIP_ID_F81966:
 	case CHIP_ID_F81866:
 	case CHIP_ID_F81865:
 		uart->port.rs485_config = fintek_8250_rs485_config;
diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
index 0804469ff052..1a74d511b02a 100644
--- a/drivers/tty/serial/8250/8250_pci.c
+++ b/drivers/tty/serial/8250/8250_pci.c
@@ -1869,12 +1869,6 @@ pci_moxa_setup(struct serial_private *priv,
 #define PCIE_DEVICE_ID_WCH_CH384_4S	0x3470
 #define PCIE_DEVICE_ID_WCH_CH382_2S	0x3253
 
-#define PCI_VENDOR_ID_PERICOM			0x12D8
-#define PCI_DEVICE_ID_PERICOM_PI7C9X7951	0x7951
-#define PCI_DEVICE_ID_PERICOM_PI7C9X7952	0x7952
-#define PCI_DEVICE_ID_PERICOM_PI7C9X7954	0x7954
-#define PCI_DEVICE_ID_PERICOM_PI7C9X7958	0x7958
-
 #define PCI_VENDOR_ID_ACCESIO			0x494f
 #define PCI_DEVICE_ID_ACCESIO_PCIE_COM_2SDB	0x1051
 #define PCI_DEVICE_ID_ACCESIO_MPCIE_COM_2S	0x1053
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index f77bf820b7a3..1632f7d25acc 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -681,6 +681,9 @@ int serial8250_em485_config(struct uart_port *port, struct serial_rs485 *rs485)
 	memset(rs485->padding, 0, sizeof(rs485->padding));
 	port->rs485 = *rs485;
 
+	gpiod_set_value(port->rs485_term_gpio,
+			rs485->flags & SER_RS485_TERMINATE_BUS);
+
 	/*
 	 * Both serial8250_em485_init() and serial8250_em485_destroy()
 	 * are idempotent.
@@ -1432,7 +1435,7 @@ static void serial8250_stop_rx(struct uart_port *port)
 
 /**
  * serial8250_em485_stop_tx() - generic ->rs485_stop_tx() callback
- * @up: uart 8250 port
+ * @p: uart 8250 port
  *
  * Generic callback usable by 8250 uart drivers to stop rs485 transmission.
  */
@@ -2615,6 +2618,8 @@ static unsigned int serial8250_get_baud_rate(struct uart_port *port,
 					     struct ktermios *termios,
 					     struct ktermios *old)
 {
+	unsigned int tolerance = port->uartclk / 100;
+
 	/*
 	 * Ask the core to calculate the divisor for us.
 	 * Allow 1% tolerance at the upper limit so uart clks marginally
@@ -2623,7 +2628,7 @@ static unsigned int serial8250_get_baud_rate(struct uart_port *port,
 	 */
 	return uart_get_baud_rate(port, termios, old,
 				  port->uartclk / 16 / UART_DIV_MAX,
-				  port->uartclk);
+				  (port->uartclk + tolerance) / 16);
 }
 
 void
diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig
index af0688156dd0..d2ae033aea40 100644
--- a/drivers/tty/serial/8250/Kconfig
+++ b/drivers/tty/serial/8250/Kconfig
@@ -9,7 +9,7 @@ config SERIAL_8250
 	depends on !S390
 	select SERIAL_CORE
 	select SERIAL_MCTRL_GPIO if GPIOLIB
-	---help---
+	help
 	  This selects whether you want to include the driver for the standard
 	  serial ports.  The standard answer is Y.  People who might say N
 	  here are those that are setting up dedicated Ethernet WWW/FTP
@@ -39,7 +39,7 @@ config SERIAL_8250_DEPRECATED_OPTIONS
 	bool "Support 8250_core.* kernel options (DEPRECATED)"
 	depends on SERIAL_8250
 	default y
-	---help---
+	help
 	  In 3.7 we renamed 8250 to 8250_core by mistake, so now we have to
 	  accept kernel parameters in both forms like 8250_core.nr_uarts=4 and
 	  8250.nr_uarts=4. We now renamed the module back to 8250, but if
@@ -56,13 +56,14 @@ config SERIAL_8250_PNP
 	bool "8250/16550 PNP device support" if EXPERT
 	depends on SERIAL_8250 && PNP
 	default y
-	---help---
+	help
 	  This builds standard PNP serial support. You may be able to
 	  disable this feature if you only need legacy serial support.
 
 config SERIAL_8250_16550A_VARIANTS
 	bool "Support for variants of the 16550A serial port"
 	depends on SERIAL_8250
+	default !X86
 	help
 	  The 8250 driver can probe for many variants of the venerable 16550A
 	  serial port. Doing so takes additional time at boot.
@@ -73,7 +74,7 @@ config SERIAL_8250_16550A_VARIANTS
 config SERIAL_8250_FINTEK
 	bool "Support for Fintek F81216A LPC to 4 UART RS485 API"
 	depends on SERIAL_8250
-	---help---
+	help
 	  Selecting this option will add support for the RS485 capabilities
 	  of the Fintek F81216A LPC to 4 UART.
 
@@ -87,7 +88,7 @@ config SERIAL_8250_CONSOLE
 	depends on SERIAL_8250=y
 	select SERIAL_CORE_CONSOLE
 	select SERIAL_EARLYCON
-	---help---
+	help
 	  If you say Y here, it will be possible to use a serial port as the
 	  system console (the system console is the device which receives all
 	  kernel messages and warnings and which allows logins in single user
@@ -157,7 +158,7 @@ config SERIAL_8250_HP300
 config SERIAL_8250_CS
 	tristate "8250/16550 PCMCIA device support"
 	depends on PCMCIA && SERIAL_8250
-	---help---
+	help
 	  Say Y here to enable support for 16-bit PCMCIA serial devices,
 	  including serial port cards, modems, and the modem functions of
 	  multi-function Ethernet/modem cards. (PCMCIA- or PC-cards are
diff --git a/drivers/tty/serial/8250/serial_cs.c b/drivers/tty/serial/8250/serial_cs.c
index c8186a05a453..e3d10794dbba 100644
--- a/drivers/tty/serial/8250/serial_cs.c
+++ b/drivers/tty/serial/8250/serial_cs.c
@@ -440,7 +440,7 @@ static int simple_config_check_notpicky(struct pcmcia_device *p_dev,
 static int simple_config(struct pcmcia_device *link)
 {
 	struct serial_info *info = link->priv;
-	int i = -ENODEV, try;
+	int ret, try;
 
 	/*
 	 * First pass: look for a config entry that looks normal.
@@ -472,8 +472,8 @@ found_port:
 	if (info->quirk && info->quirk->config)
 		info->quirk->config(link);
 
-	i = pcmcia_enable_device(link);
-	if (i != 0)
+	ret = pcmcia_enable_device(link);
+	if (ret != 0)
 		return -1;
 	return setup_serial(link, info, link->resource[0]->start, link->irq);
 }
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index adf9e80e7dc9..780908d43557 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -32,7 +32,7 @@ config SERIAL_AMBA_PL010_CONSOLE
 	bool "Support for console on AMBA serial port"
 	depends on SERIAL_AMBA_PL010=y
 	select SERIAL_CORE_CONSOLE
-	---help---
+	help
 	  Say Y here if you wish to use an AMBA PrimeCell UART as the system
 	  console (the system console is the device which receives all kernel
 	  messages and warnings and which allows logins in single user mode).
@@ -60,7 +60,7 @@ config SERIAL_AMBA_PL011_CONSOLE
 	depends on SERIAL_AMBA_PL011=y
 	select SERIAL_CORE_CONSOLE
 	select SERIAL_EARLYCON
-	---help---
+	help
 	  Say Y here if you wish to use an AMBA PrimeCell UART as the system
 	  console (the system console is the device which receives all kernel
 	  messages and warnings and which allows logins in single user mode).
@@ -101,7 +101,7 @@ config SERIAL_SB1250_DUART
 	depends on SIBYTE_SB1xxx_SOC=y
 	select SERIAL_CORE
 	default y
-	---help---
+	help
 	  Support for the asynchronous serial interface (DUART) included in
 	  the BCM1250 and derived System-On-a-Chip (SOC) devices.  Note that
 	  the letter D in DUART stands for "dual", which is how the device
@@ -116,7 +116,7 @@ config SERIAL_SB1250_DUART_CONSOLE
 	depends on SERIAL_SB1250_DUART=y
 	select SERIAL_CORE_CONSOLE
 	default y
-	---help---
+	help
 	  If you say Y here, it will be possible to use a serial port as the
 	  system console (the system console is the device which receives all
 	  kernel messages and warnings and which allows logins in single user
@@ -322,7 +322,7 @@ config SERIAL_TEGRA_TCU_CONSOLE
 	depends on SERIAL_TEGRA_TCU=y
 	select SERIAL_CORE_CONSOLE
 	default y
-	---help---
+	help
 	  If you say Y here, it will be possible to use a the Tegra TCU as the
 	  system console (the system console is the device which receives all
 	  kernel messages and warnings and which allows logins in single user
@@ -355,7 +355,7 @@ config SERIAL_DZ
 	depends on MACH_DECSTATION && 32BIT
 	select SERIAL_CORE
 	default y
-	---help---
+	help
 	  DZ11-family serial controllers for DECstations and VAXstations,
 	  including the DC7085, M7814, and M7819.
 
@@ -364,7 +364,7 @@ config SERIAL_DZ_CONSOLE
 	depends on SERIAL_DZ=y
 	select SERIAL_CORE_CONSOLE
 	default y
-	---help---
+	help
 	  If you say Y here, it will be possible to use a serial port as the
 	  system console (the system console is the device which receives all
 	  kernel messages and warnings and which allows logins in single user
@@ -380,7 +380,7 @@ config SERIAL_ZS
 	depends on MACH_DECSTATION
 	select SERIAL_CORE
 	default y
-	---help---
+	help
 	  Support for the Zilog 85C350 serial communications controller used
 	  for serial ports in newer DECstation systems.  These include the
 	  DECsystem 5900 and all models of the DECstation and DECsystem 5000
@@ -394,7 +394,7 @@ config SERIAL_ZS_CONSOLE
 	depends on SERIAL_ZS=y
 	select SERIAL_CORE_CONSOLE
 	default y
-	---help---
+	help
 	  If you say Y here, it will be possible to use a serial port as the
 	  system console (the system console is the device which receives all
 	  kernel messages and warnings and which allows logins in single user
@@ -588,7 +588,7 @@ config SERIAL_MUX
 	depends on GSC
 	select SERIAL_CORE
 	default y
-	---help---
+	help
 	  Saying Y here will enable the hardware MUX serial driver for
 	  the Nova, K class systems and D class with a 'remote control card'.
 	  The hardware MUX is not 8250/16550 compatible therefore the
@@ -1034,13 +1034,22 @@ config SERIAL_SIFIVE_CONSOLE
 	  boot time.)
 
 config SERIAL_LANTIQ
-	bool "Lantiq serial driver"
-	depends on LANTIQ
+	tristate "Lantiq serial driver"
+	depends on (LANTIQ || X86) || COMPILE_TEST
 	select SERIAL_CORE
+	help
+	  Support for UART on Lantiq and Intel SoCs.
+	  To compile this driver as a module, select M here. The
+	  module will be called lantiq.
+
+config SERIAL_LANTIQ_CONSOLE
+	bool "Console on Lantiq UART"
+	depends on SERIAL_LANTIQ=y
 	select SERIAL_CORE_CONSOLE
 	select SERIAL_EARLYCON
 	help
-	  Support for console and UART on Lantiq SoCs.
+	  Select this option if you would like to use a Lantiq UART as the
+	  system console.
 
 config SERIAL_QE
 	tristate "Freescale QUICC Engine serial port support"
@@ -1107,7 +1116,7 @@ config SERIAL_TIMBERDALE
 	tristate "Support for timberdale UART"
 	select SERIAL_CORE
 	depends on X86_32 || COMPILE_TEST
-	---help---
+	help
 	Add support for UART controller on timberdale.
 
 config SERIAL_BCM63XX
@@ -1136,7 +1145,7 @@ config SERIAL_GRLIB_GAISLER_APBUART
 	tristate "GRLIB APBUART serial support"
 	depends on OF && SPARC
 	select SERIAL_CORE
-	---help---
+	help
 	Add support for the GRLIB APBUART serial port.
 
 config SERIAL_GRLIB_GAISLER_APBUART_CONSOLE
@@ -1462,6 +1471,7 @@ config SERIAL_STM32
 	tristate "STMicroelectronics STM32 serial port support"
 	select SERIAL_CORE
 	depends on ARCH_STM32 || COMPILE_TEST
+	select SERIAL_MCTRL_GPIO if GPIOLIB
 	help
 	  This driver is for the on-chip Serial Controller on
 	  STMicroelectronics STM32 MCUs.
diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c
index c010f639298d..8efd7c2a34fe 100644
--- a/drivers/tty/serial/amba-pl011.c
+++ b/drivers/tty/serial/amba-pl011.c
@@ -2607,6 +2607,7 @@ static int pl011_setup_port(struct device *dev, struct uart_amba_port *uap,
 	uap->port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_AMBA_PL011_CONSOLE);
 	uap->port.flags = UPF_BOOT_AUTOCONF;
 	uap->port.line = index;
+	spin_lock_init(&uap->port.lock);
 
 	amba_ports[index] = uap;
 
diff --git a/drivers/tty/serial/ar933x_uart.c b/drivers/tty/serial/ar933x_uart.c
index 7e7f1398019f..0c80a79d7442 100644
--- a/drivers/tty/serial/ar933x_uart.c
+++ b/drivers/tty/serial/ar933x_uart.c
@@ -766,8 +766,6 @@ static int ar933x_uart_probe(struct platform_device *pdev)
 		goto err_disable_clk;
 	}
 
-	uart_get_rs485_mode(&pdev->dev, &port->rs485);
-
 	port->mapbase = mem_res->start;
 	port->line = id;
 	port->irq = irq_res->start;
@@ -786,6 +784,10 @@ static int ar933x_uart_probe(struct platform_device *pdev)
 	baud = ar933x_uart_get_baud(port->uartclk, 0, AR933X_UART_MAX_STEP);
 	up->max_baud = min_t(unsigned int, baud, AR933X_UART_MAX_BAUD);
 
+	ret = uart_get_rs485_mode(port);
+	if (ret)
+		goto err_disable_clk;
+
 	up->gpios = mctrl_gpio_init(port, 0);
 	if (IS_ERR(up->gpios) && PTR_ERR(up->gpios) != -ENOSYS)
 		return PTR_ERR(up->gpios);
diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c
index 8d7080efad9b..e43471b33710 100644
--- a/drivers/tty/serial/atmel_serial.c
+++ b/drivers/tty/serial/atmel_serial.c
@@ -2491,8 +2491,6 @@ static int atmel_init_port(struct atmel_uart_port *atmel_port,
 	atmel_init_property(atmel_port, pdev);
 	atmel_set_ops(port);
 
-	uart_get_rs485_mode(&mpdev->dev, &port->rs485);
-
 	port->iotype		= UPIO_MEM;
 	port->flags		= UPF_BOOT_AUTOCONF | UPF_IOREMAP;
 	port->ops		= &atmel_pops;
@@ -2506,6 +2504,10 @@ static int atmel_init_port(struct atmel_uart_port *atmel_port,
 
 	memset(&atmel_port->rx_ring, 0, sizeof(atmel_port->rx_ring));
 
+	ret = uart_get_rs485_mode(port);
+	if (ret)
+		return ret;
+
 	/* for console, the clock could already be configured */
 	if (!atmel_port->clk) {
 		atmel_port->clk = clk_get(&mpdev->dev, "usart");
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index 5d41075964f2..90298c403042 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -1231,9 +1231,7 @@ static void lpuart_dma_rx_free(struct uart_port *port)
 					struct lpuart_port, port);
 	struct dma_chan *chan = sport->dma_rx_chan;
 
-	if (chan)
-		dmaengine_terminate_all(chan);
-
+	dmaengine_terminate_all(chan);
 	dma_unmap_sg(chan->device->dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
 	kfree(sport->rx_ring.buf);
 	sport->rx_ring.tail = 0;
@@ -1514,17 +1512,17 @@ static void lpuart_request_dma(struct lpuart_port *sport)
 {
 	sport->dma_tx_chan = dma_request_chan(sport->port.dev, "tx");
 	if (IS_ERR(sport->dma_tx_chan)) {
-		dev_info_once(sport->port.dev,
-			      "DMA tx channel request failed, operating without tx DMA (%ld)\n",
-			      PTR_ERR(sport->dma_tx_chan));
+		dev_dbg_once(sport->port.dev,
+			     "DMA tx channel request failed, operating without tx DMA (%ld)\n",
+			     PTR_ERR(sport->dma_tx_chan));
 		sport->dma_tx_chan = NULL;
 	}
 
 	sport->dma_rx_chan = dma_request_chan(sport->port.dev, "rx");
 	if (IS_ERR(sport->dma_rx_chan)) {
-		dev_info_once(sport->port.dev,
-			      "DMA rx channel request failed, operating without rx DMA (%ld)\n",
-			      PTR_ERR(sport->dma_rx_chan));
+		dev_dbg_once(sport->port.dev,
+			     "DMA rx channel request failed, operating without rx DMA (%ld)\n",
+			     PTR_ERR(sport->dma_rx_chan));
 		sport->dma_rx_chan = NULL;
 	}
 }
@@ -2621,7 +2619,9 @@ static int lpuart_probe(struct platform_device *pdev)
 	if (ret)
 		goto failed_attach_port;
 
-	uart_get_rs485_mode(&pdev->dev, &sport->port.rs485);
+	ret = uart_get_rs485_mode(&sport->port);
+	if (ret)
+		goto failed_get_rs485;
 
 	if (sport->port.rs485.flags & SER_RS485_RX_DURING_TX)
 		dev_err(&pdev->dev, "driver doesn't support RX during TX\n");
@@ -2634,6 +2634,7 @@ static int lpuart_probe(struct platform_device *pdev)
 
 	return 0;
 
+failed_get_rs485:
 failed_attach_port:
 failed_irq_request:
 	lpuart_disable_clks(sport);
@@ -2664,8 +2665,7 @@ static int lpuart_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int lpuart_suspend(struct device *dev)
+static int __maybe_unused lpuart_suspend(struct device *dev)
 {
 	struct lpuart_port *sport = dev_get_drvdata(dev);
 	unsigned long temp;
@@ -2723,7 +2723,7 @@ static int lpuart_suspend(struct device *dev)
 	return 0;
 }
 
-static int lpuart_resume(struct device *dev)
+static int __maybe_unused lpuart_resume(struct device *dev)
 {
 	struct lpuart_port *sport = dev_get_drvdata(dev);
 	bool irq_wake = irqd_is_wakeup_set(irq_get_irq_data(sport->port.irq));
@@ -2754,7 +2754,6 @@ static int lpuart_resume(struct device *dev)
 
 	return 0;
 }
-#endif
 
 static SIMPLE_DEV_PM_OPS(lpuart_pm_ops, lpuart_suspend, lpuart_resume);
 
diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c
index f4d68109bc8b..1265e8d86d8a 100644
--- a/drivers/tty/serial/imx.c
+++ b/drivers/tty/serial/imx.c
@@ -909,6 +909,8 @@ static irqreturn_t imx_uart_int(int irq, void *dev_id)
 		usr2 &= ~USR2_ORE;
 
 	if (usr1 & (USR1_RRDY | USR1_AGTIM)) {
+		imx_uart_writel(sport, USR1_AGTIM, USR1);
+
 		__imx_uart_rxint(irq, dev_id);
 		ret = IRQ_HANDLED;
 	}
@@ -2252,6 +2254,8 @@ static int imx_uart_probe(struct platform_device *pdev)
 		return PTR_ERR(base);
 
 	rxirq = platform_get_irq(pdev, 0);
+	if (rxirq < 0)
+		return rxirq;
 	txirq = platform_get_irq_optional(pdev, 1);
 	rtsirq = platform_get_irq_optional(pdev, 2);
 
@@ -2302,7 +2306,11 @@ static int imx_uart_probe(struct platform_device *pdev)
 	sport->ucr4 = readl(sport->port.membase + UCR4);
 	sport->ufcr = readl(sport->port.membase + UFCR);
 
-	uart_get_rs485_mode(&pdev->dev, &sport->port.rs485);
+	ret = uart_get_rs485_mode(&sport->port);
+	if (ret) {
+		clk_disable_unprepare(sport->clk_ipg);
+		return ret;
+	}
 
 	if (sport->port.rs485.flags & SER_RS485_ENABLED &&
 	    (!sport->have_rtscts && !sport->have_rtsgpio))
@@ -2398,6 +2406,9 @@ static int imx_uart_probe(struct platform_device *pdev)
 		}
 	}
 
+	/* We need to initialize lock even for non-registered console */
+	spin_lock_init(&sport->port.lock);
+
 	imx_uart_ports[sport->port.line] = sport;
 
 	platform_set_drvdata(pdev, sport);
diff --git a/drivers/tty/serial/lantiq.c b/drivers/tty/serial/lantiq.c
index c5e46ff972e4..62813e421f12 100644
--- a/drivers/tty/serial/lantiq.c
+++ b/drivers/tty/serial/lantiq.c
@@ -15,6 +15,7 @@
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/lantiq.h>
+#include <linux/module.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
@@ -597,6 +598,7 @@ static const struct uart_ops lqasc_pops = {
 	.verify_port =	lqasc_verify_port,
 };
 
+#ifdef CONFIG_SERIAL_LANTIQ_CONSOLE
 static void
 lqasc_console_putchar(struct uart_port *port, int ch)
 {
@@ -705,6 +707,14 @@ lqasc_serial_early_console_setup(struct earlycon_device *device,
 OF_EARLYCON_DECLARE(lantiq, "lantiq,asc", lqasc_serial_early_console_setup);
 OF_EARLYCON_DECLARE(lantiq, "intel,lgm-asc", lqasc_serial_early_console_setup);
 
+#define LANTIQ_SERIAL_CONSOLE	(&lqasc_console)
+
+#else
+
+#define LANTIQ_SERIAL_CONSOLE	NULL
+
+#endif /* CONFIG_SERIAL_LANTIQ_CONSOLE */
+
 static struct uart_driver lqasc_reg = {
 	.owner =	THIS_MODULE,
 	.driver_name =	DRVNAME,
@@ -712,7 +722,7 @@ static struct uart_driver lqasc_reg = {
 	.major =	0,
 	.minor =	0,
 	.nr =		MAXPORTS,
-	.cons =		&lqasc_console,
+	.cons =		LANTIQ_SERIAL_CONSOLE,
 };
 
 static int fetch_irq_lantiq(struct device *dev, struct ltq_uart_port *ltq_port)
@@ -814,8 +824,7 @@ static void free_irq_intel(struct uart_port *port)
 	free_irq(ltq_port->common_irq, port);
 }
 
-static int __init
-lqasc_probe(struct platform_device *pdev)
+static int lqasc_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
 	struct ltq_uart_port *ltq_port;
@@ -899,6 +908,13 @@ lqasc_probe(struct platform_device *pdev)
 	return ret;
 }
 
+static int lqasc_remove(struct platform_device *pdev)
+{
+	struct uart_port *port = platform_get_drvdata(pdev);
+
+	return uart_remove_one_port(&lqasc_reg, port);
+}
+
 static const struct ltq_soc_data soc_data_lantiq = {
 	.fetch_irq = fetch_irq_lantiq,
 	.request_irq = request_irq_lantiq,
@@ -916,8 +932,11 @@ static const struct of_device_id ltq_asc_match[] = {
 	{ .compatible = "intel,lgm-asc", .data = &soc_data_intel },
 	{},
 };
+MODULE_DEVICE_TABLE(of, ltq_asc_match);
 
 static struct platform_driver lqasc_driver = {
+	.probe		= lqasc_probe,
+	.remove		= lqasc_remove,
 	.driver		= {
 		.name	= DRVNAME,
 		.of_match_table = ltq_asc_match,
@@ -933,10 +952,21 @@ init_lqasc(void)
 	if (ret != 0)
 		return ret;
 
-	ret = platform_driver_probe(&lqasc_driver, lqasc_probe);
+	ret = platform_driver_register(&lqasc_driver);
 	if (ret != 0)
 		uart_unregister_driver(&lqasc_reg);
 
 	return ret;
 }
-device_initcall(init_lqasc);
+
+static void __exit exit_lqasc(void)
+{
+	platform_driver_unregister(&lqasc_driver);
+	uart_unregister_driver(&lqasc_reg);
+}
+
+module_init(init_lqasc);
+module_exit(exit_lqasc);
+
+MODULE_DESCRIPTION("Serial driver for Lantiq & Intel gateway SoCs");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/tty/serial/lpc32xx_hs.c b/drivers/tty/serial/lpc32xx_hs.c
index 9a836dcac157..b5898c932036 100644
--- a/drivers/tty/serial/lpc32xx_hs.c
+++ b/drivers/tty/serial/lpc32xx_hs.c
@@ -23,7 +23,6 @@
 #include <linux/nmi.h>
 #include <linux/io.h>
 #include <linux/irq.h>
-#include <linux/gpio.h>
 #include <linux/of.h>
 #include <linux/sizes.h>
 #include <linux/soc/nxp/lpc32xx-misc.h>
diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c
index f7d6b3c9ea45..8573fc9cb0cd 100644
--- a/drivers/tty/serial/omap-serial.c
+++ b/drivers/tty/serial/omap-serial.c
@@ -33,8 +33,7 @@
 #include <linux/pm_wakeirq.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/platform_data/serial-omap.h>
 
 #define OMAP_MAX_HSUART_PORTS	10
@@ -153,7 +152,7 @@ struct uart_omap_port {
 	u32			errata;
 	u32			features;
 
-	int			rts_gpio;
+	struct gpio_desc	*rts_gpiod;
 
 	struct pm_qos_request	pm_qos_request;
 	u32			latency;
@@ -303,11 +302,11 @@ static void serial_omap_stop_tx(struct uart_port *port)
 			serial_out(up, UART_OMAP_SCR, up->scr);
 			res = (port->rs485.flags & SER_RS485_RTS_AFTER_SEND) ?
 				1 : 0;
-			if (gpio_get_value(up->rts_gpio) != res) {
+			if (gpiod_get_value(up->rts_gpiod) != res) {
 				if (port->rs485.delay_rts_after_send > 0)
 					mdelay(
 					port->rs485.delay_rts_after_send);
-				gpio_set_value(up->rts_gpio, res);
+				gpiod_set_value(up->rts_gpiod, res);
 			}
 		} else {
 			/* We're asked to stop, but there's still stuff in the
@@ -412,8 +411,8 @@ static void serial_omap_start_tx(struct uart_port *port)
 
 		/* if rts not already enabled */
 		res = (port->rs485.flags & SER_RS485_RTS_ON_SEND) ? 1 : 0;
-		if (gpio_get_value(up->rts_gpio) != res) {
-			gpio_set_value(up->rts_gpio, res);
+		if (gpiod_get_value(up->rts_gpiod) != res) {
+			gpiod_set_value(up->rts_gpiod, res);
 			if (port->rs485.delay_rts_before_send > 0)
 				mdelay(port->rs485.delay_rts_before_send);
 		}
@@ -1414,12 +1413,12 @@ serial_omap_config_rs485(struct uart_port *port, struct serial_rs485 *rs485)
 	 * Just as a precaution, only allow rs485
 	 * to be enabled if the gpio pin is valid
 	 */
-	if (gpio_is_valid(up->rts_gpio)) {
+	if (up->rts_gpiod) {
 		/* enable / disable rts */
 		val = (port->rs485.flags & SER_RS485_ENABLED) ?
 			SER_RS485_RTS_AFTER_SEND : SER_RS485_RTS_ON_SEND;
 		val = (port->rs485.flags & val) ? 1 : 0;
-		gpio_set_value(up->rts_gpio, val);
+		gpiod_set_value(up->rts_gpiod, val);
 	} else
 		port->rs485.flags &= ~SER_RS485_ENABLED;
 
@@ -1596,18 +1595,22 @@ static struct omap_uart_port_info *of_get_uart_port_info(struct device *dev)
 }
 
 static int serial_omap_probe_rs485(struct uart_omap_port *up,
-				   struct device_node *np)
+				   struct device *dev)
 {
 	struct serial_rs485 *rs485conf = &up->port.rs485;
+	struct device_node *np = dev->of_node;
+	enum gpiod_flags gflags;
 	int ret;
 
 	rs485conf->flags = 0;
-	up->rts_gpio = -EINVAL;
+	up->rts_gpiod = NULL;
 
 	if (!np)
 		return 0;
 
-	uart_get_rs485_mode(up->dev, rs485conf);
+	ret = uart_get_rs485_mode(&up->port);
+	if (ret)
+		return ret;
 
 	if (of_property_read_bool(np, "rs485-rts-active-high")) {
 		rs485conf->flags |= SER_RS485_RTS_ON_SEND;
@@ -1618,19 +1621,20 @@ static int serial_omap_probe_rs485(struct uart_omap_port *up,
 	}
 
 	/* check for tx enable gpio */
-	up->rts_gpio = of_get_named_gpio(np, "rts-gpio", 0);
-	if (gpio_is_valid(up->rts_gpio)) {
-		ret = devm_gpio_request(up->dev, up->rts_gpio, "omap-serial");
-		if (ret < 0)
-			return ret;
-		ret = rs485conf->flags & SER_RS485_RTS_AFTER_SEND ? 1 : 0;
-		ret = gpio_direction_output(up->rts_gpio, ret);
-		if (ret < 0)
+	gflags = rs485conf->flags & SER_RS485_RTS_AFTER_SEND ?
+		GPIOD_OUT_HIGH : GPIOD_OUT_LOW;
+	up->rts_gpiod = devm_gpiod_get_optional(dev, "rts", gflags);
+	if (IS_ERR(up->rts_gpiod)) {
+		ret = PTR_ERR(up->rts_gpiod);
+	        if (ret == -EPROBE_DEFER)
 			return ret;
-	} else if (up->rts_gpio == -EPROBE_DEFER) {
-		return -EPROBE_DEFER;
+		/*
+		 * FIXME: the code historically ignored any other error than
+		 * -EPROBE_DEFER and just went on without GPIO.
+		 */
+		up->rts_gpiod = NULL;
 	} else {
-		up->rts_gpio = -EINVAL;
+		gpiod_set_consumer_name(up->rts_gpiod, "omap-serial");
 	}
 
 	return 0;
@@ -1703,7 +1707,7 @@ static int serial_omap_probe(struct platform_device *pdev)
 		dev_info(up->port.dev, "no wakeirq for uart%d\n",
 			 up->port.line);
 
-	ret = serial_omap_probe_rs485(up, pdev->dev.of_node);
+	ret = serial_omap_probe_rs485(up, &pdev->dev);
 	if (ret < 0)
 		goto err_rs485;
 
diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c
index 6bace1c6bb09..457c0bf8cbf8 100644
--- a/drivers/tty/serial/qcom_geni_serial.c
+++ b/drivers/tty/serial/qcom_geni_serial.c
@@ -141,9 +141,10 @@ static void qcom_geni_serial_stop_rx(struct uart_port *uport);
 static void qcom_geni_serial_handle_rx(struct uart_port *uport, bool drop);
 
 static const unsigned long root_freq[] = {7372800, 14745600, 19200000, 29491200,
-					32000000, 48000000, 64000000, 80000000,
-					96000000, 100000000, 102400000,
-					112000000, 120000000, 128000000};
+					32000000, 48000000, 51200000, 64000000,
+					80000000, 96000000, 100000000,
+					102400000, 112000000, 120000000,
+					128000000};
 
 #define to_dev_port(ptr, member) \
 		container_of(ptr, struct qcom_geni_serial_port, member)
diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c
index 73f951d65b93..d913d9b2762a 100644
--- a/drivers/tty/serial/samsung_tty.c
+++ b/drivers/tty/serial/samsung_tty.c
@@ -154,10 +154,33 @@ struct s3c24xx_uart_port {
 #define portaddrl(port, reg) \
 	((unsigned long *)(unsigned long)((port)->membase + (reg)))
 
-#define rd_regb(port, reg) (readb_relaxed(portaddr(port, reg)))
+static u32 rd_reg(struct uart_port *port, u32 reg)
+{
+	switch (port->iotype) {
+	case UPIO_MEM:
+		return readb_relaxed(portaddr(port, reg));
+	case UPIO_MEM32:
+		return readl_relaxed(portaddr(port, reg));
+	default:
+		return 0;
+	}
+	return 0;
+}
+
 #define rd_regl(port, reg) (readl_relaxed(portaddr(port, reg)))
 
-#define wr_regb(port, reg, val) writeb_relaxed(val, portaddr(port, reg))
+static void wr_reg(struct uart_port *port, u32 reg, u32 val)
+{
+	switch (port->iotype) {
+	case UPIO_MEM:
+		writeb_relaxed(val, portaddr(port, reg));
+		break;
+	case UPIO_MEM32:
+		writel_relaxed(val, portaddr(port, reg));
+		break;
+	}
+}
+
 #define wr_regl(port, reg, val) writel_relaxed(val, portaddr(port, reg))
 
 /* Byte-order aware bit setting/clearing functions. */
@@ -714,7 +737,7 @@ static void s3c24xx_serial_rx_drain_fifo(struct s3c24xx_uart_port *ourport)
 		fifocnt--;
 
 		uerstat = rd_regl(port, S3C2410_UERSTAT);
-		ch = rd_regb(port, S3C2410_URXH);
+		ch = rd_reg(port, S3C2410_URXH);
 
 		if (port->flags & UPF_CONS_FLOW) {
 			int txe = s3c24xx_serial_txempty_nofifo(port);
@@ -826,7 +849,7 @@ static irqreturn_t s3c24xx_serial_tx_chars(int irq, void *id)
 	}
 
 	if (port->x_char) {
-		wr_regb(port, S3C2410_UTXH, port->x_char);
+		wr_reg(port, S3C2410_UTXH, port->x_char);
 		port->icount.tx++;
 		port->x_char = 0;
 		goto out;
@@ -852,7 +875,7 @@ static irqreturn_t s3c24xx_serial_tx_chars(int irq, void *id)
 		if (rd_regl(port, S3C2410_UFSTAT) & ourport->info->tx_fifofull)
 			break;
 
-		wr_regb(port, S3C2410_UTXH, xmit->buf[xmit->tail]);
+		wr_reg(port, S3C2410_UTXH, xmit->buf[xmit->tail]);
 		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
 		port->icount.tx++;
 		count--;
@@ -916,7 +939,7 @@ static unsigned int s3c24xx_serial_tx_empty(struct uart_port *port)
 /* no modem control lines */
 static unsigned int s3c24xx_serial_get_mctrl(struct uart_port *port)
 {
-	unsigned int umstat = rd_regb(port, S3C2410_UMSTAT);
+	unsigned int umstat = rd_reg(port, S3C2410_UMSTAT);
 
 	if (umstat & S3C2410_UMSTAT_CTS)
 		return TIOCM_CAR | TIOCM_DSR | TIOCM_CTS;
@@ -1281,14 +1304,14 @@ static unsigned int s3c24xx_serial_getclk(struct s3c24xx_uart_port *ourport,
 	struct s3c24xx_uart_info *info = ourport->info;
 	struct clk *clk;
 	unsigned long rate;
-	unsigned int cnt, baud, quot, clk_sel, best_quot = 0;
+	unsigned int cnt, baud, quot, best_quot = 0;
 	char clkname[MAX_CLK_NAME_LENGTH];
 	int calc_deviation, deviation = (1 << 30) - 1;
 
-	clk_sel = (ourport->cfg->clk_sel) ? ourport->cfg->clk_sel :
-			ourport->info->def_clk_sel;
 	for (cnt = 0; cnt < info->num_clks; cnt++) {
-		if (!(clk_sel & (1 << cnt)))
+		/* Keep selected clock if provided */
+		if (ourport->cfg->clk_sel &&
+			!(ourport->cfg->clk_sel & (1 << cnt)))
 			continue;
 
 		sprintf(clkname, "clk_uart_baud%d", cnt);
@@ -1974,7 +1997,7 @@ static int s3c24xx_serial_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 	struct s3c24xx_uart_port *ourport;
 	int index = probe_index;
-	int ret;
+	int ret, prop = 0;
 
 	if (np) {
 		ret = of_alias_get_id(np, "serial");
@@ -2000,10 +2023,27 @@ static int s3c24xx_serial_probe(struct platform_device *pdev)
 			dev_get_platdata(&pdev->dev) :
 			ourport->drv_data->def_cfg;
 
-	if (np)
+	if (np) {
 		of_property_read_u32(np,
 			"samsung,uart-fifosize", &ourport->port.fifosize);
 
+		if (of_property_read_u32(np, "reg-io-width", &prop) == 0) {
+			switch (prop) {
+			case 1:
+				ourport->port.iotype = UPIO_MEM;
+				break;
+			case 4:
+				ourport->port.iotype = UPIO_MEM32;
+				break;
+			default:
+				dev_warn(&pdev->dev, "unsupported reg-io-width (%d)\n",
+						prop);
+				ret = -EINVAL;
+				break;
+			}
+		}
+	}
+
 	if (ourport->drv_data->fifosize[index])
 		ourport->port.fifosize = ourport->drv_data->fifosize[index];
 	else if (ourport->info->fifosize)
@@ -2185,7 +2225,7 @@ static int s3c24xx_serial_get_poll_char(struct uart_port *port)
 	if (s3c24xx_serial_rx_fifocnt(ourport, ufstat) == 0)
 		return NO_POLL_CHAR;
 
-	return rd_regb(port, S3C2410_URXH);
+	return rd_reg(port, S3C2410_URXH);
 }
 
 static void s3c24xx_serial_put_poll_char(struct uart_port *port,
@@ -2200,7 +2240,7 @@ static void s3c24xx_serial_put_poll_char(struct uart_port *port,
 
 	while (!s3c24xx_serial_console_txrdy(port, ufcon))
 		cpu_relax();
-	wr_regb(port, S3C2410_UTXH, c);
+	wr_reg(port, S3C2410_UTXH, c);
 }
 
 #endif /* CONFIG_CONSOLE_POLL */
@@ -2212,7 +2252,7 @@ s3c24xx_serial_console_putchar(struct uart_port *port, int ch)
 
 	while (!s3c24xx_serial_console_txrdy(port, ufcon))
 		cpu_relax();
-	wr_regb(port, S3C2410_UTXH, ch);
+	wr_reg(port, S3C2410_UTXH, ch);
 }
 
 static void
@@ -2587,6 +2627,18 @@ module_platform_driver(samsung_serial_driver);
  * Early console.
  */
 
+static void wr_reg_barrier(struct uart_port *port, u32 reg, u32 val)
+{
+	switch (port->iotype) {
+	case UPIO_MEM:
+		writeb(val, portaddr(port, reg));
+		break;
+	case UPIO_MEM32:
+		writel(val, portaddr(port, reg));
+		break;
+	}
+}
+
 struct samsung_early_console_data {
 	u32 txfull_mask;
 };
@@ -2612,7 +2664,7 @@ static void samsung_early_putc(struct uart_port *port, int c)
 	else
 		samsung_early_busyuart(port);
 
-	writeb(c, port->membase + S3C2410_UTXH);
+	wr_reg_barrier(port, S3C2410_UTXH, c);
 }
 
 static void samsung_early_write(struct console *con, const char *s,
diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
index 06e8071d5601..d2e5c6c86643 100644
--- a/drivers/tty/serial/sc16is7xx.c
+++ b/drivers/tty/serial/sc16is7xx.c
@@ -315,6 +315,7 @@ struct sc16is7xx_one {
 	struct kthread_work		tx_work;
 	struct kthread_work		reg_work;
 	struct sc16is7xx_one_config	config;
+	bool				irda_mode;
 };
 
 struct sc16is7xx_port {
@@ -327,7 +328,6 @@ struct sc16is7xx_port {
 	unsigned char			buf[SC16IS7XX_FIFO_SIZE];
 	struct kthread_worker		kworker;
 	struct task_struct		*kworker_task;
-	struct kthread_work		irq_work;
 	struct mutex			efr_lock;
 	struct sc16is7xx_one		p[];
 };
@@ -710,9 +710,9 @@ static bool sc16is7xx_port_irq(struct sc16is7xx_port *s, int portno)
 	return true;
 }
 
-static void sc16is7xx_ist(struct kthread_work *ws)
+static irqreturn_t sc16is7xx_irq(int irq, void *dev_id)
 {
-	struct sc16is7xx_port *s = to_sc16is7xx_port(ws, irq_work);
+	struct sc16is7xx_port *s = (struct sc16is7xx_port *)dev_id;
 
 	mutex_lock(&s->efr_lock);
 
@@ -727,13 +727,6 @@ static void sc16is7xx_ist(struct kthread_work *ws)
 	}
 
 	mutex_unlock(&s->efr_lock);
-}
-
-static irqreturn_t sc16is7xx_irq(int irq, void *dev_id)
-{
-	struct sc16is7xx_port *s = (struct sc16is7xx_port *)dev_id;
-
-	kthread_queue_work(&s->kworker, &s->irq_work);
 
 	return IRQ_HANDLED;
 }
@@ -994,6 +987,7 @@ static int sc16is7xx_config_rs485(struct uart_port *port,
 
 static int sc16is7xx_startup(struct uart_port *port)
 {
+	struct sc16is7xx_one *one = to_sc16is7xx_one(port, port);
 	struct sc16is7xx_port *s = dev_get_drvdata(port->dev);
 	unsigned int val;
 
@@ -1032,6 +1026,13 @@ static int sc16is7xx_startup(struct uart_port *port)
 	/* Now, initialize the UART */
 	sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, SC16IS7XX_LCR_WORD_LEN_8);
 
+	/* Enable IrDA mode if requested in DT */
+	/* This bit must be written with LCR[7] = 0 */
+	sc16is7xx_port_update(port, SC16IS7XX_MCR_REG,
+			      SC16IS7XX_MCR_IRDA_BIT,
+			      one->irda_mode ?
+				SC16IS7XX_MCR_IRDA_BIT : 0);
+
 	/* Enable the Rx and Tx FIFO */
 	sc16is7xx_port_update(port, SC16IS7XX_EFCR_REG,
 			      SC16IS7XX_EFCR_RXDISABLE_BIT |
@@ -1176,10 +1177,11 @@ static int sc16is7xx_gpio_direction_output(struct gpio_chip *chip,
 
 static int sc16is7xx_probe(struct device *dev,
 			   const struct sc16is7xx_devtype *devtype,
-			   struct regmap *regmap, int irq, unsigned long flags)
+			   struct regmap *regmap, int irq)
 {
 	struct sched_param sched_param = { .sched_priority = MAX_RT_PRIO / 2 };
 	unsigned long freq = 0, *pfreq = dev_get_platdata(dev);
+	unsigned int val;
 	u32 uartclk = 0;
 	int i, ret;
 	struct sc16is7xx_port *s;
@@ -1187,6 +1189,16 @@ static int sc16is7xx_probe(struct device *dev,
 	if (IS_ERR(regmap))
 		return PTR_ERR(regmap);
 
+	/*
+	 * This device does not have an identification register that would
+	 * tell us if we are really connected to the correct device.
+	 * The best we can do is to check if communication is at all possible.
+	 */
+	ret = regmap_read(regmap,
+			  SC16IS7XX_LSR_REG << SC16IS7XX_REG_SHIFT, &val);
+	if (ret < 0)
+		return ret;
+
 	/* Alloc port structure */
 	s = devm_kzalloc(dev, struct_size(s, p, devtype->nr_uart), GFP_KERNEL);
 	if (!s) {
@@ -1221,7 +1233,6 @@ static int sc16is7xx_probe(struct device *dev,
 	mutex_init(&s->efr_lock);
 
 	kthread_init_worker(&s->kworker);
-	kthread_init_work(&s->irq_work, sc16is7xx_ist);
 	s->kworker_task = kthread_run(kthread_worker_fn, &s->kworker,
 				      "sc16is7xx");
 	if (IS_ERR(s->kworker_task)) {
@@ -1302,9 +1313,33 @@ static int sc16is7xx_probe(struct device *dev,
 		sc16is7xx_power(&s->p[i].port, 0);
 	}
 
-	/* Setup interrupt */
-	ret = devm_request_irq(dev, irq, sc16is7xx_irq,
-			       flags, dev_name(dev), s);
+	if (dev->of_node) {
+		struct property *prop;
+		const __be32 *p;
+		u32 u;
+
+		of_property_for_each_u32(dev->of_node, "irda-mode-ports",
+					 prop, p, u)
+			if (u < devtype->nr_uart)
+				s->p[u].irda_mode = true;
+	}
+
+	/*
+	 * Setup interrupt. We first try to acquire the IRQ line as level IRQ.
+	 * If that succeeds, we can allow sharing the interrupt as well.
+	 * In case the interrupt controller doesn't support that, we fall
+	 * back to a non-shared falling-edge trigger.
+	 */
+	ret = devm_request_threaded_irq(dev, irq, NULL, sc16is7xx_irq,
+					IRQF_TRIGGER_LOW | IRQF_SHARED |
+					IRQF_ONESHOT,
+					dev_name(dev), s);
+	if (!ret)
+		return 0;
+
+	ret = devm_request_threaded_irq(dev, irq, NULL, sc16is7xx_irq,
+					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+					dev_name(dev), s);
 	if (!ret)
 		return 0;
 
@@ -1378,7 +1413,6 @@ static struct regmap_config regcfg = {
 static int sc16is7xx_spi_probe(struct spi_device *spi)
 {
 	const struct sc16is7xx_devtype *devtype;
-	unsigned long flags = 0;
 	struct regmap *regmap;
 	int ret;
 
@@ -1399,14 +1433,13 @@ static int sc16is7xx_spi_probe(struct spi_device *spi)
 		const struct spi_device_id *id_entry = spi_get_device_id(spi);
 
 		devtype = (struct sc16is7xx_devtype *)id_entry->driver_data;
-		flags = IRQF_TRIGGER_FALLING;
 	}
 
 	regcfg.max_register = (0xf << SC16IS7XX_REG_SHIFT) |
 			      (devtype->nr_uart - 1);
 	regmap = devm_regmap_init_spi(spi, &regcfg);
 
-	return sc16is7xx_probe(&spi->dev, devtype, regmap, spi->irq, flags);
+	return sc16is7xx_probe(&spi->dev, devtype, regmap, spi->irq);
 }
 
 static int sc16is7xx_spi_remove(struct spi_device *spi)
@@ -1445,7 +1478,6 @@ static int sc16is7xx_i2c_probe(struct i2c_client *i2c,
 			       const struct i2c_device_id *id)
 {
 	const struct sc16is7xx_devtype *devtype;
-	unsigned long flags = 0;
 	struct regmap *regmap;
 
 	if (i2c->dev.of_node) {
@@ -1454,14 +1486,13 @@ static int sc16is7xx_i2c_probe(struct i2c_client *i2c,
 			return -ENODEV;
 	} else {
 		devtype = (struct sc16is7xx_devtype *)id->driver_data;
-		flags = IRQF_TRIGGER_FALLING;
 	}
 
 	regcfg.max_register = (0xf << SC16IS7XX_REG_SHIFT) |
 			      (devtype->nr_uart - 1);
 	regmap = devm_regmap_init_i2c(i2c, &regcfg);
 
-	return sc16is7xx_probe(&i2c->dev, devtype, regmap, i2c->irq, flags);
+	return sc16is7xx_probe(&i2c->dev, devtype, regmap, i2c->irq);
 }
 
 static int sc16is7xx_i2c_remove(struct i2c_client *client)
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index 66a5e2faf57e..57840cf90388 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -3295,8 +3295,10 @@ EXPORT_SYMBOL(uart_remove_one_port);
  * This function implements the device tree binding described in
  * Documentation/devicetree/bindings/serial/rs485.txt.
  */
-void uart_get_rs485_mode(struct device *dev, struct serial_rs485 *rs485conf)
+int uart_get_rs485_mode(struct uart_port *port)
 {
+	struct serial_rs485 *rs485conf = &port->rs485;
+	struct device *dev = port->dev;
 	u32 rs485_delay[2];
 	int ret;
 
@@ -3315,6 +3317,7 @@ void uart_get_rs485_mode(struct device *dev, struct serial_rs485 *rs485conf)
 	 * to get to a defined state with the following properties:
 	 */
 	rs485conf->flags &= ~(SER_RS485_RX_DURING_TX | SER_RS485_ENABLED |
+			      SER_RS485_TERMINATE_BUS |
 			      SER_RS485_RTS_AFTER_SEND);
 	rs485conf->flags |= SER_RS485_RTS_ON_SEND;
 
@@ -3328,6 +3331,23 @@ void uart_get_rs485_mode(struct device *dev, struct serial_rs485 *rs485conf)
 		rs485conf->flags &= ~SER_RS485_RTS_ON_SEND;
 		rs485conf->flags |= SER_RS485_RTS_AFTER_SEND;
 	}
+
+	/*
+	 * Disabling termination by default is the safe choice:  Else if many
+	 * bus participants enable it, no communication is possible at all.
+	 * Works fine for short cables and users may enable for longer cables.
+	 */
+	port->rs485_term_gpio = devm_gpiod_get_optional(dev, "rs485-term",
+							GPIOD_OUT_LOW);
+	if (IS_ERR(port->rs485_term_gpio)) {
+		ret = PTR_ERR(port->rs485_term_gpio);
+		port->rs485_term_gpio = NULL;
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Cannot get rs485-term-gpios\n");
+		return ret;
+	}
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(uart_get_rs485_mode);
 
diff --git a/drivers/tty/serial/sh-sci.h b/drivers/tty/serial/sh-sci.h
index 0b9e804e61a9..c0dfe4382898 100644
--- a/drivers/tty/serial/sh-sci.h
+++ b/drivers/tty/serial/sh-sci.h
@@ -2,7 +2,6 @@
 #include <linux/bitops.h>
 #include <linux/serial_core.h>
 #include <linux/io.h>
-#include <linux/gpio.h>
 
 #define SCI_MAJOR		204
 #define SCI_MINOR_START		8
diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c
index 5e93e8d40f59..8602ff357321 100644
--- a/drivers/tty/serial/stm32-usart.c
+++ b/drivers/tty/serial/stm32-usart.c
@@ -31,6 +31,7 @@
 #include <linux/tty_flip.h>
 #include <linux/tty.h>
 
+#include "serial_mctrl_gpio.h"
 #include "stm32-usart.h"
 
 static void stm32_stop_tx(struct uart_port *port);
@@ -158,9 +159,7 @@ static int stm32_init_rs485(struct uart_port *port,
 	if (!pdev->dev.of_node)
 		return -ENODEV;
 
-	uart_get_rs485_mode(&pdev->dev, rs485conf);
-
-	return 0;
+	return uart_get_rs485_mode(port);
 }
 
 static int stm32_pending_rx(struct uart_port *port, u32 *sr, int *last_res,
@@ -510,12 +509,29 @@ static void stm32_set_mctrl(struct uart_port *port, unsigned int mctrl)
 		stm32_set_bits(port, ofs->cr3, USART_CR3_RTSE);
 	else
 		stm32_clr_bits(port, ofs->cr3, USART_CR3_RTSE);
+
+	mctrl_gpio_set(stm32_port->gpios, mctrl);
 }
 
 static unsigned int stm32_get_mctrl(struct uart_port *port)
 {
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	unsigned int ret;
+
 	/* This routine is used to get signals of: DCD, DSR, RI, and CTS */
-	return TIOCM_CAR | TIOCM_DSR | TIOCM_CTS;
+	ret = TIOCM_CAR | TIOCM_DSR | TIOCM_CTS;
+
+	return mctrl_gpio_get(stm32_port->gpios, &ret);
+}
+
+static void stm32_enable_ms(struct uart_port *port)
+{
+	mctrl_gpio_enable_ms(to_stm32_port(port)->gpios);
+}
+
+static void stm32_disable_ms(struct uart_port *port)
+{
+	mctrl_gpio_disable_ms(to_stm32_port(port)->gpios);
 }
 
 /* Transmit stop */
@@ -626,6 +642,9 @@ static void stm32_shutdown(struct uart_port *port)
 	u32 val, isr;
 	int ret;
 
+	/* Disable modem control interrupts */
+	stm32_disable_ms(port);
+
 	val = USART_CR1_TXEIE | USART_CR1_TE;
 	val |= stm32_port->cr1_irq | USART_CR1_RE;
 	val |= BIT(cfg->uart_enable_bit);
@@ -764,6 +783,12 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 		cr3 |= USART_CR3_CTSE | USART_CR3_RTSE;
 	}
 
+	/* Handle modem control interrupts */
+	if (UART_ENABLE_MS(port, termios->c_cflag))
+		stm32_enable_ms(port);
+	else
+		stm32_disable_ms(port);
+
 	usartdiv = DIV_ROUND_CLOSEST(port->uartclk, baud);
 
 	/*
@@ -898,6 +923,7 @@ static const struct uart_ops stm32_uart_ops = {
 	.throttle	= stm32_throttle,
 	.unthrottle	= stm32_unthrottle,
 	.stop_rx	= stm32_stop_rx,
+	.enable_ms	= stm32_enable_ms,
 	.break_ctl	= stm32_break_ctl,
 	.startup	= stm32_startup,
 	.shutdown	= stm32_shutdown,
@@ -931,7 +957,9 @@ static int stm32_init_port(struct stm32_port *stm32port,
 
 	port->rs485_config = stm32_config_rs485;
 
-	stm32_init_rs485(port, pdev);
+	ret = stm32_init_rs485(port, pdev);
+	if (ret)
+		return ret;
 
 	if (stm32port->info->cfg.has_wakeup) {
 		stm32port->wakeirq = platform_get_irq(pdev, 1);
@@ -960,11 +988,32 @@ static int stm32_init_port(struct stm32_port *stm32port,
 
 	stm32port->port.uartclk = clk_get_rate(stm32port->clk);
 	if (!stm32port->port.uartclk) {
-		clk_disable_unprepare(stm32port->clk);
 		ret = -EINVAL;
+		goto err_clk;
+	}
+
+	stm32port->gpios = mctrl_gpio_init(&stm32port->port, 0);
+	if (IS_ERR(stm32port->gpios)) {
+		ret = PTR_ERR(stm32port->gpios);
+		goto err_clk;
+	}
+
+	/* Both CTS/RTS gpios and "st,hw-flow-ctrl" should not be specified */
+	if (stm32port->hw_flow_control) {
+		if (mctrl_gpio_to_gpiod(stm32port->gpios, UART_GPIO_CTS) ||
+		    mctrl_gpio_to_gpiod(stm32port->gpios, UART_GPIO_RTS)) {
+			dev_err(&pdev->dev, "Conflicting RTS/CTS config\n");
+			ret = -EINVAL;
+			goto err_clk;
+		}
 	}
 
 	return ret;
+
+err_clk:
+	clk_disable_unprepare(stm32port->clk);
+
+	return ret;
 }
 
 static struct stm32_port *stm32_of_get_stm32_port(struct platform_device *pdev)
@@ -1375,7 +1424,18 @@ static int __maybe_unused stm32_serial_suspend(struct device *dev)
 	else
 		stm32_serial_enable_wakeup(port, false);
 
-	pinctrl_pm_select_sleep_state(dev);
+	/*
+	 * When "no_console_suspend" is enabled, keep the pinctrl default state
+	 * and rely on bootloader stage to restore this state upon resume.
+	 * Otherwise, apply the idle or sleep states depending on wakeup
+	 * capabilities.
+	 */
+	if (console_suspend_enabled || !uart_console(port)) {
+		if (device_may_wakeup(dev))
+			pinctrl_pm_select_idle_state(dev);
+		else
+			pinctrl_pm_select_sleep_state(dev);
+	}
 
 	return 0;
 }
diff --git a/drivers/tty/serial/stm32-usart.h b/drivers/tty/serial/stm32-usart.h
index db8bf0d4982d..d4c916e78d40 100644
--- a/drivers/tty/serial/stm32-usart.h
+++ b/drivers/tty/serial/stm32-usart.h
@@ -274,6 +274,7 @@ struct stm32_port {
 	bool fifoen;
 	int wakeirq;
 	int rdr_mask;		/* receive data register mask */
+	struct mctrl_gpios *gpios; /* modem control gpios */
 };
 
 static struct stm32_port stm32_ports[STM32_MAX_PORTS];
diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c
index 35e9e8faf8de..b9d672af8b65 100644
--- a/drivers/tty/serial/xilinx_uartps.c
+++ b/drivers/tty/serial/xilinx_uartps.c
@@ -1235,9 +1235,7 @@ static void cdns_uart_console_write(struct console *co, const char *s,
 	writel(ctrl, port->membase + CDNS_UART_CR);
 
 	uart_console_write(port, s, count, cdns_uart_console_putchar);
-	while ((readl(port->membase + CDNS_UART_SR) &
-			(CDNS_UART_SR_TXEMPTY | CDNS_UART_SR_TACTIVE)) !=
-			CDNS_UART_SR_TXEMPTY)
+	while (cdns_uart_tx_empty(port) != TIOCSER_TEMT)
 		cpu_relax();
 
 	/* restore interrupt state */
@@ -1262,6 +1260,7 @@ static int cdns_uart_console_setup(struct console *co, char *options)
 	int bits = 8;
 	int parity = 'n';
 	int flow = 'n';
+	unsigned long time_out;
 
 	if (!port->membase) {
 		pr_debug("console on " CDNS_UART_TTY_NAME "%i not present\n",
@@ -1272,6 +1271,13 @@ static int cdns_uart_console_setup(struct console *co, char *options)
 	if (options)
 		uart_parse_options(options, &baud, &parity, &bits, &flow);
 
+	/* Wait for tx_empty before setting up the console */
+	time_out = jiffies + usecs_to_jiffies(TX_TIMEOUT);
+
+	while (time_before(jiffies, time_out) &&
+	       cdns_uart_tx_empty(port) != TIOCSER_TEMT)
+		cpu_relax();
+
 	return uart_set_options(port, co, baud, parity, bits, flow);
 }
 
diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c
index 0dc3878794fd..7c95afa905a0 100644
--- a/drivers/tty/sysrq.c
+++ b/drivers/tty/sysrq.c
@@ -106,7 +106,7 @@ static void sysrq_handle_loglevel(int key)
 	pr_info("Loglevel set to %d\n", i);
 	console_loglevel = i;
 }
-static struct sysrq_key_op sysrq_loglevel_op = {
+static const struct sysrq_key_op sysrq_loglevel_op = {
 	.handler	= sysrq_handle_loglevel,
 	.help_msg	= "loglevel(0-9)",
 	.action_msg	= "Changing Loglevel",
@@ -119,14 +119,14 @@ static void sysrq_handle_SAK(int key)
 	struct work_struct *SAK_work = &vc_cons[fg_console].SAK_work;
 	schedule_work(SAK_work);
 }
-static struct sysrq_key_op sysrq_SAK_op = {
+static const struct sysrq_key_op sysrq_SAK_op = {
 	.handler	= sysrq_handle_SAK,
 	.help_msg	= "sak(k)",
 	.action_msg	= "SAK",
 	.enable_mask	= SYSRQ_ENABLE_KEYBOARD,
 };
 #else
-#define sysrq_SAK_op (*(struct sysrq_key_op *)NULL)
+#define sysrq_SAK_op (*(const struct sysrq_key_op *)NULL)
 #endif
 
 #ifdef CONFIG_VT
@@ -135,14 +135,14 @@ static void sysrq_handle_unraw(int key)
 	vt_reset_unicode(fg_console);
 }
 
-static struct sysrq_key_op sysrq_unraw_op = {
+static const struct sysrq_key_op sysrq_unraw_op = {
 	.handler	= sysrq_handle_unraw,
 	.help_msg	= "unraw(r)",
 	.action_msg	= "Keyboard mode set to system default",
 	.enable_mask	= SYSRQ_ENABLE_KEYBOARD,
 };
 #else
-#define sysrq_unraw_op (*(struct sysrq_key_op *)NULL)
+#define sysrq_unraw_op (*(const struct sysrq_key_op *)NULL)
 #endif /* CONFIG_VT */
 
 static void sysrq_handle_crash(int key)
@@ -152,7 +152,7 @@ static void sysrq_handle_crash(int key)
 
 	panic("sysrq triggered crash\n");
 }
-static struct sysrq_key_op sysrq_crash_op = {
+static const struct sysrq_key_op sysrq_crash_op = {
 	.handler	= sysrq_handle_crash,
 	.help_msg	= "crash(c)",
 	.action_msg	= "Trigger a crash",
@@ -165,18 +165,20 @@ static void sysrq_handle_reboot(int key)
 	local_irq_enable();
 	emergency_restart();
 }
-static struct sysrq_key_op sysrq_reboot_op = {
+static const struct sysrq_key_op sysrq_reboot_op = {
 	.handler	= sysrq_handle_reboot,
 	.help_msg	= "reboot(b)",
 	.action_msg	= "Resetting",
 	.enable_mask	= SYSRQ_ENABLE_BOOT,
 };
 
+const struct sysrq_key_op *__sysrq_reboot_op = &sysrq_reboot_op;
+
 static void sysrq_handle_sync(int key)
 {
 	emergency_sync();
 }
-static struct sysrq_key_op sysrq_sync_op = {
+static const struct sysrq_key_op sysrq_sync_op = {
 	.handler	= sysrq_handle_sync,
 	.help_msg	= "sync(s)",
 	.action_msg	= "Emergency Sync",
@@ -188,7 +190,7 @@ static void sysrq_handle_show_timers(int key)
 	sysrq_timer_list_show();
 }
 
-static struct sysrq_key_op sysrq_show_timers_op = {
+static const struct sysrq_key_op sysrq_show_timers_op = {
 	.handler	= sysrq_handle_show_timers,
 	.help_msg	= "show-all-timers(q)",
 	.action_msg	= "Show clockevent devices & pending hrtimers (no others)",
@@ -198,7 +200,7 @@ static void sysrq_handle_mountro(int key)
 {
 	emergency_remount();
 }
-static struct sysrq_key_op sysrq_mountro_op = {
+static const struct sysrq_key_op sysrq_mountro_op = {
 	.handler	= sysrq_handle_mountro,
 	.help_msg	= "unmount(u)",
 	.action_msg	= "Emergency Remount R/O",
@@ -211,13 +213,13 @@ static void sysrq_handle_showlocks(int key)
 	debug_show_all_locks();
 }
 
-static struct sysrq_key_op sysrq_showlocks_op = {
+static const struct sysrq_key_op sysrq_showlocks_op = {
 	.handler	= sysrq_handle_showlocks,
 	.help_msg	= "show-all-locks(d)",
 	.action_msg	= "Show Locks Held",
 };
 #else
-#define sysrq_showlocks_op (*(struct sysrq_key_op *)NULL)
+#define sysrq_showlocks_op (*(const struct sysrq_key_op *)NULL)
 #endif
 
 #ifdef CONFIG_SMP
@@ -233,7 +235,7 @@ static void showacpu(void *dummy)
 
 	raw_spin_lock_irqsave(&show_lock, flags);
 	pr_info("CPU%d:\n", smp_processor_id());
-	show_stack(NULL, NULL);
+	show_stack(NULL, NULL, KERN_INFO);
 	raw_spin_unlock_irqrestore(&show_lock, flags);
 }
 
@@ -264,7 +266,7 @@ static void sysrq_handle_showallcpus(int key)
 	}
 }
 
-static struct sysrq_key_op sysrq_showallcpus_op = {
+static const struct sysrq_key_op sysrq_showallcpus_op = {
 	.handler	= sysrq_handle_showallcpus,
 	.help_msg	= "show-backtrace-all-active-cpus(l)",
 	.action_msg	= "Show backtrace of all active CPUs",
@@ -282,7 +284,7 @@ static void sysrq_handle_showregs(int key)
 		show_regs(regs);
 	perf_event_print_debug();
 }
-static struct sysrq_key_op sysrq_showregs_op = {
+static const struct sysrq_key_op sysrq_showregs_op = {
 	.handler	= sysrq_handle_showregs,
 	.help_msg	= "show-registers(p)",
 	.action_msg	= "Show Regs",
@@ -294,7 +296,7 @@ static void sysrq_handle_showstate(int key)
 	show_state();
 	show_workqueue_state();
 }
-static struct sysrq_key_op sysrq_showstate_op = {
+static const struct sysrq_key_op sysrq_showstate_op = {
 	.handler	= sysrq_handle_showstate,
 	.help_msg	= "show-task-states(t)",
 	.action_msg	= "Show State",
@@ -305,7 +307,7 @@ static void sysrq_handle_showstate_blocked(int key)
 {
 	show_state_filter(TASK_UNINTERRUPTIBLE);
 }
-static struct sysrq_key_op sysrq_showstate_blocked_op = {
+static const struct sysrq_key_op sysrq_showstate_blocked_op = {
 	.handler	= sysrq_handle_showstate_blocked,
 	.help_msg	= "show-blocked-tasks(w)",
 	.action_msg	= "Show Blocked State",
@@ -319,21 +321,21 @@ static void sysrq_ftrace_dump(int key)
 {
 	ftrace_dump(DUMP_ALL);
 }
-static struct sysrq_key_op sysrq_ftrace_dump_op = {
+static const struct sysrq_key_op sysrq_ftrace_dump_op = {
 	.handler	= sysrq_ftrace_dump,
 	.help_msg	= "dump-ftrace-buffer(z)",
 	.action_msg	= "Dump ftrace buffer",
 	.enable_mask	= SYSRQ_ENABLE_DUMP,
 };
 #else
-#define sysrq_ftrace_dump_op (*(struct sysrq_key_op *)NULL)
+#define sysrq_ftrace_dump_op (*(const struct sysrq_key_op *)NULL)
 #endif
 
 static void sysrq_handle_showmem(int key)
 {
 	show_mem(0, NULL);
 }
-static struct sysrq_key_op sysrq_showmem_op = {
+static const struct sysrq_key_op sysrq_showmem_op = {
 	.handler	= sysrq_handle_showmem,
 	.help_msg	= "show-memory-usage(m)",
 	.action_msg	= "Show Memory",
@@ -364,7 +366,7 @@ static void sysrq_handle_term(int key)
 	send_sig_all(SIGTERM);
 	console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
 }
-static struct sysrq_key_op sysrq_term_op = {
+static const struct sysrq_key_op sysrq_term_op = {
 	.handler	= sysrq_handle_term,
 	.help_msg	= "terminate-all-tasks(e)",
 	.action_msg	= "Terminate All Tasks",
@@ -394,7 +396,7 @@ static void sysrq_handle_moom(int key)
 {
 	schedule_work(&moom_work);
 }
-static struct sysrq_key_op sysrq_moom_op = {
+static const struct sysrq_key_op sysrq_moom_op = {
 	.handler	= sysrq_handle_moom,
 	.help_msg	= "memory-full-oom-kill(f)",
 	.action_msg	= "Manual OOM execution",
@@ -406,7 +408,7 @@ static void sysrq_handle_thaw(int key)
 {
 	emergency_thaw_all();
 }
-static struct sysrq_key_op sysrq_thaw_op = {
+static const struct sysrq_key_op sysrq_thaw_op = {
 	.handler	= sysrq_handle_thaw,
 	.help_msg	= "thaw-filesystems(j)",
 	.action_msg	= "Emergency Thaw of all frozen filesystems",
@@ -419,7 +421,7 @@ static void sysrq_handle_kill(int key)
 	send_sig_all(SIGKILL);
 	console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
 }
-static struct sysrq_key_op sysrq_kill_op = {
+static const struct sysrq_key_op sysrq_kill_op = {
 	.handler	= sysrq_handle_kill,
 	.help_msg	= "kill-all-tasks(i)",
 	.action_msg	= "Kill All Tasks",
@@ -430,7 +432,7 @@ static void sysrq_handle_unrt(int key)
 {
 	normalize_rt_tasks();
 }
-static struct sysrq_key_op sysrq_unrt_op = {
+static const struct sysrq_key_op sysrq_unrt_op = {
 	.handler	= sysrq_handle_unrt,
 	.help_msg	= "nice-all-RT-tasks(n)",
 	.action_msg	= "Nice All RT Tasks",
@@ -440,7 +442,7 @@ static struct sysrq_key_op sysrq_unrt_op = {
 /* Key Operations table and lock */
 static DEFINE_SPINLOCK(sysrq_key_table_lock);
 
-static struct sysrq_key_op *sysrq_key_table[36] = {
+static const struct sysrq_key_op *sysrq_key_table[36] = {
 	&sysrq_loglevel_op,		/* 0 */
 	&sysrq_loglevel_op,		/* 1 */
 	&sysrq_loglevel_op,		/* 2 */
@@ -516,9 +518,9 @@ static int sysrq_key_table_key2index(int key)
 /*
  * get and put functions for the table, exposed to modules.
  */
-struct sysrq_key_op *__sysrq_get_key_op(int key)
+static const struct sysrq_key_op *__sysrq_get_key_op(int key)
 {
-        struct sysrq_key_op *op_p = NULL;
+        const struct sysrq_key_op *op_p = NULL;
         int i;
 
 	i = sysrq_key_table_key2index(key);
@@ -528,7 +530,7 @@ struct sysrq_key_op *__sysrq_get_key_op(int key)
         return op_p;
 }
 
-static void __sysrq_put_key_op(int key, struct sysrq_key_op *op_p)
+static void __sysrq_put_key_op(int key, const struct sysrq_key_op *op_p)
 {
         int i = sysrq_key_table_key2index(key);
 
@@ -538,7 +540,7 @@ static void __sysrq_put_key_op(int key, struct sysrq_key_op *op_p)
 
 void __handle_sysrq(int key, bool check_mask)
 {
-	struct sysrq_key_op *op_p;
+	const struct sysrq_key_op *op_p;
 	int orig_log_level;
 	int orig_suppress_printk;
 	int i;
@@ -1061,8 +1063,8 @@ int sysrq_toggle_support(int enable_mask)
 }
 EXPORT_SYMBOL_GPL(sysrq_toggle_support);
 
-static int __sysrq_swap_key_ops(int key, struct sysrq_key_op *insert_op_p,
-                                struct sysrq_key_op *remove_op_p)
+static int __sysrq_swap_key_ops(int key, const struct sysrq_key_op *insert_op_p,
+                                const struct sysrq_key_op *remove_op_p)
 {
 	int retval;
 
@@ -1085,13 +1087,13 @@ static int __sysrq_swap_key_ops(int key, struct sysrq_key_op *insert_op_p,
 	return retval;
 }
 
-int register_sysrq_key(int key, struct sysrq_key_op *op_p)
+int register_sysrq_key(int key, const struct sysrq_key_op *op_p)
 {
 	return __sysrq_swap_key_ops(key, op_p, NULL);
 }
 EXPORT_SYMBOL(register_sysrq_key);
 
-int unregister_sysrq_key(int key, struct sysrq_key_op *op_p)
+int unregister_sysrq_key(int key, const struct sysrq_key_op *op_p)
 {
 	return __sysrq_swap_key_ops(key, NULL, op_p);
 }
diff --git a/drivers/tty/vcc.c b/drivers/tty/vcc.c
index d2a1e1228c82..9ffd42e333b8 100644
--- a/drivers/tty/vcc.c
+++ b/drivers/tty/vcc.c
@@ -605,6 +605,7 @@ static int vcc_probe(struct vio_dev *vdev, const struct vio_device_id *id)
 	port->index = vcc_table_add(port);
 	if (port->index == -1) {
 		pr_err("VCC: no more TTY indices left for allocation\n");
+		rv = -ENOMEM;
 		goto free_ldc;
 	}
 
diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c
index b28aa0d289f8..c1be96bb3ecf 100644
--- a/drivers/tty/vt/consolemap.c
+++ b/drivers/tty/vt/consolemap.c
@@ -12,7 +12,7 @@
  * Fix bug in inverse translation. Stanislav Voronyi <stas@cnti.uanet.kharkov.ua>, Dec 1998
  *
  * In order to prevent the following circular lock dependency:
- *   &mm->mmap_sem --> cpu_hotplug.lock --> console_lock --> &mm->mmap_sem
+ *   &mm->mmap_lock --> cpu_hotplug.lock --> console_lock --> &mm->mmap_lock
  *
  * We cannot allow page fault to happen while holding the console_lock.
  * Therefore, all the userspace copy operations have to be done outside
diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c
index 15d33fa0c925..568b2171f335 100644
--- a/drivers/tty/vt/keyboard.c
+++ b/drivers/tty/vt/keyboard.c
@@ -127,7 +127,11 @@ static DEFINE_SPINLOCK(func_buf_lock); /* guard 'func_buf'  and friends */
 static unsigned long key_down[BITS_TO_LONGS(KEY_CNT)];	/* keyboard key bitmap */
 static unsigned char shift_down[NR_SHIFT];		/* shift state counters.. */
 static bool dead_key_next;
-static int npadch = -1;					/* -1 or number assembled on pad */
+
+/* Handles a number being assembled on the number pad */
+static bool npadch_active;
+static unsigned int npadch_value;
+
 static unsigned int diacr;
 static char rep;					/* flag telling character repeat */
 
@@ -845,12 +849,12 @@ static void k_shift(struct vc_data *vc, unsigned char value, char up_flag)
 		shift_state &= ~(1 << value);
 
 	/* kludge */
-	if (up_flag && shift_state != old_state && npadch != -1) {
+	if (up_flag && shift_state != old_state && npadch_active) {
 		if (kbd->kbdmode == VC_UNICODE)
-			to_utf8(vc, npadch);
+			to_utf8(vc, npadch_value);
 		else
-			put_queue(vc, npadch & 0xff);
-		npadch = -1;
+			put_queue(vc, npadch_value & 0xff);
+		npadch_active = false;
 	}
 }
 
@@ -868,7 +872,7 @@ static void k_meta(struct vc_data *vc, unsigned char value, char up_flag)
 
 static void k_ascii(struct vc_data *vc, unsigned char value, char up_flag)
 {
-	int base;
+	unsigned int base;
 
 	if (up_flag)
 		return;
@@ -882,10 +886,12 @@ static void k_ascii(struct vc_data *vc, unsigned char value, char up_flag)
 		base = 16;
 	}
 
-	if (npadch == -1)
-		npadch = value;
-	else
-		npadch = npadch * base + value;
+	if (!npadch_active) {
+		npadch_value = 0;
+		npadch_active = true;
+	}
+
+	npadch_value = npadch_value * base + value;
 }
 
 static void k_lock(struct vc_data *vc, unsigned char value, char up_flag)
diff --git a/drivers/tty/vt/selection.c b/drivers/tty/vt/selection.c
index d54a549c5892..31bb3647a99c 100644
--- a/drivers/tty/vt/selection.c
+++ b/drivers/tty/vt/selection.c
@@ -185,47 +185,54 @@ int set_selection_user(const struct tiocl_selection __user *sel,
 	return set_selection_kernel(&v, tty);
 }
 
-static int __set_selection_kernel(struct tiocl_selection *v, struct tty_struct *tty)
+static int vc_selection_store_chars(struct vc_data *vc, bool unicode)
 {
-	struct vc_data *vc = vc_cons[fg_console].d;
-	int new_sel_start, new_sel_end, spc;
 	char *bp, *obp;
-	int i, ps, pe;
-	u32 c;
-	int ret = 0;
-	bool unicode;
-
-	poke_blanked_console();
-
-	v->xs = min_t(u16, v->xs - 1, vc->vc_cols - 1);
-	v->ys = min_t(u16, v->ys - 1, vc->vc_rows - 1);
-	v->xe = min_t(u16, v->xe - 1, vc->vc_cols - 1);
-	v->ye = min_t(u16, v->ye - 1, vc->vc_rows - 1);
-	ps = v->ys * vc->vc_size_row + (v->xs << 1);
-	pe = v->ye * vc->vc_size_row + (v->xe << 1);
+	unsigned int i;
 
-	if (v->sel_mode == TIOCL_SELCLEAR) {
-		/* useful for screendump without selection highlights */
+	/* Allocate a new buffer before freeing the old one ... */
+	/* chars can take up to 4 bytes with unicode */
+	bp = kmalloc_array((vc_sel.end - vc_sel.start) / 2 + 1, unicode ? 4 : 1,
+			   GFP_KERNEL);
+	if (!bp) {
+		printk(KERN_WARNING "selection: kmalloc() failed\n");
 		clear_selection();
-		return 0;
+		return -ENOMEM;
 	}
+	kfree(vc_sel.buffer);
+	vc_sel.buffer = bp;
 
-	if (mouse_reporting() && (v->sel_mode & TIOCL_SELMOUSEREPORT)) {
-		mouse_report(tty, v->sel_mode & TIOCL_SELBUTTONMASK, v->xs,
-			     v->ys);
-		return 0;
+	obp = bp;
+	for (i = vc_sel.start; i <= vc_sel.end; i += 2) {
+		u32 c = sel_pos(i, unicode);
+		if (unicode)
+			bp += store_utf8(c, bp);
+		else
+			*bp++ = c;
+		if (!isspace(c))
+			obp = bp;
+		if (!((i + 2) % vc->vc_size_row)) {
+			/* strip trailing blanks from line and add newline,
+			   unless non-space at end of line. */
+			if (obp != bp) {
+				bp = obp;
+				*bp++ = '\r';
+			}
+			obp = bp;
+		}
 	}
+	vc_sel.buf_len = bp - vc_sel.buffer;
 
-	if (ps > pe)	/* make vc_sel.start <= vc_sel.end */
-		swap(ps, pe);
+	return 0;
+}
 
-	if (vc_sel.cons != vc_cons[fg_console].d) {
-		clear_selection();
-		vc_sel.cons = vc_cons[fg_console].d;
-	}
-	unicode = vt_do_kdgkbmode(fg_console) == K_UNICODE;
+static int vc_do_selection(struct vc_data *vc, unsigned short mode, int ps,
+		int pe)
+{
+	int new_sel_start, new_sel_end, spc;
+	bool unicode = vt_do_kdgkbmode(fg_console) == K_UNICODE;
 
-	switch (v->sel_mode) {
+	switch (mode) {
 	case TIOCL_SELCHAR:	/* character-by-character selection */
 		new_sel_start = ps;
 		new_sel_end = pe;
@@ -303,40 +310,44 @@ static int __set_selection_kernel(struct tiocl_selection *v, struct tty_struct *
 	vc_sel.start = new_sel_start;
 	vc_sel.end = new_sel_end;
 
-	/* Allocate a new buffer before freeing the old one ... */
-	/* chars can take up to 4 bytes with unicode */
-	bp = kmalloc_array((vc_sel.end - vc_sel.start) / 2 + 1, unicode ? 4 : 1,
-			   GFP_KERNEL);
-	if (!bp) {
-		printk(KERN_WARNING "selection: kmalloc() failed\n");
+	return vc_selection_store_chars(vc, unicode);
+}
+
+static int vc_selection(struct vc_data *vc, struct tiocl_selection *v,
+		struct tty_struct *tty)
+{
+	int ps, pe;
+
+	poke_blanked_console();
+
+	if (v->sel_mode == TIOCL_SELCLEAR) {
+		/* useful for screendump without selection highlights */
 		clear_selection();
-		return -ENOMEM;
+		return 0;
 	}
-	kfree(vc_sel.buffer);
-	vc_sel.buffer = bp;
 
-	obp = bp;
-	for (i = vc_sel.start; i <= vc_sel.end; i += 2) {
-		c = sel_pos(i, unicode);
-		if (unicode)
-			bp += store_utf8(c, bp);
-		else
-			*bp++ = c;
-		if (!isspace(c))
-			obp = bp;
-		if (! ((i + 2) % vc->vc_size_row)) {
-			/* strip trailing blanks from line and add newline,
-			   unless non-space at end of line. */
-			if (obp != bp) {
-				bp = obp;
-				*bp++ = '\r';
-			}
-			obp = bp;
-		}
+	v->xs = min_t(u16, v->xs - 1, vc->vc_cols - 1);
+	v->ys = min_t(u16, v->ys - 1, vc->vc_rows - 1);
+	v->xe = min_t(u16, v->xe - 1, vc->vc_cols - 1);
+	v->ye = min_t(u16, v->ye - 1, vc->vc_rows - 1);
+
+	if (mouse_reporting() && (v->sel_mode & TIOCL_SELMOUSEREPORT)) {
+		mouse_report(tty, v->sel_mode & TIOCL_SELBUTTONMASK, v->xs,
+			     v->ys);
+		return 0;
 	}
-	vc_sel.buf_len = bp - vc_sel.buffer;
 
-	return ret;
+	ps = v->ys * vc->vc_size_row + (v->xs << 1);
+	pe = v->ye * vc->vc_size_row + (v->xe << 1);
+	if (ps > pe)	/* make vc_sel.start <= vc_sel.end */
+		swap(ps, pe);
+
+	if (vc_sel.cons != vc) {
+		clear_selection();
+		vc_sel.cons = vc;
+	}
+
+	return vc_do_selection(vc, v->sel_mode, ps, pe);
 }
 
 int set_selection_kernel(struct tiocl_selection *v, struct tty_struct *tty)
@@ -345,7 +356,7 @@ int set_selection_kernel(struct tiocl_selection *v, struct tty_struct *tty)
 
 	mutex_lock(&vc_sel.lock);
 	console_lock();
-	ret = __set_selection_kernel(v, tty);
+	ret = vc_selection(vc_cons[fg_console].d, v, tty);
 	console_unlock();
 	mutex_unlock(&vc_sel.lock);
 
diff --git a/drivers/uio/uio.c b/drivers/uio/uio.c
index 6e725c6c6256..73efb80815db 100644
--- a/drivers/uio/uio.c
+++ b/drivers/uio/uio.c
@@ -398,7 +398,7 @@ static void uio_dev_del_attributes(struct uio_device *idev)
 
 static int uio_get_minor(struct uio_device *idev)
 {
-	int retval = -ENOMEM;
+	int retval;
 
 	mutex_lock(&minor_lock);
 	retval = idr_alloc(&uio_idr, idev, 0, UIO_MAX_DEVICES, GFP_KERNEL);
diff --git a/drivers/uio/uio_dmem_genirq.c b/drivers/uio/uio_dmem_genirq.c
index f6ab3f28c838..6e27fe4fcca3 100644
--- a/drivers/uio/uio_dmem_genirq.c
+++ b/drivers/uio/uio_dmem_genirq.c
@@ -44,7 +44,6 @@ static int uio_dmem_genirq_open(struct uio_info *info, struct inode *inode)
 {
 	struct uio_dmem_genirq_platdata *priv = info->priv;
 	struct uio_mem *uiomem;
-	int ret = 0;
 	int dmem_region = priv->dmem_region_start;
 
 	uiomem = &priv->uioinfo->mem[priv->dmem_region_start];
@@ -68,7 +67,7 @@ static int uio_dmem_genirq_open(struct uio_info *info, struct inode *inode)
 	mutex_unlock(&priv->alloc_lock);
 	/* Wait until the Runtime PM code has woken up the device */
 	pm_runtime_get_sync(&priv->pdev->dev);
-	return ret;
+	return 0;
 }
 
 static int uio_dmem_genirq_release(struct uio_info *info, struct inode *inode)
diff --git a/drivers/uio/uio_hv_generic.c b/drivers/uio/uio_hv_generic.c
index 3c5169eb23f5..4dae2320b103 100644
--- a/drivers/uio/uio_hv_generic.c
+++ b/drivers/uio/uio_hv_generic.c
@@ -361,6 +361,7 @@ hv_uio_remove(struct hv_device *dev)
 	if (!pdata)
 		return 0;
 
+	sysfs_remove_bin_file(&dev->channel->kobj, &ring_buffer_bin_attr);
 	uio_unregister_device(&pdata->info);
 	hv_uio_cleanup(dev, pdata);
 	hv_set_drvdata(dev, NULL);
diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig
index 275568abc670..26475b409b53 100644
--- a/drivers/usb/Kconfig
+++ b/drivers/usb/Kconfig
@@ -30,7 +30,7 @@ menuconfig USB_SUPPORT
 	bool "USB support"
 	depends on HAS_IOMEM
 	default y
-	---help---
+	help
 	  This option adds core support for Universal Serial Bus (USB).
 	  You will also need drivers from the following menu to make use of it.
 
@@ -47,7 +47,7 @@ config USB
 	select GENERIC_ALLOCATOR
 	select USB_COMMON
 	select NLS  # for UTF-8 strings
-	---help---
+	help
 	  Universal Serial Bus (USB) is a specification for a serial bus
 	  subsystem which offers higher speeds and more features than the
 	  traditional PC serial port.  The bus supplies power to peripherals
@@ -83,7 +83,7 @@ config USB_PCI
 	bool "PCI based USB host interface"
 	depends on PCI
 	default y
-	---help---
+	help
 	  Many embedded system SOCs (e.g. freescale T2080) have both
 	  PCI and USB modules with the USB module directly controlled by
 	  registers and having no relationship to the PCI module.
@@ -133,7 +133,7 @@ config USB_USS720
 	tristate "USS720 parport driver"
 	depends on PARPORT
 	select PARPORT_NOT_PC
-	---help---
+	help
 	  This driver is for USB parallel port adapters that use the Lucent
 	  Technologies USS-720 chip. These cables are plugged into your USB
 	  port and provide USB compatibility to peripherals designed with
diff --git a/drivers/usb/cdns3/cdns3-ti.c b/drivers/usb/cdns3/cdns3-ti.c
index 5685ba11480b..e701ab56b0a7 100644
--- a/drivers/usb/cdns3/cdns3-ti.c
+++ b/drivers/usb/cdns3/cdns3-ti.c
@@ -138,7 +138,7 @@ static int cdns_ti_probe(struct platform_device *pdev)
 	error = pm_runtime_get_sync(dev);
 	if (error < 0) {
 		dev_err(dev, "pm_runtime_get_sync failed: %d\n", error);
-		goto err_get;
+		goto err;
 	}
 
 	/* assert RESET */
@@ -185,7 +185,6 @@ static int cdns_ti_probe(struct platform_device *pdev)
 
 err:
 	pm_runtime_put_sync(data->dev);
-err_get:
 	pm_runtime_disable(data->dev);
 
 	return error;
diff --git a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c
index 4aafba20f450..19bbb5b7e6b6 100644
--- a/drivers/usb/cdns3/core.c
+++ b/drivers/usb/cdns3/core.c
@@ -82,8 +82,6 @@ static void cdns3_exit_roles(struct cdns3 *cdns)
 	cdns3_drd_exit(cdns);
 }
 
-static enum usb_role cdsn3_hw_role_state_machine(struct cdns3 *cdns);
-
 /**
  * cdns3_core_init_role - initialize role of operation
  * @cdns: Pointer to cdns3 structure
@@ -193,12 +191,12 @@ err:
 }
 
 /**
- * cdsn3_hw_role_state_machine  - role switch state machine based on hw events.
+ * cdns3_hw_role_state_machine  - role switch state machine based on hw events.
  * @cdns: Pointer to controller structure.
  *
  * Returns next role to be entered based on hw events.
  */
-static enum usb_role cdsn3_hw_role_state_machine(struct cdns3 *cdns)
+static enum usb_role cdns3_hw_role_state_machine(struct cdns3 *cdns)
 {
 	enum usb_role role;
 	int id, vbus;
@@ -291,14 +289,10 @@ int cdns3_hw_role_switch(struct cdns3 *cdns)
 	enum usb_role real_role, current_role;
 	int ret = 0;
 
-	/* Do nothing if role based on syfs. */
-	if (cdns->role_override)
-		return 0;
-
 	pm_runtime_get_sync(cdns->dev);
 
 	current_role = cdns->role;
-	real_role = cdsn3_hw_role_state_machine(cdns);
+	real_role = cdns3_hw_role_state_machine(cdns);
 
 	/* Do nothing if nothing changed */
 	if (current_role == real_role)
@@ -353,39 +347,6 @@ static int cdns3_role_set(struct usb_role_switch *sw, enum usb_role role)
 
 	pm_runtime_get_sync(cdns->dev);
 
-	/*
-	 * FIXME: switch role framework should be extended to meet
-	 * requirements. Driver assumes that role can be controlled
-	 * by SW or HW. Temporary workaround is to use USB_ROLE_NONE to
-	 * switch from SW to HW control.
-	 *
-	 * For dr_mode == USB_DR_MODE_OTG:
-	 *	if user sets USB_ROLE_HOST or USB_ROLE_DEVICE then driver
-	 *	sets role_override flag and forces that role.
-	 *	if user sets USB_ROLE_NONE, driver clears role_override and lets
-	 *	HW state machine take over.
-	 *
-	 * For dr_mode != USB_DR_MODE_OTG:
-	 *	Assumptions:
-	 *	1. Restricted user control between NONE and dr_mode.
-	 *	2. Driver doesn't need to rely on role_override flag.
-	 *	3. Driver needs to ensure that HW state machine is never called
-	 *	   if dr_mode != USB_DR_MODE_OTG.
-	 */
-	if (role == USB_ROLE_NONE)
-		cdns->role_override = 0;
-	else
-		cdns->role_override = 1;
-
-	/*
-	 * HW state might have changed so driver need to trigger
-	 * HW state machine if dr_mode == USB_DR_MODE_OTG.
-	 */
-	if (!cdns->role_override && cdns->dr_mode == USB_DR_MODE_OTG) {
-		cdns3_hw_role_switch(cdns);
-		goto pm_put;
-	}
-
 	if (cdns->role == role)
 		goto pm_put;
 
@@ -528,6 +489,8 @@ static int cdns3_probe(struct platform_device *pdev)
 	sw_desc.get = cdns3_role_get;
 	sw_desc.allow_userspace_control = true;
 	sw_desc.driver_data = cdns;
+	if (device_property_read_bool(dev, "usb-role-switch"))
+		sw_desc.fwnode = dev->fwnode;
 
 	cdns->role_sw = usb_role_switch_register(dev, &sw_desc);
 	if (IS_ERR(cdns->role_sw)) {
diff --git a/drivers/usb/cdns3/core.h b/drivers/usb/cdns3/core.h
index 969eb94de204..1ad1f1fe61e9 100644
--- a/drivers/usb/cdns3/core.h
+++ b/drivers/usb/cdns3/core.h
@@ -62,7 +62,6 @@ struct cdns3_role_driver {
  *           This field based on firmware setting, kernel configuration
  *           and hardware configuration.
  * @role_sw: pointer to role switch object.
- * @role_override: set 1 if role rely on SW.
  */
 struct cdns3 {
 	struct device			*dev;
@@ -90,7 +89,6 @@ struct cdns3 {
 	struct mutex			mutex;
 	enum usb_dr_mode		dr_mode;
 	struct usb_role_switch		*role_sw;
-	int				role_override;
 };
 
 int cdns3_hw_role_switch(struct cdns3 *cdns);
diff --git a/drivers/usb/cdns3/drd.c b/drivers/usb/cdns3/drd.c
index 16ad485f0b69..58089841ed52 100644
--- a/drivers/usb/cdns3/drd.c
+++ b/drivers/usb/cdns3/drd.c
@@ -329,7 +329,7 @@ int cdns3_drd_init(struct cdns3 *cdns)
 		cdns->otg_v1_regs = NULL;
 		cdns->otg_regs = regs;
 		writel(1, &cdns->otg_v0_regs->simulate);
-		dev_info(cdns->dev, "DRD version v0 (%08x)\n",
+		dev_dbg(cdns->dev, "DRD version v0 (%08x)\n",
 			 readl(&cdns->otg_v0_regs->version));
 	} else {
 		cdns->otg_v0_regs = NULL;
@@ -337,7 +337,7 @@ int cdns3_drd_init(struct cdns3 *cdns)
 		cdns->otg_regs = (void *)&cdns->otg_v1_regs->cmd;
 		cdns->version  = CDNS3_CONTROLLER_V1;
 		writel(1, &cdns->otg_v1_regs->simulate);
-		dev_info(cdns->dev, "DRD version v1 (ID: %08x, rev: %08x)\n",
+		dev_dbg(cdns->dev, "DRD version v1 (ID: %08x, rev: %08x)\n",
 			 readl(&cdns->otg_v1_regs->did),
 			 readl(&cdns->otg_v1_regs->rid));
 	}
diff --git a/drivers/usb/cdns3/ep0.c b/drivers/usb/cdns3/ep0.c
index e71240b386b4..82645a2a0f52 100644
--- a/drivers/usb/cdns3/ep0.c
+++ b/drivers/usb/cdns3/ep0.c
@@ -332,13 +332,6 @@ static int cdns3_ep0_feature_handle_device(struct cdns3_device *priv_dev,
 		case TEST_K:
 		case TEST_SE0_NAK:
 		case TEST_PACKET:
-			cdns3_ep0_complete_setup(priv_dev, 0, 1);
-			/**
-			 *  Little delay to give the controller some time
-			 * for sending status stage.
-			 * This time should be less then 3ms.
-			 */
-			mdelay(1);
 			cdns3_set_register_bit(&priv_dev->regs->usb_cmd,
 					       USB_CMD_STMODE |
 					       USB_STS_TMODE_SEL(tmode - 1));
diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c
index 4d43f3b28309..5e24c2e57c0d 100644
--- a/drivers/usb/cdns3/gadget.c
+++ b/drivers/usb/cdns3/gadget.c
@@ -512,8 +512,8 @@ static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 }
 
 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
-					      struct cdns3_endpoint *priv_ep,
-					      struct cdns3_request *priv_req)
+						     struct cdns3_endpoint *priv_ep,
+						     struct cdns3_request *priv_req)
 {
 	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 	    priv_req->flags & REQUEST_INTERNAL) {
@@ -552,8 +552,8 @@ static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_d
 }
 
 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
-			      struct cdns3_endpoint *priv_ep,
-			      struct cdns3_request *priv_req)
+				     struct cdns3_endpoint *priv_ep,
+				     struct cdns3_request *priv_req)
 {
 	int deferred = 0;
 
@@ -1905,7 +1905,7 @@ static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
 }
 
 static void cdns3_stream_ep_reconfig(struct cdns3_device *priv_dev,
-			      struct cdns3_endpoint *priv_ep)
+				     struct cdns3_endpoint *priv_ep)
 {
 	if (!priv_ep->use_streams || priv_dev->gadget.speed < USB_SPEED_SUPER)
 		return;
@@ -1926,7 +1926,7 @@ static void cdns3_stream_ep_reconfig(struct cdns3_device *priv_dev,
 }
 
 static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
-			   struct cdns3_endpoint *priv_ep)
+				  struct cdns3_endpoint *priv_ep)
 {
 	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
 
@@ -2965,7 +2965,7 @@ static int cdns3_init_eps(struct cdns3_device *priv_dev)
 
 		priv_ep->flags = 0;
 
-		dev_info(priv_dev->dev, "Initialized  %s support: %s %s\n",
+		dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",
 			 priv_ep->name,
 			 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
 			 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
@@ -3069,6 +3069,7 @@ static int cdns3_gadget_start(struct cdns3 *cdns)
 	priv_dev->gadget.name = "usb-ss-gadget";
 	priv_dev->gadget.sg_supported = 1;
 	priv_dev->gadget.quirk_avoids_skb_reserve = 1;
+	priv_dev->gadget.irq = cdns->dev_irq;
 
 	spin_lock_init(&priv_dev->lock);
 	INIT_WORK(&priv_dev->pending_status_wq,
diff --git a/drivers/usb/chipidea/Kconfig b/drivers/usb/chipidea/Kconfig
index d53db520e209..8bafcfc6080d 100644
--- a/drivers/usb/chipidea/Kconfig
+++ b/drivers/usb/chipidea/Kconfig
@@ -18,17 +18,6 @@ config USB_CHIPIDEA
 
 if USB_CHIPIDEA
 
-config USB_CHIPIDEA_OF
-	tristate
-	depends on OF
-	default USB_CHIPIDEA
-
-config USB_CHIPIDEA_PCI
-	tristate
-	depends on USB_PCI
-	depends on NOP_USB_XCEIV
-	default USB_CHIPIDEA
-
 config USB_CHIPIDEA_UDC
 	bool "ChipIdea device controller"
 	depends on USB_GADGET
@@ -43,4 +32,30 @@ config USB_CHIPIDEA_HOST
 	help
 	  Say Y here to enable host controller functionality of the
 	  ChipIdea driver.
+
+config USB_CHIPIDEA_PCI
+	tristate "Enable PCI glue driver" if EMBEDDED
+	depends on USB_PCI
+	depends on NOP_USB_XCEIV
+	default USB_CHIPIDEA
+
+config USB_CHIPIDEA_MSM
+	tristate "Enable MSM hsusb glue driver" if EMBEDDED
+	default USB_CHIPIDEA
+
+config USB_CHIPIDEA_IMX
+	tristate "Enable i.MX USB glue driver" if EMBEDDED
+	depends on OF
+	default USB_CHIPIDEA
+
+config USB_CHIPIDEA_GENERIC
+	tristate "Enable generic USB2 glue driver" if EMBEDDED
+	default USB_CHIPIDEA
+
+config USB_CHIPIDEA_TEGRA
+	tristate "Enable Tegra UDC glue driver" if EMBEDDED
+	depends on OF
+	depends on USB_CHIPIDEA_UDC
+	default USB_CHIPIDEA
+
 endif
diff --git a/drivers/usb/chipidea/Makefile b/drivers/usb/chipidea/Makefile
index 12df94f78f72..fae779a23866 100644
--- a/drivers/usb/chipidea/Makefile
+++ b/drivers/usb/chipidea/Makefile
@@ -8,11 +8,8 @@ ci_hdrc-$(CONFIG_USB_OTG_FSM)		+= otg_fsm.o
 
 # Glue/Bridge layers go here
 
-obj-$(CONFIG_USB_CHIPIDEA)	+= ci_hdrc_usb2.o
-obj-$(CONFIG_USB_CHIPIDEA)	+= ci_hdrc_msm.o
-obj-$(CONFIG_USB_CHIPIDEA)	+= ci_hdrc_zevio.o
-
-obj-$(CONFIG_USB_CHIPIDEA_PCI)	+= ci_hdrc_pci.o
-
-obj-$(CONFIG_USB_CHIPIDEA_OF)	+= usbmisc_imx.o ci_hdrc_imx.o
-obj-$(CONFIG_USB_CHIPIDEA_OF)	+= ci_hdrc_tegra.o
+obj-$(CONFIG_USB_CHIPIDEA_GENERIC)	+= ci_hdrc_usb2.o
+obj-$(CONFIG_USB_CHIPIDEA_MSM)		+= ci_hdrc_msm.o
+obj-$(CONFIG_USB_CHIPIDEA_PCI)		+= ci_hdrc_pci.o
+obj-$(CONFIG_USB_CHIPIDEA_IMX)		+= ci_hdrc_imx.o usbmisc_imx.o
+obj-$(CONFIG_USB_CHIPIDEA_TEGRA)	+= ci_hdrc_tegra.o
diff --git a/drivers/usb/chipidea/ci.h b/drivers/usb/chipidea/ci.h
index 644ecaef17ee..0697eb980e5f 100644
--- a/drivers/usb/chipidea/ci.h
+++ b/drivers/usb/chipidea/ci.h
@@ -25,6 +25,7 @@
 #define TD_PAGE_COUNT      5
 #define CI_HDRC_PAGE_SIZE  4096ul /* page size for TD's */
 #define ENDPT_MAX          32
+#define CI_MAX_BUF_SIZE	(TD_PAGE_COUNT * CI_HDRC_PAGE_SIZE)
 
 /******************************************************************************
  * REGISTERS
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index a479af3ae31d..5ae16368a0c7 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -271,6 +271,7 @@ static int ci_hdrc_imx_notify_event(struct ci_hdrc *ci, unsigned int event)
 	struct device *dev = ci->dev->parent;
 	struct ci_hdrc_imx_data *data = dev_get_drvdata(dev);
 	int ret = 0;
+	struct imx_usbmisc_data *mdata = data->usbmisc_data;
 
 	switch (event) {
 	case CI_HDRC_IMX_HSIC_ACTIVE_EVENT:
@@ -284,11 +285,19 @@ static int ci_hdrc_imx_notify_event(struct ci_hdrc *ci, unsigned int event)
 		}
 		break;
 	case CI_HDRC_IMX_HSIC_SUSPEND_EVENT:
-		ret = imx_usbmisc_hsic_set_connect(data->usbmisc_data);
+		ret = imx_usbmisc_hsic_set_connect(mdata);
 		if (ret)
 			dev_err(dev,
 				"hsic_set_connect failed, err=%d\n", ret);
 		break;
+	case CI_HDRC_CONTROLLER_VBUS_EVENT:
+		if (ci->vbus_active)
+			ret = imx_usbmisc_charger_detection(mdata, true);
+		else
+			ret = imx_usbmisc_charger_detection(mdata, false);
+		if (ci->usb_phy)
+			schedule_work(&ci->usb_phy->chg_work);
+		break;
 	default:
 		break;
 	}
@@ -414,6 +423,8 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
 	}
 
 	pdata.usb_phy = data->phy;
+	if (data->usbmisc_data)
+		data->usbmisc_data->usb_phy = data->phy;
 
 	if ((of_device_is_compatible(np, "fsl,imx53-usb") ||
 	     of_device_is_compatible(np, "fsl,imx51-usb")) && pdata.usb_phy &&
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.h b/drivers/usb/chipidea/ci_hdrc_imx.h
index c2051aeba13f..727d02b6dbd3 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.h
+++ b/drivers/usb/chipidea/ci_hdrc_imx.h
@@ -24,6 +24,7 @@ struct imx_usbmisc_data {
 	unsigned int hsic:1; /* HSIC controlller */
 	unsigned int ext_id:1; /* ID from exteranl event */
 	unsigned int ext_vbus:1; /* Vbus from exteranl event */
+	struct usb_phy *usb_phy;
 };
 
 int imx_usbmisc_init(struct imx_usbmisc_data *data);
@@ -31,5 +32,6 @@ int imx_usbmisc_init_post(struct imx_usbmisc_data *data);
 int imx_usbmisc_set_wakeup(struct imx_usbmisc_data *data, bool enabled);
 int imx_usbmisc_hsic_set_connect(struct imx_usbmisc_data *data);
 int imx_usbmisc_hsic_set_clk(struct imx_usbmisc_data *data, bool on);
+int imx_usbmisc_charger_detection(struct imx_usbmisc_data *data, bool connect);
 
 #endif /* __DRIVER_USB_CHIPIDEA_CI_HDRC_IMX_H */
diff --git a/drivers/usb/chipidea/ci_hdrc_usb2.c b/drivers/usb/chipidea/ci_hdrc_usb2.c
index c044fba463e4..89e1d82d739b 100644
--- a/drivers/usb/chipidea/ci_hdrc_usb2.c
+++ b/drivers/usb/chipidea/ci_hdrc_usb2.c
@@ -28,13 +28,19 @@ static const struct ci_hdrc_platform_data ci_default_pdata = {
 	.flags		= CI_HDRC_DISABLE_STREAMING,
 };
 
-static struct ci_hdrc_platform_data ci_zynq_pdata = {
+static const struct ci_hdrc_platform_data ci_zynq_pdata = {
 	.capoffset	= DEF_CAPOFFSET,
 };
 
+static const struct ci_hdrc_platform_data ci_zevio_pdata = {
+	.capoffset	= DEF_CAPOFFSET,
+	.flags		= CI_HDRC_REGS_SHARED | CI_HDRC_FORCE_FULLSPEED,
+};
+
 static const struct of_device_id ci_hdrc_usb2_of_match[] = {
-	{ .compatible = "chipidea,usb2"},
-	{ .compatible = "xlnx,zynq-usb-2.20a", .data = &ci_zynq_pdata},
+	{ .compatible = "chipidea,usb2" },
+	{ .compatible = "xlnx,zynq-usb-2.20a", .data = &ci_zynq_pdata },
+	{ .compatible = "lsi,zevio-usb", .data = &ci_zevio_pdata },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ci_hdrc_usb2_of_match);
@@ -64,13 +70,14 @@ static int ci_hdrc_usb2_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
-	priv->clk = devm_clk_get(dev, NULL);
-	if (!IS_ERR(priv->clk)) {
-		ret = clk_prepare_enable(priv->clk);
-		if (ret) {
-			dev_err(dev, "failed to enable the clock: %d\n", ret);
-			return ret;
-		}
+	priv->clk = devm_clk_get_optional(dev, NULL);
+	if (IS_ERR(priv->clk))
+		return PTR_ERR(priv->clk);
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret) {
+		dev_err(dev, "failed to enable the clock: %d\n", ret);
+		return ret;
 	}
 
 	ci_pdata->name = dev_name(dev);
@@ -94,8 +101,7 @@ static int ci_hdrc_usb2_probe(struct platform_device *pdev)
 	return 0;
 
 clk_err:
-	if (!IS_ERR(priv->clk))
-		clk_disable_unprepare(priv->clk);
+	clk_disable_unprepare(priv->clk);
 	return ret;
 }
 
diff --git a/drivers/usb/chipidea/ci_hdrc_zevio.c b/drivers/usb/chipidea/ci_hdrc_zevio.c
deleted file mode 100644
index e1634da4a4b1..000000000000
--- a/drivers/usb/chipidea/ci_hdrc_zevio.c
+++ /dev/null
@@ -1,67 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *	Copyright (C) 2013 Daniel Tang <tangrs@tangrs.id.au>
- *
- * Based off drivers/usb/chipidea/ci_hdrc_msm.c
- */
-
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/usb/gadget.h>
-#include <linux/usb/chipidea.h>
-
-#include "ci.h"
-
-static struct ci_hdrc_platform_data ci_hdrc_zevio_platdata = {
-	.name			= "ci_hdrc_zevio",
-	.flags			= CI_HDRC_REGS_SHARED | CI_HDRC_FORCE_FULLSPEED,
-	.capoffset		= DEF_CAPOFFSET,
-};
-
-static int ci_hdrc_zevio_probe(struct platform_device *pdev)
-{
-	struct platform_device *ci_pdev;
-
-	dev_dbg(&pdev->dev, "ci_hdrc_zevio_probe\n");
-
-	ci_pdev = ci_hdrc_add_device(&pdev->dev,
-				pdev->resource, pdev->num_resources,
-				&ci_hdrc_zevio_platdata);
-
-	if (IS_ERR(ci_pdev)) {
-		dev_err(&pdev->dev, "ci_hdrc_add_device failed!\n");
-		return PTR_ERR(ci_pdev);
-	}
-
-	platform_set_drvdata(pdev, ci_pdev);
-
-	return 0;
-}
-
-static int ci_hdrc_zevio_remove(struct platform_device *pdev)
-{
-	struct platform_device *ci_pdev = platform_get_drvdata(pdev);
-
-	ci_hdrc_remove_device(ci_pdev);
-
-	return 0;
-}
-
-static const struct of_device_id ci_hdrc_zevio_dt_ids[] = {
-	{ .compatible = "lsi,zevio-usb", },
-	{ /* sentinel */ }
-};
-
-static struct platform_driver ci_hdrc_zevio_driver = {
-	.probe = ci_hdrc_zevio_probe,
-	.remove = ci_hdrc_zevio_remove,
-	.driver = {
-		.name = "zevio_usb",
-		.of_match_table = ci_hdrc_zevio_dt_ids,
-	},
-};
-
-MODULE_DEVICE_TABLE(of, ci_hdrc_zevio_dt_ids);
-module_platform_driver(ci_hdrc_zevio_driver);
-
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c
index ae0bdc036464..9a7c53d09ab4 100644
--- a/drivers/usb/chipidea/core.c
+++ b/drivers/usb/chipidea/core.c
@@ -3,42 +3,16 @@
  * core.c - ChipIdea USB IP core family device controller
  *
  * Copyright (C) 2008 Chipidea - MIPS Technologies, Inc. All rights reserved.
+ * Copyright (C) 2020 NXP
  *
  * Author: David Lopo
- */
-
-/*
- * Description: ChipIdea USB IP core family device controller
- *
- * This driver is composed of several blocks:
- * - HW:     hardware interface
- * - DBG:    debug facilities (optional)
- * - UTIL:   utilities
- * - ISR:    interrupts handling
- * - ENDPT:  endpoint operations (Gadget API)
- * - GADGET: gadget operations (Gadget API)
- * - BUS:    bus glue code, bus abstraction layer
+ *	   Peter Chen <peter.chen@nxp.com>
  *
- * Compile Options
- * - STALL_IN:  non-empty bulk-in pipes cannot be halted
- *              if defined mass storage compliance succeeds but with warnings
- *              => case 4: Hi >  Dn
- *              => case 5: Hi >  Di
- *              => case 8: Hi <> Do
- *              if undefined usbtest 13 fails
- * - TRACE:     enable function tracing (depends on DEBUG)
- *
- * Main Features
- * - Chapter 9 & Mass Storage Compliance with Gadget File Storage
- * - Chapter 9 Compliance with Gadget Zero (STALL_IN undefined)
- * - Normal & LPM support
- *
- * USBTEST Report
- * - OK: 0-12, 13 (STALL_IN defined) & 14
- * - Not Supported: 15 & 16 (ISO)
- *
- * TODO List
- * - Suspend & Remote Wakeup
+ * Main Features:
+ * - Four transfers are supported, usbtest is passed
+ * - USB Certification for gadget: CH9 and Mass Storage are passed
+ * - Low power mode
+ * - USB wakeup
  */
 #include <linux/delay.h>
 #include <linux/device.h>
@@ -272,7 +246,7 @@ static int hw_device_init(struct ci_hdrc *ci, void __iomem *base)
 	ci->rev = ci_get_revision(ci);
 
 	dev_dbg(ci->dev,
-		"ChipIdea HDRC found, revision: %d, lpm: %d; cap: %p op: %p\n",
+		"revision: %d, lpm: %d; cap: %px op: %px\n",
 		ci->rev, ci->hw_bank.lpm, ci->hw_bank.cap, ci->hw_bank.op);
 
 	/* setup lock mode ? */
@@ -666,6 +640,7 @@ static int ci_usb_role_switch_set(struct usb_role_switch *sw,
 static struct usb_role_switch_desc ci_role_switch = {
 	.set = ci_usb_role_switch_set,
 	.get = ci_usb_role_switch_get,
+	.allow_userspace_control = true,
 };
 
 static int ci_get_platdata(struct device *dev,
@@ -1149,8 +1124,11 @@ static int ci_hdrc_probe(struct platform_device *pdev)
 
 	if (!ci_otg_is_fsm_mode(ci)) {
 		/* only update vbus status for peripheral */
-		if (ci->role == CI_ROLE_GADGET)
+		if (ci->role == CI_ROLE_GADGET) {
+			/* Pull down DP for possible charger detection */
+			hw_write(ci, OP_USBCMD, USBCMD_RS, 0);
 			ci_handle_vbus_change(ci);
+		}
 
 		ret = ci_role_start(ci, ci->role);
 		if (ret) {
diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c
index 921bcf14dc06..db0cfde0cc3c 100644
--- a/drivers/usb/chipidea/udc.c
+++ b/drivers/usb/chipidea/udc.c
@@ -338,7 +338,7 @@ static int hw_usb_reset(struct ci_hdrc *ci)
  *****************************************************************************/
 
 static int add_td_to_list(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq,
-			  unsigned length)
+			unsigned int length, struct scatterlist *s)
 {
 	int i;
 	u32 temp;
@@ -366,7 +366,13 @@ static int add_td_to_list(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq,
 		node->ptr->token |= cpu_to_le32(mul << __ffs(TD_MULTO));
 	}
 
-	temp = (u32) (hwreq->req.dma + hwreq->req.actual);
+	if (s) {
+		temp = (u32) (sg_dma_address(s) + hwreq->req.actual);
+		node->td_remaining_size = CI_MAX_BUF_SIZE - length;
+	} else {
+		temp = (u32) (hwreq->req.dma + hwreq->req.actual);
+	}
+
 	if (length) {
 		node->ptr->page[0] = cpu_to_le32(temp);
 		for (i = 1; i < TD_PAGE_COUNT; i++) {
@@ -400,6 +406,122 @@ static inline u8 _usb_addr(struct ci_hw_ep *ep)
 	return ((ep->dir == TX) ? USB_ENDPOINT_DIR_MASK : 0) | ep->num;
 }
 
+static int prepare_td_for_non_sg(struct ci_hw_ep *hwep,
+		struct ci_hw_req *hwreq)
+{
+	unsigned int rest = hwreq->req.length;
+	int pages = TD_PAGE_COUNT;
+	int ret = 0;
+
+	if (rest == 0) {
+		ret = add_td_to_list(hwep, hwreq, 0, NULL);
+		if (ret < 0)
+			return ret;
+	}
+
+	/*
+	 * The first buffer could be not page aligned.
+	 * In that case we have to span into one extra td.
+	 */
+	if (hwreq->req.dma % PAGE_SIZE)
+		pages--;
+
+	while (rest > 0) {
+		unsigned int count = min(hwreq->req.length - hwreq->req.actual,
+			(unsigned int)(pages * CI_HDRC_PAGE_SIZE));
+
+		ret = add_td_to_list(hwep, hwreq, count, NULL);
+		if (ret < 0)
+			return ret;
+
+		rest -= count;
+	}
+
+	if (hwreq->req.zero && hwreq->req.length && hwep->dir == TX
+	    && (hwreq->req.length % hwep->ep.maxpacket == 0)) {
+		ret = add_td_to_list(hwep, hwreq, 0, NULL);
+		if (ret < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+static int prepare_td_per_sg(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq,
+		struct scatterlist *s)
+{
+	unsigned int rest = sg_dma_len(s);
+	int ret = 0;
+
+	hwreq->req.actual = 0;
+	while (rest > 0) {
+		unsigned int count = min_t(unsigned int, rest,
+				CI_MAX_BUF_SIZE);
+
+		ret = add_td_to_list(hwep, hwreq, count, s);
+		if (ret < 0)
+			return ret;
+
+		rest -= count;
+	}
+
+	return ret;
+}
+
+static void ci_add_buffer_entry(struct td_node *node, struct scatterlist *s)
+{
+	int empty_td_slot_index = (CI_MAX_BUF_SIZE - node->td_remaining_size)
+			/ CI_HDRC_PAGE_SIZE;
+	int i;
+
+	node->ptr->token +=
+		cpu_to_le32(sg_dma_len(s) << __ffs(TD_TOTAL_BYTES));
+
+	for (i = empty_td_slot_index; i < TD_PAGE_COUNT; i++) {
+		u32 page = (u32) sg_dma_address(s) +
+			(i - empty_td_slot_index) * CI_HDRC_PAGE_SIZE;
+
+		page &= ~TD_RESERVED_MASK;
+		node->ptr->page[i] = cpu_to_le32(page);
+	}
+}
+
+static int prepare_td_for_sg(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)
+{
+	struct usb_request *req = &hwreq->req;
+	struct scatterlist *s = req->sg;
+	int ret = 0, i = 0;
+	struct td_node *node = NULL;
+
+	if (!s || req->zero || req->length == 0) {
+		dev_err(hwep->ci->dev, "not supported operation for sg\n");
+		return -EINVAL;
+	}
+
+	while (i++ < req->num_mapped_sgs) {
+		if (sg_dma_address(s) % PAGE_SIZE) {
+			dev_err(hwep->ci->dev, "not page aligned sg buffer\n");
+			return -EINVAL;
+		}
+
+		if (node && (node->td_remaining_size >= sg_dma_len(s))) {
+			ci_add_buffer_entry(node, s);
+			node->td_remaining_size -= sg_dma_len(s);
+		} else {
+			ret = prepare_td_per_sg(hwep, hwreq, s);
+			if (ret)
+				return ret;
+
+			node = list_entry(hwreq->tds.prev,
+				struct td_node, td);
+		}
+
+		s = sg_next(s);
+	}
+
+	return ret;
+}
+
 /**
  * _hardware_enqueue: configures a request at hardware level
  * @hwep:   endpoint
@@ -411,8 +533,6 @@ static int _hardware_enqueue(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)
 {
 	struct ci_hdrc *ci = hwep->ci;
 	int ret = 0;
-	unsigned rest = hwreq->req.length;
-	int pages = TD_PAGE_COUNT;
 	struct td_node *firstnode, *lastnode;
 
 	/* don't queue twice */
@@ -426,35 +546,13 @@ static int _hardware_enqueue(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)
 	if (ret)
 		return ret;
 
-	/*
-	 * The first buffer could be not page aligned.
-	 * In that case we have to span into one extra td.
-	 */
-	if (hwreq->req.dma % PAGE_SIZE)
-		pages--;
-
-	if (rest == 0) {
-		ret = add_td_to_list(hwep, hwreq, 0);
-		if (ret < 0)
-			goto done;
-	}
-
-	while (rest > 0) {
-		unsigned count = min(hwreq->req.length - hwreq->req.actual,
-					(unsigned)(pages * CI_HDRC_PAGE_SIZE));
-		ret = add_td_to_list(hwep, hwreq, count);
-		if (ret < 0)
-			goto done;
-
-		rest -= count;
-	}
+	if (hwreq->req.num_mapped_sgs)
+		ret = prepare_td_for_sg(hwep, hwreq);
+	else
+		ret = prepare_td_for_non_sg(hwep, hwreq);
 
-	if (hwreq->req.zero && hwreq->req.length && hwep->dir == TX
-	    && (hwreq->req.length % hwep->ep.maxpacket == 0)) {
-		ret = add_td_to_list(hwep, hwreq, 0);
-		if (ret < 0)
-			goto done;
-	}
+	if (ret)
+		return ret;
 
 	firstnode = list_first_entry(&hwreq->tds, struct td_node, td);
 
@@ -1561,6 +1659,7 @@ static int ci_udc_vbus_session(struct usb_gadget *_gadget, int is_active)
 {
 	struct ci_hdrc *ci = container_of(_gadget, struct ci_hdrc, gadget);
 	unsigned long flags;
+	int ret = 0;
 
 	spin_lock_irqsave(&ci->lock, flags);
 	ci->vbus_active = is_active;
@@ -1570,10 +1669,14 @@ static int ci_udc_vbus_session(struct usb_gadget *_gadget, int is_active)
 		usb_phy_set_charger_state(ci->usb_phy, is_active ?
 			USB_CHARGER_PRESENT : USB_CHARGER_ABSENT);
 
+	if (ci->platdata->notify_event)
+		ret = ci->platdata->notify_event(ci,
+				CI_HDRC_CONTROLLER_VBUS_EVENT);
+
 	if (ci->driver)
 		ci_hdrc_gadget_connect(_gadget, is_active);
 
-	return 0;
+	return ret;
 }
 
 static int ci_udc_wakeup(struct usb_gadget *_gadget)
@@ -1936,6 +2039,7 @@ static int udc_start(struct ci_hdrc *ci)
 	ci->gadget.max_speed    = USB_SPEED_HIGH;
 	ci->gadget.name         = ci->platdata->name;
 	ci->gadget.otg_caps	= otg_caps;
+	ci->gadget.sg_supported = 1;
 
 	if (ci->platdata->flags & CI_HDRC_REQUIRES_ALIGNED_DMA)
 		ci->gadget.quirk_avoids_skb_reserve = 1;
diff --git a/drivers/usb/chipidea/udc.h b/drivers/usb/chipidea/udc.h
index ebb11b625bb8..5193df1e18c7 100644
--- a/drivers/usb/chipidea/udc.h
+++ b/drivers/usb/chipidea/udc.h
@@ -61,16 +61,14 @@ struct td_node {
 	struct list_head	td;
 	dma_addr_t		dma;
 	struct ci_hw_td		*ptr;
+	int			td_remaining_size;
 };
 
 /**
  * struct ci_hw_req - usb request representation
  * @req: request structure for gadget drivers
  * @queue: link to QH list
- * @ptr: transfer descriptor for this request
- * @dma: dma address for the transfer descriptor
- * @zptr: transfer descriptor for the zero packet
- * @zdma: dma address of the zero packet's transfer descriptor
+ * @tds: link to TD list
  */
 struct ci_hw_req {
 	struct usb_request	req;
diff --git a/drivers/usb/chipidea/usbmisc_imx.c b/drivers/usb/chipidea/usbmisc_imx.c
index e81e33c26e6c..f136876cb4a3 100644
--- a/drivers/usb/chipidea/usbmisc_imx.c
+++ b/drivers/usb/chipidea/usbmisc_imx.c
@@ -8,6 +8,7 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/delay.h>
+#include <linux/usb/otg.h>
 
 #include "ci_hdrc_imx.h"
 
@@ -99,6 +100,33 @@
 #define MX7D_USB_VBUS_WAKEUP_SOURCE_AVALID	MX7D_USB_VBUS_WAKEUP_SOURCE(1)
 #define MX7D_USB_VBUS_WAKEUP_SOURCE_BVALID	MX7D_USB_VBUS_WAKEUP_SOURCE(2)
 #define MX7D_USB_VBUS_WAKEUP_SOURCE_SESS_END	MX7D_USB_VBUS_WAKEUP_SOURCE(3)
+#define MX7D_USBNC_AUTO_RESUME				BIT(2)
+/* The default DM/DP value is pull-down */
+#define MX7D_USBNC_USB_CTRL2_OPMODE(v)			(v << 6)
+#define MX7D_USBNC_USB_CTRL2_OPMODE_NON_DRIVING	MX7D_USBNC_USB_CTRL2_OPMODE(1)
+#define MX7D_USBNC_USB_CTRL2_OPMODE_OVERRIDE_MASK	(BIT(7) | BIT(6))
+#define MX7D_USBNC_USB_CTRL2_OPMODE_OVERRIDE_EN		BIT(8)
+#define MX7D_USBNC_USB_CTRL2_DP_OVERRIDE_VAL		BIT(12)
+#define MX7D_USBNC_USB_CTRL2_DP_OVERRIDE_EN		BIT(13)
+#define MX7D_USBNC_USB_CTRL2_DM_OVERRIDE_VAL		BIT(14)
+#define MX7D_USBNC_USB_CTRL2_DM_OVERRIDE_EN		BIT(15)
+#define MX7D_USBNC_USB_CTRL2_DP_DM_MASK			(BIT(12) | BIT(13) | \
+							BIT(14) | BIT(15))
+
+#define MX7D_USB_OTG_PHY_CFG1		0x30
+#define MX7D_USB_OTG_PHY_CFG2_CHRG_CHRGSEL	BIT(0)
+#define MX7D_USB_OTG_PHY_CFG2_CHRG_VDATDETENB0	BIT(1)
+#define MX7D_USB_OTG_PHY_CFG2_CHRG_VDATSRCENB0	BIT(2)
+#define MX7D_USB_OTG_PHY_CFG2_CHRG_DCDENB	BIT(3)
+#define MX7D_USB_OTG_PHY_CFG2_DRVVBUS0		BIT(16)
+
+#define MX7D_USB_OTG_PHY_CFG2		0x34
+
+#define MX7D_USB_OTG_PHY_STATUS		0x3c
+#define MX7D_USB_OTG_PHY_STATUS_LINE_STATE0	BIT(0)
+#define MX7D_USB_OTG_PHY_STATUS_LINE_STATE1	BIT(1)
+#define MX7D_USB_OTG_PHY_STATUS_VBUS_VLD	BIT(3)
+#define MX7D_USB_OTG_PHY_STATUS_CHRGDET		BIT(29)
 
 #define MX6_USB_OTG_WAKEUP_BITS (MX6_BM_WAKEUP_ENABLE | MX6_BM_VBUS_WAKEUP | \
 				 MX6_BM_ID_WAKEUP)
@@ -114,6 +142,8 @@ struct usbmisc_ops {
 	int (*hsic_set_connect)(struct imx_usbmisc_data *data);
 	/* It's called during suspend/resume */
 	int (*hsic_set_clk)(struct imx_usbmisc_data *data, bool enabled);
+	/* usb charger detection */
+	int (*charger_detection)(struct imx_usbmisc_data *data);
 };
 
 struct imx_usbmisc {
@@ -609,10 +639,263 @@ static int usbmisc_imx7d_init(struct imx_usbmisc_data *data)
 		reg |= MX6_BM_PWR_POLARITY;
 	writel(reg, usbmisc->base);
 
-	reg = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
-	reg &= ~MX7D_USB_VBUS_WAKEUP_SOURCE_MASK;
-	writel(reg | MX7D_USB_VBUS_WAKEUP_SOURCE_BVALID,
-		 usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	/* SoC non-burst setting */
+	reg = readl(usbmisc->base);
+	writel(reg | MX6_BM_NON_BURST_SETTING, usbmisc->base);
+
+	if (!data->hsic) {
+		reg = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
+		reg &= ~MX7D_USB_VBUS_WAKEUP_SOURCE_MASK;
+		writel(reg | MX7D_USB_VBUS_WAKEUP_SOURCE_BVALID
+			| MX7D_USBNC_AUTO_RESUME,
+			usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	}
+
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+
+	usbmisc_imx7d_set_wakeup(data, false);
+
+	return 0;
+}
+
+static int imx7d_charger_secondary_detection(struct imx_usbmisc_data *data)
+{
+	struct imx_usbmisc *usbmisc = dev_get_drvdata(data->dev);
+	struct usb_phy *usb_phy = data->usb_phy;
+	int val;
+	unsigned long flags;
+
+	/* VDM_SRC is connected to D- and IDP_SINK is connected to D+ */
+	spin_lock_irqsave(&usbmisc->lock, flags);
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	writel(val | MX7D_USB_OTG_PHY_CFG2_CHRG_VDATSRCENB0 |
+			MX7D_USB_OTG_PHY_CFG2_CHRG_VDATDETENB0 |
+			MX7D_USB_OTG_PHY_CFG2_CHRG_CHRGSEL,
+				usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+
+	usleep_range(1000, 2000);
+
+	/*
+	 * Per BC 1.2, check voltage of D+:
+	 * DCP: if greater than VDAT_REF;
+	 * CDP: if less than VDAT_REF.
+	 */
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_STATUS);
+	if (val & MX7D_USB_OTG_PHY_STATUS_CHRGDET) {
+		dev_dbg(data->dev, "It is a dedicate charging port\n");
+		usb_phy->chg_type = DCP_TYPE;
+	} else {
+		dev_dbg(data->dev, "It is a charging downstream port\n");
+		usb_phy->chg_type = CDP_TYPE;
+	}
+
+	return 0;
+}
+
+static void imx7_disable_charger_detector(struct imx_usbmisc_data *data)
+{
+	struct imx_usbmisc *usbmisc = dev_get_drvdata(data->dev);
+	unsigned long flags;
+	u32 val;
+
+	spin_lock_irqsave(&usbmisc->lock, flags);
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	val &= ~(MX7D_USB_OTG_PHY_CFG2_CHRG_DCDENB |
+			MX7D_USB_OTG_PHY_CFG2_CHRG_VDATSRCENB0 |
+			MX7D_USB_OTG_PHY_CFG2_CHRG_VDATDETENB0 |
+			MX7D_USB_OTG_PHY_CFG2_CHRG_CHRGSEL);
+	writel(val, usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+
+	/* Set OPMODE to be 2'b00 and disable its override */
+	val = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	val &= ~MX7D_USBNC_USB_CTRL2_OPMODE_OVERRIDE_MASK;
+	writel(val, usbmisc->base + MX7D_USBNC_USB_CTRL2);
+
+	val = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	writel(val & ~MX7D_USBNC_USB_CTRL2_OPMODE_OVERRIDE_EN,
+			usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+}
+
+static int imx7d_charger_data_contact_detect(struct imx_usbmisc_data *data)
+{
+	struct imx_usbmisc *usbmisc = dev_get_drvdata(data->dev);
+	unsigned long flags;
+	u32 val;
+	int i, data_pin_contact_count = 0;
+
+	/* Enable Data Contact Detect (DCD) per the USB BC 1.2 */
+	spin_lock_irqsave(&usbmisc->lock, flags);
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	writel(val | MX7D_USB_OTG_PHY_CFG2_CHRG_DCDENB,
+			usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+
+	for (i = 0; i < 100; i = i + 1) {
+		val = readl(usbmisc->base + MX7D_USB_OTG_PHY_STATUS);
+		if (!(val & MX7D_USB_OTG_PHY_STATUS_LINE_STATE0)) {
+			if (data_pin_contact_count++ > 5)
+				/* Data pin makes contact */
+				break;
+			usleep_range(5000, 10000);
+		} else {
+			data_pin_contact_count = 0;
+			usleep_range(5000, 6000);
+		}
+	}
+
+	/* Disable DCD after finished data contact check */
+	spin_lock_irqsave(&usbmisc->lock, flags);
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	writel(val & ~MX7D_USB_OTG_PHY_CFG2_CHRG_DCDENB,
+			usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+
+	if (i == 100) {
+		dev_err(data->dev,
+			"VBUS is coming from a dedicated power supply.\n");
+		return -ENXIO;
+	}
+
+	return 0;
+}
+
+static int imx7d_charger_primary_detection(struct imx_usbmisc_data *data)
+{
+	struct imx_usbmisc *usbmisc = dev_get_drvdata(data->dev);
+	struct usb_phy *usb_phy = data->usb_phy;
+	unsigned long flags;
+	u32 val;
+
+	/* VDP_SRC is connected to D+ and IDM_SINK is connected to D- */
+	spin_lock_irqsave(&usbmisc->lock, flags);
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	val &= ~MX7D_USB_OTG_PHY_CFG2_CHRG_CHRGSEL;
+	writel(val | MX7D_USB_OTG_PHY_CFG2_CHRG_VDATSRCENB0 |
+			MX7D_USB_OTG_PHY_CFG2_CHRG_VDATDETENB0,
+				usbmisc->base + MX7D_USB_OTG_PHY_CFG2);
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+
+	usleep_range(1000, 2000);
+
+	/* Check if D- is less than VDAT_REF to determine an SDP per BC 1.2 */
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_STATUS);
+	if (!(val & MX7D_USB_OTG_PHY_STATUS_CHRGDET)) {
+		dev_dbg(data->dev, "It is a standard downstream port\n");
+		usb_phy->chg_type = SDP_TYPE;
+	}
+
+	return 0;
+}
+
+/**
+ * Whole charger detection process:
+ * 1. OPMODE override to be non-driving
+ * 2. Data contact check
+ * 3. Primary detection
+ * 4. Secondary detection
+ * 5. Disable charger detection
+ */
+static int imx7d_charger_detection(struct imx_usbmisc_data *data)
+{
+	struct imx_usbmisc *usbmisc = dev_get_drvdata(data->dev);
+	struct usb_phy *usb_phy = data->usb_phy;
+	unsigned long flags;
+	u32 val;
+	int ret;
+
+	/* Check if vbus is valid */
+	val = readl(usbmisc->base + MX7D_USB_OTG_PHY_STATUS);
+	if (!(val & MX7D_USB_OTG_PHY_STATUS_VBUS_VLD)) {
+		dev_err(data->dev, "vbus is error\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Keep OPMODE to be non-driving mode during the whole
+	 * charger detection process.
+	 */
+	spin_lock_irqsave(&usbmisc->lock, flags);
+	val = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	val &= ~MX7D_USBNC_USB_CTRL2_OPMODE_OVERRIDE_MASK;
+	val |= MX7D_USBNC_USB_CTRL2_OPMODE_NON_DRIVING;
+	writel(val, usbmisc->base + MX7D_USBNC_USB_CTRL2);
+
+	val = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	writel(val | MX7D_USBNC_USB_CTRL2_OPMODE_OVERRIDE_EN,
+			usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	spin_unlock_irqrestore(&usbmisc->lock, flags);
+
+	ret = imx7d_charger_data_contact_detect(data);
+	if (ret)
+		return ret;
+
+	ret = imx7d_charger_primary_detection(data);
+	if (!ret && usb_phy->chg_type != SDP_TYPE)
+		ret = imx7d_charger_secondary_detection(data);
+
+	imx7_disable_charger_detector(data);
+
+	return ret;
+}
+
+static int usbmisc_imx7ulp_init(struct imx_usbmisc_data *data)
+{
+	struct imx_usbmisc *usbmisc = dev_get_drvdata(data->dev);
+	unsigned long flags;
+	u32 reg;
+
+	if (data->index >= 1)
+		return -EINVAL;
+
+	spin_lock_irqsave(&usbmisc->lock, flags);
+	reg = readl(usbmisc->base);
+	if (data->disable_oc) {
+		reg |= MX6_BM_OVER_CUR_DIS;
+	} else {
+		reg &= ~MX6_BM_OVER_CUR_DIS;
+
+		/*
+		 * If the polarity is not configured keep it as setup by the
+		 * bootloader.
+		 */
+		if (data->oc_pol_configured && data->oc_pol_active_low)
+			reg |= MX6_BM_OVER_CUR_POLARITY;
+		else if (data->oc_pol_configured)
+			reg &= ~MX6_BM_OVER_CUR_POLARITY;
+	}
+	/* If the polarity is not set keep it as setup by the bootlader */
+	if (data->pwr_pol == 1)
+		reg |= MX6_BM_PWR_POLARITY;
+
+	writel(reg, usbmisc->base);
+
+	/* SoC non-burst setting */
+	reg = readl(usbmisc->base);
+	writel(reg | MX6_BM_NON_BURST_SETTING, usbmisc->base);
+
+	if (data->hsic) {
+		reg = readl(usbmisc->base);
+		writel(reg | MX6_BM_UTMI_ON_CLOCK, usbmisc->base);
+
+		reg = readl(usbmisc->base + MX6_USB_HSIC_CTRL_OFFSET);
+		reg |= MX6_BM_HSIC_EN | MX6_BM_HSIC_CLK_ON;
+		writel(reg, usbmisc->base + MX6_USB_HSIC_CTRL_OFFSET);
+
+		/*
+		 * For non-HSIC controller, the autoresume is enabled
+		 * at MXS PHY driver (usbphy_ctrl bit18).
+		 */
+		reg = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
+		writel(reg | MX7D_USBNC_AUTO_RESUME,
+			usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	} else {
+		reg = readl(usbmisc->base + MX7D_USBNC_USB_CTRL2);
+		reg &= ~MX7D_USB_VBUS_WAKEUP_SOURCE_MASK;
+		writel(reg | MX7D_USB_VBUS_WAKEUP_SOURCE_BVALID,
+			 usbmisc->base + MX7D_USBNC_USB_CTRL2);
+	}
 
 	spin_unlock_irqrestore(&usbmisc->lock, flags);
 
@@ -659,6 +942,14 @@ static const struct usbmisc_ops imx6sx_usbmisc_ops = {
 static const struct usbmisc_ops imx7d_usbmisc_ops = {
 	.init = usbmisc_imx7d_init,
 	.set_wakeup = usbmisc_imx7d_set_wakeup,
+	.charger_detection = imx7d_charger_detection,
+};
+
+static const struct usbmisc_ops imx7ulp_usbmisc_ops = {
+	.init = usbmisc_imx7ulp_init,
+	.set_wakeup = usbmisc_imx7d_set_wakeup,
+	.hsic_set_connect = usbmisc_imx6_hsic_set_connect,
+	.hsic_set_clk = usbmisc_imx6_hsic_set_clk,
 };
 
 static inline bool is_imx53_usbmisc(struct imx_usbmisc_data *data)
@@ -737,6 +1028,39 @@ int imx_usbmisc_hsic_set_clk(struct imx_usbmisc_data *data, bool on)
 	return usbmisc->ops->hsic_set_clk(data, on);
 }
 EXPORT_SYMBOL_GPL(imx_usbmisc_hsic_set_clk);
+
+int imx_usbmisc_charger_detection(struct imx_usbmisc_data *data, bool connect)
+{
+	struct imx_usbmisc *usbmisc;
+	struct usb_phy *usb_phy;
+	int ret = 0;
+
+	if (!data)
+		return -EINVAL;
+
+	usbmisc = dev_get_drvdata(data->dev);
+	usb_phy = data->usb_phy;
+	if (!usbmisc->ops->charger_detection)
+		return -ENOTSUPP;
+
+	if (connect) {
+		ret = usbmisc->ops->charger_detection(data);
+		if (ret) {
+			dev_err(data->dev,
+					"Error occurs during detection: %d\n",
+					ret);
+			usb_phy->chg_state = USB_CHARGER_ABSENT;
+		} else {
+			usb_phy->chg_state = USB_CHARGER_PRESENT;
+		}
+	} else {
+		usb_phy->chg_state = USB_CHARGER_ABSENT;
+		usb_phy->chg_type = UNKNOWN_TYPE;
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(imx_usbmisc_charger_detection);
+
 static const struct of_device_id usbmisc_imx_dt_ids[] = {
 	{
 		.compatible = "fsl,imx25-usbmisc",
@@ -780,7 +1104,7 @@ static const struct of_device_id usbmisc_imx_dt_ids[] = {
 	},
 	{
 		.compatible = "fsl,imx7ulp-usbmisc",
-		.data = &imx7d_usbmisc_ops,
+		.data = &imx7ulp_usbmisc_ops,
 	},
 	{ /* sentinel */ }
 };
diff --git a/drivers/usb/class/Kconfig b/drivers/usb/class/Kconfig
index f8a798900093..d3f5162bd67e 100644
--- a/drivers/usb/class/Kconfig
+++ b/drivers/usb/class/Kconfig
@@ -7,7 +7,7 @@ comment "USB Device Class drivers"
 config USB_ACM
 	tristate "USB Modem (CDC ACM) support"
 	depends on TTY
-	---help---
+	help
 	  This driver supports USB modems and ISDN adapters which support the
 	  Communication Device Class Abstract Control Model interface.
 	  Please read <file:Documentation/usb/acm.rst> for details.
@@ -30,7 +30,7 @@ config USB_PRINTER
 
 config USB_WDM
 	tristate "USB Wireless Device Management support"
-	---help---
+	help
 	  This driver supports the WMC Device Management functionality
 	  of cell phones compliant to the CDC WMC specification. You can use
 	  AT commands over this device.
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index ded8d93834ca..f67088bb8218 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -584,7 +584,7 @@ static void acm_softint(struct work_struct *work)
 	}
 
 	if (test_and_clear_bit(ACM_ERROR_DELAY, &acm->flags)) {
-		for (i = 0; i < ACM_NR; i++) 
+		for (i = 0; i < acm->rx_buflimit; i++)
 			if (test_and_clear_bit(i, &acm->urbs_in_error_delay))
 					acm_submit_read_urb(acm, i, GFP_NOIO);
 	}
diff --git a/drivers/usb/class/usblp.c b/drivers/usb/class/usblp.c
index 0d8e3f3804a3..084c48c5848f 100644
--- a/drivers/usb/class/usblp.c
+++ b/drivers/usb/class/usblp.c
@@ -468,7 +468,8 @@ static int usblp_release(struct inode *inode, struct file *file)
 	usb_autopm_put_interface(usblp->intf);
 
 	if (!usblp->present)		/* finish cleanup from disconnect */
-		usblp_cleanup(usblp);
+		usblp_cleanup(usblp);	/* any URBs must be dead */
+
 	mutex_unlock(&usblp_mutex);
 	return 0;
 }
@@ -1375,9 +1376,11 @@ static void usblp_disconnect(struct usb_interface *intf)
 
 	usblp_unlink_urbs(usblp);
 	mutex_unlock(&usblp->mut);
+	usb_poison_anchored_urbs(&usblp->urbs);
 
 	if (!usblp->used)
 		usblp_cleanup(usblp);
+
 	mutex_unlock(&usblp_mutex);
 }
 
diff --git a/drivers/usb/core/hcd-pci.c b/drivers/usb/core/hcd-pci.c
index f0a259937da8..1547aa6e5314 100644
--- a/drivers/usb/core/hcd-pci.c
+++ b/drivers/usb/core/hcd-pci.c
@@ -159,6 +159,7 @@ static void ehci_wait_for_companions(struct pci_dev *pdev, struct usb_hcd *hcd,
  * usb_hcd_pci_probe - initialize PCI-based HCDs
  * @dev: USB Host Controller being probed
  * @id: pci hotplug id connecting controller to HCD framework
+ * @driver: USB HC driver handle
  * Context: !in_interrupt()
  *
  * Allocates basic PCI resources for this USB host controller, and
@@ -169,9 +170,9 @@ static void ehci_wait_for_companions(struct pci_dev *pdev, struct usb_hcd *hcd,
  *
  * Return: 0 if successful.
  */
-int usb_hcd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
+int usb_hcd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id,
+		      const struct hc_driver *driver)
 {
-	struct hc_driver	*driver;
 	struct usb_hcd		*hcd;
 	int			retval;
 	int			hcd_irq = 0;
@@ -181,7 +182,7 @@ int usb_hcd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 
 	if (!id)
 		return -EINVAL;
-	driver = (struct hc_driver *)id->driver_data;
+
 	if (!driver)
 		return -EINVAL;
 
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index aa45840d8273..de624c47e190 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -31,6 +31,7 @@
 #include <linux/types.h>
 #include <linux/genalloc.h>
 #include <linux/io.h>
+#include <linux/kcov.h>
 
 #include <linux/phy/phy.h>
 #include <linux/usb.h>
@@ -1645,7 +1646,9 @@ static void __usb_hcd_giveback_urb(struct urb *urb)
 
 	/* pass ownership to the completion handler */
 	urb->status = status;
+	kcov_remote_start_usb((u64)urb->dev->bus->busnum);
 	urb->complete(urb);
+	kcov_remote_stop();
 
 	usb_anchor_resume_wakeups(anchor);
 	atomic_dec(&urb->use_count);
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index fc748c731832..b1e14beaac5f 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -93,7 +93,7 @@ module_param(old_scheme_first, bool, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(old_scheme_first,
 		 "start with the old device initialization scheme");
 
-static bool use_both_schemes = 1;
+static bool use_both_schemes = true;
 module_param(use_both_schemes, bool, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(use_both_schemes,
 		"try the other device initialization scheme if the "
diff --git a/drivers/usb/core/hub.h b/drivers/usb/core/hub.h
index a97dd1ba964e..73f4482d833a 100644
--- a/drivers/usb/core/hub.h
+++ b/drivers/usb/core/hub.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * usb hub driver head file
  *
diff --git a/drivers/usb/core/otg_whitelist.h b/drivers/usb/core/otg_whitelist.h
index 2ae90158ded7..fdd4897401e2 100644
--- a/drivers/usb/core/otg_whitelist.h
+++ b/drivers/usb/core/otg_whitelist.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * drivers/usb/core/otg_whitelist.h
  *
diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c
index 9f4320b9d7fc..a2ca38e25e0c 100644
--- a/drivers/usb/core/sysfs.c
+++ b/drivers/usb/core/sysfs.c
@@ -1262,8 +1262,10 @@ void usb_create_sysfs_intf_files(struct usb_interface *intf)
 
 	if (!alt->string && !(udev->quirks & USB_QUIRK_CONFIG_INTF_STRINGS))
 		alt->string = usb_cache_string(udev, alt->desc.iInterface);
-	if (alt->string && device_create_file(&intf->dev, &dev_attr_interface))
-		;	/* We don't actually care if the function fails. */
+	if (alt->string && device_create_file(&intf->dev, &dev_attr_interface)) {
+		/* This is not a serious error */
+		dev_dbg(&intf->dev, "interface string descriptor file not created\n");
+	}
 	intf->sysfs_files_created = 1;
 }
 
diff --git a/drivers/usb/core/usb.h b/drivers/usb/core/usb.h
index 64ed4023a8c8..19e4c550bc73 100644
--- a/drivers/usb/core/usb.h
+++ b/drivers/usb/core/usb.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Released under the GPLv2 only.
  */
diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c
index 78a4925aa118..fec17a2d2447 100644
--- a/drivers/usb/dwc2/core.c
+++ b/drivers/usb/dwc2/core.c
@@ -524,10 +524,25 @@ int dwc2_core_reset(struct dwc2_hsotg *hsotg, bool skip_wait)
 	greset |= GRSTCTL_CSFTRST;
 	dwc2_writel(hsotg, greset, GRSTCTL);
 
-	if (dwc2_hsotg_wait_bit_clear(hsotg, GRSTCTL, GRSTCTL_CSFTRST, 10000)) {
-		dev_warn(hsotg->dev, "%s: HANG! Soft Reset timeout GRSTCTL GRSTCTL_CSFTRST\n",
-			 __func__);
-		return -EBUSY;
+	if ((hsotg->hw_params.snpsid & DWC2_CORE_REV_MASK) <
+		(DWC2_CORE_REV_4_20a & DWC2_CORE_REV_MASK)) {
+		if (dwc2_hsotg_wait_bit_clear(hsotg, GRSTCTL,
+					      GRSTCTL_CSFTRST, 10000)) {
+			dev_warn(hsotg->dev, "%s: HANG! Soft Reset timeout GRSTCTL_CSFTRST\n",
+				 __func__);
+			return -EBUSY;
+		}
+	} else {
+		if (dwc2_hsotg_wait_bit_set(hsotg, GRSTCTL,
+					    GRSTCTL_CSFTRST_DONE, 10000)) {
+			dev_warn(hsotg->dev, "%s: HANG! Soft Reset timeout GRSTCTL_CSFTRST_DONE\n",
+				 __func__);
+			return -EBUSY;
+		}
+		greset = dwc2_readl(hsotg, GRSTCTL);
+		greset &= ~GRSTCTL_CSFTRST;
+		greset |= GRSTCTL_CSFTRST_DONE;
+		dwc2_writel(hsotg, greset, GRSTCTL);
 	}
 
 	/* Wait for AHB master IDLE state */
diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h
index 99b0bdfe0012..132d687f1590 100644
--- a/drivers/usb/dwc2/core.h
+++ b/drivers/usb/dwc2/core.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
 /*
  * core.h - DesignWare HS OTG Controller common declarations
  *
@@ -1103,8 +1103,10 @@ struct dwc2_hsotg {
 #define DWC2_CORE_REV_3_00a	0x4f54300a
 #define DWC2_CORE_REV_3_10a	0x4f54310a
 #define DWC2_CORE_REV_4_00a	0x4f54400a
+#define DWC2_CORE_REV_4_20a	0x4f54420a
 #define DWC2_FS_IOT_REV_1_00a	0x5531100a
 #define DWC2_HS_IOT_REV_1_00a	0x5532100a
+#define DWC2_CORE_REV_MASK	0x0000ffff
 
 	/* DWC OTG HW Core ID */
 #define DWC2_OTG_ID		0x4f540000
@@ -1309,6 +1311,8 @@ void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg);
 
 bool dwc2_is_controller_alive(struct dwc2_hsotg *hsotg);
 
+int dwc2_check_core_version(struct dwc2_hsotg *hsotg);
+
 /*
  * Common core Functions.
  * The following functions support managing the DWC_otg controller in either
diff --git a/drivers/usb/dwc2/core_intr.c b/drivers/usb/dwc2/core_intr.c
index 876ff31261d5..55f1d14fc414 100644
--- a/drivers/usb/dwc2/core_intr.c
+++ b/drivers/usb/dwc2/core_intr.c
@@ -416,10 +416,13 @@ static void dwc2_handle_wakeup_detected_intr(struct dwc2_hsotg *hsotg)
 			if (ret && (ret != -ENOTSUPP))
 				dev_err(hsotg->dev, "exit power_down failed\n");
 
+			/* Change to L0 state */
+			hsotg->lx_state = DWC2_L0;
 			call_gadget(hsotg, resume);
+		} else {
+			/* Change to L0 state */
+			hsotg->lx_state = DWC2_L0;
 		}
-		/* Change to L0 state */
-		hsotg->lx_state = DWC2_L0;
 	} else {
 		if (hsotg->params.power_down)
 			return;
diff --git a/drivers/usb/dwc2/debug.h b/drivers/usb/dwc2/debug.h
index a8c565b6bc34..47252c56d410 100644
--- a/drivers/usb/dwc2/debug.h
+++ b/drivers/usb/dwc2/debug.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * debug.h - Designware USB2 DRD controller debug header
  *
diff --git a/drivers/usb/dwc2/hcd.h b/drivers/usb/dwc2/hcd.h
index 1224fa9df604..ea02ee63ac6d 100644
--- a/drivers/usb/dwc2/hcd.h
+++ b/drivers/usb/dwc2/hcd.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
 /*
  * hcd.h - DesignWare HS OTG Controller host-mode declarations
  *
diff --git a/drivers/usb/dwc2/hw.h b/drivers/usb/dwc2/hw.h
index c4027bbcedec..c3d6dde2aca4 100644
--- a/drivers/usb/dwc2/hw.h
+++ b/drivers/usb/dwc2/hw.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
 /*
  * hw.h - DesignWare HS OTG Controller hardware definitions
  *
@@ -126,6 +126,7 @@
 #define GRSTCTL				HSOTG_REG(0x010)
 #define GRSTCTL_AHBIDLE			BIT(31)
 #define GRSTCTL_DMAREQ			BIT(30)
+#define GRSTCTL_CSFTRST_DONE		BIT(29)
 #define GRSTCTL_TXFNUM_MASK		(0x1f << 6)
 #define GRSTCTL_TXFNUM_SHIFT		6
 #define GRSTCTL_TXFNUM_LIMIT		0x1f
diff --git a/drivers/usb/dwc2/params.c b/drivers/usb/dwc2/params.c
index 8ccc83f7eb3f..ce736d67c7c3 100644
--- a/drivers/usb/dwc2/params.c
+++ b/drivers/usb/dwc2/params.c
@@ -782,25 +782,6 @@ int dwc2_get_hwparams(struct dwc2_hsotg *hsotg)
 	u32 hwcfg1, hwcfg2, hwcfg3, hwcfg4;
 	u32 grxfsiz;
 
-	/*
-	 * Attempt to ensure this device is really a DWC_otg Controller.
-	 * Read and verify the GSNPSID register contents. The value should be
-	 * 0x45f4xxxx, 0x5531xxxx or 0x5532xxxx
-	 */
-
-	hw->snpsid = dwc2_readl(hsotg, GSNPSID);
-	if ((hw->snpsid & GSNPSID_ID_MASK) != DWC2_OTG_ID &&
-	    (hw->snpsid & GSNPSID_ID_MASK) != DWC2_FS_IOT_ID &&
-	    (hw->snpsid & GSNPSID_ID_MASK) != DWC2_HS_IOT_ID) {
-		dev_err(hsotg->dev, "Bad value for GSNPSID: 0x%08x\n",
-			hw->snpsid);
-		return -ENODEV;
-	}
-
-	dev_dbg(hsotg->dev, "Core Release: %1x.%1x%1x%1x (snpsid=%x)\n",
-		hw->snpsid >> 12 & 0xf, hw->snpsid >> 8 & 0xf,
-		hw->snpsid >> 4 & 0xf, hw->snpsid & 0xf, hw->snpsid);
-
 	hwcfg1 = dwc2_readl(hsotg, GHWCFG1);
 	hwcfg2 = dwc2_readl(hsotg, GHWCFG2);
 	hwcfg3 = dwc2_readl(hsotg, GHWCFG3);
diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c
index 69972750e161..e571c8ae65ec 100644
--- a/drivers/usb/dwc2/platform.c
+++ b/drivers/usb/dwc2/platform.c
@@ -363,6 +363,37 @@ static bool dwc2_check_core_endianness(struct dwc2_hsotg *hsotg)
 }
 
 /**
+ * Check core version
+ *
+ * @hsotg: Programming view of the DWC_otg controller
+ *
+ */
+int dwc2_check_core_version(struct dwc2_hsotg *hsotg)
+{
+	struct dwc2_hw_params *hw = &hsotg->hw_params;
+
+	/*
+	 * Attempt to ensure this device is really a DWC_otg Controller.
+	 * Read and verify the GSNPSID register contents. The value should be
+	 * 0x45f4xxxx, 0x5531xxxx or 0x5532xxxx
+	 */
+
+	hw->snpsid = dwc2_readl(hsotg, GSNPSID);
+	if ((hw->snpsid & GSNPSID_ID_MASK) != DWC2_OTG_ID &&
+	    (hw->snpsid & GSNPSID_ID_MASK) != DWC2_FS_IOT_ID &&
+	    (hw->snpsid & GSNPSID_ID_MASK) != DWC2_HS_IOT_ID) {
+		dev_err(hsotg->dev, "Bad value for GSNPSID: 0x%08x\n",
+			hw->snpsid);
+		return -ENODEV;
+	}
+
+	dev_dbg(hsotg->dev, "Core Release: %1x.%1x%1x%1x (snpsid=%x)\n",
+		hw->snpsid >> 12 & 0xf, hw->snpsid >> 8 & 0xf,
+		hw->snpsid >> 4 & 0xf, hw->snpsid & 0xf, hw->snpsid);
+	return 0;
+}
+
+/**
  * dwc2_driver_probe() - Called when the DWC_otg core is bound to the DWC_otg
  * driver
  *
@@ -445,6 +476,14 @@ static int dwc2_driver_probe(struct platform_device *dev)
 				      "snps,need-phy-for-wake");
 
 	/*
+	 * Before performing any core related operations
+	 * check core version.
+	 */
+	retval = dwc2_check_core_version(hsotg);
+	if (retval)
+		goto error;
+
+	/*
 	 * Reset before dwc2_get_hwparams() then it could get power-on real
 	 * reset value form registers.
 	 */
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index edc17155cb2b..25c686a752b0 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -85,7 +85,9 @@ static int dwc3_get_dr_mode(struct dwc3 *dwc)
 		 * specified or set to OTG, then set the mode to peripheral.
 		 */
 		if (mode == USB_DR_MODE_OTG &&
-		    dwc->revision >= DWC3_REVISION_330A)
+		    (!IS_ENABLED(CONFIG_USB_ROLE_SWITCH) ||
+		     !device_property_read_bool(dwc->dev, "usb-role-switch")) &&
+		    !DWC3_VER_IS_PRIOR(DWC3, 330A))
 			mode = USB_DR_MODE_PERIPHERAL;
 	}
 
@@ -121,17 +123,19 @@ static void __dwc3_set_mode(struct work_struct *work)
 	if (dwc->dr_mode != USB_DR_MODE_OTG)
 		return;
 
+	pm_runtime_get_sync(dwc->dev);
+
 	if (dwc->current_dr_role == DWC3_GCTL_PRTCAP_OTG)
 		dwc3_otg_update(dwc, 0);
 
 	if (!dwc->desired_dr_role)
-		return;
+		goto out;
 
 	if (dwc->desired_dr_role == dwc->current_dr_role)
-		return;
+		goto out;
 
 	if (dwc->desired_dr_role == DWC3_GCTL_PRTCAP_OTG && dwc->edev)
-		return;
+		goto out;
 
 	switch (dwc->current_dr_role) {
 	case DWC3_GCTL_PRTCAP_HOST:
@@ -190,6 +194,9 @@ static void __dwc3_set_mode(struct work_struct *work)
 		break;
 	}
 
+out:
+	pm_runtime_mark_last_busy(dwc->dev);
+	pm_runtime_put_autosuspend(dwc->dev);
 }
 
 void dwc3_set_mode(struct dwc3 *dwc, u32 mode)
@@ -257,7 +264,7 @@ static int dwc3_core_soft_reset(struct dwc3 *dwc)
 	 * take a little more than 50ms. Set the polling rate at 20ms
 	 * for 10 times instead.
 	 */
-	if (dwc3_is_usb31(dwc) && dwc->revision >= DWC3_USB31_REVISION_190A)
+	if (DWC3_VER_IS_WITHIN(DWC31, 190A, ANY) || DWC3_IP_IS(DWC32))
 		retries = 10;
 
 	do {
@@ -265,8 +272,7 @@ static int dwc3_core_soft_reset(struct dwc3 *dwc)
 		if (!(reg & DWC3_DCTL_CSFTRST))
 			goto done;
 
-		if (dwc3_is_usb31(dwc) &&
-		    dwc->revision >= DWC3_USB31_REVISION_190A)
+		if (DWC3_VER_IS_WITHIN(DWC31, 190A, ANY) || DWC3_IP_IS(DWC32))
 			msleep(20);
 		else
 			udelay(1);
@@ -283,7 +289,7 @@ done:
 	 * is cleared, we must wait at least 50ms before accessing the PHY
 	 * domain (synchronization delay).
 	 */
-	if (dwc3_is_usb31(dwc) && dwc->revision <= DWC3_USB31_REVISION_180A)
+	if (DWC3_VER_IS_WITHIN(DWC31, ANY, 180A))
 		msleep(50);
 
 	return 0;
@@ -298,7 +304,7 @@ static void dwc3_frame_length_adjustment(struct dwc3 *dwc)
 	u32 reg;
 	u32 dft;
 
-	if (dwc->revision < DWC3_REVISION_250A)
+	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
 		return;
 
 	if (dwc->fladj == 0)
@@ -579,7 +585,7 @@ static int dwc3_phy_setup(struct dwc3 *dwc)
 	 * will be '0' when the core is reset. Application needs to set it
 	 * to '1' after the core initialization is completed.
 	 */
-	if (dwc->revision > DWC3_REVISION_194A)
+	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A))
 		reg |= DWC3_GUSB3PIPECTL_SUSPHY;
 
 	/*
@@ -670,7 +676,7 @@ static int dwc3_phy_setup(struct dwc3 *dwc)
 	 * be '0' when the core is reset. Application needs to set it to
 	 * '1' after the core initialization is completed.
 	 */
-	if (dwc->revision > DWC3_REVISION_194A)
+	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A))
 		reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 
 	/*
@@ -719,15 +725,13 @@ static bool dwc3_core_is_valid(struct dwc3 *dwc)
 	u32 reg;
 
 	reg = dwc3_readl(dwc->regs, DWC3_GSNPSID);
+	dwc->ip = DWC3_GSNPS_ID(reg);
 
 	/* This should read as U3 followed by revision number */
-	if ((reg & DWC3_GSNPSID_MASK) == 0x55330000) {
-		/* Detected DWC_usb3 IP */
+	if (DWC3_IP_IS(DWC3)) {
 		dwc->revision = reg;
-	} else if ((reg & DWC3_GSNPSID_MASK) == 0x33310000) {
-		/* Detected DWC_usb31 IP */
+	} else if (DWC3_IP_IS(DWC31) || DWC3_IP_IS(DWC32)) {
 		dwc->revision = dwc3_readl(dwc->regs, DWC3_VER_NUMBER);
-		dwc->revision |= DWC3_REVISION_IS_DWC31;
 		dwc->version_type = dwc3_readl(dwc->regs, DWC3_VER_TYPE);
 	} else {
 		return false;
@@ -760,8 +764,7 @@ static void dwc3_core_setup_global_control(struct dwc3 *dwc)
 		 */
 		if ((dwc->dr_mode == USB_DR_MODE_HOST ||
 				dwc->dr_mode == USB_DR_MODE_OTG) &&
-				(dwc->revision >= DWC3_REVISION_210A &&
-				dwc->revision <= DWC3_REVISION_250A))
+				DWC3_VER_IS_WITHIN(DWC3, 210A, 250A))
 			reg |= DWC3_GCTL_DSBLCLKGTNG | DWC3_GCTL_SOFITPSYNC;
 		else
 			reg &= ~DWC3_GCTL_DSBLCLKGTNG;
@@ -804,7 +807,7 @@ static void dwc3_core_setup_global_control(struct dwc3 *dwc)
 	 * and falls back to high-speed mode which causes
 	 * the device to enter a Connect/Disconnect loop
 	 */
-	if (dwc->revision < DWC3_REVISION_190A)
+	if (DWC3_VER_IS_PRIOR(DWC3, 190A))
 		reg |= DWC3_GCTL_U2RSTECN;
 
 	dwc3_writel(dwc->regs, DWC3_GCTL, reg);
@@ -957,7 +960,7 @@ static int dwc3_core_init(struct dwc3 *dwc)
 		goto err0a;
 
 	if (hw_mode == DWC3_GHWPARAMS0_MODE_DRD &&
-	    dwc->revision > DWC3_REVISION_194A) {
+	    !DWC3_VER_IS_WITHIN(DWC3, ANY, 194A)) {
 		if (!dwc->dis_u3_susphy_quirk) {
 			reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
 			reg |= DWC3_GUSB3PIPECTL_SUSPHY;
@@ -1004,20 +1007,20 @@ static int dwc3_core_init(struct dwc3 *dwc)
 	 * the DWC_usb3 controller. It is NOT available in the
 	 * DWC_usb31 controller.
 	 */
-	if (!dwc3_is_usb31(dwc) && dwc->revision >= DWC3_REVISION_310A) {
+	if (DWC3_VER_IS_WITHIN(DWC3, 310A, ANY)) {
 		reg = dwc3_readl(dwc->regs, DWC3_GUCTL2);
 		reg |= DWC3_GUCTL2_RST_ACTBITLATER;
 		dwc3_writel(dwc->regs, DWC3_GUCTL2, reg);
 	}
 
-	if (dwc->revision >= DWC3_REVISION_250A) {
+	if (!DWC3_VER_IS_PRIOR(DWC3, 250A)) {
 		reg = dwc3_readl(dwc->regs, DWC3_GUCTL1);
 
 		/*
 		 * Enable hardware control of sending remote wakeup
 		 * in HS when the device is in the L1 state.
 		 */
-		if (dwc->revision >= DWC3_REVISION_290A)
+		if (!DWC3_VER_IS_PRIOR(DWC3, 290A))
 			reg |= DWC3_GUCTL1_DEV_L1_EXIT_BY_HW;
 
 		if (dwc->dis_tx_ipgap_linecheck_quirk)
@@ -1049,7 +1052,7 @@ static int dwc3_core_init(struct dwc3 *dwc)
 	 * Must config both number of packets and max burst settings to enable
 	 * RX and/or TX threshold.
 	 */
-	if (dwc3_is_usb31(dwc) && dwc->dr_mode == USB_DR_MODE_HOST) {
+	if (!DWC3_IP_IS(DWC3) && dwc->dr_mode == USB_DR_MODE_HOST) {
 		u8 rx_thr_num = dwc->rx_thr_num_pkt_prd;
 		u8 rx_maxburst = dwc->rx_max_burst_prd;
 		u8 tx_thr_num = dwc->tx_thr_num_pkt_prd;
@@ -1371,10 +1374,9 @@ static void dwc3_get_properties(struct dwc3 *dwc)
 /* check whether the core supports IMOD */
 bool dwc3_has_imod(struct dwc3 *dwc)
 {
-	return ((dwc3_is_usb3(dwc) &&
-		 dwc->revision >= DWC3_REVISION_300A) ||
-		(dwc3_is_usb31(dwc) &&
-		 dwc->revision >= DWC3_USB31_REVISION_120A));
+	return DWC3_VER_IS_WITHIN(DWC3, 300A, ANY) ||
+		DWC3_VER_IS_WITHIN(DWC31, 120A, ANY) ||
+		DWC3_IP_IS(DWC32);
 }
 
 static void dwc3_check_params(struct dwc3 *dwc)
@@ -1395,7 +1397,7 @@ static void dwc3_check_params(struct dwc3 *dwc)
 	 * affected version.
 	 */
 	if (!dwc->imod_interval &&
-	    (dwc->revision == DWC3_REVISION_300A))
+	    DWC3_VER_IS(DWC3, 300A))
 		dwc->imod_interval = 1;
 
 	/* Check the maximum_speed parameter */
@@ -1417,7 +1419,7 @@ static void dwc3_check_params(struct dwc3 *dwc)
 		/*
 		 * default to superspeed plus if we are capable.
 		 */
-		if (dwc3_is_usb31(dwc) &&
+		if ((DWC3_IP_IS(DWC31) || DWC3_IP_IS(DWC32)) &&
 		    (DWC3_GHWPARAMS3_SSPHY_IFC(dwc->hwparams.hwparams3) ==
 		     DWC3_GHWPARAMS3_SSPHY_IFC_GEN2))
 			dwc->maximum_speed = USB_SPEED_SUPER_PLUS;
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 4c171a8e215f..013f42a2b5dc 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * core.h - DesignWare USB3 DRD Core Header
  *
@@ -69,6 +69,7 @@
 #define DWC3_GEVNTCOUNT_EHB	BIT(31)
 #define DWC3_GSNPSID_MASK	0xffff0000
 #define DWC3_GSNPSREV_MASK	0xffff
+#define DWC3_GSNPS_ID(p)	(((p) & DWC3_GSNPSID_MASK) >> 16)
 
 /* DWC3 registers memory space boundries */
 #define DWC3_XHCI_REGS_START		0x0
@@ -365,6 +366,9 @@
 #define DWC3_GHWPARAMS6_SRPSUPPORT		BIT(10)
 #define DWC3_GHWPARAMS6_EN_FPGA			BIT(7)
 
+/* DWC_usb32 only */
+#define DWC3_GHWPARAMS6_MDWIDTH(n)		((n) & (0x3 << 8))
+
 /* Global HWPARAMS7 Register */
 #define DWC3_GHWPARAMS7_RAM1_DEPTH(n)	((n) & 0xffff)
 #define DWC3_GHWPARAMS7_RAM2_DEPTH(n)	(((n) >> 16) & 0xffff)
@@ -491,6 +495,7 @@
 #define DWC3_DGCMD_SELECTED_FIFO_FLUSH	0x09
 #define DWC3_DGCMD_ALL_FIFO_FLUSH	0x0a
 #define DWC3_DGCMD_SET_ENDPOINT_NRDY	0x0c
+#define DWC3_DGCMD_SET_ENDPOINT_PRIME	0x0d
 #define DWC3_DGCMD_RUN_SOC_BUS_LOOPBACK	0x10
 
 #define DWC3_DGCMD_STATUS(n)		(((n) >> 12) & 0x0F)
@@ -697,6 +702,10 @@ struct dwc3_ep {
 #define DWC3_EP_END_TRANSFER_PENDING BIT(4)
 #define DWC3_EP_PENDING_REQUEST	BIT(5)
 #define DWC3_EP_DELAY_START	BIT(6)
+#define DWC3_EP_WAIT_TRANSFER_COMPLETE	BIT(7)
+#define DWC3_EP_IGNORE_NEXT_NOSTREAM	BIT(8)
+#define DWC3_EP_FORCE_RESTART_STREAM	BIT(9)
+#define DWC3_EP_FIRST_STREAM_PRIMED	BIT(10)
 
 	/* This last one is specific to EP0 */
 #define DWC3_EP0_DIR_IN		BIT(31)
@@ -949,7 +958,8 @@ struct dwc3_scratchpad_array {
  * @nr_scratch: number of scratch buffers
  * @u1u2: only used on revisions <1.83a for workaround
  * @maximum_speed: maximum speed requested (mainly for testing purposes)
- * @revision: revision register contents
+ * @ip: controller's ID
+ * @revision: controller's version of an IP
  * @version_type: VERSIONTYPE register contents, a sub release of a revision
  * @dr_mode: requested mode of operation
  * @current_dr_role: current role of operation when in dual-role mode
@@ -1110,15 +1120,15 @@ struct dwc3 {
 	u32			u1u2;
 	u32			maximum_speed;
 
-	/*
-	 * All 3.1 IP version constants are greater than the 3.0 IP
-	 * version constants. This works for most version checks in
-	 * dwc3. However, in the future, this may not apply as
-	 * features may be developed on newer versions of the 3.0 IP
-	 * that are not in the 3.1 IP.
-	 */
+	u32			ip;
+
+#define DWC3_IP			0x5533
+#define DWC31_IP		0x3331
+#define DWC32_IP		0x3332
+
 	u32			revision;
 
+#define DWC3_REVISION_ANY	0x0
 #define DWC3_REVISION_173A	0x5533173a
 #define DWC3_REVISION_175A	0x5533175a
 #define DWC3_REVISION_180A	0x5533180a
@@ -1143,20 +1153,20 @@ struct dwc3 {
 #define DWC3_REVISION_310A	0x5533310a
 #define DWC3_REVISION_330A	0x5533330a
 
-/*
- * NOTICE: we're using bit 31 as a "is usb 3.1" flag. This is really
- * just so dwc31 revisions are always larger than dwc3.
- */
-#define DWC3_REVISION_IS_DWC31		0x80000000
-#define DWC3_USB31_REVISION_110A	(0x3131302a | DWC3_REVISION_IS_DWC31)
-#define DWC3_USB31_REVISION_120A	(0x3132302a | DWC3_REVISION_IS_DWC31)
-#define DWC3_USB31_REVISION_160A	(0x3136302a | DWC3_REVISION_IS_DWC31)
-#define DWC3_USB31_REVISION_170A	(0x3137302a | DWC3_REVISION_IS_DWC31)
-#define DWC3_USB31_REVISION_180A	(0x3138302a | DWC3_REVISION_IS_DWC31)
-#define DWC3_USB31_REVISION_190A	(0x3139302a | DWC3_REVISION_IS_DWC31)
+#define DWC31_REVISION_ANY	0x0
+#define DWC31_REVISION_110A	0x3131302a
+#define DWC31_REVISION_120A	0x3132302a
+#define DWC31_REVISION_160A	0x3136302a
+#define DWC31_REVISION_170A	0x3137302a
+#define DWC31_REVISION_180A	0x3138302a
+#define DWC31_REVISION_190A	0x3139302a
+
+#define DWC32_REVISION_ANY	0x0
+#define DWC32_REVISION_100A	0x3130302a
 
 	u32			version_type;
 
+#define DWC31_VERSIONTYPE_ANY		0x0
 #define DWC31_VERSIONTYPE_EA01		0x65613031
 #define DWC31_VERSIONTYPE_EA02		0x65613032
 #define DWC31_VERSIONTYPE_EA03		0x65613033
@@ -1298,6 +1308,10 @@ struct dwc3_event_depevt {
 #define DEPEVT_STREAMEVT_FOUND		1
 #define DEPEVT_STREAMEVT_NOTFOUND	2
 
+/* Stream event parameter */
+#define DEPEVT_STREAM_PRIME		0xfffe
+#define DEPEVT_STREAM_NOSTREAM		0x0
+
 /* Control-only Status */
 #define DEPEVT_STATUS_CONTROL_DATA	1
 #define DEPEVT_STATUS_CONTROL_STATUS	2
@@ -1400,17 +1414,26 @@ void dwc3_set_prtcap(struct dwc3 *dwc, u32 mode);
 void dwc3_set_mode(struct dwc3 *dwc, u32 mode);
 u32 dwc3_core_fifo_space(struct dwc3_ep *dep, u8 type);
 
-/* check whether we are on the DWC_usb3 core */
-static inline bool dwc3_is_usb3(struct dwc3 *dwc)
-{
-	return !(dwc->revision & DWC3_REVISION_IS_DWC31);
-}
+#define DWC3_IP_IS(_ip)							\
+	(dwc->ip == _ip##_IP)
 
-/* check whether we are on the DWC_usb31 core */
-static inline bool dwc3_is_usb31(struct dwc3 *dwc)
-{
-	return !!(dwc->revision & DWC3_REVISION_IS_DWC31);
-}
+#define DWC3_VER_IS(_ip, _ver)						\
+	(DWC3_IP_IS(_ip) && dwc->revision == _ip##_REVISION_##_ver)
+
+#define DWC3_VER_IS_PRIOR(_ip, _ver)					\
+	(DWC3_IP_IS(_ip) && dwc->revision < _ip##_REVISION_##_ver)
+
+#define DWC3_VER_IS_WITHIN(_ip, _from, _to)				\
+	(DWC3_IP_IS(_ip) &&						\
+	 dwc->revision >= _ip##_REVISION_##_from &&			\
+	 (!(_ip##_REVISION_##_to) ||					\
+	  dwc->revision <= _ip##_REVISION_##_to))
+
+#define DWC3_VER_TYPE_IS_WITHIN(_ip, _ver, _from, _to)			\
+	(DWC3_VER_IS(_ip, _ver) &&					\
+	 dwc->version_type >= _ip##_VERSIONTYPE_##_from &&		\
+	 (!(_ip##_VERSIONTYPE_##_to) ||					\
+	  dwc->version_type <= _ip##_VERSIONTYPE_##_to))
 
 bool dwc3_has_imod(struct dwc3 *dwc);
 
diff --git a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h
index 4a13ceaf4093..d8f600e0e88f 100644
--- a/drivers/usb/dwc3/debug.h
+++ b/drivers/usb/dwc3/debug.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
  * debug.h - DesignWare USB3 DRD Controller Debug Header
  *
@@ -68,6 +68,8 @@ dwc3_gadget_generic_cmd_string(u8 cmd)
 		return "All FIFO Flush";
 	case DWC3_DGCMD_SET_ENDPOINT_NRDY:
 		return "Set Endpoint NRDY";
+	case DWC3_DGCMD_SET_ENDPOINT_PRIME:
+		return "Set Endpoint Prime";
 	case DWC3_DGCMD_RUN_SOC_BUS_LOOPBACK:
 		return "Run SoC Bus Loopback Test";
 	default:
diff --git a/drivers/usb/dwc3/debugfs.c b/drivers/usb/dwc3/debugfs.c
index 4fe8b1e1485c..6d9de334e46a 100644
--- a/drivers/usb/dwc3/debugfs.c
+++ b/drivers/usb/dwc3/debugfs.c
@@ -635,13 +635,18 @@ static int dwc3_tx_fifo_size_show(struct seq_file *s, void *unused)
 	struct dwc3_ep		*dep = s->private;
 	struct dwc3		*dwc = dep->dwc;
 	unsigned long		flags;
+	int			mdwidth;
 	u32			val;
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	val = dwc3_core_fifo_space(dep, DWC3_TXFIFO);
 
 	/* Convert to bytes */
-	val *= DWC3_MDWIDTH(dwc->hwparams.hwparams0);
+	mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
+	if (DWC3_IP_IS(DWC32))
+		mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
+
+	val *= mdwidth;
 	val >>= 3;
 	seq_printf(s, "%u\n", val);
 	spin_unlock_irqrestore(&dwc->lock, flags);
@@ -654,13 +659,18 @@ static int dwc3_rx_fifo_size_show(struct seq_file *s, void *unused)
 	struct dwc3_ep		*dep = s->private;
 	struct dwc3		*dwc = dep->dwc;
 	unsigned long		flags;
+	int			mdwidth;
 	u32			val;
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	val = dwc3_core_fifo_space(dep, DWC3_RXFIFO);
 
 	/* Convert to bytes */
-	val *= DWC3_MDWIDTH(dwc->hwparams.hwparams0);
+	mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
+	if (DWC3_IP_IS(DWC32))
+		mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
+
+	val *= mdwidth;
 	val >>= 3;
 	seq_printf(s, "%u\n", val);
 	spin_unlock_irqrestore(&dwc->lock, flags);
diff --git a/drivers/usb/dwc3/drd.c b/drivers/usb/dwc3/drd.c
index 7db1ffc92bbd..2e483448d695 100644
--- a/drivers/usb/dwc3/drd.c
+++ b/drivers/usb/dwc3/drd.c
@@ -56,7 +56,7 @@ static irqreturn_t dwc3_otg_thread_irq(int irq, void *_dwc)
 	spin_lock(&dwc->lock);
 	if (dwc->otg_restart_host) {
 		dwc3_otg_host_init(dwc);
-		dwc->otg_restart_host = 0;
+		dwc->otg_restart_host = false;
 	}
 
 	spin_unlock(&dwc->lock);
@@ -82,7 +82,7 @@ static irqreturn_t dwc3_otg_irq(int irq, void *_dwc)
 
 		if (dwc->current_otg_role == DWC3_OTG_ROLE_HOST &&
 		    !(reg & DWC3_OEVT_DEVICEMODE))
-			dwc->otg_restart_host = 1;
+			dwc->otg_restart_host = true;
 		dwc3_writel(dwc->regs, DWC3_OEVT, reg);
 		ret = IRQ_WAKE_THREAD;
 	}
@@ -653,6 +653,6 @@ void dwc3_drd_exit(struct dwc3 *dwc)
 		break;
 	}
 
-	if (!dwc->edev)
+	if (dwc->otg_irq)
 		free_irq(dwc->otg_irq, dwc);
 }
diff --git a/drivers/usb/dwc3/dwc3-keystone.c b/drivers/usb/dwc3/dwc3-keystone.c
index 1e14a6f4884b..6505f7bd69e2 100644
--- a/drivers/usb/dwc3/dwc3-keystone.c
+++ b/drivers/usb/dwc3/dwc3-keystone.c
@@ -14,6 +14,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
 #include <linux/of_platform.h>
+#include <linux/phy/phy.h>
 #include <linux/pm_runtime.h>
 
 /* USBSS register offsets */
@@ -34,6 +35,7 @@
 struct dwc3_keystone {
 	struct device			*dev;
 	void __iomem			*usbss;
+	struct phy			*usb3_phy;
 };
 
 static inline u32 kdwc3_readl(void __iomem *base, u32 offset)
@@ -95,8 +97,38 @@ static int kdwc3_probe(struct platform_device *pdev)
 	if (IS_ERR(kdwc->usbss))
 		return PTR_ERR(kdwc->usbss);
 
-	pm_runtime_enable(kdwc->dev);
+	/* PSC dependency on AM65 needs SERDES0 to be powered before USB0 */
+	kdwc->usb3_phy = devm_phy_optional_get(dev, "usb3-phy");
+	if (IS_ERR(kdwc->usb3_phy)) {
+		error = PTR_ERR(kdwc->usb3_phy);
+		if (error != -EPROBE_DEFER)
+			dev_err(dev, "couldn't get usb3 phy: %d\n", error);
+
+		return error;
+	}
+
+	phy_pm_runtime_get_sync(kdwc->usb3_phy);
 
+	error = phy_reset(kdwc->usb3_phy);
+	if (error < 0) {
+		dev_err(dev, "usb3 phy reset failed: %d\n", error);
+		return error;
+	}
+
+	error = phy_init(kdwc->usb3_phy);
+	if (error < 0) {
+		dev_err(dev, "usb3 phy init failed: %d\n", error);
+		return error;
+	}
+
+	error = phy_power_on(kdwc->usb3_phy);
+	if (error < 0) {
+		dev_err(dev, "usb3 phy power on failed: %d\n", error);
+		phy_exit(kdwc->usb3_phy);
+		return error;
+	}
+
+	pm_runtime_enable(kdwc->dev);
 	error = pm_runtime_get_sync(kdwc->dev);
 	if (error < 0) {
 		dev_err(kdwc->dev, "pm_runtime_get_sync failed, error %d\n",
@@ -138,6 +170,9 @@ err_core:
 err_irq:
 	pm_runtime_put_sync(kdwc->dev);
 	pm_runtime_disable(kdwc->dev);
+	phy_power_off(kdwc->usb3_phy);
+	phy_exit(kdwc->usb3_phy);
+	phy_pm_runtime_put_sync(kdwc->usb3_phy);
 
 	return error;
 }
@@ -163,6 +198,10 @@ static int kdwc3_remove(struct platform_device *pdev)
 	pm_runtime_put_sync(kdwc->dev);
 	pm_runtime_disable(kdwc->dev);
 
+	phy_power_off(kdwc->usb3_phy);
+	phy_exit(kdwc->usb3_phy);
+	phy_pm_runtime_put_sync(kdwc->usb3_phy);
+
 	platform_set_drvdata(pdev, NULL);
 
 	return 0;
diff --git a/drivers/usb/dwc3/dwc3-meson-g12a.c b/drivers/usb/dwc3/dwc3-meson-g12a.c
index b81d085bc534..1f7f4d88ed9d 100644
--- a/drivers/usb/dwc3/dwc3-meson-g12a.c
+++ b/drivers/usb/dwc3/dwc3-meson-g12a.c
@@ -30,7 +30,7 @@
 #include <linux/usb/role.h>
 #include <linux/regulator/consumer.h>
 
-/* USB2 Ports Control Registers */
+/* USB2 Ports Control Registers, offsets are per-port */
 
 #define U2P_REG_SIZE						0x20
 
@@ -50,14 +50,16 @@
 
 /* USB Glue Control Registers */
 
-#define USB_R0							0x80
+#define G12A_GLUE_OFFSET					0x80
+
+#define USB_R0							0x00
 	#define USB_R0_P30_LANE0_TX2RX_LOOPBACK			BIT(17)
 	#define USB_R0_P30_LANE0_EXT_PCLK_REQ			BIT(18)
 	#define USB_R0_P30_PCS_RX_LOS_MASK_VAL_MASK		GENMASK(28, 19)
 	#define USB_R0_U2D_SS_SCALEDOWN_MODE_MASK		GENMASK(30, 29)
 	#define USB_R0_U2D_ACT					BIT(31)
 
-#define USB_R1							0x84
+#define USB_R1							0x04
 	#define USB_R1_U3H_BIGENDIAN_GS				BIT(0)
 	#define USB_R1_U3H_PME_ENABLE				BIT(1)
 	#define USB_R1_U3H_HUB_PORT_OVERCURRENT_MASK		GENMASK(4, 2)
@@ -69,23 +71,23 @@
 	#define USB_R1_U3H_FLADJ_30MHZ_REG_MASK			GENMASK(24, 19)
 	#define USB_R1_P30_PCS_TX_SWING_FULL_MASK		GENMASK(31, 25)
 
-#define USB_R2							0x88
+#define USB_R2							0x08
 	#define USB_R2_P30_PCS_TX_DEEMPH_3P5DB_MASK		GENMASK(25, 20)
 	#define USB_R2_P30_PCS_TX_DEEMPH_6DB_MASK		GENMASK(31, 26)
 
-#define USB_R3							0x8c
+#define USB_R3							0x0c
 	#define USB_R3_P30_SSC_ENABLE				BIT(0)
 	#define USB_R3_P30_SSC_RANGE_MASK			GENMASK(3, 1)
 	#define USB_R3_P30_SSC_REF_CLK_SEL_MASK			GENMASK(12, 4)
 	#define USB_R3_P30_REF_SSP_EN				BIT(13)
 
-#define USB_R4							0x90
+#define USB_R4							0x10
 	#define USB_R4_P21_PORT_RESET_0				BIT(0)
 	#define USB_R4_P21_SLEEP_M0				BIT(1)
 	#define USB_R4_MEM_PD_MASK				GENMASK(3, 2)
 	#define USB_R4_P21_ONLY					BIT(4)
 
-#define USB_R5							0x94
+#define USB_R5							0x14
 	#define USB_R5_ID_DIG_SYNC				BIT(0)
 	#define USB_R5_ID_DIG_REG				BIT(1)
 	#define USB_R5_ID_DIG_CFG_MASK				GENMASK(3, 2)
@@ -96,15 +98,12 @@
 	#define USB_R5_ID_DIG_TH_MASK				GENMASK(15, 8)
 	#define USB_R5_ID_DIG_CNT_MASK				GENMASK(23, 16)
 
-enum {
-	USB2_HOST_PHY = 0,
-	USB2_OTG_PHY,
-	USB3_HOST_PHY,
-	PHY_COUNT,
-};
+#define PHY_COUNT						3
+#define USB2_OTG_PHY						1
 
-static const char *phy_names[PHY_COUNT] = {
-	"usb2-phy0", "usb2-phy1", "usb3-phy0",
+static struct clk_bulk_data meson_gxl_clocks[] = {
+	{ .id = "usb_ctrl" },
+	{ .id = "ddr" },
 };
 
 static struct clk_bulk_data meson_g12a_clocks[] = {
@@ -117,27 +116,133 @@ static struct clk_bulk_data meson_a1_clocks[] = {
 	{ .id = "xtal_usb_ctrl" },
 };
 
+static const char *meson_gxm_phy_names[] = {
+	"usb2-phy0", "usb2-phy1", "usb2-phy2",
+};
+
+static const char *meson_g12a_phy_names[] = {
+	"usb2-phy0", "usb2-phy1", "usb3-phy0",
+};
+
+/*
+ * Amlogic A1 has a single physical PHY, in slot 1, but still has the
+ * two U2 PHY controls register blocks like G12A.
+ * Handling the first PHY on slot 1 would need a large amount of code
+ * changes, and the current management is generic enough to handle it
+ * correctly when only the "usb2-phy1" phy is specified on-par with the
+ * DT bindings.
+ */
+static const char *meson_a1_phy_names[] = {
+	"usb2-phy0", "usb2-phy1"
+};
+
+struct dwc3_meson_g12a;
+
 struct dwc3_meson_g12a_drvdata {
 	bool otg_switch_supported;
+	bool otg_phy_host_port_disable;
 	struct clk_bulk_data *clks;
 	int num_clks;
+	const char **phy_names;
+	int num_phys;
+	int (*setup_regmaps)(struct dwc3_meson_g12a *priv, void __iomem *base);
+	int (*usb2_init_phy)(struct dwc3_meson_g12a *priv, int i,
+			     enum phy_mode mode);
+	int (*set_phy_mode)(struct dwc3_meson_g12a *priv, int i,
+			    enum phy_mode mode);
+	int (*usb_init)(struct dwc3_meson_g12a *priv);
+	int (*usb_post_init)(struct dwc3_meson_g12a *priv);
+};
+
+static int dwc3_meson_gxl_setup_regmaps(struct dwc3_meson_g12a *priv,
+					void __iomem *base);
+static int dwc3_meson_g12a_setup_regmaps(struct dwc3_meson_g12a *priv,
+					 void __iomem *base);
+
+static int dwc3_meson_g12a_usb2_init_phy(struct dwc3_meson_g12a *priv, int i,
+					 enum phy_mode mode);
+static int dwc3_meson_gxl_usb2_init_phy(struct dwc3_meson_g12a *priv, int i,
+					enum phy_mode mode);
+
+static int dwc3_meson_g12a_set_phy_mode(struct dwc3_meson_g12a *priv,
+					int i, enum phy_mode mode);
+static int dwc3_meson_gxl_set_phy_mode(struct dwc3_meson_g12a *priv,
+				       int i, enum phy_mode mode);
+
+static int dwc3_meson_g12a_usb_init(struct dwc3_meson_g12a *priv);
+static int dwc3_meson_gxl_usb_init(struct dwc3_meson_g12a *priv);
+
+static int dwc3_meson_gxl_usb_post_init(struct dwc3_meson_g12a *priv);
+
+/*
+ * For GXL and GXM SoCs:
+ * USB Phy muxing between the DWC2 Device controller and the DWC3 Host
+ * controller is buggy when switching from Device to Host when USB port
+ * is unpopulated, it causes the DWC3 to hard crash.
+ * When populated (including OTG switching with ID pin), the switch works
+ * like a charm like on the G12A platforms.
+ * In order to still switch from Host to Device on an USB Type-A port,
+ * an U2_PORT_DISABLE bit has been added to disconnect the DWC3 Host
+ * controller from the port, but when used the DWC3 controller must be
+ * reset to recover usage of the port.
+ */
+
+static struct dwc3_meson_g12a_drvdata gxl_drvdata = {
+	.otg_switch_supported = true,
+	.otg_phy_host_port_disable = true,
+	.clks = meson_gxl_clocks,
+	.num_clks = ARRAY_SIZE(meson_g12a_clocks),
+	.phy_names = meson_a1_phy_names,
+	.num_phys = ARRAY_SIZE(meson_a1_phy_names),
+	.setup_regmaps = dwc3_meson_gxl_setup_regmaps,
+	.usb2_init_phy = dwc3_meson_gxl_usb2_init_phy,
+	.set_phy_mode = dwc3_meson_gxl_set_phy_mode,
+	.usb_init = dwc3_meson_gxl_usb_init,
+	.usb_post_init = dwc3_meson_gxl_usb_post_init,
+};
+
+static struct dwc3_meson_g12a_drvdata gxm_drvdata = {
+	.otg_switch_supported = true,
+	.otg_phy_host_port_disable = true,
+	.clks = meson_gxl_clocks,
+	.num_clks = ARRAY_SIZE(meson_g12a_clocks),
+	.phy_names = meson_gxm_phy_names,
+	.num_phys = ARRAY_SIZE(meson_gxm_phy_names),
+	.setup_regmaps = dwc3_meson_gxl_setup_regmaps,
+	.usb2_init_phy = dwc3_meson_gxl_usb2_init_phy,
+	.set_phy_mode = dwc3_meson_gxl_set_phy_mode,
+	.usb_init = dwc3_meson_gxl_usb_init,
+	.usb_post_init = dwc3_meson_gxl_usb_post_init,
 };
 
 static struct dwc3_meson_g12a_drvdata g12a_drvdata = {
 	.otg_switch_supported = true,
 	.clks = meson_g12a_clocks,
 	.num_clks = ARRAY_SIZE(meson_g12a_clocks),
+	.phy_names = meson_g12a_phy_names,
+	.num_phys = ARRAY_SIZE(meson_g12a_phy_names),
+	.setup_regmaps = dwc3_meson_g12a_setup_regmaps,
+	.usb2_init_phy = dwc3_meson_g12a_usb2_init_phy,
+	.set_phy_mode = dwc3_meson_g12a_set_phy_mode,
+	.usb_init = dwc3_meson_g12a_usb_init,
 };
 
 static struct dwc3_meson_g12a_drvdata a1_drvdata = {
 	.otg_switch_supported = false,
 	.clks = meson_a1_clocks,
 	.num_clks = ARRAY_SIZE(meson_a1_clocks),
+	.phy_names = meson_a1_phy_names,
+	.num_phys = ARRAY_SIZE(meson_a1_phy_names),
+	.setup_regmaps = dwc3_meson_g12a_setup_regmaps,
+	.usb2_init_phy = dwc3_meson_g12a_usb2_init_phy,
+	.set_phy_mode = dwc3_meson_g12a_set_phy_mode,
+	.usb_init = dwc3_meson_g12a_usb_init,
 };
 
 struct dwc3_meson_g12a {
 	struct device		*dev;
-	struct regmap		*regmap;
+	struct regmap		*u2p_regmap[PHY_COUNT];
+	struct regmap		*usb_glue_regmap;
 	struct reset_control	*reset;
 	struct phy		*phys[PHY_COUNT];
 	enum usb_dr_mode	otg_mode;
@@ -150,49 +255,78 @@ struct dwc3_meson_g12a {
 	const struct dwc3_meson_g12a_drvdata *drvdata;
 };
 
-static void dwc3_meson_g12a_usb2_set_mode(struct dwc3_meson_g12a *priv,
-					  int i, enum phy_mode mode)
+static int dwc3_meson_gxl_set_phy_mode(struct dwc3_meson_g12a *priv,
+					 int i, enum phy_mode mode)
+{
+	return phy_set_mode(priv->phys[i], mode);
+}
+
+static int dwc3_meson_gxl_usb2_init_phy(struct dwc3_meson_g12a *priv, int i,
+					enum phy_mode mode)
+{
+	/* On GXL PHY must be started in device mode for DWC2 init */
+	return priv->drvdata->set_phy_mode(priv, i,
+				(i == USB2_OTG_PHY) ? PHY_MODE_USB_DEVICE
+						    : PHY_MODE_USB_HOST);
+}
+
+static int dwc3_meson_g12a_set_phy_mode(struct dwc3_meson_g12a *priv,
+					 int i, enum phy_mode mode)
 {
 	if (mode == PHY_MODE_USB_HOST)
-		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
+		regmap_update_bits(priv->u2p_regmap[i], U2P_R0,
 				U2P_R0_HOST_DEVICE,
 				U2P_R0_HOST_DEVICE);
 	else
-		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
+		regmap_update_bits(priv->u2p_regmap[i], U2P_R0,
 				U2P_R0_HOST_DEVICE, 0);
+
+	return 0;
 }
 
-static int dwc3_meson_g12a_usb2_init(struct dwc3_meson_g12a *priv)
+static int dwc3_meson_g12a_usb2_init_phy(struct dwc3_meson_g12a *priv, int i,
+					 enum phy_mode mode)
 {
-	int i;
+	int ret;
 
-	if (priv->otg_mode == USB_DR_MODE_PERIPHERAL)
-		priv->otg_phy_mode = PHY_MODE_USB_DEVICE;
-	else
-		priv->otg_phy_mode = PHY_MODE_USB_HOST;
+	regmap_update_bits(priv->u2p_regmap[i], U2P_R0,
+			U2P_R0_POWER_ON_RESET,
+			U2P_R0_POWER_ON_RESET);
 
-	for (i = 0 ; i < USB3_HOST_PHY ; ++i) {
-		if (!priv->phys[i])
-			continue;
+	if (priv->drvdata->otg_switch_supported && i == USB2_OTG_PHY) {
+		regmap_update_bits(priv->u2p_regmap[i], U2P_R0,
+				   U2P_R0_ID_PULLUP | U2P_R0_DRV_VBUS,
+				   U2P_R0_ID_PULLUP | U2P_R0_DRV_VBUS);
 
-		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
-				   U2P_R0_POWER_ON_RESET,
-				   U2P_R0_POWER_ON_RESET);
+		ret = priv->drvdata->set_phy_mode(priv, i, mode);
+	} else
+		ret = priv->drvdata->set_phy_mode(priv, i,
+						  PHY_MODE_USB_HOST);
 
-		if (priv->drvdata->otg_switch_supported && i == USB2_OTG_PHY) {
-			regmap_update_bits(priv->regmap,
-				U2P_R0 + (U2P_REG_SIZE * i),
-				U2P_R0_ID_PULLUP | U2P_R0_DRV_VBUS,
-				U2P_R0_ID_PULLUP | U2P_R0_DRV_VBUS);
+	if (ret)
+		return ret;
+
+	regmap_update_bits(priv->u2p_regmap[i], U2P_R0,
+			U2P_R0_POWER_ON_RESET, 0);
+
+	return 0;
+}
 
-			dwc3_meson_g12a_usb2_set_mode(priv, i,
-						      priv->otg_phy_mode);
-		} else
-			dwc3_meson_g12a_usb2_set_mode(priv, i,
-						      PHY_MODE_USB_HOST);
+static int dwc3_meson_g12a_usb2_init(struct dwc3_meson_g12a *priv,
+				     enum phy_mode mode)
+{
+	int i, ret;
 
-		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
-				   U2P_R0_POWER_ON_RESET, 0);
+	for (i = 0; i < priv->drvdata->num_phys; ++i) {
+		if (!priv->phys[i])
+			continue;
+
+		if (!strstr(priv->drvdata->phy_names[i], "usb2"))
+			continue;
+
+		ret = priv->drvdata->usb2_init_phy(priv, i, mode);
+		if (ret)
+			return ret;
 	}
 
 	return 0;
@@ -200,7 +334,7 @@ static int dwc3_meson_g12a_usb2_init(struct dwc3_meson_g12a *priv)
 
 static void dwc3_meson_g12a_usb3_init(struct dwc3_meson_g12a *priv)
 {
-	regmap_update_bits(priv->regmap, USB_R3,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R3,
 			USB_R3_P30_SSC_RANGE_MASK |
 			USB_R3_P30_REF_SSP_EN,
 			USB_R3_P30_SSC_ENABLE |
@@ -208,61 +342,77 @@ static void dwc3_meson_g12a_usb3_init(struct dwc3_meson_g12a *priv)
 			USB_R3_P30_REF_SSP_EN);
 	udelay(2);
 
-	regmap_update_bits(priv->regmap, USB_R2,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R2,
 			USB_R2_P30_PCS_TX_DEEMPH_3P5DB_MASK,
 			FIELD_PREP(USB_R2_P30_PCS_TX_DEEMPH_3P5DB_MASK, 0x15));
 
-	regmap_update_bits(priv->regmap, USB_R2,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R2,
 			USB_R2_P30_PCS_TX_DEEMPH_6DB_MASK,
 			FIELD_PREP(USB_R2_P30_PCS_TX_DEEMPH_6DB_MASK, 0x20));
 
 	udelay(2);
 
-	regmap_update_bits(priv->regmap, USB_R1,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R1,
 			USB_R1_U3H_HOST_PORT_POWER_CONTROL_PRESENT,
 			USB_R1_U3H_HOST_PORT_POWER_CONTROL_PRESENT);
 
-	regmap_update_bits(priv->regmap, USB_R1,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R1,
 			USB_R1_P30_PCS_TX_SWING_FULL_MASK,
 			FIELD_PREP(USB_R1_P30_PCS_TX_SWING_FULL_MASK, 127));
 }
 
-static void dwc3_meson_g12a_usb_otg_apply_mode(struct dwc3_meson_g12a *priv)
+static void dwc3_meson_g12a_usb_otg_apply_mode(struct dwc3_meson_g12a *priv,
+					       enum phy_mode mode)
 {
-	if (priv->otg_phy_mode == PHY_MODE_USB_DEVICE) {
-		regmap_update_bits(priv->regmap, USB_R0,
+	if (mode == PHY_MODE_USB_DEVICE) {
+		if (priv->otg_mode != USB_DR_MODE_OTG &&
+		    priv->drvdata->otg_phy_host_port_disable)
+			/* Isolate the OTG PHY port from the Host Controller */
+			regmap_update_bits(priv->usb_glue_regmap, USB_R1,
+				USB_R1_U3H_HOST_U2_PORT_DISABLE_MASK,
+				FIELD_PREP(USB_R1_U3H_HOST_U2_PORT_DISABLE_MASK,
+					   BIT(USB2_OTG_PHY)));
+
+		regmap_update_bits(priv->usb_glue_regmap, USB_R0,
 				USB_R0_U2D_ACT, USB_R0_U2D_ACT);
-		regmap_update_bits(priv->regmap, USB_R0,
+		regmap_update_bits(priv->usb_glue_regmap, USB_R0,
 				USB_R0_U2D_SS_SCALEDOWN_MODE_MASK, 0);
-		regmap_update_bits(priv->regmap, USB_R4,
+		regmap_update_bits(priv->usb_glue_regmap, USB_R4,
 				USB_R4_P21_SLEEP_M0, USB_R4_P21_SLEEP_M0);
 	} else {
-		regmap_update_bits(priv->regmap, USB_R0,
+		if (priv->otg_mode != USB_DR_MODE_OTG &&
+		    priv->drvdata->otg_phy_host_port_disable) {
+			regmap_update_bits(priv->usb_glue_regmap, USB_R1,
+				USB_R1_U3H_HOST_U2_PORT_DISABLE_MASK, 0);
+			msleep(500);
+		}
+		regmap_update_bits(priv->usb_glue_regmap, USB_R0,
 				USB_R0_U2D_ACT, 0);
-		regmap_update_bits(priv->regmap, USB_R4,
+		regmap_update_bits(priv->usb_glue_regmap, USB_R4,
 				USB_R4_P21_SLEEP_M0, 0);
 	}
 }
 
-static int dwc3_meson_g12a_usb_init(struct dwc3_meson_g12a *priv)
+static int dwc3_meson_g12a_usb_init_glue(struct dwc3_meson_g12a *priv,
+					 enum phy_mode mode)
 {
 	int ret;
 
-	ret = dwc3_meson_g12a_usb2_init(priv);
+	ret = dwc3_meson_g12a_usb2_init(priv, mode);
 	if (ret)
 		return ret;
 
-	regmap_update_bits(priv->regmap, USB_R1,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R1,
 			USB_R1_U3H_FLADJ_30MHZ_REG_MASK,
 			FIELD_PREP(USB_R1_U3H_FLADJ_30MHZ_REG_MASK, 0x20));
 
-	regmap_update_bits(priv->regmap, USB_R5,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R5,
 			USB_R5_ID_DIG_EN_0,
 			USB_R5_ID_DIG_EN_0);
-	regmap_update_bits(priv->regmap, USB_R5,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R5,
 			USB_R5_ID_DIG_EN_1,
 			USB_R5_ID_DIG_EN_1);
-	regmap_update_bits(priv->regmap, USB_R5,
+	regmap_update_bits(priv->usb_glue_regmap, USB_R5,
 			USB_R5_ID_DIG_TH_MASK,
 			FIELD_PREP(USB_R5_ID_DIG_TH_MASK, 0xff));
 
@@ -270,12 +420,13 @@ static int dwc3_meson_g12a_usb_init(struct dwc3_meson_g12a *priv)
 	if (priv->usb3_ports)
 		dwc3_meson_g12a_usb3_init(priv);
 
-	dwc3_meson_g12a_usb_otg_apply_mode(priv);
+	dwc3_meson_g12a_usb_otg_apply_mode(priv, mode);
 
 	return 0;
 }
 
-static const struct regmap_config phy_meson_g12a_usb3_regmap_conf = {
+static const struct regmap_config phy_meson_g12a_usb_glue_regmap_conf = {
+	.name = "usb-glue",
 	.reg_bits = 8,
 	.val_bits = 32,
 	.reg_stride = 4,
@@ -284,17 +435,19 @@ static const struct regmap_config phy_meson_g12a_usb3_regmap_conf = {
 
 static int dwc3_meson_g12a_get_phys(struct dwc3_meson_g12a *priv)
 {
+	const char *phy_name;
 	int i;
 
-	for (i = 0 ; i < PHY_COUNT ; ++i) {
-		priv->phys[i] = devm_phy_optional_get(priv->dev, phy_names[i]);
+	for (i = 0 ; i < priv->drvdata->num_phys ; ++i) {
+		phy_name = priv->drvdata->phy_names[i];
+		priv->phys[i] = devm_phy_optional_get(priv->dev, phy_name);
 		if (!priv->phys[i])
 			continue;
 
 		if (IS_ERR(priv->phys[i]))
 			return PTR_ERR(priv->phys[i]);
 
-		if (i == USB3_HOST_PHY)
+		if (strstr(phy_name, "usb3"))
 			priv->usb3_ports++;
 		else
 			priv->usb2_ports++;
@@ -310,7 +463,7 @@ static enum phy_mode dwc3_meson_g12a_get_id(struct dwc3_meson_g12a *priv)
 {
 	u32 reg;
 
-	regmap_read(priv->regmap, USB_R5, &reg);
+	regmap_read(priv->usb_glue_regmap, USB_R5, &reg);
 
 	if (reg & (USB_R5_ID_DIG_SYNC | USB_R5_ID_DIG_REG))
 		return PHY_MODE_USB_DEVICE;
@@ -342,9 +495,11 @@ static int dwc3_meson_g12a_otg_mode_set(struct dwc3_meson_g12a *priv,
 
 	priv->otg_phy_mode = mode;
 
-	dwc3_meson_g12a_usb2_set_mode(priv, USB2_OTG_PHY, mode);
+	ret = priv->drvdata->set_phy_mode(priv, USB2_OTG_PHY, mode);
+	if (ret)
+		return ret;
 
-	dwc3_meson_g12a_usb_otg_apply_mode(priv);
+	dwc3_meson_g12a_usb_otg_apply_mode(priv, mode);
 
 	return 0;
 }
@@ -364,6 +519,13 @@ static int dwc3_meson_g12a_role_set(struct usb_role_switch *sw,
 	if (mode == priv->otg_phy_mode)
 		return 0;
 
+	if (priv->drvdata->otg_phy_host_port_disable)
+		dev_warn_once(priv->dev, "Manual OTG switch is broken on this "\
+					 "SoC, when manual switching from "\
+					 "Host to device, DWC3 controller "\
+					 "will need to be resetted in order "\
+					 "to recover usage of the Host port");
+
 	return dwc3_meson_g12a_otg_mode_set(priv, mode);
 }
 
@@ -386,7 +548,8 @@ static irqreturn_t dwc3_meson_g12a_irq_thread(int irq, void *data)
 			dev_warn(priv->dev, "Failed to switch OTG mode\n");
 	}
 
-	regmap_update_bits(priv->regmap, USB_R5, USB_R5_ID_DIG_IRQ, 0);
+	regmap_update_bits(priv->usb_glue_regmap, USB_R5,
+			   USB_R5_ID_DIG_IRQ, 0);
 
 	return IRQ_HANDLED;
 }
@@ -421,7 +584,7 @@ static int dwc3_meson_g12a_otg_init(struct platform_device *pdev,
 
 	if (priv->otg_mode == USB_DR_MODE_OTG) {
 		/* Ack irq before registering */
-		regmap_update_bits(priv->regmap, USB_R5,
+		regmap_update_bits(priv->usb_glue_regmap, USB_R5,
 				   USB_R5_ID_DIG_IRQ, 0);
 
 		irq = platform_get_irq(pdev, 0);
@@ -457,6 +620,77 @@ static int dwc3_meson_g12a_otg_init(struct platform_device *pdev,
 	return 0;
 }
 
+static int dwc3_meson_gxl_setup_regmaps(struct dwc3_meson_g12a *priv,
+					void __iomem *base)
+{
+	/* GXL controls the PHY mode in the PHY registers unlike G12A */
+	priv->usb_glue_regmap = devm_regmap_init_mmio(priv->dev, base,
+					&phy_meson_g12a_usb_glue_regmap_conf);
+	if (IS_ERR(priv->usb_glue_regmap))
+		return PTR_ERR(priv->usb_glue_regmap);
+
+	return 0;
+}
+
+static int dwc3_meson_g12a_setup_regmaps(struct dwc3_meson_g12a *priv,
+					 void __iomem *base)
+{
+	int i;
+
+	priv->usb_glue_regmap = devm_regmap_init_mmio(priv->dev,
+					base + G12A_GLUE_OFFSET,
+					&phy_meson_g12a_usb_glue_regmap_conf);
+	if (IS_ERR(priv->usb_glue_regmap))
+		return PTR_ERR(priv->usb_glue_regmap);
+
+	/* Create a regmap for each USB2 PHY control register set */
+	for (i = 0; i < priv->usb2_ports; i++) {
+		struct regmap_config u2p_regmap_config = {
+			.reg_bits = 8,
+			.val_bits = 32,
+			.reg_stride = 4,
+			.max_register = U2P_R1,
+		};
+
+		u2p_regmap_config.name = devm_kasprintf(priv->dev, GFP_KERNEL,
+							"u2p-%d", i);
+		if (!u2p_regmap_config.name)
+			return -ENOMEM;
+
+		priv->u2p_regmap[i] = devm_regmap_init_mmio(priv->dev,
+						base + (i * U2P_REG_SIZE),
+						&u2p_regmap_config);
+		if (IS_ERR(priv->u2p_regmap[i]))
+			return PTR_ERR(priv->u2p_regmap[i]);
+	}
+
+	return 0;
+}
+
+static int dwc3_meson_g12a_usb_init(struct dwc3_meson_g12a *priv)
+{
+	return dwc3_meson_g12a_usb_init_glue(priv, priv->otg_phy_mode);
+}
+
+static int dwc3_meson_gxl_usb_init(struct dwc3_meson_g12a *priv)
+{
+	return dwc3_meson_g12a_usb_init_glue(priv, PHY_MODE_USB_DEVICE);
+}
+
+static int dwc3_meson_gxl_usb_post_init(struct dwc3_meson_g12a *priv)
+{
+	int ret;
+
+	ret = priv->drvdata->set_phy_mode(priv, USB2_OTG_PHY,
+					  priv->otg_phy_mode);
+	if (ret)
+		return ret;
+
+	dwc3_meson_g12a_usb_otg_apply_mode(priv, priv->otg_phy_mode);
+
+	return 0;
+}
+
 static int dwc3_meson_g12a_probe(struct platform_device *pdev)
 {
 	struct dwc3_meson_g12a	*priv;
@@ -473,10 +707,8 @@ static int dwc3_meson_g12a_probe(struct platform_device *pdev)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	priv->regmap = devm_regmap_init_mmio(dev, base,
-					     &phy_meson_g12a_usb3_regmap_conf);
-	if (IS_ERR(priv->regmap))
-		return PTR_ERR(priv->regmap);
+	priv->drvdata = of_device_get_match_data(&pdev->dev);
+	priv->dev = dev;
 
 	priv->vbus = devm_regulator_get_optional(dev, "vbus");
 	if (IS_ERR(priv->vbus)) {
@@ -485,8 +717,6 @@ static int dwc3_meson_g12a_probe(struct platform_device *pdev)
 		priv->vbus = NULL;
 	}
 
-	priv->drvdata = of_device_get_match_data(&pdev->dev);
-
 	ret = devm_clk_bulk_get(dev,
 				priv->drvdata->num_clks,
 				priv->drvdata->clks);
@@ -499,13 +729,12 @@ static int dwc3_meson_g12a_probe(struct platform_device *pdev)
 		return ret;
 
 	platform_set_drvdata(pdev, priv);
-	priv->dev = dev;
 
-	priv->reset = devm_reset_control_get(dev, NULL);
+	priv->reset = devm_reset_control_get_shared(dev, NULL);
 	if (IS_ERR(priv->reset)) {
 		ret = PTR_ERR(priv->reset);
 		dev_err(dev, "failed to get device reset, err=%d\n", ret);
-		return ret;
+		goto err_disable_clks;
 	}
 
 	ret = reset_control_reset(priv->reset);
@@ -516,6 +745,10 @@ static int dwc3_meson_g12a_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_disable_clks;
 
+	ret = priv->drvdata->setup_regmaps(priv, base);
+	if (ret)
+		return ret;
+
 	if (priv->vbus) {
 		ret = regulator_enable(priv->vbus);
 		if (ret)
@@ -525,7 +758,14 @@ static int dwc3_meson_g12a_probe(struct platform_device *pdev)
 	/* Get dr_mode */
 	priv->otg_mode = usb_get_dr_mode(dev);
 
-	dwc3_meson_g12a_usb_init(priv);
+	if (priv->otg_mode == USB_DR_MODE_PERIPHERAL)
+		priv->otg_phy_mode = PHY_MODE_USB_DEVICE;
+	else
+		priv->otg_phy_mode = PHY_MODE_USB_HOST;
+
+	ret = priv->drvdata->usb_init(priv);
+	if (ret)
+		goto err_disable_clks;
 
 	/* Init PHYs */
 	for (i = 0 ; i < PHY_COUNT ; ++i) {
@@ -541,6 +781,12 @@ static int dwc3_meson_g12a_probe(struct platform_device *pdev)
 			goto err_phys_exit;
 	}
 
+	if (priv->drvdata->usb_post_init) {
+		ret = priv->drvdata->usb_post_init(priv);
+		if (ret)
+			goto err_phys_power;
+	}
+
 	ret = of_platform_populate(np, NULL, NULL, dev);
 	if (ret)
 		goto err_phys_power;
@@ -642,7 +888,9 @@ static int __maybe_unused dwc3_meson_g12a_resume(struct device *dev)
 
 	reset_control_deassert(priv->reset);
 
-	dwc3_meson_g12a_usb_init(priv);
+	ret = priv->drvdata->usb_init(priv);
+	if (ret)
+		return ret;
 
 	/* Init PHYs */
 	for (i = 0 ; i < PHY_COUNT ; ++i) {
@@ -675,6 +923,14 @@ static const struct dev_pm_ops dwc3_meson_g12a_dev_pm_ops = {
 
 static const struct of_device_id dwc3_meson_g12a_match[] = {
 	{
+		.compatible = "amlogic,meson-gxl-usb-ctrl",
+		.data = &gxl_drvdata,
+	},
+	{
+		.compatible = "amlogic,meson-gxm-usb-ctrl",
+		.data = &gxm_drvdata,
+	},
+	{
 		.compatible = "amlogic,meson-g12a-usb-ctrl",
 		.data = &g12a_drvdata,
 	},
diff --git a/drivers/usb/dwc3/dwc3-of-simple.c b/drivers/usb/dwc3/dwc3-of-simple.c
index e64754be47b4..8852fbfdead4 100644
--- a/drivers/usb/dwc3/dwc3-of-simple.c
+++ b/drivers/usb/dwc3/dwc3-of-simple.c
@@ -27,7 +27,6 @@ struct dwc3_of_simple {
 	struct clk_bulk_data	*clks;
 	int			num_clocks;
 	struct reset_control	*resets;
-	bool			pulse_resets;
 	bool			need_reset;
 };
 
@@ -38,7 +37,6 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 	struct device_node	*np = dev->of_node;
 
 	int			ret;
-	bool			shared_resets = false;
 
 	simple = devm_kzalloc(dev, sizeof(*simple), GFP_KERNEL);
 	if (!simple)
@@ -54,13 +52,7 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 	if (of_device_is_compatible(np, "rockchip,rk3399-dwc3"))
 		simple->need_reset = true;
 
-	if (of_device_is_compatible(np, "amlogic,meson-axg-dwc3") ||
-	    of_device_is_compatible(np, "amlogic,meson-gxl-dwc3")) {
-		shared_resets = true;
-		simple->pulse_resets = true;
-	}
-
-	simple->resets = of_reset_control_array_get(np, shared_resets, true,
+	simple->resets = of_reset_control_array_get(np, false, true,
 						    true);
 	if (IS_ERR(simple->resets)) {
 		ret = PTR_ERR(simple->resets);
@@ -68,15 +60,9 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	if (simple->pulse_resets) {
-		ret = reset_control_reset(simple->resets);
-		if (ret)
-			goto err_resetc_put;
-	} else {
-		ret = reset_control_deassert(simple->resets);
-		if (ret)
-			goto err_resetc_put;
-	}
+	ret = reset_control_deassert(simple->resets);
+	if (ret)
+		goto err_resetc_put;
 
 	ret = clk_bulk_get_all(simple->dev, &simple->clks);
 	if (ret < 0)
@@ -102,8 +88,7 @@ err_clk_put:
 	clk_bulk_put_all(simple->num_clocks, simple->clks);
 
 err_resetc_assert:
-	if (!simple->pulse_resets)
-		reset_control_assert(simple->resets);
+	reset_control_assert(simple->resets);
 
 err_resetc_put:
 	reset_control_put(simple->resets);
@@ -118,8 +103,7 @@ static void __dwc3_of_simple_teardown(struct dwc3_of_simple *simple)
 	clk_bulk_put_all(simple->num_clocks, simple->clks);
 	simple->num_clocks = 0;
 
-	if (!simple->pulse_resets)
-		reset_control_assert(simple->resets);
+	reset_control_assert(simple->resets);
 
 	reset_control_put(simple->resets);
 
@@ -191,8 +175,6 @@ static const struct of_device_id of_dwc3_simple_match[] = {
 	{ .compatible = "xlnx,zynqmp-dwc3" },
 	{ .compatible = "cavium,octeon-7130-usb-uctl" },
 	{ .compatible = "sprd,sc9860-dwc3" },
-	{ .compatible = "amlogic,meson-axg-dwc3" },
-	{ .compatible = "amlogic,meson-gxl-dwc3" },
 	{ .compatible = "allwinner,sun50i-h6-dwc3" },
 	{ /* Sentinel */ }
 };
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 585cb3deea7a..80c3ef134e41 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -95,7 +95,7 @@ int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
 	 * Wait until device controller is ready. Only applies to 1.94a and
 	 * later RTL.
 	 */
-	if (dwc->revision >= DWC3_REVISION_194A) {
+	if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
 		while (--retries) {
 			reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 			if (reg & DWC3_DSTS_DCNRD)
@@ -122,7 +122,7 @@ int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
 	 * The following code is racy when called from dwc3_gadget_wakeup,
 	 * and is not needed, at least on newer versions
 	 */
-	if (dwc->revision >= DWC3_REVISION_194A)
+	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
 		return 0;
 
 	/* wait for a change in DSTS */
@@ -273,7 +273,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 {
 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 	struct dwc3		*dwc = dep->dwc;
-	u32			timeout = 1000;
+	u32			timeout = 5000;
 	u32			saved_config = 0;
 	u32			reg;
 
@@ -356,6 +356,8 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 				ret = 0;
 				break;
 			case DEPEVT_TRANSFER_NO_RESOURCE:
+				dev_WARN(dwc->dev, "No resource for %s\n",
+					 dep->name);
 				ret = -EINVAL;
 				break;
 			case DEPEVT_TRANSFER_BUS_EXPIRY:
@@ -387,9 +389,12 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 
 	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 
-	if (ret == 0 && DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
-		dep->flags |= DWC3_EP_TRANSFER_STARTED;
-		dwc3_gadget_ep_get_transfer_index(dep);
+	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
+		if (ret == 0)
+			dep->flags |= DWC3_EP_TRANSFER_STARTED;
+
+		if (ret != -ETIMEDOUT)
+			dwc3_gadget_ep_get_transfer_index(dep);
 	}
 
 	if (saved_config) {
@@ -415,7 +420,8 @@ static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 	 * IN transfers due to a mishandled error condition. Synopsys
 	 * STAR 9000614252.
 	 */
-	if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
+	if (dep->direction &&
+	    !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
 	    (dwc->gadget.speed >= USB_SPEED_SUPER))
 		cmd |= DWC3_DEPCMD_CLEARPENDIN;
 
@@ -573,6 +579,7 @@ static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 
 	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 		params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
+			| DWC3_DEPCFG_XFER_COMPLETE_EN
 			| DWC3_DEPCFG_STREAM_EVENT_EN;
 		dep->stream_capable = true;
 	}
@@ -603,6 +610,9 @@ static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 }
 
+static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
+		bool interrupt);
+
 /**
  * __dwc3_gadget_ep_enable - initializes a hw endpoint
  * @dep: endpoint to be initialized
@@ -663,7 +673,7 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 	 * Issue StartTransfer here with no-op TRB so we can always rely on No
 	 * Response Update Transfer command.
 	 */
-	if ((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) ||
+	if (usb_endpoint_xfer_bulk(desc) ||
 			usb_endpoint_xfer_int(desc)) {
 		struct dwc3_gadget_ep_cmd_params params;
 		struct dwc3_trb	*trb;
@@ -682,6 +692,29 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 		ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 		if (ret < 0)
 			return ret;
+
+		if (dep->stream_capable) {
+			/*
+			 * For streams, at start, there maybe a race where the
+			 * host primes the endpoint before the function driver
+			 * queues a request to initiate a stream. In that case,
+			 * the controller will not see the prime to generate the
+			 * ERDY and start stream. To workaround this, issue a
+			 * no-op TRB as normal, but end it immediately. As a
+			 * result, when the function driver queues the request,
+			 * the next START_TRANSFER command will cause the
+			 * controller to generate an ERDY to initiate the
+			 * stream.
+			 */
+			dwc3_stop_active_transfer(dep, true, true);
+
+			/*
+			 * All stream eps will reinitiate stream on NoStream
+			 * rejection until we can determine that the host can
+			 * prime after the first transfer.
+			 */
+			dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
+		}
 	}
 
 out:
@@ -690,8 +723,6 @@ out:
 	return 0;
 }
 
-static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
-		bool interrupt);
 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 {
 	struct dwc3_request		*req;
@@ -912,7 +943,8 @@ static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 
 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 		dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
-		unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
+		unsigned stream_id, unsigned short_not_ok,
+		unsigned no_interrupt, unsigned is_last)
 {
 	struct dwc3		*dwc = dep->dwc;
 	struct usb_gadget	*gadget = &dwc->gadget;
@@ -1005,6 +1037,8 @@ static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 
 	if (chain)
 		trb->ctrl |= DWC3_TRB_CTRL_CHN;
+	else if (dep->stream_capable && is_last)
+		trb->ctrl |= DWC3_TRB_CTRL_LST;
 
 	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
@@ -1032,6 +1066,7 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 	unsigned		stream_id = req->request.stream_id;
 	unsigned		short_not_ok = req->request.short_not_ok;
 	unsigned		no_interrupt = req->request.no_interrupt;
+	unsigned		is_last = req->request.is_last;
 
 	if (req->request.num_sgs > 0) {
 		length = sg_dma_len(req->start_sg);
@@ -1052,7 +1087,7 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 	req->num_trbs++;
 
 	__dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
-			stream_id, short_not_ok, no_interrupt);
+			stream_id, short_not_ok, no_interrupt, is_last);
 }
 
 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
@@ -1097,7 +1132,8 @@ static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
 					maxp - rem, false, 1,
 					req->request.stream_id,
 					req->request.short_not_ok,
-					req->request.no_interrupt);
+					req->request.no_interrupt,
+					req->request.is_last);
 		} else {
 			dwc3_prepare_one_trb(dep, req, chain, i);
 		}
@@ -1141,7 +1177,8 @@ static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
 		__dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
 				false, 1, req->request.stream_id,
 				req->request.short_not_ok,
-				req->request.no_interrupt);
+				req->request.no_interrupt,
+				req->request.is_last);
 	} else if (req->request.zero && req->request.length &&
 		   (IS_ALIGNED(req->request.length, maxp))) {
 		struct dwc3	*dwc = dep->dwc;
@@ -1158,7 +1195,8 @@ static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
 		__dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
 				false, 1, req->request.stream_id,
 				req->request.short_not_ok,
-				req->request.no_interrupt);
+				req->request.no_interrupt,
+				req->request.is_last);
 	} else {
 		dwc3_prepare_one_trb(dep, req, false, 0);
 	}
@@ -1194,6 +1232,14 @@ static void dwc3_prepare_trbs(struct dwc3_ep *dep)
 
 		if (!dwc3_calc_trbs_left(dep))
 			return;
+
+		/*
+		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
+		 * burst capability may try to read and use TRBs beyond the
+		 * active transfer instead of stopping.
+		 */
+		if (dep->stream_capable && req->request.is_last)
+			return;
 	}
 
 	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
@@ -1217,9 +1263,19 @@ static void dwc3_prepare_trbs(struct dwc3_ep *dep)
 
 		if (!dwc3_calc_trbs_left(dep))
 			return;
+
+		/*
+		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
+		 * burst capability may try to read and use TRBs beyond the
+		 * active transfer instead of stopping.
+		 */
+		if (dep->stream_capable && req->request.is_last)
+			return;
 	}
 }
 
+static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
+
 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
 {
 	struct dwc3_gadget_ep_cmd_params params;
@@ -1259,17 +1315,26 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
 
 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 	if (ret < 0) {
-		/*
-		 * FIXME we need to iterate over the list of requests
-		 * here and stop, unmap, free and del each of the linked
-		 * requests instead of what we do now.
-		 */
-		if (req->trb)
-			memset(req->trb, 0, sizeof(struct dwc3_trb));
-		dwc3_gadget_del_and_unmap_request(dep, req, ret);
+		struct dwc3_request *tmp;
+
+		if (ret == -EAGAIN)
+			return ret;
+
+		dwc3_stop_active_transfer(dep, true, true);
+
+		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
+			dwc3_gadget_move_cancelled_request(req);
+
+		/* If ep isn't started, then there's no end transfer pending */
+		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
+			dwc3_gadget_ep_cleanup_cancelled_requests(dep);
+
 		return ret;
 	}
 
+	if (dep->stream_capable && req->request.is_last)
+		dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
+
 	return 0;
 }
 
@@ -1402,17 +1467,15 @@ static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
 	int ret;
 	int i;
 
-	if (list_empty(&dep->pending_list)) {
+	if (list_empty(&dep->pending_list) &&
+	    list_empty(&dep->started_list)) {
 		dep->flags |= DWC3_EP_PENDING_REQUEST;
 		return -EAGAIN;
 	}
 
-	if (!dwc->dis_start_transfer_quirk && dwc3_is_usb31(dwc) &&
-	    (dwc->revision <= DWC3_USB31_REVISION_160A ||
-	     (dwc->revision == DWC3_USB31_REVISION_170A &&
-	      dwc->version_type >= DWC31_VERSIONTYPE_EA01 &&
-	      dwc->version_type <= DWC31_VERSIONTYPE_EA06))) {
-
+	if (!dwc->dis_start_transfer_quirk &&
+	    (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
+	     DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
 		if (dwc->gadget.speed <= USB_SPEED_HIGH && dep->direction)
 			return dwc3_gadget_start_isoc_quirk(dep);
 	}
@@ -1425,6 +1488,27 @@ static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
 			break;
 	}
 
+	/*
+	 * After a number of unsuccessful start attempts due to bus-expiry
+	 * status, issue END_TRANSFER command and retry on the next XferNotReady
+	 * event.
+	 */
+	if (ret == -EAGAIN) {
+		struct dwc3_gadget_ep_cmd_params params;
+		u32 cmd;
+
+		cmd = DWC3_DEPCMD_ENDTRANSFER |
+			DWC3_DEPCMD_CMDIOC |
+			DWC3_DEPCMD_PARAM(dep->resource_index);
+
+		dep->resource_index = 0;
+		memset(&params, 0, sizeof(params));
+
+		ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
+		if (!ret)
+			dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
+	}
+
 	return ret;
 }
 
@@ -1457,6 +1541,9 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 	list_add_tail(&req->list, &dep->pending_list);
 	req->status = DWC3_REQUEST_STATUS_QUEUED;
 
+	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
+		return 0;
+
 	/* Start the transfer only after the END_TRANSFER is completed */
 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
 		dep->flags |= DWC3_EP_DELAY_START;
@@ -1508,6 +1595,10 @@ static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *r
 {
 	int i;
 
+	/* If req->trb is not set, then the request has not started */
+	if (!req->trb)
+		return;
+
 	/*
 	 * If request was already started, this means we had to
 	 * stop the transfer. With that we also need to ignore
@@ -1556,39 +1647,40 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
 
 	spin_lock_irqsave(&dwc->lock, flags);
 
-	list_for_each_entry(r, &dep->pending_list, list) {
+	list_for_each_entry(r, &dep->cancelled_list, list) {
 		if (r == req)
-			break;
+			goto out;
 	}
 
-	if (r != req) {
-		list_for_each_entry(r, &dep->started_list, list) {
-			if (r == req)
-				break;
+	list_for_each_entry(r, &dep->pending_list, list) {
+		if (r == req) {
+			dwc3_gadget_giveback(dep, req, -ECONNRESET);
+			goto out;
 		}
+	}
+
+	list_for_each_entry(r, &dep->started_list, list) {
 		if (r == req) {
+			struct dwc3_request *t;
+
 			/* wait until it is processed */
 			dwc3_stop_active_transfer(dep, true, true);
 
-			if (!r->trb)
-				goto out0;
+			/*
+			 * Remove any started request if the transfer is
+			 * cancelled.
+			 */
+			list_for_each_entry_safe(r, t, &dep->started_list, list)
+				dwc3_gadget_move_cancelled_request(r);
 
-			dwc3_gadget_move_cancelled_request(req);
-			if (dep->flags & DWC3_EP_TRANSFER_STARTED)
-				goto out0;
-			else
-				goto out1;
+			goto out;
 		}
-		dev_err(dwc->dev, "request %pK was not queued to %s\n",
-				request, ep->name);
-		ret = -EINVAL;
-		goto out0;
 	}
 
-out1:
-	dwc3_gadget_giveback(dep, req, -ECONNRESET);
-
-out0:
+	dev_err(dwc->dev, "request %pK was not queued to %s\n",
+		request, ep->name);
+	ret = -EINVAL;
+out:
 	spin_unlock_irqrestore(&dwc->lock, flags);
 
 	return ret;
@@ -1598,6 +1690,8 @@ int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
 {
 	struct dwc3_gadget_ep_cmd_params	params;
 	struct dwc3				*dwc = dep->dwc;
+	struct dwc3_request			*req;
+	struct dwc3_request			*tmp;
 	int					ret;
 
 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
@@ -1634,13 +1728,37 @@ int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
 		else
 			dep->flags |= DWC3_EP_STALL;
 	} else {
+		/*
+		 * Don't issue CLEAR_STALL command to control endpoints. The
+		 * controller automatically clears the STALL when it receives
+		 * the SETUP token.
+		 */
+		if (dep->number <= 1) {
+			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
+			return 0;
+		}
 
 		ret = dwc3_send_clear_stall_ep_cmd(dep);
-		if (ret)
+		if (ret) {
 			dev_err(dwc->dev, "failed to clear STALL on %s\n",
 					dep->name);
-		else
-			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
+			return ret;
+		}
+
+		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
+
+		dwc3_stop_active_transfer(dep, true, true);
+
+		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
+			dwc3_gadget_move_cancelled_request(req);
+
+		list_for_each_entry_safe(req, tmp, &dep->pending_list, list)
+			dwc3_gadget_move_cancelled_request(req);
+
+		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) {
+			dep->flags &= ~DWC3_EP_DELAY_START;
+			dwc3_gadget_ep_cleanup_cancelled_requests(dep);
+		}
 	}
 
 	return ret;
@@ -1756,7 +1874,7 @@ static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
 	}
 
 	/* Recent versions do this automatically */
-	if (dwc->revision < DWC3_REVISION_194A) {
+	if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
 		/* write zeroes to Link Change Request */
 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 		reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
@@ -1818,12 +1936,12 @@ static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
 
 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 	if (is_on) {
-		if (dwc->revision <= DWC3_REVISION_187A) {
+		if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
 			reg &= ~DWC3_DCTL_TRGTULST_MASK;
 			reg |= DWC3_DCTL_TRGTULST_RX_DET;
 		}
 
-		if (dwc->revision >= DWC3_REVISION_194A)
+		if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
 		reg |= DWC3_DCTL_RUN_STOP;
 
@@ -1897,7 +2015,7 @@ static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
 			DWC3_DEVTEN_USBRSTEN |
 			DWC3_DEVTEN_DISCONNEVTEN);
 
-	if (dwc->revision < DWC3_REVISION_250A)
+	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
 		reg |= DWC3_DEVTEN_ULSTCNGEN;
 
 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
@@ -1942,6 +2060,8 @@ static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
 
 	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
 	mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
+	if (DWC3_IP_IS(DWC32))
+		mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
 
 	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
 	nump = min_t(u32, nump, 16);
@@ -1978,10 +2098,10 @@ static int __dwc3_gadget_start(struct dwc3 *dwc)
 	 * bursts of data without going through any sort of endpoint throttling.
 	 */
 	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
-	if (dwc3_is_usb31(dwc))
-		reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
-	else
+	if (DWC3_IP_IS(DWC3))
 		reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
+	else
+		reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
 
 	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
 
@@ -2154,7 +2274,7 @@ static void dwc3_gadget_set_speed(struct usb_gadget *g,
 	 * STAR#9000525659: Clock Domain Crossing on DCTL in
 	 * USB 2.0 Mode
 	 */
-	if (dwc->revision < DWC3_REVISION_220A &&
+	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
 	    !dwc->dis_metastability_quirk) {
 		reg |= DWC3_DCFG_SUPERSPEED;
 	} else {
@@ -2172,18 +2292,18 @@ static void dwc3_gadget_set_speed(struct usb_gadget *g,
 			reg |= DWC3_DCFG_SUPERSPEED;
 			break;
 		case USB_SPEED_SUPER_PLUS:
-			if (dwc3_is_usb31(dwc))
-				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
-			else
+			if (DWC3_IP_IS(DWC3))
 				reg |= DWC3_DCFG_SUPERSPEED;
+			else
+				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
 			break;
 		default:
 			dev_err(dwc->dev, "invalid speed (%d)\n", speed);
 
-			if (dwc->revision & DWC3_REVISION_IS_DWC31)
-				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
-			else
+			if (DWC3_IP_IS(DWC3))
 				reg |= DWC3_DCFG_SUPERSPEED;
+			else
+				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
 		}
 	}
 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
@@ -2226,14 +2346,17 @@ static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
 	int size;
 
 	mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
+	if (DWC3_IP_IS(DWC32))
+		mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
+
 	/* MDWIDTH is represented in bits, we need it in bytes */
 	mdwidth /= 8;
 
 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
-	if (dwc3_is_usb31(dwc))
-		size = DWC31_GTXFIFOSIZ_TXFDEP(size);
-	else
+	if (DWC3_IP_IS(DWC3))
 		size = DWC3_GTXFIFOSIZ_TXFDEP(size);
+	else
+		size = DWC31_GTXFIFOSIZ_TXFDEP(size);
 
 	/* FIFO Depth is in MDWDITH bytes. Multiply */
 	size *= mdwidth;
@@ -2270,16 +2393,18 @@ static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
 	int size;
 
 	mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
+	if (DWC3_IP_IS(DWC32))
+		mdwidth += DWC3_GHWPARAMS6_MDWIDTH(dwc->hwparams.hwparams6);
 
 	/* MDWIDTH is represented in bits, convert to bytes */
 	mdwidth /= 8;
 
 	/* All OUT endpoints share a single RxFIFO space */
 	size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
-	if (dwc3_is_usb31(dwc))
-		size = DWC31_GRXFIFOSIZ_RXFDEP(size);
-	else
+	if (DWC3_IP_IS(DWC3))
 		size = DWC3_GRXFIFOSIZ_RXFDEP(size);
+	else
+		size = DWC31_GRXFIFOSIZ_RXFDEP(size);
 
 	/* FIFO depth is in MDWDITH bytes */
 	size *= mdwidth;
@@ -2531,10 +2656,8 @@ static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
 
 	req->request.actual = req->request.length - req->remaining;
 
-	if (!dwc3_gadget_ep_request_completed(req)) {
-		__dwc3_gadget_kick_transfer(dep);
+	if (!dwc3_gadget_ep_request_completed(req))
 		goto out;
-	}
 
 	dwc3_gadget_giveback(dep, req, status);
 
@@ -2558,41 +2681,53 @@ static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
 	}
 }
 
+static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
+{
+	struct dwc3_request	*req;
+
+	if (!list_empty(&dep->pending_list))
+		return true;
+
+	/*
+	 * We only need to check the first entry of the started list. We can
+	 * assume the completed requests are removed from the started list.
+	 */
+	req = next_request(&dep->started_list);
+	if (!req)
+		return false;
+
+	return !dwc3_gadget_ep_request_completed(req);
+}
+
 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
 		const struct dwc3_event_depevt *event)
 {
 	dep->frame_number = event->parameters;
 }
 
-static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
-		const struct dwc3_event_depevt *event)
+static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
+		const struct dwc3_event_depevt *event, int status)
 {
 	struct dwc3		*dwc = dep->dwc;
-	unsigned		status = 0;
-	bool			stop = false;
-
-	dwc3_gadget_endpoint_frame_from_event(dep, event);
-
-	if (event->status & DEPEVT_STATUS_BUSERR)
-		status = -ECONNRESET;
-
-	if (event->status & DEPEVT_STATUS_MISSED_ISOC) {
-		status = -EXDEV;
-
-		if (list_empty(&dep->started_list))
-			stop = true;
-	}
+	bool			no_started_trb = true;
 
 	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
 
-	if (stop)
+	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
+		goto out;
+
+	if (status == -EXDEV && list_empty(&dep->started_list))
 		dwc3_stop_active_transfer(dep, true, true);
+	else if (dwc3_gadget_ep_should_continue(dep))
+		if (__dwc3_gadget_kick_transfer(dep) == 0)
+			no_started_trb = false;
 
+out:
 	/*
 	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
 	 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
 	 */
-	if (dwc->revision < DWC3_REVISION_183A) {
+	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
 		u32		reg;
 		int		i;
 
@@ -2603,7 +2738,7 @@ static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
 				continue;
 
 			if (!list_empty(&dep->started_list))
-				return;
+				return no_started_trb;
 		}
 
 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
@@ -2612,15 +2747,124 @@ static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
 
 		dwc->u1u2 = 0;
 	}
+
+	return no_started_trb;
+}
+
+static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
+		const struct dwc3_event_depevt *event)
+{
+	int status = 0;
+
+	if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
+		dwc3_gadget_endpoint_frame_from_event(dep, event);
+
+	if (event->status & DEPEVT_STATUS_BUSERR)
+		status = -ECONNRESET;
+
+	if (event->status & DEPEVT_STATUS_MISSED_ISOC)
+		status = -EXDEV;
+
+	dwc3_gadget_endpoint_trbs_complete(dep, event, status);
+}
+
+static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
+		const struct dwc3_event_depevt *event)
+{
+	int status = 0;
+
+	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
+
+	if (event->status & DEPEVT_STATUS_BUSERR)
+		status = -ECONNRESET;
+
+	if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
+		dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
 }
 
 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
 		const struct dwc3_event_depevt *event)
 {
 	dwc3_gadget_endpoint_frame_from_event(dep, event);
+
+	/*
+	 * The XferNotReady event is generated only once before the endpoint
+	 * starts. It will be generated again when END_TRANSFER command is
+	 * issued. For some controller versions, the XferNotReady event may be
+	 * generated while the END_TRANSFER command is still in process. Ignore
+	 * it and wait for the next XferNotReady event after the command is
+	 * completed.
+	 */
+	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
+		return;
+
 	(void) __dwc3_gadget_start_isoc(dep);
 }
 
+static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
+		const struct dwc3_event_depevt *event)
+{
+	struct dwc3 *dwc = dep->dwc;
+
+	if (event->status == DEPEVT_STREAMEVT_FOUND) {
+		dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
+		goto out;
+	}
+
+	/* Note: NoStream rejection event param value is 0 and not 0xFFFF */
+	switch (event->parameters) {
+	case DEPEVT_STREAM_PRIME:
+		/*
+		 * If the host can properly transition the endpoint state from
+		 * idle to prime after a NoStream rejection, there's no need to
+		 * force restarting the endpoint to reinitiate the stream. To
+		 * simplify the check, assume the host follows the USB spec if
+		 * it primed the endpoint more than once.
+		 */
+		if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
+			if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
+				dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
+			else
+				dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
+		}
+
+		break;
+	case DEPEVT_STREAM_NOSTREAM:
+		if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
+		    !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
+		    !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))
+			break;
+
+		/*
+		 * If the host rejects a stream due to no active stream, by the
+		 * USB and xHCI spec, the endpoint will be put back to idle
+		 * state. When the host is ready (buffer added/updated), it will
+		 * prime the endpoint to inform the usb device controller. This
+		 * triggers the device controller to issue ERDY to restart the
+		 * stream. However, some hosts don't follow this and keep the
+		 * endpoint in the idle state. No prime will come despite host
+		 * streams are updated, and the device controller will not be
+		 * triggered to generate ERDY to move the next stream data. To
+		 * workaround this and maintain compatibility with various
+		 * hosts, force to reinitate the stream until the host is ready
+		 * instead of waiting for the host to prime the endpoint.
+		 */
+		if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
+			unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
+
+			dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
+		} else {
+			dep->flags |= DWC3_EP_DELAY_START;
+			dwc3_stop_active_transfer(dep, true, true);
+			return;
+		}
+		break;
+	}
+
+out:
+	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
+}
+
 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
 		const struct dwc3_event_depevt *event)
 {
@@ -2665,8 +2909,12 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
 			dep->flags &= ~DWC3_EP_DELAY_START;
 		}
 		break;
-	case DWC3_DEPEVT_STREAMEVT:
 	case DWC3_DEPEVT_XFERCOMPLETE:
+		dwc3_gadget_endpoint_transfer_complete(dep, event);
+		break;
+	case DWC3_DEPEVT_STREAMEVT:
+		dwc3_gadget_endpoint_stream_event(dep, event);
+		break;
 	case DWC3_DEPEVT_RXTXFIFOEVT:
 		break;
 	}
@@ -2758,6 +3006,14 @@ static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
 	WARN_ON_ONCE(ret);
 	dep->resource_index = 0;
 
+	/*
+	 * The END_TRANSFER command will cause the controller to generate a
+	 * NoStream Event, and it's not due to the host DP NoStream rejection.
+	 * Ignore the next NoStream event.
+	 */
+	if (dep->stream_capable)
+		dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
+
 	if (!interrupt)
 		dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 	else
@@ -2838,7 +3094,7 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
 	 * STAR#9000466709: RTL: Device : Disconnect event not
 	 * generated if setup packet pending in FIFO
 	 */
-	if (dwc->revision < DWC3_REVISION_188A) {
+	if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
 		if (dwc->setup_packet_pending)
 			dwc3_gadget_disconnect_interrupt(dwc);
 	}
@@ -2897,7 +3153,7 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 		 * STAR#9000483510: RTL: SS : USB3 reset event may
 		 * not be generated always when the link enters poll
 		 */
-		if (dwc->revision < DWC3_REVISION_190A)
+		if (DWC3_VER_IS_PRIOR(DWC3, 190A))
 			dwc3_gadget_reset_interrupt(dwc);
 
 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
@@ -2925,7 +3181,7 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 
 	/* Enable USB2 LPM Capability */
 
-	if ((dwc->revision > DWC3_REVISION_194A) &&
+	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
 	    (speed != DWC3_DSTS_SUPERSPEED) &&
 	    (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
 		reg = dwc3_readl(dwc->regs, DWC3_DCFG);
@@ -2944,11 +3200,10 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 		 * BESL value in the LPM token is less than or equal to LPM
 		 * NYET threshold.
 		 */
-		WARN_ONCE(dwc->revision < DWC3_REVISION_240A
-				&& dwc->has_lpm_erratum,
+		WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
 				"LPM Erratum not available on dwc3 revisions < 2.40a\n");
 
-		if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
+		if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
 			reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
 
 		dwc3_gadget_dctl_write_safe(dwc, reg);
@@ -3019,7 +3274,7 @@ static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
 	 * operational mode
 	 */
 	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
-	if ((dwc->revision < DWC3_REVISION_250A) &&
+	if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
 			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
 		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
 				(next == DWC3_LINK_STATE_RESUME)) {
@@ -3045,7 +3300,7 @@ static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
 	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
 	 * core send LGO_Ux entering U0
 	 */
-	if (dwc->revision < DWC3_REVISION_183A) {
+	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
 		if (next == DWC3_LINK_STATE_U0) {
 			u32	u1u2;
 			u32	reg;
@@ -3156,7 +3411,7 @@ static void dwc3_gadget_interrupt(struct dwc3 *dwc,
 		break;
 	case DWC3_DEVICE_EVENT_EOPF:
 		/* It changed to be suspend event for version 2.30a and above */
-		if (dwc->revision >= DWC3_REVISION_230A) {
+		if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
 			/*
 			 * Ignore suspend event until the gadget enters into
 			 * USB_STATE_CONFIGURED state.
@@ -3401,7 +3656,7 @@ int dwc3_gadget_init(struct dwc3 *dwc)
 	 * is less than super speed because we don't have means, yet, to tell
 	 * composite.c that we are USB 2.0 + LPM ECN.
 	 */
-	if (dwc->revision < DWC3_REVISION_220A &&
+	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
 	    !dwc->dis_metastability_quirk)
 		dev_info(dwc->dev, "changing max_speed on rev %08x\n",
 				dwc->revision);
diff --git a/drivers/usb/dwc3/gadget.h b/drivers/usb/dwc3/gadget.h
index fbc7d8013f0b..24dca3872022 100644
--- a/drivers/usb/dwc3/gadget.h
+++ b/drivers/usb/dwc3/gadget.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * gadget.h - DesignWare USB3 DRD Gadget Header
  *
diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c
index 86dbd012b984..bef1c1ac2067 100644
--- a/drivers/usb/dwc3/host.c
+++ b/drivers/usb/dwc3/host.c
@@ -104,7 +104,7 @@ int dwc3_host_init(struct dwc3 *dwc)
 	 *
 	 * This following flag tells XHCI to do just that.
 	 */
-	if (dwc->revision <= DWC3_REVISION_300A)
+	if (DWC3_VER_IS_WITHIN(DWC3, ANY, 300A))
 		props[prop_idx++] = PROPERTY_ENTRY_BOOL("quirk-broken-port-ped");
 
 	if (prop_idx) {
diff --git a/drivers/usb/dwc3/io.h b/drivers/usb/dwc3/io.h
index 70acdf94a0bf..9bbe5d4bf076 100644
--- a/drivers/usb/dwc3/io.h
+++ b/drivers/usb/dwc3/io.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
  * io.h - DesignWare USB3 DRD IO Header
  *
diff --git a/drivers/usb/dwc3/trace.h b/drivers/usb/dwc3/trace.h
index 3054b89512ff..4c4fc6c41d9b 100644
--- a/drivers/usb/dwc3/trace.h
+++ b/drivers/usb/dwc3/trace.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
  * trace.h - DesignWare USB3 DRD Controller Trace Support
  *
diff --git a/drivers/usb/early/xhci-dbc.c b/drivers/usb/early/xhci-dbc.c
index 171280c80228..04ba11fff0ed 100644
--- a/drivers/usb/early/xhci-dbc.c
+++ b/drivers/usb/early/xhci-dbc.c
@@ -18,7 +18,6 @@
 #include <asm/fixmap.h>
 #include <linux/bcd.h>
 #include <linux/export.h>
-#include <linux/version.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/kthread.h>
diff --git a/drivers/usb/early/xhci-dbc.h b/drivers/usb/early/xhci-dbc.h
index 6e2b7266a695..8b4d71de45fc 100644
--- a/drivers/usb/early/xhci-dbc.h
+++ b/drivers/usb/early/xhci-dbc.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * xhci-dbc.h - xHCI debug capability early driver
  *
diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
index cb4950cf1cdc..5c1eb96a5c57 100644
--- a/drivers/usb/gadget/composite.c
+++ b/drivers/usb/gadget/composite.c
@@ -96,40 +96,43 @@ function_descriptors(struct usb_function *f,
 }
 
 /**
- * next_ep_desc() - advance to the next EP descriptor
+ * next_desc() - advance to the next desc_type descriptor
  * @t: currect pointer within descriptor array
+ * @desc_type: descriptor type
  *
- * Return: next EP descriptor or NULL
+ * Return: next desc_type descriptor or NULL
  *
- * Iterate over @t until either EP descriptor found or
+ * Iterate over @t until either desc_type descriptor found or
  * NULL (that indicates end of list) encountered
  */
 static struct usb_descriptor_header**
-next_ep_desc(struct usb_descriptor_header **t)
+next_desc(struct usb_descriptor_header **t, u8 desc_type)
 {
 	for (; *t; t++) {
-		if ((*t)->bDescriptorType == USB_DT_ENDPOINT)
+		if ((*t)->bDescriptorType == desc_type)
 			return t;
 	}
 	return NULL;
 }
 
 /*
- * for_each_ep_desc()- iterate over endpoint descriptors in the
- *		descriptors list
- * @start:	pointer within descriptor array.
- * @ep_desc:	endpoint descriptor to use as the loop cursor
+ * for_each_desc() - iterate over desc_type descriptors in the
+ * descriptors list
+ * @start: pointer within descriptor array.
+ * @iter_desc: desc_type descriptor to use as the loop cursor
+ * @desc_type: wanted descriptr type
  */
-#define for_each_ep_desc(start, ep_desc) \
-	for (ep_desc = next_ep_desc(start); \
-	      ep_desc; ep_desc = next_ep_desc(ep_desc+1))
+#define for_each_desc(start, iter_desc, desc_type) \
+	for (iter_desc = next_desc(start, desc_type); \
+	     iter_desc; iter_desc = next_desc(iter_desc + 1, desc_type))
 
 /**
- * config_ep_by_speed() - configures the given endpoint
+ * config_ep_by_speed_and_alt() - configures the given endpoint
  * according to gadget speed.
  * @g: pointer to the gadget
  * @f: usb function
  * @_ep: the endpoint to configure
+ * @alt: alternate setting number
  *
  * Return: error code, 0 on success
  *
@@ -142,11 +145,13 @@ next_ep_desc(struct usb_descriptor_header **t)
  * Note: the supplied function should hold all the descriptors
  * for supported speeds
  */
-int config_ep_by_speed(struct usb_gadget *g,
-			struct usb_function *f,
-			struct usb_ep *_ep)
+int config_ep_by_speed_and_alt(struct usb_gadget *g,
+				struct usb_function *f,
+				struct usb_ep *_ep,
+				u8 alt)
 {
 	struct usb_endpoint_descriptor *chosen_desc = NULL;
+	struct usb_interface_descriptor *int_desc = NULL;
 	struct usb_descriptor_header **speed_desc = NULL;
 
 	struct usb_ss_ep_comp_descriptor *comp_desc = NULL;
@@ -182,8 +187,21 @@ int config_ep_by_speed(struct usb_gadget *g,
 	default:
 		speed_desc = f->fs_descriptors;
 	}
+
+	/* find correct alternate setting descriptor */
+	for_each_desc(speed_desc, d_spd, USB_DT_INTERFACE) {
+		int_desc = (struct usb_interface_descriptor *)*d_spd;
+
+		if (int_desc->bAlternateSetting == alt) {
+			speed_desc = d_spd;
+			goto intf_found;
+		}
+	}
+	return -EIO;
+
+intf_found:
 	/* find descriptors */
-	for_each_ep_desc(speed_desc, d_spd) {
+	for_each_desc(speed_desc, d_spd, USB_DT_ENDPOINT) {
 		chosen_desc = (struct usb_endpoint_descriptor *)*d_spd;
 		if (chosen_desc->bEndpointAddress == _ep->address)
 			goto ep_found;
@@ -237,6 +255,32 @@ ep_found:
 	}
 	return 0;
 }
+EXPORT_SYMBOL_GPL(config_ep_by_speed_and_alt);
+
+/**
+ * config_ep_by_speed() - configures the given endpoint
+ * according to gadget speed.
+ * @g: pointer to the gadget
+ * @f: usb function
+ * @_ep: the endpoint to configure
+ *
+ * Return: error code, 0 on success
+ *
+ * This function chooses the right descriptors for a given
+ * endpoint according to gadget speed and saves it in the
+ * endpoint desc field. If the endpoint already has a descriptor
+ * assigned to it - overwrites it with currently corresponding
+ * descriptor. The endpoint maxpacket field is updated according
+ * to the chosen descriptor.
+ * Note: the supplied function should hold all the descriptors
+ * for supported speeds
+ */
+int config_ep_by_speed(struct usb_gadget *g,
+			struct usb_function *f,
+			struct usb_ep *_ep)
+{
+	return config_ep_by_speed_and_alt(g, f, _ep, 0);
+}
 EXPORT_SYMBOL_GPL(config_ep_by_speed);
 
 /**
diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c
index 6a9aa4413d64..9dc06a4e1b30 100644
--- a/drivers/usb/gadget/configfs.c
+++ b/drivers/usb/gadget/configfs.c
@@ -13,8 +13,6 @@
 int check_user_usb_string(const char *name,
 		struct usb_gadget_strings *stringtab_dev)
 {
-	unsigned primary_lang;
-	unsigned sub_lang;
 	u16 num;
 	int ret;
 
@@ -22,17 +20,7 @@ int check_user_usb_string(const char *name,
 	if (ret)
 		return ret;
 
-	primary_lang = num & 0x3ff;
-	sub_lang = num >> 10;
-
-	/* simple sanity check for valid langid */
-	switch (primary_lang) {
-	case 0:
-	case 0x62 ... 0xfe:
-	case 0x100 ... 0x3ff:
-		return -EINVAL;
-	}
-	if (!sub_lang)
+	if (!usb_validate_langid(num))
 		return -EINVAL;
 
 	stringtab_dev->language = num;
diff --git a/drivers/usb/gadget/function/f_acm.c b/drivers/usb/gadget/function/f_acm.c
index 7c152c28b26c..200596ea9557 100644
--- a/drivers/usb/gadget/function/f_acm.c
+++ b/drivers/usb/gadget/function/f_acm.c
@@ -723,6 +723,20 @@ static void acm_free_func(struct usb_function *f)
 	kfree(acm);
 }
 
+static void acm_resume(struct usb_function *f)
+{
+	struct f_acm *acm = func_to_acm(f);
+
+	gserial_resume(&acm->port);
+}
+
+static void acm_suspend(struct usb_function *f)
+{
+	struct f_acm *acm = func_to_acm(f);
+
+	gserial_suspend(&acm->port);
+}
+
 static struct usb_function *acm_alloc_func(struct usb_function_instance *fi)
 {
 	struct f_serial_opts *opts;
@@ -750,6 +764,8 @@ static struct usb_function *acm_alloc_func(struct usb_function_instance *fi)
 	acm->port_num = opts->port_num;
 	acm->port.func.unbind = acm_unbind;
 	acm->port.func.free_func = acm_free_func;
+	acm->port.func.resume = acm_resume;
+	acm->port.func.suspend = acm_suspend;
 
 	return &acm->port.func;
 }
diff --git a/drivers/usb/gadget/function/f_eem.c b/drivers/usb/gadget/function/f_eem.c
index b81a91d504bd..cfcc4e81fb77 100644
--- a/drivers/usb/gadget/function/f_eem.c
+++ b/drivers/usb/gadget/function/f_eem.c
@@ -291,8 +291,6 @@ static int eem_bind(struct usb_configuration *c, struct usb_function *f)
 		goto fail;
 	eem->port.out_ep = ep;
 
-	status = -ENOMEM;
-
 	/* support all relevant hardware speeds... we expect that when
 	 * hardware is dual speed, all bulk-capable endpoints work at
 	 * both speeds
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 10f01f974f67..490d353d5fde 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -32,7 +32,7 @@
 #include <linux/usb/functionfs.h>
 
 #include <linux/aio.h>
-#include <linux/mmu_context.h>
+#include <linux/kthread.h>
 #include <linux/poll.h>
 #include <linux/eventfd.h>
 
@@ -824,13 +824,9 @@ static void ffs_user_copy_worker(struct work_struct *work)
 	bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD;
 
 	if (io_data->read && ret > 0) {
-		mm_segment_t oldfs = get_fs();
-
-		set_fs(USER_DS);
-		use_mm(io_data->mm);
+		kthread_use_mm(io_data->mm);
 		ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data);
-		unuse_mm(io_data->mm);
-		set_fs(oldfs);
+		kthread_unuse_mm(io_data->mm);
 	}
 
 	io_data->kiocb->ki_complete(io_data->kiocb, ret, ret);
@@ -2508,7 +2504,7 @@ static int __ffs_data_got_descs(struct ffs_data *ffs,
 		os_descs_count = get_unaligned_le32(data);
 		data += 4;
 		len -= 4;
-	};
+	}
 
 	/* Read descriptors */
 	raw_descs = data;
diff --git a/drivers/usb/gadget/function/f_serial.c b/drivers/usb/gadget/function/f_serial.c
index 1406255d0865..e62713846350 100644
--- a/drivers/usb/gadget/function/f_serial.c
+++ b/drivers/usb/gadget/function/f_serial.c
@@ -348,6 +348,20 @@ static void gser_unbind(struct usb_configuration *c, struct usb_function *f)
 	usb_free_all_descriptors(f);
 }
 
+static void gser_resume(struct usb_function *f)
+{
+	struct f_gser *gser = func_to_gser(f);
+
+	gserial_resume(&gser->port);
+}
+
+static void gser_suspend(struct usb_function *f)
+{
+	struct f_gser *gser = func_to_gser(f);
+
+	gserial_suspend(&gser->port);
+}
+
 static struct usb_function *gser_alloc(struct usb_function_instance *fi)
 {
 	struct f_gser	*gser;
@@ -369,6 +383,8 @@ static struct usb_function *gser_alloc(struct usb_function_instance *fi)
 	gser->port.func.set_alt = gser_set_alt;
 	gser->port.func.disable = gser_disable;
 	gser->port.func.free_func = gser_free;
+	gser->port.func.resume = gser_resume;
+	gser->port.func.suspend = gser_suspend;
 
 	return &gser->port.func;
 }
diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c
index 36504931b2d1..eaf556ceac32 100644
--- a/drivers/usb/gadget/function/f_tcm.c
+++ b/drivers/usb/gadget/function/f_tcm.c
@@ -531,6 +531,7 @@ static int uasp_prepare_r_request(struct usbg_cmd *cmd)
 		stream->req_in->sg = se_cmd->t_data_sg;
 	}
 
+	stream->req_in->is_last = 1;
 	stream->req_in->complete = uasp_status_data_cmpl;
 	stream->req_in->length = se_cmd->data_length;
 	stream->req_in->context = cmd;
@@ -554,6 +555,7 @@ static void uasp_prepare_status(struct usbg_cmd *cmd)
 	 */
 	iu->len = cpu_to_be16(se_cmd->scsi_sense_length);
 	iu->status = se_cmd->scsi_status;
+	stream->req_status->is_last = 1;
 	stream->req_status->context = cmd;
 	stream->req_status->length = se_cmd->scsi_sense_length + 16;
 	stream->req_status->buf = iu;
@@ -991,6 +993,7 @@ static int usbg_prepare_w_request(struct usbg_cmd *cmd, struct usb_request *req)
 		req->sg = se_cmd->t_data_sg;
 	}
 
+	req->is_last = 1;
 	req->complete = usbg_data_write_cmpl;
 	req->length = se_cmd->data_length;
 	req->context = cmd;
@@ -1049,7 +1052,8 @@ static void usbg_cmd_work(struct work_struct *work)
 		transport_init_se_cmd(se_cmd,
 				tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
 				tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
-				cmd->prio_attr, cmd->sense_iu.sense);
+				cmd->prio_attr, cmd->sense_iu.sense,
+				cmd->unpacked_lun);
 		goto out;
 	}
 
@@ -1179,7 +1183,8 @@ static void bot_cmd_work(struct work_struct *work)
 		transport_init_se_cmd(se_cmd,
 				tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo,
 				tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE,
-				cmd->prio_attr, cmd->sense_iu.sense);
+				cmd->prio_attr, cmd->sense_iu.sense,
+				cmd->unpacked_lun);
 		goto out;
 	}
 
diff --git a/drivers/usb/gadget/function/f_uvc.h b/drivers/usb/gadget/function/f_uvc.h
index a81a17765558..1db972d4beeb 100644
--- a/drivers/usb/gadget/function/f_uvc.h
+++ b/drivers/usb/gadget/function/f_uvc.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  *	f_uvc.h  --  USB Video Class Gadget driver
  *
diff --git a/drivers/usb/gadget/function/rndis.h b/drivers/usb/gadget/function/rndis.h
index c7e3a70ce6c1..f6167f7fea82 100644
--- a/drivers/usb/gadget/function/rndis.h
+++ b/drivers/usb/gadget/function/rndis.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * RNDIS	Definitions for Remote NDIS
  *
diff --git a/drivers/usb/gadget/function/u_audio.h b/drivers/usb/gadget/function/u_audio.h
index 81d3d4ed6dfb..5ea6b86f1fda 100644
--- a/drivers/usb/gadget/function/u_audio.h
+++ b/drivers/usb/gadget/function/u_audio.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * u_audio.h -- interface to USB gadget "ALSA sound card" utilities
  *
diff --git a/drivers/usb/gadget/function/u_ecm.h b/drivers/usb/gadget/function/u_ecm.h
index 098ece573a5e..77cfb89932be 100644
--- a/drivers/usb/gadget/function/u_ecm.h
+++ b/drivers/usb/gadget/function/u_ecm.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_ecm.h
  *
diff --git a/drivers/usb/gadget/function/u_eem.h b/drivers/usb/gadget/function/u_eem.h
index 921386a375cf..3bd85dfcd71c 100644
--- a/drivers/usb/gadget/function/u_eem.h
+++ b/drivers/usb/gadget/function/u_eem.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_eem.h
  *
diff --git a/drivers/usb/gadget/function/u_ether.h b/drivers/usb/gadget/function/u_ether.h
index 332307d54292..10dd640684e2 100644
--- a/drivers/usb/gadget/function/u_ether.h
+++ b/drivers/usb/gadget/function/u_ether.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * u_ether.h -- interface to USB gadget "ethernet link" utilities
  *
diff --git a/drivers/usb/gadget/function/u_ether_configfs.h b/drivers/usb/gadget/function/u_ether_configfs.h
index d8b92485b727..bd92b5703013 100644
--- a/drivers/usb/gadget/function/u_ether_configfs.h
+++ b/drivers/usb/gadget/function/u_ether_configfs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_ether_configfs.h
  *
diff --git a/drivers/usb/gadget/function/u_fs.h b/drivers/usb/gadget/function/u_fs.h
index f9b0cf67360d..f102ec23f3af 100644
--- a/drivers/usb/gadget/function/u_fs.h
+++ b/drivers/usb/gadget/function/u_fs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_fs.h
  *
diff --git a/drivers/usb/gadget/function/u_gether.h b/drivers/usb/gadget/function/u_gether.h
index ce4f07626f96..2f7a373ed449 100644
--- a/drivers/usb/gadget/function/u_gether.h
+++ b/drivers/usb/gadget/function/u_gether.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_gether.h
  *
diff --git a/drivers/usb/gadget/function/u_hid.h b/drivers/usb/gadget/function/u_hid.h
index 1594bfa312eb..84e6da302499 100644
--- a/drivers/usb/gadget/function/u_hid.h
+++ b/drivers/usb/gadget/function/u_hid.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_hid.h
  *
diff --git a/drivers/usb/gadget/function/u_midi.h b/drivers/usb/gadget/function/u_midi.h
index 29bf006c0a13..f6e14af7f566 100644
--- a/drivers/usb/gadget/function/u_midi.h
+++ b/drivers/usb/gadget/function/u_midi.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_midi.h
  *
diff --git a/drivers/usb/gadget/function/u_ncm.h b/drivers/usb/gadget/function/u_ncm.h
index 70da3201a1d0..5408854d8407 100644
--- a/drivers/usb/gadget/function/u_ncm.h
+++ b/drivers/usb/gadget/function/u_ncm.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_ncm.h
  *
diff --git a/drivers/usb/gadget/function/u_phonet.h b/drivers/usb/gadget/function/u_phonet.h
index 12fb613f85d1..c53233b37192 100644
--- a/drivers/usb/gadget/function/u_phonet.h
+++ b/drivers/usb/gadget/function/u_phonet.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * u_phonet.h - interface to Phonet
  *
diff --git a/drivers/usb/gadget/function/u_printer.h b/drivers/usb/gadget/function/u_printer.h
index 78797764f478..318205fb778e 100644
--- a/drivers/usb/gadget/function/u_printer.h
+++ b/drivers/usb/gadget/function/u_printer.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_printer.h
  *
diff --git a/drivers/usb/gadget/function/u_rndis.h b/drivers/usb/gadget/function/u_rndis.h
index 1e148b76f339..a8c409b2f52f 100644
--- a/drivers/usb/gadget/function/u_rndis.h
+++ b/drivers/usb/gadget/function/u_rndis.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_rndis.h
  *
diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c
index 8167d379e115..3cfc6e2eba71 100644
--- a/drivers/usb/gadget/function/u_serial.c
+++ b/drivers/usb/gadget/function/u_serial.c
@@ -120,6 +120,8 @@ struct gs_port {
 	wait_queue_head_t	drain_wait;	/* wait while writes drain */
 	bool                    write_busy;
 	wait_queue_head_t	close_wait;
+	bool			suspended;	/* port suspended */
+	bool			start_delayed;	/* delay start when suspended */
 
 	/* REVISIT this state ... */
 	struct usb_cdc_line_coding port_line_coding;	/* 8-N-1 etc */
@@ -630,13 +632,19 @@ static int gs_open(struct tty_struct *tty, struct file *file)
 
 	/* if connected, start the I/O stream */
 	if (port->port_usb) {
-		struct gserial	*gser = port->port_usb;
-
-		pr_debug("gs_open: start ttyGS%d\n", port->port_num);
-		gs_start_io(port);
-
-		if (gser->connect)
-			gser->connect(gser);
+		/* if port is suspended, wait resume to start I/0 stream */
+		if (!port->suspended) {
+			struct gserial	*gser = port->port_usb;
+
+			pr_debug("gs_open: start ttyGS%d\n", port->port_num);
+			gs_start_io(port);
+
+			if (gser->connect)
+				gser->connect(gser);
+		} else {
+			pr_debug("delay start of ttyGS%d\n", port->port_num);
+			port->start_delayed = true;
+		}
 	}
 
 	pr_debug("gs_open: ttyGS%d (%p,%p)\n", port->port_num, tty, file);
@@ -680,7 +688,7 @@ raced_with_open:
 	pr_debug("gs_close: ttyGS%d (%p,%p) ...\n", port->port_num, tty, file);
 
 	gser = port->port_usb;
-	if (gser && gser->disconnect)
+	if (gser && !port->suspended && gser->disconnect)
 		gser->disconnect(gser);
 
 	/* wait for circular write buffer to drain, disconnect, or at
@@ -708,6 +716,7 @@ raced_with_open:
 	else
 		kfifo_reset(&port->port_write_buf);
 
+	port->start_delayed = false;
 	port->port.count = 0;
 	port->port.tty = NULL;
 
@@ -1403,6 +1412,38 @@ void gserial_disconnect(struct gserial *gser)
 }
 EXPORT_SYMBOL_GPL(gserial_disconnect);
 
+void gserial_suspend(struct gserial *gser)
+{
+	struct gs_port	*port = gser->ioport;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&port->port_lock, flags);
+	port->suspended = true;
+	spin_unlock_irqrestore(&port->port_lock, flags);
+}
+EXPORT_SYMBOL_GPL(gserial_suspend);
+
+void gserial_resume(struct gserial *gser)
+{
+	struct gs_port *port = gser->ioport;
+	unsigned long	flags;
+
+	spin_lock_irqsave(&port->port_lock, flags);
+	port->suspended = false;
+	if (!port->start_delayed) {
+		spin_unlock_irqrestore(&port->port_lock, flags);
+		return;
+	}
+
+	pr_debug("delayed start ttyGS%d\n", port->port_num);
+	gs_start_io(port);
+	if (gser->connect)
+		gser->connect(gser);
+	port->start_delayed = false;
+	spin_unlock_irqrestore(&port->port_lock, flags);
+}
+EXPORT_SYMBOL_GPL(gserial_resume);
+
 static int userial_init(void)
 {
 	unsigned			i;
diff --git a/drivers/usb/gadget/function/u_serial.h b/drivers/usb/gadget/function/u_serial.h
index e5b08ab8cf7a..cadb76eecbc7 100644
--- a/drivers/usb/gadget/function/u_serial.h
+++ b/drivers/usb/gadget/function/u_serial.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * u_serial.h - interface to USB gadget "serial port"/TTY utilities
  *
@@ -68,6 +68,8 @@ ssize_t gserial_get_console(unsigned char port_num, char *page);
 /* connect/disconnect is handled by individual functions */
 int gserial_connect(struct gserial *, u8 port_num);
 void gserial_disconnect(struct gserial *);
+void gserial_suspend(struct gserial *p);
+void gserial_resume(struct gserial *p);
 
 /* functions are bound to configurations by a config or gadget driver */
 int gser_bind_config(struct usb_configuration *c, u8 port_num);
diff --git a/drivers/usb/gadget/function/u_tcm.h b/drivers/usb/gadget/function/u_tcm.h
index 3f7ccecb0f9b..2cd15d9a1c0d 100644
--- a/drivers/usb/gadget/function/u_tcm.h
+++ b/drivers/usb/gadget/function/u_tcm.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_tcm.h
  *
diff --git a/drivers/usb/gadget/function/u_uac1.h b/drivers/usb/gadget/function/u_uac1.h
index 6f1a9d73defe..39c0e29e1b46 100644
--- a/drivers/usb/gadget/function/u_uac1.h
+++ b/drivers/usb/gadget/function/u_uac1.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_uac1.h - Utility definitions for UAC1 function
  *
diff --git a/drivers/usb/gadget/function/u_uac1_legacy.h b/drivers/usb/gadget/function/u_uac1_legacy.h
index 5c1bdf46fe32..b5df9bcbbeba 100644
--- a/drivers/usb/gadget/function/u_uac1_legacy.h
+++ b/drivers/usb/gadget/function/u_uac1_legacy.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * u_uac1.h -- interface to USB gadget "ALSA AUDIO" utilities
  *
diff --git a/drivers/usb/gadget/function/u_uac2.h b/drivers/usb/gadget/function/u_uac2.h
index 82048791eb6e..b5035711172d 100644
--- a/drivers/usb/gadget/function/u_uac2.h
+++ b/drivers/usb/gadget/function/u_uac2.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_uac2.h
  *
diff --git a/drivers/usb/gadget/function/u_uvc.h b/drivers/usb/gadget/function/u_uvc.h
index 16da49a2fcf2..9a01a7d4f17f 100644
--- a/drivers/usb/gadget/function/u_uvc.h
+++ b/drivers/usb/gadget/function/u_uvc.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * u_uvc.h
  *
diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h
index 1473d25ff17a..23ee25383c1f 100644
--- a/drivers/usb/gadget/function/uvc.h
+++ b/drivers/usb/gadget/function/uvc.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  *	uvc_gadget.h  --  USB Video Class Gadget driver
  *
@@ -77,6 +77,8 @@ struct uvc_video {
 	struct uvc_device *uvc;
 	struct usb_ep *ep;
 
+	struct work_struct pump;
+
 	/* Frame parameters */
 	u8 bpp;
 	u32 fcc;
diff --git a/drivers/usb/gadget/function/uvc_configfs.h b/drivers/usb/gadget/function/uvc_configfs.h
index 341391dbc81f..7e1d7ca29bf2 100644
--- a/drivers/usb/gadget/function/uvc_configfs.h
+++ b/drivers/usb/gadget/function/uvc_configfs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * uvc_configfs.h
  *
diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c
index 495f0ec663ea..4ca89eab6159 100644
--- a/drivers/usb/gadget/function/uvc_v4l2.c
+++ b/drivers/usb/gadget/function/uvc_v4l2.c
@@ -169,7 +169,9 @@ uvc_v4l2_qbuf(struct file *file, void *fh, struct v4l2_buffer *b)
 	if (ret < 0)
 		return ret;
 
-	return uvcg_video_pump(video);
+	schedule_work(&video->pump);
+
+	return ret;
 }
 
 static int
diff --git a/drivers/usb/gadget/function/uvc_v4l2.h b/drivers/usb/gadget/function/uvc_v4l2.h
index 452d71059b3f..1576005b61fd 100644
--- a/drivers/usb/gadget/function/uvc_v4l2.h
+++ b/drivers/usb/gadget/function/uvc_v4l2.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *	uvc_v4l2.h  --  USB Video Class Gadget driver
  *
diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c
index 5c042f380708..633e23d58d86 100644
--- a/drivers/usb/gadget/function/uvc_video.c
+++ b/drivers/usb/gadget/function/uvc_video.c
@@ -142,44 +142,12 @@ static int uvcg_video_ep_queue(struct uvc_video *video, struct usb_request *req)
 	return ret;
 }
 
-/*
- * I somehow feel that synchronisation won't be easy to achieve here. We have
- * three events that control USB requests submission:
- *
- * - USB request completion: the completion handler will resubmit the request
- *   if a video buffer is available.
- *
- * - USB interface setting selection: in response to a SET_INTERFACE request,
- *   the handler will start streaming if a video buffer is available and if
- *   video is not currently streaming.
- *
- * - V4L2 buffer queueing: the driver will start streaming if video is not
- *   currently streaming.
- *
- * Race conditions between those 3 events might lead to deadlocks or other
- * nasty side effects.
- *
- * The "video currently streaming" condition can't be detected by the irqqueue
- * being empty, as a request can still be in flight. A separate "queue paused"
- * flag is thus needed.
- *
- * The paused flag will be set when we try to retrieve the irqqueue head if the
- * queue is empty, and cleared when we queue a buffer.
- *
- * The USB request completion handler will get the buffer at the irqqueue head
- * under protection of the queue spinlock. If the queue is empty, the streaming
- * paused flag will be set. Right after releasing the spinlock a userspace
- * application can queue a buffer. The flag will then cleared, and the ioctl
- * handler will restart the video stream.
- */
 static void
 uvc_video_complete(struct usb_ep *ep, struct usb_request *req)
 {
 	struct uvc_video *video = req->context;
 	struct uvc_video_queue *queue = &video->queue;
-	struct uvc_buffer *buf;
 	unsigned long flags;
-	int ret;
 
 	switch (req->status) {
 	case 0:
@@ -188,39 +156,20 @@ uvc_video_complete(struct usb_ep *ep, struct usb_request *req)
 	case -ESHUTDOWN:	/* disconnect from host. */
 		uvcg_dbg(&video->uvc->func, "VS request cancelled.\n");
 		uvcg_queue_cancel(queue, 1);
-		goto requeue;
+		break;
 
 	default:
 		uvcg_info(&video->uvc->func,
 			  "VS request completed with status %d.\n",
 			  req->status);
 		uvcg_queue_cancel(queue, 0);
-		goto requeue;
 	}
 
-	spin_lock_irqsave(&video->queue.irqlock, flags);
-	buf = uvcg_queue_head(&video->queue);
-	if (buf == NULL) {
-		spin_unlock_irqrestore(&video->queue.irqlock, flags);
-		goto requeue;
-	}
-
-	video->encode(req, video, buf);
-
-	ret = uvcg_video_ep_queue(video, req);
-	spin_unlock_irqrestore(&video->queue.irqlock, flags);
-
-	if (ret < 0) {
-		uvcg_queue_cancel(queue, 0);
-		goto requeue;
-	}
-
-	return;
-
-requeue:
 	spin_lock_irqsave(&video->req_lock, flags);
 	list_add_tail(&req->list, &video->req_free);
 	spin_unlock_irqrestore(&video->req_lock, flags);
+
+	schedule_work(&video->pump);
 }
 
 static int
@@ -294,18 +243,15 @@ error:
  * This function fills the available USB requests (listed in req_free) with
  * video data from the queued buffers.
  */
-int uvcg_video_pump(struct uvc_video *video)
+static void uvcg_video_pump(struct work_struct *work)
 {
+	struct uvc_video *video = container_of(work, struct uvc_video, pump);
 	struct uvc_video_queue *queue = &video->queue;
 	struct usb_request *req;
 	struct uvc_buffer *buf;
 	unsigned long flags;
 	int ret;
 
-	/* FIXME TODO Race between uvcg_video_pump and requests completion
-	 * handler ???
-	 */
-
 	while (1) {
 		/* Retrieve the first available USB request, protected by the
 		 * request lock.
@@ -313,7 +259,7 @@ int uvcg_video_pump(struct uvc_video *video)
 		spin_lock_irqsave(&video->req_lock, flags);
 		if (list_empty(&video->req_free)) {
 			spin_unlock_irqrestore(&video->req_lock, flags);
-			return 0;
+			return;
 		}
 		req = list_first_entry(&video->req_free, struct usb_request,
 					list);
@@ -345,7 +291,7 @@ int uvcg_video_pump(struct uvc_video *video)
 	spin_lock_irqsave(&video->req_lock, flags);
 	list_add_tail(&req->list, &video->req_free);
 	spin_unlock_irqrestore(&video->req_lock, flags);
-	return 0;
+	return;
 }
 
 /*
@@ -363,6 +309,9 @@ int uvcg_video_enable(struct uvc_video *video, int enable)
 	}
 
 	if (!enable) {
+		cancel_work_sync(&video->pump);
+		uvcg_queue_cancel(&video->queue, 0);
+
 		for (i = 0; i < UVC_NUM_REQUESTS; ++i)
 			if (video->req[i])
 				usb_ep_dequeue(video->ep, video->req[i]);
@@ -384,7 +333,9 @@ int uvcg_video_enable(struct uvc_video *video, int enable)
 	} else
 		video->encode = uvc_video_encode_isoc;
 
-	return uvcg_video_pump(video);
+	schedule_work(&video->pump);
+
+	return ret;
 }
 
 /*
@@ -394,6 +345,7 @@ int uvcg_video_init(struct uvc_video *video, struct uvc_device *uvc)
 {
 	INIT_LIST_HEAD(&video->req_free);
 	spin_lock_init(&video->req_lock);
+	INIT_WORK(&video->pump, uvcg_video_pump);
 
 	video->uvc = uvc;
 	video->fcc = V4L2_PIX_FMT_YUYV;
diff --git a/drivers/usb/gadget/function/uvc_video.h b/drivers/usb/gadget/function/uvc_video.h
index dff12103f696..03adeefa343b 100644
--- a/drivers/usb/gadget/function/uvc_video.h
+++ b/drivers/usb/gadget/function/uvc_video.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *	uvc_video.h  --  USB Video Class Gadget driver
  *
@@ -14,8 +14,6 @@
 
 struct uvc_video;
 
-int uvcg_video_pump(struct uvc_video *video);
-
 int uvcg_video_enable(struct uvc_video *video, int enable);
 
 int uvcg_video_init(struct uvc_video *video, struct uvc_device *uvc);
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index 3afddd3bea6e..9ee0bfe7bcda 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -21,7 +21,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/poll.h>
-#include <linux/mmu_context.h>
+#include <linux/kthread.h>
 #include <linux/aio.h>
 #include <linux/uio.h>
 #include <linux/refcount.h>
@@ -462,9 +462,9 @@ static void ep_user_copy_worker(struct work_struct *work)
 	struct kiocb *iocb = priv->iocb;
 	size_t ret;
 
-	use_mm(mm);
+	kthread_use_mm(mm);
 	ret = copy_to_iter(priv->buf, priv->actual, &priv->to);
-	unuse_mm(mm);
+	kthread_unuse_mm(mm);
 	if (!ret)
 		ret = -EFAULT;
 
diff --git a/drivers/usb/gadget/legacy/mass_storage.c b/drivers/usb/gadget/legacy/mass_storage.c
index f18f77584fc2..9ed22c5fb7fe 100644
--- a/drivers/usb/gadget/legacy/mass_storage.c
+++ b/drivers/usb/gadget/legacy/mass_storage.c
@@ -229,18 +229,8 @@ static struct usb_composite_driver msg_driver = {
 	.unbind		= msg_unbind,
 };
 
+module_usb_composite_driver(msg_driver);
+
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_AUTHOR("Michal Nazarewicz");
 MODULE_LICENSE("GPL");
-
-static int __init msg_init(void)
-{
-	return usb_composite_probe(&msg_driver);
-}
-module_init(msg_init);
-
-static void __exit msg_cleanup(void)
-{
-	usb_composite_unregister(&msg_driver);
-}
-module_exit(msg_cleanup);
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/core.c b/drivers/usb/gadget/udc/aspeed-vhub/core.c
index f8d35dd60c34..cdf96911e4b1 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/core.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/core.c
@@ -134,11 +134,15 @@ static irqreturn_t ast_vhub_irq(int irq, void *data)
 	}
 
 	/* Handle device interrupts */
-	for (i = 0; i < vhub->max_ports; i++) {
-		u32 dev_mask = VHUB_IRQ_DEVICE1 << i;
+	if (istat & vhub->port_irq_mask) {
+		unsigned long bitmap = istat;
+		int offset = VHUB_IRQ_DEV1_BIT;
+		int size = VHUB_IRQ_DEV1_BIT + vhub->max_ports;
 
-		if (istat & dev_mask)
+		for_each_set_bit_from(offset, &bitmap, size) {
+			i = offset - VHUB_IRQ_DEV1_BIT;
 			ast_vhub_dev_irq(&vhub->ports[i].dev);
+		}
 	}
 
 	/* Handle top-level vHub EP0 interrupts */
@@ -332,6 +336,8 @@ static int ast_vhub_probe(struct platform_device *pdev)
 
 	spin_lock_init(&vhub->lock);
 	vhub->pdev = pdev;
+	vhub->port_irq_mask = GENMASK(VHUB_IRQ_DEV1_BIT + vhub->max_ports - 1,
+				      VHUB_IRQ_DEV1_BIT);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	vhub->regs = devm_ioremap_resource(&pdev->dev, res);
@@ -402,7 +408,9 @@ static int ast_vhub_probe(struct platform_device *pdev)
 		goto err;
 
 	/* Init hub emulation */
-	ast_vhub_init_hub(vhub);
+	rc = ast_vhub_init_hub(vhub);
+	if (rc)
+		goto err;
 
 	/* Initialize HW */
 	ast_vhub_init_hw(vhub);
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/hub.c b/drivers/usb/gadget/udc/aspeed-vhub/hub.c
index 6e565c3dbb5b..6497185ec4e7 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/hub.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/hub.c
@@ -50,6 +50,7 @@
 #define KERNEL_VER	bin2bcd(((LINUX_VERSION_CODE >> 8) & 0x0ff))
 
 enum {
+	AST_VHUB_STR_INDEX_MAX = 4,
 	AST_VHUB_STR_MANUF = 3,
 	AST_VHUB_STR_PRODUCT = 2,
 	AST_VHUB_STR_SERIAL = 1,
@@ -72,13 +73,6 @@ static const struct usb_device_descriptor ast_vhub_dev_desc = {
 	.bNumConfigurations	= 1,
 };
 
-/* Patches to the above when forcing USB1 mode */
-static void ast_vhub_patch_dev_desc_usb1(struct usb_device_descriptor *desc)
-{
-	desc->bcdUSB = cpu_to_le16(0x0100);
-	desc->bDeviceProtocol = 0;
-}
-
 /*
  * Configuration descriptor: same comments as above
  * regarding handling USB1 mode.
@@ -302,31 +296,81 @@ static int ast_vhub_rep_desc(struct ast_vhub_ep *ep,
 	if (len > dsize)
 		len = dsize;
 
-	/* Patch it if forcing USB1 */
-	if (desc_type == USB_DT_DEVICE && ep->vhub->force_usb1)
-		ast_vhub_patch_dev_desc_usb1(ep->buf);
-
 	/* Shoot it from the EP buffer */
 	return ast_vhub_reply(ep, NULL, len);
 }
 
+static struct usb_gadget_strings*
+ast_vhub_str_of_container(struct usb_gadget_string_container *container)
+{
+	return (struct usb_gadget_strings *)container->stash;
+}
+
+static int ast_vhub_collect_languages(struct ast_vhub *vhub, void *buf,
+				      size_t size)
+{
+	int rc, hdr_len, nlangs, max_langs;
+	struct usb_gadget_strings *lang_str;
+	struct usb_gadget_string_container *container;
+	struct usb_string_descriptor *sdesc = buf;
+
+	nlangs = 0;
+	hdr_len = sizeof(struct usb_descriptor_header);
+	max_langs = (size - hdr_len) / sizeof(sdesc->wData[0]);
+	list_for_each_entry(container, &vhub->vhub_str_desc, list) {
+		if (nlangs >= max_langs)
+			break;
+
+		lang_str = ast_vhub_str_of_container(container);
+		sdesc->wData[nlangs++] = cpu_to_le16(lang_str->language);
+	}
+
+	rc = hdr_len + nlangs * sizeof(sdesc->wData[0]);
+	sdesc->bLength = rc;
+	sdesc->bDescriptorType = USB_DT_STRING;
+
+	return rc;
+}
+
+static struct usb_gadget_strings *ast_vhub_lookup_string(struct ast_vhub *vhub,
+							 u16 lang_id)
+{
+	struct usb_gadget_strings *lang_str;
+	struct usb_gadget_string_container *container;
+
+	list_for_each_entry(container, &vhub->vhub_str_desc, list) {
+		lang_str = ast_vhub_str_of_container(container);
+		if (lang_str->language == lang_id)
+			return lang_str;
+	}
+
+	return NULL;
+}
+
 static int ast_vhub_rep_string(struct ast_vhub_ep *ep,
 			       u8 string_id, u16 lang_id,
 			       u16 len)
 {
-	int rc = usb_gadget_get_string(&ep->vhub->vhub_str_desc,
-					string_id, ep->buf);
+	int rc;
+	u8 buf[256];
+	struct ast_vhub *vhub = ep->vhub;
+	struct usb_gadget_strings *lang_str;
 
-	/*
-	 * This should never happen unless we put too big strings in
-	 * the array above
-	 */
-	BUG_ON(rc >= AST_VHUB_EP0_MAX_PACKET);
+	if (string_id == 0) {
+		rc = ast_vhub_collect_languages(vhub, buf, sizeof(buf));
+	} else {
+		lang_str = ast_vhub_lookup_string(vhub, lang_id);
+		if (!lang_str)
+			return std_req_stall;
 
-	if (rc < 0)
+		rc = usb_gadget_get_string(lang_str, string_id, buf);
+	}
+
+	if (rc < 0 || rc >= AST_VHUB_EP0_MAX_PACKET)
 		return std_req_stall;
 
 	/* Shoot it from the EP buffer */
+	memcpy(ep->buf, buf, rc);
 	return ast_vhub_reply(ep, NULL, min_t(u16, rc, len));
 }
 
@@ -832,11 +876,148 @@ void ast_vhub_hub_reset(struct ast_vhub *vhub)
 	writel(0, vhub->regs + AST_VHUB_EP1_STS_CHG);
 }
 
-static void ast_vhub_init_desc(struct ast_vhub *vhub)
+static void ast_vhub_of_parse_dev_desc(struct ast_vhub *vhub,
+				       const struct device_node *vhub_np)
+{
+	u16 id;
+	u32 data;
+
+	if (!of_property_read_u32(vhub_np, "vhub-vendor-id", &data)) {
+		id = (u16)data;
+		vhub->vhub_dev_desc.idVendor = cpu_to_le16(id);
+	}
+	if (!of_property_read_u32(vhub_np, "vhub-product-id", &data)) {
+		id = (u16)data;
+		vhub->vhub_dev_desc.idProduct = cpu_to_le16(id);
+	}
+	if (!of_property_read_u32(vhub_np, "vhub-device-revision", &data)) {
+		id = (u16)data;
+		vhub->vhub_dev_desc.bcdDevice = cpu_to_le16(id);
+	}
+}
+
+static void ast_vhub_fixup_usb1_dev_desc(struct ast_vhub *vhub)
+{
+	vhub->vhub_dev_desc.bcdUSB = cpu_to_le16(0x0100);
+	vhub->vhub_dev_desc.bDeviceProtocol = 0;
+}
+
+static struct usb_gadget_string_container*
+ast_vhub_str_container_alloc(struct ast_vhub *vhub)
+{
+	unsigned int size;
+	struct usb_string *str_array;
+	struct usb_gadget_strings *lang_str;
+	struct usb_gadget_string_container *container;
+
+	size = sizeof(*container);
+	size += sizeof(struct usb_gadget_strings);
+	size += sizeof(struct usb_string) * AST_VHUB_STR_INDEX_MAX;
+	container = devm_kzalloc(&vhub->pdev->dev, size, GFP_KERNEL);
+	if (!container)
+		return ERR_PTR(-ENOMEM);
+
+	lang_str = ast_vhub_str_of_container(container);
+	str_array = (struct usb_string *)(lang_str + 1);
+	lang_str->strings = str_array;
+	return container;
+}
+
+static void ast_vhub_str_deep_copy(struct usb_gadget_strings *dest,
+				   const struct usb_gadget_strings *src)
 {
+	struct usb_string *src_array = src->strings;
+	struct usb_string *dest_array = dest->strings;
+
+	dest->language = src->language;
+	if (src_array && dest_array) {
+		do {
+			*dest_array = *src_array;
+			dest_array++;
+			src_array++;
+		} while (src_array->s);
+	}
+}
+
+static int ast_vhub_str_alloc_add(struct ast_vhub *vhub,
+				  const struct usb_gadget_strings *src_str)
+{
+	struct usb_gadget_strings *dest_str;
+	struct usb_gadget_string_container *container;
+
+	container = ast_vhub_str_container_alloc(vhub);
+	if (IS_ERR(container))
+		return PTR_ERR(container);
+
+	dest_str = ast_vhub_str_of_container(container);
+	ast_vhub_str_deep_copy(dest_str, src_str);
+	list_add_tail(&container->list, &vhub->vhub_str_desc);
+
+	return 0;
+}
+
+static const struct {
+	const char *name;
+	u8 id;
+} str_id_map[] = {
+	{"manufacturer",	AST_VHUB_STR_MANUF},
+	{"product",		AST_VHUB_STR_PRODUCT},
+	{"serial-number",	AST_VHUB_STR_SERIAL},
+	{},
+};
+
+static int ast_vhub_of_parse_str_desc(struct ast_vhub *vhub,
+				      const struct device_node *desc_np)
+{
+	u32 langid;
+	int ret = 0;
+	int i, offset;
+	const char *str;
+	struct device_node *child;
+	struct usb_string str_array[AST_VHUB_STR_INDEX_MAX];
+	struct usb_gadget_strings lang_str = {
+		.strings = (struct usb_string *)str_array,
+	};
+
+	for_each_child_of_node(desc_np, child) {
+		if (of_property_read_u32(child, "reg", &langid))
+			continue; /* no language identifier specified */
+
+		if (!usb_validate_langid(langid))
+			continue; /* invalid language identifier */
+
+		lang_str.language = langid;
+		for (i = offset = 0; str_id_map[i].name; i++) {
+			str = of_get_property(child, str_id_map[i].name, NULL);
+			if (str) {
+				str_array[offset].s = str;
+				str_array[offset].id = str_id_map[i].id;
+				offset++;
+			}
+		}
+		str_array[offset].id = 0;
+		str_array[offset].s = NULL;
+
+		ret = ast_vhub_str_alloc_add(vhub, &lang_str);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
+static int ast_vhub_init_desc(struct ast_vhub *vhub)
+{
+	int ret;
+	struct device_node *desc_np;
+	const struct device_node *vhub_np = vhub->pdev->dev.of_node;
+
 	/* Initialize vhub Device Descriptor. */
 	memcpy(&vhub->vhub_dev_desc, &ast_vhub_dev_desc,
 		sizeof(vhub->vhub_dev_desc));
+	ast_vhub_of_parse_dev_desc(vhub, vhub_np);
+	if (vhub->force_usb1)
+		ast_vhub_fixup_usb1_dev_desc(vhub);
 
 	/* Initialize vhub Configuration Descriptor. */
 	memcpy(&vhub->vhub_conf_desc, &ast_vhub_conf_desc,
@@ -848,15 +1029,20 @@ static void ast_vhub_init_desc(struct ast_vhub *vhub)
 	vhub->vhub_hub_desc.bNbrPorts = vhub->max_ports;
 
 	/* Initialize vhub String Descriptors. */
-	memcpy(&vhub->vhub_str_desc, &ast_vhub_strings,
-		sizeof(vhub->vhub_str_desc));
+	INIT_LIST_HEAD(&vhub->vhub_str_desc);
+	desc_np = of_get_child_by_name(vhub_np, "vhub-strings");
+	if (desc_np)
+		ret = ast_vhub_of_parse_str_desc(vhub, desc_np);
+	else
+		ret = ast_vhub_str_alloc_add(vhub, &ast_vhub_strings);
+
+	return ret;
 }
 
-void ast_vhub_init_hub(struct ast_vhub *vhub)
+int ast_vhub_init_hub(struct ast_vhub *vhub)
 {
 	vhub->speed = USB_SPEED_UNKNOWN;
 	INIT_WORK(&vhub->wake_work, ast_vhub_wake_work);
 
-	ast_vhub_init_desc(vhub);
+	return ast_vhub_init_desc(vhub);
 }
-
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/vhub.h b/drivers/usb/gadget/udc/aspeed-vhub/vhub.h
index fac79ef6d669..2e5a1ef14a75 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/vhub.h
+++ b/drivers/usb/gadget/udc/aspeed-vhub/vhub.h
@@ -51,14 +51,11 @@
 #define VHUB_CTRL_UPSTREAM_CONNECT		(1 << 0)
 
 /* IER & ISR */
+#define VHUB_IRQ_DEV1_BIT			9
 #define VHUB_IRQ_USB_CMD_DEADLOCK		(1 << 18)
 #define VHUB_IRQ_EP_POOL_NAK			(1 << 17)
 #define VHUB_IRQ_EP_POOL_ACK_STALL		(1 << 16)
-#define VHUB_IRQ_DEVICE5			(1 << 13)
-#define VHUB_IRQ_DEVICE4			(1 << 12)
-#define VHUB_IRQ_DEVICE3			(1 << 11)
-#define VHUB_IRQ_DEVICE2			(1 << 10)
-#define VHUB_IRQ_DEVICE1			(1 << 9)
+#define VHUB_IRQ_DEVICE1			(1 << (VHUB_IRQ_DEV1_BIT))
 #define VHUB_IRQ_BUS_RESUME			(1 << 8)
 #define VHUB_IRQ_BUS_SUSPEND 			(1 << 7)
 #define VHUB_IRQ_BUS_RESET 			(1 << 6)
@@ -402,6 +399,7 @@ struct ast_vhub {
 	/* Per-port info */
 	struct ast_vhub_port		*ports;
 	u32				max_ports;
+	u32				port_irq_mask;
 
 	/* Generic EP data structures */
 	struct ast_vhub_ep		*epns;
@@ -423,7 +421,7 @@ struct ast_vhub {
 	struct usb_device_descriptor	vhub_dev_desc;
 	struct ast_vhub_full_cdesc	vhub_conf_desc;
 	struct usb_hub_descriptor	vhub_hub_desc;
-	struct usb_gadget_strings	vhub_str_desc;
+	struct list_head		vhub_str_desc;
 };
 
 /* Standard request handlers result codes */
@@ -533,7 +531,7 @@ int __ast_vhub_simple_reply(struct ast_vhub_ep *ep, int len, ...);
 			       __VA_ARGS__)
 
 /* hub.c */
-void ast_vhub_init_hub(struct ast_vhub *vhub);
+int ast_vhub_init_hub(struct ast_vhub *vhub);
 enum std_req_rc ast_vhub_std_hub_request(struct ast_vhub_ep *ep,
 					 struct usb_ctrlrequest *crq);
 enum std_req_rc ast_vhub_class_hub_request(struct ast_vhub_ep *ep,
diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.c b/drivers/usb/gadget/udc/atmel_usba_udc.c
index b771a854e29c..d69f61ff0181 100644
--- a/drivers/usb/gadget/udc/atmel_usba_udc.c
+++ b/drivers/usb/gadget/udc/atmel_usba_udc.c
@@ -2043,10 +2043,56 @@ static const struct usba_udc_errata at91sam9g45_errata = {
 	.pulse_bias = at91sam9g45_pulse_bias,
 };
 
+static const struct usba_ep_config ep_config_sam9[] __initconst = {
+	{ .nr_banks = 1 },				/* ep 0 */
+	{ .nr_banks = 2, .can_dma = 1, .can_isoc = 1 },	/* ep 1 */
+	{ .nr_banks = 2, .can_dma = 1, .can_isoc = 1 },	/* ep 2 */
+	{ .nr_banks = 3, .can_dma = 1 },		/* ep 3 */
+	{ .nr_banks = 3, .can_dma = 1 },		/* ep 4 */
+	{ .nr_banks = 3, .can_dma = 1, .can_isoc = 1 },	/* ep 5 */
+	{ .nr_banks = 3, .can_dma = 1, .can_isoc = 1 },	/* ep 6 */
+};
+
+static const struct usba_ep_config ep_config_sama5[] __initconst = {
+	{ .nr_banks = 1 },				/* ep 0 */
+	{ .nr_banks = 3, .can_dma = 1, .can_isoc = 1 },	/* ep 1 */
+	{ .nr_banks = 3, .can_dma = 1, .can_isoc = 1 },	/* ep 2 */
+	{ .nr_banks = 2, .can_dma = 1, .can_isoc = 1 },	/* ep 3 */
+	{ .nr_banks = 2, .can_dma = 1, .can_isoc = 1 },	/* ep 4 */
+	{ .nr_banks = 2, .can_dma = 1, .can_isoc = 1 },	/* ep 5 */
+	{ .nr_banks = 2, .can_dma = 1, .can_isoc = 1 },	/* ep 6 */
+	{ .nr_banks = 2, .can_dma = 1, .can_isoc = 1 },	/* ep 7 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 8 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 9 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 10 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 11 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 12 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 13 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 14 */
+	{ .nr_banks = 2, .can_isoc = 1 },		/* ep 15 */
+};
+
+static const struct usba_udc_config udc_at91sam9rl_cfg = {
+	.errata = &at91sam9rl_errata,
+	.config = ep_config_sam9,
+	.num_ep = ARRAY_SIZE(ep_config_sam9),
+};
+
+static const struct usba_udc_config udc_at91sam9g45_cfg = {
+	.errata = &at91sam9g45_errata,
+	.config = ep_config_sam9,
+	.num_ep = ARRAY_SIZE(ep_config_sam9),
+};
+
+static const struct usba_udc_config udc_sama5d3_cfg = {
+	.config = ep_config_sama5,
+	.num_ep = ARRAY_SIZE(ep_config_sama5),
+};
+
 static const struct of_device_id atmel_udc_dt_ids[] = {
-	{ .compatible = "atmel,at91sam9rl-udc", .data = &at91sam9rl_errata },
-	{ .compatible = "atmel,at91sam9g45-udc", .data = &at91sam9g45_errata },
-	{ .compatible = "atmel,sama5d3-udc" },
+	{ .compatible = "atmel,at91sam9rl-udc", .data = &udc_at91sam9rl_cfg },
+	{ .compatible = "atmel,at91sam9g45-udc", .data = &udc_at91sam9g45_cfg },
+	{ .compatible = "atmel,sama5d3-udc", .data = &udc_sama5d3_cfg },
 	{ /* sentinel */ }
 };
 
@@ -2055,18 +2101,19 @@ MODULE_DEVICE_TABLE(of, atmel_udc_dt_ids);
 static struct usba_ep * atmel_udc_of_init(struct platform_device *pdev,
 						    struct usba_udc *udc)
 {
-	u32 val;
 	struct device_node *np = pdev->dev.of_node;
 	const struct of_device_id *match;
 	struct device_node *pp;
 	int i, ret;
 	struct usba_ep *eps, *ep;
+	const struct usba_udc_config *udc_config;
 
 	match = of_match_node(atmel_udc_dt_ids, np);
 	if (!match)
 		return ERR_PTR(-EINVAL);
 
-	udc->errata = match->data;
+	udc_config = match->data;
+	udc->errata = udc_config->errata;
 	udc->pmc = syscon_regmap_lookup_by_compatible("atmel,at91sam9g45-pmc");
 	if (IS_ERR(udc->pmc))
 		udc->pmc = syscon_regmap_lookup_by_compatible("atmel,at91sam9rl-pmc");
@@ -2082,8 +2129,7 @@ static struct usba_ep * atmel_udc_of_init(struct platform_device *pdev,
 
 	if (fifo_mode == 0) {
 		pp = NULL;
-		while ((pp = of_get_next_child(np, pp)))
-			udc->num_ep++;
+		udc->num_ep = udc_config->num_ep;
 		udc->configured_ep = 1;
 	} else {
 		udc->num_ep = usba_config_fifo_table(udc);
@@ -2100,52 +2146,38 @@ static struct usba_ep * atmel_udc_of_init(struct platform_device *pdev,
 
 	pp = NULL;
 	i = 0;
-	while ((pp = of_get_next_child(np, pp)) && i < udc->num_ep) {
+	while (i < udc->num_ep) {
+		const struct usba_ep_config *ep_cfg = &udc_config->config[i];
+
 		ep = &eps[i];
 
-		ret = of_property_read_u32(pp, "reg", &val);
-		if (ret) {
-			dev_err(&pdev->dev, "of_probe: reg error(%d)\n", ret);
-			goto err;
-		}
-		ep->index = fifo_mode ? udc->fifo_cfg[i].hw_ep_num : val;
+		ep->index = fifo_mode ? udc->fifo_cfg[i].hw_ep_num : i;
+
+		/* Only the first EP is 64 bytes */
+		if (ep->index == 0)
+			ep->fifo_size = 64;
+		else
+			ep->fifo_size = 1024;
 
-		ret = of_property_read_u32(pp, "atmel,fifo-size", &val);
-		if (ret) {
-			dev_err(&pdev->dev, "of_probe: fifo-size error(%d)\n", ret);
-			goto err;
-		}
 		if (fifo_mode) {
-			if (val < udc->fifo_cfg[i].fifo_size) {
+			if (ep->fifo_size < udc->fifo_cfg[i].fifo_size)
 				dev_warn(&pdev->dev,
-					 "Using max fifo-size value from DT\n");
-				ep->fifo_size = val;
-			} else {
+					 "Using default max fifo-size value\n");
+			else
 				ep->fifo_size = udc->fifo_cfg[i].fifo_size;
-			}
-		} else {
-			ep->fifo_size = val;
 		}
 
-		ret = of_property_read_u32(pp, "atmel,nb-banks", &val);
-		if (ret) {
-			dev_err(&pdev->dev, "of_probe: nb-banks error(%d)\n", ret);
-			goto err;
-		}
+		ep->nr_banks = ep_cfg->nr_banks;
 		if (fifo_mode) {
-			if (val < udc->fifo_cfg[i].nr_banks) {
+			if (ep->nr_banks < udc->fifo_cfg[i].nr_banks)
 				dev_warn(&pdev->dev,
-					 "Using max nb-banks value from DT\n");
-				ep->nr_banks = val;
-			} else {
+					 "Using default max nb-banks value\n");
+			else
 				ep->nr_banks = udc->fifo_cfg[i].nr_banks;
-			}
-		} else {
-			ep->nr_banks = val;
 		}
 
-		ep->can_dma = of_property_read_bool(pp, "atmel,can-dma");
-		ep->can_isoc = of_property_read_bool(pp, "atmel,can-isoc");
+		ep->can_dma = ep_cfg->can_dma;
+		ep->can_isoc = ep_cfg->can_isoc;
 
 		sprintf(ep->name, "ep%d", ep->index);
 		ep->ep.name = ep->name;
diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.h b/drivers/usb/gadget/udc/atmel_usba_udc.h
index a0225e4543d4..48e332439ed5 100644
--- a/drivers/usb/gadget/udc/atmel_usba_udc.h
+++ b/drivers/usb/gadget/udc/atmel_usba_udc.h
@@ -290,6 +290,12 @@ struct usba_ep {
 #endif
 };
 
+struct usba_ep_config {
+	u8					nr_banks;
+	unsigned int				can_dma:1;
+	unsigned int				can_isoc:1;
+};
+
 struct usba_request {
 	struct usb_request			req;
 	struct list_head			queue;
@@ -307,6 +313,12 @@ struct usba_udc_errata {
 	void (*pulse_bias)(struct usba_udc *udc);
 };
 
+struct usba_udc_config {
+	const struct usba_udc_errata *errata;
+	const struct usba_ep_config *config;
+	const int num_ep;
+};
+
 struct usba_udc {
 	/* Protect hw registers from concurrent modifications */
 	spinlock_t lock;
diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c
index 9b11046480fe..2e28dde8376f 100644
--- a/drivers/usb/gadget/udc/core.c
+++ b/drivers/usb/gadget/udc/core.c
@@ -1297,6 +1297,8 @@ static void usb_gadget_remove_driver(struct usb_udc *udc)
 	kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE);
 
 	usb_gadget_disconnect(udc->gadget);
+	if (udc->gadget->irq)
+		synchronize_irq(udc->gadget->irq);
 	udc->driver->unbind(udc->gadget);
 	usb_gadget_udc_stop(udc);
 
diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
index 6e3e3ebf715f..0eeaead5acea 100644
--- a/drivers/usb/gadget/udc/dummy_hcd.c
+++ b/drivers/usb/gadget/udc/dummy_hcd.c
@@ -187,31 +187,31 @@ static const struct {
 		USB_EP_CAPS(USB_EP_CAPS_TYPE_BULK, USB_EP_CAPS_DIR_IN)),
 
 	/* and now some generic EPs so we have enough in multi config */
-	EP_INFO("ep3out",
+	EP_INFO("ep-aout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
-	EP_INFO("ep4in",
+	EP_INFO("ep-bin",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_IN)),
-	EP_INFO("ep5out",
+	EP_INFO("ep-cout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
-	EP_INFO("ep6out",
+	EP_INFO("ep-dout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
-	EP_INFO("ep7in",
+	EP_INFO("ep-ein",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_IN)),
-	EP_INFO("ep8out",
+	EP_INFO("ep-fout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
-	EP_INFO("ep9in",
+	EP_INFO("ep-gin",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_IN)),
-	EP_INFO("ep10out",
+	EP_INFO("ep-hout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
-	EP_INFO("ep11out",
+	EP_INFO("ep-iout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
-	EP_INFO("ep12in",
+	EP_INFO("ep-jin",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_IN)),
-	EP_INFO("ep13out",
+	EP_INFO("ep-kout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
-	EP_INFO("ep14in",
+	EP_INFO("ep-lin",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_IN)),
-	EP_INFO("ep15out",
+	EP_INFO("ep-mout",
 		USB_EP_CAPS(TYPE_BULK_OR_INT, USB_EP_CAPS_DIR_OUT)),
 
 #undef EP_INFO
@@ -427,6 +427,7 @@ static void set_link_state_by_speed(struct dummy_hcd *dum_hcd)
 
 /* caller must hold lock */
 static void set_link_state(struct dummy_hcd *dum_hcd)
+	__must_hold(&dum->lock)
 {
 	struct dummy *dum = dum_hcd->dum;
 	unsigned int power_bit;
diff --git a/drivers/usb/gadget/udc/fsl_udc_core.c b/drivers/usb/gadget/udc/fsl_udc_core.c
index febabde62f71..b2638e83bb49 100644
--- a/drivers/usb/gadget/udc/fsl_udc_core.c
+++ b/drivers/usb/gadget/udc/fsl_udc_core.c
@@ -2440,8 +2440,8 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	udc_controller->max_ep = (dccparams & DCCPARAMS_DEN_MASK) * 2;
 
 	udc_controller->irq = platform_get_irq(pdev, 0);
-	if (!udc_controller->irq) {
-		ret = -ENODEV;
+	if (udc_controller->irq <= 0) {
+		ret = udc_controller->irq ? : -ENODEV;
 		goto err_iounmap;
 	}
 
diff --git a/drivers/usb/gadget/udc/gr_udc.c b/drivers/usb/gadget/udc/gr_udc.c
index aaf975c809bf..7164ad9800f1 100644
--- a/drivers/usb/gadget/udc/gr_udc.c
+++ b/drivers/usb/gadget/udc/gr_udc.c
@@ -48,7 +48,6 @@
 #define	DRIVER_DESC	"Aeroflex Gaisler GRUSBDC USB Peripheral Controller"
 
 static const char driver_name[] = DRIVER_NAME;
-static const char driver_desc[] = DRIVER_DESC;
 
 #define gr_read32(x) (ioread32be((x)))
 #define gr_write32(x, v) (iowrite32be((v), (x)))
diff --git a/drivers/usb/gadget/udc/lpc32xx_udc.c b/drivers/usb/gadget/udc/lpc32xx_udc.c
index cb997b82c008..465d0b7c6522 100644
--- a/drivers/usb/gadget/udc/lpc32xx_udc.c
+++ b/drivers/usb/gadget/udc/lpc32xx_udc.c
@@ -1614,17 +1614,17 @@ static int lpc32xx_ep_enable(struct usb_ep *_ep,
 			     const struct usb_endpoint_descriptor *desc)
 {
 	struct lpc32xx_ep *ep = container_of(_ep, struct lpc32xx_ep, ep);
-	struct lpc32xx_udc *udc = ep->udc;
+	struct lpc32xx_udc *udc;
 	u16 maxpacket;
 	u32 tmp;
 	unsigned long flags;
 
 	/* Verify EP data */
 	if ((!_ep) || (!ep) || (!desc) ||
-	    (desc->bDescriptorType != USB_DT_ENDPOINT)) {
-		dev_dbg(udc->dev, "bad ep or descriptor\n");
+	    (desc->bDescriptorType != USB_DT_ENDPOINT))
 		return -EINVAL;
-	}
+
+	udc = ep->udc;
 	maxpacket = usb_endpoint_maxp(desc);
 	if ((maxpacket == 0) || (maxpacket > ep->maxpacket)) {
 		dev_dbg(udc->dev, "bad ep descriptor's packet size\n");
@@ -1872,7 +1872,7 @@ static int lpc32xx_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
 static int lpc32xx_ep_set_halt(struct usb_ep *_ep, int value)
 {
 	struct lpc32xx_ep *ep = container_of(_ep, struct lpc32xx_ep, ep);
-	struct lpc32xx_udc *udc = ep->udc;
+	struct lpc32xx_udc *udc;
 	unsigned long flags;
 
 	if ((!ep) || (ep->hwep_num <= 1))
@@ -1882,6 +1882,7 @@ static int lpc32xx_ep_set_halt(struct usb_ep *_ep, int value)
 	if (ep->is_in)
 		return -EAGAIN;
 
+	udc = ep->udc;
 	spin_lock_irqsave(&udc->lock, flags);
 
 	if (value == 1) {
diff --git a/drivers/usb/gadget/udc/m66592-udc.c b/drivers/usb/gadget/udc/m66592-udc.c
index 75d16a8902e6..931e6362a13d 100644
--- a/drivers/usb/gadget/udc/m66592-udc.c
+++ b/drivers/usb/gadget/udc/m66592-udc.c
@@ -1667,7 +1667,7 @@ static int m66592_probe(struct platform_device *pdev)
 
 err_add_udc:
 	m66592_free_request(&m66592->ep[0].ep, m66592->ep0_req);
-
+	m66592->ep0_req = NULL;
 clean_up3:
 	if (m66592->pdata->on_chip) {
 		clk_disable(m66592->clk);
diff --git a/drivers/usb/gadget/udc/max3420_udc.c b/drivers/usb/gadget/udc/max3420_udc.c
index 8fbc083b6732..23f33946d80c 100644
--- a/drivers/usb/gadget/udc/max3420_udc.c
+++ b/drivers/usb/gadget/udc/max3420_udc.c
@@ -901,7 +901,7 @@ loop:
 	}
 
 	set_current_state(TASK_RUNNING);
-	dev_info(udc->dev, "SPI thread exiting");
+	dev_info(udc->dev, "SPI thread exiting\n");
 	return 0;
 }
 
diff --git a/drivers/usb/gadget/udc/mv_u3d_core.c b/drivers/usb/gadget/udc/mv_u3d_core.c
index 35e02a8d0091..5bb0568b934e 100644
--- a/drivers/usb/gadget/udc/mv_u3d_core.c
+++ b/drivers/usb/gadget/udc/mv_u3d_core.c
@@ -1548,7 +1548,7 @@ static void mv_u3d_handle_setup_packet(struct mv_u3d *u3d, u8 ep_num,
 		delegate = true;
 
 	/* delegate USB standard requests to the gadget driver */
-	if (delegate == true) {
+	if (delegate) {
 		/* USB requests handled by gadget */
 		if (setup->wLength) {
 			/* DATA phase from gadget, STATUS phase from u3d */
diff --git a/drivers/usb/gadget/udc/net2272.c b/drivers/usb/gadget/udc/net2272.c
index 5af0fe9c61d7..928057b206f1 100644
--- a/drivers/usb/gadget/udc/net2272.c
+++ b/drivers/usb/gadget/udc/net2272.c
@@ -54,7 +54,7 @@ static const char * const ep_name[] = {
  *
  * If use_dma is disabled, pio will be used instead.
  */
-static bool use_dma = 0;
+static bool use_dma = false;
 module_param(use_dma, bool, 0644);
 
 /*
diff --git a/drivers/usb/gadget/udc/omap_udc.c b/drivers/usb/gadget/udc/omap_udc.c
index bf87c6c0d7f6..4139da885651 100644
--- a/drivers/usb/gadget/udc/omap_udc.c
+++ b/drivers/usb/gadget/udc/omap_udc.c
@@ -2576,7 +2576,7 @@ omap_ep_setup(char *name, u8 addr, u8 type,
 	case USB_ENDPOINT_XFER_INT:
 		ep->ep.caps.type_int = true;
 		break;
-	};
+	}
 
 	if (addr & USB_DIR_IN)
 		ep->ep.caps.dir_in = true;
diff --git a/drivers/usb/gadget/udc/s3c2410_udc.c b/drivers/usb/gadget/udc/s3c2410_udc.c
index 0507a2ca0f55..80002d97b59d 100644
--- a/drivers/usb/gadget/udc/s3c2410_udc.c
+++ b/drivers/usb/gadget/udc/s3c2410_udc.c
@@ -251,10 +251,6 @@ static void s3c2410_udc_done(struct s3c2410_ep *ep,
 static void s3c2410_udc_nuke(struct s3c2410_udc *udc,
 		struct s3c2410_ep *ep, int status)
 {
-	/* Sanity check */
-	if (&ep->queue == NULL)
-		return;
-
 	while (!list_empty(&ep->queue)) {
 		struct s3c2410_request *req;
 		req = list_entry(ep->queue.next, struct s3c2410_request,
diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c
index dfabc54cdc27..bbe1a04686da 100644
--- a/drivers/usb/gadget/udc/tegra-xudc.c
+++ b/drivers/usb/gadget/udc/tegra-xudc.c
@@ -158,6 +158,30 @@
 #define  SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
 #define  SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
 					SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
+#define SSPX_CORE_CNT56 0x6fc
+#define  SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK GENMASK(19, 0)
+#define  SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(x) ((x) & \
+				SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK)
+#define SSPX_CORE_CNT57 0x700
+#define  SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK GENMASK(19, 0)
+#define  SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(x) ((x) & \
+				SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK)
+#define SSPX_CORE_CNT65 0x720
+#define  SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK GENMASK(19, 0)
+#define  SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(x) ((x) & \
+				SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK)
+#define SSPX_CORE_CNT66 0x724
+#define  SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK GENMASK(19, 0)
+#define  SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(x) ((x) & \
+				SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK)
+#define SSPX_CORE_CNT67 0x728
+#define  SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK GENMASK(19, 0)
+#define  SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(x) ((x) & \
+				SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK)
+#define SSPX_CORE_CNT72 0x73c
+#define  SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK GENMASK(19, 0)
+#define  SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(x) ((x) & \
+				SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK)
 #define SSPX_CORE_PADCTL4 0x750
 #define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
 #define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
@@ -492,6 +516,7 @@ struct tegra_xudc {
 	bool powergated;
 
 	struct usb_phy **usbphy;
+	struct usb_phy *curr_usbphy;
 	struct notifier_block vbus_nb;
 
 	struct completion disconnect_complete;
@@ -530,6 +555,7 @@ struct tegra_xudc_soc {
 	bool invalid_seq_num;
 	bool pls_quirk;
 	bool port_reset_quirk;
+	bool port_speed_quirk;
 	bool has_ipfs;
 };
 
@@ -599,6 +625,78 @@ static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
 		trb->control);
 }
 
+static void tegra_xudc_limit_port_speed(struct tegra_xudc *xudc)
+{
+	u32 val;
+
+	/* limit port speed to gen 1 */
+	val = xudc_readl(xudc, SSPX_CORE_CNT56);
+	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
+	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x260);
+	xudc_writel(xudc, val, SSPX_CORE_CNT56);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT57);
+	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
+	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x6D6);
+	xudc_writel(xudc, val, SSPX_CORE_CNT57);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT65);
+	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
+	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0x4B0);
+	xudc_writel(xudc, val, SSPX_CORE_CNT66);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT66);
+	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
+	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x4B0);
+	xudc_writel(xudc, val, SSPX_CORE_CNT66);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT67);
+	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
+	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x4B0);
+	xudc_writel(xudc, val, SSPX_CORE_CNT67);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT72);
+	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
+	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x10);
+	xudc_writel(xudc, val, SSPX_CORE_CNT72);
+}
+
+static void tegra_xudc_restore_port_speed(struct tegra_xudc *xudc)
+{
+	u32 val;
+
+	/* restore port speed to gen2 */
+	val = xudc_readl(xudc, SSPX_CORE_CNT56);
+	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
+	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x438);
+	xudc_writel(xudc, val, SSPX_CORE_CNT56);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT57);
+	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
+	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x528);
+	xudc_writel(xudc, val, SSPX_CORE_CNT57);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT65);
+	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
+	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0xE10);
+	xudc_writel(xudc, val, SSPX_CORE_CNT66);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT66);
+	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
+	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x348);
+	xudc_writel(xudc, val, SSPX_CORE_CNT66);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT67);
+	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
+	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x5a0);
+	xudc_writel(xudc, val, SSPX_CORE_CNT67);
+
+	val = xudc_readl(xudc, SSPX_CORE_CNT72);
+	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
+	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x1c21);
+	xudc_writel(xudc, val, SSPX_CORE_CNT72);
+}
+
 static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
 {
 	int err;
@@ -631,6 +729,9 @@ static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
 
 	reinit_completion(&xudc->disconnect_complete);
 
+	if (xudc->soc->port_speed_quirk)
+		tegra_xudc_restore_port_speed(xudc);
+
 	phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG, USB_ROLE_NONE);
 
 	pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
@@ -719,6 +820,7 @@ static int tegra_xudc_vbus_notify(struct notifier_block *nb,
 	if (!xudc->suspended && phy_index != -1) {
 		xudc->curr_utmi_phy = xudc->utmi_phy[phy_index];
 		xudc->curr_usb3_phy = xudc->usb3_phy[phy_index];
+		xudc->curr_usbphy = usbphy;
 		schedule_work(&xudc->usb_role_sw_work);
 	}
 
@@ -2042,6 +2144,20 @@ static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
 	return 0;
 }
 
+static int tegra_xudc_gadget_vbus_draw(struct usb_gadget *gadget,
+						unsigned int m_a)
+{
+	int ret = 0;
+	struct tegra_xudc *xudc = to_xudc(gadget);
+
+	dev_dbg(xudc->dev, "%s: %u mA\n", __func__, m_a);
+
+	if (xudc->curr_usbphy->chg_type == SDP_TYPE)
+		ret = usb_phy_set_power(xudc->curr_usbphy, m_a);
+
+	return ret;
+}
+
 static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
 {
 	struct tegra_xudc *xudc = to_xudc(gadget);
@@ -2058,6 +2174,7 @@ static struct usb_gadget_ops tegra_xudc_gadget_ops = {
 	.pullup = tegra_xudc_gadget_pullup,
 	.udc_start = tegra_xudc_gadget_start,
 	.udc_stop = tegra_xudc_gadget_stop,
+	.vbus_draw = tegra_xudc_gadget_vbus_draw,
 	.set_selfpowered = tegra_xudc_set_selfpowered,
 };
 
@@ -3274,6 +3391,9 @@ static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
 		xudc_writel(xudc, val, BLCG);
 	}
 
+	if (xudc->soc->port_speed_quirk)
+		tegra_xudc_limit_port_speed(xudc);
+
 	/* Set a reasonable U3 exit timer value. */
 	val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
 	val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
@@ -3506,6 +3626,7 @@ static struct tegra_xudc_soc tegra210_xudc_soc_data = {
 	.invalid_seq_num = true,
 	.pls_quirk = true,
 	.port_reset_quirk = true,
+	.port_speed_quirk = false,
 	.has_ipfs = true,
 };
 
@@ -3519,6 +3640,21 @@ static struct tegra_xudc_soc tegra186_xudc_soc_data = {
 	.invalid_seq_num = false,
 	.pls_quirk = false,
 	.port_reset_quirk = false,
+	.port_speed_quirk = false,
+	.has_ipfs = false,
+};
+
+static struct tegra_xudc_soc tegra194_xudc_soc_data = {
+	.clock_names = tegra186_xudc_clock_names,
+	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
+	.num_phys = 4,
+	.u1_enable = true,
+	.u2_enable = true,
+	.lpm_enable = true,
+	.invalid_seq_num = false,
+	.pls_quirk = false,
+	.port_reset_quirk = false,
+	.port_speed_quirk = true,
 	.has_ipfs = false,
 };
 
@@ -3531,6 +3667,10 @@ static const struct of_device_id tegra_xudc_of_match[] = {
 		.compatible = "nvidia,tegra186-xudc",
 		.data = &tegra186_xudc_soc_data
 	},
+	{
+		.compatible = "nvidia,tegra194-xudc",
+		.data = &tegra194_xudc_soc_data
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
diff --git a/drivers/usb/gadget/udc/udc-xilinx.c b/drivers/usb/gadget/udc/udc-xilinx.c
index b1cfc8279c3d..709553bdb233 100644
--- a/drivers/usb/gadget/udc/udc-xilinx.c
+++ b/drivers/usb/gadget/udc/udc-xilinx.c
@@ -1732,6 +1732,7 @@ static void xudc_set_clear_feature(struct xusb_udc *udc)
  * Process setup packet and delegate to gadget layer.
  */
 static void xudc_handle_setup(struct xusb_udc *udc)
+	__must_hold(&udc->lock)
 {
 	struct xusb_ep *ep0 = &udc->ep[0];
 	struct usb_ctrlrequest setup;
diff --git a/drivers/usb/gadget/usbstring.c b/drivers/usb/gadget/usbstring.c
index 7c24d1ce1088..58a4d3325090 100644
--- a/drivers/usb/gadget/usbstring.c
+++ b/drivers/usb/gadget/usbstring.c
@@ -65,3 +65,27 @@ usb_gadget_get_string (const struct usb_gadget_strings *table, int id, u8 *buf)
 	return buf [0];
 }
 EXPORT_SYMBOL_GPL(usb_gadget_get_string);
+
+/**
+ * usb_validate_langid - validate usb language identifiers
+ * @lang: usb language identifier
+ *
+ * Returns true for valid language identifier, otherwise false.
+ */
+bool usb_validate_langid(u16 langid)
+{
+	u16 primary_lang = langid & 0x3ff;	/* bit [9:0] */
+	u16 sub_lang = langid >> 10;		/* bit [15:10] */
+
+	switch (primary_lang) {
+	case 0:
+	case 0x62 ... 0xfe:
+	case 0x100 ... 0x3ff:
+		return false;
+	}
+	if (!sub_lang)
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL_GPL(usb_validate_langid);
diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig
index 55bdfdf11e4c..1cb3004ea7b2 100644
--- a/drivers/usb/host/Kconfig
+++ b/drivers/usb/host/Kconfig
@@ -20,7 +20,7 @@ config USB_C67X00_HCD
 config USB_XHCI_HCD
 	tristate "xHCI HCD (USB 3.0) support"
 	depends on HAS_DMA && HAS_IOMEM
-	---help---
+	help
 	  The eXtensible Host Controller Interface (xHCI) is standard for USB 3.0
 	  "SuperSpeed" host controller hardware.
 
@@ -31,7 +31,7 @@ if USB_XHCI_HCD
 config USB_XHCI_DBGCAP
 	bool "xHCI support for debug capability"
 	depends on TTY
-	---help---
+	help
 	  Say 'Y' to enable the support for the xHCI debug capability. Make
 	  sure that your xHCI host supports the extended debug capability and
 	  you want a TTY serial device based on the xHCI debug capability
@@ -40,12 +40,21 @@ config USB_XHCI_DBGCAP
 config USB_XHCI_PCI
 	tristate
 	depends on USB_PCI
+	depends on USB_XHCI_PCI_RENESAS || !USB_XHCI_PCI_RENESAS
 	default y
 
+config USB_XHCI_PCI_RENESAS
+	tristate "Support for additional Renesas xHCI controller with firwmare"
+	help
+	  Say 'Y' to enable the support for the Renesas xHCI controller with
+	  firwmare. Make sure you have the firwmare for the device and
+	  installed on your system for this device to work.
+	  If unsure, say 'N'.
+
 config USB_XHCI_PLATFORM
 	tristate "Generic xHCI driver for a platform device"
 	select USB_XHCI_RCAR if ARCH_RENESAS
-	---help---
+	help
 	  Adds an xHCI host driver for a generic platform device, which
 	  provides a memory space and an irq.
 	  It is also a prerequisite for platform specific drivers that
@@ -64,7 +73,7 @@ config USB_XHCI_MTK
 	tristate "xHCI support for MediaTek SoCs"
 	select MFD_SYSCON
 	depends on (MIPS && SOC_MT7621) || ARCH_MEDIATEK || COMPILE_TEST
-	---help---
+	help
 	  Say 'Y' to enable the support for the xHCI host controller
 	  found in MediaTek SoCs.
 	  If unsure, say N.
@@ -74,7 +83,7 @@ config USB_XHCI_MVEBU
 	select USB_XHCI_PLATFORM
 	depends on HAS_IOMEM
 	depends on ARCH_MVEBU || COMPILE_TEST
-	---help---
+	help
 	  Say 'Y' to enable the support for the xHCI host controller
 	  found in Marvell Armada 375/38x/37xx ARM SOCs.
 
@@ -82,7 +91,7 @@ config USB_XHCI_RCAR
 	tristate "xHCI support for Renesas R-Car SoCs"
 	depends on USB_XHCI_PLATFORM
 	depends on ARCH_RENESAS || COMPILE_TEST
-	---help---
+	help
 	  Say 'Y' to enable the support for the xHCI host controller
 	  found in Renesas R-Car ARM SoCs.
 
@@ -91,16 +100,36 @@ config USB_XHCI_TEGRA
 	depends on PHY_TEGRA_XUSB
 	depends on RESET_CONTROLLER
 	select FW_LOADER
-	---help---
+	help
 	  Say 'Y' to enable the support for the xHCI host controller
 	  found in NVIDIA Tegra124 and later SoCs.
 
 endif # USB_XHCI_HCD
 
+config USB_EHCI_BRCMSTB
+       tristate
+
+config USB_BRCMSTB
+	tristate "Broadcom STB USB support"
+	depends on (ARCH_BRCMSTB && PHY_BRCM_USB) || COMPILE_TEST
+	select USB_OHCI_HCD_PLATFORM if USB_OHCI_HCD
+	select USB_EHCI_BRCMSTB if USB_EHCI_HCD
+	select USB_XHCI_PLATFORM if USB_XHCI_HCD
+	help
+	  Enables support for XHCI, EHCI and OHCI host controllers
+	  found in Broadcom STB SoC's.
+
+	  To compile these drivers as modules, choose M here: the
+	  modules will be called ohci-platform.ko, ehci-brcm.ko and
+	  xhci-plat-hcd.ko
+
+	  Disabling this will keep the controllers and corresponding
+	  PHYs powered down.
+
 config USB_EHCI_HCD
 	tristate "EHCI HCD (USB 2.0) support"
 	depends on HAS_DMA && HAS_IOMEM
-	---help---
+	help
 	  The Enhanced Host Controller Interface (EHCI) is standard for USB 2.0
 	  "high speed" (480 Mbit/sec, 60 Mbyte/sec) host controller hardware.
 	  If your USB host controller supports USB 2.0, you will likely want to
@@ -122,7 +151,7 @@ config USB_EHCI_HCD
 config USB_EHCI_ROOT_HUB_TT
 	bool "Root Hub Transaction Translators"
 	depends on USB_EHCI_HCD
-	---help---
+	help
 	  Some EHCI chips have vendor-specific extensions to integrate
 	  transaction translators, so that no OHCI or UHCI companion
 	  controller is needed.  It's safe to say "y" even if your
@@ -135,7 +164,7 @@ config USB_EHCI_TT_NEWSCHED
 	bool "Improved Transaction Translator scheduling"
 	depends on USB_EHCI_HCD
 	default y
-	---help---
+	help
 	  This changes the periodic scheduling code to fill more of the low
 	  and full speed bandwidth available from the Transaction Translator
 	  (TT) in USB 2.0 hubs.  Without this, only one transfer will be
@@ -163,7 +192,7 @@ config USB_EHCI_HCD_PMC_MSP
 	depends on MSP_HAS_USB
 	select USB_EHCI_BIG_ENDIAN_DESC
 	select USB_EHCI_BIG_ENDIAN_MMIO
-	---help---
+	help
 		Enables support for the onchip USB controller on the PMC_MSP7100 Family SoC's.
 		If unsure, say N.
 
@@ -172,7 +201,7 @@ config XPS_USB_HCD_XILINX
 	depends on (PPC32 || MICROBLAZE)
 	select USB_EHCI_BIG_ENDIAN_DESC
 	select USB_EHCI_BIG_ENDIAN_MMIO
-	---help---
+	help
 		Xilinx xps USB host controller core is EHCI compliant and has
 		transaction translator built-in. It can be configured to either
 		support both high speed and full speed devices, or high speed
@@ -181,14 +210,14 @@ config XPS_USB_HCD_XILINX
 config USB_EHCI_FSL
 	tristate "Support for Freescale on-chip EHCI USB controller"
 	select USB_EHCI_ROOT_HUB_TT
-	---help---
+	help
 	  Variation of ARC USB block used in some Freescale chips.
 
 config USB_EHCI_MXC
 	tristate "Support for Freescale i.MX on-chip EHCI USB controller"
 	depends on ARCH_MXC || COMPILE_TEST
 	select USB_EHCI_ROOT_HUB_TT
-	---help---
+	help
 	  Variation of ARC USB block used in some Freescale chips.
 
 config USB_EHCI_HCD_NPCM7XX
@@ -204,7 +233,7 @@ config USB_EHCI_HCD_OMAP
 	depends on ARCH_OMAP
 	depends on NOP_USB_XCEIV
 	default y
-	---help---
+	help
 	  Enables support for the on-chip EHCI controller on
 	  OMAP3 and later chips.
 
@@ -212,7 +241,7 @@ config USB_EHCI_HCD_ORION
 	tristate  "Support for Marvell EBU on-chip EHCI USB controller"
 	depends on USB_EHCI_HCD && (PLAT_ORION || ARCH_MVEBU || COMPILE_TEST)
 	default y if (PLAT_ORION || ARCH_MVEBU)
-	---help---
+	help
 	  Enables support for the on-chip EHCI controller on Marvell's
 	  embedded ARM SoCs, including Orion, Kirkwood, Dove, Armada XP,
 	  Armada 370.  This is different from the EHCI implementation
@@ -223,7 +252,7 @@ config USB_EHCI_HCD_SPEAR
 	tristate "Support for ST SPEAr on-chip EHCI USB controller"
 	depends on USB_EHCI_HCD && (PLAT_SPEAR || COMPILE_TEST)
 	default y if PLAT_SPEAR
-	---help---
+	help
 	  Enables support for the on-chip EHCI controller on
 	  ST SPEAr chips.
 
@@ -240,7 +269,7 @@ config USB_EHCI_HCD_AT91
 	tristate  "Support for Atmel on-chip EHCI USB controller"
 	depends on USB_EHCI_HCD && (ARCH_AT91 || COMPILE_TEST)
 	default y if ARCH_AT91
-	---help---
+	help
 	  Enables support for the on-chip EHCI controller on
 	  Atmel chips.
 
@@ -257,14 +286,14 @@ config USB_EHCI_HCD_PPC_OF
 	bool "EHCI support for PPC USB controller on OF platform bus"
 	depends on PPC
 	default y
-	---help---
+	help
 	  Enables support for the USB controller present on the PowerPC
 	  OpenFirmware platform bus.
 
 config USB_EHCI_SH
 	bool "EHCI support for SuperH USB controller"
 	depends on SUPERH || COMPILE_TEST
-	---help---
+	help
 	  Enables support for the on-chip EHCI controller on the SuperH.
 	  If you use the PCI EHCI controller, this option is not necessary.
 
@@ -278,7 +307,7 @@ config USB_EHCI_MV
 	tristate "EHCI support for Marvell PXA/MMP USB controller"
 	depends on ARCH_PXA || ARCH_MMP || COMPILE_TEST
 	select USB_EHCI_ROOT_HUB_TT
-	---help---
+	help
 	  Enables support for Marvell (including PXA and MMP series) on-chip
 	  USB SPH and OTG controller. SPH is a single port host, and it can
 	  only be EHCI host. OTG is controller that can switch to host mode.
@@ -291,7 +320,7 @@ config USB_CNS3XXX_EHCI
 	bool "Cavium CNS3XXX EHCI Module (DEPRECATED)"
 	depends on ARCH_CNS3XXX || COMPILE_TEST
 	select USB_EHCI_HCD_PLATFORM
-	---help---
+	help
 	  This option is deprecated now and the driver was removed, use
 	  USB_EHCI_HCD_PLATFORM instead.
 
@@ -301,7 +330,7 @@ config USB_CNS3XXX_EHCI
 
 config USB_EHCI_HCD_PLATFORM
 	tristate "Generic EHCI driver for a platform device"
-	---help---
+	help
 	  Adds an EHCI host driver for a generic platform device, which
 	  provides a memory space and an irq.
 
@@ -326,7 +355,7 @@ endif # USB_EHCI_HCD
 config USB_OXU210HP_HCD
 	tristate "OXU210HP HCD support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  The OXU210HP is an USB host/OTG/device controller. Enable this
 	  option if your board has this chip. If unsure, say N.
 
@@ -339,7 +368,7 @@ config USB_OXU210HP_HCD
 config USB_ISP116X_HCD
 	tristate "ISP116X HCD support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  The ISP1160 and ISP1161 chips are USB host controllers. Enable this
 	  option if your board has this chip. If unsure, say N.
 
@@ -352,7 +381,7 @@ config USB_ISP1362_HCD
 	tristate "ISP1362 HCD support"
 	depends on HAS_IOMEM
 	depends on COMPILE_TEST # nothing uses this
-	---help---
+	help
 	  Supports the Philips ISP1362 chip as a host controller
 
 	  This driver does not support isochronous transfers.
@@ -363,7 +392,7 @@ config USB_ISP1362_HCD
 config USB_FOTG210_HCD
 	tristate "FOTG210 HCD support"
 	depends on USB && HAS_DMA && HAS_IOMEM
-	---help---
+	help
 	  Faraday FOTG210 is an OTG controller which can be configured as
 	  an USB2.0 host. It is designed to meet USB2.0 EHCI specification
 	  with minor modification.
@@ -374,7 +403,7 @@ config USB_FOTG210_HCD
 config USB_MAX3421_HCD
 	tristate "MAX3421 HCD (USB-over-SPI) support"
 	depends on USB && SPI
-	---help---
+	help
 	  The Maxim MAX3421E chip supports standard USB 2.0-compliant
 	  full-speed devices either in host or peripheral mode.  This
 	  driver supports the host-mode of the MAX3421E only.
@@ -385,7 +414,7 @@ config USB_MAX3421_HCD
 config USB_OHCI_HCD
 	tristate "OHCI HCD (USB 1.1) support"
 	depends on HAS_DMA && HAS_IOMEM
-	---help---
+	help
 	  The Open Host Controller Interface (OHCI) is a standard for accessing
 	  USB 1.1 host controller hardware.  It does more in hardware than Intel's
 	  UHCI specification.  If your USB host controller follows the OHCI spec,
@@ -405,14 +434,14 @@ config USB_OHCI_HCD_OMAP1
 	depends on ARCH_OMAP1
 	depends on ISP1301_OMAP || !(MACH_OMAP_H2 || MACH_OMAP_H3)
 	default y
-	---help---
+	help
 	  Enables support for the OHCI controller on OMAP1/2 chips.
 
 config USB_OHCI_HCD_SPEAR
 	tristate "Support for ST SPEAr on-chip OHCI USB controller"
 	depends on USB_OHCI_HCD && (PLAT_SPEAR || COMPILE_TEST)
 	default y if PLAT_SPEAR
-	---help---
+	help
 	  Enables support for the on-chip OHCI controller on
 	  ST SPEAr chips.
 
@@ -429,7 +458,7 @@ config USB_OHCI_HCD_S3C2410
 	tristate "OHCI support for Samsung S3C24xx/S3C64xx SoC series"
 	depends on USB_OHCI_HCD && (ARCH_S3C24XX || ARCH_S3C64XX || COMPILE_TEST)
 	default y if (ARCH_S3C24XX || ARCH_S3C64XX)
-	---help---
+	help
 	  Enables support for the on-chip OHCI controller on
 	  S3C24xx/S3C64xx chips.
 
@@ -439,7 +468,7 @@ config USB_OHCI_HCD_LPC32XX
 	depends on ARCH_LPC32XX || COMPILE_TEST
 	depends on USB_ISP1301
 	default y
-	---help---
+	help
 	  Enables support for the on-chip OHCI controller on
 	  NXP chips.
 
@@ -447,7 +476,7 @@ config USB_OHCI_HCD_PXA27X
 	tristate "Support for PXA27X/PXA3XX on-chip OHCI USB controller"
 	depends on USB_OHCI_HCD && (PXA27x || PXA3xx)
 	default y
-	---help---
+	help
 	  Enables support for the on-chip OHCI controller on
 	  PXA27x/PXA3xx chips.
 
@@ -455,7 +484,7 @@ config USB_OHCI_HCD_AT91
 	tristate "Support for Atmel on-chip OHCI USB controller"
 	depends on USB_OHCI_HCD && (ARCH_AT91 || COMPILE_TEST) && OF
 	default y if ARCH_AT91
-	---help---
+	help
 	  Enables support for the on-chip OHCI controller on
 	  Atmel chips.
 
@@ -487,7 +516,7 @@ config USB_OHCI_HCD_PPC_OF_BE
 	depends on PPC
 	select USB_OHCI_BIG_ENDIAN_DESC
 	select USB_OHCI_BIG_ENDIAN_MMIO
-	---help---
+	help
 	  Enables support for big-endian USB controllers present on the
 	  OpenFirmware platform bus.
 
@@ -495,7 +524,7 @@ config USB_OHCI_HCD_PPC_OF_LE
 	bool "OHCI support for OF platform bus (little endian)"
 	depends on PPC
 	select USB_OHCI_LITTLE_ENDIAN
-	---help---
+	help
 	  Enables support for little-endian USB controllers present on the
 	  OpenFirmware platform bus.
 
@@ -509,7 +538,7 @@ config USB_OHCI_HCD_PCI
 	depends on USB_PCI
 	default y
 	select USB_OHCI_LITTLE_ENDIAN
-	---help---
+	help
 	  Enables support for PCI-bus plug-in USB controller cards.
 	  If unsure, say Y.
 
@@ -518,7 +547,7 @@ config USB_OHCI_HCD_SSB
 	depends on (SSB = y || SSB = USB_OHCI_HCD)
 	select USB_HCD_SSB
 	select USB_OHCI_HCD_PLATFORM
-	---help---
+	help
 	  This option is deprecated now and the driver was removed, use
 	  USB_HCD_SSB and USB_OHCI_HCD_PLATFORM instead.
 
@@ -534,7 +563,7 @@ config USB_OHCI_SH
 	bool "OHCI support for SuperH USB controller (DEPRECATED)"
 	depends on SUPERH || COMPILE_TEST
 	select USB_OHCI_HCD_PLATFORM
-	---help---
+	help
 	  This option is deprecated now and the driver was removed, use
 	  USB_OHCI_HCD_PLATFORM instead.
 
@@ -551,7 +580,7 @@ config USB_CNS3XXX_OHCI
 	bool "Cavium CNS3XXX OHCI Module (DEPRECATED)"
 	depends on ARCH_CNS3XXX || COMPILE_TEST
 	select USB_OHCI_HCD_PLATFORM
-	---help---
+	help
 	  This option is deprecated now and the driver was removed, use
 	  USB_OHCI_HCD_PLATFORM instead.
 
@@ -560,7 +589,7 @@ config USB_CNS3XXX_OHCI
 
 config USB_OHCI_HCD_PLATFORM
 	tristate "Generic OHCI driver for a platform device"
-	---help---
+	help
 	  Adds an OHCI host driver for a generic platform device, which
 	  provides a memory space and an irq.
 
@@ -586,7 +615,7 @@ endif # USB_OHCI_HCD
 config USB_UHCI_HCD
 	tristate "UHCI HCD (most Intel and VIA) support"
 	depends on USB_PCI || USB_UHCI_SUPPORT_NON_PCI_HC
-	---help---
+	help
 	  The Universal Host Controller Interface is a standard by Intel for
 	  accessing the USB hardware in the PC (which is also called the USB
 	  host controller). If your USB host controller conforms to this
@@ -748,7 +777,7 @@ config USB_HCD_SSB
 
 config USB_HCD_TEST_MODE
 	bool "HCD test mode support"
-	---help---
+	help
 	  Say 'Y' to enable additional software test modes that may be
 	  supported by the host controller drivers.
 
diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile
index b191361257cc..bc731332fed9 100644
--- a/drivers/usb/host/Makefile
+++ b/drivers/usb/host/Makefile
@@ -49,6 +49,7 @@ obj-$(CONFIG_USB_EHCI_HCD_STI)	+= ehci-st.o
 obj-$(CONFIG_USB_EHCI_EXYNOS)	+= ehci-exynos.o
 obj-$(CONFIG_USB_EHCI_HCD_AT91) += ehci-atmel.o
 obj-$(CONFIG_USB_EHCI_TEGRA)	+= ehci-tegra.o
+obj-$(CONFIG_USB_EHCI_BRCMSTB)	+= ehci-brcm.o
 
 obj-$(CONFIG_USB_OXU210HP_HCD)	+= oxu210hp-hcd.o
 obj-$(CONFIG_USB_ISP116X_HCD)	+= isp116x-hcd.o
@@ -71,6 +72,7 @@ obj-$(CONFIG_USB_UHCI_HCD)	+= uhci-hcd.o
 obj-$(CONFIG_USB_FHCI_HCD)	+= fhci.o
 obj-$(CONFIG_USB_XHCI_HCD)	+= xhci-hcd.o
 obj-$(CONFIG_USB_XHCI_PCI)	+= xhci-pci.o
+obj-$(CONFIG_USB_XHCI_PCI_RENESAS)	+= xhci-pci-renesas.o
 obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o
 obj-$(CONFIG_USB_XHCI_HISTB)	+= xhci-histb.o
 obj-$(CONFIG_USB_XHCI_MTK)	+= xhci-mtk.o
diff --git a/drivers/usb/host/ehci-brcm.c b/drivers/usb/host/ehci-brcm.c
new file mode 100644
index 000000000000..3e0ebe8cc649
--- /dev/null
+++ b/drivers/usb/host/ehci-brcm.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2020, Broadcom */
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/usb.h>
+#include <linux/usb/hcd.h>
+#include <linux/iopoll.h>
+
+#include "ehci.h"
+
+#define hcd_to_ehci_priv(h) ((struct brcm_priv *)hcd_to_ehci(h)->priv)
+
+struct brcm_priv {
+	struct clk *clk;
+};
+
+/*
+ * ehci_brcm_wait_for_sof
+ * Wait for start of next microframe, then wait extra delay microseconds
+ */
+static inline void ehci_brcm_wait_for_sof(struct ehci_hcd *ehci, u32 delay)
+{
+	u32 frame_idx = ehci_readl(ehci, &ehci->regs->frame_index);
+	u32 val;
+	int res;
+
+	/* Wait for next microframe (every 125 usecs) */
+	res = readl_relaxed_poll_timeout(&ehci->regs->frame_index, val,
+					 val != frame_idx, 1, 130);
+	if (res)
+		ehci_err(ehci, "Error waiting for SOF\n");
+	udelay(delay);
+}
+
+/*
+ * ehci_brcm_hub_control
+ * The EHCI controller has a bug where it can violate the SOF
+ * interval between the first two SOF's transmitted after resume
+ * if the resume occurs near the end of the microframe. This causees
+ * the controller to detect babble on the suspended port and
+ * will eventually cause the controller to reset the port.
+ * The fix is to Intercept the echi-hcd request to complete RESUME and
+ * align it to the start of the next microframe.
+ * See SWLINUX-1909 for more details
+ */
+static int ehci_brcm_hub_control(
+	struct usb_hcd	*hcd,
+	u16		typeReq,
+	u16		wValue,
+	u16		wIndex,
+	char		*buf,
+	u16		wLength)
+{
+	struct ehci_hcd	*ehci = hcd_to_ehci(hcd);
+	int		ports = HCS_N_PORTS(ehci->hcs_params);
+	u32 __iomem	*status_reg;
+	unsigned long flags;
+	int retval, irq_disabled = 0;
+
+	status_reg = &ehci->regs->port_status[(wIndex & 0xff) - 1];
+
+	/*
+	 * RESUME is cleared when GetPortStatus() is called 20ms after start
+	 * of RESUME
+	 */
+	if ((typeReq == GetPortStatus) &&
+	    (wIndex && wIndex <= ports) &&
+	    ehci->reset_done[wIndex-1] &&
+	    time_after_eq(jiffies, ehci->reset_done[wIndex-1]) &&
+	    (ehci_readl(ehci, status_reg) & PORT_RESUME)) {
+
+		/*
+		 * to make sure we are not interrupted until RESUME bit
+		 * is cleared, disable interrupts on current CPU
+		 */
+		ehci_dbg(ehci, "SOF alignment workaround\n");
+		irq_disabled = 1;
+		local_irq_save(flags);
+		ehci_brcm_wait_for_sof(ehci, 5);
+	}
+	retval = ehci_hub_control(hcd, typeReq, wValue, wIndex, buf, wLength);
+	if (irq_disabled)
+		local_irq_restore(flags);
+	return retval;
+}
+
+static int ehci_brcm_reset(struct usb_hcd *hcd)
+{
+	struct ehci_hcd *ehci = hcd_to_ehci(hcd);
+	int len;
+
+	ehci->big_endian_mmio = 1;
+
+	ehci->caps = (void __iomem *)hcd->regs;
+	len = HC_LENGTH(ehci, ehci_readl(ehci, &ehci->caps->hc_capbase));
+	ehci->regs = (void __iomem *)(hcd->regs + len);
+
+	/* This fixes the lockup during reboot due to prior interrupts */
+	ehci_writel(ehci, CMD_RESET, &ehci->regs->command);
+	mdelay(10);
+
+	/*
+	 * SWLINUX-1705: Avoid OUT packet underflows during high memory
+	 *   bus usage
+	 * port_status[0x0f] = Broadcom-proprietary USB_EHCI_INSNREG00 @ 0x90
+	 */
+	ehci_writel(ehci, 0x00800040, &ehci->regs->port_status[0x10]);
+	ehci_writel(ehci, 0x00000001, &ehci->regs->port_status[0x12]);
+
+	return ehci_setup(hcd);
+}
+
+static struct hc_driver __read_mostly ehci_brcm_hc_driver;
+
+static const struct ehci_driver_overrides brcm_overrides __initconst = {
+	.reset = ehci_brcm_reset,
+	.extra_priv_size = sizeof(struct brcm_priv),
+};
+
+static int ehci_brcm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res_mem;
+	struct brcm_priv *priv;
+	struct usb_hcd *hcd;
+	int irq;
+	int err;
+
+	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (err)
+		return err;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq <= 0)
+		return irq ? irq : -EINVAL;
+
+	/* Hook the hub control routine to work around a bug */
+	ehci_brcm_hc_driver.hub_control = ehci_brcm_hub_control;
+
+	/* initialize hcd */
+	hcd = usb_create_hcd(&ehci_brcm_hc_driver, dev, dev_name(dev));
+	if (!hcd)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, hcd);
+	priv = hcd_to_ehci_priv(hcd);
+
+	priv->clk = devm_clk_get_optional(dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		err = PTR_ERR(priv->clk);
+		goto err_hcd;
+	}
+
+	err = clk_prepare_enable(priv->clk);
+	if (err)
+		goto err_hcd;
+
+	hcd->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res_mem);
+	if (IS_ERR(hcd->regs)) {
+		err = PTR_ERR(hcd->regs);
+		goto err_clk;
+	}
+	hcd->rsrc_start = res_mem->start;
+	hcd->rsrc_len = resource_size(res_mem);
+	err = usb_add_hcd(hcd, irq, IRQF_SHARED);
+	if (err)
+		goto err_clk;
+
+	device_wakeup_enable(hcd->self.controller);
+	device_enable_async_suspend(hcd->self.controller);
+
+	return 0;
+
+err_clk:
+	clk_disable_unprepare(priv->clk);
+err_hcd:
+	usb_put_hcd(hcd);
+
+	return err;
+}
+
+static int ehci_brcm_remove(struct platform_device *dev)
+{
+	struct usb_hcd *hcd = platform_get_drvdata(dev);
+	struct brcm_priv *priv = hcd_to_ehci_priv(hcd);
+
+	usb_remove_hcd(hcd);
+	clk_disable_unprepare(priv->clk);
+	usb_put_hcd(hcd);
+	return 0;
+}
+
+static int __maybe_unused ehci_brcm_suspend(struct device *dev)
+{
+	int ret;
+	struct usb_hcd *hcd = dev_get_drvdata(dev);
+	struct brcm_priv *priv = hcd_to_ehci_priv(hcd);
+	bool do_wakeup = device_may_wakeup(dev);
+
+	ret = ehci_suspend(hcd, do_wakeup);
+	if (ret)
+		return ret;
+	clk_disable_unprepare(priv->clk);
+	return 0;
+}
+
+static int __maybe_unused ehci_brcm_resume(struct device *dev)
+{
+	struct usb_hcd *hcd = dev_get_drvdata(dev);
+	struct ehci_hcd *ehci = hcd_to_ehci(hcd);
+	struct brcm_priv *priv = hcd_to_ehci_priv(hcd);
+	int err;
+
+	err = clk_prepare_enable(priv->clk);
+	if (err)
+		return err;
+	/*
+	 * SWLINUX-1705: Avoid OUT packet underflows during high memory
+	 *   bus usage
+	 * port_status[0x0f] = Broadcom-proprietary USB_EHCI_INSNREG00
+	 * @ 0x90
+	 */
+	ehci_writel(ehci, 0x00800040, &ehci->regs->port_status[0x10]);
+	ehci_writel(ehci, 0x00000001, &ehci->regs->port_status[0x12]);
+
+	ehci_resume(hcd, false);
+
+	pm_runtime_disable(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(ehci_brcm_pm_ops, ehci_brcm_suspend,
+		ehci_brcm_resume);
+
+static const struct of_device_id brcm_ehci_of_match[] = {
+	{ .compatible = "brcm,ehci-brcm-v2", },
+	{ .compatible = "brcm,bcm7445-ehci", },
+	{}
+};
+
+static struct platform_driver ehci_brcm_driver = {
+	.probe		= ehci_brcm_probe,
+	.remove		= ehci_brcm_remove,
+	.shutdown	= usb_hcd_platform_shutdown,
+	.driver		= {
+		.name	= "ehci-brcm",
+		.pm	= &ehci_brcm_pm_ops,
+		.of_match_table = brcm_ehci_of_match,
+	}
+};
+
+static int __init ehci_brcm_init(void)
+{
+	if (usb_disabled())
+		return -ENODEV;
+
+	ehci_init_driver(&ehci_brcm_hc_driver, &brcm_overrides);
+	return platform_driver_register(&ehci_brcm_driver);
+}
+module_init(ehci_brcm_init);
+
+static void __exit ehci_brcm_exit(void)
+{
+	platform_driver_unregister(&ehci_brcm_driver);
+}
+module_exit(ehci_brcm_exit);
+
+MODULE_ALIAS("platform:ehci-brcm");
+MODULE_DESCRIPTION("EHCI Broadcom STB driver");
+MODULE_AUTHOR("Al Cooper");
+MODULE_LICENSE("GPL");
diff --git a/drivers/usb/host/ehci-fsl.h b/drivers/usb/host/ehci-fsl.h
index 9d18c6e6ab27..c95341d472f4 100644
--- a/drivers/usb/host/ehci-fsl.h
+++ b/drivers/usb/host/ehci-fsl.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /* Copyright (C) 2005-2010,2012 Freescale Semiconductor, Inc.
  * Copyright (c) 2005 MontaVista Software
  */
diff --git a/drivers/usb/host/ehci-mv.c b/drivers/usb/host/ehci-mv.c
index 1300c457d9ed..cffdc8d01b2a 100644
--- a/drivers/usb/host/ehci-mv.c
+++ b/drivers/usb/host/ehci-mv.c
@@ -108,7 +108,7 @@ static int mv_ehci_probe(struct platform_device *pdev)
 	struct ehci_hcd *ehci;
 	struct ehci_hcd_mv *ehci_mv;
 	struct resource *r;
-	int retval = -ENODEV;
+	int retval;
 	u32 offset;
 	u32 status;
 
@@ -143,8 +143,6 @@ static int mv_ehci_probe(struct platform_device *pdev)
 		goto err_put_hcd;
 	}
 
-
-
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	ehci_mv->base = devm_ioremap_resource(&pdev->dev, r);
 	if (IS_ERR(ehci_mv->base)) {
@@ -168,12 +166,10 @@ static int mv_ehci_probe(struct platform_device *pdev)
 	hcd->rsrc_len = resource_size(r);
 	hcd->regs = ehci_mv->op_regs;
 
-	hcd->irq = platform_get_irq(pdev, 0);
-	if (!hcd->irq) {
-		dev_err(&pdev->dev, "Cannot get irq.");
-		retval = -ENODEV;
+	retval = platform_get_irq(pdev, 0);
+	if (retval < 0)
 		goto err_disable_clk;
-	}
+	hcd->irq = retval;
 
 	ehci = hcd_to_ehci(hcd);
 	ehci->caps = (struct ehci_caps __iomem *) ehci_mv->cap_regs;
diff --git a/drivers/usb/host/ehci-mxc.c b/drivers/usb/host/ehci-mxc.c
index c9f91e6c72b6..dc2676320527 100644
--- a/drivers/usb/host/ehci-mxc.c
+++ b/drivers/usb/host/ehci-mxc.c
@@ -36,12 +36,12 @@ static const struct ehci_driver_overrides ehci_mxc_overrides __initconst = {
 
 static int ehci_mxc_drv_probe(struct platform_device *pdev)
 {
-	struct mxc_usbh_platform_data *pdata = dev_get_platdata(&pdev->dev);
+	struct device *dev = &pdev->dev;
+	struct mxc_usbh_platform_data *pdata = dev_get_platdata(dev);
 	struct usb_hcd *hcd;
 	struct resource *res;
 	int irq, ret;
 	struct ehci_mxc_priv *priv;
-	struct device *dev = &pdev->dev;
 	struct ehci_hcd *ehci;
 
 	if (!pdata) {
@@ -50,13 +50,15 @@ static int ehci_mxc_drv_probe(struct platform_device *pdev)
 	}
 
 	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
 
 	hcd = usb_create_hcd(&ehci_mxc_hc_driver, dev, dev_name(dev));
 	if (!hcd)
 		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	hcd->regs = devm_ioremap_resource(&pdev->dev, res);
+	hcd->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(hcd->regs)) {
 		ret = PTR_ERR(hcd->regs);
 		goto err_alloc;
@@ -69,14 +71,14 @@ static int ehci_mxc_drv_probe(struct platform_device *pdev)
 	priv = (struct ehci_mxc_priv *) ehci->priv;
 
 	/* enable clocks */
-	priv->usbclk = devm_clk_get(&pdev->dev, "ipg");
+	priv->usbclk = devm_clk_get(dev, "ipg");
 	if (IS_ERR(priv->usbclk)) {
 		ret = PTR_ERR(priv->usbclk);
 		goto err_alloc;
 	}
 	clk_prepare_enable(priv->usbclk);
 
-	priv->ahbclk = devm_clk_get(&pdev->dev, "ahb");
+	priv->ahbclk = devm_clk_get(dev, "ahb");
 	if (IS_ERR(priv->ahbclk)) {
 		ret = PTR_ERR(priv->ahbclk);
 		goto err_clk_ahb;
@@ -84,13 +86,12 @@ static int ehci_mxc_drv_probe(struct platform_device *pdev)
 	clk_prepare_enable(priv->ahbclk);
 
 	/* "dr" device has its own clock on i.MX51 */
-	priv->phyclk = devm_clk_get(&pdev->dev, "phy");
+	priv->phyclk = devm_clk_get(dev, "phy");
 	if (IS_ERR(priv->phyclk))
 		priv->phyclk = NULL;
 	if (priv->phyclk)
 		clk_prepare_enable(priv->phyclk);
 
-
 	/* call platform specific init function */
 	if (pdata->init) {
 		ret = pdata->init(pdev);
diff --git a/drivers/usb/host/ehci-pci.c b/drivers/usb/host/ehci-pci.c
index 1a48ab1bd3b2..3c3820ad9092 100644
--- a/drivers/usb/host/ehci-pci.c
+++ b/drivers/usb/host/ehci-pci.c
@@ -360,23 +360,21 @@ static int ehci_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	if (is_bypassed_id(pdev))
 		return -ENODEV;
-	return usb_hcd_pci_probe(pdev, id);
+	return usb_hcd_pci_probe(pdev, id, &ehci_pci_hc_driver);
 }
 
 static void ehci_pci_remove(struct pci_dev *pdev)
 {
 	pci_clear_mwi(pdev);
-	usb_hcd_pci_remove(pdev);	
+	usb_hcd_pci_remove(pdev);
 }
 
 /* PCI driver selection metadata; PCI hotplugging uses this */
 static const struct pci_device_id pci_ids [] = { {
 	/* handle any USB 2.0 EHCI controller */
 	PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_EHCI, ~0),
-	.driver_data =	(unsigned long) &ehci_pci_hc_driver,
 	}, {
 	PCI_VDEVICE(STMICRO, PCI_DEVICE_ID_STMICRO_USB_HOST),
-	.driver_data = (unsigned long) &ehci_pci_hc_driver,
 	},
 	{ /* end: all zeroes */ }
 };
diff --git a/drivers/usb/host/ehci-platform.c b/drivers/usb/host/ehci-platform.c
index e4fc3f66d43b..e9a49007cce4 100644
--- a/drivers/usb/host/ehci-platform.c
+++ b/drivers/usb/host/ehci-platform.c
@@ -455,6 +455,10 @@ static int ehci_platform_resume(struct device *dev)
 
 	ehci_resume(hcd, priv->reset_on_resume);
 
+	pm_runtime_disable(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
 	if (priv->quirk_poll)
 		quirk_poll_init(priv);
 
diff --git a/drivers/usb/host/ehci-tegra.c b/drivers/usb/host/ehci-tegra.c
index 10d51daa6a1b..e077b2ca53c5 100644
--- a/drivers/usb/host/ehci-tegra.c
+++ b/drivers/usb/host/ehci-tegra.c
@@ -480,7 +480,6 @@ static int tegra_ehci_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (!irq) {
-		dev_err(&pdev->dev, "Failed to get IRQ\n");
 		err = -ENODEV;
 		goto cleanup_phy;
 	}
diff --git a/drivers/usb/host/ehci.h b/drivers/usb/host/ehci.h
index 229b3de319e6..eabf22a78eae 100644
--- a/drivers/usb/host/ehci.h
+++ b/drivers/usb/host/ehci.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright (c) 2001-2002 by David Brownell
  */
diff --git a/drivers/usb/host/fhci.h b/drivers/usb/host/fhci.h
index 2ce5031d866d..81fbc019a9b3 100644
--- a/drivers/usb/host/fhci.h
+++ b/drivers/usb/host/fhci.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Freescale QUICC Engine USB Host Controller Driver
  *
diff --git a/drivers/usb/host/imx21-hcd.h b/drivers/usb/host/imx21-hcd.h
index 7b9cf0a38d6e..96d16752a73e 100644
--- a/drivers/usb/host/imx21-hcd.h
+++ b/drivers/usb/host/imx21-hcd.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Macros and prototypes for i.MX21
  *
diff --git a/drivers/usb/host/ohci-pci.c b/drivers/usb/host/ohci-pci.c
index 22117a6aeb4a..585222af24ff 100644
--- a/drivers/usb/host/ohci-pci.c
+++ b/drivers/usb/host/ohci-pci.c
@@ -277,21 +277,24 @@ static const struct ohci_driver_overrides pci_overrides __initconst = {
 static const struct pci_device_id pci_ids[] = { {
 	/* handle any USB OHCI controller */
 	PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_OHCI, ~0),
-	.driver_data =	(unsigned long) &ohci_pci_hc_driver,
 	}, {
 	/* The device in the ConneXT I/O hub has no class reg */
 	PCI_VDEVICE(STMICRO, PCI_DEVICE_ID_STMICRO_USB_OHCI),
-	.driver_data =	(unsigned long) &ohci_pci_hc_driver,
 	}, { /* end: all zeroes */ }
 };
 MODULE_DEVICE_TABLE (pci, pci_ids);
 
+static int ohci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	return usb_hcd_pci_probe(dev, id, &ohci_pci_hc_driver);
+}
+
 /* pci driver glue; this is a "new style" PCI driver module */
 static struct pci_driver ohci_pci_driver = {
 	.name =		hcd_name,
 	.id_table =	pci_ids,
 
-	.probe =	usb_hcd_pci_probe,
+	.probe =	ohci_pci_probe,
 	.remove =	usb_hcd_pci_remove,
 	.shutdown =	usb_hcd_pci_shutdown,
 
diff --git a/drivers/usb/host/ohci-platform.c b/drivers/usb/host/ohci-platform.c
index 7addfc2cbadc..4a8456f12a73 100644
--- a/drivers/usb/host/ohci-platform.c
+++ b/drivers/usb/host/ohci-platform.c
@@ -299,6 +299,11 @@ static int ohci_platform_resume(struct device *dev)
 	}
 
 	ohci_resume(hcd, false);
+
+	pm_runtime_disable(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
 	return 0;
 }
 #endif /* CONFIG_PM_SLEEP */
diff --git a/drivers/usb/host/ohci-sm501.c b/drivers/usb/host/ohci-sm501.c
index c158cda9e4b9..cff965240327 100644
--- a/drivers/usb/host/ohci-sm501.c
+++ b/drivers/usb/host/ohci-sm501.c
@@ -157,9 +157,10 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev)
 	 * the call to usb_hcd_setup_local_mem() below does just that.
 	 */
 
-	if (usb_hcd_setup_local_mem(hcd, mem->start,
-				    mem->start - mem->parent->start,
-				    resource_size(mem)) < 0)
+	retval = usb_hcd_setup_local_mem(hcd, mem->start,
+					 mem->start - mem->parent->start,
+					 resource_size(mem));
+	if (retval < 0)
 		goto err5;
 	retval = usb_add_hcd(hcd, irq, IRQF_SHARED);
 	if (retval)
diff --git a/drivers/usb/host/ohci.h b/drivers/usb/host/ohci.h
index 27c26ca10bfd..b85a39588f9d 100644
--- a/drivers/usb/host/ohci.h
+++ b/drivers/usb/host/ohci.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-1.0+
+/* SPDX-License-Identifier: GPL-1.0+ */
 /*
  * OHCI HCD (Host Controller Driver) for USB.
  *
diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c
index beb2efa71341..0b949acfa258 100644
--- a/drivers/usb/host/pci-quirks.c
+++ b/drivers/usb/host/pci-quirks.c
@@ -16,6 +16,9 @@
 #include <linux/export.h>
 #include <linux/acpi.h>
 #include <linux/dmi.h>
+
+#include <soc/bcm2835/raspberrypi-firmware.h>
+
 #include "pci-quirks.h"
 #include "xhci-ext-caps.h"
 
@@ -205,7 +208,7 @@ static void usb_amd_find_chipset_info(void)
 {
 	unsigned long flags;
 	struct amd_chipset_info info;
-	info.need_pll_quirk = 0;
+	info.need_pll_quirk = false;
 
 	spin_lock_irqsave(&amd_lock, flags);
 
@@ -229,10 +232,10 @@ static void usb_amd_find_chipset_info(void)
 	case AMD_CHIPSET_SB800:
 	case AMD_CHIPSET_HUDSON2:
 	case AMD_CHIPSET_BOLTON:
-		info.need_pll_quirk = 1;
+		info.need_pll_quirk = true;
 		break;
 	default:
-		info.need_pll_quirk = 0;
+		info.need_pll_quirk = false;
 		break;
 	}
 
@@ -529,7 +532,7 @@ void usb_amd_dev_put(void)
 	amd_chipset.nb_type = 0;
 	memset(&amd_chipset.sb_type, 0, sizeof(amd_chipset.sb_type));
 	amd_chipset.isoc_reqs = 0;
-	amd_chipset.need_pll_quirk = 0;
+	amd_chipset.need_pll_quirk = false;
 
 	spin_unlock_irqrestore(&amd_lock, flags);
 
@@ -1243,11 +1246,24 @@ iounmap:
 
 static void quirk_usb_early_handoff(struct pci_dev *pdev)
 {
+	int ret;
+
 	/* Skip Netlogic mips SoC's internal PCI USB controller.
 	 * This device does not need/support EHCI/OHCI handoff
 	 */
 	if (pdev->vendor == 0x184e)	/* vendor Netlogic */
 		return;
+
+	if (pdev->vendor == PCI_VENDOR_ID_VIA && pdev->device == 0x3483) {
+		ret = rpi_firmware_init_vl805(pdev);
+		if (ret) {
+			/* Firmware might be outdated, or something failed */
+			dev_warn(&pdev->dev,
+				 "Failed to load VL805's firmware: %d. Will continue to attempt to work, but bad things might happen. You should fix this...\n",
+				 ret);
+		}
+	}
+
 	if (pdev->class != PCI_CLASS_SERIAL_USB_UHCI &&
 			pdev->class != PCI_CLASS_SERIAL_USB_OHCI &&
 			pdev->class != PCI_CLASS_SERIAL_USB_EHCI &&
diff --git a/drivers/usb/host/r8a66597.h b/drivers/usb/host/r8a66597.h
index 51973a923526..ab081475c113 100644
--- a/drivers/usb/host/r8a66597.h
+++ b/drivers/usb/host/r8a66597.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * R8A66597 HCD (Host Controller Driver)
  *
diff --git a/drivers/usb/host/u132-hcd.c b/drivers/usb/host/u132-hcd.c
index e9209e3e6248..995bc52d2d22 100644
--- a/drivers/usb/host/u132-hcd.c
+++ b/drivers/usb/host/u132-hcd.c
@@ -81,7 +81,6 @@ static DECLARE_WAIT_QUEUE_HEAD(u132_hcd_wait);
 static struct mutex u132_module_lock;
 static int u132_exiting;
 static int u132_instances;
-static struct list_head u132_static_list;
 /*
 * end of the global variables protected by u132_module_lock
 */
@@ -177,7 +176,6 @@ struct u132_ring {
 };
 struct u132 {
 	struct kref kref;
-	struct list_head u132_list;
 	struct mutex sw_lock;
 	struct mutex scheduler_lock;
 	struct u132_platform_data *board;
@@ -254,7 +252,6 @@ static void u132_hcd_delete(struct kref *kref)
 	struct usb_hcd *hcd = u132_to_hcd(u132);
 	u132->going += 1;
 	mutex_lock(&u132_module_lock);
-	list_del_init(&u132->u132_list);
 	u132_instances -= 1;
 	mutex_unlock(&u132_module_lock);
 	dev_warn(&u132->platform_dev->dev, "FREEING the hcd=%p and thus the u13"
@@ -3089,7 +3086,6 @@ static int u132_probe(struct platform_device *pdev)
 		retval = 0;
 		hcd->rsrc_start = 0;
 		mutex_lock(&u132_module_lock);
-		list_add_tail(&u132->u132_list, &u132_static_list);
 		u132->sequence_num = ++u132_instances;
 		mutex_unlock(&u132_module_lock);
 		u132_u132_init_kref(u132);
@@ -3192,7 +3188,6 @@ static struct platform_driver u132_platform_driver = {
 static int __init u132_hcd_init(void)
 {
 	int retval;
-	INIT_LIST_HEAD(&u132_static_list);
 	u132_instances = 0;
 	u132_exiting = 0;
 	mutex_init(&u132_module_lock);
@@ -3213,14 +3208,9 @@ static int __init u132_hcd_init(void)
 module_init(u132_hcd_init);
 static void __exit u132_hcd_exit(void)
 {
-	struct u132 *u132;
-	struct u132 *temp;
 	mutex_lock(&u132_module_lock);
 	u132_exiting += 1;
 	mutex_unlock(&u132_module_lock);
-	list_for_each_entry_safe(u132, temp, &u132_static_list, u132_list) {
-		platform_device_unregister(u132->platform_dev);
-	}
 	platform_driver_unregister(&u132_platform_driver);
 	printk(KERN_INFO "u132-hcd driver deregistered\n");
 	wait_event(u132_hcd_wait, u132_instances == 0);
diff --git a/drivers/usb/host/uhci-pci.c b/drivers/usb/host/uhci-pci.c
index 957c87efc746..9b88745d247f 100644
--- a/drivers/usb/host/uhci-pci.c
+++ b/drivers/usb/host/uhci-pci.c
@@ -287,17 +287,21 @@ static const struct hc_driver uhci_driver = {
 static const struct pci_device_id uhci_pci_ids[] = { {
 	/* handle any USB UHCI controller */
 	PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_UHCI, ~0),
-	.driver_data =	(unsigned long) &uhci_driver,
 	}, { /* end: all zeroes */ }
 };
 
 MODULE_DEVICE_TABLE(pci, uhci_pci_ids);
 
+static int uhci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	return usb_hcd_pci_probe(dev, id, &uhci_driver);
+}
+
 static struct pci_driver uhci_pci_driver = {
 	.name =		hcd_name,
 	.id_table =	uhci_pci_ids,
 
-	.probe =	usb_hcd_pci_probe,
+	.probe =	uhci_pci_probe,
 	.remove =	usb_hcd_pci_remove,
 	.shutdown =	uhci_shutdown,
 
diff --git a/drivers/usb/host/xhci-debugfs.h b/drivers/usb/host/xhci-debugfs.h
index f7a4e2492b00..56db635fcd6e 100644
--- a/drivers/usb/host/xhci-debugfs.h
+++ b/drivers/usb/host/xhci-debugfs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * xhci-debugfs.h - xHCI debugfs interface
  *
diff --git a/drivers/usb/host/xhci-ext-caps.h b/drivers/usb/host/xhci-ext-caps.h
index 268328c20681..fa59b242cd51 100644
--- a/drivers/usb/host/xhci-ext-caps.h
+++ b/drivers/usb/host/xhci-ext-caps.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * xHCI host controller driver
  *
diff --git a/drivers/usb/host/xhci-mtk.h b/drivers/usb/host/xhci-mtk.h
index acd56517215a..a93cfe817904 100644
--- a/drivers/usb/host/xhci-mtk.h
+++ b/drivers/usb/host/xhci-mtk.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2015 MediaTek Inc.
  * Author:
diff --git a/drivers/usb/host/xhci-mvebu.h b/drivers/usb/host/xhci-mvebu.h
index ca0a3a5721dd..3be021793cc8 100644
--- a/drivers/usb/host/xhci-mvebu.h
+++ b/drivers/usb/host/xhci-mvebu.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2014 Marvell
  *
diff --git a/drivers/usb/host/xhci-pci-renesas.c b/drivers/usb/host/xhci-pci-renesas.c
new file mode 100644
index 000000000000..59b1965ad0a3
--- /dev/null
+++ b/drivers/usb/host/xhci-pci-renesas.c
@@ -0,0 +1,645 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2019-2020 Linaro Limited */
+
+#include <linux/acpi.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <asm/unaligned.h>
+
+#include "xhci.h"
+#include "xhci-trace.h"
+#include "xhci-pci.h"
+
+#define RENESAS_FW_VERSION				0x6C
+#define RENESAS_ROM_CONFIG				0xF0
+#define RENESAS_FW_STATUS				0xF4
+#define RENESAS_FW_STATUS_MSB				0xF5
+#define RENESAS_ROM_STATUS				0xF6
+#define RENESAS_ROM_STATUS_MSB				0xF7
+#define RENESAS_DATA0					0xF8
+#define RENESAS_DATA1					0xFC
+
+#define RENESAS_FW_VERSION_FIELD			GENMASK(23, 7)
+#define RENESAS_FW_VERSION_OFFSET			8
+
+#define RENESAS_FW_STATUS_DOWNLOAD_ENABLE		BIT(0)
+#define RENESAS_FW_STATUS_LOCK				BIT(1)
+#define RENESAS_FW_STATUS_RESULT			GENMASK(6, 4)
+  #define RENESAS_FW_STATUS_INVALID			0
+  #define RENESAS_FW_STATUS_SUCCESS			BIT(4)
+  #define RENESAS_FW_STATUS_ERROR			BIT(5)
+#define RENESAS_FW_STATUS_SET_DATA0			BIT(8)
+#define RENESAS_FW_STATUS_SET_DATA1			BIT(9)
+
+#define RENESAS_ROM_STATUS_ACCESS			BIT(0)
+#define RENESAS_ROM_STATUS_ERASE			BIT(1)
+#define RENESAS_ROM_STATUS_RELOAD			BIT(2)
+#define RENESAS_ROM_STATUS_RESULT			GENMASK(6, 4)
+  #define RENESAS_ROM_STATUS_NO_RESULT			0
+  #define RENESAS_ROM_STATUS_SUCCESS			BIT(4)
+  #define RENESAS_ROM_STATUS_ERROR			BIT(5)
+#define RENESAS_ROM_STATUS_SET_DATA0			BIT(8)
+#define RENESAS_ROM_STATUS_SET_DATA1			BIT(9)
+#define RENESAS_ROM_STATUS_ROM_EXISTS			BIT(15)
+
+#define RENESAS_ROM_ERASE_MAGIC				0x5A65726F
+#define RENESAS_ROM_WRITE_MAGIC				0x53524F4D
+
+#define RENESAS_RETRY	10000
+#define RENESAS_DELAY	10
+
+#define ROM_VALID_01 0x2013
+#define ROM_VALID_02 0x2026
+
+static int renesas_verify_fw_version(struct pci_dev *pdev, u32 version)
+{
+	switch (version) {
+	case ROM_VALID_01:
+	case ROM_VALID_02:
+		return 0;
+	}
+	dev_err(&pdev->dev, "FW has invalid version :%d\n", version);
+	return -EINVAL;
+}
+
+static int renesas_fw_download_image(struct pci_dev *dev,
+				     const u32 *fw, size_t step, bool rom)
+{
+	size_t i;
+	int err;
+	u8 fw_status;
+	bool data0_or_data1;
+	u32 status_reg;
+
+	if (rom)
+		status_reg = RENESAS_ROM_STATUS_MSB;
+	else
+		status_reg = RENESAS_FW_STATUS_MSB;
+
+	/*
+	 * The hardware does alternate between two 32-bit pages.
+	 * (This is because each row of the firmware is 8 bytes).
+	 *
+	 * for even steps we use DATA0, for odd steps DATA1.
+	 */
+	data0_or_data1 = (step & 1) == 1;
+
+	/* step+1. Read "Set DATAX" and confirm it is cleared. */
+	for (i = 0; i < RENESAS_RETRY; i++) {
+		err = pci_read_config_byte(dev, status_reg, &fw_status);
+		if (err) {
+			dev_err(&dev->dev, "Read Status failed: %d\n",
+				pcibios_err_to_errno(err));
+			return pcibios_err_to_errno(err);
+		}
+		if (!(fw_status & BIT(data0_or_data1)))
+			break;
+
+		udelay(RENESAS_DELAY);
+	}
+	if (i == RENESAS_RETRY) {
+		dev_err(&dev->dev, "Timeout for Set DATAX step: %zd\n", step);
+		return -ETIMEDOUT;
+	}
+
+	/*
+	 * step+2. Write FW data to "DATAX".
+	 * "LSB is left" => force little endian
+	 */
+	err = pci_write_config_dword(dev, data0_or_data1 ?
+				     RENESAS_DATA1 : RENESAS_DATA0,
+				     (__force u32)cpu_to_le32(fw[step]));
+	if (err) {
+		dev_err(&dev->dev, "Write to DATAX failed: %d\n",
+			pcibios_err_to_errno(err));
+		return pcibios_err_to_errno(err);
+	}
+
+	udelay(100);
+
+	/* step+3. Set "Set DATAX". */
+	err = pci_write_config_byte(dev, status_reg, BIT(data0_or_data1));
+	if (err) {
+		dev_err(&dev->dev, "Write config for DATAX failed: %d\n",
+			pcibios_err_to_errno(err));
+		return pcibios_err_to_errno(err);
+	}
+
+	return 0;
+}
+
+static int renesas_fw_verify(const void *fw_data,
+			     size_t length)
+{
+	u16 fw_version_pointer;
+	u16 fw_version;
+
+	/*
+	 * The Firmware's Data Format is describe in
+	 * "6.3 Data Format" R19UH0078EJ0500 Rev.5.00 page 124
+	 */
+
+	/*
+	 * The bootrom chips of the big brother have sizes up to 64k, let's
+	 * assume that's the biggest the firmware can get.
+	 */
+	if (length < 0x1000 || length >= 0x10000) {
+		pr_err("firmware is size %zd is not (4k - 64k).",
+			length);
+		return -EINVAL;
+	}
+
+	/* The First 2 bytes are fixed value (55aa). "LSB on Left" */
+	if (get_unaligned_le16(fw_data) != 0x55aa) {
+		pr_err("no valid firmware header found.");
+		return -EINVAL;
+	}
+
+	/* verify the firmware version position and print it. */
+	fw_version_pointer = get_unaligned_le16(fw_data + 4);
+	if (fw_version_pointer + 2 >= length) {
+		pr_err("fw ver pointer is outside of the firmware image");
+		return -EINVAL;
+	}
+
+	fw_version = get_unaligned_le16(fw_data + fw_version_pointer);
+	pr_err("got firmware version: %02x.", fw_version);
+
+	return 0;
+}
+
+static bool renesas_check_rom(struct pci_dev *pdev)
+{
+	u16 rom_status;
+	int retval;
+
+	/* Check if external ROM exists */
+	retval = pci_read_config_word(pdev, RENESAS_ROM_STATUS, &rom_status);
+	if (retval)
+		return false;
+
+	rom_status &= RENESAS_ROM_STATUS_ROM_EXISTS;
+	if (rom_status) {
+		dev_dbg(&pdev->dev, "External ROM exists\n");
+		return true; /* External ROM exists */
+	}
+
+	return false;
+}
+
+static int renesas_check_rom_state(struct pci_dev *pdev)
+{
+	u16 rom_state;
+	u32 version;
+	int err;
+
+	/* check FW version */
+	err = pci_read_config_dword(pdev, RENESAS_FW_VERSION, &version);
+	if (err)
+		return pcibios_err_to_errno(err);
+
+	version &= RENESAS_FW_VERSION_FIELD;
+	version = version >> RENESAS_FW_VERSION_OFFSET;
+
+	err = renesas_verify_fw_version(pdev, version);
+	if (err)
+		return err;
+
+	/*
+	 * Test if ROM is present and loaded, if so we can skip everything
+	 */
+	err = pci_read_config_word(pdev, RENESAS_ROM_STATUS, &rom_state);
+	if (err)
+		return pcibios_err_to_errno(err);
+
+	if (rom_state & BIT(15)) {
+		/* ROM exists */
+		dev_dbg(&pdev->dev, "ROM exists\n");
+
+		/* Check the "Result Code" Bits (6:4) and act accordingly */
+		switch (rom_state & RENESAS_ROM_STATUS_RESULT) {
+		case RENESAS_ROM_STATUS_SUCCESS:
+			return 0;
+
+		case RENESAS_ROM_STATUS_NO_RESULT: /* No result yet */
+			return 0;
+
+		case RENESAS_ROM_STATUS_ERROR: /* Error State */
+		default: /* All other states are marked as "Reserved states" */
+			dev_err(&pdev->dev, "Invalid ROM..");
+			break;
+		}
+	}
+
+	return -EIO;
+}
+
+static int renesas_fw_check_running(struct pci_dev *pdev)
+{
+	u8 fw_state;
+	int err;
+
+	/* Check if device has ROM and loaded, if so skip everything */
+	err = renesas_check_rom(pdev);
+	if (err) { /* we have rom */
+		err = renesas_check_rom_state(pdev);
+		if (!err)
+			return err;
+	}
+
+	/*
+	 * Test if the device is actually needing the firmware. As most
+	 * BIOSes will initialize the device for us. If the device is
+	 * initialized.
+	 */
+	err = pci_read_config_byte(pdev, RENESAS_FW_STATUS, &fw_state);
+	if (err)
+		return pcibios_err_to_errno(err);
+
+	/*
+	 * Check if "FW Download Lock" is locked. If it is and the FW is
+	 * ready we can simply continue. If the FW is not ready, we have
+	 * to give up.
+	 */
+	if (fw_state & RENESAS_FW_STATUS_LOCK) {
+		dev_dbg(&pdev->dev, "FW Download Lock is engaged.");
+
+		if (fw_state & RENESAS_FW_STATUS_SUCCESS)
+			return 0;
+
+		dev_err(&pdev->dev,
+			"FW Download Lock is set and FW is not ready. Giving Up.");
+		return -EIO;
+	}
+
+	/*
+	 * Check if "FW Download Enable" is set. If someone (us?) tampered
+	 * with it and it can't be reset, we have to give up too... and
+	 * ask for a forgiveness and a reboot.
+	 */
+	if (fw_state & RENESAS_FW_STATUS_DOWNLOAD_ENABLE) {
+		dev_err(&pdev->dev,
+			"FW Download Enable is stale. Giving Up (poweroff/reboot needed).");
+		return -EIO;
+	}
+
+	/* Otherwise, Check the "Result Code" Bits (6:4) and act accordingly */
+	switch (fw_state & RENESAS_FW_STATUS_RESULT) {
+	case 0: /* No result yet */
+		dev_dbg(&pdev->dev, "FW is not ready/loaded yet.");
+
+		/* tell the caller, that this device needs the firmware. */
+		return 1;
+
+	case RENESAS_FW_STATUS_SUCCESS: /* Success, device should be working. */
+		dev_dbg(&pdev->dev, "FW is ready.");
+		return 0;
+
+	case RENESAS_FW_STATUS_ERROR: /* Error State */
+		dev_err(&pdev->dev,
+			"hardware is in an error state. Giving up (poweroff/reboot needed).");
+		return -ENODEV;
+
+	default: /* All other states are marked as "Reserved states" */
+		dev_err(&pdev->dev,
+			"hardware is in an invalid state %lx. Giving up (poweroff/reboot needed).",
+			(fw_state & RENESAS_FW_STATUS_RESULT) >> 4);
+		return -EINVAL;
+	}
+}
+
+static int renesas_fw_download(struct pci_dev *pdev,
+			       const struct firmware *fw)
+{
+	const u32 *fw_data = (const u32 *)fw->data;
+	size_t i;
+	int err;
+	u8 fw_status;
+
+	/*
+	 * For more information and the big picture: please look at the
+	 * "Firmware Download Sequence" in "7.1 FW Download Interface"
+	 * of R19UH0078EJ0500 Rev.5.00 page 131
+	 */
+
+	/*
+	 * 0. Set "FW Download Enable" bit in the
+	 * "FW Download Control & Status Register" at 0xF4
+	 */
+	err = pci_write_config_byte(pdev, RENESAS_FW_STATUS,
+				    RENESAS_FW_STATUS_DOWNLOAD_ENABLE);
+	if (err)
+		return pcibios_err_to_errno(err);
+
+	/* 1 - 10 follow one step after the other. */
+	for (i = 0; i < fw->size / 4; i++) {
+		err = renesas_fw_download_image(pdev, fw_data, i, false);
+		if (err) {
+			dev_err(&pdev->dev,
+				"Firmware Download Step %zd failed at position %zd bytes with (%d).",
+				i, i * 4, err);
+			return err;
+		}
+	}
+
+	/*
+	 * This sequence continues until the last data is written to
+	 * "DATA0" or "DATA1". Naturally, we wait until "SET DATA0/1"
+	 * is cleared by the hardware beforehand.
+	 */
+	for (i = 0; i < RENESAS_RETRY; i++) {
+		err = pci_read_config_byte(pdev, RENESAS_FW_STATUS_MSB,
+					   &fw_status);
+		if (err)
+			return pcibios_err_to_errno(err);
+		if (!(fw_status & (BIT(0) | BIT(1))))
+			break;
+
+		udelay(RENESAS_DELAY);
+	}
+	if (i == RENESAS_RETRY)
+		dev_warn(&pdev->dev, "Final Firmware Download step timed out.");
+
+	/*
+	 * 11. After finishing writing the last data of FW, the
+	 * System Software must clear "FW Download Enable"
+	 */
+	err = pci_write_config_byte(pdev, RENESAS_FW_STATUS, 0);
+	if (err)
+		return pcibios_err_to_errno(err);
+
+	/* 12. Read "Result Code" and confirm it is good. */
+	for (i = 0; i < RENESAS_RETRY; i++) {
+		err = pci_read_config_byte(pdev, RENESAS_FW_STATUS, &fw_status);
+		if (err)
+			return pcibios_err_to_errno(err);
+		if (fw_status & RENESAS_FW_STATUS_SUCCESS)
+			break;
+
+		udelay(RENESAS_DELAY);
+	}
+	if (i == RENESAS_RETRY) {
+		/* Timed out / Error - let's see if we can fix this */
+		err = renesas_fw_check_running(pdev);
+		switch (err) {
+		case 0: /*
+			 * we shouldn't end up here.
+			 * maybe it took a little bit longer.
+			 * But all should be well?
+			 */
+			break;
+
+		case 1: /* (No result yet! */
+			dev_err(&pdev->dev, "FW Load timedout");
+			return -ETIMEDOUT;
+
+		default:
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static void renesas_rom_erase(struct pci_dev *pdev)
+{
+	int retval, i;
+	u8 status;
+
+	dev_dbg(&pdev->dev, "Performing ROM Erase...\n");
+	retval = pci_write_config_dword(pdev, RENESAS_DATA0,
+					RENESAS_ROM_ERASE_MAGIC);
+	if (retval) {
+		dev_err(&pdev->dev, "ROM erase, magic word write failed: %d\n",
+			pcibios_err_to_errno(retval));
+		return;
+	}
+
+	retval = pci_read_config_byte(pdev, RENESAS_ROM_STATUS, &status);
+	if (retval) {
+		dev_err(&pdev->dev, "ROM status read failed: %d\n",
+			pcibios_err_to_errno(retval));
+		return;
+	}
+	status |= RENESAS_ROM_STATUS_ERASE;
+	retval = pci_write_config_byte(pdev, RENESAS_ROM_STATUS, status);
+	if (retval) {
+		dev_err(&pdev->dev, "ROM erase set word write failed\n");
+		return;
+	}
+
+	/* sleep a bit while ROM is erased */
+	msleep(20);
+
+	for (i = 0; i < RENESAS_RETRY; i++) {
+		retval = pci_read_config_byte(pdev, RENESAS_ROM_STATUS,
+					      &status);
+		status &= RENESAS_ROM_STATUS_ERASE;
+		if (!status)
+			break;
+
+		mdelay(RENESAS_DELAY);
+	}
+
+	if (i == RENESAS_RETRY)
+		dev_dbg(&pdev->dev, "Chip erase timedout: %x\n", status);
+
+	dev_dbg(&pdev->dev, "ROM Erase... Done success\n");
+}
+
+static bool renesas_setup_rom(struct pci_dev *pdev, const struct firmware *fw)
+{
+	const u32 *fw_data = (const u32 *)fw->data;
+	int err, i;
+	u8 status;
+
+	/* 2. Write magic word to Data0 */
+	err = pci_write_config_dword(pdev, RENESAS_DATA0,
+				     RENESAS_ROM_WRITE_MAGIC);
+	if (err)
+		return false;
+
+	/* 3. Set External ROM access */
+	err = pci_write_config_byte(pdev, RENESAS_ROM_STATUS,
+				    RENESAS_ROM_STATUS_ACCESS);
+	if (err)
+		goto remove_bypass;
+
+	/* 4. Check the result */
+	err = pci_read_config_byte(pdev, RENESAS_ROM_STATUS, &status);
+	if (err)
+		goto remove_bypass;
+	status &= GENMASK(6, 4);
+	if (status) {
+		dev_err(&pdev->dev,
+			"setting external rom failed: %x\n", status);
+		goto remove_bypass;
+	}
+
+	/* 5 to 16 Write FW to DATA0/1 while checking SetData0/1 */
+	for (i = 0; i < fw->size / 4; i++) {
+		err = renesas_fw_download_image(pdev, fw_data, i, true);
+		if (err) {
+			dev_err(&pdev->dev,
+				"ROM Download Step %d failed at position %d bytes with (%d)\n",
+				 i, i * 4, err);
+			goto remove_bypass;
+		}
+	}
+
+	/*
+	 * wait till DATA0/1 is cleared
+	 */
+	for (i = 0; i < RENESAS_RETRY; i++) {
+		err = pci_read_config_byte(pdev, RENESAS_ROM_STATUS_MSB,
+					   &status);
+		if (err)
+			goto remove_bypass;
+		if (!(status & (BIT(0) | BIT(1))))
+			break;
+
+		udelay(RENESAS_DELAY);
+	}
+	if (i == RENESAS_RETRY) {
+		dev_err(&pdev->dev, "Final Firmware ROM Download step timed out\n");
+		goto remove_bypass;
+	}
+
+	/* 17. Remove bypass */
+	err = pci_write_config_byte(pdev, RENESAS_ROM_STATUS, 0);
+	if (err)
+		return false;
+
+	udelay(10);
+
+	/* 18. check result */
+	for (i = 0; i < RENESAS_RETRY; i++) {
+		err = pci_read_config_byte(pdev, RENESAS_ROM_STATUS, &status);
+		if (err) {
+			dev_err(&pdev->dev, "Read ROM status failed:%d\n",
+				pcibios_err_to_errno(err));
+			return false;
+		}
+		status &= RENESAS_ROM_STATUS_RESULT;
+		if (status ==  RENESAS_ROM_STATUS_SUCCESS) {
+			dev_dbg(&pdev->dev, "Download ROM success\n");
+			break;
+		}
+		udelay(RENESAS_DELAY);
+	}
+	if (i == RENESAS_RETRY) { /* Timed out */
+		dev_err(&pdev->dev,
+			"Download to external ROM TO: %x\n", status);
+		return false;
+	}
+
+	dev_dbg(&pdev->dev, "Download to external ROM succeeded\n");
+
+	/* Last step set Reload */
+	err = pci_write_config_byte(pdev, RENESAS_ROM_STATUS,
+				    RENESAS_ROM_STATUS_RELOAD);
+	if (err) {
+		dev_err(&pdev->dev, "Set ROM execute failed: %d\n",
+			pcibios_err_to_errno(err));
+		return false;
+	}
+
+	/*
+	 * wait till Reload is cleared
+	 */
+	for (i = 0; i < RENESAS_RETRY; i++) {
+		err = pci_read_config_byte(pdev, RENESAS_ROM_STATUS, &status);
+		if (err)
+			return false;
+		if (!(status & RENESAS_ROM_STATUS_RELOAD))
+			break;
+
+		udelay(RENESAS_DELAY);
+	}
+	if (i == RENESAS_RETRY) {
+		dev_err(&pdev->dev, "ROM Exec timed out: %x\n", status);
+		return false;
+	}
+
+	return true;
+
+remove_bypass:
+	pci_write_config_byte(pdev, RENESAS_ROM_STATUS, 0);
+	return false;
+}
+
+static int renesas_load_fw(struct pci_dev *pdev, const struct firmware *fw)
+{
+	int err = 0;
+	bool rom;
+
+	/* Check if the device has external ROM */
+	rom = renesas_check_rom(pdev);
+	if (rom) {
+		/* perform chip erase first */
+		renesas_rom_erase(pdev);
+
+		/* lets try loading fw on ROM first */
+		rom = renesas_setup_rom(pdev, fw);
+		if (!rom) {
+			dev_dbg(&pdev->dev,
+				"ROM load failed, falling back on FW load\n");
+		} else {
+			dev_dbg(&pdev->dev,
+				"ROM load success\n");
+			goto exit;
+		}
+	}
+
+	err = renesas_fw_download(pdev, fw);
+
+exit:
+	if (err)
+		dev_err(&pdev->dev, "firmware failed to download (%d).", err);
+	return err;
+}
+
+int renesas_xhci_check_request_fw(struct pci_dev *pdev,
+				  const struct pci_device_id *id)
+{
+	struct xhci_driver_data *driver_data =
+			(struct xhci_driver_data *)id->driver_data;
+	const char *fw_name = driver_data->firmware;
+	const struct firmware *fw;
+	int err;
+
+	err = renesas_fw_check_running(pdev);
+	/* Continue ahead, if the firmware is already running. */
+	if (err == 0)
+		return 0;
+
+	if (err != 1)
+		return err;
+
+	pci_dev_get(pdev);
+	err = request_firmware(&fw, fw_name, &pdev->dev);
+	pci_dev_put(pdev);
+	if (err) {
+		dev_err(&pdev->dev, "request_firmware failed: %d\n", err);
+		return err;
+	}
+
+	err = renesas_fw_verify(fw->data, fw->size);
+	if (err)
+		goto exit;
+
+	err = renesas_load_fw(pdev, fw);
+exit:
+	release_firmware(fw);
+	return err;
+}
+EXPORT_SYMBOL_GPL(renesas_xhci_check_request_fw);
+
+void renesas_xhci_pci_exit(struct pci_dev *dev)
+{
+}
+EXPORT_SYMBOL_GPL(renesas_xhci_pci_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c
index 766b74723e64..ef513c2fb843 100644
--- a/drivers/usb/host/xhci-pci.c
+++ b/drivers/usb/host/xhci-pci.c
@@ -15,6 +15,7 @@
 
 #include "xhci.h"
 #include "xhci-trace.h"
+#include "xhci-pci.h"
 
 #define SSIC_PORT_NUM		2
 #define SSIC_PORT_CFG2		0x880c
@@ -87,7 +88,16 @@ static int xhci_pci_reinit(struct xhci_hcd *xhci, struct pci_dev *pdev)
 
 static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci)
 {
-	struct pci_dev		*pdev = to_pci_dev(dev);
+	struct pci_dev                  *pdev = to_pci_dev(dev);
+	struct xhci_driver_data         *driver_data;
+	const struct pci_device_id      *id;
+
+	id = pci_match_id(pdev->driver->id_table, pdev);
+
+	if (id && id->driver_data) {
+		driver_data = (struct xhci_driver_data *)id->driver_data;
+		xhci->quirks |= driver_data->quirks;
+	}
 
 	/* Look for vendor-specific quirks */
 	if (pdev->vendor == PCI_VENDOR_ID_FRESCO_LOGIC &&
@@ -327,10 +337,15 @@ static int xhci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
 	int retval;
 	struct xhci_hcd *xhci;
-	struct hc_driver *driver;
 	struct usb_hcd *hcd;
+	struct xhci_driver_data *driver_data;
 
-	driver = (struct hc_driver *)id->driver_data;
+	driver_data = (struct xhci_driver_data *)id->driver_data;
+	if (driver_data && driver_data->quirks & XHCI_RENESAS_FW_QUIRK) {
+		retval = renesas_xhci_check_request_fw(dev, id);
+		if (retval)
+			return retval;
+	}
 
 	/* Prevent runtime suspending between USB-2 and USB-3 initialization */
 	pm_runtime_get_noresume(&dev->dev);
@@ -341,7 +356,7 @@ static int xhci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	 * to say USB 2.0, but I'm not sure what the implications would be in
 	 * the other parts of the HCD code.
 	 */
-	retval = usb_hcd_pci_probe(dev, id);
+	retval = usb_hcd_pci_probe(dev, id, &xhci_pci_hc_driver);
 
 	if (retval)
 		goto put_runtime_pm;
@@ -349,8 +364,8 @@ static int xhci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	/* USB 2.0 roothub is stored in the PCI device now. */
 	hcd = dev_get_drvdata(&dev->dev);
 	xhci = hcd_to_xhci(hcd);
-	xhci->shared_hcd = usb_create_shared_hcd(driver, &dev->dev,
-				pci_name(dev), hcd);
+	xhci->shared_hcd = usb_create_shared_hcd(&xhci_pci_hc_driver, &dev->dev,
+						 pci_name(dev), hcd);
 	if (!xhci->shared_hcd) {
 		retval = -ENOMEM;
 		goto dealloc_usb2_hcd;
@@ -392,6 +407,9 @@ static void xhci_pci_remove(struct pci_dev *dev)
 	struct xhci_hcd *xhci;
 
 	xhci = hcd_to_xhci(pci_get_drvdata(dev));
+	if (xhci->quirks & XHCI_RENESAS_FW_QUIRK)
+		renesas_xhci_pci_exit(dev);
+
 	xhci->xhc_state |= XHCI_STATE_REMOVING;
 
 	if (xhci->quirks & XHCI_DEFAULT_PM_RUNTIME_ALLOW)
@@ -543,15 +561,26 @@ static void xhci_pci_shutdown(struct usb_hcd *hcd)
 
 /*-------------------------------------------------------------------------*/
 
+static const struct xhci_driver_data reneses_data = {
+	.quirks  = XHCI_RENESAS_FW_QUIRK,
+	.firmware = "renesas_usb_fw.mem",
+};
+
 /* PCI driver selection metadata; PCI hotplugging uses this */
-static const struct pci_device_id pci_ids[] = { {
+static const struct pci_device_id pci_ids[] = {
+	{ PCI_DEVICE(0x1912, 0x0014),
+		.driver_data =  (unsigned long)&reneses_data,
+	},
+	{ PCI_DEVICE(0x1912, 0x0015),
+		.driver_data =  (unsigned long)&reneses_data,
+	},
 	/* handle any USB 3.0 xHCI controller */
-	PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_XHCI, ~0),
-	.driver_data =	(unsigned long) &xhci_pci_hc_driver,
+	{ PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_XHCI, ~0),
 	},
 	{ /* end: all zeroes */ }
 };
 MODULE_DEVICE_TABLE(pci, pci_ids);
+MODULE_FIRMWARE("renesas_usb_fw.mem");
 
 /* pci driver glue; this is a "new style" PCI driver module */
 static struct pci_driver xhci_pci_driver = {
diff --git a/drivers/usb/host/xhci-pci.h b/drivers/usb/host/xhci-pci.h
new file mode 100644
index 000000000000..acd7cf0a1706
--- /dev/null
+++ b/drivers/usb/host/xhci-pci.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2019-2020 Linaro Limited */
+
+#ifndef XHCI_PCI_H
+#define XHCI_PCI_H
+
+#if IS_ENABLED(CONFIG_USB_XHCI_PCI_RENESAS)
+int renesas_xhci_check_request_fw(struct pci_dev *dev,
+				  const struct pci_device_id *id);
+void renesas_xhci_pci_exit(struct pci_dev *dev);
+
+#else
+static int renesas_xhci_check_request_fw(struct pci_dev *dev,
+					 const struct pci_device_id *id)
+{
+	return 0;
+}
+
+static void renesas_xhci_pci_exit(struct pci_dev *dev) { };
+
+#endif
+
+struct xhci_driver_data {
+	u64 quirks;
+	const char *firmware;
+};
+
+#endif
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
index ea460b9682d5..f6b4089bfc4a 100644
--- a/drivers/usb/host/xhci-plat.c
+++ b/drivers/usb/host/xhci-plat.c
@@ -112,6 +112,10 @@ static const struct xhci_plat_priv xhci_plat_renesas_rcar_gen3 = {
 	SET_XHCI_PLAT_PRIV_FOR_RCAR(XHCI_RCAR_FIRMWARE_NAME_V3)
 };
 
+static const struct xhci_plat_priv xhci_plat_brcm = {
+	.quirks = XHCI_RESET_ON_RESUME,
+};
+
 static const struct of_device_id usb_xhci_of_match[] = {
 	{
 		.compatible = "generic-xhci",
@@ -147,6 +151,12 @@ static const struct of_device_id usb_xhci_of_match[] = {
 	}, {
 		.compatible = "renesas,rcar-gen3-xhci",
 		.data = &xhci_plat_renesas_rcar_gen3,
+	}, {
+		.compatible = "brcm,xhci-brcm-v2",
+		.data = &xhci_plat_brcm,
+	}, {
+		.compatible = "brcm,bcm7445-xhci",
+		.data = &xhci_plat_brcm,
 	},
 	{},
 };
@@ -409,7 +419,15 @@ static int __maybe_unused xhci_plat_resume(struct device *dev)
 	if (ret)
 		return ret;
 
-	return xhci_resume(xhci, 0);
+	ret = xhci_resume(xhci, 0);
+	if (ret)
+		return ret;
+
+	pm_runtime_disable(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+
+	return 0;
 }
 
 static int __maybe_unused xhci_plat_runtime_suspend(struct device *dev)
diff --git a/drivers/usb/host/xhci-plat.h b/drivers/usb/host/xhci-plat.h
index 5681723fc9cd..b49f6447bd3a 100644
--- a/drivers/usb/host/xhci-plat.h
+++ b/drivers/usb/host/xhci-plat.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * xhci-plat.h - xHCI host controller driver platform Bus Glue.
  *
diff --git a/drivers/usb/host/xhci-rcar.h b/drivers/usb/host/xhci-rcar.h
index 012744a63a49..048ad3b8a6c7 100644
--- a/drivers/usb/host/xhci-rcar.h
+++ b/drivers/usb/host/xhci-rcar.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * drivers/usb/host/xhci-rcar.h
  *
diff --git a/drivers/usb/host/xhci-trace.h b/drivers/usb/host/xhci-trace.h
index b19582b2a72c..627abd236dbe 100644
--- a/drivers/usb/host/xhci-trace.h
+++ b/drivers/usb/host/xhci-trace.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * xHCI host controller driver
  *
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index 86cfefdd6632..2c6c4f8d1ee1 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 
 /*
  * xHCI host controller driver
@@ -1873,6 +1873,7 @@ struct xhci_hcd {
 #define XHCI_DEFAULT_PM_RUNTIME_ALLOW	BIT_ULL(33)
 #define XHCI_RESET_PLL_ON_DISCONNECT	BIT_ULL(34)
 #define XHCI_SNPS_BROKEN_SUSPEND    BIT_ULL(35)
+#define XHCI_RENESAS_FW_QUIRK	BIT_ULL(36)
 
 	unsigned int		num_active_eps;
 	unsigned int		limit_active_eps;
diff --git a/drivers/usb/image/Kconfig b/drivers/usb/image/Kconfig
index 26c75f309da9..a65c638e7fc1 100644
--- a/drivers/usb/image/Kconfig
+++ b/drivers/usb/image/Kconfig
@@ -6,7 +6,7 @@ comment "USB Imaging devices"
 
 config USB_MDC800
 	tristate "USB Mustek MDC800 Digital Camera support"
-	---help---
+	help
 	  Say Y here if you want to connect this type of still camera to
 	  your computer's USB port. This driver can be used with gphoto 0.4.3
 	  and higher (look at <http://www.gphoto.org/>).
diff --git a/drivers/usb/isp1760/isp1760-core.h b/drivers/usb/isp1760/isp1760-core.h
index 97cb4d7a3e1c..d9a0a4cc467c 100644
--- a/drivers/usb/isp1760/isp1760-core.h
+++ b/drivers/usb/isp1760/isp1760-core.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Driver for the NXP ISP1760 chip
  *
diff --git a/drivers/usb/isp1760/isp1760-regs.h b/drivers/usb/isp1760/isp1760-regs.h
index 1f00c3850cf7..fedc4f5cded0 100644
--- a/drivers/usb/isp1760/isp1760-regs.h
+++ b/drivers/usb/isp1760/isp1760-regs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Driver for the NXP ISP1760 chip
  *
diff --git a/drivers/usb/isp1760/isp1760-udc.h b/drivers/usb/isp1760/isp1760-udc.h
index 2d0b88747701..d2df650d54e9 100644
--- a/drivers/usb/isp1760/isp1760-udc.h
+++ b/drivers/usb/isp1760/isp1760-udc.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Driver for the NXP ISP1761 device controller
  *
diff --git a/drivers/usb/misc/Kconfig b/drivers/usb/misc/Kconfig
index 833a460ee55a..4e48f8eed168 100644
--- a/drivers/usb/misc/Kconfig
+++ b/drivers/usb/misc/Kconfig
@@ -6,7 +6,7 @@ comment "USB Miscellaneous drivers"
 
 config USB_EMI62
 	tristate "EMI 6|2m USB Audio interface support"
-	---help---
+	help
 	  This driver loads firmware to Emagic EMI 6|2m low latency USB
 	  Audio and Midi interface.
 
@@ -20,7 +20,7 @@ config USB_EMI62
 
 config USB_EMI26
 	tristate "EMI 2|6 USB Audio interface support"
-	---help---
+	help
 	  This driver loads firmware to Emagic EMI 2|6 low latency USB
 	  Audio interface.
 
diff --git a/drivers/usb/misc/sisusbvga/Kconfig b/drivers/usb/misc/sisusbvga/Kconfig
index c16121276a21..655d9cb0651a 100644
--- a/drivers/usb/misc/sisusbvga/Kconfig
+++ b/drivers/usb/misc/sisusbvga/Kconfig
@@ -4,7 +4,7 @@ config USB_SISUSBVGA
 	tristate "USB 2.0 SVGA dongle support (Net2280/SiS315)"
 	depends on (USB_MUSB_HDRC || USB_EHCI_HCD)
 	select FONT_SUPPORT if USB_SISUSBVGA_CON
-	---help---
+	help
 	  Say Y here if you intend to attach a USB2VGA dongle based on a
 	  Net2280 and a SiS315 chip.
 
@@ -18,7 +18,7 @@ config USB_SISUSBVGA_CON
 	bool "Text console and mode switching support" if USB_SISUSBVGA
 	depends on VT
 	select FONT_8x16
-	---help---
+	help
 	  Say Y here if you want a VGA text console via the USB dongle or
 	  want to support userland applications that utilize the driver's
 	  display mode switching capabilities.
diff --git a/drivers/usb/misc/sisusbvga/sisusb.h b/drivers/usb/misc/sisusbvga/sisusb.h
index 8a5e6bb07d05..c0fb9e1c5361 100644
--- a/drivers/usb/misc/sisusbvga/sisusb.h
+++ b/drivers/usb/misc/sisusbvga/sisusb.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
 /*
  * sisusb - usb kernel driver for Net2280/SiS315 based USB2VGA dongles
  *
diff --git a/drivers/usb/misc/sisusbvga/sisusb_init.h b/drivers/usb/misc/sisusbvga/sisusb_init.h
index ace09985dae4..aa33bc81ee52 100644
--- a/drivers/usb/misc/sisusbvga/sisusb_init.h
+++ b/drivers/usb/misc/sisusbvga/sisusb_init.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
 /* $XFree86$ */
 /* $XdotOrg$ */
 /*
diff --git a/drivers/usb/misc/sisusbvga/sisusb_struct.h b/drivers/usb/misc/sisusbvga/sisusb_struct.h
index 706d77090e00..3df64d2a9d43 100644
--- a/drivers/usb/misc/sisusbvga/sisusb_struct.h
+++ b/drivers/usb/misc/sisusbvga/sisusb_struct.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
 /*
  * General structure definitions for universal mode switching modules
  *
diff --git a/drivers/usb/misc/usb_u132.h b/drivers/usb/misc/usb_u132.h
index 4bf77736914f..1584efbbd704 100644
--- a/drivers/usb/misc/usb_u132.h
+++ b/drivers/usb/misc/usb_u132.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
 * Common Header File for the Elan Digital Systems U132 adapter
 * this file should be included by both the "ftdi-u132" and
diff --git a/drivers/usb/mtu3/mtu3.h b/drivers/usb/mtu3/mtu3.h
index 6087be236a35..d49db92ab26c 100644
--- a/drivers/usb/mtu3/mtu3.h
+++ b/drivers/usb/mtu3/mtu3.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * mtu3.h - MediaTek USB3 DRD header
  *
diff --git a/drivers/usb/mtu3/mtu3_debug.h b/drivers/usb/mtu3/mtu3_debug.h
index e96a69234d05..fb6b28277c9b 100644
--- a/drivers/usb/mtu3/mtu3_debug.h
+++ b/drivers/usb/mtu3/mtu3_debug.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * mtu3_debug.h - debug header
  *
diff --git a/drivers/usb/mtu3/mtu3_dr.h b/drivers/usb/mtu3/mtu3_dr.h
index 5e58c4dbd54a..760fe7d69c6b 100644
--- a/drivers/usb/mtu3/mtu3_dr.h
+++ b/drivers/usb/mtu3/mtu3_dr.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * mtu3_dr.h - dual role switch and host glue layer header
  *
diff --git a/drivers/usb/mtu3/mtu3_hw_regs.h b/drivers/usb/mtu3/mtu3_hw_regs.h
index 8382d066749e..bf34f784f84b 100644
--- a/drivers/usb/mtu3/mtu3_hw_regs.h
+++ b/drivers/usb/mtu3/mtu3_hw_regs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * mtu3_hw_regs.h - MediaTek USB3 DRD register and field definitions
  *
diff --git a/drivers/usb/mtu3/mtu3_qmu.h b/drivers/usb/mtu3/mtu3_qmu.h
index 9cfde201db63..66e1c0ab5a99 100644
--- a/drivers/usb/mtu3/mtu3_qmu.h
+++ b/drivers/usb/mtu3/mtu3_qmu.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * mtu3_qmu.h - Queue Management Unit driver header
  *
diff --git a/drivers/usb/mtu3/mtu3_trace.h b/drivers/usb/mtu3/mtu3_trace.h
index 050e30f0fbd4..1b897636daf2 100644
--- a/drivers/usb/mtu3/mtu3_trace.h
+++ b/drivers/usb/mtu3/mtu3_trace.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
  * mtu3_trace.h - trace support
  *
diff --git a/drivers/usb/musb/davinci.h b/drivers/usb/musb/davinci.h
index e021485c83ae..c8e67d15b510 100644
--- a/drivers/usb/musb/davinci.h
+++ b/drivers/usb/musb/davinci.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2005-2006 by Texas Instruments
  */
diff --git a/drivers/usb/musb/jz4740.c b/drivers/usb/musb/jz4740.c
index e64dd30e80e7..c4fe1f4cd17a 100644
--- a/drivers/usb/musb/jz4740.c
+++ b/drivers/usb/musb/jz4740.c
@@ -30,11 +30,11 @@ static irqreturn_t jz4740_musb_interrupt(int irq, void *__hci)
 	irqreturn_t	retval = IRQ_NONE, retval_dma = IRQ_NONE;
 	struct musb	*musb = __hci;
 
-	spin_lock_irqsave(&musb->lock, flags);
-
 	if (IS_ENABLED(CONFIG_USB_INVENTRA_DMA) && musb->dma_controller)
 		retval_dma = dma_controller_irq(irq, musb->dma_controller);
 
+	spin_lock_irqsave(&musb->lock, flags);
+
 	musb->int_usb = musb_readb(musb->mregs, MUSB_INTRUSB);
 	musb->int_tx = musb_readw(musb->mregs, MUSB_INTRTX);
 	musb->int_rx = musb_readw(musb->mregs, MUSB_INTRRX);
diff --git a/drivers/usb/musb/mediatek.c b/drivers/usb/musb/mediatek.c
index 6196b0e8d77d..eebeadd26946 100644
--- a/drivers/usb/musb/mediatek.c
+++ b/drivers/usb/musb/mediatek.c
@@ -208,6 +208,12 @@ static irqreturn_t generic_interrupt(int irq, void *__hci)
 	musb->int_rx = musb_clearw(musb->mregs, MUSB_INTRRX);
 	musb->int_tx = musb_clearw(musb->mregs, MUSB_INTRTX);
 
+	if ((musb->int_usb & MUSB_INTR_RESET) && !is_host_active(musb)) {
+		/* ep0 FADDR must be 0 when (re)entering peripheral mode */
+		musb_ep_select(musb->mregs, 0);
+		musb_writeb(musb->mregs, MUSB_FADDR, 0);
+	}
+
 	if (musb->int_usb || musb->int_tx || musb->int_rx)
 		retval = musb_interrupt(musb);
 
diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c
index d590110539ab..384a8039a7fd 100644
--- a/drivers/usb/musb/musb_core.c
+++ b/drivers/usb/musb/musb_core.c
@@ -1795,7 +1795,7 @@ irqreturn_t musb_interrupt(struct musb *musb)
 EXPORT_SYMBOL_GPL(musb_interrupt);
 
 #ifndef CONFIG_MUSB_PIO_ONLY
-static bool use_dma = 1;
+static bool use_dma = true;
 
 /* "modprobe ... use_dma=0" etc */
 module_param(use_dma, bool, 0644);
@@ -2877,6 +2877,13 @@ static int musb_resume(struct device *dev)
 	musb_enable_interrupts(musb);
 	musb_platform_enable(musb);
 
+	/* session might be disabled in suspend */
+	if (musb->port_mode == MUSB_HOST &&
+	    !(musb->ops->quirks & MUSB_PRESERVE_SESSION)) {
+		devctl |= MUSB_DEVCTL_SESSION;
+		musb_writeb(musb->mregs, MUSB_DEVCTL, devctl);
+	}
+
 	spin_lock_irqsave(&musb->lock, flags);
 	error = musb_run_resume_work(musb);
 	if (error)
diff --git a/drivers/usb/musb/musb_core.h b/drivers/usb/musb/musb_core.h
index 290a2bc46606..dbe5623db1e0 100644
--- a/drivers/usb/musb/musb_core.h
+++ b/drivers/usb/musb/musb_core.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MUSB OTG driver defines
  *
diff --git a/drivers/usb/musb/musb_debug.h b/drivers/usb/musb/musb_debug.h
index c444a80fe1da..e5b3506c7b3f 100644
--- a/drivers/usb/musb/musb_debug.h
+++ b/drivers/usb/musb/musb_debug.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MUSB OTG driver debug defines
  *
diff --git a/drivers/usb/musb/musb_debugfs.c b/drivers/usb/musb/musb_debugfs.c
index 7b6281ab62ed..30a89aa8a3e7 100644
--- a/drivers/usb/musb/musb_debugfs.c
+++ b/drivers/usb/musb/musb_debugfs.c
@@ -168,6 +168,11 @@ static ssize_t musb_test_mode_write(struct file *file,
 	u8			test;
 	char			buf[24];
 
+	memset(buf, 0x00, sizeof(buf));
+
+	if (copy_from_user(buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
+		return -EFAULT;
+
 	pm_runtime_get_sync(musb->controller);
 	test = musb_readb(musb->mregs, MUSB_TESTMODE);
 	if (test) {
@@ -176,11 +181,6 @@ static ssize_t musb_test_mode_write(struct file *file,
 		goto ret;
 	}
 
-	memset(buf, 0x00, sizeof(buf));
-
-	if (copy_from_user(buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
 	if (strstarts(buf, "force host full-speed"))
 		test = MUSB_TEST_FORCE_HOST | MUSB_TEST_FORCE_FS;
 
diff --git a/drivers/usb/musb/musb_dma.h b/drivers/usb/musb/musb_dma.h
index 4b4d8dc5d3f2..7d67b69df0a0 100644
--- a/drivers/usb/musb/musb_dma.h
+++ b/drivers/usb/musb/musb_dma.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MUSB OTG driver DMA controller abstraction
  *
diff --git a/drivers/usb/musb/musb_gadget.h b/drivers/usb/musb/musb_gadget.h
index d02663660813..f49f25b3bf56 100644
--- a/drivers/usb/musb/musb_gadget.h
+++ b/drivers/usb/musb/musb_gadget.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MUSB OTG driver peripheral defines
  *
diff --git a/drivers/usb/musb/musb_host.c b/drivers/usb/musb/musb_host.c
index 8736f4251a22..8b7d22a0c0fb 100644
--- a/drivers/usb/musb/musb_host.c
+++ b/drivers/usb/musb/musb_host.c
@@ -1774,9 +1774,15 @@ void musb_host_rx(struct musb *musb, u8 epnum)
 		status = -EPIPE;
 
 	} else if (rx_csr & MUSB_RXCSR_H_ERROR) {
-		musb_dbg(musb, "end %d RX proto error", epnum);
+		dev_err(musb->controller, "ep%d RX three-strikes error", epnum);
 
-		status = -EPROTO;
+		/*
+		 * The three-strikes error could only happen when the USB
+		 * device is not accessible, for example detached or powered
+		 * off. So return the fatal error -ESHUTDOWN so hopefully the
+		 * USB device drivers won't immediately resubmit the same URB.
+		 */
+		status = -ESHUTDOWN;
 		musb_writeb(epio, MUSB_RXINTERVAL, 0);
 
 		rx_csr &= ~MUSB_RXCSR_H_ERROR;
diff --git a/drivers/usb/musb/musb_host.h b/drivers/usb/musb/musb_host.h
index 2999845632ce..32336571f05c 100644
--- a/drivers/usb/musb/musb_host.h
+++ b/drivers/usb/musb/musb_host.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MUSB OTG driver host defines
  *
diff --git a/drivers/usb/musb/musb_io.h b/drivers/usb/musb/musb_io.h
index f17aabd95a50..12874d3b2a64 100644
--- a/drivers/usb/musb/musb_io.h
+++ b/drivers/usb/musb/musb_io.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MUSB OTG driver register I/O
  *
diff --git a/drivers/usb/musb/musb_regs.h b/drivers/usb/musb/musb_regs.h
index 5cd7264fc2cb..5fa110978f1a 100644
--- a/drivers/usb/musb/musb_regs.h
+++ b/drivers/usb/musb/musb_regs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * MUSB OTG driver register defines
  *
diff --git a/drivers/usb/musb/musb_trace.h b/drivers/usb/musb/musb_trace.h
index b193daf69685..380ebc77eab1 100644
--- a/drivers/usb/musb/musb_trace.h
+++ b/drivers/usb/musb/musb_trace.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * musb_trace.h - MUSB Controller Trace Support
  *
diff --git a/drivers/usb/musb/omap2430.h b/drivers/usb/musb/omap2430.h
index 859008fa0e3c..939a0361ae88 100644
--- a/drivers/usb/musb/omap2430.h
+++ b/drivers/usb/musb/omap2430.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2005-2006 by Texas Instruments
  */
diff --git a/drivers/usb/musb/tusb6010.h b/drivers/usb/musb/tusb6010.h
index fd8025bbece7..8a253564fb18 100644
--- a/drivers/usb/musb/tusb6010.h
+++ b/drivers/usb/musb/tusb6010.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Definitions for TUSB6010 USB 2.0 OTG Dual Role controller
  *
diff --git a/drivers/usb/phy/phy-fsl-usb.h b/drivers/usb/phy/phy-fsl-usb.h
index 43d410f6641b..fbcc28ad9964 100644
--- a/drivers/usb/phy/phy-fsl-usb.h
+++ b/drivers/usb/phy/phy-fsl-usb.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /* Copyright (C) 2007,2008 Freescale Semiconductor, Inc. */
 
 #include <linux/usb/otg-fsm.h>
diff --git a/drivers/usb/phy/phy-jz4770.c b/drivers/usb/phy/phy-jz4770.c
index 3ea1f5b9bcf8..8f62dc2a90ff 100644
--- a/drivers/usb/phy/phy-jz4770.c
+++ b/drivers/usb/phy/phy-jz4770.c
@@ -125,13 +125,13 @@ static int jz4770_phy_init(struct usb_phy *phy)
 
 	err = regulator_enable(priv->vcc_supply);
 	if (err) {
-		dev_err(priv->dev, "Unable to enable VCC: %d", err);
+		dev_err(priv->dev, "Unable to enable VCC: %d\n", err);
 		return err;
 	}
 
 	err = clk_prepare_enable(priv->clk);
 	if (err) {
-		dev_err(priv->dev, "Unable to start clock: %d", err);
+		dev_err(priv->dev, "Unable to start clock: %d\n", err);
 		return err;
 	}
 
@@ -191,7 +191,7 @@ static int jz4770_phy_probe(struct platform_device *pdev)
 
 	priv->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->base)) {
-		dev_err(dev, "Failed to map registers");
+		dev_err(dev, "Failed to map registers\n");
 		return PTR_ERR(priv->base);
 	}
 
@@ -199,7 +199,7 @@ static int jz4770_phy_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->clk)) {
 		err = PTR_ERR(priv->clk);
 		if (err != -EPROBE_DEFER)
-			dev_err(dev, "Failed to get clock");
+			dev_err(dev, "Failed to get clock\n");
 		return err;
 	}
 
@@ -207,14 +207,14 @@ static int jz4770_phy_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->vcc_supply)) {
 		err = PTR_ERR(priv->vcc_supply);
 		if (err != -EPROBE_DEFER)
-			dev_err(dev, "failed to get regulator");
+			dev_err(dev, "Failed to get regulator\n");
 		return err;
 	}
 
 	err = usb_add_phy(&priv->phy, USB_PHY_TYPE_USB2);
 	if (err) {
 		if (err != -EPROBE_DEFER)
-			dev_err(dev, "Unable to register PHY");
+			dev_err(dev, "Unable to register PHY\n");
 		return err;
 	}
 
diff --git a/drivers/usb/phy/phy-mv-usb.h b/drivers/usb/phy/phy-mv-usb.h
index 96701a1229ad..5d5c0abb0c3a 100644
--- a/drivers/usb/phy/phy-mv-usb.h
+++ b/drivers/usb/phy/phy-mv-usb.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright (C) 2011 Marvell International Ltd. All rights reserved.
  */
diff --git a/drivers/usb/renesas_usbhs/common.h b/drivers/usb/renesas_usbhs/common.h
index ef1735d014da..eb34d762a63d 100644
--- a/drivers/usb/renesas_usbhs/common.h
+++ b/drivers/usb/renesas_usbhs/common.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-1.0+
+/* SPDX-License-Identifier: GPL-1.0+ */
 /*
  * Renesas USB driver
  *
diff --git a/drivers/usb/renesas_usbhs/fifo.h b/drivers/usb/renesas_usbhs/fifo.h
index c3d3cc35cee0..7d3700bf41d9 100644
--- a/drivers/usb/renesas_usbhs/fifo.h
+++ b/drivers/usb/renesas_usbhs/fifo.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-1.0+
+/* SPDX-License-Identifier: GPL-1.0+ */
 /*
  * Renesas USB driver
  *
diff --git a/drivers/usb/renesas_usbhs/mod.h b/drivers/usb/renesas_usbhs/mod.h
index 65dc19ca528e..56b7106d254d 100644
--- a/drivers/usb/renesas_usbhs/mod.h
+++ b/drivers/usb/renesas_usbhs/mod.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-1.0+
+/* SPDX-License-Identifier: GPL-1.0+ */
 /*
  * Renesas USB driver
  *
diff --git a/drivers/usb/renesas_usbhs/pipe.h b/drivers/usb/renesas_usbhs/pipe.h
index 3b130529408b..a4ae9f97d9cd 100644
--- a/drivers/usb/renesas_usbhs/pipe.h
+++ b/drivers/usb/renesas_usbhs/pipe.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-1.0+
+/* SPDX-License-Identifier: GPL-1.0+ */
 /*
  * Renesas USB driver
  *
diff --git a/drivers/usb/renesas_usbhs/rcar2.h b/drivers/usb/renesas_usbhs/rcar2.h
index 7d88732c5bff..046d07edb36f 100644
--- a/drivers/usb/renesas_usbhs/rcar2.h
+++ b/drivers/usb/renesas_usbhs/rcar2.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 #include "common.h"
 
 extern const struct renesas_usbhs_platform_info usbhs_rcar_gen2_plat_info;
diff --git a/drivers/usb/renesas_usbhs/rcar3.h b/drivers/usb/renesas_usbhs/rcar3.h
index c7c5ec1e3af2..d13db30bd21b 100644
--- a/drivers/usb/renesas_usbhs/rcar3.h
+++ b/drivers/usb/renesas_usbhs/rcar3.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 #include "common.h"
 
 extern const struct renesas_usbhs_platform_info usbhs_rcar_gen3_plat_info;
diff --git a/drivers/usb/renesas_usbhs/rza.h b/drivers/usb/renesas_usbhs/rza.h
index 1ca42a6fd480..a29b75fef057 100644
--- a/drivers/usb/renesas_usbhs/rza.h
+++ b/drivers/usb/renesas_usbhs/rza.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 #include "common.h"
 
 extern const struct renesas_usbhs_platform_info usbhs_rza1_plat_info;
diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c
index 5b17709821df..27d92af29635 100644
--- a/drivers/usb/roles/class.c
+++ b/drivers/usb/roles/class.c
@@ -49,8 +49,10 @@ int usb_role_switch_set_role(struct usb_role_switch *sw, enum usb_role role)
 	mutex_lock(&sw->lock);
 
 	ret = sw->set(sw, role);
-	if (!ret)
+	if (!ret) {
 		sw->role = role;
+		kobject_uevent(&sw->dev.kobj, KOBJ_CHANGE);
+	}
 
 	mutex_unlock(&sw->lock);
 
diff --git a/drivers/usb/serial/Kconfig b/drivers/usb/serial/Kconfig
index 25d7e0c36d38..4007fa25a8ff 100644
--- a/drivers/usb/serial/Kconfig
+++ b/drivers/usb/serial/Kconfig
@@ -6,7 +6,7 @@
 menuconfig USB_SERIAL
 	tristate "USB Serial Converter support"
 	depends on TTY
-	---help---
+	help
 	  Say Y here if you have a USB device that provides normal serial
 	  ports, or acts like a serial device, and you want to connect it to
 	  your USB bus.
@@ -23,7 +23,7 @@ if USB_SERIAL
 config USB_SERIAL_CONSOLE
 	bool "USB Serial Console device support"
 	depends on USB_SERIAL=y
-	---help---
+	help
 	  If you say Y here, it will be possible to use a USB to serial
 	  converter port as the system console (the system console is the
 	  device which receives all kernel messages and warnings and which
@@ -123,7 +123,7 @@ config USB_SERIAL_WHITEHEAT
 
 config USB_SERIAL_DIGI_ACCELEPORT
 	tristate "USB Digi International AccelePort USB Serial Driver"
-	---help---
+	help
 	  Say Y here if you want to use Digi AccelePort USB 2 or 4 devices,
 	  2 port (plus parallel port) and 4 port USB serial converters.  The
 	  parallel port on the USB 2 appears as a third serial port on Linux.
@@ -168,7 +168,7 @@ config USB_SERIAL_EMPEG
 
 config USB_SERIAL_FTDI_SIO
 	tristate "USB FTDI Single Port Serial Driver"
-	---help---
+	help
 	  Say Y here if you want to use a FTDI SIO single port USB to serial
 	  converter device. The implementation I have is called the USC-1000.
 	  This driver has also been tested with the 245 and 232 devices.
@@ -213,7 +213,7 @@ config USB_SERIAL_IR
 
 config USB_SERIAL_EDGEPORT
 	tristate "USB Inside Out Edgeport Serial Driver"
-	---help---
+	help
 	  Say Y here if you want to use any of the following devices from
 	  Inside Out Networks (Digi):
 	  Edgeport/4
@@ -311,7 +311,7 @@ config USB_SERIAL_KEYSPAN_PDA
 config USB_SERIAL_KEYSPAN
 	tristate "USB Keyspan USA-xxx Serial Driver"
 	select USB_EZUSB_FX2
-	---help---
+	help
 	  Say Y here if you want to use Keyspan USB to serial converter
 	  devices.  This driver makes use of Keyspan's official firmware
 	  and was developed with their support.  You must also include
@@ -324,7 +324,7 @@ config USB_SERIAL_KEYSPAN
 
 config USB_SERIAL_KLSI
 	tristate "USB KL5KUSB105 (Palmconnect) Driver"
-	---help---
+	help
 	  Say Y here if you want to use a KL5KUSB105 - based single port
 	  serial adapter. The most widely known -- and currently the only
 	  tested -- device in this category is the PalmConnect USB Serial
@@ -339,7 +339,7 @@ config USB_SERIAL_KLSI
 
 config USB_SERIAL_KOBIL_SCT
 	tristate "USB KOBIL chipcard reader"
-	---help---
+	help
 	  Say Y here if you want to use one of the following KOBIL USB chipcard
 	  readers:
 
@@ -356,7 +356,7 @@ config USB_SERIAL_KOBIL_SCT
 
 config USB_SERIAL_MCT_U232
 	tristate "USB MCT Single Port Serial Driver"
-	---help---
+	help
 	  Say Y here if you want to use a USB Serial single port adapter from
 	  Magic Control Technology Corp. (U232 is one of the model numbers).
 
@@ -368,7 +368,7 @@ config USB_SERIAL_MCT_U232
 
 config USB_SERIAL_METRO
 	tristate "USB Metrologic Instruments USB-POS Barcode Scanner Driver"
-	---help---
+	help
 	  Say Y here if you want to use a USB POS Metrologic barcode scanner.
 
 	  To compile this driver as a module, choose M here: the
@@ -376,7 +376,7 @@ config USB_SERIAL_METRO
 
 config USB_SERIAL_MOS7720
 	tristate "USB Moschip 7720 Serial Driver"
-	---help---
+	help
 	  Say Y here if you want to use USB Serial single and double
 	  port adapters from Moschip Semiconductor Tech.
 
@@ -388,14 +388,14 @@ config USB_SERIAL_MOS7715_PARPORT
 	depends on USB_SERIAL_MOS7720
 	depends on PARPORT=y || PARPORT=USB_SERIAL_MOS7720
 	select PARPORT_NOT_PC
-	---help---
+	help
 	Say Y if you have a Moschip 7715 device and would like to use
 	the parallel port it provides.  The port will register with
 	the parport subsystem as a low-level driver.
 
 config USB_SERIAL_MOS7840
 	tristate "USB Moschip 7840/7820 USB Serial Driver"
-	---help---
+	help
 	  Say Y here if you want to use a MCS7840 Quad-Serial or MCS7820
 	  Dual-Serial port device from MosChip Semiconductor.
 
@@ -409,7 +409,7 @@ config USB_SERIAL_MOS7840
 
 config USB_SERIAL_MXUPORT
 	tristate "USB Moxa UPORT Serial Driver"
-	---help---
+	help
 	  Say Y here if you want to use a MOXA UPort Serial hub.
 
 	  This driver supports:
@@ -528,7 +528,7 @@ config USB_SERIAL_TI
 
 config USB_SERIAL_CYBERJACK
 	tristate "USB REINER SCT cyberJack pinpad/e-com chipcard reader"
-	---help---
+	help
 	  Say Y here if you want to use a cyberJack pinpad/e-com USB chipcard
 	  reader. This is an interface to ISO 7816 compatible contact-based
 	  chipcards, e.g. GSM SIMs.
diff --git a/drivers/usb/serial/belkin_sa.h b/drivers/usb/serial/belkin_sa.h
index a13a98d284f2..89ec30c63cc6 100644
--- a/drivers/usb/serial/belkin_sa.h
+++ b/drivers/usb/serial/belkin_sa.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Definitions for Belkin USB Serial Adapter Driver
  *
diff --git a/drivers/usb/serial/ch341.c b/drivers/usb/serial/ch341.c
index c5ecdcd51ffc..89675ee29645 100644
--- a/drivers/usb/serial/ch341.c
+++ b/drivers/usb/serial/ch341.c
@@ -73,6 +73,8 @@
 #define CH341_LCR_CS6          0x01
 #define CH341_LCR_CS5          0x00
 
+#define CH341_QUIRK_LIMITED_PRESCALER	BIT(0)
+
 static const struct usb_device_id id_table[] = {
 	{ USB_DEVICE(0x4348, 0x5523) },
 	{ USB_DEVICE(0x1a86, 0x7523) },
@@ -87,6 +89,7 @@ struct ch341_private {
 	u8 mcr;
 	u8 msr;
 	u8 lcr;
+	unsigned long quirks;
 };
 
 static void ch341_set_termios(struct tty_struct *tty,
@@ -159,9 +162,11 @@ static const speed_t ch341_min_rates[] = {
  *		2 <= div <= 256 if fact = 0, or
  *		9 <= div <= 256 if fact = 1
  */
-static int ch341_get_divisor(speed_t speed)
+static int ch341_get_divisor(struct ch341_private *priv)
 {
 	unsigned int fact, div, clk_div;
+	speed_t speed = priv->baud_rate;
+	bool force_fact0 = false;
 	int ps;
 
 	/*
@@ -187,8 +192,12 @@ static int ch341_get_divisor(speed_t speed)
 	clk_div = CH341_CLK_DIV(ps, fact);
 	div = CH341_CLKRATE / (clk_div * speed);
 
+	/* Some devices require a lower base clock if ps < 3. */
+	if (ps < 3 && (priv->quirks & CH341_QUIRK_LIMITED_PRESCALER))
+		force_fact0 = true;
+
 	/* Halve base clock (fact = 0) if required. */
-	if (div < 9 || div > 255) {
+	if (div < 9 || div > 255 || force_fact0) {
 		div /= 2;
 		clk_div *= 2;
 		fact = 0;
@@ -227,7 +236,7 @@ static int ch341_set_baudrate_lcr(struct usb_device *dev,
 	if (!priv->baud_rate)
 		return -EINVAL;
 
-	val = ch341_get_divisor(priv->baud_rate);
+	val = ch341_get_divisor(priv);
 	if (val < 0)
 		return -EINVAL;
 
@@ -308,6 +317,54 @@ out:	kfree(buffer);
 	return r;
 }
 
+static int ch341_detect_quirks(struct usb_serial_port *port)
+{
+	struct ch341_private *priv = usb_get_serial_port_data(port);
+	struct usb_device *udev = port->serial->dev;
+	const unsigned int size = 2;
+	unsigned long quirks = 0;
+	char *buffer;
+	int r;
+
+	buffer = kmalloc(size, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	/*
+	 * A subset of CH34x devices does not support all features. The
+	 * prescaler is limited and there is no support for sending a RS232
+	 * break condition. A read failure when trying to set up the latter is
+	 * used to detect these devices.
+	 */
+	r = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), CH341_REQ_READ_REG,
+			    USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN,
+			    CH341_REG_BREAK, 0, buffer, size, DEFAULT_TIMEOUT);
+	if (r == -EPIPE) {
+		dev_dbg(&port->dev, "break control not supported\n");
+		quirks = CH341_QUIRK_LIMITED_PRESCALER;
+		r = 0;
+		goto out;
+	}
+
+	if (r != size) {
+		if (r >= 0)
+			r = -EIO;
+		dev_err(&port->dev, "failed to read break control: %d\n", r);
+		goto out;
+	}
+
+	r = 0;
+out:
+	kfree(buffer);
+
+	if (quirks) {
+		dev_dbg(&port->dev, "enabling quirk flags: 0x%02lx\n", quirks);
+		priv->quirks |= quirks;
+	}
+
+	return r;
+}
+
 static int ch341_port_probe(struct usb_serial_port *port)
 {
 	struct ch341_private *priv;
@@ -330,6 +387,11 @@ static int ch341_port_probe(struct usb_serial_port *port)
 		goto error;
 
 	usb_set_serial_port_data(port, priv);
+
+	r = ch341_detect_quirks(port);
+	if (r < 0)
+		goto error;
+
 	return 0;
 
 error:	kfree(priv);
diff --git a/drivers/usb/serial/io_16654.h b/drivers/usb/serial/io_16654.h
index 4980f72dc56f..f18501f056cf 100644
--- a/drivers/usb/serial/io_16654.h
+++ b/drivers/usb/serial/io_16654.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /************************************************************************
  *
  *	16654.H		Definitions for 16C654 UART used on EdgePorts
diff --git a/drivers/usb/serial/io_edgeport.h b/drivers/usb/serial/io_edgeport.h
index 2e7fedbaf2ff..43ba53a3a6fa 100644
--- a/drivers/usb/serial/io_edgeport.h
+++ b/drivers/usb/serial/io_edgeport.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /************************************************************************
  *
  *	io_edgeport.h	Edgeport Linux Interface definitions
diff --git a/drivers/usb/serial/io_ionsp.h b/drivers/usb/serial/io_ionsp.h
index 4b8e4823bd45..db4fce815c97 100644
--- a/drivers/usb/serial/io_ionsp.h
+++ b/drivers/usb/serial/io_ionsp.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /************************************************************************
  *
  *	IONSP.H		Definitions for I/O Networks Serial Protocol
diff --git a/drivers/usb/serial/io_ti.h b/drivers/usb/serial/io_ti.h
index 9bbcee37524e..50b899d55ed0 100644
--- a/drivers/usb/serial/io_ti.h
+++ b/drivers/usb/serial/io_ti.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*****************************************************************************
  *
  *	Copyright (C) 1997-2002 Inside Out Networks, Inc.
diff --git a/drivers/usb/serial/io_usbvend.h b/drivers/usb/serial/io_usbvend.h
index 0d1a5bb4636e..52cbc353051f 100644
--- a/drivers/usb/serial/io_usbvend.h
+++ b/drivers/usb/serial/io_usbvend.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /************************************************************************
  *
  *	USBVEND.H		Vendor-specific USB definitions
diff --git a/drivers/usb/serial/iuu_phoenix.h b/drivers/usb/serial/iuu_phoenix.h
index b400b262f72e..87992b24d904 100644
--- a/drivers/usb/serial/iuu_phoenix.h
+++ b/drivers/usb/serial/iuu_phoenix.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Infinity Unlimited USB Phoenix driver
  *
diff --git a/drivers/usb/serial/mct_u232.h b/drivers/usb/serial/mct_u232.h
index 0084edf518e8..e3d09a83cab1 100644
--- a/drivers/usb/serial/mct_u232.h
+++ b/drivers/usb/serial/mct_u232.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Definitions for MCT (Magic Control Technology) USB-RS232 Converter Driver
  *
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index 8bfffca3e4ae..254a8bbeea67 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -1157,6 +1157,10 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_CC864_SINGLE) },
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_DE910_DUAL) },
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_UE910_V2) },
+	{ USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1031, 0xff),	/* Telit LE910C1-EUX */
+	 .driver_info = NCTRL(0) | RSVD(3) },
+	{ USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1033, 0xff),	/* Telit LE910C1-EUX (ECM) */
+	 .driver_info = NCTRL(0) },
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG0),
 	  .driver_info = RSVD(0) | RSVD(1) | NCTRL(2) | RSVD(3) },
 	{ USB_DEVICE(TELIT_VENDOR_ID, TELIT_PRODUCT_LE922_USBCFG1),
diff --git a/drivers/usb/serial/oti6858.h b/drivers/usb/serial/oti6858.h
index 1226bf2347eb..5c25836fdcd9 100644
--- a/drivers/usb/serial/oti6858.h
+++ b/drivers/usb/serial/oti6858.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Ours Technology Inc. OTi-6858 USB to serial adapter driver.
  */
diff --git a/drivers/usb/serial/pl2303.h b/drivers/usb/serial/pl2303.h
index 52db5519aaf0..7d3090ee7e0c 100644
--- a/drivers/usb/serial/pl2303.h
+++ b/drivers/usb/serial/pl2303.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Prolific PL2303 USB to serial adaptor driver header file
  */
diff --git a/drivers/usb/serial/qcserial.c b/drivers/usb/serial/qcserial.c
index ce0401d3137f..d147feae83e6 100644
--- a/drivers/usb/serial/qcserial.c
+++ b/drivers/usb/serial/qcserial.c
@@ -173,6 +173,7 @@ static const struct usb_device_id id_table[] = {
 	{DEVICE_SWI(0x413c, 0x81b3)},	/* Dell Wireless 5809e Gobi(TM) 4G LTE Mobile Broadband Card (rev3) */
 	{DEVICE_SWI(0x413c, 0x81b5)},	/* Dell Wireless 5811e QDL */
 	{DEVICE_SWI(0x413c, 0x81b6)},	/* Dell Wireless 5811e QDL */
+	{DEVICE_SWI(0x413c, 0x81cb)},	/* Dell Wireless 5816e QDL */
 	{DEVICE_SWI(0x413c, 0x81cc)},	/* Dell Wireless 5816e */
 	{DEVICE_SWI(0x413c, 0x81cf)},   /* Dell Wireless 5819 */
 	{DEVICE_SWI(0x413c, 0x81d0)},   /* Dell Wireless 5819 */
diff --git a/drivers/usb/serial/usb_wwan.c b/drivers/usb/serial/usb_wwan.c
index 13be21aad2f4..4b9845807bee 100644
--- a/drivers/usb/serial/usb_wwan.c
+++ b/drivers/usb/serial/usb_wwan.c
@@ -270,6 +270,10 @@ static void usb_wwan_indat_callback(struct urb *urb)
 	if (status) {
 		dev_dbg(dev, "%s: nonzero status: %d on endpoint %02x.\n",
 			__func__, status, endpoint);
+
+		/* don't resubmit on fatal errors */
+		if (status == -ESHUTDOWN || status == -ENOENT)
+			return;
 	} else {
 		if (urb->actual_length) {
 			tty_insert_flip_string(&port->port, data,
diff --git a/drivers/usb/serial/visor.h b/drivers/usb/serial/visor.h
index 4bd69d047036..622d639ce74e 100644
--- a/drivers/usb/serial/visor.h
+++ b/drivers/usb/serial/visor.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * USB HandSpring Visor driver
  *
diff --git a/drivers/usb/serial/whiteheat.h b/drivers/usb/serial/whiteheat.h
index 269e727a92f9..7e63074c9128 100644
--- a/drivers/usb/serial/whiteheat.h
+++ b/drivers/usb/serial/whiteheat.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * USB ConnectTech WhiteHEAT driver
  *
diff --git a/drivers/usb/storage/Kconfig b/drivers/usb/storage/Kconfig
index 59aad38b490a..5335a7ff5d14 100644
--- a/drivers/usb/storage/Kconfig
+++ b/drivers/usb/storage/Kconfig
@@ -9,7 +9,7 @@ comment "also be needed; see USB_STORAGE Help for more info"
 config USB_STORAGE
 	tristate "USB Mass Storage support"
 	depends on SCSI
-	---help---
+	help
 	  Say Y here if you want to connect USB mass storage devices to your
 	  computer's USB port. This is the driver you need for USB
 	  floppy drives, USB hard disks, USB tape drives, USB CD-ROMs,
@@ -63,7 +63,7 @@ config USB_STORAGE_FREECOM
 
 config USB_STORAGE_ISD200
 	tristate "ISD-200 USB/ATA Bridge support"
-	---help---
+	help
 	  Say Y here if you want to use USB Mass Store devices based
 	  on the In-Systems Design ISD-200 USB/ATA bridge.
 
@@ -165,7 +165,7 @@ config USB_STORAGE_KARMA
 
 config USB_STORAGE_CYPRESS_ATACB
 	tristate "SAT emulation on Cypress USB/ATA Bridge with ATACB"
-	---help---
+	help
 	  Say Y here if you want to use SAT (ata pass through) on devices based
 	  on the Cypress USB/ATA bridge supporting ATACB. This will allow you
 	  to use tools to tune and monitor your drive (like hdparm or smartctl).
@@ -177,7 +177,7 @@ config USB_STORAGE_CYPRESS_ATACB
 
 config USB_STORAGE_ENE_UB6250
 	tristate "USB ENE card reader support"
-	---help---
+	help
 	  Say Y here if you wish to control a ENE SD/MS Card reader.
 	  Note that this driver does not support SM cards.
 
diff --git a/drivers/usb/storage/debug.h b/drivers/usb/storage/debug.h
index 16ce06039a4d..a6505ceb6693 100644
--- a/drivers/usb/storage/debug.h
+++ b/drivers/usb/storage/debug.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for USB Mass Storage compliant devices
  * Debugging Functions Header File
diff --git a/drivers/usb/storage/initializers.h b/drivers/usb/storage/initializers.h
index 2dbf9c7d9749..dcd7b7e5eda8 100644
--- a/drivers/usb/storage/initializers.h
+++ b/drivers/usb/storage/initializers.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Header file for Special Initializers for certain USB Mass Storage devices
  *
diff --git a/drivers/usb/storage/protocol.h b/drivers/usb/storage/protocol.h
index 072f1ffda2af..1d102463a66c 100644
--- a/drivers/usb/storage/protocol.h
+++ b/drivers/usb/storage/protocol.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for USB Mass Storage compliant devices
  * Protocol Functions Header File
diff --git a/drivers/usb/storage/scsiglue.h b/drivers/usb/storage/scsiglue.h
index 2bc5ea045bf7..2a79c3ed4d86 100644
--- a/drivers/usb/storage/scsiglue.h
+++ b/drivers/usb/storage/scsiglue.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for USB Mass Storage compliant devices
  * SCSI Connecting Glue Header File
diff --git a/drivers/usb/storage/sierra_ms.c b/drivers/usb/storage/sierra_ms.c
index e605cbc3d8bf..b9f78ef3edc3 100644
--- a/drivers/usb/storage/sierra_ms.c
+++ b/drivers/usb/storage/sierra_ms.c
@@ -129,15 +129,11 @@ int sierra_ms_init(struct us_data *us)
 	int result, retries;
 	struct swoc_info *swocInfo;
 	struct usb_device *udev;
-	struct Scsi_Host *sh;
 
 	retries = 3;
 	result = 0;
 	udev = us->pusb_dev;
 
-	sh = us_to_host(us);
-	scsi_get_host_dev(sh);
-
 	/* Force Modem mode */
 	if (swi_tru_install == TRU_FORCE_MODEM) {
 		usb_stor_dbg(us, "SWIMS: Forcing Modem Mode\n");
diff --git a/drivers/usb/storage/transport.h b/drivers/usb/storage/transport.h
index fb3bb4ee4ccf..74ffd0d7e7b6 100644
--- a/drivers/usb/storage/transport.h
+++ b/drivers/usb/storage/transport.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for USB Mass Storage compliant devices
  * Transport Functions Header File
diff --git a/drivers/usb/storage/unusual_alauda.h b/drivers/usb/storage/unusual_alauda.h
index 0ec8c99a4976..13f61ec88cde 100644
--- a/drivers/usb/storage/unusual_alauda.h
+++ b/drivers/usb/storage/unusual_alauda.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for the Alauda-based card readers
  */
diff --git a/drivers/usb/storage/unusual_cypress.h b/drivers/usb/storage/unusual_cypress.h
index fb99e526cd48..0547daf116a2 100644
--- a/drivers/usb/storage/unusual_cypress.h
+++ b/drivers/usb/storage/unusual_cypress.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for devices based on the Cypress USB/ATA bridge
  *	with support for ATACB
diff --git a/drivers/usb/storage/unusual_datafab.h b/drivers/usb/storage/unusual_datafab.h
index fdab5e7d68ca..5335b5d2bd79 100644
--- a/drivers/usb/storage/unusual_datafab.h
+++ b/drivers/usb/storage/unusual_datafab.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for the Datafab USB Compact Flash reader
  */
diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h
index f6c3681fa2e9..b6a9a7451620 100644
--- a/drivers/usb/storage/unusual_devs.h
+++ b/drivers/usb/storage/unusual_devs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for USB Mass Storage compliant devices
  * Unusual Devices File
diff --git a/drivers/usb/storage/unusual_ene_ub6250.h b/drivers/usb/storage/unusual_ene_ub6250.h
index 9134b91fbd73..a3b32abc2b2f 100644
--- a/drivers/usb/storage/unusual_ene_ub6250.h
+++ b/drivers/usb/storage/unusual_ene_ub6250.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 #if defined(CONFIG_USB_STORAGE_ENE_UB6250) || \
 		defined(CONFIG_USB_STORAGE_ENE_UB6250_MODULE)
 
diff --git a/drivers/usb/storage/unusual_freecom.h b/drivers/usb/storage/unusual_freecom.h
index 949231c7a36b..9ca686364a93 100644
--- a/drivers/usb/storage/unusual_freecom.h
+++ b/drivers/usb/storage/unusual_freecom.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for the Freecom USB/IDE adaptor
  */
diff --git a/drivers/usb/storage/unusual_isd200.h b/drivers/usb/storage/unusual_isd200.h
index d03a02cc904e..f248190bd666 100644
--- a/drivers/usb/storage/unusual_isd200.h
+++ b/drivers/usb/storage/unusual_isd200.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for In-System Design, Inc. ISD200 ASIC
  */
diff --git a/drivers/usb/storage/unusual_jumpshot.h b/drivers/usb/storage/unusual_jumpshot.h
index c323338881ef..44878f849c1c 100644
--- a/drivers/usb/storage/unusual_jumpshot.h
+++ b/drivers/usb/storage/unusual_jumpshot.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for the Lexar "Jumpshot" Compact Flash reader
  */
diff --git a/drivers/usb/storage/unusual_karma.h b/drivers/usb/storage/unusual_karma.h
index 8f1eebd71d2c..9fbed4cbc895 100644
--- a/drivers/usb/storage/unusual_karma.h
+++ b/drivers/usb/storage/unusual_karma.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for the Rio Karma
  */
diff --git a/drivers/usb/storage/unusual_onetouch.h b/drivers/usb/storage/unusual_onetouch.h
index c76d4e990f7b..cdfee8f6cf37 100644
--- a/drivers/usb/storage/unusual_onetouch.h
+++ b/drivers/usb/storage/unusual_onetouch.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for the Maxtor OneTouch USB hard drive's button
  */
diff --git a/drivers/usb/storage/unusual_realtek.h b/drivers/usb/storage/unusual_realtek.h
index 7e14c2d7cf73..945dcb19d31d 100644
--- a/drivers/usb/storage/unusual_realtek.h
+++ b/drivers/usb/storage/unusual_realtek.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for Realtek RTS51xx USB card reader
  *
diff --git a/drivers/usb/storage/unusual_sddr09.h b/drivers/usb/storage/unusual_sddr09.h
index 650cf2862754..bfb650974129 100644
--- a/drivers/usb/storage/unusual_sddr09.h
+++ b/drivers/usb/storage/unusual_sddr09.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for SanDisk SDDR-09 SmartMedia reader
  */
diff --git a/drivers/usb/storage/unusual_sddr55.h b/drivers/usb/storage/unusual_sddr55.h
index e89df2cea7bd..6d6f76eb0630 100644
--- a/drivers/usb/storage/unusual_sddr55.h
+++ b/drivers/usb/storage/unusual_sddr55.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for SanDisk SDDR-55 SmartMedia reader
  */
diff --git a/drivers/usb/storage/unusual_uas.h b/drivers/usb/storage/unusual_uas.h
index 37157ed9a881..162b09d69f62 100644
--- a/drivers/usb/storage/unusual_uas.h
+++ b/drivers/usb/storage/unusual_uas.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for USB Attached SCSI devices - Unusual Devices File
  *
diff --git a/drivers/usb/storage/unusual_usbat.h b/drivers/usb/storage/unusual_usbat.h
index 05abf6870b8f..f9d3e5efc39d 100644
--- a/drivers/usb/storage/unusual_usbat.h
+++ b/drivers/usb/storage/unusual_usbat.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Unusual Devices File for SCM Microsystems (a.k.a. Shuttle) USB-ATAPI cable
  */
diff --git a/drivers/usb/storage/usb.h b/drivers/usb/storage/usb.h
index 5850d624cac7..0451fac1adce 100644
--- a/drivers/usb/storage/usb.h
+++ b/drivers/usb/storage/usb.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Driver for USB Mass Storage compliant devices
  * Main Header File
diff --git a/drivers/usb/typec/Kconfig b/drivers/usb/typec/Kconfig
index b4f2aac7ae8a..559dd06117e7 100644
--- a/drivers/usb/typec/Kconfig
+++ b/drivers/usb/typec/Kconfig
@@ -64,7 +64,8 @@ config TYPEC_HD3SS3220
 config TYPEC_TPS6598X
 	tristate "TI TPS6598x USB Power Delivery controller driver"
 	depends on I2C
-	select REGMAP_I2C
+	depends on REGMAP_I2C
+	depends on USB_ROLE_SWITCH || !USB_ROLE_SWITCH
 	help
 	  Say Y or M here if your system has TI TPS65982 or TPS65983 USB Power
 	  Delivery controller.
diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c
index 8d894bdff77d..c9234748537a 100644
--- a/drivers/usb/typec/class.c
+++ b/drivers/usb/typec/class.c
@@ -917,6 +917,12 @@ EXPORT_SYMBOL_GPL(typec_unregister_cable);
 /* ------------------------------------------------------------------------- */
 /* USB Type-C ports */
 
+static const char * const typec_orientations[] = {
+	[TYPEC_ORIENTATION_NONE]	= "unknown",
+	[TYPEC_ORIENTATION_NORMAL]	= "normal",
+	[TYPEC_ORIENTATION_REVERSE]	= "reverse",
+};
+
 static const char * const typec_roles[] = {
 	[TYPEC_SINK]	= "sink",
 	[TYPEC_SOURCE]	= "source",
@@ -1248,18 +1254,9 @@ static ssize_t orientation_show(struct device *dev,
 				   struct device_attribute *attr,
 				   char *buf)
 {
-	struct typec_port *p = to_typec_port(dev);
-	enum typec_orientation orientation = typec_get_orientation(p);
-
-	switch (orientation) {
-	case TYPEC_ORIENTATION_NORMAL:
-		return sprintf(buf, "%s\n", "normal");
-	case TYPEC_ORIENTATION_REVERSE:
-		return sprintf(buf, "%s\n", "reverse");
-	case TYPEC_ORIENTATION_NONE:
-	default:
-		return sprintf(buf, "%s\n", "unknown");
-	}
+	struct typec_port *port = to_typec_port(dev);
+
+	return sprintf(buf, "%s\n", typec_orientations[port->orientation]);
 }
 static DEVICE_ATTR_RO(orientation);
 
@@ -1452,6 +1449,21 @@ void typec_set_pwr_opmode(struct typec_port *port,
 EXPORT_SYMBOL_GPL(typec_set_pwr_opmode);
 
 /**
+ * typec_find_orientation - Convert orientation string to enum typec_orientation
+ * @name: Orientation string
+ *
+ * This routine is used to find the typec_orientation by its string name @name.
+ *
+ * Returns the orientation value on success, otherwise negative error code.
+ */
+int typec_find_orientation(const char *name)
+{
+	return match_string(typec_orientations, ARRAY_SIZE(typec_orientations),
+			    name);
+}
+EXPORT_SYMBOL_GPL(typec_find_orientation);
+
+/**
  * typec_find_port_power_role - Get the typec port power capability
  * @name: port power capability string
  *
diff --git a/drivers/usb/typec/mux/intel_pmc_mux.c b/drivers/usb/typec/mux/intel_pmc_mux.c
index 1ac0a3eb7dd8..962bc69a6a59 100644
--- a/drivers/usb/typec/mux/intel_pmc_mux.c
+++ b/drivers/usb/typec/mux/intel_pmc_mux.c
@@ -92,6 +92,9 @@ struct pmc_usb_port {
 
 	u8 usb2_port;
 	u8 usb3_port;
+
+	enum typec_orientation sbu_orientation;
+	enum typec_orientation hsl_orientation;
 };
 
 struct pmc_usb {
@@ -101,6 +104,22 @@ struct pmc_usb {
 	struct pmc_usb_port *port;
 };
 
+static int sbu_orientation(struct pmc_usb_port *port)
+{
+	if (port->sbu_orientation)
+		return port->sbu_orientation - 1;
+
+	return port->orientation - 1;
+}
+
+static int hsl_orientation(struct pmc_usb_port *port)
+{
+	if (port->hsl_orientation)
+		return port->hsl_orientation - 1;
+
+	return port->orientation - 1;
+}
+
 static int pmc_usb_command(struct pmc_usb_port *port, u8 *msg, u32 len)
 {
 	u8 response[4];
@@ -152,8 +171,9 @@ pmc_usb_mux_dp(struct pmc_usb_port *port, struct typec_mux_state *state)
 
 	req.mode_data = (port->orientation - 1) << PMC_USB_ALTMODE_ORI_SHIFT;
 	req.mode_data |= (port->role - 1) << PMC_USB_ALTMODE_UFP_SHIFT;
-	req.mode_data |= (port->orientation - 1) << PMC_USB_ALTMODE_ORI_AUX_SHIFT;
-	req.mode_data |= (port->orientation - 1) << PMC_USB_ALTMODE_ORI_HSL_SHIFT;
+
+	req.mode_data |= sbu_orientation(port) << PMC_USB_ALTMODE_ORI_AUX_SHIFT;
+	req.mode_data |= hsl_orientation(port) << PMC_USB_ALTMODE_ORI_HSL_SHIFT;
 
 	req.mode_data |= (state->mode - TYPEC_STATE_MODAL) <<
 			 PMC_USB_ALTMODE_DP_MODE_SHIFT;
@@ -177,8 +197,9 @@ pmc_usb_mux_tbt(struct pmc_usb_port *port, struct typec_mux_state *state)
 
 	req.mode_data = (port->orientation - 1) << PMC_USB_ALTMODE_ORI_SHIFT;
 	req.mode_data |= (port->role - 1) << PMC_USB_ALTMODE_UFP_SHIFT;
-	req.mode_data |= (port->orientation - 1) << PMC_USB_ALTMODE_ORI_AUX_SHIFT;
-	req.mode_data |= (port->orientation - 1) << PMC_USB_ALTMODE_ORI_HSL_SHIFT;
+
+	req.mode_data |= sbu_orientation(port) << PMC_USB_ALTMODE_ORI_AUX_SHIFT;
+	req.mode_data |= hsl_orientation(port) << PMC_USB_ALTMODE_ORI_HSL_SHIFT;
 
 	if (TBT_ADAPTER(data->device_mode) == TBT_ADAPTER_TBT3)
 		req.mode_data |= PMC_USB_ALTMODE_TBT_TYPE;
@@ -215,8 +236,8 @@ static int pmc_usb_connect(struct pmc_usb_port *port)
 	msg[0] |= port->usb3_port << PMC_USB_MSG_USB3_PORT_SHIFT;
 
 	msg[1] = port->usb2_port << PMC_USB_MSG_USB2_PORT_SHIFT;
-	msg[1] |= (port->orientation - 1) << PMC_USB_MSG_ORI_HSL_SHIFT;
-	msg[1] |= (port->orientation - 1) << PMC_USB_MSG_ORI_AUX_SHIFT;
+	msg[1] |= hsl_orientation(port) << PMC_USB_MSG_ORI_HSL_SHIFT;
+	msg[1] |= sbu_orientation(port) << PMC_USB_MSG_ORI_AUX_SHIFT;
 
 	return pmc_usb_command(port, msg, sizeof(msg));
 }
@@ -300,6 +321,7 @@ static int pmc_usb_register_port(struct pmc_usb *pmc, int index,
 	struct usb_role_switch_desc desc = { };
 	struct typec_switch_desc sw_desc = { };
 	struct typec_mux_desc mux_desc = { };
+	const char *str;
 	int ret;
 
 	ret = fwnode_property_read_u8(fwnode, "usb2-port-number", &port->usb2_port);
@@ -310,6 +332,14 @@ static int pmc_usb_register_port(struct pmc_usb *pmc, int index,
 	if (ret)
 		return ret;
 
+	ret = fwnode_property_read_string(fwnode, "sbu-orientation", &str);
+	if (!ret)
+		port->sbu_orientation = typec_find_orientation(str);
+
+	ret = fwnode_property_read_string(fwnode, "hsl-orientation", &str);
+	if (!ret)
+		port->hsl_orientation = typec_find_orientation(str);
+
 	port->num = index;
 	port->pmc = pmc;
 
diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c
index b498960ff72b..b28facece43c 100644
--- a/drivers/usb/typec/tcpm/fusb302.c
+++ b/drivers/usb/typec/tcpm/fusb302.c
@@ -9,14 +9,13 @@
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/extcon.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/of_device.h>
-#include <linux/of_gpio.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/proc_fs.h>
 #include <linux/regulator/consumer.h>
@@ -83,7 +82,7 @@ struct fusb302_chip {
 	struct work_struct irq_work;
 	bool irq_suspended;
 	bool irq_while_suspended;
-	int gpio_int_n;
+	struct gpio_desc *gpio_int_n;
 	int gpio_int_n_irq;
 	struct extcon_dev *extcon;
 
@@ -1618,30 +1617,17 @@ done:
 
 static int init_gpio(struct fusb302_chip *chip)
 {
-	struct device_node *node;
+	struct device *dev = chip->dev;
 	int ret = 0;
 
-	node = chip->dev->of_node;
-	chip->gpio_int_n = of_get_named_gpio(node, "fcs,int_n", 0);
-	if (!gpio_is_valid(chip->gpio_int_n)) {
-		ret = chip->gpio_int_n;
-		dev_err(chip->dev, "cannot get named GPIO Int_N, ret=%d", ret);
-		return ret;
-	}
-	ret = devm_gpio_request(chip->dev, chip->gpio_int_n, "fcs,int_n");
-	if (ret < 0) {
-		dev_err(chip->dev, "cannot request GPIO Int_N, ret=%d", ret);
-		return ret;
-	}
-	ret = gpio_direction_input(chip->gpio_int_n);
-	if (ret < 0) {
-		dev_err(chip->dev,
-			"cannot set GPIO Int_N to input, ret=%d", ret);
-		return ret;
+	chip->gpio_int_n = devm_gpiod_get(dev, "fcs,int_n", GPIOD_IN);
+	if (IS_ERR(chip->gpio_int_n)) {
+		dev_err(dev, "failed to request gpio_int_n\n");
+		return PTR_ERR(chip->gpio_int_n);
 	}
-	ret = gpio_to_irq(chip->gpio_int_n);
+	ret = gpiod_to_irq(chip->gpio_int_n);
 	if (ret < 0) {
-		dev_err(chip->dev,
+		dev_err(dev,
 			"cannot request IRQ for GPIO Int_N, ret=%d", ret);
 		return ret;
 	}
diff --git a/drivers/usb/typec/tcpm/fusb302_reg.h b/drivers/usb/typec/tcpm/fusb302_reg.h
index 00b39d365478..edc0e4b0f1e6 100644
--- a/drivers/usb/typec/tcpm/fusb302_reg.h
+++ b/drivers/usb/typec/tcpm/fusb302_reg.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright 2016-2017 Google, Inc
  *
diff --git a/drivers/usb/typec/tps6598x.c b/drivers/usb/typec/tps6598x.c
index 0698addd1185..b7c9fe5caabe 100644
--- a/drivers/usb/typec/tps6598x.c
+++ b/drivers/usb/typec/tps6598x.c
@@ -12,6 +12,7 @@
 #include <linux/regmap.h>
 #include <linux/interrupt.h>
 #include <linux/usb/typec.h>
+#include <linux/usb/role.h>
 
 /* Register offsets */
 #define TPS_REG_VID			0x00
@@ -94,6 +95,7 @@ struct tps6598x {
 	struct typec_port *port;
 	struct typec_partner *partner;
 	struct usb_pd_identity partner_identity;
+	struct usb_role_switch *role_sw;
 };
 
 /*
@@ -190,6 +192,23 @@ static int tps6598x_read_partner_identity(struct tps6598x *tps)
 	return 0;
 }
 
+static void tps6598x_set_data_role(struct tps6598x *tps,
+				   enum typec_data_role role, bool connected)
+{
+	enum usb_role role_val;
+
+	if (role == TYPEC_HOST)
+		role_val = USB_ROLE_HOST;
+	else
+		role_val = USB_ROLE_DEVICE;
+
+	if (!connected)
+		role_val = USB_ROLE_NONE;
+
+	usb_role_switch_set_role(tps->role_sw, role_val);
+	typec_set_data_role(tps->port, role);
+}
+
 static int tps6598x_connect(struct tps6598x *tps, u32 status)
 {
 	struct typec_partner_desc desc;
@@ -220,7 +239,7 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status)
 	typec_set_pwr_opmode(tps->port, mode);
 	typec_set_pwr_role(tps->port, TPS_STATUS_PORTROLE(status));
 	typec_set_vconn_role(tps->port, TPS_STATUS_VCONN(status));
-	typec_set_data_role(tps->port, TPS_STATUS_DATAROLE(status));
+	tps6598x_set_data_role(tps, TPS_STATUS_DATAROLE(status), true);
 
 	tps->partner = typec_register_partner(tps->port, &desc);
 	if (IS_ERR(tps->partner))
@@ -240,7 +259,7 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status)
 	typec_set_pwr_opmode(tps->port, TYPEC_PWR_MODE_USB);
 	typec_set_pwr_role(tps->port, TPS_STATUS_PORTROLE(status));
 	typec_set_vconn_role(tps->port, TPS_STATUS_VCONN(status));
-	typec_set_data_role(tps->port, TPS_STATUS_DATAROLE(status));
+	tps6598x_set_data_role(tps, TPS_STATUS_DATAROLE(status), false);
 }
 
 static int tps6598x_exec_cmd(struct tps6598x *tps, const char *cmd,
@@ -328,7 +347,7 @@ static int tps6598x_dr_set(struct typec_port *port, enum typec_data_role role)
 		goto out_unlock;
 	}
 
-	typec_set_data_role(tps->port, role);
+	tps6598x_set_data_role(tps, role, true);
 
 out_unlock:
 	mutex_unlock(&tps->lock);
@@ -452,6 +471,7 @@ static int tps6598x_probe(struct i2c_client *client)
 {
 	struct typec_capability typec_cap = { };
 	struct tps6598x *tps;
+	struct fwnode_handle *fwnode;
 	u32 status;
 	u32 conf;
 	u32 vid;
@@ -495,11 +515,22 @@ static int tps6598x_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
+	fwnode = device_get_named_child_node(&client->dev, "connector");
+	if (IS_ERR(fwnode))
+		return PTR_ERR(fwnode);
+
+	tps->role_sw = fwnode_usb_role_switch_get(fwnode);
+	if (IS_ERR(tps->role_sw)) {
+		ret = PTR_ERR(tps->role_sw);
+		goto err_fwnode_put;
+	}
+
 	typec_cap.revision = USB_TYPEC_REV_1_2;
 	typec_cap.pd_revision = 0x200;
 	typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE;
 	typec_cap.driver_data = tps;
 	typec_cap.ops = &tps6598x_ops;
+	typec_cap.fwnode = fwnode;
 
 	switch (TPS_SYSCONF_PORTINFO(conf)) {
 	case TPS_PORTINFO_SINK_ACCESSORY:
@@ -525,12 +556,16 @@ static int tps6598x_probe(struct i2c_client *client)
 		typec_cap.data = TYPEC_PORT_DFP;
 		break;
 	default:
-		return -ENODEV;
+		ret = -ENODEV;
+		goto err_role_put;
 	}
 
 	tps->port = typec_register_port(&client->dev, &typec_cap);
-	if (IS_ERR(tps->port))
-		return PTR_ERR(tps->port);
+	if (IS_ERR(tps->port)) {
+		ret = PTR_ERR(tps->port);
+		goto err_role_put;
+	}
+	fwnode_handle_put(fwnode);
 
 	if (status & TPS_STATUS_PLUG_PRESENT) {
 		ret = tps6598x_connect(tps, status);
@@ -545,12 +580,19 @@ static int tps6598x_probe(struct i2c_client *client)
 	if (ret) {
 		tps6598x_disconnect(tps, 0);
 		typec_unregister_port(tps->port);
-		return ret;
+		goto err_role_put;
 	}
 
 	i2c_set_clientdata(client, tps);
 
 	return 0;
+
+err_role_put:
+	usb_role_switch_put(tps->role_sw);
+err_fwnode_put:
+	fwnode_handle_put(fwnode);
+
+	return ret;
 }
 
 static int tps6598x_remove(struct i2c_client *client)
@@ -559,10 +601,17 @@ static int tps6598x_remove(struct i2c_client *client)
 
 	tps6598x_disconnect(tps, 0);
 	typec_unregister_port(tps->port);
+	usb_role_switch_put(tps->role_sw);
 
 	return 0;
 }
 
+static const struct of_device_id tps6598x_of_match[] = {
+	{ .compatible = "ti,tps6598x", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, tps6598x_of_match);
+
 static const struct i2c_device_id tps6598x_id[] = {
 	{ "tps6598x" },
 	{ }
@@ -572,6 +621,7 @@ MODULE_DEVICE_TABLE(i2c, tps6598x_id);
 static struct i2c_driver tps6598x_i2c_driver = {
 	.driver = {
 		.name = "tps6598x",
+		.of_match_table = tps6598x_of_match,
 	},
 	.probe_new = tps6598x_probe,
 	.remove = tps6598x_remove,
diff --git a/drivers/usb/typec/ucsi/Makefile b/drivers/usb/typec/ucsi/Makefile
index b35e15a1f02c..8a8eb5cb8e0f 100644
--- a/drivers/usb/typec/ucsi/Makefile
+++ b/drivers/usb/typec/ucsi/Makefile
@@ -7,6 +7,10 @@ typec_ucsi-y				:= ucsi.o
 
 typec_ucsi-$(CONFIG_TRACING)		+= trace.o
 
+ifneq ($(CONFIG_POWER_SUPPLY),)
+	typec_ucsi-y			+= psy.o
+endif
+
 ifneq ($(CONFIG_TYPEC_DP_ALTMODE),)
 	typec_ucsi-y			+= displayport.o
 endif
diff --git a/drivers/usb/typec/ucsi/psy.c b/drivers/usb/typec/ucsi/psy.c
new file mode 100644
index 000000000000..26ed0b520749
--- /dev/null
+++ b/drivers/usb/typec/ucsi/psy.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Power Supply for UCSI
+ *
+ * Copyright (C) 2020, Intel Corporation
+ * Author: K V, Abhilash <abhilash.k.v@intel.com>
+ * Author: Heikki Krogerus <heikki.krogerus@linux.intel.com>
+ */
+
+#include <linux/property.h>
+#include <linux/usb/pd.h>
+
+#include "ucsi.h"
+
+/* Power Supply access to expose source power information */
+enum ucsi_psy_online_states {
+	UCSI_PSY_OFFLINE = 0,
+	UCSI_PSY_FIXED_ONLINE,
+	UCSI_PSY_PROG_ONLINE,
+};
+
+static enum power_supply_property ucsi_psy_props[] = {
+	POWER_SUPPLY_PROP_USB_TYPE,
+	POWER_SUPPLY_PROP_ONLINE,
+	POWER_SUPPLY_PROP_VOLTAGE_MIN,
+	POWER_SUPPLY_PROP_VOLTAGE_MAX,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_MAX,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+};
+
+static int ucsi_psy_get_online(struct ucsi_connector *con,
+			       union power_supply_propval *val)
+{
+	val->intval = UCSI_PSY_OFFLINE;
+	if (con->status.flags & UCSI_CONSTAT_CONNECTED &&
+	    (con->status.flags & UCSI_CONSTAT_PWR_DIR) == TYPEC_SINK)
+		val->intval = UCSI_PSY_FIXED_ONLINE;
+	return 0;
+}
+
+static int ucsi_psy_get_voltage_min(struct ucsi_connector *con,
+				    union power_supply_propval *val)
+{
+	u32 pdo;
+
+	switch (UCSI_CONSTAT_PWR_OPMODE(con->status.flags)) {
+	case UCSI_CONSTAT_PWR_OPMODE_PD:
+		pdo = con->src_pdos[0];
+		val->intval = pdo_fixed_voltage(pdo) * 1000;
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0:
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
+	case UCSI_CONSTAT_PWR_OPMODE_BC:
+	case UCSI_CONSTAT_PWR_OPMODE_DEFAULT:
+		val->intval = UCSI_TYPEC_VSAFE5V * 1000;
+		break;
+	default:
+		val->intval = 0;
+		break;
+	}
+	return 0;
+}
+
+static int ucsi_psy_get_voltage_max(struct ucsi_connector *con,
+				    union power_supply_propval *val)
+{
+	u32 pdo;
+
+	switch (UCSI_CONSTAT_PWR_OPMODE(con->status.flags)) {
+	case UCSI_CONSTAT_PWR_OPMODE_PD:
+		if (con->num_pdos > 0) {
+			pdo = con->src_pdos[con->num_pdos - 1];
+			val->intval = pdo_fixed_voltage(pdo) * 1000;
+		} else {
+			val->intval = 0;
+		}
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0:
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
+	case UCSI_CONSTAT_PWR_OPMODE_BC:
+	case UCSI_CONSTAT_PWR_OPMODE_DEFAULT:
+		val->intval = UCSI_TYPEC_VSAFE5V * 1000;
+		break;
+	default:
+		val->intval = 0;
+		break;
+	}
+	return 0;
+}
+
+static int ucsi_psy_get_voltage_now(struct ucsi_connector *con,
+				    union power_supply_propval *val)
+{
+	int index;
+	u32 pdo;
+
+	switch (UCSI_CONSTAT_PWR_OPMODE(con->status.flags)) {
+	case UCSI_CONSTAT_PWR_OPMODE_PD:
+		index = rdo_index(con->rdo);
+		if (index > 0) {
+			pdo = con->src_pdos[index - 1];
+			val->intval = pdo_fixed_voltage(pdo) * 1000;
+		} else {
+			val->intval = 0;
+		}
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0:
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
+	case UCSI_CONSTAT_PWR_OPMODE_BC:
+	case UCSI_CONSTAT_PWR_OPMODE_DEFAULT:
+		val->intval = UCSI_TYPEC_VSAFE5V * 1000;
+		break;
+	default:
+		val->intval = 0;
+		break;
+	}
+	return 0;
+}
+
+static int ucsi_psy_get_current_max(struct ucsi_connector *con,
+				    union power_supply_propval *val)
+{
+	u32 pdo;
+
+	switch (UCSI_CONSTAT_PWR_OPMODE(con->status.flags)) {
+	case UCSI_CONSTAT_PWR_OPMODE_PD:
+		if (con->num_pdos > 0) {
+			pdo = con->src_pdos[con->num_pdos - 1];
+			val->intval = pdo_max_current(pdo) * 1000;
+		} else {
+			val->intval = 0;
+		}
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
+		val->intval = UCSI_TYPEC_1_5_CURRENT * 1000;
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0:
+		val->intval = UCSI_TYPEC_3_0_CURRENT * 1000;
+		break;
+	case UCSI_CONSTAT_PWR_OPMODE_BC:
+	case UCSI_CONSTAT_PWR_OPMODE_DEFAULT:
+	/* UCSI can't tell b/w DCP/CDP or USB2/3x1/3x2 SDP chargers */
+	default:
+		val->intval = 0;
+		break;
+	}
+	return 0;
+}
+
+static int ucsi_psy_get_current_now(struct ucsi_connector *con,
+				    union power_supply_propval *val)
+{
+	u16 flags = con->status.flags;
+
+	if (UCSI_CONSTAT_PWR_OPMODE(flags) == UCSI_CONSTAT_PWR_OPMODE_PD)
+		val->intval = rdo_op_current(con->rdo) * 1000;
+	else
+		val->intval = 0;
+	return 0;
+}
+
+static int ucsi_psy_get_usb_type(struct ucsi_connector *con,
+				 union power_supply_propval *val)
+{
+	u16 flags = con->status.flags;
+
+	val->intval = POWER_SUPPLY_USB_TYPE_C;
+	if (flags & UCSI_CONSTAT_CONNECTED &&
+	    UCSI_CONSTAT_PWR_OPMODE(flags) == UCSI_CONSTAT_PWR_OPMODE_PD)
+		val->intval = POWER_SUPPLY_USB_TYPE_PD;
+
+	return 0;
+}
+
+static int ucsi_psy_get_prop(struct power_supply *psy,
+			     enum power_supply_property psp,
+			     union power_supply_propval *val)
+{
+	struct ucsi_connector *con = power_supply_get_drvdata(psy);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_USB_TYPE:
+		return ucsi_psy_get_usb_type(con, val);
+	case POWER_SUPPLY_PROP_ONLINE:
+		return ucsi_psy_get_online(con, val);
+	case POWER_SUPPLY_PROP_VOLTAGE_MIN:
+		return ucsi_psy_get_voltage_min(con, val);
+	case POWER_SUPPLY_PROP_VOLTAGE_MAX:
+		return ucsi_psy_get_voltage_max(con, val);
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		return ucsi_psy_get_voltage_now(con, val);
+	case POWER_SUPPLY_PROP_CURRENT_MAX:
+		return ucsi_psy_get_current_max(con, val);
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		return ucsi_psy_get_current_now(con, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static enum power_supply_usb_type ucsi_psy_usb_types[] = {
+	POWER_SUPPLY_USB_TYPE_C,
+	POWER_SUPPLY_USB_TYPE_PD,
+	POWER_SUPPLY_USB_TYPE_PD_PPS,
+};
+
+int ucsi_register_port_psy(struct ucsi_connector *con)
+{
+	struct power_supply_config psy_cfg = {};
+	struct device *dev = con->ucsi->dev;
+	char *psy_name;
+
+	psy_cfg.drv_data = con;
+	psy_cfg.fwnode = dev_fwnode(dev);
+
+	psy_name = devm_kasprintf(dev, GFP_KERNEL, "ucsi-source-psy-%s%d",
+				  dev_name(dev), con->num);
+	if (!psy_name)
+		return -ENOMEM;
+
+	con->psy_desc.name = psy_name;
+	con->psy_desc.type = POWER_SUPPLY_TYPE_USB,
+	con->psy_desc.usb_types = ucsi_psy_usb_types;
+	con->psy_desc.num_usb_types = ARRAY_SIZE(ucsi_psy_usb_types);
+	con->psy_desc.properties = ucsi_psy_props,
+	con->psy_desc.num_properties = ARRAY_SIZE(ucsi_psy_props),
+	con->psy_desc.get_property = ucsi_psy_get_prop;
+
+	con->psy = power_supply_register(dev, &con->psy_desc, &psy_cfg);
+
+	return PTR_ERR_OR_ZERO(con->psy);
+}
+
+void ucsi_unregister_port_psy(struct ucsi_connector *con)
+{
+	if (IS_ERR_OR_NULL(con->psy))
+		return;
+
+	power_supply_unregister(con->psy);
+}
diff --git a/drivers/usb/typec/ucsi/trace.c b/drivers/usb/typec/ucsi/trace.c
index 48ad1dc1b1b2..cb62ad835761 100644
--- a/drivers/usb/typec/ucsi/trace.c
+++ b/drivers/usb/typec/ucsi/trace.c
@@ -35,16 +35,16 @@ const char *ucsi_cmd_str(u64 raw_cmd)
 
 const char *ucsi_cci_str(u32 cci)
 {
-	if (cci & GENMASK(7, 0)) {
-		if (cci & BIT(29))
+	if (UCSI_CCI_CONNECTOR(cci)) {
+		if (cci & UCSI_CCI_ACK_COMPLETE)
 			return "Event pending (ACK completed)";
-		if (cci & BIT(31))
+		if (cci & UCSI_CCI_COMMAND_COMPLETE)
 			return "Event pending (command completed)";
 		return "Connector Change";
 	}
-	if (cci & BIT(29))
+	if (cci & UCSI_CCI_ACK_COMPLETE)
 		return "ACK completed";
-	if (cci & BIT(31))
+	if (cci & UCSI_CCI_COMMAND_COMPLETE)
 		return "Command completed";
 
 	return "";
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index ddf2ad3752de..d0c63afaf345 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -492,19 +492,45 @@ static void ucsi_unregister_altmodes(struct ucsi_connector *con, u8 recipient)
 	}
 }
 
+static void ucsi_get_pdos(struct ucsi_connector *con, int is_partner)
+{
+	struct ucsi *ucsi = con->ucsi;
+	u64 command;
+	int ret;
+
+	command = UCSI_COMMAND(UCSI_GET_PDOS) | UCSI_CONNECTOR_NUMBER(con->num);
+	command |= UCSI_GET_PDOS_PARTNER_PDO(is_partner);
+	command |= UCSI_GET_PDOS_NUM_PDOS(UCSI_MAX_PDOS - 1);
+	command |= UCSI_GET_PDOS_SRC_PDOS;
+	ret = ucsi_run_command(ucsi, command, con->src_pdos,
+			       sizeof(con->src_pdos));
+	if (ret < 0) {
+		dev_err(ucsi->dev, "UCSI_GET_PDOS failed (%d)\n", ret);
+		return;
+	}
+	con->num_pdos = ret / sizeof(u32); /* number of bytes to 32-bit PDOs */
+	if (ret == 0)
+		dev_warn(ucsi->dev, "UCSI_GET_PDOS returned 0 bytes\n");
+}
+
 static void ucsi_pwr_opmode_change(struct ucsi_connector *con)
 {
 	switch (UCSI_CONSTAT_PWR_OPMODE(con->status.flags)) {
 	case UCSI_CONSTAT_PWR_OPMODE_PD:
+		con->rdo = con->status.request_data_obj;
 		typec_set_pwr_opmode(con->port, TYPEC_PWR_MODE_PD);
+		ucsi_get_pdos(con, 1);
 		break;
 	case UCSI_CONSTAT_PWR_OPMODE_TYPEC1_5:
+		con->rdo = 0;
 		typec_set_pwr_opmode(con->port, TYPEC_PWR_MODE_1_5A);
 		break;
 	case UCSI_CONSTAT_PWR_OPMODE_TYPEC3_0:
+		con->rdo = 0;
 		typec_set_pwr_opmode(con->port, TYPEC_PWR_MODE_3_0A);
 		break;
 	default:
+		con->rdo = 0;
 		typec_set_pwr_opmode(con->port, TYPEC_PWR_MODE_USB);
 		break;
 	}
@@ -566,6 +592,8 @@ static void ucsi_partner_change(struct ucsi_connector *con)
 
 	switch (UCSI_CONSTAT_PARTNER_TYPE(con->status.flags)) {
 	case UCSI_CONSTAT_PARTNER_TYPE_UFP:
+	case UCSI_CONSTAT_PARTNER_TYPE_CABLE:
+	case UCSI_CONSTAT_PARTNER_TYPE_CABLE_AND_UFP:
 		typec_set_data_role(con->port, TYPEC_HOST);
 		break;
 	case UCSI_CONSTAT_PARTNER_TYPE_DFP:
@@ -611,7 +639,8 @@ static void ucsi_handle_connector_change(struct work_struct *work)
 
 	role = !!(con->status.flags & UCSI_CONSTAT_PWR_DIR);
 
-	if (con->status.change & UCSI_CONSTAT_POWER_OPMODE_CHANGE)
+	if (con->status.change & UCSI_CONSTAT_POWER_OPMODE_CHANGE ||
+	    con->status.change & UCSI_CONSTAT_POWER_LEVEL_CHANGE)
 		ucsi_pwr_opmode_change(con);
 
 	if (con->status.change & UCSI_CONSTAT_POWER_DIR_CHANGE) {
@@ -627,6 +656,8 @@ static void ucsi_handle_connector_change(struct work_struct *work)
 
 		switch (UCSI_CONSTAT_PARTNER_TYPE(con->status.flags)) {
 		case UCSI_CONSTAT_PARTNER_TYPE_UFP:
+		case UCSI_CONSTAT_PARTNER_TYPE_CABLE:
+		case UCSI_CONSTAT_PARTNER_TYPE_CABLE_AND_UFP:
 			typec_set_data_role(con->port, TYPEC_HOST);
 			break;
 		case UCSI_CONSTAT_PARTNER_TYPE_DFP:
@@ -905,6 +936,10 @@ static int ucsi_register_port(struct ucsi *ucsi, int index)
 	cap->driver_data = con;
 	cap->ops = &ucsi_ops;
 
+	ret = ucsi_register_port_psy(con);
+	if (ret)
+		return ret;
+
 	/* Register the connector */
 	con->port = typec_register_port(ucsi->dev, cap);
 	if (IS_ERR(con->port))
@@ -927,6 +962,8 @@ static int ucsi_register_port(struct ucsi *ucsi, int index)
 
 	switch (UCSI_CONSTAT_PARTNER_TYPE(con->status.flags)) {
 	case UCSI_CONSTAT_PARTNER_TYPE_UFP:
+	case UCSI_CONSTAT_PARTNER_TYPE_CABLE:
+	case UCSI_CONSTAT_PARTNER_TYPE_CABLE_AND_UFP:
 		typec_set_data_role(con->port, TYPEC_HOST);
 		break;
 	case UCSI_CONSTAT_PARTNER_TYPE_DFP:
@@ -1029,6 +1066,7 @@ err_unregister:
 	for (con = ucsi->connector; con->port; con++) {
 		ucsi_unregister_partner(con);
 		ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON);
+		ucsi_unregister_port_psy(con);
 		typec_unregister_port(con->port);
 		con->port = NULL;
 	}
@@ -1152,6 +1190,7 @@ void ucsi_unregister(struct ucsi *ucsi)
 		ucsi_unregister_partner(&ucsi->connector[i]);
 		ucsi_unregister_altmodes(&ucsi->connector[i],
 					 UCSI_RECIPIENT_CON);
+		ucsi_unregister_port_psy(&ucsi->connector[i]);
 		typec_unregister_port(ucsi->connector[i].port);
 	}
 
diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h
index 8e831108f481..cba6f77bea61 100644
--- a/drivers/usb/typec/ucsi/ucsi.h
+++ b/drivers/usb/typec/ucsi/ucsi.h
@@ -5,6 +5,7 @@
 
 #include <linux/bitops.h>
 #include <linux/device.h>
+#include <linux/power_supply.h>
 #include <linux/types.h>
 #include <linux/usb/typec.h>
 
@@ -21,7 +22,7 @@ struct ucsi_altmode;
 #define UCSI_MESSAGE_OUT		32
 
 /* Command Status and Connector Change Indication (CCI) bits */
-#define UCSI_CCI_CONNECTOR(_c_)		(((_c_) & GENMASK(7, 0)) >> 1)
+#define UCSI_CCI_CONNECTOR(_c_)		(((_c_) & GENMASK(7, 1)) >> 1)
 #define UCSI_CCI_LENGTH(_c_)		(((_c_) & GENMASK(15, 8)) >> 8)
 #define UCSI_CCI_NOT_SUPPORTED		BIT(25)
 #define UCSI_CCI_CANCEL_COMPLETE	BIT(26)
@@ -130,6 +131,11 @@ void ucsi_connector_change(struct ucsi *ucsi, u8 num);
 #define UCSI_GET_ALTMODE_OFFSET(_r_)		((u64)(_r_) << 32)
 #define UCSI_GET_ALTMODE_NUM_ALTMODES(_r_)	((u64)(_r_) << 40)
 
+/* GET_PDOS command bits */
+#define UCSI_GET_PDOS_PARTNER_PDO(_r_)		((u64)(_r_) << 23)
+#define UCSI_GET_PDOS_NUM_PDOS(_r_)		((u64)(_r_) << 32)
+#define UCSI_GET_PDOS_SRC_PDOS			((u64)1 << 34)
+
 /* -------------------------------------------------------------------------- */
 
 /* Error information returned by PPM in response to GET_ERROR_STATUS command. */
@@ -294,6 +300,11 @@ struct ucsi {
 
 #define UCSI_MAX_SVID		5
 #define UCSI_MAX_ALTMODES	(UCSI_MAX_SVID * 6)
+#define UCSI_MAX_PDOS		(4)
+
+#define UCSI_TYPEC_VSAFE5V	5000
+#define UCSI_TYPEC_1_5_CURRENT	1500
+#define UCSI_TYPEC_3_0_CURRENT	3000
 
 struct ucsi_connector {
 	int num;
@@ -313,6 +324,11 @@ struct ucsi_connector {
 
 	struct ucsi_connector_status status;
 	struct ucsi_connector_capability cap;
+	struct power_supply *psy;
+	struct power_supply_desc psy_desc;
+	u32 rdo;
+	u32 src_pdos[UCSI_MAX_PDOS];
+	int num_pdos;
 };
 
 int ucsi_send_command(struct ucsi *ucsi, u64 command,
@@ -321,6 +337,14 @@ int ucsi_send_command(struct ucsi *ucsi, u64 command,
 void ucsi_altmode_update_active(struct ucsi_connector *con);
 int ucsi_resume(struct ucsi *ucsi);
 
+#if IS_ENABLED(CONFIG_POWER_SUPPLY)
+int ucsi_register_port_psy(struct ucsi_connector *con);
+void ucsi_unregister_port_psy(struct ucsi_connector *con);
+#else
+static inline int ucsi_register_port_psy(struct ucsi_connector *con) { return 0; }
+static inline void ucsi_unregister_port_psy(struct ucsi_connector *con) { }
+#endif /* CONFIG_POWER_SUPPLY */
+
 #if IS_ENABLED(CONFIG_TYPEC_DP_ALTMODE)
 struct typec_altmode *
 ucsi_register_displayport(struct ucsi_connector *con,
diff --git a/drivers/usb/usbip/Kconfig b/drivers/usb/usbip/Kconfig
index 7bbae7a08642..b9f94e2e278d 100644
--- a/drivers/usb/usbip/Kconfig
+++ b/drivers/usb/usbip/Kconfig
@@ -5,7 +5,7 @@ config USBIP_CORE
 	depends on NET
 	select USB_COMMON
 	select SGL_ALLOC
-	---help---
+	help
 	  This enables pushing USB packets over IP to allow remote
 	  machines direct access to USB devices. It provides the
 	  USB/IP core that is required by both drivers.
@@ -21,7 +21,7 @@ config USBIP_CORE
 config USBIP_VHCI_HCD
 	tristate "VHCI hcd"
 	depends on USBIP_CORE && USB
-	---help---
+	help
 	  This enables the USB/IP virtual host controller driver,
 	  which is run on the remote machine.
 
@@ -33,7 +33,7 @@ config USBIP_VHCI_HC_PORTS
 	range 1 15
 	default 8
 	depends on USBIP_VHCI_HCD
-	---help---
+	help
 	  To increase number of ports available for USB/IP virtual
 	  host controller driver, this defines number of ports per
 	  USB/IP virtual host controller.
@@ -43,7 +43,7 @@ config USBIP_VHCI_NR_HCS
 	range 1 128
 	default 1
 	depends on USBIP_VHCI_HCD
-	---help---
+	help
 	  To increase number of ports available for USB/IP virtual
 	  host controller driver, this defines number of USB/IP
 	  virtual host controllers as if adding physical host
@@ -52,7 +52,7 @@ config USBIP_VHCI_NR_HCS
 config USBIP_HOST
 	tristate "Host driver"
 	depends on USBIP_CORE && USB
-	---help---
+	help
 	  This enables the USB/IP host driver, which is run on the
 	  machine that is sharing the USB devices.
 
@@ -62,7 +62,7 @@ config USBIP_HOST
 config USBIP_VUDC
 	tristate "VUDC driver"
 	depends on USBIP_CORE && USB_GADGET
-	---help---
+	help
 	  This enables the USB/IP virtual USB device controller
 	  driver, which is run on the host machine, allowing the
 	  machine itself to act as a device.
@@ -73,5 +73,5 @@ config USBIP_VUDC
 config USBIP_DEBUG
 	bool "Debug messages for USB/IP"
 	depends on USBIP_CORE
-	---help---
+	help
 	  This enables the debug messages from the USB/IP drivers.
diff --git a/drivers/vdpa/Kconfig b/drivers/vdpa/Kconfig
index e8140065c8a5..3e1ceb8e9f2b 100644
--- a/drivers/vdpa/Kconfig
+++ b/drivers/vdpa/Kconfig
@@ -10,7 +10,7 @@ if VDPA
 
 config VDPA_SIM
 	tristate "vDPA device simulator"
-	depends on RUNTIME_TESTING_MENU && HAS_DMA && VHOST_DPN
+	depends on RUNTIME_TESTING_MENU && HAS_DMA
 	select VHOST_RING
 	default n
 	help
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index e24371d644b5..94bf0328b68d 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -185,6 +185,9 @@ void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
 
 void ifcvf_reset(struct ifcvf_hw *hw)
 {
+	hw->config_cb.callback = NULL;
+	hw->config_cb.private = NULL;
+
 	ifcvf_set_status(hw, 0);
 	/* flush set_status, make sure VF is stopped, reset */
 	ifcvf_get_status(hw);
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index e80307092351..f4554412e607 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -27,6 +27,7 @@
 		((1ULL << VIRTIO_NET_F_MAC)			| \
 		 (1ULL << VIRTIO_F_ANY_LAYOUT)			| \
 		 (1ULL << VIRTIO_F_VERSION_1)			| \
+		 (1ULL << VIRTIO_NET_F_STATUS)			| \
 		 (1ULL << VIRTIO_F_ORDER_PLATFORM)		| \
 		 (1ULL << VIRTIO_F_IOMMU_PLATFORM)		| \
 		 (1ULL << VIRTIO_NET_F_MRG_RXBUF))
@@ -81,6 +82,9 @@ struct ifcvf_hw {
 	void __iomem *net_cfg;
 	struct vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2];
 	void __iomem * const *base;
+	char config_msix_name[256];
+	struct vdpa_callback config_cb;
+
 };
 
 struct ifcvf_adapter {
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index abf6a061cab6..f5a60c14b979 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -18,6 +18,16 @@
 #define DRIVER_AUTHOR   "Intel Corporation"
 #define IFCVF_DRIVER_NAME       "ifcvf"
 
+static irqreturn_t ifcvf_config_changed(int irq, void *arg)
+{
+	struct ifcvf_hw *vf = arg;
+
+	if (vf->config_cb.callback)
+		return vf->config_cb.callback(vf->config_cb.private);
+
+	return IRQ_HANDLED;
+}
+
 static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
 {
 	struct vring_info *vring = arg;
@@ -28,6 +38,68 @@ static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
+static void ifcvf_free_irq_vectors(void *data)
+{
+	pci_free_irq_vectors(data);
+}
+
+static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)
+{
+	struct pci_dev *pdev = adapter->pdev;
+	struct ifcvf_hw *vf = &adapter->vf;
+	int i;
+
+
+	for (i = 0; i < queues; i++)
+		devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
+
+	ifcvf_free_irq_vectors(pdev);
+}
+
+static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
+{
+	struct pci_dev *pdev = adapter->pdev;
+	struct ifcvf_hw *vf = &adapter->vf;
+	int vector, i, ret, irq;
+
+	ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+				    IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+	if (ret < 0) {
+		IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
+		return ret;
+	}
+
+	snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
+		 pci_name(pdev));
+	vector = 0;
+	irq = pci_irq_vector(pdev, vector);
+	ret = devm_request_irq(&pdev->dev, irq,
+			       ifcvf_config_changed, 0,
+			       vf->config_msix_name, vf);
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
+			 pci_name(pdev), i);
+		vector = i + IFCVF_MSI_QUEUE_OFF;
+		irq = pci_irq_vector(pdev, vector);
+		ret = devm_request_irq(&pdev->dev, irq,
+				       ifcvf_intr_handler, 0,
+				       vf->vring[i].msix_name,
+				       &vf->vring[i]);
+		if (ret) {
+			IFCVF_ERR(pdev,
+				  "Failed to request irq for vq %d\n", i);
+			ifcvf_free_irq(adapter, i);
+
+			return ret;
+		}
+
+		vf->vring[i].irq = irq;
+	}
+
+	return 0;
+}
+
 static int ifcvf_start_datapath(void *private)
 {
 	struct ifcvf_hw *vf = ifcvf_private_to_vf(private);
@@ -118,17 +190,37 @@ static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
 {
 	struct ifcvf_adapter *adapter;
 	struct ifcvf_hw *vf;
+	u8 status_old;
+	int ret;
 
 	vf  = vdpa_to_vf(vdpa_dev);
 	adapter = dev_get_drvdata(vdpa_dev->dev.parent);
+	status_old = ifcvf_get_status(vf);
 
-	if (status == 0) {
+	if (status_old == status)
+		return;
+
+	if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) &&
+	    !(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
 		ifcvf_stop_datapath(adapter);
+		ifcvf_free_irq(adapter, IFCVF_MAX_QUEUE_PAIRS * 2);
+	}
+
+	if (status == 0) {
 		ifcvf_reset_vring(adapter);
 		return;
 	}
 
-	if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+	if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
+	    !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
+		ret = ifcvf_request_irq(adapter);
+		if (ret) {
+			status = ifcvf_get_status(vf);
+			status |= VIRTIO_CONFIG_S_FAILED;
+			ifcvf_set_status(vf, status);
+			return;
+		}
+
 		if (ifcvf_start_datapath(adapter) < 0)
 			IFCVF_ERR(adapter->pdev,
 				  "Failed to set ifcvf vdpa  status %u\n",
@@ -254,7 +346,10 @@ static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,
 static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
 				     struct vdpa_callback *cb)
 {
-	/* We don't support config interrupt */
+	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+
+	vf->config_cb.callback = cb->callback;
+	vf->config_cb.private = cb->private;
 }
 
 /*
@@ -284,38 +379,6 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.set_config_cb  = ifcvf_vdpa_set_config_cb,
 };
 
-static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
-{
-	struct pci_dev *pdev = adapter->pdev;
-	struct ifcvf_hw *vf = &adapter->vf;
-	int vector, i, ret, irq;
-
-
-	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
-		snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
-			 pci_name(pdev), i);
-		vector = i + IFCVF_MSI_QUEUE_OFF;
-		irq = pci_irq_vector(pdev, vector);
-		ret = devm_request_irq(&pdev->dev, irq,
-				       ifcvf_intr_handler, 0,
-				       vf->vring[i].msix_name,
-				       &vf->vring[i]);
-		if (ret) {
-			IFCVF_ERR(pdev,
-				  "Failed to request irq for vq %d\n", i);
-			return ret;
-		}
-		vf->vring[i].irq = irq;
-	}
-
-	return 0;
-}
-
-static void ifcvf_free_irq_vectors(void *data)
-{
-	pci_free_irq_vectors(data);
-}
-
 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	struct device *dev = &pdev->dev;
@@ -349,13 +412,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		return ret;
 	}
 
-	ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
-				    IFCVF_MAX_INTR, PCI_IRQ_MSIX);
-	if (ret < 0) {
-		IFCVF_ERR(pdev, "Failed to alloc irq vectors\n");
-		return ret;
-	}
-
 	ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);
 	if (ret) {
 		IFCVF_ERR(pdev,
@@ -379,12 +435,6 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	adapter->pdev = pdev;
 	adapter->vdpa.dma_dev = &pdev->dev;
 
-	ret = ifcvf_request_irq(adapter);
-	if (ret) {
-		IFCVF_ERR(pdev, "Failed to request MSI-X irq\n");
-		goto err;
-	}
-
 	ret = ifcvf_init_hw(vf, pdev);
 	if (ret) {
 		IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 01c456f7c1f7..c7334cc65bb2 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -101,7 +101,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
 
 static void vdpasim_vq_reset(struct vdpasim_virtqueue *vq)
 {
-	vq->ready = 0;
+	vq->ready = false;
 	vq->desc_addr = 0;
 	vq->driver_addr = 0;
 	vq->device_addr = 0;
@@ -131,9 +131,10 @@ static void vdpasim_work(struct work_struct *work)
 						 vdpasim, work);
 	struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];
 	struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];
-	size_t read, write, total_write;
-	int err;
+	ssize_t read, write;
+	size_t total_write;
 	int pkts = 0;
+	int err;
 
 	spin_lock(&vdpasim->lock);
 
diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c
index 8ad14e5c02bf..917fd84c1c6f 100644
--- a/drivers/vfio/mdev/mdev_sysfs.c
+++ b/drivers/vfio/mdev/mdev_sysfs.c
@@ -110,7 +110,7 @@ static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent,
 				   "%s-%s", dev_driver_string(parent->dev),
 				   group->name);
 	if (ret) {
-		kfree(type);
+		kobject_put(&type->kobj);
 		return ERR_PTR(ret);
 	}
 
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 6c6b37b5c04e..7c0779018b1b 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -26,6 +26,7 @@
 #include <linux/vfio.h>
 #include <linux/vgaarb.h>
 #include <linux/nospec.h>
+#include <linux/sched/mm.h>
 
 #include "vfio_pci_private.h"
 
@@ -184,6 +185,7 @@ no_mmap:
 
 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);
 static void vfio_pci_disable(struct vfio_pci_device *vdev);
+static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data);
 
 /*
  * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
@@ -519,6 +521,10 @@ static void vfio_pci_release(void *device_data)
 		vfio_pci_vf_token_user_add(vdev, -1);
 		vfio_spapr_pci_eeh_release(vdev->pdev);
 		vfio_pci_disable(vdev);
+		if (vdev->err_trigger)
+			eventfd_ctx_put(vdev->err_trigger);
+		if (vdev->req_trigger)
+			eventfd_ctx_put(vdev->req_trigger);
 	}
 
 	mutex_unlock(&vdev->reflck->lock);
@@ -736,6 +742,12 @@ int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
 	return 0;
 }
 
+struct vfio_devices {
+	struct vfio_device **devices;
+	int cur_index;
+	int max_index;
+};
+
 static long vfio_pci_ioctl(void *device_data,
 			   unsigned int cmd, unsigned long arg)
 {
@@ -809,7 +821,7 @@ static long vfio_pci_ioctl(void *device_data,
 		{
 			void __iomem *io;
 			size_t size;
-			u16 orig_cmd;
+			u16 cmd;
 
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 			info.flags = 0;
@@ -829,10 +841,7 @@ static long vfio_pci_ioctl(void *device_data,
 			 * Is it really there?  Enable memory decode for
 			 * implicit access in pci_map_rom().
 			 */
-			pci_read_config_word(pdev, PCI_COMMAND, &orig_cmd);
-			pci_write_config_word(pdev, PCI_COMMAND,
-					      orig_cmd | PCI_COMMAND_MEMORY);
-
+			cmd = vfio_pci_memory_lock_and_enable(vdev);
 			io = pci_map_rom(pdev, &size);
 			if (io) {
 				info.flags = VFIO_REGION_INFO_FLAG_READ;
@@ -840,8 +849,8 @@ static long vfio_pci_ioctl(void *device_data,
 			} else {
 				info.size = 0;
 			}
+			vfio_pci_memory_unlock_and_restore(vdev, cmd);
 
-			pci_write_config_word(pdev, PCI_COMMAND, orig_cmd);
 			break;
 		}
 		case VFIO_PCI_VGA_REGION_INDEX:
@@ -984,8 +993,16 @@ static long vfio_pci_ioctl(void *device_data,
 		return ret;
 
 	} else if (cmd == VFIO_DEVICE_RESET) {
-		return vdev->reset_works ?
-			pci_try_reset_function(vdev->pdev) : -EINVAL;
+		int ret;
+
+		if (!vdev->reset_works)
+			return -EINVAL;
+
+		vfio_pci_zap_and_down_write_memory_lock(vdev);
+		ret = pci_try_reset_function(vdev->pdev);
+		up_write(&vdev->memory_lock);
+
+		return ret;
 
 	} else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
 		struct vfio_pci_hot_reset_info hdr;
@@ -1065,8 +1082,9 @@ reset_info_exit:
 		int32_t *group_fds;
 		struct vfio_pci_group_entry *groups;
 		struct vfio_pci_group_info info;
+		struct vfio_devices devs = { .cur_index = 0 };
 		bool slot = false;
-		int i, count = 0, ret = 0;
+		int i, group_idx, mem_idx = 0, count = 0, ret = 0;
 
 		minsz = offsetofend(struct vfio_pci_hot_reset, count);
 
@@ -1118,9 +1136,9 @@ reset_info_exit:
 		 * user interface and store the group and iommu ID.  This
 		 * ensures the group is held across the reset.
 		 */
-		for (i = 0; i < hdr.count; i++) {
+		for (group_idx = 0; group_idx < hdr.count; group_idx++) {
 			struct vfio_group *group;
-			struct fd f = fdget(group_fds[i]);
+			struct fd f = fdget(group_fds[group_idx]);
 			if (!f.file) {
 				ret = -EBADF;
 				break;
@@ -1133,8 +1151,9 @@ reset_info_exit:
 				break;
 			}
 
-			groups[i].group = group;
-			groups[i].id = vfio_external_user_iommu_id(group);
+			groups[group_idx].group = group;
+			groups[group_idx].id =
+					vfio_external_user_iommu_id(group);
 		}
 
 		kfree(group_fds);
@@ -1153,13 +1172,63 @@ reset_info_exit:
 		ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
 						    vfio_pci_validate_devs,
 						    &info, slot);
-		if (!ret)
-			/* User has access, do the reset */
-			ret = pci_reset_bus(vdev->pdev);
+		if (ret)
+			goto hot_reset_release;
+
+		devs.max_index = count;
+		devs.devices = kcalloc(count, sizeof(struct vfio_device *),
+				       GFP_KERNEL);
+		if (!devs.devices) {
+			ret = -ENOMEM;
+			goto hot_reset_release;
+		}
+
+		/*
+		 * We need to get memory_lock for each device, but devices
+		 * can share mmap_lock, therefore we need to zap and hold
+		 * the vma_lock for each device, and only then get each
+		 * memory_lock.
+		 */
+		ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
+					    vfio_pci_try_zap_and_vma_lock_cb,
+					    &devs, slot);
+		if (ret)
+			goto hot_reset_release;
+
+		for (; mem_idx < devs.cur_index; mem_idx++) {
+			struct vfio_pci_device *tmp;
+
+			tmp = vfio_device_data(devs.devices[mem_idx]);
+
+			ret = down_write_trylock(&tmp->memory_lock);
+			if (!ret) {
+				ret = -EBUSY;
+				goto hot_reset_release;
+			}
+			mutex_unlock(&tmp->vma_lock);
+		}
+
+		/* User has access, do the reset */
+		ret = pci_reset_bus(vdev->pdev);
 
 hot_reset_release:
-		for (i--; i >= 0; i--)
-			vfio_group_put_external_user(groups[i].group);
+		for (i = 0; i < devs.cur_index; i++) {
+			struct vfio_device *device;
+			struct vfio_pci_device *tmp;
+
+			device = devs.devices[i];
+			tmp = vfio_device_data(device);
+
+			if (i < mem_idx)
+				up_write(&tmp->memory_lock);
+			else
+				mutex_unlock(&tmp->vma_lock);
+			vfio_device_put(device);
+		}
+		kfree(devs.devices);
+
+		for (group_idx--; group_idx >= 0; group_idx--)
+			vfio_group_put_external_user(groups[group_idx].group);
 
 		kfree(groups);
 		return ret;
@@ -1299,6 +1368,202 @@ static ssize_t vfio_pci_write(void *device_data, const char __user *buf,
 	return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);
 }
 
+/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
+static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try)
+{
+	struct vfio_pci_mmap_vma *mmap_vma, *tmp;
+
+	/*
+	 * Lock ordering:
+	 * vma_lock is nested under mmap_lock for vm_ops callback paths.
+	 * The memory_lock semaphore is used by both code paths calling
+	 * into this function to zap vmas and the vm_ops.fault callback
+	 * to protect the memory enable state of the device.
+	 *
+	 * When zapping vmas we need to maintain the mmap_lock => vma_lock
+	 * ordering, which requires using vma_lock to walk vma_list to
+	 * acquire an mm, then dropping vma_lock to get the mmap_lock and
+	 * reacquiring vma_lock.  This logic is derived from similar
+	 * requirements in uverbs_user_mmap_disassociate().
+	 *
+	 * mmap_lock must always be the top-level lock when it is taken.
+	 * Therefore we can only hold the memory_lock write lock when
+	 * vma_list is empty, as we'd need to take mmap_lock to clear
+	 * entries.  vma_list can only be guaranteed empty when holding
+	 * vma_lock, thus memory_lock is nested under vma_lock.
+	 *
+	 * This enables the vm_ops.fault callback to acquire vma_lock,
+	 * followed by memory_lock read lock, while already holding
+	 * mmap_lock without risk of deadlock.
+	 */
+	while (1) {
+		struct mm_struct *mm = NULL;
+
+		if (try) {
+			if (!mutex_trylock(&vdev->vma_lock))
+				return 0;
+		} else {
+			mutex_lock(&vdev->vma_lock);
+		}
+		while (!list_empty(&vdev->vma_list)) {
+			mmap_vma = list_first_entry(&vdev->vma_list,
+						    struct vfio_pci_mmap_vma,
+						    vma_next);
+			mm = mmap_vma->vma->vm_mm;
+			if (mmget_not_zero(mm))
+				break;
+
+			list_del(&mmap_vma->vma_next);
+			kfree(mmap_vma);
+			mm = NULL;
+		}
+		if (!mm)
+			return 1;
+		mutex_unlock(&vdev->vma_lock);
+
+		if (try) {
+			if (!mmap_read_trylock(mm)) {
+				mmput(mm);
+				return 0;
+			}
+		} else {
+			mmap_read_lock(mm);
+		}
+		if (mmget_still_valid(mm)) {
+			if (try) {
+				if (!mutex_trylock(&vdev->vma_lock)) {
+					mmap_read_unlock(mm);
+					mmput(mm);
+					return 0;
+				}
+			} else {
+				mutex_lock(&vdev->vma_lock);
+			}
+			list_for_each_entry_safe(mmap_vma, tmp,
+						 &vdev->vma_list, vma_next) {
+				struct vm_area_struct *vma = mmap_vma->vma;
+
+				if (vma->vm_mm != mm)
+					continue;
+
+				list_del(&mmap_vma->vma_next);
+				kfree(mmap_vma);
+
+				zap_vma_ptes(vma, vma->vm_start,
+					     vma->vm_end - vma->vm_start);
+			}
+			mutex_unlock(&vdev->vma_lock);
+		}
+		mmap_read_unlock(mm);
+		mmput(mm);
+	}
+}
+
+void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device *vdev)
+{
+	vfio_pci_zap_and_vma_lock(vdev, false);
+	down_write(&vdev->memory_lock);
+	mutex_unlock(&vdev->vma_lock);
+}
+
+u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev)
+{
+	u16 cmd;
+
+	down_write(&vdev->memory_lock);
+	pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
+	if (!(cmd & PCI_COMMAND_MEMORY))
+		pci_write_config_word(vdev->pdev, PCI_COMMAND,
+				      cmd | PCI_COMMAND_MEMORY);
+
+	return cmd;
+}
+
+void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev, u16 cmd)
+{
+	pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
+	up_write(&vdev->memory_lock);
+}
+
+/* Caller holds vma_lock */
+static int __vfio_pci_add_vma(struct vfio_pci_device *vdev,
+			      struct vm_area_struct *vma)
+{
+	struct vfio_pci_mmap_vma *mmap_vma;
+
+	mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL);
+	if (!mmap_vma)
+		return -ENOMEM;
+
+	mmap_vma->vma = vma;
+	list_add(&mmap_vma->vma_next, &vdev->vma_list);
+
+	return 0;
+}
+
+/*
+ * Zap mmaps on open so that we can fault them in on access and therefore
+ * our vma_list only tracks mappings accessed since last zap.
+ */
+static void vfio_pci_mmap_open(struct vm_area_struct *vma)
+{
+	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
+}
+
+static void vfio_pci_mmap_close(struct vm_area_struct *vma)
+{
+	struct vfio_pci_device *vdev = vma->vm_private_data;
+	struct vfio_pci_mmap_vma *mmap_vma;
+
+	mutex_lock(&vdev->vma_lock);
+	list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
+		if (mmap_vma->vma == vma) {
+			list_del(&mmap_vma->vma_next);
+			kfree(mmap_vma);
+			break;
+		}
+	}
+	mutex_unlock(&vdev->vma_lock);
+}
+
+static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct vfio_pci_device *vdev = vma->vm_private_data;
+	vm_fault_t ret = VM_FAULT_NOPAGE;
+
+	mutex_lock(&vdev->vma_lock);
+	down_read(&vdev->memory_lock);
+
+	if (!__vfio_pci_memory_enabled(vdev)) {
+		ret = VM_FAULT_SIGBUS;
+		mutex_unlock(&vdev->vma_lock);
+		goto up_out;
+	}
+
+	if (__vfio_pci_add_vma(vdev, vma)) {
+		ret = VM_FAULT_OOM;
+		mutex_unlock(&vdev->vma_lock);
+		goto up_out;
+	}
+
+	mutex_unlock(&vdev->vma_lock);
+
+	if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+			    vma->vm_end - vma->vm_start, vma->vm_page_prot))
+		ret = VM_FAULT_SIGBUS;
+
+up_out:
+	up_read(&vdev->memory_lock);
+	return ret;
+}
+
+static const struct vm_operations_struct vfio_pci_mmap_ops = {
+	.open = vfio_pci_mmap_open,
+	.close = vfio_pci_mmap_close,
+	.fault = vfio_pci_mmap_fault,
+};
+
 static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
 {
 	struct vfio_pci_device *vdev = device_data;
@@ -1357,8 +1622,14 @@ static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
 
-	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
-			       req_len, vma->vm_page_prot);
+	/*
+	 * See remap_pfn_range(), called from vfio_pci_fault() but we can't
+	 * change vm_flags within the fault handler.  Set them now.
+	 */
+	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
+	vma->vm_ops = &vfio_pci_mmap_ops;
+
+	return 0;
 }
 
 static void vfio_pci_request(void *device_data, unsigned int count)
@@ -1608,6 +1879,9 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	spin_lock_init(&vdev->irqlock);
 	mutex_init(&vdev->ioeventfds_lock);
 	INIT_LIST_HEAD(&vdev->ioeventfds_list);
+	mutex_init(&vdev->vma_lock);
+	INIT_LIST_HEAD(&vdev->vma_list);
+	init_rwsem(&vdev->memory_lock);
 
 	ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
 	if (ret)
@@ -1861,12 +2135,6 @@ static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck)
 	kref_put_mutex(&reflck->kref, vfio_pci_reflck_release, &reflck_lock);
 }
 
-struct vfio_devices {
-	struct vfio_device **devices;
-	int cur_index;
-	int max_index;
-};
-
 static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data)
 {
 	struct vfio_devices *devs = data;
@@ -1897,6 +2165,39 @@ static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data)
 	return 0;
 }
 
+static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data)
+{
+	struct vfio_devices *devs = data;
+	struct vfio_device *device;
+	struct vfio_pci_device *vdev;
+
+	if (devs->cur_index == devs->max_index)
+		return -ENOSPC;
+
+	device = vfio_device_get_from_dev(&pdev->dev);
+	if (!device)
+		return -EINVAL;
+
+	if (pci_dev_driver(pdev) != &vfio_pci_driver) {
+		vfio_device_put(device);
+		return -EBUSY;
+	}
+
+	vdev = vfio_device_data(device);
+
+	/*
+	 * Locking multiple devices is prone to deadlock, runaway and
+	 * unwind if we hit contention.
+	 */
+	if (!vfio_pci_zap_and_vma_lock(vdev, true)) {
+		vfio_device_put(device);
+		return -EBUSY;
+	}
+
+	devs->devices[devs->cur_index++] = device;
+	return 0;
+}
+
 /*
  * If a bus or slot reset is available for the provided device and:
  *  - All of the devices affected by that bus or slot reset are unused
diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c
index 90c0b80f8acf..8746c943247a 100644
--- a/drivers/vfio/pci/vfio_pci_config.c
+++ b/drivers/vfio/pci/vfio_pci_config.c
@@ -395,6 +395,14 @@ static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write)
 	*(__le32 *)(&p->write[off]) = cpu_to_le32(write);
 }
 
+/* Caller should hold memory_lock semaphore */
+bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev)
+{
+	u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]);
+
+	return cmd & PCI_COMMAND_MEMORY;
+}
+
 /*
  * Restore the *real* BARs after we detect a FLR or backdoor reset.
  * (backdoor = some device specific technique that we didn't catch)
@@ -556,13 +564,18 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,
 
 		new_cmd = le32_to_cpu(val);
 
+		phys_io = !!(phys_cmd & PCI_COMMAND_IO);
+		virt_io = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_IO);
+		new_io = !!(new_cmd & PCI_COMMAND_IO);
+
 		phys_mem = !!(phys_cmd & PCI_COMMAND_MEMORY);
 		virt_mem = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_MEMORY);
 		new_mem = !!(new_cmd & PCI_COMMAND_MEMORY);
 
-		phys_io = !!(phys_cmd & PCI_COMMAND_IO);
-		virt_io = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_IO);
-		new_io = !!(new_cmd & PCI_COMMAND_IO);
+		if (!new_mem)
+			vfio_pci_zap_and_down_write_memory_lock(vdev);
+		else
+			down_write(&vdev->memory_lock);
 
 		/*
 		 * If the user is writing mem/io enable (new_mem/io) and we
@@ -579,8 +592,11 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,
 	}
 
 	count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
-	if (count < 0)
+	if (count < 0) {
+		if (offset == PCI_COMMAND)
+			up_write(&vdev->memory_lock);
 		return count;
+	}
 
 	/*
 	 * Save current memory/io enable bits in vconfig to allow for
@@ -591,6 +607,8 @@ static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,
 
 		*virt_cmd &= cpu_to_le16(~mask);
 		*virt_cmd |= cpu_to_le16(new_cmd & mask);
+
+		up_write(&vdev->memory_lock);
 	}
 
 	/* Emulate INTx disable */
@@ -828,8 +846,11 @@ static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos,
 						 pos - offset + PCI_EXP_DEVCAP,
 						 &cap);
 
-		if (!ret && (cap & PCI_EXP_DEVCAP_FLR))
+		if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) {
+			vfio_pci_zap_and_down_write_memory_lock(vdev);
 			pci_try_reset_function(vdev->pdev);
+			up_write(&vdev->memory_lock);
+		}
 	}
 
 	/*
@@ -907,8 +928,11 @@ static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos,
 						pos - offset + PCI_AF_CAP,
 						&cap);
 
-		if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP))
+		if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) {
+			vfio_pci_zap_and_down_write_memory_lock(vdev);
 			pci_try_reset_function(vdev->pdev);
+			up_write(&vdev->memory_lock);
+		}
 	}
 
 	return count;
@@ -1462,7 +1486,12 @@ static int vfio_cap_init(struct vfio_pci_device *vdev)
 		if (ret)
 			return ret;
 
-		if (cap <= PCI_CAP_ID_MAX) {
+		/*
+		 * ID 0 is a NULL capability, conflicting with our fake
+		 * PCI_CAP_ID_BASIC.  As it has no content, consider it
+		 * hidden for now.
+		 */
+		if (cap && cap <= PCI_CAP_ID_MAX) {
 			len = pci_cap_length[cap];
 			if (len == 0xFF) { /* Variable length */
 				len = vfio_cap_len(vdev, cap, pos);
@@ -1728,8 +1757,11 @@ void vfio_config_free(struct vfio_pci_device *vdev)
 	vdev->vconfig = NULL;
 	kfree(vdev->pci_config_map);
 	vdev->pci_config_map = NULL;
-	kfree(vdev->msi_perm);
-	vdev->msi_perm = NULL;
+	if (vdev->msi_perm) {
+		free_perm_bits(vdev->msi_perm);
+		kfree(vdev->msi_perm);
+		vdev->msi_perm = NULL;
+	}
 }
 
 /*
diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
index 2056f3f85f59..1d9fb2592945 100644
--- a/drivers/vfio/pci/vfio_pci_intrs.c
+++ b/drivers/vfio/pci/vfio_pci_intrs.c
@@ -249,6 +249,7 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
 	struct pci_dev *pdev = vdev->pdev;
 	unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
 	int ret;
+	u16 cmd;
 
 	if (!is_irq_none(vdev))
 		return -EINVAL;
@@ -258,13 +259,16 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
 		return -ENOMEM;
 
 	/* return the number of supported vectors if we can't get all: */
+	cmd = vfio_pci_memory_lock_and_enable(vdev);
 	ret = pci_alloc_irq_vectors(pdev, 1, nvec, flag);
 	if (ret < nvec) {
 		if (ret > 0)
 			pci_free_irq_vectors(pdev);
+		vfio_pci_memory_unlock_and_restore(vdev, cmd);
 		kfree(vdev->ctx);
 		return ret;
 	}
+	vfio_pci_memory_unlock_and_restore(vdev, cmd);
 
 	vdev->num_ctx = nvec;
 	vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX :
@@ -287,6 +291,7 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 	struct pci_dev *pdev = vdev->pdev;
 	struct eventfd_ctx *trigger;
 	int irq, ret;
+	u16 cmd;
 
 	if (vector < 0 || vector >= vdev->num_ctx)
 		return -EINVAL;
@@ -295,7 +300,11 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 
 	if (vdev->ctx[vector].trigger) {
 		irq_bypass_unregister_producer(&vdev->ctx[vector].producer);
+
+		cmd = vfio_pci_memory_lock_and_enable(vdev);
 		free_irq(irq, vdev->ctx[vector].trigger);
+		vfio_pci_memory_unlock_and_restore(vdev, cmd);
+
 		kfree(vdev->ctx[vector].name);
 		eventfd_ctx_put(vdev->ctx[vector].trigger);
 		vdev->ctx[vector].trigger = NULL;
@@ -323,6 +332,7 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 	 * such a reset it would be unsuccessful. To avoid this, restore the
 	 * cached value of the message prior to enabling.
 	 */
+	cmd = vfio_pci_memory_lock_and_enable(vdev);
 	if (msix) {
 		struct msi_msg msg;
 
@@ -332,6 +342,7 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
 
 	ret = request_irq(irq, vfio_msihandler, 0,
 			  vdev->ctx[vector].name, trigger);
+	vfio_pci_memory_unlock_and_restore(vdev, cmd);
 	if (ret) {
 		kfree(vdev->ctx[vector].name);
 		eventfd_ctx_put(trigger);
@@ -376,6 +387,7 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
 {
 	struct pci_dev *pdev = vdev->pdev;
 	int i;
+	u16 cmd;
 
 	for (i = 0; i < vdev->num_ctx; i++) {
 		vfio_virqfd_disable(&vdev->ctx[i].unmask);
@@ -384,7 +396,9 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
 
 	vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
 
+	cmd = vfio_pci_memory_lock_and_enable(vdev);
 	pci_free_irq_vectors(pdev);
+	vfio_pci_memory_unlock_and_restore(vdev, cmd);
 
 	/*
 	 * Both disable paths above use pci_intx_for_msi() to clear DisINTx
diff --git a/drivers/vfio/pci/vfio_pci_nvlink2.c b/drivers/vfio/pci/vfio_pci_nvlink2.c
index ed20d73cc27c..65c61710c0e9 100644
--- a/drivers/vfio/pci/vfio_pci_nvlink2.c
+++ b/drivers/vfio/pci/vfio_pci_nvlink2.c
@@ -67,7 +67,7 @@ static size_t vfio_pci_nvgpu_rw(struct vfio_pci_device *vdev,
 	 *
 	 * This is not fast path anyway.
 	 */
-	sizealigned = _ALIGN_UP(posoff + count, PAGE_SIZE);
+	sizealigned = ALIGN(posoff + count, PAGE_SIZE);
 	ptr = ioremap_cache(data->gpu_hpa + posaligned, sizealigned);
 	if (!ptr)
 		return -EFAULT;
diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h
index 36ec69081ecd..86a02aff8735 100644
--- a/drivers/vfio/pci/vfio_pci_private.h
+++ b/drivers/vfio/pci/vfio_pci_private.h
@@ -92,6 +92,11 @@ struct vfio_pci_vf_token {
 	int			users;
 };
 
+struct vfio_pci_mmap_vma {
+	struct vm_area_struct	*vma;
+	struct list_head	vma_next;
+};
+
 struct vfio_pci_device {
 	struct pci_dev		*pdev;
 	void __iomem		*barmap[PCI_STD_NUM_BARS];
@@ -132,6 +137,9 @@ struct vfio_pci_device {
 	struct list_head	ioeventfds_list;
 	struct vfio_pci_vf_token	*vf_token;
 	struct notifier_block	nb;
+	struct mutex		vma_lock;
+	struct list_head	vma_list;
+	struct rw_semaphore	memory_lock;
 };
 
 #define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
@@ -174,6 +182,13 @@ extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
 extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev,
 				    pci_power_t state);
 
+extern bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev);
+extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device
+						    *vdev);
+extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev);
+extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev,
+					       u16 cmd);
+
 #ifdef CONFIG_VFIO_PCI_IGD
 extern int vfio_pci_igd_init(struct vfio_pci_device *vdev);
 #else
diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c
index a87992892a9f..916b184df3a5 100644
--- a/drivers/vfio/pci/vfio_pci_rdwr.c
+++ b/drivers/vfio/pci/vfio_pci_rdwr.c
@@ -162,6 +162,7 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
 	size_t x_start = 0, x_end = 0;
 	resource_size_t end;
 	void __iomem *io;
+	struct resource *res = &vdev->pdev->resource[bar];
 	ssize_t done;
 
 	if (pci_resource_start(pdev, bar))
@@ -177,6 +178,14 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
 
 	count = min(count, (size_t)(end - pos));
 
+	if (res->flags & IORESOURCE_MEM) {
+		down_read(&vdev->memory_lock);
+		if (!__vfio_pci_memory_enabled(vdev)) {
+			up_read(&vdev->memory_lock);
+			return -EIO;
+		}
+	}
+
 	if (bar == PCI_ROM_RESOURCE) {
 		/*
 		 * The ROM can fill less space than the BAR, so we start the
@@ -184,13 +193,17 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
 		 * filling large ROM BARs much faster.
 		 */
 		io = pci_map_rom(pdev, &x_start);
-		if (!io)
-			return -ENOMEM;
+		if (!io) {
+			done = -ENOMEM;
+			goto out;
+		}
 		x_end = end;
 	} else {
 		int ret = vfio_pci_setup_barmap(vdev, bar);
-		if (ret)
-			return ret;
+		if (ret) {
+			done = ret;
+			goto out;
+		}
 
 		io = vdev->barmap[bar];
 	}
@@ -207,6 +220,9 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
 
 	if (bar == PCI_ROM_RESOURCE)
 		pci_unmap_rom(pdev, io);
+out:
+	if (res->flags & IORESOURCE_MEM)
+		up_read(&vdev->memory_lock);
 
 	return done;
 }
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 765e0e5d83ed..580099afeaff 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -85,6 +85,7 @@ struct vfio_group {
 	atomic_t			opened;
 	wait_queue_head_t		container_q;
 	bool				noiommu;
+	unsigned int			dev_counter;
 	struct kvm			*kvm;
 	struct blocking_notifier_head	notifier;
 };
@@ -555,6 +556,7 @@ struct vfio_device *vfio_group_create_device(struct vfio_group *group,
 
 	mutex_lock(&group->device_lock);
 	list_add(&device->group_next, &group->device_list);
+	group->dev_counter++;
 	mutex_unlock(&group->device_lock);
 
 	return device;
@@ -567,6 +569,7 @@ static void vfio_device_release(struct kref *kref)
 	struct vfio_group *group = device->group;
 
 	list_del(&device->group_next);
+	group->dev_counter--;
 	mutex_unlock(&group->device_lock);
 
 	dev_set_drvdata(device->dev, NULL);
@@ -1945,6 +1948,9 @@ int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage,
 	if (!group)
 		return -ENODEV;
 
+	if (group->dev_counter > 1)
+		return -EINVAL;
+
 	ret = vfio_group_add_container_user(group);
 	if (ret)
 		goto err_pin_pages;
@@ -1952,7 +1958,8 @@ int vfio_pin_pages(struct device *dev, unsigned long *user_pfn, int npage,
 	container = group->container;
 	driver = container->iommu_driver;
 	if (likely(driver && driver->ops->pin_pages))
-		ret = driver->ops->pin_pages(container->iommu_data, user_pfn,
+		ret = driver->ops->pin_pages(container->iommu_data,
+					     group->iommu_group, user_pfn,
 					     npage, prot, phys_pfn);
 	else
 		ret = -ENOTTY;
@@ -2050,8 +2057,8 @@ int vfio_group_pin_pages(struct vfio_group *group,
 	driver = container->iommu_driver;
 	if (likely(driver && driver->ops->pin_pages))
 		ret = driver->ops->pin_pages(container->iommu_data,
-					     user_iova_pfn, npage,
-					     prot, phys_pfn);
+					     group->iommu_group, user_iova_pfn,
+					     npage, prot, phys_pfn);
 	else
 		ret = -ENOTTY;
 
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index cc1d64765ce7..5e556ac9102a 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -27,7 +27,7 @@
 #include <linux/iommu.h>
 #include <linux/module.h>
 #include <linux/mm.h>
-#include <linux/mmu_context.h>
+#include <linux/kthread.h>
 #include <linux/rbtree.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/mm.h>
@@ -69,8 +69,11 @@ struct vfio_iommu {
 	struct rb_root		dma_list;
 	struct blocking_notifier_head notifier;
 	unsigned int		dma_avail;
+	uint64_t		pgsize_bitmap;
 	bool			v2;
 	bool			nesting;
+	bool			dirty_page_tracking;
+	bool			pinned_page_dirty_scope;
 };
 
 struct vfio_domain {
@@ -91,12 +94,14 @@ struct vfio_dma {
 	bool			lock_cap;	/* capable(CAP_IPC_LOCK) */
 	struct task_struct	*task;
 	struct rb_root		pfn_list;	/* Ex-user pinned pfn list */
+	unsigned long		*bitmap;
 };
 
 struct vfio_group {
 	struct iommu_group	*iommu_group;
 	struct list_head	next;
 	bool			mdev_group;	/* An mdev group */
+	bool			pinned_page_dirty_scope;
 };
 
 struct vfio_iova {
@@ -112,7 +117,7 @@ struct vfio_pfn {
 	struct rb_node		node;
 	dma_addr_t		iova;		/* Device address */
 	unsigned long		pfn;		/* Host pfn */
-	atomic_t		ref_count;
+	unsigned int		ref_count;
 };
 
 struct vfio_regions {
@@ -125,8 +130,25 @@ struct vfio_regions {
 #define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)	\
 					(!list_empty(&iommu->domain_list))
 
+#define DIRTY_BITMAP_BYTES(n)	(ALIGN(n, BITS_PER_TYPE(u64)) / BITS_PER_BYTE)
+
+/*
+ * Input argument of number of bits to bitmap_set() is unsigned integer, which
+ * further casts to signed integer for unaligned multi-bit operation,
+ * __bitmap_set().
+ * Then maximum bitmap size supported is 2^31 bits divided by 2^3 bits/byte,
+ * that is 2^28 (256 MB) which maps to 2^31 * 2^12 = 2^43 (8TB) on 4K page
+ * system.
+ */
+#define DIRTY_BITMAP_PAGES_MAX	 ((u64)INT_MAX)
+#define DIRTY_BITMAP_SIZE_MAX	 DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX)
+
 static int put_pfn(unsigned long pfn, int prot);
 
+static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
+					       struct iommu_group *iommu_group);
+
+static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu);
 /*
  * This code handles mapping and unmapping of user data buffers
  * into DMA'ble space using the IOMMU
@@ -175,6 +197,81 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old)
 	rb_erase(&old->node, &iommu->dma_list);
 }
 
+
+static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize)
+{
+	uint64_t npages = dma->size / pgsize;
+
+	if (npages > DIRTY_BITMAP_PAGES_MAX)
+		return -EINVAL;
+
+	/*
+	 * Allocate extra 64 bits that are used to calculate shift required for
+	 * bitmap_shift_left() to manipulate and club unaligned number of pages
+	 * in adjacent vfio_dma ranges.
+	 */
+	dma->bitmap = kvzalloc(DIRTY_BITMAP_BYTES(npages) + sizeof(u64),
+			       GFP_KERNEL);
+	if (!dma->bitmap)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void vfio_dma_bitmap_free(struct vfio_dma *dma)
+{
+	kfree(dma->bitmap);
+	dma->bitmap = NULL;
+}
+
+static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize)
+{
+	struct rb_node *p;
+	unsigned long pgshift = __ffs(pgsize);
+
+	for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) {
+		struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node);
+
+		bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1);
+	}
+}
+
+static int vfio_dma_bitmap_alloc_all(struct vfio_iommu *iommu, size_t pgsize)
+{
+	struct rb_node *n;
+
+	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
+		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
+		int ret;
+
+		ret = vfio_dma_bitmap_alloc(dma, pgsize);
+		if (ret) {
+			struct rb_node *p;
+
+			for (p = rb_prev(n); p; p = rb_prev(p)) {
+				struct vfio_dma *dma = rb_entry(n,
+							struct vfio_dma, node);
+
+				vfio_dma_bitmap_free(dma);
+			}
+			return ret;
+		}
+		vfio_dma_populate_bitmap(dma, pgsize);
+	}
+	return 0;
+}
+
+static void vfio_dma_bitmap_free_all(struct vfio_iommu *iommu)
+{
+	struct rb_node *n;
+
+	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
+		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
+
+		vfio_dma_bitmap_free(dma);
+	}
+}
+
 /*
  * Helper Functions for host iova-pfn list
  */
@@ -233,7 +330,7 @@ static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova,
 
 	vpfn->iova = iova;
 	vpfn->pfn = pfn;
-	atomic_set(&vpfn->ref_count, 1);
+	vpfn->ref_count = 1;
 	vfio_link_pfn(dma, vpfn);
 	return 0;
 }
@@ -251,7 +348,7 @@ static struct vfio_pfn *vfio_iova_get_vfio_pfn(struct vfio_dma *dma,
 	struct vfio_pfn *vpfn = vfio_find_vpfn(dma, iova);
 
 	if (vpfn)
-		atomic_inc(&vpfn->ref_count);
+		vpfn->ref_count++;
 	return vpfn;
 }
 
@@ -259,7 +356,8 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn)
 {
 	int ret = 0;
 
-	if (atomic_dec_and_test(&vpfn->ref_count)) {
+	vpfn->ref_count--;
+	if (!vpfn->ref_count) {
 		ret = put_pfn(vpfn->pfn, dma->prot);
 		vfio_remove_from_pfn_list(dma, vpfn);
 	}
@@ -278,11 +376,11 @@ static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async)
 	if (!mm)
 		return -ESRCH; /* process exited */
 
-	ret = down_write_killable(&mm->mmap_sem);
+	ret = mmap_write_lock_killable(mm);
 	if (!ret) {
 		ret = __account_locked_vm(mm, abs(npage), npage > 0, dma->task,
 					  dma->lock_cap);
-		up_write(&mm->mmap_sem);
+		mmap_write_unlock(mm);
 	}
 
 	if (async)
@@ -317,6 +415,32 @@ static int put_pfn(unsigned long pfn, int prot)
 	return 0;
 }
 
+static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
+			    unsigned long vaddr, unsigned long *pfn,
+			    bool write_fault)
+{
+	int ret;
+
+	ret = follow_pfn(vma, vaddr, pfn);
+	if (ret) {
+		bool unlocked = false;
+
+		ret = fixup_user_fault(NULL, mm, vaddr,
+				       FAULT_FLAG_REMOTE |
+				       (write_fault ?  FAULT_FLAG_WRITE : 0),
+				       &unlocked);
+		if (unlocked)
+			return -EAGAIN;
+
+		if (ret)
+			return ret;
+
+		ret = follow_pfn(vma, vaddr, pfn);
+	}
+
+	return ret;
+}
+
 static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
 			 int prot, unsigned long *pfn)
 {
@@ -328,7 +452,7 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
 	if (prot & IOMMU_WRITE)
 		flags |= FOLL_WRITE;
 
-	down_read(&mm->mmap_sem);
+	mmap_read_lock(mm);
 	ret = pin_user_pages_remote(NULL, mm, vaddr, 1, flags | FOLL_LONGTERM,
 				    page, NULL, NULL);
 	if (ret == 1) {
@@ -339,15 +463,19 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
 
 	vaddr = untagged_addr(vaddr);
 
+retry:
 	vma = find_vma_intersection(mm, vaddr, vaddr + 1);
 
 	if (vma && vma->vm_flags & VM_PFNMAP) {
-		if (!follow_pfn(vma, vaddr, pfn) &&
-		    is_invalid_reserved_pfn(*pfn))
-			ret = 0;
+		ret = follow_fault_pfn(vma, mm, vaddr, pfn, prot & IOMMU_WRITE);
+		if (ret == -EAGAIN)
+			goto retry;
+
+		if (!ret && !is_invalid_reserved_pfn(*pfn))
+			ret = -EFAULT;
 	}
 done:
-	up_read(&mm->mmap_sem);
+	mmap_read_unlock(mm);
 	return ret;
 }
 
@@ -501,11 +629,13 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova,
 }
 
 static int vfio_iommu_type1_pin_pages(void *iommu_data,
+				      struct iommu_group *iommu_group,
 				      unsigned long *user_pfn,
 				      int npage, int prot,
 				      unsigned long *phys_pfn)
 {
 	struct vfio_iommu *iommu = iommu_data;
+	struct vfio_group *group;
 	int i, j, ret;
 	unsigned long remote_vaddr;
 	struct vfio_dma *dma;
@@ -566,9 +696,26 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data,
 			vfio_unpin_page_external(dma, iova, do_accounting);
 			goto pin_unwind;
 		}
-	}
 
+		if (iommu->dirty_page_tracking) {
+			unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
+
+			/*
+			 * Bitmap populated with the smallest supported page
+			 * size
+			 */
+			bitmap_set(dma->bitmap,
+				   (iova - dma->iova) >> pgshift, 1);
+		}
+	}
 	ret = i;
+
+	group = vfio_iommu_find_iommu_group(iommu, iommu_group);
+	if (!group->pinned_page_dirty_scope) {
+		group->pinned_page_dirty_scope = true;
+		update_pinned_page_dirty_scope(iommu);
+	}
+
 	goto pin_done;
 
 pin_unwind:
@@ -800,19 +947,19 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma)
 	vfio_unmap_unpin(iommu, dma, true);
 	vfio_unlink_dma(iommu, dma);
 	put_task_struct(dma->task);
+	vfio_dma_bitmap_free(dma);
 	kfree(dma);
 	iommu->dma_avail++;
 }
 
-static unsigned long vfio_pgsize_bitmap(struct vfio_iommu *iommu)
+static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu)
 {
 	struct vfio_domain *domain;
-	unsigned long bitmap = ULONG_MAX;
 
-	mutex_lock(&iommu->lock);
+	iommu->pgsize_bitmap = ULONG_MAX;
+
 	list_for_each_entry(domain, &iommu->domain_list, next)
-		bitmap &= domain->domain->pgsize_bitmap;
-	mutex_unlock(&iommu->lock);
+		iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap;
 
 	/*
 	 * In case the IOMMU supports page sizes smaller than PAGE_SIZE
@@ -822,36 +969,143 @@ static unsigned long vfio_pgsize_bitmap(struct vfio_iommu *iommu)
 	 * granularity while iommu driver can use the sub-PAGE_SIZE size
 	 * to map the buffer.
 	 */
-	if (bitmap & ~PAGE_MASK) {
-		bitmap &= PAGE_MASK;
-		bitmap |= PAGE_SIZE;
+	if (iommu->pgsize_bitmap & ~PAGE_MASK) {
+		iommu->pgsize_bitmap &= PAGE_MASK;
+		iommu->pgsize_bitmap |= PAGE_SIZE;
 	}
+}
+
+static int update_user_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
+			      struct vfio_dma *dma, dma_addr_t base_iova,
+			      size_t pgsize)
+{
+	unsigned long pgshift = __ffs(pgsize);
+	unsigned long nbits = dma->size >> pgshift;
+	unsigned long bit_offset = (dma->iova - base_iova) >> pgshift;
+	unsigned long copy_offset = bit_offset / BITS_PER_LONG;
+	unsigned long shift = bit_offset % BITS_PER_LONG;
+	unsigned long leftover;
+
+	/*
+	 * mark all pages dirty if any IOMMU capable device is not able
+	 * to report dirty pages and all pages are pinned and mapped.
+	 */
+	if (!iommu->pinned_page_dirty_scope && dma->iommu_mapped)
+		bitmap_set(dma->bitmap, 0, nbits);
 
-	return bitmap;
+	if (shift) {
+		bitmap_shift_left(dma->bitmap, dma->bitmap, shift,
+				  nbits + shift);
+
+		if (copy_from_user(&leftover,
+				   (void __user *)(bitmap + copy_offset),
+				   sizeof(leftover)))
+			return -EFAULT;
+
+		bitmap_or(dma->bitmap, dma->bitmap, &leftover, shift);
+	}
+
+	if (copy_to_user((void __user *)(bitmap + copy_offset), dma->bitmap,
+			 DIRTY_BITMAP_BYTES(nbits + shift)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu,
+				  dma_addr_t iova, size_t size, size_t pgsize)
+{
+	struct vfio_dma *dma;
+	struct rb_node *n;
+	unsigned long pgshift = __ffs(pgsize);
+	int ret;
+
+	/*
+	 * GET_BITMAP request must fully cover vfio_dma mappings.  Multiple
+	 * vfio_dma mappings may be clubbed by specifying large ranges, but
+	 * there must not be any previous mappings bisected by the range.
+	 * An error will be returned if these conditions are not met.
+	 */
+	dma = vfio_find_dma(iommu, iova, 1);
+	if (dma && dma->iova != iova)
+		return -EINVAL;
+
+	dma = vfio_find_dma(iommu, iova + size - 1, 0);
+	if (dma && dma->iova + dma->size != iova + size)
+		return -EINVAL;
+
+	for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) {
+		struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node);
+
+		if (dma->iova < iova)
+			continue;
+
+		if (dma->iova > iova + size - 1)
+			break;
+
+		ret = update_user_bitmap(bitmap, iommu, dma, iova, pgsize);
+		if (ret)
+			return ret;
+
+		/*
+		 * Re-populate bitmap to include all pinned pages which are
+		 * considered as dirty but exclude pages which are unpinned and
+		 * pages which are marked dirty by vfio_dma_rw()
+		 */
+		bitmap_clear(dma->bitmap, 0, dma->size >> pgshift);
+		vfio_dma_populate_bitmap(dma, pgsize);
+	}
+	return 0;
+}
+
+static int verify_bitmap_size(uint64_t npages, uint64_t bitmap_size)
+{
+	if (!npages || !bitmap_size || (bitmap_size > DIRTY_BITMAP_SIZE_MAX) ||
+	    (bitmap_size < DIRTY_BITMAP_BYTES(npages)))
+		return -EINVAL;
+
+	return 0;
 }
 
 static int vfio_dma_do_unmap(struct vfio_iommu *iommu,
-			     struct vfio_iommu_type1_dma_unmap *unmap)
+			     struct vfio_iommu_type1_dma_unmap *unmap,
+			     struct vfio_bitmap *bitmap)
 {
-	uint64_t mask;
 	struct vfio_dma *dma, *dma_last = NULL;
-	size_t unmapped = 0;
+	size_t unmapped = 0, pgsize;
 	int ret = 0, retries = 0;
+	unsigned long pgshift;
+
+	mutex_lock(&iommu->lock);
 
-	mask = ((uint64_t)1 << __ffs(vfio_pgsize_bitmap(iommu))) - 1;
+	pgshift = __ffs(iommu->pgsize_bitmap);
+	pgsize = (size_t)1 << pgshift;
+
+	if (unmap->iova & (pgsize - 1)) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	if (!unmap->size || unmap->size & (pgsize - 1)) {
+		ret = -EINVAL;
+		goto unlock;
+	}
 
-	if (unmap->iova & mask)
-		return -EINVAL;
-	if (!unmap->size || unmap->size & mask)
-		return -EINVAL;
 	if (unmap->iova + unmap->size - 1 < unmap->iova ||
-	    unmap->size > SIZE_MAX)
-		return -EINVAL;
+	    unmap->size > SIZE_MAX) {
+		ret = -EINVAL;
+		goto unlock;
+	}
 
-	WARN_ON(mask & PAGE_MASK);
-again:
-	mutex_lock(&iommu->lock);
+	/* When dirty tracking is enabled, allow only min supported pgsize */
+	if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) &&
+	    (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) {
+		ret = -EINVAL;
+		goto unlock;
+	}
 
+	WARN_ON((pgsize - 1) & PAGE_MASK);
+again:
 	/*
 	 * vfio-iommu-type1 (v1) - User mappings were coalesced together to
 	 * avoid tracking individual mappings.  This means that the granularity
@@ -929,8 +1183,17 @@ again:
 			blocking_notifier_call_chain(&iommu->notifier,
 						    VFIO_IOMMU_NOTIFY_DMA_UNMAP,
 						    &nb_unmap);
+			mutex_lock(&iommu->lock);
 			goto again;
 		}
+
+		if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
+			ret = update_user_bitmap(bitmap->data, iommu, dma,
+						 unmap->iova, pgsize);
+			if (ret)
+				break;
+		}
+
 		unmapped += dma->size;
 		vfio_remove_dma(iommu, dma);
 	}
@@ -1037,31 +1300,35 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
 	unsigned long vaddr = map->vaddr;
 	size_t size = map->size;
 	int ret = 0, prot = 0;
-	uint64_t mask;
+	size_t pgsize;
 	struct vfio_dma *dma;
 
 	/* Verify that none of our __u64 fields overflow */
 	if (map->size != size || map->vaddr != vaddr || map->iova != iova)
 		return -EINVAL;
 
-	mask = ((uint64_t)1 << __ffs(vfio_pgsize_bitmap(iommu))) - 1;
-
-	WARN_ON(mask & PAGE_MASK);
-
 	/* READ/WRITE from device perspective */
 	if (map->flags & VFIO_DMA_MAP_FLAG_WRITE)
 		prot |= IOMMU_WRITE;
 	if (map->flags & VFIO_DMA_MAP_FLAG_READ)
 		prot |= IOMMU_READ;
 
-	if (!prot || !size || (size | iova | vaddr) & mask)
-		return -EINVAL;
+	mutex_lock(&iommu->lock);
 
-	/* Don't allow IOVA or virtual address wrap */
-	if (iova + size - 1 < iova || vaddr + size - 1 < vaddr)
-		return -EINVAL;
+	pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);
 
-	mutex_lock(&iommu->lock);
+	WARN_ON((pgsize - 1) & PAGE_MASK);
+
+	if (!prot || !size || (size | iova | vaddr) & (pgsize - 1)) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	/* Don't allow IOVA or virtual address wrap */
+	if (iova + size - 1 < iova || vaddr + size - 1 < vaddr) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
 
 	if (vfio_find_dma(iommu, iova, size)) {
 		ret = -EEXIST;
@@ -1129,6 +1396,12 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu,
 	else
 		ret = vfio_pin_map_dma(iommu, dma, size);
 
+	if (!ret && iommu->dirty_page_tracking) {
+		ret = vfio_dma_bitmap_alloc(dma, pgsize);
+		if (ret)
+			vfio_remove_dma(iommu, dma);
+	}
+
 out_unlock:
 	mutex_unlock(&iommu->lock);
 	return ret;
@@ -1267,6 +1540,51 @@ static struct vfio_group *find_iommu_group(struct vfio_domain *domain,
 	return NULL;
 }
 
+static struct vfio_group *vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
+					       struct iommu_group *iommu_group)
+{
+	struct vfio_domain *domain;
+	struct vfio_group *group = NULL;
+
+	list_for_each_entry(domain, &iommu->domain_list, next) {
+		group = find_iommu_group(domain, iommu_group);
+		if (group)
+			return group;
+	}
+
+	if (iommu->external_domain)
+		group = find_iommu_group(iommu->external_domain, iommu_group);
+
+	return group;
+}
+
+static void update_pinned_page_dirty_scope(struct vfio_iommu *iommu)
+{
+	struct vfio_domain *domain;
+	struct vfio_group *group;
+
+	list_for_each_entry(domain, &iommu->domain_list, next) {
+		list_for_each_entry(group, &domain->group_list, next) {
+			if (!group->pinned_page_dirty_scope) {
+				iommu->pinned_page_dirty_scope = false;
+				return;
+			}
+		}
+	}
+
+	if (iommu->external_domain) {
+		domain = iommu->external_domain;
+		list_for_each_entry(group, &domain->group_list, next) {
+			if (!group->pinned_page_dirty_scope) {
+				iommu->pinned_page_dirty_scope = false;
+				return;
+			}
+		}
+	}
+
+	iommu->pinned_page_dirty_scope = true;
+}
+
 static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,
 				  phys_addr_t *base)
 {
@@ -1667,12 +1985,23 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
 			if (!iommu->external_domain) {
 				INIT_LIST_HEAD(&domain->group_list);
 				iommu->external_domain = domain;
+				vfio_update_pgsize_bitmap(iommu);
 			} else {
 				kfree(domain);
 			}
 
 			list_add(&group->next,
 				 &iommu->external_domain->group_list);
+			/*
+			 * Non-iommu backed group cannot dirty memory directly,
+			 * it can only use interfaces that provide dirty
+			 * tracking.
+			 * The iommu scope can only be promoted with the
+			 * addition of a dirty tracking group.
+			 */
+			group->pinned_page_dirty_scope = true;
+			if (!iommu->pinned_page_dirty_scope)
+				update_pinned_page_dirty_scope(iommu);
 			mutex_unlock(&iommu->lock);
 
 			return 0;
@@ -1792,9 +2121,17 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
 	}
 
 	list_add(&domain->next, &iommu->domain_list);
+	vfio_update_pgsize_bitmap(iommu);
 done:
 	/* Delete the old one and insert new iova list */
 	vfio_iommu_iova_insert_copy(iommu, &iova_copy);
+
+	/*
+	 * An iommu backed group can dirty memory directly and therefore
+	 * demotes the iommu scope until it declares itself dirty tracking
+	 * capable via the page pinning interface.
+	 */
+	iommu->pinned_page_dirty_scope = false;
 	mutex_unlock(&iommu->lock);
 	vfio_iommu_resv_free(&group_resv_regions);
 
@@ -1947,6 +2284,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
 	struct vfio_iommu *iommu = iommu_data;
 	struct vfio_domain *domain;
 	struct vfio_group *group;
+	bool update_dirty_scope = false;
 	LIST_HEAD(iova_copy);
 
 	mutex_lock(&iommu->lock);
@@ -1954,6 +2292,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
 	if (iommu->external_domain) {
 		group = find_iommu_group(iommu->external_domain, iommu_group);
 		if (group) {
+			update_dirty_scope = !group->pinned_page_dirty_scope;
 			list_del(&group->next);
 			kfree(group);
 
@@ -1983,6 +2322,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
 			continue;
 
 		vfio_iommu_detach_group(domain, group);
+		update_dirty_scope = !group->pinned_page_dirty_scope;
 		list_del(&group->next);
 		kfree(group);
 		/*
@@ -2003,6 +2343,7 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
 			list_del(&domain->next);
 			kfree(domain);
 			vfio_iommu_aper_expand(iommu, &iova_copy);
+			vfio_update_pgsize_bitmap(iommu);
 		}
 		break;
 	}
@@ -2013,6 +2354,12 @@ static void vfio_iommu_type1_detach_group(void *iommu_data,
 		vfio_iommu_iova_free(&iova_copy);
 
 detach_group_done:
+	/*
+	 * Removal of a group without dirty tracking may allow the iommu scope
+	 * to be promoted.
+	 */
+	if (update_dirty_scope)
+		update_pinned_page_dirty_scope(iommu);
 	mutex_unlock(&iommu->lock);
 }
 
@@ -2135,8 +2482,6 @@ static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
 	size_t size;
 	int iovas = 0, i = 0, ret;
 
-	mutex_lock(&iommu->lock);
-
 	list_for_each_entry(iova, &iommu->iova_list, list)
 		iovas++;
 
@@ -2145,17 +2490,14 @@ static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
 		 * Return 0 as a container with a single mdev device
 		 * will have an empty list
 		 */
-		ret = 0;
-		goto out_unlock;
+		return 0;
 	}
 
 	size = sizeof(*cap_iovas) + (iovas * sizeof(*cap_iovas->iova_ranges));
 
 	cap_iovas = kzalloc(size, GFP_KERNEL);
-	if (!cap_iovas) {
-		ret = -ENOMEM;
-		goto out_unlock;
-	}
+	if (!cap_iovas)
+		return -ENOMEM;
 
 	cap_iovas->nr_iovas = iovas;
 
@@ -2168,11 +2510,25 @@ static int vfio_iommu_iova_build_caps(struct vfio_iommu *iommu,
 	ret = vfio_iommu_iova_add_cap(caps, cap_iovas, size);
 
 	kfree(cap_iovas);
-out_unlock:
-	mutex_unlock(&iommu->lock);
 	return ret;
 }
 
+static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu,
+					   struct vfio_info_cap *caps)
+{
+	struct vfio_iommu_type1_info_cap_migration cap_mig;
+
+	cap_mig.header.id = VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION;
+	cap_mig.header.version = 1;
+
+	cap_mig.flags = 0;
+	/* support minimum pgsize */
+	cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap);
+	cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX;
+
+	return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig));
+}
+
 static long vfio_iommu_type1_ioctl(void *iommu_data,
 				   unsigned int cmd, unsigned long arg)
 {
@@ -2214,11 +2570,18 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
 			info.cap_offset = 0; /* output, no-recopy necessary */
 		}
 
+		mutex_lock(&iommu->lock);
 		info.flags = VFIO_IOMMU_INFO_PGSIZES;
 
-		info.iova_pgsizes = vfio_pgsize_bitmap(iommu);
+		info.iova_pgsizes = iommu->pgsize_bitmap;
+
+		ret = vfio_iommu_migration_build_caps(iommu, &caps);
+
+		if (!ret)
+			ret = vfio_iommu_iova_build_caps(iommu, &caps);
+
+		mutex_unlock(&iommu->lock);
 
-		ret = vfio_iommu_iova_build_caps(iommu, &caps);
 		if (ret)
 			return ret;
 
@@ -2261,22 +2624,143 @@ static long vfio_iommu_type1_ioctl(void *iommu_data,
 
 	} else if (cmd == VFIO_IOMMU_UNMAP_DMA) {
 		struct vfio_iommu_type1_dma_unmap unmap;
-		long ret;
+		struct vfio_bitmap bitmap = { 0 };
+		int ret;
 
 		minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size);
 
 		if (copy_from_user(&unmap, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		if (unmap.argsz < minsz || unmap.flags)
+		if (unmap.argsz < minsz ||
+		    unmap.flags & ~VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP)
 			return -EINVAL;
 
-		ret = vfio_dma_do_unmap(iommu, &unmap);
+		if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) {
+			unsigned long pgshift;
+
+			if (unmap.argsz < (minsz + sizeof(bitmap)))
+				return -EINVAL;
+
+			if (copy_from_user(&bitmap,
+					   (void __user *)(arg + minsz),
+					   sizeof(bitmap)))
+				return -EFAULT;
+
+			if (!access_ok((void __user *)bitmap.data, bitmap.size))
+				return -EINVAL;
+
+			pgshift = __ffs(bitmap.pgsize);
+			ret = verify_bitmap_size(unmap.size >> pgshift,
+						 bitmap.size);
+			if (ret)
+				return ret;
+		}
+
+		ret = vfio_dma_do_unmap(iommu, &unmap, &bitmap);
 		if (ret)
 			return ret;
 
 		return copy_to_user((void __user *)arg, &unmap, minsz) ?
 			-EFAULT : 0;
+	} else if (cmd == VFIO_IOMMU_DIRTY_PAGES) {
+		struct vfio_iommu_type1_dirty_bitmap dirty;
+		uint32_t mask = VFIO_IOMMU_DIRTY_PAGES_FLAG_START |
+				VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP |
+				VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
+		int ret = 0;
+
+		if (!iommu->v2)
+			return -EACCES;
+
+		minsz = offsetofend(struct vfio_iommu_type1_dirty_bitmap,
+				    flags);
+
+		if (copy_from_user(&dirty, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (dirty.argsz < minsz || dirty.flags & ~mask)
+			return -EINVAL;
+
+		/* only one flag should be set at a time */
+		if (__ffs(dirty.flags) != __fls(dirty.flags))
+			return -EINVAL;
+
+		if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_START) {
+			size_t pgsize;
+
+			mutex_lock(&iommu->lock);
+			pgsize = 1 << __ffs(iommu->pgsize_bitmap);
+			if (!iommu->dirty_page_tracking) {
+				ret = vfio_dma_bitmap_alloc_all(iommu, pgsize);
+				if (!ret)
+					iommu->dirty_page_tracking = true;
+			}
+			mutex_unlock(&iommu->lock);
+			return ret;
+		} else if (dirty.flags & VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP) {
+			mutex_lock(&iommu->lock);
+			if (iommu->dirty_page_tracking) {
+				iommu->dirty_page_tracking = false;
+				vfio_dma_bitmap_free_all(iommu);
+			}
+			mutex_unlock(&iommu->lock);
+			return 0;
+		} else if (dirty.flags &
+				 VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP) {
+			struct vfio_iommu_type1_dirty_bitmap_get range;
+			unsigned long pgshift;
+			size_t data_size = dirty.argsz - minsz;
+			size_t iommu_pgsize;
+
+			if (!data_size || data_size < sizeof(range))
+				return -EINVAL;
+
+			if (copy_from_user(&range, (void __user *)(arg + minsz),
+					   sizeof(range)))
+				return -EFAULT;
+
+			if (range.iova + range.size < range.iova)
+				return -EINVAL;
+			if (!access_ok((void __user *)range.bitmap.data,
+				       range.bitmap.size))
+				return -EINVAL;
+
+			pgshift = __ffs(range.bitmap.pgsize);
+			ret = verify_bitmap_size(range.size >> pgshift,
+						 range.bitmap.size);
+			if (ret)
+				return ret;
+
+			mutex_lock(&iommu->lock);
+
+			iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap);
+
+			/* allow only smallest supported pgsize */
+			if (range.bitmap.pgsize != iommu_pgsize) {
+				ret = -EINVAL;
+				goto out_unlock;
+			}
+			if (range.iova & (iommu_pgsize - 1)) {
+				ret = -EINVAL;
+				goto out_unlock;
+			}
+			if (!range.size || range.size & (iommu_pgsize - 1)) {
+				ret = -EINVAL;
+				goto out_unlock;
+			}
+
+			if (iommu->dirty_page_tracking)
+				ret = vfio_iova_dirty_bitmap(range.bitmap.data,
+						iommu, range.iova, range.size,
+						range.bitmap.pgsize);
+			else
+				ret = -EINVAL;
+out_unlock:
+			mutex_unlock(&iommu->lock);
+
+			return ret;
+		}
 	}
 
 	return -ENOTTY;
@@ -2333,7 +2817,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu,
 		return -EPERM;
 
 	if (kthread)
-		use_mm(mm);
+		kthread_use_mm(mm);
 	else if (current->mm != mm)
 		goto out;
 
@@ -2344,14 +2828,23 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu,
 
 	vaddr = dma->vaddr + offset;
 
-	if (write)
+	if (write) {
 		*copied = copy_to_user((void __user *)vaddr, data,
 					 count) ? 0 : count;
-	else
+		if (*copied && iommu->dirty_page_tracking) {
+			unsigned long pgshift = __ffs(iommu->pgsize_bitmap);
+			/*
+			 * Bitmap populated with the smallest supported page
+			 * size
+			 */
+			bitmap_set(dma->bitmap, offset >> pgshift,
+				   *copied >> pgshift);
+		}
+	} else
 		*copied = copy_from_user(data, (void __user *)vaddr,
 					   count) ? 0 : count;
 	if (kthread)
-		unuse_mm(mm);
+		kthread_unuse_mm(mm);
 out:
 	mmput(mm);
 	return *copied ? 0 : -EFAULT;
diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig
index c4f273793595..d3688c6afb87 100644
--- a/drivers/vhost/Kconfig
+++ b/drivers/vhost/Kconfig
@@ -13,15 +13,6 @@ config VHOST_RING
 	  This option is selected by any driver which needs to access
 	  the host side of a virtio ring.
 
-config VHOST_DPN
-	bool
-	depends on !ARM || AEABI
-	default y
-	help
-	  Anything selecting VHOST or VHOST_RING must depend on VHOST_DPN.
-	  This excludes the deprecated ARM ABI since that forces a 4 byte
-	  alignment on all structs - incompatible with virtio spec requirements.
-
 config VHOST
 	tristate
 	select VHOST_IOTLB
@@ -37,9 +28,9 @@ if VHOST_MENU
 
 config VHOST_NET
 	tristate "Host kernel accelerator for virtio net"
-	depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP) && VHOST_DPN
+	depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP)
 	select VHOST
-	---help---
+	help
 	  This kernel module can be loaded in host kernel to accelerate
 	  guest networking with virtio_net. Not to be confused with virtio_net
 	  module itself which needs to be loaded in guest kernel.
@@ -49,20 +40,20 @@ config VHOST_NET
 
 config VHOST_SCSI
 	tristate "VHOST_SCSI TCM fabric driver"
-	depends on TARGET_CORE && EVENTFD && VHOST_DPN
+	depends on TARGET_CORE && EVENTFD
 	select VHOST
 	default n
-	---help---
+	help
 	Say M here to enable the vhost_scsi TCM fabric module
 	for use with virtio-scsi guests
 
 config VHOST_VSOCK
 	tristate "vhost virtio-vsock driver"
-	depends on VSOCKETS && EVENTFD && VHOST_DPN
+	depends on VSOCKETS && EVENTFD
 	select VHOST
 	select VIRTIO_VSOCKETS_COMMON
 	default n
-	---help---
+	help
 	This kernel module can be loaded in the host kernel to provide AF_VSOCK
 	sockets for communicating with guests.  The guests must have the
 	virtio_transport.ko driver loaded to use the virtio-vsock device.
@@ -72,7 +63,7 @@ config VHOST_VSOCK
 
 config VHOST_VDPA
 	tristate "Vhost driver for vDPA-based backend"
-	depends on EVENTFD && VHOST_DPN
+	depends on EVENTFD
 	select VHOST
 	depends on VDPA
 	help
@@ -85,7 +76,7 @@ config VHOST_VDPA
 config VHOST_CROSS_ENDIAN_LEGACY
 	bool "Cross-endian support for vhost"
 	default n
-	---help---
+	help
 	  This option allows vhost to support guests with a different byte
 	  ordering from host while using legacy virtio.
 
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 516519dcc8ff..e992decfec53 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -1327,7 +1327,7 @@ static int vhost_net_open(struct inode *inode, struct file *f)
 	}
 	vhost_dev_init(dev, vqs, VHOST_NET_VQ_MAX,
 		       UIO_MAXIOV + VHOST_NET_BATCH,
-		       VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT,
+		       VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, true,
 		       NULL);
 
 	vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev);
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index c39952243fd3..6fb4d7ecfa19 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -1628,7 +1628,7 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)
 		vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
 	}
 	vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV,
-		       VHOST_SCSI_WEIGHT, 0, NULL);
+		       VHOST_SCSI_WEIGHT, 0, true, NULL);
 
 	vhost_scsi_init_inflight(vs, NULL);
 
@@ -2280,6 +2280,7 @@ static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
 static const struct target_core_fabric_ops vhost_scsi_ops = {
 	.module				= THIS_MODULE,
 	.fabric_name			= "vhost",
+	.max_data_sg_nents		= VHOST_SCSI_PREALLOC_SGLS,
 	.tpg_get_wwn			= vhost_scsi_get_fabric_wwn,
 	.tpg_get_tag			= vhost_scsi_get_tpgt,
 	.tpg_check_demo_mode		= vhost_scsi_check_true,
diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c
index 9a3a09005e03..0466921f4772 100644
--- a/drivers/vhost/test.c
+++ b/drivers/vhost/test.c
@@ -120,7 +120,7 @@ static int vhost_test_open(struct inode *inode, struct file *f)
 	vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ];
 	n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick;
 	vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX, UIO_MAXIOV,
-		       VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, NULL);
+		       VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, true, NULL);
 
 	f->private_data = n;
 
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index 0968361e3b77..7580e34f76c1 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -15,12 +15,14 @@
 #include <linux/module.h>
 #include <linux/cdev.h>
 #include <linux/device.h>
+#include <linux/mm.h>
 #include <linux/iommu.h>
 #include <linux/uuid.h>
 #include <linux/vdpa.h>
 #include <linux/nospec.h>
 #include <linux/vhost.h>
 #include <linux/virtio_net.h>
+#include <linux/kernel.h>
 
 #include "vhost.h"
 
@@ -70,6 +72,7 @@ struct vhost_vdpa {
 	int nvqs;
 	int virtio_id;
 	int minor;
+	struct eventfd_ctx *config_ctx;
 };
 
 static DEFINE_IDA(vhost_vdpa_ida);
@@ -101,6 +104,17 @@ static irqreturn_t vhost_vdpa_virtqueue_cb(void *private)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t vhost_vdpa_config_cb(void *private)
+{
+	struct vhost_vdpa *v = private;
+	struct eventfd_ctx *config_ctx = v->config_ctx;
+
+	if (config_ctx)
+		eventfd_signal(config_ctx, 1);
+
+	return IRQ_HANDLED;
+}
+
 static void vhost_vdpa_reset(struct vhost_vdpa *v)
 {
 	struct vdpa_device *vdpa = v->vdpa;
@@ -288,6 +302,36 @@ static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)
 	return 0;
 }
 
+static void vhost_vdpa_config_put(struct vhost_vdpa *v)
+{
+	if (v->config_ctx)
+		eventfd_ctx_put(v->config_ctx);
+}
+
+static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)
+{
+	struct vdpa_callback cb;
+	int fd;
+	struct eventfd_ctx *ctx;
+
+	cb.callback = vhost_vdpa_config_cb;
+	cb.private = v->vdpa;
+	if (copy_from_user(&fd, argp, sizeof(fd)))
+		return  -EFAULT;
+
+	ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
+	swap(ctx, v->config_ctx);
+
+	if (!IS_ERR_OR_NULL(ctx))
+		eventfd_ctx_put(ctx);
+
+	if (IS_ERR(v->config_ctx))
+		return PTR_ERR(v->config_ctx);
+
+	v->vdpa->config->set_config_cb(v->vdpa, &cb);
+
+	return 0;
+}
 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 				   void __user *argp)
 {
@@ -395,6 +439,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
 	case VHOST_SET_LOG_FD:
 		r = -ENOIOCTLCMD;
 		break;
+	case VHOST_VDPA_SET_CONFIG_CALL:
+		r = vhost_vdpa_set_config_call(v, argp);
+		break;
 	default:
 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);
 		if (r == -ENOIOCTLCMD)
@@ -527,7 +574,7 @@ static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,
 	if (!npages)
 		return -EINVAL;
 
-	down_read(&dev->mm->mmap_sem);
+	mmap_read_lock(dev->mm);
 
 	locked = atomic64_add_return(npages, &dev->mm->pinned_vm);
 	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
@@ -580,7 +627,7 @@ out:
 		vhost_vdpa_unmap(v, msg->iova, msg->size);
 		atomic64_sub(npages, &dev->mm->pinned_vm);
 	}
-	up_read(&dev->mm->mmap_sem);
+	mmap_read_unlock(dev->mm);
 	free_page((unsigned long)page_list);
 	return ret;
 }
@@ -694,7 +741,7 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep)
 		vqs[i] = &v->vqs[i];
 		vqs[i]->handle_kick = handle_vq_kick;
 	}
-	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0,
+	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
 		       vhost_vdpa_process_iotlb_msg);
 
 	dev->iotlb = vhost_iotlb_alloc(0, 0);
@@ -729,6 +776,7 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)
 	vhost_dev_stop(&v->vdev);
 	vhost_vdpa_iotlb_free(v);
 	vhost_vdpa_free_domain(v);
+	vhost_vdpa_config_put(v);
 	vhost_dev_cleanup(&v->vdev);
 	kfree(v->vdev.vqs);
 	mutex_unlock(&d->mutex);
@@ -739,12 +787,74 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)
 	return 0;
 }
 
+#ifdef CONFIG_MMU
+static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
+{
+	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
+	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+	struct vdpa_notification_area notify;
+	struct vm_area_struct *vma = vmf->vma;
+	u16 index = vma->vm_pgoff;
+
+	notify = ops->get_vq_notification(vdpa, index);
+
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
+			    notify.addr >> PAGE_SHIFT, PAGE_SIZE,
+			    vma->vm_page_prot))
+		return VM_FAULT_SIGBUS;
+
+	return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct vhost_vdpa_vm_ops = {
+	.fault = vhost_vdpa_fault,
+};
+
+static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct vhost_vdpa *v = vma->vm_file->private_data;
+	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
+	struct vdpa_notification_area notify;
+	int index = vma->vm_pgoff;
+
+	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+		return -EINVAL;
+	if ((vma->vm_flags & VM_SHARED) == 0)
+		return -EINVAL;
+	if (vma->vm_flags & VM_READ)
+		return -EINVAL;
+	if (index > 65535)
+		return -EINVAL;
+	if (!ops->get_vq_notification)
+		return -ENOTSUPP;
+
+	/* To be safe and easily modelled by userspace, We only
+	 * support the doorbell which sits on the page boundary and
+	 * does not share the page with other registers.
+	 */
+	notify = ops->get_vq_notification(vdpa, index);
+	if (notify.addr & (PAGE_SIZE - 1))
+		return -EINVAL;
+	if (vma->vm_end - vma->vm_start != notify.size)
+		return -ENOTSUPP;
+
+	vma->vm_ops = &vhost_vdpa_vm_ops;
+	return 0;
+}
+#endif /* CONFIG_MMU */
+
 static const struct file_operations vhost_vdpa_fops = {
 	.owner		= THIS_MODULE,
 	.open		= vhost_vdpa_open,
 	.release	= vhost_vdpa_release,
 	.write_iter	= vhost_vdpa_chr_write_iter,
 	.unlocked_ioctl	= vhost_vdpa_unlocked_ioctl,
+#ifdef CONFIG_MMU
+	.mmap		= vhost_vdpa_mmap,
+#endif /* CONFIG_MMU */
 	.compat_ioctl	= compat_ptr_ioctl,
 };
 
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 21a59b598ed8..d7b8df3edffc 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -14,7 +14,6 @@
 #include <linux/vhost.h>
 #include <linux/uio.h>
 #include <linux/mm.h>
-#include <linux/mmu_context.h>
 #include <linux/miscdevice.h>
 #include <linux/mutex.h>
 #include <linux/poll.h>
@@ -166,11 +165,16 @@ static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync,
 			     void *key)
 {
 	struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
+	struct vhost_work *work = &poll->work;
 
 	if (!(key_to_poll(key) & poll->mask))
 		return 0;
 
-	vhost_poll_queue(poll);
+	if (!poll->dev->use_worker)
+		work->fn(work);
+	else
+		vhost_poll_queue(poll);
+
 	return 0;
 }
 
@@ -330,10 +334,8 @@ static int vhost_worker(void *data)
 	struct vhost_dev *dev = data;
 	struct vhost_work *work, *work_next;
 	struct llist_node *node;
-	mm_segment_t oldfs = get_fs();
 
-	set_fs(USER_DS);
-	use_mm(dev->mm);
+	kthread_use_mm(dev->mm);
 
 	for (;;) {
 		/* mb paired w/ kthread_stop */
@@ -361,8 +363,7 @@ static int vhost_worker(void *data)
 				schedule();
 		}
 	}
-	unuse_mm(dev->mm);
-	set_fs(oldfs);
+	kthread_unuse_mm(dev->mm);
 	return 0;
 }
 
@@ -454,6 +455,7 @@ static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,
 void vhost_dev_init(struct vhost_dev *dev,
 		    struct vhost_virtqueue **vqs, int nvqs,
 		    int iov_limit, int weight, int byte_weight,
+		    bool use_worker,
 		    int (*msg_handler)(struct vhost_dev *dev,
 				       struct vhost_iotlb_msg *msg))
 {
@@ -471,6 +473,7 @@ void vhost_dev_init(struct vhost_dev *dev,
 	dev->iov_limit = iov_limit;
 	dev->weight = weight;
 	dev->byte_weight = byte_weight;
+	dev->use_worker = use_worker;
 	dev->msg_handler = msg_handler;
 	init_llist_head(&dev->work_list);
 	init_waitqueue_head(&dev->wait);
@@ -534,6 +537,36 @@ bool vhost_dev_has_owner(struct vhost_dev *dev)
 }
 EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
 
+static void vhost_attach_mm(struct vhost_dev *dev)
+{
+	/* No owner, become one */
+	if (dev->use_worker) {
+		dev->mm = get_task_mm(current);
+	} else {
+		/* vDPA device does not use worker thead, so there's
+		 * no need to hold the address space for mm. This help
+		 * to avoid deadlock in the case of mmap() which may
+		 * held the refcnt of the file and depends on release
+		 * method to remove vma.
+		 */
+		dev->mm = current->mm;
+		mmgrab(dev->mm);
+	}
+}
+
+static void vhost_detach_mm(struct vhost_dev *dev)
+{
+	if (!dev->mm)
+		return;
+
+	if (dev->use_worker)
+		mmput(dev->mm);
+	else
+		mmdrop(dev->mm);
+
+	dev->mm = NULL;
+}
+
 /* Caller should have device mutex */
 long vhost_dev_set_owner(struct vhost_dev *dev)
 {
@@ -546,21 +579,24 @@ long vhost_dev_set_owner(struct vhost_dev *dev)
 		goto err_mm;
 	}
 
-	/* No owner, become one */
-	dev->mm = get_task_mm(current);
+	vhost_attach_mm(dev);
+
 	dev->kcov_handle = kcov_common_handle();
-	worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
-	if (IS_ERR(worker)) {
-		err = PTR_ERR(worker);
-		goto err_worker;
-	}
+	if (dev->use_worker) {
+		worker = kthread_create(vhost_worker, dev,
+					"vhost-%d", current->pid);
+		if (IS_ERR(worker)) {
+			err = PTR_ERR(worker);
+			goto err_worker;
+		}
 
-	dev->worker = worker;
-	wake_up_process(worker);	/* avoid contributing to loadavg */
+		dev->worker = worker;
+		wake_up_process(worker); /* avoid contributing to loadavg */
 
-	err = vhost_attach_cgroups(dev);
-	if (err)
-		goto err_cgroup;
+		err = vhost_attach_cgroups(dev);
+		if (err)
+			goto err_cgroup;
+	}
 
 	err = vhost_dev_alloc_iovecs(dev);
 	if (err)
@@ -568,12 +604,12 @@ long vhost_dev_set_owner(struct vhost_dev *dev)
 
 	return 0;
 err_cgroup:
-	kthread_stop(worker);
-	dev->worker = NULL;
+	if (dev->worker) {
+		kthread_stop(dev->worker);
+		dev->worker = NULL;
+	}
 err_worker:
-	if (dev->mm)
-		mmput(dev->mm);
-	dev->mm = NULL;
+	vhost_detach_mm(dev);
 	dev->kcov_handle = 0;
 err_mm:
 	return err;
@@ -670,9 +706,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev)
 		dev->worker = NULL;
 		dev->kcov_handle = 0;
 	}
-	if (dev->mm)
-		mmput(dev->mm);
-	dev->mm = NULL;
+	vhost_detach_mm(dev);
 }
 EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
 
@@ -882,7 +916,7 @@ static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
 
 #define vhost_put_user(vq, x, ptr)		\
 ({ \
-	int ret = -EFAULT; \
+	int ret; \
 	if (!vq->iotlb) { \
 		ret = __put_user(x, ptr); \
 	} else { \
@@ -1244,9 +1278,9 @@ static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
 }
 
 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
-			 struct vring_desc __user *desc,
-			 struct vring_avail __user *avail,
-			 struct vring_used __user *used)
+			 vring_desc_t __user *desc,
+			 vring_avail_t __user *avail,
+			 vring_used_t __user *used)
 
 {
 	return access_ok(desc, vhost_get_desc_size(vq, num)) &&
@@ -1574,7 +1608,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
 			r = -EFAULT;
 			break;
 		}
-		eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
+		eventfp = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_fget(f.fd);
 		if (IS_ERR(eventfp)) {
 			r = PTR_ERR(eventfp);
 			break;
@@ -1590,7 +1624,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
 			r = -EFAULT;
 			break;
 		}
-		ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
+		ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
 		if (IS_ERR(ctx)) {
 			r = PTR_ERR(ctx);
 			break;
@@ -1602,7 +1636,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
 			r = -EFAULT;
 			break;
 		}
-		ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
+		ctx = f.fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(f.fd);
 		if (IS_ERR(ctx)) {
 			r = PTR_ERR(ctx);
 			break;
@@ -1727,7 +1761,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
 		r = get_user(fd, (int __user *)argp);
 		if (r < 0)
 			break;
-		ctx = fd == -1 ? NULL : eventfd_ctx_fdget(fd);
+		ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd);
 		if (IS_ERR(ctx)) {
 			r = PTR_ERR(ctx);
 			break;
@@ -1762,15 +1796,14 @@ static int set_bit_to_user(int nr, void __user *addr)
 	int bit = nr + (log % PAGE_SIZE) * 8;
 	int r;
 
-	r = get_user_pages_fast(log, 1, FOLL_WRITE, &page);
+	r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page);
 	if (r < 0)
 		return r;
 	BUG_ON(r != 1);
 	base = kmap_atomic(page);
 	set_bit(bit, base);
 	kunmap_atomic(base);
-	set_page_dirty_lock(page);
-	put_page(page);
+	unpin_user_pages_dirty_lock(&page, 1, true);
 	return 0;
 }
 
@@ -2301,7 +2334,7 @@ static int __vhost_add_used_n(struct vhost_virtqueue *vq,
 			    struct vring_used_elem *heads,
 			    unsigned count)
 {
-	struct vring_used_elem __user *used;
+	vring_used_elem_t __user *used;
 	u16 old, new;
 	int start;
 
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index f8403bd46b85..c8e96a095d3b 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -67,9 +67,9 @@ struct vhost_virtqueue {
 	/* The actual ring of buffers. */
 	struct mutex mutex;
 	unsigned int num;
-	struct vring_desc __user *desc;
-	struct vring_avail __user *avail;
-	struct vring_used __user *used;
+	vring_desc_t __user *desc;
+	vring_avail_t __user *avail;
+	vring_used_t __user *used;
 	const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS];
 	struct file *kick;
 	struct eventfd_ctx *call_ctx;
@@ -154,6 +154,7 @@ struct vhost_dev {
 	int weight;
 	int byte_weight;
 	u64 kcov_handle;
+	bool use_worker;
 	int (*msg_handler)(struct vhost_dev *dev,
 			   struct vhost_iotlb_msg *msg);
 };
@@ -161,6 +162,7 @@ struct vhost_dev {
 bool vhost_exceeds_weight(struct vhost_virtqueue *vq, int pkts, int total_len);
 void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs,
 		    int nvqs, int iov_limit, int weight, int byte_weight,
+		    bool use_worker,
 		    int (*msg_handler)(struct vhost_dev *dev,
 				       struct vhost_iotlb_msg *msg));
 long vhost_dev_set_owner(struct vhost_dev *dev);
diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c
index ba8e0d6cfd97..e059a9a47cdf 100644
--- a/drivers/vhost/vringh.c
+++ b/drivers/vhost/vringh.c
@@ -620,9 +620,9 @@ static inline int xfer_to_user(const struct vringh *vrh,
  */
 int vringh_init_user(struct vringh *vrh, u64 features,
 		     unsigned int num, bool weak_barriers,
-		     struct vring_desc __user *desc,
-		     struct vring_avail __user *avail,
-		     struct vring_used __user *used)
+		     vring_desc_t __user *desc,
+		     vring_avail_t __user *avail,
+		     vring_used_t __user *used)
 {
 	/* Sane power of 2 please! */
 	if (!num || num > 0xffff || (num & (num - 1))) {
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index fb4e944c4d0d..a483cec31d5c 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -632,7 +632,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
 
 	vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),
 		       UIO_MAXIOV, VHOST_VSOCK_PKT_WEIGHT,
-		       VHOST_VSOCK_WEIGHT, NULL);
+		       VHOST_VSOCK_WEIGHT, true, NULL);
 
 	file->private_data = vsock;
 	spin_lock_init(&vsock->send_pkt_list_lock);
diff --git a/drivers/video/backlight/backlight.c b/drivers/video/backlight/backlight.c
index cac3e35d7630..92d80aa0c0ef 100644
--- a/drivers/video/backlight/backlight.c
+++ b/drivers/video/backlight/backlight.c
@@ -433,6 +433,27 @@ struct backlight_device *backlight_device_get_by_type(enum backlight_type type)
 EXPORT_SYMBOL(backlight_device_get_by_type);
 
 /**
+ * backlight_device_get_by_name - Get backlight device by name
+ * @name: Device name
+ *
+ * This function looks up a backlight device by its name. It obtains a reference
+ * on the backlight device and it is the caller's responsibility to drop the
+ * reference by calling backlight_put().
+ *
+ * Returns:
+ * A pointer to the backlight device if found, otherwise NULL.
+ */
+struct backlight_device *backlight_device_get_by_name(const char *name)
+{
+	struct device *dev;
+
+	dev = class_find_device_by_name(backlight_class, name);
+
+	return dev ? to_backlight_device(dev) : NULL;
+}
+EXPORT_SYMBOL(backlight_device_get_by_name);
+
+/**
  * backlight_device_unregister - unregisters a backlight device object.
  * @bd: the backlight device object to be unregistered and freed.
  *
diff --git a/drivers/video/backlight/l4f00242t03.c b/drivers/video/backlight/l4f00242t03.c
index 8554b4aa980c..46f97d1c3d21 100644
--- a/drivers/video/backlight/l4f00242t03.c
+++ b/drivers/video/backlight/l4f00242t03.c
@@ -14,13 +14,11 @@
 #include <linux/kernel.h>
 #include <linux/delay.h>
 #include <linux/module.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/lcd.h>
 #include <linux/slab.h>
 #include <linux/regulator/consumer.h>
-
 #include <linux/spi/spi.h>
-#include <linux/spi/l4f00242t03.h>
 
 struct l4f00242t03_priv {
 	struct spi_device	*spi;
@@ -28,16 +26,18 @@ struct l4f00242t03_priv {
 	int lcd_state;
 	struct regulator *io_reg;
 	struct regulator *core_reg;
+	struct gpio_desc *reset;
+	struct gpio_desc *enable;
 };
 
-static void l4f00242t03_reset(unsigned int gpio)
+static void l4f00242t03_reset(struct gpio_desc *gpiod)
 {
 	pr_debug("l4f00242t03_reset.\n");
-	gpio_set_value(gpio, 1);
+	gpiod_set_value(gpiod, 1);
 	mdelay(100);
-	gpio_set_value(gpio, 0);
+	gpiod_set_value(gpiod, 0);
 	mdelay(10);	/* tRES >= 100us */
-	gpio_set_value(gpio, 1);
+	gpiod_set_value(gpiod, 1);
 	mdelay(20);
 }
 
@@ -45,7 +45,6 @@ static void l4f00242t03_reset(unsigned int gpio)
 
 static void l4f00242t03_lcd_init(struct spi_device *spi)
 {
-	struct l4f00242t03_pdata *pdata = dev_get_platdata(&spi->dev);
 	struct l4f00242t03_priv *priv = spi_get_drvdata(spi);
 	const u16 cmd[] = { 0x36, param(0), 0x3A, param(0x60) };
 	int ret;
@@ -76,21 +75,20 @@ static void l4f00242t03_lcd_init(struct spi_device *spi)
 		return;
 	}
 
-	l4f00242t03_reset(pdata->reset_gpio);
+	l4f00242t03_reset(priv->reset);
 
-	gpio_set_value(pdata->data_enable_gpio, 1);
+	gpiod_set_value(priv->enable, 1);
 	msleep(60);
 	spi_write(spi, (const u8 *)cmd, ARRAY_SIZE(cmd) * sizeof(u16));
 }
 
 static void l4f00242t03_lcd_powerdown(struct spi_device *spi)
 {
-	struct l4f00242t03_pdata *pdata = dev_get_platdata(&spi->dev);
 	struct l4f00242t03_priv *priv = spi_get_drvdata(spi);
 
 	dev_dbg(&spi->dev, "Powering down LCD\n");
 
-	gpio_set_value(pdata->data_enable_gpio, 0);
+	gpiod_set_value(priv->enable, 0);
 
 	regulator_disable(priv->io_reg);
 	regulator_disable(priv->core_reg);
@@ -168,13 +166,6 @@ static struct lcd_ops l4f_ops = {
 static int l4f00242t03_probe(struct spi_device *spi)
 {
 	struct l4f00242t03_priv *priv;
-	struct l4f00242t03_pdata *pdata = dev_get_platdata(&spi->dev);
-	int ret;
-
-	if (pdata == NULL) {
-		dev_err(&spi->dev, "Uninitialized platform data.\n");
-		return -EINVAL;
-	}
 
 	priv = devm_kzalloc(&spi->dev, sizeof(struct l4f00242t03_priv),
 				GFP_KERNEL);
@@ -187,21 +178,21 @@ static int l4f00242t03_probe(struct spi_device *spi)
 
 	priv->spi = spi;
 
-	ret = devm_gpio_request_one(&spi->dev, pdata->reset_gpio,
-			GPIOF_OUT_INIT_HIGH, "lcd l4f00242t03 reset");
-	if (ret) {
+	priv->reset = devm_gpiod_get(&spi->dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(priv->reset)) {
 		dev_err(&spi->dev,
 			"Unable to get the lcd l4f00242t03 reset gpio.\n");
-		return ret;
+		return PTR_ERR(priv->reset);
 	}
+	gpiod_set_consumer_name(priv->reset, "lcd l4f00242t03 reset");
 
-	ret = devm_gpio_request_one(&spi->dev, pdata->data_enable_gpio,
-			GPIOF_OUT_INIT_LOW, "lcd l4f00242t03 data enable");
-	if (ret) {
+	priv->enable = devm_gpiod_get(&spi->dev, "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(priv->enable)) {
 		dev_err(&spi->dev,
 			"Unable to get the lcd l4f00242t03 data en gpio.\n");
-		return ret;
+		return PTR_ERR(priv->enable);
 	}
+	gpiod_set_consumer_name(priv->enable, "lcd l4f00242t03 data enable");
 
 	priv->io_reg = devm_regulator_get(&spi->dev, "vdd");
 	if (IS_ERR(priv->io_reg)) {
diff --git a/drivers/video/backlight/lp855x_bl.c b/drivers/video/backlight/lp855x_bl.c
index f68920131a4a..e94932c69f54 100644
--- a/drivers/video/backlight/lp855x_bl.c
+++ b/drivers/video/backlight/lp855x_bl.c
@@ -456,7 +456,7 @@ static int lp855x_probe(struct i2c_client *cl, const struct i2c_device_id *id)
 		ret = regulator_enable(lp->enable);
 		if (ret < 0) {
 			dev_err(lp->dev, "failed to enable vddio: %d\n", ret);
-			return ret;
+			goto disable_supply;
 		}
 
 		/*
@@ -471,24 +471,34 @@ static int lp855x_probe(struct i2c_client *cl, const struct i2c_device_id *id)
 	ret = lp855x_configure(lp);
 	if (ret) {
 		dev_err(lp->dev, "device config err: %d", ret);
-		return ret;
+		goto disable_vddio;
 	}
 
 	ret = lp855x_backlight_register(lp);
 	if (ret) {
 		dev_err(lp->dev,
 			"failed to register backlight. err: %d\n", ret);
-		return ret;
+		goto disable_vddio;
 	}
 
 	ret = sysfs_create_group(&lp->dev->kobj, &lp855x_attr_group);
 	if (ret) {
 		dev_err(lp->dev, "failed to register sysfs. err: %d\n", ret);
-		return ret;
+		goto disable_vddio;
 	}
 
 	backlight_update_status(lp->bl);
+
 	return 0;
+
+disable_vddio:
+	if (lp->enable)
+		regulator_disable(lp->enable);
+disable_supply:
+	if (lp->supply)
+		regulator_disable(lp->supply);
+
+	return ret;
 }
 
 static int lp855x_remove(struct i2c_client *cl)
@@ -497,6 +507,8 @@ static int lp855x_remove(struct i2c_client *cl)
 
 	lp->bl->props.brightness = 0;
 	backlight_update_status(lp->bl);
+	if (lp->enable)
+		regulator_disable(lp->enable);
 	if (lp->supply)
 		regulator_disable(lp->supply);
 	sysfs_remove_group(&lp->dev->kobj, &lp855x_attr_group);
diff --git a/drivers/video/backlight/qcom-wled.c b/drivers/video/backlight/qcom-wled.c
index 3d276b30a78c..4c8c34b99441 100644
--- a/drivers/video/backlight/qcom-wled.c
+++ b/drivers/video/backlight/qcom-wled.c
@@ -15,16 +15,21 @@
 
 /* From DT binding */
 #define WLED_MAX_STRINGS				4
+#define MOD_A						0
+#define MOD_B						1
 
 #define WLED_DEFAULT_BRIGHTNESS				2048
 #define WLED_SOFT_START_DLY_US				10000
 #define WLED3_SINK_REG_BRIGHT_MAX			0xFFF
+#define WLED5_SINK_REG_BRIGHT_MAX_12B			0xFFF
+#define WLED5_SINK_REG_BRIGHT_MAX_15B			0x7FFF
 
 /* WLED3/WLED4 control registers */
 #define WLED3_CTRL_REG_FAULT_STATUS			0x08
 #define  WLED3_CTRL_REG_ILIM_FAULT_BIT			BIT(0)
 #define  WLED3_CTRL_REG_OVP_FAULT_BIT			BIT(1)
 #define  WLED4_CTRL_REG_SC_FAULT_BIT			BIT(2)
+#define  WLED5_CTRL_REG_OVP_PRE_ALARM_BIT		BIT(4)
 
 #define WLED3_CTRL_REG_INT_RT_STS			0x10
 #define  WLED3_CTRL_REG_OVP_FAULT_STATUS		BIT(1)
@@ -40,6 +45,7 @@
 
 #define WLED3_CTRL_REG_OVP				0x4d
 #define  WLED3_CTRL_REG_OVP_MASK			GENMASK(1, 0)
+#define  WLED5_CTRL_REG_OVP_MASK			GENMASK(3, 0)
 
 #define WLED3_CTRL_REG_ILIMIT				0x4e
 #define  WLED3_CTRL_REG_ILIMIT_MASK			GENMASK(2, 0)
@@ -101,6 +107,44 @@
 
 #define WLED4_SINK_REG_BRIGHT(n)			(0x57 + (n * 0x10))
 
+/* WLED5 specific control registers */
+#define WLED5_CTRL_REG_OVP_INT_CTL			0x5f
+#define  WLED5_CTRL_REG_OVP_INT_TIMER_MASK		GENMASK(2, 0)
+
+/* WLED5 specific sink registers */
+#define WLED5_SINK_REG_MOD_A_EN				0x50
+#define WLED5_SINK_REG_MOD_B_EN				0x60
+#define  WLED5_SINK_REG_MOD_EN_MASK			BIT(7)
+
+#define WLED5_SINK_REG_MOD_A_SRC_SEL			0x51
+#define WLED5_SINK_REG_MOD_B_SRC_SEL			0x61
+#define  WLED5_SINK_REG_MOD_SRC_SEL_HIGH		0
+#define  WLED5_SINK_REG_MOD_SRC_SEL_EXT			0x03
+#define  WLED5_SINK_REG_MOD_SRC_SEL_MASK		GENMASK(1, 0)
+
+#define WLED5_SINK_REG_MOD_A_BRIGHTNESS_WIDTH_SEL	0x52
+#define WLED5_SINK_REG_MOD_B_BRIGHTNESS_WIDTH_SEL	0x62
+#define  WLED5_SINK_REG_BRIGHTNESS_WIDTH_12B		0
+#define  WLED5_SINK_REG_BRIGHTNESS_WIDTH_15B		1
+
+#define WLED5_SINK_REG_MOD_A_BRIGHTNESS_LSB		0x53
+#define WLED5_SINK_REG_MOD_A_BRIGHTNESS_MSB		0x54
+#define WLED5_SINK_REG_MOD_B_BRIGHTNESS_LSB		0x63
+#define WLED5_SINK_REG_MOD_B_BRIGHTNESS_MSB		0x64
+
+#define WLED5_SINK_REG_MOD_SYNC_BIT			0x65
+#define  WLED5_SINK_REG_SYNC_MOD_A_BIT			BIT(0)
+#define  WLED5_SINK_REG_SYNC_MOD_B_BIT			BIT(1)
+#define  WLED5_SINK_REG_SYNC_MASK			GENMASK(1, 0)
+
+/* WLED5 specific per-'string' registers below */
+#define WLED5_SINK_REG_STR_FULL_SCALE_CURR(n)		(0x72 + (n * 0x10))
+
+#define WLED5_SINK_REG_STR_SRC_SEL(n)			(0x73 + (n * 0x10))
+#define  WLED5_SINK_REG_SRC_SEL_MOD_A			0
+#define  WLED5_SINK_REG_SRC_SEL_MOD_B			1
+#define  WLED5_SINK_REG_SRC_SEL_MASK			GENMASK(1, 0)
+
 struct wled_var_cfg {
 	const u32 *values;
 	u32 (*fn)(u32);
@@ -125,6 +169,8 @@ struct wled_config {
 	u32 num_strings;
 	u32 string_i_limit;
 	u32 enabled_strings[WLED_MAX_STRINGS];
+	u32 mod_sel;
+	u32 cabc_sel;
 	bool cs_out_en;
 	bool ext_gen;
 	bool cabc;
@@ -147,14 +193,39 @@ struct wled {
 	u32 max_brightness;
 	u32 short_count;
 	u32 auto_detect_count;
+	u32 version;
 	bool disabled_by_short;
 	bool has_short_detect;
+	bool cabc_disabled;
 	int short_irq;
 	int ovp_irq;
 
 	struct wled_config cfg;
 	struct delayed_work ovp_work;
+
+	/* Configures the brightness. Applicable for wled3, wled4 and wled5 */
 	int (*wled_set_brightness)(struct wled *wled, u16 brightness);
+
+	/* Configures the cabc register. Applicable for wled4 and wled5 */
+	int (*wled_cabc_config)(struct wled *wled, bool enable);
+
+	/*
+	 * Toggles the sync bit for the brightness update to take place.
+	 * Applicable for WLED3, WLED4 and WLED5.
+	 */
+	int (*wled_sync_toggle)(struct wled *wled);
+
+	/*
+	 * Time to wait before checking the OVP status after wled module enable.
+	 * Applicable for WLED4 and WLED5.
+	 */
+	int (*wled_ovp_delay)(struct wled *wled);
+
+	/*
+	 * Determines if the auto string detection is required.
+	 * Applicable for WLED4 and WLED5
+	 */
+	bool (*wled_auto_detection_required)(struct wled *wled);
 };
 
 static int wled3_set_brightness(struct wled *wled, u16 brightness)
@@ -198,6 +269,28 @@ static int wled4_set_brightness(struct wled *wled, u16 brightness)
 	return 0;
 }
 
+static int wled5_set_brightness(struct wled *wled, u16 brightness)
+{
+	int rc, offset;
+	u16 low_limit = wled->max_brightness * 1 / 1000;
+	u8 v[2];
+
+	/* WLED5's lower limit is 0.1% */
+	if (brightness < low_limit)
+		brightness = low_limit;
+
+	v[0] = brightness & 0xff;
+	v[1] = (brightness >> 8) & 0x7f;
+
+	offset = (wled->cfg.mod_sel == MOD_A) ?
+		  WLED5_SINK_REG_MOD_A_BRIGHTNESS_LSB :
+		  WLED5_SINK_REG_MOD_B_BRIGHTNESS_LSB;
+
+	rc = regmap_bulk_write(wled->regmap, wled->sink_addr + offset,
+			       v, 2);
+	return rc;
+}
+
 static void wled_ovp_work(struct work_struct *work)
 {
 	struct wled *wled = container_of(work,
@@ -237,7 +330,7 @@ static int wled_module_enable(struct wled *wled, int val)
 	return 0;
 }
 
-static int wled_sync_toggle(struct wled *wled)
+static int wled3_sync_toggle(struct wled *wled)
 {
 	int rc;
 	unsigned int mask = GENMASK(wled->max_string_count - 1, 0);
@@ -255,6 +348,88 @@ static int wled_sync_toggle(struct wled *wled)
 	return rc;
 }
 
+static int wled5_sync_toggle(struct wled *wled)
+{
+	int rc;
+	u8 val;
+
+	val = (wled->cfg.mod_sel == MOD_A) ? WLED5_SINK_REG_SYNC_MOD_A_BIT :
+					     WLED5_SINK_REG_SYNC_MOD_B_BIT;
+	rc = regmap_update_bits(wled->regmap,
+				wled->sink_addr + WLED5_SINK_REG_MOD_SYNC_BIT,
+				WLED5_SINK_REG_SYNC_MASK, val);
+	if (rc < 0)
+		return rc;
+
+	return regmap_update_bits(wled->regmap,
+				  wled->sink_addr + WLED5_SINK_REG_MOD_SYNC_BIT,
+				  WLED5_SINK_REG_SYNC_MASK, 0);
+}
+
+static int wled_ovp_fault_status(struct wled *wled, bool *fault_set)
+{
+	int rc;
+	u32 int_rt_sts, fault_sts;
+
+	*fault_set = false;
+	rc = regmap_read(wled->regmap,
+			wled->ctrl_addr + WLED3_CTRL_REG_INT_RT_STS,
+			&int_rt_sts);
+	if (rc < 0) {
+		dev_err(wled->dev, "Failed to read INT_RT_STS rc=%d\n", rc);
+		return rc;
+	}
+
+	rc = regmap_read(wled->regmap,
+			wled->ctrl_addr + WLED3_CTRL_REG_FAULT_STATUS,
+			&fault_sts);
+	if (rc < 0) {
+		dev_err(wled->dev, "Failed to read FAULT_STATUS rc=%d\n", rc);
+		return rc;
+	}
+
+	if (int_rt_sts & WLED3_CTRL_REG_OVP_FAULT_STATUS)
+		*fault_set = true;
+
+	if (wled->version == 4 && (fault_sts & WLED3_CTRL_REG_OVP_FAULT_BIT))
+		*fault_set = true;
+
+	if (wled->version == 5 && (fault_sts & (WLED3_CTRL_REG_OVP_FAULT_BIT |
+				   WLED5_CTRL_REG_OVP_PRE_ALARM_BIT)))
+		*fault_set = true;
+
+	if (*fault_set)
+		dev_dbg(wled->dev, "WLED OVP fault detected, int_rt_sts=0x%x fault_sts=0x%x\n",
+			int_rt_sts, fault_sts);
+
+	return rc;
+}
+
+static int wled4_ovp_delay(struct wled *wled)
+{
+	return WLED_SOFT_START_DLY_US;
+}
+
+static int wled5_ovp_delay(struct wled *wled)
+{
+	int rc, delay_us;
+	u32 val;
+	u8 ovp_timer_ms[8] = {1, 2, 4, 8, 12, 16, 20, 24};
+
+	/* For WLED5, get the delay based on OVP timer */
+	rc = regmap_read(wled->regmap, wled->ctrl_addr +
+			 WLED5_CTRL_REG_OVP_INT_CTL, &val);
+	if (rc < 0)
+		delay_us =
+		ovp_timer_ms[val & WLED5_CTRL_REG_OVP_INT_TIMER_MASK] * 1000;
+	else
+		delay_us = 2 * WLED_SOFT_START_DLY_US;
+
+	dev_dbg(wled->dev, "delay_time_us: %d\n", delay_us);
+
+	return delay_us;
+}
+
 static int wled_update_status(struct backlight_device *bl)
 {
 	struct wled *wled = bl_get_data(bl);
@@ -275,7 +450,7 @@ static int wled_update_status(struct backlight_device *bl)
 			goto unlock_mutex;
 		}
 
-		rc = wled_sync_toggle(wled);
+		rc = wled->wled_sync_toggle(wled);
 		if (rc < 0) {
 			dev_err(wled->dev, "wled sync failed rc:%d\n", rc);
 			goto unlock_mutex;
@@ -298,6 +473,50 @@ unlock_mutex:
 	return rc;
 }
 
+static int wled4_cabc_config(struct wled *wled, bool enable)
+{
+	int i, j, rc;
+	u8 val;
+
+	for (i = 0; i < wled->cfg.num_strings; i++) {
+		j = wled->cfg.enabled_strings[i];
+
+		val = enable ? WLED4_SINK_REG_STR_CABC_MASK : 0;
+		rc = regmap_update_bits(wled->regmap, wled->sink_addr +
+					WLED4_SINK_REG_STR_CABC(j),
+					WLED4_SINK_REG_STR_CABC_MASK, val);
+		if (rc < 0)
+			return rc;
+	}
+
+	return 0;
+}
+
+static int wled5_cabc_config(struct wled *wled, bool enable)
+{
+	int rc, offset;
+	u8 reg;
+
+	if (wled->cabc_disabled)
+		return 0;
+
+	reg = enable ? wled->cfg.cabc_sel : 0;
+	offset = (wled->cfg.mod_sel == MOD_A) ? WLED5_SINK_REG_MOD_A_SRC_SEL :
+						WLED5_SINK_REG_MOD_B_SRC_SEL;
+
+	rc = regmap_update_bits(wled->regmap, wled->sink_addr + offset,
+				WLED5_SINK_REG_MOD_SRC_SEL_MASK, reg);
+	if (rc < 0) {
+		pr_err("Error in configuring CABC rc=%d\n", rc);
+		return rc;
+	}
+
+	if (!wled->cfg.cabc_sel)
+		wled->cabc_disabled = true;
+
+	return 0;
+}
+
 #define WLED_SHORT_DLY_MS			20
 #define WLED_SHORT_CNT_MAX			5
 #define WLED_SHORT_RESET_CNT_DLY_US		USEC_PER_SEC
@@ -345,9 +564,10 @@ unlock_mutex:
 
 static void wled_auto_string_detection(struct wled *wled)
 {
-	int rc = 0, i;
-	u32 sink_config = 0, int_sts;
+	int rc = 0, i, delay_time_us;
+	u32 sink_config = 0;
 	u8 sink_test = 0, sink_valid = 0, val;
+	bool fault_set;
 
 	/* Read configured sink configuration */
 	rc = regmap_read(wled->regmap, wled->sink_addr +
@@ -376,14 +596,9 @@ static void wled_auto_string_detection(struct wled *wled)
 	}
 
 	if (wled->cfg.cabc) {
-		for (i = 0; i < wled->cfg.num_strings; i++) {
-			rc = regmap_update_bits(wled->regmap, wled->sink_addr +
-						WLED4_SINK_REG_STR_CABC(i),
-						WLED4_SINK_REG_STR_CABC_MASK,
-						0);
-			if (rc < 0)
-				goto failed_detect;
-		}
+		rc = wled->wled_cabc_config(wled, false);
+		if (rc < 0)
+			goto failed_detect;
 	}
 
 	/* Disable all sinks */
@@ -427,18 +642,17 @@ static void wled_auto_string_detection(struct wled *wled)
 			goto failed_detect;
 		}
 
-		usleep_range(WLED_SOFT_START_DLY_US,
-			     WLED_SOFT_START_DLY_US + 1000);
+		delay_time_us = wled->wled_ovp_delay(wled);
+		usleep_range(delay_time_us, delay_time_us + 1000);
 
-		rc = regmap_read(wled->regmap, wled->ctrl_addr +
-				 WLED3_CTRL_REG_INT_RT_STS, &int_sts);
+		rc = wled_ovp_fault_status(wled, &fault_set);
 		if (rc < 0) {
-			dev_err(wled->dev, "Error in reading WLED3_CTRL_INT_RT_STS rc=%d\n",
+			dev_err(wled->dev, "Error in getting OVP fault_sts, rc=%d\n",
 				rc);
 			goto failed_detect;
 		}
 
-		if (int_sts & WLED3_CTRL_REG_OVP_FAULT_STATUS)
+		if (fault_set)
 			dev_dbg(wled->dev, "WLED OVP fault detected with SINK %d\n",
 				i + 1);
 		else
@@ -478,30 +692,30 @@ static void wled_auto_string_detection(struct wled *wled)
 	}
 
 	/* Enable valid sinks */
-	for (i = 0; i < wled->cfg.num_strings; i++) {
-		if (wled->cfg.cabc) {
-			rc = regmap_update_bits(wled->regmap, wled->sink_addr +
-						WLED4_SINK_REG_STR_CABC(i),
-						WLED4_SINK_REG_STR_CABC_MASK,
-						WLED4_SINK_REG_STR_CABC_MASK);
-			if (rc < 0)
+	if (wled->version == 4) {
+		for (i = 0; i < wled->cfg.num_strings; i++) {
+			if (sink_config &
+			    BIT(WLED4_SINK_REG_CURR_SINK_SHFT + i))
+				val = WLED4_SINK_REG_STR_MOD_MASK;
+			else
+				/* Disable modulator_en for unused sink */
+				val = 0;
+
+			rc = regmap_write(wled->regmap, wled->sink_addr +
+					  WLED4_SINK_REG_STR_MOD_EN(i), val);
+			if (rc < 0) {
+				dev_err(wled->dev, "Failed to configure MODULATOR_EN rc=%d\n",
+					rc);
 				goto failed_detect;
-		}
-
-		if (sink_config & BIT(WLED4_SINK_REG_CURR_SINK_SHFT + i))
-			val = WLED4_SINK_REG_STR_MOD_MASK;
-		else
-			val = 0x0; /* Disable modulator_en for unused sink */
-
-		rc = regmap_write(wled->regmap, wled->sink_addr +
-				  WLED4_SINK_REG_STR_MOD_EN(i), val);
-		if (rc < 0) {
-			dev_err(wled->dev, "Failed to configure MODULATOR_EN rc=%d\n",
-				rc);
-			goto failed_detect;
+			}
 		}
 	}
 
+	/* Enable CABC */
+	rc = wled->wled_cabc_config(wled, true);
+	if (rc < 0)
+		goto failed_detect;
+
 	/* Restore the feedback setting */
 	rc = regmap_write(wled->regmap,
 			  wled->ctrl_addr + WLED3_CTRL_REG_FEEDBACK_CONTROL, 0);
@@ -534,7 +748,8 @@ failed_detect:
 
 #define WLED_AUTO_DETECT_OVP_COUNT		5
 #define WLED_AUTO_DETECT_CNT_DLY_US		USEC_PER_SEC
-static bool wled_auto_detection_required(struct wled *wled)
+
+static bool wled4_auto_detection_required(struct wled *wled)
 {
 	s64 elapsed_time_us;
 
@@ -567,32 +782,39 @@ static bool wled_auto_detection_required(struct wled *wled)
 	return false;
 }
 
+static bool wled5_auto_detection_required(struct wled *wled)
+{
+	if (!wled->cfg.auto_detection_enabled)
+		return false;
+
+	/*
+	 * Unlike WLED4, WLED5 has OVP fault density interrupt configuration
+	 * i.e. to count the number of OVP alarms for a certain duration before
+	 * triggering OVP fault interrupt. By default, number of OVP fault
+	 * events counted before an interrupt is fired is 32 and the time
+	 * interval is 12 ms. If we see one OVP fault interrupt, then that
+	 * should qualify for a real OVP fault condition to run auto detection
+	 * algorithm.
+	 */
+	return true;
+}
+
 static int wled_auto_detection_at_init(struct wled *wled)
 {
 	int rc;
-	u32 fault_status, rt_status;
+	bool fault_set;
 
 	if (!wled->cfg.auto_detection_enabled)
 		return 0;
 
-	rc = regmap_read(wled->regmap,
-			 wled->ctrl_addr + WLED3_CTRL_REG_INT_RT_STS,
-			 &rt_status);
+	rc = wled_ovp_fault_status(wled, &fault_set);
 	if (rc < 0) {
-		dev_err(wled->dev, "Failed to read RT status rc=%d\n", rc);
-		return rc;
-	}
-
-	rc = regmap_read(wled->regmap,
-			 wled->ctrl_addr + WLED3_CTRL_REG_FAULT_STATUS,
-			 &fault_status);
-	if (rc < 0) {
-		dev_err(wled->dev, "Failed to read fault status rc=%d\n", rc);
+		dev_err(wled->dev, "Error in getting OVP fault_sts, rc=%d\n",
+			rc);
 		return rc;
 	}
 
-	if ((rt_status & WLED3_CTRL_REG_OVP_FAULT_STATUS) ||
-	    (fault_status & WLED3_CTRL_REG_OVP_FAULT_BIT)) {
+	if (fault_set) {
 		mutex_lock(&wled->lock);
 		wled_auto_string_detection(wled);
 		mutex_unlock(&wled->lock);
@@ -629,7 +851,7 @@ static irqreturn_t wled_ovp_irq_handler(int irq, void *_wled)
 			int_sts, fault_sts);
 
 	if (fault_sts & WLED3_CTRL_REG_OVP_FAULT_BIT) {
-		if (wled_auto_detection_required(wled)) {
+		if (wled->wled_auto_detection_required(wled)) {
 			mutex_lock(&wled->lock);
 			wled_auto_string_detection(wled);
 			mutex_unlock(&wled->lock);
@@ -811,17 +1033,12 @@ static int wled4_setup(struct wled *wled)
 					wled->cfg.string_i_limit);
 		if (rc < 0)
 			return rc;
-
-		addr = wled->sink_addr +
-				WLED4_SINK_REG_STR_CABC(j);
-		rc = regmap_update_bits(wled->regmap, addr,
-					WLED4_SINK_REG_STR_CABC_MASK,
-					wled->cfg.cabc ?
-					WLED4_SINK_REG_STR_CABC_MASK : 0);
-		if (rc < 0)
-			return rc;
 	}
 
+	rc = wled4_cabc_config(wled, wled->cfg.cabc);
+	if (rc < 0)
+		return rc;
+
 	rc = regmap_update_bits(wled->regmap, wled->ctrl_addr +
 				WLED3_CTRL_REG_MOD_EN,
 				WLED3_CTRL_REG_MOD_EN_MASK,
@@ -835,7 +1052,7 @@ static int wled4_setup(struct wled *wled)
 	if (rc < 0)
 		return rc;
 
-	rc = wled_sync_toggle(wled);
+	rc = wled->wled_sync_toggle(wled);
 	if (rc < 0) {
 		dev_err(wled->dev, "Failed to toggle sync reg rc:%d\n", rc);
 		return rc;
@@ -857,6 +1074,119 @@ static const struct wled_config wled4_config_defaults = {
 	.auto_detection_enabled = false,
 };
 
+static int wled5_setup(struct wled *wled)
+{
+	int rc, temp, i, j, offset;
+	u8 sink_en = 0;
+	u16 addr;
+	u32 val;
+
+	rc = regmap_update_bits(wled->regmap,
+				wled->ctrl_addr + WLED3_CTRL_REG_OVP,
+				WLED5_CTRL_REG_OVP_MASK, wled->cfg.ovp);
+	if (rc < 0)
+		return rc;
+
+	rc = regmap_update_bits(wled->regmap,
+				wled->ctrl_addr + WLED3_CTRL_REG_ILIMIT,
+				WLED3_CTRL_REG_ILIMIT_MASK,
+				wled->cfg.boost_i_limit);
+	if (rc < 0)
+		return rc;
+
+	rc = regmap_update_bits(wled->regmap,
+				wled->ctrl_addr + WLED3_CTRL_REG_FREQ,
+				WLED3_CTRL_REG_FREQ_MASK,
+				wled->cfg.switch_freq);
+	if (rc < 0)
+		return rc;
+
+	/* Per sink/string configuration */
+	for (i = 0; i < wled->cfg.num_strings; ++i) {
+		j = wled->cfg.enabled_strings[i];
+		addr = wled->sink_addr +
+				WLED4_SINK_REG_STR_FULL_SCALE_CURR(j);
+		rc = regmap_update_bits(wled->regmap, addr,
+					WLED4_SINK_REG_STR_FULL_SCALE_CURR_MASK,
+					wled->cfg.string_i_limit);
+		if (rc < 0)
+			return rc;
+
+		addr = wled->sink_addr + WLED5_SINK_REG_STR_SRC_SEL(j);
+		rc = regmap_update_bits(wled->regmap, addr,
+					WLED5_SINK_REG_SRC_SEL_MASK,
+					wled->cfg.mod_sel == MOD_A ?
+					WLED5_SINK_REG_SRC_SEL_MOD_A :
+					WLED5_SINK_REG_SRC_SEL_MOD_B);
+
+		temp = j + WLED4_SINK_REG_CURR_SINK_SHFT;
+		sink_en |= 1 << temp;
+	}
+
+	rc = wled5_cabc_config(wled, wled->cfg.cabc_sel ? true : false);
+	if (rc < 0)
+		return rc;
+
+	/* Enable one of the modulators A or B based on mod_sel */
+	addr = wled->sink_addr + WLED5_SINK_REG_MOD_A_EN;
+	val = (wled->cfg.mod_sel == MOD_A) ? WLED5_SINK_REG_MOD_EN_MASK : 0;
+	rc = regmap_update_bits(wled->regmap, addr,
+				WLED5_SINK_REG_MOD_EN_MASK, val);
+	if (rc < 0)
+		return rc;
+
+	addr = wled->sink_addr + WLED5_SINK_REG_MOD_B_EN;
+	val = (wled->cfg.mod_sel == MOD_B) ? WLED5_SINK_REG_MOD_EN_MASK : 0;
+	rc = regmap_update_bits(wled->regmap, addr,
+				WLED5_SINK_REG_MOD_EN_MASK, val);
+	if (rc < 0)
+		return rc;
+
+	offset = (wled->cfg.mod_sel == MOD_A) ?
+		  WLED5_SINK_REG_MOD_A_BRIGHTNESS_WIDTH_SEL :
+		  WLED5_SINK_REG_MOD_B_BRIGHTNESS_WIDTH_SEL;
+
+	addr = wled->sink_addr + offset;
+	val = (wled->max_brightness == WLED5_SINK_REG_BRIGHT_MAX_15B) ?
+		 WLED5_SINK_REG_BRIGHTNESS_WIDTH_15B :
+		 WLED5_SINK_REG_BRIGHTNESS_WIDTH_12B;
+	rc = regmap_write(wled->regmap, addr, val);
+	if (rc < 0)
+		return rc;
+
+	rc = regmap_update_bits(wled->regmap,
+				wled->sink_addr + WLED4_SINK_REG_CURR_SINK,
+				WLED4_SINK_REG_CURR_SINK_MASK, sink_en);
+	if (rc < 0)
+		return rc;
+
+	/* This updates only FSC configuration in WLED5 */
+	rc = wled->wled_sync_toggle(wled);
+	if (rc < 0) {
+		pr_err("Failed to toggle sync reg rc:%d\n", rc);
+		return rc;
+	}
+
+	rc = wled_auto_detection_at_init(wled);
+	if (rc < 0)
+		return rc;
+
+	return 0;
+}
+
+static const struct wled_config wled5_config_defaults = {
+	.boost_i_limit = 5,
+	.string_i_limit = 10,
+	.ovp = 4,
+	.num_strings = 4,
+	.switch_freq = 11,
+	.mod_sel = 0,
+	.cabc_sel = 0,
+	.cabc = false,
+	.external_pfet = false,
+	.auto_detection_enabled = false,
+};
+
 static const u32 wled3_boost_i_limit_values[] = {
 	105, 385, 525, 805, 980, 1260, 1400, 1680,
 };
@@ -875,6 +1205,16 @@ static const struct wled_var_cfg wled4_boost_i_limit_cfg = {
 	.size = ARRAY_SIZE(wled4_boost_i_limit_values),
 };
 
+static inline u32 wled5_boost_i_limit_values_fn(u32 idx)
+{
+	return 525 + (idx * 175);
+}
+
+static const struct wled_var_cfg wled5_boost_i_limit_cfg = {
+	.fn = wled5_boost_i_limit_values_fn,
+	.size = 8,
+};
+
 static const u32 wled3_ovp_values[] = {
 	35, 32, 29, 27,
 };
@@ -893,6 +1233,21 @@ static const struct wled_var_cfg wled4_ovp_cfg = {
 	.size = ARRAY_SIZE(wled4_ovp_values),
 };
 
+static inline u32 wled5_ovp_values_fn(u32 idx)
+{
+	/*
+	 * 0000 - 38.5 V
+	 * 0001 - 37 V ..
+	 * 1111 - 16 V
+	 */
+	return 38500 - (idx * 1500);
+}
+
+static const struct wled_var_cfg wled5_ovp_cfg = {
+	.fn = wled5_ovp_values_fn,
+	.size = 16,
+};
+
 static u32 wled3_num_strings_values_fn(u32 idx)
 {
 	return idx + 1;
@@ -940,6 +1295,14 @@ static const struct wled_var_cfg wled4_string_cfg = {
 	.size = 16,
 };
 
+static const struct wled_var_cfg wled5_mod_sel_cfg = {
+	.size = 2,
+};
+
+static const struct wled_var_cfg wled5_cabc_sel_cfg = {
+	.size = 4,
+};
+
 static u32 wled_values(const struct wled_var_cfg *cfg, u32 idx)
 {
 	if (idx >= cfg->size)
@@ -951,7 +1314,7 @@ static u32 wled_values(const struct wled_var_cfg *cfg, u32 idx)
 	return idx;
 }
 
-static int wled_configure(struct wled *wled, int version)
+static int wled_configure(struct wled *wled)
 {
 	struct wled_config *cfg = &wled->cfg;
 	struct device *dev = wled->dev;
@@ -1016,6 +1379,44 @@ static int wled_configure(struct wled *wled, int version)
 		},
 	};
 
+	const struct wled_u32_opts wled5_opts[] = {
+		{
+			.name = "qcom,current-boost-limit",
+			.val_ptr = &cfg->boost_i_limit,
+			.cfg = &wled5_boost_i_limit_cfg,
+		},
+		{
+			.name = "qcom,current-limit-microamp",
+			.val_ptr = &cfg->string_i_limit,
+			.cfg = &wled4_string_i_limit_cfg,
+		},
+		{
+			.name = "qcom,ovp-millivolt",
+			.val_ptr = &cfg->ovp,
+			.cfg = &wled5_ovp_cfg,
+		},
+		{
+			.name = "qcom,switching-freq",
+			.val_ptr = &cfg->switch_freq,
+			.cfg = &wled3_switch_freq_cfg,
+		},
+		{
+			.name = "qcom,num-strings",
+			.val_ptr = &cfg->num_strings,
+			.cfg = &wled4_num_strings_cfg,
+		},
+		{
+			.name = "qcom,modulator-sel",
+			.val_ptr = &cfg->mod_sel,
+			.cfg = &wled5_mod_sel_cfg,
+		},
+		{
+			.name = "qcom,cabc-sel",
+			.val_ptr = &cfg->cabc_sel,
+			.cfg = &wled5_cabc_sel_cfg,
+		},
+	};
+
 	const struct wled_bool_opts bool_opts[] = {
 		{ "qcom,cs-out", &cfg->cs_out_en, },
 		{ "qcom,ext-gen", &cfg->ext_gen, },
@@ -1035,12 +1436,13 @@ static int wled_configure(struct wled *wled, int version)
 	if (rc)
 		wled->name = devm_kasprintf(dev, GFP_KERNEL, "%pOFn", dev->of_node);
 
-	switch (version) {
+	switch (wled->version) {
 	case 3:
 		u32_opts = wled3_opts;
 		size = ARRAY_SIZE(wled3_opts);
 		*cfg = wled3_config_defaults;
 		wled->wled_set_brightness = wled3_set_brightness;
+		wled->wled_sync_toggle = wled3_sync_toggle;
 		wled->max_string_count = 3;
 		wled->sink_addr = wled->ctrl_addr;
 		break;
@@ -1050,6 +1452,31 @@ static int wled_configure(struct wled *wled, int version)
 		size = ARRAY_SIZE(wled4_opts);
 		*cfg = wled4_config_defaults;
 		wled->wled_set_brightness = wled4_set_brightness;
+		wled->wled_sync_toggle = wled3_sync_toggle;
+		wled->wled_cabc_config = wled4_cabc_config;
+		wled->wled_ovp_delay = wled4_ovp_delay;
+		wled->wled_auto_detection_required =
+					wled4_auto_detection_required;
+		wled->max_string_count = 4;
+
+		prop_addr = of_get_address(dev->of_node, 1, NULL, NULL);
+		if (!prop_addr) {
+			dev_err(wled->dev, "invalid IO resources\n");
+			return -EINVAL;
+		}
+		wled->sink_addr = be32_to_cpu(*prop_addr);
+		break;
+
+	case 5:
+		u32_opts = wled5_opts;
+		size = ARRAY_SIZE(wled5_opts);
+		*cfg = wled5_config_defaults;
+		wled->wled_set_brightness = wled5_set_brightness;
+		wled->wled_sync_toggle = wled5_sync_toggle;
+		wled->wled_cabc_config = wled5_cabc_config;
+		wled->wled_ovp_delay = wled5_ovp_delay;
+		wled->wled_auto_detection_required =
+					wled5_auto_detection_required;
 		wled->max_string_count = 4;
 
 		prop_addr = of_get_address(dev->of_node, 1, NULL, NULL);
@@ -1186,7 +1613,6 @@ static int wled_probe(struct platform_device *pdev)
 	struct backlight_device *bl;
 	struct wled *wled;
 	struct regmap *regmap;
-	int version;
 	u32 val;
 	int rc;
 
@@ -1203,18 +1629,22 @@ static int wled_probe(struct platform_device *pdev)
 	wled->regmap = regmap;
 	wled->dev = &pdev->dev;
 
-	version = (uintptr_t)of_device_get_match_data(&pdev->dev);
-	if (!version) {
+	wled->version = (uintptr_t)of_device_get_match_data(&pdev->dev);
+	if (!wled->version) {
 		dev_err(&pdev->dev, "Unknown device version\n");
 		return -ENODEV;
 	}
 
 	mutex_init(&wled->lock);
-	rc = wled_configure(wled, version);
+	rc = wled_configure(wled);
 	if (rc)
 		return rc;
 
-	switch (version) {
+	val = WLED3_SINK_REG_BRIGHT_MAX;
+	of_property_read_u32(pdev->dev.of_node, "max-brightness", &val);
+	wled->max_brightness = val;
+
+	switch (wled->version) {
 	case 3:
 		wled->cfg.auto_detection_enabled = false;
 		rc = wled3_setup(wled);
@@ -1233,6 +1663,18 @@ static int wled_probe(struct platform_device *pdev)
 		}
 		break;
 
+	case 5:
+		wled->has_short_detect = true;
+		if (wled->cfg.cabc_sel)
+			wled->max_brightness = WLED5_SINK_REG_BRIGHT_MAX_12B;
+
+		rc = wled5_setup(wled);
+		if (rc) {
+			dev_err(&pdev->dev, "wled5_setup failed\n");
+			return rc;
+		}
+		break;
+
 	default:
 		dev_err(wled->dev, "Invalid WLED version\n");
 		break;
@@ -1254,7 +1696,7 @@ static int wled_probe(struct platform_device *pdev)
 	memset(&props, 0, sizeof(struct backlight_properties));
 	props.type = BACKLIGHT_RAW;
 	props.brightness = val;
-	props.max_brightness = WLED3_SINK_REG_BRIGHT_MAX;
+	props.max_brightness = wled->max_brightness;
 	bl = devm_backlight_device_register(&pdev->dev, wled->name,
 					    &pdev->dev, wled,
 					    &wled_ops, &props);
@@ -1277,6 +1719,7 @@ static const struct of_device_id wled_match_table[] = {
 	{ .compatible = "qcom,pm8941-wled", .data = (void *)3 },
 	{ .compatible = "qcom,pmi8998-wled", .data = (void *)4 },
 	{ .compatible = "qcom,pm660l-wled", .data = (void *)4 },
+	{ .compatible = "qcom,pm8150l-wled", .data = (void *)5 },
 	{}
 };
 MODULE_DEVICE_TABLE(of, wled_match_table);
diff --git a/drivers/video/console/Kconfig b/drivers/video/console/Kconfig
index 3c01b0d2414f..5e850cc9f891 100644
--- a/drivers/video/console/Kconfig
+++ b/drivers/video/console/Kconfig
@@ -71,7 +71,7 @@ config VGACON_SOFT_SCROLLBACK_PERSISTENT_ENABLE_BY_DEFAULT
 config MDA_CONSOLE
 	depends on !M68K && !PARISC && ISA
 	tristate "MDA text console (dual-headed)"
-	---help---
+	help
 	  Say Y here if you have an old MDA or monochrome Hercules graphics
 	  adapter in your system acting as a second head ( = video card). You
 	  will then be able to use two monitors with your Linux system. Do not
@@ -128,7 +128,7 @@ config FRAMEBUFFER_CONSOLE_DETECT_PRIMARY
        bool "Map the console to the primary display device"
        depends on FRAMEBUFFER_CONSOLE
        default n
-       ---help---
+	help
 	 If this option is selected, the framebuffer console will
 	 automatically select the primary display device (if the architecture
 	 supports this feature).  Otherwise, the framebuffer console will
diff --git a/drivers/video/console/newport_con.c b/drivers/video/console/newport_con.c
index 00dddf6e08b0..504cda38763e 100644
--- a/drivers/video/console/newport_con.c
+++ b/drivers/video/console/newport_con.c
@@ -24,7 +24,6 @@
 #include <asm/io.h>
 #include <linux/uaccess.h>
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/gio_device.h>
 
 #include <video/newport.h>
diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig
index fa88e8b9a83d..0f559aeaf469 100644
--- a/drivers/video/fbdev/Kconfig
+++ b/drivers/video/fbdev/Kconfig
@@ -13,7 +13,7 @@ menuconfig FB
 	tristate "Support for frame buffer devices"
 	select FB_CMDLINE
 	select FB_NOTIFY
-	---help---
+	help
 	  The frame buffer device provides an abstraction for the graphics
 	  hardware. It represents the frame buffer of some video hardware and
 	  allows application software to access the graphics hardware through
@@ -48,7 +48,7 @@ menuconfig FB
 config FIRMWARE_EDID
 	bool "Enable firmware EDID"
 	depends on FB
-	---help---
+	help
 	  This enables access to the EDID transferred from the firmware.
 	  On the i386, this is from the Video BIOS. Enable this if DDC/I2C
 	  transfers do not work for your driver and if you are using
@@ -69,14 +69,14 @@ config FB_DDC
 config FB_BOOT_VESA_SUPPORT
 	bool
 	depends on FB
-	---help---
+	help
 	  If true, at least one selected framebuffer driver can take advantage
 	  of VESA video modes set at an early boot stage via the vga= parameter.
 
 config FB_CFB_FILLRECT
 	tristate
 	depends on FB
-	---help---
+	help
 	  Include the cfb_fillrect function for generic software rectangle
 	  filling. This is used by drivers that don't provide their own
 	  (accelerated) version.
@@ -84,7 +84,7 @@ config FB_CFB_FILLRECT
 config FB_CFB_COPYAREA
 	tristate
 	depends on FB
-	---help---
+	help
 	  Include the cfb_copyarea function for generic software area copying.
 	  This is used by drivers that don't provide their own (accelerated)
 	  version.
@@ -92,7 +92,7 @@ config FB_CFB_COPYAREA
 config FB_CFB_IMAGEBLIT
 	tristate
 	depends on FB
-	---help---
+	help
 	  Include the cfb_imageblit function for generic software image
 	  blitting. This is used by drivers that don't provide their own
 	  (accelerated) version.
@@ -100,7 +100,7 @@ config FB_CFB_IMAGEBLIT
 config FB_CFB_REV_PIXELS_IN_BYTE
 	bool
 	depends on FB
-	---help---
+	help
 	  Allow generic frame-buffer functions to work on displays with 1, 2
 	  and 4 bits per pixel depths which has opposite order of pixels in
 	  byte order to bytes in long order.
@@ -108,7 +108,7 @@ config FB_CFB_REV_PIXELS_IN_BYTE
 config FB_SYS_FILLRECT
 	tristate
 	depends on FB
-	---help---
+	help
 	  Include the sys_fillrect function for generic software rectangle
 	  filling. This is used by drivers that don't provide their own
 	  (accelerated) version and the framebuffer is in system RAM.
@@ -116,7 +116,7 @@ config FB_SYS_FILLRECT
 config FB_SYS_COPYAREA
 	tristate
 	depends on FB
-	---help---
+	help
 	  Include the sys_copyarea function for generic software area copying.
 	  This is used by drivers that don't provide their own (accelerated)
 	  version and the framebuffer is in system RAM.
@@ -124,7 +124,7 @@ config FB_SYS_COPYAREA
 config FB_SYS_IMAGEBLIT
 	tristate
 	depends on FB
-	---help---
+	help
 	  Include the sys_imageblit function for generic software image
 	  blitting. This is used by drivers that don't provide their own
 	  (accelerated) version and the framebuffer is in system RAM.
@@ -132,14 +132,14 @@ config FB_SYS_IMAGEBLIT
 config FB_PROVIDE_GET_FB_UNMAPPED_AREA
 	bool
 	depends on FB
-	---help---
+	help
 	  Allow generic frame-buffer to provide get_fb_unmapped_area
 	  function.
 
 menuconfig FB_FOREIGN_ENDIAN
 	bool "Framebuffer foreign endianness support"
 	depends on FB
-	---help---
+	help
 	  This menu will let you enable support for the framebuffers with
 	  non-native endianness (e.g. Little-Endian framebuffer on a
 	  Big-Endian machine). Most probably you don't have such hardware,
@@ -176,7 +176,7 @@ config FB_HECUBA
 config FB_SVGALIB
 	tristate
 	depends on FB
-	---help---
+	help
 	  Common utility functions useful to fbdev drivers of VGA-based
 	  cards.
 
@@ -192,7 +192,7 @@ config FB_BACKLIGHT
 config FB_MODE_HELPERS
 	bool "Enable Video Mode Handling Helpers"
 	depends on FB
-	---help---
+	help
 	  This enables functions for handling video modes using the
 	  Generalized Timing Formula and the EDID parser. A few drivers rely
 	  on this feature such as the radeonfb, rivafb, and the i810fb. If
@@ -202,7 +202,7 @@ config FB_MODE_HELPERS
 config FB_TILEBLITTING
 	bool "Enable Tile Blitting Support"
 	depends on FB
-	---help---
+	help
 	  This enables tile blitting.  Tile blitting is a drawing technique
 	  where the screen is divided into rectangular sections (tiles), whereas
 	  the standard blitting divides the screen into pixels. Because the
@@ -225,7 +225,7 @@ config FB_GRVGA
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  This enables support for the SVGACTRL framebuffer in the GRLIB IP library from Aeroflex Gaisler.
 
 config FB_CIRRUS
@@ -234,7 +234,7 @@ config FB_CIRRUS
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  This enables support for Cirrus Logic GD542x/543x based boards on
 	  Amiga: SD64, Piccolo, Picasso II/II+, Picasso IV, or EGS Spectrum.
 
@@ -553,7 +553,7 @@ config FB_STI
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
 	default y
-	---help---
+	help
 	  STI refers to the HP "Standard Text Interface" which is a set of
 	  BIOS routines contained in a ROM chip in HP PA-RISC based machines.
 	  Enabling this option will implement the linux framebuffer device
@@ -587,7 +587,7 @@ config FB_TGA
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
 	select BITREVERSE
-	---help---
+	help
 	  This is the frame buffer device driver for generic TGA and SFB+
 	  graphic cards.  These include DEC ZLXp-E1, -E2 and -E3 PCI cards,
 	  also known as PBXGA-A, -B and -C, and DEC ZLX-E1, -E2 and -E3
@@ -815,7 +815,7 @@ config FB_PVR2
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Say Y here if you have a PowerVR 2 card in your box.  If you plan to
 	  run linux on your Dreamcast, you will have to say Y here.
 	  This driver may or may not work on other PowerVR 2 cards, but is
@@ -1066,7 +1066,7 @@ config FB_INTEL
 config FB_INTEL_DEBUG
 	bool "Intel driver Debug Messages"
 	depends on FB_INTEL
-	---help---
+	help
 	  Say Y here if you want the Intel driver to output all sorts
 	  of debugging information to provide to the maintainer when
 	  something goes wrong.
@@ -1087,7 +1087,7 @@ config FB_MATROX
 	select FB_CFB_IMAGEBLIT
 	select FB_TILEBLITTING
 	select FB_MACMODES if PPC_PMAC
-	---help---
+	help
 	  Say Y here if you have a Matrox Millennium, Matrox Millennium II,
 	  Matrox Mystique, Matrox Mystique 220, Matrox Productiva G100, Matrox
 	  Mystique G200, Matrox Millennium G200, Matrox Marvel G200 video,
@@ -1123,7 +1123,7 @@ config FB_MATROX_MYSTIQUE
 config FB_MATROX_G
 	bool "G100/G200/G400/G450/G550 support"
 	depends on FB_MATROX
-	---help---
+	help
 	  Say Y here if you have a Matrox G100, G200, G400, G450 or G550 based
 	  video card. If you select "Advanced lowlevel driver options", you
 	  should check 8 bpp packed pixel, 16 bpp packed pixel, 24 bpp packed
@@ -1154,7 +1154,7 @@ config FB_MATROX_I2C
 	tristate "Matrox I2C support"
 	depends on FB_MATROX
 	select FB_DDC
-	---help---
+	help
 	  This drivers creates I2C buses which are needed for accessing the
 	  DDC (I2C) bus present on all Matroxes, an I2C bus which
 	  interconnects Matrox optional devices, like MGA-TVO on G200 and
@@ -1170,7 +1170,7 @@ config FB_MATROX_I2C
 config FB_MATROX_MAVEN
 	tristate "G400 second head support"
 	depends on FB_MATROX_G && FB_MATROX_I2C
-	---help---
+	help
 	  WARNING !!! This support does not work with G450 !!!
 
 	  Say Y or M here if you want to use a secondary head (meaning two
@@ -1321,7 +1321,7 @@ config FB_S3
 	select FB_SVGALIB
 	select VGASTATE
 	select FONT_8x16 if FRAMEBUFFER_CONSOLE
-	---help---
+	help
 	  Driver for graphics boards with S3 Trio / S3 Virge chip.
 
 config FB_S3_DDC
@@ -1484,7 +1484,7 @@ config FB_3DFX
 config FB_3DFX_ACCEL
 	bool "3Dfx Acceleration functions"
 	depends on FB_3DFX
-	---help---
+	help
 	This will compile the 3Dfx Banshee/Voodoo3/VSA-100 frame buffer
 	device driver with acceleration functions.
 
@@ -1502,7 +1502,7 @@ config FB_VOODOO1
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Say Y here if you have a 3Dfx Voodoo Graphics (Voodoo1/sst1) or
 	  Voodoo2 (cvg) based graphics card.
 
@@ -1524,7 +1524,7 @@ config FB_VT8623
 	select FB_SVGALIB
 	select VGASTATE
 	select FONT_8x16 if FRAMEBUFFER_CONSOLE
-	---help---
+	help
 	  Driver for CastleRock integrated graphics core in the
 	  VIA VT8623 [Apollo CLE266] chipset.
 
@@ -1536,7 +1536,7 @@ config FB_TRIDENT
 	select FB_CFB_IMAGEBLIT
 	select FB_DDC
 	select FB_MODE_HELPERS
-	---help---
+	help
 	  This is the frame buffer device driver for Trident PCI/AGP chipsets.
 	  Supported chipset families are TGUI 9440/96XX, 3DImage, Blade3D
 	  and Blade XP.
@@ -1560,7 +1560,7 @@ config FB_ARK
 	select FB_SVGALIB
 	select VGASTATE
 	select FONT_8x16 if FRAMEBUFFER_CONSOLE
-	---help---
+	help
 	  Driver for PCI graphics boards with ARK 2000PV chip
 	  and ICS 5342 RAMDAC.
 
@@ -1738,7 +1738,7 @@ config FB_PXA168
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Frame buffer driver for the built-in LCD controller in the Marvell
 	  MMP processor.
 
@@ -1750,7 +1750,7 @@ config FB_PXA
 	select FB_CFB_IMAGEBLIT
 	select VIDEOMODE_HELPERS if OF
 	select FB_MODE_HELPERS if OF
-	---help---
+	help
 	  Frame buffer driver for the built-in LCD controller in the Intel
 	  PXA2x0 processor.
 
@@ -1772,7 +1772,7 @@ config FB_PXA_SMARTPANEL
 config FB_PXA_PARAMETERS
 	bool "PXA LCD command line parameters"
 	depends on FB_PXA
-	---help---
+	help
 	  Enable the use of kernel command line or module parameters
 	  to configure the physical properties of the LCD panel when
 	  using the PXA LCD driver.
@@ -1801,14 +1801,14 @@ config FB_MBX
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Framebuffer driver for the Intel 2700G (Marathon) Graphics
 	  Accelerator
 
 config FB_MBX_DEBUG
 	bool "Enable debugging info via debugfs"
 	depends on FB_MBX && DEBUG_FS
-	---help---
+	help
 	  Enable this if you want debugging information using the debug
 	  filesystem (debugfs)
 
@@ -1822,7 +1822,7 @@ config FB_FSL_DIU
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
 	select PPC_LIB_RHEAP
-	---help---
+	help
 	  Framebuffer driver for the Freescale SoC DIU
 
 config FB_W100
@@ -1831,7 +1831,7 @@ config FB_W100
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Frame buffer driver for the w100 as found on the Sharp SL-Cxx series.
 	  It can also drive the w3220 chip found on iPAQ hx4700.
 
@@ -1852,7 +1852,7 @@ config FB_SH_MOBILE_LCDC
 	select FB_SYS_FOPS
 	select FB_DEFERRED_IO
 	select FB_BACKLIGHT
-	---help---
+	help
 	  Frame buffer driver for the on-chip SH-Mobile LCD controller.
 
 config FB_TMIO
@@ -1861,7 +1861,7 @@ config FB_TMIO
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Frame buffer driver for the Toshiba Mobile IO integrated as found
 	  on the Sharp SL-6000 series
 
@@ -1884,7 +1884,7 @@ config FB_S3C
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Frame buffer driver for the built-in FB controller in the Samsung
 	  SoC line from the S3C2443 onwards, including the S3C2416, S3C2450,
 	  and the S3C64XX series such as the S3C6400 and S3C6410.
@@ -1899,7 +1899,7 @@ config FB_S3C
 config FB_S3C_DEBUG_REGWRITE
 	bool "Debug register writes"
 	depends on FB_S3C
-	---help---
+	help
 	  Show all register writes via pr_debug()
 
 config FB_S3C2410
@@ -1908,7 +1908,7 @@ config FB_S3C2410
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Frame buffer driver for the built-in LCD controller in the Samsung
 	  S3C2410 processor.
 
@@ -1931,7 +1931,7 @@ config FB_SM501
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Frame buffer driver for the CRT and LCD controllers in the Silicon
 	  Motion SM501.
 
@@ -1951,7 +1951,7 @@ config FB_SMSCUFX
 	select FB_SYS_IMAGEBLIT
 	select FB_SYS_FOPS
 	select FB_DEFERRED_IO
-	---help---
+	help
 	  This is a kernel framebuffer driver for SMSC UFX USB devices.
 	  Supports fbdev clients like xf86-video-fbdev, kdrive, fbi, and
 	  mplayer -vo fbdev. Supports both UFX6000 (USB 2.0) and UFX7000
@@ -1967,7 +1967,7 @@ config FB_UDL
 	select FB_SYS_IMAGEBLIT
 	select FB_SYS_FOPS
 	select FB_DEFERRED_IO
-	---help---
+	help
 	  This is a kernel framebuffer driver for DisplayLink USB devices.
 	  Supports fbdev clients like xf86-video-fbdev, kdrive, fbi, and
 	  mplayer -vo fbdev. Supports all USB 2.0 era DisplayLink devices.
@@ -1979,7 +1979,7 @@ config FB_IBM_GXT4500
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Say Y here to enable support for the IBM GXT4000P/6000P and
 	  GXT4500P/6500P display adaptor based on Raster Engine RC1000,
 	  found on some IBM System P (pSeries) machines. This driver
@@ -1993,14 +1993,14 @@ config FB_PS3
 	select FB_SYS_COPYAREA
 	select FB_SYS_IMAGEBLIT
 	select FB_SYS_FOPS
-	---help---
+	help
 	  Include support for the virtual frame buffer in the PS3 platform.
 
 config FB_PS3_DEFAULT_SIZE_M
 	int "PS3 default frame buffer size (in MiB)"
 	depends on FB_PS3
 	default 9
-	---help---
+	help
 	  This is the default size (in MiB) of the virtual frame buffer in
 	  the PS3.
 	  The default value can be overridden on the kernel command line
@@ -2008,11 +2008,11 @@ config FB_PS3_DEFAULT_SIZE_M
 
 config FB_XILINX
 	tristate "Xilinx frame buffer support"
-	depends on FB && (XILINX_VIRTEX || MICROBLAZE || ARCH_ZYNQ || ARCH_ZYNQMP)
+	depends on FB && (MICROBLAZE || ARCH_ZYNQ || ARCH_ZYNQMP)
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Include support for the Xilinx ML300/ML403 reference design
 	  framebuffer. ML300 carries a 640*480 LCD display on the board,
 	  ML403 uses a standard DB15 VGA connector.
@@ -2024,7 +2024,7 @@ config FB_GOLDFISH
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Framebuffer driver for Goldfish Virtual Platform
 
 config FB_COBALT
@@ -2038,7 +2038,7 @@ config FB_SH7760
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Support for the SH7760/SH7763/SH7720/SH7721 integrated
 	  (D)STN/TFT LCD Controller.
 	  Supports display resolutions up to 1024x1024 pixel, grayscale and
@@ -2056,7 +2056,7 @@ config FB_DA8XX
 	select FB_CFB_REV_PIXELS_IN_BYTE
 	select FB_MODE_HELPERS
 	select VIDEOMODE_HELPERS
-	---help---
+	help
 	  This is the frame buffer device driver for the TI LCD controller
 	  found on DA8xx/OMAP-L1xx/AM335x SoCs.
 	  If unsure, say N.
@@ -2068,7 +2068,7 @@ config FB_VIRTUAL
 	select FB_SYS_COPYAREA
 	select FB_SYS_IMAGEBLIT
 	select FB_SYS_FOPS
-	---help---
+	help
 	  This is a `virtual' frame buffer device. It operates on a chunk of
 	  unswappable kernel memory instead of on the memory of a graphics
 	  board. This means you cannot see any output sent to this frame
@@ -2119,7 +2119,7 @@ config FB_MB862XX
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Frame buffer driver for Fujitsu Carmine/Coral-P(A)/Lime controllers.
 
 choice
@@ -2129,7 +2129,7 @@ choice
 config FB_MB862XX_PCI_GDC
 	bool "Carmine/Coral-P(A) GDC"
 	depends on PCI
-	---help---
+	help
 	  This enables framebuffer support for Fujitsu Carmine/Coral-P(A)
 	  PCI graphics controller devices.
 
@@ -2138,7 +2138,7 @@ config FB_MB862XX_LIME
 	depends on OF && PPC
 	select FB_FOREIGN_ENDIAN
 	select FB_LITTLE_ENDIAN
-	---help---
+	help
 	  Framebuffer support for Fujitsu Lime GDC on host CPU bus.
 
 endchoice
@@ -2159,7 +2159,7 @@ config FB_EP93XX
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Framebuffer driver for the Cirrus Logic EP93XX series of processors.
 	  This driver is also available as a module. The module will be called
 	  ep93xx-fb.
@@ -2167,7 +2167,7 @@ config FB_EP93XX
 config FB_PRE_INIT_FB
 	bool "Don't reinitialize, use bootloader's GDC/Display configuration"
 	depends on FB && FB_MB862XX_LIME
-	---help---
+	help
 	  Select this option if display contents should be inherited as set by
 	  the bootloader.
 
diff --git a/drivers/video/fbdev/acornfb.c b/drivers/video/fbdev/acornfb.c
index a3af49529173..09a9ad901dad 100644
--- a/drivers/video/fbdev/acornfb.c
+++ b/drivers/video/fbdev/acornfb.c
@@ -30,7 +30,6 @@
 #include <mach/hardware.h>
 #include <asm/irq.h>
 #include <asm/mach-types.h>
-#include <asm/pgtable.h>
 
 #include "acornfb.h"
 
diff --git a/drivers/video/fbdev/atafb.c b/drivers/video/fbdev/atafb.c
index 51f5d1c56fd9..f253daa05d9d 100644
--- a/drivers/video/fbdev/atafb.c
+++ b/drivers/video/fbdev/atafb.c
@@ -58,7 +58,6 @@
 
 #include <asm/setup.h>
 #include <linux/uaccess.h>
-#include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <asm/io.h>
 
diff --git a/drivers/video/fbdev/cirrusfb.c b/drivers/video/fbdev/cirrusfb.c
index c3a3e344cee3..3df64a973194 100644
--- a/drivers/video/fbdev/cirrusfb.c
+++ b/drivers/video/fbdev/cirrusfb.c
@@ -42,7 +42,6 @@
 #include <linux/delay.h>
 #include <linux/fb.h>
 #include <linux/init.h>
-#include <asm/pgtable.h>
 
 #ifdef CONFIG_ZORRO
 #include <linux/zorro.h>
diff --git a/drivers/video/fbdev/cyber2000fb.c b/drivers/video/fbdev/cyber2000fb.c
index 513f58f28b0f..42d37bed518a 100644
--- a/drivers/video/fbdev/cyber2000fb.c
+++ b/drivers/video/fbdev/cyber2000fb.c
@@ -47,7 +47,6 @@
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 
-#include <asm/pgtable.h>
 
 #ifdef __arm__
 #include <asm/mach-types.h>
diff --git a/drivers/video/fbdev/fb-puv3.c b/drivers/video/fbdev/fb-puv3.c
index 75df6aabac21..030e85c11a78 100644
--- a/drivers/video/fbdev/fb-puv3.c
+++ b/drivers/video/fbdev/fb-puv3.c
@@ -18,7 +18,6 @@
 #include <linux/mm.h>
 
 #include <linux/sizes.h>
-#include <asm/pgtable.h>
 #include <mach/hardware.h>
 
 /* Platform_data reserved for unifb registers. */
diff --git a/drivers/video/fbdev/geode/Kconfig b/drivers/video/fbdev/geode/Kconfig
index b36b94b3ae43..ac9c860592aa 100644
--- a/drivers/video/fbdev/geode/Kconfig
+++ b/drivers/video/fbdev/geode/Kconfig
@@ -5,7 +5,7 @@
 config FB_GEODE
 	bool "AMD Geode family framebuffer support"
 	depends on FB && PCI && (X86_32 || (X86 && COMPILE_TEST))
-	---help---
+	help
 	  Say 'Y' here to allow you to select framebuffer drivers for
 	  the AMD Geode family of processors.
 
@@ -15,7 +15,7 @@ config FB_GEODE_LX
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Framebuffer driver for the display controller integrated into the
 	  AMD Geode LX processors.
 
@@ -30,7 +30,7 @@ config FB_GEODE_GX
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Framebuffer driver for the display controller integrated into the
 	  AMD Geode GX processors.
 
@@ -45,7 +45,7 @@ config FB_GEODE_GX1
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  Framebuffer driver for the display controller integrated into the
 	  AMD Geode GX1 processor.
 
diff --git a/drivers/video/fbdev/hitfb.c b/drivers/video/fbdev/hitfb.c
index 009e5d2aa100..bbb0f1d953cc 100644
--- a/drivers/video/fbdev/hitfb.c
+++ b/drivers/video/fbdev/hitfb.c
@@ -23,7 +23,6 @@
 
 #include <asm/machvec.h>
 #include <linux/uaccess.h>
-#include <asm/pgtable.h>
 #include <asm/io.h>
 #include <asm/hd64461.h>
 #include <cpu/dac.h>
diff --git a/drivers/video/fbdev/hpfb.c b/drivers/video/fbdev/hpfb.c
index f02be0db335e..8d418abdd767 100644
--- a/drivers/video/fbdev/hpfb.c
+++ b/drivers/video/fbdev/hpfb.c
@@ -402,7 +402,7 @@ int __init hpfb_init(void)
 	if (err)
 		return err;
 
-	err = probe_kernel_read(&i, (unsigned char *)INTFBVADDR + DIO_IDOFF, 1);
+	err = copy_from_kernel_nofault(&i, (unsigned char *)INTFBVADDR + DIO_IDOFF, 1);
 
 	if (!err && (i == DIO_ID_FBUFFER) && topcat_sid_ok(sid = DIO_SECID(INTFBVADDR))) {
 		if (!request_mem_region(INTFBPADDR, DIO_DEVSIZE, "Internal Topcat"))
diff --git a/drivers/video/fbdev/neofb.c b/drivers/video/fbdev/neofb.c
index e6ea853c1723..f5a676bfd67a 100644
--- a/drivers/video/fbdev/neofb.c
+++ b/drivers/video/fbdev/neofb.c
@@ -70,7 +70,6 @@
 
 #include <asm/io.h>
 #include <asm/irq.h>
-#include <asm/pgtable.h>
 #include <video/vga.h>
 #include <video/neomagic.h>
 
diff --git a/drivers/video/fbdev/ps3fb.c b/drivers/video/fbdev/ps3fb.c
index 834f63edf700..9df78fb77267 100644
--- a/drivers/video/fbdev/ps3fb.c
+++ b/drivers/video/fbdev/ps3fb.c
@@ -44,7 +44,7 @@
 #define GPU_CMD_BUF_SIZE			(2 * 1024 * 1024)
 #define GPU_FB_START				(64 * 1024)
 #define GPU_IOIF				(0x0d000000UL)
-#define GPU_ALIGN_UP(x)				_ALIGN_UP((x), 64)
+#define GPU_ALIGN_UP(x)				ALIGN((x), 64)
 #define GPU_MAX_LINE_LENGTH			(65536 - 64)
 
 #define GPU_INTR_STATUS_VSYNC_0			0	/* vsync on head A */
@@ -1015,7 +1015,7 @@ static int ps3fb_probe(struct ps3_system_bus_device *dev)
 	}
 #endif
 
-	max_ps3fb_size = _ALIGN_UP(GPU_IOIF, 256*1024*1024) - GPU_IOIF;
+	max_ps3fb_size = ALIGN(GPU_IOIF, 256*1024*1024) - GPU_IOIF;
 	if (ps3fb_videomemory.size > max_ps3fb_size) {
 		dev_info(&dev->core, "Limiting ps3fb mem size to %lu bytes\n",
 			 max_ps3fb_size);
diff --git a/drivers/video/fbdev/q40fb.c b/drivers/video/fbdev/q40fb.c
index 79ff14a35c85..079a2a7fb2c5 100644
--- a/drivers/video/fbdev/q40fb.c
+++ b/drivers/video/fbdev/q40fb.c
@@ -23,7 +23,6 @@
 #include <asm/q40_master.h>
 #include <linux/fb.h>
 #include <linux/module.h>
-#include <asm/pgtable.h>
 
 #define Q40_PHYS_SCREEN_ADDR 0xFE800000
 
diff --git a/drivers/video/fbdev/savage/savagefb_driver.c b/drivers/video/fbdev/savage/savagefb_driver.c
index aab312a7d9da..3c8ae87f0ea7 100644
--- a/drivers/video/fbdev/savage/savagefb_driver.c
+++ b/drivers/video/fbdev/savage/savagefb_driver.c
@@ -55,7 +55,6 @@
 
 #include <asm/io.h>
 #include <asm/irq.h>
-#include <asm/pgtable.h>
 
 #include "savagefb.h"
 
diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig
index 363af2eaf2ba..cbc1f25c79ab 100644
--- a/drivers/virt/Kconfig
+++ b/drivers/virt/Kconfig
@@ -5,7 +5,7 @@
 
 menuconfig VIRT_DRIVERS
 	bool "Virtualization drivers"
-	---help---
+	help
 	  Say Y here to get to see options for device drivers that support
 	  virtualization environments.
 
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 69a32dfc318a..5809e5f5b157 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config VIRTIO
 	tristate
-	---help---
+	help
 	  This option is selected by any driver which implements the virtio
 	  bus, such as CONFIG_VIRTIO_PCI, CONFIG_VIRTIO_MMIO, CONFIG_RPMSG
 	  or CONFIG_S390_GUEST.
@@ -16,7 +16,7 @@ config VIRTIO_PCI
 	tristate "PCI driver for virtio devices"
 	depends on PCI
 	select VIRTIO
-	---help---
+	help
 	  This driver provides support for virtio based paravirtual device
 	  drivers over PCI.  This requires that your VMM has appropriate PCI
 	  virtio backends.  Most QEMU based VMMs should support these devices
@@ -28,7 +28,7 @@ config VIRTIO_PCI_LEGACY
 	bool "Support for legacy virtio draft 0.9.X and older devices"
 	default y
 	depends on VIRTIO_PCI
-	---help---
+	help
           Virtio PCI Card 0.9.X Draft (circa 2014) and older device support.
 
 	  This option enables building a transitional driver, supporting
@@ -72,17 +72,34 @@ config VIRTIO_BALLOON
 	depends on VIRTIO
 	select MEMORY_BALLOON
 	select PAGE_REPORTING
-	---help---
+	help
 	 This driver supports increasing and decreasing the amount
 	 of memory within a KVM guest.
 
 	 If unsure, say M.
 
+config VIRTIO_MEM
+	tristate "Virtio mem driver"
+	default m
+	depends on X86_64
+	depends on VIRTIO
+	depends on MEMORY_HOTPLUG_SPARSE
+	depends on MEMORY_HOTREMOVE
+	select CONTIG_ALLOC
+	help
+	 This driver provides access to virtio-mem paravirtualized memory
+	 devices, allowing to hotplug and hotunplug memory.
+
+	 This driver was only tested under x86-64, but should theoretically
+	 work on all architectures that support memory hotplug and hotremove.
+
+	 If unsure, say M.
+
 config VIRTIO_INPUT
 	tristate "Virtio input driver"
 	depends on VIRTIO
 	depends on INPUT
-	---help---
+	help
 	 This driver supports virtio input devices such as
 	 keyboards, mice and tablets.
 
@@ -92,7 +109,7 @@ config VIRTIO_MMIO
 	tristate "Platform bus driver for memory mapped virtio devices"
 	depends on HAS_IOMEM && HAS_DMA
 	select VIRTIO
-	---help---
+	help
 	 This drivers provides support for memory mapped virtio
 	 platform device driver.
 
@@ -101,7 +118,7 @@ config VIRTIO_MMIO
 config VIRTIO_MMIO_CMDLINE_DEVICES
 	bool "Memory mapped virtio devices parameter parsing"
 	depends on VIRTIO_MMIO
-	---help---
+	help
 	 Allow virtio-mmio devices instantiation via the kernel command line
 	 or module parameters. Be aware that using incorrect parameters (base
 	 address in particular) can crash your system - you have been warned.
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index 29a1386ecc03..4d993791f2d7 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -7,3 +7,4 @@ virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o
 obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
 obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o
 obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o
+obj-$(CONFIG_VIRTIO_MEM) += virtio_mem.o
diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c
index 51086a5afdd4..1f157d2f4952 100644
--- a/drivers/virtio/virtio_balloon.c
+++ b/drivers/virtio/virtio_balloon.c
@@ -1107,11 +1107,18 @@ static int virtballoon_restore(struct virtio_device *vdev)
 
 static int virtballoon_validate(struct virtio_device *vdev)
 {
-	/* Tell the host whether we care about poisoned pages. */
+	/*
+	 * Inform the hypervisor that our pages are poisoned or
+	 * initialized. If we cannot do that then we should disable
+	 * page reporting as it could potentially change the contents
+	 * of our free pages.
+	 */
 	if (!want_init_on_free() &&
 	    (IS_ENABLED(CONFIG_PAGE_POISONING_NO_SANITY) ||
 	     !page_poisoning_enabled()))
 		__virtio_clear_bit(vdev, VIRTIO_BALLOON_F_PAGE_POISON);
+	else if (!virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON))
+		__virtio_clear_bit(vdev, VIRTIO_BALLOON_F_REPORTING);
 
 	__virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM);
 	return 0;
diff --git a/drivers/virtio/virtio_mem.c b/drivers/virtio/virtio_mem.c
new file mode 100644
index 000000000000..50c689f25045
--- /dev/null
+++ b/drivers/virtio/virtio_mem.c
@@ -0,0 +1,1965 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Virtio-mem device driver.
+ *
+ * Copyright Red Hat, Inc. 2020
+ *
+ * Author(s): David Hildenbrand <david@redhat.com>
+ */
+
+#include <linux/virtio.h>
+#include <linux/virtio_mem.h>
+#include <linux/workqueue.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/memory_hotplug.h>
+#include <linux/memory.h>
+#include <linux/hrtimer.h>
+#include <linux/crash_dump.h>
+#include <linux/mutex.h>
+#include <linux/bitmap.h>
+#include <linux/lockdep.h>
+
+#include <acpi/acpi_numa.h>
+
+static bool unplug_online = true;
+module_param(unplug_online, bool, 0644);
+MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
+
+enum virtio_mem_mb_state {
+	/* Unplugged, not added to Linux. Can be reused later. */
+	VIRTIO_MEM_MB_STATE_UNUSED = 0,
+	/* (Partially) plugged, not added to Linux. Error on add_memory(). */
+	VIRTIO_MEM_MB_STATE_PLUGGED,
+	/* Fully plugged, fully added to Linux, offline. */
+	VIRTIO_MEM_MB_STATE_OFFLINE,
+	/* Partially plugged, fully added to Linux, offline. */
+	VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL,
+	/* Fully plugged, fully added to Linux, online (!ZONE_MOVABLE). */
+	VIRTIO_MEM_MB_STATE_ONLINE,
+	/* Partially plugged, fully added to Linux, online (!ZONE_MOVABLE). */
+	VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL,
+	/*
+	 * Fully plugged, fully added to Linux, online (ZONE_MOVABLE).
+	 * We are not allowed to allocate (unplug) parts of this block that
+	 * are not movable (similar to gigantic pages). We will never allow
+	 * to online OFFLINE_PARTIAL to ZONE_MOVABLE (as they would contain
+	 * unmovable parts).
+	 */
+	VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE,
+	VIRTIO_MEM_MB_STATE_COUNT
+};
+
+struct virtio_mem {
+	struct virtio_device *vdev;
+
+	/* We might first have to unplug all memory when starting up. */
+	bool unplug_all_required;
+
+	/* Workqueue that processes the plug/unplug requests. */
+	struct work_struct wq;
+	atomic_t config_changed;
+
+	/* Virtqueue for guest->host requests. */
+	struct virtqueue *vq;
+
+	/* Wait for a host response to a guest request. */
+	wait_queue_head_t host_resp;
+
+	/* Space for one guest request and the host response. */
+	struct virtio_mem_req req;
+	struct virtio_mem_resp resp;
+
+	/* The current size of the device. */
+	uint64_t plugged_size;
+	/* The requested size of the device. */
+	uint64_t requested_size;
+
+	/* The device block size (for communicating with the device). */
+	uint64_t device_block_size;
+	/* The translated node id. NUMA_NO_NODE in case not specified. */
+	int nid;
+	/* Physical start address of the memory region. */
+	uint64_t addr;
+	/* Maximum region size in bytes. */
+	uint64_t region_size;
+
+	/* The subblock size. */
+	uint64_t subblock_size;
+	/* The number of subblocks per memory block. */
+	uint32_t nb_sb_per_mb;
+
+	/* Id of the first memory block of this device. */
+	unsigned long first_mb_id;
+	/* Id of the last memory block of this device. */
+	unsigned long last_mb_id;
+	/* Id of the last usable memory block of this device. */
+	unsigned long last_usable_mb_id;
+	/* Id of the next memory bock to prepare when needed. */
+	unsigned long next_mb_id;
+
+	/* The parent resource for all memory added via this device. */
+	struct resource *parent_resource;
+
+	/* Summary of all memory block states. */
+	unsigned long nb_mb_state[VIRTIO_MEM_MB_STATE_COUNT];
+#define VIRTIO_MEM_NB_OFFLINE_THRESHOLD		10
+
+	/*
+	 * One byte state per memory block.
+	 *
+	 * Allocated via vmalloc(). When preparing new blocks, resized
+	 * (alloc+copy+free) when needed (crossing pages with the next mb).
+	 * (when crossing pages).
+	 *
+	 * With 128MB memory blocks, we have states for 512GB of memory in one
+	 * page.
+	 */
+	uint8_t *mb_state;
+
+	/*
+	 * $nb_sb_per_mb bit per memory block. Handled similar to mb_state.
+	 *
+	 * With 4MB subblocks, we manage 128GB of memory in one page.
+	 */
+	unsigned long *sb_bitmap;
+
+	/*
+	 * Mutex that protects the nb_mb_state, mb_state, and sb_bitmap.
+	 *
+	 * When this lock is held the pointers can't change, ONLINE and
+	 * OFFLINE blocks can't change the state and no subblocks will get
+	 * plugged/unplugged.
+	 */
+	struct mutex hotplug_mutex;
+	bool hotplug_active;
+
+	/* An error occurred we cannot handle - stop processing requests. */
+	bool broken;
+
+	/* The driver is being removed. */
+	spinlock_t removal_lock;
+	bool removing;
+
+	/* Timer for retrying to plug/unplug memory. */
+	struct hrtimer retry_timer;
+	unsigned int retry_timer_ms;
+#define VIRTIO_MEM_RETRY_TIMER_MIN_MS		50000
+#define VIRTIO_MEM_RETRY_TIMER_MAX_MS		300000
+
+	/* Memory notifier (online/offline events). */
+	struct notifier_block memory_notifier;
+
+	/* Next device in the list of virtio-mem devices. */
+	struct list_head next;
+};
+
+/*
+ * We have to share a single online_page callback among all virtio-mem
+ * devices. We use RCU to iterate the list in the callback.
+ */
+static DEFINE_MUTEX(virtio_mem_mutex);
+static LIST_HEAD(virtio_mem_devices);
+
+static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
+
+/*
+ * Register a virtio-mem device so it will be considered for the online_page
+ * callback.
+ */
+static int register_virtio_mem_device(struct virtio_mem *vm)
+{
+	int rc = 0;
+
+	/* First device registers the callback. */
+	mutex_lock(&virtio_mem_mutex);
+	if (list_empty(&virtio_mem_devices))
+		rc = set_online_page_callback(&virtio_mem_online_page_cb);
+	if (!rc)
+		list_add_rcu(&vm->next, &virtio_mem_devices);
+	mutex_unlock(&virtio_mem_mutex);
+
+	return rc;
+}
+
+/*
+ * Unregister a virtio-mem device so it will no longer be considered for the
+ * online_page callback.
+ */
+static void unregister_virtio_mem_device(struct virtio_mem *vm)
+{
+	/* Last device unregisters the callback. */
+	mutex_lock(&virtio_mem_mutex);
+	list_del_rcu(&vm->next);
+	if (list_empty(&virtio_mem_devices))
+		restore_online_page_callback(&virtio_mem_online_page_cb);
+	mutex_unlock(&virtio_mem_mutex);
+
+	synchronize_rcu();
+}
+
+/*
+ * Calculate the memory block id of a given address.
+ */
+static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
+{
+	return addr / memory_block_size_bytes();
+}
+
+/*
+ * Calculate the physical start address of a given memory block id.
+ */
+static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
+{
+	return mb_id * memory_block_size_bytes();
+}
+
+/*
+ * Calculate the subblock id of a given address.
+ */
+static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
+					      unsigned long addr)
+{
+	const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
+	const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
+
+	return (addr - mb_addr) / vm->subblock_size;
+}
+
+/*
+ * Set the state of a memory block, taking care of the state counter.
+ */
+static void virtio_mem_mb_set_state(struct virtio_mem *vm, unsigned long mb_id,
+				    enum virtio_mem_mb_state state)
+{
+	const unsigned long idx = mb_id - vm->first_mb_id;
+	enum virtio_mem_mb_state old_state;
+
+	old_state = vm->mb_state[idx];
+	vm->mb_state[idx] = state;
+
+	BUG_ON(vm->nb_mb_state[old_state] == 0);
+	vm->nb_mb_state[old_state]--;
+	vm->nb_mb_state[state]++;
+}
+
+/*
+ * Get the state of a memory block.
+ */
+static enum virtio_mem_mb_state virtio_mem_mb_get_state(struct virtio_mem *vm,
+							unsigned long mb_id)
+{
+	const unsigned long idx = mb_id - vm->first_mb_id;
+
+	return vm->mb_state[idx];
+}
+
+/*
+ * Prepare the state array for the next memory block.
+ */
+static int virtio_mem_mb_state_prepare_next_mb(struct virtio_mem *vm)
+{
+	unsigned long old_bytes = vm->next_mb_id - vm->first_mb_id + 1;
+	unsigned long new_bytes = vm->next_mb_id - vm->first_mb_id + 2;
+	int old_pages = PFN_UP(old_bytes);
+	int new_pages = PFN_UP(new_bytes);
+	uint8_t *new_mb_state;
+
+	if (vm->mb_state && old_pages == new_pages)
+		return 0;
+
+	new_mb_state = vzalloc(new_pages * PAGE_SIZE);
+	if (!new_mb_state)
+		return -ENOMEM;
+
+	mutex_lock(&vm->hotplug_mutex);
+	if (vm->mb_state)
+		memcpy(new_mb_state, vm->mb_state, old_pages * PAGE_SIZE);
+	vfree(vm->mb_state);
+	vm->mb_state = new_mb_state;
+	mutex_unlock(&vm->hotplug_mutex);
+
+	return 0;
+}
+
+#define virtio_mem_for_each_mb_state(_vm, _mb_id, _state) \
+	for (_mb_id = _vm->first_mb_id; \
+	     _mb_id < _vm->next_mb_id && _vm->nb_mb_state[_state]; \
+	     _mb_id++) \
+		if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
+
+#define virtio_mem_for_each_mb_state_rev(_vm, _mb_id, _state) \
+	for (_mb_id = _vm->next_mb_id - 1; \
+	     _mb_id >= _vm->first_mb_id && _vm->nb_mb_state[_state]; \
+	     _mb_id--) \
+		if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
+
+/*
+ * Mark all selected subblocks plugged.
+ *
+ * Will not modify the state of the memory block.
+ */
+static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm,
+					 unsigned long mb_id, int sb_id,
+					 int count)
+{
+	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
+
+	__bitmap_set(vm->sb_bitmap, bit, count);
+}
+
+/*
+ * Mark all selected subblocks unplugged.
+ *
+ * Will not modify the state of the memory block.
+ */
+static void virtio_mem_mb_set_sb_unplugged(struct virtio_mem *vm,
+					   unsigned long mb_id, int sb_id,
+					   int count)
+{
+	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
+
+	__bitmap_clear(vm->sb_bitmap, bit, count);
+}
+
+/*
+ * Test if all selected subblocks are plugged.
+ */
+static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm,
+					  unsigned long mb_id, int sb_id,
+					  int count)
+{
+	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
+
+	if (count == 1)
+		return test_bit(bit, vm->sb_bitmap);
+
+	/* TODO: Helper similar to bitmap_set() */
+	return find_next_zero_bit(vm->sb_bitmap, bit + count, bit) >=
+	       bit + count;
+}
+
+/*
+ * Test if all selected subblocks are unplugged.
+ */
+static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm,
+					    unsigned long mb_id, int sb_id,
+					    int count)
+{
+	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
+
+	/* TODO: Helper similar to bitmap_set() */
+	return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count;
+}
+
+/*
+ * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is
+ * none.
+ */
+static int virtio_mem_mb_first_unplugged_sb(struct virtio_mem *vm,
+					    unsigned long mb_id)
+{
+	const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb;
+
+	return find_next_zero_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) -
+	       bit;
+}
+
+/*
+ * Prepare the subblock bitmap for the next memory block.
+ */
+static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm)
+{
+	const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id;
+	const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb;
+	const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb;
+	int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
+	int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
+	unsigned long *new_sb_bitmap, *old_sb_bitmap;
+
+	if (vm->sb_bitmap && old_pages == new_pages)
+		return 0;
+
+	new_sb_bitmap = vzalloc(new_pages * PAGE_SIZE);
+	if (!new_sb_bitmap)
+		return -ENOMEM;
+
+	mutex_lock(&vm->hotplug_mutex);
+	if (new_sb_bitmap)
+		memcpy(new_sb_bitmap, vm->sb_bitmap, old_pages * PAGE_SIZE);
+
+	old_sb_bitmap = vm->sb_bitmap;
+	vm->sb_bitmap = new_sb_bitmap;
+	mutex_unlock(&vm->hotplug_mutex);
+
+	vfree(old_sb_bitmap);
+	return 0;
+}
+
+/*
+ * Try to add a memory block to Linux. This will usually only fail
+ * if out of memory.
+ *
+ * Must not be called with the vm->hotplug_mutex held (possible deadlock with
+ * onlining code).
+ *
+ * Will not modify the state of the memory block.
+ */
+static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id)
+{
+	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
+	int nid = vm->nid;
+
+	if (nid == NUMA_NO_NODE)
+		nid = memory_add_physaddr_to_nid(addr);
+
+	dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id);
+	return add_memory(nid, addr, memory_block_size_bytes());
+}
+
+/*
+ * Try to remove a memory block from Linux. Will only fail if the memory block
+ * is not offline.
+ *
+ * Must not be called with the vm->hotplug_mutex held (possible deadlock with
+ * onlining code).
+ *
+ * Will not modify the state of the memory block.
+ */
+static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id)
+{
+	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
+	int nid = vm->nid;
+
+	if (nid == NUMA_NO_NODE)
+		nid = memory_add_physaddr_to_nid(addr);
+
+	dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id);
+	return remove_memory(nid, addr, memory_block_size_bytes());
+}
+
+/*
+ * Try to offline and remove a memory block from Linux.
+ *
+ * Must not be called with the vm->hotplug_mutex held (possible deadlock with
+ * onlining code).
+ *
+ * Will not modify the state of the memory block.
+ */
+static int virtio_mem_mb_offline_and_remove(struct virtio_mem *vm,
+					    unsigned long mb_id)
+{
+	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
+	int nid = vm->nid;
+
+	if (nid == NUMA_NO_NODE)
+		nid = memory_add_physaddr_to_nid(addr);
+
+	dev_dbg(&vm->vdev->dev, "offlining and removing memory block: %lu\n",
+		mb_id);
+	return offline_and_remove_memory(nid, addr, memory_block_size_bytes());
+}
+
+/*
+ * Trigger the workqueue so the device can perform its magic.
+ */
+static void virtio_mem_retry(struct virtio_mem *vm)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&vm->removal_lock, flags);
+	if (!vm->removing)
+		queue_work(system_freezable_wq, &vm->wq);
+	spin_unlock_irqrestore(&vm->removal_lock, flags);
+}
+
+static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
+{
+	int node = NUMA_NO_NODE;
+
+#if defined(CONFIG_ACPI_NUMA)
+	if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
+		node = pxm_to_node(node_id);
+#endif
+	return node;
+}
+
+/*
+ * Test if a virtio-mem device overlaps with the given range. Can be called
+ * from (notifier) callbacks lockless.
+ */
+static bool virtio_mem_overlaps_range(struct virtio_mem *vm,
+				      unsigned long start, unsigned long size)
+{
+	unsigned long dev_start = virtio_mem_mb_id_to_phys(vm->first_mb_id);
+	unsigned long dev_end = virtio_mem_mb_id_to_phys(vm->last_mb_id) +
+				memory_block_size_bytes();
+
+	return start < dev_end && dev_start < start + size;
+}
+
+/*
+ * Test if a virtio-mem device owns a memory block. Can be called from
+ * (notifier) callbacks lockless.
+ */
+static bool virtio_mem_owned_mb(struct virtio_mem *vm, unsigned long mb_id)
+{
+	return mb_id >= vm->first_mb_id && mb_id <= vm->last_mb_id;
+}
+
+static int virtio_mem_notify_going_online(struct virtio_mem *vm,
+					  unsigned long mb_id,
+					  enum zone_type zone)
+{
+	switch (virtio_mem_mb_get_state(vm, mb_id)) {
+	case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
+		/*
+		 * We won't allow to online a partially plugged memory block
+		 * to the MOVABLE zone - it would contain unmovable parts.
+		 */
+		if (zone == ZONE_MOVABLE) {
+			dev_warn_ratelimited(&vm->vdev->dev,
+					     "memory block has holes, MOVABLE not supported\n");
+			return NOTIFY_BAD;
+		}
+		return NOTIFY_OK;
+	case VIRTIO_MEM_MB_STATE_OFFLINE:
+		return NOTIFY_OK;
+	default:
+		break;
+	}
+	dev_warn_ratelimited(&vm->vdev->dev,
+			     "memory block onlining denied\n");
+	return NOTIFY_BAD;
+}
+
+static void virtio_mem_notify_offline(struct virtio_mem *vm,
+				      unsigned long mb_id)
+{
+	switch (virtio_mem_mb_get_state(vm, mb_id)) {
+	case VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL:
+		virtio_mem_mb_set_state(vm, mb_id,
+					VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
+		break;
+	case VIRTIO_MEM_MB_STATE_ONLINE:
+	case VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE:
+		virtio_mem_mb_set_state(vm, mb_id,
+					VIRTIO_MEM_MB_STATE_OFFLINE);
+		break;
+	default:
+		BUG();
+		break;
+	}
+
+	/*
+	 * Trigger the workqueue, maybe we can now unplug memory. Also,
+	 * when we offline and remove a memory block, this will re-trigger
+	 * us immediately - which is often nice because the removal of
+	 * the memory block (e.g., memmap) might have freed up memory
+	 * on other memory blocks we manage.
+	 */
+	virtio_mem_retry(vm);
+}
+
+static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id,
+				     enum zone_type zone)
+{
+	unsigned long nb_offline;
+
+	switch (virtio_mem_mb_get_state(vm, mb_id)) {
+	case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
+		BUG_ON(zone == ZONE_MOVABLE);
+		virtio_mem_mb_set_state(vm, mb_id,
+					VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
+		break;
+	case VIRTIO_MEM_MB_STATE_OFFLINE:
+		if (zone == ZONE_MOVABLE)
+			virtio_mem_mb_set_state(vm, mb_id,
+					    VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE);
+		else
+			virtio_mem_mb_set_state(vm, mb_id,
+						VIRTIO_MEM_MB_STATE_ONLINE);
+		break;
+	default:
+		BUG();
+		break;
+	}
+	nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
+		     vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
+
+	/* see if we can add new blocks now that we onlined one block */
+	if (nb_offline == VIRTIO_MEM_NB_OFFLINE_THRESHOLD - 1)
+		virtio_mem_retry(vm);
+}
+
+static void virtio_mem_notify_going_offline(struct virtio_mem *vm,
+					    unsigned long mb_id)
+{
+	const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
+	struct page *page;
+	unsigned long pfn;
+	int sb_id, i;
+
+	for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
+		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
+			continue;
+		/*
+		 * Drop our reference to the pages so the memory can get
+		 * offlined and add the unplugged pages to the managed
+		 * page counters (so offlining code can correctly subtract
+		 * them again).
+		 */
+		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
+			       sb_id * vm->subblock_size);
+		adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
+		for (i = 0; i < nr_pages; i++) {
+			page = pfn_to_page(pfn + i);
+			if (WARN_ON(!page_ref_dec_and_test(page)))
+				dump_page(page, "unplugged page referenced");
+		}
+	}
+}
+
+static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm,
+					     unsigned long mb_id)
+{
+	const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
+	unsigned long pfn;
+	int sb_id, i;
+
+	for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
+		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
+			continue;
+		/*
+		 * Get the reference we dropped when going offline and
+		 * subtract the unplugged pages from the managed page
+		 * counters.
+		 */
+		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
+			       sb_id * vm->subblock_size);
+		adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
+		for (i = 0; i < nr_pages; i++)
+			page_ref_inc(pfn_to_page(pfn + i));
+	}
+}
+
+/*
+ * This callback will either be called synchronously from add_memory() or
+ * asynchronously (e.g., triggered via user space). We have to be careful
+ * with locking when calling add_memory().
+ */
+static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
+					 unsigned long action, void *arg)
+{
+	struct virtio_mem *vm = container_of(nb, struct virtio_mem,
+					     memory_notifier);
+	struct memory_notify *mhp = arg;
+	const unsigned long start = PFN_PHYS(mhp->start_pfn);
+	const unsigned long size = PFN_PHYS(mhp->nr_pages);
+	const unsigned long mb_id = virtio_mem_phys_to_mb_id(start);
+	enum zone_type zone;
+	int rc = NOTIFY_OK;
+
+	if (!virtio_mem_overlaps_range(vm, start, size))
+		return NOTIFY_DONE;
+
+	/*
+	 * Memory is onlined/offlined in memory block granularity. We cannot
+	 * cross virtio-mem device boundaries and memory block boundaries. Bail
+	 * out if this ever changes.
+	 */
+	if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
+			 !IS_ALIGNED(start, memory_block_size_bytes())))
+		return NOTIFY_BAD;
+
+	/*
+	 * Avoid circular locking lockdep warnings. We lock the mutex
+	 * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
+	 * blocking_notifier_call_chain() has it's own lock, which gets unlocked
+	 * between both notifier calls and will bail out. False positive.
+	 */
+	lockdep_off();
+
+	switch (action) {
+	case MEM_GOING_OFFLINE:
+		mutex_lock(&vm->hotplug_mutex);
+		if (vm->removing) {
+			rc = notifier_from_errno(-EBUSY);
+			mutex_unlock(&vm->hotplug_mutex);
+			break;
+		}
+		vm->hotplug_active = true;
+		virtio_mem_notify_going_offline(vm, mb_id);
+		break;
+	case MEM_GOING_ONLINE:
+		mutex_lock(&vm->hotplug_mutex);
+		if (vm->removing) {
+			rc = notifier_from_errno(-EBUSY);
+			mutex_unlock(&vm->hotplug_mutex);
+			break;
+		}
+		vm->hotplug_active = true;
+		zone = page_zonenum(pfn_to_page(mhp->start_pfn));
+		rc = virtio_mem_notify_going_online(vm, mb_id, zone);
+		break;
+	case MEM_OFFLINE:
+		virtio_mem_notify_offline(vm, mb_id);
+		vm->hotplug_active = false;
+		mutex_unlock(&vm->hotplug_mutex);
+		break;
+	case MEM_ONLINE:
+		zone = page_zonenum(pfn_to_page(mhp->start_pfn));
+		virtio_mem_notify_online(vm, mb_id, zone);
+		vm->hotplug_active = false;
+		mutex_unlock(&vm->hotplug_mutex);
+		break;
+	case MEM_CANCEL_OFFLINE:
+		if (!vm->hotplug_active)
+			break;
+		virtio_mem_notify_cancel_offline(vm, mb_id);
+		vm->hotplug_active = false;
+		mutex_unlock(&vm->hotplug_mutex);
+		break;
+	case MEM_CANCEL_ONLINE:
+		if (!vm->hotplug_active)
+			break;
+		vm->hotplug_active = false;
+		mutex_unlock(&vm->hotplug_mutex);
+		break;
+	default:
+		break;
+	}
+
+	lockdep_on();
+
+	return rc;
+}
+
+/*
+ * Set a range of pages PG_offline. Remember pages that were never onlined
+ * (via generic_online_page()) using PageDirty().
+ */
+static void virtio_mem_set_fake_offline(unsigned long pfn,
+					unsigned int nr_pages, bool onlined)
+{
+	for (; nr_pages--; pfn++) {
+		struct page *page = pfn_to_page(pfn);
+
+		__SetPageOffline(page);
+		if (!onlined) {
+			SetPageDirty(page);
+			/* FIXME: remove after cleanups */
+			ClearPageReserved(page);
+		}
+	}
+}
+
+/*
+ * Clear PG_offline from a range of pages. If the pages were never onlined,
+ * (via generic_online_page()), clear PageDirty().
+ */
+static void virtio_mem_clear_fake_offline(unsigned long pfn,
+					  unsigned int nr_pages, bool onlined)
+{
+	for (; nr_pages--; pfn++) {
+		struct page *page = pfn_to_page(pfn);
+
+		__ClearPageOffline(page);
+		if (!onlined)
+			ClearPageDirty(page);
+	}
+}
+
+/*
+ * Release a range of fake-offline pages to the buddy, effectively
+ * fake-onlining them.
+ */
+static void virtio_mem_fake_online(unsigned long pfn, unsigned int nr_pages)
+{
+	const int order = MAX_ORDER - 1;
+	int i;
+
+	/*
+	 * We are always called with subblock granularity, which is at least
+	 * aligned to MAX_ORDER - 1.
+	 */
+	for (i = 0; i < nr_pages; i += 1 << order) {
+		struct page *page = pfn_to_page(pfn + i);
+
+		/*
+		 * If the page is PageDirty(), it was kept fake-offline when
+		 * onlining the memory block. Otherwise, it was allocated
+		 * using alloc_contig_range(). All pages in a subblock are
+		 * alike.
+		 */
+		if (PageDirty(page)) {
+			virtio_mem_clear_fake_offline(pfn + i, 1 << order,
+						      false);
+			generic_online_page(page, order);
+		} else {
+			virtio_mem_clear_fake_offline(pfn + i, 1 << order,
+						      true);
+			free_contig_range(pfn + i, 1 << order);
+			adjust_managed_page_count(page, 1 << order);
+		}
+	}
+}
+
+static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
+{
+	const unsigned long addr = page_to_phys(page);
+	const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
+	struct virtio_mem *vm;
+	int sb_id;
+
+	/*
+	 * We exploit here that subblocks have at least MAX_ORDER - 1
+	 * size/alignment and that this callback is is called with such a
+	 * size/alignment. So we cannot cross subblocks and therefore
+	 * also not memory blocks.
+	 */
+	rcu_read_lock();
+	list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
+		if (!virtio_mem_owned_mb(vm, mb_id))
+			continue;
+
+		sb_id = virtio_mem_phys_to_sb_id(vm, addr);
+		/*
+		 * If plugged, online the pages, otherwise, set them fake
+		 * offline (PageOffline).
+		 */
+		if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
+			generic_online_page(page, order);
+		else
+			virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
+						    false);
+		rcu_read_unlock();
+		return;
+	}
+	rcu_read_unlock();
+
+	/* not virtio-mem memory, but e.g., a DIMM. online it */
+	generic_online_page(page, order);
+}
+
+static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
+					const struct virtio_mem_req *req)
+{
+	struct scatterlist *sgs[2], sg_req, sg_resp;
+	unsigned int len;
+	int rc;
+
+	/* don't use the request residing on the stack (vaddr) */
+	vm->req = *req;
+
+	/* out: buffer for request */
+	sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
+	sgs[0] = &sg_req;
+
+	/* in: buffer for response */
+	sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
+	sgs[1] = &sg_resp;
+
+	rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
+	if (rc < 0)
+		return rc;
+
+	virtqueue_kick(vm->vq);
+
+	/* wait for a response */
+	wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
+
+	return virtio16_to_cpu(vm->vdev, vm->resp.type);
+}
+
+static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
+					uint64_t size)
+{
+	const uint64_t nb_vm_blocks = size / vm->device_block_size;
+	const struct virtio_mem_req req = {
+		.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
+		.u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
+		.u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
+	};
+
+	if (atomic_read(&vm->config_changed))
+		return -EAGAIN;
+
+	switch (virtio_mem_send_request(vm, &req)) {
+	case VIRTIO_MEM_RESP_ACK:
+		vm->plugged_size += size;
+		return 0;
+	case VIRTIO_MEM_RESP_NACK:
+		return -EAGAIN;
+	case VIRTIO_MEM_RESP_BUSY:
+		return -ETXTBSY;
+	case VIRTIO_MEM_RESP_ERROR:
+		return -EINVAL;
+	default:
+		return -ENOMEM;
+	}
+}
+
+static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
+					  uint64_t size)
+{
+	const uint64_t nb_vm_blocks = size / vm->device_block_size;
+	const struct virtio_mem_req req = {
+		.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
+		.u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
+		.u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
+	};
+
+	if (atomic_read(&vm->config_changed))
+		return -EAGAIN;
+
+	switch (virtio_mem_send_request(vm, &req)) {
+	case VIRTIO_MEM_RESP_ACK:
+		vm->plugged_size -= size;
+		return 0;
+	case VIRTIO_MEM_RESP_BUSY:
+		return -ETXTBSY;
+	case VIRTIO_MEM_RESP_ERROR:
+		return -EINVAL;
+	default:
+		return -ENOMEM;
+	}
+}
+
+static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
+{
+	const struct virtio_mem_req req = {
+		.type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
+	};
+
+	switch (virtio_mem_send_request(vm, &req)) {
+	case VIRTIO_MEM_RESP_ACK:
+		vm->unplug_all_required = false;
+		vm->plugged_size = 0;
+		/* usable region might have shrunk */
+		atomic_set(&vm->config_changed, 1);
+		return 0;
+	case VIRTIO_MEM_RESP_BUSY:
+		return -ETXTBSY;
+	default:
+		return -ENOMEM;
+	}
+}
+
+/*
+ * Plug selected subblocks. Updates the plugged state, but not the state
+ * of the memory block.
+ */
+static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
+				 int sb_id, int count)
+{
+	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
+			      sb_id * vm->subblock_size;
+	const uint64_t size = count * vm->subblock_size;
+	int rc;
+
+	dev_dbg(&vm->vdev->dev, "plugging memory block: %lu : %i - %i\n", mb_id,
+		sb_id, sb_id + count - 1);
+
+	rc = virtio_mem_send_plug_request(vm, addr, size);
+	if (!rc)
+		virtio_mem_mb_set_sb_plugged(vm, mb_id, sb_id, count);
+	return rc;
+}
+
+/*
+ * Unplug selected subblocks. Updates the plugged state, but not the state
+ * of the memory block.
+ */
+static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
+				   int sb_id, int count)
+{
+	const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
+			      sb_id * vm->subblock_size;
+	const uint64_t size = count * vm->subblock_size;
+	int rc;
+
+	dev_dbg(&vm->vdev->dev, "unplugging memory block: %lu : %i - %i\n",
+		mb_id, sb_id, sb_id + count - 1);
+
+	rc = virtio_mem_send_unplug_request(vm, addr, size);
+	if (!rc)
+		virtio_mem_mb_set_sb_unplugged(vm, mb_id, sb_id, count);
+	return rc;
+}
+
+/*
+ * Unplug the desired number of plugged subblocks of a offline or not-added
+ * memory block. Will fail if any subblock cannot get unplugged (instead of
+ * skipping it).
+ *
+ * Will not modify the state of the memory block.
+ *
+ * Note: can fail after some subblocks were unplugged.
+ */
+static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm,
+				       unsigned long mb_id, uint64_t *nb_sb)
+{
+	int sb_id, count;
+	int rc;
+
+	sb_id = vm->nb_sb_per_mb - 1;
+	while (*nb_sb) {
+		/* Find the next candidate subblock */
+		while (sb_id >= 0 &&
+		       virtio_mem_mb_test_sb_unplugged(vm, mb_id, sb_id, 1))
+			sb_id--;
+		if (sb_id < 0)
+			break;
+		/* Try to unplug multiple subblocks at a time */
+		count = 1;
+		while (count < *nb_sb && sb_id > 0 &&
+		       virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
+			count++;
+			sb_id--;
+		}
+
+		rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
+		if (rc)
+			return rc;
+		*nb_sb -= count;
+		sb_id--;
+	}
+
+	return 0;
+}
+
+/*
+ * Unplug all plugged subblocks of an offline or not-added memory block.
+ *
+ * Will not modify the state of the memory block.
+ *
+ * Note: can fail after some subblocks were unplugged.
+ */
+static int virtio_mem_mb_unplug(struct virtio_mem *vm, unsigned long mb_id)
+{
+	uint64_t nb_sb = vm->nb_sb_per_mb;
+
+	return virtio_mem_mb_unplug_any_sb(vm, mb_id, &nb_sb);
+}
+
+/*
+ * Prepare tracking data for the next memory block.
+ */
+static int virtio_mem_prepare_next_mb(struct virtio_mem *vm,
+				      unsigned long *mb_id)
+{
+	int rc;
+
+	if (vm->next_mb_id > vm->last_usable_mb_id)
+		return -ENOSPC;
+
+	/* Resize the state array if required. */
+	rc = virtio_mem_mb_state_prepare_next_mb(vm);
+	if (rc)
+		return rc;
+
+	/* Resize the subblock bitmap if required. */
+	rc = virtio_mem_sb_bitmap_prepare_next_mb(vm);
+	if (rc)
+		return rc;
+
+	vm->nb_mb_state[VIRTIO_MEM_MB_STATE_UNUSED]++;
+	*mb_id = vm->next_mb_id++;
+	return 0;
+}
+
+/*
+ * Don't add too many blocks that are not onlined yet to avoid running OOM.
+ */
+static bool virtio_mem_too_many_mb_offline(struct virtio_mem *vm)
+{
+	unsigned long nb_offline;
+
+	nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
+		     vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
+	return nb_offline >= VIRTIO_MEM_NB_OFFLINE_THRESHOLD;
+}
+
+/*
+ * Try to plug the desired number of subblocks and add the memory block
+ * to Linux.
+ *
+ * Will modify the state of the memory block.
+ */
+static int virtio_mem_mb_plug_and_add(struct virtio_mem *vm,
+				      unsigned long mb_id,
+				      uint64_t *nb_sb)
+{
+	const int count = min_t(int, *nb_sb, vm->nb_sb_per_mb);
+	int rc, rc2;
+
+	if (WARN_ON_ONCE(!count))
+		return -EINVAL;
+
+	/*
+	 * Plug the requested number of subblocks before adding it to linux,
+	 * so that onlining will directly online all plugged subblocks.
+	 */
+	rc = virtio_mem_mb_plug_sb(vm, mb_id, 0, count);
+	if (rc)
+		return rc;
+
+	/*
+	 * Mark the block properly offline before adding it to Linux,
+	 * so the memory notifiers will find the block in the right state.
+	 */
+	if (count == vm->nb_sb_per_mb)
+		virtio_mem_mb_set_state(vm, mb_id,
+					VIRTIO_MEM_MB_STATE_OFFLINE);
+	else
+		virtio_mem_mb_set_state(vm, mb_id,
+					VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
+
+	/* Add the memory block to linux - if that fails, try to unplug. */
+	rc = virtio_mem_mb_add(vm, mb_id);
+	if (rc) {
+		enum virtio_mem_mb_state new_state = VIRTIO_MEM_MB_STATE_UNUSED;
+
+		dev_err(&vm->vdev->dev,
+			"adding memory block %lu failed with %d\n", mb_id, rc);
+		rc2 = virtio_mem_mb_unplug_sb(vm, mb_id, 0, count);
+
+		/*
+		 * TODO: Linux MM does not properly clean up yet in all cases
+		 * where adding of memory failed - especially on -ENOMEM.
+		 */
+		if (rc2)
+			new_state = VIRTIO_MEM_MB_STATE_PLUGGED;
+		virtio_mem_mb_set_state(vm, mb_id, new_state);
+		return rc;
+	}
+
+	*nb_sb -= count;
+	return 0;
+}
+
+/*
+ * Try to plug the desired number of subblocks of a memory block that
+ * is already added to Linux.
+ *
+ * Will modify the state of the memory block.
+ *
+ * Note: Can fail after some subblocks were successfully plugged.
+ */
+static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id,
+				     uint64_t *nb_sb, bool online)
+{
+	unsigned long pfn, nr_pages;
+	int sb_id, count;
+	int rc;
+
+	if (WARN_ON_ONCE(!*nb_sb))
+		return -EINVAL;
+
+	while (*nb_sb) {
+		sb_id = virtio_mem_mb_first_unplugged_sb(vm, mb_id);
+		if (sb_id >= vm->nb_sb_per_mb)
+			break;
+		count = 1;
+		while (count < *nb_sb &&
+		       sb_id + count < vm->nb_sb_per_mb &&
+		       !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count,
+						      1))
+			count++;
+
+		rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count);
+		if (rc)
+			return rc;
+		*nb_sb -= count;
+		if (!online)
+			continue;
+
+		/* fake-online the pages if the memory block is online */
+		pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
+			       sb_id * vm->subblock_size);
+		nr_pages = PFN_DOWN(count * vm->subblock_size);
+		virtio_mem_fake_online(pfn, nr_pages);
+	}
+
+	if (virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
+		if (online)
+			virtio_mem_mb_set_state(vm, mb_id,
+						VIRTIO_MEM_MB_STATE_ONLINE);
+		else
+			virtio_mem_mb_set_state(vm, mb_id,
+						VIRTIO_MEM_MB_STATE_OFFLINE);
+	}
+
+	return rc;
+}
+
+/*
+ * Try to plug the requested amount of memory.
+ */
+static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
+{
+	uint64_t nb_sb = diff / vm->subblock_size;
+	unsigned long mb_id;
+	int rc;
+
+	if (!nb_sb)
+		return 0;
+
+	/* Don't race with onlining/offlining */
+	mutex_lock(&vm->hotplug_mutex);
+
+	/* Try to plug subblocks of partially plugged online blocks. */
+	virtio_mem_for_each_mb_state(vm, mb_id,
+				     VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
+		rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, true);
+		if (rc || !nb_sb)
+			goto out_unlock;
+		cond_resched();
+	}
+
+	/* Try to plug subblocks of partially plugged offline blocks. */
+	virtio_mem_for_each_mb_state(vm, mb_id,
+				     VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
+		rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, false);
+		if (rc || !nb_sb)
+			goto out_unlock;
+		cond_resched();
+	}
+
+	/*
+	 * We won't be working on online/offline memory blocks from this point,
+	 * so we can't race with memory onlining/offlining. Drop the mutex.
+	 */
+	mutex_unlock(&vm->hotplug_mutex);
+
+	/* Try to plug and add unused blocks */
+	virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED) {
+		if (virtio_mem_too_many_mb_offline(vm))
+			return -ENOSPC;
+
+		rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
+		if (rc || !nb_sb)
+			return rc;
+		cond_resched();
+	}
+
+	/* Try to prepare, plug and add new blocks */
+	while (nb_sb) {
+		if (virtio_mem_too_many_mb_offline(vm))
+			return -ENOSPC;
+
+		rc = virtio_mem_prepare_next_mb(vm, &mb_id);
+		if (rc)
+			return rc;
+		rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
+		if (rc)
+			return rc;
+		cond_resched();
+	}
+
+	return 0;
+out_unlock:
+	mutex_unlock(&vm->hotplug_mutex);
+	return rc;
+}
+
+/*
+ * Unplug the desired number of plugged subblocks of an offline memory block.
+ * Will fail if any subblock cannot get unplugged (instead of skipping it).
+ *
+ * Will modify the state of the memory block. Might temporarily drop the
+ * hotplug_mutex.
+ *
+ * Note: Can fail after some subblocks were successfully unplugged.
+ */
+static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm,
+					       unsigned long mb_id,
+					       uint64_t *nb_sb)
+{
+	int rc;
+
+	rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb);
+
+	/* some subblocks might have been unplugged even on failure */
+	if (!virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))
+		virtio_mem_mb_set_state(vm, mb_id,
+					VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
+	if (rc)
+		return rc;
+
+	if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
+		/*
+		 * Remove the block from Linux - this should never fail.
+		 * Hinder the block from getting onlined by marking it
+		 * unplugged. Temporarily drop the mutex, so
+		 * any pending GOING_ONLINE requests can be serviced/rejected.
+		 */
+		virtio_mem_mb_set_state(vm, mb_id,
+					VIRTIO_MEM_MB_STATE_UNUSED);
+
+		mutex_unlock(&vm->hotplug_mutex);
+		rc = virtio_mem_mb_remove(vm, mb_id);
+		BUG_ON(rc);
+		mutex_lock(&vm->hotplug_mutex);
+	}
+	return 0;
+}
+
+/*
+ * Unplug the given plugged subblocks of an online memory block.
+ *
+ * Will modify the state of the memory block.
+ */
+static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm,
+					  unsigned long mb_id, int sb_id,
+					  int count)
+{
+	const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count;
+	unsigned long start_pfn;
+	int rc;
+
+	start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
+			     sb_id * vm->subblock_size);
+	rc = alloc_contig_range(start_pfn, start_pfn + nr_pages,
+				MIGRATE_MOVABLE, GFP_KERNEL);
+	if (rc == -ENOMEM)
+		/* whoops, out of memory */
+		return rc;
+	if (rc)
+		return -EBUSY;
+
+	/* Mark it as fake-offline before unplugging it */
+	virtio_mem_set_fake_offline(start_pfn, nr_pages, true);
+	adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages);
+
+	/* Try to unplug the allocated memory */
+	rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
+	if (rc) {
+		/* Return the memory to the buddy. */
+		virtio_mem_fake_online(start_pfn, nr_pages);
+		return rc;
+	}
+
+	virtio_mem_mb_set_state(vm, mb_id,
+				VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
+	return 0;
+}
+
+/*
+ * Unplug the desired number of plugged subblocks of an online memory block.
+ * Will skip subblock that are busy.
+ *
+ * Will modify the state of the memory block. Might temporarily drop the
+ * hotplug_mutex.
+ *
+ * Note: Can fail after some subblocks were successfully unplugged. Can
+ *       return 0 even if subblocks were busy and could not get unplugged.
+ */
+static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm,
+					      unsigned long mb_id,
+					      uint64_t *nb_sb)
+{
+	int rc, sb_id;
+
+	/* If possible, try to unplug the complete block in one shot. */
+	if (*nb_sb >= vm->nb_sb_per_mb &&
+	    virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
+		rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0,
+						    vm->nb_sb_per_mb);
+		if (!rc) {
+			*nb_sb -= vm->nb_sb_per_mb;
+			goto unplugged;
+		} else if (rc != -EBUSY)
+			return rc;
+	}
+
+	/* Fallback to single subblocks. */
+	for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
+		/* Find the next candidate subblock */
+		while (sb_id >= 0 &&
+		       !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
+			sb_id--;
+		if (sb_id < 0)
+			break;
+
+		rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, sb_id, 1);
+		if (rc == -EBUSY)
+			continue;
+		else if (rc)
+			return rc;
+		*nb_sb -= 1;
+	}
+
+unplugged:
+	/*
+	 * Once all subblocks of a memory block were unplugged, offline and
+	 * remove it. This will usually not fail, as no memory is in use
+	 * anymore - however some other notifiers might NACK the request.
+	 */
+	if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
+		mutex_unlock(&vm->hotplug_mutex);
+		rc = virtio_mem_mb_offline_and_remove(vm, mb_id);
+		mutex_lock(&vm->hotplug_mutex);
+		if (!rc)
+			virtio_mem_mb_set_state(vm, mb_id,
+						VIRTIO_MEM_MB_STATE_UNUSED);
+	}
+
+	return 0;
+}
+
+/*
+ * Try to unplug the requested amount of memory.
+ */
+static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
+{
+	uint64_t nb_sb = diff / vm->subblock_size;
+	unsigned long mb_id;
+	int rc;
+
+	if (!nb_sb)
+		return 0;
+
+	/*
+	 * We'll drop the mutex a couple of times when it is safe to do so.
+	 * This might result in some blocks switching the state (online/offline)
+	 * and we could miss them in this run - we will retry again later.
+	 */
+	mutex_lock(&vm->hotplug_mutex);
+
+	/* Try to unplug subblocks of partially plugged offline blocks. */
+	virtio_mem_for_each_mb_state_rev(vm, mb_id,
+					 VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
+		rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
+							 &nb_sb);
+		if (rc || !nb_sb)
+			goto out_unlock;
+		cond_resched();
+	}
+
+	/* Try to unplug subblocks of plugged offline blocks. */
+	virtio_mem_for_each_mb_state_rev(vm, mb_id,
+					 VIRTIO_MEM_MB_STATE_OFFLINE) {
+		rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
+							 &nb_sb);
+		if (rc || !nb_sb)
+			goto out_unlock;
+		cond_resched();
+	}
+
+	if (!unplug_online) {
+		mutex_unlock(&vm->hotplug_mutex);
+		return 0;
+	}
+
+	/* Try to unplug subblocks of partially plugged online blocks. */
+	virtio_mem_for_each_mb_state_rev(vm, mb_id,
+					 VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
+		rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
+							&nb_sb);
+		if (rc || !nb_sb)
+			goto out_unlock;
+		mutex_unlock(&vm->hotplug_mutex);
+		cond_resched();
+		mutex_lock(&vm->hotplug_mutex);
+	}
+
+	/* Try to unplug subblocks of plugged online blocks. */
+	virtio_mem_for_each_mb_state_rev(vm, mb_id,
+					 VIRTIO_MEM_MB_STATE_ONLINE) {
+		rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
+							&nb_sb);
+		if (rc || !nb_sb)
+			goto out_unlock;
+		mutex_unlock(&vm->hotplug_mutex);
+		cond_resched();
+		mutex_lock(&vm->hotplug_mutex);
+	}
+
+	mutex_unlock(&vm->hotplug_mutex);
+	return nb_sb ? -EBUSY : 0;
+out_unlock:
+	mutex_unlock(&vm->hotplug_mutex);
+	return rc;
+}
+
+/*
+ * Try to unplug all blocks that couldn't be unplugged before, for example,
+ * because the hypervisor was busy.
+ */
+static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
+{
+	unsigned long mb_id;
+	int rc;
+
+	virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_PLUGGED) {
+		rc = virtio_mem_mb_unplug(vm, mb_id);
+		if (rc)
+			return rc;
+		virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
+	}
+
+	return 0;
+}
+
+/*
+ * Update all parts of the config that could have changed.
+ */
+static void virtio_mem_refresh_config(struct virtio_mem *vm)
+{
+	const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
+	uint64_t new_plugged_size, usable_region_size, end_addr;
+
+	/* the plugged_size is just a reflection of what _we_ did previously */
+	virtio_cread(vm->vdev, struct virtio_mem_config, plugged_size,
+		     &new_plugged_size);
+	if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
+		vm->plugged_size = new_plugged_size;
+
+	/* calculate the last usable memory block id */
+	virtio_cread(vm->vdev, struct virtio_mem_config,
+		     usable_region_size, &usable_region_size);
+	end_addr = vm->addr + usable_region_size;
+	end_addr = min(end_addr, phys_limit);
+	vm->last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr) - 1;
+
+	/* see if there is a request to change the size */
+	virtio_cread(vm->vdev, struct virtio_mem_config, requested_size,
+		     &vm->requested_size);
+
+	dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
+	dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
+}
+
+/*
+ * Workqueue function for handling plug/unplug requests and config updates.
+ */
+static void virtio_mem_run_wq(struct work_struct *work)
+{
+	struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
+	uint64_t diff;
+	int rc;
+
+	hrtimer_cancel(&vm->retry_timer);
+
+	if (vm->broken)
+		return;
+
+retry:
+	rc = 0;
+
+	/* Make sure we start with a clean state if there are leftovers. */
+	if (unlikely(vm->unplug_all_required))
+		rc = virtio_mem_send_unplug_all_request(vm);
+
+	if (atomic_read(&vm->config_changed)) {
+		atomic_set(&vm->config_changed, 0);
+		virtio_mem_refresh_config(vm);
+	}
+
+	/* Unplug any leftovers from previous runs */
+	if (!rc)
+		rc = virtio_mem_unplug_pending_mb(vm);
+
+	if (!rc && vm->requested_size != vm->plugged_size) {
+		if (vm->requested_size > vm->plugged_size) {
+			diff = vm->requested_size - vm->plugged_size;
+			rc = virtio_mem_plug_request(vm, diff);
+		} else {
+			diff = vm->plugged_size - vm->requested_size;
+			rc = virtio_mem_unplug_request(vm, diff);
+		}
+	}
+
+	switch (rc) {
+	case 0:
+		vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
+		break;
+	case -ENOSPC:
+		/*
+		 * We cannot add any more memory (alignment, physical limit)
+		 * or we have too many offline memory blocks.
+		 */
+		break;
+	case -ETXTBSY:
+		/*
+		 * The hypervisor cannot process our request right now
+		 * (e.g., out of memory, migrating);
+		 */
+	case -EBUSY:
+		/*
+		 * We cannot free up any memory to unplug it (all plugged memory
+		 * is busy).
+		 */
+	case -ENOMEM:
+		/* Out of memory, try again later. */
+		hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
+			      HRTIMER_MODE_REL);
+		break;
+	case -EAGAIN:
+		/* Retry immediately (e.g., the config changed). */
+		goto retry;
+	default:
+		/* Unknown error, mark as broken */
+		dev_err(&vm->vdev->dev,
+			"unknown error, marking device broken: %d\n", rc);
+		vm->broken = true;
+	}
+}
+
+static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
+{
+	struct virtio_mem *vm = container_of(timer, struct virtio_mem,
+					     retry_timer);
+
+	virtio_mem_retry(vm);
+	vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
+				   VIRTIO_MEM_RETRY_TIMER_MAX_MS);
+	return HRTIMER_NORESTART;
+}
+
+static void virtio_mem_handle_response(struct virtqueue *vq)
+{
+	struct virtio_mem *vm = vq->vdev->priv;
+
+	wake_up(&vm->host_resp);
+}
+
+static int virtio_mem_init_vq(struct virtio_mem *vm)
+{
+	struct virtqueue *vq;
+
+	vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
+				   "guest-request");
+	if (IS_ERR(vq))
+		return PTR_ERR(vq);
+	vm->vq = vq;
+
+	return 0;
+}
+
+static int virtio_mem_init(struct virtio_mem *vm)
+{
+	const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
+	uint16_t node_id;
+
+	if (!vm->vdev->config->get) {
+		dev_err(&vm->vdev->dev, "config access disabled\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * We don't want to (un)plug or reuse any memory when in kdump. The
+	 * memory is still accessible (but not mapped).
+	 */
+	if (is_kdump_kernel()) {
+		dev_warn(&vm->vdev->dev, "disabled in kdump kernel\n");
+		return -EBUSY;
+	}
+
+	/* Fetch all properties that can't change. */
+	virtio_cread(vm->vdev, struct virtio_mem_config, plugged_size,
+		     &vm->plugged_size);
+	virtio_cread(vm->vdev, struct virtio_mem_config, block_size,
+		     &vm->device_block_size);
+	virtio_cread(vm->vdev, struct virtio_mem_config, node_id,
+		     &node_id);
+	vm->nid = virtio_mem_translate_node_id(vm, node_id);
+	virtio_cread(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
+	virtio_cread(vm->vdev, struct virtio_mem_config, region_size,
+		     &vm->region_size);
+
+	/*
+	 * We always hotplug memory in memory block granularity. This way,
+	 * we have to wait for exactly one memory block to online.
+	 */
+	if (vm->device_block_size > memory_block_size_bytes()) {
+		dev_err(&vm->vdev->dev,
+			"The block size is not supported (too big).\n");
+		return -EINVAL;
+	}
+
+	/* bad device setup - warn only */
+	if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
+		dev_warn(&vm->vdev->dev,
+			 "The alignment of the physical start address can make some memory unusable.\n");
+	if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
+		dev_warn(&vm->vdev->dev,
+			 "The alignment of the physical end address can make some memory unusable.\n");
+	if (vm->addr + vm->region_size > phys_limit)
+		dev_warn(&vm->vdev->dev,
+			 "Some memory is not addressable. This can make some memory unusable.\n");
+
+	/*
+	 * Calculate the subblock size:
+	 * - At least MAX_ORDER - 1 / pageblock_order.
+	 * - At least the device block size.
+	 * In the worst case, a single subblock per memory block.
+	 */
+	vm->subblock_size = PAGE_SIZE * 1ul << max_t(uint32_t, MAX_ORDER - 1,
+						     pageblock_order);
+	vm->subblock_size = max_t(uint64_t, vm->device_block_size,
+				  vm->subblock_size);
+	vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size;
+
+	/* Round up to the next full memory block */
+	vm->first_mb_id = virtio_mem_phys_to_mb_id(vm->addr - 1 +
+						   memory_block_size_bytes());
+	vm->next_mb_id = vm->first_mb_id;
+	vm->last_mb_id = virtio_mem_phys_to_mb_id(vm->addr +
+			 vm->region_size) - 1;
+
+	dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
+	dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
+	dev_info(&vm->vdev->dev, "device block size: 0x%llx",
+		 (unsigned long long)vm->device_block_size);
+	dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
+		 memory_block_size_bytes());
+	dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
+		 (unsigned long long)vm->subblock_size);
+	if (vm->nid != NUMA_NO_NODE)
+		dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
+
+	return 0;
+}
+
+static int virtio_mem_create_resource(struct virtio_mem *vm)
+{
+	/*
+	 * When force-unloading the driver and removing the device, we
+	 * could have a garbage pointer. Duplicate the string.
+	 */
+	const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
+
+	if (!name)
+		return -ENOMEM;
+
+	vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
+						   name, IORESOURCE_SYSTEM_RAM);
+	if (!vm->parent_resource) {
+		kfree(name);
+		dev_warn(&vm->vdev->dev, "could not reserve device region\n");
+		dev_info(&vm->vdev->dev,
+			 "reloading the driver is not supported\n");
+		return -EBUSY;
+	}
+
+	/* The memory is not actually busy - make add_memory() work. */
+	vm->parent_resource->flags &= ~IORESOURCE_BUSY;
+	return 0;
+}
+
+static void virtio_mem_delete_resource(struct virtio_mem *vm)
+{
+	const char *name;
+
+	if (!vm->parent_resource)
+		return;
+
+	name = vm->parent_resource->name;
+	release_resource(vm->parent_resource);
+	kfree(vm->parent_resource);
+	kfree(name);
+	vm->parent_resource = NULL;
+}
+
+static int virtio_mem_probe(struct virtio_device *vdev)
+{
+	struct virtio_mem *vm;
+	int rc;
+
+	BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
+	BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
+
+	vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
+	if (!vm)
+		return -ENOMEM;
+
+	init_waitqueue_head(&vm->host_resp);
+	vm->vdev = vdev;
+	INIT_WORK(&vm->wq, virtio_mem_run_wq);
+	mutex_init(&vm->hotplug_mutex);
+	INIT_LIST_HEAD(&vm->next);
+	spin_lock_init(&vm->removal_lock);
+	hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	vm->retry_timer.function = virtio_mem_timer_expired;
+	vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
+
+	/* register the virtqueue */
+	rc = virtio_mem_init_vq(vm);
+	if (rc)
+		goto out_free_vm;
+
+	/* initialize the device by querying the config */
+	rc = virtio_mem_init(vm);
+	if (rc)
+		goto out_del_vq;
+
+	/* create the parent resource for all memory */
+	rc = virtio_mem_create_resource(vm);
+	if (rc)
+		goto out_del_vq;
+
+	/*
+	 * If we still have memory plugged, we have to unplug all memory first.
+	 * Registering our parent resource makes sure that this memory isn't
+	 * actually in use (e.g., trying to reload the driver).
+	 */
+	if (vm->plugged_size) {
+		vm->unplug_all_required = 1;
+		dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
+	}
+
+	/* register callbacks */
+	vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
+	rc = register_memory_notifier(&vm->memory_notifier);
+	if (rc)
+		goto out_del_resource;
+	rc = register_virtio_mem_device(vm);
+	if (rc)
+		goto out_unreg_mem;
+
+	virtio_device_ready(vdev);
+
+	/* trigger a config update to start processing the requested_size */
+	atomic_set(&vm->config_changed, 1);
+	queue_work(system_freezable_wq, &vm->wq);
+
+	return 0;
+out_unreg_mem:
+	unregister_memory_notifier(&vm->memory_notifier);
+out_del_resource:
+	virtio_mem_delete_resource(vm);
+out_del_vq:
+	vdev->config->del_vqs(vdev);
+out_free_vm:
+	kfree(vm);
+	vdev->priv = NULL;
+
+	return rc;
+}
+
+static void virtio_mem_remove(struct virtio_device *vdev)
+{
+	struct virtio_mem *vm = vdev->priv;
+	unsigned long mb_id;
+	int rc;
+
+	/*
+	 * Make sure the workqueue won't be triggered anymore and no memory
+	 * blocks can be onlined/offlined until we're finished here.
+	 */
+	mutex_lock(&vm->hotplug_mutex);
+	spin_lock_irq(&vm->removal_lock);
+	vm->removing = true;
+	spin_unlock_irq(&vm->removal_lock);
+	mutex_unlock(&vm->hotplug_mutex);
+
+	/* wait until the workqueue stopped */
+	cancel_work_sync(&vm->wq);
+	hrtimer_cancel(&vm->retry_timer);
+
+	/*
+	 * After we unregistered our callbacks, user space can online partially
+	 * plugged offline blocks. Make sure to remove them.
+	 */
+	virtio_mem_for_each_mb_state(vm, mb_id,
+				     VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
+		rc = virtio_mem_mb_remove(vm, mb_id);
+		BUG_ON(rc);
+		virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
+	}
+	/*
+	 * After we unregistered our callbacks, user space can no longer
+	 * offline partially plugged online memory blocks. No need to worry
+	 * about them.
+	 */
+
+	/* unregister callbacks */
+	unregister_virtio_mem_device(vm);
+	unregister_memory_notifier(&vm->memory_notifier);
+
+	/*
+	 * There is no way we could reliably remove all memory we have added to
+	 * the system. And there is no way to stop the driver/device from going
+	 * away. Warn at least.
+	 */
+	if (vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] ||
+	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL] ||
+	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE] ||
+	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL] ||
+	    vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_MOVABLE])
+		dev_warn(&vdev->dev, "device still has system memory added\n");
+	else
+		virtio_mem_delete_resource(vm);
+
+	/* remove all tracking data - no locking needed */
+	vfree(vm->mb_state);
+	vfree(vm->sb_bitmap);
+
+	/* reset the device and cleanup the queues */
+	vdev->config->reset(vdev);
+	vdev->config->del_vqs(vdev);
+
+	kfree(vm);
+	vdev->priv = NULL;
+}
+
+static void virtio_mem_config_changed(struct virtio_device *vdev)
+{
+	struct virtio_mem *vm = vdev->priv;
+
+	atomic_set(&vm->config_changed, 1);
+	virtio_mem_retry(vm);
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int virtio_mem_freeze(struct virtio_device *vdev)
+{
+	/*
+	 * When restarting the VM, all memory is usually unplugged. Don't
+	 * allow to suspend/hibernate.
+	 */
+	dev_err(&vdev->dev, "save/restore not supported.\n");
+	return -EPERM;
+}
+
+static int virtio_mem_restore(struct virtio_device *vdev)
+{
+	return -EPERM;
+}
+#endif
+
+static unsigned int virtio_mem_features[] = {
+#if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
+	VIRTIO_MEM_F_ACPI_PXM,
+#endif
+};
+
+static struct virtio_device_id virtio_mem_id_table[] = {
+	{ VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
+	{ 0 },
+};
+
+static struct virtio_driver virtio_mem_driver = {
+	.feature_table = virtio_mem_features,
+	.feature_table_size = ARRAY_SIZE(virtio_mem_features),
+	.driver.name = KBUILD_MODNAME,
+	.driver.owner = THIS_MODULE,
+	.id_table = virtio_mem_id_table,
+	.probe = virtio_mem_probe,
+	.remove = virtio_mem_remove,
+	.config_changed = virtio_mem_config_changed,
+#ifdef CONFIG_PM_SLEEP
+	.freeze	=	virtio_mem_freeze,
+	.restore =	virtio_mem_restore,
+#endif
+};
+
+module_virtio_driver(virtio_mem_driver);
+MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
+MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
+MODULE_DESCRIPTION("Virtio-mem driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
index 97d5725fd9a2..9d16aaffca9d 100644
--- a/drivers/virtio/virtio_mmio.c
+++ b/drivers/virtio/virtio_mmio.c
@@ -466,10 +466,8 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 	int irq = platform_get_irq(vm_dev->pdev, 0);
 	int i, err, queue_idx = 0;
 
-	if (irq < 0) {
-		dev_err(&vdev->dev, "Cannot get IRQ resource\n");
+	if (irq < 0)
 		return irq;
-	}
 
 	err = request_irq(irq, vm_interrupt, IRQF_SHARED,
 			dev_name(&vdev->dev), vm_dev);
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index 7abcc50838b8..db93cedd262f 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -16,6 +16,7 @@
 
 #include <linux/delay.h>
 #define VIRTIO_PCI_NO_LEGACY
+#define VIRTIO_RING_NO_LEGACY
 #include "virtio_pci_common.h"
 
 /*
diff --git a/drivers/visorbus/Kconfig b/drivers/visorbus/Kconfig
index d702c44fb5d6..fa947a79b5cd 100644
--- a/drivers/visorbus/Kconfig
+++ b/drivers/visorbus/Kconfig
@@ -6,7 +6,7 @@
 config UNISYS_VISORBUS
 	tristate "Unisys visorbus driver"
 	depends on X86_64 && ACPI
-	---help---
+	help
 	The visorbus driver is a virtualized bus for the Unisys s-Par firmware.
 	Virtualized devices allow Linux guests on a system to share disks and
 	network cards that do not have SR-IOV support, and to be accessed using
diff --git a/drivers/visorbus/controlvmchannel.h b/drivers/visorbus/controlvmchannel.h
index 8c57562a070a..c87213554427 100644
--- a/drivers/visorbus/controlvmchannel.h
+++ b/drivers/visorbus/controlvmchannel.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
diff --git a/drivers/visorbus/vbuschannel.h b/drivers/visorbus/vbuschannel.h
index b1dce26166bf..4aaf6564eb9f 100644
--- a/drivers/visorbus/vbuschannel.h
+++ b/drivers/visorbus/vbuschannel.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
diff --git a/drivers/visorbus/visorbus_private.h b/drivers/visorbus/visorbus_private.h
index 366380b7f8d9..6956de605827 100644
--- a/drivers/visorbus/visorbus_private.h
+++ b/drivers/visorbus/visorbus_private.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
diff --git a/drivers/vme/Kconfig b/drivers/vme/Kconfig
index 7cb9ec6b6b41..936392ca3c8c 100644
--- a/drivers/vme/Kconfig
+++ b/drivers/vme/Kconfig
@@ -6,7 +6,7 @@
 menuconfig VME_BUS
 	bool "VME bridge support"
 	depends on PCI
-	---help---
+	help
 	  If you say Y here you get support for the VME bridge Framework.
 
 if VME_BUS
diff --git a/drivers/w1/Kconfig b/drivers/w1/Kconfig
index 3e7ad7b232fe..0d6f46702efb 100644
--- a/drivers/w1/Kconfig
+++ b/drivers/w1/Kconfig
@@ -2,7 +2,7 @@
 menuconfig W1
 	tristate "Dallas's 1-wire support"
 	depends on HAS_IOMEM
-	---help---
+	help
 	  Dallas' 1-wire bus is useful to connect slow 1-pin devices
 	  such as iButtons and thermal sensors.
 
@@ -17,7 +17,7 @@ config W1_CON
 	depends on CONNECTOR
 	bool "Userspace communication over connector"
 	default y
-	---help---
+	help
 	  This allows to communicate with userspace using connector. For more
 	  information see <file:Documentation/driver-api/connector.rst>.
 	  There are three types of messages between w1 core and userspace:
diff --git a/drivers/w1/masters/omap_hdq.c b/drivers/w1/masters/omap_hdq.c
index aa09f8527776..bf2ec59c1f9d 100644
--- a/drivers/w1/masters/omap_hdq.c
+++ b/drivers/w1/masters/omap_hdq.c
@@ -54,10 +54,10 @@ MODULE_PARM_DESC(w1_id, "1-wire id for the slave detection in HDQ mode");
 struct hdq_data {
 	struct device		*dev;
 	void __iomem		*hdq_base;
-	/* lock status update */
+	/* lock read/write/break operations */
 	struct  mutex		hdq_mutex;
+	/* interrupt status and a lock for it */
 	u8			hdq_irqstatus;
-	/* device lock */
 	spinlock_t		hdq_spinlock;
 	/* mode: 0-HDQ 1-W1 */
 	int                     mode;
@@ -120,13 +120,18 @@ static int hdq_wait_for_flag(struct hdq_data *hdq_data, u32 offset,
 }
 
 /* Clear saved irqstatus after using an interrupt */
-static void hdq_reset_irqstatus(struct hdq_data *hdq_data)
+static u8 hdq_reset_irqstatus(struct hdq_data *hdq_data, u8 bits)
 {
 	unsigned long irqflags;
+	u8 status;
 
 	spin_lock_irqsave(&hdq_data->hdq_spinlock, irqflags);
-	hdq_data->hdq_irqstatus = 0;
+	status = hdq_data->hdq_irqstatus;
+	/* this is a read-modify-write */
+	hdq_data->hdq_irqstatus &= ~bits;
 	spin_unlock_irqrestore(&hdq_data->hdq_spinlock, irqflags);
+
+	return status;
 }
 
 /* write out a byte and fill *status with HDQ_INT_STATUS */
@@ -135,6 +140,16 @@ static int hdq_write_byte(struct hdq_data *hdq_data, u8 val, u8 *status)
 	int ret;
 	u8 tmp_status;
 
+	ret = mutex_lock_interruptible(&hdq_data->hdq_mutex);
+	if (ret < 0) {
+		ret = -EINTR;
+		goto rtn;
+	}
+
+	if (hdq_data->hdq_irqstatus)
+		dev_err(hdq_data->dev, "TX irqstatus not cleared (%02x)\n",
+			hdq_data->hdq_irqstatus);
+
 	*status = 0;
 
 	hdq_reg_out(hdq_data, OMAP_HDQ_TX_DATA, val);
@@ -144,18 +159,19 @@ static int hdq_write_byte(struct hdq_data *hdq_data, u8 val, u8 *status)
 		OMAP_HDQ_CTRL_STATUS_DIR | OMAP_HDQ_CTRL_STATUS_GO);
 	/* wait for the TXCOMPLETE bit */
 	ret = wait_event_timeout(hdq_wait_queue,
-		hdq_data->hdq_irqstatus, OMAP_HDQ_TIMEOUT);
+		(hdq_data->hdq_irqstatus & OMAP_HDQ_INT_STATUS_TXCOMPLETE),
+		OMAP_HDQ_TIMEOUT);
+	*status = hdq_reset_irqstatus(hdq_data, OMAP_HDQ_INT_STATUS_TXCOMPLETE);
 	if (ret == 0) {
 		dev_dbg(hdq_data->dev, "TX wait elapsed\n");
 		ret = -ETIMEDOUT;
 		goto out;
 	}
 
-	*status = hdq_data->hdq_irqstatus;
 	/* check irqstatus */
 	if (!(*status & OMAP_HDQ_INT_STATUS_TXCOMPLETE)) {
 		dev_dbg(hdq_data->dev, "timeout waiting for"
-			" TXCOMPLETE/RXCOMPLETE, %x", *status);
+			" TXCOMPLETE/RXCOMPLETE, %x\n", *status);
 		ret = -ETIMEDOUT;
 		goto out;
 	}
@@ -166,11 +182,12 @@ static int hdq_write_byte(struct hdq_data *hdq_data, u8 val, u8 *status)
 			OMAP_HDQ_FLAG_CLEAR, &tmp_status);
 	if (ret) {
 		dev_dbg(hdq_data->dev, "timeout waiting GO bit"
-			" return to zero, %x", tmp_status);
+			" return to zero, %x\n", tmp_status);
 	}
 
 out:
-	hdq_reset_irqstatus(hdq_data);
+	mutex_unlock(&hdq_data->hdq_mutex);
+rtn:
 	return ret;
 }
 
@@ -181,9 +198,9 @@ static irqreturn_t hdq_isr(int irq, void *_hdq)
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&hdq_data->hdq_spinlock, irqflags);
-	hdq_data->hdq_irqstatus = hdq_reg_in(hdq_data, OMAP_HDQ_INT_STATUS);
+	hdq_data->hdq_irqstatus |= hdq_reg_in(hdq_data, OMAP_HDQ_INT_STATUS);
 	spin_unlock_irqrestore(&hdq_data->hdq_spinlock, irqflags);
-	dev_dbg(hdq_data->dev, "hdq_isr: %x", hdq_data->hdq_irqstatus);
+	dev_dbg(hdq_data->dev, "hdq_isr: %x\n", hdq_data->hdq_irqstatus);
 
 	if (hdq_data->hdq_irqstatus &
 		(OMAP_HDQ_INT_STATUS_TXCOMPLETE | OMAP_HDQ_INT_STATUS_RXCOMPLETE
@@ -230,6 +247,10 @@ static int omap_hdq_break(struct hdq_data *hdq_data)
 		goto rtn;
 	}
 
+	if (hdq_data->hdq_irqstatus)
+		dev_err(hdq_data->dev, "break irqstatus not cleared (%02x)\n",
+			hdq_data->hdq_irqstatus);
+
 	/* set the INIT and GO bit */
 	hdq_reg_merge(hdq_data, OMAP_HDQ_CTRL_STATUS,
 		OMAP_HDQ_CTRL_STATUS_INITIALIZATION | OMAP_HDQ_CTRL_STATUS_GO,
@@ -238,18 +259,19 @@ static int omap_hdq_break(struct hdq_data *hdq_data)
 
 	/* wait for the TIMEOUT bit */
 	ret = wait_event_timeout(hdq_wait_queue,
-		hdq_data->hdq_irqstatus, OMAP_HDQ_TIMEOUT);
+		(hdq_data->hdq_irqstatus & OMAP_HDQ_INT_STATUS_TIMEOUT),
+		OMAP_HDQ_TIMEOUT);
+	tmp_status = hdq_reset_irqstatus(hdq_data, OMAP_HDQ_INT_STATUS_TIMEOUT);
 	if (ret == 0) {
 		dev_dbg(hdq_data->dev, "break wait elapsed\n");
 		ret = -EINTR;
 		goto out;
 	}
 
-	tmp_status = hdq_data->hdq_irqstatus;
 	/* check irqstatus */
 	if (!(tmp_status & OMAP_HDQ_INT_STATUS_TIMEOUT)) {
-		dev_dbg(hdq_data->dev, "timeout waiting for TIMEOUT, %x",
-				tmp_status);
+		dev_dbg(hdq_data->dev, "timeout waiting for TIMEOUT, %x\n",
+			tmp_status);
 		ret = -ETIMEDOUT;
 		goto out;
 	}
@@ -275,10 +297,9 @@ static int omap_hdq_break(struct hdq_data *hdq_data)
 			&tmp_status);
 	if (ret)
 		dev_dbg(hdq_data->dev, "timeout waiting INIT&GO bits"
-			" return to zero, %x", tmp_status);
+			" return to zero, %x\n", tmp_status);
 
 out:
-	hdq_reset_irqstatus(hdq_data);
 	mutex_unlock(&hdq_data->hdq_mutex);
 rtn:
 	return ret;
@@ -309,12 +330,15 @@ static int hdq_read_byte(struct hdq_data *hdq_data, u8 *val)
 		 */
 		wait_event_timeout(hdq_wait_queue,
 				   (hdq_data->hdq_irqstatus
-				    & OMAP_HDQ_INT_STATUS_RXCOMPLETE),
+				    & (OMAP_HDQ_INT_STATUS_RXCOMPLETE |
+				       OMAP_HDQ_INT_STATUS_TIMEOUT)),
 				   OMAP_HDQ_TIMEOUT);
-
+		status = hdq_reset_irqstatus(hdq_data,
+					     OMAP_HDQ_INT_STATUS_RXCOMPLETE |
+					     OMAP_HDQ_INT_STATUS_TIMEOUT);
 		hdq_reg_merge(hdq_data, OMAP_HDQ_CTRL_STATUS, 0,
 			OMAP_HDQ_CTRL_STATUS_DIR);
-		status = hdq_data->hdq_irqstatus;
+
 		/* check irqstatus */
 		if (!(status & OMAP_HDQ_INT_STATUS_RXCOMPLETE)) {
 			dev_dbg(hdq_data->dev, "timeout waiting for"
@@ -322,11 +346,12 @@ static int hdq_read_byte(struct hdq_data *hdq_data, u8 *val)
 			ret = -ETIMEDOUT;
 			goto out;
 		}
+	} else { /* interrupt had occurred before hdq_read_byte was called */
+		hdq_reset_irqstatus(hdq_data, OMAP_HDQ_INT_STATUS_RXCOMPLETE);
 	}
 	/* the data is ready. Read it in! */
 	*val = hdq_reg_in(hdq_data, OMAP_HDQ_RX_DATA);
 out:
-	hdq_reset_irqstatus(hdq_data);
 	mutex_unlock(&hdq_data->hdq_mutex);
 rtn:
 	return ret;
@@ -367,15 +392,15 @@ static u8 omap_w1_triplet(void *_hdq, u8 bdir)
 				 (hdq_data->hdq_irqstatus
 				  & OMAP_HDQ_INT_STATUS_RXCOMPLETE),
 				 OMAP_HDQ_TIMEOUT);
+	/* Must clear irqstatus for another RXCOMPLETE interrupt */
+	hdq_reset_irqstatus(hdq_data, OMAP_HDQ_INT_STATUS_RXCOMPLETE);
+
 	if (err == 0) {
 		dev_dbg(hdq_data->dev, "RX wait elapsed\n");
 		goto out;
 	}
 	id_bit = (hdq_reg_in(_hdq, OMAP_HDQ_RX_DATA) & 0x01);
 
-	/* Must clear irqstatus for another RXCOMPLETE interrupt */
-	hdq_reset_irqstatus(hdq_data);
-
 	/* read comp_bit */
 	hdq_reg_merge(_hdq, OMAP_HDQ_CTRL_STATUS,
 		      ctrl | OMAP_HDQ_CTRL_STATUS_DIR, mask);
@@ -383,6 +408,9 @@ static u8 omap_w1_triplet(void *_hdq, u8 bdir)
 				 (hdq_data->hdq_irqstatus
 				  & OMAP_HDQ_INT_STATUS_RXCOMPLETE),
 				 OMAP_HDQ_TIMEOUT);
+	/* Must clear irqstatus for another RXCOMPLETE interrupt */
+	hdq_reset_irqstatus(hdq_data, OMAP_HDQ_INT_STATUS_RXCOMPLETE);
+
 	if (err == 0) {
 		dev_dbg(hdq_data->dev, "RX wait elapsed\n");
 		goto out;
@@ -409,6 +437,9 @@ static u8 omap_w1_triplet(void *_hdq, u8 bdir)
 				 (hdq_data->hdq_irqstatus
 				  & OMAP_HDQ_INT_STATUS_TXCOMPLETE),
 				 OMAP_HDQ_TIMEOUT);
+	/* Must clear irqstatus for another TXCOMPLETE interrupt */
+	hdq_reset_irqstatus(hdq_data, OMAP_HDQ_INT_STATUS_TXCOMPLETE);
+
 	if (err == 0) {
 		dev_dbg(hdq_data->dev, "TX wait elapsed\n");
 		goto out;
@@ -418,7 +449,6 @@ static u8 omap_w1_triplet(void *_hdq, u8 bdir)
 		      OMAP_HDQ_CTRL_STATUS_SINGLE);
 
 out:
-	hdq_reset_irqstatus(hdq_data);
 	mutex_unlock(&hdq_data->hdq_mutex);
 rtn:
 	pm_runtime_mark_last_busy(hdq_data->dev);
@@ -464,7 +494,7 @@ static u8 omap_w1_read_byte(void *_hdq)
 
 	ret = hdq_read_byte(hdq_data, &val);
 	if (ret)
-		ret = -1;
+		val = -1;
 
 	pm_runtime_mark_last_busy(hdq_data->dev);
 	pm_runtime_put_autosuspend(hdq_data->dev);
diff --git a/drivers/w1/slaves/w1_ds2430.c b/drivers/w1/slaves/w1_ds2430.c
index 6fb0563fb2ae..75bb8a88620b 100644
--- a/drivers/w1/slaves/w1_ds2430.c
+++ b/drivers/w1/slaves/w1_ds2430.c
@@ -290,6 +290,6 @@ static struct w1_family w1_family_14 = {
 module_w1_family(w1_family_14);
 
 MODULE_AUTHOR("Angelo Dureghello <angelo.dureghello@timesys.com>");
-MODULE_DESCRIPTION("w1 family 14 driver for DS2430, 256kb EEPROM");
+MODULE_DESCRIPTION("w1 family 14 driver for DS2430, 256b EEPROM");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("w1-family-" __stringify(W1_EEPROM_DS2430));
diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c
index e028e0092799..c1b4eda16719 100644
--- a/drivers/w1/slaves/w1_therm.c
+++ b/drivers/w1/slaves/w1_therm.c
@@ -16,6 +16,7 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/hwmon.h>
+#include <linux/string.h>
 
 #include <linux/w1.h>
 
@@ -25,7 +26,8 @@
 #define W1_THERM_DS1825		0x3B
 #define W1_THERM_DS28EA00	0x42
 
-/* Allow the strong pullup to be disabled, but default to enabled.
+/*
+ * Allow the strong pullup to be disabled, but default to enabled.
  * If it was disabled a parasite powered device might not get the require
  * current to do a temperature conversion.  If it is enabled parasite powered
  * devices have a better chance of getting the current required.
@@ -41,42 +43,211 @@
 static int w1_strong_pullup = 1;
 module_param_named(strong_pullup, w1_strong_pullup, int, 0);
 
+/* Counter for devices supporting bulk reading */
+static u16 bulk_read_device_counter; /* =0 as per C standard */
+
+/* This command should be in public header w1.h but is not */
+#define W1_RECALL_EEPROM	0xB8
+
+/* Nb of try for an operation */
+#define W1_THERM_MAX_TRY		5
+
+/* ms delay to retry bus mutex */
+#define W1_THERM_RETRY_DELAY		20
+
+/* delay in ms to write in EEPROM */
+#define W1_THERM_EEPROM_WRITE_DELAY	10
+
+#define EEPROM_CMD_WRITE    "save"	/* cmd for write eeprom sysfs */
+#define EEPROM_CMD_READ     "restore"	/* cmd for read eeprom sysfs */
+#define BULK_TRIGGER_CMD    "trigger"	/* cmd to trigger a bulk read */
+
+#define MIN_TEMP	-55	/* min temperature that can be mesured */
+#define MAX_TEMP	125	/* max temperature that can be mesured */
+
+/* Helpers Macros */
+
+/*
+ * return a pointer on the slave w1_therm_family_converter struct:
+ * always test family data existence before using this macro
+ */
+#define SLAVE_SPECIFIC_FUNC(sl) \
+	(((struct w1_therm_family_data *)(sl->family_data))->specific_functions)
+
+/*
+ * return the power mode of the sl slave : 1-ext, 0-parasite, <0 unknown
+ * always test family data existence before using this macro
+ */
+#define SLAVE_POWERMODE(sl) \
+	(((struct w1_therm_family_data *)(sl->family_data))->external_powered)
+
+/*
+ * return the resolution in bit of the sl slave : <0 unknown
+ * always test family data existence before using this macro
+ */
+#define SLAVE_RESOLUTION(sl) \
+	(((struct w1_therm_family_data *)(sl->family_data))->resolution)
+
+/*
+ * return whether or not a converT command has been issued to the slave
+ * * 0: no bulk read is pending
+ * * -1: conversion is in progress
+ * * 1: conversion done, result to be read
+ */
+#define SLAVE_CONVERT_TRIGGERED(sl) \
+	(((struct w1_therm_family_data *)(sl->family_data))->convert_triggered)
+
+/* return the address of the refcnt in the family data */
+#define THERM_REFCNT(family_data) \
+	(&((struct w1_therm_family_data *)family_data)->refcnt)
+
+/* Structs definition */
+
+/**
+ * struct w1_therm_family_converter - bind device specific functions
+ * @broken: flag for non-registred families
+ * @reserved: not used here
+ * @f: pointer to the device binding structure
+ * @convert: pointer to the device conversion function
+ * @get_conversion_time: pointer to the device conversion time function
+ * @set_resolution: pointer to the device set_resolution function
+ * @get_resolution: pointer to the device get_resolution function
+ * @write_data: pointer to the device writing function (2 or 3 bytes)
+ * @bulk_read: true if device family support bulk read, false otherwise
+ */
+struct w1_therm_family_converter {
+	u8		broken;
+	u16		reserved;
+	struct w1_family	*f;
+	int		(*convert)(u8 rom[9]);
+	int		(*get_conversion_time)(struct w1_slave *sl);
+	int		(*set_resolution)(struct w1_slave *sl, int val);
+	int		(*get_resolution)(struct w1_slave *sl);
+	int		(*write_data)(struct w1_slave *sl, const u8 *data);
+	bool		bulk_read;
+};
+
+/**
+ * struct w1_therm_family_data - device data
+ * @rom: ROM device id (64bit Lasered ROM code + 1 CRC byte)
+ * @refcnt: ref count
+ * @external_powered:	1 device powered externally,
+ *				0 device parasite powered,
+ *				-x error or undefined
+ * @resolution: current device resolution
+ * @convert_triggered: conversion state of the device
+ * @specific_functions: pointer to struct of device specific function
+ */
 struct w1_therm_family_data {
 	uint8_t rom[9];
 	atomic_t refcnt;
+	int external_powered;
+	int resolution;
+	int convert_triggered;
+	struct w1_therm_family_converter *specific_functions;
 };
 
+/**
+ * struct therm_info - store temperature reading
+ * @rom: read device data (8 data bytes + 1 CRC byte)
+ * @crc: computed crc from rom
+ * @verdict: 1 crc checked, 0 crc not matching
+ */
 struct therm_info {
 	u8 rom[9];
 	u8 crc;
 	u8 verdict;
 };
 
-/* return the address of the refcnt in the family data */
-#define THERM_REFCNT(family_data) \
-	(&((struct w1_therm_family_data *)family_data)->refcnt)
+/* Hardware Functions declaration */
 
-static int w1_therm_add_slave(struct w1_slave *sl)
-{
-	sl->family_data = kzalloc(sizeof(struct w1_therm_family_data),
-		GFP_KERNEL);
-	if (!sl->family_data)
-		return -ENOMEM;
-	atomic_set(THERM_REFCNT(sl->family_data), 1);
-	return 0;
-}
+/**
+ * reset_select_slave() - reset and select a slave
+ * @sl: the slave to select
+ *
+ * Resets the bus and select the slave by sending a ROM MATCH cmd
+ * w1_reset_select_slave() from w1_io.c could not be used here because
+ * it sent a SKIP ROM command if only one device is on the line.
+ * At the beginning of the such process, sl->master->slave_count is 1 even if
+ * more devices are on the line, causing collision on the line.
+ *
+ * Context: The w1 master lock must be held.
+ *
+ * Return: 0 if success, negative kernel error code otherwise.
+ */
+static int reset_select_slave(struct w1_slave *sl);
 
-static void w1_therm_remove_slave(struct w1_slave *sl)
-{
-	int refcnt = atomic_sub_return(1, THERM_REFCNT(sl->family_data));
+/**
+ * convert_t() - Query the device for temperature conversion and read
+ * @sl: pointer to the slave to read
+ * @info: pointer to a structure to store the read results
+ *
+ * Return: 0 if success, -kernel error code otherwise
+ */
+static int convert_t(struct w1_slave *sl, struct therm_info *info);
 
-	while (refcnt) {
-		msleep(1000);
-		refcnt = atomic_read(THERM_REFCNT(sl->family_data));
-	}
-	kfree(sl->family_data);
-	sl->family_data = NULL;
-}
+/**
+ * read_scratchpad() - read the data in device RAM
+ * @sl: pointer to the slave to read
+ * @info: pointer to a structure to store the read results
+ *
+ * Return: 0 if success, -kernel error code otherwise
+ */
+static int read_scratchpad(struct w1_slave *sl, struct therm_info *info);
+
+/**
+ * write_scratchpad() - write nb_bytes in the device RAM
+ * @sl: pointer to the slave to write in
+ * @data: pointer to an array of 3 bytes, as 3 bytes MUST be written
+ * @nb_bytes: number of bytes to be written (2 for DS18S20, 3 otherwise)
+ *
+ * Return: 0 if success, -kernel error code otherwise
+ */
+static int write_scratchpad(struct w1_slave *sl, const u8 *data, u8 nb_bytes);
+
+/**
+ * copy_scratchpad() - Copy the content of scratchpad in device EEPROM
+ * @sl: slave involved
+ *
+ * Return: 0 if success, -kernel error code otherwise
+ */
+static int copy_scratchpad(struct w1_slave *sl);
+
+/**
+ * recall_eeprom() - Restore EEPROM data to device RAM
+ * @sl: slave involved
+ *
+ * Return: 0 if success, -kernel error code otherwise
+ */
+static int recall_eeprom(struct w1_slave *sl);
+
+/**
+ * read_powermode() - Query the power mode of the slave
+ * @sl: slave to retrieve the power mode
+ *
+ * Ask the device to get its power mode (external or parasite)
+ * and store the power status in the &struct w1_therm_family_data.
+ *
+ * Return:
+ * * 0 parasite powered device
+ * * 1 externally powered device
+ * * <0 kernel error code
+ */
+static int read_powermode(struct w1_slave *sl);
+
+/**
+ * trigger_bulk_read() - function to trigger a bulk read on the bus
+ * @dev_master: the device master of the bus
+ *
+ * Send a SKIP ROM follow by a CONVERT T commmand on the bus.
+ * It also set the status flag in each slave &struct w1_therm_family_data
+ * to signal that a conversion is in progress.
+ *
+ * Return: 0 if success, -kernel error code otherwise
+ */
+static int trigger_bulk_read(struct w1_master *dev_master);
+
+/* Sysfs interface declaration */
 
 static ssize_t w1_slave_show(struct device *device,
 	struct device_attribute *attr, char *buf);
@@ -87,21 +258,103 @@ static ssize_t w1_slave_store(struct device *device,
 static ssize_t w1_seq_show(struct device *device,
 	struct device_attribute *attr, char *buf);
 
+static ssize_t temperature_show(struct device *device,
+	struct device_attribute *attr, char *buf);
+
+static ssize_t ext_power_show(struct device *device,
+	struct device_attribute *attr, char *buf);
+
+static ssize_t resolution_show(struct device *device,
+	struct device_attribute *attr, char *buf);
+
+static ssize_t resolution_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size);
+
+static ssize_t eeprom_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size);
+
+static ssize_t alarms_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size);
+
+static ssize_t alarms_show(struct device *device,
+	struct device_attribute *attr, char *buf);
+
+static ssize_t therm_bulk_read_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size);
+
+static ssize_t therm_bulk_read_show(struct device *device,
+	struct device_attribute *attr, char *buf);
+
+/* Attributes declarations */
+
 static DEVICE_ATTR_RW(w1_slave);
 static DEVICE_ATTR_RO(w1_seq);
+static DEVICE_ATTR_RO(temperature);
+static DEVICE_ATTR_RO(ext_power);
+static DEVICE_ATTR_RW(resolution);
+static DEVICE_ATTR_WO(eeprom);
+static DEVICE_ATTR_RW(alarms);
+
+static DEVICE_ATTR_RW(therm_bulk_read); /* attribut at master level */
+
+/* Interface Functions declaration */
+
+/**
+ * w1_therm_add_slave() - Called when a new slave is discovered
+ * @sl: slave just discovered by the master.
+ *
+ * Called by the master when the slave is discovered on the bus. Used to
+ * initialize slave state before the beginning of any communication.
+ *
+ * Return: 0 - If success, negative kernel code otherwise
+ */
+static int w1_therm_add_slave(struct w1_slave *sl);
+
+/**
+ * w1_therm_remove_slave() - Called when a slave is removed
+ * @sl: slave to be removed.
+ *
+ * Called by the master when the slave is considered not to be on the bus
+ * anymore. Used to free memory.
+ */
+static void w1_therm_remove_slave(struct w1_slave *sl);
+
+/* Family attributes */
 
 static struct attribute *w1_therm_attrs[] = {
 	&dev_attr_w1_slave.attr,
+	&dev_attr_temperature.attr,
+	&dev_attr_ext_power.attr,
+	&dev_attr_resolution.attr,
+	&dev_attr_eeprom.attr,
+	&dev_attr_alarms.attr,
+	NULL,
+};
+
+static struct attribute *w1_ds18s20_attrs[] = {
+	&dev_attr_w1_slave.attr,
+	&dev_attr_temperature.attr,
+	&dev_attr_ext_power.attr,
+	&dev_attr_eeprom.attr,
+	&dev_attr_alarms.attr,
 	NULL,
 };
 
 static struct attribute *w1_ds28ea00_attrs[] = {
 	&dev_attr_w1_slave.attr,
 	&dev_attr_w1_seq.attr,
+	&dev_attr_temperature.attr,
+	&dev_attr_ext_power.attr,
+	&dev_attr_resolution.attr,
+	&dev_attr_eeprom.attr,
+	&dev_attr_alarms.attr,
 	NULL,
 };
 
+/* Attribute groups */
+
 ATTRIBUTE_GROUPS(w1_therm);
+ATTRIBUTE_GROUPS(w1_ds18s20);
 ATTRIBUTE_GROUPS(w1_ds28ea00);
 
 #if IS_REACHABLE(CONFIG_HWMON)
@@ -154,6 +407,8 @@ static const struct hwmon_chip_info w1_chip_info = {
 #define W1_CHIPINFO	NULL
 #endif
 
+/* Family operations */
+
 static struct w1_family_ops w1_therm_fops = {
 	.add_slave	= w1_therm_add_slave,
 	.remove_slave	= w1_therm_remove_slave,
@@ -161,6 +416,13 @@ static struct w1_family_ops w1_therm_fops = {
 	.chip_info	= W1_CHIPINFO,
 };
 
+static struct w1_family_ops w1_ds18s20_fops = {
+	.add_slave	= w1_therm_add_slave,
+	.remove_slave	= w1_therm_remove_slave,
+	.groups		= w1_ds18s20_groups,
+	.chip_info	= W1_CHIPINFO,
+};
+
 static struct w1_family_ops w1_ds28ea00_fops = {
 	.add_slave	= w1_therm_add_slave,
 	.remove_slave	= w1_therm_remove_slave,
@@ -168,9 +430,11 @@ static struct w1_family_ops w1_ds28ea00_fops = {
 	.chip_info	= W1_CHIPINFO,
 };
 
+/* Family binding operations struct */
+
 static struct w1_family w1_therm_family_DS18S20 = {
 	.fid = W1_THERM_DS18S20,
-	.fops = &w1_therm_fops,
+	.fops = &w1_ds18s20_fops,
 };
 
 static struct w1_family w1_therm_family_DS18B20 = {
@@ -193,377 +457,813 @@ static struct w1_family w1_therm_family_DS1825 = {
 	.fops = &w1_therm_fops,
 };
 
-struct w1_therm_family_converter {
-	u8			broken;
-	u16			reserved;
-	struct w1_family	*f;
-	int			(*convert)(u8 rom[9]);
-	int			(*precision)(struct device *device, int val);
-	int			(*eeprom)(struct device *device);
-};
+/* Device dependent func */
+
+static inline int w1_DS18B20_convert_time(struct w1_slave *sl)
+{
+	int ret;
+
+	if (!sl->family_data)
+		return -ENODEV;	/* device unknown */
+
+	/* return time in ms for conversion operation */
+	switch (SLAVE_RESOLUTION(sl)) {
+	case 9:
+		ret = 95;
+		break;
+	case 10:
+		ret = 190;
+		break;
+	case 11:
+		ret = 375;
+		break;
+	case 12:
+	default:
+		ret = 750;
+	}
+	return ret;
+}
+
+static inline int w1_DS18S20_convert_time(struct w1_slave *sl)
+{
+	(void)(sl);
+	return 750; /* always 750ms for DS18S20 */
+}
+
+static inline int w1_DS18B20_write_data(struct w1_slave *sl,
+				const u8 *data)
+{
+	return write_scratchpad(sl, data, 3);
+}
+
+static inline int w1_DS18S20_write_data(struct w1_slave *sl,
+				const u8 *data)
+{
+	/* No config register */
+	return write_scratchpad(sl, data, 2);
+}
+
+static inline int w1_DS18B20_set_resolution(struct w1_slave *sl, int val)
+{
+	int ret;
+	u8 new_config_register[3];	/* array of data to be written */
+	struct therm_info info;
+
+	/* resolution of DS18B20 is in the range [9..12] bits */
+	if (val < 9 || val > 12)
+		return -EINVAL;
+
+	val -= 9; /* soustract 9 the lowest resolution in bit */
+	val = (val << 5); /* shift to position bit 5 & bit 6 */
+
+	/*
+	 * Read the scratchpad to change only the required bits
+	 * (bit5 & bit 6 from byte 4)
+	 */
+	ret = read_scratchpad(sl, &info);
+	if (!ret) {
+		new_config_register[0] = info.rom[2];
+		new_config_register[1] = info.rom[3];
+		/* config register is byte 4 & mask 0b10011111*/
+		new_config_register[2] = (info.rom[4] & 0x9F) |
+					(u8) val;
+	} else
+		return ret;
+
+	/* Write data in the device RAM */
+	ret = w1_DS18B20_write_data(sl, new_config_register);
+
+	return ret;
+}
+
+static inline int w1_DS18B20_get_resolution(struct w1_slave *sl)
+{
+	int ret;
+	u8 config_register;
+	struct therm_info info;
+
+	ret = read_scratchpad(sl, &info);
+
+	if (!ret)	{
+		config_register = info.rom[4]; /* config register is byte 4 */
+		config_register &= 0x60; /* 0b01100000 keep only bit 5 & 6 */
+		config_register = (config_register >> 5);	/* shift */
+		config_register += 9; /* add 9 the lowest resolution in bit */
+		ret = (int) config_register;
+	}
+	return ret;
+}
 
-/* write configuration to eeprom */
-static inline int w1_therm_eeprom(struct device *device);
+/**
+ * w1_DS18B20_convert_temp() - temperature computation for DS18B20
+ * @rom: data read from device RAM (8 data bytes + 1 CRC byte)
+ *
+ * Can be called for any DS18B20 compliant device.
+ *
+ * Return: value in millidegrees Celsius.
+ */
+static inline int w1_DS18B20_convert_temp(u8 rom[9])
+{
+	s16 t = le16_to_cpup((__le16 *)rom);
 
-/* Set precision for conversion */
-static inline int w1_DS18B20_precision(struct device *device, int val);
-static inline int w1_DS18S20_precision(struct device *device, int val);
+	return t*1000/16;
+}
 
-/* The return value is millidegrees Centigrade. */
-static inline int w1_DS18B20_convert_temp(u8 rom[9]);
-static inline int w1_DS18S20_convert_temp(u8 rom[9]);
+/**
+ * w1_DS18S20_convert_temp() - temperature computation for DS18S20
+ * @rom: data read from device RAM (8 data bytes + 1 CRC byte)
+ *
+ * Can be called for any DS18S20 compliant device.
+ *
+ * Return: value in millidegrees Celsius.
+ */
+static inline int w1_DS18S20_convert_temp(u8 rom[9])
+{
+	int t, h;
+
+	if (!rom[7]) {
+		pr_debug("%s: Invalid argument for conversion\n", __func__);
+		return 0;
+	}
+
+	if (rom[1] == 0)
+		t = ((s32)rom[0] >> 1)*1000;
+	else
+		t = 1000*(-1*(s32)(0x100-rom[0]) >> 1);
+
+	t -= 250;
+	h = 1000*((s32)rom[7] - (s32)rom[6]);
+	h /= (s32)rom[7];
+	t += h;
+
+	return t;
+}
+
+/* Device capability description */
 
 static struct w1_therm_family_converter w1_therm_families[] = {
 	{
-		.f		= &w1_therm_family_DS18S20,
-		.convert	= w1_DS18S20_convert_temp,
-		.precision	= w1_DS18S20_precision,
-		.eeprom		= w1_therm_eeprom
+		.f				= &w1_therm_family_DS18S20,
+		.convert			= w1_DS18S20_convert_temp,
+		.get_conversion_time	= w1_DS18S20_convert_time,
+		.set_resolution		= NULL,	/* no config register */
+		.get_resolution		= NULL,	/* no config register */
+		.write_data			= w1_DS18S20_write_data,
+		.bulk_read			= true
 	},
 	{
-		.f		= &w1_therm_family_DS1822,
-		.convert	= w1_DS18B20_convert_temp,
-		.precision	= w1_DS18S20_precision,
-		.eeprom		= w1_therm_eeprom
+		.f				= &w1_therm_family_DS1822,
+		.convert			= w1_DS18B20_convert_temp,
+		.get_conversion_time	= w1_DS18B20_convert_time,
+		.set_resolution		= w1_DS18B20_set_resolution,
+		.get_resolution		= w1_DS18B20_get_resolution,
+		.write_data			= w1_DS18B20_write_data,
+		.bulk_read			= true
 	},
 	{
-		.f		= &w1_therm_family_DS18B20,
-		.convert	= w1_DS18B20_convert_temp,
-		.precision	= w1_DS18B20_precision,
-		.eeprom		= w1_therm_eeprom
+		.f				= &w1_therm_family_DS18B20,
+		.convert			= w1_DS18B20_convert_temp,
+		.get_conversion_time	= w1_DS18B20_convert_time,
+		.set_resolution		= w1_DS18B20_set_resolution,
+		.get_resolution		= w1_DS18B20_get_resolution,
+		.write_data			= w1_DS18B20_write_data,
+		.bulk_read			= true
 	},
 	{
-		.f		= &w1_therm_family_DS28EA00,
-		.convert	= w1_DS18B20_convert_temp,
-		.precision	= w1_DS18S20_precision,
-		.eeprom		= w1_therm_eeprom
+		.f				= &w1_therm_family_DS28EA00,
+		.convert			= w1_DS18B20_convert_temp,
+		.get_conversion_time	= w1_DS18B20_convert_time,
+		.set_resolution		= w1_DS18B20_set_resolution,
+		.get_resolution		= w1_DS18B20_get_resolution,
+		.write_data			= w1_DS18B20_write_data,
+		.bulk_read			= false
 	},
 	{
-		.f		= &w1_therm_family_DS1825,
-		.convert	= w1_DS18B20_convert_temp,
-		.precision	= w1_DS18S20_precision,
-		.eeprom		= w1_therm_eeprom
+		.f				= &w1_therm_family_DS1825,
+		.convert			= w1_DS18B20_convert_temp,
+		.get_conversion_time	= w1_DS18B20_convert_time,
+		.set_resolution		= w1_DS18B20_set_resolution,
+		.get_resolution		= w1_DS18B20_get_resolution,
+		.write_data			= w1_DS18B20_write_data,
+		.bulk_read			= true
 	}
 };
 
-static inline int w1_therm_eeprom(struct device *device)
+/* Helpers Functions */
+
+/**
+ * device_family() - Retrieve a pointer on &struct w1_therm_family_converter
+ * @sl: slave to retrieve the device specific structure
+ *
+ * Return: pointer to the slaves's family converter, NULL if not known
+ */
+static struct w1_therm_family_converter *device_family(struct w1_slave *sl)
 {
-	struct w1_slave *sl = dev_to_w1_slave(device);
-	struct w1_master *dev = sl->master;
-	u8 rom[9], external_power;
-	int ret, max_trying = 10;
-	u8 *family_data = sl->family_data;
+	struct w1_therm_family_converter *ret = NULL;
+	int i;
 
-	if (!sl->family_data) {
-		ret = -ENODEV;
-		goto error;
+	for (i = 0; i < ARRAY_SIZE(w1_therm_families); ++i) {
+		if (w1_therm_families[i].f->fid == sl->family->fid) {
+			ret = &w1_therm_families[i];
+			break;
+		}
 	}
+	return ret;
+}
 
-	/* prevent the slave from going away in sleep */
-	atomic_inc(THERM_REFCNT(family_data));
+/**
+ * bus_mutex_lock() - Acquire the mutex
+ * @lock: w1 bus mutex to acquire
+ *
+ * It try to acquire the mutex W1_THERM_MAX_TRY times and wait
+ * W1_THERM_RETRY_DELAY between 2 attempts.
+ *
+ * Return: true is mutex is acquired and lock, false otherwise
+ */
+static inline bool bus_mutex_lock(struct mutex *lock)
+{
+	int max_trying = W1_THERM_MAX_TRY;
 
-	ret = mutex_lock_interruptible(&dev->bus_mutex);
-	if (ret != 0)
-		goto dec_refcnt;
+	/* try to acquire the mutex, if not, sleep retry_delay before retry) */
+	while (mutex_lock_interruptible(lock) != 0 && max_trying > 0) {
+		unsigned long sleep_rem;
 
-	memset(rom, 0, sizeof(rom));
+		sleep_rem = msleep_interruptible(W1_THERM_RETRY_DELAY);
+		if (!sleep_rem)
+			max_trying--;
+	}
 
-	while (max_trying--) {
-		if (!w1_reset_select_slave(sl)) {
-			unsigned int tm = 10;
-			unsigned long sleep_rem;
+	if (!max_trying)
+		return false;	/* Didn't acquire the bus mutex */
+
+	return true;
+}
 
-			/* check if in parasite mode */
-			w1_write_8(dev, W1_READ_PSUPPLY);
-			external_power = w1_read_8(dev);
+/**
+ * support_bulk_read() - check if slave support bulk read
+ * @sl: device to check the ability
+ *
+ * Return: true if bulk read is supported, false if not or error
+ */
+static inline bool bulk_read_support(struct w1_slave *sl)
+{
+	if (SLAVE_SPECIFIC_FUNC(sl))
+		return SLAVE_SPECIFIC_FUNC(sl)->bulk_read;
+
+	dev_info(&sl->dev,
+		"%s: Device not supported by the driver\n", __func__);
+
+	return false;  /* No device family */
+}
+
+/**
+ * conversion_time() - get the Tconv for the slave
+ * @sl: device to get the conversion time
+ *
+ * On device supporting resolution settings, conversion time depend
+ * on the resolution setting. This helper function get the slave timing,
+ * depending on its current setting.
+ *
+ * Return: conversion time in ms, negative values are kernel error code
+ */
+static inline int conversion_time(struct w1_slave *sl)
+{
+	if (SLAVE_SPECIFIC_FUNC(sl))
+		return SLAVE_SPECIFIC_FUNC(sl)->get_conversion_time(sl);
+
+	dev_info(&sl->dev,
+		"%s: Device not supported by the driver\n", __func__);
+
+	return -ENODEV;  /* No device family */
+}
+
+/**
+ * temperature_from_RAM() - Convert the read info to temperature
+ * @sl: device that sent the RAM data
+ * @rom: read value on the slave device RAM
+ *
+ * Device dependent, the function bind the correct computation method.
+ *
+ * Return: temperature in 1/1000degC, 0 on error.
+ */
+static inline int temperature_from_RAM(struct w1_slave *sl, u8 rom[9])
+{
+	if (SLAVE_SPECIFIC_FUNC(sl))
+		return SLAVE_SPECIFIC_FUNC(sl)->convert(rom);
 
-			if (w1_reset_select_slave(sl))
-				continue;
+	dev_info(&sl->dev,
+		"%s: Device not supported by the driver\n", __func__);
 
-			/* 10ms strong pullup/delay after the copy command */
-			if (w1_strong_pullup == 2 ||
-			    (!external_power && w1_strong_pullup))
-				w1_next_pullup(dev, tm);
+	return 0;  /* No device family */
+}
+
+/**
+ * int_to_short() - Safe casting of int to short
+ *
+ * @i: integer to be converted to short
+ *
+ * Device register use 1 byte to store signed integer.
+ * This helper function convert the int in a signed short,
+ * using the min/max values that device can measure as limits.
+ * min/max values are defined by macro.
+ *
+ * Return: a short in the range of min/max value
+ */
+static inline s8 int_to_short(int i)
+{
+	/* Prepare to cast to short by eliminating out of range values */
+	i = i > MAX_TEMP ? MAX_TEMP : i;
+	i = i < MIN_TEMP ? MIN_TEMP : i;
+	return (s8) i;
+}
 
-			w1_write_8(dev, W1_COPY_SCRATCHPAD);
+/* Interface Functions */
 
-			if (external_power) {
-				mutex_unlock(&dev->bus_mutex);
+static int w1_therm_add_slave(struct w1_slave *sl)
+{
+	struct w1_therm_family_converter *sl_family_conv;
+
+	/* Allocate memory */
+	sl->family_data = kzalloc(sizeof(struct w1_therm_family_data),
+		GFP_KERNEL);
+	if (!sl->family_data)
+		return -ENOMEM;
+
+	atomic_set(THERM_REFCNT(sl->family_data), 1);
+
+	/* Get a pointer to the device specific function struct */
+	sl_family_conv = device_family(sl);
+	if (!sl_family_conv) {
+		kfree(sl->family_data);
+		return -ENODEV;
+	}
+	/* save this pointer to the device structure */
+	SLAVE_SPECIFIC_FUNC(sl) = sl_family_conv;
+
+	if (bulk_read_support(sl)) {
+		/*
+		 * add the sys entry to trigger bulk_read
+		 * at master level only the 1st time
+		 */
+		if (!bulk_read_device_counter) {
+			int err = device_create_file(&sl->master->dev,
+				&dev_attr_therm_bulk_read);
+
+			if (err)
+				dev_warn(&sl->dev,
+				"%s: Device has been added, but bulk read is unavailable. err=%d\n",
+				__func__, err);
+		}
+		/* Increment the counter */
+		bulk_read_device_counter++;
+	}
+
+	/* Getting the power mode of the device {external, parasite} */
+	SLAVE_POWERMODE(sl) = read_powermode(sl);
+
+	if (SLAVE_POWERMODE(sl) < 0) {
+		/* no error returned as device has been added */
+		dev_warn(&sl->dev,
+			"%s: Device has been added, but power_mode may be corrupted. err=%d\n",
+			 __func__, SLAVE_POWERMODE(sl));
+	}
+
+	/* Getting the resolution of the device */
+	if (SLAVE_SPECIFIC_FUNC(sl)->get_resolution) {
+		SLAVE_RESOLUTION(sl) =
+			SLAVE_SPECIFIC_FUNC(sl)->get_resolution(sl);
+		if (SLAVE_RESOLUTION(sl) < 0) {
+			/* no error returned as device has been added */
+			dev_warn(&sl->dev,
+				"%s:Device has been added, but resolution may be corrupted. err=%d\n",
+				__func__, SLAVE_RESOLUTION(sl));
+		}
+	}
+
+	/* Finally initialize convert_triggered flag */
+	SLAVE_CONVERT_TRIGGERED(sl) = 0;
+
+	return 0;
+}
+
+static void w1_therm_remove_slave(struct w1_slave *sl)
+{
+	int refcnt = atomic_sub_return(1, THERM_REFCNT(sl->family_data));
+
+	if (bulk_read_support(sl)) {
+		bulk_read_device_counter--;
+		/* Delete the entry if no more device support the feature */
+		if (!bulk_read_device_counter)
+			device_remove_file(&sl->master->dev,
+				&dev_attr_therm_bulk_read);
+	}
+
+	while (refcnt) {
+		msleep(1000);
+		refcnt = atomic_read(THERM_REFCNT(sl->family_data));
+	}
+	kfree(sl->family_data);
+	sl->family_data = NULL;
+}
+
+/* Hardware Functions */
+
+/* Safe version of reset_select_slave - avoid using the one in w_io.c */
+static int reset_select_slave(struct w1_slave *sl)
+{
+	u8 match[9] = { W1_MATCH_ROM, };
+	u64 rn = le64_to_cpu(*((u64 *)&sl->reg_num));
+
+	if (w1_reset_bus(sl->master))
+		return -ENODEV;
+
+	memcpy(&match[1], &rn, 8);
+	w1_write_block(sl->master, match, 9);
+
+	return 0;
+}
+
+static int convert_t(struct w1_slave *sl, struct therm_info *info)
+{
+	struct w1_master *dev_master = sl->master;
+	int max_trying = W1_THERM_MAX_TRY;
+	int t_conv;
+	int ret = -ENODEV;
+	bool strong_pullup;
 
-				sleep_rem = msleep_interruptible(tm);
+	if (!sl->family_data)
+		goto error;
+
+	strong_pullup = (w1_strong_pullup == 2 ||
+					(!SLAVE_POWERMODE(sl) &&
+					w1_strong_pullup));
+
+	/* get conversion duration device and id dependent */
+	t_conv = conversion_time(sl);
+
+	memset(info->rom, 0, sizeof(info->rom));
+
+	/* prevent the slave from going away in sleep */
+	atomic_inc(THERM_REFCNT(sl->family_data));
+
+	if (!bus_mutex_lock(&dev_master->bus_mutex)) {
+		ret = -EAGAIN;	/* Didn't acquire the mutex */
+		goto dec_refcnt;
+	}
+
+	while (max_trying-- && ret) { /* ret should be 0 */
+
+		info->verdict = 0;
+		info->crc = 0;
+		/* safe version to select slave */
+		if (!reset_select_slave(sl)) {
+			unsigned long sleep_rem;
+
+			/* 750ms strong pullup (or delay) after the convert */
+			if (strong_pullup)
+				w1_next_pullup(dev_master, t_conv);
+
+			w1_write_8(dev_master, W1_CONVERT_TEMP);
+
+			if (strong_pullup) { /*some device need pullup */
+				sleep_rem = msleep_interruptible(t_conv);
 				if (sleep_rem != 0) {
 					ret = -EINTR;
-					goto dec_refcnt;
+					goto mt_unlock;
 				}
+				mutex_unlock(&dev_master->bus_mutex);
+			} else { /*no device need pullup */
+				mutex_unlock(&dev_master->bus_mutex);
 
-				ret = mutex_lock_interruptible(&dev->bus_mutex);
-				if (ret != 0)
-					goto dec_refcnt;
-			} else if (!w1_strong_pullup) {
-				sleep_rem = msleep_interruptible(tm);
+				sleep_rem = msleep_interruptible(t_conv);
 				if (sleep_rem != 0) {
 					ret = -EINTR;
-					goto mt_unlock;
+					goto dec_refcnt;
 				}
 			}
-
-			break;
+			ret = read_scratchpad(sl, info);
+			goto dec_refcnt;
 		}
+
 	}
 
 mt_unlock:
-	mutex_unlock(&dev->bus_mutex);
+	mutex_unlock(&dev_master->bus_mutex);
 dec_refcnt:
-	atomic_dec(THERM_REFCNT(family_data));
+	atomic_dec(THERM_REFCNT(sl->family_data));
 error:
 	return ret;
 }
 
-/* DS18S20 does not feature configuration register */
-static inline int w1_DS18S20_precision(struct device *device, int val)
+static int read_scratchpad(struct w1_slave *sl, struct therm_info *info)
 {
-	return 0;
-}
+	struct w1_master *dev_master = sl->master;
+	int max_trying = W1_THERM_MAX_TRY;
+	int ret = -ENODEV;
 
-static inline int w1_DS18B20_precision(struct device *device, int val)
-{
-	struct w1_slave *sl = dev_to_w1_slave(device);
-	struct w1_master *dev = sl->master;
-	u8 rom[9], crc;
-	int ret, max_trying = 10;
-	u8 *family_data = sl->family_data;
-	uint8_t precision_bits;
-	uint8_t mask = 0x60;
+	info->verdict = 0;
 
-	if (val > 12 || val < 9) {
-		pr_warn("Unsupported precision\n");
-		ret = -EINVAL;
+	if (!sl->family_data)
 		goto error;
-	}
 
-	if (!sl->family_data) {
-		ret = -ENODEV;
-		goto error;
-	}
+	memset(info->rom, 0, sizeof(info->rom));
 
 	/* prevent the slave from going away in sleep */
-	atomic_inc(THERM_REFCNT(family_data));
+	atomic_inc(THERM_REFCNT(sl->family_data));
 
-	ret = mutex_lock_interruptible(&dev->bus_mutex);
-	if (ret != 0)
+	if (!bus_mutex_lock(&dev_master->bus_mutex)) {
+		ret = -EAGAIN;	/* Didn't acquire the mutex */
 		goto dec_refcnt;
+	}
 
-	memset(rom, 0, sizeof(rom));
+	while (max_trying-- && ret) { /* ret should be 0 */
+		/* safe version to select slave */
+		if (!reset_select_slave(sl)) {
+			u8 nb_bytes_read;
+
+			w1_write_8(dev_master, W1_READ_SCRATCHPAD);
+
+			nb_bytes_read = w1_read_block(dev_master, info->rom, 9);
+			if (nb_bytes_read != 9) {
+				dev_warn(&sl->dev,
+					"w1_read_block(): returned %u instead of 9.\n",
+					nb_bytes_read);
+				ret = -EIO;
+			}
+
+			info->crc = w1_calc_crc8(info->rom, 8);
+
+			if (info->rom[8] == info->crc) {
+				info->verdict = 1;
+				ret = 0;
+			} else
+				ret = -EIO; /* CRC not checked */
+		}
 
-	/* translate precision to bitmask (see datasheet page 9) */
-	switch (val) {
-	case 9:
-		precision_bits = 0x00;
-		break;
-	case 10:
-		precision_bits = 0x20;
-		break;
-	case 11:
-		precision_bits = 0x40;
-		break;
-	case 12:
-	default:
-		precision_bits = 0x60;
-		break;
 	}
+	mutex_unlock(&dev_master->bus_mutex);
 
-	while (max_trying--) {
-		crc = 0;
+dec_refcnt:
+	atomic_dec(THERM_REFCNT(sl->family_data));
+error:
+	return ret;
+}
 
-		if (!w1_reset_select_slave(sl)) {
-			int count = 0;
+static int write_scratchpad(struct w1_slave *sl, const u8 *data, u8 nb_bytes)
+{
+	struct w1_master *dev_master = sl->master;
+	int max_trying = W1_THERM_MAX_TRY;
+	int ret = -ENODEV;
 
-			/* read values to only alter precision bits */
-			w1_write_8(dev, W1_READ_SCRATCHPAD);
-			count = w1_read_block(dev, rom, 9);
-			if (count != 9)
-				dev_warn(device, "w1_read_block() returned %u instead of 9.\n",	count);
+	if (!sl->family_data)
+		goto error;
 
-			crc = w1_calc_crc8(rom, 8);
-			if (rom[8] == crc) {
-				rom[4] = (rom[4] & ~mask) | (precision_bits & mask);
+	/* prevent the slave from going away in sleep */
+	atomic_inc(THERM_REFCNT(sl->family_data));
 
-				if (!w1_reset_select_slave(sl)) {
-					w1_write_8(dev, W1_WRITE_SCRATCHPAD);
-					w1_write_8(dev, rom[2]);
-					w1_write_8(dev, rom[3]);
-					w1_write_8(dev, rom[4]);
+	if (!bus_mutex_lock(&dev_master->bus_mutex)) {
+		ret = -EAGAIN;	/* Didn't acquire the mutex */
+		goto dec_refcnt;
+	}
 
-					break;
-				}
-			}
+	while (max_trying-- && ret) { /* ret should be 0 */
+		/* safe version to select slave */
+		if (!reset_select_slave(sl)) {
+			w1_write_8(dev_master, W1_WRITE_SCRATCHPAD);
+			w1_write_block(dev_master, data, nb_bytes);
+			ret = 0;
 		}
 	}
+	mutex_unlock(&dev_master->bus_mutex);
 
-	mutex_unlock(&dev->bus_mutex);
 dec_refcnt:
-	atomic_dec(THERM_REFCNT(family_data));
+	atomic_dec(THERM_REFCNT(sl->family_data));
 error:
 	return ret;
 }
 
-static inline int w1_DS18B20_convert_temp(u8 rom[9])
+static int copy_scratchpad(struct w1_slave *sl)
 {
-	s16 t = le16_to_cpup((__le16 *)rom);
+	struct w1_master *dev_master = sl->master;
+	int max_trying = W1_THERM_MAX_TRY;
+	int t_write, ret = -ENODEV;
+	bool strong_pullup;
 
-	return t*1000/16;
-}
+	if (!sl->family_data)
+		goto error;
 
-static inline int w1_DS18S20_convert_temp(u8 rom[9])
-{
-	int t, h;
+	t_write = W1_THERM_EEPROM_WRITE_DELAY;
+	strong_pullup = (w1_strong_pullup == 2 ||
+					(!SLAVE_POWERMODE(sl) &&
+					w1_strong_pullup));
 
-	if (!rom[7])
-		return 0;
+	/* prevent the slave from going away in sleep */
+	atomic_inc(THERM_REFCNT(sl->family_data));
 
-	if (rom[1] == 0)
-		t = ((s32)rom[0] >> 1)*1000;
-	else
-		t = 1000*(-1*(s32)(0x100-rom[0]) >> 1);
+	if (!bus_mutex_lock(&dev_master->bus_mutex)) {
+		ret = -EAGAIN;	/* Didn't acquire the mutex */
+		goto dec_refcnt;
+	}
 
-	t -= 250;
-	h = 1000*((s32)rom[7] - (s32)rom[6]);
-	h /= (s32)rom[7];
-	t += h;
+	while (max_trying-- && ret) { /* ret should be 0 */
+		/* safe version to select slave */
+		if (!reset_select_slave(sl)) {
+			unsigned long sleep_rem;
 
-	return t;
-}
+			/* 10ms strong pullup (or delay) after the convert */
+			if (strong_pullup)
+				w1_next_pullup(dev_master, t_write);
 
-static inline int w1_convert_temp(u8 rom[9], u8 fid)
-{
-	int i;
+			w1_write_8(dev_master, W1_COPY_SCRATCHPAD);
 
-	for (i = 0; i < ARRAY_SIZE(w1_therm_families); ++i)
-		if (w1_therm_families[i].f->fid == fid)
-			return w1_therm_families[i].convert(rom);
+			if (strong_pullup) {
+				sleep_rem = msleep_interruptible(t_write);
+				if (sleep_rem != 0) {
+					ret = -EINTR;
+					goto mt_unlock;
+				}
+			}
+			ret = 0;
+		}
 
-	return 0;
+	}
+
+mt_unlock:
+	mutex_unlock(&dev_master->bus_mutex);
+dec_refcnt:
+	atomic_dec(THERM_REFCNT(sl->family_data));
+error:
+	return ret;
 }
 
-static ssize_t w1_slave_store(struct device *device,
-			      struct device_attribute *attr, const char *buf,
-			      size_t size)
+static int recall_eeprom(struct w1_slave *sl)
 {
-	int val, ret;
-	struct w1_slave *sl = dev_to_w1_slave(device);
-	int i;
+	struct w1_master *dev_master = sl->master;
+	int max_trying = W1_THERM_MAX_TRY;
+	int ret = -ENODEV;
 
-	ret = kstrtoint(buf, 0, &val);
-	if (ret)
-		return ret;
+	if (!sl->family_data)
+		goto error;
 
-	for (i = 0; i < ARRAY_SIZE(w1_therm_families); ++i) {
-		if (w1_therm_families[i].f->fid == sl->family->fid) {
-			/* zero value indicates to write current configuration to eeprom */
-			if (val == 0)
-				ret = w1_therm_families[i].eeprom(device);
-			else
-				ret = w1_therm_families[i].precision(device, val);
-			break;
+	/* prevent the slave from going away in sleep */
+	atomic_inc(THERM_REFCNT(sl->family_data));
+
+	if (!bus_mutex_lock(&dev_master->bus_mutex)) {
+		ret = -EAGAIN;	/* Didn't acquire the mutex */
+		goto dec_refcnt;
+	}
+
+	while (max_trying-- && ret) { /* ret should be 0 */
+		/* safe version to select slave */
+		if (!reset_select_slave(sl)) {
+
+			w1_write_8(dev_master, W1_RECALL_EEPROM);
+
+			ret = 1; /* Slave will pull line to 0 */
+			while (ret)
+				ret = 1 - w1_touch_bit(dev_master, 1);
 		}
+
 	}
-	return ret ? : size;
+
+	mutex_unlock(&dev_master->bus_mutex);
+
+dec_refcnt:
+	atomic_dec(THERM_REFCNT(sl->family_data));
+error:
+	return ret;
 }
 
-static ssize_t read_therm(struct device *device,
-			  struct w1_slave *sl, struct therm_info *info)
+static int read_powermode(struct w1_slave *sl)
 {
-	struct w1_master *dev = sl->master;
-	u8 external_power;
-	int ret, max_trying = 10;
-	u8 *family_data = sl->family_data;
+	struct w1_master *dev_master = sl->master;
+	int max_trying = W1_THERM_MAX_TRY;
+	int  ret = -ENODEV;
 
-	if (!family_data) {
-		ret = -ENODEV;
+	if (!sl->family_data)
 		goto error;
-	}
 
 	/* prevent the slave from going away in sleep */
-	atomic_inc(THERM_REFCNT(family_data));
+	atomic_inc(THERM_REFCNT(sl->family_data));
 
-	ret = mutex_lock_interruptible(&dev->bus_mutex);
-	if (ret != 0)
+	if (!bus_mutex_lock(&dev_master->bus_mutex)) {
+		ret = -EAGAIN;	/* Didn't acquire the mutex */
 		goto dec_refcnt;
+	}
 
-	memset(info->rom, 0, sizeof(info->rom));
+	while ((max_trying--) && (ret < 0)) {
+		/* safe version to select slave */
+		if (!reset_select_slave(sl)) {
+			w1_write_8(dev_master, W1_READ_PSUPPLY);
+			/*
+			 * Emit a read time slot and read only one bit,
+			 * 1 is externally powered,
+			 * 0 is parasite powered
+			 */
+			ret = w1_touch_bit(dev_master, 1);
+			/* ret should be either 1 either 0 */
+		}
+	}
+	mutex_unlock(&dev_master->bus_mutex);
 
-	while (max_trying--) {
+dec_refcnt:
+	atomic_dec(THERM_REFCNT(sl->family_data));
+error:
+	return ret;
+}
 
-		info->verdict = 0;
-		info->crc = 0;
+static int trigger_bulk_read(struct w1_master *dev_master)
+{
+	struct w1_slave *sl = NULL; /* used to iterate through slaves */
+	int max_trying = W1_THERM_MAX_TRY;
+	int t_conv = 0;
+	int ret = -ENODEV;
+	bool strong_pullup = false;
+
+	/*
+	 * Check whether there are parasite powered device on the bus,
+	 * and compute duration of conversion for these devices
+	 * so we can apply a strong pullup if required
+	 */
+	list_for_each_entry(sl, &dev_master->slist, w1_slave_entry) {
+		if (!sl->family_data)
+			goto error;
+		if (bulk_read_support(sl)) {
+			int t_cur = conversion_time(sl);
+
+			t_conv = t_cur > t_conv ? t_cur : t_conv;
+			strong_pullup = strong_pullup ||
+					(w1_strong_pullup == 2 ||
+					(!SLAVE_POWERMODE(sl) &&
+					w1_strong_pullup));
+		}
+	}
 
-		if (!w1_reset_select_slave(sl)) {
-			int count = 0;
-			unsigned int tm = 750;
-			unsigned long sleep_rem;
+	/*
+	 * t_conv is the max conversion time required on the bus
+	 * If its 0, no device support the bulk read feature
+	 */
+	if (!t_conv)
+		goto error;
 
-			w1_write_8(dev, W1_READ_PSUPPLY);
-			external_power = w1_read_8(dev);
+	if (!bus_mutex_lock(&dev_master->bus_mutex)) {
+		ret = -EAGAIN;	/* Didn't acquire the mutex */
+		goto error;
+	}
 
-			if (w1_reset_select_slave(sl))
-				continue;
+	while ((max_trying--) && (ret < 0)) { /* ret should be either 0 */
 
-			/* 750ms strong pullup (or delay) after the convert */
-			if (w1_strong_pullup == 2 ||
-					(!external_power && w1_strong_pullup))
-				w1_next_pullup(dev, tm);
+		if (!w1_reset_bus(dev_master)) {	/* Just reset the bus */
+			unsigned long sleep_rem;
 
-			w1_write_8(dev, W1_CONVERT_TEMP);
+			w1_write_8(dev_master, W1_SKIP_ROM);
 
-			if (external_power) {
-				mutex_unlock(&dev->bus_mutex);
+			if (strong_pullup)	/* Apply pullup if required */
+				w1_next_pullup(dev_master, t_conv);
 
-				sleep_rem = msleep_interruptible(tm);
-				if (sleep_rem != 0) {
-					ret = -EINTR;
-					goto dec_refcnt;
-				}
+			w1_write_8(dev_master, W1_CONVERT_TEMP);
 
-				ret = mutex_lock_interruptible(&dev->bus_mutex);
-				if (ret != 0)
-					goto dec_refcnt;
-			} else if (!w1_strong_pullup) {
-				sleep_rem = msleep_interruptible(tm);
+			/* set a flag to instruct that converT pending */
+			list_for_each_entry(sl,
+				&dev_master->slist, w1_slave_entry) {
+				if (bulk_read_support(sl))
+					SLAVE_CONVERT_TRIGGERED(sl) = -1;
+			}
+
+			if (strong_pullup) { /* some device need pullup */
+				sleep_rem = msleep_interruptible(t_conv);
 				if (sleep_rem != 0) {
 					ret = -EINTR;
 					goto mt_unlock;
 				}
-			}
-
-			if (!w1_reset_select_slave(sl)) {
-
-				w1_write_8(dev, W1_READ_SCRATCHPAD);
-				count = w1_read_block(dev, info->rom, 9);
-				if (count != 9) {
-					dev_warn(device, "w1_read_block() "
-						"returned %u instead of 9.\n",
-						count);
+				mutex_unlock(&dev_master->bus_mutex);
+			} else {
+				mutex_unlock(&dev_master->bus_mutex);
+				sleep_rem = msleep_interruptible(t_conv);
+				if (sleep_rem != 0) {
+					ret = -EINTR;
+					goto set_flag;
 				}
-
-				info->crc = w1_calc_crc8(info->rom, 8);
-
-				if (info->rom[8] == info->crc)
-					info->verdict = 1;
 			}
+			ret = 0;
+			goto set_flag;
 		}
-
-		if (info->verdict)
-			break;
 	}
 
 mt_unlock:
-	mutex_unlock(&dev->bus_mutex);
-dec_refcnt:
-	atomic_dec(THERM_REFCNT(family_data));
+	mutex_unlock(&dev_master->bus_mutex);
+set_flag:
+	/* set a flag to register convsersion is done */
+	list_for_each_entry(sl, &dev_master->slist, w1_slave_entry) {
+		if (bulk_read_support(sl))
+			SLAVE_CONVERT_TRIGGERED(sl) = 1;
+	}
 error:
 	return ret;
 }
 
+/* Sysfs Interface definition */
+
 static ssize_t w1_slave_show(struct device *device,
 			     struct device_attribute *attr, char *buf)
 {
@@ -572,43 +1272,405 @@ static ssize_t w1_slave_show(struct device *device,
 	u8 *family_data = sl->family_data;
 	int ret, i;
 	ssize_t c = PAGE_SIZE;
-	u8 fid = sl->family->fid;
 
-	ret = read_therm(device, sl, &info);
-	if (ret)
-		return ret;
+	if (bulk_read_support(sl)) {
+		if (SLAVE_CONVERT_TRIGGERED(sl) < 0) {
+			dev_dbg(device,
+				"%s: Conversion in progress, retry later\n",
+				__func__);
+			return 0;
+		} else if (SLAVE_CONVERT_TRIGGERED(sl) > 0) {
+			/* A bulk read has been issued, read the device RAM */
+			ret = read_scratchpad(sl, &info);
+			SLAVE_CONVERT_TRIGGERED(sl) = 0;
+		} else
+			ret = convert_t(sl, &info);
+	} else
+		ret = convert_t(sl, &info);
+
+	if (ret < 0) {
+		dev_dbg(device,
+			"%s: Temperature data may be corrupted. err=%d\n",
+			__func__, ret);
+		return 0;
+	}
 
 	for (i = 0; i < 9; ++i)
 		c -= snprintf(buf + PAGE_SIZE - c, c, "%02x ", info.rom[i]);
 	c -= snprintf(buf + PAGE_SIZE - c, c, ": crc=%02x %s\n",
 		      info.crc, (info.verdict) ? "YES" : "NO");
+
 	if (info.verdict)
 		memcpy(family_data, info.rom, sizeof(info.rom));
 	else
-		dev_warn(device, "Read failed CRC check\n");
+		dev_warn(device, "%s:Read failed CRC check\n", __func__);
 
 	for (i = 0; i < 9; ++i)
 		c -= snprintf(buf + PAGE_SIZE - c, c, "%02x ",
 			      ((u8 *)family_data)[i]);
 
 	c -= snprintf(buf + PAGE_SIZE - c, c, "t=%d\n",
-			w1_convert_temp(info.rom, fid));
+			temperature_from_RAM(sl, info.rom));
+
 	ret = PAGE_SIZE - c;
 	return ret;
 }
 
+static ssize_t w1_slave_store(struct device *device,
+			      struct device_attribute *attr, const char *buf,
+			      size_t size)
+{
+	int val, ret = 0;
+	struct w1_slave *sl = dev_to_w1_slave(device);
+
+	ret = kstrtoint(buf, 10, &val); /* converting user entry to int */
+
+	if (ret) {	/* conversion error */
+		dev_info(device,
+			"%s: conversion error. err= %d\n", __func__, ret);
+		return size;	/* return size to avoid call back again */
+	}
+
+	if ((!sl->family_data) || (!SLAVE_SPECIFIC_FUNC(sl))) {
+		dev_info(device,
+			"%s: Device not supported by the driver\n", __func__);
+		return size;  /* No device family */
+	}
+
+	if (val == 0)	/* val=0 : trigger a EEPROM save */
+		ret = copy_scratchpad(sl);
+	else {
+		if (SLAVE_SPECIFIC_FUNC(sl)->set_resolution)
+			ret = SLAVE_SPECIFIC_FUNC(sl)->set_resolution(sl, val);
+	}
+
+	if (ret) {
+		dev_info(device,
+			"%s: writing error %d\n", __func__, ret);
+		/* return size to avoid call back again */
+	} else
+		SLAVE_RESOLUTION(sl) = val;
+
+	return size; /* always return size to avoid infinite calling */
+}
+
+static ssize_t temperature_show(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+	struct therm_info info;
+	int ret = 0;
+
+	if ((!sl->family_data) || (!SLAVE_SPECIFIC_FUNC(sl))) {
+		dev_info(device,
+			"%s: Device not supported by the driver\n", __func__);
+		return 0;  /* No device family */
+	}
+
+	if (bulk_read_support(sl)) {
+		if (SLAVE_CONVERT_TRIGGERED(sl) < 0) {
+			dev_dbg(device,
+				"%s: Conversion in progress, retry later\n",
+				__func__);
+			return 0;
+		} else if (SLAVE_CONVERT_TRIGGERED(sl) > 0) {
+			/* A bulk read has been issued, read the device RAM */
+			ret = read_scratchpad(sl, &info);
+			SLAVE_CONVERT_TRIGGERED(sl) = 0;
+		} else
+			ret = convert_t(sl, &info);
+	} else
+		ret = convert_t(sl, &info);
+
+	if (ret < 0) {
+		dev_dbg(device,
+			"%s: Temperature data may be corrupted. err=%d\n",
+			__func__, ret);
+		return 0;
+	}
+
+	return sprintf(buf, "%d\n", temperature_from_RAM(sl, info.rom));
+}
+
+static ssize_t ext_power_show(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+
+	if (!sl->family_data) {
+		dev_info(device,
+			"%s: Device not supported by the driver\n", __func__);
+		return 0;  /* No device family */
+	}
+
+	/* Getting the power mode of the device {external, parasite} */
+	SLAVE_POWERMODE(sl) = read_powermode(sl);
+
+	if (SLAVE_POWERMODE(sl) < 0) {
+		dev_dbg(device,
+			"%s: Power_mode may be corrupted. err=%d\n",
+			__func__, SLAVE_POWERMODE(sl));
+	}
+	return sprintf(buf, "%d\n", SLAVE_POWERMODE(sl));
+}
+
+static ssize_t resolution_show(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+
+	if ((!sl->family_data) || (!SLAVE_SPECIFIC_FUNC(sl))) {
+		dev_info(device,
+			"%s: Device not supported by the driver\n", __func__);
+		return 0;  /* No device family */
+	}
+
+	/* get the correct function depending on the device */
+	SLAVE_RESOLUTION(sl) = SLAVE_SPECIFIC_FUNC(sl)->get_resolution(sl);
+	if (SLAVE_RESOLUTION(sl) < 0) {
+		dev_dbg(device,
+			"%s: Resolution may be corrupted. err=%d\n",
+			__func__, SLAVE_RESOLUTION(sl));
+	}
+
+	return sprintf(buf, "%d\n", SLAVE_RESOLUTION(sl));
+}
+
+static ssize_t resolution_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+	int val;
+	int ret = 0;
+
+	ret = kstrtoint(buf, 10, &val); /* converting user entry to int */
+
+	if (ret) {	/* conversion error */
+		dev_info(device,
+			"%s: conversion error. err= %d\n", __func__, ret);
+		return size;	/* return size to avoid call back again */
+	}
+
+	if ((!sl->family_data) || (!SLAVE_SPECIFIC_FUNC(sl))) {
+		dev_info(device,
+			"%s: Device not supported by the driver\n", __func__);
+		return size;  /* No device family */
+	}
+
+	/*
+	 * Don't deal with the val enterd by user,
+	 * only device knows what is correct or not
+	 */
+
+	/* get the correct function depending on the device */
+	ret = SLAVE_SPECIFIC_FUNC(sl)->set_resolution(sl, val);
+
+	if (ret) {
+		dev_info(device,
+			"%s: writing error %d\n", __func__, ret);
+		/* return size to avoid call back again */
+	} else
+		SLAVE_RESOLUTION(sl) = val;
+
+	return size;
+}
+
+static ssize_t eeprom_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+	int ret = -EINVAL; /* Invalid argument */
+
+	if (size == sizeof(EEPROM_CMD_WRITE)) {
+		if (!strncmp(buf, EEPROM_CMD_WRITE, sizeof(EEPROM_CMD_WRITE)-1))
+			ret = copy_scratchpad(sl);
+	} else if (size == sizeof(EEPROM_CMD_READ)) {
+		if (!strncmp(buf, EEPROM_CMD_READ, sizeof(EEPROM_CMD_READ)-1))
+			ret = recall_eeprom(sl);
+	}
+
+	if (ret)
+		dev_info(device, "%s: error in process %d\n", __func__, ret);
+
+	return size;
+}
+
+static ssize_t alarms_show(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+	int ret;
+	s8 th = 0, tl = 0;
+	struct therm_info scratchpad;
+
+	ret = read_scratchpad(sl, &scratchpad);
+
+	if (!ret)	{
+		th = scratchpad.rom[2]; /* TH is byte 2 */
+		tl = scratchpad.rom[3]; /* TL is byte 3 */
+	} else {
+		dev_info(device,
+			"%s: error reading alarms register %d\n",
+			__func__, ret);
+	}
+
+	return sprintf(buf, "%hd %hd\n", tl, th);
+}
+
+static ssize_t alarms_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+	struct therm_info info;
+	u8 new_config_register[3];	/* array of data to be written */
+	int temp, ret;
+	char *token = NULL;
+	s8 tl, th, tt;	/* 1 byte per value + temp ring order */
+	char *p_args, *orig;
+
+	p_args = orig = kmalloc(size, GFP_KERNEL);
+	/* Safe string copys as buf is const */
+	if (!p_args) {
+		dev_warn(device,
+			"%s: error unable to allocate memory %d\n",
+			__func__, -ENOMEM);
+		return size;
+	}
+	strcpy(p_args, buf);
+
+	/* Split string using space char */
+	token = strsep(&p_args, " ");
+
+	if (!token)	{
+		dev_info(device,
+			"%s: error parsing args %d\n", __func__, -EINVAL);
+		goto free_m;
+	}
+
+	/* Convert 1st entry to int */
+	ret = kstrtoint (token, 10, &temp);
+	if (ret) {
+		dev_info(device,
+			"%s: error parsing args %d\n", __func__, ret);
+		goto free_m;
+	}
+
+	tl = int_to_short(temp);
+
+	/* Split string using space char */
+	token = strsep(&p_args, " ");
+	if (!token)	{
+		dev_info(device,
+			"%s: error parsing args %d\n", __func__, -EINVAL);
+		goto free_m;
+	}
+	/* Convert 2nd entry to int */
+	ret = kstrtoint (token, 10, &temp);
+	if (ret) {
+		dev_info(device,
+			"%s: error parsing args %d\n", __func__, ret);
+		goto free_m;
+	}
+
+	/* Prepare to cast to short by eliminating out of range values */
+	th = int_to_short(temp);
+
+	/* Reorder if required th and tl */
+	if (tl > th) {
+		tt = tl; tl = th; th = tt;
+	}
+
+	/*
+	 * Read the scratchpad to change only the required bits
+	 * (th : byte 2 - tl: byte 3)
+	 */
+	ret = read_scratchpad(sl, &info);
+	if (!ret) {
+		new_config_register[0] = th;	/* Byte 2 */
+		new_config_register[1] = tl;	/* Byte 3 */
+		new_config_register[2] = info.rom[4];/* Byte 4 */
+	} else {
+		dev_info(device,
+			"%s: error reading from the slave device %d\n",
+			__func__, ret);
+		goto free_m;
+	}
+
+	/* Write data in the device RAM */
+	if (!SLAVE_SPECIFIC_FUNC(sl)) {
+		dev_info(device,
+			"%s: Device not supported by the driver %d\n",
+			__func__, -ENODEV);
+		goto free_m;
+	}
+
+	ret = SLAVE_SPECIFIC_FUNC(sl)->write_data(sl, new_config_register);
+	if (ret)
+		dev_info(device,
+			"%s: error writing to the slave device %d\n",
+			__func__, ret);
+
+free_m:
+	/* free allocated memory */
+	kfree(orig);
+
+	return size;
+}
+
+static ssize_t therm_bulk_read_store(struct device *device,
+	struct device_attribute *attr, const char *buf, size_t size)
+{
+	struct w1_master *dev_master = dev_to_w1_master(device);
+	int ret = -EINVAL; /* Invalid argument */
+
+	if (size == sizeof(BULK_TRIGGER_CMD))
+		if (!strncmp(buf, BULK_TRIGGER_CMD,
+				sizeof(BULK_TRIGGER_CMD)-1))
+			ret = trigger_bulk_read(dev_master);
+
+	if (ret)
+		dev_info(device,
+			"%s: unable to trigger a bulk read on the bus. err=%d\n",
+			__func__, ret);
+
+	return size;
+}
+
+static ssize_t therm_bulk_read_show(struct device *device,
+	struct device_attribute *attr, char *buf)
+{
+	struct w1_master *dev_master = dev_to_w1_master(device);
+	struct w1_slave *sl = NULL;
+	int ret = 0;
+
+	list_for_each_entry(sl, &dev_master->slist, w1_slave_entry) {
+		if (sl->family_data) {
+			if (bulk_read_support(sl)) {
+				if (SLAVE_CONVERT_TRIGGERED(sl) == -1) {
+					ret = -1;
+					goto show_result;
+				}
+				if (SLAVE_CONVERT_TRIGGERED(sl) == 1)
+					/* continue to check other slaves */
+					ret = 1;
+			}
+		}
+	}
+show_result:
+	return sprintf(buf, "%d\n", ret);
+}
+
 #if IS_REACHABLE(CONFIG_HWMON)
 static int w1_read_temp(struct device *device, u32 attr, int channel,
 			long *val)
 {
 	struct w1_slave *sl = dev_get_drvdata(device);
 	struct therm_info info;
-	u8 fid = sl->family->fid;
 	int ret;
 
 	switch (attr) {
 	case hwmon_temp_input:
-		ret = read_therm(device, sl, &info);
+		ret = convert_t(sl, &info);
 		if (ret)
 			return ret;
 
@@ -617,7 +1679,7 @@ static int w1_read_temp(struct device *device, u32 attr, int channel,
 			return ret;
 		}
 
-		*val = w1_convert_temp(info.rom, fid);
+		*val = temperature_from_RAM(sl, info.rom);
 		ret = 0;
 		break;
 	default:
@@ -666,7 +1728,7 @@ static ssize_t w1_seq_show(struct device *device,
 	if (ack != W1_42_SUCCESS_CONFIRM_BYTE)
 		goto error;
 
-	/* In case the bus fails to send 0xFF, limit*/
+	/* In case the bus fails to send 0xFF, limit */
 	for (i = 0; i <= 64; i++) {
 		if (w1_reset_bus(sl->master))
 			goto error;
diff --git a/drivers/w1/w1_netlink.h b/drivers/w1/w1_netlink.h
index 3041092e84b3..449680a61569 100644
--- a/drivers/w1/w1_netlink.h
+++ b/drivers/w1/w1_netlink.h
@@ -73,7 +73,7 @@ struct w1_netlink_msg
 			__u32		res;
 		} mst;
 	} id;
-	__u8				data[0];
+	__u8				data[];
 };
 
 /**
@@ -122,7 +122,7 @@ struct w1_netlink_cmd
 	__u8				cmd;
 	__u8				res;
 	__u16				len;
-	__u8				data[0];
+	__u8				data[];
 };
 
 #ifdef __KERNEL__
diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
index b739c476955b..4f4687c46d38 100644
--- a/drivers/watchdog/Kconfig
+++ b/drivers/watchdog/Kconfig
@@ -6,7 +6,7 @@
 
 menuconfig WATCHDOG
 	bool "Watchdog Timer Support"
-	---help---
+	help
 	  If you say Y here (and to one of the following options) and create a
 	  character special file /dev/watchdog with major number 10 and minor
 	  number 130 using mknod ("man mknod"), you will get a watchdog, i.e.:
@@ -32,7 +32,7 @@ if WATCHDOG
 
 config WATCHDOG_CORE
 	tristate "WatchDog Timer Driver Core"
-	---help---
+	help
 	  Say Y here if you want to use the new watchdog timer driver core.
 	  This driver provides a framework for all watchdog timer drivers
 	  and gives them the /dev/watchdog interface (and later also the
@@ -678,6 +678,7 @@ config TS4800_WATCHDOG
 config TS72XX_WATCHDOG
 	tristate "TS-72XX SBC Watchdog"
 	depends on MACH_TS72XX || COMPILE_TEST
+	select WATCHDOG_CORE
 	help
 	  Technologic Systems TS-7200, TS-7250 and TS-7260 boards have
 	  watchdog timer implemented in a external CPLD chip. Say Y here
@@ -867,6 +868,19 @@ config DIGICOLOR_WATCHDOG
 	  To compile this driver as a module, choose M here: the
 	  module will be called digicolor_wdt.
 
+config ARM_SMC_WATCHDOG
+	tristate "ARM Secure Monitor Call based watchdog support"
+	depends on ARM || ARM64
+	depends on OF
+	depends on HAVE_ARM_SMCCC
+	select WATCHDOG_CORE
+	help
+	  Say Y here to include support for a watchdog timer
+	  implemented by the EL3 Secure Monitor on ARM platforms.
+	  Requires firmware support.
+	  To compile this driver as a module, choose M here: the
+	  module will be called arm_smc_wdt.
+
 config LPC18XX_WATCHDOG
 	tristate "LPC18xx/43xx Watchdog"
 	depends on ARCH_LPC18XX || COMPILE_TEST
@@ -995,7 +1009,7 @@ config PM8916_WATCHDOG
 config ACQUIRE_WDT
 	tristate "Acquire SBC Watchdog Timer"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the hardware watchdog on Single Board
 	  Computers produced by Acquire Inc (and others). This watchdog
 	  simply watches your kernel to make sure it doesn't freeze, and if
@@ -1018,7 +1032,7 @@ config ADVANTECH_WDT
 config ALIM1535_WDT
 	tristate "ALi M1535 PMU Watchdog Timer"
 	depends on X86 && PCI
-	---help---
+	help
 	  This is the driver for the hardware watchdog on the ALi M1535 PMU.
 
 	  To compile this driver as a module, choose M here: the
@@ -1064,7 +1078,7 @@ config SP5100_TCO
 	tristate "AMD/ATI SP5100 TCO Timer/Watchdog"
 	depends on X86 && PCI
 	select WATCHDOG_CORE
-	---help---
+	help
 	  Hardware watchdog driver for the AMD/ATI SP5100 chipset. The TCO
 	  (Total Cost of Ownership) timer is a watchdog timer that will reboot
 	  the machine after its expiration. The expiration time can be
@@ -1102,7 +1116,7 @@ config SC520_WDT
 config SBC_FITPC2_WATCHDOG
 	tristate "Compulab SBC-FITPC2 watchdog"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the built-in watchdog timer on the fit-PC2,
 	  fit-PC2i, CM-iAM single-board computers made by Compulab.
 
@@ -1132,7 +1146,7 @@ config EUROTECH_WDT
 config IB700_WDT
 	tristate "IB700 SBC Watchdog Timer"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the hardware watchdog on the IB700 Single
 	  Board Computer produced by TMC Technology (www.tmc-uk.com). This watchdog
 	  simply watches your kernel to make sure it doesn't freeze, and if
@@ -1170,7 +1184,7 @@ config I6300ESB_WDT
 	tristate "Intel 6300ESB Timer/Watchdog"
 	depends on PCI
 	select WATCHDOG_CORE
-	---help---
+	help
 	  Hardware driver for the watchdog timer built into the Intel
 	  6300ESB controller hub.
 
@@ -1183,7 +1197,7 @@ config IE6XX_WDT
 	select WATCHDOG_CORE
 	select MFD_CORE
 	select LPC_SCH
-	---help---
+	help
 	  Hardware driver for the watchdog timer built into the Intel
 	  Atom E6XX (TunnelCreek) processor.
 
@@ -1193,7 +1207,7 @@ config IE6XX_WDT
 config INTEL_SCU_WATCHDOG
 	bool "Intel SCU Watchdog for Mobile Platforms"
 	depends on X86_INTEL_MID
-	---help---
+	help
 	  Hardware driver for the watchdog time built into the Intel SCU
 	  for Intel Mobile Platforms.
 
@@ -1203,7 +1217,7 @@ config INTEL_MID_WATCHDOG
 	tristate "Intel MID Watchdog Timer"
 	depends on X86_INTEL_MID
 	select WATCHDOG_CORE
-	---help---
+	help
 	  Watchdog timer driver built into the Intel SCU for Intel MID
 	  Platforms.
 
@@ -1220,7 +1234,7 @@ config ITCO_WDT
 	depends on MFD_INTEL_PMC_BXT || !MFD_INTEL_PMC_BXT
 	select LPC_ICH if !EXPERT
 	select I2C_I801 if !EXPERT && I2C
-	---help---
+	help
 	  Hardware driver for the intel TCO timer based watchdog devices.
 	  These drivers are included in the Intel 82801 I/O Controller
 	  Hub family (from ICH0 up to ICH10) and in the Intel 63xxESB
@@ -1241,7 +1255,7 @@ config ITCO_WDT
 config ITCO_VENDOR_SUPPORT
 	bool "Intel TCO Timer/Watchdog Specific Vendor Support"
 	depends on ITCO_WDT
-	---help---
+	help
 	  Add vendor specific support to the intel TCO timer based watchdog
 	  devices. At this moment we only have additional support for some
 	  SuperMicro Inc. motherboards.
@@ -1249,7 +1263,7 @@ config ITCO_VENDOR_SUPPORT
 config IT8712F_WDT
 	tristate "IT8712F (Smart Guardian) Watchdog Timer"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the built-in watchdog timer on the IT8712F
 	  Super I/0 chipset used on many motherboards.
 
@@ -1263,7 +1277,7 @@ config IT87_WDT
 	tristate "IT87 Watchdog Timer"
 	depends on X86
 	select WATCHDOG_CORE
-	---help---
+	help
 	  This is the driver for the hardware watchdog on the ITE IT8607,
 	  IT8620, IT8622, IT8625, IT8628, IT8655, IT8665, IT8686, IT8702,
 	  IT8712, IT8716, IT8718, IT8720, IT8721, IT8726, IT8728, and
@@ -1333,7 +1347,7 @@ config SCx200_WDT
 config PC87413_WDT
 	tristate "NS PC87413 watchdog"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the hardware watchdog on the PC87413 chipset
 	  This watchdog simply watches your kernel to make sure it doesn't
 	  freeze, and if it does, it reboots your computer after a certain
@@ -1347,7 +1361,7 @@ config PC87413_WDT
 config NV_TCO
 	tristate "nVidia TCO Timer/Watchdog"
 	depends on X86 && PCI
-	---help---
+	help
 	  Hardware driver for the TCO timer built into the nVidia Hub family
 	  (such as the MCP51).  The TCO (Total Cost of Ownership) timer is a
 	  watchdog timer that will reboot the machine after its second
@@ -1390,7 +1404,7 @@ config 60XX_WDT
 config SBC8360_WDT
 	tristate "SBC8360 Watchdog Timer"
 	depends on X86_32
-	---help---
+	help
 
 	  This is the driver for the hardware watchdog on the SBC8360 Single
 	  Board Computer produced by Axiomtek Co., Ltd. (www.axiomtek.com).
@@ -1403,7 +1417,7 @@ config SBC8360_WDT
 config SBC7240_WDT
 	tristate "SBC Nano 7240 Watchdog Timer"
 	depends on X86_32 && !UML
-	---help---
+	help
 	  This is the driver for the hardware watchdog found on the IEI
 	  single board computers EPIC Nano 7240 (and likely others). This
 	  watchdog simply watches your kernel to make sure it doesn't freeze,
@@ -1416,7 +1430,7 @@ config SBC7240_WDT
 config CPU5_WDT
 	tristate "SMA CPU5 Watchdog"
 	depends on X86
-	---help---
+	help
 	  TBD.
 	  To compile this driver as a module, choose M here: the
 	  module will be called cpu5wdt.
@@ -1424,7 +1438,7 @@ config CPU5_WDT
 config SMSC_SCH311X_WDT
 	tristate "SMSC SCH311X Watchdog Timer"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the hardware watchdog timer on the
 	  SMSC SCH3112, SCH3114 and SCH3116 Super IO chipset
 	  (LPC IO with 8042 KBC, Reset Generation, HWM and multiple
@@ -1436,7 +1450,7 @@ config SMSC_SCH311X_WDT
 config SMSC37B787_WDT
 	tristate "Winbond SMsC37B787 Watchdog Timer"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the hardware watchdog component on the
 	  Winbond SMsC37B787 chipset as used on the NetRunner Mainboard
 	  from Vision Systems and maybe others.
@@ -1470,7 +1484,7 @@ config VIA_WDT
 	tristate "VIA Watchdog Timer"
 	depends on X86 && PCI
 	select WATCHDOG_CORE
-	---help---
+	help
 	This is the driver for the hardware watchdog timer on VIA
 	southbridge chipset CX700, VX800/VX820 or VX855/VX875.
 
@@ -1483,7 +1497,7 @@ config W83627HF_WDT
 	tristate "Watchdog timer for W83627HF/W83627DHG and compatibles"
 	depends on X86
 	select WATCHDOG_CORE
-	---help---
+	help
 	  This is the driver for the hardware watchdog on the following
 	  Super I/O chips.
 		W83627DHG/DHG-P/EHF/EHG/F/G/HF/S/SF/THF/UHG/UG
@@ -1512,7 +1526,7 @@ config W83627HF_WDT
 config W83877F_WDT
 	tristate "W83877F (EMACS) Watchdog Timer"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the hardware watchdog on the W83877F chipset
 	  as used in EMACS PC-104 motherboards (and likely others).  This
 	  watchdog simply watches your kernel to make sure it doesn't freeze,
@@ -1527,7 +1541,7 @@ config W83877F_WDT
 config W83977F_WDT
 	tristate "W83977F (PCM-5335) Watchdog Timer"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the hardware watchdog on the W83977F I/O chip
 	  as used in AAEON's PCM-5335 SBC (and likely others).  This
 	  watchdog simply watches your kernel to make sure it doesn't freeze,
@@ -1540,7 +1554,7 @@ config W83977F_WDT
 config MACHZ_WDT
 	tristate "ZF MachZ Watchdog"
 	depends on X86
-	---help---
+	help
 	  If you are using a ZF Micro MachZ processor, say Y here, otherwise
 	  N.  This is the driver for the watchdog timer built-in on that
 	  processor using ZF-Logic interface.  This watchdog simply watches
@@ -1553,7 +1567,7 @@ config MACHZ_WDT
 config SBC_EPX_C3_WATCHDOG
 	tristate "Winsystems SBC EPX-C3 watchdog"
 	depends on X86
-	---help---
+	help
 	  This is the driver for the built-in watchdog timer on the EPX-C3
 	  Single-board computer made by Winsystems, Inc.
 
@@ -1575,7 +1589,7 @@ config INTEL_MEI_WDT
 	tristate "Intel MEI iAMT Watchdog"
 	depends on INTEL_MEI && X86
 	select WATCHDOG_CORE
-	---help---
+	help
 	  A device driver for the Intel MEI iAMT watchdog.
 
 	  The Intel AMT Watchdog is an OS Health (Hang/Crash) watchdog.
@@ -1590,7 +1604,7 @@ config NI903X_WDT
 	tristate "NI 903x/913x Watchdog"
 	depends on X86 && ACPI
 	select WATCHDOG_CORE
-	---help---
+	help
 	  This is the driver for the watchdog timer on the National Instruments
 	  903x/913x real-time controllers.
 
@@ -1601,7 +1615,7 @@ config NIC7018_WDT
 	tristate "NIC7018 Watchdog"
 	depends on X86 && ACPI
 	select WATCHDOG_CORE
-	---help---
+	help
 	  Support for National Instruments NIC7018 Watchdog.
 
 	  To compile this driver as a module, choose M here: the module will be
@@ -1851,7 +1865,7 @@ config PIC32_DMT
 config GEF_WDT
 	tristate "GE Watchdog Timer"
 	depends on GE_FPGA
-	---help---
+	help
 	  Watchdog timer found in a number of GE single board computers.
 
 config MPC5200_WDT
@@ -1889,7 +1903,7 @@ config BOOKE_WDT
 	tristate "PowerPC Book-E Watchdog Timer"
 	depends on BOOKE || 4xx
 	select WATCHDOG_CORE
-	---help---
+	help
 	  Watchdog driver for PowerPC Book-E chips, such as the Freescale
 	  MPC85xx SOCs and the IBM PowerPC 440.
 
@@ -1980,7 +1994,7 @@ config SH_WDT
 config WATCHDOG_CP1XXX
 	tristate "CP1XXX Hardware Watchdog support"
 	depends on SPARC64 && PCI
-	---help---
+	help
 	  This is the driver for the hardware watchdog timers present on
 	  Sun Microsystems CompactPCI models CP1400 and CP1500.
 
@@ -2037,7 +2051,7 @@ comment "ISA-based Watchdog Cards"
 config PCWATCHDOG
 	tristate "Berkshire Products ISA-PC Watchdog"
 	depends on ISA
-	---help---
+	help
 	  This is the driver for the Berkshire Products ISA-PC Watchdog card.
 	  This card simply watches your kernel to make sure it doesn't freeze,
 	  and if it does, it reboots your computer after a certain amount of
@@ -2053,7 +2067,7 @@ config PCWATCHDOG
 config MIXCOMWD
 	tristate "Mixcom Watchdog"
 	depends on ISA
-	---help---
+	help
 	  This is a driver for the Mixcom hardware watchdog cards.  This
 	  watchdog simply watches your kernel to make sure it doesn't freeze,
 	  and if it does, it reboots your computer after a certain amount of
@@ -2067,7 +2081,7 @@ config MIXCOMWD
 config WDT
 	tristate "WDT Watchdog timer"
 	depends on ISA
-	---help---
+	help
 	  If you have a WDT500P or WDT501P watchdog board, say Y here,
 	  otherwise N. It is not possible to probe for this board, which means
 	  that you have to inform the kernel about the IO port and IRQ that
@@ -2086,7 +2100,7 @@ comment "PCI-based Watchdog Cards"
 config PCIPCWATCHDOG
 	tristate "Berkshire Products PCI-PC Watchdog"
 	depends on PCI
-	---help---
+	help
 	  This is the driver for the Berkshire Products PCI-PC Watchdog card.
 	  This card simply watches your kernel to make sure it doesn't freeze,
 	  and if it does, it reboots your computer after a certain amount of
@@ -2101,7 +2115,7 @@ config PCIPCWATCHDOG
 config WDTPCI
 	tristate "PCI-WDT500/501 Watchdog timer"
 	depends on PCI
-	---help---
+	help
 	  If you have a PCI-WDT500/501 watchdog board, say Y here, otherwise N.
 
 	  If you have a PCI-WDT501 watchdog board then you can enable the
@@ -2124,7 +2138,7 @@ comment "USB-based Watchdog Cards"
 config USBPCWATCHDOG
 	tristate "Berkshire Products USB-PC Watchdog"
 	depends on USB
-	---help---
+	help
 	  This is the driver for the Berkshire Products USB-PC Watchdog card.
 	  This card simply watches your kernel to make sure it doesn't freeze,
 	  and if it does, it reboots your computer after a certain amount of
diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile
index 6de2e4ceef19..97bed1d3d97c 100644
--- a/drivers/watchdog/Makefile
+++ b/drivers/watchdog/Makefile
@@ -94,6 +94,7 @@ obj-$(CONFIG_UNIPHIER_WATCHDOG) += uniphier_wdt.o
 obj-$(CONFIG_RTD119X_WATCHDOG) += rtd119x_wdt.o
 obj-$(CONFIG_SPRD_WATCHDOG) += sprd_wdt.o
 obj-$(CONFIG_PM8916_WATCHDOG) += pm8916_wdt.o
+obj-$(CONFIG_ARM_SMC_WATCHDOG) += arm_smc_wdt.o
 
 # X86 (i386 + ia64 + x86_64) Architecture
 obj-$(CONFIG_ACQUIRE_WDT) += acquirewdt.o
diff --git a/drivers/watchdog/arm_smc_wdt.c b/drivers/watchdog/arm_smc_wdt.c
new file mode 100644
index 000000000000..8f3d0c3a005f
--- /dev/null
+++ b/drivers/watchdog/arm_smc_wdt.c
@@ -0,0 +1,188 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ARM Secure Monitor Call watchdog driver
+ *
+ * Copyright 2020 Google LLC.
+ * Julius Werner <jwerner@chromium.org>
+ * Based on mtk_wdt.c
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+#include <linux/watchdog.h>
+#include <uapi/linux/psci.h>
+
+#define DRV_NAME		"arm_smc_wdt"
+#define DRV_VERSION		"1.0"
+
+enum smcwd_call {
+	SMCWD_INIT		= 0,
+	SMCWD_SET_TIMEOUT	= 1,
+	SMCWD_ENABLE		= 2,
+	SMCWD_PET		= 3,
+	SMCWD_GET_TIMELEFT	= 4,
+};
+
+static bool nowayout = WATCHDOG_NOWAYOUT;
+static unsigned int timeout;
+
+static int smcwd_call(struct watchdog_device *wdd, enum smcwd_call call,
+		      unsigned long arg, struct arm_smccc_res *res)
+{
+	struct arm_smccc_res local_res;
+
+	if (!res)
+		res = &local_res;
+
+	arm_smccc_smc((u32)(uintptr_t)watchdog_get_drvdata(wdd), call, arg, 0,
+		      0, 0, 0, 0, res);
+
+	if (res->a0 == PSCI_RET_NOT_SUPPORTED)
+		return -ENODEV;
+	if (res->a0 == PSCI_RET_INVALID_PARAMS)
+		return -EINVAL;
+	if (res->a0 != PSCI_RET_SUCCESS)
+		return -EIO;
+	return 0;
+}
+
+static int smcwd_ping(struct watchdog_device *wdd)
+{
+	return smcwd_call(wdd, SMCWD_PET, 0, NULL);
+}
+
+static unsigned int smcwd_get_timeleft(struct watchdog_device *wdd)
+{
+	struct arm_smccc_res res;
+
+	smcwd_call(wdd, SMCWD_GET_TIMELEFT, 0, &res);
+	if (res.a0)
+		return 0;
+	return res.a1;
+}
+
+static int smcwd_set_timeout(struct watchdog_device *wdd, unsigned int timeout)
+{
+	int res;
+
+	res = smcwd_call(wdd, SMCWD_SET_TIMEOUT, timeout, NULL);
+	if (!res)
+		wdd->timeout = timeout;
+	return res;
+}
+
+static int smcwd_stop(struct watchdog_device *wdd)
+{
+	return smcwd_call(wdd, SMCWD_ENABLE, 0, NULL);
+}
+
+static int smcwd_start(struct watchdog_device *wdd)
+{
+	return smcwd_call(wdd, SMCWD_ENABLE, 1, NULL);
+}
+
+static const struct watchdog_info smcwd_info = {
+	.identity	= DRV_NAME,
+	.options	= WDIOF_SETTIMEOUT |
+			  WDIOF_KEEPALIVEPING |
+			  WDIOF_MAGICCLOSE,
+};
+
+static const struct watchdog_ops smcwd_ops = {
+	.start		= smcwd_start,
+	.stop		= smcwd_stop,
+	.ping		= smcwd_ping,
+	.set_timeout	= smcwd_set_timeout,
+};
+
+static const struct watchdog_ops smcwd_timeleft_ops = {
+	.start		= smcwd_start,
+	.stop		= smcwd_stop,
+	.ping		= smcwd_ping,
+	.set_timeout	= smcwd_set_timeout,
+	.get_timeleft	= smcwd_get_timeleft,
+};
+
+static int smcwd_probe(struct platform_device *pdev)
+{
+	struct watchdog_device *wdd;
+	int err;
+	struct arm_smccc_res res;
+	u32 smc_func_id;
+
+	wdd = devm_kzalloc(&pdev->dev, sizeof(*wdd), GFP_KERNEL);
+	if (!wdd)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, wdd);
+
+	if (of_property_read_u32(pdev->dev.of_node, "arm,smc-id",
+				 &smc_func_id))
+		smc_func_id = 0x82003D06;
+	watchdog_set_drvdata(wdd, (void *)(uintptr_t)smc_func_id);
+
+	err = smcwd_call(wdd, SMCWD_INIT, 0, &res);
+	if (err < 0)
+		return err;
+
+	wdd->info = &smcwd_info;
+	/* get_timeleft is optional */
+	if (smcwd_call(wdd, SMCWD_GET_TIMELEFT, 0, NULL))
+		wdd->ops = &smcwd_ops;
+	else
+		wdd->ops = &smcwd_timeleft_ops;
+	wdd->timeout = res.a2;
+	wdd->max_timeout = res.a2;
+	wdd->min_timeout = res.a1;
+	wdd->parent = &pdev->dev;
+
+	watchdog_stop_on_reboot(wdd);
+	watchdog_stop_on_unregister(wdd);
+	watchdog_set_nowayout(wdd, nowayout);
+	watchdog_init_timeout(wdd, timeout, &pdev->dev);
+	err = smcwd_set_timeout(wdd, wdd->timeout);
+	if (err)
+		return err;
+
+	err = devm_watchdog_register_device(&pdev->dev, wdd);
+	if (err)
+		return err;
+
+	dev_info(&pdev->dev,
+		 "Watchdog registered (timeout=%d sec, nowayout=%d)\n",
+		 wdd->timeout, nowayout);
+
+	return 0;
+}
+
+static const struct of_device_id smcwd_dt_ids[] = {
+	{ .compatible = "arm,smc-wdt" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, smcwd_dt_ids);
+
+static struct platform_driver smcwd_driver = {
+	.probe		= smcwd_probe,
+	.driver		= {
+		.name		= DRV_NAME,
+		.of_match_table	= smcwd_dt_ids,
+	},
+};
+
+module_platform_driver(smcwd_driver);
+
+module_param(timeout, uint, 0);
+MODULE_PARM_DESC(timeout, "Watchdog heartbeat in seconds");
+
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
+			__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Julius Werner <jwerner@chromium.org>");
+MODULE_DESCRIPTION("ARM Secure Monitor Call Watchdog Driver");
+MODULE_VERSION(DRV_VERSION);
diff --git a/drivers/watchdog/da9062_wdt.c b/drivers/watchdog/da9062_wdt.c
index 0ad15d55071c..706fb09c2f24 100644
--- a/drivers/watchdog/da9062_wdt.c
+++ b/drivers/watchdog/da9062_wdt.c
@@ -35,6 +35,15 @@ struct da9062_watchdog {
 	bool use_sw_pm;
 };
 
+static unsigned int da9062_wdt_read_timeout(struct da9062_watchdog *wdt)
+{
+	unsigned int val;
+
+	regmap_read(wdt->hw->regmap, DA9062AA_CONTROL_D, &val);
+
+	return wdt_timeout[val & DA9062AA_TWDSCALE_MASK];
+}
+
 static unsigned int da9062_wdt_timeout_to_sel(unsigned int secs)
 {
 	unsigned int i;
@@ -58,11 +67,6 @@ static int da9062_wdt_update_timeout_register(struct da9062_watchdog *wdt,
 					      unsigned int regval)
 {
 	struct da9062 *chip = wdt->hw;
-	int ret;
-
-	ret = da9062_reset_watchdog_timer(wdt);
-	if (ret)
-		return ret;
 
 	regmap_update_bits(chip->regmap,
 				  DA9062AA_CONTROL_D,
@@ -183,7 +187,7 @@ MODULE_DEVICE_TABLE(of, da9062_compatible_id_table);
 static int da9062_wdt_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	int ret;
+	unsigned int timeout;
 	struct da9062 *chip;
 	struct da9062_watchdog *wdt;
 
@@ -213,11 +217,19 @@ static int da9062_wdt_probe(struct platform_device *pdev)
 	watchdog_set_drvdata(&wdt->wdtdev, wdt);
 	dev_set_drvdata(dev, &wdt->wdtdev);
 
-	ret = devm_watchdog_register_device(dev, &wdt->wdtdev);
-	if (ret < 0)
-		return ret;
+	timeout = da9062_wdt_read_timeout(wdt);
+	if (timeout)
+		wdt->wdtdev.timeout = timeout;
+
+	/* Set timeout from DT value if available */
+	watchdog_init_timeout(&wdt->wdtdev, 0, dev);
+
+	if (timeout) {
+		da9062_wdt_set_timeout(&wdt->wdtdev, wdt->wdtdev.timeout);
+		set_bit(WDOG_HW_RUNNING, &wdt->wdtdev.status);
+	}
 
-	return da9062_wdt_ping(&wdt->wdtdev);
+	return devm_watchdog_register_device(dev, &wdt->wdtdev);
 }
 
 static int __maybe_unused da9062_wdt_suspend(struct device *dev)
diff --git a/drivers/watchdog/da9063_wdt.c b/drivers/watchdog/da9063_wdt.c
index 3d65e92a4e3f..423584252606 100644
--- a/drivers/watchdog/da9063_wdt.c
+++ b/drivers/watchdog/da9063_wdt.c
@@ -46,15 +46,16 @@ static unsigned int da9063_wdt_timeout_to_sel(unsigned int secs)
 }
 
 /*
- * Return 0 if watchdog is disabled, else non zero.
+ * Read the currently active timeout.
+ * Zero means the watchdog is disabled.
  */
-static unsigned int da9063_wdt_is_running(struct da9063 *da9063)
+static unsigned int da9063_wdt_read_timeout(struct da9063 *da9063)
 {
 	unsigned int val;
 
 	regmap_read(da9063->regmap, DA9063_REG_CONTROL_D, &val);
 
-	return val & DA9063_TWDSCALE_MASK;
+	return wdt_timeout[val & DA9063_TWDSCALE_MASK];
 }
 
 static int da9063_wdt_disable_timer(struct da9063 *da9063)
@@ -191,6 +192,7 @@ static int da9063_wdt_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct da9063 *da9063;
 	struct watchdog_device *wdd;
+	unsigned int timeout;
 
 	if (!dev->parent)
 		return -EINVAL;
@@ -214,13 +216,19 @@ static int da9063_wdt_probe(struct platform_device *pdev)
 	watchdog_set_restart_priority(wdd, 128);
 	watchdog_set_drvdata(wdd, da9063);
 
-	/* Set default timeout, maybe override it with DT value, scale it */
 	wdd->timeout = DA9063_WDG_TIMEOUT;
+
+	/* Use pre-configured timeout if watchdog is already running. */
+	timeout = da9063_wdt_read_timeout(da9063);
+	if (timeout)
+		wdd->timeout = timeout;
+
+	/* Set timeout, maybe override it with DT value, scale it */
 	watchdog_init_timeout(wdd, 0, dev);
 	da9063_wdt_set_timeout(wdd, wdd->timeout);
 
-	/* Change the timeout to the default value if the watchdog is running */
-	if (da9063_wdt_is_running(da9063)) {
+	/* Update timeout if the watchdog is already running. */
+	if (timeout) {
 		da9063_wdt_update_timeout(da9063, wdd->timeout);
 		set_bit(WDOG_HW_RUNNING, &wdd->status);
 	}
diff --git a/drivers/watchdog/imx2_wdt.c b/drivers/watchdog/imx2_wdt.c
index 1fe472f56cb3..b84f80f7d342 100644
--- a/drivers/watchdog/imx2_wdt.c
+++ b/drivers/watchdog/imx2_wdt.c
@@ -2,7 +2,7 @@
 /*
  * Watchdog driver for IMX2 and later processors
  *
- *  Copyright (C) 2010 Wolfram Sang, Pengutronix e.K. <w.sang@pengutronix.de>
+ *  Copyright (C) 2010 Wolfram Sang, Pengutronix e.K. <kernel@pengutronix.de>
  *  Copyright (C) 2014 Freescale Semiconductor, Inc.
  *
  * some parts adapted by similar drivers from Darius Augulis and Vladimir
diff --git a/drivers/watchdog/imx_sc_wdt.c b/drivers/watchdog/imx_sc_wdt.c
index 60a32469f7de..e9ee22a7cb45 100644
--- a/drivers/watchdog/imx_sc_wdt.c
+++ b/drivers/watchdog/imx_sc_wdt.c
@@ -175,6 +175,11 @@ static int imx_sc_wdt_probe(struct platform_device *pdev)
 	wdog->timeout = DEFAULT_TIMEOUT;
 
 	watchdog_init_timeout(wdog, 0, dev);
+
+	ret = imx_sc_wdt_set_timeout(wdog, wdog->timeout);
+	if (ret)
+		return ret;
+
 	watchdog_stop_on_reboot(wdog);
 	watchdog_stop_on_unregister(wdog);
 
diff --git a/drivers/watchdog/m54xx_wdt.c b/drivers/watchdog/m54xx_wdt.c
index 22f335e1e164..60ed6252e5f4 100644
--- a/drivers/watchdog/m54xx_wdt.c
+++ b/drivers/watchdog/m54xx_wdt.c
@@ -29,6 +29,7 @@
 #include <linux/bitops.h>
 #include <linux/ioport.h>
 #include <linux/uaccess.h>
+#include <linux/io.h>
 
 #include <asm/coldfire.h>
 #include <asm/m54xxsim.h>
diff --git a/drivers/watchdog/omap_wdt.c b/drivers/watchdog/omap_wdt.c
index 9b91882fe3c4..1616f93dfad7 100644
--- a/drivers/watchdog/omap_wdt.c
+++ b/drivers/watchdog/omap_wdt.c
@@ -273,6 +273,7 @@ static int omap_wdt_probe(struct platform_device *pdev)
 
 	ret = watchdog_register_device(&wdev->wdog);
 	if (ret) {
+		pm_runtime_put(wdev->dev);
 		pm_runtime_disable(wdev->dev);
 		return ret;
 	}
diff --git a/drivers/watchdog/riowd.c b/drivers/watchdog/riowd.c
index dc3c06a92f93..1b9a6dc8f982 100644
--- a/drivers/watchdog/riowd.c
+++ b/drivers/watchdog/riowd.c
@@ -141,7 +141,7 @@ static long riowd_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 	default:
 		return -EINVAL;
-	};
+	}
 
 	return 0;
 }
diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig
index 61212fc7f0c7..727f11eb46b2 100644
--- a/drivers/xen/Kconfig
+++ b/drivers/xen/Kconfig
@@ -13,12 +13,16 @@ config XEN_BALLOON
 config XEN_BALLOON_MEMORY_HOTPLUG
 	bool "Memory hotplug support for Xen balloon driver"
 	depends on XEN_BALLOON && MEMORY_HOTPLUG
+	default y
 	help
 	  Memory hotplug support for Xen balloon driver allows expanding memory
 	  available for the system above limit declared at system startup.
 	  It is very useful on critical systems which require long
 	  run without rebooting.
 
+	  It's also very useful for non PV domains to obtain unpopulated physical
+	  memory ranges to use in order to map foreign memory or grants.
+
 	  Memory could be hotplugged in following steps:
 
 	    1) target domain: ensure that memory auto online policy is in
diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile
index 0c4efa6fe450..0d322f3d90cd 100644
--- a/drivers/xen/Makefile
+++ b/drivers/xen/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_HOTPLUG_CPU)		+= cpu_hotplug.o
-obj-y	+= grant-table.o features.o balloon.o manage.o preempt.o time.o
+obj-y	+= grant-table.o features.o balloon.o manage.o time.o
 obj-y	+= mem-reservation.o
 obj-y	+= events/
 obj-y	+= xenbus/
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index 0c142bcab79d..77c57568e5d7 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -59,7 +59,6 @@
 
 #include <asm/page.h>
 #include <asm/pgalloc.h>
-#include <asm/pgtable.h>
 #include <asm/tlb.h>
 
 #include <asm/xen/hypervisor.h>
diff --git a/drivers/xen/cpu_hotplug.c b/drivers/xen/cpu_hotplug.c
index ec975decb5de..b96b11e2b571 100644
--- a/drivers/xen/cpu_hotplug.c
+++ b/drivers/xen/cpu_hotplug.c
@@ -93,10 +93,8 @@ static int setup_cpu_watcher(struct notifier_block *notifier,
 	(void)register_xenbus_watch(&cpu_watch);
 
 	for_each_possible_cpu(cpu) {
-		if (vcpu_online(cpu) == 0) {
-			device_offline(get_cpu_device(cpu));
-			set_cpu_present(cpu, false);
-		}
+		if (vcpu_online(cpu) == 0)
+			disable_hotplug_cpu(cpu);
 	}
 
 	return NOTIFY_DONE;
@@ -119,5 +117,5 @@ static int __init setup_vcpu_hotplug_event(void)
 	return 0;
 }
 
-arch_initcall(setup_vcpu_hotplug_event);
+late_initcall(setup_vcpu_hotplug_event);
 
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index 3a791c8485d0..140c7bf33a98 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -37,6 +37,7 @@
 #ifdef CONFIG_X86
 #include <asm/desc.h>
 #include <asm/ptrace.h>
+#include <asm/idtentry.h>
 #include <asm/irq.h>
 #include <asm/io_apic.h>
 #include <asm/i8259.h>
@@ -1236,9 +1237,6 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
 	struct pt_regs *old_regs = set_irq_regs(regs);
 
 	irq_enter();
-#ifdef CONFIG_X86
-	inc_irq_stat(irq_hv_callback_count);
-#endif
 
 	__xen_evtchn_do_upcall();
 
@@ -1639,26 +1637,30 @@ EXPORT_SYMBOL_GPL(xen_set_callback_via);
 /* Vector callbacks are better than PCI interrupts to receive event
  * channel notifications because we can receive vector callbacks on any
  * vcpu and we don't need PCI support or APIC interactions. */
-void xen_callback_vector(void)
+void xen_setup_callback_vector(void)
 {
-	int rc;
 	uint64_t callback_via;
 
 	if (xen_have_vector_callback) {
 		callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
-		rc = xen_set_callback_via(callback_via);
-		if (rc) {
+		if (xen_set_callback_via(callback_via)) {
 			pr_err("Request for Xen HVM callback vector failed\n");
 			xen_have_vector_callback = 0;
-			return;
 		}
-		pr_info_once("Xen HVM callback vector for event delivery is enabled\n");
-		alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
-				xen_hvm_callback_vector);
 	}
 }
+
+static __init void xen_alloc_callback_vector(void)
+{
+	if (!xen_have_vector_callback)
+		return;
+
+	pr_info("Xen HVM callback vector for event delivery is enabled\n");
+	alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback);
+}
 #else
-void xen_callback_vector(void) {}
+void xen_setup_callback_vector(void) {}
+static inline void xen_alloc_callback_vector(void) {}
 #endif
 
 #undef MODULE_PARAM_PREFIX
@@ -1692,8 +1694,10 @@ void __init xen_init_IRQ(void)
 		if (xen_initial_domain())
 			pci_xen_initial_domain();
 	}
-	if (xen_feature(XENFEAT_hvm_callback_vector))
-		xen_callback_vector();
+	if (xen_feature(XENFEAT_hvm_callback_vector)) {
+		xen_setup_callback_vector();
+		xen_alloc_callback_vector();
+	}
 
 	if (xen_hvm_domain()) {
 		native_init_IRQ();
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
index 50651e566564..64a9025a87be 100644
--- a/drivers/xen/gntdev.c
+++ b/drivers/xen/gntdev.c
@@ -625,7 +625,7 @@ static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv *priv,
 		return -EFAULT;
 	pr_debug("priv %p, offset for vaddr %lx\n", priv, (unsigned long)op.vaddr);
 
-	down_read(&current->mm->mmap_sem);
+	mmap_read_lock(current->mm);
 	vma = find_vma(current->mm, op.vaddr);
 	if (!vma || vma->vm_ops != &gntdev_vmops)
 		goto out_unlock;
@@ -639,7 +639,7 @@ static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv *priv,
 	rv = 0;
 
  out_unlock:
-	up_read(&current->mm->mmap_sem);
+	mmap_read_unlock(current->mm);
 
 	if (rv == 0 && copy_to_user(u, &op, sizeof(op)) != 0)
 		return -EFAULT;
@@ -1014,7 +1014,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma)
 		 * to the PTE from going stale.
 		 *
 		 * Since this vma's mappings can't be touched without the
-		 * mmap_sem, and we are holding it now, there is no need for
+		 * mmap_lock, and we are holding it now, there is no need for
 		 * the notifier_range locking pattern.
 		 */
 		mmu_interval_read_begin(&map->notifier);
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 7b36b51cdb9f..8d06bf1cc347 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -64,7 +64,6 @@
 #include <asm/xen/hypercall.h>
 #include <asm/xen/interface.h>
 
-#include <asm/pgtable.h>
 #include <asm/sync_bitops.h>
 
 /* External tools reserve first few grant table entries. */
diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
index 59e85e408c23..dd911e1ff782 100644
--- a/drivers/xen/platform-pci.c
+++ b/drivers/xen/platform-pci.c
@@ -168,7 +168,7 @@ static const struct pci_device_id platform_pci_tbl[] = {
 	{0,}
 };
 
-static struct dev_pm_ops platform_pm_ops = {
+static const struct dev_pm_ops platform_pm_ops = {
 	.resume_noirq =   platform_pci_resume,
 };
 
diff --git a/drivers/xen/preempt.c b/drivers/xen/preempt.c
deleted file mode 100644
index 17240c5325a3..000000000000
--- a/drivers/xen/preempt.c
+++ /dev/null
@@ -1,42 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Preemptible hypercalls
- *
- * Copyright (C) 2014 Citrix Systems R&D ltd.
- */
-
-#include <linux/sched.h>
-#include <xen/xen-ops.h>
-
-#ifndef CONFIG_PREEMPTION
-
-/*
- * Some hypercalls issued by the toolstack can take many 10s of
- * seconds. Allow tasks running hypercalls via the privcmd driver to
- * be voluntarily preempted even if full kernel preemption is
- * disabled.
- *
- * Such preemptible hypercalls are bracketed by
- * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end()
- * calls.
- */
-
-DEFINE_PER_CPU(bool, xen_in_preemptible_hcall);
-EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall);
-
-asmlinkage __visible void xen_maybe_preempt_hcall(void)
-{
-	if (unlikely(__this_cpu_read(xen_in_preemptible_hcall)
-		     && need_resched())) {
-		/*
-		 * Clear flag as we may be rescheduled on a different
-		 * cpu.
-		 */
-		__this_cpu_write(xen_in_preemptible_hcall, false);
-		local_irq_enable();
-		cond_resched();
-		local_irq_disable();
-		__this_cpu_write(xen_in_preemptible_hcall, true);
-	}
-}
-#endif /* CONFIG_PREEMPTION */
diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index c6070e70dd73..a250d118144a 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -26,8 +26,6 @@
 #include <linux/moduleparam.h>
 
 #include <asm/pgalloc.h>
-#include <asm/pgtable.h>
-#include <asm/tlb.h>
 #include <asm/xen/hypervisor.h>
 #include <asm/xen/hypercall.h>
 
@@ -278,7 +276,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata)
 	if (rc || list_empty(&pagelist))
 		goto out;
 
-	down_write(&mm->mmap_sem);
+	mmap_write_lock(mm);
 
 	{
 		struct page *page = list_first_entry(&pagelist,
@@ -303,7 +301,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata)
 
 
 out_up:
-	up_write(&mm->mmap_sem);
+	mmap_write_unlock(mm);
 
 out:
 	free_page_list(&pagelist);
@@ -499,7 +497,7 @@ static long privcmd_ioctl_mmap_batch(
 		}
 	}
 
-	down_write(&mm->mmap_sem);
+	mmap_write_lock(mm);
 
 	vma = find_vma(mm, m.addr);
 	if (!vma ||
@@ -555,7 +553,7 @@ static long privcmd_ioctl_mmap_batch(
 	BUG_ON(traverse_pages_block(m.num, sizeof(xen_pfn_t),
 				    &pagelist, mmap_batch_fn, &state));
 
-	up_write(&mm->mmap_sem);
+	mmap_write_unlock(mm);
 
 	if (state.global_error) {
 		/* Write back errors in second pass. */
@@ -576,7 +574,7 @@ out:
 	return ret;
 
 out_unlock:
-	up_write(&mm->mmap_sem);
+	mmap_write_unlock(mm);
 	goto out;
 }
 
@@ -741,7 +739,7 @@ static long privcmd_ioctl_mmap_resource(struct file *file, void __user *udata)
 	if (data->domid != DOMID_INVALID && data->domid != kdata.dom)
 		return -EPERM;
 
-	down_write(&mm->mmap_sem);
+	mmap_write_lock(mm);
 
 	vma = find_vma(mm, kdata.addr);
 	if (!vma || vma->vm_ops != &privcmd_vm_ops) {
@@ -820,7 +818,7 @@ static long privcmd_ioctl_mmap_resource(struct file *file, void __user *udata)
 	}
 
 out:
-	up_write(&mm->mmap_sem);
+	mmap_write_unlock(mm);
 	kfree(pfns);
 
 	return rc;
diff --git a/drivers/xen/pvcalls-back.c b/drivers/xen/pvcalls-back.c
index cf4ce3e9358d..9eae1fceec1e 100644
--- a/drivers/xen/pvcalls-back.c
+++ b/drivers/xen/pvcalls-back.c
@@ -24,7 +24,7 @@
 #define PVCALLS_VERSIONS "1"
 #define MAX_RING_ORDER XENBUS_MAX_RING_GRANT_ORDER
 
-struct pvcalls_back_global {
+static struct pvcalls_back_global {
 	struct list_head frontends;
 	struct semaphore frontends_lock;
 } pvcalls_back_global;
@@ -1088,7 +1088,8 @@ static void set_backend_state(struct xenbus_device *dev,
 		case XenbusStateInitialised:
 			switch (state) {
 			case XenbusStateConnected:
-				backend_connect(dev);
+				if (backend_connect(dev))
+					return;
 				xenbus_switch_state(dev, XenbusStateConnected);
 				break;
 			case XenbusStateClosing:
diff --git a/drivers/xen/time.c b/drivers/xen/time.c
index 0968859c29d0..108edbcbc040 100644
--- a/drivers/xen/time.c
+++ b/drivers/xen/time.c
@@ -64,7 +64,7 @@ static void xen_get_runstate_snapshot_cpu_delta(
 	do {
 		state_time = get64(&state->state_entry_time);
 		rmb();	/* Hypervisor might update data. */
-		*res = READ_ONCE(*state);
+		*res = __READ_ONCE(*state);
 		rmb();	/* Hypervisor might update data. */
 	} while (get64(&state->state_entry_time) != state_time ||
 		 (state_time & XEN_RUNSTATE_UPDATE));
diff --git a/drivers/xen/xen-pciback/conf_space.c b/drivers/xen/xen-pciback/conf_space.c
index da51a5d34e6e..059de92aea7d 100644
--- a/drivers/xen/xen-pciback/conf_space.c
+++ b/drivers/xen/xen-pciback/conf_space.c
@@ -10,6 +10,8 @@
  * Author: Ryan Wilson <hap9@epoch.ncsc.mil>
  */
 
+#define dev_fmt(fmt) DRV_NAME ": " fmt
+
 #include <linux/kernel.h>
 #include <linux/moduleparam.h>
 #include <linux/pci.h>
@@ -154,9 +156,7 @@ int xen_pcibk_config_read(struct pci_dev *dev, int offset, int size,
 	 * (as if device didn't respond) */
 	u32 value = 0, tmp_val;
 
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: read %d bytes at 0x%x\n",
-		       pci_name(dev), size, offset);
+	dev_dbg(&dev->dev, "read %d bytes at 0x%x\n", size, offset);
 
 	if (!valid_request(offset, size)) {
 		err = XEN_PCI_ERR_invalid_offset;
@@ -195,9 +195,7 @@ int xen_pcibk_config_read(struct pci_dev *dev, int offset, int size,
 	}
 
 out:
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: read %d bytes at 0x%x = %x\n",
-		       pci_name(dev), size, offset, value);
+	dev_dbg(&dev->dev, "read %d bytes at 0x%x = %x\n", size, offset, value);
 
 	*ret_val = value;
 	return xen_pcibios_err_to_errno(err);
@@ -212,10 +210,8 @@ int xen_pcibk_config_write(struct pci_dev *dev, int offset, int size, u32 value)
 	u32 tmp_val;
 	int field_start, field_end;
 
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG
-		       DRV_NAME ": %s: write request %d bytes at 0x%x = %x\n",
-		       pci_name(dev), size, offset, value);
+	dev_dbg(&dev->dev, "write request %d bytes at 0x%x = %x\n",
+		size, offset, value);
 
 	if (!valid_request(offset, size))
 		return XEN_PCI_ERR_invalid_offset;
diff --git a/drivers/xen/xen-pciback/conf_space_header.c b/drivers/xen/xen-pciback/conf_space_header.c
index fb4fccb4aecc..ac45cdc38e85 100644
--- a/drivers/xen/xen-pciback/conf_space_header.c
+++ b/drivers/xen/xen-pciback/conf_space_header.c
@@ -6,6 +6,7 @@
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define dev_fmt pr_fmt
 
 #include <linux/kernel.h>
 #include <linux/pci.h>
@@ -67,53 +68,39 @@ static int command_write(struct pci_dev *dev, int offset, u16 value, void *data)
 
 	dev_data = pci_get_drvdata(dev);
 	if (!pci_is_enabled(dev) && is_enable_cmd(value)) {
-		if (unlikely(verbose_request))
-			printk(KERN_DEBUG DRV_NAME ": %s: enable\n",
-			       pci_name(dev));
+		dev_dbg(&dev->dev, "enable\n");
 		err = pci_enable_device(dev);
 		if (err)
 			return err;
 		if (dev_data)
 			dev_data->enable_intx = 1;
 	} else if (pci_is_enabled(dev) && !is_enable_cmd(value)) {
-		if (unlikely(verbose_request))
-			printk(KERN_DEBUG DRV_NAME ": %s: disable\n",
-			       pci_name(dev));
+		dev_dbg(&dev->dev, "disable\n");
 		pci_disable_device(dev);
 		if (dev_data)
 			dev_data->enable_intx = 0;
 	}
 
 	if (!dev->is_busmaster && is_master_cmd(value)) {
-		if (unlikely(verbose_request))
-			printk(KERN_DEBUG DRV_NAME ": %s: set bus master\n",
-			       pci_name(dev));
+		dev_dbg(&dev->dev, "set bus master\n");
 		pci_set_master(dev);
 	} else if (dev->is_busmaster && !is_master_cmd(value)) {
-		if (unlikely(verbose_request))
-			printk(KERN_DEBUG DRV_NAME ": %s: clear bus master\n",
-			       pci_name(dev));
+		dev_dbg(&dev->dev, "clear bus master\n");
 		pci_clear_master(dev);
 	}
 
 	if (!(cmd->val & PCI_COMMAND_INVALIDATE) &&
 	    (value & PCI_COMMAND_INVALIDATE)) {
-		if (unlikely(verbose_request))
-			printk(KERN_DEBUG
-			       DRV_NAME ": %s: enable memory-write-invalidate\n",
-			       pci_name(dev));
+		dev_dbg(&dev->dev, "enable memory-write-invalidate\n");
 		err = pci_set_mwi(dev);
 		if (err) {
-			pr_warn("%s: cannot enable memory-write-invalidate (%d)\n",
-				pci_name(dev), err);
+			dev_warn(&dev->dev, "cannot enable memory-write-invalidate (%d)\n",
+				err);
 			value &= ~PCI_COMMAND_INVALIDATE;
 		}
 	} else if ((cmd->val & PCI_COMMAND_INVALIDATE) &&
 		   !(value & PCI_COMMAND_INVALIDATE)) {
-		if (unlikely(verbose_request))
-			printk(KERN_DEBUG
-			       DRV_NAME ": %s: disable memory-write-invalidate\n",
-			       pci_name(dev));
+		dev_dbg(&dev->dev, "disable memory-write-invalidate\n");
 		pci_clear_mwi(dev);
 	}
 
@@ -157,8 +144,7 @@ static int rom_write(struct pci_dev *dev, int offset, u32 value, void *data)
 	struct pci_bar_info *bar = data;
 
 	if (unlikely(!bar)) {
-		pr_warn(DRV_NAME ": driver data not found for %s\n",
-		       pci_name(dev));
+		dev_warn(&dev->dev, "driver data not found\n");
 		return XEN_PCI_ERR_op_failed;
 	}
 
@@ -194,8 +180,7 @@ static int bar_write(struct pci_dev *dev, int offset, u32 value, void *data)
 	u32 mask;
 
 	if (unlikely(!bar)) {
-		pr_warn(DRV_NAME ": driver data not found for %s\n",
-		       pci_name(dev));
+		dev_warn(&dev->dev, "driver data not found\n");
 		return XEN_PCI_ERR_op_failed;
 	}
 
@@ -228,8 +213,7 @@ static int bar_read(struct pci_dev *dev, int offset, u32 * value, void *data)
 	struct pci_bar_info *bar = data;
 
 	if (unlikely(!bar)) {
-		pr_warn(DRV_NAME ": driver data not found for %s\n",
-		       pci_name(dev));
+		dev_warn(&dev->dev, "driver data not found\n");
 		return XEN_PCI_ERR_op_failed;
 	}
 
@@ -433,8 +417,8 @@ int xen_pcibk_config_header_add_fields(struct pci_dev *dev)
 
 	default:
 		err = -EINVAL;
-		pr_err("%s: Unsupported header type %d!\n",
-		       pci_name(dev), dev->hdr_type);
+		dev_err(&dev->dev, "Unsupported header type %d!\n",
+			dev->hdr_type);
 		break;
 	}
 
diff --git a/drivers/xen/xen-pciback/conf_space_quirks.c b/drivers/xen/xen-pciback/conf_space_quirks.c
index ed593d1042a6..7dc281086302 100644
--- a/drivers/xen/xen-pciback/conf_space_quirks.c
+++ b/drivers/xen/xen-pciback/conf_space_quirks.c
@@ -6,6 +6,8 @@
  * Author: Chris Bookholt <hap10@epoch.ncsc.mil>
  */
 
+#define dev_fmt(fmt) DRV_NAME ": " fmt
+
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include "pciback.h"
@@ -35,8 +37,8 @@ static struct xen_pcibk_config_quirk *xen_pcibk_find_quirk(struct pci_dev *dev)
 		if (match_one_device(&tmp_quirk->devid, dev) != NULL)
 			goto out;
 	tmp_quirk = NULL;
-	printk(KERN_DEBUG DRV_NAME
-	       ": quirk didn't match any device known\n");
+	dev_printk(KERN_DEBUG, &dev->dev,
+		   "quirk didn't match any device known\n");
 out:
 	return tmp_quirk;
 }
diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c
index 7af93d65ed51..e876c3d6dad1 100644
--- a/drivers/xen/xen-pciback/pci_stub.c
+++ b/drivers/xen/xen-pciback/pci_stub.c
@@ -6,6 +6,7 @@
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define dev_fmt pr_fmt
 
 #include <linux/module.h>
 #include <linux/init.h>
@@ -626,11 +627,11 @@ static void pcistub_remove(struct pci_dev *dev)
 		if (found_psdev->pdev) {
 			int domid = xen_find_device_domain_owner(dev);
 
-			pr_warn("****** removing device %s while still in-use by domain %d! ******\n",
+			dev_warn(&dev->dev, "****** removing device %s while still in-use by domain %d! ******\n",
 			       pci_name(found_psdev->dev), domid);
-			pr_warn("****** driver domain may still access this device's i/o resources!\n");
-			pr_warn("****** shutdown driver domain before binding device\n");
-			pr_warn("****** to other drivers or domains\n");
+			dev_warn(&dev->dev, "****** driver domain may still access this device's i/o resources!\n");
+			dev_warn(&dev->dev, "****** shutdown driver domain before binding device\n");
+			dev_warn(&dev->dev, "****** to other drivers or domains\n");
 
 			/* N.B. This ends up calling pcistub_put_pci_dev which ends up
 			 * doing the FLR. */
@@ -711,14 +712,12 @@ static pci_ers_result_t common_process(struct pcistub_device *psdev,
 	ret = xen_pcibk_get_pcifront_dev(psdev->dev, psdev->pdev,
 		&aer_op->domain, &aer_op->bus, &aer_op->devfn);
 	if (!ret) {
-		dev_err(&psdev->dev->dev,
-			DRV_NAME ": failed to get pcifront device\n");
+		dev_err(&psdev->dev->dev, "failed to get pcifront device\n");
 		return PCI_ERS_RESULT_NONE;
 	}
 	wmb();
 
-	dev_dbg(&psdev->dev->dev,
-			DRV_NAME ": aer_op %x dom %x bus %x devfn %x\n",
+	dev_dbg(&psdev->dev->dev, "aer_op %x dom %x bus %x devfn %x\n",
 			aer_cmd, aer_op->domain, aer_op->bus, aer_op->devfn);
 	/*local flag to mark there's aer request, xen_pcibk callback will use
 	* this flag to judge whether we need to check pci-front give aer
@@ -754,8 +753,7 @@ static pci_ers_result_t common_process(struct pcistub_device *psdev,
 
 	if (test_bit(_XEN_PCIF_active,
 		(unsigned long *)&sh_info->flags)) {
-		dev_dbg(&psdev->dev->dev,
-			"schedule pci_conf service in " DRV_NAME "\n");
+		dev_dbg(&psdev->dev->dev, "schedule pci_conf service\n");
 		xen_pcibk_test_and_schedule_op(psdev->pdev);
 	}
 
@@ -786,13 +784,12 @@ static pci_ers_result_t xen_pcibk_slot_reset(struct pci_dev *dev)
 				PCI_FUNC(dev->devfn));
 
 	if (!psdev || !psdev->pdev) {
-		dev_err(&dev->dev,
-			DRV_NAME " device is not found/assigned\n");
+		dev_err(&dev->dev, "device is not found/assigned\n");
 		goto end;
 	}
 
 	if (!psdev->pdev->sh_info) {
-		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
+		dev_err(&dev->dev, "device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
 		goto end;
@@ -844,13 +841,12 @@ static pci_ers_result_t xen_pcibk_mmio_enabled(struct pci_dev *dev)
 				PCI_FUNC(dev->devfn));
 
 	if (!psdev || !psdev->pdev) {
-		dev_err(&dev->dev,
-			DRV_NAME " device is not found/assigned\n");
+		dev_err(&dev->dev, "device is not found/assigned\n");
 		goto end;
 	}
 
 	if (!psdev->pdev->sh_info) {
-		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
+		dev_err(&dev->dev, "device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
 		goto end;
@@ -902,13 +898,12 @@ static pci_ers_result_t xen_pcibk_error_detected(struct pci_dev *dev,
 				PCI_FUNC(dev->devfn));
 
 	if (!psdev || !psdev->pdev) {
-		dev_err(&dev->dev,
-			DRV_NAME " device is not found/assigned\n");
+		dev_err(&dev->dev, "device is not found/assigned\n");
 		goto end;
 	}
 
 	if (!psdev->pdev->sh_info) {
-		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
+		dev_err(&dev->dev, "device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
 		goto end;
@@ -956,13 +951,12 @@ static void xen_pcibk_error_resume(struct pci_dev *dev)
 				PCI_FUNC(dev->devfn));
 
 	if (!psdev || !psdev->pdev) {
-		dev_err(&dev->dev,
-			DRV_NAME " device is not found/assigned\n");
+		dev_err(&dev->dev, "device is not found/assigned\n");
 		goto end;
 	}
 
 	if (!psdev->pdev->sh_info) {
-		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
+		dev_err(&dev->dev, "device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
 		goto end;
diff --git a/drivers/xen/xen-pciback/pciback.h b/drivers/xen/xen-pciback/pciback.h
index 7c95516a860f..f1ed2dbf685c 100644
--- a/drivers/xen/xen-pciback/pciback.h
+++ b/drivers/xen/xen-pciback/pciback.h
@@ -186,8 +186,6 @@ void xen_pcibk_do_op(struct work_struct *data);
 int xen_pcibk_xenbus_register(void);
 void xen_pcibk_xenbus_unregister(void);
 
-extern int verbose_request;
-
 void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev);
 #endif
 
diff --git a/drivers/xen/xen-pciback/pciback_ops.c b/drivers/xen/xen-pciback/pciback_ops.c
index 787966f44589..e11a7438e1a2 100644
--- a/drivers/xen/xen-pciback/pciback_ops.c
+++ b/drivers/xen/xen-pciback/pciback_ops.c
@@ -6,6 +6,7 @@
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define dev_fmt pr_fmt
 
 #include <linux/moduleparam.h>
 #include <linux/wait.h>
@@ -14,9 +15,6 @@
 #include <linux/sched.h>
 #include "pciback.h"
 
-int verbose_request;
-module_param(verbose_request, int, 0644);
-
 static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id);
 
 /* Ensure a device is has the fake IRQ handler "turned on/off" and is
@@ -147,9 +145,6 @@ int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev,
 	struct xen_pcibk_dev_data *dev_data;
 	int status;
 
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: enable MSI\n", pci_name(dev));
-
 	if (dev->msi_enabled)
 		status = -EALREADY;
 	else if (dev->msix_enabled)
@@ -158,9 +153,8 @@ int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev,
 		status = pci_enable_msi(dev);
 
 	if (status) {
-		pr_warn_ratelimited("%s: error enabling MSI for guest %u: err %d\n",
-				    pci_name(dev), pdev->xdev->otherend_id,
-				    status);
+		dev_warn_ratelimited(&dev->dev, "error enabling MSI for guest %u: err %d\n",
+				     pdev->xdev->otherend_id, status);
 		op->value = 0;
 		return XEN_PCI_ERR_op_failed;
 	}
@@ -169,9 +163,8 @@ int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev,
 	 * the local domain's IRQ number. */
 
 	op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0;
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: MSI: %d\n", pci_name(dev),
-			op->value);
+
+	dev_dbg(&dev->dev, "MSI: %d\n", op->value);
 
 	dev_data = pci_get_drvdata(dev);
 	if (dev_data)
@@ -184,10 +177,6 @@ static
 int xen_pcibk_disable_msi(struct xen_pcibk_device *pdev,
 			  struct pci_dev *dev, struct xen_pci_op *op)
 {
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: disable MSI\n",
-		       pci_name(dev));
-
 	if (dev->msi_enabled) {
 		struct xen_pcibk_dev_data *dev_data;
 
@@ -198,9 +187,9 @@ int xen_pcibk_disable_msi(struct xen_pcibk_device *pdev,
 			dev_data->ack_intr = 1;
 	}
 	op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0;
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: MSI: %d\n", pci_name(dev),
-			op->value);
+
+	dev_dbg(&dev->dev, "MSI: %d\n", op->value);
+
 	return 0;
 }
 
@@ -213,9 +202,7 @@ int xen_pcibk_enable_msix(struct xen_pcibk_device *pdev,
 	struct msix_entry *entries;
 	u16 cmd;
 
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: enable MSI-X\n",
-		       pci_name(dev));
+	dev_dbg(&dev->dev, "enable MSI-X\n");
 
 	if (op->value > SH_INFO_MAX_VEC)
 		return -EINVAL;
@@ -248,17 +235,13 @@ int xen_pcibk_enable_msix(struct xen_pcibk_device *pdev,
 			if (entries[i].vector) {
 				op->msix_entries[i].vector =
 					xen_pirq_from_irq(entries[i].vector);
-				if (unlikely(verbose_request))
-					printk(KERN_DEBUG DRV_NAME ": %s: " \
-						"MSI-X[%d]: %d\n",
-						pci_name(dev), i,
-						op->msix_entries[i].vector);
+				dev_dbg(&dev->dev, "MSI-X[%d]: %d\n", i,
+					op->msix_entries[i].vector);
 			}
 		}
 	} else
-		pr_warn_ratelimited("%s: error enabling MSI-X for guest %u: err %d!\n",
-				    pci_name(dev), pdev->xdev->otherend_id,
-				    result);
+		dev_warn_ratelimited(&dev->dev, "error enabling MSI-X for guest %u: err %d!\n",
+				     pdev->xdev->otherend_id, result);
 	kfree(entries);
 
 	op->value = result;
@@ -273,10 +256,6 @@ static
 int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev,
 			   struct pci_dev *dev, struct xen_pci_op *op)
 {
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: disable MSI-X\n",
-			pci_name(dev));
-
 	if (dev->msix_enabled) {
 		struct xen_pcibk_dev_data *dev_data;
 
@@ -291,9 +270,9 @@ int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev,
 	 * an undefined IRQ value of zero.
 	 */
 	op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0;
-	if (unlikely(verbose_request))
-		printk(KERN_DEBUG DRV_NAME ": %s: MSI-X: %d\n",
-		       pci_name(dev), op->value);
+
+	dev_dbg(&dev->dev, "MSI-X: %d\n", op->value);
+
 	return 0;
 }
 #endif
@@ -424,7 +403,7 @@ static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id)
 		dev_data->handled++;
 		if ((dev_data->handled % 1000) == 0) {
 			if (xen_test_irq_shared(irq)) {
-				pr_info("%s IRQ line is not shared "
+				dev_info(&dev->dev, "%s IRQ line is not shared "
 					"with other domains. Turning ISR off\n",
 					 dev_data->irq_name);
 				dev_data->ack_intr = 0;
diff --git a/drivers/xen/xen-pciback/vpci.c b/drivers/xen/xen-pciback/vpci.c
index f6ba18191c0f..5447b5ab7c76 100644
--- a/drivers/xen/xen-pciback/vpci.c
+++ b/drivers/xen/xen-pciback/vpci.c
@@ -7,6 +7,7 @@
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define dev_fmt pr_fmt
 
 #include <linux/list.h>
 #include <linux/slab.h>
@@ -105,9 +106,8 @@ static int __xen_pcibk_add_pci_dev(struct xen_pcibk_device *pdev,
 				       struct pci_dev_entry, list);
 
 			if (match_slot(dev, t->dev)) {
-				pr_info("vpci: %s: assign to virtual slot %d func %d\n",
-					pci_name(dev), slot,
-					PCI_FUNC(dev->devfn));
+				dev_info(&dev->dev, "vpci: assign to virtual slot %d func %d\n",
+					 slot, PCI_FUNC(dev->devfn));
 				list_add_tail(&dev_entry->list,
 					      &vpci_dev->dev_list[slot]);
 				func = PCI_FUNC(dev->devfn);
@@ -119,8 +119,8 @@ static int __xen_pcibk_add_pci_dev(struct xen_pcibk_device *pdev,
 	/* Assign to a new slot on the virtual PCI bus */
 	for (slot = 0; slot < PCI_SLOT_MAX; slot++) {
 		if (list_empty(&vpci_dev->dev_list[slot])) {
-			pr_info("vpci: %s: assign to virtual slot %d\n",
-				pci_name(dev), slot);
+			dev_info(&dev->dev, "vpci: assign to virtual slot %d\n",
+				 slot);
 			list_add_tail(&dev_entry->list,
 				      &vpci_dev->dev_list[slot]);
 			func = dev->is_virtfn ? 0 : PCI_FUNC(dev->devfn);
diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c
index 8c4d05b687b7..38725d97d909 100644
--- a/drivers/xen/xenbus/xenbus_probe.c
+++ b/drivers/xen/xenbus/xenbus_probe.c
@@ -31,6 +31,7 @@
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define dev_fmt pr_fmt
 
 #define DPRINTK(fmt, args...)				\
 	pr_debug("xenbus_probe (%s:%d) " fmt ".\n",	\
@@ -51,7 +52,6 @@
 #include <linux/module.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/xen/hypervisor.h>
 
 #include <xen/xen.h>
@@ -608,7 +608,7 @@ int xenbus_dev_suspend(struct device *dev)
 	if (drv->suspend)
 		err = drv->suspend(xdev);
 	if (err)
-		pr_warn("suspend %s failed: %i\n", dev_name(dev), err);
+		dev_warn(dev, "suspend failed: %i\n", err);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(xenbus_dev_suspend);
@@ -627,8 +627,7 @@ int xenbus_dev_resume(struct device *dev)
 	drv = to_xenbus_driver(dev->driver);
 	err = talk_to_otherend(xdev);
 	if (err) {
-		pr_warn("resume (talk_to_otherend) %s failed: %i\n",
-			dev_name(dev), err);
+		dev_warn(dev, "resume (talk_to_otherend) failed: %i\n", err);
 		return err;
 	}
 
@@ -637,15 +636,14 @@ int xenbus_dev_resume(struct device *dev)
 	if (drv->resume) {
 		err = drv->resume(xdev);
 		if (err) {
-			pr_warn("resume %s failed: %i\n", dev_name(dev), err);
+			dev_warn(dev, "resume failed: %i\n", err);
 			return err;
 		}
 	}
 
 	err = watch_otherend(xdev);
 	if (err) {
-		pr_warn("resume (watch_otherend) %s failed: %d.\n",
-			dev_name(dev), err);
+		dev_warn(dev, "resume (watch_otherend) failed: %d\n", err);
 		return err;
 	}
 
diff --git a/drivers/xen/xenbus/xenbus_probe_backend.c b/drivers/xen/xenbus/xenbus_probe_backend.c
index 9b2fbe69bccc..2ba699897e6d 100644
--- a/drivers/xen/xenbus/xenbus_probe_backend.c
+++ b/drivers/xen/xenbus/xenbus_probe_backend.c
@@ -48,7 +48,6 @@
 #include <linux/semaphore.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/xen/hypervisor.h>
 #include <asm/hypervisor.h>
 #include <xen/xenbus.h>
diff --git a/drivers/xen/xenbus/xenbus_probe_frontend.c b/drivers/xen/xenbus/xenbus_probe_frontend.c
index 8a1650bbe18f..15379089853b 100644
--- a/drivers/xen/xenbus/xenbus_probe_frontend.c
+++ b/drivers/xen/xenbus/xenbus_probe_frontend.c
@@ -19,7 +19,6 @@
 #include <linux/module.h>
 
 #include <asm/page.h>
-#include <asm/pgtable.h>
 #include <asm/xen/hypervisor.h>
 #include <xen/xenbus.h>
 #include <xen/events.h>
diff --git a/drivers/zorro/Kconfig b/drivers/zorro/Kconfig
index 2b6ed2936ce3..41475ad46e45 100644
--- a/drivers/zorro/Kconfig
+++ b/drivers/zorro/Kconfig
@@ -5,7 +5,7 @@
 config ZORRO_NAMES
 	bool "Zorro device name database"
 	depends on ZORRO
-	---help---
+	help
 	  By default, the kernel contains a database of all known Zorro device
 	  names to make the information in /proc/iomem comprehensible to the
 	  user. This database increases the size of the kernel image by about